From 379709955ff3c99976aca6f3766886fdd914597f Mon Sep 17 00:00:00 2001 From: oppo99 Date: Tue, 28 Jul 2026 13:04:31 +0200 Subject: [PATCH 001/189] Support AProjQ4 GGUFs: Q4_K dense attention projections The AProjQ4 DeepSeek V4 Flash GGUFs quantize the five dense attention projections per layer (attn_q_a, attn_q_b, attn_kv, attn_output_a, attn_output_b) as Q4_K instead of Q8_0. Loading already accepted them (tensor_expect_dense_quant_layout), but the decode graph read the Q4_K blocks through the hardcoded Q8_0 kernels and generated garbage (BOS loops), and the CPU reference died with "expected a 2D Q8_0 tensor". - Metal decode graph: gate the fused Q8_0 q_a/kv pair kernel and the plain Q8_0 matvec fallbacks on the actual tensor type, dispatching through the existing generic dense-quant path for Q4_K. Q8_0 models keep the exact kernels they used before, so their output stays bit-identical. - CPU reference: add a dense Q4_K matvec/matmul family (activations prequantized to Q8_K, rows reduced with ds4_vec_dot_q4_K_q8_K), with grouped, decode-scratch and prefill-batch variants, and dispatch the attention projection call sites on tensor type. - CUDA decode-TP attention output split now refuses non-Q8_0 output projections with a clear error instead of computing garbage. Verified on Apple M1 Pro 16 GB with --metal --ssd-streaming on DeepSeek-V4-Flash-IQ2XXS-w2Q2K-AProjQ4-SExpQ8-OutQ8-chat-v2-imatrix: coherent greedy output, --decode-consistency max_abs=0 rms=0, and no regression on the AProjQ8 gguf. Co-Authored-By: Claude Fable 5 --- ds4.c | 410 ++++++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 383 insertions(+), 27 deletions(-) diff --git a/ds4.c b/ds4.c index 7e0fad839..343870821 100644 --- a/ds4.c +++ b/ds4.c @@ -932,6 +932,7 @@ typedef struct { float *routed_mid_all; block_q8_K *routed_xq; block_q8_K *routed_midq; + block_q8_K *dense_xq; int8_t *routed_q8_xq; float *routed_q8_xscale; int8_t *routed_q8_midq; @@ -8225,6 +8226,12 @@ static void matvec_q8_0_f32_ref( } static void matvec_any(float *out, const ds4_model *m, const ds4_tensor *w, const float *x); +static void matvec_q4_K_decode_scratch( + float *out, + const ds4_model *m, + const ds4_tensor *w, + const float *x, + ds4_cpu_decode_scratch *scratch); /* Decode scratch owns this temporary activation quantization so generation * can assert that the hot path performs no malloc. */ @@ -8266,6 +8273,8 @@ static void matvec_any_decode_scratch( ds4_cpu_decode_scratch * scratch) { if (w->type == 8) { matvec_q8_0_decode_scratch(out, m, w, x, scratch); + } else if (w->type == DS4_TENSOR_Q4_K) { + matvec_q4_K_decode_scratch(out, m, w, x, scratch); } else { matvec_any(out, m, w, x); } @@ -8392,6 +8401,343 @@ static void matmul_q8_0_grouped_batch( free(xq); } +/* ========================================================================= + * Dense Q4_K matvec/matmul. + * ========================================================================= + * + * The AProjQ4 GGUFs store the dense attention projections (q_a, q_b, kv, + * output_a, output_b) as Q4_K instead of Q8_0. This family mirrors the + * Q8_0 dense functions above: the activation is quantized to Q8_K once and + * each Q4_K weight row is reduced with the shared ds4_vec_dot_q4_K_q8_K + * kernel, so the same call sites can dispatch on the tensor type. + */ + +typedef struct { + float *out; + const uint8_t *data; + const block_q8_K *xq; + uint64_t in_dim; + uint64_t row_bytes; +} matvec_q4_K_dense_ctx; + +static void matvec_q4_K_dense_worker(void *vctx, uint64_t row0, uint64_t row1) { + matvec_q4_K_dense_ctx *ctx = vctx; + for (uint64_t r = row0; r < row1; r++) { + const block_q4_K *row = (const block_q4_K *)(ctx->data + r * ctx->row_bytes); + ds4_vec_dot_q4_K_q8_K((int)ctx->in_dim, &ctx->out[r], row, ctx->xq); + } +} + +static void dense_q4_K_expect(const ds4_tensor *w, uint64_t in_dim) { + if (w->type != DS4_TENSOR_Q4_K || w->ndim != 2) ds4_die("expected a 2D Q4_K tensor"); + if (w->dim[0] != in_dim) ds4_die("Q4_K dense tensor has an unexpected width"); + if ((in_dim % QK_K) != 0) ds4_die("Q4_K dense row is not QK_K aligned"); +} + +static void matvec_q4_K_prequant( + float *out, + const ds4_model *m, + const ds4_tensor *w, + const block_q8_K *xq) { + matvec_q4_K_dense_ctx ctx = { + .out = out, + .data = tensor_data(m, w), + .xq = xq, + .in_dim = w->dim[0], + .row_bytes = (w->dim[0] / QK_K) * sizeof(block_q4_K), + }; + ds4_parallel_for(w->dim[1], matvec_q4_K_dense_worker, &ctx); +} + +static void matvec_q4_K(float *out, const ds4_model *m, const ds4_tensor *w, const float *x) { + dense_q4_K_expect(w, w->dim[0]); + const uint64_t blocks = w->dim[0] / QK_K; + block_q8_K *xq = xmalloc((size_t)blocks * sizeof(block_q8_K)); + ds4_quantize_row_q8_K(x, xq, (int64_t)w->dim[0]); + matvec_q4_K_prequant(out, m, w, xq); + free(xq); +} + +static void matvec_q4_K_decode_scratch( + float *out, + const ds4_model *m, + const ds4_tensor *w, + const float *x, + ds4_cpu_decode_scratch *scratch) { + dense_q4_K_expect(w, w->dim[0]); + if (w->dim[0] > scratch->q8_cap) ds4_die("CPU decode Q4_K scratch buffer is too small"); + ds4_quantize_row_q8_K(x, scratch->dense_xq, (int64_t)w->dim[0]); + matvec_q4_K_prequant(out, m, w, scratch->dense_xq); +} + +typedef struct { + float *out; + const uint8_t *data; + const block_q8_K *xq; + uint64_t in_dim; + uint64_t blocks; + uint64_t rank; +} matvec_q4_K_grouped_ctx; + +static void matvec_q4_K_grouped_worker(void *vctx, uint64_t r0, uint64_t r1) { + matvec_q4_K_grouped_ctx *ctx = vctx; + for (uint64_t idx = r0; idx < r1; idx++) { + const uint64_t group = idx / ctx->rank; + const block_q4_K *row = (const block_q4_K *) + (ctx->data + idx * ctx->blocks * sizeof(block_q4_K)); + ds4_vec_dot_q4_K_q8_K((int)ctx->in_dim, &ctx->out[idx], row, + ctx->xq + group * ctx->blocks); + } +} + +static void matvec_q4_K_grouped_expect( + const ds4_tensor *w, + uint32_t n_groups, + uint64_t group_dim, + uint64_t rank) { + dense_q4_K_expect(w, group_dim); + if (w->dim[1] < (uint64_t)n_groups * rank) { + ds4_die("grouped Q4_K tensor has an unexpected layout"); + } +} + +static void matvec_q4_K_grouped_rows_prequant( + float *out, + const ds4_model *m, + const ds4_tensor *w, + const block_q8_K *xq, + uint32_t n_groups, + uint64_t group_dim, + uint64_t rank) { + matvec_q4_K_grouped_ctx ctx = { + .out = out, + .data = tensor_data(m, w), + .xq = xq, + .in_dim = group_dim, + .blocks = group_dim / QK_K, + .rank = rank, + }; + ds4_parallel_for((uint64_t)n_groups * rank, matvec_q4_K_grouped_worker, &ctx); +} + +static void matvec_q4_K_grouped_rows( + float *out, + const ds4_model *m, + const ds4_tensor *w, + const float *x, + uint32_t n_groups, + uint64_t group_dim, + uint64_t rank) { + matvec_q4_K_grouped_expect(w, n_groups, group_dim, rank); + const uint64_t blocks = group_dim / QK_K; + block_q8_K *xq = xmalloc((size_t)n_groups * blocks * sizeof(block_q8_K)); + for (uint32_t g = 0; g < n_groups; g++) { + ds4_quantize_row_q8_K(x + (uint64_t)g * group_dim, + xq + (uint64_t)g * blocks, + (int64_t)group_dim); + } + matvec_q4_K_grouped_rows_prequant(out, m, w, xq, n_groups, group_dim, rank); + free(xq); +} + +static void matvec_q4_K_grouped_rows_decode_scratch( + float *out, + const ds4_model *m, + const ds4_tensor *w, + const float *x, + uint32_t n_groups, + uint64_t group_dim, + uint64_t rank, + ds4_cpu_decode_scratch *scratch) { + matvec_q4_K_grouped_expect(w, n_groups, group_dim, rank); + if ((uint64_t)n_groups * group_dim > scratch->q8_cap) { + ds4_die("CPU decode grouped Q4_K scratch buffer is too small"); + } + const uint64_t blocks = group_dim / QK_K; + for (uint32_t g = 0; g < n_groups; g++) { + ds4_quantize_row_q8_K(x + (uint64_t)g * group_dim, + scratch->dense_xq + (uint64_t)g * blocks, + (int64_t)group_dim); + } + matvec_q4_K_grouped_rows_prequant(out, m, w, scratch->dense_xq, + n_groups, group_dim, rank); +} + +typedef struct { + float *out; + const uint8_t *data; + const block_q8_K *xq; + uint64_t n_tok; + uint64_t in_dim; + uint64_t out_dim; + uint64_t blocks; +} matmul_q4_K_batch_ctx; + +static void matmul_q4_K_batch_worker(void *vctx, uint64_t r0, uint64_t r1) { + matmul_q4_K_batch_ctx *ctx = vctx; + for (uint64_t r = r0; r < r1; r++) { + const block_q4_K *row = (const block_q4_K *) + (ctx->data + r * ctx->blocks * sizeof(block_q4_K)); + for (uint64_t t = 0; t < ctx->n_tok; t++) { + ds4_vec_dot_q4_K_q8_K((int)ctx->in_dim, + &ctx->out[t * ctx->out_dim + r], + row, + ctx->xq + t * ctx->blocks); + } + } +} + +static void matmul_q4_K_batch( + float *out, + const ds4_model *m, + const ds4_tensor *w, + const float *x, + uint64_t n_tok) { + dense_q4_K_expect(w, w->dim[0]); + const uint64_t in_dim = w->dim[0]; + const uint64_t blocks = in_dim / QK_K; + block_q8_K *xq = xmalloc((size_t)n_tok * blocks * sizeof(block_q8_K)); + for (uint64_t t = 0; t < n_tok; t++) { + ds4_quantize_row_q8_K(x + t * in_dim, xq + t * blocks, (int64_t)in_dim); + } + matmul_q4_K_batch_ctx ctx = { + .out = out, + .data = tensor_data(m, w), + .xq = xq, + .n_tok = n_tok, + .in_dim = in_dim, + .out_dim = w->dim[1], + .blocks = blocks, + }; + ds4_parallel_for(w->dim[1], matmul_q4_K_batch_worker, &ctx); + free(xq); +} + +typedef struct { + float *out; + const uint8_t *data; + const block_q8_K *xq; + uint64_t n_tok; + uint64_t n_groups; + uint64_t group_dim; + uint64_t blocks; + uint64_t rank; +} matmul_q4_K_grouped_batch_ctx; + +static void matmul_q4_K_grouped_batch_worker(void *vctx, uint64_t r0, uint64_t r1) { + matmul_q4_K_grouped_batch_ctx *ctx = vctx; + for (uint64_t idx = r0; idx < r1; idx++) { + const uint64_t group = idx / ctx->rank; + const block_q4_K *row = (const block_q4_K *) + (ctx->data + idx * ctx->blocks * sizeof(block_q4_K)); + for (uint64_t t = 0; t < ctx->n_tok; t++) { + ds4_vec_dot_q4_K_q8_K((int)ctx->group_dim, + &ctx->out[t * ctx->n_groups * ctx->rank + idx], + row, + ctx->xq + (t * ctx->n_groups + group) * ctx->blocks); + } + } +} + +static void matmul_q4_K_grouped_batch( + float *out, + const ds4_model *m, + const ds4_tensor *w, + const float *x, + uint64_t n_tok, + uint32_t n_groups, + uint64_t group_dim, + uint64_t rank) { + matvec_q4_K_grouped_expect(w, n_groups, group_dim, rank); + const uint64_t blocks = group_dim / QK_K; + block_q8_K *xq = xmalloc((size_t)n_tok * n_groups * blocks * sizeof(block_q8_K)); + for (uint64_t t = 0; t < n_tok; t++) { + for (uint32_t g = 0; g < n_groups; g++) { + ds4_quantize_row_q8_K(x + (t * n_groups + g) * group_dim, + xq + (t * n_groups + g) * blocks, + (int64_t)group_dim); + } + } + matmul_q4_K_grouped_batch_ctx ctx = { + .out = out, + .data = tensor_data(m, w), + .xq = xq, + .n_tok = n_tok, + .n_groups = n_groups, + .group_dim = group_dim, + .blocks = blocks, + .rank = rank, + }; + ds4_parallel_for((uint64_t)n_groups * rank, matmul_q4_K_grouped_batch_worker, &ctx); + free(xq); +} + +/* Type dispatch for the dense attention projections: the AProjQ8 GGUFs keep + * them Q8_0, the AProjQ4 ones use Q4_K. Q8_0 stays on the exact functions + * it always used so existing models remain bit-identical. */ + +static void matvec_dense_grouped_rows( + float *out, + const ds4_model *m, + const ds4_tensor *w, + const float *x, + uint32_t n_groups, + uint64_t group_dim, + uint64_t rank) { + if (w->type == DS4_TENSOR_Q4_K) { + matvec_q4_K_grouped_rows(out, m, w, x, n_groups, group_dim, rank); + } else { + matvec_q8_0_grouped_rows(out, m, w, x, n_groups, group_dim, rank); + } +} + +static void matvec_dense_grouped_rows_decode_scratch( + float *out, + const ds4_model *m, + const ds4_tensor *w, + const float *x, + uint32_t n_groups, + uint64_t group_dim, + uint64_t rank, + ds4_cpu_decode_scratch *scratch) { + if (w->type == DS4_TENSOR_Q4_K) { + matvec_q4_K_grouped_rows_decode_scratch(out, m, w, x, n_groups, + group_dim, rank, scratch); + } else { + matvec_q8_0_grouped_rows_decode_scratch(out, m, w, x, n_groups, + group_dim, rank, scratch); + } +} + +static void matmul_dense_grouped_batch( + float *out, + const ds4_model *m, + const ds4_tensor *w, + const float *x, + uint64_t n_tok, + uint32_t n_groups, + uint64_t group_dim, + uint64_t rank) { + if (w->type == DS4_TENSOR_Q4_K) { + matmul_q4_K_grouped_batch(out, m, w, x, n_tok, n_groups, group_dim, rank); + } else { + matmul_q8_0_grouped_batch(out, m, w, x, n_tok, n_groups, group_dim, rank); + } +} + +static void matmul_dense_batch( + float *out, + const ds4_model *m, + const ds4_tensor *w, + const float *x, + uint64_t n_tok) { + if (w->type == DS4_TENSOR_Q4_K) { + matmul_q4_K_batch(out, m, w, x, n_tok); + } else { + matmul_q8_0_batch(out, m, w, x, n_tok); + } +} + typedef struct { float *out; const float *data; @@ -8424,12 +8770,14 @@ static void matvec_f32(float *out, const ds4_model *m, const ds4_tensor *w, cons ds4_parallel_for(w->dim[1], matvec_f32_worker, &ctx); } -/* Dispatch for dense F32/F16/Q8_0 tensors used by auxiliary projections. */ +/* Dispatch for dense F32/F16/Q8_0/Q4_K tensors used by the attention and + * auxiliary projections. */ static void matvec_any(float *out, const ds4_model *m, const ds4_tensor *w, const float *x) { switch (w->type) { case 0: matvec_f32(out, m, w, x); break; case 1: matvec_f16(out, m, w, x); break; case 8: matvec_q8_0(out, m, w, x); break; + case DS4_TENSOR_Q4_K: matvec_q4_K(out, m, w, x); break; default: ds4_die("unsupported tensor type for dense matvec"); } @@ -10706,9 +11054,9 @@ static void layer_q_projection_normed_one( const float *q_a_norm = tensor_data(model, layer->attn_q_a_norm); - matvec_q8_0(qr, model, layer->attn_q_a, norm); + matvec_any(qr, model, layer->attn_q_a, norm); rms_norm_weight(qr_norm, qr, q_a_norm, q_rank, DS4_RMS_EPS); - matvec_q8_0(q, model, layer->attn_q_b, qr_norm); + matvec_any(q, model, layer->attn_q_b, qr_norm); head_rms_norm_inplace(q, DS4_N_HEAD, DS4_N_HEAD_DIM, DS4_RMS_EPS); free(qr_norm); @@ -10725,9 +11073,9 @@ static void layer_q_projection_with_lora_one( float *qr = xmalloc((size_t)q_rank * sizeof(qr[0])); const float *q_a_norm = tensor_data(model, layer->attn_q_a_norm); - matvec_q8_0(qr, model, layer->attn_q_a, norm); + matvec_any(qr, model, layer->attn_q_a, norm); rms_norm_weight(qr_norm, qr, q_a_norm, q_rank, DS4_RMS_EPS); - matvec_q8_0(q, model, layer->attn_q_b, qr_norm); + matvec_any(q, model, layer->attn_q_b, qr_norm); head_rms_norm_inplace(q, DS4_N_HEAD, DS4_N_HEAD_DIM, DS4_RMS_EPS); free(qr); @@ -10743,7 +11091,7 @@ static void layer_kv_projection_normed_one( const float *kv_norm = tensor_data(model, layer->attn_kv_a_norm); - matvec_q8_0(raw, model, layer->attn_kv, normed); + matvec_any(raw, model, layer->attn_kv, normed); rms_norm_weight(kv, raw, kv_norm, DS4_N_HEAD_DIM, DS4_RMS_EPS); free(raw); @@ -10758,9 +11106,9 @@ static void layer_q_projection_with_lora_one_decode_scratch( ds4_cpu_decode_scratch * scratch) { const float *q_a_norm = tensor_data(model, layer->attn_q_a_norm); - matvec_q8_0_decode_scratch(scratch->qr, model, layer->attn_q_a, norm, scratch); + matvec_any_decode_scratch(scratch->qr, model, layer->attn_q_a, norm, scratch); rms_norm_weight(qr_norm, scratch->qr, q_a_norm, DS4_N_LORA_Q, DS4_RMS_EPS); - matvec_q8_0_decode_scratch(q, model, layer->attn_q_b, qr_norm, scratch); + matvec_any_decode_scratch(q, model, layer->attn_q_b, qr_norm, scratch); head_rms_norm_inplace(q, DS4_N_HEAD, DS4_N_HEAD_DIM, DS4_RMS_EPS); } @@ -10772,7 +11120,7 @@ static void layer_kv_projection_normed_one_decode_scratch( ds4_cpu_decode_scratch * scratch) { const float *kv_norm = tensor_data(model, layer->attn_kv_a_norm); - matvec_q8_0_decode_scratch(scratch->kv_raw, model, layer->attn_kv, normed, scratch); + matvec_any_decode_scratch(scratch->kv_raw, model, layer->attn_kv, normed, scratch); rms_norm_weight(kv, scratch->kv_raw, kv_norm, DS4_N_HEAD_DIM, DS4_RMS_EPS); } @@ -11061,9 +11409,9 @@ static void layer_grouped_out_one( float *low = xcalloc((size_t)n_groups * rank, sizeof(low[0])); - matvec_q8_0_grouped_rows(low, model, layer->attn_output_a, heads, n_groups, group_dim, rank); + matvec_dense_grouped_rows(low, model, layer->attn_output_a, heads, n_groups, group_dim, rank); - matvec_q8_0(out, model, layer->attn_output_b, low); + matvec_any(out, model, layer->attn_output_b, low); free(low); } @@ -11079,9 +11427,9 @@ static void layer_grouped_out_one_decode_scratch( const uint32_t rank = 1024; memset(scratch->attn_low, 0, (size_t)n_groups * rank * sizeof(scratch->attn_low[0])); - matvec_q8_0_grouped_rows_decode_scratch(scratch->attn_low, model, layer->attn_output_a, - heads, n_groups, group_dim, rank, scratch); - matvec_q8_0_decode_scratch(out, model, layer->attn_output_b, scratch->attn_low, scratch); + matvec_dense_grouped_rows_decode_scratch(scratch->attn_low, model, layer->attn_output_a, + heads, n_groups, group_dim, rank, scratch); + matvec_any_decode_scratch(out, model, layer->attn_output_b, scratch->attn_low, scratch); } static void layer_grouped_out_batch( @@ -11097,9 +11445,9 @@ static void layer_grouped_out_batch( float *low = xcalloc((size_t)n_tok * n_groups * rank, sizeof(low[0])); - matmul_q8_0_grouped_batch(low, model, layer->attn_output_a, heads, - n_tok, n_groups, group_dim, rank); - matmul_q8_0_batch(out, model, layer->attn_output_b, low, n_tok); + matmul_dense_grouped_batch(low, model, layer->attn_output_a, heads, + n_tok, n_groups, group_dim, rank); + matmul_dense_batch(out, model, layer->attn_output_b, low, n_tok); free(low); } @@ -12822,6 +13170,7 @@ static void cpu_decode_scratch_init(ds4_cpu_decode_scratch *scratch, uint32_t ct scratch->routed_mid_all = xmalloc((size_t)DS4_N_EXPERT_USED * DS4_N_FF_EXP * sizeof(float)); scratch->routed_xq = xmalloc((size_t)(DS4_N_EMBD / QK_K) * sizeof(block_q8_K)); scratch->routed_midq = xmalloc((size_t)DS4_N_EXPERT_USED * (DS4_N_FF_EXP / QK_K) * sizeof(block_q8_K)); + scratch->dense_xq = xmalloc((size_t)((q8_cap + QK_K - 1u) / QK_K) * sizeof(block_q8_K)); scratch->routed_q8_xq = xmalloc((size_t)routed_q8_x_blocks * 32u); scratch->routed_q8_xscale = xmalloc((size_t)routed_q8_x_blocks * sizeof(float)); scratch->routed_q8_midq = xmalloc((size_t)DS4_N_EXPERT_USED * routed_q8_mid_blocks * 32u); @@ -12852,6 +13201,7 @@ static void cpu_decode_scratch_free(ds4_cpu_decode_scratch *scratch) { free(scratch->routed_q8_midq); free(scratch->routed_q8_xscale); free(scratch->routed_q8_xq); + free(scratch->dense_xq); free(scratch->routed_midq); free(scratch->routed_xq); free(scratch->routed_mid_all); @@ -13801,7 +14151,7 @@ static void layer_attention_raw_swa_batch( if (profile) t_hc_norm = now_sec() - t0; t0 = profile ? now_sec() : 0.0; - matmul_q8_0_batch(qr, model, layer->attn_q_a, attn_norm, n_tok); + matmul_dense_batch(qr, model, layer->attn_q_a, attn_norm, n_tok); for (uint32_t t = 0; t < n_tok; t++) { rms_norm_weight(qr_norm + (uint64_t)t * q_rank, qr + (uint64_t)t * q_rank, @@ -13809,7 +14159,7 @@ static void layer_attention_raw_swa_batch( q_rank, DS4_RMS_EPS); } - matmul_q8_0_batch(q, model, layer->attn_q_b, qr_norm, n_tok); + matmul_dense_batch(q, model, layer->attn_q_b, qr_norm, n_tok); for (uint32_t t = 0; t < n_tok; t++) { head_rms_norm_inplace(q + (uint64_t)t * q_dim, DS4_N_HEAD, @@ -13819,7 +14169,7 @@ static void layer_attention_raw_swa_batch( if (profile) t_q = now_sec() - t0; t0 = profile ? now_sec() : 0.0; - matmul_q8_0_batch(kv_raw, model, layer->attn_kv, attn_norm, n_tok); + matmul_dense_batch(kv_raw, model, layer->attn_kv, attn_norm, n_tok); for (uint32_t t = 0; t < n_tok; t++) { rms_norm_weight(kv + (uint64_t)t * DS4_N_HEAD_DIM, kv_raw + (uint64_t)t * DS4_N_HEAD_DIM, @@ -22778,6 +23128,13 @@ static bool metal_graph_encode_decode_layer_phase( bool fuse_kv_rope_store = false; bool qkv_pair_quad_fused = false; if (!resume_after_qkv) { + /* AProjQ4 GGUFs carry q_a/kv (and the output pair) as Q4_K; the fused + * Q8_0 pair kernel and the plain Q8_0 matvec would read those blocks as + * Q8_0 and produce garbage, so both stay gated on the actual type and + * everything else goes through the type-dispatching dense-quant path. */ + const bool qkv_proj_q8 = + layer->attn_q_a->type == DS4_TENSOR_Q8_0 && + layer->attn_kv->type == DS4_TENSOR_Q8_0; bool qkv_pair_projected = resume_after_qa_kv_raw; /* M1-M5 decode fusion: the q_a/kv Q8 pair and the four F16 compressor * projections all read the same normalized attention input and write @@ -22789,8 +23146,7 @@ static bool metal_graph_encode_decode_layer_phase( phase == METAL_DECODE_LAYER_FULL && !g->ssd_streaming && !g->ssd_streaming_cold && ds4_layer_compress_ratio(il) == 4u && - layer->attn_q_a->type == DS4_TENSOR_Q8_0 && - layer->attn_kv->type == DS4_TENSOR_Q8_0 && + qkv_proj_q8 && g->cuda_qkv_pair && !metal_graph_use_reference_qkv_pair_proj() && !metal_graph_use_reference_compressor_pair_proj() && layer->attn_compressor_kv && layer->attn_compressor_gate && @@ -22855,8 +23211,7 @@ static bool metal_graph_encode_decode_layer_phase( phase == METAL_DECODE_LAYER_FULL && !g->ssd_streaming && !g->ssd_streaming_cold && ds4_layer_compress_ratio(il) == 128u && - layer->attn_q_a->type == DS4_TENSOR_Q8_0 && - layer->attn_kv->type == DS4_TENSOR_Q8_0 && + qkv_proj_q8 && g->cuda_qkv_pair && !metal_graph_use_reference_qkv_pair_proj() && !metal_graph_use_reference_compressor_pair_proj() && layer->attn_compressor_kv && layer->attn_compressor_gate && @@ -22909,8 +23264,7 @@ static bool metal_graph_encode_decode_layer_phase( } } if (!resume_after_qa_kv_raw && ok && !qkv_pair_quad_fused && qkv_rms_fused && - layer->attn_q_a->type == DS4_TENSOR_Q8_0 && - layer->attn_kv->type == DS4_TENSOR_Q8_0 && + qkv_proj_q8 && g->cuda_qkv_pair && !metal_graph_use_reference_qkv_pair_proj()) { qkv_pair_projected = ds4_gpu_matmul_q8_0_pair_tensor( metal_graph_qr(g), @@ -23906,7 +24260,9 @@ static bool metal_graph_encode_decode_layer_phase( cuda_tp_attn_requested && !metal_graph_directional_steering_attn_enabled(g) && cuda_tp_partner_tier >= 0 && - (n_groups % 2u) == 0u; + (n_groups % 2u) == 0u && + layer->attn_output_a->type == DS4_TENSOR_Q8_0 && + layer->attn_output_b->type == DS4_TENSOR_Q8_0; ds4_gpu_tensor *tp_attn_a = NULL; /* rank partials consumed directly */ ds4_gpu_tensor *tp_attn_b = NULL; /* by the HC expand */ const bool fuse_attn_out_hc = From f7695ea09df21ff8950d2eba6350db763aaec606 Mon Sep 17 00:00:00 2001 From: oppo99 Date: Tue, 28 Jul 2026 15:07:41 +0200 Subject: [PATCH 002/189] Metal streaming: opt-in F_NOCACHE descriptor for expert preads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DS4_METAL_STREAMING_EXPERT_NOCACHE=1 serves the streaming expert preads from a second F_NOCACHE descriptor (reopened by path: a dup would share the file description with the mmap-backed fd) and turns off the F_RDADVISE readahead hints, which only warm page cache the preads no longer consume. On tight-RAM machines the ~1 GB/token of routed-expert churn was flowing through the page cache and kept evicting the ~7 GiB of mapped dense weights that decode re-reads every token; once the dense set fell out, generation collapsed to SSD-fault speed and could never recover because the next run's expert traffic flushed it again. A/B on Apple M1 Pro 16 GB, AProjQ4 gguf, greedy 32 tokens, cold page cache: baseline 0.29/0.29 tok/s (stuck across runs); with the flag the dense set survives the expert traffic and warms across runs — 0.29, 2.09, 1.93, 1.91 tok/s. Generated tokens bit-identical to baseline in all runs; --decode-consistency max_abs=0 rms=0. Opt-in because on the >=96 GB target machines everything fits in RAM and cached preads are strictly better (second touch is free). Co-Authored-By: Claude Fable 5 --- ds4_metal.m | 53 +++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 51 insertions(+), 2 deletions(-) diff --git a/ds4_metal.m b/ds4_metal.m index 45c3a4676..ac21ae8bc 100644 --- a/ds4_metal.m +++ b/ds4_metal.m @@ -363,6 +363,9 @@ static id g_moe_q4_down_slots_buffer; static id g_attn_out_group_ids_buffer; static int g_model_fd = -1; +/* Second model descriptor with F_NOCACHE for the streaming expert preads + * (DS4_METAL_STREAMING_EXPERT_NOCACHE); -1 = use the cached g_model_fd. */ +static int g_model_fd_nocache = -1; static const void *g_model_map_ptr; static uint64_t g_model_map_size; static uint64_t g_model_mapped_offset; @@ -10447,6 +10450,10 @@ void ds4_gpu_cleanup(void) { g_moe_q4_down_slots_buffer = nil; g_attn_out_group_ids_buffer = nil; g_model_fd = -1; + if (g_model_fd_nocache >= 0) { + close(g_model_fd_nocache); + g_model_fd_nocache = -1; + } g_model_map_ptr = NULL; g_model_map_size = 0; g_model_mapped_offset = 0; @@ -11399,8 +11406,45 @@ int ds4_gpu_set_model_map(const void *model_map, uint64_t model_size) { return ds4_gpu_set_model_map_range(model_map, model_size, 0, model_size, 0); } +/* DS4_METAL_STREAMING_EXPERT_NOCACHE: serve the streaming expert preads from + * a second F_NOCACHE descriptor so the ~1 GB/token of expert churn stops + * evicting the mapped dense weights from the page cache. On tight-RAM + * machines the dense working set (attention projections, shared experts, + * routing) is re-read every token through the page cache: with the default + * cached preads the expert traffic keeps pushing it out and decode collapses + * to SSD fault speed. Opt-in: on machines where everything fits in RAM the + * cached preads are strictly better (second touch is free). */ +static int ds4_gpu_stream_expert_nocache_requested(void) { + const char *env = getenv("DS4_METAL_STREAMING_EXPERT_NOCACHE"); + return env != NULL && env[0] != '\0' && env[0] != '0'; +} + int ds4_gpu_set_model_fd(int fd) { g_model_fd = fd; + if (g_model_fd_nocache >= 0) { + close(g_model_fd_nocache); + g_model_fd_nocache = -1; + } + if (fd >= 0 && ds4_gpu_stream_expert_nocache_requested()) { + /* A dup() would share the file description (and its F_NOCACHE flag) + * with the mmap-backed descriptor, so reopen the model by path. */ + char path[1024] = {0}; + int nfd = -1; + if (fcntl(fd, F_GETPATH, path) == 0) nfd = open(path, O_RDONLY); + if (nfd >= 0) { + (void)fcntl(nfd, F_SETFD, FD_CLOEXEC); + (void)fcntl(nfd, F_NOCACHE, 1); + g_model_fd_nocache = nfd; + fprintf(stderr, + "ds4: Metal streaming expert preads on a F_NOCACHE descriptor; " + "page cache reserved for the dense weights (readahead hints off)\n"); + } else { + fprintf(stderr, + "ds4: WARNING: F_NOCACHE expert descriptor unavailable (%s); " + "using cached preads\n", + strerror(errno)); + } + } return 1; } @@ -11833,7 +11877,11 @@ static void ds4_gpu_stream_expert_timing_note_cache_class( } static int ds4_gpu_stream_expert_readahead_enabled(void) { + /* F_RDADVISE warms the PAGE CACHE: with the F_NOCACHE expert descriptor + * active those pages would never be consumed by the preads and only evict + * the dense weights — the exact pollution that mode exists to stop. */ return g_ssd_streaming_mode && + g_model_fd_nocache < 0 && getenv("DS4_METAL_DISABLE_STREAMING_EXPERT_READAHEAD") == NULL; } @@ -11985,7 +12033,8 @@ static int ds4_gpu_stream_expert_pread_into( double *ms_out) { if (read_bytes) *read_bytes = 0; if (ms_out) *ms_out = 0.0; - if (g_model_fd < 0 || + const int fd = g_model_fd_nocache >= 0 ? g_model_fd_nocache : g_model_fd; + if (fd < 0 || !dst || len == 0 || offset > (uint64_t)LLONG_MAX || @@ -12001,7 +12050,7 @@ static int ds4_gpu_stream_expert_pread_into( const size_t want = rem > (uint64_t)SSIZE_MAX ? (size_t)SSIZE_MAX : (size_t)rem; ssize_t nread; do { - nread = pread(g_model_fd, dst + pos, want, (off_t)(offset + pos)); + nread = pread(fd, dst + pos, want, (off_t)(offset + pos)); } while (nread < 0 && errno == EINTR); if (nread <= 0) { ok = 0; From 8f5a7458e4ca7a09d4882e54ce8463a839bf01fe Mon Sep 17 00:00:00 2001 From: oppo99 Date: Tue, 28 Jul 2026 16:05:45 +0200 Subject: [PATCH 003/189] Metal streaming: split expert slab preads to raise NVMe queue depth DS4_METAL_STREAMING_EXPERT_PREAD_SPLIT=N expands every expert slab pread into up to N disjoint 16 KB-aligned ranges read concurrently by the existing pread pool. Decode misses queue only a handful of slabs per layer (~4 experts x 3 slabs) while NVMe drives reach their random-read ceiling around ~24 requests in flight: splitting deepens the queue at identical bytes. Results are folded back per original slab so callers keep per-task ok/bytes/ms. Default 1 = historical path. Interleaved A/B on Apple M1 Pro 16 GB, AProjQ4 gguf, greedy 32 tokens, warm state, with DS4_METAL_STREAMING_EXPERT_NOCACHE=1: split=1 1.94/1.92/1.93 tok/s, split=4 2.24/2.20/2.24 tok/s (+16%). Tokens bit-identical in all runs; --decode-consistency max_abs=0 rms=0. Same trick as the Swift port's DS4_PREAD_SPLIT, where 4 also measured best. Co-Authored-By: Claude Fable 5 --- ds4_metal.m | 97 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 96 insertions(+), 1 deletion(-) diff --git a/ds4_metal.m b/ds4_metal.m index ac21ae8bc..010fd7fa2 100644 --- a/ds4_metal.m +++ b/ds4_metal.m @@ -12293,7 +12293,7 @@ static void ds4_gpu_stream_expert_pread_pool_shutdown(void) { pthread_mutex_unlock(&g_stream_expert_pread_pool_mutex); } -static int ds4_gpu_stream_expert_pread_tasks( +static int ds4_gpu_stream_expert_pread_tasks_run( ds4_gpu_stream_expert_pread_task *tasks, uint32_t n_tasks, uint64_t *total_bytes, @@ -12456,6 +12456,101 @@ static void ds4_gpu_stream_expert_cache_cap_budget_to_locked(void) { } } +/* DS4_METAL_STREAMING_EXPERT_PREAD_SPLIT: split every expert slab pread into + * N disjoint ranges read concurrently. Decode misses queue only a handful of + * slabs per layer (~4 experts x 3 slabs), while NVMe drives reach their + * random-read ceiling around ~24 requests in flight: splitting raises the + * queue depth at identical bytes. Boundaries are 16 KB aligned so F_NOCACHE + * never reads the same page from two jobs. 1 (default) = historical path. */ +static uint32_t ds4_gpu_stream_expert_pread_split(void) { + static int cached = -1; + if (cached < 0) { + const char *env = getenv("DS4_METAL_STREAMING_EXPERT_PREAD_SPLIT"); + int v = env ? atoi(env) : 1; + if (v < 1) v = 1; + if (v > 8) v = 8; + cached = v; + } + return (uint32_t)cached; +} + +static int ds4_gpu_stream_expert_pread_tasks( + ds4_gpu_stream_expert_pread_task *tasks, + uint32_t n_tasks, + uint64_t *total_bytes, + double *wall_ms) { + if (total_bytes) *total_bytes = 0; + if (wall_ms) *wall_ms = 0.0; + if (!tasks || n_tasks == 0) return 1; + + const uint32_t split = ds4_gpu_stream_expert_pread_split(); + const uint64_t min_split_len = 256u << 10; + if (split <= 1) { + return ds4_gpu_stream_expert_pread_tasks_run(tasks, n_tasks, total_bytes, wall_ms); + } + + uint32_t n_sub = 0; + for (uint32_t i = 0; i < n_tasks; i++) { + n_sub += tasks[i].len >= min_split_len ? split : 1; + } + if (n_sub == n_tasks || n_sub < n_tasks) { + return ds4_gpu_stream_expert_pread_tasks_run(tasks, n_tasks, total_bytes, wall_ms); + } + + ds4_gpu_stream_expert_pread_task *sub = + malloc((size_t)n_sub * sizeof(sub[0])); + uint32_t *owner = malloc((size_t)n_sub * sizeof(owner[0])); + if (!sub || !owner) { + free(owner); + free(sub); + return ds4_gpu_stream_expert_pread_tasks_run(tasks, n_tasks, total_bytes, wall_ms); + } + + const uint64_t align = 16u << 10; + uint32_t w = 0; + for (uint32_t i = 0; i < n_tasks; i++) { + const uint64_t len = tasks[i].len; + if (len < min_split_len) { + sub[w] = tasks[i]; + owner[w++] = i; + continue; + } + uint64_t chunk = (len + split - 1) / split; + chunk = (chunk + align - 1) / align * align; + uint64_t off = 0; + while (off < len) { + const uint64_t part = len - off < chunk ? len - off : chunk; + sub[w].offset = tasks[i].offset + off; + sub[w].len = part; + sub[w].dst = tasks[i].dst + off; + sub[w].read_bytes = 0; + sub[w].ms = 0.0; + sub[w].ok = 0; + owner[w++] = i; + off += part; + } + } + + const int ok = ds4_gpu_stream_expert_pread_tasks_run(sub, w, total_bytes, wall_ms); + + /* Fold the split results back so callers keep per-slab ok/bytes/ms. */ + for (uint32_t i = 0; i < n_tasks; i++) { + tasks[i].ok = 1; + tasks[i].read_bytes = 0; + tasks[i].ms = 0.0; + } + for (uint32_t j = 0; j < w; j++) { + ds4_gpu_stream_expert_pread_task *t = &tasks[owner[j]]; + if (!sub[j].ok) t->ok = 0; + t->read_bytes += sub[j].read_bytes; + if (sub[j].ms > t->ms) t->ms = sub[j].ms; + } + + free(owner); + free(sub); + return ok; +} + static id ds4_gpu_stream_expert_alloc_buffer( uint64_t len, NSString *label) { From cbe484b54be75c64397ddba7a0250d160051f8f2 Mon Sep 17 00:00:00 2001 From: oppo99 Date: Tue, 28 Jul 2026 17:34:48 +0200 Subject: [PATCH 004/189] CUDA/ROCm: dense Q4_K matmul for the AProjQ4 attention projections MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The AProjQ4 GGUFs load fine on CUDA but every dense Q4_K projection hit the "matmul_quant: unsupported type 12" error. Add the missing path: - matmul_q4_K_dense_kernel + cuda_matmul_q4_K_tensor: activations are quantized to Q8_K rows with the existing routed-MoE quantizer and each weight row is reduced with the shared dev_dot_q4_K_q8_K_block, so the numerics match the already-validated MoE Q4_K path. 8 lanes per row, same idiom as the MoE decode kernels. Wired into ds4_gpu_matmul_quant_tensor as type 12. Every token re-reads the weight rows from device memory: fine for decode, correct but unoptimized for prefill chunks (a dequant+GEMM path can follow if it shows up in profiles). ROCm compiles the same source through HIP. - ds4.c: the specialized Q4_K attention-output low projection now falls through to the generic per-group dense-quant loop when the backend returns 0 (the CUDA/ROCm stub), instead of failing the layer. Metal keeps its fast path; a quick A/B confirms bit-identical tokens. Still unsupported with Q4_K projections on CUDA: the decode-TP attention-output split and the kslice fused paths — both already refuse loudly via the type guards instead of computing garbage. NOT compile-tested on a CUDA machine (authored on a Mac): testers with NVIDIA/ROCm hardware, please build and run the AProjQ4 gguf with --temp 0 plus --decode-consistency, and confirm AProjQ8 stays bit-identical to main. Co-Authored-By: Claude Fable 5 --- ds4.c | 24 ++++++++------- ds4_cuda.cu | 86 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 99 insertions(+), 11 deletions(-) diff --git a/ds4.c b/ds4.c index 343870821..344bdef7d 100644 --- a/ds4.c +++ b/ds4.c @@ -26804,16 +26804,20 @@ static bool metal_graph_attention_output_dense_quant_low( group_cnt, heads) != 0; } - if (out_a->type == DS4_TENSOR_Q4_K) { - return ds4_gpu_attention_output_low_q4_K_slice_tensor(low, - model->map, - model->size, - out_a->abs_offset, - group_dim, - rank, - group0, - group_cnt, - heads) != 0; + /* Specialized Q4_K low projection where the backend has it (Metal); a + * zero return (the CUDA/ROCm stub) falls through to the generic + * per-group dense-quant loop below instead of failing the layer. */ + if (out_a->type == DS4_TENSOR_Q4_K && + ds4_gpu_attention_output_low_q4_K_slice_tensor(low, + model->map, + model->size, + out_a->abs_offset, + group_dim, + rank, + group0, + group_cnt, + heads) != 0) { + return true; } uint64_t row_bytes = 0; if (!metal_graph_dense_quant_row_bytes(out_a, group_dim, &row_bytes)) return false; diff --git a/ds4_cuda.cu b/ds4_cuda.cu index 19b782966..3b88abe29 100644 --- a/ds4_cuda.cu +++ b/ds4_cuda.cu @@ -32075,6 +32075,87 @@ static int cuda_matmul_mmq_dense_quant( return 1; } +/* Dense Q4_K matmul for the AProjQ4 GGUFs (attn_q_a/q_b/kv/output_a/output_b + * natively Q4_K instead of Q8_0). Activations are quantized to Q8_K rows with + * the routed-MoE quantizer and every weight row is reduced with the shared + * Q4_K x Q8_K superblock dot, so the numerics match the validated MoE Q4_K + * path. 8 lanes per row, same idiom as the MoE decode kernels. Grid: + * x = row groups of 32, y = token. Every token re-reads the weight rows from + * device memory: fine for decode (n_tok <= 8), and correct-but-unoptimized + * for prefill chunks — a dequant + GEMM path can come later if the extra + * traffic shows up in profiles. */ +__global__ static void matmul_q4_K_dense_kernel( + float *out, + const char *w_base, + const cuda_block_q8_K *xq, + uint64_t row_bytes, + uint32_t xq_blocks, + uint32_t out_dim, + uint32_t n_tok) { + uint32_t lane = threadIdx.x & 7u; + uint32_t row_lane = threadIdx.x >> 3u; /* 32 rows per 256-thread block */ + uint32_t tok = blockIdx.y; + uint32_t row = blockIdx.x * 32u + row_lane; + if (tok >= n_tok || row >= out_dim) return; + const cuda_block_q8_K *xqb = xq + (uint64_t)tok * xq_blocks; + const cuda_block_q4_K *wr = + (const cuda_block_q4_K *)(w_base + (uint64_t)row * row_bytes); + float acc = 0.0f; + for (uint32_t b = lane; b < xq_blocks; b += 8u) { + acc += dev_dot_q4_K_q8_K_block(wr + b, xqb + b); + } + acc = quarter_warp_sum_f32(acc, lane); + if (lane == 0) out[(uint64_t)tok * out_dim + row] = acc; +} + +static int cuda_matmul_q4_K_tensor( + ds4_gpu_tensor *out, + const void *model_map, + uint64_t model_size, + uint64_t weight_offset, + uint64_t in_dim, + uint64_t out_dim, + const ds4_gpu_tensor *x, + uint64_t n_tok) { + if (!out || !x || !model_map || n_tok == 0) return 0; + if (in_dim == 0 || (in_dim % CUDA_QK_K) != 0) return 0; + const uint64_t blocks = in_dim / CUDA_QK_K; + const uint64_t row_bytes = blocks * sizeof(cuda_block_q4_K); + if (weight_offset > model_size || row_bytes == 0 || + out_dim > UINT64_MAX / row_bytes) { + return 0; + } + const uint64_t weight_bytes = out_dim * row_bytes; + if (weight_bytes > model_size - weight_offset) return 0; + if (x->bytes < n_tok * in_dim * sizeof(float) || + out->bytes < n_tok * out_dim * sizeof(float)) { + return 0; + } + const int logical_tier = ds4_tensor_device_idx(out); + const char *wptr = cuda_resolve_weight_ptr(model_map, weight_offset, + weight_bytes, logical_tier, + "q4_K dense"); + if (!wptr) return 0; + void *tmp = cuda_tmp_alloc_on(logical_tier, + n_tok * blocks * sizeof(cuda_block_q8_K), + "q4_K dense prequant"); + if (!tmp) return 0; + cuda_block_q8_K *xq = (cuda_block_q8_K *)tmp; + dim3 qgrid((unsigned)blocks, (unsigned)n_tok, 1); + q8_K_quantize_kernel<<>>(xq, (const float *)x->ptr, + (uint32_t)in_dim, (uint32_t)n_tok); + if (!cuda_ok(cudaGetLastError(), "q4_K dense quantize launch")) return 0; + dim3 grid(((unsigned)out_dim + 31u) / 32u, (unsigned)n_tok, 1); + matmul_q4_K_dense_kernel<<>>((float *)out->ptr, + wptr, + xq, + row_bytes, + (uint32_t)blocks, + (uint32_t)out_dim, + (uint32_t)n_tok); + return cuda_ok(cudaGetLastError(), "q4_K dense matmul launch"); +} + extern "C" int ds4_gpu_matmul_quant_tensor( ds4_gpu_tensor *out, const void *model_map, @@ -32090,12 +32171,15 @@ extern "C" int ds4_gpu_matmul_quant_tensor( return ds4_gpu_matmul_q8_0_tensor(out, model_map, model_size, weight_offset, in_dim, out_dim, x, n_tok); + case 12u: /* Q4_K (AProjQ4 dense attention projections) */ + return cuda_matmul_q4_K_tensor(out, model_map, model_size, + weight_offset, in_dim, out_dim, + x, n_tok); case 1u: /* F16 */ return ds4_gpu_matmul_f16_tensor(out, model_map, model_size, weight_offset, in_dim, out_dim, x, n_tok); case 10u: /* Q2_K */ - case 12u: /* Q4_K */ case 16u: /* IQ2_XXS */ case 39u: /* MXFP4 */ return cuda_matmul_mmq_dense_quant( From 4a7d3aa9960fdb64f8231afd4ce6db6a747c67e4 Mon Sep 17 00:00:00 2001 From: Giorgio Oppo Date: Thu, 6 Aug 2026 21:36:23 +0200 Subject: [PATCH 005/189] Accelerate Q4 CUDA prefill and add GGUF requantization --- ds4_cuda.cu | 100 ++++++++++++++++++++---- gguf-tools/deepseek4-quantize.c | 131 +++++++++++++++++++++++++++++++- 2 files changed, 216 insertions(+), 15 deletions(-) diff --git a/ds4_cuda.cu b/ds4_cuda.cu index 3b88abe29..396c2727a 100644 --- a/ds4_cuda.cu +++ b/ds4_cuda.cu @@ -32075,15 +32075,10 @@ static int cuda_matmul_mmq_dense_quant( return 1; } -/* Dense Q4_K matmul for the AProjQ4 GGUFs (attn_q_a/q_b/kv/output_a/output_b - * natively Q4_K instead of Q8_0). Activations are quantized to Q8_K rows with - * the routed-MoE quantizer and every weight row is reduced with the shared - * Q4_K x Q8_K superblock dot, so the numerics match the validated MoE Q4_K - * path. 8 lanes per row, same idiom as the MoE decode kernels. Grid: - * x = row groups of 32, y = token. Every token re-reads the weight rows from - * device memory: fine for decode (n_tok <= 8), and correct-but-unoptimized - * for prefill chunks — a dequant + GEMM path can come later if the extra - * traffic shows up in profiles. */ +/* Dense Q4_K matvec fallback for the AProjQ4 attention projections. MMQ below + * handles prefill as a tiled matrix multiply so weights are reused across + * tokens; this bandwidth-oriented kernel remains useful for decode and for + * shapes MMQ cannot accept. */ __global__ static void matmul_q4_K_dense_kernel( float *out, const char *w_base, @@ -32136,6 +32131,23 @@ static int cuda_matmul_q4_K_tensor( weight_bytes, logical_tier, "q4_K dense"); if (!wptr) return 0; + /* The scalar-token kernel below rereads every weight row for every token, + * making prefill scale almost linearly with batch length. MMQ tiles both + * axes and shares the Q4_K weights across activation columns, matching the + * fast Q8 prefill path while retaining native quantized arithmetic. */ + if (n_tok > 1u && in_dim <= INT_MAX && out_dim <= INT_MAX && + n_tok <= INT_MAX && cuda_use_mmq()) { + const int rc = ds4_mmq_q4_K_dense( + wptr, (const float *)x->ptr, (float *)out->ptr, + (int)out_dim, (int)n_tok, (int)in_dim, (cudaStream_t)0); + if (rc == 0) return 1; + fprintf(stderr, + "ds4: ds4_mmq_q4_K_dense returned %d " + "(in=%llu out=%llu n_tok=%llu); falling back\n", + rc, (unsigned long long)in_dim, + (unsigned long long)out_dim, + (unsigned long long)n_tok); + } void *tmp = cuda_tmp_alloc_on(logical_tier, n_tok * blocks * sizeof(cuda_block_q8_K), "q4_K dense prequant"); @@ -32876,10 +32888,72 @@ extern "C" int ds4_gpu_attention_output_q4_K_batch_tensor( uint64_t out_a_offset, uint64_t out_b_offset, uint32_t out_b_type, uint64_t group_dim, uint64_t rank, uint32_t n_groups, uint64_t out_dim, const ds4_gpu_tensor *heads, uint32_t n_tokens) { - (void)out; (void)low; (void)group_tmp; (void)low_tmp; - (void)model_map; (void)model_size; (void)out_a_offset; - (void)out_b_offset; (void)out_b_type; (void)group_dim; (void)rank; - (void)n_groups; (void)out_dim; (void)heads; (void)n_tokens; + if (!out || !low || !group_tmp || !low_tmp || !heads || !model_map || + group_dim == 0 || rank == 0 || n_groups == 0 || out_dim == 0 || + n_tokens < 2u || group_dim > INT_MAX || rank > INT_MAX || + out_dim > INT_MAX || n_tokens > INT_MAX || !cuda_use_mmq()) { + return 0; + } + const uint64_t low_dim = (uint64_t)n_groups * rank; + if (low_dim > INT_MAX || (group_dim % CUDA_QK_K) != 0u || + (low_dim % CUDA_QK_K) != 0u) { + return 0; + } + const uint64_t row_a_bytes = + (group_dim / CUDA_QK_K) * sizeof(cuda_block_q4_K); + if (rank > UINT64_MAX / row_a_bytes || + n_groups > UINT64_MAX / (rank * row_a_bytes)) { + return 0; + } + const uint64_t out_a_bytes = (uint64_t)n_groups * rank * row_a_bytes; + if (out_a_offset > model_size || + out_a_bytes > model_size - out_a_offset || + heads->bytes < (uint64_t)n_tokens * n_groups * group_dim * sizeof(float) || + low->bytes < (uint64_t)n_tokens * low_dim * sizeof(float) || + out->bytes < (uint64_t)n_tokens * out_dim * sizeof(float) || + group_tmp->bytes < (uint64_t)n_tokens * group_dim * sizeof(float) || + low_tmp->bytes < (uint64_t)n_tokens * rank * sizeof(float)) { + return 0; + } + const int logical_tier = ds4_tensor_device_idx(out); + const char *out_a = cuda_resolve_weight_ptr( + model_map, out_a_offset, out_a_bytes, logical_tier, "q4 attn_out_a"); + if (!out_a) return 0; + + /* Heads are token-major with groups interleaved. Pack one group at a time + * into the graph scratch, run a true token-batched MMQ, then scatter its + * rank rows into the token-major low tensor. This keeps scratch bounded by + * one group while allowing MMQ to reuse each Q4_K tile across all tokens. */ + for (uint32_t g = 0; g < n_groups; g++) { + cudaError_t ce = cudaMemcpy2DAsync( + group_tmp->ptr, group_dim * sizeof(float), + (const float *)heads->ptr + (uint64_t)g * group_dim, + (uint64_t)n_groups * group_dim * sizeof(float), + group_dim * sizeof(float), n_tokens, + cudaMemcpyDeviceToDevice, (cudaStream_t)0); + if (!cuda_ok(ce, "q4 attention output heads pack")) return 0; + const int rc = ds4_mmq_q4_K_dense( + out_a + (uint64_t)g * rank * row_a_bytes, + (const float *)group_tmp->ptr, (float *)low_tmp->ptr, + (int)rank, (int)n_tokens, (int)group_dim, (cudaStream_t)0); + if (rc != 0) return 0; + ce = cudaMemcpy2DAsync( + (float *)low->ptr + (uint64_t)g * rank, + low_dim * sizeof(float), low_tmp->ptr, rank * sizeof(float), + rank * sizeof(float), n_tokens, + cudaMemcpyDeviceToDevice, (cudaStream_t)0); + if (!cuda_ok(ce, "q4 attention output low unpack")) return 0; + } + if (out_b_type == 12u) { + return cuda_matmul_q4_K_tensor(out, model_map, model_size, + out_b_offset, low_dim, out_dim, + low, n_tokens); + } + if (out_b_type == 8u) { + return cuda_matmul_q8_0_tensor_labeled(out, model_map, model_size, + out_b_offset, low_dim, out_dim, + low, n_tokens, "q4 attn_output_b"); + } return 0; } diff --git a/gguf-tools/deepseek4-quantize.c b/gguf-tools/deepseek4-quantize.c index cb1e99a91..ceb2ac03e 100644 --- a/gguf-tools/deepseek4-quantize.c +++ b/gguf-tools/deepseek4-quantize.c @@ -1194,6 +1194,7 @@ typedef struct { } quant_policy; static bool is_attention_projection(const char *name) { + if (strstr(name, ".indexer.")) return false; return strstr(name, ".attn_kv.weight") || strstr(name, ".attn_q_a.weight") || strstr(name, ".attn_q_b.weight") || strstr(name, ".attn_output_a.weight") || strstr(name, ".attn_output_b.weight"); @@ -1959,6 +1960,112 @@ static void write_full_gguf(st_db *db, const gguf_file *tmpl, const output_conte fclose(fp); } +static void copy_bytes(FILE *dst, FILE *src, uint64_t n, const char *path) { + uint8_t buf[1u << 20]; + while (n) { + size_t chunk = n < sizeof(buf) ? (size_t)n : sizeof(buf); + if (fread(buf, 1, chunk, src) != chunk) die_errno("read source GGUF", path); + if (fwrite(buf, 1, chunk, dst) != chunk) die("write output tensor failed"); + n -= chunk; + } +} + +static void dequantize_q8_0_rows(const uint8_t *src, float *dst, + int64_t nrows, int64_t ncols) { + const int64_t blocks = ncols / 32; + for (int64_t r = 0; r < nrows; r++) { + const uint8_t *row = src + (size_t)r * (size_t)blocks * 34u; + float *out = dst + (size_t)r * (size_t)ncols; + for (int64_t b = 0; b < blocks; b++) { + uint16_t hd; + memcpy(&hd, row + (size_t)b * 34u, sizeof(hd)); + const float d = ds4q_f16_to_f32(hd); + const int8_t *qs = (const int8_t *)(row + (size_t)b * 34u + 2u); + for (int j = 0; j < 32; j++) out[b * 32 + j] = d * (float)qs[j]; + } + } +} + +static void write_requant_gguf(const gguf_file *src_g, const output_context *out_ctx, + const char *out_path, const imatrix_store *imatrix) { + FILE *src_fp = fopen(src_g->path, "rb"); + if (!src_fp) die_errno("open source GGUF", src_g->path); + FILE *fp = fopen(out_path, "wb"); + if (!fp) die_errno("open output", out_path); + if (fwrite("GGUF", 1, 4, fp) != 4) die("write GGUF magic failed"); + write_u32(fp, src_g->version); + write_u64(fp, src_g->n_tensors); + write_u64(fp, src_g->n_kv + out_ctx->n_kv_extra); + if (fwrite(src_g->kv_raw, 1, src_g->kv_raw_len, fp) != src_g->kv_raw_len) { + die("write GGUF KV failed"); + } + write_imatrix_kvs(fp, imatrix); + for (uint64_t i = 0; i < out_ctx->n_tensors; i++) { + const tensor_meta *t = &out_ctx->tensors[i]; + write_gguf_string(fp, t->name); + write_u32(fp, (uint32_t)t->n_dims); + for (int j = 0; j < t->n_dims; j++) write_u64(fp, (uint64_t)t->ne[j]); + write_u32(fp, (uint32_t)t->type); + write_u64(fp, t->new_offset); + } + off_t pos = ftello(fp); + if (pos < 0 || (size_t)pos > out_ctx->data_offset) die("bad output metadata size"); + write_padding(fp, out_ctx->data_offset - (size_t)pos); + + for (uint64_t i = 0; i < out_ctx->n_tensors; i++) { + const tensor_meta *src = &src_g->tensors[i]; + const tensor_meta *dst = &out_ctx->tensors[i]; + fprintf(stderr, "[%4" PRIu64 "/%4" PRIu64 "] %s: %s -> %s\n", + i + 1, out_ctx->n_tensors, src->name, + ds4q_type_name(src->type), ds4q_type_name(dst->type)); + if (fseeko(src_fp, (off_t)(src_g->data_offset + src->old_offset), SEEK_SET) != 0) { + die_errno("seek source GGUF", src_g->path); + } + if (src->type == dst->type) { + copy_bytes(fp, src_fp, src->size, src_g->path); + } else { + if (src->type != DS4Q_TYPE_Q8_0 || dst->type != DS4Q_TYPE_Q4_K || + src->ne[0] % 256 != 0) { + fprintf(stderr, "error: direct requantization unsupported for %s (%s -> %s)\n", + src->name, ds4q_type_name(src->type), ds4q_type_name(dst->type)); + exit(1); + } + int64_t nrows = 1; + for (int d = 1; d < src->n_dims; d++) nrows *= src->ne[d]; + const int64_t ncols = src->ne[0]; + const size_t q8_row = ds4q_row_size(DS4Q_TYPE_Q8_0, ncols); + const size_t q4_row = ds4q_row_size(DS4Q_TYPE_Q4_K, ncols); + const int64_t batch_cap = 16; + uint8_t *q8 = xmalloc((size_t)batch_cap * q8_row); + float *f32 = xmalloc((size_t)batch_cap * (size_t)ncols * sizeof(float)); + uint8_t *q4 = xmalloc((size_t)batch_cap * q4_row); + const char *names[1] = { src->name }; + const float *imat = imatrix_find(imatrix, names, 1, ncols, -1, 0); + ds4q_quantize_init(DS4Q_TYPE_Q4_K); + for (int64_t row0 = 0; row0 < nrows; row0 += batch_cap) { + const int64_t nr = nrows - row0 < batch_cap ? nrows - row0 : batch_cap; + if (fread(q8, q8_row, (size_t)nr, src_fp) != (size_t)nr) { + die_errno("read Q8_0 source tensor", src_g->path); + } + dequantize_q8_0_rows(q8, f32, nr, ncols); + const size_t wrote = ds4q_quantize_chunk( + DS4Q_TYPE_Q4_K, f32, q4, 0, nr, ncols, imat); + if (wrote != (size_t)nr * q4_row || + fwrite(q4, 1, wrote, fp) != wrote) { + die("Q4_K requantization write failed"); + } + } + free(q4); + free(f32); + free(q8); + } + const size_t padded = ds4q_pad(dst->size, out_ctx->alignment); + write_padding(fp, padded - dst->size); + } + if (fclose(fp) != 0) die_errno("close output", out_path); + fclose(src_fp); +} + static void print_plan(const gguf_file *tmpl, const output_context *out_ctx) { size_t tensor_bytes = 0; size_t changed = 0; @@ -2008,6 +2115,7 @@ static void dspark_support_defaults(dspark_support_options *o) { typedef struct { char *hf_dir; + char *source_gguf; char *template_gguf; char *out_gguf; char *compare_gguf; @@ -2713,10 +2821,11 @@ static void free_dspark_support_plan(dspark_support_plan *plan) { } static void usage(const char *argv0) { - printf("usage: %s --hf DIR --template MODEL.gguf --out OUT.gguf [options]\n", argv0); + printf("usage: %s (--hf DIR | --source-gguf MODEL.gguf) --template MODEL.gguf --out OUT.gguf [options]\n", argv0); printf("\nDeepSeek V4 Flash/Pro safetensors -> GGUF quantizer in plain C.\n\n"); printf("options:\n"); printf(" --hf DIR Hugging Face model directory with model.safetensors.index.json\n"); + printf(" --source-gguf FILE requantize directly from a GGUF (currently Q8_0 -> Q4_K)\n"); printf(" --template FILE existing DS4 GGUF used for metadata, tensor order, shapes\n"); printf(" --out FILE output GGUF path\n"); printf(" --compare-gguf FILE reference GGUF for --compare-tensor; normal mode defaults to template\n"); @@ -2815,6 +2924,8 @@ static params parse_args(int argc, char **argv) { exit(0); } else if (strcmp(arg, "--hf") == 0) { p.hf_dir = need_value(argc, argv, &i, arg); + } else if (strcmp(arg, "--source-gguf") == 0) { + p.source_gguf = need_value(argc, argv, &i, arg); } else if (strcmp(arg, "--template") == 0) { p.template_gguf = need_value(argc, argv, &i, arg); } else if (strcmp(arg, "--out") == 0) { @@ -2880,7 +2991,12 @@ static params parse_args(int argc, char **argv) { exit(1); } } - if (!p.hf_dir) die("--hf is required"); + if (!!p.hf_dir == !!p.source_gguf) { + die("exactly one of --hf or --source-gguf is required"); + } + if (p.source_gguf && (p.dspark_manifest || p.dspark_support)) { + die("--source-gguf is not supported for DSpark modes"); + } if (p.dspark_manifest && p.dspark_support) die("--dspark-manifest and --dspark-support are mutually exclusive"); if (p.dspark_manifest) return p; if (p.dspark_support) { @@ -3065,6 +3181,17 @@ int main(int argc, char **argv) { return 0; } + if (p.source_gguf) { + write_requant_gguf(&tmpl, &out_ctx, p.out_gguf, &imatrix); + fprintf(stderr, "wrote %s\n", p.out_gguf); + imatrix_free(&imatrix); + free_gguf_file(&tmpl); + free(out_ctx.tensors); + for (int i = 0; i < p.policy.n_overrides; i++) free(p.policy.overrides[i].prefix); + free(p.policy.overrides); + return 0; + } + st_db db; db_open(&db, p.hf_dir); if (p.compare_tensor) { From ef227db1af9b90f2a03eb352148d8abb30033a85 Mon Sep 17 00:00:00 2001 From: Giorgio Oppo Date: Fri, 7 Aug 2026 08:35:23 +0200 Subject: [PATCH 006/189] Optimize Q4 CUDA prefill paths --- cuda/mmq/ds4_mmq.cu | 7 ++ cuda/mmq/ds4_mmq.h | 9 ++ ds4.c | 2 +- ds4_cuda.cu | 240 ++++++++++++++++++++++++++++++++++++++------ ds4_gpu.h | 6 +- 5 files changed, 227 insertions(+), 37 deletions(-) diff --git a/cuda/mmq/ds4_mmq.cu b/cuda/mmq/ds4_mmq.cu index 36c310441..c3a270596 100644 --- a/cuda/mmq/ds4_mmq.cu +++ b/cuda/mmq/ds4_mmq.cu @@ -4935,6 +4935,13 @@ extern "C" int ds4_mmq_q4_K_dense_pair_vec( W0, W1, X, out0, out1, M, K, stream); } +extern "C" int ds4_mmq_q4_K_dense_vec( + const void * W, const float * X, float * out, + int M, int N, int K, cudaStream_t stream) { + return ds4_mmq_dense_vec_impl( + "ds4_mmq_q4_K_dense_vec", W, X, out, M, N, K, stream); +} + // Explicit instantiations. One per quant type the public API exposes. // Each instantiation drags in the load_tiles_ + vec_dot__* // device functions from mmq.cuh, so the .o objects below contain everything diff --git a/cuda/mmq/ds4_mmq.h b/cuda/mmq/ds4_mmq.h index f22d6aa36..cf0a7e66d 100644 --- a/cuda/mmq/ds4_mmq.h +++ b/cuda/mmq/ds4_mmq.h @@ -882,6 +882,15 @@ int ds4_mmq_q4_K_dense_pair_vec( int K, cudaStream_t stream); +int ds4_mmq_q4_K_dense_vec( + const void * W_q4_K, + const float * X_f32, + float * out_f32, + int M, + int N, + int K, + cudaStream_t stream); + // Set the thread-local stream that the internal cuda pool uses for // cudaMallocAsync / cudaFreeAsync. Defaults to cudaStreamPerThread. // Step 8 (CUDA Graphs) calls this with the capture stream so pool diff --git a/ds4.c b/ds4.c index 344bdef7d..b52c6a443 100644 --- a/ds4.c +++ b/ds4.c @@ -26936,7 +26936,7 @@ static bool metal_graph_attention_output_dense_quant_batch( heads, n_tokens) != 0; } - if (out_a->type == DS4_TENSOR_Q4_K && n_tokens >= 32u) { + if (out_a->type == DS4_TENSOR_Q4_K && n_tokens >= 2u) { if (ds4_gpu_attention_output_q4_K_batch_tensor(out, low, metal_graph_batch_group_tmp(g), diff --git a/ds4_cuda.cu b/ds4_cuda.cu index 396c2727a..21078cb73 100644 --- a/ds4_cuda.cu +++ b/ds4_cuda.cu @@ -32135,14 +32135,21 @@ static int cuda_matmul_q4_K_tensor( * making prefill scale almost linearly with batch length. MMQ tiles both * axes and shares the Q4_K weights across activation columns, matching the * fast Q8 prefill path while retaining native quantized arithmetic. */ - if (n_tok > 1u && in_dim <= INT_MAX && out_dim <= INT_MAX && - n_tok <= INT_MAX && cuda_use_mmq()) { - const int rc = ds4_mmq_q4_K_dense( - wptr, (const float *)x->ptr, (float *)out->ptr, - (int)out_dim, (int)n_tok, (int)in_dim, (cudaStream_t)0); + if (in_dim <= INT_MAX && out_dim <= INT_MAX && n_tok <= INT_MAX && + cuda_use_mmq()) { + /* MMVQ has lower setup cost for decode and speculative micro-batches; + * regular MMQ wins once enough token columns can share each weight + * tile. Both consume the GGUF Q4_K layout directly. */ + const int rc = n_tok <= 8u + ? ds4_mmq_q4_K_dense_vec( + wptr, (const float *)x->ptr, (float *)out->ptr, + (int)out_dim, (int)n_tok, (int)in_dim, (cudaStream_t)0) + : ds4_mmq_q4_K_dense( + wptr, (const float *)x->ptr, (float *)out->ptr, + (int)out_dim, (int)n_tok, (int)in_dim, (cudaStream_t)0); if (rc == 0) return 1; fprintf(stderr, - "ds4: ds4_mmq_q4_K_dense returned %d " + "ds4: Q4_K MMQ returned %d " "(in=%llu out=%llu n_tok=%llu); falling back\n", rc, (unsigned long long)in_dim, (unsigned long long)out_dim, @@ -32168,6 +32175,128 @@ static int cuda_matmul_q4_K_tensor( return cuda_ok(cudaGetLastError(), "q4_K dense matmul launch"); } +/* Grouped attention-output A projection for Q4_K. Heads arrive as + * [token, group, K] and the weights as [group, rank, K]. The previous + * implementation packed one group, quantized it, launched MMQ, and unpacked + * it again for every group. This kernel consumes the native layouts + * directly and evaluates eight token columns together, so each Q4_K block is + * decoded once for eight dot products. */ +__global__ static void attention_output_q4_K_grouped_tok8_kernel( + float *low, + const char *w_base, + const cuda_block_q8_K *xq, + uint64_t row_bytes, + uint32_t xq_blocks, + uint32_t rank, + uint32_t n_groups, + uint32_t n_tokens) { + const uint32_t lane = threadIdx.x & 7u; + const uint32_t row_lane = threadIdx.x >> 3u; + const uint32_t low_dim = n_groups * rank; + const uint32_t row = blockIdx.x * 32u + row_lane; + if (row >= low_dim) return; + + const uint32_t group = row / rank; + const uint32_t tok0 = blockIdx.y * 8u; + const uint32_t remaining = n_tokens - tok0; + const uint32_t np = remaining < 8u ? remaining : 8u; + const cuda_block_q4_K *wr = (const cuda_block_q4_K *)( + w_base + (uint64_t)row * row_bytes); + const cuda_block_q8_K *xqb[8] = {NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL}; + #pragma unroll + for (uint32_t p = 0; p < 8u; p++) { + if (p < np) { + xqb[p] = xq + ((uint64_t)(tok0 + p) * n_groups + group) * + xq_blocks; + } + } + float acc[8] = {0.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 0.0f}; + for (uint32_t b = lane; b < xq_blocks; b += 8u) { + dev_dot_q4_K_q8_K_block8( + wr + b, + xqb[0] ? xqb[0] + b : NULL, xqb[1] ? xqb[1] + b : NULL, + xqb[2] ? xqb[2] + b : NULL, xqb[3] ? xqb[3] + b : NULL, + xqb[4] ? xqb[4] + b : NULL, xqb[5] ? xqb[5] + b : NULL, + xqb[6] ? xqb[6] + b : NULL, xqb[7] ? xqb[7] + b : NULL, + np, acc); + } + #pragma unroll + for (uint32_t p = 0; p < 8u; p++) { + if (p < np) { + acc[p] = quarter_warp_sum_f32(acc[p], lane); + if (lane == 0) { + low[(uint64_t)(tok0 + p) * low_dim + row] = acc[p]; + } + } + } +} + +__global__ static void matmul_q4_K_kslice_kernel( + float *out, + const char *w_base, + const cuda_block_q8_K *xq, + uint64_t full_row_bytes, + uint32_t block0, + uint32_t block_count, + uint32_t out_dim) { + const uint32_t lane = threadIdx.x & 7u; + const uint32_t row_lane = threadIdx.x >> 3u; + const uint32_t row = blockIdx.x * 32u + row_lane; + if (row >= out_dim) return; + const cuda_block_q4_K *wr = (const cuda_block_q4_K *)( + w_base + (uint64_t)row * full_row_bytes) + block0; + float acc = 0.0f; + for (uint32_t b = lane; b < block_count; b += 8u) { + acc += dev_dot_q4_K_q8_K_block(wr + b, xq + b); + } + acc = quarter_warp_sum_f32(acc, lane); + if (lane == 0) out[row] = acc; +} + +static int cuda_matmul_q4_K_kslice_tensor( + ds4_gpu_tensor *out, const void *model_map, uint64_t model_size, + uint64_t weight_offset, uint64_t full_in_dim, uint64_t k_off, + uint64_t k_cnt, uint64_t out_dim, const ds4_gpu_tensor *x, + uint64_t x_elem_off) { + if (!out || !x || !model_map || full_in_dim == 0 || k_cnt == 0 || + full_in_dim > UINT32_MAX || k_cnt > UINT32_MAX || + out_dim > UINT32_MAX || out_dim > UINT64_MAX / sizeof(float) || + (full_in_dim % CUDA_QK_K) != 0u || + (k_off % CUDA_QK_K) != 0u || (k_cnt % CUDA_QK_K) != 0u || + k_off > full_in_dim || k_cnt > full_in_dim - k_off || + x_elem_off > x->bytes / sizeof(float) || + k_cnt > x->bytes / sizeof(float) - x_elem_off || + out->bytes < out_dim * sizeof(float)) { + return 0; + } + const uint64_t full_blocks = full_in_dim / CUDA_QK_K; + const uint64_t slice_blocks = k_cnt / CUDA_QK_K; + if (full_blocks > UINT64_MAX / sizeof(cuda_block_q4_K)) return 0; + const uint64_t row_bytes = full_blocks * sizeof(cuda_block_q4_K); + if (out_dim > UINT64_MAX / row_bytes || weight_offset > model_size) return 0; + const uint64_t weight_bytes = out_dim * row_bytes; + if (weight_bytes > model_size - weight_offset) return 0; + const int logical_tier = ds4_tensor_device_idx(out); + const char *wptr = cuda_resolve_weight_ptr( + model_map, weight_offset, weight_bytes, logical_tier, "q4_K kslice"); + if (!wptr) return 0; + cuda_block_q8_K *xq = (cuda_block_q8_K *)cuda_tmp_alloc_on( + logical_tier, slice_blocks * sizeof(cuda_block_q8_K), + "q4_K kslice prequant"); + if (!xq) return 0; + const float *xptr = (const float *)x->ptr + x_elem_off; + q8_K_quantize_kernel<<<(unsigned)slice_blocks, 256>>>( + xq, xptr, (uint32_t)k_cnt, 1u); + if (!cuda_ok(cudaGetLastError(), "q4_K kslice quantize launch")) return 0; + matmul_q4_K_kslice_kernel<<<((unsigned)out_dim + 31u) / 32u, 256>>>( + (float *)out->ptr, wptr, xq, row_bytes, + (uint32_t)(k_off / CUDA_QK_K), (uint32_t)slice_blocks, + (uint32_t)out_dim); + return cuda_ok(cudaGetLastError(), "q4_K kslice matmul launch"); +} + extern "C" int ds4_gpu_matmul_quant_tensor( ds4_gpu_tensor *out, const void *model_map, @@ -32806,10 +32935,17 @@ extern "C" int ds4_gpu_matmul_quant_kslice_tensor( uint64_t out_dim, const ds4_gpu_tensor *x, uint64_t x_elem_off) { - if (weight_type != 8u) return 0; - return ds4_gpu_matmul_q8_0_kslice_tensor( - out, model_map, model_size, weight_offset, - full_in_dim, k_off, k_cnt, out_dim, x, x_elem_off); + if (weight_type == 8u) { + return ds4_gpu_matmul_q8_0_kslice_tensor( + out, model_map, model_size, weight_offset, + full_in_dim, k_off, k_cnt, out_dim, x, x_elem_off); + } + if (weight_type == 12u) { + return cuda_matmul_q4_K_kslice_tensor( + out, model_map, model_size, weight_offset, + full_in_dim, k_off, k_cnt, out_dim, x, x_elem_off); + } + return 0; } extern "C" int ds4_gpu_matmul_q8_0_f16_out_tensor( @@ -32920,29 +33056,67 @@ extern "C" int ds4_gpu_attention_output_q4_K_batch_tensor( model_map, out_a_offset, out_a_bytes, logical_tier, "q4 attn_out_a"); if (!out_a) return 0; - /* Heads are token-major with groups interleaved. Pack one group at a time - * into the graph scratch, run a true token-batched MMQ, then scatter its - * rank rows into the token-major low tensor. This keeps scratch bounded by - * one group while allowing MMQ to reuse each Q4_K tile across all tokens. */ - for (uint32_t g = 0; g < n_groups; g++) { - cudaError_t ce = cudaMemcpy2DAsync( - group_tmp->ptr, group_dim * sizeof(float), - (const float *)heads->ptr + (uint64_t)g * group_dim, - (uint64_t)n_groups * group_dim * sizeof(float), - group_dim * sizeof(float), n_tokens, - cudaMemcpyDeviceToDevice, (cudaStream_t)0); - if (!cuda_ok(ce, "q4 attention output heads pack")) return 0; - const int rc = ds4_mmq_q4_K_dense( - out_a + (uint64_t)g * rank * row_a_bytes, - (const float *)group_tmp->ptr, (float *)low_tmp->ptr, - (int)rank, (int)n_tokens, (int)group_dim, (cudaStream_t)0); - if (rc != 0) return 0; - ce = cudaMemcpy2DAsync( - (float *)low->ptr + (uint64_t)g * rank, - low_dim * sizeof(float), low_tmp->ptr, rank * sizeof(float), - rank * sizeof(float), n_tokens, - cudaMemcpyDeviceToDevice, (cudaStream_t)0); - if (!cuda_ok(ce, "q4 attention output low unpack")) return 0; + int grouped_done = 0; + if (n_tokens >= 8u && + getenv("DS4_CUDA_NO_Q4_ATTN_GROUPED_TOK8") == NULL) { + const uint64_t xq_blocks = group_dim / CUDA_QK_K; + const uint64_t x_rows = (uint64_t)n_tokens * n_groups; + if (x_rows <= UINT32_MAX && xq_blocks <= UINT32_MAX && + x_rows <= UINT64_MAX / xq_blocks && + x_rows * xq_blocks <= UINT64_MAX / sizeof(cuda_block_q8_K)) { + const uint64_t xq_bytes = + x_rows * xq_blocks * sizeof(cuda_block_q8_K); + cuda_block_q8_K *xq = (cuda_block_q8_K *)cuda_tmp_alloc_on( + logical_tier, xq_bytes, "q4 attention output grouped prequant"); + if (xq) { + dim3 qgrid((unsigned)xq_blocks, (unsigned)x_rows, 1); + q8_K_quantize_kernel<<>>( + xq, (const float *)heads->ptr, (uint32_t)group_dim, + (uint32_t)x_rows); + if (!cuda_ok(cudaGetLastError(), + "q4 attention output grouped quantize launch")) { + return 0; + } + dim3 grid(((unsigned)low_dim + 31u) / 32u, + ((unsigned)n_tokens + 7u) / 8u, 1); + attention_output_q4_K_grouped_tok8_kernel<<< + grid, 256, 0, cuda_decode_stream()>>>( + (float *)low->ptr, out_a, xq, row_a_bytes, + (uint32_t)xq_blocks, (uint32_t)rank, n_groups, + n_tokens); + if (!cuda_ok(cudaGetLastError(), + "q4 attention output grouped tok8 launch")) { + return 0; + } + grouped_done = 1; + } + } + } + + if (!grouped_done) { + /* Small batches retain MMQ: its setup cost is affordable here and the + * per-group scratch keeps the graph allocation bounded. */ + for (uint32_t g = 0; g < n_groups; g++) { + cudaError_t ce = cudaMemcpy2DAsync( + group_tmp->ptr, group_dim * sizeof(float), + (const float *)heads->ptr + (uint64_t)g * group_dim, + (uint64_t)n_groups * group_dim * sizeof(float), + group_dim * sizeof(float), n_tokens, + cudaMemcpyDeviceToDevice, cuda_decode_stream()); + if (!cuda_ok(ce, "q4 attention output heads pack")) return 0; + const int rc = ds4_mmq_q4_K_dense( + out_a + (uint64_t)g * rank * row_a_bytes, + (const float *)group_tmp->ptr, (float *)low_tmp->ptr, + (int)rank, (int)n_tokens, (int)group_dim, + cuda_decode_stream()); + if (rc != 0) return 0; + ce = cudaMemcpy2DAsync( + (float *)low->ptr + (uint64_t)g * rank, + low_dim * sizeof(float), low_tmp->ptr, rank * sizeof(float), + rank * sizeof(float), n_tokens, + cudaMemcpyDeviceToDevice, cuda_decode_stream()); + if (!cuda_ok(ce, "q4 attention output low unpack")) return 0; + } } if (out_b_type == 12u) { return cuda_matmul_q4_K_tensor(out, model_map, model_size, diff --git a/ds4_gpu.h b/ds4_gpu.h index 839f3b969..791d64db8 100644 --- a/ds4_gpu.h +++ b/ds4_gpu.h @@ -340,9 +340,9 @@ int ds4_gpu_tp_failed(void); * * ds4_gpu_matmul_q8_0_kslice_tensor computes a k-range partial matvec: * out[out_dim] = W[:, k_off : k_off + k_cnt] @ x[x_elem_off : +k_cnt] where - * W rows span full_in_dim quantized Q8_0 elements. k offsets/counts must be - * multiples of 32 (Q8_0 block). Partial results from both ranks sum to the - * full projection. + * W rows span full_in_dim quantized elements. Q8_0 slices use multiples of 32; + * the generic dispatch also accepts Q4_K slices in multiples of 256. Partial + * results from both ranks sum to the full projection. * * ds4_gpu_attention_output_q8_tp_tensor is the group-sliced attention output * pair: low projection for groups [group0, group0+group_cnt) plus the From fd1306a481aad9db001a5b5b49120e89f48bfc89 Mon Sep 17 00:00:00 2001 From: Giorgio Oppo Date: Fri, 7 Aug 2026 08:48:58 +0200 Subject: [PATCH 007/189] Reuse Q4 activations across QKV projections --- cuda/mmq/ds4_mmq.cu | 34 ++++++++++++++++++++++++++- cuda/mmq/ds4_mmq.h | 14 ++++++++++++ ds4.c | 56 +++++++++++++++++++++++++++++---------------- ds4_cuda.cu | 54 +++++++++++++++++++++++++++++++++++++++++++ ds4_gpu.h | 16 +++++++++++++ ds4_metal.m | 12 ++++++++++ 6 files changed, 165 insertions(+), 21 deletions(-) diff --git a/cuda/mmq/ds4_mmq.cu b/cuda/mmq/ds4_mmq.cu index c3a270596..eed2da98e 100644 --- a/cuda/mmq/ds4_mmq.cu +++ b/cuda/mmq/ds4_mmq.cu @@ -522,7 +522,10 @@ int ds4_mmq_dense_impl( int M, int N, int K, - cudaStream_t stream) { + cudaStream_t stream, + const void * W_pair = nullptr, + float * out_pair = nullptr, + int M_pair = 0) { if (!W || !X_f32 || !out_f32) { fprintf(stderr, "%s: null pointer\n", tag); @@ -664,6 +667,26 @@ int ds4_mmq_dense_impl( return -3; } ds4_mmq_sanitize_f32(out_f32, (uint64_t)M * (uint64_t)N, stream); + if (W_pair && out_pair && M_pair > 0) { + if (out_memset_enabled()) { + cudaMemsetAsync(out_pair, 0, + (size_t)M_pair * (size_t)N * sizeof(float), stream); + } + mmq_args pair_args = args; + pair_args.x = (const char *)W_pair; + pair_args.dst = out_pair; + pair_args.nrows_x = (int64_t)M_pair; + pair_args.nrows_dst = (int64_t)M_pair; + mul_mat_q_case(*ctx, pair_args, stream); + err = cudaGetLastError(); + if (err != cudaSuccess) { + fprintf(stderr, "%s: pair mul_mat_q_case launch failed: %s\n", + tag, cudaGetErrorString(err)); + return -4; + } + ds4_mmq_sanitize_f32(out_pair, + (uint64_t)M_pair * (uint64_t)N, stream); + } return 0; } @@ -895,6 +918,15 @@ extern "C" int ds4_mmq_q4_K_dense( return ds4_mmq_dense_impl("ds4_mmq_q4_K_dense", W, X, out, M, N, K, stream); } +extern "C" int ds4_mmq_q4_K_dense_pair( + const void * W0, const void * W1, const float * X, + float * out0, float * out1, + int M0, int M1, int N, int K, cudaStream_t stream) { + return ds4_mmq_dense_impl( + "ds4_mmq_q4_K_dense_pair", W0, X, out0, M0, N, K, stream, + W1, out1, M1); +} + extern "C" int ds4_mmq_mxfp4_dense( const void * W, const float * X, float * out, int M, int N, int K, cudaStream_t stream) { diff --git a/cuda/mmq/ds4_mmq.h b/cuda/mmq/ds4_mmq.h index cf0a7e66d..0eb00ab73 100644 --- a/cuda/mmq/ds4_mmq.h +++ b/cuda/mmq/ds4_mmq.h @@ -159,6 +159,20 @@ int ds4_mmq_q4_K_dense( int K, cudaStream_t stream); +// Two dense Q4_K projections sharing the same activation matrix. The input +// is quantized to MMQ Q8_1 once and consumed by both weight matrices. +int ds4_mmq_q4_K_dense_pair( + const void * W0_q4_K, + const void * W1_q4_K, + const float * X_f32, + float * out0_f32, + float * out1_f32, + int M0, + int M1, + int N, + int K, + cudaStream_t stream); + int ds4_mmq_mxfp4_dense( const void * W_mxfp4, const float * X_f32, diff --git a/ds4.c b/ds4.c index b52c6a443..441010b47 100644 --- a/ds4.c +++ b/ds4.c @@ -29270,32 +29270,48 @@ static bool metal_graph_encode_layer_attention_batch( } DS4_METAL_PROFILE_ATTN_STAGE("norm"); DS4_METAL_PROFILE_Q_STAGE("pre_q"); - if (ok) ok = metal_graph_matmul_q8_0_named_tensor("attn_q_a", - il, - pos0, - metal_graph_batch_qr(g), - model, - layer->attn_q_a, - DS4_N_EMBD, - q_rank, - metal_graph_batch_attn_norm(g), - n_tokens); + bool qkv_q4_pair_projected = false; + if (ok && qkv_rms_fused && n_tokens >= 2u && + getenv("DS4_CUDA_NO_Q4_QKV_PAIR") == NULL && + layer->attn_q_a->type == DS4_TENSOR_Q4_K && + layer->attn_kv->type == DS4_TENSOR_Q4_K) { + qkv_q4_pair_projected = ds4_gpu_matmul_q4_K_pair_tensor( + metal_graph_batch_qr(g), metal_graph_batch_kv_raw(g), + model->map, model->size, + layer->attn_q_a->abs_offset, layer->attn_kv->abs_offset, + DS4_N_EMBD, q_rank, DS4_N_HEAD_DIM, + metal_graph_batch_attn_norm(g), n_tokens) != 0; + } + if (ok && !qkv_q4_pair_projected) { + ok = metal_graph_matmul_q8_0_named_tensor("attn_q_a", + il, + pos0, + metal_graph_batch_qr(g), + model, + layer->attn_q_a, + DS4_N_EMBD, + q_rank, + metal_graph_batch_attn_norm(g), + n_tokens); + } if (ok) { metal_graph_debug_dump_tensor("q_lora", metal_graph_batch_qr(g), (uint64_t)n_tokens * q_rank, il, pos0); } DS4_METAL_PROFILE_Q_STAGE("q_a"); if (qkv_rms_fused) { - if (ok) ok = metal_graph_matmul_q8_0_named_tensor("attn_kv", - il, - pos0, - metal_graph_batch_kv_raw(g), - model, - layer->attn_kv, - DS4_N_EMBD, - DS4_N_HEAD_DIM, - metal_graph_batch_attn_norm(g), - n_tokens); + if (ok && !qkv_q4_pair_projected) { + ok = metal_graph_matmul_q8_0_named_tensor("attn_kv", + il, + pos0, + metal_graph_batch_kv_raw(g), + model, + layer->attn_kv, + DS4_N_EMBD, + DS4_N_HEAD_DIM, + metal_graph_batch_attn_norm(g), + n_tokens); + } if (ok) { metal_graph_debug_dump_tensor("KVraw", metal_graph_batch_kv_raw(g), (uint64_t)n_tokens * DS4_N_HEAD_DIM, il, pos0); diff --git a/ds4_cuda.cu b/ds4_cuda.cu index 21078cb73..e8ade31aa 100644 --- a/ds4_cuda.cu +++ b/ds4_cuda.cu @@ -15446,6 +15446,60 @@ extern "C" int ds4_gpu_matmul_q8_0_pair_tensor( return cuda_ok(cudaGetLastError(), "matmul_q8_0 pair warp launch"); } +extern "C" int ds4_gpu_matmul_q4_K_pair_tensor( + ds4_gpu_tensor *out0, ds4_gpu_tensor *out1, + const void *model_map, uint64_t model_size, + uint64_t weight0_offset, uint64_t weight1_offset, + uint64_t in_dim, uint64_t out0_dim, uint64_t out1_dim, + const ds4_gpu_tensor *x, uint64_t n_tok) { + if (!out0 || !out1 || !x || !model_map || n_tok < 2u || + in_dim == 0 || (in_dim % CUDA_QK_K) != 0u || + in_dim > INT_MAX || out0_dim > INT_MAX || out1_dim > INT_MAX || + n_tok > INT_MAX || !cuda_use_mmq()) { + return 0; + } + const uint64_t blocks = in_dim / CUDA_QK_K; + if (blocks > UINT64_MAX / sizeof(cuda_block_q4_K)) return 0; + const uint64_t row_bytes = blocks * sizeof(cuda_block_q4_K); + if (out0_dim > UINT64_MAX / row_bytes || + out1_dim > UINT64_MAX / row_bytes || + n_tok > UINT64_MAX / in_dim || + n_tok * in_dim > UINT64_MAX / sizeof(float) || + n_tok > UINT64_MAX / out0_dim || + n_tok > UINT64_MAX / out1_dim || + n_tok * out0_dim > UINT64_MAX / sizeof(float) || + n_tok * out1_dim > UINT64_MAX / sizeof(float)) { + return 0; + } + const uint64_t w0_bytes = out0_dim * row_bytes; + const uint64_t w1_bytes = out1_dim * row_bytes; + if (weight0_offset > model_size || w0_bytes > model_size - weight0_offset || + weight1_offset > model_size || w1_bytes > model_size - weight1_offset || + x->bytes < n_tok * in_dim * sizeof(float) || + out0->bytes < n_tok * out0_dim * sizeof(float) || + out1->bytes < n_tok * out1_dim * sizeof(float)) { + return 0; + } + const int tier = ds4_tensor_device_idx(out0); + if (ds4_tensor_device_idx(out1) != tier) return 0; + const char *w0 = cuda_resolve_weight_ptr( + model_map, weight0_offset, w0_bytes, tier, "q4_K pair0"); + const char *w1 = cuda_resolve_weight_ptr( + model_map, weight1_offset, w1_bytes, tier, "q4_K pair1"); + if (!w0 || !w1) return 0; + const int rc = ds4_mmq_q4_K_dense_pair( + w0, w1, (const float *)x->ptr, + (float *)out0->ptr, (float *)out1->ptr, + (int)out0_dim, (int)out1_dim, (int)n_tok, (int)in_dim, + cuda_decode_stream()); + if (rc == 0) return 1; + fprintf(stderr, + "ds4: Q4_K pair MMQ returned %d (in=%llu out0=%llu out1=%llu n_tok=%llu); falling back\n", + rc, (unsigned long long)in_dim, (unsigned long long)out0_dim, + (unsigned long long)out1_dim, (unsigned long long)n_tok); + return 0; +} + extern "C" int ds4_gpu_matmul_q8_0_decode_rows_exact_tensor( ds4_gpu_tensor *out, const void *model_map, diff --git a/ds4_gpu.h b/ds4_gpu.h index 791d64db8..eff6fb120 100644 --- a/ds4_gpu.h +++ b/ds4_gpu.h @@ -668,6 +668,22 @@ int ds4_gpu_matmul_q4_K_pair_decode_tensor( uint64_t out_dim, const ds4_gpu_tensor *x); +/* Optional dense Q4_K pair used by prefill q_a + kv. Both projections share + * one MMQ activation quantization; a zero return requests separate fallback + * matmuls from the graph. */ +int ds4_gpu_matmul_q4_K_pair_tensor( + ds4_gpu_tensor *out0, + ds4_gpu_tensor *out1, + const void *model_map, + uint64_t model_size, + uint64_t weight0_offset, + uint64_t weight1_offset, + uint64_t in_dim, + uint64_t out0_dim, + uint64_t out1_dim, + const ds4_gpu_tensor *x, + uint64_t n_tok); + /* Multi-row decode projections that preserve the one-row reduction order. */ int ds4_gpu_matmul_q8_0_decode_rows_exact_tensor( ds4_gpu_tensor *out, diff --git a/ds4_metal.m b/ds4_metal.m index 010fd7fa2..38bd7f1b3 100644 --- a/ds4_metal.m +++ b/ds4_metal.m @@ -19018,6 +19018,18 @@ int ds4_gpu_matmul_q8_0_pair_tensor( return 1; } +int ds4_gpu_matmul_q4_K_pair_tensor( + ds4_gpu_tensor *out0, ds4_gpu_tensor *out1, + const void *model_map, uint64_t model_size, + uint64_t weight0_offset, uint64_t weight1_offset, + uint64_t in_dim, uint64_t out0_dim, uint64_t out1_dim, + const ds4_gpu_tensor *x, uint64_t n_tok) { + (void)out0; (void)out1; (void)model_map; (void)model_size; + (void)weight0_offset; (void)weight1_offset; (void)in_dim; + (void)out0_dim; (void)out1_dim; (void)x; (void)n_tok; + return 0; +} + int ds4_gpu_matmul_q8_0_f16_out_tensor( ds4_gpu_tensor *out_h, const void *model_map, From cdbcde3a21539cb5276f790aa327133cff449ea6 Mon Sep 17 00:00:00 2001 From: Giorgio Oppo Date: Fri, 7 Aug 2026 15:31:22 +0200 Subject: [PATCH 008/189] Optimize Metal Q4 decode and SSD streaming --- ds4.c | 92 ++++++++++--- ds4_metal.m | 359 ++++++++++++++++++++++++++++++++++-------------- metal/moe.metal | 28 ++++ 3 files changed, 353 insertions(+), 126 deletions(-) diff --git a/ds4.c b/ds4.c index 441010b47..dfa7ace41 100644 --- a/ds4.c +++ b/ds4.c @@ -4781,41 +4781,51 @@ static ds4_gpu_stream_expert_table graph_stream_expert_table_make( } #endif -static uint64_t ds4_streaming_manual_cache_safe_bytes( +static bool ds4_streaming_manual_cache_safe_bytes( ds4_backend backend, int ctx_size, uint32_t prefill_chunk, - bool ssd_streaming) { + bool ssd_streaming, + uint64_t non_routed_bytes, + uint64_t *safe_bytes_out) { + if (safe_bytes_out) *safe_bytes_out = 0; #ifdef DS4_NO_GPU (void)backend; (void)ctx_size; (void)prefill_chunk; (void)ssd_streaming; - return 0; + (void)non_routed_bytes; + return false; #else - const uint64_t gib = 1024ull * 1024ull * 1024ull; const uint64_t recommended = ds4_gpu_recommended_working_set_size(); - if (recommended == 0) return 0; + if (recommended == 0 || !safe_bytes_out) return false; /* - * Explicit NGB budgets name only the routed expert cache. Keep that cache - * below the graph backend's working-set recommendation after accounting for - * the graph context/KV buffers. This is intentionally not an mlock-derived - * cap: crossing too close to the recommended working set makes short - * token-major prefill spend most of its time in VM/driver synchronization. + * Explicit NGB budgets name the total routed-expert budget (prefill + * headroom plus the decode cache). On Metal, the mmap-backed non-routed + * weights share unified memory with that budget even though the startup + * report labels only the initially mapped token span as resident. Failing + * to include them lets a seemingly safe expert cache evict the dense + * working set every token. Keep the total below the backend's recommended + * working set after accounting for both graph memory and those weights. + * + * If less than the minimum usable cache remains, the normal planner emits + * its existing "budget too small" error instead of silently exceeding the + * memory-pressure limit. */ - uint64_t target = recommended > UINT64_MAX / 7ull ? - UINT64_MAX : (recommended * 7ull) / 8ull; + const uint64_t target = recommended - recommended / 8ull; const ds4_context_memory ctx_mem = ds4_context_memory_estimate_with_prefill_mode(backend, ctx_size, prefill_chunk, ssd_streaming); - uint64_t safe = 0; - if (target > ctx_mem.total_bytes) safe = target - ctx_mem.total_bytes; - safe = (safe / gib) * gib; - if (safe == 0) safe = gib; - return safe; + uint64_t fixed = ctx_mem.total_bytes; + if (backend == DS4_BACKEND_METAL) { + fixed = fixed > UINT64_MAX - non_routed_bytes ? + UINT64_MAX : fixed + non_routed_bytes; + } + *safe_bytes_out = target > fixed ? target - fixed : 0; + return true; #endif } @@ -23135,6 +23145,9 @@ static bool metal_graph_encode_decode_layer_phase( const bool qkv_proj_q8 = layer->attn_q_a->type == DS4_TENSOR_Q8_0 && layer->attn_kv->type == DS4_TENSOR_Q8_0; + const bool qkv_proj_q4 = + layer->attn_q_a->type == DS4_TENSOR_Q4_K && + layer->attn_kv->type == DS4_TENSOR_Q4_K; bool qkv_pair_projected = resume_after_qa_kv_raw; /* M1-M5 decode fusion: the q_a/kv Q8 pair and the four F16 compressor * projections all read the same normalized attention input and write @@ -23279,6 +23292,21 @@ static bool metal_graph_encode_decode_layer_phase( metal_graph_attn_norm(g), 1) != 0; } + if (!resume_after_qa_kv_raw && ok && qkv_rms_fused && qkv_proj_q4 && + !qkv_pair_projected && !metal_graph_use_reference_qkv_pair_proj()) { + qkv_pair_projected = ds4_gpu_matmul_q4_K_pair_tensor( + metal_graph_qr(g), + metal_graph_kv_raw(g), + model->map, + model->size, + layer->attn_q_a->abs_offset, + layer->attn_kv->abs_offset, + DS4_N_EMBD, + q_rank, + DS4_N_HEAD_DIM, + metal_graph_attn_norm(g), + 1) != 0; + } if (!resume_after_qa_kv_raw && ok && !qkv_pair_projected) { ok = metal_graph_matmul_dense_quant_tensor(metal_graph_qr(g), model, @@ -62587,12 +62615,36 @@ static int ds4_engine_open_internal(ds4_engine **out, } if (e->ssd_streaming && e->ssd_streaming_cache_bytes != 0) { const uint64_t requested_cache_bytes = e->ssd_streaming_cache_bytes; - const uint64_t safe_cache_bytes = + uint64_t non_routed_bytes = 0; + const bool non_routed_known = + weights_streaming_non_routed_bytes(&e->weights, + &non_routed_bytes); + if (e->backend == DS4_BACKEND_METAL && !non_routed_known) { + fprintf(stderr, + "ds4: Metal SSD streaming could not measure non-routed " + "weights for the manual cache safety cap\n"); + ds4_engine_close(e); + *out = NULL; + return 1; + } + uint64_t safe_cache_bytes = 0; + const bool safe_cache_known = ds4_streaming_manual_cache_safe_bytes(e->backend, opt->context_size, e->prefill_chunk, - e->ssd_streaming); - if (safe_cache_bytes != 0 && + e->ssd_streaming, + non_routed_bytes, + &safe_cache_bytes); + if (safe_cache_known && safe_cache_bytes == 0) { + fprintf(stderr, + "ds4: %s SSD streaming has no safe room for the requested " + "expert cache after graph and non-routed weights\n", + ds4_backend_name(e->backend)); + ds4_engine_close(e); + *out = NULL; + return 1; + } + if (safe_cache_known && e->ssd_streaming_cache_bytes > safe_cache_bytes) { e->ssd_streaming_cache_bytes = safe_cache_bytes; fprintf(stderr, diff --git a/ds4_metal.m b/ds4_metal.m index 38bd7f1b3..b904c6ed1 100644 --- a/ds4_metal.m +++ b/ds4_metal.m @@ -619,6 +619,10 @@ static void ds4_gpu_print_device_summary(void) { DS4_METAL_STREAM_EXPERT_CACHE_MAX_SLABS = 256, DS4_METAL_STREAM_EXPERT_HOTNESS_DECAY_TOKENS = 16, DS4_METAL_STREAM_EXPERT_VALIDATE_WORDS = 16, + DS4_METAL_STREAM_EXPERT_PREAD_MAX_SPLIT = 8, + DS4_METAL_STREAM_EXPERT_PENDING_MAX_TASKS = + DS4_METAL_STREAM_EXPERT_CACHE_MAX_SELECTED * 3u * + DS4_METAL_STREAM_EXPERT_PREAD_MAX_SPLIT, }; typedef struct { @@ -11940,6 +11944,7 @@ static void ds4_gpu_stream_expert_readahead_range(uint64_t offset, uint64_t len) uint32_t source_slots[DS4_METAL_STREAM_EXPERT_CACHE_MAX_SELECTED]; uint32_t n_loads; uint32_t n_tasks; + uint32_t n_tensor_tasks; uint64_t gate_expert_bytes; uint64_t down_expert_bytes; int32_t selected_ids[DS4_METAL_STREAM_EXPERT_CACHE_MAX_SELECTED]; @@ -11952,7 +11957,8 @@ static void ds4_gpu_stream_expert_readahead_range(uint64_t offset, uint64_t len) NSUInteger gate_inners[DS4_METAL_STREAM_EXPERT_CACHE_MAX_SELECTED]; NSUInteger up_inners[DS4_METAL_STREAM_EXPERT_CACHE_MAX_SELECTED]; NSUInteger down_inners[DS4_METAL_STREAM_EXPERT_CACHE_MAX_SELECTED]; - ds4_gpu_stream_expert_pread_task tasks[DS4_METAL_STREAM_EXPERT_CACHE_MAX_SELECTED * 3u]; + ds4_gpu_stream_expert_pread_task + tasks[DS4_METAL_STREAM_EXPERT_PENDING_MAX_TASKS]; double start_ms; double prepare_ms; } ds4_gpu_stream_expert_pending_load; @@ -12358,6 +12364,172 @@ static int ds4_gpu_stream_expert_pread_tasks_run( return ok; } +/* DS4_METAL_STREAMING_EXPERT_PREAD_SPLIT: split every expert slab pread into + * N disjoint ranges read concurrently. Decode misses queue only a handful of + * slabs per layer (~4 experts x 3 slabs), while NVMe drives reach their + * random-read ceiling around ~24 requests in flight: splitting raises the + * queue depth at identical bytes. Boundaries are 16 KB aligned so F_NOCACHE + * never reads the same page from two jobs. Automatic mode uses 1 for small + * caches and 4 from 64 entries upward; an explicit value always wins. */ +static uint32_t ds4_gpu_stream_expert_pread_split(void) { + static int checked; + static uint32_t explicit_split; + if (!checked) { + const char *env = getenv("DS4_METAL_STREAMING_EXPERT_PREAD_SPLIT"); + if (env) { + int v = atoi(env); + if (v < 1) v = 1; + if (v > DS4_METAL_STREAM_EXPERT_PREAD_MAX_SPLIT) { + v = DS4_METAL_STREAM_EXPERT_PREAD_MAX_SPLIT; + } + explicit_split = (uint32_t)v; + } + checked = 1; + } + if (explicit_split != 0) return explicit_split; + + /* Four 16-KiB-aligned requests help once a larger expert cache can + * sustain enough concurrent misses. With the tiny automatic cache, the + * extra requests cost more than they overlap, so retain one read. Query + * the current budget so an engine reload can choose again. */ + return ds4_gpu_stream_expert_cache_configured_count() >= 64u ? 4u : 1u; +} + +/* Expand the persistent early-load task list in place. The pool keeps a + * pointer to this storage until pending_load_finish(), so a stack or temporary + * allocation would be unsafe here. If a future shape exceeds the fixed bound, + * preserve the original unsplit tasks instead of dropping any bytes. */ +static uint32_t ds4_gpu_stream_expert_pread_expand_tasks_bounded( + ds4_gpu_stream_expert_pread_task *tasks, + uint32_t n_tasks, + uint32_t capacity) { + const uint32_t split = ds4_gpu_stream_expert_pread_split(); + const uint64_t min_split_len = 256u << 10; + const uint64_t align = 16u << 10; + enum { + max_tensor_tasks = DS4_METAL_STREAM_EXPERT_CACHE_MAX_SELECTED * 3u, + }; + if (!tasks || n_tasks == 0 || split <= 1 || + n_tasks > max_tensor_tasks || capacity < n_tasks) { + return n_tasks; + } + + ds4_gpu_stream_expert_pread_task original[max_tensor_tasks]; + memcpy(original, tasks, (size_t)n_tasks * sizeof(original[0])); + + uint32_t out = 0; + for (uint32_t i = 0; i < n_tasks; i++) { + const uint64_t len = original[i].len; + if (len < min_split_len) { + if (out >= capacity) goto unsplit_fallback; + tasks[out++] = original[i]; + continue; + } + + uint64_t chunk = len / split + (len % split != 0); + if (chunk > UINT64_MAX - (align - 1u)) goto unsplit_fallback; + chunk = ((chunk + align - 1u) / align) * align; + uint64_t off = 0; + while (off < len) { + if (out >= capacity || original[i].offset > UINT64_MAX - off) { + goto unsplit_fallback; + } + const uint64_t part = len - off < chunk ? len - off : chunk; + tasks[out++] = (ds4_gpu_stream_expert_pread_task) { + .offset = original[i].offset + off, + .len = part, + .dst = original[i].dst + off, + .read_bytes = 0, + .ms = 0.0, + .ok = 0, + }; + off += part; + } + } + return out; + +unsplit_fallback: + memcpy(tasks, original, (size_t)n_tasks * sizeof(original[0])); + return n_tasks; +} + +static int ds4_gpu_stream_expert_pread_tasks( + ds4_gpu_stream_expert_pread_task *tasks, + uint32_t n_tasks, + uint64_t *total_bytes, + double *wall_ms) { + if (total_bytes) *total_bytes = 0; + if (wall_ms) *wall_ms = 0.0; + if (!tasks || n_tasks == 0) return 1; + + const uint32_t split = ds4_gpu_stream_expert_pread_split(); + const uint64_t min_split_len = 256u << 10; + if (split <= 1) { + return ds4_gpu_stream_expert_pread_tasks_run(tasks, n_tasks, total_bytes, wall_ms); + } + + uint32_t n_sub = 0; + for (uint32_t i = 0; i < n_tasks; i++) { + n_sub += tasks[i].len >= min_split_len ? split : 1; + } + if (n_sub == n_tasks || n_sub < n_tasks) { + return ds4_gpu_stream_expert_pread_tasks_run(tasks, n_tasks, total_bytes, wall_ms); + } + + ds4_gpu_stream_expert_pread_task *sub = + malloc((size_t)n_sub * sizeof(sub[0])); + uint32_t *owner = malloc((size_t)n_sub * sizeof(owner[0])); + if (!sub || !owner) { + free(owner); + free(sub); + return ds4_gpu_stream_expert_pread_tasks_run(tasks, n_tasks, total_bytes, wall_ms); + } + + const uint64_t align = 16u << 10; + uint32_t w = 0; + for (uint32_t i = 0; i < n_tasks; i++) { + const uint64_t len = tasks[i].len; + if (len < min_split_len) { + sub[w] = tasks[i]; + owner[w++] = i; + continue; + } + uint64_t chunk = (len + split - 1) / split; + chunk = (chunk + align - 1) / align * align; + uint64_t off = 0; + while (off < len) { + const uint64_t part = len - off < chunk ? len - off : chunk; + sub[w].offset = tasks[i].offset + off; + sub[w].len = part; + sub[w].dst = tasks[i].dst + off; + sub[w].read_bytes = 0; + sub[w].ms = 0.0; + sub[w].ok = 0; + owner[w++] = i; + off += part; + } + } + + const int ok = ds4_gpu_stream_expert_pread_tasks_run(sub, w, total_bytes, wall_ms); + + /* Fold the split results back so callers keep per-slab ok/bytes/ms. */ + for (uint32_t i = 0; i < n_tasks; i++) { + tasks[i].ok = 1; + tasks[i].read_bytes = 0; + tasks[i].ms = 0.0; + } + for (uint32_t j = 0; j < w; j++) { + ds4_gpu_stream_expert_pread_task *t = &tasks[owner[j]]; + if (!sub[j].ok) t->ok = 0; + t->read_bytes += sub[j].read_bytes; + if (sub[j].ms > t->ms) t->ms = sub[j].ms; + } + + free(owner); + free(sub); + return ok; +} + static void ds4_gpu_stream_expert_cache_warn_mlock_failure( uint64_t failed_len, int err) { @@ -12456,101 +12628,6 @@ static void ds4_gpu_stream_expert_cache_cap_budget_to_locked(void) { } } -/* DS4_METAL_STREAMING_EXPERT_PREAD_SPLIT: split every expert slab pread into - * N disjoint ranges read concurrently. Decode misses queue only a handful of - * slabs per layer (~4 experts x 3 slabs), while NVMe drives reach their - * random-read ceiling around ~24 requests in flight: splitting raises the - * queue depth at identical bytes. Boundaries are 16 KB aligned so F_NOCACHE - * never reads the same page from two jobs. 1 (default) = historical path. */ -static uint32_t ds4_gpu_stream_expert_pread_split(void) { - static int cached = -1; - if (cached < 0) { - const char *env = getenv("DS4_METAL_STREAMING_EXPERT_PREAD_SPLIT"); - int v = env ? atoi(env) : 1; - if (v < 1) v = 1; - if (v > 8) v = 8; - cached = v; - } - return (uint32_t)cached; -} - -static int ds4_gpu_stream_expert_pread_tasks( - ds4_gpu_stream_expert_pread_task *tasks, - uint32_t n_tasks, - uint64_t *total_bytes, - double *wall_ms) { - if (total_bytes) *total_bytes = 0; - if (wall_ms) *wall_ms = 0.0; - if (!tasks || n_tasks == 0) return 1; - - const uint32_t split = ds4_gpu_stream_expert_pread_split(); - const uint64_t min_split_len = 256u << 10; - if (split <= 1) { - return ds4_gpu_stream_expert_pread_tasks_run(tasks, n_tasks, total_bytes, wall_ms); - } - - uint32_t n_sub = 0; - for (uint32_t i = 0; i < n_tasks; i++) { - n_sub += tasks[i].len >= min_split_len ? split : 1; - } - if (n_sub == n_tasks || n_sub < n_tasks) { - return ds4_gpu_stream_expert_pread_tasks_run(tasks, n_tasks, total_bytes, wall_ms); - } - - ds4_gpu_stream_expert_pread_task *sub = - malloc((size_t)n_sub * sizeof(sub[0])); - uint32_t *owner = malloc((size_t)n_sub * sizeof(owner[0])); - if (!sub || !owner) { - free(owner); - free(sub); - return ds4_gpu_stream_expert_pread_tasks_run(tasks, n_tasks, total_bytes, wall_ms); - } - - const uint64_t align = 16u << 10; - uint32_t w = 0; - for (uint32_t i = 0; i < n_tasks; i++) { - const uint64_t len = tasks[i].len; - if (len < min_split_len) { - sub[w] = tasks[i]; - owner[w++] = i; - continue; - } - uint64_t chunk = (len + split - 1) / split; - chunk = (chunk + align - 1) / align * align; - uint64_t off = 0; - while (off < len) { - const uint64_t part = len - off < chunk ? len - off : chunk; - sub[w].offset = tasks[i].offset + off; - sub[w].len = part; - sub[w].dst = tasks[i].dst + off; - sub[w].read_bytes = 0; - sub[w].ms = 0.0; - sub[w].ok = 0; - owner[w++] = i; - off += part; - } - } - - const int ok = ds4_gpu_stream_expert_pread_tasks_run(sub, w, total_bytes, wall_ms); - - /* Fold the split results back so callers keep per-slab ok/bytes/ms. */ - for (uint32_t i = 0; i < n_tasks; i++) { - tasks[i].ok = 1; - tasks[i].read_bytes = 0; - tasks[i].ms = 0.0; - } - for (uint32_t j = 0; j < w; j++) { - ds4_gpu_stream_expert_pread_task *t = &tasks[owner[j]]; - if (!sub[j].ok) t->ok = 0; - t->read_bytes += sub[j].read_bytes; - if (sub[j].ms > t->ms) t->ms = sub[j].ms; - } - - free(owner); - free(sub); - return ok; -} - static id ds4_gpu_stream_expert_alloc_buffer( uint64_t len, NSString *label) { @@ -15210,9 +15287,10 @@ static int ds4_gpu_stream_expert_pending_load_install( } if (ds4_gpu_stream_expert_pending_load_profile_enabled()) { fprintf(stderr, - "ds4: Metal streaming expert early-load finish layer=%u experts=%u tensors=%u bytes=%.2f GiB wall=%.3f ms\n", + "ds4: Metal streaming expert early-load finish layer=%u experts=%u tensors=%u requests=%u bytes=%.2f GiB wall=%.3f ms\n", p->layer, p->n_loads, + p->n_tensor_tasks, p->n_tasks, ds4_gpu_gib(read_bytes), elapsed_ms); @@ -15238,6 +15316,7 @@ static int ds4_gpu_stream_expert_pending_load_finish( elapsed_ms); ds4_gpu_stream_expert_pending_load_release_buffers(p); p->n_tasks = 0; + p->n_tensor_tasks = 0; p->n_loads = 0; p->prepare_ms = 0.0; return ok; @@ -15333,6 +15412,7 @@ int ds4_gpu_stream_expert_cache_begin_selected_load( p->missing_mask = 0; p->n_loads = 0; p->n_tasks = 0; + p->n_tensor_tasks = 0; p->gate_expert_bytes = gate_expert_bytes; p->down_expert_bytes = down_expert_bytes; p->prepare_ms = 0.0; @@ -15532,6 +15612,11 @@ int ds4_gpu_stream_expert_cache_begin_selected_load( } } + p->n_tensor_tasks = p->n_tasks; + p->n_tasks = ds4_gpu_stream_expert_pread_expand_tasks_bounded( + p->tasks, + p->n_tasks, + DS4_METAL_STREAM_EXPERT_PENDING_MAX_TASKS); const uint32_t n_workers = ds4_gpu_stream_expert_pread_thread_count(p->n_tasks); p->start_ms = ds4_gpu_now_ms(); @@ -15544,9 +15629,10 @@ int ds4_gpu_stream_expert_cache_begin_selected_load( p->active = 1; if (ds4_gpu_stream_expert_pending_load_profile_enabled()) { fprintf(stderr, - "ds4: Metal streaming expert early-load begin layer=%u experts=%u tensors=%u threads=%u\n", + "ds4: Metal streaming expert early-load begin layer=%u experts=%u tensors=%u requests=%u threads=%u\n", layer, p->n_loads, + p->n_tensor_tasks, p->n_tasks, n_workers); } @@ -15555,10 +15641,10 @@ int ds4_gpu_stream_expert_cache_begin_selected_load( uint64_t read_bytes = 0; double read_ms = 0.0; - if (!ds4_gpu_stream_expert_pread_tasks(p->tasks, - p->n_tasks, - &read_bytes, - &read_ms)) { + if (!ds4_gpu_stream_expert_pread_tasks_run(p->tasks, + p->n_tasks, + &read_bytes, + &read_ms)) { ds4_gpu_stream_expert_pending_load_release_buffers(p); return 0; } @@ -15569,6 +15655,7 @@ int ds4_gpu_stream_expert_cache_begin_selected_load( } ds4_gpu_stream_expert_pending_load_release_buffers(p); p->n_tasks = 0; + p->n_tensor_tasks = 0; p->n_loads = 0; p->prepare_ms = 0.0; return 1; @@ -19024,10 +19111,70 @@ int ds4_gpu_matmul_q4_K_pair_tensor( uint64_t weight0_offset, uint64_t weight1_offset, uint64_t in_dim, uint64_t out0_dim, uint64_t out1_dim, const ds4_gpu_tensor *x, uint64_t n_tok) { - (void)out0; (void)out1; (void)model_map; (void)model_size; - (void)weight0_offset; (void)weight1_offset; (void)in_dim; - (void)out0_dim; (void)out1_dim; (void)x; (void)n_tok; - return 0; + if (!g_initialized && !ds4_gpu_init()) return 0; + if (!out0 || !out1 || !model_map || !x || n_tok == 0 || n_tok > 8u || + in_dim == 0 || (in_dim % 256u) != 0 || + in_dim > UINT32_MAX || out0_dim == 0 || out1_dim == 0 || + out0_dim > UINT32_MAX || out1_dim > UINT32_MAX || + getenv("DS4_METAL_DISABLE_Q4_DENSE_PAIR") != NULL) { + return 0; + } + @autoreleasepool { + const uint64_t row_bytes = (in_dim / 256u) * 144u; + const uint64_t w0_bytes = out0_dim * row_bytes; + const uint64_t w1_bytes = out1_dim * row_bytes; + if (weight0_offset > model_size || w0_bytes > model_size - weight0_offset || + weight1_offset > model_size || w1_bytes > model_size - weight1_offset || + ds4_gpu_tensor_bytes(x) < n_tok * in_dim * sizeof(float) || + ds4_gpu_tensor_bytes(out0) < n_tok * out0_dim * sizeof(float) || + ds4_gpu_tensor_bytes(out1) < n_tok * out1_dim * sizeof(float)) { + return 0; + } + uint64_t inner0 = 0, inner1 = 0; + id w0 = ds4_gpu_wrap_model_range( + model_map, model_size, weight0_offset, w0_bytes, &inner0); + id w1 = ds4_gpu_wrap_model_range( + model_map, model_size, weight1_offset, w1_bytes, &inner1); + id xb = ds4_gpu_tensor_buffer(x); + id o0 = ds4_gpu_tensor_buffer(out0); + id o1 = ds4_gpu_tensor_buffer(out1); + if (!w0 || !w1 || !xb || !o0 || !o1) return 0; + + const int16_t nsg = 2; + id pipeline = ds4_gpu_get_mul_mv_ext_pipeline( + "kernel_mul_mv_q4_K_dense_pair_f32", nsg, 8); + if (!pipeline) return 0; + ds4_gpu_q8_0_matvec_args args0 = { + .ne00=(int32_t)in_dim, .ne01=(int32_t)out0_dim, .ne02=1, + .nb00=1, .nb01=row_bytes, .nb02=row_bytes*out0_dim, .nb03=row_bytes*out0_dim, + .ne10=(int32_t)in_dim, .ne11=(int32_t)n_tok, .ne12=1, + .nb10=sizeof(float), .nb11=in_dim*sizeof(float), + .nb12=in_dim*n_tok*sizeof(float), .nb13=in_dim*n_tok*sizeof(float), + .ne0=(int32_t)out0_dim, .ne1=(int32_t)n_tok, .nr0=2, .r2=1, .r3=1, + }; + ds4_gpu_q8_0_matvec_args args1 = args0; + args1.ne01 = args1.ne0 = (int32_t)out1_dim; + args1.nb02 = args1.nb03 = row_bytes * out1_dim; + + int owned = 0; + id cb = ds4_gpu_command_buffer(&owned); + if (!cb) return 0; + id enc = ds4_gpu_compute_encoder(cb); + [enc setComputePipelineState:pipeline]; + [enc setBytes:&args0 length:sizeof(args0) atIndex:0]; + [enc setBytes:&args1 length:sizeof(args1) atIndex:1]; + [enc setBuffer:w0 offset:(NSUInteger)inner0 atIndex:2]; + [enc setBuffer:w1 offset:(NSUInteger)inner1 atIndex:3]; + [enc setBuffer:xb offset:ds4_gpu_tensor_offset(x) atIndex:4]; + [enc setBuffer:o0 offset:ds4_gpu_tensor_offset(out0) atIndex:5]; + [enc setBuffer:o1 offset:ds4_gpu_tensor_offset(out1) atIndex:6]; + [enc setThreadgroupMemoryLength:32 atIndex:0]; + const uint64_t max_out = out0_dim > out1_dim ? out0_dim : out1_dim; + [enc dispatchThreadgroups:MTLSizeMake((max_out + 3u) / 4u, n_tok, 1) + threadsPerThreadgroup:MTLSizeMake(32, nsg, 1)]; + ds4_gpu_end_compute_encoder(cb, enc); + return ds4_gpu_finish_command_buffer(cb, owned, "paired Q4_K matvec"); + } } int ds4_gpu_matmul_q8_0_f16_out_tensor( diff --git a/metal/moe.metal b/metal/moe.metal index 7aeb9d922..cc7065247 100644 --- a/metal/moe.metal +++ b/metal/moe.metal @@ -3381,6 +3381,34 @@ kernel void kernel_mul_mv_q4_K_dense_f32( kernel_mul_mv_q4_K_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); } +// Dense Q-A/KV pair. Both projections consume the same activation rows and +// retain the standalone Q4_K reduction order. Combining them removes one +// Metal dispatch/encoder transition while keeping independently-sized output +// matrices and byte-identical arithmetic. +kernel void kernel_mul_mv_q4_K_dense_pair_f32( + constant ds4_metal_args_mul_mv & args0, + constant ds4_metal_args_mul_mv & args1, + device const char * src0_a, + device const char * src0_b, + device const char * src1, + device char * dst_a, + device char * dst_b, + threadgroup char * shmem [[threadgroup(0)]], + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]]) { + const int first_row = + (tgpig.x * FC_mul_mv_nsg + sgitg) * N_R0_Q4_K; + if (first_row < args0.ne0) { + kernel_mul_mv_q4_K_f32_impl( + args0, src0_a, src1, dst_a, shmem, tgpig, tiisg, sgitg); + } + if (first_row < args1.ne0) { + kernel_mul_mv_q4_K_f32_impl( + args1, src0_b, src1, dst_b, shmem, tgpig, tiisg, sgitg); + } +} + // DS4 attention output low projection, specialized for the fixed block // diagonal mapping used by the model: // From 9720179399730bdbd9af02af8e4f0f4fda96f8d3 Mon Sep 17 00:00:00 2001 From: Giorgio Oppo Date: Sat, 8 Aug 2026 13:09:27 +0200 Subject: [PATCH 009/189] Stabilize Q4 CUDA paths and direct GGUF requantization --- cuda/mmq/ds4_mmq.cu | 34 +--- cuda/mmq/ds4_mmq.h | 14 -- ds4.c | 1 - ds4_cuda.cu | 190 +++--------------- ds4_gpu.h | 5 +- gguf-tools/README.md | 31 +++ gguf-tools/deepseek4-quantize.c | 114 ++++++++++- .../results/score-q4-requant-imatrix-0731.tsv | 101 ++++++++++ 8 files changed, 264 insertions(+), 226 deletions(-) create mode 100644 gguf-tools/quality-testing/results/score-q4-requant-imatrix-0731.tsv diff --git a/cuda/mmq/ds4_mmq.cu b/cuda/mmq/ds4_mmq.cu index eed2da98e..c3a270596 100644 --- a/cuda/mmq/ds4_mmq.cu +++ b/cuda/mmq/ds4_mmq.cu @@ -522,10 +522,7 @@ int ds4_mmq_dense_impl( int M, int N, int K, - cudaStream_t stream, - const void * W_pair = nullptr, - float * out_pair = nullptr, - int M_pair = 0) { + cudaStream_t stream) { if (!W || !X_f32 || !out_f32) { fprintf(stderr, "%s: null pointer\n", tag); @@ -667,26 +664,6 @@ int ds4_mmq_dense_impl( return -3; } ds4_mmq_sanitize_f32(out_f32, (uint64_t)M * (uint64_t)N, stream); - if (W_pair && out_pair && M_pair > 0) { - if (out_memset_enabled()) { - cudaMemsetAsync(out_pair, 0, - (size_t)M_pair * (size_t)N * sizeof(float), stream); - } - mmq_args pair_args = args; - pair_args.x = (const char *)W_pair; - pair_args.dst = out_pair; - pair_args.nrows_x = (int64_t)M_pair; - pair_args.nrows_dst = (int64_t)M_pair; - mul_mat_q_case(*ctx, pair_args, stream); - err = cudaGetLastError(); - if (err != cudaSuccess) { - fprintf(stderr, "%s: pair mul_mat_q_case launch failed: %s\n", - tag, cudaGetErrorString(err)); - return -4; - } - ds4_mmq_sanitize_f32(out_pair, - (uint64_t)M_pair * (uint64_t)N, stream); - } return 0; } @@ -918,15 +895,6 @@ extern "C" int ds4_mmq_q4_K_dense( return ds4_mmq_dense_impl("ds4_mmq_q4_K_dense", W, X, out, M, N, K, stream); } -extern "C" int ds4_mmq_q4_K_dense_pair( - const void * W0, const void * W1, const float * X, - float * out0, float * out1, - int M0, int M1, int N, int K, cudaStream_t stream) { - return ds4_mmq_dense_impl( - "ds4_mmq_q4_K_dense_pair", W0, X, out0, M0, N, K, stream, - W1, out1, M1); -} - extern "C" int ds4_mmq_mxfp4_dense( const void * W, const float * X, float * out, int M, int N, int K, cudaStream_t stream) { diff --git a/cuda/mmq/ds4_mmq.h b/cuda/mmq/ds4_mmq.h index 0eb00ab73..cf0a7e66d 100644 --- a/cuda/mmq/ds4_mmq.h +++ b/cuda/mmq/ds4_mmq.h @@ -159,20 +159,6 @@ int ds4_mmq_q4_K_dense( int K, cudaStream_t stream); -// Two dense Q4_K projections sharing the same activation matrix. The input -// is quantized to MMQ Q8_1 once and consumed by both weight matrices. -int ds4_mmq_q4_K_dense_pair( - const void * W0_q4_K, - const void * W1_q4_K, - const float * X_f32, - float * out0_f32, - float * out1_f32, - int M0, - int M1, - int N, - int K, - cudaStream_t stream); - int ds4_mmq_mxfp4_dense( const void * W_mxfp4, const float * X_f32, diff --git a/ds4.c b/ds4.c index dfa7ace41..c6ad7c817 100644 --- a/ds4.c +++ b/ds4.c @@ -29300,7 +29300,6 @@ static bool metal_graph_encode_layer_attention_batch( DS4_METAL_PROFILE_Q_STAGE("pre_q"); bool qkv_q4_pair_projected = false; if (ok && qkv_rms_fused && n_tokens >= 2u && - getenv("DS4_CUDA_NO_Q4_QKV_PAIR") == NULL && layer->attn_q_a->type == DS4_TENSOR_Q4_K && layer->attn_kv->type == DS4_TENSOR_Q4_K) { qkv_q4_pair_projected = ds4_gpu_matmul_q4_K_pair_tensor( diff --git a/ds4_cuda.cu b/ds4_cuda.cu index e8ade31aa..b09098d17 100644 --- a/ds4_cuda.cu +++ b/ds4_cuda.cu @@ -15452,51 +15452,9 @@ extern "C" int ds4_gpu_matmul_q4_K_pair_tensor( uint64_t weight0_offset, uint64_t weight1_offset, uint64_t in_dim, uint64_t out0_dim, uint64_t out1_dim, const ds4_gpu_tensor *x, uint64_t n_tok) { - if (!out0 || !out1 || !x || !model_map || n_tok < 2u || - in_dim == 0 || (in_dim % CUDA_QK_K) != 0u || - in_dim > INT_MAX || out0_dim > INT_MAX || out1_dim > INT_MAX || - n_tok > INT_MAX || !cuda_use_mmq()) { - return 0; - } - const uint64_t blocks = in_dim / CUDA_QK_K; - if (blocks > UINT64_MAX / sizeof(cuda_block_q4_K)) return 0; - const uint64_t row_bytes = blocks * sizeof(cuda_block_q4_K); - if (out0_dim > UINT64_MAX / row_bytes || - out1_dim > UINT64_MAX / row_bytes || - n_tok > UINT64_MAX / in_dim || - n_tok * in_dim > UINT64_MAX / sizeof(float) || - n_tok > UINT64_MAX / out0_dim || - n_tok > UINT64_MAX / out1_dim || - n_tok * out0_dim > UINT64_MAX / sizeof(float) || - n_tok * out1_dim > UINT64_MAX / sizeof(float)) { - return 0; - } - const uint64_t w0_bytes = out0_dim * row_bytes; - const uint64_t w1_bytes = out1_dim * row_bytes; - if (weight0_offset > model_size || w0_bytes > model_size - weight0_offset || - weight1_offset > model_size || w1_bytes > model_size - weight1_offset || - x->bytes < n_tok * in_dim * sizeof(float) || - out0->bytes < n_tok * out0_dim * sizeof(float) || - out1->bytes < n_tok * out1_dim * sizeof(float)) { - return 0; - } - const int tier = ds4_tensor_device_idx(out0); - if (ds4_tensor_device_idx(out1) != tier) return 0; - const char *w0 = cuda_resolve_weight_ptr( - model_map, weight0_offset, w0_bytes, tier, "q4_K pair0"); - const char *w1 = cuda_resolve_weight_ptr( - model_map, weight1_offset, w1_bytes, tier, "q4_K pair1"); - if (!w0 || !w1) return 0; - const int rc = ds4_mmq_q4_K_dense_pair( - w0, w1, (const float *)x->ptr, - (float *)out0->ptr, (float *)out1->ptr, - (int)out0_dim, (int)out1_dim, (int)n_tok, (int)in_dim, - cuda_decode_stream()); - if (rc == 0) return 1; - fprintf(stderr, - "ds4: Q4_K pair MMQ returned %d (in=%llu out0=%llu out1=%llu n_tok=%llu); falling back\n", - rc, (unsigned long long)in_dim, (unsigned long long)out0_dim, - (unsigned long long)out1_dim, (unsigned long long)n_tok); + (void)out0; (void)out1; (void)model_map; (void)model_size; + (void)weight0_offset; (void)weight1_offset; (void)in_dim; + (void)out0_dim; (void)out1_dim; (void)x; (void)n_tok; return 0; } @@ -32229,64 +32187,6 @@ static int cuda_matmul_q4_K_tensor( return cuda_ok(cudaGetLastError(), "q4_K dense matmul launch"); } -/* Grouped attention-output A projection for Q4_K. Heads arrive as - * [token, group, K] and the weights as [group, rank, K]. The previous - * implementation packed one group, quantized it, launched MMQ, and unpacked - * it again for every group. This kernel consumes the native layouts - * directly and evaluates eight token columns together, so each Q4_K block is - * decoded once for eight dot products. */ -__global__ static void attention_output_q4_K_grouped_tok8_kernel( - float *low, - const char *w_base, - const cuda_block_q8_K *xq, - uint64_t row_bytes, - uint32_t xq_blocks, - uint32_t rank, - uint32_t n_groups, - uint32_t n_tokens) { - const uint32_t lane = threadIdx.x & 7u; - const uint32_t row_lane = threadIdx.x >> 3u; - const uint32_t low_dim = n_groups * rank; - const uint32_t row = blockIdx.x * 32u + row_lane; - if (row >= low_dim) return; - - const uint32_t group = row / rank; - const uint32_t tok0 = blockIdx.y * 8u; - const uint32_t remaining = n_tokens - tok0; - const uint32_t np = remaining < 8u ? remaining : 8u; - const cuda_block_q4_K *wr = (const cuda_block_q4_K *)( - w_base + (uint64_t)row * row_bytes); - const cuda_block_q8_K *xqb[8] = {NULL, NULL, NULL, NULL, - NULL, NULL, NULL, NULL}; - #pragma unroll - for (uint32_t p = 0; p < 8u; p++) { - if (p < np) { - xqb[p] = xq + ((uint64_t)(tok0 + p) * n_groups + group) * - xq_blocks; - } - } - float acc[8] = {0.0f, 0.0f, 0.0f, 0.0f, - 0.0f, 0.0f, 0.0f, 0.0f}; - for (uint32_t b = lane; b < xq_blocks; b += 8u) { - dev_dot_q4_K_q8_K_block8( - wr + b, - xqb[0] ? xqb[0] + b : NULL, xqb[1] ? xqb[1] + b : NULL, - xqb[2] ? xqb[2] + b : NULL, xqb[3] ? xqb[3] + b : NULL, - xqb[4] ? xqb[4] + b : NULL, xqb[5] ? xqb[5] + b : NULL, - xqb[6] ? xqb[6] + b : NULL, xqb[7] ? xqb[7] + b : NULL, - np, acc); - } - #pragma unroll - for (uint32_t p = 0; p < 8u; p++) { - if (p < np) { - acc[p] = quarter_warp_sum_f32(acc[p], lane); - if (lane == 0) { - low[(uint64_t)(tok0 + p) * low_dim + row] = acc[p]; - } - } - } -} - __global__ static void matmul_q4_K_kslice_kernel( float *out, const char *w_base, @@ -33110,67 +33010,29 @@ extern "C" int ds4_gpu_attention_output_q4_K_batch_tensor( model_map, out_a_offset, out_a_bytes, logical_tier, "q4 attn_out_a"); if (!out_a) return 0; - int grouped_done = 0; - if (n_tokens >= 8u && - getenv("DS4_CUDA_NO_Q4_ATTN_GROUPED_TOK8") == NULL) { - const uint64_t xq_blocks = group_dim / CUDA_QK_K; - const uint64_t x_rows = (uint64_t)n_tokens * n_groups; - if (x_rows <= UINT32_MAX && xq_blocks <= UINT32_MAX && - x_rows <= UINT64_MAX / xq_blocks && - x_rows * xq_blocks <= UINT64_MAX / sizeof(cuda_block_q8_K)) { - const uint64_t xq_bytes = - x_rows * xq_blocks * sizeof(cuda_block_q8_K); - cuda_block_q8_K *xq = (cuda_block_q8_K *)cuda_tmp_alloc_on( - logical_tier, xq_bytes, "q4 attention output grouped prequant"); - if (xq) { - dim3 qgrid((unsigned)xq_blocks, (unsigned)x_rows, 1); - q8_K_quantize_kernel<<>>( - xq, (const float *)heads->ptr, (uint32_t)group_dim, - (uint32_t)x_rows); - if (!cuda_ok(cudaGetLastError(), - "q4 attention output grouped quantize launch")) { - return 0; - } - dim3 grid(((unsigned)low_dim + 31u) / 32u, - ((unsigned)n_tokens + 7u) / 8u, 1); - attention_output_q4_K_grouped_tok8_kernel<<< - grid, 256, 0, cuda_decode_stream()>>>( - (float *)low->ptr, out_a, xq, row_a_bytes, - (uint32_t)xq_blocks, (uint32_t)rank, n_groups, - n_tokens); - if (!cuda_ok(cudaGetLastError(), - "q4 attention output grouped tok8 launch")) { - return 0; - } - grouped_done = 1; - } - } - } - - if (!grouped_done) { - /* Small batches retain MMQ: its setup cost is affordable here and the - * per-group scratch keeps the graph allocation bounded. */ - for (uint32_t g = 0; g < n_groups; g++) { - cudaError_t ce = cudaMemcpy2DAsync( - group_tmp->ptr, group_dim * sizeof(float), - (const float *)heads->ptr + (uint64_t)g * group_dim, - (uint64_t)n_groups * group_dim * sizeof(float), - group_dim * sizeof(float), n_tokens, - cudaMemcpyDeviceToDevice, cuda_decode_stream()); - if (!cuda_ok(ce, "q4 attention output heads pack")) return 0; - const int rc = ds4_mmq_q4_K_dense( - out_a + (uint64_t)g * rank * row_a_bytes, - (const float *)group_tmp->ptr, (float *)low_tmp->ptr, - (int)rank, (int)n_tokens, (int)group_dim, - cuda_decode_stream()); - if (rc != 0) return 0; - ce = cudaMemcpy2DAsync( - (float *)low->ptr + (uint64_t)g * rank, - low_dim * sizeof(float), low_tmp->ptr, rank * sizeof(float), - rank * sizeof(float), n_tokens, - cudaMemcpyDeviceToDevice, cuda_decode_stream()); - if (!cuda_ok(ce, "q4 attention output low unpack")) return 0; - } + /* Heads are token-major with groups interleaved. Pack one group at a time + * into graph scratch, run a token-batched MMQ, then scatter its rank rows + * into the token-major low tensor. This is the stable fast path on CUDA. */ + for (uint32_t g = 0; g < n_groups; g++) { + cudaError_t ce = cudaMemcpy2DAsync( + group_tmp->ptr, group_dim * sizeof(float), + (const float *)heads->ptr + (uint64_t)g * group_dim, + (uint64_t)n_groups * group_dim * sizeof(float), + group_dim * sizeof(float), n_tokens, + cudaMemcpyDeviceToDevice, cuda_decode_stream()); + if (!cuda_ok(ce, "q4 attention output heads pack")) return 0; + const int rc = ds4_mmq_q4_K_dense( + out_a + (uint64_t)g * rank * row_a_bytes, + (const float *)group_tmp->ptr, (float *)low_tmp->ptr, + (int)rank, (int)n_tokens, (int)group_dim, + cuda_decode_stream()); + if (rc != 0) return 0; + ce = cudaMemcpy2DAsync( + (float *)low->ptr + (uint64_t)g * rank, + low_dim * sizeof(float), low_tmp->ptr, rank * sizeof(float), + rank * sizeof(float), n_tokens, + cudaMemcpyDeviceToDevice, cuda_decode_stream()); + if (!cuda_ok(ce, "q4 attention output low unpack")) return 0; } if (out_b_type == 12u) { return cuda_matmul_q4_K_tensor(out, model_map, model_size, diff --git a/ds4_gpu.h b/ds4_gpu.h index eff6fb120..c3c1b6abc 100644 --- a/ds4_gpu.h +++ b/ds4_gpu.h @@ -668,9 +668,8 @@ int ds4_gpu_matmul_q4_K_pair_decode_tensor( uint64_t out_dim, const ds4_gpu_tensor *x); -/* Optional dense Q4_K pair used by prefill q_a + kv. Both projections share - * one MMQ activation quantization; a zero return requests separate fallback - * matmuls from the graph. */ +/* Optional small-token dense Q4_K pair. Backends without a beneficial paired + * kernel return zero and request separate fallback matmuls from the graph. */ int ds4_gpu_matmul_q4_K_pair_tensor( ds4_gpu_tensor *out0, ds4_gpu_tensor *out1, diff --git a/gguf-tools/README.md b/gguf-tools/README.md index f8d68410a..56330550f 100644 --- a/gguf-tools/README.md +++ b/gguf-tools/README.md @@ -66,6 +66,37 @@ family, plus enough free disk for the temporary output. Use `--dry-run` and `--compare-tensor` before starting a full write, and use `--overwrite` only when you really mean to replace an existing GGUF. +### Requantize A Q8 GGUF Directly + +Dense attention projections can be requantized directly from an existing GGUF +without the original Hugging Face safetensors. Direct requantization currently +supports `Q8_0 -> Q4_K`; tensors not selected by the policy are copied byte for +byte. The output path must differ from the source path. For a full run, write to +a temporary output name and rename it only after validation; an interrupted run +leaves a partial output file. + +Validate the plan and all required imatrix entries first: + +```sh +gguf-tools/deepseek4-quantize \ + --source-gguf /path/to/DeepSeek-V4-Flash-AProjQ8.gguf \ + --attention-proj q4_k \ + --imatrix /path/to/DeepSeek-V4-Flash-chat-v2-routed-and-dense-ds4-220k.dat \ + --imatrix-strict \ + --dry-run +``` + +Then write the Q4 GGUF: + +```sh +gguf-tools/deepseek4-quantize \ + --source-gguf /path/to/DeepSeek-V4-Flash-AProjQ8.gguf \ + --out /path/to/DeepSeek-V4-Flash-AProjQ4.gguf \ + --attention-proj q4_k \ + --imatrix /path/to/DeepSeek-V4-Flash-chat-v2-routed-and-dense-ds4-220k.dat \ + --imatrix-strict +``` + Q2 routed experts with imatrix: ```sh diff --git a/gguf-tools/deepseek4-quantize.c b/gguf-tools/deepseek4-quantize.c index ceb2ac03e..aaabf84ef 100644 --- a/gguf-tools/deepseek4-quantize.c +++ b/gguf-tools/deepseek4-quantize.c @@ -35,6 +35,7 @@ #include #include #include +#include #include #if defined(_WIN32) @@ -1725,7 +1726,7 @@ static gguf_file load_gguf_metadata_with_override(const char *path, if (!fp) die_errno("open GGUF", path); char magic[4]; if (fread(magic, 1, sizeof(magic), fp) != sizeof(magic) || memcmp(magic, "GGUF", 4) != 0) { - die("bad GGUF template"); + die("bad GGUF"); } g.version = read_u32_le_fp(fp, "GGUF version"); g.n_tensors = read_u64_le_fp(fp, "GGUF tensor count"); @@ -1872,7 +1873,7 @@ static output_context build_output_context(const gguf_file *tmpl, const quant_po const hf_model_metadata *metadata) { output_context out = {0}; out.n_tensors = tmpl->n_tensors; - out.n_kv_extra = 1 + extra_imatrix_kv_count(im); + out.n_kv_extra = (metadata ? 1u : 0u) + extra_imatrix_kv_count(im); out.alignment = tmpl->alignment; out.tensors = xcalloc((size_t)out.n_tensors, sizeof(out.tensors[0])); size_t tensor_info = 0; @@ -1901,7 +1902,8 @@ static output_context build_output_context(const gguf_file *tmpl, const quant_po } out.tensor_bytes = off; out.meta_size = 4 + 4 + 8 + 8 + tmpl->kv_raw_len + - extra_hf_metadata_kv_size(metadata) + extra_imatrix_kv_size(im) + tensor_info; + (metadata ? extra_hf_metadata_kv_size(metadata) : 0u) + + extra_imatrix_kv_size(im) + tensor_info; out.data_offset = ds4q_pad(out.meta_size, tmpl->alignment); return out; } @@ -1986,6 +1988,34 @@ static void dequantize_q8_0_rows(const uint8_t *src, float *dst, } } +static void validate_requant_plan(const gguf_file *src_g, + const output_context *out_ctx, + const imatrix_store *imatrix) { + size_t changed = 0; + for (uint64_t i = 0; i < out_ctx->n_tensors; i++) { + const tensor_meta *src = &src_g->tensors[i]; + const tensor_meta *dst = &out_ctx->tensors[i]; + if (src->type == dst->type) continue; + changed++; + if (src->type != DS4Q_TYPE_Q8_0 || dst->type != DS4Q_TYPE_Q4_K || + src->ne[0] % 256 != 0) { + fprintf(stderr, + "error: direct requantization unsupported for %s (%s -> %s)\n", + src->name, ds4q_type_name(src->type), + ds4q_type_name(dst->type)); + exit(1); + } + const char *names[1] = { src->name }; + (void)imatrix_find(imatrix, names, 1, src->ne[0], -1, 0); + } + if (changed == 0) { + die("direct requantization plan does not change any tensors"); + } + fprintf(stderr, + "validated direct GGUF requantization plan: %zu Q8_0 -> Q4_K tensors\n", + changed); +} + static void write_requant_gguf(const gguf_file *src_g, const output_context *out_ctx, const char *out_path, const imatrix_store *imatrix) { FILE *src_fp = fopen(src_g->path, "rb"); @@ -2821,12 +2851,12 @@ static void free_dspark_support_plan(dspark_support_plan *plan) { } static void usage(const char *argv0) { - printf("usage: %s (--hf DIR | --source-gguf MODEL.gguf) --template MODEL.gguf --out OUT.gguf [options]\n", argv0); + printf("usage: %s (--hf DIR --template MODEL.gguf | --source-gguf MODEL.gguf) --out OUT.gguf [options]\n", argv0); printf("\nDeepSeek V4 Flash/Pro safetensors -> GGUF quantizer in plain C.\n\n"); printf("options:\n"); printf(" --hf DIR Hugging Face model directory with model.safetensors.index.json\n"); printf(" --source-gguf FILE requantize directly from a GGUF (currently Q8_0 -> Q4_K)\n"); - printf(" --template FILE existing DS4 GGUF used for metadata, tensor order, shapes\n"); + printf(" --template FILE GGUF metadata/layout template required with --hf\n"); printf(" --out FILE output GGUF path\n"); printf(" --compare-gguf FILE reference GGUF for --compare-tensor; normal mode defaults to template\n"); printf(" --compare-tensor NAME regenerate one tensor, checksum, optionally byte-compare, and exit\n"); @@ -2908,6 +2938,40 @@ static bool file_exists(const char *path) { return true; } +static bool same_existing_file(const char *a, const char *b) { + struct stat sa; + struct stat sb; + if (!a || !b || stat(a, &sa) != 0 || stat(b, &sb) != 0) return false; + return sa.st_dev == sb.st_dev && sa.st_ino == sb.st_ino; +} + +static void require_complete_gguf(const gguf_file *g) { + uint64_t required = (uint64_t)g->data_offset; + for (uint64_t i = 0; i < g->n_tensors; i++) { + const tensor_meta *t = &g->tensors[i]; + if (t->old_offset > UINT64_MAX - (uint64_t)t->size || + (uint64_t)g->data_offset > + UINT64_MAX - (t->old_offset + (uint64_t)t->size)) { + die("GGUF tensor extent overflows file size"); + } + const uint64_t end = + (uint64_t)g->data_offset + t->old_offset + (uint64_t)t->size; + if (end > required) required = end; + } + + struct stat st; + if (stat(g->path, &st) != 0) die_errno("stat source GGUF", g->path); + if (st.st_size < 0 || (uint64_t)st.st_size < required) { + fprintf(stderr, + "error: source GGUF is incomplete: %s has %" PRIu64 + " bytes, needs at least %" PRIu64 "\n", + g->path, + st.st_size < 0 ? 0 : (uint64_t)st.st_size, + required); + exit(1); + } +} + static params parse_args(int argc, char **argv) { params p = {0}; p.policy.routed_w1 = p.policy.routed_w2 = p.policy.routed_w3 = DS4Q_TYPE_COUNT; @@ -2997,6 +3061,15 @@ static params parse_args(int argc, char **argv) { if (p.source_gguf && (p.dspark_manifest || p.dspark_support)) { die("--source-gguf is not supported for DSpark modes"); } + if (p.source_gguf && p.template_gguf) { + die("--template is not used with --source-gguf"); + } + if (p.source_gguf && p.compare_tensor) { + die("--compare-tensor is not supported with --source-gguf"); + } + if (p.imatrix_strict && !p.imatrix_file) { + die("--imatrix-strict requires --imatrix"); + } if (p.dspark_manifest && p.dspark_support) die("--dspark-manifest and --dspark-support are mutually exclusive"); if (p.dspark_manifest) return p; if (p.dspark_support) { @@ -3008,9 +3081,13 @@ static params parse_args(int argc, char **argv) { } return p; } - if (!p.template_gguf) die("--template is required"); + if (p.hf_dir && !p.template_gguf) die("--template is required with --hf"); if (!p.dry_run && !p.compare_tensor && !p.out_gguf) die("--out is required unless --dry-run or --compare-tensor is used"); if (p.compare_tensor && !p.compare_gguf) p.compare_gguf = p.template_gguf; + if (p.source_gguf && p.out_gguf && + same_existing_file(p.source_gguf, p.out_gguf)) { + die("--out must differ from --source-gguf"); + } if (p.out_gguf && file_exists(p.out_gguf) && !p.overwrite) die("output exists; use --overwrite"); return p; } @@ -3155,12 +3232,19 @@ int main(int argc, char **argv) { return 0; } - hf_model_metadata metadata = load_hf_model_metadata(p.hf_dir); - gguf_file tmpl = load_gguf_metadata_with_override(p.template_gguf, &metadata); + hf_model_metadata metadata = {0}; + gguf_file tmpl; + if (p.source_gguf) { + tmpl = load_gguf_metadata(p.source_gguf); + require_complete_gguf(&tmpl); + } else { + metadata = load_hf_model_metadata(p.hf_dir); + tmpl = load_gguf_metadata_with_override(p.template_gguf, &metadata); + } if (p.n_experts <= 0) { if (tmpl.n_experts > 0) { p.n_experts = tmpl.n_experts; - fprintf(stderr, "using %d routed experts from template metadata\n", p.n_experts); + fprintf(stderr, "using %d routed experts from GGUF metadata\n", p.n_experts); } else { p.n_experts = 256; fprintf(stderr, "warning: template has no deepseek4.expert_count; using Flash default %d routed experts\n", p.n_experts); @@ -3168,9 +3252,16 @@ int main(int argc, char **argv) { } else { fprintf(stderr, "using %d routed experts from --n-experts\n", p.n_experts); } - output_context out_ctx = build_output_context(&tmpl, &p.policy, &imatrix, &metadata); + output_context out_ctx = build_output_context( + &tmpl, &p.policy, &imatrix, p.source_gguf ? NULL : &metadata); + if (p.source_gguf) { + validate_requant_plan(&tmpl, &out_ctx, &imatrix); + } print_plan(&tmpl, &out_ctx); - printf("compress_ratios: source=config.json count=%" PRIu64 "\n", metadata.n_compress_ratios); + if (!p.source_gguf) { + printf("compress_ratios: source=config.json count=%" PRIu64 "\n", + metadata.n_compress_ratios); + } if (p.dry_run) { free_hf_model_metadata(&metadata); imatrix_free(&imatrix); @@ -3184,6 +3275,7 @@ int main(int argc, char **argv) { if (p.source_gguf) { write_requant_gguf(&tmpl, &out_ctx, p.out_gguf, &imatrix); fprintf(stderr, "wrote %s\n", p.out_gguf); + free_hf_model_metadata(&metadata); imatrix_free(&imatrix); free_gguf_file(&tmpl); free(out_ctx.tensors); diff --git a/gguf-tools/quality-testing/results/score-q4-requant-imatrix-0731.tsv b/gguf-tools/quality-testing/results/score-q4-requant-imatrix-0731.tsv new file mode 100644 index 000000000..f6ecf067f --- /dev/null +++ b/gguf-tools/quality-testing/results/score-q4-requant-imatrix-0731.tsv @@ -0,0 +1,101 @@ +id prompt_tokens target_tokens nll avg_nll first_match greedy_lcp api_ref_tokens api_target_tokens api_target_mae api_target_mean_delta api_top_items api_top_mapped api_top_coverage api_top1_count api_top1_match api_top1_rate api_topn_ref api_topn_hit api_topn_recall api_top_logprob_count api_top_mae api_top_mean_delta api_pair_total api_pair_agree api_pair_rate +case_000 17 24 17.312644945 0.721360206 0 0 24 24 0.721360206 -0.721360206 480 480 1.000000000 24 20 0.833333333 480 359 0.747916667 480 9487.421797487 9487.349661466 456 448 0.982456140 +case_001 18 24 3.519118909 0.146629955 1 16 24 24 0.146629955 -0.146629955 480 478 0.995833333 24 22 0.916666667 478 361 0.755230126 478 9482.871267111 9482.856542764 454 452 0.995594714 +case_002 19 24 9.016919033 0.375704960 1 5 24 24 0.375704960 -0.375704960 480 478 0.995833333 24 21 0.875000000 478 384 0.803347280 478 9484.455142063 9484.417414368 454 450 0.991189427 +case_003 19 24 15.901699850 0.662570827 0 0 24 24 0.662570827 -0.662570827 480 479 0.997916667 24 20 0.833333333 479 341 0.711899791 479 9485.298247316 9485.231851910 455 446 0.980219780 +case_004 15 24 8.241609899 0.343400412 1 9 24 24 0.343400412 -0.343400412 480 479 0.997916667 24 21 0.875000000 479 361 0.753653445 479 9473.645505916 9473.611094183 473 468 0.989429175 +case_005 14 24 11.783083390 0.490961808 1 14 24 24 0.490961808 -0.490961808 480 477 0.993750000 24 20 0.833333333 477 387 0.811320755 477 9482.151350529 9482.101945567 453 448 0.988962472 +case_006 18 24 5.836509358 0.243187890 1 14 24 24 0.243187890 -0.243187890 480 478 0.995833333 24 23 0.958333333 478 372 0.778242678 478 9484.115720328 9484.091299787 454 452 0.995594714 +case_007 14 24 17.234519687 0.718104987 1 7 24 24 0.718104987 -0.718104987 480 480 1.000000000 24 17 0.708333333 480 351 0.731250000 480 9470.292701638 9470.220891139 474 464 0.978902954 +case_008 20 24 6.375413965 0.265642249 1 8 24 24 0.265642249 -0.265642249 480 480 1.000000000 24 21 0.875000000 480 363 0.756250000 480 9486.949410766 9486.922846541 456 453 0.993421053 +case_009 17 24 3.422861613 0.142619234 1 9 24 24 0.142619234 -0.142619234 480 475 0.989583333 24 22 0.916666667 475 370 0.778947368 475 9480.378199020 9480.363786971 451 449 0.995565410 +case_010 26 24 8.309394612 0.346224776 1 1 24 24 0.346224776 -0.346224776 480 477 0.993750000 24 21 0.875000000 477 370 0.775681342 477 9479.529638554 9479.494798325 453 447 0.986754967 +case_011 21 24 5.323985858 0.221832744 0 0 24 24 0.221832744 -0.221832744 480 479 0.997916667 24 22 0.916666667 479 374 0.780793319 479 9485.093759309 9485.071529723 455 453 0.995604396 +case_012 22 24 15.004470772 0.625186282 0 0 24 24 0.625186282 -0.625186282 480 480 1.000000000 24 18 0.750000000 480 363 0.756250000 480 9485.013779102 9484.951260474 456 446 0.978070175 +case_013 22 24 1.986351234 0.082764635 1 24 24 24 0.082764635 -0.082764635 480 480 1.000000000 24 24 1.000000000 480 371 0.772916667 480 9485.200640849 9485.192364385 456 456 1.000000000 +case_014 21 24 7.670987196 0.319624466 0 0 24 24 0.319624466 -0.319624466 480 479 0.997916667 24 22 0.916666667 479 368 0.768267223 479 9484.922548967 9484.890519793 455 452 0.993406593 +case_015 22 24 16.408808564 0.683700357 0 0 24 24 0.683700357 -0.683700357 480 480 1.000000000 24 20 0.833333333 480 381 0.793750000 480 9486.967876887 9486.899506851 456 443 0.971491228 +case_016 19 24 26.730866900 1.113786121 0 0 24 24 1.113786121 -1.113786121 480 479 0.997916667 24 14 0.583333333 479 350 0.730688935 479 9477.064931615 9476.953320479 473 450 0.951374207 +case_017 26 24 12.455984403 0.518999350 0 0 24 24 0.518999350 -0.518999350 480 480 1.000000000 24 17 0.708333333 480 348 0.725000000 480 9484.117106852 9484.065206917 456 449 0.984649123 +case_018 25 24 5.721043043 0.238376793 1 1 24 24 0.238376793 -0.238376793 480 479 0.997916667 24 22 0.916666667 479 369 0.770354906 479 9484.483962822 9484.460075377 455 452 0.993406593 +case_019 19 24 15.545420350 0.647725848 1 13 24 24 0.647725848 -0.647725848 480 478 0.995833333 24 18 0.750000000 478 347 0.725941423 478 9478.883681977 9478.818638377 472 459 0.972457627 +case_020 18 24 8.268487782 0.344520324 1 9 24 24 0.344520324 -0.344520324 480 480 1.000000000 24 23 0.958333333 480 370 0.770833333 480 9472.865039636 9472.830587603 474 472 0.995780591 +case_021 18 24 0.338914253 0.014121427 1 24 24 24 0.014121427 -0.014121427 480 475 0.989583333 24 24 1.000000000 475 386 0.812631579 475 9477.136670357 9477.135243349 451 451 1.000000000 +case_022 16 24 6.040604720 0.251691863 0 0 24 24 0.251691863 -0.251691863 480 479 0.997916667 24 22 0.916666667 479 360 0.751565762 479 9483.089156574 9483.063934842 455 450 0.989010989 +case_023 14 24 14.442730262 0.601780428 0 0 24 24 0.601780428 -0.601780428 480 478 0.995833333 24 19 0.791666667 478 346 0.723849372 478 9482.216005966 9482.155576133 454 444 0.977973568 +case_024 15 24 8.854954396 0.368956433 1 1 24 24 0.368956433 -0.368956433 480 478 0.995833333 24 22 0.916666667 478 384 0.803347280 478 9485.557892918 9485.520842899 454 452 0.995594714 +case_025 20 24 2.162639309 0.090109971 1 24 24 24 0.090109971 -0.090109971 480 479 0.997916667 24 24 1.000000000 479 361 0.753653445 479 9482.883694833 9482.874665024 455 455 1.000000000 +case_026 17 24 10.303078678 0.429294945 1 16 24 24 0.429294945 -0.429294945 480 480 1.000000000 24 21 0.875000000 480 365 0.760416667 480 9470.195046088 9470.152116593 492 484 0.983739837 +case_027 14 24 4.583833605 0.190993067 1 4 24 24 0.190993067 -0.190993067 480 479 0.997916667 24 22 0.916666667 479 354 0.739039666 479 9484.276314923 9484.257175743 455 453 0.995604396 +case_028 16 24 9.848800531 0.410366689 0 0 24 24 0.410366689 -0.410366689 480 479 0.997916667 24 19 0.791666667 479 339 0.707724426 479 9483.994738382 9483.953616042 455 449 0.986813187 +case_029 18 24 17.256905416 0.719037726 0 0 24 24 0.719037726 -0.719037726 480 480 1.000000000 24 17 0.708333333 480 381 0.793750000 480 9489.060572890 9488.988669117 456 442 0.969298246 +case_030 23 24 6.501602475 0.270900103 1 14 24 24 0.270900103 -0.270900103 480 480 1.000000000 24 22 0.916666667 480 382 0.795833333 480 9485.646879825 9485.619789814 456 454 0.995614035 +case_031 22 24 4.198487506 0.174936979 1 3 24 24 0.174936979 -0.174936979 480 477 0.993750000 24 22 0.916666667 477 376 0.788259958 477 9480.597036038 9480.579432317 453 451 0.995584989 +case_032 18 24 6.839275155 0.284969798 0 0 24 24 0.284969798 -0.284969798 480 480 1.000000000 24 21 0.875000000 480 365 0.760416667 480 9483.364223786 9483.335726806 456 452 0.991228070 +case_033 15 24 9.137401905 0.380725079 0 0 24 24 0.380725079 -0.380725079 480 478 0.995833333 24 21 0.875000000 478 370 0.774058577 478 9482.877146716 9482.838914909 454 447 0.984581498 +case_034 22 24 5.741615119 0.239233963 1 6 24 24 0.239233963 -0.239233963 480 479 0.997916667 24 22 0.916666667 479 380 0.793319415 479 9480.537679329 9480.513705988 455 452 0.993406593 +case_035 23 24 10.828044113 0.451168505 0 0 24 24 0.451168505 -0.451168505 480 479 0.997916667 24 21 0.875000000 479 369 0.770354906 479 9483.513360421 9483.468149381 455 448 0.984615385 +case_036 21 24 3.612839144 0.150534964 0 0 24 24 0.150534964 -0.150534964 480 479 0.997916667 24 23 0.958333333 479 364 0.759916493 479 9483.886827858 9483.871742934 455 454 0.997802198 +case_037 18 24 3.712592811 0.154691367 0 0 24 24 0.154691367 -0.154691367 480 479 0.997916667 24 22 0.916666667 479 359 0.749478079 479 9479.677214647 9479.661713215 455 451 0.991208791 +case_038 26 24 6.750856550 0.281285690 1 20 24 24 0.281285690 -0.281285690 480 479 0.997916667 24 23 0.958333333 479 343 0.716075157 479 9481.224720656 9481.196533364 455 453 0.995604396 +case_039 23 24 20.108833071 0.837868045 1 12 24 24 0.837868045 -0.837868045 480 480 1.000000000 24 19 0.791666667 480 367 0.764583333 480 9475.577381004 9475.493594199 492 475 0.965447154 +case_040 21 24 15.393823258 0.641409302 1 15 24 24 0.641409302 -0.641409302 480 479 0.997916667 24 19 0.791666667 479 372 0.776617954 479 9484.803952761 9484.739677924 455 444 0.975824176 +case_041 15 24 13.209647992 0.550402000 0 0 24 24 0.550402000 -0.550402000 480 480 1.000000000 24 19 0.791666667 480 358 0.745833333 480 9488.504924398 9488.449884198 456 449 0.984649123 +case_042 14 24 11.930940702 0.497122529 1 1 24 24 0.497122529 -0.497122529 480 480 1.000000000 24 19 0.791666667 480 390 0.812500000 480 9487.484085239 9487.434372986 456 450 0.986842105 +case_043 18 24 0.968710698 0.040362946 1 24 24 24 0.040362946 -0.040362946 480 472 0.983333333 24 24 1.000000000 472 343 0.726694915 472 9474.593028965 9474.588924259 448 448 1.000000000 +case_044 20 24 8.053645895 0.335568579 0 0 24 24 0.335568579 -0.335568579 480 479 0.997916667 24 21 0.875000000 479 379 0.791231733 479 9484.742704744 9484.709077830 455 449 0.986813187 +case_045 18 24 10.508414715 0.437850613 1 5 24 24 0.437850613 -0.437850613 480 480 1.000000000 24 21 0.875000000 480 348 0.725000000 480 9487.114973161 9487.071188099 456 451 0.989035088 +case_046 19 24 2.811793445 0.117158060 1 2 24 24 0.117158060 -0.117158060 480 476 0.991666667 24 23 0.958333333 476 349 0.733193277 476 9480.652843728 9480.641029470 452 451 0.997787611 +case_047 16 24 7.877445790 0.328226908 1 5 24 24 0.328226908 -0.328226908 480 479 0.997916667 24 20 0.833333333 479 341 0.711899791 479 9484.296954290 9484.264063076 455 450 0.989010989 +case_048 18 24 6.200179281 0.258340803 1 23 24 24 0.258340803 -0.258340803 480 479 0.997916667 24 23 0.958333333 479 374 0.780793319 479 9485.266618269 9485.240730255 455 450 0.989010989 +case_049 18 24 20.722330966 0.863430457 1 14 24 24 0.863430457 -0.863430457 480 480 1.000000000 24 19 0.791666667 480 369 0.768750000 480 9466.904310408 9466.817967362 474 458 0.966244726 +case_050 27 24 18.225562645 0.759398444 1 11 24 24 0.759398444 -0.759398444 480 480 1.000000000 24 19 0.791666667 480 350 0.729166667 480 9484.084624933 9484.008685089 456 447 0.980263158 +case_051 21 24 12.529212802 0.522050533 0 0 24 24 0.522050533 -0.522050533 480 480 1.000000000 24 20 0.833333333 480 364 0.758333333 480 9489.048368422 9488.996163368 456 450 0.986842105 +case_052 19 24 11.977788362 0.499074515 1 3 24 24 0.499074515 -0.499074515 480 479 0.997916667 24 18 0.750000000 479 362 0.755741127 479 9481.001144315 9480.951132672 491 484 0.985743381 +case_053 22 24 2.967696017 0.123654001 1 11 24 24 0.123654001 -0.123654001 480 476 0.991666667 24 23 0.958333333 476 355 0.745798319 476 9478.653084767 9478.640615456 452 451 0.997787611 +case_054 24 24 8.700215908 0.362508996 0 0 24 24 0.362508996 -0.362508996 480 480 1.000000000 24 22 0.916666667 480 384 0.800000000 480 9483.186217249 9483.149966349 456 445 0.975877193 +case_055 25 24 5.355243020 0.223135126 1 5 24 24 0.223135126 -0.223135126 480 480 1.000000000 24 21 0.875000000 480 338 0.704166667 480 9483.254834637 9483.232521124 456 453 0.993421053 +case_056 18 24 13.790494920 0.574603955 0 0 24 24 0.574603955 -0.574603955 480 479 0.997916667 24 19 0.791666667 479 348 0.726513570 479 9484.431406416 9484.373826062 455 447 0.982417582 +case_057 22 24 5.008496027 0.208687334 0 0 24 24 0.208687334 -0.208687334 480 478 0.995833333 24 23 0.958333333 478 365 0.763598326 478 9480.527223189 9480.506267139 454 453 0.997797357 +case_058 25 24 4.142684494 0.172611854 1 19 24 24 0.172611854 -0.172611854 480 480 1.000000000 24 23 0.958333333 480 369 0.768750000 480 9484.309634956 9484.292373771 456 455 0.997807018 +case_059 23 24 10.513147963 0.438047832 0 0 24 24 0.438047832 -0.438047832 480 480 1.000000000 24 19 0.791666667 480 348 0.725000000 480 9484.238997349 9484.195192566 456 450 0.986842105 +case_060 17 24 0.539267842 0.022469493 1 24 24 24 0.022469493 -0.022469493 480 473 0.985416667 24 24 1.000000000 473 375 0.792811839 473 9474.150389026 9474.148108823 449 449 1.000000000 +case_061 16 24 18.323077899 0.763461579 0 0 24 24 0.763461579 -0.763461579 480 480 1.000000000 24 17 0.708333333 480 342 0.712500000 480 9475.491756605 9475.415410447 492 476 0.967479675 +case_062 14 24 5.082559957 0.211773332 1 15 24 24 0.211773332 -0.211773332 480 480 1.000000000 24 23 0.958333333 480 352 0.733333333 480 9482.775947450 9482.754770116 456 453 0.993421053 +case_063 15 24 6.285562807 0.261898450 1 11 24 24 0.261898450 -0.261898450 480 476 0.991666667 24 22 0.916666667 476 377 0.792016807 476 9476.502627259 9476.476217331 452 446 0.986725664 +case_064 12 24 7.793198913 0.324716621 1 14 24 24 0.324716621 -0.324716621 480 477 0.993750000 24 23 0.958333333 477 333 0.698113208 477 9475.308509306 9475.275833420 471 468 0.993630573 +case_065 15 24 12.247740998 0.510322542 0 0 24 24 0.510322542 -0.510322542 480 480 1.000000000 24 18 0.750000000 480 383 0.797916667 480 9484.232162796 9484.181130542 456 449 0.984649123 +case_066 16 24 6.192415293 0.258017304 0 0 24 24 0.258017304 -0.258017304 480 480 1.000000000 24 21 0.875000000 480 393 0.818750000 480 9483.562772116 9483.536970386 474 471 0.993670886 +case_067 17 24 15.314347684 0.638097820 0 0 24 24 0.638097820 -0.638097820 480 479 0.997916667 24 20 0.833333333 479 357 0.745302714 479 9469.414218501 9469.350275504 473 463 0.978858351 +case_068 17 24 14.847894468 0.618662270 0 0 24 24 0.618662270 -0.618662270 480 480 1.000000000 24 19 0.791666667 480 338 0.704166667 480 9486.834477168 9486.772610941 456 445 0.975877193 +case_069 18 24 10.562609476 0.440108728 0 0 24 24 0.440108728 -0.440108728 480 479 0.997916667 24 19 0.791666667 479 385 0.803757829 479 9487.120359185 9487.076256431 455 449 0.986813187 +case_070 24 24 2.280622508 0.095025938 1 24 24 24 0.095025938 -0.095025938 480 476 0.991666667 24 24 1.000000000 476 381 0.800420168 476 9477.265043590 9477.255461143 452 452 1.000000000 +case_071 24 24 6.744841142 0.281035048 0 0 24 24 0.281035048 -0.281035048 480 480 1.000000000 24 22 0.916666667 480 373 0.777083333 480 9485.094612134 9485.066508629 456 453 0.993421053 +case_072 17 24 2.005355449 0.083556477 1 9 24 24 0.083556477 -0.083556477 480 478 0.995833333 24 23 0.958333333 478 361 0.755230126 478 9475.043177757 9475.034787149 472 471 0.997881356 +case_073 15 24 8.580089935 0.357503747 1 3 24 24 0.357503747 -0.357503747 480 477 0.993750000 24 21 0.875000000 477 376 0.788259958 477 9470.514505557 9470.478530337 471 466 0.989384289 +case_074 15 24 14.386974335 0.599457264 0 0 24 24 0.599457264 -0.599457264 480 478 0.995833333 24 20 0.833333333 478 345 0.721757322 478 9463.410017343 9463.349820798 471 457 0.970276008 +case_075 16 24 3.614196414 0.150591517 1 6 24 24 0.150591517 -0.150591517 480 478 0.995833333 24 22 0.916666667 478 356 0.744769874 478 9480.769879486 9480.754757326 454 452 0.995594714 +case_076 22 24 5.396436546 0.224851523 1 4 24 24 0.224851523 -0.224851523 480 477 0.993750000 24 22 0.916666667 477 366 0.767295597 477 9479.935717170 9479.913090602 453 450 0.993377483 +case_077 24 24 7.871823196 0.327992633 0 0 24 24 0.327992633 -0.327992633 480 480 1.000000000 24 20 0.833333333 480 376 0.783333333 480 9484.635272915 9484.602473651 456 452 0.991228070 +case_078 28 24 19.085377210 0.795224050 0 0 24 24 0.795224050 -0.795224050 480 479 0.997916667 24 18 0.750000000 479 338 0.705636743 479 9484.897514179 9484.817825757 455 436 0.958241758 +case_079 20 24 7.624825106 0.317701046 1 18 24 24 0.317701046 -0.317701046 480 479 0.997916667 24 22 0.916666667 479 352 0.734864301 479 9473.519303318 9473.487466888 473 470 0.993657505 +case_080 13 24 17.398640752 0.724943365 1 6 24 24 0.724943365 -0.724943365 480 478 0.995833333 24 17 0.708333333 478 356 0.744769874 478 9485.466003952 9485.393206291 454 439 0.966960352 +case_081 13 24 9.845937111 0.410247380 0 0 24 24 0.410247380 -0.410247380 480 479 0.997916667 24 21 0.875000000 479 352 0.734864301 479 9485.830540154 9485.789429769 455 450 0.989010989 +case_082 16 24 28.455881343 1.185661723 1 7 24 24 1.185661723 -1.185661723 480 477 0.993750000 24 16 0.666666667 477 328 0.687631027 477 9472.165230265 9472.045918393 471 451 0.957537155 +case_083 19 17 0.099510162 0.005853539 1 17 17 17 0.005853539 -0.005853539 340 330 0.970588235 17 17 1.000000000 330 241 0.730303030 330 9463.047109516 9463.046506424 313 313 1.000000000 +case_084 23 11 0.701877409 0.063807037 1 11 11 11 0.063807037 -0.063807037 220 215 0.977272727 11 11 1.000000000 215 163 0.758139535 215 9467.740751223 9467.734222131 204 204 1.000000000 +case_085 18 24 10.909207049 0.454550294 1 11 24 24 0.454550294 -0.454550294 480 475 0.989583333 24 21 0.875000000 475 373 0.785263158 475 9463.544781446 9463.498847943 469 464 0.989339019 +case_086 16 24 7.114907964 0.296454499 1 8 24 24 0.296454499 -0.296454499 480 480 1.000000000 24 21 0.875000000 480 376 0.783333333 480 9488.089166432 9488.059520982 456 450 0.986842105 +case_087 12 24 11.005250253 0.458552094 1 3 24 24 0.458552094 -0.458552094 480 480 1.000000000 24 20 0.833333333 480 377 0.785416667 480 9486.949239349 9486.903384140 456 448 0.982456140 +case_088 14 24 14.441297598 0.601720733 1 7 24 24 0.601720733 -0.601720733 480 480 1.000000000 24 17 0.708333333 480 393 0.818750000 480 9489.376179659 9489.316007586 456 445 0.975877193 +case_089 12 24 18.530864832 0.772119368 0 0 24 24 0.772119368 -0.772119368 480 480 1.000000000 24 19 0.791666667 480 353 0.735416667 480 9468.948623045 9468.871411109 474 462 0.974683544 +case_090 17 24 7.802819239 0.325117468 0 0 24 24 0.325117468 -0.325117468 480 480 1.000000000 24 22 0.916666667 480 344 0.716666667 480 9485.508067545 9485.475555798 456 450 0.986842105 +case_091 14 24 14.468345118 0.602847713 0 0 24 24 0.602847713 -0.602847713 480 479 0.997916667 24 21 0.875000000 479 326 0.680584551 479 9483.682717450 9483.622306824 455 437 0.960439560 +case_092 15 24 0.690675976 0.028778166 1 24 24 24 0.028778166 -0.028778166 480 475 0.989583333 24 24 1.000000000 475 379 0.797894737 475 9474.294883704 9474.291975594 451 451 1.000000000 +case_093 17 24 18.748704203 0.781196008 0 0 24 24 0.781196008 -0.781196008 480 480 1.000000000 24 17 0.708333333 480 366 0.762500000 480 9479.038790268 9478.960670668 474 463 0.976793249 +case_094 16 8 0.179222634 0.022402829 1 8 8 8 0.022402829 -0.022402829 160 155 0.968750000 8 8 1.000000000 155 123 0.793548387 155 9463.391503847 9463.389191297 147 147 1.000000000 +case_095 17 15 0.955572988 0.063704866 1 15 15 15 0.063704866 -0.063704866 300 290 0.966666667 15 15 1.000000000 290 228 0.786206897 290 9464.078747327 9464.072157169 275 275 1.000000000 +case_096 15 11 2.966251965 0.269659270 0 0 11 11 0.269659270 -0.269659270 220 213 0.968181818 11 10 0.909090909 213 140 0.657276995 213 9462.336499702 9462.308647570 202 201 0.995049505 +case_097 22 11 0.023667264 0.002151569 1 11 11 11 0.002151569 -0.002151569 220 211 0.959090909 11 11 1.000000000 211 147 0.696682464 211 9456.299221880 9456.298997545 200 200 1.000000000 +case_098 20 8 5.732133494 0.716516687 1 1 8 8 0.716516687 -0.716516687 160 152 0.950000000 8 5 0.625000000 152 100 0.657894737 152 9458.117830898 9458.042408089 144 141 0.979166667 +case_099 22 24 12.111447118 0.504643630 0 0 24 24 0.504643630 -0.504643630 480 464 0.966666667 24 21 0.875000000 464 343 0.739224138 464 9462.414465266 9462.362260752 440 437 0.993181818 From 194b801ce27b87790bea8ac6f3d0b59d3f4aade0 Mon Sep 17 00:00:00 2001 From: Giorgio Oppo Date: Sat, 8 Aug 2026 16:45:04 +0200 Subject: [PATCH 010/189] Enable DSpark SSD streaming across GPU backends --- Makefile | 46 +++- QA_BEFORE_RELEASES.md | 38 +++ README.md | 49 +++- ds4.c | 424 ++++++++++++++++++++++------- ds4_cuda.cu | 65 ++++- ds4_gpu.h | 5 + ds4_metal.m | 422 ++++++++++++++++++++++++---- rocm/ds4_rocm_attention.cuh | 79 ++++++ rocm/ds4_rocm_attention_launch.cuh | 52 ++++ rocm/ds4_rocm_runtime.cuh | 42 +++ tests/ds4_test.c | 2 +- tests/dspark_acceptance_fixture.sh | 118 ++++++-- tests/test_engine_mgpu_placement.c | 22 ++ 13 files changed, 1177 insertions(+), 187 deletions(-) diff --git a/Makefile b/Makefile index 3ab172667..2a73387c7 100644 --- a/Makefile +++ b/Makefile @@ -62,12 +62,13 @@ ROCM_LDLIBS ?= -lm -pthread -lhipblas -lhipblaslt -lrocblas ROCM_MMQ_Y ?= 64 ROCM_MMQ_FLAGS := $(ROCM_CFLAGS) -std=c++17 -DGGML_USE_HIP -DDS4_HIP_MMQ_Y=$(ROCM_MMQ_Y) $(MMQ_INCLUDES) ROCM_MMQ_OBJS := cuda/mmq/ds4_ggml_stubs.rocm.o cuda/mmq/ds4_mmq.rocm.o cuda/mmq/quantize.rocm.o cuda/mmq/mmid.rocm.o cuda/mmq/mmvq.rocm.o cuda/mmq/d2r_stubs.rocm.o +ROCM_CORE_OBJS := ds4.o ds4_image.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_rocm.o ds4_rocm_compat.o ds4_rocm_unavailable.o ds4_layer_pack.o $(ROCM_MMQ_OBJS) DS4_LINK ?= $(NVCC) $(NVCCFLAGS) DS4_LINK_LIBS ?= $(CUDA_LDLIBS) METAL_LDLIBS := $(LDLIBS) endif -.PHONY: all help clean test test-rocm test-glm53-kda-rocm test-metal-session-batch test-mxfp4-cuda test-mxfp4-rocm test-cuda-session-batch test-cuda-mixed-batch dspark-acceptance dspark-verify-depth mtp-verify-depth cpu cuda cuda-spark cuda-generic cuda-regression strix-halo rocm +.PHONY: all help clean test test-rocm test-glm53-kda-rocm test-metal-session-batch test-mxfp4-cuda test-mxfp4-rocm test-cuda-session-batch test-cuda-mixed-batch dspark-acceptance dspark-verify-depth rocm-dspark-acceptance rocm-dspark-verify-depth mtp-verify-depth cpu cuda cuda-spark cuda-generic cuda-regression strix-halo rocm ifeq ($(UNAME_S),Darwin) .PHONY: metal-decode-schedule-bench metal-prefill-variant-bench check-mxfp4-half-lut @@ -163,6 +164,8 @@ help: @echo " make rocm Alias for make strix-halo" @echo " make test-mxfp4-rocm Build and run the synthetic ROCm MXFP4 MoE test" @echo " make test-rocm Core regression suite on ROCm-only hosts" + @echo " make rocm-dspark-acceptance Build ROCm and run the DSpark acceptance fixture" + @echo " make rocm-dspark-verify-depth Build ROCm and run the DSpark verifier invariant" @echo " make cpu Build CPU-only ./ds4, ./ds4-server, ./ds4-bench, ./ds4-eval, and ./ds4-agent" @echo " make test Build and run tests" @echo " make dspark-verify-depth Run DSpark speculative verification smoke if support GGUF is present" @@ -184,8 +187,8 @@ cuda: $(MAKE) -B ds4 ds4-server ds4-bench ds4-eval ds4-agent CUDA_ARCH="$(CUDA_ARCH)" strix-halo: - $(MAKE) -B ds4 ds4-server ds4-bench ds4-eval ds4-agent \ - CORE_OBJS="ds4.o ds4_image.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_rocm.o ds4_rocm_compat.o ds4_rocm_unavailable.o ds4_layer_pack.o $(ROCM_MMQ_OBJS)" \ + $(MAKE) -B ds4 ds4-server ds4-bench ds4-eval ds4-agent ds4_test \ + CORE_OBJS="$(ROCM_CORE_OBJS)" \ CFLAGS="$(CFLAGS) $(ROCM_HOST_CFLAGS) -DDS4_ROCM_BUILD" \ DS4_LINK="$(HIPCC) $(ROCM_CFLAGS)" \ DS4_LINK_LIBS="$(ROCM_LDLIBS)" @@ -200,7 +203,7 @@ test-rocm: $(MAKE) -B ds4_test ds4_agent_test ds4-eval q4k-dot-test mxfp4-dot-test \ tests/test_layer_pack tests/test_engine_mgpu_placement tests/test_gpu_args \ ds4 ds4-server ds4-bench ds4-agent \ - CORE_OBJS="ds4.o ds4_image.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_rocm.o ds4_rocm_compat.o ds4_rocm_unavailable.o ds4_layer_pack.o $(ROCM_MMQ_OBJS)" \ + CORE_OBJS="$(ROCM_CORE_OBJS)" \ CFLAGS="$(CFLAGS) $(ROCM_HOST_CFLAGS) -DDS4_ROCM_BUILD" \ DS4_LINK="$(HIPCC) $(ROCM_CFLAGS)" \ DS4_LINK_LIBS="$(ROCM_LDLIBS)" @@ -212,6 +215,41 @@ test-rocm: ./tests/test_gpu_args ./tests/test_gpu_args_cli.sh +rocm-dspark-acceptance: + @if [ ! -f "$(DS4_DSPARK_MODEL)" ]; then \ + echo "rocm-dspark-acceptance: missing model $(DS4_DSPARK_MODEL)" >&2; \ + exit 1; \ + elif [ ! -f "$(DS4_DSPARK_SUPPORT)" ]; then \ + echo "rocm-dspark-acceptance: missing DSpark support $(DS4_DSPARK_SUPPORT)" >&2; \ + exit 1; \ + fi + $(MAKE) -B ds4 \ + CORE_OBJS="$(ROCM_CORE_OBJS)" \ + CFLAGS="$(CFLAGS) $(ROCM_HOST_CFLAGS) -DDS4_ROCM_BUILD" \ + DS4_LINK="$(HIPCC) $(ROCM_CFLAGS)" \ + DS4_LINK_LIBS="$(ROCM_LDLIBS)" + DS4_DSPARK_MODEL="$(DS4_DSPARK_MODEL)" \ + DS4_DSPARK_SUPPORT="$(DS4_DSPARK_SUPPORT)" \ + DS4_DSPARK_FIXTURE_BACKEND=rocm \ + sh tests/dspark_acceptance_fixture.sh + +rocm-dspark-verify-depth: + @if [ ! -f "$(DS4_TEST_MODEL)" ]; then \ + echo "rocm-dspark-verify-depth: missing model $(DS4_TEST_MODEL)" >&2; \ + exit 1; \ + elif [ ! -f "$(DS4_DSPARK_SUPPORT)" ]; then \ + echo "rocm-dspark-verify-depth: missing DSpark support $(DS4_DSPARK_SUPPORT)" >&2; \ + exit 1; \ + fi + $(MAKE) -B ds4_test \ + CORE_OBJS="$(ROCM_CORE_OBJS)" \ + CFLAGS="$(CFLAGS) $(ROCM_HOST_CFLAGS) -DDS4_ROCM_BUILD" \ + DS4_LINK="$(HIPCC) $(ROCM_CFLAGS)" \ + DS4_LINK_LIBS="$(ROCM_LDLIBS)" + DS4_TEST_MODEL="$(DS4_TEST_MODEL)" \ + DS4_TEST_DSPARK="$(DS4_DSPARK_SUPPORT)" \ + ./ds4_test --dspark-verify-depth + ds4: ds4_cli.o ds4_help.o linenoise.o ds4_gpu_args.o $(CORE_OBJS) $(DS4_LINK) -o $@ $^ $(DS4_LINK_LIBS) diff --git a/QA_BEFORE_RELEASES.md b/QA_BEFORE_RELEASES.md index cd96f2000..d7ca77268 100644 --- a/QA_BEFORE_RELEASES.md +++ b/QA_BEFORE_RELEASES.md @@ -267,6 +267,44 @@ than a failure. `--dspark-strict` remains the byte-identical target-only mode. `DS4_DSPARK_FIXTURE_CONFIDENCE=0 DS4_DSPARK_FIXTURE_TOKENS=8 DS4_DSPARK_FIXTURE_REQUIRE_PARTIAL=1 DS4_DSPARK_MODEL=/Users/antirez/ds4/gguf/DeepSeek-V4-Flash-IQ2XXS-w2Q2K-AProjQ8-SExpQ8-OutQ8-chat-v2-imatrix-0731.gguf DS4_DSPARK_SUPPORT=/Users/antirez/ds4/gguf/DeepSeek-V4-Flash-DSpark-support-0731.gguf make dspark-acceptance`. - DSpark verifier invariant smoke: `DS4_TEST_MODEL=/Users/antirez/ds4/gguf/DeepSeek-V4-Flash-IQ2XXS-w2Q2K-AProjQ8-SExpQ8-OutQ8-chat-v2-imatrix-0731.gguf DS4_DSPARK_SUPPORT=/Users/antirez/ds4/gguf/DeepSeek-V4-Flash-DSpark-support-0731.gguf make dspark-verify-depth`. +- When DSpark, support-model mapping, or SSD streaming changes, repeat both + the acceptance fixture and verifier invariant on every advertised graph + backend. Apply the backend and SSD options to the target-only baseline as + well as the DSpark run: + + ```sh + DS4_DSPARK_MODEL=/path/to/flash-0731.gguf \ + DS4_DSPARK_SUPPORT=/path/to/DeepSeek-V4-Flash-DSpark-support-0731.gguf \ + DS4_DSPARK_FIXTURE_BACKEND=cuda \ + DS4_DSPARK_FIXTURE_SSD_STREAMING=1 \ + DS4_DSPARK_FIXTURE_SSD_STREAMING_CACHE_EXPERTS=32 \ + DS4_DSPARK_FIXTURE_CONFIDENCE=0 \ + DS4_DSPARK_SCHEDULER=0 make dspark-acceptance + + DS4_TEST_MODEL=/path/to/flash-0731.gguf \ + DS4_DSPARK_SUPPORT=/path/to/DeepSeek-V4-Flash-DSpark-support-0731.gguf \ + DS4_TEST_SSD_STREAMING=1 \ + DS4_TEST_SSD_STREAMING_CACHE_EXPERTS=32 \ + make dspark-verify-depth + ``` + + On Strix Halo use the same variables with + `DS4_DSPARK_FIXTURE_BACKEND=rocm make rocm-dspark-acceptance` and + `make rocm-dspark-verify-depth`. Do not use the generic targets after a ROCm + build: on non-Apple hosts their default object set is CUDA. For the 0731 + Flash layout, ROCm needs at least 30 expert slots; use 32 in release tests. +- The fixture must report aggregate `proposed>0`, `accepted_draft>0`, + `verifier_unavailable=0`, and `errors=0`; stdout must remain byte-identical + to the target-only SSD baseline. The verifier smoke must report + `max_chunk>1`, `nspec>64`, and `worst_argmax_gap<=2`. +- Preserve baseline and DSpark `generation` t/s from the same fixture run, + with the same host, model, cache, scheduler, thermal state, and background + load. A DSpark path that is materially slower than the target-only SSD path + without a documented correctness tradeoff is a release blocker. +- On ROCm, also run one `DS4_DSPARK_PROBE=1 DS4_DSPARK_SCHEDULER=0` + generation and require the non-causal attention and stage-chain probes to + pass. This covers the HIP draft-attention kernel before the end-to-end + verifier gate. - If shared support-model or verifier structures changed, also run legacy MTP: `make mtp-verify-depth` with `DS4_TEST_MTP` set to a one-stage MTP support GGUF, or confirm the target skips only because the optional file is missing. diff --git a/README.md b/README.md index 009b7c833..0419507b3 100644 --- a/README.md +++ b/README.md @@ -350,10 +350,10 @@ GGUF of about 5.6 GiB. It is not a standalone model. Download it once: The support file can be used with the 0731 Flash `ds4f-q2`, `ds4f-q2-q4`, and `ds4f-q4` models listed above. It is checkpoint-specific and must not be paired with an older Flash model. For now **DeepSeek V4 PRO** -is not supported. On Metal, the main model may be resident or use -`--ssd-streaming`; the support model still adds its own weights and runtime -state to the memory requirement. DSpark replaces the legacy one-stage MTP -support model for that run rather than stacking with it. +is not supported. On Metal, CUDA, and ROCm, the main model may be resident or +use `--ssd-streaming`; the support model remains resident and adds its own +weights and runtime state to the memory requirement. DSpark replaces the +legacy one-stage MTP support model for that run rather than stacking with it. Run it with the normal sampling defaults: @@ -391,6 +391,47 @@ The same DSpark flags work with `ds4-agent` and with non-batched `ds4-server` requests. Session-batched serving currently uses ordinary target decoding. +On a single accelerator, the main model can instead stream its routed experts +from SSD while the DSpark support model remains mapped or device-cached +separately. Select the backend with `--metal`, `--cuda`, or `--rocm`: + +```sh +./ds4 -m ds4flash.gguf \ + --mtp-model gguf/DeepSeek-V4-Flash-DSpark-support-0731.gguf \ + --dspark --metal --ssd-streaming \ + --ssd-streaming-cache-experts 16 --temp 0 +``` + +Use `--cuda` in a CUDA build. On ROCm, use `--rocm` and a verification-safe +cache, for example `--ssd-streaming-cache-experts 32`. + +Tune the expert-cache count for the available accelerator memory. ROCm needs +enough slots for a whole verification block (30 for the 0731 model; use at +least 32), and currently supports the IQ2_XXS/Q2_K or all-Q2_K routed-expert +layouts. CUDA uses a transient selected-expert cache for each target block. +The DSpark support weights remain resident and are included in the startup +memory budget. This combination is single-device only; CPU, distributed or +multi-GPU placement, tensor parallelism, and legacy MTP support models remain +incompatible with DSpark plus SSD streaming. + +The acceptance fixture can exercise the same SSD path on both the target-only +baseline and the DSpark run. It also requires real proposals and accepted +draft tokens, so an unavailable verifier cannot pass as a silent no-op: + +```sh +DS4_DSPARK_FIXTURE_BACKEND=cuda \ +DS4_DSPARK_FIXTURE_SSD_STREAMING=1 \ +DS4_DSPARK_FIXTURE_SSD_STREAMING_CACHE_EXPERTS=32 \ +make dspark-acceptance +``` + +For ROCm, use `make rocm-dspark-acceptance` with the same model, support, and +SSD fixture environment variables. The ROCm-specific target preserves the HIP +object set and linker; the generic target selects CUDA objects on non-Apple +hosts. `make rocm-dspark-verify-depth` provides the corresponding verifier +invariant test. + + ## Speed The current q2 results use `ds4-bench` with the standard *Promessi sposi* diff --git a/ds4.c b/ds4.c index c6ad7c817..fca250024 100644 --- a/ds4.c +++ b/ds4.c @@ -397,6 +397,24 @@ static bool ds4_backend_uses_graph(ds4_backend backend) { return backend == DS4_BACKEND_METAL || backend == DS4_BACKEND_CUDA; } +typedef enum { + DS4_DSPARK_RUNTIME_SUPPORTED = 0, + DS4_DSPARK_RUNTIME_UNSUPPORTED_CPU, + DS4_DSPARK_RUNTIME_UNSUPPORTED_DISTRIBUTED, +} ds4_dspark_runtime_policy; + +static ds4_dspark_runtime_policy ds4_dspark_runtime_policy_for( + ds4_backend backend, + ds4_distributed_role distributed_role) { + if (backend == DS4_BACKEND_CPU) { + return DS4_DSPARK_RUNTIME_UNSUPPORTED_CPU; + } + if (distributed_role != DS4_DISTRIBUTED_NONE) { + return DS4_DSPARK_RUNTIME_UNSUPPORTED_DISTRIBUTED; + } + return DS4_DSPARK_RUNTIME_SUPPORTED; +} + static bool ds4_backend_supports_ssd_streaming(ds4_backend backend) { if (backend == DS4_BACKEND_METAL) return true; if (backend == DS4_BACKEND_CUDA) { @@ -4786,7 +4804,7 @@ static bool ds4_streaming_manual_cache_safe_bytes( int ctx_size, uint32_t prefill_chunk, bool ssd_streaming, - uint64_t non_routed_bytes, + uint64_t fixed_model_bytes, uint64_t *safe_bytes_out) { if (safe_bytes_out) *safe_bytes_out = 0; #ifdef DS4_NO_GPU @@ -4794,7 +4812,7 @@ static bool ds4_streaming_manual_cache_safe_bytes( (void)ctx_size; (void)prefill_chunk; (void)ssd_streaming; - (void)non_routed_bytes; + (void)fixed_model_bytes; return false; #else const uint64_t recommended = ds4_gpu_recommended_working_set_size(); @@ -4802,12 +4820,12 @@ static bool ds4_streaming_manual_cache_safe_bytes( /* * Explicit NGB budgets name the total routed-expert budget (prefill - * headroom plus the decode cache). On Metal, the mmap-backed non-routed - * weights share unified memory with that budget even though the startup - * report labels only the initially mapped token span as resident. Failing - * to include them lets a seemingly safe expert cache evict the dense - * working set every token. Keep the total below the backend's recommended - * working set after accounting for both graph memory and those weights. + * headroom plus the decode cache). The mmap-backed Metal views and the + * CUDA/ROCm device caches all share the accelerator memory target with + * non-routed weights and an active support model. Failing to include those + * fixed weights lets a seemingly safe expert cache evict the dense working + * set or OOM while loading DSpark. Keep the total below the backend's + * recommended working set after graph and fixed-model accounting. * * If less than the minimum usable cache remains, the normal planner emits * its existing "budget too small" error instead of silently exceeding the @@ -4820,9 +4838,9 @@ static bool ds4_streaming_manual_cache_safe_bytes( prefill_chunk, ssd_streaming); uint64_t fixed = ctx_mem.total_bytes; - if (backend == DS4_BACKEND_METAL) { - fixed = fixed > UINT64_MAX - non_routed_bytes ? - UINT64_MAX : fixed + non_routed_bytes; + if (ds4_backend_uses_graph(backend)) { + fixed = fixed > UINT64_MAX - fixed_model_bytes ? + UINT64_MAX : fixed + fixed_model_bytes; } *safe_bytes_out = target > fixed ? target - fixed : 0; return true; @@ -31375,7 +31393,7 @@ static bool metal_graph_encode_layer_ffn_batch( x_row, NULL, il, - false) != 0; + !g->ssd_streaming) != 0; ds4_gpu_tensor_free(w_row); ds4_gpu_tensor_free(sel_row); ds4_gpu_tensor_free(x_row); @@ -31417,7 +31435,7 @@ static bool metal_graph_encode_layer_ffn_batch( il, n_tokens, &g->batch_routed_mid_is_f16, - false) != 0; + !g->ssd_streaming) != 0; } if (ok) { metal_graph_debug_dump_tensor("ffn_moe_gate_clamped", metal_graph_batch_routed_gate(g), @@ -36623,6 +36641,20 @@ static bool metal_graph_verify_suffix_tops_impl( const uint32_t top_rows = n_tokens > 1 ? n_tokens - 1 : 0; if (top_rows && !row_tops) return false; + /* A dynamic SSD decode may leave only the output-head view installed. + * Verification immediately needs token, every non-routed layer weight, + * and the output head in one layer-major command stream. Install that + * complete static set explicitly; Metal keeps the separate DSpark support + * views while replacing only this target model's views. */ + if (g->ssd_streaming) { + if (!metal_graph_stream_map_decode_static_all(model, weights)) { + return false; + } + g->streaming_static_decode_map_current = + metal_graph_stream_decode_static_map_enabled() && + metal_graph_stream_decode_static_map_state_cache_enabled(); + } + const double upload_t0 = timing ? now_sec() : 0.0; bool ok = metal_graph_upload_prompt_tokens(metal_graph_prefill_tokens(g), prompt, start, n_tokens); if (ok) ok = metal_graph_upload_prompt_embeddings_hc(metal_graph_batch_cur_hc(g), @@ -38326,6 +38358,59 @@ struct ds4_engine { int placement_session_count_hint; }; +static uint64_t ds4_engine_support_model_bytes(const ds4_engine *e) { + if (!e || !e->mtp_model.map) return 0; + const bool runtime_ready = + e->mtp_ready || + (e->support_kind == DS4_SUPPORT_DSPARK && e->dspark); + if (!runtime_ready || e->mtp_model.size <= e->mtp_model.tensor_data_pos) { + return 0; + } + return e->mtp_model.size - e->mtp_model.tensor_data_pos; +} + +#ifdef DS4_ROCM_BUILD +static bool ds4_rocm_dspark_ssd_layout_supported( + const ds4_weights *weights, + uint32_t *bad_layer, + uint32_t *bad_gate_type, + uint32_t *bad_down_type) { + if (bad_layer) *bad_layer = UINT32_MAX; + if (bad_gate_type) *bad_gate_type = 0; + if (bad_down_type) *bad_down_type = 0; + if (!weights) return false; + + bool found_routed = false; + for (uint32_t il = 0; il < DS4_N_LAYER; il++) { + const ds4_layer_weights *layer = &weights->layer[il]; + if (!layer->ffn_gate_exps || !layer->ffn_up_exps || + !layer->ffn_down_exps) { + continue; + } + found_routed = true; + const uint32_t gate_type = layer->ffn_gate_exps->type; + const uint32_t up_type = layer->ffn_up_exps->type; + const uint32_t down_type = layer->ffn_down_exps->type; + const bool iq2_selected = + gate_type == DS4_TENSOR_IQ2_XXS && + up_type == DS4_TENSOR_IQ2_XXS && + (down_type == DS4_TENSOR_Q2_K || + down_type == DS4_TENSOR_IQ2_XXS); + const bool q2_selected = + gate_type == DS4_TENSOR_Q2_K && + up_type == DS4_TENSOR_Q2_K && + down_type == DS4_TENSOR_Q2_K; + if (iq2_selected || q2_selected) continue; + + if (bad_layer) *bad_layer = il; + if (bad_gate_type) *bad_gate_type = gate_type; + if (bad_down_type) *bad_down_type = down_type; + return false; + } + return found_routed; +} +#endif + static uint64_t ds4_engine_dynamic_expert_cache_bytes( const ds4_engine *e) { if (!e || !e->ssd_streaming) return 0; @@ -38397,6 +38482,8 @@ static void ds4_engine_print_startup_memory( ds4_add_sat_u64(mem.raw_bytes, mem.compressed_bytes); const uint64_t dynamic_expert_cache_bytes = ds4_engine_dynamic_expert_cache_bytes(e); + const uint64_t support_model_bytes = + ds4_engine_support_model_bytes(e); const uint64_t expert_reserved_bytes = e->ssd_streaming_prefill_headroom_bytes; uint64_t resident_model_bytes = e->startup_model_span_bytes; @@ -38415,6 +38502,7 @@ static void ds4_engine_print_startup_memory( uint64_t total = kv_bytes; total = ds4_add_sat_u64(total, mem.scratch_bytes); total = ds4_add_sat_u64(total, resident_model_bytes); + total = ds4_add_sat_u64(total, support_model_bytes); total = ds4_add_sat_u64(total, dynamic_expert_cache_bytes); total = ds4_add_sat_u64(total, expert_reserved_bytes); @@ -38432,6 +38520,11 @@ static void ds4_engine_print_startup_memory( ds4_bytes_to_gib(mem.compressed_bytes), ds4_bytes_to_gib(mem.scratch_bytes), ds4_bytes_to_gib(resident_model_bytes)); + if (support_model_bytes != 0) { + fprintf(stderr, + " + support model %.2f GiB", + ds4_bytes_to_gib(support_model_bytes)); + } if (dynamic_expert_cache_bytes != 0) { fprintf(stderr, " + expert cache %.2f GiB", @@ -59987,6 +60080,10 @@ static bool ds4_engine_configure_streaming_auto_cache(ds4_engine *e) { "ds4: SSD streaming auto cache could not measure non-routed model weights\n"); return false; } + const uint64_t support_model_bytes = + ds4_engine_support_model_bytes(e); + const uint64_t fixed_model_bytes = + ds4_add_sat_u64(non_routed_bytes, support_model_bytes); uint64_t per_expert_bytes = 0; if (!ds4_streaming_routed_expert_bytes(&e->weights, &per_expert_bytes)) { @@ -60017,7 +60114,7 @@ static bool ds4_engine_configure_streaming_auto_cache(ds4_engine *e) { } ds4_ssd_cache_plan plan; if (!ds4_ssd_auto_cache_plan(recommended, - non_routed_bytes, + fixed_model_bytes, per_expert_bytes, max_model_experts, &plan)) { @@ -60080,8 +60177,8 @@ static bool ds4_engine_configure_streaming_auto_cache(ds4_engine *e) { } uint64_t active_model_bytes = glm_graph_streaming_active_model_bytes(&e->weights); - if (non_routed_bytes > active_model_bytes) { - active_model_bytes = non_routed_bytes; + if (fixed_model_bytes > active_model_bytes) { + active_model_bytes = fixed_model_bytes; } const double fraction = glm_graph_env_double( "DS4_GLM_MEMORY_GUARD_FRACTION", 0.99, 0.50, 1.00); @@ -60145,6 +60242,11 @@ static bool ds4_engine_configure_streaming_auto_cache(ds4_engine *e) { fprintf(stderr, "ds4: non-routed weights: %.2f GiB\n", (double)non_routed_bytes / 1073741824.0); + if (support_model_bytes != 0) { + fprintf(stderr, + "ds4: support model weights: %.2f GiB\n", + (double)support_model_bytes / 1073741824.0); + } fprintf(stderr, "ds4: routed expert size: %.2f MiB\n", (double)per_expert_bytes / 1048576.0); @@ -60176,9 +60278,9 @@ static bool ds4_engine_configure_streaming_auto_cache(ds4_engine *e) { e->placement_ctx_hint > 0 ? e->placement_ctx_hint : 4096); } #endif - if (plan.model_target_bytes <= non_routed_bytes) { + if (plan.model_target_bytes <= fixed_model_bytes) { fprintf(stderr, - "ds4: note: non-routed weights already fill the 80%% target; keeping a one-expert cache\n"); + "ds4: note: fixed model weights already fill the 80%% target; keeping a one-expert cache\n"); } return true; #endif @@ -62007,6 +62109,11 @@ int ds4_test_glm_memory_guard_disabled(void) { return glm_graph_memory_guard_disabled() ? 1 : 0; } +int ds4_test_dspark_runtime_policy(ds4_backend backend, + ds4_distributed_role distributed_role) { + return (int)ds4_dspark_runtime_policy_for(backend, distributed_role); +} + static int ds4_test_make_engine( ds4_engine *eng, const ds4_test_fake_tensor *tensors, @@ -62329,6 +62436,27 @@ static int ds4_engine_open_internal(ds4_engine **out, *out = NULL; return 1; } + if (opt->dspark && !opt->inspect_only) { + const ds4_dspark_runtime_policy policy = + ds4_dspark_runtime_policy_for(opt->backend, + opt->distributed.role); + if (policy == DS4_DSPARK_RUNTIME_UNSUPPORTED_CPU) { + fprintf(stderr, + "ds4: --dspark requires a Metal, CUDA, or ROCm graph " + "backend; CPU speculative decode is not implemented\n"); + free(e); + *out = NULL; + return 1; + } + if (policy == DS4_DSPARK_RUNTIME_UNSUPPORTED_DISTRIBUTED) { + fprintf(stderr, + "ds4: --dspark is not yet compatible with distributed " + "inference\n"); + free(e); + *out = NULL; + return 1; + } + } if ((opt->directional_steering_attn != 0.0f || opt->directional_steering_ffn != 0.0f) && (!opt->directional_steering_file || !opt->directional_steering_file[0])) { @@ -62612,48 +62740,6 @@ static int ds4_engine_open_internal(ds4_engine **out, *out = NULL; return 1; } - if (e->ssd_streaming && e->ssd_streaming_cache_bytes != 0) { - const uint64_t requested_cache_bytes = e->ssd_streaming_cache_bytes; - uint64_t non_routed_bytes = 0; - const bool non_routed_known = - weights_streaming_non_routed_bytes(&e->weights, - &non_routed_bytes); - if (e->backend == DS4_BACKEND_METAL && !non_routed_known) { - fprintf(stderr, - "ds4: Metal SSD streaming could not measure non-routed " - "weights for the manual cache safety cap\n"); - ds4_engine_close(e); - *out = NULL; - return 1; - } - uint64_t safe_cache_bytes = 0; - const bool safe_cache_known = - ds4_streaming_manual_cache_safe_bytes(e->backend, - opt->context_size, - e->prefill_chunk, - e->ssd_streaming, - non_routed_bytes, - &safe_cache_bytes); - if (safe_cache_known && safe_cache_bytes == 0) { - fprintf(stderr, - "ds4: %s SSD streaming has no safe room for the requested " - "expert cache after graph and non-routed weights\n", - ds4_backend_name(e->backend)); - ds4_engine_close(e); - *out = NULL; - return 1; - } - if (safe_cache_known && - e->ssd_streaming_cache_bytes > safe_cache_bytes) { - e->ssd_streaming_cache_bytes = safe_cache_bytes; - fprintf(stderr, - "ds4: %s SSD streaming cache budget %.2f GiB capped to %.2f GiB " - "to stay below the graph working-set pressure budget\n", - ds4_backend_name(e->backend), - (double)requested_cache_bytes / 1073741824.0, - (double)e->ssd_streaming_cache_bytes / 1073741824.0); - } - } if (opt->inspect_only) { if (opt->mtp_path && opt->mtp_path[0] && opt->distributed.role == DS4_DISTRIBUTED_NONE) { @@ -62753,17 +62839,30 @@ static int ds4_engine_open_internal(ds4_engine **out, } if (opt->mtp_path && opt->mtp_path[0] && opt->distributed.role == DS4_DISTRIBUTED_NONE) { - if (e->ssd_streaming) { - fprintf(stderr, "ds4: --ssd-streaming is not compatible with --mtp-model yet\n"); - ds4_engine_close(e); - *out = NULL; - return 1; - } model_open(&e->mtp_model, opt->mtp_path, graph_backend, true); ds4_dspark_summary dspark = {0}; e->support_kind = support_model_detect(&e->mtp_model, &e->support_stages, &dspark); + if (e->dspark && e->support_kind != DS4_SUPPORT_DSPARK) { + fprintf(stderr, + "ds4: --dspark requires a DSpark support GGUF; %s was " + "detected as %s\n", + opt->mtp_path, + support_kind_name(e->support_kind)); + ds4_engine_close(e); + *out = NULL; + return 1; + } if (e->support_kind == DS4_SUPPORT_MTP_LEGACY) { + if (e->ssd_streaming) { + fprintf(stderr, + "ds4: --ssd-streaming is not compatible with the " + "legacy MTP support model; use DSpark or disable " + "streaming\n"); + ds4_engine_close(e); + *out = NULL; + return 1; + } if (opt->tp.role != DS4_TP_NONE) { fprintf(stderr, "ds4: legacy MTP support is ignored under tensor parallelism; " @@ -62779,6 +62878,15 @@ static int ds4_engine_open_internal(ds4_engine **out, e->mtp_draft_tokens); } } else if (e->support_kind == DS4_SUPPORT_DSPARK) { + if (e->ssd_streaming && e->dspark && + opt->tp.role != DS4_TP_NONE) { + fprintf(stderr, + "ds4: DSpark with --ssd-streaming is not yet " + "supported with tensor parallelism\n"); + ds4_engine_close(e); + *out = NULL; + return 1; + } dspark_weights_bind_optional(&e->dspark_weights, &e->mtp_model, &dspark); @@ -62812,6 +62920,53 @@ static int ds4_engine_open_internal(ds4_engine **out, return 1; } } + if (e->ssd_streaming && e->ssd_streaming_cache_bytes != 0) { + const uint64_t requested_cache_bytes = e->ssd_streaming_cache_bytes; + uint64_t non_routed_bytes = 0; + const bool non_routed_known = + weights_streaming_non_routed_bytes(&e->weights, + &non_routed_bytes); + if (ds4_backend_uses_graph(e->backend) && !non_routed_known) { + fprintf(stderr, + "ds4: %s SSD streaming could not measure non-routed " + "weights for the manual cache safety cap\n", + ds4_backend_name(e->backend)); + ds4_engine_close(e); + *out = NULL; + return 1; + } + const uint64_t support_model_bytes = + ds4_engine_support_model_bytes(e); + const uint64_t fixed_model_bytes = + ds4_add_sat_u64(non_routed_bytes, support_model_bytes); + uint64_t safe_cache_bytes = 0; + const bool safe_cache_known = + ds4_streaming_manual_cache_safe_bytes(e->backend, + opt->context_size, + e->prefill_chunk, + e->ssd_streaming, + fixed_model_bytes, + &safe_cache_bytes); + if (safe_cache_known && safe_cache_bytes == 0) { + fprintf(stderr, + "ds4: %s SSD streaming has no safe room for the requested " + "expert cache after graph and fixed model weights\n", + ds4_backend_name(e->backend)); + ds4_engine_close(e); + *out = NULL; + return 1; + } + if (safe_cache_known && + e->ssd_streaming_cache_bytes > safe_cache_bytes) { + e->ssd_streaming_cache_bytes = safe_cache_bytes; + fprintf(stderr, + "ds4: %s SSD streaming cache budget %.2f GiB capped to %.2f GiB " + "to stay below the graph working-set pressure budget\n", + ds4_backend_name(e->backend), + (double)requested_cache_bytes / 1073741824.0, + (double)e->ssd_streaming_cache_bytes / 1073741824.0); + } + } #ifndef DS4_NO_GPU if (e->backend == DS4_BACKEND_CUDA) { @@ -62834,13 +62989,17 @@ static int ds4_engine_open_internal(ds4_engine **out, * GPU budget, so let the residency set pin them — that is what makes * the shard actually resident. Without the sysctl, fall back to lazy * faulting (slow but functional). */ - if (graph_backend && tp_shard && glm_graph_wired_limit_bytes() == 0) { + const bool skip_model_residency = + graph_backend && tp_shard && glm_graph_wired_limit_bytes() == 0; + if (skip_model_residency) { fprintf(stderr, "ds4: iogpu.wired_limit_mb is 0 -- TP expert shard will page " "lazily; raise it (e.g. sudo sysctl iogpu.wired_limit_mb=120000) " "for full residency\n"); - ds4_gpu_model_residency_skip(1); } + /* This is process-global backend state, so every engine open must reset it + * after a previous TP/no-wired-limit engine (including failed startup). */ + ds4_gpu_model_residency_skip(skip_model_residency ? 1 : 0); if (graph_backend) { if (e->multi_tier) { /* Wave-2 multi-tier branch. @@ -62926,6 +63085,50 @@ static int ds4_engine_open_internal(ds4_engine **out, load_output || (load_output_optional && weights_have_output_head(&e->weights)), opt->context_size); +#ifdef DS4_ROCM_BUILD + if (e->backend == DS4_BACKEND_CUDA && + e->ssd_streaming && e->dspark && + e->support_kind == DS4_SUPPORT_DSPARK) { + uint32_t bad_layer = UINT32_MAX; + uint32_t bad_gate_type = 0; + uint32_t bad_down_type = 0; + if (!ds4_rocm_dspark_ssd_layout_supported( + &e->weights, + &bad_layer, + &bad_gate_type, + &bad_down_type)) { + fprintf(stderr, + "ds4: ROCm DSpark with SSD streaming currently " + "requires IQ2_XXS/Q2_K or Q2_K routed experts; " + "layer %u uses gate=%s down=%s\n", + bad_layer, + tensor_type_name(bad_gate_type), + tensor_type_name(bad_down_type)); + ds4_engine_close(e); + *out = NULL; + return 1; + } + uint64_t min_verify_experts = + (uint64_t)e->dspark_weights.block_size * + (uint64_t)DS4_N_EXPERT_USED; + if (min_verify_experts > DS4_N_EXPERT) { + min_verify_experts = DS4_N_EXPERT; + } + if (e->ssd_streaming_cache_experts < min_verify_experts) { + fprintf(stderr, + "ds4: ROCm DSpark verification with SSD streaming " + "needs at least %llu cached experts (%u draft rows x " + "%u experts); configured %u\n", + (unsigned long long)min_verify_experts, + e->dspark_weights.block_size, + DS4_N_EXPERT_USED, + e->ssd_streaming_cache_experts); + ds4_engine_close(e); + *out = NULL; + return 1; + } + } +#endif if (!ds4_engine_glm_streaming_memory_guard( e, load_slice, @@ -63205,32 +63408,6 @@ static int ds4_engine_open_internal(ds4_engine **out, const bool support_model_runtime_ready = e->mtp_ready || (e->support_kind == DS4_SUPPORT_DSPARK && e->dspark); - bool support_uses_secondary_rocm_cache = false; -#ifdef DS4_ROCM_BUILD - /* The generic range cache is already keyed by model map. Keep the - * resident target ranges when adding DSpark support on gfx1151. */ - support_uses_secondary_rocm_cache = - e->support_kind == DS4_SUPPORT_DSPARK && - ds4_gpu_dspark_gfx1151_fast_path() != 0; -#endif - if (support_model_runtime_ready && - !support_uses_secondary_rocm_cache && - !ds4_gpu_set_model_map_range(e->mtp_model.map, - e->mtp_model.size, - e->mtp_model.tensor_data_pos, - e->mtp_model.size - e->mtp_model.tensor_data_pos, - e->mtp_model.max_tensor_bytes)) - { - fprintf(stderr, - "ds4: %s failed to map support model views; aborting startup. " - "This is commonly caused by insufficient memory or accelerator VM budget.\n", - ds4_backend_name(e->backend)); - free(load_offsets); - free(load_sizes); - ds4_engine_close(e); - *out = NULL; - return 1; - } if (!ds4_engine_preload_pro_q4_expert_tables(e, load_slice, load_layer_start, @@ -63255,19 +63432,28 @@ static int ds4_engine_open_internal(ds4_engine **out, } free(load_offsets); free(load_sizes); - /* Also apply explicit optional Q8 preload settings to the runtime - * support model when loaded. */ + /* The support GGUF is deliberately fully resident. Backends prepare + * it without replacing the active target mapping: Metal installs a + * second view set, CUDA a host-base-keyed range, and ROCm a persistent + * device image that survives target layer remaps. */ if (support_model_runtime_ready) { (void)ds4_gpu_set_model_fd_for_map(e->mtp_model.fd, e->mtp_model.map); - if (!accelerator_cache_model_tensors(e->backend, &e->mtp_model, - NULL, NULL, 0)) { - fprintf(stderr, "ds4: %s failed to prepare optional support model cache\n", + const int support_ok = ds4_gpu_prepare_support_model( + e->mtp_model.map, + e->mtp_model.size, + e->mtp_model.tensor_data_pos, + e->mtp_model.size - e->mtp_model.tensor_data_pos, + e->mtp_model.max_tensor_bytes); + (void)ds4_gpu_set_model_fd_for_map(e->model.fd, e->model.map); + if (!support_ok) { + fprintf(stderr, + "ds4: %s failed to prepare the resident support model; " + "check accelerator memory and model-cache limits\n", ds4_backend_name(e->backend)); ds4_engine_close(e); *out = NULL; return 1; } - (void)ds4_gpu_set_model_fd_for_map(e->model.fd, e->model.map); } if (e->vision_ready) { #if defined(__APPLE__) @@ -63728,6 +63914,12 @@ int ds4_engine_tp_bind(ds4_engine *e, struct ds4_tp *tp, char *err, size_t errle snprintf(err, errlen, "tensor parallelism requires the Metal backend"); return 0; } + if (e->ssd_streaming && e->dspark && + e->support_kind == DS4_SUPPORT_DSPARK) { + snprintf(err, errlen, + "DSpark with SSD streaming does not yet support Metal tensor parallelism"); + return 0; + } if (e->tp.active) { snprintf(err, errlen, "tensor parallelism already bound"); return 0; @@ -63831,16 +64023,19 @@ void ds4_engine_close(ds4_engine *e) { weights_free(&e->weights); vocab_free(&e->vocab); ds4_threads_shutdown(); - if (e->mtp_model.map) model_close(&e->mtp_model); - if (e->vision_model.map) model_close(&e->vision_model); - model_close(&e->model); #ifndef DS4_NO_GPU if (e->shared_prefill_workspace_ready) { metal_graph_free_prefill_workspace(&e->shared_prefill_workspace); e->shared_prefill_workspace_ready = false; } + /* Model views are no-copy aliases of the mmap regions. Release and wait + * for the backend before munmap, especially when main and DSpark support + * views coexist in the Metal registry. */ ds4_gpu_cleanup(); #endif + if (e->mtp_model.map) model_close(&e->mtp_model); + if (e->vision_model.map) model_close(&e->vision_model); + model_close(&e->model); ds4_ssd_memory_lock_release(&e->simulated_memory); ds4_release_instance_lock(); free(e->directional_steering_dirs); @@ -67579,7 +67774,17 @@ static bool ds4_session_prepare_dspark_draft(ds4_session *s, if (ds4_gpu_set_current_device(exec_tier) != 0) return false; g->active_tier = exec_tier; } - const bool ok = ds4_session_prepare_dspark_draft_impl(s, token, pos); + bool ok = true; + if (g->ssd_streaming) { + ok = metal_graph_stream_map_decode_static_all(&s->engine->model, + &s->engine->weights); + if (ok) { + g->streaming_static_decode_map_current = + metal_graph_stream_decode_static_map_enabled() && + metal_graph_stream_decode_static_map_state_cache_enabled(); + } + } + if (ok) ok = ds4_session_prepare_dspark_draft_impl(s, token, pos); if (exec_tier != saved_tier) { g->active_tier = saved_tier; if (ds4_gpu_set_current_device(saved_tier) != 0) return false; @@ -69519,6 +69724,15 @@ static int ds4_session_eval_dspark_speculative_argmax( bool ok = have_frontier && row_logits && (draft_n <= 1 || row_tops); bool verifier_may_have_mutated = false; bool tp_verify_sent = false; + /* The target logits used for target_top already verify the first draft. + * With an SSD-backed target, running the batch verifier again for a + * one-token suffix adds no acceptance information and would require the + * multi-row selected-expert path solely to recreate those logits. Replay + * below still evaluates the accepted token exactly and advances all KV + * and compressor state. SSD+DSpark is rejected under TP, so no mirrored + * worker verify is skipped here. */ + const bool skip_single_ssd_verify = + s->graph.ssd_streaming && draft_n == 1; if (ok && ds4_session_tp_leader(s)) { /* Announce the block before mutating anything: the worker runs its * half of the verify and then waits for our commit decision. */ @@ -69531,7 +69745,7 @@ static int ds4_session_eval_dspark_speculative_argmax( } tp_verify_sent = true; } - if (ok) { + if (ok && !skip_single_ssd_verify) { for (int i = 0; i < draft_n; i++) token_vec_push(&s->checkpoint, drafts[i]); verifier_may_have_mutated = true; ds4_verify_suffix_timing verify_timing; diff --git a/ds4_cuda.cu b/ds4_cuda.cu index b09098d17..b93bdabe4 100644 --- a/ds4_cuda.cu +++ b/ds4_cuda.cu @@ -689,6 +689,8 @@ static const char *cuda_model_ptr(const void *model_map, uint64_t offset) { static const char *cuda_model_range_ptr(const void *model_map, uint64_t offset, uint64_t bytes, const char *what) { if (bytes == 0) return cuda_model_ptr(model_map, offset); + /* Whole-image ownership and host registration describe only the active + * target map. A second DSpark mmap must resolve through its own cache. */ const uint64_t end = offset + bytes; if (end < offset) return NULL; auto exact = g_model_range_by_offset.find(offset); @@ -1624,6 +1626,18 @@ static const __half *cuda_q8_f16_ptr( return r.device_ptr; } } + /* Two GGUFs can legitimately reuse the same file offset. The fast + * offset index names only one of them, so fall back to the full key. */ + for (const cuda_q8_f16_range &r : g_q8_f16_ranges) { + if (r.host_base == model_map && + r.offset == offset && + r.weight_bytes == weight_bytes && + r.in_dim == in_dim && + r.out_dim == out_dim && + r.device_id == expected_device) { + return r.device_ptr; + } + } } else { for (const cuda_q8_f16_range &r : g_q8_f16_ranges) { if (r.host_base == model_map && @@ -1739,6 +1753,16 @@ static float *cuda_q8_f32_ptr( return r.device_ptr; } } + for (const cuda_q8_f32_range &r : g_q8_f32_ranges) { + if (r.host_base == model_map && + r.offset == offset && + r.weight_bytes == weight_bytes && + r.in_dim == in_dim && + r.out_dim == out_dim && + r.device_id == expected_device) { + return r.device_ptr; + } + } } else { for (const cuda_q8_f32_range &r : g_q8_f32_ranges) { if (r.host_base == model_map && @@ -2893,6 +2917,10 @@ extern "C" void ds4_gpu_cleanup(void) { g_model_range_mapping_supported = 1; g_model_hmm_direct = 0; g_model_fd = -1; + g_model_fd_host_base = NULL; + g_support_host_base = NULL; + g_support_host_size = 0; + g_support_offset_bias = 0; if (g_model_direct_fd >= 0) { (void)close(g_model_direct_fd); g_model_direct_fd = -1; @@ -3861,6 +3889,34 @@ extern "C" int ds4_gpu_set_aux_model_map_range( return 1; } +extern "C" int ds4_gpu_prepare_support_model( + const void *model_map, + uint64_t model_size, + uint64_t map_offset, + uint64_t map_size, + uint64_t max_tensor_bytes) { + (void)max_tensor_bytes; + if (!model_map || model_size == 0 || map_offset > model_size || + map_size == 0 || map_size > model_size - map_offset) { + return 0; + } + if (g_model_fd < 0 || g_model_fd_host_base != model_map) { + fprintf(stderr, + "ds4: CUDA support model fd does not match its mmap\n"); + return 0; + } + /* Keep the target mmap active and install the support payload as one + * host-base-keyed device range. Dynamic target SSD remaps then cannot + * unregister or reinterpret the secondary GGUF. The caller has selected + * the support fd before entering this function. */ + const char *ptr = cuda_model_range_ptr(model_map, + map_offset, + map_size, + "DSpark support model"); + return ptr != NULL && + cuda_model_range_is_cached(model_map, map_offset, map_size); +} + /* Register the mmap'd host model pointer for selective-cache lookups WITHOUT * triggering any device-side copy. Used by multi-GPU placement scaffolding's * multi-tier path so DS4_CUDA_COPY_MODEL cannot reintroduce a full-model @@ -24083,7 +24139,8 @@ static int routed_moe_launch( * [n_tokens, n_expert, *] by the validation above. Any entry * failure falls through to the legacy sorted-pairs path (the * buffers are scratch there too). */ - if (iq2_path && n_tokens > 1u && !owned_filtered && cuda_use_mmq()) { + if (iq2_path && n_tokens > 1u && !owned_filtered && cuda_use_mmq() && + !(g_ssd_streaming_mode && allow_streaming)) { const uint64_t gate_total = (uint64_t)n_total_expert * gate_expert_bytes; const uint64_t down_total = (uint64_t)n_total_expert * down_expert_bytes; const int mmq_tier = ds4_tensor_device_idx(out); @@ -25662,7 +25719,6 @@ extern "C" int ds4_gpu_routed_moe_one_tensor(ds4_gpu_tensor *out, ds4_gpu_tensor layer_index, 1, force_resident ? 0 : 1, 0); } extern "C" int ds4_gpu_routed_moe_batch_tensor(ds4_gpu_tensor *out, ds4_gpu_tensor *gate, ds4_gpu_tensor *up, ds4_gpu_tensor *mid, ds4_gpu_tensor *down, const void *model_map, uint64_t model_size, uint64_t gate_offset, uint64_t up_offset, uint64_t down_offset, uint32_t gate_type, uint32_t down_type, uint64_t gate_expert_bytes, uint64_t gate_row_bytes, uint64_t down_expert_bytes, uint64_t down_row_bytes, uint32_t expert_in_dim, uint32_t expert_mid_dim, uint32_t out_dim, const ds4_gpu_tensor *selected, const ds4_gpu_tensor *weights, uint32_t n_total_expert, uint32_t n_expert, float clamp, const ds4_gpu_tensor *x, uint32_t layer_index, uint32_t n_tokens, bool *mid_is_f16, bool force_resident) { - (void)force_resident; if (mid_is_f16) *mid_is_f16 = false; return routed_moe_launch(out, gate, up, mid, down, model_map, model_size, gate_offset, up_offset, down_offset, @@ -25671,7 +25727,7 @@ extern "C" int ds4_gpu_routed_moe_batch_tensor(ds4_gpu_tensor *out, ds4_gpu_tens down_expert_bytes, down_row_bytes, expert_in_dim, expert_mid_dim, out_dim, selected, weights, n_total_expert, n_expert, clamp, x, - layer_index, n_tokens, 1, 0); + layer_index, n_tokens, force_resident ? 0 : 1, 0); } extern "C" int ds4_gpu_routed_moe_batch_owned_tensor( @@ -32305,7 +32361,8 @@ extern "C" int ds4_gpu_routed_moe_set_selected_override(const int32_t *selected, } extern "C" void ds4_gpu_set_glm_streaming_prefill_full_layer(bool enabled) { - (void)enabled; /* SSD streaming is not used on the CUDA backend */ + /* CUDA streams selected experts rather than pinning whole routed layers. */ + (void)enabled; } extern "C" void ds4_gpu_set_glm_mtp_verify_mode(bool enabled) { diff --git a/ds4_gpu.h b/ds4_gpu.h index c3c1b6abc..fce976a1e 100644 --- a/ds4_gpu.h +++ b/ds4_gpu.h @@ -113,6 +113,11 @@ int ds4_gpu_synchronize(void); int ds4_gpu_set_model_map(const void *model_map, uint64_t model_size); int ds4_gpu_set_model_fd(int fd); int ds4_gpu_set_model_fd_for_map(int fd, const void *model_map); +/* Prepare a second, fully resident support GGUF without replacing the active + * target-model mapping used by SSD streaming. */ +int ds4_gpu_prepare_support_model(const void *model_map, uint64_t model_size, + uint64_t map_offset, uint64_t map_size, + uint64_t max_tensor_bytes); int ds4_gpu_build_derived_artifacts(const void *model_map, uint64_t model_size, const char *model_path); int ds4_gpu_model_range_replaced(const void *model_map, uint64_t offset, diff --git a/ds4_metal.m b/ds4_metal.m index b904c6ed1..48fb7cf71 100644 --- a/ds4_metal.m +++ b/ds4_metal.m @@ -1686,15 +1686,68 @@ static void ds4_gpu_progress_failed(void) { fflush(stderr); } +static void ds4_gpu_model_view_clear(uint32_t i) { + if (i >= DS4_METAL_MAX_MODEL_VIEWS) return; + g_model_views[i].buffer = nil; + g_model_views[i].model_map = NULL; + g_model_views[i].model_size = 0; + g_model_views[i].model_offset = 0; + g_model_views[i].bytes = 0; +} + +static void ds4_gpu_model_views_truncate(uint32_t count) { + if (count > g_model_view_count) return; + for (uint32_t i = count; i < g_model_view_count; i++) { + ds4_gpu_model_view_clear(i); + } + g_model_view_count = count; +} + static void ds4_gpu_model_views_clear(void) { - for (uint32_t i = 0; i < g_model_view_count; i++) { - g_model_views[i].buffer = nil; - g_model_views[i].model_map = NULL; - g_model_views[i].model_size = 0; - g_model_views[i].model_offset = 0; - g_model_views[i].bytes = 0; + ds4_gpu_model_views_truncate(0); +} + +static void ds4_gpu_model_map_state_invalidate(void) { + g_model_map_ptr = NULL; + g_model_map_size = 0; + g_model_mapped_offset = 0; + g_model_mapped_size = 0; + g_model_mapped_max_tensor_bytes = 0; +} + +/* Replace only the old views for one mmap while preserving views belonging to + * another simultaneously active model (for example a DSpark support GGUF). + * New views are appended first; prefix_count marks the registry boundary from + * before the append, so a failed build can be truncated without disturbing + * the previous mapping. */ +static void ds4_gpu_model_views_remove_map_prefix( + const void *model_map, + uint64_t model_size, + uint32_t prefix_count) { + if (!model_map || g_model_view_count == 0) return; + if (prefix_count > g_model_view_count) prefix_count = g_model_view_count; + + uint32_t write = 0; + for (uint32_t read = 0; read < g_model_view_count; read++) { + const bool remove = + read < prefix_count && + g_model_views[read].model_map == model_map && + g_model_views[read].model_size == model_size; + if (remove) { + ds4_gpu_model_view_clear(read); + continue; + } + if (write != read) { + g_model_views[write].buffer = g_model_views[read].buffer; + g_model_views[write].model_map = g_model_views[read].model_map; + g_model_views[write].model_size = g_model_views[read].model_size; + g_model_views[write].model_offset = g_model_views[read].model_offset; + g_model_views[write].bytes = g_model_views[read].bytes; + ds4_gpu_model_view_clear(read); + } + write++; } - g_model_view_count = 0; + g_model_view_count = write; } static void ds4_gpu_model_residency_clear(void) { @@ -1975,26 +2028,6 @@ static int ds4_gpu_finish_model_views( return 1; } -static int ds4_gpu_map_model_views( - const void *model_map, - uint64_t model_size, - uint64_t map_offset, - uint64_t map_size, - uint64_t max_tensor_bytes) { - const double t0 = ds4_gpu_now_ms(); - uint64_t mapped_model_size = 0; - if (!ds4_gpu_add_model_view_range(model_map, - model_size, - map_offset, - map_size, - max_tensor_bytes, - false, - &mapped_model_size)) { - return 0; - } - return ds4_gpu_finish_model_views(t0, mapped_model_size, map_offset); -} - static id ds4_gpu_new_transient_buffer(NSUInteger bytes, const char *label) { if (bytes == 0) bytes = 1; @@ -11309,9 +11342,43 @@ int ds4_gpu_set_model_map_range(const void *model_map, uint64_t model_size, uint } } + const double t0 = ds4_gpu_now_ms(); + const uint32_t old_view_count = g_model_view_count; + uint64_t mapped_model_size = 0; + ds4_gpu_model_residency_clear(); + if (!ds4_gpu_add_model_view_range(model_map, + model_size, + map_offset, + map_size, + max_tensor_bytes, + false, + &mapped_model_size)) { + ds4_gpu_model_views_truncate(old_view_count); + if (!ds4_gpu_model_residency_request_views()) { + ds4_gpu_model_residency_clear(); + ds4_gpu_model_views_clear(); + ds4_gpu_model_map_state_invalidate(); + } + return 0; + } + if (!ds4_gpu_finish_model_views(t0, mapped_model_size, map_offset)) { + ds4_gpu_model_residency_clear(); + ds4_gpu_model_views_truncate(old_view_count); + if (!ds4_gpu_model_residency_request_views()) { + ds4_gpu_model_residency_clear(); + ds4_gpu_model_views_clear(); + ds4_gpu_model_map_state_invalidate(); + } + return 0; + } ds4_gpu_model_residency_clear(); - if (!ds4_gpu_map_model_views(model_map, model_size, map_offset, map_size, max_tensor_bytes)) { + ds4_gpu_model_views_remove_map_prefix(model_map, + model_size, + old_view_count); + if (!ds4_gpu_model_residency_request_views()) { ds4_gpu_model_residency_clear(); + ds4_gpu_model_views_clear(); + ds4_gpu_model_map_state_invalidate(); return 0; } g_model_map_ptr = model_map; @@ -11359,18 +11426,22 @@ int ds4_gpu_set_model_map_spans( const double t0 = ds4_gpu_now_ms(); max_tensor_bytes = ds4_gpu_effective_model_max_tensor_bytes(model_size, max_tensor_bytes); + for (uint32_t i = 0; i < count; i++) { + if (offsets[i] > model_size || sizes[i] == 0 || + sizes[i] > model_size - offsets[i]) { + fprintf(stderr, + "ds4: Metal model span %u is outside the GGUF mapping\n", + i); + return 0; + } + } + + const uint32_t old_view_count = g_model_view_count; ds4_gpu_model_residency_clear(); - ds4_gpu_model_views_clear(); uint64_t mapped_total = 0; uint64_t first_offset = UINT64_MAX; for (uint32_t i = 0; i < count; i++) { - if (offsets[i] > model_size || sizes[i] == 0 || sizes[i] > model_size - offsets[i]) { - fprintf(stderr, "ds4: Metal model span %u is outside the GGUF mapping\n", i); - ds4_gpu_model_residency_clear(); - ds4_gpu_model_views_clear(); - return 0; - } if (offsets[i] < first_offset) first_offset = offsets[i]; uint64_t effective_max = max_tensor_bytes; if (effective_max > sizes[i]) effective_max = sizes[i]; @@ -11381,14 +11452,33 @@ int ds4_gpu_set_model_map_spans( effective_max, true, &mapped_total)) { - ds4_gpu_model_residency_clear(); - ds4_gpu_model_views_clear(); + ds4_gpu_model_views_truncate(old_view_count); + if (!ds4_gpu_model_residency_request_views()) { + ds4_gpu_model_residency_clear(); + ds4_gpu_model_views_clear(); + ds4_gpu_model_map_state_invalidate(); + } return 0; } } if (!ds4_gpu_finish_model_views(t0, mapped_total, first_offset)) { + ds4_gpu_model_residency_clear(); + ds4_gpu_model_views_truncate(old_view_count); + if (!ds4_gpu_model_residency_request_views()) { + ds4_gpu_model_residency_clear(); + ds4_gpu_model_views_clear(); + ds4_gpu_model_map_state_invalidate(); + } + return 0; + } + ds4_gpu_model_residency_clear(); + ds4_gpu_model_views_remove_map_prefix(model_map, + model_size, + old_view_count); + if (!ds4_gpu_model_residency_request_views()) { ds4_gpu_model_residency_clear(); ds4_gpu_model_views_clear(); + ds4_gpu_model_map_state_invalidate(); return 0; } g_model_map_ptr = model_map; @@ -11410,6 +11500,20 @@ int ds4_gpu_set_model_map(const void *model_map, uint64_t model_size) { return ds4_gpu_set_model_map_range(model_map, model_size, 0, model_size, 0); } +int ds4_gpu_prepare_support_model(const void *model_map, + uint64_t model_size, + uint64_t map_offset, + uint64_t map_size, + uint64_t max_tensor_bytes) { + /* Metal's model-view registry is keyed by mmap identity, so installing the + * support GGUF preserves the independently replaceable target views. */ + return ds4_gpu_set_model_map_range(model_map, + model_size, + map_offset, + map_size, + max_tensor_bytes); +} + /* DS4_METAL_STREAMING_EXPERT_NOCACHE: serve the streaming expert preads from * a second F_NOCACHE descriptor so the ~1 GB/token of expert churn stops * evicting the mapped dense weights from the page cache. On tight-RAM @@ -13308,6 +13412,7 @@ static int ds4_gpu_stream_prefill_batch_selected_addr_enabled( uint32_t gate_type, uint32_t down_type) { if (!g_ssd_streaming_mode || + g_tp_split_world != 1 || n_tokens <= 1 || n_total_expert == 0 || n_expert != 6 || @@ -13320,10 +13425,12 @@ static int ds4_gpu_stream_prefill_batch_selected_addr_enabled( getenv("DS4_METAL_DISABLE_ROUTED_PAIR_SWIGLU_FUSION") != NULL) { return 0; } - /* All unique experts for one layer must fit simultaneously because the - * address-table kernels consume them in one dispatch. Once the global - * cache fills, preparation reuses entries owned by other layers. */ - if (ds4_gpu_stream_expert_cache_configured_count() < n_total_expert) { + /* Ordinary prefill needs enough persistent slots to prepare a complete + * address table without evicting protected entries. DSpark verification + * is only 2..5 rows: when that set exceeds the cache, preparation packs + * just its unique experts into transient buffers instead. */ + if (ds4_gpu_stream_expert_cache_configured_count() < n_total_expert && + n_tokens > 5u) { return 0; } if (getenv("DS4_METAL_ENABLE_STREAMING_PREFILL_BATCH_SELECTED_ADDR") != NULL) { @@ -16253,6 +16360,194 @@ static void ds4_gpu_stream_expert_cache_clear_layer(uint32_t layer) { g_stream_expert_cache_layer_count[layer] = 0; } +/* Tiny speculative batches may route to more unique experts than the user's + * persistent SSD cache can hold. Loading the entire routed tensors for that + * case defeats streaming (and is especially costly for every verifier layer). + * Pack only the selected expert slabs into transient shared buffers and give + * the address-table kernels private transient tables. This avoids leaving + * dangling GPU addresses in the persistent decode-cache tables after the + * command completes. The transient registry retains every buffer until the + * enclosing Metal command buffer completes. */ +static int ds4_gpu_stream_expert_prepare_transient_selected_batch( + uint64_t model_size, + uint32_t layer, + const int32_t *unique_ids, + uint32_t unique_count, + uint32_t n_total_expert, + uint64_t gate_offset, + uint64_t up_offset, + uint64_t down_offset, + uint64_t gate_expert_bytes, + uint64_t down_expert_bytes, + id *gate_addrs, + id *up_addrs, + id *down_addrs, + id *packed_gate, + id *packed_up, + id *packed_down) { + if (!unique_ids || unique_count == 0 || unique_count > n_total_expert || + !gate_addrs || !up_addrs || !down_addrs || + !packed_gate || !packed_up || !packed_down || + gate_expert_bytes == 0 || down_expert_bytes == 0 || + (uint64_t)unique_count > UINT64_MAX / gate_expert_bytes || + (uint64_t)unique_count > UINT64_MAX / down_expert_bytes) { + return 0; + } + + const uint64_t gate_bytes = (uint64_t)unique_count * gate_expert_bytes; + const uint64_t down_bytes = (uint64_t)unique_count * down_expert_bytes; + if (gate_bytes == 0 || down_bytes == 0 || + gate_bytes > (uint64_t)NSUIntegerMax || + down_bytes > (uint64_t)NSUIntegerMax) { + return 0; + } + + id gate_buf = + ds4_gpu_new_transient_buffer((NSUInteger)gate_bytes, + "ds4_ssd_tiny_selected_gate"); + id up_buf = + ds4_gpu_new_transient_buffer((NSUInteger)gate_bytes, + "ds4_ssd_tiny_selected_up"); + id down_buf = + ds4_gpu_new_transient_buffer((NSUInteger)down_bytes, + "ds4_ssd_tiny_selected_down"); + const uint64_t addr_bytes64 = + (uint64_t)n_total_expert * sizeof(uint64_t); + if (addr_bytes64 == 0 || addr_bytes64 > (uint64_t)NSUIntegerMax) return 0; + const NSUInteger addr_bytes = (NSUInteger)addr_bytes64; + id gate_addr_buf = + ds4_gpu_new_transient_buffer(addr_bytes, + "ds4_ssd_tiny_selected_gate_addrs"); + id up_addr_buf = + ds4_gpu_new_transient_buffer(addr_bytes, + "ds4_ssd_tiny_selected_up_addrs"); + id down_addr_buf = + ds4_gpu_new_transient_buffer(addr_bytes, + "ds4_ssd_tiny_selected_down_addrs"); + if (!gate_buf || !up_buf || !down_buf || + !gate_addr_buf || !up_addr_buf || !down_addr_buf) { + return 0; + } + + uint8_t *gate_dst = (uint8_t *)[gate_buf contents]; + uint8_t *up_dst = (uint8_t *)[up_buf contents]; + uint8_t *down_dst = (uint8_t *)[down_buf contents]; + uint64_t *gate_addr = (uint64_t *)[gate_addr_buf contents]; + uint64_t *up_addr = (uint64_t *)[up_addr_buf contents]; + uint64_t *down_addr = (uint64_t *)[down_addr_buf contents]; + if (!gate_dst || !up_dst || !down_dst || + !gate_addr || !up_addr || !down_addr) { + return 0; + } + memset(gate_addr, 0, addr_bytes); + memset(up_addr, 0, addr_bytes); + memset(down_addr, 0, addr_bytes); + + ds4_gpu_stream_expert_pread_task *tasks = + calloc((size_t)unique_count * 3u, sizeof(tasks[0])); + if (!tasks) return 0; + + uint32_t n_tasks = 0; + int ok = 1; + for (uint32_t u = 0; u < unique_count; u++) { + const int32_t selected_id = unique_ids[u]; + if (selected_id < 0 || (uint32_t)selected_id >= n_total_expert) { + ok = 0; + break; + } + const uint64_t expert = (uint64_t)(uint32_t)selected_id; + if (expert > UINT64_MAX / gate_expert_bytes || + expert > UINT64_MAX / down_expert_bytes) { + ok = 0; + break; + } + const uint64_t gate_rel = expert * gate_expert_bytes; + const uint64_t down_rel = expert * down_expert_bytes; + if (gate_rel > UINT64_MAX - gate_offset || + gate_rel > UINT64_MAX - up_offset || + down_rel > UINT64_MAX - down_offset) { + ok = 0; + break; + } + const uint64_t gate_abs = gate_offset + gate_rel; + const uint64_t up_abs = up_offset + gate_rel; + const uint64_t down_abs = down_offset + down_rel; + if (gate_abs > model_size || gate_expert_bytes > model_size - gate_abs || + up_abs > model_size || gate_expert_bytes > model_size - up_abs || + down_abs > model_size || down_expert_bytes > model_size - down_abs) { + ok = 0; + break; + } + + const uint64_t gate_inner = (uint64_t)u * gate_expert_bytes; + const uint64_t down_inner = (uint64_t)u * down_expert_bytes; + tasks[n_tasks++] = (ds4_gpu_stream_expert_pread_task) { + .offset = gate_abs, + .len = gate_expert_bytes, + .dst = gate_dst + gate_inner, + }; + tasks[n_tasks++] = (ds4_gpu_stream_expert_pread_task) { + .offset = up_abs, + .len = gate_expert_bytes, + .dst = up_dst + gate_inner, + }; + tasks[n_tasks++] = (ds4_gpu_stream_expert_pread_task) { + .offset = down_abs, + .len = down_expert_bytes, + .dst = down_dst + down_inner, + }; + } + + uint64_t read_bytes = 0; + double read_ms = 0.0; + if (ok) { + ok = ds4_gpu_stream_expert_pread_tasks(tasks, + n_tasks, + &read_bytes, + &read_ms); + } + free(tasks); + if (!ok) return 0; + + ds4_gpu_stream_expert_cache_note_pread(layer, read_bytes, read_ms); + [gate_buf didModifyRange:NSMakeRange(0, (NSUInteger)gate_bytes)]; + [up_buf didModifyRange:NSMakeRange(0, (NSUInteger)gate_bytes)]; + [down_buf didModifyRange:NSMakeRange(0, (NSUInteger)down_bytes)]; + + for (uint32_t u = 0; u < unique_count; u++) { + const uint32_t expert = (uint32_t)unique_ids[u]; + const NSUInteger gate_inner = (NSUInteger)((uint64_t)u * gate_expert_bytes); + const NSUInteger down_inner = (NSUInteger)((uint64_t)u * down_expert_bytes); + gate_addr[expert] = ds4_gpu_buffer_address(gate_buf, gate_inner); + up_addr[expert] = ds4_gpu_buffer_address(up_buf, gate_inner); + down_addr[expert] = ds4_gpu_buffer_address(down_buf, down_inner); + if (gate_addr[expert] == 0 || up_addr[expert] == 0 || + down_addr[expert] == 0) { + return 0; + } + } + [gate_addr_buf didModifyRange:NSMakeRange(0, addr_bytes)]; + [up_addr_buf didModifyRange:NSMakeRange(0, addr_bytes)]; + [down_addr_buf didModifyRange:NSMakeRange(0, addr_bytes)]; + + *gate_addrs = gate_addr_buf; + *up_addrs = up_addr_buf; + *down_addrs = down_addr_buf; + *packed_gate = gate_buf; + *packed_up = up_buf; + *packed_down = down_buf; + if (getenv("DS4_METAL_STREAMING_EXPERT_PREAD_PROFILE") != NULL) { + fprintf(stderr, + "ds4: Metal SSD tiny selected batch layer=%u experts=%u " + "bytes=%.2f MiB wall=%.3f ms\n", + layer, + unique_count, + ds4_gpu_mib(read_bytes), + read_ms); + } + return 1; +} + static int ds4_gpu_stream_expert_cache_prepare_selected_batch( const void *model_map, uint64_t model_size, @@ -16351,6 +16646,33 @@ static int ds4_gpu_stream_expert_cache_prepare_selected_batch( frequency, n_total_expert); } + const bool use_transient_selected = + ok && + n_tokens <= 5u && + ds4_gpu_stream_expert_cache_configured_count() < n_total_expert; + if (use_transient_selected) { + ok = ds4_gpu_stream_expert_prepare_transient_selected_batch( + model_size, + layer, + unique_ids, + unique_count, + n_total_expert, + gate_offset, + up_offset, + down_offset, + gate_expert_bytes, + down_expert_bytes, + gate_addrs, + up_addrs, + down_addrs, + overflow_gate, + overflow_up, + overflow_down); + free(ids); + if (!ok) return 0; + *unique_out = unique_count; + return 1; + } /* * When the layer's unique selected set does not fit the cache budget, the * extra experts are addressed straight into whole-tensor mapped model views @@ -31070,7 +31392,8 @@ static int ds4_gpu_encode_mul_mv_addr_iq2_pair_swiglu( for (uint32_t i = 0; i < n_entries; i++) { if (!entries[i] || !entries[i]->gate_buffer || !entries[i]->up_buffer) return 0; } - if (!ds4_gpu_stream_expert_cache_mark_entries_inflight(entries, + if (n_entries != 0 && + !ds4_gpu_stream_expert_cache_mark_entries_inflight(entries, n_entries, 0)) { return 0; @@ -31194,7 +31517,8 @@ static int ds4_gpu_encode_mul_mv_addr_q2_sum6( for (uint32_t i = 0; i < n_entries; i++) { if (!entries[i] || !entries[i]->down_buffer) return 0; } - if (!ds4_gpu_stream_expert_cache_mark_entries_inflight(entries, + if (n_entries != 0 && + !ds4_gpu_stream_expert_cache_mark_entries_inflight(entries, n_entries, 0)) { return 0; @@ -38580,7 +38904,9 @@ int ds4_gpu_routed_moe_one_tensor( return 0; } if ((expert_in_dim % 256u) != 0 || (expert_mid_dim % 256u) != 0) { if (getenv("DS4_GLM_TP_DEBUG")) fprintf(stderr, "ds4: routed_moe_one silent return at line %d\n", 32047); return 0; } - ds4_gpu_stream_expert_cache_note_token(layer_index); + if (!force_resident) { + ds4_gpu_stream_expert_cache_note_token(layer_index); + } @autoreleasepool { id xbuf = ds4_gpu_tensor_buffer(x); @@ -41136,7 +41462,6 @@ int ds4_gpu_routed_moe_batch_tensor( uint32_t n_tokens, bool *mid_is_f16, bool force_resident) { - (void)force_resident; if (!g_initialized && !ds4_gpu_init()) return 0; /* TP sharding (see ds4_gpu_routed_moe_one_tensor): bind from the owned * expert range and rebase ids in the kernels. */ @@ -41236,7 +41561,7 @@ int ds4_gpu_routed_moe_batch_tensor( n_tokens == 1 && n_expert == 6 && n_total_expert >= 128 && - (g_ssd_streaming_mode || + ((g_ssd_streaming_mode && !force_resident) || (gate_tensor_bytes >= q4_selected_min_tensor_bytes && down_tensor_bytes >= q4_selected_min_tensor_bytes)) && !g_quality_mode && @@ -41261,7 +41586,7 @@ int ds4_gpu_routed_moe_batch_tensor( getenv("DS4_METAL_DISABLE_ROUTED_PAIR_SWIGLU_FUSION") == NULL && g_moe_mul_mv_id_mxfp4_pair_swiglu_pipeline != nil && g_moe_mul_mv_id_mxfp4_sum6_pipeline != nil && - (!g_ssd_streaming_mode || + (force_resident || !g_ssd_streaming_mode || (g_moe_mul_mv_slots6_mxfp4_pair_swiglu_pipeline != nil && g_moe_mul_mv_slots6_mxfp4_sum6_pipeline != nil)); if (use_single_token_q4_one_tensor || use_single_token_mxfp4_one_tensor) { @@ -41293,7 +41618,7 @@ int ds4_gpu_routed_moe_batch_tensor( x, NULL, layer_index, - false); + force_resident); } @autoreleasepool { @@ -41359,6 +41684,7 @@ int ds4_gpu_routed_moe_batch_tensor( return 0; } const bool use_iq2_batch_selected_addr = + !force_resident && ds4_gpu_stream_prefill_batch_selected_addr_enabled(n_tokens, n_total_expert, n_expert, diff --git a/rocm/ds4_rocm_attention.cuh b/rocm/ds4_rocm_attention.cuh index 939bb292b..a5cb44036 100644 --- a/rocm/ds4_rocm_attention.cuh +++ b/rocm/ds4_rocm_attention.cuh @@ -204,6 +204,85 @@ __global__ static void attention_prefill_raw_kernel( } } +/* Non-causal attention used by the DSpark draft block. Every draft query sees + * the complete visible raw-KV ring plus the learned per-head sink. Keep the + * scalar accumulation order aligned with the CUDA reference path. */ +__global__ static void attention_noncausal_raw_batch_heads_kernel( + float *heads, + const float *sinks, + const float *q, + const float *raw_kv, + uint32_t n_tokens, + uint32_t n_raw, + uint32_t raw_cap, + uint32_t raw_start, + uint32_t n_head, + uint32_t head_dim) { + const uint32_t tok = blockIdx.x; + const uint32_t h = blockIdx.y; + if (tok >= n_tokens || h >= n_head) return; + + extern __shared__ float sh_scores[]; + __shared__ float partial[256]; + __shared__ float max_s; + __shared__ float denom; + const float *qh = q + ((uint64_t)tok * n_head + h) * head_dim; + const float scale = rsqrtf((float)head_dim); + + for (uint32_t r = threadIdx.x; r < n_raw; r += blockDim.x) { + const uint32_t row = (raw_start + r) % raw_cap; + const float *kv = raw_kv + (uint64_t)row * head_dim; + float dot = 0.0f; + for (uint32_t d = 0; d < head_dim; d++) dot += qh[d] * kv[d]; + sh_scores[r] = dot * scale; + } + __syncthreads(); + + float local_max = sinks[h]; + for (uint32_t r = threadIdx.x; r < n_raw; r += blockDim.x) { + local_max = fmaxf(local_max, sh_scores[r]); + } + partial[threadIdx.x] = local_max; + __syncthreads(); + for (uint32_t stride = blockDim.x >> 1u; stride > 0u; stride >>= 1u) { + if (threadIdx.x < stride) { + partial[threadIdx.x] = + fmaxf(partial[threadIdx.x], partial[threadIdx.x + stride]); + } + __syncthreads(); + } + if (threadIdx.x == 0) max_s = partial[0]; + __syncthreads(); + + float den_local = 0.0f; + for (uint32_t r = threadIdx.x; r < n_raw; r += blockDim.x) { + sh_scores[r] = expf(sh_scores[r] - max_s); + den_local += sh_scores[r]; + } + partial[threadIdx.x] = den_local; + __syncthreads(); + for (uint32_t stride = blockDim.x >> 1u; stride > 0u; stride >>= 1u) { + if (threadIdx.x < stride) { + partial[threadIdx.x] += partial[threadIdx.x + stride]; + } + __syncthreads(); + } + if (threadIdx.x == 0) { + denom = partial[0] + expf(sinks[h] - max_s); + } + __syncthreads(); + + float *oh = heads + ((uint64_t)tok * n_head + h) * head_dim; + for (uint32_t d = threadIdx.x; d < head_dim; d += blockDim.x) { + float acc = 0.0f; + for (uint32_t r = 0; r < n_raw; r++) { + const uint32_t row = (raw_start + r) % raw_cap; + acc += raw_kv[(uint64_t)row * head_dim + d] * sh_scores[r]; + } + oh[d] = acc / denom; + } +} + __global__ static void attention_prefill_mixed_kernel( float *heads, const float *sinks, diff --git a/rocm/ds4_rocm_attention_launch.cuh b/rocm/ds4_rocm_attention_launch.cuh index 3190f8bb4..1b7b79537 100644 --- a/rocm/ds4_rocm_attention_launch.cuh +++ b/rocm/ds4_rocm_attention_launch.cuh @@ -467,6 +467,58 @@ extern "C" int ds4_gpu_attention_decode_raw_batch_heads_tensor( n_head, head_dim); } +extern "C" int ds4_gpu_attention_noncausal_raw_batch_heads_tensor( + ds4_gpu_tensor *heads, + const void *model_map, + uint64_t model_size, + uint64_t sinks_offset, + const ds4_gpu_tensor *q, + const ds4_gpu_tensor *raw_kv, + uint32_t n_tokens, + uint32_t n_raw, + uint32_t raw_cap, + uint32_t raw_start, + uint32_t n_head, + uint32_t head_dim) { + uint64_t sink_bytes = 0; + uint64_t head_elems = 0; + uint64_t head_bytes = 0; + uint64_t raw_bytes = 0; + const bool sizes_ok = + cuda_u64_mul_checked(n_head, sizeof(float), &sink_bytes) && + cuda_u64_mul3_checked(n_tokens, n_head, head_dim, &head_elems) && + cuda_u64_mul_checked(head_elems, sizeof(float), &head_bytes) && + cuda_u64_mul3_checked(raw_cap, head_dim, sizeof(float), &raw_bytes); + if (!heads || !q || !raw_kv || !model_map || + !sizes_ok || + n_tokens == 0 || n_raw == 0 || raw_cap < n_raw || + raw_start >= raw_cap || n_head == 0 || head_dim == 0 || + sinks_offset > model_size || sink_bytes > model_size - sinks_offset || + heads->bytes < head_bytes || q->bytes < head_bytes || + raw_kv->bytes < raw_bytes) { + return 0; + } + const float *sinks = (const float *)cuda_model_range_ptr( + model_map, sinks_offset, sink_bytes, "dspark_attn_sinks"); + if (!sinks) return 0; + const size_t shmem = (size_t)n_raw * sizeof(float); + if (shmem > 32768u) return 0; + dim3 grid(n_tokens, n_head, 1); + attention_noncausal_raw_batch_heads_kernel<<>>( + (float *)heads->ptr, + sinks, + (const float *)q->ptr, + (const float *)raw_kv->ptr, + n_tokens, + n_raw, + raw_cap, + raw_start, + n_head, + head_dim); + return cuda_ok(cudaGetLastError(), + "attention noncausal raw batch heads launch"); +} + extern "C" int ds4_gpu_attention_decode_mixed_batch_heads_tensor( ds4_gpu_tensor *heads, const void *model_map, diff --git a/rocm/ds4_rocm_runtime.cuh b/rocm/ds4_rocm_runtime.cuh index ca8a26851..b1075d10e 100644 --- a/rocm/ds4_rocm_runtime.cuh +++ b/rocm/ds4_rocm_runtime.cuh @@ -5954,6 +5954,7 @@ extern "C" void ds4_gpu_cleanup(void) { g_model_device_owned = 0; g_model_range_mapping_supported = 1; g_model_fd = -1; + g_model_fd_host_base = NULL; if (g_model_direct_fd >= 0) { (void)close(g_model_direct_fd); g_model_direct_fd = -1; @@ -5961,6 +5962,8 @@ extern "C" void ds4_gpu_cleanup(void) { g_model_direct_align = 1; g_model_file_size = 0; g_model_cache_full = 0; + g_ssd_streaming_mode = 0; + g_stream_expert_cache_budget = 0; } __global__ static void fill_f32_kernel(float *x, uint64_t n, float v); @@ -6231,6 +6234,45 @@ extern "C" int ds4_gpu_set_aux_model_map_range( return 1; } +extern "C" int ds4_gpu_prepare_support_model( + const void *model_map, + uint64_t model_size, + uint64_t map_offset, + uint64_t map_size, + uint64_t max_tensor_bytes) { + (void)max_tensor_bytes; + if (!model_map || model_size == 0 || map_offset > model_size || + map_size == 0 || map_size > model_size - map_offset) { + return 0; + } + if (g_model_fd < 0 || g_model_fd_host_base != model_map) { + fprintf(stderr, + DS4_GPU_LOG_PREFIX "support model fd does not match its mmap\n"); + return 0; + } + + /* ROCm's streaming range cache is intentionally replaced at every target + * layer. Store the small DSpark GGUF in the persistent, mmap-keyed image + * registry instead, while leaving the target map as the active mapping. */ + const void *saved_host_base = g_model_host_base; + const char *saved_device_base = g_model_device_base; + const uint64_t saved_registered_size = g_model_registered_size; + const int saved_device_owned = g_model_device_owned; + + cuda_q8_f16_cache_release_all(); + g_q8_f16_disabled_for_multi_model = 1; + const int ok = cuda_model_copy_chunked(model_map, + model_size, + map_offset, + map_size); + + g_model_host_base = saved_host_base; + g_model_device_base = saved_device_base; + g_model_registered_size = saved_registered_size; + g_model_device_owned = saved_device_owned; + return ok; +} + extern "C" int ds4_gpu_set_model_map_spans( const void *model_map, uint64_t model_size, diff --git a/tests/ds4_test.c b/tests/ds4_test.c index cf8bca2c5..86fa3b893 100644 --- a/tests/ds4_test.c +++ b/tests/ds4_test.c @@ -6832,7 +6832,7 @@ static void test_print_help(const char *prog) { puts("\nEnvironment:"); puts(" DS4_TEST_MODEL=FILE Model path. Default: ds4flash.gguf"); puts(" DS4_TEST_BACKEND=cpu Run model tests on CPU instead of Metal/CUDA."); - puts(" DS4_TEST_SSD_STREAMING=1 Run model tests through Metal SSD streaming."); + puts(" DS4_TEST_SSD_STREAMING=1 Run model tests through backend SSD streaming."); puts(" DS4_TEST_SSD_STREAMING_CACHE_GB=N Streaming routed expert cache in GiB."); puts(" DS4_TEST_SSD_STREAMING_CACHE_EXPERTS=N Streaming routed expert cache count."); puts(" DS4_TEST_SSD_STREAMING_COLD=1 Skip streaming hot expert preload."); diff --git a/tests/dspark_acceptance_fixture.sh b/tests/dspark_acceptance_fixture.sh index 07b39f3dc..8f621baff 100644 --- a/tests/dspark_acceptance_fixture.sh +++ b/tests/dspark_acceptance_fixture.sh @@ -23,6 +23,46 @@ fi partial_cases=0 direct_partial_cases=0 direct_commits=0 +BACKEND=${DS4_DSPARK_FIXTURE_BACKEND:-auto} +SSD_STREAMING=${DS4_DSPARK_FIXTURE_SSD_STREAMING:-0} +SSD_CACHE_EXPERTS=${DS4_DSPARK_FIXTURE_SSD_STREAMING_CACHE_EXPERTS:-} +REQUIRE_ACTIVE=${DS4_DSPARK_FIXTURE_REQUIRE_ACTIVE:-1} +total_proposed=0 +total_accepted_draft=0 + +case "$BACKEND" in +auto|metal|cuda|rocm) ;; +*) + echo "dspark-fixture: invalid DS4_DSPARK_FIXTURE_BACKEND=$BACKEND" >&2 + exit 1 + ;; +esac +case "$SSD_STREAMING" in +0|1) ;; +*) + echo "dspark-fixture: DS4_DSPARK_FIXTURE_SSD_STREAMING must be 0 or 1" >&2 + exit 1 + ;; +esac +case "$REQUIRE_ACTIVE" in +0|1) ;; +*) + echo "dspark-fixture: DS4_DSPARK_FIXTURE_REQUIRE_ACTIVE must be 0 or 1" >&2 + exit 1 + ;; +esac +case "$SSD_CACHE_EXPERTS" in +""|*[!0-9]*) + if [ -n "$SSD_CACHE_EXPERTS" ]; then + echo "dspark-fixture: invalid SSD streaming expert count $SSD_CACHE_EXPERTS" >&2 + exit 1 + fi + ;; +esac +if [ "$SSD_STREAMING" = 0 ] && [ -n "$SSD_CACHE_EXPERTS" ]; then + echo "dspark-fixture: SSD cache experts requires DS4_DSPARK_FIXTURE_SSD_STREAMING=1" >&2 + exit 1 +fi proposal_quality_guard_enabled() { case "$PROPOSAL_QUALITY_GUARD" in @@ -109,6 +149,8 @@ print_metadata() { "$tail_min_tokens" "$PROPOSAL_QUALITY_GUARD" \ "$PROPOSAL_QUALITY_GUARD_ACTIVE" "$C_ADD_MIN_ACCEPTED" \ "$REQUIRE_DIRECT" "$REQUIRE_IDENTICAL" + printf '# backend=%s ssd_streaming=%s ssd_cache_experts=%s require_active=%s\n' \ + "$BACKEND" "$SSD_STREAMING" "${SSD_CACHE_EXPERTS:-auto}" "$REQUIRE_ACTIVE" printf '# baseline_command=%s -m %s --tokens %s --temp %s --top-p %s --min-p %s --seed %s --nothink -p \n' \ "$DS4_BIN" "$MODEL" "$TOKENS" "$TEMPERATURE" "$TOP_P" "$MIN_P" "$SEED" printf '# dspark_command=DS4_DSPARK_STATS=1 %s --dspark%s%s -m %s --mtp-model %s --tokens %s --temp %s --top-p %s --min-p %s --seed %s --nothink -p \n' \ @@ -133,6 +175,44 @@ fi tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/ds4-dspark-fixture.XXXXXX") trap 'rm -rf "$tmpdir"' EXIT HUP INT TERM +run_variant() { + mode=$1 + prompt=$2 + stdout_file=$3 + stderr_file=$4 + + set -- "$DS4_BIN" + case "$BACKEND" in + metal) set -- "$@" --metal ;; + cuda) set -- "$@" --cuda ;; + rocm) set -- "$@" --rocm ;; + esac + if [ "$SSD_STREAMING" = 1 ]; then + set -- "$@" --ssd-streaming + if [ -n "$SSD_CACHE_EXPERTS" ]; then + set -- "$@" --ssd-streaming-cache-experts "$SSD_CACHE_EXPERTS" + fi + fi + if [ "$mode" = dspark ]; then + set -- "$@" --dspark --mtp-model "$SUPPORT" + if [ "$EXACT_SAMPLING" != 0 ]; then + set -- "$@" --mtp-exact-sampling + fi + if [ -n "$CONFIDENCE" ]; then + set -- "$@" --dspark-confidence "$CONFIDENCE" + fi + fi + set -- "$@" -m "$MODEL" --tokens "$TOKENS" \ + --temp "$TEMPERATURE" --top-p "$TOP_P" --min-p "$MIN_P" \ + --seed "$SEED" --nothink -p "$prompt" + + if [ "$mode" = dspark ]; then + DS4_DSPARK_STATS=1 "$@" >"$stdout_file" 2>"$stderr_file" + else + "$@" >"$stdout_file" 2>"$stderr_file" + fi +} + run_case() { id=$1 prompt=$2 @@ -141,27 +221,8 @@ run_case() { dspark_out="$tmpdir/$id.dspark.out" dspark_err="$tmpdir/$id.dspark.err" - "$DS4_BIN" -m "$MODEL" \ - --tokens "$TOKENS" --temp "$TEMPERATURE" --top-p "$TOP_P" \ - --min-p "$MIN_P" --seed "$SEED" --nothink -p "$prompt" \ - >"$base_out" 2>"$base_err" - - if [ -n "$CONFIDENCE" ]; then - DS4_DSPARK_STATS=1 \ - "$DS4_BIN" --dspark $exact_sampling_arg \ - --dspark-confidence "$CONFIDENCE" \ - -m "$MODEL" --mtp-model "$SUPPORT" \ - --tokens "$TOKENS" --temp "$TEMPERATURE" --top-p "$TOP_P" \ - --min-p "$MIN_P" --seed "$SEED" --nothink -p "$prompt" \ - >"$dspark_out" 2>"$dspark_err" - else - DS4_DSPARK_STATS=1 \ - "$DS4_BIN" --dspark $exact_sampling_arg \ - -m "$MODEL" --mtp-model "$SUPPORT" \ - --tokens "$TOKENS" --temp "$TEMPERATURE" --top-p "$TOP_P" \ - --min-p "$MIN_P" --seed "$SEED" --nothink -p "$prompt" \ - >"$dspark_out" 2>"$dspark_err" - fi + run_variant baseline "$prompt" "$base_out" "$base_err" + run_variant dspark "$prompt" "$dspark_out" "$dspark_err" output_match=1 if ! cmp -s "$base_out" "$dspark_out"; then @@ -186,11 +247,15 @@ run_case() { partial=$(printf '%s\n' "$stats" | sed -n 's/.* partial=\([0-9][0-9]*\).*/\1/p') errors=$(printf '%s\n' "$stats" | sed -n 's/.*errors=\([0-9][0-9]*\).*/\1/p') + verifier_unavailable=$(printf '%s\n' "$stats" | sed -n 's/.*verifier_unavailable=\([0-9][0-9]*\).*/\1/p') + proposed=$(printf '%s\n' "$stats" | sed -n 's/.*proposed=\([0-9][0-9]*\).*/\1/p') accepted_draft=$(printf '%s\n' "$stats" | sed -n 's/.*accepted_draft=\([0-9][0-9]*\).*/\1/p') direct_full=$(printf '%s\n' "$stats" | sed -n 's/.*direct_full=\([0-9][0-9]*\).*/\1/p') direct_partial=$(printf '%s\n' "$stats" | sed -n 's/.*direct_partial=\([0-9][0-9]*\).*/\1/p') partial=${partial:-0} errors=${errors:-0} + verifier_unavailable=${verifier_unavailable:-0} + proposed=${proposed:-0} accepted_draft=${accepted_draft:-0} direct_full=${direct_full:-0} direct_partial=${direct_partial:-0} @@ -198,6 +263,12 @@ run_case() { echo "dspark-fixture: verifier errors for $id: $stats" >&2 return 1 fi + if [ "$verifier_unavailable" -ne 0 ]; then + echo "dspark-fixture: verifier unavailable for $id: $stats" >&2 + return 1 + fi + total_proposed=$((total_proposed + proposed)) + total_accepted_draft=$((total_accepted_draft + accepted_draft)) if [ "$PROPOSAL_QUALITY_GUARD_ACTIVE" -ne 0 ] && [ "$id" = c_add ] && [ "$accepted_draft" -lt "$C_ADD_MIN_ACCEPTED" ]; then echo "dspark-fixture: c_add accepted_draft $accepted_draft below required $C_ADD_MIN_ACCEPTED: $stats" >&2 @@ -235,3 +306,8 @@ if [ "$REQUIRE_DIRECT" != 0 ] && [ "$direct_commits" -eq 0 ]; then echo "dspark-fixture: expected at least one direct verifier-state commit" >&2 exit 1 fi +if [ "$REQUIRE_ACTIVE" != 0 ] && + { [ "$total_proposed" -eq 0 ] || [ "$total_accepted_draft" -eq 0 ]; }; then + echo "dspark-fixture: DSpark runtime was not active (proposed=$total_proposed accepted_draft=$total_accepted_draft)" >&2 + exit 1 +fi diff --git a/tests/test_engine_mgpu_placement.c b/tests/test_engine_mgpu_placement.c index fa4116088..0cf861ce6 100644 --- a/tests/test_engine_mgpu_placement.c +++ b/tests/test_engine_mgpu_placement.c @@ -39,6 +39,8 @@ int ds4_test_classify_multi_tier(const ds4_test_fake_tensor *tensors, int *out_multi_tier, int *out_n_entries); int ds4_test_tensor_to_entry(const char *name, int name_len); +int ds4_test_dspark_runtime_policy(ds4_backend backend, + ds4_distributed_role distributed_role); /* Ctx-aware variants and calibration helpers. Declared here (not in * ds4.h) matching the existing DS4_TEST_HOOKS pattern. */ @@ -184,6 +186,25 @@ static void test_null_config(void) { CHECK(n_entries == 0, "NULL cfg -> n_entries 0"); } +static void test_dspark_runtime_policy(void) { + fprintf(stderr, "RUN: test_dspark_runtime_policy\n"); + CHECK(ds4_test_dspark_runtime_policy(DS4_BACKEND_METAL, + DS4_DISTRIBUTED_NONE) == 0, + "DSpark supports a local Metal graph backend"); + CHECK(ds4_test_dspark_runtime_policy(DS4_BACKEND_CUDA, + DS4_DISTRIBUTED_NONE) == 0, + "DSpark supports a local CUDA/ROCm graph backend"); + CHECK(ds4_test_dspark_runtime_policy(DS4_BACKEND_CPU, + DS4_DISTRIBUTED_NONE) != 0, + "DSpark rejects the CPU backend"); + CHECK(ds4_test_dspark_runtime_policy(DS4_BACKEND_CUDA, + DS4_DISTRIBUTED_COORDINATOR) != 0, + "DSpark rejects a distributed coordinator"); + CHECK(ds4_test_dspark_runtime_policy(DS4_BACKEND_CUDA, + DS4_DISTRIBUTED_WORKER) != 0, + "DSpark rejects a distributed worker"); +} + /* Build a synthetic, model-shaped tensor list: 1 embedding + 43 layers * (each with 2 tensors of equal size) + 1 output head. Used by the * multi-tier tests to drive a realistic placement decision. */ @@ -703,6 +724,7 @@ static void test_cuda_tp_output_head_moves_to_lower_half(void) { int main(void) { test_tensor_to_entry(); test_null_config(); + test_dspark_runtime_policy(); test_forced_two_tier_no_spill(); test_cpu_spill(); test_zero_budget_guard(); From c5af87987094ce26fa15e8de014e5fca4b2be3b4 Mon Sep 17 00:00:00 2001 From: Giorgio Oppo Date: Sat, 8 Aug 2026 21:54:38 +0200 Subject: [PATCH 011/189] Optimize DSpark inference across GPU backends Reduce Metal SSD paging and verifier cache churn, remove CUDA/ROCm mid-token fences, and add CUDA DSpark verifier/proposer fast paths with guarded fallbacks. Validation: Apple M1 Pro, Metal/CPU, no model-backed quant run for this final patch; make -B, make cpu, make ds4_test, ./ds4_test --server, 106/106 engine placement checks, ROCm C syntax check, and git diff --check passed. Target model is Flash 0731 AProjQ4. CUDA/ROCm hardware validation remains pending; Metal kernel tests were unavailable because the Codex process exposed no Metal device. --- QA_BEFORE_RELEASES.md | 15 + README.md | 93 ++++- ds4.c | 772 +++++++++++++++++++++++++++++++++++------- ds4_cuda.cu | 91 +++-- ds4_metal.m | 397 ++++++++++++++++------ 5 files changed, 1104 insertions(+), 264 deletions(-) diff --git a/QA_BEFORE_RELEASES.md b/QA_BEFORE_RELEASES.md index d7ca77268..81aa0fc46 100644 --- a/QA_BEFORE_RELEASES.md +++ b/QA_BEFORE_RELEASES.md @@ -267,6 +267,21 @@ than a failure. `--dspark-strict` remains the byte-identical target-only mode. `DS4_DSPARK_FIXTURE_CONFIDENCE=0 DS4_DSPARK_FIXTURE_TOKENS=8 DS4_DSPARK_FIXTURE_REQUIRE_PARTIAL=1 DS4_DSPARK_MODEL=/Users/antirez/ds4/gguf/DeepSeek-V4-Flash-IQ2XXS-w2Q2K-AProjQ8-SExpQ8-OutQ8-chat-v2-imatrix-0731.gguf DS4_DSPARK_SUPPORT=/Users/antirez/ds4/gguf/DeepSeek-V4-Flash-DSpark-support-0731.gguf make dspark-acceptance`. - DSpark verifier invariant smoke: `DS4_TEST_MODEL=/Users/antirez/ds4/gguf/DeepSeek-V4-Flash-IQ2XXS-w2Q2K-AProjQ8-SExpQ8-OutQ8-chat-v2-imatrix-0731.gguf DS4_DSPARK_SUPPORT=/Users/antirez/ds4/gguf/DeepSeek-V4-Flash-DSpark-support-0731.gguf make dspark-verify-depth`. +- For the experimental resident-CUDA exact-2 verifier, run the active + acceptance fixture both with and without `DS4_CUDA_DSPARK_EXACT2=1` on the + same single-GPU host. Keep SSD streaming and TP disabled, require + byte-identical stdout, `errors=0`, `verifier_unavailable=0`, and record + `verify`, `replay`, acceptance, and generation t/s from both runs. +- For CUDA HC and tiny routed-MoE kernel changes, keep + `DS4_CUDA_DSPARK_EXACT2` unset and repeat the resident acceptance fixture + with these explicit A/B pairs: HC control + `DS4_CUDA_DISABLE_HC_SPLIT_NORM_FUSED=1` versus candidate with that variable + absent; routed-MoE control `DS4_CUDA_DSPARK_TINY_ALIGNED_VEC=0` versus + candidate `=1`. Require byte-identical stdout, `errors=0`, and + `verifier_unavailable=0`; record `prop_chain`, `verify_layer`, total + proposal/verify time, acceptance, and generation t/s. Also run + `--decode-consistency 64` and the logprob-vector regression before enabling + a numerically different kernel by default. - When DSpark, support-model mapping, or SSD streaming changes, repeat both the acceptance fixture and verifier invariant on every advertised graph backend. Apply the backend and SSD options to the target-only baseline as diff --git a/README.md b/README.md index 0419507b3..38cbc36ca 100644 --- a/README.md +++ b/README.md @@ -351,9 +351,10 @@ The support file can be used with the 0731 Flash `ds4f-q2`, `ds4f-q2-q4`, and `ds4f-q4` models listed above. It is checkpoint-specific and must not be paired with an older Flash model. For now **DeepSeek V4 PRO** is not supported. On Metal, CUDA, and ROCm, the main model may be resident or -use `--ssd-streaming`; the support model remains resident and adds its own -weights and runtime state to the memory requirement. DSpark replaces the -legacy one-stage MTP support model for that run rather than stacking with it. +use `--ssd-streaming`; the support model is kept separately mapped or +device-cached and adds its own weights and runtime state to the memory +requirement. DSpark replaces the legacy one-stage MTP support model for that +run rather than stacking with it. Run it with the normal sampling defaults: @@ -393,7 +394,9 @@ decoding. On a single accelerator, the main model can instead stream its routed experts from SSD while the DSpark support model remains mapped or device-cached -separately. Select the backend with `--metal`, `--cuda`, or `--rocm`: +separately. On Metal the support mapping is file-backed and pageable; CUDA and +ROCm prepare a separate device cache. Select the backend with `--metal`, +`--cuda`, or `--rocm`: ```sh ./ds4 -m ds4flash.gguf \ @@ -405,15 +408,93 @@ separately. Select the backend with `--metal`, `--cuda`, or `--rocm`: Use `--cuda` in a CUDA build. On ROCm, use `--rocm` and a verification-safe cache, for example `--ssd-streaming-cache-experts 32`. +For memory-constrained Metal systems, use a small graph workspace as well as a +small expert cache. A practical 16 GiB starting point is: + +```sh +./ds4 -m ds4flash.gguf \ + --mtp gguf/DeepSeek-V4-Flash-DSpark-support-0731.gguf \ + --dspark --metal --ssd-streaming \ + --ssd-streaming-cache-experts 16 \ + --ctx 4096 --prefill-chunk 128 --temp 0 +``` + +When DSpark+SSD runs on a Mac with at most 24 GiB and neither +`--prefill-chunk` nor `DS4_METAL_PREFILL_CHUNK` is set, the runtime selects 128 +automatically. Set `DS4_DSPARK_LOW_MEMORY_PREFILL_CHUNK=0` to retain the normal +workspace policy, or set it to another row count. + +The Metal SSD verifier automatically limits a speculative block to the number +of complete top-k rows that fit in the effective target expert cache (two rows +with 12 effective slots and top-6 routing). Override this diagnostic policy +with `DS4_DSPARK_SSD_VERIFY_BLOCK_MAX=N`. Exact file views for the two token +embedding rows and repeatedly used Q8 support tensors are automatic; the +compatibility kill switches are +`DS4_METAL_DISABLE_TOKEN_EMBED_EXACT_VIEW=1` and +`DS4_METAL_DISABLE_SUPPORT_Q8_DECODE_EXACT_VIEWS=1`. + +The low-memory Metal SSD scheduler measures proposal plus verify/replay cost +against the target decode time. After two unprofitable attempts (cost above +1.5 times the estimated saved decode work), it backs off for 4, 8, 16, 32, +then 64 target cycles, probing once before advancing to the next backoff +level. +`DS4_DSPARK_SCHEDULER_TIMING=0` restores acceptance-only scheduling; +the existing `DS4_DSPARK_SCHEDULER=0` disables all scheduler pauses. Quality +and strict DSpark modes remain target-only. + Tune the expert-cache count for the available accelerator memory. ROCm needs enough slots for a whole verification block (30 for the 0731 model; use at least 32), and currently supports the IQ2_XXS/Q2_K or all-Q2_K routed-expert layouts. CUDA uses a transient selected-expert cache for each target block. -The DSpark support weights remain resident and are included in the startup -memory budget. This combination is single-device only; CPU, distributed or +The DSpark support weights are included in the startup memory budget even when +the Metal file-backed mapping remains pageable. This combination is +single-device only; CPU, distributed or multi-GPU placement, tensor parallelism, and legacy MTP support models remain incompatible with DSpark plus SSD streaming. +Resident single-GPU CUDA skips verifier captures that rollback/replay cannot +consume, batches frontier snapshot/restore copies behind one device fence, +computes the output head only for the final replayed token, pads the five-row +Q8 proposer head to the tensor-core shape, and fuses proposer Q RMSNorm with +RoPE. CUDA and ROCm also avoid the Metal-only mid-token submission split: on +those backends the same flush is a device-wide synchronization and only drains +the launch pipeline. The two kernel-selection kill switches for before/after +measurements are +`DS4_CUDA_DSPARK_NO_PADDED_HEAD=1` and +`DS4_CUDA_DSPARK_NO_Q_NORM_ROPE_FUSION=1`. + +CUDA fuses HC split, weighted sum, and RMSNorm across multiple batch rows, +including the DSpark proposer and verifier; use +`DS4_CUDA_DISABLE_HC_SPLIT_NORM_FUSED=1` for an A/B fallback to the separate +kernels. An experimental resident-CUDA path can run the existing aligned +IQ2_XXS/Q2_K vector MoE kernels for two-to-five-token routed batches, +preserving the established fused-SoA path as an automatic fallback: + +```sh +DS4_CUDA_DSPARK_TINY_ALIGNED_VEC=1 DS4_DSPARK_STATS=1 \ +./ds4 --cuda -m ds4flash.gguf \ + --mtp gguf/DeepSeek-V4-Flash-DSpark-support-0731.gguf \ + --dspark --temp 0 -p 'Write a Python quicksort function with comments.' +``` + +Keep the aligned tiny-batch path opt-in until the same-machine acceptance, +decode-consistency, and throughput comparisons pass on CUDA hardware. + +An experimental exact two-token resident-CUDA verifier is available for a +DGX Spark A/B test. It uses the ordinary decode kernels, commits a two-token +full accept without rollback/replay, and replays only the first token on a +partial accept. The switch also caps verification to the first two drafts: + +```sh +DS4_CUDA_DSPARK_EXACT2=1 DS4_DSPARK_STATS=1 \ +./ds4 --cuda -m ds4flash.gguf \ + --mtp gguf/DeepSeek-V4-Flash-DSpark-support-0731.gguf \ + --dspark --temp 0 -p 'Write a Python quicksort function with comments.' +``` + +Keep this path opt-in until the CUDA acceptance fixture is byte-identical and +the same-machine statistics show lower `verify` plus `replay` time. + The acceptance fixture can exercise the same SSD path on both the target-only baseline and the DSpark run. It also requires real proposals and accepted draft tokens, so an unavailable verifier cannot pass as a silent no-op: diff --git a/ds4.c b/ds4.c index fca250024..555690f79 100644 --- a/ds4.c +++ b/ds4.c @@ -16002,6 +16002,10 @@ typedef struct { uint32_t spec_prefix_n_comp[DS4_SPEC_PREFIX_SLOTS][DS4_MAX_LAYER]; uint32_t spec_prefix_n_index_comp[DS4_SPEC_PREFIX_SLOTS][DS4_MAX_LAYER]; bool spec_capture_prefixes; + /* Batch verification normally takes the per-row compressor path when it + * captures intermediate frontiers. Rollback+replay DSpark verification + * needs the same arithmetic path, but not the frontier copies themselves. */ + bool spec_force_sequential_compressor; uint32_t raw_cap; /* Maximum compressed-row capacity across layers. Shared work buffers use * this worst-case size because ratio-4 indexer layers can still reach it. */ @@ -16074,7 +16078,6 @@ typedef struct { ds4_gpu_tensor *dspark_target_hc; ds4_gpu_tensor *dspark_stage_input_hc; ds4_gpu_tensor *dspark_stage_output_hc; - ds4_gpu_tensor *dspark_position_ids; ds4_gpu_tensor *dspark_raw_cache[DS4_DSPARK_MAX_STAGES]; uint32_t dspark_cache_cap; uint32_t dspark_cache_start; @@ -16830,7 +16833,6 @@ static void metal_graph_free(ds4_gpu_graph *g) { ds4_gpu_tensor_free(g->flat_hc_by_tier[t]); ds4_gpu_tensor_free(g->cur_hc_by_tier[t]); } - ds4_gpu_tensor_free(g->dspark_position_ids); ds4_gpu_tensor_free(g->dspark_stage_output_hc); ds4_gpu_tensor_free(g->dspark_stage_input_hc); ds4_gpu_tensor_free(g->dspark_target_hc); @@ -17017,12 +17019,9 @@ static bool metal_graph_configure_dspark_capture( g->dspark_stage_output_hc = ds4_gpu_tensor_alloc((uint64_t)dw->block_size * hc_dim * sizeof(float)); - g->dspark_position_ids = - ds4_gpu_tensor_alloc((uint64_t)(dw->block_size + 1u) * - sizeof(int32_t)); if (!g->dspark_draft_tokens || !g->dspark_draft_hc || !g->dspark_target_hc || !g->dspark_stage_input_hc || - !g->dspark_stage_output_hc || !g->dspark_position_ids) { + !g->dspark_stage_output_hc) { return false; } if (dw->n_stages != 0 && g->raw_cap != 0) { @@ -27801,16 +27800,22 @@ static DS4_MAYBE_UNUSED bool metal_graph_pre_m5_q2_decode_schedule_eligible( } static uint32_t metal_graph_token_split_after_layers(void) { +#if !defined(__APPLE__) + /* Metal flushes submit the encoded prefix without waiting, allowing the GPU + * to start it while the CPU encodes the suffix. CUDA and ROCm implement + * this API as a device-wide synchronization, so splitting there only drains + * the launch pipeline in the middle of every token. */ + return 0; +#else uint32_t split_after_layers = 4; -#ifndef DS4_ROCM_BUILD const char *split_env = getenv("DS4_METAL_GRAPH_TOKEN_SPLIT_LAYERS"); if (split_env && split_env[0]) { char *end = NULL; unsigned long v = strtoul(split_env, &end, 10); if (end != split_env && v <= DS4_N_LAYER) split_after_layers = (uint32_t)v; } -#endif return split_after_layers; +#endif } static uint32_t metal_graph_token_adaptive_split_after_layers( @@ -29807,6 +29812,7 @@ static bool metal_graph_encode_layer_attention_batch( const bool aligned_chunk = getenv("DS4_CUDA_NO_COMPRESSOR_PREFILL_BATCH") == NULL && !g->spec_capture_prefixes && + !g->spec_force_sequential_compressor && (pos0 % ratio) == 0u && (n_tokens % ratio) == 0u; if (aligned_chunk) { const uint32_t comp_before = g->layer_n_comp[il]; @@ -30132,6 +30138,7 @@ static bool metal_graph_encode_layer_attention_batch( const bool aligned_chunk = getenv("DS4_CUDA_NO_COMPRESSOR_PREFILL_BATCH") == NULL && !g->spec_capture_prefixes && + !g->spec_force_sequential_compressor && (pos0 % ratio) == 0u && (n_tokens % ratio) == 0u; if (aligned_chunk) { const uint32_t index_before = g->layer_n_index_comp[il]; @@ -32596,8 +32603,7 @@ static bool dspark_stage_input_ready( dw->block_size > DS4_DSPARK_MAX_BLOCK_SIZE || g->dspark_block_size != dw->block_size || !g->dspark_main_x || !g->dspark_draft_hc || - !g->dspark_target_hc || !g->dspark_stage_input_hc || - !g->dspark_position_ids) { + !g->dspark_target_hc || !g->dspark_stage_input_hc) { return false; } if (dw->block_size == UINT32_MAX) return false; @@ -32606,9 +32612,7 @@ static bool dspark_stage_input_ready( return ds4_gpu_tensor_bytes(g->dspark_target_hc) >= hc_dim * sizeof(float) && ds4_gpu_tensor_bytes(g->dspark_stage_input_hc) >= - rows * hc_dim * sizeof(float) && - ds4_gpu_tensor_bytes(g->dspark_position_ids) >= - rows * sizeof(int32_t); + rows * hc_dim * sizeof(float); } static bool dspark_stage_cache_ready( @@ -32695,6 +32699,80 @@ static bool metal_graph_probe_dspark_noncausal_attention( return ok; } +/* DSpark setup embeds one real token followed by repeated noise tokens. The + * generic batched embedding binds the complete vocabulary table even though + * setup reads only two rows. On Metal, encode the two exact rows once and + * duplicate the noise HC row on-device; non-Apple backends retain their + * established batched embedding path. */ +static bool metal_graph_embed_dspark_draft_rows( + ds4_gpu_graph *g, + const ds4_model *base_model, + const ds4_weights *base_weights, + const ds4_dspark_weights *dw, + int token) { + if (!g || !base_model || !base_weights || !base_weights->token_embd || + !dw || !g->dspark_draft_hc || dw->block_size == 0) { + return false; + } + +#if !defined(__APPLE__) + return ds4_gpu_embed_tokens_hc_tensor( + g->dspark_draft_hc, + g->dspark_draft_tokens, + base_model->map, + base_model->size, + base_weights->token_embd->abs_offset, + (uint32_t)base_weights->token_embd->dim[1], + dw->block_size, + DS4_N_EMBD, + DS4_N_HC) != 0; +#else + const uint64_t hc_bytes = + (uint64_t)DS4_N_HC * DS4_N_EMBD * sizeof(float); + ds4_gpu_tensor *token_row = + ds4_gpu_tensor_view(g->dspark_draft_hc, 0, hc_bytes); + ds4_gpu_tensor *noise_row = dw->block_size > 1u + ? ds4_gpu_tensor_view(g->dspark_draft_hc, hc_bytes, hc_bytes) + : NULL; + bool ok = token_row != NULL && + (dw->block_size == 1u || noise_row != NULL); + const uint32_t n_vocab = + (uint32_t)base_weights->token_embd->dim[1]; + if (ok) { + ok = ds4_gpu_embed_token_hc_tensor( + token_row, + base_model->map, + base_model->size, + base_weights->token_embd->abs_offset, + n_vocab, + (uint32_t)token, + DS4_N_EMBD, + DS4_N_HC) != 0; + } + if (ok && noise_row) { + ok = ds4_gpu_embed_token_hc_tensor( + noise_row, + base_model->map, + base_model->size, + base_weights->token_embd->abs_offset, + n_vocab, + dw->noise_token_id, + DS4_N_EMBD, + DS4_N_HC) != 0; + } + for (uint32_t i = 2; ok && i < dw->block_size; i++) { + ok = ds4_gpu_tensor_copy(g->dspark_draft_hc, + (uint64_t)i * hc_bytes, + noise_row, + 0, + hc_bytes) != 0; + } + ds4_gpu_tensor_free(noise_row); + ds4_gpu_tensor_free(token_row); + return ok; +#endif +} + static bool metal_graph_prepare_dspark_setup_block( ds4_gpu_graph *g, const ds4_model *base_model, @@ -32715,11 +32793,6 @@ static bool metal_graph_prepare_dspark_setup_block( const uint64_t hc_dim = (uint64_t)DS4_N_HC * DS4_N_EMBD; const uint64_t hc_bytes = hc_dim * sizeof(float); - int32_t positions[DS4_DSPARK_MAX_BLOCK_SIZE + 1u]; - positions[0] = (int32_t)pos; - for (uint32_t i = 0; i < dw->block_size; i++) { - positions[i + 1u] = (int32_t)(pos + i); - } int32_t ids[DS4_DSPARK_MAX_BLOCK_SIZE]; ids[0] = (int32_t)token; for (uint32_t i = 1; i < dw->block_size; i++) { @@ -32729,24 +32802,13 @@ static bool metal_graph_prepare_dspark_setup_block( bool ok = ds4_gpu_tensor_write(g->dspark_draft_tokens, 0, ids, - (uint64_t)dw->block_size * sizeof(ids[0])) != 0 && - ds4_gpu_tensor_write(g->dspark_position_ids, - 0, - positions, - ((uint64_t)dw->block_size + 1u) * - sizeof(positions[0])) != 0; + (uint64_t)dw->block_size * sizeof(ids[0])) != 0; if (ok) ok = ds4_gpu_begin_commands() != 0; - if (ok) { - ok = ds4_gpu_embed_tokens_hc_tensor(g->dspark_draft_hc, - g->dspark_draft_tokens, - base_model->map, - base_model->size, - base_weights->token_embd->abs_offset, - (uint32_t)base_weights->token_embd->dim[1], - dw->block_size, - DS4_N_EMBD, - DS4_N_HC) != 0; - } + if (ok) ok = metal_graph_embed_dspark_draft_rows(g, + base_model, + base_weights, + dw, + token); if (ok) { ok = ds4_gpu_repeat_hc_tensor(g->dspark_target_hc, g->dspark_main_x, @@ -32796,11 +32858,6 @@ static bool metal_graph_prepare_dspark_stage0_setup_block( const uint64_t in_dim = (uint64_t)dw->target_layer_count * DS4_N_EMBD; const uint64_t hc_dim = (uint64_t)DS4_N_HC * DS4_N_EMBD; const uint64_t hc_bytes = hc_dim * sizeof(float); - int32_t positions[DS4_DSPARK_MAX_BLOCK_SIZE + 1u]; - positions[0] = (int32_t)pos; - for (uint32_t i = 0; i < dw->block_size; i++) { - positions[i + 1u] = (int32_t)(pos + i); - } int32_t ids[DS4_DSPARK_MAX_BLOCK_SIZE]; ids[0] = (int32_t)token; for (uint32_t i = 1; i < dw->block_size; i++) { @@ -32814,12 +32871,7 @@ static bool metal_graph_prepare_dspark_stage0_setup_block( bool ok = ds4_gpu_tensor_write(g->dspark_draft_tokens, 0, ids, - (uint64_t)dw->block_size * sizeof(ids[0])) != 0 && - ds4_gpu_tensor_write(g->dspark_position_ids, - 0, - positions, - ((uint64_t)dw->block_size + 1u) * - sizeof(positions[0])) != 0; + (uint64_t)dw->block_size * sizeof(ids[0])) != 0; const double pp_t1 = prop_profile ? now_sec() : 0.0; if (ok) ok = ds4_gpu_begin_commands() != 0; const double pp_t2 = prop_profile ? now_sec() : 0.0; @@ -32841,17 +32893,11 @@ static bool metal_graph_prepare_dspark_stage0_setup_block( DS4_N_EMBD, DS4_RMS_EPS) != 0; } - if (ok) { - ok = ds4_gpu_embed_tokens_hc_tensor(g->dspark_draft_hc, - g->dspark_draft_tokens, - base_model->map, - base_model->size, - base_weights->token_embd->abs_offset, - (uint32_t)base_weights->token_embd->dim[1], - dw->block_size, - DS4_N_EMBD, - DS4_N_HC) != 0; - } + if (ok) ok = metal_graph_embed_dspark_draft_rows(g, + base_model, + base_weights, + dw, + token); if (ok) { ok = ds4_gpu_repeat_hc_tensor(g->dspark_target_hc, g->dspark_main_x, @@ -33475,25 +33521,55 @@ static bool metal_graph_eval_dspark_stage_block( q_dim, metal_graph_batch_qr_norm(g), draft); - if (ok) ok = ds4_gpu_head_rms_norm_tensor(metal_graph_batch_q(g), - draft, - DS4_N_HEAD, - DS4_N_HEAD_DIM, - DS4_RMS_EPS) != 0; - if (ok) ok = ds4_gpu_rope_tail_tensor(metal_graph_batch_q(g), + bool q_norm_rope_fused = false; +#if !defined(__APPLE__) && !defined(DS4_ROCM_BUILD) + /* DSpark's tiny CUDA batches otherwise launch separate head-normalization + * and RoPE kernels at every support stage. The fused kernel is already + * used by the target graph and preserves the same operation order within + * each head. Keep a kill switch and the established two-kernel fallback + * so unsupported shapes never make the proposer unavailable. */ + if (ok && getenv("DS4_CUDA_DSPARK_NO_Q_NORM_ROPE_FUSION") == NULL) { + q_norm_rope_fused = ds4_gpu_head_rms_norm_rope_tail_tensor( + metal_graph_batch_q(g), + draft, + DS4_N_HEAD, + DS4_N_HEAD_DIM, + DS4_N_ROT, + pos, + 0, + false, + freq_base, + freq_scale, + ext_factor, + attn_factor, + DS4_ROPE_YARN_BETA_FAST, + DS4_ROPE_YARN_BETA_SLOW, + DS4_RMS_EPS) != 0; + } +#endif + if (ok && !q_norm_rope_fused) { + ok = ds4_gpu_head_rms_norm_tensor(metal_graph_batch_q(g), draft, DS4_N_HEAD, DS4_N_HEAD_DIM, - DS4_N_ROT, - pos, - 0, - false, - freq_base, - freq_scale, - ext_factor, - attn_factor, - DS4_ROPE_YARN_BETA_FAST, - DS4_ROPE_YARN_BETA_SLOW) != 0; + DS4_RMS_EPS) != 0; + } + if (ok && !q_norm_rope_fused) { + ok = ds4_gpu_rope_tail_tensor(metal_graph_batch_q(g), + draft, + DS4_N_HEAD, + DS4_N_HEAD_DIM, + DS4_N_ROT, + pos, + 0, + false, + freq_base, + freq_scale, + ext_factor, + attn_factor, + DS4_ROPE_YARN_BETA_FAST, + DS4_ROPE_YARN_BETA_SLOW) != 0; + } DS4_DSPARK_PROFILE_STAGE("q_path"); if (ok) ok = metal_graph_matmul_plain_tensor(metal_graph_batch_kv_raw(g), @@ -34171,13 +34247,36 @@ static bool metal_graph_eval_dspark_base_logits_from_hidden( DS4_N_VOCAB * sizeof(float)); bool ok = output_norm && logits; if (ok) ok = ds4_gpu_begin_commands() != 0; - if (ok) ok = metal_graph_matmul_plain_tensor(logits, - base_model, - base_weights->output, - DS4_N_EMBD, - DS4_N_VOCAB, - output_norm, - dw->block_size); +#if !defined(__APPLE__) && !defined(DS4_ROCM_BUILD) + /* The exact CUDA Q8 tensor-core vocab kernel starts at eight rows. DSpark + * produces five, while both backing workspaces have spare rows. Reuse the + * verifier's zero-padding helper so the five meaningful rows take the MMA + * path; the padded rows are never exposed to the proposer. */ + const bool use_cuda_padded_head = + base_weights->output->type == DS4_TENSOR_Q8_0 && + dw->block_size > 1u && dw->block_size < 8u && + getenv("DS4_CUDA_DSPARK_NO_PADDED_HEAD") == NULL; +#else + const bool use_cuda_padded_head = false; +#endif + if (ok && use_cuda_padded_head) { + ok = metal_graph_output_logits_head_matmul( + g, + base_model, + base_weights, + metal_graph_batch_ffn_norm(g), + g->spec_logits, + dw->block_size, + DS4_N_VOCAB); + } else if (ok) { + ok = metal_graph_matmul_plain_tensor(logits, + base_model, + base_weights->output, + DS4_N_EMBD, + DS4_N_VOCAB, + output_norm, + dw->block_size); + } if (ok) ok = ds4_gpu_end_commands() != 0; if (!ok) (void)ds4_gpu_synchronize(); @@ -36907,13 +37006,15 @@ static bool metal_graph_read_spec_logits_row(ds4_gpu_graph *g, uint32_t row, flo * decode kernels and cache update order, but encodes the two proposed tokens * layer-by-layer in one command stream. It returns the exact target top after * token0, and exact logits after token1. */ -static bool metal_graph_verify_decode2_exact( +static bool metal_graph_verify_decode2_exact_impl( ds4_gpu_graph *g, const ds4_model *model, const ds4_weights *weights, int token0, int token1, uint32_t start, + bool capture_prefix1, + int expected_token1, int *top0, int *top1, float *logits0, @@ -36974,7 +37075,7 @@ static bool metal_graph_verify_decode2_exact( DS4_N_EMBD, DS4_N_HC) != 0; - g->spec_capture_prefixes = true; + g->spec_capture_prefixes = capture_prefix1; if (ok) ok = ds4_gpu_begin_commands() != 0; for (uint32_t il = 0; ok && il < DS4_N_LAYER; il++) { const uint32_t pos0 = start; @@ -37070,10 +37171,12 @@ static bool metal_graph_verify_decode2_exact( } } - if (ok) { + const bool need_token1_head = + !ok || expected_token1 < 0 || *top0 == expected_token1; + if (ok && need_token1_head) { ok = metal_graph_set_active_tier_no_copy(g, cur_tier); } - if (ok) { + if (ok && need_token1_head) { const bool split_top1 = logits1 == NULL && top1 != NULL && @@ -37137,6 +37240,31 @@ static bool metal_graph_verify_decode2_exact( return ok; } +static bool metal_graph_verify_decode2_exact( + ds4_gpu_graph *g, + const ds4_model *model, + const ds4_weights *weights, + int token0, + int token1, + uint32_t start, + int *top0, + int *top1, + float *logits0, + float *logits1) { + return metal_graph_verify_decode2_exact_impl(g, + model, + weights, + token0, + token1, + start, + true, + -1, + top0, + top1, + logits0, + logits1); +} + /* Pick a raw SWA cache size for Metal. During batched prefill it must cover * the previous window plus the current ubatch. */ static uint32_t metal_graph_raw_cap_for_context(int ctx_size, uint32_t prefill_cap) { @@ -41053,7 +41181,7 @@ static double glm_graph_env_double( return v; } -static uint64_t glm_graph_host_memory_bytes(void) { +static uint64_t ds4_graph_host_memory_bytes(void) { #if defined(__APPLE__) uint64_t mem = 0; size_t len = sizeof(mem); @@ -53845,6 +53973,7 @@ struct ds4_session { uint32_t dspark_sched_accepted; uint32_t dspark_sched_no_draft; uint32_t dspark_sched_skip; + uint32_t dspark_sched_backoff_level; uint32_t dspark_sched_lifetime_accepted; double dspark_sched_life_extra_ms; double dspark_sched_life_saved_ms; @@ -53919,6 +54048,82 @@ static bool ds4_dspark_scheduler_enabled(const ds4_session *s) { return true; } +/* A tiny SSD verifier may touch top-k experts for every speculative row. On + * low-memory Metal systems the useful upper bound is the number of complete + * top-k rows that fit in the configured target expert cache. Capping only + * verification (the proposer may still produce its full block) trades a small + * amount of speculative depth for substantially fewer SSD reads. */ +static uint32_t ds4_session_dspark_verify_block_cap(const ds4_session *s) { + uint32_t cap = ds4_dspark_env_u32( + "DS4_DSPARK_SSD_VERIFY_BLOCK_MAX", 0); +#if !defined(__APPLE__) && !defined(DS4_ROCM_BUILD) + /* Exact-2 is an explicit CUDA experiment. Make the single switch usable + * with the checkpoint's native five-token proposal block by verifying its + * first two drafts; an explicit block cap still takes precedence. */ + if (cap == 0 && s && s->engine && + s->engine->backend == DS4_BACKEND_CUDA && + !s->engine->multi_tier && !s->engine->tp.active && + !s->graph.ssd_streaming && s->graph.placement == NULL && + s->graph.prefill_cap >= 2u && + metal_graph_tp_env_flag("DS4_CUDA_DSPARK_EXACT2", false)) { + cap = 2; + } +#endif + if (cap == 0 && s && s->engine && + s->engine->backend == DS4_BACKEND_METAL && + s->graph.ssd_streaming) { + const uint32_t slots = + ds4_gpu_stream_expert_cache_configured_count(); + cap = DS4_N_EXPERT_USED != 0 ? slots / DS4_N_EXPERT_USED : 0; + if (cap == 0) cap = 1; + } + if (cap == 0 || cap > DS4_DSPARK_MAX_BLOCK_SIZE) { + cap = DS4_DSPARK_MAX_BLOCK_SIZE; + } + return cap; +} + +/* Tiny-batch verification writes every future raw row before evaluating the + * block. A deliberately narrow raw ring must not wrap onto rows still in the + * visible SWA window. Default padded rings satisfy this invariant; custom + * DS4_METAL_GRAPH_RAW_CAP values may not. */ +static bool ds4_session_dspark_batch_raw_safe( + const ds4_session *s, + uint32_t draft_n) { + if (!s || draft_n <= 1u) return true; + const uint64_t start = (uint64_t)s->checkpoint.len; + const uint64_t raw_cap = s->graph.raw_cap; + const uint64_t raw_window = s->graph.raw_window; + if (raw_cap == 0) return false; + if (start + draft_n <= raw_cap) return true; + return raw_cap >= raw_window && draft_n <= raw_cap - raw_window; +} + +/* Experimental resident-CUDA verifier that advances two draft tokens with + * the ordinary one-token kernels, layer by layer. Keep the first rollout + * deliberately narrow: TP needs a mirrored exact-2 worker protocol, while + * SSD streaming needs per-layer remapping that the exact verifier does not + * perform. */ +static bool ds4_session_cuda_dspark_exact2_requested( + const ds4_session *s, + uint32_t draft_n) { +#if !defined(__APPLE__) && !defined(DS4_ROCM_BUILD) + return s && s->engine && + s->engine->backend == DS4_BACKEND_CUDA && + !s->engine->multi_tier && + !s->engine->tp.active && + !s->graph.ssd_streaming && + s->graph.placement == NULL && + s->graph.prefill_cap >= 2u && + draft_n == 2u && + metal_graph_tp_env_flag("DS4_CUDA_DSPARK_EXACT2", false); +#else + (void)s; + (void)draft_n; + return false; +#endif +} + static uint32_t ds4_dspark_scheduler_window(const ds4_session *s) { const uint32_t fallback = ds4_session_dspark_rocm_gfx1151_fast_path(s) ? 16u : 4u; @@ -53957,6 +54162,70 @@ static uint32_t ds4_dspark_scheduler_break_even_window(void) { return ds4_dspark_env_u32("DS4_DSPARK_SCHEDULER_BREAK_EVEN_WINDOW", 0); } +static bool ds4_session_dspark_low_memory_timing_default( + const ds4_session *s) { +#if defined(__APPLE__) + if (!s || !s->engine || + s->engine->backend != DS4_BACKEND_METAL || + !s->graph.ssd_streaming || + s->engine->support_kind != DS4_SUPPORT_DSPARK || + s->graph.quality || s->engine->dspark_strict) { + return false; + } + static uint64_t host_bytes; + static bool host_bytes_ready; + if (!host_bytes_ready) { + host_bytes = ds4_graph_host_memory_bytes(); + host_bytes_ready = true; + } + return host_bytes != 0 && + host_bytes <= 24ull * 1024ull * 1024ull * 1024ull; +#else + (void)s; + return false; +#endif +} + +/* Low-memory Metal SSD runs can spend several target-token equivalents on a + * single verify/replay attempt. Enable the existing measured break-even + * scheduler there by default; explicit timing thresholds still take priority. + * DS4_DSPARK_SCHEDULER_TIMING=0 restores the deterministic acceptance-only + * scheduler used on resident/high-memory backends. */ +static bool ds4_session_dspark_scheduler_timing_policy( + const ds4_session *s) { + const char *env = getenv("DS4_DSPARK_SCHEDULER_TIMING"); + if (env && env[0]) return strcmp(env, "0") != 0; + return ds4_session_dspark_low_memory_timing_default(s); +} + +static uint32_t ds4_session_dspark_scheduler_extra_saved_ratio_milli( + const ds4_session *s) { + if (getenv("DS4_DSPARK_SCHEDULER_MAX_EXTRA_SAVED_RATIO_MILLI") != NULL) { + return ds4_dspark_scheduler_max_extra_saved_ratio_milli(); + } + return ds4_session_dspark_scheduler_timing_policy(s) ? 1500u : 0u; +} + +static uint32_t ds4_session_dspark_scheduler_break_even_window( + const ds4_session *s) { + if (getenv("DS4_DSPARK_SCHEDULER_BREAK_EVEN_WINDOW") != NULL) { + return ds4_dspark_scheduler_break_even_window(); + } + if (!ds4_session_dspark_scheduler_timing_policy(s)) return 0u; + /* Measure two attempts before the first decision, then one probe is + * sufficient at each exponentially longer backoff level. */ + return s && s->dspark_sched_backoff_level != 0 ? 1u : 2u; +} + +static uint32_t ds4_session_dspark_scheduler_backoff_cycles( + const ds4_session *s) { + uint32_t skip = ds4_dspark_scheduler_slow_skip_cycles(); + uint32_t level = s ? s->dspark_sched_backoff_level : 0; + if (level > 4u) level = 4u; + if (skip > (UINT32_MAX >> level)) return UINT32_MAX; + return skip << level; +} + static uint32_t ds4_dspark_scheduler_no_draft_skip_cycles(void) { return ds4_dspark_env_u32("DS4_DSPARK_SCHEDULER_NO_DRAFT_SKIP", 3); } @@ -53978,10 +54247,11 @@ static float ds4_dspark_scheduler_cold_low_confidence_threshold(void) { } /* Timing-sensitive scheduling changes which arithmetic path advances a token. - * Keep it opt-in so greedy DSpark output is reproducible across runs. */ -static bool ds4_dspark_scheduler_timing_enabled(void) { + * It is automatic only for low-memory Metal SSD runtime decode; quality and + * strict modes remain target-only. */ +static bool ds4_dspark_scheduler_timing_enabled(const ds4_session *s) { return ds4_dspark_scheduler_max_ms_per_accept_milli() != 0 || - ds4_dspark_scheduler_max_extra_saved_ratio_milli() != 0; + ds4_session_dspark_scheduler_extra_saved_ratio_milli(s) != 0; } static void ds4_session_dspark_scheduler_reset(ds4_session *s) { @@ -54098,22 +54368,29 @@ static void ds4_session_dspark_scheduler_note( const uint32_t window = ds4_dspark_scheduler_window(s); const uint32_t break_even_window = - ds4_dspark_scheduler_break_even_window(); + ds4_session_dspark_scheduler_break_even_window(s); const uint32_t max_extra_saved_ratio_milli = - ds4_dspark_scheduler_max_extra_saved_ratio_milli(); + ds4_session_dspark_scheduler_extra_saved_ratio_milli(s); const bool measured_unprofitable = max_extra_saved_ratio_milli != 0 && - s->dspark_sched_accepted != 0 && - s->dspark_sched_saved_ms > 0.0 && - s->dspark_sched_extra_ms * 1000.0 > - s->dspark_sched_saved_ms * - (double)max_extra_saved_ratio_milli; + s->dspark_sched_extra_ms > 0.0 && + (s->dspark_sched_saved_ms <= 0.0 || + s->dspark_sched_extra_ms * 1000.0 > + s->dspark_sched_saved_ms * + (double)max_extra_saved_ratio_milli); if (break_even_window != 0 && s->dspark_sched_cycles >= break_even_window && measured_unprofitable) { - s->dspark_sched_skip = ds4_dspark_scheduler_slow_skip_cycles(); + const uint32_t backoff = + ds4_session_dspark_scheduler_backoff_cycles(s); + if (s->dspark_sched_skip < backoff) { + s->dspark_sched_skip = backoff; + } + if (s->dspark_sched_backoff_level < 4u) { + s->dspark_sched_backoff_level++; + } if (getenv("DS4_DSPARK_SPEC_LOG") != NULL) { fprintf(stderr, "ds4: DSpark scheduler break-even pause cycles=%u " @@ -54128,6 +54405,18 @@ static void ds4_session_dspark_scheduler_note( return; } + if (break_even_window != 0 && + s->dspark_sched_cycles >= break_even_window && + max_extra_saved_ratio_milli != 0 && + s->dspark_sched_saved_ms > 0.0 && + !measured_unprofitable) { + if (s->dspark_sched_backoff_level != 0) { + s->dspark_sched_backoff_level--; + } + ds4_session_dspark_scheduler_reset(s); + return; + } + if (s->dspark_sched_cycles < window) return; const uint64_t avg_milli = @@ -55698,6 +55987,30 @@ static void spec_frontier_free(ds4_spec_frontier *f) { memset(f, 0, sizeof(*f)); } +/* Frontier snapshots are a burst of independent, same-device D2D copies. + * On resident single-GPU CUDA they can share the default stream and one final + * end_commands fence; retaining the synchronous primitive everywhere else is + * important because a single final device fence cannot cover multi-tier + * streams. */ +static bool spec_frontier_tensor_copy( + ds4_session *s, + ds4_gpu_tensor *dst, + const ds4_gpu_tensor *src, + uint64_t bytes) { +#if !defined(__APPLE__) && !defined(DS4_ROCM_BUILD) + if (s && s->engine && + s->engine->backend == DS4_BACKEND_CUDA && + !s->engine->multi_tier && !s->engine->tp.active && + s->graph.placement == NULL) { + if (ds4_gpu_set_current_device(0) != 0) return false; + return ds4_gpu_tensor_copy_async(dst, src, bytes) != 0; + } +#else + (void)s; +#endif + return ds4_gpu_tensor_copy(dst, 0, src, 0, bytes) != 0; +} + static bool spec_frontier_snapshot(ds4_spec_frontier *f, ds4_session *s) { memset(f, 0, sizeof(*f)); ds4_gpu_graph *g = &s->graph; @@ -55714,17 +56027,21 @@ static bool spec_frontier_snapshot(ds4_spec_frontier *f, ds4_session *s) { const uint32_t ratio = ds4_layer_compress_ratio(il); if (ratio == 0) continue; const uint64_t ab = ds4_gpu_tensor_bytes(g->layer_attn_state_kv[il]); - ok = ds4_gpu_tensor_copy(g->spec_attn_state_kv[il], 0, - g->layer_attn_state_kv[il], 0, ab) != 0 && - ds4_gpu_tensor_copy(g->spec_attn_state_score[il], 0, - g->layer_attn_state_score[il], 0, ab) != 0; + ok = spec_frontier_tensor_copy(s, + g->spec_attn_state_kv[il], + g->layer_attn_state_kv[il], ab) && + spec_frontier_tensor_copy(s, + g->spec_attn_state_score[il], + g->layer_attn_state_score[il], ab); if (ratio == 4) { const uint64_t ib = ds4_gpu_tensor_bytes(g->layer_index_state_kv[il]); ok = ok && - ds4_gpu_tensor_copy(g->spec_index_state_kv[il], 0, - g->layer_index_state_kv[il], 0, ib) != 0 && - ds4_gpu_tensor_copy(g->spec_index_state_score[il], 0, - g->layer_index_state_score[il], 0, ib) != 0; + spec_frontier_tensor_copy(s, + g->spec_index_state_kv[il], + g->layer_index_state_kv[il], ib) && + spec_frontier_tensor_copy(s, + g->spec_index_state_score[il], + g->layer_index_state_score[il], ib); } } if (ok) ok = ds4_gpu_end_commands() != 0; @@ -55756,16 +56073,20 @@ static bool spec_frontier_restore(ds4_spec_frontier *f, ds4_session *s) { const uint32_t ratio = ds4_layer_compress_ratio(il); if (ratio == 0) continue; const uint64_t ab = ds4_gpu_tensor_bytes(g->layer_attn_state_kv[il]); - ok = ds4_gpu_tensor_copy(g->layer_attn_state_kv[il], 0, - g->spec_attn_state_kv[il], 0, ab) != 0 && - ds4_gpu_tensor_copy(g->layer_attn_state_score[il], 0, - g->spec_attn_state_score[il], 0, ab) != 0; + ok = spec_frontier_tensor_copy(s, + g->layer_attn_state_kv[il], + g->spec_attn_state_kv[il], ab) && + spec_frontier_tensor_copy(s, + g->layer_attn_state_score[il], + g->spec_attn_state_score[il], ab); if (ok && ratio == 4) { const uint64_t ib = ds4_gpu_tensor_bytes(g->layer_index_state_kv[il]); - ok = ds4_gpu_tensor_copy(g->layer_index_state_kv[il], 0, - g->spec_index_state_kv[il], 0, ib) != 0 && - ds4_gpu_tensor_copy(g->layer_index_state_score[il], 0, - g->spec_index_state_score[il], 0, ib) != 0; + ok = spec_frontier_tensor_copy(s, + g->layer_index_state_kv[il], + g->spec_index_state_kv[il], ib) && + spec_frontier_tensor_copy(s, + g->layer_index_state_score[il], + g->spec_index_state_score[il], ib); } } if (ok) ok = ds4_gpu_end_commands() != 0; @@ -62920,6 +63241,32 @@ static int ds4_engine_open_internal(ds4_engine **out, return 1; } } +#if defined(__APPLE__) && !defined(DS4_NO_GPU) + /* A 4096-row graph workspace consumes several GiB on Flash and competes + * directly with the file-backed target/support working sets. When the + * user did not request a chunk size, keep DSpark+SSD viable on small Macs + * by choosing a decode-oriented workspace. Explicit CLI/env choices win; + * set DS4_DSPARK_LOW_MEMORY_PREFILL_CHUNK=0 to disable this policy. */ + if (e->backend == DS4_BACKEND_METAL && + e->ssd_streaming && e->dspark && + e->support_kind == DS4_SUPPORT_DSPARK && + opt->prefill_chunk == 0 && + getenv("DS4_METAL_PREFILL_CHUNK") == NULL) { + const uint64_t host_bytes = ds4_graph_host_memory_bytes(); + if (host_bytes != 0 && host_bytes <= 24ull * 1024ull * 1024ull * 1024ull) { + const uint32_t low_memory_chunk = ds4_dspark_env_u32( + "DS4_DSPARK_LOW_MEMORY_PREFILL_CHUNK", 128u); + if (low_memory_chunk != 0) { + e->prefill_chunk = low_memory_chunk; + fprintf(stderr, + "ds4: Metal SSD+DSpark low-memory prefill chunk set " + "to %u (override with --prefill-chunk or " + "DS4_DSPARK_LOW_MEMORY_PREFILL_CHUNK=0)\n", + e->prefill_chunk); + } + } + } +#endif if (e->ssd_streaming && e->ssd_streaming_cache_bytes != 0) { const uint64_t requested_cache_bytes = e->ssd_streaming_cache_bytes; uint64_t non_routed_bytes = 0; @@ -67230,7 +67577,7 @@ static bool ds4_session_prepare_dspark_draft_impl(ds4_session *s, const bool scheduler_enabled = ds4_dspark_scheduler_enabled(s); const bool time_enabled = stats_enabled || - (scheduler_enabled && ds4_dspark_scheduler_timing_enabled()); + (scheduler_enabled && ds4_dspark_scheduler_timing_enabled(s)); const double stats_t0 = time_enabled ? now_sec() : 0.0; #define DS4_DSPARK_PROP_T0() (stats_enabled ? now_sec() : 0.0) #define DS4_DSPARK_PROP_ADD(field_, t0_) do { \ @@ -67949,7 +68296,7 @@ static int ds4_session_eval_internal(ds4_session *s, int token, bool probe_mtp, e->support_kind == DS4_SUPPORT_DSPARK && (ds4_dspark_stats_enabled() || (ds4_dspark_scheduler_enabled(s) && - ds4_dspark_scheduler_timing_enabled())); + ds4_dspark_scheduler_timing_enabled(s))); const double target_t0 = dspark_target_timing ? now_sec() : 0.0; if (!metal_graph_eval_token_raw_swa(&s->graph, &e->model, &e->weights, (uint32_t)token, @@ -69585,7 +69932,7 @@ static int ds4_session_eval_dspark_speculative_argmax( const bool scheduler_enabled = s && ds4_dspark_scheduler_enabled(s); const double stats_t0 = (stats_enabled || - (scheduler_enabled && ds4_dspark_scheduler_timing_enabled())) + (scheduler_enabled && ds4_dspark_scheduler_timing_enabled(s))) ? now_sec() : 0.0; #define DS4_DSPARK_STATS_FINISH() do { \ if (stats_enabled) { \ @@ -69629,6 +69976,21 @@ static int ds4_session_eval_dspark_speculative_argmax( if (draft_n > accepted_cap - n_accept) draft_n = accepted_cap - n_accept; int room = s->ctx_size - s->checkpoint.len; if (draft_n > room - 1) draft_n = room - 1; + const uint32_t verify_cap = + ds4_session_dspark_verify_block_cap(s); + if (draft_n > (int)verify_cap) draft_n = (int)verify_cap; + if (draft_n > 1 && + !ds4_session_dspark_batch_raw_safe(s, (uint32_t)draft_n)) { + if (spec_log) { + fprintf(stderr, + "ds4: DSpark verifier capped to one token for raw ring " + "safety cap=%u window=%u pos=%d\n", + s->graph.raw_cap, + s->graph.raw_window, + s->checkpoint.len); + } + draft_n = 1; + } if (draft_n <= 0) { s->dspark_draft_valid = false; s->dspark_draft_len = 0; @@ -69716,23 +70078,155 @@ static int ds4_session_eval_dspark_speculative_argmax( int *row_tops = draft_n > 1 ? row_tops_buf : NULL; float *row_logits = s->spec_row_logits; const int start = s->checkpoint.len; +#ifndef DS4_ROCM_BUILD + const bool cuda_rollback_replay = + e->backend == DS4_BACKEND_CUDA && !e->multi_tier && !e->tp.active; +#else + const bool cuda_rollback_replay = false; +#endif + /* The current target logits already verify a one-token draft. With no + * mirrored TP worker there is no speculative verifier state to roll back: + * replay that token directly and avoid copying every compressor frontier. */ + const bool skip_single_verify = + draft_n == 1 && (s->graph.ssd_streaming || cuda_rollback_replay) && + metal_graph_dspark_cache_current_window_valid(&s->graph); const double snapshot_t0 = stats_enabled ? now_sec() : 0.0; - bool have_frontier = spec_frontier_snapshot(&frontier, s); + bool have_frontier = skip_single_verify || + spec_frontier_snapshot(&frontier, s); if (stats_enabled) { s->dspark_stats.snapshot_ms += (now_sec() - snapshot_t0) * 1000.0; } bool ok = have_frontier && row_logits && (draft_n <= 1 || row_tops); bool verifier_may_have_mutated = false; bool tp_verify_sent = false; - /* The target logits used for target_top already verify the first draft. - * With an SSD-backed target, running the batch verifier again for a - * one-token suffix adds no acceptance information and would require the - * multi-row selected-expert path solely to recreate those logits. Replay - * below still evaluates the accepted token exactly and advances all KV - * and compressor state. SSD+DSpark is rejected under TP, so no mirrored - * worker verify is skipped here. */ - const bool skip_single_ssd_verify = - s->graph.ssd_streaming && draft_n == 1; + const bool cuda_exact2 = + ok && ds4_session_cuda_dspark_exact2_requested( + s, (uint32_t)draft_n); + if (cuda_exact2) { + /* sample_probs is a session-owned vocab-sized scratch row. Keeping + * row0 there leaves the pre-cycle s->logits intact if exact-2 fails + * and the legacy verifier has to take over. */ + int exact_top0 = -1; + const double exact_t0 = stats_enabled ? now_sec() : 0.0; + bool exact_ok = metal_graph_verify_decode2_exact_impl( + &s->graph, + &e->model, + &e->weights, + drafts[0], + drafts[1], + (uint32_t)start, + false, + drafts[1], + &exact_top0, + NULL, + s->sample_probs, + row_logits); + if (stats_enabled) { + s->dspark_stats.verify_ms += + (now_sec() - exact_t0) * 1000.0; + } + + const int exact_commit = + exact_ok && exact_top0 == drafts[1] ? 2 : 1; + bool exact_partial_replayed = false; + if (exact_ok && exact_commit == 1) { + /* Prefix-state capture adds dozens of D2D operations to every + * full accept. Exact-2 therefore skips it and pays a single + * headless token0 replay only on a partial accept. The already- + * computed exact row0 logits remain valid. */ + s->checkpoint.len = start; + ds4_session_dspark_capture_invalidate(s); + exact_ok = spec_frontier_restore(&frontier, s); + const double replay_t0 = stats_enabled ? now_sec() : 0.0; + if (exact_ok) { + exact_ok = metal_graph_eval_token_raw_swa(&s->graph, + &e->model, + &e->weights, + drafts[0], + (uint32_t)start, + NULL); + exact_partial_replayed = exact_ok; + } + if (stats_enabled) { + s->dspark_stats.replay_ms += + (now_sec() - replay_t0) * 1000.0; + } + } + if (exact_ok) { + const float *continuation_logits = + exact_commit == 2 ? row_logits : s->sample_probs; + memcpy(s->logits, + continuation_logits, + (size_t)DS4_N_VOCAB * sizeof(s->logits[0])); + + /* exact-2 uses private row views and therefore does not refresh + * the DSpark target-hidden capture. Never relabel the old hidden + * row as current: the ordinary first-token decode of the next + * cycle will capture the new state before running the proposer. */ + if (!exact_partial_replayed) { + ds4_session_dspark_capture_invalidate(s); + } + for (int i = 0; i < exact_commit; i++) { + token_vec_push(&s->checkpoint, drafts[i]); + accepted[n_accept++] = drafts[i]; + } + s->checkpoint_valid = true; + ds4_session_dspark_capture_note_checkpoint(s); + if (stats_enabled) { + if (exact_commit == draft_n) { + s->dspark_stats.full_accepts++; + } else { + s->dspark_stats.partial_accepts++; + } + s->dspark_stats.accepted_draft_tokens += + (uint64_t)exact_commit; + ds4_dspark_stats_note_len( + s->dspark_stats.accepted_len_hist, + (uint32_t)exact_commit); + } + ds4_session_dspark_scheduler_note( + s, + (uint32_t)exact_commit, + false, + DS4_DSPARK_SCHED_EXTRA_MS()); + if (spec_log) { + fprintf(stderr, + "ds4: DSpark CUDA exact2 drafted=2 verified_next=%d " + "accepted_draft=%d accepted_total=%d\n", + exact_top0, + exact_commit, + n_accept); + } + spec_frontier_free(&frontier); + DS4_DSPARK_STATS_FINISH(); + return n_accept; + } + + /* A backend error may happen after exact-2 has touched persistent KV + * state. Restore the pre-verify snapshot and retain the established + * batch-verify/replay path as a correctness fallback. */ + s->checkpoint.len = start; + ds4_session_dspark_capture_invalidate(s); + if (!have_frontier || !spec_frontier_restore(&frontier, s)) { + snprintf(err, errlen, "DSpark CUDA exact2 rollback failed"); + s->checkpoint_valid = false; + if (stats_enabled) { + s->dspark_stats.verifier_errors++; + ds4_dspark_stats_note_len( + s->dspark_stats.accepted_len_hist, 0); + } + spec_frontier_free(&frontier); + DS4_DSPARK_STATS_FINISH(); + return -1; + } + if (spec_log) { + fprintf(stderr, + "ds4: DSpark CUDA exact2 unavailable; falling back to " + "batch verify plus exact replay\n"); + } + } + /* Replay below still evaluates a skipped one-token draft exactly and + * advances all KV and compressor state. */ if (ok && ds4_session_tp_leader(s)) { /* Announce the block before mutating anything: the worker runs its * half of the verify and then waits for our commit decision. */ @@ -69745,24 +70239,32 @@ static int ds4_session_eval_dspark_speculative_argmax( } tp_verify_sent = true; } - if (ok && !skip_single_ssd_verify) { + if (ok && !skip_single_verify) { for (int i = 0; i < draft_n; i++) token_vec_push(&s->checkpoint, drafts[i]); verifier_may_have_mutated = true; ds4_verify_suffix_timing verify_timing; const double verify_t0 = stats_enabled ? now_sec() : 0.0; + const bool saved_force_sequential = + s->graph.spec_force_sequential_compressor; + if (cuda_rollback_replay) { + s->graph.spec_force_sequential_compressor = true; + } ok = metal_graph_verify_suffix_tops(&s->graph, &e->model, &e->weights, &s->checkpoint, (uint32_t)start, (uint32_t)draft_n, - draft_n > 1 && + !cuda_rollback_replay && + draft_n > 1 && draft_n <= (int)DS4_SPEC_PREFIX_SLOTS + 1, - true, + !cuda_rollback_replay, row_tops, NULL, stats_enabled ? &verify_timing : NULL); + s->graph.spec_force_sequential_compressor = + saved_force_sequential; if (stats_enabled) { s->dspark_stats.verify_ms += (now_sec() - verify_t0) * 1000.0; s->dspark_stats.verify_upload_ms += verify_timing.upload_ms; @@ -70011,8 +70513,8 @@ static int ds4_session_eval_dspark_speculative_argmax( return n_accept; } - /* Precompute the exact replay count (cap + eos cuts) so the worker can - * run the same gated replay evals in lockstep. */ + /* Precompute the exact replay count (capacity and EOS cuts) so a TP worker + * can mirror the same fallback decode sequence in lockstep. */ int replay_budget = commit_drafts; if (replay_budget > accepted_cap - n_accept) replay_budget = accepted_cap - n_accept; @@ -70020,6 +70522,8 @@ static int ds4_session_eval_dspark_speculative_argmax( for (int i = 0; i < replay_budget; i++) { if (drafts[i] == eos_token) { replay_budget = i + 1; break; } } + + /* Tell the worker how many exact replay evals to mirror under TP. */ if (tp_verify_sent && !ds4_tp_send_verify_commit(e->tp.ctx, 0, replay_budget)) { snprintf(err, errlen, "tp: verify commit send failed"); @@ -70033,12 +70537,18 @@ static int ds4_session_eval_dspark_speculative_argmax( s->dspark_stats.replay_fallbacks++; } for (int i = 0; i < replay_budget; i++) { + /* Only the final replayed token supplies the continuation logits. + * CUDA can therefore skip the output head and readback for every + * accepted prefix token while preserving the exact decode/KV path. */ + float *replay_logits = + !cuda_rollback_replay || i + 1 == replay_budget ? + row_logits : NULL; ok = metal_graph_eval_token_raw_swa(&s->graph, &e->model, &e->weights, drafts[i], (uint32_t)s->checkpoint.len, - row_logits); + replay_logits); if (!ok) { snprintf(err, errlen, "%s decode failed", ds4_backend_name(e->backend)); s->checkpoint_valid = false; diff --git a/ds4_cuda.cu b/ds4_cuda.cu index b93bdabe4..b14f4ad36 100644 --- a/ds4_cuda.cu +++ b/ds4_cuda.cu @@ -23893,8 +23893,12 @@ static int routed_moe_launch( if (gate_aligned && up_aligned && down_aligned) { const cudaStream_t aligned_stream = n_tokens == 1u ? cuda_decode_stream() : (cudaStream_t)0; - int rc; - if (n_tokens == 1u) { + const int dspark_tiny_aligned_vec = + n_tokens >= 2u && n_tokens <= 5u && + cuda_env_flag_enabled( + "DS4_CUDA_DSPARK_TINY_ALIGNED_VEC", 0); + int rc = -1; + if (n_tokens == 1u || dspark_tiny_aligned_vec) { rc = ds4_mmq_iq2_xxs_aligned_moe_gate_up_mid_vec( gate_aligned, up_aligned, (const float *)x->ptr, @@ -23915,7 +23919,20 @@ static int routed_moe_launch( /*n_expert_used=*/1, aligned_stream); } - } else { + if (rc == 0 && dspark_tiny_aligned_vec) { + static int logged_dspark_tiny_aligned_vec = 0; + if (!logged_dspark_tiny_aligned_vec) { + logged_dspark_tiny_aligned_vec = 1; + fprintf(stderr, + "ds4: CUDA DSpark tiny batches using " + "aligned vector MoE\n"); + } + } + } + /* A successful tiny-vector run is final. All other multi-token + * shapes retain the direct-prefill path and fused-SoA fallback. */ + if (n_tokens > 1u && + (!dspark_tiny_aligned_vec || rc != 0)) { rc = 1; const uint64_t assignments = (uint64_t)n_tokens * n_expert; @@ -25923,37 +25940,50 @@ extern "C" int ds4_gpu_hc_split_weighted_sum_norm_tensor( (uint64_t)n_embd * sizeof(float) > model_size - norm_weight_offset) { return 0; } - uint64_t n_rows = out->bytes / out_row_bytes; - if (n_rows == 1) { - if (mix->bytes < n_rows * mix_bytes || - split->bytes < n_rows * mix_bytes || - residual_hc->bytes < n_rows * residual_row_bytes) { - return 0; - } - const int logical_tier = ds4_tensor_device_idx(out); - const float *scale = (const float *)cuda_resolve_weight_ptr(model_map, scale_offset, - 3ull * sizeof(float), logical_tier, "hc_scale"); - const float *base = (const float *)cuda_resolve_weight_ptr(model_map, base_offset, - mix_bytes, logical_tier, "hc_base"); - const float *norm_w = (const float *)cuda_resolve_weight_ptr(model_map, norm_weight_offset, - (uint64_t)n_embd * sizeof(float), logical_tier, "hc_norm_weight"); - if (!scale || !base || !norm_w) return 0; - hc_split_weighted_sum_norm_fused_kernel<<<(uint32_t)n_rows, 256, 0, cuda_decode_stream()>>>( - (float *)out->ptr, - (float *)norm_out->ptr, - (float *)split->ptr, - (const float *)mix->ptr, - (const float *)residual_hc->ptr, - scale, - base, - norm_w, - n_embd, n_hc, (uint32_t)n_rows, sinkhorn_iters, eps, norm_eps); - return cuda_ok(cudaGetLastError(), "hc split weighted sum norm launch"); + const uint64_t n_rows = out->bytes / out_row_bytes; + if (n_rows > (uint64_t)INT_MAX || + n_rows > UINT64_MAX / mix_bytes || + n_rows > UINT64_MAX / residual_row_bytes) { + return 0; + } + const uint64_t mix_total_bytes = n_rows * mix_bytes; + const uint64_t residual_total_bytes = n_rows * residual_row_bytes; + if (mix->bytes < mix_total_bytes || + split->bytes < mix_total_bytes || + residual_hc->bytes < residual_total_bytes) { + return 0; } + const int logical_tier = ds4_tensor_device_idx(out); + const float *scale = (const float *)cuda_resolve_weight_ptr(model_map, scale_offset, + 3ull * sizeof(float), logical_tier, "hc_scale"); + const float *base = (const float *)cuda_resolve_weight_ptr(model_map, base_offset, + mix_bytes, logical_tier, "hc_base"); + const float *norm_w = (const float *)cuda_resolve_weight_ptr(model_map, norm_weight_offset, + (uint64_t)n_embd * sizeof(float), logical_tier, "hc_norm_weight"); + if (!scale || !base || !norm_w) return 0; + hc_split_weighted_sum_norm_fused_kernel<<<(uint32_t)n_rows, 256, 0, cuda_decode_stream()>>>( + (float *)out->ptr, + (float *)norm_out->ptr, + (float *)split->ptr, + (const float *)mix->ptr, + (const float *)residual_hc->ptr, + scale, + base, + norm_w, + n_embd, n_hc, (uint32_t)n_rows, sinkhorn_iters, eps, norm_eps); + return cuda_ok(cudaGetLastError(), "hc split weighted sum norm launch"); } /* Multi-row fallback: norm EVERY row (rms_norm_weight_tensor is the * single-row entry and would leave rows 1..n-1 of norm_out untouched). */ if (!out || n_embd == 0) return 0; + const uint64_t fallback_row_bytes = (uint64_t)n_embd * sizeof(float); + if (out->bytes < fallback_row_bytes || + out->bytes % fallback_row_bytes != 0 || + out->bytes / fallback_row_bytes > (uint64_t)INT_MAX) { + return 0; + } + const uint32_t fallback_rows = + (uint32_t)(out->bytes / fallback_row_bytes); return ds4_gpu_hc_split_weighted_sum_tensor(out, split, mix, residual_hc, model_map, model_size, scale_offset, base_offset, @@ -25962,8 +25992,7 @@ extern "C" int ds4_gpu_hc_split_weighted_sum_norm_tensor( ds4_gpu_rms_norm_weight_rows_tensor( norm_out, out, model_map, model_size, norm_weight_offset, n_embd, - (uint32_t)(out->bytes / - ((uint64_t)n_embd * sizeof(float))), + fallback_rows, norm_eps); } extern "C" int ds4_gpu_output_hc_weights_tensor( diff --git a/ds4_metal.m b/ds4_metal.m index 48fb7cf71..cbd61ddbd 100644 --- a/ds4_metal.m +++ b/ds4_metal.m @@ -371,6 +371,11 @@ static uint64_t g_model_mapped_offset; static uint64_t g_model_mapped_size; static uint64_t g_model_mapped_max_tensor_bytes; +/* Keep the support GGUF identity separate from the most recently installed + * model view. The target and DSpark mappings coexist, and g_model_map_ptr is + * intentionally updated whenever either mapping is prepared. */ +static const void *g_support_model_map_ptr; +static uint64_t g_support_model_map_size; static uint64_t g_tensor_alloc_live_bytes; static uint64_t g_tensor_alloc_peak_bytes; static pthread_mutex_t g_tensor_mu = PTHREAD_MUTEX_INITIALIZER; @@ -10496,6 +10501,8 @@ void ds4_gpu_cleanup(void) { g_model_mapped_offset = 0; g_model_mapped_size = 0; g_model_mapped_max_tensor_bytes = 0; + g_support_model_map_ptr = NULL; + g_support_model_map_size = 0; ds4_gpu_tensor_tracking_reset(); g_flash_attn_mask_bytes = 0; g_flash_attn_zero_mask_bytes = 0; @@ -11167,13 +11174,44 @@ int ds4_gpu_embed_token_hc_tensor( return 0; } + const uint64_t src_row_bytes = (uint64_t)n_embd * sizeof(uint16_t); + const uint64_t token_rel = (uint64_t)token * src_row_bytes; uint64_t inner_offset = 0; - id wbuf = - ds4_gpu_wrap_model_range(model_map, - model_size, - weight_offset, - weight_bytes, - &inner_offset); + uint32_t token_for_kernel = token; + id wbuf = nil; + const bool exact_token_row = + getenv("DS4_METAL_DISABLE_TOKEN_EMBED_EXACT_VIEW") == NULL && + (g_ssd_streaming_mode || + getenv("DS4_METAL_ENABLE_TOKEN_EMBED_EXACT_VIEW") != NULL); + if (exact_token_row) { + if (token_rel > weight_bytes || + src_row_bytes > weight_bytes - token_rel) { + fprintf(stderr, + "ds4: Metal graph embedding token row is outside the mapped table\n"); + return 0; + } + wbuf = ds4_gpu_wrap_model_exact_range(model_map, + model_size, + weight_offset + token_rel, + src_row_bytes, + &inner_offset); + if (wbuf) { + token_for_kernel = 0; + } else { + inner_offset = 0; + wbuf = ds4_gpu_wrap_model_range(model_map, + model_size, + weight_offset, + weight_bytes, + &inner_offset); + } + } else { + wbuf = ds4_gpu_wrap_model_range(model_map, + model_size, + weight_offset, + weight_bytes, + &inner_offset); + } if (!wbuf) return 0; const NSUInteger row_bytes = (NSUInteger)n_embd * sizeof(float); @@ -11188,8 +11226,7 @@ int ds4_gpu_embed_token_hc_tensor( id cb = ds4_gpu_command_buffer(&owned); if (!cb) return 0; - const int32_t token_i32 = (int32_t)token; - const uint64_t src_row_bytes = (uint64_t)n_embd * sizeof(uint16_t); + const int32_t token_i32 = (int32_t)token_for_kernel; const uint64_t dst_row_bytes = (uint64_t)n_embd * sizeof(float); ds4_gpu_get_rows_args args = { .ne00t = (int32_t)n_embd, @@ -11507,11 +11544,16 @@ int ds4_gpu_prepare_support_model(const void *model_map, uint64_t max_tensor_bytes) { /* Metal's model-view registry is keyed by mmap identity, so installing the * support GGUF preserves the independently replaceable target views. */ - return ds4_gpu_set_model_map_range(model_map, - model_size, - map_offset, - map_size, - max_tensor_bytes); + const int ok = ds4_gpu_set_model_map_range(model_map, + model_size, + map_offset, + map_size, + max_tensor_bytes); + if (ok) { + g_support_model_map_ptr = model_map; + g_support_model_map_size = model_size; + } + return ok; } /* DS4_METAL_STREAMING_EXPERT_NOCACHE: serve the streaming expert preads from @@ -11764,6 +11806,17 @@ int ds4_gpu_set_model_fd_for_map(int fd, const void *model_map) { DS4_GPU_EXACT_VIEW_OWNED); } +/* Keep the modern target-model view policy. Only DSpark's separately mapped + * support GGUF gets exact Q8 views during SSD streaming. */ +static bool ds4_gpu_support_q8_decode_exact_views_enabled( + const void *model_map, + uint64_t model_size) { + return model_map != NULL && model_size != 0 && + g_ssd_streaming_mode && + model_map == g_support_model_map_ptr && + model_size == g_support_model_map_size && + getenv("DS4_METAL_DISABLE_SUPPORT_Q8_DECODE_EXACT_VIEWS") == NULL; +} uint32_t ds4_gpu_stream_expert_cache_configured_count(void) { uint32_t budget = ds4_gpu_stream_expert_cache_configured_budget(); if (budget > DS4_METAL_STREAM_EXPERT_CACHE_MAX_ENTRIES) { @@ -16363,17 +16416,22 @@ static void ds4_gpu_stream_expert_cache_clear_layer(uint32_t layer) { /* Tiny speculative batches may route to more unique experts than the user's * persistent SSD cache can hold. Loading the entire routed tensors for that * case defeats streaming (and is especially costly for every verifier layer). - * Pack only the selected expert slabs into transient shared buffers and give - * the address-table kernels private transient tables. This avoids leaving + * Reuse matching persistent-cache entries, pack only the remaining selected + * expert slabs into transient shared buffers, and give the address-table + * kernels private transient tables. This avoids both redundant preads and * dangling GPU addresses in the persistent decode-cache tables after the - * command completes. The transient registry retains every buffer until the - * enclosing Metal command buffer completes. */ + * command completes. Cached entries are returned as explicit resources and + * marked in-flight by the encoder; the transient registry retains every miss + * buffer until the enclosing Metal command buffer completes. */ static int ds4_gpu_stream_expert_prepare_transient_selected_batch( + const void *model_map, uint64_t model_size, uint32_t layer, const int32_t *unique_ids, + const uint32_t *frequency, uint32_t unique_count, uint32_t n_total_expert, + uint32_t n_selected, uint64_t gate_offset, uint64_t up_offset, uint64_t down_offset, @@ -16384,33 +16442,115 @@ static int ds4_gpu_stream_expert_prepare_transient_selected_batch( id *down_addrs, id *packed_gate, id *packed_up, - id *packed_down) { - if (!unique_ids || unique_count == 0 || unique_count > n_total_expert || + id *packed_down, + ds4_gpu_stream_expert_cache_entry **resources, + uint32_t *n_resources) { + if (!model_map || !unique_ids || !frequency || + unique_count == 0 || unique_count > n_total_expert || !gate_addrs || !up_addrs || !down_addrs || !packed_gate || !packed_up || !packed_down || + !resources || !n_resources || + layer >= DS4_METAL_STREAM_EXPERT_CACHE_MAX_LAYER || + n_total_expert > DS4_METAL_STREAM_EXPERT_CACHE_MAX_EXPERT || + n_selected == 0 || gate_expert_bytes == 0 || down_expert_bytes == 0 || (uint64_t)unique_count > UINT64_MAX / gate_expert_bytes || (uint64_t)unique_count > UINT64_MAX / down_expert_bytes) { return 0; } - const uint64_t gate_bytes = (uint64_t)unique_count * gate_expert_bytes; - const uint64_t down_bytes = (uint64_t)unique_count * down_expert_bytes; - if (gate_bytes == 0 || down_bytes == 0 || - gate_bytes > (uint64_t)NSUIntegerMax || + *gate_addrs = nil; + *up_addrs = nil; + *down_addrs = nil; + *packed_gate = nil; + *packed_up = nil; + *packed_down = nil; + *n_resources = 0; + + uint64_t gate_abs_offsets[DS4_METAL_STREAM_EXPERT_CACHE_MAX_EXPERT]; + uint64_t up_abs_offsets[DS4_METAL_STREAM_EXPERT_CACHE_MAX_EXPERT]; + uint64_t down_abs_offsets[DS4_METAL_STREAM_EXPERT_CACHE_MAX_EXPERT]; + uint32_t miss_slots[DS4_METAL_STREAM_EXPERT_CACHE_MAX_EXPERT]; + ds4_gpu_stream_expert_cache_entry + *hit_entries[DS4_METAL_STREAM_EXPERT_CACHE_MAX_EXPERT] = { NULL }; + uint32_t miss_count = 0; + int ok = 1; + for (uint32_t u = 0; u < unique_count; u++) { + const int32_t selected_id = unique_ids[u]; + if (selected_id < 0 || (uint32_t)selected_id >= n_total_expert) { + ok = 0; + break; + } + const uint64_t expert = (uint64_t)(uint32_t)selected_id; + if (expert > UINT64_MAX / gate_expert_bytes || + expert > UINT64_MAX / down_expert_bytes) { + ok = 0; + break; + } + const uint64_t gate_rel = expert * gate_expert_bytes; + const uint64_t down_rel = expert * down_expert_bytes; + if (gate_rel > UINT64_MAX - gate_offset || + gate_rel > UINT64_MAX - up_offset || + down_rel > UINT64_MAX - down_offset) { + ok = 0; + break; + } + const uint64_t gate_abs = gate_offset + gate_rel; + const uint64_t up_abs = up_offset + gate_rel; + const uint64_t down_abs = down_offset + down_rel; + if (gate_abs > model_size || gate_expert_bytes > model_size - gate_abs || + up_abs > model_size || gate_expert_bytes > model_size - up_abs || + down_abs > model_size || down_expert_bytes > model_size - down_abs) { + ok = 0; + break; + } + gate_abs_offsets[u] = gate_abs; + up_abs_offsets[u] = up_abs; + down_abs_offsets[u] = down_abs; + + ds4_gpu_stream_expert_cache_entry *entry = + ds4_gpu_stream_expert_cache_peek(model_map, + model_size, + layer, + (uint32_t)expert, + n_total_expert, + n_selected, + gate_abs, + up_abs, + down_abs, + gate_expert_bytes, + down_expert_bytes); + hit_entries[u] = entry; + if (entry) { + miss_slots[u] = UINT32_MAX; + } else { + miss_slots[u] = miss_count++; + } + } + if (!ok) return 0; + + const uint64_t gate_bytes = (uint64_t)miss_count * gate_expert_bytes; + const uint64_t down_bytes = (uint64_t)miss_count * down_expert_bytes; + if (gate_bytes > (uint64_t)NSUIntegerMax || down_bytes > (uint64_t)NSUIntegerMax) { return 0; } - id gate_buf = - ds4_gpu_new_transient_buffer((NSUInteger)gate_bytes, - "ds4_ssd_tiny_selected_gate"); - id up_buf = - ds4_gpu_new_transient_buffer((NSUInteger)gate_bytes, - "ds4_ssd_tiny_selected_up"); - id down_buf = - ds4_gpu_new_transient_buffer((NSUInteger)down_bytes, - "ds4_ssd_tiny_selected_down"); + id gate_buf = nil; + id up_buf = nil; + id down_buf = nil; + if (miss_count != 0) { + gate_buf = + ds4_gpu_new_transient_buffer((NSUInteger)gate_bytes, + "ds4_ssd_tiny_selected_gate"); + up_buf = + ds4_gpu_new_transient_buffer((NSUInteger)gate_bytes, + "ds4_ssd_tiny_selected_up"); + down_buf = + ds4_gpu_new_transient_buffer((NSUInteger)down_bytes, + "ds4_ssd_tiny_selected_down"); + if (!gate_buf || !up_buf || !down_buf) return 0; + } const uint64_t addr_bytes64 = (uint64_t)n_total_expert * sizeof(uint64_t); if (addr_bytes64 == 0 || addr_bytes64 > (uint64_t)NSUIntegerMax) return 0; @@ -16424,18 +16564,17 @@ static int ds4_gpu_stream_expert_prepare_transient_selected_batch( id down_addr_buf = ds4_gpu_new_transient_buffer(addr_bytes, "ds4_ssd_tiny_selected_down_addrs"); - if (!gate_buf || !up_buf || !down_buf || - !gate_addr_buf || !up_addr_buf || !down_addr_buf) { + if (!gate_addr_buf || !up_addr_buf || !down_addr_buf) { return 0; } - uint8_t *gate_dst = (uint8_t *)[gate_buf contents]; - uint8_t *up_dst = (uint8_t *)[up_buf contents]; - uint8_t *down_dst = (uint8_t *)[down_buf contents]; + uint8_t *gate_dst = miss_count != 0 ? (uint8_t *)[gate_buf contents] : NULL; + uint8_t *up_dst = miss_count != 0 ? (uint8_t *)[up_buf contents] : NULL; + uint8_t *down_dst = miss_count != 0 ? (uint8_t *)[down_buf contents] : NULL; uint64_t *gate_addr = (uint64_t *)[gate_addr_buf contents]; uint64_t *up_addr = (uint64_t *)[up_addr_buf contents]; uint64_t *down_addr = (uint64_t *)[down_addr_buf contents]; - if (!gate_dst || !up_dst || !down_dst || + if ((miss_count != 0 && (!gate_dst || !up_dst || !down_dst)) || !gate_addr || !up_addr || !down_addr) { return 0; } @@ -16443,56 +16582,31 @@ static int ds4_gpu_stream_expert_prepare_transient_selected_batch( memset(up_addr, 0, addr_bytes); memset(down_addr, 0, addr_bytes); - ds4_gpu_stream_expert_pread_task *tasks = - calloc((size_t)unique_count * 3u, sizeof(tasks[0])); - if (!tasks) return 0; + ds4_gpu_stream_expert_pread_task *tasks = NULL; + if (miss_count != 0) { + tasks = calloc((size_t)miss_count * 3u, sizeof(tasks[0])); + if (!tasks) return 0; + } uint32_t n_tasks = 0; - int ok = 1; for (uint32_t u = 0; u < unique_count; u++) { - const int32_t selected_id = unique_ids[u]; - if (selected_id < 0 || (uint32_t)selected_id >= n_total_expert) { - ok = 0; - break; - } - const uint64_t expert = (uint64_t)(uint32_t)selected_id; - if (expert > UINT64_MAX / gate_expert_bytes || - expert > UINT64_MAX / down_expert_bytes) { - ok = 0; - break; - } - const uint64_t gate_rel = expert * gate_expert_bytes; - const uint64_t down_rel = expert * down_expert_bytes; - if (gate_rel > UINT64_MAX - gate_offset || - gate_rel > UINT64_MAX - up_offset || - down_rel > UINT64_MAX - down_offset) { - ok = 0; - break; - } - const uint64_t gate_abs = gate_offset + gate_rel; - const uint64_t up_abs = up_offset + gate_rel; - const uint64_t down_abs = down_offset + down_rel; - if (gate_abs > model_size || gate_expert_bytes > model_size - gate_abs || - up_abs > model_size || gate_expert_bytes > model_size - up_abs || - down_abs > model_size || down_expert_bytes > model_size - down_abs) { - ok = 0; - break; - } - - const uint64_t gate_inner = (uint64_t)u * gate_expert_bytes; - const uint64_t down_inner = (uint64_t)u * down_expert_bytes; + if (hit_entries[u]) continue; + const uint64_t gate_inner = + (uint64_t)miss_slots[u] * gate_expert_bytes; + const uint64_t down_inner = + (uint64_t)miss_slots[u] * down_expert_bytes; tasks[n_tasks++] = (ds4_gpu_stream_expert_pread_task) { - .offset = gate_abs, + .offset = gate_abs_offsets[u], .len = gate_expert_bytes, .dst = gate_dst + gate_inner, }; tasks[n_tasks++] = (ds4_gpu_stream_expert_pread_task) { - .offset = up_abs, + .offset = up_abs_offsets[u], .len = gate_expert_bytes, .dst = up_dst + gate_inner, }; tasks[n_tasks++] = (ds4_gpu_stream_expert_pread_task) { - .offset = down_abs, + .offset = down_abs_offsets[u], .len = down_expert_bytes, .dst = down_dst + down_inner, }; @@ -16509,18 +16623,80 @@ static int ds4_gpu_stream_expert_prepare_transient_selected_batch( free(tasks); if (!ok) return 0; - ds4_gpu_stream_expert_cache_note_pread(layer, read_bytes, read_ms); - [gate_buf didModifyRange:NSMakeRange(0, (NSUInteger)gate_bytes)]; - [up_buf didModifyRange:NSMakeRange(0, (NSUInteger)gate_bytes)]; - [down_buf didModifyRange:NSMakeRange(0, (NSUInteger)down_bytes)]; + if (miss_count != 0) { + ds4_gpu_stream_expert_cache_note_pread(layer, read_bytes, read_ms); + if (g_stream_expert_cache_misses > UINT64_MAX - miss_count) { + g_stream_expert_cache_misses = UINT64_MAX; + } else { + g_stream_expert_cache_misses += miss_count; + } + if (g_stream_expert_cache_layer_misses[layer] > + UINT64_MAX - miss_count) { + g_stream_expert_cache_layer_misses[layer] = UINT64_MAX; + } else { + g_stream_expert_cache_layer_misses[layer] += miss_count; + } + [gate_buf didModifyRange:NSMakeRange(0, (NSUInteger)gate_bytes)]; + [up_buf didModifyRange:NSMakeRange(0, (NSUInteger)gate_bytes)]; + [down_buf didModifyRange:NSMakeRange(0, (NSUInteger)down_bytes)]; + } for (uint32_t u = 0; u < unique_count; u++) { const uint32_t expert = (uint32_t)unique_ids[u]; - const NSUInteger gate_inner = (NSUInteger)((uint64_t)u * gate_expert_bytes); - const NSUInteger down_inner = (NSUInteger)((uint64_t)u * down_expert_bytes); - gate_addr[expert] = ds4_gpu_buffer_address(gate_buf, gate_inner); - up_addr[expert] = ds4_gpu_buffer_address(up_buf, gate_inner); - down_addr[expert] = ds4_gpu_buffer_address(down_buf, down_inner); + ds4_gpu_stream_expert_cache_entry *entry = hit_entries[u]; + if (entry) { + gate_addr[expert] = + ds4_gpu_buffer_address(entry->gate_buffer, entry->gate_inner); + up_addr[expert] = + ds4_gpu_buffer_address(entry->up_buffer, entry->up_inner); + down_addr[expert] = + ds4_gpu_buffer_address(entry->down_buffer, entry->down_inner); + const uint32_t extra_uses = + frequency[expert] > 0 ? frequency[expert] - 1u : 0; + if (extra_uses != 0) { + if (entry->use_count > UINT64_MAX - extra_uses) { + entry->use_count = UINT64_MAX; + } else { + entry->use_count += extra_uses; + } + if (g_stream_expert_cache_hits > UINT64_MAX - extra_uses) { + g_stream_expert_cache_hits = UINT64_MAX; + } else { + g_stream_expert_cache_hits += extra_uses; + } + if (g_stream_expert_cache_layer_hits[layer] > + UINT64_MAX - extra_uses) { + g_stream_expert_cache_layer_hits[layer] = UINT64_MAX; + } else { + g_stream_expert_cache_layer_hits[layer] += extra_uses; + } + } + resources[*n_resources] = entry; + (*n_resources)++; + } else { + const NSUInteger gate_inner = (NSUInteger)( + (uint64_t)miss_slots[u] * gate_expert_bytes); + const NSUInteger down_inner = (NSUInteger)( + (uint64_t)miss_slots[u] * down_expert_bytes); + gate_addr[expert] = ds4_gpu_buffer_address(gate_buf, gate_inner); + up_addr[expert] = ds4_gpu_buffer_address(up_buf, gate_inner); + down_addr[expert] = ds4_gpu_buffer_address(down_buf, down_inner); + const uint32_t extra_uses = + frequency[expert] > 0 ? frequency[expert] - 1u : 0; + if (extra_uses != 0) { + if (g_stream_expert_cache_hits > UINT64_MAX - extra_uses) { + g_stream_expert_cache_hits = UINT64_MAX; + } else { + g_stream_expert_cache_hits += extra_uses; + } + if (g_stream_expert_cache_layer_hits[layer] > + UINT64_MAX - extra_uses) { + g_stream_expert_cache_layer_hits[layer] = UINT64_MAX; + } else { + g_stream_expert_cache_layer_hits[layer] += extra_uses; + } + } + } if (gate_addr[expert] == 0 || up_addr[expert] == 0 || down_addr[expert] == 0) { return 0; @@ -16539,9 +16715,12 @@ static int ds4_gpu_stream_expert_prepare_transient_selected_batch( if (getenv("DS4_METAL_STREAMING_EXPERT_PREAD_PROFILE") != NULL) { fprintf(stderr, "ds4: Metal SSD tiny selected batch layer=%u experts=%u " + "cache_hits=%u transient_misses=%u " "bytes=%.2f MiB wall=%.3f ms\n", layer, unique_count, + *n_resources, + miss_count, ds4_gpu_mib(read_bytes), read_ms); } @@ -16652,11 +16831,14 @@ static int ds4_gpu_stream_expert_cache_prepare_selected_batch( ds4_gpu_stream_expert_cache_configured_count() < n_total_expert; if (use_transient_selected) { ok = ds4_gpu_stream_expert_prepare_transient_selected_batch( + model_map, model_size, layer, unique_ids, + frequency, unique_count, n_total_expert, + n_selected, gate_offset, up_offset, down_offset, @@ -16667,7 +16849,9 @@ static int ds4_gpu_stream_expert_cache_prepare_selected_batch( down_addrs, overflow_gate, overflow_up, - overflow_down); + overflow_down, + resources, + n_resources); free(ids); if (!ok) return 0; *unique_out = unique_count; @@ -19562,18 +19746,39 @@ static int ds4_gpu_shared_gate_up_swiglu_q8_0_impl( return 0; } + const bool exact_decode_views = + ds4_gpu_support_q8_decode_exact_views_enabled(model_map, + model_size); uint64_t gate_inner = 0; uint64_t up_inner = 0; - id gate_wbuf = ds4_gpu_wrap_model_range(model_map, - model_size, - gate_offset, - weight_bytes, - &gate_inner); - id up_wbuf = ds4_gpu_wrap_model_range(model_map, - model_size, - up_offset, - weight_bytes, - &up_inner); + id gate_wbuf = nil; + id up_wbuf = nil; + if (exact_decode_views) { + gate_wbuf = ds4_gpu_wrap_model_exact_range(model_map, + model_size, + gate_offset, + weight_bytes, + &gate_inner); + up_wbuf = ds4_gpu_wrap_model_exact_range(model_map, + model_size, + up_offset, + weight_bytes, + &up_inner); + } + if (!gate_wbuf || !up_wbuf) { + gate_inner = 0; + up_inner = 0; + gate_wbuf = ds4_gpu_wrap_model_range(model_map, + model_size, + gate_offset, + weight_bytes, + &gate_inner); + up_wbuf = ds4_gpu_wrap_model_range(model_map, + model_size, + up_offset, + weight_bytes, + &up_inner); + } if (!gate_wbuf || !up_wbuf) return 0; ds4_gpu_q8_0_matvec_args args = ds4_gpu_make_q8_0_mv_args(in_dim, out_dim); From c14523e66c500bf185b6ed401f4a3d9b237ce583 Mon Sep 17 00:00:00 2001 From: Giorgio Oppo Date: Sat, 8 Aug 2026 22:51:31 +0200 Subject: [PATCH 012/189] Remove DSpark scheduler --- QA_BEFORE_RELEASES.md | 16 +- README.md | 11 +- ds4.c | 637 ++--------------------------- tests/ds4_test.c | 4 - tests/dspark_acceptance_fixture.sh | 17 +- 5 files changed, 43 insertions(+), 642 deletions(-) diff --git a/QA_BEFORE_RELEASES.md b/QA_BEFORE_RELEASES.md index 81aa0fc46..8e30aaa5b 100644 --- a/QA_BEFORE_RELEASES.md +++ b/QA_BEFORE_RELEASES.md @@ -232,8 +232,8 @@ Use the normal Flash GGUF that 128 GB users run. ### DSpark / DeepSpec Runtime DSpark is opt-in, but it mutates the verifier, target-hidden capture, support -model loading, and scheduler paths. Run these whenever DSpark support, -speculative verification, confidence/scheduler policy, target hidden capture, +model loading, and proposal paths. Run these whenever DSpark support, +speculative verification, confidence policy, target hidden capture, tiny routed-MoE verifier kernels, or shared `--mtp-model` support-model code changes: Use the 0731 DSpark support GGUF only with a Flash 0731 target. A support model @@ -294,7 +294,7 @@ than a failure. `--dspark-strict` remains the byte-identical target-only mode. DS4_DSPARK_FIXTURE_SSD_STREAMING=1 \ DS4_DSPARK_FIXTURE_SSD_STREAMING_CACHE_EXPERTS=32 \ DS4_DSPARK_FIXTURE_CONFIDENCE=0 \ - DS4_DSPARK_SCHEDULER=0 make dspark-acceptance + make dspark-acceptance DS4_TEST_MODEL=/path/to/flash-0731.gguf \ DS4_DSPARK_SUPPORT=/path/to/DeepSeek-V4-Flash-DSpark-support-0731.gguf \ @@ -313,10 +313,10 @@ than a failure. `--dspark-strict` remains the byte-identical target-only mode. to the target-only SSD baseline. The verifier smoke must report `max_chunk>1`, `nspec>64`, and `worst_argmax_gap<=2`. - Preserve baseline and DSpark `generation` t/s from the same fixture run, - with the same host, model, cache, scheduler, thermal state, and background - load. A DSpark path that is materially slower than the target-only SSD path - without a documented correctness tradeoff is a release blocker. -- On ROCm, also run one `DS4_DSPARK_PROBE=1 DS4_DSPARK_SCHEDULER=0` + with the same host, model, cache, runtime settings, thermal state, and + background load. A DSpark path that is materially slower than the target-only + SSD path without a documented correctness tradeoff is a release blocker. +- On ROCm, also run one `DS4_DSPARK_PROBE=1` generation and require the non-causal attention and stage-chain probes to pass. This covers the HIP draft-attention kernel before the end-to-end verifier gate. @@ -327,7 +327,7 @@ than a failure. `--dspark-strict` remains the byte-identical target-only mode. `replay_fallbacks`, `errors=0`, `verify_layer`, `net_saved`, and `output_match` for both 32-token and 64-token runs. At least one direct commit must occur. A faster run with lower proposal quality is a regression unless - it was an intentional scheduler change. + it was an intentional confidence-policy change. - If verifier MoE kernels changed, run one diagnostic `c_add` profile with `DS4_DSPARK_VERIFY_SELECTED_PROFILE=1` or the Metal MoE stage profiler and record the selected-expert footprint or stage timing in the DSpark log. diff --git a/README.md b/README.md index 38cbc36ca..c22f11ae1 100644 --- a/README.md +++ b/README.md @@ -433,14 +433,9 @@ compatibility kill switches are `DS4_METAL_DISABLE_TOKEN_EMBED_EXACT_VIEW=1` and `DS4_METAL_DISABLE_SUPPORT_Q8_DECODE_EXACT_VIEWS=1`. -The low-memory Metal SSD scheduler measures proposal plus verify/replay cost -against the target decode time. After two unprofitable attempts (cost above -1.5 times the estimated saved decode work), it backs off for 4, 8, 16, 32, -then 64 target cycles, probing once before advancing to the next backoff -level. -`DS4_DSPARK_SCHEDULER_TIMING=0` restores acceptance-only scheduling; -the existing `DS4_DSPARK_SCHEDULER=0` disables all scheduler pauses. Quality -and strict DSpark modes remain target-only. +DSpark attempts a proposal on every eligible cycle. Proposal cadence is not +adaptively throttled, so the reported acceptance rate covers the full runtime +sample. Quality and strict DSpark modes remain target-only. Tune the expert-cache count for the available accelerator memory. ROCm needs enough slots for a whole verification block (30 for the 0731 model; use at diff --git a/ds4.c b/ds4.c index 555690f79..33646f74b 100644 --- a/ds4.c +++ b/ds4.c @@ -33822,106 +33822,6 @@ static bool metal_graph_eval_dspark_stage_chain( return true; } -/* Keep the support KV ring aligned while the scheduler skips proposals. */ -static bool metal_graph_dspark_ring_maintain( - ds4_gpu_graph *g, - const ds4_model *dspark_model, - const ds4_dspark_weights *dw, - uint32_t pos) { - if (!g || !dspark_model || !dw || - !g->dspark_capture_valid || - g->dspark_cache_len == 0 || - !metal_graph_dspark_cache_ends_at(g, pos) || - !dspark_stage0_weights_ready(g, dw) || - !dspark_stage_cache_ready(g, dw) || - !metal_graph_batch_kv_raw(g) || !metal_graph_batch_kv(g)) { - return false; - } - for (uint32_t stage = 0; stage < dw->n_stages; stage++) { - if (!dspark_stage_block_ready(g, dw, stage)) return false; - } - - const ds4_dspark_stage_weights *stage0 = &dw->stage[0]; - const uint64_t in_dim = (uint64_t)dw->target_layer_count * DS4_N_EMBD; - ds4_gpu_tensor *kv_raw_view = - ds4_gpu_tensor_view(metal_graph_batch_kv_raw(g), - 0, - (uint64_t)DS4_N_HEAD_DIM * sizeof(float)); - ds4_gpu_tensor *kv_view = - ds4_gpu_tensor_view(metal_graph_batch_kv(g), - 0, - (uint64_t)DS4_N_HEAD_DIM * sizeof(float)); - bool ok = kv_raw_view && kv_view && ds4_gpu_begin_commands() != 0; - if (ok) { - ok = metal_graph_matmul_plain_tensor(g->dspark_stage0_proj, - dspark_model, - stage0->main_proj, - in_dim, - DS4_N_EMBD, - g->dspark_target_hidden, - 1); - } - if (ok) { - ok = ds4_gpu_rms_norm_weight_tensor(g->dspark_main_x, - g->dspark_stage0_proj, - dspark_model->map, - dspark_model->size, - stage0->main_norm->abs_offset, - DS4_N_EMBD, - DS4_RMS_EPS) != 0; - } - for (uint32_t stage = 0; ok && stage < dw->n_stages; stage++) { - const ds4_layer_weights *block = &dw->stage[stage].block; - ok = metal_graph_matmul_plain_tensor(kv_raw_view, - dspark_model, - block->attn_kv, - DS4_N_EMBD, - DS4_N_HEAD_DIM, - g->dspark_main_x, - 1); - if (ok) ok = ds4_gpu_rms_norm_weight_rows_tensor( - kv_view, - kv_raw_view, - dspark_model->map, - dspark_model->size, - block->attn_kv_a_norm->abs_offset, - DS4_N_HEAD_DIM, - 1, - DS4_RMS_EPS) != 0; - if (ok) ok = ds4_gpu_rope_tail_tensor(kv_view, - 1, - 1, - DS4_N_HEAD_DIM, - DS4_N_ROT, - pos, - 0, - false, - DS4_ROPE_FREQ_BASE, - 1.0f, - 0.0f, - 1.0f, - DS4_ROPE_YARN_BETA_FAST, - DS4_ROPE_YARN_BETA_SLOW) != 0; - if (ok) ok = ds4_gpu_dsv4_fp8_kv_quantize_tensor(kv_view, - 1, - DS4_N_HEAD_DIM, - DS4_N_ROT) != 0; - if (ok) ok = ds4_gpu_store_raw_kv_batch_tensor( - g->dspark_raw_cache[stage], - kv_view, - g->dspark_cache_cap, - pos, - 1, - DS4_N_HEAD_DIM) != 0; - } - if (ok) ok = ds4_gpu_end_commands() != 0; - else (void)ds4_gpu_synchronize(); - ds4_gpu_tensor_free(kv_view); - ds4_gpu_tensor_free(kv_raw_view); - if (ok) (void)metal_graph_dspark_cache_claim_appended_row(g, pos); - return ok; -} - static ds4_gpu_tensor *metal_graph_dspark_final_output_hc(const ds4_gpu_graph *g) { if (!g) return NULL; if (getenv("DS4_DSPARK_DISABLE_FINAL_OUTPUT_ALIAS") == NULL && @@ -53887,8 +53787,6 @@ typedef struct ds4_dspark_spec_stats { uint64_t invalid_draft; uint64_t draft_len_hist[DS4_DSPARK_MAX_BLOCK_SIZE + 1u]; uint64_t accepted_len_hist[DS4_DSPARK_MAX_BLOCK_SIZE + 1u]; - uint64_t scheduler_skips; - uint64_t tail_skips; uint64_t verifier_unavailable; uint64_t verifier_errors; double target_ms; @@ -53969,27 +53867,11 @@ struct ds4_session { #ifndef DS4_NO_GPU int dspark_draft_tokens[DS4_DSPARK_MAX_BLOCK_SIZE]; uint32_t dspark_draft_len; - uint32_t dspark_sched_cycles; - uint32_t dspark_sched_accepted; - uint32_t dspark_sched_no_draft; - uint32_t dspark_sched_skip; - uint32_t dspark_sched_backoff_level; - uint32_t dspark_sched_lifetime_accepted; - double dspark_sched_life_extra_ms; - double dspark_sched_life_saved_ms; - double dspark_sched_extra_ms; - double dspark_sched_saved_ms; double dspark_last_target_eval_ms; - double dspark_last_propose_ms; - float dspark_last_confidence0; float dspark_sample_temperature; uint64_t *dspark_sample_rng; bool dspark_draft_valid; bool dspark_stochastic_draft; - bool dspark_sched_skipped_cycle; - bool dspark_sched_long_accept_seen; - bool dspark_sched_bypass; - bool dspark_last_confidence0_valid; ds4_dspark_spec_stats dspark_stats; #endif uint64_t mtp_probe_total; @@ -54040,14 +53922,6 @@ static bool ds4_session_dspark_seed_batch_enabled( if (env && env[0]) return env[0] != '0'; return ds4_session_dspark_rocm_gfx1151_fast_path(s); } - -static bool ds4_dspark_scheduler_enabled(const ds4_session *s) { - const char *env = getenv("DS4_DSPARK_SCHEDULER"); - if (env && env[0]) return strcmp(env, "0") != 0; - (void)s; - return true; -} - /* A tiny SSD verifier may touch top-k experts for every speculative row. On * low-memory Metal systems the useful upper bound is the number of complete * top-k rows that fit in the configured target expert cache. Capping only @@ -54124,356 +53998,15 @@ static bool ds4_session_cuda_dspark_exact2_requested( #endif } -static uint32_t ds4_dspark_scheduler_window(const ds4_session *s) { - const uint32_t fallback = - ds4_session_dspark_rocm_gfx1151_fast_path(s) ? 16u : 4u; - uint32_t v = ds4_dspark_env_u32("DS4_DSPARK_SCHEDULER_WINDOW", fallback); - return v ? v : 4; -} - -static uint32_t ds4_dspark_scheduler_skip_cycles(void) { - const uint32_t fallback = - ds4_dspark_rocm_gfx1151_fast_path() ? 4u : 2u; - return ds4_dspark_env_u32("DS4_DSPARK_SCHEDULER_SKIP", fallback); -} - -static uint32_t ds4_dspark_scheduler_slow_skip_cycles(void) { - return ds4_dspark_env_u32("DS4_DSPARK_SCHEDULER_SLOW_SKIP", 4); -} - -static uint32_t ds4_dspark_scheduler_min_avg_milli(void) { - const uint32_t fallback = - ds4_dspark_rocm_gfx1151_fast_path() ? 4000u : 1500u; - return ds4_dspark_env_u32( - "DS4_DSPARK_SCHEDULER_MIN_AVG_MILLI", fallback); -} - -static uint32_t ds4_dspark_scheduler_max_ms_per_accept_milli(void) { - return ds4_dspark_env_u32( - "DS4_DSPARK_SCHEDULER_MAX_MS_PER_ACCEPT_MILLI", 0); -} - -static uint32_t ds4_dspark_scheduler_max_extra_saved_ratio_milli(void) { - return ds4_dspark_env_u32( - "DS4_DSPARK_SCHEDULER_MAX_EXTRA_SAVED_RATIO_MILLI", 0); -} - -static uint32_t ds4_dspark_scheduler_break_even_window(void) { - return ds4_dspark_env_u32("DS4_DSPARK_SCHEDULER_BREAK_EVEN_WINDOW", 0); -} - -static bool ds4_session_dspark_low_memory_timing_default( - const ds4_session *s) { -#if defined(__APPLE__) - if (!s || !s->engine || - s->engine->backend != DS4_BACKEND_METAL || - !s->graph.ssd_streaming || - s->engine->support_kind != DS4_SUPPORT_DSPARK || - s->graph.quality || s->engine->dspark_strict) { - return false; - } - static uint64_t host_bytes; - static bool host_bytes_ready; - if (!host_bytes_ready) { - host_bytes = ds4_graph_host_memory_bytes(); - host_bytes_ready = true; - } - return host_bytes != 0 && - host_bytes <= 24ull * 1024ull * 1024ull * 1024ull; -#else - (void)s; - return false; -#endif -} - -/* Low-memory Metal SSD runs can spend several target-token equivalents on a - * single verify/replay attempt. Enable the existing measured break-even - * scheduler there by default; explicit timing thresholds still take priority. - * DS4_DSPARK_SCHEDULER_TIMING=0 restores the deterministic acceptance-only - * scheduler used on resident/high-memory backends. */ -static bool ds4_session_dspark_scheduler_timing_policy( - const ds4_session *s) { - const char *env = getenv("DS4_DSPARK_SCHEDULER_TIMING"); - if (env && env[0]) return strcmp(env, "0") != 0; - return ds4_session_dspark_low_memory_timing_default(s); -} - -static uint32_t ds4_session_dspark_scheduler_extra_saved_ratio_milli( - const ds4_session *s) { - if (getenv("DS4_DSPARK_SCHEDULER_MAX_EXTRA_SAVED_RATIO_MILLI") != NULL) { - return ds4_dspark_scheduler_max_extra_saved_ratio_milli(); - } - return ds4_session_dspark_scheduler_timing_policy(s) ? 1500u : 0u; -} - -static uint32_t ds4_session_dspark_scheduler_break_even_window( - const ds4_session *s) { - if (getenv("DS4_DSPARK_SCHEDULER_BREAK_EVEN_WINDOW") != NULL) { - return ds4_dspark_scheduler_break_even_window(); - } - if (!ds4_session_dspark_scheduler_timing_policy(s)) return 0u; - /* Measure two attempts before the first decision, then one probe is - * sufficient at each exponentially longer backoff level. */ - return s && s->dspark_sched_backoff_level != 0 ? 1u : 2u; -} - -static uint32_t ds4_session_dspark_scheduler_backoff_cycles( - const ds4_session *s) { - uint32_t skip = ds4_dspark_scheduler_slow_skip_cycles(); - uint32_t level = s ? s->dspark_sched_backoff_level : 0; - if (level > 4u) level = 4u; - if (skip > (UINT32_MAX >> level)) return UINT32_MAX; - return skip << level; -} - -static uint32_t ds4_dspark_scheduler_no_draft_skip_cycles(void) { - return ds4_dspark_env_u32("DS4_DSPARK_SCHEDULER_NO_DRAFT_SKIP", 3); -} - -static uint32_t ds4_dspark_scheduler_short_accept_no_draft_skip_cycles(void) { - return ds4_dspark_env_u32("DS4_DSPARK_SCHEDULER_SHORT_ACCEPT_NO_DRAFT_SKIP", 4); -} - -static uint32_t ds4_dspark_scheduler_cold_low_confidence_skip_cycles(void) { - return ds4_dspark_env_u32("DS4_DSPARK_SCHEDULER_COLD_LOW_CONFIDENCE_SKIP", 7); -} - -static uint32_t ds4_dspark_scheduler_tail_min_tokens(void) { - return ds4_dspark_env_u32("DS4_DSPARK_SCHEDULER_TAIL_MIN_TOKENS", 10); -} - -static float ds4_dspark_scheduler_cold_low_confidence_threshold(void) { - return (float)ds4_dspark_env_u32("DS4_DSPARK_SCHEDULER_COLD_LOW_CONFIDENCE_MILLI", 500) / 1000.0f; -} - -/* Timing-sensitive scheduling changes which arithmetic path advances a token. - * It is automatic only for low-memory Metal SSD runtime decode; quality and - * strict modes remain target-only. */ -static bool ds4_dspark_scheduler_timing_enabled(const ds4_session *s) { - return ds4_dspark_scheduler_max_ms_per_accept_milli() != 0 || - ds4_session_dspark_scheduler_extra_saved_ratio_milli(s) != 0; -} - -static void ds4_session_dspark_scheduler_reset(ds4_session *s) { - if (!s) return; - s->dspark_sched_cycles = 0; - s->dspark_sched_accepted = 0; - s->dspark_sched_no_draft = 0; - s->dspark_sched_extra_ms = 0.0; - s->dspark_sched_saved_ms = 0.0; -} - -static void ds4_session_dspark_scheduler_begin_request(ds4_session *s) { - if (!s) return; - ds4_session_dspark_scheduler_reset(s); - s->dspark_sched_skip = 0; - s->dspark_sched_lifetime_accepted = 0; - s->dspark_sched_life_extra_ms = 0.0; - s->dspark_sched_life_saved_ms = 0.0; - s->dspark_sched_skipped_cycle = false; - s->dspark_sched_long_accept_seen = false; - s->dspark_sched_bypass = false; -} - -static bool ds4_session_dspark_scheduler_should_skip(ds4_session *s) { - if (!s || !ds4_dspark_scheduler_enabled(s)) return false; - s->dspark_sched_skipped_cycle = false; - if (s->dspark_sched_bypass) { - s->dspark_sched_skipped_cycle = true; - s->dspark_stats.scheduler_skips++; - return true; - } - if (s->dspark_sched_skip == 0) return false; - s->dspark_sched_skip--; - s->dspark_sched_skipped_cycle = true; - s->dspark_stats.scheduler_skips++; - if (getenv("DS4_DSPARK_SPEC_LOG") != NULL) { - fprintf(stderr, - "ds4: DSpark scheduler skip remaining=%u\n", - s->dspark_sched_skip); - } - return true; -} - -static void ds4_session_dspark_scheduler_note( +static void ds4_session_dspark_stats_note_saved( ds4_session *s, - uint32_t accepted_drafts, - bool no_draft, - double extra_ms) { - if (!s || !ds4_dspark_scheduler_enabled(s)) return; - if (s->dspark_sched_skipped_cycle) { - s->dspark_sched_skipped_cycle = false; - return; - } - - s->dspark_sched_cycles++; - s->dspark_sched_accepted += accepted_drafts; - if (accepted_drafts != 0) { - if (s->dspark_sched_lifetime_accepted <= - UINT32_MAX - accepted_drafts) { - s->dspark_sched_lifetime_accepted += accepted_drafts; - } else { - s->dspark_sched_lifetime_accepted = UINT32_MAX; - } - if (accepted_drafts > 2u) { - s->dspark_sched_long_accept_seen = true; - } - } - if (no_draft) s->dspark_sched_no_draft++; - if (extra_ms > 0.0 && isfinite(extra_ms)) { - s->dspark_sched_extra_ms += extra_ms; - } - if (accepted_drafts != 0 && - s->dspark_last_target_eval_ms > 0.0 && + uint32_t accepted_drafts) { + if (!s || accepted_drafts == 0 || !ds4_dspark_stats_enabled()) return; + if (s->dspark_last_target_eval_ms > 0.0 && isfinite(s->dspark_last_target_eval_ms)) { - const double saved_ms = + s->dspark_stats.saved_ms += s->dspark_last_target_eval_ms * (double)accepted_drafts; - s->dspark_sched_saved_ms += saved_ms; - if (ds4_dspark_stats_enabled()) { - s->dspark_stats.saved_ms += saved_ms; - } - } - - const uint32_t no_draft_skip = - ds4_dspark_scheduler_no_draft_skip_cycles(); - if (no_draft && no_draft_skip != 0) { - uint32_t skip = no_draft_skip; - if (s->dspark_sched_lifetime_accepted != 0 && - !s->dspark_sched_long_accept_seen) { - const uint32_t short_accept_skip = - ds4_dspark_scheduler_short_accept_no_draft_skip_cycles(); - if (skip < short_accept_skip) skip = short_accept_skip; - } else if (s->dspark_sched_lifetime_accepted == 0 && - s->dspark_last_confidence0_valid && - s->dspark_last_confidence0 <= - ds4_dspark_scheduler_cold_low_confidence_threshold()) { - const uint32_t cold_low_conf_skip = - ds4_dspark_scheduler_cold_low_confidence_skip_cycles(); - if (skip < cold_low_conf_skip) skip = cold_low_conf_skip; - } - if (s->dspark_sched_skip < skip) { - s->dspark_sched_skip = skip; - } - if (getenv("DS4_DSPARK_SPEC_LOG") != NULL) { - fprintf(stderr, - "ds4: DSpark scheduler no-draft pause skip=%u " - "accepted_total=%u long_accept=%d confidence0=%s%.3f\n", - s->dspark_sched_skip, - s->dspark_sched_lifetime_accepted, - s->dspark_sched_long_accept_seen ? 1 : 0, - s->dspark_last_confidence0_valid ? "" : "n/a:", - s->dspark_last_confidence0); - } - } - - const uint32_t window = ds4_dspark_scheduler_window(s); - const uint32_t break_even_window = - ds4_session_dspark_scheduler_break_even_window(s); - - const uint32_t max_extra_saved_ratio_milli = - ds4_session_dspark_scheduler_extra_saved_ratio_milli(s); - const bool measured_unprofitable = - max_extra_saved_ratio_milli != 0 && - s->dspark_sched_extra_ms > 0.0 && - (s->dspark_sched_saved_ms <= 0.0 || - s->dspark_sched_extra_ms * 1000.0 > - s->dspark_sched_saved_ms * - (double)max_extra_saved_ratio_milli); - - if (break_even_window != 0 && - s->dspark_sched_cycles >= break_even_window && - measured_unprofitable) { - const uint32_t backoff = - ds4_session_dspark_scheduler_backoff_cycles(s); - if (s->dspark_sched_skip < backoff) { - s->dspark_sched_skip = backoff; - } - if (s->dspark_sched_backoff_level < 4u) { - s->dspark_sched_backoff_level++; - } - if (getenv("DS4_DSPARK_SPEC_LOG") != NULL) { - fprintf(stderr, - "ds4: DSpark scheduler break-even pause cycles=%u " - "accepted=%u saved=%.3fms extra=%.3fms skip=%u\n", - s->dspark_sched_cycles, - s->dspark_sched_accepted, - s->dspark_sched_saved_ms, - s->dspark_sched_extra_ms, - s->dspark_sched_skip); - } - ds4_session_dspark_scheduler_reset(s); - return; - } - - if (break_even_window != 0 && - s->dspark_sched_cycles >= break_even_window && - max_extra_saved_ratio_milli != 0 && - s->dspark_sched_saved_ms > 0.0 && - !measured_unprofitable) { - if (s->dspark_sched_backoff_level != 0) { - s->dspark_sched_backoff_level--; - } - ds4_session_dspark_scheduler_reset(s); - return; - } - - if (s->dspark_sched_cycles < window) return; - - const uint64_t avg_milli = - ((uint64_t)s->dspark_sched_accepted * 1000ull) / - (uint64_t)s->dspark_sched_cycles; - const uint32_t min_avg_milli = - ds4_dspark_scheduler_min_avg_milli(); - const bool low_accept = avg_milli < min_avg_milli; - const bool many_no_draft = - s->dspark_sched_no_draft * 2u >= s->dspark_sched_cycles; - const uint32_t max_ms_per_accept_milli = - ds4_dspark_scheduler_max_ms_per_accept_milli(); - const double extra_per_accept_ms = - s->dspark_sched_accepted != 0 ? - s->dspark_sched_extra_ms / (double)s->dspark_sched_accepted : 0.0; - const bool slow_accept = - max_ms_per_accept_milli != 0 && - s->dspark_sched_accepted != 0 && - extra_per_accept_ms * 1000.0 > (double)max_ms_per_accept_milli; - if (low_accept || many_no_draft || slow_accept || measured_unprofitable) { - if (ds4_session_dspark_rocm_gfx1151_fast_path(s)) { - s->dspark_sched_bypass = true; - s->dspark_sched_skip = 0; - if (getenv("DS4_DSPARK_SPEC_LOG") != NULL) { - fprintf(stderr, - "ds4: DSpark scheduler bypass accepted=%u avg=%.3f " - "no_draft=%u\n", - s->dspark_sched_accepted, - (double)avg_milli / 1000.0, - s->dspark_sched_no_draft); - } - ds4_session_dspark_scheduler_reset(s); - return; - } - s->dspark_sched_skip = ds4_dspark_scheduler_skip_cycles(); - if (many_no_draft || slow_accept || measured_unprofitable) { - const uint32_t slow_skip = ds4_dspark_scheduler_slow_skip_cycles(); - if (s->dspark_sched_skip < slow_skip) { - s->dspark_sched_skip = slow_skip; - } - } - if (getenv("DS4_DSPARK_SPEC_LOG") != NULL) { - fprintf(stderr, - "ds4: DSpark scheduler pause cycles=%u accepted=%u " - "avg=%.3f no_draft=%u extra_per_accept=%.3fms " - "saved=%.3fms extra=%.3fms skip=%u\n", - s->dspark_sched_cycles, - s->dspark_sched_accepted, - (double)avg_milli / 1000.0, - s->dspark_sched_no_draft, - extra_per_accept_ms, - s->dspark_sched_saved_ms, - s->dspark_sched_extra_ms, - s->dspark_sched_skip); - } } - ds4_session_dspark_scheduler_reset(s); } #endif @@ -64445,8 +63978,8 @@ static void ds4_session_print_dspark_stats(const ds4_session *s) { "accepted_draft=%llu accept_rate=%.2f%% avg_accept=%.3f " "full=%llu partial=%llu direct_full=%llu direct_partial=%llu " "replay_fallbacks=%llu miss_first=%llu no_draft=%llu " - "no_room=%llu invalid=%llu scheduler_skips=%llu " - "tail_skips=%llu verifier_unavailable=%llu errors=%llu time_ms propose=%.3f " + "no_room=%llu invalid=%llu verifier_unavailable=%llu " + "errors=%llu time_ms propose=%.3f " "prop_stage0=%.3f prop_setup=%.3f prop_cache=%.3f " "prop_chain=%.3f prop_hidden=%.3f prop_conf0=%.3f " "prop_logits=%.3f prop_markov=%.3f prop_confidence=%.3f " @@ -64470,8 +64003,6 @@ static void ds4_session_print_dspark_stats(const ds4_session *s) { (unsigned long long)st->no_draft, (unsigned long long)st->no_room, (unsigned long long)st->invalid_draft, - (unsigned long long)st->scheduler_skips, - (unsigned long long)st->tail_skips, (unsigned long long)st->verifier_unavailable, (unsigned long long)st->verifier_errors, st->propose_ms, @@ -66180,9 +65711,6 @@ int ds4_session_sync(ds4_session *s, const ds4_tokens *prompt, char *err, size_t s, s->sync_images, s->sync_image_count)) { ds4_session_invalidate(s); } -#ifndef DS4_NO_GPU - ds4_session_dspark_scheduler_begin_request(s); -#endif const bool mirror = ds4_session_tp_leader(s); if (mirror && prompt && prompt->len > 0) { if (s->sync_image_count > UINT32_MAX) { @@ -67574,11 +67102,7 @@ static bool ds4_session_prepare_dspark_draft_impl(ds4_session *s, s->engine->dspark_confidence_threshold < 0.8f ? 0.8f : s->engine->dspark_confidence_threshold; const bool stats_enabled = ds4_dspark_stats_enabled(); - const bool scheduler_enabled = ds4_dspark_scheduler_enabled(s); - const bool time_enabled = - stats_enabled || - (scheduler_enabled && ds4_dspark_scheduler_timing_enabled(s)); - const double stats_t0 = time_enabled ? now_sec() : 0.0; + const double stats_t0 = stats_enabled ? now_sec() : 0.0; #define DS4_DSPARK_PROP_T0() (stats_enabled ? now_sec() : 0.0) #define DS4_DSPARK_PROP_ADD(field_, t0_) do { \ if (stats_enabled) { \ @@ -67588,23 +67112,6 @@ static bool ds4_session_prepare_dspark_draft_impl(ds4_session *s, s->dspark_draft_valid = false; s->dspark_draft_len = 0; s->dspark_stochastic_draft = false; - s->dspark_last_confidence0 = 0.0f; - s->dspark_last_confidence0_valid = false; - if (scheduler_enabled) s->dspark_last_propose_ms = 0.0; - if (enabled && !fake_argmax_enabled && - ds4_session_dspark_scheduler_should_skip(s)) { - (void)metal_graph_dspark_ring_maintain(&s->graph, - &s->engine->mtp_model, - &s->engine->dspark_weights, - pos); - const double propose_ms = - time_enabled ? (now_sec() - stats_t0) * 1000.0 : 0.0; - if (scheduler_enabled) s->dspark_last_propose_ms = propose_ms; - if (stats_enabled) { - s->dspark_stats.propose_ms += propose_ms; - } - return false; - } if (probe_log || enabled) { const bool capture_ok = ds4_session_dspark_capture_current(s); const bool batch_capture_ok = @@ -67973,10 +67480,6 @@ static bool ds4_session_prepare_dspark_draft_impl(ds4_session *s, s->dspark_stochastic_draft = s->dspark_draft_valid && stochastic_requested; } - if (confidence_ok && confidence_len != 0) { - s->dspark_last_confidence0 = confidence0; - s->dspark_last_confidence0_valid = true; - } bool fake_argmax_ok = false; if (!s->dspark_draft_valid && fake_argmax_enabled) { s->dspark_draft_tokens[0] = sample_argmax(s->logits, DS4_N_VOCAB); @@ -68098,10 +67601,8 @@ static bool ds4_session_prepare_dspark_draft_impl(ds4_session *s, dw->metadata_errors); } } - if (time_enabled) { - const double propose_ms = (now_sec() - stats_t0) * 1000.0; - if (scheduler_enabled) s->dspark_last_propose_ms = propose_ms; - if (stats_enabled) s->dspark_stats.propose_ms += propose_ms; + if (stats_enabled) { + s->dspark_stats.propose_ms += (now_sec() - stats_t0) * 1000.0; } #undef DS4_DSPARK_PROP_ADD #undef DS4_DSPARK_PROP_T0 @@ -68294,9 +67795,7 @@ static int ds4_session_eval_internal(ds4_session *s, int token, bool probe_mtp, } const bool dspark_target_timing = e->support_kind == DS4_SUPPORT_DSPARK && - (ds4_dspark_stats_enabled() || - (ds4_dspark_scheduler_enabled(s) && - ds4_dspark_scheduler_timing_enabled(s))); + ds4_dspark_stats_enabled(); const double target_t0 = dspark_target_timing ? now_sec() : 0.0; if (!metal_graph_eval_token_raw_swa(&s->graph, &e->model, &e->weights, (uint32_t)token, @@ -68310,9 +67809,7 @@ static int ds4_session_eval_internal(ds4_session *s, int token, bool probe_mtp, if (dspark_target_timing) { const double target_ms = (now_sec() - target_t0) * 1000.0; s->dspark_last_target_eval_ms = target_ms; - if (ds4_dspark_stats_enabled()) { - s->dspark_stats.target_ms += target_ms; - } + s->dspark_stats.target_ms += target_ms; } token_vec_push(&s->checkpoint, token); s->checkpoint_valid = true; @@ -69929,19 +69426,12 @@ static int ds4_session_eval_dspark_speculative_argmax( size_t errlen) { const bool spec_log = getenv("DS4_DSPARK_SPEC_LOG") != NULL; const bool stats_enabled = s && ds4_dspark_stats_enabled(); - const bool scheduler_enabled = s && ds4_dspark_scheduler_enabled(s); - const double stats_t0 = - (stats_enabled || - (scheduler_enabled && ds4_dspark_scheduler_timing_enabled(s))) - ? now_sec() : 0.0; + const double stats_t0 = stats_enabled ? now_sec() : 0.0; #define DS4_DSPARK_STATS_FINISH() do { \ if (stats_enabled) { \ s->dspark_stats.total_ms += (now_sec() - stats_t0) * 1000.0; \ } \ } while (0) -#define DS4_DSPARK_SCHED_EXTRA_MS() \ - ((scheduler_enabled && stats_t0 != 0.0) ? \ - s->dspark_last_propose_ms + (now_sec() - stats_t0) * 1000.0 : 0.0) if (stats_enabled) { s->dspark_stats.cycles++; if (n_accept > 0) s->dspark_stats.first_tokens++; @@ -69960,10 +69450,6 @@ static int ds4_session_eval_dspark_speculative_argmax( s->dspark_stats.no_draft++; ds4_dspark_stats_note_len(s->dspark_stats.accepted_len_hist, 0); } - if (s) { - ds4_session_dspark_scheduler_note( - s, 0, true, DS4_DSPARK_SCHED_EXTRA_MS()); - } if (spec_log) { fprintf(stderr, "ds4: DSpark spec skip no-draft\n"); } @@ -70046,8 +69532,6 @@ static int ds4_session_eval_dspark_speculative_argmax( ds4_dspark_stats_note_len( s->dspark_stats.accepted_len_hist, 0); } - ds4_session_dspark_scheduler_note( - s, 0, false, DS4_DSPARK_SCHED_EXTRA_MS()); DS4_DSPARK_STATS_FINISH(); return n_accept; } @@ -70059,8 +69543,6 @@ static int ds4_session_eval_dspark_speculative_argmax( s->dspark_stats.first_misses++; ds4_dspark_stats_note_len(s->dspark_stats.accepted_len_hist, 0); } - ds4_session_dspark_scheduler_note( - s, 0, false, DS4_DSPARK_SCHED_EXTRA_MS()); if (spec_log) { fprintf(stderr, "ds4: DSpark spec miss first draft=%d base=%d\n", @@ -70184,11 +69666,8 @@ static int ds4_session_eval_dspark_speculative_argmax( s->dspark_stats.accepted_len_hist, (uint32_t)exact_commit); } - ds4_session_dspark_scheduler_note( - s, - (uint32_t)exact_commit, - false, - DS4_DSPARK_SCHED_EXTRA_MS()); + ds4_session_dspark_stats_note_saved( + s, (uint32_t)exact_commit); if (spec_log) { fprintf(stderr, "ds4: DSpark CUDA exact2 drafted=2 verified_next=%d " @@ -70330,9 +69809,8 @@ static int ds4_session_eval_dspark_speculative_argmax( ds4_dspark_stats_note_len(s->dspark_stats.accepted_len_hist, (uint32_t)emitted_drafts); } - ds4_session_dspark_scheduler_note( - s, (uint32_t)emitted_drafts, false, - DS4_DSPARK_SCHED_EXTRA_MS()); + ds4_session_dspark_stats_note_saved( + s, (uint32_t)emitted_drafts); if (spec_log) { fprintf(stderr, "ds4: DSpark spec direct-full drafted=%d accepted=%d\n", @@ -70392,9 +69870,8 @@ static int ds4_session_eval_dspark_speculative_argmax( s->dspark_stats.accepted_len_hist, (uint32_t)emitted_drafts); } - ds4_session_dspark_scheduler_note( - s, (uint32_t)emitted_drafts, false, - DS4_DSPARK_SCHED_EXTRA_MS()); + ds4_session_dspark_stats_note_saved( + s, (uint32_t)emitted_drafts); if (spec_log) { fprintf(stderr, "ds4: DSpark spec direct-partial drafted=%d committed=%d accepted=%d\n", @@ -70454,9 +69931,8 @@ static int ds4_session_eval_dspark_speculative_argmax( s->dspark_stats.accepted_len_hist, (uint32_t)emitted_drafts); } - ds4_session_dspark_scheduler_note( - s, (uint32_t)emitted_drafts, false, - DS4_DSPARK_SCHED_EXTRA_MS()); + ds4_session_dspark_stats_note_saved( + s, (uint32_t)emitted_drafts); if (spec_log) { fprintf(stderr, "ds4: DSpark spec prefix-extended drafted=%d committed=%d accepted=%d\n", @@ -70595,11 +70071,8 @@ static int ds4_session_eval_dspark_speculative_argmax( } else if (stats_enabled) { ds4_dspark_stats_note_len(s->dspark_stats.accepted_len_hist, 0); } - ds4_session_dspark_scheduler_note( - s, - (uint32_t)replayed_drafts, - false, - DS4_DSPARK_SCHED_EXTRA_MS()); + ds4_session_dspark_stats_note_saved( + s, (uint32_t)replayed_drafts); if (spec_log) { fprintf(stderr, "ds4: DSpark spec partial drafted=%d verified=%d accepted=%d\n", @@ -70609,7 +70082,6 @@ static int ds4_session_eval_dspark_speculative_argmax( } spec_frontier_free(&frontier); DS4_DSPARK_STATS_FINISH(); -#undef DS4_DSPARK_SCHED_EXTRA_MS #undef DS4_DSPARK_STATS_FINISH return n_accept; } @@ -70646,23 +70118,15 @@ static int ds4_session_eval_dspark_speculative_stochastic( uint64_t *rng, int *accepted, int accepted_cap, - char *err, - size_t errlen) { + char *err, + size_t errlen) { const bool stats_enabled = s && ds4_dspark_stats_enabled(); - const bool scheduler_enabled = s && ds4_dspark_scheduler_enabled(s); - const double stats_t0 = - (stats_enabled || - (scheduler_enabled && ds4_dspark_scheduler_timing_enabled())) - ? now_sec() : 0.0; + const double stats_t0 = stats_enabled ? now_sec() : 0.0; #define DS4_DSPARK_STOCH_FINISH() do { \ if (stats_enabled) { \ s->dspark_stats.total_ms += (now_sec() - stats_t0) * 1000.0; \ } \ } while (0) -#define DS4_DSPARK_STOCH_EXTRA_MS() \ - ((scheduler_enabled && stats_t0 != 0.0) ? \ - s->dspark_last_propose_ms + (now_sec() - stats_t0) * 1000.0 : 0.0) - if (stats_enabled) { s->dspark_stats.cycles++; if (n_accept > 0) s->dspark_stats.first_tokens++; @@ -70673,10 +70137,6 @@ static int ds4_session_eval_dspark_speculative_stochastic( s->dspark_stats.no_draft++; ds4_dspark_stats_note_len(s->dspark_stats.accepted_len_hist, 0); } - if (s) { - ds4_session_dspark_scheduler_note( - s, 0, true, DS4_DSPARK_STOCH_EXTRA_MS()); - } DS4_DSPARK_STOCH_FINISH(); return n_accept; } @@ -70749,8 +70209,6 @@ static int ds4_session_eval_dspark_speculative_stochastic( s->dspark_stats.first_misses++; ds4_dspark_stats_note_len(s->dspark_stats.accepted_len_hist, 0); } - ds4_session_dspark_scheduler_note( - s, 0, false, DS4_DSPARK_STOCH_EXTRA_MS()); DS4_DSPARK_STOCH_FINISH(); return n_accept; } @@ -70845,9 +70303,7 @@ static int ds4_session_eval_dspark_speculative_stochastic( ds4_dspark_stats_note_len(s->dspark_stats.accepted_len_hist, (uint32_t)emitted); } - ds4_session_dspark_scheduler_note( - s, (uint32_t)emitted, false, - DS4_DSPARK_STOCH_EXTRA_MS()); + ds4_session_dspark_stats_note_saved(s, (uint32_t)emitted); spec_frontier_free(&frontier); DS4_DSPARK_STOCH_FINISH(); return n_accept; @@ -70966,11 +70422,9 @@ static int ds4_session_eval_dspark_speculative_stochastic( ds4_dspark_stats_note_len(s->dspark_stats.accepted_len_hist, (uint32_t)emitted); } - ds4_session_dspark_scheduler_note( - s, (uint32_t)emitted, false, DS4_DSPARK_STOCH_EXTRA_MS()); + ds4_session_dspark_stats_note_saved(s, (uint32_t)emitted); spec_frontier_free(&frontier); DS4_DSPARK_STOCH_FINISH(); -#undef DS4_DSPARK_STOCH_EXTRA_MS #undef DS4_DSPARK_STOCH_FINISH return n_accept; } @@ -73986,34 +73440,14 @@ static int ds4_session_eval_speculative_argmax_impl( const bool strict_dspark = e->support_kind == DS4_SUPPORT_DSPARK && (e->quality || e->dspark_strict); - const bool dspark_scheduler_bypass = - e->support_kind == DS4_SUPPORT_DSPARK && - s->dspark_sched_bypass; bool can_prepare_support_draft = !strict_dspark && - !dspark_scheduler_bypass && first_token != eos_token && max_tokens > 1 && accepted_cap > 1; if (can_prepare_support_draft && e->tp.active && e->support_kind == DS4_SUPPORT_MTP_LEGACY) { can_prepare_support_draft = false; } - bool dspark_tail_skip = false; - if (can_prepare_support_draft && e->support_kind == DS4_SUPPORT_DSPARK && - ds4_dspark_scheduler_enabled(s)) { - const uint32_t tail_min = ds4_dspark_scheduler_tail_min_tokens(); - if (tail_min != 0 && (uint32_t)max_tokens < tail_min) { - can_prepare_support_draft = false; - dspark_tail_skip = true; - if (ds4_dspark_stats_enabled()) s->dspark_stats.tail_skips++; - if (getenv("DS4_DSPARK_SPEC_LOG") != NULL) { - fprintf(stderr, - "ds4: DSpark scheduler tail skip max=%d min=%u\n", - max_tokens, - tail_min); - } - } - } const bool seed_batch_dspark = can_prepare_support_draft && e->support_kind == DS4_SUPPORT_DSPARK && @@ -74050,8 +73484,6 @@ static int ds4_session_eval_speculative_argmax_impl( accepted[n_accept++] = first_token; if (first_token == eos_token || max_tokens == 1 || n_accept >= accepted_cap) return n_accept; if (strict_dspark) return n_accept; - if (dspark_scheduler_bypass) return n_accept; - if (dspark_tail_skip) return n_accept; if (e->support_kind == DS4_SUPPORT_DSPARK) { return ds4_session_eval_dspark_speculative_argmax(s, @@ -74781,18 +74213,9 @@ int ds4_session_eval_speculative(ds4_session *s, int first_token, const bool stochastic_dspark = e && e->support_kind == DS4_SUPPORT_DSPARK && e->dspark && !e->quality && !e->dspark_strict && e->dspark_exact_sampling; - bool can_prepare = stochastic_dspark && !s->dspark_sched_bypass && + bool can_prepare = stochastic_dspark && first_token != eos_token && max_tokens > 1 && accepted_cap > 1; - bool tail_skip = false; - if (can_prepare && ds4_dspark_scheduler_enabled(s)) { - const uint32_t tail_min = ds4_dspark_scheduler_tail_min_tokens(); - if (tail_min != 0 && (uint32_t)max_tokens < tail_min) { - can_prepare = false; - tail_skip = true; - if (ds4_dspark_stats_enabled()) s->dspark_stats.tail_skips++; - } - } s->dspark_sample_temperature = temperature; s->dspark_sample_rng = can_prepare ? rng : NULL; @@ -74806,7 +74229,7 @@ int ds4_session_eval_speculative(ds4_session *s, int first_token, int n_accept = 0; accepted[n_accept++] = first_token; - if (!can_prepare || tail_skip || first_token == eos_token || + if (!can_prepare || first_token == eos_token || max_tokens == 1 || n_accept >= accepted_cap) { s->dspark_sample_temperature = 0.0f; return n_accept; diff --git a/tests/ds4_test.c b/tests/ds4_test.c index 86fa3b893..4ca112a57 100644 --- a/tests/ds4_test.c +++ b/tests/ds4_test.c @@ -6731,9 +6731,6 @@ static void test_dspark_verify_depth(void) { return; } - char *saved_scheduler = test_save_env("DS4_DSPARK_SCHEDULER"); - setenv("DS4_DSPARK_SCHEDULER", "0", 1); - ds4_engine *engine = test_open_dspark_engine(support); ds4_tokens prompt = {0}; int *spec = NULL; @@ -6776,7 +6773,6 @@ static void test_dspark_verify_depth(void) { free(spec); ds4_tokens_free(&prompt); ds4_engine_close(engine); - test_restore_env("DS4_DSPARK_SCHEDULER", saved_scheduler); } #endif diff --git a/tests/dspark_acceptance_fixture.sh b/tests/dspark_acceptance_fixture.sh index 8f621baff..a82e1b452 100644 --- a/tests/dspark_acceptance_fixture.sh +++ b/tests/dspark_acceptance_fixture.sh @@ -97,11 +97,6 @@ if proposal_quality_guard_enabled; then PROPOSAL_QUALITY_GUARD_ACTIVE=1 fi -if [ "$REQUIRE_PARTIAL" != 0 ] && [ "${DS4_DSPARK_SCHEDULER_TAIL_MIN_TOKENS+x}" != x ]; then - DS4_DSPARK_SCHEDULER_TAIL_MIN_TOKENS=0 - export DS4_DSPARK_SCHEDULER_TAIL_MIN_TOKENS -fi - file_bytes() { if stat -L -f %z "$1" >/dev/null 2>&1; then stat -L -f %z "$1" @@ -129,12 +124,6 @@ print_metadata() { hw_model=$(sysctl -n hw.model 2>/dev/null || true) hw_cpu=$(sysctl -n machdep.cpu.brand_string 2>/dev/null || true) confidence=${CONFIDENCE:-default} - scheduler=${DS4_DSPARK_SCHEDULER:-default} - no_draft_skip=${DS4_DSPARK_SCHEDULER_NO_DRAFT_SKIP:-default} - short_accept_skip=${DS4_DSPARK_SCHEDULER_SHORT_ACCEPT_NO_DRAFT_SKIP:-default} - cold_low_conf_skip=${DS4_DSPARK_SCHEDULER_COLD_LOW_CONFIDENCE_SKIP:-default} - cold_low_conf_milli=${DS4_DSPARK_SCHEDULER_COLD_LOW_CONFIDENCE_MILLI:-default} - tail_min_tokens=${DS4_DSPARK_SCHEDULER_TAIL_MIN_TOKENS:-default} printf '# commit=%s\n' "$(git_commit_label)" printf '# hardware_os=%s hardware_model=%s hardware_cpu=%s\n' \ @@ -142,11 +131,9 @@ print_metadata() { printf '# model=%s model_bytes=%s support=%s support_bytes=%s\n' \ "$MODEL" "$(file_bytes "$MODEL")" \ "$SUPPORT" "$(file_bytes "$SUPPORT")" - printf '# tokens=%s ctx=default flags="--temp %s --top-p %s --min-p %s --seed %s --nothink" exact_sampling=%s confidence=%s scheduler=%s no_draft_skip=%s short_accept_no_draft_skip=%s cold_low_confidence_skip=%s cold_low_confidence_milli=%s tail_min_tokens=%s proposal_quality_guard=%s proposal_quality_active=%s c_add_min_accepted=%s require_direct=%s require_identical=%s\n' \ + printf '# tokens=%s ctx=default flags="--temp %s --top-p %s --min-p %s --seed %s --nothink" exact_sampling=%s confidence=%s proposal_quality_guard=%s proposal_quality_active=%s c_add_min_accepted=%s require_direct=%s require_identical=%s\n' \ "$TOKENS" "$TEMPERATURE" "$TOP_P" "$MIN_P" "$SEED" \ - "$EXACT_SAMPLING" "$confidence" "$scheduler" "$no_draft_skip" \ - "$short_accept_skip" "$cold_low_conf_skip" "$cold_low_conf_milli" \ - "$tail_min_tokens" "$PROPOSAL_QUALITY_GUARD" \ + "$EXACT_SAMPLING" "$confidence" "$PROPOSAL_QUALITY_GUARD" \ "$PROPOSAL_QUALITY_GUARD_ACTIVE" "$C_ADD_MIN_ACCEPTED" \ "$REQUIRE_DIRECT" "$REQUIRE_IDENTICAL" printf '# backend=%s ssd_streaming=%s ssd_cache_experts=%s require_active=%s\n' \ From b6eeda504a4b30d4eca7ce8edac2b2ce8cb2fd97 Mon Sep 17 00:00:00 2001 From: Giorgio Oppo Date: Sun, 9 Aug 2026 15:01:06 +0200 Subject: [PATCH 013/189] Optimize DSpark exact verification and Q4 GPU paths --- Makefile | 21 +- QA_BEFORE_RELEASES.md | 98 +- README.md | 142 ++- cuda/mmq/ds4_mmq.cu | 139 +++ cuda/mmq/ds4_mmq.h | 16 + ds4.c | 1757 ++++++++++++++++++++++++++-- ds4_cuda.cu | 515 +++++++- ds4_gpu.h | 72 ++ ds4_metal.m | 849 +++++++++++++- metal/dsv4_hc.metal | 2 - metal/moe.metal | 210 ++++ tests/dspark_acceptance_fixture.sh | 62 +- tests/test_metal_exactn_oracle.c | 516 ++++++++ 13 files changed, 4195 insertions(+), 204 deletions(-) create mode 100644 tests/test_metal_exactn_oracle.c diff --git a/Makefile b/Makefile index 2a73387c7..f79d15f92 100644 --- a/Makefile +++ b/Makefile @@ -68,10 +68,10 @@ DS4_LINK_LIBS ?= $(CUDA_LDLIBS) METAL_LDLIBS := $(LDLIBS) endif -.PHONY: all help clean test test-rocm test-glm53-kda-rocm test-metal-session-batch test-mxfp4-cuda test-mxfp4-rocm test-cuda-session-batch test-cuda-mixed-batch dspark-acceptance dspark-verify-depth rocm-dspark-acceptance rocm-dspark-verify-depth mtp-verify-depth cpu cuda cuda-spark cuda-generic cuda-regression strix-halo rocm +.PHONY: all help clean test test-rocm test-glm53-kda-rocm test-metal-session-batch test-metal-exactn-oracle test-mxfp4-cuda test-mxfp4-rocm test-cuda-session-batch test-cuda-mixed-batch dspark-acceptance dspark-verify-depth rocm-dspark-acceptance rocm-dspark-verify-depth mtp-verify-depth cpu cuda cuda-spark cuda-generic cuda-regression strix-halo rocm ifeq ($(UNAME_S),Darwin) -.PHONY: metal-decode-schedule-bench metal-prefill-variant-bench check-mxfp4-half-lut +.PHONY: metal-decode-schedule-bench metal-prefill-variant-bench check-mxfp4-half-lut test-mxfp4-metal all: ds4 ds4-server ds4-bench ds4-eval ds4-agent @@ -84,6 +84,7 @@ help: @echo " make metal-prefill-variant-bench Build the balanced Metal prefill variant benchmark" @echo " make check-mxfp4-half-lut Verify the checked-in MXFP4 half LUT matches the generator" @echo " make test-mxfp4-metal Check the MXFP4 half LUT, then run Metal MXFP4 exactness tests" + @echo " make test-metal-exactn-oracle Compare Metal exact-N state with sequential decode" @echo " make dspark-verify-depth Run DSpark speculative verification smoke if support GGUF is present" @echo " make mtp-verify-depth Run legacy MTP speculative verification smoke if MTP GGUF is present" @echo " make clean Remove build outputs" @@ -131,6 +132,20 @@ speed-bench/metal_prefill_variant_bench: speed-bench/metal_prefill_variant_bench metal-prefill-variant-bench: speed-bench/metal_prefill_variant_bench +ds4_metal_test_hooks.o: ds4.c ds4.h ds4_gpu.h ds4_gpu_mgpu.h ds4_image.h ds4_layer_pack.h + $(CC) $(CFLAGS) -Wno-unused-function -DDS4_TEST_HOOKS -c -o $@ ds4.c + +tests/test_metal_exactn_oracle.o: tests/test_metal_exactn_oracle.c ds4.h + $(CC) $(CFLAGS) -DDS4_TEST_HOOKS -I. -c -o $@ $< + +tests/test_metal_exactn_oracle: tests/test_metal_exactn_oracle.o ds4_metal_test_hooks.o ds4_image.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_metal.o ds4_layer_pack.o + $(CC) $(CFLAGS) -o $@ $^ $(METAL_LDLIBS) + +test-metal-exactn-oracle: tests/test_metal_exactn_oracle + DS4_TEST_REQUIRE_MODEL=1 \ + DS4_TEST_MODEL="$(DS4_TEST_MODEL)" \ + ./tests/test_metal_exactn_oracle + tests/test_mxfp4_metal.o: tests/test_mxfp4_metal.c ds4_gpu.h $(CC) $(CFLAGS) -I. -c -o $@ $< @@ -631,4 +646,4 @@ mxfp4-dot-test: tests/test_mxfp4_dot.c ./tests/test_mxfp4_dot clean: - rm -f ds4 ds4-server ds4-bench ds4-eval ds4-agent ds4_cpu ds4_native ds4_server_test ds4_test ds4_agent_test gguf-tools/quality-testing/score_official gguf-tools/quality-testing/score_official.o speed-bench/metal_decode_schedule_bench speed-bench/metal_prefill_variant_bench speed-bench/*.o tests/test_q4k_dot tests/test_mxfp4_dot tests/test_mxfp4_metal tests/test_mxfp4_rocm tests/test_mxfp4_cuda tests/test_metal_session_batch tests/test_glm53_kda tests/test_glm53_kda_rocm tests/test_glm53_vision_engine tests/test_glm53_vision_prompt tests/test_gpu_xdev tests/test_gpu_model_cache tests/test_gpu_lookup_cache_strict tests/test_engine_mgpu_refusal tests/test_engine_mgpu_runtime tests/test_engine_correctness tests/test_sampling tests/test_cuda_session_batch tests/test_cuda_mixed_batch tests/*.o *.o tests/cuda_long_context_smoke tests/cuda_long_context_smoke.o + rm -f ds4 ds4-server ds4-bench ds4-eval ds4-agent ds4_cpu ds4_native ds4_server_test ds4_test ds4_agent_test gguf-tools/quality-testing/score_official gguf-tools/quality-testing/score_official.o speed-bench/metal_decode_schedule_bench speed-bench/metal_prefill_variant_bench speed-bench/*.o tests/test_q4k_dot tests/test_mxfp4_dot tests/test_mxfp4_metal tests/test_mxfp4_rocm tests/test_mxfp4_cuda tests/test_metal_session_batch tests/test_metal_exactn_oracle tests/test_glm53_kda tests/test_glm53_kda_rocm tests/test_glm53_vision_engine tests/test_glm53_vision_prompt tests/test_gpu_xdev tests/test_gpu_model_cache tests/test_gpu_lookup_cache_strict tests/test_engine_mgpu_refusal tests/test_engine_mgpu_runtime tests/test_engine_correctness tests/test_sampling tests/test_cuda_session_batch tests/test_cuda_mixed_batch tests/*.o *.o tests/cuda_long_context_smoke tests/cuda_long_context_smoke.o diff --git a/QA_BEFORE_RELEASES.md b/QA_BEFORE_RELEASES.md index 8e30aaa5b..701fb27f2 100644 --- a/QA_BEFORE_RELEASES.md +++ b/QA_BEFORE_RELEASES.md @@ -267,11 +267,85 @@ than a failure. `--dspark-strict` remains the byte-identical target-only mode. `DS4_DSPARK_FIXTURE_CONFIDENCE=0 DS4_DSPARK_FIXTURE_TOKENS=8 DS4_DSPARK_FIXTURE_REQUIRE_PARTIAL=1 DS4_DSPARK_MODEL=/Users/antirez/ds4/gguf/DeepSeek-V4-Flash-IQ2XXS-w2Q2K-AProjQ8-SExpQ8-OutQ8-chat-v2-imatrix-0731.gguf DS4_DSPARK_SUPPORT=/Users/antirez/ds4/gguf/DeepSeek-V4-Flash-DSpark-support-0731.gguf make dspark-acceptance`. - DSpark verifier invariant smoke: `DS4_TEST_MODEL=/Users/antirez/ds4/gguf/DeepSeek-V4-Flash-IQ2XXS-w2Q2K-AProjQ8-SExpQ8-OutQ8-chat-v2-imatrix-0731.gguf DS4_DSPARK_SUPPORT=/Users/antirez/ds4/gguf/DeepSeek-V4-Flash-DSpark-support-0731.gguf make dspark-verify-depth`. -- For the experimental resident-CUDA exact-2 verifier, run the active - acceptance fixture both with and without `DS4_CUDA_DSPARK_EXACT2=1` on the - same single-GPU host. Keep SSD streaming and TP disabled, require - byte-identical stdout, `errors=0`, `verifier_unavailable=0`, and record - `verify`, `replay`, acceptance, and generation t/s from both runs. +- For Metal DSpark verifier/proposer/replay changes, run this same-machine A/B + matrix with `DS4_DSPARK_STATS=1`, greedy decoding, the same prompt and token + limit, and no other environment changes: + + | Target expert cache | Expected DSpark depth | Legacy control | Candidate | + | ---: | ---: | --- | --- | + | 16 | 2 | `DS4_METAL_DSPARK_PROPOSER_BLOCK_MAX=0 DS4_METAL_DSPARK_ACCEPTANCE_ONLY_VERIFY=0 DS4_METAL_DSPARK_HEADLESS_REPLAY=0` | Leave proposer/headless unset; keep acceptance-only `=0` | + | 32 | 5 | `DS4_DSPARK_SSD_VERIFY_BLOCK_MAX=5 DS4_METAL_DSPARK_ACCEPTANCE_ONLY_VERIFY=0 DS4_METAL_DSPARK_HEADLESS_REPLAY=0` | Keep the verifier cap, set acceptance-only `=1`, and leave proposer/headless unset | + + Use `--ssd-streaming-cache-experts 16` or `32` to match the row. The 0731 + top-6 verifier needs 30 effective slots for five draft rows; 32 leaves a small + margin. Require byte-identical stdout between control and candidate and + `errors=0`, `verifier_unavailable=0`, `proposed>0`, and + `accepted_draft>0`. Record generation t/s, acceptance, `propose`, `verify`, + `replay`, `prop_capped`, `prop_scheduled_rows`, `metal_accept_only`, + `metal_verify_rows_saved`, and `metal_replay_headless`. In the candidate, + eligible `N >= 3` verification cycles should save one target row; aligned + ratio-4 boundaries intentionally remain on the legacy path. The + depth-2 run exercises proposer capping and headless replay while retaining + the legacy verifier; the depth-5 run should retain the checkpoint's native + five proposal rows and exercise acceptance-only verification. + On low-memory Metal, repeat the depth-5 candidate once with + `DS4_METAL_DSPARK_PIN_MAIN_PROJ=1`. Require a startup log confirming the + locked byte count, identical stdout and acceptance, and compare + `prop_setup`, total `propose`, page faults, and generation t/s. A lock + failure or a slower median keeps this optimization opt-in. +- For the experimental Metal SSD exact-2 verifier, repeat the depth-2 row + above with `DS4_METAL_DSPARK_EXACT2=0` as the control and `=1` as the only + candidate change. Set `DS4_DSPARK_FIXTURE_REQUIRE_EXACT2=1` only on the + candidate. Require byte-identical stdout against both control and + target-only output, `exact2_attempt>0`, `exact2_full>0`, + `exact2_fallback=0`, and `errors=0`. Record generation t/s, `verify`, and + `replay`; then repeat for at least 100 generated tokens to catch cumulative + state drift. Do not infer that the generic five-row batch state is directly + committable from this two-row result. +- For Metal exact-union or the AProjQ4/HC decode fusions, first run the + model-backed oracle with the target AProjQ4 GGUF: + `DS4_TEST_MODEL=/path/to/deepseek-v4-flash-aprojq4.gguf make test-metal-exactn-oracle`. + Require its N=2..5 cases to be byte-identical to sequential decode for + serialized KV/compressor state, logits, and the four-token continuation. + The matrix must include full accepts for N=2,3,4,5, all N=5 partial prefixes + 1..4, and EOS in the first and a middle row. This is a correctness gate, not + evidence of a speedup. +- Then run isolated, same-machine greedy A/B pairs with identical prompt, + context, cache, token limit, and `DS4_DSPARK_STATS=1`. Change only the gate + named by the row: + + | Metal fusion | Reference control | Candidate | + | --- | --- | --- | + | HC RMSNorm + F16 mixer on M1-M4 | `DS4_METAL_DISABLE_PRE_M5_HC_NORM_MIX_FUSE=1` | Leave the disable switch unset | + | HC RMSNorm + F16 mixer on another Apple generation | Leave both HC norm/mix switches unset | `DS4_METAL_ENABLE_HC_NORM_MIX_FUSE=1` | + | Q4 Q-A/KV + compressor store in exact-union | `DS4_METAL_DSPARK_EXACTN_UNION=1` with the Q4 enable switch unset | Keep exact-union `=1`; set `DS4_METAL_ENABLE_Q4_QKV_COMPRESSOR_FUSE=1` | + | Q4 Q-A/KV + compressor store in ordinary `FULL` decode | Leave `DS4_METAL_ENABLE_Q4_QKV_COMPRESSOR_FUSE` unset | Set `DS4_METAL_ENABLE_Q4_QKV_COMPRESSOR_FUSE=1` | + | Q4 attention-output B + HC expansion | `DS4_METAL_DISABLE_Q4_ATTN_OUT_HC_FUSE=1` | Leave the disable switch unset | + + The Q4 Q-A/KV compound is opt-in in both exact-union and ordinary `FULL` + decode; it is enabled only when the explicit enable variable is present. + Require byte-identical stdout and `errors=0`; for exact-union also + require `exactn_union_attempt>0` and `exactn_union_error_fallback=0`. + Partial-accept fallback is expected when the draft diverges. Record + `exactn_union_full`, `exactn_union_partial_fallback`, `propose`, `verify`, + `replay`, stage timings, page faults, and generation t/s. A candidate that + is correct but slower remains disabled or opt-in according to its gate. +- For the experimental resident-CUDA exact-2 verifier, use three controlled + runs with verifier cap two on the same single-GPU host: native proposer plus + legacy verifier + (`DS4_CUDA_DSPARK_EXACT2=0 DS4_CUDA_DSPARK_PROPOSER_BLOCK_MAX=0 DS4_DSPARK_SSD_VERIFY_BLOCK_MAX=2`), + two-row proposer plus legacy verifier + (`DS4_CUDA_DSPARK_EXACT2=0 DS4_CUDA_DSPARK_PROPOSER_BLOCK_MAX=2 DS4_DSPARK_SSD_VERIFY_BLOCK_MAX=2`), + and two-row proposer plus exact-2 + (`DS4_CUDA_DSPARK_EXACT2=1 DS4_CUDA_DSPARK_PROPOSER_BLOCK_MAX=2 DS4_DSPARK_SSD_VERIFY_BLOCK_MAX=2`). + This separates the non-causal proposer-width change from the verifier and + replay change. Then compare uncapped legacy DSpark against exact-2 as an + end-to-end policy test. Keep SSD streaming and TP disabled. Require + byte-identical stdout, `errors=0`, and `verifier_unavailable=0` from every + run; set `DS4_DSPARK_FIXTURE_REQUIRE_EXACT2=1` on the exact-2 run so the + fixture enforces `exact2_attempt>0` and `exact2_fallback=0`. + Record `prop_scheduled_rows/cycles`, `propose`, `verify`, `replay`, `net_saved`, + `miss_first`, `no_draft`, `avg_accept`, and generation t/s from every run. - For CUDA HC and tiny routed-MoE kernel changes, keep `DS4_CUDA_DSPARK_EXACT2` unset and repeat the resident acceptance fixture with these explicit A/B pairs: HC control @@ -282,6 +356,20 @@ than a failure. `--dspark-strict` remains the byte-identical target-only mode. proposal/verify time, acceptance, and generation t/s. Also run `--decode-consistency 64` and the logprob-vector regression before enabling a numerically different kernel by default. +- For the CUDA AProjQ4 ports, run an isolated A/B for each dispatch: + Q-A/KV pair control `DS4_CUDA_DISABLE_Q4_DENSE_PAIR=1` versus candidate with + that variable absent; HC norm/mix control + `DS4_CUDA_DISABLE_HC_NORM_MIX_FUSE=1` versus candidate + `DS4_CUDA_ENABLE_HC_NORM_MIX_FUSE=1 DS4_CUDA_NO_F16_CUBLAS_ONE=1`; and Q4 + attention-output/HC control `DS4_CUDA_DISABLE_Q4_ATTN_OUT_HC_FUSE=1` versus + candidate `DS4_CUDA_ENABLE_Q4_ATTN_OUT_HC_FUSE=1`. Repeat the pair and + attention-output cases with `DS4_CUDA_MMQ=0` to exercise the Q8_K fallback + arithmetic separately from the default MMVQ/Q8_1 path. Require + byte-identical stdout and full-logit/tensor equivalence before promoting an + opt-in gate. Run with decode graphs both enabled and disabled, and record + target, proposer, verifier, replay, acceptance, and generation t/s. A CUDA + build and hardware run are mandatory; a host-only build does not compile + the device kernels. - When DSpark, support-model mapping, or SSD streaming changes, repeat both the acceptance fixture and verifier invariant on every advertised graph backend. Apply the backend and SSD options to the target-only baseline as diff --git a/README.md b/README.md index c22f11ae1..b8fe9f322 100644 --- a/README.md +++ b/README.md @@ -413,7 +413,7 @@ small expert cache. A practical 16 GiB starting point is: ```sh ./ds4 -m ds4flash.gguf \ - --mtp gguf/DeepSeek-V4-Flash-DSpark-support-0731.gguf \ + --mtp-model gguf/DeepSeek-V4-Flash-DSpark-support-0731.gguf \ --dspark --metal --ssd-streaming \ --ssd-streaming-cache-experts 16 \ --ctx 4096 --prefill-chunk 128 --temp 0 @@ -424,12 +424,97 @@ When DSpark+SSD runs on a Mac with at most 24 GiB and neither automatically. Set `DS4_DSPARK_LOW_MEMORY_PREFILL_CHUNK=0` to retain the normal workspace policy, or set it to another row count. -The Metal SSD verifier automatically limits a speculative block to the number -of complete top-k rows that fit in the effective target expert cache (two rows -with 12 effective slots and top-6 routing). Override this diagnostic policy -with `DS4_DSPARK_SSD_VERIFY_BLOCK_MAX=N`. Exact file views for the two token -embedding rows and repeatedly used Q8 support tensors are automatic; the -compatibility kill switches are +The Metal SSD verifier already supports the checkpoint's full five-draft +speculative block. With top-6 routing it needs at least 30 effective target +expert-cache slots; `--ssd-streaming-cache-experts 32` is the practical +five-draft setting. Smaller caches automatically limit the verifier to the +number of complete top-k rows that fit (a 16-expert cache normally selects two +rows). The Metal proposer follows that effective verifier/cache cap, avoiding +work on a suffix that cannot be consumed. Set +`DS4_METAL_DSPARK_PROPOSER_BLOCK_MAX=0` to restore the checkpoint's native +five-row proposer for an A/B control, or set a positive value to cap it +explicitly. Override the verifier policy independently with +`DS4_DSPARK_SSD_VERIFY_BLOCK_MAX=N`. + +An experimental single-device Metal verifier can use the current target +logits for the first draft and evaluate only the remaining `N-1` target rows. +Enable it with `DS4_METAL_DSPARK_ACCEPTANCE_ONLY_VERIFY=1`; it remains opt-in +because the smaller batch did not improve throughput on the measured M1 Pro +SSD path. Two-draft blocks retain the legacy verifier +because its one-row SSD routed-FFN path does not yet use the tiny-batch expert +table. A five-draft block starting exactly on a ratio-4 compressor boundary +also retains the legacy path so acceptance arithmetic does not switch to the +aligned compressor kernel. After verification rolls back, +the exact replay also skips the output head and logits readback for accepted +prefix tokens whose logits would be discarded; set +`DS4_METAL_DSPARK_HEADLESS_REPLAY=0` to restore the legacy replay. With +`DS4_DSPARK_STATS=1`, `metal_accept_only`, `metal_verify_rows_saved`, and +`metal_replay_headless` show how often these paths were exercised. + +On very small unified-memory Macs, an additional diagnostic can keep only the +stage-0 `main_norm` and `main_proj` support tensors resident. Set +`DS4_METAL_DSPARK_PIN_MAIN_PROJ=1`; the 0731 support file locks about 51 MiB, +not the full 5.6 GiB GGUF. A failed lock is non-fatal and leaves the existing +pageable path active. Keep this opt-in until a same-machine A/B shows lower +`prop_setup`/generation time without reducing target-only throughput. + +An experimental two-draft Metal SSD verifier can commit a full accept without +the normal rollback/replay pass: + +```sh +DS4_METAL_DSPARK_EXACT2=1 DS4_DSPARK_STATS=1 \ +./ds4 -m ds4flash.gguf \ + --mtp-model gguf/DeepSeek-V4-Flash-DSpark-support-0731.gguf \ + --dspark --metal --ssd-streaming \ + --ssd-streaming-cache-experts 16 --ctx 4096 --prefill-chunk 128 \ + --temp 0 +``` + +It uses the canonical one-row decode kernels in layer order, restores and +replays token zero on a partial accept, and defaults both proposal and verify +width to two. Keep it opt-in until a long same-machine run is byte-identical, +has `exact2_attempt>0` and `exact2_fallback=0`, and improves throughput. The +generic Metal verifier can already evaluate five drafts together with a +32-expert cache, but its batch state is not numerically interchangeable with +ordinary decode and therefore still requires rollback plus exact replay. + +`DS4_METAL_DSPARK_EXACTN_UNION=1` enables a separate experimental Metal SSD +verifier for two through five draft tokens. It executes canonical one-row +target decode in layer order, loads the union of the rows' routed experts once +per layer, and commits directly only after a full accept; partial accepts keep +the established rollback/replay fallback. The model-backed +`test-metal-exactn-oracle` is byte-identical to sequential decode for N=2..5, +including every N=5 partial prefix, EOS in the first or a middle row, serialized +KV/compressor state, logits, and a four-token continuation. Five drafts plus +the target token already available at the start of the cycle cover the +six-token speculative-cycle limit. Exact-union remains opt-in: correctness +does not imply a throughput improvement on a particular memory configuration. + +The AProjQ4 Metal decode path has three exact dispatch fusions relevant to this +verifier: + +- HC RMSNorm plus the narrow F16 HC mixer is the M1-M4 default, including SSD + split phases such as `TO_ROUTER`. Use + `DS4_METAL_DISABLE_PRE_M5_HC_NORM_MIX_FUSE=1` for the reference control; + `DS4_METAL_ENABLE_HC_NORM_MIX_FUSE=1` is the explicit non-default gate on + other Apple generations. +- The Q4 Q-A/KV projections can share a dispatch with eligible F16 compressor + projection/store work. It remains opt-in in both exact-union and ordinary + `FULL` decode via `DS4_METAL_ENABLE_Q4_QKV_COMPRESSOR_FUSE=1`; the first M1 + Pro SSD A/B reduced dispatch count but did not improve verifier time. In + either scope, + `DS4_METAL_DISABLE_Q4_QKV_COMPRESSOR_FUSE=1` selects the existing fallback. +- The eligible Q4 attention-output B projection can perform the following HC + expansion in the same dispatch. Use + `DS4_METAL_DISABLE_Q4_ATTN_OUT_HC_FUSE=1` as its isolated A/B control. + +These gates change dispatch and intermediate-memory traffic, not model +arithmetic. Compare byte-identical output, exact-union counters, stage timings, +and generation rate on the same machine; do not infer a speedup from a lower +dispatch count alone. + +Exact file views for the two token embedding rows and repeatedly used Q8 +support tensors are automatic; the compatibility kill switches are `DS4_METAL_DISABLE_TOKEN_EMBED_EXACT_VIEW=1` and `DS4_METAL_DISABLE_SUPPORT_Q8_DECODE_EXACT_VIEWS=1`. @@ -461,14 +546,32 @@ measurements are CUDA fuses HC split, weighted sum, and RMSNorm across multiple batch rows, including the DSpark proposer and verifier; use `DS4_CUDA_DISABLE_HC_SPLIT_NORM_FUSED=1` for an A/B fallback to the separate -kernels. An experimental resident-CUDA path can run the existing aligned -IQ2_XXS/Q2_K vector MoE kernels for two-to-five-token routed batches, +kernels. AProjQ4 CUDA decode can also share the activation quantization for +the Q-A/KV dense pair; `DS4_CUDA_DISABLE_Q4_DENSE_PAIR=1` selects the two +standalone projections. The canonical Q4 path submits to the decode stream so +these projections and the attention-output tail can participate in CUDA +decode graphs. + +Two additional CUDA fusions remain experimental until a device oracle passes +on the target GPU. `DS4_CUDA_ENABLE_HC_NORM_MIX_FUSE=1` combines HC RMSNorm +with the narrow F16 mixer when the selected standalone kernels have the same +reduction order; with the normal one-token cuBLAS path, also set +`DS4_CUDA_NO_F16_CUBLAS_ONE=1` to exercise it. The controls are +`DS4_CUDA_DISABLE_HC_NORM_MIX_FUSE=1` and +`DS4_CUDA_DISABLE_Q4_ATTN_OUT_HC_FUSE=1`. The Q4 attention-output B plus HC +expansion is automatic only when MMQ is disabled, where it preserves the +existing Q8_K activation quantizer. With the normal MMVQ/Q8_1 decode path it +requires the explicit `DS4_CUDA_ENABLE_Q4_ATTN_OUT_HC_FUSE=1` experiment and +may not be numerically identical until validated on CUDA hardware. + +An experimental resident-CUDA path can run the existing aligned +IQ2_XXS/Q2_K vector MoE kernels for two-to-five-draft routed batches, preserving the established fused-SoA path as an automatic fallback: ```sh DS4_CUDA_DSPARK_TINY_ALIGNED_VEC=1 DS4_DSPARK_STATS=1 \ ./ds4 --cuda -m ds4flash.gguf \ - --mtp gguf/DeepSeek-V4-Flash-DSpark-support-0731.gguf \ + --mtp-model gguf/DeepSeek-V4-Flash-DSpark-support-0731.gguf \ --dspark --temp 0 -p 'Write a Python quicksort function with comments.' ``` @@ -478,17 +581,29 @@ decode-consistency, and throughput comparisons pass on CUDA hardware. An experimental exact two-token resident-CUDA verifier is available for a DGX Spark A/B test. It uses the ordinary decode kernels, commits a two-token full accept without rollback/replay, and replays only the first token on a -partial accept. The switch also caps verification to the first two drafts: +partial accept. By default the switch runs both the proposer and verifier at +width two, instead of evaluating the checkpoint's native five-row proposal +when only two drafts can be consumed: ```sh DS4_CUDA_DSPARK_EXACT2=1 DS4_DSPARK_STATS=1 \ ./ds4 --cuda -m ds4flash.gguf \ - --mtp gguf/DeepSeek-V4-Flash-DSpark-support-0731.gguf \ + --mtp-model gguf/DeepSeek-V4-Flash-DSpark-support-0731.gguf \ --dspark --temp 0 -p 'Write a Python quicksort function with comments.' ``` +The support model uses non-causal attention across the proposal block, so a +two-row proposal is not guaranteed to be a prefix-identical version of its +native five-row proposal. The target verifier still protects the emitted +greedy continuation. For isolated A/B tests, +`DS4_CUDA_DSPARK_PROPOSER_BLOCK_MAX=0` preserves the native proposer and an +explicit value such as `2` caps it independently of exact-2. + Keep this path opt-in until the CUDA acceptance fixture is byte-identical and -the same-machine statistics show lower `verify` plus `replay` time. +the same-machine statistics show lower `propose`, `verify` plus `replay` time. +The stats line reports `prop_capped`, `prop_scheduled_rows`, `exact2_attempt`, +`exact2_full`, `exact2_partial`, and `exact2_fallback`; a valid run must +exercise exact-2 and leave its fallback counter at zero. The acceptance fixture can exercise the same SSD path on both the target-only baseline and the DSpark run. It also requires real proposals and accepted @@ -507,7 +622,6 @@ object set and linker; the generic target selects CUDA objects on non-Apple hosts. `make rocm-dspark-verify-depth` provides the corresponding verifier invariant test. - ## Speed The current q2 results use `ds4-bench` with the standard *Promessi sposi* diff --git a/cuda/mmq/ds4_mmq.cu b/cuda/mmq/ds4_mmq.cu index c3a270596..f4e3f61bb 100644 --- a/cuda/mmq/ds4_mmq.cu +++ b/cuda/mmq/ds4_mmq.cu @@ -3153,6 +3153,136 @@ int ds4_mmq_dense_vec_impl( return 0; } +template +int ds4_mmq_dense_pair_vec_impl( + const char * tag, + const void * W0, + const void * W1, + const float * X_f32, + float * out0_f32, + float * out1_f32, + int M0, + int M1, + int N, + int K, + cudaStream_t stream) { + + if (!W0 || !W1 || !X_f32 || !out0_f32 || !out1_f32) { + fprintf(stderr, "%s: null pointer\n", tag); + return -1; + } + if (M0 <= 0 || M1 <= 0 || N <= 0 || K <= 0) { + fprintf(stderr, "%s: bad shape M0=%d M1=%d N=%d K=%d\n", + tag, M0, M1, N, K); + return -1; + } + if (K % 256 != 0) { + fprintf(stderr, "%s: K=%d must be a multiple of 256\n", tag, K); + return -1; + } + if (N > MMVQ_MAX_BATCH_SIZE) { + fprintf(stderr, "%s: N=%d exceeds MMVQ_MAX_BATCH_SIZE=%d\n", + tag, N, MMVQ_MAX_BATCH_SIZE); + return -1; + } + + const int dev = ggml_cuda_get_device(); + ggml_backend_cuda_context * ctx = get_ctx_for_device(dev); + if (!ctx) { + fprintf(stderr, "%s: failed to get cuda context for device %d\n", + tag, dev); + return -1; + } + + ds4_pool_set_stream(stream); + + /* Match ds4_mmq_dense_vec_impl's activation layout and quantizer exactly, + * but retain the Q8_1 row for both projections. */ + const int64_t ne10_padded = GGML_PAD((int64_t)K, MATRIX_ROW_PADDING); + const size_t nbytes_q8_1 = (size_t)N * ne10_padded * + sizeof(block_q8_1) / QK8_1; + ggml_cuda_pool_alloc src1_q8_1(ctx->pool(), nbytes_q8_1); + + quantize_row_q8_1_cuda( + X_f32, /*ids=*/nullptr, (void *)src1_q8_1.get(), + type, /*ne00=*/K, + /*s11=*/(int64_t)K, /*s12=*/(int64_t)K * N, + /*s13=*/(int64_t)K * N, + /*ne0=*/ne10_padded, /*ne1=*/N, /*ne2=*/1, /*ne3=*/1, + stream); + + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) { + fprintf(stderr, "%s: quantize_row_q8_1_cuda failed: %s\n", + tag, cudaGetErrorString(err)); + return -2; + } + + const int64_t blck = ggml_blck_size(type); + const int64_t s01_row = (int64_t)K / blck; + const int64_t s11_y = ne10_padded / QK8_1; + const int64_t s12_y = (int64_t)N * s11_y; + ggml_cuda_mm_fusion_args_device fusion = {}; + + /* Keep each leg's memset, canonical MMVQ dispatch, error check, and + * sanitizer in the same order as two dense_vec calls. Only the activation + * quantization/allocation above is shared. */ + cudaMemsetAsync(out0_f32, 0, + (size_t)M0 * (size_t)N * sizeof(float), stream); + mul_mat_vec_q_switch_type( + /*vx=*/W0, /*type_x=*/type, + /*vy=*/(const void *)src1_q8_1.get(), + /*ids=*/nullptr, /*fusion=*/fusion, + /*dst=*/out0_f32, + /*ncols_x=*/K, /*nrows_x=*/M0, /*ncols_dst=*/N, + /*stride_row_x=*/(int)s01_row, + /*stride_col_y=*/(int)s11_y, + /*stride_col_dst=*/M0, + /*nchannels_x=*/1, /*nchannels_y=*/1, /*nchannels_dst=*/1, + /*stride_channel_x=*/0, + /*stride_channel_y=*/(int)s12_y, + /*stride_channel_dst=*/0, + /*nsamples_x=*/1, /*nsamples_dst=*/1, + /*stride_sample_x=*/0, /*stride_sample_y=*/0, + /*stride_sample_dst=*/0, + /*ids_stride=*/0, stream); + err = cudaGetLastError(); + if (err != cudaSuccess) { + fprintf(stderr, "%s: first dense MMVQ launch failed: %s\n", + tag, cudaGetErrorString(err)); + return -3; + } + ds4_mmq_sanitize_f32(out0_f32, (uint64_t)M0 * (uint64_t)N, stream); + + cudaMemsetAsync(out1_f32, 0, + (size_t)M1 * (size_t)N * sizeof(float), stream); + mul_mat_vec_q_switch_type( + /*vx=*/W1, /*type_x=*/type, + /*vy=*/(const void *)src1_q8_1.get(), + /*ids=*/nullptr, /*fusion=*/fusion, + /*dst=*/out1_f32, + /*ncols_x=*/K, /*nrows_x=*/M1, /*ncols_dst=*/N, + /*stride_row_x=*/(int)s01_row, + /*stride_col_y=*/(int)s11_y, + /*stride_col_dst=*/M1, + /*nchannels_x=*/1, /*nchannels_y=*/1, /*nchannels_dst=*/1, + /*stride_channel_x=*/0, + /*stride_channel_y=*/(int)s12_y, + /*stride_channel_dst=*/0, + /*nsamples_x=*/1, /*nsamples_dst=*/1, + /*stride_sample_x=*/0, /*stride_sample_y=*/0, + /*stride_sample_dst=*/0, + /*ids_stride=*/0, stream); + err = cudaGetLastError(); + if (err != cudaSuccess) { + fprintf(stderr, "%s: second dense MMVQ launch failed: %s\n", + tag, cudaGetErrorString(err)); + return -4; + } + ds4_mmq_sanitize_f32(out1_f32, (uint64_t)M1 * (uint64_t)N, stream); + return 0; +} + template struct ds4_mmq_vdr_mmvq_value; template <> struct ds4_mmq_vdr_mmvq_value { static constexpr int value = VDR_IQ2_XXS_Q8_1_MMVQ; }; template <> struct ds4_mmq_vdr_mmvq_value { static constexpr int value = VDR_Q2_K_Q8_1_MMVQ; }; @@ -4942,6 +5072,15 @@ extern "C" int ds4_mmq_q4_K_dense_vec( "ds4_mmq_q4_K_dense_vec", W, X, out, M, N, K, stream); } +extern "C" int ds4_mmq_q4_K_dense_pair_vec( + const void * W0, const void * W1, const float * X, + float * out0, float * out1, + int M0, int M1, int N, int K, cudaStream_t stream) { + return ds4_mmq_dense_pair_vec_impl( + "ds4_mmq_q4_K_dense_pair_vec", W0, W1, X, out0, out1, + M0, M1, N, K, stream); +} + // Explicit instantiations. One per quant type the public API exposes. // Each instantiation drags in the load_tiles_ + vec_dot__* // device functions from mmq.cuh, so the .o objects below contain everything diff --git a/cuda/mmq/ds4_mmq.h b/cuda/mmq/ds4_mmq.h index cf0a7e66d..ed4c19f35 100644 --- a/cuda/mmq/ds4_mmq.h +++ b/cuda/mmq/ds4_mmq.h @@ -891,6 +891,22 @@ int ds4_mmq_q4_K_dense_vec( int K, cudaStream_t stream); +// Two independent dense Q4_K projections that share the canonical Q8_1 +// activation quantization. Each output is dispatched through the same MMVQ +// entry as ds4_mmq_q4_K_dense_vec, so its reduction and output bits are +// unchanged; M0 and M1 may differ (the DS4 Q-A/KV decode shape does). +int ds4_mmq_q4_K_dense_pair_vec( + const void * W0_q4_K, + const void * W1_q4_K, + const float * X_f32, + float * out0_f32, + float * out1_f32, + int M0, + int M1, + int N, + int K, + cudaStream_t stream); + // Set the thread-local stream that the internal cuda pool uses for // cudaMallocAsync / cudaFreeAsync. Defaults to cudaStreamPerThread. // Step 8 (CUDA Graphs) calls this with the capture stream so pool diff --git a/ds4.c b/ds4.c index 33646f74b..9dd5142a5 100644 --- a/ds4.c +++ b/ds4.c @@ -16002,6 +16002,11 @@ typedef struct { uint32_t spec_prefix_n_comp[DS4_SPEC_PREFIX_SLOTS][DS4_MAX_LAYER]; uint32_t spec_prefix_n_index_comp[DS4_SPEC_PREFIX_SLOTS][DS4_MAX_LAYER]; bool spec_capture_prefixes; + /* While exact-N collects GPU router rows, hash routing must not start the + * ordinary per-token SSD early load or publish a process-global selected + * override. The routed tails consume the GPU-selected matrix through the + * exact-row union scope instead. */ + bool spec_exactn_union_collect_routes; /* Batch verification normally takes the per-row compressor path when it * captures intermediate frontiers. Rollback+replay DSpark verification * needs the same arithmetic path, but not the frontier copies themselves. */ @@ -16022,6 +16027,8 @@ typedef struct { * predictable: each pointer names an actual DS4 stage. */ ds4_gpu_tensor *comp_kv_cur_by_tier[DS4_MAX_GPUS]; ds4_gpu_tensor *comp_sc_cur_by_tier[DS4_MAX_GPUS]; + /* The Q4 Q-A/KV compound projects both compressor pairs in one dispatch. + * Attention and indexer outputs must therefore have disjoint scratch. */ ds4_gpu_tensor *index_comp_kv_cur_by_tier[DS4_MAX_GPUS]; ds4_gpu_tensor *index_comp_sc_cur_by_tier[DS4_MAX_GPUS]; ds4_gpu_tensor *attn_comp_stage_by_tier[DS4_MAX_GPUS]; @@ -16765,9 +16772,9 @@ static void metal_graph_free(ds4_gpu_graph *g) { ds4_gpu_tensor_free(g->attn_out_by_tier[t]); ds4_gpu_tensor_free(g->attn_low_by_tier[t]); ds4_gpu_tensor_free(g->heads_by_tier[t]); - ds4_gpu_tensor_free(g->comp_sc_cur_by_tier[t]); - ds4_gpu_tensor_free(g->index_comp_kv_cur_by_tier[t]); ds4_gpu_tensor_free(g->index_comp_sc_cur_by_tier[t]); + ds4_gpu_tensor_free(g->index_comp_kv_cur_by_tier[t]); + ds4_gpu_tensor_free(g->comp_sc_cur_by_tier[t]); ds4_gpu_tensor_free(g->comp_kv_cur_by_tier[t]); ds4_gpu_tensor_free(g->attn_comp_stage_by_tier[t]); ds4_gpu_tensor_free(g->comp_mask_by_tier[t]); @@ -20715,6 +20722,25 @@ static bool metal_graph_use_reference_hc_decode(void) { return metal_graph_env_flag("DS4_METAL_DISABLE_HC_FUSION", &cache); } +static bool metal_graph_use_hc_norm_mix_f16( + const ds4_tensor *weight, + uint64_t in_dim, + uint64_t out_dim) { +#if !defined(DS4_NO_GPU) && !defined(DS4_ROCM_BUILD) + return weight != NULL && + in_dim == 16384u && + out_dim == 24u && + weight->type == DS4_TENSOR_F16 && + !metal_graph_use_reference_hc_decode() && + ds4_gpu_hc_rms_norm_mix_f16_available() != 0; +#else + (void)weight; + (void)in_dim; + (void)out_dim; + return false; +#endif +} + static bool metal_graph_use_reference_kv_decode(void) { static int cache = -1; return metal_graph_env_flag("DS4_METAL_DISABLE_KV_FUSION", &cache); @@ -21865,6 +21891,13 @@ static bool metal_graph_decode_set_hash_selected_override( return true; } + /* ds4_gpu_router_select_tensor() has already encoded the hash lookup into + * this row's router_selected/router_weights views. Exact-N reads those + * GPU-produced rows together after the prefix batch, so publishing the + * ordinary host override here would overwrite each preceding row and its + * early-load request would serialize/load the same layer once per token. */ + if (g && g->spec_exactn_union_collect_routes) return true; + int selected[DS4_MAX_EXPERT_USED]; int32_t selected_i32[DS4_MAX_EXPERT_USED]; layer_hash_selected_experts(selected, model, layer, (int)token); @@ -23024,13 +23057,12 @@ static bool metal_graph_encode_decode_layer_phase( /* Fused norm+mix removes one decode dispatch per layer; the kernel * reproduces both reduction trees bit-exactly (see dsv4_hc.metal). */ const bool fuse_norm_mix = - hc_dim == 16384u && mix_hc == 24u && - layer->hc_attn_fn->type == DS4_TENSOR_F16 && - !metal_graph_use_reference_hc_decode() && + metal_graph_use_hc_norm_mix_f16(layer->hc_attn_fn, + hc_dim, + mix_hc) && getenv("DS4_METAL_DISABLE_PRE_M5_HC_NORM_MIX_FUSE") == NULL && (ds4_gpu_device_is_pre_m5_apple_silicon() || - ds4_gpu_device_is_m5_apple_silicon()) && - ds4_gpu_hc_rms_norm_mix_f16_available() != 0; + ds4_gpu_device_is_m5_apple_silicon()); #if defined(__APPLE__) const bool fuse_producer_pre_norm = fuse_norm_mix && fuse_hc_norm && @@ -23166,6 +23198,93 @@ static bool metal_graph_encode_decode_layer_phase( layer->attn_q_a->type == DS4_TENSOR_Q4_K && layer->attn_kv->type == DS4_TENSOR_Q4_K; bool qkv_pair_projected = resume_after_qa_kv_raw; +#if defined(__APPLE__) && !defined(DS4_NO_GPU) + /* Fuse the Q4 Q-A/KV pair with the compressor projections that consume + * the same normalized row. Keep the larger compound dispatch opt-in in + * both exact-union and ordinary FULL decode: on the measured M1 Pro SSD + * path it reduced dispatch count but did not improve verifier time. */ + const uint32_t q4_compound_ratio = + compressed ? ds4_layer_compress_ratio(il) : 0u; + const uint32_t q4_compound_width0 = + q4_compound_ratio == 4u ? 2u * DS4_N_HEAD_DIM : DS4_N_HEAD_DIM; + const uint32_t q4_compound_width1 = + q4_compound_ratio == 4u ? 2u * DS4_N_INDEXER_HEAD_DIM : 0u; + const bool q4_compound_scope = + getenv("DS4_METAL_ENABLE_Q4_QKV_COMPRESSOR_FUSE") != NULL && + ((phase == METAL_DECODE_LAYER_TO_ROUTER && + g->spec_exactn_union_collect_routes) || + phase == METAL_DECODE_LAYER_FULL); + const bool q4_compound_index_ok = + q4_compound_ratio != 4u || + (layer->indexer_compressor_kv && layer->indexer_compressor_gate && + layer->indexer_compressor_ape && + layer->indexer_compressor_kv->type == DS4_TENSOR_F16 && + layer->indexer_compressor_gate->type == DS4_TENSOR_F16 && + layer->indexer_compressor_kv->dim[0] == DS4_N_EMBD && + layer->indexer_compressor_gate->dim[0] == DS4_N_EMBD && + layer->indexer_compressor_kv->dim[1] == q4_compound_width1 && + layer->indexer_compressor_gate->dim[1] == q4_compound_width1); + if (!resume_after_qa_kv_raw && ok && qkv_rms_fused && qkv_proj_q4 && + compressed && q4_compound_scope && + (q4_compound_ratio == 4u || q4_compound_ratio == 128u) && + !metal_graph_use_reference_qkv_pair_proj() && + !metal_graph_use_reference_compressor_pair_proj() && + layer->attn_compressor_kv && layer->attn_compressor_gate && + layer->attn_compressor_ape && + layer->attn_compressor_kv->type == DS4_TENSOR_F16 && + layer->attn_compressor_gate->type == DS4_TENSOR_F16 && + layer->attn_compressor_kv->dim[0] == DS4_N_EMBD && + layer->attn_compressor_gate->dim[0] == DS4_N_EMBD && + layer->attn_compressor_kv->dim[1] == q4_compound_width0 && + layer->attn_compressor_gate->dim[1] == q4_compound_width0 && + q4_compound_index_ok) { + ds4_gpu_tensor *out1_kv = q4_compound_width1 + ? metal_graph_index_comp_kv_cur(g) : metal_graph_comp_kv_cur(g); + ds4_gpu_tensor *out1_sc = q4_compound_width1 + ? metal_graph_index_comp_sc_cur(g) : metal_graph_comp_sc_cur(g); + ds4_gpu_tensor *state1_kv = q4_compound_width1 + ? g->layer_index_state_kv[il] : g->layer_attn_state_kv[il]; + ds4_gpu_tensor *state1_sc = q4_compound_width1 + ? g->layer_index_state_score[il] : g->layer_attn_state_score[il]; + const uint64_t weight1_kv = q4_compound_width1 + ? layer->indexer_compressor_kv->abs_offset + : layer->attn_compressor_kv->abs_offset; + const uint64_t weight1_sc = q4_compound_width1 + ? layer->indexer_compressor_gate->abs_offset + : layer->attn_compressor_gate->abs_offset; + const uint64_t ape1 = q4_compound_width1 + ? layer->indexer_compressor_ape->abs_offset + : layer->attn_compressor_ape->abs_offset; + const uint32_t ape1_type = q4_compound_width1 + ? layer->indexer_compressor_ape->type + : layer->attn_compressor_ape->type; + const int fused = ds4_gpu_q4_K_pair_quad_compressor_store_tensor( + metal_graph_qr(g), metal_graph_kv_raw(g), + metal_graph_comp_kv_cur(g), metal_graph_comp_sc_cur(g), + out1_kv, out1_sc, + g->layer_attn_state_kv[il], + g->layer_attn_state_score[il], + state1_kv, state1_sc, + model->map, model->size, + layer->attn_q_a->abs_offset, + layer->attn_kv->abs_offset, + layer->attn_compressor_kv->abs_offset, + layer->attn_compressor_gate->abs_offset, + weight1_kv, weight1_sc, + layer->attn_compressor_ape->abs_offset, + layer->attn_compressor_ape->type, + ape1, ape1_type, + DS4_N_EMBD, (uint32_t)q_rank, DS4_N_HEAD_DIM, + q4_compound_width0, q4_compound_width1, + metal_graph_attn_norm(g), q4_compound_ratio, pos); + if (fused < 0) { + ok = false; + } else if (fused > 0) { + qkv_pair_projected = true; + qkv_pair_quad_fused = true; + } + } +#endif /* M1-M5 decode fusion: the q_a/kv Q8 pair and the four F16 compressor * projections all read the same normalized attention input and write * disjoint outputs, so one dispatch covers both stages with unchanged @@ -23293,8 +23412,8 @@ static bool metal_graph_encode_decode_layer_phase( qkv_pair_quad_fused = true; } } - if (!resume_after_qa_kv_raw && ok && !qkv_pair_quad_fused && qkv_rms_fused && - qkv_proj_q8 && + if (!resume_after_qa_kv_raw && ok && !qkv_pair_quad_fused && + qkv_rms_fused && qkv_proj_q8 && g->cuda_qkv_pair && !metal_graph_use_reference_qkv_pair_proj()) { qkv_pair_projected = ds4_gpu_matmul_q8_0_pair_tensor( metal_graph_qr(g), @@ -23310,6 +23429,7 @@ static bool metal_graph_encode_decode_layer_phase( 1) != 0; } if (!resume_after_qa_kv_raw && ok && qkv_rms_fused && qkv_proj_q4 && + g->cuda_qkv_pair && !qkv_pair_projected && !metal_graph_use_reference_qkv_pair_proj()) { qkv_pair_projected = ds4_gpu_matmul_q4_K_pair_tensor( metal_graph_qr(g), @@ -23647,7 +23767,8 @@ static bool metal_graph_encode_decode_layer_phase( } else if (quad_store > 0) { comp_state_already_stored = true; } - if (ok && quad_store == 0 && !metal_graph_use_reference_compressor_pair_proj()) { + if (ok && !comp_state_already_stored && + !metal_graph_use_reference_compressor_pair_proj()) { const int fused_store = ds4_gpu_matmul_f16_pair_compressor_store_tensor( metal_graph_comp_kv_cur(g), @@ -23681,7 +23802,7 @@ static bool metal_graph_encode_decode_layer_phase( metal_graph_attn_norm(g), 1) != 0; } - } else if (quad_store == 0) { + } else if (!comp_state_already_stored) { if (ok) ok = ds4_gpu_matmul_f16_tensor(metal_graph_comp_kv_cur(g), model->map, model->size, layer->attn_compressor_kv->abs_offset, DS4_N_EMBD, comp_width, @@ -23776,11 +23897,18 @@ static bool metal_graph_encode_decode_layer_phase( ok = false; } bool index_state_already_stored = quad_store > 0; - if (ok && quad_store == 0 && !metal_graph_use_reference_compressor_pair_proj()) { + ds4_gpu_tensor *index_comp_kv = quad_store > 0 + ? metal_graph_index_comp_kv_cur(g) + : metal_graph_comp_kv_cur(g); + ds4_gpu_tensor *index_comp_sc = quad_store > 0 + ? metal_graph_index_comp_sc_cur(g) + : metal_graph_comp_sc_cur(g); + if (ok && !index_state_already_stored && + !metal_graph_use_reference_compressor_pair_proj()) { const int fused_store = ds4_gpu_matmul_f16_pair_compressor_store_tensor( - metal_graph_comp_kv_cur(g), - metal_graph_comp_sc_cur(g), + index_comp_kv, + index_comp_sc, g->layer_index_state_kv[il], g->layer_index_state_score[il], model->map, @@ -23799,8 +23927,8 @@ static bool metal_graph_encode_decode_layer_phase( } else if (fused_store > 0) { index_state_already_stored = true; } else { - ok = ds4_gpu_matmul_f16_pair_tensor(metal_graph_comp_kv_cur(g), - metal_graph_comp_sc_cur(g), + ok = ds4_gpu_matmul_f16_pair_tensor(index_comp_kv, + index_comp_sc, model->map, model->size, layer->indexer_compressor_kv->abs_offset, @@ -23810,20 +23938,20 @@ static bool metal_graph_encode_decode_layer_phase( metal_graph_attn_norm(g), 1) != 0; } - } else if (quad_store == 0) { - if (ok) ok = ds4_gpu_matmul_f16_tensor(metal_graph_comp_kv_cur(g), model->map, model->size, + } else if (!index_state_already_stored) { + if (ok) ok = ds4_gpu_matmul_f16_tensor(index_comp_kv, model->map, model->size, layer->indexer_compressor_kv->abs_offset, DS4_N_EMBD, index_width, metal_graph_attn_norm(g), 1) != 0; - if (ok) ok = ds4_gpu_matmul_f16_tensor(metal_graph_comp_sc_cur(g), model->map, model->size, + if (ok) ok = ds4_gpu_matmul_f16_tensor(index_comp_sc, model->map, model->size, layer->indexer_compressor_gate->abs_offset, DS4_N_EMBD, index_width, metal_graph_attn_norm(g), 1) != 0; } DS4_METAL_PROFILE_DECODE_STAGE("indexer_compressor_proj"); const uint32_t index_row = g->layer_n_index_comp[il]; - if (ok) ok = ds4_gpu_compressor_update_tensor(metal_graph_comp_kv_cur(g), - metal_graph_comp_sc_cur(g), + if (ok) ok = ds4_gpu_compressor_update_tensor(index_comp_kv, + index_comp_sc, g->layer_index_state_kv[il], g->layer_index_state_score[il], g->layer_index_comp_cache[il], @@ -24310,13 +24438,27 @@ static bool metal_graph_encode_decode_layer_phase( layer->attn_output_b->type == DS4_TENSOR_Q8_0; ds4_gpu_tensor *tp_attn_a = NULL; /* rank partials consumed directly */ ds4_gpu_tensor *tp_attn_b = NULL; /* by the HC expand */ - const bool fuse_attn_out_hc = + const bool fuse_attn_out_hc_q8 = !cuda_tp_attn && g->tp_world < 2 && layer->attn_output_a->type == DS4_TENSOR_Q8_0 && layer->attn_output_b->type == DS4_TENSOR_Q8_0 && !metal_graph_directional_steering_attn_enabled(g) && !metal_graph_use_reference_attn_out_hc(); +#if !defined(DS4_NO_GPU) && !defined(DS4_ROCM_BUILD) + const bool fuse_attn_out_hc_q4 = + !cuda_tp_attn && + g->tp_world < 2 && + layer->attn_output_a->type == DS4_TENSOR_Q4_K && + layer->attn_output_b->type == DS4_TENSOR_Q4_K && + !metal_graph_directional_steering_attn_enabled(g) && + !metal_graph_use_reference_attn_out_hc() && + ds4_gpu_matmul_q4_K_hc_expand_available() != 0; +#else + const bool fuse_attn_out_hc_q4 = false; +#endif + const bool fuse_attn_out_hc = + fuse_attn_out_hc_q8 || fuse_attn_out_hc_q4; const bool fuse_tp_attn_out_hc = cuda_tp_attn && !metal_graph_use_reference_attn_out_hc() && @@ -24487,7 +24629,7 @@ static bool metal_graph_encode_decode_layer_phase( DS4_N_HC) != 0; if (ok) cuda_tp_attn_hc_fused = true; } - } else if (ok && fuse_attn_out_hc) { + } else if (ok && fuse_attn_out_hc_q8) { ok = ds4_gpu_attention_output_low_q8_tensor(metal_graph_attn_low(g), model->map, model->size, @@ -24510,6 +24652,29 @@ static bool metal_graph_encode_decode_layer_phase( DS4_N_EMBD, DS4_N_HC) != 0; } + } else if (ok && fuse_attn_out_hc_q4) { + ok = metal_graph_attention_output_dense_quant_low( + metal_graph_attn_low(g), g, model, + layer->attn_output_a, + group_dim, rank, 0, n_groups, + metal_graph_heads(g)); +#if !defined(DS4_NO_GPU) && !defined(DS4_ROCM_BUILD) + if (ok) { + ok = ds4_gpu_matmul_q4_K_hc_expand_tensor( + metal_graph_after_attn_hc(g), + metal_graph_attn_out(g), + model->map, model->size, + layer->attn_output_b->abs_offset, + (uint64_t)n_groups * rank, + DS4_N_EMBD, + metal_graph_attn_low(g), + metal_graph_cur_hc(g), + metal_graph_hc_split(g), + DS4_N_EMBD, DS4_N_HC) != 0; + } +#else + ok = false; +#endif } else if (ok && g->tp_world == 2) { /* Group-sliced attention output: this rank computes its half of the * output groups and the matching k-window of the expand projection, @@ -24618,13 +24783,12 @@ static bool metal_graph_encode_decode_layer_phase( bool ffn_hc_producer_pre_norm_fused = false; if (ok && !tp_ablate_hcpre) { const bool fuse_norm_mix = - hc_dim == 16384u && mix_hc == 24u && - layer->hc_ffn_fn->type == DS4_TENSOR_F16 && - !metal_graph_use_reference_hc_decode() && + metal_graph_use_hc_norm_mix_f16(layer->hc_ffn_fn, + hc_dim, + mix_hc) && getenv("DS4_METAL_DISABLE_PRE_M5_HC_NORM_MIX_FUSE") == NULL && (ds4_gpu_device_is_pre_m5_apple_silicon() || - ds4_gpu_device_is_m5_apple_silicon()) && - ds4_gpu_hc_rms_norm_mix_f16_available() != 0; + ds4_gpu_device_is_m5_apple_silicon()); #if defined(__APPLE__) const bool fuse_producer_pre_norm = fuse_norm_mix && fuse_hc_norm && @@ -25309,6 +25473,7 @@ static bool metal_graph_encode_decode_layer_phase( metal_graph_decode_cuda_selected_slots_expected(g, layer); const bool overlap_selected_shared = ok && + !external_routed && g->tp_world < 2 && !decode_stage_profile && !metal_graph_decode_cpu_router_applicable(g, layer) && @@ -25327,6 +25492,7 @@ static bool metal_graph_encode_decode_layer_phase( cuda_selected_shared_overlap); const bool selected_readahead_shared_delay = ok && + !external_routed && g->tp_world < 2 && !overlap_selected_shared && !decode_stage_profile && @@ -25337,6 +25503,7 @@ static bool metal_graph_encode_decode_layer_phase( getenv("DS4_MOE_REPLAY_SELECTED_IDS") == NULL; const bool cuda_stream_selected_load = ok && + !external_routed && !overlap_selected_shared && !selected_readahead_shared_delay && g->ssd_streaming && @@ -29322,7 +29489,8 @@ static bool metal_graph_encode_layer_attention_batch( DS4_METAL_PROFILE_ATTN_STAGE("norm"); DS4_METAL_PROFILE_Q_STAGE("pre_q"); bool qkv_q4_pair_projected = false; - if (ok && qkv_rms_fused && n_tokens >= 2u && + if (ok && qkv_rms_fused && g->cuda_qkv_pair && + !metal_graph_use_reference_qkv_pair_proj() && n_tokens >= 2u && layer->attn_q_a->type == DS4_TENSOR_Q4_K && layer->attn_kv->type == DS4_TENSOR_Q4_K) { qkv_q4_pair_projected = ds4_gpu_matmul_q4_K_pair_tensor( @@ -32585,7 +32753,7 @@ static bool dspark_draft_block_ready( !g->dspark_draft_tokens || !g->dspark_draft_hc || dw->block_size == 0 || dw->block_size > DS4_DSPARK_MAX_BLOCK_SIZE || - g->dspark_block_size != dw->block_size || + g->dspark_block_size < dw->block_size || !dw->has_noise_token_id) { return false; } @@ -32601,7 +32769,7 @@ static bool dspark_stage_input_ready( if (!g || !dw || dw->block_size == 0 || dw->block_size > DS4_DSPARK_MAX_BLOCK_SIZE || - g->dspark_block_size != dw->block_size || + g->dspark_block_size < dw->block_size || !g->dspark_main_x || !g->dspark_draft_hc || !g->dspark_target_hc || !g->dspark_stage_input_hc) { return false; @@ -32710,8 +32878,8 @@ static bool metal_graph_embed_dspark_draft_rows( const ds4_weights *base_weights, const ds4_dspark_weights *dw, int token) { - if (!g || !base_model || !base_weights || !base_weights->token_embd || - !dw || !g->dspark_draft_hc || dw->block_size == 0) { + if (!base_model || + !dspark_draft_block_ready(g, base_weights, dw, token)) { return false; } @@ -36628,16 +36796,18 @@ static bool metal_graph_verify_suffix_tops_impl( const ds4_weights *weights, const token_vec *prompt, uint32_t start, - uint32_t n_tokens, + uint32_t eval_rows, + uint32_t top_rows, bool capture_prefix1, bool capture_dspark_hidden, int *row_tops, float *row_logits, ds4_verify_suffix_timing *timing) { if (timing) memset(timing, 0, sizeof(*timing)); - if (n_tokens == 0 || n_tokens > g->prefill_cap || !g->spec_logits) return false; - if (start > (uint32_t)prompt->len || n_tokens > (uint32_t)prompt->len - start) return false; - const uint32_t top_rows = n_tokens > 1 ? n_tokens - 1 : 0; + if (eval_rows == 0 || eval_rows > g->prefill_cap || + top_rows > eval_rows || !g->spec_logits) return false; + if (start > (uint32_t)prompt->len || + eval_rows > (uint32_t)prompt->len - start) return false; if (top_rows && !row_tops) return false; /* A dynamic SSD decode may leave only the output-head view installed. @@ -36655,20 +36825,23 @@ static bool metal_graph_verify_suffix_tops_impl( } const double upload_t0 = timing ? now_sec() : 0.0; - bool ok = metal_graph_upload_prompt_tokens(metal_graph_prefill_tokens(g), prompt, start, n_tokens); + bool ok = metal_graph_upload_prompt_tokens(metal_graph_prefill_tokens(g), + prompt, + start, + eval_rows); if (ok) ok = metal_graph_upload_prompt_embeddings_hc(metal_graph_batch_cur_hc(g), metal_graph_prefill_tokens(g), model, weights, prompt, start, - n_tokens); + eval_rows); if (!ok) return false; const bool saved_capture = g->spec_capture_prefixes; g->spec_capture_prefixes = - capture_prefix1 && n_tokens > 1u && - n_tokens <= DS4_SPEC_PREFIX_SLOTS + 1u; + capture_prefix1 && eval_rows > 1u && + eval_rows <= DS4_SPEC_PREFIX_SLOTS + 1u; const char *split_head_env = getenv("DS4_DSPARK_VERIFY_SPLIT_HEAD"); const bool fuse_head = !split_head_env || !split_head_env[0] || @@ -36687,8 +36860,8 @@ static bool metal_graph_verify_suffix_tops_impl( * layer reconstructs the routed result while preserving their KV state. */ g->tp_batch_rows = (g->tp_world == 2 && g->tp_batch_out != NULL && g->tp_batch_in != NULL && - n_tokens <= (uint32_t)DS4_TP_BATCH_MAX_ROWS) - ? n_tokens : 0; + eval_rows <= (uint32_t)DS4_TP_BATCH_MAX_ROWS) + ? eval_rows : 0; #ifdef DS4_ROCM_BUILD bool rocm_dspark_fast = false; const char *verify_fast_env = @@ -36698,7 +36871,7 @@ static bool metal_graph_verify_suffix_tops_impl( verify_fast_env[0] != '0' : ds4_dspark_rocm_gfx1151_fast_path(); rocm_dspark_fast = - n_tokens >= 2u && n_tokens <= 6u && verify_fast_enabled; + eval_rows >= 2u && eval_rows <= 6u && verify_fast_enabled; if (rocm_dspark_fast) ds4_gpu_set_dspark_verify_mode(true); #endif const double layer_t0 = timing ? now_sec() : 0.0; @@ -36708,7 +36881,7 @@ static bool metal_graph_verify_suffix_tops_impl( capture_dspark_hidden && metal_graph_dspark_capture_verified_suffix_begin(g, start, - n_tokens, + eval_rows, true); static int verify_profile_left = -1; if (verify_profile_left < 0) { @@ -36729,12 +36902,12 @@ static bool metal_graph_verify_suffix_tops_impl( &weights->layer[il], il, start, - n_tokens); + eval_rows); if (ok && dspark_capture_active) { ok = metal_graph_dspark_capture_verified_suffix_layer(g, il, start, - n_tokens); + eval_rows); } if (ok && selected_profile) { ok = ds4_gpu_end_commands() != 0 && @@ -36742,7 +36915,7 @@ static bool metal_graph_verify_suffix_tops_impl( g, &weights->layer[il], il, - n_tokens, + eval_rows, "DSpark verifier selected profile") && ds4_gpu_begin_commands() != 0; } @@ -36762,7 +36935,7 @@ static bool metal_graph_verify_suffix_tops_impl( ok = metal_graph_encode_output_head_batch(g, model, weights, - n_tokens, + eval_rows, weights->output->dim[1]); } if (ok && fuse_head) { @@ -36801,7 +36974,7 @@ static bool metal_graph_verify_suffix_tops_impl( if (ok) ok = metal_graph_encode_output_head_batch(g, model, weights, - n_tokens, + eval_rows, weights->output->dim[1]); if (ok) { if (top_rows == 1) { @@ -36857,7 +37030,8 @@ static bool metal_graph_verify_suffix_tops_impl( ok = ds4_gpu_tensor_read(g->spec_logits, 0, row_logits, - (uint64_t)n_tokens * DS4_N_VOCAB * sizeof(row_logits[0])) != 0; + (uint64_t)eval_rows * DS4_N_VOCAB * + sizeof(row_logits[0])) != 0; } if (timing) timing->read_ms += (now_sec() - read_t0) * 1000.0; return ok; @@ -36881,6 +37055,8 @@ static bool metal_graph_verify_suffix_tops( const bool ok = metal_graph_verify_suffix_tops_impl(g, model, weights, prompt, start, n_tokens, + n_tokens > 1u ? + n_tokens - 1u : 0u, capture_prefix1, capture_dspark_hidden, row_tops, row_logits, @@ -36889,6 +37065,36 @@ static bool metal_graph_verify_suffix_tops( return ok; } +/* DSpark's current target logits already verify draft[0]. A rollback/replay + * cycle therefore needs target rows only for draft[0..N-2], whose N-1 tops + * verify draft[1..N-1]. This dedicated entry point deliberately disables + * transient prefix/hidden captures because every verifier side effect is + * rolled back before the exact replay. */ +static bool metal_graph_verify_suffix_acceptance_tops( + ds4_gpu_graph *g, + const ds4_model *model, + const ds4_weights *weights, + const token_vec *prompt, + uint32_t start, + uint32_t draft_n, + int *row_tops, + ds4_verify_suffix_timing *timing) { + if (draft_n <= 1u) return false; + const uint32_t eval_rows = draft_n - 1u; + ds4_gpu_tp_keepalive_pause(1); + const bool ok = metal_graph_verify_suffix_tops_impl(g, model, weights, + prompt, start, + eval_rows, + eval_rows, + false, + false, + row_tops, + NULL, + timing); + ds4_gpu_tp_keepalive_pause(0); + return ok; +} + static bool metal_graph_read_spec_logits_row(ds4_gpu_graph *g, uint32_t row, float *logits) { if (!g || !g->spec_logits || !logits || row >= g->prefill_cap) return false; const uint64_t row_bytes = (uint64_t)DS4_N_VOCAB * sizeof(float); @@ -37140,6 +37346,539 @@ static bool metal_graph_verify_decode2_exact_impl( return ok; } +enum { DS4_METAL_EXACTN_UNION_MAX_ROWS = 5 }; + +/* Row-local aliases needed across the TO_ROUTER/FROM_ROUTER split. The + * ordinary decode tape intentionally reuses its Class-P tensors at every + * dispatch. That is safe for a complete row, but not when N router prefixes + * are encoded before any routed tail consumes them. These views borrow the + * existing prefill workspace, so no new resident allocation is needed. */ +typedef struct { + int tier; + bool active; + ds4_gpu_tensor *after_attn_hc; + ds4_gpu_tensor *ffn_cur; + ds4_gpu_tensor *ffn_norm; + ds4_gpu_tensor *hc_split; + ds4_gpu_tensor *hc_pre; + ds4_gpu_tensor *hc_post; + ds4_gpu_tensor *hc_comb; + ds4_gpu_tensor *router_selected; + ds4_gpu_tensor *router_weights; + ds4_gpu_tensor *saved_cur_hc; + ds4_gpu_tensor *saved_after_ffn_hc; + ds4_gpu_tensor *saved_after_attn_hc; + ds4_gpu_tensor *saved_ffn_cur; + ds4_gpu_tensor *saved_ffn_norm; + ds4_gpu_tensor *saved_hc_split; + ds4_gpu_tensor *saved_hc_pre; + ds4_gpu_tensor *saved_hc_post; + ds4_gpu_tensor *saved_hc_comb; + ds4_gpu_tensor *saved_router_selected; + ds4_gpu_tensor *saved_router_weights; +} metal_graph_exactn_union_row_alias; + +static void metal_graph_release_exactn_union_row_alias( + ds4_gpu_graph *g, + metal_graph_exactn_union_row_alias *alias) { + if (!alias) return; + if (g && alias->active) { + const int t = alias->tier; + g->cur_hc_by_tier[t] = alias->saved_cur_hc; + g->after_ffn_hc_by_tier[t] = alias->saved_after_ffn_hc; + g->after_attn_hc_by_tier[t] = alias->saved_after_attn_hc; + g->ffn_cur_by_tier[t] = alias->saved_ffn_cur; + g->ffn_norm_by_tier[t] = alias->saved_ffn_norm; + g->hc_split_by_tier[t] = alias->saved_hc_split; + g->hc_pre_by_tier[t] = alias->saved_hc_pre; + g->hc_post_by_tier[t] = alias->saved_hc_post; + g->hc_comb_by_tier[t] = alias->saved_hc_comb; + g->router_selected_by_tier[t] = alias->saved_router_selected; + g->router_weights_by_tier[t] = alias->saved_router_weights; + } + ds4_gpu_tensor_free(alias->router_weights); + ds4_gpu_tensor_free(alias->router_selected); + ds4_gpu_tensor_free(alias->hc_comb); + ds4_gpu_tensor_free(alias->hc_post); + ds4_gpu_tensor_free(alias->hc_pre); + ds4_gpu_tensor_free(alias->hc_split); + ds4_gpu_tensor_free(alias->ffn_norm); + ds4_gpu_tensor_free(alias->ffn_cur); + ds4_gpu_tensor_free(alias->after_attn_hc); + memset(alias, 0, sizeof(*alias)); +} + +static DS4_MAYBE_UNUSED bool metal_graph_bind_exactn_union_row( + ds4_gpu_graph *g, + uint32_t row, + ds4_gpu_tensor *cur_hc, + ds4_gpu_tensor *next_hc, + metal_graph_exactn_union_row_alias *alias) { + if (!alias) return false; + if (!g || !cur_hc || !next_hc || + g->active_tier < 0 || row >= g->prefill_cap || alias->active) { + return false; + } + if (!alias->after_attn_hc) { + memset(alias, 0, sizeof(*alias)); + alias->tier = g->active_tier; + const uint64_t hc_dim = (uint64_t)DS4_N_HC * DS4_N_EMBD; + const uint64_t mix_hc = 2ull * DS4_N_HC + + (uint64_t)DS4_N_HC * DS4_N_HC; + alias->after_attn_hc = ds4_gpu_tensor_view( + metal_graph_batch_after_attn_hc(g), + (uint64_t)row * hc_dim * sizeof(float), + hc_dim * sizeof(float)); + alias->ffn_cur = ds4_gpu_tensor_view( + metal_graph_batch_ffn_cur(g), + (uint64_t)row * DS4_N_EMBD * sizeof(float), + (uint64_t)DS4_N_EMBD * sizeof(float)); + alias->ffn_norm = ds4_gpu_tensor_view( + metal_graph_batch_ffn_norm(g), + (uint64_t)row * DS4_N_EMBD * sizeof(float), + (uint64_t)DS4_N_EMBD * sizeof(float)); + alias->hc_split = ds4_gpu_tensor_view( + metal_graph_batch_hc_split(g), + (uint64_t)row * mix_hc * sizeof(float), + mix_hc * sizeof(float)); + alias->hc_pre = ds4_gpu_tensor_view( + alias->hc_split, + 0, + (uint64_t)DS4_N_HC * sizeof(float)); + alias->hc_post = ds4_gpu_tensor_view( + alias->hc_split, + (uint64_t)DS4_N_HC * sizeof(float), + (uint64_t)DS4_N_HC * sizeof(float)); + alias->hc_comb = ds4_gpu_tensor_view( + alias->hc_split, + 2ull * DS4_N_HC * sizeof(float), + (uint64_t)DS4_N_HC * DS4_N_HC * sizeof(float)); + alias->router_selected = ds4_gpu_tensor_view( + metal_graph_batch_router_selected(g), + (uint64_t)row * DS4_N_EXPERT_USED * sizeof(int32_t), + (uint64_t)DS4_N_EXPERT_USED * sizeof(int32_t)); + alias->router_weights = ds4_gpu_tensor_view( + metal_graph_batch_router_weights(g), + (uint64_t)row * DS4_N_EXPERT_USED * sizeof(float), + (uint64_t)DS4_N_EXPERT_USED * sizeof(float)); + } + if (!alias->after_attn_hc || !alias->ffn_cur || !alias->ffn_norm || + !alias->hc_split || !alias->hc_pre || !alias->hc_post || + !alias->hc_comb || !alias->router_selected || + !alias->router_weights || alias->tier != g->active_tier) { + metal_graph_release_exactn_union_row_alias(g, alias); + return false; + } + + const int t = alias->tier; + alias->saved_cur_hc = g->cur_hc_by_tier[t]; + alias->saved_after_ffn_hc = g->after_ffn_hc_by_tier[t]; + alias->saved_after_attn_hc = g->after_attn_hc_by_tier[t]; + alias->saved_ffn_cur = g->ffn_cur_by_tier[t]; + alias->saved_ffn_norm = g->ffn_norm_by_tier[t]; + alias->saved_hc_split = g->hc_split_by_tier[t]; + alias->saved_hc_pre = g->hc_pre_by_tier[t]; + alias->saved_hc_post = g->hc_post_by_tier[t]; + alias->saved_hc_comb = g->hc_comb_by_tier[t]; + alias->saved_router_selected = g->router_selected_by_tier[t]; + alias->saved_router_weights = g->router_weights_by_tier[t]; + g->cur_hc_by_tier[t] = cur_hc; + g->after_ffn_hc_by_tier[t] = next_hc; + g->after_attn_hc_by_tier[t] = alias->after_attn_hc; + g->ffn_cur_by_tier[t] = alias->ffn_cur; + g->ffn_norm_by_tier[t] = alias->ffn_norm; + g->hc_split_by_tier[t] = alias->hc_split; + g->hc_pre_by_tier[t] = alias->hc_pre; + g->hc_post_by_tier[t] = alias->hc_post; + g->hc_comb_by_tier[t] = alias->hc_comb; + g->router_selected_by_tier[t] = alias->router_selected; + g->router_weights_by_tier[t] = alias->router_weights; + alias->active = true; + return true; +} + +static DS4_MAYBE_UNUSED void metal_graph_unbind_exactn_union_row( + ds4_gpu_graph *g, + metal_graph_exactn_union_row_alias *alias) { + if (!g || !alias || !alias->active) return; + const int t = alias->tier; + g->cur_hc_by_tier[t] = alias->saved_cur_hc; + g->after_ffn_hc_by_tier[t] = alias->saved_after_ffn_hc; + g->after_attn_hc_by_tier[t] = alias->saved_after_attn_hc; + g->ffn_cur_by_tier[t] = alias->saved_ffn_cur; + g->ffn_norm_by_tier[t] = alias->saved_ffn_norm; + g->hc_split_by_tier[t] = alias->saved_hc_split; + g->hc_pre_by_tier[t] = alias->saved_hc_pre; + g->hc_post_by_tier[t] = alias->saved_hc_post; + g->hc_comb_by_tier[t] = alias->saved_hc_comb; + g->router_selected_by_tier[t] = alias->saved_router_selected; + g->router_weights_by_tier[t] = alias->saved_router_weights; + alias->active = false; +} + +/* Exact speculative Metal verifier with one selected-expert union per layer. + * + * Attention and router prefixes remain canonical one-token decode dispatches, + * in autoregressive row order. A hard router readback boundary then lets the + * backend load the union of N x top-k experts once and expose immutable + * per-row address tables to the canonical routed tails. No command boundary + * is permitted between set_row() and routed MoE, or between routed rows. + * + * The helper only reports acceptance. Callers own the pre-cycle frontier and + * must restore it on a partial match or backend error. */ +static bool metal_graph_verify_decode_exactn_union_impl( + ds4_gpu_graph *g, + const ds4_model *model, + const ds4_weights *weights, + const int *tokens, + uint32_t n_tokens, + uint32_t start, + int *row_tops, + float *last_logits) { +#if defined(__APPLE__) + const bool exact_rows_profile = + getenv("DS4_METAL_DSPARK_EXACT_ROWS_PROFILE") != NULL; + if (!g || !model || !weights || !tokens || !row_tops || !last_logits || + n_tokens < 2u || n_tokens > DS4_METAL_EXACTN_UNION_MAX_ROWS || + n_tokens > g->prefill_cap || g->raw_cap == 0 || + !g->ssd_streaming || g->placement != NULL || g->tp_world > 1u || + g->active_tier != 0 || g->emb_tier != 0 || g->head_tier != 0 || + g->decode_stage_profile || g_expert_profile.active || + metal_graph_debug_get_config()->prefix != NULL || + metal_graph_hc_norm_fusion_check_enabled() || + g->spec_exactn_union_collect_routes || + getenv("DS4_TP_ABLATE") != NULL || + getenv("DS4_METAL_SELECTED_PROFILE") != NULL || + getenv("DS4_METAL_Q4_SELECTED_PROFILE") != NULL || + !g->spec_logits || + ds4_gpu_tensor_bytes(g->spec_logits) < + (uint64_t)n_tokens * DS4_N_VOCAB * sizeof(float)) { + if (exact_rows_profile) { + fprintf(stderr, + "ds4: Metal exact-row union preflight rejected " + "rows=%u prefill=%u raw=%u ssd=%u placement=%u tp=%u " + "active=%d emb=%d head=%d spec_logits=%llu\n", + n_tokens, + g ? g->prefill_cap : 0u, + g ? g->raw_cap : 0u, + g ? (unsigned)g->ssd_streaming : 0u, + g && g->placement ? 1u : 0u, + g ? g->tp_world : 0u, + g ? g->active_tier : -1, + g ? g->emb_tier : -1, + g ? g->head_tier : -1, + (unsigned long long)(g && g->spec_logits + ? ds4_gpu_tensor_bytes(g->spec_logits) : 0u)); + } + return false; + } + for (uint32_t il = 0; il < DS4_N_LAYER; il++) { + const ds4_layer_weights *layer = &weights->layer[il]; + if (!weights_layer_has_required(layer, il) || + !metal_graph_decode_iq2_selected_slots_expected(g, layer) || + metal_graph_decode_cpu_router_applicable(g, layer)) { + if (exact_rows_profile) { + fprintf(stderr, + "ds4: Metal exact-row union layout rejected layer=%u " + "required=%u iq2_slots=%u cpu_router=%u hash_router=%u\n", + il, + (unsigned)weights_layer_has_required(layer, il), + (unsigned)metal_graph_decode_iq2_selected_slots_expected( + g, layer), + (unsigned)metal_graph_decode_cpu_router_applicable( + g, layer), + layer->ffn_gate_tid2eid != NULL ? 1u : 0u); + } + return false; + } + } + + const uint64_t hc_dim = (uint64_t)DS4_N_HC * DS4_N_EMBD; + ds4_gpu_tensor *cur_rows[DS4_METAL_EXACTN_UNION_MAX_ROWS] = {0}; + ds4_gpu_tensor *next_rows[DS4_METAL_EXACTN_UNION_MAX_ROWS] = {0}; + metal_graph_exactn_union_row_alias + row_aliases[DS4_METAL_EXACTN_UNION_MAX_ROWS] = {0}; + bool ok = true; + for (uint32_t row = 0; ok && row < n_tokens; row++) { + cur_rows[row] = ds4_gpu_tensor_view( + metal_graph_batch_cur_hc(g), + (uint64_t)row * hc_dim * sizeof(float), + hc_dim * sizeof(float)); + next_rows[row] = ds4_gpu_tensor_view( + metal_graph_batch_next_hc(g), + (uint64_t)row * hc_dim * sizeof(float), + hc_dim * sizeof(float)); + ok = cur_rows[row] && next_rows[row]; + } + + ds4_gpu_tensor *selected_rows = NULL; + if (ok) { + selected_rows = ds4_gpu_tensor_view( + metal_graph_batch_router_selected(g), + 0, + (uint64_t)n_tokens * DS4_N_EXPERT_USED * sizeof(int32_t)); + ok = selected_rows != NULL; + } + + /* Embeddings are independent rows and can share the same command batch; + * the layer tape below never uses the ordinary cur_hc allocation. */ + if (ok) ok = ds4_gpu_begin_commands() != 0; + bool commands_open = ok; + for (uint32_t row = 0; ok && row < n_tokens; row++) { + ok = ds4_gpu_embed_token_hc_tensor( + cur_rows[row], + model->map, + model->size, + weights->token_embd->abs_offset, + (uint32_t)weights->token_embd->dim[1], + (uint32_t)tokens[row], + DS4_N_EMBD, + DS4_N_HC) != 0; + } + + const bool saved_capture = g->spec_capture_prefixes; + g->spec_capture_prefixes = false; + bool exact_rows_active = false; + bool exact_rows_collecting = false; + for (uint32_t il = 0; ok && il < DS4_N_LAYER; il++) { + const ds4_layer_weights *layer = &weights->layer[il]; + if (ok) { + ok = ds4_gpu_stream_expert_exact_rows_begin_collect() != 0; + exact_rows_collecting = ok; + } + g->spec_exactn_union_collect_routes = ok; + for (uint32_t row = 0; ok && row < n_tokens; row++) { + const uint32_t pos = start + row; + ok = metal_graph_bind_exactn_union_row( + g, row, cur_rows[row], next_rows[row], + &row_aliases[row]); + if (ok) { + ok = metal_graph_encode_decode_layer_phase( + g, + model, + layer, + il, + pos, + g->layer_raw_cache[il], + g->raw_cap, + pos % g->raw_cap, + metal_graph_raw_span_for_batch(g, pos, 1), + tokens[row], + METAL_DECODE_LAYER_TO_ROUTER); + } + metal_graph_unbind_exactn_union_row(g, &row_aliases[row]); + } + g->spec_exactn_union_collect_routes = false; + + const uint64_t gate_row_bytes = + routed_expert_row_bytes(layer->ffn_gate_exps); + const uint64_t down_row_bytes = + routed_expert_row_bytes(layer->ffn_down_exps); + if (ok && + (gate_row_bytes == 0 || down_row_bytes == 0 || + layer->ffn_gate_exps->dim[1] > UINT64_MAX / gate_row_bytes || + layer->ffn_down_exps->dim[1] > UINT64_MAX / down_row_bytes)) { + ok = false; + } + const uint64_t gate_expert_bytes = ok + ? layer->ffn_gate_exps->dim[1] * gate_row_bytes : 0; + const uint64_t down_expert_bytes = ok + ? layer->ffn_down_exps->dim[1] * down_row_bytes : 0; + if (ok) { + const ds4_gpu_stream_expert_table table = + graph_stream_expert_table_make(model, + layer, + il, + gate_expert_bytes, + down_expert_bytes); + ok = ds4_gpu_stream_expert_exact_rows_prepare( + &table, + selected_rows, + n_tokens, + DS4_N_EXPERT_USED) != 0; + exact_rows_active = ok; + } + + for (uint32_t row = 0; ok && row < n_tokens; row++) { + const uint32_t pos = start + row; + ok = metal_graph_bind_exactn_union_row( + g, row, cur_rows[row], next_rows[row], + &row_aliases[row]); + if (ok) { + ok = ds4_gpu_stream_expert_exact_rows_set_row(row) != 0; + } + if (ok) { + ok = metal_graph_encode_decode_layer_phase( + g, + model, + layer, + il, + pos, + g->layer_raw_cache[il], + g->raw_cap, + pos % g->raw_cap, + metal_graph_raw_span_for_batch(g, pos, 1), + tokens[row], + METAL_DECODE_LAYER_FROM_ROUTER); + } + metal_graph_unbind_exactn_union_row(g, &row_aliases[row]); + } + /* The exact-row scope owns the private address buffers referenced by + * every routed tail. Drain that command buffer before dropping its + * strong refs; the next layer starts in a fresh batch. */ + if (exact_rows_active) { + if (commands_open) { + if (ok) { + const bool end_ok = ds4_gpu_end_commands() != 0; + if (!end_ok) (void)ds4_gpu_synchronize(); + ok = end_ok; + } else { + (void)ds4_gpu_end_commands(); + (void)ds4_gpu_synchronize(); + } + commands_open = false; + } + } + if (exact_rows_collecting) { + ds4_gpu_stream_expert_exact_rows_release(); + exact_rows_collecting = false; + exact_rows_active = false; + } + if (ok) { + for (uint32_t row = 0; row < n_tokens; row++) { + ds4_gpu_tensor *tmp = cur_rows[row]; + cur_rows[row] = next_rows[row]; + next_rows[row] = tmp; + } + if (il + 1u < DS4_N_LAYER) { + ok = ds4_gpu_begin_commands() != 0; + commands_open = ok; + } + } + } + g->spec_exactn_union_collect_routes = false; + if (exact_rows_active) { + if (commands_open) { + (void)ds4_gpu_end_commands(); + (void)ds4_gpu_synchronize(); + commands_open = false; + } + } + if (exact_rows_collecting) { + ds4_gpu_stream_expert_exact_rows_release(); + exact_rows_collecting = false; + exact_rows_active = false; + } + if (commands_open) { + if (ok) { + ok = ds4_gpu_end_commands() != 0; + } else { + (void)ds4_gpu_synchronize(); + } + commands_open = false; + } + g->spec_capture_prefixes = saved_capture; + + /* Preserve every output row in spec_logits so all canonical output-head + * dispatches and argmax reductions can remain in one command batch. */ + ds4_gpu_tensor *logits_rows[DS4_METAL_EXACTN_UNION_MAX_ROWS] = {0}; + ds4_gpu_tensor *top_rows[DS4_METAL_EXACTN_UNION_MAX_ROWS - 1u] = {0}; + ds4_gpu_tensor *top_span = NULL; + if (ok) { + top_span = ds4_gpu_tensor_view( + metal_graph_batch_router_selected(g), + 0, + (uint64_t)(n_tokens - 1u) * sizeof(int32_t)); + ok = top_span != NULL; + } + for (uint32_t row = 0; ok && row < n_tokens; row++) { + logits_rows[row] = ds4_gpu_tensor_view( + g->spec_logits, + (uint64_t)row * DS4_N_VOCAB * sizeof(float), + (uint64_t)DS4_N_VOCAB * sizeof(float)); + ok = logits_rows[row] != NULL; + if (ok && row + 1u < n_tokens) { + top_rows[row] = ds4_gpu_tensor_view( + top_span, + (uint64_t)row * sizeof(int32_t), + sizeof(int32_t)); + ok = top_rows[row] != NULL; + } + } + + ds4_gpu_tensor *saved_cur = g->cur_hc_by_tier[0]; + ds4_gpu_tensor *saved_after = g->after_ffn_hc_by_tier[0]; + ds4_gpu_tensor *saved_logits = g->logits_by_tier[0]; + if (ok) ok = ds4_gpu_begin_commands() != 0; + commands_open = ok; + for (uint32_t row = 0; ok && row < n_tokens; row++) { + g->cur_hc_by_tier[0] = cur_rows[row]; + g->logits_by_tier[0] = logits_rows[row]; + ok = metal_graph_encode_output_head( + g, model, weights, weights->output->dim[1]); + if (ok && row + 1u < n_tokens) { + ok = ds4_gpu_argmax_tensor(top_rows[row], + logits_rows[row], + DS4_N_VOCAB) != 0; + } + } + g->cur_hc_by_tier[0] = saved_cur; + g->after_ffn_hc_by_tier[0] = saved_after; + g->logits_by_tier[0] = saved_logits; + if (commands_open) { + if (ok) { + ok = ds4_gpu_end_commands() != 0; + } else { + (void)ds4_gpu_synchronize(); + } + } + + int32_t tops_i32[DS4_METAL_EXACTN_UNION_MAX_ROWS - 1u] = {0}; + if (ok) { + ok = ds4_gpu_tensor_read( + top_span, + 0, + tops_i32, + (uint64_t)(n_tokens - 1u) * sizeof(tops_i32[0])) != 0 && + ds4_gpu_tensor_read( + logits_rows[n_tokens - 1u], + 0, + last_logits, + (uint64_t)DS4_N_VOCAB * sizeof(last_logits[0])) != 0; + } + if (ok) { + for (uint32_t row = 0; row + 1u < n_tokens; row++) { + row_tops[row] = tops_i32[row]; + } + } + for (uint32_t row = 0; row + 1u < n_tokens; row++) { + ds4_gpu_tensor_free(top_rows[row]); + } + ds4_gpu_tensor_free(top_span); + for (uint32_t row = 0; row < n_tokens; row++) { + ds4_gpu_tensor_free(logits_rows[row]); + } + ds4_gpu_tensor_free(selected_rows); + for (uint32_t row = 0; row < n_tokens; row++) { + metal_graph_release_exactn_union_row_alias( + g, &row_aliases[row]); + ds4_gpu_tensor_free(next_rows[row]); + ds4_gpu_tensor_free(cur_rows[row]); + } + return ok; +#else + (void)g; + (void)model; + (void)weights; + (void)tokens; + (void)n_tokens; + (void)start; + (void)row_tops; + (void)last_logits; + return false; +#endif +} + static bool metal_graph_verify_decode2_exact( ds4_gpu_graph *g, const ds4_model *model, @@ -38362,6 +39101,10 @@ struct ds4_engine { bool vision_ready; bool vision_map_ready; bool share_session_prefill_workspace; +#if defined(__APPLE__) && !defined(DS4_NO_GPU) + void *dspark_hot_lock_addr; + size_t dspark_hot_lock_len; +#endif #ifndef DS4_NO_GPU bool shared_prefill_workspace_ready; ds4_gpu_graph shared_prefill_workspace; @@ -38386,6 +39129,78 @@ struct ds4_engine { int placement_session_count_hint; }; +#if defined(__APPLE__) && !defined(DS4_NO_GPU) +/* The first DSpark projection is reused by every proposal and, under Metal + * SSD streaming, otherwise remains a pageable no-copy view of the support + * GGUF. This opt-in lock keeps only stage-0 main_norm + main_proj hot instead + * of pinning the full 5.6 GiB support model. Failure is deliberately + * non-fatal: the existing pageable path remains correct. */ +static void ds4_engine_metal_dspark_lock_stage0_hotset(ds4_engine *e) { + const char *env = getenv("DS4_METAL_DSPARK_PIN_MAIN_PROJ"); + if (!e || !env || !env[0] || strcmp(env, "0") == 0 || + e->backend != DS4_BACKEND_METAL || !e->ssd_streaming || !e->dspark || + e->support_kind != DS4_SUPPORT_DSPARK || e->multi_tier || + e->tp.active || !e->mtp_model.map || e->dspark_weights.n_stages == 0) { + return; + } + + const ds4_dspark_stage_weights *stage0 = &e->dspark_weights.stage[0]; + const ds4_tensor *tensors[2] = {stage0->main_norm, stage0->main_proj}; + uint64_t lo = UINT64_MAX; + uint64_t hi = 0; + for (uint32_t i = 0; i < 2; i++) { + const ds4_tensor *t = tensors[i]; + if (!t || t->bytes == 0 || t->abs_offset > e->mtp_model.size || + t->bytes > e->mtp_model.size - t->abs_offset) { + fprintf(stderr, + "ds4: WARNING: Metal DSpark stage-0 hot-set tensor is " + "missing or outside the support mapping; leaving it pageable\n"); + return; + } + const uint64_t end = t->abs_offset + t->bytes; + if (t->abs_offset < lo) lo = t->abs_offset; + if (end > hi) hi = end; + } + + const long page_long = sysconf(_SC_PAGESIZE); + if (page_long <= 0 || lo == UINT64_MAX || hi <= lo) return; + const uint64_t page = (uint64_t)page_long; + const uint64_t aligned_lo = (lo / page) * page; + if (hi > UINT64_MAX - (page - 1u)) return; + const uint64_t aligned_hi = ((hi + page - 1u) / page) * page; + if (aligned_hi <= aligned_lo || aligned_hi - aligned_lo > SIZE_MAX) return; + const size_t lock_len = (size_t)(aligned_hi - aligned_lo); + const size_t max_hotset = 128u * 1024u * 1024u; + if (lock_len > max_hotset) { + fprintf(stderr, + "ds4: WARNING: Metal DSpark stage-0 hot set is %.2f MiB " + "(limit %.2f MiB); leaving it pageable\n", + (double)lock_len / 1048576.0, + (double)max_hotset / 1048576.0); + return; + } + + void *addr = (void *)(e->mtp_model.map + aligned_lo); + const double t0 = now_sec(); + if (mlock(addr, lock_len) != 0) { + const int saved_errno = errno; + fprintf(stderr, + "ds4: WARNING: Metal DSpark could not pin the %.2f MiB " + "stage-0 main projection hot set: %s; leaving it pageable\n", + (double)lock_len / 1048576.0, + strerror(saved_errno)); + return; + } + e->dspark_hot_lock_addr = addr; + e->dspark_hot_lock_len = lock_len; + fprintf(stderr, + "ds4: Metal DSpark pinned %.2f MiB stage-0 main projection hot " + "set in %.3fs\n", + (double)lock_len / 1048576.0, + now_sec() - t0); +} +#endif + static uint64_t ds4_engine_support_model_bytes(const ds4_engine *e) { if (!e || !e->mtp_model.map) return 0; const bool runtime_ready = @@ -53781,6 +54596,23 @@ typedef struct ds4_dspark_spec_stats { uint64_t direct_full_commits; uint64_t direct_partial_commits; uint64_t replay_fallbacks; + uint64_t proposer_capped; + uint64_t proposer_scheduled_rows; + uint64_t exact2_attempts; + uint64_t exact2_full_accepts; + uint64_t exact2_partial_accepts; + uint64_t exact2_fallbacks; + uint64_t exactn_union_attempts; + uint64_t exactn_union_full_accepts; + uint64_t exactn_union_fallbacks; + uint64_t exactn_union_partial_fallbacks; + uint64_t exactn_union_error_fallbacks; + uint64_t exactn_attempts; + uint64_t exactn_full_accepts; + uint64_t exactn_fallbacks; + uint64_t exactn_partial_fallbacks; + uint64_t exactn_error_fallbacks; + uint64_t exactn_boundary_rows; uint64_t first_misses; uint64_t no_draft; uint64_t no_room; @@ -53808,6 +54640,9 @@ typedef struct ds4_dspark_spec_stats { double verify_head_ms; double verify_read_ms; uint64_t verifier_fused_head; + uint64_t metal_acceptance_only_attempts; + uint64_t metal_acceptance_only_rows_saved; + uint64_t metal_replay_headless_tokens; double replay_ms; double total_ms; } ds4_dspark_spec_stats; @@ -53922,27 +54757,150 @@ static bool ds4_session_dspark_seed_batch_enabled( if (env && env[0]) return env[0] != '0'; return ds4_session_dspark_rocm_gfx1151_fast_path(s); } + +/* Exact-2 is currently a resident, single-GPU CUDA experiment. Keep its hard + * backend eligibility in one place; proposal and verification widths remain + * independently controllable for attribution tests. */ +static bool ds4_session_cuda_dspark_exact2_enabled( + const ds4_session *s) { +#if !defined(__APPLE__) && !defined(DS4_ROCM_BUILD) + return s && s->engine && + s->engine->backend == DS4_BACKEND_CUDA && + !s->engine->multi_tier && + !s->engine->tp.active && + !s->graph.ssd_streaming && + s->graph.placement == NULL && + s->graph.prefill_cap >= 3u && + metal_graph_tp_env_flag("DS4_CUDA_DSPARK_EXACT2", false); +#else + (void)s; + return false; +#endif +} + +/* Metal exact-2 reuses the same decode-exact body as CUDA, but starts as an + * SSD-only opt-in until its selected-expert barriers and throughput are + * measured on real Apple hardware. */ +static bool ds4_session_metal_dspark_exact2_enabled( + const ds4_session *s) { +#if defined(__APPLE__) + return s && s->engine && + s->engine->backend == DS4_BACKEND_METAL && + !s->engine->multi_tier && + !s->engine->tp.active && + s->graph.ssd_streaming && + s->graph.placement == NULL && + s->graph.prefill_cap >= 3u && + metal_graph_tp_env_flag("DS4_METAL_DSPARK_EXACT2", false); +#else + (void)s; + return false; +#endif +} + +/* Correctness oracle for a future exact Metal microbatch. It deliberately + * advances each row through the ordinary one-token SSD decode entry point, + * whose command completion and logits readback form a hard boundary between + * rows. Only a fully accepted block is committed directly; partial blocks + * are rolled back and left to the established verifier/replay path. */ +static bool ds4_session_metal_dspark_exactn_enabled( + const ds4_session *s) { +#if defined(__APPLE__) + return s && s->engine && + s->engine->backend == DS4_BACKEND_METAL && + !s->engine->multi_tier && + !s->engine->tp.active && + s->graph.ssd_streaming && + s->graph.placement == NULL && + metal_graph_tp_env_flag("DS4_METAL_DSPARK_EXACTN", false); +#else + (void)s; + return false; +#endif +} + +/* Fast exact-N experiment: keep the canonical one-token tape, but split every + * layer at the router so all speculative rows share one immutable selected- + * expert union. It stays independent of the boundary oracle until both state + * and output comparisons pass on real Metal hardware. */ +static bool ds4_session_metal_dspark_exactn_union_enabled( + const ds4_session *s) { +#if defined(__APPLE__) + return s && s->engine && + s->engine->backend == DS4_BACKEND_METAL && + !s->engine->multi_tier && + !s->engine->tp.active && + s->graph.ssd_streaming && + s->graph.placement == NULL && + metal_graph_tp_env_flag( + "DS4_METAL_DSPARK_EXACTN_UNION", false); +#else + (void)s; + return false; +#endif +} + +/* Keep the proposer width independently controllable from the verifier so a + * CUDA A/B can attribute the gain. An explicit zero keeps the checkpoint's + * native width; otherwise exact-2 defaults to a two-row proposal. */ +static uint32_t ds4_session_cuda_dspark_proposer_block_cap( + const ds4_session *s, + uint32_t native_block_size) { +#if !defined(__APPLE__) && !defined(DS4_ROCM_BUILD) + if (!s || !s->engine || native_block_size == 0 || + s->engine->backend != DS4_BACKEND_CUDA || + s->engine->multi_tier || s->engine->tp.active || + s->graph.ssd_streaming || s->graph.placement != NULL) { + return native_block_size; + } + + const char *env = getenv("DS4_CUDA_DSPARK_PROPOSER_BLOCK_MAX"); + if (env && env[0]) { + char *end = NULL; + errno = 0; + const unsigned long value = strtoul(env, &end, 10); + if (end != env && *end == '\0' && errno == 0 && + value <= UINT32_MAX) { + if (value == 0) return native_block_size; + uint32_t cap = (uint32_t)value; + if (cap > DS4_DSPARK_MAX_BLOCK_SIZE) { + cap = DS4_DSPARK_MAX_BLOCK_SIZE; + } + return cap < native_block_size ? cap : native_block_size; + } + return native_block_size; + } + + const uint32_t verify_cap = ds4_dspark_env_u32( + "DS4_DSPARK_SSD_VERIFY_BLOCK_MAX", 0); + if (s->graph.prefill_cap >= 3u && + ds4_session_cuda_dspark_exact2_enabled(s) && + (verify_cap == 0u || verify_cap == 2u)) { + return native_block_size > 2u ? 2u : native_block_size; + } +#else + (void)s; +#endif + return native_block_size; +} + /* A tiny SSD verifier may touch top-k experts for every speculative row. On * low-memory Metal systems the useful upper bound is the number of complete - * top-k rows that fit in the configured target expert cache. Capping only - * verification (the proposer may still produce its full block) trades a small - * amount of speculative depth for substantially fewer SSD reads. */ + * top-k rows that fit in the configured target expert cache. The Metal + * proposer normally inherits this limit below, while its explicit zero + * override preserves the old full-proposal/capped-verifier A/B path. */ static uint32_t ds4_session_dspark_verify_block_cap(const ds4_session *s) { uint32_t cap = ds4_dspark_env_u32( "DS4_DSPARK_SSD_VERIFY_BLOCK_MAX", 0); -#if !defined(__APPLE__) && !defined(DS4_ROCM_BUILD) - /* Exact-2 is an explicit CUDA experiment. Make the single switch usable - * with the checkpoint's native five-token proposal block by verifying its - * first two drafts; an explicit block cap still takes precedence. */ - if (cap == 0 && s && s->engine && - s->engine->backend == DS4_BACKEND_CUDA && - !s->engine->multi_tier && !s->engine->tp.active && - !s->graph.ssd_streaming && s->graph.placement == NULL && - s->graph.prefill_cap >= 2u && - metal_graph_tp_env_flag("DS4_CUDA_DSPARK_EXACT2", false)) { + /* Exact-2 defaults verification to two rows. The independent proposer + * cap may preserve the checkpoint's native proposal width for A/B tests; + * an explicit verifier cap still takes precedence. */ + if (cap == 0 && + !ds4_session_metal_dspark_exactn_enabled(s) && + (ds4_session_cuda_dspark_exact2_enabled(s) || + ds4_session_metal_dspark_exact2_enabled(s))) { cap = 2; } -#endif if (cap == 0 && s && s->engine && s->engine->backend == DS4_BACKEND_METAL && s->graph.ssd_streaming) { @@ -53957,6 +54915,85 @@ static uint32_t ds4_session_dspark_verify_block_cap(const ds4_session *s) { return cap; } +/* On single-device Metal, schedule only rows the target verifier can consume. + * The automatic SSD cap is cache_slots / top_k (so 16 slots schedule two rows, + * while 30/32 slots retain a native five-row DSpark block). An explicit + * DS4_METAL_DSPARK_PROPOSER_BLOCK_MAX overrides that policy; zero is the A/B + * kill switch and restores the checkpoint's native proposal width. */ +static uint32_t ds4_session_metal_dspark_proposer_block_cap( + const ds4_session *s, + uint32_t native_block_size) { +#if defined(__APPLE__) + if (!s || !s->engine || native_block_size == 0 || + s->engine->backend != DS4_BACKEND_METAL || + s->engine->multi_tier || s->engine->tp.active || + s->graph.placement != NULL) { + return native_block_size; + } + + const char *env = getenv("DS4_METAL_DSPARK_PROPOSER_BLOCK_MAX"); + uint32_t cap = 0; + if (env && env[0]) { + char *end = NULL; + errno = 0; + const unsigned long value = strtoul(env, &end, 10); + if (end == env || *end != '\0' || errno != 0 || + value > UINT32_MAX) { + return native_block_size; + } + if (value == 0) return native_block_size; + cap = (uint32_t)value; + if (cap > DS4_DSPARK_MAX_BLOCK_SIZE) { + cap = DS4_DSPARK_MAX_BLOCK_SIZE; + } + } else { + cap = ds4_session_dspark_verify_block_cap(s); + } + return cap < native_block_size ? cap : native_block_size; +#else + (void)s; + return native_block_size; +#endif +} + +/* Intermediate accepted tokens only advance target state; their logits are + * discarded. Single-device Metal can therefore omit those output heads and + * readbacks. Keep an explicit switch for byte-for-byte A/B diagnostics. */ +static bool ds4_session_metal_dspark_headless_replay_enabled( + const ds4_session *s) { +#if defined(__APPLE__) + return s && s->engine && + s->engine->backend == DS4_BACKEND_METAL && + !s->engine->multi_tier && !s->engine->tp.active && + s->graph.placement == NULL && + metal_graph_tp_env_flag( + "DS4_METAL_DSPARK_HEADLESS_REPLAY", true); +#else + (void)s; + return false; +#endif +} + +/* The base target logits have already accepted draft[0]. A single-device + * Metal cycle that unconditionally rolls verification back can therefore + * evaluate only draft[0..N-2] and still obtain every remaining acceptance + * top. Keep the contraction opt-in: on the M1 Pro SSD path the smaller batch + * did not beat the legacy N-row verifier despite doing less row work. */ +static bool ds4_session_metal_dspark_acceptance_only_verify_enabled( + const ds4_session *s) { +#if defined(__APPLE__) + return s && s->engine && + s->engine->backend == DS4_BACKEND_METAL && + !s->engine->multi_tier && !s->engine->tp.active && + s->graph.placement == NULL && + metal_graph_tp_env_flag( + "DS4_METAL_DSPARK_ACCEPTANCE_ONLY_VERIFY", false); +#else + (void)s; + return false; +#endif +} + /* Tiny-batch verification writes every future raw row before evaluating the * block. A deliberately narrow raw ring must not wrap onto rows still in the * visible SWA window. Default padded rings satisfy this invariant; custom @@ -53973,29 +55010,33 @@ static bool ds4_session_dspark_batch_raw_safe( return raw_cap >= raw_window && draft_n <= raw_cap - raw_window; } -/* Experimental resident-CUDA verifier that advances two draft tokens with - * the ordinary one-token kernels, layer by layer. Keep the first rollout - * deliberately narrow: TP needs a mirrored exact-2 worker protocol, while - * SSD streaming needs per-layer remapping that the exact verifier does not - * perform. */ -static bool ds4_session_cuda_dspark_exact2_requested( +/* Experimental exact verifier that advances two draft tokens with the + * ordinary one-token kernels, layer by layer. CUDA remains resident-only; + * the Metal caller installs its complete SSD target map before entering. */ +static bool ds4_session_dspark_exact2_requested( const ds4_session *s, uint32_t draft_n) { -#if !defined(__APPLE__) && !defined(DS4_ROCM_BUILD) - return s && s->engine && - s->engine->backend == DS4_BACKEND_CUDA && - !s->engine->multi_tier && - !s->engine->tp.active && - !s->graph.ssd_streaming && - s->graph.placement == NULL && - s->graph.prefill_cap >= 2u && - draft_n == 2u && - metal_graph_tp_env_flag("DS4_CUDA_DSPARK_EXACT2", false); -#else - (void)s; - (void)draft_n; - return false; -#endif + return draft_n == 2u && + (ds4_session_cuda_dspark_exact2_enabled(s) || + ds4_session_metal_dspark_exact2_enabled(s)); +} + +static bool ds4_session_metal_dspark_exactn_requested( + const ds4_session *s, + uint32_t draft_n) { + return draft_n >= 2u && + draft_n <= DS4_DSPARK_MAX_BLOCK_SIZE && + draft_n <= 5u && + ds4_session_metal_dspark_exactn_enabled(s); +} + +static bool ds4_session_metal_dspark_exactn_union_requested( + const ds4_session *s, + uint32_t draft_n) { + return draft_n >= 2u && + draft_n <= DS4_DSPARK_MAX_BLOCK_SIZE && + draft_n <= DS4_METAL_EXACTN_UNION_MAX_ROWS && + ds4_session_metal_dspark_exactn_union_enabled(s); } static void ds4_session_dspark_stats_note_saved( @@ -63334,6 +64375,9 @@ static int ds4_engine_open_internal(ds4_engine **out, *out = NULL; return 1; } +#if defined(__APPLE__) && !defined(DS4_NO_GPU) + ds4_engine_metal_dspark_lock_stage0_hotset(e); +#endif } if (e->vision_ready) { #if defined(__APPLE__) @@ -63912,6 +64956,17 @@ void ds4_engine_close(ds4_engine *e) { * for the backend before munmap, especially when main and DSpark support * views coexist in the Metal registry. */ ds4_gpu_cleanup(); +#endif +#if defined(__APPLE__) && !defined(DS4_NO_GPU) + if (e->dspark_hot_lock_addr && e->dspark_hot_lock_len != 0) { + if (munlock(e->dspark_hot_lock_addr, e->dspark_hot_lock_len) != 0) { + fprintf(stderr, + "ds4: WARNING: Metal DSpark stage-0 hot-set unlock failed: %s\n", + strerror(errno)); + } + e->dspark_hot_lock_addr = NULL; + e->dspark_hot_lock_len = 0; + } #endif if (e->mtp_model.map) model_close(&e->mtp_model); if (e->vision_model.map) model_close(&e->vision_model); @@ -63977,7 +65032,17 @@ static void ds4_session_print_dspark_stats(const ds4_session *s) { "ds4: DSpark stats cycles=%llu first_tokens=%llu proposed=%llu " "accepted_draft=%llu accept_rate=%.2f%% avg_accept=%.3f " "full=%llu partial=%llu direct_full=%llu direct_partial=%llu " - "replay_fallbacks=%llu miss_first=%llu no_draft=%llu " + "replay_fallbacks=%llu prop_capped=%llu " + "prop_scheduled_rows=%llu " + "exact2_attempt=%llu exact2_full=%llu exact2_partial=%llu " + "exact2_fallback=%llu exactn_union_attempt=%llu " + "exactn_union_full=%llu exactn_union_fallback=%llu " + "exactn_union_partial_fallback=%llu " + "exactn_union_error_fallback=%llu " + "exactn_attempt=%llu exactn_full=%llu " + "exactn_fallback=%llu exactn_partial_fallback=%llu " + "exactn_error_fallback=%llu exactn_boundary_rows=%llu " + "miss_first=%llu no_draft=%llu " "no_room=%llu invalid=%llu verifier_unavailable=%llu " "errors=%llu time_ms propose=%.3f " "prop_stage0=%.3f prop_setup=%.3f prop_cache=%.3f " @@ -63985,7 +65050,9 @@ static void ds4_session_print_dspark_stats(const ds4_session *s) { "prop_logits=%.3f prop_markov=%.3f prop_confidence=%.3f " "snapshot=%.3f verify=%.3f verify_upload=%.3f " "verify_layer=%.3f verify_head=%.3f verify_read=%.3f " - "verify_fused_head=%llu replay=%.3f spec_total=%.3f " + "verify_fused_head=%llu metal_accept_only=%llu " + "metal_verify_rows_saved=%llu metal_replay_headless=%llu " + "replay=%.3f spec_total=%.3f " "target=%.3f saved=%.3f net_saved=%.3f " "draft_len_hist=%s accepted_len_hist=%s\n", (unsigned long long)st->cycles, @@ -63999,6 +65066,23 @@ static void ds4_session_print_dspark_stats(const ds4_session *s) { (unsigned long long)st->direct_full_commits, (unsigned long long)st->direct_partial_commits, (unsigned long long)st->replay_fallbacks, + (unsigned long long)st->proposer_capped, + (unsigned long long)st->proposer_scheduled_rows, + (unsigned long long)st->exact2_attempts, + (unsigned long long)st->exact2_full_accepts, + (unsigned long long)st->exact2_partial_accepts, + (unsigned long long)st->exact2_fallbacks, + (unsigned long long)st->exactn_union_attempts, + (unsigned long long)st->exactn_union_full_accepts, + (unsigned long long)st->exactn_union_fallbacks, + (unsigned long long)st->exactn_union_partial_fallbacks, + (unsigned long long)st->exactn_union_error_fallbacks, + (unsigned long long)st->exactn_attempts, + (unsigned long long)st->exactn_full_accepts, + (unsigned long long)st->exactn_fallbacks, + (unsigned long long)st->exactn_partial_fallbacks, + (unsigned long long)st->exactn_error_fallbacks, + (unsigned long long)st->exactn_boundary_rows, (unsigned long long)st->first_misses, (unsigned long long)st->no_draft, (unsigned long long)st->no_room, @@ -64022,6 +65106,9 @@ static void ds4_session_print_dspark_stats(const ds4_session *s) { st->verify_head_ms, st->verify_read_ms, (unsigned long long)st->verifier_fused_head, + (unsigned long long)st->metal_acceptance_only_attempts, + (unsigned long long)st->metal_acceptance_only_rows_saved, + (unsigned long long)st->metal_replay_headless_tokens, st->replay_ms, st->total_ms, st->target_ms, @@ -64218,7 +65305,11 @@ int ds4_session_create(ds4_session **out, ds4_engine *e, int ctx_size) { const bool need_spec_verifier = e->mtp_ready || (e->support_kind == DS4_SUPPORT_DSPARK && e->dspark) || - e->tp.active; /* TP worker mirrors the leader's verify blocks */ + e->tp.active /* TP worker mirrors the leader's verify blocks */ +#ifdef DS4_TEST_HOOKS + || getenv("DS4_TEST_METAL_EXACTN_ORACLE") != NULL +#endif + ; const int *placement = e->multi_tier ? e->placement : NULL; const ds4_gpu_graph *shared_prefill_workspace = e->share_session_prefill_workspace && @@ -67116,7 +68207,28 @@ static bool ds4_session_prepare_dspark_draft_impl(ds4_session *s, const bool capture_ok = ds4_session_dspark_capture_current(s); const bool batch_capture_ok = ds4_session_dspark_capture_batch_current(s); - const ds4_dspark_weights *dw = &s->engine->dspark_weights; + const ds4_dspark_weights *native_dw = &s->engine->dspark_weights; + ds4_dspark_weights capped_dw; + const ds4_dspark_weights *dw = native_dw; + /* Backends choose their proposal width independently. CUDA exact-2 + * keeps its isolated A/B control; Metal defaults to the effective + * verifier/cache budget. Buffers retain their native capacity. */ + uint32_t proposer_cap = native_dw->block_size; + if (enabled) { + proposer_cap = ds4_session_cuda_dspark_proposer_block_cap( + s, proposer_cap); + proposer_cap = ds4_session_metal_dspark_proposer_block_cap( + s, proposer_cap); + } + if (proposer_cap < native_dw->block_size) { + capped_dw = *native_dw; + capped_dw.block_size = proposer_cap; + dw = &capped_dw; + if (stats_enabled) s->dspark_stats.proposer_capped++; + } + if (stats_enabled) { + s->dspark_stats.proposer_scheduled_rows += dw->block_size; + } const uint32_t verify_cap = ds4_dspark_env_u32( "DS4_DSPARK_VERIFY_CAP", ds4_session_dspark_rocm_gfx1151_fast_path(s) ? 5u : 0u); @@ -69552,7 +70664,16 @@ static int ds4_session_eval_dspark_speculative_argmax( DS4_DSPARK_STATS_FINISH(); return n_accept; } - if (drafts[0] == eos_token) draft_n = 1; + /* Never verify or commit state beyond EOS. The verifier must still + * evaluate the EOS row itself so its directly committed state matches + * ordinary decode, but every later proposal is outside the generation. */ + for (int i = 0; i < draft_n; i++) { + if (drafts[i] == eos_token) { + draft_n = i + 1; + break; + } + } + ds4_engine *e = s->engine; ds4_spec_frontier frontier; memset(&frontier, 0, sizeof(frontier)); @@ -69566,6 +70687,22 @@ static int ds4_session_eval_dspark_speculative_argmax( #else const bool cuda_rollback_replay = false; #endif + const bool metal_headless_replay = + ds4_session_metal_dspark_headless_replay_enabled(s); + /* A four-row chunk at a ratio-4 boundary selects the aligned compressor + * kernel, while the legacy capture path intentionally uses sequential + * updates. Preserve acceptance arithmetic by retaining the legacy N-row + * verifier on those boundaries; all other N>=3 suffixes naturally take + * the same sequential compressor path after contracting to N-1 rows. */ + const uint32_t metal_acceptance_rows = + draft_n > 1 ? (uint32_t)draft_n - 1u : 0u; + const bool metal_acceptance_aligned_ratio4 = + metal_acceptance_rows != 0u && + (metal_acceptance_rows % 4u) == 0u && + ((uint32_t)start % 4u) == 0u; + const bool metal_acceptance_only_verify = + draft_n > 2 && !metal_acceptance_aligned_ratio4 && + ds4_session_metal_dspark_acceptance_only_verify_enabled(s); /* The current target logits already verify a one-token draft. With no * mirrored TP worker there is no speculative verifier state to roll back: * replay that token directly and avoid copying every compressor frontier. */ @@ -69581,28 +70718,309 @@ static int ds4_session_eval_dspark_speculative_argmax( bool ok = have_frontier && row_logits && (draft_n <= 1 || row_tops); bool verifier_may_have_mutated = false; bool tp_verify_sent = false; - const bool cuda_exact2 = - ok && ds4_session_cuda_dspark_exact2_requested( + + /* Fast Metal exact-N experiment. Unlike the token-major oracle below, + * this advances all rows layer-major and shares one immutable selected- + * expert union per layer. It may directly commit only a complete match; + * every partial/error path restores the pre-cycle frontier before trying + * the oracle (when separately enabled) or the established legacy path. */ + const bool exactn_union = + ok && ds4_session_metal_dspark_exactn_union_requested( + s, (uint32_t)draft_n); + if (exactn_union) { + if (stats_enabled) s->dspark_stats.exactn_union_attempts++; + bool union_ok = true; + bool union_mismatch = false; + int union_accepted_prefix = 1; + int union_last_top = -1; + const double union_t0 = stats_enabled ? now_sec() : 0.0; + + const bool static_map_cache = + metal_graph_stream_decode_static_map_enabled() && + metal_graph_stream_decode_static_map_state_cache_enabled(); + if (!static_map_cache || + !s->graph.streaming_static_decode_map_current) { + union_ok = metal_graph_stream_map_decode_static_all( + &e->model, &e->weights); + if (union_ok) { + s->graph.streaming_static_decode_map_current = + static_map_cache; + } + } + if (union_ok) { + union_ok = metal_graph_verify_decode_exactn_union_impl( + &s->graph, + &e->model, + &e->weights, + drafts, + (uint32_t)draft_n, + (uint32_t)start, + row_tops, + row_logits); + } + if (union_ok) { + for (int row = 0; row + 1 < draft_n; row++) { + union_last_top = row_tops[row]; + if (union_last_top != drafts[row + 1]) { + union_mismatch = true; + break; + } + union_accepted_prefix = row + 2; + } + } + if (stats_enabled) { + s->dspark_stats.verify_ms += + (now_sec() - union_t0) * 1000.0; + } + + if (union_ok && !union_mismatch) { + memcpy(s->logits, + row_logits, + (size_t)DS4_N_VOCAB * sizeof(s->logits[0])); + /* The union tape does not refresh the DSpark target-hidden rows. + * Never relabel the old capture as the newly committed frontier. */ + ds4_session_dspark_capture_invalidate(s); + for (int i = 0; i < draft_n; i++) { + token_vec_push(&s->checkpoint, drafts[i]); + accepted[n_accept++] = drafts[i]; + } + s->checkpoint_valid = true; + ds4_session_dspark_capture_note_checkpoint(s); + if (stats_enabled) { + s->dspark_stats.full_accepts++; + s->dspark_stats.exactn_union_full_accepts++; + s->dspark_stats.accepted_draft_tokens += + (uint64_t)draft_n; + ds4_dspark_stats_note_len( + s->dspark_stats.accepted_len_hist, + (uint32_t)draft_n); + } + ds4_session_dspark_stats_note_saved(s, (uint32_t)draft_n); + if (spec_log) { + fprintf(stderr, + "ds4: DSpark Metal exactN union drafted=%d " + "accepted_draft=%d accepted_total=%d\n", + draft_n, + draft_n, + n_accept); + } + spec_frontier_free(&frontier); + DS4_DSPARK_STATS_FINISH(); + return n_accept; + } + + if (stats_enabled) { + s->dspark_stats.exactn_union_fallbacks++; + if (union_mismatch) { + s->dspark_stats.exactn_union_partial_fallbacks++; + } else { + s->dspark_stats.exactn_union_error_fallbacks++; + } + } + s->checkpoint.len = start; + ds4_session_dspark_capture_invalidate(s); + if (!have_frontier || !spec_frontier_restore(&frontier, s)) { + snprintf(err, errlen, + "DSpark Metal exactN union rollback failed"); + s->checkpoint_valid = false; + if (stats_enabled) { + s->dspark_stats.verifier_errors++; + ds4_dspark_stats_note_len( + s->dspark_stats.accepted_len_hist, 0); + } + spec_frontier_free(&frontier); + DS4_DSPARK_STATS_FINISH(); + return -1; + } + if (spec_log) { + if (union_mismatch) { + fprintf(stderr, + "ds4: DSpark Metal exactN union partial " + "accepted_prefix=%d verified_next=%d; restored " + "frontier for fallback\n", + union_accepted_prefix, + union_last_top); + } else { + fprintf(stderr, + "ds4: DSpark Metal exactN union unavailable; " + "restored frontier for fallback\n"); + } + } + } + + /* Correctness-first exact-N oracle for Metal SSD streaming. Every row + * uses the ordinary decode entry point and completes before the next row + * starts. This intentionally has no microbatch speed claim: it establishes + * the byte-identical state/output reference that a union-route kernel must + * match before its state can be committed. */ + const bool exactn = + ok && ds4_session_metal_dspark_exactn_requested( s, (uint32_t)draft_n); - if (cuda_exact2) { + if (exactn) { + if (stats_enabled) s->dspark_stats.exactn_attempts++; + bool exactn_ok = true; + bool exactn_mismatch = false; + int exactn_accepted_prefix = 1; + int exactn_last_top = -1; + const double exactn_t0 = stats_enabled ? now_sec() : 0.0; + + /* Evaluating draft[i] produces the target logits that verify + * draft[i+1]. sample_probs is private scratch, so s->logits remains + * the pre-cycle row until a full block is known to be committable. */ + for (int i = 0; exactn_ok && i + 1 < draft_n; i++) { + exactn_ok = metal_graph_eval_token_raw_swa( + &s->graph, + &e->model, + &e->weights, + drafts[i], + (uint32_t)(start + i), + s->sample_probs); + if (!exactn_ok) break; + if (stats_enabled) s->dspark_stats.exactn_boundary_rows++; + exactn_last_top = sample_argmax(s->sample_probs, DS4_N_VOCAB); + if (exactn_last_top != drafts[i + 1]) { + exactn_mismatch = true; + break; + } + exactn_accepted_prefix = i + 2; + } + + /* All inter-row tops matched. Advance the last accepted token too, + * yielding both its exact persistent state and continuation logits. */ + if (exactn_ok && !exactn_mismatch) { + exactn_ok = metal_graph_eval_token_raw_swa( + &s->graph, + &e->model, + &e->weights, + drafts[draft_n - 1], + (uint32_t)(start + draft_n - 1), + row_logits); + if (exactn_ok && stats_enabled) { + s->dspark_stats.exactn_boundary_rows++; + } + } + if (stats_enabled) { + s->dspark_stats.verify_ms += + (now_sec() - exactn_t0) * 1000.0; + } + + if (exactn_ok && !exactn_mismatch) { + memcpy(s->logits, + row_logits, + (size_t)DS4_N_VOCAB * sizeof(s->logits[0])); + for (int i = 0; i < draft_n; i++) { + token_vec_push(&s->checkpoint, drafts[i]); + accepted[n_accept++] = drafts[i]; + } + s->checkpoint_valid = true; + ds4_session_dspark_capture_note_checkpoint(s); + if (stats_enabled) { + s->dspark_stats.full_accepts++; + s->dspark_stats.exactn_full_accepts++; + s->dspark_stats.accepted_draft_tokens += + (uint64_t)draft_n; + ds4_dspark_stats_note_len( + s->dspark_stats.accepted_len_hist, + (uint32_t)draft_n); + } + ds4_session_dspark_stats_note_saved(s, (uint32_t)draft_n); + if (spec_log) { + fprintf(stderr, + "ds4: DSpark Metal exactN boundary oracle drafted=%d " + "accepted_draft=%d accepted_total=%d\n", + draft_n, + draft_n, + n_accept); + } + spec_frontier_free(&frontier); + DS4_DSPARK_STATS_FINISH(); + return n_accept; + } + + /* A partial accept is deliberately not committed by this diagnostic + * path. Restore every pre-cycle frontier and let the proven legacy + * verifier plus exact replay decide and commit the accepted prefix. */ + if (stats_enabled) { + s->dspark_stats.exactn_fallbacks++; + if (exactn_mismatch) { + s->dspark_stats.exactn_partial_fallbacks++; + } else { + s->dspark_stats.exactn_error_fallbacks++; + } + } + s->checkpoint.len = start; + ds4_session_dspark_capture_invalidate(s); + if (!have_frontier || !spec_frontier_restore(&frontier, s)) { + snprintf(err, errlen, + "DSpark Metal exactN boundary rollback failed"); + s->checkpoint_valid = false; + if (stats_enabled) { + s->dspark_stats.verifier_errors++; + ds4_dspark_stats_note_len( + s->dspark_stats.accepted_len_hist, 0); + } + spec_frontier_free(&frontier); + DS4_DSPARK_STATS_FINISH(); + return -1; + } + if (spec_log) { + if (exactn_mismatch) { + fprintf(stderr, + "ds4: DSpark Metal exactN boundary partial " + "accepted_prefix=%d verified_next=%d; falling back " + "to legacy verify/replay\n", + exactn_accepted_prefix, + exactn_last_top); + } else { + fprintf(stderr, + "ds4: DSpark Metal exactN boundary decode failed; " + "falling back to legacy verify/replay\n"); + } + } + } + + const bool exact2 = + !exactn && ok && ds4_session_dspark_exact2_requested( + s, (uint32_t)draft_n); + if (exact2) { + if (stats_enabled) s->dspark_stats.exact2_attempts++; /* sample_probs is a session-owned vocab-sized scratch row. Keeping * row0 there leaves the pre-cycle s->logits intact if exact-2 fails * and the legacy verifier has to take over. */ int exact_top0 = -1; const double exact_t0 = stats_enabled ? now_sec() : 0.0; - bool exact_ok = metal_graph_verify_decode2_exact_impl( - &s->graph, - &e->model, - &e->weights, - drafts[0], - drafts[1], - (uint32_t)start, - false, - drafts[1], - &exact_top0, - NULL, - s->sample_probs, - row_logits); + const bool metal_exact2 = + ds4_session_metal_dspark_exact2_enabled(s); + bool exact_ok = true; + if (metal_exact2) { + const bool static_map_cache = + metal_graph_stream_decode_static_map_enabled() && + metal_graph_stream_decode_static_map_state_cache_enabled(); + if (!static_map_cache || + !s->graph.streaming_static_decode_map_current) { + exact_ok = metal_graph_stream_map_decode_static_all( + &e->model, &e->weights); + if (exact_ok) { + s->graph.streaming_static_decode_map_current = + static_map_cache; + } + } + } + if (exact_ok) { + exact_ok = metal_graph_verify_decode2_exact_impl( + &s->graph, + &e->model, + &e->weights, + drafts[0], + drafts[1], + (uint32_t)start, + false, + drafts[1], + &exact_top0, + NULL, + s->sample_probs, + row_logits); + } if (stats_enabled) { s->dspark_stats.verify_ms += (now_sec() - exact_t0) * 1000.0; @@ -69657,8 +71075,10 @@ static int ds4_session_eval_dspark_speculative_argmax( if (stats_enabled) { if (exact_commit == draft_n) { s->dspark_stats.full_accepts++; + s->dspark_stats.exact2_full_accepts++; } else { s->dspark_stats.partial_accepts++; + s->dspark_stats.exact2_partial_accepts++; } s->dspark_stats.accepted_draft_tokens += (uint64_t)exact_commit; @@ -69670,8 +71090,9 @@ static int ds4_session_eval_dspark_speculative_argmax( s, (uint32_t)exact_commit); if (spec_log) { fprintf(stderr, - "ds4: DSpark CUDA exact2 drafted=2 verified_next=%d " + "ds4: DSpark %s exact2 drafted=2 verified_next=%d " "accepted_draft=%d accepted_total=%d\n", + ds4_backend_name(e->backend), exact_top0, exact_commit, n_accept); @@ -69684,10 +71105,12 @@ static int ds4_session_eval_dspark_speculative_argmax( /* A backend error may happen after exact-2 has touched persistent KV * state. Restore the pre-verify snapshot and retain the established * batch-verify/replay path as a correctness fallback. */ + if (stats_enabled) s->dspark_stats.exact2_fallbacks++; s->checkpoint.len = start; ds4_session_dspark_capture_invalidate(s); if (!have_frontier || !spec_frontier_restore(&frontier, s)) { - snprintf(err, errlen, "DSpark CUDA exact2 rollback failed"); + snprintf(err, errlen, "DSpark %s exact2 rollback failed", + ds4_backend_name(e->backend)); s->checkpoint_valid = false; if (stats_enabled) { s->dspark_stats.verifier_errors++; @@ -69700,10 +71123,12 @@ static int ds4_session_eval_dspark_speculative_argmax( } if (spec_log) { fprintf(stderr, - "ds4: DSpark CUDA exact2 unavailable; falling back to " - "batch verify plus exact replay\n"); + "ds4: DSpark %s exact2 unavailable; falling back to " + "batch verify plus exact replay\n", + ds4_backend_name(e->backend)); } } + /* Replay below still evaluates a skipped one-token draft exactly and * advances all KV and compressor state. */ if (ok && ds4_session_tp_leader(s)) { @@ -69728,20 +71153,36 @@ static int ds4_session_eval_dspark_speculative_argmax( if (cuda_rollback_replay) { s->graph.spec_force_sequential_compressor = true; } - ok = metal_graph_verify_suffix_tops(&s->graph, - &e->model, - &e->weights, - &s->checkpoint, - (uint32_t)start, - (uint32_t)draft_n, - !cuda_rollback_replay && - draft_n > 1 && - draft_n <= - (int)DS4_SPEC_PREFIX_SLOTS + 1, - !cuda_rollback_replay, - row_tops, - NULL, - stats_enabled ? &verify_timing : NULL); + if (metal_acceptance_only_verify) { + if (stats_enabled) { + s->dspark_stats.metal_acceptance_only_attempts++; + s->dspark_stats.metal_acceptance_only_rows_saved++; + } + ok = metal_graph_verify_suffix_acceptance_tops( + &s->graph, + &e->model, + &e->weights, + &s->checkpoint, + (uint32_t)start, + (uint32_t)draft_n, + row_tops, + stats_enabled ? &verify_timing : NULL); + } else { + ok = metal_graph_verify_suffix_tops( + &s->graph, + &e->model, + &e->weights, + &s->checkpoint, + (uint32_t)start, + (uint32_t)draft_n, + !cuda_rollback_replay && + draft_n > 1 && + draft_n <= (int)DS4_SPEC_PREFIX_SLOTS + 1, + !cuda_rollback_replay, + row_tops, + NULL, + stats_enabled ? &verify_timing : NULL); + } s->graph.spec_force_sequential_compressor = saved_force_sequential; if (stats_enabled) { @@ -70014,11 +71455,15 @@ static int ds4_session_eval_dspark_speculative_argmax( } for (int i = 0; i < replay_budget; i++) { /* Only the final replayed token supplies the continuation logits. - * CUDA can therefore skip the output head and readback for every - * accepted prefix token while preserving the exact decode/KV path. */ - float *replay_logits = - !cuda_rollback_replay || i + 1 == replay_budget ? - row_logits : NULL; + * Eligible single-device backends can skip the output head and + * readback for accepted prefix tokens without changing decode/KV. */ + const bool headless_prefix = + i + 1 < replay_budget && + (cuda_rollback_replay || metal_headless_replay); + float *replay_logits = headless_prefix ? NULL : row_logits; + if (stats_enabled && metal_headless_replay && headless_prefix) { + s->dspark_stats.metal_replay_headless_tokens++; + } ok = metal_graph_eval_token_raw_swa(&s->graph, &e->model, &e->weights, @@ -70430,6 +71875,58 @@ static int ds4_session_eval_dspark_speculative_stochastic( } #endif +#if defined(DS4_TEST_HOOKS) && !defined(DS4_NO_GPU) +/* Test-only injection point for deterministic exact-N full/partial/EOS + * fixtures. It calls the production speculative cycle after installing the + * requested draft block; no verifier state transition is duplicated here. */ +int ds4_test_session_eval_exact_drafts( + ds4_session *s, + const int *drafts, + int draft_n, + int eos_token, + int *accepted, + int accepted_cap, + char *err, + size_t errlen) { + if (!s || !drafts || !accepted || draft_n < 1 || draft_n > 5 || + draft_n > (int)DS4_DSPARK_MAX_BLOCK_SIZE || + accepted_cap < draft_n) { + if (err && errlen) { + snprintf(err, errlen, "invalid exact-N test draft block"); + } + return -1; + } + for (int i = 0; i < draft_n; i++) { + s->dspark_draft_tokens[i] = drafts[i]; + } + s->dspark_draft_len = (uint32_t)draft_n; + s->dspark_draft_valid = true; + return ds4_session_eval_dspark_speculative_argmax( + s, + 0, + draft_n, + eos_token, + accepted, + accepted_cap, + err, + errlen); +} + +/* Keep the model-backed oracle independent of stderr formatting. The fixed + * order is attempt, full, fallback, partial fallback, error fallback. */ +int ds4_test_session_exactn_union_stats( + const ds4_session *s, + uint64_t out[5]) { + if (!s || !out) return -1; + out[0] = s->dspark_stats.exactn_union_attempts; + out[1] = s->dspark_stats.exactn_union_full_accepts; + out[2] = s->dspark_stats.exactn_union_fallbacks; + out[3] = s->dspark_stats.exactn_union_partial_fallbacks; + out[4] = s->dspark_stats.exactn_union_error_fallbacks; + return 0; +} +#endif + /* TP worker side of a mirrored speculative-verify block. Runs its half of the * same batch verify as the leader, including per-layer combine gates, purely * for KV, compressor, and indexer side effects; then it obeys the commit diff --git a/ds4_cuda.cu b/ds4_cuda.cu index b14f4ad36..2f6561b47 100644 --- a/ds4_cuda.cu +++ b/ds4_cuda.cu @@ -4842,6 +4842,99 @@ __global__ static void matmul_f16_small_out_hx_ordered_chunks_kernel( } } +/* One-row DS4 HC prelude: unweighted RMSNorm (16384 floats) followed by the + * narrow 24-row F16 mixer. Every block recreates the 256-thread reduction of + * rms_norm_plain_batch8_kernel, then two warps reproduce the contiguous + * 32-chunk accumulation and lane-ordered final sum used by the established + * one-token F16 matvec. Recomputing the tiny norm in twelve blocks keeps all + * mixer rows parallel while avoiding the normalized 64 KiB device round trip + * and one launch. */ +__global__ static void hc_rms_norm_mix_f16_kernel( + float *out, + const float *x, + const __half *w, + float eps, + int round_x_to_f16) { + constexpr uint32_t N = 16384u; + constexpr uint32_t OUT_DIM = 24u; + constexpr uint32_t NORM_THREADS = 256u; + constexpr uint32_t ROWS_PER_BLOCK = 2u; + constexpr uint32_t MATVEC_THREADS = 32u; + constexpr uint32_t MATVEC_CHUNK = N / MATVEC_THREADS; + + const uint32_t tid = threadIdx.x; + float norm_sum = 0.0f; + if (tid < NORM_THREADS) { +#pragma unroll 1 + for (uint32_t i = tid; i < N; i += 2048u) { + const float v0 = x[i]; + const float v1 = x[i + 256u]; + const float v2 = x[i + 512u]; + const float v3 = x[i + 768u]; + const float v4 = x[i + 1024u]; + const float v5 = x[i + 1280u]; + const float v6 = x[i + 1536u]; + const float v7 = x[i + 1792u]; + norm_sum += v0 * v0; + norm_sum += v1 * v1; + norm_sum += v2 * v2; + norm_sum += v3 * v3; + norm_sum += v4 * v4; + norm_sum += v5 * v5; + norm_sum += v6 * v6; + norm_sum += v7 * v7; + } + } + + __shared__ float norm_partial[NORM_THREADS]; + __shared__ float mv_partial[ROWS_PER_BLOCK * MATVEC_THREADS]; + if (tid < NORM_THREADS) norm_partial[tid] = norm_sum; + __syncthreads(); + for (uint32_t stride = NORM_THREADS >> 1u; stride > 0u; stride >>= 1u) { + if (tid < stride) { + norm_partial[tid] += norm_partial[tid + stride]; + } + __syncthreads(); + } + const float scale = rsqrtf(norm_partial[0] / (float)N + eps); + + if (tid < ROWS_PER_BLOCK * MATVEC_THREADS) { + const uint32_t local_row = tid / MATVEC_THREADS; + const uint32_t lane = tid % MATVEC_THREADS; + const uint32_t row = blockIdx.x * ROWS_PER_BLOCK + local_row; + float sum = 0.0f; + if (row < OUT_DIM) { + const uint32_t k0 = lane * MATVEC_CHUNK; + const uint32_t k1 = k0 + MATVEC_CHUNK; + const __half *wr = w + (uint64_t)row * N; +#pragma unroll 1 + for (uint32_t i = k0; i < k1; i++) { + float xv = x[i] * scale; + if (round_x_to_f16) { + xv = __half2float(__float2half(xv)); + } + sum += __half2float(wr[i]) * xv; + } + } + mv_partial[local_row * MATVEC_THREADS + lane] = sum; + } + __syncthreads(); + + if (tid < ROWS_PER_BLOCK * MATVEC_THREADS && + (tid % MATVEC_THREADS) == 0u) { + const uint32_t local_row = tid / MATVEC_THREADS; + const uint32_t row = blockIdx.x * ROWS_PER_BLOCK + local_row; + if (row < OUT_DIM) { + float total = 0.0f; +#pragma unroll 1 + for (uint32_t lane = 0; lane < MATVEC_THREADS; lane++) { + total += mv_partial[local_row * MATVEC_THREADS + lane]; + } + out[row] = total; + } + } +} + __global__ static void matmul_f16_small_out_batch_kernel( float *out, const __half *w, @@ -15502,16 +15595,24 @@ extern "C" int ds4_gpu_matmul_q8_0_pair_tensor( return cuda_ok(cudaGetLastError(), "matmul_q8_0 pair warp launch"); } +static int cuda_matmul_q4_K_pair_tensor_impl( + ds4_gpu_tensor *out0, ds4_gpu_tensor *out1, + const void *model_map, uint64_t model_size, + uint64_t weight0_offset, uint64_t weight1_offset, + uint64_t in_dim, uint64_t out0_dim, uint64_t out1_dim, + const ds4_gpu_tensor *x, uint64_t n_tok); + extern "C" int ds4_gpu_matmul_q4_K_pair_tensor( ds4_gpu_tensor *out0, ds4_gpu_tensor *out1, const void *model_map, uint64_t model_size, uint64_t weight0_offset, uint64_t weight1_offset, uint64_t in_dim, uint64_t out0_dim, uint64_t out1_dim, const ds4_gpu_tensor *x, uint64_t n_tok) { - (void)out0; (void)out1; (void)model_map; (void)model_size; - (void)weight0_offset; (void)weight1_offset; (void)in_dim; - (void)out0_dim; (void)out1_dim; (void)x; (void)n_tok; - return 0; + if (getenv("DS4_CUDA_DISABLE_Q4_DENSE_PAIR") != NULL) return 0; + return cuda_matmul_q4_K_pair_tensor_impl( + out0, out1, model_map, model_size, + weight0_offset, weight1_offset, + in_dim, out0_dim, out1_dim, x, n_tok); } extern "C" int ds4_gpu_matmul_q8_0_decode_rows_exact_tensor( @@ -15838,6 +15939,78 @@ extern "C" int ds4_gpu_matmul_f16_tensor(ds4_gpu_tensor *out, const void *model_ return cuda_ok(cudaGetLastError(), "matmul_f16 launch"); } +/* Return the activation mode that exactly matches the currently selected + * standalone one-token path: 1 keeps normalized activations in F32, 2 rounds + * them through F16. cuBLAS and alternate reduction modes deliberately reject + * the fusion, so the graph caller retains its established fallback. */ +static int cuda_hc_rms_norm_mix_f16_mode(void) { + const char *enable = getenv("DS4_CUDA_ENABLE_HC_NORM_MIX_FUSE"); + if (!enable || !enable[0] || + (enable[0] == '0' && enable[1] == '\0') || + getenv("DS4_CUDA_DISABLE_HC_NORM_MIX_FUSE") != NULL || + getenv("DS4_CUDA_SERIAL_F16_MATMUL") != NULL || + getenv("DS4_CUDA_NO_ORDERED_F16_MATMUL") != NULL) { + return 0; + } + + const int small_out_one_token = + !g_quality_mode && + getenv("DS4_CUDA_F16_SMALL_OUT") != NULL && + getenv("DS4_CUDA_NO_F16_SMALL_OUT") == NULL; + if (small_out_one_token) return 2; + + const int cublas_one_token = + g_cublas_ready && + getenv("DS4_CUDA_NO_F16_CUBLAS_ONE") == NULL && + (!g_quality_mode || getenv("DS4_CUDA_F16_CUBLAS_ONE") != NULL); + if (cublas_one_token) return 0; + return 1; +} + +extern "C" int ds4_gpu_hc_rms_norm_mix_f16_available(void) { + return cuda_hc_rms_norm_mix_f16_mode() != 0; +} + +extern "C" int ds4_gpu_hc_rms_norm_mix_f16_tensor( + ds4_gpu_tensor *out, + const ds4_gpu_tensor *x, + const void *model_map, + uint64_t model_size, + uint64_t weight_offset, + uint32_t n, + uint32_t out_dim, + float eps) { + const int mode = cuda_hc_rms_norm_mix_f16_mode(); + if (mode == 0 || !out || !x || !model_map || + n != 16384u || out_dim != 24u || + weight_offset > model_size) { + return 0; + } + + const uint64_t weight_bytes = + (uint64_t)n * out_dim * sizeof(uint16_t); + if (weight_bytes > model_size - weight_offset || + x->bytes < (uint64_t)n * sizeof(float) || + out->bytes < (uint64_t)out_dim * sizeof(float) || + ds4_tensor_device_idx(x) != ds4_tensor_device_idx(out)) { + return 0; + } + + const int logical_tier = ds4_tensor_device_idx(out); + const char *wptr = cuda_resolve_weight_ptr( + model_map, weight_offset, weight_bytes, logical_tier, + "hc rms-norm/f16-mix"); + if (!wptr) return 0; + + hc_rms_norm_mix_f16_kernel<<<12u, 256u, 0, cuda_decode_stream()>>>( + (float *)out->ptr, + (const float *)x->ptr, + (const __half *)wptr, + eps, + mode == 2 ? 1 : 0); + return cuda_ok(cudaGetLastError(), "hc rms-norm/f16-mix launch"); +} + extern "C" int ds4_gpu_matmul_f16_rms_fold_tensor( ds4_gpu_tensor *out, const void *model_map, @@ -32200,6 +32373,105 @@ __global__ static void matmul_q4_K_dense_kernel( if (lane == 0) out[(uint64_t)tok * out_dim + row] = acc; } +/* Two independently-sized Q4_K projections over one canonical Q8_K row. + * Each accumulator keeps the same block walk and quarter-warp reduction as + * matmul_q4_K_dense_kernel; interleaving the weight loads does not alter either + * output's arithmetic order. */ +__global__ static void matmul_q4_K_dense_pair_kernel( + float *out0, + float *out1, + const char *w0_base, + const char *w1_base, + const cuda_block_q8_K *xq, + uint64_t row_bytes, + uint32_t xq_blocks, + uint32_t out0_dim, + uint32_t out1_dim, + uint32_t n_tok) { + const uint32_t lane = threadIdx.x & 7u; + const uint32_t row_lane = threadIdx.x >> 3u; + const uint32_t tok = blockIdx.y; + const uint32_t row = blockIdx.x * 32u + row_lane; + if (tok >= n_tok || (row >= out0_dim && row >= out1_dim)) return; + + const cuda_block_q8_K *xqb = xq + (uint64_t)tok * xq_blocks; + const cuda_block_q4_K *wr0 = row < out0_dim + ? (const cuda_block_q4_K *)(w0_base + (uint64_t)row * row_bytes) + : NULL; + const cuda_block_q4_K *wr1 = row < out1_dim + ? (const cuda_block_q4_K *)(w1_base + (uint64_t)row * row_bytes) + : NULL; + float acc0 = 0.0f; + float acc1 = 0.0f; + for (uint32_t b = lane; b < xq_blocks; b += 8u) { + if (wr0) acc0 += dev_dot_q4_K_q8_K_block(wr0 + b, xqb + b); + if (wr1) acc1 += dev_dot_q4_K_q8_K_block(wr1 + b, xqb + b); + } + acc0 = quarter_warp_sum_f32(acc0, lane); + acc1 = quarter_warp_sum_f32(acc1, lane); + if (lane == 0u) { + if (row < out0_dim) out0[(uint64_t)tok * out0_dim + row] = acc0; + if (row < out1_dim) out1[(uint64_t)tok * out1_dim + row] = acc1; + } +} + +/* Decode attention-output tail for AProjQ4: + * + * block_out = input @ Wob(Q4_K) + * out_hc = HCPost(block_out, residual_hc, split) + * + * Keep the Q4_K dot-product and quarter-warp reduction identical to + * matmul_q4_K_dense_kernel. The row-owning lane then materializes the + * diagnostic block output and immediately expands the same F32 value into + * the four HC streams. This removes the standalone hc_expand launch without + * changing the Q4 accumulation order used by the non-MMQ fallback. */ +__global__ static void matmul_q4_K_hc_expand4_kernel( + float *out_hc, + float *block_out, + const float *residual_hc, + const float *split, + const char *w_base, + const cuda_block_q8_K *xq, + uint64_t row_bytes, + uint32_t xq_blocks, + uint32_t out_dim, + uint32_t n_embd) { + const uint32_t lane = threadIdx.x & 7u; + const uint32_t row_lane = threadIdx.x >> 3u; + const uint32_t row = blockIdx.x * 32u + row_lane; + if (row >= out_dim) return; + + const cuda_block_q4_K *wr = + (const cuda_block_q4_K *)(w_base + (uint64_t)row * row_bytes); + float acc = 0.0f; + const bool vector_aligned = (((uintptr_t)w_base & 15u) == 0u); + for (uint32_t b = lane; b < xq_blocks; b += 8u) { + if (vector_aligned) { + dev_dot_q4_K_q8_K_block_vec(wr + b, xq + b, &acc); + } else { + acc += dev_dot_q4_K_q8_K_block(wr + b, xq + b); + } + } + acc = quarter_warp_sum_f32(acc, lane); + + if (lane == 0u) { + block_out[row] = acc; + + const float *post = split + 4u; + const float *comb = split + 8u; +#pragma unroll + for (uint32_t dst_hc = 0; dst_hc < 4u; dst_hc++) { + float hc_acc = acc * post[dst_hc]; +#pragma unroll + for (uint32_t src_hc = 0; src_hc < 4u; src_hc++) { + hc_acc += comb[dst_hc + src_hc * 4u] * + residual_hc[(uint64_t)src_hc * n_embd + row]; + } + out_hc[(uint64_t)dst_hc * n_embd + row] = hc_acc; + } + } +} + static int cuda_matmul_q4_K_tensor( ds4_gpu_tensor *out, const void *model_map, @@ -32240,10 +32512,12 @@ static int cuda_matmul_q4_K_tensor( const int rc = n_tok <= 8u ? ds4_mmq_q4_K_dense_vec( wptr, (const float *)x->ptr, (float *)out->ptr, - (int)out_dim, (int)n_tok, (int)in_dim, (cudaStream_t)0) + (int)out_dim, (int)n_tok, (int)in_dim, + cuda_decode_stream()) : ds4_mmq_q4_K_dense( wptr, (const float *)x->ptr, (float *)out->ptr, - (int)out_dim, (int)n_tok, (int)in_dim, (cudaStream_t)0); + (int)out_dim, (int)n_tok, (int)in_dim, + cuda_decode_stream()); if (rc == 0) return 1; fprintf(stderr, "ds4: Q4_K MMQ returned %d " @@ -32258,20 +32532,231 @@ static int cuda_matmul_q4_K_tensor( if (!tmp) return 0; cuda_block_q8_K *xq = (cuda_block_q8_K *)tmp; dim3 qgrid((unsigned)blocks, (unsigned)n_tok, 1); - q8_K_quantize_kernel<<>>(xq, (const float *)x->ptr, - (uint32_t)in_dim, (uint32_t)n_tok); + q8_K_quantize_kernel<<>>( + xq, (const float *)x->ptr, + (uint32_t)in_dim, (uint32_t)n_tok); if (!cuda_ok(cudaGetLastError(), "q4_K dense quantize launch")) return 0; dim3 grid(((unsigned)out_dim + 31u) / 32u, (unsigned)n_tok, 1); - matmul_q4_K_dense_kernel<<>>((float *)out->ptr, - wptr, - xq, - row_bytes, - (uint32_t)blocks, - (uint32_t)out_dim, - (uint32_t)n_tok); + matmul_q4_K_dense_kernel<<>>( + (float *)out->ptr, + wptr, + xq, + row_bytes, + (uint32_t)blocks, + (uint32_t)out_dim, + (uint32_t)n_tok); return cuda_ok(cudaGetLastError(), "q4_K dense matmul launch"); } +static int cuda_matmul_q4_K_pair_tensor_impl( + ds4_gpu_tensor *out0, + ds4_gpu_tensor *out1, + const void *model_map, + uint64_t model_size, + uint64_t weight0_offset, + uint64_t weight1_offset, + uint64_t in_dim, + uint64_t out0_dim, + uint64_t out1_dim, + const ds4_gpu_tensor *x, + uint64_t n_tok) { + if (!out0 || !out1 || !x || !model_map || n_tok == 0u || n_tok > 8u || + in_dim == 0u || (in_dim % CUDA_QK_K) != 0u || + in_dim > INT_MAX || out0_dim == 0u || out1_dim == 0u || + out0_dim > INT_MAX || out1_dim > INT_MAX) { + return 0; + } + + const uint64_t blocks = in_dim / CUDA_QK_K; + if (blocks == 0u || blocks > UINT64_MAX / sizeof(cuda_block_q4_K)) { + return 0; + } + const uint64_t row_bytes = blocks * sizeof(cuda_block_q4_K); + if (out0_dim > UINT64_MAX / row_bytes || + out1_dim > UINT64_MAX / row_bytes || + weight0_offset > model_size || weight1_offset > model_size) { + return 0; + } + const uint64_t weight0_bytes = out0_dim * row_bytes; + const uint64_t weight1_bytes = out1_dim * row_bytes; + if (weight0_bytes > model_size - weight0_offset || + weight1_bytes > model_size - weight1_offset || + n_tok > UINT64_MAX / in_dim || + n_tok * in_dim > UINT64_MAX / sizeof(float) || + n_tok > UINT64_MAX / out0_dim || + n_tok * out0_dim > UINT64_MAX / sizeof(float) || + n_tok > UINT64_MAX / out1_dim || + n_tok * out1_dim > UINT64_MAX / sizeof(float)) { + return 0; + } + const uint64_t x_bytes = n_tok * in_dim * sizeof(float); + const uint64_t out0_bytes = n_tok * out0_dim * sizeof(float); + const uint64_t out1_bytes = n_tok * out1_dim * sizeof(float); + if (x->bytes < x_bytes || out0->bytes < out0_bytes || + out1->bytes < out1_bytes) { + return 0; + } + + const int logical_tier = ds4_tensor_device_idx(out0); + if (logical_tier < 0 || logical_tier >= g_n_gpus || + ds4_tensor_device_idx(out1) != logical_tier || + ds4_tensor_device_idx(x) != logical_tier) { + return 0; + } + const char *w0 = cuda_resolve_weight_ptr( + model_map, weight0_offset, weight0_bytes, logical_tier, + "q4_K dense pair0"); + const char *w1 = cuda_resolve_weight_ptr( + model_map, weight1_offset, weight1_bytes, logical_tier, + "q4_K dense pair1"); + if (!w0 || !w1) return 0; + + if (cuda_use_mmq()) { + const int rc = ds4_mmq_q4_K_dense_pair_vec( + w0, w1, (const float *)x->ptr, + (float *)out0->ptr, (float *)out1->ptr, + (int)out0_dim, (int)out1_dim, (int)n_tok, (int)in_dim, + cuda_decode_stream()); + if (rc == 0) return 1; + fprintf(stderr, + "ds4: Q4_K MMVQ pair returned %d " + "(in=%llu out0=%llu out1=%llu n_tok=%llu); falling back\n", + rc, (unsigned long long)in_dim, + (unsigned long long)out0_dim, + (unsigned long long)out1_dim, + (unsigned long long)n_tok); + } + + if (n_tok > UINT64_MAX / blocks || + n_tok * blocks > UINT64_MAX / sizeof(cuda_block_q8_K)) { + return 0; + } + cuda_block_q8_K *xq = (cuda_block_q8_K *)cuda_tmp_alloc_on( + logical_tier, n_tok * blocks * sizeof(cuda_block_q8_K), + "q4_K dense pair prequant"); + if (!xq) return 0; + + const dim3 qgrid((unsigned)blocks, (unsigned)n_tok, 1u); + q8_K_quantize_kernel<<>>( + xq, (const float *)x->ptr, (uint32_t)in_dim, (uint32_t)n_tok); + if (!cuda_ok(cudaGetLastError(), "q4_K dense pair quantize launch")) { + return 0; + } + + const uint64_t max_out = out0_dim > out1_dim ? out0_dim : out1_dim; + const dim3 grid(((unsigned)max_out + 31u) / 32u, + (unsigned)n_tok, 1u); + matmul_q4_K_dense_pair_kernel<<>>( + (float *)out0->ptr, + (float *)out1->ptr, + w0, + w1, + xq, + row_bytes, + (uint32_t)blocks, + (uint32_t)out0_dim, + (uint32_t)out1_dim, + (uint32_t)n_tok); + return cuda_ok(cudaGetLastError(), "q4_K dense pair matmul launch"); +} + +extern "C" int ds4_gpu_matmul_q4_K_hc_expand_available(void) { + if (getenv("DS4_CUDA_DISABLE_Q4_ATTN_OUT_HC_FUSE") != NULL) return 0; + /* The fused implementation is byte-compatible with the native Q8_K + * fallback. Single-GPU decode normally selects the vendored MMVQ/Q8_1 + * path instead, whose activation quantizer intentionally has different + * numerics. Preserve that default until an on-device oracle validates a + * switch; multi-GPU/quality/DS4_CUDA_MMQ=0 configurations already use + * Q8_K and can enable the compound transparently. */ + return getenv("DS4_CUDA_ENABLE_Q4_ATTN_OUT_HC_FUSE") != NULL || + !cuda_use_mmq(); +} + +extern "C" int ds4_gpu_matmul_q4_K_hc_expand_tensor( + ds4_gpu_tensor *out_hc, + ds4_gpu_tensor *block_out, + const void *model_map, + uint64_t model_size, + uint64_t weight_offset, + uint64_t in_dim, + uint64_t out_dim, + const ds4_gpu_tensor *x, + const ds4_gpu_tensor *residual_hc, + const ds4_gpu_tensor *split, + uint32_t n_embd, + uint32_t n_hc) { + if (!ds4_gpu_matmul_q4_K_hc_expand_available() || + !out_hc || !block_out || !model_map || !x || !residual_hc || !split || + in_dim == 0u || (in_dim % CUDA_QK_K) != 0u || + out_dim == 0u || out_dim != n_embd || (out_dim & 1u) != 0u || + n_hc != 4u || in_dim > UINT32_MAX || out_dim > UINT32_MAX) { + return 0; + } + + const uint64_t blocks = in_dim / CUDA_QK_K; + if (blocks == 0u || blocks > UINT64_MAX / sizeof(cuda_block_q4_K)) { + return 0; + } + const uint64_t row_bytes = blocks * sizeof(cuda_block_q4_K); + if (out_dim > UINT64_MAX / row_bytes || weight_offset > model_size) { + return 0; + } + const uint64_t weight_bytes = out_dim * row_bytes; + if (weight_bytes > model_size - weight_offset || + in_dim > UINT64_MAX / sizeof(float) || + out_dim > UINT64_MAX / sizeof(float)) { + return 0; + } + const uint64_t x_bytes = in_dim * sizeof(float); + const uint64_t embd_bytes = out_dim * sizeof(float); + const uint64_t hc_bytes = 4u * embd_bytes; + const uint64_t split_bytes = 24u * sizeof(float); + if (x->bytes < x_bytes || block_out->bytes < embd_bytes || + residual_hc->bytes < hc_bytes || split->bytes < split_bytes || + out_hc->bytes < hc_bytes) { + return 0; + } + + const int logical_tier = ds4_tensor_device_idx(out_hc); + if (ds4_tensor_device_idx(block_out) != logical_tier || + ds4_tensor_device_idx(x) != logical_tier || + ds4_tensor_device_idx(residual_hc) != logical_tier || + ds4_tensor_device_idx(split) != logical_tier) { + return 0; + } + const char *wptr = cuda_resolve_weight_ptr( + model_map, weight_offset, weight_bytes, logical_tier, + "q4_K hc expand"); + if (!wptr) return 0; + + cuda_block_q8_K *xq = (cuda_block_q8_K *)cuda_tmp_alloc_on( + logical_tier, blocks * sizeof(cuda_block_q8_K), + "q4_K hc expand prequant"); + if (!xq) return 0; + + q8_K_quantize_kernel<<<(unsigned)blocks, 256, 0, + cuda_decode_stream()>>>( + xq, (const float *)x->ptr, (uint32_t)in_dim, 1u); + if (!cuda_ok(cudaGetLastError(), + "q4_K hc expand quantize launch")) { + return 0; + } + + matmul_q4_K_hc_expand4_kernel<<<((unsigned)out_dim + 31u) / 32u, + 256, 0, cuda_decode_stream()>>>( + (float *)out_hc->ptr, + (float *)block_out->ptr, + (const float *)residual_hc->ptr, + (const float *)split->ptr, + wptr, + xq, + row_bytes, + (uint32_t)blocks, + (uint32_t)out_dim, + n_embd); + return cuda_ok(cudaGetLastError(), "q4_K hc expand launch"); +} + __global__ static void matmul_q4_K_kslice_kernel( float *out, const char *w_base, diff --git a/ds4_gpu.h b/ds4_gpu.h index fce976a1e..c8d2aad6f 100644 --- a/ds4_gpu.h +++ b/ds4_gpu.h @@ -270,6 +270,23 @@ int ds4_gpu_stream_expert_cache_seed_experts_gpu_copy( const int32_t *expert_ids, const uint32_t *expert_priorities, uint32_t n_experts); +/* Exact speculative decode may compute up to five independent router rows + * before executing their routed MoE tails. begin_collect() first isolates + * this layer from ordinary decode's pending selected-expert load and global + * selected-id override. prepare() then builds one immutable SSD address table + * for the union of those rows, and set_row() arms exactly one routed-MoE call + * at a time in increasing row order. prepare() is a Metal command-stream + * boundary: it waits for the router rows to become CPU-visible and reopens the + * command batch. release() is required on every success/error exit after + * begin_collect(), and before collecting the next layer. */ +int ds4_gpu_stream_expert_exact_rows_begin_collect(void); +int ds4_gpu_stream_expert_exact_rows_prepare( + const ds4_gpu_stream_expert_table *table, + const ds4_gpu_tensor *selected_rows, + uint32_t n_rows, + uint32_t n_selected); +int ds4_gpu_stream_expert_exact_rows_set_row(uint32_t row); +void ds4_gpu_stream_expert_exact_rows_release(void); #endif void ds4_gpu_print_memory_report(const char *label); @@ -688,6 +705,41 @@ int ds4_gpu_matmul_q4_K_pair_tensor( const ds4_gpu_tensor *x, uint64_t n_tok); +/* Metal decode compound for AProjQ4: Q-A/KV Q4_K pair plus the attention + * and indexer F16 compressor pairs/state stores. Returns 1 when encoded, + * 0 to use the separate fallback, and -1 on an attempted-path error. */ +int ds4_gpu_q4_K_pair_quad_compressor_store_tensor( + ds4_gpu_tensor *qr, + ds4_gpu_tensor *kv_raw, + ds4_gpu_tensor *out0_kv, + ds4_gpu_tensor *out0_score, + ds4_gpu_tensor *out1_kv, + ds4_gpu_tensor *out1_score, + ds4_gpu_tensor *state0_kv, + ds4_gpu_tensor *state0_score, + ds4_gpu_tensor *state1_kv, + ds4_gpu_tensor *state1_score, + const void *model_map, + uint64_t model_size, + uint64_t q_a_offset, + uint64_t kv_offset, + uint64_t weight0_kv_offset, + uint64_t weight0_score_offset, + uint64_t weight1_kv_offset, + uint64_t weight1_score_offset, + uint64_t ape0_offset, + uint32_t ape0_type, + uint64_t ape1_offset, + uint32_t ape1_type, + uint32_t in_dim, + uint32_t q_rank, + uint32_t kv_dim, + uint32_t width0, + uint32_t width1, + const ds4_gpu_tensor *x, + uint32_t ratio, + uint32_t pos); + /* Multi-row decode projections that preserve the one-row reduction order. */ int ds4_gpu_matmul_q8_0_decode_rows_exact_tensor( ds4_gpu_tensor *out, @@ -2752,6 +2804,10 @@ int ds4_gpu_hc_split_weighted_sum_norm_tensor( float eps, float norm_eps); +/* Exact one-row HC decode fusion: unweighted RMSNorm followed by the narrow + * F16 HC-mix projection. The Metal and CUDA implementations are specialized + * for the 16384 -> 24 DS4 Flash shape and preserve their standalone reduction + * trees. */ int ds4_gpu_hc_rms_norm_mix_f16_available(void); int ds4_gpu_hc_rms_norm_mix_f16_tensor( ds4_gpu_tensor *out, @@ -3077,6 +3133,22 @@ int ds4_gpu_glm53_kda_prefill( float gate_lower_bound, float norm_eps); +/* Q4_K sibling of the decode attention-output/HC compound. */ +int ds4_gpu_matmul_q4_K_hc_expand_available(void); +int ds4_gpu_matmul_q4_K_hc_expand_tensor( + ds4_gpu_tensor *out_hc, + ds4_gpu_tensor *block_out, + const void *model_map, + uint64_t model_size, + uint64_t weight_offset, + uint64_t in_dim, + uint64_t out_dim, + const ds4_gpu_tensor *x, + const ds4_gpu_tensor *residual_hc, + const ds4_gpu_tensor *split, + uint32_t n_embd, + uint32_t n_hc); + /* Decode-island CUDA graph capture (CUDA backend; Metal/ROCm/CPU stub it * out and stay eager). Design ported from the Entrpi/ds4 batched-serving * fork's per-layer decode graph capture. The key identifies a captured diff --git a/ds4_metal.m b/ds4_metal.m index cbd61ddbd..6e4a6015b 100644 --- a/ds4_metal.m +++ b/ds4_metal.m @@ -674,6 +674,79 @@ static void ds4_gpu_print_device_summary(void) { NSUInteger down_inner; } ds4_gpu_stream_expert_reusable_buffers; +enum { + DS4_METAL_EXACT_ROWS_MAX = 5, + DS4_METAL_EXACT_ROWS_MAX_RESOURCES = + DS4_METAL_EXACT_ROWS_MAX * DS4_METAL_STREAM_EXPERT_CACHE_MAX_SELECTED, +}; + +/* A layer-local immutable view of all experts selected by an exact speculative + * microbatch. The selected matrix and address tables are private to the + * scope: ordinary decode's mutable per-layer table can therefore be updated by + * neither a later row nor cache pruning while earlier rows remain in flight. */ +typedef struct { + int collecting; + int active; + int row_armed; + uint32_t row; + uint32_t next_row; + uint32_t n_rows; + uint32_t n_selected; + ds4_gpu_stream_expert_table table; + int32_t selected_ids[DS4_METAL_EXACT_ROWS_MAX_RESOURCES]; + __strong id selected_buffer; + NSUInteger selected_offset; + __strong id gate_addrs; + __strong id up_addrs; + __strong id down_addrs; + __strong id overflow_gate; + __strong id overflow_up; + __strong id overflow_down; + ds4_gpu_stream_expert_cache_entry + *resources[DS4_METAL_EXACT_ROWS_MAX_RESOURCES]; + __strong id + resource_gate[DS4_METAL_EXACT_ROWS_MAX_RESOURCES]; + __strong id + resource_up[DS4_METAL_EXACT_ROWS_MAX_RESOURCES]; + __strong id + resource_down[DS4_METAL_EXACT_ROWS_MAX_RESOURCES]; + uint32_t n_resources; + uint32_t unique_count; +} ds4_gpu_stream_expert_exact_rows_scope; + +static ds4_gpu_stream_expert_exact_rows_scope + g_stream_expert_exact_rows_scope; + +static void ds4_gpu_stream_expert_exact_rows_clear(void) { + ds4_gpu_stream_expert_exact_rows_scope *scope = + &g_stream_expert_exact_rows_scope; + scope->selected_buffer = nil; + scope->gate_addrs = nil; + scope->up_addrs = nil; + scope->down_addrs = nil; + scope->overflow_gate = nil; + scope->overflow_up = nil; + scope->overflow_down = nil; + for (uint32_t i = 0; i < DS4_METAL_EXACT_ROWS_MAX_RESOURCES; i++) { + scope->resources[i] = NULL; + scope->resource_gate[i] = nil; + scope->resource_up[i] = nil; + scope->resource_down[i] = nil; + scope->selected_ids[i] = -1; + } + memset(&scope->table, 0, sizeof(scope->table)); + scope->collecting = 0; + scope->active = 0; + scope->row_armed = 0; + scope->row = 0; + scope->next_row = 0; + scope->n_rows = 0; + scope->n_selected = 0; + scope->selected_offset = 0; + scope->n_resources = 0; + scope->unique_count = 0; +} + static ds4_gpu_stream_expert_cache_entry g_stream_expert_cache[DS4_METAL_STREAM_EXPERT_CACHE_MAX_LAYER][DS4_METAL_STREAM_EXPERT_CACHE_MAX_EXPERT]; static ds4_gpu_stream_expert_cache_entry @@ -14185,6 +14258,7 @@ static void ds4_gpu_stream_expert_cache_clear_entry( } static void ds4_gpu_stream_expert_cache_clear_all(int reset_stats) { + ds4_gpu_stream_expert_exact_rows_clear(); ds4_gpu_stream_expert_pending_load_clear(); g_stream_expert_cache_done_seq = g_stream_expert_cache_cb_seq; g_stream_expert_cache_batch_seq = 0; @@ -16748,7 +16822,8 @@ static int ds4_gpu_stream_expert_cache_prepare_selected_batch( uint32_t *unique_out, id *overflow_gate, id *overflow_up, - id *overflow_down) { + id *overflow_down, + bool require_private_addr_tables) { if (overflow_gate) *overflow_gate = nil; if (overflow_up) *overflow_up = nil; if (overflow_down) *overflow_down = nil; @@ -16796,7 +16871,7 @@ static int ds4_gpu_stream_expert_cache_prepare_selected_batch( uint32_t unique_count = 0; *n_resources = 0; *unique_out = 0; - if (ok) { + if (ok && !require_private_addr_tables) { if (!ds4_gpu_stream_expert_cache_ensure_addr_buffers(layer)) { ok = 0; } @@ -16828,7 +16903,8 @@ static int ds4_gpu_stream_expert_cache_prepare_selected_batch( const bool use_transient_selected = ok && n_tokens <= 5u && - ds4_gpu_stream_expert_cache_configured_count() < n_total_expert; + (require_private_addr_tables || + ds4_gpu_stream_expert_cache_configured_count() < n_total_expert); if (use_transient_selected) { ok = ds4_gpu_stream_expert_prepare_transient_selected_batch( model_map, @@ -17242,6 +17318,211 @@ static int ds4_gpu_stream_expert_cache_prepare_selected_batch( return 1; } +int ds4_gpu_stream_expert_exact_rows_begin_collect(void) { + if (!g_initialized && !ds4_gpu_init()) return 0; + ds4_gpu_stream_expert_exact_rows_scope *scope = + &g_stream_expert_exact_rows_scope; + if (!g_ssd_streaming_mode || scope->collecting || scope->active || + scope->row_armed) { + return 0; + } + + /* A failed ordinary decode must not leak host routing state into the + * matrix collected below. Finish any outstanding read before the exact + * scope begins so it cannot install/evict entries while the union address + * table is being assembled. */ + g_routed_moe_selected_override_n = 0; + if (!ds4_gpu_stream_expert_pending_load_finish(NULL)) { + ds4_gpu_stream_expert_exact_rows_clear(); + return 0; + } + scope->collecting = 1; + return 1; +} + +int ds4_gpu_stream_expert_exact_rows_prepare( + const ds4_gpu_stream_expert_table *table, + const ds4_gpu_tensor *selected_rows, + uint32_t n_rows, + uint32_t n_selected) { + if (!g_initialized && !ds4_gpu_init()) return 0; + if (!g_ssd_streaming_mode || !table || !selected_rows || + !table->model_map || + table->layer >= DS4_METAL_STREAM_EXPERT_CACHE_MAX_LAYER || + table->n_total_expert == 0 || + table->n_total_expert > DS4_METAL_STREAM_EXPERT_CACHE_MAX_EXPERT || + table->gate_expert_bytes == 0 || table->down_expert_bytes == 0 || + n_rows == 0 || n_rows > DS4_METAL_EXACT_ROWS_MAX || + n_selected == 0 || + n_selected > DS4_METAL_STREAM_EXPERT_CACHE_MAX_SELECTED || + n_rows > UINT32_MAX / n_selected) { + return 0; + } + const uint64_t n_ids = (uint64_t)n_rows * n_selected; + const uint64_t selected_bytes = n_ids * sizeof(int32_t); + if (ds4_gpu_tensor_bytes(selected_rows) < selected_bytes) return 0; + + ds4_gpu_stream_expert_exact_rows_scope *scope = + &g_stream_expert_exact_rows_scope; + if (!scope->collecting || scope->active || scope->row_armed || !g_batch_cb) { + fprintf(stderr, + "ds4: Metal exact-row expert union requires one active, un-nested command batch\n"); + return 0; + } + + /* Router rows are GPU-produced shared memory. A plain end/commit is not + * sufficient for a CPU read; wait on a signal encoded after all routers, + * then continue the MoE tails in a fresh ordered command buffer. */ + if (!ds4_gpu_signal_batch_and_wait_event( + "exact-row expert union router readback")) { + return 0; + } + /* The signal is encoded at the end of the router command buffer, hence + * all resources registered before it are no longer in use. Retire that + * completed buffer now so layer-local packed experts do not accumulate + * over the model's full layer count. */ + if (!ds4_gpu_wait_pending_command_buffers( + "exact-row expert union router boundary")) { + return 0; + } + [g_transient_buffers removeAllObjects]; + if (!ds4_gpu_tensor_read(selected_rows, + 0, + scope->selected_ids, + selected_bytes)) { + ds4_gpu_stream_expert_exact_rows_clear(); + return 0; + } + + uint32_t n_resources = 0; + uint32_t unique_count = 0; + id gate_addrs = nil; + id up_addrs = nil; + id down_addrs = nil; + id overflow_gate = nil; + id overflow_up = nil; + id overflow_down = nil; + if (!ds4_gpu_stream_expert_cache_prepare_selected_batch( + table->model_map, + table->model_size, + table->layer, + selected_rows, + n_rows, + table->n_total_expert, + n_selected, + table->gate_offset, + table->up_offset, + table->down_offset, + table->gate_expert_bytes, + table->down_expert_bytes, + &gate_addrs, + &up_addrs, + &down_addrs, + scope->resources, + &n_resources, + &unique_count, + &overflow_gate, + &overflow_up, + &overflow_down, + true)) { + ds4_gpu_stream_expert_exact_rows_clear(); + return 0; + } + if (n_resources > DS4_METAL_EXACT_ROWS_MAX_RESOURCES || + unique_count == 0 || !gate_addrs || !up_addrs || !down_addrs || + gate_addrs == g_stream_expert_cache_gate_addr_buffers[table->layer] || + up_addrs == g_stream_expert_cache_up_addr_buffers[table->layer] || + down_addrs == g_stream_expert_cache_down_addr_buffers[table->layer] || + gate_addrs == g_stream_compact_gate_addr_buffers[table->layer] || + up_addrs == g_stream_compact_up_addr_buffers[table->layer] || + down_addrs == g_stream_compact_down_addr_buffers[table->layer]) { + fprintf(stderr, + "ds4: Metal exact-row expert union did not produce private address tables\n"); + ds4_gpu_stream_expert_exact_rows_clear(); + return 0; + } + + for (uint32_t i = 0; i < n_resources; i++) { + ds4_gpu_stream_expert_cache_entry *entry = scope->resources[i]; + if (!entry || !entry->valid || !entry->gate_buffer || + !entry->up_buffer || !entry->down_buffer) { + ds4_gpu_stream_expert_exact_rows_clear(); + return 0; + } + /* Retain immutable resources independently of cache ownership. The + * address table remains valid even if a later implementation permits + * cache maintenance while the exact-row command buffer is in flight. */ + scope->resource_gate[i] = entry->gate_buffer; + scope->resource_up[i] = entry->up_buffer; + scope->resource_down[i] = entry->down_buffer; + } + /* Pin every persistent-cache hit to the command-buffer epoch before the + * scope becomes visible. Exact rows deliberately bypass all ordinary + * cache setup/pruning below, but this also makes an accidental cache + * maintenance attempt fail safely instead of invalidating an entry whose + * GPU address is already published in the private table. */ + if (n_resources != 0 && + !ds4_gpu_stream_expert_cache_mark_entries_inflight(scope->resources, + n_resources, + 0)) { + fprintf(stderr, + "ds4: Metal exact-row expert union could not pin cache resources\n"); + ds4_gpu_stream_expert_exact_rows_clear(); + return 0; + } + scope->table = *table; + scope->selected_buffer = ds4_gpu_tensor_buffer(selected_rows); + scope->selected_offset = ds4_gpu_tensor_offset(selected_rows); + scope->gate_addrs = gate_addrs; + scope->up_addrs = up_addrs; + scope->down_addrs = down_addrs; + scope->overflow_gate = overflow_gate; + scope->overflow_up = overflow_up; + scope->overflow_down = overflow_down; + scope->n_rows = n_rows; + scope->n_selected = n_selected; + scope->n_resources = n_resources; + scope->unique_count = unique_count; + scope->row = 0; + scope->next_row = 0; + scope->row_armed = 0; + scope->active = 1; + + if (getenv("DS4_METAL_DSPARK_EXACT_ROWS_PROFILE") != NULL) { + fprintf(stderr, + "ds4: Metal exact-row expert union layer=%u rows=%u " + "selected=%u unique=%u resident=%u transient=%u\n", + table->layer, + n_rows, + n_selected, + unique_count, + n_resources, + scope->overflow_gate != nil); + } + return 1; +} + +int ds4_gpu_stream_expert_exact_rows_set_row(uint32_t row) { + ds4_gpu_stream_expert_exact_rows_scope *scope = + &g_stream_expert_exact_rows_scope; + if (!scope->active || scope->row_armed || row >= scope->n_rows || + row != scope->next_row) { + return 0; + } + scope->row = row; + scope->row_armed = 1; + return 1; +} + +void ds4_gpu_stream_expert_exact_rows_release(void) { + /* release() is also the fail-clean exit for a collection that did not + * reach prepare(). Neither ordinary selected state is allowed to escape + * the exact scope. */ + g_routed_moe_selected_override_n = 0; + ds4_gpu_stream_expert_pending_load_clear(); + ds4_gpu_stream_expert_exact_rows_clear(); +} + int ds4_gpu_stream_expert_cache_seed_selected( const ds4_gpu_stream_expert_table *table, const int32_t *selected_ids, @@ -19683,6 +19964,241 @@ int ds4_gpu_matmul_q4_K_pair_tensor( } } +int ds4_gpu_q4_K_pair_quad_compressor_store_tensor( + ds4_gpu_tensor *qr, + ds4_gpu_tensor *kv_raw, + ds4_gpu_tensor *out0_kv, + ds4_gpu_tensor *out0_score, + ds4_gpu_tensor *out1_kv, + ds4_gpu_tensor *out1_score, + ds4_gpu_tensor *state0_kv, + ds4_gpu_tensor *state0_score, + ds4_gpu_tensor *state1_kv, + ds4_gpu_tensor *state1_score, + const void *model_map, + uint64_t model_size, + uint64_t q_a_offset, + uint64_t kv_offset, + uint64_t weight0_kv_offset, + uint64_t weight0_score_offset, + uint64_t weight1_kv_offset, + uint64_t weight1_score_offset, + uint64_t ape0_offset, + uint32_t ape0_type, + uint64_t ape1_offset, + uint32_t ape1_type, + uint32_t in_dim, + uint32_t q_rank, + uint32_t kv_dim, + uint32_t width0, + uint32_t width1, + const ds4_gpu_tensor *x, + uint32_t ratio, + uint32_t pos) { + if (!g_initialized && !ds4_gpu_init()) return -1; + if (getenv("DS4_METAL_DISABLE_Q4_QKV_COMPRESSOR_FUSE") != NULL) return 0; + if (!qr || !kv_raw || !out0_kv || !out0_score || + !state0_kv || !state0_score || !model_map || !x || + in_dim == 0u || (in_dim % 256u) != 0u || + q_rank == 0u || kv_dim == 0u || (q_rank & 1u) != 0u || + (kv_dim & 1u) != 0u || width0 == 0u || + (width0 & 1u) != 0u || (width1 & 1u) != 0u || + (ratio != 4u && ratio != 128u) || + (ape0_type != 0u && ape0_type != 1u) || + (ape1_type != 0u && ape1_type != 1u)) { + return 0; + } + if (width1 != 0u && + (!out1_kv || !out1_score || !state1_kv || !state1_score)) { + return -1; + } + + @autoreleasepool { + const uint64_t q4_row_bytes = ((uint64_t)in_dim / 256u) * 144u; + const uint64_t f16_row_bytes = (uint64_t)in_dim * sizeof(uint16_t); + if ((uint64_t)q_rank > UINT64_MAX / q4_row_bytes || + (uint64_t)kv_dim > UINT64_MAX / q4_row_bytes || + (uint64_t)width0 > UINT64_MAX / f16_row_bytes || + (width1 != 0u && + (uint64_t)width1 > UINT64_MAX / f16_row_bytes)) { + return -1; + } + const uint64_t q_a_bytes = (uint64_t)q_rank * q4_row_bytes; + const uint64_t kv_bytes = (uint64_t)kv_dim * q4_row_bytes; + const uint64_t weight0_bytes = (uint64_t)width0 * f16_row_bytes; + const uint64_t weight1_bytes = (uint64_t)width1 * f16_row_bytes; + const uint64_t state_rows = ratio == 4u ? 2u * ratio : ratio; + const uint64_t ape0_elem = ape0_type == 1u ? sizeof(uint16_t) : sizeof(float); + const uint64_t ape1_elem = ape1_type == 1u ? sizeof(uint16_t) : sizeof(float); + const uint64_t ape0_bytes = (uint64_t)width0 * ratio * ape0_elem; + const uint64_t ape1_bytes = (uint64_t)width1 * ratio * ape1_elem; + if (q_a_offset > model_size || q_a_bytes > model_size - q_a_offset || + kv_offset > model_size || kv_bytes > model_size - kv_offset || + weight0_kv_offset > model_size || + weight0_bytes > model_size - weight0_kv_offset || + weight0_score_offset > model_size || + weight0_bytes > model_size - weight0_score_offset || + ape0_offset > model_size || ape0_bytes > model_size - ape0_offset || + (width1 != 0u && + (weight1_kv_offset > model_size || + weight1_bytes > model_size - weight1_kv_offset || + weight1_score_offset > model_size || + weight1_bytes > model_size - weight1_score_offset || + ape1_offset > model_size || + ape1_bytes > model_size - ape1_offset))) { + return -1; + } + + const uint64_t state0_bytes = state_rows * width0 * sizeof(float); + const uint64_t state1_bytes = state_rows * width1 * sizeof(float); + if (ds4_gpu_tensor_bytes(qr) < (uint64_t)q_rank * sizeof(float) || + ds4_gpu_tensor_bytes(kv_raw) < (uint64_t)kv_dim * sizeof(float) || + ds4_gpu_tensor_bytes(out0_kv) < (uint64_t)width0 * sizeof(float) || + ds4_gpu_tensor_bytes(out0_score) < (uint64_t)width0 * sizeof(float) || + ds4_gpu_tensor_bytes(state0_kv) < state0_bytes || + ds4_gpu_tensor_bytes(state0_score) < state0_bytes || + ds4_gpu_tensor_bytes(x) < (uint64_t)in_dim * sizeof(float) || + (width1 != 0u && + (ds4_gpu_tensor_bytes(out1_kv) < (uint64_t)width1 * sizeof(float) || + ds4_gpu_tensor_bytes(out1_score) < (uint64_t)width1 * sizeof(float) || + ds4_gpu_tensor_bytes(state1_kv) < state1_bytes || + ds4_gpu_tensor_bytes(state1_score) < state1_bytes))) { + return -1; + } + + uint64_t q_a_inner = 0, kv_inner = 0; + uint64_t w0kv_inner = 0, w0sc_inner = 0; + uint64_t w1kv_inner = 0, w1sc_inner = 0; + uint64_t ape0_inner = 0, ape1_inner = 0; + id qw0buf = ds4_gpu_wrap_model_range( + model_map, model_size, q_a_offset, q_a_bytes, &q_a_inner); + id qw1buf = ds4_gpu_wrap_model_range( + model_map, model_size, kv_offset, kv_bytes, &kv_inner); + id w0kvbuf = ds4_gpu_wrap_model_range( + model_map, model_size, weight0_kv_offset, weight0_bytes, &w0kv_inner); + id w0scbuf = ds4_gpu_wrap_model_range( + model_map, model_size, weight0_score_offset, weight0_bytes, &w0sc_inner); + id ape0buf = ds4_gpu_wrap_model_range( + model_map, model_size, ape0_offset, ape0_bytes, &ape0_inner); + id w1kvbuf = width1 != 0u + ? ds4_gpu_wrap_model_range(model_map, model_size, + weight1_kv_offset, weight1_bytes, + &w1kv_inner) + : w0kvbuf; + id w1scbuf = width1 != 0u + ? ds4_gpu_wrap_model_range(model_map, model_size, + weight1_score_offset, weight1_bytes, + &w1sc_inner) + : w0scbuf; + id ape1buf = width1 != 0u + ? ds4_gpu_wrap_model_range(model_map, model_size, + ape1_offset, ape1_bytes, &ape1_inner) + : ape0buf; + if (width1 == 0u) { + w1kv_inner = w0kv_inner; + w1sc_inner = w0sc_inner; + ape1_inner = ape0_inner; + out1_kv = out0_kv; + out1_score = out0_score; + state1_kv = state0_kv; + state1_score = state0_score; + } + if (!qw0buf || !qw1buf || !w0kvbuf || !w0scbuf || + !w1kvbuf || !w1scbuf || !ape0buf || !ape1buf) return -1; + + id xbuf = ds4_gpu_tensor_buffer(x); + id qrbuf = ds4_gpu_tensor_buffer(qr); + id kvbuf = ds4_gpu_tensor_buffer(kv_raw); + id out0kvbuf = ds4_gpu_tensor_buffer(out0_kv); + id out0scbuf = ds4_gpu_tensor_buffer(out0_score); + id out1kvbuf = ds4_gpu_tensor_buffer(out1_kv); + id out1scbuf = ds4_gpu_tensor_buffer(out1_score); + id state0kvbuf = ds4_gpu_tensor_buffer(state0_kv); + id state0scbuf = ds4_gpu_tensor_buffer(state0_score); + id state1kvbuf = ds4_gpu_tensor_buffer(state1_kv); + id state1scbuf = ds4_gpu_tensor_buffer(state1_score); + if (!xbuf || !qrbuf || !kvbuf || !out0kvbuf || !out0scbuf || + !out1kvbuf || !out1scbuf || !state0kvbuf || !state0scbuf || + !state1kvbuf || !state1scbuf) return -1; + + ds4_gpu_q8_0_matvec_args args0 = { + .ne00 = (int32_t)in_dim, .ne01 = (int32_t)q_rank, .ne02 = 1, + .nb00 = 1, .nb01 = q4_row_bytes, + .nb02 = q4_row_bytes * q_rank, .nb03 = q4_row_bytes * q_rank, + .ne10 = (int32_t)in_dim, .ne11 = 1, .ne12 = 1, + .nb10 = sizeof(float), .nb11 = (uint64_t)in_dim * sizeof(float), + .nb12 = (uint64_t)in_dim * sizeof(float), + .nb13 = (uint64_t)in_dim * sizeof(float), + .ne0 = (int32_t)q_rank, .ne1 = 1, .nr0 = 2, .r2 = 1, .r3 = 1, + }; + ds4_gpu_q8_0_matvec_args args1 = args0; + args1.ne01 = args1.ne0 = (int32_t)kv_dim; + args1.nb02 = args1.nb03 = q4_row_bytes * kv_dim; + ds4_gpu_f16_matvec_args cargs = + ds4_gpu_make_f16_mv_args(in_dim, width0); + cargs.nr0 = 2; + ds4_gpu_dsv4_compressor_store_one_args store0_args = { + .width = width0, .ratio = ratio, .pos = pos, + .ape_type = ape0_type, + }; + ds4_gpu_dsv4_compressor_store_one_args store1_args = { + .width = width1, .ratio = ratio, .pos = pos, + .ape_type = ape1_type, + }; + const uint32_t max_out = q_rank > kv_dim ? q_rank : kv_dim; + const uint32_t pair_tgs = (max_out + 15u) / 16u; + id pipeline = ds4_gpu_get_mul_mv_pipeline( + "kernel_dsv4_q4_K_qkv_pair_quad_compressor_store", 8); + if (!pipeline) return 0; + + int owned = 0; + id cb = ds4_gpu_command_buffer(&owned); + if (!cb) return -1; + id enc = ds4_gpu_compute_encoder(cb); + [enc setComputePipelineState:pipeline]; + [enc setBytes:&args0 length:sizeof(args0) atIndex:0]; + [enc setBytes:&args1 length:sizeof(args1) atIndex:1]; + [enc setBytes:&cargs length:sizeof(cargs) atIndex:2]; + [enc setBytes:&store0_args length:sizeof(store0_args) atIndex:3]; + [enc setBytes:&store1_args length:sizeof(store1_args) atIndex:4]; + [enc setBytes:&pair_tgs length:sizeof(pair_tgs) atIndex:5]; + [enc setBuffer:qw0buf offset:(NSUInteger)q_a_inner atIndex:6]; + [enc setBuffer:qw1buf offset:(NSUInteger)kv_inner atIndex:7]; + [enc setBuffer:w0kvbuf offset:(NSUInteger)w0kv_inner atIndex:8]; + [enc setBuffer:w0scbuf offset:(NSUInteger)w0sc_inner atIndex:9]; + [enc setBuffer:w1kvbuf offset:(NSUInteger)w1kv_inner atIndex:10]; + [enc setBuffer:w1scbuf offset:(NSUInteger)w1sc_inner atIndex:11]; + [enc setBuffer:xbuf offset:ds4_gpu_tensor_offset(x) atIndex:12]; + [enc setBuffer:qrbuf offset:ds4_gpu_tensor_offset(qr) atIndex:13]; + [enc setBuffer:kvbuf offset:ds4_gpu_tensor_offset(kv_raw) atIndex:14]; + [enc setBuffer:out0kvbuf offset:ds4_gpu_tensor_offset(out0_kv) atIndex:15]; + [enc setBuffer:out0scbuf offset:ds4_gpu_tensor_offset(out0_score) atIndex:16]; + [enc setBuffer:out1kvbuf offset:ds4_gpu_tensor_offset(out1_kv) atIndex:17]; + [enc setBuffer:out1scbuf offset:ds4_gpu_tensor_offset(out1_score) atIndex:18]; + [enc setBuffer:ape0buf offset:(NSUInteger)ape0_inner atIndex:19]; + [enc setBuffer:ape1buf offset:(NSUInteger)ape1_inner atIndex:20]; + [enc setBuffer:state0kvbuf offset:ds4_gpu_tensor_offset(state0_kv) atIndex:21]; + [enc setBuffer:state0scbuf offset:ds4_gpu_tensor_offset(state0_score) atIndex:22]; + [enc setBuffer:state1kvbuf offset:ds4_gpu_tensor_offset(state1_kv) atIndex:23]; + [enc setBuffer:state1scbuf offset:ds4_gpu_tensor_offset(state1_score) atIndex:24]; + [enc setThreadgroupMemoryLength:32u * 2u * sizeof(float) atIndex:0]; + [enc dispatchThreadgroups:MTLSizeMake( + (NSUInteger)pair_tgs + + ((NSUInteger)width0 + 1u) / 2u + + ((NSUInteger)width1 + 1u) / 2u, + 1, 1) + threadsPerThreadgroup:MTLSizeMake(32, 8, 1)]; + ds4_gpu_end_compute_encoder(cb, enc); + + if (!ds4_gpu_finish_command_buffer( + cb, owned, "Q4_K QKV pair + compressor store")) { + return -1; + } + } + + return 1; +} + int ds4_gpu_matmul_q8_0_f16_out_tensor( ds4_gpu_tensor *out_h, const void *model_map, @@ -38489,7 +39005,8 @@ static int ds4_gpu_glm_routed_moe_batch_tensor_impl( &stream_unique, &stream_overflow_gate, &stream_overflow_up, - &stream_overflow_down)) { + &stream_overflow_down, + false)) { return 0; } if (stream_unique == 0) { @@ -39162,6 +39679,9 @@ int ds4_gpu_routed_moe_one_tensor( __unsafe_unretained id up_slot_bufs[DS4_METAL_MAX_ROUTED_EXPERT_USED] = { nil }; __unsafe_unretained id down_slot_bufs[DS4_METAL_MAX_ROUTED_EXPERT_USED] = { nil }; ds4_gpu_stream_expert_cache_entry *stream_slot_entries[DS4_METAL_MAX_ROUTED_EXPERT_USED] = { NULL }; + ds4_gpu_stream_expert_cache_entry * const *stream_addr_resources = + stream_slot_entries; + uint32_t stream_addr_resource_count = n_expert; NSUInteger gate_slot_offsets[DS4_METAL_MAX_ROUTED_EXPERT_USED] = { 0 }; NSUInteger up_slot_offsets[DS4_METAL_MAX_ROUTED_EXPERT_USED] = { 0 }; NSUInteger down_slot_offsets[DS4_METAL_MAX_ROUTED_EXPERT_USED] = { 0 }; @@ -39171,6 +39691,9 @@ int ds4_gpu_routed_moe_one_tensor( id stream_gate_addr_buf = nil; id stream_up_addr_buf = nil; id stream_down_addr_buf = nil; + id stream_overflow_gate = nil; + id stream_overflow_up = nil; + id stream_overflow_down = nil; bool use_stream_expert_addr_table = false; bool use_stream_expert_masked_addr_table = false; bool use_stream_compact_addr_table = false; @@ -39705,6 +40228,95 @@ int ds4_gpu_routed_moe_one_tensor( const bool use_selected_slots = use_q4_selected_slots || use_iq2_selected_slots || use_mxfp4_selected_slots || use_iq2_stream_addr_table; + ds4_gpu_stream_expert_exact_rows_scope *exact_rows_scope = + &g_stream_expert_exact_rows_scope; + bool use_exact_rows_scope = false; + if (exact_rows_scope->active) { + uint64_t expected_selected_offset = exact_rows_scope->selected_offset; + const uint64_t row_selected_bytes = + (uint64_t)exact_rows_scope->n_selected * sizeof(int32_t); + if ((uint64_t)exact_rows_scope->row > + (UINT64_MAX - expected_selected_offset) / row_selected_bytes) { + fprintf(stderr, "ds4: Metal exact-row selected offset overflow\n"); + return 0; + } + expected_selected_offset += + (uint64_t)exact_rows_scope->row * row_selected_bytes; + bool exact_rows_supported = + exact_rows_scope->row_armed && + exact_rows_scope->row < exact_rows_scope->n_rows && + exact_rows_scope->n_selected == n_expert && + exact_rows_scope->table.model_map == model_map && + exact_rows_scope->table.model_size == model_size && + exact_rows_scope->table.layer == layer_index && + exact_rows_scope->table.n_total_expert == n_total_expert && + exact_rows_scope->table.gate_offset == gate_offset && + exact_rows_scope->table.up_offset == up_offset && + exact_rows_scope->table.down_offset == down_offset && + exact_rows_scope->table.gate_expert_bytes == gate_expert_bytes && + exact_rows_scope->table.down_expert_bytes == down_expert_bytes && + exact_rows_scope->selected_buffer == selectedbuf && + expected_selected_offset == ds4_gpu_tensor_offset(selected) && + g_tp_split_world == 1 && + gate_type == DS4_METAL_TENSOR_IQ2_XXS && + down_type == DS4_METAL_TENSOR_Q2_K && + use_iq2_selected_slots && + g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_pipeline != nil && + g_moe_mul_mv_addr_q2_k_sum6_pipeline != nil; + if (exact_rows_supported) { + exact_rows_supported = + exact_rows_scope->unique_count != 0 && + exact_rows_scope->n_resources <= + exact_rows_scope->unique_count && + exact_rows_scope->gate_addrs && + exact_rows_scope->up_addrs && + exact_rows_scope->down_addrs && + exact_rows_scope->gate_addrs != + g_stream_expert_cache_gate_addr_buffers[layer_index] && + exact_rows_scope->up_addrs != + g_stream_expert_cache_up_addr_buffers[layer_index] && + exact_rows_scope->down_addrs != + g_stream_expert_cache_down_addr_buffers[layer_index] && + exact_rows_scope->gate_addrs != + g_stream_compact_gate_addr_buffers[layer_index] && + exact_rows_scope->up_addrs != + g_stream_compact_up_addr_buffers[layer_index] && + exact_rows_scope->down_addrs != + g_stream_compact_down_addr_buffers[layer_index] && + (exact_rows_scope->n_resources == + exact_rows_scope->unique_count || + (exact_rows_scope->overflow_gate && + exact_rows_scope->overflow_up && + exact_rows_scope->overflow_down)); + } + for (uint32_t i = 0; + exact_rows_supported && i < exact_rows_scope->n_resources; + i++) { + ds4_gpu_stream_expert_cache_entry *entry = + exact_rows_scope->resources[i]; + exact_rows_supported = + entry && entry->valid && + entry->gate_buffer == exact_rows_scope->resource_gate[i] && + entry->up_buffer == exact_rows_scope->resource_up[i] && + entry->down_buffer == exact_rows_scope->resource_down[i]; + } + if (!exact_rows_supported) { + fprintf(stderr, + "ds4: Metal exact-row expert union does not match routed MoE layer=%u row=%u\n", + layer_index, + exact_rows_scope->row); + return 0; + } + use_exact_rows_scope = true; + stream_addr_resources = exact_rows_scope->resources; + stream_addr_resource_count = exact_rows_scope->n_resources; + stream_gate_addr_buf = exact_rows_scope->gate_addrs; + stream_up_addr_buf = exact_rows_scope->up_addrs; + stream_down_addr_buf = exact_rows_scope->down_addrs; + stream_overflow_gate = exact_rows_scope->overflow_gate; + stream_overflow_up = exact_rows_scope->overflow_up; + stream_overflow_down = exact_rows_scope->overflow_down; + } id slots_pair_swiglu_pipeline = use_iq2_selected_slots ? g_moe_mul_mv_slots6_iq2_xxs_pair_swiglu_pipeline : (use_mxfp4_selected_slots ? g_moe_mul_mv_slots6_mxfp4_pair_swiglu_pipeline : @@ -40040,21 +40652,23 @@ int ds4_gpu_routed_moe_one_tensor( bool selected_ids_available = true; bool selected_exec_ids_from_host = false; const int stream_expert_cache_size_known = + use_exact_rows_scope ? 1 : ds4_gpu_stream_expert_cache_note_expert_size(gate_expert_bytes, down_expert_bytes); const bool use_iq2_full_expert_addr_table = - use_iq2_selected_slots && + !use_exact_rows_scope && use_iq2_selected_slots && ds4_gpu_stream_full_expert_addr_table_requested() && g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_pipeline != nil && g_moe_mul_mv_addr_q2_k_sum6_pipeline != nil; use_stream_expert_cache = - !use_iq2_full_expert_addr_table && - (use_iq2_selected_slots || use_iq2_stream_addr_table || - use_q4_selected_slots || use_mxfp4_selected_slots) && - stream_expert_cache_size_known && - ds4_gpu_stream_expert_cache_effective_cap(layer_index, - n_total_expert, - n_expert) != 0; + use_exact_rows_scope || + (!use_iq2_full_expert_addr_table && + (use_iq2_selected_slots || use_iq2_stream_addr_table || + use_q4_selected_slots || use_mxfp4_selected_slots) && + stream_expert_cache_size_known && + ds4_gpu_stream_expert_cache_effective_cap(layer_index, + n_total_expert, + n_expert) != 0); if (use_iq2_stream_addr_table && !use_stream_expert_cache) { fprintf(stderr, "ds4: Metal IQ2/IQ2 streaming decode requires a non-empty expert cache\n"); @@ -40062,10 +40676,10 @@ int ds4_gpu_routed_moe_one_tensor( return 0; } const bool stream_split_ready = - use_stream_expert_cache && + !use_exact_rows_scope && use_stream_expert_cache && ds4_gpu_stream_expert_split_ready(); const bool use_stream_compact_addr = - use_stream_expert_cache && + !use_exact_rows_scope && use_stream_expert_cache && use_iq2_selected_slots && ds4_gpu_stream_compact_addr_requested() && !stream_split_ready && @@ -40073,14 +40687,14 @@ int ds4_gpu_routed_moe_one_tensor( g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_pipeline != nil && g_moe_mul_mv_addr_q2_k_sum6_pipeline != nil; use_stream_expert_split_candidate = - use_stream_expert_cache && + !use_exact_rows_scope && use_stream_expert_cache && use_iq2_selected_slots && !use_stream_compact_addr && stream_split_ready && g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_masked_pipeline != nil && g_moe_mul_mv_addr_q2_k_sum6_masked_pipeline != nil; const bool use_stream_hit_validator = - use_stream_expert_cache && + !use_exact_rows_scope && use_stream_expert_cache && use_iq2_selected_slots && ds4_gpu_stream_expert_hit_validator_requested() && g_moe_stream_expert_cache_validate_pipeline != nil && @@ -40090,7 +40704,19 @@ int ds4_gpu_routed_moe_one_tensor( &stream_gate_addr_buf, &stream_up_addr_buf, &stream_down_addr_buf); - if (use_iq2_full_expert_addr_table) { + if (use_exact_rows_scope) { + selected_id_source = "exact-union"; + memcpy(selected_ids, + exact_rows_scope->selected_ids + + (uint64_t)exact_rows_scope->row * n_expert, + (size_t)n_expert * sizeof(selected_ids[0])); + g_routed_moe_selected_override_n = 0; + use_stream_expert_cache = true; + use_stream_expert_addr_table = true; + use_stream_expert_masked_addr_table = false; + use_stream_expert_split_candidate = false; + use_stream_expert_split_deferred = false; + } else if (use_iq2_full_expert_addr_table) { selected_id_source = "gpu-full-addr"; selected_ids_available = false; g_routed_moe_selected_override_n = 0; @@ -40213,7 +40839,7 @@ int ds4_gpu_routed_moe_one_tensor( selected_t0 = ds4_gpu_now_ms(); } - if (selected_ids_available) { + if (selected_ids_available && !use_exact_rows_scope) { for (uint32_t i = 0; i < n_expert; i++) { if (selected_ids[i] < 0 || (uint32_t)selected_ids[i] >= n_total_expert) { fprintf(stderr, @@ -40346,7 +40972,40 @@ int ds4_gpu_routed_moe_one_tensor( } } } - if (use_stream_expert_cache) { + if (use_exact_rows_scope) { + /* The union scope owns a full, private address table for the + * whole speculative microbatch. Do not run any ordinary + * per-token cache setup here: it mutates the shared per-layer + * tables and may prune entries already referenced by an + * earlier exact row in this command buffer. */ + stream_addr_resources = exact_rows_scope->resources; + stream_addr_resource_count = exact_rows_scope->n_resources; + stream_gate_addr_buf = exact_rows_scope->gate_addrs; + stream_up_addr_buf = exact_rows_scope->up_addrs; + stream_down_addr_buf = exact_rows_scope->down_addrs; + stream_overflow_gate = exact_rows_scope->overflow_gate; + stream_overflow_up = exact_rows_scope->overflow_up; + stream_overflow_down = exact_rows_scope->overflow_down; + selected_exec_buf = selectedbuf; + selected_exec_off = ds4_gpu_tensor_offset(selected); + use_stream_expert_cache = true; + use_stream_expert_addr_table = true; + use_stream_expert_masked_addr_table = false; + use_stream_compact_addr_table = false; + use_stream_expert_split_candidate = false; + use_stream_expert_split_deferred = false; + if (!stream_gate_addr_buf || !stream_up_addr_buf || + !stream_down_addr_buf || + (stream_addr_resource_count == 0 && + (!stream_overflow_gate || !stream_overflow_up || + !stream_overflow_down))) { + fprintf(stderr, + "ds4: Metal exact-row expert union lost private resources at layer=%u row=%u\n", + layer_index, + exact_rows_scope->row); + return 0; + } + } else if (use_stream_expert_cache) { use_stream_expert_addr_table = ((use_iq2_selected_slots && ds4_gpu_stream_expert_addr_table_kernel_requested() && @@ -41194,8 +41853,8 @@ int ds4_gpu_routed_moe_one_tensor( g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_pipeline, &gate_args, &act_args, - stream_slot_entries, - n_expert, + stream_addr_resources, + stream_addr_resource_count, stream_gate_addr_buf, stream_up_addr_buf, xbuf, @@ -41213,8 +41872,8 @@ int ds4_gpu_routed_moe_one_tensor( gate_smem, 2, false, - nil, - nil); + stream_overflow_gate, + stream_overflow_up); } } else { ok = (!use_stream_expert_cache || @@ -41543,8 +42202,8 @@ int ds4_gpu_routed_moe_one_tensor( ok = ds4_gpu_encode_mul_mv_addr_q2_sum6(cb, g_moe_mul_mv_addr_q2_k_sum6_pipeline, &down_args, - stream_slot_entries, - n_expert, + stream_addr_resources, + stream_addr_resource_count, stream_down_addr_buf, midbuf, ds4_gpu_tensor_offset(mid), @@ -41554,7 +42213,7 @@ int ds4_gpu_routed_moe_one_tensor( selected_exec_off, down_smem, 2, - nil); + stream_overflow_down); } } else { ok = (!use_stream_expert_cache || @@ -41630,6 +42289,10 @@ int ds4_gpu_routed_moe_one_tensor( return 0; } } + if (use_exact_rows_scope) { + exact_rows_scope->row_armed = 0; + exact_rows_scope->next_row++; + } #undef DS4_METAL_PROFILE_MOE_ONE_STAGE } @@ -42236,7 +42899,8 @@ int ds4_gpu_routed_moe_batch_tensor( &stream_unique, &stream_overflow_gate, &stream_overflow_up, - &stream_overflow_down)) { + &stream_overflow_down, + false)) { g_stream_prefill_batch_selected_addr_building--; return 0; } @@ -43506,7 +44170,6 @@ int ds4_gpu_hc_rms_norm_mix_f16_tensor( return 1; } - int ds4_gpu_hc_rms_norm_mix_split_norm_f16_tensor( ds4_gpu_tensor *mix, ds4_gpu_tensor *out, @@ -44574,7 +45237,137 @@ int ds4_gpu_matmul_q8_0_hc_expand_tensor( return 1; } +int ds4_gpu_matmul_q4_K_hc_expand_available(void) { + if (!g_initialized && !ds4_gpu_init()) return 0; + if (getenv("DS4_METAL_DISABLE_Q4_ATTN_OUT_HC_FUSE") != NULL) return 0; + return ds4_gpu_get_mul_mv_ext_pipeline( + "kernel_dsv4_q4_K_hc_expand4", 2, 8) != nil; +} + +int ds4_gpu_matmul_q4_K_hc_expand_tensor( + ds4_gpu_tensor *out_hc, + ds4_gpu_tensor *block_out, + const void *model_map, + uint64_t model_size, + uint64_t weight_offset, + uint64_t in_dim, + uint64_t out_dim, + const ds4_gpu_tensor *x, + const ds4_gpu_tensor *residual_hc, + const ds4_gpu_tensor *split, + uint32_t n_embd, + uint32_t n_hc) { + if (!g_initialized && !ds4_gpu_init()) return 0; + if (getenv("DS4_METAL_DISABLE_Q4_ATTN_OUT_HC_FUSE") != NULL) return 0; + if (!out_hc || !block_out || !model_map || !x || !residual_hc || !split || + n_embd == 0u || n_hc != 4u || out_dim != n_embd || + in_dim == 0u || (in_dim % 256u) != 0u || + (out_dim & 1u) != 0u || + in_dim > UINT32_MAX || out_dim > UINT32_MAX) { + return 0; + } + + @autoreleasepool { + const uint64_t row_bytes = (in_dim / 256u) * 144u; + if (out_dim > UINT64_MAX / row_bytes) return 0; + const uint64_t weight_bytes = out_dim * row_bytes; + const uint64_t x_bytes = in_dim * sizeof(float); + const uint64_t embd_bytes = out_dim * sizeof(float); + const uint64_t hc_bytes = (uint64_t)n_hc * n_embd * sizeof(float); + const uint64_t mix_hc = 2ull * n_hc + (uint64_t)n_hc * n_hc; + const uint64_t split_bytes = mix_hc * sizeof(float); + if (weight_offset > model_size || + weight_bytes > model_size - weight_offset) { + return 0; + } + + id xbuf = ds4_gpu_tensor_buffer(x); + id blockbuf = ds4_gpu_tensor_buffer(block_out); + id resbuf = ds4_gpu_tensor_buffer(residual_hc); + id splitbuf = ds4_gpu_tensor_buffer(split); + id outbuf = ds4_gpu_tensor_buffer(out_hc); + if (!xbuf || !blockbuf || !resbuf || !splitbuf || !outbuf || + ds4_gpu_tensor_bytes(x) < x_bytes || + ds4_gpu_tensor_bytes(block_out) < embd_bytes || + ds4_gpu_tensor_bytes(residual_hc) < hc_bytes || + ds4_gpu_tensor_bytes(split) < split_bytes || + ds4_gpu_tensor_bytes(out_hc) < hc_bytes) { + return 0; + } + + uint64_t inner_offset = 0; + id wbuf = ds4_gpu_wrap_model_range( + model_map, model_size, weight_offset, weight_bytes, + &inner_offset); + if (!wbuf) return 0; + + ds4_gpu_q8_0_matvec_args mv_args = { + .ne00 = (int32_t)in_dim, .ne01 = (int32_t)out_dim, .ne02 = 1, + .nb00 = 1, .nb01 = row_bytes, + .nb02 = row_bytes * out_dim, .nb03 = row_bytes * out_dim, + .ne10 = (int32_t)in_dim, .ne11 = 1, .ne12 = 1, + .nb10 = sizeof(float), .nb11 = in_dim * sizeof(float), + .nb12 = in_dim * sizeof(float), .nb13 = in_dim * sizeof(float), + .ne0 = (int32_t)out_dim, .ne1 = 1, .nr0 = 2, + .r2 = 1, .r3 = 1, + }; + ds4_gpu_hc_expand_args hc_args = { + .n_embd = n_embd, + .n_hc = n_hc, + .n_tokens = 1, + .nb_block0 = sizeof(float), + .nb_block1 = (uint64_t)n_embd * sizeof(float), + .nb_add0 = sizeof(float), + .nb_add1 = (uint64_t)n_embd * sizeof(float), + .nb_res0 = sizeof(float), + .nb_res1 = (uint64_t)n_embd * sizeof(float), + .nb_res2 = (uint64_t)n_hc * n_embd * sizeof(float), + .nb_post0 = sizeof(float), + .nb_post1 = mix_hc * sizeof(float), + .nb_comb0 = sizeof(float), + .nb_comb1 = (uint64_t)n_hc * sizeof(float), + .nb_comb2 = mix_hc * sizeof(float), + .nb0 = sizeof(float), + .nb1 = (uint64_t)n_embd * sizeof(float), + .nb2 = (uint64_t)n_hc * n_embd * sizeof(float), + .has_add = 0, + }; + + const int16_t nsg = 2; + id pipeline = ds4_gpu_get_mul_mv_ext_pipeline( + "kernel_dsv4_q4_K_hc_expand4", nsg, 8); + if (!pipeline) return 0; + + int owned = 0; + id cb = ds4_gpu_command_buffer(&owned); + if (!cb) return 0; + id enc = ds4_gpu_compute_encoder(cb); + [enc setComputePipelineState:pipeline]; + [enc setBytes:&mv_args length:sizeof(mv_args) atIndex:0]; + [enc setBytes:&hc_args length:sizeof(hc_args) atIndex:1]; + [enc setBuffer:wbuf offset:(NSUInteger)inner_offset atIndex:2]; + [enc setBuffer:xbuf offset:ds4_gpu_tensor_offset(x) atIndex:3]; + [enc setBuffer:blockbuf offset:ds4_gpu_tensor_offset(block_out) atIndex:4]; + [enc setBuffer:resbuf offset:ds4_gpu_tensor_offset(residual_hc) atIndex:5]; + [enc setBuffer:splitbuf + offset:ds4_gpu_tensor_offset(split) + + (NSUInteger)n_hc * sizeof(float) + atIndex:6]; + [enc setBuffer:splitbuf + offset:ds4_gpu_tensor_offset(split) + + (NSUInteger)(2u * n_hc) * sizeof(float) + atIndex:7]; + [enc setBuffer:outbuf offset:ds4_gpu_tensor_offset(out_hc) atIndex:8]; + [enc setThreadgroupMemoryLength:32u atIndex:0]; + [enc dispatchThreadgroups:MTLSizeMake( + ((NSUInteger)out_dim + 3u) / 4u, 1, 1) + threadsPerThreadgroup:MTLSizeMake(32, (NSUInteger)nsg, 1)]; + ds4_gpu_end_compute_encoder(cb, enc); + return ds4_gpu_finish_command_buffer( + cb, owned, "Q4_K HC expand fused"); + } +} /* Kimi Delta Attention primitive shared with the GLM-5.3 graph. */ static int glm53_gpu_mul_u64(uint64_t a, uint64_t b, uint64_t *out) { diff --git a/metal/dsv4_hc.metal b/metal/dsv4_hc.metal index c46794675..bf1b3a316 100644 --- a/metal/dsv4_hc.metal +++ b/metal/dsv4_hc.metal @@ -1106,7 +1106,6 @@ kernel void kernel_dsv4_output_hc_weights4( } } - struct ds4_metal_args_hc_norm_mix { int32_t n; int32_t out_dim; @@ -1202,7 +1201,6 @@ kernel void kernel_dsv4_hc_rms_norm_mix_f16( FOR_UNROLL (short i = 0; i < NF4; ++i) { sumq += dot(float4(xb4[i]), yl4[i]); } - sumf_mv[row] += sumq; } } diff --git a/metal/moe.metal b/metal/moe.metal index cc7065247..904d33281 100644 --- a/metal/moe.metal +++ b/metal/moe.metal @@ -3409,6 +3409,216 @@ kernel void kernel_mul_mv_q4_K_dense_pair_f32( } } +// Decode Q4_K Q-A/KV pair plus the attention/indexer F16 compressor pairs. +// The Q4_K range calls the same per-row implementation as the standalone +// dense-pair kernel. With NSG=8 each simdgroup still owns an independent +// two-row cohort, so changing the number of cohorts in a threadgroup does not +// change the K walk or the simd reduction for any output row. The shifted +// compressor ranges are the standalone F16 pair/store body verbatim. The +// fused dispatch therefore removes encoder/dispatch transitions while +// preserving the bits produced by all three original kernels. +kernel void kernel_dsv4_q4_K_qkv_pair_quad_compressor_store( + constant ds4_metal_args_mul_mv & args0, + constant ds4_metal_args_mul_mv & args1, + constant ds4_metal_args_mul_mv & cargs, + constant ds4_metal_args_compressor_pair_store & store0, + constant ds4_metal_args_compressor_pair_store & store1, + constant uint & pair_tgs, + device const char * qw0, + device const char * qw1, + device const char * cw0a, + device const char * cw0b, + device const char * cw1a, + device const char * cw1b, + device const char * src1, + device char * dst0, + device char * dst1, + device char * cdst_a0, + device char * cdst_b0, + device char * cdst_a1, + device char * cdst_b1, + device const char * ape0, + device const char * ape1, + device float * state0_kv, + device float * state0_score, + device float * state1_kv, + device float * state1_score, + threadgroup char * shmem [[threadgroup(0)]], + uint3 tgpig [[threadgroup_position_in_grid]], + ushort tiitg [[thread_index_in_threadgroup]], + ushort tiisg [[thread_index_in_simdgroup]], + ushort sgitg [[simdgroup_index_in_threadgroup]]) { + if (tgpig.x < pair_tgs) { + const int first_row = + (tgpig.x * FC_mul_mv_nsg + sgitg) * N_R0_Q4_K; + if (first_row < args0.ne0) { + kernel_mul_mv_q4_K_f32_impl( + args0, qw0, src1, dst0, shmem, tgpig, tiisg, sgitg); + } + if (first_row < args1.ne0) { + kernel_mul_mv_q4_K_f32_impl( + args1, qw1, src1, dst1, shmem, tgpig, tiisg, sgitg); + } + return; + } + + constexpr short NR0 = 2; + const uint lx = tgpig.x - pair_tgs; + const uint tgs0 = ((uint)store0.width + NR0 - 1u) / NR0; + const uint tgs1 = ((uint)store1.width + NR0 - 1u) / NR0; + if (lx >= tgs0 + tgs1) return; + const bool second = lx >= tgs0; + + uint3 local_tgpig = tgpig; + local_tgpig.x = second ? lx - tgs0 : lx; + + ds4_metal_args_mul_mv largs = cargs; + largs.nr0 = NR0; + largs.ne01 = second ? (int32_t)store1.width : (int32_t)store0.width; + + if (!second) { + kernel_mul_mv_f16_f32_pair_4_impl( + largs, cw0a, cw0b, src1, cdst_a0, cdst_b0, + shmem, local_tgpig, tiisg, sgitg); + } else { + kernel_mul_mv_f16_f32_pair_4_impl( + largs, cw1a, cw1b, src1, cdst_a1, cdst_b1, + shmem, local_tgpig, tiisg, sgitg); + } + + threadgroup_barrier(mem_flags::mem_device); + + constant ds4_metal_args_compressor_pair_store & store = + second ? store1 : store0; + if (tiitg >= NR0 || store.width == 0u || store.ratio == 0u) { + return; + } + const uint col = local_tgpig.x * (uint)NR0 + tiitg; + if (col >= store.width) return; + + const uint pos_mod = store.pos % store.ratio; + const uint dst_row = store.ratio == 4u ? store.ratio + pos_mod : pos_mod; + const uint dst = dst_row * store.width + col; + const uint ape_i = pos_mod * store.width + col; + + device volatile const float * projected_kv = second + ? (device volatile const float *)cdst_a1 + : (device volatile const float *)cdst_a0; + device volatile const float * projected_score = second + ? (device volatile const float *)cdst_b1 + : (device volatile const float *)cdst_b0; + device const char * ape = second ? ape1 : ape0; + device float * state_kv = second ? state1_kv : state0_kv; + device float * state_score = second ? state1_score : state0_score; + + float ape_v; + if (store.ape_type == 1u) { + ape_v = (float)(((device const half *)ape)[ape_i]); + } else { + ape_v = ((device const float *)ape)[ape_i]; + } + + state_kv[dst] = projected_kv[col]; + state_score[dst] = projected_score[col] + ape_v; +} + +// ABI-compatible subset of ds4_metal_args_dsv4_hc_expand. The Q4_K kernel +// lives in this source file beside the classic Q4 implementation, while the +// generic HC kernels are concatenated later from dsv4_hc.metal. +struct ds4_metal_args_q4_hc_expand { + int64_t n_embd; + int64_t n_hc; + int64_t n_tokens; + uint64_t nb_block0; + uint64_t nb_block1; + uint64_t nb_add0; + uint64_t nb_add1; + uint64_t nb_res0; + uint64_t nb_res1; + uint64_t nb_res2; + uint64_t nb_post0; + uint64_t nb_post1; + uint64_t nb_comb0; + uint64_t nb_comb1; + uint64_t nb_comb2; + uint64_t nb0; + uint64_t nb1; + uint64_t nb2; + int32_t has_add; +}; + +// Decode attention-output tail for AProjQ4: +// +// block_out = input @ Wob(Q4_K) +// out_hc = HCPost(block_out, residual_hc, split) +// +// The matvec is the exact standalone classic Q4_K implementation. Once its +// stored F32 row is visible to the threadgroup, the owning simdgroup expands +// the same value into the four HC streams. Materializing block_out preserves +// diagnostics and makes an A/B memcmp possible. +kernel void kernel_dsv4_q4_K_hc_expand4( + constant ds4_metal_args_mul_mv & mv, + constant ds4_metal_args_q4_hc_expand & hc, + device const char * weight, + device const char * input, + device char * block_out, + device const char * residual, + device const char * post, + device const char * comb, + device char * dst, + threadgroup char * shmem [[threadgroup(0)]], + uint3 tgpig [[threadgroup_position_in_grid]], + ushort tiisg [[thread_index_in_simdgroup]], + ushort sgitg [[simdgroup_index_in_threadgroup]]) { + if (hc.n_hc != 4 || hc.n_tokens != 1 || hc.has_add != 0 || + mv.ne0 != hc.n_embd || (mv.ne0 & 1) != 0 || + hc.nb_block0 != sizeof(float)) { + return; + } + + const int first_row = + (tgpig.x * FC_mul_mv_nsg + sgitg) * N_R0_Q4_K; + if (first_row < mv.ne0) { + kernel_mul_mv_q4_K_f32_impl( + mv, weight, input, block_out, shmem, + tgpig, tiisg, sgitg); + } + + threadgroup_barrier(mem_flags::mem_device); + + if (tiisg != 0) return; + FOR_UNROLL(short row = 0; row < N_R0_Q4_K; ++row) { + const int d = first_row + row; + if (d >= mv.ne0) continue; + + const float block_v = *((device const float *)( + block_out + (uint64_t)d * hc.nb_block0)); + const float r0 = *((device const float *)( + residual + (uint64_t)d * hc.nb_res0 + 0 * hc.nb_res1)); + const float r1 = *((device const float *)( + residual + (uint64_t)d * hc.nb_res0 + 1 * hc.nb_res1)); + const float r2 = *((device const float *)( + residual + (uint64_t)d * hc.nb_res0 + 2 * hc.nb_res1)); + const float r3 = *((device const float *)( + residual + (uint64_t)d * hc.nb_res0 + 3 * hc.nb_res1)); + + FOR_UNROLL(short dst_hc = 0; dst_hc < 4; ++dst_hc) { + float acc = block_v * *((device const float *)( + post + (uint64_t)dst_hc * hc.nb_post0)); + acc += *((device const float *)( + comb + (uint64_t)dst_hc * hc.nb_comb0 + 0 * hc.nb_comb1)) * r0; + acc += *((device const float *)( + comb + (uint64_t)dst_hc * hc.nb_comb0 + 1 * hc.nb_comb1)) * r1; + acc += *((device const float *)( + comb + (uint64_t)dst_hc * hc.nb_comb0 + 2 * hc.nb_comb1)) * r2; + acc += *((device const float *)( + comb + (uint64_t)dst_hc * hc.nb_comb0 + 3 * hc.nb_comb1)) * r3; + *((device float *)(dst + (uint64_t)d * hc.nb0 + + (uint64_t)dst_hc * hc.nb1)) = acc; + } + } +} + // DS4 attention output low projection, specialized for the fixed block // diagonal mapping used by the model: // diff --git a/tests/dspark_acceptance_fixture.sh b/tests/dspark_acceptance_fixture.sh index a82e1b452..7c9edd6b5 100644 --- a/tests/dspark_acceptance_fixture.sh +++ b/tests/dspark_acceptance_fixture.sh @@ -27,8 +27,23 @@ BACKEND=${DS4_DSPARK_FIXTURE_BACKEND:-auto} SSD_STREAMING=${DS4_DSPARK_FIXTURE_SSD_STREAMING:-0} SSD_CACHE_EXPERTS=${DS4_DSPARK_FIXTURE_SSD_STREAMING_CACHE_EXPERTS:-} REQUIRE_ACTIVE=${DS4_DSPARK_FIXTURE_REQUIRE_ACTIVE:-1} +REQUIRE_EXACT2=${DS4_DSPARK_FIXTURE_REQUIRE_EXACT2:-0} total_proposed=0 total_accepted_draft=0 +total_exact2_attempt=0 +total_exact2_fallback=0 + +stats_field() { + printf '%s\n' "$1" | awk -v key="$2" ' + { prefix = key "=" + for (i = 1; i <= NF; i++) { + if (index($i, prefix) == 1) { + print substr($i, length(prefix) + 1) + exit + } + } + }' +} case "$BACKEND" in auto|metal|cuda|rocm) ;; @@ -51,6 +66,13 @@ case "$REQUIRE_ACTIVE" in exit 1 ;; esac +case "$REQUIRE_EXACT2" in +0|1) ;; +*) + echo "dspark-fixture: DS4_DSPARK_FIXTURE_REQUIRE_EXACT2 must be 0 or 1" >&2 + exit 1 + ;; +esac case "$SSD_CACHE_EXPERTS" in ""|*[!0-9]*) if [ -n "$SSD_CACHE_EXPERTS" ]; then @@ -124,6 +146,11 @@ print_metadata() { hw_model=$(sysctl -n hw.model 2>/dev/null || true) hw_cpu=$(sysctl -n machdep.cpu.brand_string 2>/dev/null || true) confidence=${CONFIDENCE:-default} + exact2_cuda=${DS4_CUDA_DSPARK_EXACT2:-unset} + exact2_metal=${DS4_METAL_DSPARK_EXACT2:-unset} + proposer_cap_cuda=${DS4_CUDA_DSPARK_PROPOSER_BLOCK_MAX:-unset} + proposer_cap_metal=${DS4_METAL_DSPARK_PROPOSER_BLOCK_MAX:-unset} + verifier_cap=${DS4_DSPARK_SSD_VERIFY_BLOCK_MAX:-unset} printf '# commit=%s\n' "$(git_commit_label)" printf '# hardware_os=%s hardware_model=%s hardware_cpu=%s\n' \ @@ -144,6 +171,9 @@ print_metadata() { "$DS4_BIN" "${exact_sampling_arg:+ $exact_sampling_arg}" \ "${CONFIDENCE:+ --dspark-confidence $CONFIDENCE}" \ "$MODEL" "$SUPPORT" "$TOKENS" "$TEMPERATURE" "$TOP_P" "$MIN_P" "$SEED" + printf '# exact2_cuda=%s exact2_metal=%s proposer_block_max_cuda=%s proposer_block_max_metal=%s verifier_block_max=%s require_exact2=%s\n' \ + "$exact2_cuda" "$exact2_metal" "$proposer_cap_cuda" \ + "$proposer_cap_metal" "$verifier_cap" "$REQUIRE_EXACT2" } if [ ! -x "$DS4_BIN" ]; then @@ -232,13 +262,15 @@ run_case() { return 1 fi - partial=$(printf '%s\n' "$stats" | sed -n 's/.* partial=\([0-9][0-9]*\).*/\1/p') - errors=$(printf '%s\n' "$stats" | sed -n 's/.*errors=\([0-9][0-9]*\).*/\1/p') - verifier_unavailable=$(printf '%s\n' "$stats" | sed -n 's/.*verifier_unavailable=\([0-9][0-9]*\).*/\1/p') - proposed=$(printf '%s\n' "$stats" | sed -n 's/.*proposed=\([0-9][0-9]*\).*/\1/p') - accepted_draft=$(printf '%s\n' "$stats" | sed -n 's/.*accepted_draft=\([0-9][0-9]*\).*/\1/p') - direct_full=$(printf '%s\n' "$stats" | sed -n 's/.*direct_full=\([0-9][0-9]*\).*/\1/p') - direct_partial=$(printf '%s\n' "$stats" | sed -n 's/.*direct_partial=\([0-9][0-9]*\).*/\1/p') + partial=$(stats_field "$stats" partial) + errors=$(stats_field "$stats" errors) + verifier_unavailable=$(stats_field "$stats" verifier_unavailable) + proposed=$(stats_field "$stats" proposed) + accepted_draft=$(stats_field "$stats" accepted_draft) + direct_full=$(stats_field "$stats" direct_full) + direct_partial=$(stats_field "$stats" direct_partial) + exact2_attempt=$(stats_field "$stats" exact2_attempt) + exact2_fallback=$(stats_field "$stats" exact2_fallback) partial=${partial:-0} errors=${errors:-0} verifier_unavailable=${verifier_unavailable:-0} @@ -246,6 +278,8 @@ run_case() { accepted_draft=${accepted_draft:-0} direct_full=${direct_full:-0} direct_partial=${direct_partial:-0} + exact2_attempt=${exact2_attempt:-0} + exact2_fallback=${exact2_fallback:-0} if [ "$errors" -ne 0 ]; then echo "dspark-fixture: verifier errors for $id: $stats" >&2 return 1 @@ -256,6 +290,12 @@ run_case() { fi total_proposed=$((total_proposed + proposed)) total_accepted_draft=$((total_accepted_draft + accepted_draft)) + total_exact2_attempt=$((total_exact2_attempt + exact2_attempt)) + total_exact2_fallback=$((total_exact2_fallback + exact2_fallback)) + if [ "$REQUIRE_EXACT2" != 0 ] && [ "$exact2_fallback" -ne 0 ]; then + echo "dspark-fixture: exact2 fallback for $id: $stats" >&2 + return 1 + fi if [ "$PROPOSAL_QUALITY_GUARD_ACTIVE" -ne 0 ] && [ "$id" = c_add ] && [ "$accepted_draft" -lt "$C_ADD_MIN_ACCEPTED" ]; then echo "dspark-fixture: c_add accepted_draft $accepted_draft below required $C_ADD_MIN_ACCEPTED: $stats" >&2 @@ -298,3 +338,11 @@ if [ "$REQUIRE_ACTIVE" != 0 ] && echo "dspark-fixture: DSpark runtime was not active (proposed=$total_proposed accepted_draft=$total_accepted_draft)" >&2 exit 1 fi +if [ "$REQUIRE_EXACT2" != 0 ] && [ "$total_exact2_attempt" -eq 0 ]; then + echo "dspark-fixture: exact2 was required but never attempted" >&2 + exit 1 +fi +if [ "$REQUIRE_EXACT2" != 0 ] && [ "$total_exact2_fallback" -ne 0 ]; then + echo "dspark-fixture: exact2 fallback count=$total_exact2_fallback" >&2 + exit 1 +fi diff --git a/tests/test_metal_exactn_oracle.c b/tests/test_metal_exactn_oracle.c new file mode 100644 index 000000000..10b3a0f6b --- /dev/null +++ b/tests/test_metal_exactn_oracle.c @@ -0,0 +1,516 @@ +/* Model-backed correctness oracle for the Metal exact-N speculative path. + * + * The DSpark proposer is intentionally not involved: the test derives five + * target-greedy tokens, injects controlled full/partial/EOS draft blocks, and + * sends them through the same production verifier/commit function. Each + * resulting session is compared with ordinary one-token decode at three + * levels: serialized KV/compressor state, continuation logits, and a short + * greedy continuation. + * + * Run with: + * DS4_TEST_MODEL=/path/to/model.gguf make test-metal-exactn-oracle + * + * Running the binary directly without a model is a developer-friendly skip. + * The Make target is the release gate: it sets DS4_TEST_REQUIRE_MODEL=1 and + * therefore fails when the configured model is absent. + */ + +#include "ds4.h" + +#include +#include +#include +#include +#include +#include +#include + +#define TEST_CTX 512 +#define TEST_PREFILL_CHUNK 128u +/* Partial-N deliberately falls back to the legacy five-row verifier, whose + * routed union may need 5 * top6 slots before replaying the accepted prefix. */ +#define TEST_EXPERT_CACHE 32u +#define MAX_DRAFT 5 +#define CONTINUATION_TOKENS 4 + +/* This symbol exists only in the ds4 core compiled with DS4_TEST_HOOKS. */ +int ds4_test_session_eval_exact_drafts( + ds4_session *s, + const int *drafts, + int draft_n, + int eos_token, + int *accepted, + int accepted_cap, + char *err, + size_t errlen); + +enum { + EXACTN_UNION_ATTEMPTS = 0, + EXACTN_UNION_FULL_ACCEPTS, + EXACTN_UNION_FALLBACKS, + EXACTN_UNION_PARTIAL_FALLBACKS, + EXACTN_UNION_ERROR_FALLBACKS, + EXACTN_UNION_COUNTER_COUNT +}; + +int ds4_test_session_exactn_union_stats( + const ds4_session *s, + uint64_t out[EXACTN_UNION_COUNTER_COUNT]); + +typedef struct { + const char *name; + int draft_n; + int reject_at; /* -1 means every draft is target-greedy. */ + int eos_at; /* -1 uses the model EOS; otherwise a synthetic EOS row. */ +} exactn_case; + +static void fail(const char *what, const char *case_name, const char *detail) { + fprintf(stderr, "FAIL: %s case=%s%s%s\n", + what, + case_name ? case_name : "setup", + detail && detail[0] ? ": " : "", + detail && detail[0] ? detail : ""); + exit(1); +} + +static void restore_snapshot(ds4_session *s, + const ds4_session_snapshot *snap, + const char *case_name) { + char err[256] = ""; + if (ds4_session_load_snapshot(s, snap, err, sizeof(err)) != 0) { + fail("snapshot restore", case_name, err); + } +} + +static void save_snapshot(ds4_session *s, + ds4_session_snapshot *snap, + const char *case_name) { + char err[256] = ""; + if (ds4_session_save_snapshot(s, snap, err, sizeof(err)) != 0) { + fail("snapshot save", case_name, err); + } +} + +static void eval_tokens(ds4_session *s, + const int *tokens, + int count, + const char *case_name) { + char err[256] = ""; + for (int i = 0; i < count; i++) { + if (ds4_session_eval(s, tokens[i], err, sizeof(err)) != 0) { + char detail[320]; + snprintf(detail, sizeof(detail), "row=%d token=%d err=%s", + i, tokens[i], err); + fail("sequential decode", case_name, detail); + } + } +} + +static int copy_logits(ds4_session *s, float *out, int vocab, + const char *case_name) { + const int copied = ds4_session_copy_logits(s, out, vocab); + if (copied != vocab) { + char detail[96]; + snprintf(detail, sizeof(detail), "copied=%d vocab=%d", copied, vocab); + fail("logits read", case_name, detail); + } + return copied; +} + +static int lowest_finite_token(const float *logits, int vocab, + int excluded_a, int excluded_b) { + int token = -1; + float value = FLT_MAX; + for (int i = 0; i < vocab; i++) { + if (i == excluded_a || i == excluded_b) continue; + if (token < 0 || logits[i] < value) { + token = i; + value = logits[i]; + } + } + return token; +} + +static int greedy_continuation(ds4_session *s, int eos, + int out[CONTINUATION_TOKENS], + const char *case_name) { + char err[256] = ""; + int n = 0; + while (n < CONTINUATION_TOKENS) { + const int token = ds4_session_argmax(s); + if (token < 0) fail("continuation argmax", case_name, "negative token"); + out[n++] = token; + if (token == eos || n == CONTINUATION_TOKENS) break; + if (ds4_session_eval(s, token, err, sizeof(err)) != 0) { + fail("continuation decode", case_name, err); + } + } + return n; +} + +static void compare_logits(const float *expected, const float *actual, + int vocab, const char *case_name) { + if (memcmp(expected, actual, (size_t)vocab * sizeof(*actual)) == 0) return; + + int differing = 0; + int first = -1; + float max_abs = 0.0f; + for (int i = 0; i < vocab; i++) { + if (memcmp(&expected[i], &actual[i], sizeof(actual[i])) != 0) { + if (first < 0) first = i; + differing++; + } + float delta = fabsf(expected[i] - actual[i]); + if (!isfinite(delta)) delta = FLT_MAX; + if (delta > max_abs) max_abs = delta; + } + char detail[256]; + snprintf(detail, sizeof(detail), + "differing=%d first=%d expected=%g actual=%g max_abs=%g", + differing, first, + first >= 0 ? expected[first] : 0.0f, + first >= 0 ? actual[first] : 0.0f, + max_abs); + fail("continuation logits mismatch", case_name, detail); +} + +static void compare_snapshots(const ds4_session_snapshot *expected, + const ds4_session_snapshot *actual, + const char *case_name) { + if (expected->len != actual->len) { + char detail[160]; + snprintf(detail, sizeof(detail), "expected_bytes=%llu actual_bytes=%llu", + (unsigned long long)expected->len, + (unsigned long long)actual->len); + fail("state snapshot length mismatch", case_name, detail); + } + if (memcmp(expected->ptr, actual->ptr, (size_t)expected->len) == 0) return; + + uint64_t first = 0; + while (first < expected->len && + expected->ptr[first] == actual->ptr[first]) { + first++; + } + char detail[192]; + snprintf(detail, sizeof(detail), + "first_byte=%llu expected=0x%02x actual=0x%02x bytes=%llu", + (unsigned long long)first, + first < expected->len ? expected->ptr[first] : 0, + first < actual->len ? actual->ptr[first] : 0, + (unsigned long long)expected->len); + fail("KV/compressor snapshot mismatch", case_name, detail); +} + +static void compare_continuations(const int *expected, int expected_n, + const int *actual, int actual_n, + const char *case_name) { + if (expected_n == actual_n && + memcmp(expected, actual, (size_t)expected_n * sizeof(*actual)) == 0) { + return; + } + int first = 0; + const int common = expected_n < actual_n ? expected_n : actual_n; + while (first < common && expected[first] == actual[first]) first++; + char detail[192]; + snprintf(detail, sizeof(detail), + "expected_n=%d actual_n=%d first=%d expected=%d actual=%d", + expected_n, actual_n, first, + first < expected_n ? expected[first] : -1, + first < actual_n ? actual[first] : -1); + fail("greedy continuation mismatch", case_name, detail); +} + +static void read_union_stats( + const ds4_session *session, + uint64_t out[EXACTN_UNION_COUNTER_COUNT], + const char *case_name) { + if (ds4_test_session_exactn_union_stats(session, out) != 0) { + fail("exact-N union stats", case_name, "hook failed"); + } +} + +static void check_union_stats_delta( + const uint64_t before[EXACTN_UNION_COUNTER_COUNT], + const uint64_t after[EXACTN_UNION_COUNTER_COUNT], + const exactn_case *tc) { + uint64_t expected[EXACTN_UNION_COUNTER_COUNT] = {0}; + /* EOS in the first draft row truncates the block to N=1 before exact-N + * dispatch. Every other full block (including middle EOS) is committed + * by the union path; a deliberately wrong row falls back to the slow + * exact-N oracle after recording a partial mismatch. */ + if (tc->eos_at != 0) { + expected[EXACTN_UNION_ATTEMPTS] = 1; + if (tc->reject_at >= 0) { + expected[EXACTN_UNION_FALLBACKS] = 1; + expected[EXACTN_UNION_PARTIAL_FALLBACKS] = 1; + } else { + expected[EXACTN_UNION_FULL_ACCEPTS] = 1; + } + } + + static const char *const names[EXACTN_UNION_COUNTER_COUNT] = { + "attempt", "full", "fallback", "partial", "error" + }; + for (int i = 0; i < EXACTN_UNION_COUNTER_COUNT; i++) { + if (after[i] < before[i] || after[i] - before[i] != expected[i]) { + char detail[192]; + snprintf(detail, sizeof(detail), + "%s before=%llu after=%llu expected_delta=%llu", + names[i], + (unsigned long long)before[i], + (unsigned long long)after[i], + (unsigned long long)expected[i]); + fail("exact-N union stats delta", tc->name, detail); + } + } +} + +static void run_case(ds4_session *session, + const ds4_session_snapshot *base, + int base_pos, + ds4_session_snapshot *expected_state, + ds4_session_snapshot *actual_state, + const int correct[MAX_DRAFT], + const int wrong[MAX_DRAFT], + int model_eos, + int vocab, + float *expected_logits, + float *actual_logits, + const exactn_case *tc) { + int drafts[MAX_DRAFT]; + memcpy(drafts, correct, (size_t)tc->draft_n * sizeof(drafts[0])); + if (tc->reject_at >= 0) drafts[tc->reject_at] = wrong[tc->reject_at]; + + int cycle_eos = model_eos; + if (tc->eos_at >= 0) { + cycle_eos = drafts[tc->eos_at]; + } + + /* Run the production path first. A partial fallback may conservatively + * accept fewer correct prefix rows if the legacy batch top differs near a + * tie; it must never accept the deliberately wrong row. The sequential + * oracle is therefore built for the prefix production actually commits. */ + restore_snapshot(session, base, tc->name); + int accepted[MAX_DRAFT] = {-1, -1, -1, -1, -1}; + char err[256] = ""; + uint64_t union_before[EXACTN_UNION_COUNTER_COUNT]; + uint64_t union_after[EXACTN_UNION_COUNTER_COUNT]; + read_union_stats(session, union_before, tc->name); + const int accepted_n = ds4_test_session_eval_exact_drafts( + session, drafts, tc->draft_n, cycle_eos, + accepted, MAX_DRAFT, err, sizeof(err)); + if (accepted_n < 0) fail("exact-N cycle", tc->name, err); + read_union_stats(session, union_after, tc->name); + check_union_stats_delta(union_before, union_after, tc); + int full_expected = tc->draft_n; + if (tc->eos_at >= 0) { + /* Production truncates at the first occurrence of EOS. The fixture + * chooses a unique middle token below, but computing the first row + * here keeps the oracle correct if a future prompt changes. */ + for (int i = 0; i < tc->draft_n; i++) { + if (drafts[i] == cycle_eos) { + full_expected = i + 1; + break; + } + } + } + if (tc->reject_at < 0 && accepted_n != full_expected) { + char detail[160]; + snprintf(detail, sizeof(detail), "expected=%d actual=%d err=%s", + full_expected, accepted_n, err); + fail("accepted prefix length", tc->name, detail); + } + if (tc->reject_at >= 0 && + (accepted_n <= 0 || accepted_n > tc->reject_at)) { + char detail[160]; + snprintf(detail, sizeof(detail), + "reject_at=%d accepted=%d err=%s", + tc->reject_at, accepted_n, err); + fail("partial accepted wrong row", tc->name, detail); + } + for (int i = 0; i < accepted_n; i++) { + if (accepted[i] != correct[i]) { + char detail[128]; + snprintf(detail, sizeof(detail), + "row=%d expected=%d actual=%d", + i, correct[i], accepted[i]); + fail("accepted token", tc->name, detail); + } + } + if (ds4_session_pos(session) != base_pos + accepted_n) { + fail("checkpoint length", tc->name, "unexpected committed position"); + } + + copy_logits(session, actual_logits, vocab, tc->name); + save_snapshot(session, actual_state, tc->name); + int actual_cont[CONTINUATION_TOKENS]; + const int actual_cont_n = tc->eos_at >= 0 ? 0 : + greedy_continuation(session, model_eos, actual_cont, tc->name); + + restore_snapshot(session, base, tc->name); + eval_tokens(session, correct, accepted_n, tc->name); + copy_logits(session, expected_logits, vocab, tc->name); + save_snapshot(session, expected_state, tc->name); + int expected_cont[CONTINUATION_TOKENS]; + const int expected_cont_n = tc->eos_at >= 0 ? 0 : + greedy_continuation(session, model_eos, expected_cont, tc->name); + + compare_logits(expected_logits, actual_logits, vocab, tc->name); + compare_snapshots(expected_state, actual_state, tc->name); + compare_continuations(expected_cont, expected_cont_n, + actual_cont, actual_cont_n, tc->name); + fprintf(stderr, + "PASS: exact-N oracle case=%s drafted=%d committed=%d " + "snapshot_bytes=%llu continuation=%d\n", + tc->name, tc->draft_n, accepted_n, + (unsigned long long)actual_state->len, actual_cont_n); +} + +int main(void) { +#ifndef __APPLE__ + fprintf(stderr, "test_metal_exactn_oracle: skipped (Metal requires macOS)\n"); + return 0; +#else + const char *model = getenv("DS4_TEST_MODEL"); + if (!model || !model[0] || access(model, R_OK) != 0) { + const char *required = getenv("DS4_TEST_REQUIRE_MODEL"); + fprintf(stderr, + "test_metal_exactn_oracle: %s " + "(set DS4_TEST_MODEL to a readable target GGUF)\n", + required && required[0] && strcmp(required, "0") != 0 + ? "FAIL: required model is missing" + : "skipped"); + return required && required[0] && strcmp(required, "0") != 0 ? 1 : 0; + } + + setenv("DS4_TEST_METAL_EXACTN_ORACLE", "1", 1); + setenv("DS4_METAL_DSPARK_EXACTN_UNION", "1", 1); + setenv("DS4_METAL_DSPARK_EXACTN", "1", 1); + setenv("DS4_DSPARK_STATS", "1", 1); + setenv("DS4_DSPARK_SSD_VERIFY_BLOCK_MAX", "5", 1); + setenv("DS4_METAL_DSPARK_ACCEPTANCE_ONLY_VERIFY", "0", 1); + setenv("DS4_METAL_DSPARK_EXACT2", "0", 1); + + ds4_engine_options opt; + memset(&opt, 0, sizeof(opt)); + opt.model_path = model; + opt.backend = DS4_BACKEND_METAL; + opt.context_size = TEST_CTX; + opt.prefill_chunk = TEST_PREFILL_CHUNK; + opt.ssd_streaming = true; + opt.ssd_streaming_cold = true; + opt.ssd_streaming_cache_experts = TEST_EXPERT_CACHE; + + ds4_engine *engine = NULL; + if (ds4_engine_open(&engine, &opt) != 0 || !engine) { + fail("engine open", NULL, model); + } + + ds4_tokens prompt = {0}; + ds4_encode_chat_prompt( + engine, + NULL, + "Continue this sequence concisely: 1, 2, 3, 4, 5,", + DS4_THINK_NONE, + &prompt); + if (prompt.len <= 0 || prompt.len >= TEST_CTX - 16) { + fail("prompt tokenization", NULL, "invalid prompt length"); + } + + ds4_session *session = NULL; + if (ds4_session_create(&session, engine, TEST_CTX) != 0 || !session) { + fail("session create", NULL, "failed"); + } + char err[256] = ""; + if (ds4_session_sync(session, &prompt, err, sizeof(err)) != 0) { + fail("session prefill", NULL, err); + } + + ds4_session_snapshot base = {0}; + ds4_session_snapshot expected_state = {0}; + ds4_session_snapshot actual_state = {0}; + save_snapshot(session, &base, "setup"); + + const int vocab = ds4_engine_vocab_size(engine); + const int model_eos = ds4_token_eos(engine); + float *probe_logits = malloc((size_t)vocab * sizeof(*probe_logits)); + float *expected_logits = malloc((size_t)vocab * sizeof(*expected_logits)); + float *actual_logits = malloc((size_t)vocab * sizeof(*actual_logits)); + if (!probe_logits || !expected_logits || !actual_logits) { + fail("host allocation", NULL, "logits"); + } + + int correct[MAX_DRAFT]; + int wrong[MAX_DRAFT]; + for (int i = 0; i < MAX_DRAFT; i++) { + copy_logits(session, probe_logits, vocab, "draft derivation"); + correct[i] = ds4_session_argmax(session); + if (correct[i] < 0 || correct[i] == model_eos) { + fail("draft derivation", NULL, "unexpected early EOS"); + } + wrong[i] = lowest_finite_token(probe_logits, vocab, + correct[i], model_eos); + if (wrong[i] < 0) fail("draft derivation", NULL, "no rejection token"); + eval_tokens(session, &correct[i], 1, "draft derivation"); + } + restore_snapshot(session, &base, "setup"); + + int eos_middle_at = -1; + for (int candidate = 2; candidate < MAX_DRAFT && eos_middle_at < 0; + candidate++) { + bool unique = true; + for (int previous = 0; previous < candidate; previous++) { + if (correct[previous] == correct[candidate]) { + unique = false; + break; + } + } + if (unique) eos_middle_at = candidate; + } + for (int candidate = 1; candidate < 2 && eos_middle_at < 0; candidate++) { + if (correct[candidate] != correct[0]) eos_middle_at = candidate; + } + if (eos_middle_at < 1) { + fail("EOS fixture derivation", NULL, + "no target token unique after the first draft row"); + } + + const exactn_case cases[] = { + {"full-2", 2, -1, -1}, + {"full-3", 3, -1, -1}, + {"full-4", 4, -1, -1}, + /* Five drafts plus the already-generated target token exercise the + * requested six-token speculative cycle. */ + {"full-5", 5, -1, -1}, + {"partial-5-at1", 5, 1, -1}, + {"partial-5-at2", 5, 2, -1}, + {"partial-5-at3", 5, 3, -1}, + {"partial-5-at4", 5, 4, -1}, + {"eos-first", 5, -1, 0}, + {"eos-middle", 5, -1, eos_middle_at}, + }; + + for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { + run_case(session, &base, prompt.len, &expected_state, &actual_state, + correct, wrong, model_eos, vocab, + expected_logits, actual_logits, &cases[i]); + } + + fprintf(stderr, + "test_metal_exactn_oracle PASS cases=%zu N=2..5 " + "(six-token cycle at N=5) partial_prefixes=1..4 " + "eos=first,middle\n", + sizeof(cases) / sizeof(cases[0])); + + free(actual_logits); + free(expected_logits); + free(probe_logits); + ds4_session_snapshot_free(&actual_state); + ds4_session_snapshot_free(&expected_state); + ds4_session_snapshot_free(&base); + ds4_session_free(session); + ds4_tokens_free(&prompt); + ds4_engine_close(engine); + return 0; +#endif +} From 3a383e96363da6bb36dcff40eb153b5d65741345 Mon Sep 17 00:00:00 2001 From: Giorgio Oppo Date: Sun, 9 Aug 2026 20:43:30 +0200 Subject: [PATCH 014/189] Snapshot GPU optimizations before upstream merge --- QA_BEFORE_RELEASES.md | 70 +++ README.md | 109 ++++- ds4.c | 443 ++++++++++++++++- ds4_cuda.cu | 214 ++++++++- ds4_gpu.h | 37 ++ ds4_metal.m | 155 +++++- metal/dsv4_hc.metal | 274 +++++++++++ tests/ds4_test.c | 742 ++++++++++++++++++++++++++++- tests/dspark_acceptance_fixture.sh | 52 ++ 9 files changed, 2036 insertions(+), 60 deletions(-) diff --git a/QA_BEFORE_RELEASES.md b/QA_BEFORE_RELEASES.md index 701fb27f2..ba14e6207 100644 --- a/QA_BEFORE_RELEASES.md +++ b/QA_BEFORE_RELEASES.md @@ -318,9 +318,16 @@ than a failure. `--dspark-strict` remains the byte-identical target-only mode. | --- | --- | --- | | HC RMSNorm + F16 mixer on M1-M4 | `DS4_METAL_DISABLE_PRE_M5_HC_NORM_MIX_FUSE=1` | Leave the disable switch unset | | HC RMSNorm + F16 mixer on another Apple generation | Leave both HC norm/mix switches unset | `DS4_METAL_ENABLE_HC_NORM_MIX_FUSE=1` | + | HC producer + split/Sinkhorn/destination RMSNorm on M1-M5 | `DS4_METAL_DISABLE_HC_PRODUCER_PRE_NORM_FUSE=1` | Leave the disable switch unset | | Q4 Q-A/KV + compressor store in exact-union | `DS4_METAL_DSPARK_EXACTN_UNION=1` with the Q4 enable switch unset | Keep exact-union `=1`; set `DS4_METAL_ENABLE_Q4_QKV_COMPRESSOR_FUSE=1` | | Q4 Q-A/KV + compressor store in ordinary `FULL` decode | Leave `DS4_METAL_ENABLE_Q4_QKV_COMPRESSOR_FUSE` unset | Set `DS4_METAL_ENABLE_Q4_QKV_COMPRESSOR_FUSE=1` | + | F16 attention+indexer quad compressor store in `FULL` decode | `DS4_METAL_DISABLE_COMPRESSOR_QUAD_STORE=1` | Leave the disable switch unset | + | F16 attention+indexer quad compressor store in exact-union | `DS4_METAL_DSPARK_EXACTN_UNION=1 DS4_METAL_DISABLE_COMPRESSOR_QUAD_STORE=1` | Keep exact-union `=1`; leave the quad disable switch unset | + | Exact ratio-4 one-row compressor pool on M1-M5 | `DS4_METAL_DISABLE_COMPRESSOR_EXACT_POOL_RATIO4=1` | Leave the disable switch unset | | Q4 attention-output B + HC expansion | `DS4_METAL_DISABLE_Q4_ATTN_OUT_HC_FUSE=1` | Leave the disable switch unset | + | FlashAttention pad/block PSO memo | `DS4_METAL_DISABLE_PRE_M5_FLASH_ATTN_PAD_BLK_MEMO=1` | Leave the disable switch unset | + | FlashAttention batched/vector PSO memo | `DS4_METAL_DISABLE_PRE_M5_FLASH_ATTN_BATCHED_MEMO=1` | Leave the disable switch unset | + | Exact-union asynchronous routed tails | `DS4_METAL_DSPARK_EXACTN_UNION=1` with `DS4_METAL_DSPARK_EXACT_ROWS_ASYNC_TAILS` unset | Keep exact-union `=1`; set `DS4_METAL_DSPARK_EXACT_ROWS_ASYNC_TAILS=1` | The Q4 Q-A/KV compound is opt-in in both exact-union and ordinary `FULL` decode; it is enabled only when the explicit enable variable is present. @@ -330,6 +337,33 @@ than a failure. `--dspark-strict` remains the byte-identical target-only mode. `exactn_union_full`, `exactn_union_partial_fallback`, `propose`, `verify`, `replay`, stage timings, page faults, and generation t/s. A candidate that is correct but slower remains disabled or opt-in according to its gate. + For asynchronous tails, also repeat the model-backed oracle with the switch + set and run enough exact-union cycles to cross cache eviction and raw-ring + wrap boundaries. The candidate removes a CPU wait but retains private expert + buffers until command-buffer completion; serialized state and process memory + after synchronization must match the synchronous control. + Before the model-backed runs, build `ds4_test` and run + `./ds4_test --metal-kernels`. This covers the isolated compound HC, F16 quad + compressor-store, exact ratio-4 pool, and tie-heavy Metal routing kernels. + For the exact one-row pool candidate, repeat once with + `DS4_METAL_REQUIRE_COMPRESSOR_EXACT_POOL_RATIO4=1`; the run must exercise the + specialization instead of silently falling back. Also exercise the global + kill switches plus the matching pre-M5 or M5 HC/pool rollback on the target + machine. Treat the FlashAttention memo rows as host-dispatch A/B tests: the + selected specialization and output must remain identical, and any timing + comparison must use repeated warm runs. +- For the removed Metal 512-column streaming top-k path, there is no runtime + candidate gate. Compare the current binary with a build immediately before + its removal only if historical timing is needed. First require + `./ds4_test --metal-kernels` to pass, including tie-heavy routing cases, then + require identical selected expert ids and greedy output. Correct deterministic + ordering takes precedence over a timing difference. +- For the default CPU unrolled argmax, run `tests/test_sampling`, then compare + an otherwise identical greedy workload with + `DS4_CPU_DISABLE_UNROLLED_ARGMAX=1` (scalar control) and with the variable + unset (candidate). Require identical tokens for ordinary, excluded-id, + cross-lane-tie, and vocabulary-tail cases; record median generation t/s over + repeated runs without claiming a speedup from the implementation alone. - For the experimental resident-CUDA exact-2 verifier, use three controlled runs with verifier cap two on the same single-GPU host: native proposer plus legacy verifier @@ -346,6 +380,42 @@ than a failure. `--dspark-strict` remains the byte-identical target-only mode. fixture enforces `exact2_attempt>0` and `exact2_fallback=0`. Record `prop_scheduled_rows/cycles`, `propose`, `verify`, `replay`, `net_saved`, `miss_first`, `no_draft`, `avg_accept`, and generation t/s from every run. +- For resident CUDA exact-N, keep exact-2 disabled and compare + `DS4_CUDA_DSPARK_EXACTN=0` against `=1` with the native five-row proposer + and `DS4_DSPARK_SSD_VERIFY_BLOCK_MAX=5`. Repeat N=2,3,4,5 with explicit + proposer/verifier caps, then exercise the kill switch with both + `DS4_CUDA_DSPARK_EXACTN=1` and + `DS4_CUDA_DISABLE_DSPARK_EXACTN=1`. Require byte-identical greedy stdout, + `errors=0`, `verifier_unavailable=0`, `cuda_exactn_attempt>0`, and at least + one `cuda_exactn_full`; partial cases must increment the partial and + aggregate fallback counters, never the error counter, and continue + identically through legacy replay. Include + EOS as the first and a middle draft, a raw-ring wrap boundary, a context + capacity cut, and prefill workspaces below five rows. Record + `cuda_exactn_rows`, its full/partial/error counters, `snapshot`, `verify`, + `replay`, acceptance, and generation t/s. Run with CUDA decode graphs both + enabled and disabled. Do not promote the gate without a CUDA device build + and serialized KV/compressor-state oracle; host syntax tests do not execute + this path. On candidate fixture runs set + `DS4_DSPARK_FIXTURE_REQUIRE_CUDA_EXACTN=1`; it requires aggregate + `cuda_exactn_attempt>0` and `cuda_exactn_error_fallback=0`. It reports but + does not reject aggregate `cuda_exactn_fallback`, because valid partial + matches increment both the partial and aggregate fallback counters before + legacy replay. +- For CUDA DSpark non-causal proposer attention, compare the reference with + `DS4_CUDA_ENABLE_DSPARK_NONCAUSAL_ONLINE=0` against the candidate with `=1`. + Repeat at proposal depths two and five, across every raw-ring start index, + and once with both the enable variable and + `DS4_CUDA_DISABLE_DSPARK_NONCAUSAL_ONLINE=1` to prove the kill switch restores + the reference dispatch. On the short diagnostic runs also set + `DS4_DSPARK_VERIFY_NONCAUSAL=1`; record all three reported `max_abs` and + `max_rel` comparisons and reject non-finite values or a material error + regression. Then run the acceptance fixture without the diagnostic host + readbacks and require byte-identical target stdout, `errors=0`, and + `verifier_unavailable=0`. Record proposal time, acceptance, generation t/s, + and the startup dispatch log. Draft logits or acceptance may differ slightly + because online softmax changes the floating-point reduction order; that is + not permission for the verified target continuation to differ. - For CUDA HC and tiny routed-MoE kernel changes, keep `DS4_CUDA_DSPARK_EXACT2` unset and repeat the resident acceptance fixture with these explicit A/B pairs: HC control diff --git a/README.md b/README.md index b8fe9f322..eab6eb9c6 100644 --- a/README.md +++ b/README.md @@ -490,8 +490,17 @@ the target token already available at the start of the cycle cover the six-token speculative-cycle limit. Exact-union remains opt-in: correctness does not imply a throughput improvement on a particular memory configuration. -The AProjQ4 Metal decode path has three exact dispatch fusions relevant to this -verifier: +Exact-union normally waits for every layer's routed-tail command buffer before +releasing its private expert-address scope. For an isolated A/B, +`DS4_METAL_DSPARK_EXACT_ROWS_ASYNC_TAILS=1` commits that tail without the CPU +wait, retains all scope resources until command-buffer completion, and lets the +next layer's router boundary provide the required ordering. The switch has no +effect outside exact-union and is also opt-in; unset it for the synchronous +control. Validate serialized state and greedy output as well as verifier time, +because removing a host wait is not by itself evidence of an end-to-end gain. + +The AProjQ4 Metal decode path has several exact dispatch fusions relevant to +this verifier: - HC RMSNorm plus the narrow F16 HC mixer is the M1-M4 default, including SSD split phases such as `TO_ROUTER`. Use @@ -508,11 +517,63 @@ verifier: expansion in the same dispatch. Use `DS4_METAL_DISABLE_Q4_ATTN_OUT_HC_FUSE=1` as its isolated A/B control. +Additional PR #755 ports keep their established kernels as shape/resource +fallbacks: + +- On Apple M1 through M5, an eligible one-row HC producer combines the F16 + RMSNorm/mixer, HC split and Sinkhorn-weighted sum, and destination RMSNorm in + one compound dispatch for both attention and FFN producers. The global + rollback is `DS4_METAL_DISABLE_HC_PRODUCER_PRE_NORM_FUSE=1`; the narrower + controls are `DS4_METAL_DISABLE_PRE_M5_HC_PRODUCER_PRE_NORM_FUSE=1` and + `DS4_METAL_DISABLE_M5_HC_PRODUCER_PRE_NORM_FUSE=1`. The existing + `DS4_METAL_DISABLE_PRE_M5_DECODE_PORTS=1` umbrella also disables it before + M5. `DS4_METAL_ENABLE_HC_PRODUCER_PRE_NORM_FUSE=1` permits a focused trial + on another eligible Metal device. +- For an eligible ratio-4 layer on Apple M1 through M5, the standalone F16 compressor path can + project the attention and indexer KV/gate pairs and append both recurrent + states in one quad dispatch. It is the default in ordinary `FULL` decode and + the exact-union collection prefix when the larger Q4 compound dispatch did + not already store those states. Use + `DS4_METAL_DISABLE_COMPRESSOR_QUAD_STORE=1` for the reference path; + `DS4_METAL_DISABLE_PRE_M5_COMPRESSOR_QUAD_STORE=1` is an additional + compatibility rollback. `DS4_METAL_ENABLE_COMPRESSOR_QUAD_STORE=1` permits + a focused trial on another Metal device and widens the phase scope for + diagnostics. +- The exact ratio-4, one-compressed-row pool specialization is the M1-M5 + default for supported 128- and 512-element head shapes. Disable it globally + with `DS4_METAL_DISABLE_COMPRESSOR_EXACT_POOL_RATIO4=1`, or use the + pre-M5/M5 controls + `DS4_METAL_DISABLE_PRE_M5_COMPRESSOR_EXACT_POOL_RATIO4=1` and + `DS4_METAL_DISABLE_M5_COMPRESSOR_EXACT_POOL_RATIO4=1`. For a diagnostic run, + `DS4_METAL_REQUIRE_COMPRESSOR_EXACT_POOL_RATIO4=1` turns an unavailable + exact dispatch into a visible failure instead of silently selecting the + legacy reduction sequence. + +Metal FlashAttention pipeline selection also keeps a generation-aware +one-entry host memo for hot specializations. This changes pipeline lookup, not +kernel arithmetic. Disable the pad/block memo with +`DS4_METAL_DISABLE_PRE_M5_FLASH_ATTN_PAD_BLK_MEMO=1` and the batched/vector +memo with `DS4_METAL_DISABLE_PRE_M5_FLASH_ATTN_BATCHED_MEMO=1` when isolating +host-side dispatch overhead. + +The former 512-column streaming Metal top-k specialization has been removed; +its ordering was not deterministic for every input. The regular deterministic +top-k implementation is now used instead and has no runtime re-enable switch. +Use a previous binary only as a performance control, and require identical +selected ids on tie-heavy inputs before comparing timing. + These gates change dispatch and intermediate-memory traffic, not model arithmetic. Compare byte-identical output, exact-union counters, stage timings, and generation rate on the same machine; do not infer a speedup from a lower dispatch count alone. +CPU greedy decoding and the verifier's excluding-argmax scan use an unrolled +eight-lane implementation by default, including scalar tail handling and +first-index tie semantics. Set `DS4_CPU_DISABLE_UNROLLED_ARGMAX=1` to restore +the scalar scan for an isolated A/B. `tests/test_sampling` compares both paths, +including cross-lane ties, excluded ids, and non-multiple-of-eight vocabulary +sizes. + Exact file views for the two token embedding rows and repeatedly used Q8 support tensors are automatic; the compatibility kill switches are `DS4_METAL_DISABLE_TOKEN_EMBED_EXACT_VIEW=1` and @@ -599,12 +660,56 @@ greedy continuation. For isolated A/B tests, `DS4_CUDA_DSPARK_PROPOSER_BLOCK_MAX=0` preserves the native proposer and an explicit value such as `2` caps it independently of exact-2. +CUDA also has an opt-in tiled online-softmax kernel for this non-causal support +attention. It shares each raw KV row across a group of attention heads and is +selected only for the DSpark raw-ring/head geometry it supports; every other +shape keeps the reference kernel. Enable it with +`DS4_CUDA_ENABLE_DSPARK_NONCAUSAL_ONLINE=1`. The emergency control +`DS4_CUDA_DISABLE_DSPARK_NONCAUSAL_ONLINE=1` wins when both variables are set. +The online reduction order can change draft floating-point results even though +the target verifier still protects greedy output. Use +`DS4_DSPARK_VERIFY_NONCAUSAL=1` to print the first three comparisons against a +host double-precision reference, and require the final target continuation to +remain byte-identical in the performance A/B. + Keep this path opt-in until the CUDA acceptance fixture is byte-identical and the same-machine statistics show lower `propose`, `verify` plus `replay` time. The stats line reports `prop_capped`, `prop_scheduled_rows`, `exact2_attempt`, `exact2_full`, `exact2_partial`, and `exact2_fallback`; a valid run must exercise exact-2 and leave its fallback counter at zero. +A separate resident exact-N CUDA experiment extends the same canonical +one-token tape to two through five draft rows. It leaves hidden rows and all +target weights on one GPU, submits the per-row ordinary decode kernels in one +stream, and reads back only the `N-1` acceptance ids plus final logits. A full +match therefore commits its already-exact KV/compressor state without replay; +a partial match or backend error restores the pre-cycle frontier and uses the +legacy verifier/replay fallback. Enable it independently with: + +```sh +DS4_CUDA_DSPARK_EXACTN=1 DS4_DSPARK_STATS=1 \ +./ds4 --cuda -m ds4flash.gguf \ + --mtp gguf/DeepSeek-V4-Flash-DSpark-support-0731.gguf \ + --dspark --temp 0 -p 'Write a Python quicksort function with comments.' +``` + +The default verifier cap under this gate is five (or the available prefill +workspace when smaller). The proposer uses at most one fewer workspace row, +because its support stage also carries the current target row; the existing +explicit proposer and verifier cap variables still take precedence. +`DS4_CUDA_DISABLE_DSPARK_EXACTN=1` is the +kill switch and restores the previous path without changing the enable +variable. Track `cuda_exactn_attempt`, `cuda_exactn_full`, +`cuda_exactn_fallback`, its partial/error split, and `cuda_exactn_rows`. Keep +this experiment disabled by default until a real CUDA oracle and a long greedy +A/B show byte-identical output, no fallback errors, and a throughput win. + +Set `DS4_DSPARK_FIXTURE_REQUIRE_CUDA_EXACTN=1` on the candidate acceptance +fixture to require at least one `cuda_exactn_attempt` and zero +`cuda_exactn_error_fallback`. The aggregate `cuda_exactn_fallback` is reported +but is not required to be zero: it also includes valid partial draft matches, +which deliberately restore the frontier and use legacy replay. + The acceptance fixture can exercise the same SSD path on both the target-only baseline and the DSpark run. It also requires real proposals and accepted draft tokens, so an unavailable verifier cannot pass as a silent no-op: diff --git a/ds4.c b/ds4.c index 9dd5142a5..129b691a2 100644 --- a/ds4.c +++ b/ds4.c @@ -16007,6 +16007,12 @@ typedef struct { * override. The routed tails consume the GPU-selected matrix through the * exact-row union scope instead. */ bool spec_exactn_union_collect_routes; + /* Exact decode tapes bind short-lived row views as cur/after_ffn. CUDA + * decode-graph keys identify those wrapper handles, so caching an island + * across tape invocations could replay a graph after the handle address + * has been recycled for a different row. Keep graph capture/replay off + * only while such a tape is active. */ + bool spec_disable_decode_graphs; /* Batch verification normally takes the per-row compressor path when it * captures intermediate frontiers. Rollback+replay DSpark verification * needs the same arithmetic path, but not the frontier copies themselves. */ @@ -22971,6 +22977,7 @@ static bool metal_graph_encode_decode_layer_phase( g->tp_world <= 1 && !g->cuda_tp_decode && !g->ssd_streaming && + !g->spec_disable_decode_graphs && !g->materialize_ffn_out && !decode_stage_profile && !g_expert_profile.active && @@ -23716,25 +23723,39 @@ static bool metal_graph_encode_decode_layer_phase( ok = false; } bool comp_state_already_stored = qkv_pair_quad_fused; - /* Quad projection: the attention and indexer compressor pairs share - * the normalized input and the F16 matvec shape, so a single dispatch - * covers all four matrices with unchanged per-row reduction trees. - * Removes one dispatch per decode layer. Either preceding fused - * QKV path may already have performed the same work. */ + /* Ratio-4 attention and indexer compressor pairs consume the same + * normalized row. On the normal FULL path and the DSpark exact-union + * prefix, one Metal dispatch can project all four F16 matrices and + * append both recurrent states without changing either reduction + * tree. The disable is authoritative; the enable also permits + * focused experiments in other decode phases. */ int quad_store = comp_state_already_stored ? 1 : 0; - if (ok && quad_store == 0 && ratio == 4u && +#if defined(__APPLE__) && !defined(DS4_NO_GPU) + const bool quad_store_forced = + getenv("DS4_METAL_ENABLE_COMPRESSOR_QUAD_STORE") != NULL; + const bool quad_store_scope = + phase == METAL_DECODE_LAYER_FULL || + (phase == METAL_DECODE_LAYER_TO_ROUTER && + g->spec_exactn_union_collect_routes) || + quad_store_forced; + const bool quad_store_device = + quad_store_forced || + ds4_gpu_f16_quad_compressor_store_auto_available() != 0; + if (ok && quad_store == 0 && ratio == 4u && quad_store_scope && + quad_store_device && !metal_graph_use_reference_compressor_pair_proj() && + getenv("DS4_METAL_DISABLE_COMPRESSOR_QUAD_STORE") == NULL && getenv("DS4_METAL_DISABLE_PRE_M5_COMPRESSOR_QUAD_STORE") == NULL && - (ds4_gpu_device_is_pre_m5_apple_silicon() || - ds4_gpu_device_is_m5_apple_silicon()) && layer->indexer_compressor_kv && layer->indexer_compressor_gate && layer->indexer_compressor_ape && layer->indexer_compressor_kv->type == DS4_TENSOR_F16 && layer->indexer_compressor_gate->type == DS4_TENSOR_F16 && layer->indexer_compressor_kv->dim[0] == DS4_N_EMBD && layer->indexer_compressor_gate->dim[0] == DS4_N_EMBD && - layer->indexer_compressor_kv->dim[1] == 2u * DS4_N_INDEXER_HEAD_DIM && - layer->indexer_compressor_gate->dim[1] == 2u * DS4_N_INDEXER_HEAD_DIM) { + layer->indexer_compressor_kv->dim[1] == + 2u * DS4_N_INDEXER_HEAD_DIM && + layer->indexer_compressor_gate->dim[1] == + 2u * DS4_N_INDEXER_HEAD_DIM) { quad_store = ds4_gpu_matmul_f16_quad_compressor_store_tensor( metal_graph_comp_kv_cur(g), @@ -23762,6 +23783,7 @@ static bool metal_graph_encode_decode_layer_phase( ratio, pos); } +#endif if (quad_store < 0) { ok = false; } else if (quad_store > 0) { @@ -37137,6 +37159,8 @@ static bool metal_graph_verify_decode2_exact_impl( ds4_gpu_tensor *saved_after_by_tier[DS4_MAX_GPUS] = {0}; const int saved_active_tier = g->active_tier; const bool saved_capture = g->spec_capture_prefixes; + const bool saved_disable_decode_graphs = + g->spec_disable_decode_graphs; bool ok = true; for (int t = 0; t < DS4_MAX_GPUS; t++) { @@ -37182,6 +37206,7 @@ static bool metal_graph_verify_decode2_exact_impl( DS4_N_HC) != 0; g->spec_capture_prefixes = capture_prefix1; + g->spec_disable_decode_graphs = true; if (ok) ok = ds4_gpu_begin_commands() != 0; for (uint32_t il = 0; ok && il < DS4_N_LAYER; il++) { const uint32_t pos0 = start; @@ -37246,6 +37271,7 @@ static bool metal_graph_verify_decode2_exact_impl( if (ok) ok = ds4_gpu_end_commands() != 0; else (void)ds4_gpu_synchronize(); g->spec_capture_prefixes = saved_capture; + g->spec_disable_decode_graphs = saved_disable_decode_graphs; if (ok) { ok = metal_graph_set_active_tier_no_copy(g, cur_tier); @@ -37346,6 +37372,201 @@ static bool metal_graph_verify_decode2_exact_impl( return ok; } +enum { DS4_CUDA_EXACTN_MAX_ROWS = 5 }; + +/* Resident exact-N verifier for single-device CUDA. + * + * This is the N-row form of the canonical exact-2 tape above: every row still + * executes the ordinary one-token layer and output-head kernels, in the same + * autoregressive order. The only batching is lifetime/dispatch batching: + * row-local hidden states stay in the existing prefill workspace, all layer + * launches share one command stream, and the CPU reads N-1 top ids plus the + * final logits after the whole block. Persistent KV/compressor state is + * therefore directly committable only when every draft matches. The caller + * owns a pre-cycle frontier and must restore it on a partial match or error. */ +static bool metal_graph_verify_decode_exactn_cuda_resident_impl( + ds4_gpu_graph *g, + const ds4_model *model, + const ds4_weights *weights, + const int *tokens, + uint32_t n_tokens, + uint32_t start, + int *row_tops, + float *last_logits) { + if (!g || !model || !weights || !tokens || !row_tops || !last_logits || + n_tokens < 2u || n_tokens > DS4_CUDA_EXACTN_MAX_ROWS || + n_tokens > g->prefill_cap || g->raw_cap == 0 || + g->ssd_streaming || g->placement != NULL || g->tp_world > 1u || + g->active_tier != 0 || g->emb_tier != 0 || g->head_tier != 0 || + !g->batch_cur_hc_by_tier[0] || !g->batch_next_hc_by_tier[0] || + !g->spec_logits || + ds4_gpu_tensor_bytes(g->spec_logits) < + (uint64_t)n_tokens * DS4_N_VOCAB * sizeof(float) || + !g->batch_router_selected_by_tier[0] || + ds4_gpu_tensor_bytes(g->batch_router_selected_by_tier[0]) < + (uint64_t)(n_tokens - 1u) * sizeof(int32_t)) { + return false; + } + + const uint64_t hc_dim = (uint64_t)DS4_N_HC * DS4_N_EMBD; + ds4_gpu_tensor *cur_rows[DS4_CUDA_EXACTN_MAX_ROWS] = {0}; + ds4_gpu_tensor *next_rows[DS4_CUDA_EXACTN_MAX_ROWS] = {0}; + ds4_gpu_tensor *logits_rows[DS4_CUDA_EXACTN_MAX_ROWS] = {0}; + ds4_gpu_tensor *top_rows[DS4_CUDA_EXACTN_MAX_ROWS - 1u] = {0}; + ds4_gpu_tensor *top_span = NULL; + ds4_gpu_tensor *saved_cur = g->cur_hc_by_tier[0]; + ds4_gpu_tensor *saved_after = g->after_ffn_hc_by_tier[0]; + ds4_gpu_tensor *saved_logits = g->logits_by_tier[0]; + const bool saved_capture = g->spec_capture_prefixes; + const bool saved_disable_decode_graphs = + g->spec_disable_decode_graphs; + bool commands_open = false; + bool ok = metal_graph_set_active_tier_no_copy(g, 0); + + for (uint32_t row = 0; ok && row < n_tokens; row++) { + cur_rows[row] = ds4_gpu_tensor_view( + g->batch_cur_hc_by_tier[0], + (uint64_t)row * hc_dim * sizeof(float), + hc_dim * sizeof(float)); + next_rows[row] = ds4_gpu_tensor_view( + g->batch_next_hc_by_tier[0], + (uint64_t)row * hc_dim * sizeof(float), + hc_dim * sizeof(float)); + ok = cur_rows[row] && next_rows[row]; + if (ok) { + ok = ds4_gpu_embed_token_hc_tensor( + cur_rows[row], + model->map, + model->size, + weights->token_embd->abs_offset, + (uint32_t)weights->token_embd->dim[1], + (uint32_t)tokens[row], + DS4_N_EMBD, + DS4_N_HC) != 0; + } + } + + g->spec_capture_prefixes = false; + g->spec_disable_decode_graphs = true; + if (ok) { + ok = ds4_gpu_begin_commands() != 0; + commands_open = ok; + } + for (uint32_t il = 0; ok && il < DS4_N_LAYER; il++) { + for (uint32_t row = 0; ok && row < n_tokens; row++) { + const uint32_t pos = start + row; + g->cur_hc_by_tier[0] = cur_rows[row]; + g->after_ffn_hc_by_tier[0] = next_rows[row]; + ok = metal_graph_encode_decode_layer( + g, + model, + &weights->layer[il], + il, + pos, + g->layer_raw_cache[il], + g->raw_cap, + pos % g->raw_cap, + metal_graph_raw_span_for_batch(g, pos, 1), + tokens[row]); + } + if (ok) { + for (uint32_t row = 0; row < n_tokens; row++) { + ds4_gpu_tensor *tmp = cur_rows[row]; + cur_rows[row] = next_rows[row]; + next_rows[row] = tmp; + } + } + } + if (commands_open) { + if (ok) { + ok = ds4_gpu_end_commands() != 0; + } else { + (void)ds4_gpu_synchronize(); + } + commands_open = false; + } + + if (ok) { + top_span = ds4_gpu_tensor_view( + g->batch_router_selected_by_tier[0], + 0, + (uint64_t)(n_tokens - 1u) * sizeof(int32_t)); + ok = top_span != NULL; + } + for (uint32_t row = 0; ok && row < n_tokens; row++) { + logits_rows[row] = ds4_gpu_tensor_view( + g->spec_logits, + (uint64_t)row * DS4_N_VOCAB * sizeof(float), + (uint64_t)DS4_N_VOCAB * sizeof(float)); + ok = logits_rows[row] != NULL; + if (ok && row + 1u < n_tokens) { + top_rows[row] = ds4_gpu_tensor_view( + top_span, + (uint64_t)row * sizeof(int32_t), + sizeof(int32_t)); + ok = top_rows[row] != NULL; + } + } + + if (ok) { + ok = ds4_gpu_begin_commands() != 0; + commands_open = ok; + } + for (uint32_t row = 0; ok && row < n_tokens; row++) { + g->cur_hc_by_tier[0] = cur_rows[row]; + g->logits_by_tier[0] = logits_rows[row]; + ok = metal_graph_encode_output_head( + g, model, weights, weights->output->dim[1]); + if (ok && row + 1u < n_tokens) { + ok = ds4_gpu_argmax_tensor( + top_rows[row], logits_rows[row], DS4_N_VOCAB) != 0; + } + } + if (commands_open) { + if (ok) { + ok = ds4_gpu_end_commands() != 0; + } else { + (void)ds4_gpu_synchronize(); + } + commands_open = false; + } + + int32_t tops_i32[DS4_CUDA_EXACTN_MAX_ROWS - 1u] = {0}; + if (ok) { + ok = ds4_gpu_tensor_read( + top_span, + 0, + tops_i32, + (uint64_t)(n_tokens - 1u) * sizeof(tops_i32[0])) != 0 && + ds4_gpu_tensor_read( + logits_rows[n_tokens - 1u], + 0, + last_logits, + (uint64_t)DS4_N_VOCAB * sizeof(last_logits[0])) != 0; + } + if (ok) { + for (uint32_t row = 0; row + 1u < n_tokens; row++) { + row_tops[row] = tops_i32[row]; + } + } + + g->spec_capture_prefixes = saved_capture; + g->spec_disable_decode_graphs = saved_disable_decode_graphs; + g->cur_hc_by_tier[0] = saved_cur; + g->after_ffn_hc_by_tier[0] = saved_after; + g->logits_by_tier[0] = saved_logits; + for (uint32_t row = 0; row + 1u < n_tokens; row++) { + ds4_gpu_tensor_free(top_rows[row]); + } + ds4_gpu_tensor_free(top_span); + for (uint32_t row = 0; row < n_tokens; row++) { + ds4_gpu_tensor_free(logits_rows[row]); + ds4_gpu_tensor_free(next_rows[row]); + ds4_gpu_tensor_free(cur_rows[row]); + } + return ok; +} + enum { DS4_METAL_EXACTN_UNION_MAX_ROWS = 5 }; /* Row-local aliases needed across the TO_ROUTER/FROM_ROUTER split. The @@ -37538,6 +37759,8 @@ static bool metal_graph_verify_decode_exactn_union_impl( #if defined(__APPLE__) const bool exact_rows_profile = getenv("DS4_METAL_DSPARK_EXACT_ROWS_PROFILE") != NULL; + const bool async_exact_rows_tails = + getenv("DS4_METAL_DSPARK_EXACT_ROWS_ASYNC_TAILS") != NULL; if (!g || !model || !weights || !tokens || !row_tops || !last_logits || n_tokens < 2u || n_tokens > DS4_METAL_EXACTN_UNION_MAX_ROWS || n_tokens > g->prefill_cap || g->raw_cap == 0 || @@ -37724,12 +37947,17 @@ static bool metal_graph_verify_decode_exactn_union_impl( metal_graph_unbind_exactn_union_row(g, &row_aliases[row]); } /* The exact-row scope owns the private address buffers referenced by - * every routed tail. Drain that command buffer before dropping its - * strong refs; the next layer starts in a fresh batch. */ + * every routed tail. The default path drains that command buffer + * before dropping its strong refs. The opt-in path commits without a + * CPU wait and retains the whole scope through CB completion; queue + * order makes the next layer's router-readback event cover these + * earlier tails as well. */ if (exact_rows_active) { if (commands_open) { if (ok) { - const bool end_ok = ds4_gpu_end_commands() != 0; + const bool end_ok = async_exact_rows_tails + ? ds4_gpu_stream_expert_exact_rows_end_async() != 0 + : ds4_gpu_end_commands() != 0; if (!end_ok) (void)ds4_gpu_synchronize(); ok = end_ok; } else { @@ -42012,7 +42240,7 @@ static bool glm_graph_memory_guard_budget( (void)load_slice; (void)ssd_streaming; #endif - uint64_t budget_base = glm_graph_host_memory_bytes(); + uint64_t budget_base = ds4_graph_host_memory_bytes(); if (budget_base == 0) { budget_base = ds4_gpu_recommended_working_set_size(); } @@ -54602,6 +54830,12 @@ typedef struct ds4_dspark_spec_stats { uint64_t exact2_full_accepts; uint64_t exact2_partial_accepts; uint64_t exact2_fallbacks; + uint64_t cuda_exactn_attempts; + uint64_t cuda_exactn_full_accepts; + uint64_t cuda_exactn_fallbacks; + uint64_t cuda_exactn_partial_fallbacks; + uint64_t cuda_exactn_error_fallbacks; + uint64_t cuda_exactn_rows; uint64_t exactn_union_attempts; uint64_t exactn_union_full_accepts; uint64_t exactn_union_fallbacks; @@ -54778,6 +55012,29 @@ static bool ds4_session_cuda_dspark_exact2_enabled( #endif } +/* Resident single-GPU exact-N is deliberately independent of exact-2 so the + * proposal width and verifier state transition can be attributed separately + * on CUDA hardware. It is opt-in and has an explicit emergency kill switch; + * neither variable changes Metal or ROCm behavior. */ +static bool ds4_session_cuda_dspark_exactn_enabled( + const ds4_session *s) { +#if !defined(__APPLE__) && !defined(DS4_ROCM_BUILD) + return s && s->engine && + s->engine->backend == DS4_BACKEND_CUDA && + !s->engine->multi_tier && + !s->engine->tp.active && + !s->graph.ssd_streaming && + s->graph.placement == NULL && + s->graph.prefill_cap >= 3u && + metal_graph_tp_env_flag("DS4_CUDA_DSPARK_EXACTN", false) && + !metal_graph_tp_env_flag( + "DS4_CUDA_DISABLE_DSPARK_EXACTN", false); +#else + (void)s; + return false; +#endif +} + /* Metal exact-2 reuses the same decode-exact body as CUDA, but starts as an * SSD-only opt-in until its selected-expert barriers and throughput are * measured on real Apple hardware. */ @@ -54842,7 +55099,8 @@ static bool ds4_session_metal_dspark_exactn_union_enabled( /* Keep the proposer width independently controllable from the verifier so a * CUDA A/B can attribute the gain. An explicit zero keeps the checkpoint's - * native width; otherwise exact-2 defaults to a two-row proposal. */ + * native width; otherwise exact-2 defaults to two rows and exact-N to the + * largest resident width it can consume (at most five). */ static uint32_t ds4_session_cuda_dspark_proposer_block_cap( const ds4_session *s, uint32_t native_block_size) { @@ -54871,6 +55129,14 @@ static uint32_t ds4_session_cuda_dspark_proposer_block_cap( return native_block_size; } + if (ds4_session_cuda_dspark_exactn_enabled(s)) { + /* The support stage carries the current target row plus every draft, + * so a D-row proposal needs D+1 rows of prefill workspace. */ + uint32_t cap = s->graph.prefill_cap - 1u; + if (cap > 5u) cap = 5u; + return cap < native_block_size ? cap : native_block_size; + } + const uint32_t verify_cap = ds4_dspark_env_u32( "DS4_DSPARK_SSD_VERIFY_BLOCK_MAX", 0); if (s->graph.prefill_cap >= 3u && @@ -54892,6 +55158,10 @@ static uint32_t ds4_session_cuda_dspark_proposer_block_cap( static uint32_t ds4_session_dspark_verify_block_cap(const ds4_session *s) { uint32_t cap = ds4_dspark_env_u32( "DS4_DSPARK_SSD_VERIFY_BLOCK_MAX", 0); + if (cap == 0 && ds4_session_cuda_dspark_exactn_enabled(s)) { + cap = s->graph.prefill_cap; + if (cap > 5u) cap = 5u; + } /* Exact-2 defaults verification to two rows. The independent proposer * cap may preserve the checkpoint's native proposal width for A/B tests; * an explicit verifier cap still takes precedence. */ @@ -55021,6 +55291,16 @@ static bool ds4_session_dspark_exact2_requested( ds4_session_metal_dspark_exact2_enabled(s)); } +static bool ds4_session_cuda_dspark_exactn_requested( + const ds4_session *s, + uint32_t draft_n) { + return draft_n >= 2u && + draft_n <= DS4_DSPARK_MAX_BLOCK_SIZE && + draft_n <= 5u && + s && draft_n <= s->graph.prefill_cap && + ds4_session_cuda_dspark_exactn_enabled(s); +} + static bool ds4_session_metal_dspark_exactn_requested( const ds4_session *s, uint32_t draft_n) { @@ -65035,7 +65315,11 @@ static void ds4_session_print_dspark_stats(const ds4_session *s) { "replay_fallbacks=%llu prop_capped=%llu " "prop_scheduled_rows=%llu " "exact2_attempt=%llu exact2_full=%llu exact2_partial=%llu " - "exact2_fallback=%llu exactn_union_attempt=%llu " + "exact2_fallback=%llu cuda_exactn_attempt=%llu " + "cuda_exactn_full=%llu cuda_exactn_fallback=%llu " + "cuda_exactn_partial_fallback=%llu " + "cuda_exactn_error_fallback=%llu cuda_exactn_rows=%llu " + "exactn_union_attempt=%llu " "exactn_union_full=%llu exactn_union_fallback=%llu " "exactn_union_partial_fallback=%llu " "exactn_union_error_fallback=%llu " @@ -65072,6 +65356,12 @@ static void ds4_session_print_dspark_stats(const ds4_session *s) { (unsigned long long)st->exact2_full_accepts, (unsigned long long)st->exact2_partial_accepts, (unsigned long long)st->exact2_fallbacks, + (unsigned long long)st->cuda_exactn_attempts, + (unsigned long long)st->cuda_exactn_full_accepts, + (unsigned long long)st->cuda_exactn_fallbacks, + (unsigned long long)st->cuda_exactn_partial_fallbacks, + (unsigned long long)st->cuda_exactn_error_fallbacks, + (unsigned long long)st->cuda_exactn_rows, (unsigned long long)st->exactn_union_attempts, (unsigned long long)st->exactn_union_full_accepts, (unsigned long long)st->exactn_union_fallbacks, @@ -70719,6 +71009,124 @@ static int ds4_session_eval_dspark_speculative_argmax( bool verifier_may_have_mutated = false; bool tp_verify_sent = false; + /* Resident CUDA exact-N keeps the canonical one-token arithmetic while + * removing the per-token command/readback boundary. A full match already + * owns the exact target state and commits immediately. Partial/error + * attempts restore the pre-cycle frontier before entering the established + * batch verifier plus exact replay path below. */ + const bool cuda_exactn = + ok && ds4_session_cuda_dspark_exactn_requested( + s, (uint32_t)draft_n); + if (cuda_exactn) { + if (stats_enabled) { + s->dspark_stats.cuda_exactn_attempts++; + s->dspark_stats.cuda_exactn_rows += (uint64_t)draft_n; + } + bool exactn_ok = true; + bool exactn_mismatch = false; + int exactn_accepted_prefix = 1; + int exactn_last_top = -1; + const double exactn_t0 = stats_enabled ? now_sec() : 0.0; + + exactn_ok = metal_graph_verify_decode_exactn_cuda_resident_impl( + &s->graph, + &e->model, + &e->weights, + drafts, + (uint32_t)draft_n, + (uint32_t)start, + row_tops, + row_logits); + if (exactn_ok) { + for (int row = 0; row + 1 < draft_n; row++) { + exactn_last_top = row_tops[row]; + if (exactn_last_top != drafts[row + 1]) { + exactn_mismatch = true; + break; + } + exactn_accepted_prefix = row + 2; + } + } + if (stats_enabled) { + s->dspark_stats.verify_ms += + (now_sec() - exactn_t0) * 1000.0; + } + + if (exactn_ok && !exactn_mismatch) { + memcpy(s->logits, + row_logits, + (size_t)DS4_N_VOCAB * sizeof(s->logits[0])); + /* Exact-N uses private hidden rows and cannot refresh the target + * hidden capture consumed by the next DSpark proposal. */ + ds4_session_dspark_capture_invalidate(s); + for (int i = 0; i < draft_n; i++) { + token_vec_push(&s->checkpoint, drafts[i]); + accepted[n_accept++] = drafts[i]; + } + s->checkpoint_valid = true; + ds4_session_dspark_capture_note_checkpoint(s); + if (stats_enabled) { + s->dspark_stats.full_accepts++; + s->dspark_stats.cuda_exactn_full_accepts++; + s->dspark_stats.accepted_draft_tokens += + (uint64_t)draft_n; + ds4_dspark_stats_note_len( + s->dspark_stats.accepted_len_hist, + (uint32_t)draft_n); + } + ds4_session_dspark_stats_note_saved(s, (uint32_t)draft_n); + if (spec_log) { + fprintf(stderr, + "ds4: DSpark CUDA exactN drafted=%d " + "accepted_draft=%d accepted_total=%d\n", + draft_n, + draft_n, + n_accept); + } + spec_frontier_free(&frontier); + DS4_DSPARK_STATS_FINISH(); + return n_accept; + } + + if (stats_enabled) { + s->dspark_stats.cuda_exactn_fallbacks++; + if (exactn_mismatch) { + s->dspark_stats.cuda_exactn_partial_fallbacks++; + } else { + s->dspark_stats.cuda_exactn_error_fallbacks++; + } + } + s->checkpoint.len = start; + ds4_session_dspark_capture_invalidate(s); + if (!have_frontier || !spec_frontier_restore(&frontier, s)) { + snprintf(err, errlen, + "DSpark CUDA exactN rollback failed"); + s->checkpoint_valid = false; + if (stats_enabled) { + s->dspark_stats.verifier_errors++; + ds4_dspark_stats_note_len( + s->dspark_stats.accepted_len_hist, 0); + } + spec_frontier_free(&frontier); + DS4_DSPARK_STATS_FINISH(); + return -1; + } + if (spec_log) { + if (exactn_mismatch) { + fprintf(stderr, + "ds4: DSpark CUDA exactN partial " + "accepted_prefix=%d verified_next=%d; falling back " + "to legacy verify/replay\n", + exactn_accepted_prefix, + exactn_last_top); + } else { + fprintf(stderr, + "ds4: DSpark CUDA exactN unavailable; restored " + "frontier for legacy verify/replay\n"); + } + } + } + /* Fast Metal exact-N experiment. Unlike the token-major oracle below, * this advances all rows layer-major and shares one immutable selected- * expert union per layer. It may directly commit only a complete match; @@ -70980,7 +71388,8 @@ static int ds4_session_eval_dspark_speculative_argmax( } const bool exact2 = - !exactn && ok && ds4_session_dspark_exact2_requested( + !cuda_exactn && !exactn && ok && + ds4_session_dspark_exact2_requested( s, (uint32_t)draft_n); if (exact2) { if (stats_enabled) s->dspark_stats.exact2_attempts++; diff --git a/ds4_cuda.cu b/ds4_cuda.cu index 2f6561b47..909440784 100644 --- a/ds4_cuda.cu +++ b/ds4_cuda.cu @@ -16443,6 +16443,131 @@ __global__ static void attention_noncausal_raw_batch_heads_kernel( } } +/* Experimental DSpark verifier attention for the native 512-wide head. Eight + * heads share each staged raw row, while every warp maintains its own online + * softmax state. Seeding (max,sum) with the sink gives the sink a zero value + * contribution without materializing scores or a sink row. Keep this behind + * an explicit gate: the online recurrence is mathematically equivalent to the + * reference kernel, but its floating-point accumulation order is different. */ +template +__global__ static void __launch_bounds__(256, 2) +attention_noncausal_raw_batch_heads_online_kernel( + float *heads, + const float *sinks, + const float *q, + const float *raw_kv, + uint32_t n_tokens, + uint32_t n_raw, + uint32_t raw_cap, + uint32_t raw_start, + uint32_t n_head, + uint32_t head_dim) { + const uint32_t tok = blockIdx.x; + const uint32_t head_group = blockIdx.y; + if (tok >= n_tokens || raw_cap == 0u || raw_start >= raw_cap || + n_raw == 0u || n_raw > raw_cap || head_dim != 512u) { + return; + } + + const uint32_t lane = threadIdx.x & 31u; + const uint32_t warp = threadIdx.x >> 5u; + const uint32_t h = head_group * HEADS_PER_GROUP + warp; + const bool valid_head = warp < HEADS_PER_GROUP && h < n_head; + + /* One 512-float row is 128 float4 values. Staging it once lets every + * verifier head reuse the same raw-ring traffic. */ + __shared__ float4 kv_shared[ROWS_PER_STAGE * 128u]; + + const float4 *q4 = valid_head + ? (const float4 *)(q + ((uint64_t)tok * n_head + h) * head_dim) + : NULL; + float4 q0 = make_float4(0.0f, 0.0f, 0.0f, 0.0f); + float4 q1 = q0, q2 = q0, q3 = q0; + if (valid_head) { + q0 = q4[lane + 0u]; + q1 = q4[lane + 32u]; + q2 = q4[lane + 64u]; + q3 = q4[lane + 96u]; + } + + const float scale = rsqrtf((float)head_dim); + float max_s = valid_head ? sinks[h] : -INFINITY; + float sum_s = valid_head ? 1.0f : 0.0f; + float4 o0 = make_float4(0.0f, 0.0f, 0.0f, 0.0f); + float4 o1 = o0, o2 = o0, o3 = o0; + + for (uint32_t row0 = 0; row0 < n_raw; row0 += ROWS_PER_STAGE) { + const uint32_t nr = n_raw - row0 < ROWS_PER_STAGE + ? n_raw - row0 : ROWS_PER_STAGE; + for (uint32_t off = threadIdx.x; + off < nr * 128u; + off += blockDim.x) { + const uint32_t rr = off >> 7u; + const uint32_t c4 = off & 127u; + const uint32_t logical_row = row0 + rr; + /* n_raw <= raw_cap means the visible interval wraps at most once. + * Use 64-bit addition so a future large ring cannot overflow here. */ + const uint32_t physical_row = (uint32_t)( + ((uint64_t)raw_start + logical_row) % raw_cap); + const float4 *src = (const float4 *)( + raw_kv + (uint64_t)physical_row * head_dim); + kv_shared[off] = src[c4]; + } + __syncthreads(); + + if (valid_head) { + for (uint32_t rr = 0; rr < nr; rr++) { + const float4 *kv4 = kv_shared + rr * 128u; + const float4 k0 = kv4[lane + 0u]; + const float4 k1 = kv4[lane + 32u]; + const float4 k2 = kv4[lane + 64u]; + const float4 k3 = kv4[lane + 96u]; + float score = dot4_f32(q0, k0) + dot4_f32(q1, k1) + + dot4_f32(q2, k2) + dot4_f32(q3, k3); + score = warp_sum_f32(score) * scale; + score = __shfl_sync(0xffffffffu, score, 0); + + const float new_m = fmaxf(max_s, score); + const float old_scale = expf(max_s - new_m); + const float row_scale = expf(score - new_m); + sum_s = sum_s * old_scale + row_scale; + o0.x = o0.x * old_scale + k0.x * row_scale; + o0.y = o0.y * old_scale + k0.y * row_scale; + o0.z = o0.z * old_scale + k0.z * row_scale; + o0.w = o0.w * old_scale + k0.w * row_scale; + o1.x = o1.x * old_scale + k1.x * row_scale; + o1.y = o1.y * old_scale + k1.y * row_scale; + o1.z = o1.z * old_scale + k1.z * row_scale; + o1.w = o1.w * old_scale + k1.w * row_scale; + o2.x = o2.x * old_scale + k2.x * row_scale; + o2.y = o2.y * old_scale + k2.y * row_scale; + o2.z = o2.z * old_scale + k2.z * row_scale; + o2.w = o2.w * old_scale + k2.w * row_scale; + o3.x = o3.x * old_scale + k3.x * row_scale; + o3.y = o3.y * old_scale + k3.y * row_scale; + o3.z = o3.z * old_scale + k3.z * row_scale; + o3.w = o3.w * old_scale + k3.w * row_scale; + max_s = new_m; + } + } + __syncthreads(); + } + + if (valid_head) { + const float inv_s = sum_s == 0.0f ? 0.0f : 1.0f / sum_s; + o0.x *= inv_s; o0.y *= inv_s; o0.z *= inv_s; o0.w *= inv_s; + o1.x *= inv_s; o1.y *= inv_s; o1.z *= inv_s; o1.w *= inv_s; + o2.x *= inv_s; o2.y *= inv_s; o2.z *= inv_s; o2.w *= inv_s; + o3.x *= inv_s; o3.y *= inv_s; o3.z *= inv_s; o3.w *= inv_s; + float4 *out4 = (float4 *)( + heads + ((uint64_t)tok * n_head + h) * head_dim); + out4[lane + 0u] = o0; + out4[lane + 32u] = o1; + out4[lane + 64u] = o2; + out4[lane + 96u] = o3; + } +} + extern "C" int ds4_gpu_attention_noncausal_raw_batch_heads_tensor( ds4_gpu_tensor *heads, const void *model_map, @@ -16456,14 +16581,27 @@ extern "C" int ds4_gpu_attention_noncausal_raw_batch_heads_tensor( uint32_t raw_start, uint32_t n_head, uint32_t head_dim) { - if (!heads || !q || !raw_kv || !model_map || - n_tokens == 0 || n_raw == 0 || raw_cap < n_raw || - raw_start >= raw_cap || n_head == 0 || head_dim == 0 || - sinks_offset > model_size || - (uint64_t)n_head * sizeof(float) > model_size - sinks_offset || - heads->bytes < (uint64_t)n_tokens * n_head * head_dim * sizeof(float) || - q->bytes < (uint64_t)n_tokens * n_head * head_dim * sizeof(float) || - raw_kv->bytes < (uint64_t)raw_cap * head_dim * sizeof(float)) { + if (!heads || !q || !raw_kv || !heads->ptr || !q->ptr || !raw_kv->ptr || + !model_map || + n_tokens == 0u || n_raw == 0u || raw_cap < n_raw || + raw_start >= raw_cap || n_head == 0u || head_dim == 0u) { + return 0; + } + const uint64_t sink_bytes = (uint64_t)n_head * sizeof(float); + if ((uint64_t)n_tokens > UINT64_MAX / n_head) return 0; + const uint64_t head_rows = (uint64_t)n_tokens * n_head; + if (head_rows > UINT64_MAX / head_dim) return 0; + const uint64_t head_elems = head_rows * head_dim; + if (head_elems > UINT64_MAX / sizeof(float) || + (uint64_t)raw_cap > UINT64_MAX / head_dim) { + return 0; + } + const uint64_t raw_elems = (uint64_t)raw_cap * head_dim; + if (raw_elems > UINT64_MAX / sizeof(float) || + sinks_offset > model_size || sink_bytes > model_size - sinks_offset || + heads->bytes < head_elems * sizeof(float) || + q->bytes < head_elems * sizeof(float) || + raw_kv->bytes < raw_elems * sizeof(float)) { return 0; } const int logical_tier = ds4_tensor_device_idx(heads); @@ -16471,16 +16609,56 @@ extern "C" int ds4_gpu_attention_noncausal_raw_batch_heads_tensor( model_map, sinks_offset, (uint64_t)n_head * sizeof(float), logical_tier, "dspark_attn_sinks"); if (!sinks) return 0; - const size_t shmem = (size_t)n_raw * sizeof(float); - if (shmem > 32768) return 0; /* draft blocks are tiny; guard anyway */ - dim3 grid(n_tokens, n_head, 1); - attention_noncausal_raw_batch_heads_kernel<<>>( - (float *)heads->ptr, - sinks, - (const float *)q->ptr, - (const float *)raw_kv->ptr, - n_tokens, n_raw, raw_cap, raw_start, n_head, head_dim); - if (!cuda_ok(cudaGetLastError(), "attention noncausal raw batch heads launch")) return 0; + + /* The reference path remains the unconditional fallback. The online path + * is deliberately narrow: DSpark's raw ring is eight rows, its current + * attention head is 512 floats, and float4 staging requires aligned device + * views. The disable flag always wins over the enable flag. */ + const bool online_requested = + cuda_env_flag_enabled("DS4_CUDA_ENABLE_DSPARK_NONCAUSAL_ONLINE", 0) && + !cuda_env_flag_enabled("DS4_CUDA_DISABLE_DSPARK_NONCAUSAL_ONLINE", 0); + const bool online_shape = + n_tokens <= 8u && n_raw <= 8u && head_dim == 512u && + n_head <= 256u && + ds4_tensor_device_idx(q) == logical_tier && + ds4_tensor_device_idx(raw_kv) == logical_tier && + ((((uintptr_t)heads->ptr | (uintptr_t)q->ptr | + (uintptr_t)raw_kv->ptr) & 15u) == 0u); + if (online_requested && online_shape) { + static int logged_online = 0; + if (!logged_online) { + logged_online = 1; + fprintf(stderr, + "ds4: CUDA DSpark noncausal attention using tiled " + "online softmax\n"); + } + dim3 grid(n_tokens, (n_head + 7u) / 8u, 1u); + attention_noncausal_raw_batch_heads_online_kernel<4, 8> + <<>>( + (float *)heads->ptr, + sinks, + (const float *)q->ptr, + (const float *)raw_kv->ptr, + n_tokens, n_raw, raw_cap, raw_start, n_head, head_dim); + if (!cuda_ok(cudaGetLastError(), + "attention noncausal raw batch heads online launch")) { + return 0; + } + } else { + const size_t shmem = (size_t)n_raw * sizeof(float); + if (shmem > 32768u) return 0; + dim3 grid(n_tokens, n_head, 1u); + attention_noncausal_raw_batch_heads_kernel<<>>( + (float *)heads->ptr, + sinks, + (const float *)q->ptr, + (const float *)raw_kv->ptr, + n_tokens, n_raw, raw_cap, raw_start, n_head, head_dim); + if (!cuda_ok(cudaGetLastError(), + "attention noncausal raw batch heads launch")) { + return 0; + } + } static int verify_left = -1; if (verify_left < 0) { verify_left = getenv("DS4_DSPARK_VERIFY_NONCAUSAL") != NULL ? 3 : 0; diff --git a/ds4_gpu.h b/ds4_gpu.h index c8d2aad6f..ece57a895 100644 --- a/ds4_gpu.h +++ b/ds4_gpu.h @@ -286,6 +286,12 @@ int ds4_gpu_stream_expert_exact_rows_prepare( uint32_t n_rows, uint32_t n_selected); int ds4_gpu_stream_expert_exact_rows_set_row(uint32_t row); +/* Commit the completed routed-tail command buffer without waiting. The + * backend retains every private address/overflow/cache buffer owned by the + * exact-row scope until that command buffer completes, so release() may be + * called immediately after a successful return. This is an experimental + * boundary used only when the caller explicitly enables asynchronous tails. */ +int ds4_gpu_stream_expert_exact_rows_end_async(void); void ds4_gpu_stream_expert_exact_rows_release(void); #endif void ds4_gpu_print_memory_report(const char *label); @@ -947,6 +953,11 @@ int ds4_gpu_matmul_f16_pair_compressor_store_tensor( uint32_t ratio, uint32_t pos); +/* Optional Metal ratio-4 decode fusion. Returns 1 when both paired + * compressor projections and their recurrent-state stores were encoded in a + * single dispatch, 0 when the caller should keep the established separate + * paths, and -1 on an attempted-path error. */ +int ds4_gpu_f16_quad_compressor_store_auto_available(void); int ds4_gpu_matmul_f16_quad_compressor_store_tensor( ds4_gpu_tensor *out0_kv, ds4_gpu_tensor *out0_score, @@ -2819,6 +2830,32 @@ int ds4_gpu_hc_rms_norm_mix_f16_tensor( uint32_t out_dim, float eps); +#ifdef __APPLE__ +/* Exact one-row continuation of HC RMSNorm+mix: split/Sinkhorn, HC collapse, + * and the following weighted RMSNorm are encoded in the producer dispatch. */ +int ds4_gpu_hc_rms_norm_mix_split_norm_f16_available(void); +int ds4_gpu_hc_rms_norm_mix_split_norm_f16_tensor( + ds4_gpu_tensor *mix, + ds4_gpu_tensor *out, + ds4_gpu_tensor *norm_out, + ds4_gpu_tensor *split, + const ds4_gpu_tensor *residual_hc, + const void *model_map, + uint64_t model_size, + uint64_t mix_weight_offset, + uint64_t scale_offset, + uint64_t base_offset, + uint64_t norm_weight_offset, + uint32_t n, + uint32_t mix_dim, + uint32_t n_embd, + uint32_t n_hc, + uint32_t sinkhorn_iters, + float eps, + float hc_eps, + float norm_eps); +#endif + /* Batched HC RMSNorm followed by its narrow F16 mixer projection. On the * tuned Metal path, scale_scratch stores one float per row instead of the * full normalized HC tensor; other shapes retain the established fallback. */ diff --git a/ds4_metal.m b/ds4_metal.m index 6e4a6015b..0cf905882 100644 --- a/ds4_metal.m +++ b/ds4_metal.m @@ -264,6 +264,7 @@ static id g_dsv4_router_weights_batch_pipeline; static id g_dsv4_hc_expand4_pipeline; static NSMutableDictionary> *g_pipeline_cache; +static uint64_t g_pipeline_cache_generation; enum { DS4_METAL_DECODE_PIPELINE_FAST_CACHE_SLOTS = 64, @@ -3198,11 +3199,14 @@ int ds4_gpu_test_decode_pipeline_fast_lookup_ext(void) { static struct { bool m; int32_t nc; + uint64_t generation; id pipeline; } memo; const bool memo_disabled = getenv("DS4_METAL_DISABLE_PRE_M5_FLASH_ATTN_PAD_BLK_MEMO") != NULL; - if (!memo_disabled && memo.pipeline && memo.m == has_mask && memo.nc == ncpsg) { + if (!memo_disabled && memo.pipeline && + memo.generation == g_pipeline_cache_generation && + memo.m == has_mask && memo.nc == ncpsg) { return memo.pipeline; } @@ -3211,7 +3215,9 @@ int ds4_gpu_test_decode_pipeline_fast_lookup_ext(void) { id cached = [g_pipeline_cache objectForKey:key]; if (cached) { if (!memo_disabled) { - memo = (typeof(memo)){ has_mask, ncpsg, cached }; + memo = (typeof(memo)){ + has_mask, ncpsg, g_pipeline_cache_generation, cached + }; } return cached; } @@ -3240,7 +3246,9 @@ int ds4_gpu_test_decode_pipeline_fast_lookup_ext(void) { [g_pipeline_cache setObject:pipeline forKey:key]; if (!memo_disabled) { - memo = (typeof(memo)){ has_mask, ncpsg, pipeline }; + memo = (typeof(memo)){ + has_mask, ncpsg, g_pipeline_cache_generation, pipeline + }; } return pipeline; } @@ -3257,11 +3265,14 @@ int ds4_gpu_test_decode_pipeline_fast_lookup_ext(void) { static struct { int32_t nq; int32_t nc; + uint64_t generation; id pipeline; } memo; const bool memo_disabled = getenv("DS4_METAL_DISABLE_PRE_M5_FLASH_ATTN_PAD_BLK_MEMO") != NULL; - if (!memo_disabled && memo.pipeline && memo.nq == nqptg && memo.nc == ncpsg) { + if (!memo_disabled && memo.pipeline && + memo.generation == g_pipeline_cache_generation && + memo.nq == nqptg && memo.nc == ncpsg) { return memo.pipeline; } @@ -3270,7 +3281,9 @@ int ds4_gpu_test_decode_pipeline_fast_lookup_ext(void) { id cached = [g_pipeline_cache objectForKey:key]; if (cached) { if (!memo_disabled) { - memo = (typeof(memo)){ nqptg, ncpsg, cached }; + memo = (typeof(memo)){ + nqptg, ncpsg, g_pipeline_cache_generation, cached + }; } return cached; } @@ -3299,7 +3312,9 @@ int ds4_gpu_test_decode_pipeline_fast_lookup_ext(void) { [g_pipeline_cache setObject:pipeline forKey:key]; if (!memo_disabled) { - memo = (typeof(memo)){ nqptg, ncpsg, pipeline }; + memo = (typeof(memo)){ + nqptg, ncpsg, g_pipeline_cache_generation, pipeline + }; } return pipeline; } @@ -3326,11 +3341,13 @@ int ds4_gpu_test_decode_pipeline_fast_lookup_ext(void) { const char *fn; bool m, s, b, c, k, bc; int32_t n10, n20, sg; + uint64_t generation; id pipeline; } memo; const bool memo_disabled = getenv("DS4_METAL_DISABLE_PRE_M5_FLASH_ATTN_BATCHED_MEMO") != NULL; - if (!memo_disabled && memo.pipeline && memo.fn != NULL && + if (!memo_disabled && memo.pipeline && + memo.generation == g_pipeline_cache_generation && memo.fn != NULL && strcmp(memo.fn, function_name) == 0 && memo.m == has_mask && memo.s == has_sinks && memo.b == has_bias && memo.c == has_scap && memo.k == has_kvpad && memo.bc == bc_mask && @@ -3354,7 +3371,7 @@ int ds4_gpu_test_decode_pipeline_fast_lookup_ext(void) { if (!memo_disabled) { memo = (typeof(memo)){ function_name, has_mask, has_sinks, has_bias, has_scap, has_kvpad, bc_mask, ns10, ns20, - nsg, cached }; + nsg, g_pipeline_cache_generation, cached }; } return cached; } @@ -3393,7 +3410,7 @@ int ds4_gpu_test_decode_pipeline_fast_lookup_ext(void) { if (!memo_disabled) { memo = (typeof(memo)){ function_name, has_mask, has_sinks, has_bias, has_scap, has_kvpad, bc_mask, ns10, ns20, - nsg, pipeline }; + nsg, g_pipeline_cache_generation, pipeline }; } return pipeline; } @@ -3419,9 +3436,14 @@ int ds4_gpu_test_decode_pipeline_fast_lookup_ext(void) { const char *fn; bool m, s, b, c, k, sp; int32_t n10, n20, sg, wg; + uint64_t generation; id pipeline; } memo; - if (memo.pipeline && memo.fn != NULL && strcmp(memo.fn, function_name) == 0 && + const bool memo_disabled = + getenv("DS4_METAL_DISABLE_PRE_M5_FLASH_ATTN_BATCHED_MEMO") != NULL; + if (!memo_disabled && memo.pipeline && + memo.generation == g_pipeline_cache_generation && memo.fn != NULL && + strcmp(memo.fn, function_name) == 0 && memo.m == has_mask && memo.s == has_sinks && memo.b == has_bias && memo.c == has_scap && memo.k == has_kvpad && memo.sp == shared_kvpad && memo.n10 == ns10 && memo.n20 == ns20 && memo.sg == nsg && memo.wg == nwg) { @@ -3442,9 +3464,13 @@ int ds4_gpu_test_decode_pipeline_fast_lookup_ext(void) { (int)nwg]; id cached = [g_pipeline_cache objectForKey:key]; if (cached) { - memo = (typeof(memo)){ function_name, has_mask, has_sinks, has_bias, - has_scap, has_kvpad, shared_kvpad, ns10, ns20, - nsg, nwg, cached }; + if (!memo_disabled) { + memo = (typeof(memo)){ + function_name, has_mask, has_sinks, has_bias, has_scap, + has_kvpad, shared_kvpad, ns10, ns20, nsg, nwg, + g_pipeline_cache_generation, cached + }; + } return cached; } @@ -3480,9 +3506,13 @@ int ds4_gpu_test_decode_pipeline_fast_lookup_ext(void) { } [g_pipeline_cache setObject:pipeline forKey:key]; - memo = (typeof(memo)){ function_name, has_mask, has_sinks, has_bias, - has_scap, has_kvpad, shared_kvpad, ns10, ns20, - nsg, nwg, pipeline }; + if (!memo_disabled) { + memo = (typeof(memo)){ + function_name, has_mask, has_sinks, has_bias, has_scap, + has_kvpad, shared_kvpad, ns10, ns20, nsg, nwg, + g_pipeline_cache_generation, pipeline + }; + } return pipeline; } @@ -3540,17 +3570,28 @@ int ds4_gpu_decode_attn_rope_fuse_available(void) { int32_t dv, int32_t nwg) { /* Same per-layer memo pattern as the vec getter above. */ - static int32_t memo_dv, memo_nwg; - static id memo_pipeline; - if (memo_pipeline && memo_dv == dv && memo_nwg == nwg) { - return memo_pipeline; + static struct { + int32_t dv, nwg; + uint64_t generation; + id pipeline; + } memo; + const bool memo_disabled = + getenv("DS4_METAL_DISABLE_PRE_M5_FLASH_ATTN_BATCHED_MEMO") != NULL; + if (!memo_disabled && memo.pipeline && + memo.generation == g_pipeline_cache_generation && + memo.dv == dv && memo.nwg == nwg) { + return memo.pipeline; } NSString *key = [NSString stringWithFormat:@"kernel_flash_attn_ext_vec_reduce_dv=%d_nwg=%d", (int)dv, (int)nwg]; id cached = [g_pipeline_cache objectForKey:key]; if (cached) { - memo_dv = dv; memo_nwg = nwg; memo_pipeline = cached; + if (!memo_disabled) { + memo = (typeof(memo)){ + dv, nwg, g_pipeline_cache_generation, cached + }; + } return cached; } @@ -3577,7 +3618,11 @@ int ds4_gpu_decode_attn_rope_fuse_available(void) { } [g_pipeline_cache setObject:pipeline forKey:key]; - memo_dv = dv; memo_nwg = nwg; memo_pipeline = pipeline; + if (!memo_disabled) { + memo = (typeof(memo)){ + dv, nwg, g_pipeline_cache_generation, pipeline + }; + } return pipeline; } @@ -6517,6 +6562,8 @@ int ds4_gpu_init(void) { g_q4_expert_table_cache = [NSMutableDictionary dictionary]; g_q4_expert_layer_residency_cache = [NSMutableDictionary dictionary]; g_pipeline_cache = [NSMutableDictionary dictionary]; + g_pipeline_cache_generation++; + if (g_pipeline_cache_generation == 0) g_pipeline_cache_generation++; g_dsv4_completion_cache = [NSCache new]; g_dsv4_completion_cache.countLimit = 256u; g_transient_buffers = [NSMutableArray array]; @@ -10619,6 +10666,9 @@ void ds4_gpu_cleanup(void) { g_model_buffer_cache_over_limit = 0; ds4_gpu_model_residency_clear(); ds4_gpu_model_views_clear(); + g_dsv4_hc_producer_last_completion = nil; + g_dsv4_hc_producer_last_mix_buffer = nil; + g_dsv4_hc_producer_last_mix_offset = 0; [g_pipeline_cache removeAllObjects]; g_pipeline_cache = nil; [g_q4_expert_layer_residency_cache removeAllObjects]; @@ -17514,6 +17564,58 @@ int ds4_gpu_stream_expert_exact_rows_set_row(uint32_t row) { return 1; } +int ds4_gpu_stream_expert_exact_rows_end_async(void) { + if (!g_initialized && !ds4_gpu_init()) return 0; + ds4_gpu_stream_expert_exact_rows_scope *scope = + &g_stream_expert_exact_rows_scope; + if (!scope->collecting || !scope->active || scope->row_armed || + scope->next_row != scope->n_rows || !g_batch_cb) { + return 0; + } + + /* Command buffers normally retain encoded resources, but the diagnostic + * DS4_METAL_UNRETAINED_COMMAND_BUFFERS mode deliberately disables that. + * Capture the complete immutable scope in the completion handler so the + * asynchronous boundary is correct in both modes. Cache-entry structs + * themselves live in the static cache; their three buffers are retained + * independently so later cache maintenance cannot invalidate an address + * already published in a private table. */ + NSMutableArray> *resources = [NSMutableArray array]; +#define DS4_EXACT_ROWS_RETAIN(buffer_) do { \ + id retained_buffer_ = (buffer_); \ + if (retained_buffer_) [resources addObject:retained_buffer_]; \ + } while (0) + DS4_EXACT_ROWS_RETAIN(scope->selected_buffer); + DS4_EXACT_ROWS_RETAIN(scope->gate_addrs); + DS4_EXACT_ROWS_RETAIN(scope->up_addrs); + DS4_EXACT_ROWS_RETAIN(scope->down_addrs); + DS4_EXACT_ROWS_RETAIN(scope->overflow_gate); + DS4_EXACT_ROWS_RETAIN(scope->overflow_up); + DS4_EXACT_ROWS_RETAIN(scope->overflow_down); + for (uint32_t i = 0; i < scope->n_resources; i++) { + DS4_EXACT_ROWS_RETAIN(scope->resource_gate[i]); + DS4_EXACT_ROWS_RETAIN(scope->resource_up[i]); + DS4_EXACT_ROWS_RETAIN(scope->resource_down[i]); + } +#undef DS4_EXACT_ROWS_RETAIN + NSArray> *retained_resources = [resources copy]; + + ds4_gpu_close_batch_encoder(); + id cb = g_batch_cb; + g_batch_cb = nil; + g_batch_has_work = NO; + [cb addCompletedHandler:^(id completed) { + (void)completed; + /* Referencing the array is intentional: the block owns the last + * scope-independent strong refs until Metal invokes it. */ + (void)[retained_resources count]; + }]; + [cb commit]; + [g_pending_cbs addObject:cb]; + ds4_gpu_stream_expert_cache_note_batch_committed(); + return 1; +} + void ds4_gpu_stream_expert_exact_rows_release(void) { /* release() is also the fail-clean exit for a collection that did not * reach prepare(). Neither ordinary selected state is allowed to escape @@ -21271,6 +21373,15 @@ int ds4_gpu_matmul_f16_pair_compressor_store_tensor( return 1; } +int ds4_gpu_f16_quad_compressor_store_auto_available(void) { + if (!g_initialized && !ds4_gpu_init()) return 0; + return strncmp(g_metal_device_name, "Apple M", 7) == 0 && + g_metal_device_name[7] >= '1' && + g_metal_device_name[7] <= '5' && + (g_metal_device_name[8] == '\0' || + g_metal_device_name[8] == ' '); +} + /* Quad variant of the paired compressor projection: the attention compressor * and indexer compressor pairs share the input activation and F16 matvec * shape, so one dispatch covers all four matrices. Bit-exact by diff --git a/metal/dsv4_hc.metal b/metal/dsv4_hc.metal index bf1b3a316..8b3aead3c 100644 --- a/metal/dsv4_hc.metal +++ b/metal/dsv4_hc.metal @@ -1540,3 +1540,277 @@ kernel void kernel_dsv4_hc_rms_norm_mix_f16_cluster2_pre_norm( thread_scope_device); atomic_store_explicit(completion, 0u, memory_order_relaxed); } + +/* The compound producer writes the 4x4 combination logits from four + * independent threadgroups. The final arriving group performs exactly the + * same Sinkhorn sequence used by the standalone HC split kernel. */ +static __attribute__((always_inline)) inline void +ds4_hc_comb_weights4_exact_continuation( + constant ds4_metal_args_dsv4_hc_split_weighted_sum_norm & args, + device volatile const float *mix, + device const float *scale, + device const float *base, + device float *out) { + const float epsv = args.eps; + const float comb_scale = scale[2]; + + float4 r0 = *((device volatile const float4 *)(mix + 8)) * comb_scale + + *((device const float4 *)(base + 8)); + float4 r1 = *((device volatile const float4 *)(mix + 12)) * comb_scale + + *((device const float4 *)(base + 12)); + float4 r2 = *((device volatile const float4 *)(mix + 16)) * comb_scale + + *((device const float4 *)(base + 16)); + float4 r3 = *((device volatile const float4 *)(mix + 20)) * comb_scale + + *((device const float4 *)(base + 20)); + + const float m0 = max(max(r0.x, r0.y), max(r0.z, r0.w)); + const float m1 = max(max(r1.x, r1.y), max(r1.z, r1.w)); + const float m2 = max(max(r2.x, r2.y), max(r2.z, r2.w)); + const float m3 = max(max(r3.x, r3.y), max(r3.z, r3.w)); + + r0 = exp(r0 - m0); + r1 = exp(r1 - m1); + r2 = exp(r2 - m2); + r3 = exp(r3 - m3); + + r0 = r0 * (1.0f / (r0.x + r0.y + r0.z + r0.w)) + epsv; + r1 = r1 * (1.0f / (r1.x + r1.y + r1.z + r1.w)) + epsv; + r2 = r2 * (1.0f / (r2.x + r2.y + r2.z + r2.w)) + epsv; + r3 = r3 * (1.0f / (r3.x + r3.y + r3.z + r3.w)) + epsv; + + float4 col_inv = 1.0f / (r0 + r1 + r2 + r3 + epsv); + r0 *= col_inv; + r1 *= col_inv; + r2 *= col_inv; + r3 *= col_inv; + + for (int iter = 1; iter < args.sinkhorn_iters; ++iter) { + r0 *= 1.0f / (r0.x + r0.y + r0.z + r0.w + epsv); + r1 *= 1.0f / (r1.x + r1.y + r1.z + r1.w + epsv); + r2 *= 1.0f / (r2.x + r2.y + r2.z + r2.w + epsv); + r3 *= 1.0f / (r3.x + r3.y + r3.z + r3.w + epsv); + + col_inv = 1.0f / (r0 + r1 + r2 + r3 + epsv); + r0 *= col_inv; + r1 *= col_inv; + r2 *= col_inv; + r3 *= col_inv; + } + + *((device float4 *)(out + 8)) = r0; + *((device float4 *)(out + 12)) = r1; + *((device float4 *)(out + 16)) = r2; + *((device float4 *)(out + 20)) = r3; +} + +/* Exact DS4 decode compound for the fixed 16384 -> 24 HC producer shape. + * Six 512-thread groups reproduce the original RMSNorm and F16 matvec trees. + * Group zero continues directly into pre weighting, HC collapse, and the next + * weighted RMSNorm; group one emits post weights; groups two through five + * publish the combination rows and the last arrival runs Sinkhorn. */ +kernel void kernel_dsv4_hc_rms_norm_mix_f16_cluster2_pre_norm( + constant ds4_metal_args_hc_norm_mix & args, + constant ds4_metal_args_dsv4_hc_split_weighted_sum_norm & split_args, + device const char * x, + device const char * weight, + device char * dst, + device const float * hc_scale, + device const float * hc_base, + device char * split, + device char * collapse_dst, + device const char * norm_weight, + device char * norm_dst, + device atomic_uint * completion, + threadgroup char * shmem [[threadgroup(0)]], + uint3 tgpig [[threadgroup_position_in_grid]], + ushort tiisg [[thread_index_in_simdgroup]], + ushort sgitg [[simdgroup_index_in_threadgroup]]) { + constexpr short NSG_CLUSTER = 8; + constexpr short NCLUSTER = 2; + constexpr short NSG_TOTAL = NSG_CLUSTER * NCLUSTER; + constexpr short NW = N_SIMDWIDTH; + constexpr short NR0 = 2; + constexpr short NB = 32; + constexpr short NF = 16; + constexpr short NF4 = NF/4; + constexpr uint VTHREADS = 1024u; + constexpr short VSLICES = VTHREADS/(NSG_TOTAL*NW); + + const uint n = (uint)args.n; + const uint n4 = n >> 2; + device const float4 *x4 = (device const float4 *)x; + threadgroup float *norm_shmem = (threadgroup float *)shmem; + threadgroup float *mv_shmem = norm_shmem + NW; + + for (short v = 0; v < VSLICES; ++v) { + const uint vt = (uint)(sgitg + NSG_TOTAL*v)*NW + tiisg; + float sumf = 0.0f; + for (uint i00 = vt; i00 < n4; i00 += VTHREADS) { + sumf += dot(x4[i00], x4[i00]); + } + sumf = simd_sum(sumf); + if (tiisg == 0) { + norm_shmem[sgitg + NSG_TOTAL*v] = sumf; + } + } + threadgroup_barrier(mem_flags::mem_threadgroup); + + float total = norm_shmem[tiisg]; + total = simd_sum(total); + const float mean = total/(float)args.n; + const float scale = 1.0f/sqrt(mean + args.eps); + + const short cluster = sgitg / NSG_CLUSTER; + const short local_sg = sgitg - cluster*NSG_CLUSTER; + const int nb = args.n/NB; + const int r0 = (int)tgpig.x*(NCLUSTER*NR0) + cluster*NR0; + + device const half4 *ax4[NR0]; + FOR_UNROLL(short row = 0; row < NR0; ++row) { + ax4[row] = (device const half4 *)( + weight + (uint64_t)(r0 + row)*(uint64_t)n*sizeof(half)); + } + + float sumf_mv[NR0] = { 0.0f }; + const short ix = tiisg/(NW/NF); + const short il = tiisg%(NW/NF); + const int ib0 = local_sg*NF + ix; + for (int ib = ib0; ib < nb; ib += NSG_CLUSTER*NF) { + float4 yl4[NF4]; + FOR_UNROLL(short i = 0; i < NF4; ++i) { + yl4[i] = x4[(ib*NB + il*NF)/4 + i]*scale; + } + FOR_UNROLL(short row = 0; row < NR0; ++row) { + device const half4 *xb4 = ax4[row] + (ib*NB + il*NF)/4; + float sumq = 0.0f; + FOR_UNROLL(short i = 0; i < NF4; ++i) { + sumq += dot(float4(xb4[i]), yl4[i]); + } + sumf_mv[row] += sumq; + } + } + + threadgroup float *cluster_shmem[NR0]; + FOR_UNROLL(short row = 0; row < NR0; ++row) { + cluster_shmem[row] = mv_shmem + ((uint)cluster*NR0 + row)*NW; + if (local_sg == 0) { + cluster_shmem[row][tiisg] = 0.0f; + } + sumf_mv[row] = simd_sum(sumf_mv[row]); + } + threadgroup_barrier(mem_flags::mem_threadgroup); + FOR_UNROLL(short row = 0; row < NR0; ++row) { + if (tiisg == 0) { + cluster_shmem[row][local_sg] = sumf_mv[row]; + } + } + threadgroup_barrier(mem_flags::mem_threadgroup); + + device volatile float *mixes_f32 = (device volatile float *)dst; + if (local_sg == 0) { + FOR_UNROLL(short row = 0; row < NR0; ++row) { + const float tot = simd_sum(cluster_shmem[row][tiisg]); + if (tiisg == 0 && r0 + row < args.out_dim) { + mixes_f32[r0 + row] = tot; + } + } + } + + threadgroup_barrier(mem_flags::mem_device_and_threadgroup); + const uint tid = (uint)sgitg*(uint)NW + (uint)tiisg; + threadgroup float *pre_shmem = norm_shmem + 32u + 4u*NW; + threadgroup float *sum_shmem = pre_shmem + 4; + + if (tgpig.x == 0) { + device float *out = (device float *)split; + if (tid == 0) { + const float4 pre_z = + *((device volatile const float4 *)mixes_f32)*hc_scale[0] + + *((device const float4 *)hc_base); + const float4 pre = + 1.0f/(1.0f + exp(-pre_z)) + split_args.eps; + *((device float4 *)out) = pre; + pre_shmem[0] = pre.x; + pre_shmem[1] = pre.y; + pre_shmem[2] = pre.z; + pre_shmem[3] = pre.w; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + + const uint n4_collapse = uint(split_args.n_embd) >> 2; + const uint i0 = tid; + const uint i1 = tid + 512u; + device const float4 *x0 = + (device const float4 *)(x + 0*split_args.nb_x1); + device const float4 *x1 = + (device const float4 *)(x + 1*split_args.nb_x1); + device const float4 *x2 = + (device const float4 *)(x + 2*split_args.nb_x1); + device const float4 *x3 = + (device const float4 *)(x + 3*split_args.nb_x1); + + float4 v0 = 0.0f; + v0 += x0[i0]*pre_shmem[0]; + v0 += x1[i0]*pre_shmem[1]; + v0 += x2[i0]*pre_shmem[2]; + v0 += x3[i0]*pre_shmem[3]; + const float sum0 = simd_sum(dot(v0, v0)); + + float4 v1 = 0.0f; + if (i1 < n4_collapse) { + v1 += x0[i1]*pre_shmem[0]; + v1 += x1[i1]*pre_shmem[1]; + v1 += x2[i1]*pre_shmem[2]; + v1 += x3[i1]*pre_shmem[3]; + } + const float sum1 = simd_sum(dot(v1, v1)); + if (tiisg == 0) { + sum_shmem[sgitg] = sum0; + sum_shmem[sgitg + 16] = sum1; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + + float sumf = sum_shmem[tiisg]; + sumf = simd_sum(sumf); + const float norm_arg = + sumf/float(split_args.n_embd) + split_args.norm_eps; + const float norm_scale = rsqrt(norm_arg); + device float4 *dst4 = (device float4 *)collapse_dst; + device const float4 *w4 = (device const float4 *)norm_weight; + device float4 *norm4 = (device float4 *)norm_dst; + dst4[i0] = v0; + norm4[i0] = (v0*norm_scale)*w4[i0]; + if (i1 < n4_collapse) { + dst4[i1] = v1; + norm4[i1] = (v1*norm_scale)*w4[i1]; + } + } else if (tgpig.x == 1 && tid == 0) { + device float *out = (device float *)split; + const float4 post_z = + *((device volatile const float4 *)(mixes_f32 + 4))*hc_scale[1] + + *((device const float4 *)(hc_base + 4)); + *((device float4 *)(out + 4)) = 2.0f/(1.0f + exp(-post_z)); + } + + atomic_thread_fence(mem_flags::mem_device, + memory_order_seq_cst, + thread_scope_device); + if (tgpig.x < 2 || tid != 0) { + return; + } + + const uint old = atomic_fetch_add_explicit( + completion, 1u, memory_order_relaxed); + if (old + 1u != 4u) { + return; + } + atomic_thread_fence(mem_flags::mem_device, + memory_order_seq_cst, + thread_scope_device); + ds4_hc_comb_weights4_exact_continuation( + split_args, mixes_f32, hc_scale, hc_base, (device float *)split); + atomic_thread_fence(mem_flags::mem_device, + memory_order_seq_cst, + thread_scope_device); + atomic_store_explicit(completion, 0u, memory_order_relaxed); +} diff --git a/tests/ds4_test.c b/tests/ds4_test.c index 4ca112a57..da986f177 100644 --- a/tests/ds4_test.c +++ b/tests/ds4_test.c @@ -1392,6 +1392,292 @@ static void test_metal_f16_compressor_pair_state_store_exact(void) { 512, 128, 255, 1, 43, false); } +static void test_metal_f16_compressor_quad_state_store_exact_case( + uint32_t width0, + uint32_t pos, + uint32_t ape0_type, + uint32_t ape1_type, + uint32_t seed) { + const uint32_t in_dim = 4096u; + const uint32_t ratio = 4u; + const uint32_t width1 = 256u; + const uint32_t state_rows = 2u * ratio; + const uint32_t widths[4] = {width0, width0, width1, width1}; + const uint32_t ape_types[2] = {ape0_type, ape1_type}; + const uint64_t page = (uint64_t)getpagesize(); + const uint64_t weight_bytes[4] = { + (uint64_t)width0 * in_dim * sizeof(uint16_t), + (uint64_t)width0 * in_dim * sizeof(uint16_t), + (uint64_t)width1 * in_dim * sizeof(uint16_t), + (uint64_t)width1 * in_dim * sizeof(uint16_t), + }; + uint64_t weight_offsets[4] = {0}; + for (uint32_t i = 1; i < 4u; i++) { + weight_offsets[i] = test_round_up_u64( + weight_offsets[i - 1u] + weight_bytes[i - 1u], page); + } + const uint64_t ape0_offset = test_round_up_u64( + weight_offsets[3] + weight_bytes[3], page); + const uint64_t ape0_bytes = (uint64_t)ratio * width0 * + (ape0_type == 1u ? sizeof(uint16_t) : sizeof(float)); + const uint64_t ape1_offset = test_round_up_u64( + ape0_offset + ape0_bytes, page); + const uint64_t ape1_bytes = (uint64_t)ratio * width1 * + (ape1_type == 1u ? sizeof(uint16_t) : sizeof(float)); + const uint64_t ape_offsets[2] = {ape0_offset, ape1_offset}; + const uint32_t ape_widths[2] = {width0, width1}; + const uint64_t model_bytes = test_round_up_u64( + ape1_offset + ape1_bytes, page); + const uint64_t x_bytes = (uint64_t)in_dim * sizeof(float); + const uint64_t max_state_count = (uint64_t)state_rows * width0; + + void *model_raw = NULL; + TEST_ASSERT(posix_memalign( + &model_raw, (size_t)page, (size_t)model_bytes) == 0); + ds4_gpu_tensor *x = ds4_gpu_tensor_alloc(x_bytes); + ds4_gpu_tensor *ref_out[4] = {0}; + ds4_gpu_tensor *quad_out[4] = {0}; + ds4_gpu_tensor *ref_state[4] = {0}; + ds4_gpu_tensor *quad_state[4] = {0}; + bool allocated = model_raw != NULL && x != NULL; + for (uint32_t i = 0; i < 4u; i++) { + const uint64_t out_bytes = + (uint64_t)widths[i] * sizeof(float); + const uint64_t state_bytes = + (uint64_t)state_rows * widths[i] * sizeof(float); + ref_out[i] = ds4_gpu_tensor_alloc(out_bytes); + quad_out[i] = ds4_gpu_tensor_alloc(out_bytes); + ref_state[i] = ds4_gpu_tensor_alloc(state_bytes); + quad_state[i] = ds4_gpu_tensor_alloc(state_bytes); + TEST_ASSERT(ref_out[i] != NULL); + TEST_ASSERT(quad_out[i] != NULL); + TEST_ASSERT(ref_state[i] != NULL); + TEST_ASSERT(quad_state[i] != NULL); + allocated = allocated && ref_out[i] && quad_out[i] && + ref_state[i] && quad_state[i]; + } + + float *x_host = malloc((size_t)x_bytes); + float *ref_host = malloc((size_t)max_state_count * sizeof(float)); + float *quad_host = malloc((size_t)max_state_count * sizeof(float)); + TEST_ASSERT(x_host != NULL); + TEST_ASSERT(ref_host != NULL); + TEST_ASSERT(quad_host != NULL); + allocated = allocated && x_host && ref_host && quad_host; + + const char *force_pair_env = + "DS4_METAL_ENABLE_COMPRESSOR_PAIR_STATE_STORE"; + const char *disable_pair_state_env = + "DS4_METAL_DISABLE_M3_COMPRESSOR_PAIR_STATE_STORE"; + const char *disable_pair_proj_env = + "DS4_METAL_DISABLE_COMPRESSOR_PAIR_PROJ"; + const char *disable_store_env = + "DS4_METAL_DISABLE_COMPRESSOR_STORE_ONE"; + char *saved_force_pair = test_save_env(force_pair_env); + char *saved_disable_pair_state = test_save_env(disable_pair_state_env); + char *saved_disable_pair_proj = test_save_env(disable_pair_proj_env); + char *saved_disable_store = test_save_env(disable_store_env); + + test_float_compare_stats out_stats[4] = {{0}}; + test_float_compare_stats state_stats[4] = {{0}}; + if (allocated) { + memset(model_raw, 0, (size_t)model_bytes); + for (uint32_t matrix = 0; matrix < 4u; matrix++) { + uint16_t *weights = (uint16_t *)( + (uint8_t *)model_raw + weight_offsets[matrix]); + for (uint32_t o = 0; o < widths[matrix]; o++) { + for (uint32_t i = 0; i < in_dim; i++) { + const int value = + (int)((o * (17u + 2u * matrix) + + i * (23u + 4u * matrix) + + (o ^ (i >> (matrix & 3u))) * + (3u + 2u * matrix) + + seed * (29u + matrix)) % 127u) - 63; + weights[(uint64_t)o * in_dim + i] = + test_float_to_f16( + (float)value / (96.0f + 8.0f * matrix)); + } + } + } + + for (uint32_t which = 0; which < 2u; which++) { + const uint64_t count = + (uint64_t)ratio * ape_widths[which]; + if (ape_types[which] == 1u) { + uint16_t *ape = (uint16_t *)( + (uint8_t *)model_raw + ape_offsets[which]); + for (uint64_t i = 0; i < count; i++) { + const int value = + (int)((i * (13u + 2u * which) + + (i ^ (i >> 3u)) * (7u + 2u * which) + + seed * (17u + which)) % 61u) - 30; + ape[i] = test_float_to_f16( + (float)value / (80.0f + 8.0f * which)); + } + } else { + float *ape = (float *)( + (uint8_t *)model_raw + ape_offsets[which]); + for (uint64_t i = 0; i < count; i++) { + const int value = + (int)((i * (13u + 2u * which) + + (i ^ (i >> 3u)) * (7u + 2u * which) + + seed * (17u + which)) % 61u) - 30; + ape[i] = + (float)value / (80.0f + 8.0f * which); + } + } + } + + for (uint32_t i = 0; i < in_dim; i++) { + const int value = + (int)((i * 29u + (i ^ (i >> 4u)) * 9u + + seed * 11u) % 127u) - 63; + x_host[i] = (float)value / 88.0f; + } + TEST_ASSERT(ds4_gpu_tensor_write(x, 0, x_host, x_bytes) != 0); + + for (uint32_t buffer = 0; buffer < 4u; buffer++) { + const uint64_t out_bytes = + (uint64_t)widths[buffer] * sizeof(float); + const uint64_t state_count = + (uint64_t)state_rows * widths[buffer]; + const uint64_t state_bytes = state_count * sizeof(float); + for (uint32_t i = 0; i < widths[buffer]; i++) { + const uint32_t poison = + 0x7fc00001u + ((buffer * 1024u + i) & 0x3fffu); + memcpy(ref_host + i, &poison, sizeof(poison)); + } + TEST_ASSERT(ds4_gpu_tensor_write( + ref_out[buffer], 0, ref_host, + out_bytes) != 0); + TEST_ASSERT(ds4_gpu_tensor_write( + quad_out[buffer], 0, ref_host, + out_bytes) != 0); + + for (uint64_t i = 0; i < state_count; i++) { + const int value = + (int)((i * (5u + 2u * buffer) + + seed * (13u + buffer)) % 193u) - 96; + ref_host[i] = + (float)value / (64.0f + 8.0f * buffer); + } + TEST_ASSERT(ds4_gpu_tensor_write( + ref_state[buffer], 0, ref_host, + state_bytes) != 0); + TEST_ASSERT(ds4_gpu_tensor_write( + quad_state[buffer], 0, ref_host, + state_bytes) != 0); + } + + TEST_ASSERT(ds4_gpu_set_model_map(model_raw, model_bytes) != 0); + ds4_gpu_set_quality(false); + TEST_ASSERT(setenv(force_pair_env, "1", 1) == 0); + TEST_ASSERT(unsetenv(disable_pair_state_env) == 0); + TEST_ASSERT(unsetenv(disable_pair_proj_env) == 0); + TEST_ASSERT(unsetenv(disable_store_env) == 0); + + TEST_ASSERT(ds4_gpu_matmul_f16_pair_compressor_store_tensor( + ref_out[0], ref_out[1], + ref_state[0], ref_state[1], + model_raw, model_bytes, + weight_offsets[0], weight_offsets[1], + ape_offsets[0], ape_types[0], + in_dim, width0, x, ratio, pos) == 1); + TEST_ASSERT(ds4_gpu_matmul_f16_pair_compressor_store_tensor( + ref_out[2], ref_out[3], + ref_state[2], ref_state[3], + model_raw, model_bytes, + weight_offsets[2], weight_offsets[3], + ape_offsets[1], ape_types[1], + in_dim, width1, x, ratio, pos) == 1); + TEST_ASSERT(ds4_gpu_matmul_f16_quad_compressor_store_tensor( + quad_out[0], quad_out[1], + quad_out[2], quad_out[3], + quad_state[0], quad_state[1], + quad_state[2], quad_state[3], + model_raw, model_bytes, + weight_offsets[0], weight_offsets[1], + weight_offsets[2], weight_offsets[3], + ape_offsets[0], ape_types[0], + ape_offsets[1], ape_types[1], + in_dim, width0, width1, x, ratio, pos) == 1); + + for (uint32_t buffer = 0; buffer < 4u; buffer++) { + const uint64_t out_bytes = + (uint64_t)widths[buffer] * sizeof(float); + const uint64_t state_count = + (uint64_t)state_rows * widths[buffer]; + const uint64_t state_bytes = state_count * sizeof(float); + TEST_ASSERT(ds4_gpu_tensor_read( + ref_out[buffer], 0, ref_host, + out_bytes) != 0); + TEST_ASSERT(ds4_gpu_tensor_read( + quad_out[buffer], 0, quad_host, + out_bytes) != 0); + out_stats[buffer] = test_compare_float_bits( + ref_host, quad_host, widths[buffer]); + TEST_ASSERT(ds4_gpu_tensor_read( + ref_state[buffer], 0, ref_host, + state_bytes) != 0); + TEST_ASSERT(ds4_gpu_tensor_read( + quad_state[buffer], 0, quad_host, + state_bytes) != 0); + state_stats[buffer] = test_compare_float_bits( + ref_host, quad_host, (size_t)state_count); + } + } + + test_restore_env(force_pair_env, saved_force_pair); + test_restore_env(disable_pair_state_env, saved_disable_pair_state); + test_restore_env(disable_pair_proj_env, saved_disable_pair_proj); + test_restore_env(disable_store_env, saved_disable_store); + + size_t out_mismatches = 0; + size_t state_mismatches = 0; + uint32_t max_out_ulp = 0; + uint32_t max_state_ulp = 0; + for (uint32_t i = 0; i < 4u; i++) { + out_mismatches += out_stats[i].mismatch_count; + state_mismatches += state_stats[i].mismatch_count; + if (out_stats[i].max_ulp > max_out_ulp) { + max_out_ulp = out_stats[i].max_ulp; + } + if (state_stats[i].max_ulp > max_state_ulp) { + max_state_ulp = state_stats[i].max_ulp; + } + TEST_ASSERT(out_stats[i].mismatch_count == 0); + TEST_ASSERT(state_stats[i].mismatch_count == 0); + } + fprintf(stderr, + "ds4-test: compressor quad state-store exact width=%u/256 " + "pos_mod4=%u ape=%s/%s outputs=%zu states=%zu " + "max_ulp=%u/%u\n", + width0, pos % ratio, + ape0_type == 1u ? "f16" : "f32", + ape1_type == 1u ? "f16" : "f32", + out_mismatches, state_mismatches, + max_out_ulp, max_state_ulp); + + free(quad_host); + free(ref_host); + free(x_host); + for (uint32_t i = 0; i < 4u; i++) { + ds4_gpu_tensor_free(quad_state[i]); + ds4_gpu_tensor_free(ref_state[i]); + ds4_gpu_tensor_free(quad_out[i]); + ds4_gpu_tensor_free(ref_out[i]); + } + ds4_gpu_tensor_free(x); + free(model_raw); +} + +static void test_metal_f16_compressor_quad_state_store_exact(void) { + test_metal_f16_compressor_quad_state_store_exact_case( + 1024u, 8u, 1u, 0u, 59u); + test_metal_f16_compressor_quad_state_store_exact_case( + 512u, 11u, 0u, 1u, 71u); +} + static void test_metal_compressor_ape_add_exact_case( uint32_t head_dim, uint32_t ratio, @@ -2267,7 +2553,7 @@ static void test_metal_compressor_ratio4_direct_pool_exact_case( } static void test_metal_compressor_ratio4_direct_pool_exact(void) { - /* n_comp == 1 deliberately stays on the exact GGML reduction path. */ + /* Prefill direct-pool coverage, including its legacy n_comp == 1 case. */ test_metal_compressor_ratio4_direct_pool_exact_case( 512, 0, 4, 1, false, 59); test_metal_compressor_ratio4_direct_pool_exact_case( @@ -2282,6 +2568,269 @@ static void test_metal_compressor_ratio4_direct_pool_exact(void) { 128, 8, 4, 1, true, 79); } +static void test_metal_compressor_ratio4_exact_pool_decode_case( + uint32_t head_dim, + uint32_t ape_type, + uint32_t seed) { + const uint32_t ratio = 4u; + const uint32_t width = 2u * head_dim; + const uint32_t state_rows = 8u; + const uint32_t pos = 3u; + const uint64_t cur_count = width; + const uint64_t state_count = (uint64_t)state_rows * width; + const uint64_t cur_bytes = cur_count * sizeof(float); + const uint64_t state_bytes = state_count * sizeof(float); + const uint64_t comp_bytes = (uint64_t)head_dim * sizeof(float); + const uint64_t ape_elem_bytes = ape_type == 1u ? 2u : 4u; + const uint64_t ape_bytes = (uint64_t)ratio * width * ape_elem_bytes; + const uint64_t page = (uint64_t)getpagesize(); + const uint64_t norm_offset = test_round_up_u64(ape_bytes, page); + const uint64_t model_bytes = test_round_up_u64( + norm_offset + (uint64_t)head_dim * sizeof(float), page); + + ds4_gpu_tensor *kv_cur = ds4_gpu_tensor_alloc(cur_bytes); + ds4_gpu_tensor *sc_cur = ds4_gpu_tensor_alloc(cur_bytes); + ds4_gpu_tensor *ref_state_kv = ds4_gpu_tensor_alloc(state_bytes); + ds4_gpu_tensor *ref_state_score = ds4_gpu_tensor_alloc(state_bytes); + ds4_gpu_tensor *exact_state_kv = ds4_gpu_tensor_alloc(state_bytes); + ds4_gpu_tensor *exact_state_score = ds4_gpu_tensor_alloc(state_bytes); + ds4_gpu_tensor *ref_comp = ds4_gpu_tensor_alloc(comp_bytes); + ds4_gpu_tensor *exact_comp = ds4_gpu_tensor_alloc(comp_bytes); + float *kv_cur_host = malloc((size_t)cur_bytes); + float *sc_cur_host = malloc((size_t)cur_bytes); + float *state_kv_host = malloc((size_t)state_bytes); + float *state_score_host = malloc((size_t)state_bytes); + float *ref_state_kv_host = malloc((size_t)state_bytes); + float *ref_state_score_host = malloc((size_t)state_bytes); + float *exact_state_kv_host = malloc((size_t)state_bytes); + float *exact_state_score_host = malloc((size_t)state_bytes); + float *ref_comp_host = malloc((size_t)comp_bytes); + float *exact_comp_host = malloc((size_t)comp_bytes); + void *model_raw = NULL; + const int model_alloc_ok = posix_memalign( + &model_raw, (size_t)page, (size_t)model_bytes) == 0; + + TEST_ASSERT(kv_cur != NULL); + TEST_ASSERT(sc_cur != NULL); + TEST_ASSERT(ref_state_kv != NULL); + TEST_ASSERT(ref_state_score != NULL); + TEST_ASSERT(exact_state_kv != NULL); + TEST_ASSERT(exact_state_score != NULL); + TEST_ASSERT(ref_comp != NULL); + TEST_ASSERT(exact_comp != NULL); + TEST_ASSERT(kv_cur_host != NULL); + TEST_ASSERT(sc_cur_host != NULL); + TEST_ASSERT(state_kv_host != NULL); + TEST_ASSERT(state_score_host != NULL); + TEST_ASSERT(ref_state_kv_host != NULL); + TEST_ASSERT(ref_state_score_host != NULL); + TEST_ASSERT(exact_state_kv_host != NULL); + TEST_ASSERT(exact_state_score_host != NULL); + TEST_ASSERT(ref_comp_host != NULL); + TEST_ASSERT(exact_comp_host != NULL); + TEST_ASSERT(model_alloc_ok); + + const char *enable_env = + "DS4_METAL_ENABLE_COMPRESSOR_EXACT_POOL_RATIO4"; + const char *disable_env = + "DS4_METAL_DISABLE_COMPRESSOR_EXACT_POOL_RATIO4"; + const char *pre_m5_disable_env = + "DS4_METAL_DISABLE_PRE_M5_COMPRESSOR_EXACT_POOL_RATIO4"; + const char *m3_disable_env = + "DS4_METAL_DISABLE_M3_COMPRESSOR_EXACT_POOL_RATIO4"; + const char *m5_disable_env = + "DS4_METAL_DISABLE_M5_COMPRESSOR_EXACT_POOL_RATIO4"; + const char *all_pre_m5_disable_env = + "DS4_METAL_DISABLE_PRE_M5_DECODE_PORTS"; + const char *require_env = + "DS4_METAL_REQUIRE_COMPRESSOR_EXACT_POOL_RATIO4"; + const char *poison_env = + "DS4_METAL_TEST_POISON_COMPRESSOR_EXACT_REDUCTION_SCRATCH"; + char *saved_enable = test_save_env(enable_env); + char *saved_disable = test_save_env(disable_env); + char *saved_pre_m5_disable = test_save_env(pre_m5_disable_env); + char *saved_m3_disable = test_save_env(m3_disable_env); + char *saved_m5_disable = test_save_env(m5_disable_env); + char *saved_all_pre_m5_disable = test_save_env(all_pre_m5_disable_env); + char *saved_require = test_save_env(require_env); + char *saved_poison = test_save_env(poison_env); + test_float_compare_stats comp_stats = {0}; + test_float_compare_stats state_kv_stats = {0}; + test_float_compare_stats state_score_stats = {0}; + + const bool allocated = kv_cur && sc_cur && ref_state_kv && + ref_state_score && exact_state_kv && exact_state_score && ref_comp && + exact_comp && kv_cur_host && sc_cur_host && state_kv_host && + state_score_host && ref_state_kv_host && ref_state_score_host && + exact_state_kv_host && exact_state_score_host && ref_comp_host && + exact_comp_host && model_alloc_ok; + if (allocated) { + memset(model_raw, 0, (size_t)model_bytes); + if (ape_type == 1u) { + uint16_t *ape = model_raw; + for (uint64_t i = 0; i < (uint64_t)ratio * width; i++) { + const int value = + (int)((i * 17u + (i >> 3u) * 13u + seed * 11u) % + 101u) - 50; + ape[i] = test_float_to_f16((float)value / 96.0f); + } + } else { + float *ape = model_raw; + for (uint64_t i = 0; i < (uint64_t)ratio * width; i++) { + const int value = + (int)((i * 17u + (i >> 3u) * 13u + seed * 11u) % + 101u) - 50; + ape[i] = (float)value / 96.0f; + } + } + float *norm = + (float *)((uint8_t *)model_raw + norm_offset); + for (uint32_t i = 0; i < head_dim; i++) { + norm[i] = 0.75f + + (float)((i * 7u + seed * 3u) % 29u) / 128.0f; + } + + for (uint32_t i = 0; i < width; i++) { + const int kv_value = + (int)(((uint64_t)i * 29u + seed * 37u) % 211u) - 105; + const int score_value = + (int)(((uint64_t)i * 31u + seed * 19u) % 181u) - 90; + kv_cur_host[i] = (float)kv_value / 112.0f; + sc_cur_host[i] = (float)score_value / 48.0f; + } + for (uint32_t row = 0; row < state_rows; row++) { + for (uint32_t col = 0; col < width; col++) { + const uint64_t i = (uint64_t)row * width + col; + const int kv_value = + (int)(((uint64_t)row * 43u + (uint64_t)col * 23u + + seed * 41u) % 257u) - 128; + const int score_value = + (int)(((uint64_t)row * 47u + (uint64_t)col * 17u + + seed * 31u) % 193u) - 96; + state_kv_host[i] = (float)kv_value / 128.0f; + state_score_host[i] = (float)score_value / 56.0f; + } + } + + TEST_ASSERT(ds4_gpu_tensor_write( + kv_cur, 0, kv_cur_host, cur_bytes) != 0); + TEST_ASSERT(ds4_gpu_tensor_write( + sc_cur, 0, sc_cur_host, cur_bytes) != 0); + TEST_ASSERT(ds4_gpu_tensor_write( + ref_state_kv, 0, state_kv_host, state_bytes) != 0); + TEST_ASSERT(ds4_gpu_tensor_write( + ref_state_score, 0, state_score_host, + state_bytes) != 0); + TEST_ASSERT(ds4_gpu_tensor_write( + exact_state_kv, 0, state_kv_host, state_bytes) != 0); + TEST_ASSERT(ds4_gpu_tensor_write( + exact_state_score, 0, state_score_host, + state_bytes) != 0); + TEST_ASSERT(ds4_gpu_set_model_map(model_raw, model_bytes) != 0); + ds4_gpu_set_quality(false); + + /* Force-enable and force-disable together: the kill switch must win. */ + TEST_ASSERT(setenv(enable_env, "1", 1) == 0); + TEST_ASSERT(setenv(disable_env, "1", 1) == 0); + TEST_ASSERT(unsetenv(require_env) == 0); + TEST_ASSERT(unsetenv(poison_env) == 0); + const int ref_ok = ds4_gpu_compressor_update_tensor( + kv_cur, sc_cur, ref_state_kv, ref_state_score, ref_comp, + model_raw, model_bytes, 0, ape_type, norm_offset, 0, + head_dim, ratio, pos, 0, 0, 0, + 10000.0f, 1.0f, 0.0f, 1.0f, 32.0f, 1.0f, 1.0e-6f, false); + TEST_ASSERT(ref_ok != 0); + + TEST_ASSERT(unsetenv(disable_env) == 0); + TEST_ASSERT(unsetenv(pre_m5_disable_env) == 0); + TEST_ASSERT(unsetenv(m3_disable_env) == 0); + TEST_ASSERT(unsetenv(m5_disable_env) == 0); + TEST_ASSERT(unsetenv(all_pre_m5_disable_env) == 0); + TEST_ASSERT(setenv(require_env, "1", 1) == 0); + TEST_ASSERT(setenv(poison_env, "1", 1) == 0); + const int exact_ok = ds4_gpu_compressor_update_tensor( + kv_cur, sc_cur, exact_state_kv, exact_state_score, exact_comp, + model_raw, model_bytes, 0, ape_type, norm_offset, 0, + head_dim, ratio, pos, 0, 0, 0, + 10000.0f, 1.0f, 0.0f, 1.0f, 32.0f, 1.0f, 1.0e-6f, false); + TEST_ASSERT(exact_ok != 0); + + TEST_ASSERT(ds4_gpu_tensor_read( + ref_comp, 0, ref_comp_host, comp_bytes) != 0); + TEST_ASSERT(ds4_gpu_tensor_read( + exact_comp, 0, exact_comp_host, comp_bytes) != 0); + TEST_ASSERT(ds4_gpu_tensor_read( + ref_state_kv, 0, ref_state_kv_host, + state_bytes) != 0); + TEST_ASSERT(ds4_gpu_tensor_read( + exact_state_kv, 0, exact_state_kv_host, + state_bytes) != 0); + TEST_ASSERT(ds4_gpu_tensor_read( + ref_state_score, 0, ref_state_score_host, + state_bytes) != 0); + TEST_ASSERT(ds4_gpu_tensor_read( + exact_state_score, 0, exact_state_score_host, + state_bytes) != 0); + + comp_stats = test_compare_float_bits( + ref_comp_host, exact_comp_host, head_dim); + state_kv_stats = test_compare_float_bits( + ref_state_kv_host, exact_state_kv_host, (size_t)state_count); + state_score_stats = test_compare_float_bits( + ref_state_score_host, exact_state_score_host, + (size_t)state_count); + } + + test_restore_env(enable_env, saved_enable); + test_restore_env(disable_env, saved_disable); + test_restore_env(pre_m5_disable_env, saved_pre_m5_disable); + test_restore_env(m3_disable_env, saved_m3_disable); + test_restore_env(m5_disable_env, saved_m5_disable); + test_restore_env(all_pre_m5_disable_env, saved_all_pre_m5_disable); + test_restore_env(require_env, saved_require); + test_restore_env(poison_env, saved_poison); + fprintf(stderr, + "ds4-test: compressor ratio4 exact decode pool head=%u ape=%s " + "comp=%zu/%u state_kv=%zu/%llu state_score=%zu/%llu " + "max_ulp=%u/%u/%u\n", + head_dim, ape_type == 1u ? "f16" : "f32", + comp_stats.mismatch_count, head_dim, + state_kv_stats.mismatch_count, + (unsigned long long)state_count, + state_score_stats.mismatch_count, + (unsigned long long)state_count, + comp_stats.max_ulp, state_kv_stats.max_ulp, + state_score_stats.max_ulp); + TEST_ASSERT(comp_stats.mismatch_count == 0); + TEST_ASSERT(state_kv_stats.mismatch_count == 0); + TEST_ASSERT(state_score_stats.mismatch_count == 0); + + free(model_raw); + free(exact_comp_host); + free(ref_comp_host); + free(exact_state_score_host); + free(exact_state_kv_host); + free(ref_state_score_host); + free(ref_state_kv_host); + free(state_score_host); + free(state_kv_host); + free(sc_cur_host); + free(kv_cur_host); + ds4_gpu_tensor_free(exact_comp); + ds4_gpu_tensor_free(ref_comp); + ds4_gpu_tensor_free(exact_state_score); + ds4_gpu_tensor_free(exact_state_kv); + ds4_gpu_tensor_free(ref_state_score); + ds4_gpu_tensor_free(ref_state_kv); + ds4_gpu_tensor_free(sc_cur); + ds4_gpu_tensor_free(kv_cur); +} + +static void test_metal_compressor_ratio4_exact_pool_decode(void) { + test_metal_compressor_ratio4_exact_pool_decode_case(128u, 1u, 83u); + test_metal_compressor_ratio4_exact_pool_decode_case(512u, 0u, 89u); +} + static void test_metal_inplace_rope_pair_exact(void) { typedef struct { uint32_t head_dim; @@ -3836,6 +4385,194 @@ static void test_metal_hc_split_weighted_sum_norm_batch_exact(void) { free(model_raw); } +static void test_metal_hc_producer_pre_norm_compound_exact(void) { + const uint32_t n = 16384u; + const uint32_t mix_dim = 24u; + const uint32_t n_embd = 4096u; + const uint32_t n_hc = 4u; + const uint32_t sinkhorn_iters = 20u; + const float eps = 1.0e-6f; + const uint64_t page = (uint64_t)getpagesize(); + const uint64_t weight_offset = page; + const uint64_t weight_bytes = + (uint64_t)n * mix_dim * sizeof(uint16_t); + const uint64_t scale_offset = + test_round_up_u64(weight_offset + weight_bytes, page); + const uint64_t base_offset = + test_round_up_u64(scale_offset + 3u*sizeof(float), page); + const uint64_t norm_offset = + test_round_up_u64(base_offset + mix_dim*sizeof(float), page); + const uint64_t model_bytes = + test_round_up_u64(norm_offset + n_embd*sizeof(float), page); + const uint64_t residual_bytes = (uint64_t)n*sizeof(float); + const uint64_t mix_bytes = (uint64_t)mix_dim*sizeof(float); + const uint64_t out_bytes = (uint64_t)n_embd*sizeof(float); + + const char *force_env = + "DS4_METAL_ENABLE_HC_PRODUCER_PRE_NORM_FUSE"; + const char *disable_env = + "DS4_METAL_DISABLE_HC_PRODUCER_PRE_NORM_FUSE"; + const char *disable_pre_env = + "DS4_METAL_DISABLE_PRE_M5_HC_PRODUCER_PRE_NORM_FUSE"; + const char *disable_m5_env = + "DS4_METAL_DISABLE_M5_HC_PRODUCER_PRE_NORM_FUSE"; + const char *disable_ports_env = + "DS4_METAL_DISABLE_PRE_M5_DECODE_PORTS"; + char *saved_force = test_save_env(force_env); + char *saved_disable = test_save_env(disable_env); + char *saved_disable_pre = test_save_env(disable_pre_env); + char *saved_disable_m5 = test_save_env(disable_m5_env); + char *saved_disable_ports = test_save_env(disable_ports_env); + + void *model_raw = NULL; + float *residual_host = NULL; + float *ref_host = NULL; + float *fused_host = NULL; + ds4_gpu_tensor *residual = NULL; + ds4_gpu_tensor *ref_mix = NULL; + ds4_gpu_tensor *fused_mix = NULL; + ds4_gpu_tensor *ref_split = NULL; + ds4_gpu_tensor *fused_split = NULL; + ds4_gpu_tensor *ref_out = NULL; + ds4_gpu_tensor *fused_out = NULL; + ds4_gpu_tensor *ref_norm = NULL; + ds4_gpu_tensor *fused_norm = NULL; + + TEST_ASSERT(setenv(force_env, "1", 1) == 0); + TEST_ASSERT(unsetenv(disable_env) == 0); + TEST_ASSERT(unsetenv(disable_pre_env) == 0); + TEST_ASSERT(unsetenv(disable_m5_env) == 0); + TEST_ASSERT(unsetenv(disable_ports_env) == 0); + TEST_ASSERT(posix_memalign( + &model_raw, (size_t)page, (size_t)model_bytes) == 0); + if (!model_raw) goto cleanup; + memset(model_raw, 0, (size_t)model_bytes); + + uint16_t *weight = + (uint16_t *)((uint8_t *)model_raw + weight_offset); + float *hc_scale = (float *)((uint8_t *)model_raw + scale_offset); + float *hc_base = (float *)((uint8_t *)model_raw + base_offset); + float *norm_weight = (float *)((uint8_t *)model_raw + norm_offset); + for (uint32_t o = 0; o < mix_dim; o++) { + for (uint32_t i = 0; i < n; i++) { + const uint32_t key = i*37u + o*1009u + ((i >> 3u) ^ (o*19u)); + const int value = (int)(key % 127u) - 63; + weight[(uint64_t)o*n + i] = + test_float_to_f16((float)value/256.0f); + } + } + hc_scale[0] = 0.625f; + hc_scale[1] = -0.75f; + hc_scale[2] = 0.4375f; + for (uint32_t i = 0; i < mix_dim; i++) { + const int value = (int)((i*17u + 5u) % 29u) - 14; + hc_base[i] = (float)value/32.0f; + } + for (uint32_t i = 0; i < n_embd; i++) { + norm_weight[i] = 0.5f + (float)((i*13u + 7u) % 31u)/32.0f; + } + + residual_host = malloc((size_t)residual_bytes); + ref_host = malloc((size_t)out_bytes); + fused_host = malloc((size_t)out_bytes); + TEST_ASSERT(residual_host != NULL); + TEST_ASSERT(ref_host != NULL); + TEST_ASSERT(fused_host != NULL); + if (!residual_host || !ref_host || !fused_host) goto cleanup; + for (uint32_t i = 0; i < n; i++) { + const uint32_t key = i*131u + ((i >> 4u) ^ (i*7u)); + const int value = (int)(key % 4093u) - 2046; + residual_host[i] = (float)value/1024.0f; + } + + residual = ds4_gpu_tensor_alloc(residual_bytes); + ref_mix = ds4_gpu_tensor_alloc(mix_bytes); + fused_mix = ds4_gpu_tensor_alloc(mix_bytes); + ref_split = ds4_gpu_tensor_alloc(mix_bytes); + fused_split = ds4_gpu_tensor_alloc(mix_bytes); + ref_out = ds4_gpu_tensor_alloc(out_bytes); + fused_out = ds4_gpu_tensor_alloc(out_bytes); + ref_norm = ds4_gpu_tensor_alloc(out_bytes); + fused_norm = ds4_gpu_tensor_alloc(out_bytes); + TEST_ASSERT(residual && ref_mix && fused_mix && ref_split && fused_split && + ref_out && fused_out && ref_norm && fused_norm); + if (!residual || !ref_mix || !fused_mix || !ref_split || !fused_split || + !ref_out || !fused_out || !ref_norm || !fused_norm) { + goto cleanup; + } + + TEST_ASSERT(ds4_gpu_tensor_write( + residual, 0, residual_host, residual_bytes) != 0); + TEST_ASSERT(ds4_gpu_set_model_map(model_raw, model_bytes) != 0); + TEST_ASSERT(ds4_gpu_hc_rms_norm_mix_f16_tensor( + ref_mix, residual, model_raw, model_bytes, + weight_offset, n, mix_dim, eps) != 0); + TEST_ASSERT(ds4_gpu_hc_split_weighted_sum_norm_tensor( + ref_out, ref_norm, ref_split, ref_mix, residual, + model_raw, model_bytes, scale_offset, base_offset, + norm_offset, n_embd, n_hc, sinkhorn_iters, + eps, eps) != 0); + TEST_ASSERT(ds4_gpu_hc_rms_norm_mix_split_norm_f16_available() != 0); + TEST_ASSERT(ds4_gpu_hc_rms_norm_mix_split_norm_f16_tensor( + fused_mix, fused_out, fused_norm, fused_split, residual, + model_raw, model_bytes, weight_offset, scale_offset, + base_offset, norm_offset, n, mix_dim, n_embd, n_hc, + sinkhorn_iters, eps, eps, eps) > 0); + + test_float_compare_stats mix_stats = {0}; + test_float_compare_stats split_stats = {0}; + test_float_compare_stats out_stats = {0}; + test_float_compare_stats norm_stats = {0}; +#define TEST_HC_COMPOUND_COMPARE(ref_, fused_, bytes_, count_, stats_) do { \ + TEST_ASSERT(ds4_gpu_tensor_read((ref_), 0, ref_host, (bytes_)) != 0); \ + TEST_ASSERT(ds4_gpu_tensor_read((fused_), 0, fused_host, (bytes_)) != 0); \ + (stats_) = test_compare_float_bits(ref_host, fused_host, (count_)); \ + } while (0) + TEST_HC_COMPOUND_COMPARE( + ref_mix, fused_mix, mix_bytes, mix_dim, mix_stats); + TEST_HC_COMPOUND_COMPARE( + ref_split, fused_split, mix_bytes, mix_dim, split_stats); + TEST_HC_COMPOUND_COMPARE( + ref_out, fused_out, out_bytes, n_embd, out_stats); + TEST_HC_COMPOUND_COMPARE( + ref_norm, fused_norm, out_bytes, n_embd, norm_stats); +#undef TEST_HC_COMPOUND_COMPARE + + fprintf(stderr, + "ds4-test: HC producer/pre-norm compound exact " + "mix=%zu/%u split=%zu/%u collapse=%zu/%u norm=%zu/%u\n", + mix_stats.mismatch_count, mix_dim, + split_stats.mismatch_count, mix_dim, + out_stats.mismatch_count, n_embd, + norm_stats.mismatch_count, n_embd); + TEST_ASSERT(mix_stats.mismatch_count == 0 && mix_stats.max_ulp == 0); + TEST_ASSERT(split_stats.mismatch_count == 0 && split_stats.max_ulp == 0); + TEST_ASSERT(out_stats.mismatch_count == 0 && out_stats.max_ulp == 0); + TEST_ASSERT(norm_stats.mismatch_count == 0 && norm_stats.max_ulp == 0); + TEST_ASSERT(setenv(disable_env, "1", 1) == 0); + TEST_ASSERT(ds4_gpu_hc_rms_norm_mix_split_norm_f16_available() == 0); + +cleanup: + ds4_gpu_tensor_free(fused_norm); + ds4_gpu_tensor_free(ref_norm); + ds4_gpu_tensor_free(fused_out); + ds4_gpu_tensor_free(ref_out); + ds4_gpu_tensor_free(fused_split); + ds4_gpu_tensor_free(ref_split); + ds4_gpu_tensor_free(fused_mix); + ds4_gpu_tensor_free(ref_mix); + ds4_gpu_tensor_free(residual); + free(fused_host); + free(ref_host); + free(residual_host); + free(model_raw); + test_restore_env(disable_ports_env, saved_disable_ports); + test_restore_env(disable_m5_env, saved_disable_m5); + test_restore_env(disable_pre_env, saved_disable_pre); + test_restore_env(disable_env, saved_disable); + test_restore_env(force_env, saved_force); +} + static void test_metal_output_hc_weights4_exact(void) { const uint32_t n_hc = 4; const float eps = 1.0e-6f; @@ -4731,10 +5468,12 @@ static void test_metal_kernel_group(void) { test_metal_q8_0_decode_pair_exact(); #if defined(__APPLE__) test_metal_f16_compressor_pair_state_store_exact(); + test_metal_f16_compressor_quad_state_store_exact(); test_metal_compressor_ape_add_exact(); test_metal_compressor_ratio4_pack_exact(); test_metal_compressor_ratio4_replay_pack_exact(); test_metal_compressor_ratio4_direct_pool_exact(); + test_metal_compressor_ratio4_exact_pool_decode(); test_metal_inplace_rope_pair_exact(); test_metal_contiguous_f32_f16_roundtrip_exact(); test_metal_gathered_kv_stage_exact(); @@ -4742,6 +5481,7 @@ static void test_metal_kernel_group(void) { test_metal_persistent_zero_attention_mask_exact(); test_metal_zero_prefix_prefill_mask_cache_exact(); test_metal_hc_split_weighted_sum_norm_batch_exact(); + test_metal_hc_producer_pre_norm_compound_exact(); test_metal_output_hc_weights4_exact(); test_metal_hc_rms_scale_project_f16_exact(); test_metal_router_simd_finalize_exact(); diff --git a/tests/dspark_acceptance_fixture.sh b/tests/dspark_acceptance_fixture.sh index 7c9edd6b5..423e45721 100644 --- a/tests/dspark_acceptance_fixture.sh +++ b/tests/dspark_acceptance_fixture.sh @@ -28,10 +28,14 @@ SSD_STREAMING=${DS4_DSPARK_FIXTURE_SSD_STREAMING:-0} SSD_CACHE_EXPERTS=${DS4_DSPARK_FIXTURE_SSD_STREAMING_CACHE_EXPERTS:-} REQUIRE_ACTIVE=${DS4_DSPARK_FIXTURE_REQUIRE_ACTIVE:-1} REQUIRE_EXACT2=${DS4_DSPARK_FIXTURE_REQUIRE_EXACT2:-0} +REQUIRE_CUDA_EXACTN=${DS4_DSPARK_FIXTURE_REQUIRE_CUDA_EXACTN:-0} total_proposed=0 total_accepted_draft=0 total_exact2_attempt=0 total_exact2_fallback=0 +total_cuda_exactn_attempt=0 +total_cuda_exactn_fallback=0 +total_cuda_exactn_error_fallback=0 stats_field() { printf '%s\n' "$1" | awk -v key="$2" ' @@ -73,6 +77,13 @@ case "$REQUIRE_EXACT2" in exit 1 ;; esac +case "$REQUIRE_CUDA_EXACTN" in +0|1) ;; +*) + echo "dspark-fixture: DS4_DSPARK_FIXTURE_REQUIRE_CUDA_EXACTN must be 0 or 1" >&2 + exit 1 + ;; +esac case "$SSD_CACHE_EXPERTS" in ""|*[!0-9]*) if [ -n "$SSD_CACHE_EXPERTS" ]; then @@ -148,6 +159,13 @@ print_metadata() { confidence=${CONFIDENCE:-default} exact2_cuda=${DS4_CUDA_DSPARK_EXACT2:-unset} exact2_metal=${DS4_METAL_DSPARK_EXACT2:-unset} + exactn_cuda=${DS4_CUDA_DSPARK_EXACTN:-unset} + exactn_cuda_disable=${DS4_CUDA_DISABLE_DSPARK_EXACTN:-unset} + exactn_union_metal=${DS4_METAL_DSPARK_EXACTN_UNION:-unset} + noncausal_online_cuda=${DS4_CUDA_ENABLE_DSPARK_NONCAUSAL_ONLINE:-unset} + noncausal_online_cuda_disable=${DS4_CUDA_DISABLE_DSPARK_NONCAUSAL_ONLINE:-unset} + verify_noncausal=${DS4_DSPARK_VERIFY_NONCAUSAL:-unset} + exact_rows_async_tails_metal=${DS4_METAL_DSPARK_EXACT_ROWS_ASYNC_TAILS:-unset} proposer_cap_cuda=${DS4_CUDA_DSPARK_PROPOSER_BLOCK_MAX:-unset} proposer_cap_metal=${DS4_METAL_DSPARK_PROPOSER_BLOCK_MAX:-unset} verifier_cap=${DS4_DSPARK_SSD_VERIFY_BLOCK_MAX:-unset} @@ -174,6 +192,11 @@ print_metadata() { printf '# exact2_cuda=%s exact2_metal=%s proposer_block_max_cuda=%s proposer_block_max_metal=%s verifier_block_max=%s require_exact2=%s\n' \ "$exact2_cuda" "$exact2_metal" "$proposer_cap_cuda" \ "$proposer_cap_metal" "$verifier_cap" "$REQUIRE_EXACT2" + printf '# exactn_cuda=%s exactn_cuda_disable=%s require_cuda_exactn=%s exactn_union_metal=%s noncausal_online_cuda=%s noncausal_online_cuda_disable=%s verify_noncausal=%s exact_rows_async_tails_metal=%s\n' \ + "$exactn_cuda" "$exactn_cuda_disable" "$REQUIRE_CUDA_EXACTN" \ + "$exactn_union_metal" "$noncausal_online_cuda" \ + "$noncausal_online_cuda_disable" "$verify_noncausal" \ + "$exact_rows_async_tails_metal" } if [ ! -x "$DS4_BIN" ]; then @@ -271,6 +294,9 @@ run_case() { direct_partial=$(stats_field "$stats" direct_partial) exact2_attempt=$(stats_field "$stats" exact2_attempt) exact2_fallback=$(stats_field "$stats" exact2_fallback) + cuda_exactn_attempt=$(stats_field "$stats" cuda_exactn_attempt) + cuda_exactn_fallback=$(stats_field "$stats" cuda_exactn_fallback) + cuda_exactn_error_fallback=$(stats_field "$stats" cuda_exactn_error_fallback) partial=${partial:-0} errors=${errors:-0} verifier_unavailable=${verifier_unavailable:-0} @@ -280,6 +306,9 @@ run_case() { direct_partial=${direct_partial:-0} exact2_attempt=${exact2_attempt:-0} exact2_fallback=${exact2_fallback:-0} + cuda_exactn_attempt=${cuda_exactn_attempt:-0} + cuda_exactn_fallback=${cuda_exactn_fallback:-0} + cuda_exactn_error_fallback=${cuda_exactn_error_fallback:-0} if [ "$errors" -ne 0 ]; then echo "dspark-fixture: verifier errors for $id: $stats" >&2 return 1 @@ -292,10 +321,18 @@ run_case() { total_accepted_draft=$((total_accepted_draft + accepted_draft)) total_exact2_attempt=$((total_exact2_attempt + exact2_attempt)) total_exact2_fallback=$((total_exact2_fallback + exact2_fallback)) + total_cuda_exactn_attempt=$((total_cuda_exactn_attempt + cuda_exactn_attempt)) + total_cuda_exactn_fallback=$((total_cuda_exactn_fallback + cuda_exactn_fallback)) + total_cuda_exactn_error_fallback=$((total_cuda_exactn_error_fallback + cuda_exactn_error_fallback)) if [ "$REQUIRE_EXACT2" != 0 ] && [ "$exact2_fallback" -ne 0 ]; then echo "dspark-fixture: exact2 fallback for $id: $stats" >&2 return 1 fi + if [ "$REQUIRE_CUDA_EXACTN" != 0 ] && + [ "$cuda_exactn_error_fallback" -ne 0 ]; then + echo "dspark-fixture: CUDA exact-N error fallback for $id: $stats" >&2 + return 1 + fi if [ "$PROPOSAL_QUALITY_GUARD_ACTIVE" -ne 0 ] && [ "$id" = c_add ] && [ "$accepted_draft" -lt "$C_ADD_MIN_ACCEPTED" ]; then echo "dspark-fixture: c_add accepted_draft $accepted_draft below required $C_ADD_MIN_ACCEPTED: $stats" >&2 @@ -346,3 +383,18 @@ if [ "$REQUIRE_EXACT2" != 0 ] && [ "$total_exact2_fallback" -ne 0 ]; then echo "dspark-fixture: exact2 fallback count=$total_exact2_fallback" >&2 exit 1 fi +if [ "$REQUIRE_CUDA_EXACTN" != 0 ]; then + printf '# cuda_exactn_attempt=%s cuda_exactn_fallback=%s cuda_exactn_error_fallback=%s\n' \ + "$total_cuda_exactn_attempt" "$total_cuda_exactn_fallback" \ + "$total_cuda_exactn_error_fallback" +fi +if [ "$REQUIRE_CUDA_EXACTN" != 0 ] && + [ "$total_cuda_exactn_attempt" -eq 0 ]; then + echo "dspark-fixture: CUDA exact-N was required but never attempted" >&2 + exit 1 +fi +if [ "$REQUIRE_CUDA_EXACTN" != 0 ] && + [ "$total_cuda_exactn_error_fallback" -ne 0 ]; then + echo "dspark-fixture: CUDA exact-N error fallback count=$total_cuda_exactn_error_fallback" >&2 + exit 1 +fi From da17afc631e615caff4a307fda740106fd85ee98 Mon Sep 17 00:00:00 2001 From: Giorgio Oppo Date: Mon, 10 Aug 2026 16:00:24 +0200 Subject: [PATCH 015/189] Optimize speculative decode and GB10 CUDA kernels Port antirez/ds4#766 commit d99de5c while preserving the local Metal/CUDA DSpark exact-N, tiny-batch, and Q4 paths. Add fail-closed diagnostics, parity tests, and GB10 benchmark documentation. --- QA_BEFORE_RELEASES.md | 51 +- README.md | 171 +- cuda/mmq/ds4_mmq.cu | 223 ++- cuda/mmq/ds4_mmq.h | 11 + ds4.c | 1016 +++++++++- ds4_cuda.cu | 1754 +++++++++++++++++- ds4_gpu.h | 48 + ds4_metal.m | 493 ++++- metal/dsv4_misc.metal | 242 +++ speed-bench/ds4_gb10_q2_cuda_port_results.md | 324 ++++ speed-bench/gb10.csv | 64 +- tests/ds4_test.c | 1165 ++++++++++++ tests/dspark_acceptance_fixture.sh | 261 ++- tests/test_metal_exactn_oracle.c | 68 +- 14 files changed, 5545 insertions(+), 346 deletions(-) create mode 100644 speed-bench/ds4_gb10_q2_cuda_port_results.md diff --git a/QA_BEFORE_RELEASES.md b/QA_BEFORE_RELEASES.md index ba14e6207..33a87c687 100644 --- a/QA_BEFORE_RELEASES.md +++ b/QA_BEFORE_RELEASES.md @@ -310,6 +310,10 @@ than a failure. `--dspark-strict` remains the byte-identical target-only mode. The matrix must include full accepts for N=2,3,4,5, all N=5 partial prefixes 1..4, and EOS in the first and a middle row. This is a correctness gate, not evidence of a speedup. +- The Q8 Q-A/KV compound rows below require a separate AProjQ8 target whose + metadata includes both ratio-4 and ratio-128 compressor layers. An AProjQ4 + oracle cannot exercise that compound and is a failed coverage gate even if + greedy output remains correct. - Then run isolated, same-machine greedy A/B pairs with identical prompt, context, cache, token limit, and `DS4_DSPARK_STATS=1`. Change only the gate named by the row: @@ -321,6 +325,9 @@ than a failure. `--dspark-strict` remains the byte-identical target-only mode. | HC producer + split/Sinkhorn/destination RMSNorm on M1-M5 | `DS4_METAL_DISABLE_HC_PRODUCER_PRE_NORM_FUSE=1` | Leave the disable switch unset | | Q4 Q-A/KV + compressor store in exact-union | `DS4_METAL_DSPARK_EXACTN_UNION=1` with the Q4 enable switch unset | Keep exact-union `=1`; set `DS4_METAL_ENABLE_Q4_QKV_COMPRESSOR_FUSE=1` | | Q4 Q-A/KV + compressor store in ordinary `FULL` decode | Leave `DS4_METAL_ENABLE_Q4_QKV_COMPRESSOR_FUSE` unset | Set `DS4_METAL_ENABLE_Q4_QKV_COMPRESSOR_FUSE=1` | + | Q8 Q-A/KV + compressor store in SSD `FULL` (AProjQ8) | Leave the Q8 enable/require switches unset | Set `DS4_METAL_ENABLE_Q8_QKV_COMPRESSOR_FUSE=1 DS4_METAL_REQUIRE_Q8_QKV_COMPRESSOR_FUSE=1` | + | Q8 Q-A/KV + compressor store in SSD exact-union (AProjQ8) | `DS4_METAL_DSPARK_EXACTN_UNION=1` with the Q8 enable/require switches unset | Keep exact-union `=1`; set `DS4_METAL_ENABLE_Q8_QKV_COMPRESSOR_FUSE=1 DS4_METAL_REQUIRE_Q8_QKV_COMPRESSOR_FUSE=1` | + | Q4 attention-output tiny batch in the generic verifier | Set `DS4_METAL_DSPARK_EXACTN_UNION=0 DS4_METAL_DSPARK_EXACTN=0 DS4_METAL_DSPARK_EXACT2=0`; leave tiny enable/require unset | Keep all three exact gates `=0`; set `DS4_METAL_REQUIRE_Q4_ATTN_OUT_TINY_BATCH=1` and require at least one proposed block of depth 3–5 (the acceptance-only suffix evaluates one fewer row) | | F16 attention+indexer quad compressor store in `FULL` decode | `DS4_METAL_DISABLE_COMPRESSOR_QUAD_STORE=1` | Leave the disable switch unset | | F16 attention+indexer quad compressor store in exact-union | `DS4_METAL_DSPARK_EXACTN_UNION=1 DS4_METAL_DISABLE_COMPRESSOR_QUAD_STORE=1` | Keep exact-union `=1`; leave the quad disable switch unset | | Exact ratio-4 one-row compressor pool on M1-M5 | `DS4_METAL_DISABLE_COMPRESSOR_EXACT_POOL_RATIO4=1` | Leave the disable switch unset | @@ -432,14 +439,20 @@ than a failure. `--dspark-strict` remains the byte-identical target-only mode. `DS4_CUDA_DISABLE_HC_NORM_MIX_FUSE=1` versus candidate `DS4_CUDA_ENABLE_HC_NORM_MIX_FUSE=1 DS4_CUDA_NO_F16_CUBLAS_ONE=1`; and Q4 attention-output/HC control `DS4_CUDA_DISABLE_Q4_ATTN_OUT_HC_FUSE=1` versus - candidate `DS4_CUDA_ENABLE_Q4_ATTN_OUT_HC_FUSE=1`. Repeat the pair and - attention-output cases with `DS4_CUDA_MMQ=0` to exercise the Q8_K fallback - arithmetic separately from the default MMVQ/Q8_1 path. Require - byte-identical stdout and full-logit/tensor equivalence before promoting an - opt-in gate. Run with decode graphs both enabled and disabled, and record - target, proposer, verifier, replay, acceptance, and generation t/s. A CUDA - build and hardware run are mandatory; a host-only build does not compile - the device kernels. + MMVQ-safe candidate `DS4_CUDA_ENABLE_Q4_ATTN_OUT_HC_FUSE=1`. Run a separate + non-captured diagnostic with `DS4_CUDA_Q4_ATTN_OUT_HC_ORACLE=1`; require + the summary to be present with `calls>0`, `skips=0`, and + `epilogue_mismatches=0`, while `q8k_mismatches` records the expected + numerical distance from the optional one-dispatch Q8_K experiment. A + zero-call summary is a failed coverage gate. Only + test `DS4_CUDA_Q4_ATTN_OUT_HC_Q8K_EXPERIMENT=1` as a promotion candidate if + its oracle mismatches are also zero. Repeat the pair and attention-output + cases with `DS4_CUDA_MMQ=0` to exercise the canonical Q8_K fallback + separately from the default MMVQ/Q8_1 path. Require byte-identical stdout + and full-logit/tensor equivalence before promoting an opt-in gate. Run with + decode graphs both enabled and disabled, and record target, proposer, + verifier, replay, acceptance, and generation t/s. A CUDA build and hardware + run are mandatory; a host-only build does not compile the device kernels. - When DSpark, support-model mapping, or SSD streaming changes, repeat both the acceptance fixture and verifier invariant on every advertised graph backend. Apply the backend and SSD options to the target-only baseline as @@ -948,6 +961,28 @@ Do not use high-performance Hugging Face Xet mode while vLLM is resident. receiving explicit permission to use `192.168.60.250` for this QA pass. - Run: `make cuda-regression`. +- On a single GB10 (`sm_121`), validate the imported Q2 decode fast paths with + the AProjQ8/OutQ8 Flash GGUF. Compare the default against a rollback process + that sets all of: + `DS4_CUDA_NO_DIRECT_Q2_PREFILL=1`, + `DS4_CUDA_NO_F16_PAIR_COMPRESSOR_STORE=1`, + `DS4_CUDA_NO_F16_PAIR_COMPRESSOR_TRANSPOSE=1`, + `DS4_CUDA_NO_F16_PAIR_COMPRESSOR_TRANSPOSE_PREFETCH8=1`, + `DS4_CUDA_NO_Q8_FUSED_ALIGNED=1`, + `DS4_CUDA_NO_Q8_ALIGNED_PERSISTENT=1`, + `DS4_CUDA_NO_Q8_ALIGNED_DENSE_SCRATCH=1`, and + `DS4_CUDA_NO_HC_SPLIT_NORM_SPLIT4096=1`. Use separate processes, require + byte-identical greedy stdout and per-token logprobs, then run the same pair + under Compute Sanitizer. Record prefill, decode, and steady decode rather + than copying the upstream PR numbers into a release claim. +- Repeat the GB10 comparison with AProjQ4/OutQ8. The persistent vocabulary, + compressor, HC split, scratch, and direct routed-MoE paths remain relevant, + while Q8 attention-projection consumers are intentionally ineligible. +- Exercise CUDA DSpark at verifier/proposer depth 5 with the fast paths enabled + and disabled. Require identical final output, zero verifier errors, and + matching full/partial acceptance histograms. Test both the generic batch + verifier (direct Q2 path) and CUDA exact-N (one-row decode paths); do not + infer speculative speedup from the target-only benchmark. - For native MXFP4 changes, run `make test-mxfp4-cuda CUDA_ARCH=native` on the multi-GPU CUDA host only after receiving explicit permission for `192.168.60.250`, and diff --git a/README.md b/README.md index eab6eb9c6..d0c2e6235 100644 --- a/README.md +++ b/README.md @@ -481,8 +481,15 @@ ordinary decode and therefore still requires rollback plus exact replay. `DS4_METAL_DSPARK_EXACTN_UNION=1` enables a separate experimental Metal SSD verifier for two through five draft tokens. It executes canonical one-row target decode in layer order, loads the union of the rows' routed experts once -per layer, and commits directly only after a full accept; partial accepts keep -the established rollback/replay fallback. The model-backed +per layer, and commits its verifier state directly after a full accept. On a +partial match it restores the frontier once, skips the boundary oracle, +exact-two path, and legacy token-by-token verifier, then exactly replays only +the already verified prefix. With `DS4_DSPARK_STATS=1`, +`exactn_union_partial_replay` and `exactn_union_verify_skip` should advance +together; `exactn_union_partial_replay_ms` isolates the required commit replay. +Set `DS4_DSPARK_FIXTURE_REQUIRE_METAL_EXACTN_PARTIAL=1` to require at least one +such partial match, equal replay/skip counts, no exact-union error fallback, +and byte-identical fixture output. The model-backed `test-metal-exactn-oracle` is byte-identical to sequential decode for N=2..5, including every N=5 partial prefix, EOS in the first or a middle row, serialized KV/compressor state, logits, and a four-token continuation. Five drafts plus @@ -490,6 +497,51 @@ the target token already available at the start of the cycle cover the six-token speculative-cycle limit. Exact-union remains opt-in: correctness does not imply a throughput improvement on a particular memory configuration. +For an independent Q8 output-head A/B inside exact-union, set +`DS4_METAL_DSPARK_EXACTN_BATCH_HEAD=1`. HC collapse and normalization remain on +the canonical one-row kernels, while one bit-exact decode-row dispatch projects +all two through five verifier rows to vocabulary logits. Non-Q8 output weights +are ineligible and a dispatch failure falls back to the ordinary per-row heads. +`DS4_METAL_DISABLE_DSPARK_EXACTN_BATCH_HEAD=1` is the unconditional kill switch +and wins if both variables are set. The +`metal_exactn_batch_head_attempt`, `metal_exactn_batch_head_use`, and +`metal_exactn_batch_head_fallback` counters identify the selected path; set +`DS4_DSPARK_FIXTURE_REQUIRE_METAL_EXACTN_BATCH_HEAD=1` to require a nonzero, +fallback-free use with byte-identical output. + +The generic model-backed exact-N oracle keeps this Q8-only experiment disabled, +so it remains valid for target models with another output quantization. To add +model-backed batch-head coverage, use an OutQ8 target explicitly: + +```sh +DS4_TEST_METAL_EXACTN_BATCH_HEAD=1 \ +DS4_TEST_MODEL=/path/to/target-OutQ8.gguf \ +make test-metal-exactn-oracle +``` + +The Metal proposer also has an independent experiment for confidence/Markov +synchronization overhead. Set `DS4_METAL_DSPARK_DEVICE_PROPOSER=1` when the +final confidence projection and both Markov matrices are Q8_0. On an eligible +single-device, tier-zero run it keeps the previous token, confidence decisions, +and Markov argmax chain on Metal, reuses the first confidence already computed +by the proposer, and returns one result for the complete draft block. Unlike +the CUDA experiment, the Metal path is eligible with SSD streaming; tensor +placement and proposal-quality mode remain excluded. It stops at the first +rejected confidence row and preserves the smaller-token argmax tie break. +Unsupported layouts, an incomplete result, or a CPU sigmoid-policy mismatch +fall back to the existing per-row implementation. + +`DS4_METAL_DSPARK_NO_DEVICE_PROPOSER=1` is the unconditional kill switch; +`DS4_DSPARK_NO_GPU_MARKOV=1` and `DS4_DSPARK_NO_MARKOV=1` also keep the path +disabled. This remains opt-in because Q8 confidence accumulation moves from the +host CPU to Metal and therefore needs a same-machine greedy oracle and A/B. +With `DS4_DSPARK_STATS=1`, require +`metal_device_proposer_attempt == metal_device_proposer_use > 0`, +`metal_device_proposer_fallback=0`, and +`metal_device_proposer_policy_mismatch=0`. The acceptance fixture enforces +those conditions and byte-identical output with +`DS4_DSPARK_FIXTURE_REQUIRE_METAL_DEVICE_PROPOSER=1`. + Exact-union normally waits for every layer's routed-tail command buffer before releasing its private expert-address scope. For an isolated A/B, `DS4_METAL_DSPARK_EXACT_ROWS_ASYNC_TAILS=1` commits that tail without the CPU @@ -516,6 +568,34 @@ this verifier: - The eligible Q4 attention-output B projection can perform the following HC expansion in the same dispatch. Use `DS4_METAL_DISABLE_Q4_ATTN_OUT_HC_FUSE=1` as its isolated A/B control. +- For an AProjQ4 multi-row attention-output batch with either attention B in + Q8 or Q4_K, the opt-in + `DS4_METAL_ENABLE_Q4_ATTN_OUT_TINY_BATCH=1` evaluates two through five rows + in two dispatches while retaining the canonical one-row reduction order for + every row. This covers the generic suffix verifier; the exact-union tape is + intentionally still row-by-row and does not select this helper. Other + output formats, unsupported shapes, a disabled Q4 classic matvec, or a + dispatch setup failure return to the existing row-wise path. + `DS4_METAL_DISABLE_Q4_ATTN_OUT_TINY_BATCH=1` is the unconditional kill + switch and wins when both variables are set. For a fail-closed model-backed + generic-verifier test, `DS4_METAL_REQUIRE_Q4_ATTN_OUT_TINY_BATCH=1` implies + the enable gate for N=2..5 and turns an ineligible shape, the kill switch, or + a dispatch failure into a hard error instead of a silent row-wise fallback. + +The AProjQ8 Q-A/KV plus compressor compound remains the M1-M5 default for +eligible resident `FULL` decode. For an SSD-streaming A/B, including the +exact-union `TO_ROUTER` collection prefix, set +`DS4_METAL_ENABLE_Q8_QKV_COMPRESSOR_FUSE=1`. Ratio-4 layers combine the Q8 +Q-A/KV pair with both attention and indexer F16 compressor pairs; ratio-128 +layers combine it with the attention pair. The kernel preserves the canonical +NSG=4 Q8 and NR0=2 F16 reduction trees. A diagnostic Q8 NSG override or the +experimental NR0=4 compressor schedule therefore selects the separate +dispatches. `DS4_METAL_REQUIRE_Q8_QKV_COMPRESSOR_FUSE=1` turns such a fallback +into a visible error for model-backed tests. The existing ratio-specific +pre-M5/M5 QKV compound disable variables remain authoritative. Keep the SSD +extension opt-in until warm and cold A/B runs show a gain: it removes a launch +per row and layer but reads the same model bytes, and a compound grid can +change the order in which distant GGUF pages are faulted. Additional PR #755 ports keep their established kernels as shape/resource fallbacks: @@ -620,10 +700,24 @@ reduction order; with the normal one-token cuBLAS path, also set `DS4_CUDA_NO_F16_CUBLAS_ONE=1` to exercise it. The controls are `DS4_CUDA_DISABLE_HC_NORM_MIX_FUSE=1` and `DS4_CUDA_DISABLE_Q4_ATTN_OUT_HC_FUSE=1`. The Q4 attention-output B plus HC -expansion is automatic only when MMQ is disabled, where it preserves the -existing Q8_K activation quantizer. With the normal MMVQ/Q8_1 decode path it -requires the explicit `DS4_CUDA_ENABLE_Q4_ATTN_OUT_HC_FUSE=1` experiment and -may not be numerically identical until validated on CUDA hardware. +path is automatic when MMQ is disabled, where the existing one-dispatch Q8_K +implementation is bit-compatible with its fallback. With the normal +MMVQ/Q8_1 decode path, `DS4_CUDA_ENABLE_Q4_ATTN_OUT_HC_FUSE=1` now preserves +the canonical MMVQ projection and replaces only the following HC expansion +with a row-packed epilogue that reads each projected row once. It therefore +does not change activation quantization. The older, truly single-dispatch +Q8_K experiment is isolated behind +`DS4_CUDA_Q4_ATTN_OUT_HC_Q8K_EXPERIMENT=1` and may differ numerically from +MMVQ/Q8_1. + +For a fail-closed hardware comparison, set +`DS4_CUDA_Q4_ATTN_OUT_HC_ORACLE=1`. It retains the canonical MMVQ/Q8_1 output, +compares both the row-packed epilogue and the Q8_K compound bit-for-bit, and +prints `epilogue_mismatches`, `q8k_mismatches`, and `skips` at exit. The +oracle avoids readback while CUDA graph capture is active; run a separate +non-captured diagnostic and require `calls>0`, `skips=0`, and +`epilogue_mismatches=0` before promoting the row-packed path. A zero-call +summary is therefore an explicit failed coverage gate, not a silent pass. An experimental resident-CUDA path can run the existing aligned IQ2_XXS/Q2_K vector MoE kernels for two-to-five-draft routed batches, @@ -683,13 +777,15 @@ one-token tape to two through five draft rows. It leaves hidden rows and all target weights on one GPU, submits the per-row ordinary decode kernels in one stream, and reads back only the `N-1` acceptance ids plus final logits. A full match therefore commits its already-exact KV/compressor state without replay; -a partial match or backend error restores the pre-cycle frontier and uses the -legacy verifier/replay fallback. Enable it independently with: +a partial match restores the pre-cycle frontier once and replays the prefix +already proven by exact-N, without running the legacy verifier a second time. +Only a backend error retains the legacy verifier/replay fallback. Enable it +independently with: ```sh DS4_CUDA_DSPARK_EXACTN=1 DS4_DSPARK_STATS=1 \ ./ds4 --cuda -m ds4flash.gguf \ - --mtp gguf/DeepSeek-V4-Flash-DSpark-support-0731.gguf \ + --mtp-model gguf/DeepSeek-V4-Flash-DSpark-support-0731.gguf \ --dspark --temp 0 -p 'Write a Python quicksort function with comments.' ``` @@ -704,11 +800,54 @@ variable. Track `cuda_exactn_attempt`, `cuda_exactn_full`, this experiment disabled by default until a real CUDA oracle and a long greedy A/B show byte-identical output, no fallback errors, and a throughput win. +For a separate output-head A/B, set +`DS4_CUDA_DSPARK_EXACTN_BATCH_HEAD=1`. The experiment keeps HC collapse and +normalization on the canonical one-row kernels, then runs the Q8 vocabulary +projection for all exact-N rows through the bit-exact decode-row kernel. It is +automatically ineligible for non-Q8 output weights and falls back to the +ordinary per-row heads on a dispatch failure. The emergency kill switch is +`DS4_CUDA_DISABLE_DSPARK_EXACTN_BATCH_HEAD=1` and wins when both variables are +present. + +The CUDA proposer tail has a second, independent experiment for the fixed +confidence/Markov synchronization overhead: + +```sh +DS4_CUDA_DSPARK_DEVICE_PROPOSER=1 +``` + +When the final confidence projection and both Markov matrices are Q8_0, this +keeps the previous token, confidence decisions, and all Markov argmax steps on +the decode stream and reads one 64-byte result for the whole draft block. It +stops at the first rejected confidence row, preserves the Markov smaller-token +tie break, and rechecks the returned confidence prefix with the established +CPU sigmoid policy. Unsupported layouts, an incomplete result, or a policy +mismatch fall back to the per-row implementation. The unconditional kill +switch is `DS4_CUDA_DSPARK_NO_DEVICE_PROPOSER=1`; the older +`DS4_DSPARK_NO_GPU_MARKOV` switch also keeps this path disabled. +The initial gate is intentionally limited to resident, single-GPU, non-quality +CUDA and reuses the already-computed first confidence value. + +This remains opt-in because the Q8 confidence accumulation moves from the host +CPU to CUDA and must pass the DGX proposal/acceptance oracle before promotion. +With `DS4_DSPARK_STATS=1`, require +`cuda_device_proposer_attempt == cuda_device_proposer_use > 0`, +`cuda_device_proposer_fallback=0`, and +`cuda_device_proposer_policy_mismatch=0`. The acceptance fixture can enforce +those conditions with +`DS4_DSPARK_FIXTURE_REQUIRE_CUDA_DEVICE_PROPOSER=1`. + +The stats line separates `cuda_exactn_ms` into setup, layer, head, and read +components, and reports restore, legacy-error-fallback verification, and +partial replay time independently. `cuda_exactn_partial_replay` and +`cuda_exactn_verify_skip` should advance together on valid partial matches; +the batch-head attempt/use/fallback counters make its dispatch unambiguous. + Set `DS4_DSPARK_FIXTURE_REQUIRE_CUDA_EXACTN=1` on the candidate acceptance fixture to require at least one `cuda_exactn_attempt` and zero `cuda_exactn_error_fallback`. The aggregate `cuda_exactn_fallback` is reported but is not required to be zero: it also includes valid partial draft matches, -which deliberately restore the frontier and use legacy replay. +which deliberately restore the frontier and use exact replay. The acceptance fixture can exercise the same SSD path on both the target-only baseline and the DSpark run. It also requires real proposals and accepted @@ -733,7 +872,9 @@ The current q2 results use `ds4-bench` with the standard *Promessi sposi* input, 2048-token context steps, and 128 greedy generation tokens at every frontier. Each prefill number is for the next 2048-token chunk. The complete sweeps are in [m5_max.csv](speed-bench/m5_max.csv) and -[gb10.csv](speed-bench/gb10.csv). +[gb10.csv](speed-bench/gb10.csv). The GB10 optimization methodology and +validation are documented in +[ds4_gb10_q2_cuda_port_results.md](speed-bench/ds4_gb10_q2_cuda_port_results.md). | Machine | Backend | Context | Prefill | Generation | | --- | --- | ---: | ---: | ---: | @@ -741,10 +882,10 @@ sweeps are in [m5_max.csv](speed-bench/m5_max.csv) and | MacBook Pro M5 Max, 128 GB | Metal | 16384 | 572.53 t/s | 36.14 t/s | | MacBook Pro M5 Max, 128 GB | Metal | 32768 | 557.04 t/s | 34.36 t/s | | MacBook Pro M5 Max, 128 GB | Metal | 65536 | 398.50 t/s | 27.64 t/s | -| DGX Spark GB10, 128 GB | CUDA | 2048 | 825.76 t/s | 18.05 t/s | -| DGX Spark GB10, 128 GB | CUDA | 16384 | 872.44 t/s | 15.10 t/s | -| DGX Spark GB10, 128 GB | CUDA | 32768 | 855.94 t/s | 14.43 t/s | -| DGX Spark GB10, 128 GB | CUDA | 65536 | 822.98 t/s | 13.84 t/s | +| DGX Spark GB10, 128 GB | CUDA | 2048 | 832.86 t/s | 20.58 t/s | +| DGX Spark GB10, 128 GB | CUDA | 16384 | 883.81 t/s | 16.80 t/s | +| DGX Spark GB10, 128 GB | CUDA | 32768 | 865.40 t/s | 15.99 t/s | +| DGX Spark GB10, 128 GB | CUDA | 65536 | 833.44 t/s | 15.27 t/s | Older measurements for machines and model variants not rerun in this pass are kept for reference. They used the earlier CLI prompt procedure and are not diff --git a/cuda/mmq/ds4_mmq.cu b/cuda/mmq/ds4_mmq.cu index f4e3f61bb..c6c7b4842 100644 --- a/cuda/mmq/ds4_mmq.cu +++ b/cuda/mmq/ds4_mmq.cu @@ -2135,11 +2135,30 @@ extern "C" int ds4_mmq_iq2_xxs_q2_K_moe_fused_direct_soa( !input_q8_scratch || input_q8_scratch_bytes == 0 || !down_q8_scratch || down_q8_scratch_bytes == 0 || !work_scratch || work_scratch_bytes == 0 || !down) { - return -1; - } - const size_t down_bytes = - (size_t)n_tokens * (size_t)n_expert_used * - (size_t)out_dim * sizeof(float); + return DS4_MMQ_NOT_APPLICABLE; + } + const size_t nt = (size_t)n_tokens; + const size_t nu = (size_t)n_expert_used; + const size_t od = (size_t)out_dim; + if (nt > SIZE_MAX / nu || nt * nu > SIZE_MAX / od || + nt * nu * od > SIZE_MAX / sizeof(float)) { + return DS4_MMQ_NOT_APPLICABLE; + } + const size_t assignments = nt * nu; + const size_t expert_in = (size_t)expert_in_dim; + const size_t expert_mid = (size_t)expert_mid_dim; + if (assignments > SIZE_MAX / expert_in || + assignments > SIZE_MAX / expert_mid) { + return DS4_MMQ_NOT_APPLICABLE; + } + /* The internal MMQ producer sizes multiply these logical element counts + * by block structs before dividing by their values-per-block. Keep ample + * headroom for that multiplication and its fixed tail allocation. */ + if (assignments * expert_in > SIZE_MAX / 512u || + assignments * expert_mid > SIZE_MAX / 512u) { + return DS4_MMQ_NOT_APPLICABLE; + } + const size_t down_bytes = assignments * od * sizeof(float); if (ds4_mmq_scratch_overlaps( input_q8_scratch, input_q8_scratch_bytes, down_q8_scratch, down_q8_scratch_bytes) || @@ -2155,12 +2174,20 @@ extern "C" int ds4_mmq_iq2_xxs_q2_K_moe_fused_direct_soa( down_q8_scratch, down_q8_scratch_bytes, down, down_bytes) || ds4_mmq_scratch_overlaps( work_scratch, work_scratch_bytes, down, down_bytes)) { - return -1; + return DS4_MMQ_NOT_APPLICABLE; + } + const int64_t iq2_k_blocks = expert_in_dim / 256; + const int64_t q2_k_blocks = expert_mid_dim / 256; + if ((int64_t)n_experts > INT64_MAX / expert_mid_dim || + (int64_t)n_experts * expert_mid_dim > INT64_MAX / iq2_k_blocks || + (int64_t)n_experts > INT64_MAX / (out_dim / 2) || + (int64_t)n_experts * (out_dim / 2) > INT64_MAX / q2_k_blocks) { + return DS4_MMQ_NOT_APPLICABLE; } const int64_t iq2_blocks = - (int64_t)n_experts * expert_mid_dim * (expert_in_dim / 256); + (int64_t)n_experts * expert_mid_dim * iq2_k_blocks; const int64_t q2_pairs = - (int64_t)n_experts * (out_dim / 2) * (expert_mid_dim / 256); + (int64_t)n_experts * (out_dim / 2) * q2_k_blocks; const ds4_mmq_fused_down fused_down = { W_down, (const char *)W_down, @@ -2180,13 +2207,17 @@ extern "C" int ds4_mmq_iq2_xxs_q2_K_moe_fused_direct_soa( input_q8_ext, input_q8_ext_bytes, }; - return ds4_mmq_moe_pair_impl( + const int rc = ds4_mmq_moe_pair_impl( "ds4_mmq_iq2_xxs_q2_K_moe_fused_direct_soa", W_gate, W_up, X, ids, nullptr, nullptr, expert_mid_dim, expert_in_dim, n_tokens, n_experts, n_expert_used, stream, (const char *)W_gate, (const char *)W_up, iq2_blocks, /*sanitize_out=*/false, &fused_down); + if (rc == -1 || (rc <= -91 && rc >= -97)) { + return DS4_MMQ_NOT_APPLICABLE; + } + return rc; } extern "C" int ds4_mmq_q4_K_moe_pair( @@ -4062,6 +4093,12 @@ extern "C" int ds4_mmq_iq2_xxs_aligned_derepack( return 0; } +static int g_gb10_optimizations = 0; + +extern "C" void ds4_mmq_set_gb10_optimizations(int enabled) { + g_gb10_optimizations = enabled != 0; +} + // --------------------------------------------------------------------------- // Aligned-SoA Q8_0 dense decode matvec (megakernel program M1-Inc3). // @@ -4114,6 +4151,96 @@ __global__ void q8_0_aligned_dense_vec_kernel( if (lane == 0) out[row] = acc; } +/* K=1024 decode specialization. Eight persistent row warps per CTA hoist the + * 32 Q8_1 activation blocks into registers and walk output rows at a grid + * stride. Each lane still owns the same single block term and the warp tree is + * unchanged, so output bits match q8_0_aligned_dense_vec_kernel. */ +__global__ __launch_bounds__(256, 6) void q8_0_aligned_dense_vec_k1024_persistent_kernel( + float *out, + const int4 *qs, + const __half *dq, + const block_q8_1 *x8, + int M) +{ + const int lane = threadIdx.x & 31; + const int warp = threadIdx.x >> 5; + const int *u = (const int *)x8[lane].qs; + const int u0 = u[0]; + const int u1 = u[1]; + const int u2 = u[2]; + const int u3 = u[3]; + const int u4 = u[4]; + const int u5 = u[5]; + const int u6 = u[6]; + const int u7 = u[7]; + const float dx = __low2float(x8[lane].ds); + const int64_t row0 = (int64_t)blockIdx.x * 8 + warp; + const int64_t row_stride = (int64_t)gridDim.x * 8; + + for (int64_t row = row0; row < (int64_t)M; row += row_stride) { + const long long block = (long long)row * 32 + lane; + const int4 w0 = qs[block * 2 + 0]; + const int4 w1 = qs[block * 2 + 1]; + int s0 = ggml_cuda_dp4a(w0.x, u0, 0); + s0 = ggml_cuda_dp4a(w0.y, u1, s0); + int s1 = ggml_cuda_dp4a(w0.z, u2, 0); + s1 = ggml_cuda_dp4a(w0.w, u3, s1); + int s2 = ggml_cuda_dp4a(w1.x, u4, 0); + s2 = ggml_cuda_dp4a(w1.y, u5, s2); + int s3 = ggml_cuda_dp4a(w1.z, u6, 0); + s3 = ggml_cuda_dp4a(w1.w, u7, s3); + const int sumi = (s0 + s1) + (s2 + s3); + float acc = 0.0f; + acc += __half2float(dq[block]) * dx * (float)sumi; +#pragma unroll + for (int off = 16; off > 0; off >>= 1) + acc += __shfl_down_sync(0xffffffffu, acc, off); + if (lane == 0) out[row] = acc; + } +} + +/* Persistent-CTA form for the K=4096 vocabulary projection. It preserves + * the original lane/block assignment, per-lane term order, and warp tree; + * grouping eight row warps removes the one-warp CTA occupancy ceiling. */ +__global__ __launch_bounds__(256, 6) void q8_0_aligned_dense_vec_persistent_kernel( + float *out, + const int4 *qs, + const __half *dq, + const block_q8_1 *x8, + int M, + int nb) +{ + const int lane = threadIdx.x & 31; + const int warp = threadIdx.x >> 5; + const int64_t row0 = (int64_t)blockIdx.x * 8 + warp; + const int64_t row_stride = (int64_t)gridDim.x * 8; + for (int64_t row = row0; row < (int64_t)M; row += row_stride) { + const long long rbase = (long long)row * nb; + float acc = 0.0f; + for (int b0 = 0; b0 < nb; b0 += 32) { + const int b = b0 + lane; + const int4 w0 = qs[(rbase + b) * 2 + 0]; + const int4 w1 = qs[(rbase + b) * 2 + 1]; + const int *u = (const int *)x8[b].qs; + int sumi = 0; + sumi = ggml_cuda_dp4a(w0.x, u[0], sumi); + sumi = ggml_cuda_dp4a(w0.y, u[1], sumi); + sumi = ggml_cuda_dp4a(w0.z, u[2], sumi); + sumi = ggml_cuda_dp4a(w0.w, u[3], sumi); + sumi = ggml_cuda_dp4a(w1.x, u[4], sumi); + sumi = ggml_cuda_dp4a(w1.y, u[5], sumi); + sumi = ggml_cuda_dp4a(w1.z, u[6], sumi); + sumi = ggml_cuda_dp4a(w1.w, u[7], sumi); + acc += __half2float(dq[rbase + b]) * + __low2float(x8[b].ds) * (float)sumi; + } +#pragma unroll + for (int off = 16; off > 0; off >>= 1) + acc += __shfl_down_sync(0xffffffffu, acc, off); + if (lane == 0) out[row] = acc; + } +} + // Verify-width variant (v0.4 dense chase, proto_q8_aligned_nc): same aligned // weight stream read ONCE per row, NC output columns accumulated per lane // against col-strided q8_1 activations (which L1/L2-broadcast across rows). @@ -4335,7 +4462,11 @@ extern "C" int ds4_mmq_q8_0_aligned_dense_vec( char *x8 = N == 1 ? ds4_mmq_folded_q81(X_f32, K, 1, ne10_padded) : NULL; cudaError_t err; if (!x8) { - if (g_q81_scratch_enabled && g_q81_scratch_ptr && g_q81_scratch_bytes >= nbytes_q8_1) { + if (getenv("DS4_CUDA_NO_Q8_ALIGNED_DENSE_SCRATCH") == NULL && + g_aligned_q81_scratch_ptr && + g_aligned_q81_scratch_bytes >= nbytes_q8_1) { + x8 = (char *)g_aligned_q81_scratch_ptr; + } else if (g_q81_scratch_enabled && g_q81_scratch_ptr && g_q81_scratch_bytes >= nbytes_q8_1) { x8 = (char *)g_q81_scratch_ptr; } else { q8_pool.alloc(ctx->pool(), nbytes_q8_1); @@ -4361,33 +4492,51 @@ extern "C" int ds4_mmq_q8_0_aligned_dense_vec( const block_q8_1 *x8p = (const block_q8_1 *)x8; switch (N) { case 1: - switch (ds4_q8_aligned_warps_per_block( - ggml_cuda_info().devices[dev].cc)) { - case 16: - q8_0_aligned_dense_vec_kernel<16> - <<<((unsigned)M + 15u) / 16u, 512, 0, stream>>>( - out_f32, qsp, dqp, x8p, M, K / 32); - break; - case 8: - q8_0_aligned_dense_vec_kernel<8> - <<<((unsigned)M + 7u) / 8u, 256, 0, stream>>>( - out_f32, qsp, dqp, x8p, M, K / 32); - break; - case 4: - q8_0_aligned_dense_vec_kernel<4> - <<<((unsigned)M + 3u) / 4u, 128, 0, stream>>>( - out_f32, qsp, dqp, x8p, M, K / 32); - break; - case 2: - q8_0_aligned_dense_vec_kernel<2> - <<<((unsigned)M + 1u) / 2u, 64, 0, stream>>>( - out_f32, qsp, dqp, x8p, M, K / 32); - break; - default: - q8_0_aligned_dense_vec_kernel<1> - <<<(unsigned)M, 32, 0, stream>>>( - out_f32, qsp, dqp, x8p, M, K / 32); - break; + if (g_gb10_optimizations && + getenv("DS4_CUDA_NO_Q8_ALIGNED_PERSISTENT") == NULL && + (K == 1024 || K == 4096) && M >= 32768) { + const uint64_t row_blocks = ((uint64_t)(unsigned)M + 7u) / 8u; + /* 288 = 48 GB10 SMs x __launch_bounds__(256, 6) resident CTAs. */ + const unsigned persistent_blocks = + row_blocks < 288u ? (unsigned)row_blocks : 288u; + if (K == 1024) { + q8_0_aligned_dense_vec_k1024_persistent_kernel<<< + persistent_blocks, 256, 0, stream>>>( + out_f32, qsp, dqp, x8p, M); + } else { + q8_0_aligned_dense_vec_persistent_kernel<<< + persistent_blocks, 256, 0, stream>>>( + out_f32, qsp, dqp, x8p, M, K / 32); + } + } else { + switch (ds4_q8_aligned_warps_per_block( + ggml_cuda_info().devices[dev].cc)) { + case 16: + q8_0_aligned_dense_vec_kernel<16> + <<<((unsigned)M + 15u) / 16u, 512, 0, stream>>>( + out_f32, qsp, dqp, x8p, M, K / 32); + break; + case 8: + q8_0_aligned_dense_vec_kernel<8> + <<<((unsigned)M + 7u) / 8u, 256, 0, stream>>>( + out_f32, qsp, dqp, x8p, M, K / 32); + break; + case 4: + q8_0_aligned_dense_vec_kernel<4> + <<<((unsigned)M + 3u) / 4u, 128, 0, stream>>>( + out_f32, qsp, dqp, x8p, M, K / 32); + break; + case 2: + q8_0_aligned_dense_vec_kernel<2> + <<<((unsigned)M + 1u) / 2u, 64, 0, stream>>>( + out_f32, qsp, dqp, x8p, M, K / 32); + break; + default: + q8_0_aligned_dense_vec_kernel<1> + <<<(unsigned)M, 32, 0, stream>>>( + out_f32, qsp, dqp, x8p, M, K / 32); + break; + } } break; case 2: q8_0_aligned_dense_vec_nc_kernel<2><<<(unsigned)M, 32, 0, stream>>>(out_f32, qsp, dqp, x8p, M, K / 32); break; diff --git a/cuda/mmq/ds4_mmq.h b/cuda/mmq/ds4_mmq.h index ed4c19f35..7c3247dc3 100644 --- a/cuda/mmq/ds4_mmq.h +++ b/cuda/mmq/ds4_mmq.h @@ -344,6 +344,12 @@ int ds4_mmq_iq2_xxs_q2_K_moe_fused_soa( float clamp, cudaStream_t stream); +/* Optional fused entries return this before enqueueing work when their + * capability/shape/scratch preflight cannot engage. Callers may safely retry + * a materialized fallback only for this result; zero is success and negative + * values may follow partial enqueue. */ +#define DS4_MMQ_NOT_APPLICABLE 1 + /* Aligned-artifact production fast path: gate/up stay in registers, weighted * SwiGLU is quantized directly into down_q8_scratch, and only the pair-major * down output is materialized. Caller-owned scratch keeps this hot path free @@ -578,6 +584,11 @@ int ds4_mmq_q2_K_aligned_derepack( // return non-zero so the caller can fall back to ds4_mmq_q8_0_dense_vec. uint64_t ds4_mmq_q8_0_aligned_bytes(int M, int K); +// Enable decode shapes validated on integrated sm_121 (GB10). The CUDA +// backend sets this after device discovery; other devices retain the generic +// aligned kernel. DS4_CUDA_NO_Q8_ALIGNED_PERSISTENT is the runtime rollback. +void ds4_mmq_set_gb10_optimizations(int enabled); + int ds4_mmq_q8_0_aligned_dense_vec( const void * W_aligned, const float * X_f32, diff --git a/ds4.c b/ds4.c index 129b691a2..6cd8f88cb 100644 --- a/ds4.c +++ b/ds4.c @@ -16087,6 +16087,10 @@ typedef struct { ds4_gpu_tensor *dspark_stage0_proj; ds4_gpu_tensor *dspark_main_x; ds4_gpu_tensor *dspark_draft_tokens; + /* Device-resident confidence/Markov returns one compact result for + * the whole speculative block. Keep it separate from draft_tokens: that + * buffer is only block_size * i32 and is too small for the 64-byte ABI. */ + ds4_gpu_tensor *dspark_device_proposal; ds4_gpu_tensor *dspark_draft_hc; ds4_gpu_tensor *dspark_target_hc; ds4_gpu_tensor *dspark_stage_input_hc; @@ -16850,6 +16854,7 @@ static void metal_graph_free(ds4_gpu_graph *g) { ds4_gpu_tensor_free(g->dspark_stage_input_hc); ds4_gpu_tensor_free(g->dspark_target_hc); ds4_gpu_tensor_free(g->dspark_draft_hc); + ds4_gpu_tensor_free(g->dspark_device_proposal); ds4_gpu_tensor_free(g->dspark_draft_tokens); for (uint32_t stage = 0; stage < DS4_DSPARK_MAX_STAGES; stage++) { ds4_gpu_tensor_free(g->dspark_raw_cache[stage]); @@ -17022,6 +17027,12 @@ static bool metal_graph_configure_dspark_capture( const uint64_t hc_dim = (uint64_t)DS4_N_HC * DS4_N_EMBD; g->dspark_draft_tokens = ds4_gpu_tensor_alloc((uint64_t)dw->block_size * sizeof(int32_t)); +#if !defined(DS4_ROCM_BUILD) + /* Opportunistic CUDA/Metal accelerator scratch. Allocation failure + * must not make the ordinary DSpark graph unavailable. */ + g->dspark_device_proposal = ds4_gpu_tensor_alloc( + DS4_GPU_DSPARK_DEVICE_PROPOSAL_BYTES); +#endif g->dspark_draft_hc = ds4_gpu_tensor_alloc((uint64_t)dw->block_size * hc_dim * sizeof(float)); g->dspark_target_hc = @@ -23205,6 +23216,129 @@ static bool metal_graph_encode_decode_layer_phase( layer->attn_q_a->type == DS4_TENSOR_Q4_K && layer->attn_kv->type == DS4_TENSOR_Q4_K; bool qkv_pair_projected = resume_after_qa_kv_raw; +#if defined(__APPLE__) && !defined(DS4_NO_GPU) + /* M1-M5 decode fusion: the q_a/kv Q8 pair and the eligible F16 + * compressor projections all read the same normalized attention row and + * write disjoint outputs. Resident FULL decode retains its established + * default. SSD FULL decode and exact-union's TO_ROUTER prefix stay + * opt-in because one compound grid can change model-page scheduling even + * though it reads exactly the same bytes as the separate dispatches. */ + const uint32_t q8_compound_ratio = + compressed ? ds4_layer_compress_ratio(il) : 0u; + const uint32_t q8_compound_width0 = + q8_compound_ratio == 4u ? 2u * DS4_N_HEAD_DIM : DS4_N_HEAD_DIM; + const uint32_t q8_compound_width1 = + q8_compound_ratio == 4u ? 2u * DS4_N_INDEXER_HEAD_DIM : 0u; + const bool q8_compound_resident_scope = + phase == METAL_DECODE_LAYER_FULL && + !g->ssd_streaming && !g->ssd_streaming_cold; + const bool q8_compound_required = + metal_graph_tp_env_flag( + "DS4_METAL_REQUIRE_Q8_QKV_COMPRESSOR_FUSE", false); + const bool q8_compound_stream_enabled = + q8_compound_required || + metal_graph_tp_env_flag( + "DS4_METAL_ENABLE_Q8_QKV_COMPRESSOR_FUSE", false); + const bool q8_compound_stream_scope = + q8_compound_stream_enabled && + ((phase == METAL_DECODE_LAYER_FULL && + (g->ssd_streaming || g->ssd_streaming_cold)) || + (phase == METAL_DECODE_LAYER_TO_ROUTER && + g->spec_exactn_union_collect_routes)); + const bool q8_compound_scope = + q8_compound_resident_scope || q8_compound_stream_scope; + const bool q8_compound_device = q8_compound_ratio == 4u + ? metal_graph_ported_m5_decode_feature_enabled( + "DS4_METAL_DISABLE_PRE_M5_QKV_PAIR_QUAD_FUSE", + "DS4_METAL_DISABLE_M5_QKV_PAIR_QUAD_FUSE") + : (q8_compound_ratio == 128u && + metal_graph_ported_m5_decode_feature_enabled( + "DS4_METAL_DISABLE_PRE_M5_QKV_PAIR_COMPRESSOR_FUSE", + "DS4_METAL_DISABLE_M5_QKV_PAIR_COMPRESSOR_FUSE")); + const bool q8_compound_index_ok = + q8_compound_ratio != 4u || + (layer->indexer_compressor_kv && layer->indexer_compressor_gate && + layer->indexer_compressor_ape && + layer->indexer_compressor_kv->type == DS4_TENSOR_F16 && + layer->indexer_compressor_gate->type == DS4_TENSOR_F16 && + layer->indexer_compressor_kv->dim[0] == DS4_N_EMBD && + layer->indexer_compressor_gate->dim[0] == DS4_N_EMBD && + layer->indexer_compressor_kv->dim[1] == q8_compound_width1 && + layer->indexer_compressor_gate->dim[1] == q8_compound_width1); + const bool q8_compound_eligible = + !resume_after_qa_kv_raw && ok && qkv_rms_fused && qkv_proj_q8 && + compressed && q8_compound_scope && q8_compound_device && + (q8_compound_ratio == 4u || q8_compound_ratio == 128u) && + g->cuda_qkv_pair && !metal_graph_use_reference_qkv_pair_proj() && + !metal_graph_use_reference_compressor_pair_proj() && + layer->attn_compressor_kv && layer->attn_compressor_gate && + layer->attn_compressor_ape && + layer->attn_compressor_kv->type == DS4_TENSOR_F16 && + layer->attn_compressor_gate->type == DS4_TENSOR_F16 && + layer->attn_compressor_kv->dim[0] == DS4_N_EMBD && + layer->attn_compressor_gate->dim[0] == DS4_N_EMBD && + layer->attn_compressor_kv->dim[1] == q8_compound_width0 && + layer->attn_compressor_gate->dim[1] == q8_compound_width0 && + q8_compound_index_ok; + if (q8_compound_eligible) { + ds4_gpu_tensor *out1_kv = q8_compound_width1 + ? metal_graph_index_comp_kv_cur(g) : metal_graph_comp_kv_cur(g); + ds4_gpu_tensor *out1_sc = q8_compound_width1 + ? metal_graph_index_comp_sc_cur(g) : metal_graph_comp_sc_cur(g); + ds4_gpu_tensor *state1_kv = q8_compound_width1 + ? g->layer_index_state_kv[il] : g->layer_attn_state_kv[il]; + ds4_gpu_tensor *state1_sc = q8_compound_width1 + ? g->layer_index_state_score[il] : g->layer_attn_state_score[il]; + const uint64_t weight1_kv = q8_compound_width1 + ? layer->indexer_compressor_kv->abs_offset + : layer->attn_compressor_kv->abs_offset; + const uint64_t weight1_sc = q8_compound_width1 + ? layer->indexer_compressor_gate->abs_offset + : layer->attn_compressor_gate->abs_offset; + const uint64_t ape1 = q8_compound_width1 + ? layer->indexer_compressor_ape->abs_offset + : layer->attn_compressor_ape->abs_offset; + const uint32_t ape1_type = q8_compound_width1 + ? layer->indexer_compressor_ape->type + : layer->attn_compressor_ape->type; + const int fused = ds4_gpu_qkv_pair_quad_compressor_store_tensor( + metal_graph_qr(g), metal_graph_kv_raw(g), + metal_graph_comp_kv_cur(g), metal_graph_comp_sc_cur(g), + out1_kv, out1_sc, + g->layer_attn_state_kv[il], + g->layer_attn_state_score[il], + state1_kv, state1_sc, + model->map, model->size, + layer->attn_q_a->abs_offset, + layer->attn_kv->abs_offset, + layer->attn_compressor_kv->abs_offset, + layer->attn_compressor_gate->abs_offset, + weight1_kv, weight1_sc, + layer->attn_compressor_ape->abs_offset, + layer->attn_compressor_ape->type, + ape1, ape1_type, + DS4_N_EMBD, (uint32_t)q_rank, DS4_N_HEAD_DIM, + q8_compound_width0, q8_compound_width1, + metal_graph_attn_norm(g), q8_compound_ratio, pos); + if (fused < 0) { + ok = false; + } else if (fused > 0) { + qkv_pair_projected = true; + qkv_pair_quad_fused = true; + } + } + if (ok && !qkv_pair_quad_fused && + q8_compound_required && + !resume_after_qa_kv_raw && qkv_proj_q8 && compressed && + q8_compound_scope && + (q8_compound_ratio == 4u || q8_compound_ratio == 128u)) { + fprintf(stderr, + "ds4: required Metal Q8 QKV/compressor compound was not selected " + "at layer %u ratio=%u phase=%u\n", + il, q8_compound_ratio, (unsigned)phase); + ok = false; + } +#endif #if defined(__APPLE__) && !defined(DS4_NO_GPU) /* Fuse the Q4 Q-A/KV pair with the compressor projections that consume * the same normalized row. Keep the larger compound dispatch opt-in in @@ -23298,7 +23432,8 @@ static bool metal_graph_encode_decode_layer_phase( * per-row reduction trees (see the kernel comment). Restricted to the * resident FULL phase so split-phase / CUDA / SSD flows keep their * original ordering. */ - if (!resume_after_qa_kv_raw && ok && qkv_rms_fused && compressed && + if (!resume_after_qa_kv_raw && ok && !qkv_pair_quad_fused && + qkv_rms_fused && compressed && phase == METAL_DECODE_LAYER_FULL && !g->ssd_streaming && !g->ssd_streaming_cold && ds4_layer_compress_ratio(il) == 4u && @@ -27171,23 +27306,24 @@ static bool metal_graph_attention_output_dense_quant_batch( n_tokens) != 0; } if (out_a->type == DS4_TENSOR_Q4_K && n_tokens >= 2u) { - if (ds4_gpu_attention_output_q4_K_batch_tensor(out, - low, - metal_graph_batch_group_tmp(g), - metal_graph_batch_low_tmp(g), - model->map, - model->size, - out_a->abs_offset, - out_b->abs_offset, - out_b->type, - group_dim, - rank, - n_groups, - out_dim, - heads, - n_tokens) != 0) { - return true; - } + const int tiny_rc = ds4_gpu_attention_output_q4_K_batch_tensor( + out, + low, + metal_graph_batch_group_tmp(g), + metal_graph_batch_low_tmp(g), + model->map, + model->size, + out_a->abs_offset, + out_b->abs_offset, + out_b->type, + group_dim, + rank, + n_groups, + out_dim, + heads, + n_tokens); + if (tiny_rc > 0) return true; + if (tiny_rc < 0) return false; } const uint64_t heads_row_elems = (uint64_t)n_groups * group_dim; @@ -27989,14 +28125,15 @@ static DS4_MAYBE_UNUSED bool metal_graph_pre_m5_q2_decode_schedule_eligible( } static uint32_t metal_graph_token_split_after_layers(void) { -#if !defined(__APPLE__) +#if defined(__APPLE__) + uint32_t split_after_layers = 4; +#else /* Metal flushes submit the encoded prefix without waiting, allowing the GPU * to start it while the CPU encodes the suffix. CUDA and ROCm implement * this API as a device-wide synchronization, so splitting there only drains * the launch pipeline in the middle of every token. */ - return 0; -#else - uint32_t split_after_layers = 4; + uint32_t split_after_layers = 0; +#endif const char *split_env = getenv("DS4_METAL_GRAPH_TOKEN_SPLIT_LAYERS"); if (split_env && split_env[0]) { char *end = NULL; @@ -28004,7 +28141,6 @@ static uint32_t metal_graph_token_split_after_layers(void) { if (end != split_env && v <= DS4_N_LAYER) split_after_layers = (uint32_t)v; } return split_after_layers; -#endif } static uint32_t metal_graph_token_adaptive_split_after_layers( @@ -35024,6 +35160,166 @@ static uint32_t dspark_confident_prefix_len( return confidence_len; } +/* Keep the Q8 confidence/Markov chain on the GPU and read one compact result at + * the end. Unsupported layouts and every validation discrepancy are + * fail-closed: the caller immediately runs the established per-row path. + * + * This is deliberately opt-in. The GPU reproduces the Q8 activation + * quantization and Markov tie-break, but the confidence accumulation runs on + * a different processor and therefore still needs the backend parity oracle + * before this can become a default. */ +static bool dspark_apply_markov_confidence_device_runtime( + ds4_gpu_graph *g, + const ds4_model *dspark_model, + const ds4_dspark_weights *dw, + int first_prev_token, + float confidence_threshold, + int32_t proposal[DS4_DSPARK_MAX_BLOCK_SIZE], + uint32_t *proposal_len, + uint32_t *confidence_len, + uint32_t *confidence_prefix_len, + bool reuse_first_confidence, + float *confidence0, + bool *attempted, + bool *policy_mismatch) { + if (attempted) *attempted = false; + if (policy_mismatch) *policy_mismatch = false; +#if !defined(DS4_ROCM_BUILD) +#if defined(__APPLE__) + const char *device_enable = + getenv("DS4_METAL_DSPARK_DEVICE_PROPOSER"); + const char *device_disable = + getenv("DS4_METAL_DSPARK_NO_DEVICE_PROPOSER"); +#else + const char *device_enable = + getenv("DS4_CUDA_DSPARK_DEVICE_PROPOSER"); +#endif + const bool device_requested = + device_enable && device_enable[0] && + strcmp(device_enable, "0") != 0 && + strcmp(device_enable, "off") != 0 && + strcmp(device_enable, "false") != 0 && +#if defined(__APPLE__) + device_disable == NULL; +#else + getenv("DS4_CUDA_DSPARK_NO_DEVICE_PROPOSER") == NULL; +#endif + if (!g || !g->dspark_device_proposal || !g->spec_logits || + !metal_graph_batch_ffn_norm(g) || !dspark_model || !dw || + !proposal || !proposal_len || !confidence_len || + !confidence_prefix_len || !confidence0 || + first_prev_token < 0 || + (uint32_t)first_prev_token >= DS4_N_VOCAB || + confidence_threshold <= 0.0f || confidence_threshold > 1.0f || + dw->block_size == 0 || + dw->block_size > DS4_GPU_DSPARK_MAX_DRAFTS || + g->placement != NULL || +#if !defined(__APPLE__) + g->ssd_streaming || +#endif + g->quality || + g->active_tier != 0 || g->dspark_exec_tier != 0 || + g->tp_world != 0 || !reuse_first_confidence || + !device_requested || + getenv("DS4_DSPARK_NO_GPU_MARKOV") != NULL || + dspark_markov_bias_disabled() || + !dspark_markov_probe_ready(dw) || + !dspark_confidence_probe_ready(dw)) { + return false; + } + + const ds4_dspark_stage_weights *final = + &dw->stage[dw->n_stages - 1u]; + if (final->markov_w1->type != DS4_TENSOR_Q8_0 || + final->markov_w2->type != DS4_TENSOR_Q8_0 || + final->confidence_proj->type != DS4_TENSOR_Q8_0) { + return false; + } + + if (attempted) *attempted = true; + ds4_gpu_dspark_device_proposal result; + memset(&result, 0, sizeof(result)); + if (!ds4_gpu_dspark_markov_confidence_q8_tensor( + g->dspark_device_proposal, + g->spec_logits, + metal_graph_batch_ffn_norm(g), + dspark_model->map, + dspark_model->size, + final->markov_w1->abs_offset, + final->markov_w2->abs_offset, + final->confidence_proj->abs_offset, + (uint32_t)first_prev_token, + DS4_N_VOCAB, + dw->markov_rank, + DS4_N_EMBD, + dw->block_size, + confidence_threshold, + reuse_first_confidence ? 1 : 0, + *confidence0) || + !ds4_gpu_tensor_read(g->dspark_device_proposal, + 0, + &result, + sizeof(result))) { + return false; + } + + if (result.status != 1u || result.reserved != 0u || + result.confidence_len == 0u || + result.confidence_len > dw->block_size || + result.proposal_len > result.confidence_len || + result.proposal_len > dw->block_size || + !((result.proposal_len == dw->block_size && + result.confidence_len == dw->block_size) || + (result.proposal_len < dw->block_size && + result.confidence_len == result.proposal_len + 1u))) { + return false; + } + for (uint32_t i = 0; i < result.confidence_len; i++) { + if (!isfinite(result.confidence_logits[i])) return false; + } + for (uint32_t i = 0; i < result.proposal_len; i++) { + if (result.tokens[i] < 0 || + (uint32_t)result.tokens[i] >= DS4_N_VOCAB) { + return false; + } + } + + const uint32_t cpu_prefix = + dspark_confident_prefix_len(result.confidence_logits, + result.confidence_len, + confidence_threshold); + /* The device may stop on a libdevice sigmoid rounding difference. If + * the established CPU policy accepts that row, the device did not + * compute its Markov token and the result is incomplete. */ + if (cpu_prefix > result.proposal_len) { + if (policy_mismatch) *policy_mismatch = true; + return false; + } + + for (uint32_t i = 0; i < result.proposal_len; i++) { + proposal[i] = result.tokens[i]; + } + *proposal_len = result.proposal_len; + *confidence_len = result.confidence_len; + *confidence_prefix_len = cpu_prefix; + *confidence0 = result.confidence_logits[0]; + return true; +#else + (void)g; + (void)dspark_model; + (void)dw; + (void)first_prev_token; + (void)confidence_threshold; + (void)proposal; + (void)proposal_len; + (void)confidence_len; + (void)confidence_prefix_len; + (void)reuse_first_confidence; + (void)confidence0; + return false; +#endif +} + static bool metal_graph_eval_mtp_draft_from_hc( ds4_gpu_graph *g, const ds4_model *base_model, @@ -37374,11 +37670,126 @@ static bool metal_graph_verify_decode2_exact_impl( enum { DS4_CUDA_EXACTN_MAX_ROWS = 5 }; +typedef struct { + double setup_ms; + double layer_ms; + double head_ms; + double read_ms; + bool batch_head_attempted; + bool batch_head_used; + bool batch_head_fallback; +} ds4_cuda_exactn_timing; + +/* Keep the exact-N head experiment narrower than the resident verifier gate: + * only Q8 output weights have a multi-row CUDA entry point that explicitly + * preserves the ordinary one-row reduction order. The disable variable wins + * so a deployed command line can retain the enable flag during an A/B. */ +static bool metal_graph_cuda_exactn_batch_head_requested( + const ds4_gpu_graph *g, + const ds4_weights *weights, + uint32_t n_tokens) { +#if !defined(__APPLE__) && !defined(DS4_ROCM_BUILD) + return g && weights && weights->output && n_tokens >= 2u && + n_tokens <= DS4_CUDA_EXACTN_MAX_ROWS && + g->placement == NULL && g->active_tier == 0 && + g->head_tier == 0 && + weights->output->type == DS4_TENSOR_Q8_0 && + weights->output->ndim == 2 && + weights->output->dim[0] == DS4_N_EMBD && + weights->output->dim[1] == DS4_N_VOCAB && + metal_graph_batch_ffn_norm(g) && g->spec_logits && + ds4_gpu_tensor_bytes(metal_graph_batch_ffn_norm(g)) >= + (uint64_t)n_tokens * DS4_N_EMBD * sizeof(float) && + ds4_gpu_tensor_bytes(g->spec_logits) >= + (uint64_t)n_tokens * DS4_N_VOCAB * sizeof(float) && + metal_graph_tp_env_flag( + "DS4_CUDA_DSPARK_EXACTN_BATCH_HEAD", false) && + !metal_graph_tp_env_flag( + "DS4_CUDA_DISABLE_DSPARK_EXACTN_BATCH_HEAD", false); +#else + (void)g; + (void)weights; + (void)n_tokens; + return false; +#endif +} + +/* Canonical one-row output-head prefix, stopping immediately before the vocab + * projection. exact-N invokes it once per row, but binds output_norm to a + * distinct contiguous row. Thus HC collapse and normalization retain the + * exact one-row kernels while the expensive Q8 vocabulary projection can be + * issued once for all rows. */ +static bool metal_graph_encode_output_head_norm_exact_row( + ds4_gpu_graph *g, + const ds4_model *model, + const ds4_weights *weights, + ds4_gpu_tensor *norm_dst) { + if (!g || !model || !weights || !norm_dst || g->placement != NULL || + g->active_tier != 0 || g->head_tier != 0 || + ds4_gpu_tensor_bytes(norm_dst) < + (uint64_t)DS4_N_EMBD * sizeof(float)) { + return false; + } + + const uint64_t hc_dim = (uint64_t)DS4_N_HC * DS4_N_EMBD; + ds4_gpu_tensor *saved_norm = g->output_norm_by_tier[0]; + g->output_norm_by_tier[0] = norm_dst; + bool ok = ds4_gpu_rms_norm_plain_tensor( + metal_graph_flat_hc(g), + metal_graph_cur_hc(g), + (uint32_t)hc_dim, + DS4_RMS_EPS) != 0; + if (ok) { + ok = ds4_gpu_matmul_f16_tensor( + metal_graph_output_pre(g), + model->map, + model->size, + weights->output_hc_fn->abs_offset, + hc_dim, + DS4_N_HC, + metal_graph_flat_hc(g), + 1) != 0; + } + if (ok) { + ok = ds4_gpu_output_hc_weights_tensor( + metal_graph_output_weights(g), + metal_graph_output_pre(g), + model->map, + model->size, + weights->output_hc_scale->abs_offset, + weights->output_hc_base->abs_offset, + DS4_N_HC, + DS4_HC_EPS) != 0; + } + if (ok) { + ok = ds4_gpu_hc_weighted_sum_tensor( + metal_graph_output_embd(g), + metal_graph_cur_hc(g), + metal_graph_output_weights(g), + DS4_N_EMBD, + DS4_N_HC) != 0; + } + if (ok) { + ok = ds4_gpu_rms_norm_weight_tensor( + norm_dst, + metal_graph_output_embd(g), + model->map, + model->size, + weights->output_norm->abs_offset, + DS4_N_EMBD, + DS4_RMS_EPS) != 0; + } + g->output_norm_by_tier[0] = saved_norm; + return ok; +} + /* Resident exact-N verifier for single-device CUDA. * * This is the N-row form of the canonical exact-2 tape above: every row still - * executes the ordinary one-token layer and output-head kernels, in the same - * autoregressive order. The only batching is lifetime/dispatch batching: + * executes the ordinary one-token layers in the same autoregressive order. + * By default its output head is one-row too; the Q8-only opt-in preserves the + * one-row HC/norm prefix and batches just the exact-row vocab projection. The + * remaining batching is lifetime/dispatch batching: * row-local hidden states stay in the existing prefill workspace, all layer * launches share one command stream, and the CPU reads N-1 top ids plus the * final logits after the whole block. Persistent KV/compressor state is @@ -37392,7 +37803,9 @@ static bool metal_graph_verify_decode_exactn_cuda_resident_impl( uint32_t n_tokens, uint32_t start, int *row_tops, - float *last_logits) { + float *last_logits, + ds4_cuda_exactn_timing *timing) { + if (timing) memset(timing, 0, sizeof(*timing)); if (!g || !model || !weights || !tokens || !row_tops || !last_logits || n_tokens < 2u || n_tokens > DS4_CUDA_EXACTN_MAX_ROWS || n_tokens > g->prefill_cap || g->raw_cap == 0 || @@ -37408,6 +37821,7 @@ static bool metal_graph_verify_decode_exactn_cuda_resident_impl( return false; } + const double setup_t0 = timing ? now_sec() : 0.0; const uint64_t hc_dim = (uint64_t)DS4_N_HC * DS4_N_EMBD; ds4_gpu_tensor *cur_rows[DS4_CUDA_EXACTN_MAX_ROWS] = {0}; ds4_gpu_tensor *next_rows[DS4_CUDA_EXACTN_MAX_ROWS] = {0}; @@ -37448,6 +37862,8 @@ static bool metal_graph_verify_decode_exactn_cuda_resident_impl( g->spec_capture_prefixes = false; g->spec_disable_decode_graphs = true; + if (timing) timing->setup_ms += (now_sec() - setup_t0) * 1000.0; + const double layer_t0 = timing ? now_sec() : 0.0; if (ok) { ok = ds4_gpu_begin_commands() != 0; commands_open = ok; @@ -37485,7 +37901,9 @@ static bool metal_graph_verify_decode_exactn_cuda_resident_impl( } commands_open = false; } + if (timing) timing->layer_ms += (now_sec() - layer_t0) * 1000.0; + const double head_t0 = timing ? now_sec() : 0.0; if (ok) { top_span = ds4_gpu_tensor_view( g->batch_router_selected_by_tier[0], @@ -37508,30 +37926,87 @@ static bool metal_graph_verify_decode_exactn_cuda_resident_impl( } } - if (ok) { + const bool try_batch_head = ok && + metal_graph_cuda_exactn_batch_head_requested(g, weights, n_tokens); + bool batch_head_ok = false; + if (try_batch_head) { + if (timing) timing->batch_head_attempted = true; + batch_head_ok = ds4_gpu_begin_commands() != 0; + commands_open = batch_head_ok; + for (uint32_t row = 0; batch_head_ok && row < n_tokens; row++) { + ds4_gpu_tensor *norm_row = ds4_gpu_tensor_view( + metal_graph_batch_ffn_norm(g), + (uint64_t)row * DS4_N_EMBD * sizeof(float), + (uint64_t)DS4_N_EMBD * sizeof(float)); + g->cur_hc_by_tier[0] = cur_rows[row]; + batch_head_ok = norm_row && + metal_graph_encode_output_head_norm_exact_row( + g, model, weights, norm_row); + ds4_gpu_tensor_free(norm_row); + } + if (batch_head_ok) { + batch_head_ok = + ds4_gpu_matmul_q8_0_decode_rows_exact_tensor( + g->spec_logits, + model->map, + model->size, + weights->output->abs_offset, + DS4_N_EMBD, + DS4_N_VOCAB, + metal_graph_batch_ffn_norm(g), + n_tokens) != 0; + } + for (uint32_t row = 0; + batch_head_ok && row + 1u < n_tokens; + row++) { + batch_head_ok = ds4_gpu_argmax_tensor( + top_rows[row], logits_rows[row], DS4_N_VOCAB) != 0; + } + if (commands_open) { + if (batch_head_ok) { + batch_head_ok = ds4_gpu_end_commands() != 0; + if (!batch_head_ok) (void)ds4_gpu_synchronize(); + } else { + (void)ds4_gpu_synchronize(); + } + commands_open = false; + } + if (timing) { + timing->batch_head_used = batch_head_ok; + timing->batch_head_fallback = !batch_head_ok; + } + } + + /* An ineligible or failed experimental head always returns to the proven + * one-row sequence. Head scratch is non-persistent, so retrying does not + * alter the verifier frontier. */ + if (ok && !batch_head_ok) { ok = ds4_gpu_begin_commands() != 0; commands_open = ok; - } - for (uint32_t row = 0; ok && row < n_tokens; row++) { - g->cur_hc_by_tier[0] = cur_rows[row]; - g->logits_by_tier[0] = logits_rows[row]; - ok = metal_graph_encode_output_head( - g, model, weights, weights->output->dim[1]); - if (ok && row + 1u < n_tokens) { - ok = ds4_gpu_argmax_tensor( - top_rows[row], logits_rows[row], DS4_N_VOCAB) != 0; + for (uint32_t row = 0; ok && row < n_tokens; row++) { + g->cur_hc_by_tier[0] = cur_rows[row]; + g->logits_by_tier[0] = logits_rows[row]; + ok = metal_graph_encode_output_head( + g, model, weights, weights->output->dim[1]); + if (ok && row + 1u < n_tokens) { + ok = ds4_gpu_argmax_tensor( + top_rows[row], logits_rows[row], + DS4_N_VOCAB) != 0; + } } - } - if (commands_open) { - if (ok) { - ok = ds4_gpu_end_commands() != 0; - } else { - (void)ds4_gpu_synchronize(); + if (commands_open) { + if (ok) { + ok = ds4_gpu_end_commands() != 0; + } else { + (void)ds4_gpu_synchronize(); + } + commands_open = false; } - commands_open = false; } + if (timing) timing->head_ms += (now_sec() - head_t0) * 1000.0; int32_t tops_i32[DS4_CUDA_EXACTN_MAX_ROWS - 1u] = {0}; + const double read_t0 = timing ? now_sec() : 0.0; if (ok) { ok = ds4_gpu_tensor_read( top_span, @@ -37549,6 +38024,7 @@ static bool metal_graph_verify_decode_exactn_cuda_resident_impl( row_tops[row] = tops_i32[row]; } } + if (timing) timing->read_ms += (now_sec() - read_t0) * 1000.0; g->spec_capture_prefixes = saved_capture; g->spec_disable_decode_graphs = saved_disable_decode_graphs; @@ -37569,6 +38045,45 @@ static bool metal_graph_verify_decode_exactn_cuda_resident_impl( enum { DS4_METAL_EXACTN_UNION_MAX_ROWS = 5 }; +typedef struct { + bool batch_head_attempted; + bool batch_head_used; + bool batch_head_fallback; +} ds4_metal_exactn_union_timing; + +static DS4_MAYBE_UNUSED bool metal_graph_metal_exactn_batch_head_requested( + const ds4_gpu_graph *g, + const ds4_weights *weights, + uint32_t n_tokens) { +#if defined(__APPLE__) + return g && weights && weights->output && + n_tokens >= 2u && + n_tokens <= DS4_METAL_EXACTN_UNION_MAX_ROWS && + n_tokens <= g->prefill_cap && + g->ssd_streaming && g->placement == NULL && + g->tp_world <= 1u && g->active_tier == 0 && + g->head_tier == 0 && + weights->output->type == DS4_TENSOR_Q8_0 && + weights->output->ndim == 2 && + weights->output->dim[0] == DS4_N_EMBD && + weights->output->dim[1] == DS4_N_VOCAB && + metal_graph_batch_ffn_norm(g) && g->spec_logits && + ds4_gpu_tensor_bytes(metal_graph_batch_ffn_norm(g)) >= + (uint64_t)n_tokens * DS4_N_EMBD * sizeof(float) && + ds4_gpu_tensor_bytes(g->spec_logits) >= + (uint64_t)n_tokens * DS4_N_VOCAB * sizeof(float) && + metal_graph_tp_env_flag( + "DS4_METAL_DSPARK_EXACTN_BATCH_HEAD", false) && + !metal_graph_tp_env_flag( + "DS4_METAL_DISABLE_DSPARK_EXACTN_BATCH_HEAD", false); +#else + (void)g; + (void)weights; + (void)n_tokens; + return false; +#endif +} + /* Row-local aliases needed across the TO_ROUTER/FROM_ROUTER split. The * ordinary decode tape intentionally reuses its Class-P tensors at every * dispatch. That is safe for a complete row, but not when N router prefixes @@ -37755,8 +38270,10 @@ static bool metal_graph_verify_decode_exactn_union_impl( uint32_t n_tokens, uint32_t start, int *row_tops, - float *last_logits) { + float *last_logits, + ds4_metal_exactn_union_timing *timing) { #if defined(__APPLE__) + if (timing) memset(timing, 0, sizeof(*timing)); const bool exact_rows_profile = getenv("DS4_METAL_DSPARK_EXACT_ROWS_PROFILE") != NULL; const bool async_exact_rows_tails = @@ -38037,27 +38554,90 @@ static bool metal_graph_verify_decode_exactn_union_impl( ds4_gpu_tensor *saved_cur = g->cur_hc_by_tier[0]; ds4_gpu_tensor *saved_after = g->after_ffn_hc_by_tier[0]; ds4_gpu_tensor *saved_logits = g->logits_by_tier[0]; - if (ok) ok = ds4_gpu_begin_commands() != 0; - commands_open = ok; - for (uint32_t row = 0; ok && row < n_tokens; row++) { - g->cur_hc_by_tier[0] = cur_rows[row]; - g->logits_by_tier[0] = logits_rows[row]; - ok = metal_graph_encode_output_head( - g, model, weights, weights->output->dim[1]); - if (ok && row + 1u < n_tokens) { - ok = ds4_gpu_argmax_tensor(top_rows[row], - logits_rows[row], - DS4_N_VOCAB) != 0; + const bool try_batch_head = ok && + metal_graph_metal_exactn_batch_head_requested( + g, weights, n_tokens); + bool batch_head_ok = false; + if (try_batch_head) { + if (timing) timing->batch_head_attempted = true; + batch_head_ok = ds4_gpu_begin_commands() != 0; + commands_open = batch_head_ok; + for (uint32_t row = 0; + batch_head_ok && row < n_tokens; + row++) { + ds4_gpu_tensor *norm_row = ds4_gpu_tensor_view( + metal_graph_batch_ffn_norm(g), + (uint64_t)row * DS4_N_EMBD * sizeof(float), + (uint64_t)DS4_N_EMBD * sizeof(float)); + g->cur_hc_by_tier[0] = cur_rows[row]; + batch_head_ok = norm_row && + metal_graph_encode_output_head_norm_exact_row( + g, model, weights, norm_row); + ds4_gpu_tensor_free(norm_row); + } + if (batch_head_ok) { + batch_head_ok = + ds4_gpu_matmul_q8_0_decode_rows_exact_tensor( + g->spec_logits, + model->map, + model->size, + weights->output->abs_offset, + DS4_N_EMBD, + DS4_N_VOCAB, + metal_graph_batch_ffn_norm(g), + n_tokens) != 0; + } + for (uint32_t row = 0; + batch_head_ok && row + 1u < n_tokens; + row++) { + batch_head_ok = ds4_gpu_argmax_tensor( + top_rows[row], logits_rows[row], DS4_N_VOCAB) != 0; + } + g->cur_hc_by_tier[0] = saved_cur; + g->after_ffn_hc_by_tier[0] = saved_after; + g->logits_by_tier[0] = saved_logits; + if (commands_open) { + if (batch_head_ok) { + batch_head_ok = ds4_gpu_end_commands() != 0; + if (!batch_head_ok) (void)ds4_gpu_synchronize(); + } else { + (void)ds4_gpu_synchronize(); + } + commands_open = false; + } + if (timing) { + timing->batch_head_used = batch_head_ok; + timing->batch_head_fallback = !batch_head_ok; } } - g->cur_hc_by_tier[0] = saved_cur; - g->after_ffn_hc_by_tier[0] = saved_after; - g->logits_by_tier[0] = saved_logits; - if (commands_open) { - if (ok) { - ok = ds4_gpu_end_commands() != 0; - } else { - (void)ds4_gpu_synchronize(); + + /* The experimental multi-row projection only touches head scratch. A + * failed or ineligible attempt can therefore retry the established + * one-row sequence without restoring the target frontier. */ + if (ok && !batch_head_ok) { + ok = ds4_gpu_begin_commands() != 0; + commands_open = ok; + for (uint32_t row = 0; ok && row < n_tokens; row++) { + g->cur_hc_by_tier[0] = cur_rows[row]; + g->logits_by_tier[0] = logits_rows[row]; + ok = metal_graph_encode_output_head( + g, model, weights, weights->output->dim[1]); + if (ok && row + 1u < n_tokens) { + ok = ds4_gpu_argmax_tensor(top_rows[row], + logits_rows[row], + DS4_N_VOCAB) != 0; + } + } + g->cur_hc_by_tier[0] = saved_cur; + g->after_ffn_hc_by_tier[0] = saved_after; + g->logits_by_tier[0] = saved_logits; + if (commands_open) { + if (ok) { + ok = ds4_gpu_end_commands() != 0; + } else { + (void)ds4_gpu_synchronize(); + } + commands_open = false; } } @@ -38103,6 +38683,7 @@ static bool metal_graph_verify_decode_exactn_union_impl( (void)start; (void)row_tops; (void)last_logits; + (void)timing; return false; #endif } @@ -54836,13 +55417,32 @@ typedef struct ds4_dspark_spec_stats { uint64_t cuda_exactn_partial_fallbacks; uint64_t cuda_exactn_error_fallbacks; uint64_t cuda_exactn_rows; + uint64_t cuda_exactn_partial_replays; + uint64_t cuda_exactn_legacy_verify_skips; + uint64_t cuda_exactn_batch_head_attempts; + uint64_t cuda_exactn_batch_head_uses; + uint64_t cuda_exactn_batch_head_fallbacks; + uint64_t cuda_device_proposer_attempts; + uint64_t cuda_device_proposer_uses; + uint64_t cuda_device_proposer_fallbacks; + uint64_t cuda_device_proposer_policy_mismatches; + uint64_t metal_device_proposer_attempts; + uint64_t metal_device_proposer_uses; + uint64_t metal_device_proposer_fallbacks; + uint64_t metal_device_proposer_policy_mismatches; uint64_t exactn_union_attempts; uint64_t exactn_union_full_accepts; uint64_t exactn_union_fallbacks; uint64_t exactn_union_partial_fallbacks; uint64_t exactn_union_error_fallbacks; + uint64_t exactn_union_partial_replays; + uint64_t exactn_union_legacy_verify_skips; + uint64_t metal_exactn_batch_head_attempts; + uint64_t metal_exactn_batch_head_uses; + uint64_t metal_exactn_batch_head_fallbacks; uint64_t exactn_attempts; uint64_t exactn_full_accepts; + uint64_t exactn_partial_accepts; uint64_t exactn_fallbacks; uint64_t exactn_partial_fallbacks; uint64_t exactn_error_fallbacks; @@ -54873,6 +55473,15 @@ typedef struct ds4_dspark_spec_stats { double verify_layer_ms; double verify_head_ms; double verify_read_ms; + double cuda_exactn_ms; + double cuda_exactn_setup_ms; + double cuda_exactn_layer_ms; + double cuda_exactn_head_ms; + double cuda_exactn_read_ms; + double cuda_exactn_restore_ms; + double cuda_exactn_legacy_verify_ms; + double cuda_exactn_partial_replay_ms; + double exactn_union_partial_replay_ms; uint64_t verifier_fused_head; uint64_t metal_acceptance_only_attempts; uint64_t metal_acceptance_only_rows_saved; @@ -65319,11 +65928,30 @@ static void ds4_session_print_dspark_stats(const ds4_session *s) { "cuda_exactn_full=%llu cuda_exactn_fallback=%llu " "cuda_exactn_partial_fallback=%llu " "cuda_exactn_error_fallback=%llu cuda_exactn_rows=%llu " + "cuda_exactn_partial_replay=%llu " + "cuda_exactn_verify_skip=%llu " + "cuda_exactn_batch_head_attempt=%llu " + "cuda_exactn_batch_head_use=%llu " + "cuda_exactn_batch_head_fallback=%llu " + "cuda_device_proposer_attempt=%llu " + "cuda_device_proposer_use=%llu " + "cuda_device_proposer_fallback=%llu " + "cuda_device_proposer_policy_mismatch=%llu " + "metal_device_proposer_attempt=%llu " + "metal_device_proposer_use=%llu " + "metal_device_proposer_fallback=%llu " + "metal_device_proposer_policy_mismatch=%llu " "exactn_union_attempt=%llu " "exactn_union_full=%llu exactn_union_fallback=%llu " "exactn_union_partial_fallback=%llu " "exactn_union_error_fallback=%llu " + "exactn_union_partial_replay=%llu " + "exactn_union_verify_skip=%llu " + "metal_exactn_batch_head_attempt=%llu " + "metal_exactn_batch_head_use=%llu " + "metal_exactn_batch_head_fallback=%llu " "exactn_attempt=%llu exactn_full=%llu " + "exactn_partial=%llu " "exactn_fallback=%llu exactn_partial_fallback=%llu " "exactn_error_fallback=%llu exactn_boundary_rows=%llu " "miss_first=%llu no_draft=%llu " @@ -65334,6 +65962,12 @@ static void ds4_session_print_dspark_stats(const ds4_session *s) { "prop_logits=%.3f prop_markov=%.3f prop_confidence=%.3f " "snapshot=%.3f verify=%.3f verify_upload=%.3f " "verify_layer=%.3f verify_head=%.3f verify_read=%.3f " + "cuda_exactn_ms=%.3f cuda_exactn_setup=%.3f " + "cuda_exactn_layer=%.3f cuda_exactn_head=%.3f " + "cuda_exactn_read=%.3f cuda_exactn_restore=%.3f " + "cuda_exactn_legacy_verify=%.3f " + "cuda_exactn_partial_replay_ms=%.3f " + "exactn_union_partial_replay_ms=%.3f " "verify_fused_head=%llu metal_accept_only=%llu " "metal_verify_rows_saved=%llu metal_replay_headless=%llu " "replay=%.3f spec_total=%.3f " @@ -65362,13 +65996,32 @@ static void ds4_session_print_dspark_stats(const ds4_session *s) { (unsigned long long)st->cuda_exactn_partial_fallbacks, (unsigned long long)st->cuda_exactn_error_fallbacks, (unsigned long long)st->cuda_exactn_rows, + (unsigned long long)st->cuda_exactn_partial_replays, + (unsigned long long)st->cuda_exactn_legacy_verify_skips, + (unsigned long long)st->cuda_exactn_batch_head_attempts, + (unsigned long long)st->cuda_exactn_batch_head_uses, + (unsigned long long)st->cuda_exactn_batch_head_fallbacks, + (unsigned long long)st->cuda_device_proposer_attempts, + (unsigned long long)st->cuda_device_proposer_uses, + (unsigned long long)st->cuda_device_proposer_fallbacks, + (unsigned long long)st->cuda_device_proposer_policy_mismatches, + (unsigned long long)st->metal_device_proposer_attempts, + (unsigned long long)st->metal_device_proposer_uses, + (unsigned long long)st->metal_device_proposer_fallbacks, + (unsigned long long)st->metal_device_proposer_policy_mismatches, (unsigned long long)st->exactn_union_attempts, (unsigned long long)st->exactn_union_full_accepts, (unsigned long long)st->exactn_union_fallbacks, (unsigned long long)st->exactn_union_partial_fallbacks, (unsigned long long)st->exactn_union_error_fallbacks, + (unsigned long long)st->exactn_union_partial_replays, + (unsigned long long)st->exactn_union_legacy_verify_skips, + (unsigned long long)st->metal_exactn_batch_head_attempts, + (unsigned long long)st->metal_exactn_batch_head_uses, + (unsigned long long)st->metal_exactn_batch_head_fallbacks, (unsigned long long)st->exactn_attempts, (unsigned long long)st->exactn_full_accepts, + (unsigned long long)st->exactn_partial_accepts, (unsigned long long)st->exactn_fallbacks, (unsigned long long)st->exactn_partial_fallbacks, (unsigned long long)st->exactn_error_fallbacks, @@ -65395,6 +66048,15 @@ static void ds4_session_print_dspark_stats(const ds4_session *s) { st->verify_layer_ms, st->verify_head_ms, st->verify_read_ms, + st->cuda_exactn_ms, + st->cuda_exactn_setup_ms, + st->cuda_exactn_layer_ms, + st->cuda_exactn_head_ms, + st->cuda_exactn_read_ms, + st->cuda_exactn_restore_ms, + st->cuda_exactn_legacy_verify_ms, + st->cuda_exactn_partial_replay_ms, + st->exactn_union_partial_replay_ms, (unsigned long long)st->verifier_fused_head, (unsigned long long)st->metal_acceptance_only_attempts, (unsigned long long)st->metal_acceptance_only_rows_saved, @@ -68726,8 +69388,49 @@ static bool ds4_session_prepare_dspark_draft_impl(ds4_session *s, markov_ready && !probe_log && confidence_threshold > 0.0f; if (lazy_runtime_confidence) { const double markov_t0 = DS4_DSPARK_PROP_T0(); - markov_ok = - dspark_apply_markov_confidence_lazy_runtime( + bool device_attempted = false; + bool device_policy_mismatch = false; + markov_ok = dspark_apply_markov_confidence_device_runtime( + &s->graph, + &s->engine->mtp_model, + dw, + token, + confidence_threshold, + markov_proposal, + &markov_proposal_len, + &confidence_len, + &confidence_prefix_len, + reuse_confidence0_markov, + &confidence0, + &device_attempted, + &device_policy_mismatch); + if (stats_enabled && device_attempted) { +#if defined(__APPLE__) + s->dspark_stats.metal_device_proposer_attempts++; + if (markov_ok) { + s->dspark_stats.metal_device_proposer_uses++; + } else { + s->dspark_stats.metal_device_proposer_fallbacks++; + if (device_policy_mismatch) { + s->dspark_stats + .metal_device_proposer_policy_mismatches++; + } + } +#else + s->dspark_stats.cuda_device_proposer_attempts++; + if (markov_ok) { + s->dspark_stats.cuda_device_proposer_uses++; + } else { + s->dspark_stats.cuda_device_proposer_fallbacks++; + if (device_policy_mismatch) { + s->dspark_stats + .cuda_device_proposer_policy_mismatches++; + } + } +#endif + } + if (!markov_ok) { + markov_ok = dspark_apply_markov_confidence_lazy_runtime( &s->graph, &s->engine->mtp_model, dw, @@ -68743,6 +69446,7 @@ static bool ds4_session_prepare_dspark_draft_impl(ds4_session *s, &confidence_prefix_len, reuse_confidence0_markov, &confidence0); + } DS4_DSPARK_PROP_ADD(propose_markov_ms, markov_t0); confidence_ok = markov_ok; } else if (markov_ready) { @@ -71008,6 +71712,9 @@ static int ds4_session_eval_dspark_speculative_argmax( bool ok = have_frontier && row_logits && (draft_n <= 1 || row_tops); bool verifier_may_have_mutated = false; bool tp_verify_sent = false; + bool cuda_exactn_verified_partial = false; + bool metal_exactn_union_verified_partial = false; + int preverified_commit_drafts = 0; /* Resident CUDA exact-N keeps the canonical one-token arithmetic while * removing the per-token command/readback boundary. A full match already @@ -71027,6 +71734,7 @@ static int ds4_session_eval_dspark_speculative_argmax( int exactn_accepted_prefix = 1; int exactn_last_top = -1; const double exactn_t0 = stats_enabled ? now_sec() : 0.0; + ds4_cuda_exactn_timing exactn_timing; exactn_ok = metal_graph_verify_decode_exactn_cuda_resident_impl( &s->graph, @@ -71036,7 +71744,8 @@ static int ds4_session_eval_dspark_speculative_argmax( (uint32_t)draft_n, (uint32_t)start, row_tops, - row_logits); + row_logits, + stats_enabled ? &exactn_timing : NULL); if (exactn_ok) { for (int row = 0; row + 1 < draft_n; row++) { exactn_last_top = row_tops[row]; @@ -71048,8 +71757,22 @@ static int ds4_session_eval_dspark_speculative_argmax( } } if (stats_enabled) { - s->dspark_stats.verify_ms += - (now_sec() - exactn_t0) * 1000.0; + const double exactn_ms = (now_sec() - exactn_t0) * 1000.0; + s->dspark_stats.verify_ms += exactn_ms; + s->dspark_stats.cuda_exactn_ms += exactn_ms; + s->dspark_stats.cuda_exactn_setup_ms += exactn_timing.setup_ms; + s->dspark_stats.cuda_exactn_layer_ms += exactn_timing.layer_ms; + s->dspark_stats.cuda_exactn_head_ms += exactn_timing.head_ms; + s->dspark_stats.cuda_exactn_read_ms += exactn_timing.read_ms; + if (exactn_timing.batch_head_attempted) { + s->dspark_stats.cuda_exactn_batch_head_attempts++; + } + if (exactn_timing.batch_head_used) { + s->dspark_stats.cuda_exactn_batch_head_uses++; + } + if (exactn_timing.batch_head_fallback) { + s->dspark_stats.cuda_exactn_batch_head_fallbacks++; + } } if (exactn_ok && !exactn_mismatch) { @@ -71098,7 +71821,14 @@ static int ds4_session_eval_dspark_speculative_argmax( } s->checkpoint.len = start; ds4_session_dspark_capture_invalidate(s); - if (!have_frontier || !spec_frontier_restore(&frontier, s)) { + const double exactn_restore_t0 = stats_enabled ? now_sec() : 0.0; + const bool exactn_restored = + have_frontier && spec_frontier_restore(&frontier, s); + if (stats_enabled) { + s->dspark_stats.cuda_exactn_restore_ms += + (now_sec() - exactn_restore_t0) * 1000.0; + } + if (!exactn_restored) { snprintf(err, errlen, "DSpark CUDA exactN rollback failed"); s->checkpoint_valid = false; @@ -71111,12 +71841,24 @@ static int ds4_session_eval_dspark_speculative_argmax( DS4_DSPARK_STATS_FINISH(); return -1; } + if (exactn_mismatch) { + /* row_tops already proves the exact accepted prefix. The + * frontier above is the only rollback needed; skip the legacy + * N-row verifier and let the common replay tail advance exactly + * those accepted rows. Backend errors still use the legacy + * verifier because their row_tops are not trustworthy. */ + cuda_exactn_verified_partial = true; + if (stats_enabled) { + s->dspark_stats.cuda_exactn_partial_replays++; + s->dspark_stats.cuda_exactn_legacy_verify_skips++; + } + } if (spec_log) { if (exactn_mismatch) { fprintf(stderr, "ds4: DSpark CUDA exactN partial " - "accepted_prefix=%d verified_next=%d; falling back " - "to legacy verify/replay\n", + "accepted_prefix=%d verified_next=%d; restored " + "frontier and replaying without legacy verify\n", exactn_accepted_prefix, exactn_last_top); } else { @@ -71129,9 +71871,9 @@ static int ds4_session_eval_dspark_speculative_argmax( /* Fast Metal exact-N experiment. Unlike the token-major oracle below, * this advances all rows layer-major and shares one immutable selected- - * expert union per layer. It may directly commit only a complete match; - * every partial/error path restores the pre-cycle frontier before trying - * the oracle (when separately enabled) or the established legacy path. */ + * expert union per layer. It may directly commit only a complete match. + * A verified partial restores once and replays only the proven prefix; + * backend errors still fall through to the oracle or legacy verifier. */ const bool exactn_union = ok && ds4_session_metal_dspark_exactn_union_requested( s, (uint32_t)draft_n); @@ -71142,6 +71884,7 @@ static int ds4_session_eval_dspark_speculative_argmax( int union_accepted_prefix = 1; int union_last_top = -1; const double union_t0 = stats_enabled ? now_sec() : 0.0; + ds4_metal_exactn_union_timing union_timing = {0}; const bool static_map_cache = metal_graph_stream_decode_static_map_enabled() && @@ -71164,7 +71907,8 @@ static int ds4_session_eval_dspark_speculative_argmax( (uint32_t)draft_n, (uint32_t)start, row_tops, - row_logits); + row_logits, + stats_enabled ? &union_timing : NULL); } if (union_ok) { for (int row = 0; row + 1 < draft_n; row++) { @@ -71179,6 +71923,15 @@ static int ds4_session_eval_dspark_speculative_argmax( if (stats_enabled) { s->dspark_stats.verify_ms += (now_sec() - union_t0) * 1000.0; + if (union_timing.batch_head_attempted) { + s->dspark_stats.metal_exactn_batch_head_attempts++; + } + if (union_timing.batch_head_used) { + s->dspark_stats.metal_exactn_batch_head_uses++; + } + if (union_timing.batch_head_fallback) { + s->dspark_stats.metal_exactn_batch_head_fallbacks++; + } } if (union_ok && !union_mismatch) { @@ -71240,12 +71993,24 @@ static int ds4_session_eval_dspark_speculative_argmax( DS4_DSPARK_STATS_FINISH(); return -1; } + if (union_mismatch) { + /* The union verifier completed every row and its top-1 results + * already prove the accepted prefix. The restored pre-cycle + * frontier is the only rollback required; replay that prefix + * canonically and avoid a second N-row SSD verifier pass. */ + metal_exactn_union_verified_partial = true; + preverified_commit_drafts = union_accepted_prefix; + if (stats_enabled) { + s->dspark_stats.exactn_union_partial_replays++; + s->dspark_stats.exactn_union_legacy_verify_skips++; + } + } if (spec_log) { if (union_mismatch) { fprintf(stderr, "ds4: DSpark Metal exactN union partial " "accepted_prefix=%d verified_next=%d; restored " - "frontier for fallback\n", + "frontier and replaying without legacy verify\n", union_accepted_prefix, union_last_top); } else { @@ -71262,7 +72027,8 @@ static int ds4_session_eval_dspark_speculative_argmax( * the byte-identical state/output reference that a union-route kernel must * match before its state can be committed. */ const bool exactn = - ok && ds4_session_metal_dspark_exactn_requested( + !metal_exactn_union_verified_partial && ok && + ds4_session_metal_dspark_exactn_requested( s, (uint32_t)draft_n); if (exactn) { if (stats_enabled) s->dspark_stats.exactn_attempts++; @@ -71295,6 +72061,47 @@ static int ds4_session_eval_dspark_speculative_argmax( /* All inter-row tops matched. Advance the last accepted token too, * yielding both its exact persistent state and continuation logits. */ + if (exactn_ok && exactn_mismatch) { + /* The boundary oracle stops immediately after the last accepted + * token, so unlike the union tape its live frontier and + * sample_probs already are the canonical partial commit. */ + memcpy(s->logits, + s->sample_probs, + (size_t)DS4_N_VOCAB * sizeof(s->logits[0])); + for (int i = 0; i < exactn_accepted_prefix; i++) { + token_vec_push(&s->checkpoint, drafts[i]); + accepted[n_accept++] = drafts[i]; + } + s->checkpoint_valid = true; + ds4_session_dspark_capture_note_checkpoint(s); + if (stats_enabled) { + s->dspark_stats.partial_accepts++; + s->dspark_stats.exactn_partial_accepts++; + s->dspark_stats.accepted_draft_tokens += + (uint64_t)exactn_accepted_prefix; + ds4_dspark_stats_note_len( + s->dspark_stats.accepted_len_hist, + (uint32_t)exactn_accepted_prefix); + } + ds4_session_dspark_stats_note_saved( + s, (uint32_t)exactn_accepted_prefix); + if (stats_enabled) { + s->dspark_stats.verify_ms += + (now_sec() - exactn_t0) * 1000.0; + } + if (spec_log) { + fprintf(stderr, + "ds4: DSpark Metal exactN boundary direct-partial " + "drafted=%d committed=%d accepted_total=%d\n", + draft_n, + exactn_accepted_prefix, + n_accept); + } + spec_frontier_free(&frontier); + DS4_DSPARK_STATS_FINISH(); + return n_accept; + } + if (exactn_ok && !exactn_mismatch) { exactn_ok = metal_graph_eval_token_raw_swa( &s->graph, @@ -71345,9 +72152,9 @@ static int ds4_session_eval_dspark_speculative_argmax( return n_accept; } - /* A partial accept is deliberately not committed by this diagnostic - * path. Restore every pre-cycle frontier and let the proven legacy - * verifier plus exact replay decide and commit the accepted prefix. */ + /* Only a backend error reaches this point: partial accepts commit + * directly above. Restore every pre-cycle frontier and let the + * established legacy verifier/replay recover conservatively. */ if (stats_enabled) { s->dspark_stats.exactn_fallbacks++; if (exactn_mismatch) { @@ -71388,7 +72195,8 @@ static int ds4_session_eval_dspark_speculative_argmax( } const bool exact2 = - !cuda_exactn && !exactn && ok && + !cuda_exactn && !metal_exactn_union_verified_partial && + !exactn && ok && ds4_session_dspark_exact2_requested( s, (uint32_t)draft_n); if (exact2) { @@ -71552,7 +72360,8 @@ static int ds4_session_eval_dspark_speculative_argmax( } tp_verify_sent = true; } - if (ok && !skip_single_verify) { + if (ok && !skip_single_verify && !cuda_exactn_verified_partial && + !metal_exactn_union_verified_partial) { for (int i = 0; i < draft_n; i++) token_vec_push(&s->checkpoint, drafts[i]); verifier_may_have_mutated = true; ds4_verify_suffix_timing verify_timing; @@ -71595,7 +72404,13 @@ static int ds4_session_eval_dspark_speculative_argmax( s->graph.spec_force_sequential_compressor = saved_force_sequential; if (stats_enabled) { - s->dspark_stats.verify_ms += (now_sec() - verify_t0) * 1000.0; + const double legacy_verify_ms = + (now_sec() - verify_t0) * 1000.0; + s->dspark_stats.verify_ms += legacy_verify_ms; + if (cuda_exactn) { + s->dspark_stats.cuda_exactn_legacy_verify_ms += + legacy_verify_ms; + } s->dspark_stats.verify_upload_ms += verify_timing.upload_ms; s->dspark_stats.verify_layer_ms += verify_timing.layer_ms; s->dspark_stats.verify_head_ms += verify_timing.head_ms; @@ -71606,8 +72421,8 @@ static int ds4_session_eval_dspark_speculative_argmax( } } - int commit_drafts = 0; - if (ok) { + int commit_drafts = preverified_commit_drafts; + if (ok && commit_drafts == 0) { commit_drafts = 1; for (int i = 1; i < draft_n; i++) { if (row_tops[i - 1] != drafts[i]) break; @@ -71884,7 +72699,17 @@ static int ds4_session_eval_dspark_speculative_argmax( s->checkpoint_valid = false; if (stats_enabled) { s->dspark_stats.verifier_errors++; - s->dspark_stats.replay_ms += (now_sec() - replay_t0) * 1000.0; + const double failed_replay_ms = + (now_sec() - replay_t0) * 1000.0; + s->dspark_stats.replay_ms += failed_replay_ms; + if (cuda_exactn_verified_partial) { + s->dspark_stats.cuda_exactn_partial_replay_ms += + failed_replay_ms; + } + if (metal_exactn_union_verified_partial) { + s->dspark_stats.exactn_union_partial_replay_ms += + failed_replay_ms; + } ds4_dspark_stats_note_len(s->dspark_stats.accepted_len_hist, 0); } spec_frontier_free(&frontier); @@ -71897,7 +72722,14 @@ static int ds4_session_eval_dspark_speculative_argmax( if (drafts[i] == eos_token) break; } if (stats_enabled) { - s->dspark_stats.replay_ms += (now_sec() - replay_t0) * 1000.0; + const double replay_ms = (now_sec() - replay_t0) * 1000.0; + s->dspark_stats.replay_ms += replay_ms; + if (cuda_exactn_verified_partial) { + s->dspark_stats.cuda_exactn_partial_replay_ms += replay_ms; + } + if (metal_exactn_union_verified_partial) { + s->dspark_stats.exactn_union_partial_replay_ms += replay_ms; + } } /* Vocab-split head: the last replay eval produced only our logits half; * merge the worker's before installing them as the session logits. */ @@ -72321,17 +73153,21 @@ int ds4_test_session_eval_exact_drafts( errlen); } -/* Keep the model-backed oracle independent of stderr formatting. The fixed - * order is attempt, full, fallback, partial fallback, error fallback. */ +/* Keep the model-backed oracle independent of stderr formatting. */ int ds4_test_session_exactn_union_stats( const ds4_session *s, - uint64_t out[5]) { + uint64_t out[10]) { if (!s || !out) return -1; out[0] = s->dspark_stats.exactn_union_attempts; out[1] = s->dspark_stats.exactn_union_full_accepts; out[2] = s->dspark_stats.exactn_union_fallbacks; out[3] = s->dspark_stats.exactn_union_partial_fallbacks; out[4] = s->dspark_stats.exactn_union_error_fallbacks; + out[5] = s->dspark_stats.exactn_union_partial_replays; + out[6] = s->dspark_stats.exactn_union_legacy_verify_skips; + out[7] = s->dspark_stats.metal_exactn_batch_head_attempts; + out[8] = s->dspark_stats.metal_exactn_batch_head_uses; + out[9] = s->dspark_stats.metal_exactn_batch_head_fallbacks; return 0; } #endif diff --git a/ds4_cuda.cu b/ds4_cuda.cu index 909440784..fde3d404d 100644 --- a/ds4_cuda.cu +++ b/ds4_cuda.cu @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -125,6 +126,9 @@ static int g_cuda_exact_score_split_vec4_plain; static int g_cuda_exact_score_split_dim2; static int g_cuda_exact_score_split_fuse_inv_rope; static int g_cuda_moe_decode_graph; +static int g_cuda_direct_q2_prefill; +static int g_cuda_f16_pair_compressor_store; +static int g_cuda_is_gb10[DS4_MAX_GPUS]; static int g_current_logical_tier = -1; static int g_ssd_streaming_mode; @@ -282,6 +286,14 @@ static void cuda_decode_dispatch_env_refresh(void) { g_cuda_exact_score_split_fuse_inv_rope = getenv("DS4_CUDA_EXACT_SCORE_SPLIT_FUSE_INV_ROPE") != NULL; g_cuda_moe_decode_graph = getenv("DS4_CUDA_MOE_DECODE_GRAPH") != NULL; + /* Graph dumps consume the gate/up/mid intermediates that direct Q2 + * prefill intentionally reuses as scratch. */ + g_cuda_direct_q2_prefill = + getenv("DS4_CUDA_NO_DIRECT_Q2_PREFILL") == NULL && + getenv("DS4_METAL_GRAPH_DUMP_PREFIX") == NULL && + getenv("DS4_ROCM_GRAPH_DUMP_PREFIX") == NULL; + g_cuda_f16_pair_compressor_store = + getenv("DS4_CUDA_NO_F16_PAIR_COMPRESSOR_STORE") == NULL; } /* WITH_DEVICE(d) { ... } scope macro. @@ -375,6 +387,19 @@ struct cuda_q8_f32_range { int device_id; /* physical CUDA device id; 0 in single-tier */ }; +/* Decode-only exact compressor layout. Each lane retains its original + * contiguous 128-element accumulation chunk, while the 32 lanes' weights at + * a given iteration are interleaved for one coalesced transaction. */ +struct cuda_f16_pair_chunk32_range { + const void *host_base; + uint64_t weight0_offset; + uint64_t weight1_offset; + uint64_t in_dim; + uint32_t width; + __half2 *device_ptr; + int device_id; +}; + enum cuda_derived_kind { CUDA_DERIVED_IQ2_XXS_ALIGNED_MOE = 4, CUDA_DERIVED_Q8_0_ALIGNED_DENSE = 5, @@ -400,6 +425,8 @@ static std::vector g_q8_f16_ranges; static std::unordered_map g_q8_f16_by_offset; static std::vector g_q8_f32_ranges; static std::unordered_map g_q8_f32_by_offset; +static std::vector g_f16_pair_chunk32_ranges; +static int g_f16_pair_chunk32_disabled_after_oom; static std::vector g_derived_ranges; static const void *g_derived_replace_map; static uint64_t g_derived_artifact_bytes; @@ -909,7 +936,16 @@ extern "C" int ds4_gpu_decode_graphs_supported(void) { strcmp(s, "off") == 0 || strcmp(s, "OFF") == 0 || strcmp(s, "no") == 0 || strcmp(s, "NO") == 0 || strcmp(s, "false") == 0 || strcmp(s, "FALSE") == 0); - if (off) { + const char *oracle = + getenv("DS4_CUDA_Q4_ATTN_OUT_HC_ORACLE"); + const int oracle_on = oracle && *oracle && + strcmp(oracle, "0") != 0; + if (oracle_on) { + fprintf(stderr, + "ds4: CUDA decode graph capture disabled for Q4 " + "attention-output/HC oracle\n"); + enabled = 0; + } else if (off) { fprintf(stderr, "ds4: DS4_CUDA_DECODE_GRAPHS=%s - decode graph capture disabled\n", s); enabled = 0; } else { @@ -2575,6 +2611,19 @@ static void cuda_model_range_release_all(void) { cuda_model_load_progress_reset(); } +static void cuda_f16_pair_chunk32_release_all(void) { + int previous_device = -1; + (void)cudaGetDevice(&previous_device); + for (const cuda_f16_pair_chunk32_range &r : g_f16_pair_chunk32_ranges) { + if (!r.device_ptr) continue; + (void)cudaSetDevice(r.device_id); + (void)cudaFree(r.device_ptr); + } + if (previous_device >= 0) (void)cudaSetDevice(previous_device); + g_f16_pair_chunk32_ranges.clear(); + g_f16_pair_chunk32_disabled_after_oom = 0; +} + static void cuda_derived_range_release_all(void) { ds4_mmq_set_aligned_q81_scratch(NULL, 0); if (g_aligned_q81_scratch) { @@ -2598,7 +2647,9 @@ static int cublas_ok(cublasStatus_t st, const char *what) { } extern "C" int ds4_gpu_init_multi(const ds4_gpu_config *cfg) { + ds4_mmq_set_gb10_optimizations(0); if (!cfg || cfg->n_gpus < 1 || cfg->n_gpus > DS4_MAX_GPUS) return 0; + memset(g_cuda_is_gb10, 0, sizeof(g_cuda_is_gb10)); cuda_xdev_env_refresh(); cuda_decode_dispatch_env_refresh(); g_current_logical_tier = -1; @@ -2621,6 +2672,8 @@ extern "C" int ds4_gpu_init_multi(const ds4_gpu_config *cfg) { if (!cuda_ok(cudaSetDevice(c->device_id), "init set device")) return 0; cudaDeviceProp prop; if (cudaGetDeviceProperties(&prop, c->device_id) == cudaSuccess) { + g_cuda_is_gb10[i] = + prop.major == 12 && prop.minor == 1 && prop.integrated; fprintf(stderr, "ds4: CUDA backend initialized on %s (sm_%d%d) dev=%d\n", prop.name, prop.major, prop.minor, c->device_id); } @@ -2648,6 +2701,11 @@ extern "C" int ds4_gpu_init_multi(const ds4_gpu_config *cfg) { c->scratch = NULL; c->scratch_bytes = 0; } + int all_gb10 = cfg->n_gpus == 1; + for (int i = 0; i < cfg->n_gpus; i++) { + if (!g_cuda_is_gb10[i]) all_gb10 = 0; + } + ds4_mmq_set_gb10_optimizations(all_gb10); /* NxN peer-access matrix. * @@ -2852,6 +2910,7 @@ extern "C" void ds4_gpu_cleanup(void) { } cuda_stream_selected_cache_release(); cuda_stream_selected_stage_release(); + ds4_mmq_set_gb10_optimizations(0); g_n_gpus = 0; g_cublas_ready = 0; @@ -2872,6 +2931,7 @@ extern "C" void ds4_gpu_cleanup(void) { /* Continue with legacy global teardown below. */ cuda_model_range_release_all(); + cuda_f16_pair_chunk32_release_all(); cuda_derived_range_release_all(); cuda_q8_f16_cache_release_all(); g_q8_f16_disabled_after_oom = 0; @@ -3729,6 +3789,7 @@ extern "C" int ds4_gpu_set_model_map(const void *model_map, uint64_t model_size) if (!model_map || model_size == 0) return 0; if (g_model_host_base == model_map && g_model_registered_size == model_size) return 1; cuda_stream_selected_cache_release(); + cuda_f16_pair_chunk32_release_all(); cuda_model_range_release_all(); cuda_q8_f16_cache_release_all(); g_q8_f16_disabled_after_oom = 0; @@ -3930,6 +3991,7 @@ extern "C" int ds4_gpu_register_model_map_no_copy(const void *model_map, uint64_ if (g_model_host_base == model_map && g_model_registered_size == model_size) return 1; cuda_stream_selected_cache_release(); + cuda_f16_pair_chunk32_release_all(); cuda_model_range_release_all(); cuda_q8_f16_cache_release_all(); g_q8_f16_disabled_after_oom = 0; @@ -5031,6 +5093,290 @@ __global__ static void matmul_f16_pair_ordered_chunks_kernel( } } +/* Decode compressor projection with the state-store epilogue folded into + * the ordered F16 pair matvec. The dot products and final lane-ordered sums + * are verbatim matmul_f16_pair_ordered_chunks_kernel; only lane 0 performs + * the independent state writes that otherwise require a second launch. */ +__global__ static void matmul_f16_pair_compressor_store_ordered_chunks_kernel( + float *out_kv, + float *out_score, + float *state_kv, + float *state_score, + const __half *w_kv, + const __half *w_score, + const float *x, + const void *ape, + uint32_t ape_type, + uint64_t in_dim, + uint32_t width, + uint32_t ratio, + uint32_t pos) { + const uint32_t row = blockIdx.x; + if (row >= width) return; + + __shared__ float partial_kv[32]; + __shared__ float partial_score[32]; + const uint32_t tid = threadIdx.x; + float sum_kv = 0.0f; + float sum_score = 0.0f; + const uint64_t chunk = (in_dim + 31u) / 32u; + const uint64_t k0 = (uint64_t)tid * chunk; + uint64_t k1 = k0 + chunk; + if (k1 > in_dim) k1 = in_dim; + const __half *wr_kv = w_kv + (uint64_t)row * in_dim; + const __half *wr_score = w_score + (uint64_t)row * in_dim; + for (uint64_t i = k0; i < k1; i++) { + const float xv = x[i]; + sum_kv += __half2float(wr_kv[i]) * xv; + sum_score += __half2float(wr_score[i]) * xv; + } + partial_kv[tid] = sum_kv; + partial_score[tid] = sum_score; + __syncthreads(); + if (tid == 0u) { + float total_kv = 0.0f; + float total_score = 0.0f; + for (uint32_t i = 0; i < 32u; i++) { + total_kv += partial_kv[i]; + total_score += partial_score[i]; + } + out_kv[row] = total_kv; + out_score[row] = total_score; + const uint32_t pos_mod = pos % ratio; + const uint32_t dst_row = ratio == 4u ? ratio + pos_mod : pos_mod; + const uint64_t ape_index = (uint64_t)pos_mod * width + row; + const float ape_value = ape_type == 1u + ? __half2float(((const __half *)ape)[ape_index]) + : ((const float *)ape)[ape_index]; + state_kv[(uint64_t)dst_row * width + row] = total_kv; + state_score[(uint64_t)dst_row * width + row] = + total_score + ape_value; + } +} + +__global__ static void f16_pair_chunk32_repack_kernel( + __half2 *dst, + const __half *w0, + const __half *w1, + uint64_t in_dim, + uint32_t width) { + const uint64_t gid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + const uint64_t count = (uint64_t)width * in_dim; + if (gid >= count) return; + const uint64_t row = gid / in_dim; + const uint64_t k = gid - row * in_dim; + const uint64_t chunk = in_dim / 32u; + const uint64_t lane = k / chunk; + const uint64_t iter = k - lane * chunk; + const uint64_t dst_idx = (row * chunk + iter) * 32u + lane; + dst[dst_idx] = __halves2half2(w0[gid], w1[gid]); +} + +__global__ static void matmul_f16_pair_compressor_store_chunk32_kernel( + float *out_kv, + float *out_score, + float *state_kv, + float *state_score, + const __half2 *w_pair, + const float *x, + const void *ape, + uint32_t ape_type, + uint64_t in_dim, + uint32_t width, + uint32_t ratio, + uint32_t pos) { + const uint32_t row = blockIdx.x; + if (row >= width) return; + + __shared__ float partial_kv[32]; + __shared__ float partial_score[32]; + const uint32_t tid = threadIdx.x; + float sum_kv = 0.0f; + float sum_score = 0.0f; + const uint64_t chunk = in_dim / 32u; + const uint64_t k0 = (uint64_t)tid * chunk; + const __half2 *wr = w_pair + (uint64_t)row * in_dim; + for (uint64_t j = 0; j < chunk; j++) { + const float xv = x[k0 + j]; + const __half2 wp = wr[j * 32u + tid]; + sum_kv += __half2float(__low2half(wp)) * xv; + sum_score += __half2float(__high2half(wp)) * xv; + } + partial_kv[tid] = sum_kv; + partial_score[tid] = sum_score; + __syncthreads(); + if (tid == 0u) { + float total_kv = 0.0f; + float total_score = 0.0f; + for (uint32_t i = 0; i < 32u; i++) { + total_kv += partial_kv[i]; + total_score += partial_score[i]; + } + out_kv[row] = total_kv; + out_score[row] = total_score; + const uint32_t pos_mod = pos % ratio; + const uint32_t dst_row = ratio == 4u ? ratio + pos_mod : pos_mod; + const uint64_t ape_index = (uint64_t)pos_mod * width + row; + const float ape_value = ape_type == 1u + ? __half2float(((const __half *)ape)[ape_index]) + : ((const float *)ape)[ape_index]; + state_kv[(uint64_t)dst_row * width + row] = total_kv; + state_score[(uint64_t)dst_row * width + row] = total_score + ape_value; + } +} + +__global__ static void matmul_f16_pair_compressor_store_chunk32_prefetch8_kernel( + float *out_kv, + float *out_score, + float *state_kv, + float *state_score, + const __half2 *w_pair, + const float *x, + const void *ape, + uint32_t ape_type, + uint64_t in_dim, + uint32_t width, + uint32_t ratio, + uint32_t pos) { + const uint32_t row = blockIdx.x; + if (row >= width) return; + + __shared__ float partial_kv[32]; + __shared__ float partial_score[32]; + const uint32_t tid = threadIdx.x; + float sum_kv = 0.0f; + float sum_score = 0.0f; + const uint64_t chunk = in_dim / 32u; + const uint64_t k0 = (uint64_t)tid * chunk; + const __half2 *wr = w_pair + (uint64_t)row * in_dim; + for (uint64_t j = 0; j < chunk; j += 8u) { + const float xv0 = x[k0 + j + 0u]; + const float xv1 = x[k0 + j + 1u]; + const float xv2 = x[k0 + j + 2u]; + const float xv3 = x[k0 + j + 3u]; + const float xv4 = x[k0 + j + 4u]; + const float xv5 = x[k0 + j + 5u]; + const float xv6 = x[k0 + j + 6u]; + const float xv7 = x[k0 + j + 7u]; + const __half2 wp0 = wr[(j + 0u) * 32u + tid]; + const __half2 wp1 = wr[(j + 1u) * 32u + tid]; + const __half2 wp2 = wr[(j + 2u) * 32u + tid]; + const __half2 wp3 = wr[(j + 3u) * 32u + tid]; + const __half2 wp4 = wr[(j + 4u) * 32u + tid]; + const __half2 wp5 = wr[(j + 5u) * 32u + tid]; + const __half2 wp6 = wr[(j + 6u) * 32u + tid]; + const __half2 wp7 = wr[(j + 7u) * 32u + tid]; + sum_kv += __half2float(__low2half(wp0)) * xv0; + sum_score += __half2float(__high2half(wp0)) * xv0; + sum_kv += __half2float(__low2half(wp1)) * xv1; + sum_score += __half2float(__high2half(wp1)) * xv1; + sum_kv += __half2float(__low2half(wp2)) * xv2; + sum_score += __half2float(__high2half(wp2)) * xv2; + sum_kv += __half2float(__low2half(wp3)) * xv3; + sum_score += __half2float(__high2half(wp3)) * xv3; + sum_kv += __half2float(__low2half(wp4)) * xv4; + sum_score += __half2float(__high2half(wp4)) * xv4; + sum_kv += __half2float(__low2half(wp5)) * xv5; + sum_score += __half2float(__high2half(wp5)) * xv5; + sum_kv += __half2float(__low2half(wp6)) * xv6; + sum_score += __half2float(__high2half(wp6)) * xv6; + sum_kv += __half2float(__low2half(wp7)) * xv7; + sum_score += __half2float(__high2half(wp7)) * xv7; + } + partial_kv[tid] = sum_kv; + partial_score[tid] = sum_score; + __syncthreads(); + if (tid == 0u) { + float total_kv = 0.0f; + float total_score = 0.0f; + for (uint32_t i = 0; i < 32u; i++) { + total_kv += partial_kv[i]; + total_score += partial_score[i]; + } + out_kv[row] = total_kv; + out_score[row] = total_score; + const uint32_t pos_mod = pos % ratio; + const uint32_t dst_row = ratio == 4u ? ratio + pos_mod : pos_mod; + const uint64_t ape_index = (uint64_t)pos_mod * width + row; + const float ape_value = ape_type == 1u + ? __half2float(((const __half *)ape)[ape_index]) + : ((const float *)ape)[ape_index]; + state_kv[(uint64_t)dst_row * width + row] = total_kv; + state_score[(uint64_t)dst_row * width + row] = total_score + ape_value; + } +} + +static const __half2 *cuda_f16_pair_chunk32_get( + const void *model_map, + uint64_t weight0_offset, + uint64_t weight1_offset, + const __half *w0, + const __half *w1, + uint64_t in_dim, + uint32_t width, + int logical_tier) { + const int device_id = (logical_tier >= 0 && logical_tier < g_n_gpus) + ? g_gpu[logical_tier].device_id : logical_tier; + for (const cuda_f16_pair_chunk32_range &r : g_f16_pair_chunk32_ranges) { + if (r.host_base == model_map && + r.weight0_offset == weight0_offset && + r.weight1_offset == weight1_offset && + r.in_dim == in_dim && r.width == width && + r.device_id == device_id) { + return r.device_ptr; + } + } + if (g_f16_pair_chunk32_disabled_after_oom) return NULL; + /* Lazy cudaMalloc is forbidden while the decode island capture is active. + * Querying the legacy stream would miss capture on g_decode_graph_stream. */ + if (g_decode_graph_capturing) return NULL; + + cudaStreamCaptureStatus capture_status = cudaStreamCaptureStatusNone; + if (cudaStreamIsCapturing(0, &capture_status) != cudaSuccess || + capture_status != cudaStreamCaptureStatusNone) { + (void)cudaGetLastError(); + return NULL; + } + if (in_dim == 0u || (in_dim % 32u) != 0u || + (uint64_t)width > UINT64_MAX / in_dim) return NULL; + const uint64_t count = (uint64_t)width * in_dim; + if (count > SIZE_MAX / sizeof(__half2)) return NULL; + + int previous_device = -1; + if (cudaGetDevice(&previous_device) != cudaSuccess || + cudaSetDevice(device_id) != cudaSuccess) { + (void)cudaGetLastError(); + return NULL; + } + __half2 *device_ptr = NULL; + cudaError_t err = cudaMalloc(&device_ptr, (size_t)count * sizeof(__half2)); + if (err != cudaSuccess) { + fprintf(stderr, + "ds4: CUDA exact F16 compressor transpose disabled after " + "allocation failure (%.2f MiB): %s\n", + (double)(count * sizeof(__half2)) / 1048576.0, + cudaGetErrorString(err)); + (void)cudaGetLastError(); + g_f16_pair_chunk32_disabled_after_oom = 1; + if (previous_device >= 0) (void)cudaSetDevice(previous_device); + return NULL; + } + const uint64_t blocks = (count + 255u) / 256u; + f16_pair_chunk32_repack_kernel<<<(unsigned)blocks, 256>>>( + device_ptr, w0, w1, in_dim, width); + err = cudaGetLastError(); + if (err != cudaSuccess) { + (void)cudaFree(device_ptr); + if (previous_device >= 0) (void)cudaSetDevice(previous_device); + return NULL; + } + g_f16_pair_chunk32_ranges.push_back({ + model_map, weight0_offset, weight1_offset, in_dim, width, + device_ptr, device_id}); + if (previous_device >= 0) (void)cudaSetDevice(previous_device); + return device_ptr; +} + __global__ static void matmul_f32_kernel( float *out, const float *w, @@ -5139,6 +5485,21 @@ __device__ __forceinline__ static int32_t dot_i8_block(const int8_t *a, const in return dot; } +__device__ __forceinline__ static int32_t dot_i8x32_aligned_int4( + const int4 &a0, const int4 &a1, const int8_t *b) { + const int32_t *bi = (const int32_t *)b; + int32_t dot = 0; + dot = __dp4a(a0.x, bi[0], dot); + dot = __dp4a(a0.y, bi[1], dot); + dot = __dp4a(a0.z, bi[2], dot); + dot = __dp4a(a0.w, bi[3], dot); + dot = __dp4a(a1.x, bi[4], dot); + dot = __dp4a(a1.y, bi[5], dot); + dot = __dp4a(a1.z, bi[6], dot); + dot = __dp4a(a1.w, bi[7], dot); + return dot; +} + __global__ static DS4_CUDA_UNUSED void matmul_q8_0_kernel( float *out, const unsigned char *w, @@ -5487,6 +5848,51 @@ __global__ static void matmul_q8_0_pair_preq_warp8_kernel( } } +__global__ static void matmul_q8_0_pair_aligned_preq_warp8_kernel( + float *out0, + float *out1, + const int4 *w0_qs, + const __half *w0_dq, + const int4 *w1_qs, + const __half *w1_dq, + const int8_t *xq, + const float *xscale, + uint64_t out0_dim, + uint64_t out1_dim, + uint64_t blocks) { + const uint64_t row = (uint64_t)blockIdx.x * 8u + (threadIdx.x >> 5u); + const uint64_t tok = (uint64_t)blockIdx.y; + const uint32_t lane = threadIdx.x & 31u; + if (row >= out0_dim && row >= out1_dim) return; + float acc0 = 0.0f; + float acc1 = 0.0f; + const int8_t *xqr = xq + tok * blocks * 32u; + const float *xsr = xscale + tok * blocks; + const uint64_t rbase = row * blocks; + for (uint64_t b = lane; b < blocks; b += 32u) { + const int8_t *xqb = xqr + b * 32u; + const float xs = xsr[b]; + if (row < out0_dim) { + const int4 q0 = w0_qs[(rbase + b) * 2u]; + const int4 q1 = w0_qs[(rbase + b) * 2u + 1u]; + const int32_t dot = dot_i8x32_aligned_int4(q0, q1, xqb); + acc0 += __half2float(w0_dq[rbase + b]) * xs * (float)dot; + } + if (row < out1_dim) { + const int4 q0 = w1_qs[(rbase + b) * 2u]; + const int4 q1 = w1_qs[(rbase + b) * 2u + 1u]; + const int32_t dot = dot_i8x32_aligned_int4(q0, q1, xqb); + acc1 += __half2float(w1_dq[rbase + b]) * xs * (float)dot; + } + } + acc0 = warp_sum_f32(acc0); + acc1 = warp_sum_f32(acc1); + if (lane == 0u) { + if (row < out0_dim) out0[tok * out0_dim + row] = acc0; + if (row < out1_dim) out1[tok * out1_dim + row] = acc1; + } +} + __global__ static void shared_mid_q8_0_preq_warp8_exact_kernel( float *mid, const unsigned char *gate_w, @@ -5786,6 +6192,72 @@ __global__ static void matmul_q8_0_hc_expand_preq_warp8_kernel( } } +__global__ static void matmul_q8_0_hc_expand_aligned_preq_warp8_kernel( + float *out_hc, + float *block_out, + const float *block_add, + const float *block_add2, + const float *owned_home_slots, + const float *owned_peer_packed, + const int32_t *owned_selected, + const float *residual_hc, + const float *split, + const int4 *w_qs, + const __half *w_dq, + const int8_t *xq, + const float *xscale, + uint64_t out_dim, + uint32_t n_embd, + uint32_t n_hc, + uint64_t blocks, + int has_add, + int has_add2, + int has_owned_slots, + uint32_t owned_expert_split) { + const uint64_t row = (uint64_t)blockIdx.x * 8u + (threadIdx.x >> 5u); + const uint32_t lane = threadIdx.x & 31u; + if (row >= out_dim) return; + const uint64_t rbase = row * blocks; + float acc = 0.0f; + for (uint64_t b = lane; b < blocks; b += 32u) { + const int4 w0 = w_qs[(rbase + b) * 2u]; + const int4 w1 = w_qs[(rbase + b) * 2u + 1u]; + const int32_t dot = dot_i8x32_aligned_int4(w0, w1, xq + b * 32u); + acc += __half2float(w_dq[rbase + b]) * xscale[b] * (float)dot; + } + acc = warp_sum_f32(acc); + if (lane == 0) { + const uint32_t d = (uint32_t)row; + block_out[d] = acc; + float block_v = acc; + if (has_owned_slots) { + const float routed = moe_owned_packed_combine_row( + owned_home_slots, + owned_peer_packed, + owned_selected, + d, + (uint32_t)out_dim, + owned_expert_split); + block_v = __fadd_rn(block_v, routed); + } else if (has_add) { + float add_v = block_add[d]; + if (has_add2) add_v += block_add2[d]; + block_v += add_v; + } + const float *post = split + n_hc; + const float *comb = split + 2u * n_hc; + for (uint32_t dst_hc = 0; dst_hc < n_hc; dst_hc++) { + float hc_acc = block_v * post[dst_hc]; + for (uint32_t src_hc = 0; src_hc < n_hc; src_hc++) { + const float comb_v = comb[dst_hc + (uint64_t)src_hc * n_hc]; + const float res_v = residual_hc[(uint64_t)src_hc * n_embd + d]; + hc_acc += comb_v * res_v; + } + out_hc[(uint64_t)dst_hc * n_embd + d] = hc_acc; + } + } +} + __global__ static void matmul_q8_0_kslice_hc_expand_add_preq_warp8_kernel( float *out_hc, float *block_out, @@ -6520,6 +6992,39 @@ __global__ static void grouped_q8_0_a_preq_warp8_kernel( if (lane == 0) low[tok * low_dim + row] = acc; } +__global__ static void grouped_q8_0_a_aligned_preq_warp8_kernel( + float *low, + const int4 *w_qs, + const __half *w_dq, + const int8_t *xq, + const float *xscale, + uint64_t rank, + uint32_t n_groups, + uint32_t n_tokens, + uint64_t blocks) { + const uint64_t row = (uint64_t)blockIdx.x * 8u + (threadIdx.x >> 5u); + const uint64_t tok = (uint64_t)blockIdx.y; + const uint32_t lane = threadIdx.x & 31u; + const uint64_t low_dim = (uint64_t)n_groups * rank; + if (row >= low_dim || tok >= n_tokens) return; + + const uint64_t group = row / rank; + const uint64_t xrow = tok * (uint64_t)n_groups + group; + const int8_t *xqr = xq + xrow * blocks * 32u; + const float *xsr = xscale + xrow * blocks; + const uint64_t rbase = row * blocks; + float acc = 0.0f; + for (uint64_t b = lane; b < blocks; b += 32u) { + const int4 q0 = w_qs[(rbase + b) * 2u]; + const int4 q1 = w_qs[(rbase + b) * 2u + 1u]; + const int32_t dot = dot_i8x32_aligned_int4( + q0, q1, xqr + b * 32u); + acc += __half2float(w_dq[rbase + b]) * xsr[b] * (float)dot; + } + acc = warp_sum_f32(acc); + if (lane == 0u) low[tok * low_dim + row] = acc; +} + __global__ static void grouped_q8_0_a_preq_warp8_tok2_kernel( float *low, const unsigned char *w, @@ -11875,6 +12380,90 @@ __global__ static void hc_split_weighted_sum_norm_fused_kernel( } } +/* Split form of the exact single-row HC weighted-sum + RMS kernel. The + * reference kernel uses one 256-thread CTA; at n_embd=4096 that leaves one CTA + * to read 64 KiB and serialize both phases. Sixteen partial CTAs compute the + * same per-column weighted sums. A small reduction CTA then replays the + * original per-thread ascending-column FMA chain and shared-memory tree from + * those exact outputs, and a sixteen-CTA store writes the normalized values. + * The output tensors and floating-point reduction DAG are bit-identical. */ +__global__ static void hc_split_weighted_sum_norm_fused_partial4096_kernel( + float *out, + float *split, + const float *mix, + const float *residual_hc, + const float *scale, + const float *base, + uint32_t sinkhorn_iters, + float epsv) { + constexpr uint32_t n_embd = 4096u; + constexpr uint32_t mix_hc = 24u; + constexpr uint32_t tile = 256u; + const uint32_t d = threadIdx.x; + const uint32_t tile0 = blockIdx.x * tile; + + __shared__ float sp[mix_hc]; + if (d == 0u) { + hc4_split_one(sp, mix, scale, base, sinkhorn_iters, epsv); + } + __syncthreads(); + if (blockIdx.x == 0u && d < mix_hc) { + split[d] = sp[d]; + } + +#pragma unroll + for (uint32_t j = 0; j < tile / 256u; j++) { + const uint32_t col = tile0 + j * 256u + d; + float acc = 0.0f; +#pragma unroll + for (uint32_t h = 0; h < 4u; h++) { + acc += residual_hc[(uint64_t)h * n_embd + col] * sp[h]; + } + out[col] = acc; + } +} + +__global__ static void hc_split_weighted_sum_norm_fused_reduce4096_kernel( + const float *out, + float *norm_scale, + float norm_eps) { + constexpr uint32_t n_embd = 4096u; + const uint32_t d = threadIdx.x; + + __shared__ float partial[256]; + float sum = 0.0f; +#pragma unroll + for (uint32_t j = 0; j < n_embd / 256u; j++) { + const uint32_t col = d + j * 256u; + sum += out[col] * out[col]; + } + partial[d] = sum; + __syncthreads(); + for (uint32_t stride = blockDim.x >> 1; stride > 0; stride >>= 1) { + if (d < stride) partial[d] += partial[d + stride]; + __syncthreads(); + } + if (d == 0u) { + norm_scale[0] = rsqrtf(partial[0] / (float)n_embd + norm_eps); + } +} + +__global__ static void hc_split_weighted_sum_norm_fused_store4096_kernel( + const float *out, + float *norm_out, + const float *norm_w, + const float *norm_scale) { + constexpr uint32_t tile = 256u; + const uint32_t d = threadIdx.x; + const uint32_t tile0 = blockIdx.x * tile; + const float scale = norm_scale[0]; +#pragma unroll + for (uint32_t j = 0; j < tile / 256u; j++) { + const uint32_t col = tile0 + j * 256u + d; + norm_out[col] = out[col] * scale * norm_w[col]; + } +} + __global__ static void output_hc_weights_kernel( float *out, const float *pre, @@ -13352,18 +13941,32 @@ __device__ __forceinline__ static void top2_insert_candidate( } /* DSpark markov chain step: out = argmax_i(logits[i] + dot(w2[i], w1[prev])) - * over the vocab, entirely on-device (logits row stays resident; the chain - * loop only reads back 4 bytes per draft). w1/w2 are q8_0 with 272-byte rows - * (8 blocks of 32). Single block; ~35 MB w2 read per step. */ + * over the vocab, entirely on-device. w1/w2 are q8_0 with 272-byte rows + * (8 blocks of 32); the legacy wrapper reads one key after each step, while + * the batched proposer below feeds the selected token directly to the next + * launch and performs one result readback for the whole chain. */ __global__ static void dspark_markov_argmax_kernel( unsigned long long *out_key, const float *logits, - const unsigned char *w1_row, + const unsigned char *w1, const unsigned char *w2, uint32_t vocab, - uint32_t rank_blocks) { + uint32_t rank_blocks, + const uint32_t *prev_token_device, + const uint32_t *active_device) { __shared__ float state[256]; + __shared__ uint32_t prev_token; + __shared__ uint32_t active; const uint32_t tid = threadIdx.x; + if (tid == 0u) { + if (prev_token_device) prev_token = *prev_token_device; + active = active_device ? *active_device : 1u; + } + __syncthreads(); + if (active == 0u) return; + const unsigned char *w1_row = prev_token_device + ? w1 + (uint64_t)prev_token * rank_blocks * 34u + : w1; if (tid < rank_blocks * 32u) { const uint32_t b = tid >> 5, k = tid & 31u; const unsigned char *blk = w1_row + (uint64_t)b * 34u; @@ -13420,6 +14023,168 @@ __global__ static void dspark_markov_argmax_kernel( } } +/* Device-resident DSpark proposer tail. ds4_cuda.cu intentionally does not + * include ds4_gpu.h, so keep this result declaration byte-for-byte identical + * to ds4_gpu_dspark_device_proposal there. */ +#define DS4_GPU_DSPARK_MAX_DRAFTS 6u +#define DS4_GPU_DSPARK_DEVICE_PROPOSAL_BYTES 2048u +typedef struct { + int32_t tokens[DS4_GPU_DSPARK_MAX_DRAFTS]; + float confidence_logits[DS4_GPU_DSPARK_MAX_DRAFTS]; + uint32_t proposal_len; + uint32_t confidence_len; + uint32_t status; + uint32_t reserved; +} ds4_gpu_dspark_device_proposal; +static_assert(sizeof(ds4_gpu_dspark_device_proposal) == 64u, + "DSpark device proposal ABI must match ds4_gpu.h"); +static_assert(offsetof(ds4_gpu_dspark_device_proposal, tokens) == 0u && + offsetof(ds4_gpu_dspark_device_proposal, + confidence_logits) == 24u && + offsetof(ds4_gpu_dspark_device_proposal, proposal_len) == 48u && + offsetof(ds4_gpu_dspark_device_proposal, confidence_len) == 52u && + offsetof(ds4_gpu_dspark_device_proposal, status) == 56u && + offsetof(ds4_gpu_dspark_device_proposal, reserved) == 60u, + "DSpark device proposal field offsets must match ds4_gpu.h"); + +typedef struct { + unsigned long long best_key[DS4_GPU_DSPARK_MAX_DRAFTS]; + uint32_t prev_token; + uint32_t status; + uint32_t active; + uint32_t proposal_len; + uint32_t confidence_len; + int32_t tokens[DS4_GPU_DSPARK_MAX_DRAFTS]; + float confidence_logits[DS4_GPU_DSPARK_MAX_DRAFTS]; +} cuda_dspark_device_proposal_state; +static_assert(sizeof(ds4_gpu_dspark_device_proposal) + + sizeof(cuda_dspark_device_proposal_state) <= + DS4_GPU_DSPARK_DEVICE_PROPOSAL_BYTES, + "DSpark device proposal storage is too small"); + +__global__ static void dspark_device_proposal_init_kernel( + cuda_dspark_device_proposal_state *state, + uint32_t first_prev_token) { + if (blockIdx.x != 0u || threadIdx.x != 0u) return; + state->prev_token = first_prev_token; + state->status = 1u; + state->active = 1u; + state->proposal_len = 0u; + state->confidence_len = 0u; + for (uint32_t i = 0; i < DS4_GPU_DSPARK_MAX_DRAFTS; i++) { + state->best_key[i] = 0ull; + state->tokens[i] = -1; + state->confidence_logits[i] = 0.0f; + } +} + +__device__ __forceinline__ static float dspark_q8_value( + const unsigned char *row, + uint32_t index) { + const unsigned char *blk = row + (uint64_t)(index >> 5u) * 34u; + const float d = __half2float(*(const __half *)blk); + return d * (float)((const int8_t *)(blk + 2))[index & 31u]; +} + +/* The confidence projection has one Q8_0 output row. A single thread keeps + * activation quantization and block accumulation in a stable, explicit + * order; this tail is tiny beside the vocab-wide Markov scan and avoids + * materializing or reading back the [hidden || w1(prev)] feature vector. */ +__global__ static void dspark_device_confidence_q8_kernel( + cuda_dspark_device_proposal_state *state, + const float *hidden_rows, + const unsigned char *w1, + const unsigned char *confidence, + uint32_t rank_blocks, + uint32_t hidden_dim, + uint32_t draft, + float confidence_threshold, + int reuse_confidence0, + float confidence0) { + if (blockIdx.x != 0u || threadIdx.x != 0u || state->status == 0u) return; + if (state->active == 0u) return; + float acc = 0.0f; + if (draft == 0u && reuse_confidence0) { + acc = confidence0; + } else { + const uint32_t hidden_blocks = hidden_dim / 32u; + const uint32_t feature_blocks = hidden_blocks + rank_blocks; + const uint64_t w1_row_bytes = (uint64_t)rank_blocks * 34u; + const unsigned char *w1_row = + w1 + (uint64_t)state->prev_token * w1_row_bytes; + const float *hidden = hidden_rows + (uint64_t)draft * hidden_dim; + + for (uint32_t b = 0; b < feature_blocks; b++) { + float values[32]; + float amax = 0.0f; + for (uint32_t k = 0; k < 32u; k++) { + const uint32_t feature = b * 32u + k; + const float v = feature < hidden_dim + ? hidden[feature] + : dspark_q8_value(w1_row, feature - hidden_dim); + values[k] = v; + const float av = fabsf(v); + if (av > amax) amax = av; + } + const float xscale = amax / 127.0f; + const float inv = xscale != 0.0f ? 1.0f / xscale : 0.0f; + const unsigned char *wblk = confidence + (uint64_t)b * 34u; + const float wscale = __half2float(*(const __half *)wblk); + const int8_t *wq = (const int8_t *)(wblk + 2); + int isum = 0; + for (uint32_t k = 0; k < 32u; k++) { + int q = __float2int_rn(values[k] * inv); + if (q > 127) q = 127; + if (q < -128) q = -128; + isum += (int)wq[k] * q; + } + /* Explicit rounded operations prevent contraction from silently + * changing the documented block-by-block accumulation order. */ + const float scaled = __fmul_rn(__fmul_rn(wscale, xscale), + (float)isum); + acc = __fadd_rn(acc, scaled); + } + } + state->confidence_logits[draft] = acc; + state->confidence_len = draft + 1u; + const float e = expf(acc >= 0.0f ? -acc : acc); + const float confidence_value = acc >= 0.0f + ? 1.0f / (1.0f + e) + : e / (1.0f + e); + if (confidence_value < confidence_threshold) state->active = 0u; +} + +__global__ static void dspark_device_proposal_advance_kernel( + cuda_dspark_device_proposal_state *state, + uint32_t vocab, + uint32_t draft) { + if (blockIdx.x != 0u || threadIdx.x != 0u || state->status == 0u || + state->active == 0u) return; + const unsigned long long key = state->best_key[draft]; + const uint32_t token = ~(uint32_t)(key & 0xffffffffu); + if (key == 0ull || token >= vocab) { + state->status = 0u; + return; + } + state->tokens[draft] = (int32_t)token; + state->prev_token = token; + state->proposal_len = draft + 1u; +} + +__global__ static void dspark_device_proposal_export_kernel( + ds4_gpu_dspark_device_proposal *out, + const cuda_dspark_device_proposal_state *state) { + if (blockIdx.x != 0u || threadIdx.x != 0u) return; + for (uint32_t i = 0; i < DS4_GPU_DSPARK_MAX_DRAFTS; i++) { + out->tokens[i] = state->tokens[i]; + out->confidence_logits[i] = state->confidence_logits[i]; + } + out->proposal_len = state->proposal_len; + out->confidence_len = state->confidence_len; + out->status = state->status; + out->reserved = 0u; +} + __global__ static void indexer_top1_kernel( uint32_t *selected, const float *scores, @@ -14329,13 +15094,171 @@ extern "C" int ds4_gpu_dspark_markov_argmax_tensor( dspark_markov_argmax_kernel<<<128, 256>>>( (unsigned long long *)out_idx->ptr, (const float *)logits_row->ptr, - w1_row, w2, vocab, rank_blocks); + w1_row, w2, vocab, rank_blocks, NULL, NULL); rc = cuda_ok(cudaGetLastError(), "dspark markov argmax launch"); } if (logical_tier != dev_save) (void)cudaSetDevice(dev_save); return rc; } +static int dspark_device_proposer_env_enabled(void) { + const char *enable = getenv("DS4_CUDA_DSPARK_DEVICE_PROPOSER"); + if (!enable || !enable[0] || strcmp(enable, "0") == 0 || + strcmp(enable, "off") == 0 || strcmp(enable, "false") == 0) { + return 0; + } + return getenv("DS4_CUDA_DSPARK_NO_DEVICE_PROPOSER") == NULL; +} + +extern "C" int ds4_gpu_dspark_markov_confidence_q8_tensor( + ds4_gpu_tensor *out_result, + const ds4_gpu_tensor *logits_rows, + const ds4_gpu_tensor *hidden_rows, + const void *model_map, + uint64_t model_size, + uint64_t w1_offset, + uint64_t w2_offset, + uint64_t confidence_offset, + uint32_t first_prev_token, + uint32_t vocab, + uint32_t rank, + uint32_t hidden_dim, + uint32_t n_drafts, + float confidence_threshold, + int reuse_confidence0, + float confidence0) { + if (!dspark_device_proposer_env_enabled() || + !out_result || !logits_rows || !hidden_rows || !model_map || + vocab == 0u || first_prev_token >= vocab || + rank == 0u || rank > 256u || (rank & 31u) != 0u || + hidden_dim == 0u || (hidden_dim & 31u) != 0u || + n_drafts == 0u || n_drafts > DS4_GPU_DSPARK_MAX_DRAFTS || + !(confidence_threshold > 0.0f && confidence_threshold <= 1.0f) || + g_decode_graph_capturing || + out_result->bytes < DS4_GPU_DSPARK_DEVICE_PROPOSAL_BYTES) { + return 0; + } + if ((uint64_t)n_drafts > UINT64_MAX / vocab / sizeof(float) || + logits_rows->bytes < + (uint64_t)n_drafts * vocab * sizeof(float) || + (uint64_t)n_drafts > UINT64_MAX / hidden_dim / sizeof(float) || + hidden_rows->bytes < + (uint64_t)n_drafts * hidden_dim * sizeof(float)) { + return 0; + } + + const int logical_tier = ds4_tensor_device_idx(logits_rows); + if (logical_tier < 0 || logical_tier >= g_n_gpus || + ds4_tensor_device_idx(hidden_rows) != logical_tier || + ds4_tensor_device_idx(out_result) != logical_tier) { + return 0; + } + const uint32_t rank_blocks = rank / 32u; + const uint64_t markov_row_bytes = (uint64_t)rank_blocks * 34u; + const uint64_t markov_bytes = (uint64_t)vocab * markov_row_bytes; + const uint64_t feature_dim = (uint64_t)hidden_dim + rank; + if (feature_dim > UINT32_MAX || (feature_dim & 31u) != 0u) return 0; + const uint64_t confidence_bytes = (feature_dim / 32u) * 34u; + if (w1_offset > model_size || markov_bytes > model_size - w1_offset || + w2_offset > model_size || markov_bytes > model_size - w2_offset || + confidence_offset > model_size || + confidence_bytes > model_size - confidence_offset) { + return 0; + } + + int saved_device = -1; + if (cudaGetDevice(&saved_device) != cudaSuccess) return 0; + const int target_device = g_gpu[logical_tier].device_id; + if (target_device != saved_device && + cudaSetDevice(target_device) != cudaSuccess) { + return 0; + } + + const unsigned char *w1 = + (const unsigned char *)cuda_resolve_weight_ptr( + model_map, w1_offset, markov_bytes, logical_tier, + "dspark device proposer w1"); + const unsigned char *w2 = + (const unsigned char *)cuda_resolve_weight_ptr( + model_map, w2_offset, markov_bytes, logical_tier, + "dspark device proposer w2"); + const unsigned char *confidence = + (const unsigned char *)cuda_resolve_weight_ptr( + model_map, confidence_offset, confidence_bytes, logical_tier, + "dspark device proposer confidence"); + if (!w1 || !w2 || !confidence) { + if (target_device != saved_device) (void)cudaSetDevice(saved_device); + return 0; + } + + /* Session-owned storage prevents concurrent proposer calls from aliasing + * the backend's global temporary slab. The compact public result occupies + * the first 64 bytes; the private state starts at an 8-byte-aligned offset + * immediately after it and lives through the caller's synchronous read. */ + cuda_dspark_device_proposal_state *state = + (cuda_dspark_device_proposal_state *)( + (unsigned char *)out_result->ptr + + sizeof(ds4_gpu_dspark_device_proposal)); + + int rc = 0; + const cudaStream_t stream = cuda_decode_stream(); + dspark_device_proposal_init_kernel<<<1, 1, 0, stream>>>( + state, first_prev_token); + rc = cuda_ok(cudaGetLastError(), "dspark device proposer init launch"); + for (uint32_t draft = 0; rc && draft < n_drafts; draft++) { + dspark_device_confidence_q8_kernel<<<1, 1, 0, stream>>>( + state, + (const float *)hidden_rows->ptr, + w1, + confidence, + rank_blocks, + hidden_dim, + draft, + confidence_threshold, + reuse_confidence0, + confidence0); + rc = cuda_ok(cudaGetLastError(), + "dspark device confidence launch"); + if (!rc) break; + dspark_markov_argmax_kernel<<<128, 256, 0, stream>>>( + &state->best_key[draft], + (const float *)logits_rows->ptr + (uint64_t)draft * vocab, + w1, + w2, + vocab, + rank_blocks, + &state->prev_token, + &state->active); + rc = cuda_ok(cudaGetLastError(), + "dspark device markov launch"); + if (!rc) break; + dspark_device_proposal_advance_kernel<<<1, 1, 0, stream>>>( + state, vocab, draft); + rc = cuda_ok(cudaGetLastError(), + "dspark device proposer advance launch"); + } + if (rc) { + dspark_device_proposal_export_kernel<<<1, 1, 0, stream>>>( + (ds4_gpu_dspark_device_proposal *)out_result->ptr, + state); + rc = cuda_ok(cudaGetLastError(), + "dspark device proposer export launch"); + } + if (target_device != saved_device) (void)cudaSetDevice(saved_device); + + if (rc) { + static int logged = 0; + if (!logged) { + logged = 1; + fprintf(stderr, + "ds4: CUDA DSpark device proposer enabled " + "(Q8 confidence/Markov, max_drafts=%u)\n", + DS4_GPU_DSPARK_MAX_DRAFTS); + } + } + return rc; +} + extern "C" int ds4_gpu_indexer_topk_tensor( ds4_gpu_tensor *selected, const ds4_gpu_tensor *scores, @@ -15385,6 +16308,32 @@ extern "C" int ds4_gpu_matmul_q8_0_pair_tensor( const char *w0 = cuda_resolve_weight_ptr(model_map, weight0_offset, weight0_bytes, logical_tier, "q8_0_pair0"); const char *w1 = cuda_resolve_weight_ptr(model_map, weight1_offset, weight1_bytes, logical_tier, "q8_0_pair1"); if (!w0 || !w1) return 0; + const bool fused_aligned_candidate = + g_n_gpus == 1 && + logical_tier >= 0 && logical_tier < DS4_MAX_GPUS && + g_cuda_is_gb10[logical_tier] && + getenv("DS4_CUDA_NO_Q8_FUSED_ALIGNED") == NULL && + n_tok == 1u && in_dim <= INT_MAX && + out0_dim <= INT_MAX && out1_dim <= INT_MAX && + (in_dim % 1024u) == 0u && + (out0_dim % 128u) == 0u && (out1_dim % 128u) == 0u && + cuda_aligned_q8_enabled() && cuda_q8_use_dp4a(); + const uint64_t aligned0_bytes = fused_aligned_candidate + ? ds4_mmq_q8_0_aligned_bytes((int)out0_dim, (int)in_dim) : 0u; + const uint64_t aligned1_bytes = fused_aligned_candidate + ? ds4_mmq_q8_0_aligned_bytes((int)out1_dim, (int)in_dim) : 0u; + const char *aligned0 = aligned0_bytes + ? cuda_derived_weight_ptr( + model_map, weight0_offset, weight0_bytes, + CUDA_DERIVED_Q8_0_ALIGNED_DENSE, + in_dim, out0_dim, 1u, aligned0_bytes) + : NULL; + const char *aligned1 = aligned1_bytes + ? cuda_derived_weight_ptr( + model_map, weight1_offset, weight1_bytes, + CUDA_DERIVED_Q8_0_ALIGNED_DENSE, + in_dim, out1_dim, 1u, aligned1_bytes) + : NULL; const bool force_decode_warp = n_tok == 2u && g_glm_mtp_verify_mode; @@ -15580,18 +16529,32 @@ extern "C" int ds4_gpu_matmul_q8_0_pair_tensor( return cuda_ok(cudaGetLastError(), "matmul_q8_0 pair1 batch launch"); } const uint64_t max_out = out0_dim > out1_dim ? out0_dim : out1_dim; - matmul_q8_0_pair_preq_warp8_kernel<<<((unsigned)max_out + 7u) / 8u, 256, 0, cuda_decode_stream()>>>( - (float *)out0->ptr, - (float *)out1->ptr, - reinterpret_cast(w0), - reinterpret_cast(w1), - xq, - xscale, - in_dim, - out0_dim, - out1_dim, - blocks, - use_dp4a); + const dim3 decode_grid(((unsigned)max_out + 7u) / 8u, 1u, 1u); + if (aligned0 && aligned1) { + const uint64_t nblk0 = out0_dim * blocks; + const uint64_t nblk1 = out1_dim * blocks; + const uint64_t dq0_bytes = + (nblk0 * sizeof(__half) + 63u) & ~63ull; + const uint64_t dq1_bytes = + (nblk1 * sizeof(__half) + 63u) & ~63ull; + matmul_q8_0_pair_aligned_preq_warp8_kernel<<< + decode_grid, 256, 0, cuda_decode_stream()>>>( + (float *)out0->ptr, + (float *)out1->ptr, + (const int4 *)(aligned0 + dq0_bytes), + (const __half *)aligned0, + (const int4 *)(aligned1 + dq1_bytes), + (const __half *)aligned1, + xq, xscale, out0_dim, out1_dim, blocks); + } else { + matmul_q8_0_pair_preq_warp8_kernel<<< + decode_grid, 256, 0, cuda_decode_stream()>>>( + (float *)out0->ptr, + (float *)out1->ptr, + reinterpret_cast(w0), + reinterpret_cast(w1), + xq, xscale, in_dim, out0_dim, out1_dim, blocks, use_dp4a); + } return cuda_ok(cudaGetLastError(), "matmul_q8_0 pair warp launch"); } @@ -15642,6 +16605,31 @@ extern "C" int ds4_gpu_matmul_q8_0_decode_rows_exact_tensor( ds4_tensor_device_idx(x) != logical_tier) { return 0; } + /* Match the canonical one-row decode path when an aligned Q8 artifact is + * present. Its NC kernel reads each weight row once while preserving the + * same per-column block walk and warp reduction, so exact-N can batch the + * vocab head without silently switching to the raw warp8 arithmetic. */ + const uint64_t aligned_bytes = + (in_dim % 1024u) == 0u && (out_dim % 128u) == 0u + ? ds4_mmq_q8_0_aligned_bytes((int)out_dim, (int)in_dim) + : 0u; + const char *aligned = aligned_bytes && cuda_aligned_q8_enabled() + ? cuda_derived_weight_ptr( + model_map, weight_offset, weight_bytes, + CUDA_DERIVED_Q8_0_ALIGNED_DENSE, + in_dim, out_dim, 1u, aligned_bytes) + : NULL; + if (aligned && n_rows <= 8u) { + const int aligned_rc = ds4_mmq_q8_0_aligned_dense_vec( + aligned, + (const float *)x->ptr, + (float *)out->ptr, + (int)out_dim, + (int)n_rows, + (int)in_dim, + cuda_decode_stream()); + if (aligned_rc == 0) return 1; + } const char *wptr = cuda_resolve_weight_ptr( model_map, weight_offset, weight_bytes, logical_tier, "q8_0 decode rows exact"); @@ -15657,14 +16645,14 @@ extern "C" int ds4_gpu_matmul_q8_0_decode_rows_exact_tensor( int8_t *xq = (int8_t *)tmp; float *xscale = (float *)((char *)tmp + scale_offset); dim3 qgrid((unsigned)blocks, n_rows, 1u); - quantize_q8_0_f32_kernel<<>>( + quantize_q8_0_f32_kernel<<>>( xq, xscale, (const float *)x->ptr, in_dim, blocks); if (!cuda_ok(cudaGetLastError(), "q8_0 decode rows exact quantize launch")) { return 0; } dim3 grid(((unsigned)out_dim + 7u) / 8u, n_rows, 1u); - matmul_q8_0_preq_warp8_kernel<<>>( + matmul_q8_0_preq_warp8_kernel<<>>( (float *)out->ptr, reinterpret_cast(wptr), xq, xscale, in_dim, out_dim, blocks, cuda_q8_use_dp4a()); @@ -15795,6 +16783,23 @@ static int cuda_matmul_q8_0_hc_expand_tensor_labeled( const int logical_tier = ds4_tensor_device_idx(out_hc); const char *wptr = cuda_resolve_weight_ptr(model_map, weight_offset, weight_bytes, logical_tier, label ? label : "q8_0_hc_expand"); if (!wptr) return 0; + const uint64_t aligned_bytes = + in_dim <= INT_MAX && out_dim <= INT_MAX && + (in_dim % 1024u) == 0u && (out_dim % 128u) == 0u + ? ds4_mmq_q8_0_aligned_bytes((int)out_dim, (int)in_dim) + : 0u; + const char *aligned = + g_n_gpus == 1 && + logical_tier >= 0 && logical_tier < DS4_MAX_GPUS && + g_cuda_is_gb10[logical_tier] && + getenv("DS4_CUDA_NO_Q8_FUSED_ALIGNED") == NULL && + aligned_bytes != 0u && cuda_aligned_q8_enabled() && + cuda_q8_use_dp4a() + ? cuda_derived_weight_ptr( + model_map, weight_offset, weight_bytes, + CUDA_DERIVED_Q8_0_ALIGNED_DENSE, + in_dim, out_dim, 1u, aligned_bytes) + : NULL; const uint64_t xq_bytes = blocks * 32u; const uint64_t scale_offset = (xq_bytes + 15u) & ~15ull; @@ -15806,7 +16811,29 @@ static int cuda_matmul_q8_0_hc_expand_tensor_labeled( const int use_dp4a = cuda_q8_use_dp4a(); quantize_q8_0_f32_kernel<<<(unsigned)blocks, 32, 0, cuda_decode_stream()>>>(xq, xscale, (const float *)x->ptr, in_dim, blocks); if (!cuda_ok(cudaGetLastError(), "matmul_q8_0_hc_expand quantize launch")) return 0; - matmul_q8_0_hc_expand_preq_warp8_kernel<<<((unsigned)out_dim + 7u) / 8u, 256, 0, cuda_decode_stream()>>>( + const dim3 grid(((unsigned)out_dim + 7u) / 8u, 1u, 1u); + if (aligned) { + const uint64_t nblk = out_dim * blocks; + const uint64_t dq_bytes = (nblk * sizeof(__half) + 63u) & ~63ull; + matmul_q8_0_hc_expand_aligned_preq_warp8_kernel<<>>( + (float *)out_hc->ptr, + (float *)block_out->ptr, + block_add ? (const float *)block_add->ptr : (const float *)block_out->ptr, + block_add2 ? (const float *)block_add2->ptr : (const float *)block_out->ptr, + owned_home_slots ? (const float *)owned_home_slots->ptr : NULL, + owned_peer_packed ? (const float *)owned_peer_packed->ptr : NULL, + owned_selected ? (const int32_t *)owned_selected->ptr : NULL, + (const float *)residual_hc->ptr, + (const float *)split->ptr, + (const int4 *)(aligned + dq_bytes), + (const __half *)aligned, + xq, xscale, out_dim, n_embd, n_hc, blocks, + block_add ? 1 : 0, + block_add2 ? 1 : 0, + owned_home_slots ? 1 : 0, + owned_expert_split); + } else { + matmul_q8_0_hc_expand_preq_warp8_kernel<<>>( (float *)out_hc->ptr, (float *)block_out->ptr, block_add ? (const float *)block_add->ptr : (const float *)block_out->ptr, @@ -15817,18 +16844,13 @@ static int cuda_matmul_q8_0_hc_expand_tensor_labeled( (const float *)residual_hc->ptr, (const float *)split->ptr, reinterpret_cast(wptr), - xq, - xscale, - in_dim, - out_dim, - n_embd, - n_hc, - blocks, + xq, xscale, in_dim, out_dim, n_embd, n_hc, blocks, block_add ? 1 : 0, block_add2 ? 1 : 0, owned_home_slots ? 1 : 0, owned_expert_split, use_dp4a); + } return cuda_ok(cudaGetLastError(), "matmul_q8_0_hc_expand launch"); } @@ -16300,22 +17322,145 @@ extern "C" int ds4_gpu_matmul_f16_pair_compressor_store_tensor( const ds4_gpu_tensor *x, uint32_t ratio, uint32_t pos) { - (void)out_kv; - (void)out_score; - (void)state_kv; - (void)state_score; - (void)model_map; - (void)model_size; - (void)weight_kv_offset; - (void)weight_score_offset; - (void)ape_offset; - (void)ape_type; - (void)in_dim; - (void)width; - (void)x; - (void)ratio; - (void)pos; - return 0; + if (!g_cuda_f16_pair_compressor_store || + getenv("DS4_CUDA_NO_F16_PAIR_MATMUL") != NULL || + getenv("DS4_CUDA_SERIAL_F16_MATMUL") != NULL || + getenv("DS4_CUDA_SERIAL_ROUTER") != NULL || + getenv("DS4_CUDA_NO_ORDERED_F16_MATMUL") != NULL) { + return 0; + } + if (!out_kv || !out_score || !state_kv || !state_score || !x || + !model_map || in_dim == 0u || width == 0u || ratio == 0u || + (ape_type != 0u && ape_type != 1u)) { + return -1; + } + /* These are the three resident Flash/Pro compressor projection shapes + * validated on GB10. Other layouts retain the ordinary pair+store path. */ + if (in_dim != 4096u || + !((ratio == 4u && (width == 256u || width == 1024u)) || + (ratio == 128u && width == 512u))) { + return 0; + } + if ((uint64_t)width > UINT64_MAX / in_dim || + weight_kv_offset > model_size || + weight_score_offset > model_size || + ape_offset > model_size) { + return -1; + } + const uint64_t weight_elems = (uint64_t)width * in_dim; + if (weight_elems > UINT64_MAX / sizeof(uint16_t) || + in_dim > UINT64_MAX / sizeof(float) || + (uint64_t)width > UINT64_MAX / sizeof(float)) { + return -1; + } + const uint64_t weight_bytes = weight_elems * sizeof(uint16_t); + const uint64_t output_bytes = (uint64_t)width * sizeof(float); + const uint64_t input_bytes = in_dim * sizeof(float); + const uint64_t ape_elem_bytes = ape_type == 1u ? 2u : 4u; + if ((uint64_t)width > UINT64_MAX / ratio) return -1; + const uint64_t ape_elems = (uint64_t)width * ratio; + if (ape_elems > UINT64_MAX / ape_elem_bytes) return -1; + const uint64_t ape_bytes = ape_elems * ape_elem_bytes; + const uint64_t coff = ratio == 4u ? 2u : 1u; + const uint64_t state_rows = coff * ratio; + if (state_rows > UINT64_MAX / width) return -1; + const uint64_t state_elems = state_rows * width; + if (state_elems > UINT64_MAX / sizeof(float)) return -1; + const uint64_t state_bytes = state_elems * sizeof(float); + if (weight_bytes > model_size - weight_kv_offset || + weight_bytes > model_size - weight_score_offset || + ape_bytes > model_size - ape_offset || + x->bytes < input_bytes || + out_kv->bytes < output_bytes || + out_score->bytes < output_bytes || + state_kv->bytes < state_bytes || + state_score->bytes < state_bytes) { + return -1; + } + const int logical_tier = ds4_tensor_device_idx(out_kv); + if (logical_tier < 0 || logical_tier >= DS4_MAX_GPUS) return -1; + if (!g_cuda_is_gb10[logical_tier]) return 0; + if (ds4_tensor_device_idx(out_score) != logical_tier || + ds4_tensor_device_idx(state_kv) != logical_tier || + ds4_tensor_device_idx(state_score) != logical_tier || + ds4_tensor_device_idx(x) != logical_tier) { + return -1; + } + const int expected_device = logical_tier < g_n_gpus + ? g_gpu[logical_tier].device_id : -1; + int active_device = -1; + if (expected_device < 0 || + cudaGetDevice(&active_device) != cudaSuccess || + active_device != expected_device) { + (void)cudaGetLastError(); + return -1; + } + const __half *w_kv = (const __half *)cuda_resolve_weight_ptr( + model_map, weight_kv_offset, weight_bytes, logical_tier, + "f16 compressor kv"); + const __half *w_score = (const __half *)cuda_resolve_weight_ptr( + model_map, weight_score_offset, weight_bytes, logical_tier, + "f16 compressor score"); + const void *ape = cuda_resolve_weight_ptr( + model_map, ape_offset, ape_bytes, logical_tier, "compressor ape"); + if (!w_kv || !w_score || !ape) return -1; + + const __half2 *w_pair_chunk32 = NULL; + if (g_n_gpus == 1 && + getenv("DS4_CUDA_NO_F16_PAIR_COMPRESSOR_TRANSPOSE") == NULL) { + w_pair_chunk32 = cuda_f16_pair_chunk32_get( + model_map, weight_kv_offset, weight_score_offset, + w_kv, w_score, in_dim, width, logical_tier); + } + if (w_pair_chunk32) { + if (getenv("DS4_CUDA_NO_F16_PAIR_COMPRESSOR_TRANSPOSE_PREFETCH8") == NULL && + (in_dim % 256u) == 0u) { + matmul_f16_pair_compressor_store_chunk32_prefetch8_kernel<<>>( + (float *)out_kv->ptr, + (float *)out_score->ptr, + (float *)state_kv->ptr, + (float *)state_score->ptr, + w_pair_chunk32, + (const float *)x->ptr, + ape, + ape_type, + in_dim, + width, + ratio, + pos); + } else { + matmul_f16_pair_compressor_store_chunk32_kernel<<>>( + (float *)out_kv->ptr, + (float *)out_score->ptr, + (float *)state_kv->ptr, + (float *)state_score->ptr, + w_pair_chunk32, + (const float *)x->ptr, + ape, + ape_type, + in_dim, + width, + ratio, + pos); + } + } else { + matmul_f16_pair_compressor_store_ordered_chunks_kernel<<>>( + (float *)out_kv->ptr, + (float *)out_score->ptr, + (float *)state_kv->ptr, + (float *)state_score->ptr, + w_kv, + w_score, + (const float *)x->ptr, + ape, + ape_type, + in_dim, + width, + ratio, + pos); + } + return cuda_ok(cudaGetLastError(), + "f16 pair compressor/store launch") ? 1 : -1; } extern "C" int ds4_gpu_matmul_f32_tensor(ds4_gpu_tensor *out, const void *model_map, uint64_t model_size, uint64_t weight_offset, uint64_t in_dim, uint64_t out_dim, const ds4_gpu_tensor *x, uint64_t n_tok) { @@ -18818,6 +19963,22 @@ extern "C" int ds4_gpu_attention_output_q8_batch_tensor( const unsigned char *out_b = reinterpret_cast( cuda_resolve_weight_ptr(model_map, out_b_offset, out_b_bytes, logical_tier, "attn_out_b")); if (!out_a || !out_b) return 0; + const bool out_a_aligned_candidate = + g_n_gpus == 1 && + logical_tier >= 0 && logical_tier < DS4_MAX_GPUS && + g_cuda_is_gb10[logical_tier] && + getenv("DS4_CUDA_NO_Q8_FUSED_ALIGNED") == NULL && + n_tokens == 1u && group_dim <= INT_MAX && low_dim <= INT_MAX && + (group_dim % 1024u) == 0u && (low_dim % 128u) == 0u && + cuda_aligned_q8_enabled() && cuda_q8_use_dp4a(); + const uint64_t out_a_aligned_bytes = out_a_aligned_candidate + ? ds4_mmq_q8_0_aligned_bytes((int)low_dim, (int)group_dim) : 0u; + const char *out_a_aligned = out_a_aligned_bytes + ? cuda_derived_weight_ptr( + model_map, out_a_offset, out_a_bytes, + CUDA_DERIVED_Q8_0_ALIGNED_DENSE, + group_dim, low_dim, 1u, out_a_aligned_bytes) + : NULL; const uint32_t profile = getenv("DS4_CUDA_ATTN_OUTPUT_PROFILE") != NULL; cudaEvent_t prof_ev[3] = {NULL, NULL, NULL}; @@ -18947,16 +20108,29 @@ extern "C" int ds4_gpu_attention_output_q8_batch_tensor( use_dp4a); } else { dim3 grid_a(((unsigned)low_dim + 7u) / 8u, (unsigned)n_tokens, 1); - grouped_q8_0_a_preq_warp8_kernel<<>>((float *)low->ptr, - out_a, - xq, - xscale, - group_dim, - rank, - n_groups, - n_tokens, - blocks_a, - use_dp4a); + if (out_a_aligned) { + const uint64_t nblk = low_dim * blocks_a; + const uint64_t dq_bytes = + (nblk * sizeof(__half) + 63u) & ~63ull; + grouped_q8_0_a_aligned_preq_warp8_kernel<<< + grid_a, 256, 0, cuda_decode_stream()>>>( + (float *)low->ptr, + (const int4 *)(out_a_aligned + dq_bytes), + (const __half *)out_a_aligned, + xq, xscale, rank, n_groups, n_tokens, blocks_a); + } else { + grouped_q8_0_a_preq_warp8_kernel<<>>( + (float *)low->ptr, + out_a, + xq, + xscale, + group_dim, + rank, + n_groups, + n_tokens, + blocks_a, + use_dp4a); + } } if (!cuda_ok(cudaGetLastError(), "attention_output_q8_a preq launch")) return 0; } @@ -19031,6 +20205,23 @@ extern "C" int ds4_gpu_attention_output_low_q8_rows_exact_tensor( cuda_resolve_weight_ptr(model_map, a_offset, out_a_bytes, logical_tier, "attn_out_a_rows")); if (!out_a) return 0; + const bool aligned_candidate = + g_n_gpus == 1 && + logical_tier >= 0 && logical_tier < DS4_MAX_GPUS && + g_cuda_is_gb10[logical_tier] && + getenv("DS4_CUDA_NO_Q8_FUSED_ALIGNED") == NULL && + n_rows == 1u && group0 == 0u && group_cnt == n_groups_total && + group_dim <= INT_MAX && low_dim <= INT_MAX && + (group_dim % 1024u) == 0u && (low_dim % 128u) == 0u && + cuda_aligned_q8_enabled() && cuda_q8_use_dp4a(); + const uint64_t aligned_bytes = aligned_candidate + ? ds4_mmq_q8_0_aligned_bytes((int)low_dim, (int)group_dim) : 0u; + const char *aligned = aligned_bytes + ? cuda_derived_weight_ptr( + model_map, a_offset, out_a_bytes, + CUDA_DERIVED_Q8_0_ALIGNED_DENSE, + group_dim, low_dim, 1u, aligned_bytes) + : NULL; const uint64_t x_rows = (uint64_t)n_rows * group_cnt; const uint64_t xq_bytes = x_rows * blocks_a * 32u; @@ -19054,16 +20245,29 @@ extern "C" int ds4_gpu_attention_output_low_q8_rows_exact_tensor( if (!cuda_ok(cudaGetLastError(), "attention_output_low_q8 rows prequant launch")) return 0; dim3 grid_a(((unsigned)low_dim + 7u) / 8u, n_rows, 1u); - grouped_q8_0_a_preq_warp8_kernel<<>>((float *)low->ptr, - out_a, - xq, - xscale, - group_dim, - rank, - group_cnt, - n_rows, - blocks_a, - use_dp4a); + if (aligned) { + const uint64_t nblk = low_dim * blocks_a; + const uint64_t dq_bytes = + (nblk * sizeof(__half) + 63u) & ~63ull; + grouped_q8_0_a_aligned_preq_warp8_kernel<<< + grid_a, 256, 0, cuda_decode_stream()>>>( + (float *)low->ptr, + (const int4 *)(aligned + dq_bytes), + (const __half *)aligned, + xq, xscale, rank, group_cnt, n_rows, blocks_a); + } else { + grouped_q8_0_a_preq_warp8_kernel<<>>( + (float *)low->ptr, + out_a, + xq, + xscale, + group_dim, + rank, + group_cnt, + n_rows, + blocks_a, + use_dp4a); + } return cuda_ok(cudaGetLastError(), "attention_output_low_q8 rows launch"); } @@ -24280,14 +25484,27 @@ static int routed_moe_launch( } } } - /* A successful tiny-vector run is final. All other multi-token - * shapes retain the direct-prefill path and fused-SoA fallback. */ + /* A successful tiny-vector run is final. For every remaining + * multi-token shape, keep the exact direct-prefill scratch sizing. + * Large prefill stays device-generic; GB10 also uses the direct + * producer for smaller speculative batches. */ if (n_tokens > 1u && (!dspark_tiny_aligned_vec || rc != 0)) { - rc = 1; + rc = DS4_MMQ_NOT_APPLICABLE; const uint64_t assignments = (uint64_t)n_tokens * n_expert; - if (assignments >= 1024u) { + const int direct_tier = ds4_tensor_device_idx(out); + const int direct_gb10 = + direct_tier >= 0 && direct_tier < DS4_MAX_GPUS && + g_cuda_is_gb10[direct_tier]; + const int direct_shape_fits = + n_tokens <= INT_MAX && n_total_expert <= INT_MAX && + n_expert <= INT_MAX && expert_in_dim <= INT_MAX && + expert_mid_dim <= INT_MAX && out_dim <= INT_MAX; + const int direct_applicable = + g_cuda_direct_q2_prefill && direct_shape_fits && + (assignments >= 1024u || direct_gb10); + if (direct_applicable) { size_t input_q8_bytes = 0; size_t down_q8_bytes = 0; size_t work_bytes = 0; @@ -24323,7 +25540,9 @@ static int routed_moe_launch( } } } - if (rc != 0) { + /* Only a pre-enqueue NOT_APPLICABLE result may retry the + * materialized SoA path. Other failures may follow enqueue. */ + if (rc == DS4_MMQ_NOT_APPLICABLE) { rc = ds4_mmq_iq2_xxs_q2_K_moe_fused_soa( gate_aligned, up_aligned, down_aligned, (const float *)x->ptr, @@ -26312,6 +27531,46 @@ extern "C" int ds4_gpu_hc_split_weighted_sum_norm_tensor( const float *norm_w = (const float *)cuda_resolve_weight_ptr(model_map, norm_weight_offset, (uint64_t)n_embd * sizeof(float), logical_tier, "hc_norm_weight"); if (!scale || !base || !norm_w) return 0; + if (n_embd == 4096u && n_rows == 1u && + out->ptr != norm_out->ptr && + getenv("DS4_CUDA_NO_HC_SPLIT_NORM_SPLIT4096") == NULL) { + bool split_ok = true; + hc_split_weighted_sum_norm_fused_partial4096_kernel<<<16u, 256, 0, cuda_decode_stream()>>>( + (float *)out->ptr, + (float *)split->ptr, + (const float *)mix->ptr, + (const float *)residual_hc->ptr, + scale, + base, + sinkhorn_iters, + eps); + split_ok = cudaGetLastError() == cudaSuccess; + float *norm_scale = NULL; + if (split_ok) { + norm_scale = (float *)cuda_tmp_alloc_on( + logical_tier, sizeof(float), "hc split norm scale"); + split_ok = norm_scale != NULL; + } + if (split_ok) { + hc_split_weighted_sum_norm_fused_reduce4096_kernel<<<1u, 256, 0, cuda_decode_stream()>>>( + (const float *)out->ptr, + norm_scale, + norm_eps); + split_ok = cudaGetLastError() == cudaSuccess; + } + if (split_ok) { + hc_split_weighted_sum_norm_fused_store4096_kernel<<<16u, 256, 0, cuda_decode_stream()>>>( + (const float *)out->ptr, + (float *)norm_out->ptr, + norm_w, + norm_scale); + split_ok = cudaGetLastError() == cudaSuccess; + } + if (split_ok) return 1; + /* The split launches are adjacent, so any pre-enqueue failure + * leaves a valid fused-reference retry: it rewrites out, split, + * and norm_out without consuming partial state. */ + } hc_split_weighted_sum_norm_fused_kernel<<<(uint32_t)n_rows, 256, 0, cuda_decode_stream()>>>( (float *)out->ptr, (float *)norm_out->ptr, @@ -32650,6 +33909,47 @@ __global__ static void matmul_q4_K_hc_expand4_kernel( } } +/* HC epilogue for the canonical MMVQ/Q8_1 Q4_K path. MMVQ materializes + * block_out using its established activation quantizer and reduction order; + * one thread per embedding row then reuses that value for all four HC + * destinations. Each destination retains hc_expand_kernel's exact + * multiply/add order, while block_out is read once instead of four times. */ +__global__ static void q4_K_hc_expand4_rows_kernel( + float *out_hc, + const float *block_out, + const float *residual_hc, + const float *split, + uint32_t n_embd) { + const uint32_t row = blockIdx.x * blockDim.x + threadIdx.x; + if (row >= n_embd) return; + + const float block_v = block_out[row]; + const float *post = split + 4u; + const float *comb = split + 8u; +#pragma unroll + for (uint32_t dst_hc = 0; dst_hc < 4u; dst_hc++) { + float acc = block_v * post[dst_hc]; +#pragma unroll + for (uint32_t src_hc = 0; src_hc < 4u; src_hc++) { + acc += comb[dst_hc + src_hc * 4u] * + residual_hc[(uint64_t)src_hc * n_embd + row]; + } + out_hc[(uint64_t)dst_hc * n_embd + row] = acc; + } +} + +__global__ static void q4_K_attn_hc_bitwise_compare_kernel( + uint32_t *mismatch, + const float *reference, + const float *candidate, + uint64_t count) { + const uint64_t i = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (i >= count) return; + if (__float_as_uint(reference[i]) != __float_as_uint(candidate[i])) { + atomicExch(mismatch, 1u); + } +} + static int cuda_matmul_q4_K_tensor( ds4_gpu_tensor *out, const void *model_map, @@ -32838,16 +34138,130 @@ static int cuda_matmul_q4_K_pair_tensor_impl( return cuda_ok(cudaGetLastError(), "q4_K dense pair matmul launch"); } +static uint64_t g_q4_attn_hc_oracle_calls; +static uint64_t g_q4_attn_hc_oracle_epilogue_mismatches; +static uint64_t g_q4_attn_hc_oracle_q8k_mismatches; +static uint64_t g_q4_attn_hc_oracle_skips; +static int g_q4_attn_hc_oracle_report_registered; +static int g_q4_attn_hc_oracle_epilogue_reported; +static int g_q4_attn_hc_oracle_q8k_reported; + +static void cuda_q4_attn_hc_oracle_report(void) { + fprintf(stderr, + "ds4: CUDA Q4 attention-output/HC oracle: " + "calls=%llu epilogue_mismatches=%llu " + "q8k_mismatches=%llu skips=%llu " + "(canonical output retained)\n", + (unsigned long long)g_q4_attn_hc_oracle_calls, + (unsigned long long)g_q4_attn_hc_oracle_epilogue_mismatches, + (unsigned long long)g_q4_attn_hc_oracle_q8k_mismatches, + (unsigned long long)g_q4_attn_hc_oracle_skips); +} + +static void cuda_q4_attn_hc_oracle_register_report(void) { + if (!g_q4_attn_hc_oracle_report_registered) { + g_q4_attn_hc_oracle_report_registered = 1; + (void)atexit(cuda_q4_attn_hc_oracle_report); + } +} + +static int cuda_q4_K_hc_expand_canonical( + ds4_gpu_tensor *out_hc, + ds4_gpu_tensor *block_out, + const void *model_map, + uint64_t model_size, + uint64_t weight_offset, + uint64_t in_dim, + uint64_t out_dim, + const ds4_gpu_tensor *x, + const ds4_gpu_tensor *residual_hc, + const ds4_gpu_tensor *split, + int row_packed_epilogue) { + /* Reuse the exact ordinary Q4 dispatcher. With MMQ enabled this is the + * canonical MMVQ/Q8_1 path (including sanitize); if MMVQ rejects the + * shape it retains the ordinary Q8_K materialized fallback. */ + if (!cuda_matmul_q4_K_tensor(block_out, model_map, model_size, + weight_offset, in_dim, out_dim, x, 1u)) { + return 0; + } + if (row_packed_epilogue) { + q4_K_hc_expand4_rows_kernel<<<((unsigned)out_dim + 255u) / 256u, + 256, 0, cuda_decode_stream()>>>( + (float *)out_hc->ptr, + (const float *)block_out->ptr, + (const float *)residual_hc->ptr, + (const float *)split->ptr, + (uint32_t)out_dim); + } else { + /* Oracle reference: byte-for-byte the ordinary one-token + * ds4_gpu_hc_expand_tensor launch and argument layout. */ + const uint64_t n_elem = 4u * out_dim; + const float *base = (const float *)split->ptr; + hc_expand_kernel<<<(unsigned)((n_elem + 255u) / 256u), + 256, 0, cuda_decode_stream()>>>( + (float *)out_hc->ptr, + (const float *)block_out->ptr, + (const float *)block_out->ptr, + (const float *)block_out->ptr, + (const float *)residual_hc->ptr, + base + 4u, + base + 8u, + (uint32_t)out_dim, 4u, 1u, 4u, 16u, 0, 0); + } + return cuda_ok(cudaGetLastError(), + "q4_K canonical MMVQ hc expand launch"); +} + +static int cuda_q4_K_hc_expand_q8k_launch( + ds4_gpu_tensor *out_hc, + ds4_gpu_tensor *block_out, + const ds4_gpu_tensor *x, + const ds4_gpu_tensor *residual_hc, + const ds4_gpu_tensor *split, + const char *wptr, + cuda_block_q8_K *xq, + uint64_t row_bytes, + uint32_t blocks, + uint32_t out_dim, + uint32_t n_embd) { + q8_K_quantize_kernel<<>>( + xq, (const float *)x->ptr, blocks * CUDA_QK_K, 1u); + if (!cuda_ok(cudaGetLastError(), + "q4_K Q8_K hc expand quantize launch")) { + return 0; + } + + matmul_q4_K_hc_expand4_kernel<<<(out_dim + 31u) / 32u, + 256, 0, cuda_decode_stream()>>>( + (float *)out_hc->ptr, + (float *)block_out->ptr, + (const float *)residual_hc->ptr, + (const float *)split->ptr, + wptr, + xq, + row_bytes, + blocks, + out_dim, + n_embd); + return cuda_ok(cudaGetLastError(), "q4_K Q8_K hc expand launch"); +} + extern "C" int ds4_gpu_matmul_q4_K_hc_expand_available(void) { if (getenv("DS4_CUDA_DISABLE_Q4_ATTN_OUT_HC_FUSE") != NULL) return 0; - /* The fused implementation is byte-compatible with the native Q8_K - * fallback. Single-GPU decode normally selects the vendored MMVQ/Q8_1 - * path instead, whose activation quantizer intentionally has different - * numerics. Preserve that default until an on-device oracle validates a - * switch; multi-GPU/quality/DS4_CUDA_MMQ=0 configurations already use - * Q8_K and can enable the compound transparently. */ - return getenv("DS4_CUDA_ENABLE_Q4_ATTN_OUT_HC_FUSE") != NULL || - !cuda_use_mmq(); + if (cuda_env_flag_enabled("DS4_CUDA_Q4_ATTN_OUT_HC_ORACLE", 0)) { + /* Always emit a summary when the oracle was requested, including a + * zero-call summary that makes an ineligible/unused A/B visible. */ + cuda_q4_attn_hc_oracle_register_report(); + } + /* MMQ decode stays opt-in until its row-packed HC epilogue is timed on + * CUDA hardware. Unlike the old experiment, the normal enable switch no + * longer changes Q8_1 activation quantization. The explicitly named + * Q8_K experiment remains available behind a separate gate and oracle. */ + return !cuda_use_mmq() || + cuda_env_flag_enabled("DS4_CUDA_ENABLE_Q4_ATTN_OUT_HC_FUSE", 0) || + cuda_env_flag_enabled( + "DS4_CUDA_Q4_ATTN_OUT_HC_Q8K_EXPERIMENT", 0) || + cuda_env_flag_enabled("DS4_CUDA_Q4_ATTN_OUT_HC_ORACLE", 0); } extern "C" int ds4_gpu_matmul_q4_K_hc_expand_tensor( @@ -32907,32 +34321,174 @@ extern "C" int ds4_gpu_matmul_q4_K_hc_expand_tensor( "q4_K hc expand"); if (!wptr) return 0; + const int q8k_experiment = + !cuda_use_mmq() || + cuda_env_flag_enabled( + "DS4_CUDA_Q4_ATTN_OUT_HC_Q8K_EXPERIMENT", 0); + const int oracle = cuda_env_flag_enabled( + "DS4_CUDA_Q4_ATTN_OUT_HC_ORACLE", 0); + + if (!q8k_experiment && !oracle) { + return cuda_q4_K_hc_expand_canonical( + out_hc, block_out, model_map, model_size, weight_offset, + in_dim, out_dim, x, residual_hc, split, 1); + } + + /* The oracle is deliberately diagnostic and fail-closed: compute and + * retain canonical output first, then compare the true one-kernel Q8_K + * candidate bit-for-bit. It synchronizes only when explicitly enabled + * and never attempts a host read while the decode stream is captured. */ + if (oracle) { + cuda_q4_attn_hc_oracle_register_report(); + if (!cuda_q4_K_hc_expand_canonical( + out_hc, block_out, model_map, model_size, weight_offset, + in_dim, out_dim, x, residual_hc, split, 0)) { + return 0; + } + + cudaStream_t stream = cuda_decode_stream(); + cudaStreamCaptureStatus capture = cudaStreamCaptureStatusNone; + const cudaError_t capture_err = + cudaStreamIsCapturing(stream, &capture); + if (capture_err != cudaSuccess || + capture != cudaStreamCaptureStatusNone) { + (void)cudaGetLastError(); + g_q4_attn_hc_oracle_skips++; + return 1; + } + + const uint64_t xq_bytes = + blocks * sizeof(cuda_block_q8_K); + const uint64_t candidate_block_off = + (xq_bytes + 255u) & ~255ull; + const uint64_t candidate_q8k_hc_off = + (candidate_block_off + embd_bytes + 255u) & ~255ull; + const uint64_t candidate_epilogue_hc_off = + (candidate_q8k_hc_off + hc_bytes + 255u) & ~255ull; + const uint64_t mismatch_off = + (candidate_epilogue_hc_off + hc_bytes + 255u) & ~255ull; + const uint64_t scratch_bytes = + mismatch_off + 2u * sizeof(uint32_t); + unsigned char *scratch = (unsigned char *)cuda_tmp_alloc_on( + logical_tier, scratch_bytes, + "q4_K attention-output HC oracle"); + if (!scratch) { + g_q4_attn_hc_oracle_skips++; + return 1; + } + + cuda_block_q8_K *candidate_xq = + (cuda_block_q8_K *)scratch; + ds4_gpu_tensor candidate_block = *block_out; + candidate_block.ptr = scratch + candidate_block_off; + candidate_block.bytes = embd_bytes; + candidate_block.owner = 0; + ds4_gpu_tensor candidate_q8k_hc = *out_hc; + candidate_q8k_hc.ptr = scratch + candidate_q8k_hc_off; + candidate_q8k_hc.bytes = hc_bytes; + candidate_q8k_hc.owner = 0; + ds4_gpu_tensor candidate_epilogue_hc = *out_hc; + candidate_epilogue_hc.ptr = scratch + candidate_epilogue_hc_off; + candidate_epilogue_hc.bytes = hc_bytes; + candidate_epilogue_hc.owner = 0; + uint32_t *mismatch_device = + (uint32_t *)(scratch + mismatch_off); + + if (!cuda_ok(cudaMemsetAsync(mismatch_device, 0, + 2u * sizeof(uint32_t), + stream), + "clear q4_K attention-output HC oracle") || + !cuda_q4_K_hc_expand_q8k_launch( + &candidate_q8k_hc, &candidate_block, x, residual_hc, split, + wptr, candidate_xq, row_bytes, (uint32_t)blocks, + (uint32_t)out_dim, n_embd)) { + g_q4_attn_hc_oracle_skips++; + return 1; + } + + q4_K_hc_expand4_rows_kernel<<<((unsigned)out_dim + 255u) / 256u, + 256, 0, stream>>>( + (float *)candidate_epilogue_hc.ptr, + (const float *)block_out->ptr, + (const float *)residual_hc->ptr, + (const float *)split->ptr, + (uint32_t)out_dim); + + q4_K_attn_hc_bitwise_compare_kernel + <<<(embd_bytes / sizeof(float) + 255u) / 256u, + 256, 0, stream>>>( + mismatch_device, + (const float *)block_out->ptr, + (const float *)candidate_block.ptr, + embd_bytes / sizeof(float)); + q4_K_attn_hc_bitwise_compare_kernel + <<<(hc_bytes / sizeof(float) + 255u) / 256u, + 256, 0, stream>>>( + mismatch_device, + (const float *)out_hc->ptr, + (const float *)candidate_q8k_hc.ptr, + hc_bytes / sizeof(float)); + q4_K_attn_hc_bitwise_compare_kernel + <<<(hc_bytes / sizeof(float) + 255u) / 256u, + 256, 0, stream>>>( + mismatch_device + 1u, + (const float *)out_hc->ptr, + (const float *)candidate_epilogue_hc.ptr, + hc_bytes / sizeof(float)); + if (!cuda_ok(cudaGetLastError(), + "q4_K attention-output HC oracle compare launch")) { + return 0; + } + + uint32_t mismatch_host[2] = {0u, 0u}; + if (!cuda_ok(cudaMemcpyAsync(mismatch_host, mismatch_device, + sizeof(mismatch_host), + cudaMemcpyDeviceToHost, stream), + "read q4_K attention-output HC oracle") || + !cuda_ok(cudaStreamSynchronize(stream), + "synchronize q4_K attention-output HC oracle")) { + return 0; + } + g_q4_attn_hc_oracle_calls++; + if (mismatch_host[0] != 0u) { + g_q4_attn_hc_oracle_q8k_mismatches++; + if (!g_q4_attn_hc_oracle_q8k_reported) { + g_q4_attn_hc_oracle_q8k_reported = 1; + fprintf(stderr, + "ds4: CUDA Q4 attention-output/HC Q8_K oracle " + "found a bitwise mismatch; retaining canonical " + "MMVQ/Q8_1 output\n"); + } + } + if (mismatch_host[1] != 0u) { + g_q4_attn_hc_oracle_epilogue_mismatches++; + if (!g_q4_attn_hc_oracle_epilogue_reported) { + g_q4_attn_hc_oracle_epilogue_reported = 1; + fprintf(stderr, + "ds4: CUDA Q4 attention-output/HC row-packed " + "epilogue oracle found a bitwise mismatch; " + "retaining the ordinary HC expansion\n"); + } + } + return 1; + } + cuda_block_q8_K *xq = (cuda_block_q8_K *)cuda_tmp_alloc_on( logical_tier, blocks * sizeof(cuda_block_q8_K), "q4_K hc expand prequant"); - if (!xq) return 0; - - q8_K_quantize_kernel<<<(unsigned)blocks, 256, 0, - cuda_decode_stream()>>>( - xq, (const float *)x->ptr, (uint32_t)in_dim, 1u); - if (!cuda_ok(cudaGetLastError(), - "q4_K hc expand quantize launch")) { - return 0; + if (xq && cuda_q4_K_hc_expand_q8k_launch( + out_hc, block_out, x, residual_hc, split, + wptr, xq, row_bytes, (uint32_t)blocks, + (uint32_t)out_dim, n_embd)) { + return 1; } - matmul_q4_K_hc_expand4_kernel<<<((unsigned)out_dim + 31u) / 32u, - 256, 0, cuda_decode_stream()>>>( - (float *)out_hc->ptr, - (float *)block_out->ptr, - (const float *)residual_hc->ptr, - (const float *)split->ptr, - wptr, - xq, - row_bytes, - (uint32_t)blocks, - (uint32_t)out_dim, - n_embd); - return cuda_ok(cudaGetLastError(), "q4_K hc expand launch"); + /* Allocation/launch rejection of the optional candidate must not make + * decoding unavailable. Re-materialize through the ordinary dispatcher + * and finish with the exact row-packed HC epilogue. */ + return cuda_q4_K_hc_expand_canonical( + out_hc, block_out, model_map, model_size, weight_offset, + in_dim, out_dim, x, residual_hc, split, 1); } __global__ static void matmul_q4_K_kslice_kernel( diff --git a/ds4_gpu.h b/ds4_gpu.h index ece57a895..c2b7fb15a 100644 --- a/ds4_gpu.h +++ b/ds4_gpu.h @@ -540,6 +540,51 @@ int ds4_gpu_dspark_markov_argmax_tensor(ds4_gpu_tensor *out_idx, uint32_t prev_token, uint32_t vocab, uint32_t rank); + +/* Optional GPU-resident DSpark proposer tail. The backend keeps the Markov + * token chain on-device across all draft rows, stops it at the first rejected + * confidence row, and returns proposals plus the evaluated confidence logits + * in one result/readback. Callers must re-evaluate the returned prefix with + * the established CPU sigmoid policy and fall back if the device stopped a + * row that policy accepts. + * + * This acceleration hook is deliberately fail-closed: it returns zero for + * disabled or unsupported inputs and callers must use the established + * per-row path. CUDA requires DS4_CUDA_DSPARK_DEVICE_PROPOSER=1; Metal uses + * DS4_METAL_DSPARK_DEVICE_PROPOSER=1. The corresponding NO_DEVICE_PROPOSER + * variables are unconditional kill switches. The first implementation + * supports Q8_0 w1, w2, and confidence weights with 32-aligned hidden/rank + * dimensions. The caller-owned result tensor must provide + * DS4_GPU_DSPARK_DEVICE_PROPOSAL_BYTES; the first sizeof(result) bytes are the + * public payload and the remainder is private per-call backend state. */ +#define DS4_GPU_DSPARK_MAX_DRAFTS 6u +#define DS4_GPU_DSPARK_DEVICE_PROPOSAL_BYTES 2048u +typedef struct { + int32_t tokens[DS4_GPU_DSPARK_MAX_DRAFTS]; + float confidence_logits[DS4_GPU_DSPARK_MAX_DRAFTS]; + uint32_t proposal_len; + uint32_t confidence_len; + uint32_t status; /* 1: complete; 0: device-side failure */ + uint32_t reserved; +} ds4_gpu_dspark_device_proposal; + +int ds4_gpu_dspark_markov_confidence_q8_tensor( + ds4_gpu_tensor *out_result, + const ds4_gpu_tensor *logits_rows, + const ds4_gpu_tensor *hidden_rows, + const void *model_map, + uint64_t model_size, + uint64_t w1_offset, + uint64_t w2_offset, + uint64_t confidence_offset, + uint32_t first_prev_token, + uint32_t vocab, + uint32_t rank, + uint32_t hidden_dim, + uint32_t n_drafts, + float confidence_threshold, + int reuse_confidence0, + float confidence0); int ds4_gpu_indexer_topk_tensor( ds4_gpu_tensor *selected, const ds4_gpu_tensor *scores, @@ -2305,6 +2350,9 @@ int ds4_gpu_attention_output_q8_batch_tensor( uint64_t out_dim, const ds4_gpu_tensor *heads, uint32_t n_tokens); +/* Returns 1 when the batch path ran, 0 for the ordinary row fallback, and -1 + * when the Metal REQUIRE diagnostic made an ineligible/failed tiny batch a + * hard error. */ int ds4_gpu_attention_output_q4_K_batch_tensor( ds4_gpu_tensor *out, ds4_gpu_tensor *low, diff --git a/ds4_metal.m b/ds4_metal.m index 0cf905882..c7fadc6c7 100644 --- a/ds4_metal.m +++ b/ds4_metal.m @@ -11940,6 +11940,42 @@ static bool ds4_gpu_support_q8_decode_exact_views_enabled( model_size == g_support_model_map_size && getenv("DS4_METAL_DISABLE_SUPPORT_Q8_DECODE_EXACT_VIEWS") == NULL; } + +static id ds4_gpu_wrap_q8_decode_model_range( + const void *model_map, + uint64_t model_size, + uint64_t offset, + uint64_t len, + uint64_t n_tokens, + uint64_t *inner_offset) { + const uint64_t exact_decode_max_mib = + ds4_gpu_env_u64("DS4_METAL_Q8_DECODE_EXACT_VIEW_MAX_MIB", + 1024u, + 1u, + 4096u); + const uint64_t exact_decode_max_bytes = + exact_decode_max_mib * 1024ull * 1024ull; + const bool exact_support_batch = + n_tokens >= 1u && n_tokens <= 6u && + len <= exact_decode_max_bytes && + ds4_gpu_support_q8_decode_exact_views_enabled(model_map, model_size); + if (exact_support_batch) { + id exact = + ds4_gpu_wrap_model_exact_range(model_map, + model_size, + offset, + len, + inner_offset); + if (exact) return exact; + if (inner_offset) *inner_offset = 0; + } + return ds4_gpu_wrap_model_range(model_map, + model_size, + offset, + len, + inner_offset); +} + uint32_t ds4_gpu_stream_expert_cache_configured_count(void) { uint32_t budget = ds4_gpu_stream_expert_cache_configured_budget(); if (budget > DS4_METAL_STREAM_EXPERT_CACHE_MAX_ENTRIES) { @@ -18830,6 +18866,230 @@ int ds4_gpu_indexer_scores_decode_batch_tensor( scale); } +typedef struct { + uint32_t vocab; + uint32_t rank_blocks; + uint32_t hidden_dim; + uint32_t n_drafts; + uint32_t draft; + uint32_t reuse_confidence0; + float confidence_threshold; + float confidence0; +} ds4_metal_dspark_device_args; + +_Static_assert(sizeof(ds4_gpu_dspark_device_proposal) == 64u, + "DSpark device proposal ABI must stay 64 bytes"); +_Static_assert(offsetof(ds4_gpu_dspark_device_proposal, tokens) == 0u && + offsetof(ds4_gpu_dspark_device_proposal, + confidence_logits) == 24u && + offsetof(ds4_gpu_dspark_device_proposal, proposal_len) == 48u && + offsetof(ds4_gpu_dspark_device_proposal, confidence_len) == 52u && + offsetof(ds4_gpu_dspark_device_proposal, status) == 56u && + offsetof(ds4_gpu_dspark_device_proposal, reserved) == 60u, + "DSpark device proposal field offsets changed"); +_Static_assert(DS4_GPU_DSPARK_DEVICE_PROPOSAL_BYTES >= 1168u, + "DSpark Metal device proposal scratch is too small"); + +static int ds4_gpu_metal_dspark_device_proposer_enabled(void) { + return ds4_gpu_env_bool("DS4_METAL_DSPARK_DEVICE_PROPOSER") == 1 && + getenv("DS4_METAL_DSPARK_NO_DEVICE_PROPOSER") == NULL; +} + +int ds4_gpu_dspark_markov_confidence_q8_tensor( + ds4_gpu_tensor *out_result, + const ds4_gpu_tensor *logits_rows, + const ds4_gpu_tensor *hidden_rows, + const void *model_map, + uint64_t model_size, + uint64_t w1_offset, + uint64_t w2_offset, + uint64_t confidence_offset, + uint32_t first_prev_token, + uint32_t vocab, + uint32_t rank, + uint32_t hidden_dim, + uint32_t n_drafts, + float confidence_threshold, + int reuse_confidence0, + float confidence0) { + if (!g_initialized && !ds4_gpu_init()) return 0; + if (!ds4_gpu_metal_dspark_device_proposer_enabled() || + !out_result || !logits_rows || !hidden_rows || !model_map || + g_batch_cb || g_quality_mode || vocab == 0u || + first_prev_token >= vocab || rank == 0u || rank > 256u || + (rank & 31u) != 0u || hidden_dim == 0u || + (hidden_dim & 31u) != 0u || n_drafts == 0u || + n_drafts > DS4_GPU_DSPARK_MAX_DRAFTS || + !(confidence_threshold > 0.0f && confidence_threshold <= 1.0f) || + ds4_gpu_tensor_bytes(out_result) < + DS4_GPU_DSPARK_DEVICE_PROPOSAL_BYTES) { + return 0; + } + if ((uint64_t)n_drafts > UINT64_MAX / vocab / sizeof(float) || + ds4_gpu_tensor_bytes(logits_rows) < + (uint64_t)n_drafts * vocab * sizeof(float) || + (uint64_t)n_drafts > UINT64_MAX / hidden_dim / sizeof(float) || + ds4_gpu_tensor_bytes(hidden_rows) < + (uint64_t)n_drafts * hidden_dim * sizeof(float)) { + return 0; + } + + const uint32_t rank_blocks = rank / 32u; + const uint64_t markov_row_bytes = (uint64_t)rank_blocks * 34u; + if ((uint64_t)vocab > UINT64_MAX / markov_row_bytes) return 0; + const uint64_t markov_bytes = (uint64_t)vocab * markov_row_bytes; + const uint64_t feature_dim = (uint64_t)hidden_dim + rank; + if (feature_dim > UINT32_MAX || (feature_dim & 31u) != 0u) return 0; + const uint64_t confidence_bytes = (feature_dim / 32u) * 34u; + if (w1_offset > model_size || markov_bytes > model_size - w1_offset || + w2_offset > model_size || markov_bytes > model_size - w2_offset || + confidence_offset > model_size || + confidence_bytes > model_size - confidence_offset) { + return 0; + } + + @autoreleasepool { + id resultbuf = ds4_gpu_tensor_buffer(out_result); + id logitsbuf = ds4_gpu_tensor_buffer(logits_rows); + id hiddenbuf = ds4_gpu_tensor_buffer(hidden_rows); + if (!resultbuf || !logitsbuf || !hiddenbuf) return 0; + + uint64_t w1_inner = 0; + uint64_t w2_inner = 0; + uint64_t confidence_inner = 0; + id w1buf = ds4_gpu_wrap_q8_decode_model_range( + model_map, model_size, w1_offset, markov_bytes, n_drafts, + &w1_inner); + id w2buf = ds4_gpu_wrap_q8_decode_model_range( + model_map, model_size, w2_offset, markov_bytes, n_drafts, + &w2_inner); + id confidencebuf = + ds4_gpu_wrap_q8_decode_model_range( + model_map, model_size, confidence_offset, confidence_bytes, + n_drafts, &confidence_inner); + if (!w1buf || !w2buf || !confidencebuf) return 0; + + id init_pipeline = ds4_gpu_get_pipeline( + "kernel_dsv4_dspark_device_proposal_init"); + id confidence_pipeline = + ds4_gpu_get_pipeline( + "kernel_dsv4_dspark_device_confidence_q8"); + id scan_pipeline = ds4_gpu_get_pipeline( + "kernel_dsv4_dspark_device_markov_scan_q8"); + id reduce_pipeline = ds4_gpu_get_pipeline( + "kernel_dsv4_dspark_device_markov_reduce"); + id export_pipeline = ds4_gpu_get_pipeline( + "kernel_dsv4_dspark_device_proposal_export"); + if (!init_pipeline || !confidence_pipeline || !scan_pipeline || + !reduce_pipeline || !export_pipeline) { + return 0; + } + + int owned = 0; + id cb = ds4_gpu_command_buffer(&owned); + if (!cb || !owned) return 0; + + const NSUInteger result_offset = + ds4_gpu_tensor_offset(out_result); + const NSUInteger state_offset = + result_offset + sizeof(ds4_gpu_dspark_device_proposal); + const NSUInteger logits_offset = + ds4_gpu_tensor_offset(logits_rows); + const NSUInteger hidden_offset = + ds4_gpu_tensor_offset(hidden_rows); + const NSUInteger logits_row_bytes = + (NSUInteger)vocab * sizeof(float); + id state_barrier[1] = { resultbuf }; + + id enc = ds4_gpu_compute_encoder(cb); + [enc setComputePipelineState:init_pipeline]; + [enc setBytes:&first_prev_token + length:sizeof(first_prev_token) + atIndex:0]; + [enc setBuffer:resultbuf offset:state_offset atIndex:1]; + [enc dispatchThreadgroups:MTLSizeMake(1, 1, 1) + threadsPerThreadgroup:MTLSizeMake(1, 1, 1)]; + [enc memoryBarrierWithResources:state_barrier count:1u]; + ds4_gpu_end_compute_encoder(cb, enc); + + ds4_metal_dspark_device_args args = { + .vocab = vocab, + .rank_blocks = rank_blocks, + .hidden_dim = hidden_dim, + .n_drafts = n_drafts, + .draft = 0, + .reuse_confidence0 = reuse_confidence0 ? 1u : 0u, + .confidence_threshold = confidence_threshold, + .confidence0 = confidence0, + }; + for (uint32_t draft = 0; draft < n_drafts; draft++) { + args.draft = draft; + + enc = ds4_gpu_compute_encoder(cb); + [enc setComputePipelineState:confidence_pipeline]; + [enc setBytes:&args length:sizeof(args) atIndex:0]; + [enc setBuffer:hiddenbuf offset:hidden_offset atIndex:1]; + [enc setBuffer:w1buf offset:(NSUInteger)w1_inner atIndex:2]; + [enc setBuffer:confidencebuf + offset:(NSUInteger)confidence_inner + atIndex:3]; + [enc setBuffer:resultbuf offset:state_offset atIndex:4]; + [enc dispatchThreadgroups:MTLSizeMake(1, 1, 1) + threadsPerThreadgroup:MTLSizeMake(1, 1, 1)]; + [enc memoryBarrierWithResources:state_barrier count:1u]; + ds4_gpu_end_compute_encoder(cb, enc); + + enc = ds4_gpu_compute_encoder(cb); + [enc setComputePipelineState:scan_pipeline]; + [enc setBytes:&args length:sizeof(args) atIndex:0]; + [enc setBuffer:logitsbuf + offset:logits_offset + + (NSUInteger)draft * logits_row_bytes + atIndex:1]; + [enc setBuffer:w1buf offset:(NSUInteger)w1_inner atIndex:2]; + [enc setBuffer:w2buf offset:(NSUInteger)w2_inner atIndex:3]; + [enc setBuffer:resultbuf offset:state_offset atIndex:4]; + [enc dispatchThreadgroups:MTLSizeMake(128, 1, 1) + threadsPerThreadgroup:MTLSizeMake(256, 1, 1)]; + [enc memoryBarrierWithResources:state_barrier count:1u]; + ds4_gpu_end_compute_encoder(cb, enc); + + enc = ds4_gpu_compute_encoder(cb); + [enc setComputePipelineState:reduce_pipeline]; + [enc setBytes:&args length:sizeof(args) atIndex:0]; + [enc setBuffer:resultbuf offset:state_offset atIndex:1]; + [enc dispatchThreadgroups:MTLSizeMake(1, 1, 1) + threadsPerThreadgroup:MTLSizeMake(128, 1, 1)]; + [enc memoryBarrierWithResources:state_barrier count:1u]; + ds4_gpu_end_compute_encoder(cb, enc); + } + + enc = ds4_gpu_compute_encoder(cb); + [enc setComputePipelineState:export_pipeline]; + [enc setBuffer:resultbuf offset:result_offset atIndex:0]; + [enc setBuffer:resultbuf offset:state_offset atIndex:1]; + [enc dispatchThreadgroups:MTLSizeMake(1, 1, 1) + threadsPerThreadgroup:MTLSizeMake(1, 1, 1)]; + [enc memoryBarrierWithResources:state_barrier count:1u]; + ds4_gpu_end_compute_encoder(cb, enc); + + if (!ds4_gpu_finish_command_buffer( + cb, owned, "DSpark Metal device proposer")) { + return 0; + } + + static int logged = 0; + if (!logged) { + logged = 1; + fprintf(stderr, + "ds4: Metal DSpark device proposer enabled " + "(Q8 confidence/Markov, max_drafts=%u)\n", + DS4_GPU_DSPARK_MAX_DRAFTS); + } + } + return 1; +} + int ds4_gpu_indexer_topk_tensor( ds4_gpu_tensor *selected, const ds4_gpu_tensor *scores, @@ -20114,7 +20374,6 @@ int ds4_gpu_q4_K_pair_quad_compressor_store_tensor( (!out1_kv || !out1_score || !state1_kv || !state1_score)) { return -1; } - @autoreleasepool { const uint64_t q4_row_bytes = ((uint64_t)in_dim / 256u) * 144u; const uint64_t f16_row_bytes = (uint64_t)in_dim * sizeof(uint16_t); @@ -21694,51 +21953,95 @@ int ds4_gpu_qkv_pair_quad_compressor_store_tensor( const ds4_gpu_tensor *x, uint32_t ratio, uint32_t pos) { - if (!g_initialized && !ds4_gpu_init()) return 0; - if (!qr || !kv_raw || !out0_kv || !out0_score || !out1_kv || !out1_score || - !state0_kv || !state0_score || !state1_kv || !state1_score || - !model_map || !x || ratio == 0u || (in_dim & 31u) != 0 || - (width0 & 1u) != 0 || (width1 & 1u) != 0) { + if (!g_initialized && !ds4_gpu_init()) return -1; + if (!qr || !kv_raw || !out0_kv || !out0_score || + !state0_kv || !state0_score || !model_map || !x || + in_dim == 0u || (in_dim & 31u) != 0u || + q_rank == 0u || kv_dim == 0u || width0 == 0u || + q_rank > INT32_MAX || kv_dim > INT32_MAX || + width0 > INT32_MAX || width1 > INT32_MAX || + (width0 & 1u) != 0u || (width1 & 1u) != 0u || + (ratio != 4u && ratio != 128u) || + (ratio == 4u && width1 == 0u) || + (ratio == 128u && width1 != 0u) || + (ape0_type != 0u && ape0_type != 1u) || + (ape1_type != 0u && ape1_type != 1u)) { + return 0; + } + if (width1 != 0u && + (!out1_kv || !out1_score || !state1_kv || !state1_score)) { + return -1; + } + /* The compound kernel embeds the canonical NSG=4 Q8 pair and NR0=2 F16 + * reduction trees. Respect diagnostic schedule overrides by selecting + * the separate dispatches instead of silently changing their arithmetic. */ + if (ds4_gpu_make_q8_0_mv_dispatch().nsg != 4 || + ds4_gpu_use_compressor_pair_nr4()) { return 0; } @autoreleasepool { const uint64_t q8_row_bytes = (in_dim / 32u) * 34u; const uint64_t f16_row_bytes = (uint64_t)in_dim * sizeof(uint16_t); + if ((uint64_t)q_rank > UINT64_MAX / q8_row_bytes || + (uint64_t)kv_dim > UINT64_MAX / q8_row_bytes || + (uint64_t)width0 > UINT64_MAX / f16_row_bytes || + (width1 != 0u && + (uint64_t)width1 > UINT64_MAX / f16_row_bytes)) { + return -1; + } const uint64_t q_a_bytes = (uint64_t)q_rank * q8_row_bytes; const uint64_t kv_bytes = (uint64_t)kv_dim * q8_row_bytes; const uint64_t weight0_bytes = (uint64_t)width0 * f16_row_bytes; const uint64_t weight1_bytes = (uint64_t)width1 * f16_row_bytes; const uint64_t state_rows = ratio == 4u ? 2u * ratio : ratio; - const uint64_t ape0_bytes = (uint64_t)width0 * ratio * - (ape0_type == 1u ? 2u : 4u); - const uint64_t ape1_bytes = (uint64_t)width1 * ratio * - (ape1_type == 1u ? 2u : 4u); + const uint64_t ape0_elem = + ape0_type == 1u ? sizeof(uint16_t) : sizeof(float); + const uint64_t ape1_elem = + ape1_type == 1u ? sizeof(uint16_t) : sizeof(float); + if ((uint64_t)width0 > UINT64_MAX / ratio / ape0_elem || + (width1 != 0u && + (uint64_t)width1 > UINT64_MAX / ratio / ape1_elem)) { + return -1; + } + const uint64_t ape0_bytes = + (uint64_t)width0 * ratio * ape0_elem; + const uint64_t ape1_bytes = + (uint64_t)width1 * ratio * ape1_elem; if (q_a_offset > model_size || q_a_bytes > model_size - q_a_offset || kv_offset > model_size || kv_bytes > model_size - kv_offset || weight0_kv_offset > model_size || weight0_bytes > model_size - weight0_kv_offset || weight0_score_offset > model_size || weight0_bytes > model_size - weight0_score_offset || - weight1_kv_offset > model_size || - weight1_bytes > model_size - weight1_kv_offset || - weight1_score_offset > model_size || - weight1_bytes > model_size - weight1_score_offset || ape0_offset > model_size || ape0_bytes > model_size - ape0_offset || - ape1_offset > model_size || ape1_bytes > model_size - ape1_offset) { + (width1 != 0u && + (weight1_kv_offset > model_size || + weight1_bytes > model_size - weight1_kv_offset || + weight1_score_offset > model_size || + weight1_bytes > model_size - weight1_score_offset || + ape1_offset > model_size || + ape1_bytes > model_size - ape1_offset))) { return -1; } + const uint64_t state0_bytes = + state_rows * (uint64_t)width0 * sizeof(float); + const uint64_t state1_bytes = + state_rows * (uint64_t)width1 * sizeof(float); if (ds4_gpu_tensor_bytes(qr) < (uint64_t)q_rank * sizeof(float) || ds4_gpu_tensor_bytes(kv_raw) < (uint64_t)kv_dim * sizeof(float) || ds4_gpu_tensor_bytes(out0_kv) < (uint64_t)width0 * sizeof(float) || ds4_gpu_tensor_bytes(out0_score) < (uint64_t)width0 * sizeof(float) || - ds4_gpu_tensor_bytes(out1_kv) < (uint64_t)width1 * sizeof(float) || - ds4_gpu_tensor_bytes(out1_score) < (uint64_t)width1 * sizeof(float) || ds4_gpu_tensor_bytes(x) < (uint64_t)in_dim * sizeof(float) || - ds4_gpu_tensor_bytes(state0_kv) < state_rows * width0 * sizeof(float) || - ds4_gpu_tensor_bytes(state0_score) < state_rows * width0 * sizeof(float) || - ds4_gpu_tensor_bytes(state1_kv) < state_rows * width1 * sizeof(float) || - ds4_gpu_tensor_bytes(state1_score) < state_rows * width1 * sizeof(float)) { + ds4_gpu_tensor_bytes(state0_kv) < state0_bytes || + ds4_gpu_tensor_bytes(state0_score) < state0_bytes || + (width1 != 0u && + (ds4_gpu_tensor_bytes(out1_kv) < + (uint64_t)width1 * sizeof(float) || + ds4_gpu_tensor_bytes(out1_score) < + (uint64_t)width1 * sizeof(float) || + ds4_gpu_tensor_bytes(state1_kv) < state1_bytes || + ds4_gpu_tensor_bytes(state1_score) < state1_bytes))) { return -1; } @@ -21766,6 +22069,10 @@ int ds4_gpu_qkv_pair_quad_compressor_store_tensor( w1kv_inner = w0kv_inner; w1sc_inner = w0sc_inner; ape1_inner = ape0_inner; + out1_kv = out0_kv; + out1_score = out0_score; + state1_kv = state0_kv; + state1_score = state0_score; } if (!qw0buf || !qw1buf || !w0kvbuf || !w0scbuf || !w1kvbuf || !w1scbuf || !ape0buf || !ape1buf) return -1; @@ -26026,25 +26333,63 @@ int ds4_gpu_attention_output_q4_K_batch_tensor( uint64_t out_dim, const ds4_gpu_tensor *heads, uint32_t n_tokens) { - if (!g_initialized && !ds4_gpu_init()) return 0; + const bool tiny_scope = n_tokens >= 2u && n_tokens <= 5u; + const bool require_tiny = + tiny_scope && + ds4_gpu_env_bool("DS4_METAL_REQUIRE_Q4_ATTN_OUT_TINY_BATCH") == 1; + const int failure_rc = require_tiny ? -1 : 0; + if (!g_initialized && !ds4_gpu_init()) return failure_rc; if (!out || !low || !heads || !model_map || group_dim == 0 || rank == 0 || n_groups == 0 || out_dim == 0 || n_tokens == 0 || group_dim > UINT32_MAX || rank > UINT32_MAX || out_dim > UINT32_MAX) { - return 0; + return failure_rc; + } + + /* + * DSpark exact-N only needs two through five rows. The Q4 attention-A + * decode kernel already carries a token index in the same dispatch. The + * Q8 exact-row output projection and the Q4_K classic multi-row matvec + * both preserve the canonical one-row arithmetic independently for each + * token. Keep their composition experimental until it wins an end-to-end + * A/B: unsupported shapes and either disabled gate normally return zero + * to the caller's established row-wise fallback. The REQUIRE diagnostic + * implies the enable gate inside this tiny scope and turns any ineligible + * or failed candidate into -1 so a model-backed A/B cannot false-green. + * Respect the classic-Q4 kill switch so an explicitly selected alternate + * schedule stays canonical. + */ + const bool tiny_disabled = + ds4_gpu_env_bool("DS4_METAL_DISABLE_Q4_ATTN_OUT_TINY_BATCH") == 1; + const bool tiny_out_b_supported = + out_b_type == DS4_METAL_TENSOR_Q8_0 || + (out_b_type == DS4_METAL_TENSOR_Q4_K && + getenv("DS4_METAL_DISABLE_Q4_MV_CLASSIC") == NULL); + const bool use_tiny_exact = + tiny_scope && + tiny_out_b_supported && + (require_tiny || + ds4_gpu_env_bool("DS4_METAL_ENABLE_Q4_ATTN_OUT_TINY_BATCH") == 1) && + !tiny_disabled; + if (require_tiny && !use_tiny_exact) { + fprintf(stderr, + "ds4: required Metal Q4 attention-output tiny batch is " + "ineligible (rows=%u out_b_type=%u disabled=%u)\n", + n_tokens, out_b_type, tiny_disabled ? 1u : 0u); + return -1; } - if (n_tokens < 32u) return 0; + if (n_tokens < 32u && !use_tiny_exact) return 0; @autoreleasepool { const uint64_t low_dim = (uint64_t)n_groups * rank; if ((group_dim % 256u) != 0 || (low_dim % 256u) != 0 || low_dim > UINT32_MAX) { - return 0; + return failure_rc; } uint64_t row_a_bytes = 0; uint64_t row_b_bytes = 0; if (!ds4_gpu_quant_row_bytes(DS4_METAL_TENSOR_Q4_K, (uint32_t)group_dim, &row_a_bytes) || !ds4_gpu_quant_row_bytes(out_b_type, (uint32_t)low_dim, &row_b_bytes)) { - return 0; + return failure_rc; } const uint64_t out_a_bytes = (uint64_t)n_groups * rank * row_a_bytes; @@ -26052,7 +26397,7 @@ int ds4_gpu_attention_output_q4_K_batch_tensor( if (out_a_offset > model_size || out_a_bytes > model_size - out_a_offset || out_b_offset > model_size || out_b_bytes > model_size - out_b_offset) { fprintf(stderr, "ds4: Metal Q4 attention output batch weights are outside the mapped model\n"); - return 0; + return failure_rc; } const uint64_t heads_bytes = (uint64_t)n_tokens * n_groups * group_dim * sizeof(float); @@ -26062,7 +26407,7 @@ int ds4_gpu_attention_output_q4_K_batch_tensor( ds4_gpu_tensor_bytes(low) < low_bytes || ds4_gpu_tensor_bytes(out) < out_bytes) { fprintf(stderr, "ds4: Metal Q4 attention output batch received undersized buffers\n"); - return 0; + return failure_rc; } (void)group_tmp; (void)low_tmp; @@ -26101,7 +26446,7 @@ int ds4_gpu_attention_output_q4_K_batch_tensor( } const NSUInteger ids_bytes = (NSUInteger)n_tokens * (NSUInteger)n_groups * sizeof(int32_t); id group_ids_buffer = nil; - if (!use_mpp_low) { + if (!use_mpp_low && !use_tiny_exact) { if (getenv("DS4_METAL_DISABLE_ATTN_OUT_IDS_CACHE") != NULL) { group_ids_buffer = ds4_gpu_new_transient_buffer(ids_bytes, "attention output Q4 group ids"); @@ -26111,7 +26456,7 @@ int ds4_gpu_attention_output_q4_K_batch_tensor( "ds4_attention_output_group_ids")) { group_ids_buffer = g_attn_out_group_ids_buffer; } - if (!group_ids_buffer) return 0; + if (!group_ids_buffer) return failure_rc; int32_t *ids = (int32_t *)[group_ids_buffer contents]; for (uint32_t t = 0; t < n_tokens; t++) { @@ -26126,10 +26471,10 @@ int ds4_gpu_attention_output_q4_K_batch_tensor( ds4_gpu_wrap_model_range(model_map, model_size, out_a_offset, out_a_bytes, &out_a_inner); - if (!out_a_buf) return 0; + if (!out_a_buf) return failure_rc; const bool had_batch = g_batch_cb != nil; - if (!had_batch && ds4_gpu_begin_commands() == 0) return 0; + if (!had_batch && ds4_gpu_begin_commands() == 0) return failure_rc; bool ok = true; int owned = 0; @@ -26148,7 +26493,47 @@ int ds4_gpu_attention_output_q4_K_batch_tensor( use_mpp_low && use_mpp_padding ? padded_n_tokens : n_tokens); - if (use_mpp_low) { + if (use_tiny_exact) { + ds4_gpu_mul_mv_id_args args = { + .nei0 = (int32_t)n_groups, + .nei1 = (int32_t)n_tokens, + .nbi1 = 0, + .ne00 = (int32_t)group_dim, + .ne01 = (int32_t)rank, + .ne02 = (int32_t)n_groups, + .nb00 = 1, + .nb01 = row_a_bytes, + .nb02 = (uint64_t)rank * row_a_bytes, + .ne10 = (int32_t)group_dim, + .ne11 = (int32_t)n_groups, + .ne12 = (int32_t)n_tokens, + .ne13 = 1, + .nb10 = sizeof(float), + .nb11 = (uint64_t)group_dim * sizeof(float), + .nb12 = (uint64_t)n_groups * group_dim * sizeof(float), + .ne0 = (int32_t)rank, + .ne1 = (int32_t)n_groups, + .nb1 = (uint64_t)rank * sizeof(float), + .nr0 = 2, + }; + const NSUInteger nsg = 2u; + id pipeline = + ds4_gpu_get_mul_mv_pipeline( + "kernel_dsv4_attn_out_low_q4_K_f32", (int16_t)nsg); + ok = ds4_gpu_encode_attn_out_low_q8_direct( + cb, + pipeline, + &args, + out_a_buf, + (NSUInteger)out_a_inner, + ds4_gpu_tensor_buffer(heads), + ds4_gpu_tensor_offset(heads), + ds4_gpu_tensor_buffer(low), + ds4_gpu_tensor_offset(low), + 32u, + nsg, + false) != 0; + } else if (use_mpp_low) { ok = ds4_gpu_encode_attn_out_low_mpp( cb, mpp_low_pipeline, @@ -26187,21 +26572,43 @@ int ds4_gpu_attention_output_q4_K_batch_tensor( } if (ok) { - ok = ds4_gpu_matmul_quant_tensor(out, - model_map, - model_size, - out_b_offset, - out_b_type, - low_dim, - out_dim, - low, - n_tokens) != 0; + if (use_tiny_exact && out_b_type == DS4_METAL_TENSOR_Q8_0) { + ok = ds4_gpu_matmul_q8_0_decode_rows_exact_tensor( + out, + model_map, + model_size, + out_b_offset, + low_dim, + out_dim, + low, + n_tokens) != 0; + } else if (use_tiny_exact) { + ok = ds4_gpu_matmul_quant_tensor(out, + model_map, + model_size, + out_b_offset, + out_b_type, + low_dim, + out_dim, + low, + n_tokens) != 0; + } else { + ok = ds4_gpu_matmul_quant_tensor(out, + model_map, + model_size, + out_b_offset, + out_b_type, + low_dim, + out_dim, + low, + n_tokens) != 0; + } } if (!had_batch) { ok = ds4_gpu_end_commands() != 0 && ok; } - return ok ? 1 : 0; + return ok ? 1 : failure_rc; } } diff --git a/metal/dsv4_misc.metal b/metal/dsv4_misc.metal index 567b4e026..4d7e8c60d 100644 --- a/metal/dsv4_misc.metal +++ b/metal/dsv4_misc.metal @@ -6894,3 +6894,245 @@ kernel void kernel_dsv4_softmax_pool_ratio4_direct( dst[ic * args.head_dim + id] = acc/sum; } + +/* DSpark confidence/Markov tail. The public result occupies the first + * 64 bytes of a shared buffer; this private state starts immediately after + * it. Keeping the previous token and the 128 group candidates here lets a + * complete draft block use one CPU readback instead of one per row. */ +#define DS4_METAL_DSPARK_MAX_DRAFTS 6u +#define DS4_METAL_DSPARK_MARKOV_GROUPS 128u + +struct ds4_metal_dspark_device_args { + uint vocab; + uint rank_blocks; + uint hidden_dim; + uint n_drafts; + uint draft; + uint reuse_confidence0; + float confidence_threshold; + float confidence0; +}; + +struct ds4_metal_dspark_device_result { + int tokens[DS4_METAL_DSPARK_MAX_DRAFTS]; + float confidence_logits[DS4_METAL_DSPARK_MAX_DRAFTS]; + uint proposal_len; + uint confidence_len; + uint status; + uint reserved; +}; + +struct ds4_metal_dspark_device_state { + uint prev_token; + uint status; + uint active; + uint proposal_len; + uint confidence_len; + uint pad0; + uint pad1; + uint pad2; + int tokens[DS4_METAL_DSPARK_MAX_DRAFTS]; + float confidence_logits[DS4_METAL_DSPARK_MAX_DRAFTS]; + float group_values[DS4_METAL_DSPARK_MARKOV_GROUPS]; + uint group_indices[DS4_METAL_DSPARK_MARKOV_GROUPS]; +}; + +kernel void kernel_dsv4_dspark_device_proposal_init( + constant uint &first_prev_token [[buffer(0)]], + device ds4_metal_dspark_device_state *state [[buffer(1)]], + uint gid [[thread_position_in_grid]]) { + if (gid != 0u) return; + state->prev_token = first_prev_token; + state->status = 1u; + state->active = 1u; + state->proposal_len = 0u; + state->confidence_len = 0u; + for (uint i = 0u; i < DS4_METAL_DSPARK_MAX_DRAFTS; i++) { + state->tokens[i] = -1; + state->confidence_logits[i] = 0.0f; + } +} + +static inline float ds4_metal_dspark_q8_value( + device const block_q8_0 *row, + uint index) { + const uint b = index >> 5u; + return float(row[b].d) * float(row[b].qs[index & 31u]); +} + +kernel void kernel_dsv4_dspark_device_confidence_q8( + constant ds4_metal_dspark_device_args &args [[buffer(0)]], + device const float *hidden_rows [[buffer(1)]], + device const block_q8_0 *w1 [[buffer(2)]], + device const block_q8_0 *confidence [[buffer(3)]], + device ds4_metal_dspark_device_state *state [[buffer(4)]], + uint gid [[thread_position_in_grid]]) { + if (gid != 0u || state->status == 0u || state->active == 0u || + args.draft >= args.n_drafts) return; + + float acc = 0.0f; + if (args.draft == 0u && args.reuse_confidence0 != 0u) { + acc = args.confidence0; + } else { + const uint hidden_blocks = args.hidden_dim >> 5u; + const uint feature_blocks = hidden_blocks + args.rank_blocks; + device const block_q8_0 *w1_row = + w1 + (ulong)state->prev_token * args.rank_blocks; + device const float *hidden = + hidden_rows + (ulong)args.draft * args.hidden_dim; + + for (uint b = 0u; b < feature_blocks; b++) { + float values[32]; + float amax = 0.0f; + for (uint k = 0u; k < 32u; k++) { + const uint feature = b * 32u + k; + const float v = feature < args.hidden_dim + ? hidden[feature] + : ds4_metal_dspark_q8_value( + w1_row, feature - args.hidden_dim); + values[k] = v; + amax = max(amax, abs(v)); + } + const float xscale = amax / 127.0f; + const float inv = xscale != 0.0f ? 1.0f / xscale : 0.0f; + int isum = 0; + for (uint k = 0u; k < 32u; k++) { + int q = int(rint(values[k] * inv)); + q = clamp(q, -128, 127); + isum += int(confidence[b].qs[k]) * q; + } + acc += float(confidence[b].d) * xscale * float(isum); + } + } + + state->confidence_logits[args.draft] = acc; + state->confidence_len = args.draft + 1u; + const float e = exp(acc >= 0.0f ? -acc : acc); + const float probability = acc >= 0.0f + ? 1.0f / (1.0f + e) + : e / (1.0f + e); + if (probability < args.confidence_threshold) state->active = 0u; +} + +static inline bool ds4_metal_dspark_score_better( + float av, uint ai, float bv, uint bi) { + return av > bv || (av == bv && ai < bi); +} + +kernel void kernel_dsv4_dspark_device_markov_scan_q8( + constant ds4_metal_dspark_device_args &args [[buffer(0)]], + device const float *logits [[buffer(1)]], + device const block_q8_0 *w1 [[buffer(2)]], + device const block_q8_0 *w2 [[buffer(3)]], + device ds4_metal_dspark_device_state *state [[buffer(4)]], + uint tid [[thread_index_in_threadgroup]], + uint group [[threadgroup_position_in_grid]]) { + threadgroup float markov_state[256]; + threadgroup float values[256]; + threadgroup uint indices[256]; + + if (state->active == 0u || state->status == 0u) { + if (tid == 0u && group < DS4_METAL_DSPARK_MARKOV_GROUPS) { + state->group_values[group] = -INFINITY; + state->group_indices[group] = 0u; + } + return; + } + + const uint rank = args.rank_blocks * 32u; + device const block_q8_0 *w1_row = + w1 + (ulong)state->prev_token * args.rank_blocks; + if (tid < rank) { + markov_state[tid] = ds4_metal_dspark_q8_value(w1_row, tid); + } + threadgroup_barrier(mem_flags::mem_threadgroup); + + float best_v = -INFINITY; + uint best_i = 0u; + const uint first = group * 256u + tid; + const uint stride = DS4_METAL_DSPARK_MARKOV_GROUPS * 256u; + for (uint i = first; i < args.vocab; i += stride) { + device const block_q8_0 *row = + w2 + (ulong)i * args.rank_blocks; + float acc = 0.0f; + for (uint b = 0u; b < args.rank_blocks; b++) { + float s = 0.0f; + for (uint k = 0u; k < 32u; k++) { + s += float(row[b].qs[k]) * markov_state[b * 32u + k]; + } + acc += float(row[b].d) * s; + } + const float v = logits[i] + acc; + if (ds4_metal_dspark_score_better(v, i, best_v, best_i)) { + best_v = v; + best_i = i; + } + } + + values[tid] = best_v; + indices[tid] = best_i; + threadgroup_barrier(mem_flags::mem_threadgroup); + for (uint step = 128u; step != 0u; step >>= 1u) { + if (tid < step && + ds4_metal_dspark_score_better( + values[tid + step], indices[tid + step], + values[tid], indices[tid])) { + values[tid] = values[tid + step]; + indices[tid] = indices[tid + step]; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + } + if (tid == 0u) { + state->group_values[group] = values[0]; + state->group_indices[group] = indices[0]; + } +} + +kernel void kernel_dsv4_dspark_device_markov_reduce( + constant ds4_metal_dspark_device_args &args [[buffer(0)]], + device ds4_metal_dspark_device_state *state [[buffer(1)]], + uint tid [[thread_index_in_threadgroup]]) { + threadgroup float values[DS4_METAL_DSPARK_MARKOV_GROUPS]; + threadgroup uint indices[DS4_METAL_DSPARK_MARKOV_GROUPS]; + if (state->active == 0u || state->status == 0u) return; + + values[tid] = state->group_values[tid]; + indices[tid] = state->group_indices[tid]; + threadgroup_barrier(mem_flags::mem_threadgroup); + for (uint step = DS4_METAL_DSPARK_MARKOV_GROUPS / 2u; + step != 0u; step >>= 1u) { + if (tid < step && + ds4_metal_dspark_score_better( + values[tid + step], indices[tid + step], + values[tid], indices[tid])) { + values[tid] = values[tid + step]; + indices[tid] = indices[tid + step]; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + } + if (tid == 0u) { + const uint token = indices[0]; + if (!isfinite(values[0]) || token >= args.vocab) { + state->status = 0u; + return; + } + state->tokens[args.draft] = int(token); + state->prev_token = token; + state->proposal_len = args.draft + 1u; + } +} + +kernel void kernel_dsv4_dspark_device_proposal_export( + device ds4_metal_dspark_device_result *out [[buffer(0)]], + device const ds4_metal_dspark_device_state *state [[buffer(1)]], + uint gid [[thread_position_in_grid]]) { + if (gid != 0u) return; + for (uint i = 0u; i < DS4_METAL_DSPARK_MAX_DRAFTS; i++) { + out->tokens[i] = state->tokens[i]; + out->confidence_logits[i] = state->confidence_logits[i]; + } + out->proposal_len = state->proposal_len; + out->confidence_len = state->confidence_len; + out->status = state->status; + out->reserved = 0u; +} diff --git a/speed-bench/ds4_gb10_q2_cuda_port_results.md b/speed-bench/ds4_gb10_q2_cuda_port_results.md new file mode 100644 index 000000000..18d0cf4c3 --- /dev/null +++ b/speed-bench/ds4_gb10_q2_cuda_port_results.md @@ -0,0 +1,324 @@ +# GB10 Q2 CUDA port results + +This records the Q2 CUDA baseline and the exact kernel work promoted for the +NVIDIA GB10 / DGX Spark path. The starting tree was `b030961` (`main`) and the +model was: + +```text +gguf/DeepSeek-V4-Flash-IQ2XXS-w2Q2K-AProjQ8-SExpQ8-OutQ8-chat-v2-imatrix-0731.gguf +``` + +The test machine was an NVIDIA GB10 (`sm_121`) with nominal 128 GB unified +memory, driver 580.173.02, and CUDA 13.0. Builds used: + +```sh +make cuda-spark +``` + +## Branch audit and selected work + +`origin/mxfp4-m3` is 24 commits ahead of the starting tree, but its net CUDA +backend delta is only four compatibility lines. Its throughput work is in +Metal. The MXFP4 LUT/tile kernels are not directly applicable to this +IQ2_XXS/Q2_K model, and CUDA already has equivalents for several exact Metal +fusions (router top-k/weights, Q/KV RMS+RoPE, compressor pool math, and HC +split/weighted-sum/norm). + +The two useful remaining ideas were removal of intermediate materialization and +an exact producer/store epilogue: + +1. **Direct Q2 prefill.** The existing dormant direct D2R MMQ now has a + production dispatch. IQ2 gate/up accumulators remain in registers and + weighted SwiGLU is quantized directly to Q8_1 for the Q2_K down MMQ. It + avoids materializing F32 gate, up, and mid tensors. +2. **F16 compressor projection/store.** The established ordered F16 pair + matvec now optionally writes the compressor state in lane 0 of the same + kernel. This removes 62 dependent state-store launches per Flash decode + token (42 ratio-4 attention/indexer calls and 20 ratio-128 attention calls). + +Both defaults are limited to GB10. The direct path has shape, residency, +aligned-artifact, streaming, scratch, and token-count fallbacks. Its public MMQ +contract returns `DS4_MMQ_NOT_APPLICABLE` only before any enqueue; negative +results are never retried. It is disabled while graph intermediates are being +dumped because its scratch intentionally overwrites the otherwise-dead +gate/up/mid buffers. The compressor fusion is limited to the three validated +4096-wide Flash/Pro calls: `(ratio,width) = (4,256), (4,1024), (128,512)`. + +Rollback switches are: + +```text +DS4_CUDA_NO_DIRECT_Q2_PREFILL=1 +DS4_CUDA_NO_F16_PAIR_COMPRESSOR_STORE=1 +``` + +The existing ordered-pair rollback switches also dominate the compressor +fusion. + +Two experiments were not promoted: exact score split DIM2 was neutral (808.41 +prefill tok/s in its initial 2K screen), and a QKV RoPE/FP8/cache-store +prototype preserved output but did not provide a worthwhile speedup. + +## Reproducible baseline + +The original `b030961` benchmark binary was retained on the author's test +host at `/tmp/ds4-bench-main-baseline` with SHA-256: + +```text +6cf857252c1dbbc64add2b94857b3be568615c6e6e863961d4df938ad9f3ec98 +``` + +Four interleaved 2,048-token runs of the ordinary path used +`speed-bench/promessi_sposi.txt`: + +| run | prefill tok/s | decode tok/s | steady decode tok/s | +|---:|---:|---:|---:| +| 1 | 810.26 | 17.78 | 17.93 | +| 2 | 809.01 | 17.78 | 17.94 | +| 3 | 815.58 | 17.73 | 17.89 | +| 4 | 807.88 | 17.61 | 17.77 | +| **mean** | **810.6825** | **17.725** | **17.8825** | + +## Performance + +The final aggregate was measured in ABBA order with 2,048 prefill tokens and +512 generated tokens. The control set both rollback variables; the optimized +arm unset both. The per-run command body was: + +```sh +./ds4-bench -m ds4flash.gguf --cuda \ + --prompt-file speed-bench/promessi_sposi.txt \ + --ctx-start 2048 --ctx-max 2048 --step-incr 2048 \ + --gen-tokens 512 --csv OUT.csv +``` + +| arm/run | prefill tok/s | decode tok/s | steady decode tok/s | +|---|---:|---:|---:| +| rollback 1 | 808.86 | 17.66 | 17.78 | +| default 1 | 829.89 | 17.85 | 17.97 | +| default 2 | 829.70 | 17.84 | 17.96 | +| rollback 2 | 805.29 | 17.48 | 17.60 | +| **rollback mean** | **807.075** | **17.570** | **17.690** | +| **default mean** | **829.795** | **17.845** | **17.965** | +| **gain** | **+2.82%** | **+1.57%** | **+1.55%** | + +Isolated screens agree with the attribution: + +- Direct prefill: 830.6275 versus 810.6825 tok/s over four interleaved runs, + **+2.46%**; decode was neutral (+0.06%). +- Compressor/store: 17.935 versus 17.765 decode tok/s and 18.055 versus 17.885 + steady tok/s over a 512-token ABBA screen, **+0.96% / +0.95%**. + +Author-local raw artifacts are under `/tmp/ds4-opt/`, notably `final-abba/`, +`final-correct/`, `abba/`, and `compstore/abba/`; these paths are provenance +notes, not required to reproduce the checked-in CSVs. + +## Exactness and safety validation + +Frontier hashes alone were not used as the decode criterion. + +- A 2,782-token prompt with a 128-token greedy decode limit (56 tokens + emitted through EOS) produced byte-identical stdout and full per-token + logprob JSON with both defaults on versus both rollbacks. SHA-256: + `6519499391d81b625344a8335ba10fde02d1a2c4e414eb65e6ee711fa3b37d14`. +- The isolated compressor A/B generated-token/logprob artifact is + byte-identical with SHA-256 + `9b76a6b7579a192d0aeb04bb48616944313cee805f7cbba158c99655ad657868`. +- The direct prefill frontier JSON was fully parsed and byte-identical, + including every logit and selected ID, with SHA-256 + `e7b419e8ebbcb6c40a5eccfe8784645d910dbf203df7960500d4d8d133f061a8`. +- `DS4_MMQ_YIND_VERIFY=1` checked all 43 routed layers at 2,048 tokens; every + layer reported `bad=0/393216` Q8 staging values. +- Small/ragged batches fall back normally; disabling D2R and ordered pair MMQ + together also completed without an aligned-path error. +- `compute-sanitizer --tool memcheck` completed the fused decode path with + `ERROR SUMMARY: 0 errors`. +- `make cuda-regression` passed its long-context/top-k smoke. +- `make test CUDA_ARCH=sm_121` passed in full. This included a 30,474-token + long-context run (direct batches at 4,096 and the final ragged tail), five + logprob vectors, exact tensor-equivalence vectors, local golden vectors, + server tests, sampling tests, and all ordinary unit tests. +- Two independent source/safety reviews found no release blocker after the + scratch-overflow, tri-state, debug, target-admission, and fallback-contract + guards were applied. + +Nsight Compute hardware counters were unavailable on this system because +`RmProfilingAdminOnly=1`; performance conclusions therefore use controlled +wall-clock ABBA measurements plus exact output and sanitizer validation. + +## Decode phase: approximately 20 steady tok/s + +A second profiling/optimization pass targeted the remaining decode path. Nsight +Systems showed that the token was dominated by exact, bandwidth-bound Q8 and +compressor projections. Four changes were promoted: + +1. **No CUDA mid-token split synchronization.** Metal's four-layer split + asynchronously commits a command buffer, but the CUDA implementation of the + same flush calls `cudaDeviceSynchronize()`. CUDA now defaults the split to + zero; Apple, ROCm, and CPU builds retain four. The existing + `DS4_METAL_GRAPH_TOKEN_SPLIT_LAYERS` override remains available. +2. **Coalesced exact F16 compressor pairs.** On first use, each validated + compressor KV/score weight pair is repacked into an interleaved chunk-32 + layout. Every lane still accumulates its original contiguous 128 values and + lane 0 performs the same ordered sum, but weights at a given iteration are + coalesced. An eight-value load prefetch preserves the arithmetic chain. + The transpose is enabled only on the validated single-GB10 path. The cache + is device-qualified, released on map change/cleanup, refuses graph capture + allocation, and permanently falls back after an allocation failure. +3. **Aligned Q8 fused consumers.** Decode Q8 pair, HC-expand, and grouped + attention-A kernels now consume the already-built aligned Q8 artifacts + directly instead of returning to misaligned raw 34-byte blocks. DP4A term + order, per-lane accumulation, warp reduction, and all epilogues are + unchanged. Admission is limited to one validated GB10, exact aligned + artifacts, full tensors, and dimensions divisible by the proven tile sizes; + multi-GPU and unsupported shapes fall back to the raw kernels. +4. **Persistent aligned Q8 projections.** The K=1024 Q-b projection and K=4096 + vocabulary projection use eight row warps per persistent CTA on one GB10. The + K=1024 kernel hoists the immutable activation into registers. Lane/block + assignment and the float warp tree remain identical; integer dot regrouping + is overflow-safe. The generic aligned kernel remains the fallback. + +New CUDA rollback switches are: + +```text +DS4_CUDA_NO_F16_PAIR_COMPRESSOR_TRANSPOSE=1 +DS4_CUDA_NO_F16_PAIR_COMPRESSOR_TRANSPOSE_PREFETCH8=1 +DS4_CUDA_NO_Q8_FUSED_ALIGNED=1 +DS4_CUDA_NO_Q8_ALIGNED_PERSISTENT=1 +``` + +`DS4_METAL_GRAPH_TOKEN_SPLIT_LAYERS=4` is the existing, Metal-named token-split +control; setting it to `4` restores the old blocking split behavior on CUDA. + +### Initial ABBA result + +The final clean-build comparison used the same 2,048-token prompt frontier and +512 generated tokens. The rollback arm set all five switches above; the default +arm set none. + +| arm/run | prefill tok/s | decode tok/s | steady decode tok/s | +|---|---:|---:|---:| +| rollback 1 | 832.25 | 17.98 | 18.10 | +| default 1 | 830.41 | 19.85 | 20.01 | +| default 2 | 829.24 | 19.81 | 19.97 | +| rollback 2 | 832.36 | 17.94 | 18.06 | +| **rollback mean** | **832.305** | **17.960** | **18.080** | +| **default mean** | **829.825** | **19.830** | **19.990** | +| **gain** | **-0.30%** | **+10.41%** | **+10.56%** | + +A post-audit clean run measured **19.92 decode / 20.08 steady tok/s**. Using +the ABBA default means (19.830 decode and 19.990 steady) relative to the +original `b030961` mean, the complete port is **+11.88% decode** and +**+11.79% steady decode**, while retaining a **+2.36% prefill** gain. Two +1,024-generation runs measured 19.73/19.88 and 19.71/19.86 decode/steady as the +attention context grew from 2,048 to 3,072 tokens. + +Profiler attribution matched the wall-clock result. The ordered F16 compressor +family fell from about 5.14 ms/token to about 2.96 ms/token. The persistent +K=1024 Q8 kernel fell from roughly 165 us to 150 us per layer, while the aligned +fused Q8 kernels removed the raw-block penalty from pair, HC-expand, and +attention-A projections. + +### Pre-target-21 exactness and validation + +- A 7,000-byte *Promessi sposi* prompt plus up to 256 greedy generated tokens + produced byte-identical text and full per-token logprob JSON with all defaults + versus all five decode rollbacks. SHA-256: + `128362d060d18e38ebadc7649c18ca8db625f53b5ee3b12762db9859d508d174`. +- A post-audit 64-token default/rollback repeat was also byte-identical, SHA-256 + `3f3d8890e13d4118b6a8964ed9ab3e2d11c8f0e3f8ab718d85a574bcf8cad6e5`. +- `compute-sanitizer --tool memcheck` completed the promoted path with + `ERROR SUMMARY: 0 errors`. +- A clean `make cuda-spark`, `make cuda-regression CUDA_ARCH=sm_121`, and + `make test CUDA_ARCH=sm_121` all passed after the final + safety hardening. The full test again included the 30,474-token context and + exact tensor-equivalence suite. +- Final source audit added single-device derived-artifact admission, active + device validation, OOM negative caching, capture and divisibility guards, + persistent-grid overflow guards, signed-zero preservation, init/reset + hardening, and direct-MMQ scratch-size overflow admission. + +Neutral or regressive experiments were not promoted: grouped top-6 MoE CTAs, +stream-0 graph replay, extending graph island A through Q/KV, small-output F16, +exact-score split LDG/vector variants, compressor CTA grouping, persistent-grid +retuning, and HC RMS-fold continuation. + +## README speed-table replication + +The upstream README's GB10 sweep was repeated with the final target-only build +(no DSpark support model or speculative flags), using the same 2,048-token +frontiers and 128 greedy generation tokens: + +```sh +./ds4-bench -m ds4flash.gguf --cuda \ + --prompt-file speed-bench/promessi_sposi.txt \ + --ctx-start 2048 --ctx-max 65536 --step-incr 2048 \ + --gen-tokens 128 \ + --csv speed-bench/gb10.csv +``` + +| Context | README prefill | Final prefill | README generation | Final generation | Final steady | +|---:|---:|---:|---:|---:|---:| +| 2,048 | 825.76 | 832.86 | 18.05 | 20.58 | 20.69 | +| 16,384 | 872.44 | 883.81 | 15.10 | 16.80 | 16.81 | +| 32,768 | 855.94 | 865.40 | 14.43 | 15.99 | 16.00 | +| 65,536 | 822.98 | 833.44 | 13.84 | 15.27 | 15.28 | + +Generation gains at those four table rows are respectively **+14.02%**, +**+11.26%**, **+10.81%**, and **+10.33%**. The complete 32-frontier sweep is in +`speed-bench/gb10.csv`; benchmark binary SHA-256 was +`04d5321402dc073b1f0300a19063e95262a81cc20831fa4f083cf9147fcc145f`. + +## Target-21 follow-up + +A review of `eugr/spark-vllm-docker` at commit +`e5f3cf9e5320d9a424966a801570bf452405d122` led to the referenced B12X SM121 +kernel package at `7cecbb2c4819636ae7f05f8b116f2c45ee2cff7b`. The applicable +ideas were stable caller-owned scratch (no per-token asynchronous allocation), +the split/parallel MHC decode structure, and measured GB10 launch geometry. +Its tensor-core FP4/FP8, sparse-MLA, and MoE scheduler machinery is not a +drop-in for this IQ2_XXS/Q2_K/F32-exact path. + +Two exact changes were promoted: + +1. The dense aligned-Q8 wrapper now prefers the already-owned 256 KiB aligned + Q8_1 scratch allocation instead of recording a per-token `cudaMallocAsync` + pool node. Rollback: `DS4_CUDA_NO_Q8_ALIGNED_DENSE_SCRATCH=1`. +2. The single-row 4K HC weighted-sum + RMS kernel now uses 16 partial CTAs, + a one-CTA exact reduction replay, and 16 store CTAs. The reduction retains + the original ascending-column FMA chain and 256-lane shared tree. Rollback: + `DS4_CUDA_NO_HC_SPLIT_NORM_SPLIT4096=1`. + +Matched 512-generation AB results at 2,048 context: + +| Variant | Steady runs | Median steady | +|---|---:|---:| +| Both target-21 rollbacks | 19.78, 19.89 | 19.835 t/s | +| Aligned scratch only | 20.68, 20.69 | 20.685 t/s | +| Final 16-CTA HC split | 21.01, 21.06 | 21.035 t/s | + +A 1,024-generation run measured 20.73 decode and 20.89 steady t/s as context +grew from 2,048 to 3,072 tokens. The full 32-frontier sweep in +`gb10.csv` was rerun with this final build. + +The 7,000-byte prompt plus 256 greedy generated tokens remained byte-identical +to the HC rollback, including full per-token logprob JSON. SHA-256: +`952799babb7f421cb0e2e75e6ede9de73c40304da279ef1d8d99042ef684be62`. +`make test CUDA_ARCH=sm_121`, `make cuda-regression CUDA_ARCH=sm_121`, and +`compute-sanitizer --tool memcheck` all passed; sanitizer reported +`ERROR SUMMARY: 0 errors`. + +## Post-rebase validation + +The branch was rebased onto `upstream/main` commit +`84cc882352757baf628a1776badf7cc54d584e28` and retested. The rebased CUDA +build passed `make cuda-spark`, `make test CUDA_ARCH=sm_121`, and +`make cuda-regression CUDA_ARCH=sm_121`. Compute Sanitizer reported +`ERROR SUMMARY: 0 errors`. The 7,000-byte prompt plus 256 greedy +tokens remained byte-identical to the rollback logprobs, SHA-256 +`952799babb7f421cb0e2e75e6ede9de73c40304da279ef1d8d99042ef684be62`. + +Same-day 512-generation runs after the rebase measured 20.69, 20.65, and 20.74 +steady tok/s at 2,048 context. A preserved pre-rebase worktree measured 20.67 +and 20.67 steady tok/s in the same thermal window, so the rebase itself is +performance-neutral within noise. The full sweep and README table above were +refreshed with the rebased build. diff --git a/speed-bench/gb10.csv b/speed-bench/gb10.csv index a59f1dfe4..60faa6b37 100644 --- a/speed-bench/gb10.csv +++ b/speed-bench/gb10.csv @@ -1,33 +1,33 @@ ctx_tokens,prefill_tokens,prefill_tps,gen_tokens,gen_tps,gen_first_ms,gen_steady_tokens,gen_steady_tps,kvcache_bytes -2048,2048,825.76,128,18.05,71.721,127,18.20,52184460 -4096,2048,899.52,128,15.47,57.061,127,15.54,80373132 -6144,2048,888.62,128,15.37,65.899,127,15.45,108561804 -8192,2048,882.80,128,15.21,65.667,127,15.29,136750476 -10240,2048,874.95,128,15.13,67.430,127,15.21,164939148 -12288,2048,872.74,128,15.08,67.877,127,15.16,193127820 -14336,2048,867.76,128,15.02,67.140,127,15.10,221316492 -16384,2048,872.44,128,15.10,67.128,127,15.18,249505164 -18432,2048,871.75,128,15.04,67.562,127,15.12,277693836 -20480,2048,869.55,128,15.00,67.678,127,15.08,305882508 -22528,2048,866.37,128,14.96,68.248,127,15.04,334071180 -24576,2048,864.65,128,14.91,68.667,127,14.99,362259852 -26624,2048,861.34,128,14.86,68.919,127,14.94,390448524 -28672,2048,859.07,128,14.82,69.044,127,14.90,418637196 -30720,2048,856.88,128,14.80,68.233,127,14.87,446825868 -32768,2048,855.94,128,14.43,68.704,127,14.51,475014540 -34816,2048,850.96,128,14.36,70.661,127,14.43,503203212 -36864,2048,849.83,128,14.34,70.827,127,14.41,531391884 -38912,2048,849.59,128,14.30,70.971,127,14.38,559580556 -40960,2048,847.40,128,14.28,70.881,127,14.36,587769228 -43008,2048,843.24,128,14.24,71.968,127,14.32,615957900 -45056,2048,840.41,128,14.19,71.406,127,14.26,644146572 -47104,2048,839.95,128,14.16,71.523,127,14.23,672335244 -49152,2048,836.64,128,14.12,72.153,127,14.19,700523916 -51200,2048,837.72,128,14.08,71.579,127,14.15,728712588 -53248,2048,837.01,128,14.04,72.138,127,14.11,756901260 -55296,2048,829.92,128,14.00,73.733,127,14.08,785089932 -57344,2048,830.16,128,13.98,72.862,127,14.05,813278604 -59392,2048,829.11,128,13.94,73.157,127,14.01,841467276 -61440,2048,827.26,128,13.90,73.924,127,13.97,869655948 -63488,2048,825.04,128,13.86,73.787,127,13.93,897844620 -65536,2048,822.98,128,13.84,73.966,127,13.91,0 +2048,2048,832.86,128,20.58,77.503,127,20.69,52184460 +4096,2048,909.55,128,17.28,50.694,127,17.28,80373132 +6144,2048,901.83,128,17.15,59.791,127,17.16,108561804 +8192,2048,894.75,128,16.96,60.946,127,16.97,136750476 +10240,2048,884.88,128,16.85,60.291,127,16.86,164939148 +12288,2048,879.46,128,16.78,61.277,127,16.79,193127820 +14336,2048,876.32,128,16.71,61.579,127,16.72,221316492 +16384,2048,883.81,128,16.80,60.258,127,16.81,249505164 +18432,2048,881.73,128,16.73,61.953,127,16.74,277693836 +20480,2048,879.01,128,16.68,59.971,127,16.69,305882508 +22528,2048,875.57,128,16.64,61.660,127,16.65,334071180 +24576,2048,872.93,128,16.60,61.166,127,16.61,362259852 +26624,2048,871.46,128,16.54,62.080,127,16.55,390448524 +28672,2048,869.00,128,16.49,60.657,127,16.50,418637196 +30720,2048,865.56,128,16.46,62.215,127,16.47,446825868 +32768,2048,865.40,128,15.99,63.022,127,16.00,475014540 +34816,2048,859.60,128,15.91,64.303,127,15.92,503203212 +36864,2048,860.68,128,15.87,65.082,127,15.88,531391884 +38912,2048,857.03,128,15.83,63.978,127,15.84,559580556 +40960,2048,854.89,128,15.79,64.615,127,15.80,587769228 +43008,2048,853.18,128,15.76,64.846,127,15.76,615957900 +45056,2048,850.05,128,15.70,65.960,127,15.71,644146572 +47104,2048,847.88,128,15.66,65.232,127,15.67,672335244 +49152,2048,845.96,128,15.63,65.233,127,15.64,700523916 +51200,2048,846.48,128,15.58,65.126,127,15.59,728712588 +53248,2048,844.07,128,15.53,65.666,127,15.54,756901260 +55296,2048,839.85,128,15.48,66.767,127,15.49,785089932 +57344,2048,839.90,128,15.45,66.645,127,15.46,813278604 +59392,2048,835.70,128,15.41,65.679,127,15.42,841467276 +61440,2048,837.29,128,15.36,66.754,127,15.37,869655948 +63488,2048,832.94,128,15.32,67.365,127,15.33,897844620 +65536,2048,833.44,128,15.27,67.665,127,15.28,0 diff --git a/tests/ds4_test.c b/tests/ds4_test.c index da986f177..1ffac1c9e 100644 --- a/tests/ds4_test.c +++ b/tests/ds4_test.c @@ -1007,6 +1007,790 @@ static void test_metal_q8_0_decode_pair_exact(void) { } #if defined(__APPLE__) +static void test_fill_q4_K_weights(uint8_t *weights, + uint32_t in_dim, + uint32_t out_dim, + uint32_t seed) { + /* GGUF Q4_K block: f16 d, f16 dmin, 12 packed scales, 128 quants. */ + const uint32_t block_elems = 256u; + const uint32_t block_bytes = 144u; + TEST_ASSERT(weights != NULL); + TEST_ASSERT((in_dim % block_elems) == 0u); + if (!weights || (in_dim % block_elems) != 0u) return; + + const uint32_t blocks_per_row = in_dim / block_elems; + for (uint32_t row = 0; row < out_dim; row++) { + for (uint32_t block = 0; block < blocks_per_row; block++) { + uint8_t *q = weights + + ((uint64_t)row * blocks_per_row + block) * block_bytes; + const uint32_t key = + seed + row * 1009u + block * 313u + (row ^ (block * 17u)); + const uint16_t d = test_float_to_f16( + 0.0025f + (float)(key % 13u) / 4096.0f); + const uint16_t dmin = test_float_to_f16( + 0.0010f + (float)((key >> 3u) % 7u) / 8192.0f); + memcpy(q + 0u, &d, sizeof(d)); + memcpy(q + 2u, &dmin, sizeof(dmin)); + for (uint32_t i = 0; i < 12u; i++) { + q[4u + i] = (uint8_t)( + 1u + ((key + i * 23u + (i ^ row) * 5u) % 0xfeu)); + } + for (uint32_t i = 0; i < 128u; i++) { + q[16u + i] = (uint8_t)( + key + i * 37u + (i >> 2u) * 11u + row * 3u); + } + } + } +} + +static void test_metal_q8_0_decode_rows_exact(void) { + /* The exact-N output head must preserve the canonical one-row Q8_0 + * arithmetic for every speculative row, including the output tail. */ + const uint32_t in_dim = 4096u; + const uint32_t out_dim = 259u; + const uint32_t max_rows = 5u; + const uint64_t page = (uint64_t)getpagesize(); + const uint64_t row_bytes = (uint64_t)(in_dim / 32u) * 34u; + const uint64_t weight_bytes = (uint64_t)out_dim * row_bytes; + const uint64_t weight_alloc = test_round_up_u64(weight_bytes, page); + const uint64_t x_row_bytes = (uint64_t)in_dim * sizeof(float); + const uint64_t out_row_bytes = (uint64_t)out_dim * sizeof(float); + const uint64_t x_bytes = (uint64_t)max_rows * x_row_bytes; + const uint64_t out_bytes = (uint64_t)max_rows * out_row_bytes; + + void *weights_raw = NULL; + TEST_ASSERT(posix_memalign(&weights_raw, + (size_t)page, + (size_t)weight_alloc) == 0); + if (!weights_raw) return; + memset(weights_raw, 0, (size_t)weight_alloc); + test_fill_q8_0_weights(weights_raw, in_dim, out_dim, 173u); + + ds4_gpu_tensor *x = ds4_gpu_tensor_alloc(x_bytes); + ds4_gpu_tensor *reference = ds4_gpu_tensor_alloc(out_bytes); + ds4_gpu_tensor *batched = ds4_gpu_tensor_alloc(out_bytes); + TEST_ASSERT(x != NULL); + TEST_ASSERT(reference != NULL); + TEST_ASSERT(batched != NULL); + if (!x || !reference || !batched) { + ds4_gpu_tensor_free(x); + ds4_gpu_tensor_free(reference); + ds4_gpu_tensor_free(batched); + free(weights_raw); + return; + } + + float *x_host = malloc((size_t)x_bytes); + float *reference_host = malloc((size_t)out_bytes); + float *batched_host = malloc((size_t)out_bytes); + TEST_ASSERT(x_host != NULL); + TEST_ASSERT(reference_host != NULL); + TEST_ASSERT(batched_host != NULL); + if (!x_host || !reference_host || !batched_host) { + free(x_host); + free(reference_host); + free(batched_host); + ds4_gpu_tensor_free(x); + ds4_gpu_tensor_free(reference); + ds4_gpu_tensor_free(batched); + free(weights_raw); + return; + } + + for (uint32_t row = 0; row < max_rows; row++) { + for (uint32_t i = 0; i < in_dim; i++) { + const uint32_t mix = + i * 29u + row * 101u + ((i + row * 17u) ^ (i >> 3u)) * 7u; + x_host[(uint64_t)row * in_dim + i] = + (float)((int)(mix % 191u) - 95) / 113.0f; + } + } + + TEST_ASSERT(ds4_gpu_tensor_write(x, 0, x_host, x_bytes) != 0); + TEST_ASSERT(ds4_gpu_set_model_map(weights_raw, weight_alloc) != 0); + ds4_gpu_set_quality(false); + + for (uint32_t row = 0; row < max_rows; row++) { + ds4_gpu_tensor *x_row = ds4_gpu_tensor_view( + x, (uint64_t)row * x_row_bytes, x_row_bytes); + ds4_gpu_tensor *out_row = ds4_gpu_tensor_view( + reference, (uint64_t)row * out_row_bytes, out_row_bytes); + TEST_ASSERT(x_row != NULL); + TEST_ASSERT(out_row != NULL); + if (x_row && out_row) { + TEST_ASSERT(ds4_gpu_matmul_q8_0_tensor( + out_row, weights_raw, weight_alloc, 0, + in_dim, out_dim, x_row, 1) != 0); + } + ds4_gpu_tensor_free(x_row); + ds4_gpu_tensor_free(out_row); + } + TEST_ASSERT(ds4_gpu_tensor_read( + reference, 0, reference_host, out_bytes) != 0); + + for (uint32_t n_rows = 2u; n_rows <= max_rows; n_rows++) { + memset(batched_host, 0xa5, (size_t)out_bytes); + TEST_ASSERT(ds4_gpu_tensor_write( + batched, 0, batched_host, out_bytes) != 0); + TEST_ASSERT(ds4_gpu_matmul_q8_0_decode_rows_exact_tensor( + batched, weights_raw, weight_alloc, 0, + in_dim, out_dim, x, n_rows) != 0); + TEST_ASSERT(ds4_gpu_tensor_read( + batched, 0, batched_host, out_bytes) != 0); + + const size_t compared = (size_t)n_rows * out_dim; + const test_float_compare_stats stats = test_compare_float_bits( + reference_host, batched_host, compared); + if (stats.mismatch_count != 0) { + fprintf(stderr, + "ds4-test: Metal Q8_0 exact-%u rows mismatches=%zu/%zu " + "max_ulp=%u max_abs=%g\n", + n_rows, + stats.mismatch_count, + compared, + stats.max_ulp, + stats.max_abs); + } + TEST_ASSERT(memcmp(reference_host, + batched_host, + compared * sizeof(float)) == 0); + } + + free(x_host); + free(reference_host); + free(batched_host); + ds4_gpu_tensor_free(x); + ds4_gpu_tensor_free(reference); + ds4_gpu_tensor_free(batched); + free(weights_raw); +} + +static void test_metal_q4_attention_output_tiny_batch_exact_case( + uint32_t out_b_type) { + /* + * The opt-in AProjQ4 tiny batch must be the exact composition of the two + * canonical one-row projections for both deployed output quantizations. + * Keep a sixth poisoned row in every destination so each N=2..5 case also + * checks dispatch bounds. + */ + TEST_ASSERT(out_b_type == 8u || out_b_type == 12u); + if (out_b_type != 8u && out_b_type != 12u) return; + const uint32_t group_dim = 256u; + const uint32_t rank = 128u; + const uint32_t n_groups = 2u; + const uint32_t low_dim = n_groups * rank; + const uint32_t out_dim = 67u; + const uint32_t max_rows = 5u; + const uint32_t alloc_rows = max_rows + 1u; + const uint64_t page = (uint64_t)getpagesize(); + const uint64_t row_a_bytes = (uint64_t)(group_dim / 256u) * 144u; + const uint64_t out_a_bytes = + (uint64_t)n_groups * rank * row_a_bytes; + const uint64_t out_b_offset = test_round_up_u64(out_a_bytes, page); + const uint64_t row_b_bytes = out_b_type == 8u + ? (uint64_t)(low_dim / 32u) * 34u + : (uint64_t)(low_dim / 256u) * 144u; + const uint64_t out_b_bytes = (uint64_t)out_dim * row_b_bytes; + const uint64_t model_bytes = + test_round_up_u64(out_b_offset + out_b_bytes, page); + const uint64_t heads_row_bytes = + (uint64_t)n_groups * group_dim * sizeof(float); + const uint64_t low_row_bytes = (uint64_t)low_dim * sizeof(float); + const uint64_t out_row_bytes = (uint64_t)out_dim * sizeof(float); + const uint64_t heads_bytes = (uint64_t)alloc_rows * heads_row_bytes; + const uint64_t low_bytes = (uint64_t)alloc_rows * low_row_bytes; + const uint64_t out_bytes = (uint64_t)alloc_rows * out_row_bytes; + const char *enable_env = + "DS4_METAL_ENABLE_Q4_ATTN_OUT_TINY_BATCH"; + const char *disable_env = + "DS4_METAL_DISABLE_Q4_ATTN_OUT_TINY_BATCH"; + const char *require_env = + "DS4_METAL_REQUIRE_Q4_ATTN_OUT_TINY_BATCH"; + const char *disable_classic_env = + "DS4_METAL_DISABLE_Q4_MV_CLASSIC"; + + char *saved_enable = test_save_env(enable_env); + char *saved_disable = test_save_env(disable_env); + char *saved_require = test_save_env(require_env); + char *saved_disable_classic = test_save_env(disable_classic_env); + void *model_raw = NULL; + float *heads_host = NULL; + float *reference_low_host = NULL; + float *reference_out_host = NULL; + float *candidate_low_host = NULL; + float *candidate_out_host = NULL; + ds4_gpu_tensor *heads = NULL; + ds4_gpu_tensor *reference_low = NULL; + ds4_gpu_tensor *reference_out = NULL; + ds4_gpu_tensor *candidate_low = NULL; + ds4_gpu_tensor *candidate_out = NULL; + ds4_gpu_tensor *group_tmp = NULL; + ds4_gpu_tensor *low_tmp = NULL; + + TEST_ASSERT(posix_memalign( + &model_raw, (size_t)page, (size_t)model_bytes) == 0); + if (!model_raw) goto cleanup; + memset(model_raw, 0, (size_t)model_bytes); + test_fill_q4_K_weights((uint8_t *)model_raw, + group_dim, + n_groups * rank, + 211u); + if (out_b_type == 8u) { + test_fill_q8_0_weights( + (uint8_t *)model_raw + out_b_offset, low_dim, out_dim, 307u); + } else { + test_fill_q4_K_weights( + (uint8_t *)model_raw + out_b_offset, low_dim, out_dim, 307u); + } + + heads_host = malloc((size_t)heads_bytes); + reference_low_host = malloc((size_t)low_bytes); + reference_out_host = malloc((size_t)out_bytes); + candidate_low_host = malloc((size_t)low_bytes); + candidate_out_host = malloc((size_t)out_bytes); + heads = ds4_gpu_tensor_alloc(heads_bytes); + reference_low = ds4_gpu_tensor_alloc(low_bytes); + reference_out = ds4_gpu_tensor_alloc(out_bytes); + candidate_low = ds4_gpu_tensor_alloc(low_bytes); + candidate_out = ds4_gpu_tensor_alloc(out_bytes); + group_tmp = ds4_gpu_tensor_alloc( + (uint64_t)alloc_rows * group_dim * sizeof(float)); + low_tmp = ds4_gpu_tensor_alloc( + (uint64_t)alloc_rows * rank * sizeof(float)); + TEST_ASSERT(heads_host && reference_low_host && reference_out_host && + candidate_low_host && candidate_out_host && heads && + reference_low && reference_out && candidate_low && + candidate_out && group_tmp && low_tmp); + if (!heads_host || !reference_low_host || !reference_out_host || + !candidate_low_host || !candidate_out_host || !heads || + !reference_low || !reference_out || !candidate_low || + !candidate_out || !group_tmp || !low_tmp) { + goto cleanup; + } + + for (uint32_t row = 0; row < alloc_rows; row++) { + for (uint32_t i = 0; i < n_groups * group_dim; i++) { + const uint32_t key = + i * 41u + row * 271u + ((i >> 2u) ^ (row * 19u)); + heads_host[(uint64_t)row * n_groups * group_dim + i] = + (float)((int)(key % 257u) - 128) / 137.0f; + } + } + memset(reference_low_host, 0, (size_t)low_bytes); + memset(reference_out_host, 0, (size_t)out_bytes); + TEST_ASSERT(ds4_gpu_tensor_write( + heads, 0, heads_host, heads_bytes) != 0); + TEST_ASSERT(ds4_gpu_tensor_write( + reference_low, 0, reference_low_host, low_bytes) != 0); + TEST_ASSERT(ds4_gpu_tensor_write( + reference_out, 0, reference_out_host, out_bytes) != 0); + TEST_ASSERT(ds4_gpu_set_model_map(model_raw, model_bytes) != 0); + ds4_gpu_set_quality(false); + TEST_ASSERT(unsetenv(disable_classic_env) == 0); + + for (uint32_t row = 0; row < max_rows; row++) { + ds4_gpu_tensor *heads_row = ds4_gpu_tensor_view( + heads, (uint64_t)row * heads_row_bytes, heads_row_bytes); + ds4_gpu_tensor *low_row = ds4_gpu_tensor_view( + reference_low, (uint64_t)row * low_row_bytes, low_row_bytes); + ds4_gpu_tensor *out_row = ds4_gpu_tensor_view( + reference_out, (uint64_t)row * out_row_bytes, out_row_bytes); + TEST_ASSERT(heads_row && low_row && out_row); + if (heads_row && low_row && out_row) { + TEST_ASSERT(ds4_gpu_attention_output_low_q4_K_slice_tensor( + low_row, + model_raw, + model_bytes, + 0, + group_dim, + rank, + 0, + n_groups, + heads_row) != 0); + if (out_b_type == 8u) { + TEST_ASSERT(ds4_gpu_matmul_q8_0_tensor( + out_row, + model_raw, + model_bytes, + out_b_offset, + low_dim, + out_dim, + low_row, + 1) != 0); + } else { + TEST_ASSERT(ds4_gpu_matmul_quant_tensor( + out_row, + model_raw, + model_bytes, + out_b_offset, + out_b_type, + low_dim, + out_dim, + low_row, + 1) != 0); + } + } + ds4_gpu_tensor_free(heads_row); + ds4_gpu_tensor_free(low_row); + ds4_gpu_tensor_free(out_row); + } + TEST_ASSERT(ds4_gpu_tensor_read( + reference_low, 0, reference_low_host, low_bytes) != 0); + TEST_ASSERT(ds4_gpu_tensor_read( + reference_out, 0, reference_out_host, out_bytes) != 0); + + /* The API itself is fail-closed; the graph caller owns the row fallback. */ + TEST_ASSERT(unsetenv(enable_env) == 0); + TEST_ASSERT(unsetenv(disable_env) == 0); + TEST_ASSERT(unsetenv(require_env) == 0); + TEST_ASSERT(ds4_gpu_attention_output_q4_K_batch_tensor( + candidate_out, + candidate_low, + group_tmp, + low_tmp, + model_raw, + model_bytes, + 0, + out_b_offset, + out_b_type, + group_dim, + rank, + n_groups, + out_dim, + heads, + 2u) == 0); + + TEST_ASSERT(setenv(enable_env, "1", 1) == 0); + for (uint32_t n_rows = 2u; n_rows <= max_rows; n_rows++) { + for (uint64_t i = 0; i < (uint64_t)alloc_rows * low_dim; i++) { + const uint32_t bits = 0x7fc10000u + (uint32_t)(i & 0xffffu); + memcpy(&candidate_low_host[i], &bits, sizeof(bits)); + } + for (uint64_t i = 0; i < (uint64_t)alloc_rows * out_dim; i++) { + const uint32_t bits = 0x7fc20000u + (uint32_t)(i & 0xffffu); + memcpy(&candidate_out_host[i], &bits, sizeof(bits)); + } + TEST_ASSERT(ds4_gpu_tensor_write( + candidate_low, 0, candidate_low_host, low_bytes) != 0); + TEST_ASSERT(ds4_gpu_tensor_write( + candidate_out, 0, candidate_out_host, out_bytes) != 0); + TEST_ASSERT(ds4_gpu_attention_output_q4_K_batch_tensor( + candidate_out, + candidate_low, + group_tmp, + low_tmp, + model_raw, + model_bytes, + 0, + out_b_offset, + out_b_type, + group_dim, + rank, + n_groups, + out_dim, + heads, + n_rows) == 1); + TEST_ASSERT(ds4_gpu_tensor_read( + candidate_low, 0, candidate_low_host, low_bytes) != 0); + TEST_ASSERT(ds4_gpu_tensor_read( + candidate_out, 0, candidate_out_host, out_bytes) != 0); + + const size_t low_count = (size_t)n_rows * low_dim; + const size_t out_count = (size_t)n_rows * out_dim; + const test_float_compare_stats low_stats = test_compare_float_bits( + reference_low_host, candidate_low_host, low_count); + const test_float_compare_stats out_stats = test_compare_float_bits( + reference_out_host, candidate_out_host, out_count); + size_t low_oob_mismatch = 0; + size_t out_oob_mismatch = 0; + for (uint64_t i = (uint64_t)n_rows * low_dim; + i < (uint64_t)alloc_rows * low_dim; + i++) { + uint32_t bits = 0; + memcpy(&bits, &candidate_low_host[i], sizeof(bits)); + if (bits != 0x7fc10000u + (uint32_t)(i & 0xffffu)) { + low_oob_mismatch++; + } + } + for (uint64_t i = (uint64_t)n_rows * out_dim; + i < (uint64_t)alloc_rows * out_dim; + i++) { + uint32_t bits = 0; + memcpy(&bits, &candidate_out_host[i], sizeof(bits)); + if (bits != 0x7fc20000u + (uint32_t)(i & 0xffffu)) { + out_oob_mismatch++; + } + } + fprintf(stderr, + "ds4-test: Metal Q4 attention-output out_b=%s exact-%u " + "low=%zu/%zu out=%zu/%zu low_oob=%zu out_oob=%zu\n", + out_b_type == 8u ? "Q8_0" : "Q4_K", + n_rows, + low_stats.mismatch_count, + low_count, + out_stats.mismatch_count, + out_count, + low_oob_mismatch, + out_oob_mismatch); + TEST_ASSERT(low_stats.mismatch_count == 0 && low_stats.max_ulp == 0); + TEST_ASSERT(out_stats.mismatch_count == 0 && out_stats.max_ulp == 0); + TEST_ASSERT(low_oob_mismatch == 0); + TEST_ASSERT(out_oob_mismatch == 0); + } + + /* REQUIRE is a fail-closed model-oracle gate and implies enable for the + * exact N=2..5 scope. */ + TEST_ASSERT(unsetenv(enable_env) == 0); + TEST_ASSERT(setenv(require_env, "1", 1) == 0); + TEST_ASSERT(ds4_gpu_attention_output_q4_K_batch_tensor( + candidate_out, + candidate_low, + group_tmp, + low_tmp, + model_raw, + model_bytes, + 0, + out_b_offset, + out_b_type, + group_dim, + rank, + n_groups, + out_dim, + heads, + 2u) == 1); + + if (out_b_type == 12u) { + /* A required candidate must hard-fail rather than bypass an explicit + * request for the alternate Q4 schedule. */ + TEST_ASSERT(unsetenv(disable_env) == 0); + TEST_ASSERT(setenv(disable_classic_env, "1", 1) == 0); + TEST_ASSERT(ds4_gpu_attention_output_q4_K_batch_tensor( + candidate_out, + candidate_low, + group_tmp, + low_tmp, + model_raw, + model_bytes, + 0, + out_b_offset, + out_b_type, + group_dim, + rank, + n_groups, + out_dim, + heads, + 2u) == -1); + TEST_ASSERT(unsetenv(disable_classic_env) == 0); + } + + /* The unconditional kill switch wins over REQUIRE and makes the + * model-oracle run fail instead of silently selecting the fallback. */ + TEST_ASSERT(setenv(disable_env, "1", 1) == 0); + TEST_ASSERT(ds4_gpu_attention_output_q4_K_batch_tensor( + candidate_out, + candidate_low, + group_tmp, + low_tmp, + model_raw, + model_bytes, + 0, + out_b_offset, + out_b_type, + group_dim, + rank, + n_groups, + out_dim, + heads, + 2u) == -1); + + /* Without REQUIRE the same kill switch retains the ordinary fallback. */ + TEST_ASSERT(unsetenv(require_env) == 0); + TEST_ASSERT(setenv(enable_env, "1", 1) == 0); + TEST_ASSERT(ds4_gpu_attention_output_q4_K_batch_tensor( + candidate_out, + candidate_low, + group_tmp, + low_tmp, + model_raw, + model_bytes, + 0, + out_b_offset, + out_b_type, + group_dim, + rank, + n_groups, + out_dim, + heads, + 2u) == 0); + +cleanup: + ds4_gpu_tensor_free(low_tmp); + ds4_gpu_tensor_free(group_tmp); + ds4_gpu_tensor_free(candidate_out); + ds4_gpu_tensor_free(candidate_low); + ds4_gpu_tensor_free(reference_out); + ds4_gpu_tensor_free(reference_low); + ds4_gpu_tensor_free(heads); + free(candidate_out_host); + free(candidate_low_host); + free(reference_out_host); + free(reference_low_host); + free(heads_host); + free(model_raw); + test_restore_env(disable_classic_env, saved_disable_classic); + test_restore_env(require_env, saved_require); + test_restore_env(disable_env, saved_disable); + test_restore_env(enable_env, saved_enable); +} + +static void test_metal_q4_attention_output_tiny_batch_exact(void) { + test_metal_q4_attention_output_tiny_batch_exact_case(8u); + test_metal_q4_attention_output_tiny_batch_exact_case(12u); +} + +static void test_metal_dspark_device_proposer_q8(void) { + /* Keep this fixture small while exercising the complete public contract: + * six on-device draft steps, a confidence stop after a verified prefix, + * and stable lowest-token tie breaking in the Markov argmax. */ + _Static_assert(sizeof(ds4_gpu_dspark_device_proposal) == 64u, + "DSpark device proposal ABI must stay 64 bytes"); + _Static_assert(DS4_GPU_DSPARK_MAX_DRAFTS == 6u, + "DSpark device proposer test covers the six-token limit"); + _Static_assert(offsetof(ds4_gpu_dspark_device_proposal, tokens) == 0u && + offsetof(ds4_gpu_dspark_device_proposal, + confidence_logits) == 24u && + offsetof(ds4_gpu_dspark_device_proposal, + proposal_len) == 48u && + offsetof(ds4_gpu_dspark_device_proposal, + confidence_len) == 52u && + offsetof(ds4_gpu_dspark_device_proposal, status) == 56u && + offsetof(ds4_gpu_dspark_device_proposal, reserved) == 60u, + "DSpark device proposal field offsets changed"); + + const uint32_t vocab = 64u; + const uint32_t rank = 32u; + const uint32_t hidden_dim = 32u; + const uint32_t max_drafts = DS4_GPU_DSPARK_MAX_DRAFTS; + const uint32_t first_prev_token = 17u; + const uint32_t tied_token_lo = 3u; + const uint32_t tied_token_hi = 11u; + const float reused_confidence = 8.0f; + const uint64_t page = (uint64_t)getpagesize(); + const uint64_t markov_row_bytes = (uint64_t)(rank / 32u) * 34u; + const uint64_t markov_bytes = (uint64_t)vocab * markov_row_bytes; + const uint64_t confidence_bytes = + (uint64_t)((hidden_dim + rank) / 32u) * 34u; + const uint64_t w1_offset = 0u; + const uint64_t w2_offset = test_round_up_u64(markov_bytes, page); + const uint64_t confidence_offset = + test_round_up_u64(w2_offset + markov_bytes, page); + const uint64_t model_bytes = test_round_up_u64( + confidence_offset + confidence_bytes, page); + const uint64_t logits_bytes = + (uint64_t)max_drafts * vocab * sizeof(float); + const uint64_t hidden_bytes = + (uint64_t)max_drafts * hidden_dim * sizeof(float); + + void *model_raw = NULL; + TEST_ASSERT(posix_memalign(&model_raw, + (size_t)page, + (size_t)model_bytes) == 0); + if (!model_raw) return; + /* Zero is a valid Q8_0 block (zero half scale and zero quants). It makes + * every Markov correction and computed confidence exactly zero while the + * kernels still traverse all Q8 blocks. */ + memset(model_raw, 0, (size_t)model_bytes); + + ds4_gpu_tensor *logits = ds4_gpu_tensor_alloc(logits_bytes); + ds4_gpu_tensor *hidden = ds4_gpu_tensor_alloc(hidden_bytes); + ds4_gpu_tensor *result = + ds4_gpu_tensor_alloc(DS4_GPU_DSPARK_DEVICE_PROPOSAL_BYTES); + TEST_ASSERT(logits != NULL); + TEST_ASSERT(hidden != NULL); + TEST_ASSERT(result != NULL); + if (!logits || !hidden || !result) { + ds4_gpu_tensor_free(logits); + ds4_gpu_tensor_free(hidden); + ds4_gpu_tensor_free(result); + free(model_raw); + return; + } + + float *logits_host = malloc((size_t)logits_bytes); + float *hidden_host = malloc((size_t)hidden_bytes); + TEST_ASSERT(logits_host != NULL); + TEST_ASSERT(hidden_host != NULL); + if (!logits_host || !hidden_host) { + free(logits_host); + free(hidden_host); + ds4_gpu_tensor_free(logits); + ds4_gpu_tensor_free(hidden); + ds4_gpu_tensor_free(result); + free(model_raw); + return; + } + + for (uint32_t draft = 0u; draft < max_drafts; draft++) { + for (uint32_t token = 0u; token < vocab; token++) { + logits_host[(uint64_t)draft * vocab + token] = + -4.0f - (float)(token % 7u) * 0.03125f; + } + logits_host[(uint64_t)draft * vocab + tied_token_lo] = 5.0f; + logits_host[(uint64_t)draft * vocab + tied_token_hi] = 5.0f; + for (uint32_t i = 0u; i < hidden_dim; i++) { + hidden_host[(uint64_t)draft * hidden_dim + i] = + (float)((int)((draft * 19u + i * 13u) % 37u) - 18) / 23.0f; + } + } + + TEST_ASSERT(ds4_gpu_tensor_write( + logits, 0, logits_host, logits_bytes) != 0); + TEST_ASSERT(ds4_gpu_tensor_write( + hidden, 0, hidden_host, hidden_bytes) != 0); + TEST_ASSERT(ds4_gpu_set_model_map(model_raw, model_bytes) != 0); + ds4_gpu_set_quality(false); + + char *saved_enable = + test_save_env("DS4_METAL_DSPARK_DEVICE_PROPOSER"); + char *saved_kill = + test_save_env("DS4_METAL_DSPARK_NO_DEVICE_PROPOSER"); + setenv("DS4_METAL_DSPARK_DEVICE_PROPOSER", "1", 1); + unsetenv("DS4_METAL_DSPARK_NO_DEVICE_PROPOSER"); + + ds4_gpu_dspark_device_proposal public_result; + bool full_six_ok = false; + for (uint32_t n_drafts = 1u; n_drafts <= max_drafts; n_drafts++) { + memset(&public_result, 0xa5, sizeof(public_result)); + TEST_ASSERT(ds4_gpu_tensor_write(result, + 0, + &public_result, + sizeof(public_result)) != 0); + const int ok = ds4_gpu_dspark_markov_confidence_q8_tensor( + result, + logits, + hidden, + model_raw, + model_bytes, + w1_offset, + w2_offset, + confidence_offset, + first_prev_token, + vocab, + rank, + hidden_dim, + n_drafts, + 0.25f, + 1, + reused_confidence); + TEST_ASSERT(ok != 0); + if (!ok) break; + TEST_ASSERT(ds4_gpu_tensor_read(result, + 0, + &public_result, + sizeof(public_result)) != 0); + TEST_ASSERT(public_result.status == 1u); + TEST_ASSERT(public_result.reserved == 0u); + TEST_ASSERT(public_result.proposal_len == n_drafts); + TEST_ASSERT(public_result.confidence_len == n_drafts); + for (uint32_t i = 0u; i < max_drafts; i++) { + if (i < n_drafts) { + TEST_ASSERT(public_result.tokens[i] == + (int32_t)tied_token_lo); + TEST_ASSERT(public_result.confidence_logits[i] == + (i == 0u ? reused_confidence : 0.0f)); + } else { + TEST_ASSERT(public_result.tokens[i] == -1); + TEST_ASSERT(public_result.confidence_logits[i] == 0.0f); + } + } + if (n_drafts == max_drafts) full_six_ok = true; + } + TEST_ASSERT(full_six_ok); + + /* Draft zero passes via the supplied confidence. Draft one computes a + * zero logit (p=0.5), fails the 0.75 threshold, and therefore leaves a + * one-token proposal with two evaluated confidence rows. */ + memset(&public_result, 0xa5, sizeof(public_result)); + TEST_ASSERT(ds4_gpu_tensor_write(result, + 0, + &public_result, + sizeof(public_result)) != 0); + const int stop_ok = ds4_gpu_dspark_markov_confidence_q8_tensor( + result, + logits, + hidden, + model_raw, + model_bytes, + w1_offset, + w2_offset, + confidence_offset, + first_prev_token, + vocab, + rank, + hidden_dim, + max_drafts, + 0.75f, + 1, + reused_confidence); + TEST_ASSERT(stop_ok != 0); + if (stop_ok) { + TEST_ASSERT(ds4_gpu_tensor_read(result, + 0, + &public_result, + sizeof(public_result)) != 0); + TEST_ASSERT(public_result.status == 1u); + TEST_ASSERT(public_result.reserved == 0u); + TEST_ASSERT(public_result.proposal_len == 1u); + TEST_ASSERT(public_result.confidence_len == 2u); + TEST_ASSERT(public_result.tokens[0] == (int32_t)tied_token_lo); + TEST_ASSERT(public_result.confidence_logits[0] == reused_confidence); + TEST_ASSERT(public_result.confidence_logits[1] == 0.0f); + for (uint32_t i = 1u; i < max_drafts; i++) { + TEST_ASSERT(public_result.tokens[i] == -1); + if (i > 1u) { + TEST_ASSERT(public_result.confidence_logits[i] == 0.0f); + } + } + } + + setenv("DS4_METAL_DSPARK_NO_DEVICE_PROPOSER", "1", 1); + TEST_ASSERT(ds4_gpu_dspark_markov_confidence_q8_tensor( + result, + logits, + hidden, + model_raw, + model_bytes, + w1_offset, + w2_offset, + confidence_offset, + first_prev_token, + vocab, + rank, + hidden_dim, + 1u, + 0.25f, + 1, + reused_confidence) == 0); + test_restore_env("DS4_METAL_DSPARK_NO_DEVICE_PROPOSER", saved_kill); + test_restore_env("DS4_METAL_DSPARK_DEVICE_PROPOSER", saved_enable); + + fprintf(stderr, + "ds4-test: Metal DSpark device proposer full6=%d stop_prefix=%u " + "confidence_rows=%u tie=%u<%u\n", + full_six_ok ? 1 : 0, + stop_ok ? public_result.proposal_len : 0u, + stop_ok ? public_result.confidence_len : 0u, + tied_token_lo, + tied_token_hi); + + free(logits_host); + free(hidden_host); + ds4_gpu_tensor_free(logits); + ds4_gpu_tensor_free(hidden); + ds4_gpu_tensor_free(result); + free(model_raw); +} + static void test_metal_f16_compressor_pair_state_store_exact_case( uint32_t width, uint32_t ratio, @@ -1678,6 +2462,383 @@ static void test_metal_f16_compressor_quad_state_store_exact(void) { 512u, 11u, 0u, 1u, 71u); } +static void test_metal_q8_qkv_compressor_compound_exact_case( + uint32_t ratio, + uint32_t pos, + uint32_t ape0_type, + uint32_t ape1_type, + uint32_t seed, + bool batched_commands) { + enum { + Q_A_WEIGHT = 0, + KV_WEIGHT, + COMP0_KV_WEIGHT, + COMP0_SCORE_WEIGHT, + COMP1_KV_WEIGHT, + COMP1_SCORE_WEIGHT, + COMP0_APE, + COMP1_APE, + MODEL_RANGE_COUNT, + }; + const uint32_t in_dim = 4096u; + const uint32_t q_rank = 77u; + const uint32_t kv_dim = 19u; + const uint32_t width0 = ratio == 4u ? 1024u : 512u; + const uint32_t width1 = ratio == 4u ? 256u : 0u; + const uint32_t state_rows = ratio == 4u ? 2u * ratio : ratio; + const uint32_t n_comp_outputs = width1 != 0u ? 4u : 2u; + const uint32_t comp_widths[4] = { + width0, width0, width1, width1, + }; + const uint64_t page = (uint64_t)getpagesize(); + const uint64_t q8_row_bytes = (uint64_t)(in_dim / 32u) * 34u; + const uint64_t f16_row_bytes = + (uint64_t)in_dim * sizeof(uint16_t); + uint64_t range_bytes[MODEL_RANGE_COUNT] = { + (uint64_t)q_rank * q8_row_bytes, + (uint64_t)kv_dim * q8_row_bytes, + (uint64_t)width0 * f16_row_bytes, + (uint64_t)width0 * f16_row_bytes, + (uint64_t)width1 * f16_row_bytes, + (uint64_t)width1 * f16_row_bytes, + (uint64_t)ratio * width0 * + (ape0_type == 1u ? sizeof(uint16_t) : sizeof(float)), + (uint64_t)ratio * width1 * + (ape1_type == 1u ? sizeof(uint16_t) : sizeof(float)), + }; + uint64_t range_offsets[MODEL_RANGE_COUNT] = {0}; + uint64_t cursor = 0; + for (uint32_t i = 0; i < MODEL_RANGE_COUNT; i++) { + if (range_bytes[i] == 0u) continue; + range_offsets[i] = test_round_up_u64(cursor, page); + cursor = range_offsets[i] + range_bytes[i]; + } + const uint64_t model_bytes = test_round_up_u64(cursor, page); + const uint64_t x_bytes = (uint64_t)in_dim * sizeof(float); + + void *model_raw = NULL; + TEST_ASSERT(posix_memalign( + &model_raw, (size_t)page, (size_t)model_bytes) == 0); + ds4_gpu_tensor *x = ds4_gpu_tensor_alloc(x_bytes); + ds4_gpu_tensor *ref_q = + ds4_gpu_tensor_alloc((uint64_t)q_rank * sizeof(float)); + ds4_gpu_tensor *ref_kv = + ds4_gpu_tensor_alloc((uint64_t)kv_dim * sizeof(float)); + ds4_gpu_tensor *fused_q = + ds4_gpu_tensor_alloc((uint64_t)q_rank * sizeof(float)); + ds4_gpu_tensor *fused_kv = + ds4_gpu_tensor_alloc((uint64_t)kv_dim * sizeof(float)); + ds4_gpu_tensor *ref_out[4] = {0}; + ds4_gpu_tensor *fused_out[4] = {0}; + ds4_gpu_tensor *ref_state[4] = {0}; + ds4_gpu_tensor *fused_state[4] = {0}; + bool allocated = model_raw && x && ref_q && ref_kv && fused_q && fused_kv; + for (uint32_t i = 0; i < n_comp_outputs; i++) { + const uint64_t out_bytes = + (uint64_t)comp_widths[i] * sizeof(float); + const uint64_t state_bytes = + (uint64_t)state_rows * comp_widths[i] * sizeof(float); + ref_out[i] = ds4_gpu_tensor_alloc(out_bytes); + fused_out[i] = ds4_gpu_tensor_alloc(out_bytes); + ref_state[i] = ds4_gpu_tensor_alloc(state_bytes); + fused_state[i] = ds4_gpu_tensor_alloc(state_bytes); + TEST_ASSERT(ref_out[i] && fused_out[i] && + ref_state[i] && fused_state[i]); + allocated = allocated && ref_out[i] && fused_out[i] && + ref_state[i] && fused_state[i]; + } + + uint64_t host_count = (uint64_t)state_rows * width0; + if (host_count < q_rank) host_count = q_rank; + if (host_count < kv_dim) host_count = kv_dim; + float *x_host = malloc((size_t)x_bytes); + float *ref_host = malloc((size_t)host_count * sizeof(float)); + float *fused_host = malloc((size_t)host_count * sizeof(float)); + TEST_ASSERT(x_host && ref_host && fused_host); + allocated = allocated && x_host && ref_host && fused_host; + + const char *force_pair_env = + "DS4_METAL_ENABLE_COMPRESSOR_PAIR_STATE_STORE"; + const char *disable_pair_state_env = + "DS4_METAL_DISABLE_M3_COMPRESSOR_PAIR_STATE_STORE"; + const char *disable_pair_proj_env = + "DS4_METAL_DISABLE_COMPRESSOR_PAIR_PROJ"; + const char *disable_store_env = + "DS4_METAL_DISABLE_COMPRESSOR_STORE_ONE"; + const char *q8_nsg_env = "DS4_METAL_Q8_MV_NSG"; + char *saved_force_pair = test_save_env(force_pair_env); + char *saved_disable_pair_state = test_save_env(disable_pair_state_env); + char *saved_disable_pair_proj = test_save_env(disable_pair_proj_env); + char *saved_disable_store = test_save_env(disable_store_env); + char *saved_q8_nsg = test_save_env(q8_nsg_env); + + size_t q_mismatches = 0; + size_t kv_mismatches = 0; + size_t out_mismatches = 0; + size_t state_mismatches = 0; + uint32_t max_ulp = 0; + if (allocated) { + memset(model_raw, 0, (size_t)model_bytes); + test_fill_q8_0_weights( + (uint8_t *)model_raw + range_offsets[Q_A_WEIGHT], + in_dim, q_rank, seed + 1u); + test_fill_q8_0_weights( + (uint8_t *)model_raw + range_offsets[KV_WEIGHT], + in_dim, kv_dim, seed + 3u); + + for (uint32_t matrix = 0; matrix < n_comp_outputs; matrix++) { + uint16_t *weights = (uint16_t *)( + (uint8_t *)model_raw + + range_offsets[COMP0_KV_WEIGHT + matrix]); + const uint32_t width = comp_widths[matrix]; + for (uint32_t o = 0; o < width; o++) { + for (uint32_t i = 0; i < in_dim; i++) { + const int value = + (int)((o * (13u + 2u * matrix) + + i * (19u + 4u * matrix) + + (o ^ (i >> (matrix & 3u))) * + (5u + 2u * matrix) + + seed * (23u + matrix)) % 127u) - 63; + weights[(uint64_t)o * in_dim + i] = + test_float_to_f16( + (float)value / (88.0f + 8.0f * matrix)); + } + } + } + + const uint32_t ape_types[2] = {ape0_type, ape1_type}; + const uint32_t ape_widths[2] = {width0, width1}; + const uint32_t ape_ranges[2] = {COMP0_APE, COMP1_APE}; + for (uint32_t which = 0; which < 2u; which++) { + if (ape_widths[which] == 0u) continue; + const uint64_t count = + (uint64_t)ratio * ape_widths[which]; + if (ape_types[which] == 1u) { + uint16_t *ape = (uint16_t *)( + (uint8_t *)model_raw + range_offsets[ape_ranges[which]]); + for (uint64_t i = 0; i < count; i++) { + const int value = + (int)((i * (11u + 2u * which) + + (i ^ (i >> 3u)) * (7u + 2u * which) + + seed * (17u + which)) % 61u) - 30; + ape[i] = test_float_to_f16( + (float)value / (72.0f + 8.0f * which)); + } + } else { + float *ape = (float *)( + (uint8_t *)model_raw + range_offsets[ape_ranges[which]]); + for (uint64_t i = 0; i < count; i++) { + const int value = + (int)((i * (11u + 2u * which) + + (i ^ (i >> 3u)) * (7u + 2u * which) + + seed * (17u + which)) % 61u) - 30; + ape[i] = + (float)value / (72.0f + 8.0f * which); + } + } + } + + for (uint32_t i = 0; i < in_dim; i++) { + const int value = + (int)((i * 29u + (i ^ (i >> 4u)) * 9u + + seed * 11u) % 127u) - 63; + x_host[i] = (float)value / 84.0f; + } + TEST_ASSERT(ds4_gpu_tensor_write(x, 0, x_host, x_bytes) != 0); + + for (uint32_t buffer = 0; buffer < n_comp_outputs; buffer++) { + const uint32_t width = comp_widths[buffer]; + const uint64_t out_bytes = + (uint64_t)width * sizeof(float); + const uint64_t state_count = + (uint64_t)state_rows * width; + const uint64_t state_bytes = state_count * sizeof(float); + for (uint32_t i = 0; i < width; i++) { + const uint32_t poison = + 0x7fc10001u + ((buffer * 2048u + i) & 0x7fffu); + memcpy(ref_host + i, &poison, sizeof(poison)); + } + TEST_ASSERT(ds4_gpu_tensor_write( + ref_out[buffer], 0, ref_host, + out_bytes) != 0); + TEST_ASSERT(ds4_gpu_tensor_write( + fused_out[buffer], 0, ref_host, + out_bytes) != 0); + for (uint64_t i = 0; i < state_count; i++) { + const int value = + (int)((i * (5u + 2u * buffer) + + seed * (13u + buffer)) % 193u) - 96; + ref_host[i] = + (float)value / (64.0f + 8.0f * buffer); + } + TEST_ASSERT(ds4_gpu_tensor_write( + ref_state[buffer], 0, ref_host, + state_bytes) != 0); + TEST_ASSERT(ds4_gpu_tensor_write( + fused_state[buffer], 0, ref_host, + state_bytes) != 0); + } + + TEST_ASSERT(ds4_gpu_set_model_map(model_raw, model_bytes) != 0); + ds4_gpu_set_quality(false); + TEST_ASSERT(setenv(force_pair_env, "1", 1) == 0); + TEST_ASSERT(unsetenv(disable_pair_state_env) == 0); + TEST_ASSERT(unsetenv(disable_pair_proj_env) == 0); + TEST_ASSERT(unsetenv(disable_store_env) == 0); + TEST_ASSERT(unsetenv(q8_nsg_env) == 0); + + TEST_ASSERT(ds4_gpu_matmul_q8_0_pair_tensor( + ref_q, ref_kv, + model_raw, model_bytes, + range_offsets[Q_A_WEIGHT], + range_offsets[KV_WEIGHT], + in_dim, q_rank, kv_dim, x, 1u) == 1); + TEST_ASSERT(ds4_gpu_matmul_f16_pair_compressor_store_tensor( + ref_out[0], ref_out[1], + ref_state[0], ref_state[1], + model_raw, model_bytes, + range_offsets[COMP0_KV_WEIGHT], + range_offsets[COMP0_SCORE_WEIGHT], + range_offsets[COMP0_APE], ape0_type, + in_dim, width0, x, ratio, pos) == 1); + if (width1 != 0u) { + TEST_ASSERT(ds4_gpu_matmul_f16_pair_compressor_store_tensor( + ref_out[2], ref_out[3], + ref_state[2], ref_state[3], + model_raw, model_bytes, + range_offsets[COMP1_KV_WEIGHT], + range_offsets[COMP1_SCORE_WEIGHT], + range_offsets[COMP1_APE], ape1_type, + in_dim, width1, x, ratio, pos) == 1); + } + + bool commands_open = false; + if (batched_commands) { + commands_open = ds4_gpu_begin_commands() != 0; + TEST_ASSERT(commands_open); + } + const int fused = + ds4_gpu_qkv_pair_quad_compressor_store_tensor( + fused_q, fused_kv, + fused_out[0], fused_out[1], + width1 != 0u ? fused_out[2] : NULL, + width1 != 0u ? fused_out[3] : NULL, + fused_state[0], fused_state[1], + width1 != 0u ? fused_state[2] : NULL, + width1 != 0u ? fused_state[3] : NULL, + model_raw, model_bytes, + range_offsets[Q_A_WEIGHT], + range_offsets[KV_WEIGHT], + range_offsets[COMP0_KV_WEIGHT], + range_offsets[COMP0_SCORE_WEIGHT], + range_offsets[COMP1_KV_WEIGHT], + range_offsets[COMP1_SCORE_WEIGHT], + range_offsets[COMP0_APE], ape0_type, + range_offsets[COMP1_APE], ape1_type, + in_dim, q_rank, kv_dim, width0, width1, + x, ratio, pos); + if (commands_open) { + TEST_ASSERT(ds4_gpu_end_commands() != 0); + } + TEST_ASSERT(fused == 1); + + TEST_ASSERT(ds4_gpu_tensor_read( + ref_q, 0, ref_host, + (uint64_t)q_rank * sizeof(float)) != 0); + TEST_ASSERT(ds4_gpu_tensor_read( + fused_q, 0, fused_host, + (uint64_t)q_rank * sizeof(float)) != 0); + test_float_compare_stats stats = + test_compare_float_bits(ref_host, fused_host, q_rank); + q_mismatches = stats.mismatch_count; + if (stats.max_ulp > max_ulp) max_ulp = stats.max_ulp; + + TEST_ASSERT(ds4_gpu_tensor_read( + ref_kv, 0, ref_host, + (uint64_t)kv_dim * sizeof(float)) != 0); + TEST_ASSERT(ds4_gpu_tensor_read( + fused_kv, 0, fused_host, + (uint64_t)kv_dim * sizeof(float)) != 0); + stats = test_compare_float_bits(ref_host, fused_host, kv_dim); + kv_mismatches = stats.mismatch_count; + if (stats.max_ulp > max_ulp) max_ulp = stats.max_ulp; + + for (uint32_t buffer = 0; buffer < n_comp_outputs; buffer++) { + const uint32_t width = comp_widths[buffer]; + const uint64_t out_bytes = + (uint64_t)width * sizeof(float); + const uint64_t state_count = + (uint64_t)state_rows * width; + const uint64_t state_bytes = state_count * sizeof(float); + TEST_ASSERT(ds4_gpu_tensor_read( + ref_out[buffer], 0, ref_host, + out_bytes) != 0); + TEST_ASSERT(ds4_gpu_tensor_read( + fused_out[buffer], 0, fused_host, + out_bytes) != 0); + stats = test_compare_float_bits(ref_host, fused_host, width); + out_mismatches += stats.mismatch_count; + if (stats.max_ulp > max_ulp) max_ulp = stats.max_ulp; + + TEST_ASSERT(ds4_gpu_tensor_read( + ref_state[buffer], 0, ref_host, + state_bytes) != 0); + TEST_ASSERT(ds4_gpu_tensor_read( + fused_state[buffer], 0, fused_host, + state_bytes) != 0); + stats = test_compare_float_bits( + ref_host, fused_host, (size_t)state_count); + state_mismatches += stats.mismatch_count; + if (stats.max_ulp > max_ulp) max_ulp = stats.max_ulp; + } + } + + fprintf(stderr, + "ds4-test: Q8 QKV/compressor compound exact ratio=%u " + "pos_mod=%u batch=%u q=%zu kv=%zu outputs=%zu states=%zu " + "max_ulp=%u\n", + ratio, pos % ratio, batched_commands ? 1u : 0u, + q_mismatches, kv_mismatches, + out_mismatches, state_mismatches, max_ulp); + TEST_ASSERT(q_mismatches == 0); + TEST_ASSERT(kv_mismatches == 0); + TEST_ASSERT(out_mismatches == 0); + TEST_ASSERT(state_mismatches == 0); + TEST_ASSERT(max_ulp == 0); + + test_restore_env(q8_nsg_env, saved_q8_nsg); + test_restore_env(disable_store_env, saved_disable_store); + test_restore_env(disable_pair_proj_env, saved_disable_pair_proj); + test_restore_env(disable_pair_state_env, saved_disable_pair_state); + test_restore_env(force_pair_env, saved_force_pair); + free(fused_host); + free(ref_host); + free(x_host); + for (uint32_t i = 0; i < 4u; i++) { + ds4_gpu_tensor_free(fused_state[i]); + ds4_gpu_tensor_free(ref_state[i]); + ds4_gpu_tensor_free(fused_out[i]); + ds4_gpu_tensor_free(ref_out[i]); + } + ds4_gpu_tensor_free(fused_kv); + ds4_gpu_tensor_free(fused_q); + ds4_gpu_tensor_free(ref_kv); + ds4_gpu_tensor_free(ref_q); + ds4_gpu_tensor_free(x); + free(model_raw); +} + +static void test_metal_q8_qkv_compressor_compound_exact(void) { + test_metal_q8_qkv_compressor_compound_exact_case( + 4u, 11u, 1u, 0u, 79u, false); + test_metal_q8_qkv_compressor_compound_exact_case( + 4u, 11u, 1u, 0u, 79u, true); + test_metal_q8_qkv_compressor_compound_exact_case( + 128u, 255u, 0u, 1u, 83u, false); + test_metal_q8_qkv_compressor_compound_exact_case( + 128u, 255u, 0u, 1u, 83u, true); +} + static void test_metal_compressor_ape_add_exact_case( uint32_t head_dim, uint32_t ratio, @@ -5467,8 +6628,12 @@ static void test_metal_kernel_group(void) { test_dspark_cache_window_crop(); test_metal_q8_0_decode_pair_exact(); #if defined(__APPLE__) + test_metal_q8_0_decode_rows_exact(); + test_metal_q4_attention_output_tiny_batch_exact(); + test_metal_dspark_device_proposer_q8(); test_metal_f16_compressor_pair_state_store_exact(); test_metal_f16_compressor_quad_state_store_exact(); + test_metal_q8_qkv_compressor_compound_exact(); test_metal_compressor_ape_add_exact(); test_metal_compressor_ratio4_pack_exact(); test_metal_compressor_ratio4_replay_pack_exact(); diff --git a/tests/dspark_acceptance_fixture.sh b/tests/dspark_acceptance_fixture.sh index 423e45721..8a19d022c 100644 --- a/tests/dspark_acceptance_fixture.sh +++ b/tests/dspark_acceptance_fixture.sh @@ -29,6 +29,11 @@ SSD_CACHE_EXPERTS=${DS4_DSPARK_FIXTURE_SSD_STREAMING_CACHE_EXPERTS:-} REQUIRE_ACTIVE=${DS4_DSPARK_FIXTURE_REQUIRE_ACTIVE:-1} REQUIRE_EXACT2=${DS4_DSPARK_FIXTURE_REQUIRE_EXACT2:-0} REQUIRE_CUDA_EXACTN=${DS4_DSPARK_FIXTURE_REQUIRE_CUDA_EXACTN:-0} +REQUIRE_CUDA_EXACTN_BATCH_HEAD=${DS4_DSPARK_FIXTURE_REQUIRE_CUDA_EXACTN_BATCH_HEAD:-0} +REQUIRE_CUDA_DEVICE_PROPOSER=${DS4_DSPARK_FIXTURE_REQUIRE_CUDA_DEVICE_PROPOSER:-0} +REQUIRE_METAL_EXACTN_BATCH_HEAD=${DS4_DSPARK_FIXTURE_REQUIRE_METAL_EXACTN_BATCH_HEAD:-0} +REQUIRE_METAL_EXACTN_PARTIAL=${DS4_DSPARK_FIXTURE_REQUIRE_METAL_EXACTN_PARTIAL:-0} +REQUIRE_METAL_DEVICE_PROPOSER=${DS4_DSPARK_FIXTURE_REQUIRE_METAL_DEVICE_PROPOSER:-0} total_proposed=0 total_accepted_draft=0 total_exact2_attempt=0 @@ -36,6 +41,23 @@ total_exact2_fallback=0 total_cuda_exactn_attempt=0 total_cuda_exactn_fallback=0 total_cuda_exactn_error_fallback=0 +total_cuda_exactn_batch_head_attempt=0 +total_cuda_exactn_batch_head_use=0 +total_cuda_exactn_batch_head_fallback=0 +total_cuda_device_proposer_attempt=0 +total_cuda_device_proposer_use=0 +total_cuda_device_proposer_fallback=0 +total_cuda_device_proposer_policy_mismatch=0 +total_exactn_union_error_fallback=0 +total_exactn_union_partial_replay=0 +total_exactn_union_verify_skip=0 +total_metal_exactn_batch_head_attempt=0 +total_metal_exactn_batch_head_use=0 +total_metal_exactn_batch_head_fallback=0 +total_metal_device_proposer_attempt=0 +total_metal_device_proposer_use=0 +total_metal_device_proposer_fallback=0 +total_metal_device_proposer_policy_mismatch=0 stats_field() { printf '%s\n' "$1" | awk -v key="$2" ' @@ -84,6 +106,65 @@ case "$REQUIRE_CUDA_EXACTN" in exit 1 ;; esac +case "$REQUIRE_CUDA_EXACTN_BATCH_HEAD" in +0|1) ;; +*) + echo "dspark-fixture: DS4_DSPARK_FIXTURE_REQUIRE_CUDA_EXACTN_BATCH_HEAD must be 0 or 1" >&2 + exit 1 + ;; +esac +if [ "$REQUIRE_CUDA_EXACTN_BATCH_HEAD" != 0 ]; then + REQUIRE_CUDA_EXACTN=1 +fi +case "$REQUIRE_CUDA_DEVICE_PROPOSER" in +0|1) ;; +*) + echo "dspark-fixture: DS4_DSPARK_FIXTURE_REQUIRE_CUDA_DEVICE_PROPOSER must be 0 or 1" >&2 + exit 1 + ;; +esac +case "$REQUIRE_METAL_EXACTN_BATCH_HEAD" in +0|1) ;; +*) + echo "dspark-fixture: DS4_DSPARK_FIXTURE_REQUIRE_METAL_EXACTN_BATCH_HEAD must be 0 or 1" >&2 + exit 1 + ;; +esac +case "$REQUIRE_METAL_EXACTN_PARTIAL" in +0|1) ;; +*) + echo "dspark-fixture: DS4_DSPARK_FIXTURE_REQUIRE_METAL_EXACTN_PARTIAL must be 0 or 1" >&2 + exit 1 + ;; +esac +case "$REQUIRE_METAL_DEVICE_PROPOSER" in +0|1) ;; +*) + echo "dspark-fixture: DS4_DSPARK_FIXTURE_REQUIRE_METAL_DEVICE_PROPOSER must be 0 or 1" >&2 + exit 1 + ;; +esac +case "$REQUIRE_DIRECT" in +0|1) ;; +*) + echo "dspark-fixture: DS4_DSPARK_FIXTURE_REQUIRE_DIRECT_COMMIT must be 0 or 1" >&2 + exit 1 + ;; +esac +case "$REQUIRE_IDENTICAL" in +0|1) ;; +*) + echo "dspark-fixture: DS4_DSPARK_FIXTURE_REQUIRE_IDENTICAL must be 0 or 1" >&2 + exit 1 + ;; +esac +if [ "$REQUIRE_EXACT2" != 0 ] || [ "$REQUIRE_CUDA_EXACTN" != 0 ] || + [ "$REQUIRE_CUDA_DEVICE_PROPOSER" != 0 ] || + [ "$REQUIRE_METAL_EXACTN_BATCH_HEAD" != 0 ] || + [ "$REQUIRE_METAL_EXACTN_PARTIAL" != 0 ] || + [ "$REQUIRE_METAL_DEVICE_PROPOSER" != 0 ]; then + REQUIRE_IDENTICAL=1 +fi case "$SSD_CACHE_EXPERTS" in ""|*[!0-9]*) if [ -n "$SSD_CACHE_EXPERTS" ]; then @@ -161,7 +242,15 @@ print_metadata() { exact2_metal=${DS4_METAL_DSPARK_EXACT2:-unset} exactn_cuda=${DS4_CUDA_DSPARK_EXACTN:-unset} exactn_cuda_disable=${DS4_CUDA_DISABLE_DSPARK_EXACTN:-unset} + exactn_cuda_batch_head=${DS4_CUDA_DSPARK_EXACTN_BATCH_HEAD:-unset} + exactn_cuda_batch_head_disable=${DS4_CUDA_DISABLE_DSPARK_EXACTN_BATCH_HEAD:-unset} + cuda_device_proposer=${DS4_CUDA_DSPARK_DEVICE_PROPOSER:-unset} + cuda_device_proposer_disable=${DS4_CUDA_DSPARK_NO_DEVICE_PROPOSER:-unset} exactn_union_metal=${DS4_METAL_DSPARK_EXACTN_UNION:-unset} + exactn_metal_batch_head=${DS4_METAL_DSPARK_EXACTN_BATCH_HEAD:-unset} + exactn_metal_batch_head_disable=${DS4_METAL_DISABLE_DSPARK_EXACTN_BATCH_HEAD:-unset} + metal_device_proposer=${DS4_METAL_DSPARK_DEVICE_PROPOSER:-unset} + metal_device_proposer_disable=${DS4_METAL_DSPARK_NO_DEVICE_PROPOSER:-unset} noncausal_online_cuda=${DS4_CUDA_ENABLE_DSPARK_NONCAUSAL_ONLINE:-unset} noncausal_online_cuda_disable=${DS4_CUDA_DISABLE_DSPARK_NONCAUSAL_ONLINE:-unset} verify_noncausal=${DS4_DSPARK_VERIFY_NONCAUSAL:-unset} @@ -192,11 +281,20 @@ print_metadata() { printf '# exact2_cuda=%s exact2_metal=%s proposer_block_max_cuda=%s proposer_block_max_metal=%s verifier_block_max=%s require_exact2=%s\n' \ "$exact2_cuda" "$exact2_metal" "$proposer_cap_cuda" \ "$proposer_cap_metal" "$verifier_cap" "$REQUIRE_EXACT2" - printf '# exactn_cuda=%s exactn_cuda_disable=%s require_cuda_exactn=%s exactn_union_metal=%s noncausal_online_cuda=%s noncausal_online_cuda_disable=%s verify_noncausal=%s exact_rows_async_tails_metal=%s\n' \ + printf '# exactn_cuda=%s exactn_cuda_disable=%s require_cuda_exactn=%s exactn_cuda_batch_head=%s exactn_cuda_batch_head_disable=%s require_cuda_exactn_batch_head=%s cuda_device_proposer=%s cuda_device_proposer_disable=%s require_cuda_device_proposer=%s exactn_union_metal=%s noncausal_online_cuda=%s noncausal_online_cuda_disable=%s verify_noncausal=%s exact_rows_async_tails_metal=%s\n' \ "$exactn_cuda" "$exactn_cuda_disable" "$REQUIRE_CUDA_EXACTN" \ + "$exactn_cuda_batch_head" "$exactn_cuda_batch_head_disable" \ + "$REQUIRE_CUDA_EXACTN_BATCH_HEAD" \ + "$cuda_device_proposer" "$cuda_device_proposer_disable" \ + "$REQUIRE_CUDA_DEVICE_PROPOSER" \ "$exactn_union_metal" "$noncausal_online_cuda" \ "$noncausal_online_cuda_disable" "$verify_noncausal" \ "$exact_rows_async_tails_metal" + printf '# exactn_metal_batch_head=%s exactn_metal_batch_head_disable=%s require_metal_exactn_batch_head=%s require_metal_exactn_partial=%s metal_device_proposer=%s metal_device_proposer_disable=%s require_metal_device_proposer=%s\n' \ + "$exactn_metal_batch_head" "$exactn_metal_batch_head_disable" \ + "$REQUIRE_METAL_EXACTN_BATCH_HEAD" "$REQUIRE_METAL_EXACTN_PARTIAL" \ + "$metal_device_proposer" "$metal_device_proposer_disable" \ + "$REQUIRE_METAL_DEVICE_PROPOSER" } if [ ! -x "$DS4_BIN" ]; then @@ -297,6 +395,27 @@ run_case() { cuda_exactn_attempt=$(stats_field "$stats" cuda_exactn_attempt) cuda_exactn_fallback=$(stats_field "$stats" cuda_exactn_fallback) cuda_exactn_error_fallback=$(stats_field "$stats" cuda_exactn_error_fallback) + cuda_exactn_batch_head_attempt=$(stats_field "$stats" cuda_exactn_batch_head_attempt) + cuda_exactn_batch_head_use=$(stats_field "$stats" cuda_exactn_batch_head_use) + cuda_exactn_batch_head_fallback=$(stats_field "$stats" cuda_exactn_batch_head_fallback) + cuda_device_proposer_attempt=$(stats_field "$stats" cuda_device_proposer_attempt) + cuda_device_proposer_use=$(stats_field "$stats" cuda_device_proposer_use) + cuda_device_proposer_fallback=$(stats_field "$stats" cuda_device_proposer_fallback) + cuda_device_proposer_policy_mismatch=$(stats_field "$stats" cuda_device_proposer_policy_mismatch) + exactn_union_error_fallback=$(stats_field "$stats" exactn_union_error_fallback) + exactn_union_partial_replay=$(stats_field "$stats" exactn_union_partial_replay) + exactn_union_verify_skip=$(stats_field "$stats" exactn_union_verify_skip) + metal_exactn_batch_head_attempt=$(stats_field "$stats" metal_exactn_batch_head_attempt) + metal_exactn_batch_head_use=$(stats_field "$stats" metal_exactn_batch_head_use) + metal_exactn_batch_head_fallback=$(stats_field "$stats" metal_exactn_batch_head_fallback) + metal_device_proposer_attempt=$(stats_field "$stats" metal_device_proposer_attempt) + metal_device_proposer_use=$(stats_field "$stats" metal_device_proposer_use) + metal_device_proposer_fallback=$(stats_field "$stats" metal_device_proposer_fallback) + metal_device_proposer_policy_mismatch=$(stats_field "$stats" metal_device_proposer_policy_mismatch) + exact2_full=$(stats_field "$stats" exact2_full) + cuda_exactn_full=$(stats_field "$stats" cuda_exactn_full) + exactn_union_full=$(stats_field "$stats" exactn_union_full) + exactn_full=$(stats_field "$stats" exactn_full) partial=${partial:-0} errors=${errors:-0} verifier_unavailable=${verifier_unavailable:-0} @@ -309,6 +428,27 @@ run_case() { cuda_exactn_attempt=${cuda_exactn_attempt:-0} cuda_exactn_fallback=${cuda_exactn_fallback:-0} cuda_exactn_error_fallback=${cuda_exactn_error_fallback:-0} + cuda_exactn_batch_head_attempt=${cuda_exactn_batch_head_attempt:-0} + cuda_exactn_batch_head_use=${cuda_exactn_batch_head_use:-0} + cuda_exactn_batch_head_fallback=${cuda_exactn_batch_head_fallback:-0} + cuda_device_proposer_attempt=${cuda_device_proposer_attempt:-0} + cuda_device_proposer_use=${cuda_device_proposer_use:-0} + cuda_device_proposer_fallback=${cuda_device_proposer_fallback:-0} + cuda_device_proposer_policy_mismatch=${cuda_device_proposer_policy_mismatch:-0} + exactn_union_error_fallback=${exactn_union_error_fallback:-0} + exactn_union_partial_replay=${exactn_union_partial_replay:-0} + exactn_union_verify_skip=${exactn_union_verify_skip:-0} + metal_exactn_batch_head_attempt=${metal_exactn_batch_head_attempt:-0} + metal_exactn_batch_head_use=${metal_exactn_batch_head_use:-0} + metal_exactn_batch_head_fallback=${metal_exactn_batch_head_fallback:-0} + metal_device_proposer_attempt=${metal_device_proposer_attempt:-0} + metal_device_proposer_use=${metal_device_proposer_use:-0} + metal_device_proposer_fallback=${metal_device_proposer_fallback:-0} + metal_device_proposer_policy_mismatch=${metal_device_proposer_policy_mismatch:-0} + exact2_full=${exact2_full:-0} + cuda_exactn_full=${cuda_exactn_full:-0} + exactn_union_full=${exactn_union_full:-0} + exactn_full=${exactn_full:-0} if [ "$errors" -ne 0 ]; then echo "dspark-fixture: verifier errors for $id: $stats" >&2 return 1 @@ -324,6 +464,23 @@ run_case() { total_cuda_exactn_attempt=$((total_cuda_exactn_attempt + cuda_exactn_attempt)) total_cuda_exactn_fallback=$((total_cuda_exactn_fallback + cuda_exactn_fallback)) total_cuda_exactn_error_fallback=$((total_cuda_exactn_error_fallback + cuda_exactn_error_fallback)) + total_cuda_exactn_batch_head_attempt=$((total_cuda_exactn_batch_head_attempt + cuda_exactn_batch_head_attempt)) + total_cuda_exactn_batch_head_use=$((total_cuda_exactn_batch_head_use + cuda_exactn_batch_head_use)) + total_cuda_exactn_batch_head_fallback=$((total_cuda_exactn_batch_head_fallback + cuda_exactn_batch_head_fallback)) + total_cuda_device_proposer_attempt=$((total_cuda_device_proposer_attempt + cuda_device_proposer_attempt)) + total_cuda_device_proposer_use=$((total_cuda_device_proposer_use + cuda_device_proposer_use)) + total_cuda_device_proposer_fallback=$((total_cuda_device_proposer_fallback + cuda_device_proposer_fallback)) + total_cuda_device_proposer_policy_mismatch=$((total_cuda_device_proposer_policy_mismatch + cuda_device_proposer_policy_mismatch)) + total_exactn_union_error_fallback=$((total_exactn_union_error_fallback + exactn_union_error_fallback)) + total_exactn_union_partial_replay=$((total_exactn_union_partial_replay + exactn_union_partial_replay)) + total_exactn_union_verify_skip=$((total_exactn_union_verify_skip + exactn_union_verify_skip)) + total_metal_exactn_batch_head_attempt=$((total_metal_exactn_batch_head_attempt + metal_exactn_batch_head_attempt)) + total_metal_exactn_batch_head_use=$((total_metal_exactn_batch_head_use + metal_exactn_batch_head_use)) + total_metal_exactn_batch_head_fallback=$((total_metal_exactn_batch_head_fallback + metal_exactn_batch_head_fallback)) + total_metal_device_proposer_attempt=$((total_metal_device_proposer_attempt + metal_device_proposer_attempt)) + total_metal_device_proposer_use=$((total_metal_device_proposer_use + metal_device_proposer_use)) + total_metal_device_proposer_fallback=$((total_metal_device_proposer_fallback + metal_device_proposer_fallback)) + total_metal_device_proposer_policy_mismatch=$((total_metal_device_proposer_policy_mismatch + metal_device_proposer_policy_mismatch)) if [ "$REQUIRE_EXACT2" != 0 ] && [ "$exact2_fallback" -ne 0 ]; then echo "dspark-fixture: exact2 fallback for $id: $stats" >&2 return 1 @@ -333,6 +490,34 @@ run_case() { echo "dspark-fixture: CUDA exact-N error fallback for $id: $stats" >&2 return 1 fi + if [ "$REQUIRE_CUDA_EXACTN_BATCH_HEAD" != 0 ] && + [ "$cuda_exactn_batch_head_fallback" -ne 0 ]; then + echo "dspark-fixture: CUDA exact-N batch-head fallback for $id: $stats" >&2 + return 1 + fi + if [ "$REQUIRE_CUDA_DEVICE_PROPOSER" != 0 ] && + { [ "$cuda_device_proposer_fallback" -ne 0 ] || + [ "$cuda_device_proposer_policy_mismatch" -ne 0 ]; }; then + echo "dspark-fixture: CUDA device proposer fallback/mismatch for $id: $stats" >&2 + return 1 + fi + if [ "$REQUIRE_METAL_EXACTN_BATCH_HEAD" != 0 ] && + [ "$metal_exactn_batch_head_fallback" -ne 0 ]; then + echo "dspark-fixture: Metal exact-N batch-head fallback for $id: $stats" >&2 + return 1 + fi + if [ "$REQUIRE_METAL_EXACTN_PARTIAL" != 0 ] && + { [ "$exactn_union_error_fallback" -ne 0 ] || + [ "$exactn_union_partial_replay" -ne "$exactn_union_verify_skip" ]; }; then + echo "dspark-fixture: Metal exact-N partial replay/skip mismatch for $id: $stats" >&2 + return 1 + fi + if [ "$REQUIRE_METAL_DEVICE_PROPOSER" != 0 ] && + { [ "$metal_device_proposer_fallback" -ne 0 ] || + [ "$metal_device_proposer_policy_mismatch" -ne 0 ]; }; then + echo "dspark-fixture: Metal device proposer fallback/mismatch for $id: $stats" >&2 + return 1 + fi if [ "$PROPOSAL_QUALITY_GUARD_ACTIVE" -ne 0 ] && [ "$id" = c_add ] && [ "$accepted_draft" -lt "$C_ADD_MIN_ACCEPTED" ]; then echo "dspark-fixture: c_add accepted_draft $accepted_draft below required $C_ADD_MIN_ACCEPTED: $stats" >&2 @@ -344,7 +529,8 @@ run_case() { if [ "$direct_partial" -gt 0 ]; then direct_partial_cases=$((direct_partial_cases + 1)) fi - direct_commits=$((direct_commits + direct_full + direct_partial)) + direct_commits=$((direct_commits + direct_full + direct_partial + \ + exact2_full + cuda_exactn_full + exactn_union_full + exactn_full)) printf '%s\toutput_match=%s\tbaseline_tps=%s\tdspark_tps=%s\t%s\n' \ "$id" "$output_match" "${base_tps:-n/a}" "${dspark_tps:-n/a}" "$stats" @@ -388,6 +574,38 @@ if [ "$REQUIRE_CUDA_EXACTN" != 0 ]; then "$total_cuda_exactn_attempt" "$total_cuda_exactn_fallback" \ "$total_cuda_exactn_error_fallback" fi +if [ "$REQUIRE_CUDA_EXACTN_BATCH_HEAD" != 0 ]; then + printf '# cuda_exactn_batch_head_attempt=%s cuda_exactn_batch_head_use=%s cuda_exactn_batch_head_fallback=%s\n' \ + "$total_cuda_exactn_batch_head_attempt" \ + "$total_cuda_exactn_batch_head_use" \ + "$total_cuda_exactn_batch_head_fallback" +fi +if [ "$REQUIRE_CUDA_DEVICE_PROPOSER" != 0 ]; then + printf '# cuda_device_proposer_attempt=%s cuda_device_proposer_use=%s cuda_device_proposer_fallback=%s cuda_device_proposer_policy_mismatch=%s\n' \ + "$total_cuda_device_proposer_attempt" \ + "$total_cuda_device_proposer_use" \ + "$total_cuda_device_proposer_fallback" \ + "$total_cuda_device_proposer_policy_mismatch" +fi +if [ "$REQUIRE_METAL_EXACTN_BATCH_HEAD" != 0 ]; then + printf '# metal_exactn_batch_head_attempt=%s metal_exactn_batch_head_use=%s metal_exactn_batch_head_fallback=%s\n' \ + "$total_metal_exactn_batch_head_attempt" \ + "$total_metal_exactn_batch_head_use" \ + "$total_metal_exactn_batch_head_fallback" +fi +if [ "$REQUIRE_METAL_EXACTN_PARTIAL" != 0 ]; then + printf '# exactn_union_error_fallback=%s exactn_union_partial_replay=%s exactn_union_verify_skip=%s\n' \ + "$total_exactn_union_error_fallback" \ + "$total_exactn_union_partial_replay" \ + "$total_exactn_union_verify_skip" +fi +if [ "$REQUIRE_METAL_DEVICE_PROPOSER" != 0 ]; then + printf '# metal_device_proposer_attempt=%s metal_device_proposer_use=%s metal_device_proposer_fallback=%s metal_device_proposer_policy_mismatch=%s\n' \ + "$total_metal_device_proposer_attempt" \ + "$total_metal_device_proposer_use" \ + "$total_metal_device_proposer_fallback" \ + "$total_metal_device_proposer_policy_mismatch" +fi if [ "$REQUIRE_CUDA_EXACTN" != 0 ] && [ "$total_cuda_exactn_attempt" -eq 0 ]; then echo "dspark-fixture: CUDA exact-N was required but never attempted" >&2 @@ -398,3 +616,42 @@ if [ "$REQUIRE_CUDA_EXACTN" != 0 ] && echo "dspark-fixture: CUDA exact-N error fallback count=$total_cuda_exactn_error_fallback" >&2 exit 1 fi +if [ "$REQUIRE_CUDA_EXACTN_BATCH_HEAD" != 0 ] && + { [ "$total_cuda_exactn_batch_head_attempt" -eq 0 ] || + [ "$total_cuda_exactn_batch_head_use" -eq 0 ] || + [ "$total_cuda_exactn_batch_head_fallback" -ne 0 ]; }; then + echo "dspark-fixture: CUDA exact-N batch head not cleanly exercised (attempt=$total_cuda_exactn_batch_head_attempt use=$total_cuda_exactn_batch_head_use fallback=$total_cuda_exactn_batch_head_fallback)" >&2 + exit 1 +fi +if [ "$REQUIRE_CUDA_DEVICE_PROPOSER" != 0 ] && + { [ "$total_cuda_device_proposer_attempt" -eq 0 ] || + [ "$total_cuda_device_proposer_use" -eq 0 ] || + [ "$total_cuda_device_proposer_attempt" -ne "$total_cuda_device_proposer_use" ] || + [ "$total_cuda_device_proposer_fallback" -ne 0 ] || + [ "$total_cuda_device_proposer_policy_mismatch" -ne 0 ]; }; then + echo "dspark-fixture: CUDA device proposer not cleanly exercised (attempt=$total_cuda_device_proposer_attempt use=$total_cuda_device_proposer_use fallback=$total_cuda_device_proposer_fallback policy_mismatch=$total_cuda_device_proposer_policy_mismatch)" >&2 + exit 1 +fi +if [ "$REQUIRE_METAL_EXACTN_BATCH_HEAD" != 0 ] && + { [ "$total_metal_exactn_batch_head_attempt" -eq 0 ] || + [ "$total_metal_exactn_batch_head_use" -eq 0 ] || + [ "$total_metal_exactn_batch_head_fallback" -ne 0 ]; }; then + echo "dspark-fixture: Metal exact-N batch head not cleanly exercised (attempt=$total_metal_exactn_batch_head_attempt use=$total_metal_exactn_batch_head_use fallback=$total_metal_exactn_batch_head_fallback)" >&2 + exit 1 +fi +if [ "$REQUIRE_METAL_EXACTN_PARTIAL" != 0 ] && + { [ "$total_exactn_union_partial_replay" -eq 0 ] || + [ "$total_exactn_union_partial_replay" -ne "$total_exactn_union_verify_skip" ] || + [ "$total_exactn_union_error_fallback" -ne 0 ]; }; then + echo "dspark-fixture: Metal exact-N partial path not cleanly exercised (replay=$total_exactn_union_partial_replay verify_skip=$total_exactn_union_verify_skip error_fallback=$total_exactn_union_error_fallback)" >&2 + exit 1 +fi +if [ "$REQUIRE_METAL_DEVICE_PROPOSER" != 0 ] && + { [ "$total_metal_device_proposer_attempt" -eq 0 ] || + [ "$total_metal_device_proposer_use" -eq 0 ] || + [ "$total_metal_device_proposer_attempt" -ne "$total_metal_device_proposer_use" ] || + [ "$total_metal_device_proposer_fallback" -ne 0 ] || + [ "$total_metal_device_proposer_policy_mismatch" -ne 0 ]; }; then + echo "dspark-fixture: Metal device proposer not cleanly exercised (attempt=$total_metal_device_proposer_attempt use=$total_metal_device_proposer_use fallback=$total_metal_device_proposer_fallback policy_mismatch=$total_metal_device_proposer_policy_mismatch)" >&2 + exit 1 +fi diff --git a/tests/test_metal_exactn_oracle.c b/tests/test_metal_exactn_oracle.c index 10b3a0f6b..6198caa14 100644 --- a/tests/test_metal_exactn_oracle.c +++ b/tests/test_metal_exactn_oracle.c @@ -50,6 +50,11 @@ enum { EXACTN_UNION_FALLBACKS, EXACTN_UNION_PARTIAL_FALLBACKS, EXACTN_UNION_ERROR_FALLBACKS, + EXACTN_UNION_PARTIAL_REPLAYS, + EXACTN_UNION_VERIFY_SKIPS, + EXACTN_UNION_BATCH_HEAD_ATTEMPTS, + EXACTN_UNION_BATCH_HEAD_USES, + EXACTN_UNION_BATCH_HEAD_FALLBACKS, EXACTN_UNION_COUNTER_COUNT }; @@ -230,26 +235,35 @@ static void read_union_stats( } static void check_union_stats_delta( - const uint64_t before[EXACTN_UNION_COUNTER_COUNT], - const uint64_t after[EXACTN_UNION_COUNTER_COUNT], - const exactn_case *tc) { + const uint64_t before[EXACTN_UNION_COUNTER_COUNT], + const uint64_t after[EXACTN_UNION_COUNTER_COUNT], + const exactn_case *tc, + bool expect_batch_head) { uint64_t expected[EXACTN_UNION_COUNTER_COUNT] = {0}; /* EOS in the first draft row truncates the block to N=1 before exact-N * dispatch. Every other full block (including middle EOS) is committed - * by the union path; a deliberately wrong row falls back to the slow - * exact-N oracle after recording a partial mismatch. */ + * by the union path; a deliberately wrong row restores once and exactly + * replays the already verified prefix. */ if (tc->eos_at != 0) { expected[EXACTN_UNION_ATTEMPTS] = 1; if (tc->reject_at >= 0) { expected[EXACTN_UNION_FALLBACKS] = 1; expected[EXACTN_UNION_PARTIAL_FALLBACKS] = 1; + expected[EXACTN_UNION_PARTIAL_REPLAYS] = 1; + expected[EXACTN_UNION_VERIFY_SKIPS] = 1; } else { expected[EXACTN_UNION_FULL_ACCEPTS] = 1; } + if (expect_batch_head) { + expected[EXACTN_UNION_BATCH_HEAD_ATTEMPTS] = 1; + expected[EXACTN_UNION_BATCH_HEAD_USES] = 1; + } } static const char *const names[EXACTN_UNION_COUNTER_COUNT] = { - "attempt", "full", "fallback", "partial", "error" + "attempt", "full", "fallback", "partial", "error", + "partial-replay", "verify-skip", "batch-head-attempt", + "batch-head-use", "batch-head-fallback" }; for (int i = 0; i < EXACTN_UNION_COUNTER_COUNT; i++) { if (after[i] < before[i] || after[i] - before[i] != expected[i]) { @@ -273,10 +287,11 @@ static void run_case(ds4_session *session, const int correct[MAX_DRAFT], const int wrong[MAX_DRAFT], int model_eos, - int vocab, - float *expected_logits, - float *actual_logits, - const exactn_case *tc) { + int vocab, + float *expected_logits, + float *actual_logits, + const exactn_case *tc, + bool expect_batch_head) { int drafts[MAX_DRAFT]; memcpy(drafts, correct, (size_t)tc->draft_n * sizeof(drafts[0])); if (tc->reject_at >= 0) drafts[tc->reject_at] = wrong[tc->reject_at]; @@ -286,10 +301,9 @@ static void run_case(ds4_session *session, cycle_eos = drafts[tc->eos_at]; } - /* Run the production path first. A partial fallback may conservatively - * accept fewer correct prefix rows if the legacy batch top differs near a - * tie; it must never accept the deliberately wrong row. The sequential - * oracle is therefore built for the prefix production actually commits. */ + /* Run the production path first. A partial union result already proves + * the complete correct prefix and skips the legacy batch verifier, so the + * commit must stop exactly at the deliberately wrong row. */ restore_snapshot(session, base, tc->name); int accepted[MAX_DRAFT] = {-1, -1, -1, -1, -1}; char err[256] = ""; @@ -301,7 +315,8 @@ static void run_case(ds4_session *session, accepted, MAX_DRAFT, err, sizeof(err)); if (accepted_n < 0) fail("exact-N cycle", tc->name, err); read_union_stats(session, union_after, tc->name); - check_union_stats_delta(union_before, union_after, tc); + check_union_stats_delta(union_before, union_after, tc, + expect_batch_head); int full_expected = tc->draft_n; if (tc->eos_at >= 0) { /* Production truncates at the first occurrence of EOS. The fixture @@ -320,8 +335,7 @@ static void run_case(ds4_session *session, full_expected, accepted_n, err); fail("accepted prefix length", tc->name, detail); } - if (tc->reject_at >= 0 && - (accepted_n <= 0 || accepted_n > tc->reject_at)) { + if (tc->reject_at >= 0 && accepted_n != tc->reject_at) { char detail[160]; snprintf(detail, sizeof(detail), "reject_at=%d accepted=%d err=%s", @@ -383,9 +397,21 @@ int main(void) { return required && required[0] && strcmp(required, "0") != 0 ? 1 : 0; } + const char *batch_head_env = + getenv("DS4_TEST_METAL_EXACTN_BATCH_HEAD"); + const bool expect_batch_head = + batch_head_env && batch_head_env[0] && + strcmp(batch_head_env, "0") != 0; + setenv("DS4_TEST_METAL_EXACTN_ORACLE", "1", 1); setenv("DS4_METAL_DSPARK_EXACTN_UNION", "1", 1); setenv("DS4_METAL_DSPARK_EXACTN", "1", 1); + if (expect_batch_head) { + setenv("DS4_METAL_DSPARK_EXACTN_BATCH_HEAD", "1", 1); + unsetenv("DS4_METAL_DISABLE_DSPARK_EXACTN_BATCH_HEAD"); + } else { + unsetenv("DS4_METAL_DSPARK_EXACTN_BATCH_HEAD"); + } setenv("DS4_DSPARK_STATS", "1", 1); setenv("DS4_DSPARK_SSD_VERIFY_BLOCK_MAX", "5", 1); setenv("DS4_METAL_DSPARK_ACCEPTANCE_ONLY_VERIFY", "0", 1); @@ -493,14 +519,16 @@ int main(void) { for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { run_case(session, &base, prompt.len, &expected_state, &actual_state, correct, wrong, model_eos, vocab, - expected_logits, actual_logits, &cases[i]); + expected_logits, actual_logits, &cases[i], + expect_batch_head); } fprintf(stderr, "test_metal_exactn_oracle PASS cases=%zu N=2..5 " "(six-token cycle at N=5) partial_prefixes=1..4 " - "eos=first,middle\n", - sizeof(cases) / sizeof(cases[0])); + "eos=first,middle batch_head=%s\n", + sizeof(cases) / sizeof(cases[0]), + expect_batch_head ? "required" : "disabled"); free(actual_logits); free(expected_logits); From db331f468fca6e9af945f700367b3604711d2a78 Mon Sep 17 00:00:00 2001 From: Giorgio Oppo Date: Mon, 10 Aug 2026 17:02:28 +0200 Subject: [PATCH 016/189] cuda: optimize AProjQ4 decode on GB10 Reuse canonical Q8_1 scratch for Q4 MMVQ, add grouped attention-A and the exact K1024 persistent path, and cover the new dispatches with rollback gates and parity tests. --- Makefile | 9 +- QA_BEFORE_RELEASES.md | 18 +- README.md | 25 +++ cuda/mmq/ds4_mmq.cu | 333 ++++++++++++++++++++++++++++--- cuda/mmq/ds4_mmq.h | 28 ++- cuda/mmq/test/test_mmq_parity.cu | 211 ++++++++++++++++++++ ds4_cuda.cu | 305 ++++++++++++++++++++++++---- ds4_gpu.h | 4 +- 8 files changed, 852 insertions(+), 81 deletions(-) diff --git a/Makefile b/Makefile index f79d15f92..4b8485d22 100644 --- a/Makefile +++ b/Makefile @@ -68,7 +68,7 @@ DS4_LINK_LIBS ?= $(CUDA_LDLIBS) METAL_LDLIBS := $(LDLIBS) endif -.PHONY: all help clean test test-rocm test-glm53-kda-rocm test-metal-session-batch test-metal-exactn-oracle test-mxfp4-cuda test-mxfp4-rocm test-cuda-session-batch test-cuda-mixed-batch dspark-acceptance dspark-verify-depth rocm-dspark-acceptance rocm-dspark-verify-depth mtp-verify-depth cpu cuda cuda-spark cuda-generic cuda-regression strix-halo rocm +.PHONY: all help clean test test-rocm test-glm53-kda-rocm test-metal-session-batch test-metal-exactn-oracle test-mxfp4-cuda test-mxfp4-rocm test-mmq-parity-cuda test-cuda-session-batch test-cuda-mixed-batch dspark-acceptance dspark-verify-depth rocm-dspark-acceptance rocm-dspark-verify-depth mtp-verify-depth cpu cuda cuda-spark cuda-generic cuda-regression strix-halo rocm ifeq ($(UNAME_S),Darwin) .PHONY: metal-decode-schedule-bench metal-prefill-variant-bench check-mxfp4-half-lut test-mxfp4-metal @@ -175,6 +175,7 @@ help: @echo " make cuda-spark Build CUDA for DGX Spark / GB10" @echo " make cuda-generic Build CUDA for a generic local CUDA GPU" @echo " make cuda CUDA_ARCH=sm_N Build CUDA with an explicit nvcc -arch value" + @echo " make test-mmq-parity-cuda CUDA_ARCH=sm_N Run quantized CUDA kernel parity tests" @echo " make strix-halo Build ROCm for Strix Halo / gfx1151" @echo " make rocm Alias for make strix-halo" @echo " make test-mxfp4-rocm Build and run the synthetic ROCm MXFP4 MoE test" @@ -301,6 +302,12 @@ tests/test_mxfp4_cuda: tests/test_mxfp4_cuda.cu $(MMQ_OBJS) test-mxfp4-cuda: tests/test_mxfp4_cuda ./tests/test_mxfp4_cuda + +cuda/mmq/test/test_mmq_parity: cuda/mmq/test/test_mmq_parity.cu $(MMQ_OBJS) + $(NVCC) $(NVCCFLAGS) -std=c++17 $(MMQ_INCLUDES) -o $@ $^ $(CUDA_LDLIBS) + +test-mmq-parity-cuda: cuda/mmq/test/test_mmq_parity + ./cuda/mmq/test/test_mmq_parity endif ds4.o: ds4.c ds4.h ds4_ssd.h ds4_distributed.h ds4_gpu.h diff --git a/QA_BEFORE_RELEASES.md b/QA_BEFORE_RELEASES.md index 33a87c687..ee1ed4fe7 100644 --- a/QA_BEFORE_RELEASES.md +++ b/QA_BEFORE_RELEASES.md @@ -975,9 +975,21 @@ Do not use high-performance Hugging Face Xet mode while vLLM is resident. byte-identical greedy stdout and per-token logprobs, then run the same pair under Compute Sanitizer. Record prefill, decode, and steady decode rather than copying the upstream PR numbers into a release claim. -- Repeat the GB10 comparison with AProjQ4/OutQ8. The persistent vocabulary, - compressor, HC split, scratch, and direct routed-MoE paths remain relevant, - while Q8 attention-projection consumers are intentionally ineligible. +- Repeat the GB10 comparison with AProjQ4/OutQ8. First run + `make test-mmq-parity-cuda CUDA_ARCH=sm_121`; its Q4 cases must report zero + bit mismatches for persistent scratch, grouped attention-A, and the + opt-in K1024 persistent kernel. For the model A/B, use + `DS4_CUDA_NO_Q4_GB10_FAST=1` in the control and leave it unset in the + candidate. Run a separate candidate with + `DS4_CUDA_Q4_GROUPED_ATTN_A_ORACLE=1` and require `calls>0`, + `mismatches=0`, and `skips=0`. Benchmark the K1024 persistent kernel as a + separate fail-closed arm with both + `DS4_CUDA_ENABLE_Q4_K1024_PERSISTENT=1` and + `DS4_CUDA_REQUIRE_Q4_K1024_PERSISTENT=1`; its rollback is + `DS4_CUDA_NO_Q4_K1024_PERSISTENT=1`. The persistent OutQ8 + vocabulary, compressor, HC split, direct routed-MoE paths, Q4 scratch, + grouped attention-A, and canonical B+HC epilogue remain relevant, while + the Q8-only attention-projection consumers are intentionally ineligible. - Exercise CUDA DSpark at verifier/proposer depth 5 with the fast paths enabled and disabled. Require identical final output, zero verifier errors, and matching full/partial acceptance histograms. Test both the generic batch diff --git a/README.md b/README.md index d0c2e6235..ed35bbb69 100644 --- a/README.md +++ b/README.md @@ -693,6 +693,31 @@ standalone projections. The canonical Q4 path submits to the decode stream so these projections and the attention-output tail can participate in CUDA decode graphs. +On a single DGX Spark/GB10, the AProjQ4 path also mirrors the safe parts of +the aligned-Q8 decode work while retaining canonical Q4_K MMVQ/Q8_1 +arithmetic: + +- dense and paired Q4 projections reuse the persistent 256-KiB Q8_1 scratch; + `DS4_CUDA_NO_Q4_DENSE_SCRATCH=1` restores pool allocation; +- attention-output A evaluates all output groups through one channel-grouped + MMVQ dispatch per token, preserving the one-row reduction tree of every + group; `DS4_CUDA_NO_Q4_GROUPED_ATTN_A=1` restores the per-group loop; +- attention-output B keeps its canonical MMVQ result and automatically uses + the row-packed HC epilogue described below; +- the exact Q-b shape `32768x1024` has an experimental persistent-CTA kernel + behind `DS4_CUDA_ENABLE_Q4_K1024_PERSISTENT=1`, with + `DS4_CUDA_NO_Q4_K1024_PERSISTENT=1` taking precedence. Tests can add + `DS4_CUDA_REQUIRE_Q4_K1024_PERSISTENT=1` to fail instead of silently using + canonical MMVQ when the persistent dispatch is unavailable. + +`DS4_CUDA_NO_Q4_GB10_FAST=1` is the umbrella rollback for these new GB10 +choices; it does not disable the older cross-CUDA Q-A/KV pair itself. For a +fail-closed grouped attention comparison, +`DS4_CUDA_Q4_GROUPED_ATTN_A_ORACLE=1` computes the established per-group +MMVQ reference, reports calls/mismatches/skips, and retains the reference on +any mismatch. The scratch, grouped, and persistent paths are also covered by +`make test-mmq-parity-cuda CUDA_ARCH=sm_121`. + Two additional CUDA fusions remain experimental until a device oracle passes on the target GPU. `DS4_CUDA_ENABLE_HC_NORM_MIX_FUSE=1` combines HC RMSNorm with the narrow F16 mixer when the selected standalone kernels have the same diff --git a/cuda/mmq/ds4_mmq.cu b/cuda/mmq/ds4_mmq.cu index c6c7b4842..57d7f45da 100644 --- a/cuda/mmq/ds4_mmq.cu +++ b/cuda/mmq/ds4_mmq.cu @@ -139,6 +139,7 @@ struct mmq_pair_map_scratch { }; static mmq_pair_map_scratch g_mmq_pair_maps[GGML_CUDA_MAX_DEVICES] = {}; +static int g_gb10_optimizations = 0; extern "C" void ds4_mmq_set_aligned_q81_scratch(void *ptr, size_t bytes) { g_aligned_q81_scratch_ptr = ptr; @@ -3078,6 +3079,102 @@ int ds4_mmq_moe_pair_vec_impl( return 0; } +/* GB10 AProjQ4 Q-b decode specialization (M=32768, N=1, K=1024). + * + * The canonical MMVQ small-K launch uses four warps to evaluate four rows: + * warp 0 owns Q4_K superblocks 0/1, warp 1 owns 2/3, and warps 2/3 + * contribute +0.0f. Its reduction first adds the three peer-warp partials + * lane by lane, then applies warp_reduce_sum's XOR tree. Two independent + * four-warp groups below preserve that assignment and arithmetic order while + * persistent CTAs walk eight-row tiles at a grid stride. The immutable + * canonical Q8_1 activation is staged once per CTA; no Q8_K re-quantization + * or Q4_K weight repack is involved. + * + * Keep this kernel paired with the exact M/N/K admission in + * ds4_mmq_dense_vec_impl. Generalizing the row-warp mapping would change + * floating-point association relative to MMVQ. */ +static __global__ __launch_bounds__(256, 4) void +q4_K_dense_vec_k1024_persistent_kernel( + const block_q4_K * __restrict__ W, + const block_q8_1 * __restrict__ x8, + float * __restrict__ out, + int M) { + constexpr int k_q4_blocks = 4; /* 1024 / QK_K */ + constexpr int k_q8_blocks = 32; /* 1024 / QK8_1 */ + constexpr int k_rows_per_group = 4; /* canonical MMVQ small-K tile */ + constexpr int k_groups = 2; + + /* block_q8_1 is 36 bytes. A uint32_t backing array both copies it + * efficiently and preserves the alignment required by vec_dot's int + * loads from qs. */ + __shared__ __align__(16) uint32_t x8_words[ + (k_q8_blocks * sizeof(block_q8_1)) / sizeof(uint32_t)]; + __shared__ float partial[k_groups][3][k_rows_per_group][32]; + + const uint32_t *x8_src = (const uint32_t *)x8; + for (uint32_t i = threadIdx.x; + i < (uint32_t)(sizeof(x8_words) / sizeof(x8_words[0])); + i += blockDim.x) { + x8_words[i] = x8_src[i]; + } + __syncthreads(); + + const block_q8_1 *x8_shared = (const block_q8_1 *)x8_words; + const uint32_t lane = threadIdx.x & 31u; + const uint32_t warp = threadIdx.x >> 5u; + const uint32_t group = warp >> 2u; + const uint32_t warp_in_group = warp & 3u; + const uint32_t group_tid = warp_in_group * 32u + lane; + const uint64_t row_tiles = ((uint64_t)(uint32_t)M + 7u) / 8u; + + for (uint64_t tile = blockIdx.x; tile < row_tiles; tile += gridDim.x) { + const uint32_t row0 = (uint32_t)(tile * 8u) + + group * k_rows_per_group; + float tmp[k_rows_per_group] = {0.0f}; + + /* This is the canonical N=1, K=1024 MMVQ small-K loop verbatim: + * qi/vdr = 16 and blocks_per_iter = 8 for Q4_K. */ + const int kqs = VDR_Q4_K_Q8_1_MMVQ * (int)(group_tid % 16u); + for (int kbx = (int)(group_tid / 16u); + kbx < k_q4_blocks; + kbx += 8) { + const int kby = kbx * (QK_K / QK8_1); +#pragma unroll + for (int i = 0; i < k_rows_per_group; ++i) { + tmp[i] += vec_dot_q4_K_q8_1( + W, &x8_shared[kby], + (int)((uint64_t)(row0 + (uint32_t)i) * k_q4_blocks) + kbx, + kqs); + } + } + + if (warp_in_group > 0u) { +#pragma unroll + for (int i = 0; i < k_rows_per_group; ++i) { + partial[group][warp_in_group - 1u][i][lane] = tmp[i]; + } + } + __syncthreads(); + + if (warp_in_group == 0u) { +#pragma unroll + for (int i = 0; i < k_rows_per_group; ++i) { +#pragma unroll + for (int peer = 0; peer < 3; ++peer) { + tmp[i] += partial[group][peer][i][lane]; + } + tmp[i] = warp_reduce_sum<32>(tmp[i]); + } + if (lane < k_rows_per_group) { + out[row0 + lane] = tmp[lane]; + } + } + /* Both groups must finish consuming partial before the next + * grid-stride tile reuses it. */ + __syncthreads(); + } +} + template int ds4_mmq_dense_vec_impl( const char * tag, @@ -3122,11 +3219,23 @@ int ds4_mmq_dense_vec_impl( const int64_t ne10_padded = GGML_PAD((int64_t)K, MATRIX_ROW_PADDING); const size_t nbytes_q8_1 = (size_t)N * ne10_padded * sizeof(block_q8_1) / QK8_1; - ggml_cuda_pool_alloc src1_q8_1(ctx->pool(), nbytes_q8_1); + ggml_cuda_pool_alloc src1_q8_1; + char *x8 = nullptr; + if constexpr (type == GGML_TYPE_Q4_K) { + if (g_gb10_optimizations && + getenv("DS4_CUDA_NO_Q4_GB10_FAST") == nullptr && + getenv("DS4_CUDA_NO_Q4_DENSE_SCRATCH") == nullptr) { + x8 = (char *)ds4_mmq_aligned_q81_scratch(dev, nbytes_q8_1); + } + } + if (!x8) { + src1_q8_1.alloc(ctx->pool(), nbytes_q8_1); + x8 = src1_q8_1.get(); + } // Dense src1 layout: K innermost, N next; ne11=N, ne12=1, ne13=1. quantize_row_q8_1_cuda( - X_f32, /*ids=*/nullptr, (void *)src1_q8_1.get(), + X_f32, /*ids=*/nullptr, (void *)x8, type, /*ne00=*/K, /*s11=*/(int64_t)K, /*s12=*/(int64_t)K * N, /*s13=*/(int64_t)K * N, /*ne0=*/ne10_padded, /*ne1=*/N, /*ne2=*/1, /*ne3=*/1, @@ -3155,24 +3264,61 @@ int ds4_mmq_dense_vec_impl( (void)cudaMemsetAsync(out_f32, 0, (size_t)M * (size_t)N * sizeof(float), stream); - mul_mat_vec_q_switch_type( - /*vx=*/W, /*type_x=*/type, - /*vy=*/(const void *)src1_q8_1.get(), - /*ids=*/nullptr, /*fusion=*/fusion, - /*dst=*/out_f32, - /*ncols_x=*/K, /*nrows_x=*/M, /*ncols_dst=*/N, - /*stride_row_x=*/(int)s01_row, - /*stride_col_y=*/(int)s11_y, - /*stride_col_dst=*/(int)s1_dst, - /*nchannels_x=*/1, - /*nchannels_y=*/1, - /*nchannels_dst=*/1, - /*stride_channel_x=*/0, - /*stride_channel_y=*/(int)s12_y, - /*stride_channel_dst=*/0, - /*nsamples_x=*/1, /*nsamples_dst=*/1, - /*stride_sample_x=*/0, /*stride_sample_y=*/0, /*stride_sample_dst=*/0, - /*ids_stride=*/0, stream); + bool q4_k1024_persistent = false; + if constexpr (type == GGML_TYPE_Q4_K) { + const bool exact_shape = M == 32768 && N == 1 && K == 1024; + const bool enable = + getenv("DS4_CUDA_ENABLE_Q4_K1024_PERSISTENT") != nullptr; + const bool disable = + getenv("DS4_CUDA_NO_Q4_K1024_PERSISTENT") != nullptr || + getenv("DS4_CUDA_NO_Q4_GB10_FAST") != nullptr; + if (g_gb10_optimizations && enable && !disable && exact_shape && + (((uintptr_t)W & 15u) == 0u)) { + const uint64_t row_tiles = ((uint64_t)(uint32_t)M + 7u) / 8u; + const int nsm = ggml_cuda_info().devices[dev].nsm; + const uint64_t resident_blocks = + nsm > 0 ? (uint64_t)(uint32_t)nsm * 4u : 0u; + const uint64_t grid64 = row_tiles < resident_blocks + ? row_tiles : resident_blocks; + if (grid64 > 0u && grid64 <= UINT32_MAX) { + q4_K_dense_vec_k1024_persistent_kernel<<< + (unsigned)grid64, 256, 0, stream>>>( + (const block_q4_K *)W, + (const block_q8_1 *)x8, + out_f32, M); + q4_k1024_persistent = true; + } + } + if (exact_shape && + getenv("DS4_CUDA_REQUIRE_Q4_K1024_PERSISTENT") != nullptr && + !q4_k1024_persistent) { + fprintf(stderr, + "%s: required Q4_K K1024 persistent path unavailable\n", + tag); + return -4; + } + } + + if (!q4_k1024_persistent) { + mul_mat_vec_q_switch_type( + /*vx=*/W, /*type_x=*/type, + /*vy=*/(const void *)x8, + /*ids=*/nullptr, /*fusion=*/fusion, + /*dst=*/out_f32, + /*ncols_x=*/K, /*nrows_x=*/M, /*ncols_dst=*/N, + /*stride_row_x=*/(int)s01_row, + /*stride_col_y=*/(int)s11_y, + /*stride_col_dst=*/(int)s1_dst, + /*nchannels_x=*/1, + /*nchannels_y=*/1, + /*nchannels_dst=*/1, + /*stride_channel_x=*/0, + /*stride_channel_y=*/(int)s12_y, + /*stride_channel_dst=*/0, + /*nsamples_x=*/1, /*nsamples_dst=*/1, + /*stride_sample_x=*/0, /*stride_sample_y=*/0, /*stride_sample_dst=*/0, + /*ids_stride=*/0, stream); + } err = cudaGetLastError(); if (err != cudaSuccess) { @@ -3232,10 +3378,22 @@ int ds4_mmq_dense_pair_vec_impl( const int64_t ne10_padded = GGML_PAD((int64_t)K, MATRIX_ROW_PADDING); const size_t nbytes_q8_1 = (size_t)N * ne10_padded * sizeof(block_q8_1) / QK8_1; - ggml_cuda_pool_alloc src1_q8_1(ctx->pool(), nbytes_q8_1); + ggml_cuda_pool_alloc src1_q8_1; + char *x8 = nullptr; + if constexpr (type == GGML_TYPE_Q4_K) { + if (g_gb10_optimizations && + getenv("DS4_CUDA_NO_Q4_GB10_FAST") == nullptr && + getenv("DS4_CUDA_NO_Q4_DENSE_SCRATCH") == nullptr) { + x8 = (char *)ds4_mmq_aligned_q81_scratch(dev, nbytes_q8_1); + } + } + if (!x8) { + src1_q8_1.alloc(ctx->pool(), nbytes_q8_1); + x8 = src1_q8_1.get(); + } quantize_row_q8_1_cuda( - X_f32, /*ids=*/nullptr, (void *)src1_q8_1.get(), + X_f32, /*ids=*/nullptr, (void *)x8, type, /*ne00=*/K, /*s11=*/(int64_t)K, /*s12=*/(int64_t)K * N, /*s13=*/(int64_t)K * N, @@ -3262,7 +3420,7 @@ int ds4_mmq_dense_pair_vec_impl( (size_t)M0 * (size_t)N * sizeof(float), stream); mul_mat_vec_q_switch_type( /*vx=*/W0, /*type_x=*/type, - /*vy=*/(const void *)src1_q8_1.get(), + /*vy=*/(const void *)x8, /*ids=*/nullptr, /*fusion=*/fusion, /*dst=*/out0_f32, /*ncols_x=*/K, /*nrows_x=*/M0, /*ncols_dst=*/N, @@ -3289,7 +3447,7 @@ int ds4_mmq_dense_pair_vec_impl( (size_t)M1 * (size_t)N * sizeof(float), stream); mul_mat_vec_q_switch_type( /*vx=*/W1, /*type_x=*/type, - /*vy=*/(const void *)src1_q8_1.get(), + /*vy=*/(const void *)x8, /*ids=*/nullptr, /*fusion=*/fusion, /*dst=*/out1_f32, /*ncols_x=*/K, /*nrows_x=*/M1, /*ncols_dst=*/N, @@ -3314,6 +3472,117 @@ int ds4_mmq_dense_pair_vec_impl( return 0; } +__global__ static void ds4_mmq_identity_i32_kernel( + int32_t *ids, int n) { + const int i = (int)threadIdx.x; + if (i < n) ids[i] = i; +} + +/* Grouped AProjQ4 attention-A projection. Treat each attention group as an + * MMVQ channel (not as a column): ncols_dst stays one, so each channel uses + * exactly the same one-row Q4_K MMVQ specialization, K partition, peer-warp + * fold, and reduction tree as the canonical per-group loop. Only activation + * quantization and launch setup are shared across groups. */ +static int ds4_mmq_q4_K_grouped_vec_impl( + const void *W, + const float *X, + float *out, + int M, + int K, + int n_groups, + cudaStream_t stream) { + const char *tag = "ds4_mmq_q4_K_grouped_vec"; + if (!W || !X || !out) { + fprintf(stderr, "%s: null pointer\n", tag); + return -1; + } + if (!g_gb10_optimizations || + getenv("DS4_CUDA_NO_Q4_GB10_FAST") != nullptr || + getenv("DS4_CUDA_NO_Q4_GROUPED_ATTN_A") != nullptr || + M <= 0 || K <= 0 || n_groups <= 0 || n_groups > 16 || + K % 256 != 0) { + return DS4_MMQ_NOT_APPLICABLE; + } + + const int64_t row_blocks = (int64_t)K / ggml_blck_size(GGML_TYPE_Q4_K); + const int64_t weight_channel_stride = (int64_t)M * row_blocks; + if (row_blocks <= 0 || row_blocks > INT_MAX || + weight_channel_stride > INT_MAX) { + return DS4_MMQ_NOT_APPLICABLE; + } + + const int64_t ne10_padded = GGML_PAD((int64_t)K, MATRIX_ROW_PADDING); + const size_t nbytes_q8_1 = (size_t)n_groups * (size_t)ne10_padded * + sizeof(block_q8_1) / QK8_1; + if (nbytes_q8_1 > SIZE_MAX - 15u) return DS4_MMQ_NOT_APPLICABLE; + const size_t ids_offset = (nbytes_q8_1 + 15u) & ~(size_t)15u; + const size_t ids_bytes = (size_t)n_groups * sizeof(int32_t); + if (ids_offset > SIZE_MAX - ids_bytes) { + return DS4_MMQ_NOT_APPLICABLE; + } + + const int dev = ggml_cuda_get_device(); + char *x8 = (char *)ds4_mmq_aligned_q81_scratch( + dev, ids_offset + ids_bytes); + if (!x8) return DS4_MMQ_NOT_APPLICABLE; + int32_t *ids = (int32_t *)(x8 + ids_offset); + ds4_mmq_identity_i32_kernel<<<1, 32, 0, stream>>>(ids, n_groups); + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) { + fprintf(stderr, "%s: identity launch failed: %s\n", + tag, cudaGetErrorString(err)); + return -2; + } + + quantize_row_q8_1_cuda( + X, /*ids=*/nullptr, (void *)x8, + GGML_TYPE_Q4_K, /*ne00=*/K, + /*s11=*/(int64_t)K, + /*s12=*/(int64_t)K * n_groups, + /*s13=*/(int64_t)K * n_groups, + /*ne0=*/ne10_padded, /*ne1=*/n_groups, /*ne2=*/1, /*ne3=*/1, + stream); + err = cudaGetLastError(); + if (err != cudaSuccess) { + fprintf(stderr, "%s: quantize_row_q8_1_cuda failed: %s\n", + tag, cudaGetErrorString(err)); + return -3; + } + + const int64_t y_channel_stride = ne10_padded / QK8_1; + ggml_cuda_mm_fusion_args_device fusion = {}; + cudaMemsetAsync(out, 0, + (size_t)n_groups * (size_t)M * sizeof(float), stream); + mul_mat_vec_q_switch_type( + /*vx=*/W, /*type_x=*/GGML_TYPE_Q4_K, + /*vy=*/(const void *)x8, + /*ids=*/ids, /*fusion=*/fusion, + /*dst=*/out, + /*ncols_x=*/K, /*nrows_x=*/M, /*ncols_dst=*/1, + /*stride_row_x=*/(int)row_blocks, + /*stride_col_y=*/(int)y_channel_stride, + /*stride_col_dst=*/M, + /*nchannels_x=*/n_groups, + /*nchannels_y=*/n_groups, + /*nchannels_dst=*/n_groups, + /*stride_channel_x=*/(int)weight_channel_stride, + /*stride_channel_y=*/(int)y_channel_stride, + /*stride_channel_dst=*/M, + /*nsamples_x=*/1, /*nsamples_dst=*/1, + /*stride_sample_x=*/0, /*stride_sample_y=*/0, + /*stride_sample_dst=*/0, + /*ids_stride=*/1, stream); + err = cudaGetLastError(); + if (err != cudaSuccess) { + fprintf(stderr, "%s: grouped MMVQ launch failed: %s\n", + tag, cudaGetErrorString(err)); + return -4; + } + ds4_mmq_sanitize_f32( + out, (uint64_t)(uint32_t)n_groups * (uint64_t)(uint32_t)M, stream); + return 0; +} + template struct ds4_mmq_vdr_mmvq_value; template <> struct ds4_mmq_vdr_mmvq_value { static constexpr int value = VDR_IQ2_XXS_Q8_1_MMVQ; }; template <> struct ds4_mmq_vdr_mmvq_value { static constexpr int value = VDR_Q2_K_Q8_1_MMVQ; }; @@ -4093,8 +4362,6 @@ extern "C" int ds4_mmq_iq2_xxs_aligned_derepack( return 0; } -static int g_gb10_optimizations = 0; - extern "C" void ds4_mmq_set_gb10_optimizations(int enabled) { g_gb10_optimizations = enabled != 0; } @@ -5207,13 +5474,6 @@ extern "C" int ds4_mmq_q8_0_dense_vec( "ds4_mmq_q8_0_dense_vec", W, X, out, M, N, K, stream); } -extern "C" int ds4_mmq_q4_K_dense_pair_vec( - const void *W0, const void *W1, const float *X, - float *out0, float *out1, int M, int K, cudaStream_t stream) { - return ds4_mmq_q4_K_dense_pair_vec_impl( - W0, W1, X, out0, out1, M, K, stream); -} - extern "C" int ds4_mmq_q4_K_dense_vec( const void * W, const float * X, float * out, int M, int N, int K, cudaStream_t stream) { @@ -5221,6 +5481,13 @@ extern "C" int ds4_mmq_q4_K_dense_vec( "ds4_mmq_q4_K_dense_vec", W, X, out, M, N, K, stream); } +extern "C" int ds4_mmq_q4_K_grouped_vec( + const void *W, const float *X, float *out, + int M, int K, int n_groups, cudaStream_t stream) { + return ds4_mmq_q4_K_grouped_vec_impl( + W, X, out, M, K, n_groups, stream); +} + extern "C" int ds4_mmq_q4_K_dense_pair_vec( const void * W0, const void * W1, const float * X, float * out0, float * out1, diff --git a/cuda/mmq/ds4_mmq.h b/cuda/mmq/ds4_mmq.h index 7c3247dc3..87437946a 100644 --- a/cuda/mmq/ds4_mmq.h +++ b/cuda/mmq/ds4_mmq.h @@ -883,25 +883,39 @@ int ds4_mmq_q8_0_dense_vec( int K, cudaStream_t stream); -int ds4_mmq_q4_K_dense_pair_vec( - const void * W0_q4_K, - const void * W1_q4_K, +int ds4_mmq_q4_K_dense_vec( + const void * W_q4_K, const float * X_f32, - float * out0_f32, - float * out1_f32, + float * out_f32, int M, + int N, int K, cudaStream_t stream); -int ds4_mmq_q4_K_dense_vec( +// Exact grouped one-row Q4_K MMVQ for AProjQ4 attention-A on a single GB10. +// W is [n_groups][M][K], X is [n_groups][K], and out is +// [n_groups][M]. Each group retains the canonical dense_vec reduction tree; +// only Q8_1 quantization and launch setup are shared. Returns +// DS4_MMQ_NOT_APPLICABLE before enqueue when its GB10/scratch/shape gates do +// not hold. +int ds4_mmq_q4_K_grouped_vec( const void * W_q4_K, const float * X_f32, float * out_f32, int M, - int N, int K, + int n_groups, cudaStream_t stream); +// On a single GB10, the exact AProjQ4 Q-b shape (M=32768, N=1, K=1024) +// can opt into a persistent-CTA form with +// DS4_CUDA_ENABLE_Q4_K1024_PERSISTENT=1. The rollback switch +// DS4_CUDA_NO_Q4_K1024_PERSISTENT=1 is authoritative when both are set. +// DS4_CUDA_REQUIRE_Q4_K1024_PERSISTENT=1 makes an unavailable exact-shape +// dispatch fail closed instead of silently running canonical MMVQ. +// DS4_CUDA_NO_Q4_GB10_FAST=1 is the umbrella rollback for this and the +// GB10 Q4 activation scratch. Other shapes and devices retain canonical MMVQ. + // Two independent dense Q4_K projections that share the canonical Q8_1 // activation quantization. Each output is dispatched through the same MMVQ // entry as ds4_mmq_q4_K_dense_vec, so its reduction and output bits are diff --git a/cuda/mmq/test/test_mmq_parity.cu b/cuda/mmq/test/test_mmq_parity.cu index 3a7098151..05018ce37 100644 --- a/cuda/mmq/test/test_mmq_parity.cu +++ b/cuda/mmq/test/test_mmq_parity.cu @@ -1106,6 +1106,205 @@ bool run_q8_0_dense_vec(int M, int N, int K, uint32_t seed) { return ok; } +bool run_q4_K_dense_vec_gb10_parity( + int M, int N, int K, uint32_t seed, bool persistent_k1024) { + fprintf(stderr, + "=== Q4_K/DENSE_VEC_%s M=%d N=%d K=%d seed=%u ===\n", + persistent_k1024 ? "PERSISTENT" : "SCRATCH", + M, N, K, seed); + + std::mt19937 rng(seed); + std::normal_distribution nd(0.0f, 1.0f); + const int blocks_per_row = K / QK_K_LOCAL; + std::vector W((size_t)M * blocks_per_row); + for (auto &blk : W) generate_random_block_q4_K(&blk, rng); + std::vector X((size_t)N * K); + for (float &v : X) v = nd(rng); + + cudaStream_t stream = nullptr; + void *dW = nullptr; + void *scratch = nullptr; + float *dX = nullptr; + float *dRef = nullptr; + float *dGot = nullptr; + bool ok = cudaStreamCreate(&stream) == cudaSuccess && + cudaMalloc(&dW, W.size() * sizeof(block_q4_K)) == cudaSuccess && + cudaMalloc(&dX, X.size() * sizeof(float)) == cudaSuccess && + cudaMalloc(&dRef, (size_t)M * N * sizeof(float)) == cudaSuccess && + cudaMalloc(&dGot, (size_t)M * N * sizeof(float)) == cudaSuccess && + cudaMalloc(&scratch, 256u * 1024u) == cudaSuccess; + if (!ok) { + fprintf(stderr, "Q4_K dense vec parity allocation failed\n"); + if (scratch) cudaFree(scratch); + if (dGot) cudaFree(dGot); + if (dRef) cudaFree(dRef); + if (dX) cudaFree(dX); + if (dW) cudaFree(dW); + if (stream) cudaStreamDestroy(stream); + return false; + } + cudaMemcpyAsync(dW, W.data(), W.size() * sizeof(block_q4_K), + cudaMemcpyHostToDevice, stream); + cudaMemcpyAsync(dX, X.data(), X.size() * sizeof(float), + cudaMemcpyHostToDevice, stream); + + unsetenv("DS4_CUDA_NO_Q4_GB10_FAST"); + unsetenv("DS4_CUDA_NO_Q4_DENSE_SCRATCH"); + unsetenv("DS4_CUDA_NO_Q4_K1024_PERSISTENT"); + unsetenv("DS4_CUDA_ENABLE_Q4_K1024_PERSISTENT"); + unsetenv("DS4_CUDA_REQUIRE_Q4_K1024_PERSISTENT"); + ds4_mmq_set_gb10_optimizations(persistent_k1024 ? 1 : 0); + ds4_mmq_set_aligned_q81_scratch( + persistent_k1024 ? scratch : nullptr, + persistent_k1024 ? 256u * 1024u : 0u); + const int rc_ref = ds4_mmq_q4_K_dense_vec( + dW, dX, dRef, M, N, K, stream); + + ds4_mmq_set_gb10_optimizations(1); + ds4_mmq_set_aligned_q81_scratch(scratch, 256u * 1024u); + if (persistent_k1024) { + setenv("DS4_CUDA_ENABLE_Q4_K1024_PERSISTENT", "1", 1); + setenv("DS4_CUDA_REQUIRE_Q4_K1024_PERSISTENT", "1", 1); + } + const int rc_got = ds4_mmq_q4_K_dense_vec( + dW, dX, dGot, M, N, K, stream); + + std::vector ref((size_t)M * N); + std::vector got((size_t)M * N); + cudaMemcpyAsync(ref.data(), dRef, ref.size() * sizeof(float), + cudaMemcpyDeviceToHost, stream); + cudaMemcpyAsync(got.data(), dGot, got.size() * sizeof(float), + cudaMemcpyDeviceToHost, stream); + int rc_required_disabled = 0; + if (persistent_k1024) { + setenv("DS4_CUDA_NO_Q4_K1024_PERSISTENT", "1", 1); + rc_required_disabled = ds4_mmq_q4_K_dense_vec( + dW, dX, dGot, M, N, K, stream); + unsetenv("DS4_CUDA_NO_Q4_K1024_PERSISTENT"); + } + const cudaError_t sync_err = cudaStreamSynchronize(stream); + size_t mismatches = 0; + for (size_t i = 0; i < ref.size(); i++) { + if (std::memcmp(&ref[i], &got[i], sizeof(float)) != 0) mismatches++; + } + ok = rc_ref == 0 && rc_got == 0 && + (!persistent_k1024 || rc_required_disabled != 0) && + sync_err == cudaSuccess && + mismatches == 0; + fprintf(stderr, + "rc_ref=%d rc_candidate=%d rc_required_disabled=%d " + "mismatches=%zu sync=%s\n%s\n\n", + rc_ref, rc_got, rc_required_disabled, mismatches, + cudaGetErrorString(sync_err), + ok ? "PASS" : "FAIL"); + + unsetenv("DS4_CUDA_ENABLE_Q4_K1024_PERSISTENT"); + unsetenv("DS4_CUDA_REQUIRE_Q4_K1024_PERSISTENT"); + ds4_mmq_set_aligned_q81_scratch(nullptr, 0u); + ds4_mmq_set_gb10_optimizations(0); + cudaFree(scratch); + cudaFree(dGot); + cudaFree(dRef); + cudaFree(dX); + cudaFree(dW); + cudaStreamDestroy(stream); + return ok; +} + +bool run_q4_K_grouped_vec_parity( + int M, int K, int n_groups, uint32_t seed) { + fprintf(stderr, + "=== Q4_K/GROUPED_VEC M=%d K=%d groups=%d seed=%u ===\n", + M, K, n_groups, seed); + std::mt19937 rng(seed); + std::normal_distribution nd(0.0f, 1.0f); + const int blocks_per_row = K / QK_K_LOCAL; + const size_t blocks_per_group = (size_t)M * blocks_per_row; + std::vector W((size_t)n_groups * blocks_per_group); + for (auto &blk : W) generate_random_block_q4_K(&blk, rng); + std::vector X((size_t)n_groups * K); + for (float &v : X) v = nd(rng); + + cudaStream_t stream = nullptr; + void *dW = nullptr; + void *scratch = nullptr; + float *dX = nullptr; + float *dRef = nullptr; + float *dGot = nullptr; + bool ok = cudaStreamCreate(&stream) == cudaSuccess && + cudaMalloc(&dW, W.size() * sizeof(block_q4_K)) == cudaSuccess && + cudaMalloc(&dX, X.size() * sizeof(float)) == cudaSuccess && + cudaMalloc(&dRef, (size_t)n_groups * M * sizeof(float)) == cudaSuccess && + cudaMalloc(&dGot, (size_t)n_groups * M * sizeof(float)) == cudaSuccess && + cudaMalloc(&scratch, 256u * 1024u) == cudaSuccess; + if (!ok) { + fprintf(stderr, "Q4_K grouped vec parity allocation failed\n"); + if (scratch) cudaFree(scratch); + if (dGot) cudaFree(dGot); + if (dRef) cudaFree(dRef); + if (dX) cudaFree(dX); + if (dW) cudaFree(dW); + if (stream) cudaStreamDestroy(stream); + return false; + } + cudaMemcpyAsync(dW, W.data(), W.size() * sizeof(block_q4_K), + cudaMemcpyHostToDevice, stream); + cudaMemcpyAsync(dX, X.data(), X.size() * sizeof(float), + cudaMemcpyHostToDevice, stream); + unsetenv("DS4_CUDA_NO_Q4_GB10_FAST"); + unsetenv("DS4_CUDA_NO_Q4_GROUPED_ATTN_A"); + ds4_mmq_set_gb10_optimizations(1); + ds4_mmq_set_aligned_q81_scratch(scratch, 256u * 1024u); + + int rc_ref = 0; + for (int g = 0; g < n_groups && rc_ref == 0; g++) { + rc_ref = ds4_mmq_q4_K_dense_vec( + (const char *)dW + (size_t)g * blocks_per_group * + sizeof(block_q4_K), + dX + (size_t)g * K, + dRef + (size_t)g * M, + M, 1, K, stream); + } + const int rc_got = ds4_mmq_q4_K_grouped_vec( + dW, dX, dGot, M, K, n_groups, stream); + setenv("DS4_CUDA_NO_Q4_GB10_FAST", "1", 1); + const int rc_disabled = ds4_mmq_q4_K_grouped_vec( + dW, dX, dGot, M, K, n_groups, stream); + unsetenv("DS4_CUDA_NO_Q4_GB10_FAST"); + + std::vector ref((size_t)n_groups * M); + std::vector got((size_t)n_groups * M); + cudaMemcpyAsync(ref.data(), dRef, ref.size() * sizeof(float), + cudaMemcpyDeviceToHost, stream); + cudaMemcpyAsync(got.data(), dGot, got.size() * sizeof(float), + cudaMemcpyDeviceToHost, stream); + const cudaError_t sync_err = cudaStreamSynchronize(stream); + size_t mismatches = 0; + for (size_t i = 0; i < ref.size(); i++) { + if (std::memcmp(&ref[i], &got[i], sizeof(float)) != 0) mismatches++; + } + ok = rc_ref == 0 && rc_got == 0 && + rc_disabled == DS4_MMQ_NOT_APPLICABLE && + sync_err == cudaSuccess && + mismatches == 0; + fprintf(stderr, + "rc_ref=%d rc_grouped=%d rc_disabled=%d " + "mismatches=%zu sync=%s\n%s\n\n", + rc_ref, rc_got, rc_disabled, mismatches, + cudaGetErrorString(sync_err), + ok ? "PASS" : "FAIL"); + + ds4_mmq_set_aligned_q81_scratch(nullptr, 0u); + ds4_mmq_set_gb10_optimizations(0); + cudaFree(scratch); + cudaFree(dGot); + cudaFree(dRef); + cudaFree(dX); + cudaFree(dW); + cudaStreamDestroy(stream); + return ok; +} + } // namespace int main(int argc, char ** argv) { @@ -1238,6 +1437,18 @@ int main(int argc, char ** argv) { all_ok &= run_q8_0_dense_vec(/*M=*/64, /*N=*/1, /*K=*/256, 0xC0FE40); all_ok &= run_q8_0_dense_vec(/*M=*/256, /*N=*/1, /*K=*/512, 0xC0FE41); all_ok &= run_q8_0_dense_vec(/*M=*/1024, /*N=*/1, /*K=*/4096, 0xC0FE42); + all_ok &= run_q4_K_dense_vec_gb10_parity( + /*M=*/1024, /*N=*/5, /*K=*/4096, 0xC4FE40, false); + all_ok &= run_q4_K_dense_vec_gb10_parity( + /*M=*/32768, /*N=*/1, /*K=*/1024, 0xC4FE41, true); + all_ok &= run_q4_K_grouped_vec_parity( + /*M=*/256, /*K=*/8192, /*groups=*/4, 0xC4FE42); + // DeepSeek-V4 Flash AProjQ4 attention-A production shape. + all_ok &= run_q4_K_grouped_vec_parity( + /*M=*/128, /*K=*/4096, /*groups=*/8, 0xC4FE43); + // Pro-style maximum group count accepted by the grouped entry. + all_ok &= run_q4_K_grouped_vec_parity( + /*M=*/64, /*K=*/4096, /*groups=*/16, 0xC4FE44); fprintf(stderr, "===================\n"); fprintf(stderr, "%s\n", all_ok ? "ALL PASS" : "SOME FAILED"); diff --git a/ds4_cuda.cu b/ds4_cuda.cu index fde3d404d..21b807805 100644 --- a/ds4_cuda.cu +++ b/ds4_cuda.cu @@ -938,12 +938,16 @@ extern "C" int ds4_gpu_decode_graphs_supported(void) { strcmp(s, "false") == 0 || strcmp(s, "FALSE") == 0); const char *oracle = getenv("DS4_CUDA_Q4_ATTN_OUT_HC_ORACLE"); - const int oracle_on = oracle && *oracle && - strcmp(oracle, "0") != 0; + const char *grouped_oracle = + getenv("DS4_CUDA_Q4_GROUPED_ATTN_A_ORACLE"); + const int oracle_on = + (oracle && *oracle && strcmp(oracle, "0") != 0) || + (grouped_oracle && *grouped_oracle && + strcmp(grouped_oracle, "0") != 0); if (oracle_on) { fprintf(stderr, "ds4: CUDA decode graph capture disabled for Q4 " - "attention-output/HC oracle\n"); + "attention oracle\n"); enabled = 0; } else if (off) { fprintf(stderr, "ds4: DS4_CUDA_DECODE_GRAPHS=%s - decode graph capture disabled\n", s); @@ -1410,6 +1414,24 @@ static int cuda_env_flag_enabled(const char *name, int fallback) { return strcmp(env, "0") != 0; } +/* Conservative AProjQ4 port of the GB10 Q8 decode dispatch ideas. Q4_K + * must keep the canonical MMVQ Q8_1 activation quantizer and reduction DAG; + * switching these projections to the local Q8_K fallback changes output + * bits. The helpers below therefore only select launch/packing consumers + * around the existing MMVQ entries. Non-GB10, multi-GPU, quality mode and + * DS4_CUDA_MMQ=0 all retain the ordinary caller fallback. The global kill + * switch is intentionally checked in addition to each path's local rollback. + */ +static int cuda_q4_gb10_fast_path_enabled( + int logical_tier, const char *local_rollback_env) { + if (g_n_gpus != 1) return 0; + if (logical_tier < 0) logical_tier = 0; + if (logical_tier != 0 || !g_cuda_is_gb10[logical_tier]) return 0; + if (getenv("DS4_CUDA_NO_Q4_GB10_FAST") != NULL) return 0; + if (local_rollback_env && getenv(local_rollback_env) != NULL) return 0; + return cuda_use_mmq(); +} + extern "C" int ds4_gpu_set_decode_fast_attention(int enabled) { const int old = g_decode_fast_attention; g_decode_fast_attention = enabled != 0; @@ -16266,7 +16288,8 @@ extern "C" int ds4_gpu_matmul_q4_K_pair_decode_tensor( return ds4_mmq_q4_K_dense_pair_vec( w0, w1, (const float *)x->ptr, (float *)out0->ptr, (float *)out1->ptr, - (int)out_dim, (int)in_dim, cuda_decode_stream()) == 0; + (int)out_dim, (int)out_dim, /*N=*/1, (int)in_dim, + cuda_decode_stream()) == 0; } extern "C" int ds4_gpu_matmul_q8_0_pair_tensor( @@ -16571,6 +16594,8 @@ extern "C" int ds4_gpu_matmul_q4_K_pair_tensor( uint64_t weight0_offset, uint64_t weight1_offset, uint64_t in_dim, uint64_t out0_dim, uint64_t out1_dim, const ds4_gpu_tensor *x, uint64_t n_tok) { + /* Cross-CUDA pair dispatch predates the GB10 specialization. Preserve it + * verbatim outside GB10; this switch remains its established rollback. */ if (getenv("DS4_CUDA_DISABLE_Q4_DENSE_PAIR") != NULL) return 0; return cuda_matmul_q4_K_pair_tensor_impl( out0, out1, model_map, model_size, @@ -34003,6 +34028,10 @@ static int cuda_matmul_q4_K_tensor( rc, (unsigned long long)in_dim, (unsigned long long)out_dim, (unsigned long long)n_tok); + if (in_dim == 1024u && out_dim == 32768u && n_tok == 1u && + getenv("DS4_CUDA_REQUIRE_Q4_K1024_PERSISTENT") != NULL) { + return 0; + } } void *tmp = cuda_tmp_alloc_on(logical_tier, n_tok * blocks * sizeof(cuda_block_q8_K), @@ -34089,7 +34118,13 @@ static int cuda_matmul_q4_K_pair_tensor_impl( "q4_K dense pair1"); if (!w0 || !w1) return 0; + const int gb10_canonical = cuda_q4_gb10_fast_path_enabled( + logical_tier, "DS4_CUDA_DISABLE_Q4_DENSE_PAIR"); if (cuda_use_mmq()) { + /* One Q8_1 activation allocation/quantization is shared by both + * canonical MMVQ legs. On GB10 a rejected launch fails closed to + * ds4.c's independent projections; the Q8_K pair fallback below is + * intentionally retained only for the pre-existing non-GB10 path. */ const int rc = ds4_mmq_q4_K_dense_pair_vec( w0, w1, (const float *)x->ptr, (float *)out0->ptr, (float *)out1->ptr, @@ -34103,6 +34138,7 @@ static int cuda_matmul_q4_K_pair_tensor_impl( (unsigned long long)out0_dim, (unsigned long long)out1_dim, (unsigned long long)n_tok); + if (gb10_canonical) return 0; } if (n_tok > UINT64_MAX / blocks || @@ -34253,10 +34289,15 @@ extern "C" int ds4_gpu_matmul_q4_K_hc_expand_available(void) { * zero-call summary that makes an ineligible/unused A/B visible. */ cuda_q4_attn_hc_oracle_register_report(); } - /* MMQ decode stays opt-in until its row-packed HC epilogue is timed on - * CUDA hardware. Unlike the old experiment, the normal enable switch no - * longer changes Q8_1 activation quantization. The explicitly named - * Q8_K experiment remains available behind a separate gate and oracle. */ + /* On GB10 the default candidate keeps the ordinary Q4_K MMVQ result and + * only replaces the four-way HC launch with a row-packed epilogue. The + * existing disable variable (plus DS4_CUDA_NO_Q4_GB10_FAST) is a complete + * rollback to the caller's separate B projection + HC expansion. Legacy + * explicit experiment/oracle switches remain usable for diagnostics. */ + if (cuda_q4_gb10_fast_path_enabled( + g_current_logical_tier, NULL)) { + return 1; + } return !cuda_use_mmq() || cuda_env_flag_enabled("DS4_CUDA_ENABLE_Q4_ATTN_OUT_HC_FUSE", 0) || cuda_env_flag_enabled( @@ -35315,29 +35356,55 @@ extern "C" int ds4_gpu_attention_output_q4_K_batch_tensor( model_map, out_a_offset, out_a_bytes, logical_tier, "q4 attn_out_a"); if (!out_a) return 0; - /* Heads are token-major with groups interleaved. Pack one group at a time - * into graph scratch, run a token-batched MMQ, then scatter its rank rows - * into the token-major low tensor. This is the stable fast path on CUDA. */ - for (uint32_t g = 0; g < n_groups; g++) { - cudaError_t ce = cudaMemcpy2DAsync( - group_tmp->ptr, group_dim * sizeof(float), - (const float *)heads->ptr + (uint64_t)g * group_dim, - (uint64_t)n_groups * group_dim * sizeof(float), - group_dim * sizeof(float), n_tokens, - cudaMemcpyDeviceToDevice, cuda_decode_stream()); - if (!cuda_ok(ce, "q4 attention output heads pack")) return 0; - const int rc = ds4_mmq_q4_K_dense( - out_a + (uint64_t)g * rank * row_a_bytes, - (const float *)group_tmp->ptr, (float *)low_tmp->ptr, - (int)rank, (int)n_tokens, (int)group_dim, - cuda_decode_stream()); - if (rc != 0) return 0; - ce = cudaMemcpy2DAsync( - (float *)low->ptr + (uint64_t)g * rank, - low_dim * sizeof(float), low_tmp->ptr, rank * sizeof(float), - rank * sizeof(float), n_tokens, - cudaMemcpyDeviceToDevice, cuda_decode_stream()); - if (!cuda_ok(ce, "q4 attention output low unpack")) return 0; + const int grouped_gb10 = + n_tokens <= 8u && n_groups <= INT_MAX && + ds4_tensor_device_idx(low) == logical_tier && + ds4_tensor_device_idx(heads) == logical_tier && + cuda_q4_gb10_fast_path_enabled( + logical_tier, "DS4_CUDA_NO_Q4_GROUPED_ATTN_A"); + if (grouped_gb10) { + /* DSpark verification stores [token][group][K]. Dispatch one exact + * grouped MMVQ per token: each group keeps its own Q8_1 row and Q4_K + * weights, while the grouped entry amortizes setup/launch overhead. + * A preflight rejection returns 0 to the unchanged row-wise caller; + * a negative post-launch result is fatal (the caller understands the + * tri-state contract for this tiny-batch hook). */ + for (uint32_t t = 0; t < n_tokens; t++) { + const int rc = ds4_mmq_q4_K_grouped_vec( + out_a, + (const float *)heads->ptr + + (uint64_t)t * n_groups * group_dim, + (float *)low->ptr + (uint64_t)t * low_dim, + (int)rank, (int)group_dim, (int)n_groups, + cuda_decode_stream()); + if (rc == DS4_MMQ_NOT_APPLICABLE) return 0; + if (rc != 0) return -1; + } + } else { + /* Existing cross-CUDA path: pack one group at a time, run a + * token-batched MMQ, then scatter rank rows back to token-major low. + * Keep this byte-for-byte outside the new GB10 dispatch. */ + for (uint32_t g = 0; g < n_groups; g++) { + cudaError_t ce = cudaMemcpy2DAsync( + group_tmp->ptr, group_dim * sizeof(float), + (const float *)heads->ptr + (uint64_t)g * group_dim, + (uint64_t)n_groups * group_dim * sizeof(float), + group_dim * sizeof(float), n_tokens, + cudaMemcpyDeviceToDevice, cuda_decode_stream()); + if (!cuda_ok(ce, "q4 attention output heads pack")) return 0; + const int rc = ds4_mmq_q4_K_dense( + out_a + (uint64_t)g * rank * row_a_bytes, + (const float *)group_tmp->ptr, (float *)low_tmp->ptr, + (int)rank, (int)n_tokens, (int)group_dim, + cuda_decode_stream()); + if (rc != 0) return 0; + ce = cudaMemcpy2DAsync( + (float *)low->ptr + (uint64_t)g * rank, + low_dim * sizeof(float), low_tmp->ptr, rank * sizeof(float), + rank * sizeof(float), n_tokens, + cudaMemcpyDeviceToDevice, cuda_decode_stream()); + if (!cuda_ok(ce, "q4 attention output low unpack")) return 0; + } } if (out_b_type == 12u) { return cuda_matmul_q4_K_tensor(out, model_map, model_size, @@ -35352,15 +35419,183 @@ extern "C" int ds4_gpu_attention_output_q4_K_batch_tensor( return 0; } +static uint64_t g_q4_grouped_attn_a_oracle_calls; +static uint64_t g_q4_grouped_attn_a_oracle_mismatches; +static uint64_t g_q4_grouped_attn_a_oracle_skips; +static int g_q4_grouped_attn_a_oracle_report_registered; +static int g_q4_grouped_attn_a_oracle_mismatch_reported; + +static void cuda_q4_grouped_attn_a_oracle_report(void) { + fprintf(stderr, + "ds4: CUDA Q4 grouped attention-A oracle: " + "calls=%llu mismatches=%llu skips=%llu " + "(canonical MMVQ output retained)\n", + (unsigned long long)g_q4_grouped_attn_a_oracle_calls, + (unsigned long long)g_q4_grouped_attn_a_oracle_mismatches, + (unsigned long long)g_q4_grouped_attn_a_oracle_skips); +} + +static void cuda_q4_grouped_attn_a_oracle_register_report(void) { + if (!g_q4_grouped_attn_a_oracle_report_registered) { + g_q4_grouped_attn_a_oracle_report_registered = 1; + (void)atexit(cuda_q4_grouped_attn_a_oracle_report); + } +} + extern "C" int ds4_gpu_attention_output_low_q4_K_slice_tensor( ds4_gpu_tensor *low, const void *model_map, uint64_t model_size, uint64_t out_a_offset, uint64_t group_dim, uint64_t rank, uint32_t group0, uint32_t group_cnt, const ds4_gpu_tensor *heads) { - (void)low; (void)model_map; (void)model_size; (void)out_a_offset; - (void)group_dim; (void)rank; (void)group0; (void)group_cnt; - (void)heads; - return 0; + const int oracle = cuda_env_flag_enabled( + "DS4_CUDA_Q4_GROUPED_ATTN_A_ORACLE", 0); + if (oracle) cuda_q4_grouped_attn_a_oracle_register_report(); + if (!low || !heads || !model_map || group_dim == 0u || rank == 0u || + group_cnt == 0u || group0 > UINT32_MAX - group_cnt || + group_dim > INT_MAX || rank > INT_MAX || group_cnt > INT_MAX || + (group_dim % CUDA_QK_K) != 0u) { + return 0; + } + const int logical_tier = ds4_tensor_device_idx(low); + if (ds4_tensor_device_idx(heads) != logical_tier || + !cuda_q4_gb10_fast_path_enabled( + logical_tier, "DS4_CUDA_NO_Q4_GROUPED_ATTN_A")) { + return 0; + } + + const uint64_t blocks = group_dim / CUDA_QK_K; + if (blocks == 0u || + blocks > UINT64_MAX / sizeof(cuda_block_q4_K)) { + return 0; + } + const uint64_t row_bytes = blocks * sizeof(cuda_block_q4_K); + if (rank > UINT64_MAX / row_bytes || + group_dim > UINT64_MAX / group_cnt || + rank > UINT64_MAX / group_cnt) { + return 0; + } + const uint64_t group_weight_bytes = rank * row_bytes; + if ((uint64_t)group0 > UINT64_MAX / group_weight_bytes || + (uint64_t)group_cnt > UINT64_MAX / group_weight_bytes) { + return 0; + } + const uint64_t group_weight_offset = + (uint64_t)group0 * group_weight_bytes; + const uint64_t selected_weight_bytes = + (uint64_t)group_cnt * group_weight_bytes; + if (out_a_offset > model_size || + group_weight_offset > model_size - out_a_offset) { + return 0; + } + const uint64_t selected_offset = out_a_offset + group_weight_offset; + if (selected_weight_bytes > model_size - selected_offset) return 0; + + const uint64_t heads_elems = (uint64_t)group_cnt * group_dim; + const uint64_t low_elems = (uint64_t)group_cnt * rank; + if (heads_elems > UINT64_MAX / sizeof(float) || + low_elems > UINT64_MAX / sizeof(float) || + heads->bytes < heads_elems * sizeof(float) || + low->bytes < low_elems * sizeof(float)) { + return 0; + } + + const char *out_a = cuda_resolve_weight_ptr( + model_map, selected_offset, selected_weight_bytes, logical_tier, + "q4 grouped attn_out_a"); + if (!out_a) return 0; + cudaStream_t stream = cuda_decode_stream(); + const int rc = ds4_mmq_q4_K_grouped_vec( + out_a, (const float *)heads->ptr, (float *)low->ptr, + (int)rank, (int)group_dim, (int)group_cnt, stream); + if (rc != 0) { + /* NOT_APPLICABLE is pre-enqueue and cleanly retries the established + * per-group loop. A negative result may follow a launch; returning + * zero still avoids consuming a possibly incomplete candidate. */ + return 0; + } + + if (!oracle) return 1; + + cudaStreamCaptureStatus capture = cudaStreamCaptureStatusNone; + const cudaError_t capture_err = cudaStreamIsCapturing(stream, &capture); + if (capture_err != cudaSuccess || + capture != cudaStreamCaptureStatusNone) { + (void)cudaGetLastError(); + g_q4_grouped_attn_a_oracle_skips++; + return 1; + } + + const uint64_t reference_bytes = low_elems * sizeof(float); + const uint64_t mismatch_offset = + (reference_bytes + 255u) & ~255ull; + if (mismatch_offset < reference_bytes || + mismatch_offset > UINT64_MAX - sizeof(uint32_t)) { + g_q4_grouped_attn_a_oracle_skips++; + return 1; + } + unsigned char *scratch = (unsigned char *)cuda_tmp_alloc_on( + logical_tier, mismatch_offset + sizeof(uint32_t), + "q4 grouped attention-A oracle"); + if (!scratch) { + g_q4_grouped_attn_a_oracle_skips++; + return 1; + } + float *reference = (float *)scratch; + uint32_t *mismatch_device = + (uint32_t *)(scratch + mismatch_offset); + + int reference_ok = 1; + for (uint32_t g = 0; g < group_cnt; g++) { + const int group_rc = ds4_mmq_q4_K_dense_vec( + out_a + (uint64_t)g * group_weight_bytes, + (const float *)heads->ptr + (uint64_t)g * group_dim, + reference + (uint64_t)g * rank, + (int)rank, 1, (int)group_dim, stream); + if (group_rc != 0) { + reference_ok = 0; + break; + } + } + if (!reference_ok || + !cuda_ok(cudaMemsetAsync(mismatch_device, 0, sizeof(uint32_t), + stream), + "clear q4 grouped attention-A oracle")) { + g_q4_grouped_attn_a_oracle_skips++; + return 1; + } + q4_K_attn_hc_bitwise_compare_kernel + <<<(low_elems + 255u) / 256u, 256, 0, stream>>>( + mismatch_device, reference, (const float *)low->ptr, low_elems); + if (!cuda_ok(cudaGetLastError(), + "q4 grouped attention-A oracle compare launch")) { + return 0; + } + + uint32_t mismatch_host = 0u; + if (!cuda_ok(cudaMemcpyAsync(&mismatch_host, mismatch_device, + sizeof(mismatch_host), + cudaMemcpyDeviceToHost, stream), + "read q4 grouped attention-A oracle") || + !cuda_ok(cudaStreamSynchronize(stream), + "synchronize q4 grouped attention-A oracle")) { + return 0; + } + g_q4_grouped_attn_a_oracle_calls++; + if (mismatch_host != 0u) { + g_q4_grouped_attn_a_oracle_mismatches++; + if (!g_q4_grouped_attn_a_oracle_mismatch_reported) { + g_q4_grouped_attn_a_oracle_mismatch_reported = 1; + fprintf(stderr, + "ds4: CUDA Q4 grouped attention-A oracle found a " + "bitwise mismatch; retaining per-group MMVQ output\n"); + } + if (!cuda_ok(cudaMemcpyAsync(low->ptr, reference, reference_bytes, + cudaMemcpyDeviceToDevice, stream), + "restore q4 grouped attention-A oracle reference")) { + return 0; + } + } + return 1; } extern "C" int ds4_gpu_hc_expand_split_half_tensor( diff --git a/ds4_gpu.h b/ds4_gpu.h index c2b7fb15a..6ad4d5168 100644 --- a/ds4_gpu.h +++ b/ds4_gpu.h @@ -2351,8 +2351,8 @@ int ds4_gpu_attention_output_q8_batch_tensor( const ds4_gpu_tensor *heads, uint32_t n_tokens); /* Returns 1 when the batch path ran, 0 for the ordinary row fallback, and -1 - * when the Metal REQUIRE diagnostic made an ineligible/failed tiny batch a - * hard error. */ + * for a required Metal path that is unavailable or a CUDA grouped launch + * that failed after enqueue and therefore cannot safely fall back. */ int ds4_gpu_attention_output_q4_K_batch_tensor( ds4_gpu_tensor *out, ds4_gpu_tensor *low, From 40fa5793009792920a1bf25c2cef7361638432b6 Mon Sep 17 00:00:00 2001 From: Giorgio Oppo Date: Mon, 10 Aug 2026 20:35:18 +0200 Subject: [PATCH 017/189] Optimize DSpark verification and GB10 GPU paths --- Makefile | 13 +- QA_BEFORE_RELEASES.md | 10 +- README.md | 65 ++- cuda/mmq/ds4_mmq.cu | 127 +++--- cuda/mmq/ds4_mmq.h | 24 ++ cuda/mmq/test/test_mmq_parity.cu | 105 +++-- ds4.c | 661 +++++++++++++++++++++++++---- ds4_cuda.cu | 334 +++++++++++++-- ds4_gpu.h | 90 ++-- ds4_help.c | 2 +- ds4_metal.m | 59 ++- metal/dsv4_hc.metal | 32 ++ tests/dspark_acceptance_fixture.sh | 64 +++ tests/test_engine_mgpu_placement.c | 129 ++++++ tests/test_metal_dspark_capture.c | 204 +++++++++ 15 files changed, 1652 insertions(+), 267 deletions(-) create mode 100644 tests/test_metal_dspark_capture.c diff --git a/Makefile b/Makefile index 4b8485d22..14aa79977 100644 --- a/Makefile +++ b/Makefile @@ -68,7 +68,7 @@ DS4_LINK_LIBS ?= $(CUDA_LDLIBS) METAL_LDLIBS := $(LDLIBS) endif -.PHONY: all help clean test test-rocm test-glm53-kda-rocm test-metal-session-batch test-metal-exactn-oracle test-mxfp4-cuda test-mxfp4-rocm test-mmq-parity-cuda test-cuda-session-batch test-cuda-mixed-batch dspark-acceptance dspark-verify-depth rocm-dspark-acceptance rocm-dspark-verify-depth mtp-verify-depth cpu cuda cuda-spark cuda-generic cuda-regression strix-halo rocm +.PHONY: all help clean test test-rocm test-glm53-kda-rocm test-metal-session-batch test-metal-exactn-oracle test-metal-dspark-capture test-mxfp4-cuda test-mxfp4-rocm test-mmq-parity-cuda test-cuda-session-batch test-cuda-mixed-batch dspark-acceptance dspark-verify-depth rocm-dspark-acceptance rocm-dspark-verify-depth mtp-verify-depth cpu cuda cuda-spark cuda-generic cuda-regression strix-halo rocm ifeq ($(UNAME_S),Darwin) .PHONY: metal-decode-schedule-bench metal-prefill-variant-bench check-mxfp4-half-lut test-mxfp4-metal @@ -80,6 +80,7 @@ help: @echo " make Build Metal ./ds4, ./ds4-server, ./ds4-bench, ./ds4-eval, and ./ds4-agent" @echo " make cpu Build CPU-only ./ds4, ./ds4-server, ./ds4-bench, ./ds4-eval, and ./ds4-agent" @echo " make test Build and run tests" + @echo " make test-metal-dspark-capture Check fused DSpark HC capture bitwise" @echo " make metal-decode-schedule-bench Build the balanced Metal decode schedule benchmark" @echo " make metal-prefill-variant-bench Build the balanced Metal prefill variant benchmark" @echo " make check-mxfp4-half-lut Verify the checked-in MXFP4 half LUT matches the generator" @@ -116,6 +117,14 @@ tests/test_metal_session_batch: tests/test_metal_session_batch.o $(CORE_OBJS) test-metal-session-batch: tests/test_metal_session_batch DS4_TEST_MODEL="$(DS4_TEST_MODEL)" ./tests/test_metal_session_batch +tests/test_metal_dspark_capture.o: tests/test_metal_dspark_capture.c ds4_gpu.h + $(CC) $(CFLAGS) -I. -c -o $@ $< + +tests/test_metal_dspark_capture: tests/test_metal_dspark_capture.o ds4_metal.o + $(CC) $(CFLAGS) -o $@ $^ $(METAL_LDLIBS) + +test-metal-dspark-capture: tests/test_metal_dspark_capture + ./tests/test_metal_dspark_capture speed-bench/metal_decode_schedule_bench.o: speed-bench/metal_decode_schedule_bench.c ds4.h $(CC) $(CFLAGS) -I. -c -o $@ $< @@ -653,4 +662,4 @@ mxfp4-dot-test: tests/test_mxfp4_dot.c ./tests/test_mxfp4_dot clean: - rm -f ds4 ds4-server ds4-bench ds4-eval ds4-agent ds4_cpu ds4_native ds4_server_test ds4_test ds4_agent_test gguf-tools/quality-testing/score_official gguf-tools/quality-testing/score_official.o speed-bench/metal_decode_schedule_bench speed-bench/metal_prefill_variant_bench speed-bench/*.o tests/test_q4k_dot tests/test_mxfp4_dot tests/test_mxfp4_metal tests/test_mxfp4_rocm tests/test_mxfp4_cuda tests/test_metal_session_batch tests/test_metal_exactn_oracle tests/test_glm53_kda tests/test_glm53_kda_rocm tests/test_glm53_vision_engine tests/test_glm53_vision_prompt tests/test_gpu_xdev tests/test_gpu_model_cache tests/test_gpu_lookup_cache_strict tests/test_engine_mgpu_refusal tests/test_engine_mgpu_runtime tests/test_engine_correctness tests/test_sampling tests/test_cuda_session_batch tests/test_cuda_mixed_batch tests/*.o *.o tests/cuda_long_context_smoke tests/cuda_long_context_smoke.o + rm -f ds4 ds4-server ds4-bench ds4-eval ds4-agent ds4_cpu ds4_native ds4_server_test ds4_test ds4_agent_test gguf-tools/quality-testing/score_official gguf-tools/quality-testing/score_official.o speed-bench/metal_decode_schedule_bench speed-bench/metal_prefill_variant_bench speed-bench/*.o tests/test_q4k_dot tests/test_mxfp4_dot tests/test_mxfp4_metal tests/test_mxfp4_rocm tests/test_mxfp4_cuda tests/test_metal_session_batch tests/test_metal_exactn_oracle tests/test_metal_dspark_capture tests/test_glm53_kda tests/test_glm53_kda_rocm tests/test_glm53_vision_engine tests/test_glm53_vision_prompt tests/test_gpu_xdev tests/test_gpu_model_cache tests/test_gpu_lookup_cache_strict tests/test_engine_mgpu_refusal tests/test_engine_mgpu_runtime tests/test_engine_correctness tests/test_sampling tests/test_cuda_session_batch tests/test_cuda_mixed_batch tests/*.o *.o tests/cuda_long_context_smoke tests/cuda_long_context_smoke.o diff --git a/QA_BEFORE_RELEASES.md b/QA_BEFORE_RELEASES.md index ee1ed4fe7..b1f171487 100644 --- a/QA_BEFORE_RELEASES.md +++ b/QA_BEFORE_RELEASES.md @@ -980,9 +980,13 @@ Do not use high-performance Hugging Face Xet mode while vLLM is resident. bit mismatches for persistent scratch, grouped attention-A, and the opt-in K1024 persistent kernel. For the model A/B, use `DS4_CUDA_NO_Q4_GB10_FAST=1` in the control and leave it unset in the - candidate. Run a separate candidate with - `DS4_CUDA_Q4_GROUPED_ATTN_A_ORACLE=1` and require `calls>0`, - `mismatches=0`, and `skips=0`. Benchmark the K1024 persistent kernel as a + candidate. Run a separate generic-verifier candidate with + `DS4_CUDA_DISABLE_DSPARK_EXACTN=1`, + `DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_BATCH=1`, + `DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_BATCH=1`, and + `DS4_CUDA_Q4_GROUPED_ATTN_A_ORACLE=1`; require `batch_candidates>0`, + `batch_calls>0`, `batch_mismatches=0`, and `batch_skips=0`, plus + byte-identical stdout. Benchmark the K1024 persistent kernel as a separate fail-closed arm with both `DS4_CUDA_ENABLE_Q4_K1024_PERSISTENT=1` and `DS4_CUDA_REQUIRE_Q4_K1024_PERSISTENT=1`; its rollback is diff --git a/README.md b/README.md index ed35bbb69..bf42c0924 100644 --- a/README.md +++ b/README.md @@ -436,6 +436,14 @@ five-row proposer for an A/B control, or set a positive value to cap it explicitly. Override the verifier policy independently with `DS4_DSPARK_SSD_VERIFY_BLOCK_MAX=N`. +Metal can experimentally mirror the final target-hidden prefill row from the +HC weighted-sum kernel itself, avoiding a separate 16 KiB blit and +compute/blit encoder transition on each captured target layer. Enable it with +`DS4_METAL_ENABLE_DSPARK_CAPTURE_FUSED_LAST=1`; the historical +weighted-sum-plus-blit sequence remains the default because short M1 Pro SSD +A/B runs were bit-identical but did not show a repeatable throughput win. +`DS4_METAL_DISABLE_DSPARK_CAPTURE_FUSED_LAST=1` is the dominant kill switch. + An experimental single-device Metal verifier can use the current target logits for the first draft and evaluate only the remaining `N-1` target rows. Enable it with `DS4_METAL_DSPARK_ACCEPTANCE_ONLY_VERIFY=1`; it remains opt-in @@ -697,11 +705,16 @@ On a single DGX Spark/GB10, the AProjQ4 path also mirrors the safe parts of the aligned-Q8 decode work while retaining canonical Q4_K MMVQ/Q8_1 arithmetic: -- dense and paired Q4 projections reuse the persistent 256-KiB Q8_1 scratch; +- dense and paired Q4 projections reuse the persistent 1-MiB Q8_1 scratch; `DS4_CUDA_NO_Q4_DENSE_SCRATCH=1` restores pool allocation; - attention-output A evaluates all output groups through one channel-grouped MMVQ dispatch per token, preserving the one-row reduction tree of every - group; `DS4_CUDA_NO_Q4_GROUPED_ATTN_A=1` restores the per-group loop; + group. For DSpark verification widths 2--8, + `DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_BATCH=1` flattens `(token, group)` into + MMVQ channels and replaces the per-token loop with one grouped MMVQ + dispatch while keeping `ncols_dst=1`; + `DS4_CUDA_NO_Q4_GROUPED_ATTN_A_BATCH=1` restores the per-token grouped loop + and `DS4_CUDA_NO_Q4_GROUPED_ATTN_A=1` restores the per-group loop; - attention-output B keeps its canonical MMVQ result and automatically uses the row-packed HC epilogue described below; - the exact Q-b shape `32768x1024` has an experimental persistent-CTA kernel @@ -712,10 +725,19 @@ arithmetic: `DS4_CUDA_NO_Q4_GB10_FAST=1` is the umbrella rollback for these new GB10 choices; it does not disable the older cross-CUDA Q-A/KV pair itself. For a -fail-closed grouped attention comparison, -`DS4_CUDA_Q4_GROUPED_ATTN_A_ORACLE=1` computes the established per-group -MMVQ reference, reports calls/mismatches/skips, and retains the reference on -any mismatch. The scratch, grouped, and persistent paths are also covered by +fail-closed grouped attention comparison, set +`DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_BATCH=1`, +`DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_BATCH=1`, and +`DS4_CUDA_Q4_GROUPED_ATTN_A_ORACLE=1`. The oracle computes the established per-group +MMVQ reference (or the established per-token grouped loop for a multi-token +candidate), reports aggregate calls/mismatches/skips plus separate +batch_candidates/batch_calls/batch_mismatches/batch_skips, and retains +canonical output. A valid multi-token test has nonzero candidates/calls and +zero batch mismatches/skips. The +oracle disables decode-graph capture. For a multi-token candidate, if it +encounters another active capture or cannot allocate comparison scratch, it +directly enqueues the canonical reference instead of consuming an unchecked +candidate. The scratch, grouped, and persistent paths are also covered by `make test-mmq-parity-cuda CUDA_ARCH=sm_121`. Two additional CUDA fusions remain experimental until a device oracle passes @@ -825,6 +847,24 @@ variable. Track `cuda_exactn_attempt`, `cuda_exactn_full`, this experiment disabled by default until a real CUDA oracle and a long greedy A/B show byte-identical output, no fallback errors, and a throughput win. +The layer tape can separately reuse its position-independent CUDA decode +islands with `DS4_CUDA_DSPARK_EXACTN_GRAPHS=1`. Graph keys use the stable +device address (including each batch-row offset), not the short-lived tensor +view wrapper. Four cache entries per layer/island remain reserved for ordinary +decode and five more are isolated for exact-N rows, so a width-five verifier +cannot evict the normal decode keys. The first encounter warms lazy allocators, +the second captures/instantiates, and only later encounters are pure replay; +benchmark at least 128--256 generated tokens rather than judging a short +capture-heavy run. `DS4_CUDA_DISABLE_DSPARK_EXACTN_GRAPHS=1` is the dedicated +kill switch, while `DS4_CUDA_DECODE_GRAPHS=0` still disables all decode graphs. +The stats fields `cuda_exactn_graph_attempt`, `..._use`, `..._warm`, +`..._capture`, `..._replay`, `..._no_slot`, and `..._failure` expose warmup, +reuse, capacity misses, and retired captures. This first rollout is for +serialized, single-session DGX testing; the graph cache and cuBLAS capture +state remain process-global. The fixture can require a clean post-warmup +replay (including zero no-slot/failure events) with +`DS4_DSPARK_FIXTURE_REQUIRE_CUDA_EXACTN_GRAPHS=1`. + For a separate output-head A/B, set `DS4_CUDA_DSPARK_EXACTN_BATCH_HEAD=1`. The experiment keeps HC collapse and normalization on the canonical one-row kernels, then runs the Q8 vocabulary @@ -966,7 +1006,18 @@ context, and backend working-set limit leave less room. A plain number such as `--ssd-streaming-cache-experts 4000` requests 4000 dynamic expert slots without the two-layer reserve, but it can be reduced by the same final memory check. Non-routed weights, KV cache, graph scratch, and activations need additional -memory. The automatic cache budget takes +memory. + +Metal SSD+DSpark also has an experimental, support-aware pre-cap for A/B tests. +Set `DS4_METAL_DSPARK_SAFE_EXPERT_COUNT=1` to convert a numeric count to bytes +and cap it, when measurable, after accounting for the target's non-routed +weights, the context/KV estimate, and a 2 GiB active reserve for the mmap-backed +support model. It does not yet price the complete batch-prefill workspace or a +separate routed-prefill transient reserve. Startup reports requested/effective +slots and the support reserve. If this policy cannot measure safe room, it +retains the explicit count with a warning; the normal final memory check remains +authoritative. The experiment does not affect `NGB` budgets or CUDA/ROCm. +Prefer an `NGB` budget for normal use. The automatic cache budget takes 80% of the backend's recommended working set, subtracts non-routed weights, then applies the same routed-prefill headroom before sizing the dynamic cache. Leave the hot expert preload enabled for normal use; use `--ssd-streaming-cold` and diff --git a/cuda/mmq/ds4_mmq.cu b/cuda/mmq/ds4_mmq.cu index 57d7f45da..6febcdec8 100644 --- a/cuda/mmq/ds4_mmq.cu +++ b/cuda/mmq/ds4_mmq.cu @@ -100,19 +100,13 @@ private: // Init // ---------------------------------------------------------------------------- -// Step 7 task #29: experimental persistent Q8_1 scratch buffer. -// -// Hypothesis: ggml_cuda_pool_alloc inside ds4_mmq_moe_vec_impl records a -// cudaMallocAsync graph node into the captured layer graph. At replay -// time the alloc node returns a (potentially different) address, but the -// matvec kernel's pointer argument was baked in at capture time. Result: -// the matvec reads stale/wrong memory and produces a different output -// than eager execution, even with identical inputs. -// -// Mitigation under test: pre-allocate a persistent device buffer at -// startup via plain cudaMalloc (NOT cudaMallocAsync, NOT inside any -// capture). When the env flag DS4_CUDA_MMQ_Q81_PERSISTENT=1 is set, -// ds4_mmq_moe_vec_impl uses this persistent buffer instead of pool_alloc. +// Step 7 task #29: experimental persistent Q8_1 scratch buffer. CUDA graph +// memory nodes already preserve the allocation's virtual address for the +// lifetime of the graph, so the ordinary same-stream pool path is correct. +// This startup allocation is instead an optimization experiment: it removes +// captured alloc/free nodes and their allocator/instantiation overhead. When +// DS4_CUDA_MMQ_Q81_PERSISTENT=1 is set, ds4_mmq_moe_vec_impl uses this buffer +// instead of pool_alloc. // // Sized for V4 Flash decode shapes: gate Q8_1 ~8 KB, down Q8_1 ~14 KB. // 256 KB allocation gives generous headroom for short prefill batches. @@ -351,9 +345,10 @@ extern "C" int ds4_mmq_init(int device) { } // Step 7 task #29: pre-allocate persistent Q8_1 scratch if enabled. - // Must happen here (before any layer-graph capture) so the cudaMalloc - // is not forbidden by capture-mode restrictions, and so the kernel - // pointer arg baked into the captured graph stays valid at replay. + // This happens before layer-graph capture to keep allocator nodes and + // their instantiation overhead out of the graph. The ordinary same-stream + // cudaMallocAsync path remains correct: CUDA graph memory nodes preserve + // their virtual address for the lifetime of the graph. if (getenv("DS4_CUDA_MMQ_Q81_PERSISTENT") && !g_q81_scratch_ptr) { const size_t bytes = 256 * 1024; cudaError_t err = cudaMalloc(&g_q81_scratch_ptr, bytes); @@ -2323,11 +2318,10 @@ int ds4_mmq_moe_vec_impl( const int64_t ne10_padded = GGML_PAD((int64_t)K, MATRIX_ROW_PADDING); const size_t nbytes_q8_1 = (size_t)n_tokens * ne10_padded * sizeof(block_q8_1) / QK8_1; - // Step 7 task #29: experimental persistent Q8_1 scratch. Avoids - // pool_alloc (cudaMallocAsync) graph nodes whose pointer baked at - // capture time may not match the address resolved at replay. When - // disabled (default) or when the persistent buffer is too small, - // fall back to the pool path. See ds4_mmq_init for setup. + // Step 7 task #29: experimental persistent Q8_1 scratch. It avoids + // captured pool alloc/free nodes as a performance experiment. When + // disabled (default) or too small, the valid same-stream graph-memory + // pool path remains the fallback. See ds4_mmq_init for setup. ggml_cuda_pool_alloc src1_q8_1_pool; char *src1_q8_1_ptr = nullptr; if (g_q81_scratch_enabled && g_q81_scratch_ptr && @@ -3472,26 +3466,31 @@ int ds4_mmq_dense_pair_vec_impl( return 0; } -__global__ static void ds4_mmq_identity_i32_kernel( - int32_t *ids, int n) { - const int i = (int)threadIdx.x; - if (i < n) ids[i] = i; +__global__ static void ds4_mmq_group_ids_i32_kernel( + int32_t *ids, int n, int n_groups) { + const int i = (int)(blockIdx.x * blockDim.x + threadIdx.x); + if (i < n) ids[i] = i % n_groups; } -/* Grouped AProjQ4 attention-A projection. Treat each attention group as an - * MMVQ channel (not as a column): ncols_dst stays one, so each channel uses - * exactly the same one-row Q4_K MMVQ specialization, K partition, peer-warp - * fold, and reduction tree as the canonical per-group loop. Only activation - * quantization and launch setup are shared across groups. */ -static int ds4_mmq_q4_K_grouped_vec_impl( +/* Grouped AProjQ4 attention-A projection. Flatten (token, group) into the + * MMVQ channel dimension (never the column dimension): ncols_dst stays one, + * so every pair uses exactly the same one-row Q4_K MMVQ specialization, K + * partition, peer-warp fold, and reduction tree as the canonical nested + * token/group loop. The repeated ids select W[group], while channel_y and + * channel_dst retain the token-major flat index. Only activation + * quantization and launch setup are shared. */ +static int ds4_mmq_q4_K_grouped_batch_vec_impl( const void *W, const float *X, float *out, int M, int K, + int n_tokens, int n_groups, cudaStream_t stream) { - const char *tag = "ds4_mmq_q4_K_grouped_vec"; + const char *tag = n_tokens == 1 + ? "ds4_mmq_q4_K_grouped_vec" + : "ds4_mmq_q4_K_grouped_batch_vec"; if (!W || !X || !out) { fprintf(stderr, "%s: null pointer\n", tag); return -1; @@ -3499,10 +3498,21 @@ static int ds4_mmq_q4_K_grouped_vec_impl( if (!g_gb10_optimizations || getenv("DS4_CUDA_NO_Q4_GB10_FAST") != nullptr || getenv("DS4_CUDA_NO_Q4_GROUPED_ATTN_A") != nullptr || - M <= 0 || K <= 0 || n_groups <= 0 || n_groups > 16 || + M <= 0 || K <= 0 || n_tokens <= 0 || n_tokens > 8 || + n_groups <= 0 || n_groups > 16 || K % 256 != 0) { return DS4_MMQ_NOT_APPLICABLE; } + if (n_tokens > 1) { + const char *enable = + getenv("DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_BATCH"); + if (!enable || !enable[0] || strcmp(enable, "0") == 0 || + getenv("DS4_CUDA_NO_Q4_GROUPED_ATTN_A_BATCH") != nullptr) { + return DS4_MMQ_NOT_APPLICABLE; + } + } + + const int flat_channels = n_tokens * n_groups; /* <= 8 * 16 */ const int64_t row_blocks = (int64_t)K / ggml_blck_size(GGML_TYPE_Q4_K); const int64_t weight_channel_stride = (int64_t)M * row_blocks; @@ -3512,11 +3522,15 @@ static int ds4_mmq_q4_K_grouped_vec_impl( } const int64_t ne10_padded = GGML_PAD((int64_t)K, MATRIX_ROW_PADDING); - const size_t nbytes_q8_1 = (size_t)n_groups * (size_t)ne10_padded * - sizeof(block_q8_1) / QK8_1; + const size_t q8_row_bytes = (size_t)ne10_padded * + sizeof(block_q8_1) / QK8_1; + if ((size_t)flat_channels > SIZE_MAX / q8_row_bytes) { + return DS4_MMQ_NOT_APPLICABLE; + } + const size_t nbytes_q8_1 = (size_t)flat_channels * q8_row_bytes; if (nbytes_q8_1 > SIZE_MAX - 15u) return DS4_MMQ_NOT_APPLICABLE; const size_t ids_offset = (nbytes_q8_1 + 15u) & ~(size_t)15u; - const size_t ids_bytes = (size_t)n_groups * sizeof(int32_t); + const size_t ids_bytes = (size_t)flat_channels * sizeof(int32_t); if (ids_offset > SIZE_MAX - ids_bytes) { return DS4_MMQ_NOT_APPLICABLE; } @@ -3526,10 +3540,12 @@ static int ds4_mmq_q4_K_grouped_vec_impl( dev, ids_offset + ids_bytes); if (!x8) return DS4_MMQ_NOT_APPLICABLE; int32_t *ids = (int32_t *)(x8 + ids_offset); - ds4_mmq_identity_i32_kernel<<<1, 32, 0, stream>>>(ids, n_groups); + ds4_mmq_group_ids_i32_kernel<<< + (unsigned)(flat_channels + 31) / 32u, 32, 0, stream>>>( + ids, flat_channels, n_groups); cudaError_t err = cudaGetLastError(); if (err != cudaSuccess) { - fprintf(stderr, "%s: identity launch failed: %s\n", + fprintf(stderr, "%s: group-id launch failed: %s\n", tag, cudaGetErrorString(err)); return -2; } @@ -3538,9 +3554,9 @@ static int ds4_mmq_q4_K_grouped_vec_impl( X, /*ids=*/nullptr, (void *)x8, GGML_TYPE_Q4_K, /*ne00=*/K, /*s11=*/(int64_t)K, - /*s12=*/(int64_t)K * n_groups, - /*s13=*/(int64_t)K * n_groups, - /*ne0=*/ne10_padded, /*ne1=*/n_groups, /*ne2=*/1, /*ne3=*/1, + /*s12=*/(int64_t)K * flat_channels, + /*s13=*/(int64_t)K * flat_channels, + /*ne0=*/ne10_padded, /*ne1=*/flat_channels, /*ne2=*/1, /*ne3=*/1, stream); err = cudaGetLastError(); if (err != cudaSuccess) { @@ -3551,8 +3567,13 @@ static int ds4_mmq_q4_K_grouped_vec_impl( const int64_t y_channel_stride = ne10_padded / QK8_1; ggml_cuda_mm_fusion_args_device fusion = {}; - cudaMemsetAsync(out, 0, - (size_t)n_groups * (size_t)M * sizeof(float), stream); + err = cudaMemsetAsync( + out, 0, (size_t)flat_channels * (size_t)M * sizeof(float), stream); + if (err != cudaSuccess) { + fprintf(stderr, "%s: output clear failed: %s\n", + tag, cudaGetErrorString(err)); + return -4; + } mul_mat_vec_q_switch_type( /*vx=*/W, /*type_x=*/GGML_TYPE_Q4_K, /*vy=*/(const void *)x8, @@ -3563,8 +3584,8 @@ static int ds4_mmq_q4_K_grouped_vec_impl( /*stride_col_y=*/(int)y_channel_stride, /*stride_col_dst=*/M, /*nchannels_x=*/n_groups, - /*nchannels_y=*/n_groups, - /*nchannels_dst=*/n_groups, + /*nchannels_y=*/flat_channels, + /*nchannels_dst=*/flat_channels, /*stride_channel_x=*/(int)weight_channel_stride, /*stride_channel_y=*/(int)y_channel_stride, /*stride_channel_dst=*/M, @@ -3576,10 +3597,11 @@ static int ds4_mmq_q4_K_grouped_vec_impl( if (err != cudaSuccess) { fprintf(stderr, "%s: grouped MMVQ launch failed: %s\n", tag, cudaGetErrorString(err)); - return -4; + return -5; } ds4_mmq_sanitize_f32( - out, (uint64_t)(uint32_t)n_groups * (uint64_t)(uint32_t)M, stream); + out, (uint64_t)(uint32_t)flat_channels * (uint64_t)(uint32_t)M, + stream); return 0; } @@ -5484,8 +5506,15 @@ extern "C" int ds4_mmq_q4_K_dense_vec( extern "C" int ds4_mmq_q4_K_grouped_vec( const void *W, const float *X, float *out, int M, int K, int n_groups, cudaStream_t stream) { - return ds4_mmq_q4_K_grouped_vec_impl( - W, X, out, M, K, n_groups, stream); + return ds4_mmq_q4_K_grouped_batch_vec_impl( + W, X, out, M, K, 1, n_groups, stream); +} + +extern "C" int ds4_mmq_q4_K_grouped_batch_vec( + const void *W, const float *X, float *out, + int M, int K, int n_tokens, int n_groups, cudaStream_t stream) { + return ds4_mmq_q4_K_grouped_batch_vec_impl( + W, X, out, M, K, n_tokens, n_groups, stream); } extern "C" int ds4_mmq_q4_K_dense_pair_vec( diff --git a/cuda/mmq/ds4_mmq.h b/cuda/mmq/ds4_mmq.h index 87437946a..5b838481d 100644 --- a/cuda/mmq/ds4_mmq.h +++ b/cuda/mmq/ds4_mmq.h @@ -907,6 +907,30 @@ int ds4_mmq_q4_K_grouped_vec( int n_groups, cudaStream_t stream); +// Token-aware form of the exact grouped Q4_K entry above. X and out are +// token-major: [n_tokens][n_groups][K] and +// [n_tokens][n_groups][M]. Internally (token, group) is flattened into the +// MMVQ channel dimension while ncols_dst remains one. This is deliberate: +// every pair therefore retains the canonical one-row Q8_1 quantization, +// Q4_K K partition, peer-warp fold and reduction tree. The GB10 path accepts +// at most eight tokens and is opt-in with +// DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_BATCH=1. Either +// DS4_CUDA_NO_Q4_GROUPED_ATTN_A_BATCH=1 or the existing grouped/global kill +// switches disables it. DS4_MMQ_NOT_APPLICABLE is returned before enqueue +// whenever a gate or scratch-capacity check fails. +// The graph-level diagnostic +// DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_BATCH=1 turns such ineligibility into a +// visible failure when this attention-output path is reached. +int ds4_mmq_q4_K_grouped_batch_vec( + const void * W_q4_K, + const float * X_f32, + float * out_f32, + int M, + int K, + int n_tokens, + int n_groups, + cudaStream_t stream); + // On a single GB10, the exact AProjQ4 Q-b shape (M=32768, N=1, K=1024) // can opt into a persistent-CTA form with // DS4_CUDA_ENABLE_Q4_K1024_PERSISTENT=1. The rollback switch diff --git a/cuda/mmq/test/test_mmq_parity.cu b/cuda/mmq/test/test_mmq_parity.cu index 05018ce37..98f946e5a 100644 --- a/cuda/mmq/test/test_mmq_parity.cu +++ b/cuda/mmq/test/test_mmq_parity.cu @@ -1212,17 +1212,17 @@ bool run_q4_K_dense_vec_gb10_parity( } bool run_q4_K_grouped_vec_parity( - int M, int K, int n_groups, uint32_t seed) { + int M, int N, int K, int n_groups, uint32_t seed) { fprintf(stderr, - "=== Q4_K/GROUPED_VEC M=%d K=%d groups=%d seed=%u ===\n", - M, K, n_groups, seed); + "=== Q4_K/GROUPED_VEC M=%d N=%d K=%d groups=%d seed=%u ===\n", + M, N, K, n_groups, seed); std::mt19937 rng(seed); std::normal_distribution nd(0.0f, 1.0f); const int blocks_per_row = K / QK_K_LOCAL; const size_t blocks_per_group = (size_t)M * blocks_per_row; std::vector W((size_t)n_groups * blocks_per_group); for (auto &blk : W) generate_random_block_q4_K(&blk, rng); - std::vector X((size_t)n_groups * K); + std::vector X((size_t)N * n_groups * K); for (float &v : X) v = nd(rng); cudaStream_t stream = nullptr; @@ -1231,12 +1231,15 @@ bool run_q4_K_grouped_vec_parity( float *dX = nullptr; float *dRef = nullptr; float *dGot = nullptr; + const size_t output_count = (size_t)N * n_groups * M; + /* Covers the N=8, G=16, K=4096 parity envelope with room for ids. */ + const size_t scratch_bytes = 1024u * 1024u; bool ok = cudaStreamCreate(&stream) == cudaSuccess && cudaMalloc(&dW, W.size() * sizeof(block_q4_K)) == cudaSuccess && cudaMalloc(&dX, X.size() * sizeof(float)) == cudaSuccess && - cudaMalloc(&dRef, (size_t)n_groups * M * sizeof(float)) == cudaSuccess && - cudaMalloc(&dGot, (size_t)n_groups * M * sizeof(float)) == cudaSuccess && - cudaMalloc(&scratch, 256u * 1024u) == cudaSuccess; + cudaMalloc(&dRef, output_count * sizeof(float)) == cudaSuccess && + cudaMalloc(&dGot, output_count * sizeof(float)) == cudaSuccess && + cudaMalloc(&scratch, scratch_bytes) == cudaSuccess; if (!ok) { fprintf(stderr, "Q4_K grouped vec parity allocation failed\n"); if (scratch) cudaFree(scratch); @@ -1253,27 +1256,64 @@ bool run_q4_K_grouped_vec_parity( cudaMemcpyHostToDevice, stream); unsetenv("DS4_CUDA_NO_Q4_GB10_FAST"); unsetenv("DS4_CUDA_NO_Q4_GROUPED_ATTN_A"); + unsetenv("DS4_CUDA_NO_Q4_GROUPED_ATTN_A_BATCH"); + unsetenv("DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_BATCH"); ds4_mmq_set_gb10_optimizations(1); - ds4_mmq_set_aligned_q81_scratch(scratch, 256u * 1024u); + ds4_mmq_set_aligned_q81_scratch(scratch, scratch_bytes); int rc_ref = 0; - for (int g = 0; g < n_groups && rc_ref == 0; g++) { - rc_ref = ds4_mmq_q4_K_dense_vec( - (const char *)dW + (size_t)g * blocks_per_group * - sizeof(block_q4_K), - dX + (size_t)g * K, - dRef + (size_t)g * M, - M, 1, K, stream); + for (int t = 0; t < N && rc_ref == 0; t++) { + for (int g = 0; g < n_groups && rc_ref == 0; g++) { + const size_t channel = (size_t)t * n_groups + g; + rc_ref = ds4_mmq_q4_K_dense_vec( + (const char *)dW + (size_t)g * blocks_per_group * + sizeof(block_q4_K), + dX + channel * K, + dRef + channel * M, + M, 1, K, stream); + } } - const int rc_got = ds4_mmq_q4_K_grouped_vec( - dW, dX, dGot, M, K, n_groups, stream); - setenv("DS4_CUDA_NO_Q4_GB10_FAST", "1", 1); - const int rc_disabled = ds4_mmq_q4_K_grouped_vec( - dW, dX, dGot, M, K, n_groups, stream); + + int rc_opt_out = DS4_MMQ_NOT_APPLICABLE; + int rc_short_scratch = DS4_MMQ_NOT_APPLICABLE; + if (N > 1) { + rc_opt_out = ds4_mmq_q4_K_grouped_batch_vec( + dW, dX, dGot, M, K, N, n_groups, stream); + setenv("DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_BATCH", "1", 1); + const size_t padded_k = ((size_t)K + 511u) & ~(size_t)511u; + const size_t q8_row_bytes = + padded_k * sizeof(block_q8_1) / QK8_1; + const size_t q8_bytes = (size_t)N * n_groups * q8_row_bytes; + const size_t ids_offset = (q8_bytes + 15u) & ~(size_t)15u; + const size_t required_bytes = + ids_offset + (size_t)N * n_groups * sizeof(int32_t); + if (required_bytes > 0u) { + ds4_mmq_set_aligned_q81_scratch(scratch, required_bytes - 1u); + rc_short_scratch = ds4_mmq_q4_K_grouped_batch_vec( + dW, dX, dGot, M, K, N, n_groups, stream); + ds4_mmq_set_aligned_q81_scratch(scratch, scratch_bytes); + } + } + const int rc_got = N == 1 + ? ds4_mmq_q4_K_grouped_vec( + dW, dX, dGot, M, K, n_groups, stream) + : ds4_mmq_q4_K_grouped_batch_vec( + dW, dX, dGot, M, K, N, n_groups, stream); + if (N == 1) { + setenv("DS4_CUDA_NO_Q4_GB10_FAST", "1", 1); + } else { + setenv("DS4_CUDA_NO_Q4_GROUPED_ATTN_A_BATCH", "1", 1); + } + const int rc_disabled = N == 1 + ? ds4_mmq_q4_K_grouped_vec( + dW, dX, dGot, M, K, n_groups, stream) + : ds4_mmq_q4_K_grouped_batch_vec( + dW, dX, dGot, M, K, N, n_groups, stream); unsetenv("DS4_CUDA_NO_Q4_GB10_FAST"); + unsetenv("DS4_CUDA_NO_Q4_GROUPED_ATTN_A_BATCH"); - std::vector ref((size_t)n_groups * M); - std::vector got((size_t)n_groups * M); + std::vector ref(output_count); + std::vector got(output_count); cudaMemcpyAsync(ref.data(), dRef, ref.size() * sizeof(float), cudaMemcpyDeviceToHost, stream); cudaMemcpyAsync(got.data(), dGot, got.size() * sizeof(float), @@ -1284,16 +1324,20 @@ bool run_q4_K_grouped_vec_parity( if (std::memcmp(&ref[i], &got[i], sizeof(float)) != 0) mismatches++; } ok = rc_ref == 0 && rc_got == 0 && + rc_opt_out == DS4_MMQ_NOT_APPLICABLE && + rc_short_scratch == DS4_MMQ_NOT_APPLICABLE && rc_disabled == DS4_MMQ_NOT_APPLICABLE && sync_err == cudaSuccess && mismatches == 0; fprintf(stderr, - "rc_ref=%d rc_grouped=%d rc_disabled=%d " + "rc_ref=%d rc_grouped=%d rc_opt_out=%d rc_short_scratch=%d " + "rc_disabled=%d " "mismatches=%zu sync=%s\n%s\n\n", - rc_ref, rc_got, rc_disabled, mismatches, + rc_ref, rc_got, rc_opt_out, rc_short_scratch, rc_disabled, mismatches, cudaGetErrorString(sync_err), ok ? "PASS" : "FAIL"); + unsetenv("DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_BATCH"); ds4_mmq_set_aligned_q81_scratch(nullptr, 0u); ds4_mmq_set_gb10_optimizations(0); cudaFree(scratch); @@ -1441,14 +1485,17 @@ int main(int argc, char ** argv) { /*M=*/1024, /*N=*/5, /*K=*/4096, 0xC4FE40, false); all_ok &= run_q4_K_dense_vec_gb10_parity( /*M=*/32768, /*N=*/1, /*K=*/1024, 0xC4FE41, true); + // Preserve coverage of the original one-token grouped ABI. + all_ok &= run_q4_K_grouped_vec_parity( + /*M=*/64, /*N=*/1, /*K=*/512, /*groups=*/4, 0xC4FE45); all_ok &= run_q4_K_grouped_vec_parity( - /*M=*/256, /*K=*/8192, /*groups=*/4, 0xC4FE42); - // DeepSeek-V4 Flash AProjQ4 attention-A production shape. + /*M=*/256, /*N=*/2, /*K=*/8192, /*groups=*/4, 0xC4FE42); + // DeepSeek-V4 Flash AProjQ4 attention-A production shape at DSpark N=5. all_ok &= run_q4_K_grouped_vec_parity( - /*M=*/128, /*K=*/4096, /*groups=*/8, 0xC4FE43); - // Pro-style maximum group count accepted by the grouped entry. + /*M=*/128, /*N=*/5, /*K=*/4096, /*groups=*/8, 0xC4FE43); + // Pro-style maximum group count and the N=8 API ceiling. all_ok &= run_q4_K_grouped_vec_parity( - /*M=*/64, /*K=*/4096, /*groups=*/16, 0xC4FE44); + /*M=*/64, /*N=*/8, /*K=*/4096, /*groups=*/16, 0xC4FE44); fprintf(stderr, "===================\n"); fprintf(stderr, "%s\n", all_ok ? "ALL PASS" : "SOME FAILED"); diff --git a/ds4.c b/ds4.c index 6cd8f88cb..34d3e578b 100644 --- a/ds4.c +++ b/ds4.c @@ -4825,7 +4825,9 @@ static bool ds4_streaming_manual_cache_safe_bytes( * non-routed weights and an active support model. Failing to include those * fixed weights lets a seemingly safe expert cache evict the dense working * set or OOM while loading DSpark. Keep the total below the backend's - * recommended working set after graph and fixed-model accounting. + * recommended working set after the context/KV estimate and fixed-model + * accounting. This estimate deliberately does not claim the complete + * per-tier batch-prefill workspace or routed-prefill transient reserve. * * If less than the minimum usable cache remains, the normal planner emits * its existing "budget too small" error instead of silently exceeding the @@ -4847,6 +4849,90 @@ static bool ds4_streaming_manual_cache_safe_bytes( #endif } +#define DS4_METAL_DSPARK_SAFE_EXPERT_COUNT_ENV \ + "DS4_METAL_DSPARK_SAFE_EXPERT_COUNT" + +/* + * A plain --ssd-streaming-cache-experts N value names dynamic cache slots, + * unlike an NGB value. Convert it to bytes before comparing it with the same + * working-set limit used by manual NGB budgets, then convert the safe result + * back to whole slots. Keeping this arithmetic in one checked helper avoids + * wrapping a large count into a deceptively small byte request. + */ +static bool ds4_streaming_manual_cache_cap_count( + uint32_t requested_count, + uint64_t per_expert_bytes, + uint64_t safe_cache_bytes, + uint32_t *effective_count_out, + uint64_t *requested_bytes_out, + uint64_t *effective_bytes_out) { + if (effective_count_out) *effective_count_out = 0; + if (requested_bytes_out) *requested_bytes_out = 0; + if (effective_bytes_out) *effective_bytes_out = 0; + if (requested_count == 0 || per_expert_bytes == 0 || + !effective_count_out || !requested_bytes_out || + !effective_bytes_out) { + return false; + } + if ((uint64_t)requested_count > UINT64_MAX / per_expert_bytes) { + return false; + } + + const uint64_t requested_bytes = + (uint64_t)requested_count * per_expert_bytes; + uint64_t safe_count = safe_cache_bytes / per_expert_bytes; + if (safe_count > UINT32_MAX) safe_count = UINT32_MAX; + const uint32_t effective_count = + safe_count < requested_count ? (uint32_t)safe_count : requested_count; + /* effective_count <= requested_count, whose product was checked above. */ + const uint64_t effective_bytes = + (uint64_t)effective_count * per_expert_bytes; + + *effective_count_out = effective_count; + *requested_bytes_out = requested_bytes; + *effective_bytes_out = effective_bytes; + return true; +} + +static bool ds4_streaming_manual_cache_count_cap_enabled(void) { + const char *env = getenv(DS4_METAL_DSPARK_SAFE_EXPERT_COUNT_ENV); + /* Explicit opt-in while the policy is being validated on real hardware. */ + return env && strcmp(env, "1") == 0; +} + +static bool ds4_streaming_manual_cache_count_cap_eligible( + ds4_backend backend, + bool ssd_streaming, + bool dspark_enabled, + ds4_support_kind support_kind, + uint32_t cache_experts, + uint64_t cache_bytes) { + return backend == DS4_BACKEND_METAL && + ssd_streaming && + dspark_enabled && + support_kind == DS4_SUPPORT_DSPARK && + cache_experts != 0 && + cache_bytes == 0; +} + +static uint64_t ds4_streaming_dspark_active_support_reserve_bytes( + uint64_t support_model_bytes) { + const uint64_t max_active_support_reserve = + 2ull * 1024ull * 1024ull * 1024ull; + return support_model_bytes < max_active_support_reserve ? + support_model_bytes : max_active_support_reserve; +} + +/* Unknown or sub-slot room must not make an existing explicit-count command + * fail. A measured non-zero candidate may only reduce the requested count. */ +static uint32_t ds4_streaming_manual_cache_nonfatal_effective_count( + uint32_t requested_count, + bool safe_cache_known, + uint32_t candidate_count) { + if (!safe_cache_known || candidate_count == 0) return requested_count; + return candidate_count < requested_count ? candidate_count : requested_count; +} + static uint64_t ds4_add_sat_u64(uint64_t a, uint64_t b) { return a > UINT64_MAX - b ? UINT64_MAX : a + b; } @@ -16007,12 +16093,14 @@ typedef struct { * override. The routed tails consume the GPU-selected matrix through the * exact-row union scope instead. */ bool spec_exactn_union_collect_routes; - /* Exact decode tapes bind short-lived row views as cur/after_ffn. CUDA - * decode-graph keys identify those wrapper handles, so caching an island - * across tape invocations could replay a graph after the handle address - * has been recycled for a different row. Keep graph capture/replay off - * only while such a tape is active. */ + /* Emergency/tape-local CUDA graph gate. Graph keys now use stable device + * storage identities rather than short-lived wrapper handles, so the + * resident exact-N tape may opt back in. Other experimental tapes remain + * eager until they receive the same lifetime and oracle coverage. */ bool spec_disable_decode_graphs; + /* Zero is ordinary decode. Exact-N assigns a stable domain-tagged row id + * while encoding each speculative row, then restores this field. */ + uint32_t decode_graph_variant; /* Batch verification normally takes the per-row compressor path when it * captures intermediate frontiers. Rollback+replay DSpark verification * needs the same arithmetic path, but not the frontier copies themselves. */ @@ -22457,7 +22545,6 @@ static DS4_MAYBE_UNUSED bool metal_graph_selected_async_load_start_tensor( job->event_value = event_value; job->gate_expert_bytes = gate_expert_bytes; job->down_expert_bytes = down_expert_bytes; - pthread_mutex_lock(&g_metal_graph_selected_async_load_mutex); if (g_metal_graph_selected_async_load_has_job || g_metal_graph_selected_async_load_done) { @@ -22983,6 +23070,10 @@ static bool metal_graph_encode_decode_layer_phase( * eagerly, byte-identical to before. A replayed graph re-runs the * captured kernels byte-for-byte, so replay output is bit-identical * to the eager encode it recorded. */ + /* Equivalent tensor views are frequently recreated by speculative tapes. + * The CUDA graph bakes the device address, not the host-side wrapper, into + * its nodes; key the cache by that same stable address. Metal/ROCm/CPU do + * not execute these graphs and retain the historical handle identity. */ const bool decode_graphs_common_ok = !g->placement && g->tp_world <= 1 && @@ -23007,10 +23098,20 @@ static bool metal_graph_encode_decode_layer_phase( memset(&isla_key, 0, sizeof(isla_key)); isla_key.il = il; isla_key.island = 0u; - isla_key.cur_hc = (void *)metal_graph_cur_hc(g); - isla_key.after_attn_hc = (void *)metal_graph_after_attn_hc(g); - isla_key.after_ffn_hc = (void *)metal_graph_after_ffn_hc(g); - isla_key.attn_norm = (void *)metal_graph_attn_norm(g); + isla_key.variant = g->decode_graph_variant; +#if !defined(DS4_NO_GPU) && !defined(__APPLE__) && !defined(DS4_ROCM_BUILD) +#define DS4_DECODE_GRAPH_TENSOR_KEY(t) \ + ((void *)(uintptr_t)ds4_gpu_tensor_storage_key((t))) +#else +#define DS4_DECODE_GRAPH_TENSOR_KEY(t) ((void *)(t)) +#endif + isla_key.cur_hc = DS4_DECODE_GRAPH_TENSOR_KEY(metal_graph_cur_hc(g)); + isla_key.after_attn_hc = + DS4_DECODE_GRAPH_TENSOR_KEY(metal_graph_after_attn_hc(g)); + isla_key.after_ffn_hc = + DS4_DECODE_GRAPH_TENSOR_KEY(metal_graph_after_ffn_hc(g)); + isla_key.attn_norm = + DS4_DECODE_GRAPH_TENSOR_KEY(metal_graph_attn_norm(g)); for (;;) { const int isla_state = ds4_gpu_decode_graph_begin(&isla_key); if (isla_state == 1) { @@ -24640,10 +24741,14 @@ static bool metal_graph_encode_decode_layer_phase( memset(&islb_key, 0, sizeof(islb_key)); islb_key.il = il; islb_key.island = 1u; - islb_key.cur_hc = (void *)metal_graph_cur_hc(g); - islb_key.after_attn_hc = (void *)metal_graph_after_attn_hc(g); - islb_key.after_ffn_hc = (void *)metal_graph_after_ffn_hc(g); - islb_key.attn_norm = (void *)metal_graph_attn_norm(g); + islb_key.variant = g->decode_graph_variant; + islb_key.cur_hc = DS4_DECODE_GRAPH_TENSOR_KEY(metal_graph_cur_hc(g)); + islb_key.after_attn_hc = + DS4_DECODE_GRAPH_TENSOR_KEY(metal_graph_after_attn_hc(g)); + islb_key.after_ffn_hc = + DS4_DECODE_GRAPH_TENSOR_KEY(metal_graph_after_ffn_hc(g)); + islb_key.attn_norm = + DS4_DECODE_GRAPH_TENSOR_KEY(metal_graph_attn_norm(g)); for (;;) { const int islb_state = ds4_gpu_decode_graph_begin(&islb_key); if (islb_state == 1) return ok; @@ -24669,6 +24774,7 @@ static bool metal_graph_encode_decode_layer_phase( cuda_tp_home_tier, cuda_tp_partner_tier, n_groups); ok = false; } +#undef DS4_DECODE_GRAPH_TENSOR_KEY if (ok && cuda_tp_attn) { const uint32_t tp_groups = n_groups / 2u; const uint64_t tp_heads_bytes = (uint64_t)tp_groups * group_dim * sizeof(float); @@ -28561,6 +28667,51 @@ static bool metal_graph_dspark_capture_decode_layer( return metal_graph_dspark_capture_hc(g, metal_graph_cur_hc(g), (uint32_t)slot); } +/* A DSpark prefill capture needs the complete row matrix for stage-0 and the + * final row for the next decode proposal. On Metal, mirror the last value + * from the weighted-sum kernel itself: the fallback below would otherwise + * force a compute-encoder close plus a separate blit encoder for 16 KiB. + * Keep the historical sequence selectable as a correctness/perf kill switch. */ +static bool metal_graph_dspark_capture_rows_and_last( + ds4_gpu_tensor *batch_dst, + ds4_gpu_tensor *last_dst, + const ds4_gpu_tensor *hc, + const ds4_gpu_tensor *weights, + uint32_t n_tokens) { + if (!batch_dst || !last_dst || !hc || !weights || n_tokens == 0) { + return false; + } +#ifdef __APPLE__ + const bool fused_last_enabled = + metal_graph_tp_env_flag( + "DS4_METAL_ENABLE_DSPARK_CAPTURE_FUSED_LAST", false) && + !metal_graph_tp_env_flag( + "DS4_METAL_DISABLE_DSPARK_CAPTURE_FUSED_LAST", false); + if (fused_last_enabled) { + return ds4_gpu_hc_weighted_sum_capture_last_tensor(batch_dst, + last_dst, + hc, + weights, + DS4_N_EMBD, + DS4_N_HC) != 0; + } +#endif + const uint64_t embd_bytes = (uint64_t)DS4_N_EMBD * sizeof(float); + ds4_gpu_tensor *last_src = + ds4_gpu_tensor_view(batch_dst, + (uint64_t)(n_tokens - 1u) * embd_bytes, + embd_bytes); + const bool ok = last_src && + ds4_gpu_hc_weighted_sum_tensor(batch_dst, + hc, + weights, + DS4_N_EMBD, + DS4_N_HC) != 0 && + ds4_gpu_tensor_copy(last_dst, 0, last_src, 0, embd_bytes) != 0; + ds4_gpu_tensor_free(last_src); + return ok; +} + static bool metal_graph_dspark_capture_prefill_layer( ds4_gpu_graph *g, uint32_t il, @@ -28580,28 +28731,17 @@ static bool metal_graph_dspark_capture_prefill_layer( ((uint64_t)slot * g->prefill_cap * DS4_N_EMBD) * sizeof(float), (uint64_t)n_tokens * embd_bytes); - ds4_gpu_tensor *last_src = - batch_dst ? - ds4_gpu_tensor_view(batch_dst, - (uint64_t)(n_tokens - 1u) * embd_bytes, - embd_bytes) : NULL; ds4_gpu_tensor *last_dst = ds4_gpu_tensor_view(g->dspark_target_hidden, (uint64_t)slot * embd_bytes, embd_bytes); - bool ok = batch_dst && last_src && last_dst && - ds4_gpu_hc_weighted_sum_tensor(batch_dst, - metal_graph_batch_cur_hc(g), - g->dspark_hc_mean_rows, - DS4_N_EMBD, - DS4_N_HC) != 0 && - ds4_gpu_tensor_copy(last_dst, - 0, - last_src, - 0, - embd_bytes) != 0; + bool ok = metal_graph_dspark_capture_rows_and_last( + batch_dst, + last_dst, + metal_graph_batch_cur_hc(g), + g->dspark_hc_mean_rows, + n_tokens); ds4_gpu_tensor_free(last_dst); - ds4_gpu_tensor_free(last_src); ds4_gpu_tensor_free(batch_dst); if (ok) { metal_graph_dspark_capture_note_slot(g, (uint32_t)slot); @@ -28650,35 +28790,34 @@ static bool metal_graph_dspark_capture_prefill_rows( (((uint64_t)(uint32_t)slot * g->prefill_cap + row0) * DS4_N_EMBD) * sizeof(float), (uint64_t)n_tokens * embd_bytes); - bool ok = batch_dst && - ds4_gpu_hc_weighted_sum_tensor(batch_dst, - metal_graph_batch_cur_hc(g), - g->dspark_hc_mean_rows, - DS4_N_EMBD, - DS4_N_HC) != 0; + const bool reaches_last = row0 + n_tokens == chunk_len; + ds4_gpu_tensor *last_dst = reaches_last ? + ds4_gpu_tensor_view(g->dspark_target_hidden, + (uint64_t)(uint32_t)slot * embd_bytes, + embd_bytes) : NULL; + bool ok = reaches_last ? + metal_graph_dspark_capture_rows_and_last( + batch_dst, + last_dst, + metal_graph_batch_cur_hc(g), + g->dspark_hc_mean_rows, + n_tokens) : + batch_dst && + ds4_gpu_hc_weighted_sum_tensor(batch_dst, + metal_graph_batch_cur_hc(g), + g->dspark_hc_mean_rows, + DS4_N_EMBD, + DS4_N_HC) != 0; if (!ok) fprintf(stderr, "ds4: pipeline capture rows FAIL il=%u row0=%u n=%u dst=%d\n", il, row0, n_tokens, batch_dst != NULL); - if (ok && row0 + n_tokens == chunk_len) { - ds4_gpu_tensor *last_src = - ds4_gpu_tensor_view(batch_dst, - (uint64_t)(n_tokens - 1u) * embd_bytes, - embd_bytes); - ds4_gpu_tensor *last_dst = - ds4_gpu_tensor_view(g->dspark_target_hidden, - (uint64_t)(uint32_t)slot * embd_bytes, - embd_bytes); - ok = last_src && last_dst && - ds4_gpu_tensor_copy(last_dst, 0, last_src, 0, embd_bytes) != 0; - ds4_gpu_tensor_free(last_dst); - ds4_gpu_tensor_free(last_src); - if (ok) { - metal_graph_dspark_capture_note_slot(g, (uint32_t)slot); - ok = metal_graph_dspark_capture_batch_note_slot(g, - (uint32_t)slot, - chunk_start, - chunk_len); - } + if (ok && reaches_last) { + metal_graph_dspark_capture_note_slot(g, (uint32_t)slot); + ok = metal_graph_dspark_capture_batch_note_slot(g, + (uint32_t)slot, + chunk_start, + chunk_len); } + ds4_gpu_tensor_free(last_dst); ds4_gpu_tensor_free(batch_dst); return ok; } @@ -28753,28 +28892,17 @@ static bool metal_graph_dspark_capture_verified_suffix_layer( (((uint64_t)(uint32_t)slot * g->prefill_cap + 1u) * DS4_N_EMBD) * sizeof(float), (uint64_t)n_tokens * embd_bytes); - ds4_gpu_tensor *last_src = - batch_dst ? - ds4_gpu_tensor_view(batch_dst, - (uint64_t)(n_tokens - 1u) * embd_bytes, - embd_bytes) : NULL; ds4_gpu_tensor *last_dst = ds4_gpu_tensor_view(g->dspark_target_hidden, (uint64_t)(uint32_t)slot * embd_bytes, embd_bytes); - bool ok = batch_dst && last_src && last_dst && - ds4_gpu_hc_weighted_sum_tensor(batch_dst, - metal_graph_batch_cur_hc(g), - g->dspark_hc_mean_rows, - DS4_N_EMBD, - DS4_N_HC) != 0 && - ds4_gpu_tensor_copy(last_dst, - 0, - last_src, - 0, - embd_bytes) != 0; + bool ok = metal_graph_dspark_capture_rows_and_last( + batch_dst, + last_dst, + metal_graph_batch_cur_hc(g), + g->dspark_hc_mean_rows, + n_tokens); ds4_gpu_tensor_free(last_dst); - ds4_gpu_tensor_free(last_src); ds4_gpu_tensor_free(batch_dst); if (ok) { metal_graph_dspark_capture_note_slot(g, (uint32_t)slot); @@ -33848,7 +33976,7 @@ static bool metal_graph_eval_dspark_stage_block( metal_graph_batch_qr_norm(g), draft); bool q_norm_rope_fused = false; -#if !defined(__APPLE__) && !defined(DS4_ROCM_BUILD) +#if !defined(DS4_NO_GPU) && !defined(__APPLE__) && !defined(DS4_ROCM_BUILD) /* DSpark's tiny CUDA batches otherwise launch separate head-normalization * and RoPE kernels at every support stage. The fused kernel is already * used by the target graph and preserves the same operation order within @@ -34473,7 +34601,7 @@ static bool metal_graph_eval_dspark_base_logits_from_hidden( DS4_N_VOCAB * sizeof(float)); bool ok = output_norm && logits; if (ok) ok = ds4_gpu_begin_commands() != 0; -#if !defined(__APPLE__) && !defined(DS4_ROCM_BUILD) +#if !defined(DS4_NO_GPU) && !defined(__APPLE__) && !defined(DS4_ROCM_BUILD) /* The exact CUDA Q8 tensor-core vocab kernel starts at eight rows. DSpark * produces five, while both backing workspaces have spare rows. Reuse the * verifier's zero-padding helper so the five meaningful rows take the MMA @@ -37675,11 +37803,41 @@ typedef struct { double layer_ms; double head_ms; double read_ms; + uint64_t graph_captures; + uint64_t graph_replays; + uint64_t graph_warms; + uint64_t graph_no_slots; + uint64_t graph_failures; + bool graphs_attempted; + bool graphs_used; bool batch_head_attempted; bool batch_head_used; bool batch_head_fallback; } ds4_cuda_exactn_timing; +/* CUDA Graphs are an independent exact-N experiment. Stable storage-address + * keys make recreated row views safe, while the global disable remains the + * emergency rollback for graph capture itself. */ +static bool metal_graph_cuda_exactn_graphs_requested( + const ds4_gpu_graph *g, + uint32_t n_tokens) { +#if !defined(DS4_NO_GPU) && !defined(__APPLE__) && !defined(DS4_ROCM_BUILD) + return g && n_tokens >= 2u && + n_tokens <= DS4_CUDA_EXACTN_MAX_ROWS && + g->placement == NULL && g->active_tier == 0 && + g->tp_world <= 1u && !g->ssd_streaming && + metal_graph_tp_env_flag( + "DS4_CUDA_DSPARK_EXACTN_GRAPHS", false) && + !metal_graph_tp_env_flag( + "DS4_CUDA_DISABLE_DSPARK_EXACTN_GRAPHS", false) && + ds4_gpu_decode_graphs_supported() != 0; +#else + (void)g; + (void)n_tokens; + return false; +#endif +} + /* Keep the exact-N head experiment narrower than the resident verifier gate: * only Q8 output weights have a multi-row CUDA entry point that explicitly * preserves the ordinary one-row reduction order. The disable variable wins @@ -37834,16 +37992,53 @@ static bool metal_graph_verify_decode_exactn_cuda_resident_impl( const bool saved_capture = g->spec_capture_prefixes; const bool saved_disable_decode_graphs = g->spec_disable_decode_graphs; + const uint32_t saved_decode_graph_variant = + g->decode_graph_variant; + const bool try_exactn_graphs = + metal_graph_cuda_exactn_graphs_requested(g, n_tokens); +#if !defined(DS4_NO_GPU) && !defined(__APPLE__) && !defined(DS4_ROCM_BUILD) + uint64_t graph_captures_before = 0; + uint64_t graph_replays_before = 0; + uint64_t graph_warms_before = 0; + uint64_t graph_no_slots_before = 0; + uint64_t graph_failures_before = 0; + if (try_exactn_graphs) { + ds4_gpu_decode_graph_counters(&graph_captures_before, + &graph_replays_before, + &graph_warms_before, + &graph_no_slots_before, + &graph_failures_before); + } +#endif + if (timing) timing->graphs_attempted = try_exactn_graphs; bool commands_open = false; bool ok = metal_graph_set_active_tier_no_copy(g, 0); + /* The generic batch path may swap these two scratch owners. Exact-N + * graph keys have five row slots, so canonicalize their backing order + * before creating views; otherwise an intervening odd-layer batch could + * invert every address and permanently exhaust the disjoint cache with a + * second set of equivalent keys. Both buffers are scratch and every cur + * row is overwritten by the embedding below. */ + ds4_gpu_tensor *exactn_cur_base = g->batch_cur_hc_by_tier[0]; + ds4_gpu_tensor *exactn_next_base = g->batch_next_hc_by_tier[0]; +#if !defined(DS4_NO_GPU) && !defined(__APPLE__) && !defined(DS4_ROCM_BUILD) + if (try_exactn_graphs && + ds4_gpu_tensor_storage_key(exactn_next_base) < + ds4_gpu_tensor_storage_key(exactn_cur_base)) { + ds4_gpu_tensor *tmp = exactn_cur_base; + exactn_cur_base = exactn_next_base; + exactn_next_base = tmp; + } +#endif + for (uint32_t row = 0; ok && row < n_tokens; row++) { cur_rows[row] = ds4_gpu_tensor_view( - g->batch_cur_hc_by_tier[0], + exactn_cur_base, (uint64_t)row * hc_dim * sizeof(float), hc_dim * sizeof(float)); next_rows[row] = ds4_gpu_tensor_view( - g->batch_next_hc_by_tier[0], + exactn_next_base, (uint64_t)row * hc_dim * sizeof(float), hc_dim * sizeof(float)); ok = cur_rows[row] && next_rows[row]; @@ -37861,7 +38056,8 @@ static bool metal_graph_verify_decode_exactn_cuda_resident_impl( } g->spec_capture_prefixes = false; - g->spec_disable_decode_graphs = true; + g->spec_disable_decode_graphs = + saved_disable_decode_graphs || !try_exactn_graphs; if (timing) timing->setup_ms += (now_sec() - setup_t0) * 1000.0; const double layer_t0 = timing ? now_sec() : 0.0; if (ok) { @@ -37871,6 +38067,8 @@ static bool metal_graph_verify_decode_exactn_cuda_resident_impl( for (uint32_t il = 0; ok && il < DS4_N_LAYER; il++) { for (uint32_t row = 0; ok && row < n_tokens; row++) { const uint32_t pos = start + row; + g->decode_graph_variant = + DS4_DECODE_GRAPH_VARIANT_EXACTN | (row + 1u); g->cur_hc_by_tier[0] = cur_rows[row]; g->after_ffn_hc_by_tier[0] = next_rows[row]; ok = metal_graph_encode_decode_layer( @@ -37893,6 +38091,7 @@ static bool metal_graph_verify_decode_exactn_cuda_resident_impl( } } } + g->decode_graph_variant = saved_decode_graph_variant; if (commands_open) { if (ok) { ok = ds4_gpu_end_commands() != 0; @@ -37901,6 +38100,37 @@ static bool metal_graph_verify_decode_exactn_cuda_resident_impl( } commands_open = false; } +#if !defined(DS4_NO_GPU) && !defined(__APPLE__) && !defined(DS4_ROCM_BUILD) + if (try_exactn_graphs && timing) { + uint64_t graph_captures_after = 0; + uint64_t graph_replays_after = 0; + uint64_t graph_warms_after = 0; + uint64_t graph_no_slots_after = 0; + uint64_t graph_failures_after = 0; + ds4_gpu_decode_graph_counters(&graph_captures_after, + &graph_replays_after, + &graph_warms_after, + &graph_no_slots_after, + &graph_failures_after); + timing->graph_captures = + graph_captures_after >= graph_captures_before ? + graph_captures_after - graph_captures_before : 0; + timing->graph_replays = + graph_replays_after >= graph_replays_before ? + graph_replays_after - graph_replays_before : 0; + timing->graph_warms = + graph_warms_after >= graph_warms_before ? + graph_warms_after - graph_warms_before : 0; + timing->graph_no_slots = + graph_no_slots_after >= graph_no_slots_before ? + graph_no_slots_after - graph_no_slots_before : 0; + timing->graph_failures = + graph_failures_after >= graph_failures_before ? + graph_failures_after - graph_failures_before : 0; + timing->graphs_used = + timing->graph_captures != 0 || timing->graph_replays != 0; + } +#endif if (timing) timing->layer_ms += (now_sec() - layer_t0) * 1000.0; const double head_t0 = timing ? now_sec() : 0.0; @@ -38028,6 +38258,7 @@ static bool metal_graph_verify_decode_exactn_cuda_resident_impl( g->spec_capture_prefixes = saved_capture; g->spec_disable_decode_graphs = saved_disable_decode_graphs; + g->decode_graph_variant = saved_decode_graph_variant; g->cur_hc_by_tier[0] = saved_cur; g->after_ffn_hc_by_tier[0] = saved_after; g->logits_by_tier[0] = saved_logits; @@ -40021,6 +40252,154 @@ static uint64_t ds4_engine_support_model_bytes(const ds4_engine *e) { return e->mtp_model.size - e->mtp_model.tensor_data_pos; } +/* + * Keep the long-standing exact-count behavior unless the experimental policy + * is explicitly selected. Metal SSD+DSpark is the problematic combination: + * target dense weights, the pageable support GGUF, graph buffers, and a + * numeric expert cache all compete for one unified-memory working set. An + * NGB input is already handled by the manual-byte-budget path below. + * + * The support GGUF is mmap-backed and its three stages execute sequentially; + * charging all 5+ GiB as resident makes every cache size impossible on a + * 16-GiB Mac. Reserve at most 2 GiB as its active pageable working set until + * hardware A/B data justifies a dynamic per-stage estimate. This policy must + * never turn a previously valid explicit-count command into a startup error. + */ +static bool ds4_engine_cap_metal_dspark_manual_cache_count( + ds4_engine *e, + int ctx_size) { + if (!e || + !ds4_streaming_manual_cache_count_cap_eligible( + e->backend, + e->ssd_streaming, + e->dspark, + e->support_kind, + e->ssd_streaming_cache_experts, + e->ssd_streaming_cache_bytes)) { + return true; + } + if (!ds4_streaming_manual_cache_count_cap_enabled()) return true; + + const uint32_t requested_count = e->ssd_streaming_cache_experts; + uint64_t per_expert_bytes = 0; + if (!ds4_streaming_routed_expert_bytes(&e->weights, + &per_expert_bytes) || + per_expert_bytes == 0) { + fprintf(stderr, + "ds4: WARNING: Metal SSD+DSpark safe numeric-cache policy " + "could not measure routed expert size; keeping requested=%u " + "effective=%u slots\n", + requested_count, + requested_count); + return true; + } + + uint64_t non_routed_bytes = 0; + if (!weights_streaming_non_routed_bytes(&e->weights, + &non_routed_bytes)) { + fprintf(stderr, + "ds4: WARNING: Metal SSD+DSpark safe numeric-cache policy " + "could not measure non-routed weights; keeping requested=%u " + "effective=%u slots\n", + requested_count, + requested_count); + return true; + } + const uint64_t support_model_bytes = + ds4_engine_support_model_bytes(e); + const uint64_t active_support_reserve = + ds4_streaming_dspark_active_support_reserve_bytes( + support_model_bytes); + const uint64_t fixed_model_bytes = + ds4_add_sat_u64(non_routed_bytes, active_support_reserve); + uint64_t safe_cache_bytes = 0; + const bool safe_cache_known = + ds4_streaming_manual_cache_safe_bytes(e->backend, + ctx_size, + e->prefill_chunk, + e->ssd_streaming, + fixed_model_bytes, + &safe_cache_bytes); + + uint32_t candidate_count = 0; + uint64_t requested_bytes = 0; + uint64_t effective_bytes = 0; + if (!ds4_streaming_manual_cache_cap_count( + requested_count, + per_expert_bytes, + safe_cache_known ? safe_cache_bytes : UINT64_MAX, + &candidate_count, + &requested_bytes, + &effective_bytes)) { + fprintf(stderr, + "ds4: WARNING: Metal SSD+DSpark numeric expert cache %u x " + "%.2f MiB overflows byte accounting; keeping requested=%u " + "effective=%u slots (support reserve %.2f GiB)\n", + requested_count, + (double)per_expert_bytes / 1048576.0, + requested_count, + requested_count, + (double)active_support_reserve / 1073741824.0); + return true; + } + + const uint32_t effective_count = + ds4_streaming_manual_cache_nonfatal_effective_count( + requested_count, + safe_cache_known, + candidate_count); + + if (!safe_cache_known) { + fprintf(stderr, + "ds4: WARNING: Metal SSD+DSpark working-set recommendation " + "is unavailable; keeping requested=%u effective=%u slots " + "(%.2f GiB, support reserve %.2f GiB)\n", + requested_count, + requested_count, + (double)requested_bytes / 1073741824.0, + (double)active_support_reserve / 1073741824.0); + return true; + } + if (candidate_count == 0) { + fprintf(stderr, + "ds4: WARNING: Metal SSD+DSpark safe numeric-cache policy " + "found no measured room after the context/KV estimate, " + "target weights, and " + "the %.2f GiB support reserve; keeping requested=%u " + "effective=%u slots (%.2f GiB)\n", + (double)active_support_reserve / 1073741824.0, + requested_count, + requested_count, + (double)requested_bytes / 1073741824.0); + return true; + } + if (effective_count < requested_count) { + e->ssd_streaming_cache_experts = effective_count; + fprintf(stderr, + "ds4: WARNING: Metal SSD+DSpark safe numeric-cache policy " + "capped requested=%u slots (%.2f GiB) to effective=%u slots " + "(%.2f GiB); active support reserve %.2f GiB of %.2f GiB " + "mapped support\n", + requested_count, + (double)requested_bytes / 1073741824.0, + effective_count, + (double)effective_bytes / 1073741824.0, + (double)active_support_reserve / 1073741824.0, + (double)support_model_bytes / 1073741824.0); + } else { + fprintf(stderr, + "ds4: Metal SSD+DSpark safe numeric-cache policy kept " + "requested=%u effective=%u slots (%.2f GiB); active support " + "reserve %.2f GiB of %.2f GiB mapped support\n", + requested_count, + effective_count, + (double)effective_bytes / 1073741824.0, + (double)active_support_reserve / 1073741824.0, + (double)support_model_bytes / 1073741824.0); + } + return true; +} + #ifdef DS4_ROCM_BUILD static bool ds4_rocm_dspark_ssd_layout_supported( const ds4_weights *weights, @@ -54664,6 +55043,21 @@ static int generate_metal_graph_raw_swa( const bool memory_report = getenv("DS4_METAL_MEMORY_REPORT") != NULL; if (memory_report) ds4_gpu_print_memory_report("after graph alloc"); + /* Match the session frontend's bounded first-submission warmup. Without + * this call the legacy greedy frontend reaches the warmup from inside + * metal_graph_prefill_layer_major() and uses the whole prompt width. The + * scratch result is overwritten by the real prefill, so warming thousands + * of rows is redundant work as well as making its timing incomparable to + * the session/DSpark frontend. */ + const uint32_t warmup_rows = prefill_cap < 32u ? prefill_cap : 32u; + if (!metal_graph_warmup_prefill_kernels(&g, + model, + weights, + warmup_rows)) { + metal_graph_free(&g); + return 1; + } + float *logits = xmalloc((size_t)DS4_N_VOCAB * sizeof(logits[0])); const bool trace_top = getenv("DS4_TRACE_TOP") != NULL; const bool token_timing = getenv("DS4_TOKEN_TIMING") != NULL; @@ -55422,6 +55816,13 @@ typedef struct ds4_dspark_spec_stats { uint64_t cuda_exactn_batch_head_attempts; uint64_t cuda_exactn_batch_head_uses; uint64_t cuda_exactn_batch_head_fallbacks; + uint64_t cuda_exactn_graph_attempts; + uint64_t cuda_exactn_graph_uses; + uint64_t cuda_exactn_graph_captures; + uint64_t cuda_exactn_graph_replays; + uint64_t cuda_exactn_graph_warms; + uint64_t cuda_exactn_graph_no_slots; + uint64_t cuda_exactn_graph_failures; uint64_t cuda_device_proposer_attempts; uint64_t cuda_device_proposer_uses; uint64_t cuda_device_proposer_fallbacks; @@ -63898,6 +64299,57 @@ int ds4_test_dspark_runtime_policy(ds4_backend backend, return (int)ds4_dspark_runtime_policy_for(backend, distributed_role); } +bool ds4_test_streaming_manual_cache_cap_count( + uint32_t requested_count, + uint64_t per_expert_bytes, + uint64_t safe_cache_bytes, + uint32_t *effective_count_out, + uint64_t *requested_bytes_out, + uint64_t *effective_bytes_out) { + return ds4_streaming_manual_cache_cap_count(requested_count, + per_expert_bytes, + safe_cache_bytes, + effective_count_out, + requested_bytes_out, + effective_bytes_out); +} + +bool ds4_test_streaming_manual_cache_count_cap_enabled(void) { + return ds4_streaming_manual_cache_count_cap_enabled(); +} + +bool ds4_test_streaming_manual_cache_count_cap_eligible( + ds4_backend backend, + bool ssd_streaming, + bool dspark_enabled, + bool support_is_dspark, + uint32_t cache_experts, + uint64_t cache_bytes) { + return ds4_streaming_manual_cache_count_cap_eligible( + backend, + ssd_streaming, + dspark_enabled, + support_is_dspark ? DS4_SUPPORT_DSPARK : DS4_SUPPORT_NONE, + cache_experts, + cache_bytes); +} + +uint64_t ds4_test_streaming_dspark_active_support_reserve_bytes( + uint64_t support_model_bytes) { + return ds4_streaming_dspark_active_support_reserve_bytes( + support_model_bytes); +} + +uint32_t ds4_test_streaming_manual_cache_nonfatal_effective_count( + uint32_t requested_count, + bool safe_cache_known, + uint32_t candidate_count) { + return ds4_streaming_manual_cache_nonfatal_effective_count( + requested_count, + safe_cache_known, + candidate_count); +} + static int ds4_test_make_engine( ds4_engine *eng, const ds4_test_fake_tensor *tensors, @@ -64730,6 +65182,12 @@ static int ds4_engine_open_internal(ds4_engine **out, } } #endif + if (!ds4_engine_cap_metal_dspark_manual_cache_count(e, + opt->context_size)) { + ds4_engine_close(e); + *out = NULL; + return 1; + } if (e->ssd_streaming && e->ssd_streaming_cache_bytes != 0) { const uint64_t requested_cache_bytes = e->ssd_streaming_cache_bytes; uint64_t non_routed_bytes = 0; @@ -64760,7 +65218,8 @@ static int ds4_engine_open_internal(ds4_engine **out, if (safe_cache_known && safe_cache_bytes == 0) { fprintf(stderr, "ds4: %s SSD streaming has no safe room for the requested " - "expert cache after graph and fixed model weights\n", + "expert cache after the context/KV estimate and fixed " + "model weights\n", ds4_backend_name(e->backend)); ds4_engine_close(e); *out = NULL; @@ -64771,7 +65230,8 @@ static int ds4_engine_open_internal(ds4_engine **out, e->ssd_streaming_cache_bytes = safe_cache_bytes; fprintf(stderr, "ds4: %s SSD streaming cache budget %.2f GiB capped to %.2f GiB " - "to stay below the graph working-set pressure budget\n", + "to stay below the context-estimated working-set pressure " + "budget\n", ds4_backend_name(e->backend), (double)requested_cache_bytes / 1073741824.0, (double)e->ssd_streaming_cache_bytes / 1073741824.0); @@ -65933,6 +66393,13 @@ static void ds4_session_print_dspark_stats(const ds4_session *s) { "cuda_exactn_batch_head_attempt=%llu " "cuda_exactn_batch_head_use=%llu " "cuda_exactn_batch_head_fallback=%llu " + "cuda_exactn_graph_attempt=%llu " + "cuda_exactn_graph_use=%llu " + "cuda_exactn_graph_capture=%llu " + "cuda_exactn_graph_replay=%llu " + "cuda_exactn_graph_warm=%llu " + "cuda_exactn_graph_no_slot=%llu " + "cuda_exactn_graph_failure=%llu " "cuda_device_proposer_attempt=%llu " "cuda_device_proposer_use=%llu " "cuda_device_proposer_fallback=%llu " @@ -66001,6 +66468,13 @@ static void ds4_session_print_dspark_stats(const ds4_session *s) { (unsigned long long)st->cuda_exactn_batch_head_attempts, (unsigned long long)st->cuda_exactn_batch_head_uses, (unsigned long long)st->cuda_exactn_batch_head_fallbacks, + (unsigned long long)st->cuda_exactn_graph_attempts, + (unsigned long long)st->cuda_exactn_graph_uses, + (unsigned long long)st->cuda_exactn_graph_captures, + (unsigned long long)st->cuda_exactn_graph_replays, + (unsigned long long)st->cuda_exactn_graph_warms, + (unsigned long long)st->cuda_exactn_graph_no_slots, + (unsigned long long)st->cuda_exactn_graph_failures, (unsigned long long)st->cuda_device_proposer_attempts, (unsigned long long)st->cuda_device_proposer_uses, (unsigned long long)st->cuda_device_proposer_fallbacks, @@ -66337,7 +66811,10 @@ int ds4_session_create(ds4_session **out, ds4_engine *e, int ctx_size) { free(s); return 1; } - if (e->support_kind == DS4_SUPPORT_DSPARK) { + /* Loading a DSpark support GGUF is not, by itself, a request to run the + * speculative runtime. Avoid allocating capture buffers and reducing + * every prompt row unless --dspark is actually enabled. */ + if (e->support_kind == DS4_SUPPORT_DSPARK && e->dspark) { if (!metal_graph_configure_dspark_capture(&s->graph, &e->dspark_weights)) { fprintf(stderr, "ds4: failed to configure DSpark target-hidden capture\n"); @@ -71773,6 +72250,22 @@ static int ds4_session_eval_dspark_speculative_argmax( if (exactn_timing.batch_head_fallback) { s->dspark_stats.cuda_exactn_batch_head_fallbacks++; } + if (exactn_timing.graphs_attempted) { + s->dspark_stats.cuda_exactn_graph_attempts++; + } + if (exactn_timing.graphs_used) { + s->dspark_stats.cuda_exactn_graph_uses++; + } + s->dspark_stats.cuda_exactn_graph_captures += + exactn_timing.graph_captures; + s->dspark_stats.cuda_exactn_graph_replays += + exactn_timing.graph_replays; + s->dspark_stats.cuda_exactn_graph_warms += + exactn_timing.graph_warms; + s->dspark_stats.cuda_exactn_graph_no_slots += + exactn_timing.graph_no_slots; + s->dspark_stats.cuda_exactn_graph_failures += + exactn_timing.graph_failures; } if (exactn_ok && !exactn_mismatch) { @@ -73147,6 +73640,8 @@ int ds4_test_session_eval_exact_drafts( 0, draft_n, eos_token, + false, + DS4_THINK_HIGH, accepted, accepted_cap, err, diff --git a/ds4_cuda.cu b/ds4_cuda.cu index 21b807805..dc7060cb0 100644 --- a/ds4_cuda.cu +++ b/ds4_cuda.cu @@ -433,6 +433,10 @@ static uint64_t g_derived_artifact_bytes; static double g_derived_artifact_build_secs; static int g_derived_replaces_complete; static void *g_aligned_q81_scratch; +/* Preserve the larger direct-prefill arena while also covering token-aware + * Q4 grouped attention-A (8 tokens * 16 groups * K=4096 plus ids/alignment). */ +static const size_t CUDA_ALIGNED_Q81_SCRATCH_BYTES = + 96u * 1024u * 1024u; static uint64_t g_model_range_bytes; static uint64_t g_q8_f16_bytes; static uint64_t g_q8_f32_bytes; @@ -465,6 +469,7 @@ static cudaStream_t g_stream_selected_upload_stream; static int cuda_ok(cudaError_t err, const char *what); extern "C" void ds4_gpu_decode_graphs_invalidate(void); +static void cuda_decode_graphs_shutdown(void); static const char *cuda_model_range_ptr_from_fd( const void *model_map, uint64_t offset, @@ -890,7 +895,11 @@ static inline cublasHandle_t cuda_cublas_for_tier(int logical_tier) { * DS4_CUDA_DECODE_GRAPHS=0 (or off/no/false) disables everything. */ #define CUDA_DECODE_GRAPH_LAYERS 64u #define CUDA_DECODE_GRAPH_ISLANDS 2u -#define CUDA_DECODE_GRAPH_VARIANTS 4u +#define CUDA_DECODE_GRAPH_BASE_VARIANTS 4u +#define CUDA_DECODE_GRAPH_EXACTN_VARIANTS 5u +#define CUDA_DECODE_GRAPH_VARIANTS \ + (CUDA_DECODE_GRAPH_BASE_VARIANTS + CUDA_DECODE_GRAPH_EXACTN_VARIANTS) +#define CUDA_DECODE_GRAPH_VARIANT_EXACTN 0x80000000u /* Mirrors the public `struct ds4_decode_graph_key` decl in ds4_gpu.h * byte-for-byte (ds4_cuda.cu does not include that header; it carries @@ -924,6 +933,22 @@ static cudaStream_t g_decode_graph_stream = NULL; static int g_decode_graph_capturing = 0; static uint64_t g_decode_graph_replays = 0; static uint64_t g_decode_graph_captures = 0; +static uint64_t g_decode_graph_warms = 0; +static uint64_t g_decode_graph_no_slots = 0; +static uint64_t g_decode_graph_failures = 0; + +extern "C" void ds4_gpu_decode_graph_counters( + uint64_t *captures, + uint64_t *replays, + uint64_t *warms, + uint64_t *no_slots, + uint64_t *failures) { + if (captures) *captures = g_decode_graph_captures; + if (replays) *replays = g_decode_graph_replays; + if (warms) *warms = g_decode_graph_warms; + if (no_slots) *no_slots = g_decode_graph_no_slots; + if (failures) *failures = g_decode_graph_failures; +} extern "C" int ds4_gpu_decode_graphs_supported(void) { static int init = 0; @@ -991,12 +1016,35 @@ extern "C" void ds4_gpu_decode_graphs_invalidate(void) { } } +static void cuda_decode_graphs_shutdown(void) { + if (g_decode_graph_stream) { + (void)cudaStreamSynchronize(g_decode_graph_stream); + } + ds4_gpu_decode_graphs_invalidate(); + if (g_decode_graph_stream) { + (void)cudaStreamDestroy(g_decode_graph_stream); + g_decode_graph_stream = NULL; + } + g_decode_graph_capturing = 0; + g_decode_graph_replays = 0; + g_decode_graph_captures = 0; + g_decode_graph_warms = 0; + g_decode_graph_no_slots = 0; + g_decode_graph_failures = 0; +} + static cuda_decode_graph_entry *cuda_decode_graph_find( const ds4_decode_graph_key *key) { if (key->il >= CUDA_DECODE_GRAPH_LAYERS || key->island >= CUDA_DECODE_GRAPH_ISLANDS) return NULL; + const bool exactn_domain = + (key->variant & CUDA_DECODE_GRAPH_VARIANT_EXACTN) != 0u; + const uint32_t first = exactn_domain ? + CUDA_DECODE_GRAPH_BASE_VARIANTS : 0u; + const uint32_t end = exactn_domain ? + CUDA_DECODE_GRAPH_VARIANTS : CUDA_DECODE_GRAPH_BASE_VARIANTS; cuda_decode_graph_entry *slot = NULL; - for (uint32_t v = 0; v < CUDA_DECODE_GRAPH_VARIANTS; v++) { + for (uint32_t v = first; v < end; v++) { cuda_decode_graph_entry *e = &g_decode_graphs[key->il][key->island][v]; if (e->state != 0 && memcmp(&e->key, key, sizeof(*key)) == 0) return e; @@ -1014,11 +1062,23 @@ extern "C" int ds4_gpu_decode_graph_begin(const ds4_decode_graph_key *key) { if (!key || !ds4_gpu_decode_graphs_supported()) return -1; if (g_decode_graph_capturing) return -1; /* no nesting */ cuda_decode_graph_entry *e = cuda_decode_graph_find(key); - if (!e || e->state == 3) return -1; + if (!e) { + g_decode_graph_no_slots++; + if (getenv("DS4_CUDA_DECODE_GRAPH_LOG") != NULL) { + fprintf(stderr, + "ds4: decode graph no slot il=%u island=%u variant=0x%08x domain=%s\n", + key->il, key->island, key->variant, + (key->variant & CUDA_DECODE_GRAPH_VARIANT_EXACTN) ? + "exactn" : "decode"); + } + return -1; + } + if (e->state == 3) return -1; if (e->state == 0) { /* Warm pass: run eagerly once so lazy allocators (tmp scratch, * cuBLAS workspaces) reach steady-state sizes before capture. */ e->state = 1; + g_decode_graph_warms++; return -1; } if (e->state == 2) { @@ -1028,6 +1088,7 @@ extern "C" int ds4_gpu_decode_graph_begin(const ds4_decode_graph_key *key) { key->il, key->island, cudaGetErrorString(err)); (void)cudaGetLastError(); cuda_decode_graph_entry_kill(e); + g_decode_graph_failures++; return -1; /* caller encodes eagerly; nothing was consumed */ } e->hits++; @@ -1040,6 +1101,7 @@ extern "C" int ds4_gpu_decode_graph_begin(const ds4_decode_graph_key *key) { "decode graph stream create")) { g_decode_graph_stream = NULL; cuda_decode_graph_entry_kill(e); + g_decode_graph_failures++; return -1; } } @@ -1051,6 +1113,7 @@ extern "C" int ds4_gpu_decode_graph_begin(const ds4_decode_graph_key *key) { "decode graph begin capture")) { (void)cublasSetStream(cuda_cublas_for_tier(0), NULL); cuda_decode_graph_entry_kill(e); + g_decode_graph_failures++; return -1; } g_decode_graph_capturing = 1; @@ -1070,10 +1133,12 @@ extern "C" int ds4_gpu_decode_graph_end(const ds4_decode_graph_key *key) { (void)cudaGetLastError(); if (graph) (void)cudaGraphDestroy(graph); if (e) cuda_decode_graph_entry_kill(e); + g_decode_graph_failures++; return -1; /* caller re-encodes the island eagerly */ } if (!e) { /* cannot happen: begin() found it */ (void)cudaGraphDestroy(graph); + g_decode_graph_failures++; return -1; } cudaGraphExec_t exec = NULL; @@ -1084,6 +1149,7 @@ extern "C" int ds4_gpu_decode_graph_end(const ds4_decode_graph_key *key) { key->il, key->island, cudaGetErrorString(err)); (void)cudaGetLastError(); cuda_decode_graph_entry_kill(e); + g_decode_graph_failures++; return -1; } /* Capture recorded the work without executing it: launch now so this @@ -1095,14 +1161,17 @@ extern "C" int ds4_gpu_decode_graph_end(const ds4_decode_graph_key *key) { (void)cudaGetLastError(); (void)cudaGraphExecDestroy(exec); cuda_decode_graph_entry_kill(e); + g_decode_graph_failures++; return -1; } e->exec = exec; e->state = 2; g_decode_graph_captures++; if (getenv("DS4_CUDA_DECODE_GRAPH_LOG") != NULL) { - fprintf(stderr, "ds4: decode graph captured il=%u island=%u (total %llu)\n", - key->il, key->island, + fprintf(stderr, "ds4: decode graph captured il=%u island=%u variant=0x%08x domain=%s (total %llu)\n", + key->il, key->island, key->variant, + (key->variant & CUDA_DECODE_GRAPH_VARIANT_EXACTN) ? + "exactn" : "decode", (unsigned long long)g_decode_graph_captures); } return 0; @@ -1247,6 +1316,7 @@ extern "C" void ds4_gpu_decode_graph_abort(const ds4_decode_graph_key *key) { cuda_decode_graph_entry *e = cuda_decode_graph_find(key); if (e) cuda_decode_graph_entry_kill(e); } + g_decode_graph_failures++; } /* Multi-tier-aware weight pointer resolver. @@ -2878,6 +2948,7 @@ extern "C" int ds4_gpu_init(void) { extern "C" void ds4_gpu_cleanup(void) { (void)cudaDeviceSynchronize(); + cuda_decode_graphs_shutdown(); g_current_logical_tier = -1; /* Multi-GPU teardown: events, streams, cublas handles, scratch @@ -3216,6 +3287,11 @@ extern "C" uint64_t ds4_gpu_tensor_bytes(const ds4_gpu_tensor *tensor) { return tensor ? tensor->bytes : 0; } +extern "C" uintptr_t ds4_gpu_tensor_storage_key( + const ds4_gpu_tensor *tensor) { + return tensor ? (uintptr_t)tensor->ptr : (uintptr_t)0; +} + extern "C" void *ds4_gpu_tensor_contents(ds4_gpu_tensor *tensor) { if (!tensor) return NULL; /* Full-device sync preserves legacy semantics. */ @@ -4668,12 +4744,11 @@ extern "C" int ds4_gpu_build_derived_artifacts( g_derived_artifact_bytes = built_bytes; g_derived_artifact_build_secs = cuda_wall_sec() - t0; if (!g_aligned_q81_scratch) { - const size_t scratch_bytes = 96u * 1024u * 1024u; cudaError_t scratch_err = cudaMalloc(&g_aligned_q81_scratch, - scratch_bytes); + CUDA_ALIGNED_Q81_SCRATCH_BYTES); if (scratch_err == cudaSuccess) { ds4_mmq_set_aligned_q81_scratch(g_aligned_q81_scratch, - scratch_bytes); + CUDA_ALIGNED_Q81_SCRATCH_BYTES); } else { g_aligned_q81_scratch = NULL; (void)cudaGetLastError(); @@ -35317,6 +35392,22 @@ extern "C" int ds4_gpu_attention_output_q8_batch_f16_tensor( return 0; } +static void cuda_q4_grouped_attn_a_oracle_register_report(void); +static uint64_t g_q4_grouped_attn_a_oracle_calls; +static uint64_t g_q4_grouped_attn_a_oracle_mismatches; +static uint64_t g_q4_grouped_attn_a_oracle_skips; +static uint64_t g_q4_grouped_attn_a_oracle_batch_candidates; +static uint64_t g_q4_grouped_attn_a_oracle_batch_calls; +static uint64_t g_q4_grouped_attn_a_oracle_batch_mismatches; +static uint64_t g_q4_grouped_attn_a_oracle_batch_skips; +static int g_q4_grouped_attn_a_oracle_report_registered; +static int g_q4_grouped_attn_a_oracle_mismatch_reported; +static int cuda_q4_grouped_attn_a_batch_oracle( + const char *out_a, const float *heads, float *low, + uint32_t n_tokens, uint32_t n_groups, + uint32_t group_dim, uint32_t rank, + int logical_tier, cudaStream_t stream); + extern "C" int ds4_gpu_attention_output_q4_K_batch_tensor( ds4_gpu_tensor *out, ds4_gpu_tensor *low, ds4_gpu_tensor *group_tmp, ds4_gpu_tensor *low_tmp, @@ -35324,22 +35415,24 @@ extern "C" int ds4_gpu_attention_output_q4_K_batch_tensor( uint64_t out_a_offset, uint64_t out_b_offset, uint32_t out_b_type, uint64_t group_dim, uint64_t rank, uint32_t n_groups, uint64_t out_dim, const ds4_gpu_tensor *heads, uint32_t n_tokens) { + const int grouped_batch_require = cuda_env_flag_enabled( + "DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_BATCH", 0); if (!out || !low || !group_tmp || !low_tmp || !heads || !model_map || group_dim == 0 || rank == 0 || n_groups == 0 || out_dim == 0 || n_tokens < 2u || group_dim > INT_MAX || rank > INT_MAX || out_dim > INT_MAX || n_tokens > INT_MAX || !cuda_use_mmq()) { - return 0; + return grouped_batch_require ? -1 : 0; } const uint64_t low_dim = (uint64_t)n_groups * rank; if (low_dim > INT_MAX || (group_dim % CUDA_QK_K) != 0u || (low_dim % CUDA_QK_K) != 0u) { - return 0; + return grouped_batch_require ? -1 : 0; } const uint64_t row_a_bytes = (group_dim / CUDA_QK_K) * sizeof(cuda_block_q4_K); if (rank > UINT64_MAX / row_a_bytes || n_groups > UINT64_MAX / (rank * row_a_bytes)) { - return 0; + return grouped_batch_require ? -1 : 0; } const uint64_t out_a_bytes = (uint64_t)n_groups * rank * row_a_bytes; if (out_a_offset > model_size || @@ -35349,12 +35442,17 @@ extern "C" int ds4_gpu_attention_output_q4_K_batch_tensor( out->bytes < (uint64_t)n_tokens * out_dim * sizeof(float) || group_tmp->bytes < (uint64_t)n_tokens * group_dim * sizeof(float) || low_tmp->bytes < (uint64_t)n_tokens * rank * sizeof(float)) { - return 0; + return grouped_batch_require ? -1 : 0; } const int logical_tier = ds4_tensor_device_idx(out); const char *out_a = cuda_resolve_weight_ptr( model_map, out_a_offset, out_a_bytes, logical_tier, "q4 attn_out_a"); - if (!out_a) return 0; + if (!out_a) return grouped_batch_require ? -1 : 0; + const int grouped_oracle = cuda_env_flag_enabled( + "DS4_CUDA_Q4_GROUPED_ATTN_A_ORACLE", 0); + const int grouped_batch_enable = cuda_env_flag_enabled( + "DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_BATCH", 0); + if (grouped_oracle) cuda_q4_grouped_attn_a_oracle_register_report(); const int grouped_gb10 = n_tokens <= 8u && n_groups <= INT_MAX && @@ -35362,23 +35460,54 @@ extern "C" int ds4_gpu_attention_output_q4_K_batch_tensor( ds4_tensor_device_idx(heads) == logical_tier && cuda_q4_gb10_fast_path_enabled( logical_tier, "DS4_CUDA_NO_Q4_GROUPED_ATTN_A"); + if (grouped_batch_require && + (!grouped_batch_enable || !grouped_gb10)) { + fprintf(stderr, + "ds4: required CUDA Q4 grouped attention-A batch path is " + "not eligible\n"); + return -1; + } if (grouped_gb10) { - /* DSpark verification stores [token][group][K]. Dispatch one exact - * grouped MMVQ per token: each group keeps its own Q8_1 row and Q4_K - * weights, while the grouped entry amortizes setup/launch overhead. - * A preflight rejection returns 0 to the unchanged row-wise caller; - * a negative post-launch result is fatal (the caller understands the - * tri-state contract for this tiny-batch hook). */ - for (uint32_t t = 0; t < n_tokens; t++) { - const int rc = ds4_mmq_q4_K_grouped_vec( - out_a, - (const float *)heads->ptr + - (uint64_t)t * n_groups * group_dim, - (float *)low->ptr + (uint64_t)t * low_dim, - (int)rank, (int)group_dim, (int)n_groups, - cuda_decode_stream()); - if (rc == DS4_MMQ_NOT_APPLICABLE) return 0; - if (rc != 0) return -1; + /* DSpark verification stores [token][group][K]. The opt-in batch + * entry flattens (token, group) into channels while keeping + * ncols_dst=1, preserving the canonical per-pair Q8_1 quantization + * and MMVQ reduction. NOT_APPLICABLE is guaranteed pre-enqueue, so + * it safely falls back to the established per-token grouped loop. + * Any other error may follow an enqueue and therefore fails closed. */ + const int batch_rc = ds4_mmq_q4_K_grouped_batch_vec( + out_a, (const float *)heads->ptr, (float *)low->ptr, + (int)rank, (int)group_dim, (int)n_tokens, (int)n_groups, + cuda_decode_stream()); + if (batch_rc == DS4_MMQ_NOT_APPLICABLE) { + if (grouped_batch_require) { + fprintf(stderr, + "ds4: required CUDA Q4 grouped attention-A batch " + "dispatch was not applicable\n"); + return -1; + } + for (uint32_t t = 0; t < n_tokens; t++) { + const int rc = ds4_mmq_q4_K_grouped_vec( + out_a, + (const float *)heads->ptr + + (uint64_t)t * n_groups * group_dim, + (float *)low->ptr + (uint64_t)t * low_dim, + (int)rank, (int)group_dim, (int)n_groups, + cuda_decode_stream()); + if (rc == DS4_MMQ_NOT_APPLICABLE) return 0; + if (rc != 0) return -1; + } + } else if (batch_rc != 0) { + return -1; + } else if (grouped_oracle) { + g_q4_grouped_attn_a_oracle_batch_candidates++; + } + if (batch_rc == 0 && grouped_oracle && + !cuda_q4_grouped_attn_a_batch_oracle( + out_a, (const float *)heads->ptr, + (float *)low->ptr, n_tokens, n_groups, + (uint32_t)group_dim, (uint32_t)rank, + logical_tier, cuda_decode_stream())) { + return -1; } } else { /* Existing cross-CUDA path: pack one group at a time, run a @@ -35419,20 +35548,20 @@ extern "C" int ds4_gpu_attention_output_q4_K_batch_tensor( return 0; } -static uint64_t g_q4_grouped_attn_a_oracle_calls; -static uint64_t g_q4_grouped_attn_a_oracle_mismatches; -static uint64_t g_q4_grouped_attn_a_oracle_skips; -static int g_q4_grouped_attn_a_oracle_report_registered; -static int g_q4_grouped_attn_a_oracle_mismatch_reported; - static void cuda_q4_grouped_attn_a_oracle_report(void) { fprintf(stderr, "ds4: CUDA Q4 grouped attention-A oracle: " "calls=%llu mismatches=%llu skips=%llu " + "batch_candidates=%llu batch_calls=%llu " + "batch_mismatches=%llu batch_skips=%llu " "(canonical MMVQ output retained)\n", (unsigned long long)g_q4_grouped_attn_a_oracle_calls, (unsigned long long)g_q4_grouped_attn_a_oracle_mismatches, - (unsigned long long)g_q4_grouped_attn_a_oracle_skips); + (unsigned long long)g_q4_grouped_attn_a_oracle_skips, + (unsigned long long)g_q4_grouped_attn_a_oracle_batch_candidates, + (unsigned long long)g_q4_grouped_attn_a_oracle_batch_calls, + (unsigned long long)g_q4_grouped_attn_a_oracle_batch_mismatches, + (unsigned long long)g_q4_grouped_attn_a_oracle_batch_skips); } static void cuda_q4_grouped_attn_a_oracle_register_report(void) { @@ -35442,6 +35571,139 @@ static void cuda_q4_grouped_attn_a_oracle_register_report(void) { } } +static int cuda_q4_grouped_attn_a_batch_reference( + const char *out_a, const float *heads, float *reference, + uint32_t n_tokens, uint32_t n_groups, + uint32_t group_dim, uint32_t rank, + cudaStream_t stream) { + const uint64_t heads_token_stride = + (uint64_t)n_groups * group_dim; + const uint64_t low_token_stride = + (uint64_t)n_groups * rank; + for (uint32_t t = 0; t < n_tokens; t++) { + const int rc = ds4_mmq_q4_K_grouped_vec( + out_a, + heads + (uint64_t)t * heads_token_stride, + reference + (uint64_t)t * low_token_stride, + (int)rank, (int)group_dim, (int)n_groups, stream); + if (rc != 0) return 0; + } + return 1; +} + +/* Diagnostic oracle for the token-aware grouped dispatch. The old + * per-token grouped loop is the canonical reference. With the oracle on, + * that reference always replaces the candidate before the output-B consumer; + * the candidate is used only for a bitwise comparison. Decode graphs are + * disabled globally for this env, but a capture check remains here so dynamic + * env changes and foreign captures still retain canonical output without a + * host synchronization. */ +static int cuda_q4_grouped_attn_a_batch_oracle( + const char *out_a, const float *heads, float *low, + uint32_t n_tokens, uint32_t n_groups, + uint32_t group_dim, uint32_t rank, + int logical_tier, cudaStream_t stream) { + cudaStreamCaptureStatus capture = cudaStreamCaptureStatusNone; + const cudaError_t capture_err = cudaStreamIsCapturing(stream, &capture); + if (capture_err != cudaSuccess || + capture != cudaStreamCaptureStatusNone) { + (void)cudaGetLastError(); + g_q4_grouped_attn_a_oracle_skips++; + g_q4_grouped_attn_a_oracle_batch_skips++; + return cuda_q4_grouped_attn_a_batch_reference( + out_a, heads, low, n_tokens, n_groups, + group_dim, rank, stream); + } + + const uint64_t low_elems = + (uint64_t)n_tokens * n_groups * rank; + if (low_elems > UINT64_MAX / sizeof(float)) return 0; + const uint64_t reference_bytes = low_elems * sizeof(float); + if (reference_bytes > UINT64_MAX - 255u) return 0; + const uint64_t mismatch_offset = + (reference_bytes + 255u) & ~255ull; + if (mismatch_offset > UINT64_MAX - sizeof(uint32_t)) return 0; + unsigned char *scratch = (unsigned char *)cuda_tmp_alloc_on( + logical_tier, mismatch_offset + sizeof(uint32_t), + "q4 grouped attention-A batch oracle"); + if (!scratch) { + g_q4_grouped_attn_a_oracle_skips++; + g_q4_grouped_attn_a_oracle_batch_skips++; + return cuda_q4_grouped_attn_a_batch_reference( + out_a, heads, low, n_tokens, n_groups, + group_dim, rank, stream); + } + float *reference = (float *)scratch; + uint32_t *mismatch_device = + (uint32_t *)(scratch + mismatch_offset); + if (!cuda_q4_grouped_attn_a_batch_reference( + out_a, heads, reference, n_tokens, n_groups, + group_dim, rank, stream)) { + return 0; + } + + if (!cuda_ok(cudaMemsetAsync(mismatch_device, 0, sizeof(uint32_t), + stream), + "clear q4 grouped attention-A batch oracle")) { + g_q4_grouped_attn_a_oracle_skips++; + g_q4_grouped_attn_a_oracle_batch_skips++; + return cuda_ok(cudaMemcpyAsync( + low, reference, reference_bytes, + cudaMemcpyDeviceToDevice, stream), + "retain q4 grouped attention-A batch reference"); + } + q4_K_attn_hc_bitwise_compare_kernel + <<<(unsigned)((low_elems + 255u) / 256u), 256, 0, stream>>>( + mismatch_device, reference, low, low_elems); + if (!cuda_ok(cudaGetLastError(), + "q4 grouped attention-A batch oracle compare launch")) { + g_q4_grouped_attn_a_oracle_skips++; + g_q4_grouped_attn_a_oracle_batch_skips++; + return cuda_ok(cudaMemcpyAsync( + low, reference, reference_bytes, + cudaMemcpyDeviceToDevice, stream), + "retain q4 grouped attention-A batch reference"); + } + + uint32_t mismatch_host = 0u; + const cudaError_t retain_err = cudaMemcpyAsync( + low, reference, reference_bytes, cudaMemcpyDeviceToDevice, stream); + if (retain_err != cudaSuccess) { + return cuda_ok(retain_err, + "retain q4 grouped attention-A batch reference"); + } + const cudaError_t read_err = cudaMemcpyAsync( + &mismatch_host, mismatch_device, sizeof(mismatch_host), + cudaMemcpyDeviceToHost, stream); + if (read_err != cudaSuccess) { + (void)cuda_ok(read_err, + "read q4 grouped attention-A batch oracle"); + g_q4_grouped_attn_a_oracle_skips++; + g_q4_grouped_attn_a_oracle_batch_skips++; + return cuda_ok(cudaStreamSynchronize(stream), + "synchronize retained q4 grouped batch reference"); + } + if (!cuda_ok(cudaStreamSynchronize(stream), + "synchronize q4 grouped attention-A batch oracle")) { + return 0; + } + + g_q4_grouped_attn_a_oracle_calls++; + g_q4_grouped_attn_a_oracle_batch_calls++; + if (mismatch_host != 0u) { + g_q4_grouped_attn_a_oracle_mismatches++; + g_q4_grouped_attn_a_oracle_batch_mismatches++; + if (!g_q4_grouped_attn_a_oracle_mismatch_reported) { + g_q4_grouped_attn_a_oracle_mismatch_reported = 1; + fprintf(stderr, + "ds4: CUDA Q4 grouped attention-A oracle found a " + "bitwise batch mismatch; retained per-token MMVQ " + "output\n"); + } + } + return 1; +} + extern "C" int ds4_gpu_attention_output_low_q4_K_slice_tensor( ds4_gpu_tensor *low, const void *model_map, uint64_t model_size, uint64_t out_a_offset, uint64_t group_dim, uint64_t rank, diff --git a/ds4_gpu.h b/ds4_gpu.h index 6ad4d5168..c725e12ca 100644 --- a/ds4_gpu.h +++ b/ds4_gpu.h @@ -50,6 +50,12 @@ ds4_gpu_tensor *ds4_gpu_tensor_view(const ds4_gpu_tensor *base, uint64_t offset, void ds4_gpu_tensor_free(ds4_gpu_tensor *tensor); uint64_t ds4_gpu_tensor_bytes(const ds4_gpu_tensor *tensor); void *ds4_gpu_tensor_contents(ds4_gpu_tensor *tensor); +#if !defined(__APPLE__) && !defined(DS4_ROCM_BUILD) +/* Stable CUDA allocation identity, including a view's byte offset. Unlike + * the wrapper handle, this stays unchanged when an equivalent tensor view is + * recreated and is therefore suitable for CUDA graph-cache keys. */ +uintptr_t ds4_gpu_tensor_storage_key(const ds4_gpu_tensor *tensor); +#endif int ds4_gpu_tensor_fill_f32(ds4_gpu_tensor *tensor, float value, uint64_t count); int ds4_gpu_tensor_write(ds4_gpu_tensor *tensor, uint64_t offset, const void *data, uint64_t bytes); int ds4_gpu_tensor_read(const ds4_gpu_tensor *tensor, uint64_t offset, void *data, uint64_t bytes); @@ -412,21 +418,6 @@ int ds4_gpu_matmul_quant_kslice_tensor( uint64_t out_dim, const ds4_gpu_tensor *x, uint64_t x_elem_off); -int ds4_gpu_attention_output_q8_tp_tensor( - ds4_gpu_tensor *out, - ds4_gpu_tensor *low, - const void *model_map, - uint64_t model_size, - uint64_t out_a_offset, - uint64_t out_b_offset, - uint64_t group_dim, - uint64_t rank, - uint32_t n_groups_total, - uint32_t group0, - uint32_t group_cnt, - uint64_t out_dim, - const ds4_gpu_tensor *heads); - /* ========================================================================= * Embeddings and Indexer Helpers. * ========================================================================= @@ -2351,8 +2342,8 @@ int ds4_gpu_attention_output_q8_batch_tensor( const ds4_gpu_tensor *heads, uint32_t n_tokens); /* Returns 1 when the batch path ran, 0 for the ordinary row fallback, and -1 - * for a required Metal path that is unavailable or a CUDA grouped launch - * that failed after enqueue and therefore cannot safely fall back. */ + * for a post-enqueue failure or a backend REQUIRE diagnostic. The caller + * must not retry the row fallback after -1. */ int ds4_gpu_attention_output_q4_K_batch_tensor( ds4_gpu_tensor *out, ds4_gpu_tensor *low, @@ -2823,6 +2814,19 @@ int ds4_gpu_hc_weighted_sum_tensor( uint32_t n_embd, uint32_t n_hc); +#ifdef __APPLE__ +/* Metal DSpark prefill capture: materialize every reduced HC row and mirror + * the final row in the same compute dispatch. `out` and `last_out` must refer + * to non-overlapping storage; production uses separate persistent tensors. */ +int ds4_gpu_hc_weighted_sum_capture_last_tensor( + ds4_gpu_tensor *out, + ds4_gpu_tensor *last_out, + const ds4_gpu_tensor *residual_hc, + const ds4_gpu_tensor *weights, + uint32_t n_embd, + uint32_t n_hc); +#endif + int ds4_gpu_hc_weighted_sum_split_tensor( ds4_gpu_tensor *out, const ds4_gpu_tensor *residual_hc, @@ -2919,29 +2923,6 @@ int ds4_gpu_hc_rms_scale_project_f16_tensor( uint32_t n_rows, float eps); -#ifdef __APPLE__ -int ds4_gpu_hc_rms_norm_mix_split_norm_f16_tensor( - ds4_gpu_tensor *mix, - ds4_gpu_tensor *out, - ds4_gpu_tensor *norm_out, - ds4_gpu_tensor *split, - const ds4_gpu_tensor *residual_hc, - const void *model_map, - uint64_t model_size, - uint64_t mix_weight_offset, - uint64_t scale_offset, - uint64_t base_offset, - uint64_t norm_weight_offset, - uint32_t n, - uint32_t mix_dim, - uint32_t n_embd, - uint32_t n_hc, - uint32_t sinkhorn_iters, - float eps, - float hc_eps, - float norm_eps); - -#endif int ds4_gpu_output_hc_weights_tensor( ds4_gpu_tensor *out, const ds4_gpu_tensor *pre, @@ -2969,18 +2950,6 @@ int ds4_gpu_hc_expand_add_tensor( const ds4_gpu_tensor *comb, uint32_t n_embd, uint32_t n_hc); - - -int ds4_gpu_hc_expand_add_tensor( - ds4_gpu_tensor *out_hc, - const ds4_gpu_tensor *block_out, - const ds4_gpu_tensor *block_add, - const ds4_gpu_tensor *residual_hc, - const ds4_gpu_tensor *post, - const ds4_gpu_tensor *comb, - uint32_t n_embd, - uint32_t n_hc); - int ds4_gpu_hc_expand_split_tensor( ds4_gpu_tensor *out_hc, const ds4_gpu_tensor *block_out, @@ -3237,8 +3206,9 @@ int ds4_gpu_matmul_q4_K_hc_expand_tensor( /* Decode-island CUDA graph capture (CUDA backend; Metal/ROCm/CPU stub it * out and stay eager). Design ported from the Entrpi/ds4 batched-serving * fork's per-layer decode graph capture. The key identifies a captured - * island: layer, island index, and the activation buffers whose addresses - * the captured kernels bake in. ds4_cuda.cu mirrors this struct + * island: layer, island index, and the stable device-storage addresses that + * the captured kernels bake in (never short-lived view-wrapper addresses). + * ds4_cuda.cu mirrors this struct * byte-for-byte (it does not include this header); keep both in sync. */ typedef struct ds4_decode_graph_key { uint32_t il; @@ -3251,6 +3221,10 @@ typedef struct ds4_decode_graph_key { void *attn_norm; } ds4_decode_graph_key; +/* Exact-N uses a disjoint CUDA graph-cache domain so its five batch-row + * activation addresses cannot evict the ordinary decode variants. */ +#define DS4_DECODE_GRAPH_VARIANT_EXACTN 0x80000000u + int ds4_gpu_decode_graphs_supported(void); /* 1: replayed (island already executed; skip encoding it) * 0: capturing (encode the island, then call _end) @@ -3261,6 +3235,14 @@ int ds4_gpu_decode_graph_begin(const ds4_decode_graph_key *key); int ds4_gpu_decode_graph_end(const ds4_decode_graph_key *key); void ds4_gpu_decode_graph_abort(const ds4_decode_graph_key *key); void ds4_gpu_decode_graphs_invalidate(void); +#if !defined(__APPLE__) && !defined(DS4_ROCM_BUILD) +void ds4_gpu_decode_graph_counters( + uint64_t *captures, + uint64_t *replays, + uint64_t *warms, + uint64_t *no_slots, + uint64_t *failures); +#endif #ifdef __cplusplus } diff --git a/ds4_help.c b/ds4_help.c index c0ee9e3cd..b6155d9bf 100644 --- a/ds4_help.c +++ b/ds4_help.c @@ -172,7 +172,7 @@ static void print_model_runtime(FILE *fp, const help_colors *c, opt(fp, c, "--power N", "GPU duty-cycle target, 1..100. Default: 100"); opt(fp, c, "--ssd-streaming", "Metal/CUDA/ROCm: opt in to SSD-backed model streaming instead of full residency."); opt(fp, c, "--ssd-streaming-cold", "SSD streaming: skip default popularity-based expert-cache preload."); - opt(fp, c, "--ssd-streaming-cache-experts N|NGB", "SSD streaming cache target. N requests dynamic expert slots; NGB also reserves two full prefill layers. Either may be reduced to fit the model, graph, context, and backend working set."); + opt(fp, c, "--ssd-streaming-cache-experts N|NGB", "SSD streaming cache target. N requests dynamic expert slots without the two-layer reserve; NGB is an accounted routed budget that includes it. Either may be reduced by the final memory check. Metal SSD+DSpark can A/B a support-aware pre-cap with DS4_METAL_DSPARK_SAFE_EXPERT_COUNT=1. Auto: 80% working set minus fixed weights."); opt(fp, c, "--ssd-streaming-full-layers N", "GLM Metal streaming: keep the first N routed layers fully resident. Default: auto from NGB expert budget; use 0 to disable."); opt(fp, c, "--ssd-streaming-preload-experts N", "SSD streaming: upfront popularity preload count. DeepSeek auto-seeds by default; GLM demand-fills unless N is explicit."); opt(fp, c, "--simulate-used-memory NGB", "Diagnostic: lock N GiB before model load to simulate a smaller-memory machine."); diff --git a/ds4_metal.m b/ds4_metal.m index c7fadc6c7..2e75fcf8a 100644 --- a/ds4_metal.m +++ b/ds4_metal.m @@ -95,6 +95,7 @@ static id g_hc_split_weighted_sum_norm_pipeline; static id g_dsv4_hc_producer_pre_norm_pipeline; static id g_hc_weighted_sum_pipeline; +static id g_hc_weighted_sum_capture_last_pipeline; static id g_output_hc_weights4_pipeline; static uint32_t g_test_flags; static id g_hc_expand_pipeline; @@ -8363,6 +8364,25 @@ int ds4_gpu_init(void) { return 0; } + error = nil; + fn = [library newFunctionWithName:@"kernel_dsv4_hc_weighted_sum_capture_last"]; + if (!fn) { + fprintf(stderr, "ds4: Metal kernel_dsv4_hc_weighted_sum_capture_last function not found\n"); + g_queue = nil; + g_device = nil; + return 0; + } + g_hc_weighted_sum_capture_last_pipeline = + [g_device newComputePipelineStateWithFunction:fn error:&error]; + if (!g_hc_weighted_sum_capture_last_pipeline) { + fprintf(stderr, + "ds4: Metal kernel_dsv4_hc_weighted_sum_capture_last pipeline failed: %s\n", + [[error localizedDescription] UTF8String]); + g_queue = nil; + g_device = nil; + return 0; + } + error = nil; fn = [library newFunctionWithName:@"kernel_dsv4_output_hc_weights4"]; if (fn) { @@ -10421,6 +10441,7 @@ void ds4_gpu_cleanup(void) { g_hc_split_weighted_sum_norm_pipeline = nil; g_dsv4_hc_producer_pre_norm_pipeline = nil; g_hc_weighted_sum_pipeline = nil; + g_hc_weighted_sum_capture_last_pipeline = nil; g_output_hc_weights4_pipeline = nil; g_hc_expand_pipeline = nil; g_moe_mul_mv_id_iq2_xxs_pipeline = nil; @@ -44209,6 +44230,7 @@ int ds4_gpu_hc_split_sinkhorn_tensor( static int ds4_gpu_hc_weighted_sum_strided( ds4_gpu_tensor *out, + ds4_gpu_tensor *last_out, const ds4_gpu_tensor *residual_hc, const ds4_gpu_tensor *weights, uint64_t weight_offset, @@ -44226,6 +44248,8 @@ static int ds4_gpu_hc_weighted_sum_strided( id xbuf = ds4_gpu_tensor_buffer(residual_hc); id wbuf = ds4_gpu_tensor_buffer(weights); id outbuf = ds4_gpu_tensor_buffer(out); + id lastbuf = last_out ? + ds4_gpu_tensor_buffer(last_out) : nil; const uint64_t out_row_bytes = (uint64_t)n_embd * sizeof(float); const uint64_t out_tensor_bytes = ds4_gpu_tensor_bytes(out); if (out_row_bytes == 0 || out_tensor_bytes < out_row_bytes || out_tensor_bytes % out_row_bytes != 0) { @@ -44252,9 +44276,10 @@ static int ds4_gpu_hc_weighted_sum_strided( const uint64_t w_last = weight_offset + (n_tokens64 - 1u) * weight_row_stride + (uint64_t)n_hc * sizeof(float); - if (!xbuf || !wbuf || !outbuf || + if (!xbuf || !wbuf || !outbuf || (last_out && !lastbuf) || ds4_gpu_tensor_bytes(residual_hc) < x_bytes || - ds4_gpu_tensor_bytes(weights) < w_last) { + ds4_gpu_tensor_bytes(weights) < w_last || + (last_out && ds4_gpu_tensor_bytes(last_out) < out_row_bytes)) { fprintf(stderr, "ds4: Metal HC weighted sum received undersized activation buffers\n"); return 0; } @@ -44279,11 +44304,18 @@ static int ds4_gpu_hc_weighted_sum_strided( if (!cb) return 0; id enc = ds4_gpu_compute_encoder(cb); - [enc setComputePipelineState:g_hc_weighted_sum_pipeline]; + [enc setComputePipelineState:last_out ? + g_hc_weighted_sum_capture_last_pipeline : + g_hc_weighted_sum_pipeline]; [enc setBytes:&args length:sizeof(args) atIndex:0]; [enc setBuffer:xbuf offset:ds4_gpu_tensor_offset(residual_hc) atIndex:1]; [enc setBuffer:wbuf offset:ds4_gpu_tensor_offset(weights) + (NSUInteger)weight_offset atIndex:2]; [enc setBuffer:outbuf offset:ds4_gpu_tensor_offset(out) atIndex:3]; + if (last_out) { + [enc setBuffer:lastbuf + offset:ds4_gpu_tensor_offset(last_out) + atIndex:4]; + } [enc dispatchThreadgroups:MTLSizeMake(n_tg, 1, 1) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)]; ds4_gpu_end_compute_encoder(cb, enc); @@ -44301,6 +44333,7 @@ int ds4_gpu_hc_weighted_sum_tensor( uint32_t n_embd, uint32_t n_hc) { return ds4_gpu_hc_weighted_sum_strided(out, + NULL, residual_hc, weights, 0, @@ -44310,6 +44343,25 @@ int ds4_gpu_hc_weighted_sum_tensor( "HC weighted sum"); } +int ds4_gpu_hc_weighted_sum_capture_last_tensor( + ds4_gpu_tensor *out, + ds4_gpu_tensor *last_out, + const ds4_gpu_tensor *residual_hc, + const ds4_gpu_tensor *weights, + uint32_t n_embd, + uint32_t n_hc) { + if (!last_out) return 0; + return ds4_gpu_hc_weighted_sum_strided(out, + last_out, + residual_hc, + weights, + 0, + (uint64_t)n_hc * sizeof(float), + n_embd, + n_hc, + "HC weighted sum/capture last"); +} + int ds4_gpu_hc_weighted_sum_split_tensor( ds4_gpu_tensor *out, const ds4_gpu_tensor *residual_hc, @@ -44318,6 +44370,7 @@ int ds4_gpu_hc_weighted_sum_split_tensor( uint32_t n_hc) { const uint64_t mix_hc = 2ull * n_hc + (uint64_t)n_hc * n_hc; return ds4_gpu_hc_weighted_sum_strided(out, + NULL, residual_hc, split, 0, diff --git a/metal/dsv4_hc.metal b/metal/dsv4_hc.metal index 8b3aead3c..dfcefdded 100644 --- a/metal/dsv4_hc.metal +++ b/metal/dsv4_hc.metal @@ -1070,6 +1070,38 @@ kernel void kernel_dsv4_hc_weighted_sum( *((device float *) (dst + d*args.nb0 + t*args.nb1)) = acc; } +// DSpark needs both the complete prompt matrix and its final reduced row. +// Keep this as a separate entry point so the ordinary HC weighted-sum shader +// retains its original ABI and instruction stream. The extra store reuses the +// same accumulator before it leaves the register and replaces a 16 KiB blit. +kernel void kernel_dsv4_hc_weighted_sum_capture_last( + constant ds4_metal_args_dsv4_hc_weighted_sum & args, + device const char * x, + device const char * weights, + device char * dst, + device char * last_dst, + uint gid [[thread_position_in_grid]]) { + const int64_t n_elem = args.n_embd * args.n_tokens; + if ((int64_t) gid >= n_elem) { + return; + } + + const int64_t d = ((int64_t) gid) % args.n_embd; + const int64_t t = ((int64_t) gid) / args.n_embd; + + float acc = 0.0f; + for (int64_t h = 0; h < args.n_hc; ++h) { + const float xv = *((device const float *) (x + d*args.nb_x0 + h*args.nb_x1 + t*args.nb_x2)); + const float wv = *((device const float *) (weights + h*args.nb_w0 + t*args.nb_w1)); + acc += xv * wv; + } + + *((device float *) (dst + d*args.nb0 + t*args.nb1)) = acc; + if (t + 1 == args.n_tokens) { + *((device float *) (last_dst + d*args.nb0)) = acc; + } +} + // The one-row HC=4 output head historically materializes four device-F32 // stages across separate launches. Collapse those launches into one tiny // two-thread group while preserving the scalar/vector lane mapping and every diff --git a/tests/dspark_acceptance_fixture.sh b/tests/dspark_acceptance_fixture.sh index 8a19d022c..4cac4f7d5 100644 --- a/tests/dspark_acceptance_fixture.sh +++ b/tests/dspark_acceptance_fixture.sh @@ -30,6 +30,7 @@ REQUIRE_ACTIVE=${DS4_DSPARK_FIXTURE_REQUIRE_ACTIVE:-1} REQUIRE_EXACT2=${DS4_DSPARK_FIXTURE_REQUIRE_EXACT2:-0} REQUIRE_CUDA_EXACTN=${DS4_DSPARK_FIXTURE_REQUIRE_CUDA_EXACTN:-0} REQUIRE_CUDA_EXACTN_BATCH_HEAD=${DS4_DSPARK_FIXTURE_REQUIRE_CUDA_EXACTN_BATCH_HEAD:-0} +REQUIRE_CUDA_EXACTN_GRAPHS=${DS4_DSPARK_FIXTURE_REQUIRE_CUDA_EXACTN_GRAPHS:-0} REQUIRE_CUDA_DEVICE_PROPOSER=${DS4_DSPARK_FIXTURE_REQUIRE_CUDA_DEVICE_PROPOSER:-0} REQUIRE_METAL_EXACTN_BATCH_HEAD=${DS4_DSPARK_FIXTURE_REQUIRE_METAL_EXACTN_BATCH_HEAD:-0} REQUIRE_METAL_EXACTN_PARTIAL=${DS4_DSPARK_FIXTURE_REQUIRE_METAL_EXACTN_PARTIAL:-0} @@ -44,6 +45,13 @@ total_cuda_exactn_error_fallback=0 total_cuda_exactn_batch_head_attempt=0 total_cuda_exactn_batch_head_use=0 total_cuda_exactn_batch_head_fallback=0 +total_cuda_exactn_graph_attempt=0 +total_cuda_exactn_graph_use=0 +total_cuda_exactn_graph_capture=0 +total_cuda_exactn_graph_replay=0 +total_cuda_exactn_graph_warm=0 +total_cuda_exactn_graph_no_slot=0 +total_cuda_exactn_graph_failure=0 total_cuda_device_proposer_attempt=0 total_cuda_device_proposer_use=0 total_cuda_device_proposer_fallback=0 @@ -116,6 +124,16 @@ esac if [ "$REQUIRE_CUDA_EXACTN_BATCH_HEAD" != 0 ]; then REQUIRE_CUDA_EXACTN=1 fi +case "$REQUIRE_CUDA_EXACTN_GRAPHS" in +0|1) ;; +*) + echo "dspark-fixture: DS4_DSPARK_FIXTURE_REQUIRE_CUDA_EXACTN_GRAPHS must be 0 or 1" >&2 + exit 1 + ;; +esac +if [ "$REQUIRE_CUDA_EXACTN_GRAPHS" != 0 ]; then + REQUIRE_CUDA_EXACTN=1 +fi case "$REQUIRE_CUDA_DEVICE_PROPOSER" in 0|1) ;; *) @@ -244,6 +262,8 @@ print_metadata() { exactn_cuda_disable=${DS4_CUDA_DISABLE_DSPARK_EXACTN:-unset} exactn_cuda_batch_head=${DS4_CUDA_DSPARK_EXACTN_BATCH_HEAD:-unset} exactn_cuda_batch_head_disable=${DS4_CUDA_DISABLE_DSPARK_EXACTN_BATCH_HEAD:-unset} + exactn_cuda_graphs=${DS4_CUDA_DSPARK_EXACTN_GRAPHS:-unset} + exactn_cuda_graphs_disable=${DS4_CUDA_DISABLE_DSPARK_EXACTN_GRAPHS:-unset} cuda_device_proposer=${DS4_CUDA_DSPARK_DEVICE_PROPOSER:-unset} cuda_device_proposer_disable=${DS4_CUDA_DSPARK_NO_DEVICE_PROPOSER:-unset} exactn_union_metal=${DS4_METAL_DSPARK_EXACTN_UNION:-unset} @@ -290,6 +310,9 @@ print_metadata() { "$exactn_union_metal" "$noncausal_online_cuda" \ "$noncausal_online_cuda_disable" "$verify_noncausal" \ "$exact_rows_async_tails_metal" + printf '# exactn_cuda_graphs=%s exactn_cuda_graphs_disable=%s require_cuda_exactn_graphs=%s\n' \ + "$exactn_cuda_graphs" "$exactn_cuda_graphs_disable" \ + "$REQUIRE_CUDA_EXACTN_GRAPHS" printf '# exactn_metal_batch_head=%s exactn_metal_batch_head_disable=%s require_metal_exactn_batch_head=%s require_metal_exactn_partial=%s metal_device_proposer=%s metal_device_proposer_disable=%s require_metal_device_proposer=%s\n' \ "$exactn_metal_batch_head" "$exactn_metal_batch_head_disable" \ "$REQUIRE_METAL_EXACTN_BATCH_HEAD" "$REQUIRE_METAL_EXACTN_PARTIAL" \ @@ -398,6 +421,13 @@ run_case() { cuda_exactn_batch_head_attempt=$(stats_field "$stats" cuda_exactn_batch_head_attempt) cuda_exactn_batch_head_use=$(stats_field "$stats" cuda_exactn_batch_head_use) cuda_exactn_batch_head_fallback=$(stats_field "$stats" cuda_exactn_batch_head_fallback) + cuda_exactn_graph_attempt=$(stats_field "$stats" cuda_exactn_graph_attempt) + cuda_exactn_graph_use=$(stats_field "$stats" cuda_exactn_graph_use) + cuda_exactn_graph_capture=$(stats_field "$stats" cuda_exactn_graph_capture) + cuda_exactn_graph_replay=$(stats_field "$stats" cuda_exactn_graph_replay) + cuda_exactn_graph_warm=$(stats_field "$stats" cuda_exactn_graph_warm) + cuda_exactn_graph_no_slot=$(stats_field "$stats" cuda_exactn_graph_no_slot) + cuda_exactn_graph_failure=$(stats_field "$stats" cuda_exactn_graph_failure) cuda_device_proposer_attempt=$(stats_field "$stats" cuda_device_proposer_attempt) cuda_device_proposer_use=$(stats_field "$stats" cuda_device_proposer_use) cuda_device_proposer_fallback=$(stats_field "$stats" cuda_device_proposer_fallback) @@ -431,6 +461,13 @@ run_case() { cuda_exactn_batch_head_attempt=${cuda_exactn_batch_head_attempt:-0} cuda_exactn_batch_head_use=${cuda_exactn_batch_head_use:-0} cuda_exactn_batch_head_fallback=${cuda_exactn_batch_head_fallback:-0} + cuda_exactn_graph_attempt=${cuda_exactn_graph_attempt:-0} + cuda_exactn_graph_use=${cuda_exactn_graph_use:-0} + cuda_exactn_graph_capture=${cuda_exactn_graph_capture:-0} + cuda_exactn_graph_replay=${cuda_exactn_graph_replay:-0} + cuda_exactn_graph_warm=${cuda_exactn_graph_warm:-0} + cuda_exactn_graph_no_slot=${cuda_exactn_graph_no_slot:-0} + cuda_exactn_graph_failure=${cuda_exactn_graph_failure:-0} cuda_device_proposer_attempt=${cuda_device_proposer_attempt:-0} cuda_device_proposer_use=${cuda_device_proposer_use:-0} cuda_device_proposer_fallback=${cuda_device_proposer_fallback:-0} @@ -467,6 +504,13 @@ run_case() { total_cuda_exactn_batch_head_attempt=$((total_cuda_exactn_batch_head_attempt + cuda_exactn_batch_head_attempt)) total_cuda_exactn_batch_head_use=$((total_cuda_exactn_batch_head_use + cuda_exactn_batch_head_use)) total_cuda_exactn_batch_head_fallback=$((total_cuda_exactn_batch_head_fallback + cuda_exactn_batch_head_fallback)) + total_cuda_exactn_graph_attempt=$((total_cuda_exactn_graph_attempt + cuda_exactn_graph_attempt)) + total_cuda_exactn_graph_use=$((total_cuda_exactn_graph_use + cuda_exactn_graph_use)) + total_cuda_exactn_graph_capture=$((total_cuda_exactn_graph_capture + cuda_exactn_graph_capture)) + total_cuda_exactn_graph_replay=$((total_cuda_exactn_graph_replay + cuda_exactn_graph_replay)) + total_cuda_exactn_graph_warm=$((total_cuda_exactn_graph_warm + cuda_exactn_graph_warm)) + total_cuda_exactn_graph_no_slot=$((total_cuda_exactn_graph_no_slot + cuda_exactn_graph_no_slot)) + total_cuda_exactn_graph_failure=$((total_cuda_exactn_graph_failure + cuda_exactn_graph_failure)) total_cuda_device_proposer_attempt=$((total_cuda_device_proposer_attempt + cuda_device_proposer_attempt)) total_cuda_device_proposer_use=$((total_cuda_device_proposer_use + cuda_device_proposer_use)) total_cuda_device_proposer_fallback=$((total_cuda_device_proposer_fallback + cuda_device_proposer_fallback)) @@ -580,6 +624,16 @@ if [ "$REQUIRE_CUDA_EXACTN_BATCH_HEAD" != 0 ]; then "$total_cuda_exactn_batch_head_use" \ "$total_cuda_exactn_batch_head_fallback" fi +if [ "$REQUIRE_CUDA_EXACTN_GRAPHS" != 0 ]; then + printf '# cuda_exactn_graph_attempt=%s cuda_exactn_graph_use=%s cuda_exactn_graph_capture=%s cuda_exactn_graph_replay=%s cuda_exactn_graph_warm=%s cuda_exactn_graph_no_slot=%s cuda_exactn_graph_failure=%s\n' \ + "$total_cuda_exactn_graph_attempt" \ + "$total_cuda_exactn_graph_use" \ + "$total_cuda_exactn_graph_capture" \ + "$total_cuda_exactn_graph_replay" \ + "$total_cuda_exactn_graph_warm" \ + "$total_cuda_exactn_graph_no_slot" \ + "$total_cuda_exactn_graph_failure" +fi if [ "$REQUIRE_CUDA_DEVICE_PROPOSER" != 0 ]; then printf '# cuda_device_proposer_attempt=%s cuda_device_proposer_use=%s cuda_device_proposer_fallback=%s cuda_device_proposer_policy_mismatch=%s\n' \ "$total_cuda_device_proposer_attempt" \ @@ -623,6 +677,16 @@ if [ "$REQUIRE_CUDA_EXACTN_BATCH_HEAD" != 0 ] && echo "dspark-fixture: CUDA exact-N batch head not cleanly exercised (attempt=$total_cuda_exactn_batch_head_attempt use=$total_cuda_exactn_batch_head_use fallback=$total_cuda_exactn_batch_head_fallback)" >&2 exit 1 fi +if [ "$REQUIRE_CUDA_EXACTN_GRAPHS" != 0 ] && + { [ "$total_cuda_exactn_graph_attempt" -eq 0 ] || + [ "$total_cuda_exactn_graph_use" -eq 0 ] || + [ "$total_cuda_exactn_graph_capture" -eq 0 ] || + [ "$total_cuda_exactn_graph_replay" -eq 0 ] || + [ "$total_cuda_exactn_graph_no_slot" -ne 0 ] || + [ "$total_cuda_exactn_graph_failure" -ne 0 ]; }; then + echo "dspark-fixture: CUDA exact-N graphs were not cleanly replayed after warmup (attempt=$total_cuda_exactn_graph_attempt use=$total_cuda_exactn_graph_use warm=$total_cuda_exactn_graph_warm capture=$total_cuda_exactn_graph_capture replay=$total_cuda_exactn_graph_replay no_slot=$total_cuda_exactn_graph_no_slot failure=$total_cuda_exactn_graph_failure)" >&2 + exit 1 +fi if [ "$REQUIRE_CUDA_DEVICE_PROPOSER" != 0 ] && { [ "$total_cuda_device_proposer_attempt" -eq 0 ] || [ "$total_cuda_device_proposer_use" -eq 0 ] || diff --git a/tests/test_engine_mgpu_placement.c b/tests/test_engine_mgpu_placement.c index 0cf861ce6..89c4cf685 100644 --- a/tests/test_engine_mgpu_placement.c +++ b/tests/test_engine_mgpu_placement.c @@ -41,6 +41,27 @@ int ds4_test_classify_multi_tier(const ds4_test_fake_tensor *tensors, int ds4_test_tensor_to_entry(const char *name, int name_len); int ds4_test_dspark_runtime_policy(ds4_backend backend, ds4_distributed_role distributed_role); +bool ds4_test_streaming_manual_cache_cap_count( + uint32_t requested_count, + uint64_t per_expert_bytes, + uint64_t safe_cache_bytes, + uint32_t *effective_count_out, + uint64_t *requested_bytes_out, + uint64_t *effective_bytes_out); +bool ds4_test_streaming_manual_cache_count_cap_enabled(void); +bool ds4_test_streaming_manual_cache_count_cap_eligible( + ds4_backend backend, + bool ssd_streaming, + bool dspark_enabled, + bool support_is_dspark, + uint32_t cache_experts, + uint64_t cache_bytes); +uint64_t ds4_test_streaming_dspark_active_support_reserve_bytes( + uint64_t support_model_bytes); +uint32_t ds4_test_streaming_manual_cache_nonfatal_effective_count( + uint32_t requested_count, + bool safe_cache_known, + uint32_t candidate_count); /* Ctx-aware variants and calibration helpers. Declared here (not in * ds4.h) matching the existing DS4_TEST_HOOKS pattern. */ @@ -600,6 +621,113 @@ static void test_glm_memory_guard_budget(void) { restore_env_value("DS4_GLM_MEMORY_GUARD", old_guard); } +static void test_streaming_manual_cache_count_cap(void) { + fprintf(stderr, "RUN: test_streaming_manual_cache_count_cap\n"); + const uint64_t mib = 1024ull * 1024ull; + const uint64_t per_expert = 7ull * mib; + uint32_t count = UINT32_MAX; + uint64_t requested = UINT64_MAX; + uint64_t effective = UINT64_MAX; + + CHECK(ds4_test_streaming_manual_cache_cap_count( + 100, per_expert, 100ull * per_expert, + &count, &requested, &effective), + "numeric cache count converts to bytes without a cap"); + CHECK(count == 100, "safe numeric cache count is preserved"); + CHECK(requested == 100ull * per_expert, + "numeric request byte conversion is exact"); + CHECK(effective == requested, + "uncapped effective bytes equal requested bytes"); + + CHECK(ds4_test_streaming_manual_cache_cap_count( + 100, per_expert, 15ull * per_expert + per_expert / 2ull, + &count, &requested, &effective), + "numeric cache count accepts a fractional-slot byte limit"); + CHECK(count == 15, "safe byte limit rounds down to whole expert slots"); + CHECK(effective == 15ull * per_expert, + "capped count converts back to exact bytes"); + + CHECK(ds4_test_streaming_manual_cache_cap_count( + 3, per_expert, per_expert - 1ull, + &count, &requested, &effective), + "sub-slot safe limit is represented without overflow"); + CHECK(count == 0 && effective == 0, + "sub-slot safe limit never forces one unsafe expert"); + + count = 123; + requested = 456; + effective = 789; + CHECK(!ds4_test_streaming_manual_cache_cap_count( + UINT32_MAX, UINT64_MAX, UINT64_MAX, + &count, &requested, &effective), + "numeric request multiplication overflow is rejected"); + CHECK(count == 0 && requested == 0 && effective == 0, + "overflow failure clears all conversion outputs"); + + const char *env = "DS4_METAL_DSPARK_SAFE_EXPERT_COUNT"; + char *saved = save_env_value(env); + unsetenv(env); + CHECK(!ds4_test_streaming_manual_cache_count_cap_enabled(), + "numeric cache safety cap is disabled by default during A/B"); + setenv(env, "0", 1); + CHECK(!ds4_test_streaming_manual_cache_count_cap_enabled(), + "zero does not enable the numeric cache safety cap"); + setenv(env, "true", 1); + CHECK(!ds4_test_streaming_manual_cache_count_cap_enabled(), + "ambiguous truthy text does not enable the safety cap"); + setenv(env, "1", 1); + CHECK(ds4_test_streaming_manual_cache_count_cap_enabled(), + "the documented exact value 1 enables the safety cap"); + restore_env_value(env, saved); + + CHECK(ds4_test_streaming_manual_cache_count_cap_eligible( + DS4_BACKEND_METAL, true, true, true, 1032, 0), + "Metal SSD+DSpark numeric count is eligible for the opt-in cap"); + CHECK(!ds4_test_streaming_manual_cache_count_cap_eligible( + DS4_BACKEND_CUDA, true, true, true, 1032, 0), + "CUDA numeric count is not eligible"); + CHECK(!ds4_test_streaming_manual_cache_count_cap_eligible( + DS4_BACKEND_METAL, false, true, true, 1032, 0), + "non-streaming Metal is not eligible"); + CHECK(!ds4_test_streaming_manual_cache_count_cap_eligible( + DS4_BACKEND_METAL, true, false, true, 1032, 0), + "Metal support model without --dspark is not eligible"); + CHECK(!ds4_test_streaming_manual_cache_count_cap_eligible( + DS4_BACKEND_METAL, true, true, false, 1032, 0), + "non-DSpark support model is not eligible"); + CHECK(!ds4_test_streaming_manual_cache_count_cap_eligible( + DS4_BACKEND_METAL, true, true, true, 0, 0), + "automatic cache selection is not a numeric-count request"); + CHECK(!ds4_test_streaming_manual_cache_count_cap_eligible( + DS4_BACKEND_METAL, true, true, true, 0, 32ull << 30), + "NGB cache budget is not eligible"); + + const uint64_t gib = 1024ull * 1024ull * 1024ull; + CHECK(ds4_test_streaming_dspark_active_support_reserve_bytes(0) == 0, + "absent support mapping reserves zero bytes"); + CHECK(ds4_test_streaming_dspark_active_support_reserve_bytes(gib) == gib, + "small support mapping is reserved in full"); + CHECK(ds4_test_streaming_dspark_active_support_reserve_bytes(2ull * gib) == + 2ull * gib, + "support reserve includes the exact 2 GiB boundary"); + CHECK(ds4_test_streaming_dspark_active_support_reserve_bytes(6ull * gib) == + 2ull * gib, + "large mmap-backed support model is limited to 2 GiB active reserve"); + + CHECK(ds4_test_streaming_manual_cache_nonfatal_effective_count( + 1032, true, 341) == 341, + "measured non-zero candidate may cap a numeric request"); + CHECK(ds4_test_streaming_manual_cache_nonfatal_effective_count( + 1032, false, 341) == 1032, + "unknown safe budget preserves the explicit request"); + CHECK(ds4_test_streaming_manual_cache_nonfatal_effective_count( + 1032, true, 0) == 1032, + "sub-slot safe budget remains non-fatal"); + CHECK(ds4_test_streaming_manual_cache_nonfatal_effective_count( + 100, true, 200) == 100, + "policy never grows an explicit numeric request"); +} + static void test_cuda_tp_prefill_default_accounting(void) { fprintf(stderr, "RUN: test_cuda_tp_prefill_default_accounting\n"); @@ -734,6 +862,7 @@ int main(void) { test_glm_per_layer_cache_accounting(); test_glm_session_count_accounting(); test_glm_memory_guard_budget(); + test_streaming_manual_cache_count_cap(); test_cuda_tp_prefill_default_accounting(); test_cuda_tp_output_head_moves_to_lower_half(); diff --git a/tests/test_metal_dspark_capture.c b/tests/test_metal_dspark_capture.c new file mode 100644 index 000000000..dcbc850bf --- /dev/null +++ b/tests/test_metal_dspark_capture.c @@ -0,0 +1,204 @@ +#define _DARWIN_C_SOURCE + +#include "ds4_gpu.h" + +#include +#include +#include +#include +#include + +enum { + TEST_HC = 4, + GUARD_FLOATS = 11, +}; + +static const uint32_t guard_bits = 0x7fc12345u; + +bool ds4_log_is_tty(FILE *fp) { + (void)fp; + return false; +} + +static void fill_guard(uint32_t *dst, size_t count) { + for (size_t i = 0; i < count; i++) dst[i] = guard_bits; +} + +static int guard_is_intact(const char *name, + const uint32_t *storage, + size_t payload_floats) { + for (size_t i = 0; i < GUARD_FLOATS; i++) { + if (storage[i] != guard_bits) { + fprintf(stderr, "%s prefix guard changed at %zu: 0x%08x\n", + name, i, storage[i]); + return 0; + } + } + for (size_t i = 0; i < GUARD_FLOATS; i++) { + const size_t at = GUARD_FLOATS + payload_floats + i; + if (storage[at] != guard_bits) { + fprintf(stderr, "%s suffix guard changed at %zu: 0x%08x\n", + name, i, storage[at]); + return 0; + } + } + return 1; +} + +static ds4_gpu_tensor *make_guarded_tensor(size_t payload_floats, + uint32_t **host_storage) { + const size_t total = payload_floats + 2u * GUARD_FLOATS; + uint32_t *storage = malloc(total * sizeof(*storage)); + if (!storage) return NULL; + fill_guard(storage, total); + + ds4_gpu_tensor *base = ds4_gpu_tensor_alloc(total * sizeof(float)); + if (!base || !ds4_gpu_tensor_write(base, 0, storage, + total * sizeof(*storage))) { + ds4_gpu_tensor_free(base); + free(storage); + return NULL; + } + *host_storage = storage; + return base; +} + +static int read_guarded_tensor(ds4_gpu_tensor *base, + uint32_t *storage, + size_t payload_floats) { + const size_t total = payload_floats + 2u * GUARD_FLOATS; + return ds4_gpu_tensor_read(base, 0, storage, + total * sizeof(*storage)); +} + +static int run_case(uint32_t rows, uint32_t n_embd) { + const size_t x_floats = (size_t)rows * TEST_HC * n_embd; + const size_t weights_floats = (size_t)rows * TEST_HC; + const size_t out_floats = (size_t)rows * n_embd; + const size_t last_floats = n_embd; + uint32_t *x_storage = NULL; + uint32_t *weights_storage = NULL; + uint32_t *ref_storage = NULL; + uint32_t *fused_storage = NULL; + uint32_t *last_storage = NULL; + ds4_gpu_tensor *x_base = make_guarded_tensor(x_floats, &x_storage); + ds4_gpu_tensor *weights_base = + make_guarded_tensor(weights_floats, &weights_storage); + ds4_gpu_tensor *ref_base = make_guarded_tensor(out_floats, &ref_storage); + ds4_gpu_tensor *fused_base = + make_guarded_tensor(out_floats, &fused_storage); + ds4_gpu_tensor *last_base = + make_guarded_tensor(last_floats, &last_storage); + ds4_gpu_tensor *x = NULL; + ds4_gpu_tensor *weights = NULL; + ds4_gpu_tensor *ref = NULL; + ds4_gpu_tensor *fused = NULL; + ds4_gpu_tensor *last = NULL; + int ok = x_base && weights_base && ref_base && fused_base && last_base; + + if (ok) { + float *x_values = (float *)(x_storage + GUARD_FLOATS); + float *weight_values = (float *)(weights_storage + GUARD_FLOATS); + for (size_t i = 0; i < x_floats; i++) { + const int32_t centered = (int32_t)((i * 37u + 13u) % 257u) - 128; + x_values[i] = (float)centered / 32.0f; + } + for (size_t i = 0; i < weights_floats; i++) { + weight_values[i] = 0.25f; + } + ok = ds4_gpu_tensor_write(x_base, 0, x_storage, + (x_floats + 2u * GUARD_FLOATS) * sizeof(float)) && + ds4_gpu_tensor_write(weights_base, 0, weights_storage, + (weights_floats + 2u * GUARD_FLOATS) * sizeof(float)); + } + if (ok) { + x = ds4_gpu_tensor_view(x_base, + (uint64_t)GUARD_FLOATS * sizeof(float), + (uint64_t)x_floats * sizeof(float)); + weights = ds4_gpu_tensor_view(weights_base, + (uint64_t)GUARD_FLOATS * sizeof(float), + (uint64_t)weights_floats * sizeof(float)); + ref = ds4_gpu_tensor_view(ref_base, + (uint64_t)GUARD_FLOATS * sizeof(float), + (uint64_t)out_floats * sizeof(float)); + fused = ds4_gpu_tensor_view(fused_base, + (uint64_t)GUARD_FLOATS * sizeof(float), + (uint64_t)out_floats * sizeof(float)); + last = ds4_gpu_tensor_view(last_base, + (uint64_t)GUARD_FLOATS * sizeof(float), + (uint64_t)last_floats * sizeof(float)); + ok = x && weights && ref && fused && last; + } + if (ok) { + ok = ds4_gpu_hc_weighted_sum_tensor(ref, x, weights, + n_embd, TEST_HC) && + ds4_gpu_hc_weighted_sum_capture_last_tensor(fused, last, + x, weights, + n_embd, TEST_HC) && + ds4_gpu_synchronize(); + } + if (ok) { + ok = read_guarded_tensor(ref_base, ref_storage, out_floats) && + read_guarded_tensor(fused_base, fused_storage, out_floats) && + read_guarded_tensor(last_base, last_storage, last_floats) && + read_guarded_tensor(x_base, x_storage, x_floats) && + read_guarded_tensor(weights_base, weights_storage, weights_floats); + } + if (ok && memcmp(ref_storage + GUARD_FLOATS, + fused_storage + GUARD_FLOATS, + out_floats * sizeof(float)) != 0) { + fprintf(stderr, "DSpark capture batch mismatch rows=%u embd=%u\n", + rows, n_embd); + ok = 0; + } + if (ok && memcmp(ref_storage + GUARD_FLOATS + + (size_t)(rows - 1u) * n_embd, + last_storage + GUARD_FLOATS, + last_floats * sizeof(float)) != 0) { + fprintf(stderr, "DSpark capture last-row mismatch rows=%u embd=%u\n", + rows, n_embd); + ok = 0; + } + if (ok) { + ok = guard_is_intact("input", x_storage, x_floats) && + guard_is_intact("weights", weights_storage, weights_floats) && + guard_is_intact("reference", ref_storage, out_floats) && + guard_is_intact("fused batch", fused_storage, out_floats) && + guard_is_intact("fused last", last_storage, last_floats); + } + + ds4_gpu_tensor_free(last); + ds4_gpu_tensor_free(fused); + ds4_gpu_tensor_free(ref); + ds4_gpu_tensor_free(weights); + ds4_gpu_tensor_free(x); + ds4_gpu_tensor_free(last_base); + ds4_gpu_tensor_free(fused_base); + ds4_gpu_tensor_free(ref_base); + ds4_gpu_tensor_free(weights_base); + ds4_gpu_tensor_free(x_base); + free(last_storage); + free(fused_storage); + free(ref_storage); + free(weights_storage); + free(x_storage); + + if (ok) { + fprintf(stderr, "DSpark Metal capture rows=%u embd=%u bitwise PASS\n", + rows, n_embd); + } + return ok; +} + +int main(void) { + int ok = ds4_gpu_init(); + const uint32_t rows[] = {1u, 2u, 5u}; + const uint32_t embd[] = {17u, 4096u}; + for (size_t d = 0; ok && d < sizeof(embd) / sizeof(embd[0]); d++) { + for (size_t r = 0; ok && r < sizeof(rows) / sizeof(rows[0]); r++) { + ok = run_case(rows[r], embd[d]); + } + } + ds4_gpu_cleanup(); + return ok ? 0 : 1; +} From d89028f82e99a5e7c13a2ae2cd0aa6e8ac94f4f6 Mon Sep 17 00:00:00 2001 From: Giorgio Oppo Date: Tue, 11 Aug 2026 11:33:20 +0200 Subject: [PATCH 018/189] cuda: preserve canonical Q4 GB10 numerics --- QA_BEFORE_RELEASES.md | 42 ++- README.md | 20 +- cuda/mmq/test/test_mmq_parity.cu | 13 +- ds4_cuda.cu | 30 +- tests/cuda_q4_gb10_fast_matrix.sh | 449 ++++++++++++++++++++++++++++++ 5 files changed, 530 insertions(+), 24 deletions(-) create mode 100755 tests/cuda_q4_gb10_fast_matrix.sh diff --git a/QA_BEFORE_RELEASES.md b/QA_BEFORE_RELEASES.md index b1f171487..4cd7a8d32 100644 --- a/QA_BEFORE_RELEASES.md +++ b/QA_BEFORE_RELEASES.md @@ -439,7 +439,7 @@ than a failure. `--dspark-strict` remains the byte-identical target-only mode. `DS4_CUDA_DISABLE_HC_NORM_MIX_FUSE=1` versus candidate `DS4_CUDA_ENABLE_HC_NORM_MIX_FUSE=1 DS4_CUDA_NO_F16_CUBLAS_ONE=1`; and Q4 attention-output/HC control `DS4_CUDA_DISABLE_Q4_ATTN_OUT_HC_FUSE=1` versus - MMVQ-safe candidate `DS4_CUDA_ENABLE_Q4_ATTN_OUT_HC_FUSE=1`. Run a separate + the graph-compatible canonical candidate with that variable absent. Run a separate non-captured diagnostic with `DS4_CUDA_Q4_ATTN_OUT_HC_ORACLE=1`; require the summary to be present with `calls>0`, `skips=0`, and `epilogue_mismatches=0`, while `q8k_mismatches` records the expected @@ -994,6 +994,46 @@ Do not use high-performance Hugging Face Xet mode while vLLM is resident. vocabulary, compressor, HC split, direct routed-MoE paths, Q4 scratch, grouped attention-A, and canonical B+HC epilogue remain relevant, while the Q8-only attention-projection consumers are intentionally ineligible. +- If the umbrella AProjQ4 A/B changes logits, do not attribute that change to + "the Q4 fast path" as a unit. Run the fail-closed component matrix from a + clean `cuda-spark` build. The output directory is intentionally explicit so + the six independent-process arms, two oracle arms, raw logs, and diffs are + retained: + + ```sh + make clean && make cuda-spark + make gguf-tools/quality-testing/score_official CUDA_ARCH=sm_121 + + DS4_CUDA_Q4_MATRIX_SSD_STREAMING=1 \ + DS4_CUDA_Q4_MATRIX_SSD_CACHE=16GB \ + DS4_CUDA_Q4_MATRIX_DECODE_GRAPHS=1 \ + tests/cuda_q4_gb10_fast_matrix.sh \ + /path/to/DeepSeek-V4-Flash-AProjQ4-OutQ8.gguf \ + gguf-tools/quality-testing/data/flash/manifest.tsv \ + /tmp/q4-gb10-graphs-on + + DS4_CUDA_Q4_MATRIX_SSD_STREAMING=1 \ + DS4_CUDA_Q4_MATRIX_SSD_CACHE=16GB \ + DS4_CUDA_Q4_MATRIX_DECODE_GRAPHS=0 \ + tests/cuda_q4_gb10_fast_matrix.sh \ + /path/to/DeepSeek-V4-Flash-AProjQ4-OutQ8.gguf \ + gguf-tools/quality-testing/data/flash/manifest.tsv \ + /tmp/q4-gb10-graphs-off + ``` + + The matrix first proves that the three local rollback switches reproduce + `DS4_CUDA_NO_Q4_GB10_FAST=1`; failure of `local_control` means the matrix is + incomplete and no component claim is valid. It then enables exactly one of + persistent Q8_1 scratch, grouped attention-A, or the graph-compatible B+HC + call, with K1024 persistent kept disabled because it is a separate opt-in. + The grouped and HC oracle summaries must have `calls>0`, `skips=0`, and zero + relevant mismatches. `summary.txt` must say `promotion_gate=pass`. When it + is blocked, use the named `*_differences` arms and their `.diff` or + `.comparison.txt` files to identify a component; if no single arm differs + but `default_fast` does, report an interaction rather than blaming an + individual kernel. The tensor oracles and synthetic parity test are the + bit-exact component gates; the top-128 smoke dump and every scorer TSV row + are complementary end-to-end drift detectors, not a full-logit proof. - Exercise CUDA DSpark at verifier/proposer depth 5 with the fast paths enabled and disabled. Require identical final output, zero verifier errors, and matching full/partial acceptance histograms. Test both the generic batch diff --git a/README.md b/README.md index bf42c0924..70da4679d 100644 --- a/README.md +++ b/README.md @@ -715,8 +715,9 @@ arithmetic: dispatch while keeping `ncols_dst=1`; `DS4_CUDA_NO_Q4_GROUPED_ATTN_A_BATCH=1` restores the per-token grouped loop and `DS4_CUDA_NO_Q4_GROUPED_ATTN_A=1` restores the per-group loop; -- attention-output B keeps its canonical MMVQ result and automatically uses - the row-packed HC epilogue described below; +- attention-output B keeps its canonical MMVQ result and the ordinary HC + epilogue inside the graph-compatible fused call. The row-packed epilogue + remains oracle-only until a GB10 device run proves it bit-exact; - the exact Q-b shape `32768x1024` has an experimental persistent-CTA kernel behind `DS4_CUDA_ENABLE_Q4_K1024_PERSISTENT=1`, with `DS4_CUDA_NO_Q4_K1024_PERSISTENT=1` taking precedence. Tests can add @@ -749,11 +750,11 @@ reduction order; with the normal one-token cuBLAS path, also set `DS4_CUDA_DISABLE_Q4_ATTN_OUT_HC_FUSE=1`. The Q4 attention-output B plus HC path is automatic when MMQ is disabled, where the existing one-dispatch Q8_K implementation is bit-compatible with its fallback. With the normal -MMVQ/Q8_1 decode path, `DS4_CUDA_ENABLE_Q4_ATTN_OUT_HC_FUSE=1` now preserves -the canonical MMVQ projection and replaces only the following HC expansion -with a row-packed epilogue that reads each projected row once. It therefore -does not change activation quantization. The older, truly single-dispatch -Q8_K experiment is isolated behind +MMVQ/Q8_1 decode path, the GB10 graph-compatible call preserves both the +canonical MMVQ projection and the ordinary HC expansion. The specialized +row-packed epilogue is evaluated only by the oracle below and is never +consumed by normal decoding. The older, truly single-dispatch Q8_K experiment +is isolated behind `DS4_CUDA_Q4_ATTN_OUT_HC_Q8K_EXPERIMENT=1` and may differ numerically from MMVQ/Q8_1. @@ -763,8 +764,9 @@ compares both the row-packed epilogue and the Q8_K compound bit-for-bit, and prints `epilogue_mismatches`, `q8k_mismatches`, and `skips` at exit. The oracle avoids readback while CUDA graph capture is active; run a separate non-captured diagnostic and require `calls>0`, `skips=0`, and -`epilogue_mismatches=0` before promoting the row-packed path. A zero-call -summary is therefore an explicit failed coverage gate, not a silent pass. +`epilogue_mismatches=0` before promoting the row-packed path back into normal +decoding. A zero-call summary is therefore an explicit failed coverage gate, +not a silent pass. An experimental resident-CUDA path can run the existing aligned IQ2_XXS/Q2_K vector MoE kernels for two-to-five-draft routed batches, diff --git a/cuda/mmq/test/test_mmq_parity.cu b/cuda/mmq/test/test_mmq_parity.cu index 98f946e5a..6cb4d1f8b 100644 --- a/cuda/mmq/test/test_mmq_parity.cu +++ b/cuda/mmq/test/test_mmq_parity.cu @@ -1490,12 +1490,23 @@ int main(int argc, char ** argv) { /*M=*/64, /*N=*/1, /*K=*/512, /*groups=*/4, 0xC4FE45); all_ok &= run_q4_K_grouped_vec_parity( /*M=*/256, /*N=*/2, /*K=*/8192, /*groups=*/4, 0xC4FE42); - // DeepSeek-V4 Flash AProjQ4 attention-A production shape at DSpark N=5. + // Small synthetic Flash-like case for quick token-aware coverage. all_ok &= run_q4_K_grouped_vec_parity( /*M=*/128, /*N=*/5, /*K=*/4096, /*groups=*/8, 0xC4FE43); + // DeepSeek-V4 Flash AProjQ4 attention-A production shape: each of the + // eight output groups owns a [rank=1024, group_dim=4096] Q4_K matrix. + // Cover both ordinary one-token decode and the maximum DSpark proposal + // width used by the verifier. + all_ok &= run_q4_K_grouped_vec_parity( + /*M=*/1024, /*N=*/1, /*K=*/4096, /*groups=*/8, 0xC4FE46); + all_ok &= run_q4_K_grouped_vec_parity( + /*M=*/1024, /*N=*/5, /*K=*/4096, /*groups=*/8, 0xC4FE47); // Pro-style maximum group count and the N=8 API ceiling. all_ok &= run_q4_K_grouped_vec_parity( /*M=*/64, /*N=*/8, /*K=*/4096, /*groups=*/16, 0xC4FE44); + // DeepSeek-V4 Pro has the same rank/group_dim as Flash and 16 groups. + all_ok &= run_q4_K_grouped_vec_parity( + /*M=*/1024, /*N=*/1, /*K=*/4096, /*groups=*/16, 0xC4FE48); fprintf(stderr, "===================\n"); fprintf(stderr, "%s\n", all_ok ? "ALL PASS" : "SOME FAILED"); diff --git a/ds4_cuda.cu b/ds4_cuda.cu index dc7060cb0..8eb38e44f 100644 --- a/ds4_cuda.cu +++ b/ds4_cuda.cu @@ -34009,11 +34009,13 @@ __global__ static void matmul_q4_K_hc_expand4_kernel( } } -/* HC epilogue for the canonical MMVQ/Q8_1 Q4_K path. MMVQ materializes - * block_out using its established activation quantizer and reduction order; - * one thread per embedding row then reuses that value for all four HC - * destinations. Each destination retains hc_expand_kernel's exact - * multiply/add order, while block_out is read once instead of four times. */ +/* Diagnostic row-packed HC epilogue for the canonical MMVQ/Q8_1 Q4_K path. + * MMVQ materializes block_out using its established activation quantizer and + * reduction order; one thread per embedding row then reuses that value for + * all four HC destinations. Its source-level multiply/add sequence mirrors + * hc_expand_kernel, but full unrolling under --use_fast_math is not assumed + * bit-exact. Normal decoding uses hc_expand_kernel; the oracle below is the + * only consumer of this candidate until device parity is proven. */ __global__ static void q4_K_hc_expand4_rows_kernel( float *out_hc, const float *block_out, @@ -34364,11 +34366,13 @@ extern "C" int ds4_gpu_matmul_q4_K_hc_expand_available(void) { * zero-call summary that makes an ineligible/unused A/B visible. */ cuda_q4_attn_hc_oracle_register_report(); } - /* On GB10 the default candidate keeps the ordinary Q4_K MMVQ result and - * only replaces the four-way HC launch with a row-packed epilogue. The - * existing disable variable (plus DS4_CUDA_NO_Q4_GB10_FAST) is a complete - * rollback to the caller's separate B projection + HC expansion. Legacy - * explicit experiment/oracle switches remain usable for diagnostics. */ + /* On GB10 the default fused call path keeps both the ordinary Q4_K MMVQ + * result and the ordinary HC epilogue. This preserves decode-graph island + * B without claiming that the row-packed epilogue is bit-exact. The + * row-packed and Q8_K candidates remain oracle-only diagnostics until + * device tests prove their numerical contract. The existing disable + * variable (plus DS4_CUDA_NO_Q4_GB10_FAST) is a complete rollback to the + * caller's separate B projection + HC expansion. */ if (cuda_q4_gb10_fast_path_enabled( g_current_logical_tier, NULL)) { return 1; @@ -34447,7 +34451,7 @@ extern "C" int ds4_gpu_matmul_q4_K_hc_expand_tensor( if (!q8k_experiment && !oracle) { return cuda_q4_K_hc_expand_canonical( out_hc, block_out, model_map, model_size, weight_offset, - in_dim, out_dim, x, residual_hc, split, 1); + in_dim, out_dim, x, residual_hc, split, 0); } /* The oracle is deliberately diagnostic and fail-closed: compute and @@ -34601,10 +34605,10 @@ extern "C" int ds4_gpu_matmul_q4_K_hc_expand_tensor( /* Allocation/launch rejection of the optional candidate must not make * decoding unavailable. Re-materialize through the ordinary dispatcher - * and finish with the exact row-packed HC epilogue. */ + * and finish with the ordinary HC epilogue. */ return cuda_q4_K_hc_expand_canonical( out_hc, block_out, model_map, model_size, weight_offset, - in_dim, out_dim, x, residual_hc, split, 1); + in_dim, out_dim, x, residual_hc, split, 0); } __global__ static void matmul_q4_K_kslice_kernel( diff --git a/tests/cuda_q4_gb10_fast_matrix.sh b/tests/cuda_q4_gb10_fast_matrix.sh new file mode 100755 index 000000000..86404a227 --- /dev/null +++ b/tests/cuda_q4_gb10_fast_matrix.sh @@ -0,0 +1,449 @@ +#!/bin/sh +# Isolate the numerical effect of each AProjQ4 GB10 fast-path component. +# +# Usage: +# tests/cuda_q4_gb10_fast_matrix.sh MODEL MANIFEST [OUTPUT_DIR] +# +# Every non-oracle arm is run through score_official. The script deliberately +# uses separate processes: +# these switches are cached during CUDA/MMQ initialization and cannot be +# compared safely in one process. + +set -eu + +usage() { + cat >&2 <<'EOF' +usage: tests/cuda_q4_gb10_fast_matrix.sh MODEL MANIFEST [OUTPUT_DIR] + +Environment: + DS4_BIN ds4 executable (default: ./ds4) + DS4_CUDA_Q4_MATRIX_SCORER score_official executable + DS4_CUDA_Q4_MATRIX_CTX context size (default: 4096) + DS4_CUDA_Q4_MATRIX_TOKENS smoke continuation length (default: 32) + DS4_CUDA_Q4_MATRIX_TOP_K smoke top-logprobs (default: 128) + DS4_CUDA_Q4_MATRIX_PROMPT deterministic smoke prompt + DS4_CUDA_Q4_MATRIX_SSD_STREAMING 0 or 1 (default: 0) + DS4_CUDA_Q4_MATRIX_SSD_CACHE expert count or NGB; required with streaming + DS4_CUDA_Q4_MATRIX_SSD_PRELOAD optional expert preload count + DS4_CUDA_Q4_MATRIX_DECODE_GRAPHS default, 0, or 1 + DS4_CUDA_Q4_MATRIX_SKIP_PARITY 0 or 1 (default: 0; skip is incomplete QA) + +The output directory must not already contain matrix results. Exit status is +zero only when tensor-oracle coverage passes, scorer rows match, and every +recorded smoke result is byte-identical to the umbrella rollback. A numerical +difference is preserved in *.diff and causes a nonzero exit after all arms +have completed. +EOF +} + +if [ "$#" -lt 2 ] || [ "$#" -gt 3 ]; then + usage + exit 2 +fi + +MODEL=$1 +MANIFEST=$2 +OUT_DIR=${3:-} +DS4_BIN=${DS4_BIN:-./ds4} +SCORER=${DS4_CUDA_Q4_MATRIX_SCORER:-gguf-tools/quality-testing/score_official} +CTX=${DS4_CUDA_Q4_MATRIX_CTX:-4096} +TOKENS=${DS4_CUDA_Q4_MATRIX_TOKENS:-32} +TOP_K=${DS4_CUDA_Q4_MATRIX_TOP_K:-128} +PROMPT=${DS4_CUDA_Q4_MATRIX_PROMPT:-Write a complete Python quicksort function with comments.} +SSD_STREAMING=${DS4_CUDA_Q4_MATRIX_SSD_STREAMING:-0} +SSD_CACHE=${DS4_CUDA_Q4_MATRIX_SSD_CACHE:-} +SSD_PRELOAD=${DS4_CUDA_Q4_MATRIX_SSD_PRELOAD:-} +DECODE_GRAPHS=${DS4_CUDA_Q4_MATRIX_DECODE_GRAPHS:-default} +SKIP_PARITY=${DS4_CUDA_Q4_MATRIX_SKIP_PARITY:-0} + +case "$CTX:$TOKENS:$TOP_K" in + *[!0-9:]*|:*|*::*|*:|0:*|*:0:*|*:0) + echo "q4-gb10-matrix: ctx, tokens, and top-k must be positive integers" >&2 + exit 2 + ;; +esac +if [ "$TOP_K" -gt 128 ]; then + echo "q4-gb10-matrix: top-k cannot exceed the ds4 dump limit (128)" >&2 + exit 2 +fi +case "$SSD_STREAMING" in + 0|1) ;; + *) echo "q4-gb10-matrix: SSD_STREAMING must be 0 or 1" >&2; exit 2 ;; +esac +case "$SKIP_PARITY" in + 0|1) ;; + *) echo "q4-gb10-matrix: SKIP_PARITY must be 0 or 1" >&2; exit 2 ;; +esac +case "$DECODE_GRAPHS" in + default|0|1) ;; + *) echo "q4-gb10-matrix: DECODE_GRAPHS must be default, 0, or 1" >&2; exit 2 ;; +esac +if [ "$SSD_STREAMING" = 1 ] && [ -z "$SSD_CACHE" ]; then + echo "q4-gb10-matrix: set DS4_CUDA_Q4_MATRIX_SSD_CACHE when streaming" >&2 + exit 2 +fi +if [ "$SSD_STREAMING" = 0 ] && { [ -n "$SSD_CACHE" ] || [ -n "$SSD_PRELOAD" ]; }; then + echo "q4-gb10-matrix: SSD cache/preload requires SSD_STREAMING=1" >&2 + exit 2 +fi +if [ ! -x "$DS4_BIN" ]; then + echo "q4-gb10-matrix: ds4 executable not found: $DS4_BIN" >&2 + exit 2 +fi +if [ ! -r "$MODEL" ]; then + echo "q4-gb10-matrix: model is not readable: $MODEL" >&2 + exit 2 +fi +if [ ! -r "$MANIFEST" ]; then + echo "q4-gb10-matrix: manifest is not readable: $MANIFEST" >&2 + exit 2 +fi +if [ ! -x "$SCORER" ]; then + echo "q4-gb10-matrix: scorer not found: $SCORER" >&2 + echo "build it with: make gguf-tools/quality-testing/score_official" >&2 + exit 2 +fi + +if [ -z "$OUT_DIR" ]; then + OUT_DIR=$(mktemp -d "${TMPDIR:-/tmp}/ds4-q4-gb10-matrix.XXXXXX") +else + mkdir -p "$OUT_DIR" +fi +if [ -e "$OUT_DIR/umbrella_control.log" ] || + [ -e "$OUT_DIR/umbrella_control.json" ] || + [ -e "$OUT_DIR/umbrella_control.tsv" ]; then + echo "q4-gb10-matrix: output directory already contains matrix results: $OUT_DIR" >&2 + exit 2 +fi + +case "$DECODE_GRAPHS" in + default) GRAPH_ENV=; GRAPH_LOG_ENV= ;; + 0) GRAPH_ENV=DS4_CUDA_DECODE_GRAPHS=0; GRAPH_LOG_ENV= ;; + 1) GRAPH_ENV=DS4_CUDA_DECODE_GRAPHS=1; GRAPH_LOG_ENV=DS4_CUDA_DECODE_GRAPH_LOG=1 ;; +esac + +# Strip every selector that could leak from the caller and turn an apparently +# isolated arm into a compound experiment. Arm-specific assignments follow +# these -u options. +clean_env() { + env \ + -u DS4_CUDA_MMQ \ + -u DS4_CUDA_DISABLE_Q4_DENSE_PAIR \ + -u DS4_CUDA_NO_Q4_GB10_FAST \ + -u DS4_CUDA_NO_Q4_DENSE_SCRATCH \ + -u DS4_CUDA_NO_Q4_GROUPED_ATTN_A \ + -u DS4_CUDA_NO_Q4_GROUPED_ATTN_A_BATCH \ + -u DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_BATCH \ + -u DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_BATCH \ + -u DS4_CUDA_Q4_GROUPED_ATTN_A_ORACLE \ + -u DS4_CUDA_DISABLE_Q4_ATTN_OUT_HC_FUSE \ + -u DS4_CUDA_ENABLE_Q4_ATTN_OUT_HC_FUSE \ + -u DS4_CUDA_Q4_ATTN_OUT_HC_Q8K_EXPERIMENT \ + -u DS4_CUDA_Q4_ATTN_OUT_HC_ORACLE \ + -u DS4_CUDA_NO_Q4_K1024_PERSISTENT \ + -u DS4_CUDA_ENABLE_Q4_K1024_PERSISTENT \ + -u DS4_CUDA_REQUIRE_Q4_K1024_PERSISTENT \ + -u DS4_CUDA_DECODE_GRAPHS \ + -u DS4_CUDA_DECODE_GRAPH_LOG \ + "$@" +} + +check_gb10_log() { + log=$1 + cuda_count=$(grep -c 'ds4: CUDA backend initialized on ' "$log" || true) + if [ "$cuda_count" -ne 1 ] || + ! grep -Eq 'ds4: CUDA backend initialized on .*\(sm_121\) dev=' "$log"; then + echo "q4-gb10-matrix: $log did not initialize exactly one sm_121 CUDA GPU" >&2 + return 1 + fi +} + +check_graph_log() { + arm=$1 + log=$2 + case "$arm" in + grouped_oracle|hc_oracle) + if ! grep -q 'decode graph capture disabled for Q4 attention oracle' "$log"; then + echo "q4-gb10-matrix: $arm did not prove oracle graph exclusion" >&2 + return 1 + fi + ;; + *) + case "$DECODE_GRAPHS" in + 0) + if ! grep -q 'decode graph capture disabled' "$log"; then + echo "q4-gb10-matrix: $arm did not prove graphs-off dispatch" >&2 + return 1 + fi + ;; + 1) + if ! grep -q 'ds4: decode graph captured ' "$log"; then + echo "q4-gb10-matrix: $arm had zero decode-graph captures" >&2 + return 1 + fi + if grep -Eq 'decode graph (capture|instantiate|first launch|replay) failed' "$log"; then + echo "q4-gb10-matrix: $arm reported a decode-graph failure" >&2 + return 1 + fi + ;; + esac + ;; + esac +} + +run_smoke() { + arm=$1 + shift + json="$OUT_DIR/$arm.json" + stdout="$OUT_DIR/$arm.stdout" + log="$OUT_DIR/$arm.log" + echo "q4-gb10-matrix: smoke arm=$arm" + + if [ "$SSD_STREAMING" = 1 ]; then + if [ -n "$SSD_PRELOAD" ]; then + clean_env ${GRAPH_ENV:+"$GRAPH_ENV"} ${GRAPH_LOG_ENV:+"$GRAPH_LOG_ENV"} \ + "$@" "$DS4_BIN" \ + --cuda -m "$MODEL" --ctx "$CTX" --tokens "$TOKENS" \ + --nothink --temp 0 --dump-logprobs "$json" \ + --logprobs-top-k "$TOP_K" --ssd-streaming \ + --ssd-streaming-cache-experts "$SSD_CACHE" \ + --ssd-streaming-preload-experts "$SSD_PRELOAD" \ + -p "$PROMPT" >"$stdout" 2>"$log" + else + clean_env ${GRAPH_ENV:+"$GRAPH_ENV"} ${GRAPH_LOG_ENV:+"$GRAPH_LOG_ENV"} \ + "$@" "$DS4_BIN" \ + --cuda -m "$MODEL" --ctx "$CTX" --tokens "$TOKENS" \ + --nothink --temp 0 --dump-logprobs "$json" \ + --logprobs-top-k "$TOP_K" --ssd-streaming \ + --ssd-streaming-cache-experts "$SSD_CACHE" \ + -p "$PROMPT" >"$stdout" 2>"$log" + fi + else + clean_env ${GRAPH_ENV:+"$GRAPH_ENV"} ${GRAPH_LOG_ENV:+"$GRAPH_LOG_ENV"} \ + "$@" "$DS4_BIN" \ + --cuda -m "$MODEL" --ctx "$CTX" --tokens "$TOKENS" \ + --nothink --temp 0 --dump-logprobs "$json" \ + --logprobs-top-k "$TOP_K" -p "$PROMPT" \ + >"$stdout" 2>"$log" + fi + check_gb10_log "$log" + check_graph_log "$arm" "$log" + if [ ! -s "$json" ]; then + echo "q4-gb10-matrix: arm $arm produced no logprob dump" >&2 + return 1 + fi +} + +run_score() { + arm=$1 + shift + tsv="$OUT_DIR/$arm.tsv" + log="$OUT_DIR/$arm.score.log" + echo "q4-gb10-matrix: quality arm=$arm" + + if [ "$SSD_STREAMING" = 1 ]; then + if [ -n "$SSD_PRELOAD" ]; then + clean_env ${GRAPH_ENV:+"$GRAPH_ENV"} ${GRAPH_LOG_ENV:+"$GRAPH_LOG_ENV"} \ + "$@" "$SCORER" \ + "$MODEL" "$MANIFEST" "$tsv" "$CTX" --ssd-streaming \ + --ssd-streaming-cache-experts "$SSD_CACHE" \ + --ssd-streaming-preload-experts "$SSD_PRELOAD" \ + >"$OUT_DIR/$arm.score.stdout" 2>"$log" + else + clean_env ${GRAPH_ENV:+"$GRAPH_ENV"} ${GRAPH_LOG_ENV:+"$GRAPH_LOG_ENV"} \ + "$@" "$SCORER" \ + "$MODEL" "$MANIFEST" "$tsv" "$CTX" --ssd-streaming \ + --ssd-streaming-cache-experts "$SSD_CACHE" \ + >"$OUT_DIR/$arm.score.stdout" 2>"$log" + fi + else + clean_env ${GRAPH_ENV:+"$GRAPH_ENV"} ${GRAPH_LOG_ENV:+"$GRAPH_LOG_ENV"} \ + "$@" "$SCORER" \ + "$MODEL" "$MANIFEST" "$tsv" "$CTX" \ + >"$OUT_DIR/$arm.score.stdout" 2>"$log" + fi + check_gb10_log "$log" + check_graph_log "$arm" "$log" + if [ ! -s "$tsv" ]; then + echo "q4-gb10-matrix: quality arm $arm produced no TSV" >&2 + return 1 + fi +} + +field() { + printf '%s\n' "$1" | tr ' ' '\n' | awk -F= -v key="$2" ' + $1 == key { gsub(/[^0-9].*$/, "", $2); print $2; exit } + ' +} + +check_grouped_oracle() { + log=$1 + line=$(grep 'ds4: CUDA Q4 grouped attention-A oracle:' "$log" | tail -n 1 || true) + calls=$(field "$line" calls) + mismatches=$(field "$line" mismatches) + skips=$(field "$line" skips) + if [ -z "$line" ] || [ -z "$calls" ] || [ "$calls" -le 0 ] || + [ "$skips" != 0 ]; then + echo "q4-gb10-matrix: grouped oracle coverage failed: ${line:-missing summary}" >&2 + return 1 + fi + if [ "$mismatches" != 0 ]; then + echo "q4-gb10-matrix: grouped oracle found mismatches: $line" >&2 + oracle_mismatch_arms="$oracle_mismatch_arms grouped" + fi +} + +check_hc_oracle() { + log=$1 + line=$(grep 'ds4: CUDA Q4 attention-output/HC oracle:' "$log" | tail -n 1 || true) + calls=$(field "$line" calls) + mismatches=$(field "$line" epilogue_mismatches) + skips=$(field "$line" skips) + if [ -z "$line" ] || [ -z "$calls" ] || [ "$calls" -le 0 ] || + [ "$skips" != 0 ]; then + echo "q4-gb10-matrix: HC epilogue oracle coverage failed: ${line:-missing summary}" >&2 + return 1 + fi + if [ "$mismatches" != 0 ]; then + echo "q4-gb10-matrix: HC epilogue oracle found mismatches: $line" >&2 + oracle_mismatch_arms="$oracle_mismatch_arms hc_epilogue" + fi +} + +LOCAL_ROLLBACK="DS4_CUDA_NO_Q4_DENSE_SCRATCH=1 +DS4_CUDA_NO_Q4_GROUPED_ATTN_A=1 +DS4_CUDA_NO_Q4_GROUPED_ATTN_A_BATCH=1 +DS4_CUDA_DISABLE_Q4_ATTN_OUT_HC_FUSE=1 +DS4_CUDA_NO_Q4_K1024_PERSISTENT=1" + +# Do not stop at the first numerical difference: completing all arms is what +# identifies a single culprit versus an interaction. Execution/coverage +# failures still stop immediately because subsequent comparisons would lie. +if [ "$SKIP_PARITY" = 0 ]; then + echo "q4-gb10-matrix: synthetic MMQ parity" + clean_env make test-mmq-parity-cuda CUDA_ARCH=sm_121 \ + >"$OUT_DIR/mmq-parity.log" 2>&1 +else + echo "q4-gb10-matrix: WARNING synthetic parity skipped (result is incomplete)" >&2 +fi + +# `set -- $LOCAL_ROLLBACK` intentionally splits the newline-delimited list +# into environment assignments. Values and names contain no shell metacharacters. +set -- $LOCAL_ROLLBACK +run_smoke umbrella_control DS4_CUDA_NO_Q4_GB10_FAST=1 +run_smoke local_control "$@" +run_smoke scratch_only \ + DS4_CUDA_NO_Q4_GROUPED_ATTN_A=1 \ + DS4_CUDA_NO_Q4_GROUPED_ATTN_A_BATCH=1 \ + DS4_CUDA_DISABLE_Q4_ATTN_OUT_HC_FUSE=1 \ + DS4_CUDA_NO_Q4_K1024_PERSISTENT=1 +run_smoke grouped_only \ + DS4_CUDA_NO_Q4_DENSE_SCRATCH=1 \ + DS4_CUDA_NO_Q4_GROUPED_ATTN_A_BATCH=1 \ + DS4_CUDA_DISABLE_Q4_ATTN_OUT_HC_FUSE=1 \ + DS4_CUDA_NO_Q4_K1024_PERSISTENT=1 +run_smoke hc_only \ + DS4_CUDA_NO_Q4_DENSE_SCRATCH=1 \ + DS4_CUDA_NO_Q4_GROUPED_ATTN_A=1 \ + DS4_CUDA_NO_Q4_GROUPED_ATTN_A_BATCH=1 \ + DS4_CUDA_NO_Q4_K1024_PERSISTENT=1 +run_smoke default_fast DS4_CUDA_NO_Q4_K1024_PERSISTENT=1 + +status=0 +oracle_mismatch_arms= +run_smoke grouped_oracle \ + DS4_CUDA_NO_Q4_DENSE_SCRATCH=1 \ + DS4_CUDA_NO_Q4_GROUPED_ATTN_A_BATCH=1 \ + DS4_CUDA_DISABLE_Q4_ATTN_OUT_HC_FUSE=1 \ + DS4_CUDA_NO_Q4_K1024_PERSISTENT=1 \ + DS4_CUDA_Q4_GROUPED_ATTN_A_ORACLE=1 +check_grouped_oracle "$OUT_DIR/grouped_oracle.log" + +run_smoke hc_oracle \ + DS4_CUDA_NO_Q4_DENSE_SCRATCH=1 \ + DS4_CUDA_NO_Q4_GROUPED_ATTN_A=1 \ + DS4_CUDA_NO_Q4_GROUPED_ATTN_A_BATCH=1 \ + DS4_CUDA_NO_Q4_K1024_PERSISTENT=1 \ + DS4_CUDA_Q4_ATTN_OUT_HC_ORACLE=1 +check_hc_oracle "$OUT_DIR/hc_oracle.log" + +if [ -n "$oracle_mismatch_arms" ]; then + status=1 +fi +changed_smoke= +compare_smoke() { + arm=$1 + if cmp -s "$OUT_DIR/umbrella_control.json" "$OUT_DIR/$arm.json"; then + echo "q4-gb10-matrix: smoke $arm: EXACT" + else + echo "q4-gb10-matrix: smoke $arm: DIFFERENT" + diff -u "$OUT_DIR/umbrella_control.json" "$OUT_DIR/$arm.json" \ + >"$OUT_DIR/$arm.diff" || true + changed_smoke="$changed_smoke $arm" + status=1 + fi +} + +for arm in local_control scratch_only grouped_only hc_only default_fast \ + grouped_oracle hc_oracle; do + compare_smoke "$arm" +done + +changed_quality= +run_score umbrella_control DS4_CUDA_NO_Q4_GB10_FAST=1 +set -- $LOCAL_ROLLBACK +run_score local_control "$@" +run_score scratch_only \ + DS4_CUDA_NO_Q4_GROUPED_ATTN_A=1 \ + DS4_CUDA_NO_Q4_GROUPED_ATTN_A_BATCH=1 \ + DS4_CUDA_DISABLE_Q4_ATTN_OUT_HC_FUSE=1 \ + DS4_CUDA_NO_Q4_K1024_PERSISTENT=1 +run_score grouped_only \ + DS4_CUDA_NO_Q4_DENSE_SCRATCH=1 \ + DS4_CUDA_NO_Q4_GROUPED_ATTN_A_BATCH=1 \ + DS4_CUDA_DISABLE_Q4_ATTN_OUT_HC_FUSE=1 \ + DS4_CUDA_NO_Q4_K1024_PERSISTENT=1 +run_score hc_only \ + DS4_CUDA_NO_Q4_DENSE_SCRATCH=1 \ + DS4_CUDA_NO_Q4_GROUPED_ATTN_A=1 \ + DS4_CUDA_NO_Q4_GROUPED_ATTN_A_BATCH=1 \ + DS4_CUDA_NO_Q4_K1024_PERSISTENT=1 +run_score default_fast DS4_CUDA_NO_Q4_K1024_PERSISTENT=1 + +for arm in local_control scratch_only grouped_only hc_only default_fast; do + if cmp -s "$OUT_DIR/umbrella_control.tsv" "$OUT_DIR/$arm.tsv"; then + echo "q4-gb10-matrix: quality $arm: EXACT" + else + echo "q4-gb10-matrix: quality $arm: DIFFERENT" + python3 gguf-tools/quality-testing/compare_scores.py \ + "$OUT_DIR/umbrella_control.tsv" "$OUT_DIR/$arm.tsv" \ + >"$OUT_DIR/$arm.comparison.txt" + changed_quality="$changed_quality $arm" + status=1 + fi +done + +if [ "$SKIP_PARITY" != 0 ]; then + status=1 +fi + +{ + echo "output_dir=$OUT_DIR" + echo "decode_graphs=$DECODE_GRAPHS" + echo "smoke_differences=${changed_smoke# }" + echo "quality_differences=${changed_quality# }" + echo "oracle_mismatches=${oracle_mismatch_arms# }" + echo "quality_manifest=$MANIFEST" + if [ "$SKIP_PARITY" = 0 ]; then + echo "synthetic_parity=pass" + else + echo "synthetic_parity=skipped" + fi + if [ "$status" -eq 0 ]; then + echo "promotion_gate=pass" + else + echo "promotion_gate=blocked" + fi +} | tee "$OUT_DIR/summary.txt" + +exit "$status" From fc33759a796ca7902886acef0996019f5542904f Mon Sep 17 00:00:00 2001 From: Giorgio Oppo Date: Tue, 11 Aug 2026 12:45:08 +0200 Subject: [PATCH 019/189] cuda: harden Q4 K1024 persistent path --- QA_BEFORE_RELEASES.md | 10 +- README.md | 8 +- cuda/mmq/ds4_mmq.cu | 248 +++++++++++++++++++++++++++---- cuda/mmq/ds4_mmq.h | 14 +- cuda/mmq/test/test_mmq_parity.cu | 60 +++++++- 5 files changed, 303 insertions(+), 37 deletions(-) diff --git a/QA_BEFORE_RELEASES.md b/QA_BEFORE_RELEASES.md index 4cd7a8d32..68d80c436 100644 --- a/QA_BEFORE_RELEASES.md +++ b/QA_BEFORE_RELEASES.md @@ -990,7 +990,15 @@ Do not use high-performance Hugging Face Xet mode while vLLM is resident. separate fail-closed arm with both `DS4_CUDA_ENABLE_Q4_K1024_PERSISTENT=1` and `DS4_CUDA_REQUIRE_Q4_K1024_PERSISTENT=1`; its rollback is - `DS4_CUDA_NO_Q4_K1024_PERSISTENT=1`. The persistent OutQ8 + `DS4_CUDA_NO_Q4_K1024_PERSISTENT=1`. Then run a non-captured oracle process + with `DS4_CUDA_DECODE_GRAPHS=0`, + `DS4_CUDA_Q4_K1024_PERSISTENT_ORACLE=1`, and + `DS4_CUDA_Q4_K1024_PERSISTENT_STATS=1`; require `candidates>0`, `uses>0`, + `oracle_calls>0`, `oracle_mismatches=0`, and `oracle_skips=0`. The parity + test must also show a nonzero REQUIRE failure with the local kill set, + proving admission fails before enqueue, and a canonical reference forced by + that same kill. The counters are host dispatches and intentionally exclude + CUDA graph replays. The persistent OutQ8 vocabulary, compressor, HC split, direct routed-MoE paths, Q4 scratch, grouped attention-A, and canonical B+HC epilogue remain relevant, while the Q8-only attention-projection consumers are intentionally ineligible. diff --git a/README.md b/README.md index 70da4679d..1e36a2ad0 100644 --- a/README.md +++ b/README.md @@ -722,7 +722,13 @@ arithmetic: behind `DS4_CUDA_ENABLE_Q4_K1024_PERSISTENT=1`, with `DS4_CUDA_NO_Q4_K1024_PERSISTENT=1` taking precedence. Tests can add `DS4_CUDA_REQUIRE_Q4_K1024_PERSISTENT=1` to fail instead of silently using - canonical MMVQ when the persistent dispatch is unavailable. + canonical MMVQ when the persistent dispatch is unavailable; this admission + now fails before quantization, output clearing, or any kernel enqueue. Set + `DS4_CUDA_Q4_K1024_PERSISTENT_STATS=1` for host-dispatch candidate/use/ + fallback counters. For a bitwise model-backed check, run with + `DS4_CUDA_DECODE_GRAPHS=0 DS4_CUDA_Q4_K1024_PERSISTENT_ORACLE=1`; the oracle + forces the candidate, compares it with canonical MMVQ, and always retains + canonical output. `DS4_CUDA_NO_Q4_GB10_FAST=1` is the umbrella rollback for these new GB10 choices; it does not disable the older cross-CUDA Q-A/KV pair itself. For a diff --git a/cuda/mmq/ds4_mmq.cu b/cuda/mmq/ds4_mmq.cu index 6febcdec8..b77a62cdd 100644 --- a/cuda/mmq/ds4_mmq.cu +++ b/cuda/mmq/ds4_mmq.cu @@ -3073,6 +3073,63 @@ int ds4_mmq_moe_pair_vec_impl( return 0; } +/* Diagnostic counters are host-dispatch counters. CUDA graph replays do not + * re-enter this wrapper, so they are deliberately not presented as kernel + * execution counts. They are still a fail-closed coverage signal: a GB10 + * model run must observe at least one candidate and one use before this path + * can be promoted from opt-in to default. */ +static uint64_t g_q4_k1024_persistent_candidates; +static uint64_t g_q4_k1024_persistent_uses; +static uint64_t g_q4_k1024_persistent_fallbacks; +static uint64_t g_q4_k1024_persistent_require_failures; +static uint64_t g_q4_k1024_persistent_oracle_calls; +static uint64_t g_q4_k1024_persistent_oracle_mismatches; +static uint64_t g_q4_k1024_persistent_oracle_skips; +static int g_q4_k1024_persistent_report_registered; +static int g_q4_k1024_persistent_oracle_mismatch_reported; + +static bool q4_k1024_env_flag(const char *name) { + const char *value = getenv(name); + return value && value[0] && strcmp(value, "0") != 0; +} + +static void q4_k1024_persistent_report(void) { + fprintf(stderr, + "ds4: CUDA Q4 K1024 persistent: " + "candidates=%llu uses=%llu fallbacks=%llu " + "require_failures=%llu oracle_calls=%llu " + "oracle_mismatches=%llu oracle_skips=%llu " + "(host dispatches; graph replays excluded, canonical oracle output retained)\n", + (unsigned long long)g_q4_k1024_persistent_candidates, + (unsigned long long)g_q4_k1024_persistent_uses, + (unsigned long long)g_q4_k1024_persistent_fallbacks, + (unsigned long long)g_q4_k1024_persistent_require_failures, + (unsigned long long)g_q4_k1024_persistent_oracle_calls, + (unsigned long long)g_q4_k1024_persistent_oracle_mismatches, + (unsigned long long)g_q4_k1024_persistent_oracle_skips); +} + +static void q4_k1024_persistent_maybe_register_report(void) { + if (!g_q4_k1024_persistent_report_registered && + (q4_k1024_env_flag("DS4_CUDA_Q4_K1024_PERSISTENT_STATS") || + q4_k1024_env_flag("DS4_CUDA_Q4_K1024_PERSISTENT_ORACLE"))) { + g_q4_k1024_persistent_report_registered = 1; + (void)atexit(q4_k1024_persistent_report); + } +} + +__global__ static void q4_K_k1024_bitwise_compare_kernel( + uint32_t *mismatch, + const float *candidate, + const float *reference, + uint64_t count) { + const uint64_t i = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (i < count && + __float_as_uint(candidate[i]) != __float_as_uint(reference[i])) { + atomicExch(mismatch, 1u); + } +} + /* GB10 AProjQ4 Q-b decode specialization (M=32768, N=1, K=1024). * * The canonical MMVQ small-K launch uses four warps to evaluate four rows: @@ -3205,10 +3262,92 @@ int ds4_mmq_dense_vec_impl( return -1; } + /* Resolve the exact-shape admission before allocating pool storage, + * quantizing X, clearing output, or launching any kernel. REQUIRE and + * the oracle are coverage gates: an ineligible candidate must therefore + * fail without leaving work queued on the caller's stream. */ + bool q4_k1024_exact = false; + bool q4_k1024_eligible = false; + bool q4_k1024_oracle = false; + unsigned q4_k1024_grid = 0u; + if constexpr (type == GGML_TYPE_Q4_K) { + q4_k1024_persistent_maybe_register_report(); + q4_k1024_exact = M == 32768 && N == 1 && K == 1024; + q4_k1024_oracle = q4_k1024_exact && + q4_k1024_env_flag("DS4_CUDA_Q4_K1024_PERSISTENT_ORACLE"); + const bool enable = + getenv("DS4_CUDA_ENABLE_Q4_K1024_PERSISTENT") != nullptr; + const bool disable = + getenv("DS4_CUDA_NO_Q4_K1024_PERSISTENT") != nullptr || + getenv("DS4_CUDA_NO_Q4_GB10_FAST") != nullptr; + if (q4_k1024_exact) { + g_q4_k1024_persistent_candidates++; + } + if (q4_k1024_exact && g_gb10_optimizations && + (enable || q4_k1024_oracle) && !disable && + (((uintptr_t)W & 15u) == 0u)) { + const uint64_t row_tiles = ((uint64_t)(uint32_t)M + 7u) / 8u; + const int nsm = ggml_cuda_info().devices[dev].nsm; + const uint64_t resident_blocks = + nsm > 0 ? (uint64_t)(uint32_t)nsm * 4u : 0u; + const uint64_t grid64 = row_tiles < resident_blocks + ? row_tiles : resident_blocks; + if (grid64 > 0u && grid64 <= UINT32_MAX) { + q4_k1024_grid = (unsigned)grid64; + q4_k1024_eligible = true; + } + } + if (q4_k1024_exact && !q4_k1024_eligible) { + g_q4_k1024_persistent_fallbacks++; + const bool require = + getenv("DS4_CUDA_REQUIRE_Q4_K1024_PERSISTENT") != nullptr; + if (require || q4_k1024_oracle) { + g_q4_k1024_persistent_require_failures++; + if (q4_k1024_oracle) { + g_q4_k1024_persistent_oracle_skips++; + } + fprintf(stderr, + "%s: required Q4_K K1024 persistent path unavailable " + "before enqueue\n", + tag); + return -4; + } + } + if (q4_k1024_eligible && q4_k1024_oracle) { + cudaStreamCaptureStatus capture = cudaStreamCaptureStatusNone; + const cudaError_t capture_err = + cudaStreamIsCapturing(stream, &capture); + if (capture_err != cudaSuccess || + capture != cudaStreamCaptureStatusNone) { + (void)cudaGetLastError(); + g_q4_k1024_persistent_fallbacks++; + g_q4_k1024_persistent_require_failures++; + g_q4_k1024_persistent_oracle_skips++; + fprintf(stderr, + "%s: Q4_K K1024 persistent oracle refuses CUDA " + "graph capture before enqueue; run the oracle with " + "DS4_CUDA_DECODE_GRAPHS=0\n", + tag); + return -5; + } + } + } + // Route the pool's cudaMallocAsync through the caller-supplied stream // for Step 8 / CUDA Graph compatibility. See ds4_mmq_moe_vec_impl. ds4_pool_set_stream(stream); + /* Oracle-only storage. Graph capture was rejected above, so the pool + * allocations and the host readback below cannot become graph nodes. + * The persistent candidate writes here; canonical MMVQ always owns the + * caller-visible output. */ + ggml_cuda_pool_alloc q4_k1024_candidate; + ggml_cuda_pool_alloc q4_k1024_mismatch; + if (q4_k1024_eligible && q4_k1024_oracle) { + q4_k1024_candidate.alloc(ctx->pool(), (size_t)M); + q4_k1024_mismatch.alloc(ctx->pool(), 1u); + } + // Dense: no MoE, ids=null. Layout [K, N, 1, 1] for src1. const int64_t ne10_padded = GGML_PAD((int64_t)K, MATRIX_ROW_PADDING); const size_t nbytes_q8_1 = (size_t)N * ne10_padded * @@ -3260,36 +3399,16 @@ int ds4_mmq_dense_vec_impl( bool q4_k1024_persistent = false; if constexpr (type == GGML_TYPE_Q4_K) { - const bool exact_shape = M == 32768 && N == 1 && K == 1024; - const bool enable = - getenv("DS4_CUDA_ENABLE_Q4_K1024_PERSISTENT") != nullptr; - const bool disable = - getenv("DS4_CUDA_NO_Q4_K1024_PERSISTENT") != nullptr || - getenv("DS4_CUDA_NO_Q4_GB10_FAST") != nullptr; - if (g_gb10_optimizations && enable && !disable && exact_shape && - (((uintptr_t)W & 15u) == 0u)) { - const uint64_t row_tiles = ((uint64_t)(uint32_t)M + 7u) / 8u; - const int nsm = ggml_cuda_info().devices[dev].nsm; - const uint64_t resident_blocks = - nsm > 0 ? (uint64_t)(uint32_t)nsm * 4u : 0u; - const uint64_t grid64 = row_tiles < resident_blocks - ? row_tiles : resident_blocks; - if (grid64 > 0u && grid64 <= UINT32_MAX) { - q4_K_dense_vec_k1024_persistent_kernel<<< - (unsigned)grid64, 256, 0, stream>>>( - (const block_q4_K *)W, - (const block_q8_1 *)x8, - out_f32, M); - q4_k1024_persistent = true; - } - } - if (exact_shape && - getenv("DS4_CUDA_REQUIRE_Q4_K1024_PERSISTENT") != nullptr && - !q4_k1024_persistent) { - fprintf(stderr, - "%s: required Q4_K K1024 persistent path unavailable\n", - tag); - return -4; + if (q4_k1024_eligible) { + float *candidate_out = q4_k1024_oracle + ? q4_k1024_candidate.get() : out_f32; + q4_K_dense_vec_k1024_persistent_kernel<<< + q4_k1024_grid, 256, 0, stream>>>( + (const block_q4_K *)W, + (const block_q8_1 *)x8, + candidate_out, M); + g_q4_k1024_persistent_uses++; + q4_k1024_persistent = !q4_k1024_oracle; } } @@ -3320,7 +3439,53 @@ int ds4_mmq_dense_vec_impl( tag, cudaGetErrorString(err)); return -3; } - ds4_mmq_sanitize_f32(out_f32, (uint64_t)M * (uint64_t)N, stream); + const uint64_t out_count = (uint64_t)M * (uint64_t)N; + ds4_mmq_sanitize_f32(out_f32, out_count, stream); + if (q4_k1024_oracle) { + ds4_mmq_sanitize_f32(q4_k1024_candidate.get(), out_count, stream); + if (cudaGetLastError() != cudaSuccess) { + fprintf(stderr, "%s: Q4_K K1024 oracle sanitize failed\n", tag); + g_q4_k1024_persistent_oracle_skips++; + return -6; + } + cudaError_t oracle_err = cudaMemsetAsync( + q4_k1024_mismatch.get(), 0, sizeof(uint32_t), stream); + if (oracle_err == cudaSuccess) { + q4_K_k1024_bitwise_compare_kernel<<< + (unsigned)((out_count + 255u) / 256u), 256, 0, stream>>>( + q4_k1024_mismatch.get(), q4_k1024_candidate.get(), + out_f32, out_count); + oracle_err = cudaGetLastError(); + } + uint32_t mismatch_host = 0u; + if (oracle_err == cudaSuccess) { + oracle_err = cudaMemcpyAsync( + &mismatch_host, q4_k1024_mismatch.get(), sizeof(uint32_t), + cudaMemcpyDeviceToHost, stream); + } + if (oracle_err == cudaSuccess) { + oracle_err = cudaStreamSynchronize(stream); + } + if (oracle_err != cudaSuccess) { + fprintf(stderr, + "%s: Q4_K K1024 persistent oracle failed: %s\n", + tag, cudaGetErrorString(oracle_err)); + (void)cudaGetLastError(); + g_q4_k1024_persistent_oracle_skips++; + return -6; + } + g_q4_k1024_persistent_oracle_calls++; + if (mismatch_host != 0u) { + g_q4_k1024_persistent_oracle_mismatches++; + if (!g_q4_k1024_persistent_oracle_mismatch_reported) { + g_q4_k1024_persistent_oracle_mismatch_reported = 1; + fprintf(stderr, + "%s: Q4_K K1024 persistent oracle found a bitwise " + "mismatch; retained canonical MMVQ output\n", + tag); + } + } + } return 0; } @@ -4288,6 +4453,27 @@ static int ds4_mmq_q4_K_dense_pair_vec_impl( } // anonymous namespace +extern "C" void ds4_mmq_q4_K_k1024_persistent_counters( + uint64_t *candidates, + uint64_t *uses, + uint64_t *fallbacks, + uint64_t *require_failures, + uint64_t *oracle_calls, + uint64_t *oracle_mismatches, + uint64_t *oracle_skips) { + if (candidates) *candidates = g_q4_k1024_persistent_candidates; + if (uses) *uses = g_q4_k1024_persistent_uses; + if (fallbacks) *fallbacks = g_q4_k1024_persistent_fallbacks; + if (require_failures) { + *require_failures = g_q4_k1024_persistent_require_failures; + } + if (oracle_calls) *oracle_calls = g_q4_k1024_persistent_oracle_calls; + if (oracle_mismatches) { + *oracle_mismatches = g_q4_k1024_persistent_oracle_mismatches; + } + if (oracle_skips) *oracle_skips = g_q4_k1024_persistent_oracle_skips; +} + extern "C" int ds4_mmq_q8_0_moe_vec( const void * W, const float * X, const int32_t * ids, float * out, int M, int K, int n_tokens, int n_experts, int n_expert_used, diff --git a/cuda/mmq/ds4_mmq.h b/cuda/mmq/ds4_mmq.h index 5b838481d..173120df0 100644 --- a/cuda/mmq/ds4_mmq.h +++ b/cuda/mmq/ds4_mmq.h @@ -936,9 +936,21 @@ int ds4_mmq_q4_K_grouped_batch_vec( // DS4_CUDA_ENABLE_Q4_K1024_PERSISTENT=1. The rollback switch // DS4_CUDA_NO_Q4_K1024_PERSISTENT=1 is authoritative when both are set. // DS4_CUDA_REQUIRE_Q4_K1024_PERSISTENT=1 makes an unavailable exact-shape -// dispatch fail closed instead of silently running canonical MMVQ. +// dispatch fail before any CUDA work is enqueued instead of silently running +// canonical MMVQ. DS4_CUDA_Q4_K1024_PERSISTENT_ORACLE=1 forces a candidate, +// compares it bit-for-bit with canonical MMVQ, and retains canonical output; +// run it with DS4_CUDA_DECODE_GRAPHS=0. Set +// DS4_CUDA_Q4_K1024_PERSISTENT_STATS=1 for the atexit counter summary. // DS4_CUDA_NO_Q4_GB10_FAST=1 is the umbrella rollback for this and the // GB10 Q4 activation scratch. Other shapes and devices retain canonical MMVQ. +void ds4_mmq_q4_K_k1024_persistent_counters( + uint64_t *candidates, + uint64_t *uses, + uint64_t *fallbacks, + uint64_t *require_failures, + uint64_t *oracle_calls, + uint64_t *oracle_mismatches, + uint64_t *oracle_skips); // Two independent dense Q4_K projections that share the canonical Q8_1 // activation quantization. Each output is dispatched through the same MMVQ diff --git a/cuda/mmq/test/test_mmq_parity.cu b/cuda/mmq/test/test_mmq_parity.cu index 6cb4d1f8b..3e88fb0f0 100644 --- a/cuda/mmq/test/test_mmq_parity.cu +++ b/cuda/mmq/test/test_mmq_parity.cu @@ -1148,17 +1148,32 @@ bool run_q4_K_dense_vec_gb10_parity( cudaMemcpyAsync(dX, X.data(), X.size() * sizeof(float), cudaMemcpyHostToDevice, stream); + uint64_t candidates0 = 0, uses0 = 0, fallbacks0 = 0; + uint64_t require_failures0 = 0, oracle_calls0 = 0; + uint64_t oracle_mismatches0 = 0, oracle_skips0 = 0; + ds4_mmq_q4_K_k1024_persistent_counters( + &candidates0, &uses0, &fallbacks0, &require_failures0, + &oracle_calls0, &oracle_mismatches0, &oracle_skips0); + unsetenv("DS4_CUDA_NO_Q4_GB10_FAST"); unsetenv("DS4_CUDA_NO_Q4_DENSE_SCRATCH"); unsetenv("DS4_CUDA_NO_Q4_K1024_PERSISTENT"); unsetenv("DS4_CUDA_ENABLE_Q4_K1024_PERSISTENT"); unsetenv("DS4_CUDA_REQUIRE_Q4_K1024_PERSISTENT"); + unsetenv("DS4_CUDA_Q4_K1024_PERSISTENT_ORACLE"); ds4_mmq_set_gb10_optimizations(persistent_k1024 ? 1 : 0); ds4_mmq_set_aligned_q81_scratch( persistent_k1024 ? scratch : nullptr, persistent_k1024 ? 256u * 1024u : 0u); + /* The reference must remain canonical even after a future default-on + * promotion. The authoritative kill switch makes this a real + * reference-vs-candidate comparison rather than candidate-vs-candidate. */ + if (persistent_k1024) { + setenv("DS4_CUDA_NO_Q4_K1024_PERSISTENT", "1", 1); + } const int rc_ref = ds4_mmq_q4_K_dense_vec( dW, dX, dRef, M, N, K, stream); + unsetenv("DS4_CUDA_NO_Q4_K1024_PERSISTENT"); ds4_mmq_set_gb10_optimizations(1); ds4_mmq_set_aligned_q81_scratch(scratch, 256u * 1024u); @@ -1176,30 +1191,69 @@ bool run_q4_K_dense_vec_gb10_parity( cudaMemcpyAsync(got.data(), dGot, got.size() * sizeof(float), cudaMemcpyDeviceToHost, stream); int rc_required_disabled = 0; + int rc_oracle = 0; + std::vector oracle((size_t)M * N); if (persistent_k1024) { setenv("DS4_CUDA_NO_Q4_K1024_PERSISTENT", "1", 1); rc_required_disabled = ds4_mmq_q4_K_dense_vec( dW, dX, dGot, M, N, K, stream); unsetenv("DS4_CUDA_NO_Q4_K1024_PERSISTENT"); + setenv("DS4_CUDA_Q4_K1024_PERSISTENT_ORACLE", "1", 1); + rc_oracle = ds4_mmq_q4_K_dense_vec( + dW, dX, dGot, M, N, K, stream); + unsetenv("DS4_CUDA_Q4_K1024_PERSISTENT_ORACLE"); + cudaMemcpyAsync(oracle.data(), dGot, oracle.size() * sizeof(float), + cudaMemcpyDeviceToHost, stream); } const cudaError_t sync_err = cudaStreamSynchronize(stream); size_t mismatches = 0; + size_t oracle_output_mismatches = 0; for (size_t i = 0; i < ref.size(); i++) { if (std::memcmp(&ref[i], &got[i], sizeof(float)) != 0) mismatches++; + if (persistent_k1024 && + std::memcmp(&ref[i], &oracle[i], sizeof(float)) != 0) { + oracle_output_mismatches++; + } } + + uint64_t candidates1 = 0, uses1 = 0, fallbacks1 = 0; + uint64_t require_failures1 = 0, oracle_calls1 = 0; + uint64_t oracle_mismatches1 = 0, oracle_skips1 = 0; + ds4_mmq_q4_K_k1024_persistent_counters( + &candidates1, &uses1, &fallbacks1, &require_failures1, + &oracle_calls1, &oracle_mismatches1, &oracle_skips1); + const bool counter_ok = !persistent_k1024 || + (candidates1 - candidates0 >= 4u && + uses1 - uses0 >= 2u && + fallbacks1 - fallbacks0 >= 2u && + require_failures1 - require_failures0 >= 1u && + oracle_calls1 - oracle_calls0 >= 1u && + oracle_mismatches1 == oracle_mismatches0 && + oracle_skips1 == oracle_skips0); ok = rc_ref == 0 && rc_got == 0 && (!persistent_k1024 || rc_required_disabled != 0) && + (!persistent_k1024 || rc_oracle == 0) && sync_err == cudaSuccess && - mismatches == 0; + mismatches == 0 && oracle_output_mismatches == 0 && counter_ok; fprintf(stderr, "rc_ref=%d rc_candidate=%d rc_required_disabled=%d " - "mismatches=%zu sync=%s\n%s\n\n", - rc_ref, rc_got, rc_required_disabled, mismatches, + "rc_oracle=%d mismatches=%zu oracle_output_mismatches=%zu " + "counter_delta=%llu/%llu/%llu/%llu/%llu/%llu/%llu sync=%s\n%s\n\n", + rc_ref, rc_got, rc_required_disabled, rc_oracle, mismatches, + oracle_output_mismatches, + (unsigned long long)(candidates1 - candidates0), + (unsigned long long)(uses1 - uses0), + (unsigned long long)(fallbacks1 - fallbacks0), + (unsigned long long)(require_failures1 - require_failures0), + (unsigned long long)(oracle_calls1 - oracle_calls0), + (unsigned long long)(oracle_mismatches1 - oracle_mismatches0), + (unsigned long long)(oracle_skips1 - oracle_skips0), cudaGetErrorString(sync_err), ok ? "PASS" : "FAIL"); unsetenv("DS4_CUDA_ENABLE_Q4_K1024_PERSISTENT"); unsetenv("DS4_CUDA_REQUIRE_Q4_K1024_PERSISTENT"); + unsetenv("DS4_CUDA_Q4_K1024_PERSISTENT_ORACLE"); ds4_mmq_set_aligned_q81_scratch(nullptr, 0u); ds4_mmq_set_gb10_optimizations(0); cudaFree(scratch); From 0976f67a6c573eeff6795897c97fb363e5e44a2a Mon Sep 17 00:00:00 2001 From: Giorgio Oppo Date: Tue, 11 Aug 2026 12:48:40 +0200 Subject: [PATCH 020/189] cuda: add opt-in canonical Q8_1 producer fold --- QA_BEFORE_RELEASES.md | 14 + README.md | 16 + cuda/mmq/ds4_ggml_stubs.cu | 14 + cuda/mmq/ds4_mmq.cu | 608 +++++++++++++++++++---- cuda/mmq/test/d2r_stubs.cu | 6 +- cuda/mmq/test/proto_gemm_dense_q8_d2r.cu | 7 +- cuda/mmq/test/test_mmq_soa_tiles.cu | 7 +- ds4_cuda.cu | 376 +++++++++++++- tests/test_mxfp4_cuda.cu | 4 +- 9 files changed, 936 insertions(+), 116 deletions(-) diff --git a/QA_BEFORE_RELEASES.md b/QA_BEFORE_RELEASES.md index 68d80c436..ea6cd74a8 100644 --- a/QA_BEFORE_RELEASES.md +++ b/QA_BEFORE_RELEASES.md @@ -1002,6 +1002,20 @@ Do not use high-performance Hugging Face Xet mode while vLLM is resident. vocabulary, compressor, HC split, direct routed-MoE paths, Q4 scratch, grouped attention-A, and canonical B+HC epilogue remain relevant, while the Q8-only attention-projection consumers are intentionally ineligible. +- Validate the experimental HC-to-consumer Q8_1 producer fold in separate + processes. Use `DS4_CUDA_NO_Q8_FOLD=1` for the control and + `DS4_CUDA_ENABLE_Q8_FOLD=1` for the candidate, first with the normal graph + setting and then with `DS4_CUDA_DECODE_GRAPHS=0`. For the non-captured arm, + also set `DS4_CUDA_Q8_FOLD_ORACLE=1` and require `hits>0`, `byte_calls>0`, + `output_calls>0`, `byte_mismatches=0`, `output_mismatches=0`, and `skips=0`. + The reached consumer must be reported as aligned Q8 or IQ2 MoE rather than + inferred from producer counters alone. Require byte-identical greedy stdout + and per-token logprobs, then repeat the control/candidate pair under Compute + Sanitizer. Keep the oracle off for the graph-on timing arm: capture is an + intentional fail-closed miss and is checked for safety, not fold coverage. + Run these arms through the ordinary serialized inference dispatcher; the + opt-in fold does not support concurrent host-thread submission to one CUDA + stream. - If the umbrella AProjQ4 A/B changes logits, do not attribute that change to "the Q4 fast path" as a unit. Run the fail-closed component matrix from a clean `cuda-spark` build. The output directory is intentionally explicit so diff --git a/README.md b/README.md index 1e36a2ad0..6a682925d 100644 --- a/README.md +++ b/README.md @@ -701,6 +701,22 @@ standalone projections. The canonical Q4 path submits to the decode stream so these projections and the attention-output tail can participate in CUDA decode graphs. +Single-token resident CUDA can also experiment with folding the canonical +Q8_1 activation emitted by the 4096-wide HC split plus RMSNorm stage into its +next MMVQ consumer. This remains opt-in pending GB10/DGX validation: set +`DS4_CUDA_ENABLE_Q8_FOLD=1`; `DS4_CUDA_NO_Q8_FOLD=1` is the dominant kill +switch. The sidecar is one-shot and keyed by model map, physical device, +stream, source pointer, and session epoch. Capture, scratch growth, a model-map +or device transition, and every lookup mismatch reject or invalidate it and +fall back to the established quantizer. For a diagnostic run, disable decode +graphs and add `DS4_CUDA_Q8_FOLD_ORACLE=1`; the oracle compares canonical Q8_1 +bytes and the reached aligned-Q8 or IQ2 MoE consumer output, then always keeps +the freshly quantized reference. Require nonzero fold hits, `byte_calls`, and +`output_calls`, with zero mismatches and skips before considering promotion. +The experiment supports ds4's serialized, single-inference-host-thread CUDA +runtime only; embeddings that submit concurrently to the same CUDA stream +must leave it disabled. + On a single DGX Spark/GB10, the AProjQ4 path also mirrors the safe parts of the aligned-Q8 decode work while retaining canonical Q4_K MMVQ/Q8_1 arithmetic: diff --git a/cuda/mmq/ds4_ggml_stubs.cu b/cuda/mmq/ds4_ggml_stubs.cu index 99c5b8ed3..18f139bac 100644 --- a/cuda/mmq/ds4_ggml_stubs.cu +++ b/cuda/mmq/ds4_ggml_stubs.cu @@ -22,6 +22,20 @@ #include #include +/* Standalone MMQ tests do not link ds4_cuda.cu. Full ds4 links its strong, + * stream-aware registry implementation over this fail-closed weak miss. */ +#if defined(__GNUC__) +extern "C" __attribute__((weak)) int ds4_cuda_q8_fold_take_q81( + const void *src, uint64_t in_dim, cudaStream_t stream, + const void **q81) { + (void)src; + (void)in_dim; + (void)stream; + if (q81) *q81 = nullptr; + return 0; +} +#endif + // ---------------------------------------------------------------------------- // Device info singleton. // diff --git a/cuda/mmq/ds4_mmq.cu b/cuda/mmq/ds4_mmq.cu index b77a62cdd..86b3974ef 100644 --- a/cuda/mmq/ds4_mmq.cu +++ b/cuda/mmq/ds4_mmq.cu @@ -167,12 +167,135 @@ extern "C" void *ds4_mmq_q81_scratch_ptr(void) { // (ne10_padded == K); the registry itself guarantees freshness (slots are // reset by the producing entry every layer and pops are one-shot). extern "C" int ds4_cuda_q8_fold_take_q81(const void *src, uint64_t in_dim, + cudaStream_t stream, const void **q81); + +static uint64_t g_q8_fold_oracle_byte_calls; +static uint64_t g_q8_fold_oracle_byte_mismatches; +static uint64_t g_q8_fold_oracle_output_calls; +static uint64_t g_q8_fold_oracle_output_mismatches; +static uint64_t g_q8_fold_oracle_raw_moe_calls; +static uint64_t g_q8_fold_oracle_aligned_q8_calls; +static uint64_t g_q8_fold_oracle_aligned_iq2_calls; +static uint64_t g_q8_fold_oracle_skips; +static int g_q8_fold_oracle_report_registered; + +static void ds4_mmq_q8_fold_oracle_report(void) { + fprintf(stderr, + "ds4: CUDA Q8_1 fold oracle: byte_calls=%llu " + "byte_mismatches=%llu output_calls=%llu " + "output_mismatches=%llu raw_moe_calls=%llu " + "aligned_q8_calls=%llu aligned_iq2_calls=%llu skips=%llu " + "(canonical reference retained)\n", + (unsigned long long)g_q8_fold_oracle_byte_calls, + (unsigned long long)g_q8_fold_oracle_byte_mismatches, + (unsigned long long)g_q8_fold_oracle_output_calls, + (unsigned long long)g_q8_fold_oracle_output_mismatches, + (unsigned long long)g_q8_fold_oracle_raw_moe_calls, + (unsigned long long)g_q8_fold_oracle_aligned_q8_calls, + (unsigned long long)g_q8_fold_oracle_aligned_iq2_calls, + (unsigned long long)g_q8_fold_oracle_skips); +} + +static bool ds4_mmq_q8_fold_oracle_enabled() { + const char *env = getenv("DS4_CUDA_Q8_FOLD_ORACLE"); + const bool enabled = env && strcmp(env, "1") == 0; + if (enabled && !g_q8_fold_oracle_report_registered) { + g_q8_fold_oracle_report_registered = 1; + (void)atexit(ds4_mmq_q8_fold_oracle_report); + } + return enabled; +} + +__global__ static void q8_fold_output_compare_kernel( + uint32_t *mismatch, const float *candidate, + const float *reference, uint64_t n) { + const uint64_t i = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (i < n && __float_as_uint(candidate[i]) != + __float_as_uint(reference[i])) { + atomicExch(mismatch, 1u); + } +} + +/* Byte oracle for every consumer. On mismatch the fresh canonical bytes + * overwrite the sidecar before it is consumed. Any setup/capture failure + * rejects the fold entirely so the established prelude quantizes again. */ +static bool ds4_mmq_q8_fold_oracle_bytes( + const float *X_f32, int64_t K, int64_t ne10_padded, + char *folded, cudaStream_t stream) { + if (!ds4_mmq_q8_fold_oracle_enabled()) return true; + if (!X_f32 || !folded || K <= 0 || ne10_padded != K || + (K % QK8_1) != 0) { + g_q8_fold_oracle_skips++; + return false; + } + cudaStreamCaptureStatus capture = cudaStreamCaptureStatusNone; + if (cudaStreamIsCapturing(stream, &capture) != cudaSuccess || + capture != cudaStreamCaptureStatusNone) { + (void)cudaGetLastError(); + g_q8_fold_oracle_skips++; + return false; + } + const size_t bytes = (size_t)ne10_padded * sizeof(block_q8_1) / QK8_1; + if (bytes == 0u || bytes > 16384u) { + g_q8_fold_oracle_skips++; + return false; + } + char *fresh = nullptr; + char *host = (char *)malloc(bytes * 2u); + if (!host || cudaMalloc((void **)&fresh, bytes) != cudaSuccess || !fresh) { + free(host); + (void)cudaGetLastError(); + g_q8_fold_oracle_skips++; + return false; + } + quantize_row_q8_1_cuda( + X_f32, /*ids=*/nullptr, fresh, GGML_TYPE_Q8_0, + /*ne00=*/K, /*s11=*/K, /*s12=*/K, /*s13=*/K, + /*ne0=*/ne10_padded, /*ne1=*/1, /*ne2=*/1, /*ne3=*/1, + stream); + bool setup_ok = cudaGetLastError() == cudaSuccess && + cudaStreamSynchronize(stream) == cudaSuccess && + cudaMemcpy(host, folded, bytes, + cudaMemcpyDeviceToHost) == cudaSuccess && + cudaMemcpy(host + bytes, fresh, bytes, + cudaMemcpyDeviceToHost) == cudaSuccess; + if (!setup_ok) { + (void)cudaGetLastError(); + (void)cudaFree(fresh); + free(host); + g_q8_fold_oracle_skips++; + return false; + } + const bool match = memcmp(host, host + bytes, bytes) == 0; + g_q8_fold_oracle_byte_calls++; + if (!match) { + g_q8_fold_oracle_byte_mismatches++; + if (cudaMemcpyAsync(folded, fresh, bytes, + cudaMemcpyDeviceToDevice, stream) != cudaSuccess || + cudaStreamSynchronize(stream) != cudaSuccess) { + (void)cudaGetLastError(); + (void)cudaFree(fresh); + free(host); + g_q8_fold_oracle_skips++; + return false; + } + } + (void)cudaFree(fresh); + free(host); + return true; +} + static char *ds4_mmq_folded_q81(const float *X_f32, int64_t K, int n_tokens, - int64_t ne10_padded) { + int64_t ne10_padded, cudaStream_t stream) { if (n_tokens != 1 || ne10_padded != K) return nullptr; const void *p = nullptr; - if (!ds4_cuda_q8_fold_take_q81((const void *)X_f32, (uint64_t)K, &p)) return nullptr; + if (!ds4_cuda_q8_fold_take_q81( + (const void *)X_f32, (uint64_t)K, stream, &p)) return nullptr; + if (!ds4_mmq_q8_fold_oracle_bytes( + X_f32, K, ne10_padded, (char *)(uintptr_t)p, stream)) { + return nullptr; + } static int logged = 0; if (!logged) { logged = 1; @@ -3276,10 +3399,10 @@ int ds4_mmq_dense_vec_impl( q4_k1024_oracle = q4_k1024_exact && q4_k1024_env_flag("DS4_CUDA_Q4_K1024_PERSISTENT_ORACLE"); const bool enable = - getenv("DS4_CUDA_ENABLE_Q4_K1024_PERSISTENT") != nullptr; + q4_k1024_env_flag("DS4_CUDA_ENABLE_Q4_K1024_PERSISTENT"); const bool disable = - getenv("DS4_CUDA_NO_Q4_K1024_PERSISTENT") != nullptr || - getenv("DS4_CUDA_NO_Q4_GB10_FAST") != nullptr; + q4_k1024_env_flag("DS4_CUDA_NO_Q4_K1024_PERSISTENT") || + q4_k1024_env_flag("DS4_CUDA_NO_Q4_GB10_FAST"); if (q4_k1024_exact) { g_q4_k1024_persistent_candidates++; } @@ -3300,7 +3423,8 @@ int ds4_mmq_dense_vec_impl( if (q4_k1024_exact && !q4_k1024_eligible) { g_q4_k1024_persistent_fallbacks++; const bool require = - getenv("DS4_CUDA_REQUIRE_Q4_K1024_PERSISTENT") != nullptr; + q4_k1024_env_flag( + "DS4_CUDA_REQUIRE_Q4_K1024_PERSISTENT"); if (require || q4_k1024_oracle) { g_q4_k1024_persistent_require_failures++; if (q4_k1024_oracle) { @@ -4296,7 +4420,9 @@ int ds4_mmq_moe_gate_up_mid_vec_impl( ggml_cuda_pool_alloc src1_q8_1_pool; // M2-Inc2a: the fused HC stage may have emitted this activation's q8_1 // codes already (ffn_norm) -- take them and skip the quantize prelude. - char *src1_q8_1_ptr = ds4_mmq_folded_q81(X_f32, K, n_tokens, ne10_padded); + char *src1_q8_1_ptr = ds4_mmq_folded_q81( + X_f32, K, n_tokens, ne10_padded, stream); + const bool folded_hit = src1_q8_1_ptr != nullptr; cudaError_t err; if (!src1_q8_1_ptr) { if (g_q81_scratch_enabled && g_q81_scratch_ptr && g_q81_scratch_bytes >= nbytes_q8_1) { @@ -4328,6 +4454,98 @@ int ds4_mmq_moe_gate_up_mid_vec_impl( const dim3 block_nums((M + 63) / 64, n_tokens * n_expert_used); const dim3 block_dims(256); + if (folded_hit && ds4_mmq_q8_fold_oracle_enabled()) { + const size_t q8_bytes = + (size_t)ne10_padded * sizeof(block_q8_1) / QK8_1; + const uint64_t mid_count = + (uint64_t)M * (uint64_t)n_tokens * (uint64_t)n_expert_used; + const size_t mid_bytes = (size_t)mid_count * sizeof(float); + block_q8_1 *fresh = nullptr; + float *reference = nullptr; + uint32_t *mismatch_device = nullptr; + const bool allocated = + cudaMalloc((void **)&fresh, q8_bytes) == cudaSuccess && + cudaMalloc((void **)&reference, mid_bytes) == cudaSuccess && + cudaMalloc((void **)&mismatch_device, sizeof(uint32_t)) == cudaSuccess && + fresh && reference && mismatch_device; + if (!allocated) { + (void)cudaGetLastError(); + if (fresh) (void)cudaFree(fresh); + if (reference) (void)cudaFree(reference); + if (mismatch_device) (void)cudaFree(mismatch_device); + g_q8_fold_oracle_skips++; + } else { + cudaError_t oracle_err = cudaMemsetAsync( + mismatch_device, 0, sizeof(uint32_t), stream); + if (oracle_err == cudaSuccess) { + quantize_row_q8_1_cuda( + X_f32, /*ids=*/nullptr, fresh, type, + /*ne00=*/K, /*s11=*/K, /*s12=*/K, /*s13=*/K, + /*ne0=*/ne10_padded, /*ne1=*/1, /*ne2=*/1, /*ne3=*/1, + stream); + oracle_err = cudaGetLastError(); + } + if (oracle_err == cudaSuccess) { + ds4_mmq_moe_gate_up_mid_q8_1_qwarp32_kernel<<< + block_nums, block_dims, 0, stream>>>( + W_gate, W_up, + (const block_q8_1 *)src1_q8_1_ptr, + ids, weights, mid_f32, + (uint32_t)K, (uint32_t)M, + (uint32_t)n_tokens, (uint32_t)n_experts, + stride_row_x, stride_col_y, stride_channel_x, clamp); + oracle_err = cudaGetLastError(); + } + if (oracle_err == cudaSuccess) { + ds4_mmq_moe_gate_up_mid_q8_1_qwarp32_kernel<<< + block_nums, block_dims, 0, stream>>>( + W_gate, W_up, fresh, ids, weights, reference, + (uint32_t)K, (uint32_t)M, + (uint32_t)n_tokens, (uint32_t)n_experts, + stride_row_x, stride_col_y, stride_channel_x, clamp); + oracle_err = cudaGetLastError(); + } + if (oracle_err == cudaSuccess) { + q8_fold_output_compare_kernel<<< + (unsigned)((mid_count + 255u) / 256u), 256, 0, stream>>>( + mismatch_device, mid_f32, reference, mid_count); + oracle_err = cudaGetLastError(); + } + if (oracle_err == cudaSuccess) { + oracle_err = cudaMemcpyAsync( + mid_f32, reference, mid_bytes, + cudaMemcpyDeviceToDevice, stream); + } + uint32_t mismatch_host = 0u; + if (oracle_err == cudaSuccess) { + oracle_err = cudaMemcpyAsync( + &mismatch_host, mismatch_device, sizeof(mismatch_host), + cudaMemcpyDeviceToHost, stream); + } + if (oracle_err == cudaSuccess) { + oracle_err = cudaStreamSynchronize(stream); + } + (void)cudaFree(fresh); + (void)cudaFree(reference); + (void)cudaFree(mismatch_device); + if (oracle_err != cudaSuccess) { + (void)cudaGetLastError(); + g_q8_fold_oracle_skips++; + fprintf(stderr, "%s: fold consumer oracle failed\n", tag); + return -3; + } + g_q8_fold_oracle_output_calls++; + g_q8_fold_oracle_raw_moe_calls++; + if (mismatch_host != 0u) { + g_q8_fold_oracle_output_mismatches++; + fprintf(stderr, + "ds4: CUDA Q8_1 fold oracle found a raw MoE " + "consumer output mismatch; retained canonical " + "output\n"); + } + return 0; + } + } ds4_mmq_moe_gate_up_mid_q8_1_qwarp32_kernel<<>>( W_gate, W_up, (const block_q8_1 *)src1_q8_1_ptr, ids, weights, mid_f32, (uint32_t)K, (uint32_t)M, (uint32_t)n_tokens, (uint32_t)n_experts, @@ -4416,7 +4634,7 @@ static int ds4_mmq_q4_K_dense_pair_vec_impl( const size_t qbytes = (size_t)padded * sizeof(block_q8_1) / QK8_1; ggml_cuda_pool_alloc q8_pool; - char *x8 = ds4_mmq_folded_q81(X, K, 1, padded); + char *x8 = ds4_mmq_folded_q81(X, K, 1, padded, stream); if (!x8) { if (void *scratch = ds4_mmq_aligned_q81_scratch(dev, qbytes)) { x8 = (char *)scratch; @@ -4770,6 +4988,162 @@ __global__ void q8_0_aligned_dense_vec_nc_kernel( } } +static int ds4_q8_aligned_warps_per_block(int cc); + +static cudaError_t q8_0_aligned_dense_vec_launch( + float *out, const int4 *qs, const __half *dq, + const block_q8_1 *x8, int M, int N, int K, + cudaStream_t stream) { + switch (N) { + case 1: + if (g_gb10_optimizations && + getenv("DS4_CUDA_NO_Q8_ALIGNED_PERSISTENT") == NULL && + (K == 1024 || K == 4096) && M >= 32768) { + const uint64_t row_blocks = ((uint64_t)(unsigned)M + 7u) / 8u; + const unsigned persistent_blocks = + row_blocks < 288u ? (unsigned)row_blocks : 288u; + if (K == 1024) { + q8_0_aligned_dense_vec_k1024_persistent_kernel<<< + persistent_blocks, 256, 0, stream>>>( + out, qs, dq, x8, M); + } else { + q8_0_aligned_dense_vec_persistent_kernel<<< + persistent_blocks, 256, 0, stream>>>( + out, qs, dq, x8, M, K / 32); + } + } else { + switch (ds4_q8_aligned_warps_per_block( + ggml_cuda_info().devices[ggml_cuda_get_device()].cc)) { + case 16: + q8_0_aligned_dense_vec_kernel<16> + <<<((unsigned)M + 15u) / 16u, 512, 0, stream>>>( + out, qs, dq, x8, M, K / 32); + break; + case 8: + q8_0_aligned_dense_vec_kernel<8> + <<<((unsigned)M + 7u) / 8u, 256, 0, stream>>>( + out, qs, dq, x8, M, K / 32); + break; + case 4: + q8_0_aligned_dense_vec_kernel<4> + <<<((unsigned)M + 3u) / 4u, 128, 0, stream>>>( + out, qs, dq, x8, M, K / 32); + break; + case 2: + q8_0_aligned_dense_vec_kernel<2> + <<<((unsigned)M + 1u) / 2u, 64, 0, stream>>>( + out, qs, dq, x8, M, K / 32); + break; + default: + q8_0_aligned_dense_vec_kernel<1> + <<<(unsigned)M, 32, 0, stream>>>( + out, qs, dq, x8, M, K / 32); + break; + } + } + break; + case 2: q8_0_aligned_dense_vec_nc_kernel<2><<<(unsigned)M, 32, 0, stream>>>(out, qs, dq, x8, M, K / 32); break; + case 3: q8_0_aligned_dense_vec_nc_kernel<3><<<(unsigned)M, 32, 0, stream>>>(out, qs, dq, x8, M, K / 32); break; + case 4: q8_0_aligned_dense_vec_nc_kernel<4><<<(unsigned)M, 32, 0, stream>>>(out, qs, dq, x8, M, K / 32); break; + case 5: q8_0_aligned_dense_vec_nc_kernel<5><<<(unsigned)M, 32, 0, stream>>>(out, qs, dq, x8, M, K / 32); break; + case 6: q8_0_aligned_dense_vec_nc_kernel<6><<<(unsigned)M, 32, 0, stream>>>(out, qs, dq, x8, M, K / 32); break; + case 7: q8_0_aligned_dense_vec_nc_kernel<7><<<(unsigned)M, 32, 0, stream>>>(out, qs, dq, x8, M, K / 32); break; + case 8: q8_0_aligned_dense_vec_nc_kernel<8><<<(unsigned)M, 32, 0, stream>>>(out, qs, dq, x8, M, K / 32); break; + default: return cudaErrorInvalidValue; + } + return cudaGetLastError(); +} + +/* Full consumer oracle for the folded single-column Q8_0 aligned entry. + * It regenerates canonical Q8_1, runs the exact same consumer twice, compares + * output bits, and always leaves the freshly quantized reference output in + * the caller buffer. Return 1 when handled, 0 when diagnostics could not be + * set up before enqueue, and -1 after a CUDA failure. */ +static int q8_fold_q8_aligned_output_oracle( + const float *X_f32, const block_q8_1 *folded, + float *out, const int4 *qs, const __half *dq, + int M, int K, cudaStream_t stream) { + if (!ds4_mmq_q8_fold_oracle_enabled() || !folded || M <= 0 || K <= 0) { + return 0; + } + cudaStreamCaptureStatus capture = cudaStreamCaptureStatusNone; + if (cudaStreamIsCapturing(stream, &capture) != cudaSuccess || + capture != cudaStreamCaptureStatusNone) { + (void)cudaGetLastError(); + g_q8_fold_oracle_skips++; + return 0; + } + const size_t q8_bytes = (size_t)K * sizeof(block_q8_1) / QK8_1; + const size_t out_bytes = (size_t)M * sizeof(float); + block_q8_1 *fresh = nullptr; + float *reference = nullptr; + uint32_t *mismatch_device = nullptr; + if (cudaMalloc((void **)&fresh, q8_bytes) != cudaSuccess || + cudaMalloc((void **)&reference, out_bytes) != cudaSuccess || + cudaMalloc((void **)&mismatch_device, sizeof(uint32_t)) != cudaSuccess || + !fresh || !reference || !mismatch_device) { + (void)cudaGetLastError(); + if (fresh) (void)cudaFree(fresh); + if (reference) (void)cudaFree(reference); + if (mismatch_device) (void)cudaFree(mismatch_device); + g_q8_fold_oracle_skips++; + return 0; + } + + cudaError_t err = cudaMemsetAsync( + mismatch_device, 0, sizeof(uint32_t), stream); + if (err == cudaSuccess) { + quantize_row_q8_1_cuda( + X_f32, /*ids=*/nullptr, fresh, GGML_TYPE_Q8_0, + /*ne00=*/K, /*s11=*/K, /*s12=*/K, /*s13=*/K, + /*ne0=*/K, /*ne1=*/1, /*ne2=*/1, /*ne3=*/1, stream); + err = cudaGetLastError(); + } + if (err == cudaSuccess) { + err = q8_0_aligned_dense_vec_launch( + out, qs, dq, folded, M, 1, K, stream); + } + if (err == cudaSuccess) { + err = q8_0_aligned_dense_vec_launch( + reference, qs, dq, fresh, M, 1, K, stream); + } + if (err == cudaSuccess) { + q8_fold_output_compare_kernel<<< + (unsigned)(((uint64_t)M + 255u) / 256u), 256, 0, stream>>>( + mismatch_device, out, reference, (uint64_t)M); + err = cudaGetLastError(); + } + if (err == cudaSuccess) { + err = cudaMemcpyAsync(out, reference, out_bytes, + cudaMemcpyDeviceToDevice, stream); + } + uint32_t mismatch_host = 0u; + if (err == cudaSuccess) { + err = cudaMemcpyAsync(&mismatch_host, mismatch_device, + sizeof(mismatch_host), + cudaMemcpyDeviceToHost, stream); + } + if (err == cudaSuccess) err = cudaStreamSynchronize(stream); + + (void)cudaFree(fresh); + (void)cudaFree(reference); + (void)cudaFree(mismatch_device); + if (err != cudaSuccess) { + (void)cudaGetLastError(); + g_q8_fold_oracle_skips++; + return -1; + } + g_q8_fold_oracle_output_calls++; + g_q8_fold_oracle_aligned_q8_calls++; + if (mismatch_host != 0u) { + g_q8_fold_oracle_output_mismatches++; + fprintf(stderr, + "ds4: CUDA Q8_1 fold oracle found a Q8 aligned consumer " + "output mismatch; retained canonical output\n"); + } + return 1; +} + extern "C" uint64_t ds4_mmq_q8_0_aligned_bytes(int M, int K) { if (M <= 0 || K <= 0 || K % 1024 != 0) return 0; const uint64_t nblk = (uint64_t)M * (uint64_t)(K / 32); @@ -4854,7 +5228,7 @@ extern "C" int ds4_mmq_q8_0_aligned_dense_vec_pair( const size_t qbytes = (size_t)padded * sizeof(block_q8_1) / QK8_1; ggml_cuda_pool_alloc q8_pool; - char *x8 = ds4_mmq_folded_q81(X_f32, K, 1, padded); + char *x8 = ds4_mmq_folded_q81(X_f32, K, 1, padded, stream); if (!x8) { if (g_q81_scratch_enabled && g_q81_scratch_ptr && g_q81_scratch_bytes >= qbytes) { @@ -4934,30 +5308,37 @@ extern "C" int ds4_mmq_q8_0_aligned_dense_vec( // M2-Inc2a: producer-emitted q8_1 codes (qr_norm from the qkv-rms // kernel) -- take them and skip the quantize prelude. Single-column // producers only; verify widths always quantize. - char *x8 = N == 1 ? ds4_mmq_folded_q81(X_f32, K, 1, ne10_padded) : NULL; + char *x8 = N == 1 + ? ds4_mmq_folded_q81(X_f32, K, 1, ne10_padded, stream) + : NULL; + const bool folded_hit = x8 != NULL; cudaError_t err; if (!x8) { - if (getenv("DS4_CUDA_NO_Q8_ALIGNED_DENSE_SCRATCH") == NULL && - g_aligned_q81_scratch_ptr && - g_aligned_q81_scratch_bytes >= nbytes_q8_1) { - x8 = (char *)g_aligned_q81_scratch_ptr; - } else if (g_q81_scratch_enabled && g_q81_scratch_ptr && g_q81_scratch_bytes >= nbytes_q8_1) { - x8 = (char *)g_q81_scratch_ptr; - } else { - q8_pool.alloc(ctx->pool(), nbytes_q8_1); - x8 = q8_pool.get(); - } - quantize_row_q8_1_cuda( - X_f32, /*ids=*/nullptr, (void *)x8, - GGML_TYPE_Q8_0, /*ne00=*/K, - /*s11=*/(int64_t)K, /*s12=*/(int64_t)K * N, /*s13=*/(int64_t)K * N, - /*ne0=*/ne10_padded, /*ne1=*/N, /*ne2=*/1, /*ne3=*/1, - stream); - err = cudaGetLastError(); - if (err != cudaSuccess) { - fprintf(stderr, "%s: quantize_row_q8_1_cuda failed: %s\n", tag, cudaGetErrorString(err)); - return -2; - } + if (getenv("DS4_CUDA_NO_Q8_ALIGNED_DENSE_SCRATCH") == NULL) { + x8 = (char *)ds4_mmq_aligned_q81_scratch(dev, nbytes_q8_1); + } + if (!x8 && g_q81_scratch_enabled && g_q81_scratch_ptr && + g_q81_scratch_bytes >= nbytes_q8_1) { + x8 = (char *)g_q81_scratch_ptr; + } + if (!x8) { + q8_pool.alloc(ctx->pool(), nbytes_q8_1); + x8 = q8_pool.get(); + } + quantize_row_q8_1_cuda( + X_f32, /*ids=*/nullptr, (void *)x8, + GGML_TYPE_Q8_0, /*ne00=*/K, + /*s11=*/(int64_t)K, /*s12=*/(int64_t)K * N, + /*s13=*/(int64_t)K * N, + /*ne0=*/ne10_padded, /*ne1=*/N, /*ne2=*/1, /*ne3=*/1, + stream); + err = cudaGetLastError(); + if (err != cudaSuccess) { + fprintf(stderr, + "%s: quantize_row_q8_1_cuda failed: %s\n", + tag, cudaGetErrorString(err)); + return -2; + } } const uint64_t nblk = (uint64_t)M * (uint64_t)(K / 32); @@ -4965,64 +5346,17 @@ extern "C" int ds4_mmq_q8_0_aligned_dense_vec( const int4 *qsp = (const int4 *)((const char *)W_aligned + dq_bytes); const __half *dqp = (const __half *)W_aligned; const block_q8_1 *x8p = (const block_q8_1 *)x8; - switch (N) { - case 1: - if (g_gb10_optimizations && - getenv("DS4_CUDA_NO_Q8_ALIGNED_PERSISTENT") == NULL && - (K == 1024 || K == 4096) && M >= 32768) { - const uint64_t row_blocks = ((uint64_t)(unsigned)M + 7u) / 8u; - /* 288 = 48 GB10 SMs x __launch_bounds__(256, 6) resident CTAs. */ - const unsigned persistent_blocks = - row_blocks < 288u ? (unsigned)row_blocks : 288u; - if (K == 1024) { - q8_0_aligned_dense_vec_k1024_persistent_kernel<<< - persistent_blocks, 256, 0, stream>>>( - out_f32, qsp, dqp, x8p, M); - } else { - q8_0_aligned_dense_vec_persistent_kernel<<< - persistent_blocks, 256, 0, stream>>>( - out_f32, qsp, dqp, x8p, M, K / 32); - } - } else { - switch (ds4_q8_aligned_warps_per_block( - ggml_cuda_info().devices[dev].cc)) { - case 16: - q8_0_aligned_dense_vec_kernel<16> - <<<((unsigned)M + 15u) / 16u, 512, 0, stream>>>( - out_f32, qsp, dqp, x8p, M, K / 32); - break; - case 8: - q8_0_aligned_dense_vec_kernel<8> - <<<((unsigned)M + 7u) / 8u, 256, 0, stream>>>( - out_f32, qsp, dqp, x8p, M, K / 32); - break; - case 4: - q8_0_aligned_dense_vec_kernel<4> - <<<((unsigned)M + 3u) / 4u, 128, 0, stream>>>( - out_f32, qsp, dqp, x8p, M, K / 32); - break; - case 2: - q8_0_aligned_dense_vec_kernel<2> - <<<((unsigned)M + 1u) / 2u, 64, 0, stream>>>( - out_f32, qsp, dqp, x8p, M, K / 32); - break; - default: - q8_0_aligned_dense_vec_kernel<1> - <<<(unsigned)M, 32, 0, stream>>>( - out_f32, qsp, dqp, x8p, M, K / 32); - break; - } + if (folded_hit && N == 1 && ds4_mmq_q8_fold_oracle_enabled()) { + const int oracle_rc = q8_fold_q8_aligned_output_oracle( + X_f32, x8p, out_f32, qsp, dqp, M, K, stream); + if (oracle_rc > 0) return 0; + if (oracle_rc < 0) { + fprintf(stderr, "%s: fold consumer oracle failed\n", tag); + return -3; } - break; - case 2: q8_0_aligned_dense_vec_nc_kernel<2><<<(unsigned)M, 32, 0, stream>>>(out_f32, qsp, dqp, x8p, M, K / 32); break; - case 3: q8_0_aligned_dense_vec_nc_kernel<3><<<(unsigned)M, 32, 0, stream>>>(out_f32, qsp, dqp, x8p, M, K / 32); break; - case 4: q8_0_aligned_dense_vec_nc_kernel<4><<<(unsigned)M, 32, 0, stream>>>(out_f32, qsp, dqp, x8p, M, K / 32); break; - case 5: q8_0_aligned_dense_vec_nc_kernel<5><<<(unsigned)M, 32, 0, stream>>>(out_f32, qsp, dqp, x8p, M, K / 32); break; - case 6: q8_0_aligned_dense_vec_nc_kernel<6><<<(unsigned)M, 32, 0, stream>>>(out_f32, qsp, dqp, x8p, M, K / 32); break; - case 7: q8_0_aligned_dense_vec_nc_kernel<7><<<(unsigned)M, 32, 0, stream>>>(out_f32, qsp, dqp, x8p, M, K / 32); break; - case 8: q8_0_aligned_dense_vec_nc_kernel<8><<<(unsigned)M, 32, 0, stream>>>(out_f32, qsp, dqp, x8p, M, K / 32); break; } - err = cudaGetLastError(); + err = q8_0_aligned_dense_vec_launch( + out_f32, qsp, dqp, x8p, M, N, K, stream); if (err != cudaSuccess) { fprintf(stderr, "%s: kernel launch failed: %s\n", tag, cudaGetErrorString(err)); return -3; @@ -5333,7 +5667,9 @@ extern "C" uint64_t ds4_mmq_iq2_xxs_aligned_bytes(int M, int K, int n_experts) { // pool otherwise) or nullptr on failure; *pool must outlive the launches. static char *iq2_aligned_quantize_xn( const char *tag, const float *X_f32, int K, int n_tokens, - ggml_cuda_pool_alloc *pool, cudaStream_t stream) { + ggml_cuda_pool_alloc *pool, cudaStream_t stream, + bool *was_folded) { + if (was_folded) *was_folded = false; const int dev = ggml_cuda_get_device(); ggml_backend_cuda_context * ctx = get_ctx_for_device(dev); if (!ctx) { @@ -5345,8 +5681,10 @@ static char *iq2_aligned_quantize_xn( const size_t nbytes_q8_1 = (size_t)n_tokens * ne10_padded * sizeof(block_q8_1) / QK8_1; // M2-Inc2a: producer-emitted q8_1 codes (ffn_norm from the fused HC // stage) -- take them and skip the quantize prelude. - char *folded = ds4_mmq_folded_q81(X_f32, K, n_tokens, ne10_padded); + char *folded = ds4_mmq_folded_q81( + X_f32, K, n_tokens, ne10_padded, stream); if (folded) { + if (was_folded) *was_folded = true; // C3-Inc4 fold twin selftest (DS4_Q8_FOLD_SELFTEST=, // eager legs only -- syncs the stream): the taken sidecar must be // byte-identical to the fresh quantize this prelude would have run. @@ -5424,7 +5762,8 @@ extern "C" int ds4_mmq_iq2_xxs_aligned_moe_pair_vec( return -1; } ggml_cuda_pool_alloc q8_pool; - char *x8 = iq2_aligned_quantize_xn(tag, X_f32, K, n_tokens, &q8_pool, stream); + char *x8 = iq2_aligned_quantize_xn( + tag, X_f32, K, n_tokens, &q8_pool, stream, nullptr); if (!x8) return -2; const uint64_t nblk = (uint64_t)n_experts * (uint64_t)M * (uint64_t)(K / 256); @@ -5461,7 +5800,9 @@ extern "C" int ds4_mmq_iq2_xxs_aligned_moe_gate_up_mid_vec( return -1; } ggml_cuda_pool_alloc q8_pool; - char *x8 = iq2_aligned_quantize_xn(tag, X_f32, K, n_tokens, &q8_pool, stream); + bool folded_hit = false; + char *x8 = iq2_aligned_quantize_xn( + tag, X_f32, K, n_tokens, &q8_pool, stream, &folded_hit); if (!x8) return -2; const uint64_t nblk = (uint64_t)n_experts * (uint64_t)M * (uint64_t)(K / 256); @@ -5471,6 +5812,91 @@ extern "C" int ds4_mmq_iq2_xxs_aligned_moe_gate_up_mid_vec( const __half *dq_g = (const __half *)W_gate_aligned; const uint2 *qs_u = (const uint2 *)((const char *)W_up_aligned + dq_bytes); const __half *dq_u = (const __half *)W_up_aligned; + if (folded_hit && n_tokens == 1 && + ds4_mmq_q8_fold_oracle_enabled()) { + const size_t q8_bytes = (size_t)K * sizeof(block_q8_1) / QK8_1; + const uint64_t mid_count = + (uint64_t)M * (uint64_t)n_expert_used; + const size_t mid_bytes = (size_t)mid_count * sizeof(float); + block_q8_1 *fresh = nullptr; + float *reference = nullptr; + uint32_t *mismatch_device = nullptr; + const bool allocated = + cudaMalloc((void **)&fresh, q8_bytes) == cudaSuccess && + cudaMalloc((void **)&reference, mid_bytes) == cudaSuccess && + cudaMalloc((void **)&mismatch_device, sizeof(uint32_t)) == cudaSuccess && + fresh && reference && mismatch_device; + if (!allocated) { + (void)cudaGetLastError(); + if (fresh) (void)cudaFree(fresh); + if (reference) (void)cudaFree(reference); + if (mismatch_device) (void)cudaFree(mismatch_device); + g_q8_fold_oracle_skips++; + } else { + cudaError_t oracle_err = cudaMemsetAsync( + mismatch_device, 0, sizeof(uint32_t), stream); + if (oracle_err == cudaSuccess) { + quantize_row_q8_1_cuda( + X_f32, /*ids=*/nullptr, fresh, GGML_TYPE_IQ2_XXS, + /*ne00=*/K, /*s11=*/K, /*s12=*/K, /*s13=*/K, + /*ne0=*/K, /*ne1=*/1, /*ne2=*/1, /*ne3=*/1, stream); + oracle_err = cudaGetLastError(); + } + if (oracle_err == cudaSuccess) { + iq2_xxs_aligned_moe_gate_up_mid_kernel<<>>( + mid_f32, qs_g, dq_g, qs_u, dq_u, + (const block_q8_1 *)x8, ids, weights, + M, K / 256, K / 32, n_expert_used, clamp); + oracle_err = cudaGetLastError(); + } + if (oracle_err == cudaSuccess) { + iq2_xxs_aligned_moe_gate_up_mid_kernel<<>>( + reference, qs_g, dq_g, qs_u, dq_u, fresh, + ids, weights, M, K / 256, K / 32, + n_expert_used, clamp); + oracle_err = cudaGetLastError(); + } + if (oracle_err == cudaSuccess) { + q8_fold_output_compare_kernel<<< + (unsigned)((mid_count + 255u) / 256u), 256, 0, stream>>>( + mismatch_device, mid_f32, reference, mid_count); + oracle_err = cudaGetLastError(); + } + if (oracle_err == cudaSuccess) { + oracle_err = cudaMemcpyAsync( + mid_f32, reference, mid_bytes, + cudaMemcpyDeviceToDevice, stream); + } + uint32_t mismatch_host = 0u; + if (oracle_err == cudaSuccess) { + oracle_err = cudaMemcpyAsync( + &mismatch_host, mismatch_device, sizeof(mismatch_host), + cudaMemcpyDeviceToHost, stream); + } + if (oracle_err == cudaSuccess) { + oracle_err = cudaStreamSynchronize(stream); + } + (void)cudaFree(fresh); + (void)cudaFree(reference); + (void)cudaFree(mismatch_device); + if (oracle_err != cudaSuccess) { + (void)cudaGetLastError(); + g_q8_fold_oracle_skips++; + fprintf(stderr, "%s: fold consumer oracle failed\n", tag); + return -3; + } + g_q8_fold_oracle_output_calls++; + g_q8_fold_oracle_aligned_iq2_calls++; + if (mismatch_host != 0u) { + g_q8_fold_oracle_output_mismatches++; + fprintf(stderr, + "ds4: CUDA Q8_1 fold oracle found an IQ2 MoE " + "consumer output mismatch; retained canonical " + "output\n"); + } + return 0; + } + } /* v0.4 V6: verify widths dedup expert overlap (see the dedup kernel's * header comment). n_tokens==1 has no cross-token overlap and keeps * the per-slot kernel; widths beyond the verify envelope likewise. diff --git a/cuda/mmq/test/d2r_stubs.cu b/cuda/mmq/test/d2r_stubs.cu index e9249e790..81fb1a9e3 100644 --- a/cuda/mmq/test/d2r_stubs.cu +++ b/cuda/mmq/test/d2r_stubs.cu @@ -1,10 +1,12 @@ #include "ds4_mmq_d2r.cuh" extern "C" int ds4_cuda_q8_fold_take_q81( - const void *src, uint64_t in_dim, const void **q81) { + const void *src, uint64_t in_dim, cudaStream_t stream, + const void **q81) { (void)src; (void)in_dim; - (void)q81; + (void)stream; + if (q81) *q81 = nullptr; return 0; } diff --git a/cuda/mmq/test/proto_gemm_dense_q8_d2r.cu b/cuda/mmq/test/proto_gemm_dense_q8_d2r.cu index 78496c5a1..0fec3eb0f 100644 --- a/cuda/mmq/test/proto_gemm_dense_q8_d2r.cu +++ b/cuda/mmq/test/proto_gemm_dense_q8_d2r.cu @@ -42,10 +42,13 @@ #include "ds4_mmq.h" #include "quantize.cuh" -extern "C" int ds4_cuda_q8_fold_take_q81(const void *src, uint64_t in_dim, void *out) { +extern "C" int ds4_cuda_q8_fold_take_q81( + const void *src, uint64_t in_dim, cudaStream_t stream, + const void **q81) { (void)src; (void)in_dim; - (void)out; + (void)stream; + if (q81) *q81 = nullptr; return 0; } diff --git a/cuda/mmq/test/test_mmq_soa_tiles.cu b/cuda/mmq/test/test_mmq_soa_tiles.cu index 3f5f1d1e3..da7385932 100644 --- a/cuda/mmq/test/test_mmq_soa_tiles.cu +++ b/cuda/mmq/test/test_mmq_soa_tiles.cu @@ -39,8 +39,11 @@ // libds4mmq.a references this ds4_cuda.cu symbol from the q8-fold vec paths // (C3 Inc4); the entries under test never reach it, so a "no fold available" // stub satisfies the link. -extern "C" int ds4_cuda_q8_fold_take_q81(const void *src, uint64_t in_dim, void *out) { - (void)src; (void)in_dim; (void)out; +extern "C" int ds4_cuda_q8_fold_take_q81( + const void *src, uint64_t in_dim, cudaStream_t stream, + const void **q81) { + (void)src; (void)in_dim; (void)stream; + if (q81) *q81 = nullptr; return 0; } diff --git a/ds4_cuda.cu b/ds4_cuda.cu index 8eb38e44f..d5a84bec4 100644 --- a/ds4_cuda.cu +++ b/ds4_cuda.cu @@ -21,6 +21,8 @@ #include #include #include +#include +#include #include "cuda/mmq/ds4_mmq.h" #include "cuda/mmq/ds4_repack.h" @@ -72,6 +74,18 @@ typedef struct { int16_t bsums[CUDA_QK_K / 16]; } cuda_block_q8_K; +/* Canonical activation layout consumed by the MMVQ entries in + * cuda/mmq/. Keep this private copy byte-for-byte aligned with + * block_q8_1 in ggml-common.h: half2(d, sum), then 32 signed codes. */ +typedef struct { + __half2 ds; + int8_t qs[32]; +} cuda_block_q8_1; +static_assert(sizeof(cuda_block_q8_1) == 36u, + "canonical Q8_1 block layout drift"); +static_assert(offsetof(cuda_block_q8_1, qs) == 4u, + "canonical Q8_1 code offset drift"); + typedef struct { uint16_t d; uint16_t qs[CUDA_QK_K / 8]; @@ -437,6 +451,46 @@ static void *g_aligned_q81_scratch; * Q4 grouped attention-A (8 tokens * 16 groups * K=4096 plus ids/alignment). */ static const size_t CUDA_ALIGNED_Q81_SCRATCH_BYTES = 96u * 1024u * 1024u; + +/* Opt-in producer-fold sidecars. A slot belongs permanently to one + * (physical device, stream) for the lifetime of a CUDA session. That makes + * reuse stream ordered without events, while a one-shot publication prevents + * a stale activation pointer from being accepted by a later consumer. + * + * Host contract: ds4's CUDA inference dispatcher serializes a session on one + * host thread. The mutex protects registry state and cross-stream/device + * invalidation, but it does not make prepare -> producer enqueue -> publish or + * take -> consumer enqueue atomic when arbitrary host threads share one CUDA + * stream. Such external multi-thread stream submission is unsupported for + * this experiment; keep the fold disabled in that embedding. */ +#define CUDA_Q8_FOLD_SLOTS 16u +#define CUDA_Q8_FOLD_SLOT_BYTES 16384u +struct cuda_q8_fold_slot { + void *q81; + size_t capacity; + const void *src; + const void *model_map; + uint64_t in_dim; + uint64_t epoch; + cudaStream_t stream; + int device; + int owner_valid; + int ready; +}; +static cuda_q8_fold_slot g_q8_fold_slots[CUDA_Q8_FOLD_SLOTS]; +static std::mutex g_q8_fold_mutex; +static uint64_t g_q8_fold_epoch = 1u; +static int g_q8_fold_last_device = -1; +static uint64_t g_q8_fold_prepares; +static uint64_t g_q8_fold_publishes; +static uint64_t g_q8_fold_hits; +static uint64_t g_q8_fold_misses; +static uint64_t g_q8_fold_capture_rejects; +static uint64_t g_q8_fold_invalidations; +static int g_q8_fold_report_registered; +static std::once_flag g_q8_fold_mode_once; +static int g_q8_fold_mode; +static std::atomic g_q8_fold_ever_enabled{0}; static uint64_t g_model_range_bytes; static uint64_t g_q8_f16_bytes; static uint64_t g_q8_f32_bytes; @@ -482,6 +536,234 @@ extern "C" int ds4_gpu_lookup_cache_strict(uint64_t source_offset, uint64_t bytes, int expected_device, void **out_device_ptr); + +static void cuda_q8_fold_report(void) { + fprintf(stderr, + "ds4: CUDA Q8_1 producer fold: prepares=%llu publishes=%llu " + "hits=%llu misses=%llu capture_rejects=%llu invalidations=%llu\n", + (unsigned long long)g_q8_fold_prepares, + (unsigned long long)g_q8_fold_publishes, + (unsigned long long)g_q8_fold_hits, + (unsigned long long)g_q8_fold_misses, + (unsigned long long)g_q8_fold_capture_rejects, + (unsigned long long)g_q8_fold_invalidations); +} + +/* Experimental until the CUDA matrix has run on GB10/DGX. The explicit + * disable is dominant even when the enable variable is present. */ +static void cuda_q8_fold_init_mode(void) { + const char *enable = getenv("DS4_CUDA_ENABLE_Q8_FOLD"); + const char *disable = getenv("DS4_CUDA_NO_Q8_FOLD"); + const int disabled = disable && disable[0] && + strcmp(disable, "0") != 0; + const int enabled = enable && strcmp(enable, "1") == 0; + g_q8_fold_mode = enabled && !disabled; + g_q8_fold_ever_enabled.store(g_q8_fold_mode, + std::memory_order_release); + if (enabled && !disabled && !g_q8_fold_report_registered) { + g_q8_fold_report_registered = 1; + (void)atexit(cuda_q8_fold_report); + fprintf(stderr, + "ds4: DS4_CUDA_ENABLE_Q8_FOLD=1 - experimental canonical " + "Q8_1 producer fold enabled\n"); + } +} + +static int cuda_q8_fold_enabled(void) { + std::call_once(g_q8_fold_mode_once, cuda_q8_fold_init_mode); + return g_q8_fold_mode; +} + +static void cuda_q8_fold_invalidate_locked(void) { + g_q8_fold_epoch++; + if (g_q8_fold_epoch == 0u) g_q8_fold_epoch = 1u; + for (unsigned i = 0; i < CUDA_Q8_FOLD_SLOTS; i++) { + g_q8_fold_slots[i].src = NULL; + g_q8_fold_slots[i].model_map = NULL; + g_q8_fold_slots[i].in_dim = 0u; + g_q8_fold_slots[i].epoch = 0u; + g_q8_fold_slots[i].ready = 0; + } + g_q8_fold_invalidations++; +} + +static void cuda_q8_fold_invalidate_all(void) { + if (!g_q8_fold_ever_enabled.load(std::memory_order_acquire)) return; + std::lock_guard lock(g_q8_fold_mutex); + cuda_q8_fold_invalidate_locked(); +} + +static void cuda_q8_fold_release_all(void) { + std::lock_guard lock(g_q8_fold_mutex); + int previous_device = -1; + (void)cudaGetDevice(&previous_device); + for (unsigned i = 0; i < CUDA_Q8_FOLD_SLOTS; i++) { + cuda_q8_fold_slot *slot = &g_q8_fold_slots[i]; + if (slot->q81) { + if (slot->owner_valid) (void)cudaSetDevice(slot->device); + (void)cudaFree(slot->q81); + } + memset(slot, 0, sizeof(*slot)); + slot->device = -1; + } + if (previous_device >= 0) (void)cudaSetDevice(previous_device); + g_q8_fold_last_device = -1; + cuda_q8_fold_invalidate_locked(); +} + +static int cuda_q8_fold_stream_is_eager(cudaStream_t stream) { + cudaStreamCaptureStatus status = cudaStreamCaptureStatusNone; + const cudaError_t err = cudaStreamIsCapturing(stream, &status); + if (err == cudaSuccess && status == cudaStreamCaptureStatusNone) return 1; + (void)cudaGetLastError(); + std::lock_guard lock(g_q8_fold_mutex); + g_q8_fold_capture_rejects++; + cuda_q8_fold_invalidate_locked(); + return 0; +} + +/* Reserve the stream-owned sidecar before launching a producer. The + * allocation is deliberately forbidden during capture. New producers on + * the same stream may replace an untaken publication: CUDA stream ordering + * guarantees that the old consumer, when there was one, has already read it. */ +static cuda_block_q8_1 *cuda_q8_fold_prepare( + const void *src, uint64_t in_dim, const void *model_map, + cudaStream_t stream) { + if (!cuda_q8_fold_enabled() || !src || in_dim != 4096u || + (in_dim & 31u) != 0u || g_n_gpus != 1 || + !model_map || model_map != g_model_host_base) { + return NULL; + } + if (!cuda_q8_fold_stream_is_eager(stream)) return NULL; + + int device = -1; + if (cudaGetDevice(&device) != cudaSuccess) { + (void)cudaGetLastError(); + cuda_q8_fold_invalidate_all(); + return NULL; + } + const uint64_t blocks = in_dim / 32u; + if (blocks > SIZE_MAX / sizeof(cuda_block_q8_1)) return NULL; + const size_t bytes = (size_t)blocks * sizeof(cuda_block_q8_1); + if (bytes > CUDA_Q8_FOLD_SLOT_BYTES) return NULL; + + std::lock_guard lock(g_q8_fold_mutex); + if (g_q8_fold_last_device >= 0 && + g_q8_fold_last_device != device) { + cuda_q8_fold_invalidate_locked(); + } + g_q8_fold_last_device = device; + cuda_q8_fold_slot *slot = NULL; + for (unsigned i = 0; i < CUDA_Q8_FOLD_SLOTS; i++) { + cuda_q8_fold_slot *candidate = &g_q8_fold_slots[i]; + if (candidate->owner_valid && candidate->device == device && + candidate->stream == stream) { + slot = candidate; + break; + } + if (!slot && !candidate->owner_valid) slot = candidate; + } + if (!slot) return NULL; + + if (!slot->owner_valid) { + void *sidecar = NULL; + const cudaError_t alloc_err = + cudaMalloc(&sidecar, CUDA_Q8_FOLD_SLOT_BYTES); + if (alloc_err != cudaSuccess || !sidecar) { + (void)cudaGetLastError(); + return NULL; + } + slot->q81 = sidecar; + slot->capacity = CUDA_Q8_FOLD_SLOT_BYTES; + slot->stream = stream; + slot->device = device; + slot->owner_valid = 1; + } + if (!slot->q81 || slot->capacity < bytes) return NULL; + + slot->src = src; + slot->model_map = model_map; + slot->in_dim = in_dim; + slot->epoch = g_q8_fold_epoch; + slot->ready = 0; + g_q8_fold_prepares++; + return (cuda_block_q8_1 *)slot->q81; +} + +static void cuda_q8_fold_publish( + const void *src, uint64_t in_dim, const void *model_map, + cudaStream_t stream, const void *q81) { + if (!q81) return; + if (!cuda_q8_fold_stream_is_eager(stream)) return; + int device = -1; + if (cudaGetDevice(&device) != cudaSuccess) { + (void)cudaGetLastError(); + cuda_q8_fold_invalidate_all(); + return; + } + std::lock_guard lock(g_q8_fold_mutex); + if (g_q8_fold_last_device >= 0 && + g_q8_fold_last_device != device) { + cuda_q8_fold_invalidate_locked(); + } + g_q8_fold_last_device = device; + for (unsigned i = 0; i < CUDA_Q8_FOLD_SLOTS; i++) { + cuda_q8_fold_slot *slot = &g_q8_fold_slots[i]; + if (slot->q81 == q81 && slot->device == device && + slot->src == src && + slot->model_map == model_map && slot->in_dim == in_dim && + slot->stream == stream && slot->epoch == g_q8_fold_epoch) { + slot->ready = 1; + g_q8_fold_publishes++; + return; + } + } +} + +static int cuda_q8_fold_take( + const void *src, uint64_t in_dim, cudaStream_t stream, + const void **q81) { + if (q81) *q81 = NULL; + if (!q81 || !cuda_q8_fold_enabled() || !src || in_dim == 0u || + g_n_gpus != 1 || !g_model_host_base) { + return 0; + } + if (!cuda_q8_fold_stream_is_eager(stream)) return 0; + int device = -1; + if (cudaGetDevice(&device) != cudaSuccess) { + (void)cudaGetLastError(); + cuda_q8_fold_invalidate_all(); + return 0; + } + + std::lock_guard lock(g_q8_fold_mutex); + if (g_q8_fold_last_device >= 0 && + g_q8_fold_last_device != device) { + cuda_q8_fold_invalidate_locked(); + } + g_q8_fold_last_device = device; + for (unsigned i = 0; i < CUDA_Q8_FOLD_SLOTS; i++) { + cuda_q8_fold_slot *slot = &g_q8_fold_slots[i]; + if (slot->ready && slot->src == src && slot->in_dim == in_dim && + slot->stream == stream && slot->device == device && + slot->model_map == g_model_host_base && + slot->epoch == g_q8_fold_epoch) { + *q81 = slot->q81; + slot->ready = 0; + slot->src = NULL; + g_q8_fold_hits++; + return 1; + } + } + g_q8_fold_misses++; + /* A pointer miss is a sequencing mismatch, not merely an absent key. + * Tensor scratch addresses can recur after their contents were replaced; + * retaining any ready publication across that mismatch could therefore + * turn a later address match into a stale hit. Bump the session epoch + * and discard every publication while already holding the registry lock. */ + cuda_q8_fold_invalidate_locked(); + return 0; +} __global__ static void dequant_q8_0_to_f16_kernel( __half *out, const unsigned char *w, @@ -586,6 +868,7 @@ static int cuda_span_fully_replaced( static void *cuda_tmp_alloc(uint64_t bytes, const char *what) { if (bytes == 0) return NULL; if (g_cuda_tmp_bytes >= bytes) return g_cuda_tmp; + cuda_q8_fold_invalidate_all(); if (g_cuda_tmp) { if (!cuda_ok(cudaDeviceSynchronize(), "synchronize CUDA scratch growth")) { @@ -612,6 +895,7 @@ static void *cuda_tmp_alloc(uint64_t bytes, const char *what) { static void *tt_scratch_ensure(uint64_t bytes, const char *what) { if (bytes == 0) return NULL; if (g_tt_scratch_bytes >= bytes) return g_tt_scratch; + cuda_q8_fold_invalidate_all(); int device = -1; if (!cuda_ok(cudaGetDevice(&device), "get device for token-tile scratch")) { @@ -669,6 +953,7 @@ static void *cuda_tmp_alloc_on(int logical_tier, uint64_t bytes, const char *wha } ds4_gpu_ctx *ctx = &g_gpu[logical_tier]; if (ctx->scratch_bytes >= bytes) return ctx->scratch; + cuda_q8_fold_invalidate_all(); int prev = -1; cudaError_t derr = cudaGetDevice(&prev); if (derr != cudaSuccess) { @@ -1061,6 +1346,9 @@ static cuda_decode_graph_entry *cuda_decode_graph_find( extern "C" int ds4_gpu_decode_graph_begin(const ds4_decode_graph_key *key) { if (!key || !ds4_gpu_decode_graphs_supported()) return -1; if (g_decode_graph_capturing) return -1; /* no nesting */ + /* A captured/replayed island owns its stream schedule. Never carry an + * eager host-side sidecar publication across that boundary. */ + cuda_q8_fold_invalidate_all(); cuda_decode_graph_entry *e = cuda_decode_graph_find(key); if (!e) { g_decode_graph_no_slots++; @@ -1192,14 +1480,13 @@ extern "C" int ds4_gpu_decode_graph_end(const ds4_decode_graph_key *key) { * prefill logits drift at ULP scale: validated against the official * continuation vectors rather than byte-diffs. DS4_CUDA_MMQ=0 restores * the legacy dispatch. */ -/* Producer-fold registry lookup the vendored ds4_mmq.cu entries probe for - * pre-quantized q8_1 activations (the fork's flat-pool M2-Inc2a fold). - * The flat-pool producers are not ported, so nothing is ever registered: - * always miss, and the entries run their own activation quantize. */ +/* Producer-fold registry lookup used by the vendored MMVQ entries. Stream + * identity is part of the ABI: accepting a sidecar emitted on another stream + * without an event would be a data race, so such a lookup must miss. */ extern "C" int ds4_cuda_q8_fold_take_q81(const void *src, uint64_t in_dim, + cudaStream_t stream, const void **q81) { - (void)src; (void)in_dim; (void)q81; - return 0; + return cuda_q8_fold_take(src, in_dim, stream, q81); } static int cuda_use_mmq(void) { @@ -2739,6 +3026,7 @@ static int cublas_ok(cublasStatus_t st, const char *what) { } extern "C" int ds4_gpu_init_multi(const ds4_gpu_config *cfg) { + cuda_q8_fold_invalidate_all(); ds4_mmq_set_gb10_optimizations(0); if (!cfg || cfg->n_gpus < 1 || cfg->n_gpus > DS4_MAX_GPUS) return 0; memset(g_cuda_is_gb10, 0, sizeof(g_cuda_is_gb10)); @@ -2949,6 +3237,7 @@ extern "C" int ds4_gpu_init(void) { extern "C" void ds4_gpu_cleanup(void) { (void)cudaDeviceSynchronize(); cuda_decode_graphs_shutdown(); + cuda_q8_fold_release_all(); g_current_logical_tier = -1; /* Multi-GPU teardown: events, streams, cublas handles, scratch @@ -3886,6 +4175,7 @@ extern "C" int ds4_gpu_synchronize(void) { return cuda_ok(cudaDeviceSynchronize( extern "C" int ds4_gpu_set_model_map(const void *model_map, uint64_t model_size) { if (!model_map || model_size == 0) return 0; if (g_model_host_base == model_map && g_model_registered_size == model_size) return 1; + cuda_q8_fold_invalidate_all(); cuda_stream_selected_cache_release(); cuda_f16_pair_chunk32_release_all(); cuda_model_range_release_all(); @@ -4087,6 +4377,7 @@ extern "C" int ds4_gpu_prepare_support_model( extern "C" int ds4_gpu_register_model_map_no_copy(const void *model_map, uint64_t model_size) { if (!model_map || model_size == 0) return 0; if (g_model_host_base == model_map && g_model_registered_size == model_size) return 1; + cuda_q8_fold_invalidate_all(); cuda_stream_selected_cache_release(); cuda_f16_pair_chunk32_release_all(); @@ -12430,6 +12721,26 @@ __global__ static void hc_split_weighted_sum_fused_kernel( } } +/* Emit one canonical block_q8_1 from a warp holding 32 consecutive normalized + * values. This is intentionally the same XOR butterfly, scale, rounding and + * half2 conversion as cuda/mmq/quantize.cu::quantize_q8_1. */ +__device__ static __forceinline__ void cuda_q8_fold_store_warp( + cuda_block_q8_1 *q81, uint64_t element, float value) { + const uint32_t lane = threadIdx.x & 31u; + float amax = fabsf(value); + float sum = value; +#pragma unroll + for (uint32_t offset = 16u; offset > 0u; offset >>= 1u) { + amax = fmaxf(amax, __shfl_xor_sync(0xffffffffu, amax, offset, 32)); + sum += __shfl_xor_sync(0xffffffffu, sum, offset, 32); + } + const float d = amax / 127.0f; + const int8_t q = amax == 0.0f ? (int8_t)0 : (int8_t)roundf(value / d); + cuda_block_q8_1 *block = q81 + element / 32u; + block->qs[lane] = q; + if (lane == 0u) block->ds = __floats2half2_rn(d, sum); +} + __global__ static void hc_split_weighted_sum_norm_fused_kernel( float *out, float *norm_out, @@ -12444,7 +12755,8 @@ __global__ static void hc_split_weighted_sum_norm_fused_kernel( uint32_t n_rows, uint32_t sinkhorn_iters, float epsv, - float norm_eps) { + float norm_eps, + cuda_block_q8_1 *q81) { const uint32_t t = blockIdx.x; const uint32_t d = threadIdx.x; if (t >= n_rows || n_hc != 4) return; @@ -12473,7 +12785,10 @@ __global__ static void hc_split_weighted_sum_norm_fused_kernel( const float norm_scale = rsqrtf(partial[0] / (float)n_embd + norm_eps); for (uint32_t col = d; col < n_embd; col += blockDim.x) { const float v = out[(uint64_t)t * n_embd + col]; - norm_out[(uint64_t)t * n_embd + col] = v * norm_scale * norm_w[col]; + const uint64_t element = (uint64_t)t * n_embd + col; + const float normalized = v * norm_scale * norm_w[col]; + norm_out[element] = normalized; + if (q81) cuda_q8_fold_store_warp(q81, element, normalized); } } @@ -12549,7 +12864,8 @@ __global__ static void hc_split_weighted_sum_norm_fused_store4096_kernel( const float *out, float *norm_out, const float *norm_w, - const float *norm_scale) { + const float *norm_scale, + cuda_block_q8_1 *q81) { constexpr uint32_t tile = 256u; const uint32_t d = threadIdx.x; const uint32_t tile0 = blockIdx.x * tile; @@ -12557,7 +12873,9 @@ __global__ static void hc_split_weighted_sum_norm_fused_store4096_kernel( #pragma unroll for (uint32_t j = 0; j < tile / 256u; j++) { const uint32_t col = tile0 + j * 256u + d; - norm_out[col] = out[col] * scale * norm_w[col]; + const float normalized = out[col] * scale * norm_w[col]; + norm_out[col] = normalized; + if (q81) cuda_q8_fold_store_warp(q81, col, normalized); } } @@ -13891,6 +14209,7 @@ static void *indexer_mxf4_scratch_alloc(uint64_t bytes) { if (g_indexer_mxf4_scratch_bytes >= bytes) { return g_indexer_mxf4_scratch; } + cuda_q8_fold_invalidate_all(); int device = -1; if (!cuda_ok(cudaGetDevice(&device), @@ -27631,11 +27950,12 @@ extern "C" int ds4_gpu_hc_split_weighted_sum_norm_tensor( const float *norm_w = (const float *)cuda_resolve_weight_ptr(model_map, norm_weight_offset, (uint64_t)n_embd * sizeof(float), logical_tier, "hc_norm_weight"); if (!scale || !base || !norm_w) return 0; + const cudaStream_t stream = cuda_decode_stream(); if (n_embd == 4096u && n_rows == 1u && out->ptr != norm_out->ptr && getenv("DS4_CUDA_NO_HC_SPLIT_NORM_SPLIT4096") == NULL) { bool split_ok = true; - hc_split_weighted_sum_norm_fused_partial4096_kernel<<<16u, 256, 0, cuda_decode_stream()>>>( + hc_split_weighted_sum_norm_fused_partial4096_kernel<<<16u, 256, 0, stream>>>( (float *)out->ptr, (float *)split->ptr, (const float *)mix->ptr, @@ -27652,26 +27972,38 @@ extern "C" int ds4_gpu_hc_split_weighted_sum_norm_tensor( split_ok = norm_scale != NULL; } if (split_ok) { - hc_split_weighted_sum_norm_fused_reduce4096_kernel<<<1u, 256, 0, cuda_decode_stream()>>>( + hc_split_weighted_sum_norm_fused_reduce4096_kernel<<<1u, 256, 0, stream>>>( (const float *)out->ptr, norm_scale, norm_eps); split_ok = cudaGetLastError() == cudaSuccess; } + cuda_block_q8_1 *fold_q81 = NULL; if (split_ok) { - hc_split_weighted_sum_norm_fused_store4096_kernel<<<16u, 256, 0, cuda_decode_stream()>>>( + fold_q81 = cuda_q8_fold_prepare( + norm_out->ptr, n_embd, model_map, stream); + hc_split_weighted_sum_norm_fused_store4096_kernel<<<16u, 256, 0, stream>>>( (const float *)out->ptr, (float *)norm_out->ptr, norm_w, - norm_scale); + norm_scale, + fold_q81); split_ok = cudaGetLastError() == cudaSuccess; } - if (split_ok) return 1; + if (split_ok) { + cuda_q8_fold_publish(norm_out->ptr, n_embd, model_map, + stream, fold_q81); + return 1; + } + if (fold_q81) cuda_q8_fold_invalidate_all(); /* The split launches are adjacent, so any pre-enqueue failure * leaves a valid fused-reference retry: it rewrites out, split, * and norm_out without consuming partial state. */ } - hc_split_weighted_sum_norm_fused_kernel<<<(uint32_t)n_rows, 256, 0, cuda_decode_stream()>>>( + cuda_block_q8_1 *fold_q81 = n_rows == 1u + ? cuda_q8_fold_prepare(norm_out->ptr, n_embd, model_map, stream) + : NULL; + hc_split_weighted_sum_norm_fused_kernel<<<(uint32_t)n_rows, 256, 0, stream>>>( (float *)out->ptr, (float *)norm_out->ptr, (float *)split->ptr, @@ -27680,8 +28012,16 @@ extern "C" int ds4_gpu_hc_split_weighted_sum_norm_tensor( scale, base, norm_w, - n_embd, n_hc, (uint32_t)n_rows, sinkhorn_iters, eps, norm_eps); - return cuda_ok(cudaGetLastError(), "hc split weighted sum norm launch"); + n_embd, n_hc, (uint32_t)n_rows, sinkhorn_iters, eps, + norm_eps, fold_q81); + const cudaError_t launch_err = cudaGetLastError(); + if (launch_err == cudaSuccess) { + cuda_q8_fold_publish(norm_out->ptr, n_embd, model_map, + stream, fold_q81); + } else if (fold_q81) { + cuda_q8_fold_invalidate_all(); + } + return cuda_ok(launch_err, "hc split weighted sum norm launch"); } /* Multi-row fallback: norm EVERY row (rms_norm_weight_tensor is the * single-row entry and would leave rows 1..n-1 of norm_out untouched). */ diff --git a/tests/test_mxfp4_cuda.cu b/tests/test_mxfp4_cuda.cu index 9a1c3e8d7..be81a5442 100644 --- a/tests/test_mxfp4_cuda.cu +++ b/tests/test_mxfp4_cuda.cu @@ -11,9 +11,11 @@ #include extern "C" int ds4_cuda_q8_fold_take_q81( - const void *src, uint64_t in_dim, const void **q81) { + const void *src, uint64_t in_dim, cudaStream_t stream, + const void **q81) { (void)src; (void)in_dim; + (void)stream; if (q81) *q81 = nullptr; return 0; } From 5363e0b93f12bcb6906704496f9a5601e9e4e93f Mon Sep 17 00:00:00 2001 From: Giorgio Oppo Date: Tue, 11 Aug 2026 12:49:07 +0200 Subject: [PATCH 021/189] metal: add opt-in M1 IQ2 SSD mid-only producer --- Makefile | 15 +- QA_BEFORE_RELEASES.md | 16 ++ README.md | 21 ++ ds4_gpu.h | 20 ++ ds4_metal.m | 490 ++++++++++++++++++++++++++++++++- metal/moe.metal | 287 +++++++++++++++++++ tests/test_metal_iq2_midonly.c | 76 +++++ 7 files changed, 919 insertions(+), 6 deletions(-) create mode 100644 tests/test_metal_iq2_midonly.c diff --git a/Makefile b/Makefile index 14aa79977..d77d2f79f 100644 --- a/Makefile +++ b/Makefile @@ -68,7 +68,7 @@ DS4_LINK_LIBS ?= $(CUDA_LDLIBS) METAL_LDLIBS := $(LDLIBS) endif -.PHONY: all help clean test test-rocm test-glm53-kda-rocm test-metal-session-batch test-metal-exactn-oracle test-metal-dspark-capture test-mxfp4-cuda test-mxfp4-rocm test-mmq-parity-cuda test-cuda-session-batch test-cuda-mixed-batch dspark-acceptance dspark-verify-depth rocm-dspark-acceptance rocm-dspark-verify-depth mtp-verify-depth cpu cuda cuda-spark cuda-generic cuda-regression strix-halo rocm +.PHONY: all help clean test test-rocm test-glm53-kda-rocm test-metal-session-batch test-metal-exactn-oracle test-metal-dspark-capture test-metal-iq2-midonly test-mxfp4-cuda test-mxfp4-rocm test-mmq-parity-cuda test-cuda-session-batch test-cuda-mixed-batch dspark-acceptance dspark-verify-depth rocm-dspark-acceptance rocm-dspark-verify-depth mtp-verify-depth cpu cuda cuda-spark cuda-generic cuda-regression strix-halo rocm ifeq ($(UNAME_S),Darwin) .PHONY: metal-decode-schedule-bench metal-prefill-variant-bench check-mxfp4-half-lut test-mxfp4-metal @@ -81,6 +81,7 @@ help: @echo " make cpu Build CPU-only ./ds4, ./ds4-server, ./ds4-bench, ./ds4-eval, and ./ds4-agent" @echo " make test Build and run tests" @echo " make test-metal-dspark-capture Check fused DSpark HC capture bitwise" + @echo " make test-metal-iq2-midonly Check M1 IQ2 addr mid-only output and sentinels" @echo " make metal-decode-schedule-bench Build the balanced Metal decode schedule benchmark" @echo " make metal-prefill-variant-bench Build the balanced Metal prefill variant benchmark" @echo " make check-mxfp4-half-lut Verify the checked-in MXFP4 half LUT matches the generator" @@ -125,6 +126,16 @@ tests/test_metal_dspark_capture: tests/test_metal_dspark_capture.o ds4_metal.o test-metal-dspark-capture: tests/test_metal_dspark_capture ./tests/test_metal_dspark_capture + +tests/test_metal_iq2_midonly.o: tests/test_metal_iq2_midonly.c ds4_gpu.h + $(CC) $(CFLAGS) -I. -c -o $@ $< + +tests/test_metal_iq2_midonly: tests/test_metal_iq2_midonly.o ds4_metal.o + $(CC) $(CFLAGS) -o $@ $^ $(METAL_LDLIBS) + +test-metal-iq2-midonly: tests/test_metal_iq2_midonly + ./tests/test_metal_iq2_midonly + speed-bench/metal_decode_schedule_bench.o: speed-bench/metal_decode_schedule_bench.c ds4.h $(CC) $(CFLAGS) -I. -c -o $@ $< @@ -662,4 +673,4 @@ mxfp4-dot-test: tests/test_mxfp4_dot.c ./tests/test_mxfp4_dot clean: - rm -f ds4 ds4-server ds4-bench ds4-eval ds4-agent ds4_cpu ds4_native ds4_server_test ds4_test ds4_agent_test gguf-tools/quality-testing/score_official gguf-tools/quality-testing/score_official.o speed-bench/metal_decode_schedule_bench speed-bench/metal_prefill_variant_bench speed-bench/*.o tests/test_q4k_dot tests/test_mxfp4_dot tests/test_mxfp4_metal tests/test_mxfp4_rocm tests/test_mxfp4_cuda tests/test_metal_session_batch tests/test_metal_exactn_oracle tests/test_metal_dspark_capture tests/test_glm53_kda tests/test_glm53_kda_rocm tests/test_glm53_vision_engine tests/test_glm53_vision_prompt tests/test_gpu_xdev tests/test_gpu_model_cache tests/test_gpu_lookup_cache_strict tests/test_engine_mgpu_refusal tests/test_engine_mgpu_runtime tests/test_engine_correctness tests/test_sampling tests/test_cuda_session_batch tests/test_cuda_mixed_batch tests/*.o *.o tests/cuda_long_context_smoke tests/cuda_long_context_smoke.o + rm -f ds4 ds4-server ds4-bench ds4-eval ds4-agent ds4_cpu ds4_native ds4_server_test ds4_test ds4_agent_test gguf-tools/quality-testing/score_official gguf-tools/quality-testing/score_official.o speed-bench/metal_decode_schedule_bench speed-bench/metal_prefill_variant_bench speed-bench/*.o tests/test_q4k_dot tests/test_mxfp4_dot tests/test_mxfp4_metal tests/test_mxfp4_rocm tests/test_mxfp4_cuda tests/test_metal_session_batch tests/test_metal_exactn_oracle tests/test_metal_dspark_capture tests/test_metal_iq2_midonly tests/test_glm53_kda tests/test_glm53_kda_rocm tests/test_glm53_vision_engine tests/test_glm53_vision_prompt tests/test_gpu_xdev tests/test_gpu_model_cache tests/test_gpu_lookup_cache_strict tests/test_engine_mgpu_refusal tests/test_engine_mgpu_runtime tests/test_engine_correctness tests/test_sampling tests/test_cuda_session_batch tests/test_cuda_mixed_batch tests/*.o *.o tests/cuda_long_context_smoke tests/cuda_long_context_smoke.o diff --git a/QA_BEFORE_RELEASES.md b/QA_BEFORE_RELEASES.md index ea6cd74a8..7a39f794f 100644 --- a/QA_BEFORE_RELEASES.md +++ b/QA_BEFORE_RELEASES.md @@ -359,6 +359,22 @@ than a failure. `--dspark-strict` remains the byte-identical target-only mode. machine. Treat the FlashAttention memo rows as host-dispatch A/B tests: the selected specialization and output must remain identical, and any timing comparison must use repeated warm runs. +- For the M1 IQ2 address-table mid-only experiment, first run + `make test-metal-iq2-midonly`. It must cover 12,288 full-shape top-6 mid + words in both unmasked and complementary masked address-table modes with + both mid mismatch counters at zero, no canonical unwritten rows, zero + candidate gate/up writes, and zero guard mismatches. Then use the same greedy + IQ2_XXS/Q2_K SSD-streaming model, prompt, cache state, and token count for + three decode runs: leave both switches unset for the canonical control, set + `DS4_METAL_REQUIRE_M1_IQ2_MID_ONLY=1` for the fail-closed candidate, and set both enable + and `DS4_METAL_DISABLE_M1_IQ2_MID_ONLY=1` for the kill-switch fallback. + Enable the routed-MoE stage profiler on one candidate layer and require path + `iq2_stream_addr_mid_only_4096x2048` or + `iq2_stream_addr_mask_mid_only_4096x2048`; absence of both is failed model + coverage. Require byte-identical greedy output and top-logprobs, and report + prefill separately from decode: this one-token routed producer is not a + prefill optimization. Compare repeated hot-cache medians, then repeat a + cold-cache sanity run to exclude a change in SSD cache behavior. - For the removed Metal 512-column streaming top-k path, there is no runtime candidate gate. Compare the current binary with a build immediately before its removal only if historical timing is needed. First require diff --git a/README.md b/README.md index 6a682925d..1c966ddaa 100644 --- a/README.md +++ b/README.md @@ -650,6 +650,27 @@ top-k implementation is now used instead and has no runtime re-enable switch. Use a previous binary only as a performance control, and require identical selected ids on tie-heavy inputs before comparing timing. +Apple M1 has an opt-in SSD-streaming decode experiment for the exact +IQ2_XXS/Q2_K routed-MoE shape with 256 experts, top-6 routing, and a +4096-to-2048 gate/up projection. Set +`DS4_METAL_ENABLE_M1_IQ2_MID_ONLY=1` to replace the IQ2 address-table +pair-SwiGLU producer, including complementary resident/missing cache masks. +It preserves the canonical dot-product, +reduction, clamp, activation, and route-weight order but writes `mid` directly +instead of materializing the otherwise unused gate/up rows. Every other +device, shape, streaming mode, unsupported mask/accumulate mode, or unavailable +pipeline keeps the canonical producer; `DS4_METAL_DISABLE_M1_IQ2_MID_ONLY=1` takes precedence as +the kill switch. For fail-closed model coverage, +`DS4_METAL_REQUIRE_M1_IQ2_MID_ONLY=1` implies the enable gate and rejects an +ineligible supported address-table dispatch; the kill switch still takes +precedence. `make test-metal-iq2-midonly` compares all 12,288 top-6 +output words bitwise at full shape for both unmasked and complementary masked +address tables, verifies that the candidates leave gate/up sentinels untouched, +and checks output guards. The routed-MoE stage profiler reports +`iq2_stream_addr_mid_only_4096x2048` or +`iq2_stream_addr_mask_mid_only_4096x2048` when the model path is actually +covered. + These gates change dispatch and intermediate-memory traffic, not model arithmetic. Compare byte-identical output, exact-union counters, stage timings, and generation rate on the same machine; do not infer a speedup from a lower diff --git a/ds4_gpu.h b/ds4_gpu.h index c725e12ca..c33a1f3e7 100644 --- a/ds4_gpu.h +++ b/ds4_gpu.h @@ -189,6 +189,26 @@ int ds4_gpu_test_decode_pipeline_fast_lookup_ext(void); /* Strict test oracle for the generated resident-prefill MXFP4 half LUT. */ int ds4_gpu_test_mxfp4_down_half_lut(uint16_t *legacy_bits, uint16_t *lut_bits); +typedef struct ds4_gpu_iq2_mid_only_oracle_report { + uint64_t mid_words; + uint64_t mid_mismatches; + uint64_t canonical_gate_unwritten; + uint64_t canonical_up_unwritten; + uint64_t candidate_gate_writes; + uint64_t candidate_up_writes; + uint64_t masked_mid_mismatches; + uint64_t masked_inactive_writes; + uint64_t masked_canonical_gate_unwritten; + uint64_t masked_canonical_up_unwritten; + uint64_t masked_gate_writes; + uint64_t masked_up_writes; + uint64_t guard_byte_mismatches; +} ds4_gpu_iq2_mid_only_oracle_report; +/* Full-shape address-table oracle for the experimental M1 IQ2 mid-only + * producer. Return zero means setup/execution failure; numerical and + * sentinel failures are reported explicitly in `report`. */ +int ds4_gpu_test_iq2_addr_mid_only_oracle( + ds4_gpu_iq2_mid_only_oracle_report *report); enum { DS4_GPU_TEST_MXFP4_PAIR_TAIL_CULL = 1u << 0, DS4_GPU_TEST_MXFP4_PAIR_COMPACT_TILE = 1u << 1, diff --git a/ds4_metal.m b/ds4_metal.m index 2e75fcf8a..9a0d098b9 100644 --- a/ds4_metal.m +++ b/ds4_metal.m @@ -147,6 +147,8 @@ static id g_moe_mul_mv_slots6_mxfp4_pair_swiglu_pipeline; static id g_moe_mul_mv_slots6_mxfp4_sum6_pipeline; static id g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_pipeline; +static id g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_mid_only_4096x2048_pipeline; +static id g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_mid_only_4096x2048_masked_pipeline; static id g_moe_mul_mv_addr_iq2_xxs_pipeline; static id g_moe_mul_mv_addr_q2_k_sum6_pipeline; static id g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_masked_pipeline; @@ -2502,6 +2504,12 @@ int ds4_gpu_device_is_pre_m5_apple_silicon(void) { g_metal_device_name[8] == ' '); } +static int ds4_gpu_device_is_m1_apple_silicon(void) { + return strncmp(g_metal_device_name, "Apple M1", 8) == 0 && + (g_metal_device_name[8] == '\0' || + g_metal_device_name[8] == ' '); +} + int ds4_gpu_device_is_m5_apple_silicon(void) { return strncmp(g_metal_device_name, "Apple M5", 8) == 0 && (g_metal_device_name[8] == '\0' || @@ -7533,6 +7541,38 @@ int ds4_gpu_init(void) { return 0; } + error = nil; + fn = [library + newFunctionWithName:@"kernel_mul_mv_addr_iq2_xxs_pair_swiglu_mid_only_4096x2048_f32" + constantValues:moe_mv_id_constants + error:&error]; + if (fn) { + g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_mid_only_4096x2048_pipeline = + [g_device newComputePipelineStateWithFunction:fn error:&error]; + } + if (!g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_mid_only_4096x2048_pipeline) { + fprintf(stderr, + "ds4: optional Metal M1 IQ2 addr mid-only pipeline unavailable: %s\n", + error ? [[error localizedDescription] UTF8String] : + "function not found"); + } + + error = nil; + fn = [library + newFunctionWithName:@"kernel_mul_mv_addr_iq2_xxs_pair_swiglu_mid_only_4096x2048_masked_f32" + constantValues:moe_mv_id_constants + error:&error]; + if (fn) { + g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_mid_only_4096x2048_masked_pipeline = + [g_device newComputePipelineStateWithFunction:fn error:&error]; + } + if (!g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_mid_only_4096x2048_masked_pipeline) { + fprintf(stderr, + "ds4: optional Metal M1 IQ2 addr masked mid-only pipeline unavailable: %s\n", + error ? [[error localizedDescription] UTF8String] : + "function not found"); + } + error = nil; fn = [library newFunctionWithName:@"kernel_mul_mv_addr_iq2_xxs_f32" constantValues:moe_mv_id_constants @@ -8902,6 +8942,366 @@ int ds4_gpu_test_mxfp4_down_half_lut(uint16_t *legacy_bits, return 1; } +static uint64_t ds4_gpu_test_count_non_sentinel_bytes( + const uint8_t *p, size_t n, uint8_t sentinel) { + uint64_t count = 0; + for (size_t i = 0; i < n; i++) { + count += p[i] != sentinel; + } + return count; +} + +int ds4_gpu_test_iq2_addr_mid_only_oracle( + ds4_gpu_iq2_mid_only_oracle_report *report) { + enum { + N_TOTAL_EXPERT = 256, + N_SELECTED = 6, + IN_DIM = 4096, + MID_DIM = 2048, + IQ2_BLOCK = 256, + IQ2_BLOCK_BYTES = 66, + GUARD_BYTES = 64, + }; + typedef struct { + uint16_t d; + uint16_t qs[IQ2_BLOCK / 8]; + } ds4_test_block_iq2_xxs; + + if (!report || sizeof(ds4_test_block_iq2_xxs) != IQ2_BLOCK_BYTES) return 0; + memset(report, 0, sizeof(*report)); + if (!g_initialized && !ds4_gpu_init()) return 0; + if (!g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_pipeline || + !g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_mid_only_4096x2048_pipeline || + !g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_masked_pipeline || + !g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_mid_only_4096x2048_masked_pipeline) { + fprintf(stderr, "ds4: Metal IQ2 addr mid-only oracle pipelines unavailable\n"); + return 0; + } + + @autoreleasepool { + const NSUInteger row_bytes = + (IN_DIM / IQ2_BLOCK) * sizeof(ds4_test_block_iq2_xxs); + const NSUInteger matrix_bytes = (NSUInteger)MID_DIM * row_bytes; + const NSUInteger x_bytes = (NSUInteger)IN_DIM * sizeof(float); + const NSUInteger payload_bytes = + (NSUInteger)N_SELECTED * MID_DIM * sizeof(float); + const NSUInteger guarded_bytes = payload_bytes + 2u * GUARD_BYTES; + const NSUInteger table_bytes = + (NSUInteger)N_TOTAL_EXPERT * sizeof(uint64_t); + const NSUInteger ids_bytes = N_SELECTED * sizeof(int32_t); + const NSUInteger weights_bytes = N_SELECTED * sizeof(float); + const uint8_t sentinel = 0xa5u; + + id gate_weights = [g_device newBufferWithLength:matrix_bytes + options:MTLResourceStorageModeShared]; + id up_weights = [g_device newBufferWithLength:matrix_bytes + options:MTLResourceStorageModeShared]; + id x = [g_device newBufferWithLength:x_bytes + options:MTLResourceStorageModeShared]; + id gate_addrs = [g_device newBufferWithLength:table_bytes + options:MTLResourceStorageModeShared]; + id up_addrs = [g_device newBufferWithLength:table_bytes + options:MTLResourceStorageModeShared]; + id ids = [g_device newBufferWithLength:ids_bytes + options:MTLResourceStorageModeShared]; + id weights = [g_device newBufferWithLength:weights_bytes + options:MTLResourceStorageModeShared]; + id canonical_gate = [g_device newBufferWithLength:guarded_bytes + options:MTLResourceStorageModeShared]; + id canonical_up = [g_device newBufferWithLength:guarded_bytes + options:MTLResourceStorageModeShared]; + id canonical_mid = [g_device newBufferWithLength:guarded_bytes + options:MTLResourceStorageModeShared]; + id candidate_gate = [g_device newBufferWithLength:guarded_bytes + options:MTLResourceStorageModeShared]; + id candidate_up = [g_device newBufferWithLength:guarded_bytes + options:MTLResourceStorageModeShared]; + id candidate_mid = [g_device newBufferWithLength:guarded_bytes + options:MTLResourceStorageModeShared]; + id masked_canonical_gate = [g_device newBufferWithLength:guarded_bytes + options:MTLResourceStorageModeShared]; + id masked_canonical_up = [g_device newBufferWithLength:guarded_bytes + options:MTLResourceStorageModeShared]; + id masked_canonical_mid = [g_device newBufferWithLength:guarded_bytes + options:MTLResourceStorageModeShared]; + id masked_candidate_gate = [g_device newBufferWithLength:guarded_bytes + options:MTLResourceStorageModeShared]; + id masked_candidate_up = [g_device newBufferWithLength:guarded_bytes + options:MTLResourceStorageModeShared]; + id masked_candidate_mid = [g_device newBufferWithLength:guarded_bytes + options:MTLResourceStorageModeShared]; + if (!gate_weights || !up_weights || !x || !gate_addrs || !up_addrs || + !ids || !weights || !canonical_gate || !canonical_up || + !canonical_mid || !candidate_gate || !candidate_up || !candidate_mid || + !masked_canonical_gate || !masked_canonical_up || !masked_canonical_mid || + !masked_candidate_gate || !masked_candidate_up || !masked_candidate_mid) { + fprintf(stderr, "ds4: Metal IQ2 addr mid-only oracle allocation failed\n"); + return 0; + } + + ds4_test_block_iq2_xxs *gate_blocks = + (ds4_test_block_iq2_xxs *)[gate_weights contents]; + ds4_test_block_iq2_xxs *up_blocks = + (ds4_test_block_iq2_xxs *)[up_weights contents]; + const NSUInteger n_blocks = matrix_bytes / sizeof(*gate_blocks); + uint32_t gate_state = 0x91e10da5u; + uint32_t up_state = 0x6d2b79f5u; + for (NSUInteger i = 0; i < n_blocks; i++) { + /* Positive, finite half scales in [0.03125, 0.234375]. */ + gate_blocks[i].d = (uint16_t)(0x2800u + ((i % 7u) << 7u)); + up_blocks[i].d = (uint16_t)(0x2a00u + ((i % 5u) << 7u)); + for (NSUInteger q = 0; q < IQ2_BLOCK / 8; q++) { + gate_state ^= gate_state << 13; + gate_state ^= gate_state >> 17; + gate_state ^= gate_state << 5; + up_state ^= up_state << 13; + up_state ^= up_state >> 17; + up_state ^= up_state << 5; + gate_blocks[i].qs[q] = (uint16_t)gate_state; + up_blocks[i].qs[q] = (uint16_t)up_state; + } + } + float *x_f32 = (float *)[x contents]; + for (NSUInteger i = 0; i < IN_DIM; i++) { + x_f32[i] = ((int)(i % 37u) - 18) * (1.0f / 256.0f); + } + + static const int32_t selected_ids[N_SELECTED] = { 3, 17, 41, 89, 137, 251 }; + static const float route_weights[N_SELECTED] = { + 1.0f, 0.875f, 0.75f, 0.625f, 0.5f, 0.375f, + }; + memcpy([ids contents], selected_ids, sizeof(selected_ids)); + memcpy([weights contents], route_weights, sizeof(route_weights)); + memset([gate_addrs contents], 0, table_bytes); + memset([up_addrs contents], 0, table_bytes); + uint64_t *gate_table = (uint64_t *)[gate_addrs contents]; + uint64_t *up_table = (uint64_t *)[up_addrs contents]; + const uint64_t gate_gpu_addr = (uint64_t)[gate_weights gpuAddress]; + const uint64_t up_gpu_addr = (uint64_t)[up_weights gpuAddress]; + if (gate_gpu_addr == 0 || up_gpu_addr == 0) { + fprintf(stderr, "ds4: Metal IQ2 addr mid-only oracle has no GPU addresses\n"); + return 0; + } + for (NSUInteger i = 0; i < N_SELECTED; i++) { + gate_table[(uint32_t)selected_ids[i]] = gate_gpu_addr; + up_table[(uint32_t)selected_ids[i]] = up_gpu_addr; + } + + memset([canonical_gate contents], sentinel, guarded_bytes); + memset([canonical_up contents], sentinel, guarded_bytes); + memset([canonical_mid contents], sentinel, guarded_bytes); + memset([candidate_gate contents], sentinel, guarded_bytes); + memset([candidate_up contents], sentinel, guarded_bytes); + memset([candidate_mid contents], sentinel, guarded_bytes); + memset([masked_canonical_gate contents], sentinel, guarded_bytes); + memset([masked_canonical_up contents], sentinel, guarded_bytes); + memset([masked_canonical_mid contents], sentinel, guarded_bytes); + memset([masked_candidate_gate contents], sentinel, guarded_bytes); + memset([masked_candidate_up contents], sentinel, guarded_bytes); + memset([masked_candidate_mid contents], sentinel, guarded_bytes); + + ds4_gpu_mul_mv_id_args args = { + .nei0 = N_SELECTED, + .nei1 = 1, + .nbi1 = N_SELECTED * sizeof(int32_t), + .ne00 = IN_DIM, + .ne01 = MID_DIM, + .ne02 = N_TOTAL_EXPERT, + .nb00 = sizeof(ds4_test_block_iq2_xxs), + .nb01 = row_bytes, + .nb02 = matrix_bytes, + .ne10 = IN_DIM, + .ne11 = 1, + .ne12 = 1, + .ne13 = 1, + .nb10 = sizeof(float), + .nb11 = x_bytes, + .nb12 = x_bytes, + .ne0 = MID_DIM, + .ne1 = N_SELECTED, + .nb1 = (uint64_t)MID_DIM * sizeof(float), + .nr0 = 4, + .tp_world = 1, + }; + ds4_gpu_dsv4_moe_swiglu_weight_args act = { + .width = MID_DIM, + .rows = N_SELECTED, + .gate_row_stride = (uint64_t)MID_DIM * sizeof(float), + .up_row_stride = (uint64_t)MID_DIM * sizeof(float), + .mid_row_stride = (uint64_t)MID_DIM * sizeof(float), + .weight_stride = sizeof(float), + .write_clamped = 0, + .clamp_value = 6.0f, + }; + + id cb = ds4_gpu_new_command_buffer(); + id enc = cb ? [cb computeCommandEncoder] : nil; + if (!cb || !enc) return 0; + [enc useResource:gate_weights usage:MTLResourceUsageRead]; + [enc useResource:up_weights usage:MTLResourceUsageRead]; + [enc setThreadgroupMemoryLength:256u * sizeof(uint64_t) + 128u * sizeof(uint8_t) + atIndex:0]; + +#define DS4_ENCODE_IQ2_ADDR_ORACLE(PIPELINE, GATE, UP, MID) do { \ + [enc setComputePipelineState:(PIPELINE)]; \ + [enc setBytes:&args length:sizeof(args) atIndex:0]; \ + [enc setBytes:&act length:sizeof(act) atIndex:1]; \ + [enc setBuffer:gate_addrs offset:0 atIndex:2]; \ + [enc setBuffer:up_addrs offset:0 atIndex:3]; \ + [enc setBuffer:x offset:0 atIndex:4]; \ + [enc setBuffer:(GATE) offset:GUARD_BYTES atIndex:5]; \ + [enc setBuffer:(UP) offset:GUARD_BYTES atIndex:6]; \ + [enc setBuffer:(MID) offset:GUARD_BYTES atIndex:7]; \ + [enc setBuffer:ids offset:0 atIndex:8]; \ + [enc setBuffer:weights offset:0 atIndex:9]; \ + [enc dispatchThreadgroups:MTLSizeMake(MID_DIM / 8u, 1, N_SELECTED) \ + threadsPerThreadgroup:MTLSizeMake(32u, 2u, 1u)]; \ + } while (0) + DS4_ENCODE_IQ2_ADDR_ORACLE( + g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_pipeline, + canonical_gate, canonical_up, canonical_mid); + DS4_ENCODE_IQ2_ADDR_ORACLE( + g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_mid_only_4096x2048_pipeline, + candidate_gate, candidate_up, candidate_mid); +#undef DS4_ENCODE_IQ2_ADDR_ORACLE + const ds4_gpu_stream_expert_split_args split_even = { + .active_mask = 0x15u, + .accumulate = 0u, + }; + const ds4_gpu_stream_expert_split_args split_odd = { + .active_mask = 0x2au, + .accumulate = 0u, + }; +#define DS4_ENCODE_IQ2_ADDR_MASKED_ORACLE(PIPELINE, SPLIT, GATE, UP, MID) do { \ + [enc setComputePipelineState:(PIPELINE)]; \ + [enc setBytes:&args length:sizeof(args) atIndex:0]; \ + [enc setBytes:&act length:sizeof(act) atIndex:1]; \ + [enc setBytes:(SPLIT) length:sizeof(*(SPLIT)) atIndex:2]; \ + [enc setBuffer:gate_addrs offset:0 atIndex:3]; \ + [enc setBuffer:up_addrs offset:0 atIndex:4]; \ + [enc setBuffer:x offset:0 atIndex:5]; \ + [enc setBuffer:(GATE) offset:GUARD_BYTES atIndex:6]; \ + [enc setBuffer:(UP) offset:GUARD_BYTES atIndex:7]; \ + [enc setBuffer:(MID) offset:GUARD_BYTES atIndex:8]; \ + [enc setBuffer:ids offset:0 atIndex:9]; \ + [enc setBuffer:weights offset:0 atIndex:10]; \ + [enc dispatchThreadgroups:MTLSizeMake(MID_DIM / 8u, 1, N_SELECTED) \ + threadsPerThreadgroup:MTLSizeMake(32u, 2u, 1u)]; \ + } while (0) + DS4_ENCODE_IQ2_ADDR_MASKED_ORACLE( + g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_masked_pipeline, + &split_even, masked_canonical_gate, masked_canonical_up, masked_canonical_mid); + DS4_ENCODE_IQ2_ADDR_MASKED_ORACLE( + g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_mid_only_4096x2048_masked_pipeline, + &split_even, masked_candidate_gate, masked_candidate_up, masked_candidate_mid); + [enc endEncoding]; + [cb commit]; + if (!ds4_gpu_wait_command_buffer(cb, "IQ2 addr mid-only masked-even oracle")) { + return 0; + } + + /* Validate active_mask before its complementary leg fills the other + * rows. A final-only comparison after 0x15 + 0x2a cannot detect a + * kernel that simply ignores the mask. */ + const NSUInteger masked_row_bytes = (NSUInteger)MID_DIM * sizeof(float); + const uint8_t *masked_even_outputs[] = { + (const uint8_t *)[masked_canonical_gate contents] + GUARD_BYTES, + (const uint8_t *)[masked_canonical_up contents] + GUARD_BYTES, + (const uint8_t *)[masked_canonical_mid contents] + GUARD_BYTES, + (const uint8_t *)[masked_candidate_gate contents] + GUARD_BYTES, + (const uint8_t *)[masked_candidate_up contents] + GUARD_BYTES, + (const uint8_t *)[masked_candidate_mid contents] + GUARD_BYTES, + }; + for (NSUInteger output = 0; + output < sizeof(masked_even_outputs) / sizeof(masked_even_outputs[0]); + output++) { + for (NSUInteger row = 1u; row < N_SELECTED; row += 2u) { + report->masked_inactive_writes += + ds4_gpu_test_count_non_sentinel_bytes( + masked_even_outputs[output] + row * masked_row_bytes, + masked_row_bytes, sentinel); + } + } + + cb = ds4_gpu_new_command_buffer(); + enc = cb ? [cb computeCommandEncoder] : nil; + if (!cb || !enc) return 0; + [enc useResource:gate_weights usage:MTLResourceUsageRead]; + [enc useResource:up_weights usage:MTLResourceUsageRead]; + [enc setThreadgroupMemoryLength:256u * sizeof(uint64_t) + 128u * sizeof(uint8_t) + atIndex:0]; + DS4_ENCODE_IQ2_ADDR_MASKED_ORACLE( + g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_masked_pipeline, + &split_odd, masked_canonical_gate, masked_canonical_up, masked_canonical_mid); + DS4_ENCODE_IQ2_ADDR_MASKED_ORACLE( + g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_mid_only_4096x2048_masked_pipeline, + &split_odd, masked_candidate_gate, masked_candidate_up, masked_candidate_mid); +#undef DS4_ENCODE_IQ2_ADDR_MASKED_ORACLE + [enc endEncoding]; + [cb commit]; + if (!ds4_gpu_wait_command_buffer(cb, "IQ2 addr mid-only tensor oracle")) { + return 0; + } + + const uint8_t *cg = (const uint8_t *)[canonical_gate contents]; + const uint8_t *cu = (const uint8_t *)[canonical_up contents]; + const uint8_t *cm = (const uint8_t *)[canonical_mid contents]; + const uint8_t *ng = (const uint8_t *)[candidate_gate contents]; + const uint8_t *nu = (const uint8_t *)[candidate_up contents]; + const uint8_t *nm = (const uint8_t *)[candidate_mid contents]; + const uint8_t *mcg = (const uint8_t *)[masked_canonical_gate contents]; + const uint8_t *mcu = (const uint8_t *)[masked_canonical_up contents]; + const uint8_t *mcm = (const uint8_t *)[masked_canonical_mid contents]; + const uint8_t *mng = (const uint8_t *)[masked_candidate_gate contents]; + const uint8_t *mnu = (const uint8_t *)[masked_candidate_up contents]; + const uint8_t *mnm = (const uint8_t *)[masked_candidate_mid contents]; + const uint32_t sentinel_word = 0xa5a5a5a5u; + const uint32_t *cg_words = (const uint32_t *)(cg + GUARD_BYTES); + const uint32_t *cu_words = (const uint32_t *)(cu + GUARD_BYTES); + const uint32_t *cm_words = (const uint32_t *)(cm + GUARD_BYTES); + const uint32_t *nm_words = (const uint32_t *)(nm + GUARD_BYTES); + const uint32_t *mcg_words = (const uint32_t *)(mcg + GUARD_BYTES); + const uint32_t *mcu_words = (const uint32_t *)(mcu + GUARD_BYTES); + const uint32_t *mcm_words = (const uint32_t *)(mcm + GUARD_BYTES); + const uint32_t *mnm_words = (const uint32_t *)(mnm + GUARD_BYTES); + report->mid_words = payload_bytes / sizeof(uint32_t); + for (uint64_t i = 0; i < report->mid_words; i++) { + report->mid_mismatches += cm_words[i] != nm_words[i]; + report->canonical_gate_unwritten += cg_words[i] == sentinel_word; + report->canonical_up_unwritten += cu_words[i] == sentinel_word; + report->masked_mid_mismatches += mcm_words[i] != mnm_words[i]; + report->masked_canonical_gate_unwritten += mcg_words[i] == sentinel_word; + report->masked_canonical_up_unwritten += mcu_words[i] == sentinel_word; + } + report->candidate_gate_writes = + ds4_gpu_test_count_non_sentinel_bytes(ng, guarded_bytes, sentinel); + report->candidate_up_writes = + ds4_gpu_test_count_non_sentinel_bytes(nu, guarded_bytes, sentinel); + report->masked_gate_writes = + ds4_gpu_test_count_non_sentinel_bytes(mng, guarded_bytes, sentinel); + report->masked_up_writes = + ds4_gpu_test_count_non_sentinel_bytes(mnu, guarded_bytes, sentinel); + +#define DS4_COUNT_GUARDS(P) do { \ + report->guard_byte_mismatches += \ + ds4_gpu_test_count_non_sentinel_bytes((P), GUARD_BYTES, sentinel); \ + report->guard_byte_mismatches += \ + ds4_gpu_test_count_non_sentinel_bytes( \ + (P) + GUARD_BYTES + payload_bytes, GUARD_BYTES, sentinel); \ + } while (0) + DS4_COUNT_GUARDS(cg); + DS4_COUNT_GUARDS(cu); + DS4_COUNT_GUARDS(cm); + DS4_COUNT_GUARDS(nm); + DS4_COUNT_GUARDS(mcg); + DS4_COUNT_GUARDS(mcu); + DS4_COUNT_GUARDS(mcm); + DS4_COUNT_GUARDS(mnm); +#undef DS4_COUNT_GUARDS + } + return 1; +} + void ds4_gpu_test_set_flags(uint32_t flags) { g_test_flags = flags; } @@ -10482,6 +10882,8 @@ void ds4_gpu_cleanup(void) { g_moe_mul_mv_slots6_mxfp4_pair_swiglu_pipeline = nil; g_moe_mul_mv_slots6_mxfp4_sum6_pipeline = nil; g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_pipeline = nil; + g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_mid_only_4096x2048_pipeline = nil; + g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_mid_only_4096x2048_masked_pipeline = nil; g_moe_mul_mv_addr_iq2_xxs_pipeline = nil; g_moe_mul_mv_addr_q2_k_sum6_pipeline = nil; g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_masked_pipeline = nil; @@ -13563,6 +13965,13 @@ static void ds4_gpu_stream_expert_cache_note_decode_token(void) { ds4_gpu_stream_expert_cache_maybe_decay_route_hotness(); } +static int ds4_gpu_m1_iq2_mid_only_requested(void) { + return ds4_gpu_device_is_m1_apple_silicon() && + ds4_gpu_env_bool("DS4_METAL_DISABLE_M1_IQ2_MID_ONLY") != 1 && + (ds4_gpu_env_bool("DS4_METAL_ENABLE_M1_IQ2_MID_ONLY") == 1 || + ds4_gpu_env_bool("DS4_METAL_REQUIRE_M1_IQ2_MID_ONLY") == 1); +} + static int ds4_gpu_stream_compact_addr_requested(void) { return g_ssd_streaming_mode && getenv("DS4_METAL_ENABLE_STREAMING_COMPACT_ADDR") != NULL && @@ -13573,6 +13982,7 @@ static int ds4_gpu_stream_compact_addr_requested(void) { static int ds4_gpu_stream_expert_addr_table_requested(void) { return g_ssd_streaming_mode && (getenv("DS4_METAL_ENABLE_STREAMING_EXPERT_ADDR_TABLE") != NULL || + ds4_gpu_m1_iq2_mid_only_requested() || getenv("DS4_METAL_ENABLE_STREAMING_EXPERT_HIT_VALIDATOR") != NULL || getenv("DS4_METAL_ENABLE_STREAMING_EXPERT_MASKED_ADDR") != NULL || g_stream_prefill_batch_selected_addr_building || @@ -13586,6 +13996,7 @@ static int ds4_gpu_stream_expert_addr_table_requested(void) { static int ds4_gpu_stream_expert_addr_table_kernel_requested(void) { return g_ssd_streaming_mode && (getenv("DS4_METAL_ENABLE_STREAMING_EXPERT_ADDR_TABLE") != NULL || + ds4_gpu_m1_iq2_mid_only_requested() || getenv("DS4_METAL_ENABLE_STREAMING_EXPERT_HIT_VALIDATOR") != NULL || getenv("DS4_METAL_ENABLE_STREAMING_EXPERT_MASKED_ADDR") != NULL || ds4_gpu_stream_expert_split_ready()) && @@ -40112,6 +40523,12 @@ static bool ds4_gpu_mxfp4_moe_decode_nsg1_enabled(uint32_t n_tokens) { getenv("DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_DECODE_NSG1") == NULL; } +static bool ds4_gpu_m1_iq2_mid_only_split_supported( + const ds4_gpu_stream_expert_split_args *split) { + return split && split->accumulate == 0u && split->active_mask != 0u && + (split->active_mask & ~0x3fu) == 0u; +} + int ds4_gpu_routed_moe_one_tensor( ds4_gpu_tensor *out, ds4_gpu_tensor *gate, @@ -40449,6 +40866,29 @@ int ds4_gpu_routed_moe_one_tensor( (n_expert == 6 || (n_expert == 8 && g_tp_split_world == 2)) && n_tokens == 1 && down_sum6_pipeline != nil; + /* Experimental M1 SSD-streaming producer. The eventual dispatch is + * additionally required to be an address-table path supported by the + * unmasked or masked mid-only pipeline, where the following Q2 sum + * consumes only `mid`. Presence of the disable switch always wins, + * and every failed predicate retains the canonical address-table + * kernel. */ + const bool m1_iq2_mid_only_disabled = + ds4_gpu_env_bool("DS4_METAL_DISABLE_M1_IQ2_MID_ONLY") == 1; + const bool m1_iq2_mid_only_required = + ds4_gpu_env_bool("DS4_METAL_REQUIRE_M1_IQ2_MID_ONLY") == 1; + const bool m1_iq2_addr_mid_only_candidate = + ds4_gpu_m1_iq2_mid_only_requested() && + g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_mid_only_4096x2048_pipeline != nil && + !force_resident && g_ssd_streaming_mode && + gate_type == DS4_METAL_TENSOR_IQ2_XXS && + down_type == DS4_METAL_TENSOR_Q2_K && + n_tokens == 1 && n_expert == 6 && n_total_expert == 256 && + expert_in_dim == 4096 && expert_mid_dim == 2048 && + gate_row_bytes == 1056 && gate_expert_bytes == 2162688 && + gate_args.ne00 == 4096 && gate_args.ne01 == 2048 && + gate_args.ne0 == 2048 && gate_args.nr0 == 4 && + g_tp_split_rank == 0 && g_tp_split_world == 1 && add_in == NULL && + fuse_pair_swiglu && direct_down_sum; if (g_parallel_q8_pending) { /* A concurrent encoder invalidates every implicit dependency in @@ -41766,6 +42206,33 @@ int ds4_gpu_routed_moe_one_tensor( [cb useResidencySet:q4_table_layer_residency]; } + const bool use_m1_iq2_addr_mid_only = + m1_iq2_addr_mid_only_candidate && + use_iq2_selected_slots && use_stream_expert_addr_table && + (!use_stream_expert_masked_addr_table || + g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_mid_only_4096x2048_masked_pipeline != nil); + if (m1_iq2_mid_only_required && !m1_iq2_mid_only_disabled && + !use_m1_iq2_addr_mid_only) { + fprintf(stderr, + "ds4: required Metal M1 IQ2 address-table mid-only producer was not selected " + "candidate=%d selected_slots=%d addr=%d masked=%d split=%d " + "gate=%u down=%u experts=%u/%u shape=%ux%u ssd=%d resident=%d\n", + m1_iq2_addr_mid_only_candidate ? 1 : 0, + use_iq2_selected_slots ? 1 : 0, + use_stream_expert_addr_table ? 1 : 0, + use_stream_expert_masked_addr_table ? 1 : 0, + use_stream_expert_split_deferred ? 1 : 0, + gate_type, + down_type, + n_expert, + n_total_expert, + expert_in_dim, + expert_mid_dim, + g_ssd_streaming_mode ? 1 : 0, + force_resident ? 1 : 0); + return 0; + } + const bool moe_one_stage_profile = g_batch_cb != nil && ds4_gpu_stage_profile_enabled_for_layer("DS4_METAL_MOE_ONE_STAGE_PROFILE", @@ -41781,6 +42248,10 @@ int ds4_gpu_routed_moe_one_tensor( use_q4_expert_address_table ? "q4_addr_pair_swiglu" : use_q4_expert_table ? "q4_table_pair_swiglu" : use_q4_gather_slots ? "q4_gather_slots6_pair_swiglu" : + use_m1_iq2_addr_mid_only ? + (use_stream_expert_masked_addr_table ? + "iq2_stream_addr_mask_mid_only_4096x2048" : + "iq2_stream_addr_mid_only_4096x2048") : use_stream_expert_split_deferred ? "iq2_stream_split_pair_swiglu" : use_stream_expert_masked_addr_table ? "iq2_stream_addr_mask_pair_swiglu" : use_stream_expert_addr_table ? "iq2_stream_addr_pair_swiglu" : @@ -42150,7 +42621,10 @@ int ds4_gpu_routed_moe_one_tensor( .accumulate = 0u, }; ok = ds4_gpu_encode_mul_mv_addr_iq2_pair_swiglu_masked(cb, - g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_masked_pipeline, + use_m1_iq2_addr_mid_only && + ds4_gpu_m1_iq2_mid_only_split_supported(&resident_pair_args) ? + g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_mid_only_4096x2048_masked_pipeline : + g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_masked_pipeline, &gate_args, &act_args, &resident_pair_args, @@ -42294,7 +42768,10 @@ int ds4_gpu_routed_moe_one_tensor( }; if (ok) { ok = ds4_gpu_encode_mul_mv_addr_iq2_pair_swiglu_masked(cb, - g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_masked_pipeline, + use_m1_iq2_addr_mid_only && + ds4_gpu_m1_iq2_mid_only_split_supported(&missing_pair_args) ? + g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_mid_only_4096x2048_masked_pipeline : + g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_masked_pipeline, &gate_args, &act_args, &missing_pair_args, @@ -42364,7 +42841,10 @@ int ds4_gpu_routed_moe_one_tensor( .accumulate = 0u, }; ok = ds4_gpu_encode_mul_mv_addr_iq2_pair_swiglu_masked(cb, - g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_masked_pipeline, + use_m1_iq2_addr_mid_only && + ds4_gpu_m1_iq2_mid_only_split_supported(&split_args) ? + g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_mid_only_4096x2048_masked_pipeline : + g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_masked_pipeline, &gate_args, &act_args, &split_args, @@ -42389,7 +42869,9 @@ int ds4_gpu_routed_moe_one_tensor( } } else { ok = ds4_gpu_encode_mul_mv_addr_iq2_pair_swiglu(cb, - g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_pipeline, + use_m1_iq2_addr_mid_only ? + g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_mid_only_4096x2048_pipeline : + g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_pipeline, &gate_args, &act_args, stream_addr_resources, diff --git a/metal/moe.metal b/metal/moe.metal index 904d33281..eac36e3ec 100644 --- a/metal/moe.metal +++ b/metal/moe.metal @@ -3247,6 +3247,139 @@ void kernel_mul_mv_iq2_xxs_pair_f32_impl( } } +// Address-table decode specialization for the DeepSeek Flash IQ2 gate/up +// shape. It deliberately keeps the exact dot-product, simd_sum, scale, +// clamp, exp, and route-weight order of kernel_mul_mv_iq2_xxs_pair_f32_impl +// followed by the canonical fused SwiGLU epilogue. The only removed work is +// the round trip through the gate/up destination buffers: downstream consumes +// `mid` exclusively on this dispatch. +template +void kernel_mul_mv_iq2_xxs_pair_swiglu_mid_only_4096x2048_impl( + ds4_metal_args_mul_mv args, + constant ds4_metal_dsv4_moe_swiglu_weight_args & act, + device const char * src0_gate, + device const char * src0_up, + device const char * src1, + device char * dst_mid, + device const char * weights, + uint64_t pair_row, + threadgroup char * shmem, + uint3 tgpig, + ushort tiisg, + ushort sgitg) { + constexpr int ne00 = 4096; + constexpr int ne0 = 2048; + constexpr int nb = ne00 / QK_K; + const short NSG = FC_mul_mv_nsg; + + const int r0 = tgpig.x; + const int r1 = tgpig.y; + const int im = tgpig.z; + const int first_row = (r0 * NSG + sgitg) * nr0; + + const uint i12 = im % args.ne12; + const uint i13 = im / args.ne12; + const uint64_t offset0 = first_row * args.nb01 + + (i12 / args.r2) * args.nb02 + (i13 / args.r3) * args.nb03; + const uint64_t offset1 = r1 * args.nb11 + i12 * args.nb12 + i13 * args.nb13; + + device const block_iq2_xxs *xg = + (device const block_iq2_xxs *)(src0_gate + offset0); + device const block_iq2_xxs *xu = + (device const block_iq2_xxs *)(src0_up + offset0); + device const float *y = (device const float *)(src1 + offset1); + + float yl[32]; + float sumg[nr0] = {0.f}; + float sumu[nr0] = {0.f}; + constexpr int nb32 = nb * (QK_K / 32); + + threadgroup uint64_t *svalues = (threadgroup uint64_t *)(shmem); + threadgroup uint8_t *ssigns = (threadgroup uint8_t *)(svalues + 256); + { + int nval = 4; + int pos = (32 * sgitg + tiisg) * nval; + for (int i = 0; i < nval; ++i) svalues[pos + i] = ds4_metal_iq2xxs_grid[pos + i]; + nval = 2; + pos = (32 * sgitg + tiisg) * nval; + for (int i = 0; i < nval; ++i) ssigns[pos + i] = ds4_metal_ksigns_iq2xs[pos + i]; + threadgroup_barrier(mem_flags::mem_threadgroup); + } + + const int ix = tiisg; + device const float *y4 = y + 32 * ix; + for (int ib32 = ix; ib32 < nb32; ib32 += 32) { + for (short i = 0; i < 32; ++i) { + yl[i] = y4[i]; + } + + const int ibl = ib32 / (QK_K / 32); + const int ib = ib32 % (QK_K / 32); + device const block_iq2_xxs *xgr = xg + ibl; + device const block_iq2_xxs *xur = xu + ibl; + device const uint16_t *qg = xgr->qs + 4 * ib; + device const uint16_t *qu = xur->qs + 4 * ib; + device const half *dhg = &xgr->d; + device const half *dhu = &xur->d; + + for (short row = 0; row < nr0; ++row) { + device const uint8_t *aux8g = (device const uint8_t *)qg; + device const uint8_t *aux8u = (device const uint8_t *)qu; + const uint32_t aux32g = qg[2] | (qg[3] << 16); + const uint32_t aux32u = qu[2] | (qu[3] << 16); + const float dg = (float)dhg[0] * (0.5f + (aux32g >> 28)); + const float du = (float)dhu[0] * (0.5f + (aux32u >> 28)); + + float sg = 0; + float su = 0; + for (short l = 0; l < 4; ++l) { + const threadgroup uint8_t *gridg = + (const threadgroup uint8_t *)(svalues + aux8g[l]); + const threadgroup uint8_t *gridu = + (const threadgroup uint8_t *)(svalues + aux8u[l]); + const uint8_t signg = ssigns[(aux32g >> 7 * l) & 127]; + const uint8_t signu = ssigns[(aux32u >> 7 * l) & 127]; + for (short j = 0; j < 8; ++j) { + const float v = yl[8 * l + j]; + sg += v * gridg[j] * + (signg & ds4_metal_kmask_iq2xs[j] ? -1.f : 1.f); + su += v * gridu[j] * + (signu & ds4_metal_kmask_iq2xs[j] ? -1.f : 1.f); + } + } + sumg[row] += dg * sg; + sumu[row] += du * su; + + dhg += args.nb01 / 2; + dhu += args.nb01 / 2; + qg += args.nb01 / 2; + qu += args.nb01 / 2; + } + y4 += 32 * 32; + } + + device float *mid_f32 = + (device float *)(dst_mid + pair_row * act.mid_row_stride); + device const float *route_w = + (device const float *)(weights + pair_row * act.weight_stride); + const float c = act.clamp_value; + const float route_weight = route_w[0]; + for (int row = 0; row < nr0 && first_row + row < ne0; ++row) { + const float sum_gate = simd_sum(sumg[row]); + const float sum_up = simd_sum(sumu[row]); + if (tiisg == 0) { + float g = sum_gate * 0.25f; + float u = sum_up * 0.25f; + if (c > 1.0e-6f) { + g = min(g, c); + u = clamp(u, -c, c); + } + const float silu = g / (1.0f + exp(-g)); + mid_f32[first_row + row] = silu * u * route_weight; + } + } +} + typedef void (kernel_mul_mv2_disp_t)( ds4_metal_args_mul_mv args, device const char * src0, @@ -4299,6 +4432,160 @@ kernel void kernel_mul_mv_addr_iq2_xxs_pair_swiglu_f32( (void)tiitg; } +// M1 SSD-streaming candidate for the exact Flash 4096 -> 2048 gate/up shape. +// `dst_gate` and `dst_up` remain in the ABI so the host can switch pipelines +// without changing resource bindings; this kernel intentionally never writes +// either buffer. Host dispatch is opt-in and falls back to the kernel above +// for every other shape or execution mode. +kernel void kernel_mul_mv_addr_iq2_xxs_pair_swiglu_mid_only_4096x2048_f32( + constant ds4_metal_args_mul_mv_id & args, + constant ds4_metal_dsv4_moe_swiglu_weight_args & act, + device const uint64_t * gate_addrs, + device const uint64_t * up_addrs, + device const char * src1, + device char * dst_gate, + device char * dst_up, + device char * dst_mid, + device const char * ids, + device const char * weights, + threadgroup char * shmem [[threadgroup(0)]], + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiitg[[thread_index_in_threadgroup]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]]) { + if (args.ne00 != 4096 || args.ne01 != 2048 || args.ne0 != 2048 || + args.nr0 != N_R0_IQ2_XXS || args.nei1 != 1) { + return; + } + + const int iid1 = tgpig.z / args.nei0; + const int idx = tgpig.z % args.nei0; + tgpig.z = 0; + + const int32_t i02 = ((device const int32_t *)(ids + iid1 * args.nbi1))[idx]; + if (i02 < 0 || i02 >= args.ne02 || i02 >= 384) { + return; + } + const uint64_t gate_addr = gate_addrs[(uint)i02]; + const uint64_t up_addr = up_addrs[(uint)i02]; + if (gate_addr == 0 || up_addr == 0) { + return; + } + + const int64_t i11 = idx % args.ne11; + const int64_t i12 = iid1; + device const char *src0_gate_cur = + reinterpret_cast(gate_addr); + device const char *src0_up_cur = + reinterpret_cast(up_addr); + device const char *src1_cur = src1 + i11 * args.nb11 + i12 * args.nb12; + const uint64_t pair_row = (uint64_t)i12 * (uint64_t)args.nei0 + (uint64_t)idx; + + ds4_metal_args_mul_mv args0 = { + args.ne00, args.ne01, 1, + args.nb00, args.nb01, args.nb02, args.nb02, + args.ne10, 1, 1, + args.nb10, args.nb11, args.nb12, args.nb12, + args.ne0, 1, args.nr0, 1, 1, + }; + kernel_mul_mv_iq2_xxs_pair_swiglu_mid_only_4096x2048_impl( + args0, + act, + src0_gate_cur, + src0_up_cur, + src1_cur, + dst_mid, + weights, + pair_row, + shmem, + tgpig, + tiisg, + sgitg); + + (void)dst_gate; + (void)dst_up; + (void)tiitg; +} + +// Same mid-only arithmetic for the ordinary SSD cache split. Complementary +// resident/missing masks may execute in separate command buffers; each active +// slot owns a disjoint mid row, so no gate/up materialization is required. +kernel void kernel_mul_mv_addr_iq2_xxs_pair_swiglu_mid_only_4096x2048_masked_f32( + constant ds4_metal_args_mul_mv_id & args, + constant ds4_metal_dsv4_moe_swiglu_weight_args & act, + constant ds4_metal_stream_expert_split_args & split, + device const uint64_t * gate_addrs, + device const uint64_t * up_addrs, + device const char * src1, + device char * dst_gate, + device char * dst_up, + device char * dst_mid, + device const char * ids, + device const char * weights, + threadgroup char * shmem [[threadgroup(0)]], + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiitg[[thread_index_in_threadgroup]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]]) { + if (args.ne00 != 4096 || args.ne01 != 2048 || args.ne0 != 2048 || + args.nr0 != N_R0_IQ2_XXS || args.nei1 != 1 || + split.accumulate != 0u || split.active_mask == 0u || + (split.active_mask & ~0x3fu) != 0u) { + return; + } + + const int iid1 = tgpig.z / args.nei0; + const int idx = tgpig.z % args.nei0; + if ((split.active_mask & (1u << (uint)idx)) == 0) { + return; + } + tgpig.z = 0; + + const int32_t i02 = ((device const int32_t *)(ids + iid1 * args.nbi1))[idx]; + if (i02 < 0 || i02 >= args.ne02 || i02 >= 384) { + return; + } + const uint64_t gate_addr = gate_addrs[(uint)i02]; + const uint64_t up_addr = up_addrs[(uint)i02]; + if (gate_addr == 0 || up_addr == 0) { + return; + } + + const int64_t i11 = idx % args.ne11; + const int64_t i12 = iid1; + device const char *src0_gate_cur = + reinterpret_cast(gate_addr); + device const char *src0_up_cur = + reinterpret_cast(up_addr); + device const char *src1_cur = src1 + i11 * args.nb11 + i12 * args.nb12; + const uint64_t pair_row = (uint64_t)i12 * (uint64_t)args.nei0 + (uint64_t)idx; + + ds4_metal_args_mul_mv args0 = { + args.ne00, args.ne01, 1, + args.nb00, args.nb01, args.nb02, args.nb02, + args.ne10, 1, 1, + args.nb10, args.nb11, args.nb12, args.nb12, + args.ne0, 1, args.nr0, 1, 1, + }; + kernel_mul_mv_iq2_xxs_pair_swiglu_mid_only_4096x2048_impl( + args0, + act, + src0_gate_cur, + src0_up_cur, + src1_cur, + dst_mid, + weights, + pair_row, + shmem, + tgpig, + tiisg, + sgitg); + + (void)dst_gate; + (void)dst_up; + (void)tiitg; +} + kernel void kernel_mul_mv_addr_iq2_xxs_f32( constant ds4_metal_args_mul_mv_id & args, device const uint64_t * addrs, diff --git a/tests/test_metal_iq2_midonly.c b/tests/test_metal_iq2_midonly.c new file mode 100644 index 000000000..6ff74c4a4 --- /dev/null +++ b/tests/test_metal_iq2_midonly.c @@ -0,0 +1,76 @@ +#include "ds4_gpu.h" + +#include +#include + +#ifdef __APPLE__ + +/* ds4_metal.m references the CLI logger hook; the standalone tensor oracle + * does not need terminal detection. */ +bool ds4_log_is_tty(FILE *fp) { + (void)fp; + return false; +} + +int main(void) { + ds4_gpu_iq2_mid_only_oracle_report report; + if (!ds4_gpu_test_iq2_addr_mid_only_oracle(&report)) { + fprintf(stderr, "test_metal_iq2_midonly: setup/execution failed\n"); + ds4_gpu_cleanup(); + return 1; + } + + const int pass = + report.mid_words == 6u * 2048u && + report.mid_mismatches == 0 && + report.canonical_gate_unwritten == 0 && + report.canonical_up_unwritten == 0 && + report.candidate_gate_writes == 0 && + report.candidate_up_writes == 0 && + report.masked_mid_mismatches == 0 && + report.masked_inactive_writes == 0 && + report.masked_canonical_gate_unwritten == 0 && + report.masked_canonical_up_unwritten == 0 && + report.masked_gate_writes == 0 && + report.masked_up_writes == 0 && + report.guard_byte_mismatches == 0; + fprintf(stderr, + "test_metal_iq2_midonly: %s mid_words=%" PRIu64 + " mid_mismatches=%" PRIu64 + " canonical_gate_unwritten=%" PRIu64 + " canonical_up_unwritten=%" PRIu64 + " candidate_gate_writes=%" PRIu64 + " candidate_up_writes=%" PRIu64 + " masked_mid_mismatches=%" PRIu64 + " masked_inactive_writes=%" PRIu64 + " masked_canonical_gate_unwritten=%" PRIu64 + " masked_canonical_up_unwritten=%" PRIu64 + " masked_gate_writes=%" PRIu64 + " masked_up_writes=%" PRIu64 + " guard_byte_mismatches=%" PRIu64 "\n", + pass ? "PASS" : "FAIL", + report.mid_words, + report.mid_mismatches, + report.canonical_gate_unwritten, + report.canonical_up_unwritten, + report.candidate_gate_writes, + report.candidate_up_writes, + report.masked_mid_mismatches, + report.masked_inactive_writes, + report.masked_canonical_gate_unwritten, + report.masked_canonical_up_unwritten, + report.masked_gate_writes, + report.masked_up_writes, + report.guard_byte_mismatches); + ds4_gpu_cleanup(); + return pass ? 0 : 1; +} + +#else + +int main(void) { + fprintf(stderr, "test_metal_iq2_midonly: skipped (Metal requires macOS)\n"); + return 0; +} + +#endif From f9535a44855b0ba04793b097c8256c681b324086 Mon Sep 17 00:00:00 2001 From: Giorgio Oppo Date: Fri, 14 Aug 2026 14:02:40 +0200 Subject: [PATCH 022/189] metal: fix direct-RHS threadgroup memory sizing --- ds4_metal.m | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/ds4_metal.m b/ds4_metal.m index 9a0d098b9..1e9c81504 100644 --- a/ds4_metal.m +++ b/ds4_metal.m @@ -45,6 +45,11 @@ DS4_METAL_TENSOR_MXFP4 = 39, }; +/* kernel_mul_mm_mpp_direct_rhs double-buffers two 64x32 half tiles. */ +enum { + DS4_METAL_MPP_DIRECT_RHS_SMEM = 2u * 64u * 32u * sizeof(uint16_t), +}; + @class DS4MetalQ4ExpertTable; static id g_device; @@ -19794,7 +19799,7 @@ static int ds4_gpu_matmul_q8_0_legacy_tensor( [enc setBuffer:wbuf offset:(NSUInteger)inner_offset atIndex:1]; [enc setBuffer:xbuf offset:ds4_gpu_tensor_offset(x) atIndex:2]; [enc setBuffer:outbuf offset:ds4_gpu_tensor_offset(out) atIndex:3]; - [enc setThreadgroupMemoryLength:64u * 32u * sizeof(uint16_t) atIndex:0]; + [enc setThreadgroupMemoryLength:DS4_METAL_MPP_DIRECT_RHS_SMEM atIndex:0]; [enc dispatchThreadgroups:MTLSizeMake(1u, ((NSUInteger)out_dim + 63u) / 64u, 1u) @@ -19919,7 +19924,7 @@ static int ds4_gpu_matmul_q8_0_legacy_tensor( [enc setBuffer:wbuf offset:(NSUInteger)inner_offset atIndex:1]; [enc setBuffer:xbuf offset:ds4_gpu_tensor_offset(x) atIndex:2]; [enc setBuffer:outbuf offset:ds4_gpu_tensor_offset(out) atIndex:3]; - [enc setThreadgroupMemoryLength:2u * 64u * 32u * sizeof(uint16_t) atIndex:0]; + [enc setThreadgroupMemoryLength:DS4_METAL_MPP_DIRECT_RHS_SMEM atIndex:0]; [enc dispatchThreadgroups:MTLSizeMake((NSUInteger)(nax_rows / nax_tile_n), (NSUInteger)out_dim / 64u, 1) @@ -20379,7 +20384,7 @@ static int ds4_gpu_matmul_quant_impl_tensor( [enc setBuffer:wbuf offset:(NSUInteger)inner_offset atIndex:1]; [enc setBuffer:xbuf offset:ds4_gpu_tensor_offset(x) atIndex:2]; [enc setBuffer:outbuf offset:ds4_gpu_tensor_offset(out) atIndex:3]; - [enc setThreadgroupMemoryLength:64u * 32u * sizeof(uint16_t) atIndex:0]; + [enc setThreadgroupMemoryLength:DS4_METAL_MPP_DIRECT_RHS_SMEM atIndex:0]; [enc dispatchThreadgroups:MTLSizeMake((NSUInteger)(n_tok / nax_tile_n), (NSUInteger)out_dim / 64u, 1) @@ -21801,7 +21806,7 @@ int ds4_gpu_matmul_f16_tensor( [enc setBuffer:wbuf offset:(NSUInteger)inner_offset atIndex:1]; [enc setBuffer:xbuf offset:ds4_gpu_tensor_offset(x) atIndex:2]; [enc setBuffer:outbuf offset:ds4_gpu_tensor_offset(out) atIndex:3]; - [enc setThreadgroupMemoryLength:2u * 64u * 32u * sizeof(uint16_t) atIndex:0]; + [enc setThreadgroupMemoryLength:DS4_METAL_MPP_DIRECT_RHS_SMEM atIndex:0]; [enc dispatchThreadgroups:MTLSizeMake((NSUInteger)(n_tok / nax_tile_n), (NSUInteger)out_dim / 64u, 1) From 41b8265b39d750ee494afc17db4b9167768b063a Mon Sep 17 00:00:00 2001 From: Giorgio Oppo Date: Fri, 14 Aug 2026 14:07:25 +0200 Subject: [PATCH 023/189] metal: merge IQ2 selected router and shared work --- ds4.c | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/ds4.c b/ds4.c index 34d3e578b..01eecab77 100644 --- a/ds4.c +++ b/ds4.c @@ -21429,6 +21429,25 @@ static bool metal_graph_use_iq2_selected_async_load(const ds4_gpu_graph *g) { static bool metal_graph_use_iq2_selected_async_early_commit( const ds4_gpu_graph *g) { + /* The selected-id event already lets the service thread start the IQ2 + * expert load while router and shared-expert work remain in one command + * buffer. Keep the historical extra frontier as an explicit rollback + * for isolated A/Bs instead of paying it by default. */ + return g && + g->ssd_streaming && +#ifndef DS4_ROCM_BUILD + getenv("DS4_METAL_ENABLE_IQ2_SELECTED_ASYNC_EARLY_COMMIT") != NULL && + getenv("DS4_METAL_DISABLE_STREAMING_SELECTED_ASYNC_EARLY_COMMIT") == NULL; +#else + false; +#endif +} + +static bool metal_graph_use_selected_async_early_commit_legacy( + const ds4_gpu_graph *g) { + /* MXFP4 and native CUDA were not part of the IQ2 Metal scheduling A/B. + * Preserve their established default while the IQ2-only rollback above + * selects the merged Metal boundary. */ return g && g->ssd_streaming && #ifndef DS4_ROCM_BUILD @@ -25920,11 +25939,19 @@ static bool metal_graph_encode_decode_layer_phase( if (ok) ok = ds4_gpu_signal_selected_readback_ready(&selected_event) != 0; metal_graph_selected_async_load async_load = {0}; bool async_load_started = false; + const bool iq2_metal_merged_boundary = + iq2_selected_shared_overlap && + !cuda_selected_shared_overlap; const bool async_early_commit = async_selected_load && - metal_graph_use_iq2_selected_async_early_commit(g); + (iq2_metal_merged_boundary ? + metal_graph_use_iq2_selected_async_early_commit(g) : + metal_graph_use_selected_async_early_commit_legacy(g)); if (ok && async_selected_load) { - ok = metal_graph_selected_async_load_start(&async_load, + /* Failure to acquire the optional worker retains the established + * synchronous selected-id/load fallback below. */ + async_load_started = + metal_graph_selected_async_load_start(&async_load, g, model, layer, @@ -25932,9 +25959,8 @@ static bool metal_graph_encode_decode_layer_phase( selected_event, gate_expert_bytes, down_expert_bytes); - async_load_started = ok; } - if (ok && async_early_commit) { + if (ok && async_early_commit && async_load_started) { ok = ds4_gpu_flush_commands() != 0; } if (ok && fuse_shared_gate_up) { From 61e35e223afdb053d9bf89e226cb05c5dc2c2d88 Mon Sep 17 00:00:00 2001 From: Giorgio Oppo Date: Fri, 14 Aug 2026 14:28:12 +0200 Subject: [PATCH 024/189] metal: index live IQ2 SSD cache entries --- Makefile | 14 +- ds4_gpu.h | 25 ++ ds4_metal.m | 722 ++++++++++++++++++++++++------ tests/test_metal_iq2_live_index.c | 240 ++++++++++ 4 files changed, 857 insertions(+), 144 deletions(-) create mode 100644 tests/test_metal_iq2_live_index.c diff --git a/Makefile b/Makefile index d77d2f79f..67c399ffa 100644 --- a/Makefile +++ b/Makefile @@ -68,7 +68,7 @@ DS4_LINK_LIBS ?= $(CUDA_LDLIBS) METAL_LDLIBS := $(LDLIBS) endif -.PHONY: all help clean test test-rocm test-glm53-kda-rocm test-metal-session-batch test-metal-exactn-oracle test-metal-dspark-capture test-metal-iq2-midonly test-mxfp4-cuda test-mxfp4-rocm test-mmq-parity-cuda test-cuda-session-batch test-cuda-mixed-batch dspark-acceptance dspark-verify-depth rocm-dspark-acceptance rocm-dspark-verify-depth mtp-verify-depth cpu cuda cuda-spark cuda-generic cuda-regression strix-halo rocm +.PHONY: all help clean test test-rocm test-glm53-kda-rocm test-metal-session-batch test-metal-exactn-oracle test-metal-dspark-capture test-metal-iq2-midonly test-metal-iq2-live-index test-mxfp4-cuda test-mxfp4-rocm test-mmq-parity-cuda test-cuda-session-batch test-cuda-mixed-batch dspark-acceptance dspark-verify-depth rocm-dspark-acceptance rocm-dspark-verify-depth mtp-verify-depth cpu cuda cuda-spark cuda-generic cuda-regression strix-halo rocm ifeq ($(UNAME_S),Darwin) .PHONY: metal-decode-schedule-bench metal-prefill-variant-bench check-mxfp4-half-lut test-mxfp4-metal @@ -82,6 +82,7 @@ help: @echo " make test Build and run tests" @echo " make test-metal-dspark-capture Check fused DSpark HC capture bitwise" @echo " make test-metal-iq2-midonly Check M1 IQ2 addr mid-only output and sentinels" + @echo " make test-metal-iq2-live-index Check IQ2 SSD live-cache index policy and fallback" @echo " make metal-decode-schedule-bench Build the balanced Metal decode schedule benchmark" @echo " make metal-prefill-variant-bench Build the balanced Metal prefill variant benchmark" @echo " make check-mxfp4-half-lut Verify the checked-in MXFP4 half LUT matches the generator" @@ -136,6 +137,15 @@ tests/test_metal_iq2_midonly: tests/test_metal_iq2_midonly.o ds4_metal.o test-metal-iq2-midonly: tests/test_metal_iq2_midonly ./tests/test_metal_iq2_midonly +tests/test_metal_iq2_live_index.o: tests/test_metal_iq2_live_index.c ds4_gpu.h + $(CC) $(CFLAGS) -I. -c -o $@ $< + +tests/test_metal_iq2_live_index: tests/test_metal_iq2_live_index.o ds4_metal.o + $(CC) $(CFLAGS) -o $@ $^ $(METAL_LDLIBS) + +test-metal-iq2-live-index: tests/test_metal_iq2_live_index + ./tests/test_metal_iq2_live_index + speed-bench/metal_decode_schedule_bench.o: speed-bench/metal_decode_schedule_bench.c ds4.h $(CC) $(CFLAGS) -I. -c -o $@ $< @@ -673,4 +683,4 @@ mxfp4-dot-test: tests/test_mxfp4_dot.c ./tests/test_mxfp4_dot clean: - rm -f ds4 ds4-server ds4-bench ds4-eval ds4-agent ds4_cpu ds4_native ds4_server_test ds4_test ds4_agent_test gguf-tools/quality-testing/score_official gguf-tools/quality-testing/score_official.o speed-bench/metal_decode_schedule_bench speed-bench/metal_prefill_variant_bench speed-bench/*.o tests/test_q4k_dot tests/test_mxfp4_dot tests/test_mxfp4_metal tests/test_mxfp4_rocm tests/test_mxfp4_cuda tests/test_metal_session_batch tests/test_metal_exactn_oracle tests/test_metal_dspark_capture tests/test_metal_iq2_midonly tests/test_glm53_kda tests/test_glm53_kda_rocm tests/test_glm53_vision_engine tests/test_glm53_vision_prompt tests/test_gpu_xdev tests/test_gpu_model_cache tests/test_gpu_lookup_cache_strict tests/test_engine_mgpu_refusal tests/test_engine_mgpu_runtime tests/test_engine_correctness tests/test_sampling tests/test_cuda_session_batch tests/test_cuda_mixed_batch tests/*.o *.o tests/cuda_long_context_smoke tests/cuda_long_context_smoke.o + rm -f ds4 ds4-server ds4-bench ds4-eval ds4-agent ds4_cpu ds4_native ds4_server_test ds4_test ds4_agent_test gguf-tools/quality-testing/score_official gguf-tools/quality-testing/score_official.o speed-bench/metal_decode_schedule_bench speed-bench/metal_prefill_variant_bench speed-bench/*.o tests/test_q4k_dot tests/test_mxfp4_dot tests/test_mxfp4_metal tests/test_mxfp4_rocm tests/test_mxfp4_cuda tests/test_metal_session_batch tests/test_metal_exactn_oracle tests/test_metal_dspark_capture tests/test_metal_iq2_midonly tests/test_metal_iq2_live_index tests/test_glm53_kda tests/test_glm53_kda_rocm tests/test_glm53_vision_engine tests/test_glm53_vision_prompt tests/test_gpu_xdev tests/test_gpu_model_cache tests/test_gpu_lookup_cache_strict tests/test_engine_mgpu_refusal tests/test_engine_mgpu_runtime tests/test_engine_correctness tests/test_sampling tests/test_cuda_session_batch tests/test_cuda_mixed_batch tests/*.o *.o tests/cuda_long_context_smoke tests/cuda_long_context_smoke.o diff --git a/ds4_gpu.h b/ds4_gpu.h index c33a1f3e7..a461bb17b 100644 --- a/ds4_gpu.h +++ b/ds4_gpu.h @@ -209,6 +209,30 @@ typedef struct ds4_gpu_iq2_mid_only_oracle_report { * sentinel failures are reported explicitly in `report`. */ int ds4_gpu_test_iq2_addr_mid_only_oracle( ds4_gpu_iq2_mid_only_oracle_report *report); +typedef struct ds4_gpu_stream_expert_live_index_report { + uint64_t scans; + uint64_t entries; + uint64_t fallbacks; + uint64_t inserts; + uint64_t removes; + uint64_t reuse_scan_calls; + uint64_t reuse_scan_entries; + uint64_t resident_hash; + uint32_t live_count; + uint32_t cache_entries; + uint32_t eligible; + uint32_t active; + uint32_t broken; +} ds4_gpu_stream_expert_live_index_report; +/* Test-only policy/state hooks for the IQ2 production-size SSD cache index. */ +int ds4_gpu_test_stream_expert_live_index_policy( + int ssd_streaming, + uint64_t gate_expert_bytes, + uint64_t down_expert_bytes, + int enable, + int disable); +void ds4_gpu_test_stream_expert_live_index_report( + ds4_gpu_stream_expert_live_index_report *report); enum { DS4_GPU_TEST_MXFP4_PAIR_TAIL_CULL = 1u << 0, DS4_GPU_TEST_MXFP4_PAIR_COMPACT_TILE = 1u << 1, @@ -217,6 +241,7 @@ enum { DS4_GPU_TEST_MXFP4_DOWN_HALF_LUT = 1u << 4, DS4_GPU_TEST_OUTPUT_HC_WEIGHTS4 = 1u << 5, DS4_GPU_TEST_HC_RMS_SCALE_PROJ = 1u << 6, + DS4_GPU_TEST_STREAMING_LIVE_INDEX_FAILURE = 1u << 7, }; void ds4_gpu_test_set_flags(uint32_t flags); void ds4_gpu_release_zero_prefix_prefill_mask_cache(void); diff --git a/ds4_metal.m b/ds4_metal.m index 1e9c81504..e8dd88e52 100644 --- a/ds4_metal.m +++ b/ds4_metal.m @@ -400,6 +400,9 @@ static int g_model_buffer_cache_over_limit; static uint64_t g_stream_expert_cache_bytes; static uint64_t g_stream_expert_cache_expert_bytes; +static uint64_t g_stream_expert_cache_gate_class_bytes; +static uint64_t g_stream_expert_cache_down_class_bytes; +static int g_stream_expert_cache_class_conflict; static uint32_t g_stream_expert_cache_entry_count; static uint32_t g_stream_expert_cache_budget_override; static uint64_t g_stream_expert_cache_hits; @@ -445,6 +448,11 @@ static uint64_t g_stream_expert_timing_reuse_scan_entries; static double g_stream_expert_timing_reuse_scan_ms; static double g_stream_expert_timing_reuse_clear_ms; +static uint64_t g_stream_expert_live_index_scans; +static uint64_t g_stream_expert_live_index_entries; +static uint64_t g_stream_expert_live_index_fallbacks; +static uint64_t g_stream_expert_live_index_inserts; +static uint64_t g_stream_expert_live_index_removes; static uint64_t g_stream_expert_timing_readahead_calls; static uint64_t g_stream_expert_timing_readahead_bytes; static double g_stream_expert_timing_readahead_ms; @@ -516,6 +524,7 @@ static int ds4_gpu_stream_expert_cache_note_expert_size( static void ds4_gpu_stream_expert_cache_clear_all(int reset_stats); static void ds4_gpu_stream_expert_pending_load_clear(void); static void ds4_gpu_stream_expert_pread_pool_shutdown(void); +static void ds4_gpu_stream_expert_cache_live_release(void); static int ds4_gpu_stream_expert_timing_summary_enabled(void); static int ds4_gpu_stream_expert_cache_entry_protected( uint32_t layer, @@ -4412,6 +4421,12 @@ void ds4_gpu_set_streaming_expert_cache_expert_bytes(uint64_t bytes) { * deterministically from startup, instead of depending on which layer * happens to touch the cache first. */ + if (bytes != g_stream_expert_cache_expert_bytes) { + ds4_gpu_stream_expert_cache_live_release(); + g_stream_expert_cache_gate_class_bytes = 0; + g_stream_expert_cache_down_class_bytes = 0; + g_stream_expert_cache_class_conflict = 0; + } g_stream_expert_cache_expert_bytes = bytes; } @@ -10789,6 +10804,10 @@ void ds4_gpu_cleanup(void) { [g_transient_buffers removeAllObjects]; ds4_gpu_stream_expert_pread_pool_shutdown(); ds4_gpu_stream_expert_cache_clear_all(1); + ds4_gpu_stream_expert_cache_live_release(); + g_stream_expert_cache_gate_class_bytes = 0; + g_stream_expert_cache_down_class_bytes = 0; + g_stream_expert_cache_class_conflict = 0; for (uint32_t layer = 0; layer < DS4_METAL_STREAM_EXPERT_CACHE_MAX_LAYER; layer++) { g_stream_expert_cache_gate_addr_buffers[layer] = nil; g_stream_expert_cache_up_addr_buffers[layer] = nil; @@ -12445,9 +12464,19 @@ static int ds4_gpu_stream_expert_cache_note_expert_size( const uint64_t bytes = gate_expert_bytes * 2ull + down_expert_bytes; if (g_stream_expert_cache_expert_bytes == 0) { g_stream_expert_cache_expert_bytes = bytes; - return 1; } - return bytes == g_stream_expert_cache_expert_bytes; + if (bytes != g_stream_expert_cache_expert_bytes) return 0; + if (g_stream_expert_cache_gate_class_bytes == 0 && + g_stream_expert_cache_down_class_bytes == 0) { + g_stream_expert_cache_gate_class_bytes = gate_expert_bytes; + g_stream_expert_cache_down_class_bytes = down_expert_bytes; + } else if (g_stream_expert_cache_gate_class_bytes != gate_expert_bytes || + g_stream_expert_cache_down_class_bytes != down_expert_bytes) { + /* Preserve the cache's historical total-size admission contract, but + * never use the IQ2-only live index for an ambiguous size class. */ + g_stream_expert_cache_class_conflict = 1; + } + return 1; } static uint32_t ds4_gpu_stream_expert_cache_requested_budget(void) { @@ -14673,6 +14702,406 @@ static int ds4_gpu_stream_expert_cache_validate_selected( return 1; } +enum { + DS4_METAL_IQ2_LIVE_GATE_EXPERT_BYTES = 2162688u, + DS4_METAL_IQ2_LIVE_DOWN_EXPERT_BYTES = 2752512u, + DS4_METAL_IQ2_LIVE_TOTAL_EXPERT_BYTES = 7077888u, +}; + +/* Dense resident IDs avoid the historical 80 x 384 victim scan. The + * optimization is intentionally limited to the measured production IQ2 + * cache shape; every failure falls back to the authoritative cache matrix. */ +static uint32_t *g_stream_expert_cache_live_ids; +static uint32_t *g_stream_expert_cache_live_pos_plus_one; +static uint32_t g_stream_expert_cache_live_count; +static int g_stream_expert_cache_live_broken; +static int g_stream_expert_cache_live_warned; +static int g_stream_expert_cache_live_selftest_checked; +static int g_stream_expert_cache_live_rebuild_required; + +static int ds4_gpu_stream_expert_cache_live_policy( + int ssd_streaming, + uint64_t gate_expert_bytes, + uint64_t down_expert_bytes, + int enable, + int disable) { + if (!ssd_streaming || + gate_expert_bytes != DS4_METAL_IQ2_LIVE_GATE_EXPERT_BYTES || + down_expert_bytes != DS4_METAL_IQ2_LIVE_DOWN_EXPERT_BYTES || + gate_expert_bytes > (UINT64_MAX - down_expert_bytes) / 2u || + 2u * gate_expert_bytes + down_expert_bytes != + DS4_METAL_IQ2_LIVE_TOTAL_EXPERT_BYTES) { + return 0; + } + if (disable == 1) return 0; + if (enable >= 0) return enable == 1; + return 1; +} + +int ds4_gpu_test_stream_expert_live_index_policy( + int ssd_streaming, + uint64_t gate_expert_bytes, + uint64_t down_expert_bytes, + int enable, + int disable) { + return ds4_gpu_stream_expert_cache_live_policy(ssd_streaming, + gate_expert_bytes, + down_expert_bytes, + enable, + disable); +} + +static int ds4_gpu_stream_expert_cache_live_eligible(void) { + return !g_stream_expert_cache_class_conflict && + g_stream_expert_cache_expert_bytes == + DS4_METAL_IQ2_LIVE_TOTAL_EXPERT_BYTES && + ds4_gpu_stream_expert_cache_live_policy( + g_ssd_streaming_mode, + g_stream_expert_cache_gate_class_bytes, + g_stream_expert_cache_down_class_bytes, + -1, + 0); +} + +static int ds4_gpu_stream_expert_cache_live_requested(void) { + if (g_stream_expert_cache_class_conflict || + g_stream_expert_cache_expert_bytes != + DS4_METAL_IQ2_LIVE_TOTAL_EXPERT_BYTES) { + return 0; + } + return ds4_gpu_stream_expert_cache_live_policy( + g_ssd_streaming_mode, + g_stream_expert_cache_gate_class_bytes, + g_stream_expert_cache_down_class_bytes, + ds4_gpu_env_bool("DS4_METAL_ENABLE_STREAMING_EXPERT_LIVE_INDEX"), + ds4_gpu_env_bool("DS4_METAL_DISABLE_STREAMING_EXPERT_LIVE_INDEX")); +} + +static int ds4_gpu_stream_expert_cache_live_add_raw( + uint32_t *ids, + uint32_t *pos_plus_one, + uint32_t capacity, + uint32_t *count, + uint32_t id) { + if (!ids || !pos_plus_one || !count || id >= capacity) return 0; + if (pos_plus_one[id] != 0) return 1; + if (*count >= capacity) return 0; + ids[*count] = id; + pos_plus_one[id] = *count + 1u; + (*count)++; + return 1; +} + +static int ds4_gpu_stream_expert_cache_live_remove_raw( + uint32_t *ids, + uint32_t *pos_plus_one, + uint32_t capacity, + uint32_t *count, + uint32_t id) { + if (!ids || !pos_plus_one || !count || id >= capacity || + pos_plus_one[id] == 0 || *count == 0) { + return 0; + } + const uint32_t pos = pos_plus_one[id] - 1u; + if (pos >= *count) return 0; + const uint32_t last_pos = *count - 1u; + const uint32_t moved = ids[last_pos]; + if (moved >= capacity) return 0; + ids[pos] = moved; + pos_plus_one[moved] = pos + 1u; + pos_plus_one[id] = 0; + *count = last_pos; + return 1; +} + +static int ds4_gpu_stream_expert_cache_live_selftest(void) { + uint32_t ids[8] = {0}; + uint32_t pos[8] = {0}; + uint32_t count = 0; + if (!ds4_gpu_stream_expert_cache_live_add_raw(ids, pos, 8, &count, 1) || + !ds4_gpu_stream_expert_cache_live_add_raw(ids, pos, 8, &count, 3) || + !ds4_gpu_stream_expert_cache_live_add_raw(ids, pos, 8, &count, 5) || + !ds4_gpu_stream_expert_cache_live_add_raw(ids, pos, 8, &count, 3) || + count != 3 || pos[1] != 1 || pos[3] != 2 || pos[5] != 3) { + return 0; + } + if (!ds4_gpu_stream_expert_cache_live_remove_raw(ids, pos, 8, &count, 3) || + count != 2 || pos[3] != 0 || ids[1] != 5 || pos[5] != 2) { + return 0; + } + return ds4_gpu_stream_expert_cache_live_remove_raw( + ids, pos, 8, &count, 1) && + ds4_gpu_stream_expert_cache_live_remove_raw( + ids, pos, 8, &count, 5) && + count == 0 && pos[1] == 0 && pos[5] == 0; +} + +static void ds4_gpu_stream_expert_cache_live_mark_broken( + const char *reason) { + if (!g_stream_expert_cache_live_broken) { + g_stream_expert_live_index_fallbacks++; + } + g_stream_expert_cache_live_broken = 1; + if (!g_stream_expert_cache_live_warned) { + fprintf(stderr, + "ds4: Metal IQ2 streaming expert live index disabled (%s); using full cache scan\n", + reason ? reason : "invariant failure"); + g_stream_expert_cache_live_warned = 1; + } +} + +static int ds4_gpu_stream_expert_cache_live_rebuild(void) { + const uint32_t capacity = DS4_METAL_STREAM_EXPERT_CACHE_MAX_ENTRIES; + memset(g_stream_expert_cache_live_pos_plus_one, + 0, + (size_t)capacity * + sizeof(g_stream_expert_cache_live_pos_plus_one[0])); + g_stream_expert_cache_live_count = 0; + for (uint32_t layer = 0; + layer < DS4_METAL_STREAM_EXPERT_CACHE_MAX_LAYER; + layer++) { + for (uint32_t expert = 0; + expert < DS4_METAL_STREAM_EXPERT_CACHE_MAX_EXPERT; + expert++) { + if (!g_stream_expert_cache[layer][expert].valid) continue; + const uint32_t id = + layer * DS4_METAL_STREAM_EXPERT_CACHE_MAX_EXPERT + expert; + if (!ds4_gpu_stream_expert_cache_live_add_raw( + g_stream_expert_cache_live_ids, + g_stream_expert_cache_live_pos_plus_one, + capacity, + &g_stream_expert_cache_live_count, + id)) { + return 0; + } + } + } + g_stream_expert_cache_live_rebuild_required = 0; + return 1; +} + +static int ds4_gpu_stream_expert_cache_live_ensure(void) { + if (!ds4_gpu_stream_expert_cache_live_requested() || + g_stream_expert_cache_live_broken) { + return 0; + } + if ((g_test_flags & DS4_GPU_TEST_STREAMING_LIVE_INDEX_FAILURE) != 0u) { + ds4_gpu_stream_expert_cache_live_mark_broken("test fault injection"); + return 0; + } + if (!g_stream_expert_cache_live_selftest_checked) { + g_stream_expert_cache_live_selftest_checked = 1; + if (!ds4_gpu_stream_expert_cache_live_selftest()) { + ds4_gpu_stream_expert_cache_live_mark_broken("selftest failure"); + return 0; + } + } + + const size_t capacity = DS4_METAL_STREAM_EXPERT_CACHE_MAX_ENTRIES; + if (!g_stream_expert_cache_live_ids) { + g_stream_expert_cache_live_ids = + calloc(capacity, sizeof(g_stream_expert_cache_live_ids[0])); + g_stream_expert_cache_live_pos_plus_one = + calloc(capacity, + sizeof(g_stream_expert_cache_live_pos_plus_one[0])); + if (!g_stream_expert_cache_live_ids || + !g_stream_expert_cache_live_pos_plus_one) { + free(g_stream_expert_cache_live_ids); + free(g_stream_expert_cache_live_pos_plus_one); + g_stream_expert_cache_live_ids = NULL; + g_stream_expert_cache_live_pos_plus_one = NULL; + ds4_gpu_stream_expert_cache_live_mark_broken( + "allocation failure"); + return 0; + } + g_stream_expert_cache_live_rebuild_required = 1; + } + if (g_stream_expert_cache_live_rebuild_required && + !ds4_gpu_stream_expert_cache_live_rebuild()) { + ds4_gpu_stream_expert_cache_live_mark_broken("rebuild failure"); + return 0; + } + return 1; +} + +static int ds4_gpu_stream_expert_cache_live_ready(void) { + if (!ds4_gpu_stream_expert_cache_live_ensure()) return 0; + if (g_stream_expert_cache_live_count != + g_stream_expert_cache_entry_count) { + ds4_gpu_stream_expert_cache_live_mark_broken("count invariant"); + return 0; + } + for (uint32_t pos = 0; pos < g_stream_expert_cache_live_count; pos++) { + const uint32_t id = g_stream_expert_cache_live_ids[pos]; + if (id >= DS4_METAL_STREAM_EXPERT_CACHE_MAX_ENTRIES || + g_stream_expert_cache_live_pos_plus_one[id] != pos + 1u) { + ds4_gpu_stream_expert_cache_live_mark_broken( + "position invariant"); + return 0; + } + const uint32_t layer = + id / DS4_METAL_STREAM_EXPERT_CACHE_MAX_EXPERT; + const uint32_t expert = + id % DS4_METAL_STREAM_EXPERT_CACHE_MAX_EXPERT; + if (!g_stream_expert_cache[layer][expert].valid) { + ds4_gpu_stream_expert_cache_live_mark_broken( + "resident invariant"); + return 0; + } + } + return 1; +} + +static void ds4_gpu_stream_expert_cache_live_insert( + uint32_t layer, + uint32_t expert) { + if (!ds4_gpu_stream_expert_cache_live_requested()) { + if (g_stream_expert_cache_live_ids) { + g_stream_expert_cache_live_rebuild_required = 1; + } + return; + } + if (!ds4_gpu_stream_expert_cache_live_ensure()) return; + const uint32_t id = + layer * DS4_METAL_STREAM_EXPERT_CACHE_MAX_EXPERT + expert; + if (!ds4_gpu_stream_expert_cache_live_add_raw( + g_stream_expert_cache_live_ids, + g_stream_expert_cache_live_pos_plus_one, + DS4_METAL_STREAM_EXPERT_CACHE_MAX_ENTRIES, + &g_stream_expert_cache_live_count, + id)) { + ds4_gpu_stream_expert_cache_live_mark_broken("insert failure"); + return; + } + g_stream_expert_live_index_inserts++; +} + +static void ds4_gpu_stream_expert_cache_live_remove( + uint32_t layer, + uint32_t expert) { + if (!ds4_gpu_stream_expert_cache_live_requested()) { + if (g_stream_expert_cache_live_ids) { + g_stream_expert_cache_live_rebuild_required = 1; + } + return; + } + if (!ds4_gpu_stream_expert_cache_live_ensure()) return; + const uint32_t id = + layer * DS4_METAL_STREAM_EXPERT_CACHE_MAX_EXPERT + expert; + if (!ds4_gpu_stream_expert_cache_live_remove_raw( + g_stream_expert_cache_live_ids, + g_stream_expert_cache_live_pos_plus_one, + DS4_METAL_STREAM_EXPERT_CACHE_MAX_ENTRIES, + &g_stream_expert_cache_live_count, + id)) { + ds4_gpu_stream_expert_cache_live_mark_broken("remove failure"); + return; + } + g_stream_expert_live_index_removes++; +} + +static void ds4_gpu_stream_expert_cache_live_reset(void) { + g_stream_expert_cache_live_count = 0; + g_stream_expert_cache_live_broken = 0; + g_stream_expert_cache_live_warned = 0; + g_stream_expert_cache_live_rebuild_required = 0; + if (g_stream_expert_cache_live_pos_plus_one) { + memset(g_stream_expert_cache_live_pos_plus_one, + 0, + (size_t)DS4_METAL_STREAM_EXPERT_CACHE_MAX_ENTRIES * + sizeof(g_stream_expert_cache_live_pos_plus_one[0])); + } +} + +static void ds4_gpu_stream_expert_cache_live_release(void) { + free(g_stream_expert_cache_live_ids); + free(g_stream_expert_cache_live_pos_plus_one); + g_stream_expert_cache_live_ids = NULL; + g_stream_expert_cache_live_pos_plus_one = NULL; + g_stream_expert_cache_live_count = 0; + g_stream_expert_cache_live_broken = 0; + g_stream_expert_cache_live_warned = 0; + g_stream_expert_cache_live_selftest_checked = 0; + g_stream_expert_cache_live_rebuild_required = 0; +} + +typedef struct { + uint32_t cursor; + uint32_t limit; + int live; +} ds4_gpu_stream_expert_cache_iterator; + +static ds4_gpu_stream_expert_cache_iterator +ds4_gpu_stream_expert_cache_iterator_begin(void) { + ds4_gpu_stream_expert_cache_iterator it = { + .cursor = 0, + .limit = DS4_METAL_STREAM_EXPERT_CACHE_MAX_ENTRIES, + .live = 0, + }; + if (ds4_gpu_stream_expert_cache_live_requested() && + ds4_gpu_stream_expert_cache_live_ready()) { + it.limit = g_stream_expert_cache_live_count; + it.live = 1; + g_stream_expert_live_index_scans++; + } + return it; +} + +static int ds4_gpu_stream_expert_cache_iterator_next( + ds4_gpu_stream_expert_cache_iterator *it, + uint32_t *layer, + uint32_t *expert) { + if (!it || !layer || !expert || it->cursor >= it->limit) return 0; + const uint32_t pos = it->cursor++; + const uint32_t id = + it->live ? g_stream_expert_cache_live_ids[pos] : pos; + if (it->live) g_stream_expert_live_index_entries++; + *layer = id / DS4_METAL_STREAM_EXPERT_CACHE_MAX_EXPERT; + *expert = id % DS4_METAL_STREAM_EXPERT_CACHE_MAX_EXPERT; + return 1; +} + +void ds4_gpu_test_stream_expert_live_index_report( + ds4_gpu_stream_expert_live_index_report *report) { + if (!report) return; + uint64_t resident_hash = UINT64_C(1469598103934665603); + for (uint32_t layer = 0; + layer < DS4_METAL_STREAM_EXPERT_CACHE_MAX_LAYER; + layer++) { + for (uint32_t expert = 0; + expert < DS4_METAL_STREAM_EXPERT_CACHE_MAX_EXPERT; + expert++) { + if (!g_stream_expert_cache[layer][expert].valid) continue; + const uint64_t id = + (uint64_t)layer * DS4_METAL_STREAM_EXPERT_CACHE_MAX_EXPERT + + expert; + resident_hash ^= id + UINT64_C(0x9e3779b97f4a7c15); + resident_hash *= UINT64_C(1099511628211); + } + } + *report = (ds4_gpu_stream_expert_live_index_report) { + .scans = g_stream_expert_live_index_scans, + .entries = g_stream_expert_live_index_entries, + .fallbacks = g_stream_expert_live_index_fallbacks, + .inserts = g_stream_expert_live_index_inserts, + .removes = g_stream_expert_live_index_removes, + .reuse_scan_calls = g_stream_expert_timing_reuse_scan_calls, + .reuse_scan_entries = g_stream_expert_timing_reuse_scan_entries, + .resident_hash = resident_hash, + .live_count = g_stream_expert_cache_live_count, + .cache_entries = g_stream_expert_cache_entry_count, + .eligible = (uint32_t)ds4_gpu_stream_expert_cache_live_eligible(), + .active = (uint32_t)( + ds4_gpu_stream_expert_cache_live_requested() && + !g_stream_expert_cache_live_broken && + !g_stream_expert_cache_live_rebuild_required && + g_stream_expert_cache_live_ids != NULL && + g_stream_expert_cache_live_count == + g_stream_expert_cache_entry_count), + .broken = (uint32_t)(g_stream_expert_cache_live_broken != 0), + }; +} + static void ds4_gpu_stream_expert_cache_clear_entry_internal( uint32_t layer, uint32_t expert, @@ -14698,6 +15127,8 @@ static void ds4_gpu_stream_expert_cache_clear_entry_internal( return; } + ds4_gpu_stream_expert_cache_live_remove(layer, expert); + const uint64_t bytes = e->logical_bytes; ds4_gpu_stream_expert_evict_dontneed_range(e->model_map, e->model_size, @@ -14809,6 +15240,7 @@ static void ds4_gpu_stream_expert_cache_clear_all(int reset_stats) { } g_stream_expert_cache_bytes = 0; g_stream_expert_cache_entry_count = 0; + ds4_gpu_stream_expert_cache_live_reset(); for (uint32_t i = 0; i < g_stream_expert_cache_slab_count; i++) { g_stream_expert_cache_slabs[i] = nil; g_stream_expert_cache_slab_start_slot[i] = 0; @@ -14876,6 +15308,11 @@ static void ds4_gpu_stream_expert_cache_clear_all(int reset_stats) { g_stream_expert_timing_reuse_scan_entries = 0; g_stream_expert_timing_reuse_scan_ms = 0.0; g_stream_expert_timing_reuse_clear_ms = 0.0; + g_stream_expert_live_index_scans = 0; + g_stream_expert_live_index_entries = 0; + g_stream_expert_live_index_fallbacks = 0; + g_stream_expert_live_index_inserts = 0; + g_stream_expert_live_index_removes = 0; g_stream_expert_timing_readahead_calls = 0; g_stream_expert_timing_readahead_bytes = 0; g_stream_expert_timing_readahead_ms = 0.0; @@ -15024,40 +15461,40 @@ static int ds4_gpu_stream_expert_cache_take_reusable( const int timing = ds4_gpu_stream_expert_timing_summary_enabled(); const double scan_t0 = timing ? ds4_gpu_now_ms() : 0.0; uint64_t scan_entries = 0; - for (uint32_t layer = 0; - layer < DS4_METAL_STREAM_EXPERT_CACHE_MAX_LAYER; - layer++) { - for (uint32_t expert = 0; - expert < DS4_METAL_STREAM_EXPERT_CACHE_MAX_EXPERT; - expert++) { - scan_entries++; - ds4_gpu_stream_expert_cache_entry *e = - &g_stream_expert_cache[layer][expert]; - if (!ds4_gpu_stream_expert_cache_entry_reusable(e, - gate_expert_bytes, - down_expert_bytes)) { - continue; - } - if (ds4_gpu_stream_expert_cache_entry_inflight(e)) { - skipped_inflight = 1; - continue; - } - if (ds4_gpu_stream_expert_cache_entry_protected(layer, - expert, - protect_layer, - protect_ids, - n_protect)) { - continue; - } - const uint32_t hotness = - g_stream_expert_cache_route_hotness[layer][expert]; - if (hotness < lowest_hotness || - (hotness == lowest_hotness && e->last_used < oldest)) { - lowest_hotness = hotness; - oldest = e->last_used; - victim_layer = layer; - victim_expert = expert; - } + ds4_gpu_stream_expert_cache_iterator scan_it = + ds4_gpu_stream_expert_cache_iterator_begin(); + uint32_t layer; + uint32_t expert; + while (ds4_gpu_stream_expert_cache_iterator_next(&scan_it, + &layer, + &expert)) { + scan_entries++; + ds4_gpu_stream_expert_cache_entry *e = + &g_stream_expert_cache[layer][expert]; + if (!ds4_gpu_stream_expert_cache_entry_reusable(e, + gate_expert_bytes, + down_expert_bytes)) { + continue; + } + if (ds4_gpu_stream_expert_cache_entry_inflight(e)) { + skipped_inflight = 1; + continue; + } + if (ds4_gpu_stream_expert_cache_entry_protected(layer, + expert, + protect_layer, + protect_ids, + n_protect)) { + continue; + } + const uint32_t hotness = + g_stream_expert_cache_route_hotness[layer][expert]; + if (hotness < lowest_hotness || + (hotness == lowest_hotness && e->last_used < oldest)) { + lowest_hotness = hotness; + oldest = e->last_used; + victim_layer = layer; + victim_expert = expert; } } if (timing) { @@ -15148,61 +15585,61 @@ static uint32_t ds4_gpu_stream_expert_cache_take_reusable_batch( victim_last_used[i] = UINT64_MAX; } - for (uint32_t layer = 0; - layer < DS4_METAL_STREAM_EXPERT_CACHE_MAX_LAYER; - layer++) { - for (uint32_t expert = 0; - expert < DS4_METAL_STREAM_EXPERT_CACHE_MAX_EXPERT; - expert++) { - scan_entries++; - ds4_gpu_stream_expert_cache_entry *e = - &g_stream_expert_cache[layer][expert]; - if (!ds4_gpu_stream_expert_cache_entry_reusable(e, - gate_expert_bytes, - down_expert_bytes)) { - continue; - } - if (ds4_gpu_stream_expert_cache_entry_inflight(e)) { - skipped_inflight = 1; - continue; - } - if (ds4_gpu_stream_expert_cache_entry_protected(layer, - expert, - protect_layer, - protect_ids, - n_protect)) { - continue; - } + ds4_gpu_stream_expert_cache_iterator scan_it = + ds4_gpu_stream_expert_cache_iterator_begin(); + uint32_t layer; + uint32_t expert; + while (ds4_gpu_stream_expert_cache_iterator_next(&scan_it, + &layer, + &expert)) { + scan_entries++; + ds4_gpu_stream_expert_cache_entry *e = + &g_stream_expert_cache[layer][expert]; + if (!ds4_gpu_stream_expert_cache_entry_reusable(e, + gate_expert_bytes, + down_expert_bytes)) { + continue; + } + if (ds4_gpu_stream_expert_cache_entry_inflight(e)) { + skipped_inflight = 1; + continue; + } + if (ds4_gpu_stream_expert_cache_entry_protected(layer, + expert, + protect_layer, + protect_ids, + n_protect)) { + continue; + } - const uint32_t hotness = - g_stream_expert_cache_route_hotness[layer][expert]; - const uint64_t last_used = e->last_used; - if (victim_count < n_needed) { - victim_layers[victim_count] = layer; - victim_experts[victim_count] = expert; - victim_hotness[victim_count] = hotness; - victim_last_used[victim_count] = last_used; - victim_count++; - continue; - } + const uint32_t hotness = + g_stream_expert_cache_route_hotness[layer][expert]; + const uint64_t last_used = e->last_used; + if (victim_count < n_needed) { + victim_layers[victim_count] = layer; + victim_experts[victim_count] = expert; + victim_hotness[victim_count] = hotness; + victim_last_used[victim_count] = last_used; + victim_count++; + continue; + } - uint32_t worst = 0; - for (uint32_t i = 1; i < victim_count; i++) { - if (victim_hotness[i] > victim_hotness[worst] || - (victim_hotness[i] == victim_hotness[worst] && - victim_last_used[i] > victim_last_used[worst])) { - worst = i; - } - } - if (hotness < victim_hotness[worst] || - (hotness == victim_hotness[worst] && - last_used < victim_last_used[worst])) { - victim_layers[worst] = layer; - victim_experts[worst] = expert; - victim_hotness[worst] = hotness; - victim_last_used[worst] = last_used; + uint32_t worst = 0; + for (uint32_t i = 1; i < victim_count; i++) { + if (victim_hotness[i] > victim_hotness[worst] || + (victim_hotness[i] == victim_hotness[worst] && + victim_last_used[i] > victim_last_used[worst])) { + worst = i; } } + if (hotness < victim_hotness[worst] || + (hotness == victim_hotness[worst] && + last_used < victim_last_used[worst])) { + victim_layers[worst] = layer; + victim_experts[worst] = expert; + victim_hotness[worst] = hotness; + victim_last_used[worst] = last_used; + } } if (timing) { ds4_gpu_stream_expert_timing_note_reuse_scan(scan_entries, @@ -15296,36 +15733,36 @@ static uint32_t ds4_gpu_stream_expert_cache_release_mlock_margin( uint32_t lowest_hotness = UINT32_MAX; uint64_t oldest = UINT64_MAX; - for (uint32_t layer = 0; - layer < DS4_METAL_STREAM_EXPERT_CACHE_MAX_LAYER; - layer++) { - for (uint32_t expert = 0; - expert < DS4_METAL_STREAM_EXPERT_CACHE_MAX_EXPERT; - expert++) { - ds4_gpu_stream_expert_cache_entry *e = - &g_stream_expert_cache[layer][expert]; - if (!e->valid || - !e->slab_backed || - e->slab_slot >= DS4_METAL_STREAM_EXPERT_CACHE_MAX_ENTRIES || - !g_stream_expert_cache_slab_slot_locked[e->slab_slot] || - ds4_gpu_stream_expert_cache_entry_inflight(e) || - ds4_gpu_stream_expert_cache_entry_protected(layer, - expert, - protect_layer, - protect_ids, - n_protect)) { - continue; - } - const uint32_t hotness = - g_stream_expert_cache_route_hotness[layer][expert]; - if (hotness < lowest_hotness || - (hotness == lowest_hotness && e->last_used < oldest)) { - lowest_hotness = hotness; - oldest = e->last_used; - victim_layer = layer; - victim_expert = expert; - victim_slot = e->slab_slot; - } + ds4_gpu_stream_expert_cache_iterator scan_it = + ds4_gpu_stream_expert_cache_iterator_begin(); + uint32_t layer; + uint32_t expert; + while (ds4_gpu_stream_expert_cache_iterator_next(&scan_it, + &layer, + &expert)) { + ds4_gpu_stream_expert_cache_entry *e = + &g_stream_expert_cache[layer][expert]; + if (!e->valid || + !e->slab_backed || + e->slab_slot >= DS4_METAL_STREAM_EXPERT_CACHE_MAX_ENTRIES || + !g_stream_expert_cache_slab_slot_locked[e->slab_slot] || + ds4_gpu_stream_expert_cache_entry_inflight(e) || + ds4_gpu_stream_expert_cache_entry_protected(layer, + expert, + protect_layer, + protect_ids, + n_protect)) { + continue; + } + const uint32_t hotness = + g_stream_expert_cache_route_hotness[layer][expert]; + if (hotness < lowest_hotness || + (hotness == lowest_hotness && e->last_used < oldest)) { + lowest_hotness = hotness; + oldest = e->last_used; + victim_layer = layer; + victim_expert = expert; + victim_slot = e->slab_slot; } } @@ -15582,31 +16019,31 @@ static void ds4_gpu_stream_expert_cache_prune_global( uint32_t victim_expert = UINT32_MAX; uint32_t lowest_hotness = UINT32_MAX; uint64_t oldest = UINT64_MAX; - for (uint32_t layer = 0; - layer < DS4_METAL_STREAM_EXPERT_CACHE_MAX_LAYER; - layer++) { - for (uint32_t expert = 0; - expert < DS4_METAL_STREAM_EXPERT_CACHE_MAX_EXPERT; - expert++) { - ds4_gpu_stream_expert_cache_entry *e = - &g_stream_expert_cache[layer][expert]; - if (!e->valid || - ds4_gpu_stream_expert_cache_entry_protected(layer, - expert, - protect_layer, - protect_ids, - n_protect)) { - continue; - } - const uint32_t hotness = - g_stream_expert_cache_route_hotness[layer][expert]; - if (hotness < lowest_hotness || - (hotness == lowest_hotness && e->last_used < oldest)) { - lowest_hotness = hotness; - oldest = e->last_used; - victim_layer = layer; - victim_expert = expert; - } + ds4_gpu_stream_expert_cache_iterator scan_it = + ds4_gpu_stream_expert_cache_iterator_begin(); + uint32_t layer; + uint32_t expert; + while (ds4_gpu_stream_expert_cache_iterator_next(&scan_it, + &layer, + &expert)) { + ds4_gpu_stream_expert_cache_entry *e = + &g_stream_expert_cache[layer][expert]; + if (!e->valid || + ds4_gpu_stream_expert_cache_entry_protected(layer, + expert, + protect_layer, + protect_ids, + n_protect)) { + continue; + } + const uint32_t hotness = + g_stream_expert_cache_route_hotness[layer][expert]; + if (hotness < lowest_hotness || + (hotness == lowest_hotness && e->last_used < oldest)) { + lowest_hotness = hotness; + oldest = e->last_used; + victim_layer = layer; + victim_expert = expert; } } if (victim_layer == UINT32_MAX || victim_expert == UINT32_MAX) break; @@ -15774,6 +16211,7 @@ static int ds4_gpu_stream_expert_cache_entry_matches( } else { g_stream_expert_cache_bytes += logical_bytes; } + ds4_gpu_stream_expert_cache_live_insert(layer, expert); g_stream_expert_cache_misses++; g_stream_expert_cache_layer_misses[layer]++; g_stream_expert_cache_wraps += 3; diff --git a/tests/test_metal_iq2_live_index.c b/tests/test_metal_iq2_live_index.c new file mode 100644 index 000000000..2098a71a7 --- /dev/null +++ b/tests/test_metal_iq2_live_index.c @@ -0,0 +1,240 @@ +#define _DARWIN_C_SOURCE + +#include "ds4_gpu.h" + +#include +#include +#include +#include +#include +#include +#include + +#ifdef __APPLE__ + +enum { + N_TOTAL_EXPERT = 6, + N_SELECTED = 6, + CACHE_BUDGET = 3, +}; + +static const uint64_t GATE_EXPERT_BYTES = UINT64_C(2162688); +static const uint64_t DOWN_EXPERT_BYTES = UINT64_C(2752512); +static const uint64_t TOTAL_EXPERT_BYTES = UINT64_C(7077888); + +bool ds4_log_is_tty(FILE *fp) { + (void)fp; + return false; +} + +static int check_policy(void) { + int ok = 1; +#define CHECK_POLICY(expected, ssd, gate, down, enable, disable) do { \ + const int got = ds4_gpu_test_stream_expert_live_index_policy( \ + (ssd), (gate), (down), (enable), (disable)); \ + if (got != (expected)) { \ + fprintf(stderr, \ + "policy FAIL ssd=%d gate=%llu down=%llu enable=%d " \ + "disable=%d got=%d expected=%d\n", \ + (ssd), (unsigned long long)(gate), \ + (unsigned long long)(down), (enable), (disable), \ + got, (expected)); \ + ok = 0; \ + } \ +} while (0) + + CHECK_POLICY(1, 1, GATE_EXPERT_BYTES, DOWN_EXPERT_BYTES, -1, -1); + CHECK_POLICY(1, 1, GATE_EXPERT_BYTES, DOWN_EXPERT_BYTES, 1, 0); + CHECK_POLICY(0, 0, GATE_EXPERT_BYTES, DOWN_EXPERT_BYTES, 1, 0); + CHECK_POLICY(0, 1, GATE_EXPERT_BYTES - 1u, + DOWN_EXPERT_BYTES + 2u, 1, 0); + CHECK_POLICY(0, 1, GATE_EXPERT_BYTES, DOWN_EXPERT_BYTES, 0, 0); + CHECK_POLICY(0, 1, GATE_EXPERT_BYTES, DOWN_EXPERT_BYTES, 1, 1); +#undef CHECK_POLICY + return ok; +} + +static int seed_route(const ds4_gpu_stream_expert_table *table, + const int32_t route[N_SELECTED]) { + return ds4_gpu_stream_expert_cache_seed_selected(table, + route, + N_SELECTED); +} + +static int run_churn(const ds4_gpu_stream_expert_table *table) { + static const int32_t route_a[N_SELECTED] = {0, 1, 2, 0, 1, 2}; + static const int32_t route_b[N_SELECTED] = {3, 4, 5, 3, 4, 5}; + return seed_route(table, route_a) && + seed_route(table, route_b) && + seed_route(table, route_a); +} + +static void print_report( + const char *name, + const ds4_gpu_stream_expert_live_index_report *r) { + fprintf(stderr, + "%s scans=%llu entries=%llu fallbacks=%llu inserts=%llu " + "removes=%llu reuse_calls=%llu reuse_entries=%llu " + "live=%u cache=%u eligible=%u active=%u broken=%u hash=%016llx\n", + name, + (unsigned long long)r->scans, + (unsigned long long)r->entries, + (unsigned long long)r->fallbacks, + (unsigned long long)r->inserts, + (unsigned long long)r->removes, + (unsigned long long)r->reuse_scan_calls, + (unsigned long long)r->reuse_scan_entries, + r->live_count, + r->cache_entries, + r->eligible, + r->active, + r->broken, + (unsigned long long)r->resident_hash); +} + +int main(void) { + int ok = check_policy(); + int fd = -1; + void *model = MAP_FAILED; + char path[] = "/tmp/ds4-metal-iq2-live-index.XXXXXX"; + + if (2u * GATE_EXPERT_BYTES + DOWN_EXPERT_BYTES != + TOTAL_EXPERT_BYTES) { + fprintf(stderr, "production size constants FAIL\n"); + return 1; + } + + const uint64_t gate_tensor_bytes = + (uint64_t)N_TOTAL_EXPERT * GATE_EXPERT_BYTES; + const uint64_t down_tensor_bytes = + (uint64_t)N_TOTAL_EXPERT * DOWN_EXPERT_BYTES; + const uint64_t gate_offset = 0; + const uint64_t up_offset = gate_tensor_bytes; + const uint64_t down_offset = up_offset + gate_tensor_bytes; + const uint64_t model_size = down_offset + down_tensor_bytes; + + fd = mkstemp(path); + if (fd < 0 || ftruncate(fd, (off_t)model_size) != 0) { + perror("IQ2 live-index fixture"); + ok = 0; + goto cleanup; + } + model = mmap(NULL, (size_t)model_size, + PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + if (model == MAP_FAILED) { + perror("IQ2 live-index mmap"); + ok = 0; + goto cleanup; + } + + if (!ds4_gpu_init() || !ds4_gpu_set_model_map(model, model_size)) { + fprintf(stderr, "Metal initialization/model map FAIL\n"); + ok = 0; + goto cleanup; + } + ds4_gpu_set_ssd_streaming(true); + ds4_gpu_set_model_fd(fd); + ds4_gpu_set_streaming_expert_cache_expert_bytes(TOTAL_EXPERT_BYTES); + setenv("DS4_METAL_STREAMING_EXPERT_TIMING_SUMMARY", "1", 1); + + const ds4_gpu_stream_expert_table table = { + .model_map = model, + .model_size = model_size, + .layer = 0, + .n_total_expert = N_TOTAL_EXPERT, + .gate_offset = gate_offset, + .up_offset = up_offset, + .down_offset = down_offset, + .gate_expert_bytes = GATE_EXPERT_BYTES, + .down_expert_bytes = DOWN_EXPERT_BYTES, + }; + + setenv("DS4_METAL_DISABLE_STREAMING_EXPERT_LIVE_INDEX", "1", 1); + unsetenv("DS4_METAL_ENABLE_STREAMING_EXPERT_LIVE_INDEX"); + ds4_gpu_set_streaming_expert_cache_budget(CACHE_BUDGET); + ok = ok && run_churn(&table); + ds4_gpu_stream_expert_live_index_report control = {0}; + ds4_gpu_test_stream_expert_live_index_report(&control); + print_report("control", &control); + if (control.scans != 0 || control.entries != 0 || + control.cache_entries != CACHE_BUDGET || control.broken != 0 || + control.reuse_scan_calls == 0 || + control.reuse_scan_entries < 30720u) { + fprintf(stderr, "control coverage FAIL\n"); + ok = 0; + } + + unsetenv("DS4_METAL_DISABLE_STREAMING_EXPERT_LIVE_INDEX"); + unsetenv("DS4_METAL_ENABLE_STREAMING_EXPERT_LIVE_INDEX"); + ds4_gpu_set_streaming_expert_cache_budget(CACHE_BUDGET); + ok = ok && run_churn(&table); + ds4_gpu_stream_expert_live_index_report candidate = {0}; + ds4_gpu_test_stream_expert_live_index_report(&candidate); + print_report("candidate", &candidate); + if (candidate.scans == 0 || candidate.entries == 0 || + candidate.fallbacks != 0 || candidate.inserts < CACHE_BUDGET || + candidate.removes == 0 || + candidate.live_count != candidate.cache_entries || + candidate.cache_entries != CACHE_BUDGET || + candidate.eligible == 0 || candidate.active == 0 || + candidate.broken != 0 || + candidate.reuse_scan_calls == 0 || + candidate.reuse_scan_entries >= control.reuse_scan_entries || + candidate.entries > candidate.scans * CACHE_BUDGET || + candidate.resident_hash != control.resident_hash) { + fprintf(stderr, "candidate coverage/state FAIL\n"); + ok = 0; + } + + ds4_gpu_set_streaming_expert_cache_budget(CACHE_BUDGET); + static const int32_t route_a[N_SELECTED] = {0, 1, 2, 0, 1, 2}; + static const int32_t route_b[N_SELECTED] = {3, 4, 5, 3, 4, 5}; + ok = ok && seed_route(&table, route_a); + ds4_gpu_test_set_flags(DS4_GPU_TEST_STREAMING_LIVE_INDEX_FAILURE); + ok = ok && seed_route(&table, route_b); + ds4_gpu_test_set_flags(0); + ds4_gpu_stream_expert_live_index_report fault = {0}; + ds4_gpu_test_stream_expert_live_index_report(&fault); + print_report("fault-fallback", &fault); + if (fault.fallbacks == 0 || fault.broken == 0 || + fault.active != 0 || fault.cache_entries != CACHE_BUDGET) { + fprintf(stderr, "fault fallback FAIL\n"); + ok = 0; + } + + ds4_gpu_set_streaming_expert_cache_budget(CACHE_BUDGET); + ds4_gpu_stream_expert_live_index_report reset = {0}; + ds4_gpu_test_stream_expert_live_index_report(&reset); + print_report("post-reset", &reset); + if (reset.broken != 0 || reset.live_count != 0 || + reset.cache_entries != 0 || reset.fallbacks != 0) { + fprintf(stderr, "reset lifecycle FAIL\n"); + ok = 0; + } + +cleanup: + ds4_gpu_test_set_flags(0); + unsetenv("DS4_METAL_ENABLE_STREAMING_EXPERT_LIVE_INDEX"); + unsetenv("DS4_METAL_DISABLE_STREAMING_EXPERT_LIVE_INDEX"); + unsetenv("DS4_METAL_STREAMING_EXPERT_TIMING_SUMMARY"); + ds4_gpu_set_streaming_expert_cache_budget(0); + ds4_gpu_set_model_fd(-1); + ds4_gpu_set_ssd_streaming(false); + ds4_gpu_cleanup(); + if (model != MAP_FAILED) munmap(model, (size_t)model_size); + if (fd >= 0) close(fd); + unlink(path); + + fprintf(stderr, "IQ2 Metal streaming expert live-index %s\n", + ok ? "PASS" : "FAIL"); + return ok ? 0 : 1; +} + +#else + +int main(void) { + fprintf(stderr, "test_metal_iq2_live_index: SKIP (requires Apple Metal)\n"); + return 0; +} + +#endif From 801be18bfc9369c926143fd002926553313bf482 Mon Sep 17 00:00:00 2001 From: Giorgio Oppo Date: Fri, 14 Aug 2026 14:35:24 +0200 Subject: [PATCH 025/189] metal: pack routed prefill maps without losing duplicates --- ds4_metal.m | 156 +++++++++++++++++++++----------------- metal/moe.metal | 198 +++++++++++++++++++++++++++++++++++------------- 2 files changed, 230 insertions(+), 124 deletions(-) diff --git a/ds4_metal.m b/ds4_metal.m index e8dd88e52..d7614a4ea 100644 --- a/ds4_metal.m +++ b/ds4_metal.m @@ -34026,6 +34026,60 @@ static int ds4_gpu_encode_mul_mm_id( dst_off); } +typedef struct ds4_gpu_mul_mm_id_map_layout { + NSUInteger tpe_bytes; + NSUInteger hids_bytes; + NSUInteger work_offset; + NSUInteger total_bytes; + uint64_t work_cap; +} ds4_gpu_mul_mm_id_map_layout; + +/* The map stores uint2(count, packed-route-base) for every model expert, then + * exactly ne20 * ne21 token/slot route IDs. Packed ranges are required for + * duplicate expert IDs: a fixed ne21 slice cannot represent the same expert + * appearing in more than one top-k slot of a token. */ +static int ds4_gpu_make_mul_mm_id_map_layout( + const ds4_gpu_mul_mm_id_args *args, + ds4_gpu_mul_mm_id_map_layout *layout) { + if (!args || !layout || + args->ne02 <= 0 || args->ne20 <= 0 || args->ne21 <= 0) { + return 0; + } + + const uint64_t ne02 = (uint32_t)args->ne02; + const uint64_t pair_rows = + (uint64_t)(uint32_t)args->ne20 * (uint32_t)args->ne21; + if (ne02 > (uint64_t)NSUIntegerMax / (2u * sizeof(uint32_t)) || + pair_rows > (uint64_t)NSUIntegerMax / sizeof(int32_t) || + ne02 > (UINT64_MAX - pair_rows - 31u) / 31u) { + return 0; + } + + const NSUInteger tpe_bytes = + (NSUInteger)ne02 * 2u * sizeof(uint32_t); + const NSUInteger hids_bytes = + (NSUInteger)pair_rows * sizeof(int32_t); + if (tpe_bytes > NSUIntegerMax - hids_bytes - 7u) return 0; + const NSUInteger work_offset = (tpe_bytes + hids_bytes + 7u) & ~7u; + const uint64_t work_cap = (pair_rows + 31u * ne02 + 31u) / 32u; + const NSUInteger work_item_bytes = 2u * sizeof(uint32_t); + if (work_cap > (uint64_t)NSUIntegerMax || + (NSUInteger)work_cap > + (NSUIntegerMax - work_offset - 8u) / work_item_bytes) { + return 0; + } + + *layout = (ds4_gpu_mul_mm_id_map_layout) { + .tpe_bytes = tpe_bytes, + .hids_bytes = hids_bytes, + .work_offset = work_offset, + .total_bytes = work_offset + 8u + + (NSUInteger)work_cap * work_item_bytes, + .work_cap = work_cap, + }; + return 1; +} + static int ds4_gpu_encode_mul_mm_id_map( id cb, id map_pipeline, @@ -34038,26 +34092,11 @@ static int ds4_gpu_encode_mul_mm_id_map( return 0; } - const NSUInteger tpe_bytes = (NSUInteger)mm_args->ne02 * sizeof(int32_t); - const NSUInteger hids_bytes = - (NSUInteger)mm_args->ne02 * (NSUInteger)mm_args->ne21 * sizeof(int32_t); - if (tpe_bytes > NSUIntegerMax - hids_bytes) return 0; - const NSUInteger work_offset = (tpe_bytes + hids_bytes + 7u) & ~7u; - const uint64_t pair_rows = - (uint64_t)(uint32_t)mm_args->ne20 * (uint32_t)mm_args->ne21; - const uint64_t work_cap = - (pair_rows + 31u * (uint32_t)mm_args->ne02 + 31u) / 32u; - const NSUInteger work_item_bytes = 2u * sizeof(uint32_t); - if (work_cap > (NSUIntegerMax - 8u) / work_item_bytes || - work_offset > NSUIntegerMax - 8u - - (NSUInteger)work_cap * work_item_bytes) { - return 0; - } - const NSUInteger total_bytes = - work_offset + 8u + (NSUInteger)work_cap * work_item_bytes; + ds4_gpu_mul_mm_id_map_layout layout; + if (!ds4_gpu_make_mul_mm_id_map_layout(mm_args, &layout)) return 0; if (!ds4_gpu_ensure_scratch_buffer(&g_moe_id_map_buffer, &g_moe_id_map_bytes, - total_bytes, + layout.total_bytes, "ds4_moe_id_map")) { return 0; } @@ -34067,9 +34106,16 @@ static int ds4_gpu_encode_mul_mm_id_map( [enc setBytes:map_args length:sizeof(*map_args) atIndex:0]; [enc setBuffer:ids offset:ids_off atIndex:1]; [enc setBuffer:g_moe_id_map_buffer offset:0 atIndex:2]; - [enc setBuffer:g_moe_id_map_buffer offset:tpe_bytes atIndex:3]; - [enc setBuffer:g_moe_id_map_buffer offset:work_offset atIndex:4]; - [enc setThreadgroupMemoryLength:(NSUInteger)mm_args->ne02 * (NSUInteger)mm_args->ne20 * sizeof(uint16_t) atIndex:0]; + [enc setBuffer:g_moe_id_map_buffer offset:layout.tpe_bytes atIndex:3]; + [enc setBuffer:g_moe_id_map_buffer offset:layout.work_offset atIndex:4]; + const NSUInteger staging_bytes = + (NSUInteger)mm_args->ne02 * (NSUInteger)mm_args->ne20 * + sizeof(uint16_t); + const NSUInteger scatter_bytes = + (NSUInteger)mm_args->ne02 * 2u * sizeof(uint32_t); + [enc setThreadgroupMemoryLength: + staging_bytes > scatter_bytes ? staging_bytes : scatter_bytes + atIndex:0]; [enc dispatchThreadgroups:MTLSizeMake(1, 1, 1) threadsPerThreadgroup:MTLSizeMake((NSUInteger)mm_args->ne02, 1, 1)]; ds4_gpu_end_compute_encoder(cb, enc); @@ -34102,23 +34148,9 @@ static int ds4_gpu_encode_mul_mm_id_mapped_tile( getenv("DS4_METAL_MOE_MM_ID_USE_RESOURCES") != NULL && getenv("DS4_METAL_DISABLE_MOE_MM_ID_USE_RESOURCES") == NULL; - const NSUInteger tpe_bytes = (NSUInteger)mm_args->ne02 * sizeof(int32_t); - const NSUInteger hids_bytes = (NSUInteger)mm_args->ne02 * (NSUInteger)mm_args->ne21 * sizeof(int32_t); - if (tpe_bytes > NSUIntegerMax - hids_bytes) { - return 0; - } - const NSUInteger work_offset = (tpe_bytes + hids_bytes + 7u) & ~7u; - const uint64_t pair_rows = - (uint64_t)(uint32_t)mm_args->ne20 * (uint32_t)mm_args->ne21; - const uint64_t work_cap = - (pair_rows + 31u * (uint32_t)mm_args->ne02 + 31u) / 32u; - const NSUInteger work_item_bytes = 2u * sizeof(uint32_t); - if (work_cap > NSUIntegerMax || - work_offset > NSUIntegerMax - 8u || - (NSUInteger)work_cap > - (NSUIntegerMax - work_offset - 8u) / work_item_bytes || - g_moe_id_map_bytes < - work_offset + 8u + (NSUInteger)work_cap * work_item_bytes) { + ds4_gpu_mul_mm_id_map_layout layout; + if (!ds4_gpu_make_mul_mm_id_map_layout(mm_args, &layout) || + g_moe_id_map_bytes < layout.total_bytes) { return 0; } @@ -34128,14 +34160,14 @@ static int ds4_gpu_encode_mul_mm_id_mapped_tile( [enc setBuffer:src0 offset:src0_off atIndex:1]; [enc setBuffer:src1 offset:src1_off atIndex:2]; [enc setBuffer:g_moe_id_map_buffer offset:0 atIndex:3]; - [enc setBuffer:g_moe_id_map_buffer offset:tpe_bytes atIndex:4]; + [enc setBuffer:g_moe_id_map_buffer offset:layout.tpe_bytes atIndex:4]; [enc setBuffer:dst offset:dst_off atIndex:5]; - [enc setBuffer:g_moe_id_map_buffer offset:work_offset atIndex:6]; + [enc setBuffer:g_moe_id_map_buffer offset:layout.work_offset atIndex:6]; if (use_resource_hints) { [enc useResource:src0 usage:MTLResourceUsageRead]; } [enc setThreadgroupMemoryLength:threadgroup_bytes atIndex:0]; - [enc dispatchThreadgroups:MTLSizeMake((NSUInteger)work_cap, + [enc dispatchThreadgroups:MTLSizeMake((NSUInteger)layout.work_cap, ((NSUInteger)mm_args->ne0 + 63u) / 64u, 1) threadsPerThreadgroup:MTLSizeMake(128, 1, 1)]; @@ -34164,12 +34196,9 @@ static int ds4_gpu_encode_mul_mm_id_addr_mapped_tile( return 0; } - const NSUInteger tile_n = 32u; - const NSUInteger tpe_bytes = (NSUInteger)mm_args->ne02 * sizeof(int32_t); - const NSUInteger hids_bytes = - (NSUInteger)mm_args->ne02 * (NSUInteger)mm_args->ne21 * sizeof(int32_t); - if (tpe_bytes > NSUIntegerMax - hids_bytes || - g_moe_id_map_bytes < tpe_bytes + hids_bytes) { + ds4_gpu_mul_mm_id_map_layout layout; + if (!ds4_gpu_make_mul_mm_id_map_layout(mm_args, &layout) || + g_moe_id_map_bytes < layout.total_bytes) { return 0; } @@ -34179,8 +34208,9 @@ static int ds4_gpu_encode_mul_mm_id_addr_mapped_tile( [enc setBuffer:src0_addrs offset:0 atIndex:1]; [enc setBuffer:src1 offset:src1_off atIndex:2]; [enc setBuffer:g_moe_id_map_buffer offset:0 atIndex:3]; - [enc setBuffer:g_moe_id_map_buffer offset:tpe_bytes atIndex:4]; + [enc setBuffer:g_moe_id_map_buffer offset:layout.tpe_bytes atIndex:4]; [enc setBuffer:dst offset:dst_off atIndex:5]; + [enc setBuffer:g_moe_id_map_buffer offset:layout.work_offset atIndex:6]; [enc useResource:src0_addrs usage:MTLResourceUsageRead]; for (uint32_t i = 0; resources && i < resource_count; i++) { ds4_gpu_stream_expert_cache_entry *entry = resources[i]; @@ -34195,9 +34225,9 @@ static int ds4_gpu_encode_mul_mm_id_addr_mapped_tile( [enc useResource:overflow_resource usage:MTLResourceUsageRead]; } [enc setThreadgroupMemoryLength:threadgroup_bytes atIndex:0]; - [enc dispatchThreadgroups:MTLSizeMake(((NSUInteger)mm_args->ne21 + tile_n - 1u) / tile_n, + [enc dispatchThreadgroups:MTLSizeMake((NSUInteger)layout.work_cap, ((NSUInteger)mm_args->ne0 + 63u) / 64u, - (NSUInteger)mm_args->ne02) + 1) threadsPerThreadgroup:MTLSizeMake(128, 1, 1)]; ds4_gpu_end_compute_encoder(cb, enc); return 1; @@ -34227,23 +34257,9 @@ static int ds4_gpu_encode_mul_mm_id_iq2_pair_swiglu_f16( return 0; } - const NSUInteger tpe_bytes = (NSUInteger)mm_args->ne02 * sizeof(int32_t); - const NSUInteger hids_bytes = (NSUInteger)mm_args->ne02 * (NSUInteger)mm_args->ne21 * sizeof(int32_t); - if (tpe_bytes > NSUIntegerMax - hids_bytes) { - return 0; - } - const NSUInteger work_offset = (tpe_bytes + hids_bytes + 7u) & ~7u; - const uint64_t pair_rows = - (uint64_t)(uint32_t)mm_args->ne20 * (uint32_t)mm_args->ne21; - const uint64_t work_cap = - (pair_rows + 31u * (uint32_t)mm_args->ne02 + 31u) / 32u; - const NSUInteger work_item_bytes = 2u * sizeof(uint32_t); - if (work_cap > NSUIntegerMax || - work_offset > NSUIntegerMax - 8u || - (NSUInteger)work_cap > - (NSUIntegerMax - work_offset - 8u) / work_item_bytes || - g_moe_id_map_bytes < - work_offset + 8u + (NSUInteger)work_cap * work_item_bytes) { + ds4_gpu_mul_mm_id_map_layout layout; + if (!ds4_gpu_make_mul_mm_id_map_layout(mm_args, &layout) || + g_moe_id_map_bytes < layout.total_bytes) { return 0; } @@ -34255,13 +34271,13 @@ static int ds4_gpu_encode_mul_mm_id_iq2_pair_swiglu_f16( [enc setBuffer:up_src0 offset:up_src0_off atIndex:3]; [enc setBuffer:src1 offset:src1_off atIndex:4]; [enc setBuffer:g_moe_id_map_buffer offset:0 atIndex:5]; - [enc setBuffer:g_moe_id_map_buffer offset:tpe_bytes atIndex:6]; + [enc setBuffer:g_moe_id_map_buffer offset:layout.tpe_bytes atIndex:6]; [enc setBuffer:mid offset:mid_off atIndex:7]; [enc setBuffer:weights offset:weights_off atIndex:8]; - [enc setBuffer:g_moe_id_map_buffer offset:work_offset atIndex:9]; + [enc setBuffer:g_moe_id_map_buffer offset:layout.work_offset atIndex:9]; const NSUInteger tile_m = compact_tile ? 32u : 64u; [enc setThreadgroupMemoryLength:compact_tile ? 8192u : 16384u atIndex:0]; - [enc dispatchThreadgroups:MTLSizeMake((NSUInteger)work_cap, + [enc dispatchThreadgroups:MTLSizeMake((NSUInteger)layout.work_cap, ((NSUInteger)mm_args->ne0 + tile_m - 1u) / tile_m, 1) threadsPerThreadgroup:MTLSizeMake(compact_tile ? 64u : 128u, 1, 1)]; diff --git a/metal/moe.metal b/metal/moe.metal index eac36e3ec..b578f066d 100644 --- a/metal/moe.metal +++ b/metal/moe.metal @@ -8188,8 +8188,7 @@ kernel void kernel_mul_mm_id_map0( const short ide = tpitg; uint32_t n_all = 0; - - device int32_t * ids_i32 = (device int32_t *) hids + ide*args.ne21; + device int32_t * ids_i32 = (device int32_t *) hids; for (int i21 = 0; i21 < args.ne21; i21 += ntg) { if (i21 + tpitg < args.ne21) { @@ -8212,36 +8211,81 @@ kernel void kernel_mul_mm_id_map0( threadgroup const uint16_t * sids = (threadgroup const uint16_t *) shmem + t*ne20; - short sel = 0; + uint32_t matches = 0; #pragma unroll(ne20) for (short i20 = 0; i20 < ne20; i20++) { - sel += (sids[i20] == ide)*(i20 + 1); + matches += sids[i20] == ide; } - - ids_i32[n_all] = (i21 + t)*ne20 + sel - 1; - - n_all += sel > 0; + n_all += matches; } threadgroup_barrier(mem_flags::mem_threadgroup); } + /* Store a packed route list, not a fixed n_tokens slice per expert. A + * malformed/synthetic top-k list may select the same expert in multiple + * slots of one token. The historical `sel += slot + 1` collapsed those + * routes and could even point at a different slot. Counting every match + * and assigning prefix-sum ranges preserves the original (token, slot) + * identity while keeping total storage bounded by ne21 * ne20. */ + threadgroup uint32_t * route_counts = + (threadgroup uint32_t *) shmem; + route_counts[ide] = n_all; + threadgroup_barrier(mem_flags::mem_threadgroup); + + uint32_t route_base = 0; + for (ushort i = 0; i < ide; i++) { + route_base += route_counts[i]; + } + uint32_t tile_base = 0; + for (ushort i = 0; i < ide; i++) { + tile_base += (route_counts[i] + 31u) / 32u; + } + device uint32_t * tpe_u32 = (device uint32_t *) (htpe); - tpe_u32[ide] = n_all; + tpe_u32[2u*ide + 0u] = n_all; + tpe_u32[2u*ide + 1u] = route_base; + threadgroup_barrier(mem_flags::mem_threadgroup); + + uint32_t route = 0; + for (int i21 = 0; i21 < args.ne21; i21 += ntg) { + if (i21 + tpitg < args.ne21) { + device const int32_t * src2_i32 = + (device const int32_t *)(src2 + + (i21 + tpitg)*args.nb21); + threadgroup uint16_t * sids = + (threadgroup uint16_t *) shmem + tpitg*ne20; + + #pragma unroll(ne20) + for (short i20 = 0; i20 < ne20; i20++) { + sids[i20] = src2_i32[i20]; + } + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + + for (short t = 0; t < ntg; t++) { + if (i21 + t >= args.ne21) break; + threadgroup const uint16_t * sids = + (threadgroup const uint16_t *) shmem + t*ne20; + + #pragma unroll(ne20) + for (short i20 = 0; i20 < ne20; i20++) { + if (sids[i20] == ide) { + ids_i32[route_base + route++] = + (i21 + t)*ne20 + i20; + } + } + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + } // Reuse the route-id staging memory after the map is complete to build a // compact list of non-empty 32-row matmul tiles. The old dispatch covered // every possible token tile for every expert, even though most experts // receive only a small fraction of the prompt rows. - threadgroup uint16_t * tile_counts = (threadgroup uint16_t *) shmem; const uint16_t n_tiles = (uint16_t)((n_all + 31u) / 32u); - tile_counts[ide] = n_tiles; - threadgroup_barrier(mem_flags::mem_threadgroup); - - uint32_t tile_base = 0; - for (ushort i = 0; i < ide; i++) { - tile_base += tile_counts[i]; - } device uint32_t * work_count = (device uint32_t *) work; device uint2 * work_items = (device uint2 *)(work + 8); @@ -8285,8 +8329,8 @@ kernel void kernel_mul_mm_id_map_scatter_work( ushort ntg[[threads_per_threadgroup]]) { threadgroup atomic_uint * counts = (threadgroup atomic_uint *) shmem; - threadgroup uint16_t * tile_counts = - (threadgroup uint16_t *)(shmem + + threadgroup uint32_t * route_bases = + (threadgroup uint32_t *)(shmem + (uint32_t)args.ne02*sizeof(uint32_t)); device uint32_t * tpe_u32 = (device uint32_t *) htpe; device int32_t * ids_i32 = (device int32_t *) hids; @@ -8307,32 +8351,56 @@ kernel void kernel_mul_mm_id_map_scatter_work( continue; } - const uint32_t row = atomic_fetch_add_explicit( - counts + expert, 1u, memory_order_relaxed); - // Production top-k selections are unique. Keep malformed or - // synthetic duplicates from exceeding the fixed expert slice. - if (row < (uint32_t)args.ne21) { - ids_i32[(uint32_t)expert*args.ne21 + row] = - i21*ne20 + i20; - } + atomic_fetch_add_explicit(counts + expert, + 1u, + memory_order_relaxed); } } threadgroup_barrier(mem_flags::mem_threadgroup); const short ide = tpitg; - const uint32_t n_all = min( - atomic_load_explicit(counts + ide, memory_order_relaxed), - (uint32_t)args.ne21); - tpe_u32[ide] = n_all; + const uint32_t n_all = + atomic_load_explicit(counts + ide, memory_order_relaxed); + uint32_t route_base = 0; + for (ushort i = 0; i < ide; i++) { + route_base += atomic_load_explicit(counts + i, + memory_order_relaxed); + } + tpe_u32[2u*ide + 0u] = n_all; + tpe_u32[2u*ide + 1u] = route_base; + route_bases[ide] = route_base; + threadgroup_barrier(mem_flags::mem_threadgroup); - const uint16_t n_tiles = (uint16_t)((n_all + 31u) / 32u); - tile_counts[ide] = n_tiles; + /* Reuse the counters as per-expert write cursors for a second, stable-ABI + * scatter. The packed ranges have room for every route, including + * repeated expert IDs in different top-k slots. */ + atomic_store_explicit(counts + ide, 0u, memory_order_relaxed); threadgroup_barrier(mem_flags::mem_threadgroup); + for (int i21 = tpitg; i21 < args.ne21; i21 += ntg) { + device const int32_t * src2_i32 = + (device const int32_t *)(src2 + i21*args.nb21); + + #pragma unroll(ne20) + for (short i20 = 0; i20 < ne20; i20++) { + const int32_t expert = src2_i32[i20]; + if ((uint32_t)expert >= (uint32_t)args.ne02) continue; + const uint32_t row = atomic_fetch_add_explicit( + counts + expert, 1u, memory_order_relaxed); + const uint32_t base = route_bases[(uint32_t)expert]; + ids_i32[base + row] = i21*ne20 + i20; + } + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + + const uint16_t n_tiles = (uint16_t)((n_all + 31u) / 32u); uint32_t tile_base = 0; for (ushort i = 0; i < ide; i++) { - tile_base += tile_counts[i]; + const uint32_t count_i = + atomic_load_explicit(counts + i, memory_order_relaxed); + tile_base += (count_i + 31u) / 32u; } device uint32_t * work_count = (device uint32_t *) work; @@ -8396,7 +8464,8 @@ kernel void kernel_mul_mm_id( device const uint32_t * tpe_u32 = (device const uint32_t *) (htpe); device const int32_t * ids_i32 = (device const int32_t *) (hids); - const int32_t neh1 = tpe_u32[im]; + const int32_t neh1 = tpe_u32[2u*(uint)im + 0u]; + const uint32_t route_base = tpe_u32[2u*(uint)im + 1u]; if (r1 >= neh1) { return; @@ -8415,7 +8484,7 @@ kernel void kernel_mul_mm_id( * the downstream swiglu/sum stages stay unchanged. Each (token,slot) * row belongs to exactly one expert, so nothing else writes them. */ for (short j = sgitg; j < nr1; j += 4) { - const int idj = ids_i32[im*args.ne21 + r1 + j]; + const int idj = ids_i32[route_base + r1 + j]; const short ide = idj % args.ne20; const short idt = idj / args.ne20; @@ -8436,7 +8505,7 @@ kernel void kernel_mul_mm_id( short il = il0; - const int id = ids_i32[im*args.ne21 + r1 + lr1]; + const int id = ids_i32[route_base + r1 + lr1]; const short i11 = (id % args.ne20) % args.ne11; const short i12 = (id / args.ne20); @@ -8570,7 +8639,7 @@ kernel void kernel_mul_mm_id( threadgroup_barrier(mem_flags::mem_threadgroup); for (short j = sgitg; j < nr1; j += 4) { - const int idj = ids_i32[im*args.ne21 + r1 + j]; + const int idj = ids_i32[route_base + r1 + j]; const short ide = idj % args.ne20; const short idt = idj / args.ne20; @@ -8604,6 +8673,7 @@ kernel void kernel_mul_mm_id_addr( device const char * htpe, device const char * hids, device char * dst, + device const char * work, threadgroup char * shmem [[threadgroup(0)]], uint3 tgpig[[threadgroup_position_in_grid]], ushort tiitg[[thread_index_in_threadgroup]], @@ -8620,14 +8690,31 @@ kernel void kernel_mul_mm_id_addr( threadgroup S0 * sa = (threadgroup S0 *)(shmem); threadgroup S1 * sb = (threadgroup S1 *)(shmem + SA_BYTES); - const int im = tgpig.z; + // The SSD address table can contain hundreds of experts while a prompt + // routes only top-k rows per token. Consume the compact non-empty tile + // list emitted by kernel_mul_mm_id_map0 instead of launching the full + // expert x token-tile Cartesian grid. + device const uint32_t * work_count = + (device const uint32_t *)work; + const uint32_t work_index = tgpig.x; + if (work_index >= work_count[0]) { + return; + } + device const uint2 * work_items = + (device const uint2 *)(work + 8); + const uint2 item = work_items[work_index]; + const int im = (int)item.x; + if ((uint)im >= (uint)args.ne02) { + return; + } const int r0 = tgpig.y*NR0; - const int r1 = tgpig.x*NR1; + const int r1 = (int)item.y; device const uint32_t * tpe_u32 = (device const uint32_t *) (htpe); device const int32_t * ids_i32 = (device const int32_t *) (hids); - const int32_t neh1 = tpe_u32[im]; + const int32_t neh1 = tpe_u32[2u*(uint)im + 0u]; + const uint32_t route_base = tpe_u32[2u*(uint)im + 1u]; if (r1 >= neh1) { return; @@ -8649,7 +8736,7 @@ kernel void kernel_mul_mm_id_addr( short il = il0; - const int id = ids_i32[im*args.ne21 + r1 + lr1]; + const int id = ids_i32[route_base + r1 + lr1]; const short i11 = (id % args.ne20) % args.ne11; const short i12 = (id / args.ne20); @@ -8778,7 +8865,7 @@ kernel void kernel_mul_mm_id_addr( threadgroup_barrier(mem_flags::mem_threadgroup); for (short j = sgitg; j < nr1; j += 4) { - const int idj = ids_i32[im*args.ne21 + r1 + j]; + const int idj = ids_i32[route_base + r1 + j]; const short ide = idj % args.ne20; const short idt = idj / args.ne20; @@ -8847,7 +8934,8 @@ kernel void kernel_mul_mm_id_pair_swiglu_f16_impl( device const uint32_t * tpe_u32 = (device const uint32_t *) (htpe); device const int32_t * ids_i32 = (device const int32_t *) (hids); - const int32_t neh1 = tpe_u32[im]; + const int32_t neh1 = tpe_u32[2u*(uint)im + 0u]; + const uint32_t route_base = tpe_u32[2u*(uint)im + 1u]; if (r1 >= neh1) { return; @@ -8867,7 +8955,7 @@ kernel void kernel_mul_mm_id_pair_swiglu_f16_impl( const short il0 = (tiitg % NL0); short il = il0; - const int id = ids_i32[im*args.ne21 + r1 + lr1]; + const int id = ids_i32[route_base + r1 + lr1]; const short i11 = (id % args.ne20) % args.ne11; const short i12 = (id / args.ne20); @@ -8987,7 +9075,7 @@ kernel void kernel_mul_mm_id_pair_swiglu_f16_impl( const float c = act.clamp_value; for (short j = sgitg; j < nr1; j += 4) { - const int idj = ids_i32[im*args.ne21 + r1 + j]; + const int idj = ids_i32[route_base + r1 + j]; const short ide = idj % args.ne20; const short idt = idj / args.ne20; @@ -9062,7 +9150,8 @@ kernel void kernel_mul_mm_id_pair_swiglu_f16_compact_tail_impl( device const uint32_t * tpe_u32 = (device const uint32_t *) (htpe); device const int32_t * ids_i32 = (device const int32_t *) (hids); - const int32_t neh1 = tpe_u32[im]; + const int32_t neh1 = tpe_u32[2u*(uint)im + 0u]; + const uint32_t route_base = tpe_u32[2u*(uint)im + 1u]; if (r1 >= neh1) { return; } @@ -9080,8 +9169,8 @@ kernel void kernel_mul_mm_id_pair_swiglu_f16_compact_tail_impl( const short il0 = (tiitg % NL0); short il = il0; - const int id_b0 = ids_i32[im*args.ne21 + r1 + lr1_b0]; - const int id_b1 = ids_i32[im*args.ne21 + r1 + lr1_b1]; + const int id_b0 = ids_i32[route_base + r1 + lr1_b0]; + const int id_b1 = ids_i32[route_base + r1 + lr1_b1]; const short i11_b0 = (id_b0 % args.ne20) % args.ne11; const short i12_b0 = (id_b0 / args.ne20); @@ -9211,7 +9300,7 @@ kernel void kernel_mul_mm_id_pair_swiglu_f16_compact_tail_impl( const float c = act.clamp_value; for (short j = sgitg; j < nr1; j += 2) { - const int idj = ids_i32[im*args.ne21 + r1 + j]; + const int idj = ids_i32[route_base + r1 + j]; const short ide = idj % args.ne20; const short idt = idj / args.ne20; @@ -9457,7 +9546,8 @@ kernel void kernel_mul_mm_id_mpp( device const uint32_t * tpe_u32 = (device const uint32_t *) (htpe); device const int32_t * ids_i32 = (device const int32_t *) (hids); - const int32_t neh1 = tpe_u32[im]; + const int32_t neh1 = tpe_u32[2u*(uint)im + 0u]; + const uint32_t route_base = tpe_u32[2u*(uint)im + 1u]; if (r1 >= neh1) { return; @@ -9468,7 +9558,7 @@ kernel void kernel_mul_mm_id_mpp( if (!ds4_tp_owns_expert(im, args.ne02, args.tp_rank, args.tp_world)) { for (short j = sgitg; j < nr1; j += 4) { - const int idj = ids_i32[im*args.ne21 + r1 + j]; + const int idj = ids_i32[route_base + r1 + j]; const short ide = idj % args.ne20; const short idt = idj / args.ne20; device float *D = (device float *)dst + r0 + ide*args.ne0 + @@ -9484,7 +9574,7 @@ kernel void kernel_mul_mm_id_mpp( const short il0 = (tiitg % NL0); short il = il0; - const int id = ids_i32[im*args.ne21 + r1 + lr1]; + const int id = ids_i32[route_base + r1 + lr1]; const short i11 = (id % args.ne20) % args.ne11; const short i12 = (id / args.ne20); @@ -9591,7 +9681,7 @@ kernel void kernel_mul_mm_id_mpp( threadgroup_barrier(mem_flags::mem_threadgroup); for (short j = tiitg/32; j < nr1; j += 4) { - const int idj = ids_i32[im*args.ne21 + r1 + j]; + const int idj = ids_i32[route_base + r1 + j]; const short ide = idj % args.ne20; const short idt = idj / args.ne20; From 753b82f6eccecb05303008f0b5c196b4f0ce4cdf Mon Sep 17 00:00:00 2001 From: Giorgio Oppo Date: Fri, 14 Aug 2026 14:46:53 +0200 Subject: [PATCH 026/189] metal: group IQ2 SSD prefill address matmuls --- Makefile | 13 +- ds4_metal.m | 239 ++++- metal/moe.metal | 1 + tests/test_metal_iq2_ssd_grouped_mm.c | 1414 +++++++++++++++++++++++++ 4 files changed, 1654 insertions(+), 13 deletions(-) create mode 100644 tests/test_metal_iq2_ssd_grouped_mm.c diff --git a/Makefile b/Makefile index 67c399ffa..e12967758 100644 --- a/Makefile +++ b/Makefile @@ -68,7 +68,7 @@ DS4_LINK_LIBS ?= $(CUDA_LDLIBS) METAL_LDLIBS := $(LDLIBS) endif -.PHONY: all help clean test test-rocm test-glm53-kda-rocm test-metal-session-batch test-metal-exactn-oracle test-metal-dspark-capture test-metal-iq2-midonly test-metal-iq2-live-index test-mxfp4-cuda test-mxfp4-rocm test-mmq-parity-cuda test-cuda-session-batch test-cuda-mixed-batch dspark-acceptance dspark-verify-depth rocm-dspark-acceptance rocm-dspark-verify-depth mtp-verify-depth cpu cuda cuda-spark cuda-generic cuda-regression strix-halo rocm +.PHONY: all help clean test test-rocm test-glm53-kda-rocm test-metal-session-batch test-metal-exactn-oracle test-metal-dspark-capture test-metal-iq2-midonly test-metal-iq2-ssd-grouped-mm test-metal-iq2-live-index test-mxfp4-cuda test-mxfp4-rocm test-mmq-parity-cuda test-cuda-session-batch test-cuda-mixed-batch dspark-acceptance dspark-verify-depth rocm-dspark-acceptance rocm-dspark-verify-depth mtp-verify-depth cpu cuda cuda-spark cuda-generic cuda-regression strix-halo rocm ifeq ($(UNAME_S),Darwin) .PHONY: metal-decode-schedule-bench metal-prefill-variant-bench check-mxfp4-half-lut test-mxfp4-metal @@ -137,6 +137,15 @@ tests/test_metal_iq2_midonly: tests/test_metal_iq2_midonly.o ds4_metal.o test-metal-iq2-midonly: tests/test_metal_iq2_midonly ./tests/test_metal_iq2_midonly +tests/test_metal_iq2_ssd_grouped_mm.o: tests/test_metal_iq2_ssd_grouped_mm.c ds4_gpu.h + $(CC) $(CFLAGS) -I. -c -o $@ $< + +tests/test_metal_iq2_ssd_grouped_mm: tests/test_metal_iq2_ssd_grouped_mm.o ds4_metal.o + $(CC) $(CFLAGS) -o $@ $^ $(METAL_LDLIBS) + +test-metal-iq2-ssd-grouped-mm: tests/test_metal_iq2_ssd_grouped_mm + ./tests/test_metal_iq2_ssd_grouped_mm + tests/test_metal_iq2_live_index.o: tests/test_metal_iq2_live_index.c ds4_gpu.h $(CC) $(CFLAGS) -I. -c -o $@ $< @@ -683,4 +692,4 @@ mxfp4-dot-test: tests/test_mxfp4_dot.c ./tests/test_mxfp4_dot clean: - rm -f ds4 ds4-server ds4-bench ds4-eval ds4-agent ds4_cpu ds4_native ds4_server_test ds4_test ds4_agent_test gguf-tools/quality-testing/score_official gguf-tools/quality-testing/score_official.o speed-bench/metal_decode_schedule_bench speed-bench/metal_prefill_variant_bench speed-bench/*.o tests/test_q4k_dot tests/test_mxfp4_dot tests/test_mxfp4_metal tests/test_mxfp4_rocm tests/test_mxfp4_cuda tests/test_metal_session_batch tests/test_metal_exactn_oracle tests/test_metal_dspark_capture tests/test_metal_iq2_midonly tests/test_metal_iq2_live_index tests/test_glm53_kda tests/test_glm53_kda_rocm tests/test_glm53_vision_engine tests/test_glm53_vision_prompt tests/test_gpu_xdev tests/test_gpu_model_cache tests/test_gpu_lookup_cache_strict tests/test_engine_mgpu_refusal tests/test_engine_mgpu_runtime tests/test_engine_correctness tests/test_sampling tests/test_cuda_session_batch tests/test_cuda_mixed_batch tests/*.o *.o tests/cuda_long_context_smoke tests/cuda_long_context_smoke.o + rm -f ds4 ds4-server ds4-bench ds4-eval ds4-agent ds4_cpu ds4_native ds4_server_test ds4_test ds4_agent_test gguf-tools/quality-testing/score_official gguf-tools/quality-testing/score_official.o speed-bench/metal_decode_schedule_bench speed-bench/metal_prefill_variant_bench speed-bench/*.o tests/test_q4k_dot tests/test_mxfp4_dot tests/test_mxfp4_metal tests/test_mxfp4_rocm tests/test_mxfp4_cuda tests/test_metal_session_batch tests/test_metal_exactn_oracle tests/test_metal_dspark_capture tests/test_metal_iq2_midonly tests/test_metal_iq2_ssd_grouped_mm tests/test_metal_iq2_live_index tests/test_glm53_kda tests/test_glm53_kda_rocm tests/test_glm53_vision_engine tests/test_glm53_vision_prompt tests/test_gpu_xdev tests/test_gpu_model_cache tests/test_gpu_lookup_cache_strict tests/test_engine_mgpu_refusal tests/test_engine_mgpu_runtime tests/test_engine_correctness tests/test_sampling tests/test_cuda_session_batch tests/test_cuda_mixed_batch tests/*.o *.o tests/cuda_long_context_smoke tests/cuda_long_context_smoke.o diff --git a/ds4_metal.m b/ds4_metal.m index d7614a4ea..0d55e660d 100644 --- a/ds4_metal.m +++ b/ds4_metal.m @@ -504,6 +504,17 @@ static ds4_gpu_stream_expert_timing_snapshot g_stream_expert_timing_last_report; static int g_stream_prefill_batch_selected_addr_building; static int g_glm_stream_expert_addr_table_building; +/* Coverage-only counters for the IQ2_XXS/Q2_K SSD grouped address-MM path. + * They are updated on the host after an otherwise successful routed batch + * and never add a command-buffer boundary or GPU synchronization. */ +static int g_iq2_stream_addr_mm_stats_enabled; +static uint64_t g_iq2_stream_addr_mm_candidate_calls; +static uint64_t g_iq2_stream_addr_mm_calls; +static uint64_t g_iq2_stream_addr_mm_tokens; +static uint64_t g_iq2_stream_addr_mm_rows; +static uint64_t g_iq2_stream_addr_mm_require_failures; +static uint32_t g_iq2_stream_addr_mm_min_tokens; +static uint32_t g_iq2_stream_addr_mm_max_tokens; static uint64_t g_model_residency_count; static int g_model_residency_added_to_queue; static int g_glm_model_mode; @@ -2438,6 +2449,81 @@ static int ds4_gpu_env_bool(const char *name) { return 1; } +static void ds4_gpu_iq2_stream_addr_mm_stats_reset(void) { + g_iq2_stream_addr_mm_candidate_calls = 0; + g_iq2_stream_addr_mm_calls = 0; + g_iq2_stream_addr_mm_tokens = 0; + g_iq2_stream_addr_mm_rows = 0; + g_iq2_stream_addr_mm_require_failures = 0; + g_iq2_stream_addr_mm_min_tokens = 0; + g_iq2_stream_addr_mm_max_tokens = 0; +} + +static void ds4_gpu_iq2_stream_addr_mm_stats_note_candidate(void) { + if (g_iq2_stream_addr_mm_stats_enabled) { + g_iq2_stream_addr_mm_candidate_calls++; + } +} + +static void ds4_gpu_iq2_stream_addr_mm_stats_note_selected( + uint32_t n_tokens, + uint32_t n_expert) { + if (!g_iq2_stream_addr_mm_stats_enabled) return; + + if (g_iq2_stream_addr_mm_calls == 0 || + n_tokens < g_iq2_stream_addr_mm_min_tokens) { + g_iq2_stream_addr_mm_min_tokens = n_tokens; + } + if (n_tokens > g_iq2_stream_addr_mm_max_tokens) { + g_iq2_stream_addr_mm_max_tokens = n_tokens; + } + g_iq2_stream_addr_mm_calls++; + g_iq2_stream_addr_mm_tokens += n_tokens; + g_iq2_stream_addr_mm_rows += (uint64_t)n_tokens * n_expert; +} + +static void ds4_gpu_iq2_stream_addr_mm_stats_note_require_failure(void) { + if (g_iq2_stream_addr_mm_stats_enabled) { + g_iq2_stream_addr_mm_require_failures++; + } +} + +static void ds4_gpu_iq2_stream_addr_mm_stats_print(void) { + if (!g_iq2_stream_addr_mm_stats_enabled) return; + + fprintf(stderr, + "ds4: Metal IQ2_XXS SSD prefill MM stats candidates=%llu " + "calls=%llu tokens=%llu rows=%llu min_tokens=%u max_tokens=%u " + "require_failures=%llu\n", + (unsigned long long)g_iq2_stream_addr_mm_candidate_calls, + (unsigned long long)g_iq2_stream_addr_mm_calls, + (unsigned long long)g_iq2_stream_addr_mm_tokens, + (unsigned long long)g_iq2_stream_addr_mm_rows, + g_iq2_stream_addr_mm_min_tokens, + g_iq2_stream_addr_mm_max_tokens, + (unsigned long long)g_iq2_stream_addr_mm_require_failures); +} + +/* Standalone Metal oracle hook. Production code only updates the counters; + * the dedicated test reads them without changing path selection. */ +int ds4_gpu_test_iq2_stream_addr_mm_stats( + uint64_t *candidate_calls, + uint64_t *calls, + uint64_t *tokens, + uint64_t *rows, + uint64_t *require_failures, + uint32_t *min_tokens, + uint32_t *max_tokens) { + if (candidate_calls) *candidate_calls = g_iq2_stream_addr_mm_candidate_calls; + if (calls) *calls = g_iq2_stream_addr_mm_calls; + if (tokens) *tokens = g_iq2_stream_addr_mm_tokens; + if (rows) *rows = g_iq2_stream_addr_mm_rows; + if (require_failures) *require_failures = g_iq2_stream_addr_mm_require_failures; + if (min_tokens) *min_tokens = g_iq2_stream_addr_mm_min_tokens; + if (max_tokens) *max_tokens = g_iq2_stream_addr_mm_max_tokens; + return 1; +} + static uint64_t ds4_gpu_env_u64(const char *name, uint64_t fallback, uint64_t min_value, @@ -6568,6 +6654,10 @@ static int ds4_gpu_encode_rope_tail_inplace( int ds4_gpu_init(void) { if (g_initialized) return 1; + ds4_gpu_iq2_stream_addr_mm_stats_reset(); + g_iq2_stream_addr_mm_stats_enabled = + ds4_gpu_env_bool("DS4_METAL_IQ2_XXS_SSD_PREFILL_MM_STATS") > 0; + @autoreleasepool { ds4_gpu_decode_pipeline_fast_cache_reset(); g_device = MTLCreateSystemDefaultDevice(); @@ -10779,7 +10869,11 @@ int ds4_gpu_synchronize(void) { } void ds4_gpu_cleanup(void) { - if (!g_initialized) return; + if (!g_initialized) { + g_iq2_stream_addr_mm_stats_enabled = 0; + ds4_gpu_iq2_stream_addr_mm_stats_reset(); + return; + } @autoreleasepool { ds4_gpu_decode_pipeline_fast_cache_reset(); @@ -10795,6 +10889,7 @@ void ds4_gpu_cleanup(void) { g_stream_expert_cache_batch_seq = 0; } (void)ds4_gpu_wait_pending_command_buffers("cleanup"); + ds4_gpu_iq2_stream_addr_mm_stats_print(); if (ds4_gpu_stream_expert_timing_summary_enabled() && getenv("DS4_METAL_MEMORY_REPORT") == NULL) { ds4_gpu_print_memory_report("at cleanup"); @@ -11129,6 +11224,8 @@ void ds4_gpu_cleanup(void) { g_library = nil; g_queue = nil; g_device = nil; + g_iq2_stream_addr_mm_stats_enabled = 0; + ds4_gpu_iq2_stream_addr_mm_stats_reset(); g_initialized = 0; } } @@ -32609,6 +32706,8 @@ static int ds4_gpu_routed_mm_mpp_mask(void) { static id ds4_gpu_routed_mm_addr_pipeline(uint32_t type) { switch (type) { + case DS4_METAL_TENSOR_IQ2_XXS: + return ds4_gpu_get_mul_mm_id_pipeline("kernel_mul_mm_id_addr_iq2_xxs_f32", false); case DS4_METAL_TENSOR_Q2_K: return ds4_gpu_get_mul_mm_id_pipeline("kernel_mul_mm_id_addr_q2_K_f32", false); case DS4_METAL_TENSOR_Q4_K: @@ -44046,6 +44145,67 @@ int ds4_gpu_routed_moe_batch_tensor( g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_pipeline != nil && g_moe_mul_mv_addr_q2_k_sum6_pipeline != nil; + /* IQ2 grouped-MM controls are value-aware. The sparse address-MM + * path is the default for eligible IQ2_XXS/Q2_K SSD prefill; + * ENABLE=0 is the legacy-MV control, REQUIRE forces coverage, and + * the dedicated disable always wins. */ + const bool require_iq2_batch_addr_mm = + ds4_gpu_env_bool("DS4_METAL_REQUIRE_IQ2_XXS_SSD_PREFILL_MM") == 1; + const int enable_iq2_batch_addr_mm = + ds4_gpu_env_bool("DS4_METAL_ENABLE_IQ2_XXS_SSD_PREFILL_MM"); + const bool disable_iq2_batch_addr_mm = + ds4_gpu_env_bool("DS4_METAL_DISABLE_IQ2_XXS_SSD_PREFILL_MM") == 1; + const bool request_iq2_batch_addr_mm = + (enable_iq2_batch_addr_mm != 0 || require_iq2_batch_addr_mm) && + !disable_iq2_batch_addr_mm; + const bool iq2_batch_addr_mm_candidate = + !force_resident && g_ssd_streaming_mode && + gate_type == DS4_METAL_TENSOR_IQ2_XXS && + down_type == DS4_METAL_TENSOR_Q2_K && + n_tokens >= 32u && n_expert == 6 && + n_total_expert != 0 && + n_total_expert <= DS4_METAL_STREAM_EXPERT_CACHE_MAX_EXPERT && + g_tp_split_world == 1 && + !g_quality_mode && + getenv("DS4_METAL_MOE_WRITE_CLAMPED_ACT") == NULL && + getenv("DS4_METAL_DISABLE_ROUTED_PAIR_SWIGLU_FUSION") == NULL && + !ds4_gpu_stage_profile_enabled_for_layer( + "DS4_METAL_MOE_STAGE_PROFILE", + "DS4_METAL_MOE_STAGE_PROFILE_LAYER", + layer_index) && + getenv("DS4_METAL_GRAPH_DUMP_PREFIX") == NULL; + if (iq2_batch_addr_mm_candidate) { + ds4_gpu_iq2_stream_addr_mm_stats_note_candidate(); + } + const bool iq2_batch_addr_mm_policy = + request_iq2_batch_addr_mm && + iq2_batch_addr_mm_candidate && + use_iq2_batch_selected_addr; + id iq2_gate_addr_mm_pipeline = + iq2_batch_addr_mm_policy ? + ds4_gpu_routed_mm_addr_pipeline(gate_type) : nil; + id iq2_down_addr_mm_pipeline = + iq2_batch_addr_mm_policy ? + ds4_gpu_routed_mm_addr_pipeline(down_type) : nil; + const bool use_iq2_batch_addr_mm = + iq2_batch_addr_mm_policy && + iq2_gate_addr_mm_pipeline != nil && + iq2_down_addr_mm_pipeline != nil; + + /* REQUIRE only asserts calls that satisfy the specialization's + * candidate contract; short final chunks keep the established path. */ + if (require_iq2_batch_addr_mm && + !disable_iq2_batch_addr_mm && + iq2_batch_addr_mm_candidate && + !use_iq2_batch_addr_mm) { + ds4_gpu_iq2_stream_addr_mm_stats_note_require_failure(); + fprintf(stderr, + "ds4: required Metal IQ2_XXS SSD grouped address-MM path " + "was not selected tokens=%u gate=%u down=%u\n", + n_tokens, gate_type, down_type); + return 0; + } + ds4_gpu_mul_mv_id_args gate_args = ds4_gpu_make_mul_mv_id_args(expert_in_dim, expert_mid_dim, n_total_expert, gate_row_bytes, gate_expert_bytes, @@ -44259,7 +44419,7 @@ int ds4_gpu_routed_moe_batch_tensor( g_tp_split_world == 1 && (use_pre_m5_mxfp4_mm_id_down_half_lut_default || (g_test_flags & DS4_GPU_TEST_MXFP4_DOWN_HALF_LUT) != 0u); - if (use_mm_id) { + if (use_mm_id || use_iq2_batch_addr_mm) { gate_map_args = ds4_gpu_make_mul_mm_id_map_args(expert_in_dim, n_total_expert, 1, n_expert, n_tokens); gate_mm_args = @@ -44282,9 +44442,13 @@ int ds4_gpu_routed_moe_batch_tensor( use_mxfp4_mm_id_map_scatter ? "kernel_mul_mm_id_map_scatter_work_ne20_6" : ds4_gpu_mul_mm_id_map0_name(n_expert)); - gate_mm_pipeline = ds4_gpu_routed_mm_pipeline(gate_type); - up_mm_pipeline = ds4_gpu_routed_mm_pipeline(gate_type); - down_mm_pipeline = use_mxfp4_mm_id_down_half_lut ? + gate_mm_pipeline = use_iq2_batch_addr_mm ? + iq2_gate_addr_mm_pipeline : + ds4_gpu_routed_mm_pipeline(gate_type); + up_mm_pipeline = gate_mm_pipeline; + down_mm_pipeline = use_iq2_batch_addr_mm ? + iq2_down_addr_mm_pipeline : + use_mxfp4_mm_id_down_half_lut ? ds4_gpu_get_mul_mm_id_pipeline( use_mxfp4_mm_id_down_tail_simdgroup_cull ? "kernel_mul_mm_id_mxfp4_f16_half_lut_tail_cull" : @@ -44297,7 +44461,8 @@ int ds4_gpu_routed_moe_batch_tensor( ds4_gpu_routed_mm_f16_rhs_pipeline(down_type) : ds4_gpu_routed_mm_pipeline(down_type); const int mpp_mask = ds4_gpu_routed_mm_mpp_mask(); - if (mpp_mask && gate_type == DS4_METAL_TENSOR_IQ2_XXS) { + if (!use_iq2_batch_addr_mm && + mpp_mask && gate_type == DS4_METAL_TENSOR_IQ2_XXS) { id mpp = ds4_gpu_get_mul_mm_id_pipeline("kernel_mul_mm_id_iq2_xxs_f32_mpp", false); if (mpp) { @@ -44305,7 +44470,8 @@ int ds4_gpu_routed_moe_batch_tensor( if (mpp_mask & 2) up_mm_pipeline = mpp; } } - if ((mpp_mask & 4) && request_mid_f16 && + if (!use_iq2_batch_addr_mm && + (mpp_mask & 4) && request_mid_f16 && (down_type == DS4_METAL_TENSOR_Q2_K || down_type == DS4_METAL_TENSOR_IQ2_XXS)) { id mpp = ds4_gpu_get_mul_mm_id_pipeline( down_type == DS4_METAL_TENSOR_Q2_K ? @@ -44522,6 +44688,14 @@ int ds4_gpu_routed_moe_batch_tensor( int owned = 0; id cb = ds4_gpu_command_buffer(&owned); if (!cb) return 0; + if (use_iq2_batch_addr_mm && stream_resource_count != 0 && + !ds4_gpu_stream_expert_cache_mark_entries_inflight( + stream_resources, stream_resource_count, 0)) { + fprintf(stderr, + "ds4: Metal IQ2_XXS SSD grouped address-MM could not pin " + "expert resources for the command buffer\n"); + return 0; + } if (use_q4_batch_expert_table && !ds4_gpu_use_model_residency_set(cb)) { return 0; } @@ -44538,6 +44712,7 @@ int ds4_gpu_routed_moe_batch_tensor( const char *moe_stage_filter = getenv("DS4_METAL_MOE_STAGE_PROFILE_FILTER"); const char *moe_path = use_q4_batch_expert_table ? "q4_table_pair_swiglu" : + use_iq2_batch_addr_mm ? "iq2_batch_stream_addr_mm" : use_iq2_batch_selected_addr ? "iq2_batch_stream_addr" : use_mm_id_pair_swiglu ? "mm_id_pair_swiglu" : use_mm_id ? "mm_id" : @@ -44606,7 +44781,31 @@ int ds4_gpu_routed_moe_batch_tensor( n_tokens <= 4u && down_sum6_pipeline != nil; int ok = 0; - if (use_iq2_batch_selected_addr) { + if (use_iq2_batch_addr_mm) { + ok = ds4_gpu_encode_mul_mm_id_map(cb, + map_pipeline, + &gate_map_args, + &gate_mm_args, + selectedbuf, + ds4_gpu_tensor_offset(selected)); + DS4_METAL_PROFILE_MOE_STAGE("map"); + if (ok) { + ok = ds4_gpu_encode_mul_mm_id_addr_mapped_tile( + cb, gate_mm_pipeline, &gate_mm_args, + stream_gate_addr_buf, + xbuf, ds4_gpu_tensor_offset(x), + gatebuf, ds4_gpu_tensor_offset(gate), + 8192u, stream_resources, stream_resource_count, + 0u, stream_overflow_gate) && + ds4_gpu_encode_mul_mm_id_addr_mapped_tile( + cb, up_mm_pipeline, &gate_mm_args, + stream_up_addr_buf, + xbuf, ds4_gpu_tensor_offset(x), + upbuf, ds4_gpu_tensor_offset(up), + 8192u, stream_resources, stream_resource_count, + 1u, stream_overflow_up); + } + } else if (use_iq2_batch_selected_addr) { ds4_gpu_dsv4_moe_swiglu_weight_args act_args = { .width = expert_mid_dim, .rows = pair_rows, @@ -44844,7 +45043,7 @@ int ds4_gpu_routed_moe_batch_tensor( use_fused_activation && request_mid_f16; if (mid_is_f16) *mid_is_f16 = use_mid_f16; - if (ok && use_iq2_batch_selected_addr) { + if (ok && use_iq2_batch_selected_addr && !use_iq2_batch_addr_mm) { /* The address-table pair kernel already wrote weighted SwiGLU rows into mid. */ } else if (ok && use_q4_batch_expert_table) { /* The table pair kernel already wrote weighted SwiGLU rows into mid. */ @@ -44946,7 +45145,22 @@ int ds4_gpu_routed_moe_batch_tensor( NSUInteger down_dst_off = n_expert == 1 ? ds4_gpu_tensor_offset(out) : (expertsbuf ? ds4_gpu_tensor_offset(experts) : 0); if (ok) { - if (use_iq2_batch_selected_addr) { + if (use_iq2_batch_addr_mm) { + ok = ds4_gpu_encode_mul_mm_id_addr_mapped_tile( + cb, + down_mm_pipeline, + &down_mm_args, + stream_down_addr_buf, + midbuf, + ds4_gpu_tensor_offset(mid), + down_dst, + down_dst_off, + 8192u, + stream_resources, + stream_resource_count, + 2u, + stream_overflow_down); + } else if (use_iq2_batch_selected_addr) { ok = ds4_gpu_encode_mul_mv_addr_q2_sum6( cb, g_moe_mul_mv_addr_q2_k_sum6_pipeline, @@ -45034,7 +45248,7 @@ int ds4_gpu_routed_moe_batch_tensor( n_expert > 1 && !direct_down_sum && !use_q4_batch_expert_table && - !use_iq2_batch_selected_addr) { + (!use_iq2_batch_selected_addr || use_iq2_batch_addr_mm)) { ok = ds4_gpu_encode_moe_sum_experts(cb, down_dst, down_dst_off, @@ -45078,6 +45292,9 @@ int ds4_gpu_routed_moe_batch_tensor( } } } + if (use_iq2_batch_addr_mm) { + ds4_gpu_iq2_stream_addr_mm_stats_note_selected(n_tokens, n_expert); + } if (q4_batch_table_boundary) { if (ds4_gpu_end_commands() == 0 || ds4_gpu_begin_commands() == 0) { return 0; diff --git a/metal/moe.metal b/metal/moe.metal index b578f066d..c14df610f 100644 --- a/metal/moe.metal +++ b/metal/moe.metal @@ -9371,6 +9371,7 @@ template [[host_name("kernel_mul_mm_id_mxfp4_f16_half_lut")]] kernel mul_mm_id_m template [[host_name("kernel_mul_mm_id_mxfp4_f16_half_lut_tail_cull")]] kernel mul_mm_id_mxfp4_f16_rhs_half_lut_tail_cull kernel_mul_mm_id<32, half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, block_mxfp4, 2, dequantize_mxfp4_half_lut, half, half4x4, half, half2x4, true>; template [[host_name("kernel_mul_mm_id_addr_q2_K_f32")]] kernel mul_mm_id_addr kernel_mul_mm_id_addr<32, half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, block_q2_K, QK_NL, dequantize_q2_K, float, float4x4, float, float2x4>; +template [[host_name("kernel_mul_mm_id_addr_iq2_xxs_f32")]] kernel mul_mm_id_addr kernel_mul_mm_id_addr<32, half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, block_iq2_xxs, QK_NL, dequantize_iq2_xxs, float, float4x4, float, float2x4>; template [[host_name("kernel_mul_mm_id_addr_q4_K_f32")]] kernel mul_mm_id_addr kernel_mul_mm_id_addr<32, half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, block_q4_K, QK_NL, dequantize_q4_K, float, float4x4, float, float2x4>; template [[host_name("kernel_mul_mm_id_addr_mxfp4_f32")]] kernel mul_mm_id_addr kernel_mul_mm_id_addr<32, half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, block_mxfp4, 2, dequantize_mxfp4, float, float4x4, float, float2x4>; template [[host_name("kernel_mul_mm_id_addr_q2_K_f16")]] kernel mul_mm_id_addr_f16_rhs kernel_mul_mm_id_addr<32, half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, block_q2_K, QK_NL, dequantize_q2_K, half, half4x4, half, half2x4>; diff --git a/tests/test_metal_iq2_ssd_grouped_mm.c b/tests/test_metal_iq2_ssd_grouped_mm.c new file mode 100644 index 000000000..096aae69b --- /dev/null +++ b/tests/test_metal_iq2_ssd_grouped_mm.c @@ -0,0 +1,1414 @@ +#define _DARWIN_C_SOURCE + +#include "ds4_gpu.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef __APPLE__ + +#define IQ2_XXS_TYPE 16u +#define Q2_K_TYPE 10u +#define QK_K 256u +#define IN_DIM 256u +#define MID_DIM 256u +#define OUT_DIM 64u +#define N_TOTAL_EXPERT 8u +#define N_EXPERT 6u +#define MAX_TOKENS 33u +#define N_TOTAL_EXPERT_256 256u +#define HIGH_EXPERT_ID 255u +#define CLAMP 4.0f +#define SENTINEL 1234567.0f + +/* Production Flash routed-expert geometry. Eight physical experts keep the + * standalone fixture bounded while retaining the production top-6 routing, + * 32-token grouped tile, and exact quantized row strides. */ +#define FULL_IN_DIM 4096u +#define FULL_MID_DIM 2048u +#define FULL_OUT_DIM 4096u +#define FULL_TOKENS 32u +#define FULL_GUARD_WORDS 64u +#define FULL_GUARD_BITS 0x51a7c3e9u + +typedef struct { + uint16_t d; + uint16_t qs[QK_K / 8u]; +} block_iq2_xxs; + +typedef struct { + uint8_t scales[QK_K / 16u]; + uint8_t qs[QK_K / 4u]; + uint16_t d; + uint16_t dmin; +} block_q2_K; + +typedef struct { + uint32_t tokens; + uint64_t pair_count; + uint64_t out_count; + float *gate; + float *up; + float *mid; + float *out; + uint64_t guard_mismatches; +} run_result; + +typedef struct { + uint64_t pair_count; + uint64_t out_count; + float *gate; + float *up; + float *mid; + float *out; + uint64_t guard_mismatches; +} full_run_result; + +typedef struct { + uint64_t candidate_calls; + uint64_t calls; + uint64_t tokens; + uint64_t rows; + uint64_t require_failures; + uint32_t min_tokens; + uint32_t max_tokens; +} mm_stats_snapshot; + +static uint32_t *full_guarded_payload(uint64_t payload_words); +static int full_check_guard(const char *run_name, + const char *tensor_name, + const ds4_gpu_tensor *tensor, + uint64_t payload_words, + uint64_t *mismatches); + +/* Test-only counter reader implemented by the Metal backend. */ +int ds4_gpu_test_iq2_stream_addr_mm_stats( + uint64_t *candidate_calls, uint64_t *calls, uint64_t *tokens, + uint64_t *rows, uint64_t *require_failures, uint32_t *min_tokens, + uint32_t *max_tokens); + +bool ds4_log_is_tty(FILE *fp) { + (void)fp; + return false; +} + +static uint64_t align_up(uint64_t value, uint64_t alignment) { + return (value + alignment - 1u) / alignment * alignment; +} + +static void fill_iq2(block_iq2_xxs *matrix, uint32_t salt, + uint32_t n_total_expert) { + for (uint32_t expert = 0; expert < n_total_expert; expert++) { + for (uint32_t row = 0; row < MID_DIM; row++) { + block_iq2_xxs *b = matrix + (uint64_t)expert * MID_DIM + row; + const uint32_t key = salt * 977u + expert * 431u + row * 37u; + b->d = (uint16_t)(0x1800u + ((key & 1u) ? 0x0200u : 0u)); + for (uint32_t i = 0; i < QK_K / 8u; i++) { + b->qs[i] = (uint16_t)(key + i * 509u + (i >> 2u) * 131u); + } + } + } +} + +static void fill_q2(block_q2_K *matrix, uint32_t n_total_expert) { + for (uint32_t expert = 0; expert < n_total_expert; expert++) { + for (uint32_t row = 0; row < OUT_DIM; row++) { + block_q2_K *b = matrix + (uint64_t)expert * OUT_DIM + row; + const uint32_t key = expert * 617u + row * 73u; + for (uint32_t group = 0; group < QK_K / 16u; group++) { + const uint8_t scale = + (uint8_t)(1u + (key + 3u * group) % 7u); + const uint8_t min = (uint8_t)((key / 5u + group) % 4u); + b->scales[group] = (uint8_t)(scale | (min << 4u)); + } + for (uint32_t i = 0; i < QK_K / 4u; i++) { + b->qs[i] = (uint8_t)(key + 29u * i + (i >> 1u) * 7u); + } + b->d = 0x1800u; + b->dmin = 0x1400u; + } + } +} + +static void fill_iq2_full(block_iq2_xxs *matrix, uint32_t salt) { + const uint32_t blocks_per_row = FULL_IN_DIM / QK_K; + for (uint32_t expert = 0; expert < N_TOTAL_EXPERT; expert++) { + for (uint32_t row = 0; row < FULL_MID_DIM; row++) { + for (uint32_t block = 0; block < blocks_per_row; block++) { + block_iq2_xxs *b = matrix + + ((uint64_t)expert * FULL_MID_DIM + row) * + blocks_per_row + block; + const uint32_t key = salt * 977u + expert * 431u + + row * 37u + block * 811u; + b->d = (uint16_t)(0x1800u + + ((key & 1u) ? 0x0200u : 0u)); + for (uint32_t i = 0; i < QK_K / 8u; i++) { + b->qs[i] = (uint16_t)(key + i * 509u + + (i >> 2u) * 131u); + } + } + } + } +} + +static void fill_q2_full(block_q2_K *matrix) { + const uint32_t blocks_per_row = FULL_MID_DIM / QK_K; + for (uint32_t expert = 0; expert < N_TOTAL_EXPERT; expert++) { + for (uint32_t row = 0; row < FULL_OUT_DIM; row++) { + for (uint32_t block = 0; block < blocks_per_row; block++) { + block_q2_K *b = matrix + + ((uint64_t)expert * FULL_OUT_DIM + row) * + blocks_per_row + block; + const uint32_t key = expert * 617u + row * 73u + + block * 991u; + for (uint32_t group = 0; group < QK_K / 16u; group++) { + const uint8_t scale = + (uint8_t)(1u + (key + 3u * group) % 7u); + const uint8_t min = + (uint8_t)((key / 5u + group) % 4u); + b->scales[group] = + (uint8_t)(scale | (uint8_t)(min << 4u)); + } + for (uint32_t i = 0; i < QK_K / 4u; i++) { + b->qs[i] = + (uint8_t)(key + 29u * i + (i >> 1u) * 7u); + } + b->d = 0x1800u; + b->dmin = 0x1400u; + } + } + } +} + +static int result_alloc(run_result *r, uint32_t tokens) { + memset(r, 0, sizeof(*r)); + r->tokens = tokens; + r->pair_count = (uint64_t)tokens * N_EXPERT * MID_DIM; + r->out_count = (uint64_t)tokens * OUT_DIM; + r->gate = calloc((size_t)r->pair_count, sizeof(float)); + r->up = calloc((size_t)r->pair_count, sizeof(float)); + r->mid = calloc((size_t)r->pair_count, sizeof(float)); + r->out = calloc((size_t)r->out_count, sizeof(float)); + return r->gate && r->up && r->mid && r->out; +} + +static void result_free(run_result *r) { + free(r->gate); + free(r->up); + free(r->mid); + free(r->out); + memset(r, 0, sizeof(*r)); +} + +static int run_once( + const char *run_name, + run_result *result, + const void *model, + uint64_t model_size, + uint64_t gate_offset, + uint64_t up_offset, + uint64_t down_offset, + uint64_t gate_expert_bytes, + uint64_t gate_row_bytes, + uint64_t down_expert_bytes, + uint64_t down_row_bytes, + uint32_t n_total_expert, + const float *x, + const int32_t *selected, + const float *weights) { + const uint32_t tokens = result->tokens; + const uint64_t x_count = (uint64_t)tokens * IN_DIM; + const uint64_t route_count = (uint64_t)tokens * N_EXPERT; + const uint64_t expert_count = (uint64_t)tokens * N_EXPERT * OUT_DIM; + const uint64_t guard_bytes = + (uint64_t)FULL_GUARD_WORDS * sizeof(uint32_t); + uint32_t *pair_init = full_guarded_payload(result->pair_count); + uint32_t *expert_init = full_guarded_payload(expert_count); + uint32_t *out_init = full_guarded_payload(result->out_count); + if (!pair_init || !expert_init || !out_init) { + free(pair_init); + free(expert_init); + free(out_init); + return 0; + } + + ds4_gpu_tensor *x_t = ds4_gpu_tensor_alloc(x_count * sizeof(float)); + ds4_gpu_tensor *selected_t = + ds4_gpu_tensor_alloc(route_count * sizeof(int32_t)); + ds4_gpu_tensor *weights_t = + ds4_gpu_tensor_alloc(route_count * sizeof(float)); + ds4_gpu_tensor *gate_t = + ds4_gpu_tensor_alloc(result->pair_count * sizeof(float) + guard_bytes); + ds4_gpu_tensor *up_t = + ds4_gpu_tensor_alloc(result->pair_count * sizeof(float) + guard_bytes); + ds4_gpu_tensor *mid_t = + ds4_gpu_tensor_alloc(result->pair_count * sizeof(float) + guard_bytes); + ds4_gpu_tensor *experts_t = + ds4_gpu_tensor_alloc(expert_count * sizeof(float) + guard_bytes); + ds4_gpu_tensor *out_t = + ds4_gpu_tensor_alloc(result->out_count * sizeof(float) + guard_bytes); + int ok = x_t && selected_t && weights_t && gate_t && up_t && mid_t && + experts_t && out_t; + ok = ok && ds4_gpu_tensor_write(x_t, 0, x, x_count * sizeof(float)); + ok = ok && ds4_gpu_tensor_write(selected_t, 0, selected, + route_count * sizeof(int32_t)); + ok = ok && ds4_gpu_tensor_write(weights_t, 0, weights, + route_count * sizeof(float)); + ok = ok && ds4_gpu_tensor_write( + gate_t, 0, pair_init, + result->pair_count * sizeof(float) + guard_bytes); + ok = ok && ds4_gpu_tensor_write( + up_t, 0, pair_init, + result->pair_count * sizeof(float) + guard_bytes); + ok = ok && ds4_gpu_tensor_write( + mid_t, 0, pair_init, + result->pair_count * sizeof(float) + guard_bytes); + ok = ok && ds4_gpu_tensor_write( + experts_t, 0, expert_init, + expert_count * sizeof(float) + guard_bytes); + ok = ok && ds4_gpu_tensor_write( + out_t, 0, out_init, + result->out_count * sizeof(float) + guard_bytes); + + bool mid_is_f16 = true; + if (ok) { + ok = ds4_gpu_routed_moe_batch_tensor( + out_t, gate_t, up_t, mid_t, experts_t, model, model_size, + gate_offset, up_offset, down_offset, IQ2_XXS_TYPE, Q2_K_TYPE, + gate_expert_bytes, gate_row_bytes, down_expert_bytes, + down_row_bytes, IN_DIM, MID_DIM, OUT_DIM, selected_t, weights_t, + n_total_expert, N_EXPERT, CLAMP, x_t, 0u, tokens, + &mid_is_f16, false); + } + if (ok && mid_is_f16) { + fprintf(stderr, + "IQ2_XXS SSD grouped-MM oracle unexpectedly selected f16 mid\n"); + ok = 0; + } + ok = ok && ds4_gpu_tensor_read(gate_t, 0, result->gate, + result->pair_count * sizeof(float)); + ok = ok && ds4_gpu_tensor_read(up_t, 0, result->up, + result->pair_count * sizeof(float)); + ok = ok && ds4_gpu_tensor_read(mid_t, 0, result->mid, + result->pair_count * sizeof(float)); + ok = ok && ds4_gpu_tensor_read(out_t, 0, result->out, + result->out_count * sizeof(float)); + + result->guard_mismatches = 0; + if (gate_t) { + const int guard_ok = full_check_guard( + run_name, "gate", gate_t, result->pair_count, + &result->guard_mismatches); + ok = guard_ok && ok; + } + if (up_t) { + const int guard_ok = full_check_guard( + run_name, "up", up_t, result->pair_count, + &result->guard_mismatches); + ok = guard_ok && ok; + } + if (mid_t) { + const int guard_ok = full_check_guard( + run_name, "mid", mid_t, result->pair_count, + &result->guard_mismatches); + ok = guard_ok && ok; + } + if (experts_t) { + const int guard_ok = full_check_guard( + run_name, "experts", experts_t, expert_count, + &result->guard_mismatches); + ok = guard_ok && ok; + } + if (out_t) { + const int guard_ok = full_check_guard( + run_name, "out", out_t, result->out_count, + &result->guard_mismatches); + ok = guard_ok && ok; + } + + ds4_gpu_tensor_free(x_t); + ds4_gpu_tensor_free(selected_t); + ds4_gpu_tensor_free(weights_t); + ds4_gpu_tensor_free(gate_t); + ds4_gpu_tensor_free(up_t); + ds4_gpu_tensor_free(mid_t); + ds4_gpu_tensor_free(experts_t); + ds4_gpu_tensor_free(out_t); + free(pair_init); + free(expert_init); + free(out_init); + return ok; +} + +static int compare_array(const char *case_name, const char *tensor_name, + const float *candidate, const float *control, + uint64_t count, double max_limit, + double rms_limit) { + double max_abs = 0.0; + long double sum_sq = 0.0; + uint64_t nonfinite = 0; + uint64_t unwritten = 0; + uint32_t sentinel_bits = 0; + memcpy(&sentinel_bits, &(float){ SENTINEL }, sizeof(sentinel_bits)); + for (uint64_t i = 0; i < count; i++) { + uint32_t candidate_bits = 0; + uint32_t control_bits = 0; + memcpy(&candidate_bits, candidate + i, sizeof(candidate_bits)); + memcpy(&control_bits, control + i, sizeof(control_bits)); + if (candidate_bits == sentinel_bits || control_bits == sentinel_bits) { + unwritten++; + continue; + } + if ((candidate_bits & 0x7f800000u) == 0x7f800000u || + (control_bits & 0x7f800000u) == 0x7f800000u) { + nonfinite++; + continue; + } + const double diff = fabs((double)candidate[i] - (double)control[i]); + if (diff > max_abs) max_abs = diff; + sum_sq += (long double)diff * (long double)diff; + } + const double rms = count ? sqrt((double)(sum_sq / count)) : 0.0; + const int pass = nonfinite == 0 && unwritten == 0 && + max_abs <= max_limit && rms <= rms_limit; + fprintf(stderr, + "IQ2_XXS SSD grouped-MM %-12s %-4s %s count=%llu " + "max_abs=%.9g rms=%.9g limits=%.9g/%.9g nonfinite=%llu " + "unwritten=%llu\n", + case_name, tensor_name, pass ? "PASS" : "FAIL", + (unsigned long long)count, max_abs, rms, max_limit, rms_limit, + (unsigned long long)nonfinite, + (unsigned long long)unwritten); + return pass; +} + +static int compare_results(const char *name, const run_result *candidate, + const run_result *control) { + if (candidate->pair_count != control->pair_count || + candidate->out_count != control->out_count || + candidate->guard_mismatches != 0 || + control->guard_mismatches != 0) { + return 0; + } + int ok = compare_array(name, "gate", candidate->gate, control->gate, + candidate->pair_count, 0.025, 0.004); + ok = compare_array(name, "up", candidate->up, control->up, + candidate->pair_count, 0.025, 0.004) && ok; + ok = compare_array(name, "mid", candidate->mid, control->mid, + candidate->pair_count, 0.08, 0.015) && ok; + ok = compare_array(name, "out", candidate->out, control->out, + candidate->out_count, 0.08, 0.015) && ok; + return ok; +} + +static int run_pair( + const char *name, + uint32_t tokens, + const void *model, + uint64_t model_size, + uint64_t gate_offset, + uint64_t up_offset, + uint64_t down_offset, + uint64_t gate_expert_bytes, + uint64_t gate_row_bytes, + uint64_t down_expert_bytes, + uint64_t down_row_bytes, + uint32_t n_total_expert, + uint32_t cache_budget, + const float *x, + const int32_t *selected, + const float *weights) { + run_result control; + run_result candidate; + memset(&control, 0, sizeof(control)); + memset(&candidate, 0, sizeof(candidate)); + if (!result_alloc(&control, tokens) || !result_alloc(&candidate, tokens)) { + result_free(&control); + result_free(&candidate); + return 0; + } + + unsetenv("DS4_METAL_ENABLE_IQ2_XXS_SSD_PREFILL_MM"); + unsetenv("DS4_METAL_REQUIRE_IQ2_XXS_SSD_PREFILL_MM"); + setenv("DS4_METAL_DISABLE_IQ2_XXS_SSD_PREFILL_MM", "1", 1); + ds4_gpu_set_streaming_expert_cache_budget(cache_budget); + char control_name[64]; + char candidate_name[64]; + snprintf(control_name, sizeof(control_name), "%s-control", name); + snprintf(candidate_name, sizeof(candidate_name), "%s-candidate", name); + int ok = run_once(control_name, &control, + model, model_size, gate_offset, up_offset, + down_offset, gate_expert_bytes, gate_row_bytes, + down_expert_bytes, down_row_bytes, n_total_expert, + x, selected, weights); + + unsetenv("DS4_METAL_DISABLE_IQ2_XXS_SSD_PREFILL_MM"); + setenv("DS4_METAL_ENABLE_IQ2_XXS_SSD_PREFILL_MM", "1", 1); + setenv("DS4_METAL_REQUIRE_IQ2_XXS_SSD_PREFILL_MM", "1", 1); + ds4_gpu_set_streaming_expert_cache_budget(cache_budget); + ok = ok && run_once(candidate_name, &candidate, + model, model_size, gate_offset, up_offset, + down_offset, gate_expert_bytes, gate_row_bytes, + down_expert_bytes, down_row_bytes, n_total_expert, + x, selected, weights); + if (ok) ok = compare_results(name, &candidate, &control); + + result_free(&control); + result_free(&candidate); + return ok; +} + +static int read_mm_stats(mm_stats_snapshot *s) { + memset(s, 0, sizeof(*s)); + return ds4_gpu_test_iq2_stream_addr_mm_stats( + &s->candidate_calls, &s->calls, &s->tokens, &s->rows, + &s->require_failures, &s->min_tokens, &s->max_tokens); +} + +static int check_mm_stats_delta(const char *name, + const mm_stats_snapshot *before, + const mm_stats_snapshot *after, + uint64_t expected_tokens) { + const int monotonic = + after->candidate_calls >= before->candidate_calls && + after->calls >= before->calls && + after->tokens >= before->tokens && + after->rows >= before->rows && + after->require_failures >= before->require_failures; + const uint64_t candidates = monotonic ? + after->candidate_calls - before->candidate_calls : UINT64_MAX; + const uint64_t calls = monotonic ? + after->calls - before->calls : UINT64_MAX; + const uint64_t tokens = monotonic ? + after->tokens - before->tokens : UINT64_MAX; + const uint64_t rows = monotonic ? + after->rows - before->rows : UINT64_MAX; + const uint64_t failures = monotonic ? + after->require_failures - before->require_failures : UINT64_MAX; + const int ok = monotonic && candidates == 2u && calls == 1u && + tokens == expected_tokens && + rows == expected_tokens * N_EXPERT && failures == 0u; + fprintf(stderr, + "IQ2_XXS SSD grouped-MM %-15s coverage %s candidates=%llu " + "calls=%llu tokens=%llu rows=%llu require_failures=%llu\n", + name, ok ? "PASS" : "FAIL", + (unsigned long long)candidates, + (unsigned long long)calls, + (unsigned long long)tokens, + (unsigned long long)rows, + (unsigned long long)failures); + return ok; +} + +static int full_result_alloc(full_run_result *r) { + memset(r, 0, sizeof(*r)); + r->pair_count = + (uint64_t)FULL_TOKENS * N_EXPERT * FULL_MID_DIM; + r->out_count = (uint64_t)FULL_TOKENS * FULL_OUT_DIM; + r->gate = calloc((size_t)r->pair_count, sizeof(float)); + r->up = calloc((size_t)r->pair_count, sizeof(float)); + r->mid = calloc((size_t)r->pair_count, sizeof(float)); + r->out = calloc((size_t)r->out_count, sizeof(float)); + return r->gate && r->up && r->mid && r->out; +} + +static void full_result_free(full_run_result *r) { + free(r->gate); + free(r->up); + free(r->mid); + free(r->out); + memset(r, 0, sizeof(*r)); +} + +static uint32_t *full_guarded_payload(uint64_t payload_words) { + if (payload_words > (SIZE_MAX / sizeof(uint32_t)) - FULL_GUARD_WORDS) { + return NULL; + } + uint32_t *words = malloc( + (size_t)(payload_words + FULL_GUARD_WORDS) * sizeof(uint32_t)); + if (!words) return NULL; + + uint32_t sentinel_bits = 0; + const float sentinel = SENTINEL; + memcpy(&sentinel_bits, &sentinel, sizeof(sentinel_bits)); + for (uint64_t i = 0; i < payload_words; i++) words[i] = sentinel_bits; + for (uint32_t i = 0; i < FULL_GUARD_WORDS; i++) { + words[payload_words + i] = FULL_GUARD_BITS; + } + return words; +} + +static int full_check_guard(const char *run_name, + const char *tensor_name, + const ds4_gpu_tensor *tensor, + uint64_t payload_words, + uint64_t *mismatches) { + uint32_t words[FULL_GUARD_WORDS]; + if (!ds4_gpu_tensor_read(tensor, + payload_words * sizeof(uint32_t), + words, + sizeof(words))) { + fprintf(stderr, + "IQ2_XXS SSD grouped-MM %s %s guard readback FAIL\n", + run_name, tensor_name); + return 0; + } + uint64_t local = 0; + for (uint32_t i = 0; i < FULL_GUARD_WORDS; i++) { + if (words[i] != FULL_GUARD_BITS) local++; + } + *mismatches += local; + fprintf(stderr, + "IQ2_XXS SSD grouped-MM %-24s %-7s guard_%s=%llu\n", + run_name, tensor_name, local == 0 ? "PASS" : "FAIL", + (unsigned long long)local); + return local == 0; +} + +static int run_full_once( + const char *run_name, + full_run_result *result, + const void *model, + uint64_t model_size, + uint64_t gate_offset, + uint64_t up_offset, + uint64_t down_offset, + uint64_t gate_expert_bytes, + uint64_t gate_row_bytes, + uint64_t down_expert_bytes, + uint64_t down_row_bytes, + const float *x, + const int32_t *selected, + const float *weights) { + const uint64_t x_count = (uint64_t)FULL_TOKENS * FULL_IN_DIM; + const uint64_t route_count = (uint64_t)FULL_TOKENS * N_EXPERT; + const uint64_t expert_count = + (uint64_t)FULL_TOKENS * N_EXPERT * FULL_OUT_DIM; + const uint64_t pair_bytes = result->pair_count * sizeof(float); + const uint64_t expert_bytes = expert_count * sizeof(float); + const uint64_t out_bytes = result->out_count * sizeof(float); + const uint64_t guard_bytes = + (uint64_t)FULL_GUARD_WORDS * sizeof(uint32_t); + + uint32_t *pair_init = full_guarded_payload(result->pair_count); + uint32_t *expert_init = full_guarded_payload(expert_count); + uint32_t *out_init = full_guarded_payload(result->out_count); + ds4_gpu_tensor *x_t = NULL; + ds4_gpu_tensor *selected_t = NULL; + ds4_gpu_tensor *weights_t = NULL; + ds4_gpu_tensor *gate_t = NULL; + ds4_gpu_tensor *up_t = NULL; + ds4_gpu_tensor *mid_t = NULL; + ds4_gpu_tensor *experts_t = NULL; + ds4_gpu_tensor *out_t = NULL; + int ok = pair_init && expert_init && out_init; + + if (ok) x_t = ds4_gpu_tensor_alloc(x_count * sizeof(float)); + if (ok) selected_t = + ds4_gpu_tensor_alloc(route_count * sizeof(int32_t)); + if (ok) weights_t = ds4_gpu_tensor_alloc(route_count * sizeof(float)); + if (ok) gate_t = ds4_gpu_tensor_alloc(pair_bytes + guard_bytes); + if (ok) up_t = ds4_gpu_tensor_alloc(pair_bytes + guard_bytes); + if (ok) mid_t = ds4_gpu_tensor_alloc(pair_bytes + guard_bytes); + if (ok) experts_t = ds4_gpu_tensor_alloc(expert_bytes + guard_bytes); + if (ok) out_t = ds4_gpu_tensor_alloc(out_bytes + guard_bytes); + ok = ok && x_t && selected_t && weights_t && gate_t && up_t && mid_t && + experts_t && out_t; + ok = ok && ds4_gpu_tensor_write(x_t, 0, x, + x_count * sizeof(float)); + ok = ok && ds4_gpu_tensor_write(selected_t, 0, selected, + route_count * sizeof(int32_t)); + ok = ok && ds4_gpu_tensor_write(weights_t, 0, weights, + route_count * sizeof(float)); + ok = ok && ds4_gpu_tensor_write(gate_t, 0, pair_init, + pair_bytes + guard_bytes); + ok = ok && ds4_gpu_tensor_write(up_t, 0, pair_init, + pair_bytes + guard_bytes); + ok = ok && ds4_gpu_tensor_write(mid_t, 0, pair_init, + pair_bytes + guard_bytes); + ok = ok && ds4_gpu_tensor_write(experts_t, 0, expert_init, + expert_bytes + guard_bytes); + ok = ok && ds4_gpu_tensor_write(out_t, 0, out_init, + out_bytes + guard_bytes); + + bool mid_is_f16 = true; + if (ok) { + ok = ds4_gpu_routed_moe_batch_tensor( + out_t, gate_t, up_t, mid_t, experts_t, model, model_size, + gate_offset, up_offset, down_offset, IQ2_XXS_TYPE, Q2_K_TYPE, + gate_expert_bytes, gate_row_bytes, down_expert_bytes, + down_row_bytes, FULL_IN_DIM, FULL_MID_DIM, FULL_OUT_DIM, + selected_t, weights_t, N_TOTAL_EXPERT, N_EXPERT, CLAMP, x_t, + 0u, FULL_TOKENS, &mid_is_f16, false); + } + if (ok && mid_is_f16) { + fprintf(stderr, + "IQ2_XXS SSD full-shape %s unexpectedly selected f16 mid\n", + run_name); + ok = 0; + } + if (ok) { + ok = ds4_gpu_tensor_read(gate_t, 0, result->gate, pair_bytes) && + ds4_gpu_tensor_read(up_t, 0, result->up, pair_bytes) && + ds4_gpu_tensor_read(mid_t, 0, result->mid, pair_bytes) && + ds4_gpu_tensor_read(out_t, 0, result->out, out_bytes); + } + + result->guard_mismatches = 0; + if (gate_t) { + const int guard_ok = full_check_guard( + run_name, "gate", gate_t, result->pair_count, + &result->guard_mismatches); + ok = guard_ok && ok; + } + if (up_t) { + const int guard_ok = full_check_guard( + run_name, "up", up_t, result->pair_count, + &result->guard_mismatches); + ok = guard_ok && ok; + } + if (mid_t) { + const int guard_ok = full_check_guard( + run_name, "mid", mid_t, result->pair_count, + &result->guard_mismatches); + ok = guard_ok && ok; + } + if (experts_t) { + const int guard_ok = full_check_guard( + run_name, "experts", experts_t, expert_count, + &result->guard_mismatches); + ok = guard_ok && ok; + } + if (out_t) { + const int guard_ok = full_check_guard( + run_name, "out", out_t, result->out_count, + &result->guard_mismatches); + ok = guard_ok && ok; + } + + ds4_gpu_tensor_free(x_t); + ds4_gpu_tensor_free(selected_t); + ds4_gpu_tensor_free(weights_t); + ds4_gpu_tensor_free(gate_t); + ds4_gpu_tensor_free(up_t); + ds4_gpu_tensor_free(mid_t); + ds4_gpu_tensor_free(experts_t); + ds4_gpu_tensor_free(out_t); + free(pair_init); + free(expert_init); + free(out_init); + return ok; +} + +static int compare_full_results(const full_run_result *candidate, + const full_run_result *control) { + if (candidate->pair_count != control->pair_count || + candidate->out_count != control->out_count || + candidate->guard_mismatches != 0 || + control->guard_mismatches != 0) { + return 0; + } + + /* The first M1 production-shape run measured max errors below 0.0036 for + * every tensor (gate/up RMS below 0.00057, mid below 0.00018, out below + * 0.0016). The absolute bounds below retain roughly 3x max headroom and + * 2.5x-or-better RMS headroom for compiler/device variation while still + * rejecting a material change in accumulation or intermediate precision. */ + int ok = compare_array("full-4096", "gate", candidate->gate, + control->gate, candidate->pair_count, + 0.012, 0.002); + ok = compare_array("full-4096", "up", candidate->up, + control->up, candidate->pair_count, + 0.012, 0.002) && ok; + ok = compare_array("full-4096", "mid", candidate->mid, + control->mid, candidate->pair_count, + 0.012, 0.001) && ok; + ok = compare_array("full-4096", "out", candidate->out, + control->out, candidate->out_count, + 0.012, 0.004) && ok; + return ok; +} + +static int run_full_pair( + const void *model, + uint64_t model_size, + uint64_t gate_offset, + uint64_t up_offset, + uint64_t down_offset, + uint64_t gate_expert_bytes, + uint64_t gate_row_bytes, + uint64_t down_expert_bytes, + uint64_t down_row_bytes, + const float *x, + const int32_t *selected, + const float *weights) { + full_run_result control; + full_run_result candidate; + memset(&control, 0, sizeof(control)); + memset(&candidate, 0, sizeof(candidate)); + if (!full_result_alloc(&control) || !full_result_alloc(&candidate)) { + full_result_free(&control); + full_result_free(&candidate); + return 0; + } + + unsetenv("DS4_METAL_ENABLE_IQ2_XXS_SSD_PREFILL_MM"); + unsetenv("DS4_METAL_REQUIRE_IQ2_XXS_SSD_PREFILL_MM"); + setenv("DS4_METAL_DISABLE_IQ2_XXS_SSD_PREFILL_MM", "1", 1); + ds4_gpu_set_streaming_expert_cache_budget(N_TOTAL_EXPERT); + const int control_ok = run_full_once( + "control", &control, model, model_size, gate_offset, up_offset, + down_offset, gate_expert_bytes, gate_row_bytes, down_expert_bytes, + down_row_bytes, x, selected, weights); + + unsetenv("DS4_METAL_DISABLE_IQ2_XXS_SSD_PREFILL_MM"); + setenv("DS4_METAL_ENABLE_IQ2_XXS_SSD_PREFILL_MM", "1", 1); + setenv("DS4_METAL_REQUIRE_IQ2_XXS_SSD_PREFILL_MM", "1", 1); + ds4_gpu_set_streaming_expert_cache_budget(N_TOTAL_EXPERT); + const int candidate_ok = run_full_once( + "candidate", &candidate, model, model_size, gate_offset, up_offset, + down_offset, gate_expert_bytes, gate_row_bytes, down_expert_bytes, + down_row_bytes, x, selected, weights); + + int ok = control_ok && candidate_ok; + if (control_ok && candidate_ok) { + ok = compare_full_results(&candidate, &control); + } + full_result_free(&control); + full_result_free(&candidate); + return ok; +} + +static int run_full_shape_oracle(const void *restore_model, + uint64_t restore_model_size, + int restore_model_fd, + uint64_t restore_cache_expert_bytes) { + const uint64_t page = (uint64_t)getpagesize(); + const uint64_t gate_row_bytes = + (FULL_IN_DIM / QK_K) * sizeof(block_iq2_xxs); + const uint64_t gate_expert_bytes = + (uint64_t)FULL_MID_DIM * gate_row_bytes; + const uint64_t gate_tensor_bytes = + (uint64_t)N_TOTAL_EXPERT * gate_expert_bytes; + const uint64_t down_row_bytes = + (FULL_MID_DIM / QK_K) * sizeof(block_q2_K); + const uint64_t down_expert_bytes = + (uint64_t)FULL_OUT_DIM * down_row_bytes; + const uint64_t down_tensor_bytes = + (uint64_t)N_TOTAL_EXPERT * down_expert_bytes; + const uint64_t gate_offset = 0; + const uint64_t up_offset = align_up(gate_tensor_bytes, page); + const uint64_t down_offset = + align_up(up_offset + gate_tensor_bytes, page); + const uint64_t full_model_size = + align_up(down_offset + down_tensor_bytes, page); + + if (gate_row_bytes != 1056u || gate_expert_bytes != 2162688u || + down_row_bytes != 672u || down_expert_bytes != 2752512u) { + fprintf(stderr, + "IQ2_XXS SSD full-shape layout FAIL gate_row=%llu " + "gate_expert=%llu down_row=%llu down_expert=%llu\n", + (unsigned long long)gate_row_bytes, + (unsigned long long)gate_expert_bytes, + (unsigned long long)down_row_bytes, + (unsigned long long)down_expert_bytes); + return 0; + } + fprintf(stderr, + "IQ2_XXS SSD full-shape layout PASS in=%u mid=%u out=%u " + "tokens=%u topk=%u experts=%u gate_row=%llu down_row=%llu " + "model_bytes=%llu\n", + FULL_IN_DIM, FULL_MID_DIM, FULL_OUT_DIM, FULL_TOKENS, + N_EXPERT, N_TOTAL_EXPERT, + (unsigned long long)gate_row_bytes, + (unsigned long long)down_row_bytes, + (unsigned long long)full_model_size); + + int ok = 1; + int backend_switched = 0; + void *full_model = NULL; + float *x = NULL; + int32_t *selected = NULL; + float *weights = NULL; + char tmp_path[] = "/tmp/ds4-iq2-ssd-full-mm.XXXXXX"; + int full_fd = -1; + + if (posix_memalign(&full_model, (size_t)page, + (size_t)full_model_size) != 0) { + return 0; + } + memset(full_model, 0, (size_t)full_model_size); + block_iq2_xxs *gate = + (block_iq2_xxs *)((uint8_t *)full_model + gate_offset); + block_iq2_xxs *up = + (block_iq2_xxs *)((uint8_t *)full_model + up_offset); + block_q2_K *down = + (block_q2_K *)((uint8_t *)full_model + down_offset); + fill_iq2_full(gate, 19u); + fill_iq2_full(up, 47u); + fill_q2_full(down); + + const uint64_t x_count = (uint64_t)FULL_TOKENS * FULL_IN_DIM; + const uint64_t route_count = (uint64_t)FULL_TOKENS * N_EXPERT; + x = calloc((size_t)x_count, sizeof(float)); + selected = calloc((size_t)route_count, sizeof(int32_t)); + weights = calloc((size_t)route_count, sizeof(float)); + if (!x || !selected || !weights) { + ok = 0; + goto cleanup; + } + for (uint32_t token = 0; token < FULL_TOKENS; token++) { + for (uint32_t k = 0; k < FULL_IN_DIM; k++) { + const int32_t v = + (int32_t)((token * 43u + k * 29u + + (k >> 3u) * 17u) % 251u) - 125; + x[(uint64_t)token * FULL_IN_DIM + k] = (float)v / 256.0f; + } + for (uint32_t slot = 0; slot < N_EXPERT; slot++) { + const uint64_t route = (uint64_t)token * N_EXPERT + slot; + selected[route] = + (int32_t)((token * 3u + slot) % N_TOTAL_EXPERT); + weights[route] = (float)(slot + 1u) / 21.0f; + } + } + + full_fd = mkstemp(tmp_path); + if (full_fd < 0 || ftruncate(full_fd, (off_t)full_model_size) != 0) { + fprintf(stderr, "IQ2_XXS SSD full-shape fixture creation FAIL\n"); + ok = 0; + goto cleanup; + } + uint64_t written = 0; + while (written < full_model_size) { + const size_t chunk = full_model_size - written > (1u << 20) ? + (1u << 20) : (size_t)(full_model_size - written); + const ssize_t n = pwrite(full_fd, + (const uint8_t *)full_model + written, + chunk, + (off_t)written); + if (n <= 0) { + fprintf(stderr, "IQ2_XXS SSD full-shape fixture write FAIL\n"); + ok = 0; + goto cleanup; + } + written += (uint64_t)n; + } + + /* The streaming cache deliberately freezes one expert-size slab class. + * Re-seed it for the production 6.75 MiB expert after clearing the compact + * fixture, then restore the compact class before returning. */ + ds4_gpu_set_streaming_expert_cache_budget(N_TOTAL_EXPERT); + ds4_gpu_set_streaming_expert_cache_expert_bytes( + 2u * gate_expert_bytes + down_expert_bytes); + if (!ds4_gpu_set_model_fd(full_fd)) { + ok = 0; + goto cleanup; + } + backend_switched = 1; + if (!ds4_gpu_set_model_map(full_model, full_model_size)) { + ok = 0; + goto cleanup; + } + + uint64_t before_candidates = 0; + uint64_t before_calls = 0; + uint64_t before_tokens = 0; + uint64_t before_rows = 0; + uint64_t before_failures = 0; + uint32_t before_min = 0; + uint32_t before_max = 0; + if (!ds4_gpu_test_iq2_stream_addr_mm_stats( + &before_candidates, &before_calls, &before_tokens, &before_rows, + &before_failures, &before_min, &before_max)) { + ok = 0; + goto cleanup; + } + + const int pair_ok = run_full_pair( + full_model, full_model_size, gate_offset, up_offset, down_offset, + gate_expert_bytes, gate_row_bytes, down_expert_bytes, down_row_bytes, + x, selected, weights); + + uint64_t after_candidates = 0; + uint64_t after_calls = 0; + uint64_t after_tokens = 0; + uint64_t after_rows = 0; + uint64_t after_failures = 0; + uint32_t after_min = 0; + uint32_t after_max = 0; + const int stats_ok = ds4_gpu_test_iq2_stream_addr_mm_stats( + &after_candidates, &after_calls, &after_tokens, &after_rows, + &after_failures, &after_min, &after_max); + const int monotonic = stats_ok && + after_candidates >= before_candidates && + after_calls >= before_calls && + after_tokens >= before_tokens && + after_rows >= before_rows && + after_failures >= before_failures; + const uint64_t delta_candidates = monotonic ? + after_candidates - before_candidates : UINT64_MAX; + const uint64_t delta_calls = monotonic ? + after_calls - before_calls : UINT64_MAX; + const uint64_t delta_tokens = monotonic ? + after_tokens - before_tokens : UINT64_MAX; + const uint64_t delta_rows = monotonic ? + after_rows - before_rows : UINT64_MAX; + const uint64_t delta_failures = monotonic ? + after_failures - before_failures : UINT64_MAX; + const int coverage_ok = monotonic && + delta_candidates == 2u && + delta_calls == 1u && + delta_tokens == FULL_TOKENS && + delta_rows == (uint64_t)FULL_TOKENS * N_EXPERT && + delta_failures == 0u; + fprintf(stderr, + "IQ2_XXS SSD full-shape coverage %s candidates=%llu calls=%llu " + "tokens=%llu rows=%llu require_failures=%llu " + "global_min=%u->%u global_max=%u->%u\n", + coverage_ok ? "PASS" : "FAIL", + (unsigned long long)delta_candidates, + (unsigned long long)delta_calls, + (unsigned long long)delta_tokens, + (unsigned long long)delta_rows, + (unsigned long long)delta_failures, + before_min, after_min, before_max, after_max); + ok = pair_ok && coverage_ok && ok; + +cleanup: + unsetenv("DS4_METAL_ENABLE_IQ2_XXS_SSD_PREFILL_MM"); + unsetenv("DS4_METAL_REQUIRE_IQ2_XXS_SSD_PREFILL_MM"); + unsetenv("DS4_METAL_DISABLE_IQ2_XXS_SSD_PREFILL_MM"); + if (backend_switched) { + ds4_gpu_set_streaming_expert_cache_budget(N_TOTAL_EXPERT); + ds4_gpu_set_streaming_expert_cache_expert_bytes( + restore_cache_expert_bytes); + const int fd_ok = ds4_gpu_set_model_fd(restore_model_fd); + const int map_ok = ds4_gpu_set_model_map( + restore_model, restore_model_size); + if (!fd_ok || !map_ok) { + fprintf(stderr, + "IQ2_XXS SSD full-shape backend restoration FAIL\n"); + ok = 0; + } + } + if (full_fd >= 0) close(full_fd); + if (full_fd >= 0) unlink(tmp_path); + free(x); + free(selected); + free(weights); + free(full_model); + fprintf(stderr, + "IQ2_XXS/Q2_K Metal SSD full production-shape oracle %s\n", + ok ? "PASS" : "FAIL"); + return ok; +} + +static int run_256_expert_oracle( + const void *restore_model, + uint64_t restore_model_size, + int restore_model_fd, + uint64_t restore_gate_expert_bytes, + uint64_t restore_gate_row_bytes, + uint64_t restore_down_expert_bytes, + uint64_t restore_down_row_bytes) { + const uint64_t page = (uint64_t)getpagesize(); + const uint64_t gate_row_bytes = sizeof(block_iq2_xxs); + const uint64_t gate_expert_bytes = MID_DIM * gate_row_bytes; + const uint64_t gate_tensor_bytes = + (uint64_t)N_TOTAL_EXPERT_256 * gate_expert_bytes; + const uint64_t down_row_bytes = sizeof(block_q2_K); + const uint64_t down_expert_bytes = OUT_DIM * down_row_bytes; + const uint64_t down_tensor_bytes = + (uint64_t)N_TOTAL_EXPERT_256 * down_expert_bytes; + const uint64_t gate_offset = 0; + const uint64_t up_offset = align_up(gate_tensor_bytes, page); + const uint64_t down_offset = + align_up(up_offset + gate_tensor_bytes, page); + const uint64_t model_size = + align_up(down_offset + down_tensor_bytes, page); + + int ok = 1; + int backend_switched = 0; + void *model = NULL; + float *x = NULL; + int32_t *selected = NULL; + float *weights = NULL; + char tmp_path[] = "/tmp/ds4-iq2-ssd-256-mm.XXXXXX"; + int model_fd = -1; + + if (posix_memalign(&model, (size_t)page, (size_t)model_size) != 0) { + return 0; + } + memset(model, 0, (size_t)model_size); + block_iq2_xxs *gate = + (block_iq2_xxs *)((uint8_t *)model + gate_offset); + block_iq2_xxs *up = + (block_iq2_xxs *)((uint8_t *)model + up_offset); + block_q2_K *down = + (block_q2_K *)((uint8_t *)model + down_offset); + fill_iq2(gate, 23u, N_TOTAL_EXPERT_256); + fill_iq2(up, 53u, N_TOTAL_EXPERT_256); + fill_q2(down, N_TOTAL_EXPERT_256); + + const uint32_t tokens = 33u; + const uint64_t x_count = (uint64_t)tokens * IN_DIM; + const uint64_t route_count = (uint64_t)tokens * N_EXPERT; + x = calloc((size_t)x_count, sizeof(float)); + selected = calloc((size_t)route_count, sizeof(int32_t)); + weights = calloc((size_t)route_count, sizeof(float)); + if (!x || !selected || !weights) { + ok = 0; + goto cleanup; + } + uint32_t high_id_routes = 0; + for (uint32_t token = 0; token < tokens; token++) { + for (uint32_t k = 0; k < IN_DIM; k++) { + const int32_t v = + (int32_t)((token * 31u + k * 17u + (k >> 2u) * 7u) % + 127u) - 63; + x[(uint64_t)token * IN_DIM + k] = (float)v / 96.0f; + } + for (uint32_t slot = 0; slot < N_EXPERT; slot++) { + const uint64_t route = (uint64_t)token * N_EXPERT + slot; + selected[route] = slot == 0u ? (int32_t)HIGH_EXPERT_ID : + (int32_t)(slot - 1u); + weights[route] = (float)(slot + 1u) / 21.0f; + if (selected[route] == (int32_t)HIGH_EXPERT_ID) high_id_routes++; + } + } + const uint32_t high_id_work_items = (high_id_routes + 31u) / 32u; + if (high_id_routes < 33u || high_id_work_items < 2u) { + fprintf(stderr, + "IQ2_XXS SSD 256-expert route construction FAIL " + "id255_rows=%u work_items=%u\n", + high_id_routes, high_id_work_items); + ok = 0; + goto cleanup; + } + fprintf(stderr, + "IQ2_XXS SSD 256-expert address table prepared model_bytes=%llu " + "max_id=%u id255_rows=%u work_items=%u second_tile_r1=32\n", + (unsigned long long)model_size, HIGH_EXPERT_ID, + high_id_routes, high_id_work_items); + + model_fd = mkstemp(tmp_path); + if (model_fd < 0 || ftruncate(model_fd, (off_t)model_size) != 0) { + ok = 0; + goto cleanup; + } + uint64_t written = 0; + while (written < model_size) { + const size_t chunk = model_size - written > (1u << 20) ? + (1u << 20) : (size_t)(model_size - written); + const ssize_t n = pwrite(model_fd, + (const uint8_t *)model + written, + chunk, + (off_t)written); + if (n <= 0) { + ok = 0; + goto cleanup; + } + written += (uint64_t)n; + } + + ds4_gpu_set_streaming_expert_cache_budget(N_TOTAL_EXPERT_256); + ds4_gpu_set_streaming_expert_cache_expert_bytes( + 2u * gate_expert_bytes + down_expert_bytes); + if (!ds4_gpu_set_model_fd(model_fd)) { + ok = 0; + goto cleanup; + } + backend_switched = 1; + if (!ds4_gpu_set_model_map(model, model_size)) { + ok = 0; + goto cleanup; + } + + mm_stats_snapshot before; + mm_stats_snapshot after; + if (!read_mm_stats(&before)) { + ok = 0; + goto cleanup; + } + const int pair_ok = run_pair( + "id255-hot-33", tokens, model, model_size, + gate_offset, up_offset, down_offset, + gate_expert_bytes, gate_row_bytes, + down_expert_bytes, down_row_bytes, + N_TOTAL_EXPERT_256, N_TOTAL_EXPERT_256, + x, selected, weights); + const int stats_ok = read_mm_stats(&after) && + check_mm_stats_delta("id255-hot-33", &before, &after, tokens); + ok = pair_ok && stats_ok && ok; + +cleanup: + unsetenv("DS4_METAL_ENABLE_IQ2_XXS_SSD_PREFILL_MM"); + unsetenv("DS4_METAL_REQUIRE_IQ2_XXS_SSD_PREFILL_MM"); + unsetenv("DS4_METAL_DISABLE_IQ2_XXS_SSD_PREFILL_MM"); + if (backend_switched) { + ds4_gpu_set_streaming_expert_cache_budget(N_TOTAL_EXPERT); + ds4_gpu_set_streaming_expert_cache_expert_bytes( + 2u * restore_gate_expert_bytes + restore_down_expert_bytes); + const int fd_ok = ds4_gpu_set_model_fd(restore_model_fd); + const int map_ok = ds4_gpu_set_model_map( + restore_model, restore_model_size); + if (!fd_ok || !map_ok || + restore_gate_row_bytes != sizeof(block_iq2_xxs) || + restore_down_row_bytes != sizeof(block_q2_K)) { + fprintf(stderr, + "IQ2_XXS SSD 256-expert backend restoration FAIL\n"); + ok = 0; + } + } + if (model_fd >= 0) close(model_fd); + if (model_fd >= 0) unlink(tmp_path); + free(x); + free(selected); + free(weights); + free(model); + fprintf(stderr, + "IQ2_XXS/Q2_K Metal compact 256-expert ID255 oracle %s\n", + ok ? "PASS" : "FAIL"); + return ok; +} + +int main(void) { + if (sizeof(block_iq2_xxs) != 66u || sizeof(block_q2_K) != 84u) { + fprintf(stderr, + "IQ2_XXS SSD grouped-MM unexpected block sizes iq2=%zu q2=%zu\n", + sizeof(block_iq2_xxs), sizeof(block_q2_K)); + return 1; + } + + const uint64_t gate_row_bytes = sizeof(block_iq2_xxs); + const uint64_t gate_expert_bytes = MID_DIM * gate_row_bytes; + const uint64_t gate_tensor_bytes = N_TOTAL_EXPERT * gate_expert_bytes; + const uint64_t down_row_bytes = sizeof(block_q2_K); + const uint64_t down_expert_bytes = OUT_DIM * down_row_bytes; + const uint64_t down_tensor_bytes = N_TOTAL_EXPERT * down_expert_bytes; + const uint64_t page = (uint64_t)getpagesize(); + const uint64_t gate_offset = 0; + const uint64_t up_offset = align_up(gate_tensor_bytes, page); + const uint64_t down_offset = align_up(up_offset + gate_tensor_bytes, page); + const uint64_t model_size = align_up(down_offset + down_tensor_bytes, page); + + void *model = NULL; + if (posix_memalign(&model, (size_t)page, (size_t)model_size) != 0) { + return 1; + } + memset(model, 0, (size_t)model_size); + block_iq2_xxs *gate = + (block_iq2_xxs *)((uint8_t *)model + gate_offset); + block_iq2_xxs *up = + (block_iq2_xxs *)((uint8_t *)model + up_offset); + block_q2_K *down = (block_q2_K *)((uint8_t *)model + down_offset); + fill_iq2(gate, 3u, N_TOTAL_EXPERT); + fill_iq2(up, 11u, N_TOTAL_EXPERT); + fill_q2(down, N_TOTAL_EXPERT); + + float *x = calloc((size_t)MAX_TOKENS * IN_DIM, sizeof(float)); + int32_t *selected = + calloc((size_t)MAX_TOKENS * N_EXPERT, sizeof(int32_t)); + int32_t *selected_duplicate = + calloc((size_t)MAX_TOKENS * N_EXPERT, sizeof(int32_t)); + int32_t *selected_hot = + calloc((size_t)MAX_TOKENS * N_EXPERT, sizeof(int32_t)); + float *weights = + calloc((size_t)MAX_TOKENS * N_EXPERT, sizeof(float)); + int ok = x && selected && selected_duplicate && selected_hot && weights; + for (uint32_t token = 0; ok && token < MAX_TOKENS; token++) { + for (uint32_t k = 0; k < IN_DIM; k++) { + const int32_t v = (int32_t)((token * 17u + k * 11u + + (token ^ k) * 3u) % 97u) - 48; + x[(uint64_t)token * IN_DIM + k] = (float)v / 64.0f; + } + for (uint32_t slot = 0; slot < N_EXPERT; slot++) { + const uint64_t route = (uint64_t)token * N_EXPERT + slot; + selected[route] = (int32_t)((token + slot) % N_TOTAL_EXPERT); + selected_duplicate[route] = selected[route]; + /* Expert zero receives exactly one route per token. At 33 tokens + * this forces work items at r1=0 and r1=32 without relying on + * duplicate IDs inside one token. */ + selected_hot[route] = slot == 0u ? 0 : + (int32_t)(1u + ((token + slot - 1u) % + (N_TOTAL_EXPERT - 1u))); + weights[route] = 1.0f / N_EXPERT; + } + selected_duplicate[(uint64_t)token * N_EXPERT + N_EXPERT - 1u] = + selected_duplicate[(uint64_t)token * N_EXPERT]; + } + + char tmp_path[] = "/tmp/ds4-iq2-ssd-grouped-mm.XXXXXX"; + int model_fd = ok ? mkstemp(tmp_path) : -1; + if (model_fd < 0 || ftruncate(model_fd, (off_t)model_size) != 0) { + fprintf(stderr, "IQ2_XXS SSD grouped-MM could not create fixture\n"); + ok = 0; + } + uint64_t written = 0; + while (ok && written < model_size) { + const size_t chunk = model_size - written > (1u << 20) ? + (1u << 20) : (size_t)(model_size - written); + const ssize_t n = pwrite(model_fd, (const uint8_t *)model + written, + chunk, (off_t)written); + if (n <= 0) { + fprintf(stderr, "IQ2_XXS SSD grouped-MM fixture write failed\n"); + ok = 0; + } else { + written += (uint64_t)n; + } + } + + unsetenv("DS4_METAL_DISABLE_STREAMING_PREFILL_BATCH_SELECTED_ADDR"); + unsetenv("DS4_METAL_DISABLE_STREAMING_EXPERT_ADDR_TABLE"); + unsetenv("DS4_METAL_DISABLE_ROUTED_PAIR_SWIGLU_FUSION"); + unsetenv("DS4_METAL_MOE_WRITE_CLAMPED_ACT"); + unsetenv("DS4_METAL_GRAPH_DUMP_PREFIX"); + setenv("DS4_METAL_ENABLE_STREAMING_PREFILL_BATCH_SELECTED_ADDR", "1", 1); + setenv("DS4_METAL_ENABLE_STREAMING_EXPERT_ADDR_TABLE", "1", 1); + setenv("DS4_METAL_IQ2_XXS_SSD_PREFILL_MM_STATS", "1", 1); + + ok = ok && ds4_gpu_init() && ds4_gpu_set_model_map(model, model_size); + ds4_gpu_set_quality(false); + ds4_gpu_set_ssd_streaming(true); + ds4_gpu_set_streaming_expert_cache_budget(N_TOTAL_EXPERT); + ok = ok && ds4_gpu_set_model_fd(model_fd); + const int backend_ready = ok; + + if (backend_ready) { + const int case_ok = run_pair("tile-32", 32u, model, model_size, + gate_offset, up_offset, down_offset, + gate_expert_bytes, gate_row_bytes, + down_expert_bytes, down_row_bytes, + N_TOTAL_EXPERT, N_TOTAL_EXPERT, + x, selected, weights); + ok = case_ok && ok; + } + if (backend_ready) { + const int case_ok = run_pair("tail-33", 33u, model, model_size, + gate_offset, up_offset, down_offset, + gate_expert_bytes, gate_row_bytes, + down_expert_bytes, down_row_bytes, + N_TOTAL_EXPERT, N_TOTAL_EXPERT, + x, selected, weights); + ok = case_ok && ok; + } + if (backend_ready) { + const int case_ok = run_pair("duplicate-32", 32u, model, model_size, + gate_offset, up_offset, down_offset, + gate_expert_bytes, gate_row_bytes, + down_expert_bytes, down_row_bytes, + N_TOTAL_EXPERT, N_TOTAL_EXPERT, + x, selected_duplicate, weights); + ok = case_ok && ok; + } + + uint64_t candidate_calls = 0; + uint64_t calls = 0; + uint64_t tokens = 0; + uint64_t rows = 0; + uint64_t require_failures = 0; + uint32_t min_tokens = 0; + uint32_t max_tokens = 0; + const int stats_ok = ds4_gpu_test_iq2_stream_addr_mm_stats( + &candidate_calls, &calls, &tokens, &rows, &require_failures, + &min_tokens, &max_tokens); + const int counters_ok = + candidate_calls == 6u && calls == 3u && tokens == 97u && + rows == 582u && require_failures == 0u && + min_tokens == 32u && max_tokens == 33u; + fprintf(stderr, + "IQ2_XXS SSD grouped-MM coverage %s candidates=%llu calls=%llu " + "tokens=%llu rows=%llu require_failures=%llu min=%u max=%u\n", + counters_ok ? "PASS" : "FAIL", + (unsigned long long)candidate_calls, + (unsigned long long)calls, + (unsigned long long)tokens, + (unsigned long long)rows, + (unsigned long long)require_failures, + min_tokens, max_tokens); + ok = stats_ok && counters_ok && ok; + + if (backend_ready) { + uint32_t hot_rows = 0; + for (uint32_t token = 0; token < 33u; token++) { + for (uint32_t slot = 0; slot < N_EXPERT; slot++) { + if (selected_hot[(uint64_t)token * N_EXPERT + slot] == 0) { + hot_rows++; + } + } + } + const uint32_t hot_work_items = (hot_rows + 31u) / 32u; + const int route_shape_ok = hot_rows >= 33u && hot_work_items >= 2u; + fprintf(stderr, + "IQ2_XXS SSD compact hot-route %s expert=0 rows=%u " + "work_items=%u second_tile_r1=32\n", + route_shape_ok ? "PASS" : "FAIL", + hot_rows, hot_work_items); + mm_stats_snapshot before; + mm_stats_snapshot after; + const int before_ok = read_mm_stats(&before); + const int pair_ok = route_shape_ok && run_pair( + "hot-r1-32", 33u, model, model_size, + gate_offset, up_offset, down_offset, + gate_expert_bytes, gate_row_bytes, + down_expert_bytes, down_row_bytes, + N_TOTAL_EXPERT, N_TOTAL_EXPERT, + x, selected_hot, weights); + const int delta_ok = before_ok && read_mm_stats(&after) && + check_mm_stats_delta("hot-r1-32", &before, &after, 33u); + ok = pair_ok && delta_ok && ok; + } + + if (backend_ready) { + const int id255_ok = run_256_expert_oracle( + model, model_size, model_fd, + gate_expert_bytes, gate_row_bytes, + down_expert_bytes, down_row_bytes); + ok = id255_ok && ok; + } + + /* Keep the full production geometry as a distinct oracle and report its + * counter deltas independently from the compact 32/33 smoke cases. */ + if (backend_ready) { + const int full_ok = + run_full_shape_oracle( + model, model_size, model_fd, + 2u * gate_expert_bytes + down_expert_bytes); + ok = full_ok && ok; + } + + unsetenv("DS4_METAL_ENABLE_IQ2_XXS_SSD_PREFILL_MM"); + unsetenv("DS4_METAL_REQUIRE_IQ2_XXS_SSD_PREFILL_MM"); + unsetenv("DS4_METAL_DISABLE_IQ2_XXS_SSD_PREFILL_MM"); + ds4_gpu_set_model_fd(-1); + ds4_gpu_set_ssd_streaming(false); + ds4_gpu_cleanup(); + if (model_fd >= 0) close(model_fd); + if (tmp_path[0]) unlink(tmp_path); + free(x); + free(selected); + free(selected_duplicate); + free(selected_hot); + free(weights); + free(model); + + fprintf(stderr, "IQ2_XXS/Q2_K Metal SSD grouped address-MM %s\n", + ok ? "PASS" : "FAIL"); + return ok ? 0 : 1; +} + +#else + +int main(void) { + fprintf(stderr, + "test_metal_iq2_ssd_grouped_mm: skipped (Metal requires macOS)\n"); + return 0; +} + +#endif From 6ed2199e657ce74ecdc979e9bd6ddbb330e6c633 Mon Sep 17 00:00:00 2001 From: Giorgio Oppo Date: Fri, 14 Aug 2026 15:16:27 +0200 Subject: [PATCH 027/189] metal: isolate stream state and persistent exact-row cache --- ds4_gpu.h | 36 ++ ds4_metal.m | 1220 +++++++++++++++++++++++++++++++++++++++------------ 2 files changed, 971 insertions(+), 285 deletions(-) diff --git a/ds4_gpu.h b/ds4_gpu.h index a461bb17b..e4595030a 100644 --- a/ds4_gpu.h +++ b/ds4_gpu.h @@ -114,6 +114,14 @@ int ds4_gpu_tensor_read_after_selected_event(const ds4_gpu_tensor *tensor, const char *label); #endif int ds4_gpu_end_commands(void); +#ifdef __APPLE__ +/* Metal-only asynchronous command streams. Command encoding remains + * serialized; at most eight committed streams may execute concurrently. */ +void ds4_gpu_set_stream(int idx); +int ds4_gpu_current_stream(void); +int ds4_gpu_end_commands_async(void); +int ds4_gpu_wait_stream(int idx); +#endif int ds4_gpu_synchronize(void); int ds4_gpu_set_model_map(const void *model_map, uint64_t model_size); @@ -233,6 +241,34 @@ int ds4_gpu_test_stream_expert_live_index_policy( int disable); void ds4_gpu_test_stream_expert_live_index_report( ds4_gpu_stream_expert_live_index_report *report); +typedef struct ds4_gpu_exact_rows_persistent_report { + uint64_t persistent_calls; + uint64_t transient_calls; + uint64_t persistent_fallbacks; + uint64_t persistent_failures; + uint64_t mapped_view_calls; + uint32_t max_unique; +} ds4_gpu_exact_rows_persistent_report; +/* Test-only policy and counters for exact-row private cache snapshots. */ +int ds4_gpu_test_exact_rows_persistent_policy( + uint32_t configured_count, + uint32_t unique_count, + int size_class_ok); +void ds4_gpu_test_exact_rows_persistent_report( + ds4_gpu_exact_rows_persistent_report *report); +typedef struct ds4_gpu_stream_test_stats { + uint64_t tensor_live_bytes; + uint64_t transient_references; + uint32_t tensor_live_count; + uint32_t pending_command_buffers; + uint32_t last_command_buffers; + uint32_t active_queue_mask; + uint32_t model_residency_queue_mask; + uint32_t q4_residency_queue_mask; +} ds4_gpu_stream_test_stats; +/* Test-only observability and lifetime injection for Metal stream oracles. */ +void ds4_gpu_test_stream_stats(ds4_gpu_stream_test_stats *stats); +int ds4_gpu_test_hold_stream_transient(uint64_t bytes); enum { DS4_GPU_TEST_MXFP4_PAIR_TAIL_CULL = 1u << 0, DS4_GPU_TEST_MXFP4_PAIR_COMPACT_TILE = 1u << 1, diff --git a/ds4_metal.m b/ds4_metal.m index 0d55e660d..f71e3858b 100644 --- a/ds4_metal.m +++ b/ds4_metal.m @@ -53,14 +53,20 @@ @class DS4MetalQ4ExpertTable; static id g_device; +#define DS4_GPU_MAX_STREAMS 8 +static _Thread_local int g_ds4_stream = 0; static id g_queue; +static id g_stream_queues[DS4_GPU_MAX_STREAMS]; +static id g_stream_last_cb[DS4_GPU_MAX_STREAMS]; static id g_library; static id g_batch_cb; static id g_batch_enc; static BOOL g_batch_encoder_concurrent; static BOOL g_batch_has_work; static void ds4_gpu_parallel_ffn_reset_state(BOOL close_encoder); -static NSMutableArray> *g_pending_cbs; +static NSMutableArray> + *g_pending_cbs_by_stream[DS4_GPU_MAX_STREAMS]; +#define g_pending_cbs (g_pending_cbs_by_stream[g_ds4_stream]) static id g_selected_readback_event; static uint64_t g_selected_readback_event_value; static id g_set_rows_f32_i32_pipeline; @@ -309,7 +315,9 @@ static NSMutableDictionary> *g_model_buffer_cache; static NSMutableDictionary *g_q4_expert_table_cache; static NSMutableDictionary *g_q4_expert_layer_residency_cache; -static NSMutableArray> *g_transient_buffers; +static NSMutableArray> + *g_transient_buffers_by_stream[DS4_GPU_MAX_STREAMS]; +#define g_transient_buffers (g_transient_buffers_by_stream[g_ds4_stream]) static id g_model_residency_set; typedef struct { @@ -341,36 +349,133 @@ static ds4_gpu_zero_prefix_prefill_mask_cache_entry g_zero_prefix_prefill_mask_cache[DS4_GPU_PREFILL_MASK_CACHE_SLOTS]; static void ds4_gpu_invalidate_zero_prefix_prefill_block_maps(void); -static id g_flash_attn_mask_buffer; -static id g_flash_attn_zero_mask_buffer; -static id g_flash_attn_pad_buffer; -static id g_flash_attn_tmp_buffer; -static id g_flash_attn_blk_buffer; -static id g_flash_attn_ring_buffer; -static id g_flash_attn_kv_buffer; -static id g_glm_flash_attn_mask_buffer; -static id g_compressor_pool_kv_buffer; -static id g_compressor_pool_score_buffer; -static id g_compressor_pool_score_cont_buffer; -static id g_compressor_pool_softmax_buffer; -static id g_compressor_pool_product_buffer; -static id g_compressor_store_ape_buffer; -static id g_compressor_store_score_buffer; -static id g_embed_rows_buffer; -static id g_router_selection_buffer; -static id g_router_weight_sum_buffer; -static id g_indexer_head_scores_buffer; -static id g_indexer_topk_buffer; -static id g_indexed_topk_buffer; -static id g_f16_round_scratch_buffer; -static id g_raw_store_round_buffer; -static id g_moe_gate_scratch_buffer; -static id g_moe_down_scratch_buffer; -static id g_moe_id_map_buffer; -static id g_moe_q4_gate_slots_buffer; -static id g_moe_q4_up_slots_buffer; -static id g_moe_q4_down_slots_buffer; -static id g_attn_out_group_ids_buffer; +typedef struct { + __strong id flash_attn_mask_buffer; + __strong id flash_attn_zero_mask_buffer; + __strong id flash_attn_pad_buffer; + __strong id flash_attn_tmp_buffer; + __strong id flash_attn_blk_buffer; + __strong id flash_attn_ring_buffer; + __strong id flash_attn_kv_buffer; + __strong id glm_flash_attn_mask_buffer; + __strong id compressor_pool_kv_buffer; + __strong id compressor_pool_score_buffer; + __strong id compressor_pool_score_cont_buffer; + __strong id compressor_pool_softmax_buffer; + __strong id compressor_pool_product_buffer; + __strong id compressor_store_ape_buffer; + __strong id compressor_store_score_buffer; + __strong id embed_rows_buffer; + __strong id router_selection_buffer; + __strong id router_weight_sum_buffer; + __strong id indexer_head_scores_buffer; + __strong id indexer_topk_buffer; + __strong id indexed_topk_buffer; + __strong id f16_round_scratch_buffer; + __strong id raw_store_round_buffer; + __strong id moe_gate_scratch_buffer; + __strong id moe_down_scratch_buffer; + __strong id moe_id_map_buffer; + __strong id moe_q4_gate_slots_buffer; + __strong id moe_q4_up_slots_buffer; + __strong id moe_q4_down_slots_buffer; + __strong id attn_out_group_ids_buffer; + NSUInteger flash_attn_mask_bytes; + NSUInteger flash_attn_zero_mask_bytes; + NSUInteger flash_attn_pad_bytes; + NSUInteger flash_attn_tmp_bytes; + NSUInteger flash_attn_blk_bytes; + NSUInteger flash_attn_ring_bytes; + NSUInteger flash_attn_kv_bytes; + NSUInteger glm_flash_attn_mask_bytes; + NSUInteger compressor_pool_kv_bytes; + NSUInteger compressor_pool_score_bytes; + NSUInteger compressor_pool_score_cont_bytes; + NSUInteger compressor_pool_softmax_bytes; + NSUInteger compressor_pool_product_bytes; + NSUInteger compressor_store_ape_bytes; + NSUInteger compressor_store_score_bytes; + NSUInteger embed_rows_bytes; + NSUInteger router_selection_bytes; + NSUInteger router_weight_sum_bytes; + NSUInteger indexer_head_scores_bytes; + NSUInteger indexer_topk_bytes; + NSUInteger indexed_topk_bytes; + NSUInteger f16_round_scratch_bytes; + NSUInteger raw_store_round_bytes; + NSUInteger moe_gate_scratch_bytes; + NSUInteger moe_down_scratch_bytes; + NSUInteger moe_id_map_bytes; + NSUInteger moe_q4_gate_slots_bytes; + NSUInteger moe_q4_up_slots_bytes; + NSUInteger moe_q4_down_slots_bytes; + NSUInteger attn_out_group_ids_bytes; +} ds4_gpu_stream_scratch_state; + +static ds4_gpu_stream_scratch_state + g_stream_scratch[DS4_GPU_MAX_STREAMS]; + +#define DS4_STREAM_SCRATCH(field_) (g_stream_scratch[g_ds4_stream].field_) +#define g_flash_attn_mask_buffer DS4_STREAM_SCRATCH(flash_attn_mask_buffer) +#define g_flash_attn_zero_mask_buffer DS4_STREAM_SCRATCH(flash_attn_zero_mask_buffer) +#define g_flash_attn_pad_buffer DS4_STREAM_SCRATCH(flash_attn_pad_buffer) +#define g_flash_attn_tmp_buffer DS4_STREAM_SCRATCH(flash_attn_tmp_buffer) +#define g_flash_attn_blk_buffer DS4_STREAM_SCRATCH(flash_attn_blk_buffer) +#define g_flash_attn_ring_buffer DS4_STREAM_SCRATCH(flash_attn_ring_buffer) +#define g_flash_attn_kv_buffer DS4_STREAM_SCRATCH(flash_attn_kv_buffer) +#define g_glm_flash_attn_mask_buffer DS4_STREAM_SCRATCH(glm_flash_attn_mask_buffer) +#define g_compressor_pool_kv_buffer DS4_STREAM_SCRATCH(compressor_pool_kv_buffer) +#define g_compressor_pool_score_buffer DS4_STREAM_SCRATCH(compressor_pool_score_buffer) +#define g_compressor_pool_score_cont_buffer DS4_STREAM_SCRATCH(compressor_pool_score_cont_buffer) +#define g_compressor_pool_softmax_buffer DS4_STREAM_SCRATCH(compressor_pool_softmax_buffer) +#define g_compressor_pool_product_buffer DS4_STREAM_SCRATCH(compressor_pool_product_buffer) +#define g_compressor_store_ape_buffer DS4_STREAM_SCRATCH(compressor_store_ape_buffer) +#define g_compressor_store_score_buffer DS4_STREAM_SCRATCH(compressor_store_score_buffer) +#define g_embed_rows_buffer DS4_STREAM_SCRATCH(embed_rows_buffer) +#define g_router_selection_buffer DS4_STREAM_SCRATCH(router_selection_buffer) +#define g_router_weight_sum_buffer DS4_STREAM_SCRATCH(router_weight_sum_buffer) +#define g_indexer_head_scores_buffer DS4_STREAM_SCRATCH(indexer_head_scores_buffer) +#define g_indexer_topk_buffer DS4_STREAM_SCRATCH(indexer_topk_buffer) +#define g_indexed_topk_buffer DS4_STREAM_SCRATCH(indexed_topk_buffer) +#define g_f16_round_scratch_buffer DS4_STREAM_SCRATCH(f16_round_scratch_buffer) +#define g_raw_store_round_buffer DS4_STREAM_SCRATCH(raw_store_round_buffer) +#define g_moe_gate_scratch_buffer DS4_STREAM_SCRATCH(moe_gate_scratch_buffer) +#define g_moe_down_scratch_buffer DS4_STREAM_SCRATCH(moe_down_scratch_buffer) +#define g_moe_id_map_buffer DS4_STREAM_SCRATCH(moe_id_map_buffer) +#define g_moe_q4_gate_slots_buffer DS4_STREAM_SCRATCH(moe_q4_gate_slots_buffer) +#define g_moe_q4_up_slots_buffer DS4_STREAM_SCRATCH(moe_q4_up_slots_buffer) +#define g_moe_q4_down_slots_buffer DS4_STREAM_SCRATCH(moe_q4_down_slots_buffer) +#define g_attn_out_group_ids_buffer DS4_STREAM_SCRATCH(attn_out_group_ids_buffer) +#define g_flash_attn_mask_bytes DS4_STREAM_SCRATCH(flash_attn_mask_bytes) +#define g_flash_attn_zero_mask_bytes DS4_STREAM_SCRATCH(flash_attn_zero_mask_bytes) +#define g_flash_attn_pad_bytes DS4_STREAM_SCRATCH(flash_attn_pad_bytes) +#define g_flash_attn_tmp_bytes DS4_STREAM_SCRATCH(flash_attn_tmp_bytes) +#define g_flash_attn_blk_bytes DS4_STREAM_SCRATCH(flash_attn_blk_bytes) +#define g_flash_attn_ring_bytes DS4_STREAM_SCRATCH(flash_attn_ring_bytes) +#define g_flash_attn_kv_bytes DS4_STREAM_SCRATCH(flash_attn_kv_bytes) +#define g_glm_flash_attn_mask_bytes DS4_STREAM_SCRATCH(glm_flash_attn_mask_bytes) +#define g_compressor_pool_kv_bytes DS4_STREAM_SCRATCH(compressor_pool_kv_bytes) +#define g_compressor_pool_score_bytes DS4_STREAM_SCRATCH(compressor_pool_score_bytes) +#define g_compressor_pool_score_cont_bytes DS4_STREAM_SCRATCH(compressor_pool_score_cont_bytes) +#define g_compressor_pool_softmax_bytes DS4_STREAM_SCRATCH(compressor_pool_softmax_bytes) +#define g_compressor_pool_product_bytes DS4_STREAM_SCRATCH(compressor_pool_product_bytes) +#define g_compressor_store_ape_bytes DS4_STREAM_SCRATCH(compressor_store_ape_bytes) +#define g_compressor_store_score_bytes DS4_STREAM_SCRATCH(compressor_store_score_bytes) +#define g_embed_rows_bytes DS4_STREAM_SCRATCH(embed_rows_bytes) +#define g_router_selection_bytes DS4_STREAM_SCRATCH(router_selection_bytes) +#define g_router_weight_sum_bytes DS4_STREAM_SCRATCH(router_weight_sum_bytes) +#define g_indexer_head_scores_bytes DS4_STREAM_SCRATCH(indexer_head_scores_bytes) +#define g_indexer_topk_bytes DS4_STREAM_SCRATCH(indexer_topk_bytes) +#define g_indexed_topk_bytes DS4_STREAM_SCRATCH(indexed_topk_bytes) +#define g_f16_round_scratch_bytes DS4_STREAM_SCRATCH(f16_round_scratch_bytes) +#define g_raw_store_round_bytes DS4_STREAM_SCRATCH(raw_store_round_bytes) +#define g_moe_gate_scratch_bytes DS4_STREAM_SCRATCH(moe_gate_scratch_bytes) +#define g_moe_down_scratch_bytes DS4_STREAM_SCRATCH(moe_down_scratch_bytes) +#define g_moe_id_map_bytes DS4_STREAM_SCRATCH(moe_id_map_bytes) +#define g_moe_q4_gate_slots_bytes DS4_STREAM_SCRATCH(moe_q4_gate_slots_bytes) +#define g_moe_q4_up_slots_bytes DS4_STREAM_SCRATCH(moe_q4_up_slots_bytes) +#define g_moe_q4_down_slots_bytes DS4_STREAM_SCRATCH(moe_q4_down_slots_bytes) +#define g_attn_out_group_ids_bytes DS4_STREAM_SCRATCH(attn_out_group_ids_bytes) static int g_model_fd = -1; /* Second model descriptor with F_NOCACHE for the streaming expert preads * (DS4_METAL_STREAMING_EXPERT_NOCACHE); -1 = use the cached g_model_fd. */ @@ -453,6 +558,12 @@ static uint64_t g_stream_expert_live_index_fallbacks; static uint64_t g_stream_expert_live_index_inserts; static uint64_t g_stream_expert_live_index_removes; +static uint64_t g_stream_expert_exact_persistent_calls; +static uint64_t g_stream_expert_exact_transient_calls; +static uint64_t g_stream_expert_exact_persistent_fallbacks; +static uint64_t g_stream_expert_exact_persistent_failures; +static uint64_t g_stream_expert_exact_mapped_view_calls; +static uint32_t g_stream_expert_exact_max_unique; static uint64_t g_stream_expert_timing_readahead_calls; static uint64_t g_stream_expert_timing_readahead_bytes; static double g_stream_expert_timing_readahead_ms; @@ -517,6 +628,7 @@ static uint32_t g_iq2_stream_addr_mm_max_tokens; static uint64_t g_model_residency_count; static int g_model_residency_added_to_queue; +static uint32_t g_model_residency_queue_mask; static int g_glm_model_mode; static int g_ssd_streaming_mode; static int g_glm_streaming_prefill_full_layer_runtime; @@ -559,40 +671,10 @@ static int ds4_gpu_stream_expert_cache_on_service_thread(void) { return g_stream_expert_service_thread_set && pthread_equal(pthread_self(), g_stream_expert_service_thread); } -static NSUInteger g_flash_attn_mask_bytes; -static NSUInteger g_flash_attn_zero_mask_bytes; -static NSUInteger g_flash_attn_pad_bytes; -static NSUInteger g_flash_attn_tmp_bytes; -static NSUInteger g_flash_attn_blk_bytes; -static NSUInteger g_flash_attn_ring_bytes; -static NSUInteger g_flash_attn_kv_bytes; -static NSUInteger g_glm_flash_attn_mask_bytes; static uint32_t g_glm_flash_attn_mask_pos0; static uint32_t g_glm_flash_attn_mask_tokens; static uint32_t g_glm_flash_attn_mask_cache_len; static int g_glm_flash_attn_mask_valid; -static NSUInteger g_compressor_pool_kv_bytes; -static NSUInteger g_compressor_pool_score_bytes; -static NSUInteger g_compressor_pool_score_cont_bytes; -static NSUInteger g_compressor_pool_softmax_bytes; -static NSUInteger g_compressor_pool_product_bytes; -static NSUInteger g_compressor_store_ape_bytes; -static NSUInteger g_compressor_store_score_bytes; -static NSUInteger g_embed_rows_bytes; -static NSUInteger g_router_selection_bytes; -static NSUInteger g_router_weight_sum_bytes; -static NSUInteger g_indexer_head_scores_bytes; -static NSUInteger g_indexer_topk_bytes; -static NSUInteger g_indexed_topk_bytes; -static NSUInteger g_f16_round_scratch_bytes; -static NSUInteger g_raw_store_round_bytes; -static NSUInteger g_moe_gate_scratch_bytes; -static NSUInteger g_moe_down_scratch_bytes; -static NSUInteger g_moe_id_map_bytes; -static NSUInteger g_moe_q4_gate_slots_bytes; -static NSUInteger g_moe_q4_up_slots_bytes; -static NSUInteger g_moe_q4_down_slots_bytes; -static NSUInteger g_attn_out_group_ids_bytes; static int g_initialized; static int g_quality_mode; static int g_mpp_invalid_env_reported; @@ -840,7 +922,7 @@ @interface DS4MetalQ4ExpertTable : NSObject @property(nonatomic, strong) id addressBuffer; @property(nonatomic, strong) NSMutableArray> *expertBuffers; @property(nonatomic, strong) id residencySet; -@property(nonatomic, assign) BOOL residencySetAddedToQueue; +@property(nonatomic, assign) uint32_t residencySetQueueMask; @property(nonatomic, assign) uint32_t nExpert; @property(nonatomic, assign) uint64_t expertBytes; @end @@ -850,10 +932,14 @@ - (void)dealloc { #if TARGET_OS_OSX if (@available(macOS 15.0, *)) { if (_residencySet) { - if (_residencySetAddedToQueue && - g_queue && - [g_queue respondsToSelector:@selector(removeResidencySet:)]) { - [g_queue removeResidencySet:_residencySet]; + for (int i = 0; i < DS4_GPU_MAX_STREAMS; i++) { + if ((_residencySetQueueMask & (1u << i)) == 0) continue; + id queue = + i == 0 ? g_queue : g_stream_queues[i]; + if (queue && + [queue respondsToSelector:@selector(removeResidencySet:)]) { + [queue removeResidencySet:_residencySet]; + } } [_residencySet endResidency]; } @@ -864,7 +950,8 @@ - (void)dealloc { @interface DS4MetalQ4LayerResidency : NSObject @property(nonatomic, strong) id residencySet; -@property(nonatomic, assign) BOOL addedToQueue; +@property(nonatomic, assign) BOOL queueResident; +@property(nonatomic, assign) uint32_t queueMask; @end @implementation DS4MetalQ4LayerResidency @@ -872,10 +959,14 @@ - (void)dealloc { #if TARGET_OS_OSX if (@available(macOS 15.0, *)) { if (_residencySet) { - if (_addedToQueue && - g_queue && - [g_queue respondsToSelector:@selector(removeResidencySet:)]) { - [g_queue removeResidencySet:_residencySet]; + for (int i = 0; i < DS4_GPU_MAX_STREAMS; i++) { + if ((_queueMask & (1u << i)) == 0) continue; + id queue = + i == 0 ? g_queue : g_stream_queues[i]; + if (queue && + [queue respondsToSelector:@selector(removeResidencySet:)]) { + [queue removeResidencySet:_residencySet]; + } } [_residencySet endResidency]; } @@ -1063,6 +1154,86 @@ static NSUInteger ds4_gpu_tensor_offset(const ds4_gpu_tensor *tensor) { } static id ds4_gpu_new_command_buffer(void); + +/* Command encoding remains serialized through the process-global batch, but + * committed command buffers can execute concurrently on independent queues. + * The TLS selector also selects the stream-local scratch/transient state. */ +void ds4_gpu_set_stream(int idx) { + if (g_batch_cb) return; + if (idx < 0 || idx >= DS4_GPU_MAX_STREAMS) idx = 0; + g_ds4_stream = idx; +} + +int ds4_gpu_current_stream(void) { return g_ds4_stream; } + +static uint32_t ds4_gpu_active_queue_mask(void) { + uint32_t mask = g_queue ? 1u : 0u; + for (int i = 1; i < DS4_GPU_MAX_STREAMS; i++) { + if (g_stream_queues[i]) mask |= 1u << i; + } + return mask; +} + +static void ds4_gpu_q4_attach_layer_residency_to_queue( + DS4MetalQ4LayerResidency *entry, + id queue, + int stream) { +#if TARGET_OS_OSX + if (!entry || !entry.queueResident || !entry.residencySet || !queue || + stream < 0 || stream >= DS4_GPU_MAX_STREAMS) { + return; + } + if (@available(macOS 15.0, *)) { + const uint32_t bit = 1u << stream; + if ((entry.queueMask & bit) == 0 && + [queue respondsToSelector:@selector(addResidencySet:)]) { + [queue addResidencySet:entry.residencySet]; + entry.queueMask |= bit; + } + } +#else + (void)entry; + (void)queue; + (void)stream; +#endif +} + +static void ds4_gpu_q4_attach_cached_residency_to_queue( + id queue, + int stream) { +#if TARGET_OS_OSX + if (!queue || stream < 0 || stream >= DS4_GPU_MAX_STREAMS || + !g_q4_expert_layer_residency_cache) { + return; + } + if (@available(macOS 15.0, *)) { + for (DS4MetalQ4LayerResidency *entry in + [g_q4_expert_layer_residency_cache allValues]) { + ds4_gpu_q4_attach_layer_residency_to_queue(entry, queue, stream); + } + } +#else + (void)queue; + (void)stream; +#endif +} + +static id ds4_gpu_active_queue(void) { + if (g_ds4_stream == 0) return g_queue; + if (!g_stream_queues[g_ds4_stream]) { + id queue = [g_device newCommandQueue]; + if (queue && g_model_residency_set && + g_model_residency_added_to_queue && + [queue respondsToSelector:@selector(addResidencySet:)]) { + [queue addResidencySet:g_model_residency_set]; + g_model_residency_queue_mask |= 1u << g_ds4_stream; + } + g_stream_queues[g_ds4_stream] = queue; + ds4_gpu_q4_attach_cached_residency_to_queue(queue, g_ds4_stream); + } + return g_stream_queues[g_ds4_stream]; +} + static void ds4_gpu_stream_expert_cache_note_owned_created(void); static id ds4_gpu_command_buffer(int *owned) { @@ -1143,9 +1314,9 @@ static int ds4_gpu_wait_command_buffer(id cb, const char *labe initialized = 1; } if (use_unretained) { - return [g_queue commandBufferWithUnretainedReferences]; + return [ds4_gpu_active_queue() commandBufferWithUnretainedReferences]; } - return [g_queue commandBuffer]; + return [ds4_gpu_active_queue() commandBuffer]; } static uint64_t ds4_gpu_exact_view_cache_limit_bytes(void) { @@ -1217,9 +1388,16 @@ static void ds4_gpu_model_buffer_cache_clear(const char *reason) { } static void ds4_gpu_model_buffer_cache_maybe_evict(const char *reason) { - if (g_model_buffer_cache_over_limit) { - ds4_gpu_model_buffer_cache_clear(reason); + if (!g_model_buffer_cache_over_limit || g_batch_cb) return; + /* Exact model views can be the only strong references when command + * buffers are unretained. Defer eviction until every stream is drained. */ + for (int i = 0; i < DS4_GPU_MAX_STREAMS; i++) { + if (g_stream_last_cb[i] || + [g_pending_cbs_by_stream[i] count] != 0) { + return; + } } + ds4_gpu_model_buffer_cache_clear(reason); } static uint64_t ds4_gpu_stream_expert_cache_next_cb_seq(void) { @@ -1304,17 +1482,31 @@ static int ds4_gpu_stream_expert_cache_mark_entries_inflight( static int ds4_gpu_stream_expert_cache_wait_inflight(const char *label); -static int ds4_gpu_wait_pending_command_buffers(const char *label) { +static int ds4_gpu_wait_pending_command_buffers_for_stream( + int idx, const char *label) { + if (idx < 0 || idx >= DS4_GPU_MAX_STREAMS) return 0; int ok = 1; - for (id pending in g_pending_cbs) { + for (id pending in g_pending_cbs_by_stream[idx]) { if (!ds4_gpu_wait_command_buffer(pending, label)) ok = 0; } - [g_pending_cbs removeAllObjects]; + [g_pending_cbs_by_stream[idx] removeAllObjects]; ds4_gpu_stream_expert_cache_note_pending_completed(); if (!ok) ds4_gpu_invalidate_zero_prefix_prefill_block_maps(); return ok; } +static int ds4_gpu_wait_pending_command_buffers(const char *label) { + return ds4_gpu_wait_pending_command_buffers_for_stream( + g_ds4_stream, label); +} + +static int ds4_gpu_pending_command_buffers_any(void) { + for (int i = 0; i < DS4_GPU_MAX_STREAMS; i++) { + if ([g_pending_cbs_by_stream[i] count] != 0) return 1; + } + return 0; +} + static int ds4_gpu_finish_command_buffer(id cb, int owned, const char *label) { if (!owned) return 1; @@ -1861,10 +2053,14 @@ static void ds4_gpu_model_residency_clear(void) { #if TARGET_OS_OSX if (@available(macOS 15.0, *)) { if (g_model_residency_set) { - if (g_model_residency_added_to_queue && - g_queue && - [g_queue respondsToSelector:@selector(removeResidencySet:)]) { - [g_queue removeResidencySet:g_model_residency_set]; + for (int i = 0; i < DS4_GPU_MAX_STREAMS; i++) { + if ((g_model_residency_queue_mask & (1u << i)) == 0) continue; + id queue = + i == 0 ? g_queue : g_stream_queues[i]; + if (queue && + [queue respondsToSelector:@selector(removeResidencySet:)]) { + [queue removeResidencySet:g_model_residency_set]; + } } [g_model_residency_set endResidency]; [g_model_residency_set removeAllAllocations]; @@ -1874,6 +2070,7 @@ static void ds4_gpu_model_residency_clear(void) { #endif g_model_residency_count = 0; g_model_residency_added_to_queue = 0; + g_model_residency_queue_mask = 0; } /* TP sharding keeps only this rank's expert ranges warm, @@ -1921,11 +2118,20 @@ static int ds4_gpu_model_residency_request_views(void) { } [g_model_residency_set commit]; [g_model_residency_set requestResidency]; - if (getenv("DS4_METAL_DISABLE_QUEUE_RESIDENCY_SET") == NULL && - g_queue && - [g_queue respondsToSelector:@selector(addResidencySet:)]) { - [g_queue addResidencySet:g_model_residency_set]; - g_model_residency_added_to_queue = 1; + g_model_residency_queue_mask = 0; + if (getenv("DS4_METAL_DISABLE_QUEUE_RESIDENCY_SET") == NULL) { + for (int i = 0; i < DS4_GPU_MAX_STREAMS; i++) { + id queue = + i == 0 ? g_queue : g_stream_queues[i]; + if (!queue || + ![queue respondsToSelector:@selector(addResidencySet:)]) { + continue; + } + [queue addResidencySet:g_model_residency_set]; + g_model_residency_queue_mask |= 1u << i; + } + g_model_residency_added_to_queue = + g_model_residency_queue_mask != 0; } g_model_residency_count = g_model_view_count; } @@ -2156,6 +2362,42 @@ static int ds4_gpu_finish_model_views( return buffer; } +int ds4_gpu_test_hold_stream_transient(uint64_t bytes) { + if (!g_initialized && !ds4_gpu_init()) return 0; + if (bytes > (uint64_t)NSUIntegerMax) return 0; + return ds4_gpu_new_transient_buffer( + (NSUInteger)bytes, "ds4_stream_test_transient") != nil; +} + +void ds4_gpu_test_stream_stats(ds4_gpu_stream_test_stats *stats) { + if (!stats) return; + memset(stats, 0, sizeof(*stats)); + pthread_mutex_lock(&g_tensor_mu); + stats->tensor_live_bytes = g_tensor_alloc_live_bytes; + stats->tensor_live_count = + g_tensor_live_count > UINT32_MAX ? + UINT32_MAX : (uint32_t)g_tensor_live_count; + pthread_mutex_unlock(&g_tensor_mu); + + uint64_t pending = 0; + for (int i = 0; i < DS4_GPU_MAX_STREAMS; i++) { + stats->transient_references += + (uint64_t)[g_transient_buffers_by_stream[i] count]; + pending += (uint64_t)[g_pending_cbs_by_stream[i] count]; + if (g_stream_last_cb[i]) stats->last_command_buffers++; + } + stats->pending_command_buffers = + pending > UINT32_MAX ? UINT32_MAX : (uint32_t)pending; + stats->active_queue_mask = ds4_gpu_active_queue_mask(); + stats->model_residency_queue_mask = g_model_residency_queue_mask; + if (g_q4_expert_layer_residency_cache) { + for (DS4MetalQ4LayerResidency *entry in + [g_q4_expert_layer_residency_cache allValues]) { + stats->q4_residency_queue_mask |= entry.queueMask; + } + } +} + static int ds4_gpu_zero_prefix_prefill_mask_cache_enabled(void) { if (getenv("DS4_METAL_DISABLE_ZERO_PREFIX_PREFILL_MASK_CACHE") != NULL || getenv("DS4_METAL_FLASH_ATTN_STAGE_PROFILE") != NULL) { @@ -2279,9 +2521,12 @@ void ds4_gpu_release_zero_prefix_prefill_mask_cache(void) { * release point has no outstanding cache users. Keep the guard here for * diagnostic callers that may have an open or asynchronously flushed CB. */ if (!g_initialized || g_batch_cb || - (g_pending_cbs && [g_pending_cbs count] != 0)) { + ds4_gpu_pending_command_buffers_any()) { return; } + for (int i = 0; i < DS4_GPU_MAX_STREAMS; i++) { + if (g_stream_last_cb[i]) return; + } ds4_gpu_clear_zero_prefix_prefill_mask_cache(); } @@ -4151,38 +4396,55 @@ void ds4_gpu_print_memory_report(const char *label) { if (entry->mask) cached_prefill_mask_bytes += entry->mask_bytes; if (entry->blk) cached_prefill_blk_bytes += entry->blk_bytes; } - const uint64_t scratch = - (uint64_t)g_flash_attn_mask_bytes + - (uint64_t)g_flash_attn_zero_mask_bytes + - cached_prefill_mask_bytes + - (uint64_t)g_flash_attn_pad_bytes + - (uint64_t)g_flash_attn_tmp_bytes + - (uint64_t)g_flash_attn_blk_bytes + - cached_prefill_blk_bytes + - (uint64_t)g_flash_attn_ring_bytes + - (uint64_t)g_flash_attn_kv_bytes + - (uint64_t)g_glm_flash_attn_mask_bytes + - (uint64_t)g_compressor_pool_kv_bytes + - (uint64_t)g_compressor_pool_score_bytes + - (uint64_t)g_compressor_pool_score_cont_bytes + - (uint64_t)g_compressor_pool_softmax_bytes + - (uint64_t)g_compressor_pool_product_bytes + - (uint64_t)g_compressor_store_ape_bytes + - (uint64_t)g_compressor_store_score_bytes + - (uint64_t)g_embed_rows_bytes + - (uint64_t)g_router_selection_bytes + - (uint64_t)g_router_weight_sum_bytes + - (uint64_t)g_indexer_head_scores_bytes + - (uint64_t)g_indexer_topk_bytes + - (uint64_t)g_indexed_topk_bytes + - (uint64_t)g_f16_round_scratch_bytes + - (uint64_t)g_raw_store_round_bytes + - (uint64_t)g_moe_gate_scratch_bytes + - (uint64_t)g_moe_down_scratch_bytes + - (uint64_t)g_moe_id_map_bytes + - (uint64_t)g_moe_q4_gate_slots_bytes + - (uint64_t)g_moe_q4_up_slots_bytes + - (uint64_t)g_moe_q4_down_slots_bytes; + uint64_t flash_mask = cached_prefill_mask_bytes; + uint64_t flash_pad = 0; + uint64_t flash_tmp = 0; + uint64_t flash_blk = cached_prefill_blk_bytes; + uint64_t flash_ring = 0; + uint64_t flash_kv = 0; + uint64_t compressor = 0; + uint64_t router = 0; + uint64_t indexer = 0; + uint64_t moe = 0; + uint64_t f16_round = 0; + uint64_t raw_store = 0; + for (int si = 0; si < DS4_GPU_MAX_STREAMS; si++) { + const ds4_gpu_stream_scratch_state *scratch_state = + &g_stream_scratch[si]; + flash_mask += (uint64_t)scratch_state->flash_attn_mask_bytes + + (uint64_t)scratch_state->glm_flash_attn_mask_bytes + + (uint64_t)scratch_state->flash_attn_zero_mask_bytes; + flash_pad += (uint64_t)scratch_state->flash_attn_pad_bytes; + flash_tmp += (uint64_t)scratch_state->flash_attn_tmp_bytes; + flash_blk += (uint64_t)scratch_state->flash_attn_blk_bytes; + flash_ring += (uint64_t)scratch_state->flash_attn_ring_bytes; + flash_kv += (uint64_t)scratch_state->flash_attn_kv_bytes; + compressor += (uint64_t)scratch_state->compressor_pool_kv_bytes + + (uint64_t)scratch_state->compressor_pool_score_bytes + + (uint64_t)scratch_state->compressor_pool_score_cont_bytes + + (uint64_t)scratch_state->compressor_pool_softmax_bytes + + (uint64_t)scratch_state->compressor_pool_product_bytes + + (uint64_t)scratch_state->compressor_store_ape_bytes + + (uint64_t)scratch_state->compressor_store_score_bytes + + (uint64_t)scratch_state->embed_rows_bytes; + router += (uint64_t)scratch_state->router_selection_bytes + + (uint64_t)scratch_state->router_weight_sum_bytes + + (uint64_t)scratch_state->attn_out_group_ids_bytes; + indexer += (uint64_t)scratch_state->indexer_head_scores_bytes + + (uint64_t)scratch_state->indexer_topk_bytes + + (uint64_t)scratch_state->indexed_topk_bytes; + moe += (uint64_t)scratch_state->moe_gate_scratch_bytes + + (uint64_t)scratch_state->moe_down_scratch_bytes + + (uint64_t)scratch_state->moe_id_map_bytes + + (uint64_t)scratch_state->moe_q4_gate_slots_bytes + + (uint64_t)scratch_state->moe_q4_up_slots_bytes + + (uint64_t)scratch_state->moe_q4_down_slots_bytes; + f16_round += (uint64_t)scratch_state->f16_round_scratch_bytes; + raw_store += (uint64_t)scratch_state->raw_store_round_bytes; + } + const uint64_t scratch = flash_mask + flash_pad + flash_tmp + flash_blk + + flash_ring + flash_kv + compressor + router + + indexer + moe + f16_round + raw_store; pthread_mutex_lock(&g_tensor_mu); const uint64_t tensor_live_snap = g_tensor_alloc_live_bytes; @@ -4405,6 +4667,23 @@ void ds4_gpu_print_memory_report(const char *label) { } } } + if (g_stream_expert_exact_persistent_calls != 0 || + g_stream_expert_exact_transient_calls != 0 || + g_stream_expert_exact_persistent_fallbacks != 0 || + g_stream_expert_exact_persistent_failures != 0 || + g_stream_expert_exact_mapped_view_calls != 0 || + g_stream_expert_exact_max_unique != 0) { + fprintf(stderr, + "ds4: exact-row cache persistent=%llu transient=%llu " + "fallbacks=%llu failures=%llu mapped_views=%llu " + "max_unique=%u\n", + (unsigned long long)g_stream_expert_exact_persistent_calls, + (unsigned long long)g_stream_expert_exact_transient_calls, + (unsigned long long)g_stream_expert_exact_persistent_fallbacks, + (unsigned long long)g_stream_expert_exact_persistent_failures, + (unsigned long long)g_stream_expert_exact_mapped_view_calls, + g_stream_expert_exact_max_unique); + } fprintf(stderr, "ds4: model residency requests %llu%s\n", (unsigned long long)g_model_residency_count, @@ -4436,37 +4715,18 @@ void ds4_gpu_print_memory_report(const char *label) { fprintf(stderr, "ds4: scratch %.2f MiB (flash mask %.2f, pad %.2f, tmp %.2f, blk %.2f, ring %.2f, kv %.2f, compressor %.2f, router %.2f, indexer %.2f, moe %.2f, f16 %.2f, raw-store %.2f)\n", ds4_gpu_mib(scratch), - ds4_gpu_mib((uint64_t)g_flash_attn_mask_bytes + - (uint64_t)g_glm_flash_attn_mask_bytes + - (uint64_t)g_flash_attn_zero_mask_bytes + - cached_prefill_mask_bytes), - ds4_gpu_mib((uint64_t)g_flash_attn_pad_bytes), - ds4_gpu_mib((uint64_t)g_flash_attn_tmp_bytes), - ds4_gpu_mib((uint64_t)g_flash_attn_blk_bytes + - cached_prefill_blk_bytes), - ds4_gpu_mib((uint64_t)g_flash_attn_ring_bytes), - ds4_gpu_mib((uint64_t)g_flash_attn_kv_bytes), - ds4_gpu_mib((uint64_t)g_compressor_pool_kv_bytes + - (uint64_t)g_compressor_pool_score_bytes + - (uint64_t)g_compressor_pool_score_cont_bytes + - (uint64_t)g_compressor_pool_softmax_bytes + - (uint64_t)g_compressor_pool_product_bytes + - (uint64_t)g_compressor_store_ape_bytes + - (uint64_t)g_compressor_store_score_bytes + - (uint64_t)g_embed_rows_bytes), - ds4_gpu_mib((uint64_t)g_router_selection_bytes + - (uint64_t)g_router_weight_sum_bytes), - ds4_gpu_mib((uint64_t)g_indexer_head_scores_bytes + - (uint64_t)g_indexer_topk_bytes + - (uint64_t)g_indexed_topk_bytes), - ds4_gpu_mib((uint64_t)g_moe_gate_scratch_bytes + - (uint64_t)g_moe_down_scratch_bytes + - (uint64_t)g_moe_id_map_bytes + - (uint64_t)g_moe_q4_gate_slots_bytes + - (uint64_t)g_moe_q4_up_slots_bytes + - (uint64_t)g_moe_q4_down_slots_bytes), - ds4_gpu_mib((uint64_t)g_f16_round_scratch_bytes), - ds4_gpu_mib((uint64_t)g_raw_store_round_bytes)); + ds4_gpu_mib(flash_mask), + ds4_gpu_mib(flash_pad), + ds4_gpu_mib(flash_tmp), + ds4_gpu_mib(flash_blk), + ds4_gpu_mib(flash_ring), + ds4_gpu_mib(flash_kv), + ds4_gpu_mib(compressor), + ds4_gpu_mib(router), + ds4_gpu_mib(indexer), + ds4_gpu_mib(moe), + ds4_gpu_mib(f16_round), + ds4_gpu_mib(raw_store)); if (color) fputs(reset, stderr); } @@ -6685,15 +6945,20 @@ int ds4_gpu_init(void) { if (g_pipeline_cache_generation == 0) g_pipeline_cache_generation++; g_dsv4_completion_cache = [NSCache new]; g_dsv4_completion_cache.countLimit = 256u; - g_transient_buffers = [NSMutableArray array]; - g_pending_cbs = [NSMutableArray array]; + for (int si = 0; si < DS4_GPU_MAX_STREAMS; si++) { + g_transient_buffers_by_stream[si] = [NSMutableArray array]; + g_pending_cbs_by_stream[si] = [NSMutableArray array]; + } if (!g_model_buffer_cache || !g_q4_expert_table_cache || !g_q4_expert_layer_residency_cache || !g_pipeline_cache || !g_dsv4_completion_cache || - !g_transient_buffers || !g_pending_cbs) { + !g_transient_buffers_by_stream[0] || + !g_pending_cbs_by_stream[0]) { fprintf(stderr, "ds4: Metal bookkeeping allocation failed\n"); - g_pending_cbs = nil; - g_transient_buffers = nil; + for (int si = 0; si < DS4_GPU_MAX_STREAMS; si++) { + g_pending_cbs_by_stream[si] = nil; + g_transient_buffers_by_stream[si] = nil; + } g_dsv4_completion_cache = nil; g_pipeline_cache = nil; g_q4_expert_layer_residency_cache = nil; @@ -9697,7 +9962,7 @@ int ds4_gpu_begin_commands(void) { if (!g_initialized && !ds4_gpu_init()) return 0; /* A failed concurrent FFN must never affect the next command batch. */ ds4_gpu_parallel_ffn_reset_state(YES); - if (g_batch_cb) return 0; + if (g_batch_cb || g_stream_last_cb[g_ds4_stream]) return 0; /* Refresh once per command batch so same-engine A/B runs can toggle the * static PSO without paying for environment lookups in every layer. */ g_use_dsv4_head_rms_norm_rope_tail_pipeline = @@ -10808,6 +11073,37 @@ int ds4_gpu_end_commands(void) { return ds4_gpu_finish_command_buffer(cb, 1, "command batch"); } +int ds4_gpu_end_commands_async(void) { + if (!g_initialized || !g_batch_cb || + g_stream_last_cb[g_ds4_stream]) { + return 0; + } + ds4_gpu_parallel_ffn_reset_state(YES); + ds4_gpu_close_batch_encoder(); + id cb = g_batch_cb; + g_batch_cb = nil; + g_batch_has_work = NO; + [cb commit]; + g_stream_last_cb[g_ds4_stream] = cb; + ds4_gpu_stream_expert_cache_note_batch_committed(); + return 1; +} + +int ds4_gpu_wait_stream(int idx) { + if (idx < 0 || idx >= DS4_GPU_MAX_STREAMS) return 0; + int ok = ds4_gpu_wait_pending_command_buffers_for_stream( + idx, "stream pending command batch"); + id cb = g_stream_last_cb[idx]; + if (cb) { + if (!ds4_gpu_wait_command_buffer(cb, "stream command batch")) ok = 0; + g_stream_last_cb[idx] = nil; + ds4_gpu_stream_expert_cache_note_owned_completed(); + } + [g_transient_buffers_by_stream[idx] removeAllObjects]; + ds4_gpu_model_buffer_cache_maybe_evict("stream command batch"); + return ok; +} + static int ds4_gpu_flash_attn_stage_profile_boundary( id __strong *cbp, const char *mode, @@ -10854,14 +11150,18 @@ static int ds4_gpu_flash_attn_stage_profile_boundary( int ds4_gpu_synchronize(void) { if (!g_initialized && !ds4_gpu_init()) return 0; - if (g_batch_cb) return ds4_gpu_end_commands(); + int ok = 1; + if (g_batch_cb && !ds4_gpu_end_commands()) ok = 0; ds4_gpu_parallel_ffn_reset_state(YES); - if ([g_pending_cbs count] != 0) { - int ok = ds4_gpu_wait_pending_command_buffers("synchronize"); - [g_transient_buffers removeAllObjects]; - ds4_gpu_model_buffer_cache_maybe_evict("synchronize"); - return ok; + int had_stream_work = 0; + for (int i = 0; i < DS4_GPU_MAX_STREAMS; i++) { + if (g_stream_last_cb[i] || + [g_pending_cbs_by_stream[i] count] != 0) { + had_stream_work = 1; + if (!ds4_gpu_wait_stream(i)) ok = 0; + } } + if (had_stream_work || !ok) return ok; id cb = ds4_gpu_new_command_buffer(); if (!cb) return 0; @@ -10887,8 +11187,11 @@ void ds4_gpu_cleanup(void) { g_stream_expert_cache_done_seq = g_stream_expert_cache_batch_seq; } g_stream_expert_cache_batch_seq = 0; + [g_transient_buffers removeAllObjects]; + } + for (int si = 0; si < DS4_GPU_MAX_STREAMS; si++) { + (void)ds4_gpu_wait_stream(si); } - (void)ds4_gpu_wait_pending_command_buffers("cleanup"); ds4_gpu_iq2_stream_addr_mm_stats_print(); if (ds4_gpu_stream_expert_timing_summary_enabled() && getenv("DS4_METAL_MEMORY_REPORT") == NULL) { @@ -10896,7 +11199,6 @@ void ds4_gpu_cleanup(void) { } g_selected_readback_event = nil; g_selected_readback_event_value = 0; - [g_transient_buffers removeAllObjects]; ds4_gpu_stream_expert_pread_pool_shutdown(); ds4_gpu_stream_expert_cache_clear_all(1); ds4_gpu_stream_expert_cache_live_release(); @@ -11121,38 +11423,42 @@ void ds4_gpu_cleanup(void) { g_glm_q6_k_down_f32_pipeline = nil; g_dsv4_router_weights_batch_pipeline = nil; g_dsv4_hc_expand4_pipeline = nil; - g_flash_attn_mask_buffer = nil; - g_flash_attn_zero_mask_buffer = nil; - g_flash_attn_pad_buffer = nil; - g_flash_attn_tmp_buffer = nil; - g_flash_attn_blk_buffer = nil; ds4_gpu_clear_zero_prefix_prefill_mask_cache(); - g_flash_attn_ring_buffer = nil; - g_flash_attn_kv_buffer = nil; - g_glm_flash_attn_mask_buffer = nil; - g_compressor_pool_kv_buffer = nil; - g_compressor_pool_score_buffer = nil; - g_compressor_pool_score_cont_buffer = nil; - g_compressor_pool_softmax_buffer = nil; - g_compressor_pool_product_buffer = nil; - g_compressor_store_ape_buffer = nil; - g_compressor_store_score_buffer = nil; - g_embed_rows_buffer = nil; - g_router_selection_buffer = nil; - g_router_weight_sum_buffer = nil; - g_indexer_head_scores_buffer = nil; - g_indexer_topk_buffer = nil; - g_indexed_topk_buffer = nil; g_stream_expert_validate_status_buffer = nil; - g_f16_round_scratch_buffer = nil; - g_raw_store_round_buffer = nil; - g_moe_gate_scratch_buffer = nil; - g_moe_down_scratch_buffer = nil; - g_moe_id_map_buffer = nil; - g_moe_q4_gate_slots_buffer = nil; - g_moe_q4_up_slots_buffer = nil; - g_moe_q4_down_slots_buffer = nil; - g_attn_out_group_ids_buffer = nil; + for (int si = 0; si < DS4_GPU_MAX_STREAMS; si++) { + g_ds4_stream = si; + g_flash_attn_mask_buffer = nil; + g_flash_attn_zero_mask_buffer = nil; + g_flash_attn_pad_buffer = nil; + g_flash_attn_tmp_buffer = nil; + g_flash_attn_blk_buffer = nil; + g_flash_attn_ring_buffer = nil; + g_flash_attn_kv_buffer = nil; + g_glm_flash_attn_mask_buffer = nil; + g_compressor_pool_kv_buffer = nil; + g_compressor_pool_score_buffer = nil; + g_compressor_pool_score_cont_buffer = nil; + g_compressor_pool_softmax_buffer = nil; + g_compressor_pool_product_buffer = nil; + g_compressor_store_ape_buffer = nil; + g_compressor_store_score_buffer = nil; + g_embed_rows_buffer = nil; + g_router_selection_buffer = nil; + g_router_weight_sum_buffer = nil; + g_indexer_head_scores_buffer = nil; + g_indexer_topk_buffer = nil; + g_indexed_topk_buffer = nil; + g_f16_round_scratch_buffer = nil; + g_raw_store_round_buffer = nil; + g_moe_gate_scratch_buffer = nil; + g_moe_down_scratch_buffer = nil; + g_moe_id_map_buffer = nil; + g_moe_q4_gate_slots_buffer = nil; + g_moe_q4_up_slots_buffer = nil; + g_moe_q4_down_slots_buffer = nil; + g_attn_out_group_ids_buffer = nil; + } + g_ds4_stream = 0; g_model_fd = -1; if (g_model_fd_nocache >= 0) { close(g_model_fd_nocache); @@ -11166,40 +11472,44 @@ void ds4_gpu_cleanup(void) { g_support_model_map_ptr = NULL; g_support_model_map_size = 0; ds4_gpu_tensor_tracking_reset(); - g_flash_attn_mask_bytes = 0; - g_flash_attn_zero_mask_bytes = 0; - g_flash_attn_pad_bytes = 0; - g_flash_attn_tmp_bytes = 0; - g_flash_attn_blk_bytes = 0; - g_flash_attn_ring_bytes = 0; - g_flash_attn_kv_bytes = 0; - g_glm_flash_attn_mask_bytes = 0; + for (int si = 0; si < DS4_GPU_MAX_STREAMS; si++) { + g_ds4_stream = si; + g_flash_attn_mask_bytes = 0; + g_flash_attn_zero_mask_bytes = 0; + g_flash_attn_pad_bytes = 0; + g_flash_attn_tmp_bytes = 0; + g_flash_attn_blk_bytes = 0; + g_flash_attn_ring_bytes = 0; + g_flash_attn_kv_bytes = 0; + g_glm_flash_attn_mask_bytes = 0; + g_compressor_pool_kv_bytes = 0; + g_compressor_pool_score_bytes = 0; + g_compressor_pool_score_cont_bytes = 0; + g_compressor_pool_softmax_bytes = 0; + g_compressor_pool_product_bytes = 0; + g_compressor_store_ape_bytes = 0; + g_compressor_store_score_bytes = 0; + g_embed_rows_bytes = 0; + g_router_selection_bytes = 0; + g_router_weight_sum_bytes = 0; + g_indexer_head_scores_bytes = 0; + g_indexer_topk_bytes = 0; + g_indexed_topk_bytes = 0; + g_f16_round_scratch_bytes = 0; + g_raw_store_round_bytes = 0; + g_moe_gate_scratch_bytes = 0; + g_moe_down_scratch_bytes = 0; + g_moe_id_map_bytes = 0; + g_moe_q4_gate_slots_bytes = 0; + g_moe_q4_up_slots_bytes = 0; + g_moe_q4_down_slots_bytes = 0; + g_attn_out_group_ids_bytes = 0; + } + g_ds4_stream = 0; g_glm_flash_attn_mask_valid = 0; g_glm_flash_attn_mask_pos0 = 0; g_glm_flash_attn_mask_tokens = 0; g_glm_flash_attn_mask_cache_len = 0; - g_compressor_pool_kv_bytes = 0; - g_compressor_pool_score_bytes = 0; - g_compressor_pool_score_cont_bytes = 0; - g_compressor_pool_softmax_bytes = 0; - g_compressor_pool_product_bytes = 0; - g_compressor_store_ape_bytes = 0; - g_compressor_store_score_bytes = 0; - g_embed_rows_bytes = 0; - g_router_selection_bytes = 0; - g_router_weight_sum_bytes = 0; - g_indexer_head_scores_bytes = 0; - g_indexer_topk_bytes = 0; - g_indexed_topk_bytes = 0; - g_f16_round_scratch_bytes = 0; - g_raw_store_round_bytes = 0; - g_moe_gate_scratch_bytes = 0; - g_moe_down_scratch_bytes = 0; - g_moe_id_map_bytes = 0; - g_moe_q4_gate_slots_bytes = 0; - g_moe_q4_up_slots_bytes = 0; - g_moe_q4_down_slots_bytes = 0; - g_attn_out_group_ids_bytes = 0; g_model_wrap_count = 0; g_model_wrap_bytes = 0; g_model_wrap_max_bytes = 0; @@ -11219,11 +11529,17 @@ void ds4_gpu_cleanup(void) { g_q4_expert_table_cache = nil; [g_model_buffer_cache removeAllObjects]; g_model_buffer_cache = nil; - g_transient_buffers = nil; - g_pending_cbs = nil; + for (int si = 0; si < DS4_GPU_MAX_STREAMS; si++) { + [g_transient_buffers_by_stream[si] removeAllObjects]; + g_transient_buffers_by_stream[si] = nil; + g_pending_cbs_by_stream[si] = nil; + g_stream_last_cb[si] = nil; + g_stream_queues[si] = nil; + } g_library = nil; g_queue = nil; g_device = nil; + g_ds4_stream = 0; g_iq2_stream_addr_mm_stats_enabled = 0; ds4_gpu_iq2_stream_addr_mm_stats_reset(); g_initialized = 0; @@ -12594,6 +12910,53 @@ static uint32_t ds4_gpu_stream_expert_cache_configured_budget(void) { return budget; } +static int ds4_gpu_exact_rows_persistent_env_enabled(const char *name) { + const char *value = name ? getenv(name) : NULL; + return value && value[0] && strcmp(value, "0") != 0; +} + +/* Persistent private snapshots are the default when the cache can retain the + * full exact-row union. DISABLE is the A/B rollback arm and always wins; + * REQUIRE fails closed instead of silently using transient packed experts. */ +static int ds4_gpu_exact_rows_persistent_policy( + uint32_t configured_count, + uint32_t unique_count, + int size_class_ok) { + const int disabled = ds4_gpu_exact_rows_persistent_env_enabled( + "DS4_METAL_DISABLE_EXACT_ROWS_PERSISTENT_CACHE"); + const int required = ds4_gpu_exact_rows_persistent_env_enabled( + "DS4_METAL_REQUIRE_EXACT_ROWS_PERSISTENT_CACHE"); + if (disabled) return required ? -1 : 0; + + const int eligible = size_class_ok && unique_count != 0 && + configured_count >= unique_count; + if (eligible) return 1; + return required ? -1 : 0; +} + +int ds4_gpu_test_exact_rows_persistent_policy( + uint32_t configured_count, + uint32_t unique_count, + int size_class_ok) { + return ds4_gpu_exact_rows_persistent_policy(configured_count, + unique_count, + size_class_ok); +} + +void ds4_gpu_test_exact_rows_persistent_report( + ds4_gpu_exact_rows_persistent_report *report) { + if (!report) return; + *report = (ds4_gpu_exact_rows_persistent_report) { + .persistent_calls = g_stream_expert_exact_persistent_calls, + .transient_calls = g_stream_expert_exact_transient_calls, + .persistent_fallbacks = + g_stream_expert_exact_persistent_fallbacks, + .persistent_failures = g_stream_expert_exact_persistent_failures, + .mapped_view_calls = g_stream_expert_exact_mapped_view_calls, + .max_unique = g_stream_expert_exact_max_unique, + }; +} + static uint32_t ds4_gpu_stream_expert_cache_effective_cap( uint32_t layer, uint32_t n_total_expert, @@ -13667,6 +14030,36 @@ static void ds4_gpu_stream_expert_slab_push_free_slot(uint32_t slot) { slot; } +/* Prepared misses own their buffers until install_loaded() publishes them. + * A failed exact batch must return slab slots or unlock standalone buffers, + * otherwise repeated I/O failures silently shrink the cache. */ +static void ds4_gpu_stream_expert_cache_release_prepared_buffers( + id __strong *gate_buf, + id __strong *up_buf, + id __strong *down_buf, + NSUInteger gate_inner) { + if (!gate_buf || !up_buf || !down_buf) return; + + id gate = *gate_buf; + id up = *up_buf; + id down = *down_buf; + uint32_t slab_slot = UINT32_MAX; + if (gate && gate == up && gate == down && + ds4_gpu_stream_expert_slab_slot_for_buffer( + gate, gate_inner, &slab_slot)) { + ds4_gpu_stream_expert_slab_push_free_slot(slab_slot); + } else { + ds4_gpu_stream_expert_unlock_explicit_buffer(gate); + if (up != gate) ds4_gpu_stream_expert_unlock_explicit_buffer(up); + if (down != gate && down != up) { + ds4_gpu_stream_expert_unlock_explicit_buffer(down); + } + } + *gate_buf = nil; + *up_buf = nil; + *down_buf = nil; +} + static int ds4_gpu_stream_expert_slab_lock_slot(uint32_t slot) { if (slot >= DS4_METAL_STREAM_EXPERT_CACHE_MAX_ENTRIES || g_stream_expert_cache_slab_slot_locked[slot]) { @@ -13815,15 +14208,19 @@ static int ds4_gpu_stream_expert_alloc_slab_slot( if (g_stream_expert_cache_free_slot_count != 0) { const uint32_t slot = g_stream_expert_cache_free_slots[--g_stream_expert_cache_free_slot_count]; - return ds4_gpu_stream_expert_slab_slot_buffers(slot, - gate_expert_bytes, - down_expert_bytes, - gate_buf, - up_buf, - down_buf, - gate_inner, - up_inner, - down_inner); + if (ds4_gpu_stream_expert_slab_slot_buffers(slot, + gate_expert_bytes, + down_expert_bytes, + gate_buf, + up_buf, + down_buf, + gate_inner, + up_inner, + down_inner)) { + return 1; + } + ds4_gpu_stream_expert_slab_push_free_slot(slot); + return 0; } uint32_t slab = g_stream_expert_cache_slab_count; @@ -13877,15 +14274,19 @@ static int ds4_gpu_stream_expert_alloc_slab_slot( const uint32_t local_slot = g_stream_expert_cache_slab_slots_used[slab]++; const uint32_t slot = g_stream_expert_cache_slab_start_slot[slab] + local_slot; - return ds4_gpu_stream_expert_slab_slot_buffers(slot, - gate_expert_bytes, - down_expert_bytes, - gate_buf, - up_buf, - down_buf, - gate_inner, - up_inner, - down_inner); + if (ds4_gpu_stream_expert_slab_slot_buffers(slot, + gate_expert_bytes, + down_expert_bytes, + gate_buf, + up_buf, + down_buf, + gate_inner, + up_inner, + down_inner)) { + return 1; + } + ds4_gpu_stream_expert_slab_push_free_slot(slot); + return 0; } static uint64_t ds4_gpu_stream_expert_buffer_object_count( @@ -14506,9 +14907,9 @@ static int ds4_gpu_stream_expert_cache_set_addr_slot( static int ds4_gpu_stream_expert_cache_addr_buffers( uint32_t layer, - id *gate, - id *up, - id *down) { + id __strong *gate, + id __strong *up, + id __strong *down) { if (!ds4_gpu_stream_expert_cache_ensure_addr_buffers(layer)) return 0; if (gate) *gate = g_stream_expert_cache_gate_addr_buffers[layer]; if (up) *up = g_stream_expert_cache_up_addr_buffers[layer]; @@ -14518,6 +14919,88 @@ static int ds4_gpu_stream_expert_cache_addr_buffers( g_stream_expert_cache_down_addr_buffers[layer]; } +/* Exact-row tails cannot borrow mutable cache tables. Publish an immutable + * layer-local snapshot after every selected expert is resident and pinned. */ +static int ds4_gpu_stream_expert_exact_snapshot_addr_buffers( + uint32_t n_total_expert, + const int32_t *unique_ids, + uint32_t unique_count, + ds4_gpu_stream_expert_cache_entry * const *entries, + id __strong *gate_out, + id __strong *up_out, + id __strong *down_out) { + if (gate_out) *gate_out = nil; + if (up_out) *up_out = nil; + if (down_out) *down_out = nil; + if (!unique_ids || !entries || !gate_out || !up_out || !down_out || + n_total_expert == 0 || + n_total_expert > DS4_METAL_STREAM_EXPERT_CACHE_MAX_EXPERT || + unique_count == 0 || unique_count > n_total_expert) { + return 0; + } + + const NSUInteger bytes = + (NSUInteger)n_total_expert * sizeof(uint64_t); + id gate = ds4_gpu_new_transient_buffer( + bytes, "ds4_exact_rows_gate_addr_snapshot"); + id up = ds4_gpu_new_transient_buffer( + bytes, "ds4_exact_rows_up_addr_snapshot"); + id down = ds4_gpu_new_transient_buffer( + bytes, "ds4_exact_rows_down_addr_snapshot"); + if (!gate || !up || !down) return 0; + + uint64_t *gate_addrs = (uint64_t *)[gate contents]; + uint64_t *up_addrs = (uint64_t *)[up contents]; + uint64_t *down_addrs = (uint64_t *)[down contents]; + if (!gate_addrs || !up_addrs || !down_addrs) return 0; + memset(gate_addrs, 0, bytes); + memset(up_addrs, 0, bytes); + memset(down_addrs, 0, bytes); + + for (uint32_t u = 0; u < unique_count; u++) { + const int32_t selected_id = unique_ids[u]; + ds4_gpu_stream_expert_cache_entry *entry = entries[u]; + if (selected_id < 0 || + (uint32_t)selected_id >= n_total_expert || + !entry || !entry->valid || + !entry->gate_buffer || !entry->up_buffer || + !entry->down_buffer) { + return 0; + } + const uint64_t gate_addr = + ds4_gpu_buffer_address(entry->gate_buffer, entry->gate_inner); + const uint64_t up_addr = + ds4_gpu_buffer_address(entry->up_buffer, entry->up_inner); + const uint64_t down_addr = + ds4_gpu_buffer_address(entry->down_buffer, entry->down_inner); + if (gate_addr == 0 || up_addr == 0 || down_addr == 0) return 0; + + const uint32_t expert = (uint32_t)selected_id; + if ((gate_addrs[expert] != 0 && + gate_addrs[expert] != gate_addr) || + (up_addrs[expert] != 0 && up_addrs[expert] != up_addr) || + (down_addrs[expert] != 0 && + down_addrs[expert] != down_addr)) { + fprintf(stderr, + "ds4: Metal exact-row snapshot found inconsistent " + "duplicate expert %u\n", + expert); + return 0; + } + gate_addrs[expert] = gate_addr; + up_addrs[expert] = up_addr; + down_addrs[expert] = down_addr; + } + + [gate didModifyRange:NSMakeRange(0, bytes)]; + [up didModifyRange:NSMakeRange(0, bytes)]; + [down didModifyRange:NSMakeRange(0, bytes)]; + *gate_out = gate; + *up_out = up; + *down_out = down; + return 1; +} + static void ds4_gpu_stream_full_expert_addr_clear_layer(uint32_t layer) { if (layer >= DS4_METAL_STREAM_EXPERT_CACHE_MAX_LAYER) return; ds4_gpu_stream_expert_cache_entry *e = &g_stream_full_expert_addr_entry[layer]; @@ -15410,6 +15893,12 @@ static void ds4_gpu_stream_expert_cache_clear_all(int reset_stats) { g_stream_expert_live_index_fallbacks = 0; g_stream_expert_live_index_inserts = 0; g_stream_expert_live_index_removes = 0; + g_stream_expert_exact_persistent_calls = 0; + g_stream_expert_exact_transient_calls = 0; + g_stream_expert_exact_persistent_fallbacks = 0; + g_stream_expert_exact_persistent_failures = 0; + g_stream_expert_exact_mapped_view_calls = 0; + g_stream_expert_exact_max_unique = 0; g_stream_expert_timing_readahead_calls = 0; g_stream_expert_timing_readahead_bytes = 0; g_stream_expert_timing_readahead_ms = 0.0; @@ -17569,12 +18058,12 @@ static int ds4_gpu_stream_expert_prepare_transient_selected_batch( uint64_t down_offset, uint64_t gate_expert_bytes, uint64_t down_expert_bytes, - id *gate_addrs, - id *up_addrs, - id *down_addrs, - id *packed_gate, - id *packed_up, - id *packed_down, + id __strong *gate_addrs, + id __strong *up_addrs, + id __strong *down_addrs, + id __strong *packed_gate, + id __strong *packed_up, + id __strong *packed_down, ds4_gpu_stream_expert_cache_entry **resources, uint32_t *n_resources) { if (!model_map || !unique_ids || !frequency || @@ -17872,15 +18361,15 @@ static int ds4_gpu_stream_expert_cache_prepare_selected_batch( uint64_t down_offset, uint64_t gate_expert_bytes, uint64_t down_expert_bytes, - id *gate_addrs, - id *up_addrs, - id *down_addrs, + id __strong *gate_addrs, + id __strong *up_addrs, + id __strong *down_addrs, ds4_gpu_stream_expert_cache_entry **resources, uint32_t *n_resources, uint32_t *unique_out, - id *overflow_gate, - id *overflow_up, - id *overflow_down, + id __strong *overflow_gate, + id __strong *overflow_up, + id __strong *overflow_down, bool require_private_addr_tables) { if (overflow_gate) *overflow_gate = nil; if (overflow_up) *overflow_up = nil; @@ -17957,12 +18446,60 @@ static int ds4_gpu_stream_expert_cache_prepare_selected_batch( ds4_gpu_stream_expert_cache_note_frequency_hotness(layer, frequency, n_total_expert); + if (require_private_addr_tables && + unique_count > g_stream_expert_exact_max_unique) { + g_stream_expert_exact_max_unique = unique_count; + } + } + const uint32_t configured_count = + ds4_gpu_stream_expert_cache_configured_count(); + const int private_persistent_disabled = + require_private_addr_tables && + ds4_gpu_exact_rows_persistent_env_enabled( + "DS4_METAL_DISABLE_EXACT_ROWS_PERSISTENT_CACHE"); + const uint32_t private_size_class_count = + ok && require_private_addr_tables && !private_persistent_disabled ? + ds4_gpu_stream_expert_cache_budget_for_expert_size( + gate_expert_bytes, down_expert_bytes) : 0; + const int private_size_class_ok = + private_size_class_count != 0 && + !g_stream_expert_cache_class_conflict && + g_stream_expert_cache_gate_class_bytes == gate_expert_bytes && + g_stream_expert_cache_down_class_bytes == down_expert_bytes; + const int private_persistent_policy = + ok && require_private_addr_tables ? + ds4_gpu_exact_rows_persistent_policy(private_size_class_count, + unique_count, + private_size_class_ok) : 0; + if (ok && require_private_addr_tables && + private_persistent_policy < 0) { + g_stream_expert_exact_persistent_failures++; + fprintf(stderr, + "ds4: Metal exact-row persistent cache is required but " + "disabled or ineligible (cache_budget=%u " + "size_class_budget=%u unique=%u)\n", + configured_count, + private_size_class_count, + unique_count); + free(ids); + return 0; + } + const bool persistent_private_tables = + private_persistent_policy > 0; + if (ok && require_private_addr_tables) { + if (persistent_private_tables) { + g_stream_expert_exact_persistent_calls++; + } else { + g_stream_expert_exact_transient_calls++; + g_stream_expert_exact_persistent_fallbacks++; + } } const bool use_transient_selected = ok && n_tokens <= 5u && - (require_private_addr_tables || - ds4_gpu_stream_expert_cache_configured_count() < n_total_expert); + ((require_private_addr_tables && !persistent_private_tables) || + (!require_private_addr_tables && + configured_count < n_total_expert)); if (use_transient_selected) { ok = ds4_gpu_stream_expert_prepare_transient_selected_batch( model_map, @@ -18034,6 +18571,7 @@ static int ds4_gpu_stream_expert_cache_prepare_selected_batch( ds4_gpu_stream_expert_pread_task *tasks = NULL; uint32_t n_loads = 0; + uint32_t n_prepared = 0; uint32_t n_tasks = 0; double load_prepare_ms = 0.0; double load_modify_ms = 0.0; @@ -18107,12 +18645,12 @@ static int ds4_gpu_stream_expert_cache_prepare_selected_batch( gate_expert_bytes, down_expert_bytes, force_reuse, - &gate_bufs[n_loads], - &up_bufs[n_loads], - &down_bufs[n_loads], - &gate_inners[n_loads], - &up_inners[n_loads], - &down_inners[n_loads]); + &gate_bufs[n_prepared], + &up_bufs[n_prepared], + &down_bufs[n_prepared], + &gate_inners[n_prepared], + &up_inners[n_prepared], + &down_inners[n_prepared]); if (load_timing) { ds4_gpu_stream_expert_timing_note_prepare_buffer( ds4_gpu_now_ms() - buffer_t0); @@ -18121,28 +18659,32 @@ static int ds4_gpu_stream_expert_cache_prepare_selected_batch( ok = 0; break; } + const uint32_t prepared_i = n_prepared++; + load_unique[prepared_i] = u; if (!force_reuse && reserved_entries < UINT32_MAX) { reserved_entries++; } - if (!gate_bufs[n_loads] || - !up_bufs[n_loads] || - !down_bufs[n_loads]) { + if (!gate_bufs[prepared_i] || + !up_bufs[prepared_i] || + !down_bufs[prepared_i]) { ok = 0; break; } - uint8_t *gate_dst = (uint8_t *)[gate_bufs[n_loads] contents] + - gate_inners[n_loads]; - uint8_t *up_dst = (uint8_t *)[up_bufs[n_loads] contents] + - up_inners[n_loads]; - uint8_t *down_dst = (uint8_t *)[down_bufs[n_loads] contents] + - down_inners[n_loads]; + uint8_t *gate_dst = + (uint8_t *)[gate_bufs[prepared_i] contents] + + gate_inners[prepared_i]; + uint8_t *up_dst = + (uint8_t *)[up_bufs[prepared_i] contents] + + up_inners[prepared_i]; + uint8_t *down_dst = + (uint8_t *)[down_bufs[prepared_i] contents] + + down_inners[prepared_i]; if (!gate_dst || !up_dst || !down_dst) { ok = 0; break; } - load_unique[n_loads] = u; const double task_t0 = load_timing ? ds4_gpu_now_ms() : 0.0; tasks[n_tasks++] = (ds4_gpu_stream_expert_pread_task) { .offset = unique_gate_offsets[u], @@ -18243,6 +18785,40 @@ static int ds4_gpu_stream_expert_cache_prepare_selected_batch( load_install_ms); } } + if (!ok) { + /* install_loaded() may fail after publishing an entry. Classify every + * prepared tuple by authoritative cache identity before recycling; + * clear_layer() below owns all tuples already transferred to cache. */ + for (uint32_t load_i = 0; load_i < n_prepared; load_i++) { + bool cache_owned = false; + const uint32_t u = load_unique[load_i]; + if (u < unique_count && unique_ids[u] >= 0 && + (uint32_t)unique_ids[u] < + DS4_METAL_STREAM_EXPERT_CACHE_MAX_EXPERT) { + const ds4_gpu_stream_expert_cache_entry *entry = + &g_stream_expert_cache[layer][(uint32_t)unique_ids[u]]; + cache_owned = + entry->valid && + entry->gate_buffer == gate_bufs[load_i] && + entry->up_buffer == up_bufs[load_i] && + entry->down_buffer == down_bufs[load_i] && + entry->gate_inner == gate_inners[load_i] && + entry->up_inner == up_inners[load_i] && + entry->down_inner == down_inners[load_i]; + } + if (!cache_owned) { + ds4_gpu_stream_expert_cache_release_prepared_buffers( + &gate_bufs[load_i], + &up_bufs[load_i], + &down_bufs[load_i], + gate_inners[load_i]); + } else { + gate_bufs[load_i] = nil; + up_bufs[load_i] = nil; + down_bufs[load_i] = nil; + } + } + } if (tasks) free(tasks); if (ok) { for (uint32_t u = 0; u < unique_count; u++) { @@ -18252,6 +18828,15 @@ static int ds4_gpu_stream_expert_cache_prepare_selected_batch( const uint64_t gate_rel = unique_gate_offsets[u] - gate_offset; const uint64_t down_rel = unique_down_offsets[u] - down_offset; if (!*overflow_gate) { + if (gate_expert_bytes > UINT64_MAX / n_total_expert || + down_expert_bytes > UINT64_MAX / n_total_expert) { + fprintf(stderr, + "ds4: Metal streaming prefill batch selected addr " + "expert tensor size overflow at layer %u\n", + layer); + ok = 0; + break; + } const uint64_t gate_tensor_bytes = (uint64_t)n_total_expert * gate_expert_bytes; const uint64_t down_tensor_bytes = @@ -18328,13 +18913,41 @@ static int ds4_gpu_stream_expert_cache_prepare_selected_batch( (*n_resources)++; } } + if (require_private_addr_tables && view_served != 0) { + g_stream_expert_exact_mapped_view_calls++; + } free(ids); if (!ok || (*n_resources == 0 && view_served == 0)) { + if (persistent_private_tables) { + g_stream_expert_exact_persistent_failures++; + } ds4_gpu_stream_expert_cache_clear_layer(layer); return 0; } - if (!ds4_gpu_stream_expert_cache_addr_buffers(layer, + if (require_private_addr_tables && + (!persistent_private_tables || view_served != 0 || + *n_resources != unique_count || + !ds4_gpu_stream_expert_exact_snapshot_addr_buffers( + n_total_expert, + unique_ids, + unique_count, + unique_entries, + gate_addrs, + up_addrs, + down_addrs))) { + fprintf(stderr, + "ds4: Metal exact-row persistent cache could not publish " + "a complete private address snapshot at layer %u\n", + layer); + if (persistent_private_tables) { + g_stream_expert_exact_persistent_failures++; + } + ds4_gpu_stream_expert_cache_clear_layer(layer); + return 0; + } + if (!require_private_addr_tables && + !ds4_gpu_stream_expert_cache_addr_buffers(layer, gate_addrs, up_addrs, down_addrs)) { @@ -18398,7 +19011,7 @@ int ds4_gpu_stream_expert_exact_rows_begin_collect(void) { return 1; } -int ds4_gpu_stream_expert_exact_rows_prepare( +static int ds4_gpu_stream_expert_exact_rows_prepare_impl( const ds4_gpu_stream_expert_table *table, const ds4_gpu_tensor *selected_rows, uint32_t n_rows, @@ -18560,6 +19173,20 @@ int ds4_gpu_stream_expert_exact_rows_prepare( return 1; } +int ds4_gpu_stream_expert_exact_rows_prepare( + const ds4_gpu_stream_expert_table *table, + const ds4_gpu_tensor *selected_rows, + uint32_t n_rows, + uint32_t n_selected) { + /* ARC may autorelease intermediate Metal objects created by the selected + * loader. The exact scope has already retained everything needed by the + * GPU, so drain those temporary references at every layer boundary. */ + @autoreleasepool { + return ds4_gpu_stream_expert_exact_rows_prepare_impl( + table, selected_rows, n_rows, n_selected); + } +} + int ds4_gpu_stream_expert_exact_rows_set_row(uint32_t row) { ds4_gpu_stream_expert_exact_rows_scope *scope = &g_stream_expert_exact_rows_scope; @@ -18572,7 +19199,7 @@ int ds4_gpu_stream_expert_exact_rows_set_row(uint32_t row) { return 1; } -int ds4_gpu_stream_expert_exact_rows_end_async(void) { +static int ds4_gpu_stream_expert_exact_rows_end_async_impl(void) { if (!g_initialized && !ds4_gpu_init()) return 0; ds4_gpu_stream_expert_exact_rows_scope *scope = &g_stream_expert_exact_rows_scope; @@ -18624,6 +19251,14 @@ int ds4_gpu_stream_expert_exact_rows_end_async(void) { return 1; } +int ds4_gpu_stream_expert_exact_rows_end_async(void) { + /* The completion block owns its copied array; drain the temporary mutable + * array instead of retaining all expert buffers in the C caller's pool. */ + @autoreleasepool { + return ds4_gpu_stream_expert_exact_rows_end_async_impl(); + } +} + void ds4_gpu_stream_expert_exact_rows_release(void) { /* release() is also the fail-clean exit for a collection that did not * reach prepare(). Neither ordinary selected state is allowed to escape @@ -19099,7 +19734,18 @@ static id ds4_gpu_q4_expert_layer_residency_set(DS4MetalQ4ExpertTable *gate_tabl gate_table, up_table, down_table]; DS4MetalQ4LayerResidency *cached = [g_q4_expert_layer_residency_cache objectForKey:key]; - if (cached) return cached.residencySet; + if (cached) { + if (queue_residency) cached.queueResident = YES; + if (cached.queueResident) { + for (int i = 0; i < DS4_GPU_MAX_STREAMS; i++) { + id queue = + i == 0 ? g_queue : g_stream_queues[i]; + ds4_gpu_q4_attach_layer_residency_to_queue( + cached, queue, i); + } + } + return cached.residencySet; + } const NSUInteger capacity = [gate_table.expertBuffers count] + @@ -19125,9 +19771,13 @@ static id ds4_gpu_q4_expert_layer_residency_set(DS4MetalQ4ExpertTable *gate_tabl DS4MetalQ4LayerResidency *entry = [DS4MetalQ4LayerResidency new]; entry.residencySet = residency_set; + entry.queueResident = queue_residency; if (queue_residency) { - [g_queue addResidencySet:residency_set]; - entry.addedToQueue = YES; + for (int i = 0; i < DS4_GPU_MAX_STREAMS; i++) { + id queue = + i == 0 ? g_queue : g_stream_queues[i]; + ds4_gpu_q4_attach_layer_residency_to_queue(entry, queue, i); + } } [g_q4_expert_layer_residency_cache setObject:entry forKey:key]; return residency_set; From 6d39838f53c1c67c7307ebfa63e1d26a5f0702a7 Mon Sep 17 00:00:00 2001 From: Giorgio Oppo Date: Fri, 14 Aug 2026 15:41:18 +0200 Subject: [PATCH 028/189] metal: add opt-in resident Q4 session overlap --- Makefile | 15 +- ds4.c | 176 ++++++- ds4.h | 2 + tests/test_metal_q4_streams.c | 874 ++++++++++++++++++++++++++++++++++ 4 files changed, 1059 insertions(+), 8 deletions(-) create mode 100644 tests/test_metal_q4_streams.c diff --git a/Makefile b/Makefile index e12967758..6495a0b26 100644 --- a/Makefile +++ b/Makefile @@ -68,7 +68,7 @@ DS4_LINK_LIBS ?= $(CUDA_LDLIBS) METAL_LDLIBS := $(LDLIBS) endif -.PHONY: all help clean test test-rocm test-glm53-kda-rocm test-metal-session-batch test-metal-exactn-oracle test-metal-dspark-capture test-metal-iq2-midonly test-metal-iq2-ssd-grouped-mm test-metal-iq2-live-index test-mxfp4-cuda test-mxfp4-rocm test-mmq-parity-cuda test-cuda-session-batch test-cuda-mixed-batch dspark-acceptance dspark-verify-depth rocm-dspark-acceptance rocm-dspark-verify-depth mtp-verify-depth cpu cuda cuda-spark cuda-generic cuda-regression strix-halo rocm +.PHONY: all help clean test test-rocm test-glm53-kda-rocm test-metal-session-batch test-metal-q4-streams test-metal-exactn-oracle test-metal-dspark-capture test-metal-iq2-midonly test-metal-iq2-ssd-grouped-mm test-metal-iq2-live-index test-mxfp4-cuda test-mxfp4-rocm test-mmq-parity-cuda test-cuda-session-batch test-cuda-mixed-batch dspark-acceptance dspark-verify-depth rocm-dspark-acceptance rocm-dspark-verify-depth mtp-verify-depth cpu cuda cuda-spark cuda-generic cuda-regression strix-halo rocm ifeq ($(UNAME_S),Darwin) .PHONY: metal-decode-schedule-bench metal-prefill-variant-bench check-mxfp4-half-lut test-mxfp4-metal @@ -80,6 +80,7 @@ help: @echo " make Build Metal ./ds4, ./ds4-server, ./ds4-bench, ./ds4-eval, and ./ds4-agent" @echo " make cpu Build CPU-only ./ds4, ./ds4-server, ./ds4-bench, ./ds4-eval, and ./ds4-agent" @echo " make test Build and run tests" + @echo " make test-metal-q4-streams Check resident Q4 Metal stream overlap" @echo " make test-metal-dspark-capture Check fused DSpark HC capture bitwise" @echo " make test-metal-iq2-midonly Check M1 IQ2 addr mid-only output and sentinels" @echo " make test-metal-iq2-live-index Check IQ2 SSD live-cache index policy and fallback" @@ -119,6 +120,16 @@ tests/test_metal_session_batch: tests/test_metal_session_batch.o $(CORE_OBJS) test-metal-session-batch: tests/test_metal_session_batch DS4_TEST_MODEL="$(DS4_TEST_MODEL)" ./tests/test_metal_session_batch +tests/test_metal_q4_streams.o: tests/test_metal_q4_streams.c ds4.h ds4_gpu.h + $(CC) $(CFLAGS) -DDS4_TEST_HOOKS -I. -c -o $@ $< + +tests/test_metal_q4_streams: tests/test_metal_q4_streams.o ds4_metal_test_hooks.o ds4_image.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_metal.o ds4_layer_pack.o + $(CC) $(CFLAGS) -o $@ $^ $(METAL_LDLIBS) + +test-metal-q4-streams: tests/test_metal_q4_streams + env -u DS4_METAL_MODEL_UNTRACKED ./tests/test_metal_q4_streams + DS4_METAL_MODEL_UNTRACKED=1 ./tests/test_metal_q4_streams + tests/test_metal_dspark_capture.o: tests/test_metal_dspark_capture.c ds4_gpu.h $(CC) $(CFLAGS) -I. -c -o $@ $< @@ -692,4 +703,4 @@ mxfp4-dot-test: tests/test_mxfp4_dot.c ./tests/test_mxfp4_dot clean: - rm -f ds4 ds4-server ds4-bench ds4-eval ds4-agent ds4_cpu ds4_native ds4_server_test ds4_test ds4_agent_test gguf-tools/quality-testing/score_official gguf-tools/quality-testing/score_official.o speed-bench/metal_decode_schedule_bench speed-bench/metal_prefill_variant_bench speed-bench/*.o tests/test_q4k_dot tests/test_mxfp4_dot tests/test_mxfp4_metal tests/test_mxfp4_rocm tests/test_mxfp4_cuda tests/test_metal_session_batch tests/test_metal_exactn_oracle tests/test_metal_dspark_capture tests/test_metal_iq2_midonly tests/test_metal_iq2_ssd_grouped_mm tests/test_metal_iq2_live_index tests/test_glm53_kda tests/test_glm53_kda_rocm tests/test_glm53_vision_engine tests/test_glm53_vision_prompt tests/test_gpu_xdev tests/test_gpu_model_cache tests/test_gpu_lookup_cache_strict tests/test_engine_mgpu_refusal tests/test_engine_mgpu_runtime tests/test_engine_correctness tests/test_sampling tests/test_cuda_session_batch tests/test_cuda_mixed_batch tests/*.o *.o tests/cuda_long_context_smoke tests/cuda_long_context_smoke.o + rm -f ds4 ds4-server ds4-bench ds4-eval ds4-agent ds4_cpu ds4_native ds4_server_test ds4_test ds4_agent_test gguf-tools/quality-testing/score_official gguf-tools/quality-testing/score_official.o speed-bench/metal_decode_schedule_bench speed-bench/metal_prefill_variant_bench speed-bench/*.o tests/test_q4k_dot tests/test_mxfp4_dot tests/test_mxfp4_metal tests/test_mxfp4_rocm tests/test_mxfp4_cuda tests/test_metal_session_batch tests/test_metal_q4_streams tests/test_metal_exactn_oracle tests/test_metal_dspark_capture tests/test_metal_iq2_midonly tests/test_metal_iq2_ssd_grouped_mm tests/test_metal_iq2_live_index tests/test_glm53_kda tests/test_glm53_kda_rocm tests/test_glm53_vision_engine tests/test_glm53_vision_prompt tests/test_gpu_xdev tests/test_gpu_model_cache tests/test_gpu_lookup_cache_strict tests/test_engine_mgpu_refusal tests/test_engine_mgpu_runtime tests/test_engine_correctness tests/test_sampling tests/test_cuda_session_batch tests/test_cuda_mixed_batch tests/*.o *.o tests/cuda_long_context_smoke tests/cuda_long_context_smoke.o diff --git a/ds4.c b/ds4.c index 01eecab77..c56addf7d 100644 --- a/ds4.c +++ b/ds4.c @@ -55994,6 +55994,8 @@ struct ds4_session { bool greedy_splitkv_anchor_valid; }; +enum { DS4_METAL_SESSION_STREAMS = 8 }; + #ifndef DS4_NO_GPU static bool ds4_dspark_stats_enabled(void); @@ -70477,6 +70479,92 @@ static int ds4_session_eval_probe_tp(ds4_session *s, int token, bool probe_mtp, return rc; } +#ifndef DS4_NO_GPU +/* Small fail-closed policy shared by runtime admission and its test hook. + * Detailed graph/layout checks remain below; this layer owns the externally + * controllable arm and the state classes that must never reach multi-queue + * execution. */ +static bool ds4_q4_stream_overlap_policy( + int count, + bool resident, + bool ssd_streaming, + bool quality) { + const bool enabled = metal_graph_tp_env_flag( + "DS4_METAL_ENABLE_Q4_STREAM_OVERLAP", false); + const bool disabled = metal_graph_tp_env_flag( + "DS4_METAL_DISABLE_Q4_STREAM_OVERLAP", false); + return enabled && !disabled && + count >= 2 && count <= DS4_METAL_SESSION_STREAMS && + resident && !ssd_streaming && !quality; +} + +#ifdef DS4_TEST_HOOKS +int ds4_test_q4_stream_overlap_policy( + int count, + bool resident, + bool ssd_streaming, + bool quality) { + return ds4_q4_stream_overlap_policy(count, + resident, + ssd_streaming, + quality) ? 1 : 0; +} +#endif +#endif + +#if !defined(DS4_NO_GPU) && defined(__APPLE__) +static bool ds4_engine_has_q4_stream_overlap_weights(const ds4_engine *e) { + if (!e) return false; + for (uint32_t il = 0; il < DS4_N_LAYER; il++) { + const ds4_layer_weights *layer = &e->weights.layer[il]; + const bool aproj_q4 = + layer->attn_q_a && layer->attn_kv && + layer->attn_q_a->type == DS4_TENSOR_Q4_K && + layer->attn_kv->type == DS4_TENSOR_Q4_K; + const bool routed_q4 = + layer->ffn_gate_exps && layer->ffn_up_exps && + layer->ffn_down_exps && + layer->ffn_gate_exps->type == DS4_TENSOR_Q4_K && + layer->ffn_up_exps->type == DS4_TENSOR_Q4_K && + layer->ffn_down_exps->type == DS4_TENSOR_Q4_K; + if (aproj_q4 || routed_q4) return true; + } + return false; +} + +static bool ds4_session_q4_stream_overlap_eligible(ds4_session *s) { + if (!s || !s->engine) return false; + const ds4_engine *e = s->engine; + const ds4_gpu_graph *g = &s->graph; + return e->backend == DS4_BACKEND_METAL && + e->support_kind == DS4_SUPPORT_NONE && + !e->tp.active && !e->multi_tier && !e->ssd_streaming && + !s->distributed && !ds4_session_is_cpu(s) && + !ds4_session_is_glm(s) && !ds4_session_cancelled(s) && + s->checkpoint_valid && + !g->ssd_streaming && !g->placement && !g->quality && + g->tp_world < 2 && !g->materialize_ffn_out && + !g->decode_stage_profile && + !g->decode_index_stage_profile && + !g->output_stage_profile && !g_expert_profile.active && + !metal_graph_hc_norm_fusion_check_enabled() && + !metal_graph_use_reference_shared_down_hc() && + !metal_graph_use_pro_q4_cpu_router() && + !metal_graph_use_q4_selected_shared_overlap(g) && + !metal_graph_q4_non_streaming_opt_in_enabled() && + !metal_graph_directional_steering_attn_enabled(g) && + !metal_graph_directional_steering_ffn_enabled(g) && + getenv("DS4_METAL_GRAPH_DUMP_PREFIX") == NULL && + getenv("DS4_METAL_DECODE_STAGE_PROFILE") == NULL && + getenv("DS4_METAL_LAYER_STAGE_PROFILE") == NULL && + getenv("DS4_METAL_ATTN_OUT_STAGE_PROFILE") == NULL && + getenv("DS4_METAL_FLASH_ATTN_STAGE_PROFILE") == NULL && + getenv("DS4_METAL_MOE_ONE_STAGE_PROFILE") == NULL && + getenv("DS4_METAL_MOE_STAGE_PROFILE") == NULL && + ds4_engine_has_q4_stream_overlap_weights(e); +} +#endif + int ds4_session_eval(ds4_session *s, int token, char *err, size_t errlen) { bool probe_mtp = true; #ifndef DS4_NO_GPU @@ -71150,6 +71238,42 @@ static bool ds4_sessions_eval_batch_metal_supported( return true; } +static bool ds4_sessions_q4_stream_overlap_supported( + ds4_decode_item *items, + int count, + ds4_engine *e) { +#if !defined(__APPLE__) + (void)items; + (void)count; + (void)e; + return false; +#else + if (!items || count <= 0 || !e || !items[0].session) { + return false; + } + const ds4_gpu_graph *first = &items[0].session->graph; + const bool resident = + e->backend == DS4_BACKEND_METAL && + !e->multi_tier && !first->placement; + const bool ssd_streaming = e->ssd_streaming || first->ssd_streaming; + if (!ds4_q4_stream_overlap_policy(count, + resident, + ssd_streaming, + first->quality) || + !ds4_engine_has_q4_stream_overlap_weights(e)) { + return false; + } + for (int i = 0; i < count; i++) { + ds4_session *s = items[i].session; + if (!s || s->engine != e || + !ds4_session_q4_stream_overlap_eligible(s)) { + return false; + } + } + return true; +#endif +} + static bool metal_graph_native_session_batch_shared_supported( ds4_decode_item *items, int count, @@ -71534,12 +71658,45 @@ static int ds4_sessions_eval_batch_metal( #if defined(__APPLE__) if (e->tp.active) ds4_gpu_tp_set_session_batch_mode(1); #endif - bool ok = ds4_gpu_begin_commands() != 0; - const bool native_glm53 = ok && + bool native_glm53 = false; + bool native_shared = false; + bool native_qkv = false; + const bool stream_overlap = + ds4_sessions_q4_stream_overlap_supported(items, count, e); + bool ok = true; +#if defined(__APPLE__) + if (stream_overlap) { + int started = 0; + for (int i = 0; ok && i < count; i++) { + ds4_session *s = items[i].session; + ds4_gpu_set_stream(i); + ok = ds4_gpu_begin_commands() != 0; + if (ok) ok = metal_graph_encode_token_raw_swa(&s->graph, + &e->model, + &e->weights, + items[i].token, + (uint32_t)s->checkpoint.len, + true, + false); + if (ok) ok = ds4_gpu_end_commands_async() != 0; + if (ok) started = i + 1; + } + if (!ok && ds4_gpu_commands_active()) { + (void)ds4_gpu_synchronize(); + } + for (int i = 0; i < started; i++) { + if (ds4_gpu_wait_stream(i) == 0) ok = false; + } + ds4_gpu_set_stream(0); + if (!ok) (void)ds4_gpu_synchronize(); + } else { +#endif + ok = ds4_gpu_begin_commands() != 0; + native_glm53 = ok && glm53_graph_native_session_batch_supported(items, count); - const bool native_shared = ok && !native_glm53 && + native_shared = ok && !native_glm53 && metal_graph_native_session_batch_shared_supported(items, count, e); - const bool native_qkv = native_shared && + native_qkv = native_shared && metal_graph_native_session_batch_qkv_supported(items, count, e); if (native_glm53) { ok = glm53_graph_encode_native_session_batch( @@ -71574,6 +71731,9 @@ static int ds4_sessions_eval_batch_metal( } if (ok) ok = ds4_gpu_end_commands() != 0; else (void)ds4_gpu_synchronize(); +#if defined(__APPLE__) + } +#endif #if defined(__APPLE__) if (e->tp.active) ds4_gpu_tp_set_session_batch_mode(0); if (ok && e->tp.active && ds4_gpu_tp_failed()) { @@ -71625,12 +71785,16 @@ static int ds4_sessions_eval_batch_metal( if (getenv("DS4_METAL_SESSION_BATCH_LOG") != NULL) { fprintf(stderr, "ds4: Metal session batch rows=%d family=%s " - "native_glm53=%d native_shared=%d native_qkv=%d\n", + "native_glm53=%d native_shared=%d native_qkv=%d " + "stream_overlap=%d " + "streams=%d\n", count, ds4_session_is_glm(items[0].session) ? "glm" : "deepseek", native_glm53 ? 1 : 0, native_shared ? 1 : 0, - native_qkv ? 1 : 0); + native_qkv ? 1 : 0, + stream_overlap ? 1 : 0, + stream_overlap ? count : 1); } return 0; } diff --git a/ds4.h b/ds4.h index 853202c75..4a5f2dfe2 100644 --- a/ds4.h +++ b/ds4.h @@ -467,6 +467,8 @@ int ds4_test_speculative_delta_sample(const float *target_logits, int ds4_test_argmax_excluding_logits(const float *logits, uint32_t n_vocab, int excluded_id); uint64_t ds4_test_mixed_native_count(void); +int ds4_test_q4_stream_overlap_policy( + int count, bool resident, bool ssd_streaming, bool quality); #endif int ds4_session_top_logprobs(ds4_session *s, ds4_token_score *out, int k); int ds4_session_token_logprob(ds4_session *s, int token, ds4_token_score *out); diff --git a/tests/test_metal_q4_streams.c b/tests/test_metal_q4_streams.c new file mode 100644 index 000000000..25739f1eb --- /dev/null +++ b/tests/test_metal_q4_streams.c @@ -0,0 +1,874 @@ +#define _DARWIN_C_SOURCE + +/* Resident Q4_K oracle for the per-stream Metal command queues. + * + * This deliberately uses a tiny synthetic mmap-shaped model instead of a + * production GGUF: the local Q4 target is much larger than unified memory. + * Each row is evaluated three ways: synchronous FIFO, one native row batch, + * and one command buffer per stream. All three retain the same Q4_K kernel + * reduction order and therefore must be bit-identical. + * + * The default is a short correctness/leak smoke. Set + * DS4_TEST_Q4_STREAM_SOAK=N for a bounded longer overlap soak, and + * DS4_TEST_Q4_STREAM_TIMING=1 to report wall-clock A/B numbers. + */ + +#include "ds4.h" +#include "ds4_gpu.h" + +#include +#include +#include +#include +#include +#include +#include + +#ifdef __APPLE__ + +#include +#include +#include + +#define Q4_K_TYPE 12u +#define QK_K 256u +/* DeepSeek-V4 Flash AProjQ4 q_a/attn_kv projection geometry. */ +#define IN_DIM 4096u +#define OUT0_DIM 1024u +#define OUT1_DIM 512u +#define MAX_STREAMS 8u +#define GUARD_FLOATS 64u +#define MAX_TIMING_BLOCKS 51u + +typedef struct { + uint16_t d; + uint16_t dmin; + uint8_t scales[12]; + uint8_t qs[QK_K / 2u]; +} block_q4_K; + +typedef enum { + ARM_FIFO, + ARM_NATIVE, + ARM_OVERLAP, +} test_arm; + +typedef struct { + uint64_t footprint; + uint64_t resident; + uint64_t virtual_size; + uint64_t max_rss; +} task_memory; + +typedef struct { + void *model; + uint64_t model_size; + uint64_t weight0_offset; + uint64_t weight1_offset; + uint64_t row_bytes; + ds4_gpu_tensor *x; + ds4_gpu_tensor *x_row[MAX_STREAMS]; + ds4_gpu_tensor *out[3][2]; + ds4_gpu_tensor *out_row[3][2][MAX_STREAMS]; + float *host[3][2]; + uint64_t out_count[2]; + float *x_host; +} fixture; + +static void fail(const char *what) { + fprintf(stderr, "Q4 stream oracle FAIL: %s\n", what); + exit(1); +} + +typedef struct { + const char *name; + char *value; + bool present; +} saved_env; + +static saved_env save_env(const char *name) { + const char *value = getenv(name); + saved_env saved = { + .name = name, + .value = value ? strdup(value) : NULL, + .present = value != NULL, + }; + if (value && !saved.value) fail("environment snapshot"); + return saved; +} + +static void restore_env(saved_env *saved) { + const int rc = saved->present + ? setenv(saved->name, saved->value, 1) + : unsetenv(saved->name); + free(saved->value); + saved->value = NULL; + if (rc != 0) fail("environment restore"); +} + +static void expect_overlap_policy(const char *label, int expected, + int count, bool resident, + bool ssd_streaming, bool quality) { + const int actual = ds4_test_q4_stream_overlap_policy( + count, resident, ssd_streaming, quality); + if (actual != expected) { + fprintf(stderr, + "Q4 stream policy %s: got=%d expected=%d " + "count=%d resident=%d ssd=%d quality=%d\n", + label, actual, expected, count, + resident ? 1 : 0, ssd_streaming ? 1 : 0, + quality ? 1 : 0); + fail("scheduler admission policy"); + } +} + +static void test_overlap_policy(void) { + saved_env enabled = save_env("DS4_METAL_ENABLE_Q4_STREAM_OVERLAP"); + saved_env disabled = save_env("DS4_METAL_DISABLE_Q4_STREAM_OVERLAP"); + + if (unsetenv(enabled.name) != 0 || unsetenv(disabled.name) != 0) { + fail("environment clear"); + } + expect_overlap_policy("default-off", 0, 2, true, false, false); + + if (setenv(enabled.name, "1", 1) != 0) fail("enable policy"); + expect_overlap_policy("minimum-count", 1, 2, true, false, false); + expect_overlap_policy("maximum-count", 1, 8, true, false, false); + expect_overlap_policy("count-one", 0, 1, true, false, false); + expect_overlap_policy("count-nine", 0, 9, true, false, false); + expect_overlap_policy("nonresident", 0, 2, false, false, false); + expect_overlap_policy("ssd", 0, 2, true, true, false); + expect_overlap_policy("quality", 0, 2, true, false, true); + + if (setenv(disabled.name, "1", 1) != 0) fail("disable policy"); + expect_overlap_policy("disable-precedence", 0, 2, true, false, false); + if (setenv(disabled.name, "0", 1) != 0) fail("clear disable policy"); + expect_overlap_policy("disable-zero", 1, 2, true, false, false); + if (setenv(enabled.name, "0", 1) != 0) fail("clear enable policy"); + expect_overlap_policy("enable-zero", 0, 2, true, false, false); + + restore_env(&disabled); + restore_env(&enabled); + fprintf(stderr, + "Q4 stream policy PASS default-off=1 disable-precedence=1 " + "count-bounds=1 resident-only=1 ssd-fallback=1 quality-fallback=1\n"); +} + +static uint64_t env_u64(const char *name, uint64_t fallback, + uint64_t minimum, uint64_t maximum) { + const char *value = getenv(name); + if (!value || !value[0]) return fallback; + char *end = NULL; + unsigned long long parsed = strtoull(value, &end, 10); + if (end == value || *end != '\0' || parsed < minimum || parsed > maximum) { + fprintf(stderr, "Q4 stream oracle invalid %s=%s (range %llu..%llu)\n", + name, value, + (unsigned long long)minimum, + (unsigned long long)maximum); + exit(1); + } + return (uint64_t)parsed; +} + +static uint64_t align_up(uint64_t value, uint64_t alignment) { + return (value + alignment - 1u) / alignment * alignment; +} + +static double now_seconds(void) { + struct timespec ts; + if (clock_gettime(CLOCK_MONOTONIC, &ts) != 0) fail("clock_gettime"); + return (double)ts.tv_sec + (double)ts.tv_nsec * 1e-9; +} + +static task_memory read_task_memory(void) { + task_memory result = {0}; + task_vm_info_data_t info; + mach_msg_type_number_t count = TASK_VM_INFO_COUNT; + if (task_info(mach_task_self(), TASK_VM_INFO, + (task_info_t)&info, &count) == KERN_SUCCESS) { + result.footprint = (uint64_t)info.phys_footprint; + result.resident = (uint64_t)info.resident_size; + result.virtual_size = (uint64_t)info.virtual_size; + } + struct rusage usage; + if (getrusage(RUSAGE_SELF, &usage) == 0) { + /* ru_maxrss is bytes on Darwin. */ + result.max_rss = (uint64_t)usage.ru_maxrss; + } + return result; +} + +static void print_task_memory(const char *label, task_memory memory) { + const double mib = 1024.0 * 1024.0; + fprintf(stderr, + "Q4 stream memory %-12s footprint=%.2f MiB resident=%.2f MiB " + "virtual=%.2f MiB peak_rss=%.2f MiB\n", + label, + (double)memory.footprint / mib, + (double)memory.resident / mib, + (double)memory.virtual_size / mib, + (double)memory.max_rss / mib); +} + +static float f16_to_f32(uint16_t h) { + const uint32_t sign = (uint32_t)(h & 0x8000u) << 16u; + uint32_t exp = (h >> 10u) & 0x1fu; + uint32_t mant = h & 0x03ffu; + uint32_t bits; + if (exp == 0u) { + if (mant == 0u) { + bits = sign; + } else { + exp = 1u; + while ((mant & 0x0400u) == 0u) { + mant <<= 1u; + exp--; + } + mant &= 0x03ffu; + bits = sign | ((exp + 127u - 15u) << 23u) | (mant << 13u); + } + } else if (exp == 31u) { + bits = sign | 0x7f800000u | (mant << 13u); + } else { + bits = sign | ((exp + 127u - 15u) << 23u) | (mant << 13u); + } + float value; + memcpy(&value, &bits, sizeof(value)); + return value; +} + +static void q4_scale_min(const uint8_t packed[12], uint32_t group, + uint8_t *scale, uint8_t *minimum) { + if (group < 4u) { + *scale = packed[group] & 63u; + *minimum = packed[group + 4u] & 63u; + } else { + *scale = (packed[group + 4u] & 15u) | + ((packed[group - 4u] >> 6u) << 4u); + *minimum = (packed[group + 4u] >> 4u) | + ((packed[group] >> 6u) << 4u); + } +} + +static void q4_pack_scales(uint8_t packed[12], const uint8_t scale[8], + const uint8_t minimum[8]) { + memset(packed, 0, 12u); + for (uint32_t group = 0; group < 4u; group++) { + packed[group] = scale[group] & 63u; + packed[group + 4u] = minimum[group] & 63u; + } + for (uint32_t group = 4u; group < 8u; group++) { + packed[group + 4u] = (scale[group] & 15u) | + ((minimum[group] & 15u) << 4u); + packed[group - 4u] |= (scale[group] >> 4u) << 6u; + packed[group] |= (minimum[group] >> 4u) << 6u; + } +} + +static void fill_q4_matrix(block_q4_K *matrix, uint32_t rows, uint32_t salt) { + const uint32_t blocks_per_row = IN_DIM / QK_K; + for (uint32_t row = 0; row < rows; row++) { + for (uint32_t block = 0; block < blocks_per_row; block++) { + block_q4_K *b = matrix + (uint64_t)row * blocks_per_row + block; + const uint32_t key = salt + row * 1009u + block * 313u; + uint8_t scale[8]; + uint8_t minimum[8]; + for (uint32_t group = 0; group < 8u; group++) { + scale[group] = (uint8_t)(1u + (key + group * 7u) % 31u); + minimum[group] = (uint8_t)((key / 3u + group * 5u) % 17u); + } + q4_pack_scales(b->scales, scale, minimum); + for (uint32_t i = 0; i < QK_K / 2u; i++) { + b->qs[i] = (uint8_t)(key + i * 37u + (i >> 2u) * 11u); + } + /* Exact binary scales: 2^-5 and 2^-7. */ + b->d = 0x2800u; + b->dmin = 0x2000u; + } + } +} + +static float q4_dot(const block_q4_K *row, const float *x) { + float sum = 0.0f; + for (uint32_t k = 0; k < IN_DIM; k++) { + const block_q4_K *b = row + k / QK_K; + const uint32_t in_block = k % QK_K; + const uint32_t group = in_block / 32u; + const uint32_t lane = in_block % 32u; + uint8_t scale, minimum; + q4_scale_min(b->scales, group, &scale, &minimum); + const uint32_t byte_offset = (group >> 1u) * 32u + lane; + const uint32_t shift = (group & 1u) * 4u; + const uint32_t q = (b->qs[byte_offset] >> shift) & 15u; + const float w = f16_to_f32(b->d) * (float)scale * (float)q - + f16_to_f32(b->dmin) * (float)minimum; + sum += w * x[k]; + } + return sum; +} + +static uint64_t checksum(const float *values, uint64_t count) { + const uint8_t *bytes = (const uint8_t *)values; + const uint64_t byte_count = count * sizeof(float); + uint64_t hash = UINT64_C(1469598103934665603); + for (uint64_t i = 0; i < byte_count; i++) { + hash ^= bytes[i]; + hash *= UINT64_C(1099511628211); + } + return hash; +} + +static const char *arm_name(test_arm arm) { + switch (arm) { + case ARM_FIFO: return "fifo"; + case ARM_NATIVE: return "native"; + case ARM_OVERLAP: return "overlap"; + } + return "unknown"; +} + +static int encode_pair(fixture *f, ds4_gpu_tensor *out0, + ds4_gpu_tensor *out1, const ds4_gpu_tensor *x, + uint32_t rows) { + return ds4_gpu_matmul_q4_K_pair_tensor( + out0, out1, f->model, f->model_size, + f->weight0_offset, f->weight1_offset, + IN_DIM, OUT0_DIM, OUT1_DIM, x, rows); +} + +static int run_arm_once_with_options(fixture *f, test_arm arm, + uint32_t streams, + bool hold_test_transient) { + if (arm == ARM_NATIVE) { + ds4_gpu_set_stream(0); + return encode_pair(f, f->out[arm][0], f->out[arm][1], f->x, streams); + } + if (arm == ARM_FIFO) { + ds4_gpu_set_stream(0); + for (uint32_t i = 0; i < streams; i++) { + if (!encode_pair(f, f->out_row[arm][0][i], + f->out_row[arm][1][i], f->x_row[i], 1u)) { + return 0; + } + } + return 1; + } + + uint32_t submitted = 0; + for (uint32_t i = 0; i < streams; i++) { + ds4_gpu_set_stream((int)i); + if (!ds4_gpu_begin_commands() || + (hold_test_transient && + !ds4_gpu_test_hold_stream_transient(4096u)) || + !encode_pair(f, f->out_row[arm][0][i], + f->out_row[arm][1][i], f->x_row[i], 1u) || + !ds4_gpu_end_commands_async()) { + goto fail_overlap; + } + submitted++; + } + for (uint32_t i = 0; i < submitted; i++) { + if (!ds4_gpu_wait_stream((int)i)) goto fail_overlap; + } + ds4_gpu_set_stream(0); + return 1; + +fail_overlap: + for (uint32_t i = 0; i < submitted; i++) { + (void)ds4_gpu_wait_stream((int)i); + } + ds4_gpu_set_stream(0); + return 0; +} + +static int run_arm_once(fixture *f, test_arm arm, uint32_t streams) { + return run_arm_once_with_options(f, arm, streams, true); +} + +/* Exercise the intermediate-command-buffer lifetime too: the first pair and + * its test resource must survive flush, while wait_stream must retire both + * the pending and final command buffers before releasing transients. */ +static int run_overlap_flush_once(fixture *f, uint32_t streams) { + uint32_t submitted = 0; + for (uint32_t i = 0; i < streams; i++) { + ds4_gpu_set_stream((int)i); + if (!ds4_gpu_begin_commands() || + !ds4_gpu_test_hold_stream_transient(4096u) || + !encode_pair(f, f->out_row[ARM_OVERLAP][0][i], + f->out_row[ARM_OVERLAP][1][i], f->x_row[i], 1u) || + !ds4_gpu_flush_commands() || + !encode_pair(f, f->out_row[ARM_OVERLAP][0][i], + f->out_row[ARM_OVERLAP][1][i], f->x_row[i], 1u) || + !ds4_gpu_end_commands_async()) { + goto fail_overlap; + } + submitted++; + } + for (uint32_t i = 0; i < submitted; i++) { + if (!ds4_gpu_wait_stream((int)i)) goto fail_overlap; + } + ds4_gpu_set_stream(0); + return 1; + +fail_overlap: + for (uint32_t i = 0; i < submitted; i++) { + (void)ds4_gpu_wait_stream((int)i); + } + ds4_gpu_set_stream(0); + return 0; +} + +static void poison_arm(fixture *f, test_arm arm, float poison) { + for (uint32_t output = 0; output < 2u; output++) { + for (uint64_t i = 0; i < f->out_count[output]; i++) { + f->host[arm][output][i] = poison; + } + if (!ds4_gpu_tensor_write(f->out[arm][output], 0, + f->host[arm][output], + f->out_count[output] * sizeof(float))) { + fail("output poison write"); + } + } +} + +static void read_arm(fixture *f, test_arm arm) { + for (uint32_t output = 0; output < 2u; output++) { + if (!ds4_gpu_tensor_read(f->out[arm][output], 0, + f->host[arm][output], + f->out_count[output] * sizeof(float))) { + fail("output read"); + } + } +} + +static void check_outputs(fixture *f, uint32_t streams, float poison) { + const uint32_t dims[2] = {OUT0_DIM, OUT1_DIM}; + const uint64_t offsets[2] = {f->weight0_offset, f->weight1_offset}; + for (test_arm arm = ARM_FIFO; arm <= ARM_OVERLAP; arm++) read_arm(f, arm); + + for (uint32_t output = 0; output < 2u; output++) { + const uint64_t active = (uint64_t)streams * dims[output]; + const block_q4_K *matrix = (const block_q4_K *) + ((const uint8_t *)f->model + offsets[output]); + float max_abs = 0.0f; + float max_rel = 0.0f; + for (uint32_t stream = 0; stream < streams; stream++) { + for (uint32_t row = 0; row < dims[output]; row++) { + const uint64_t index = (uint64_t)stream * dims[output] + row; + const float expected = q4_dot( + matrix + (uint64_t)row * (IN_DIM / QK_K), + f->x_host + (uint64_t)stream * IN_DIM); + const float actual = f->host[ARM_FIFO][output][index]; + const float absolute = fabsf(actual - expected); + const float relative = absolute / fmaxf(1.0f, fabsf(expected)); + if (absolute > max_abs) max_abs = absolute; + if (relative > max_rel) max_rel = relative; + if (!isfinite(actual) || (absolute > 0.004f && relative > 2e-5f)) { + fprintf(stderr, + "Q4 stream CPU mismatch output=%u stream=%u row=%u " + "expected=%g actual=%g abs=%g rel=%g\n", + output, stream, row, expected, actual, + absolute, relative); + fail("CPU tolerance"); + } + } + } + for (uint64_t i = active; i < f->out_count[output]; i++) { + for (test_arm arm = ARM_FIFO; arm <= ARM_OVERLAP; arm++) { + if (memcmp(&f->host[arm][output][i], &poison, + sizeof(poison)) != 0) { + fprintf(stderr, + "Q4 stream canary mismatch arm=%s output=%u index=%llu\n", + arm_name(arm), output, (unsigned long long)i); + fail("output canary"); + } + } + } + for (test_arm arm = ARM_NATIVE; arm <= ARM_OVERLAP; arm++) { + if (memcmp(f->host[ARM_FIFO][output], f->host[arm][output], + active * sizeof(float)) != 0) { + uint64_t first = 0; + while (first < active && + memcmp(&f->host[ARM_FIFO][output][first], + &f->host[arm][output][first], sizeof(float)) == 0) { + first++; + } + fprintf(stderr, + "Q4 stream exact mismatch arm=%s output=%u first=%llu " + "fifo=%g actual=%g\n", + arm_name(arm), output, (unsigned long long)first, + first < active ? f->host[ARM_FIFO][output][first] : 0.0f, + first < active ? f->host[arm][output][first] : 0.0f); + fail("bitwise parity"); + } + } + fprintf(stderr, + "Q4 stream N=%u output=%u CPU max_abs=%g max_rel=%g " + "checksum=%016llx bitwise=1 canary=1\n", + streams, output, max_abs, max_rel, + (unsigned long long)checksum(f->host[ARM_FIFO][output], active)); + } +} + +static void check_stats_equal(const ds4_gpu_stream_test_stats *before, + const ds4_gpu_stream_test_stats *after, + const char *scope) { + if (before->tensor_live_bytes != after->tensor_live_bytes || + before->tensor_live_count != after->tensor_live_count || + after->transient_references != 0u || + after->pending_command_buffers != 0u || + after->last_command_buffers != 0u) { + fprintf(stderr, + "Q4 stream stats leak scope=%s live_bytes=%llu->%llu " + "live_count=%u->%u transient=%llu->%llu pending=%u last=%u\n", + scope, + (unsigned long long)before->tensor_live_bytes, + (unsigned long long)after->tensor_live_bytes, + before->tensor_live_count, after->tensor_live_count, + (unsigned long long)before->transient_references, + (unsigned long long)after->transient_references, + after->pending_command_buffers, + after->last_command_buffers); + fail("Metal allocation/transient counters"); + } +} + +static int compare_double(const void *a, const void *b) { + const double av = *(const double *)a; + const double bv = *(const double *)b; + return (av > bv) - (av < bv); +} + +static double percentile(const double *samples, uint32_t count, double p) { + double sorted[MAX_TIMING_BLOCKS]; + memcpy(sorted, samples, (size_t)count * sizeof(sorted[0])); + qsort(sorted, count, sizeof(sorted[0]), compare_double); + const double rank = p * (double)(count - 1u); + const uint32_t lo = (uint32_t)rank; + const uint32_t hi = lo + 1u < count ? lo + 1u : lo; + const double fraction = rank - (double)lo; + return sorted[lo] + (sorted[hi] - sorted[lo]) * fraction; +} + +static double run_timing_group(fixture *f, test_arm arm, uint32_t streams, + uint64_t iterations) { + const double start = now_seconds(); + for (uint64_t i = 0; i < iterations; i++) { + /* The transient hook is a lifecycle oracle, not production work. Do + * not charge its allocation/retain overhead to the overlap arm. */ + if (!run_arm_once_with_options(f, arm, streams, false)) { + fail("timing arm"); + } + } + return (now_seconds() - start) / (double)iterations; +} + +static void run_timing_pair(fixture *f, test_arm baseline, test_arm candidate, + uint32_t streams, uint32_t blocks, + uint64_t group_iterations) { + double baseline_samples[MAX_TIMING_BLOCKS]; + double candidate_samples[MAX_TIMING_BLOCKS]; + for (uint32_t block = 0; block < blocks; block++) { + double base_total = 0.0; + double candidate_total = 0.0; + if ((block & 1u) == 0u) { + base_total += run_timing_group( + f, baseline, streams, group_iterations); + candidate_total += run_timing_group( + f, candidate, streams, group_iterations); + candidate_total += run_timing_group( + f, candidate, streams, group_iterations); + base_total += run_timing_group( + f, baseline, streams, group_iterations); + } else { + candidate_total += run_timing_group( + f, candidate, streams, group_iterations); + base_total += run_timing_group( + f, baseline, streams, group_iterations); + base_total += run_timing_group( + f, baseline, streams, group_iterations); + candidate_total += run_timing_group( + f, candidate, streams, group_iterations); + } + baseline_samples[block] = base_total * 0.5; + candidate_samples[block] = candidate_total * 0.5; + } + + const double baseline_median = percentile(baseline_samples, blocks, 0.50); + const double candidate_median = percentile(candidate_samples, blocks, 0.50); + const double speedup = candidate_median > 0.0 ? + baseline_median / candidate_median : 0.0; + fprintf(stderr, + "Q4 stream timing microkernel=%s/%s N=%u blocks=%u group_iters=%llu " + "baseline_ms[p25/med/p75]=%.3f/%.3f/%.3f " + "candidate_ms[p25/med/p75]=%.3f/%.3f/%.3f speedup=%.3fx " + "aggregate_candidate=%.2f rows/s wall=encode+GPU full_logits=0\n", + arm_name(baseline), arm_name(candidate), streams, blocks, + (unsigned long long)group_iterations, + percentile(baseline_samples, blocks, 0.25) * 1000.0, + baseline_median * 1000.0, + percentile(baseline_samples, blocks, 0.75) * 1000.0, + percentile(candidate_samples, blocks, 0.25) * 1000.0, + candidate_median * 1000.0, + percentile(candidate_samples, blocks, 0.75) * 1000.0, + speedup, + candidate_median > 0.0 ? (double)streams / candidate_median : 0.0); +} + +static void fixture_init(fixture *f) { + memset(f, 0, sizeof(*f)); + if (sizeof(block_q4_K) != 144u) fail("unexpected Q4_K block size"); + + f->row_bytes = (IN_DIM / QK_K) * sizeof(block_q4_K); + f->weight0_offset = 0; + f->weight1_offset = OUT0_DIM * f->row_bytes; + const uint64_t weights_end = f->weight1_offset + OUT1_DIM * f->row_bytes; + const uint64_t page = (uint64_t)getpagesize(); + f->model_size = align_up(weights_end, page); + if (posix_memalign(&f->model, (size_t)page, (size_t)f->model_size) != 0) { + fail("model allocation"); + } + memset(f->model, 0, (size_t)f->model_size); + fill_q4_matrix((block_q4_K *)((uint8_t *)f->model + f->weight0_offset), + OUT0_DIM, 17u); + fill_q4_matrix((block_q4_K *)((uint8_t *)f->model + f->weight1_offset), + OUT1_DIM, 7919u); + + const uint64_t x_count = (uint64_t)MAX_STREAMS * IN_DIM; + f->x_host = malloc((size_t)x_count * sizeof(float)); + if (!f->x_host) fail("host activation allocation"); + for (uint32_t stream = 0; stream < MAX_STREAMS; stream++) { + for (uint32_t k = 0; k < IN_DIM; k++) { + const int32_t value = (int32_t)( + (stream * 19u + k * 7u + (stream ^ k) * 3u) % 127u) - 63; + f->x_host[(uint64_t)stream * IN_DIM + k] = (float)value / 128.0f; + } + } + + if (!ds4_gpu_init() || !ds4_gpu_set_model_map(f->model, f->model_size)) { + fail("Metal init/model map"); + } + ds4_gpu_set_quality(false); + f->x = ds4_gpu_tensor_alloc(x_count * sizeof(float)); + if (!f->x || !ds4_gpu_tensor_write(f->x, 0, f->x_host, + x_count * sizeof(float))) { + fail("activation tensor"); + } + for (uint32_t stream = 0; stream < MAX_STREAMS; stream++) { + f->x_row[stream] = ds4_gpu_tensor_view( + f->x, (uint64_t)stream * IN_DIM * sizeof(float), + IN_DIM * sizeof(float)); + if (!f->x_row[stream]) fail("activation row view"); + } + + const uint32_t dims[2] = {OUT0_DIM, OUT1_DIM}; + for (uint32_t output = 0; output < 2u; output++) { + f->out_count[output] = (uint64_t)MAX_STREAMS * dims[output] + GUARD_FLOATS; + for (test_arm arm = ARM_FIFO; arm <= ARM_OVERLAP; arm++) { + f->host[arm][output] = malloc( + (size_t)f->out_count[output] * sizeof(float)); + f->out[arm][output] = ds4_gpu_tensor_alloc( + f->out_count[output] * sizeof(float)); + if (!f->host[arm][output] || !f->out[arm][output]) { + fail("output allocation"); + } + for (uint32_t stream = 0; stream < MAX_STREAMS; stream++) { + f->out_row[arm][output][stream] = ds4_gpu_tensor_view( + f->out[arm][output], + (uint64_t)stream * dims[output] * sizeof(float), + (uint64_t)dims[output] * sizeof(float)); + if (!f->out_row[arm][output][stream]) { + fail("output row view"); + } + } + } + } +} + +static void fixture_release_tensors(fixture *f) { + for (uint32_t output = 0; output < 2u; output++) { + for (test_arm arm = ARM_FIFO; arm <= ARM_OVERLAP; arm++) { + for (uint32_t stream = 0; stream < MAX_STREAMS; stream++) { + ds4_gpu_tensor_free(f->out_row[arm][output][stream]); + } + ds4_gpu_tensor_free(f->out[arm][output]); + free(f->host[arm][output]); + } + } + for (uint32_t stream = 0; stream < MAX_STREAMS; stream++) { + ds4_gpu_tensor_free(f->x_row[stream]); + } + ds4_gpu_tensor_free(f->x); + f->x = NULL; +} + +static void fixture_destroy(fixture *f) { + fixture_release_tensors(f); + + /* Cleanup itself is responsible for every stream. Leave one empty async + * command buffer and one retained test resource on the last stream so a + * cleanup implementation that only drains stream zero cannot false-pass. */ + ds4_gpu_set_stream((int)MAX_STREAMS - 1); + if (!ds4_gpu_begin_commands() || + !ds4_gpu_test_hold_stream_transient(4096u) || + !ds4_gpu_end_commands_async()) { + fail("cleanup in-flight setup"); + } + ds4_gpu_stream_test_stats inflight; + ds4_gpu_test_stream_stats(&inflight); + if (inflight.last_command_buffers == 0u || + inflight.transient_references == 0u) { + fail("cleanup in-flight state was not observable"); + } + ds4_gpu_cleanup(); + free(f->x_host); + free(f->model); + memset(f, 0, sizeof(*f)); +} + +int main(void) { + static const uint32_t stream_counts[] = {2u, 4u, 8u}; + const float poison = -12345.25f; + const uint64_t warmup = env_u64("DS4_TEST_Q4_STREAM_WARMUP", 1u, 1u, 64u); + const uint64_t iterations = env_u64( + "DS4_TEST_Q4_STREAM_ITERS", 2u, 1u, 10000u); + const uint64_t soak = env_u64( + "DS4_TEST_Q4_STREAM_SOAK", 8u, 1u, 100000u); + const bool timing = getenv("DS4_TEST_Q4_STREAM_TIMING") != NULL; + const uint32_t timing_blocks = (uint32_t)env_u64( + "DS4_TEST_Q4_STREAM_TIMING_BLOCKS", 5u, 5u, MAX_TIMING_BLOCKS); + const uint64_t timing_iterations = env_u64( + "DS4_TEST_Q4_STREAM_TIMING_ITERS", 20u, 1u, 10000u); + + test_overlap_policy(); + + fprintf(stderr, + "Q4 stream oracle model_untracked=%s warmup=%llu iterations=%llu " + "soak=%llu timing=%d footprint=synth-resident\n", + getenv("DS4_METAL_MODEL_UNTRACKED") ? "on" : "off", + (unsigned long long)warmup, + (unsigned long long)iterations, + (unsigned long long)soak, + timing ? 1 : 0); + const task_memory process_before = read_task_memory(); + print_task_memory("process-start", process_before); + + fixture f; + fixture_init(&f); + ds4_gpu_stream_test_stats initial; + ds4_gpu_test_stream_stats(&initial); + if (initial.active_queue_mask != 1u || + initial.model_residency_queue_mask != 1u) { + fail("initial model residency"); + } + for (uint32_t c = 0; c < sizeof(stream_counts) / sizeof(stream_counts[0]); c++) { + const uint32_t streams = stream_counts[c]; + for (test_arm arm = ARM_FIFO; arm <= ARM_OVERLAP; arm++) { + poison_arm(&f, arm, poison); + for (uint64_t i = 0; i < warmup; i++) { + if (!run_arm_once(&f, arm, streams)) fail("warmup arm"); + } + } + + ds4_gpu_stream_test_stats stats_before; + ds4_gpu_test_stream_stats(&stats_before); + if (stats_before.transient_references != 0u) { + fail("transients retained after warmup"); + } + + for (test_arm arm = ARM_FIFO; arm <= ARM_OVERLAP; arm++) { + for (uint64_t i = 0; i < iterations; i++) { + if (!run_arm_once(&f, arm, streams)) fail("test arm"); + } + } + check_outputs(&f, streams, poison); + if (timing) { + run_timing_pair(&f, ARM_FIFO, ARM_OVERLAP, streams, + timing_blocks, timing_iterations); + run_timing_pair(&f, ARM_NATIVE, ARM_OVERLAP, streams, + timing_blocks, timing_iterations); + } + + ds4_gpu_stream_test_stats stats_after; + ds4_gpu_test_stream_stats(&stats_after); + check_stats_equal(&stats_before, &stats_after, "A/B"); + + if (!run_overlap_flush_once(&f, streams)) { + fail("overlap flush lifecycle"); + } + ds4_gpu_stream_test_stats flush_after; + ds4_gpu_test_stream_stats(&flush_after); + check_stats_equal(&stats_before, &flush_after, "flush"); + check_outputs(&f, streams, poison); + const uint32_t expected_mask = (1u << streams) - 1u; + if ((stats_after.active_queue_mask & expected_mask) != expected_mask || + (stats_after.model_residency_queue_mask & expected_mask) != expected_mask) { + fprintf(stderr, + "Q4 stream queue residency N=%u active=0x%02x " + "model=0x%02x expected=0x%02x\n", + streams, stats_after.active_queue_mask, + stats_after.model_residency_queue_mask, expected_mask); + fail("residency missing from an active queue"); + } + + ds4_gpu_stream_test_stats soak_before; + ds4_gpu_test_stream_stats(&soak_before); + for (uint64_t i = 0; i < soak; i++) { + if (!run_arm_once(&f, ARM_OVERLAP, streams)) fail("overlap soak"); + } + ds4_gpu_stream_test_stats soak_after; + ds4_gpu_test_stream_stats(&soak_after); + check_stats_equal(&soak_before, &soak_after, "soak"); + fprintf(stderr, + "Q4 stream N=%u soak=%llu active_queues=0x%02x " + "model_residency=0x%02x transient=%llu counters=stable\n", + streams, (unsigned long long)soak, + soak_after.active_queue_mask, + soak_after.model_residency_queue_mask, + (unsigned long long)soak_after.transient_references); + } + + /* Rebuild residency after all queues already exist. Model replacement in + * a long-lived engine must attach the new set to every existing queue. */ + if (!ds4_gpu_set_model_map(f.model, f.model_size)) { + fail("model residency rebuild"); + } + ds4_gpu_stream_test_stats rebuilt; + ds4_gpu_test_stream_stats(&rebuilt); + if (rebuilt.active_queue_mask != 0xffu || + rebuilt.model_residency_queue_mask != 0xffu) { + fprintf(stderr, + "Q4 residency rebuild active=0x%02x model=0x%02x\n", + rebuilt.active_queue_mask, + rebuilt.model_residency_queue_mask); + fail("model residency rebuild queue coverage"); + } + + print_task_memory("before-cleanup", read_task_memory()); + fixture_destroy(&f); + ds4_gpu_stream_test_stats cleanup_stats; + ds4_gpu_test_stream_stats(&cleanup_stats); + if (cleanup_stats.tensor_live_bytes != 0u || + cleanup_stats.tensor_live_count != 0u || + cleanup_stats.transient_references != 0u || + cleanup_stats.pending_command_buffers != 0u || + cleanup_stats.last_command_buffers != 0u || + cleanup_stats.active_queue_mask != 0u) { + fail("cleanup counters did not return to zero"); + } + print_task_memory("after-cleanup", read_task_memory()); + fprintf(stderr, + "test_metal_q4_streams PASS counts=2,4,8 bitwise=1 canary=1 " + "model_residency=1 counters=stable model_untracked=%s\n", + getenv("DS4_METAL_MODEL_UNTRACKED") ? "on" : "off"); + return 0; +} + +#else + +int main(void) { + fprintf(stderr, "test_metal_q4_streams skipped (Metal requires macOS)\n"); + return 0; +} + +#endif From 7bdbff75eac84877798e6c65fb767510a1e9af8c Mon Sep 17 00:00:00 2001 From: Giorgio Oppo Date: Fri, 14 Aug 2026 16:12:09 +0200 Subject: [PATCH 029/189] metal: add opt-in Q4 SSD session union --- Makefile | 29 +- ds4.c | 479 ++++++++++++++++++++++++++++++- tests/test_metal_session_batch.c | 226 +++++++++++++-- 3 files changed, 696 insertions(+), 38 deletions(-) diff --git a/Makefile b/Makefile index 6495a0b26..e918897f8 100644 --- a/Makefile +++ b/Makefile @@ -68,7 +68,7 @@ DS4_LINK_LIBS ?= $(CUDA_LDLIBS) METAL_LDLIBS := $(LDLIBS) endif -.PHONY: all help clean test test-rocm test-glm53-kda-rocm test-metal-session-batch test-metal-q4-streams test-metal-exactn-oracle test-metal-dspark-capture test-metal-iq2-midonly test-metal-iq2-ssd-grouped-mm test-metal-iq2-live-index test-mxfp4-cuda test-mxfp4-rocm test-mmq-parity-cuda test-cuda-session-batch test-cuda-mixed-batch dspark-acceptance dspark-verify-depth rocm-dspark-acceptance rocm-dspark-verify-depth mtp-verify-depth cpu cuda cuda-spark cuda-generic cuda-regression strix-halo rocm +.PHONY: all help clean test test-rocm test-glm53-kda-rocm test-metal-session-batch test-metal-session-batch-ssd test-metal-q4-streams test-metal-exactn-oracle test-metal-dspark-capture test-metal-iq2-midonly test-metal-iq2-ssd-grouped-mm test-metal-iq2-live-index test-mxfp4-cuda test-mxfp4-rocm test-mmq-parity-cuda test-cuda-session-batch test-cuda-mixed-batch dspark-acceptance dspark-verify-depth rocm-dspark-acceptance rocm-dspark-verify-depth mtp-verify-depth cpu cuda cuda-spark cuda-generic cuda-regression strix-halo rocm ifeq ($(UNAME_S),Darwin) .PHONY: metal-decode-schedule-bench metal-prefill-variant-bench check-mxfp4-half-lut test-mxfp4-metal @@ -80,6 +80,7 @@ help: @echo " make Build Metal ./ds4, ./ds4-server, ./ds4-bench, ./ds4-eval, and ./ds4-agent" @echo " make cpu Build CPU-only ./ds4, ./ds4-server, ./ds4-bench, ./ds4-eval, and ./ds4-agent" @echo " make test Build and run tests" + @echo " make test-metal-session-batch-ssd Exact-logit Metal SSD union control/candidate oracle" @echo " make test-metal-q4-streams Check resident Q4 Metal stream overlap" @echo " make test-metal-dspark-capture Check fused DSpark HC capture bitwise" @echo " make test-metal-iq2-midonly Check M1 IQ2 addr mid-only output and sentinels" @@ -120,6 +121,32 @@ tests/test_metal_session_batch: tests/test_metal_session_batch.o $(CORE_OBJS) test-metal-session-batch: tests/test_metal_session_batch DS4_TEST_MODEL="$(DS4_TEST_MODEL)" ./tests/test_metal_session_batch +test-metal-session-batch-ssd: tests/test_metal_session_batch + env -u DS4_METAL_ENABLE_Q4_SSD_SESSION_UNION \ + -u DS4_METAL_REQUIRE_EXACT_ROWS_PERSISTENT_CACHE \ + -u DS4_TEST_SSD_UNION_POLICY_SWITCH \ + DS4_METAL_DISABLE_Q4_SSD_SESSION_UNION=1 \ + DS4_METAL_REQUIRE_Q4_SSD_SESSION_UNION=1 \ + DS4_METAL_DISABLE_EXACT_ROWS_PERSISTENT_CACHE=1 \ + DS4_TEST_SSD_STREAMING=1 DS4_TEST_SESSION_COUNT=5 \ + DS4_TEST_SSD_CACHE_EXPERTS="$(DS4_TEST_SSD_CACHE_EXPERTS)" \ + DS4_TEST_SESSION_BATCH_TIMING=1 \ + DS4_TEST_SESSION_BATCH_ARM=control \ + DS4_TEST_MODEL="$(DS4_TEST_MODEL)" \ + ./tests/test_metal_session_batch + env -u DS4_METAL_DISABLE_Q4_SSD_SESSION_UNION \ + -u DS4_METAL_ENABLE_Q4_SSD_SESSION_UNION \ + -u DS4_METAL_DISABLE_EXACT_ROWS_PERSISTENT_CACHE \ + DS4_METAL_REQUIRE_Q4_SSD_SESSION_UNION=1 \ + DS4_METAL_REQUIRE_EXACT_ROWS_PERSISTENT_CACHE=1 \ + DS4_TEST_SSD_STREAMING=1 DS4_TEST_SESSION_COUNT=5 \ + DS4_TEST_SSD_UNION_POLICY_SWITCH=1 \ + DS4_TEST_SSD_CACHE_EXPERTS="$(DS4_TEST_SSD_CACHE_EXPERTS)" \ + DS4_TEST_SESSION_BATCH_TIMING=1 \ + DS4_TEST_SESSION_BATCH_ARM=candidate \ + DS4_TEST_MODEL="$(DS4_TEST_MODEL)" \ + ./tests/test_metal_session_batch + tests/test_metal_q4_streams.o: tests/test_metal_q4_streams.c ds4.h ds4_gpu.h $(CC) $(CFLAGS) -DDS4_TEST_HOOKS -I. -c -o $@ $< diff --git a/ds4.c b/ds4.c index c56addf7d..45db9399f 100644 --- a/ds4.c +++ b/ds4.c @@ -38403,13 +38403,15 @@ static void metal_graph_release_exactn_union_row_alias( static DS4_MAYBE_UNUSED bool metal_graph_bind_exactn_union_row( ds4_gpu_graph *g, + const ds4_gpu_graph *workspace, uint32_t row, ds4_gpu_tensor *cur_hc, ds4_gpu_tensor *next_hc, metal_graph_exactn_union_row_alias *alias) { if (!alias) return false; - if (!g || !cur_hc || !next_hc || - g->active_tier < 0 || row >= g->prefill_cap || alias->active) { + if (!g || !workspace || !cur_hc || !next_hc || + g->active_tier < 0 || workspace->active_tier != g->active_tier || + row >= workspace->prefill_cap || alias->active) { return false; } if (!alias->after_attn_hc) { @@ -38419,19 +38421,19 @@ static DS4_MAYBE_UNUSED bool metal_graph_bind_exactn_union_row( const uint64_t mix_hc = 2ull * DS4_N_HC + (uint64_t)DS4_N_HC * DS4_N_HC; alias->after_attn_hc = ds4_gpu_tensor_view( - metal_graph_batch_after_attn_hc(g), + metal_graph_batch_after_attn_hc(workspace), (uint64_t)row * hc_dim * sizeof(float), hc_dim * sizeof(float)); alias->ffn_cur = ds4_gpu_tensor_view( - metal_graph_batch_ffn_cur(g), + metal_graph_batch_ffn_cur(workspace), (uint64_t)row * DS4_N_EMBD * sizeof(float), (uint64_t)DS4_N_EMBD * sizeof(float)); alias->ffn_norm = ds4_gpu_tensor_view( - metal_graph_batch_ffn_norm(g), + metal_graph_batch_ffn_norm(workspace), (uint64_t)row * DS4_N_EMBD * sizeof(float), (uint64_t)DS4_N_EMBD * sizeof(float)); alias->hc_split = ds4_gpu_tensor_view( - metal_graph_batch_hc_split(g), + metal_graph_batch_hc_split(workspace), (uint64_t)row * mix_hc * sizeof(float), mix_hc * sizeof(float)); alias->hc_pre = ds4_gpu_tensor_view( @@ -38447,11 +38449,11 @@ static DS4_MAYBE_UNUSED bool metal_graph_bind_exactn_union_row( 2ull * DS4_N_HC * sizeof(float), (uint64_t)DS4_N_HC * DS4_N_HC * sizeof(float)); alias->router_selected = ds4_gpu_tensor_view( - metal_graph_batch_router_selected(g), + metal_graph_batch_router_selected(workspace), (uint64_t)row * DS4_N_EXPERT_USED * sizeof(int32_t), (uint64_t)DS4_N_EXPERT_USED * sizeof(int32_t)); alias->router_weights = ds4_gpu_tensor_view( - metal_graph_batch_router_weights(g), + metal_graph_batch_router_weights(workspace), (uint64_t)row * DS4_N_EXPERT_USED * sizeof(float), (uint64_t)DS4_N_EXPERT_USED * sizeof(float)); } @@ -38647,7 +38649,7 @@ static bool metal_graph_verify_decode_exactn_union_impl( for (uint32_t row = 0; ok && row < n_tokens; row++) { const uint32_t pos = start + row; ok = metal_graph_bind_exactn_union_row( - g, row, cur_rows[row], next_rows[row], + g, g, row, cur_rows[row], next_rows[row], &row_aliases[row]); if (ok) { ok = metal_graph_encode_decode_layer_phase( @@ -38699,7 +38701,7 @@ static bool metal_graph_verify_decode_exactn_union_impl( for (uint32_t row = 0; ok && row < n_tokens; row++) { const uint32_t pos = start + row; ok = metal_graph_bind_exactn_union_row( - g, row, cur_rows[row], next_rows[row], + g, g, row, cur_rows[row], next_rows[row], &row_aliases[row]); if (ok) { ok = ds4_gpu_stream_expert_exact_rows_set_row(row) != 0; @@ -70513,6 +70515,10 @@ int ds4_test_q4_stream_overlap_policy( #endif #if !defined(DS4_NO_GPU) && defined(__APPLE__) +static bool metal_graph_mixed_workspace_compatible( + const ds4_gpu_graph *owner, + const ds4_gpu_graph *member); + static bool ds4_engine_has_q4_stream_overlap_weights(const ds4_engine *e) { if (!e) return false; for (uint32_t il = 0; il < DS4_N_LAYER; il++) { @@ -70563,6 +70569,142 @@ static bool ds4_session_q4_stream_overlap_eligible(ds4_session *s) { getenv("DS4_METAL_MOE_STAGE_PROFILE") == NULL && ds4_engine_has_q4_stream_overlap_weights(e); } + +static bool ds4_q4_ssd_session_union_required(void) { + if (metal_graph_tp_env_flag( + "DS4_METAL_DISABLE_Q4_SSD_SESSION_UNION", false)) { + return false; + } + return metal_graph_tp_env_flag( + "DS4_METAL_REQUIRE_Q4_SSD_SESSION_UNION", false); +} + +static bool ds4_q4_ssd_session_union_requested(void) { + const bool disabled = metal_graph_tp_env_flag( + "DS4_METAL_DISABLE_Q4_SSD_SESSION_UNION", false); + const bool enabled = metal_graph_tp_env_flag( + "DS4_METAL_ENABLE_Q4_SSD_SESSION_UNION", false) || + ds4_q4_ssd_session_union_required(); + return enabled && !disabled; +} + +static bool ds4_q4_ssd_session_union_workspace_compatible( + const ds4_gpu_graph *owner, + const ds4_gpu_graph *member) { + if (!owner || !member || + !metal_graph_mixed_workspace_compatible(owner, member)) { + return false; + } + const int t = owner->active_tier; + return t == 0 && member->active_tier == t && + owner->batch_after_attn_hc_by_tier[t] == + member->batch_after_attn_hc_by_tier[t] && + owner->batch_ffn_cur_by_tier[t] == + member->batch_ffn_cur_by_tier[t] && + owner->batch_ffn_norm_by_tier[t] == + member->batch_ffn_norm_by_tier[t] && + owner->batch_hc_split_by_tier[t] == + member->batch_hc_split_by_tier[t] && + owner->batch_router_selected_by_tier[t] == + member->batch_router_selected_by_tier[t] && + owner->batch_router_weights_by_tier[t] == + member->batch_router_weights_by_tier[t]; +} + +static bool ds4_engine_q4_ssd_session_union_layout_supported( + const ds4_engine *e, + const ds4_gpu_graph *g) { + if (!e || !g || DS4_MODEL_FAMILY != DS4_MODEL_FAMILY_DEEPSEEK4 || + DS4_N_LAYER == 0u || DS4_N_EXPERT_USED != 6u || + DS4_N_EXPERT < 128u || DS4_N_EXPERT > 384u || + !e->weights.output || + e->weights.output->type != DS4_TENSOR_Q8_0) { + return false; + } + for (uint32_t il = 0; il < DS4_N_LAYER; il++) { + const ds4_layer_weights *layer = &e->weights.layer[il]; + if (!weights_layer_has_required(layer, il) || + !layer->attn_q_a || !layer->attn_kv || + layer->attn_q_a->type != DS4_TENSOR_Q4_K || + layer->attn_kv->type != DS4_TENSOR_Q4_K || + !layer->ffn_gate_shexp || !layer->ffn_up_shexp || + !layer->ffn_down_shexp || + layer->ffn_gate_shexp->type != DS4_TENSOR_Q8_0 || + layer->ffn_up_shexp->type != DS4_TENSOR_Q8_0 || + layer->ffn_down_shexp->type != DS4_TENSOR_Q8_0 || + !layer->ffn_gate_exps || !layer->ffn_up_exps || + !layer->ffn_down_exps || + layer->ffn_gate_exps->type != DS4_TENSOR_IQ2_XXS || + layer->ffn_up_exps->type != DS4_TENSOR_IQ2_XXS || + layer->ffn_down_exps->type != DS4_TENSOR_Q2_K || + !weights_streaming_layer_experts_uniform(&e->weights, il) || + !metal_graph_decode_iq2_selected_slots_expected(g, layer) || + metal_graph_decode_cpu_router_applicable(g, layer)) { + return false; + } + } + return true; +} + +static bool ds4_session_q4_ssd_session_union_eligible( + ds4_session *s, + const ds4_gpu_graph *workspace, + int count) { + if (!s || !s->engine || !workspace || count < 2 || + count > DS4_METAL_EXACTN_UNION_MAX_ROWS) { + return false; + } + const ds4_engine *e = s->engine; + const ds4_gpu_graph *g = &s->graph; + const uint64_t working_set = + (uint64_t)(uint32_t)count * (uint64_t)DS4_N_EXPERT_USED; + return ds4_q4_ssd_session_union_requested() && + e->backend == DS4_BACKEND_METAL && + e->support_kind == DS4_SUPPORT_NONE && + e->ssd_streaming && !e->tp.active && !e->multi_tier && + e->share_session_prefill_workspace && + e->shared_prefill_workspace_ready && + engine_placement_session_count(e) >= (uint32_t)count && + ds4_q4_ssd_session_union_workspace_compatible( + &e->shared_prefill_workspace, workspace) && + working_set <= UINT32_MAX && + e->ssd_streaming_cache_experts >= (uint32_t)working_set && + ds4_gpu_stream_expert_cache_configured_count() >= + (uint32_t)working_set && + !s->distributed && !ds4_session_is_cpu(s) && + !ds4_session_is_glm(s) && !ds4_session_cancelled(s) && + s->checkpoint_valid && + g->ssd_streaming && !g->placement && !g->quality && + g->tp_world < 2u && g->raw_cap != 0u && + g->active_tier == 0 && workspace->active_tier == 0 && + workspace->prefill_cap >= (uint32_t)count && + !g->materialize_ffn_out && !g->decode_stage_profile && + !g->decode_index_stage_profile && !g->output_stage_profile && + !g_expert_profile.active && + !graph_power_throttle_enabled(g) && + !g->spec_exactn_union_collect_routes && + !metal_graph_hc_norm_fusion_check_enabled() && + !metal_graph_use_reference_shared_down_hc() && + !metal_graph_use_pro_q4_cpu_router() && + !metal_graph_directional_steering_attn_enabled(g) && + !metal_graph_directional_steering_ffn_enabled(g) && + metal_graph_stream_decode_static_map_enabled() && + metal_graph_debug_get_config()->prefix == NULL && + getenv("DS4_METAL_GRAPH_DUMP_PREFIX") == NULL && + getenv("DS4_METAL_DECODE_STAGE_PROFILE") == NULL && + getenv("DS4_METAL_LAYER_STAGE_PROFILE") == NULL && + getenv("DS4_METAL_ATTN_OUT_STAGE_PROFILE") == NULL && + getenv("DS4_METAL_FLASH_ATTN_STAGE_PROFILE") == NULL && + getenv("DS4_METAL_MOE_ONE_STAGE_PROFILE") == NULL && + getenv("DS4_METAL_MOE_STAGE_PROFILE") == NULL && + getenv("DS4_METAL_SELECTED_PROFILE") == NULL && + getenv("DS4_METAL_Q4_SELECTED_PROFILE") == NULL && + getenv("DS4_METAL_DSPARK_EXACT_ROWS_ASYNC_TAILS") == NULL && + getenv("DS4_TP_ABLATE") == NULL && + getenv("DS4_MOE_REPLAY_SELECTED_IDS") == NULL && + getenv("DS4_MOE_RECORD_SELECTED_IDS") == NULL && + ds4_engine_q4_ssd_session_union_layout_supported(e, g); +} #endif int ds4_session_eval(ds4_session *s, int token, char *err, size_t errlen) { @@ -71238,6 +71380,60 @@ static bool ds4_sessions_eval_batch_metal_supported( return true; } +static bool ds4_sessions_q4_ssd_session_union_supported( + ds4_decode_item *items, + int count, + ds4_engine *e) { +#if !defined(__APPLE__) + (void)items; + (void)count; + (void)e; + return false; +#else + if (!items || count < 2 || + count > DS4_METAL_EXACTN_UNION_MAX_ROWS || !e || + !items[0].session) { + return false; + } + ds4_gpu_graph *workspace = &items[0].session->graph; + const uint64_t rows = (uint64_t)(uint32_t)count; + const uint64_t hc_dim = (uint64_t)DS4_N_HC * DS4_N_EMBD; + const uint64_t mix_hc = 2ull * DS4_N_HC + + (uint64_t)DS4_N_HC * DS4_N_HC; + if (!metal_graph_batch_after_attn_hc(workspace) || + !metal_graph_batch_ffn_cur(workspace) || + !metal_graph_batch_ffn_norm(workspace) || + !metal_graph_batch_hc_split(workspace) || + !metal_graph_batch_router_selected(workspace) || + !metal_graph_batch_router_weights(workspace) || + ds4_gpu_tensor_bytes(metal_graph_batch_after_attn_hc(workspace)) < + rows * hc_dim * sizeof(float) || + ds4_gpu_tensor_bytes(metal_graph_batch_ffn_cur(workspace)) < + rows * DS4_N_EMBD * sizeof(float) || + ds4_gpu_tensor_bytes(metal_graph_batch_ffn_norm(workspace)) < + rows * DS4_N_EMBD * sizeof(float) || + ds4_gpu_tensor_bytes(metal_graph_batch_hc_split(workspace)) < + rows * mix_hc * sizeof(float) || + ds4_gpu_tensor_bytes(metal_graph_batch_router_selected(workspace)) < + rows * DS4_N_EXPERT_USED * sizeof(int32_t) || + ds4_gpu_tensor_bytes(metal_graph_batch_router_weights(workspace)) < + rows * DS4_N_EXPERT_USED * sizeof(float)) { + return false; + } + for (int i = 0; i < count; i++) { + ds4_session *s = items[i].session; + if (!s || s->engine != e || + !ds4_q4_ssd_session_union_workspace_compatible( + &e->shared_prefill_workspace, &s->graph) || + !ds4_session_q4_ssd_session_union_eligible( + s, workspace, count)) { + return false; + } + } + return true; +#endif +} + static bool ds4_sessions_q4_stream_overlap_supported( ds4_decode_item *items, int count, @@ -71633,12 +71829,246 @@ static bool ds4_sessions_tp_recv_logits( return true; } +#if defined(__APPLE__) +/* Queue-0 SSD batch for the AProjQ4 + routed IQ2/Q2 layout. Each layer first + * encodes every independent session through its router into distinct rows of + * one borrowed prefill workspace. The established exact-row backend then + * loads one immutable union of selected experts and runs every routed tail. + * A synchronous boundary retires that layer before the shared cache can be + * reused, avoiding both a second cache and the unordered multi-queue epoch. */ +static bool metal_graph_encode_q4_ssd_session_union( + ds4_decode_item *items, + int count, + const ds4_model *model, + const ds4_weights *weights) { + if (!items || count < 2 || + count > DS4_METAL_EXACTN_UNION_MAX_ROWS || + !items[0].session || !model || !weights) { + return false; + } + + ds4_gpu_graph *workspace = &items[0].session->graph; + metal_graph_exactn_union_row_alias + aliases[DS4_METAL_EXACTN_UNION_MAX_ROWS] = {0}; + bool saved_capture[DS4_METAL_EXACTN_UNION_MAX_ROWS] = {0}; + bool collecting = false; + bool ok = true; + + for (int i = 0; i < count; i++) { + saved_capture[i] = + items[i].session->graph.spec_capture_prefixes; + } + + ds4_gpu_tensor *selected_rows = ds4_gpu_tensor_view( + metal_graph_batch_router_selected(workspace), + 0, + (uint64_t)(uint32_t)count * DS4_N_EXPERT_USED * + sizeof(int32_t)); + ok = selected_rows != NULL; + + /* Allocate and validate every row view before model mapping, KV writes, + * or command submission. Later binds cannot fail due to allocation. */ + for (int i = 0; ok && i < count; i++) { + ds4_gpu_graph *g = &items[i].session->graph; + ok = metal_graph_bind_exactn_union_row( + g, + workspace, + (uint32_t)i, + metal_graph_cur_hc(g), + metal_graph_after_ffn_hc(g), + &aliases[i]); + metal_graph_unbind_exactn_union_row(g, &aliases[i]); + } + + /* The model-view registry is process-global. Reinstall the complete + * static decode set before opening queue 0 so no layer remap can retire a + * view referenced by this batch. */ + if (ok) ok = metal_graph_stream_map_decode_static_all(model, weights); + if (ok) { + const bool cache_map_state = + metal_graph_stream_decode_static_map_state_cache_enabled(); + for (int i = 0; i < count; i++) { + items[i].session->graph.streaming_static_decode_map_current = + cache_map_state; + } + } + + ds4_gpu_set_stream(0); + if (ok) ok = ds4_gpu_begin_commands() != 0; + for (int i = 0; ok && i < count; i++) { + ds4_gpu_graph *g = &items[i].session->graph; + g->spec_capture_prefixes = false; + metal_graph_dspark_capture_begin(g); + ok = ds4_gpu_embed_token_hc_tensor( + metal_graph_cur_hc(g), + model->map, + model->size, + weights->token_embd->abs_offset, + (uint32_t)weights->token_embd->dim[1], + (uint32_t)items[i].token, + DS4_N_EMBD, + DS4_N_HC) != 0; + } + + for (uint32_t il = 0; ok && il < DS4_N_LAYER; il++) { + const ds4_layer_weights *layer = &weights->layer[il]; + ok = ds4_gpu_stream_expert_exact_rows_begin_collect() != 0; + collecting = ok; + + for (int i = 0; ok && i < count; i++) { + ds4_session *s = items[i].session; + ds4_gpu_graph *g = &s->graph; + const uint32_t pos = (uint32_t)s->checkpoint.len; + g->spec_exactn_union_collect_routes = true; + ok = metal_graph_bind_exactn_union_row( + g, + workspace, + (uint32_t)i, + metal_graph_cur_hc(g), + metal_graph_after_ffn_hc(g), + &aliases[i]); + if (ok) { + ok = metal_graph_encode_decode_layer_phase( + g, + model, + layer, + il, + pos, + g->layer_raw_cache[il], + g->raw_cap, + pos % g->raw_cap, + metal_graph_raw_span_for_batch(g, pos, 1), + items[i].token, + METAL_DECODE_LAYER_TO_ROUTER); + } + metal_graph_unbind_exactn_union_row(g, &aliases[i]); + g->spec_exactn_union_collect_routes = false; + } + + uint64_t gate_expert_bytes = 0; + uint64_t down_expert_bytes = 0; + if (ok) { + ok = streaming_layer_gate_down_expert_bytes( + layer, &gate_expert_bytes, &down_expert_bytes); + } + if (ok) { + const ds4_gpu_stream_expert_table table = + graph_stream_expert_table_make(model, + layer, + il, + gate_expert_bytes, + down_expert_bytes); + ok = ds4_gpu_stream_expert_exact_rows_prepare( + &table, + selected_rows, + (uint32_t)count, + DS4_N_EXPERT_USED) != 0; + } + + for (int i = 0; ok && i < count; i++) { + ds4_session *s = items[i].session; + ds4_gpu_graph *g = &s->graph; + const uint32_t pos = (uint32_t)s->checkpoint.len; + ok = metal_graph_bind_exactn_union_row( + g, + workspace, + (uint32_t)i, + metal_graph_cur_hc(g), + metal_graph_after_ffn_hc(g), + &aliases[i]); + if (ok) { + ok = ds4_gpu_stream_expert_exact_rows_set_row( + (uint32_t)i) != 0; + } + if (ok) { + ok = metal_graph_encode_decode_layer_phase( + g, + model, + layer, + il, + pos, + g->layer_raw_cache[il], + g->raw_cap, + pos % g->raw_cap, + metal_graph_raw_span_for_batch(g, pos, 1), + items[i].token, + METAL_DECODE_LAYER_FROM_ROUTER); + } + metal_graph_unbind_exactn_union_row(g, &aliases[i]); + } + + if (ok) { + ok = ds4_gpu_end_commands() != 0; + } else { + (void)ds4_gpu_synchronize(); + } + if (!ok) (void)ds4_gpu_synchronize(); + if (collecting) { + ds4_gpu_stream_expert_exact_rows_release(); + collecting = false; + } + if (!ok) break; + + for (int i = 0; ok && i < count; i++) { + ds4_gpu_graph *g = &items[i].session->graph; + ds4_gpu_tensor *tmp = metal_graph_cur_hc(g); + g->cur_hc_by_tier[g->active_tier] = + metal_graph_after_ffn_hc(g); + g->after_ffn_hc_by_tier[g->active_tier] = tmp; + ok = metal_graph_dspark_capture_decode_layer(g, il); + } + if (ok && il + 1u < DS4_N_LAYER) { + ok = ds4_gpu_begin_commands() != 0; + } + } + + if (ok) ok = ds4_gpu_begin_commands() != 0; + for (int i = 0; ok && i < count; i++) { + ok = metal_graph_encode_output_head( + &items[i].session->graph, + model, + weights, + weights->output->dim[1]); + } + if (ok) { + ok = ds4_gpu_end_commands() != 0; + } else { + (void)ds4_gpu_synchronize(); + } + + /* No view or exact-row resource may outlive work that references it. */ + if (!ok) (void)ds4_gpu_synchronize(); + if (collecting) ds4_gpu_stream_expert_exact_rows_release(); + for (int i = 0; i < count; i++) { + ds4_gpu_graph *g = &items[i].session->graph; + g->spec_exactn_union_collect_routes = false; + g->spec_capture_prefixes = saved_capture[i]; + metal_graph_release_exactn_union_row_alias(g, &aliases[i]); + } + ds4_gpu_tensor_free(selected_rows); + return ok; +} +#endif + static int ds4_sessions_eval_batch_metal( ds4_decode_item *items, int count, ds4_engine *e, char *err, size_t errlen) { + const bool ssd_session_union = + ds4_sessions_q4_ssd_session_union_supported(items, count, e); + /* The generic Metal batch tape assumes resident model ranges. If the + * opt-in changed between admission and execution, fail closed instead of + * accidentally running that tape against an SSD-streamed graph. */ + if (e && e->ssd_streaming && !ssd_session_union) { + if (err && errlen) { + snprintf(err, errlen, + "Metal SSD session union changed or became ineligible " + "before execution"); + } + return 1; + } const bool mirror = e->tp.active && e->tp.rank == 0; if (mirror) { ds4_tp_batch_item *wire = ds4_sessions_tp_batch_items(items, count); @@ -71662,10 +72092,14 @@ static int ds4_sessions_eval_batch_metal( bool native_shared = false; bool native_qkv = false; const bool stream_overlap = + !ssd_session_union && ds4_sessions_q4_stream_overlap_supported(items, count, e); bool ok = true; #if defined(__APPLE__) - if (stream_overlap) { + if (ssd_session_union) { + ok = metal_graph_encode_q4_ssd_session_union( + items, count, &e->model, &e->weights); + } else if (stream_overlap) { int started = 0; for (int i = 0; ok && i < count; i++) { ds4_session *s = items[i].session; @@ -71787,14 +72221,15 @@ static int ds4_sessions_eval_batch_metal( "ds4: Metal session batch rows=%d family=%s " "native_glm53=%d native_shared=%d native_qkv=%d " "stream_overlap=%d " - "streams=%d\n", + "streams=%d ssd_session_union=%d\n", count, ds4_session_is_glm(items[0].session) ? "glm" : "deepseek", native_glm53 ? 1 : 0, native_shared ? 1 : 0, native_qkv ? 1 : 0, stream_overlap ? 1 : 0, - stream_overlap ? count : 1); + stream_overlap ? count : 1, + ssd_session_union ? 1 : 0); } return 0; } @@ -72101,9 +72536,25 @@ int ds4_sessions_eval_batch(ds4_decode_item *items, int count, if (e->backend == DS4_BACKEND_CUDA) { return ds4_sessions_eval_batch_cuda(items, count, err, errlen); } - if (ds4_sessions_eval_batch_metal_supported(items, count, e)) { + const bool q4_ssd_session_union = + ds4_sessions_q4_ssd_session_union_supported(items, count, e); + if (q4_ssd_session_union || + ds4_sessions_eval_batch_metal_supported(items, count, e)) { return ds4_sessions_eval_batch_metal(items, count, e, err, errlen); } +#if defined(__APPLE__) + if (e->backend == DS4_BACKEND_METAL && e->ssd_streaming && + ds4_q4_ssd_session_union_required()) { + if (err && errlen) { + snprintf(err, errlen, + "required Metal Q4 SSD session union is ineligible " + "(need 2..5 valid sessions, AProjQ4 + routed " + "IQ2_XXS/Q2_K, static decode map, and >= rows*6 " + "cached experts)"); + } + return 1; + } +#endif #endif /* Preserve logical all-or-nothing behavior even on the serialized path. diff --git a/tests/test_metal_session_batch.c b/tests/test_metal_session_batch.c index 33eceff49..5bae38904 100644 --- a/tests/test_metal_session_batch.c +++ b/tests/test_metal_session_batch.c @@ -9,6 +9,7 @@ */ #include "ds4.h" +#include "ds4_gpu.h" #include "ds4_tp.h" #include @@ -39,12 +40,23 @@ static const char *prompts[MAX_SESSION_COUNT] = { static float observed_max_abs; static bool compare_argmax_only; +static bool env_flag(const char *name) { + const char *value = getenv(name); + return value && value[0] && strcmp(value, "0") != 0; +} + static double now_seconds(void) { struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts); return (double)ts.tv_sec + (double)ts.tv_nsec / 1000000000.0; } +static double monotonic_ms(void) { + struct timespec ts; + if (clock_gettime(CLOCK_MONOTONIC, &ts) != 0) return 0.0; + return (double)ts.tv_sec * 1000.0 + (double)ts.tv_nsec / 1000000.0; +} + static void fail(const char *what, int session, int step) { fprintf(stderr, "FAIL: %s session=%d step=%d\n", what, session, step); exit(1); @@ -172,6 +184,22 @@ static float steering_scale_from_env(const char *name, float fallback) { return scale; } +static uint32_t ssd_cache_experts_from_env(void) { + const char *value = getenv("DS4_TEST_SSD_CACHE_EXPERTS"); + if (!value || !value[0]) return 30u; + char *end = NULL; + unsigned long count = strtoul(value, &end, 10); + if (end == value || *end != '\0' || count < 30ul || + count > (unsigned long)UINT32_MAX) { + fprintf(stderr, + "FAIL: invalid DS4_TEST_SSD_CACHE_EXPERTS=%s " + "(expected 30..%u)\n", + value, UINT32_MAX); + exit(1); + } + return (uint32_t)count; +} + static void archive_logits(ds4_session *session, float *dst, int vocab, int session_id, int step) { if (ds4_session_copy_logits(session, dst, vocab) != vocab) { @@ -226,6 +254,38 @@ int main(void) { char *prompt_file_text = read_prompt_file(getenv("DS4_TEST_PROMPT_FILE")); const char *argmax_only = getenv("DS4_TEST_ARGMAX_ONLY"); compare_argmax_only = argmax_only && strcmp(argmax_only, "0") != 0; + const bool ssd_streaming = env_flag("DS4_TEST_SSD_STREAMING"); + const uint32_t ssd_cache_experts = + ssd_streaming ? ssd_cache_experts_from_env() : 0u; + const bool batch_timing = env_flag("DS4_TEST_SESSION_BATCH_TIMING"); + const bool ssd_union_policy_switch = + env_flag("DS4_TEST_SSD_UNION_POLICY_SWITCH"); + const char *batch_arm = getenv("DS4_TEST_SESSION_BATCH_ARM"); + if (!batch_arm || !batch_arm[0]) batch_arm = "unspecified"; + if (ssd_streaming && session_count < 5) { + fprintf(stderr, + "FAIL: DS4_TEST_SSD_STREAMING needs " + "DS4_TEST_SESSION_COUNT=5 to cover N=2..5\n"); + return 1; + } + if (ssd_streaming && decode_steps < 4) { + fprintf(stderr, + "FAIL: DS4_TEST_SSD_STREAMING needs " + "DS4_TEST_DECODE_STEPS>=4 to cover N=2..5\n"); + return 1; + } + if (ssd_streaming && live_controls) { + fprintf(stderr, + "FAIL: DS4_TEST_LIVE_CONTROLS would contaminate " + "SSD-union cache coverage counters\n"); + return 1; + } + if (ssd_union_policy_switch && !ssd_streaming) { + fprintf(stderr, + "FAIL: DS4_TEST_SSD_UNION_POLICY_SWITCH requires " + "DS4_TEST_SSD_STREAMING=1\n"); + return 1; + } const char *tp_mode = getenv("DS4_TEST_TP_MODE"); const bool tp_leader = tp_mode && strcmp(tp_mode, "leader") == 0; @@ -234,12 +294,23 @@ int main(void) { fprintf(stderr, "FAIL: invalid DS4_TEST_TP_MODE=%s\n", tp_mode); return 1; } + if (ssd_streaming && (tp_leader || tp_worker)) { + fprintf(stderr, + "FAIL: DS4_TEST_SSD_STREAMING does not support TP mode\n"); + return 1; + } const int tp_port = tp_port_from_env(); ds4_engine_options opt = { .model_path = model, .backend = DS4_BACKEND_METAL, .n_threads = 1, .context_size = context_size, + .prefill_chunk = ssd_streaming ? 128u : 0u, + .ssd_streaming_cache_experts = ssd_cache_experts, + .ssd_streaming = ssd_streaming, + .ssd_streaming_cold = ssd_streaming, + .placement_session_count_hint = ssd_streaming ? session_count : 0, + .share_session_prefill_workspace = ssd_streaming, }; const char *steering_file = getenv("DS4_TEST_DIRECTIONAL_STEERING_FILE"); @@ -250,6 +321,15 @@ int main(void) { opt.directional_steering_ffn = steering_scale_from_env( "DS4_TEST_DIRECTIONAL_STEERING_FFN", 1.0f); } + fprintf(stderr, + "test_metal_session_batch setup mode=%s arm=%s sessions=%d " + "cache_experts=%u cold=%d shared_workspace=%d\n", + ssd_streaming ? "ssd" : "resident", + batch_arm, + session_count, + opt.ssd_streaming_cache_experts, + opt.ssd_streaming_cold ? 1 : 0, + opt.share_session_prefill_workspace ? 1 : 0); if (tp_leader) { opt.tp.role = DS4_TP_LEADER; opt.tp.listen_host = getenv("DS4_TEST_TP_LISTEN_HOST"); @@ -368,10 +448,16 @@ int main(void) { float *actual = malloc((size_t)vocab * sizeof(float)); int *argmax = malloc(frontier_count * sizeof(int)); int generated[MAX_SESSION_COUNT][MAX_DECODE_STEPS]; + int final_pos[MAX_SESSION_COUNT] = {0}; ds4_session *live_control[MAX_SESSION_COUNT] = {0}; double control_seconds = 0.0; double live_control_seconds = 0.0; if (!expected || !actual || !argmax) fail("oracle allocation", -1, -1); + for (int i = 0; i < MAX_SESSION_COUNT; i++) { + for (int step = 0; step < MAX_DECODE_STEPS; step++) { + generated[i][step] = -1; + } + } #define FRONTIER(step_, session_) \ ((size_t)(step_) * (size_t)session_count + (size_t)(session_)) @@ -400,37 +486,62 @@ int main(void) { } } - const double batch_t0 = now_seconds(); + const double batch_wall_t0 = now_seconds(); + double batch_ms = 0.0; + uint64_t batch_rows_total = 0; + uint32_t ssd_rows_coverage = 0; + ds4_gpu_exact_rows_persistent_report exact_before = {0}; + ds4_gpu_exact_rows_persistent_report exact_after = {0}; + if (ssd_streaming) { + ds4_gpu_test_exact_rows_persistent_report(&exact_before); + } for (int step = 0; step < decode_steps; step++) { + if (ssd_union_policy_switch && step == decode_steps / 2) { + if (unsetenv("DS4_METAL_REQUIRE_Q4_SSD_SESSION_UNION") != 0 || + setenv("DS4_METAL_ENABLE_Q4_SSD_SESSION_UNION", "1", 1) != 0) { + fail("SSD union policy switch", -1, step); + } + fprintf(stderr, + "test_metal_session_batch policy-switch step=%d " + "REQUIRE=unset ENABLE=1\n", + step); + } + const int batch_rows = ssd_streaming ? 2 + step % 4 : session_count; ds4_decode_item items[MAX_SESSION_COUNT]; - for (int row = 0; row < session_count; row++) { - int i = (step & 1) ? session_count - 1 - row : row; + for (int row = 0; row < batch_rows; row++) { + int i = (step & 1) ? batch_rows - 1 - row : row; int token = ds4_session_argmax(batched[i]); generated[i][step] = token; items[row].session = batched[i]; items[row].token = token; } - if (ds4_sessions_eval_batch(items, session_count, + const double batch_t0 = batch_timing ? monotonic_ms() : 0.0; + if (ds4_sessions_eval_batch(items, batch_rows, err, sizeof(err)) != 0) { fprintf(stderr, "FAIL: batch step=%d: %s\n", step, err); return 1; } + if (batch_timing) batch_ms += monotonic_ms() - batch_t0; + batch_rows_total += (uint64_t)batch_rows; + if (ssd_streaming) ssd_rows_coverage |= 1u << (uint32_t)batch_rows; for (int i = 0; i < session_count; i++) { size_t f = FRONTIER(step + 1, i); archive_logits(batched[i], expected + f * (size_t)vocab, vocab, i, step + 1); argmax[f] = ds4_session_argmax(batched[i]); if (live_controls) { - const double eval_t0 = now_seconds(); - const int eval_rc = ds4_session_eval( - live_control[i], generated[i][step], - err, sizeof(err)); - live_control_seconds += now_seconds() - eval_t0; - if (eval_rc != 0) { - fprintf(stderr, - "FAIL: live control eval session=%d step=%d: %s\n", - i, step, err); - return 1; + if (generated[i][step] >= 0) { + const double eval_t0 = now_seconds(); + const int eval_rc = ds4_session_eval( + live_control[i], generated[i][step], + err, sizeof(err)); + live_control_seconds += now_seconds() - eval_t0; + if (eval_rc != 0) { + fprintf(stderr, + "FAIL: live control eval session=%d step=%d: %s\n", + i, step, err); + return 1; + } } compare_logits(live_control[i], expected + f * (size_t)vocab, @@ -440,13 +551,61 @@ int main(void) { } } const double batch_seconds = - now_seconds() - batch_t0 - live_control_seconds; + now_seconds() - batch_wall_t0 - live_control_seconds; + if (ssd_streaming && + (ssd_rows_coverage & ((1u << 2) | (1u << 3) | + (1u << 4) | (1u << 5))) != + ((1u << 2) | (1u << 3) | (1u << 4) | (1u << 5))) { + fail("SSD batch coverage N=2..5", -1, -1); + } + if (ssd_streaming) { + ds4_gpu_test_exact_rows_persistent_report(&exact_after); + const uint64_t persistent = + exact_after.persistent_calls - exact_before.persistent_calls; + const uint64_t transient = + exact_after.transient_calls - exact_before.transient_calls; + const uint64_t fallbacks = + exact_after.persistent_fallbacks - + exact_before.persistent_fallbacks; + const uint64_t failures = + exact_after.persistent_failures - exact_before.persistent_failures; + const uint64_t mapped_views = + exact_after.mapped_view_calls - exact_before.mapped_view_calls; + const int layer_count = ds4_engine_layer_count(engine); + const uint64_t expected_persistent = layer_count > 0 + ? (uint64_t)decode_steps * (uint64_t)(uint32_t)layer_count + : 0u; + fprintf(stderr, + "test_metal_session_batch exact-cache persistent=%llu " + "expected_persistent=%llu " + "transient=%llu fallbacks=%llu failures=%llu " + "mapped_views=%llu max_unique=%u\n", + (unsigned long long)persistent, + (unsigned long long)expected_persistent, + (unsigned long long)transient, + (unsigned long long)fallbacks, + (unsigned long long)failures, + (unsigned long long)mapped_views, + exact_after.max_unique); + if (env_flag("DS4_METAL_REQUIRE_EXACT_ROWS_PERSISTENT_CACHE") && + (persistent != expected_persistent || transient != 0u || + fallbacks != 0u || failures != 0u || mapped_views != 0u)) { + fail("SSD exact persistent cache coverage", -1, -1); + } + if (env_flag("DS4_METAL_DISABLE_Q4_SSD_SESSION_UNION") && + env_flag("DS4_METAL_REQUIRE_Q4_SSD_SESSION_UNION") && + (persistent != 0u || transient != 0u || fallbacks != 0u || + failures != 0u || mapped_views != 0u)) { + fail("SSD control did not serialize", -1, -1); + } + } for (int i = 0; i < session_count; i++) { + final_pos[i] = ds4_session_pos(batched[i]); ds4_session_free(batched[i]); if (live_control[i]) ds4_session_free(live_control[i]); } - if (!skip_mixed) { + if (!skip_mixed && !ssd_streaming) { ds4_tokens mixed_prompt = {0}; ds4_tokens suffix = {0}; ds4_tokens_copy(&mixed_prompt, &prompt[0]); @@ -474,7 +633,7 @@ int main(void) { return 1; } ds4_decode_item mixed_items[MAX_SESSION_COUNT]; - for (int i = 0; !live_controls && i < session_count; i++) { + for (int i = 0; i < session_count; i++) { if (ds4_session_create(&mixed_decode[i], engine, context_size) != 0) { fail("mixed decode create", i, -1); } @@ -561,7 +720,7 @@ int main(void) { compare_logits(control, expected + f * (size_t)vocab, actual, vocab, argmax[f], logit_tolerance, i, step); - if (step < decode_steps) { + if (step < decode_steps && generated[i][step] >= 0) { const double eval_t0 = now_seconds(); const int eval_rc = ds4_session_eval( control, generated[i][step], err, sizeof(err)); @@ -574,6 +733,9 @@ int main(void) { } } } + if (ds4_session_pos(control) != final_pos[i]) { + fail("final checkpoint", i, decode_steps); + } ds4_session_free(control); ds4_tokens_free(&prompt[i]); } @@ -585,17 +747,35 @@ int main(void) { if (tp) (void)ds4_tp_send_stop(tp); ds4_engine_close(engine); ds4_tp_free(tp); + if (batch_timing) { + const double rows_per_sec = batch_ms > 0.0 + ? (double)batch_rows_total * 1000.0 / batch_ms : 0.0; + fprintf(stderr, + "test_metal_session_batch timing arm=%s batch_ms=%.3f " + "rows=%llu steps=%d rows_per_sec=%.3f\n", + batch_arm, + batch_ms, + (unsigned long long)batch_rows_total, + decode_steps, + rows_per_sec); + } fprintf(stderr, "test_metal_session_batch PASS sessions=%d steps=%d mixed_suffix=%d " "comparison=%s logit_tolerance=%g max_abs=%g batch=%.2f rows/s " - "serial=%.2f rows/s speedup=%.2fx\n", + "serial=%.2f rows/s speedup=%.2fx mode=%s arm=%s " + "ssd_rows=%s batch_rows=%llu batch_ms=%s\n", session_count, decode_steps, - skip_mixed ? 0 : MIXED_SUFFIX_TOKENS, + (skip_mixed || ssd_streaming) ? 0 : MIXED_SUFFIX_TOKENS, compare_argmax_only ? "argmax" : "logits", logit_tolerance, observed_max_abs, - (double)(session_count * decode_steps) / batch_seconds, - (double)(session_count * decode_steps) / control_seconds, - control_seconds / batch_seconds); + batch_seconds > 0.0 ? (double)batch_rows_total / batch_seconds : 0.0, + control_seconds > 0.0 ? (double)batch_rows_total / control_seconds : 0.0, + batch_seconds > 0.0 ? control_seconds / batch_seconds : 0.0, + ssd_streaming ? "ssd" : "resident", + batch_arm, + ssd_streaming ? "2,3,4,5" : "fixed", + (unsigned long long)batch_rows_total, + batch_timing ? "reported-above" : "disabled"); return 0; #undef FRONTIER } From 8498c81212ba7537637c717495bf4aab8612ec44 Mon Sep 17 00:00:00 2001 From: Giorgio Oppo Date: Fri, 14 Aug 2026 17:23:34 +0200 Subject: [PATCH 030/189] cuda: batch selected expert SSD uploads --- ds4_cuda.cu | 1231 ++++++++++++++++++++++++++++++++-- ds4_gpu.h | 31 + tests/test_gpu_model_cache.c | 40 ++ 3 files changed, 1235 insertions(+), 67 deletions(-) diff --git a/ds4_cuda.cu b/ds4_cuda.cu index d5a84bec4..7f300a410 100644 --- a/ds4_cuda.cu +++ b/ds4_cuda.cu @@ -15,6 +15,8 @@ #include #include #include +#include +#include #include #include #include @@ -515,11 +517,31 @@ static void *g_model_stage_raw[4]; static void *g_model_stage[4]; static cudaEvent_t g_model_stage_event[4]; static uint64_t g_model_stage_bytes; +static uint64_t g_model_stage_align = 1; static void *g_stream_selected_stage_raw[4]; static void *g_stream_selected_stage[4]; static cudaEvent_t g_stream_selected_stage_event[4]; static uint64_t g_stream_selected_stage_bytes; +static uint64_t g_stream_selected_stage_align = 1; static cudaStream_t g_stream_selected_upload_stream; +static std::once_flag g_stream_selected_batch_io_once; +static int g_stream_selected_batch_io_enabled; +static int g_stream_selected_batch_io_required; +static int g_stream_selected_batch_io_oracle; +static int g_stream_selected_batch_io_report_registered; +static uint64_t g_stream_selected_batch_io_candidates; +static uint64_t g_stream_selected_batch_io_attempts; +static uint64_t g_stream_selected_batch_io_completed; +static uint64_t g_stream_selected_batch_io_legacy; +static uint64_t g_stream_selected_batch_io_safe_fallbacks; +static uint64_t g_stream_selected_batch_io_failures; +static uint64_t g_stream_selected_batch_io_required_failures; +static uint64_t g_stream_selected_batch_io_oracle_runs; +static uint64_t g_stream_selected_batch_io_oracle_failures; +static uint64_t g_stream_selected_batch_io_tasks; +static uint64_t g_stream_selected_batch_io_segments; +static uint64_t g_stream_selected_batch_io_reads; +static uint64_t g_stream_selected_batch_io_bytes; static int cuda_ok(cudaError_t err, const char *what); extern "C" void ds4_gpu_decode_graphs_invalidate(void); @@ -2469,15 +2491,60 @@ static uint64_t cuda_round_up(uint64_t v, uint64_t align) { return rem == 0 ? v : v + (align - rem); } -static void *cuda_align_ptr(void *ptr, uint64_t align) { - if (align <= 1) return ptr; - uintptr_t p = (uintptr_t)ptr; - uintptr_t a = (uintptr_t)align; - return (void *)(((p + a - 1u) / a) * a); +/* The aligned pointer can advance by align-1 bytes from the allocation base. + * Allocate that prefix explicitly and validate every representation boundary + * before exposing the requested usable span to O_DIRECT or CUDA DMA. */ +static int cuda_host_stage_allocation_bytes(uint64_t usable_bytes, + uint64_t align, + size_t *allocation_bytes) { + if (!allocation_bytes || usable_bytes == 0) return 0; + const uint64_t a = align > 1 ? align : 1; + const uint64_t extra = a - 1u; + if (a > (uint64_t)UINTPTR_MAX || + usable_bytes > (uint64_t)SIZE_MAX || + extra > (uint64_t)SIZE_MAX - usable_bytes) { + return 0; + } + *allocation_bytes = (size_t)(usable_bytes + extra); + return 1; } -static int cuda_model_stage_pool_alloc(uint64_t bytes) { - if (g_model_stage_bytes >= bytes) return 1; +static int cuda_host_stage_bytes_for_chunk(uint64_t chunk, uint64_t align, + uint64_t *usable_bytes) { + if (!usable_bytes || chunk == 0) return 0; + const uint64_t a = align > 1 ? align : 1; + if (chunk > UINT64_MAX - a || chunk + a > (uint64_t)SIZE_MAX) { + return 0; + } + *usable_bytes = chunk + a; + return 1; +} + +static int cuda_host_stage_aligned_view(void *raw, + size_t allocation_bytes, + uint64_t usable_bytes, + uint64_t align, + void **aligned_out) { + if (!raw || !aligned_out || usable_bytes == 0 || + usable_bytes > (uint64_t)allocation_bytes) { + return 0; + } + const uint64_t align64 = align > 1 ? align : 1; + if (align64 > (uint64_t)UINTPTR_MAX) return 0; + const uintptr_t a = (uintptr_t)align64; + if (a == 0) return 0; + const uintptr_t p = (uintptr_t)raw; + const uintptr_t rem = p % a; + const size_t delta = rem == 0 ? 0u : (size_t)(a - rem); + if (delta > allocation_bytes || + usable_bytes > (uint64_t)(allocation_bytes - delta)) { + return 0; + } + *aligned_out = (char *)raw + delta; + return 1; +} + +static void cuda_model_stage_pool_release(void) { for (size_t i = 0; i < 4; i++) { if (g_model_stage_event[i]) { (void)cudaEventDestroy(g_model_stage_event[i]); @@ -2490,6 +2557,20 @@ static int cuda_model_stage_pool_alloc(uint64_t bytes) { } } g_model_stage_bytes = 0; + g_model_stage_align = 1; +} + +static int cuda_model_stage_pool_alloc(uint64_t bytes) { + const uint64_t align = g_model_direct_align > 1 ? + g_model_direct_align : 1; + size_t allocation_bytes = 0; + if (!cuda_host_stage_allocation_bytes(bytes, align, + &allocation_bytes)) { + return 0; + } + if (g_model_stage_bytes >= bytes && + g_model_stage_align == align) return 1; + cuda_model_stage_pool_release(); if (!g_model_upload_stream) { cudaError_t err = cudaStreamCreateWithFlags(&g_model_upload_stream, cudaStreamNonBlocking); if (err != cudaSuccess) { @@ -2499,24 +2580,52 @@ static int cuda_model_stage_pool_alloc(uint64_t bytes) { } } for (size_t i = 0; i < 4; i++) { - cudaError_t err = cudaMallocHost(&g_model_stage_raw[i], (size_t)bytes); + cudaError_t err = cudaMallocHost(&g_model_stage_raw[i], + allocation_bytes); if (err != cudaSuccess) { fprintf(stderr, "ds4: CUDA pinned model staging allocation failed: %s\n", cudaGetErrorString(err)); (void)cudaGetLastError(); + cuda_model_stage_pool_release(); + return 0; + } + if (!cuda_host_stage_aligned_view(g_model_stage_raw[i], + allocation_bytes, bytes, align, + &g_model_stage[i])) { + fprintf(stderr, + "ds4: CUDA pinned model staging alignment rejected\n"); + cuda_model_stage_pool_release(); return 0; } - g_model_stage[i] = cuda_align_ptr(g_model_stage_raw[i], g_model_direct_align); err = cudaEventCreateWithFlags(&g_model_stage_event[i], cudaEventDisableTiming); if (err != cudaSuccess) { fprintf(stderr, "ds4: CUDA model staging event creation failed: %s\n", cudaGetErrorString(err)); (void)cudaGetLastError(); + cuda_model_stage_pool_release(); return 0; } } g_model_stage_bytes = bytes; + g_model_stage_align = align; return 1; } +/* A range load can fail after prior chunks were queued from the pinned ring. + * Drain before any caller can reuse or free that storage. */ +static int cuda_model_upload_fail(const char *what) { + if (g_model_upload_stream) { + const cudaError_t sync_err = + cudaStreamSynchronize(g_model_upload_stream); + if (sync_err != cudaSuccess) { + fprintf(stderr, + "ds4: CUDA model upload abort sync failed for %s: %s\n", + what ? what : "weights", + cudaGetErrorString(sync_err)); + (void)cudaGetLastError(); + } + } + return 0; +} + static int cuda_pread_full(int fd, void *buf, uint64_t bytes, uint64_t offset) { uint64_t done = 0; while (done < bytes) { @@ -2582,6 +2691,7 @@ static void cuda_stream_selected_stage_release(void) { } } g_stream_selected_stage_bytes = 0; + g_stream_selected_stage_align = 1; if (g_stream_selected_upload_stream) { (void)cudaStreamDestroy(g_stream_selected_upload_stream); g_stream_selected_upload_stream = NULL; @@ -2589,7 +2699,15 @@ static void cuda_stream_selected_stage_release(void) { } static int cuda_stream_selected_stage_pool_alloc(uint64_t bytes) { - if (g_stream_selected_stage_bytes >= bytes) return 1; + const uint64_t align = g_model_direct_align > 1 ? + g_model_direct_align : 1; + size_t allocation_bytes = 0; + if (!cuda_host_stage_allocation_bytes(bytes, align, + &allocation_bytes)) { + return 0; + } + if (g_stream_selected_stage_bytes >= bytes && + g_stream_selected_stage_align == align) return 1; cuda_stream_selected_stage_release(); cudaError_t err = cudaStreamCreateWithFlags( &g_stream_selected_upload_stream, cudaStreamNonBlocking); @@ -2601,7 +2719,8 @@ static int cuda_stream_selected_stage_pool_alloc(uint64_t bytes) { return 0; } for (size_t i = 0; i < 4; i++) { - err = cudaMallocHost(&g_stream_selected_stage_raw[i], (size_t)bytes); + err = cudaMallocHost(&g_stream_selected_stage_raw[i], + allocation_bytes); if (err != cudaSuccess) { fprintf(stderr, "ds4: CUDA streaming selected staging allocation failed: %s\n", @@ -2610,8 +2729,14 @@ static int cuda_stream_selected_stage_pool_alloc(uint64_t bytes) { cuda_stream_selected_stage_release(); return 0; } - g_stream_selected_stage[i] = cuda_align_ptr( - g_stream_selected_stage_raw[i], g_model_direct_align); + if (!cuda_host_stage_aligned_view( + g_stream_selected_stage_raw[i], allocation_bytes, + bytes, align, &g_stream_selected_stage[i])) { + fprintf(stderr, + "ds4: CUDA streaming selected staging alignment rejected\n"); + cuda_stream_selected_stage_release(); + return 0; + } err = cudaEventCreateWithFlags(&g_stream_selected_stage_event[i], cudaEventDisableTiming); if (err != cudaSuccess) { @@ -2624,9 +2749,898 @@ static int cuda_stream_selected_stage_pool_alloc(uint64_t bytes) { } } g_stream_selected_stage_bytes = bytes; + g_stream_selected_stage_align = align; + return 1; +} + +typedef struct { + char *dst; + uint64_t offset; + uint64_t bytes; + uint32_t ordinal; +} cuda_stream_selected_copy_task; + +typedef struct { + uint32_t task_index; + uint64_t task_offset; + uint64_t offset; + uint64_t bytes; +} cuda_stream_selected_copy_segment; + +typedef struct { + uint32_t segment_begin; + uint32_t segment_end; + uint64_t offset; + uint64_t bytes; +} cuda_stream_selected_copy_group; + +struct ds4_cuda_stream_selected_batch_io_report; +typedef struct ds4_cuda_stream_selected_batch_io_report + ds4_cuda_stream_selected_batch_io_report; +typedef struct { + uint64_t candidates; + uint64_t attempts; + uint64_t completed; + uint64_t legacy_batches; + uint64_t safe_fallbacks; + uint64_t failures; + uint64_t required_failures; + uint64_t oracle_runs; + uint64_t oracle_failures; + uint64_t tasks; + uint64_t segments; + uint64_t reads; + uint64_t bytes; + int enabled; + int required; + int oracle; +} cuda_stream_selected_batch_io_report_layout; + +static int cuda_stream_selected_env_value_enabled(const char *value) { + if (!value || !value[0]) return 0; + if (strcmp(value, "0") == 0 || strcasecmp(value, "false") == 0 || + strcasecmp(value, "no") == 0 || strcasecmp(value, "off") == 0) { + return 0; + } + return 1; +} + +static int cuda_stream_selected_env_flag(const char *name) { + return cuda_stream_selected_env_value_enabled( + name ? getenv(name) : NULL); +} + +extern "C" int ds4_cuda_test_stream_selected_batch_env_value( + const char *value) { + return cuda_stream_selected_env_value_enabled(value); +} + +static void cuda_stream_selected_batch_io_resolve_policy( + int enable, int disable, int require, int oracle, + int *enabled_out, int *required_out, int *oracle_out) { + const int disabled = disable != 0; + if (enabled_out) { + *enabled_out = !disabled && (enable || require || oracle); + } + if (required_out) *required_out = !disabled && require; + if (oracle_out) *oracle_out = !disabled && oracle; +} + +extern "C" int ds4_cuda_test_stream_selected_batch_policy( + int enable, int disable, int require, int oracle, + int *enabled_out, int *required_out, int *oracle_out) { + if (!enabled_out || !required_out || !oracle_out) return 0; + cuda_stream_selected_batch_io_resolve_policy( + enable, disable, require, oracle, + enabled_out, required_out, oracle_out); + return 1; +} + +static void cuda_stream_selected_batch_io_report_at_exit(void) { + fprintf(stderr, + "ds4: CUDA selected-expert batched I/O: candidates=%llu " + "attempts=%llu completed=%llu legacy=%llu safe_fallbacks=%llu " + "failures=%llu required_failures=%llu oracle_runs=%llu " + "oracle_failures=%llu tasks=%llu segments=%llu reads=%llu " + "bytes=%.2f MiB\n", + (unsigned long long)g_stream_selected_batch_io_candidates, + (unsigned long long)g_stream_selected_batch_io_attempts, + (unsigned long long)g_stream_selected_batch_io_completed, + (unsigned long long)g_stream_selected_batch_io_legacy, + (unsigned long long)g_stream_selected_batch_io_safe_fallbacks, + (unsigned long long)g_stream_selected_batch_io_failures, + (unsigned long long)g_stream_selected_batch_io_required_failures, + (unsigned long long)g_stream_selected_batch_io_oracle_runs, + (unsigned long long)g_stream_selected_batch_io_oracle_failures, + (unsigned long long)g_stream_selected_batch_io_tasks, + (unsigned long long)g_stream_selected_batch_io_segments, + (unsigned long long)g_stream_selected_batch_io_reads, + (double)g_stream_selected_batch_io_bytes / 1048576.0); +} + +static void cuda_stream_selected_batch_io_init(void) { + const int enable = cuda_stream_selected_env_flag( + "DS4_CUDA_ENABLE_STREAMING_SELECTED_BATCH_IO"); + const int disable = cuda_stream_selected_env_flag( + "DS4_CUDA_DISABLE_STREAMING_SELECTED_BATCH_IO") || + cuda_stream_selected_env_flag( + "DS4_CUDA_NO_STREAMING_SELECTED_BATCH_IO"); + const int require = cuda_stream_selected_env_flag( + "DS4_CUDA_REQUIRE_STREAMING_SELECTED_BATCH_IO"); + const int oracle = cuda_stream_selected_env_flag( + "DS4_CUDA_STREAMING_SELECTED_BATCH_IO_ORACLE"); + cuda_stream_selected_batch_io_resolve_policy( + enable, disable, require, oracle, + &g_stream_selected_batch_io_enabled, + &g_stream_selected_batch_io_required, + &g_stream_selected_batch_io_oracle); + if ((enable || require || oracle) && + !g_stream_selected_batch_io_report_registered) { + g_stream_selected_batch_io_report_registered = 1; + (void)atexit(cuda_stream_selected_batch_io_report_at_exit); + } + if (g_stream_selected_batch_io_enabled) { + fprintf(stderr, + "ds4: CUDA selected-expert batched I/O enabled%s%s\n", + g_stream_selected_batch_io_required ? " (required)" : "", + g_stream_selected_batch_io_oracle ? + " with full byte oracle" : ""); + } else if (disable && (enable || require || oracle)) { + fprintf(stderr, + "ds4: CUDA selected-expert batched I/O disabled by rollback" + " override\n"); + } +} + +static int cuda_stream_selected_batch_io_requested(void) { + std::call_once(g_stream_selected_batch_io_once, + cuda_stream_selected_batch_io_init); + return g_stream_selected_batch_io_enabled; +} + +static int cuda_stream_selected_batch_io_require_requested(void) { + std::call_once(g_stream_selected_batch_io_once, + cuda_stream_selected_batch_io_init); + return g_stream_selected_batch_io_required; +} + +static int cuda_stream_selected_batch_io_oracle_requested(void) { + std::call_once(g_stream_selected_batch_io_once, + cuda_stream_selected_batch_io_init); + return g_stream_selected_batch_io_oracle; +} + +extern "C" void ds4_cuda_stream_selected_batch_io_get_report( + ds4_cuda_stream_selected_batch_io_report *report) { + if (!report) return; + cuda_stream_selected_batch_io_report_layout out = {}; + out.candidates = g_stream_selected_batch_io_candidates; + out.attempts = g_stream_selected_batch_io_attempts; + out.completed = g_stream_selected_batch_io_completed; + out.legacy_batches = g_stream_selected_batch_io_legacy; + out.safe_fallbacks = g_stream_selected_batch_io_safe_fallbacks; + out.failures = g_stream_selected_batch_io_failures; + out.required_failures = + g_stream_selected_batch_io_required_failures; + out.oracle_runs = g_stream_selected_batch_io_oracle_runs; + out.oracle_failures = g_stream_selected_batch_io_oracle_failures; + out.tasks = g_stream_selected_batch_io_tasks; + out.segments = g_stream_selected_batch_io_segments; + out.reads = g_stream_selected_batch_io_reads; + out.bytes = g_stream_selected_batch_io_bytes; + out.enabled = cuda_stream_selected_batch_io_requested(); + out.required = cuda_stream_selected_batch_io_require_requested(); + out.oracle = cuda_stream_selected_batch_io_oracle_requested(); + memcpy(report, &out, sizeof(out)); +} + +/* Build a deterministic source-ordered plan without changing compact-slot + * order. Adjacent source spans can share one bounded read even when their + * device destinations are discontiguous. Overlaps are rejected because the + * selected-expert list has already been deduplicated. */ +static int cuda_stream_selected_copy_plan( + const std::vector &tasks, + uint64_t model_size, + uint64_t chunk, + std::vector *sorted_out, + std::vector *segments_out, + std::vector *groups_out) { + if (!sorted_out || !segments_out || !groups_out || + tasks.empty() || chunk == 0) { + return 0; + } + + try { + *sorted_out = tasks; + std::stable_sort( + sorted_out->begin(), sorted_out->end(), + [](const cuda_stream_selected_copy_task &a, + const cuda_stream_selected_copy_task &b) { + return a.offset != b.offset ? a.offset < b.offset : + a.ordinal < b.ordinal; + }); + segments_out->clear(); + groups_out->clear(); + segments_out->reserve(tasks.size()); + groups_out->reserve(tasks.size()); + } catch (...) { + return 0; + } + if (sorted_out->size() >= UINT32_MAX) return 0; + + uint64_t previous_end = 0; + for (uint32_t i = 0; i < sorted_out->size(); i++) { + const cuda_stream_selected_copy_task &task = (*sorted_out)[i]; + if (!task.dst || task.bytes == 0 || task.offset > model_size || + task.bytes > model_size - task.offset || + task.bytes > (uint64_t)SIZE_MAX || + (i != 0 && task.offset < previous_end)) { + return 0; + } + previous_end = task.offset + task.bytes; + + uint64_t done = 0; + while (done < task.bytes) { + const uint64_t n = task.bytes - done < chunk ? + task.bytes - done : chunk; + try { + segments_out->push_back( + { i, done, task.offset + done, n }); + } catch (...) { + return 0; + } + done += n; + } + } + if (segments_out->empty() || + segments_out->size() >= UINT32_MAX) return 0; + + uint32_t begin = 0; + while (begin < segments_out->size()) { + const cuda_stream_selected_copy_segment &first = + (*segments_out)[begin]; + uint64_t group_bytes = first.bytes; + uint32_t end = begin + 1u; + while (end < segments_out->size()) { + const cuda_stream_selected_copy_segment &next = + (*segments_out)[end]; + if (next.offset != first.offset + group_bytes || + next.bytes > chunk - group_bytes) { + break; + } + group_bytes += next.bytes; + end++; + } + try { + groups_out->push_back( + { begin, end, first.offset, group_bytes }); + } catch (...) { + return 0; + } + begin = end; + } + return !groups_out->empty(); +} + +static int cuda_stream_selected_upload_fail(const char *what) { + if (g_stream_selected_upload_stream) { + const cudaError_t sync_err = + cudaStreamSynchronize(g_stream_selected_upload_stream); + if (sync_err != cudaSuccess) { + fprintf(stderr, + "ds4: CUDA selected batched I/O abort sync failed for %s: %s\n", + what ? what : "expert batch", + cudaGetErrorString(sync_err)); + (void)cudaGetLastError(); + } + } + return 0; +} + +static int cuda_stream_selected_copy_oracle( + const std::vector &tasks, + const void *model_map, + uint64_t model_size, + const int32_t *remap_src, + const int32_t *remap_dst, + uint64_t remap_count, + int report) { + if (!model_map || tasks.empty() || + (remap_count != 0 && (!remap_src || !remap_dst)) || + remap_count > SIZE_MAX / sizeof(int32_t)) { + return 0; + } + + uint64_t largest_task = 0; + for (const cuda_stream_selected_copy_task &task : tasks) { + if (!task.dst || task.offset > model_size || task.bytes == 0 || + task.bytes > model_size - task.offset) { + return 0; + } + if (task.bytes > largest_task) largest_task = task.bytes; + } + const uint64_t scratch_bytes = largest_task < 4u * 1024u * 1024u ? + largest_task : 4u * 1024u * 1024u; + if (scratch_bytes == 0 || scratch_bytes > SIZE_MAX) return 0; + + std::vector got; + try { + got.resize((size_t)scratch_bytes); + } catch (...) { + if (report) { + fprintf(stderr, + "ds4: CUDA selected batched I/O byte oracle allocation failed\n"); + } + return 0; + } + + for (const cuda_stream_selected_copy_task &task : tasks) { + uint64_t done = 0; + while (done < task.bytes) { + const uint64_t n = task.bytes - done < scratch_bytes ? + task.bytes - done : scratch_bytes; + const cudaError_t err = cudaMemcpy( + got.data(), task.dst + done, (size_t)n, + cudaMemcpyDeviceToHost); + if (err != cudaSuccess) { + if (report) { + fprintf(stderr, + "ds4: CUDA selected batched I/O byte oracle read " + "failed at model offset %llu: %s\n", + (unsigned long long)(task.offset + done), + cudaGetErrorString(err)); + } + (void)cudaGetLastError(); + return 0; + } + const unsigned char *expected = + (const unsigned char *)model_map + task.offset + done; + if (memcmp(got.data(), expected, (size_t)n) != 0) { + if (report) { + size_t mismatch = 0; + while (mismatch < (size_t)n && + got[mismatch] == expected[mismatch]) { + mismatch++; + } + fprintf(stderr, + "ds4: CUDA selected batched I/O byte oracle " + "mismatch at model offset %llu " + "(expected=0x%02x got=0x%02x)\n", + (unsigned long long)(task.offset + done + mismatch), + (unsigned)expected[mismatch], + (unsigned)got[mismatch]); + } + return 0; + } + done += n; + } + } + + if (remap_count != 0) { + std::vector got_remap; + try { + got_remap.resize((size_t)remap_count); + } catch (...) { + return 0; + } + const size_t remap_bytes = + (size_t)remap_count * sizeof(got_remap[0]); + const cudaError_t err = cudaMemcpy( + got_remap.data(), remap_dst, remap_bytes, + cudaMemcpyDeviceToHost); + if (err != cudaSuccess || + memcmp(got_remap.data(), remap_src, remap_bytes) != 0) { + if (report) { + fprintf(stderr, + "ds4: CUDA selected batched I/O remap byte oracle %s\n", + err == cudaSuccess ? "mismatch" : + cudaGetErrorString(err)); + } + if (err != cudaSuccess) (void)cudaGetLastError(); + return 0; + } + } return 1; } +/* Queue every unique gate/up/down span plus the selected-id remap into one + * upload epoch. The ring events protect staging reuse; the sole final stream + * synchronization publishes the complete compact table atomically to the + * caller. submitted_out distinguishes safe pre-enqueue fallback from a + * partial upload, which must always fail closed. */ +static int cuda_model_copy_tasks_to_device_streamed( + const std::vector &tasks, + const void *model_map, + uint64_t model_size, + int32_t *remap_dst, + const int32_t *remap_src, + uint64_t remap_count, + int run_oracle, + uint64_t chunk_override, + int *submitted_out, + const char *what) { + if (submitted_out) *submitted_out = 0; + if (!model_map || tasks.empty() || !remap_dst || !remap_src || + remap_count == 0 || remap_count > SIZE_MAX / sizeof(int32_t)) { + return 0; + } + + const uint64_t chunk = chunk_override != 0 ? + chunk_override : cuda_model_copy_chunk_bytes(); + std::vector sorted; + std::vector segments; + std::vector groups; + if (!cuda_stream_selected_copy_plan(tasks, model_size, chunk, + &sorted, &segments, &groups)) { + fprintf(stderr, + "ds4: CUDA selected batched I/O could not build a valid " + "copy plan\n"); + return 0; + } + + uint64_t stage_bytes = 0; + if (!cuda_host_stage_bytes_for_chunk(chunk, g_model_direct_align, + &stage_bytes) || + !cuda_stream_selected_stage_pool_alloc(stage_bytes)) { + return 0; + } + + const bool profile = cuda_stream_selected_env_flag( + "DS4_CUDA_STREAMING_SELECTED_BATCH_IO_PROFILE"); + const double t0 = profile ? cuda_wall_sec() : 0.0; + uint64_t submitted_bytes = 0; + const bool use_fd = + g_model_fd >= 0 && + (g_model_fd_host_base == NULL || g_model_fd_host_base == model_map); + + if (use_fd) { + for (uint32_t gi = 0; gi < groups.size(); gi++) { + const cuda_stream_selected_copy_group &group = groups[gi]; + const uint64_t bi = gi % 4u; + if (gi >= 4u) { + const cudaError_t err = + cudaEventSynchronize(g_stream_selected_stage_event[bi]); + if (err != cudaSuccess) { + fprintf(stderr, + "ds4: CUDA selected batched I/O staging wait " + "failed for %s: %s\n", + what ? what : "expert batch", + cudaGetErrorString(err)); + (void)cudaGetLastError(); + return cuda_stream_selected_upload_fail(what); + } + } + + const char *payload = NULL; + if (!cuda_model_stage_read(g_stream_selected_stage[bi], + g_stream_selected_stage_bytes, + group.offset, group.bytes, &payload)) { + fprintf(stderr, + "ds4: CUDA selected batched I/O read failed for %s " + "at model offset %llu: %s\n", + what ? what : "expert batch", + (unsigned long long)group.offset, + strerror(errno)); + return cuda_stream_selected_upload_fail(what); + } + + for (uint32_t si = group.segment_begin; + si < group.segment_end; si++) { + const cuda_stream_selected_copy_segment &segment = + segments[si]; + const cuda_stream_selected_copy_task &task = + sorted[segment.task_index]; + const uint64_t stage_offset = segment.offset - group.offset; + const cudaError_t err = cudaMemcpyAsync( + task.dst + segment.task_offset, + payload + stage_offset, + (size_t)segment.bytes, + cudaMemcpyHostToDevice, + g_stream_selected_upload_stream); + if (err != cudaSuccess) { + fprintf(stderr, + "ds4: CUDA selected batched I/O copy failed for " + "%s at model offset %llu: %s\n", + what ? what : "expert batch", + (unsigned long long)segment.offset, + cudaGetErrorString(err)); + (void)cudaGetLastError(); + return cuda_stream_selected_upload_fail(what); + } + if (submitted_out) *submitted_out = 1; + if (segment.bytes > UINT64_MAX - submitted_bytes) { + return cuda_stream_selected_upload_fail(what); + } + submitted_bytes += segment.bytes; + } + const cudaError_t event_err = cudaEventRecord( + g_stream_selected_stage_event[bi], + g_stream_selected_upload_stream); + if (event_err != cudaSuccess) { + fprintf(stderr, + "ds4: CUDA selected batched I/O staging record failed " + "for %s: %s\n", + what ? what : "expert batch", + cudaGetErrorString(event_err)); + (void)cudaGetLastError(); + return cuda_stream_selected_upload_fail(what); + } + cuda_model_drop_file_pages(group.offset, group.bytes); + cuda_model_discard_source_pages(model_map, model_size, + group.offset, group.bytes); + } + } else { + for (const cuda_stream_selected_copy_task &task : sorted) { + const cudaError_t err = cudaMemcpyAsync( + task.dst, + (const char *)model_map + task.offset, + (size_t)task.bytes, + cudaMemcpyHostToDevice, + g_stream_selected_upload_stream); + if (err != cudaSuccess) { + fprintf(stderr, + "ds4: CUDA selected batched host copy failed for %s " + "at model offset %llu: %s\n", + what ? what : "expert batch", + (unsigned long long)task.offset, + cudaGetErrorString(err)); + (void)cudaGetLastError(); + return cuda_stream_selected_upload_fail(what); + } + if (submitted_out) *submitted_out = 1; + if (task.bytes > UINT64_MAX - submitted_bytes) { + return cuda_stream_selected_upload_fail(what); + } + submitted_bytes += task.bytes; + } + } + + const size_t remap_bytes = + (size_t)remap_count * sizeof(remap_src[0]); + cudaError_t err = cudaMemcpyAsync( + remap_dst, remap_src, remap_bytes, + cudaMemcpyHostToDevice, g_stream_selected_upload_stream); + if (err != cudaSuccess) { + fprintf(stderr, + "ds4: CUDA selected batched I/O remap copy failed: %s\n", + cudaGetErrorString(err)); + (void)cudaGetLastError(); + return cuda_stream_selected_upload_fail(what); + } + if (submitted_out) *submitted_out = 1; + + /* Async handoff seam: a future boundary-only change can replace this + * wait with an upload-done event recorded here and waited by the compute + * stream. All weight scatters and the remap are ordered before it. */ + err = cudaStreamSynchronize(g_stream_selected_upload_stream); + if (err != cudaSuccess) { + fprintf(stderr, + "ds4: CUDA selected batched I/O final sync failed for %s: %s\n", + what ? what : "expert batch", cudaGetErrorString(err)); + (void)cudaGetLastError(); + return 0; + } + + if (run_oracle) { + g_stream_selected_batch_io_oracle_runs++; + if (!cuda_stream_selected_copy_oracle( + tasks, model_map, model_size, remap_src, remap_dst, + remap_count, /*report=*/1)) { + g_stream_selected_batch_io_oracle_failures++; + fprintf(stderr, + "ds4: CUDA selected batched I/O failed the byte oracle; " + "compact table rejected\n"); + return 0; + } + } + + g_stream_selected_batch_io_tasks += tasks.size(); + g_stream_selected_batch_io_segments += segments.size(); + g_stream_selected_batch_io_reads += use_fd ? groups.size() : sorted.size(); + if (submitted_bytes <= UINT64_MAX - g_stream_selected_batch_io_bytes) { + g_stream_selected_batch_io_bytes += submitted_bytes; + } else { + g_stream_selected_batch_io_bytes = UINT64_MAX; + } + if (profile) { + const double t1 = cuda_wall_sec(); + fprintf(stderr, + "ds4: CUDA selected batched I/O tasks=%zu segments=%zu " + "reads=%zu bytes=%.2f MiB total=%.3f ms%s\n", + tasks.size(), segments.size(), + use_fd ? groups.size() : sorted.size(), + (double)submitted_bytes / 1048576.0, + (t1 - t0) * 1000.0, + run_oracle ? " (includes oracle)" : ""); + } + return 1; +} + +extern "C" int ds4_cuda_test_stream_selected_batch_plan(void) { + try { + const uint64_t layout_align = 64u; + const uint64_t layout_usable = 513u; + size_t layout_allocation = 0; + std::vector layout_storage( + (size_t)(layout_usable + 2u * layout_align)); + const uintptr_t layout_base = (uintptr_t)layout_storage.data(); + const size_t to_boundary = (size_t)( + (layout_align - layout_base % layout_align) % layout_align); + void *layout_raw = layout_storage.data() + to_boundary + 1u; + void *layout_aligned = NULL; + uint64_t overflow_stage = 0; + if (!cuda_host_stage_allocation_bytes( + layout_usable, layout_align, &layout_allocation) || + layout_allocation != layout_usable + layout_align - 1u || + !cuda_host_stage_aligned_view( + layout_raw, layout_allocation, layout_usable, + layout_align, &layout_aligned) || + (uintptr_t)layout_raw % layout_align != 1u || + (uintptr_t)layout_aligned % layout_align != 0u || + (char *)layout_aligned + layout_usable != + (char *)layout_raw + layout_allocation || + cuda_host_stage_aligned_view( + layout_raw, layout_allocation - 1u, layout_usable, + layout_align, &layout_aligned) || + cuda_host_stage_allocation_bytes( + UINT64_MAX, 2u, &layout_allocation) || + cuda_host_stage_bytes_for_chunk( + UINT64_MAX, 2u, &overflow_stage)) { + return 0; + } + memset(layout_aligned, 0xa5, (size_t)layout_usable); + + std::vector storage(64u * 1024u); + std::vector tasks = { + { storage.data() + 0u, 8192u, 4096u, 0u }, + { storage.data() + 4096u, 0u, 4096u, 1u }, + { storage.data() + 8192u, 4096u, 4096u, 2u }, + { storage.data() + 12288u, 12288u, 4096u, 3u }, + { storage.data() + 16384u, 32768u, 4096u, 4u }, + }; + std::vector sorted; + std::vector segments; + std::vector groups; + if (!cuda_stream_selected_copy_plan( + tasks, storage.size(), 12288u, + &sorted, &segments, &groups) || + sorted.size() != 5u || segments.size() != 5u || + groups.size() != 3u || + sorted[0].ordinal != 1u || sorted[1].ordinal != 2u || + sorted[2].ordinal != 0u || sorted[3].ordinal != 3u || + sorted[4].ordinal != 4u || + groups[0].offset != 0u || groups[0].bytes != 12288u || + groups[0].segment_begin != 0u || + groups[0].segment_end != 3u) { + return 0; + } + + tasks = { + { storage.data(), 0u, 20000u, 0u }, + { storage.data() + 20000u, 20000u, 4000u, 1u }, + }; + if (!cuda_stream_selected_copy_plan( + tasks, storage.size(), 8192u, + &sorted, &segments, &groups) || + segments.size() != 4u || groups.size() != 3u || + groups[2].offset != 16384u || groups[2].bytes != 7616u) { + return 0; + } + + tasks = { + { storage.data(), 0u, 4096u, 0u }, + { storage.data() + 4096u, 2048u, 4096u, 1u }, + }; + if (cuda_stream_selected_copy_plan( + tasks, storage.size(), 8192u, + &sorted, &segments, &groups)) { + return 0; + } + + tasks = { + { storage.data(), UINT64_MAX - 1u, 2u, 0u }, + }; + if (cuda_stream_selected_copy_plan( + tasks, UINT64_MAX, 8192u, + &sorted, &segments, &groups)) { + return 0; + } + } catch (...) { + return 0; + } + return 1; +} + +/* Exercise aligned staging, one-read scatter, ring wrap, remap upload and a + * deliberate byte-oracle failure on a real CUDA device. The test restores + * every temporary file-global before returning. */ +extern "C" int ds4_cuda_test_stream_selected_batch_copy(void) { + if (g_n_gpus < 1) return 0; + int saved_device = -1; + (void)cudaGetDevice(&saved_device); + const int saved_logical_tier = g_current_logical_tier; + const auto restore_device = [saved_device, saved_logical_tier]() { + if (saved_device >= 0) (void)cudaSetDevice(saved_device); + g_current_logical_tier = saved_logical_tier; + }; + if (ds4_gpu_set_current_device(0) != 0) { + restore_device(); + return 0; + } + if (g_model_stage_bytes != 0 || g_model_upload_stream || + g_stream_selected_stage_bytes != 0 || + g_stream_selected_upload_stream) { + restore_device(); + return 0; + } + + const size_t model_bytes = 16u * 1024u; + std::vector source; + try { + source.resize(model_bytes); + } catch (...) { + restore_device(); + return 0; + } + for (size_t i = 0; i < source.size(); i++) { + source[i] = (unsigned char)((i * 37u + 11u) & 0xffu); + } + + FILE *fp = tmpfile(); + if (!fp || fwrite(source.data(), 1, source.size(), fp) != source.size() || + fflush(fp) != 0) { + if (fp) fclose(fp); + restore_device(); + return 0; + } + void *map = mmap(NULL, model_bytes, PROT_READ, MAP_PRIVATE, + fileno(fp), 0); + if (map == MAP_FAILED) { + fclose(fp); + restore_device(); + return 0; + } + + char *dst[6] = { NULL, NULL, NULL, NULL, NULL, NULL }; + int32_t *remap_dst = NULL; + int ok = 1; + for (uint32_t i = 0; i < 6u; i++) { + if (cudaMalloc((void **)&dst[i], 4096u) != cudaSuccess) ok = 0; + } + const int32_t remap_src[6] = { 2, 0, 1, 2, 2, 0 }; + if (cudaMalloc((void **)&remap_dst, sizeof(remap_src)) != cudaSuccess) { + ok = 0; + } + + const int saved_fd = g_model_fd; + const void *saved_fd_host_base = g_model_fd_host_base; + const int saved_direct_fd = g_model_direct_fd; + const uint64_t saved_direct_align = g_model_direct_align; + const uint64_t saved_file_size = g_model_file_size; + g_model_fd = fileno(fp); + g_model_fd_host_base = map; + g_model_direct_fd = -1; + g_model_direct_align = 257u; + g_model_file_size = model_bytes; + + if (ok) { + uint64_t stage_bytes = 0; + size_t allocation_bytes = 0; + ok = cuda_host_stage_bytes_for_chunk( + 4096u, g_model_direct_align, &stage_bytes) && + cuda_host_stage_allocation_bytes( + stage_bytes, g_model_direct_align, &allocation_bytes) && + cuda_model_stage_pool_alloc(stage_bytes) && + g_model_stage_bytes == stage_bytes && + g_model_stage_align == g_model_direct_align; + for (uint32_t i = 0; ok && i < 4u; i++) { + const size_t delta = (size_t)( + (char *)g_model_stage[i] - + (char *)g_model_stage_raw[i]); + ok = (uintptr_t)g_model_stage[i] % g_model_direct_align == 0u && + delta <= allocation_bytes && + stage_bytes <= allocation_bytes - delta; + } + cuda_model_stage_pool_release(); + } + + if (ok) { + try { + std::vector tasks = { + { dst[0], 8192u, 4096u, 0u }, + { dst[1], 0u, 4096u, 1u }, + { dst[2], 4096u, 4096u, 2u }, + }; + int submitted = 0; + ok = cuda_model_copy_tasks_to_device_streamed( + tasks, map, model_bytes, + remap_dst, remap_src, 6u, + /*run_oracle=*/1, + /*chunk_override=*/12288u, + &submitted, "selected batch selftest") && submitted; + if (ok) { + size_t selected_allocation = 0; + ok = cuda_host_stage_allocation_bytes( + g_stream_selected_stage_bytes, + g_stream_selected_stage_align, + &selected_allocation); + void *first_ring_raw[4]; + for (uint32_t i = 0; ok && i < 4u; i++) { + first_ring_raw[i] = g_stream_selected_stage_raw[i]; + const size_t delta = (size_t)( + (char *)g_stream_selected_stage[i] - + (char *)g_stream_selected_stage_raw[i]); + ok = (uintptr_t)g_stream_selected_stage[i] % + g_model_direct_align == 0u && + delta <= selected_allocation && + g_stream_selected_stage_bytes <= + selected_allocation - delta; + } + const int32_t wrap_remap[6] = { 5, 4, 3, 2, 1, 0 }; + std::vector wrap_tasks = { + { dst[0], 0u, 1024u, 0u }, + { dst[1], 2048u, 1024u, 1u }, + { dst[2], 4096u, 1024u, 2u }, + { dst[3], 6144u, 1024u, 3u }, + { dst[4], 8192u, 1024u, 4u }, + { dst[5], 10240u, 1024u, 5u }, + }; + std::vector wrap_sorted; + std::vector wrap_segments; + std::vector wrap_groups; + submitted = 0; + ok = ok && cuda_stream_selected_copy_plan( + wrap_tasks, model_bytes, 4096u, + &wrap_sorted, &wrap_segments, &wrap_groups) && + wrap_groups.size() == 6u && + cuda_model_copy_tasks_to_device_streamed( + wrap_tasks, map, model_bytes, + remap_dst, wrap_remap, 6u, + /*run_oracle=*/1, + /*chunk_override=*/4096u, + &submitted, + "selected batch ring-wrap selftest") && + submitted; + for (uint32_t i = 0; ok && i < 4u; i++) { + ok = first_ring_raw[i] == + g_stream_selected_stage_raw[i]; + } + if (ok) { + const unsigned char corrupt = + (unsigned char)(source[8192u] ^ 0xffu); + ok = cudaMemcpy(dst[4], &corrupt, sizeof(corrupt), + cudaMemcpyHostToDevice) == cudaSuccess && + !cuda_stream_selected_copy_oracle( + wrap_tasks, map, model_bytes, + wrap_remap, remap_dst, 6u, /*report=*/0); + } + } + } catch (...) { + ok = 0; + } + } + + g_model_fd = saved_fd; + g_model_fd_host_base = saved_fd_host_base; + g_model_direct_fd = saved_direct_fd; + g_model_direct_align = saved_direct_align; + g_model_file_size = saved_file_size; + cuda_model_stage_pool_release(); + if (g_model_upload_stream) { + (void)cudaStreamDestroy(g_model_upload_stream); + g_model_upload_stream = NULL; + } + cuda_stream_selected_stage_release(); + for (uint32_t i = 0; i < 6u; i++) { + if (dst[i]) (void)cudaFree(dst[i]); + } + if (remap_dst) (void)cudaFree(remap_dst); + (void)munmap(map, model_bytes); + (void)fclose(fp); + restore_device(); + return ok; +} + static int cuda_model_copy_to_device_streamed( char *dst, const void *model_map, @@ -2649,9 +3663,10 @@ static int cuda_model_copy_to_device_streamed( } const uint64_t chunk = cuda_model_copy_chunk_bytes(); - const uint64_t stage_bytes = - chunk + (g_model_direct_align > 1 ? g_model_direct_align : 1); - if (!cuda_stream_selected_stage_pool_alloc(stage_bytes)) return 0; + uint64_t stage_bytes = 0; + if (!cuda_host_stage_bytes_for_chunk(chunk, g_model_direct_align, + &stage_bytes) || + !cuda_stream_selected_stage_pool_alloc(stage_bytes)) return 0; uint64_t copied = 0; uint64_t chunk_idx = 0; @@ -2666,7 +3681,7 @@ static int cuda_model_copy_to_device_streamed( "ds4: CUDA streaming selected staging wait failed for %s: %s\n", what ? what : "expert", cudaGetErrorString(err)); (void)cudaGetLastError(); - return 0; + return cuda_stream_selected_upload_fail(what); } } const char *payload = NULL; @@ -2677,7 +3692,7 @@ static int cuda_model_copy_to_device_streamed( "ds4: CUDA streaming selected read failed for %s at %.2f MiB: %s\n", what ? what : "expert", (double)copied / 1048576.0, strerror(errno)); - return 0; + return cuda_stream_selected_upload_fail(what); } err = cudaMemcpyAsync(dst + copied, payload, (size_t)n, cudaMemcpyHostToDevice, @@ -2688,7 +3703,7 @@ static int cuda_model_copy_to_device_streamed( what ? what : "expert", (double)copied / 1048576.0, cudaGetErrorString(err)); (void)cudaGetLastError(); - return 0; + return cuda_stream_selected_upload_fail(what); } err = cudaEventRecord(g_stream_selected_stage_event[bi], g_stream_selected_upload_stream); @@ -2697,7 +3712,7 @@ static int cuda_model_copy_to_device_streamed( "ds4: CUDA streaming selected staging record failed for %s: %s\n", what ? what : "expert", cudaGetErrorString(err)); (void)cudaGetLastError(); - return 0; + return cuda_stream_selected_upload_fail(what); } cuda_model_drop_file_pages(offset + copied, n); cuda_model_discard_source_pages(model_map, model_size, @@ -2815,8 +3830,10 @@ static const char *cuda_model_range_ptr_from_fd( cudaError_t err = cudaSuccess; const uint64_t chunk = cuda_model_copy_chunk_bytes(); - const uint64_t stage_bytes = chunk + (g_model_direct_align > 1 ? g_model_direct_align : 1); - if (!cuda_model_stage_pool_alloc(stage_bytes)) return NULL; + uint64_t stage_bytes = 0; + if (!cuda_host_stage_bytes_for_chunk(chunk, g_model_direct_align, + &stage_bytes) || + !cuda_model_stage_pool_alloc(stage_bytes)) return NULL; uint64_t copied = 0; uint64_t chunk_idx = 0; @@ -2829,6 +3846,7 @@ static const char *cuda_model_range_ptr_from_fd( fprintf(stderr, "ds4: CUDA model staging wait failed for %s: %s\n", what ? what : "weights", cudaGetErrorString(err)); (void)cudaGetLastError(); + (void)cuda_model_upload_fail(what); return NULL; } } @@ -2839,6 +3857,7 @@ static const char *cuda_model_range_ptr_from_fd( what ? what : "weights", (double)copied / 1048576.0, strerror(errno)); + (void)cuda_model_upload_fail(what); return NULL; } err = cudaMemcpyAsync(dev + copied, payload, (size_t)n, @@ -2849,6 +3868,7 @@ static const char *cuda_model_range_ptr_from_fd( (double)copied / 1048576.0, cudaGetErrorString(err)); (void)cudaGetLastError(); + (void)cuda_model_upload_fail(what); return NULL; } err = cudaEventRecord(g_model_stage_event[bi], g_model_upload_stream); @@ -2856,6 +3876,7 @@ static const char *cuda_model_range_ptr_from_fd( fprintf(stderr, "ds4: CUDA model staging record failed for %s: %s\n", what ? what : "weights", cudaGetErrorString(err)); (void)cudaGetLastError(); + (void)cuda_model_upload_fail(what); return NULL; } cuda_model_drop_file_pages(offset + copied, n); @@ -3329,18 +4350,7 @@ extern "C" void ds4_gpu_cleanup(void) { g_cuda_tmp = NULL; g_cuda_tmp_bytes = 0; } - for (size_t i = 0; i < 4; i++) { - if (g_model_stage_event[i]) { - (void)cudaEventDestroy(g_model_stage_event[i]); - g_model_stage_event[i] = NULL; - } - if (g_model_stage_raw[i]) { - (void)cudaFreeHost(g_model_stage_raw[i]); - g_model_stage_raw[i] = NULL; - g_model_stage[i] = NULL; - } - } - g_model_stage_bytes = 0; + cuda_model_stage_pool_release(); if (g_model_upload_stream) { (void)cudaStreamDestroy(g_model_upload_stream); g_model_upload_stream = NULL; @@ -28558,43 +29568,130 @@ static int cuda_stream_selected_cache_begin_load( return 0; } - for (uint32_t i = 0; i < compact_ids.size(); i++) { - const uint64_t expert = (uint32_t)compact_ids[i]; - const uint64_t gate_src = - table->gate_offset + expert * table->gate_expert_bytes; - const uint64_t up_src = - table->up_offset + expert * table->gate_expert_bytes; - const uint64_t down_src = - table->down_offset + expert * table->down_expert_bytes; - const uint64_t gate_dst = (uint64_t)i * table->gate_expert_bytes; - const uint64_t down_dst = (uint64_t)i * table->down_expert_bytes; - if (!cuda_model_copy_to_device_streamed( - g_stream_selected_cache.gate_ptr + gate_dst, - table->model_map, table->model_size, - gate_src, table->gate_expert_bytes, - "stream gate expert copy") || - !cuda_model_copy_to_device_streamed( - g_stream_selected_cache.up_ptr + gate_dst, - table->model_map, table->model_size, - up_src, table->gate_expert_bytes, - "stream up expert copy") || - !cuda_model_copy_to_device_streamed( - g_stream_selected_cache.down_ptr + down_dst, - table->model_map, table->model_size, - down_src, table->down_expert_bytes, - "stream down expert copy")) { + g_stream_selected_batch_io_candidates++; + const int batch_io = cuda_stream_selected_batch_io_requested(); + const int batch_io_required = + cuda_stream_selected_batch_io_require_requested(); + int copied = 0; + if (batch_io) { + std::vector tasks; + int task_plan_ready = 1; + try { + if (compact_ids.size() > SIZE_MAX / 3u || + compact_ids.size() > UINT32_MAX / 3u) { + task_plan_ready = 0; + } else { + tasks.reserve(compact_ids.size() * 3u); + uint32_t ordinal = 0; + for (uint32_t i = 0; i < compact_ids.size(); i++) { + const uint64_t expert = (uint32_t)compact_ids[i]; + const uint64_t gate_src = table->gate_offset + + expert * table->gate_expert_bytes; + const uint64_t up_src = table->up_offset + + expert * table->gate_expert_bytes; + const uint64_t down_src = table->down_offset + + expert * table->down_expert_bytes; + const uint64_t gate_dst = + (uint64_t)i * table->gate_expert_bytes; + const uint64_t down_dst = + (uint64_t)i * table->down_expert_bytes; + tasks.push_back({ + g_stream_selected_cache.gate_ptr + gate_dst, + gate_src, table->gate_expert_bytes, ordinal++, + }); + tasks.push_back({ + g_stream_selected_cache.up_ptr + gate_dst, + up_src, table->gate_expert_bytes, ordinal++, + }); + tasks.push_back({ + g_stream_selected_cache.down_ptr + down_dst, + down_src, table->down_expert_bytes, ordinal++, + }); + } + } + } catch (...) { + task_plan_ready = 0; + } + + if (task_plan_ready) { + int submitted = 0; + g_stream_selected_batch_io_attempts++; + copied = cuda_model_copy_tasks_to_device_streamed( + tasks, table->model_map, table->model_size, + g_stream_selected_cache.slot_selected_ptr, + slot_ids.data(), slot_count, + cuda_stream_selected_batch_io_oracle_requested(), + /*chunk_override=*/0, &submitted, + "selected expert batch"); + if (copied) { + g_stream_selected_batch_io_completed++; + } else { + g_stream_selected_batch_io_failures++; + if (batch_io_required) { + g_stream_selected_batch_io_required_failures++; + } + if (submitted || batch_io_required) { + cuda_stream_selected_cache_invalidate(); + return 0; + } + g_stream_selected_batch_io_safe_fallbacks++; + } + } else { + g_stream_selected_batch_io_failures++; + if (batch_io_required) { + g_stream_selected_batch_io_required_failures++; + fprintf(stderr, + "ds4: required CUDA selected batched I/O could not " + "allocate its copy plan\n"); + cuda_stream_selected_cache_invalidate(); + return 0; + } + g_stream_selected_batch_io_safe_fallbacks++; + } + } + + if (!copied) { + g_stream_selected_batch_io_legacy++; + for (uint32_t i = 0; i < compact_ids.size(); i++) { + const uint64_t expert = (uint32_t)compact_ids[i]; + const uint64_t gate_src = + table->gate_offset + expert * table->gate_expert_bytes; + const uint64_t up_src = + table->up_offset + expert * table->gate_expert_bytes; + const uint64_t down_src = + table->down_offset + expert * table->down_expert_bytes; + const uint64_t gate_dst = + (uint64_t)i * table->gate_expert_bytes; + const uint64_t down_dst = + (uint64_t)i * table->down_expert_bytes; + if (!cuda_model_copy_to_device_streamed( + g_stream_selected_cache.gate_ptr + gate_dst, + table->model_map, table->model_size, + gate_src, table->gate_expert_bytes, + "stream gate expert copy") || + !cuda_model_copy_to_device_streamed( + g_stream_selected_cache.up_ptr + gate_dst, + table->model_map, table->model_size, + up_src, table->gate_expert_bytes, + "stream up expert copy") || + !cuda_model_copy_to_device_streamed( + g_stream_selected_cache.down_ptr + down_dst, + table->model_map, table->model_size, + down_src, table->down_expert_bytes, + "stream down expert copy")) { + cuda_stream_selected_cache_invalidate(); + return 0; + } + } + if (!cuda_ok(cudaMemcpy(g_stream_selected_cache.slot_selected_ptr, + slot_ids.data(), + (size_t)slot_count * sizeof(int32_t), + cudaMemcpyHostToDevice), + "stream selected-id remap copy")) { cuda_stream_selected_cache_invalidate(); return 0; } } - if (!cuda_ok(cudaMemcpy(g_stream_selected_cache.slot_selected_ptr, - slot_ids.data(), - (size_t)slot_count * sizeof(int32_t), - cudaMemcpyHostToDevice), - "stream selected-id remap copy")) { - cuda_stream_selected_cache_invalidate(); - return 0; - } g_stream_selected_cache.logical_tier = logical_tier; g_stream_selected_cache.model_map = table->model_map; diff --git a/ds4_gpu.h b/ds4_gpu.h index e4595030a..732b51592 100644 --- a/ds4_gpu.h +++ b/ds4_gpu.h @@ -285,6 +285,37 @@ void ds4_gpu_release_zero_prefix_prefill_mask_cache(void); static inline int ds4_gpu_device_is_pre_m5_apple_silicon(void) { return 0; } static inline int ds4_gpu_device_is_m5_apple_silicon(void) { return 0; } #endif +#if !defined(__APPLE__) && !defined(DS4_ROCM_BUILD) && !defined(DS4_NO_GPU) +typedef struct ds4_cuda_stream_selected_batch_io_report { + uint64_t candidates; + uint64_t attempts; + uint64_t completed; + uint64_t legacy_batches; + uint64_t safe_fallbacks; + uint64_t failures; + uint64_t required_failures; + uint64_t oracle_runs; + uint64_t oracle_failures; + uint64_t tasks; + uint64_t segments; + uint64_t reads; + uint64_t bytes; + int enabled; + int required; + int oracle; +} ds4_cuda_stream_selected_batch_io_report; +/* CUDA-only policy/planner/scatter hooks. The policy hook takes explicit + * values so tests do not need to mutate process environment around once_flag + * initialization. */ +int ds4_cuda_test_stream_selected_batch_policy( + int enable, int disable, int require, int oracle, + int *enabled_out, int *required_out, int *oracle_out); +int ds4_cuda_test_stream_selected_batch_env_value(const char *value); +int ds4_cuda_test_stream_selected_batch_plan(void); +int ds4_cuda_test_stream_selected_batch_copy(void); +void ds4_cuda_stream_selected_batch_io_get_report( + ds4_cuda_stream_selected_batch_io_report *report); +#endif void ds4_gpu_set_streaming_expert_cache_budget(uint32_t experts); void ds4_gpu_set_streaming_expert_cache_expert_bytes(uint64_t bytes); uint64_t ds4_gpu_recommended_working_set_size(void); diff --git a/tests/test_gpu_model_cache.c b/tests/test_gpu_model_cache.c index ebbd0c4e0..53cecf2ce 100644 --- a/tests/test_gpu_model_cache.c +++ b/tests/test_gpu_model_cache.c @@ -6,6 +6,7 @@ * - ds4_gpu_lookup_cache at range bases and at interior offsets * (proves the subrange pointer offset arithmetic is right) * - device-id resolution + * - selected-expert batched-I/O policy, planner, scatter and byte oracle * - on multi-GPU boxes: caching on device 1 and active-device * preference in lookup */ @@ -26,6 +27,34 @@ } while (0) int main(void) { + int enabled = -1; + int required = -1; + int oracle = -1; + CHECK(!ds4_cuda_test_stream_selected_batch_env_value(NULL) && + !ds4_cuda_test_stream_selected_batch_env_value("") && + !ds4_cuda_test_stream_selected_batch_env_value("0") && + !ds4_cuda_test_stream_selected_batch_env_value("false") && + !ds4_cuda_test_stream_selected_batch_env_value("FALSE") && + !ds4_cuda_test_stream_selected_batch_env_value("no") && + !ds4_cuda_test_stream_selected_batch_env_value("off") && + ds4_cuda_test_stream_selected_batch_env_value("1") && + ds4_cuda_test_stream_selected_batch_env_value("true"), + "selected-expert batched-I/O value-aware environment parser"); + CHECK(ds4_cuda_test_stream_selected_batch_policy( + 1, 0, 0, 0, &enabled, &required, &oracle) && + enabled == 1 && required == 0 && oracle == 0, + "selected-expert batched-I/O enable policy"); + CHECK(ds4_cuda_test_stream_selected_batch_policy( + 0, 0, 1, 0, &enabled, &required, &oracle) && + enabled == 1 && required == 1 && oracle == 0, + "selected-expert batched-I/O require policy"); + CHECK(ds4_cuda_test_stream_selected_batch_policy( + 1, 1, 1, 1, &enabled, &required, &oracle) && + enabled == 0 && required == 0 && oracle == 0, + "selected-expert batched-I/O disable-dominant policy"); + CHECK(ds4_cuda_test_stream_selected_batch_plan(), + "selected-expert batched-I/O planner"); + int dev_count = 0; (void)cudaGetDeviceCount(&dev_count); fprintf(stderr, "test_gpu_model_cache: %d CUDA devices visible\n", @@ -36,6 +65,17 @@ int main(void) { } CHECK(ds4_gpu_init(), "ds4_gpu_init"); + CHECK(ds4_cuda_test_stream_selected_batch_copy(), + "selected-expert batched-I/O scatter + byte oracle"); + ds4_cuda_stream_selected_batch_io_report batch_report; + memset(&batch_report, 0, sizeof(batch_report)); + ds4_cuda_stream_selected_batch_io_get_report(&batch_report); + CHECK(batch_report.oracle_runs >= 2 && + batch_report.oracle_failures == 0 && + batch_report.tasks >= 9 && + batch_report.segments >= batch_report.tasks && + batch_report.reads >= 7 && batch_report.bytes >= 18u * 1024u, + "selected-expert batched-I/O coverage counters"); /* Build a synthetic 1-MiB "model" in host memory. */ const size_t total = 1024 * 1024; From ed1cab3773dd08e3e539edd330cafd7d2d87fe50 Mon Sep 17 00:00:00 2001 From: Giorgio Oppo Date: Fri, 14 Aug 2026 18:48:03 +0200 Subject: [PATCH 031/189] cuda: overlap selected expert I/O with shared compute --- ds4.c | 211 ++++++++- ds4_cuda.cu | 872 +++++++++++++++++++++++++++++++++-- ds4_gpu.h | 45 +- tests/test_gpu_model_cache.c | 38 ++ 4 files changed, 1121 insertions(+), 45 deletions(-) diff --git a/ds4.c b/ds4.c index 45db9399f..9727b8e5e 100644 --- a/ds4.c +++ b/ds4.c @@ -22408,11 +22408,15 @@ typedef struct metal_graph_selected_async_load { /* Selected ids remain usable for a synchronous retry if the service * thread cannot stage the cache load without waiting on GPU work. */ bool ids_ok; + bool cuda_event_pipeline; + bool cuda_event_required; + bool fail_closed; ds4_gpu_tensor *router_selected; const ds4_model *model; const ds4_layer_weights *layer; uint32_t il; uint64_t event_value; + uint64_t upload_event_value; uint64_t gate_expert_bytes; uint64_t down_expert_bytes; int32_t selected_ids[DS4_MAX_EXPERT_USED]; @@ -22439,8 +22443,54 @@ static void metal_graph_selected_async_load_run( DS4_N_EXPERT_USED == 0 || DS4_N_EXPERT_USED > DS4_MAX_EXPERT_USED) { return; } +#if !defined(__APPLE__) && !defined(DS4_ROCM_BUILD) && !defined(DS4_NO_GPU) + if (job->cuda_event_pipeline && + ds4_gpu_cuda_stream_selected_set_owner_device() == 0) { + job->fail_closed = job->cuda_event_required; + return; + } +#endif if (job->event_value != 0) { -#ifdef DS4_ROCM_BUILD +#if !defined(__APPLE__) && !defined(DS4_ROCM_BUILD) && !defined(DS4_NO_GPU) + if (job->cuda_event_pipeline) { + if (ds4_gpu_tensor_read_after_selected_event( + job->router_selected, + 0, + job->selected_ids, + (uint64_t)DS4_N_EXPERT_USED * + sizeof(job->selected_ids[0]), + job->event_value, + "selected-id async expert load") == 0) { + if (job->cuda_event_required) { + job->fail_closed = true; + return; + } + ds4_gpu_cuda_stream_selected_event_note_fallback(); + if (ds4_gpu_synchronize() == 0 || + ds4_gpu_tensor_read( + job->router_selected, + 0, + job->selected_ids, + (uint64_t)DS4_N_EXPERT_USED * + sizeof(job->selected_ids[0])) == 0) { + return; + } + job->cuda_event_pipeline = false; + } + } else { + if (ds4_gpu_wait_selected_readback_ready( + job->event_value, + "selected-id async expert load") == 0 || + ds4_gpu_tensor_read( + job->router_selected, + 0, + job->selected_ids, + (uint64_t)DS4_N_EXPERT_USED * + sizeof(job->selected_ids[0])) == 0) { + return; + } + } +#elif defined(DS4_ROCM_BUILD) if (ds4_gpu_tensor_read_after_selected_event( job->router_selected, 0, @@ -22452,15 +22502,15 @@ static void metal_graph_selected_async_load_run( return; } #else - if (ds4_gpu_wait_selected_readback_ready(job->event_value, - "selected-id async expert load") == 0) { - return; - } - if (ds4_gpu_tensor_read(job->router_selected, - 0, - job->selected_ids, - (uint64_t)DS4_N_EXPERT_USED * - sizeof(job->selected_ids[0])) == 0) { + if (ds4_gpu_wait_selected_readback_ready( + job->event_value, + "selected-id async expert load") == 0 || + ds4_gpu_tensor_read( + job->router_selected, + 0, + job->selected_ids, + (uint64_t)DS4_N_EXPERT_USED * + sizeof(job->selected_ids[0])) == 0) { return; } #endif @@ -22483,6 +22533,32 @@ static void metal_graph_selected_async_load_run( job->il, job->gate_expert_bytes, job->down_expert_bytes); +#if !defined(__APPLE__) && !defined(DS4_ROCM_BUILD) && !defined(DS4_NO_GPU) + if (job->cuda_event_pipeline) { + const int load_rc = + ds4_gpu_stream_expert_cache_begin_selected_load_async( + &table, + job->selected_ids, + DS4_N_EXPERT_USED, + &job->upload_event_value); + if (load_rc <= 0) { + if (load_rc < 0 || job->cuda_event_required) { + job->fail_closed = true; + return; + } + /* A zero result guarantees that no H2D operation was submitted, + * so the established synchronous loader is a safe rollback. */ + ds4_gpu_cuda_stream_selected_event_note_fallback(); + job->cuda_event_pipeline = false; + if (ds4_gpu_stream_expert_cache_begin_selected_load( + &table, + job->selected_ids, + DS4_N_EXPERT_USED) == 0) { + return; + } + } + } else +#endif if (ds4_gpu_stream_expert_cache_begin_selected_load( &table, job->selected_ids, @@ -22552,6 +22628,8 @@ static DS4_MAYBE_UNUSED bool metal_graph_selected_async_load_start_tensor( const ds4_layer_weights *layer, uint32_t il, uint64_t event_value, + bool cuda_event_pipeline, + bool cuda_event_required, uint64_t gate_expert_bytes, uint64_t down_expert_bytes) { if (!job || !router_selected || event_value == 0) return false; @@ -22562,6 +22640,8 @@ static DS4_MAYBE_UNUSED bool metal_graph_selected_async_load_start_tensor( job->layer = layer; job->il = il; job->event_value = event_value; + job->cuda_event_pipeline = cuda_event_pipeline; + job->cuda_event_required = cuda_event_required; job->gate_expert_bytes = gate_expert_bytes; job->down_expert_bytes = down_expert_bytes; pthread_mutex_lock(&g_metal_graph_selected_async_load_mutex); @@ -22586,6 +22666,8 @@ static DS4_MAYBE_UNUSED bool metal_graph_selected_async_load_start( const ds4_layer_weights *layer, uint32_t il, uint64_t event_value, + bool cuda_event_pipeline, + bool cuda_event_required, uint64_t gate_expert_bytes, uint64_t down_expert_bytes) { return metal_graph_selected_async_load_start_tensor( @@ -22595,6 +22677,8 @@ static DS4_MAYBE_UNUSED bool metal_graph_selected_async_load_start( layer, il, event_value, + cuda_event_pipeline, + cuda_event_required, gate_expert_bytes, down_expert_bytes); } @@ -25772,6 +25856,27 @@ static bool metal_graph_encode_decode_layer_phase( (mxfp4_selected_shared_overlap && metal_graph_use_iq2_selected_async_load(g)) || cuda_selected_shared_overlap); + bool cuda_selected_event_pipeline = false; + bool cuda_selected_event_required = false; +#if !defined(__APPLE__) && !defined(DS4_ROCM_BUILD) && !defined(DS4_NO_GPU) + cuda_selected_event_required = + ds4_gpu_cuda_stream_selected_event_pipeline_required() != 0; + cuda_selected_event_pipeline = + ds4_gpu_cuda_stream_selected_event_pipeline_enabled() != 0 && + cuda_selected_shared_overlap && + async_selected_load; + if (cuda_selected_event_required && + !cuda_selected_event_pipeline) { + ds4_gpu_cuda_stream_selected_event_note_candidate(); + ds4_gpu_cuda_stream_selected_event_note_failure(1); + fprintf(stderr, + "ds4: required CUDA selected-expert event pipeline is not " + "eligible at layer %u\n", + il); + (void)ds4_gpu_cuda_stream_selected_event_abort(); + return false; + } +#endif const bool selected_readahead_shared_delay = ok && !external_routed && @@ -25936,7 +26041,29 @@ static bool metal_graph_encode_decode_layer_phase( } if (overlap_selected_shared) { uint64_t selected_event = 0; - if (ok) ok = ds4_gpu_signal_selected_readback_ready(&selected_event) != 0; +#if !defined(__APPLE__) && !defined(DS4_ROCM_BUILD) && !defined(DS4_NO_GPU) + if (ok && cuda_selected_event_pipeline) { + ds4_gpu_cuda_stream_selected_event_note_candidate(); + if (ds4_gpu_signal_selected_readback_ready_async( + &selected_event) == 0) { + ds4_gpu_cuda_stream_selected_event_note_failure( + cuda_selected_event_required ? 1 : 0); + if (cuda_selected_event_required) { + fprintf(stderr, + "ds4: required CUDA selected compute-ready " + "event failed at layer %u\n", + il); + (void)ds4_gpu_cuda_stream_selected_event_abort(); + return false; + } + ds4_gpu_cuda_stream_selected_event_note_fallback(); + cuda_selected_event_pipeline = false; + } + } +#endif + if (ok && !cuda_selected_event_pipeline) { + ok = ds4_gpu_signal_selected_readback_ready(&selected_event) != 0; + } metal_graph_selected_async_load async_load = {0}; bool async_load_started = false; const bool iq2_metal_merged_boundary = @@ -25944,6 +26071,7 @@ static bool metal_graph_encode_decode_layer_phase( !cuda_selected_shared_overlap; const bool async_early_commit = async_selected_load && + !cuda_selected_event_pipeline && (iq2_metal_merged_boundary ? metal_graph_use_iq2_selected_async_early_commit(g) : metal_graph_use_selected_async_early_commit_legacy(g)); @@ -25957,8 +26085,26 @@ static bool metal_graph_encode_decode_layer_phase( layer, il, selected_event, + cuda_selected_event_pipeline, + cuda_selected_event_required, gate_expert_bytes, down_expert_bytes); +#if !defined(__APPLE__) && !defined(DS4_ROCM_BUILD) && !defined(DS4_NO_GPU) + if (!async_load_started && cuda_selected_event_pipeline) { + ds4_gpu_cuda_stream_selected_event_note_failure( + cuda_selected_event_required ? 1 : 0); + if (cuda_selected_event_required) { + fprintf(stderr, + "ds4: required CUDA selected-expert worker " + "could not start at layer %u\n", + il); + (void)ds4_gpu_cuda_stream_selected_event_abort(); + return false; + } + ds4_gpu_cuda_stream_selected_event_note_fallback(); + cuda_selected_event_pipeline = false; + } +#endif } if (ok && async_early_commit && async_load_started) { ok = ds4_gpu_flush_commands() != 0; @@ -26005,13 +26151,22 @@ static bool metal_graph_encode_decode_layer_phase( } DS4_METAL_PROFILE_DECODE_STAGE("shared_down"); if (async_load_started) { - const bool flush_ok = ds4_gpu_flush_commands() != 0; + const bool flush_ok = cuda_selected_event_pipeline || + ds4_gpu_flush_commands() != 0; bool finish_ok = metal_graph_selected_async_load_finish(&async_load); - if (!finish_ok && async_load.ids_ok) { + if (!finish_ok && async_load.ids_ok && + !async_load.fail_closed && + !cuda_selected_event_required && + async_load.upload_event_value == 0) { /* The worker read valid ids but could not stage the load * (it is not allowed to wait on in-flight cache entries). * This thread is, so retry the same load synchronously. */ +#if !defined(__APPLE__) && !defined(DS4_ROCM_BUILD) && !defined(DS4_NO_GPU) + if (cuda_selected_event_pipeline) { + ds4_gpu_cuda_stream_selected_event_note_fallback(); + } +#endif const ds4_gpu_stream_expert_table retry_table = graph_stream_expert_table_make(model, layer, @@ -26027,6 +26182,29 @@ static bool metal_graph_encode_decode_layer_phase( async_load.selected_ids, DS4_N_EXPERT_USED) != 0; } +#if !defined(__APPLE__) && !defined(DS4_ROCM_BUILD) && !defined(DS4_NO_GPU) + if (cuda_selected_event_pipeline && !finish_ok) { + ds4_gpu_cuda_stream_selected_event_note_failure( + cuda_selected_event_required ? 1 : 0); + } + if (cuda_selected_event_pipeline && finish_ok && + async_load.upload_event_value != 0 && + ds4_gpu_stream_expert_cache_wait_selected_upload( + async_load.upload_event_value, + "selected expert upload") == 0) { + ds4_gpu_cuda_stream_selected_event_note_failure( + cuda_selected_event_required ? 1 : 0); + if (cuda_selected_event_required) { + finish_ok = false; + } else { + ds4_gpu_cuda_stream_selected_event_note_fallback(); + finish_ok = ds4_gpu_synchronize() != 0; + } + } + if (cuda_selected_event_pipeline && !finish_ok) { + (void)ds4_gpu_cuda_stream_selected_event_abort(); + } +#endif ok = ok && flush_ok && finish_ok; } else if (ok) { ok = ds4_gpu_commit_and_wait_selected_readback(selected_event, @@ -26053,6 +26231,11 @@ static bool metal_graph_encode_decode_layer_phase( DS4_N_EXPERT_USED) != 0; } } +#if !defined(__APPLE__) && !defined(DS4_ROCM_BUILD) && !defined(DS4_NO_GPU) + if (!ok && cuda_selected_event_pipeline) { + (void)ds4_gpu_cuda_stream_selected_event_abort(); + } +#endif if (ok) ok = ds4_gpu_routed_moe_one_tensor(metal_graph_routed_out(g), metal_graph_routed_gate(g), metal_graph_routed_up(g), @@ -46798,6 +46981,8 @@ static bool glm_graph_encode_sparse_ffn_one( l, il, selected_event, + false, + false, gate_out * gate_row_bytes, down_out * down_row_bytes); async_path_profiled = async_profile && async_load_started; diff --git a/ds4_cuda.cu b/ds4_cuda.cu index 7f300a410..ba43dbef8 100644 --- a/ds4_cuda.cu +++ b/ds4_cuda.cu @@ -173,12 +173,14 @@ typedef struct { } cuda_stream_selected_cache; static cuda_stream_selected_cache g_stream_selected_cache; +static void cuda_stream_selected_upload_drain(void); static void cuda_stream_selected_cache_invalidate(void) { g_stream_selected_cache.valid = 0; } static void cuda_stream_selected_cache_release(void) { + cuda_stream_selected_upload_drain(); const int tier = g_stream_selected_cache.logical_tier; if (tier >= 0 && tier < g_n_gpus) { (void)ds4_gpu_set_current_device(tier); @@ -524,6 +526,18 @@ static cudaEvent_t g_stream_selected_stage_event[4]; static uint64_t g_stream_selected_stage_bytes; static uint64_t g_stream_selected_stage_align = 1; static cudaStream_t g_stream_selected_upload_stream; +static int g_stream_selected_upload_owner_device = -1; +static int32_t *g_stream_selected_remap_stage; +static uint64_t g_stream_selected_remap_stage_capacity; +static cudaStream_t g_stream_selected_readback_stream; +static void *g_stream_selected_readback_stage; +static uint64_t g_stream_selected_readback_stage_capacity; +static cudaEvent_t g_stream_selected_compute_ready_event; +static cudaEvent_t g_stream_selected_readback_done_event; +static cudaEvent_t g_stream_selected_upload_done_event; +static int g_stream_selected_event_owner_device = -1; +static uint64_t g_stream_selected_compute_event_value; +static uint64_t g_stream_selected_upload_event_value; static std::once_flag g_stream_selected_batch_io_once; static int g_stream_selected_batch_io_enabled; static int g_stream_selected_batch_io_required; @@ -542,8 +556,26 @@ static uint64_t g_stream_selected_batch_io_tasks; static uint64_t g_stream_selected_batch_io_segments; static uint64_t g_stream_selected_batch_io_reads; static uint64_t g_stream_selected_batch_io_bytes; +static std::once_flag g_stream_selected_event_pipeline_once; +static int g_stream_selected_event_pipeline_enabled; +static int g_stream_selected_event_pipeline_required; +static int g_stream_selected_event_pipeline_oracle; +static int g_stream_selected_event_pipeline_report_registered; +static std::atomic g_stream_selected_event_candidates{0}; +static std::atomic g_stream_selected_event_signals{0}; +static std::atomic g_stream_selected_event_readbacks{0}; +static std::atomic g_stream_selected_event_uploads{0}; +static std::atomic g_stream_selected_event_compute_waits{0}; +static std::atomic g_stream_selected_event_safe_fallbacks{0}; +static std::atomic g_stream_selected_event_failures{0}; +static std::atomic g_stream_selected_event_required_failures{0}; +static std::atomic g_stream_selected_event_oracle_runs{0}; +static std::atomic g_stream_selected_event_oracle_failures{0}; static int cuda_ok(cudaError_t err, const char *what); +static void cuda_stream_selected_event_pipeline_release(void); +extern "C" int ds4_gpu_stream_expert_cache_wait_selected_upload( + uint64_t event_value, const char *label); extern "C" void ds4_gpu_decode_graphs_invalidate(void); static void cuda_decode_graphs_shutdown(void); static const char *cuda_model_range_ptr_from_fd( @@ -2678,7 +2710,30 @@ static int cuda_model_stage_read(void *stage, uint64_t stage_bytes, return cuda_pread_full(g_model_fd, stage, bytes, offset); } +static void cuda_stream_selected_upload_drain(void) { + if (!g_stream_selected_upload_stream) return; + int previous_device = -1; + (void)cudaGetDevice(&previous_device); + if (g_stream_selected_upload_owner_device >= 0) { + (void)cudaSetDevice(g_stream_selected_upload_owner_device); + } + (void)cudaStreamSynchronize(g_stream_selected_upload_stream); + if (previous_device >= 0) (void)cudaSetDevice(previous_device); +} + static void cuda_stream_selected_stage_release(void) { + const int had_upload_stream = g_stream_selected_upload_stream != NULL; + int previous_device = -1; + (void)cudaGetDevice(&previous_device); + if (g_stream_selected_upload_owner_device >= 0) { + (void)cudaSetDevice(g_stream_selected_upload_owner_device); + } + /* Once event publication is enabled, resize/teardown may arrive while + * the last H2D epoch is still in flight. Drain before destroying the + * ring events or freeing their pinned payloads. */ + if (g_stream_selected_upload_stream) { + (void)cudaStreamSynchronize(g_stream_selected_upload_stream); + } for (size_t i = 0; i < 4; i++) { if (g_stream_selected_stage_event[i]) { (void)cudaEventDestroy(g_stream_selected_stage_event[i]); @@ -2692,10 +2747,21 @@ static void cuda_stream_selected_stage_release(void) { } g_stream_selected_stage_bytes = 0; g_stream_selected_stage_align = 1; + if (g_stream_selected_remap_stage) { + (void)cudaFreeHost(g_stream_selected_remap_stage); + g_stream_selected_remap_stage = NULL; + } + g_stream_selected_remap_stage_capacity = 0; if (g_stream_selected_upload_stream) { (void)cudaStreamDestroy(g_stream_selected_upload_stream); g_stream_selected_upload_stream = NULL; } + g_stream_selected_upload_owner_device = -1; + if (had_upload_stream) { + uint64_t value = ++g_stream_selected_upload_event_value; + if (value == 0) ++g_stream_selected_upload_event_value; + } + if (previous_device >= 0) (void)cudaSetDevice(previous_device); } static int cuda_stream_selected_stage_pool_alloc(uint64_t bytes) { @@ -2718,6 +2784,11 @@ static int cuda_stream_selected_stage_pool_alloc(uint64_t bytes) { (void)cudaGetLastError(); return 0; } + if (cudaGetDevice(&g_stream_selected_upload_owner_device) != cudaSuccess) { + (void)cudaGetLastError(); + cuda_stream_selected_stage_release(); + return 0; + } for (size_t i = 0; i < 4; i++) { err = cudaMallocHost(&g_stream_selected_stage_raw[i], allocation_bytes); @@ -2753,6 +2824,39 @@ static int cuda_stream_selected_stage_pool_alloc(uint64_t bytes) { return 1; } +static int cuda_stream_selected_remap_stage_ensure(uint64_t count) { + if (count == 0 || count > SIZE_MAX / sizeof(int32_t)) return 0; + if (g_stream_selected_remap_stage && + g_stream_selected_remap_stage_capacity >= count) { + return 1; + } + /* The caller invokes this before submitting the new epoch. The pool + * release drains an older epoch before replacing its pinned storage. */ + if (g_stream_selected_remap_stage) { + if (g_stream_selected_upload_stream && + cudaStreamSynchronize(g_stream_selected_upload_stream) != + cudaSuccess) { + (void)cudaGetLastError(); + return 0; + } + (void)cudaFreeHost(g_stream_selected_remap_stage); + g_stream_selected_remap_stage = NULL; + g_stream_selected_remap_stage_capacity = 0; + } + const size_t bytes = (size_t)count * sizeof(int32_t); + cudaError_t err = cudaMallocHost( + (void **)&g_stream_selected_remap_stage, bytes); + if (err != cudaSuccess) { + fprintf(stderr, + "ds4: CUDA selected remap staging allocation failed: %s\n", + cudaGetErrorString(err)); + (void)cudaGetLastError(); + return 0; + } + g_stream_selected_remap_stage_capacity = count; + return 1; +} + typedef struct { char *dst; uint64_t offset; @@ -2815,6 +2919,11 @@ extern "C" int ds4_cuda_test_stream_selected_batch_env_value( return cuda_stream_selected_env_value_enabled(value); } +extern "C" int ds4_cuda_test_stream_selected_event_env_value( + const char *value) { + return cuda_stream_selected_env_value_enabled(value); +} + static void cuda_stream_selected_batch_io_resolve_policy( int enable, int disable, int require, int oracle, int *enabled_out, int *required_out, int *oracle_out) { @@ -2934,6 +3043,267 @@ extern "C" void ds4_cuda_stream_selected_batch_io_get_report( memcpy(report, &out, sizeof(out)); } +struct ds4_cuda_stream_selected_event_pipeline_report; +typedef struct ds4_cuda_stream_selected_event_pipeline_report + ds4_cuda_stream_selected_event_pipeline_report; +typedef struct { + uint64_t candidates; + uint64_t signals; + uint64_t readbacks; + uint64_t uploads; + uint64_t compute_waits; + uint64_t safe_fallbacks; + uint64_t failures; + uint64_t required_failures; + uint64_t oracle_runs; + uint64_t oracle_failures; + int enabled; + int required; + int oracle; +} cuda_stream_selected_event_pipeline_report_layout; + +static void cuda_stream_selected_event_pipeline_resolve_policy( + int enable, int disable, int require, int oracle, + int *enabled_out, int *required_out, int *oracle_out) { + const int disabled = disable != 0; + if (enabled_out) *enabled_out = !disabled && (enable || require || oracle); + if (required_out) *required_out = !disabled && require; + if (oracle_out) *oracle_out = !disabled && oracle; +} + +extern "C" int ds4_cuda_test_stream_selected_event_pipeline_policy( + int enable, int disable, int require, int oracle, + int *enabled_out, int *required_out, int *oracle_out) { + if (!enabled_out || !required_out || !oracle_out) return 0; + cuda_stream_selected_event_pipeline_resolve_policy( + enable, disable, require, oracle, + enabled_out, required_out, oracle_out); + return 1; +} + +static void cuda_stream_selected_event_pipeline_report_at_exit(void) { + fprintf(stderr, + "ds4: CUDA selected-expert event pipeline: candidates=%llu " + "signals=%llu readbacks=%llu uploads=%llu compute_waits=%llu " + "safe_fallbacks=%llu failures=%llu required_failures=%llu " + "oracle_runs=%llu oracle_failures=%llu\n", + (unsigned long long)g_stream_selected_event_candidates.load(), + (unsigned long long)g_stream_selected_event_signals.load(), + (unsigned long long)g_stream_selected_event_readbacks.load(), + (unsigned long long)g_stream_selected_event_uploads.load(), + (unsigned long long)g_stream_selected_event_compute_waits.load(), + (unsigned long long)g_stream_selected_event_safe_fallbacks.load(), + (unsigned long long)g_stream_selected_event_failures.load(), + (unsigned long long)g_stream_selected_event_required_failures.load(), + (unsigned long long)g_stream_selected_event_oracle_runs.load(), + (unsigned long long)g_stream_selected_event_oracle_failures.load()); +} + +static void cuda_stream_selected_event_pipeline_init(void) { + const int enable = cuda_stream_selected_env_flag( + "DS4_CUDA_ENABLE_STREAMING_SELECTED_EVENT_PIPELINE"); + const int disable = cuda_stream_selected_env_flag( + "DS4_CUDA_DISABLE_STREAMING_SELECTED_EVENT_PIPELINE") || + cuda_stream_selected_env_flag( + "DS4_CUDA_NO_STREAMING_SELECTED_EVENT_PIPELINE"); + const int require = cuda_stream_selected_env_flag( + "DS4_CUDA_REQUIRE_STREAMING_SELECTED_EVENT_PIPELINE"); + const int oracle = cuda_stream_selected_env_flag( + "DS4_CUDA_STREAMING_SELECTED_EVENT_PIPELINE_ORACLE"); + const int stats = cuda_stream_selected_env_flag( + "DS4_CUDA_STREAMING_SELECTED_EVENT_PIPELINE_STATS"); + cuda_stream_selected_event_pipeline_resolve_policy( + enable, disable, require, oracle, + &g_stream_selected_event_pipeline_enabled, + &g_stream_selected_event_pipeline_required, + &g_stream_selected_event_pipeline_oracle); + if ((enable || require || oracle || stats) && + !g_stream_selected_event_pipeline_report_registered) { + g_stream_selected_event_pipeline_report_registered = 1; + (void)atexit(cuda_stream_selected_event_pipeline_report_at_exit); + } + if (g_stream_selected_event_pipeline_enabled) { + fprintf(stderr, + "ds4: CUDA selected-expert event pipeline enabled%s%s\n", + g_stream_selected_event_pipeline_required ? " (required)" : "", + g_stream_selected_event_pipeline_oracle ? + " with synchronous oracle" : ""); + } else if (disable && (enable || require || oracle)) { + fprintf(stderr, + "ds4: CUDA selected-expert event pipeline disabled by " + "rollback override\n"); + } +} + +extern "C" int ds4_gpu_cuda_stream_selected_event_pipeline_enabled(void) { + std::call_once(g_stream_selected_event_pipeline_once, + cuda_stream_selected_event_pipeline_init); + return g_stream_selected_event_pipeline_enabled; +} + +extern "C" int ds4_gpu_cuda_stream_selected_event_pipeline_required(void) { + std::call_once(g_stream_selected_event_pipeline_once, + cuda_stream_selected_event_pipeline_init); + return g_stream_selected_event_pipeline_required; +} + +static int cuda_stream_selected_event_pipeline_oracle_requested(void) { + std::call_once(g_stream_selected_event_pipeline_once, + cuda_stream_selected_event_pipeline_init); + return g_stream_selected_event_pipeline_oracle; +} + +extern "C" void ds4_gpu_cuda_stream_selected_event_note_candidate(void) { + g_stream_selected_event_candidates.fetch_add(1, std::memory_order_relaxed); +} + +extern "C" void ds4_gpu_cuda_stream_selected_event_note_fallback(void) { + g_stream_selected_event_safe_fallbacks.fetch_add( + 1, std::memory_order_relaxed); +} + +extern "C" void ds4_gpu_cuda_stream_selected_event_note_failure( + int required) { + g_stream_selected_event_failures.fetch_add(1, std::memory_order_relaxed); + if (required) { + g_stream_selected_event_required_failures.fetch_add( + 1, std::memory_order_relaxed); + } +} + +extern "C" void ds4_cuda_stream_selected_event_pipeline_get_report( + ds4_cuda_stream_selected_event_pipeline_report *report) { + if (!report) return; + cuda_stream_selected_event_pipeline_report_layout out = {}; + out.candidates = g_stream_selected_event_candidates.load(); + out.signals = g_stream_selected_event_signals.load(); + out.readbacks = g_stream_selected_event_readbacks.load(); + out.uploads = g_stream_selected_event_uploads.load(); + out.compute_waits = g_stream_selected_event_compute_waits.load(); + out.safe_fallbacks = g_stream_selected_event_safe_fallbacks.load(); + out.failures = g_stream_selected_event_failures.load(); + out.required_failures = g_stream_selected_event_required_failures.load(); + out.oracle_runs = g_stream_selected_event_oracle_runs.load(); + out.oracle_failures = g_stream_selected_event_oracle_failures.load(); + out.enabled = ds4_gpu_cuda_stream_selected_event_pipeline_enabled(); + out.required = ds4_gpu_cuda_stream_selected_event_pipeline_required(); + out.oracle = cuda_stream_selected_event_pipeline_oracle_requested(); + memcpy(report, &out, sizeof(out)); +} + +static void cuda_stream_selected_event_pipeline_release(void) { + int previous_device = -1; + (void)cudaGetDevice(&previous_device); + if (g_stream_selected_event_owner_device >= 0) { + (void)cudaSetDevice(g_stream_selected_event_owner_device); + } + if (g_stream_selected_upload_stream) { + (void)cudaStreamSynchronize(g_stream_selected_upload_stream); + } + if (g_stream_selected_readback_stream) { + (void)cudaStreamSynchronize(g_stream_selected_readback_stream); + (void)cudaStreamDestroy(g_stream_selected_readback_stream); + g_stream_selected_readback_stream = NULL; + } + if (g_stream_selected_readback_stage) { + (void)cudaFreeHost(g_stream_selected_readback_stage); + g_stream_selected_readback_stage = NULL; + } + g_stream_selected_readback_stage_capacity = 0; + if (g_stream_selected_compute_ready_event) { + (void)cudaEventDestroy(g_stream_selected_compute_ready_event); + g_stream_selected_compute_ready_event = NULL; + } + if (g_stream_selected_readback_done_event) { + (void)cudaEventDestroy(g_stream_selected_readback_done_event); + g_stream_selected_readback_done_event = NULL; + } + if (g_stream_selected_upload_done_event) { + (void)cudaEventDestroy(g_stream_selected_upload_done_event); + g_stream_selected_upload_done_event = NULL; + } + g_stream_selected_event_owner_device = -1; + uint64_t compute_value = ++g_stream_selected_compute_event_value; + if (compute_value == 0) ++g_stream_selected_compute_event_value; + uint64_t upload_value = ++g_stream_selected_upload_event_value; + if (upload_value == 0) ++g_stream_selected_upload_event_value; + if (previous_device >= 0) (void)cudaSetDevice(previous_device); +} + +static int cuda_stream_selected_event_pipeline_ensure(void) { + if (g_n_gpus != 1) return 0; + int device = -1; + if (cudaGetDevice(&device) != cudaSuccess) { + (void)cudaGetLastError(); + return 0; + } + if (g_stream_selected_event_owner_device >= 0 && + g_stream_selected_event_owner_device != device) { + cuda_stream_selected_event_pipeline_release(); + } + if (g_stream_selected_readback_stream && + g_stream_selected_compute_ready_event && + g_stream_selected_readback_done_event && + g_stream_selected_upload_done_event) { + return 1; + } + cuda_stream_selected_event_pipeline_release(); + g_stream_selected_event_owner_device = device; + cudaError_t err = cudaStreamCreateWithFlags( + &g_stream_selected_readback_stream, cudaStreamNonBlocking); + if (err == cudaSuccess) { + err = cudaEventCreateWithFlags( + &g_stream_selected_compute_ready_event, cudaEventDisableTiming); + } + if (err == cudaSuccess) { + err = cudaEventCreateWithFlags( + &g_stream_selected_readback_done_event, cudaEventDisableTiming); + } + if (err == cudaSuccess) { + err = cudaEventCreateWithFlags( + &g_stream_selected_upload_done_event, cudaEventDisableTiming); + } + if (err != cudaSuccess) { + fprintf(stderr, + "ds4: CUDA selected event resource creation failed: %s\n", + cudaGetErrorString(err)); + (void)cudaGetLastError(); + cuda_stream_selected_event_pipeline_release(); + return 0; + } + return 1; +} + +static int cuda_stream_selected_readback_stage_ensure(uint64_t bytes) { + if (bytes == 0 || bytes > SIZE_MAX) return 0; + if (g_stream_selected_readback_stage && + g_stream_selected_readback_stage_capacity >= bytes) { + return 1; + } + if (g_stream_selected_readback_stream && + cudaStreamSynchronize(g_stream_selected_readback_stream) != + cudaSuccess) { + (void)cudaGetLastError(); + return 0; + } + if (g_stream_selected_readback_stage) { + (void)cudaFreeHost(g_stream_selected_readback_stage); + g_stream_selected_readback_stage = NULL; + g_stream_selected_readback_stage_capacity = 0; + } + cudaError_t err = cudaMallocHost( + &g_stream_selected_readback_stage, (size_t)bytes); + if (err != cudaSuccess) { + fprintf(stderr, + "ds4: CUDA selected readback staging allocation failed: %s\n", + cudaGetErrorString(err)); + (void)cudaGetLastError(); + return 0; + } + g_stream_selected_readback_stage_capacity = bytes; + return 1; +} + /* Build a deterministic source-ordered plan without changing compact-slot * order. Adjacent source spans can share one bounded read even when their * device destinations are discontiguous. Overlaps are rejected because the @@ -3144,10 +3514,11 @@ static int cuda_stream_selected_copy_oracle( } /* Queue every unique gate/up/down span plus the selected-id remap into one - * upload epoch. The ring events protect staging reuse; the sole final stream - * synchronization publishes the complete compact table atomically to the - * caller. submitted_out distinguishes safe pre-enqueue fallback from a - * partial upload, which must always fail closed. */ + * upload epoch. The ring events protect staging reuse. Ordinary callers + * retain the final stream synchronization; the decode worker may instead + * request an upload-done event which the compute stream consumes before the + * routed kernels. submitted_out distinguishes safe pre-enqueue fallback + * from a partial upload, which must always fail closed. */ static int cuda_model_copy_tasks_to_device_streamed( const std::vector &tasks, const void *model_map, @@ -3158,8 +3529,10 @@ static int cuda_model_copy_tasks_to_device_streamed( int run_oracle, uint64_t chunk_override, int *submitted_out, + uint64_t *upload_event_out, const char *what) { if (submitted_out) *submitted_out = 0; + if (upload_event_out) *upload_event_out = 0; if (!model_map || tasks.empty() || !remap_dst || !remap_src || remap_count == 0 || remap_count > SIZE_MAX / sizeof(int32_t)) { return 0; @@ -3178,6 +3551,11 @@ static int cuda_model_copy_tasks_to_device_streamed( return 0; } + /* Both diagnostic oracles deliberately retain a synchronous publication + * boundary. This keeps their reference observation deterministic and + * makes the ordinary event path the only mode which returns a token. */ + const int async_publish = upload_event_out != NULL && !run_oracle && + !cuda_stream_selected_event_pipeline_oracle_requested(); uint64_t stage_bytes = 0; if (!cuda_host_stage_bytes_for_chunk(chunk, g_model_direct_align, &stage_bytes) || @@ -3185,6 +3563,21 @@ static int cuda_model_copy_tasks_to_device_streamed( return 0; } + /* Pool allocation is first: its cold-start/re-size path releases every + * selected-upload staging allocation, including the persistent remap. + * Only publish/copy the remap after that destructive boundary. */ + if (async_publish && + (!cuda_stream_selected_event_pipeline_ensure() || + !cuda_stream_selected_remap_stage_ensure(remap_count))) { + return 0; + } + const int32_t *remap_upload_src = remap_src; + if (async_publish) { + memcpy(g_stream_selected_remap_stage, remap_src, + (size_t)remap_count * sizeof(remap_src[0])); + remap_upload_src = g_stream_selected_remap_stage; + } + const bool profile = cuda_stream_selected_env_flag( "DS4_CUDA_STREAMING_SELECTED_BATCH_IO_PROFILE"); const double t0 = profile ? cuda_wall_sec() : 0.0; @@ -3296,9 +3689,9 @@ static int cuda_model_copy_tasks_to_device_streamed( } const size_t remap_bytes = - (size_t)remap_count * sizeof(remap_src[0]); + (size_t)remap_count * sizeof(remap_upload_src[0]); cudaError_t err = cudaMemcpyAsync( - remap_dst, remap_src, remap_bytes, + remap_dst, remap_upload_src, remap_bytes, cudaMemcpyHostToDevice, g_stream_selected_upload_stream); if (err != cudaSuccess) { fprintf(stderr, @@ -3309,16 +3702,30 @@ static int cuda_model_copy_tasks_to_device_streamed( } if (submitted_out) *submitted_out = 1; - /* Async handoff seam: a future boundary-only change can replace this - * wait with an upload-done event recorded here and waited by the compute - * stream. All weight scatters and the remap are ordered before it. */ - err = cudaStreamSynchronize(g_stream_selected_upload_stream); - if (err != cudaSuccess) { - fprintf(stderr, - "ds4: CUDA selected batched I/O final sync failed for %s: %s\n", - what ? what : "expert batch", cudaGetErrorString(err)); - (void)cudaGetLastError(); - return 0; + if (async_publish) { + err = cudaEventRecord(g_stream_selected_upload_done_event, + g_stream_selected_upload_stream); + if (err != cudaSuccess) { + fprintf(stderr, + "ds4: CUDA selected upload event record failed for %s: %s\n", + what ? what : "expert batch", cudaGetErrorString(err)); + (void)cudaGetLastError(); + return cuda_stream_selected_upload_fail(what); + } + uint64_t value = ++g_stream_selected_upload_event_value; + if (value == 0) value = ++g_stream_selected_upload_event_value; + *upload_event_out = value; + g_stream_selected_event_uploads.fetch_add( + 1, std::memory_order_relaxed); + } else { + err = cudaStreamSynchronize(g_stream_selected_upload_stream); + if (err != cudaSuccess) { + fprintf(stderr, + "ds4: CUDA selected batched I/O final sync failed for %s: %s\n", + what ? what : "expert batch", cudaGetErrorString(err)); + (void)cudaGetLastError(); + return 0; + } } if (run_oracle) { @@ -3551,13 +3958,36 @@ extern "C" int ds4_cuda_test_stream_selected_batch_copy(void) { { dst[1], 0u, 4096u, 1u }, { dst[2], 4096u, 4096u, 2u }, }; + /* Force the production cold-start order. stage_pool_alloc() + * must run before the persistent remap is allocated/copied; the + * inverse order used to free the remap and enqueue from a stale + * host pointer on the first async batch. */ + cuda_stream_selected_stage_release(); int submitted = 0; + uint64_t cold_upload_event = 0; ok = cuda_model_copy_tasks_to_device_streamed( + tasks, map, model_bytes, + remap_dst, remap_src, 6u, + /*run_oracle=*/0, + /*chunk_override=*/12288u, + &submitted, &cold_upload_event, + "selected batch cold-start event selftest") && + submitted && + ds4_gpu_stream_expert_cache_wait_selected_upload( + cold_upload_event, + "selected batch cold-start event selftest") && + cudaStreamSynchronize(cuda_decode_stream()) == cudaSuccess && + cuda_stream_selected_copy_oracle( + tasks, map, model_bytes, + remap_src, remap_dst, 6u, /*report=*/1); + submitted = 0; + ok = ok && cuda_model_copy_tasks_to_device_streamed( tasks, map, model_bytes, remap_dst, remap_src, 6u, /*run_oracle=*/1, /*chunk_override=*/12288u, - &submitted, "selected batch selftest") && submitted; + &submitted, /*upload_event_out=*/NULL, + "selected batch selftest") && submitted; if (ok) { size_t selected_allocation = 0; ok = cuda_host_stage_allocation_bytes( @@ -3599,6 +4029,7 @@ extern "C" int ds4_cuda_test_stream_selected_batch_copy(void) { /*run_oracle=*/1, /*chunk_override=*/4096u, &submitted, + /*upload_event_out=*/NULL, "selected batch ring-wrap selftest") && submitted; for (uint32_t i = 0; ok && i < 4u; i++) { @@ -4311,8 +4742,12 @@ extern "C" void ds4_gpu_cleanup(void) { } } } - cuda_stream_selected_cache_release(); + /* The selected cache may still be the destination of an event-published + * upload. Drain and retire its auxiliary streams before freeing device + * storage or invalidating the owner-device context. */ cuda_stream_selected_stage_release(); + cuda_stream_selected_event_pipeline_release(); + cuda_stream_selected_cache_release(); ds4_mmq_set_gb10_optimizations(0); g_n_gpus = 0; g_cublas_ready = 0; @@ -29491,10 +29926,14 @@ static int cuda_stream_selected_ranges_valid( down_bytes <= table->model_size - table->down_offset; } -static int cuda_stream_selected_cache_begin_load( +static int cuda_stream_selected_cache_begin_load_impl( const ds4_gpu_stream_expert_table *table, const int32_t *selected_ids, - uint32_t slot_count) { + uint32_t slot_count, + uint64_t *upload_event_out, + int *submitted_any_out) { + if (upload_event_out) *upload_event_out = 0; + if (submitted_any_out) *submitted_any_out = 0; cuda_stream_selected_cache_invalidate(); if (!g_ssd_streaming_mode) return 1; if (!cuda_stream_selected_ranges_valid(table) || !selected_ids || @@ -29506,6 +29945,17 @@ static int cuda_stream_selected_cache_begin_load( "ds4: CUDA SSD streaming requires single-GPU placement\n"); return 0; } + /* This function also runs on the selected-load service thread. CUDA's + * current device is thread-local, while g_current_logical_tier is a + * process-global launch cache and may already say tier 0 from the main + * thread. Always select the physical owner explicitly here. */ + if (cudaSetDevice(g_gpu[0].device_id) != cudaSuccess) { + fprintf(stderr, + "ds4: CUDA selected cache could not select owner device %d\n", + g_gpu[0].device_id); + (void)cudaGetLastError(); + return 0; + } std::vector expert_to_slot; std::vector compact_ids; @@ -29573,6 +30023,12 @@ static int cuda_stream_selected_cache_begin_load( const int batch_io_required = cuda_stream_selected_batch_io_require_requested(); int copied = 0; + if (upload_event_out && !batch_io) { + /* The explicit async contract never falls into the per-expert + * synchronous loader. Its caller may retry through the legacy API + * only after learning that no upload was submitted. */ + return 0; + } if (batch_io) { std::vector tasks; int task_plan_ready = 1; @@ -29622,7 +30078,9 @@ static int cuda_stream_selected_cache_begin_load( slot_ids.data(), slot_count, cuda_stream_selected_batch_io_oracle_requested(), /*chunk_override=*/0, &submitted, + upload_event_out, "selected expert batch"); + if (submitted_any_out) *submitted_any_out = submitted; if (copied) { g_stream_selected_batch_io_completed++; } else { @@ -29630,7 +30088,7 @@ static int cuda_stream_selected_cache_begin_load( if (batch_io_required) { g_stream_selected_batch_io_required_failures++; } - if (submitted || batch_io_required) { + if (submitted || batch_io_required || upload_event_out) { cuda_stream_selected_cache_invalidate(); return 0; } @@ -29640,9 +30098,13 @@ static int cuda_stream_selected_cache_begin_load( g_stream_selected_batch_io_failures++; if (batch_io_required) { g_stream_selected_batch_io_required_failures++; - fprintf(stderr, - "ds4: required CUDA selected batched I/O could not " - "allocate its copy plan\n"); + } + if (batch_io_required || upload_event_out) { + if (batch_io_required) { + fprintf(stderr, + "ds4: required CUDA selected batched I/O could " + "not allocate its copy plan\n"); + } cuda_stream_selected_cache_invalidate(); return 0; } @@ -29650,6 +30112,10 @@ static int cuda_stream_selected_cache_begin_load( } } + if (!copied && upload_event_out) { + cuda_stream_selected_cache_invalidate(); + return 0; + } if (!copied) { g_stream_selected_batch_io_legacy++; for (uint32_t i = 0; i < compact_ids.size(); i++) { @@ -29714,6 +30180,14 @@ static int cuda_stream_selected_cache_begin_load( return 1; } +static int cuda_stream_selected_cache_begin_load( + const ds4_gpu_stream_expert_table *table, + const int32_t *selected_ids, + uint32_t slot_count) { + return cuda_stream_selected_cache_begin_load_impl( + table, selected_ids, slot_count, NULL, NULL); +} + __device__ __forceinline__ static float glm_rope_yarn_corr_factor_dev( int n_dims, int n_ctx_orig, float n_rot, float base) { return n_dims * logf(n_ctx_orig / (n_rot * 2.0f * (float)M_PI)) / @@ -36484,6 +36958,92 @@ extern "C" int ds4_gpu_signal_selected_readback_ready(uint64_t *event_value) { return cuda_ok(cudaDeviceSynchronize(), "selected readback signal"); } +extern "C" int ds4_gpu_cuda_stream_selected_set_owner_device(void) { + if (g_n_gpus != 1) return 0; + const int owner = g_stream_selected_event_owner_device >= 0 ? + g_stream_selected_event_owner_device : g_gpu[0].device_id; + if (owner != g_gpu[0].device_id) return 0; + const cudaError_t err = cudaSetDevice(owner); + if (err != cudaSuccess) { + fprintf(stderr, + "ds4: CUDA selected event owner-device switch to %d failed: %s\n", + owner, cudaGetErrorString(err)); + (void)cudaGetLastError(); + return 0; + } + return 1; +} + +extern "C" int ds4_cuda_test_stream_selected_owner_device(void) { + if (g_n_gpus != 1) return 0; + int saved_device = -1; + int device_count = 0; + if (cudaGetDevice(&saved_device) != cudaSuccess || + cudaGetDeviceCount(&device_count) != cudaSuccess) { + (void)cudaGetLastError(); + return 0; + } + const int saved_logical_tier = g_current_logical_tier; + const int owner = g_gpu[0].device_id; + int wrong_device = owner; + for (int device = 0; device < device_count; device++) { + if (device != owner) { + wrong_device = device; + break; + } + } + /* Reproduce the service-thread hazard: the process-global logical cache + * claims tier 0 even though this thread is current on another physical + * device. The owner setter must still issue cudaSetDevice(owner). */ + g_current_logical_tier = 0; + int ok = cudaSetDevice(wrong_device) == cudaSuccess && + ds4_gpu_cuda_stream_selected_set_owner_device() != 0; + int got_device = -1; + if (ok) ok = cudaGetDevice(&got_device) == cudaSuccess && + got_device == owner; + if (saved_device >= 0) (void)cudaSetDevice(saved_device); + g_current_logical_tier = saved_logical_tier; + if (!ok) (void)cudaGetLastError(); + return ok; +} + +/* Event-only boundary used by the DeepSeek SSD selected-expert worker. It is + * intentionally separate from the compatibility API above: callers which do + * not opt in retain its full-device synchronization semantics. */ +extern "C" int ds4_gpu_signal_selected_readback_ready_async( + uint64_t *event_value) { + if (event_value) *event_value = 0; + if (!event_value || + !ds4_gpu_cuda_stream_selected_event_pipeline_enabled() || + g_n_gpus != 1 || + !ds4_gpu_cuda_stream_selected_set_owner_device() || + !cuda_stream_selected_event_pipeline_ensure()) { + (void)cudaGetLastError(); + return 0; + } + + const cudaStream_t stream = cuda_decode_stream(); + cudaStreamCaptureStatus capture_status = cudaStreamCaptureStatusNone; + cudaError_t err = cudaStreamIsCapturing(stream, &capture_status); + if (err != cudaSuccess || capture_status != cudaStreamCaptureStatusNone) { + if (err != cudaSuccess) (void)cudaGetLastError(); + return 0; + } + err = cudaEventRecord(g_stream_selected_compute_ready_event, stream); + if (err != cudaSuccess) { + fprintf(stderr, + "ds4: CUDA selected compute-ready event record failed: %s\n", + cudaGetErrorString(err)); + (void)cudaGetLastError(); + return 0; + } + uint64_t value = ++g_stream_selected_compute_event_value; + if (value == 0) value = ++g_stream_selected_compute_event_value; + *event_value = value; + g_stream_selected_event_signals.fetch_add(1, std::memory_order_relaxed); + return 1; +} + extern "C" int ds4_gpu_stream_expert_cache_begin_selected_load( const ds4_gpu_stream_expert_table *table, const int32_t *selected_ids, @@ -36492,6 +37052,29 @@ extern "C" int ds4_gpu_stream_expert_cache_begin_selected_load( n_selected); } +/* Tri-state result: 1 published (event_value may be zero for an oracle), + * 0 failed before enqueue and is safe for the caller's legacy retry, -1 + * failed after enqueue and must fail closed. */ +extern "C" int ds4_gpu_stream_expert_cache_begin_selected_load_async( + const ds4_gpu_stream_expert_table *table, + const int32_t *selected_ids, + uint32_t n_selected, + uint64_t *upload_event_value) { + if (upload_event_value) *upload_event_value = 0; + if (!upload_event_value || + !ds4_gpu_cuda_stream_selected_event_pipeline_enabled() || + g_n_gpus != 1 || + !ds4_gpu_cuda_stream_selected_set_owner_device()) { + (void)cudaGetLastError(); + return 0; + } + int submitted = 0; + const int ok = cuda_stream_selected_cache_begin_load_impl( + table, selected_ids, n_selected, upload_event_value, &submitted); + if (ok) return 1; + return submitted ? -1 : 0; +} + extern "C" uint32_t ds4_gpu_stream_expert_cache_budget_for_expert_size( uint64_t gate_expert_bytes, uint64_t down_expert_bytes) { @@ -36533,18 +37116,88 @@ extern "C" int ds4_gpu_tensor_read_after_selected_event(const ds4_gpu_tensor *te uint64_t bytes, uint64_t event_value, const char *label) { - (void)event_value; if (!tensor || !data || offset > tensor->bytes || - bytes > tensor->bytes - offset) { + bytes > tensor->bytes - offset || bytes > SIZE_MAX) { + return 0; + } + const int tier = ds4_tensor_device_idx(tensor); + if (event_value == 0 || + event_value != g_stream_selected_compute_event_value || + tier < 0 || tier >= g_n_gpus || + g_stream_selected_event_owner_device != g_gpu[tier].device_id || + !g_stream_selected_readback_stream || + !g_stream_selected_compute_ready_event || + !g_stream_selected_readback_done_event || + cudaSetDevice(g_stream_selected_event_owner_device) != cudaSuccess || + (bytes != 0 && !cuda_stream_selected_readback_stage_ensure(bytes))) { + (void)cudaGetLastError(); return 0; } - if (!cuda_ok(cudaDeviceSynchronize(), - label ? label : "selected readback wait")) { + + cudaError_t err = cudaStreamWaitEvent( + g_stream_selected_readback_stream, + g_stream_selected_compute_ready_event, 0); + if (err == cudaSuccess && bytes != 0) { + err = cudaMemcpyAsync( + g_stream_selected_readback_stage, + (const char *)tensor->ptr + offset, + (size_t)bytes, + cudaMemcpyDeviceToHost, + g_stream_selected_readback_stream); + } + if (err == cudaSuccess) { + err = cudaEventRecord(g_stream_selected_readback_done_event, + g_stream_selected_readback_stream); + } + if (err == cudaSuccess) { + err = cudaEventSynchronize(g_stream_selected_readback_done_event); + } + if (err != cudaSuccess) { + fprintf(stderr, + "ds4: CUDA %s failed: %s\n", + label ? label : "selected event readback", + cudaGetErrorString(err)); + (void)cudaGetLastError(); return 0; } - return cuda_ok(cudaMemcpy(data, (const char *)tensor->ptr + offset, - (size_t)bytes, cudaMemcpyDeviceToHost), - "selected tensor read"); + if (bytes != 0) { + memcpy(data, g_stream_selected_readback_stage, (size_t)bytes); + } + g_stream_selected_event_readbacks.fetch_add( + 1, std::memory_order_relaxed); + + if (cuda_stream_selected_event_pipeline_oracle_requested()) { + g_stream_selected_event_oracle_runs.fetch_add( + 1, std::memory_order_relaxed); + std::vector reference; + try { + reference.resize((size_t)bytes); + } catch (...) { + g_stream_selected_event_oracle_failures.fetch_add( + 1, std::memory_order_relaxed); + return 0; + } + err = cudaDeviceSynchronize(); + if (err == cudaSuccess && bytes != 0) { + err = cudaMemcpy(reference.data(), + (const char *)tensor->ptr + offset, + (size_t)bytes, + cudaMemcpyDeviceToHost); + } + if (err != cudaSuccess || + (bytes != 0 && memcmp(reference.data(), data, + (size_t)bytes) != 0)) { + fprintf(stderr, + "ds4: CUDA selected event readback oracle %s\n", + err == cudaSuccess ? "mismatch" : + cudaGetErrorString(err)); + if (err != cudaSuccess) (void)cudaGetLastError(); + g_stream_selected_event_oracle_failures.fetch_add( + 1, std::memory_order_relaxed); + return 0; + } + } + return 1; } extern "C" int ds4_gpu_tp_big_gate_encode(uint32_t layer, uint32_t rows, @@ -36570,6 +37223,163 @@ extern "C" int ds4_gpu_wait_selected_readback_ready(uint64_t event_value, const label ? label : "selected readback wait"); } +extern "C" int ds4_gpu_stream_expert_cache_wait_selected_upload( + uint64_t event_value, const char *label) { + if (event_value == 0) return 1; + if (event_value != g_stream_selected_upload_event_value || + !g_stream_selected_upload_done_event || + g_stream_selected_event_owner_device < 0 || + g_n_gpus != 1 || + g_stream_selected_event_owner_device != g_gpu[0].device_id || + g_stream_selected_upload_owner_device != + g_stream_selected_event_owner_device || + cudaSetDevice(g_stream_selected_event_owner_device) != cudaSuccess) { + (void)cudaGetLastError(); + return 0; + } + const cudaError_t err = cudaStreamWaitEvent( + cuda_decode_stream(), g_stream_selected_upload_done_event, 0); + if (err != cudaSuccess) { + fprintf(stderr, + "ds4: CUDA %s failed: %s\n", + label ? label : "selected upload wait", + cudaGetErrorString(err)); + (void)cudaGetLastError(); + return 0; + } + g_stream_selected_event_compute_waits.fetch_add( + 1, std::memory_order_relaxed); + return 1; +} + +extern "C" int ds4_gpu_cuda_stream_selected_event_abort(void) { + int device = g_stream_selected_event_owner_device; + if (device < 0 && g_n_gpus == 1) device = g_gpu[0].device_id; + if (device >= 0 && cudaSetDevice(device) != cudaSuccess) { + (void)cudaGetLastError(); + cuda_stream_selected_cache_invalidate(); + return 0; + } + const cudaError_t err = cudaDeviceSynchronize(); + cuda_stream_selected_cache_invalidate(); + uint64_t compute_value = ++g_stream_selected_compute_event_value; + if (compute_value == 0) ++g_stream_selected_compute_event_value; + uint64_t upload_value = ++g_stream_selected_upload_event_value; + if (upload_value == 0) ++g_stream_selected_upload_event_value; + if (err != cudaSuccess) { + fprintf(stderr, + "ds4: CUDA selected event abort sync failed: %s\n", + cudaGetErrorString(err)); + (void)cudaGetLastError(); + return 0; + } + return 1; +} + +/* Device-backed ordering oracle for CI/DGX bring-up. It exercises the same + * event objects and streams as decode without requiring a GGUF or mutating + * process-environment policy after its once_flag has resolved. */ +extern "C" int ds4_cuda_test_stream_selected_event_pipeline(void) { + if (g_n_gpus != 1 || + cudaSetDevice(g_gpu[0].device_id) != cudaSuccess || + !cuda_stream_selected_event_pipeline_ensure() || + !cuda_stream_selected_stage_pool_alloc(4096u) || + !cuda_stream_selected_remap_stage_ensure(1u)) { + (void)cudaGetLastError(); + return 0; + } + + int32_t *source = NULL; + int32_t *uploaded = NULL; + int32_t *consumed = NULL; + int32_t *host = NULL; + int ok = cudaMalloc((void **)&source, sizeof(*source)) == cudaSuccess && + cudaMalloc((void **)&uploaded, sizeof(*uploaded)) == cudaSuccess && + cudaMalloc((void **)&consumed, sizeof(*consumed)) == cudaSuccess && + cudaMallocHost((void **)&host, 2u * sizeof(*host)) == cudaSuccess; + if (!ok) (void)cudaGetLastError(); + + ds4_gpu_tensor tensor = {}; + tensor.ptr = source; + tensor.bytes = sizeof(*source); + tensor.owner = 0; + tensor.device_id = 0; + if (ok) { + host[0] = 0x13572468; + host[1] = 0x24681357; + ok = cudaMemcpyAsync(source, &host[0], sizeof(host[0]), + cudaMemcpyHostToDevice, + cuda_decode_stream()) == cudaSuccess && + cudaEventRecord(g_stream_selected_compute_ready_event, + cuda_decode_stream()) == cudaSuccess; + } + uint64_t compute_value = 0; + if (ok) { + compute_value = ++g_stream_selected_compute_event_value; + if (compute_value == 0) { + compute_value = ++g_stream_selected_compute_event_value; + } + g_stream_selected_event_candidates.fetch_add( + 1, std::memory_order_relaxed); + g_stream_selected_event_signals.fetch_add( + 1, std::memory_order_relaxed); + int32_t got = 0; + ok = ds4_gpu_tensor_read_after_selected_event( + &tensor, 0, &got, sizeof(got), compute_value, + "selected event selftest readback") != 0 && + got == host[0]; + } + + uint64_t upload_value = 0; + if (ok) { + g_stream_selected_remap_stage[0] = host[1]; + ok = cudaMemcpyAsync(uploaded, + g_stream_selected_remap_stage, + sizeof(g_stream_selected_remap_stage[0]), + cudaMemcpyHostToDevice, + g_stream_selected_upload_stream) == cudaSuccess && + cudaEventRecord(g_stream_selected_upload_done_event, + g_stream_selected_upload_stream) == cudaSuccess; + } + if (ok) { + upload_value = ++g_stream_selected_upload_event_value; + if (upload_value == 0) { + upload_value = ++g_stream_selected_upload_event_value; + } + g_stream_selected_event_uploads.fetch_add( + 1, std::memory_order_relaxed); + ok = ds4_gpu_stream_expert_cache_wait_selected_upload( + upload_value, "selected event selftest upload") != 0 && + cudaMemcpyAsync(consumed, uploaded, sizeof(*consumed), + cudaMemcpyDeviceToDevice, + cuda_decode_stream()) == cudaSuccess && + cudaStreamSynchronize(cuda_decode_stream()) == cudaSuccess; + } + if (ok) { + int32_t got = 0; + ok = cudaMemcpy(&got, consumed, sizeof(got), + cudaMemcpyDeviceToHost) == cudaSuccess && + got == host[1]; + } + g_stream_selected_event_oracle_runs.fetch_add( + 1, std::memory_order_relaxed); + if (!ok) { + (void)cudaGetLastError(); + g_stream_selected_event_oracle_failures.fetch_add( + 1, std::memory_order_relaxed); + (void)cudaDeviceSynchronize(); + } + if (source) (void)cudaFree(source); + if (uploaded) (void)cudaFree(uploaded); + if (consumed) (void)cudaFree(consumed); + if (host) (void)cudaFreeHost(host); + /* Keep the compute/readback/upload-done event resources for the report, + * but restore the selected-upload pool to the same cold state expected by + * the production batch-copy selftest which runs next. */ + cuda_stream_selected_stage_release(); + return ok; +} + /* Compatibility surface shared with the canonical Metal/ROCm graph. CUDA * either delegates to its equivalent primitive or reports an unavailable * optional fast path so the graph can use its established fallback. */ diff --git a/ds4_gpu.h b/ds4_gpu.h index 732b51592..c2a4bed41 100644 --- a/ds4_gpu.h +++ b/ds4_gpu.h @@ -105,7 +105,8 @@ int ds4_gpu_parallel_ffn_start( int ds4_gpu_signal_selected_readback_ready(uint64_t *event_value); int ds4_gpu_commit_and_wait_selected_readback(uint64_t event_value, const char *label); int ds4_gpu_wait_selected_readback_ready(uint64_t event_value, const char *label); -#ifdef DS4_ROCM_BUILD +#if defined(DS4_ROCM_BUILD) || \ + (!defined(__APPLE__) && !defined(DS4_NO_GPU)) int ds4_gpu_tensor_read_after_selected_event(const ds4_gpu_tensor *tensor, uint64_t offset, void *data, @@ -304,6 +305,21 @@ typedef struct ds4_cuda_stream_selected_batch_io_report { int required; int oracle; } ds4_cuda_stream_selected_batch_io_report; +typedef struct ds4_cuda_stream_selected_event_pipeline_report { + uint64_t candidates; + uint64_t signals; + uint64_t readbacks; + uint64_t uploads; + uint64_t compute_waits; + uint64_t safe_fallbacks; + uint64_t failures; + uint64_t required_failures; + uint64_t oracle_runs; + uint64_t oracle_failures; + int enabled; + int required; + int oracle; +} ds4_cuda_stream_selected_event_pipeline_report; /* CUDA-only policy/planner/scatter hooks. The policy hook takes explicit * values so tests do not need to mutate process environment around once_flag * initialization. */ @@ -315,6 +331,24 @@ int ds4_cuda_test_stream_selected_batch_plan(void); int ds4_cuda_test_stream_selected_batch_copy(void); void ds4_cuda_stream_selected_batch_io_get_report( ds4_cuda_stream_selected_batch_io_report *report); +int ds4_cuda_test_stream_selected_event_pipeline_policy( + int enable, int disable, int require, int oracle, + int *enabled_out, int *required_out, int *oracle_out); +int ds4_cuda_test_stream_selected_event_env_value(const char *value); +int ds4_cuda_test_stream_selected_event_pipeline(void); +int ds4_cuda_test_stream_selected_owner_device(void); +void ds4_cuda_stream_selected_event_pipeline_get_report( + ds4_cuda_stream_selected_event_pipeline_report *report); +int ds4_gpu_cuda_stream_selected_event_pipeline_enabled(void); +int ds4_gpu_cuda_stream_selected_event_pipeline_required(void); +int ds4_gpu_cuda_stream_selected_set_owner_device(void); +void ds4_gpu_cuda_stream_selected_event_note_candidate(void); +void ds4_gpu_cuda_stream_selected_event_note_fallback(void); +void ds4_gpu_cuda_stream_selected_event_note_failure(int required); +int ds4_gpu_signal_selected_readback_ready_async(uint64_t *event_value); +int ds4_gpu_stream_expert_cache_wait_selected_upload( + uint64_t event_value, const char *label); +int ds4_gpu_cuda_stream_selected_event_abort(void); #endif void ds4_gpu_set_streaming_expert_cache_budget(uint32_t experts); void ds4_gpu_set_streaming_expert_cache_expert_bytes(uint64_t bytes); @@ -347,6 +381,15 @@ int ds4_gpu_stream_expert_cache_begin_selected_load( const ds4_gpu_stream_expert_table *table, const int32_t *selected_ids, uint32_t n_selected); +#if !defined(__APPLE__) && !defined(DS4_ROCM_BUILD) && !defined(DS4_NO_GPU) +/* Returns 1 on publication, 0 on a safe pre-enqueue rejection, and -1 on + * a post-enqueue failure which callers must not retry. */ +int ds4_gpu_stream_expert_cache_begin_selected_load_async( + const ds4_gpu_stream_expert_table *table, + const int32_t *selected_ids, + uint32_t n_selected, + uint64_t *upload_event_value); +#endif int ds4_gpu_glm_stream_expert_cache_begin_selected_load_tensor( const ds4_gpu_stream_expert_table *table, const ds4_gpu_tensor *selected, diff --git a/tests/test_gpu_model_cache.c b/tests/test_gpu_model_cache.c index 53cecf2ce..8e459b1ee 100644 --- a/tests/test_gpu_model_cache.c +++ b/tests/test_gpu_model_cache.c @@ -52,6 +52,31 @@ int main(void) { 1, 1, 1, 1, &enabled, &required, &oracle) && enabled == 0 && required == 0 && oracle == 0, "selected-expert batched-I/O disable-dominant policy"); + CHECK(!ds4_cuda_test_stream_selected_event_env_value(NULL) && + !ds4_cuda_test_stream_selected_event_env_value("") && + !ds4_cuda_test_stream_selected_event_env_value("0") && + !ds4_cuda_test_stream_selected_event_env_value("false") && + !ds4_cuda_test_stream_selected_event_env_value("NO") && + !ds4_cuda_test_stream_selected_event_env_value("off") && + ds4_cuda_test_stream_selected_event_env_value("1") && + ds4_cuda_test_stream_selected_event_env_value("true"), + "selected-expert event-pipeline value-aware environment parser"); + CHECK(ds4_cuda_test_stream_selected_event_pipeline_policy( + 1, 0, 0, 0, &enabled, &required, &oracle) && + enabled == 1 && required == 0 && oracle == 0, + "selected-expert event-pipeline enable policy"); + CHECK(ds4_cuda_test_stream_selected_event_pipeline_policy( + 0, 0, 1, 0, &enabled, &required, &oracle) && + enabled == 1 && required == 1 && oracle == 0, + "selected-expert event-pipeline require policy"); + CHECK(ds4_cuda_test_stream_selected_event_pipeline_policy( + 0, 0, 0, 1, &enabled, &required, &oracle) && + enabled == 1 && required == 0 && oracle == 1, + "selected-expert event-pipeline oracle policy"); + CHECK(ds4_cuda_test_stream_selected_event_pipeline_policy( + 1, 1, 1, 1, &enabled, &required, &oracle) && + enabled == 0 && required == 0 && oracle == 0, + "selected-expert event-pipeline disable-dominant policy"); CHECK(ds4_cuda_test_stream_selected_batch_plan(), "selected-expert batched-I/O planner"); @@ -65,6 +90,19 @@ int main(void) { } CHECK(ds4_gpu_init(), "ds4_gpu_init"); + CHECK(ds4_cuda_test_stream_selected_event_pipeline(), + "selected-expert compute/readback/upload event ordering oracle"); + ds4_cuda_stream_selected_event_pipeline_report event_report; + memset(&event_report, 0, sizeof(event_report)); + ds4_cuda_stream_selected_event_pipeline_get_report(&event_report); + CHECK(event_report.candidates >= 1 && + event_report.signals >= 1 && + event_report.readbacks >= 1 && + event_report.uploads >= 1 && + event_report.compute_waits >= 1 && + event_report.oracle_runs >= 1 && + event_report.oracle_failures == 0, + "selected-expert event-pipeline coverage counters"); CHECK(ds4_cuda_test_stream_selected_batch_copy(), "selected-expert batched-I/O scatter + byte oracle"); ds4_cuda_stream_selected_batch_io_report batch_report; From 95c78d4e87805f12be78d8b6c1675c0a8498e4d0 Mon Sep 17 00:00:00 2001 From: Giorgio Oppo Date: Fri, 14 Aug 2026 19:00:59 +0200 Subject: [PATCH 032/189] cuda: add raw fused IQ2 Q2 MoE MMQ path --- cuda/mmq/ds4_mmq.cu | 94 ++++++++ cuda/mmq/ds4_mmq.h | 30 +++ cuda/mmq/test/test_mmq_parity.cu | 365 +++++++++++++++++++++++++++++++ 3 files changed, 489 insertions(+) diff --git a/cuda/mmq/ds4_mmq.cu b/cuda/mmq/ds4_mmq.cu index 86b3974ef..ea5c02e8a 100644 --- a/cuda/mmq/ds4_mmq.cu +++ b/cuda/mmq/ds4_mmq.cu @@ -2229,6 +2229,100 @@ extern "C" int ds4_mmq_iq2_xxs_q2_K_moe_fused_direct_scratch_sizes( return 0; } +/* Canonical-GGUF/raw counterpart of the materialized aligned-SoA pipeline. + * SSD streaming compacts the selected experts and remaps ids before this + * boundary, so n_experts describes the compact table rather than the model's + * global expert count. Keep all preflight ahead of the single pair_impl + * invocation: after that point map/quantize/MMQ work may already be queued and + * a negative result must be propagated instead of being converted into the + * retryable NOT_APPLICABLE result. */ +extern "C" int ds4_mmq_iq2_xxs_q2_K_moe_fused_raw( + const void * W_gate, const void * W_up, const void * W_down, + const float * X, const int32_t * ids, const float * router_weights, + float * gate, float * up, float * mid_f32, float * down, + int expert_mid_dim, int expert_in_dim, int out_dim, + int n_tokens, int n_experts, int n_expert_used, + float clamp, cudaStream_t stream) { + if (!W_gate || !W_up || !W_down || !X || !ids || !router_weights || + !gate || !up || !mid_f32 || !down || + expert_mid_dim <= 0 || expert_in_dim <= 0 || out_dim <= 0 || + n_tokens <= 0 || n_experts <= 0 || n_expert_used <= 0 || + n_expert_used > n_experts || n_experts == INT_MAX || + n_tokens >= (1 << 22) || n_expert_used >= (1 << 10) || + expert_in_dim % 256 != 0 || expert_mid_dim % 256 != 0) { + return DS4_MMQ_NOT_APPLICABLE; + } + + const size_t nt = (size_t)n_tokens; + const size_t nu = (size_t)n_expert_used; + const size_t mid = (size_t)expert_mid_dim; + const size_t in = (size_t)expert_in_dim; + const size_t out = (size_t)out_dim; + if (nt > SIZE_MAX / nu) return DS4_MMQ_NOT_APPLICABLE; + const size_t assignments = nt * nu; + if (assignments > SIZE_MAX / mid || + assignments * mid > SIZE_MAX / sizeof(float) || + assignments > SIZE_MAX / out || + assignments * out > SIZE_MAX / sizeof(float) || + assignments > SIZE_MAX / in || + assignments * in > SIZE_MAX / 512u || + assignments * mid > SIZE_MAX / 512u) { + return DS4_MMQ_NOT_APPLICABLE; + } + + /* Bound the raw per-expert strides used by both MMQs before any pool or + * stream operation. All dimensions enter as int, but their products do + * not necessarily fit int64_t. */ + const int64_t iq2_k_blocks = expert_in_dim / 256; + const int64_t q2_k_blocks = expert_mid_dim / 256; + if ((int64_t)n_experts > INT64_MAX / expert_mid_dim || + (int64_t)n_experts * expert_mid_dim > INT64_MAX / iq2_k_blocks || + (int64_t)n_experts > INT64_MAX / out_dim || + (int64_t)n_experts * out_dim > INT64_MAX / q2_k_blocks) { + return DS4_MMQ_NOT_APPLICABLE; + } + + const int dev = ggml_cuda_get_device(); + if (dev < 0 || dev >= GGML_CUDA_MAX_DEVICES) { + return DS4_MMQ_NOT_APPLICABLE; + } + const int cc = ggml_cuda_info().devices[dev].cc; + if (!ds4_mmq_k_tile_supported( + "ds4_mmq_iq2_xxs_q2_K_moe_fused_raw", expert_in_dim, cc) || + !get_ctx_for_device(dev) || + ((size_t)n_tokens * 4u > ggml_cuda_info().devices[dev].smpbo && + !ds4_mmid_large_enabled())) { + return DS4_MMQ_NOT_APPLICABLE; + } + + const ds4_mmq_fused_down fused_down = { + W_down, + nullptr, + 0, + router_weights, + mid_f32, + down, + out_dim, + clamp, + false, + nullptr, + 0, + nullptr, + 0, + nullptr, + 0, + nullptr, + 0, + }; + return ds4_mmq_moe_pair_impl( + "ds4_mmq_iq2_xxs_q2_K_moe_fused_raw", + W_gate, W_up, X, ids, gate, up, + expert_mid_dim, expert_in_dim, n_tokens, n_experts, n_expert_used, + stream, + nullptr, nullptr, 0, + /*sanitize_out=*/false, &fused_down); +} + /* Aligned-artifact production fast path: gate/up accumulators stay in * registers, weighted SwiGLU is quantized directly into down_q8_scratch by * the fused D2R kernel, and only the pair-major down output is materialized. diff --git a/cuda/mmq/ds4_mmq.h b/cuda/mmq/ds4_mmq.h index 173120df0..82d021821 100644 --- a/cuda/mmq/ds4_mmq.h +++ b/cuda/mmq/ds4_mmq.h @@ -350,6 +350,36 @@ int ds4_mmq_iq2_xxs_q2_K_moe_fused_soa( * values may follow partial enqueue. */ #define DS4_MMQ_NOT_APPLICABLE 1 +/* Raw-layout twin of ds4_mmq_iq2_xxs_q2_K_moe_fused_soa. This is the + * grouped SSD entry: callers may pass a compact expert table together with + * ids remapped into [0, n_experts). Gate/up/down use canonical GGUF block + * layouts while the routing map, activation quantize, and expert bounds are + * built only once for the complete fused pipeline. As with router top-k, + * ids for one token must be unique. + * + * Capability/shape failures return DS4_MMQ_NOT_APPLICABLE before enqueue. + * Once work has been submitted, failures are negative and must not be + * retried on the same stream as a materialized fallback. */ +int ds4_mmq_iq2_xxs_q2_K_moe_fused_raw( + const void * W_gate_raw, + const void * W_up_raw, + const void * W_down_raw, + const float * X_f32, + const int32_t * ids, + const float * router_weights, + float * gate_f32, + float * up_f32, + float * mid_f32, + float * down_f32, + int expert_mid_dim, + int expert_in_dim, + int out_dim, + int n_tokens, + int n_experts, + int n_expert_used, + float clamp, + cudaStream_t stream); + /* Aligned-artifact production fast path: gate/up stay in registers, weighted * SwiGLU is quantized directly into down_q8_scratch, and only the pair-major * down output is materialized. Caller-owned scratch keeps this hot path free diff --git a/cuda/mmq/test/test_mmq_parity.cu b/cuda/mmq/test/test_mmq_parity.cu index 3e88fb0f0..850a6aa13 100644 --- a/cuda/mmq/test/test_mmq_parity.cu +++ b/cuda/mmq/test/test_mmq_parity.cu @@ -807,6 +807,364 @@ bool run_moe_pair_generic( return ok; } +// Reference glue for the raw fused IQ2/Q2 pipeline. Keep the expression and +// non-finite handling identical to ds4_swiglu_weighted_f32 in ds4_mmq.cu so +// parity below isolates routing-map reuse rather than activation math. +__global__ void test_swiglu_weighted_f32( + const float * gate, const float * up, const float * router_weights, + float * mid, uint64_t n, int K, float clamp) { + const uint64_t i = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + const uint64_t pair = i / (uint64_t)K; + float g = isfinite(gate[i]) ? gate[i] : 0.0f; + float u = isfinite(up[i]) ? up[i] : 0.0f; + if (clamp > 1.0e-6f) { + g = fminf(g, clamp); + u = fminf(fmaxf(u, -clamp), clamp); + } + mid[i] = (g / (1.0f + expf(-g))) * u * router_weights[pair]; +} + +// Exercise the SSD-facing raw fused ABI with a non-identity compact expert +// table. The reference is the established materialized chain: +// raw pair -> weighted SwiGLU -> raw down. +// Every token has six distinct experts. Flattening ids to [assignments, 1] +// for the reference down leg produces the same stable expert ordering as the +// single routing map reused by the fused candidate. +bool run_iq2_xxs_q2_K_fused_raw_parity(int n_tokens, uint32_t seed) { + constexpr int global_experts = 13; + constexpr int compact_experts = 8; + constexpr int n_expert_used = 6; + // Deliberately asymmetric and multi-block in both legs. A 256x256 + // fixture degenerates every raw row to one GGUF block and cannot catch a + // bad row/expert stride in either the IQ2 gate/up or Q2 down tensor. + constexpr int expert_mid_dim = 512; + constexpr int expert_in_dim = 768; + constexpr int out_dim = 768; + constexpr float clamp = 6.0f; + const int compact_to_global[compact_experts] = {11, 2, 9, 0, 7, 12, 4, 6}; + + fprintf(stderr, + "=== IQ2_XXS+Q2_K/FUSED_RAW compact-remap ntok=%d " + "nexp=%d nused=%d seed=%u ===\n", + n_tokens, compact_experts, n_expert_used, seed); + + std::mt19937 rng(seed); + std::normal_distribution activation(0.0f, 0.05f); + const size_t iq2_blocks_per_expert = + (size_t)expert_mid_dim * (expert_in_dim / QK_K_LOCAL); + const size_t q2_blocks_per_expert = + (size_t)out_dim * (expert_mid_dim / QK_K_LOCAL); + + std::vector gate_global( + (size_t)global_experts * iq2_blocks_per_expert); + std::vector up_global( + (size_t)global_experts * iq2_blocks_per_expert); + std::vector down_global( + (size_t)global_experts * q2_blocks_per_expert); + for (auto & block : gate_global) generate_random_block_iq2_xxs(&block, rng); + for (auto & block : up_global) generate_random_block_iq2_xxs(&block, rng); + for (auto & block : down_global) generate_random_block_q2_K(&block, rng); + + std::vector gate_compact( + (size_t)compact_experts * iq2_blocks_per_expert); + std::vector up_compact( + (size_t)compact_experts * iq2_blocks_per_expert); + std::vector down_compact( + (size_t)compact_experts * q2_blocks_per_expert); + int global_to_compact[global_experts]; + std::fill(global_to_compact, global_to_compact + global_experts, -1); + for (int compact = 0; compact < compact_experts; compact++) { + const int global = compact_to_global[compact]; + global_to_compact[global] = compact; + std::memcpy(gate_compact.data() + (size_t)compact * iq2_blocks_per_expert, + gate_global.data() + (size_t)global * iq2_blocks_per_expert, + iq2_blocks_per_expert * sizeof(block_iq2_xxs)); + std::memcpy(up_compact.data() + (size_t)compact * iq2_blocks_per_expert, + up_global.data() + (size_t)global * iq2_blocks_per_expert, + iq2_blocks_per_expert * sizeof(block_iq2_xxs)); + std::memcpy(down_compact.data() + (size_t)compact * q2_blocks_per_expert, + down_global.data() + (size_t)global * q2_blocks_per_expert, + q2_blocks_per_expert * sizeof(block_q2_K)); + } + + const size_t assignments = (size_t)n_tokens * n_expert_used; + std::vector global_ids(assignments); + std::vector remapped_ids(assignments); + std::vector router_weights(assignments); + for (int token = 0; token < n_tokens; token++) { + bool seen[compact_experts] = {}; + float router_sum = 0.0f; + for (int slot = 0; slot < n_expert_used; slot++) { + // Three is coprime with eight, so the six positions are unique; + // the global round-trip makes this an explicit compact-remap test. + const int compact = (token * 5 + slot * 3) % compact_experts; + const int global = compact_to_global[compact]; + if (seen[compact] || global_to_compact[global] != compact) { + fprintf(stderr, "invalid compact routing fixture\n"); + return false; + } + seen[compact] = true; + const size_t pair = (size_t)token * n_expert_used + slot; + global_ids[pair] = global; + remapped_ids[pair] = global_to_compact[global]; + // Token- and slot-varying values make a pair-stride bug visible; + // normalize per token to retain the production router contract. + const float raw_weight = + (float)(1 + ((token * 11 + slot * 7) % 23)); + router_weights[pair] = raw_weight; + router_sum += raw_weight; + } + for (int slot = 0; slot < n_expert_used; slot++) { + const size_t pair = (size_t)token * n_expert_used + slot; + router_weights[pair] /= router_sum; + } + } + std::vector X((size_t)n_tokens * expert_in_dim); + for (float & value : X) value = activation(rng); + + const size_t mid_count = assignments * expert_mid_dim; + const size_t down_count = assignments * out_dim; + cudaStream_t stream = nullptr; + void *d_gate_w = nullptr, *d_up_w = nullptr, *d_down_w = nullptr; + void *d_gate_global_w = nullptr, *d_up_global_w = nullptr, + *d_down_global_w = nullptr; + float *d_x = nullptr, *d_router = nullptr; + int32_t *d_ids = nullptr, *d_global_ids = nullptr; + float *d_gate_ref = nullptr, *d_up_ref = nullptr, *d_mid_ref = nullptr, + *d_down_ref = nullptr; + float *d_gate_got = nullptr, *d_up_got = nullptr, *d_mid_got = nullptr, + *d_down_got = nullptr; + float *d_gate_global = nullptr, *d_up_global = nullptr, + *d_mid_global = nullptr, *d_down_global = nullptr; + + bool allocated = cudaStreamCreate(&stream) == cudaSuccess && + cudaMalloc(&d_gate_w, gate_compact.size() * sizeof(block_iq2_xxs)) == cudaSuccess && + cudaMalloc(&d_up_w, up_compact.size() * sizeof(block_iq2_xxs)) == cudaSuccess && + cudaMalloc(&d_down_w, down_compact.size() * sizeof(block_q2_K)) == cudaSuccess && + cudaMalloc(&d_gate_global_w, gate_global.size() * sizeof(block_iq2_xxs)) == cudaSuccess && + cudaMalloc(&d_up_global_w, up_global.size() * sizeof(block_iq2_xxs)) == cudaSuccess && + cudaMalloc(&d_down_global_w, down_global.size() * sizeof(block_q2_K)) == cudaSuccess && + cudaMalloc(&d_x, X.size() * sizeof(float)) == cudaSuccess && + cudaMalloc(&d_ids, remapped_ids.size() * sizeof(int32_t)) == cudaSuccess && + cudaMalloc(&d_global_ids, global_ids.size() * sizeof(int32_t)) == cudaSuccess && + cudaMalloc(&d_router, router_weights.size() * sizeof(float)) == cudaSuccess && + cudaMalloc(&d_gate_ref, mid_count * sizeof(float)) == cudaSuccess && + cudaMalloc(&d_up_ref, mid_count * sizeof(float)) == cudaSuccess && + cudaMalloc(&d_mid_ref, mid_count * sizeof(float)) == cudaSuccess && + cudaMalloc(&d_down_ref, down_count * sizeof(float)) == cudaSuccess && + cudaMalloc(&d_gate_got, mid_count * sizeof(float)) == cudaSuccess && + cudaMalloc(&d_up_got, mid_count * sizeof(float)) == cudaSuccess && + cudaMalloc(&d_mid_got, mid_count * sizeof(float)) == cudaSuccess && + cudaMalloc(&d_down_got, down_count * sizeof(float)) == cudaSuccess && + cudaMalloc(&d_gate_global, mid_count * sizeof(float)) == cudaSuccess && + cudaMalloc(&d_up_global, mid_count * sizeof(float)) == cudaSuccess && + cudaMalloc(&d_mid_global, mid_count * sizeof(float)) == cudaSuccess && + cudaMalloc(&d_down_global, down_count * sizeof(float)) == cudaSuccess; + + auto cleanup = [&]() { + if (d_down_global) cudaFree(d_down_global); + if (d_mid_global) cudaFree(d_mid_global); + if (d_up_global) cudaFree(d_up_global); + if (d_gate_global) cudaFree(d_gate_global); + if (d_down_got) cudaFree(d_down_got); + if (d_mid_got) cudaFree(d_mid_got); + if (d_up_got) cudaFree(d_up_got); + if (d_gate_got) cudaFree(d_gate_got); + if (d_down_ref) cudaFree(d_down_ref); + if (d_mid_ref) cudaFree(d_mid_ref); + if (d_up_ref) cudaFree(d_up_ref); + if (d_gate_ref) cudaFree(d_gate_ref); + if (d_router) cudaFree(d_router); + if (d_global_ids) cudaFree(d_global_ids); + if (d_ids) cudaFree(d_ids); + if (d_x) cudaFree(d_x); + if (d_down_global_w) cudaFree(d_down_global_w); + if (d_up_global_w) cudaFree(d_up_global_w); + if (d_gate_global_w) cudaFree(d_gate_global_w); + if (d_down_w) cudaFree(d_down_w); + if (d_up_w) cudaFree(d_up_w); + if (d_gate_w) cudaFree(d_gate_w); + if (stream) cudaStreamDestroy(stream); + }; + if (!allocated) { + fprintf(stderr, "fused raw parity allocation failed\nFAIL\n\n"); + cleanup(); + return false; + } + + cudaMemcpyAsync(d_gate_w, gate_compact.data(), + gate_compact.size() * sizeof(block_iq2_xxs), + cudaMemcpyHostToDevice, stream); + cudaMemcpyAsync(d_up_w, up_compact.data(), + up_compact.size() * sizeof(block_iq2_xxs), + cudaMemcpyHostToDevice, stream); + cudaMemcpyAsync(d_down_w, down_compact.data(), + down_compact.size() * sizeof(block_q2_K), + cudaMemcpyHostToDevice, stream); + cudaMemcpyAsync(d_gate_global_w, gate_global.data(), + gate_global.size() * sizeof(block_iq2_xxs), + cudaMemcpyHostToDevice, stream); + cudaMemcpyAsync(d_up_global_w, up_global.data(), + up_global.size() * sizeof(block_iq2_xxs), + cudaMemcpyHostToDevice, stream); + cudaMemcpyAsync(d_down_global_w, down_global.data(), + down_global.size() * sizeof(block_q2_K), + cudaMemcpyHostToDevice, stream); + cudaMemcpyAsync(d_x, X.data(), X.size() * sizeof(float), + cudaMemcpyHostToDevice, stream); + cudaMemcpyAsync(d_ids, remapped_ids.data(), + remapped_ids.size() * sizeof(int32_t), + cudaMemcpyHostToDevice, stream); + cudaMemcpyAsync(d_global_ids, global_ids.data(), + global_ids.size() * sizeof(int32_t), + cudaMemcpyHostToDevice, stream); + cudaMemcpyAsync(d_router, router_weights.data(), + router_weights.size() * sizeof(float), + cudaMemcpyHostToDevice, stream); + + // A rejected shape must be retryable and must not enqueue writes. Check + // every materialized output, not just the final down buffer, with canaries. + cudaMemsetAsync(d_gate_got, 0xA5, mid_count * sizeof(float), stream); + cudaMemsetAsync(d_up_got, 0xA5, mid_count * sizeof(float), stream); + cudaMemsetAsync(d_mid_got, 0xA5, mid_count * sizeof(float), stream); + cudaMemsetAsync(d_down_got, 0xA5, down_count * sizeof(float), stream); + const int rc_na = ds4_mmq_iq2_xxs_q2_K_moe_fused_raw( + d_gate_w, d_up_w, d_down_w, d_x, d_ids, d_router, + d_gate_got, d_up_got, d_mid_got, d_down_got, + expert_mid_dim, expert_in_dim, /*out_dim=*/0, + n_tokens, compact_experts, n_expert_used, clamp, stream); + std::vector gate_canary(mid_count * sizeof(float)); + std::vector up_canary(mid_count * sizeof(float)); + std::vector mid_canary(mid_count * sizeof(float)); + std::vector down_canary(down_count * sizeof(float)); + cudaMemcpyAsync(gate_canary.data(), d_gate_got, gate_canary.size(), + cudaMemcpyDeviceToHost, stream); + cudaMemcpyAsync(up_canary.data(), d_up_got, up_canary.size(), + cudaMemcpyDeviceToHost, stream); + cudaMemcpyAsync(mid_canary.data(), d_mid_got, mid_canary.size(), + cudaMemcpyDeviceToHost, stream); + cudaMemcpyAsync(down_canary.data(), d_down_got, down_canary.size(), + cudaMemcpyDeviceToHost, stream); + cudaError_t sync_err = cudaStreamSynchronize(stream); + const auto canary_intact = [](const std::vector & bytes) { + return std::all_of(bytes.begin(), bytes.end(), + [](uint8_t value) { return value == 0xA5; }); + }; + const bool na_ok = rc_na == DS4_MMQ_NOT_APPLICABLE && + sync_err == cudaSuccess && canary_intact(gate_canary) && + canary_intact(up_canary) && canary_intact(mid_canary) && + canary_intact(down_canary); + + cudaMemsetAsync(d_gate_ref, 0, mid_count * sizeof(float), stream); + cudaMemsetAsync(d_up_ref, 0, mid_count * sizeof(float), stream); + cudaMemsetAsync(d_mid_ref, 0, mid_count * sizeof(float), stream); + cudaMemsetAsync(d_down_ref, 0, down_count * sizeof(float), stream); + cudaMemsetAsync(d_gate_got, 0, mid_count * sizeof(float), stream); + cudaMemsetAsync(d_up_got, 0, mid_count * sizeof(float), stream); + cudaMemsetAsync(d_mid_got, 0, mid_count * sizeof(float), stream); + cudaMemsetAsync(d_down_got, 0, down_count * sizeof(float), stream); + cudaMemsetAsync(d_gate_global, 0, mid_count * sizeof(float), stream); + cudaMemsetAsync(d_up_global, 0, mid_count * sizeof(float), stream); + cudaMemsetAsync(d_mid_global, 0, mid_count * sizeof(float), stream); + cudaMemsetAsync(d_down_global, 0, down_count * sizeof(float), stream); + + const int rc_pair = ds4_mmq_iq2_xxs_moe_pair( + d_gate_w, d_up_w, d_x, d_ids, d_gate_ref, d_up_ref, + expert_mid_dim, expert_in_dim, n_tokens, compact_experts, + n_expert_used, stream); + test_swiglu_weighted_f32<<< + (unsigned)((mid_count + 255u) / 256u), 256, 0, stream>>>( + d_gate_ref, d_up_ref, d_router, d_mid_ref, mid_count, + expert_mid_dim, clamp); + const cudaError_t swiglu_err = cudaGetLastError(); + const int rc_down = ds4_mmq_q2_K_moe( + d_down_w, d_mid_ref, d_ids, d_down_ref, + out_dim, expert_mid_dim, (int)assignments, compact_experts, + /*n_expert_used=*/1, stream); + const int rc_fused = ds4_mmq_iq2_xxs_q2_K_moe_fused_raw( + d_gate_w, d_up_w, d_down_w, d_x, d_ids, d_router, + d_gate_got, d_up_got, d_mid_got, d_down_got, + expert_mid_dim, expert_in_dim, out_dim, + n_tokens, compact_experts, n_expert_used, clamp, stream); + // Run the same fused path against the original global expert table and + // unremapped ids. Bitwise equality with the compact result validates the + // full-expert copies and, with the multi-block shape above, both raw + // channel strides independently of the materialized compact reference. + const int rc_global = ds4_mmq_iq2_xxs_q2_K_moe_fused_raw( + d_gate_global_w, d_up_global_w, d_down_global_w, + d_x, d_global_ids, d_router, + d_gate_global, d_up_global, d_mid_global, d_down_global, + expert_mid_dim, expert_in_dim, out_dim, + n_tokens, global_experts, n_expert_used, clamp, stream); + + std::vector gate_ref(mid_count), up_ref(mid_count), + mid_ref(mid_count), down_ref(down_count); + std::vector gate_got(mid_count), up_got(mid_count), + mid_got(mid_count), down_got(down_count); + std::vector gate_global_out(mid_count), up_global_out(mid_count), + mid_global_out(mid_count), down_global_out(down_count); + cudaMemcpyAsync(gate_ref.data(), d_gate_ref, mid_count * sizeof(float), + cudaMemcpyDeviceToHost, stream); + cudaMemcpyAsync(up_ref.data(), d_up_ref, mid_count * sizeof(float), + cudaMemcpyDeviceToHost, stream); + cudaMemcpyAsync(mid_ref.data(), d_mid_ref, mid_count * sizeof(float), + cudaMemcpyDeviceToHost, stream); + cudaMemcpyAsync(down_ref.data(), d_down_ref, down_count * sizeof(float), + cudaMemcpyDeviceToHost, stream); + cudaMemcpyAsync(gate_got.data(), d_gate_got, mid_count * sizeof(float), + cudaMemcpyDeviceToHost, stream); + cudaMemcpyAsync(up_got.data(), d_up_got, mid_count * sizeof(float), + cudaMemcpyDeviceToHost, stream); + cudaMemcpyAsync(mid_got.data(), d_mid_got, mid_count * sizeof(float), + cudaMemcpyDeviceToHost, stream); + cudaMemcpyAsync(down_got.data(), d_down_got, down_count * sizeof(float), + cudaMemcpyDeviceToHost, stream); + cudaMemcpyAsync(gate_global_out.data(), d_gate_global, + mid_count * sizeof(float), cudaMemcpyDeviceToHost, stream); + cudaMemcpyAsync(up_global_out.data(), d_up_global, + mid_count * sizeof(float), cudaMemcpyDeviceToHost, stream); + cudaMemcpyAsync(mid_global_out.data(), d_mid_global, + mid_count * sizeof(float), cudaMemcpyDeviceToHost, stream); + cudaMemcpyAsync(down_global_out.data(), d_down_global, + down_count * sizeof(float), cudaMemcpyDeviceToHost, stream); + sync_err = cudaStreamSynchronize(stream); + + const auto mismatches = [](const std::vector & a, + const std::vector & b) { + size_t bad = 0; + for (size_t i = 0; i < a.size(); i++) { + if (std::memcmp(&a[i], &b[i], sizeof(float)) != 0) bad++; + } + return bad; + }; + const size_t gate_bad = mismatches(gate_got, gate_ref); + const size_t up_bad = mismatches(up_got, up_ref); + const size_t mid_bad = mismatches(mid_got, mid_ref); + const size_t down_bad = mismatches(down_got, down_ref); + const size_t gate_remap_bad = mismatches(gate_got, gate_global_out); + const size_t up_remap_bad = mismatches(up_got, up_global_out); + const size_t mid_remap_bad = mismatches(mid_got, mid_global_out); + const size_t down_remap_bad = mismatches(down_got, down_global_out); + const bool ok = na_ok && rc_pair == 0 && swiglu_err == cudaSuccess && + rc_down == 0 && rc_fused == 0 && rc_global == 0 && + sync_err == cudaSuccess && gate_bad == 0 && up_bad == 0 && + mid_bad == 0 && down_bad == 0 && gate_remap_bad == 0 && + up_remap_bad == 0 && mid_remap_bad == 0 && down_remap_bad == 0; + fprintf(stderr, + "rc_na=%d canary=%s rc_pair=%d swiglu=%s rc_down=%d " + "rc_fused=%d rc_global=%d mismatches(g/u/m/d)=%zu/%zu/%zu/%zu " + "remap_mismatches(g/u/m/d)=%zu/%zu/%zu/%zu sync=%s\n%s\n\n", + rc_na, na_ok ? "intact" : "FAILED", rc_pair, + cudaGetErrorString(swiglu_err), rc_down, rc_fused, rc_global, + gate_bad, up_bad, mid_bad, down_bad, + gate_remap_bad, up_remap_bad, mid_remap_bad, down_remap_bad, + cudaGetErrorString(sync_err), ok ? "PASS" : "FAIL"); + + cleanup(); + return ok; +} + bool run_q4_K_moe(int M, int K, int nt, int ne, int nu, uint32_t seed) { auto fn = [](block_q4_K * blk, float * out, int n_experts, int M, int K, int blocks_per_expert, @@ -1499,6 +1857,13 @@ int main(int argc, char ** argv) { /*ne=*/16, /*nu=*/6, 0xC4FE10, gen_q4k, ds4_mmq_q4_K_moe_pair, ds4_mmq_q4_K_moe); + // SSD compact-table raw fusion: one expert map/activation quantize for + // IQ2 gate+up and Q2 down. Cover the production top-6 routing shape at + // each target prefill width. + all_ok &= run_iq2_xxs_q2_K_fused_raw_parity(/*nt=*/8, 0xC2F008); + all_ok &= run_iq2_xxs_q2_K_fused_raw_parity(/*nt=*/32, 0xC2F020); + all_ok &= run_iq2_xxs_q2_K_fused_raw_parity(/*nt=*/128, 0xC2F080); + // Step 6 - mmvq vector matmul tests. // // Single-W _moe_vec tests. Two shape classes per type: From 7c5b9a3adcce188f5333fa0bdcadf2ee8517b32e Mon Sep 17 00:00:00 2001 From: Giorgio Oppo Date: Fri, 14 Aug 2026 21:22:27 +0200 Subject: [PATCH 033/189] cuda: group IQ2 SSD prefill MMQ --- ds4_cuda.cu | 1507 ++++++++++++++++++++++++++++++---- ds4_gpu.h | 38 + tests/test_gpu_model_cache.c | 82 ++ 3 files changed, 1463 insertions(+), 164 deletions(-) diff --git a/ds4_cuda.cu b/ds4_cuda.cu index ba43dbef8..ad6d09734 100644 --- a/ds4_cuda.cu +++ b/ds4_cuda.cu @@ -25,6 +25,8 @@ #include #include #include +#include +#include #include "cuda/mmq/ds4_mmq.h" #include "cuda/mmq/ds4_repack.h" @@ -92,6 +94,10 @@ typedef struct { uint16_t d; uint16_t qs[CUDA_QK_K / 8]; } cuda_block_iq2_xxs; +static_assert(sizeof(cuda_block_iq2_xxs) == 66u, + "canonical IQ2_XXS block layout drift"); +static_assert(sizeof(cuda_block_q2_K) == 84u, + "canonical Q2_K block layout drift"); #include "ds4_gpu_mgpu.h" #include "ds4_iq2_tables_cuda.inc" @@ -150,12 +156,17 @@ static int g_ssd_streaming_mode; typedef struct { int valid; + int top6_unique; int logical_tier; const void *model_map; uint32_t layer; uint32_t n_total_expert; uint32_t slot_count; uint32_t compact_count; + uint32_t slot_base; + uint32_t weight_domain; + uint64_t generation; + uint64_t upload_event_value; uint64_t gate_offset; uint64_t up_offset; uint64_t down_offset; @@ -174,31 +185,13 @@ typedef struct { static cuda_stream_selected_cache g_stream_selected_cache; static void cuda_stream_selected_upload_drain(void); +static int cuda_stream_selected_consume_drain(void); +static void cuda_stream_selected_consume_release(void); +static void cuda_stream_selected_cache_release(void); static void cuda_stream_selected_cache_invalidate(void) { g_stream_selected_cache.valid = 0; -} - -static void cuda_stream_selected_cache_release(void) { - cuda_stream_selected_upload_drain(); - const int tier = g_stream_selected_cache.logical_tier; - if (tier >= 0 && tier < g_n_gpus) { - (void)ds4_gpu_set_current_device(tier); - } - if (g_stream_selected_cache.gate_ptr) { - (void)cudaFree(g_stream_selected_cache.gate_ptr); - } - if (g_stream_selected_cache.up_ptr) { - (void)cudaFree(g_stream_selected_cache.up_ptr); - } - if (g_stream_selected_cache.down_ptr) { - (void)cudaFree(g_stream_selected_cache.down_ptr); - } - if (g_stream_selected_cache.slot_selected_ptr) { - (void)cudaFree(g_stream_selected_cache.slot_selected_ptr); - } - memset(&g_stream_selected_cache, 0, sizeof(g_stream_selected_cache)); - g_stream_selected_cache.logical_tier = -1; + g_stream_selected_cache.upload_event_value = 0; } typedef struct { @@ -535,9 +528,19 @@ static uint64_t g_stream_selected_readback_stage_capacity; static cudaEvent_t g_stream_selected_compute_ready_event; static cudaEvent_t g_stream_selected_readback_done_event; static cudaEvent_t g_stream_selected_upload_done_event; +static cudaEvent_t g_stream_selected_consume_done_event; static int g_stream_selected_event_owner_device = -1; +static int g_stream_selected_consume_owner_device = -1; static uint64_t g_stream_selected_compute_event_value; static uint64_t g_stream_selected_upload_event_value; +static uint64_t g_stream_selected_cache_generation; +static uint64_t g_stream_selected_consume_generation; +static int g_stream_selected_consume_pending; +static int g_stream_selected_consume_poisoned; +static std::mutex g_stream_selected_consume_mutex; +static std::condition_variable g_stream_selected_consume_cv; +static uint32_t g_stream_selected_consume_host_readers; +static int g_stream_selected_writer_active; static std::once_flag g_stream_selected_batch_io_once; static int g_stream_selected_batch_io_enabled; static int g_stream_selected_batch_io_required; @@ -571,9 +574,28 @@ static std::atomic g_stream_selected_event_failures{0}; static std::atomic g_stream_selected_event_required_failures{0}; static std::atomic g_stream_selected_event_oracle_runs{0}; static std::atomic g_stream_selected_event_oracle_failures{0}; +static std::once_flag g_iq2_ssd_grouped_once; +static int g_iq2_ssd_grouped_enabled; +static int g_iq2_ssd_grouped_required; +static int g_iq2_ssd_grouped_stats; +static int g_iq2_ssd_grouped_report_registered; +static std::atomic g_iq2_ssd_grouped_candidates{0}; +static std::atomic g_iq2_ssd_grouped_eligible{0}; +static std::atomic g_iq2_ssd_grouped_attempts{0}; +static std::atomic g_iq2_ssd_grouped_completed{0}; +static std::atomic g_iq2_ssd_grouped_not_applicable{0}; +static std::atomic g_iq2_ssd_grouped_safe_fallbacks{0}; +static std::atomic g_iq2_ssd_grouped_failures{0}; +static std::atomic g_iq2_ssd_grouped_required_failures{0}; +static std::atomic g_iq2_ssd_grouped_upload_waits{0}; +static std::atomic g_iq2_ssd_grouped_lease_waits{0}; +static std::atomic g_iq2_ssd_grouped_lease_records{0}; +static std::atomic g_iq2_ssd_grouped_lease_drains{0}; static int cuda_ok(cudaError_t err, const char *what); static void cuda_stream_selected_event_pipeline_release(void); +static int cuda_stream_selected_wait_upload_on( + uint64_t event_value, cudaStream_t stream, const char *label); extern "C" int ds4_gpu_stream_expert_cache_wait_selected_upload( uint64_t event_value, const char *label); extern "C" void ds4_gpu_decode_graphs_invalidate(void); @@ -2924,6 +2946,174 @@ extern "C" int ds4_cuda_test_stream_selected_event_env_value( return cuda_stream_selected_env_value_enabled(value); } +struct ds4_cuda_iq2_ssd_grouped_report; +typedef struct ds4_cuda_iq2_ssd_grouped_report + ds4_cuda_iq2_ssd_grouped_report; +typedef struct { + uint64_t candidates; + uint64_t eligible; + uint64_t attempts; + uint64_t completed; + uint64_t not_applicable; + uint64_t safe_fallbacks; + uint64_t failures; + uint64_t required_failures; + uint64_t upload_waits; + uint64_t lease_waits; + uint64_t lease_records; + uint64_t lease_drains; + int enabled; + int required; + int stats; +} cuda_iq2_ssd_grouped_report_layout; + +static void cuda_iq2_ssd_grouped_resolve_policy( + int enable, int disable, int require, int stats, + int *enabled_out, int *required_out, int *stats_out) { + const int disabled = disable != 0; + if (enabled_out) *enabled_out = !disabled && (enable || require); + if (required_out) *required_out = !disabled && require; + if (stats_out) *stats_out = !disabled && stats; +} + +extern "C" int ds4_cuda_test_iq2_ssd_grouped_policy( + int enable, int disable, int require, int stats, + int *enabled_out, int *required_out, int *stats_out) { + if (!enabled_out || !required_out || !stats_out) return 0; + cuda_iq2_ssd_grouped_resolve_policy( + enable, disable, require, stats, + enabled_out, required_out, stats_out); + return 1; +} + +/* Keep the production gate decomposed so host-only policy tests can prove + * that every exclusion remains fail-safe without spoofing CUDA properties. + * Raw-layout validation itself is checked separately against exact GGUF + * strides in routed_moe_launch(). */ +static int cuda_iq2_ssd_grouped_eligible_values( + int enabled, int ssd_streaming, int single_gpu, int gb10, + int quality, int owned_filtered, int capture, int mmq, + uint32_t n_tokens, uint32_t n_expert, int top6_unique, + int raw_layout, int binding_valid) { + return enabled && ssd_streaming && single_gpu && gb10 && !quality && + !owned_filtered && !capture && mmq && n_tokens >= 32u && + n_expert == 6u && top6_unique && raw_layout && binding_valid; +} + +extern "C" int ds4_cuda_test_iq2_ssd_grouped_eligibility( + int enabled, int ssd_streaming, int single_gpu, int gb10, + int quality, int owned_filtered, int capture, int mmq, + uint32_t n_tokens, uint32_t n_expert, int top6_unique, + int raw_layout, int binding_valid) { + return cuda_iq2_ssd_grouped_eligible_values( + enabled, ssd_streaming, single_gpu, gb10, quality, + owned_filtered, capture, mmq, n_tokens, n_expert, + top6_unique, raw_layout, binding_valid); +} + +static int cuda_iq2_ssd_grouped_candidate_values( + int iq2_path, int ssd_streaming, int allow_streaming, + int owned_filtered, uint32_t n_tokens, uint32_t n_expert, + int top6_unique, int raw_layout, int binding_valid) { + return iq2_path && ssd_streaming && allow_streaming && + !owned_filtered && n_tokens >= 32u && n_expert == 6u && + top6_unique && raw_layout && binding_valid; +} + +extern "C" int ds4_cuda_test_iq2_ssd_grouped_candidate( + int iq2_path, int ssd_streaming, int allow_streaming, + int owned_filtered, uint32_t n_tokens, uint32_t n_expert, + int top6_unique, int raw_layout, int binding_valid) { + return cuda_iq2_ssd_grouped_candidate_values( + iq2_path, ssd_streaming, allow_streaming, owned_filtered, + n_tokens, n_expert, top6_unique, raw_layout, binding_valid); +} + +static void cuda_iq2_ssd_grouped_report_at_exit(void) { + fprintf(stderr, + "ds4: CUDA IQ2 SSD grouped MMQ: candidates=%llu eligible=%llu " + "attempts=%llu completed=%llu not_applicable=%llu " + "safe_fallbacks=%llu failures=%llu required_failures=%llu " + "upload_waits=%llu lease_waits=%llu lease_records=%llu " + "lease_drains=%llu\n", + (unsigned long long)g_iq2_ssd_grouped_candidates.load(), + (unsigned long long)g_iq2_ssd_grouped_eligible.load(), + (unsigned long long)g_iq2_ssd_grouped_attempts.load(), + (unsigned long long)g_iq2_ssd_grouped_completed.load(), + (unsigned long long)g_iq2_ssd_grouped_not_applicable.load(), + (unsigned long long)g_iq2_ssd_grouped_safe_fallbacks.load(), + (unsigned long long)g_iq2_ssd_grouped_failures.load(), + (unsigned long long)g_iq2_ssd_grouped_required_failures.load(), + (unsigned long long)g_iq2_ssd_grouped_upload_waits.load(), + (unsigned long long)g_iq2_ssd_grouped_lease_waits.load(), + (unsigned long long)g_iq2_ssd_grouped_lease_records.load(), + (unsigned long long)g_iq2_ssd_grouped_lease_drains.load()); +} + +static void cuda_iq2_ssd_grouped_init(void) { + const int enable = cuda_stream_selected_env_flag( + "DS4_CUDA_ENABLE_IQ2_XXS_SSD_PREFILL_MMQ"); + const int disable = cuda_stream_selected_env_flag( + "DS4_CUDA_DISABLE_IQ2_XXS_SSD_PREFILL_MMQ") || + cuda_stream_selected_env_flag( + "DS4_CUDA_NO_IQ2_XXS_SSD_PREFILL_MMQ"); + const int require = cuda_stream_selected_env_flag( + "DS4_CUDA_REQUIRE_IQ2_XXS_SSD_PREFILL_MMQ"); + const int stats = cuda_stream_selected_env_flag( + "DS4_CUDA_IQ2_XXS_SSD_PREFILL_MMQ_STATS"); + cuda_iq2_ssd_grouped_resolve_policy( + enable, disable, require, stats, + &g_iq2_ssd_grouped_enabled, + &g_iq2_ssd_grouped_required, + &g_iq2_ssd_grouped_stats); + if ((enable || require || stats) && + !g_iq2_ssd_grouped_report_registered) { + g_iq2_ssd_grouped_report_registered = 1; + (void)atexit(cuda_iq2_ssd_grouped_report_at_exit); + } + if (g_iq2_ssd_grouped_enabled) { + fprintf(stderr, + "ds4: CUDA IQ2_XXS/Q2_K SSD grouped prefill MMQ enabled%s\n", + g_iq2_ssd_grouped_required ? " (required)" : ""); + } else if (disable && (enable || require)) { + fprintf(stderr, + "ds4: CUDA IQ2 SSD grouped MMQ disabled by rollback " + "override\n"); + } +} + +static int cuda_iq2_ssd_grouped_enabled(void) { + std::call_once(g_iq2_ssd_grouped_once, cuda_iq2_ssd_grouped_init); + return g_iq2_ssd_grouped_enabled; +} + +static int cuda_iq2_ssd_grouped_required(void) { + std::call_once(g_iq2_ssd_grouped_once, cuda_iq2_ssd_grouped_init); + return g_iq2_ssd_grouped_required; +} + +extern "C" void ds4_cuda_iq2_ssd_grouped_get_report( + ds4_cuda_iq2_ssd_grouped_report *report) { + if (!report) return; + cuda_iq2_ssd_grouped_report_layout out = {}; + out.candidates = g_iq2_ssd_grouped_candidates.load(); + out.eligible = g_iq2_ssd_grouped_eligible.load(); + out.attempts = g_iq2_ssd_grouped_attempts.load(); + out.completed = g_iq2_ssd_grouped_completed.load(); + out.not_applicable = g_iq2_ssd_grouped_not_applicable.load(); + out.safe_fallbacks = g_iq2_ssd_grouped_safe_fallbacks.load(); + out.failures = g_iq2_ssd_grouped_failures.load(); + out.required_failures = g_iq2_ssd_grouped_required_failures.load(); + out.upload_waits = g_iq2_ssd_grouped_upload_waits.load(); + out.lease_waits = g_iq2_ssd_grouped_lease_waits.load(); + out.lease_records = g_iq2_ssd_grouped_lease_records.load(); + out.lease_drains = g_iq2_ssd_grouped_lease_drains.load(); + out.enabled = cuda_iq2_ssd_grouped_enabled(); + out.required = cuda_iq2_ssd_grouped_required(); + out.stats = g_iq2_ssd_grouped_stats; + memcpy(report, &out, sizeof(out)); +} + static void cuda_stream_selected_batch_io_resolve_policy( int enable, int disable, int require, int oracle, int *enabled_out, int *required_out, int *oracle_out) { @@ -3274,6 +3464,288 @@ static int cuda_stream_selected_event_pipeline_ensure(void) { return 1; } +/* A compact selected cache is a transient binding, not ordinary scratch: + * the next SSD load overwrites the same gate/up/down arrays and remap. The + * reusable consume event is recorded after the last grouped-MMQ consumer and + * imported into the upload stream before the following H2D epoch. Allocation + * growth and teardown use the host drain because cudaFree cannot be ordered + * behind an event in the upload stream. */ +static int cuda_stream_selected_consume_ensure_locked(void) { + if (g_n_gpus != 1 || g_stream_selected_consume_poisoned) return 0; + const int owner = g_gpu[0].device_id; + if (cudaSetDevice(owner) != cudaSuccess) { + (void)cudaGetLastError(); + return 0; + } + if (g_stream_selected_consume_done_event && + g_stream_selected_consume_owner_device == owner) { + return 1; + } + if (g_stream_selected_consume_pending && + g_stream_selected_consume_done_event) { + const cudaError_t sync_err = + cudaEventSynchronize(g_stream_selected_consume_done_event); + if (sync_err != cudaSuccess) { + fprintf(stderr, + "ds4: CUDA compact-cache consume drain failed while " + "changing owner: %s\n", + cudaGetErrorString(sync_err)); + (void)cudaGetLastError(); + return 0; + } + g_stream_selected_consume_pending = 0; + g_iq2_ssd_grouped_lease_drains.fetch_add( + 1, std::memory_order_relaxed); + } + if (g_stream_selected_consume_done_event) { + (void)cudaEventDestroy(g_stream_selected_consume_done_event); + g_stream_selected_consume_done_event = NULL; + } + const cudaError_t err = cudaEventCreateWithFlags( + &g_stream_selected_consume_done_event, cudaEventDisableTiming); + if (err != cudaSuccess) { + fprintf(stderr, + "ds4: CUDA compact-cache consume event creation failed: %s\n", + cudaGetErrorString(err)); + (void)cudaGetLastError(); + g_stream_selected_consume_owner_device = -1; + return 0; + } + g_stream_selected_consume_owner_device = owner; + return 1; +} + +static int cuda_stream_selected_consume_prepare(void) { + std::lock_guard lock(g_stream_selected_consume_mutex); + return cuda_stream_selected_consume_ensure_locked(); +} + +/* Exclusive host-side writer lease for the transient binding. The flag is + * held for the entire load, publication, release or abort transaction, not + * just while valid is toggled. That closes both directions of the TOCTOU: + * readers cannot enter after a loader observes zero readers, and a reader + * cannot observe partially replaced metadata while a writer is active. */ +class cuda_stream_selected_writer_guard { +public: + explicit cuda_stream_selected_writer_guard( + std::atomic *waiting_probe = nullptr) { + std::unique_lock lock( + g_stream_selected_consume_mutex); + if (waiting_probe) { + waiting_probe->store(1, std::memory_order_release); + } + g_stream_selected_consume_cv.wait(lock, [] { + return !g_stream_selected_writer_active && + g_stream_selected_consume_host_readers == 0u; + }); + g_stream_selected_writer_active = 1; + cuda_stream_selected_cache_invalidate(); + held_ = true; + } + + cuda_stream_selected_writer_guard( + const cuda_stream_selected_writer_guard &) = delete; + cuda_stream_selected_writer_guard &operator=( + const cuda_stream_selected_writer_guard &) = delete; + + void publish_valid() { + std::lock_guard lock( + g_stream_selected_consume_mutex); + if (!held_) return; + g_stream_selected_cache.valid = 1; + g_stream_selected_writer_active = 0; + held_ = false; + g_stream_selected_consume_cv.notify_all(); + } + + ~cuda_stream_selected_writer_guard() { + std::lock_guard lock( + g_stream_selected_consume_mutex); + if (!held_) return; + cuda_stream_selected_cache_invalidate(); + g_stream_selected_writer_active = 0; + held_ = false; + g_stream_selected_consume_cv.notify_all(); + } + +private: + bool held_ = false; +}; + +static int cuda_stream_selected_consume_drain(void) { + std::lock_guard lock(g_stream_selected_consume_mutex); + if (g_stream_selected_consume_poisoned) { + if (g_stream_selected_consume_owner_device < 0 || + cudaSetDevice(g_stream_selected_consume_owner_device) != + cudaSuccess) { + (void)cudaGetLastError(); + return 0; + } + const cudaError_t device_err = cudaDeviceSynchronize(); + if (device_err != cudaSuccess) { + fprintf(stderr, + "ds4: CUDA compact-cache poisoned drain failed: %s\n", + cudaGetErrorString(device_err)); + (void)cudaGetLastError(); + return 0; + } + g_stream_selected_consume_poisoned = 0; + g_stream_selected_consume_pending = 0; + g_iq2_ssd_grouped_lease_drains.fetch_add( + 1, std::memory_order_relaxed); + return 1; + } + if (!g_stream_selected_consume_pending) return 1; + if (!g_stream_selected_consume_done_event || + g_stream_selected_consume_owner_device < 0 || + cudaSetDevice(g_stream_selected_consume_owner_device) != cudaSuccess) { + (void)cudaGetLastError(); + return 0; + } + const cudaError_t err = + cudaEventSynchronize(g_stream_selected_consume_done_event); + if (err != cudaSuccess) { + fprintf(stderr, + "ds4: CUDA compact-cache consume drain failed: %s\n", + cudaGetErrorString(err)); + (void)cudaGetLastError(); + return 0; + } + g_stream_selected_consume_pending = 0; + g_iq2_ssd_grouped_lease_drains.fetch_add( + 1, std::memory_order_relaxed); + return 1; +} + +/* Caller owns cuda_stream_selected_writer_guard, so no reader can acquire + * these pointers while they are drained and freed. */ +static int cuda_stream_selected_cache_free_storage_writer(void) { + if (!g_stream_selected_writer_active || + !cuda_stream_selected_consume_drain()) { + /* Never free a compact binding whose last consumer could not be + * drained. A bounded leak is safer than a cross-stream UAF when the + * CUDA context is already reporting an unrecoverable error. */ + cuda_stream_selected_cache_invalidate(); + return 0; + } + cuda_stream_selected_upload_drain(); + const int has_storage = g_stream_selected_cache.gate_ptr || + g_stream_selected_cache.up_ptr || + g_stream_selected_cache.down_ptr || + g_stream_selected_cache.slot_selected_ptr; + if (has_storage && + (g_n_gpus != 1 || + cudaSetDevice(g_gpu[0].device_id) != cudaSuccess)) { + (void)cudaGetLastError(); + return 0; + } + if (g_stream_selected_cache.gate_ptr) { + (void)cudaFree(g_stream_selected_cache.gate_ptr); + } + if (g_stream_selected_cache.up_ptr) { + (void)cudaFree(g_stream_selected_cache.up_ptr); + } + if (g_stream_selected_cache.down_ptr) { + (void)cudaFree(g_stream_selected_cache.down_ptr); + } + if (g_stream_selected_cache.slot_selected_ptr) { + (void)cudaFree(g_stream_selected_cache.slot_selected_ptr); + } + memset(&g_stream_selected_cache, 0, sizeof(g_stream_selected_cache)); + g_stream_selected_cache.logical_tier = -1; + return 1; +} + +static void cuda_stream_selected_cache_release(void) { + cuda_stream_selected_writer_guard writer; + (void)cuda_stream_selected_cache_free_storage_writer(); +} + +static void cuda_stream_selected_consume_release(void) { + std::lock_guard lock(g_stream_selected_consume_mutex); + if (g_stream_selected_consume_owner_device >= 0) { + (void)cudaSetDevice(g_stream_selected_consume_owner_device); + } + if (g_stream_selected_consume_pending && + g_stream_selected_consume_done_event) { + (void)cudaEventSynchronize(g_stream_selected_consume_done_event); + } + if (g_stream_selected_consume_done_event) { + (void)cudaEventDestroy(g_stream_selected_consume_done_event); + g_stream_selected_consume_done_event = NULL; + } + g_stream_selected_consume_owner_device = -1; + g_stream_selected_consume_pending = 0; + g_stream_selected_consume_poisoned = 0; + g_stream_selected_consume_generation = 0; +} + +static int cuda_stream_selected_consume_wait_on_upload(void) { + std::lock_guard lock(g_stream_selected_consume_mutex); + if (g_stream_selected_consume_poisoned) return 0; + if (!g_stream_selected_consume_pending) return 1; + if (!g_stream_selected_consume_done_event || + !g_stream_selected_upload_stream || + g_stream_selected_consume_owner_device < 0 || + g_stream_selected_upload_owner_device != + g_stream_selected_consume_owner_device || + cudaSetDevice(g_stream_selected_consume_owner_device) != cudaSuccess) { + (void)cudaGetLastError(); + return 0; + } + const cudaError_t err = cudaStreamWaitEvent( + g_stream_selected_upload_stream, + g_stream_selected_consume_done_event, 0); + if (err != cudaSuccess) { + fprintf(stderr, + "ds4: CUDA compact-cache upload lease wait failed: %s\n", + cudaGetErrorString(err)); + (void)cudaGetLastError(); + return 0; + } + g_iq2_ssd_grouped_lease_waits.fetch_add( + 1, std::memory_order_relaxed); + return 1; +} + +static int cuda_stream_selected_consume_record( + uint64_t binding_generation, cudaStream_t stream) { + std::lock_guard lock(g_stream_selected_consume_mutex); + if (!cuda_stream_selected_consume_ensure_locked()) return 0; + const cudaError_t err = cudaEventRecord( + g_stream_selected_consume_done_event, stream); + if (err != cudaSuccess) { + fprintf(stderr, + "ds4: CUDA compact-cache consume event record failed: %s\n", + cudaGetErrorString(err)); + (void)cudaGetLastError(); + /* A failed record must never expose an unfenced binding to the next + * loader. Draining the exact consumer stream is the safe fallback. */ + const cudaError_t sync_err = cudaStreamSynchronize(stream); + if (sync_err != cudaSuccess) { + fprintf(stderr, + "ds4: CUDA compact-cache consumer drain failed: %s\n", + cudaGetErrorString(sync_err)); + (void)cudaGetLastError(); + const cudaError_t device_err = cudaDeviceSynchronize(); + if (device_err != cudaSuccess) { + fprintf(stderr, + "ds4: CUDA compact-cache device drain failed: %s\n", + cudaGetErrorString(device_err)); + (void)cudaGetLastError(); + g_stream_selected_consume_poisoned = 1; + } + } + g_stream_selected_consume_pending = 0; + return 0; + } + g_stream_selected_consume_generation = binding_generation; + g_stream_selected_consume_pending = 1; + g_iq2_ssd_grouped_lease_records.fetch_add( + 1, std::memory_order_relaxed); + return 1; +} + static int cuda_stream_selected_readback_stage_ensure(uint64_t bytes) { if (bytes == 0 || bytes > SIZE_MAX) return 0; if (g_stream_selected_readback_stage && @@ -3562,6 +4034,12 @@ static int cuda_model_copy_tasks_to_device_streamed( !cuda_stream_selected_stage_pool_alloc(stage_bytes)) { return 0; } + if (!cuda_stream_selected_consume_wait_on_upload()) { + fprintf(stderr, + "ds4: CUDA selected batch refused to overwrite an active " + "compact-cache lease\n"); + return 0; + } /* Pool allocation is first: its cold-start/re-size path releases every * selected-upload staging allocation, including the persistent remap. @@ -4097,7 +4575,8 @@ static int cuda_model_copy_to_device_streamed( uint64_t stage_bytes = 0; if (!cuda_host_stage_bytes_for_chunk(chunk, g_model_direct_align, &stage_bytes) || - !cuda_stream_selected_stage_pool_alloc(stage_bytes)) return 0; + !cuda_stream_selected_stage_pool_alloc(stage_bytes) || + !cuda_stream_selected_consume_wait_on_upload()) return 0; uint64_t copied = 0; uint64_t chunk_idx = 0; @@ -4748,6 +5227,7 @@ extern "C" void ds4_gpu_cleanup(void) { cuda_stream_selected_stage_release(); cuda_stream_selected_event_pipeline_release(); cuda_stream_selected_cache_release(); + cuda_stream_selected_consume_release(); ds4_mmq_set_gb10_optimizations(0); g_n_gpus = 0; g_cublas_ready = 0; @@ -27234,6 +27714,135 @@ __global__ static void moe_down_f32_kernel( if (threadIdx.x == 0) down_out[(uint64_t)pair * out_dim + row] = partial[0]; } +typedef struct { + int valid; + int top6_unique; + const char *gate; + const char *up; + const char *down; + const ds4_gpu_tensor *selected; + uint32_t slot_base; + uint32_t weight_domain; + uint64_t generation; + uint64_t upload_event_value; +} cuda_stream_selected_binding; + +static int cuda_stream_selected_binding_acquire( + cuda_stream_selected_binding *binding, + int logical_tier, + const void *model_map, + uint32_t layer_index, + uint32_t n_total_expert, + uint64_t gate_offset, + uint64_t up_offset, + uint64_t down_offset, + uint64_t gate_expert_bytes, + uint64_t down_expert_bytes, + uint64_t required_slot_count) { + if (!binding) return 0; + memset(binding, 0, sizeof(*binding)); + if (gate_expert_bytes == 0 || down_expert_bytes == 0 || + required_slot_count > UINT64_MAX / sizeof(int32_t) || + !g_stream_selected_cache.valid || + g_stream_selected_cache.logical_tier != logical_tier || + g_stream_selected_cache.model_map != model_map || + g_stream_selected_cache.layer != layer_index || + g_stream_selected_cache.n_total_expert != n_total_expert || + g_stream_selected_cache.slot_count != required_slot_count || + g_stream_selected_cache.gate_offset != gate_offset || + g_stream_selected_cache.up_offset != up_offset || + g_stream_selected_cache.down_offset != down_offset || + g_stream_selected_cache.gate_expert_bytes != gate_expert_bytes || + g_stream_selected_cache.down_expert_bytes != down_expert_bytes || + !g_stream_selected_cache.gate_ptr || + !g_stream_selected_cache.up_ptr || + !g_stream_selected_cache.down_ptr || + !g_stream_selected_cache.slot_selected_tensor.ptr || + g_stream_selected_cache.slot_selected_tensor.bytes < + required_slot_count * sizeof(int32_t) || + g_stream_selected_cache.generation == 0 || + g_stream_selected_cache.weight_domain == 0 || + g_stream_selected_cache.slot_base > UINT32_MAX - + g_stream_selected_cache.weight_domain) { + return 0; + } + const uint64_t end_slot = + (uint64_t)g_stream_selected_cache.slot_base + + g_stream_selected_cache.weight_domain; + if (end_slot > UINT64_MAX / gate_expert_bytes || + end_slot * gate_expert_bytes > + g_stream_selected_cache.gate_capacity || + end_slot * gate_expert_bytes > + g_stream_selected_cache.up_capacity || + end_slot > UINT64_MAX / down_expert_bytes || + end_slot * down_expert_bytes > + g_stream_selected_cache.down_capacity) { + return 0; + } + binding->valid = 1; + binding->top6_unique = g_stream_selected_cache.top6_unique; + binding->gate = g_stream_selected_cache.gate_ptr + + (uint64_t)g_stream_selected_cache.slot_base * gate_expert_bytes; + binding->up = g_stream_selected_cache.up_ptr + + (uint64_t)g_stream_selected_cache.slot_base * gate_expert_bytes; + binding->down = g_stream_selected_cache.down_ptr + + (uint64_t)g_stream_selected_cache.slot_base * down_expert_bytes; + binding->selected = &g_stream_selected_cache.slot_selected_tensor; + binding->slot_base = g_stream_selected_cache.slot_base; + binding->weight_domain = g_stream_selected_cache.weight_domain; + binding->generation = g_stream_selected_cache.generation; + binding->upload_event_value = + g_stream_selected_cache.upload_event_value; + return 1; +} + +static int cuda_iq2_ssd_grouped_raw_layout( + uint32_t gate_type, uint32_t down_type, + uint64_t gate_expert_bytes, uint64_t gate_row_bytes, + uint64_t down_expert_bytes, uint64_t down_row_bytes, + uint32_t expert_in_dim, uint32_t expert_mid_dim, + uint32_t out_dim) { + if (gate_type != 16u || down_type != 10u || + expert_in_dim == 0u || expert_mid_dim == 0u || out_dim == 0u || + (expert_in_dim % CUDA_QK_K) != 0u || + (expert_mid_dim % CUDA_QK_K) != 0u) { + return 0; + } + const uint64_t iq2_blocks = expert_in_dim / CUDA_QK_K; + const uint64_t q2_blocks = expert_mid_dim / CUDA_QK_K; + if (iq2_blocks > UINT64_MAX / sizeof(cuda_block_iq2_xxs) || + q2_blocks > UINT64_MAX / sizeof(cuda_block_q2_K)) { + return 0; + } + const uint64_t canonical_gate_row = + iq2_blocks * sizeof(cuda_block_iq2_xxs); + const uint64_t canonical_down_row = + q2_blocks * sizeof(cuda_block_q2_K); + if ((uint64_t)expert_mid_dim > + UINT64_MAX / canonical_gate_row || + (uint64_t)out_dim > UINT64_MAX / canonical_down_row) { + return 0; + } + return gate_row_bytes == canonical_gate_row && + gate_expert_bytes == + (uint64_t)expert_mid_dim * canonical_gate_row && + down_row_bytes == canonical_down_row && + down_expert_bytes == + (uint64_t)out_dim * canonical_down_row; +} + +extern "C" int ds4_cuda_test_iq2_ssd_grouped_raw_layout( + uint32_t gate_type, uint32_t down_type, + uint64_t gate_expert_bytes, uint64_t gate_row_bytes, + uint64_t down_expert_bytes, uint64_t down_row_bytes, + uint32_t expert_in_dim, uint32_t expert_mid_dim, + uint32_t out_dim) { + return cuda_iq2_ssd_grouped_raw_layout( + gate_type, down_type, gate_expert_bytes, gate_row_bytes, + down_expert_bytes, down_row_bytes, + expert_in_dim, expert_mid_dim, out_dim); +} + static int routed_moe_launch( ds4_gpu_tensor *out, ds4_gpu_tensor *gate, @@ -27446,6 +28055,59 @@ static int routed_moe_launch( } } + /* Resolve one immutable raw-weight binding before choosing MXFP4, the + * IQ2 grouped tier, or the established scratch pipeline. SSD loads + * publish compact weights plus an ID remap; resident models retain their + * full domain. The public routed wrapper holds the matching reader lease + * for every return below. */ + if (gate_expert_bytes == 0 || down_expert_bytes == 0 || + (uint64_t)n_total_expert > + UINT64_MAX / gate_expert_bytes || + (uint64_t)n_total_expert > + UINT64_MAX / down_expert_bytes) { + return 0; + } + const uint64_t gate_bytes = + (uint64_t)n_total_expert * gate_expert_bytes; + const uint64_t down_bytes = + (uint64_t)n_total_expert * down_expert_bytes; + if (gate_bytes > model_size - gate_offset || + gate_bytes > model_size - up_offset || + down_bytes > model_size - down_offset) { + return 0; + } + const uint64_t required_slot_count = (uint64_t)n_tokens * n_expert; + const int logical_tier = ds4_tensor_device_idx(out); + cuda_stream_selected_binding stream_binding = {}; + const int use_stream_selected_cache = + allow_streaming && g_ssd_streaming_mode && + cuda_stream_selected_binding_acquire( + &stream_binding, logical_tier, model_map, layer_index, + n_total_expert, gate_offset, up_offset, down_offset, + gate_expert_bytes, down_expert_bytes, required_slot_count); + if (g_ssd_streaming_mode && allow_streaming && + !use_stream_selected_cache) { + fprintf(stderr, + "ds4: CUDA streaming selected experts are unavailable for " + "layer %u\n", + layer_index); + return 0; + } + if (use_stream_selected_cache) selected = stream_binding.selected; + const char *gate_w = use_stream_selected_cache + ? stream_binding.gate + : cuda_resolve_weight_ptr(model_map, gate_offset, gate_bytes, + logical_tier, "moe_gate"); + const char *up_w = use_stream_selected_cache + ? stream_binding.up + : cuda_resolve_weight_ptr(model_map, up_offset, gate_bytes, + logical_tier, "moe_up"); + const char *down_w = use_stream_selected_cache + ? stream_binding.down + : cuda_resolve_weight_ptr(model_map, down_offset, down_bytes, + logical_tier, "moe_down"); + if (!gate_w || !up_w || !down_w) return 0; + /* Native MXFP4 routed experts use the vendored MMVQ decode kernels and * MMQ matrix kernels. On Blackwell the latter dispatch to FP4 MMA; older * CUDA devices use the mathematically equivalent DP4A implementation. @@ -27456,63 +28118,18 @@ static int routed_moe_launch( fprintf(stderr, "ds4: CUDA MXFP4 requires the MMQ backend\n"); return 0; } - const uint64_t gate_total = - (uint64_t)n_total_expert * gate_expert_bytes; - const uint64_t down_total = - (uint64_t)n_total_expert * down_expert_bytes; - if (gate_total > model_size - gate_offset || - gate_total > model_size - up_offset || - down_total > model_size - down_offset) { - return 0; - } - - const uint64_t slot_count = (uint64_t)n_tokens * n_expert; - const int logical_tier = ds4_tensor_device_idx(out); - const int use_stream_selected_cache = - allow_streaming && - g_ssd_streaming_mode && - g_stream_selected_cache.valid && - g_stream_selected_cache.logical_tier == logical_tier && - g_stream_selected_cache.model_map == model_map && - g_stream_selected_cache.layer == layer_index && - g_stream_selected_cache.n_total_expert == n_total_expert && - g_stream_selected_cache.slot_count >= slot_count && - g_stream_selected_cache.gate_offset == gate_offset && - g_stream_selected_cache.up_offset == up_offset && - g_stream_selected_cache.down_offset == down_offset && - g_stream_selected_cache.gate_expert_bytes == gate_expert_bytes && - g_stream_selected_cache.down_expert_bytes == down_expert_bytes && - g_stream_selected_cache.gate_ptr && - g_stream_selected_cache.up_ptr && - g_stream_selected_cache.down_ptr && - g_stream_selected_cache.slot_selected_tensor.ptr && - g_stream_selected_cache.slot_selected_tensor.bytes >= - slot_count * sizeof(int32_t); - if (g_ssd_streaming_mode && allow_streaming && - !use_stream_selected_cache) { - fprintf(stderr, - "ds4: CUDA streaming MXFP4 experts are unavailable for layer %u\n", - layer_index); + /* MMQ lazy initialization may leave this host thread current on a + * different physical device. Do not trust the logical-device cache + * after it; select the owner explicitly before any launch. */ + if (logical_tier < 0 || logical_tier >= g_n_gpus || + cudaSetDevice(g_gpu[logical_tier].device_id) != cudaSuccess) { + (void)cudaGetLastError(); return 0; } - - const ds4_gpu_tensor *mx_selected = use_stream_selected_cache ? - &g_stream_selected_cache.slot_selected_tensor : selected; + const ds4_gpu_tensor *mx_selected = selected; const uint32_t weight_experts = use_stream_selected_cache ? - g_stream_selected_cache.compact_count : n_total_expert; - const char *gate_w = use_stream_selected_cache ? - g_stream_selected_cache.gate_ptr : - cuda_resolve_weight_ptr(model_map, gate_offset, gate_total, - logical_tier, "mxfp4 moe gate"); - const char *up_w = use_stream_selected_cache ? - g_stream_selected_cache.up_ptr : - cuda_resolve_weight_ptr(model_map, up_offset, gate_total, - logical_tier, "mxfp4 moe up"); - const char *down_w = use_stream_selected_cache ? - g_stream_selected_cache.down_ptr : - cuda_resolve_weight_ptr(model_map, down_offset, down_total, - logical_tier, "mxfp4 moe down"); - if (!gate_w || !up_w || !down_w || weight_experts == 0u) return 0; + stream_binding.weight_domain : n_total_expert; + if (weight_experts == 0u) return 0; const cudaStream_t stream = n_tokens == 1u ? cuda_decode_stream() : (cudaStream_t)0; @@ -27543,7 +28160,7 @@ static int routed_moe_launch( stream); if (rc == 0) { const uint64_t mid_floats = - slot_count * expert_mid_dim; + required_slot_count * expert_mid_dim; moe_mmq_swiglu_weighted_clamp_kernel<<< (uint32_t)((mid_floats + 255u) / 256u), 256, 0, stream>>>( (float *)mid->ptr, @@ -27559,7 +28176,7 @@ static int routed_moe_launch( (const int32_t *)mx_selected->ptr, (float *)down->ptr, (int)out_dim, (int)expert_mid_dim, - (int)slot_count, (int)weight_experts, + (int)required_slot_count, (int)weight_experts, /*n_expert_used=*/1, stream); } if (rc == 0) { @@ -27582,6 +28199,187 @@ static int routed_moe_launch( rc, layer_index, n_tokens); return 0; } + + /* Opt-in GB10 grouped SSD prefill. The raw fused entry builds routing + * once for gate/up/down over the compact weight domain. Only its explicit + * pre-enqueue NOT_APPLICABLE result may enter the legacy scratch path; + * every other error is fenced and fails closed. */ + const int grouped_enabled = cuda_iq2_ssd_grouped_enabled(); + const int grouped_required = cuda_iq2_ssd_grouped_required(); + const int grouped_raw_layout = cuda_iq2_ssd_grouped_raw_layout( + gate_type, down_type, + gate_expert_bytes, gate_row_bytes, + down_expert_bytes, down_row_bytes, + expert_in_dim, expert_mid_dim, out_dim); + const int grouped_candidate_domain = + cuda_iq2_ssd_grouped_candidate_values( + iq2_path, g_ssd_streaming_mode, allow_streaming, + owned_filtered, n_tokens, n_expert, + stream_binding.top6_unique, grouped_raw_layout, + use_stream_selected_cache); + if (grouped_candidate_domain && + (grouped_enabled || grouped_required)) { + g_iq2_ssd_grouped_candidates.fetch_add( + 1, std::memory_order_relaxed); + const cudaStream_t grouped_stream = (cudaStream_t)0; + cudaStreamCaptureStatus capture = cudaStreamCaptureStatusNone; + const cudaError_t capture_err = + cudaStreamIsCapturing(grouped_stream, &capture); + const int capture_active = capture_err != cudaSuccess || + capture != cudaStreamCaptureStatusNone; + if (capture_err != cudaSuccess) (void)cudaGetLastError(); + const int tensor_devices_ok = logical_tier == 0 && + ds4_tensor_device_idx(gate) == logical_tier && + ds4_tensor_device_idx(up) == logical_tier && + ds4_tensor_device_idx(mid) == logical_tier && + ds4_tensor_device_idx(down) == logical_tier && + ds4_tensor_device_idx(x) == logical_tier && + ds4_tensor_device_idx(weights) == logical_tier && + stream_binding.selected && + ds4_tensor_device_idx(stream_binding.selected) == logical_tier; + /* ds4_mmq_init() may change the thread's physical CUDA device. It is + * part of preflight, and the owner is selected again before waiting + * on the upload token or enqueueing the fused pipeline. */ + const int mmq_ready = cuda_use_mmq(); + const int eligible = cuda_iq2_ssd_grouped_eligible_values( + grouped_enabled, + g_ssd_streaming_mode, + g_n_gpus == 1 && logical_tier == 0, + logical_tier >= 0 && logical_tier < DS4_MAX_GPUS && + g_cuda_is_gb10[logical_tier], + g_quality_mode, + owned_filtered, + capture_active, + mmq_ready, + n_tokens, + n_expert, + stream_binding.top6_unique, + grouped_raw_layout, + use_stream_selected_cache && tensor_devices_ok && + stream_binding.slot_base == 0u && + stream_binding.weight_domain > 0u && + stream_binding.weight_domain <= INT_MAX && + n_tokens <= INT_MAX && n_expert <= INT_MAX && + expert_in_dim <= INT_MAX && expert_mid_dim <= INT_MAX && + out_dim <= INT_MAX && + ((uint64_t)n_tokens * out_dim + 255u) / 256u <= + UINT32_MAX); + if (!eligible) { + if (grouped_required) { + g_iq2_ssd_grouped_failures.fetch_add( + 1, std::memory_order_relaxed); + g_iq2_ssd_grouped_required_failures.fetch_add( + 1, std::memory_order_relaxed); + fprintf(stderr, + "ds4: required CUDA IQ2 SSD grouped MMQ is not " + "eligible at layer %u (n_tokens=%u)\n", + layer_index, n_tokens); + return 0; + } + g_iq2_ssd_grouped_safe_fallbacks.fetch_add( + 1, std::memory_order_relaxed); + } else { + g_iq2_ssd_grouped_eligible.fetch_add( + 1, std::memory_order_relaxed); + if (cudaSetDevice(g_gpu[0].device_id) != cudaSuccess) { + (void)cudaGetLastError(); + g_iq2_ssd_grouped_failures.fetch_add( + 1, std::memory_order_relaxed); + if (grouped_required) { + g_iq2_ssd_grouped_required_failures.fetch_add( + 1, std::memory_order_relaxed); + } + return 0; + } + if (stream_binding.upload_event_value != 0) { + if (!cuda_stream_selected_wait_upload_on( + stream_binding.upload_event_value, + grouped_stream, + "IQ2 SSD grouped-MMQ upload wait")) { + g_iq2_ssd_grouped_failures.fetch_add( + 1, std::memory_order_relaxed); + if (grouped_required) { + g_iq2_ssd_grouped_required_failures.fetch_add( + 1, std::memory_order_relaxed); + } + return 0; + } + g_iq2_ssd_grouped_upload_waits.fetch_add( + 1, std::memory_order_relaxed); + } + if (!g_stream_selected_cache.valid || + g_stream_selected_cache.generation != + stream_binding.generation || + g_stream_selected_cache.upload_event_value != + stream_binding.upload_event_value) { + g_iq2_ssd_grouped_failures.fetch_add( + 1, std::memory_order_relaxed); + if (grouped_required) { + g_iq2_ssd_grouped_required_failures.fetch_add( + 1, std::memory_order_relaxed); + } + return 0; + } + g_iq2_ssd_grouped_attempts.fetch_add( + 1, std::memory_order_relaxed); + int rc = ds4_mmq_iq2_xxs_q2_K_moe_fused_raw( + gate_w, up_w, down_w, + (const float *)x->ptr, + (const int32_t *)selected->ptr, + (const float *)weights->ptr, + (float *)gate->ptr, (float *)up->ptr, + (float *)mid->ptr, (float *)down->ptr, + (int)expert_mid_dim, (int)expert_in_dim, (int)out_dim, + (int)n_tokens, (int)stream_binding.weight_domain, + (int)n_expert, clamp, grouped_stream); + if (rc == DS4_MMQ_NOT_APPLICABLE) { + g_iq2_ssd_grouped_not_applicable.fetch_add( + 1, std::memory_order_relaxed); + if (grouped_required) { + g_iq2_ssd_grouped_failures.fetch_add( + 1, std::memory_order_relaxed); + g_iq2_ssd_grouped_required_failures.fetch_add( + 1, std::memory_order_relaxed); + fprintf(stderr, + "ds4: required CUDA IQ2 SSD grouped MMQ was " + "not applicable at layer %u\n", + layer_index); + return 0; + } + g_iq2_ssd_grouped_safe_fallbacks.fetch_add( + 1, std::memory_order_relaxed); + } else { + if (rc == 0) { + const uint64_t n = (uint64_t)n_tokens * out_dim; + moe_mmq_sum_kernel<<< + (uint32_t)((n + 255u) / 256u), 256, 0, + grouped_stream>>>( + (float *)out->ptr, (const float *)down->ptr, + NULL, out_dim, n_expert, n_tokens, + /*guard_nonfinite=*/1); + rc = cuda_ok(cudaGetLastError(), + "IQ2 SSD grouped moe sum launch") ? 0 : -1; + } + if (rc != 0) { + g_iq2_ssd_grouped_failures.fetch_add( + 1, std::memory_order_relaxed); + if (grouped_required) { + g_iq2_ssd_grouped_required_failures.fetch_add( + 1, std::memory_order_relaxed); + } + fprintf(stderr, + "ds4: CUDA IQ2 SSD grouped MMQ failed closed " + "with rc=%d at layer %u\n", + rc, layer_index); + return 0; + } + g_iq2_ssd_grouped_completed.fetch_add( + 1, std::memory_order_relaxed); + return 1; + } + } + } + /* mmq routed-MoE prefill tier (ported from the Entrpi/ds4 fork). * IQ2_XXS gate/up pair (one shared activation quantize + routing * pass) -> SwiGLU + clamp + router weight -> Q2_K down, treating @@ -27592,12 +28390,11 @@ static int routed_moe_launch( * buffers are scratch there too). */ if (iq2_path && n_tokens > 1u && !owned_filtered && cuda_use_mmq() && !(g_ssd_streaming_mode && allow_streaming)) { - const uint64_t gate_total = (uint64_t)n_total_expert * gate_expert_bytes; - const uint64_t down_total = (uint64_t)n_total_expert * down_expert_bytes; - const int mmq_tier = ds4_tensor_device_idx(out); - const char *gate_w = cuda_resolve_weight_ptr(model_map, gate_offset, gate_total, mmq_tier, "moe gate mmq"); - const char *up_w = gate_w ? cuda_resolve_weight_ptr(model_map, up_offset, gate_total, mmq_tier, "moe up mmq") : NULL; - const char *down_w = up_w ? cuda_resolve_weight_ptr(model_map, down_offset, down_total, mmq_tier, "moe down mmq") : NULL; + if (logical_tier < 0 || logical_tier >= g_n_gpus || + cudaSetDevice(g_gpu[logical_tier].device_id) != cudaSuccess) { + (void)cudaGetLastError(); + return 0; + } if (down_w) { const uint64_t n_assignments = (uint64_t)n_tokens * n_expert; int rc = ds4_mmq_iq2_xxs_moe_pair( @@ -27654,59 +28451,6 @@ static int routed_moe_launch( * pairs by expert and uses Q4_K tile8 gate/up + down kernels * (`DS4_CUDA_MOE_NO_Q4_SORTED=1` restores the older * token-indexed decode-style prefill kernels). */ - const uint64_t gate_bytes = (uint64_t)n_total_expert * gate_expert_bytes; - const uint64_t down_bytes = (uint64_t)n_total_expert * down_expert_bytes; - if (gate_bytes > model_size - gate_offset || - gate_bytes > model_size - up_offset || - down_bytes > model_size - down_offset) { - return 0; - } - const uint64_t required_slot_count = (uint64_t)n_tokens * n_expert; - const int logical_tier = ds4_tensor_device_idx(out); - const int use_stream_selected_cache = - allow_streaming && - g_ssd_streaming_mode && - g_stream_selected_cache.valid && - g_stream_selected_cache.logical_tier == logical_tier && - g_stream_selected_cache.model_map == model_map && - g_stream_selected_cache.layer == layer_index && - g_stream_selected_cache.n_total_expert == n_total_expert && - g_stream_selected_cache.slot_count >= required_slot_count && - g_stream_selected_cache.gate_offset == gate_offset && - g_stream_selected_cache.up_offset == up_offset && - g_stream_selected_cache.down_offset == down_offset && - g_stream_selected_cache.gate_expert_bytes == gate_expert_bytes && - g_stream_selected_cache.down_expert_bytes == down_expert_bytes && - g_stream_selected_cache.gate_ptr && - g_stream_selected_cache.up_ptr && - g_stream_selected_cache.down_ptr && - g_stream_selected_cache.slot_selected_tensor.ptr && - g_stream_selected_cache.slot_selected_tensor.bytes >= - required_slot_count * sizeof(int32_t); - if (g_ssd_streaming_mode && allow_streaming && - !use_stream_selected_cache) { - fprintf(stderr, - "ds4: CUDA streaming selected experts are unavailable for layer %u\n", - layer_index); - return 0; - } - if (use_stream_selected_cache) { - selected = &g_stream_selected_cache.slot_selected_tensor; - } - const char *gate_w = use_stream_selected_cache ? - g_stream_selected_cache.gate_ptr : - cuda_resolve_weight_ptr(model_map, gate_offset, gate_bytes, - logical_tier, "moe_gate"); - const char *up_w = use_stream_selected_cache ? - g_stream_selected_cache.up_ptr : - cuda_resolve_weight_ptr(model_map, up_offset, gate_bytes, - logical_tier, "moe_up"); - const char *down_w = use_stream_selected_cache ? - g_stream_selected_cache.down_ptr : - cuda_resolve_weight_ptr(model_map, down_offset, down_bytes, - logical_tier, "moe_down"); - if (!gate_w || !up_w || !down_w) return 0; - int ok = 1; const uint32_t xq_blocks = expert_in_dim / CUDA_QK_K; const uint32_t midq_blocks = expert_mid_dim / CUDA_QK_K; @@ -29152,6 +29896,85 @@ extern "C" int ds4_gpu_routed_moe_owned_packed_combine_tensor( "owned routed_moe packed combine launch"); } +/* Central lifetime guard for every transient SSD binding consumer. It is + * deliberately outside routed_moe_launch(): all of that function's early + * success and error returns then pass through one consume-event publication, + * including Q4, MXFP4, grouped IQ2 and the legacy IQ2 fallback. */ +static int cuda_stream_selected_consumer_begin( + int allow_streaming, cudaStream_t stream, + uint64_t *generation_out) { + if (generation_out) *generation_out = 0; + if (!allow_streaming || !g_ssd_streaming_mode) { + return 1; + } + if (!generation_out || g_n_gpus != 1 || + cudaSetDevice(g_gpu[0].device_id) != cudaSuccess || + !cuda_stream_selected_consume_prepare()) { + (void)cudaGetLastError(); + return 0; + } + uint64_t generation = 0; + uint64_t upload_event = 0; + { + std::unique_lock lock(g_stream_selected_consume_mutex); + /* A single reusable event represents the whole consume frontier. + * Serialize host acquisitions, then chain this consumer behind the + * previous record before the event is reused on its stream. */ + g_stream_selected_consume_cv.wait(lock, [] { + return !g_stream_selected_writer_active && + g_stream_selected_consume_host_readers == 0u; + }); + if (!g_stream_selected_cache.valid || + g_stream_selected_cache.generation == 0) { + return 0; + } + if (g_stream_selected_consume_pending) { + if (!g_stream_selected_consume_done_event || + g_stream_selected_consume_owner_device != + g_gpu[0].device_id) { + return 0; + } + const cudaError_t consume_wait = cudaStreamWaitEvent( + stream, g_stream_selected_consume_done_event, 0); + if (consume_wait != cudaSuccess) { + fprintf(stderr, + "ds4: CUDA compact-cache consumer-chain wait " + "failed: %s\n", + cudaGetErrorString(consume_wait)); + (void)cudaGetLastError(); + return 0; + } + } + generation = g_stream_selected_cache.generation; + upload_event = g_stream_selected_cache.upload_event_value; + g_stream_selected_consume_host_readers++; + } + if (!cuda_stream_selected_wait_upload_on( + upload_event, stream, "selected-cache consumer upload wait")) { + std::lock_guard lock(g_stream_selected_consume_mutex); + g_stream_selected_consume_host_readers--; + g_stream_selected_consume_cv.notify_all(); + return 0; + } + *generation_out = generation; + return 1; +} + +static int cuda_stream_selected_consumer_end( + uint64_t generation, cudaStream_t stream) { + if (generation == 0) return 1; + const int recorded = + cuda_stream_selected_consume_record(generation, stream); + { + std::lock_guard lock(g_stream_selected_consume_mutex); + if (g_stream_selected_consume_host_readers != 0u) { + g_stream_selected_consume_host_readers--; + } + g_stream_selected_consume_cv.notify_all(); + } + return recorded; +} + extern "C" int ds4_gpu_routed_moe_one_tensor(ds4_gpu_tensor *out, ds4_gpu_tensor *gate, ds4_gpu_tensor *up, ds4_gpu_tensor *mid, ds4_gpu_tensor *down, const void *model_map, uint64_t model_size, uint64_t gate_offset, uint64_t up_offset, uint64_t down_offset, uint32_t gate_type, uint32_t down_type, uint64_t gate_expert_bytes, uint64_t gate_row_bytes, uint64_t down_expert_bytes, uint64_t down_row_bytes, uint32_t expert_in_dim, uint32_t expert_mid_dim, uint32_t out_dim, const ds4_gpu_tensor *selected, const ds4_gpu_tensor *weights, uint32_t n_total_expert, uint32_t n_expert, float clamp, const ds4_gpu_tensor *x, const ds4_gpu_tensor *add_in, uint32_t layer_index, @@ -29160,25 +29983,46 @@ extern "C" int ds4_gpu_routed_moe_one_tensor(ds4_gpu_tensor *out, ds4_gpu_tensor if (!ds4_gpu_add_tensor(out, out, add_in, (uint32_t)(out->bytes / sizeof(float)))) return 0; } - return routed_moe_launch(out, gate, up, mid, down, model_map, model_size, - gate_offset, up_offset, down_offset, - gate_type, down_type, - gate_expert_bytes, gate_row_bytes, - down_expert_bytes, down_row_bytes, - expert_in_dim, expert_mid_dim, out_dim, - selected, weights, n_total_expert, n_expert, clamp, x, - layer_index, 1, force_resident ? 0 : 1, 0); + const int allow_streaming = force_resident ? 0 : 1; + const cudaStream_t stream = cuda_decode_stream(); + uint64_t consume_generation = 0; + if (!cuda_stream_selected_consumer_begin( + allow_streaming, stream, &consume_generation)) return 0; + const int rc = routed_moe_launch( + out, gate, up, mid, down, model_map, model_size, + gate_offset, up_offset, down_offset, + gate_type, down_type, + gate_expert_bytes, gate_row_bytes, + down_expert_bytes, down_row_bytes, + expert_in_dim, expert_mid_dim, out_dim, + selected, weights, n_total_expert, n_expert, clamp, x, + layer_index, 1, allow_streaming, 0); + if (!cuda_stream_selected_consumer_end(consume_generation, stream)) { + return 0; + } + return rc; } extern "C" int ds4_gpu_routed_moe_batch_tensor(ds4_gpu_tensor *out, ds4_gpu_tensor *gate, ds4_gpu_tensor *up, ds4_gpu_tensor *mid, ds4_gpu_tensor *down, const void *model_map, uint64_t model_size, uint64_t gate_offset, uint64_t up_offset, uint64_t down_offset, uint32_t gate_type, uint32_t down_type, uint64_t gate_expert_bytes, uint64_t gate_row_bytes, uint64_t down_expert_bytes, uint64_t down_row_bytes, uint32_t expert_in_dim, uint32_t expert_mid_dim, uint32_t out_dim, const ds4_gpu_tensor *selected, const ds4_gpu_tensor *weights, uint32_t n_total_expert, uint32_t n_expert, float clamp, const ds4_gpu_tensor *x, uint32_t layer_index, uint32_t n_tokens, bool *mid_is_f16, bool force_resident) { if (mid_is_f16) *mid_is_f16 = false; - return routed_moe_launch(out, gate, up, mid, down, model_map, model_size, - gate_offset, up_offset, down_offset, - gate_type, down_type, - gate_expert_bytes, gate_row_bytes, - down_expert_bytes, down_row_bytes, - expert_in_dim, expert_mid_dim, out_dim, - selected, weights, n_total_expert, n_expert, clamp, x, - layer_index, n_tokens, force_resident ? 0 : 1, 0); + const int allow_streaming = force_resident ? 0 : 1; + const cudaStream_t stream = n_tokens == 1u + ? cuda_decode_stream() : (cudaStream_t)0; + uint64_t consume_generation = 0; + if (!cuda_stream_selected_consumer_begin( + allow_streaming, stream, &consume_generation)) return 0; + const int rc = routed_moe_launch( + out, gate, up, mid, down, model_map, model_size, + gate_offset, up_offset, down_offset, + gate_type, down_type, + gate_expert_bytes, gate_row_bytes, + down_expert_bytes, down_row_bytes, + expert_in_dim, expert_mid_dim, out_dim, + selected, weights, n_total_expert, n_expert, clamp, x, + layer_index, n_tokens, allow_streaming, 0); + if (!cuda_stream_selected_consumer_end(consume_generation, stream)) { + return 0; + } + return rc; } extern "C" int ds4_gpu_routed_moe_batch_owned_tensor( @@ -29934,7 +30778,7 @@ static int cuda_stream_selected_cache_begin_load_impl( int *submitted_any_out) { if (upload_event_out) *upload_event_out = 0; if (submitted_any_out) *submitted_any_out = 0; - cuda_stream_selected_cache_invalidate(); + cuda_stream_selected_writer_guard writer; if (!g_ssd_streaming_mode) return 1; if (!cuda_stream_selected_ranges_valid(table) || !selected_ids || slot_count == 0) { @@ -29984,6 +30828,17 @@ static int cuda_stream_selected_cache_begin_load_impl( } slot_ids[i] = compact; } + int top6_unique = (slot_count % 6u) == 0u; + for (uint32_t base = 0; top6_unique && base < slot_count; base += 6u) { + for (uint32_t i = 0; top6_unique && i < 6u; i++) { + for (uint32_t j = i + 1u; j < 6u; j++) { + if (selected_ids[base + i] == selected_ids[base + j]) { + top6_unique = 0; + break; + } + } + } + } if (compact_ids.empty() || compact_ids.size() > UINT32_MAX) return 0; const uint64_t compact_count = compact_ids.size(); if (compact_count > UINT64_MAX / table->gate_expert_bytes || @@ -29998,8 +30853,23 @@ static int cuda_stream_selected_cache_begin_load_impl( g_stream_selected_cache.up_ptr || g_stream_selected_cache.down_ptr || g_stream_selected_cache.slot_selected_ptr)) { - cuda_stream_selected_cache_release(); + if (!cuda_stream_selected_cache_free_storage_writer()) return 0; + } + const uint64_t remap_bytes = (uint64_t)slot_count * sizeof(int32_t); + const int resize_binding = + (g_stream_selected_cache.gate_ptr && + g_stream_selected_cache.gate_capacity < gate_bytes) || + (g_stream_selected_cache.up_ptr && + g_stream_selected_cache.up_capacity < gate_bytes) || + (g_stream_selected_cache.down_ptr && + g_stream_selected_cache.down_capacity < down_bytes) || + (g_stream_selected_cache.slot_selected_ptr && + g_stream_selected_cache.slot_selected_capacity < remap_bytes); + if (resize_binding && !cuda_stream_selected_consume_drain()) { + cuda_stream_selected_cache_invalidate(); + return 0; } + if (resize_binding) cuda_stream_selected_upload_drain(); if (ds4_gpu_set_current_device(logical_tier) != 0 || !cuda_stream_selected_ensure_bytes( &g_stream_selected_cache.gate_ptr, @@ -30117,6 +30987,13 @@ static int cuda_stream_selected_cache_begin_load_impl( return 0; } if (!copied) { + /* The legacy copy path may use blocking cudaMemcpy before an upload + * stream exists. It therefore needs the host-side lease boundary; + * the batched path imports the same event directly into its stream. */ + if (!cuda_stream_selected_consume_drain()) { + cuda_stream_selected_cache_invalidate(); + return 0; + } g_stream_selected_batch_io_legacy++; for (uint32_t i = 0; i < compact_ids.size(); i++) { const uint64_t expert = (uint32_t)compact_ids[i]; @@ -30165,6 +31042,14 @@ static int cuda_stream_selected_cache_begin_load_impl( g_stream_selected_cache.n_total_expert = table->n_total_expert; g_stream_selected_cache.slot_count = slot_count; g_stream_selected_cache.compact_count = (uint32_t)compact_count; + g_stream_selected_cache.slot_base = 0; + g_stream_selected_cache.weight_domain = (uint32_t)compact_count; + g_stream_selected_cache.top6_unique = top6_unique; + uint64_t generation = ++g_stream_selected_cache_generation; + if (generation == 0) generation = ++g_stream_selected_cache_generation; + g_stream_selected_cache.generation = generation; + g_stream_selected_cache.upload_event_value = + upload_event_out ? *upload_event_out : 0; g_stream_selected_cache.gate_offset = table->gate_offset; g_stream_selected_cache.up_offset = table->up_offset; g_stream_selected_cache.down_offset = table->down_offset; @@ -30176,7 +31061,10 @@ static int cuda_stream_selected_cache_begin_load_impl( (uint64_t)slot_count * sizeof(int32_t); g_stream_selected_cache.slot_selected_tensor.owner = 0; g_stream_selected_cache.slot_selected_tensor.device_id = logical_tier; - g_stream_selected_cache.valid = 1; + /* Publish all binding metadata atomically with respect to consumer + * acquisition. The upload itself may still be in flight; its exact + * completion token is part of the metadata guarded by this release. */ + writer.publish_valid(); return 1; } @@ -37225,6 +38113,12 @@ extern "C" int ds4_gpu_wait_selected_readback_ready(uint64_t event_value, const extern "C" int ds4_gpu_stream_expert_cache_wait_selected_upload( uint64_t event_value, const char *label) { + return cuda_stream_selected_wait_upload_on( + event_value, cuda_decode_stream(), label); +} + +static int cuda_stream_selected_wait_upload_on( + uint64_t event_value, cudaStream_t stream, const char *label) { if (event_value == 0) return 1; if (event_value != g_stream_selected_upload_event_value || !g_stream_selected_upload_done_event || @@ -37238,7 +38132,7 @@ extern "C" int ds4_gpu_stream_expert_cache_wait_selected_upload( return 0; } const cudaError_t err = cudaStreamWaitEvent( - cuda_decode_stream(), g_stream_selected_upload_done_event, 0); + stream, g_stream_selected_upload_done_event, 0); if (err != cudaSuccess) { fprintf(stderr, "ds4: CUDA %s failed: %s\n", @@ -37253,15 +38147,16 @@ extern "C" int ds4_gpu_stream_expert_cache_wait_selected_upload( } extern "C" int ds4_gpu_cuda_stream_selected_event_abort(void) { + /* Stop new consumers and wait until every routed caller has published + * its consume frontier before the device-wide abort drain. */ + cuda_stream_selected_writer_guard writer; int device = g_stream_selected_event_owner_device; if (device < 0 && g_n_gpus == 1) device = g_gpu[0].device_id; if (device >= 0 && cudaSetDevice(device) != cudaSuccess) { (void)cudaGetLastError(); - cuda_stream_selected_cache_invalidate(); return 0; } const cudaError_t err = cudaDeviceSynchronize(); - cuda_stream_selected_cache_invalidate(); uint64_t compute_value = ++g_stream_selected_compute_event_value; if (compute_value == 0) ++g_stream_selected_compute_event_value; uint64_t upload_value = ++g_stream_selected_upload_event_value; @@ -37276,6 +38171,286 @@ extern "C" int ds4_gpu_cuda_stream_selected_event_abort(void) { return 1; } +__global__ static void cuda_stream_selected_lease_delay_store_kernel( + int32_t *dst, int32_t value, uint64_t delay_clocks) { + if (blockIdx.x != 0 || threadIdx.x != 0) return; + const uint64_t begin = clock64(); + while (clock64() - begin < delay_clocks) { + __nanosleep(64u); + } + *dst = value; +} + +/* End-to-end compact-binding protocol oracle. It publishes generation A + * through the real begin_load_impl call site, acquires it through the real + * consumer and binding helpers, then starts a second writer. A probe inside + * guard proves that writer B is waiting while reader A is held. Once A + * records consume_done, B imports that event into the nonblocking upload + * stream, overwrites the remap, and publishes generation B before upload + * completion. A second consumer validates both generation and upload token. + * fails if writer exclusion, publish_valid(), consume record/wait, generation + * matching, or upload completion are bypassed. */ +extern "C" int ds4_cuda_test_iq2_ssd_grouped_lease(void) { + if (g_n_gpus != 1 || + cudaSetDevice(g_gpu[0].device_id) != cudaSuccess || + !cuda_stream_selected_consume_drain() || + !cuda_stream_selected_consume_prepare()) { + (void)cudaGetLastError(); + return 0; + } + + const int saved_ssd_streaming_mode = g_ssd_streaming_mode; + const int saved_model_fd = g_model_fd; + const void *saved_model_fd_host_base = g_model_fd_host_base; + const int saved_model_direct_fd = g_model_direct_fd; + const uint64_t saved_model_direct_align = g_model_direct_align; + const uint64_t saved_model_file_size = g_model_file_size; + cuda_stream_selected_cache_release(); + { + cuda_stream_selected_writer_guard writer; + g_ssd_streaming_mode = 1; + } + /* Force begin_load_impl through its host-map copy contract regardless of + * any model fd configured by a surrounding process. No saved descriptor + * is closed; the exact globals are restored at the single cleanup exit. */ + g_model_fd = -1; + g_model_fd_host_base = NULL; + g_model_direct_fd = -1; + g_model_direct_align = 1; + + unsigned char *test_model = NULL; + int32_t *host_words = NULL; + const uint64_t expert_bytes = 16u; + const uint32_t weight_domain = 6u; + const uint32_t test_total_expert = 8u; + const uint64_t full_table_bytes = + expert_bytes * test_total_expert; + const uint64_t test_gate_offset = 0u; + const uint64_t test_up_offset = full_table_bytes; + const uint64_t test_down_offset = 2u * full_table_bytes; + const uint64_t test_model_bytes = 3u * full_table_bytes; + g_model_file_size = test_model_bytes; + int ok = cuda_stream_selected_stage_pool_alloc(4096u) && + cuda_stream_selected_event_pipeline_ensure() && + cudaMallocHost((void **)&test_model, + (size_t)test_model_bytes) == cudaSuccess && + cudaMallocHost((void **)&host_words, + 3u * sizeof(*host_words)) == cudaSuccess; + if (!ok) (void)cudaGetLastError(); + if (ok) { + for (uint64_t i = 0; i < test_model_bytes; i++) { + test_model[i] = (unsigned char)(i * 29u + 7u); + } + host_words[0] = 0x13572468; + host_words[1] = 0x24681357; + host_words[2] = 0; + } + + const void *test_model_map = test_model; + const uint32_t test_layer = 0x7f00u; + const int32_t selected_ids[6] = {0, 1, 2, 3, 4, 5}; + ds4_gpu_stream_expert_table table = {}; + table.model_map = test_model_map; + table.model_size = test_model_bytes; + table.layer = test_layer; + table.n_total_expert = test_total_expert; + table.gate_offset = test_gate_offset; + table.up_offset = test_up_offset; + table.down_offset = test_down_offset; + table.gate_expert_bytes = expert_bytes; + table.down_expert_bytes = expert_bytes; + if (ok) { + ok = cuda_stream_selected_cache_begin_load_impl( + &table, selected_ids, weight_domain, + /*upload_event_out=*/NULL, + /*submitted_any_out=*/NULL); + } + const uint64_t generation_a = ok + ? g_stream_selected_cache.generation : 0; + if (ok && (!g_stream_selected_cache.valid || generation_a == 0 || + g_stream_selected_cache.upload_event_value != 0 || + g_stream_selected_cache.weight_domain != weight_domain)) { + ok = 0; + } + + uint64_t reader_generation = 0; + int reader_held = 0; + cuda_stream_selected_binding binding_a = {}; + if (ok) { + ok = cuda_stream_selected_consumer_begin( + /*allow_streaming=*/1, (cudaStream_t)0, + &reader_generation) && + reader_generation == generation_a; + reader_held = reader_generation != 0; + } + if (ok) { + ok = cuda_stream_selected_binding_acquire( + &binding_a, 0, test_model_map, test_layer, + test_total_expert, + test_gate_offset, test_up_offset, test_down_offset, + expert_bytes, expert_bytes, weight_domain) && + binding_a.generation == generation_a && + binding_a.weight_domain == weight_domain && + binding_a.selected && + binding_a.selected->ptr == + g_stream_selected_cache.slot_selected_ptr; + } + if (ok) { + /* A is deliberately longer than writer B's 1M-clock store. If B + * omits the consume fence it writes first and A restores sentinel A; + * with the fence B cannot start until A has completed. */ + cuda_stream_selected_lease_delay_store_kernel<<<1, 1, 0, 0>>>( + (int32_t *)binding_a.selected->ptr, + host_words[0], 50000000u); + ok = cudaGetLastError() == cudaSuccess; + } + + uint64_t generation_b = ++g_stream_selected_cache_generation; + if (generation_b == 0) { + generation_b = ++g_stream_selected_cache_generation; + } + std::atomic writer_waiting{0}; + std::atomic writer_acquired{0}; + std::atomic writer_published{0}; + std::atomic writer_ok{1}; + std::thread writer_thread; + if (reader_held) { + try { + writer_thread = std::thread([&] { + cuda_stream_selected_writer_guard writer(&writer_waiting); + writer_acquired.store(1, std::memory_order_release); + int local_ok = + cudaSetDevice(g_gpu[0].device_id) == cudaSuccess && + cuda_stream_selected_consume_wait_on_upload(); + if (local_ok) { + /* Keep publication observably ahead of upload completion. + * Without consumer B's upload-token wait, its default- + * stream read runs after consume A but during this delay + * and deterministically observes sentinel A. */ + cuda_stream_selected_lease_delay_store_kernel<<< + 1, 1, 0, g_stream_selected_upload_stream>>>( + g_stream_selected_cache.slot_selected_ptr, + host_words[1], 1000000u); + local_ok = cudaGetLastError() == cudaSuccess; + } + if (local_ok) { + local_ok = cudaEventRecord( + g_stream_selected_upload_done_event, + g_stream_selected_upload_stream) == cudaSuccess; + } + if (local_ok) { + uint64_t upload_value = + ++g_stream_selected_upload_event_value; + if (upload_value == 0) { + upload_value = + ++g_stream_selected_upload_event_value; + } + g_stream_selected_cache.generation = generation_b; + g_stream_selected_cache.upload_event_value = + upload_value; + writer.publish_valid(); + writer_published.store(1, std::memory_order_release); + } else { + (void)cudaGetLastError(); + } + writer_ok.store(local_ok, std::memory_order_release); + }); + } catch (...) { + ok = 0; + } + } + + if (writer_thread.joinable()) { + uint32_t yields = 0; + while (!writer_waiting.load(std::memory_order_acquire) && + yields++ < 1000000u) { + std::this_thread::yield(); + } + /* waiting is set while the writer owns the protocol mutex, before + * its predicate wait. With reader A held, acquired must still be 0 + * and generation A must remain the published binding. */ + if (!writer_waiting.load(std::memory_order_acquire) || + writer_acquired.load(std::memory_order_acquire) || + !g_stream_selected_cache.valid || + g_stream_selected_cache.generation != generation_a) { + ok = 0; + } + } + + if (reader_held) { + if (!cuda_stream_selected_consumer_end( + reader_generation, (cudaStream_t)0)) { + ok = 0; + } + reader_held = 0; + } + if (writer_thread.joinable()) writer_thread.join(); + if (!writer_ok.load(std::memory_order_acquire) || + !writer_acquired.load(std::memory_order_acquire) || + !writer_published.load(std::memory_order_acquire) || + !g_stream_selected_cache.valid || + g_stream_selected_cache.generation != generation_b) { + ok = 0; + } + + uint64_t reader_generation_b = 0; + cuda_stream_selected_binding binding_b = {}; + int reader_b_held = 0; + if (writer_published.load(std::memory_order_acquire)) { + const int began = cuda_stream_selected_consumer_begin( + /*allow_streaming=*/1, (cudaStream_t)0, + &reader_generation_b); + reader_b_held = reader_generation_b != 0; + if (!began || reader_generation_b != generation_b || + !cuda_stream_selected_binding_acquire( + &binding_b, 0, test_model_map, test_layer, + test_total_expert, + test_gate_offset, test_up_offset, test_down_offset, + expert_bytes, expert_bytes, weight_domain) || + binding_b.generation != generation_b || + binding_b.upload_event_value == 0) { + ok = 0; + } else if (cudaMemcpyAsync( + &host_words[2], binding_b.selected->ptr, + sizeof(host_words[2]), cudaMemcpyDeviceToHost, + (cudaStream_t)0) != cudaSuccess) { + (void)cudaGetLastError(); + ok = 0; + } + } + if (reader_b_held) { + if (!cuda_stream_selected_consumer_end( + reader_generation_b, (cudaStream_t)0)) { + ok = 0; + } + reader_b_held = 0; + } + if (cudaStreamSynchronize((cudaStream_t)0) != cudaSuccess || + !host_words || host_words[2] != host_words[1]) { + (void)cudaGetLastError(); + ok = 0; + } + + if (!ok) { + (void)cudaGetLastError(); + (void)cudaDeviceSynchronize(); + } + cuda_stream_selected_cache_release(); + if (test_model) (void)cudaFreeHost(test_model); + if (host_words) (void)cudaFreeHost(host_words); + cuda_stream_selected_stage_release(); + { + cuda_stream_selected_writer_guard writer; + g_ssd_streaming_mode = saved_ssd_streaming_mode; + } + g_model_fd = saved_model_fd; + g_model_fd_host_base = saved_model_fd_host_base; + g_model_direct_fd = saved_model_direct_fd; + g_model_direct_align = saved_model_direct_align; + g_model_file_size = saved_model_file_size; + return ok; +} + /* Device-backed ordering oracle for CI/DGX bring-up. It exercises the same * event objects and streams as decode without requiring a GGUF or mutating * process-environment policy after its once_flag has resolved. */ @@ -37417,9 +38592,11 @@ extern "C" void ds4_gpu_set_glm_model(bool enabled) { } extern "C" void ds4_gpu_set_ssd_streaming(bool enabled) { + cuda_stream_selected_writer_guard writer; g_ssd_streaming_mode = enabled ? 1 : 0; - cuda_stream_selected_cache_invalidate(); - if (!g_ssd_streaming_mode) cuda_stream_selected_cache_release(); + if (!g_ssd_streaming_mode) { + (void)cuda_stream_selected_cache_free_storage_writer(); + } } extern "C" void ds4_gpu_set_streaming_expert_cache_budget(uint32_t experts) { @@ -37435,8 +38612,10 @@ extern "C" uint32_t ds4_gpu_stream_expert_cache_configured_count(void) { } extern "C" uint32_t ds4_gpu_stream_expert_cache_current_count(void) { - return g_stream_selected_cache.valid ? - g_stream_selected_cache.compact_count : 0; + std::lock_guard lock(g_stream_selected_consume_mutex); + return !g_stream_selected_writer_active && + g_stream_selected_cache.valid + ? g_stream_selected_cache.compact_count : 0; } extern "C" void ds4_gpu_stream_expert_cache_reset_route_hotness(void) { diff --git a/ds4_gpu.h b/ds4_gpu.h index c2a4bed41..102e7eec5 100644 --- a/ds4_gpu.h +++ b/ds4_gpu.h @@ -320,6 +320,23 @@ typedef struct ds4_cuda_stream_selected_event_pipeline_report { int required; int oracle; } ds4_cuda_stream_selected_event_pipeline_report; +typedef struct ds4_cuda_iq2_ssd_grouped_report { + uint64_t candidates; + uint64_t eligible; + uint64_t attempts; + uint64_t completed; + uint64_t not_applicable; + uint64_t safe_fallbacks; + uint64_t failures; + uint64_t required_failures; + uint64_t upload_waits; + uint64_t lease_waits; + uint64_t lease_records; + uint64_t lease_drains; + int enabled; + int required; + int stats; +} ds4_cuda_iq2_ssd_grouped_report; /* CUDA-only policy/planner/scatter hooks. The policy hook takes explicit * values so tests do not need to mutate process environment around once_flag * initialization. */ @@ -339,6 +356,27 @@ int ds4_cuda_test_stream_selected_event_pipeline(void); int ds4_cuda_test_stream_selected_owner_device(void); void ds4_cuda_stream_selected_event_pipeline_get_report( ds4_cuda_stream_selected_event_pipeline_report *report); +int ds4_cuda_test_iq2_ssd_grouped_policy( + int enable, int disable, int require, int stats, + int *enabled_out, int *required_out, int *stats_out); +int ds4_cuda_test_iq2_ssd_grouped_eligibility( + int enabled, int ssd_streaming, int single_gpu, int gb10, + int quality, int owned_filtered, int capture, int mmq, + uint32_t n_tokens, uint32_t n_expert, int top6_unique, + int raw_layout, int binding_valid); +int ds4_cuda_test_iq2_ssd_grouped_candidate( + int iq2_path, int ssd_streaming, int allow_streaming, + int owned_filtered, uint32_t n_tokens, uint32_t n_expert, + int top6_unique, int raw_layout, int binding_valid); +int ds4_cuda_test_iq2_ssd_grouped_raw_layout( + uint32_t gate_type, uint32_t down_type, + uint64_t gate_expert_bytes, uint64_t gate_row_bytes, + uint64_t down_expert_bytes, uint64_t down_row_bytes, + uint32_t expert_in_dim, uint32_t expert_mid_dim, + uint32_t out_dim); +int ds4_cuda_test_iq2_ssd_grouped_lease(void); +void ds4_cuda_iq2_ssd_grouped_get_report( + ds4_cuda_iq2_ssd_grouped_report *report); int ds4_gpu_cuda_stream_selected_event_pipeline_enabled(void); int ds4_gpu_cuda_stream_selected_event_pipeline_required(void); int ds4_gpu_cuda_stream_selected_set_owner_device(void); diff --git a/tests/test_gpu_model_cache.c b/tests/test_gpu_model_cache.c index 8e459b1ee..daab21226 100644 --- a/tests/test_gpu_model_cache.c +++ b/tests/test_gpu_model_cache.c @@ -77,6 +77,76 @@ int main(void) { 1, 1, 1, 1, &enabled, &required, &oracle) && enabled == 0 && required == 0 && oracle == 0, "selected-expert event-pipeline disable-dominant policy"); + int stats = -1; + CHECK(ds4_cuda_test_iq2_ssd_grouped_policy( + 1, 0, 0, 0, &enabled, &required, &stats) && + enabled == 1 && required == 0 && stats == 0, + "IQ2 SSD grouped-MMQ enable policy"); + CHECK(ds4_cuda_test_iq2_ssd_grouped_policy( + 0, 0, 1, 1, &enabled, &required, &stats) && + enabled == 1 && required == 1 && stats == 1, + "IQ2 SSD grouped-MMQ require and stats policy"); + CHECK(ds4_cuda_test_iq2_ssd_grouped_policy( + 1, 1, 1, 1, &enabled, &required, &stats) && + enabled == 0 && required == 0 && stats == 0, + "IQ2 SSD grouped-MMQ disable-dominant policy"); + CHECK(ds4_cuda_test_iq2_ssd_grouped_eligibility( + 1, 1, 1, 1, 0, 0, 0, 1, + 32u, 6u, 1, 1, 1), + "IQ2 SSD grouped-MMQ canonical eligibility"); + CHECK(!ds4_cuda_test_iq2_ssd_grouped_candidate( + 1, 1, 1, 0, 1u, 6u, 1, 1, 1) && + !ds4_cuda_test_iq2_ssd_grouped_candidate( + 1, 1, 1, 0, 31u, 6u, 1, 1, 1) && + ds4_cuda_test_iq2_ssd_grouped_candidate( + 1, 1, 1, 0, 32u, 6u, 1, 1, 1), + "IQ2 SSD grouped-MMQ REQUIRE candidate excludes decode/tail"); + CHECK(!ds4_cuda_test_iq2_ssd_grouped_candidate( + 1, 1, 1, 0, 32u, 5u, 1, 1, 1) && + !ds4_cuda_test_iq2_ssd_grouped_candidate( + 1, 1, 1, 0, 32u, 6u, 0, 1, 1) && + !ds4_cuda_test_iq2_ssd_grouped_candidate( + 1, 1, 1, 0, 32u, 6u, 1, 0, 1) && + !ds4_cuda_test_iq2_ssd_grouped_candidate( + 1, 1, 1, 0, 32u, 6u, 1, 1, 0), + "IQ2 SSD grouped-MMQ REQUIRE excludes non-candidate layouts"); + CHECK(!ds4_cuda_test_iq2_ssd_grouped_eligibility( + 1, 1, 1, 1, 0, 0, 0, 1, + 31u, 6u, 1, 1, 1) && + !ds4_cuda_test_iq2_ssd_grouped_eligibility( + 1, 1, 1, 1, 0, 0, 0, 1, + 32u, 5u, 1, 1, 1) && + !ds4_cuda_test_iq2_ssd_grouped_eligibility( + 1, 1, 1, 1, 0, 0, 0, 1, + 32u, 6u, 0, 1, 1) && + !ds4_cuda_test_iq2_ssd_grouped_eligibility( + 1, 1, 1, 1, 1, 0, 0, 1, + 32u, 6u, 1, 1, 1) && + !ds4_cuda_test_iq2_ssd_grouped_eligibility( + 1, 1, 1, 1, 0, 0, 1, 1, + 32u, 6u, 1, 1, 1) && + !ds4_cuda_test_iq2_ssd_grouped_eligibility( + 1, 1, 1, 1, 0, 0, 0, 1, + 32u, 6u, 1, 1, 0), + "IQ2 SSD grouped-MMQ exclusion matrix"); + const uint64_t iq2_row = 16u * 66u; + const uint64_t iq2_expert = 2048u * iq2_row; + const uint64_t q2_row = 8u * 84u; + const uint64_t q2_expert = 4096u * q2_row; + CHECK(ds4_cuda_test_iq2_ssd_grouped_raw_layout( + 16u, 10u, iq2_expert, iq2_row, + q2_expert, q2_row, 4096u, 2048u, 4096u), + "IQ2 SSD grouped-MMQ canonical raw layout"); + CHECK(!ds4_cuda_test_iq2_ssd_grouped_raw_layout( + 12u, 10u, iq2_expert, iq2_row, + q2_expert, q2_row, 4096u, 2048u, 4096u) && + !ds4_cuda_test_iq2_ssd_grouped_raw_layout( + 16u, 10u, iq2_expert, iq2_row + 2u, + q2_expert, q2_row, 4096u, 2048u, 4096u) && + !ds4_cuda_test_iq2_ssd_grouped_raw_layout( + 16u, 10u, iq2_expert, iq2_row, + q2_expert + 84u, q2_row, 4096u, 2048u, 4096u), + "IQ2 SSD grouped-MMQ raw-layout rejection matrix"); CHECK(ds4_cuda_test_stream_selected_batch_plan(), "selected-expert batched-I/O planner"); @@ -90,6 +160,18 @@ int main(void) { } CHECK(ds4_gpu_init(), "ds4_gpu_init"); + ds4_cuda_iq2_ssd_grouped_report lease_before; + memset(&lease_before, 0, sizeof(lease_before)); + ds4_cuda_iq2_ssd_grouped_get_report(&lease_before); + CHECK(ds4_cuda_test_iq2_ssd_grouped_lease(), + "IQ2 SSD compact-binding consume/reuse lease oracle"); + ds4_cuda_iq2_ssd_grouped_report lease_after; + memset(&lease_after, 0, sizeof(lease_after)); + ds4_cuda_iq2_ssd_grouped_get_report(&lease_after); + CHECK(lease_after.lease_records > lease_before.lease_records && + lease_after.lease_waits > lease_before.lease_waits && + lease_after.lease_drains > lease_before.lease_drains, + "IQ2 SSD compact-binding lease coverage counters"); CHECK(ds4_cuda_test_stream_selected_event_pipeline(), "selected-expert compute/readback/upload event ordering oracle"); ds4_cuda_stream_selected_event_pipeline_report event_report; From f49cc0e482aaf001dd6beff5443c56c3e517a945 Mon Sep 17 00:00:00 2001 From: Giorgio Oppo Date: Fri, 14 Aug 2026 21:41:48 +0200 Subject: [PATCH 034/189] cuda: model persistent expert cache planning --- ds4_cuda.cu | 1123 +++++++++++++++++++++++++++++++++- ds4_gpu.h | 31 + tests/test_gpu_model_cache.c | 61 ++ 3 files changed, 1203 insertions(+), 12 deletions(-) diff --git a/ds4_cuda.cu b/ds4_cuda.cu index ad6d09734..da80aba5b 100644 --- a/ds4_cuda.cu +++ b/ds4_cuda.cu @@ -27,6 +27,7 @@ #include #include #include +#include #include "cuda/mmq/ds4_mmq.h" #include "cuda/mmq/ds4_repack.h" @@ -591,6 +592,29 @@ static std::atomic g_iq2_ssd_grouped_upload_waits{0}; static std::atomic g_iq2_ssd_grouped_lease_waits{0}; static std::atomic g_iq2_ssd_grouped_lease_records{0}; static std::atomic g_iq2_ssd_grouped_lease_drains{0}; +static std::once_flag g_stream_expert_persistent_once; +static int g_stream_expert_persistent_enabled; +static int g_stream_expert_persistent_required; +static int g_stream_expert_persistent_stats; +static int g_stream_expert_persistent_oracle; +static int g_stream_expert_persistent_report_registered; +static std::atomic g_stream_expert_persistent_plan_attempts{0}; +static std::atomic g_stream_expert_persistent_plans_built{0}; +static std::atomic g_stream_expert_persistent_commits{0}; +static std::atomic g_stream_expert_persistent_rollbacks{0}; +static std::atomic g_stream_expert_persistent_hits{0}; +static std::atomic g_stream_expert_persistent_misses{0}; +static std::atomic g_stream_expert_persistent_duplicates{0}; +static std::atomic g_stream_expert_persistent_free_assignments{0}; +static std::atomic g_stream_expert_persistent_evictions{0}; +static std::atomic g_stream_expert_persistent_rejects{0}; +static std::atomic g_stream_expert_persistent_budget_rejects{0}; +static std::atomic g_stream_expert_persistent_class_rejects{0}; +static std::atomic g_stream_expert_persistent_protected_rejects{0}; +static std::atomic g_stream_expert_persistent_key_misses{0}; +static std::atomic g_stream_expert_persistent_overflow_rejects{0}; +static std::atomic g_stream_expert_persistent_oracle_runs{0}; +static std::atomic g_stream_expert_persistent_oracle_failures{0}; static int cuda_ok(cudaError_t err, const char *what); static void cuda_stream_selected_event_pipeline_release(void); @@ -2946,6 +2970,1093 @@ extern "C" int ds4_cuda_test_stream_selected_event_env_value( return cuda_stream_selected_env_value_enabled(value); } +/* This tagged compatibility type is normally declared beside the CUDA + * streaming implementation below. The phase-one host planner lives here so + * its value semantics and policy hooks can be tested before any CUDA arena is + * introduced. */ +typedef struct ds4_gpu_stream_expert_table { + const void *model_map; + uint64_t model_size; + uint32_t layer; + uint32_t n_total_expert; + uint64_t gate_offset; + uint64_t up_offset; + uint64_t down_offset; + uint64_t gate_expert_bytes; + uint64_t down_expert_bytes; +} ds4_gpu_stream_expert_table; + +struct ds4_cuda_stream_expert_persistent_report; +typedef struct ds4_cuda_stream_expert_persistent_report + ds4_cuda_stream_expert_persistent_report; +typedef struct { + uint64_t plan_attempts; + uint64_t plans_built; + uint64_t commits; + uint64_t rollbacks; + uint64_t hits; + uint64_t misses; + uint64_t duplicates; + uint64_t free_assignments; + uint64_t evictions; + uint64_t rejects; + uint64_t budget_rejects; + uint64_t class_rejects; + uint64_t protected_rejects; + uint64_t key_misses; + uint64_t overflow_rejects; + uint64_t oracle_runs; + uint64_t oracle_failures; + int enabled; + int required; + int stats; + int oracle; +} cuda_stream_expert_persistent_report_layout; + +/* Phase one of the CUDA resident-expert cache is deliberately host-only. + * It defines and tests the transaction which a later device arena will + * execute, without allocating storage or changing the transient cache path. + * An arena has one immutable byte-size class. Full entry keys prevent a + * layer, model, or table-layout change from being mistaken for a hit. */ +typedef struct { + uint64_t gate_expert_bytes; + uint64_t down_expert_bytes; + uint64_t bytes_per_slot; + uint64_t arena_bytes; + uint32_t capacity; +} cuda_stream_expert_persistent_class; + +typedef struct { + const void *model_map; + uint64_t model_size; + uint64_t gate_offset; + uint64_t up_offset; + uint64_t down_offset; + uint64_t gate_expert_bytes; + uint64_t down_expert_bytes; + uint32_t layer; + uint32_t n_total_expert; + uint32_t expert_id; +} cuda_stream_expert_persistent_key; + +typedef struct { + cuda_stream_expert_persistent_key key; + uint64_t last_use; + uint32_t pin_count; + int valid; +} cuda_stream_expert_persistent_entry; + +typedef struct { + cuda_stream_expert_persistent_class size_class; + std::vector slots; + /* Stored in reverse order so pop_back() deterministically yields the + * lowest free slot. */ + std::vector free_slots; + uint64_t lru_clock; +} cuda_stream_expert_persistent_state; + +typedef struct { + cuda_stream_expert_persistent_key key; + cuda_stream_expert_persistent_key victim; + uint32_t slot; + int had_victim; +} cuda_stream_expert_persistent_load; + +typedef struct { + cuda_stream_expert_persistent_state next; + std::vector remap; + std::vector loads; + uint32_t slot_base; + uint32_t weight_domain; + uint32_t unique_count; + uint32_t hit_count; + uint32_t miss_count; + uint32_t duplicate_count; + int built; +} cuda_stream_expert_persistent_plan; + +static int cuda_stream_expert_persistent_add_u64( + uint64_t a, uint64_t b, uint64_t *out) { + if (!out || a > UINT64_MAX - b) return 0; + *out = a + b; + return 1; +} + +static int cuda_stream_expert_persistent_mul_u64( + uint64_t a, uint64_t b, uint64_t *out) { + if (!out || (a != 0 && b > UINT64_MAX / a)) return 0; + *out = a * b; + return 1; +} + +static int cuda_stream_expert_persistent_class_make( + cuda_stream_expert_persistent_class *out, + uint32_t capacity, + uint64_t gate_expert_bytes, + uint64_t down_expert_bytes) { + if (!out || capacity == 0 || gate_expert_bytes == 0 || + down_expert_bytes == 0) { + return 0; + } + uint64_t two_gate = 0; + uint64_t bytes_per_slot = 0; + uint64_t arena_bytes = 0; + if (!cuda_stream_expert_persistent_mul_u64( + gate_expert_bytes, 2u, &two_gate) || + !cuda_stream_expert_persistent_add_u64( + two_gate, down_expert_bytes, &bytes_per_slot) || + !cuda_stream_expert_persistent_mul_u64( + bytes_per_slot, capacity, &arena_bytes) || + arena_bytes > SIZE_MAX) { + return 0; + } + out->gate_expert_bytes = gate_expert_bytes; + out->down_expert_bytes = down_expert_bytes; + out->bytes_per_slot = bytes_per_slot; + out->arena_bytes = arena_bytes; + out->capacity = capacity; + return 1; +} + +static int cuda_stream_expert_persistent_class_equal( + const cuda_stream_expert_persistent_class *a, + const cuda_stream_expert_persistent_class *b) { + return a && b && + a->gate_expert_bytes == b->gate_expert_bytes && + a->down_expert_bytes == b->down_expert_bytes && + a->bytes_per_slot == b->bytes_per_slot && + a->arena_bytes == b->arena_bytes && + a->capacity == b->capacity; +} + +static int cuda_stream_expert_persistent_key_equal( + const cuda_stream_expert_persistent_key *a, + const cuda_stream_expert_persistent_key *b) { + return a && b && a->model_map == b->model_map && + a->model_size == b->model_size && + a->gate_offset == b->gate_offset && + a->up_offset == b->up_offset && + a->down_offset == b->down_offset && + a->gate_expert_bytes == b->gate_expert_bytes && + a->down_expert_bytes == b->down_expert_bytes && + a->layer == b->layer && + a->n_total_expert == b->n_total_expert && + a->expert_id == b->expert_id; +} + +static int cuda_stream_expert_persistent_table_range_ok( + uint64_t offset, uint64_t expert_bytes, + uint32_t n_total_expert, uint64_t model_size) { + uint64_t table_bytes = 0; + return cuda_stream_expert_persistent_mul_u64( + expert_bytes, n_total_expert, &table_bytes) && + offset <= model_size && table_bytes <= model_size - offset; +} + +static int cuda_stream_expert_persistent_key_make( + cuda_stream_expert_persistent_key *out, + const ds4_gpu_stream_expert_table *table, + int32_t expert_id) { + if (!out || !table || !table->model_map || table->model_size == 0 || + table->n_total_expert == 0 || table->gate_expert_bytes == 0 || + table->down_expert_bytes == 0 || expert_id < 0 || + (uint32_t)expert_id >= table->n_total_expert || + !cuda_stream_expert_persistent_table_range_ok( + table->gate_offset, table->gate_expert_bytes, + table->n_total_expert, table->model_size) || + !cuda_stream_expert_persistent_table_range_ok( + table->up_offset, table->gate_expert_bytes, + table->n_total_expert, table->model_size) || + !cuda_stream_expert_persistent_table_range_ok( + table->down_offset, table->down_expert_bytes, + table->n_total_expert, table->model_size)) { + return 0; + } + out->model_map = table->model_map; + out->model_size = table->model_size; + out->gate_offset = table->gate_offset; + out->up_offset = table->up_offset; + out->down_offset = table->down_offset; + out->gate_expert_bytes = table->gate_expert_bytes; + out->down_expert_bytes = table->down_expert_bytes; + out->layer = table->layer; + out->n_total_expert = table->n_total_expert; + out->expert_id = (uint32_t)expert_id; + return 1; +} + +static int cuda_stream_expert_persistent_state_init( + cuda_stream_expert_persistent_state *state, + uint32_t capacity, + uint64_t gate_expert_bytes, + uint64_t down_expert_bytes) { + if (!state || !cuda_stream_expert_persistent_class_make( + &state->size_class, capacity, + gate_expert_bytes, down_expert_bytes)) { + return 0; + } + try { + state->slots.assign(capacity, {}); + state->free_slots.clear(); + state->free_slots.reserve(capacity); + for (uint32_t slot = capacity; slot != 0; slot--) { + state->free_slots.push_back(slot - 1u); + } + } catch (...) { + state->slots.clear(); + state->free_slots.clear(); + memset(&state->size_class, 0, sizeof(state->size_class)); + return 0; + } + state->lru_clock = 0; + return 1; +} + +static int cuda_stream_expert_persistent_state_valid( + const cuda_stream_expert_persistent_state *state) { + if (!state || state->size_class.capacity == 0 || + state->slots.size() != state->size_class.capacity || + state->free_slots.size() > state->slots.size()) { + return 0; + } + std::vector free_seen; + try { + free_seen.assign(state->slots.size(), 0u); + } catch (...) { + return 0; + } + for (uint32_t slot : state->free_slots) { + if (slot >= state->slots.size() || free_seen[slot] || + state->slots[slot].valid || state->slots[slot].pin_count != 0) { + return 0; + } + free_seen[slot] = 1u; + } + for (size_t slot = 0; slot < state->slots.size(); slot++) { + const cuda_stream_expert_persistent_entry *entry = + &state->slots[slot]; + if (entry->valid) { + if (free_seen[slot] || entry->last_use == 0 || + entry->last_use > state->lru_clock) { + return 0; + } + } else if (!free_seen[slot] || entry->last_use != 0 || + entry->pin_count != 0) { + return 0; + } + } + return 1; +} + +static int cuda_stream_expert_persistent_state_equal( + const cuda_stream_expert_persistent_state *a, + const cuda_stream_expert_persistent_state *b) { + if (!a || !b || + !cuda_stream_expert_persistent_class_equal( + &a->size_class, &b->size_class) || + a->lru_clock != b->lru_clock || + a->free_slots != b->free_slots || + a->slots.size() != b->slots.size()) { + return 0; + } + for (size_t i = 0; i < a->slots.size(); i++) { + const cuda_stream_expert_persistent_entry *ea = &a->slots[i]; + const cuda_stream_expert_persistent_entry *eb = &b->slots[i]; + if (ea->valid != eb->valid || ea->last_use != eb->last_use || + ea->pin_count != eb->pin_count || + (ea->valid && !cuda_stream_expert_persistent_key_equal( + &ea->key, &eb->key))) { + return 0; + } + } + return 1; +} + +static int cuda_stream_expert_persistent_state_copy( + cuda_stream_expert_persistent_state *dst, + const cuda_stream_expert_persistent_state *src) { + if (!dst || !src) return 0; + try { + *dst = *src; + } catch (...) { + return 0; + } + return 1; +} + +static int cuda_stream_expert_persistent_find_key( + const cuda_stream_expert_persistent_state *state, + const cuda_stream_expert_persistent_key *key, + uint32_t *slot_out) { + if (!state || !key) return 0; + for (uint32_t slot = 0; slot < state->slots.size(); slot++) { + const cuda_stream_expert_persistent_entry *entry = + &state->slots[slot]; + if (entry->valid && cuda_stream_expert_persistent_key_equal( + &entry->key, key)) { + if (slot_out) *slot_out = slot; + return 1; + } + } + return 0; +} + +static int cuda_stream_expert_persistent_touch( + cuda_stream_expert_persistent_state *state, uint32_t slot) { + if (!state || slot >= state->slots.size() || + !state->slots[slot].valid || state->lru_clock == UINT64_MAX) { + return 0; + } + state->slots[slot].last_use = ++state->lru_clock; + return 1; +} + +static int cuda_stream_expert_persistent_pin_key( + cuda_stream_expert_persistent_state *state, + const cuda_stream_expert_persistent_key *key, + int pin) { + uint32_t slot = 0; + if (!state || !cuda_stream_expert_persistent_find_key( + state, key, &slot)) { + return 0; + } + if (pin) { + if (state->slots[slot].pin_count == UINT32_MAX) return 0; + state->slots[slot].pin_count++; + } else { + if (state->slots[slot].pin_count == 0) return 0; + state->slots[slot].pin_count--; + } + return 1; +} + +static void cuda_stream_expert_persistent_note_reject( + std::atomic *specific) { + g_stream_expert_persistent_rejects.fetch_add( + 1, std::memory_order_relaxed); + if (specific) specific->fetch_add(1, std::memory_order_relaxed); +} + +/* Build against a private state copy. Until commit(), both LRU touches and + * victim selection are invisible to the live cache, so every rejection and + * every abandoned upload plan is a true rollback. */ +static int cuda_stream_expert_persistent_plan_build( + const cuda_stream_expert_persistent_state *state, + const ds4_gpu_stream_expert_table *table, + const int32_t *selected_ids, + uint32_t n_selected, + cuda_stream_expert_persistent_plan *plan) { + g_stream_expert_persistent_plan_attempts.fetch_add( + 1, std::memory_order_relaxed); + if (!state || !table || !selected_ids || n_selected == 0 || !plan || + !cuda_stream_expert_persistent_state_valid(state)) { + cuda_stream_expert_persistent_note_reject(NULL); + return 0; + } + + cuda_stream_expert_persistent_class request_class = {}; + if (!cuda_stream_expert_persistent_class_make( + &request_class, state->size_class.capacity, + table->gate_expert_bytes, table->down_expert_bytes)) { + cuda_stream_expert_persistent_note_reject( + &g_stream_expert_persistent_overflow_rejects); + return 0; + } + if (!cuda_stream_expert_persistent_class_equal( + &state->size_class, &request_class)) { + cuda_stream_expert_persistent_note_reject( + &g_stream_expert_persistent_class_rejects); + return 0; + } + + std::vector unique_keys; + std::vector input_unique; + uint32_t duplicate_count = 0; + try { + unique_keys.reserve(n_selected); + input_unique.reserve(n_selected); + for (uint32_t i = 0; i < n_selected; i++) { + cuda_stream_expert_persistent_key key = {}; + if (!cuda_stream_expert_persistent_key_make( + &key, table, selected_ids[i])) { + cuda_stream_expert_persistent_note_reject( + &g_stream_expert_persistent_overflow_rejects); + return 0; + } + uint32_t unique_index = UINT32_MAX; + for (uint32_t u = 0; u < unique_keys.size(); u++) { + if (cuda_stream_expert_persistent_key_equal( + &unique_keys[u], &key)) { + unique_index = u; + break; + } + } + if (unique_index == UINT32_MAX) { + if (unique_keys.size() >= UINT32_MAX) { + cuda_stream_expert_persistent_note_reject( + &g_stream_expert_persistent_overflow_rejects); + return 0; + } + unique_index = (uint32_t)unique_keys.size(); + unique_keys.push_back(key); + } else { + duplicate_count++; + } + input_unique.push_back(unique_index); + } + } catch (...) { + cuda_stream_expert_persistent_note_reject(NULL); + return 0; + } + if (unique_keys.size() > state->size_class.capacity) { + cuda_stream_expert_persistent_note_reject( + &g_stream_expert_persistent_budget_rejects); + return 0; + } + + cuda_stream_expert_persistent_plan candidate = {}; + if (!cuda_stream_expert_persistent_state_copy( + &candidate.next, state)) { + cuda_stream_expert_persistent_note_reject(NULL); + return 0; + } + std::vector protected_slots; + std::vector unique_slots; + try { + protected_slots.assign(candidate.next.slots.size(), 0u); + unique_slots.assign(unique_keys.size(), UINT32_MAX); + candidate.remap.resize(n_selected); + candidate.loads.reserve(unique_keys.size()); + } catch (...) { + cuda_stream_expert_persistent_note_reject(NULL); + return 0; + } + for (uint32_t slot = 0; slot < candidate.next.slots.size(); slot++) { + if (candidate.next.slots[slot].pin_count != 0) { + protected_slots[slot] = 1u; + } + } + + uint32_t hit_count = 0; + uint32_t miss_count = 0; + uint32_t key_miss_count = 0; + uint32_t free_assignment_count = 0; + uint32_t eviction_count = 0; + /* Resolve every hit before considering a victim. A request may put a + * cold miss before a resident key; a one-pass planner would otherwise be + * able to evict that later requested resident slot. */ + for (uint32_t u = 0; u < unique_keys.size(); u++) { + uint32_t slot = 0; + if (cuda_stream_expert_persistent_find_key( + &candidate.next, &unique_keys[u], &slot)) { + protected_slots[slot] = 1u; + if (!cuda_stream_expert_persistent_touch( + &candidate.next, slot)) { + cuda_stream_expert_persistent_note_reject( + &g_stream_expert_persistent_overflow_rejects); + return 0; + } + unique_slots[u] = slot; + hit_count++; + continue; + } + miss_count++; + for (const cuda_stream_expert_persistent_entry &entry : + candidate.next.slots) { + if (entry.valid && + entry.key.expert_id == unique_keys[u].expert_id && + !cuda_stream_expert_persistent_key_equal( + &entry.key, &unique_keys[u])) { + key_miss_count++; + break; + } + } + } + + /* Only slots outside the complete request hit-set and outside active + * leases are eligible LRU victims. Newly assigned miss slots are also + * protected so later misses in the transaction cannot replace them. */ + for (uint32_t u = 0; u < unique_keys.size(); u++) { + if (unique_slots[u] != UINT32_MAX) continue; + uint32_t slot = 0; + int had_victim = 0; + cuda_stream_expert_persistent_key victim = {}; + if (!candidate.next.free_slots.empty()) { + slot = candidate.next.free_slots.back(); + candidate.next.free_slots.pop_back(); + if (slot >= candidate.next.slots.size() || + candidate.next.slots[slot].valid || + candidate.next.slots[slot].pin_count != 0) { + cuda_stream_expert_persistent_note_reject(NULL); + return 0; + } + free_assignment_count++; + } else { + uint64_t oldest = UINT64_MAX; + slot = UINT32_MAX; + for (uint32_t s = 0; s < candidate.next.slots.size(); s++) { + const cuda_stream_expert_persistent_entry *entry = + &candidate.next.slots[s]; + if (!entry->valid || protected_slots[s] || + entry->pin_count != 0) { + continue; + } + if (slot == UINT32_MAX || entry->last_use < oldest || + (entry->last_use == oldest && s < slot)) { + oldest = entry->last_use; + slot = s; + } + } + if (slot == UINT32_MAX) { + cuda_stream_expert_persistent_note_reject( + &g_stream_expert_persistent_protected_rejects); + return 0; + } + victim = candidate.next.slots[slot].key; + had_victim = 1; + eviction_count++; + } + cuda_stream_expert_persistent_entry *entry = + &candidate.next.slots[slot]; + entry->key = unique_keys[u]; + entry->valid = 1; + entry->pin_count = 0; + if (!cuda_stream_expert_persistent_touch(&candidate.next, slot)) { + cuda_stream_expert_persistent_note_reject( + &g_stream_expert_persistent_overflow_rejects); + return 0; + } + protected_slots[slot] = 1u; + unique_slots[u] = slot; + cuda_stream_expert_persistent_load load = {}; + load.key = unique_keys[u]; + load.victim = victim; + load.slot = slot; + load.had_victim = had_victim; + try { + candidate.loads.push_back(load); + } catch (...) { + cuda_stream_expert_persistent_note_reject(NULL); + return 0; + } + } + + uint32_t min_slot = UINT32_MAX; + uint32_t max_slot = 0; + for (uint32_t slot : unique_slots) { + if (slot == UINT32_MAX || slot >= candidate.next.slots.size()) { + cuda_stream_expert_persistent_note_reject(NULL); + return 0; + } + min_slot = std::min(min_slot, slot); + max_slot = std::max(max_slot, slot); + } + if (min_slot == UINT32_MAX || max_slot < min_slot) { + cuda_stream_expert_persistent_note_reject(NULL); + return 0; + } + const uint64_t domain64 = + (uint64_t)max_slot - min_slot + 1u; + if (domain64 == 0 || domain64 > UINT32_MAX) { + cuda_stream_expert_persistent_note_reject( + &g_stream_expert_persistent_overflow_rejects); + return 0; + } + for (uint32_t i = 0; i < n_selected; i++) { + const uint32_t unique_index = input_unique[i]; + if (unique_index >= unique_slots.size() || + unique_slots[unique_index] < min_slot) { + cuda_stream_expert_persistent_note_reject(NULL); + return 0; + } + candidate.remap[i] = unique_slots[unique_index] - min_slot; + if (candidate.remap[i] >= domain64) { + cuda_stream_expert_persistent_note_reject(NULL); + return 0; + } + } + if (!cuda_stream_expert_persistent_state_valid(&candidate.next)) { + cuda_stream_expert_persistent_note_reject(NULL); + return 0; + } + candidate.slot_base = min_slot; + candidate.weight_domain = (uint32_t)domain64; + candidate.unique_count = (uint32_t)unique_keys.size(); + candidate.hit_count = hit_count; + candidate.miss_count = miss_count; + candidate.duplicate_count = duplicate_count; + candidate.built = 1; + try { + *plan = std::move(candidate); + } catch (...) { + cuda_stream_expert_persistent_note_reject(NULL); + return 0; + } + + g_stream_expert_persistent_plans_built.fetch_add( + 1, std::memory_order_relaxed); + g_stream_expert_persistent_hits.fetch_add( + hit_count, std::memory_order_relaxed); + g_stream_expert_persistent_misses.fetch_add( + miss_count, std::memory_order_relaxed); + g_stream_expert_persistent_duplicates.fetch_add( + duplicate_count, std::memory_order_relaxed); + g_stream_expert_persistent_free_assignments.fetch_add( + free_assignment_count, std::memory_order_relaxed); + g_stream_expert_persistent_evictions.fetch_add( + eviction_count, std::memory_order_relaxed); + g_stream_expert_persistent_key_misses.fetch_add( + key_miss_count, std::memory_order_relaxed); + return 1; +} + +static int cuda_stream_expert_persistent_plan_commit( + cuda_stream_expert_persistent_state *state, + cuda_stream_expert_persistent_plan *plan) { + if (!state || !plan || !plan->built || + !cuda_stream_expert_persistent_state_valid(&plan->next)) { + return 0; + } + state->size_class = plan->next.size_class; + state->slots.swap(plan->next.slots); + state->free_slots.swap(plan->next.free_slots); + state->lru_clock = plan->next.lru_clock; + plan->built = 0; + g_stream_expert_persistent_commits.fetch_add( + 1, std::memory_order_relaxed); + return 1; +} + +static void cuda_stream_expert_persistent_plan_rollback( + cuda_stream_expert_persistent_plan *plan) { + if (!plan || !plan->built) return; + plan->built = 0; + plan->next.slots.clear(); + plan->next.free_slots.clear(); + plan->remap.clear(); + plan->loads.clear(); + g_stream_expert_persistent_rollbacks.fetch_add( + 1, std::memory_order_relaxed); +} + +static void cuda_stream_expert_persistent_resolve_policy( + int enable, int disable, int require, int stats, int oracle, + int *enabled_out, int *required_out, int *stats_out, + int *oracle_out) { + const int disabled = disable != 0; + if (enabled_out) { + *enabled_out = !disabled && (enable || require || oracle); + } + if (required_out) *required_out = !disabled && require; + if (stats_out) *stats_out = !disabled && stats; + if (oracle_out) *oracle_out = !disabled && oracle; +} + +extern "C" int ds4_cuda_test_stream_expert_persistent_env_value( + const char *value) { + return cuda_stream_selected_env_value_enabled(value); +} + +extern "C" int ds4_cuda_test_stream_expert_persistent_policy( + int enable, int disable, int require, int stats, int oracle, + int *enabled_out, int *required_out, int *stats_out, + int *oracle_out) { + if (!enabled_out || !required_out || !stats_out || !oracle_out) return 0; + cuda_stream_expert_persistent_resolve_policy( + enable, disable, require, stats, oracle, + enabled_out, required_out, stats_out, oracle_out); + return 1; +} + +static void cuda_stream_expert_persistent_report_at_exit(void) { + fprintf(stderr, + "ds4: CUDA persistent expert planner: attempts=%llu built=%llu " + "commits=%llu rollbacks=%llu hits=%llu misses=%llu " + "duplicates=%llu free=%llu evictions=%llu rejects=%llu " + "budget_rejects=%llu class_rejects=%llu protected_rejects=%llu " + "key_misses=%llu overflow_rejects=%llu oracle=%llu/%llu\n", + (unsigned long long)g_stream_expert_persistent_plan_attempts.load(), + (unsigned long long)g_stream_expert_persistent_plans_built.load(), + (unsigned long long)g_stream_expert_persistent_commits.load(), + (unsigned long long)g_stream_expert_persistent_rollbacks.load(), + (unsigned long long)g_stream_expert_persistent_hits.load(), + (unsigned long long)g_stream_expert_persistent_misses.load(), + (unsigned long long)g_stream_expert_persistent_duplicates.load(), + (unsigned long long)g_stream_expert_persistent_free_assignments.load(), + (unsigned long long)g_stream_expert_persistent_evictions.load(), + (unsigned long long)g_stream_expert_persistent_rejects.load(), + (unsigned long long)g_stream_expert_persistent_budget_rejects.load(), + (unsigned long long)g_stream_expert_persistent_class_rejects.load(), + (unsigned long long)g_stream_expert_persistent_protected_rejects.load(), + (unsigned long long)g_stream_expert_persistent_key_misses.load(), + (unsigned long long)g_stream_expert_persistent_overflow_rejects.load(), + (unsigned long long)g_stream_expert_persistent_oracle_runs.load(), + (unsigned long long)g_stream_expert_persistent_oracle_failures.load()); +} + +static void cuda_stream_expert_persistent_init(void) { + const int enable = cuda_stream_selected_env_flag( + "DS4_CUDA_ENABLE_STREAMING_EXPERT_PERSISTENT_CACHE"); + const int disable = cuda_stream_selected_env_flag( + "DS4_CUDA_DISABLE_STREAMING_EXPERT_PERSISTENT_CACHE") || + cuda_stream_selected_env_flag( + "DS4_CUDA_NO_STREAMING_EXPERT_PERSISTENT_CACHE"); + const int require = cuda_stream_selected_env_flag( + "DS4_CUDA_REQUIRE_STREAMING_EXPERT_PERSISTENT_CACHE"); + const int stats = cuda_stream_selected_env_flag( + "DS4_CUDA_STREAMING_EXPERT_PERSISTENT_CACHE_STATS"); + const int oracle = cuda_stream_selected_env_flag( + "DS4_CUDA_STREAMING_EXPERT_PERSISTENT_CACHE_ORACLE"); + cuda_stream_expert_persistent_resolve_policy( + enable, disable, require, stats, oracle, + &g_stream_expert_persistent_enabled, + &g_stream_expert_persistent_required, + &g_stream_expert_persistent_stats, + &g_stream_expert_persistent_oracle); + if ((enable || require || stats || oracle) && + !g_stream_expert_persistent_report_registered) { + g_stream_expert_persistent_report_registered = 1; + (void)atexit(cuda_stream_expert_persistent_report_at_exit); + } + if (g_stream_expert_persistent_enabled) { + fprintf(stderr, + "ds4: CUDA persistent expert planner enabled%s%s\n", + g_stream_expert_persistent_required ? " (required)" : "", + g_stream_expert_persistent_oracle ? " with oracle" : ""); + } else if (disable && (enable || require || oracle)) { + fprintf(stderr, + "ds4: CUDA persistent expert planner disabled by rollback " + "override\n"); + } +} + +extern "C" void ds4_cuda_stream_expert_persistent_get_report( + ds4_cuda_stream_expert_persistent_report *report) { + if (!report) return; + std::call_once(g_stream_expert_persistent_once, + cuda_stream_expert_persistent_init); + cuda_stream_expert_persistent_report_layout out = {}; + out.plan_attempts = g_stream_expert_persistent_plan_attempts.load(); + out.plans_built = g_stream_expert_persistent_plans_built.load(); + out.commits = g_stream_expert_persistent_commits.load(); + out.rollbacks = g_stream_expert_persistent_rollbacks.load(); + out.hits = g_stream_expert_persistent_hits.load(); + out.misses = g_stream_expert_persistent_misses.load(); + out.duplicates = g_stream_expert_persistent_duplicates.load(); + out.free_assignments = + g_stream_expert_persistent_free_assignments.load(); + out.evictions = g_stream_expert_persistent_evictions.load(); + out.rejects = g_stream_expert_persistent_rejects.load(); + out.budget_rejects = + g_stream_expert_persistent_budget_rejects.load(); + out.class_rejects = g_stream_expert_persistent_class_rejects.load(); + out.protected_rejects = + g_stream_expert_persistent_protected_rejects.load(); + out.key_misses = g_stream_expert_persistent_key_misses.load(); + out.overflow_rejects = + g_stream_expert_persistent_overflow_rejects.load(); + out.oracle_runs = g_stream_expert_persistent_oracle_runs.load(); + out.oracle_failures = + g_stream_expert_persistent_oracle_failures.load(); + out.enabled = g_stream_expert_persistent_enabled; + out.required = g_stream_expert_persistent_required; + out.stats = g_stream_expert_persistent_stats; + out.oracle = g_stream_expert_persistent_oracle; + memcpy(report, &out, sizeof(out)); +} + +static int cuda_stream_expert_persistent_test_table_make( + ds4_gpu_stream_expert_table *table, + const void *model_map, + uint32_t layer, + uint32_t n_total_expert, + uint64_t gate_expert_bytes, + uint64_t down_expert_bytes) { + if (!table || !model_map || n_total_expert == 0) return 0; + uint64_t gate_table_bytes = 0; + uint64_t down_table_bytes = 0; + uint64_t down_offset = 0; + uint64_t model_size = 0; + if (!cuda_stream_expert_persistent_mul_u64( + gate_expert_bytes, n_total_expert, &gate_table_bytes) || + !cuda_stream_expert_persistent_mul_u64( + down_expert_bytes, n_total_expert, &down_table_bytes) || + !cuda_stream_expert_persistent_mul_u64( + gate_table_bytes, 2u, &down_offset) || + !cuda_stream_expert_persistent_add_u64( + down_offset, down_table_bytes, &model_size)) { + return 0; + } + memset(table, 0, sizeof(*table)); + table->model_map = model_map; + table->model_size = model_size; + table->layer = layer; + table->n_total_expert = n_total_expert; + table->gate_offset = 0; + table->up_offset = gate_table_bytes; + table->down_offset = down_offset; + table->gate_expert_bytes = gate_expert_bytes; + table->down_expert_bytes = down_expert_bytes; + return 1; +} + +static int cuda_stream_expert_persistent_test_basic(void) { + unsigned char model_marker = 0; + ds4_gpu_stream_expert_table table = {}; + cuda_stream_expert_persistent_state state = {}; + cuda_stream_expert_persistent_state before = {}; + if (!cuda_stream_expert_persistent_test_table_make( + &table, &model_marker, 7u, 8u, 16u, 8u) || + !cuda_stream_expert_persistent_state_init(&state, 4u, 16u, 8u) || + !cuda_stream_expert_persistent_state_copy(&before, &state)) { + return 0; + } + + const int32_t cold_ids[3] = {2, 3, 2}; + cuda_stream_expert_persistent_plan cold = {}; + if (!cuda_stream_expert_persistent_plan_build( + &state, &table, cold_ids, 3u, &cold) || + !cuda_stream_expert_persistent_state_equal(&state, &before) || + cold.unique_count != 2u || cold.hit_count != 0u || + cold.miss_count != 2u || cold.duplicate_count != 1u || + cold.loads.size() != 2u || cold.loads[0].slot != 0u || + cold.loads[1].slot != 1u || cold.slot_base != 0u || + cold.weight_domain != 2u || cold.remap.size() != 3u || + cold.remap[0] != 0u || cold.remap[1] != 1u || + cold.remap[2] != 0u || + !cuda_stream_expert_persistent_plan_commit(&state, &cold) || + !cuda_stream_expert_persistent_state_valid(&state)) { + return 0; + } + + const int32_t fill_ids[2] = {4, 5}; + cuda_stream_expert_persistent_plan fill = {}; + if (!cuda_stream_expert_persistent_plan_build( + &state, &table, fill_ids, 2u, &fill) || + fill.loads.size() != 2u || fill.loads[0].slot != 2u || + fill.loads[1].slot != 3u || + !cuda_stream_expert_persistent_plan_commit(&state, &fill)) { + return 0; + } + + if (!cuda_stream_expert_persistent_state_copy(&before, &state)) return 0; + const int32_t sparse_ids[3] = {3, 5, 3}; + cuda_stream_expert_persistent_plan sparse = {}; + if (!cuda_stream_expert_persistent_plan_build( + &state, &table, sparse_ids, 3u, &sparse) || + !cuda_stream_expert_persistent_state_equal(&state, &before) || + sparse.unique_count != 2u || sparse.hit_count != 2u || + sparse.miss_count != 0u || sparse.duplicate_count != 1u || + !sparse.loads.empty() || sparse.slot_base != 1u || + sparse.weight_domain != 3u || sparse.remap.size() != 3u || + sparse.remap[0] != 0u || sparse.remap[1] != 2u || + sparse.remap[2] != 0u || + !cuda_stream_expert_persistent_plan_commit(&state, &sparse) || + !cuda_stream_expert_persistent_state_valid(&state)) { + return 0; + } + return 1; +} + +static int cuda_stream_expert_persistent_test_protection(void) { + unsigned char model_marker = 0; + ds4_gpu_stream_expert_table table = {}; + cuda_stream_expert_persistent_state state = {}; + if (!cuda_stream_expert_persistent_test_table_make( + &table, &model_marker, 11u, 8u, 16u, 8u) || + !cuda_stream_expert_persistent_state_init(&state, 3u, 16u, 8u)) { + return 0; + } + const int32_t initial_ids[3] = {0, 1, 2}; + cuda_stream_expert_persistent_plan initial = {}; + if (!cuda_stream_expert_persistent_plan_build( + &state, &table, initial_ids, 3u, &initial) || + !cuda_stream_expert_persistent_plan_commit(&state, &initial)) { + return 0; + } + + cuda_stream_expert_persistent_key key0 = {}; + cuda_stream_expert_persistent_key key1 = {}; + cuda_stream_expert_persistent_key key2 = {}; + if (!cuda_stream_expert_persistent_key_make(&key0, &table, 0) || + !cuda_stream_expert_persistent_key_make(&key1, &table, 1) || + !cuda_stream_expert_persistent_key_make(&key2, &table, 2)) { + return 0; + } + cuda_stream_expert_persistent_state before = {}; + if (!cuda_stream_expert_persistent_state_copy(&before, &state)) return 0; + + /* The hit is deliberately ordered after the miss. Planning all hits + * first must protect resident slot zero before the miss selects a victim. */ + const int32_t miss_then_hit[2] = {3, 0}; + cuda_stream_expert_persistent_plan requested = {}; + if (!cuda_stream_expert_persistent_plan_build( + &state, &table, miss_then_hit, 2u, &requested) || + requested.hit_count != 1u || requested.miss_count != 1u || + requested.loads.size() != 1u || requested.loads[0].slot != 1u || + !requested.loads[0].had_victim || + requested.loads[0].victim.expert_id != 1u || + requested.slot_base != 0u || requested.weight_domain != 2u || + requested.remap.size() != 2u || requested.remap[0] != 1u || + requested.remap[1] != 0u || + !cuda_stream_expert_persistent_state_equal(&state, &before)) { + return 0; + } + cuda_stream_expert_persistent_plan_rollback(&requested); + if (!cuda_stream_expert_persistent_state_equal(&state, &before) || + !cuda_stream_expert_persistent_pin_key(&state, &key0, 1) || + !cuda_stream_expert_persistent_state_copy(&before, &state)) { + return 0; + } + + /* Slot zero is the LRU, but its active lease protects it. The planner + * must choose the next-oldest unpinned slot, then rollback without an LRU + * or free-list mutation. */ + const int32_t miss_id[1] = {3}; + cuda_stream_expert_persistent_plan replacement = {}; + if (!cuda_stream_expert_persistent_plan_build( + &state, &table, miss_id, 1u, &replacement) || + replacement.loads.size() != 1u || + replacement.loads[0].slot != 1u || + !replacement.loads[0].had_victim || + replacement.loads[0].victim.expert_id != 1u || + !cuda_stream_expert_persistent_state_equal(&state, &before)) { + return 0; + } + cuda_stream_expert_persistent_plan_rollback(&replacement); + if (!cuda_stream_expert_persistent_state_equal(&state, &before)) return 0; + + if (!cuda_stream_expert_persistent_pin_key(&state, &key1, 1) || + !cuda_stream_expert_persistent_pin_key(&state, &key2, 1) || + !cuda_stream_expert_persistent_state_copy(&before, &state)) { + return 0; + } + cuda_stream_expert_persistent_plan blocked = {}; + if (cuda_stream_expert_persistent_plan_build( + &state, &table, miss_id, 1u, &blocked) || + !cuda_stream_expert_persistent_state_equal(&state, &before)) { + return 0; + } + return 1; +} + +static int cuda_stream_expert_persistent_test_rejections(void) { + unsigned char model_a = 0; + unsigned char model_b = 0; + ds4_gpu_stream_expert_table table = {}; + cuda_stream_expert_persistent_state state = {}; + if (!cuda_stream_expert_persistent_test_table_make( + &table, &model_a, 3u, 8u, 16u, 8u) || + !cuda_stream_expert_persistent_state_init(&state, 2u, 16u, 8u)) { + return 0; + } + const int32_t initial_ids[2] = {0, 1}; + cuda_stream_expert_persistent_plan initial = {}; + if (!cuda_stream_expert_persistent_plan_build( + &state, &table, initial_ids, 2u, &initial) || + !cuda_stream_expert_persistent_plan_commit(&state, &initial)) { + return 0; + } + cuda_stream_expert_persistent_state before = {}; + if (!cuda_stream_expert_persistent_state_copy(&before, &state)) return 0; + + ds4_gpu_stream_expert_table wrong_class = {}; + const int32_t one_id[1] = {0}; + cuda_stream_expert_persistent_plan rejected = {}; + if (!cuda_stream_expert_persistent_test_table_make( + &wrong_class, &model_a, 3u, 8u, 32u, 8u) || + cuda_stream_expert_persistent_plan_build( + &state, &wrong_class, one_id, 1u, &rejected) || + !cuda_stream_expert_persistent_state_equal(&state, &before)) { + return 0; + } + /* Exact class identity matters even when 2*gate+down has the same total + * bytes per slot (16/8 and 12/16 are both 40 bytes). */ + ds4_gpu_stream_expert_table same_total_wrong_class = {}; + if (!cuda_stream_expert_persistent_test_table_make( + &same_total_wrong_class, &model_a, 3u, 8u, 12u, 16u) || + cuda_stream_expert_persistent_plan_build( + &state, &same_total_wrong_class, one_id, 1u, &rejected) || + !cuda_stream_expert_persistent_state_equal(&state, &before)) { + return 0; + } + + const int32_t over_budget[3] = {0, 1, 2}; + if (cuda_stream_expert_persistent_plan_build( + &state, &table, over_budget, 3u, &rejected) || + !cuda_stream_expert_persistent_state_equal(&state, &before)) { + return 0; + } + + /* Same expert number in another model/layer/table is a miss, never an + * aliasing hit. The discarded transaction must preserve the old key. */ + ds4_gpu_stream_expert_table other_key = {}; + if (!cuda_stream_expert_persistent_test_table_make( + &other_key, &model_b, 4u, 8u, 16u, 8u)) { + return 0; + } + const uint64_t key_misses_before = + g_stream_expert_persistent_key_misses.load(); + cuda_stream_expert_persistent_plan key_plan = {}; + if (!cuda_stream_expert_persistent_plan_build( + &state, &other_key, one_id, 1u, &key_plan) || + key_plan.hit_count != 0u || key_plan.miss_count != 1u || + key_plan.loads.size() != 1u || !key_plan.loads[0].had_victim || + key_plan.loads[0].key.model_map != &model_b || + key_plan.loads[0].victim.model_map != &model_a || + g_stream_expert_persistent_key_misses.load() != + key_misses_before + 1u || + !cuda_stream_expert_persistent_state_equal(&state, &before)) { + return 0; + } + cuda_stream_expert_persistent_plan_rollback(&key_plan); + if (!cuda_stream_expert_persistent_state_equal(&state, &before)) return 0; + + ds4_gpu_stream_expert_table bad_range = table; + bad_range.gate_offset = UINT64_MAX - 7u; + if (cuda_stream_expert_persistent_plan_build( + &state, &bad_range, one_id, 1u, &rejected) || + !cuda_stream_expert_persistent_state_equal(&state, &before)) { + return 0; + } + + cuda_stream_expert_persistent_class impossible = {}; + if (cuda_stream_expert_persistent_class_make( + &impossible, 2u, UINT64_MAX, 1u)) { + return 0; + } + + /* A wrapped LRU clock is fail-closed on the private transaction. */ + cuda_stream_expert_persistent_state wrapped = {}; + cuda_stream_expert_persistent_state wrapped_before = {}; + if (!cuda_stream_expert_persistent_state_copy(&wrapped, &state)) return 0; + wrapped.lru_clock = UINT64_MAX; + if (!cuda_stream_expert_persistent_state_copy( + &wrapped_before, &wrapped)) { + return 0; + } + if (cuda_stream_expert_persistent_plan_build( + &wrapped, &table, one_id, 1u, &rejected) || + !cuda_stream_expert_persistent_state_equal( + &wrapped, &wrapped_before)) { + return 0; + } + return 1; +} + +extern "C" int ds4_cuda_test_stream_expert_persistent_planner(void) { + g_stream_expert_persistent_oracle_runs.fetch_add( + 1, std::memory_order_relaxed); + const int ok = cuda_stream_expert_persistent_test_basic() && + cuda_stream_expert_persistent_test_protection() && + cuda_stream_expert_persistent_test_rejections(); + if (!ok) { + g_stream_expert_persistent_oracle_failures.fetch_add( + 1, std::memory_order_relaxed); + } + return ok; +} + struct ds4_cuda_iq2_ssd_grouped_report; typedef struct ds4_cuda_iq2_ssd_grouped_report ds4_cuda_iq2_ssd_grouped_report; @@ -30703,18 +31814,6 @@ extern "C" int ds4_gpu_args_probe_auto_cuda(const int *device_filter, return 0; } -typedef struct ds4_gpu_stream_expert_table { - const void *model_map; - uint64_t model_size; - uint32_t layer; - uint32_t n_total_expert; - uint64_t gate_offset; - uint64_t up_offset; - uint64_t down_offset; - uint64_t gate_expert_bytes; - uint64_t down_expert_bytes; -} ds4_gpu_stream_expert_table; - static int cuda_stream_selected_ensure_bytes( char **ptr, uint64_t *capacity, uint64_t bytes, const char *label) { if (*ptr && *capacity >= bytes) return 1; diff --git a/ds4_gpu.h b/ds4_gpu.h index 102e7eec5..bd7731933 100644 --- a/ds4_gpu.h +++ b/ds4_gpu.h @@ -337,6 +337,29 @@ typedef struct ds4_cuda_iq2_ssd_grouped_report { int required; int stats; } ds4_cuda_iq2_ssd_grouped_report; +typedef struct ds4_cuda_stream_expert_persistent_report { + uint64_t plan_attempts; + uint64_t plans_built; + uint64_t commits; + uint64_t rollbacks; + uint64_t hits; + uint64_t misses; + uint64_t duplicates; + uint64_t free_assignments; + uint64_t evictions; + uint64_t rejects; + uint64_t budget_rejects; + uint64_t class_rejects; + uint64_t protected_rejects; + uint64_t key_misses; + uint64_t overflow_rejects; + uint64_t oracle_runs; + uint64_t oracle_failures; + int enabled; + int required; + int stats; + int oracle; +} ds4_cuda_stream_expert_persistent_report; /* CUDA-only policy/planner/scatter hooks. The policy hook takes explicit * values so tests do not need to mutate process environment around once_flag * initialization. */ @@ -377,6 +400,14 @@ int ds4_cuda_test_iq2_ssd_grouped_raw_layout( int ds4_cuda_test_iq2_ssd_grouped_lease(void); void ds4_cuda_iq2_ssd_grouped_get_report( ds4_cuda_iq2_ssd_grouped_report *report); +int ds4_cuda_test_stream_expert_persistent_policy( + int enable, int disable, int require, int stats, int oracle, + int *enabled_out, int *required_out, int *stats_out, + int *oracle_out); +int ds4_cuda_test_stream_expert_persistent_env_value(const char *value); +int ds4_cuda_test_stream_expert_persistent_planner(void); +void ds4_cuda_stream_expert_persistent_get_report( + ds4_cuda_stream_expert_persistent_report *report); int ds4_gpu_cuda_stream_selected_event_pipeline_enabled(void); int ds4_gpu_cuda_stream_selected_event_pipeline_required(void); int ds4_gpu_cuda_stream_selected_set_owner_device(void); diff --git a/tests/test_gpu_model_cache.c b/tests/test_gpu_model_cache.c index daab21226..608efad0b 100644 --- a/tests/test_gpu_model_cache.c +++ b/tests/test_gpu_model_cache.c @@ -78,6 +78,67 @@ int main(void) { enabled == 0 && required == 0 && oracle == 0, "selected-expert event-pipeline disable-dominant policy"); int stats = -1; + CHECK(!ds4_cuda_test_stream_expert_persistent_env_value(NULL) && + !ds4_cuda_test_stream_expert_persistent_env_value("") && + !ds4_cuda_test_stream_expert_persistent_env_value("0") && + !ds4_cuda_test_stream_expert_persistent_env_value("false") && + !ds4_cuda_test_stream_expert_persistent_env_value("NO") && + !ds4_cuda_test_stream_expert_persistent_env_value("off") && + ds4_cuda_test_stream_expert_persistent_env_value("1") && + ds4_cuda_test_stream_expert_persistent_env_value("true"), + "persistent expert planner value-aware environment parser"); + CHECK(ds4_cuda_test_stream_expert_persistent_policy( + 1, 0, 0, 0, 0, + &enabled, &required, &stats, &oracle) && + enabled == 1 && required == 0 && stats == 0 && oracle == 0, + "persistent expert planner enable policy"); + CHECK(ds4_cuda_test_stream_expert_persistent_policy( + 0, 0, 1, 1, 1, + &enabled, &required, &stats, &oracle) && + enabled == 1 && required == 1 && stats == 1 && oracle == 1, + "persistent expert planner require/stats/oracle policy"); + CHECK(ds4_cuda_test_stream_expert_persistent_policy( + 0, 0, 0, 1, 0, + &enabled, &required, &stats, &oracle) && + enabled == 0 && required == 0 && stats == 1 && oracle == 0, + "persistent expert planner stats-only policy"); + CHECK(ds4_cuda_test_stream_expert_persistent_policy( + 1, 1, 1, 1, 1, + &enabled, &required, &stats, &oracle) && + enabled == 0 && required == 0 && stats == 0 && oracle == 0, + "persistent expert planner disable-dominant policy"); + ds4_cuda_stream_expert_persistent_report persistent_before; + memset(&persistent_before, 0, sizeof(persistent_before)); + ds4_cuda_stream_expert_persistent_get_report(&persistent_before); + CHECK(ds4_cuda_test_stream_expert_persistent_planner(), + "persistent expert LRU/free-list transaction planner oracle"); + ds4_cuda_stream_expert_persistent_report persistent_after; + memset(&persistent_after, 0, sizeof(persistent_after)); + ds4_cuda_stream_expert_persistent_get_report(&persistent_after); + CHECK(persistent_after.oracle_runs == persistent_before.oracle_runs + 1u && + persistent_after.oracle_failures == + persistent_before.oracle_failures && + persistent_after.plan_attempts > persistent_before.plan_attempts && + persistent_after.plans_built > persistent_before.plans_built && + persistent_after.commits > persistent_before.commits && + persistent_after.rollbacks > persistent_before.rollbacks && + persistent_after.hits > persistent_before.hits && + persistent_after.misses > persistent_before.misses && + persistent_after.duplicates > persistent_before.duplicates && + persistent_after.free_assignments > + persistent_before.free_assignments && + persistent_after.evictions > persistent_before.evictions && + persistent_after.rejects > persistent_before.rejects && + persistent_after.budget_rejects > + persistent_before.budget_rejects && + persistent_after.class_rejects > + persistent_before.class_rejects && + persistent_after.protected_rejects > + persistent_before.protected_rejects && + persistent_after.key_misses > persistent_before.key_misses && + persistent_after.overflow_rejects > + persistent_before.overflow_rejects, + "persistent expert planner invariant coverage counters"); CHECK(ds4_cuda_test_iq2_ssd_grouped_policy( 1, 0, 0, 0, &enabled, &required, &stats) && enabled == 1 && required == 0 && stats == 0, From d8958e17302b53073e248c9cc47d68fbf21d5bfa Mon Sep 17 00:00:00 2001 From: Giorgio Oppo Date: Fri, 14 Aug 2026 22:14:00 +0200 Subject: [PATCH 035/189] cuda: allocate persistent expert cache arena --- ds4_cuda.cu | 382 ++++++++++++++++++++++++++++++++++- ds4_gpu.h | 7 + tests/test_gpu_model_cache.c | 24 +++ 3 files changed, 403 insertions(+), 10 deletions(-) diff --git a/ds4_cuda.cu b/ds4_cuda.cu index da80aba5b..bec2b21e7 100644 --- a/ds4_cuda.cu +++ b/ds4_cuda.cu @@ -190,6 +190,28 @@ static int cuda_stream_selected_consume_drain(void); static void cuda_stream_selected_consume_release(void); static void cuda_stream_selected_cache_release(void); +typedef struct { + void *base; + char *gate; + char *up; + char *down; + uint64_t bytes; + uint64_t up_offset; + uint64_t down_offset; + uint64_t gate_expert_bytes; + uint64_t down_expert_bytes; + uint64_t configured_expert_bytes; + uint32_t capacity; + uint32_t valid_count; + uint32_t configured_budget; + int owner_device; + int poisoned; +} cuda_stream_expert_persistent_arena; + +static cuda_stream_expert_persistent_arena g_stream_expert_persistent_arena; +static std::mutex g_stream_expert_persistent_arena_mutex; +static void cuda_stream_expert_persistent_arena_release(int reset_class); + static void cuda_stream_selected_cache_invalidate(void) { g_stream_selected_cache.valid = 0; g_stream_selected_cache.upload_event_value = 0; @@ -615,6 +637,13 @@ static std::atomic g_stream_expert_persistent_key_misses{0}; static std::atomic g_stream_expert_persistent_overflow_rejects{0}; static std::atomic g_stream_expert_persistent_oracle_runs{0}; static std::atomic g_stream_expert_persistent_oracle_failures{0}; +static std::atomic g_stream_expert_persistent_arena_allocations{0}; +static std::atomic g_stream_expert_persistent_arena_reuses{0}; +static std::atomic g_stream_expert_persistent_arena_releases{0}; +static std::atomic g_stream_expert_persistent_arena_failures{0}; +static std::atomic g_stream_expert_persistent_arena_oracle_runs{0}; +static std::atomic g_stream_expert_persistent_arena_oracle_failures{0}; +static int g_stream_expert_persistent_runtime_ready; static int cuda_ok(cudaError_t err, const char *what); static void cuda_stream_selected_event_pipeline_release(void); @@ -3007,6 +3036,12 @@ typedef struct { uint64_t overflow_rejects; uint64_t oracle_runs; uint64_t oracle_failures; + uint64_t arena_allocations; + uint64_t arena_reuses; + uint64_t arena_releases; + uint64_t arena_failures; + uint64_t arena_oracle_runs; + uint64_t arena_oracle_failures; int enabled; int required; int stats; @@ -3730,6 +3765,12 @@ static void cuda_stream_expert_persistent_init(void) { } } +static int cuda_stream_expert_persistent_requested(void) { + std::call_once(g_stream_expert_persistent_once, + cuda_stream_expert_persistent_init); + return g_stream_expert_persistent_enabled; +} + extern "C" void ds4_cuda_stream_expert_persistent_get_report( ds4_cuda_stream_expert_persistent_report *report) { if (!report) return; @@ -3758,6 +3799,15 @@ extern "C" void ds4_cuda_stream_expert_persistent_get_report( out.oracle_runs = g_stream_expert_persistent_oracle_runs.load(); out.oracle_failures = g_stream_expert_persistent_oracle_failures.load(); + out.arena_allocations = + g_stream_expert_persistent_arena_allocations.load(); + out.arena_reuses = g_stream_expert_persistent_arena_reuses.load(); + out.arena_releases = g_stream_expert_persistent_arena_releases.load(); + out.arena_failures = g_stream_expert_persistent_arena_failures.load(); + out.arena_oracle_runs = + g_stream_expert_persistent_arena_oracle_runs.load(); + out.arena_oracle_failures = + g_stream_expert_persistent_arena_oracle_failures.load(); out.enabled = g_stream_expert_persistent_enabled; out.required = g_stream_expert_persistent_required; out.stats = g_stream_expert_persistent_stats; @@ -6335,6 +6385,7 @@ extern "C" void ds4_gpu_cleanup(void) { /* The selected cache may still be the destination of an event-published * upload. Drain and retire its auxiliary streams before freeing device * storage or invalidating the owner-device context. */ + cuda_stream_expert_persistent_arena_release(1); cuda_stream_selected_stage_release(); cuda_stream_selected_event_pipeline_release(); cuda_stream_selected_cache_release(); @@ -7213,6 +7264,7 @@ extern "C" int ds4_gpu_set_model_map(const void *model_map, uint64_t model_size) if (g_model_host_base == model_map && g_model_registered_size == model_size) return 1; cuda_q8_fold_invalidate_all(); cuda_stream_selected_cache_release(); + cuda_stream_expert_persistent_arena_release(1); cuda_f16_pair_chunk32_release_all(); cuda_model_range_release_all(); cuda_q8_f16_cache_release_all(); @@ -7416,6 +7468,7 @@ extern "C" int ds4_gpu_register_model_map_no_copy(const void *model_map, uint64_ cuda_q8_fold_invalidate_all(); cuda_stream_selected_cache_release(); + cuda_stream_expert_persistent_arena_release(1); cuda_f16_pair_chunk32_release_all(); cuda_model_range_release_all(); cuda_q8_f16_cache_release_all(); @@ -39062,12 +39115,299 @@ extern "C" int ds4_gpu_stream_expert_cache_begin_selected_load_async( return submitted ? -1 : 0; } +static int cuda_stream_expert_persistent_align256( + uint64_t value, uint64_t *out) { + if (!out || value > UINT64_MAX - 255u) return 0; + *out = (value + 255u) & ~UINT64_C(255); + return 1; +} + +static int cuda_stream_expert_persistent_arena_layout( + uint32_t capacity, + uint64_t gate_expert_bytes, + uint64_t down_expert_bytes, + uint64_t *up_offset, + uint64_t *down_offset, + uint64_t *total_bytes) { + uint64_t gate_plane = 0; + uint64_t down_plane = 0; + uint64_t after_up = 0; + uint64_t after_down = 0; + if (!up_offset || !down_offset || !total_bytes || capacity == 0 || + gate_expert_bytes == 0 || down_expert_bytes == 0 || + !cuda_stream_expert_persistent_mul_u64( + capacity, gate_expert_bytes, &gate_plane) || + !cuda_stream_expert_persistent_mul_u64( + capacity, down_expert_bytes, &down_plane) || + !cuda_stream_expert_persistent_align256(gate_plane, up_offset) || + !cuda_stream_expert_persistent_add_u64( + *up_offset, gate_plane, &after_up) || + !cuda_stream_expert_persistent_align256(after_up, down_offset) || + !cuda_stream_expert_persistent_add_u64( + *down_offset, down_plane, &after_down) || + !cuda_stream_expert_persistent_align256( + after_down, total_bytes) || + *total_bytes > SIZE_MAX) { + return 0; + } + return *up_offset >= gate_plane && *down_offset >= after_up && + *total_bytes >= after_down; +} + +static int cuda_stream_expert_persistent_arena_release_locked( + int reset_class) { + cuda_stream_expert_persistent_arena *arena = + &g_stream_expert_persistent_arena; + const uint32_t configured_budget = arena->configured_budget; + /* The engine seeds the majority total before installing a new model map. + * Preserve that startup pin while resetting the learned exact split. */ + const uint64_t configured_expert_bytes = + arena->configured_expert_bytes; + const uint64_t gate_expert_bytes = reset_class ? 0 : + arena->gate_expert_bytes; + const uint64_t down_expert_bytes = reset_class ? 0 : + arena->down_expert_bytes; + int previous_device = -1; + if (arena->base) { + (void)cudaGetDevice(&previous_device); + /* The arena is not a transient-cache alias, but it shares the upload + * and consumer epoch boundary which its future loader will use. */ + int drained = cuda_stream_selected_consume_drain(); + if (g_stream_selected_upload_stream) { + if (g_stream_selected_upload_owner_device < 0 || + cudaSetDevice(g_stream_selected_upload_owner_device) != + cudaSuccess || + cudaStreamSynchronize(g_stream_selected_upload_stream) != + cudaSuccess) { + (void)cudaGetLastError(); + drained = 0; + } + } + if (!drained) { + drained = cudaSetDevice(arena->owner_device) == cudaSuccess && + cudaDeviceSynchronize() == cudaSuccess; + if (!drained) (void)cudaGetLastError(); + } + if (!drained || + cudaSetDevice(arena->owner_device) != cudaSuccess) { + (void)cudaGetLastError(); + arena->poisoned = 1; + g_stream_expert_persistent_arena_failures.fetch_add( + 1, std::memory_order_relaxed); + if (previous_device >= 0) (void)cudaSetDevice(previous_device); + return 0; + } + if (cudaFree(arena->base) != cudaSuccess) { + (void)cudaGetLastError(); + arena->poisoned = 1; + g_stream_expert_persistent_arena_failures.fetch_add( + 1, std::memory_order_relaxed); + if (previous_device >= 0) (void)cudaSetDevice(previous_device); + return 0; + } + g_stream_expert_persistent_arena_releases.fetch_add( + 1, std::memory_order_relaxed); + } + memset(arena, 0, sizeof(*arena)); + arena->owner_device = -1; + arena->configured_budget = configured_budget; + arena->configured_expert_bytes = configured_expert_bytes; + arena->gate_expert_bytes = gate_expert_bytes; + arena->down_expert_bytes = down_expert_bytes; + if (previous_device >= 0) (void)cudaSetDevice(previous_device); + return 1; +} + +static void cuda_stream_expert_persistent_arena_release(int reset_class) { + std::lock_guard lock( + g_stream_expert_persistent_arena_mutex); + (void)cuda_stream_expert_persistent_arena_release_locked(reset_class); +} + +static int cuda_stream_expert_persistent_arena_ensure_locked(void) { + cuda_stream_expert_persistent_arena *arena = + &g_stream_expert_persistent_arena; + if (arena->poisoned || g_n_gpus != 1 || + arena->configured_budget == 0 || + arena->gate_expert_bytes == 0 || arena->down_expert_bytes == 0) { + return 0; + } + uint64_t up_offset = 0; + uint64_t down_offset = 0; + uint64_t total_bytes = 0; + if (!cuda_stream_expert_persistent_arena_layout( + arena->configured_budget, arena->gate_expert_bytes, + arena->down_expert_bytes, &up_offset, &down_offset, + &total_bytes)) { + g_stream_expert_persistent_arena_failures.fetch_add( + 1, std::memory_order_relaxed); + return 0; + } + const int owner = g_gpu[0].device_id; + if (arena->base && arena->owner_device == owner && + arena->capacity == arena->configured_budget && + arena->bytes == total_bytes && arena->up_offset == up_offset && + arena->down_offset == down_offset) { + g_stream_expert_persistent_arena_reuses.fetch_add( + 1, std::memory_order_relaxed); + return 1; + } + if (!cuda_stream_expert_persistent_arena_release_locked(0)) return 0; + int previous_device = -1; + (void)cudaGetDevice(&previous_device); + void *base = NULL; + cudaError_t err = cudaSetDevice(owner); + if (err == cudaSuccess) err = cudaMalloc(&base, (size_t)total_bytes); + if (err != cudaSuccess || !base || + (((uintptr_t)base) & 255u) != 0u) { + if (base) (void)cudaFree(base); + (void)cudaGetLastError(); + g_stream_expert_persistent_arena_failures.fetch_add( + 1, std::memory_order_relaxed); + if (previous_device >= 0) (void)cudaSetDevice(previous_device); + return 0; + } + arena->base = base; + arena->gate = (char *)base; + arena->up = (char *)base + up_offset; + arena->down = (char *)base + down_offset; + arena->bytes = total_bytes; + arena->up_offset = up_offset; + arena->down_offset = down_offset; + arena->capacity = arena->configured_budget; + arena->valid_count = 0; + arena->owner_device = owner; + g_stream_expert_persistent_arena_allocations.fetch_add( + 1, std::memory_order_relaxed); + if (previous_device >= 0) (void)cudaSetDevice(previous_device); + return 1; +} + +extern "C" int ds4_cuda_test_stream_expert_persistent_arena(void) { + g_stream_expert_persistent_arena_oracle_runs.fetch_add( + 1, std::memory_order_relaxed); + std::lock_guard lock( + g_stream_expert_persistent_arena_mutex); + cuda_stream_expert_persistent_arena *arena = + &g_stream_expert_persistent_arena; + const uint32_t saved_budget = arena->configured_budget; + const uint64_t saved_total = arena->configured_expert_bytes; + const uint64_t saved_gate = arena->gate_expert_bytes; + const uint64_t saved_down = arena->down_expert_bytes; + int ok = g_n_gpus == 1 && !arena->base && !arena->poisoned; + arena->configured_budget = 3u; + arena->configured_expert_bytes = 40u; + arena->gate_expert_bytes = 16u; + arena->down_expert_bytes = 8u; + + void *first_base = NULL; + if (ok) { + ok = cuda_stream_expert_persistent_arena_ensure_locked(); + } + uint64_t expected_up = 0; + uint64_t expected_down = 0; + uint64_t expected_total = 0; + if (ok) { + first_base = arena->base; + ok = cuda_stream_expert_persistent_arena_layout( + 3u, 16u, 8u, &expected_up, &expected_down, + &expected_total) && + arena->owner_device == g_gpu[0].device_id && + arena->capacity == 3u && arena->bytes == expected_total && + arena->up_offset == expected_up && + arena->down_offset == expected_down && + arena->gate == (char *)arena->base && + arena->up == (char *)arena->base + expected_up && + arena->down == (char *)arena->base + expected_down && + (((uintptr_t)arena->base | (uintptr_t)arena->gate | + (uintptr_t)arena->up | (uintptr_t)arena->down) & 255u) == 0u; + } + + int previous_device = -1; + unsigned char got[3] = {0, 0, 0}; + if (ok) { + (void)cudaGetDevice(&previous_device); + ok = cudaSetDevice(arena->owner_device) == cudaSuccess && + cudaMemset(arena->base, 0, (size_t)arena->bytes) == cudaSuccess && + cudaMemset(arena->gate + 47u, 0xa1, 1u) == cudaSuccess && + cudaMemset(arena->up + 47u, 0xb2, 1u) == cudaSuccess && + cudaMemset(arena->down + 23u, 0xc3, 1u) == cudaSuccess && + cudaMemcpy(&got[0], arena->gate + 47u, 1u, + cudaMemcpyDeviceToHost) == cudaSuccess && + cudaMemcpy(&got[1], arena->up + 47u, 1u, + cudaMemcpyDeviceToHost) == cudaSuccess && + cudaMemcpy(&got[2], arena->down + 23u, 1u, + cudaMemcpyDeviceToHost) == cudaSuccess && + got[0] == 0xa1 && got[1] == 0xb2 && got[2] == 0xc3; + if (!ok) (void)cudaGetLastError(); + if (previous_device >= 0) (void)cudaSetDevice(previous_device); + } + if (ok) { + ok = cuda_stream_expert_persistent_arena_ensure_locked() && + arena->base == first_base; + } + if (ok) { + ok = cuda_stream_expert_persistent_arena_release_locked(0) && + !arena->base && arena->gate_expert_bytes == 16u && + arena->down_expert_bytes == 8u && + cuda_stream_expert_persistent_arena_ensure_locked() && + arena->base; + } + const int cleanup_ok = + cuda_stream_expert_persistent_arena_release_locked(1); + if (cleanup_ok) { + ok = ok && !arena->base && arena->gate_expert_bytes == 0 && + arena->down_expert_bytes == 0 && + arena->configured_expert_bytes == 40u; + arena->configured_budget = saved_budget; + arena->configured_expert_bytes = saved_total; + arena->gate_expert_bytes = saved_gate; + arena->down_expert_bytes = saved_down; + } else { + ok = 0; + } + if (!ok) { + g_stream_expert_persistent_arena_oracle_failures.fetch_add( + 1, std::memory_order_relaxed); + } + return ok; +} + extern "C" uint32_t ds4_gpu_stream_expert_cache_budget_for_expert_size( uint64_t gate_expert_bytes, uint64_t down_expert_bytes) { - (void)gate_expert_bytes; - (void)down_expert_bytes; - return 0; + uint64_t two_gate = 0; + uint64_t total_class = 0; + if (gate_expert_bytes == 0 || down_expert_bytes == 0 || + !cuda_stream_expert_persistent_mul_u64( + gate_expert_bytes, 2u, &two_gate) || + !cuda_stream_expert_persistent_add_u64( + two_gate, down_expert_bytes, &total_class) || + total_class == 0) { + return 0; + } + std::lock_guard lock( + g_stream_expert_persistent_arena_mutex); + cuda_stream_expert_persistent_arena *arena = + &g_stream_expert_persistent_arena; + if (arena->configured_expert_bytes == 0) { + arena->configured_expert_bytes = total_class; + } + if (arena->configured_expert_bytes != total_class) return 0; + if (arena->gate_expert_bytes == 0 && arena->down_expert_bytes == 0) { + arena->gate_expert_bytes = gate_expert_bytes; + arena->down_expert_bytes = down_expert_bytes; + } else if (arena->gate_expert_bytes != gate_expert_bytes || + arena->down_expert_bytes != down_expert_bytes) { + return 0; + } + if (!g_stream_expert_persistent_runtime_ready || + !g_ssd_streaming_mode || arena->configured_budget == 0 || + !cuda_stream_expert_persistent_requested()) { + return 0; + } + return cuda_stream_expert_persistent_arena_ensure_locked() ? + arena->configured_budget : 0; } extern "C" int ds4_gpu_tensor_copy_f32_to_f16(ds4_gpu_tensor *dst, uint64_t dst_offset, @@ -39695,26 +40035,47 @@ extern "C" void ds4_gpu_set_ssd_streaming(bool enabled) { g_ssd_streaming_mode = enabled ? 1 : 0; if (!g_ssd_streaming_mode) { (void)cuda_stream_selected_cache_free_storage_writer(); + cuda_stream_expert_persistent_arena_release(0); } } extern "C" void ds4_gpu_set_streaming_expert_cache_budget(uint32_t experts) { - (void)experts; + std::lock_guard lock( + g_stream_expert_persistent_arena_mutex); + if (experts == g_stream_expert_persistent_arena.configured_budget) return; + if (!cuda_stream_expert_persistent_arena_release_locked(0)) return; + g_stream_expert_persistent_arena.configured_budget = experts; } extern "C" void ds4_gpu_set_streaming_expert_cache_expert_bytes(uint64_t bytes) { - (void)bytes; + std::lock_guard lock( + g_stream_expert_persistent_arena_mutex); + if (bytes == + g_stream_expert_persistent_arena.configured_expert_bytes) return; + if (!cuda_stream_expert_persistent_arena_release_locked(1)) return; + g_stream_expert_persistent_arena.configured_expert_bytes = bytes; } extern "C" uint32_t ds4_gpu_stream_expert_cache_configured_count(void) { - return 0; + std::lock_guard lock( + g_stream_expert_persistent_arena_mutex); + return g_stream_expert_persistent_runtime_ready && + g_ssd_streaming_mode && + cuda_stream_expert_persistent_requested() ? + g_stream_expert_persistent_arena.configured_budget : 0; } extern "C" uint32_t ds4_gpu_stream_expert_cache_current_count(void) { - std::lock_guard lock(g_stream_selected_consume_mutex); - return !g_stream_selected_writer_active && - g_stream_selected_cache.valid - ? g_stream_selected_cache.compact_count : 0; + if (!g_stream_expert_persistent_runtime_ready) { + std::lock_guard lock(g_stream_selected_consume_mutex); + return !g_stream_selected_writer_active && + g_stream_selected_cache.valid ? + g_stream_selected_cache.compact_count : 0; + } + std::lock_guard lock( + g_stream_expert_persistent_arena_mutex); + return g_stream_expert_persistent_arena.base ? + g_stream_expert_persistent_arena.valid_count : 0; } extern "C" void ds4_gpu_stream_expert_cache_reset_route_hotness(void) { @@ -39722,6 +40083,7 @@ extern "C" void ds4_gpu_stream_expert_cache_reset_route_hotness(void) { extern "C" void ds4_gpu_stream_expert_cache_release_resident(void) { cuda_stream_selected_cache_release(); + cuda_stream_expert_persistent_arena_release(1); } extern "C" int ds4_gpu_stream_expert_cache_seed_selected( diff --git a/ds4_gpu.h b/ds4_gpu.h index bd7731933..f0dc50fd9 100644 --- a/ds4_gpu.h +++ b/ds4_gpu.h @@ -355,6 +355,12 @@ typedef struct ds4_cuda_stream_expert_persistent_report { uint64_t overflow_rejects; uint64_t oracle_runs; uint64_t oracle_failures; + uint64_t arena_allocations; + uint64_t arena_reuses; + uint64_t arena_releases; + uint64_t arena_failures; + uint64_t arena_oracle_runs; + uint64_t arena_oracle_failures; int enabled; int required; int stats; @@ -406,6 +412,7 @@ int ds4_cuda_test_stream_expert_persistent_policy( int *oracle_out); int ds4_cuda_test_stream_expert_persistent_env_value(const char *value); int ds4_cuda_test_stream_expert_persistent_planner(void); +int ds4_cuda_test_stream_expert_persistent_arena(void); void ds4_cuda_stream_expert_persistent_get_report( ds4_cuda_stream_expert_persistent_report *report); int ds4_gpu_cuda_stream_selected_event_pipeline_enabled(void); diff --git a/tests/test_gpu_model_cache.c b/tests/test_gpu_model_cache.c index 608efad0b..288996729 100644 --- a/tests/test_gpu_model_cache.c +++ b/tests/test_gpu_model_cache.c @@ -221,6 +221,30 @@ int main(void) { } CHECK(ds4_gpu_init(), "ds4_gpu_init"); + ds4_gpu_set_streaming_expert_cache_budget(3u); + ds4_gpu_set_streaming_expert_cache_expert_bytes(40u); + CHECK(ds4_gpu_stream_expert_cache_configured_count() == 0u && + ds4_gpu_stream_expert_cache_budget_for_expert_size(16u, 8u) == 0u && + ds4_gpu_stream_expert_cache_current_count() == 0u, + "persistent expert arena remains runtime-inert before loader wiring"); + ds4_cuda_stream_expert_persistent_report arena_before; + memset(&arena_before, 0, sizeof(arena_before)); + ds4_cuda_stream_expert_persistent_get_report(&arena_before); + CHECK(ds4_cuda_test_stream_expert_persistent_arena(), + "persistent expert device arena allocation/reuse/reinit oracle"); + ds4_cuda_stream_expert_persistent_report arena_after; + memset(&arena_after, 0, sizeof(arena_after)); + ds4_cuda_stream_expert_persistent_get_report(&arena_after); + CHECK(arena_after.arena_allocations >= + arena_before.arena_allocations + 2u && + arena_after.arena_reuses > arena_before.arena_reuses && + arena_after.arena_releases >= arena_before.arena_releases + 2u && + arena_after.arena_failures == arena_before.arena_failures && + arena_after.arena_oracle_runs == + arena_before.arena_oracle_runs + 1u && + arena_after.arena_oracle_failures == + arena_before.arena_oracle_failures, + "persistent expert device arena coverage counters"); ds4_cuda_iq2_ssd_grouped_report lease_before; memset(&lease_before, 0, sizeof(lease_before)); ds4_cuda_iq2_ssd_grouped_get_report(&lease_before); From 34d365d7db1aa48d19617e368137ecdddf6448a1 Mon Sep 17 00:00:00 2001 From: Giorgio Oppo Date: Fri, 14 Aug 2026 22:32:31 +0200 Subject: [PATCH 036/189] cuda: separate selected cache binding ownership --- ds4_cuda.cu | 587 +++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 489 insertions(+), 98 deletions(-) diff --git a/ds4_cuda.cu b/ds4_cuda.cu index bec2b21e7..3689c561c 100644 --- a/ds4_cuda.cu +++ b/ds4_cuda.cu @@ -158,6 +158,7 @@ static int g_ssd_streaming_mode; typedef struct { int valid; int top6_unique; + int storage_kind; int logical_tier; const void *model_map; uint32_t layer; @@ -185,22 +186,51 @@ typedef struct { } cuda_stream_selected_cache; static cuda_stream_selected_cache g_stream_selected_cache; -static void cuda_stream_selected_upload_drain(void); + +enum { + CUDA_STREAM_SELECTED_STORAGE_NONE = 0, + CUDA_STREAM_SELECTED_STORAGE_TRANSIENT = 1, + CUDA_STREAM_SELECTED_STORAGE_PERSISTENT = 2, +}; + +/* Sole owner of allocations used by the transient selected-expert binding. + * g_stream_selected_cache only publishes non-owning views into this storage. */ +typedef struct { + char *gate; + char *up; + char *down; + int32_t *remap; + uint64_t gate_capacity; + uint64_t up_capacity; + uint64_t down_capacity; + uint64_t remap_capacity; + int owner_device; + int poisoned; +} cuda_stream_selected_transient_storage; + +static cuda_stream_selected_transient_storage + g_stream_selected_transient_storage = { + NULL, NULL, NULL, NULL, 0, 0, 0, 0, -1, 0, + }; +static int cuda_stream_selected_upload_drain_checked(void); static int cuda_stream_selected_consume_drain(void); static void cuda_stream_selected_consume_release(void); static void cuda_stream_selected_cache_release(void); +static void cuda_stream_expert_storage_release(int reset_class); typedef struct { void *base; char *gate; char *up; char *down; + int32_t *remap; uint64_t bytes; uint64_t up_offset; uint64_t down_offset; uint64_t gate_expert_bytes; uint64_t down_expert_bytes; uint64_t configured_expert_bytes; + uint64_t remap_capacity; uint32_t capacity; uint32_t valid_count; uint32_t configured_budget; @@ -214,7 +244,19 @@ static void cuda_stream_expert_persistent_arena_release(int reset_class); static void cuda_stream_selected_cache_invalidate(void) { g_stream_selected_cache.valid = 0; + g_stream_selected_cache.storage_kind = + CUDA_STREAM_SELECTED_STORAGE_NONE; g_stream_selected_cache.upload_event_value = 0; + g_stream_selected_cache.gate_ptr = NULL; + g_stream_selected_cache.up_ptr = NULL; + g_stream_selected_cache.down_ptr = NULL; + g_stream_selected_cache.gate_capacity = 0; + g_stream_selected_cache.up_capacity = 0; + g_stream_selected_cache.down_capacity = 0; + g_stream_selected_cache.slot_selected_ptr = NULL; + g_stream_selected_cache.slot_selected_capacity = 0; + memset(&g_stream_selected_cache.slot_selected_tensor, 0, + sizeof(g_stream_selected_cache.slot_selected_tensor)); } typedef struct { @@ -2785,15 +2827,27 @@ static int cuda_model_stage_read(void *stage, uint64_t stage_bytes, return cuda_pread_full(g_model_fd, stage, bytes, offset); } -static void cuda_stream_selected_upload_drain(void) { - if (!g_stream_selected_upload_stream) return; +static int cuda_stream_selected_upload_drain_checked(void) { + if (!g_stream_selected_upload_stream) return 1; int previous_device = -1; (void)cudaGetDevice(&previous_device); - if (g_stream_selected_upload_owner_device >= 0) { - (void)cudaSetDevice(g_stream_selected_upload_owner_device); + if (g_stream_selected_upload_owner_device < 0 || + cudaSetDevice(g_stream_selected_upload_owner_device) != cudaSuccess) { + (void)cudaGetLastError(); + if (previous_device >= 0) (void)cudaSetDevice(previous_device); + return 0; } - (void)cudaStreamSynchronize(g_stream_selected_upload_stream); + const cudaError_t err = + cudaStreamSynchronize(g_stream_selected_upload_stream); if (previous_device >= 0) (void)cudaSetDevice(previous_device); + if (err != cudaSuccess) { + fprintf(stderr, + "ds4: CUDA compact-cache upload drain failed: %s\n", + cudaGetErrorString(err)); + (void)cudaGetLastError(); + return 0; + } + return 1; } static void cuda_stream_selected_stage_release(void) { @@ -4781,37 +4835,70 @@ static int cuda_stream_selected_consume_drain(void) { /* Caller owns cuda_stream_selected_writer_guard, so no reader can acquire * these pointers while they are drained and freed. */ static int cuda_stream_selected_cache_free_storage_writer(void) { + cuda_stream_selected_transient_storage *storage = + &g_stream_selected_transient_storage; if (!g_stream_selected_writer_active || - !cuda_stream_selected_consume_drain()) { + !cuda_stream_selected_consume_drain() || + !cuda_stream_selected_upload_drain_checked()) { /* Never free a compact binding whose last consumer could not be * drained. A bounded leak is safer than a cross-stream UAF when the * CUDA context is already reporting an unrecoverable error. */ cuda_stream_selected_cache_invalidate(); + storage->poisoned = 1; return 0; } - cuda_stream_selected_upload_drain(); - const int has_storage = g_stream_selected_cache.gate_ptr || - g_stream_selected_cache.up_ptr || - g_stream_selected_cache.down_ptr || - g_stream_selected_cache.slot_selected_ptr; + const int has_storage = storage->gate || storage->up || + storage->down || storage->remap; + int previous_device = -1; + if (has_storage) (void)cudaGetDevice(&previous_device); if (has_storage && - (g_n_gpus != 1 || - cudaSetDevice(g_gpu[0].device_id) != cudaSuccess)) { + (storage->owner_device < 0 || + cudaSetDevice(storage->owner_device) != cudaSuccess)) { (void)cudaGetLastError(); + cuda_stream_selected_cache_invalidate(); + storage->poisoned = 1; + if (previous_device >= 0) (void)cudaSetDevice(previous_device); return 0; } - if (g_stream_selected_cache.gate_ptr) { - (void)cudaFree(g_stream_selected_cache.gate_ptr); + if (storage->gate && cudaFree(storage->gate) != cudaSuccess) { + (void)cudaGetLastError(); + storage->poisoned = 1; + cuda_stream_selected_cache_invalidate(); + if (previous_device >= 0) (void)cudaSetDevice(previous_device); + return 0; } - if (g_stream_selected_cache.up_ptr) { - (void)cudaFree(g_stream_selected_cache.up_ptr); + storage->gate = NULL; + storage->gate_capacity = 0; + if (storage->up && cudaFree(storage->up) != cudaSuccess) { + (void)cudaGetLastError(); + storage->poisoned = 1; + cuda_stream_selected_cache_invalidate(); + if (previous_device >= 0) (void)cudaSetDevice(previous_device); + return 0; } - if (g_stream_selected_cache.down_ptr) { - (void)cudaFree(g_stream_selected_cache.down_ptr); + storage->up = NULL; + storage->up_capacity = 0; + if (storage->down && cudaFree(storage->down) != cudaSuccess) { + (void)cudaGetLastError(); + storage->poisoned = 1; + cuda_stream_selected_cache_invalidate(); + if (previous_device >= 0) (void)cudaSetDevice(previous_device); + return 0; } - if (g_stream_selected_cache.slot_selected_ptr) { - (void)cudaFree(g_stream_selected_cache.slot_selected_ptr); + storage->down = NULL; + storage->down_capacity = 0; + if (storage->remap && cudaFree(storage->remap) != cudaSuccess) { + (void)cudaGetLastError(); + storage->poisoned = 1; + cuda_stream_selected_cache_invalidate(); + if (previous_device >= 0) (void)cudaSetDevice(previous_device); + return 0; } + storage->remap = NULL; + storage->remap_capacity = 0; + storage->owner_device = -1; + storage->poisoned = 0; + if (previous_device >= 0) (void)cudaSetDevice(previous_device); memset(&g_stream_selected_cache, 0, sizeof(g_stream_selected_cache)); g_stream_selected_cache.logical_tier = -1; return 1; @@ -4822,6 +4909,55 @@ static void cuda_stream_selected_cache_release(void) { (void)cuda_stream_selected_cache_free_storage_writer(); } +static int cuda_stream_selected_cache_bind_transient_storage( + int logical_tier, + uint64_t gate_bytes, + uint64_t down_bytes, + uint64_t remap_bytes) { + const cuda_stream_selected_transient_storage *storage = + &g_stream_selected_transient_storage; + if (storage->poisoned || !storage->gate || !storage->up || + !storage->down || !storage->remap || + storage->gate_capacity < gate_bytes || + storage->up_capacity < gate_bytes || + storage->down_capacity < down_bytes || + storage->remap_capacity < remap_bytes) { + return 0; + } + g_stream_selected_cache.storage_kind = + CUDA_STREAM_SELECTED_STORAGE_TRANSIENT; + g_stream_selected_cache.gate_ptr = storage->gate; + g_stream_selected_cache.up_ptr = storage->up; + g_stream_selected_cache.down_ptr = storage->down; + g_stream_selected_cache.gate_capacity = storage->gate_capacity; + g_stream_selected_cache.up_capacity = storage->up_capacity; + g_stream_selected_cache.down_capacity = storage->down_capacity; + g_stream_selected_cache.slot_selected_ptr = storage->remap; + g_stream_selected_cache.slot_selected_capacity = + storage->remap_capacity; + g_stream_selected_cache.slot_selected_tensor.ptr = storage->remap; + g_stream_selected_cache.slot_selected_tensor.bytes = remap_bytes; + g_stream_selected_cache.slot_selected_tensor.owner = 0; + g_stream_selected_cache.slot_selected_tensor.device_id = logical_tier; + return 1; +} + +/* Invalidate the public binding under one writer epoch, retire its transient + * owner, and only then retire the persistent owner it may alias in phase 3. */ +static int cuda_stream_expert_storage_release_writer(int reset_class) { + if (!g_stream_selected_writer_active || + !cuda_stream_selected_cache_free_storage_writer()) { + return 0; + } + cuda_stream_expert_persistent_arena_release(reset_class); + return 1; +} + +static void cuda_stream_expert_storage_release(int reset_class) { + cuda_stream_selected_writer_guard writer; + (void)cuda_stream_expert_storage_release_writer(reset_class); +} + static void cuda_stream_selected_consume_release(void) { std::lock_guard lock(g_stream_selected_consume_mutex); if (g_stream_selected_consume_owner_device >= 0) { @@ -6385,10 +6521,9 @@ extern "C" void ds4_gpu_cleanup(void) { /* The selected cache may still be the destination of an event-published * upload. Drain and retire its auxiliary streams before freeing device * storage or invalidating the owner-device context. */ - cuda_stream_expert_persistent_arena_release(1); + cuda_stream_expert_storage_release(1); cuda_stream_selected_stage_release(); cuda_stream_selected_event_pipeline_release(); - cuda_stream_selected_cache_release(); cuda_stream_selected_consume_release(); ds4_mmq_set_gb10_optimizations(0); g_n_gpus = 0; @@ -7263,8 +7398,7 @@ extern "C" int ds4_gpu_set_model_map(const void *model_map, uint64_t model_size) if (!model_map || model_size == 0) return 0; if (g_model_host_base == model_map && g_model_registered_size == model_size) return 1; cuda_q8_fold_invalidate_all(); - cuda_stream_selected_cache_release(); - cuda_stream_expert_persistent_arena_release(1); + cuda_stream_expert_storage_release(1); cuda_f16_pair_chunk32_release_all(); cuda_model_range_release_all(); cuda_q8_f16_cache_release_all(); @@ -7467,8 +7601,7 @@ extern "C" int ds4_gpu_register_model_map_no_copy(const void *model_map, uint64_ if (g_model_host_base == model_map && g_model_registered_size == model_size) return 1; cuda_q8_fold_invalidate_all(); - cuda_stream_selected_cache_release(); - cuda_stream_expert_persistent_arena_release(1); + cuda_stream_expert_storage_release(1); cuda_f16_pair_chunk32_release_all(); cuda_model_range_release_all(); cuda_q8_f16_cache_release_all(); @@ -28881,6 +29014,7 @@ __global__ static void moe_down_f32_kernel( typedef struct { int valid; int top6_unique; + int storage_kind; const char *gate; const char *up; const char *down; @@ -28908,6 +29042,8 @@ static int cuda_stream_selected_binding_acquire( if (gate_expert_bytes == 0 || down_expert_bytes == 0 || required_slot_count > UINT64_MAX / sizeof(int32_t) || !g_stream_selected_cache.valid || + g_stream_selected_cache.storage_kind != + CUDA_STREAM_SELECTED_STORAGE_TRANSIENT || g_stream_selected_cache.logical_tier != logical_tier || g_stream_selected_cache.model_map != model_map || g_stream_selected_cache.layer != layer_index || @@ -28945,6 +29081,7 @@ static int cuda_stream_selected_binding_acquire( } binding->valid = 1; binding->top6_unique = g_stream_selected_cache.top6_unique; + binding->storage_kind = g_stream_selected_cache.storage_kind; binding->gate = g_stream_selected_cache.gate_ptr + (uint64_t)g_stream_selected_cache.slot_base * gate_expert_bytes; binding->up = g_stream_selected_cache.up_ptr + @@ -31871,7 +32008,14 @@ static int cuda_stream_selected_ensure_bytes( char **ptr, uint64_t *capacity, uint64_t bytes, const char *label) { if (*ptr && *capacity >= bytes) return 1; if (*ptr) { - (void)cudaFree(*ptr); + const cudaError_t free_err = cudaFree(*ptr); + if (free_err != cudaSuccess) { + fprintf(stderr, + "ds4: CUDA streaming %s resize free failed: %s\n", + label, cudaGetErrorString(free_err)); + (void)cudaGetLastError(); + return 0; + } *ptr = NULL; *capacity = 0; } @@ -31887,12 +32031,15 @@ static int cuda_stream_selected_ensure_bytes( return 1; } -static int cuda_stream_selected_ensure_i32(uint64_t count) { +static int cuda_stream_selected_ensure_i32( + cuda_stream_selected_transient_storage *storage, + uint64_t count) { + if (!storage) return 0; if (count == 0 || count > UINT64_MAX / sizeof(int32_t)) return 0; const uint64_t bytes = count * sizeof(int32_t); return cuda_stream_selected_ensure_bytes( - (char **)&g_stream_selected_cache.slot_selected_ptr, - &g_stream_selected_cache.slot_selected_capacity, + (char **)&storage->remap, + &storage->remap_capacity, bytes, "selected-id remap"); } @@ -31931,6 +32078,8 @@ static int cuda_stream_selected_cache_begin_load_impl( if (upload_event_out) *upload_event_out = 0; if (submitted_any_out) *submitted_any_out = 0; cuda_stream_selected_writer_guard writer; + cuda_stream_selected_transient_storage *storage = + &g_stream_selected_transient_storage; if (!g_ssd_streaming_mode) return 1; if (!cuda_stream_selected_ranges_valid(table) || !selected_ids || slot_count == 0) { @@ -31952,6 +32101,10 @@ static int cuda_stream_selected_cache_begin_load_impl( (void)cudaGetLastError(); return 0; } + if (storage->poisoned && + !cuda_stream_selected_cache_free_storage_writer()) { + return 0; + } std::vector expert_to_slot; std::vector compact_ids; @@ -32000,42 +32153,40 @@ static int cuda_stream_selected_cache_begin_load_impl( const uint64_t gate_bytes = compact_count * table->gate_expert_bytes; const uint64_t down_bytes = compact_count * table->down_expert_bytes; const int logical_tier = 0; - if (g_stream_selected_cache.logical_tier != logical_tier && - (g_stream_selected_cache.gate_ptr || - g_stream_selected_cache.up_ptr || - g_stream_selected_cache.down_ptr || - g_stream_selected_cache.slot_selected_ptr)) { + if ((storage->gate || storage->up || storage->down || storage->remap) && + storage->owner_device != g_gpu[0].device_id) { if (!cuda_stream_selected_cache_free_storage_writer()) return 0; } const uint64_t remap_bytes = (uint64_t)slot_count * sizeof(int32_t); const int resize_binding = - (g_stream_selected_cache.gate_ptr && - g_stream_selected_cache.gate_capacity < gate_bytes) || - (g_stream_selected_cache.up_ptr && - g_stream_selected_cache.up_capacity < gate_bytes) || - (g_stream_selected_cache.down_ptr && - g_stream_selected_cache.down_capacity < down_bytes) || - (g_stream_selected_cache.slot_selected_ptr && - g_stream_selected_cache.slot_selected_capacity < remap_bytes); + (storage->gate && storage->gate_capacity < gate_bytes) || + (storage->up && storage->up_capacity < gate_bytes) || + (storage->down && storage->down_capacity < down_bytes) || + (storage->remap && storage->remap_capacity < remap_bytes); if (resize_binding && !cuda_stream_selected_consume_drain()) { cuda_stream_selected_cache_invalidate(); return 0; } - if (resize_binding) cuda_stream_selected_upload_drain(); + if (resize_binding && !cuda_stream_selected_upload_drain_checked()) { + cuda_stream_selected_cache_invalidate(); + storage->poisoned = 1; + return 0; + } + storage->owner_device = g_gpu[0].device_id; if (ds4_gpu_set_current_device(logical_tier) != 0 || !cuda_stream_selected_ensure_bytes( - &g_stream_selected_cache.gate_ptr, - &g_stream_selected_cache.gate_capacity, + &storage->gate, + &storage->gate_capacity, gate_bytes, "gate experts") || !cuda_stream_selected_ensure_bytes( - &g_stream_selected_cache.up_ptr, - &g_stream_selected_cache.up_capacity, + &storage->up, + &storage->up_capacity, gate_bytes, "up experts") || !cuda_stream_selected_ensure_bytes( - &g_stream_selected_cache.down_ptr, - &g_stream_selected_cache.down_capacity, + &storage->down, + &storage->down_capacity, down_bytes, "down experts") || - !cuda_stream_selected_ensure_i32(slot_count)) { + !cuda_stream_selected_ensure_i32(storage, slot_count)) { cuda_stream_selected_cache_invalidate(); return 0; } @@ -32074,15 +32225,15 @@ static int cuda_stream_selected_cache_begin_load_impl( const uint64_t down_dst = (uint64_t)i * table->down_expert_bytes; tasks.push_back({ - g_stream_selected_cache.gate_ptr + gate_dst, + storage->gate + gate_dst, gate_src, table->gate_expert_bytes, ordinal++, }); tasks.push_back({ - g_stream_selected_cache.up_ptr + gate_dst, + storage->up + gate_dst, up_src, table->gate_expert_bytes, ordinal++, }); tasks.push_back({ - g_stream_selected_cache.down_ptr + down_dst, + storage->down + down_dst, down_src, table->down_expert_bytes, ordinal++, }); } @@ -32096,7 +32247,7 @@ static int cuda_stream_selected_cache_begin_load_impl( g_stream_selected_batch_io_attempts++; copied = cuda_model_copy_tasks_to_device_streamed( tasks, table->model_map, table->model_size, - g_stream_selected_cache.slot_selected_ptr, + storage->remap, slot_ids.data(), slot_count, cuda_stream_selected_batch_io_oracle_requested(), /*chunk_override=*/0, &submitted, @@ -32160,17 +32311,17 @@ static int cuda_stream_selected_cache_begin_load_impl( const uint64_t down_dst = (uint64_t)i * table->down_expert_bytes; if (!cuda_model_copy_to_device_streamed( - g_stream_selected_cache.gate_ptr + gate_dst, + storage->gate + gate_dst, table->model_map, table->model_size, gate_src, table->gate_expert_bytes, "stream gate expert copy") || !cuda_model_copy_to_device_streamed( - g_stream_selected_cache.up_ptr + gate_dst, + storage->up + gate_dst, table->model_map, table->model_size, up_src, table->gate_expert_bytes, "stream up expert copy") || !cuda_model_copy_to_device_streamed( - g_stream_selected_cache.down_ptr + down_dst, + storage->down + down_dst, table->model_map, table->model_size, down_src, table->down_expert_bytes, "stream down expert copy")) { @@ -32178,7 +32329,7 @@ static int cuda_stream_selected_cache_begin_load_impl( return 0; } } - if (!cuda_ok(cudaMemcpy(g_stream_selected_cache.slot_selected_ptr, + if (!cuda_ok(cudaMemcpy(storage->remap, slot_ids.data(), (size_t)slot_count * sizeof(int32_t), cudaMemcpyHostToDevice), @@ -32207,12 +32358,11 @@ static int cuda_stream_selected_cache_begin_load_impl( g_stream_selected_cache.down_offset = table->down_offset; g_stream_selected_cache.gate_expert_bytes = table->gate_expert_bytes; g_stream_selected_cache.down_expert_bytes = table->down_expert_bytes; - g_stream_selected_cache.slot_selected_tensor.ptr = - g_stream_selected_cache.slot_selected_ptr; - g_stream_selected_cache.slot_selected_tensor.bytes = - (uint64_t)slot_count * sizeof(int32_t); - g_stream_selected_cache.slot_selected_tensor.owner = 0; - g_stream_selected_cache.slot_selected_tensor.device_id = logical_tier; + if (!cuda_stream_selected_cache_bind_transient_storage( + logical_tier, gate_bytes, down_bytes, remap_bytes)) { + cuda_stream_selected_cache_invalidate(); + return 0; + } /* Publish all binding metadata atomically with respect to consumer * acquisition. The upload itself may still be in flight; its exact * completion token is part of the metadata guarded by this release. */ @@ -39168,21 +39318,13 @@ static int cuda_stream_expert_persistent_arena_release_locked( const uint64_t down_expert_bytes = reset_class ? 0 : arena->down_expert_bytes; int previous_device = -1; - if (arena->base) { + const int has_storage = arena->base || arena->remap; + if (has_storage) { (void)cudaGetDevice(&previous_device); /* The arena is not a transient-cache alias, but it shares the upload * and consumer epoch boundary which its future loader will use. */ - int drained = cuda_stream_selected_consume_drain(); - if (g_stream_selected_upload_stream) { - if (g_stream_selected_upload_owner_device < 0 || - cudaSetDevice(g_stream_selected_upload_owner_device) != - cudaSuccess || - cudaStreamSynchronize(g_stream_selected_upload_stream) != - cudaSuccess) { - (void)cudaGetLastError(); - drained = 0; - } - } + int drained = cuda_stream_selected_consume_drain() && + cuda_stream_selected_upload_drain_checked(); if (!drained) { drained = cudaSetDevice(arena->owner_device) == cudaSuccess && cudaDeviceSynchronize() == cudaSuccess; @@ -39197,7 +39339,17 @@ static int cuda_stream_expert_persistent_arena_release_locked( if (previous_device >= 0) (void)cudaSetDevice(previous_device); return 0; } - if (cudaFree(arena->base) != cudaSuccess) { + if (arena->remap && cudaFree(arena->remap) != cudaSuccess) { + (void)cudaGetLastError(); + arena->poisoned = 1; + g_stream_expert_persistent_arena_failures.fetch_add( + 1, std::memory_order_relaxed); + if (previous_device >= 0) (void)cudaSetDevice(previous_device); + return 0; + } + arena->remap = NULL; + arena->remap_capacity = 0; + if (arena->base && cudaFree(arena->base) != cudaSuccess) { (void)cudaGetLastError(); arena->poisoned = 1; g_stream_expert_persistent_arena_failures.fetch_add( @@ -39283,6 +39435,61 @@ static int cuda_stream_expert_persistent_arena_ensure_locked(void) { return 1; } +/* The persistent remap has a different lifetime/shape from the three weight + * planes, so it is a distinct allocation owned by the arena. Phase 3a only + * exercises this through the device oracle; the production loader remains + * on g_stream_selected_transient_storage. */ +static int cuda_stream_expert_persistent_remap_ensure_locked( + uint64_t count) { + cuda_stream_expert_persistent_arena *arena = + &g_stream_expert_persistent_arena; + uint64_t bytes = 0; + if (!arena->base || arena->poisoned || arena->owner_device < 0 || + count == 0 || + !cuda_stream_expert_persistent_mul_u64( + count, sizeof(int32_t), &bytes) || + bytes > SIZE_MAX) { + return 0; + } + if (arena->remap && arena->remap_capacity >= bytes) return 1; + + int previous_device = -1; + (void)cudaGetDevice(&previous_device); + int32_t *replacement = NULL; + cudaError_t err = cudaSetDevice(arena->owner_device); + if (err == cudaSuccess) { + err = cudaMalloc((void **)&replacement, (size_t)bytes); + } + if (err != cudaSuccess || !replacement) { + if (replacement) (void)cudaFree(replacement); + (void)cudaGetLastError(); + g_stream_expert_persistent_arena_failures.fetch_add( + 1, std::memory_order_relaxed); + if (previous_device >= 0) (void)cudaSetDevice(previous_device); + return 0; + } + if (arena->remap) { + int drained = cuda_stream_selected_consume_drain() && + cuda_stream_selected_upload_drain_checked(); + if (!drained) { + drained = cudaDeviceSynchronize() == cudaSuccess; + } + if (!drained || cudaFree(arena->remap) != cudaSuccess) { + (void)cudaGetLastError(); + (void)cudaFree(replacement); + arena->poisoned = 1; + g_stream_expert_persistent_arena_failures.fetch_add( + 1, std::memory_order_relaxed); + if (previous_device >= 0) (void)cudaSetDevice(previous_device); + return 0; + } + } + arena->remap = replacement; + arena->remap_capacity = bytes; + if (previous_device >= 0) (void)cudaSetDevice(previous_device); + return 1; +} + extern "C" int ds4_cuda_test_stream_expert_persistent_arena(void) { g_stream_expert_persistent_arena_oracle_runs.fetch_add( 1, std::memory_order_relaxed); @@ -39294,21 +39501,25 @@ extern "C" int ds4_cuda_test_stream_expert_persistent_arena(void) { const uint64_t saved_total = arena->configured_expert_bytes; const uint64_t saved_gate = arena->gate_expert_bytes; const uint64_t saved_down = arena->down_expert_bytes; - int ok = g_n_gpus == 1 && !arena->base && !arena->poisoned; + int ok = g_n_gpus == 1 && !arena->base && !arena->remap && + !arena->poisoned; arena->configured_budget = 3u; arena->configured_expert_bytes = 40u; arena->gate_expert_bytes = 16u; arena->down_expert_bytes = 8u; void *first_base = NULL; + int32_t *first_remap = NULL; if (ok) { - ok = cuda_stream_expert_persistent_arena_ensure_locked(); + ok = cuda_stream_expert_persistent_arena_ensure_locked() && + cuda_stream_expert_persistent_remap_ensure_locked(7u); } uint64_t expected_up = 0; uint64_t expected_down = 0; uint64_t expected_total = 0; if (ok) { first_base = arena->base; + first_remap = arena->remap; ok = cuda_stream_expert_persistent_arena_layout( 3u, 16u, 8u, &expected_up, &expected_down, &expected_total) && @@ -39319,12 +39530,17 @@ extern "C" int ds4_cuda_test_stream_expert_persistent_arena(void) { arena->gate == (char *)arena->base && arena->up == (char *)arena->base + expected_up && arena->down == (char *)arena->base + expected_down && + arena->remap && + arena->remap_capacity >= 7u * sizeof(int32_t) && (((uintptr_t)arena->base | (uintptr_t)arena->gate | - (uintptr_t)arena->up | (uintptr_t)arena->down) & 255u) == 0u; + (uintptr_t)arena->up | (uintptr_t)arena->down | + (uintptr_t)arena->remap) & 255u) == 0u; } int previous_device = -1; unsigned char got[3] = {0, 0, 0}; + int32_t remap_got = 0; + const int32_t remap_canary = INT32_C(0x13572468); if (ok) { (void)cudaGetDevice(&previous_device); ok = cudaSetDevice(arena->owner_device) == cudaSuccess && @@ -39332,31 +39548,44 @@ extern "C" int ds4_cuda_test_stream_expert_persistent_arena(void) { cudaMemset(arena->gate + 47u, 0xa1, 1u) == cudaSuccess && cudaMemset(arena->up + 47u, 0xb2, 1u) == cudaSuccess && cudaMemset(arena->down + 23u, 0xc3, 1u) == cudaSuccess && + cudaMemcpy(arena->remap + 6u, &remap_canary, + sizeof(remap_canary), + cudaMemcpyHostToDevice) == cudaSuccess && cudaMemcpy(&got[0], arena->gate + 47u, 1u, cudaMemcpyDeviceToHost) == cudaSuccess && cudaMemcpy(&got[1], arena->up + 47u, 1u, cudaMemcpyDeviceToHost) == cudaSuccess && cudaMemcpy(&got[2], arena->down + 23u, 1u, cudaMemcpyDeviceToHost) == cudaSuccess && - got[0] == 0xa1 && got[1] == 0xb2 && got[2] == 0xc3; + cudaMemcpy(&remap_got, arena->remap + 6u, + sizeof(remap_got), + cudaMemcpyDeviceToHost) == cudaSuccess && + got[0] == 0xa1 && got[1] == 0xb2 && got[2] == 0xc3 && + remap_got == remap_canary; if (!ok) (void)cudaGetLastError(); if (previous_device >= 0) (void)cudaSetDevice(previous_device); } if (ok) { ok = cuda_stream_expert_persistent_arena_ensure_locked() && - arena->base == first_base; + cuda_stream_expert_persistent_remap_ensure_locked(3u) && + arena->base == first_base && arena->remap == first_remap; } if (ok) { ok = cuda_stream_expert_persistent_arena_release_locked(0) && - !arena->base && arena->gate_expert_bytes == 16u && + !arena->base && !arena->remap && + arena->remap_capacity == 0 && + arena->gate_expert_bytes == 16u && arena->down_expert_bytes == 8u && cuda_stream_expert_persistent_arena_ensure_locked() && - arena->base; + cuda_stream_expert_persistent_remap_ensure_locked(9u) && + arena->base && arena->remap; } const int cleanup_ok = cuda_stream_expert_persistent_arena_release_locked(1); if (cleanup_ok) { - ok = ok && !arena->base && arena->gate_expert_bytes == 0 && + ok = ok && !arena->base && !arena->remap && + arena->remap_capacity == 0 && + arena->gate_expert_bytes == 0 && arena->down_expert_bytes == 0 && arena->configured_expert_bytes == 40u; arena->configured_budget = saved_budget; @@ -39645,6 +39874,63 @@ extern "C" int ds4_cuda_test_iq2_ssd_grouped_lease(void) { const uint64_t saved_model_direct_align = g_model_direct_align; const uint64_t saved_model_file_size = g_model_file_size; cuda_stream_selected_cache_release(); + if (g_stream_selected_transient_storage.gate || + g_stream_selected_transient_storage.up || + g_stream_selected_transient_storage.down || + g_stream_selected_transient_storage.remap || + g_stream_selected_cache.valid || + g_stream_selected_cache.storage_kind != + CUDA_STREAM_SELECTED_STORAGE_NONE) { + return 0; + } + + uint32_t saved_arena_budget = 0; + uint64_t saved_arena_total = 0; + uint64_t saved_arena_gate = 0; + uint64_t saved_arena_down = 0; + void *persistent_base = NULL; + int32_t *persistent_remap = NULL; + uint64_t persistent_bytes = 0; + const unsigned char persistent_base_canary = 0x5au; + const int32_t persistent_remap_canary = INT32_C(0x31415926); + int persistent_touched = 0; + int ok = 1; + { + std::lock_guard lock( + g_stream_expert_persistent_arena_mutex); + cuda_stream_expert_persistent_arena *arena = + &g_stream_expert_persistent_arena; + saved_arena_budget = arena->configured_budget; + saved_arena_total = arena->configured_expert_bytes; + saved_arena_gate = arena->gate_expert_bytes; + saved_arena_down = arena->down_expert_bytes; + if (arena->base || arena->remap || arena->poisoned) { + ok = 0; + } else { + persistent_touched = 1; + arena->configured_budget = 3u; + arena->configured_expert_bytes = 40u; + arena->gate_expert_bytes = 16u; + arena->down_expert_bytes = 8u; + ok = cuda_stream_expert_persistent_arena_ensure_locked() && + cuda_stream_expert_persistent_remap_ensure_locked(7u); + if (ok) { + persistent_base = arena->base; + persistent_remap = arena->remap; + persistent_bytes = arena->bytes; + ok = persistent_base && persistent_remap && + persistent_bytes != 0 && + cudaMemset((char *)persistent_base + + persistent_bytes - 1u, + persistent_base_canary, 1u) == cudaSuccess && + cudaMemcpy(persistent_remap + 6u, + &persistent_remap_canary, + sizeof(persistent_remap_canary), + cudaMemcpyHostToDevice) == cudaSuccess; + if (!ok) (void)cudaGetLastError(); + } + } + } { cuda_stream_selected_writer_guard writer; g_ssd_streaming_mode = 1; @@ -39669,7 +39955,7 @@ extern "C" int ds4_cuda_test_iq2_ssd_grouped_lease(void) { const uint64_t test_down_offset = 2u * full_table_bytes; const uint64_t test_model_bytes = 3u * full_table_bytes; g_model_file_size = test_model_bytes; - int ok = cuda_stream_selected_stage_pool_alloc(4096u) && + ok = ok && cuda_stream_selected_stage_pool_alloc(4096u) && cuda_stream_selected_event_pipeline_ensure() && cudaMallocHost((void **)&test_model, (size_t)test_model_bytes) == cudaSuccess && @@ -39704,11 +39990,42 @@ extern "C" int ds4_cuda_test_iq2_ssd_grouped_lease(void) { /*upload_event_out=*/NULL, /*submitted_any_out=*/NULL); } + if (ok) { + std::lock_guard lock( + g_stream_expert_persistent_arena_mutex); + const cuda_stream_expert_persistent_arena *arena = + &g_stream_expert_persistent_arena; + const uintptr_t arena_begin = (uintptr_t)persistent_base; + const uintptr_t arena_end = arena_begin + persistent_bytes; + const uintptr_t transient_gate = + (uintptr_t)g_stream_selected_transient_storage.gate; + const uintptr_t transient_up = + (uintptr_t)g_stream_selected_transient_storage.up; + const uintptr_t transient_down = + (uintptr_t)g_stream_selected_transient_storage.down; + ok = arena->base == persistent_base && + arena->remap == persistent_remap && + arena_begin < arena_end && + (transient_gate < arena_begin || transient_gate >= arena_end) && + (transient_up < arena_begin || transient_up >= arena_end) && + (transient_down < arena_begin || transient_down >= arena_end) && + g_stream_selected_transient_storage.remap != persistent_remap; + } const uint64_t generation_a = ok ? g_stream_selected_cache.generation : 0; if (ok && (!g_stream_selected_cache.valid || generation_a == 0 || + g_stream_selected_cache.storage_kind != + CUDA_STREAM_SELECTED_STORAGE_TRANSIENT || g_stream_selected_cache.upload_event_value != 0 || - g_stream_selected_cache.weight_domain != weight_domain)) { + g_stream_selected_cache.weight_domain != weight_domain || + g_stream_selected_cache.gate_ptr != + g_stream_selected_transient_storage.gate || + g_stream_selected_cache.up_ptr != + g_stream_selected_transient_storage.up || + g_stream_selected_cache.down_ptr != + g_stream_selected_transient_storage.down || + g_stream_selected_cache.slot_selected_ptr != + g_stream_selected_transient_storage.remap)) { ok = 0; } @@ -39729,10 +40046,12 @@ extern "C" int ds4_cuda_test_iq2_ssd_grouped_lease(void) { test_gate_offset, test_up_offset, test_down_offset, expert_bytes, expert_bytes, weight_domain) && binding_a.generation == generation_a && + binding_a.storage_kind == + CUDA_STREAM_SELECTED_STORAGE_TRANSIENT && binding_a.weight_domain == weight_domain && binding_a.selected && binding_a.selected->ptr == - g_stream_selected_cache.slot_selected_ptr; + g_stream_selected_transient_storage.remap; } if (ok) { /* A is deliberately longer than writer B's 1M-clock store. If B @@ -39768,7 +40087,7 @@ extern "C" int ds4_cuda_test_iq2_ssd_grouped_lease(void) { * and deterministically observes sentinel A. */ cuda_stream_selected_lease_delay_store_kernel<<< 1, 1, 0, g_stream_selected_upload_stream>>>( - g_stream_selected_cache.slot_selected_ptr, + g_stream_selected_transient_storage.remap, host_words[1], 1000000u); local_ok = cudaGetLastError() == cudaSuccess; } @@ -39777,13 +40096,21 @@ extern "C" int ds4_cuda_test_iq2_ssd_grouped_lease(void) { g_stream_selected_upload_done_event, g_stream_selected_upload_stream) == cudaSuccess; } + uint64_t upload_value = 0; if (local_ok) { - uint64_t upload_value = - ++g_stream_selected_upload_event_value; + upload_value = ++g_stream_selected_upload_event_value; if (upload_value == 0) { upload_value = ++g_stream_selected_upload_event_value; } + local_ok = + cuda_stream_selected_cache_bind_transient_storage( + /*logical_tier=*/0, + expert_bytes * weight_domain, + expert_bytes * weight_domain, + sizeof(selected_ids)); + } + if (local_ok) { g_stream_selected_cache.generation = generation_b; g_stream_selected_cache.upload_event_value = upload_value; @@ -39847,6 +40174,8 @@ extern "C" int ds4_cuda_test_iq2_ssd_grouped_lease(void) { test_gate_offset, test_up_offset, test_down_offset, expert_bytes, expert_bytes, weight_domain) || binding_b.generation != generation_b || + binding_b.storage_kind != + CUDA_STREAM_SELECTED_STORAGE_TRANSIENT || binding_b.upload_event_value == 0) { ok = 0; } else if (cudaMemcpyAsync( @@ -39875,6 +40204,70 @@ extern "C" int ds4_cuda_test_iq2_ssd_grouped_lease(void) { (void)cudaDeviceSynchronize(); } cuda_stream_selected_cache_release(); + if (g_stream_selected_transient_storage.gate || + g_stream_selected_transient_storage.up || + g_stream_selected_transient_storage.down || + g_stream_selected_transient_storage.remap || + g_stream_selected_cache.valid || g_stream_selected_cache.gate_ptr || + g_stream_selected_cache.up_ptr || g_stream_selected_cache.down_ptr || + g_stream_selected_cache.slot_selected_ptr || + g_stream_selected_cache.storage_kind != + CUDA_STREAM_SELECTED_STORAGE_NONE) { + ok = 0; + } + if (persistent_touched) { + std::lock_guard lock( + g_stream_expert_persistent_arena_mutex); + cuda_stream_expert_persistent_arena *arena = + &g_stream_expert_persistent_arena; + unsigned char base_got = 0; + int32_t remap_got = 0; + const int32_t resized_canary = INT32_C(0x27182818); + int isolation_ok = arena->base == persistent_base && + arena->remap == persistent_remap && + arena->bytes == persistent_bytes && + arena->remap_capacity >= + 7u * sizeof(int32_t) && + cudaSetDevice(arena->owner_device) == cudaSuccess && + cudaMemcpy(&base_got, + (char *)arena->base + + arena->bytes - 1u, + 1u, cudaMemcpyDeviceToHost) == + cudaSuccess && + cudaMemcpy(&remap_got, arena->remap + 6u, + sizeof(remap_got), + cudaMemcpyDeviceToHost) == cudaSuccess && + base_got == persistent_base_canary && + remap_got == persistent_remap_canary; + if (isolation_ok) { + isolation_ok = + cuda_stream_expert_persistent_remap_ensure_locked(9u) && + arena->base == persistent_base && + arena->remap && arena->remap != persistent_remap && + arena->remap_capacity >= 9u * sizeof(int32_t) && + cudaMemcpy(arena->remap + 8u, &resized_canary, + sizeof(resized_canary), + cudaMemcpyHostToDevice) == cudaSuccess && + cudaMemcpy(&remap_got, arena->remap + 8u, + sizeof(remap_got), + cudaMemcpyDeviceToHost) == cudaSuccess && + cudaMemcpy(&base_got, + (char *)arena->base + arena->bytes - 1u, + 1u, cudaMemcpyDeviceToHost) == cudaSuccess && + remap_got == resized_canary && + base_got == persistent_base_canary; + } + if (!isolation_ok) (void)cudaGetLastError(); + const int arena_cleanup_ok = + cuda_stream_expert_persistent_arena_release_locked(1); + if (arena_cleanup_ok) { + arena->configured_budget = saved_arena_budget; + arena->configured_expert_bytes = saved_arena_total; + arena->gate_expert_bytes = saved_arena_gate; + arena->down_expert_bytes = saved_arena_down; + } + ok = ok && isolation_ok && arena_cleanup_ok; + } if (test_model) (void)cudaFreeHost(test_model); if (host_words) (void)cudaFreeHost(host_words); cuda_stream_selected_stage_release(); @@ -40034,8 +40427,7 @@ extern "C" void ds4_gpu_set_ssd_streaming(bool enabled) { cuda_stream_selected_writer_guard writer; g_ssd_streaming_mode = enabled ? 1 : 0; if (!g_ssd_streaming_mode) { - (void)cuda_stream_selected_cache_free_storage_writer(); - cuda_stream_expert_persistent_arena_release(0); + (void)cuda_stream_expert_storage_release_writer(0); } } @@ -40082,8 +40474,7 @@ extern "C" void ds4_gpu_stream_expert_cache_reset_route_hotness(void) { } extern "C" void ds4_gpu_stream_expert_cache_release_resident(void) { - cuda_stream_selected_cache_release(); - cuda_stream_expert_persistent_arena_release(1); + cuda_stream_expert_storage_release(1); } extern "C" int ds4_gpu_stream_expert_cache_seed_selected( From 794847857b216629dd390dfaaf817ac80f9da6e2 Mon Sep 17 00:00:00 2001 From: Giorgio Oppo Date: Fri, 14 Aug 2026 23:06:18 +0200 Subject: [PATCH 037/189] cuda: cache streaming experts persistently --- ds4_cuda.cu | 930 ++++++++++++++++++++++++++++++++--- ds4_gpu.h | 16 + tests/test_gpu_model_cache.c | 32 ++ 3 files changed, 913 insertions(+), 65 deletions(-) diff --git a/ds4_cuda.cu b/ds4_cuda.cu index 3689c561c..8a78f1d1f 100644 --- a/ds4_cuda.cu +++ b/ds4_cuda.cu @@ -216,7 +216,7 @@ static int cuda_stream_selected_upload_drain_checked(void); static int cuda_stream_selected_consume_drain(void); static void cuda_stream_selected_consume_release(void); static void cuda_stream_selected_cache_release(void); -static void cuda_stream_expert_storage_release(int reset_class); +static int cuda_stream_expert_storage_release(int reset_class); typedef struct { void *base; @@ -240,7 +240,7 @@ typedef struct { static cuda_stream_expert_persistent_arena g_stream_expert_persistent_arena; static std::mutex g_stream_expert_persistent_arena_mutex; -static void cuda_stream_expert_persistent_arena_release(int reset_class); +static int cuda_stream_expert_persistent_arena_release(int reset_class); static void cuda_stream_selected_cache_invalidate(void) { g_stream_selected_cache.valid = 0; @@ -585,6 +585,8 @@ static uint64_t g_stream_selected_stage_bytes; static uint64_t g_stream_selected_stage_align = 1; static cudaStream_t g_stream_selected_upload_stream; static int g_stream_selected_upload_owner_device = -1; +static int g_stream_selected_stage_poisoned; +static int g_stream_selected_stage_test_fail_drain; static int32_t *g_stream_selected_remap_stage; static uint64_t g_stream_selected_remap_stage_capacity; static cudaStream_t g_stream_selected_readback_stream; @@ -685,6 +687,22 @@ static std::atomic g_stream_expert_persistent_arena_releases{0}; static std::atomic g_stream_expert_persistent_arena_failures{0}; static std::atomic g_stream_expert_persistent_arena_oracle_runs{0}; static std::atomic g_stream_expert_persistent_arena_oracle_failures{0}; +static std::atomic g_stream_expert_persistent_epochs_attempted{0}; +static std::atomic g_stream_expert_persistent_epochs_published{0}; +static std::atomic g_stream_expert_persistent_all_hit_epochs{0}; +static std::atomic g_stream_expert_persistent_miss_epochs{0}; +static std::atomic g_stream_expert_persistent_miss_experts{0}; +static std::atomic g_stream_expert_persistent_weight_bytes{0}; +static std::atomic g_stream_expert_persistent_remap_bytes{0}; +static std::atomic g_stream_expert_persistent_upload_failures{0}; +static std::atomic g_stream_expert_persistent_fallbacks{0}; +static std::atomic g_stream_expert_persistent_slot_invalidations{0}; +static std::atomic g_stream_expert_persistent_poisons{0}; +static std::atomic g_stream_expert_persistent_dispatches{0}; +static std::atomic g_stream_expert_transient_dispatches{0}; +static std::atomic g_stream_expert_persistent_runtime_oracle_runs{0}; +static std::atomic g_stream_expert_persistent_runtime_oracle_failures{0}; +static int g_stream_expert_persistent_test_fail_after_enqueue; static int g_stream_expert_persistent_runtime_ready; static int cuda_ok(cudaError_t err, const char *what); @@ -2850,47 +2868,99 @@ static int cuda_stream_selected_upload_drain_checked(void) { return 1; } -static void cuda_stream_selected_stage_release(void) { +static int cuda_stream_selected_stage_release_checked(void) { const int had_upload_stream = g_stream_selected_upload_stream != NULL; int previous_device = -1; (void)cudaGetDevice(&previous_device); - if (g_stream_selected_upload_owner_device >= 0) { - (void)cudaSetDevice(g_stream_selected_upload_owner_device); + if (had_upload_stream && + (g_stream_selected_upload_owner_device < 0 || + cudaSetDevice(g_stream_selected_upload_owner_device) != + cudaSuccess)) { + (void)cudaGetLastError(); + g_stream_selected_stage_poisoned = 1; + if (previous_device >= 0) (void)cudaSetDevice(previous_device); + return 0; } /* Once event publication is enabled, resize/teardown may arrive while * the last H2D epoch is still in flight. Drain before destroying the * ring events or freeing their pinned payloads. */ if (g_stream_selected_upload_stream) { - (void)cudaStreamSynchronize(g_stream_selected_upload_stream); + if (g_stream_selected_stage_test_fail_drain) { + g_stream_selected_stage_test_fail_drain = 0; + g_stream_selected_stage_poisoned = 1; + if (previous_device >= 0) (void)cudaSetDevice(previous_device); + return 0; + } + const cudaError_t sync_err = + cudaStreamSynchronize(g_stream_selected_upload_stream); + if (sync_err != cudaSuccess) { + fprintf(stderr, + "ds4: CUDA selected staging drain failed: %s\n", + cudaGetErrorString(sync_err)); + (void)cudaGetLastError(); + g_stream_selected_stage_poisoned = 1; + if (previous_device >= 0) (void)cudaSetDevice(previous_device); + return 0; + } } + int released = 1; for (size_t i = 0; i < 4; i++) { if (g_stream_selected_stage_event[i]) { - (void)cudaEventDestroy(g_stream_selected_stage_event[i]); - g_stream_selected_stage_event[i] = NULL; + if (cudaEventDestroy(g_stream_selected_stage_event[i]) == + cudaSuccess) { + g_stream_selected_stage_event[i] = NULL; + } else { + (void)cudaGetLastError(); + released = 0; + } } if (g_stream_selected_stage_raw[i]) { - (void)cudaFreeHost(g_stream_selected_stage_raw[i]); - g_stream_selected_stage_raw[i] = NULL; - g_stream_selected_stage[i] = NULL; + if (cudaFreeHost(g_stream_selected_stage_raw[i]) == + cudaSuccess) { + g_stream_selected_stage_raw[i] = NULL; + g_stream_selected_stage[i] = NULL; + } else { + (void)cudaGetLastError(); + released = 0; + } } } - g_stream_selected_stage_bytes = 0; - g_stream_selected_stage_align = 1; if (g_stream_selected_remap_stage) { - (void)cudaFreeHost(g_stream_selected_remap_stage); - g_stream_selected_remap_stage = NULL; + if (cudaFreeHost(g_stream_selected_remap_stage) == cudaSuccess) { + g_stream_selected_remap_stage = NULL; + g_stream_selected_remap_stage_capacity = 0; + } else { + (void)cudaGetLastError(); + released = 0; + } } - g_stream_selected_remap_stage_capacity = 0; - if (g_stream_selected_upload_stream) { - (void)cudaStreamDestroy(g_stream_selected_upload_stream); + if (released && g_stream_selected_upload_stream && + cudaStreamDestroy(g_stream_selected_upload_stream) == cudaSuccess) { g_stream_selected_upload_stream = NULL; + } else if (released && g_stream_selected_upload_stream) { + (void)cudaGetLastError(); + released = 0; } + if (!released) { + g_stream_selected_stage_poisoned = 1; + if (previous_device >= 0) (void)cudaSetDevice(previous_device); + return 0; + } + g_stream_selected_stage_bytes = 0; + g_stream_selected_stage_align = 1; + g_stream_selected_remap_stage_capacity = 0; g_stream_selected_upload_owner_device = -1; + g_stream_selected_stage_poisoned = 0; if (had_upload_stream) { uint64_t value = ++g_stream_selected_upload_event_value; if (value == 0) ++g_stream_selected_upload_event_value; } if (previous_device >= 0) (void)cudaSetDevice(previous_device); + return 1; +} + +static void cuda_stream_selected_stage_release(void) { + (void)cuda_stream_selected_stage_release_checked(); } static int cuda_stream_selected_stage_pool_alloc(uint64_t bytes) { @@ -2901,9 +2971,11 @@ static int cuda_stream_selected_stage_pool_alloc(uint64_t bytes) { &allocation_bytes)) { return 0; } - if (g_stream_selected_stage_bytes >= bytes && + if (!g_stream_selected_stage_poisoned && + !g_stream_selected_stage_test_fail_drain && + g_stream_selected_stage_bytes >= bytes && g_stream_selected_stage_align == align) return 1; - cuda_stream_selected_stage_release(); + if (!cuda_stream_selected_stage_release_checked()) return 0; cudaError_t err = cudaStreamCreateWithFlags( &g_stream_selected_upload_stream, cudaStreamNonBlocking); if (err != cudaSuccess) { @@ -2966,9 +3038,14 @@ static int cuda_stream_selected_remap_stage_ensure(uint64_t count) { cudaStreamSynchronize(g_stream_selected_upload_stream) != cudaSuccess) { (void)cudaGetLastError(); + g_stream_selected_stage_poisoned = 1; + return 0; + } + if (cudaFreeHost(g_stream_selected_remap_stage) != cudaSuccess) { + (void)cudaGetLastError(); + g_stream_selected_stage_poisoned = 1; return 0; } - (void)cudaFreeHost(g_stream_selected_remap_stage); g_stream_selected_remap_stage = NULL; g_stream_selected_remap_stage_capacity = 0; } @@ -3096,6 +3173,21 @@ typedef struct { uint64_t arena_failures; uint64_t arena_oracle_runs; uint64_t arena_oracle_failures; + uint64_t epochs_attempted; + uint64_t epochs_published; + uint64_t all_hit_epochs; + uint64_t miss_epochs; + uint64_t miss_experts; + uint64_t weight_bytes_uploaded; + uint64_t remap_bytes_uploaded; + uint64_t upload_failures; + uint64_t fallbacks; + uint64_t slot_invalidations; + uint64_t poisons; + uint64_t persistent_dispatches; + uint64_t transient_dispatches; + uint64_t runtime_oracle_runs; + uint64_t runtime_oracle_failures; int enabled; int required; int stats; @@ -3144,6 +3236,9 @@ typedef struct { uint64_t lru_clock; } cuda_stream_expert_persistent_state; +static cuda_stream_expert_persistent_state + g_stream_expert_persistent_state; + typedef struct { cuda_stream_expert_persistent_key key; cuda_stream_expert_persistent_key victim; @@ -3301,6 +3396,22 @@ static int cuda_stream_expert_persistent_state_init( return 1; } +static void cuda_stream_expert_persistent_state_clear( + cuda_stream_expert_persistent_state *state) { + if (!state) return; + state->slots.clear(); + state->free_slots.clear(); + memset(&state->size_class, 0, sizeof(state->size_class)); + state->lru_clock = 0; +} + +static uint32_t cuda_stream_expert_persistent_state_count( + const cuda_stream_expert_persistent_state *state) { + if (!state || state->free_slots.size() > state->slots.size()) return 0; + const size_t count = state->slots.size() - state->free_slots.size(); + return count <= UINT32_MAX ? (uint32_t)count : 0; +} + static int cuda_stream_expert_persistent_state_valid( const cuda_stream_expert_persistent_state *state) { if (!state || state->size_class.capacity == 0 || @@ -3646,7 +3757,7 @@ static int cuda_stream_expert_persistent_plan_build( } const uint64_t domain64 = (uint64_t)max_slot - min_slot + 1u; - if (domain64 == 0 || domain64 > UINT32_MAX) { + if (domain64 == 0 || domain64 > INT_MAX) { cuda_stream_expert_persistent_note_reject( &g_stream_expert_persistent_overflow_rejects); return 0; @@ -3825,6 +3936,12 @@ static int cuda_stream_expert_persistent_requested(void) { return g_stream_expert_persistent_enabled; } +static int cuda_stream_expert_persistent_require_requested(void) { + std::call_once(g_stream_expert_persistent_once, + cuda_stream_expert_persistent_init); + return g_stream_expert_persistent_required; +} + extern "C" void ds4_cuda_stream_expert_persistent_get_report( ds4_cuda_stream_expert_persistent_report *report) { if (!report) return; @@ -3862,6 +3979,32 @@ extern "C" void ds4_cuda_stream_expert_persistent_get_report( g_stream_expert_persistent_arena_oracle_runs.load(); out.arena_oracle_failures = g_stream_expert_persistent_arena_oracle_failures.load(); + out.epochs_attempted = + g_stream_expert_persistent_epochs_attempted.load(); + out.epochs_published = + g_stream_expert_persistent_epochs_published.load(); + out.all_hit_epochs = + g_stream_expert_persistent_all_hit_epochs.load(); + out.miss_epochs = g_stream_expert_persistent_miss_epochs.load(); + out.miss_experts = g_stream_expert_persistent_miss_experts.load(); + out.weight_bytes_uploaded = + g_stream_expert_persistent_weight_bytes.load(); + out.remap_bytes_uploaded = + g_stream_expert_persistent_remap_bytes.load(); + out.upload_failures = + g_stream_expert_persistent_upload_failures.load(); + out.fallbacks = g_stream_expert_persistent_fallbacks.load(); + out.slot_invalidations = + g_stream_expert_persistent_slot_invalidations.load(); + out.poisons = g_stream_expert_persistent_poisons.load(); + out.persistent_dispatches = + g_stream_expert_persistent_dispatches.load(); + out.transient_dispatches = + g_stream_expert_transient_dispatches.load(); + out.runtime_oracle_runs = + g_stream_expert_persistent_runtime_oracle_runs.load(); + out.runtime_oracle_failures = + g_stream_expert_persistent_runtime_oracle_failures.load(); out.enabled = g_stream_expert_persistent_enabled; out.required = g_stream_expert_persistent_required; out.stats = g_stream_expert_persistent_stats; @@ -4949,13 +5092,12 @@ static int cuda_stream_expert_storage_release_writer(int reset_class) { !cuda_stream_selected_cache_free_storage_writer()) { return 0; } - cuda_stream_expert_persistent_arena_release(reset_class); - return 1; + return cuda_stream_expert_persistent_arena_release(reset_class); } -static void cuda_stream_expert_storage_release(int reset_class) { +static int cuda_stream_expert_storage_release(int reset_class) { cuda_stream_selected_writer_guard writer; - (void)cuda_stream_expert_storage_release_writer(reset_class); + return cuda_stream_expert_storage_release_writer(reset_class); } static void cuda_stream_selected_consume_release(void) { @@ -6254,6 +6396,7 @@ static int cublas_ok(cublasStatus_t st, const char *what) { } extern "C" int ds4_gpu_init_multi(const ds4_gpu_config *cfg) { + g_stream_expert_persistent_runtime_ready = 0; cuda_q8_fold_invalidate_all(); ds4_mmq_set_gb10_optimizations(0); if (!cfg || cfg->n_gpus < 1 || cfg->n_gpus > DS4_MAX_GPUS) return 0; @@ -6451,6 +6594,7 @@ extern "C" int ds4_gpu_init_multi(const ds4_gpu_config *cfg) { } g_cublas_ready = 1; + g_stream_expert_persistent_runtime_ready = g_n_gpus == 1; return 1; } @@ -6463,6 +6607,7 @@ extern "C" int ds4_gpu_init(void) { } extern "C" void ds4_gpu_cleanup(void) { + g_stream_expert_persistent_runtime_ready = 0; (void)cudaDeviceSynchronize(); cuda_decode_graphs_shutdown(); cuda_q8_fold_release_all(); @@ -6521,7 +6666,7 @@ extern "C" void ds4_gpu_cleanup(void) { /* The selected cache may still be the destination of an event-published * upload. Drain and retire its auxiliary streams before freeing device * storage or invalidating the owner-device context. */ - cuda_stream_expert_storage_release(1); + (void)cuda_stream_expert_storage_release(1); cuda_stream_selected_stage_release(); cuda_stream_selected_event_pipeline_release(); cuda_stream_selected_consume_release(); @@ -7398,7 +7543,7 @@ extern "C" int ds4_gpu_set_model_map(const void *model_map, uint64_t model_size) if (!model_map || model_size == 0) return 0; if (g_model_host_base == model_map && g_model_registered_size == model_size) return 1; cuda_q8_fold_invalidate_all(); - cuda_stream_expert_storage_release(1); + if (!cuda_stream_expert_storage_release(1)) return 0; cuda_f16_pair_chunk32_release_all(); cuda_model_range_release_all(); cuda_q8_f16_cache_release_all(); @@ -7601,7 +7746,7 @@ extern "C" int ds4_gpu_register_model_map_no_copy(const void *model_map, uint64_ if (g_model_host_base == model_map && g_model_registered_size == model_size) return 1; cuda_q8_fold_invalidate_all(); - cuda_stream_expert_storage_release(1); + if (!cuda_stream_expert_storage_release(1)) return 0; cuda_f16_pair_chunk32_release_all(); cuda_model_range_release_all(); cuda_q8_f16_cache_release_all(); @@ -29042,8 +29187,10 @@ static int cuda_stream_selected_binding_acquire( if (gate_expert_bytes == 0 || down_expert_bytes == 0 || required_slot_count > UINT64_MAX / sizeof(int32_t) || !g_stream_selected_cache.valid || - g_stream_selected_cache.storage_kind != - CUDA_STREAM_SELECTED_STORAGE_TRANSIENT || + (g_stream_selected_cache.storage_kind != + CUDA_STREAM_SELECTED_STORAGE_TRANSIENT && + g_stream_selected_cache.storage_kind != + CUDA_STREAM_SELECTED_STORAGE_PERSISTENT) || g_stream_selected_cache.logical_tier != logical_tier || g_stream_selected_cache.model_map != model_map || g_stream_selected_cache.layer != layer_index || @@ -29094,6 +29241,14 @@ static int cuda_stream_selected_binding_acquire( binding->generation = g_stream_selected_cache.generation; binding->upload_event_value = g_stream_selected_cache.upload_event_value; + if (binding->storage_kind == + CUDA_STREAM_SELECTED_STORAGE_PERSISTENT) { + g_stream_expert_persistent_dispatches.fetch_add( + 1, std::memory_order_relaxed); + } else { + g_stream_expert_transient_dispatches.fetch_add( + 1, std::memory_order_relaxed); + } return 1; } @@ -29408,6 +29563,9 @@ static int routed_moe_launch( : cuda_resolve_weight_ptr(model_map, down_offset, down_bytes, logical_tier, "moe_down"); if (!gate_w || !up_w || !down_w) return 0; + const uint32_t raw_weight_domain = use_stream_selected_cache ? + stream_binding.weight_domain : n_total_expert; + if (raw_weight_domain == 0u || raw_weight_domain > INT_MAX) return 0; /* Native MXFP4 routed experts use the vendored MMVQ decode kernels and * MMQ matrix kernels. On Blackwell the latter dispatch to FP4 MMA; older @@ -29428,9 +29586,7 @@ static int routed_moe_launch( return 0; } const ds4_gpu_tensor *mx_selected = selected; - const uint32_t weight_experts = use_stream_selected_cache ? - stream_binding.weight_domain : n_total_expert; - if (weight_experts == 0u) return 0; + const uint32_t weight_experts = raw_weight_domain; const cudaStream_t stream = n_tokens == 1u ? cuda_decode_stream() : (cudaStream_t)0; @@ -29557,7 +29713,6 @@ static int routed_moe_launch( stream_binding.top6_unique, grouped_raw_layout, use_stream_selected_cache && tensor_devices_ok && - stream_binding.slot_base == 0u && stream_binding.weight_domain > 0u && stream_binding.weight_domain <= INT_MAX && n_tokens <= INT_MAX && n_expert <= INT_MAX && @@ -29822,7 +29977,7 @@ static int routed_moe_launch( n_tokens >= 128u && getenv("DS4_CUDA_MOE_NO_DOWN_TILE16") == NULL; const uint32_t use_small_sorted_prep = owned_filtered && q4k_path && n_tokens <= 16u && pair_count <= 96u && - n_total_expert <= 128u && use_sorted_pairs && use_expert_tiles && + raw_weight_domain <= 128u && use_sorted_pairs && use_expert_tiles && getenv("DS4_CUDA_MOE_NO_SMALL_SORTED_PREP") == NULL; const uint32_t force_q4_down_rowspan = getenv("DS4_CUDA_MOE_DOWN_ROW512") != NULL || @@ -29944,17 +30099,17 @@ static int routed_moe_launch( ok = cuda_ok(cudaGetLastError(), "routed_moe x quantize launch"); if (prof_ev[1]) (void)cudaEventRecord(prof_ev[1], 0); if (ok && use_sorted_pairs) { - const uint64_t counts_bytes = (uint64_t)n_total_expert * sizeof(uint32_t); - const uint64_t offsets_bytes = ((uint64_t)n_total_expert + 1ull) * sizeof(uint32_t); - const uint64_t cursors_bytes = (uint64_t)n_total_expert * sizeof(uint32_t); + const uint64_t counts_bytes = (uint64_t)raw_weight_domain * sizeof(uint32_t); + const uint64_t offsets_bytes = ((uint64_t)raw_weight_domain + 1ull) * sizeof(uint32_t); + const uint64_t cursors_bytes = (uint64_t)raw_weight_domain * sizeof(uint32_t); const uint64_t sorted_bytes = (uint64_t)pair_count * sizeof(uint32_t); - tile_capacity = (pair_count + expert_tile_m - 1u) / expert_tile_m + n_total_expert; - tile16_capacity = (use_down_tile16 || use_q4_mma_tiles16) ? ((pair_count + 15u) / 16u + n_total_expert) : 0u; - const uint64_t tile_offsets_bytes = ((uint64_t)n_total_expert + 1ull) * sizeof(uint32_t); + tile_capacity = (pair_count + expert_tile_m - 1u) / expert_tile_m + raw_weight_domain; + tile16_capacity = (use_down_tile16 || use_q4_mma_tiles16) ? ((pair_count + 15u) / 16u + raw_weight_domain) : 0u; + const uint64_t tile_offsets_bytes = ((uint64_t)raw_weight_domain + 1ull) * sizeof(uint32_t); const uint64_t tile_total_bytes = sizeof(uint32_t); const uint64_t tile_experts_bytes = (uint64_t)tile_capacity * sizeof(uint32_t); const uint64_t tile_starts_bytes = (uint64_t)tile_capacity * sizeof(uint32_t); - const uint64_t tile16_offsets_bytes = (use_down_tile16 || use_q4_mma_tiles16) ? (((uint64_t)n_total_expert + 1ull) * sizeof(uint32_t)) : 0u; + const uint64_t tile16_offsets_bytes = (use_down_tile16 || use_q4_mma_tiles16) ? (((uint64_t)raw_weight_domain + 1ull) * sizeof(uint32_t)) : 0u; const uint64_t tile16_total_bytes = (use_down_tile16 || use_q4_mma_tiles16) ? sizeof(uint32_t) : 0u; const uint64_t tile16_experts_bytes = (uint64_t)tile16_capacity * sizeof(uint32_t); const uint64_t tile16_starts_bytes = (uint64_t)tile16_capacity * sizeof(uint32_t); @@ -29991,7 +30146,7 @@ static int routed_moe_launch( counts, offsets, cursors, sorted_pairs, tile_offsets, tile_total, tile_experts, tile_starts, tile16_offsets, tile16_total, tile16_experts, tile16_starts, - (const int32_t *)selected->ptr, pair_count, n_total_expert, + (const int32_t *)selected->ptr, pair_count, raw_weight_domain, expert_tile_m, use_down_tile16 || use_q4_mma_tiles16); ok = cuda_ok(cudaGetLastError(), "routed_moe small sorted setup launch"); @@ -30004,11 +30159,11 @@ static int routed_moe_launch( counts, (const int32_t *)selected->ptr, pair_count, - n_total_expert); + raw_weight_domain); ok = cuda_ok(cudaGetLastError(), "routed_moe sorted count launch"); } if (ok && !use_small_sorted_prep) { - moe_prefix_sorted_pairs_kernel<<<1, 1, 0, cuda_decode_stream()>>>(offsets, cursors, counts, n_total_expert); + moe_prefix_sorted_pairs_kernel<<<1, 1, 0, cuda_decode_stream()>>>(offsets, cursors, counts, raw_weight_domain); ok = cuda_ok(cudaGetLastError(), "routed_moe sorted prefix launch"); } if (ok && !use_small_sorted_prep) { @@ -30017,27 +30172,27 @@ static int routed_moe_launch( cursors, (const int32_t *)selected->ptr, pair_count, - n_total_expert); + raw_weight_domain); ok = cuda_ok(cudaGetLastError(), "routed_moe sorted scatter launch"); } if (ok && use_expert_tiles && !use_small_sorted_prep) { - moe_build_expert_tile_offsets_kernel<<<1, 1, 0, cuda_decode_stream()>>>(tile_offsets, tile_total, counts, expert_tile_m, n_total_expert); + moe_build_expert_tile_offsets_kernel<<<1, 1, 0, cuda_decode_stream()>>>(tile_offsets, tile_total, counts, expert_tile_m, raw_weight_domain); ok = cuda_ok(cudaGetLastError(), "routed_moe expert tile offsets launch"); } if (ok && use_expert_tiles && !use_small_sorted_prep) { - moe_build_expert_tiles_kernel<<<(n_total_expert + 255u) / 256u, 256, 0, cuda_decode_stream()>>>( - tile_experts, tile_starts, tile_offsets, counts, expert_tile_m, n_total_expert); + moe_build_expert_tiles_kernel<<<(raw_weight_domain + 255u) / 256u, 256, 0, cuda_decode_stream()>>>( + tile_experts, tile_starts, tile_offsets, counts, expert_tile_m, raw_weight_domain); ok = cuda_ok(cudaGetLastError(), "routed_moe expert tiles launch"); } if (ok && use_expert_tiles && !use_small_sorted_prep && (use_down_tile16 || use_q4_mma_tiles16)) { - moe_build_expert_tile_offsets_kernel<<<1, 1, 0, cuda_decode_stream()>>>(tile16_offsets, tile16_total, counts, 16u, n_total_expert); + moe_build_expert_tile_offsets_kernel<<<1, 1, 0, cuda_decode_stream()>>>(tile16_offsets, tile16_total, counts, 16u, raw_weight_domain); ok = cuda_ok(cudaGetLastError(), "routed_moe expert tile16 offsets launch"); } if (ok && use_expert_tiles && !use_small_sorted_prep && (use_down_tile16 || use_q4_mma_tiles16)) { - moe_build_expert_tiles_kernel<<<(n_total_expert + 255u) / 256u, 256, 0, cuda_decode_stream()>>>( - tile16_experts, tile16_starts, tile16_offsets, counts, 16u, n_total_expert); + moe_build_expert_tiles_kernel<<<(raw_weight_domain + 255u) / 256u, 256, 0, cuda_decode_stream()>>>( + tile16_experts, tile16_starts, tile16_offsets, counts, 16u, raw_weight_domain); ok = cuda_ok(cudaGetLastError(), "routed_moe expert tile16 launch"); } } @@ -30062,7 +30217,7 @@ static int routed_moe_launch( tile16_total && tile16_experts && tile16_starts && xq_blocks == 16u && cuda_q4_mma_tile16_shmem_ok(0); if (use_q4_mma_t16 && use_gate_row2048) { - const unsigned t16cap = (unsigned)((pair_count + 15u) / 16u + n_total_expert); + const unsigned t16cap = (unsigned)((pair_count + 15u) / 16u + raw_weight_domain); const size_t t16sh = 16u * 16u * sizeof(cuda_block_q8_K); if (gate_row_span == 512u) { dim3 tgrid((expert_mid_dim + 511u) / 512u, t16cap, 1); @@ -30432,7 +30587,7 @@ static int routed_moe_launch( expert_mid_dim, n_tokens * n_expert, 0u, - n_total_expert); + raw_weight_domain); ok = cuda_ok(cudaGetLastError(), "owned routed_moe active mid quantize launch"); } else { @@ -30537,7 +30692,7 @@ static int routed_moe_launch( tile16_total && tile16_experts && tile16_starts && midq_blocks <= 16u && cuda_q4_mma_tile16_shmem_ok(1); if (use_q4_down_t16 && use_q4_down_rowspan) { - const unsigned t16cap = (unsigned)((pair_count + 15u) / 16u + n_total_expert); + const unsigned t16cap = (unsigned)((pair_count + 15u) / 16u + raw_weight_domain); const size_t dt16sh = 16u * (size_t)midq_blocks * sizeof(cuda_block_q8_K); if (down_row_span == 512u) { dim3 tgrid((out_dim + 511u) / 512u, t16cap, 1); @@ -32069,6 +32224,16 @@ static int cuda_stream_selected_ranges_valid( down_bytes <= table->model_size - table->down_offset; } +/* 1 publishes persistent metadata, 0 is a safe pre-enqueue rejection, and + * -1 is a post-enqueue/fail-closed result. Caller owns the writer epoch. */ +static int cuda_stream_expert_persistent_try_load_writer( + const ds4_gpu_stream_expert_table *table, + const int32_t *selected_ids, + uint32_t slot_count, + uint64_t *upload_event_out, + int *submitted_any_out, + int force_for_oracle); + static int cuda_stream_selected_cache_begin_load_impl( const ds4_gpu_stream_expert_table *table, const int32_t *selected_ids, @@ -32101,6 +32266,24 @@ static int cuda_stream_selected_cache_begin_load_impl( (void)cudaGetLastError(); return 0; } + if (g_stream_expert_persistent_runtime_ready && + cuda_stream_expert_persistent_requested()) { + const int persistent_rc = + cuda_stream_expert_persistent_try_load_writer( + table, selected_ids, slot_count, + upload_event_out, submitted_any_out, + /*force_for_oracle=*/0); + if (persistent_rc > 0) { + writer.publish_valid(); + return 1; + } + if (persistent_rc < 0 || + cuda_stream_expert_persistent_require_requested()) { + return 0; + } + g_stream_expert_persistent_fallbacks.fetch_add( + 1, std::memory_order_relaxed); + } if (storage->poisoned && !cuda_stream_selected_cache_free_storage_writer()) { return 0; @@ -39360,6 +39543,8 @@ static int cuda_stream_expert_persistent_arena_release_locked( g_stream_expert_persistent_arena_releases.fetch_add( 1, std::memory_order_relaxed); } + cuda_stream_expert_persistent_state_clear( + &g_stream_expert_persistent_state); memset(arena, 0, sizeof(*arena)); arena->owner_device = -1; arena->configured_budget = configured_budget; @@ -39370,10 +39555,10 @@ static int cuda_stream_expert_persistent_arena_release_locked( return 1; } -static void cuda_stream_expert_persistent_arena_release(int reset_class) { +static int cuda_stream_expert_persistent_arena_release(int reset_class) { std::lock_guard lock( g_stream_expert_persistent_arena_mutex); - (void)cuda_stream_expert_persistent_arena_release_locked(reset_class); + return cuda_stream_expert_persistent_arena_release_locked(reset_class); } static int cuda_stream_expert_persistent_arena_ensure_locked(void) { @@ -39490,6 +39675,609 @@ static int cuda_stream_expert_persistent_remap_ensure_locked( return 1; } +static int cuda_stream_expert_persistent_remap_upload( + int32_t *dst, + const uint32_t *src, + uint64_t count, + uint64_t *upload_event_out, + int *submitted_out) { + if (upload_event_out) *upload_event_out = 0; + if (submitted_out) *submitted_out = 0; + if (!dst || !src || count == 0 || + count > SIZE_MAX / sizeof(int32_t)) { + return 0; + } + const size_t bytes = (size_t)count * sizeof(int32_t); + if (!upload_event_out || + cuda_stream_selected_event_pipeline_oracle_requested()) { + if (!cuda_stream_selected_consume_drain()) return 0; + if (submitted_out) *submitted_out = 1; + const cudaError_t err = cudaMemcpy( + dst, src, bytes, cudaMemcpyHostToDevice); + if (err != cudaSuccess) { + fprintf(stderr, + "ds4: CUDA persistent expert remap copy failed: %s\n", + cudaGetErrorString(err)); + (void)cudaGetLastError(); + return 0; + } + return 1; + } + + if (!cuda_stream_selected_stage_pool_alloc(1u) || + !cuda_stream_selected_event_pipeline_ensure() || + !cuda_stream_selected_remap_stage_ensure(count) || + !cuda_stream_selected_consume_wait_on_upload()) { + return 0; + } + memcpy(g_stream_selected_remap_stage, src, bytes); + cudaError_t err = cudaMemcpyAsync( + dst, g_stream_selected_remap_stage, bytes, + cudaMemcpyHostToDevice, g_stream_selected_upload_stream); + if (err != cudaSuccess) { + (void)cudaGetLastError(); + return 0; + } + if (submitted_out) *submitted_out = 1; + err = cudaEventRecord(g_stream_selected_upload_done_event, + g_stream_selected_upload_stream); + if (err != cudaSuccess) { + (void)cudaGetLastError(); + return cuda_stream_selected_upload_fail( + "persistent expert remap"); + } + uint64_t value = ++g_stream_selected_upload_event_value; + if (value == 0) value = ++g_stream_selected_upload_event_value; + *upload_event_out = value; + g_stream_selected_event_uploads.fetch_add( + 1, std::memory_order_relaxed); + return 1; +} + +static void cuda_stream_expert_persistent_invalidate_all_locked( + cuda_stream_expert_persistent_arena *arena) { + const uint32_t invalidated = + cuda_stream_expert_persistent_state_count( + &g_stream_expert_persistent_state); + if (!cuda_stream_expert_persistent_state_init( + &g_stream_expert_persistent_state, + arena->capacity, arena->gate_expert_bytes, + arena->down_expert_bytes)) { + cuda_stream_expert_persistent_state_clear( + &g_stream_expert_persistent_state); + } + arena->valid_count = 0; + g_stream_expert_persistent_slot_invalidations.fetch_add( + invalidated, std::memory_order_relaxed); +} + +static int cuda_stream_expert_persistent_try_load_writer( + const ds4_gpu_stream_expert_table *table, + const int32_t *selected_ids, + uint32_t slot_count, + uint64_t *upload_event_out, + int *submitted_any_out, + int force_for_oracle) { + if (upload_event_out) *upload_event_out = 0; + if (submitted_any_out) *submitted_any_out = 0; + g_stream_expert_persistent_epochs_attempted.fetch_add( + 1, std::memory_order_relaxed); + if (!g_stream_selected_writer_active || !g_ssd_streaming_mode || + g_n_gpus != 1 || !cuda_stream_selected_ranges_valid(table) || + !selected_ids || slot_count == 0 || + (!force_for_oracle && + (!g_stream_expert_persistent_runtime_ready || + !cuda_stream_expert_persistent_requested()))) { + return 0; + } + + std::lock_guard lock( + g_stream_expert_persistent_arena_mutex); + cuda_stream_expert_persistent_arena *arena = + &g_stream_expert_persistent_arena; + uint64_t two_gate = 0; + uint64_t total_class = 0; + if (arena->poisoned) return -1; + if (g_stream_selected_stage_poisoned && + !cuda_stream_selected_stage_release_checked()) { + arena->poisoned = 1; + g_stream_expert_persistent_poisons.fetch_add( + 1, std::memory_order_relaxed); + return -1; + } + if (arena->configured_budget == 0 || + !cuda_stream_expert_persistent_mul_u64( + table->gate_expert_bytes, 2u, &two_gate) || + !cuda_stream_expert_persistent_add_u64( + two_gate, table->down_expert_bytes, &total_class)) { + return 0; + } + if (arena->configured_expert_bytes == 0) { + arena->configured_expert_bytes = total_class; + } + if (arena->configured_expert_bytes != total_class) return 0; + if (arena->gate_expert_bytes == 0 && arena->down_expert_bytes == 0) { + arena->gate_expert_bytes = table->gate_expert_bytes; + arena->down_expert_bytes = table->down_expert_bytes; + } + if (arena->gate_expert_bytes != table->gate_expert_bytes || + arena->down_expert_bytes != table->down_expert_bytes) { + return 0; + } + if (!cuda_stream_expert_persistent_arena_ensure_locked()) { + return arena->poisoned ? -1 : 0; + } + + cuda_stream_expert_persistent_class expected_class = {}; + if (!cuda_stream_expert_persistent_class_make( + &expected_class, arena->capacity, + arena->gate_expert_bytes, arena->down_expert_bytes)) { + return 0; + } + if (g_stream_expert_persistent_state.slots.empty()) { + if (!cuda_stream_expert_persistent_state_init( + &g_stream_expert_persistent_state, + arena->capacity, arena->gate_expert_bytes, + arena->down_expert_bytes)) { + return 0; + } + } else if (!cuda_stream_expert_persistent_class_equal( + &g_stream_expert_persistent_state.size_class, + &expected_class) || + !cuda_stream_expert_persistent_state_valid( + &g_stream_expert_persistent_state)) { + cuda_stream_expert_persistent_invalidate_all_locked(arena); + if (!cuda_stream_expert_persistent_state_valid( + &g_stream_expert_persistent_state)) { + return 0; + } + } + + cuda_stream_expert_persistent_plan plan = {}; + if (!cuda_stream_expert_persistent_plan_build( + &g_stream_expert_persistent_state, table, + selected_ids, slot_count, &plan)) { + cuda_stream_expert_persistent_plan_rollback(&plan); + return 0; + } + if (!cuda_stream_expert_persistent_remap_ensure_locked(slot_count)) { + cuda_stream_expert_persistent_plan_rollback(&plan); + return arena->poisoned ? -1 : 0; + } + + std::vector tasks; + uint64_t weight_bytes = 0; + if (plan.loads.size() > UINT32_MAX / 3u) { + cuda_stream_expert_persistent_plan_rollback(&plan); + return 0; + } + int task_plan_ok = 1; + try { + tasks.reserve(plan.loads.size() * 3u); + uint32_t ordinal = 0; + for (const cuda_stream_expert_persistent_load &load : plan.loads) { + if (load.slot >= arena->capacity) { + task_plan_ok = 0; + break; + } + const uint64_t gate_dst = + (uint64_t)load.slot * table->gate_expert_bytes; + const uint64_t down_dst = + (uint64_t)load.slot * table->down_expert_bytes; + const uint64_t expert = load.key.expert_id; + tasks.push_back({arena->gate + gate_dst, + table->gate_offset + expert * table->gate_expert_bytes, + table->gate_expert_bytes, ordinal++}); + tasks.push_back({arena->up + gate_dst, + table->up_offset + expert * table->gate_expert_bytes, + table->gate_expert_bytes, ordinal++}); + tasks.push_back({arena->down + down_dst, + table->down_offset + expert * table->down_expert_bytes, + table->down_expert_bytes, ordinal++}); + } + } catch (...) { + task_plan_ok = 0; + } + if (!task_plan_ok) { + cuda_stream_expert_persistent_plan_rollback(&plan); + return 0; + } + if (!cuda_stream_expert_persistent_mul_u64( + plan.loads.size(), expected_class.bytes_per_slot, + &weight_bytes)) { + cuda_stream_expert_persistent_plan_rollback(&plan); + return 0; + } + uint64_t gate_capacity = 0; + uint64_t down_capacity = 0; + if (!cuda_stream_expert_persistent_mul_u64( + arena->capacity, table->gate_expert_bytes, &gate_capacity) || + !cuda_stream_expert_persistent_mul_u64( + arena->capacity, table->down_expert_bytes, &down_capacity)) { + cuda_stream_expert_persistent_plan_rollback(&plan); + return 0; + } + const uint64_t remap_bytes = + (uint64_t)slot_count * sizeof(int32_t); + int submitted = 0; + int copied = 0; + if (tasks.empty()) { + copied = cuda_stream_expert_persistent_remap_upload( + arena->remap, plan.remap.data(), plan.remap.size(), + upload_event_out, &submitted); + } else { + copied = cuda_model_copy_tasks_to_device_streamed( + tasks, table->model_map, table->model_size, + arena->remap, (const int32_t *)plan.remap.data(), + plan.remap.size(), /*run_oracle=*/0, + /*chunk_override=*/0, &submitted, + upload_event_out, "persistent expert cache"); + } + if (submitted_any_out) *submitted_any_out = submitted; + if (g_stream_expert_persistent_test_fail_after_enqueue && submitted) { + copied = 0; + g_stream_expert_persistent_test_fail_after_enqueue = 0; + } + const int expect_async_token = upload_event_out != NULL && + !cuda_stream_selected_event_pipeline_oracle_requested(); + if (!copied || + (expect_async_token && *upload_event_out == 0)) { + if (!submitted) { + cuda_stream_expert_persistent_plan_rollback(&plan); + if (g_stream_selected_stage_poisoned) { + arena->poisoned = 1; + g_stream_expert_persistent_poisons.fetch_add( + 1, std::memory_order_relaxed); + return -1; + } + return 0; + } + const int drained = cuda_stream_selected_upload_drain_checked(); + cuda_stream_expert_persistent_invalidate_all_locked(arena); + if (!drained) { + arena->poisoned = 1; + g_stream_expert_persistent_poisons.fetch_add( + 1, std::memory_order_relaxed); + } + cuda_stream_expert_persistent_plan_rollback(&plan); + g_stream_expert_persistent_upload_failures.fetch_add( + 1, std::memory_order_relaxed); + return -1; + } + + if (!cuda_stream_expert_persistent_plan_commit( + &g_stream_expert_persistent_state, &plan)) { + const int drained = cuda_stream_selected_upload_drain_checked(); + cuda_stream_expert_persistent_invalidate_all_locked(arena); + if (!drained) { + arena->poisoned = 1; + g_stream_expert_persistent_poisons.fetch_add( + 1, std::memory_order_relaxed); + } + cuda_stream_expert_persistent_plan_rollback(&plan); + g_stream_expert_persistent_upload_failures.fetch_add( + 1, std::memory_order_relaxed); + return -1; + } + arena->valid_count = cuda_stream_expert_persistent_state_count( + &g_stream_expert_persistent_state); + + int top6_unique = (slot_count % 6u) == 0u; + for (uint32_t base = 0; top6_unique && base < slot_count; base += 6u) { + for (uint32_t i = 0; top6_unique && i < 6u; i++) { + for (uint32_t j = i + 1u; j < 6u; j++) { + if (selected_ids[base + i] == selected_ids[base + j]) { + top6_unique = 0; + break; + } + } + } + } + g_stream_selected_cache.storage_kind = + CUDA_STREAM_SELECTED_STORAGE_PERSISTENT; + g_stream_selected_cache.logical_tier = 0; + g_stream_selected_cache.model_map = table->model_map; + g_stream_selected_cache.layer = table->layer; + g_stream_selected_cache.n_total_expert = table->n_total_expert; + g_stream_selected_cache.slot_count = slot_count; + g_stream_selected_cache.compact_count = plan.unique_count; + g_stream_selected_cache.slot_base = plan.slot_base; + g_stream_selected_cache.weight_domain = plan.weight_domain; + g_stream_selected_cache.top6_unique = top6_unique; + uint64_t generation = ++g_stream_selected_cache_generation; + if (generation == 0) generation = ++g_stream_selected_cache_generation; + g_stream_selected_cache.generation = generation; + g_stream_selected_cache.upload_event_value = + upload_event_out ? *upload_event_out : 0; + g_stream_selected_cache.gate_offset = table->gate_offset; + g_stream_selected_cache.up_offset = table->up_offset; + g_stream_selected_cache.down_offset = table->down_offset; + g_stream_selected_cache.gate_expert_bytes = table->gate_expert_bytes; + g_stream_selected_cache.down_expert_bytes = table->down_expert_bytes; + g_stream_selected_cache.gate_ptr = arena->gate; + g_stream_selected_cache.up_ptr = arena->up; + g_stream_selected_cache.down_ptr = arena->down; + g_stream_selected_cache.gate_capacity = gate_capacity; + g_stream_selected_cache.up_capacity = gate_capacity; + g_stream_selected_cache.down_capacity = down_capacity; + g_stream_selected_cache.slot_selected_ptr = arena->remap; + g_stream_selected_cache.slot_selected_capacity = arena->remap_capacity; + g_stream_selected_cache.slot_selected_tensor.ptr = arena->remap; + g_stream_selected_cache.slot_selected_tensor.bytes = remap_bytes; + g_stream_selected_cache.slot_selected_tensor.owner = 0; + g_stream_selected_cache.slot_selected_tensor.device_id = 0; + + g_stream_expert_persistent_epochs_published.fetch_add( + 1, std::memory_order_relaxed); + if (tasks.empty()) { + g_stream_expert_persistent_all_hit_epochs.fetch_add( + 1, std::memory_order_relaxed); + } else { + g_stream_expert_persistent_miss_epochs.fetch_add( + 1, std::memory_order_relaxed); + } + g_stream_expert_persistent_miss_experts.fetch_add( + plan.loads.size(), std::memory_order_relaxed); + g_stream_expert_persistent_weight_bytes.fetch_add( + weight_bytes, std::memory_order_relaxed); + g_stream_expert_persistent_remap_bytes.fetch_add( + remap_bytes, std::memory_order_relaxed); + return 1; +} + +extern "C" int ds4_cuda_test_stream_expert_persistent_runtime(void) { + g_stream_expert_persistent_runtime_oracle_runs.fetch_add( + 1, std::memory_order_relaxed); + if (g_n_gpus != 1) { + g_stream_expert_persistent_runtime_oracle_failures.fetch_add( + 1, std::memory_order_relaxed); + return 0; + } + + cuda_stream_expert_persistent_arena *arena = + &g_stream_expert_persistent_arena; + uint32_t saved_budget = 0; + uint64_t saved_total = 0; + uint64_t saved_gate = 0; + uint64_t saved_down = 0; + { + std::lock_guard lock( + g_stream_expert_persistent_arena_mutex); + saved_budget = arena->configured_budget; + saved_total = arena->configured_expert_bytes; + saved_gate = arena->gate_expert_bytes; + saved_down = arena->down_expert_bytes; + } + const int saved_ssd = g_ssd_streaming_mode; + const int saved_model_fd = g_model_fd; + int ok = cuda_stream_expert_storage_release(1); + g_ssd_streaming_mode = 1; + g_model_fd = -1; + { + std::lock_guard lock( + g_stream_expert_persistent_arena_mutex); + arena->configured_budget = 4u; + arena->configured_expert_bytes = 40u; + arena->gate_expert_bytes = 0; + arena->down_expert_bytes = 0; + } + + unsigned char *model = NULL; + const uint32_t n_total = 6u; + const uint64_t gate_stride = 16u; + const uint64_t down_stride = 8u; + const uint64_t gate_offset = 0u; + const uint64_t up_offset = n_total * gate_stride; + const uint64_t down_offset = up_offset + n_total * gate_stride; + const uint64_t model_size = down_offset + n_total * down_stride; + if (ok) { + ok = cudaMallocHost((void **)&model, (size_t)model_size) == + cudaSuccess; + if (!ok) (void)cudaGetLastError(); + } + if (ok) { + for (uint32_t expert = 0; expert < n_total; expert++) { + memset(model + gate_offset + expert * gate_stride, + 0x10 + expert, (size_t)gate_stride); + memset(model + up_offset + expert * gate_stride, + 0x40 + expert, (size_t)gate_stride); + memset(model + down_offset + expert * down_stride, + 0x70 + expert, (size_t)down_stride); + } + } + ds4_gpu_stream_expert_table table = {}; + table.model_map = model; + table.model_size = model_size; + table.layer = 17u; + table.n_total_expert = n_total; + table.gate_offset = gate_offset; + table.up_offset = up_offset; + table.down_offset = down_offset; + table.gate_expert_bytes = gate_stride; + table.down_expert_bytes = down_stride; + + auto run_epoch = [&](const int32_t *ids, uint32_t count, + uint64_t *upload_event_out, + int *submitted_out) { + cuda_stream_selected_writer_guard writer; + const int rc = cuda_stream_expert_persistent_try_load_writer( + &table, ids, count, upload_event_out, submitted_out, + /*force_for_oracle=*/1); + if (rc > 0) writer.publish_valid(); + return rc; + }; + auto verify_epoch = [&](const int32_t *ids, + const int32_t *expected_remap, + uint32_t count, uint32_t expected_slot_base, + uint32_t expected_domain) { + cuda_stream_selected_binding binding = {}; + if (!cuda_stream_selected_binding_acquire( + &binding, 0, table.model_map, table.layer, + table.n_total_expert, table.gate_offset, + table.up_offset, table.down_offset, + table.gate_expert_bytes, table.down_expert_bytes, + count) || + binding.storage_kind != + CUDA_STREAM_SELECTED_STORAGE_PERSISTENT || + binding.slot_base != expected_slot_base || + binding.weight_domain != expected_domain || + binding.gate != arena->gate + + (uint64_t)expected_slot_base * gate_stride || + binding.up != arena->up + + (uint64_t)expected_slot_base * gate_stride || + binding.down != arena->down + + (uint64_t)expected_slot_base * down_stride) { + return 0; + } + std::vector remap; + try { + remap.resize(count); + } catch (...) { + return 0; + } + if (cudaMemcpy(remap.data(), binding.selected->ptr, + (size_t)count * sizeof(int32_t), + cudaMemcpyDeviceToHost) != cudaSuccess) { + (void)cudaGetLastError(); + return 0; + } + for (uint32_t i = 0; i < count; i++) { + if (remap[i] != expected_remap[i] || remap[i] < 0 || + (uint32_t)remap[i] >= binding.weight_domain || + (uint64_t)binding.slot_base + (uint32_t)remap[i] >= + arena->capacity) { + return 0; + } + unsigned char gate_got[16] = {}; + unsigned char up_got[16] = {}; + unsigned char down_got[8] = {}; + const uint64_t gate_delta = + (uint64_t)(uint32_t)remap[i] * gate_stride; + const uint64_t down_delta = + (uint64_t)(uint32_t)remap[i] * down_stride; + if (cudaMemcpy(gate_got, binding.gate + gate_delta, + sizeof(gate_got), + cudaMemcpyDeviceToHost) != cudaSuccess || + cudaMemcpy(up_got, binding.up + gate_delta, + sizeof(up_got), + cudaMemcpyDeviceToHost) != cudaSuccess || + cudaMemcpy(down_got, binding.down + down_delta, + sizeof(down_got), + cudaMemcpyDeviceToHost) != cudaSuccess) { + (void)cudaGetLastError(); + return 0; + } + for (uint32_t b = 0; b < sizeof(gate_got); b++) { + if (gate_got[b] != (unsigned char)(0x10 + ids[i]) || + up_got[b] != (unsigned char)(0x40 + ids[i])) { + return 0; + } + } + for (uint32_t b = 0; b < sizeof(down_got); b++) { + if (down_got[b] != (unsigned char)(0x70 + ids[i])) { + return 0; + } + } + } + return 1; + }; + + const int32_t cold[] = {0, 1}; + const int32_t mixed[] = {1, 2}; + const int32_t all_hit_shifted[] = {1, 2, 1}; + const int32_t all_hit_sparse[] = {0, 2}; + const int32_t eviction[] = {3, 4}; + const int32_t fault[] = {5}; + const int32_t remap01[] = {0, 1}; + const int32_t remap010[] = {0, 1, 0}; + const int32_t remap02[] = {0, 2}; + const int32_t remap20[] = {2, 0}; + int submitted = 0; + uint64_t upload_event = 0; + if (ok) ok = run_epoch(cold, 2u, &upload_event, &submitted) == 1 && + submitted && + (upload_event != 0 || + cuda_stream_selected_event_pipeline_oracle_requested()); + uint64_t consume_generation = 0; + if (ok) { + ok = cuda_stream_selected_consumer_begin( + /*allow_streaming=*/1, cuda_decode_stream(), + &consume_generation) && + cuda_stream_selected_consumer_end( + consume_generation, cuda_decode_stream()) && + cuda_stream_selected_consume_drain() && + verify_epoch(cold, remap01, 2u, 0u, 2u); + } + submitted = 0; + if (ok) ok = run_epoch(mixed, 2u, NULL, &submitted) == 1 && + submitted && verify_epoch(mixed, remap01, 2u, 1u, 2u); + submitted = 0; + if (ok) ok = run_epoch(all_hit_shifted, 3u, NULL, &submitted) == 1 && + submitted && verify_epoch( + all_hit_shifted, remap010, 3u, 1u, 2u); + submitted = 0; + if (ok) ok = run_epoch(all_hit_sparse, 2u, NULL, &submitted) == 1 && + submitted && verify_epoch( + all_hit_sparse, remap02, 2u, 0u, 3u); + submitted = 0; + if (ok) ok = run_epoch(eviction, 2u, NULL, &submitted) == 1 && + submitted && verify_epoch( + eviction, remap20, 2u, 1u, 3u); + if (ok) g_stream_expert_persistent_test_fail_after_enqueue = 1; + submitted = 0; + upload_event = 0; + if (ok) { + ok = run_epoch(fault, 1u, &upload_event, &submitted) == -1 && + submitted && + (upload_event != 0 || + cuda_stream_selected_event_pipeline_oracle_requested()) && + !g_stream_selected_cache.valid; + } + if (ok) { + std::lock_guard lock( + g_stream_expert_persistent_arena_mutex); + ok = arena->valid_count == 0u && + cuda_stream_expert_persistent_state_count( + &g_stream_expert_persistent_state) == 0u; + } + if (ok) g_stream_selected_stage_test_fail_drain = 1; + submitted = 0; + if (ok) ok = run_epoch(cold, 2u, NULL, &submitted) == -1 && !submitted; + { + std::lock_guard lock( + g_stream_expert_persistent_arena_mutex); + ok = ok && arena->poisoned && + g_stream_selected_stage_poisoned; + } + + const int released = cuda_stream_expert_storage_release(1); + const int stage_released = + cuda_stream_selected_stage_release_checked(); + ok = ok && released && stage_released && + !g_stream_selected_stage_poisoned; + if (released) { + std::lock_guard lock( + g_stream_expert_persistent_arena_mutex); + ok = ok && !arena->base && !arena->remap && + arena->valid_count == 0u && + g_stream_expert_persistent_state.slots.empty(); + arena->configured_budget = saved_budget; + arena->configured_expert_bytes = saved_total; + arena->gate_expert_bytes = saved_gate; + arena->down_expert_bytes = saved_down; + } + g_stream_expert_persistent_test_fail_after_enqueue = 0; + g_stream_selected_stage_test_fail_drain = 0; + g_model_fd = saved_model_fd; + g_ssd_streaming_mode = saved_ssd; + if (model) (void)cudaFreeHost(model); + if (!ok) { + g_stream_expert_persistent_runtime_oracle_failures.fetch_add( + 1, std::memory_order_relaxed); + } + return ok; +} + extern "C" int ds4_cuda_test_stream_expert_persistent_arena(void) { g_stream_expert_persistent_arena_oracle_runs.fetch_add( 1, std::memory_order_relaxed); @@ -39635,8 +40423,7 @@ extern "C" uint32_t ds4_gpu_stream_expert_cache_budget_for_expert_size( !cuda_stream_expert_persistent_requested()) { return 0; } - return cuda_stream_expert_persistent_arena_ensure_locked() ? - arena->configured_budget : 0; + return arena->configured_budget; } extern "C" int ds4_gpu_tensor_copy_f32_to_f16(ds4_gpu_tensor *dst, uint64_t dst_offset, @@ -40432,18 +41219,29 @@ extern "C" void ds4_gpu_set_ssd_streaming(bool enabled) { } extern "C" void ds4_gpu_set_streaming_expert_cache_budget(uint32_t experts) { + { + std::lock_guard lock( + g_stream_expert_persistent_arena_mutex); + if (experts == + g_stream_expert_persistent_arena.configured_budget) return; + } + cuda_stream_selected_writer_guard writer; std::lock_guard lock( g_stream_expert_persistent_arena_mutex); - if (experts == g_stream_expert_persistent_arena.configured_budget) return; if (!cuda_stream_expert_persistent_arena_release_locked(0)) return; g_stream_expert_persistent_arena.configured_budget = experts; } extern "C" void ds4_gpu_set_streaming_expert_cache_expert_bytes(uint64_t bytes) { + { + std::lock_guard lock( + g_stream_expert_persistent_arena_mutex); + if (bytes == + g_stream_expert_persistent_arena.configured_expert_bytes) return; + } + cuda_stream_selected_writer_guard writer; std::lock_guard lock( g_stream_expert_persistent_arena_mutex); - if (bytes == - g_stream_expert_persistent_arena.configured_expert_bytes) return; if (!cuda_stream_expert_persistent_arena_release_locked(1)) return; g_stream_expert_persistent_arena.configured_expert_bytes = bytes; } @@ -40458,7 +41256,9 @@ extern "C" uint32_t ds4_gpu_stream_expert_cache_configured_count(void) { } extern "C" uint32_t ds4_gpu_stream_expert_cache_current_count(void) { - if (!g_stream_expert_persistent_runtime_ready) { + if (!g_stream_expert_persistent_runtime_ready || + !g_ssd_streaming_mode || + !cuda_stream_expert_persistent_requested()) { std::lock_guard lock(g_stream_selected_consume_mutex); return !g_stream_selected_writer_active && g_stream_selected_cache.valid ? @@ -40474,7 +41274,7 @@ extern "C" void ds4_gpu_stream_expert_cache_reset_route_hotness(void) { } extern "C" void ds4_gpu_stream_expert_cache_release_resident(void) { - cuda_stream_expert_storage_release(1); + (void)cuda_stream_expert_storage_release(1); } extern "C" int ds4_gpu_stream_expert_cache_seed_selected( diff --git a/ds4_gpu.h b/ds4_gpu.h index f0dc50fd9..11cf4c421 100644 --- a/ds4_gpu.h +++ b/ds4_gpu.h @@ -361,6 +361,21 @@ typedef struct ds4_cuda_stream_expert_persistent_report { uint64_t arena_failures; uint64_t arena_oracle_runs; uint64_t arena_oracle_failures; + uint64_t epochs_attempted; + uint64_t epochs_published; + uint64_t all_hit_epochs; + uint64_t miss_epochs; + uint64_t miss_experts; + uint64_t weight_bytes_uploaded; + uint64_t remap_bytes_uploaded; + uint64_t upload_failures; + uint64_t fallbacks; + uint64_t slot_invalidations; + uint64_t poisons; + uint64_t persistent_dispatches; + uint64_t transient_dispatches; + uint64_t runtime_oracle_runs; + uint64_t runtime_oracle_failures; int enabled; int required; int stats; @@ -413,6 +428,7 @@ int ds4_cuda_test_stream_expert_persistent_policy( int ds4_cuda_test_stream_expert_persistent_env_value(const char *value); int ds4_cuda_test_stream_expert_persistent_planner(void); int ds4_cuda_test_stream_expert_persistent_arena(void); +int ds4_cuda_test_stream_expert_persistent_runtime(void); void ds4_cuda_stream_expert_persistent_get_report( ds4_cuda_stream_expert_persistent_report *report); int ds4_gpu_cuda_stream_selected_event_pipeline_enabled(void); diff --git a/tests/test_gpu_model_cache.c b/tests/test_gpu_model_cache.c index 288996729..82659b090 100644 --- a/tests/test_gpu_model_cache.c +++ b/tests/test_gpu_model_cache.c @@ -245,6 +245,38 @@ int main(void) { arena_after.arena_oracle_failures == arena_before.arena_oracle_failures, "persistent expert device arena coverage counters"); + ds4_cuda_stream_expert_persistent_report runtime_before; + memset(&runtime_before, 0, sizeof(runtime_before)); + ds4_cuda_stream_expert_persistent_get_report(&runtime_before); + CHECK(ds4_cuda_test_stream_expert_persistent_runtime(), + "persistent expert cold/hit/mixed/eviction/fault runtime oracle"); + ds4_cuda_stream_expert_persistent_report runtime_after; + memset(&runtime_after, 0, sizeof(runtime_after)); + ds4_cuda_stream_expert_persistent_get_report(&runtime_after); + CHECK(runtime_after.runtime_oracle_runs == + runtime_before.runtime_oracle_runs + 1u && + runtime_after.runtime_oracle_failures == + runtime_before.runtime_oracle_failures && + runtime_after.epochs_attempted >= + runtime_before.epochs_attempted + 7u && + runtime_after.epochs_published == + runtime_before.epochs_published + 5u && + runtime_after.all_hit_epochs == + runtime_before.all_hit_epochs + 2u && + runtime_after.miss_epochs == runtime_before.miss_epochs + 3u && + runtime_after.miss_experts >= runtime_before.miss_experts + 5u && + runtime_after.weight_bytes_uploaded >= + runtime_before.weight_bytes_uploaded + 200u && + runtime_after.remap_bytes_uploaded >= + runtime_before.remap_bytes_uploaded + 44u && + runtime_after.upload_failures == + runtime_before.upload_failures + 1u && + runtime_after.slot_invalidations >= + runtime_before.slot_invalidations + 2u && + runtime_after.poisons >= runtime_before.poisons + 1u && + runtime_after.persistent_dispatches >= + runtime_before.persistent_dispatches + 5u, + "persistent expert runtime coverage counters"); ds4_cuda_iq2_ssd_grouped_report lease_before; memset(&lease_before, 0, sizeof(lease_before)); ds4_cuda_iq2_ssd_grouped_get_report(&lease_before); From ad6ea5f2a361a608492baadf3fe92f37429b1b20 Mon Sep 17 00:00:00 2001 From: Giorgio Oppo Date: Fri, 14 Aug 2026 23:23:00 +0200 Subject: [PATCH 038/189] cuda/mmq: persist grouped Q8_1 scratch --- cuda/mmq/ds4_mmq.cu | 461 +++++++++++++++++++++++++++---- cuda/mmq/ds4_mmq.h | 17 ++ cuda/mmq/test/test_mmq_parity.cu | 164 ++++++++++- ds4_cuda.cu | 9 +- 4 files changed, 586 insertions(+), 65 deletions(-) diff --git a/cuda/mmq/ds4_mmq.cu b/cuda/mmq/ds4_mmq.cu index ea5c02e8a..878b2ddc1 100644 --- a/cuda/mmq/ds4_mmq.cu +++ b/cuda/mmq/ds4_mmq.cu @@ -29,7 +29,7 @@ #include #include #include -#include +#include #if defined(__has_include) #if __has_include() @@ -100,19 +100,38 @@ private: // Init // ---------------------------------------------------------------------------- -// Step 7 task #29: experimental persistent Q8_1 scratch buffer. CUDA graph -// memory nodes already preserve the allocation's virtual address for the -// lifetime of the graph, so the ordinary same-stream pool path is correct. -// This startup allocation is instead an optimization experiment: it removes -// captured alloc/free nodes and their allocator/instantiation overhead. When -// DS4_CUDA_MMQ_Q81_PERSISTENT=1 is set, ds4_mmq_moe_vec_impl uses this buffer -// instead of pool_alloc. -// -// Sized for V4 Flash decode shapes: gate Q8_1 ~8 KB, down Q8_1 ~14 KB. -// 256 KB allocation gives generous headroom for short prefill batches. +// Experimental persistent Q8_1 scratch. The grouped raw prefill path below +// aliases its input and down-Q8 staging in this arena: both ranges have the +// same layout but disjoint lifetimes on the default stream. The feature stays +// opt-in because a process-global address is only safe for the single-owner +// GB10 dispatch covered by q81_grouped_persistent_acquire(). static void *g_q81_scratch_ptr = nullptr; static size_t g_q81_scratch_bytes = 0; -static bool g_q81_scratch_enabled = false; +// A failed resize retirement can leave both allocations live. Keep the +// unpublished replacement owned here so cleanup/reinit can retry its free. +static void *g_q81_unpublished_replacement_ptr = nullptr; +// Older vector wrappers still contain a generic persistent branch. Keep that +// branch disabled: unlike grouped fused_raw it has no default-stream lease or +// capture exclusion and therefore cannot safely share the owned arena. +static constexpr bool g_q81_scratch_enabled = false; +static bool g_q81_grouped_enabled = false; +static bool g_q81_scratch_poisoned = false; +static int g_q81_scratch_device = -1; +static std::mutex g_q81_scratch_mutex; + +static uint64_t g_q81_grouped_candidates; +static uint64_t g_q81_grouped_uses; +static uint64_t g_q81_grouped_hits; +static uint64_t g_q81_grouped_pool_fallbacks; +static uint64_t g_q81_grouped_allocations; +static uint64_t g_q81_grouped_resizes; +static uint64_t g_q81_grouped_owner_rejects; +static uint64_t g_q81_grouped_stream_rejects; +static uint64_t g_q81_grouped_capture_rejects; +static uint64_t g_q81_grouped_device_rejects; +static uint64_t g_q81_grouped_size_rejects; +static size_t g_q81_grouped_high_water; + static void *g_aligned_q81_scratch_ptr = nullptr; static size_t g_aligned_q81_scratch_bytes = 0; static int g_aligned_q81_scratch_device = -1; @@ -135,6 +154,162 @@ struct mmq_pair_map_scratch { static mmq_pair_map_scratch g_mmq_pair_maps[GGML_CUDA_MAX_DEVICES] = {}; static int g_gb10_optimizations = 0; +static constexpr size_t DS4_MMQ_Q81_ARENA_MIN_BYTES = 4u * 1024u * 1024u; + +enum q81_arena_result { + Q81_ARENA_REJECTED = 0, + Q81_ARENA_HIT, + Q81_ARENA_ALLOCATED, + Q81_ARENA_RESIZED, +}; + +static bool q81_persistent_requested() { + const char *value = getenv("DS4_CUDA_MMQ_Q81_PERSISTENT"); + if (!value || value[0] == '\0' || strcmp(value, "0") == 0 || + strcmp(value, "off") == 0 || strcmp(value, "OFF") == 0 || + strcmp(value, "false") == 0 || strcmp(value, "FALSE") == 0 || + strcmp(value, "no") == 0 || strcmp(value, "NO") == 0) { + return false; + } + return strcmp(value, "1") == 0 || + strcmp(value, "on") == 0 || strcmp(value, "ON") == 0 || + strcmp(value, "true") == 0 || strcmp(value, "TRUE") == 0 || + strcmp(value, "yes") == 0 || strcmp(value, "YES") == 0; +} + +static bool q81_is_gb10_owner(int device) { + if (!g_gb10_optimizations || device < 0 || + device >= ggml_cuda_info().device_count) { + return false; + } + const auto &info = ggml_cuda_info().devices[device]; + return info.integrated && info.cc == GGML_CUDA_CC_DGX_SPARK; +} + +static q81_arena_result q81_arena_ensure_locked(int device, size_t required) { + if (g_q81_scratch_poisoned || required > SIZE_MAX - 255u || + (g_q81_scratch_ptr && g_q81_scratch_device != device)) { + return Q81_ARENA_REJECTED; + } + if (g_q81_scratch_ptr && required <= g_q81_scratch_bytes) { + g_q81_grouped_enabled = true; + return Q81_ARENA_HIT; + } + + const size_t aligned = (required + 255u) & ~(size_t)255u; + const size_t bytes = aligned > DS4_MMQ_Q81_ARENA_MIN_BYTES + ? aligned : DS4_MMQ_Q81_ARENA_MIN_BYTES; + void *replacement = nullptr; + cudaError_t err = cudaMalloc(&replacement, bytes); + if (err != cudaSuccess || !replacement) { + fprintf(stderr, + "ds4_mmq: cudaMalloc(persistent Q8_1 arena %zu B) failed: %s; " + "using stream pool\n", + bytes, cudaGetErrorString(err)); + (void)cudaGetLastError(); + return Q81_ARENA_REJECTED; + } + + if (g_q81_scratch_ptr) { + /* The lease excludes another host submission and the persistent path + * only accepts the legacy default stream. Drain the device before + * retiring the old address: allocation succeeds before any state is + * changed, so OOM leaves the previous arena usable. A drain/free + * failure is different -- pointer liveness is ambiguous and the + * feature stays poisoned until explicit cleanup. */ + err = cudaDeviceSynchronize(); + if (err == cudaSuccess) { + err = cudaFree(g_q81_scratch_ptr); + } + if (err != cudaSuccess) { + fprintf(stderr, + "ds4_mmq: persistent Q8_1 arena resize from %zu B to " + "%zu B failed while retiring the old arena: %s; " + "persistent reuse poisoned\n", + g_q81_scratch_bytes, bytes, cudaGetErrorString(err)); + (void)cudaGetLastError(); + const cudaError_t replacement_free_err = cudaFree(replacement); + if (replacement_free_err != cudaSuccess) { + fprintf(stderr, + "ds4_mmq: freeing unpublished Q8_1 replacement " + "failed: %s\n", + cudaGetErrorString(replacement_free_err)); + (void)cudaGetLastError(); + g_q81_unpublished_replacement_ptr = replacement; + } + g_q81_grouped_enabled = false; + g_q81_scratch_poisoned = true; + return Q81_ARENA_REJECTED; + } + } + + const bool resized = g_q81_scratch_ptr != nullptr; + g_q81_scratch_ptr = replacement; + g_q81_scratch_bytes = bytes; + g_q81_scratch_device = device; + g_q81_grouped_enabled = true; + g_q81_grouped_allocations++; + if (resized) g_q81_grouped_resizes++; + fprintf(stderr, + "ds4_mmq: persistent Q8_1 arena %s (%zu B at %p, device %d)\n", + resized ? "resized" : "enabled", bytes, + g_q81_scratch_ptr, device); + return resized ? Q81_ARENA_RESIZED : Q81_ARENA_ALLOCATED; +} + +// On success, lease remains locked until the complete host dispatch has been +// submitted. A following dispatch therefore cannot interleave its writes; +// default-stream ordering protects the device-side lifetime after unlock. +static char *q81_grouped_persistent_acquire( + int device, cudaStream_t stream, size_t required, + std::unique_lock *lease) { + if (!lease || !q81_persistent_requested()) return nullptr; + lease->lock(); + g_q81_grouped_candidates++; + if (required > g_q81_grouped_high_water) { + g_q81_grouped_high_water = required; + } + if (!q81_is_gb10_owner(device)) { + g_q81_grouped_device_rejects++; + } else if (stream != (cudaStream_t)0) { + g_q81_grouped_stream_rejects++; + } else { + int active_device = -1; + const cudaError_t device_err = cudaGetDevice(&active_device); + if (device_err != cudaSuccess || active_device != device || + (g_q81_scratch_device >= 0 && + g_q81_scratch_device != device)) { + (void)cudaGetLastError(); + g_q81_grouped_owner_rejects++; + } else { + cudaStreamCaptureStatus capture = cudaStreamCaptureStatusNone; + const cudaError_t capture_err = + cudaStreamIsCapturing(stream, &capture); + if (capture_err != cudaSuccess || + capture != cudaStreamCaptureStatusNone) { + (void)cudaGetLastError(); + g_q81_grouped_capture_rejects++; + } else { + const q81_arena_result arena = + q81_arena_ensure_locked(device, required); + if (arena != Q81_ARENA_REJECTED && + g_q81_grouped_enabled && !g_q81_scratch_poisoned && + required <= g_q81_scratch_bytes) { + g_q81_grouped_uses++; + if (arena == Q81_ARENA_HIT) g_q81_grouped_hits++; + return (char *)g_q81_scratch_ptr; + } + if (g_q81_scratch_ptr && required > g_q81_scratch_bytes) { + g_q81_grouped_size_rejects++; + } + } + } + } + g_q81_grouped_pool_fallbacks++; + lease->unlock(); + return nullptr; +} + extern "C" void ds4_mmq_set_aligned_q81_scratch(void *ptr, size_t bytes) { g_aligned_q81_scratch_ptr = ptr; g_aligned_q81_scratch_bytes = ptr ? bytes : 0; @@ -149,17 +324,43 @@ static void *ds4_mmq_aligned_q81_scratch(int device, size_t bytes) { ? g_aligned_q81_scratch_ptr : nullptr; } -// Read by ds4_mmq_moe_vec_impl; non-zero means use the persistent buffer. -// Set by ds4_mmq_init once based on env. (Single-threaded GPU work; no -// atomicity needed.) +// Legacy generic-scratch ABI. It must stay false: the grouped arena has a +// stricter lease contract and callers should inspect its counters/report API. extern "C" int ds4_mmq_q81_persistent_enabled(void) { - return g_q81_scratch_enabled ? 1 : 0; + return 0; } extern "C" void *ds4_mmq_q81_scratch_ptr(void) { return g_q81_scratch_ptr; } +/* Test-only preflight hook. It deliberately traverses the production + * acquire path (including owner/default-stream/capture checks and resize + * retirement) without enqueueing a synthetic large MMQ fixture. */ +extern "C" int ds4_mmq_q81_persistent_preflight_for_test( + int device, size_t required) { + int previous = -1; + if (cudaGetDevice(&previous) != cudaSuccess || + cudaSetDevice(device) != cudaSuccess) { + (void)cudaGetLastError(); + return -1; + } + char *arena = nullptr; + { + std::unique_lock lease( + g_q81_scratch_mutex, std::defer_lock); + arena = q81_grouped_persistent_acquire( + device, (cudaStream_t)0, required, &lease); + } + const cudaError_t restore_err = previous != device + ? cudaSetDevice(previous) : cudaSuccess; + if (restore_err != cudaSuccess) { + (void)cudaGetLastError(); + return -2; + } + return arena ? 0 : -3; +} + // M2-Inc2a: registry of producer-emitted q8_1 activations (ds4_cuda.cu). // A hit returns canonical block_q8_1 codes for this exact activation // pointer (bit-exact vs quantize_row_q8_1_cuda), letting the caller skip @@ -467,30 +668,128 @@ extern "C" int ds4_mmq_init(int device) { maps.expert_bounds = base + 2u * MMQ_GFX1151_PAIR_MAP_ROWS; } - // Step 7 task #29: pre-allocate persistent Q8_1 scratch if enabled. - // This happens before layer-graph capture to keep allocator nodes and - // their instantiation overhead out of the graph. The ordinary same-stream - // cudaMallocAsync path remains correct: CUDA graph memory nodes preserve - // their virtual address for the lifetime of the graph. - if (getenv("DS4_CUDA_MMQ_Q81_PERSISTENT") && !g_q81_scratch_ptr) { - const size_t bytes = 256 * 1024; - cudaError_t err = cudaMalloc(&g_q81_scratch_ptr, bytes); - if (err != cudaSuccess) { - fprintf(stderr, "ds4_mmq_init: cudaMalloc(q81_scratch %zu B) failed: %s; " - "falling back to pool_alloc\n", - bytes, cudaGetErrorString(err)); - g_q81_scratch_ptr = nullptr; - g_q81_scratch_enabled = false; - } else { - g_q81_scratch_bytes = bytes; - g_q81_scratch_enabled = true; - fprintf(stderr, "ds4_mmq_init: persistent Q8_1 scratch enabled (%zu B at %p)\n", - bytes, g_q81_scratch_ptr); + // Allocation is intentionally lazy. The first eligible grouped dispatch + // knows its exact maximum input/down staging requirement and resolves the + // arena during preflight, before it submits any device operation. + { + std::lock_guard lock(g_q81_scratch_mutex); + const bool requested = q81_persistent_requested(); + g_q81_grouped_enabled = requested && q81_is_gb10_owner(device) && + g_q81_scratch_ptr && !g_q81_scratch_poisoned && + g_q81_scratch_device == device; + } + return 0; +} + +extern "C" int ds4_mmq_q81_persistent_cleanup(void) { + std::lock_guard lock(g_q81_scratch_mutex); + g_q81_grouped_enabled = false; + if (!g_q81_scratch_ptr && !g_q81_unpublished_replacement_ptr) { + g_q81_scratch_bytes = 0; + g_q81_scratch_device = -1; + g_q81_scratch_poisoned = false; + return 0; + } + + int previous = -1; + if (cudaGetDevice(&previous) != cudaSuccess || + g_q81_scratch_device < 0 || + cudaSetDevice(g_q81_scratch_device) != cudaSuccess) { + (void)cudaGetLastError(); + g_q81_scratch_poisoned = true; + return -1; + } + cudaError_t sync_err = cudaDeviceSynchronize(); + cudaError_t arena_free_err = cudaSuccess; + cudaError_t replacement_free_err = cudaSuccess; + if (sync_err == cudaSuccess) { + if (g_q81_scratch_ptr) { + arena_free_err = cudaFree(g_q81_scratch_ptr); + if (arena_free_err == cudaSuccess) { + g_q81_scratch_ptr = nullptr; + g_q81_scratch_bytes = 0; + } else { + (void)cudaGetLastError(); + } + } + if (g_q81_unpublished_replacement_ptr) { + replacement_free_err = cudaFree( + g_q81_unpublished_replacement_ptr); + if (replacement_free_err == cudaSuccess) { + g_q81_unpublished_replacement_ptr = nullptr; + } else { + (void)cudaGetLastError(); + } } + } else { + (void)cudaGetLastError(); + } + const cudaError_t restore_err = previous != g_q81_scratch_device + ? cudaSetDevice(previous) : cudaSuccess; + if (sync_err != cudaSuccess || arena_free_err != cudaSuccess || + replacement_free_err != cudaSuccess) { + fprintf(stderr, + "ds4_mmq: persistent Q8_1 arena cleanup failed: " + "sync=%s arena_free=%s replacement_free=%s\n", + cudaGetErrorString(sync_err), + cudaGetErrorString(arena_free_err), + cudaGetErrorString(replacement_free_err)); + g_q81_scratch_poisoned = true; + return -1; + } + // Both owned allocations are now retired; only now may reinit clear the + // poison and admit a new lazy allocation. + g_q81_scratch_device = -1; + g_q81_scratch_poisoned = false; + if (restore_err != cudaSuccess) { + fprintf(stderr, + "ds4_mmq: persistent Q8_1 arena freed, but restoring CUDA " + "device %d failed: %s\n", + previous, cudaGetErrorString(restore_err)); + (void)cudaGetLastError(); + return -2; } return 0; } +extern "C" void ds4_mmq_q81_persistent_counters( + uint64_t *candidates, uint64_t *uses, uint64_t *hits, + uint64_t *pool_fallbacks, uint64_t *allocations, uint64_t *resizes, + size_t *arena_bytes, size_t *high_water) { + std::lock_guard lock(g_q81_scratch_mutex); + if (candidates) *candidates = g_q81_grouped_candidates; + if (uses) *uses = g_q81_grouped_uses; + if (hits) *hits = g_q81_grouped_hits; + if (pool_fallbacks) *pool_fallbacks = g_q81_grouped_pool_fallbacks; + if (allocations) *allocations = g_q81_grouped_allocations; + if (resizes) *resizes = g_q81_grouped_resizes; + if (arena_bytes) *arena_bytes = g_q81_scratch_bytes; + if (high_water) *high_water = g_q81_grouped_high_water; +} + +extern "C" void ds4_mmq_q81_persistent_report(void) { + std::lock_guard lock(g_q81_scratch_mutex); + fprintf(stderr, + "ds4: CUDA MMQ grouped Q8_1 persistent: candidates=%llu " + "uses=%llu hits=%llu pool_fallbacks=%llu allocations=%llu " + "resizes=%llu " + "arena=%zu high_water=%zu rejects(device/owner/stream/capture/size)=" + "%llu/%llu/%llu/%llu/%llu poisoned=%d\n", + (unsigned long long)g_q81_grouped_candidates, + (unsigned long long)g_q81_grouped_uses, + (unsigned long long)g_q81_grouped_hits, + (unsigned long long)g_q81_grouped_pool_fallbacks, + (unsigned long long)g_q81_grouped_allocations, + (unsigned long long)g_q81_grouped_resizes, + g_q81_scratch_bytes, g_q81_grouped_high_water, + (unsigned long long)g_q81_grouped_device_rejects, + (unsigned long long)g_q81_grouped_owner_rejects, + (unsigned long long)g_q81_grouped_stream_rejects, + (unsigned long long)g_q81_grouped_capture_rejects, + (unsigned long long)g_q81_grouped_size_rejects, + g_q81_scratch_poisoned ? 1 : 0); +} + // ---------------------------------------------------------------------------- // Gating: when should the caller choose mmq over dequant+cublas? // @@ -1455,6 +1754,16 @@ int ds4_mmq_moe_pair_impl( const int64_t s01 = (int64_t)K / blck; const int64_t s02 = (int64_t)M * s01; + // Past the shared-memory cap the launcher takes the bit-identical global + // variant unless the authoritative large-map kill switch rejects it. + if ((size_t)n_tokens * 4u > ggml_cuda_info().devices[dev].smpbo && + !ds4_mmid_large_enabled()) { + fprintf(stderr, + "%s: n_tokens=%d exceeds mm_ids_helper shared-mem cap; " + "falling back\n", tag, n_tokens); + return -1; + } + ggml_cuda_pool_alloc ids_src1_alloc; ggml_cuda_pool_alloc ids_dst_alloc; ggml_cuda_pool_alloc expert_bounds_alloc; @@ -1527,12 +1836,52 @@ int ds4_mmq_moe_pair_impl( ids_src1 = (int32_t *)ids_src1_raw; ids_dst = (int32_t *)ids_dst_raw; expert_bounds = (int32_t *)expert_bounds_raw; - } else if (persistent_pair_maps) { + } + + /* `fused_raw` is the only grouped path that owns neither aligned weights + * nor caller scratch. Its input Q8 is dead after gate/up, before down Q8 + * is produced, so one max-sized range can back both phases. Resolve the + * opt-in arena (or allocate its one-block pool fallback) before the first + * expert-map enqueue; no mid-pipeline allocation failure can strand a + * partially submitted candidate. */ + const bool grouped_raw_q81 = profile_fused_prefill && + type == GGML_TYPE_IQ2_XXS && fused_down != nullptr && + !direct_gateup_q8 && xa_soa == nullptr && xb_soa == nullptr && + fused_down->W_soa == nullptr; + size_t grouped_down_q8_bytes = 0; + size_t grouped_q81_required = 0; + std::unique_lock grouped_q81_lease( + g_q81_scratch_mutex, std::defer_lock); + ggml_cuda_pool_alloc grouped_q81_pool; + char *grouped_q81_scratch = nullptr; + if (grouped_raw_q81 && q81_persistent_requested()) { + const int64_t down_padded = GGML_PAD((int64_t)M, MATRIX_ROW_PADDING); + const size_t tail = + (size_t)get_mmq_x_max_host(cc) * sizeof(block_q8_1_mmq); + if ((size_t)ne_get_rows > SIZE_MAX / (size_t)down_padded || + (size_t)ne_get_rows * (size_t)down_padded > + (SIZE_MAX - tail) / sizeof(block_q8_1)) { + return -98; + } + grouped_down_q8_bytes = + (size_t)ne_get_rows * (size_t)down_padded * + sizeof(block_q8_1) / QK8_1 + tail; + grouped_q81_required = nbytes_src1_q8_1 > grouped_down_q8_bytes + ? nbytes_src1_q8_1 : grouped_down_q8_bytes; + grouped_q81_scratch = q81_grouped_persistent_acquire( + dev, stream, grouped_q81_required, &grouped_q81_lease); + if (!grouped_q81_scratch) { + grouped_q81_scratch = grouped_q81_pool.alloc( + ctx->pool(), grouped_q81_required); + } + } + + if (persistent_pair_maps) { const auto & maps = g_mmq_pair_maps[dev]; ids_src1 = maps.ids_src1; ids_dst = maps.ids_dst; expert_bounds = maps.expert_bounds; - } else { + } else if (!direct_gateup_q8) { ids_src1 = ids_src1_alloc.alloc(ctx->pool(), ne_get_rows); ids_dst = ids_dst_alloc.alloc(ctx->pool(), ne_get_rows); expert_bounds = expert_bounds_alloc.alloc(ctx->pool(), n_experts + 1); @@ -1541,15 +1890,6 @@ int ds4_mmq_moe_pair_impl( const int si1 = n_expert_used; const int sis1 = 1; - // Same cap guard as ds4_mmq_moe_impl (see comment there): past the smem - // cap the launcher takes the bit-identical global variant (P5); only - // refuse with DS4_MMID_LARGE=0. - if ((size_t)n_tokens * 4u > ggml_cuda_info().devices[dev].smpbo && !ds4_mmid_large_enabled()) { - fprintf(stderr, "%s: n_tokens=%d exceeds mm_ids_helper shared-mem cap; falling back\n", - tag, n_tokens); - return -1; - } - cudaError_t err = cudaSuccess; { ds4_mmq_nvtx_scope stage( @@ -1597,7 +1937,9 @@ int ds4_mmq_moe_pair_impl( ggml_cuda_pool_alloc src1_q8_1_alloc; char *src1_q8_1 = direct_gateup_q8 ? (char *)fused_down->input_q8_scratch - : src1_q8_1_alloc.alloc(ctx->pool(), nbytes_src1_q8_1); + : (grouped_q81_scratch + ? grouped_q81_scratch + : src1_q8_1_alloc.alloc(ctx->pool(), nbytes_src1_q8_1)); // S1.1a fix (same as the dense/moe paths): zero the over-allocated mmq Y buffer // so the kernel's unconditional masked-out tail-tile read (mmq.cuh:3528) returns @@ -1879,8 +2221,11 @@ int ds4_mmq_moe_pair_impl( (size_t)ne_get_rows * (size_t)down_ne10_padded * sizeof(block_q8_1) / QK8_1; const size_t tail_q8_bytes = (size_t)get_mmq_x_max_host(cc) * sizeof(block_q8_1_mmq); - ggml_cuda_pool_alloc down_q8_1( - ctx->pool(), logical_q8_bytes + tail_q8_bytes); + const size_t down_q8_bytes = logical_q8_bytes + tail_q8_bytes; + ggml_cuda_pool_alloc down_q8_1_pool; + char *down_q8_1 = grouped_q81_scratch + ? grouped_q81_scratch + : down_q8_1_pool.alloc(ctx->pool(), down_q8_bytes); const uint64_t mid_values = (uint64_t)ne_get_rows * (uint64_t)M; { @@ -1888,7 +2233,7 @@ int ds4_mmq_moe_pair_impl( "ds4/prefill/moe/swiglu_down_quant", ds4_mmq_nvtx_payload((uint32_t)ne_get_rows, (uint32_t)M), nvtx_prefill); - ybuf_memset(down_q8_1.get(), logical_q8_bytes + tail_q8_bytes, stream); + ybuf_memset(down_q8_1, down_q8_bytes, stream); ds4_swiglu_weighted_f32<<< (uint32_t)((mid_values + 255u) / 256u), 256, 0, stream>>>( out_a, out_b, fused_down->router_weights, @@ -1901,7 +2246,7 @@ int ds4_mmq_moe_pair_impl( } quantize_mmq_q8_1_cuda( - fused_down->mid_f32, ids_dst, (void *)down_q8_1.get(), + fused_down->mid_f32, ids_dst, (void *)down_q8_1, GGML_TYPE_Q2_K, /*ne00=*/M, /*s01=*/M, /*s02=*/(int64_t)M, /*s03=*/(int64_t)M * ne_get_rows, /*ne0=*/down_ne10_padded, /*ne1=*/ne_get_rows, @@ -1926,7 +2271,7 @@ int ds4_mmq_moe_pair_impl( const mmq_args down_args = { /*x=*/(const char *)fused_down->W, /*type_x=*/GGML_TYPE_Q2_K, - /*y=*/(const int *)down_q8_1.get(), + /*y=*/(const int *)down_q8_1, /*ids_dst=*/ids_dst, /*expert_bounds=*/expert_bounds, /*dst=*/fused_down->out, @@ -1967,7 +2312,7 @@ int ds4_mmq_moe_pair_impl( down_done = ds4_mmq_q2_K_moe_d2r_launch( fused_down->W_soa, fused_down->soa_blocks, - down_q8_1.get(), + down_q8_1, ids_dst, expert_bounds, fused_down->out, @@ -4883,7 +5228,15 @@ extern "C" int ds4_mmq_iq2_xxs_aligned_derepack( } extern "C" void ds4_mmq_set_gb10_optimizations(int enabled) { - g_gb10_optimizations = enabled != 0; + { + std::lock_guard lock(g_q81_scratch_mutex); + g_gb10_optimizations = enabled != 0; + } + if (!enabled) { + // Backend teardown/reinit already funnels through this setter. Keep + // MMQ's owned arena lifecycle local to this translation unit. + (void)ds4_mmq_q81_persistent_cleanup(); + } } // --------------------------------------------------------------------------- diff --git a/cuda/mmq/ds4_mmq.h b/cuda/mmq/ds4_mmq.h index 82d021821..6756b57bb 100644 --- a/cuda/mmq/ds4_mmq.h +++ b/cuda/mmq/ds4_mmq.h @@ -34,6 +34,23 @@ extern "C" { int ds4_mmq_init(int device); void ds4_mmq_set_aligned_q81_scratch(void *ptr, size_t bytes); +// Opt-in grouped-MMQ Q8_1 arena controlled by +// DS4_CUDA_MMQ_Q81_PERSISTENT. Unset and =0 keep the stream-pool path. +// cleanup drains the owner device before freeing and is safe across reinit; +// report/counters expose host-dispatch coverage (graph replays excluded). +int ds4_mmq_q81_persistent_cleanup(void); +int ds4_mmq_q81_persistent_preflight_for_test(int device, size_t required); +void ds4_mmq_q81_persistent_report(void); +void ds4_mmq_q81_persistent_counters( + uint64_t *candidates, + uint64_t *uses, + uint64_t *hits, + uint64_t *pool_fallbacks, + uint64_t *allocations, + uint64_t *resizes, + size_t *arena_bytes, + size_t *high_water); + // Query whether ds4_mmq is willing to handle a given matmul. Returns // 1 if mmq is faster than dequant+cublas for this shape on this device, // 0 otherwise (caller should fall back to its existing dequant+cublas path). diff --git a/cuda/mmq/test/test_mmq_parity.cu b/cuda/mmq/test/test_mmq_parity.cu index 850a6aa13..120ed016f 100644 --- a/cuda/mmq/test/test_mmq_parity.cu +++ b/cuda/mmq/test/test_mmq_parity.cu @@ -831,7 +831,8 @@ __global__ void test_swiglu_weighted_f32( // Every token has six distinct experts. Flattening ids to [assignments, 1] // for the reference down leg produces the same stable expert ordering as the // single routing map reused by the fused candidate. -bool run_iq2_xxs_q2_K_fused_raw_parity(int n_tokens, uint32_t seed) { +bool run_iq2_xxs_q2_K_fused_raw_parity( + int n_tokens, uint32_t seed, bool persistent_q81 = false) { constexpr int global_experts = 13; constexpr int compact_experts = 8; constexpr int n_expert_used = 6; @@ -843,12 +844,49 @@ bool run_iq2_xxs_q2_K_fused_raw_parity(int n_tokens, uint32_t seed) { constexpr int out_dim = 768; constexpr float clamp = 6.0f; const int compact_to_global[compact_experts] = {11, 2, 9, 0, 7, 12, 4, 6}; + int persistent_device = 0; + + if (persistent_q81) { + cudaDeviceProp prop = {}; + if (cudaGetDevice(&persistent_device) != cudaSuccess || + cudaGetDeviceProperties(&prop, persistent_device) != cudaSuccess || + !prop.integrated || prop.major != 12 || prop.minor != 1) { + (void)cudaGetLastError(); + fprintf(stderr, + "=== IQ2_XXS+Q2_K/FUSED_RAW persistent Q8_1: " + "SKIP (requires integrated sm_121) ===\n\n"); + return true; + } + } fprintf(stderr, - "=== IQ2_XXS+Q2_K/FUSED_RAW compact-remap ntok=%d " + "=== IQ2_XXS+Q2_K/FUSED_RAW%s compact-remap ntok=%d " "nexp=%d nused=%d seed=%u ===\n", + persistent_q81 ? "/PERSISTENT_Q81" : "", n_tokens, compact_experts, n_expert_used, seed); + int initial_arena_cleanup = 0; + int q81_lazy_init_rc = 0; + uint64_t q81_init_allocations0 = 0, q81_init_resizes0 = 0; + uint64_t q81_init_allocations1 = 0, q81_init_resizes1 = 0; + size_t q81_init_arena0 = 0, q81_init_arena1 = 0; + if (persistent_q81) { + initial_arena_cleanup = ds4_mmq_q81_persistent_cleanup(); + ds4_mmq_set_gb10_optimizations(1); + ds4_mmq_q81_persistent_counters( + nullptr, nullptr, nullptr, nullptr, + &q81_init_allocations0, &q81_init_resizes0, + &q81_init_arena0, nullptr); + setenv("DS4_CUDA_MMQ_Q81_PERSISTENT", "1", 1); + q81_lazy_init_rc = ds4_mmq_init(persistent_device); + ds4_mmq_q81_persistent_counters( + nullptr, nullptr, nullptr, nullptr, + &q81_init_allocations1, &q81_init_resizes1, + &q81_init_arena1, nullptr); + // A real `=0` dispatch below is the value-aware opt-out oracle. + setenv("DS4_CUDA_MMQ_Q81_PERSISTENT", "0", 1); + } + std::mt19937 rng(seed); std::normal_distribution activation(0.0f, 0.05f); const size_t iq2_blocks_per_expert = @@ -938,7 +976,8 @@ bool run_iq2_xxs_q2_K_fused_raw_parity(int n_tokens, uint32_t seed) { float *d_gate_global = nullptr, *d_up_global = nullptr, *d_mid_global = nullptr, *d_down_global = nullptr; - bool allocated = cudaStreamCreate(&stream) == cudaSuccess && + bool allocated = (persistent_q81 || + cudaStreamCreate(&stream) == cudaSuccess) && cudaMalloc(&d_gate_w, gate_compact.size() * sizeof(block_iq2_xxs)) == cudaSuccess && cudaMalloc(&d_up_w, up_compact.size() * sizeof(block_iq2_xxs)) == cudaSuccess && cudaMalloc(&d_down_w, down_compact.size() * sizeof(block_q2_K)) == cudaSuccess && @@ -986,6 +1025,10 @@ bool run_iq2_xxs_q2_K_fused_raw_parity(int n_tokens, uint32_t seed) { if (d_up_w) cudaFree(d_up_w); if (d_gate_w) cudaFree(d_gate_w); if (stream) cudaStreamDestroy(stream); + if (persistent_q81) { + unsetenv("DS4_CUDA_MMQ_Q81_PERSISTENT"); + ds4_mmq_set_gb10_optimizations(0); + } }; if (!allocated) { fprintf(stderr, "fused raw parity allocation failed\nFAIL\n\n"); @@ -1082,11 +1125,31 @@ bool run_iq2_xxs_q2_K_fused_raw_parity(int n_tokens, uint32_t seed) { d_down_w, d_mid_ref, d_ids, d_down_ref, out_dim, expert_mid_dim, (int)assignments, compact_experts, /*n_expert_used=*/1, stream); + uint64_t q81_candidates0 = 0, q81_uses0 = 0, q81_hits0 = 0; + uint64_t q81_fallbacks0 = 0, q81_allocations0 = 0, q81_resizes0 = 0; + size_t q81_arena0 = 0, q81_high_water0 = 0; + uint64_t q81_candidates_off = 0, q81_uses_off = 0, q81_hits_off = 0; + uint64_t q81_fallbacks_off = 0, q81_allocations_off = 0; + uint64_t q81_resizes_off = 0; + size_t q81_arena_off = 0, q81_high_water_off = 0; + if (persistent_q81) { + ds4_mmq_q81_persistent_counters( + &q81_candidates0, &q81_uses0, &q81_hits0, + &q81_fallbacks0, &q81_allocations0, &q81_resizes0, + &q81_arena0, &q81_high_water0); + } const int rc_fused = ds4_mmq_iq2_xxs_q2_K_moe_fused_raw( d_gate_w, d_up_w, d_down_w, d_x, d_ids, d_router, d_gate_got, d_up_got, d_mid_got, d_down_got, expert_mid_dim, expert_in_dim, out_dim, n_tokens, compact_experts, n_expert_used, clamp, stream); + if (persistent_q81) { + ds4_mmq_q81_persistent_counters( + &q81_candidates_off, &q81_uses_off, &q81_hits_off, + &q81_fallbacks_off, &q81_allocations_off, &q81_resizes_off, + &q81_arena_off, &q81_high_water_off); + setenv("DS4_CUDA_MMQ_Q81_PERSISTENT", "1", 1); + } // Run the same fused path against the original global expert table and // unremapped ids. Bitwise equality with the compact result validates the // full-expert copies and, with the multi-block shape above, both raw @@ -1097,6 +1160,25 @@ bool run_iq2_xxs_q2_K_fused_raw_parity(int n_tokens, uint32_t seed) { d_gate_global, d_up_global, d_mid_global, d_down_global, expert_mid_dim, expert_in_dim, out_dim, n_tokens, global_experts, n_expert_used, clamp, stream); + // Traverse the real acquire/grow path with a deterministic requirement + // just above the lazy minimum. This drains the first dispatch before + // retiring its arena without making the parity fixture itself enormous. + constexpr size_t q81_growth_required = + 4u * 1024u * 1024u + 257u; + const int rc_q81_grow = persistent_q81 + ? ds4_mmq_q81_persistent_preflight_for_test( + persistent_device, q81_growth_required) + : 0; + // The second identical dispatch must reuse the same owned arena and raises + // the hit counter; default-stream order makes overwriting its Q8 input safe. + const int rc_global_reuse = persistent_q81 + ? ds4_mmq_iq2_xxs_q2_K_moe_fused_raw( + d_gate_global_w, d_up_global_w, d_down_global_w, + d_x, d_global_ids, d_router, + d_gate_global, d_up_global, d_mid_global, d_down_global, + expert_mid_dim, expert_in_dim, out_dim, + n_tokens, global_experts, n_expert_used, clamp, stream) + : 0; std::vector gate_ref(mid_count), up_ref(mid_count), mid_ref(mid_count), down_ref(down_count); @@ -1130,6 +1212,71 @@ bool run_iq2_xxs_q2_K_fused_raw_parity(int n_tokens, uint32_t seed) { down_count * sizeof(float), cudaMemcpyDeviceToHost, stream); sync_err = cudaStreamSynchronize(stream); + uint64_t q81_candidates1 = 0, q81_uses1 = 0, q81_hits1 = 0; + uint64_t q81_fallbacks1 = 0, q81_allocations1 = 0, q81_resizes1 = 0; + size_t q81_arena1 = 0, q81_high_water1 = 0; + if (persistent_q81) { + ds4_mmq_q81_persistent_counters( + &q81_candidates1, &q81_uses1, &q81_hits1, + &q81_fallbacks1, &q81_allocations1, &q81_resizes1, + &q81_arena1, &q81_high_water1); + ds4_mmq_q81_persistent_report(); + } + const size_t q81_growth_aligned = + (q81_growth_required + 255u) & ~(size_t)255u; + const bool q81_counters_ok = !persistent_q81 || + (q81_lazy_init_rc == 0 && + q81_init_allocations1 == q81_init_allocations0 && + q81_init_resizes1 == q81_init_resizes0 && + q81_init_arena0 == 0 && q81_init_arena1 == 0 && + q81_candidates_off == q81_candidates0 && + q81_uses_off == q81_uses0 && q81_hits_off == q81_hits0 && + q81_fallbacks_off == q81_fallbacks0 && + q81_allocations_off == q81_allocations0 && + q81_resizes_off == q81_resizes0 && + q81_arena_off == 0 && + q81_candidates1 - q81_candidates_off == 3u && + q81_uses1 - q81_uses_off == 3u && + q81_hits1 - q81_hits_off == 1u && + q81_fallbacks1 == q81_fallbacks_off && + q81_allocations1 - q81_allocations_off == 2u && + q81_resizes1 - q81_resizes_off == 1u && + q81_arena1 == q81_growth_aligned && + q81_high_water1 >= q81_growth_required && + q81_high_water1 >= q81_high_water0); + if (persistent_q81) { + fprintf(stderr, + "q81 persistent lazy_init=%d arena=%zu->%zu alloc=%llu->%llu " + "resize=%llu->%llu " + "delta(off c/u/h/f/a/r)=%llu/%llu/%llu/%llu/%llu/%llu " + "delta(on c/u/h/f/a/r)=%llu/%llu/%llu/%llu/%llu/%llu " + "arena=%zu high=%zu\n", + q81_lazy_init_rc, q81_init_arena0, q81_init_arena1, + (unsigned long long)q81_init_allocations0, + (unsigned long long)q81_init_allocations1, + (unsigned long long)q81_init_resizes0, + (unsigned long long)q81_init_resizes1, + (unsigned long long)(q81_candidates_off - q81_candidates0), + (unsigned long long)(q81_uses_off - q81_uses0), + (unsigned long long)(q81_hits_off - q81_hits0), + (unsigned long long)(q81_fallbacks_off - q81_fallbacks0), + (unsigned long long)(q81_allocations_off - q81_allocations0), + (unsigned long long)(q81_resizes_off - q81_resizes0), + (unsigned long long)(q81_candidates1 - q81_candidates_off), + (unsigned long long)(q81_uses1 - q81_uses_off), + (unsigned long long)(q81_hits1 - q81_hits_off), + (unsigned long long)(q81_fallbacks1 - q81_fallbacks_off), + (unsigned long long)(q81_allocations1 - q81_allocations_off), + (unsigned long long)(q81_resizes1 - q81_resizes_off), + q81_arena1, q81_high_water1); + } + int final_arena_cleanup = 0; + if (persistent_q81) { + unsetenv("DS4_CUDA_MMQ_Q81_PERSISTENT"); + final_arena_cleanup = ds4_mmq_q81_persistent_cleanup(); + ds4_mmq_set_gb10_optimizations(0); + } + const auto mismatches = [](const std::vector & a, const std::vector & b) { size_t bad = 0; @@ -1148,15 +1295,20 @@ bool run_iq2_xxs_q2_K_fused_raw_parity(int n_tokens, uint32_t seed) { const size_t down_remap_bad = mismatches(down_got, down_global_out); const bool ok = na_ok && rc_pair == 0 && swiglu_err == cudaSuccess && rc_down == 0 && rc_fused == 0 && rc_global == 0 && + rc_global_reuse == 0 && rc_q81_grow == 0 && + initial_arena_cleanup == 0 && + final_arena_cleanup == 0 && q81_counters_ok && sync_err == cudaSuccess && gate_bad == 0 && up_bad == 0 && mid_bad == 0 && down_bad == 0 && gate_remap_bad == 0 && up_remap_bad == 0 && mid_remap_bad == 0 && down_remap_bad == 0; fprintf(stderr, "rc_na=%d canary=%s rc_pair=%d swiglu=%s rc_down=%d " - "rc_fused=%d rc_global=%d mismatches(g/u/m/d)=%zu/%zu/%zu/%zu " + "rc_fused=%d rc_global=%d rc_grow=%d rc_reuse=%d " + "mismatches(g/u/m/d)=%zu/%zu/%zu/%zu " "remap_mismatches(g/u/m/d)=%zu/%zu/%zu/%zu sync=%s\n%s\n\n", - rc_na, na_ok ? "intact" : "FAILED", rc_pair, + rc_na, na_ok ? "intact" : "FAILED", rc_pair, cudaGetErrorString(swiglu_err), rc_down, rc_fused, rc_global, + rc_q81_grow, rc_global_reuse, gate_bad, up_bad, mid_bad, down_bad, gate_remap_bad, up_remap_bad, mid_remap_bad, down_remap_bad, cudaGetErrorString(sync_err), ok ? "PASS" : "FAIL"); @@ -1863,6 +2015,8 @@ int main(int argc, char ** argv) { all_ok &= run_iq2_xxs_q2_K_fused_raw_parity(/*nt=*/8, 0xC2F008); all_ok &= run_iq2_xxs_q2_K_fused_raw_parity(/*nt=*/32, 0xC2F020); all_ok &= run_iq2_xxs_q2_K_fused_raw_parity(/*nt=*/128, 0xC2F080); + all_ok &= run_iq2_xxs_q2_K_fused_raw_parity( + /*nt=*/32, 0xC2F021, /*persistent_q81=*/true); // Step 6 - mmvq vector matmul tests. // diff --git a/ds4_cuda.cu b/ds4_cuda.cu index 8a78f1d1f..0b24131e0 100644 --- a/ds4_cuda.cu +++ b/ds4_cuda.cu @@ -1701,8 +1701,9 @@ static int cuda_use_mmq(void) { /* MXFP4 has no dequant+cublas fallback, so it must retain MMQ on multi-GPU * placements where the optional Q8/IQ2 prefill tier stays disabled. MMQ * resolves the active CUDA device on every call; initialization only warms - * its device-info singleton. The experimental global persistent scratch is - * intentionally rejected because one pointer cannot span CUDA devices. */ + * its device-info singleton. The grouped persistent Q8_1 arena is device-bound + * and is never consumed by the MXFP4 wrappers, so it does not constrain this + * multi-GPU path. */ static int cuda_use_mxfp4_mmq(void) { static int init = 0; static int use = 0; @@ -1712,10 +1713,6 @@ static int cuda_use_mxfp4_mmq(void) { if (s && s[0] == '0') { fprintf(stderr, "ds4: DS4_CUDA_MMQ=0 - MXFP4 MMQ disabled\n"); - } else if (g_n_gpus > 1 && - getenv("DS4_CUDA_MMQ_Q81_PERSISTENT") != NULL) { - fprintf(stderr, - "ds4: persistent MMQ Q8_1 scratch is unavailable with multi-GPU MXFP4\n"); } else { int device = 0; if (cudaGetDevice(&device) == cudaSuccess && From db41283523bee3af6abe82f27ad69407ee18c966 Mon Sep 17 00:00:00 2001 From: Giorgio Oppo Date: Fri, 14 Aug 2026 23:41:10 +0200 Subject: [PATCH 039/189] cuda: add bit-exact Q8 HC split experiment --- ds4_cuda.cu | 923 ++++++++++++++++++++++++++++++----- ds4_gpu.h | 24 + tests/test_gpu_model_cache.c | 55 +++ 3 files changed, 893 insertions(+), 109 deletions(-) diff --git a/ds4_cuda.cu b/ds4_cuda.cu index 0b24131e0..07382069e 100644 --- a/ds4_cuda.cu +++ b/ds4_cuda.cu @@ -39,6 +39,19 @@ #define CUDA_QK_K 256 #define DS4_CUDA_UNUSED __attribute__((unused)) +/* Environment switches used by opt-in CUDA experiments are value-aware: + * spelling a switch as 0/off/no/false must behave like leaving it unset. + * Keep this helper above the MMQ policy code so early dispatch gates and + * later streaming policies share the exact same interpretation. */ +static int cuda_env_value_enabled(const char *value) { + if (!value || !value[0]) return 0; + if (strcmp(value, "0") == 0 || strcasecmp(value, "false") == 0 || + strcasecmp(value, "no") == 0 || strcasecmp(value, "off") == 0) { + return 0; + } + return 1; +} + enum { /* attention_decode_mixed_kernel stores raw-window scores plus visible * compressed scores in shared memory. The host routes larger unmasked @@ -704,6 +717,22 @@ static std::atomic g_stream_expert_persistent_runtime_oracle_runs{0}; static std::atomic g_stream_expert_persistent_runtime_oracle_failures{0}; static int g_stream_expert_persistent_test_fail_after_enqueue; static int g_stream_expert_persistent_runtime_ready; +static std::once_flag g_q8_hc_expand_policy_once; +static int g_q8_hc_expand_force_fused; +static int g_q8_hc_expand_split_requested; +static int g_q8_hc_expand_stats; +static int g_q8_hc_expand_report_registered; +static std::atomic g_q8_hc_expand_candidates{0}; +static std::atomic g_q8_hc_expand_fused_attempts{0}; +static std::atomic g_q8_hc_expand_fused_completed{0}; +static std::atomic g_q8_hc_expand_split_attempts{0}; +static std::atomic g_q8_hc_expand_split_completed{0}; +static std::atomic g_q8_hc_expand_failures{0}; +static std::atomic g_q8_hc_expand_capture_candidates{0}; +static std::atomic g_q8_hc_expand_owned_forced_fused{0}; +static std::atomic g_q8_hc_expand_multi_gpu_forced_fused{0}; +static std::atomic g_q8_hc_expand_oracle_runs{0}; +static std::atomic g_q8_hc_expand_oracle_failures{0}; static int cuda_ok(cudaError_t err, const char *what); static void cuda_stream_selected_event_pipeline_release(void); @@ -1701,9 +1730,9 @@ static int cuda_use_mmq(void) { /* MXFP4 has no dequant+cublas fallback, so it must retain MMQ on multi-GPU * placements where the optional Q8/IQ2 prefill tier stays disabled. MMQ * resolves the active CUDA device on every call; initialization only warms - * its device-info singleton. The grouped persistent Q8_1 arena is device-bound - * and is never consumed by the MXFP4 wrappers, so it does not constrain this - * multi-GPU path. */ + * its device-info singleton. The grouped persistent Q8_1 arena is owned by + * MMQ, device-bound, and never consumed by the MXFP4 wrappers, so it does not + * constrain this required multi-GPU path. */ static int cuda_use_mxfp4_mmq(void) { static int init = 0; static int use = 0; @@ -1957,6 +1986,190 @@ static int cuda_env_flag_enabled(const char *name, int fallback) { return strcmp(env, "0") != 0; } +enum cuda_q8_hc_expand_path { + CUDA_Q8_HC_EXPAND_FUSED = 1, + CUDA_Q8_HC_EXPAND_SPLIT = 2, +}; + +struct ds4_cuda_q8_hc_expand_report; +typedef struct ds4_cuda_q8_hc_expand_report + ds4_cuda_q8_hc_expand_report; +typedef struct { + uint64_t candidates; + uint64_t fused_attempts; + uint64_t fused_completed; + uint64_t split_attempts; + uint64_t split_completed; + uint64_t failures; + uint64_t capture_candidates; + uint64_t owned_forced_fused; + uint64_t multi_gpu_forced_fused; + uint64_t oracle_runs; + uint64_t oracle_failures; + int force_fused; + int split_requested; + int stats; +} cuda_q8_hc_expand_report_layout; + +/* Resolve the Q8 shared-down/HC A/B policy without consulting global state. + * The opt-in split is deliberately narrow: single GPU and non-owned only. + * Force-fused wins conflicts, while owned/multi-GPU dispatches retain their + * existing fused implementation instead of turning an A/B switch into a + * hard inference failure. Capture is observable but does not change the + * selected graph, so eager warm/capture record the same kernel sequence. */ +static int cuda_q8_hc_expand_resolve_policy( + int force_fused, int disable_fused, int n_gpus, int owned, + int capture, int *owned_forced_out, int *multi_gpu_forced_out, + int *capture_out) { + const int split_requested = disable_fused && !force_fused; + const int owned_forced = split_requested && owned; + const int multi_gpu_forced = + split_requested && !owned && n_gpus != 1; + if (owned_forced_out) *owned_forced_out = owned_forced; + if (multi_gpu_forced_out) *multi_gpu_forced_out = multi_gpu_forced; + if (capture_out) *capture_out = capture != 0; + return split_requested && n_gpus == 1 && !owned + ? CUDA_Q8_HC_EXPAND_SPLIT : CUDA_Q8_HC_EXPAND_FUSED; +} + +extern "C" int ds4_cuda_test_q8_hc_expand_policy( + int force_fused, int disable_fused, int n_gpus, int owned, + int capture, int *fused_out, int *owned_forced_out, + int *multi_gpu_forced_out, int *capture_out) { + if (!fused_out || !owned_forced_out || !multi_gpu_forced_out || + !capture_out) { + return 0; + } + const int path = cuda_q8_hc_expand_resolve_policy( + force_fused, disable_fused, n_gpus, owned, capture, + owned_forced_out, multi_gpu_forced_out, capture_out); + *fused_out = path == CUDA_Q8_HC_EXPAND_FUSED; + return 1; +} + +extern "C" int ds4_cuda_test_q8_hc_expand_env_value( + const char *value) { + return cuda_env_value_enabled(value); +} + +static void cuda_q8_hc_expand_report_at_exit(void) { + fprintf(stderr, + "ds4: CUDA Q8 shared-down/HC: candidates=%llu " + "fused=%llu/%llu split=%llu/%llu failures=%llu " + "capture=%llu owned_forced=%llu multi_gpu_forced=%llu " + "oracle=%llu/%llu\n", + (unsigned long long)g_q8_hc_expand_candidates.load(), + (unsigned long long)g_q8_hc_expand_fused_completed.load(), + (unsigned long long)g_q8_hc_expand_fused_attempts.load(), + (unsigned long long)g_q8_hc_expand_split_completed.load(), + (unsigned long long)g_q8_hc_expand_split_attempts.load(), + (unsigned long long)g_q8_hc_expand_failures.load(), + (unsigned long long)g_q8_hc_expand_capture_candidates.load(), + (unsigned long long)g_q8_hc_expand_owned_forced_fused.load(), + (unsigned long long)g_q8_hc_expand_multi_gpu_forced_fused.load(), + (unsigned long long)g_q8_hc_expand_oracle_runs.load(), + (unsigned long long)g_q8_hc_expand_oracle_failures.load()); +} + +static void cuda_q8_hc_expand_policy_init(void) { + std::call_once(g_q8_hc_expand_policy_once, []() { + g_q8_hc_expand_force_fused = cuda_env_value_enabled( + getenv("DS4_CUDA_Q8_HC_EXPAND_FUSED")); + const int disable_fused = cuda_env_value_enabled( + getenv("DS4_CUDA_DISABLE_Q8_HC_EXPAND_FUSED")); + g_q8_hc_expand_split_requested = + disable_fused && !g_q8_hc_expand_force_fused; + g_q8_hc_expand_stats = cuda_env_value_enabled( + getenv("DS4_CUDA_Q8_HC_EXPAND_STATS")); + if ((g_q8_hc_expand_force_fused || disable_fused || + g_q8_hc_expand_stats) && !g_q8_hc_expand_report_registered) { + g_q8_hc_expand_report_registered = 1; + (void)atexit(cuda_q8_hc_expand_report_at_exit); + } + if (g_q8_hc_expand_force_fused) { + fprintf(stderr, + "ds4: CUDA Q8 shared-down/HC fused path forced%s\n", + disable_fused ? " (overrides split request)" : ""); + } else if (g_q8_hc_expand_split_requested) { + fprintf(stderr, + "ds4: CUDA Q8 shared-down/HC split A/B enabled " + "for single-GPU non-owned dispatches\n"); + } + }); +} + +static int cuda_q8_hc_expand_policy_path(int owned) { + cuda_q8_hc_expand_policy_init(); + int owned_forced = 0; + int multi_gpu_forced = 0; + int capture = 0; + const int path = cuda_q8_hc_expand_resolve_policy( + g_q8_hc_expand_force_fused, + g_q8_hc_expand_split_requested, + g_n_gpus, owned, g_decode_graph_capturing, + &owned_forced, &multi_gpu_forced, &capture); + if (!g_q8_hc_expand_report_registered) return path; + g_q8_hc_expand_candidates.fetch_add(1u, std::memory_order_relaxed); + if (capture) { + g_q8_hc_expand_capture_candidates.fetch_add( + 1u, std::memory_order_relaxed); + } + if (owned_forced) { + g_q8_hc_expand_owned_forced_fused.fetch_add( + 1u, std::memory_order_relaxed); + } + if (multi_gpu_forced) { + g_q8_hc_expand_multi_gpu_forced_fused.fetch_add( + 1u, std::memory_order_relaxed); + } + if (path == CUDA_Q8_HC_EXPAND_SPLIT) { + g_q8_hc_expand_split_attempts.fetch_add( + 1u, std::memory_order_relaxed); + } else { + g_q8_hc_expand_fused_attempts.fetch_add( + 1u, std::memory_order_relaxed); + } + return path; +} + +static int cuda_q8_hc_expand_policy_complete(int path, int result) { + if (!g_q8_hc_expand_report_registered) return result; + if (result) { + if (path == CUDA_Q8_HC_EXPAND_SPLIT) { + g_q8_hc_expand_split_completed.fetch_add( + 1u, std::memory_order_relaxed); + } else { + g_q8_hc_expand_fused_completed.fetch_add( + 1u, std::memory_order_relaxed); + } + } else { + g_q8_hc_expand_failures.fetch_add(1u, std::memory_order_relaxed); + } + return result; +} + +extern "C" void ds4_cuda_q8_hc_expand_get_report( + ds4_cuda_q8_hc_expand_report *report) { + if (!report) return; + cuda_q8_hc_expand_policy_init(); + cuda_q8_hc_expand_report_layout out = {}; + out.candidates = g_q8_hc_expand_candidates.load(); + out.fused_attempts = g_q8_hc_expand_fused_attempts.load(); + out.fused_completed = g_q8_hc_expand_fused_completed.load(); + out.split_attempts = g_q8_hc_expand_split_attempts.load(); + out.split_completed = g_q8_hc_expand_split_completed.load(); + out.failures = g_q8_hc_expand_failures.load(); + out.capture_candidates = g_q8_hc_expand_capture_candidates.load(); + out.owned_forced_fused = g_q8_hc_expand_owned_forced_fused.load(); + out.multi_gpu_forced_fused = g_q8_hc_expand_multi_gpu_forced_fused.load(); + out.oracle_runs = g_q8_hc_expand_oracle_runs.load(); + out.oracle_failures = g_q8_hc_expand_oracle_failures.load(); + out.force_fused = g_q8_hc_expand_force_fused; + out.split_requested = g_q8_hc_expand_split_requested; + out.stats = g_q8_hc_expand_stats; + memcpy(report, &out, sizeof(out)); +} + /* Conservative AProjQ4 port of the GB10 Q8 decode dispatch ideas. Q4_K * must keep the canonical MMVQ Q8_1 activation quantizer and reduction DAG; * switching these projections to the local Q8_K fallback changes output @@ -3104,12 +3317,7 @@ typedef struct { } cuda_stream_selected_batch_io_report_layout; static int cuda_stream_selected_env_value_enabled(const char *value) { - if (!value || !value[0]) return 0; - if (strcmp(value, "0") == 0 || strcasecmp(value, "false") == 0 || - strcasecmp(value, "no") == 0 || strcasecmp(value, "off") == 0) { - return 0; - } - return 1; + return cuda_env_value_enabled(value); } static int cuda_stream_selected_env_flag(const char *name) { @@ -10010,6 +10218,37 @@ __global__ static void matmul_q8_0_hc_expand_aligned_preq_warp8_kernel( } } +/* Split A/B matmul leg for the aligned artifact. Unlike the general MMQ + * aligned consumer, this intentionally keeps the fused baseline's canonical + * Q8_0 activation codes and FP32 scale. Lane/block assignment, multiply + * order and warp reduction are identical to the fused aligned kernel above; + * only the HC epilogue moves to its own launch. */ +__global__ static void matmul_q8_0_aligned_preq_warp8_kernel( + float *out, + const int4 *w_qs, + const __half *w_dq, + const int8_t *xq, + const float *xscale, + uint64_t out_dim, + uint64_t blocks) { + const uint64_t row = + (uint64_t)blockIdx.x * 8u + (threadIdx.x >> 5u); + const uint32_t lane = threadIdx.x & 31u; + if (row >= out_dim) return; + const uint64_t rbase = row * blocks; + float acc = 0.0f; + for (uint64_t b = lane; b < blocks; b += 32u) { + const int4 w0 = w_qs[(rbase + b) * 2u]; + const int4 w1 = w_qs[(rbase + b) * 2u + 1u]; + const int32_t dot = + dot_i8x32_aligned_int4(w0, w1, xq + b * 32u); + acc += __half2float(w_dq[rbase + b]) * + xscale[b] * (float)dot; + } + acc = warp_sum_f32(acc); + if (lane == 0u) out[row] = acc; +} + __global__ static void matmul_q8_0_kslice_hc_expand_add_preq_warp8_kernel( float *out_hc, float *block_out, @@ -20517,6 +20756,95 @@ extern "C" int ds4_gpu_matmul_q8_0_pair_decode_rows_exact_tensor( "q8_0 pair decode rows exact warp launch"); } +typedef struct { + const char *raw; + const char *aligned; + int8_t *xq; + float *xscale; + uint64_t blocks; + int use_dp4a; +} cuda_q8_hc_matmul_prepared; + +/* One preparation contract feeds both sides of the HC A/B: identical weight + * resolution, aligned-artifact selection, scratch layout and Q8_0 activation + * quantization. Keeping this centralized prevents the split experiment from + * silently inheriting MMQ's half-scale Q8_1 numerics. */ +static int cuda_q8_hc_matmul_prepare( + ds4_gpu_tensor *block_out, + const void *model_map, + uint64_t model_size, + uint64_t weight_offset, + uint64_t in_dim, + uint64_t out_dim, + const ds4_gpu_tensor *x, + int logical_tier, + const char *label, + cuda_q8_hc_matmul_prepared *prepared) { + if (!block_out || !model_map || !x || !prepared || + in_dim == 0u || out_dim == 0u) { + return 0; + } + const uint64_t blocks = (in_dim + 31u) / 32u; + if (blocks == 0u || blocks > UINT64_MAX / 34u || + out_dim > UINT64_MAX / (blocks * 34u) || + weight_offset > model_size) { + return 0; + } + const uint64_t weight_bytes = out_dim * blocks * 34u; + if (weight_bytes > model_size - weight_offset || + in_dim > UINT64_MAX / sizeof(float) || + out_dim > UINT64_MAX / sizeof(float) || + x->bytes < in_dim * sizeof(float) || + block_out->bytes < out_dim * sizeof(float)) { + return 0; + } + const char *raw = cuda_resolve_weight_ptr( + model_map, weight_offset, weight_bytes, logical_tier, + label ? label : "q8_0_hc_expand"); + if (!raw) return 0; + const uint64_t aligned_bytes = + in_dim <= INT_MAX && out_dim <= INT_MAX && + (in_dim % 1024u) == 0u && (out_dim % 128u) == 0u + ? ds4_mmq_q8_0_aligned_bytes((int)out_dim, (int)in_dim) + : 0u; + const int use_dp4a = cuda_q8_use_dp4a(); + const char *aligned = + g_n_gpus == 1 && + logical_tier >= 0 && logical_tier < DS4_MAX_GPUS && + g_cuda_is_gb10[logical_tier] && + getenv("DS4_CUDA_NO_Q8_FUSED_ALIGNED") == NULL && + aligned_bytes != 0u && cuda_aligned_q8_enabled() && use_dp4a + ? cuda_derived_weight_ptr( + model_map, weight_offset, weight_bytes, + CUDA_DERIVED_Q8_0_ALIGNED_DENSE, + in_dim, out_dim, 1u, aligned_bytes) + : NULL; + + const uint64_t xq_bytes = blocks * 32u; + const uint64_t scale_offset = (xq_bytes + 15u) & ~15ull; + if (blocks > (UINT64_MAX - scale_offset) / sizeof(float)) return 0; + const uint64_t tmp_bytes = scale_offset + blocks * sizeof(float); + void *tmp = cuda_tmp_alloc_on( + logical_tier, tmp_bytes, "q8_0 hc expand prequant"); + if (!tmp) return 0; + int8_t *xq = (int8_t *)tmp; + float *xscale = (float *)((char *)tmp + scale_offset); + quantize_q8_0_f32_kernel<<< + (unsigned)blocks, 32, 0, cuda_decode_stream()>>>( + xq, xscale, (const float *)x->ptr, in_dim, blocks); + if (!cuda_ok(cudaGetLastError(), + "matmul_q8_0_hc_expand quantize launch")) { + return 0; + } + prepared->raw = raw; + prepared->aligned = aligned; + prepared->xq = xq; + prepared->xscale = xscale; + prepared->blocks = blocks; + prepared->use_dp4a = use_dp4a; + return 1; +} + static int cuda_matmul_q8_0_hc_expand_tensor_labeled( ds4_gpu_tensor *out_hc, ds4_gpu_tensor *block_out, @@ -20542,15 +20870,9 @@ static int cuda_matmul_q8_0_hc_expand_tensor_labeled( out_dim != (uint64_t)n_embd) { return 0; } - const uint64_t blocks = (in_dim + 31) / 32; - if (weight_offset > model_size || out_dim > UINT64_MAX / (blocks * 34)) return 0; - const uint64_t weight_bytes = out_dim * blocks * 34; const uint64_t hc_bytes = (uint64_t)n_hc * n_embd * sizeof(float); const uint64_t split_bytes = (uint64_t)(2u * n_hc + n_hc * n_hc) * sizeof(float); - if (weight_bytes > model_size - weight_offset || - x->bytes < in_dim * sizeof(float) || - block_out->bytes < out_dim * sizeof(float) || - residual_hc->bytes < hc_bytes || + if (residual_hc->bytes < hc_bytes || split->bytes < split_bytes || out_hc->bytes < hc_bytes || (block_add && block_add->bytes < out_dim * sizeof(float)) || @@ -20560,40 +20882,22 @@ static int cuda_matmul_q8_0_hc_expand_tensor_labeled( owned_expert_split == 0u || owned_home_slots->bytes < 6u * out_dim * sizeof(float) || owned_peer_packed->bytes < 4u * out_dim * sizeof(float) || - owned_selected->bytes < 6u * sizeof(int32_t)))) { + owned_selected->bytes < 6u * sizeof(int32_t)))) { return 0; } - const int logical_tier = ds4_tensor_device_idx(out_hc); - const char *wptr = cuda_resolve_weight_ptr(model_map, weight_offset, weight_bytes, logical_tier, label ? label : "q8_0_hc_expand"); - if (!wptr) return 0; - const uint64_t aligned_bytes = - in_dim <= INT_MAX && out_dim <= INT_MAX && - (in_dim % 1024u) == 0u && (out_dim % 128u) == 0u - ? ds4_mmq_q8_0_aligned_bytes((int)out_dim, (int)in_dim) - : 0u; - const char *aligned = - g_n_gpus == 1 && - logical_tier >= 0 && logical_tier < DS4_MAX_GPUS && - g_cuda_is_gb10[logical_tier] && - getenv("DS4_CUDA_NO_Q8_FUSED_ALIGNED") == NULL && - aligned_bytes != 0u && cuda_aligned_q8_enabled() && - cuda_q8_use_dp4a() - ? cuda_derived_weight_ptr( - model_map, weight_offset, weight_bytes, - CUDA_DERIVED_Q8_0_ALIGNED_DENSE, - in_dim, out_dim, 1u, aligned_bytes) - : NULL; - - const uint64_t xq_bytes = blocks * 32u; - const uint64_t scale_offset = (xq_bytes + 15u) & ~15ull; - const uint64_t tmp_bytes = scale_offset + blocks * sizeof(float); - void *tmp = cuda_tmp_alloc_on(logical_tier, tmp_bytes, "q8_0 hc expand prequant"); - if (!tmp) return 0; - int8_t *xq = (int8_t *)tmp; - float *xscale = (float *)((char *)tmp + scale_offset); - const int use_dp4a = cuda_q8_use_dp4a(); - quantize_q8_0_f32_kernel<<<(unsigned)blocks, 32, 0, cuda_decode_stream()>>>(xq, xscale, (const float *)x->ptr, in_dim, blocks); - if (!cuda_ok(cudaGetLastError(), "matmul_q8_0_hc_expand quantize launch")) return 0; + cuda_q8_hc_matmul_prepared prepared = {}; + if (!cuda_q8_hc_matmul_prepare( + block_out, model_map, model_size, weight_offset, + in_dim, out_dim, x, ds4_tensor_device_idx(out_hc), + label, &prepared)) { + return 0; + } + const char *wptr = prepared.raw; + const char *aligned = prepared.aligned; + int8_t *xq = prepared.xq; + float *xscale = prepared.xscale; + const uint64_t blocks = prepared.blocks; + const int use_dp4a = prepared.use_dp4a; const dim3 grid(((unsigned)out_dim + 7u) / 8u, 1u, 1u); if (aligned) { const uint64_t nblk = out_dim * blocks; @@ -20637,6 +20941,46 @@ static int cuda_matmul_q8_0_hc_expand_tensor_labeled( return cuda_ok(cudaGetLastError(), "matmul_q8_0_hc_expand launch"); } +static int cuda_matmul_q8_0_hc_split_matmul_tensor_labeled( + ds4_gpu_tensor *block_out, + const void *model_map, + uint64_t model_size, + uint64_t weight_offset, + uint64_t in_dim, + uint64_t out_dim, + const ds4_gpu_tensor *x, + int logical_tier, + const char *label) { + cuda_q8_hc_matmul_prepared prepared = {}; + if (!cuda_q8_hc_matmul_prepare( + block_out, model_map, model_size, weight_offset, + in_dim, out_dim, x, logical_tier, label, &prepared)) { + return 0; + } + const cudaStream_t stream = cuda_decode_stream(); + if (prepared.aligned) { + const uint64_t nblk = out_dim * prepared.blocks; + const uint64_t dq_bytes = + (nblk * sizeof(__half) + 63u) & ~63ull; + matmul_q8_0_aligned_preq_warp8_kernel<<< + ((unsigned)out_dim + 7u) / 8u, 256, 0, stream>>>( + (float *)block_out->ptr, + (const int4 *)(prepared.aligned + dq_bytes), + (const __half *)prepared.aligned, + prepared.xq, prepared.xscale, + out_dim, prepared.blocks); + } else { + matmul_q8_0_preq_warp8_kernel<<< + ((unsigned)out_dim + 7u) / 8u, 256, 0, stream>>>( + (float *)block_out->ptr, + reinterpret_cast(prepared.raw), + prepared.xq, prepared.xscale, + in_dim, out_dim, prepared.blocks, prepared.use_dp4a); + } + return cuda_ok(cudaGetLastError(), + "matmul_q8_0_hc_split matmul launch"); +} + extern "C" int ds4_gpu_matmul_f16_tensor(ds4_gpu_tensor *out, const void *model_map, uint64_t model_size, uint64_t weight_offset, uint64_t in_dim, uint64_t out_dim, const ds4_gpu_tensor *x, uint64_t n_tok) { if (!out || !x || !model_map) return 0; if (weight_offset > model_size || out_dim > UINT64_MAX / in_dim) return 0; @@ -31857,7 +32201,7 @@ extern "C" int ds4_gpu_hc_expand_split_tensor(ds4_gpu_tensor *out_hc, const ds4_ uint32_t mix_hc = 2u * n_hc + n_hc * n_hc; uint64_t n_elem = (uint64_t)n_tokens * n_hc * n_embd; const float *base = (const float *)split->ptr; - hc_expand_kernel<<<(n_elem + 255) / 256, 256>>>((float *)out_hc->ptr, + hc_expand_kernel<<<(n_elem + 255) / 256, 256, 0, cuda_decode_stream()>>>((float *)out_hc->ptr, (const float *)block_out->ptr, (const float *)block_out->ptr, (const float *)block_out->ptr, @@ -31893,7 +32237,7 @@ extern "C" int ds4_gpu_hc_expand_add2_split_tensor(ds4_gpu_tensor *out_hc, const uint32_t mix_hc = 2u * n_hc + n_hc * n_hc; uint64_t n_elem = (uint64_t)n_tokens * n_hc * n_embd; const float *base = (const float *)split->ptr; - hc_expand_kernel<<<(n_elem + 255) / 256, 256>>>((float *)out_hc->ptr, + hc_expand_kernel<<<(n_elem + 255) / 256, 256, 0, cuda_decode_stream()>>>((float *)out_hc->ptr, (const float *)block_out->ptr, (const float *)block_add->ptr, (const float *)block_add2->ptr, @@ -31919,25 +32263,22 @@ extern "C" int ds4_gpu_shared_down_hc_expand_q8_0_tensor( const ds4_gpu_tensor *split, uint32_t n_embd, uint32_t n_hc) { - if (getenv("DS4_CUDA_DISABLE_Q8_HC_EXPAND_FUSED") == NULL) { - return cuda_matmul_q8_0_hc_expand_tensor_labeled(out_hc, shared_out, - model_map, model_size, - weight_offset, - in_dim, out_dim, - shared_mid, - routed_out, - NULL, - NULL, NULL, NULL, 0, - residual_hc, - split, - n_embd, n_hc, - "shared_down_hc_expand"); - } - return ds4_gpu_matmul_q8_0_tensor(shared_out, model_map, model_size, - weight_offset, in_dim, out_dim, - shared_mid, 1) && - ds4_gpu_hc_expand_add_split_tensor(out_hc, shared_out, routed_out, - residual_hc, split, n_embd, n_hc); + const int path = cuda_q8_hc_expand_policy_path(/*owned=*/0); + const int result = path == CUDA_Q8_HC_EXPAND_FUSED + ? cuda_matmul_q8_0_hc_expand_tensor_labeled( + out_hc, shared_out, model_map, model_size, weight_offset, + in_dim, out_dim, shared_mid, routed_out, NULL, + NULL, NULL, NULL, 0, residual_hc, split, n_embd, n_hc, + "shared_down_hc_expand") + : (cuda_matmul_q8_0_hc_split_matmul_tensor_labeled( + shared_out, model_map, model_size, weight_offset, + in_dim, out_dim, shared_mid, + ds4_tensor_device_idx(out_hc), + "shared_down_hc_expand_split") && + ds4_gpu_hc_expand_add_split_tensor( + out_hc, shared_out, routed_out, residual_hc, split, + n_embd, n_hc)); + return cuda_q8_hc_expand_policy_complete(path, result); } extern "C" int ds4_gpu_shared_down_hc_expand_add_q8_0_tensor( @@ -31955,26 +32296,22 @@ extern "C" int ds4_gpu_shared_down_hc_expand_add_q8_0_tensor( const ds4_gpu_tensor *split, uint32_t n_embd, uint32_t n_hc) { - if (getenv("DS4_CUDA_DISABLE_Q8_HC_EXPAND_FUSED") == NULL) { - return cuda_matmul_q8_0_hc_expand_tensor_labeled(out_hc, shared_out, - model_map, model_size, - weight_offset, - in_dim, out_dim, - shared_mid, - routed_out, - routed_add, - NULL, NULL, NULL, 0, - residual_hc, - split, - n_embd, n_hc, - "shared_down_hc_expand_add"); - } - return ds4_gpu_matmul_q8_0_tensor(shared_out, model_map, model_size, - weight_offset, in_dim, out_dim, - shared_mid, 1) && - ds4_gpu_hc_expand_add2_split_tensor(out_hc, shared_out, routed_out, - routed_add, residual_hc, split, - n_embd, n_hc); + const int path = cuda_q8_hc_expand_policy_path(/*owned=*/0); + const int result = path == CUDA_Q8_HC_EXPAND_FUSED + ? cuda_matmul_q8_0_hc_expand_tensor_labeled( + out_hc, shared_out, model_map, model_size, weight_offset, + in_dim, out_dim, shared_mid, routed_out, routed_add, + NULL, NULL, NULL, 0, residual_hc, split, n_embd, n_hc, + "shared_down_hc_expand_add") + : (cuda_matmul_q8_0_hc_split_matmul_tensor_labeled( + shared_out, model_map, model_size, weight_offset, + in_dim, out_dim, shared_mid, + ds4_tensor_device_idx(out_hc), + "shared_down_hc_expand_add_split") && + ds4_gpu_hc_expand_add2_split_tensor( + out_hc, shared_out, routed_out, routed_add, residual_hc, + split, n_embd, n_hc)); + return cuda_q8_hc_expand_policy_complete(path, result); } extern "C" int ds4_gpu_shared_down_hc_expand_owned_q8_0_tensor( @@ -31994,8 +32331,8 @@ extern "C" int ds4_gpu_shared_down_hc_expand_owned_q8_0_tensor( const ds4_gpu_tensor *split, uint32_t n_embd, uint32_t n_hc) { - if (getenv("DS4_CUDA_DISABLE_Q8_HC_EXPAND_FUSED") != NULL) return 0; - return cuda_matmul_q8_0_hc_expand_tensor_labeled( + const int path = cuda_q8_hc_expand_policy_path(/*owned=*/1); + const int result = cuda_matmul_q8_0_hc_expand_tensor_labeled( out_hc, shared_out, model_map, @@ -32015,6 +32352,7 @@ extern "C" int ds4_gpu_shared_down_hc_expand_owned_q8_0_tensor( n_embd, n_hc, "shared_down_hc_expand_owned"); + return cuda_q8_hc_expand_policy_complete(path, result); } extern "C" int ds4_gpu_matmul_q8_0_hc_expand_tensor( @@ -32030,24 +32368,391 @@ extern "C" int ds4_gpu_matmul_q8_0_hc_expand_tensor( const ds4_gpu_tensor *split, uint32_t n_embd, uint32_t n_hc) { - if (getenv("DS4_CUDA_DISABLE_Q8_HC_EXPAND_FUSED") == NULL) { - return cuda_matmul_q8_0_hc_expand_tensor_labeled(out_hc, block_out, - model_map, model_size, - weight_offset, - in_dim, out_dim, - x, - NULL, - NULL, - NULL, NULL, NULL, 0, - residual_hc, - split, - n_embd, n_hc, - "q8_hc_expand"); - } - return ds4_gpu_matmul_q8_0_tensor(block_out, model_map, model_size, - weight_offset, in_dim, out_dim, x, 1) && - ds4_gpu_hc_expand_split_tensor(out_hc, block_out, residual_hc, - split, n_embd, n_hc); + const int path = cuda_q8_hc_expand_policy_path(/*owned=*/0); + const int result = path == CUDA_Q8_HC_EXPAND_FUSED + ? cuda_matmul_q8_0_hc_expand_tensor_labeled( + out_hc, block_out, model_map, model_size, weight_offset, + in_dim, out_dim, x, NULL, NULL, NULL, NULL, NULL, 0, + residual_hc, split, n_embd, n_hc, "q8_hc_expand") + : (cuda_matmul_q8_0_hc_split_matmul_tensor_labeled( + block_out, model_map, model_size, weight_offset, + in_dim, out_dim, x, ds4_tensor_device_idx(out_hc), + "q8_hc_expand_split") && + ds4_gpu_hc_expand_split_tensor( + out_hc, block_out, residual_hc, split, n_embd, n_hc)); + return cuda_q8_hc_expand_policy_complete(path, result); +} + +/* Device oracle for the Q8 shared-down/HC A/B boundary. Its 128x1024 + * fixture satisfies the production aligned-artifact predicate and installs + * temporary raw/derived resolver entries so the real fused and split helpers + * run. It checks both aligned and raw paths bit-for-bit; the aligned split + * plus both HC epilogues are also recorded in a CUDA graph to catch accidental + * legacy-stream launches. */ +extern "C" int ds4_cuda_test_q8_hc_expand_oracle(void) { + g_q8_hc_expand_oracle_runs.fetch_add(1u, std::memory_order_relaxed); + + constexpr uint32_t in_dim = 1024u; + constexpr uint32_t n_embd = 128u; + constexpr uint32_t n_hc = 4u; + constexpr uint32_t blocks = in_dim / 32u; + constexpr uint64_t n_weight_blocks = + (uint64_t)n_embd * blocks; + constexpr uint64_t weight_bytes = + n_weight_blocks * 34u; + constexpr uint64_t aligned_dq_bytes = + (n_weight_blocks * sizeof(__half) + 63u) & ~63ull; + constexpr uint64_t aligned_bytes = + aligned_dq_bytes + n_weight_blocks * 32u; + constexpr uint64_t x_bytes = (uint64_t)in_dim * sizeof(float); + constexpr uint64_t row_bytes = (uint64_t)n_embd * sizeof(float); + constexpr uint64_t hc_bytes = + (uint64_t)n_hc * n_embd * sizeof(float); + constexpr uint64_t split_count = 2u * n_hc + n_hc * n_hc; + constexpr uint64_t split_bytes = split_count * sizeof(float); + + void *d_aligned = NULL; + void *d_raw = NULL; + void *d_x = NULL; + void *d_residual = NULL; + void *d_split = NULL; + void *d_add = NULL; + void *d_add2 = NULL; + void *d_block_fused_add = NULL; + void *d_block_fused_plain = NULL; + void *d_block_split = NULL; + void *d_hc_fused_add = NULL; + void *d_hc_fused_plain = NULL; + void *d_hc_split_add = NULL; + void *d_hc_split_plain = NULL; + cudaStream_t stream = NULL; + cudaGraph_t graph = NULL; + cudaGraphExec_t exec = NULL; + cudaStream_t saved_decode_graph_stream = g_decode_graph_stream; + const int saved_decode_graph_capturing = g_decode_graph_capturing; + const size_t saved_model_range_count = g_model_ranges.size(); + const size_t saved_derived_range_count = g_derived_ranges.size(); + const int saved_gb10_tier0 = g_cuda_is_gb10[0]; + int oracle_ranges_installed = 0; + int capture_started = 0; + int ok = 0; + + std::vector allocations; + auto alloc = [&allocations](void **ptr, uint64_t bytes) -> int { + if (cudaMalloc(ptr, (size_t)bytes) != cudaSuccess) return 0; + allocations.push_back(*ptr); + return 1; + }; + + do { + if (saved_decode_graph_capturing) break; + if (!alloc(&d_aligned, aligned_bytes) || + !alloc(&d_raw, weight_bytes) || + !alloc(&d_x, x_bytes) || + !alloc(&d_residual, hc_bytes) || + !alloc(&d_split, split_bytes) || + !alloc(&d_add, row_bytes) || + !alloc(&d_add2, row_bytes) || + !alloc(&d_block_fused_add, row_bytes) || + !alloc(&d_block_fused_plain, row_bytes) || + !alloc(&d_block_split, row_bytes) || + !alloc(&d_hc_fused_add, hc_bytes) || + !alloc(&d_hc_fused_plain, hc_bytes) || + !alloc(&d_hc_split_add, hc_bytes) || + !alloc(&d_hc_split_plain, hc_bytes)) { + break; + } + if (cudaStreamCreate(&stream) != cudaSuccess) break; + + std::vector h_weight((size_t)weight_bytes); + for (uint32_t row = 0; row < n_embd; row++) { + for (uint32_t block = 0; block < blocks; block++) { + unsigned char *dst = h_weight.data() + + ((uint64_t)row * blocks + block) * 34u; + const __half scale = __float2half( + 0.00390625f * (float)(1u + ((row + block) % 3u))); + memcpy(dst, &scale, sizeof(scale)); + int8_t *codes = (int8_t *)(dst + sizeof(scale)); + for (uint32_t i = 0; i < 32u; i++) { + codes[i] = (int8_t)( + (int)((row * 5u + block * 7u + i * 3u) % 23u) - 11); + } + } + } + std::vector h_aligned((size_t)aligned_bytes); + for (uint64_t block = 0; block < n_weight_blocks; block++) { + const unsigned char *src = h_weight.data() + block * 34u; + memcpy(h_aligned.data() + block * sizeof(__half), + src, sizeof(__half)); + memcpy(h_aligned.data() + aligned_dq_bytes + block * 32u, + src + sizeof(__half), 32u); + } + std::vector h_x(in_dim); + std::vector h_residual((size_t)n_hc * n_embd); + std::vector h_split((size_t)split_count); + constexpr float host_guard = 12345.5f; + std::vector h_add_storage(n_embd + 2u, host_guard); + std::vector h_add2_storage(n_embd + 2u, host_guard); + float *h_add = h_add_storage.data() + 1u; + float *h_add2 = h_add2_storage.data() + 1u; + for (uint32_t i = 0; i < in_dim; i++) { + h_x[i] = (float)((int)(i % 17u) - 8) * 0.0625f; + } + for (uint32_t i = 0; i < n_embd; i++) { + h_add[i] = (float)((int)(i % 7u) - 3) * 0.03125f; + h_add2[i] = (float)((int)(i % 5u) - 2) * 0.015625f; + } + if (h_add_storage.front() != host_guard || + h_add_storage.back() != host_guard || + h_add2_storage.front() != host_guard || + h_add2_storage.back() != host_guard) { + break; + } + for (uint32_t h = 0; h < n_hc; h++) { + for (uint32_t d = 0; d < n_embd; d++) { + h_residual[(uint64_t)h * n_embd + d] = + (float)((int)((h * 13u + d) % 19u) - 9) * 0.0078125f; + } + h_split[n_hc + h] = 0.25f + (float)h * 0.0625f; + } + for (uint32_t src = 0; src < n_hc; src++) { + for (uint32_t dst = 0; dst < n_hc; dst++) { + h_split[2u * n_hc + dst + (uint64_t)src * n_hc] = + src == dst ? 0.75f : 0.03125f * (float)(src + dst + 1u); + } + } + + if (cudaMemcpy(d_aligned, h_aligned.data(), (size_t)aligned_bytes, + cudaMemcpyHostToDevice) != cudaSuccess || + cudaMemcpy(d_raw, h_weight.data(), (size_t)weight_bytes, + cudaMemcpyHostToDevice) != cudaSuccess || + cudaMemcpy(d_x, h_x.data(), (size_t)x_bytes, + cudaMemcpyHostToDevice) != cudaSuccess || + cudaMemcpy(d_residual, h_residual.data(), (size_t)hc_bytes, + cudaMemcpyHostToDevice) != cudaSuccess || + cudaMemcpy(d_split, h_split.data(), (size_t)split_bytes, + cudaMemcpyHostToDevice) != cudaSuccess || + cudaMemcpy(d_add, h_add, (size_t)row_bytes, + cudaMemcpyHostToDevice) != cudaSuccess || + cudaMemcpy(d_add2, h_add2, (size_t)row_bytes, + cudaMemcpyHostToDevice) != cudaSuccess) { + break; + } + + if (g_n_gpus != 1 || !cuda_q8_use_dp4a() || + !cuda_aligned_q8_enabled() || + getenv("DS4_CUDA_NO_Q8_FUSED_ALIGNED") != NULL || + getenv("DS4_CUDA_NO_DERIVED_WEIGHTS") != NULL) { + break; + } + const void *oracle_model_map = h_weight.data(); + g_model_ranges.push_back({ + oracle_model_map, 0u, weight_bytes, (char *)d_raw, + NULL, NULL, 0u, 0, 1}); + g_derived_ranges.push_back({ + oracle_model_map, 0u, weight_bytes, + CUDA_DERIVED_Q8_0_ALIGNED_DENSE, + in_dim, n_embd, 1u, aligned_bytes, (char *)d_aligned}); + g_cuda_is_gb10[0] = 1; + oracle_ranges_installed = 1; + + ds4_gpu_tensor x = {}; + ds4_gpu_tensor block_fused_add = {}; + ds4_gpu_tensor block_fused_plain = {}; + ds4_gpu_tensor block_split = {}; + ds4_gpu_tensor add = {}; + ds4_gpu_tensor add2 = {}; + ds4_gpu_tensor residual = {}; + ds4_gpu_tensor split = {}; + ds4_gpu_tensor hc_fused_add = {}; + ds4_gpu_tensor hc_fused_plain = {}; + ds4_gpu_tensor hc_split_add = {}; + ds4_gpu_tensor hc_split_plain = {}; + x.ptr = d_x; + x.bytes = x_bytes; + x.device_id = 0; + block_fused_add.ptr = d_block_fused_add; + block_fused_add.bytes = row_bytes; + block_fused_add.device_id = 0; + block_fused_plain.ptr = d_block_fused_plain; + block_fused_plain.bytes = row_bytes; + block_fused_plain.device_id = 0; + block_split.ptr = d_block_split; + block_split.bytes = row_bytes; + block_split.device_id = 0; + add.ptr = d_add; + add.bytes = row_bytes; + add.device_id = 0; + add2.ptr = d_add2; + add2.bytes = row_bytes; + add2.device_id = 0; + residual.ptr = d_residual; + residual.bytes = hc_bytes; + residual.device_id = 0; + split.ptr = d_split; + split.bytes = split_bytes; + split.device_id = 0; + hc_fused_add.ptr = d_hc_fused_add; + hc_fused_add.bytes = hc_bytes; + hc_fused_add.device_id = 0; + hc_fused_plain.ptr = d_hc_fused_plain; + hc_fused_plain.bytes = hc_bytes; + hc_fused_plain.device_id = 0; + hc_split_add.ptr = d_hc_split_add; + hc_split_add.bytes = hc_bytes; + hc_split_add.device_id = 0; + hc_split_plain.ptr = d_hc_split_plain; + hc_split_plain.bytes = hc_bytes; + hc_split_plain.device_id = 0; + + /* Exercise the actual fused preparation/resolver wrapper and warm + * its reusable Q8 scratch before graph capture. */ + const cudaStream_t eager_helper_stream = cuda_decode_stream(); + if (!cuda_matmul_q8_0_hc_expand_tensor_labeled( + &hc_fused_add, &block_fused_add, + oracle_model_map, weight_bytes, 0u, + in_dim, n_embd, &x, &add, &add2, + NULL, NULL, NULL, 0u, &residual, &split, + n_embd, n_hc, "q8_hc_oracle_aligned_add") || + !cuda_matmul_q8_0_hc_expand_tensor_labeled( + &hc_fused_plain, &block_fused_plain, + oracle_model_map, weight_bytes, 0u, + in_dim, n_embd, &x, NULL, NULL, + NULL, NULL, NULL, 0u, &residual, &split, + n_embd, n_hc, "q8_hc_oracle_aligned_plain") || + cudaStreamSynchronize(eager_helper_stream) != cudaSuccess) { + break; + } + + if (cudaStreamBeginCapture(stream, + cudaStreamCaptureModeGlobal) != cudaSuccess) { + break; + } + capture_started = 1; + g_decode_graph_stream = stream; + g_decode_graph_capturing = 1; + const int matmul_ok = + cuda_matmul_q8_0_hc_split_matmul_tensor_labeled( + &block_split, oracle_model_map, weight_bytes, 0u, + in_dim, n_embd, &x, 0, + "q8_hc_oracle_aligned_split"); + const int add_ok = ds4_gpu_hc_expand_add2_split_tensor( + &hc_split_add, &block_split, &add, &add2, &residual, &split, + n_embd, n_hc); + const int plain_ok = ds4_gpu_hc_expand_split_tensor( + &hc_split_plain, &block_split, &residual, &split, + n_embd, n_hc); + g_decode_graph_capturing = saved_decode_graph_capturing; + g_decode_graph_stream = saved_decode_graph_stream; + const cudaError_t capture_end = cudaStreamEndCapture(stream, &graph); + capture_started = 0; + if (!matmul_ok || !add_ok || !plain_ok || + capture_end != cudaSuccess || !graph) { + break; + } + if (cudaGraphInstantiate(&exec, graph, NULL, NULL, 0) != cudaSuccess || + !exec || cudaGraphLaunch(exec, stream) != cudaSuccess || + cudaStreamSynchronize(stream) != cudaSuccess) { + break; + } + + auto outputs_match = [&]() -> int { + std::vector h_block_fused_add(n_embd); + std::vector h_block_fused_plain(n_embd); + std::vector h_block_split(n_embd); + std::vector h_fused_add((size_t)n_hc * n_embd); + std::vector h_split_add((size_t)n_hc * n_embd); + std::vector h_fused_plain((size_t)n_hc * n_embd); + std::vector h_split_plain((size_t)n_hc * n_embd); + if (cudaMemcpy(h_block_fused_add.data(), d_block_fused_add, + (size_t)row_bytes, cudaMemcpyDeviceToHost) != + cudaSuccess || + cudaMemcpy(h_block_fused_plain.data(), d_block_fused_plain, + (size_t)row_bytes, cudaMemcpyDeviceToHost) != + cudaSuccess || + cudaMemcpy(h_block_split.data(), d_block_split, + (size_t)row_bytes, cudaMemcpyDeviceToHost) != + cudaSuccess || + cudaMemcpy(h_fused_add.data(), d_hc_fused_add, + (size_t)hc_bytes, cudaMemcpyDeviceToHost) != + cudaSuccess || + cudaMemcpy(h_split_add.data(), d_hc_split_add, + (size_t)hc_bytes, cudaMemcpyDeviceToHost) != + cudaSuccess || + cudaMemcpy(h_fused_plain.data(), d_hc_fused_plain, + (size_t)hc_bytes, cudaMemcpyDeviceToHost) != + cudaSuccess || + cudaMemcpy(h_split_plain.data(), d_hc_split_plain, + (size_t)hc_bytes, cudaMemcpyDeviceToHost) != + cudaSuccess) { + return 0; + } + return memcmp(h_block_fused_add.data(), h_block_split.data(), + (size_t)row_bytes) == 0 && + memcmp(h_block_fused_plain.data(), h_block_split.data(), + (size_t)row_bytes) == 0 && + memcmp(h_fused_add.data(), h_split_add.data(), + (size_t)hc_bytes) == 0 && + memcmp(h_fused_plain.data(), h_split_plain.data(), + (size_t)hc_bytes) == 0; + }; + if (!outputs_match()) break; + + /* Remove only the temporary derived entry to force the same public + * helpers through their raw-weight side, then repeat bit parity. */ + g_derived_ranges.resize(saved_derived_range_count); + g_cuda_is_gb10[0] = 0; + const cudaStream_t raw_helper_stream = cuda_decode_stream(); + if (!cuda_matmul_q8_0_hc_expand_tensor_labeled( + &hc_fused_add, &block_fused_add, + oracle_model_map, weight_bytes, 0u, + in_dim, n_embd, &x, &add, &add2, + NULL, NULL, NULL, 0u, &residual, &split, + n_embd, n_hc, "q8_hc_oracle_raw_add") || + !cuda_matmul_q8_0_hc_expand_tensor_labeled( + &hc_fused_plain, &block_fused_plain, + oracle_model_map, weight_bytes, 0u, + in_dim, n_embd, &x, NULL, NULL, + NULL, NULL, NULL, 0u, &residual, &split, + n_embd, n_hc, "q8_hc_oracle_raw_plain") || + !cuda_matmul_q8_0_hc_split_matmul_tensor_labeled( + &block_split, oracle_model_map, weight_bytes, 0u, + in_dim, n_embd, &x, 0, "q8_hc_oracle_raw_split") || + !ds4_gpu_hc_expand_add2_split_tensor( + &hc_split_add, &block_split, &add, &add2, + &residual, &split, n_embd, n_hc) || + !ds4_gpu_hc_expand_split_tensor( + &hc_split_plain, &block_split, &residual, &split, + n_embd, n_hc) || + cudaStreamSynchronize(raw_helper_stream) != cudaSuccess) { + break; + } + ok = outputs_match(); + } while (0); + + g_decode_graph_capturing = saved_decode_graph_capturing; + g_decode_graph_stream = saved_decode_graph_stream; + if (capture_started && stream) { + cudaGraph_t abandoned = NULL; + (void)cudaStreamEndCapture(stream, &abandoned); + if (abandoned) (void)cudaGraphDestroy(abandoned); + } + if (oracle_ranges_installed) { + g_model_ranges.resize(saved_model_range_count); + g_derived_ranges.resize(saved_derived_range_count); + g_cuda_is_gb10[0] = saved_gb10_tier0; + oracle_ranges_installed = 0; + } + if (exec) (void)cudaGraphExecDestroy(exec); + if (graph) (void)cudaGraphDestroy(graph); + if (stream) (void)cudaStreamDestroy(stream); + for (void *ptr : allocations) (void)cudaFree(ptr); + if (!ok) { + (void)cudaGetLastError(); + g_q8_hc_expand_oracle_failures.fetch_add( + 1u, std::memory_order_relaxed); + } + return ok; } /* --gpu-vram auto probe. Defined here (in the .cu unit) so the diff --git a/ds4_gpu.h b/ds4_gpu.h index 11cf4c421..cc33c8219 100644 --- a/ds4_gpu.h +++ b/ds4_gpu.h @@ -381,9 +381,33 @@ typedef struct ds4_cuda_stream_expert_persistent_report { int stats; int oracle; } ds4_cuda_stream_expert_persistent_report; +typedef struct ds4_cuda_q8_hc_expand_report { + uint64_t candidates; + uint64_t fused_attempts; + uint64_t fused_completed; + uint64_t split_attempts; + uint64_t split_completed; + uint64_t failures; + uint64_t capture_candidates; + uint64_t owned_forced_fused; + uint64_t multi_gpu_forced_fused; + uint64_t oracle_runs; + uint64_t oracle_failures; + int force_fused; + int split_requested; + int stats; +} ds4_cuda_q8_hc_expand_report; /* CUDA-only policy/planner/scatter hooks. The policy hook takes explicit * values so tests do not need to mutate process environment around once_flag * initialization. */ +int ds4_cuda_test_q8_hc_expand_policy( + int force_fused, int disable_fused, int n_gpus, int owned, + int capture, int *fused_out, int *owned_forced_out, + int *multi_gpu_forced_out, int *capture_out); +int ds4_cuda_test_q8_hc_expand_env_value(const char *value); +int ds4_cuda_test_q8_hc_expand_oracle(void); +void ds4_cuda_q8_hc_expand_get_report( + ds4_cuda_q8_hc_expand_report *report); int ds4_cuda_test_stream_selected_batch_policy( int enable, int disable, int require, int oracle, int *enabled_out, int *required_out, int *oracle_out); diff --git a/tests/test_gpu_model_cache.c b/tests/test_gpu_model_cache.c index 82659b090..c8cd7324d 100644 --- a/tests/test_gpu_model_cache.c +++ b/tests/test_gpu_model_cache.c @@ -30,6 +30,50 @@ int main(void) { int enabled = -1; int required = -1; int oracle = -1; + int fused = -1; + int owned_forced = -1; + int multi_gpu_forced = -1; + int capture = -1; + CHECK(!ds4_cuda_test_q8_hc_expand_env_value(NULL) && + !ds4_cuda_test_q8_hc_expand_env_value("") && + !ds4_cuda_test_q8_hc_expand_env_value("0") && + !ds4_cuda_test_q8_hc_expand_env_value("false") && + !ds4_cuda_test_q8_hc_expand_env_value("NO") && + !ds4_cuda_test_q8_hc_expand_env_value("off") && + ds4_cuda_test_q8_hc_expand_env_value("1") && + ds4_cuda_test_q8_hc_expand_env_value("true"), + "Q8 HC value-aware environment parser"); + CHECK(ds4_cuda_test_q8_hc_expand_policy( + 0, 0, 1, 0, 0, &fused, &owned_forced, + &multi_gpu_forced, &capture) && + fused == 1 && owned_forced == 0 && multi_gpu_forced == 0 && + capture == 0, + "Q8 HC defaults to fused"); + CHECK(ds4_cuda_test_q8_hc_expand_policy( + 0, 1, 1, 0, 0, &fused, &owned_forced, + &multi_gpu_forced, &capture) && + fused == 0 && owned_forced == 0 && multi_gpu_forced == 0, + "Q8 HC split opt-in on single GPU"); + CHECK(ds4_cuda_test_q8_hc_expand_policy( + 1, 1, 1, 0, 0, &fused, &owned_forced, + &multi_gpu_forced, &capture) && + fused == 1 && owned_forced == 0 && multi_gpu_forced == 0, + "Q8 HC force-fused dominates conflicting split request"); + CHECK(ds4_cuda_test_q8_hc_expand_policy( + 0, 1, 2, 0, 0, &fused, &owned_forced, + &multi_gpu_forced, &capture) && + fused == 1 && owned_forced == 0 && multi_gpu_forced == 1, + "Q8 HC multi-GPU remains fused"); + CHECK(ds4_cuda_test_q8_hc_expand_policy( + 0, 1, 1, 1, 0, &fused, &owned_forced, + &multi_gpu_forced, &capture) && + fused == 1 && owned_forced == 1 && multi_gpu_forced == 0, + "Q8 HC owned dispatch remains fused"); + CHECK(ds4_cuda_test_q8_hc_expand_policy( + 0, 1, 1, 0, 1, &fused, &owned_forced, + &multi_gpu_forced, &capture) && + fused == 0 && capture == 1, + "Q8 HC split graph-capture policy matches eager policy"); CHECK(!ds4_cuda_test_stream_selected_batch_env_value(NULL) && !ds4_cuda_test_stream_selected_batch_env_value("") && !ds4_cuda_test_stream_selected_batch_env_value("0") && @@ -221,6 +265,17 @@ int main(void) { } CHECK(ds4_gpu_init(), "ds4_gpu_init"); + ds4_cuda_q8_hc_expand_report q8_hc_before; + memset(&q8_hc_before, 0, sizeof(q8_hc_before)); + ds4_cuda_q8_hc_expand_get_report(&q8_hc_before); + CHECK(ds4_cuda_test_q8_hc_expand_oracle(), + "Q8 HC fused/split graph-capture parity oracle"); + ds4_cuda_q8_hc_expand_report q8_hc_after; + memset(&q8_hc_after, 0, sizeof(q8_hc_after)); + ds4_cuda_q8_hc_expand_get_report(&q8_hc_after); + CHECK(q8_hc_after.oracle_runs == q8_hc_before.oracle_runs + 1u && + q8_hc_after.oracle_failures == q8_hc_before.oracle_failures, + "Q8 HC oracle coverage counters"); ds4_gpu_set_streaming_expert_cache_budget(3u); ds4_gpu_set_streaming_expert_cache_expert_bytes(40u); CHECK(ds4_gpu_stream_expert_cache_configured_count() == 0u && From 448ce12b75567d979d71bed22f317cf0ae448103 Mon Sep 17 00:00:00 2001 From: Giorgio Oppo Date: Sat, 15 Aug 2026 08:42:38 +0200 Subject: [PATCH 040/189] metal: promote IQ2 SSD grouped prefill defaults --- README.md | 16 ++ ds4_gpu.h | 1 + ds4_metal.m | 91 +++++++++-- tests/test_metal_iq2_ssd_grouped_mm.c | 209 +++++++++++++++++++++++++- 4 files changed, 296 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 1c966ddaa..d6ffdad4c 100644 --- a/README.md +++ b/README.md @@ -1068,6 +1068,22 @@ applies the same routed-prefill headroom before sizing the dynamic cache. Leave the hot expert preload enabled for normal use; use `--ssd-streaming-cold` and `--ssd-streaming-preload-experts N` only for measurements. +For Metal IQ2_XXS/Q2_K models, eligible SSD prefill chunks automatically use +grouped address matmuls when the dynamic cache can retain the complete expert +domain. This includes normal 128-token chunks; the automatic range is 32–760 +tokens on the 256-expert Flash model and requires, for example, +`--ssd-streaming-cache-experts 256`. Once that material condition is met, +selection is fail-closed by default; no environment prefix is required. Set +`DS4_METAL_ENABLE_IQ2_XXS_SSD_PREFILL_MM=0` or +`DS4_METAL_DISABLE_IQ2_XXS_SSD_PREFILL_MM=1` for the legacy sparse-matvec +rollback. `DS4_METAL_REQUIRE_IQ2_XXS_SSD_PREFILL_MM=0` keeps automatic +selection but permits a fallback, while explicit `REQUIRE=1` also rejects an +insufficient cache. Combining `REQUIRE=1` with `DISABLE=1` fails on an eligible +grouped-MM candidate, while short tail chunks retain their normal fallback. The +IQ2 live cache index remains automatic for its production shape, selected-load +early commit remains off unless explicitly enabled, and grouped-MM statistics +plus streaming timing summaries remain opt-in diagnostics. + ### Practical SSD streaming examples On 64GB MacBooks, start with the 2-bit Flash GGUF and a moderate expert cache: diff --git a/ds4_gpu.h b/ds4_gpu.h index cc33c8219..999c73c58 100644 --- a/ds4_gpu.h +++ b/ds4_gpu.h @@ -279,6 +279,7 @@ enum { DS4_GPU_TEST_OUTPUT_HC_WEIGHTS4 = 1u << 5, DS4_GPU_TEST_HC_RMS_SCALE_PROJ = 1u << 6, DS4_GPU_TEST_STREAMING_LIVE_INDEX_FAILURE = 1u << 7, + DS4_GPU_TEST_IQ2_SSD_GROUPED_PIPELINE_FAILURE = 1u << 8, }; void ds4_gpu_test_set_flags(uint32_t flags); void ds4_gpu_release_zero_prefix_prefill_mask_cache(void); diff --git a/ds4_metal.m b/ds4_metal.m index f71e3858b..23d826690 100644 --- a/ds4_metal.m +++ b/ds4_metal.m @@ -2694,6 +2694,54 @@ static int ds4_gpu_env_bool(const char *name) { return 1; } +/* IQ2_XXS/Q2_K SSD grouped address-MM is the production default. Its + * implicit fail-closed arm is deliberately narrower than an explicit + * REQUIRE: it applies only after the complete selected-address domain is + * materially available. This keeps byte/automatic cache budgets that cannot + * retain every expert on the established sparse-MV fallback. ENABLE=0 and + * DISABLE=1 are rollback controls; REQUIRE=0 retains automatic selection but + * permits fallback, while explicit REQUIRE=1 remains strong. */ +static int ds4_gpu_iq2_stream_addr_mm_resolve_policy( + int enable, + int require, + int disable, + int material_ready, + int *requested_out, + int *required_out) { + if (!requested_out || !required_out) return 0; + + const int explicitly_required = require == 1; + const int disabled = disable == 1; + if (disabled) { + *requested_out = 0; + *required_out = explicitly_required; + return 1; + } + + const int requested = explicitly_required || enable != 0; + const int implicitly_required = + requested && require < 0 && material_ready != 0; + *requested_out = requested; + *required_out = explicitly_required || implicitly_required; + return 1; +} + +/* Standalone policy hook: values use ds4_gpu_env_bool's -1/0/1 convention. */ +int ds4_gpu_test_iq2_stream_addr_mm_policy( + int enable, + int require, + int disable, + int material_ready, + int *requested_out, + int *required_out) { + return ds4_gpu_iq2_stream_addr_mm_resolve_policy(enable, + require, + disable, + material_ready, + requested_out, + required_out); +} + static void ds4_gpu_iq2_stream_addr_mm_stats_reset(void) { g_iq2_stream_addr_mm_candidate_calls = 0; g_iq2_stream_addr_mm_calls = 0; @@ -44796,18 +44844,28 @@ int ds4_gpu_routed_moe_batch_tensor( g_moe_mul_mv_addr_q2_k_sum6_pipeline != nil; /* IQ2 grouped-MM controls are value-aware. The sparse address-MM - * path is the default for eligible IQ2_XXS/Q2_K SSD prefill; - * ENABLE=0 is the legacy-MV control, REQUIRE forces coverage, and - * the dedicated disable always wins. */ - const bool require_iq2_batch_addr_mm = - ds4_gpu_env_bool("DS4_METAL_REQUIRE_IQ2_XXS_SSD_PREFILL_MM") == 1; + * path is automatic for eligible IQ2_XXS/Q2_K SSD prefill. Once the + * full selected-address domain is available, the automatic choice is + * fail-closed; smaller byte/automatic caches retain the sparse-MV + * fallback. Explicit REQUIRE is intentionally stronger. */ const int enable_iq2_batch_addr_mm = ds4_gpu_env_bool("DS4_METAL_ENABLE_IQ2_XXS_SSD_PREFILL_MM"); - const bool disable_iq2_batch_addr_mm = - ds4_gpu_env_bool("DS4_METAL_DISABLE_IQ2_XXS_SSD_PREFILL_MM") == 1; - const bool request_iq2_batch_addr_mm = - (enable_iq2_batch_addr_mm != 0 || require_iq2_batch_addr_mm) && - !disable_iq2_batch_addr_mm; + const int require_iq2_batch_addr_mm_env = + ds4_gpu_env_bool("DS4_METAL_REQUIRE_IQ2_XXS_SSD_PREFILL_MM"); + const int disable_iq2_batch_addr_mm = + ds4_gpu_env_bool("DS4_METAL_DISABLE_IQ2_XXS_SSD_PREFILL_MM"); + int request_iq2_batch_addr_mm = 0; + int require_iq2_batch_addr_mm = 0; + if (!ds4_gpu_iq2_stream_addr_mm_resolve_policy( + enable_iq2_batch_addr_mm, + require_iq2_batch_addr_mm_env, + disable_iq2_batch_addr_mm, + use_iq2_batch_selected_addr, + &request_iq2_batch_addr_mm, + &require_iq2_batch_addr_mm)) { + fprintf(stderr, "ds4: invalid Metal IQ2_XXS SSD grouped-MM policy\n"); + return 0; + } const bool iq2_batch_addr_mm_candidate = !force_resident && g_ssd_streaming_mode && gate_type == DS4_METAL_TENSOR_IQ2_XXS && @@ -44832,18 +44890,23 @@ int ds4_gpu_routed_moe_batch_tensor( iq2_batch_addr_mm_candidate && use_iq2_batch_selected_addr; id iq2_gate_addr_mm_pipeline = - iq2_batch_addr_mm_policy ? + iq2_batch_addr_mm_policy && + (g_test_flags & + DS4_GPU_TEST_IQ2_SSD_GROUPED_PIPELINE_FAILURE) == 0u ? ds4_gpu_routed_mm_addr_pipeline(gate_type) : nil; id iq2_down_addr_mm_pipeline = - iq2_batch_addr_mm_policy ? + iq2_batch_addr_mm_policy && + (g_test_flags & + DS4_GPU_TEST_IQ2_SSD_GROUPED_PIPELINE_FAILURE) == 0u ? ds4_gpu_routed_mm_addr_pipeline(down_type) : nil; const bool use_iq2_batch_addr_mm = iq2_batch_addr_mm_policy && iq2_gate_addr_mm_pipeline != nil && iq2_down_addr_mm_pipeline != nil; - /* REQUIRE only asserts calls that satisfy the specialization's - * candidate contract; short final chunks keep the established path. */ + /* Both explicit and materially-ready implicit REQUIRE only assert + * calls satisfying the specialization contract. Short final chunks + * keep the established path. */ if (require_iq2_batch_addr_mm && !disable_iq2_batch_addr_mm && iq2_batch_addr_mm_candidate && diff --git a/tests/test_metal_iq2_ssd_grouped_mm.c b/tests/test_metal_iq2_ssd_grouped_mm.c index 096aae69b..fc0df99f4 100644 --- a/tests/test_metal_iq2_ssd_grouped_mm.c +++ b/tests/test_metal_iq2_ssd_grouped_mm.c @@ -92,6 +92,9 @@ int ds4_gpu_test_iq2_stream_addr_mm_stats( uint64_t *candidate_calls, uint64_t *calls, uint64_t *tokens, uint64_t *rows, uint64_t *require_failures, uint32_t *min_tokens, uint32_t *max_tokens); +int ds4_gpu_test_iq2_stream_addr_mm_policy( + int enable, int require, int disable, int material_ready, + int *requested_out, int *required_out); bool ds4_log_is_tty(FILE *fp) { (void)fp; @@ -102,6 +105,43 @@ static uint64_t align_up(uint64_t value, uint64_t alignment) { return (value + alignment - 1u) / alignment * alignment; } +static int check_grouped_mm_policy(void) { + int ok = 1; +#define CHECK_POLICY(label, valid_expected, request_expected, \ + require_expected, enable, require, disable, ready) do { \ + int requested = -1; \ + int required = -1; \ + const int valid = ds4_gpu_test_iq2_stream_addr_mm_policy( \ + (enable), (require), (disable), (ready), &requested, &required); \ + const int case_ok = valid == (valid_expected) && \ + requested == (request_expected) && required == (require_expected); \ + fprintf(stderr, \ + "IQ2 grouped-MM policy %-28s %s valid=%d request=%d " \ + "require=%d\n", \ + (label), case_ok ? "PASS" : "FAIL", valid, requested, required);\ + ok = case_ok && ok; \ +} while (0) + + /* Unset defaults: automatic selection, with fail-closed coverage only + * after the complete selected-address domain is materially ready. */ + CHECK_POLICY("default-ready", 1, 1, 1, -1, -1, -1, 1); + CHECK_POLICY("default-small-cache", 1, 1, 0, -1, -1, -1, 0); + + /* Value-aware rollback and explicit fallback controls. */ + CHECK_POLICY("enable-zero", 1, 0, 0, 0, -1, -1, 1); + CHECK_POLICY("enable-one", 1, 1, 1, 1, -1, -1, 1); + CHECK_POLICY("require-zero", 1, 1, 0, -1, 0, -1, 1); + CHECK_POLICY("require-one-not-ready", 1, 1, 1, -1, 1, -1, 0); + CHECK_POLICY("require-over-enable-zero", 1, 1, 1, 0, 1, -1, 0); + CHECK_POLICY("disable-one", 1, 0, 0, -1, -1, 1, 1); + CHECK_POLICY("disable-zero", 1, 1, 1, -1, -1, 0, 1); + /* The explicit REQUIRE is retained so an eligible IQ2 prefill fails at + * the candidate boundary; unrelated shapes and short tails stay valid. */ + CHECK_POLICY("require-disable-conflict", 1, 0, 1, -1, 1, 1, 1); +#undef CHECK_POLICY + return ok; +} + static void fill_iq2(block_iq2_xxs *matrix, uint32_t salt, uint32_t n_total_expert) { for (uint32_t expert = 0; expert < n_total_expert; expert++) { @@ -221,7 +261,8 @@ static int run_once( uint32_t n_total_expert, const float *x, const int32_t *selected, - const float *weights) { + const float *weights, + bool allow_mid_f16) { const uint32_t tokens = result->tokens; const uint64_t x_count = (uint64_t)tokens * IN_DIM; const uint64_t route_count = (uint64_t)tokens * N_EXPERT; @@ -286,7 +327,7 @@ static int run_once( n_total_expert, N_EXPERT, CLAMP, x_t, 0u, tokens, &mid_is_f16, false); } - if (ok && mid_is_f16) { + if (ok && mid_is_f16 && !allow_mid_f16) { fprintf(stderr, "IQ2_XXS SSD grouped-MM oracle unexpectedly selected f16 mid\n"); ok = 0; @@ -446,17 +487,17 @@ static int run_pair( model, model_size, gate_offset, up_offset, down_offset, gate_expert_bytes, gate_row_bytes, down_expert_bytes, down_row_bytes, n_total_expert, - x, selected, weights); + x, selected, weights, false); unsetenv("DS4_METAL_DISABLE_IQ2_XXS_SSD_PREFILL_MM"); - setenv("DS4_METAL_ENABLE_IQ2_XXS_SSD_PREFILL_MM", "1", 1); - setenv("DS4_METAL_REQUIRE_IQ2_XXS_SSD_PREFILL_MM", "1", 1); + unsetenv("DS4_METAL_ENABLE_IQ2_XXS_SSD_PREFILL_MM"); + unsetenv("DS4_METAL_REQUIRE_IQ2_XXS_SSD_PREFILL_MM"); ds4_gpu_set_streaming_expert_cache_budget(cache_budget); ok = ok && run_once(candidate_name, &candidate, model, model_size, gate_offset, up_offset, down_offset, gate_expert_bytes, gate_row_bytes, down_expert_bytes, down_row_bytes, n_total_expert, - x, selected, weights); + x, selected, weights, false); if (ok) ok = compare_results(name, &candidate, &control); result_free(&control); @@ -506,6 +547,150 @@ static int check_mm_stats_delta(const char *name, return ok; } +static int check_implicit_require_fault( + const void *model, + uint64_t model_size, + uint64_t gate_offset, + uint64_t up_offset, + uint64_t down_offset, + uint64_t gate_expert_bytes, + uint64_t gate_row_bytes, + uint64_t down_expert_bytes, + uint64_t down_row_bytes, + const float *x, + const int32_t *selected, + const float *weights) { + run_result result; + run_result tail_result; + memset(&result, 0, sizeof(result)); + memset(&tail_result, 0, sizeof(tail_result)); + if (!result_alloc(&result, 32u) || !result_alloc(&tail_result, 31u)) { + result_free(&result); + result_free(&tail_result); + return 0; + } + + mm_stats_snapshot before; + mm_stats_snapshot after; + int ok = read_mm_stats(&before); + ds4_gpu_test_set_flags( + DS4_GPU_TEST_IQ2_SSD_GROUPED_PIPELINE_FAILURE); + + unsetenv("DS4_METAL_ENABLE_IQ2_XXS_SSD_PREFILL_MM"); + unsetenv("DS4_METAL_REQUIRE_IQ2_XXS_SSD_PREFILL_MM"); + unsetenv("DS4_METAL_DISABLE_IQ2_XXS_SSD_PREFILL_MM"); + ds4_gpu_set_streaming_expert_cache_budget(N_TOTAL_EXPERT); + const int default_failed = !run_once( + "default-fail-closed", &result, model, model_size, + gate_offset, up_offset, down_offset, gate_expert_bytes, + gate_row_bytes, down_expert_bytes, down_row_bytes, + N_TOTAL_EXPERT, x, selected, weights, false); + + setenv("DS4_METAL_REQUIRE_IQ2_XXS_SSD_PREFILL_MM", "0", 1); + ds4_gpu_set_streaming_expert_cache_budget(N_TOTAL_EXPERT); + const int require_zero_fallback = run_once( + "require-zero-fallback", &result, model, model_size, + gate_offset, up_offset, down_offset, gate_expert_bytes, + gate_row_bytes, down_expert_bytes, down_row_bytes, + N_TOTAL_EXPERT, x, selected, weights, false); + + unsetenv("DS4_METAL_REQUIRE_IQ2_XXS_SSD_PREFILL_MM"); + setenv("DS4_METAL_DISABLE_IQ2_XXS_SSD_PREFILL_MM", "1", 1); + ds4_gpu_set_streaming_expert_cache_budget(N_TOTAL_EXPERT); + const int disable_fallback = run_once( + "disable-fallback", &result, model, model_size, + gate_offset, up_offset, down_offset, gate_expert_bytes, + gate_row_bytes, down_expert_bytes, down_row_bytes, + N_TOTAL_EXPERT, x, selected, weights, false); + + setenv("DS4_METAL_REQUIRE_IQ2_XXS_SSD_PREFILL_MM", "1", 1); + ds4_gpu_set_streaming_expert_cache_budget(N_TOTAL_EXPERT); + const int require_disable_failed = !run_once( + "require-disable-fail", &result, model, model_size, + gate_offset, up_offset, down_offset, gate_expert_bytes, + gate_row_bytes, down_expert_bytes, down_row_bytes, + N_TOTAL_EXPERT, x, selected, weights, false); + + /* The automatic fail-closed arm must not turn a cache that cannot hold + * the complete expert domain into an error. Explicit REQUIRE remains + * strong for exactly that same materially-ineligible configuration. */ + unsetenv("DS4_METAL_REQUIRE_IQ2_XXS_SSD_PREFILL_MM"); + unsetenv("DS4_METAL_DISABLE_IQ2_XXS_SSD_PREFILL_MM"); + ds4_gpu_set_streaming_expert_cache_budget(N_TOTAL_EXPERT - 1u); + const int small_cache_fallback = run_once( + "small-cache-fallback", &result, model, model_size, + gate_offset, up_offset, down_offset, gate_expert_bytes, + gate_row_bytes, down_expert_bytes, down_row_bytes, + N_TOTAL_EXPERT, x, selected, weights, true); + + setenv("DS4_METAL_REQUIRE_IQ2_XXS_SSD_PREFILL_MM", "1", 1); + ds4_gpu_set_streaming_expert_cache_budget(N_TOTAL_EXPERT - 1u); + const int small_cache_require_failed = !run_once( + "small-cache-require-fail", &result, model, model_size, + gate_offset, up_offset, down_offset, gate_expert_bytes, + gate_row_bytes, down_expert_bytes, down_row_bytes, + N_TOTAL_EXPERT, x, selected, weights, false); + + /* A short final chunk is outside the grouped-MM candidate contract even + * with a full cache and the injected missing pipeline. It must retain the + * established path, including for the contradictory explicit controls. */ + unsetenv("DS4_METAL_REQUIRE_IQ2_XXS_SSD_PREFILL_MM"); + ds4_gpu_set_streaming_expert_cache_budget(N_TOTAL_EXPERT); + const int short_tail_fallback = run_once( + "short-tail-fallback", &tail_result, model, model_size, + gate_offset, up_offset, down_offset, gate_expert_bytes, + gate_row_bytes, down_expert_bytes, down_row_bytes, + N_TOTAL_EXPERT, x, selected, weights, false); + + setenv("DS4_METAL_REQUIRE_IQ2_XXS_SSD_PREFILL_MM", "1", 1); + setenv("DS4_METAL_DISABLE_IQ2_XXS_SSD_PREFILL_MM", "1", 1); + ds4_gpu_set_streaming_expert_cache_budget(N_TOTAL_EXPERT); + const int short_tail_conflict_fallback = run_once( + "short-tail-conflict-fallback", &tail_result, model, model_size, + gate_offset, up_offset, down_offset, gate_expert_bytes, + gate_row_bytes, down_expert_bytes, down_row_bytes, + N_TOTAL_EXPERT, x, selected, weights, false); + + ds4_gpu_test_set_flags(0); + unsetenv("DS4_METAL_ENABLE_IQ2_XXS_SSD_PREFILL_MM"); + unsetenv("DS4_METAL_REQUIRE_IQ2_XXS_SSD_PREFILL_MM"); + unsetenv("DS4_METAL_DISABLE_IQ2_XXS_SSD_PREFILL_MM"); + ok = read_mm_stats(&after) && ok; + + const int monotonic = + after.candidate_calls >= before.candidate_calls && + after.calls >= before.calls && + after.require_failures >= before.require_failures; + const uint64_t candidate_delta = monotonic ? + after.candidate_calls - before.candidate_calls : UINT64_MAX; + const uint64_t call_delta = monotonic ? + after.calls - before.calls : UINT64_MAX; + const uint64_t failure_delta = monotonic ? + after.require_failures - before.require_failures : UINT64_MAX; + const int coverage_ok = monotonic && candidate_delta == 6u && + call_delta == 0u && failure_delta == 3u; + ok = default_failed && require_zero_fallback && disable_fallback && + require_disable_failed && small_cache_fallback && + small_cache_require_failed && short_tail_fallback && + short_tail_conflict_fallback && coverage_ok && ok; + fprintf(stderr, + "IQ2 grouped-MM implicit REQUIRE integration %s " + "default_fail=%d require0_fallback=%d disable_fallback=%d " + "require_disable_fail=%d small_fallback=%d " + "small_require_fail=%d tail_fallback=%d tail_conflict=%d " + "candidates=%llu calls=%llu require_failures=%llu\n", + ok ? "PASS" : "FAIL", default_failed, require_zero_fallback, + disable_fallback, require_disable_failed, small_cache_fallback, + small_cache_require_failed, short_tail_fallback, + short_tail_conflict_fallback, + (unsigned long long)candidate_delta, + (unsigned long long)call_delta, + (unsigned long long)failure_delta); + result_free(&result); + result_free(&tail_result); + return ok; +} + static int full_result_alloc(full_run_result *r) { memset(r, 0, sizeof(*r)); r->pair_count = @@ -1186,6 +1371,8 @@ int main(void) { return 1; } + int ok = check_grouped_mm_policy(); + const uint64_t gate_row_bytes = sizeof(block_iq2_xxs); const uint64_t gate_expert_bytes = MID_DIM * gate_row_bytes; const uint64_t gate_tensor_bytes = N_TOTAL_EXPERT * gate_expert_bytes; @@ -1221,7 +1408,7 @@ int main(void) { calloc((size_t)MAX_TOKENS * N_EXPERT, sizeof(int32_t)); float *weights = calloc((size_t)MAX_TOKENS * N_EXPERT, sizeof(float)); - int ok = x && selected && selected_duplicate && selected_hot && weights; + ok = x && selected && selected_duplicate && selected_hot && weights && ok; for (uint32_t token = 0; ok && token < MAX_TOKENS; token++) { for (uint32_t k = 0; k < IN_DIM; k++) { const int32_t v = (int32_t)((token * 17u + k * 11u + @@ -1334,6 +1521,14 @@ int main(void) { min_tokens, max_tokens); ok = stats_ok && counters_ok && ok; + if (backend_ready) { + const int policy_fault_ok = check_implicit_require_fault( + model, model_size, gate_offset, up_offset, down_offset, + gate_expert_bytes, gate_row_bytes, down_expert_bytes, + down_row_bytes, x, selected, weights); + ok = policy_fault_ok && ok; + } + if (backend_ready) { uint32_t hot_rows = 0; for (uint32_t token = 0; token < 33u; token++) { From 377b1aea0641ba93717c7c78cc2e89df3a3d3e0d Mon Sep 17 00:00:00 2001 From: Giorgio Oppo Date: Mon, 17 Aug 2026 09:08:38 +0200 Subject: [PATCH 041/189] metal: skip redundant expert readahead during prefill --- README.md | 7 ++++++- ds4_metal.m | 30 ++++++++++++++++++++++++------ 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index d6ffdad4c..cad16b31a 100644 --- a/README.md +++ b/README.md @@ -1082,7 +1082,12 @@ insufficient cache. Combining `REQUIRE=1` with `DISABLE=1` fails on an eligible grouped-MM candidate, while short tail chunks retain their normal fallback. The IQ2 live cache index remains automatic for its production shape, selected-load early commit remains off unless explicitly enabled, and grouped-MM statistics -plus streaming timing summaries remain opt-in diagnostics. +plus streaming timing summaries remain opt-in diagnostics. The grouped prefill +loader also skips `F_RDADVISE` for chunks of at least 32 tokens because it +immediately reads the same expert ranges with parallel `pread`; short chunks +retain the hint. Set +`DS4_METAL_ENABLE_STREAMING_PREFILL_EXPERT_READAHEAD=1` to restore the old +hint-plus-read sequence for cold-storage A/B tests. ### Practical SSD streaming examples diff --git a/ds4_metal.m b/ds4_metal.m index 23d826690..801165a78 100644 --- a/ds4_metal.m +++ b/ds4_metal.m @@ -13170,6 +13170,22 @@ static int ds4_gpu_stream_expert_readahead_enabled(void) { getenv("DS4_METAL_DISABLE_STREAMING_EXPERT_READAHEAD") == NULL; } +static int ds4_gpu_stream_prefill_expert_readahead_enabled( + uint32_t n_tokens) { + /* + * The selected-batch prefill loader immediately follows this hint with + * parallel pread() calls for the exact same ranges. On macOS, + * F_RDADVISE is costly enough that issuing both operations serially slows + * time-to-first-token. Keep the hint for asynchronous decode loaders, + * but require an explicit opt-in for this immediate-pread path so the old + * policy remains available for cold-storage A/B tests. + */ + if (!ds4_gpu_stream_expert_readahead_enabled()) return 0; + if (n_tokens < 32u) return 1; + return ds4_gpu_env_bool( + "DS4_METAL_ENABLE_STREAMING_PREFILL_EXPERT_READAHEAD") > 0; +} + static void ds4_gpu_stream_expert_readahead_range(uint64_t offset, uint64_t len) { if (!ds4_gpu_stream_expert_readahead_enabled() || g_model_fd < 0 || len == 0) { return; @@ -18677,12 +18693,14 @@ static int ds4_gpu_stream_expert_cache_prepare_selected_batch( const int force_reuse = cache_budget != 0 && reserved_entries >= cache_budget; - ds4_gpu_stream_expert_readahead_range(unique_gate_offsets[u], - gate_expert_bytes); - ds4_gpu_stream_expert_readahead_range(unique_up_offsets[u], - gate_expert_bytes); - ds4_gpu_stream_expert_readahead_range(unique_down_offsets[u], - down_expert_bytes); + if (ds4_gpu_stream_prefill_expert_readahead_enabled(n_tokens)) { + ds4_gpu_stream_expert_readahead_range(unique_gate_offsets[u], + gate_expert_bytes); + ds4_gpu_stream_expert_readahead_range(unique_up_offsets[u], + gate_expert_bytes); + ds4_gpu_stream_expert_readahead_range(unique_down_offsets[u], + down_expert_bytes); + } const double buffer_t0 = load_timing ? ds4_gpu_now_ms() : 0.0; const int prepared = ds4_gpu_stream_expert_cache_prepare_load_buffers(layer, From f34265bc47373749db3746a4c9bd5c8b604f6162 Mon Sep 17 00:00:00 2001 From: Giorgio Oppo Date: Tue, 18 Aug 2026 22:21:53 +0200 Subject: [PATCH 042/189] rocm: add dense Q4_K projection support --- Makefile | 54 ++- ds4_rocm.cu | 2 + ds4_rocm_compat.cu | 15 +- ds4_rocm_unavailable.cu | 1 + rocm/ds4_rocm_q4.cuh | 199 +++++++++ tests/test_rocm_q4_dense_pair.cpp | 693 ++++++++++++++++++++++++++++++ 6 files changed, 957 insertions(+), 7 deletions(-) create mode 100644 rocm/ds4_rocm_q4.cuh create mode 100644 tests/test_rocm_q4_dense_pair.cpp diff --git a/Makefile b/Makefile index e918897f8..0a4e9b051 100644 --- a/Makefile +++ b/Makefile @@ -68,7 +68,7 @@ DS4_LINK_LIBS ?= $(CUDA_LDLIBS) METAL_LDLIBS := $(LDLIBS) endif -.PHONY: all help clean test test-rocm test-glm53-kda-rocm test-metal-session-batch test-metal-session-batch-ssd test-metal-q4-streams test-metal-exactn-oracle test-metal-dspark-capture test-metal-iq2-midonly test-metal-iq2-ssd-grouped-mm test-metal-iq2-live-index test-mxfp4-cuda test-mxfp4-rocm test-mmq-parity-cuda test-cuda-session-batch test-cuda-mixed-batch dspark-acceptance dspark-verify-depth rocm-dspark-acceptance rocm-dspark-verify-depth mtp-verify-depth cpu cuda cuda-spark cuda-generic cuda-regression strix-halo rocm +.PHONY: all help clean test test-rocm test-glm53-kda-rocm test-metal-session-batch test-metal-session-batch-ssd test-metal-q4-streams test-metal-exactn-oracle test-metal-dspark-capture test-metal-iq2-midonly test-metal-iq2-ssd-grouped-mm test-metal-iq2-live-index test-mxfp4-cuda test-mxfp4-rocm test-mmq-parity-cuda test-rocm-q4-parity test-rocm-q4-dense test-rocm-q4-pair test-strix-rocm-q4-parity test-cuda-session-batch test-cuda-mixed-batch dspark-acceptance dspark-verify-depth rocm-dspark-acceptance rocm-dspark-verify-depth mtp-verify-depth cpu cuda cuda-spark cuda-generic cuda-regression strix-halo rocm ifeq ($(UNAME_S),Darwin) .PHONY: metal-decode-schedule-bench metal-prefill-variant-bench check-mxfp4-half-lut test-mxfp4-metal @@ -85,6 +85,7 @@ help: @echo " make test-metal-dspark-capture Check fused DSpark HC capture bitwise" @echo " make test-metal-iq2-midonly Check M1 IQ2 addr mid-only output and sentinels" @echo " make test-metal-iq2-live-index Check IQ2 SSD live-cache index policy and fallback" + @echo " make test-rocm-q4-parity Run ROCm Q4_K dense/pair CPU-oracle tests (or SKIP without HIP)" @echo " make metal-decode-schedule-bench Build the balanced Metal decode schedule benchmark" @echo " make metal-prefill-variant-bench Build the balanced Metal prefill variant benchmark" @echo " make check-mxfp4-half-lut Verify the checked-in MXFP4 half LUT matches the generator" @@ -253,6 +254,8 @@ help: @echo " make cuda-generic Build CUDA for a generic local CUDA GPU" @echo " make cuda CUDA_ARCH=sm_N Build CUDA with an explicit nvcc -arch value" @echo " make test-mmq-parity-cuda CUDA_ARCH=sm_N Run quantized CUDA kernel parity tests" + @echo " make test-rocm-q4-parity Run ROCm Q4_K dense/pair CPU-oracle tests" + @echo " make test-strix-rocm-q4-parity Require a visible gfx1151 device and run the Q4 tests" @echo " make strix-halo Build ROCm for Strix Halo / gfx1151" @echo " make rocm Alias for make strix-halo" @echo " make test-mxfp4-rocm Build and run the synthetic ROCm MXFP4 MoE test" @@ -581,6 +584,53 @@ ds4_rocm_compat.o: ds4_rocm_compat.cu ds4_gpu.h ds4_gpu_mgpu.h ds4_gpu_args.h ds4_rocm_unavailable.o: ds4_rocm_unavailable.cu $(HIPCC) $(ROCM_CFLAGS) -c -o $@ ds4_rocm_unavailable.cu +tests/test_rocm_q4_dense_pair.o: tests/test_rocm_q4_dense_pair.cpp ds4_gpu.h + $(HIPCC) $(ROCM_CFLAGS) -DDS4_ROCM_BUILD -std=c++17 -fno-fast-math -I. -c -o $@ $< + +tests/test_rocm_q4_dense_pair: tests/test_rocm_q4_dense_pair.o ds4_rocm.o ds4_rocm_compat.o ds4_rocm_unavailable.o + $(HIPCC) $(ROCM_CFLAGS) -o $@ $^ $(ROCM_LDLIBS) + +# Keep the public test target usable on development hosts without ROCm. The +# binary itself exits 77 when HIP is installed but no device is visible; an +# explicitly required Strix run converts that condition into a hard failure. +ROCM_Q4_TEST_ARGS ?= --all +test-rocm-q4-parity: + @rocm_test_hipcc="$(strip $(HIPCC))"; \ + if [ -z "$$rocm_test_hipcc" ]; then \ + rocm_test_hipcc="$$(command -v hipcc 2>/dev/null || true)"; \ + fi; \ + rocm_test_probe="$${rocm_test_hipcc%% *}"; \ + if [ -z "$$rocm_test_probe" ] || ! command -v "$$rocm_test_probe" >/dev/null 2>&1; then \ + if [ -n "$(strip $(DS4_TEST_REQUIRE_ROCM_DEVICE))" ] && [ "$(strip $(DS4_TEST_REQUIRE_ROCM_DEVICE))" != "0" ]; then \ + echo "ROCm Q4 dense/pair oracle: FAIL (hipcc not found, device required)"; \ + exit 1; \ + fi; \ + echo "ROCm Q4 dense/pair oracle: SKIP (hipcc not found)"; exit 0; \ + fi; \ + $(MAKE) --no-print-directory tests/test_rocm_q4_dense_pair HIPCC="$$rocm_test_hipcc" || exit $$?; \ + if [ -n "$(strip $(DS4_TEST_REQUIRE_ROCM_DEVICE))" ] && [ "$(strip $(DS4_TEST_REQUIRE_ROCM_DEVICE))" != "0" ]; then \ + DS4_TEST_REQUIRE_ROCM_DEVICE="$(strip $(DS4_TEST_REQUIRE_ROCM_DEVICE))" \ + ./tests/test_rocm_q4_dense_pair $(ROCM_Q4_TEST_ARGS); \ + else \ + env -u DS4_TEST_REQUIRE_ROCM_DEVICE \ + ./tests/test_rocm_q4_dense_pair $(ROCM_Q4_TEST_ARGS); \ + fi; \ + rc=$$?; \ + if [ $$rc -eq 77 ]; then \ + echo "ROCm Q4 dense/pair oracle: SKIP (no visible HIP device)"; \ + exit 0; \ + fi; \ + exit $$rc + +test-rocm-q4-dense: + $(MAKE) --no-print-directory test-rocm-q4-parity ROCM_Q4_TEST_ARGS=--dense + +test-rocm-q4-pair: + $(MAKE) --no-print-directory test-rocm-q4-parity ROCM_Q4_TEST_ARGS=--pair + +test-strix-rocm-q4-parity: + $(MAKE) --no-print-directory -B test-rocm-q4-parity ROCM_ARCH=gfx1151 DS4_TEST_REQUIRE_ROCM_DEVICE=1 + tests/cuda_long_context_smoke: tests/cuda_long_context_smoke.o ds4_cuda.o $(MMQ_OBJS) $(NVCC) $(NVCCFLAGS) -o $@ $^ $(CUDA_LDLIBS) @@ -730,4 +780,4 @@ mxfp4-dot-test: tests/test_mxfp4_dot.c ./tests/test_mxfp4_dot clean: - rm -f ds4 ds4-server ds4-bench ds4-eval ds4-agent ds4_cpu ds4_native ds4_server_test ds4_test ds4_agent_test gguf-tools/quality-testing/score_official gguf-tools/quality-testing/score_official.o speed-bench/metal_decode_schedule_bench speed-bench/metal_prefill_variant_bench speed-bench/*.o tests/test_q4k_dot tests/test_mxfp4_dot tests/test_mxfp4_metal tests/test_mxfp4_rocm tests/test_mxfp4_cuda tests/test_metal_session_batch tests/test_metal_q4_streams tests/test_metal_exactn_oracle tests/test_metal_dspark_capture tests/test_metal_iq2_midonly tests/test_metal_iq2_ssd_grouped_mm tests/test_metal_iq2_live_index tests/test_glm53_kda tests/test_glm53_kda_rocm tests/test_glm53_vision_engine tests/test_glm53_vision_prompt tests/test_gpu_xdev tests/test_gpu_model_cache tests/test_gpu_lookup_cache_strict tests/test_engine_mgpu_refusal tests/test_engine_mgpu_runtime tests/test_engine_correctness tests/test_sampling tests/test_cuda_session_batch tests/test_cuda_mixed_batch tests/*.o *.o tests/cuda_long_context_smoke tests/cuda_long_context_smoke.o + rm -f ds4 ds4-server ds4-bench ds4-eval ds4-agent ds4_cpu ds4_native ds4_server_test ds4_test ds4_agent_test gguf-tools/quality-testing/score_official gguf-tools/quality-testing/score_official.o speed-bench/metal_decode_schedule_bench speed-bench/metal_prefill_variant_bench speed-bench/*.o tests/test_q4k_dot tests/test_mxfp4_dot tests/test_mxfp4_metal tests/test_mxfp4_rocm tests/test_mxfp4_cuda tests/test_rocm_q4_dense_pair tests/test_metal_session_batch tests/test_metal_q4_streams tests/test_metal_exactn_oracle tests/test_metal_dspark_capture tests/test_metal_iq2_midonly tests/test_metal_iq2_ssd_grouped_mm tests/test_metal_iq2_live_index tests/test_glm53_kda tests/test_glm53_kda_rocm tests/test_glm53_vision_engine tests/test_glm53_vision_prompt tests/test_gpu_xdev tests/test_gpu_model_cache tests/test_gpu_lookup_cache_strict tests/test_engine_mgpu_refusal tests/test_engine_mgpu_runtime tests/test_engine_correctness tests/test_sampling tests/test_cuda_session_batch tests/test_cuda_mixed_batch tests/*.o *.o tests/cuda_long_context_smoke tests/cuda_long_context_smoke.o diff --git a/ds4_rocm.cu b/ds4_rocm.cu index 3e7368021..996776611 100644 --- a/ds4_rocm.cu +++ b/ds4_rocm.cu @@ -162,6 +162,8 @@ extern "C" int ds4_gpu_dspark_gfx1151_fast_path(void) { #include "rocm/ds4_rocm_moe.cuh" +#include "rocm/ds4_rocm_q4.cuh" + #include "rocm/ds4_rocm_moe_launch.cuh" #include "rocm/ds4_rocm_glm.cuh" diff --git a/ds4_rocm_compat.cu b/ds4_rocm_compat.cu index 554745379..7227bb8b2 100644 --- a/ds4_rocm_compat.cu +++ b/ds4_rocm_compat.cu @@ -8,6 +8,11 @@ #include "ds4_gpu.h" #include "ds4_gpu_args.h" +extern "C" int ds4_rocm_matmul_q4_K_tensor( + ds4_gpu_tensor *out, const void *model_map, uint64_t model_size, + uint64_t weight_offset, uint64_t in_dim, uint64_t out_dim, + const ds4_gpu_tensor *x, uint64_t n_tok); + ds4_gpu_ctx g_gpu[DS4_MAX_GPUS] = {}; int g_n_gpus = 1; int g_gpu_peer_ok[DS4_MAX_GPUS][DS4_MAX_GPUS] = {{1}}; @@ -299,16 +304,16 @@ extern "C" int ds4_gpu_matmul_quant_tensor( weight_offset, in_dim, out_dim, x, n_tok); } + if (weight_type == 12u) { + return ds4_rocm_matmul_q4_K_tensor(out, model_map, model_size, + weight_offset, in_dim, out_dim, x, + n_tok); + } if (weight_type == 1u) { return ds4_gpu_matmul_f16_tensor(out, model_map, model_size, weight_offset, in_dim, out_dim, x, n_tok); } - if (weight_type == 12u) { - return ds4_gpu_matmul_q4_K_tensor(out, model_map, model_size, - weight_offset, in_dim, out_dim, x, - n_tok); - } return 0; } diff --git a/ds4_rocm_unavailable.cu b/ds4_rocm_unavailable.cu index 311c8004d..6a9937e61 100644 --- a/ds4_rocm_unavailable.cu +++ b/ds4_rocm_unavailable.cu @@ -22,6 +22,7 @@ ROCM_UNAVAILABLE_INT(ds4_gpu_matmul_q8_0_kslice_hc_expand_add_tensor) ROCM_UNAVAILABLE_INT(ds4_gpu_matmul_q8_0_kslice_rows_tensor) ROCM_UNAVAILABLE_INT(ds4_gpu_matmul_q8_0_top1_tensor) ROCM_UNAVAILABLE_INT(ds4_gpu_matmul_quant_kslice_tensor) +ROCM_UNAVAILABLE_INT(ds4_gpu_matmul_quant_rows_scalar_tensor) ROCM_UNAVAILABLE_INT(ds4_gpu_moe_handoff_pack_tensor) ROCM_UNAVAILABLE_INT(ds4_gpu_register_model_map_no_copy) ROCM_UNAVAILABLE_INT(ds4_gpu_register_support_map) diff --git a/rocm/ds4_rocm_q4.cuh b/rocm/ds4_rocm_q4.cuh new file mode 100644 index 000000000..ef11cc410 --- /dev/null +++ b/rocm/ds4_rocm_q4.cuh @@ -0,0 +1,199 @@ +// DS4 ROCm dense Q4_K kernels and launch wrappers. +// +// This module is included after ds4_rocm_moe.cuh so it can reuse the +// canonical Q8_K activation quantizer, Q4_K/Q8_K block dot product, and +// quarter-wave reduction already used by the routed-MoE implementation. +// Launches intentionally stay on ROCm's default stream: cuda_tmp_alloc() is a +// single reusable scratch arena whose lifetime is protected by that ordering. + +static_assert(sizeof(cuda_block_q4_K) == 144u, + "ROCm Q4_K block layout must match GGUF"); +static_assert(sizeof(cuda_block_q8_K) == 292u, + "ROCm Q8_K activation block layout must match the dot kernel"); + +__global__ static void rocm_matmul_q4_K_dense_kernel( + float *out, + const char *w_base, + const cuda_block_q8_K *xq, + uint64_t row_bytes, + uint32_t xq_blocks, + uint32_t out_dim, + uint32_t n_tok) { + const uint32_t lane = threadIdx.x & 7u; + const uint32_t row_lane = threadIdx.x >> 3u; + const uint32_t tok = blockIdx.y; + const uint32_t row = blockIdx.x * 32u + row_lane; + if (tok >= n_tok || row >= out_dim) return; + + const cuda_block_q8_K *xqb = xq + (uint64_t)tok * xq_blocks; + const cuda_block_q4_K *wr = reinterpret_cast( + w_base + (uint64_t)row * row_bytes); + float acc = 0.0f; + for (uint32_t b = lane; b < xq_blocks; b += 8u) { + acc += dev_dot_q4_K_q8_K_block(wr + b, xqb + b); + } + acc = quarter_warp_sum_f32(acc, lane); + if (lane == 0u) out[(uint64_t)tok * out_dim + row] = acc; +} + +static int rocm_q4_K_dense_validate( + const ds4_gpu_tensor *out, + const void *model_map, + uint64_t model_size, + uint64_t weight_offset, + uint64_t in_dim, + uint64_t out_dim, + const ds4_gpu_tensor *x, + uint64_t n_tok, + uint64_t *blocks_out, + uint64_t *row_bytes_out, + uint64_t *weight_bytes_out) { + if (!out || !x || !model_map || !blocks_out || !row_bytes_out || + !weight_bytes_out || in_dim == 0u || out_dim == 0u || n_tok == 0u || + in_dim > UINT32_MAX || out_dim > UINT32_MAX || n_tok > UINT32_MAX || + (in_dim % CUDA_QK_K) != 0u) { + return 0; + } + + const uint64_t blocks = in_dim / CUDA_QK_K; + uint64_t row_bytes = 0; + uint64_t weight_bytes = 0; + if (blocks == 0u || + !cuda_u64_mul_checked(blocks, sizeof(cuda_block_q4_K), &row_bytes) || + !cuda_u64_mul_checked(out_dim, row_bytes, &weight_bytes) || + !cuda_model_range_fits(model_size, weight_offset, weight_bytes) || + !cuda_tensor_has_elems2(x, n_tok, in_dim, sizeof(float)) || + !cuda_tensor_has_elems2(out, n_tok, out_dim, sizeof(float))) { + return 0; + } + + *blocks_out = blocks; + *row_bytes_out = row_bytes; + *weight_bytes_out = weight_bytes; + return 1; +} + +static cuda_block_q8_K *rocm_q4_K_prequant_alloc( + uint64_t n_tok, + uint64_t blocks, + const char *what) { + uint64_t bytes = 0; + if (!cuda_u64_mul3_checked(n_tok, blocks, + sizeof(cuda_block_q8_K), &bytes)) { + return NULL; + } + return reinterpret_cast(cuda_tmp_alloc(bytes, what)); +} + +static int rocm_q4_K_dense_pair_requested(void) { + return getenv("DS4_ROCM_ENABLE_Q4_DENSE_PAIR") != NULL && + getenv("DS4_ROCM_DISABLE_Q4_DENSE_PAIR") == NULL; +} + +extern "C" int ds4_rocm_matmul_q4_K_tensor( + ds4_gpu_tensor *out, + const void *model_map, + uint64_t model_size, + uint64_t weight_offset, + uint64_t in_dim, + uint64_t out_dim, + const ds4_gpu_tensor *x, + uint64_t n_tok) { + uint64_t blocks = 0; + uint64_t row_bytes = 0; + uint64_t weight_bytes = 0; + if (!rocm_q4_K_dense_validate(out, model_map, model_size, weight_offset, + in_dim, out_dim, x, n_tok, &blocks, + &row_bytes, &weight_bytes)) { + return 0; + } + + const char *wptr = cuda_model_range_ptr(model_map, weight_offset, + weight_bytes, "q4_K dense"); + if (!wptr) return 0; + cuda_block_q8_K *xq = rocm_q4_K_prequant_alloc( + n_tok, blocks, "q4_K dense prequant"); + if (!xq) return 0; + + const dim3 qgrid((unsigned)blocks, (unsigned)n_tok, 1u); + q8_K_quantize_kernel<<>>( + xq, reinterpret_cast(x->ptr), + (uint32_t)in_dim, (uint32_t)n_tok); + if (!cuda_ok(cudaGetLastError(), "q4_K dense quantize launch")) return 0; + + const dim3 grid((unsigned)((out_dim - 1u) / 32u + 1u), + (unsigned)n_tok, 1u); + rocm_matmul_q4_K_dense_kernel<<>>( + reinterpret_cast(out->ptr), wptr, xq, row_bytes, + (uint32_t)blocks, (uint32_t)out_dim, (uint32_t)n_tok); + return cuda_ok(cudaGetLastError(), "q4_K dense matmul launch"); +} + +extern "C" int ds4_gpu_matmul_q4_K_pair_tensor( + ds4_gpu_tensor *out0, + ds4_gpu_tensor *out1, + const void *model_map, + uint64_t model_size, + uint64_t weight0_offset, + uint64_t weight1_offset, + uint64_t in_dim, + uint64_t out0_dim, + uint64_t out1_dim, + const ds4_gpu_tensor *x, + uint64_t n_tok) { + // Keep this optional hook scoped to decode/speculative micro-batches. Larger + // prefill batches fall back to the two required dense primitives. + if (!rocm_q4_K_dense_pair_requested() || + n_tok > 8u || !out0 || !out1 || out0 == out1 || + (out0 && out1 && out0->ptr == out1->ptr)) { + return 0; + } + + uint64_t blocks0 = 0, blocks1 = 0; + uint64_t row_bytes0 = 0, row_bytes1 = 0; + uint64_t weight0_bytes = 0, weight1_bytes = 0; + if (!rocm_q4_K_dense_validate(out0, model_map, model_size, weight0_offset, + in_dim, out0_dim, x, n_tok, &blocks0, + &row_bytes0, &weight0_bytes) || + !rocm_q4_K_dense_validate(out1, model_map, model_size, weight1_offset, + in_dim, out1_dim, x, n_tok, &blocks1, + &row_bytes1, &weight1_bytes) || + blocks0 != blocks1 || row_bytes0 != row_bytes1) { + return 0; + } + + const char *w0 = cuda_model_range_ptr(model_map, weight0_offset, + weight0_bytes, "q4_K dense pair0"); + const char *w1 = cuda_model_range_ptr(model_map, weight1_offset, + weight1_bytes, "q4_K dense pair1"); + if (!w0 || !w1) return 0; + cuda_block_q8_K *xq = rocm_q4_K_prequant_alloc( + n_tok, blocks0, "q4_K dense pair prequant"); + if (!xq) return 0; + + const dim3 qgrid((unsigned)blocks0, (unsigned)n_tok, 1u); + q8_K_quantize_kernel<<>>( + xq, reinterpret_cast(x->ptr), + (uint32_t)in_dim, (uint32_t)n_tok); + if (!cuda_ok(cudaGetLastError(), "q4_K dense pair quantize launch")) { + return 0; + } + + // Use the exact standalone kernel twice. This preserves its block walk and + // reduction path while still eliminating the second Q8_K quantization. + const dim3 grid0((unsigned)((out0_dim - 1u) / 32u + 1u), + (unsigned)n_tok, 1u); + rocm_matmul_q4_K_dense_kernel<<>>( + reinterpret_cast(out0->ptr), w0, xq, row_bytes0, + (uint32_t)blocks0, (uint32_t)out0_dim, (uint32_t)n_tok); + if (!cuda_ok(cudaGetLastError(), "q4_K dense pair0 matmul launch")) { + return 0; + } + + const dim3 grid1((unsigned)((out1_dim - 1u) / 32u + 1u), + (unsigned)n_tok, 1u); + rocm_matmul_q4_K_dense_kernel<<>>( + reinterpret_cast(out1->ptr), w1, xq, row_bytes1, + (uint32_t)blocks1, (uint32_t)out1_dim, (uint32_t)n_tok); + return cuda_ok(cudaGetLastError(), "q4_K dense pair1 matmul launch"); +} diff --git a/tests/test_rocm_q4_dense_pair.cpp b/tests/test_rocm_q4_dense_pair.cpp new file mode 100644 index 000000000..c379059c5 --- /dev/null +++ b/tests/test_rocm_q4_dense_pair.cpp @@ -0,0 +1,693 @@ +// SPDX-License-Identifier: MIT +// Deterministic ROCm Q4_K dense/pair oracle. +// +// The test deliberately goes through the public tensor/model-map API. Weight +// rows use the raw 144-byte GGUF Q4_K layout, while the CPU reference mirrors +// the backend's F32 -> Q8_K quantizer and Q4_K x Q8_K integer dot product. + +#include "ds4_gpu.h" + +#if defined(__has_include) +# if __has_include() +# include +# define DS4_TEST_HAS_HIP_RUNTIME 1 +# endif +#endif +#ifndef DS4_TEST_HAS_HIP_RUNTIME +# define DS4_TEST_HAS_HIP_RUNTIME 0 +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +constexpr uint32_t kQkK = 256u; +constexpr uint32_t kK = 4096u; +constexpr uint32_t kM0 = 65u; +constexpr uint32_t kM1 = 33u; +constexpr uint32_t kQ4Type = 12u; +constexpr float kCpuAbsTolerance = 2.0e-3f; +constexpr float kCpuRelTolerance = 3.0e-5f; +constexpr int kSkip = 77; + +struct block_q4_K_test { + uint16_t d; + uint16_t dmin; + uint8_t scales[12]; + uint8_t qs[kQkK / 2u]; +}; + +struct block_q8_K_test { + float d; + int8_t qs[kQkK]; + int16_t bsums[kQkK / 16u]; +}; + +static_assert(sizeof(block_q4_K_test) == 144u, + "Q4_K fixture must match the raw GGUF layout"); +static_assert(sizeof(block_q8_K_test) == 292u, + "Q8_K oracle must match the ROCm activation layout"); + +struct tensor_owner { + ds4_gpu_tensor *ptr = nullptr; + + explicit tensor_owner(uint64_t bytes) : ptr(ds4_gpu_tensor_alloc(bytes)) {} + ~tensor_owner() { ds4_gpu_tensor_free(ptr); } + + tensor_owner(const tensor_owner &) = delete; + tensor_owner &operator=(const tensor_owner &) = delete; +}; + +struct aligned_model { + uint8_t *data = nullptr; + uint64_t size = 0; + uint64_t weight0_offset = 0; + uint64_t weight1_offset = 0; + + ~aligned_model() { std::free(data); } + + aligned_model(const aligned_model &) = delete; + aligned_model &operator=(const aligned_model &) = delete; + aligned_model() = default; +}; + +struct env_snapshot { + const char *name; + bool was_set; + std::string value; + + explicit env_snapshot(const char *key) + : name(key), was_set(std::getenv(key) != nullptr), + value(was_set ? std::getenv(key) : "") {} + ~env_snapshot() { + if (was_set) { + (void)setenv(name, value.c_str(), 1); + } else { + (void)unsetenv(name); + } + } + + env_snapshot(const env_snapshot &) = delete; + env_snapshot &operator=(const env_snapshot &) = delete; +}; + +uint64_t round_up(uint64_t value, uint64_t alignment) { + return (value + alignment - 1u) / alignment * alignment; +} + +uint32_t lcg_next(uint32_t &state) { + state = state * 1664525u + 1013904223u; + return state; +} + +float fp16_to_float(uint16_t h) { + const uint32_t sign = (uint32_t)(h & 0x8000u) << 16u; + uint32_t exp = (h >> 10u) & 0x1fu; + uint32_t mant = h & 0x3ffu; + uint32_t bits; + if (exp == 0u) { + if (mant == 0u) { + bits = sign; + } else { + int shift = 0; + while ((mant & 0x400u) == 0u) { + mant <<= 1u; + shift++; + } + mant &= 0x3ffu; + bits = sign | (uint32_t)(127 - 14 - shift) << 23u | mant << 13u; + } + } else if (exp == 31u) { + bits = sign | 0x7f800000u | mant << 13u; + } else { + bits = sign | (exp + 112u) << 23u | mant << 13u; + } + float out; + std::memcpy(&out, &bits, sizeof(out)); + return out; +} + +uint16_t float_to_fp16(float value) { + uint32_t bits; + std::memcpy(&bits, &value, sizeof(bits)); + const uint32_t sign = bits >> 31u; + int32_t exp = (int32_t)((bits >> 23u) & 0xffu) - 127 + 15; + uint32_t mant = bits & 0x7fffffu; + if (exp >= 31) return (uint16_t)((sign << 15u) | 0x7c00u); + if (exp <= 0) { + if (exp < -10) return (uint16_t)(sign << 15u); + mant |= 0x800000u; + const uint32_t shift = (uint32_t)(14 - exp); + uint32_t rounded = mant >> shift; + const uint32_t halfway = 1u << (shift - 1u); + if ((mant & halfway) && + ((mant & (halfway - 1u)) || (rounded & 1u))) { + rounded++; + } + return (uint16_t)((sign << 15u) | rounded); + } + uint32_t rounded = mant + 0x0fffu + ((mant >> 13u) & 1u); + if (rounded & 0x800000u) { + rounded = 0u; + exp++; + if (exp >= 31) return (uint16_t)((sign << 15u) | 0x7c00u); + } + return (uint16_t)((sign << 15u) | (uint32_t)exp << 10u | + rounded >> 13u); +} + +void q4_scale_min(uint32_t j, const uint8_t *scales, + uint8_t *scale, uint8_t *minimum) { + if (j < 4u) { + *scale = scales[j] & 63u; + *minimum = scales[j + 4u] & 63u; + } else { + *scale = (scales[j + 4u] & 0x0fu) | + (uint8_t)((scales[j - 4u] >> 6u) << 4u); + *minimum = (scales[j + 4u] >> 4u) | + (uint8_t)((scales[j] >> 6u) << 4u); + } +} + +void fill_q4_rows(block_q4_K_test *rows, uint32_t n_rows, uint32_t seed) { + uint32_t state = seed; + const uint32_t blocks_per_row = kK / kQkK; + for (uint32_t row = 0; row < n_rows; row++) { + for (uint32_t b = 0; b < blocks_per_row; b++) { + block_q4_K_test &block = rows[(uint64_t)row * blocks_per_row + b]; + const float d = 0.0025f + + 0.00025f * (float)(1u + (lcg_next(state) % 23u)); + const float dmin = 0.0010f + + 0.00020f * (float)(1u + (lcg_next(state) % 19u)); + block.d = float_to_fp16(d); + block.dmin = float_to_fp16(dmin); + for (uint8_t &v : block.scales) v = (uint8_t)(lcg_next(state) >> 24u); + for (uint8_t &v : block.qs) v = (uint8_t)(lcg_next(state) >> 24u); + } + } +} + +bool make_model(aligned_model *model) { + constexpr uint64_t page = 4096u; + const uint64_t row_bytes = (kK / kQkK) * sizeof(block_q4_K_test); + const uint64_t weight0_bytes = kM0 * row_bytes; + const uint64_t weight1_bytes = kM1 * row_bytes; + model->weight0_offset = 0u; + model->weight1_offset = round_up(weight0_bytes, page); + model->size = round_up(model->weight1_offset + weight1_bytes, page); + void *storage = nullptr; + if (posix_memalign(&storage, (size_t)page, (size_t)model->size) != 0) { + return false; + } + model->data = static_cast(storage); + std::memset(model->data, 0xa5, (size_t)model->size); + fill_q4_rows(reinterpret_cast( + model->data + model->weight0_offset), + kM0, 0x41c64e6du); + fill_q4_rows(reinterpret_cast( + model->data + model->weight1_offset), + kM1, 0x9e3779b9u); + return true; +} + +void fill_activation(std::vector *x, uint32_t n_tokens) { + x->resize((uint64_t)n_tokens * kK); + for (uint32_t token = 0; token < n_tokens; token++) { + for (uint32_t b = 0; b < kK / kQkK; b++) { + float *block = x->data() + (uint64_t)token * kK + b * kQkK; + for (uint32_t i = 0; i < kQkK; i++) { + const int q = (int)((i * 73u + token * 37u + b * 19u) % 241u) - 120; + block[i] = (float)q / 32.0f; + } + // A unique, exactly representable maximum makes the CPU and GPU + // quantizers select the same signed scale without tie ambiguity. + block[0] = ((token + b) & 1u) ? 127.0f / 32.0f + : -127.0f / 32.0f; + } + } +} + +void quantize_q8_K_cpu(const float *x, block_q8_K_test *out) { + float amax = 0.0f; + float maxv = 0.0f; + for (uint32_t i = 0; i < kQkK; i++) { + const float av = std::fabs(x[i]); + if (av > amax) { + amax = av; + maxv = x[i]; + } + } + if (amax == 0.0f) { + std::memset(out, 0, sizeof(*out)); + return; + } + const float iscale = -127.0f / maxv; + for (uint32_t i = 0; i < kQkK; i++) { + int q = (int)std::lrint(iscale * x[i]); + q = std::max(-128, std::min(127, q)); + out->qs[i] = (int8_t)q; + } + for (uint32_t group = 0; group < kQkK / 16u; group++) { + int sum = 0; + for (uint32_t i = 0; i < 16u; i++) { + sum += out->qs[group * 16u + i]; + } + out->bsums[group] = (int16_t)sum; + } + out->d = 1.0f / iscale; +} + +float dot_q4_q8_raw(const block_q4_K_test &weight, + const block_q8_K_test &activation) { + int isum = 0; + int summs = 0; + for (uint32_t j = 0; j < 8u; j++) { + uint8_t scale = 0; + uint8_t minimum = 0; + q4_scale_min(j, weight.scales, &scale, &minimum); + summs += (int)minimum * + ((int)activation.bsums[2u * j] + + (int)activation.bsums[2u * j + 1u]); + const uint32_t byte_offset = (j >> 1u) * 32u; + const uint32_t shift = (j & 1u) ? 4u : 0u; + int group_dot = 0; + for (uint32_t i = 0; i < 32u; i++) { + const int q4 = (weight.qs[byte_offset + i] >> shift) & 0x0f; + group_dot += q4 * (int)activation.qs[j * 32u + i]; + } + isum += (int)scale * group_dot; + } + const float d = fp16_to_float(weight.d); + const float dmin = fp16_to_float(weight.dmin); + return activation.d * d * (float)isum - + activation.d * dmin * (float)summs; +} + +std::vector dense_reference(const uint8_t *weight_base, + const std::vector &x, + uint32_t out_dim, + uint32_t n_tokens) { + const auto *weights = reinterpret_cast(weight_base); + constexpr uint32_t blocks_per_row = kK / kQkK; + std::vector xq((uint64_t)n_tokens * blocks_per_row); + for (uint32_t token = 0; token < n_tokens; token++) { + for (uint32_t b = 0; b < blocks_per_row; b++) { + quantize_q8_K_cpu(x.data() + (uint64_t)token * kK + b * kQkK, + &xq[(uint64_t)token * blocks_per_row + b]); + } + } + std::vector result((uint64_t)n_tokens * out_dim, 0.0f); + for (uint32_t token = 0; token < n_tokens; token++) { + for (uint32_t row = 0; row < out_dim; row++) { + // Mirror the kernel's b=lane; b+=8 walk and width-8 shuffle tree. + // This makes raw-bit diagnostics meaningful even at the real + // K=4096 shape while the tolerance remains the promotion gate. + float lane_sum[8] = {}; + for (uint32_t lane = 0; lane < 8u; lane++) { + for (uint32_t b = lane; b < blocks_per_row; b += 8u) { + lane_sum[lane] += dot_q4_q8_raw( + weights[(uint64_t)row * blocks_per_row + b], + xq[(uint64_t)token * blocks_per_row + b]); + } + } + for (uint32_t offset = 4u; offset > 0u; offset >>= 1u) { + for (uint32_t lane = 0; lane + offset < 8u; lane++) { + lane_sum[lane] += lane_sum[lane + offset]; + } + } + result[(uint64_t)token * out_dim + row] = lane_sum[0]; + } + } + return result; +} + +bool close_to_cpu(const std::vector &got, + const std::vector &expected, + const char *label) { + uint64_t raw_mismatches = 0; + uint64_t tolerance_failures = 0; + float max_abs = 0.0f; + float max_rel = 0.0f; + size_t worst = 0; + for (size_t i = 0; i < got.size(); i++) { + if (std::memcmp(&got[i], &expected[i], sizeof(float)) != 0) { + raw_mismatches++; + } + const float diff = std::fabs(got[i] - expected[i]); + const float rel = diff / std::max(1.0f, std::fabs(expected[i])); + if (diff > max_abs) { + max_abs = diff; + worst = i; + } + max_rel = std::max(max_rel, rel); + const float limit = kCpuAbsTolerance + + kCpuRelTolerance * std::fabs(expected[i]); + if (!std::isfinite(got[i]) || diff > limit) tolerance_failures++; + } + std::fprintf(stderr, + "%s: raw_mismatches=%llu/%zu max_abs=%g max_rel=%g " + "worst=%zu tolerance(abs=%g rel=%g) %s\n", + label, (unsigned long long)raw_mismatches, got.size(), + max_abs, max_rel, worst, kCpuAbsTolerance, + kCpuRelTolerance, tolerance_failures == 0 ? "PASS" : "FAIL"); + if (tolerance_failures != 0 && worst < got.size()) { + std::fprintf(stderr, " worst got=%g cpu=%g delta=%g\n", + got[worst], expected[worst], got[worst] - expected[worst]); + } + return tolerance_failures == 0; +} + +bool bitwise_equal(const std::vector &got, + const std::vector &expected, + const char *label) { + uint64_t mismatches = 0; + size_t first = 0; + for (size_t i = 0; i < got.size(); i++) { + if (std::memcmp(&got[i], &expected[i], sizeof(float)) != 0) { + if (mismatches == 0) first = i; + mismatches++; + } + } + std::fprintf(stderr, "%s: raw_mismatches=%llu/%zu %s\n", + label, (unsigned long long)mismatches, got.size(), + mismatches == 0 ? "PASS" : "FAIL"); + if (mismatches != 0) { + uint32_t got_bits = 0; + uint32_t expected_bits = 0; + std::memcpy(&got_bits, &got[first], sizeof(got_bits)); + std::memcpy(&expected_bits, &expected[first], sizeof(expected_bits)); + std::fprintf(stderr, + " first=%zu got=%g/0x%08x expected=%g/0x%08x\n", + first, got[first], got_bits, expected[first], expected_bits); + } + return mismatches == 0; +} + +bool write_tensor(ds4_gpu_tensor *tensor, const std::vector &values) { + return tensor && ds4_gpu_tensor_write( + tensor, 0, values.data(), values.size() * sizeof(float)) != 0; +} + +bool read_tensor(const ds4_gpu_tensor *tensor, std::vector *values) { + return tensor && ds4_gpu_tensor_read( + tensor, 0, values->data(), values->size() * sizeof(float)) != 0; +} + +std::vector sentinel_values(size_t count) { + std::vector result(count); + for (size_t i = 0; i < count; i++) { + const uint32_t bits = 0x4b000000u + (uint32_t)i; + std::memcpy(&result[i], &bits, sizeof(bits)); + } + return result; +} + +bool run_dense_case(const aligned_model &model, uint32_t n_tokens, + uint64_t offset, uint32_t out_dim, const char *label) { + std::vector x; + fill_activation(&x, n_tokens); + tensor_owner x_gpu(x.size() * sizeof(float)); + tensor_owner out_gpu((uint64_t)n_tokens * out_dim * sizeof(float)); + if (!x_gpu.ptr || !out_gpu.ptr || !write_tensor(x_gpu.ptr, x)) { + std::fprintf(stderr, "%s: tensor allocation/write FAIL\n", label); + return false; + } + const int rc = ds4_gpu_matmul_quant_tensor( + out_gpu.ptr, model.data, model.size, offset, kQ4Type, + kK, out_dim, x_gpu.ptr, n_tokens); + std::vector got((uint64_t)n_tokens * out_dim); + if (rc == 0 || !read_tensor(out_gpu.ptr, &got)) { + std::fprintf(stderr, "%s: dense dispatch rc=%d FAIL\n", label, rc); + return false; + } + const std::vector cpu = dense_reference( + model.data + offset, x, out_dim, n_tokens); + return close_to_cpu(got, cpu, label); +} + +bool run_pair_case(const aligned_model &model, uint32_t n_tokens, + const char *label) { + std::vector x; + fill_activation(&x, n_tokens); + tensor_owner x_gpu(x.size() * sizeof(float)); + tensor_owner dense0((uint64_t)n_tokens * kM0 * sizeof(float)); + tensor_owner dense1((uint64_t)n_tokens * kM1 * sizeof(float)); + tensor_owner pair0((uint64_t)n_tokens * kM0 * sizeof(float)); + tensor_owner pair1((uint64_t)n_tokens * kM1 * sizeof(float)); + if (!x_gpu.ptr || !dense0.ptr || !dense1.ptr || !pair0.ptr || !pair1.ptr || + !write_tensor(x_gpu.ptr, x)) { + std::fprintf(stderr, "%s: tensor allocation/write FAIL\n", label); + return false; + } + const int dense_rc0 = ds4_gpu_matmul_quant_tensor( + dense0.ptr, model.data, model.size, model.weight0_offset, kQ4Type, + kK, kM0, x_gpu.ptr, n_tokens); + const int dense_rc1 = ds4_gpu_matmul_quant_tensor( + dense1.ptr, model.data, model.size, model.weight1_offset, kQ4Type, + kK, kM1, x_gpu.ptr, n_tokens); + const int pair_rc = ds4_gpu_matmul_q4_K_pair_tensor( + pair0.ptr, pair1.ptr, model.data, model.size, + model.weight0_offset, model.weight1_offset, + kK, kM0, kM1, x_gpu.ptr, n_tokens); + std::vector dense0_host((uint64_t)n_tokens * kM0); + std::vector dense1_host((uint64_t)n_tokens * kM1); + std::vector pair0_host(dense0_host.size()); + std::vector pair1_host(dense1_host.size()); + if (dense_rc0 == 0 || dense_rc1 == 0 || pair_rc == 0 || + !read_tensor(dense0.ptr, &dense0_host) || + !read_tensor(dense1.ptr, &dense1_host) || + !read_tensor(pair0.ptr, &pair0_host) || + !read_tensor(pair1.ptr, &pair1_host)) { + std::fprintf(stderr, + "%s: dispatch/read dense=(%d,%d) pair=%d FAIL\n", + label, dense_rc0, dense_rc1, pair_rc); + return false; + } + const std::vector cpu0 = dense_reference( + model.data + model.weight0_offset, x, kM0, n_tokens); + const std::vector cpu1 = dense_reference( + model.data + model.weight1_offset, x, kM1, n_tokens); + bool ok = close_to_cpu(dense0_host, cpu0, "pair control dense0 vs CPU"); + ok = close_to_cpu(dense1_host, cpu1, "pair control dense1 vs CPU") && ok; + ok = bitwise_equal(pair0_host, dense0_host, "pair0 vs standalone dense0") && ok; + ok = bitwise_equal(pair1_host, dense1_host, "pair1 vs standalone dense1") && ok; + std::fprintf(stderr, "%s: %s\n", label, ok ? "PASS" : "FAIL"); + return ok; +} + +bool unchanged_after_rejected_call(ds4_gpu_tensor *tensor, + const std::vector &sentinel, + const char *label) { + std::vector after(sentinel.size()); + if (!read_tensor(tensor, &after)) { + std::fprintf(stderr, "%s: readback FAIL\n", label); + return false; + } + return bitwise_equal(after, sentinel, label); +} + +bool run_dense_guards(const aligned_model &model) { + std::vector x; + fill_activation(&x, 1u); + tensor_owner x_gpu(x.size() * sizeof(float)); + tensor_owner out_gpu((uint64_t)kM0 * sizeof(float)); + const std::vector sentinel = sentinel_values(kM0); + if (!x_gpu.ptr || !out_gpu.ptr || !write_tensor(x_gpu.ptr, x) || + !write_tensor(out_gpu.ptr, sentinel)) { + std::fprintf(stderr, "dense guards: setup FAIL\n"); + return false; + } + const int bad_k_rc = ds4_gpu_matmul_quant_tensor( + out_gpu.ptr, model.data, model.size, model.weight0_offset, kQ4Type, + kK - 1u, kM0, x_gpu.ptr, 1u); + bool ok = bad_k_rc == 0 && unchanged_after_rejected_call( + out_gpu.ptr, sentinel, "dense K%256 guard preserves output"); + if (bad_k_rc != 0) { + std::fprintf(stderr, "dense K%%256 guard: expected rc=0 got=%d FAIL\n", + bad_k_rc); + } + if (!write_tensor(out_gpu.ptr, sentinel)) return false; + const int range_rc = ds4_gpu_matmul_quant_tensor( + out_gpu.ptr, model.data, model.size, model.size - 16u, kQ4Type, + kK, kM0, x_gpu.ptr, 1u); + ok = (range_rc == 0) && unchanged_after_rejected_call( + out_gpu.ptr, sentinel, "dense model-range guard preserves output") && ok; + if (range_rc != 0) { + std::fprintf(stderr, "dense model-range guard: expected rc=0 got=%d FAIL\n", + range_rc); + } + return ok; +} + +bool run_pair_guards(const aligned_model &model) { + constexpr uint32_t n_tokens = 9u; + std::vector x; + fill_activation(&x, n_tokens); + tensor_owner x_gpu(x.size() * sizeof(float)); + tensor_owner out0((uint64_t)n_tokens * kM0 * sizeof(float)); + tensor_owner out1((uint64_t)n_tokens * kM1 * sizeof(float)); + const std::vector sentinel0 = sentinel_values((uint64_t)n_tokens * kM0); + const std::vector sentinel1 = sentinel_values((uint64_t)n_tokens * kM1); + if (!x_gpu.ptr || !out0.ptr || !out1.ptr || !write_tensor(x_gpu.ptr, x) || + !write_tensor(out0.ptr, sentinel0) || !write_tensor(out1.ptr, sentinel1)) { + std::fprintf(stderr, "pair guards: setup FAIL\n"); + return false; + } + const int rc = ds4_gpu_matmul_q4_K_pair_tensor( + out0.ptr, out1.ptr, model.data, model.size, + model.weight0_offset, model.weight1_offset, + kK, kM0, kM1, x_gpu.ptr, n_tokens); + bool ok = rc == 0; + if (rc != 0) { + std::fprintf(stderr, "pair n_tok=9 guard: expected rc=0 got=%d FAIL\n", rc); + } + ok = unchanged_after_rejected_call( + out0.ptr, sentinel0, "pair n_tok=9 preserves out0") && ok; + ok = unchanged_after_rejected_call( + out1.ptr, sentinel1, "pair n_tok=9 preserves out1") && ok; + return ok; +} + +bool run_pair_opt_in_guards(const aligned_model &model) { + constexpr uint32_t n_tokens = 1u; + std::vector x; + fill_activation(&x, n_tokens); + tensor_owner x_gpu(x.size() * sizeof(float)); + tensor_owner out0((uint64_t)n_tokens * kM0 * sizeof(float)); + tensor_owner out1((uint64_t)n_tokens * kM1 * sizeof(float)); + const std::vector sentinel0 = sentinel_values(kM0); + const std::vector sentinel1 = sentinel_values(kM1); + if (!x_gpu.ptr || !out0.ptr || !out1.ptr || !write_tensor(x_gpu.ptr, x)) { + std::fprintf(stderr, "pair opt-in guards: setup FAIL\n"); + return false; + } + + auto rejected_call = [&](const char *label) { + if (!write_tensor(out0.ptr, sentinel0) || + !write_tensor(out1.ptr, sentinel1)) { + return false; + } + const int rc = ds4_gpu_matmul_q4_K_pair_tensor( + out0.ptr, out1.ptr, model.data, model.size, + model.weight0_offset, model.weight1_offset, + kK, kM0, kM1, x_gpu.ptr, n_tokens); + bool guard_ok = rc == 0; + if (rc != 0) { + std::fprintf(stderr, "%s: expected rc=0 got=%d FAIL\n", label, rc); + } + guard_ok = unchanged_after_rejected_call(out0.ptr, sentinel0, label) && + guard_ok; + guard_ok = unchanged_after_rejected_call(out1.ptr, sentinel1, label) && + guard_ok; + return guard_ok; + }; + + env_snapshot enable("DS4_ROCM_ENABLE_Q4_DENSE_PAIR"); + env_snapshot disable("DS4_ROCM_DISABLE_Q4_DENSE_PAIR"); + (void)unsetenv("DS4_ROCM_ENABLE_Q4_DENSE_PAIR"); + (void)unsetenv("DS4_ROCM_DISABLE_Q4_DENSE_PAIR"); + bool ok = rejected_call("pair disabled-by-default preserves outputs"); + (void)setenv("DS4_ROCM_ENABLE_Q4_DENSE_PAIR", "1", 1); + (void)setenv("DS4_ROCM_DISABLE_Q4_DENSE_PAIR", "1", 1); + ok = rejected_call("pair DISABLE dominates ENABLE") && ok; + return ok; +} + +int detect_rocm_device() { +#if DS4_TEST_HAS_HIP_RUNTIME + int count = 0; + const hipError_t err = hipGetDeviceCount(&count); + if (err != hipSuccess || count <= 0) { + std::fprintf(stderr, + "ROCm Q4 dense/pair: SKIP (HIP runtime has no visible device: %s)\n", + err == hipSuccess ? "device count is zero" : hipGetErrorString(err)); + return 0; + } + return count; +#else + std::fprintf(stderr, + "ROCm Q4 dense/pair: SKIP (compiled without HIP runtime headers)\n"); + return 0; +#endif +} + +} // namespace + +int main(int argc, char **argv) { + bool run_dense = true; + bool run_pair = true; + if (argc == 2 && std::strcmp(argv[1], "--dense") == 0) { + run_pair = false; + } else if (argc == 2 && std::strcmp(argv[1], "--pair") == 0) { + run_dense = false; + } else if (argc > 1 && + !(argc == 2 && std::strcmp(argv[1], "--all") == 0)) { + std::fprintf(stderr, "usage: %s [--all|--dense|--pair]\n", argv[0]); + return 2; + } + + if (detect_rocm_device() <= 0) { + const char *require_device = + std::getenv("DS4_TEST_REQUIRE_ROCM_DEVICE"); + const bool required = require_device && require_device[0] != '\0' && + std::strcmp(require_device, "0") != 0; + return required ? 1 : kSkip; + } + if (!ds4_gpu_init()) { + std::fprintf(stderr, + "ROCm Q4 dense/pair: FAIL (device is visible but ds4_gpu_init failed)\n"); + return 1; + } + + aligned_model model; + bool ok = make_model(&model); + if (!ok) { + std::fprintf(stderr, "ROCm Q4 dense/pair: fixture allocation FAIL\n"); + } else if (!ds4_gpu_set_model_map(model.data, model.size)) { + std::fprintf(stderr, "ROCm Q4 dense/pair: model-map registration FAIL\n"); + ok = false; + } + + const bool model_ready = ok; + if (model_ready && run_dense) { + std::fprintf(stderr, "ROCm Q4 dense oracle (raw GGUF Q4_K x Q8_K):\n"); + ok = run_dense_case(model, 1u, model.weight0_offset, kM0, + "dense n_tok=1") && ok; + ok = run_dense_case(model, 3u, model.weight1_offset, kM1, + "dense n_tok=3") && ok; + ok = run_dense_case(model, 9u, model.weight1_offset, kM1, + "dense n_tok=9") && ok; + ok = run_dense_case(model, 128u, model.weight1_offset, kM1, + "dense n_tok=128") && ok; + ok = run_dense_guards(model) && ok; + } + if (model_ready && run_pair) { + std::fprintf(stderr, "ROCm Q4 pair parity (pair vs two dense):\n"); + env_snapshot enable("DS4_ROCM_ENABLE_Q4_DENSE_PAIR"); + env_snapshot disable("DS4_ROCM_DISABLE_Q4_DENSE_PAIR"); + (void)setenv("DS4_ROCM_ENABLE_Q4_DENSE_PAIR", "1", 1); + (void)unsetenv("DS4_ROCM_DISABLE_Q4_DENSE_PAIR"); + const bool pair1_ok = run_pair_case(model, 1u, "pair n_tok=1"); + const bool pair3_ok = run_pair_case(model, 3u, "pair n_tok=3"); + const bool pair8_ok = run_pair_case(model, 8u, "pair n_tok=8"); + const bool pair_guard_ok = run_pair_guards(model); + const bool pair_opt_in_ok = run_pair_opt_in_guards(model); + ok = pair1_ok && pair3_ok && pair8_ok && pair_guard_ok && + pair_opt_in_ok && ok; + } + + // Registered host ranges must be released before their aligned backing + // allocation is destroyed. + ds4_gpu_cleanup(); + std::fprintf(stderr, "ROCm Q4 dense/pair oracle: %s\n", + ok ? "PASS" : "FAIL"); + return ok ? 0 : 1; +} From 28e6a6eb5bf51464985ab2be543a63eaf8a75add Mon Sep 17 00:00:00 2001 From: Giorgio Oppo Date: Thu, 20 Aug 2026 13:30:05 +0200 Subject: [PATCH 043/189] rocm: add tiled Q4 prefill path --- Makefile | 25 +- ds4_rocm_unavailable.cu | 1 - rocm/ds4_rocm_q4.cuh | 498 +++++++++++++++++++++- tests/test_rocm_q4_dense_pair.cpp | 659 +++++++++++++++++++++++++++++- 4 files changed, 1151 insertions(+), 32 deletions(-) diff --git a/Makefile b/Makefile index 0a4e9b051..461b31b10 100644 --- a/Makefile +++ b/Makefile @@ -68,7 +68,7 @@ DS4_LINK_LIBS ?= $(CUDA_LDLIBS) METAL_LDLIBS := $(LDLIBS) endif -.PHONY: all help clean test test-rocm test-glm53-kda-rocm test-metal-session-batch test-metal-session-batch-ssd test-metal-q4-streams test-metal-exactn-oracle test-metal-dspark-capture test-metal-iq2-midonly test-metal-iq2-ssd-grouped-mm test-metal-iq2-live-index test-mxfp4-cuda test-mxfp4-rocm test-mmq-parity-cuda test-rocm-q4-parity test-rocm-q4-dense test-rocm-q4-pair test-strix-rocm-q4-parity test-cuda-session-batch test-cuda-mixed-batch dspark-acceptance dspark-verify-depth rocm-dspark-acceptance rocm-dspark-verify-depth mtp-verify-depth cpu cuda cuda-spark cuda-generic cuda-regression strix-halo rocm +.PHONY: all help clean test test-rocm test-glm53-kda-rocm test-metal-session-batch test-metal-session-batch-ssd test-metal-q4-streams test-metal-exactn-oracle test-metal-dspark-capture test-metal-iq2-midonly test-metal-iq2-ssd-grouped-mm test-metal-iq2-live-index test-mxfp4-cuda test-mxfp4-rocm test-mmq-parity-cuda test-rocm-q4-parity test-rocm-q4-dense test-rocm-q4-pair test-rocm-q4-prefill test-strix-rocm-q4-parity test-strix-rocm-q4-prefill test-strix-rocm-q4-prefill-long test-cuda-session-batch test-cuda-mixed-batch dspark-acceptance dspark-verify-depth rocm-dspark-acceptance rocm-dspark-verify-depth mtp-verify-depth cpu cuda cuda-spark cuda-generic cuda-regression strix-halo rocm ifeq ($(UNAME_S),Darwin) .PHONY: metal-decode-schedule-bench metal-prefill-variant-bench check-mxfp4-half-lut test-mxfp4-metal @@ -85,7 +85,8 @@ help: @echo " make test-metal-dspark-capture Check fused DSpark HC capture bitwise" @echo " make test-metal-iq2-midonly Check M1 IQ2 addr mid-only output and sentinels" @echo " make test-metal-iq2-live-index Check IQ2 SSD live-cache index policy and fallback" - @echo " make test-rocm-q4-parity Run ROCm Q4_K dense/pair CPU-oracle tests (or SKIP without HIP)" + @echo " make test-rocm-q4-parity Run ROCm Q4_K dense/pair/prefill oracle (or SKIP without HIP)" + @echo " make test-rocm-q4-prefill Run ROCm Q4 tiled-prefill parity/canary oracle" @echo " make metal-decode-schedule-bench Build the balanced Metal decode schedule benchmark" @echo " make metal-prefill-variant-bench Build the balanced Metal prefill variant benchmark" @echo " make check-mxfp4-half-lut Verify the checked-in MXFP4 half LUT matches the generator" @@ -254,7 +255,8 @@ help: @echo " make cuda-generic Build CUDA for a generic local CUDA GPU" @echo " make cuda CUDA_ARCH=sm_N Build CUDA with an explicit nvcc -arch value" @echo " make test-mmq-parity-cuda CUDA_ARCH=sm_N Run quantized CUDA kernel parity tests" - @echo " make test-rocm-q4-parity Run ROCm Q4_K dense/pair CPU-oracle tests" + @echo " make test-rocm-q4-parity Run ROCm Q4_K dense/pair/prefill oracle" + @echo " make test-strix-rocm-q4-prefill Require gfx1151 and run tiled-prefill oracle" @echo " make test-strix-rocm-q4-parity Require a visible gfx1151 device and run the Q4 tests" @echo " make strix-halo Build ROCm for Strix Halo / gfx1151" @echo " make rocm Alias for make strix-halo" @@ -602,10 +604,10 @@ test-rocm-q4-parity: rocm_test_probe="$${rocm_test_hipcc%% *}"; \ if [ -z "$$rocm_test_probe" ] || ! command -v "$$rocm_test_probe" >/dev/null 2>&1; then \ if [ -n "$(strip $(DS4_TEST_REQUIRE_ROCM_DEVICE))" ] && [ "$(strip $(DS4_TEST_REQUIRE_ROCM_DEVICE))" != "0" ]; then \ - echo "ROCm Q4 dense/pair oracle: FAIL (hipcc not found, device required)"; \ + echo "ROCm Q4 dense/pair/prefill oracle: FAIL (hipcc not found, device required)"; \ exit 1; \ fi; \ - echo "ROCm Q4 dense/pair oracle: SKIP (hipcc not found)"; exit 0; \ + echo "ROCm Q4 dense/pair/prefill oracle: SKIP (hipcc not found)"; exit 0; \ fi; \ $(MAKE) --no-print-directory tests/test_rocm_q4_dense_pair HIPCC="$$rocm_test_hipcc" || exit $$?; \ if [ -n "$(strip $(DS4_TEST_REQUIRE_ROCM_DEVICE))" ] && [ "$(strip $(DS4_TEST_REQUIRE_ROCM_DEVICE))" != "0" ]; then \ @@ -617,7 +619,7 @@ test-rocm-q4-parity: fi; \ rc=$$?; \ if [ $$rc -eq 77 ]; then \ - echo "ROCm Q4 dense/pair oracle: SKIP (no visible HIP device)"; \ + echo "ROCm Q4 dense/pair/prefill oracle: SKIP (no visible HIP device)"; \ exit 0; \ fi; \ exit $$rc @@ -628,9 +630,20 @@ test-rocm-q4-dense: test-rocm-q4-pair: $(MAKE) --no-print-directory test-rocm-q4-parity ROCM_Q4_TEST_ARGS=--pair +test-rocm-q4-prefill: + $(MAKE) --no-print-directory test-rocm-q4-parity ROCM_Q4_TEST_ARGS=--prefill + test-strix-rocm-q4-parity: $(MAKE) --no-print-directory -B test-rocm-q4-parity ROCM_ARCH=gfx1151 DS4_TEST_REQUIRE_ROCM_DEVICE=1 +test-strix-rocm-q4-prefill: + $(MAKE) --no-print-directory -B test-rocm-q4-parity ROCM_ARCH=gfx1151 \ + DS4_TEST_REQUIRE_ROCM_DEVICE=1 ROCM_Q4_TEST_ARGS=--prefill + +test-strix-rocm-q4-prefill-long: + $(MAKE) --no-print-directory -B test-rocm-q4-parity ROCM_ARCH=gfx1151 \ + DS4_TEST_REQUIRE_ROCM_DEVICE=1 ROCM_Q4_TEST_ARGS=--prefill-long + tests/cuda_long_context_smoke: tests/cuda_long_context_smoke.o ds4_cuda.o $(MMQ_OBJS) $(NVCC) $(NVCCFLAGS) -o $@ $^ $(CUDA_LDLIBS) diff --git a/ds4_rocm_unavailable.cu b/ds4_rocm_unavailable.cu index 6a9937e61..ed91a7697 100644 --- a/ds4_rocm_unavailable.cu +++ b/ds4_rocm_unavailable.cu @@ -11,7 +11,6 @@ ROCM_UNAVAILABLE_INT(ds4_gpu_add_xdev_tensor) ROCM_UNAVAILABLE_INT(ds4_gpu_attention_decode_rows_rope_tensor) ROCM_UNAVAILABLE_INT(ds4_gpu_attention_output_low_q4_K_slice_tensor) ROCM_UNAVAILABLE_INT(ds4_gpu_attention_output_low_q8_rows_exact_tensor) -ROCM_UNAVAILABLE_INT(ds4_gpu_attention_output_q4_K_batch_tensor) ROCM_UNAVAILABLE_INT(ds4_gpu_attention_prefill_raw_heads_range_tensor) ROCM_UNAVAILABLE_INT(ds4_gpu_attention_prefill_static_mixed_heads_range_tensor) ROCM_UNAVAILABLE_INT(ds4_gpu_device_cache_support_tensors) diff --git a/rocm/ds4_rocm_q4.cuh b/rocm/ds4_rocm_q4.cuh index ef11cc410..a5e193018 100644 --- a/rocm/ds4_rocm_q4.cuh +++ b/rocm/ds4_rocm_q4.cuh @@ -36,6 +36,193 @@ __global__ static void rocm_matmul_q4_K_dense_kernel( if (lane == 0u) out[(uint64_t)tok * out_dim + row] = acc; } +/* Dense Q4_K prefill tile for RDNA/ROCm. + * + * The legacy kernel gives one eight-lane group a single (token,row) dot. As + * a result every token walks the complete Q4_K row independently and every + * one of the 32 row groups in a workgroup also fetches the same Q8_K input + * blocks. Prefill is therefore dominated by redundant global reads. + * + * This kernel keeps the legacy eight-lane block assignment and reduction + * order, but computes eight token columns at once. A Q4_K block is decoded + * once into eight integer dot products, and an 8x8 token/K-block tile of the + * canonical Q8_K activations is staged in LDS for reuse by all 32 rows. The + * K loop advances in groups of eight so lane L still accumulates blocks + * L,L+8,... in exactly the order used by rocm_matmul_q4_K_dense_kernel. + * + * LDS footprint: 8 tokens * 8 K blocks * 292 bytes = 18,688 bytes. + * The final, partial token and K tiles are handled without an early return; + * every thread must reach both barriers. + */ +enum { + ROCM_Q4_PREFILL_TOKEN_TILE = 8u, + ROCM_Q4_PREFILL_KBLOCK_TILE = 8u, + ROCM_Q4_Q8K_WORDS = sizeof(cuda_block_q8_K) / sizeof(uint32_t), +}; +static_assert((sizeof(cuda_block_q8_K) % sizeof(uint32_t)) == 0u, + "ROCm Q8_K LDS copies require a whole number of words"); + +__device__ __forceinline__ static void +rocm_dot_q4_K_q8_K_block8_reuse_weights( + const cuda_block_q4_K *x, + const cuda_block_q8_K *y0, + const cuda_block_q8_K *y1, + const cuda_block_q8_K *y2, + const cuda_block_q8_K *y3, + const cuda_block_q8_K *y4, + const cuda_block_q8_K *y5, + const cuda_block_q8_K *y6, + const cuda_block_q8_K *y7, + uint32_t n, + float acc[ROCM_Q4_PREFILL_TOKEN_TILE]) { + const cuda_block_q8_K *ys[ROCM_Q4_PREFILL_TOKEN_TILE] = { + y0, y1, y2, y3, y4, y5, y6, y7, + }; + const float xd = dev_f16_to_f32(x->d); + const float xmin = dev_f16_to_f32(x->dmin); + int32_t isum[ROCM_Q4_PREFILL_TOKEN_TILE] = {0, 0, 0, 0, 0, 0, 0, 0}; + int32_t summs[ROCM_Q4_PREFILL_TOKEN_TILE] = {0, 0, 0, 0, 0, 0, 0, 0}; + + /* A 32-byte Q4 payload stores the low and high nibbles for two adjacent + * 32-value groups. Load those eight packed words once, then reuse them + * for both groups and every token in the tile. This makes weight reuse + * explicit instead of relying on the compiler or vector cache to hoist + * repeated loads out of the token loop. */ + #pragma unroll + for (uint32_t jp = 0u; jp < 4u; jp++) { + const uint32_t j0 = 2u * jp; + const uint32_t j1 = j0 + 1u; + uint8_t sc0, m0, sc1, m1; + dev_q4_K_get_scale_min(j0, x->scales, &sc0, &m0); + dev_q4_K_get_scale_min(j1, x->scales, &sc1, &m1); + + int32_t qw[8]; + #pragma unroll + for (uint32_t i = 0u; i < 8u; i++) { + qw[i] = *reinterpret_cast( + x->qs + jp * 32u + i * 4u); + } + + #pragma unroll + for (uint32_t p = 0u; p < ROCM_Q4_PREFILL_TOKEN_TILE; p++) { + if (p < n) { + const cuda_block_q8_K *y = ys[p]; + int32_t dot0 = 0; + int32_t dot1 = 0; + #pragma unroll + for (uint32_t i = 0u; i < 8u; i++) { + const int32_t w0 = qw[i] & 0x0f0f0f0f; + const int32_t w1 = (qw[i] >> 4) & 0x0f0f0f0f; + dot0 = __dp4a(w0, *reinterpret_cast( + y->qs + j0 * 32u + i * 4u), dot0); + dot1 = __dp4a(w1, *reinterpret_cast( + y->qs + j1 * 32u + i * 4u), dot1); + } + isum[p] += (int32_t)sc0 * dot0; + isum[p] += (int32_t)sc1 * dot1; + summs[p] += (int32_t)m0 * + (int32_t)(y->bsums[2u * j0] + y->bsums[2u * j0 + 1u]); + summs[p] += (int32_t)m1 * + (int32_t)(y->bsums[2u * j1] + y->bsums[2u * j1 + 1u]); + } + } + } + + #pragma unroll + for (uint32_t p = 0u; p < ROCM_Q4_PREFILL_TOKEN_TILE; p++) { + if (p < n) { + const float yd = ys[p]->d; + acc[p] += yd * xd * (float)isum[p] - + yd * xmin * (float)summs[p]; + } + } +} + +__global__ static void rocm_matmul_q4_K_prefill_tile8_strided_kernel( + float *out, + const char *w_base, + const cuda_block_q8_K *xq, + uint64_t row_bytes, + uint32_t xq_blocks, + uint32_t out_dim, + uint32_t n_tok, + uint64_t xq_token_stride, + uint64_t out_token_stride) { + __shared__ cuda_block_q8_K sxq[ROCM_Q4_PREFILL_TOKEN_TILE] + [ROCM_Q4_PREFILL_KBLOCK_TILE]; + + const uint32_t tid = threadIdx.x; + const uint32_t lane = tid & 7u; + const uint32_t row_lane = tid >> 3u; + const uint32_t row = blockIdx.x * 32u + row_lane; + const uint32_t tok0 = blockIdx.y * ROCM_Q4_PREFILL_TOKEN_TILE; + const uint32_t group = blockIdx.z; + const uint32_t nt = n_tok - tok0 < ROCM_Q4_PREFILL_TOKEN_TILE + ? n_tok - tok0 : ROCM_Q4_PREFILL_TOKEN_TILE; + const bool row_valid = row < out_dim; + const cuda_block_q4_K *wr = row_valid + ? reinterpret_cast( + w_base + ((uint64_t)group * out_dim + row) * row_bytes) + : NULL; + float acc[ROCM_Q4_PREFILL_TOKEN_TILE] = { + 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, + }; + + for (uint32_t b0 = 0u; b0 < xq_blocks; + b0 += ROCM_Q4_PREFILL_KBLOCK_TILE) { + const uint32_t nb = xq_blocks - b0 < ROCM_Q4_PREFILL_KBLOCK_TILE + ? xq_blocks - b0 + : ROCM_Q4_PREFILL_KBLOCK_TILE; + + /* Copy consecutive 32-bit words cooperatively. Assigning one 292-B + * struct per lane makes adjacent lanes issue 292-B-strided global + * loads; flattening the packed blocks gives the memory coalescer long + * contiguous runs while preserving the fixed eight-block LDS layout. */ + const uint32_t tile_words = nt * ROCM_Q4_PREFILL_KBLOCK_TILE * + ROCM_Q4_Q8K_WORDS; + uint32_t *const sxq_words = reinterpret_cast(sxq); + for (uint32_t i = tid; i < tile_words; i += blockDim.x) { + const uint32_t block_slot = i / ROCM_Q4_Q8K_WORDS; + const uint32_t word = i - block_slot * ROCM_Q4_Q8K_WORDS; + const uint32_t p = block_slot >> 3u; + const uint32_t bb = block_slot & 7u; + if (bb < nb) { + const uint64_t src_block = + (uint64_t)(tok0 + p) * xq_token_stride + + (uint64_t)group * xq_blocks + b0 + bb; + const uint32_t *const src_words = + reinterpret_cast(xq + src_block); + sxq_words[i] = src_words[word]; + } + } + __syncthreads(); + + if (row_valid && lane < nb) { + rocm_dot_q4_K_q8_K_block8_reuse_weights( + wr + b0 + lane, + sxq[0] + lane, sxq[1] + lane, + sxq[2] + lane, sxq[3] + lane, + sxq[4] + lane, sxq[5] + lane, + sxq[6] + lane, sxq[7] + lane, + nt, acc); + } + __syncthreads(); + } + + if (row_valid) { + #pragma unroll + for (uint32_t p = 0u; p < ROCM_Q4_PREFILL_TOKEN_TILE; p++) { + if (p < nt) { + const float v = quarter_warp_sum_f32(acc[p], lane); + if (lane == 0u) { + out[(uint64_t)(tok0 + p) * out_token_stride + + (uint64_t)group * out_dim + row] = v; + } + } + } + } +} + static int rocm_q4_K_dense_validate( const ds4_gpu_tensor *out, const void *model_map, @@ -90,6 +277,55 @@ static int rocm_q4_K_dense_pair_requested(void) { getenv("DS4_ROCM_DISABLE_Q4_DENSE_PAIR") == NULL; } +static int rocm_q4_K_prefill_tile8_scope(uint64_t n_tok) { + /* Keep decode/speculative micro-batches on the latency-oriented legacy + * kernel. 4096 is DS4's largest supported prefill chunk and bounds the + * A/B surface while this path remains opt-in. */ + return n_tok > 8u && n_tok <= 4096u; +} + +static int rocm_q4_K_prefill_tile8_requested(void) { + return getenv("DS4_ROCM_ENABLE_Q4_PREFILL_TILE8") != NULL && + getenv("DS4_ROCM_DISABLE_Q4_PREFILL_TILE8") == NULL; +} + +static int rocm_q4_K_prefill_tile8_required(void) { + return getenv("DS4_ROCM_REQUIRE_Q4_PREFILL_TILE8") != NULL; +} + +static uint64_t g_rocm_q4_prefill_tile8_dense_calls; +static uint64_t g_rocm_q4_prefill_tile8_pair_calls; +static uint64_t g_rocm_q4_prefill_tile8_attention_batch_calls; +static uint64_t g_rocm_q4_prefill_tile8_tokens; +static int g_rocm_q4_prefill_tile8_report_registered; + +static void rocm_q4_K_prefill_tile8_report(void) { + fprintf(stderr, + "ds4: ROCm Q4_K prefill tile8 stats: " + "dense_calls=%llu pair_calls=%llu attention_batch_calls=%llu " + "tokens=%llu\n", + (unsigned long long)g_rocm_q4_prefill_tile8_dense_calls, + (unsigned long long)g_rocm_q4_prefill_tile8_pair_calls, + (unsigned long long)g_rocm_q4_prefill_tile8_attention_batch_calls, + (unsigned long long)g_rocm_q4_prefill_tile8_tokens); +} + +static void rocm_q4_K_prefill_tile8_note( + uint32_t dense_calls, + uint32_t pair_calls, + uint32_t attention_batch_calls, + uint64_t tokens) { + if (getenv("DS4_ROCM_Q4_PREFILL_TILE8_STATS") == NULL) return; + if (!g_rocm_q4_prefill_tile8_report_registered) { + g_rocm_q4_prefill_tile8_report_registered = 1; + (void)atexit(rocm_q4_K_prefill_tile8_report); + } + g_rocm_q4_prefill_tile8_dense_calls += dense_calls; + g_rocm_q4_prefill_tile8_pair_calls += pair_calls; + g_rocm_q4_prefill_tile8_attention_batch_calls += attention_batch_calls; + g_rocm_q4_prefill_tile8_tokens += tokens; +} + extern "C" int ds4_rocm_matmul_q4_K_tensor( ds4_gpu_tensor *out, const void *model_map, @@ -108,6 +344,17 @@ extern "C" int ds4_rocm_matmul_q4_K_tensor( return 0; } + const int prefill_scope = rocm_q4_K_prefill_tile8_scope(n_tok); + const int prefill_tile8 = rocm_q4_K_prefill_tile8_requested(); + const int prefill_tile8_required = rocm_q4_K_prefill_tile8_required(); + if (prefill_scope && prefill_tile8_required && !prefill_tile8) { + fprintf(stderr, + "ds4: required ROCm Q4_K prefill tile8 is disabled " + "(n_tok=%llu)\n", + (unsigned long long)n_tok); + return 0; + } + const char *wptr = cuda_model_range_ptr(model_map, weight_offset, weight_bytes, "q4_K dense"); if (!wptr) return 0; @@ -121,6 +368,21 @@ extern "C" int ds4_rocm_matmul_q4_K_tensor( (uint32_t)in_dim, (uint32_t)n_tok); if (!cuda_ok(cudaGetLastError(), "q4_K dense quantize launch")) return 0; + if (prefill_scope && prefill_tile8) { + const dim3 tiled_grid((unsigned)((out_dim - 1u) / 32u + 1u), + (unsigned)((n_tok - 1u) / + ROCM_Q4_PREFILL_TOKEN_TILE + 1u), + 1u); + rocm_matmul_q4_K_prefill_tile8_strided_kernel<<>>( + reinterpret_cast(out->ptr), wptr, xq, row_bytes, + (uint32_t)blocks, (uint32_t)out_dim, (uint32_t)n_tok, + blocks, out_dim); + const int ok = cuda_ok(cudaGetLastError(), + "q4_K dense prefill tile8 launch"); + if (ok) rocm_q4_K_prefill_tile8_note(1u, 0u, 0u, n_tok); + return ok; + } + const dim3 grid((unsigned)((out_dim - 1u) / 32u + 1u), (unsigned)n_tok, 1u); rocm_matmul_q4_K_dense_kernel<<>>( @@ -141,10 +403,26 @@ extern "C" int ds4_gpu_matmul_q4_K_pair_tensor( uint64_t out1_dim, const ds4_gpu_tensor *x, uint64_t n_tok) { - // Keep this optional hook scoped to decode/speculative micro-batches. Larger - // prefill batches fall back to the two required dense primitives. - if (!rocm_q4_K_dense_pair_requested() || - n_tok > 8u || !out0 || !out1 || out0 == out1 || + const int prefill_scope = rocm_q4_K_prefill_tile8_scope(n_tok); + const int prefill_pair = prefill_scope && + rocm_q4_K_prefill_tile8_requested(); + const int prefill_required = prefill_scope && + rocm_q4_K_prefill_tile8_required(); + if (prefill_required && !prefill_pair) { + fprintf(stderr, + "ds4: required ROCm Q4_K prefill tile8 pair is disabled " + "(n_tok=%llu)\n", + (unsigned long long)n_tok); + return 0; + } + + /* Decode keeps its original, separately gated pair path. Prefill uses + * the common tile8 gate and shares one canonical Q8_K quantization across + * the two projections, then issues one tiled launch per weight matrix. */ + const int decode_pair = n_tok <= 8u && + rocm_q4_K_dense_pair_requested(); + if ((!prefill_pair && !decode_pair) || + !out0 || !out1 || out0 == out1 || (out0 && out1 && out0->ptr == out1->ptr)) { return 0; } @@ -179,6 +457,34 @@ extern "C" int ds4_gpu_matmul_q4_K_pair_tensor( return 0; } + if (prefill_pair) { + const dim3 grid0((unsigned)((out0_dim - 1u) / 32u + 1u), + (unsigned)((n_tok - 1u) / + ROCM_Q4_PREFILL_TOKEN_TILE + 1u), + 1u); + rocm_matmul_q4_K_prefill_tile8_strided_kernel<<>>( + reinterpret_cast(out0->ptr), w0, xq, row_bytes0, + (uint32_t)blocks0, (uint32_t)out0_dim, (uint32_t)n_tok, + blocks0, out0_dim); + if (!cuda_ok(cudaGetLastError(), + "q4_K dense prefill pair0 tile8 launch")) { + return 0; + } + + const dim3 grid1((unsigned)((out1_dim - 1u) / 32u + 1u), + (unsigned)((n_tok - 1u) / + ROCM_Q4_PREFILL_TOKEN_TILE + 1u), + 1u); + rocm_matmul_q4_K_prefill_tile8_strided_kernel<<>>( + reinterpret_cast(out1->ptr), w1, xq, row_bytes1, + (uint32_t)blocks1, (uint32_t)out1_dim, (uint32_t)n_tok, + blocks1, out1_dim); + const int ok = cuda_ok(cudaGetLastError(), + "q4_K dense prefill pair1 tile8 launch"); + if (ok) rocm_q4_K_prefill_tile8_note(0u, 1u, 0u, n_tok); + return ok; + } + // Use the exact standalone kernel twice. This preserves its block walk and // reduction path while still eliminating the second Q8_K quantization. const dim3 grid0((unsigned)((out0_dim - 1u) / 32u + 1u), @@ -197,3 +503,187 @@ extern "C" int ds4_gpu_matmul_q4_K_pair_tensor( (uint32_t)blocks1, (uint32_t)out1_dim, (uint32_t)n_tok); return cuda_ok(cudaGetLastError(), "q4_K dense pair1 matmul launch"); } + +/* Quantize token-major [token][group][K] rows once, then apply group-major + * [group][out_row][K] Q4_K weights directly into token-major output. A + * return of -1 means the quantize launch was accepted and callers must not + * replay a row fallback over potentially submitted work. */ +static int rocm_q4_K_prefill_tile8_quant_launch( + float *out, + const char *w, + const float *x, + uint32_t n_tok, + uint32_t n_groups, + uint32_t in_dim, + uint32_t out_dim, + uint64_t row_bytes, + const char *label) { + uint64_t n_rows = 0; + uint64_t xq_token_stride = 0; + uint64_t out_token_stride = 0; + const uint64_t blocks = in_dim / CUDA_QK_K; + if (!out || !w || !x || n_tok == 0u || n_groups == 0u || + in_dim == 0u || out_dim == 0u || blocks == 0u || + (in_dim % CUDA_QK_K) != 0u || + !cuda_u64_mul_checked(n_tok, n_groups, &n_rows) || + /* HIP keeps the portable grid-y limit at 65535. Real AProjQ4 uses + * eight groups, so even the 4096-token ceiling remains in range. */ + n_rows > UINT16_MAX || + !cuda_u64_mul_checked(n_groups, blocks, &xq_token_stride) || + !cuda_u64_mul_checked(n_groups, out_dim, &out_token_stride)) { + return 0; + } + + cuda_block_q8_K *xq = rocm_q4_K_prequant_alloc( + n_rows, blocks, label ? label : "q4_K prefill tile8 prequant"); + if (!xq) return 0; + + const dim3 qgrid((unsigned)blocks, (unsigned)n_rows, 1u); + q8_K_quantize_kernel<<>>( + xq, x, in_dim, (uint32_t)n_rows); + if (!cuda_ok(cudaGetLastError(), + "q4_K prefill tile8 quantize launch")) { + return 0; + } + + const dim3 grid((unsigned)((out_dim - 1u) / 32u + 1u), + (unsigned)((n_tok - 1u) / + ROCM_Q4_PREFILL_TOKEN_TILE + 1u), + n_groups); + rocm_matmul_q4_K_prefill_tile8_strided_kernel<<>>( + out, w, xq, row_bytes, (uint32_t)blocks, out_dim, n_tok, + xq_token_stride, out_token_stride); + if (!cuda_ok(cudaGetLastError(), + "q4_K prefill tile8 matmul launch")) { + return -1; + } + return 1; +} + +extern "C" int ds4_gpu_attention_output_q4_K_batch_tensor( + ds4_gpu_tensor *out, + ds4_gpu_tensor *low, + ds4_gpu_tensor *group_tmp, + ds4_gpu_tensor *low_tmp, + const void *model_map, + uint64_t model_size, + uint64_t out_a_offset, + uint64_t out_b_offset, + uint32_t out_b_type, + uint64_t group_dim, + uint64_t rank, + uint32_t n_groups, + uint64_t out_dim, + const ds4_gpu_tensor *heads, + uint32_t n_tokens) { + (void)group_tmp; + (void)low_tmp; + + const int tile8_scope = rocm_q4_K_prefill_tile8_scope(n_tokens); + const int tile8_requested = rocm_q4_K_prefill_tile8_requested(); + const int tile8_required = tile8_scope && + rocm_q4_K_prefill_tile8_required(); + if (!tile8_scope) return 0; + if (!tile8_requested) { + if (tile8_required) { + fprintf(stderr, + "ds4: required ROCm Q4_K attention-output prefill " + "tile8 is disabled (n_tok=%u)\n", + n_tokens); + return -1; + } + return 0; + } + const int pre_enqueue_failure = tile8_required ? -1 : 0; + + if (!out || !low || !heads || !model_map || group_dim == 0u || + rank == 0u || n_groups == 0u || out_dim == 0u || + n_groups > UINT16_MAX || group_dim > UINT32_MAX || + rank > UINT32_MAX || out_dim > UINT32_MAX || + (group_dim % CUDA_QK_K) != 0u || + (out_b_type != 12u && out_b_type != 8u)) { + return pre_enqueue_failure; + } + + uint64_t low_dim = 0; + uint64_t heads_rows = 0; + uint64_t heads_bytes = 0; + uint64_t low_bytes = 0; + uint64_t out_bytes = 0; + if (!cuda_u64_mul_checked(n_groups, rank, &low_dim) || + low_dim == 0u || low_dim > UINT32_MAX || + !cuda_u64_mul_checked(n_tokens, n_groups, &heads_rows) || + !cuda_u64_mul3_checked(heads_rows, group_dim, + sizeof(float), &heads_bytes) || + !cuda_u64_mul3_checked(n_tokens, low_dim, + sizeof(float), &low_bytes) || + !cuda_u64_mul3_checked(n_tokens, out_dim, + sizeof(float), &out_bytes) || + heads->bytes < heads_bytes || low->bytes < low_bytes || + out->bytes < out_bytes) { + return pre_enqueue_failure; + } + + const uint64_t a_blocks = group_dim / CUDA_QK_K; + uint64_t row_a_bytes = 0; + uint64_t out_a_bytes = 0; + if (!cuda_u64_mul_checked(a_blocks, sizeof(cuda_block_q4_K), + &row_a_bytes) || + !cuda_u64_mul_checked(low_dim, row_a_bytes, &out_a_bytes) || + !cuda_model_range_fits(model_size, out_a_offset, out_a_bytes)) { + return pre_enqueue_failure; + } + + uint64_t row_b_bytes = 0; + uint64_t out_b_bytes = 0; + if (out_b_type == 12u) { + if ((low_dim % CUDA_QK_K) != 0u || + !cuda_u64_mul_checked(low_dim / CUDA_QK_K, + sizeof(cuda_block_q4_K), &row_b_bytes)) { + return pre_enqueue_failure; + } + } else { + const uint64_t b_blocks = (low_dim + 31u) / 32u; + if (!cuda_u64_mul_checked(b_blocks, 34u, &row_b_bytes)) { + return pre_enqueue_failure; + } + } + if (!cuda_u64_mul_checked(out_dim, row_b_bytes, &out_b_bytes) || + !cuda_model_range_fits(model_size, out_b_offset, out_b_bytes)) { + return pre_enqueue_failure; + } + + const char *out_a = cuda_model_range_ptr( + model_map, out_a_offset, out_a_bytes, "q4_K attention output A"); + const char *out_b = cuda_model_range_ptr( + model_map, out_b_offset, out_b_bytes, "q4_K attention output B"); + if (!out_a || !out_b) return pre_enqueue_failure; + + /* A: one quantization over [token,group] plus one z-grouped tile8 launch; + * no group pack/unpack buffers or n_tokens*n_groups dispatch loop. */ + const int a_rc = rocm_q4_K_prefill_tile8_quant_launch( + reinterpret_cast(low->ptr), out_a, + reinterpret_cast(heads->ptr), n_tokens, n_groups, + (uint32_t)group_dim, (uint32_t)rank, row_a_bytes, + "q4_K attention output A prequant"); + if (a_rc <= 0) { + return a_rc < 0 ? -1 : pre_enqueue_failure; + } + + int b_rc = 0; + if (out_b_type == 12u) { + b_rc = rocm_q4_K_prefill_tile8_quant_launch( + reinterpret_cast(out->ptr), out_b, + reinterpret_cast(low->ptr), n_tokens, 1u, + (uint32_t)low_dim, (uint32_t)out_dim, row_b_bytes, + "q4_K attention output B prequant"); + } else { + b_rc = ds4_gpu_matmul_q8_0_tensor( + out, model_map, model_size, out_b_offset, low_dim, out_dim, + low, n_tokens); + } + if (b_rc <= 0) return -1; + + rocm_q4_K_prefill_tile8_note(0u, 0u, 1u, n_tokens); + return 1; +} diff --git a/tests/test_rocm_q4_dense_pair.cpp b/tests/test_rocm_q4_dense_pair.cpp index c379059c5..9c43e28f3 100644 --- a/tests/test_rocm_q4_dense_pair.cpp +++ b/tests/test_rocm_q4_dense_pair.cpp @@ -1,9 +1,12 @@ // SPDX-License-Identifier: MIT -// Deterministic ROCm Q4_K dense/pair oracle. +// Deterministic ROCm Q4_K dense/pair/tiled-prefill oracle. // // The test deliberately goes through the public tensor/model-map API. Weight // rows use the raw 144-byte GGUF Q4_K layout, while the CPU reference mirrors // the backend's F32 -> Q8_K quantizer and Q4_K x Q8_K integer dot product. +// Prefill controls are forced through the rollback path before the TILE8 +// REQUIRE path so a future default promotion cannot turn parity into a +// candidate-vs-candidate false green. #include "ds4_gpu.h" @@ -34,10 +37,25 @@ constexpr uint32_t kK = 4096u; constexpr uint32_t kM0 = 65u; constexpr uint32_t kM1 = 33u; constexpr uint32_t kQ4Type = 12u; +constexpr uint32_t kQ8Type = 8u; +constexpr uint32_t kTailK = 1024u; +constexpr uint32_t kAttnGroupDim = 4096u; +constexpr uint32_t kAttnRank = 32u; +constexpr uint32_t kAttnGroups = 8u; +constexpr uint32_t kAttnLowDim = kAttnGroups * kAttnRank; +constexpr uint32_t kAttnOutDim = 65u; +constexpr size_t kOutputGuardFloats = 257u; constexpr float kCpuAbsTolerance = 2.0e-3f; constexpr float kCpuRelTolerance = 3.0e-5f; constexpr int kSkip = 77; +constexpr const char *kPrefillEnable = + "DS4_ROCM_ENABLE_Q4_PREFILL_TILE8"; +constexpr const char *kPrefillDisable = + "DS4_ROCM_DISABLE_Q4_PREFILL_TILE8"; +constexpr const char *kPrefillRequire = + "DS4_ROCM_REQUIRE_Q4_PREFILL_TILE8"; + struct block_q4_K_test { uint16_t d; uint16_t dmin; @@ -51,15 +69,23 @@ struct block_q8_K_test { int16_t bsums[kQkK / 16u]; }; +struct block_q8_0_test { + uint16_t d; + int8_t qs[32]; +}; + static_assert(sizeof(block_q4_K_test) == 144u, "Q4_K fixture must match the raw GGUF layout"); static_assert(sizeof(block_q8_K_test) == 292u, "Q8_K oracle must match the ROCm activation layout"); +static_assert(sizeof(block_q8_0_test) == 34u, + "Q8_0 fixture must match the raw GGUF layout"); struct tensor_owner { ds4_gpu_tensor *ptr = nullptr; explicit tensor_owner(uint64_t bytes) : ptr(ds4_gpu_tensor_alloc(bytes)) {} + explicit tensor_owner(ds4_gpu_tensor *owned) : ptr(owned) {} ~tensor_owner() { ds4_gpu_tensor_free(ptr); } tensor_owner(const tensor_owner &) = delete; @@ -71,6 +97,10 @@ struct aligned_model { uint64_t size = 0; uint64_t weight0_offset = 0; uint64_t weight1_offset = 0; + uint64_t attn_a_offset = 0; + uint64_t attn_b_offset = 0; + uint64_t attn_b_q8_offset = 0; + uint64_t tail_k1024_offset = 0; ~aligned_model() { std::free(data); } @@ -177,9 +207,10 @@ void q4_scale_min(uint32_t j, const uint8_t *scales, } } -void fill_q4_rows(block_q4_K_test *rows, uint32_t n_rows, uint32_t seed) { +void fill_q4_rows(block_q4_K_test *rows, uint32_t n_rows, + uint32_t in_dim, uint32_t seed) { uint32_t state = seed; - const uint32_t blocks_per_row = kK / kQkK; + const uint32_t blocks_per_row = in_dim / kQkK; for (uint32_t row = 0; row < n_rows; row++) { for (uint32_t b = 0; b < blocks_per_row; b++) { block_q4_K_test &block = rows[(uint64_t)row * blocks_per_row + b]; @@ -195,14 +226,56 @@ void fill_q4_rows(block_q4_K_test *rows, uint32_t n_rows, uint32_t seed) { } } +void fill_q8_0_rows(block_q8_0_test *rows, uint32_t n_rows, + uint32_t in_dim, uint32_t seed) { + uint32_t state = seed; + const uint32_t blocks_per_row = in_dim / 32u; + for (uint32_t row = 0; row < n_rows; row++) { + for (uint32_t b = 0; b < blocks_per_row; b++) { + block_q8_0_test &block = + rows[(uint64_t)row * blocks_per_row + b]; + const float scale = 0.0015f + + 0.000125f * (float)(1u + (lcg_next(state) % 29u)); + block.d = float_to_fp16(scale); + for (int8_t &q : block.qs) { + q = (int8_t)((int)(lcg_next(state) % 255u) - 127); + } + } + } +} + bool make_model(aligned_model *model) { constexpr uint64_t page = 4096u; const uint64_t row_bytes = (kK / kQkK) * sizeof(block_q4_K_test); const uint64_t weight0_bytes = kM0 * row_bytes; const uint64_t weight1_bytes = kM1 * row_bytes; + const uint64_t attn_a_row_bytes = + (kAttnGroupDim / kQkK) * sizeof(block_q4_K_test); + const uint64_t attn_a_bytes = + (uint64_t)kAttnGroups * kAttnRank * attn_a_row_bytes; + const uint64_t attn_b_row_bytes = + (kAttnLowDim / kQkK) * sizeof(block_q4_K_test); + const uint64_t attn_b_bytes = + (uint64_t)kAttnOutDim * attn_b_row_bytes; + const uint64_t tail_row_bytes = + (kTailK / kQkK) * sizeof(block_q4_K_test); + const uint64_t tail_bytes = (uint64_t)kM0 * tail_row_bytes; + const uint64_t attn_b_q8_row_bytes = + (kAttnLowDim / 32u) * sizeof(block_q8_0_test); + const uint64_t attn_b_q8_bytes = + (uint64_t)kAttnOutDim * attn_b_q8_row_bytes; model->weight0_offset = 0u; model->weight1_offset = round_up(weight0_bytes, page); - model->size = round_up(model->weight1_offset + weight1_bytes, page); + model->attn_a_offset = round_up( + model->weight1_offset + weight1_bytes, page); + model->attn_b_offset = round_up( + model->attn_a_offset + attn_a_bytes, page); + model->tail_k1024_offset = round_up( + model->attn_b_offset + attn_b_bytes, page); + model->attn_b_q8_offset = round_up( + model->tail_k1024_offset + tail_bytes, page); + model->size = round_up( + model->attn_b_q8_offset + attn_b_q8_bytes, page); void *storage = nullptr; if (posix_memalign(&storage, (size_t)page, (size_t)model->size) != 0) { return false; @@ -211,18 +284,32 @@ bool make_model(aligned_model *model) { std::memset(model->data, 0xa5, (size_t)model->size); fill_q4_rows(reinterpret_cast( model->data + model->weight0_offset), - kM0, 0x41c64e6du); + kM0, kK, 0x41c64e6du); fill_q4_rows(reinterpret_cast( model->data + model->weight1_offset), - kM1, 0x9e3779b9u); + kM1, kK, 0x9e3779b9u); + fill_q4_rows(reinterpret_cast( + model->data + model->attn_a_offset), + kAttnGroups * kAttnRank, kAttnGroupDim, 0x243f6a88u); + fill_q4_rows(reinterpret_cast( + model->data + model->attn_b_offset), + kAttnOutDim, kAttnLowDim, 0x85a308d3u); + fill_q4_rows(reinterpret_cast( + model->data + model->tail_k1024_offset), + kM0, kTailK, 0x13198a2eu); + fill_q8_0_rows(reinterpret_cast( + model->data + model->attn_b_q8_offset), + kAttnOutDim, kAttnLowDim, 0x03707344u); return true; } -void fill_activation(std::vector *x, uint32_t n_tokens) { - x->resize((uint64_t)n_tokens * kK); +void fill_activation(std::vector *x, uint32_t n_tokens, + uint32_t in_dim = kK) { + x->resize((uint64_t)n_tokens * in_dim); for (uint32_t token = 0; token < n_tokens; token++) { - for (uint32_t b = 0; b < kK / kQkK; b++) { - float *block = x->data() + (uint64_t)token * kK + b * kQkK; + for (uint32_t b = 0; b < in_dim / kQkK; b++) { + float *block = + x->data() + (uint64_t)token * in_dim + b * kQkK; for (uint32_t i = 0; i < kQkK; i++) { const int q = (int)((i * 73u + token * 37u + b * 19u) % 241u) - 120; block[i] = (float)q / 32.0f; @@ -294,14 +381,16 @@ float dot_q4_q8_raw(const block_q4_K_test &weight, std::vector dense_reference(const uint8_t *weight_base, const std::vector &x, uint32_t out_dim, - uint32_t n_tokens) { + uint32_t n_tokens, + uint32_t in_dim = kK) { const auto *weights = reinterpret_cast(weight_base); - constexpr uint32_t blocks_per_row = kK / kQkK; + const uint32_t blocks_per_row = in_dim / kQkK; std::vector xq((uint64_t)n_tokens * blocks_per_row); for (uint32_t token = 0; token < n_tokens; token++) { for (uint32_t b = 0; b < blocks_per_row; b++) { - quantize_q8_K_cpu(x.data() + (uint64_t)token * kK + b * kQkK, - &xq[(uint64_t)token * blocks_per_row + b]); + quantize_q8_K_cpu( + x.data() + (uint64_t)token * in_dim + b * kQkK, + &xq[(uint64_t)token * blocks_per_row + b]); } } std::vector result((uint64_t)n_tokens * out_dim, 0.0f); @@ -494,6 +583,446 @@ bool unchanged_after_rejected_call(ds4_gpu_tensor *tensor, return bitwise_equal(after, sentinel, label); } +bool output_guard_unchanged(const std::vector &values, + const std::vector &sentinel, + size_t logical_count, + const char *label) { + if (values.size() != sentinel.size() || + logical_count > values.size()) { + std::fprintf(stderr, "%s: invalid guard geometry FAIL\n", label); + return false; + } + uint64_t mismatches = 0; + size_t first = logical_count; + for (size_t i = logical_count; i < values.size(); i++) { + if (std::memcmp(&values[i], &sentinel[i], sizeof(float)) != 0) { + if (mismatches == 0) first = i; + mismatches++; + } + } + std::fprintf(stderr, "%s: mismatches=%llu/%zu %s\n", + label, (unsigned long long)mismatches, + values.size() - logical_count, + mismatches == 0 ? "PASS" : "FAIL"); + if (mismatches != 0) { + std::fprintf(stderr, " first guard overwrite at float %zu\n", first); + } + return mismatches == 0; +} + +bool run_prefill_parity_case(const aligned_model &model, uint32_t n_tokens, + uint64_t offset, uint32_t out_dim, + bool compare_cpu, const char *label, + uint32_t in_dim = kK) { + std::vector x; + fill_activation(&x, n_tokens, in_dim); + const size_t logical_count = (size_t)n_tokens * out_dim; + const size_t allocation_count = logical_count + kOutputGuardFloats; + const std::vector sentinel = sentinel_values(allocation_count); + + tensor_owner x_gpu(x.size() * sizeof(float)); + tensor_owner legacy_gpu(allocation_count * sizeof(float)); + tensor_owner candidate_gpu(allocation_count * sizeof(float)); + if (!x_gpu.ptr || !legacy_gpu.ptr || !candidate_gpu.ptr || + !write_tensor(x_gpu.ptr, x) || + !write_tensor(legacy_gpu.ptr, sentinel) || + !write_tensor(candidate_gpu.ptr, sentinel)) { + std::fprintf(stderr, "%s: tensor allocation/write FAIL\n", label); + return false; + } + + env_snapshot enable(kPrefillEnable); + env_snapshot disable(kPrefillDisable); + env_snapshot require(kPrefillRequire); + + // The authoritative rollback is the reference even after a future + // default-on promotion of the tiled path. + (void)unsetenv(kPrefillEnable); + (void)setenv(kPrefillDisable, "1", 1); + (void)unsetenv(kPrefillRequire); + const int legacy_rc = ds4_gpu_matmul_quant_tensor( + legacy_gpu.ptr, model.data, model.size, offset, kQ4Type, + in_dim, out_dim, x_gpu.ptr, n_tokens); + + // REQUIRE makes a silently ineligible candidate a test failure instead + // of comparing the legacy kernel with itself. + (void)setenv(kPrefillEnable, "1", 1); + (void)unsetenv(kPrefillDisable); + (void)setenv(kPrefillRequire, "1", 1); + const int candidate_rc = ds4_gpu_matmul_quant_tensor( + candidate_gpu.ptr, model.data, model.size, offset, kQ4Type, + in_dim, out_dim, x_gpu.ptr, n_tokens); + + std::vector legacy_all(allocation_count); + std::vector candidate_all(allocation_count); + if (legacy_rc == 0 || candidate_rc == 0 || + !read_tensor(legacy_gpu.ptr, &legacy_all) || + !read_tensor(candidate_gpu.ptr, &candidate_all)) { + std::fprintf(stderr, + "%s: dispatch/read legacy=%d candidate=%d FAIL\n", + label, legacy_rc, candidate_rc); + return false; + } + + bool ok = output_guard_unchanged( + legacy_all, sentinel, logical_count, "prefill legacy output canary"); + ok = output_guard_unchanged( + candidate_all, sentinel, logical_count, + "prefill candidate output canary") && ok; + + legacy_all.resize(logical_count); + candidate_all.resize(logical_count); + ok = bitwise_equal(candidate_all, legacy_all, + "prefill candidate vs forced legacy") && ok; + if (compare_cpu) { + const std::vector cpu = dense_reference( + model.data + offset, x, out_dim, n_tokens, in_dim); + ok = close_to_cpu(legacy_all, cpu, + "prefill forced legacy vs CPU") && ok; + ok = close_to_cpu(candidate_all, cpu, + "prefill candidate vs CPU") && ok; + } + std::fprintf(stderr, + "%s: legacy_rc=%d candidate_rc=%d logical=%zu guard=%zu %s\n", + label, legacy_rc, candidate_rc, logical_count, + kOutputGuardFloats, ok ? "PASS" : "FAIL"); + return ok; +} + +bool run_prefill_gate_guards(const aligned_model &model) { + constexpr uint32_t n_tokens = 9u; + std::vector x; + fill_activation(&x, n_tokens); + const size_t output_count = (size_t)n_tokens * kM1 + kOutputGuardFloats; + const std::vector sentinel = sentinel_values(output_count); + tensor_owner x_gpu(x.size() * sizeof(float)); + tensor_owner out_gpu(output_count * sizeof(float)); + if (!x_gpu.ptr || !out_gpu.ptr || !write_tensor(x_gpu.ptr, x) || + !write_tensor(out_gpu.ptr, sentinel)) { + std::fprintf(stderr, "prefill gate guards: setup FAIL\n"); + return false; + } + + env_snapshot enable(kPrefillEnable); + env_snapshot disable(kPrefillDisable); + env_snapshot require(kPrefillRequire); + (void)setenv(kPrefillEnable, "1", 1); + (void)setenv(kPrefillDisable, "1", 1); + (void)setenv(kPrefillRequire, "1", 1); + const int rc = ds4_gpu_matmul_quant_tensor( + out_gpu.ptr, model.data, model.size, model.weight1_offset, kQ4Type, + kK, kM1, x_gpu.ptr, n_tokens); + bool ok = rc == 0; + if (rc != 0) { + std::fprintf(stderr, + "prefill DISABLE+REQUIRE: expected rc=0 got=%d FAIL\n", + rc); + } + ok = unchanged_after_rejected_call( + out_gpu.ptr, sentinel, + "prefill DISABLE dominates REQUIRE and preserves output") && ok; + return ok; +} + +bool run_prefill_pair_case(const aligned_model &model) { + constexpr uint32_t n_tokens = 128u; + std::vector x; + fill_activation(&x, n_tokens); + const size_t count0 = (size_t)n_tokens * kM0; + const size_t count1 = (size_t)n_tokens * kM1; + const std::vector sentinel0 = + sentinel_values(count0 + kOutputGuardFloats); + const std::vector sentinel1 = + sentinel_values(count1 + kOutputGuardFloats); + + tensor_owner x_gpu(x.size() * sizeof(float)); + tensor_owner legacy0(sentinel0.size() * sizeof(float)); + tensor_owner legacy1(sentinel1.size() * sizeof(float)); + tensor_owner pair0(sentinel0.size() * sizeof(float)); + tensor_owner pair1(sentinel1.size() * sizeof(float)); + if (!x_gpu.ptr || !legacy0.ptr || !legacy1.ptr || !pair0.ptr || + !pair1.ptr || !write_tensor(x_gpu.ptr, x) || + !write_tensor(legacy0.ptr, sentinel0) || + !write_tensor(legacy1.ptr, sentinel1) || + !write_tensor(pair0.ptr, sentinel0) || + !write_tensor(pair1.ptr, sentinel1)) { + std::fprintf(stderr, "prefill pair n_tok=128: setup FAIL\n"); + return false; + } + + env_snapshot prefill_enable(kPrefillEnable); + env_snapshot prefill_disable(kPrefillDisable); + env_snapshot prefill_require(kPrefillRequire); + env_snapshot pair_enable("DS4_ROCM_ENABLE_Q4_DENSE_PAIR"); + env_snapshot pair_disable("DS4_ROCM_DISABLE_Q4_DENSE_PAIR"); + + (void)unsetenv(kPrefillEnable); + (void)setenv(kPrefillDisable, "1", 1); + (void)unsetenv(kPrefillRequire); + const int legacy_rc0 = ds4_gpu_matmul_quant_tensor( + legacy0.ptr, model.data, model.size, model.weight0_offset, kQ4Type, + kK, kM0, x_gpu.ptr, n_tokens); + const int legacy_rc1 = ds4_gpu_matmul_quant_tensor( + legacy1.ptr, model.data, model.size, model.weight1_offset, kQ4Type, + kK, kM1, x_gpu.ptr, n_tokens); + + // The prefill pair is a distinct path: it must not depend on the legacy + // decode-pair opt-in, whose <=8-token behavior is tested separately. + (void)setenv(kPrefillEnable, "1", 1); + (void)unsetenv(kPrefillDisable); + (void)setenv(kPrefillRequire, "1", 1); + (void)unsetenv("DS4_ROCM_ENABLE_Q4_DENSE_PAIR"); + (void)unsetenv("DS4_ROCM_DISABLE_Q4_DENSE_PAIR"); + const int pair_rc = ds4_gpu_matmul_q4_K_pair_tensor( + pair0.ptr, pair1.ptr, model.data, model.size, + model.weight0_offset, model.weight1_offset, + kK, kM0, kM1, x_gpu.ptr, n_tokens); + + std::vector legacy0_host(sentinel0.size()); + std::vector legacy1_host(sentinel1.size()); + std::vector pair0_host(sentinel0.size()); + std::vector pair1_host(sentinel1.size()); + if (legacy_rc0 == 0 || legacy_rc1 == 0 || pair_rc == 0 || + !read_tensor(legacy0.ptr, &legacy0_host) || + !read_tensor(legacy1.ptr, &legacy1_host) || + !read_tensor(pair0.ptr, &pair0_host) || + !read_tensor(pair1.ptr, &pair1_host)) { + std::fprintf(stderr, + "prefill pair n_tok=128: dispatch/read legacy=(%d,%d) " + "pair=%d FAIL\n", + legacy_rc0, legacy_rc1, pair_rc); + return false; + } + + bool ok = output_guard_unchanged( + pair0_host, sentinel0, count0, "prefill pair0 output canary"); + ok = output_guard_unchanged( + pair1_host, sentinel1, count1, + "prefill pair1 output canary") && ok; + pair0_host.resize(count0); + pair1_host.resize(count1); + legacy0_host.resize(count0); + legacy1_host.resize(count1); + ok = bitwise_equal(pair0_host, legacy0_host, + "prefill pair0 vs forced legacy dense0") && ok; + ok = bitwise_equal(pair1_host, legacy1_host, + "prefill pair1 vs forced legacy dense1") && ok; + std::fprintf(stderr, + "prefill pair K=4096 M=(65,33) n_tok=128 " + "legacy=(%d,%d) pair=%d %s\n", + legacy_rc0, legacy_rc1, pair_rc, ok ? "PASS" : "FAIL"); + return ok; +} + +bool run_attention_rowwise_reference(const aligned_model &model, + const ds4_gpu_tensor *heads, + ds4_gpu_tensor *low, + ds4_gpu_tensor *out, + uint32_t n_tokens, + uint64_t out_b_offset, + uint32_t out_b_type) { + const uint64_t heads_group_bytes = + (uint64_t)kAttnGroupDim * sizeof(float); + const uint64_t heads_token_bytes = + (uint64_t)kAttnGroups * heads_group_bytes; + const uint64_t low_group_bytes = + (uint64_t)kAttnRank * sizeof(float); + const uint64_t low_token_bytes = + (uint64_t)kAttnLowDim * sizeof(float); + const uint64_t out_token_bytes = + (uint64_t)kAttnOutDim * sizeof(float); + const uint64_t row_a_bytes = + (kAttnGroupDim / kQkK) * sizeof(block_q4_K_test); + const uint64_t group_a_bytes = (uint64_t)kAttnRank * row_a_bytes; + + for (uint32_t token = 0; token < n_tokens; token++) { + for (uint32_t group = 0; group < kAttnGroups; group++) { + tensor_owner heads_group(ds4_gpu_tensor_view( + heads, + (uint64_t)token * heads_token_bytes + + (uint64_t)group * heads_group_bytes, + heads_group_bytes)); + tensor_owner low_group(ds4_gpu_tensor_view( + low, + (uint64_t)token * low_token_bytes + + (uint64_t)group * low_group_bytes, + low_group_bytes)); + if (!heads_group.ptr || !low_group.ptr || + ds4_gpu_matmul_quant_tensor( + low_group.ptr, model.data, model.size, + model.attn_a_offset + (uint64_t)group * group_a_bytes, + kQ4Type, kAttnGroupDim, kAttnRank, + heads_group.ptr, 1u) == 0) { + std::fprintf(stderr, + "attention row reference A token=%u group=%u FAIL\n", + token, group); + return false; + } + } + tensor_owner low_row(ds4_gpu_tensor_view( + low, (uint64_t)token * low_token_bytes, low_token_bytes)); + tensor_owner out_row(ds4_gpu_tensor_view( + out, (uint64_t)token * out_token_bytes, out_token_bytes)); + if (!low_row.ptr || !out_row.ptr || + ds4_gpu_matmul_quant_tensor( + out_row.ptr, model.data, model.size, out_b_offset, + out_b_type, kAttnLowDim, kAttnOutDim, + low_row.ptr, 1u) == 0) { + std::fprintf(stderr, + "attention row reference B token=%u FAIL\n", token); + return false; + } + } + return true; +} + +bool run_attention_prefill_case(const aligned_model &model, + uint32_t n_tokens, + const char *label, + uint32_t out_b_type = kQ4Type) { + if (out_b_type != kQ4Type && out_b_type != kQ8Type) { + std::fprintf(stderr, "%s: unsupported output-B type %u FAIL\n", + label, out_b_type); + return false; + } + const uint64_t out_b_offset = out_b_type == kQ8Type + ? model.attn_b_q8_offset : model.attn_b_offset; + const size_t heads_count = + (size_t)n_tokens * kAttnGroups * kAttnGroupDim; + const size_t low_count = (size_t)n_tokens * kAttnLowDim; + const size_t out_count = (size_t)n_tokens * kAttnOutDim; + const size_t group_tmp_count = (size_t)n_tokens * kAttnGroupDim; + const size_t low_tmp_count = (size_t)n_tokens * kAttnRank; + std::vector heads_host; + fill_activation(&heads_host, n_tokens * kAttnGroups); + const std::vector low_sentinel = + sentinel_values(low_count + kOutputGuardFloats); + const std::vector out_sentinel = + sentinel_values(out_count + kOutputGuardFloats); + const std::vector group_tmp_sentinel = + sentinel_values(group_tmp_count + kOutputGuardFloats); + const std::vector low_tmp_sentinel = + sentinel_values(low_tmp_count + kOutputGuardFloats); + + tensor_owner heads_gpu(heads_count * sizeof(float)); + tensor_owner reference_low(low_sentinel.size() * sizeof(float)); + tensor_owner reference_out(out_sentinel.size() * sizeof(float)); + tensor_owner candidate_low(low_sentinel.size() * sizeof(float)); + tensor_owner candidate_out(out_sentinel.size() * sizeof(float)); + tensor_owner group_tmp(group_tmp_sentinel.size() * sizeof(float)); + tensor_owner low_tmp(low_tmp_sentinel.size() * sizeof(float)); + if (!heads_gpu.ptr || !reference_low.ptr || !reference_out.ptr || + !candidate_low.ptr || !candidate_out.ptr || !group_tmp.ptr || + !low_tmp.ptr || !write_tensor(heads_gpu.ptr, heads_host) || + !write_tensor(reference_low.ptr, low_sentinel) || + !write_tensor(reference_out.ptr, out_sentinel) || + !write_tensor(candidate_low.ptr, low_sentinel) || + !write_tensor(candidate_out.ptr, out_sentinel) || + !write_tensor(group_tmp.ptr, group_tmp_sentinel) || + !write_tensor(low_tmp.ptr, low_tmp_sentinel)) { + std::fprintf(stderr, "%s: setup FAIL\n", label); + return false; + } + + env_snapshot enable(kPrefillEnable); + env_snapshot disable(kPrefillDisable); + env_snapshot require(kPrefillRequire); + (void)unsetenv(kPrefillEnable); + (void)setenv(kPrefillDisable, "1", 1); + (void)unsetenv(kPrefillRequire); + if (!run_attention_rowwise_reference( + model, heads_gpu.ptr, reference_low.ptr, reference_out.ptr, + n_tokens, out_b_offset, out_b_type)) { + std::fprintf(stderr, "%s: row-wise reference FAIL\n", label); + return false; + } + + (void)setenv(kPrefillEnable, "1", 1); + (void)unsetenv(kPrefillDisable); + (void)setenv(kPrefillRequire, "1", 1); + const int candidate_rc = ds4_gpu_attention_output_q4_K_batch_tensor( + candidate_out.ptr, candidate_low.ptr, group_tmp.ptr, low_tmp.ptr, + model.data, model.size, model.attn_a_offset, out_b_offset, + out_b_type, kAttnGroupDim, kAttnRank, kAttnGroups, kAttnOutDim, + heads_gpu.ptr, n_tokens); + + std::vector reference_low_host(low_sentinel.size()); + std::vector reference_out_host(out_sentinel.size()); + std::vector candidate_low_host(low_sentinel.size()); + std::vector candidate_out_host(out_sentinel.size()); + std::vector group_tmp_host(group_tmp_sentinel.size()); + std::vector low_tmp_host(low_tmp_sentinel.size()); + if (candidate_rc != 1 || + !read_tensor(reference_low.ptr, &reference_low_host) || + !read_tensor(reference_out.ptr, &reference_out_host) || + !read_tensor(candidate_low.ptr, &candidate_low_host) || + !read_tensor(candidate_out.ptr, &candidate_out_host) || + !read_tensor(group_tmp.ptr, &group_tmp_host) || + !read_tensor(low_tmp.ptr, &low_tmp_host)) { + std::fprintf(stderr, "%s: candidate dispatch/read rc=%d FAIL\n", + label, candidate_rc); + return false; + } + + bool ok = output_guard_unchanged( + candidate_low_host, low_sentinel, low_count, + "attention candidate low canary"); + ok = output_guard_unchanged( + candidate_out_host, out_sentinel, out_count, + "attention candidate out canary") && ok; + ok = output_guard_unchanged( + group_tmp_host, group_tmp_sentinel, group_tmp_count, + "attention group scratch canary") && ok; + ok = output_guard_unchanged( + low_tmp_host, low_tmp_sentinel, low_tmp_count, + "attention low scratch canary") && ok; + reference_low_host.resize(low_count); + reference_out_host.resize(out_count); + candidate_low_host.resize(low_count); + candidate_out_host.resize(out_count); + ok = bitwise_equal(candidate_low_host, reference_low_host, + "attention candidate low vs 8x row-wise A") && ok; + ok = bitwise_equal(candidate_out_host, reference_out_host, + "attention candidate out vs row-wise B") && ok; + + // The batch API promises -1 for a REQUIRE diagnostic so the graph does + // not replay its row fallback after a forced-candidate failure. + if (!write_tensor(candidate_low.ptr, low_sentinel) || + !write_tensor(candidate_out.ptr, out_sentinel) || + !write_tensor(group_tmp.ptr, group_tmp_sentinel) || + !write_tensor(low_tmp.ptr, low_tmp_sentinel)) { + return false; + } + (void)setenv(kPrefillDisable, "1", 1); + const int rejected_rc = ds4_gpu_attention_output_q4_K_batch_tensor( + candidate_out.ptr, candidate_low.ptr, group_tmp.ptr, low_tmp.ptr, + model.data, model.size, model.attn_a_offset, out_b_offset, + out_b_type, kAttnGroupDim, kAttnRank, kAttnGroups, kAttnOutDim, + heads_gpu.ptr, n_tokens); + if (rejected_rc != -1) { + std::fprintf(stderr, + "%s: DISABLE+REQUIRE expected rc=-1 got=%d FAIL\n", + label, rejected_rc); + ok = false; + } + ok = unchanged_after_rejected_call( + candidate_low.ptr, low_sentinel, + "attention rejected call preserves low") && ok; + ok = unchanged_after_rejected_call( + candidate_out.ptr, out_sentinel, + "attention rejected call preserves out") && ok; + ok = unchanged_after_rejected_call( + group_tmp.ptr, group_tmp_sentinel, + "attention rejected call preserves group scratch") && ok; + ok = unchanged_after_rejected_call( + low_tmp.ptr, low_tmp_sentinel, + "attention rejected call preserves low scratch") && ok; + std::fprintf(stderr, + "%s: candidate_rc=%d rejected_rc=%d %s\n", + label, candidate_rc, rejected_rc, ok ? "PASS" : "FAIL"); + return ok; +} + bool run_dense_guards(const aligned_model &model) { std::vector x; fill_activation(&x, 1u); @@ -607,14 +1136,16 @@ int detect_rocm_device() { const hipError_t err = hipGetDeviceCount(&count); if (err != hipSuccess || count <= 0) { std::fprintf(stderr, - "ROCm Q4 dense/pair: SKIP (HIP runtime has no visible device: %s)\n", + "ROCm Q4 dense/pair/prefill: SKIP " + "(HIP runtime has no visible device: %s)\n", err == hipSuccess ? "device count is zero" : hipGetErrorString(err)); return 0; } return count; #else std::fprintf(stderr, - "ROCm Q4 dense/pair: SKIP (compiled without HIP runtime headers)\n"); + "ROCm Q4 dense/pair/prefill: SKIP " + "(compiled without HIP runtime headers)\n"); return 0; #endif } @@ -624,16 +1155,37 @@ int detect_rocm_device() { int main(int argc, char **argv) { bool run_dense = true; bool run_pair = true; + bool run_prefill = true; + bool run_prefill_long = false; if (argc == 2 && std::strcmp(argv[1], "--dense") == 0) { run_pair = false; + run_prefill = false; } else if (argc == 2 && std::strcmp(argv[1], "--pair") == 0) { run_dense = false; + run_prefill = false; + } else if (argc == 2 && std::strcmp(argv[1], "--prefill") == 0) { + run_dense = false; + run_pair = false; + } else if (argc == 2 && + std::strcmp(argv[1], "--prefill-long") == 0) { + run_dense = false; + run_pair = false; + run_prefill_long = true; } else if (argc > 1 && !(argc == 2 && std::strcmp(argv[1], "--all") == 0)) { - std::fprintf(stderr, "usage: %s [--all|--dense|--pair]\n", argv[0]); + std::fprintf(stderr, + "usage: %s [--all|--dense|--pair|--prefill|--prefill-long]\n", + argv[0]); return 2; } + env_snapshot prefill_enable(kPrefillEnable); + env_snapshot prefill_disable(kPrefillDisable); + env_snapshot prefill_require(kPrefillRequire); + (void)unsetenv(kPrefillEnable); + (void)unsetenv(kPrefillDisable); + (void)unsetenv(kPrefillRequire); + if (detect_rocm_device() <= 0) { const char *require_device = std::getenv("DS4_TEST_REQUIRE_ROCM_DEVICE"); @@ -643,16 +1195,19 @@ int main(int argc, char **argv) { } if (!ds4_gpu_init()) { std::fprintf(stderr, - "ROCm Q4 dense/pair: FAIL (device is visible but ds4_gpu_init failed)\n"); + "ROCm Q4 dense/pair/prefill: FAIL " + "(device is visible but ds4_gpu_init failed)\n"); return 1; } aligned_model model; bool ok = make_model(&model); if (!ok) { - std::fprintf(stderr, "ROCm Q4 dense/pair: fixture allocation FAIL\n"); + std::fprintf(stderr, + "ROCm Q4 dense/pair/prefill: fixture allocation FAIL\n"); } else if (!ds4_gpu_set_model_map(model.data, model.size)) { - std::fprintf(stderr, "ROCm Q4 dense/pair: model-map registration FAIL\n"); + std::fprintf(stderr, + "ROCm Q4 dense/pair/prefill: model-map registration FAIL\n"); ok = false; } @@ -683,11 +1238,73 @@ int main(int argc, char **argv) { ok = pair1_ok && pair3_ok && pair8_ok && pair_guard_ok && pair_opt_in_ok && ok; } + if (model_ready && run_prefill) { + std::fprintf(stderr, + "ROCm Q4 tiled prefill parity " + "(forced legacy vs ENABLE+REQUIRE):\n"); + const bool prefill9_ok = run_prefill_parity_case( + model, 9u, model.weight0_offset, kM0, true, + "prefill K=4096 M=65 n_tok=9"); + const bool prefill30_ok = run_prefill_parity_case( + model, 30u, model.weight0_offset, kM0, true, + "prefill K=4096 M=65 n_tok=30 (token-tail nt=6)"); + const bool prefill128_ok = run_prefill_parity_case( + model, 128u, model.weight1_offset, kM1, true, + "prefill K=4096 M=33 n_tok=128"); + const bool prefill_tail9_ok = run_prefill_parity_case( + model, 9u, model.tail_k1024_offset, kM0, true, + "prefill K=1024 M=65 n_tok=9 (K-tail nb=4)", kTailK); + const bool prefill_tail128_ok = run_prefill_parity_case( + model, 128u, model.tail_k1024_offset, kM0, true, + "prefill K=1024 M=65 n_tok=128 (K-tail nb=4)", kTailK); + const bool prefill_single9_ok = run_prefill_parity_case( + model, 9u, model.attn_b_offset, kAttnOutDim, true, + "prefill K=256 M=65 n_tok=9 (K-tail nb=1)", kAttnLowDim); + const bool prefill_single128_ok = run_prefill_parity_case( + model, 128u, model.attn_b_offset, kAttnOutDim, true, + "prefill K=256 M=65 n_tok=128 (K-tail nb=1)", kAttnLowDim); + const bool prefill_pair_ok = run_prefill_pair_case(model); + const bool attention9_ok = run_attention_prefill_case( + model, 9u, + "attention prefill groups=8 K=4096 rank=32 M=65 n_tok=9"); + const bool attention30_ok = run_attention_prefill_case( + model, 30u, + "attention prefill groups=8 K=4096 rank=32 M=65 " + "n_tok=30 (token-tail nt=6)"); + const bool attention128_ok = run_attention_prefill_case( + model, 128u, + "attention prefill groups=8 K=4096 rank=32 M=65 n_tok=128"); + const bool attention_q8_9_ok = run_attention_prefill_case( + model, 9u, + "attention prefill Q4-A/Q8-B groups=8 K=4096 rank=32 M=65 " + "n_tok=9", + kQ8Type); + const bool attention_q8_30_ok = run_attention_prefill_case( + model, 30u, + "attention prefill Q4-A/Q8-B groups=8 K=4096 rank=32 M=65 " + "n_tok=30 (token-tail nt=6)", + kQ8Type); + const bool gate_ok = run_prefill_gate_guards(model); + ok = prefill9_ok && prefill30_ok && prefill128_ok && + prefill_tail9_ok && + prefill_tail128_ok && prefill_single9_ok && + prefill_single128_ok && prefill_pair_ok && attention9_ok && + attention30_ok && attention128_ok && attention_q8_9_ok && + attention_q8_30_ok && gate_ok && ok; + if (run_prefill_long) { + // A 64 MiB activation and a roughly 0.5 Gi-op projection stress + // arbitrary token-grid tails without the much slower CPU oracle. + const bool long_ok = run_prefill_parity_case( + model, 4096u, model.weight1_offset, kM1, false, + "prefill stress K=4096 M=33 n_tok=4096"); + ok = long_ok && ok; + } + } // Registered host ranges must be released before their aligned backing // allocation is destroyed. ds4_gpu_cleanup(); - std::fprintf(stderr, "ROCm Q4 dense/pair oracle: %s\n", + std::fprintf(stderr, "ROCm Q4 dense/pair/prefill oracle: %s\n", ok ? "PASS" : "FAIL"); return ok ? 0 : 1; } From a7d41d4c1ad58c8faf53f4607d9aeb4fec050507 Mon Sep 17 00:00:00 2001 From: Giorgio Oppo Date: Thu, 20 Aug 2026 13:51:50 +0200 Subject: [PATCH 044/189] rocm: guard Metal-only Q4 overlap policy --- ds4.c | 2 +- ds4.h | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/ds4.c b/ds4.c index 9727b8e5e..33d137209 100644 --- a/ds4.c +++ b/ds4.c @@ -70666,7 +70666,7 @@ static int ds4_session_eval_probe_tp(ds4_session *s, int token, bool probe_mtp, return rc; } -#ifndef DS4_NO_GPU +#if !defined(DS4_NO_GPU) && defined(__APPLE__) /* Small fail-closed policy shared by runtime admission and its test hook. * Detailed graph/layout checks remain below; this layer owns the externally * controllable arm and the state classes that must never reach multi-queue diff --git a/ds4.h b/ds4.h index 4a5f2dfe2..ba2195d71 100644 --- a/ds4.h +++ b/ds4.h @@ -467,9 +467,11 @@ int ds4_test_speculative_delta_sample(const float *target_logits, int ds4_test_argmax_excluding_logits(const float *logits, uint32_t n_vocab, int excluded_id); uint64_t ds4_test_mixed_native_count(void); +#if defined(__APPLE__) int ds4_test_q4_stream_overlap_policy( int count, bool resident, bool ssd_streaming, bool quality); #endif +#endif int ds4_session_top_logprobs(ds4_session *s, ds4_token_score *out, int k); int ds4_session_token_logprob(ds4_session *s, int token, ds4_token_score *out); int ds4_session_copy_logits(ds4_session *s, float *out, int cap); From f0c70f90e41ca303bf8c10a9786382c161d75a05 Mon Sep 17 00:00:00 2001 From: Giorgio Oppo Date: Thu, 20 Aug 2026 19:16:44 +0200 Subject: [PATCH 045/189] rocm: default Q4 TILE8 and group attention decode --- ds4.c | 30 +-- ds4_cuda.cu | 6 +- ds4_gpu.h | 2 + ds4_rocm_unavailable.cu | 1 - rocm/ds4_rocm_q4.cuh | 143 ++++++++++++- tests/test_rocm_q4_dense_pair.cpp | 320 +++++++++++++++++++++++++++++- 6 files changed, 470 insertions(+), 32 deletions(-) diff --git a/ds4.c b/ds4.c index 33d137209..4f571679a 100644 --- a/ds4.c +++ b/ds4.c @@ -27488,20 +27488,22 @@ static bool metal_graph_attention_output_dense_quant_low( group_cnt, heads) != 0; } - /* Specialized Q4_K low projection where the backend has it (Metal); a - * zero return (the CUDA/ROCm stub) falls through to the generic - * per-group dense-quant loop below instead of failing the layer. */ - if (out_a->type == DS4_TENSOR_Q4_K && - ds4_gpu_attention_output_low_q4_K_slice_tensor(low, - model->map, - model->size, - out_a->abs_offset, - group_dim, - rank, - group0, - group_cnt, - heads) != 0) { - return true; + /* Specialized Q4_K low projection where the backend has it. Zero falls + * through to the generic per-group loop; a negative REQUIRE/error result + * fails closed after a backend may have submitted work. */ + if (out_a->type == DS4_TENSOR_Q4_K) { + const int q4_slice_rc = + ds4_gpu_attention_output_low_q4_K_slice_tensor(low, + model->map, + model->size, + out_a->abs_offset, + group_dim, + rank, + group0, + group_cnt, + heads); + if (q4_slice_rc > 0) return true; + if (q4_slice_rc < 0) return false; } uint64_t row_bytes = 0; if (!metal_graph_dense_quant_row_bytes(out_a, group_dim, &row_bytes)) return false; diff --git a/ds4_cuda.cu b/ds4_cuda.cu index 07382069e..ce6a9cdf4 100644 --- a/ds4_cuda.cu +++ b/ds4_cuda.cu @@ -42555,9 +42555,9 @@ extern "C" int ds4_gpu_attention_output_low_q4_K_slice_tensor( (int)rank, (int)group_dim, (int)group_cnt, stream); if (rc != 0) { /* NOT_APPLICABLE is pre-enqueue and cleanly retries the established - * per-group loop. A negative result may follow a launch; returning - * zero still avoids consuming a possibly incomplete candidate. */ - return 0; + * per-group loop. A negative result may follow a launch and must + * propagate so the graph cannot replay work over a partial result. */ + return rc < 0 ? -1 : 0; } if (!oracle) return 1; diff --git a/ds4_gpu.h b/ds4_gpu.h index 999c73c58..1331176dc 100644 --- a/ds4_gpu.h +++ b/ds4_gpu.h @@ -2656,6 +2656,8 @@ int ds4_gpu_attention_output_low_q8_tensor( uint64_t rank, uint32_t n_groups, const ds4_gpu_tensor *heads); +/* Q4_K grouped low projection: positive is success, zero is a clean + * pre-enqueue fallback, and negative is a required/possibly-enqueued failure. */ int ds4_gpu_attention_output_low_q4_K_slice_tensor( ds4_gpu_tensor *low, const void *model_map, diff --git a/ds4_rocm_unavailable.cu b/ds4_rocm_unavailable.cu index ed91a7697..2282971fc 100644 --- a/ds4_rocm_unavailable.cu +++ b/ds4_rocm_unavailable.cu @@ -9,7 +9,6 @@ ROCM_UNAVAILABLE_INT(ds4_gpu_add_xdev_tensor) ROCM_UNAVAILABLE_INT(ds4_gpu_attention_decode_rows_rope_tensor) -ROCM_UNAVAILABLE_INT(ds4_gpu_attention_output_low_q4_K_slice_tensor) ROCM_UNAVAILABLE_INT(ds4_gpu_attention_output_low_q8_rows_exact_tensor) ROCM_UNAVAILABLE_INT(ds4_gpu_attention_prefill_raw_heads_range_tensor) ROCM_UNAVAILABLE_INT(ds4_gpu_attention_prefill_static_mixed_heads_range_tensor) diff --git a/rocm/ds4_rocm_q4.cuh b/rocm/ds4_rocm_q4.cuh index a5e193018..de28e792b 100644 --- a/rocm/ds4_rocm_q4.cuh +++ b/rocm/ds4_rocm_q4.cuh @@ -36,6 +36,28 @@ __global__ static void rocm_matmul_q4_K_dense_kernel( if (lane == 0u) out[(uint64_t)tok * out_dim + row] = acc; } +/* One independent activation row and Q4_K matrix per output group. This is + * the canonical dense decode walk with only a group grid dimension added. */ +__global__ static void rocm_matmul_q4_K_dense_grouped_decode_kernel( + float *out, const char *w_base, const cuda_block_q8_K *xq, + uint64_t row_bytes, uint32_t xq_blocks, uint32_t out_dim, + uint32_t n_groups) { + const uint32_t lane = threadIdx.x & 7u; + const uint32_t row_lane = threadIdx.x >> 3u; + const uint32_t row = blockIdx.x * 32u + row_lane; + const uint32_t group = blockIdx.z; + if (group >= n_groups || row >= out_dim) return; + const cuda_block_q8_K *xqb = xq + (uint64_t)group * xq_blocks; + const cuda_block_q4_K *wr = reinterpret_cast( + w_base + ((uint64_t)group * out_dim + row) * row_bytes); + float acc = 0.0f; + for (uint32_t b = lane; b < xq_blocks; b += 8u) { + acc += dev_dot_q4_K_q8_K_block(wr + b, xqb + b); + } + acc = quarter_warp_sum_f32(acc, lane); + if (lane == 0u) out[(uint64_t)group * out_dim + row] = acc; +} + /* Dense Q4_K prefill tile for RDNA/ROCm. * * The legacy kernel gives one eight-lane group a single (token,row) dot. As @@ -280,13 +302,14 @@ static int rocm_q4_K_dense_pair_requested(void) { static int rocm_q4_K_prefill_tile8_scope(uint64_t n_tok) { /* Keep decode/speculative micro-batches on the latency-oriented legacy * kernel. 4096 is DS4's largest supported prefill chunk and bounds the - * A/B surface while this path remains opt-in. */ + * validated tiled-prefill surface. */ return n_tok > 8u && n_tok <= 4096u; } static int rocm_q4_K_prefill_tile8_requested(void) { - return getenv("DS4_ROCM_ENABLE_Q4_PREFILL_TILE8") != NULL && - getenv("DS4_ROCM_DISABLE_Q4_PREFILL_TILE8") == NULL; + /* TILE8 is the ROCm Q4 prefill default. Keep the old ENABLE variable + * harmlessly compatible and retain one authoritative rollback switch. */ + return getenv("DS4_ROCM_DISABLE_Q4_PREFILL_TILE8") == NULL; } static int rocm_q4_K_prefill_tile8_required(void) { @@ -299,6 +322,43 @@ static uint64_t g_rocm_q4_prefill_tile8_attention_batch_calls; static uint64_t g_rocm_q4_prefill_tile8_tokens; static int g_rocm_q4_prefill_tile8_report_registered; +static uint64_t g_rocm_q4_grouped_attn_a_calls; +static uint64_t g_rocm_q4_grouped_attn_a_dispatches; +static uint64_t g_rocm_q4_grouped_attn_a_groups; +static uint64_t g_rocm_q4_grouped_attn_a_fallbacks; +static uint64_t g_rocm_q4_grouped_attn_a_failures; +static int g_rocm_q4_grouped_attn_a_report_registered; + +static void rocm_q4_K_grouped_attn_a_report(void) { + fprintf(stderr, + "ds4: ROCm Q4_K grouped attention-A decode stats: " + "calls=%llu dispatches=%llu groups=%llu fallbacks=%llu failures=%llu\n", + (unsigned long long)g_rocm_q4_grouped_attn_a_calls, + (unsigned long long)g_rocm_q4_grouped_attn_a_dispatches, + (unsigned long long)g_rocm_q4_grouped_attn_a_groups, + (unsigned long long)g_rocm_q4_grouped_attn_a_fallbacks, + (unsigned long long)g_rocm_q4_grouped_attn_a_failures); +} + +static int rocm_q4_K_grouped_attn_a_result(int rc, uint32_t n_groups) { + if (getenv("DS4_ROCM_Q4_GROUPED_ATTN_A_STATS") != NULL) { + if (!g_rocm_q4_grouped_attn_a_report_registered) { + g_rocm_q4_grouped_attn_a_report_registered = 1; + (void)atexit(rocm_q4_K_grouped_attn_a_report); + } + g_rocm_q4_grouped_attn_a_calls++; + if (rc > 0) { + g_rocm_q4_grouped_attn_a_dispatches++; + g_rocm_q4_grouped_attn_a_groups += n_groups; + } else if (rc < 0) { + g_rocm_q4_grouped_attn_a_failures++; + } else { + g_rocm_q4_grouped_attn_a_fallbacks++; + } + } + return rc; +} + static void rocm_q4_K_prefill_tile8_report(void) { fprintf(stderr, "ds4: ROCm Q4_K prefill tile8 stats: " @@ -504,6 +564,83 @@ extern "C" int ds4_gpu_matmul_q4_K_pair_tensor( return cuda_ok(cudaGetLastError(), "q4_K dense pair1 matmul launch"); } +extern "C" int ds4_gpu_attention_output_low_q4_K_slice_tensor( + ds4_gpu_tensor *low, const void *model_map, uint64_t model_size, + uint64_t out_a_offset, uint64_t group_dim, uint64_t rank, + uint32_t group0, uint32_t group_cnt, + const ds4_gpu_tensor *heads) { + const int disabled = + getenv("DS4_ROCM_DISABLE_Q4_GROUPED_ATTN_A") != NULL; + const int required = + getenv("DS4_ROCM_REQUIRE_Q4_GROUPED_ATTN_A") != NULL; + const int enabled = + getenv("DS4_ROCM_ENABLE_Q4_GROUPED_ATTN_A") != NULL; + /* DISABLE is authoritative; REQUIRE reports that rollback as a failure + * instead of allowing the graph to false-green through its fallback. */ + if (disabled) { + if (required) { + fprintf(stderr, + "ds4: required ROCm Q4_K grouped attention-A decode " + "is disabled\n"); + } + return rocm_q4_K_grouped_attn_a_result(required ? -1 : 0, 0u); + } + if (!enabled && !required) { + return rocm_q4_K_grouped_attn_a_result(0, 0u); + } + const int pre_enqueue_failure = required ? -1 : 0; + if (!low || !heads || !model_map || group_dim == 0u || rank == 0u || + group_cnt == 0u || group_dim > UINT32_MAX || rank > UINT32_MAX || + group_cnt > UINT16_MAX || (group_dim % CUDA_QK_K) != 0u || + group0 > UINT32_MAX - group_cnt) { + return rocm_q4_K_grouped_attn_a_result(pre_enqueue_failure, 0u); + } + + const uint64_t blocks = group_dim / CUDA_QK_K; + uint64_t row_bytes = 0, group_weight_bytes = 0, group_skip = 0; + uint64_t selected_weight_bytes = 0, selected_offset = 0; + if (blocks == 0u || + !cuda_u64_mul_checked(blocks, sizeof(cuda_block_q4_K), &row_bytes) || + !cuda_u64_mul_checked(rank, row_bytes, &group_weight_bytes) || + !cuda_u64_mul_checked(group0, group_weight_bytes, &group_skip) || + !cuda_u64_mul_checked(group_cnt, group_weight_bytes, + &selected_weight_bytes) || + !cuda_u64_add_checked(out_a_offset, group_skip, &selected_offset) || + !cuda_model_range_fits(model_size, selected_offset, + selected_weight_bytes) || + !cuda_tensor_has_elems2(heads, group_cnt, group_dim, sizeof(float)) || + !cuda_tensor_has_elems2(low, group_cnt, rank, sizeof(float))) { + return rocm_q4_K_grouped_attn_a_result(pre_enqueue_failure, 0u); + } + + const char *w = cuda_model_range_ptr( + model_map, selected_offset, selected_weight_bytes, + "q4_K grouped attention output A decode"); + cuda_block_q8_K *xq = rocm_q4_K_prequant_alloc( + group_cnt, blocks, "q4_K grouped attention output A decode prequant"); + if (!w || !xq) { + return rocm_q4_K_grouped_attn_a_result(pre_enqueue_failure, 0u); + } + + const dim3 qgrid((unsigned)blocks, group_cnt, 1u); + q8_K_quantize_kernel<<>>( + xq, reinterpret_cast(heads->ptr), + (uint32_t)group_dim, group_cnt); + if (!cuda_ok(cudaGetLastError(), + "q4_K grouped attention output A decode quantize launch")) { + return rocm_q4_K_grouped_attn_a_result(-1, 0u); + } + const dim3 grid((unsigned)((rank - 1u) / 32u + 1u), 1u, group_cnt); + rocm_matmul_q4_K_dense_grouped_decode_kernel<<>>( + reinterpret_cast(low->ptr), w, xq, row_bytes, + (uint32_t)blocks, (uint32_t)rank, group_cnt); + if (!cuda_ok(cudaGetLastError(), + "q4_K grouped attention output A decode matmul launch")) { + return rocm_q4_K_grouped_attn_a_result(-1, 0u); + } + return rocm_q4_K_grouped_attn_a_result(1, group_cnt); +} + /* Quantize token-major [token][group][K] rows once, then apply group-major * [group][out_row][K] Q4_K weights directly into token-major output. A * return of -1 means the quantize launch was accepted and callers must not diff --git a/tests/test_rocm_q4_dense_pair.cpp b/tests/test_rocm_q4_dense_pair.cpp index 9c43e28f3..bbc0df1e6 100644 --- a/tests/test_rocm_q4_dense_pair.cpp +++ b/tests/test_rocm_q4_dense_pair.cpp @@ -44,6 +44,11 @@ constexpr uint32_t kAttnRank = 32u; constexpr uint32_t kAttnGroups = 8u; constexpr uint32_t kAttnLowDim = kAttnGroups * kAttnRank; constexpr uint32_t kAttnOutDim = 65u; +constexpr uint32_t kDecodeAttnGroupDim = 4096u; +constexpr uint32_t kDecodeAttnRank = 1024u; +constexpr uint32_t kDecodeAttnGroups = 8u; +constexpr uint32_t kDecodeAttnLowDim = + kDecodeAttnGroups * kDecodeAttnRank; constexpr size_t kOutputGuardFloats = 257u; constexpr float kCpuAbsTolerance = 2.0e-3f; constexpr float kCpuRelTolerance = 3.0e-5f; @@ -55,6 +60,14 @@ constexpr const char *kPrefillDisable = "DS4_ROCM_DISABLE_Q4_PREFILL_TILE8"; constexpr const char *kPrefillRequire = "DS4_ROCM_REQUIRE_Q4_PREFILL_TILE8"; +constexpr const char *kGroupedDecodeEnable = + "DS4_ROCM_ENABLE_Q4_GROUPED_ATTN_A"; +constexpr const char *kGroupedDecodeDisable = + "DS4_ROCM_DISABLE_Q4_GROUPED_ATTN_A"; +constexpr const char *kGroupedDecodeRequire = + "DS4_ROCM_REQUIRE_Q4_GROUPED_ATTN_A"; +constexpr const char *kGroupedDecodeStats = + "DS4_ROCM_Q4_GROUPED_ATTN_A_STATS"; struct block_q4_K_test { uint16_t d; @@ -98,6 +111,7 @@ struct aligned_model { uint64_t weight0_offset = 0; uint64_t weight1_offset = 0; uint64_t attn_a_offset = 0; + uint64_t decode_attn_a_offset = 0; uint64_t attn_b_offset = 0; uint64_t attn_b_q8_offset = 0; uint64_t tail_k1024_offset = 0; @@ -253,6 +267,12 @@ bool make_model(aligned_model *model) { (kAttnGroupDim / kQkK) * sizeof(block_q4_K_test); const uint64_t attn_a_bytes = (uint64_t)kAttnGroups * kAttnRank * attn_a_row_bytes; + const uint64_t decode_attn_a_row_bytes = + (kDecodeAttnGroupDim / kQkK) * sizeof(block_q4_K_test); + const uint64_t decode_attn_a_group_bytes = + (uint64_t)kDecodeAttnRank * decode_attn_a_row_bytes; + const uint64_t decode_attn_a_bytes = + (uint64_t)kDecodeAttnGroups * decode_attn_a_group_bytes; const uint64_t attn_b_row_bytes = (kAttnLowDim / kQkK) * sizeof(block_q4_K_test); const uint64_t attn_b_bytes = @@ -268,8 +288,10 @@ bool make_model(aligned_model *model) { model->weight1_offset = round_up(weight0_bytes, page); model->attn_a_offset = round_up( model->weight1_offset + weight1_bytes, page); - model->attn_b_offset = round_up( + model->decode_attn_a_offset = round_up( model->attn_a_offset + attn_a_bytes, page); + model->attn_b_offset = round_up( + model->decode_attn_a_offset + decode_attn_a_bytes, page); model->tail_k1024_offset = round_up( model->attn_b_offset + attn_b_bytes, page); model->attn_b_q8_offset = round_up( @@ -291,6 +313,13 @@ bool make_model(aligned_model *model) { fill_q4_rows(reinterpret_cast( model->data + model->attn_a_offset), kAttnGroups * kAttnRank, kAttnGroupDim, 0x243f6a88u); + for (uint32_t group = 0; group < kDecodeAttnGroups; group++) { + fill_q4_rows(reinterpret_cast( + model->data + model->decode_attn_a_offset + + (uint64_t)group * decode_attn_a_group_bytes), + kDecodeAttnRank, kDecodeAttnGroupDim, + 0xd1b54a35u ^ (group * 0x9e3779b9u)); + } fill_q4_rows(reinterpret_cast( model->data + model->attn_b_offset), kAttnOutDim, kAttnLowDim, 0x85a308d3u); @@ -635,8 +664,8 @@ bool run_prefill_parity_case(const aligned_model &model, uint32_t n_tokens, env_snapshot disable(kPrefillDisable); env_snapshot require(kPrefillRequire); - // The authoritative rollback is the reference even after a future - // default-on promotion of the tiled path. + // The authoritative rollback remains the reference now that the tiled + // path is default-on. (void)unsetenv(kPrefillEnable); (void)setenv(kPrefillDisable, "1", 1); (void)unsetenv(kPrefillRequire); @@ -644,9 +673,10 @@ bool run_prefill_parity_case(const aligned_model &model, uint32_t n_tokens, legacy_gpu.ptr, model.data, model.size, offset, kQ4Type, in_dim, out_dim, x_gpu.ptr, n_tokens); - // REQUIRE makes a silently ineligible candidate a test failure instead - // of comparing the legacy kernel with itself. - (void)setenv(kPrefillEnable, "1", 1); + // TILE8 is default-on. Leave the legacy ENABLE unset and use REQUIRE so + // a silently ineligible default cannot compare the legacy kernel with + // itself. + (void)unsetenv(kPrefillEnable); (void)unsetenv(kPrefillDisable); (void)setenv(kPrefillRequire, "1", 1); const int candidate_rc = ds4_gpu_matmul_quant_tensor( @@ -706,13 +736,35 @@ bool run_prefill_gate_guards(const aligned_model &model) { env_snapshot enable(kPrefillEnable); env_snapshot disable(kPrefillDisable); env_snapshot require(kPrefillRequire); + + /* REQUIRE with neither ENABLE nor DISABLE proves that TILE8 is selected + * by the default policy. */ + (void)unsetenv(kPrefillEnable); + (void)unsetenv(kPrefillDisable); + (void)setenv(kPrefillRequire, "1", 1); + const int default_rc = ds4_gpu_matmul_quant_tensor( + out_gpu.ptr, model.data, model.size, model.weight1_offset, kQ4Type, + kK, kM1, x_gpu.ptr, n_tokens); + bool ok = default_rc != 0; + if (default_rc == 0) { + std::fprintf(stderr, + "prefill default-on REQUIRE: expected success got=%d FAIL\n", + default_rc); + } + std::vector default_out(output_count); + if (!read_tensor(out_gpu.ptr, &default_out)) return false; + ok = output_guard_unchanged( + default_out, sentinel, (size_t)n_tokens * kM1, + "prefill default-on output canary") && ok; + if (!write_tensor(out_gpu.ptr, sentinel)) return false; + (void)setenv(kPrefillEnable, "1", 1); (void)setenv(kPrefillDisable, "1", 1); (void)setenv(kPrefillRequire, "1", 1); const int rc = ds4_gpu_matmul_quant_tensor( out_gpu.ptr, model.data, model.size, model.weight1_offset, kQ4Type, kK, kM1, x_gpu.ptr, n_tokens); - bool ok = rc == 0; + ok = rc == 0 && ok; if (rc != 0) { std::fprintf(stderr, "prefill DISABLE+REQUIRE: expected rc=0 got=%d FAIL\n", @@ -768,7 +820,7 @@ bool run_prefill_pair_case(const aligned_model &model) { // The prefill pair is a distinct path: it must not depend on the legacy // decode-pair opt-in, whose <=8-token behavior is tested separately. - (void)setenv(kPrefillEnable, "1", 1); + (void)unsetenv(kPrefillEnable); (void)unsetenv(kPrefillDisable); (void)setenv(kPrefillRequire, "1", 1); (void)unsetenv("DS4_ROCM_ENABLE_Q4_DENSE_PAIR"); @@ -937,7 +989,7 @@ bool run_attention_prefill_case(const aligned_model &model, return false; } - (void)setenv(kPrefillEnable, "1", 1); + (void)unsetenv(kPrefillEnable); (void)unsetenv(kPrefillDisable); (void)setenv(kPrefillRequire, "1", 1); const int candidate_rc = ds4_gpu_attention_output_q4_K_batch_tensor( @@ -1023,6 +1075,227 @@ bool run_attention_prefill_case(const aligned_model &model, return ok; } +bool run_grouped_attention_decode_case(const aligned_model &model) { + const uint64_t row_bytes = + (kDecodeAttnGroupDim / kQkK) * sizeof(block_q4_K_test); + const uint64_t group_weight_bytes = + (uint64_t)kDecodeAttnRank * row_bytes; + const size_t heads_count = + (size_t)kDecodeAttnGroups * kDecodeAttnGroupDim; + const size_t logical_count = kDecodeAttnLowDim; + const size_t allocation_count = logical_count + kOutputGuardFloats; + std::vector heads_host; + fill_activation(&heads_host, kDecodeAttnGroups, kDecodeAttnGroupDim); + const std::vector sentinel = sentinel_values(allocation_count); + + tensor_owner heads_gpu(heads_count * sizeof(float)); + tensor_owner legacy_gpu(allocation_count * sizeof(float)); + tensor_owner candidate_gpu(allocation_count * sizeof(float)); + if (!heads_gpu.ptr || !legacy_gpu.ptr || !candidate_gpu.ptr || + !write_tensor(heads_gpu.ptr, heads_host) || + !write_tensor(legacy_gpu.ptr, sentinel) || + !write_tensor(candidate_gpu.ptr, sentinel)) { + std::fprintf(stderr, "grouped attention-A decode: setup FAIL\n"); + return false; + } + + /* Eight standalone decode calls are the bitwise oracle. Per-group seeds + * and activation rows make a wrong weight/input group immediately visible. */ + for (uint32_t group = 0; group < kDecodeAttnGroups; group++) { + tensor_owner head_group(ds4_gpu_tensor_view( + heads_gpu.ptr, + (uint64_t)group * kDecodeAttnGroupDim * sizeof(float), + (uint64_t)kDecodeAttnGroupDim * sizeof(float))); + tensor_owner low_group(ds4_gpu_tensor_view( + legacy_gpu.ptr, + (uint64_t)group * kDecodeAttnRank * sizeof(float), + (uint64_t)kDecodeAttnRank * sizeof(float))); + if (!head_group.ptr || !low_group.ptr || + ds4_gpu_matmul_quant_tensor( + low_group.ptr, model.data, model.size, + model.decode_attn_a_offset + + (uint64_t)group * group_weight_bytes, + kQ4Type, kDecodeAttnGroupDim, kDecodeAttnRank, + head_group.ptr, 1u) == 0) { + std::fprintf(stderr, + "grouped attention-A legacy group=%u FAIL\n", group); + return false; + } + } + + env_snapshot enable(kGroupedDecodeEnable); + env_snapshot disable(kGroupedDecodeDisable); + env_snapshot require(kGroupedDecodeRequire); + env_snapshot stats(kGroupedDecodeStats); + (void)setenv(kGroupedDecodeStats, "1", 1); + (void)unsetenv(kGroupedDecodeEnable); + (void)unsetenv(kGroupedDecodeDisable); + (void)unsetenv(kGroupedDecodeRequire); + + const int default_rc = ds4_gpu_attention_output_low_q4_K_slice_tensor( + candidate_gpu.ptr, model.data, model.size, + model.decode_attn_a_offset, kDecodeAttnGroupDim, kDecodeAttnRank, + 0u, kDecodeAttnGroups, heads_gpu.ptr); + bool ok = default_rc == 0; + if (default_rc != 0) { + std::fprintf(stderr, + "grouped attention-A default gate: expected rc=0 got=%d FAIL\n", + default_rc); + } + ok = unchanged_after_rejected_call( + candidate_gpu.ptr, sentinel, + "grouped attention-A disabled-by-default preserves output") && ok; + + (void)setenv(kGroupedDecodeEnable, "1", 1); + (void)setenv(kGroupedDecodeDisable, "1", 1); + const int disabled_only_rc = + ds4_gpu_attention_output_low_q4_K_slice_tensor( + candidate_gpu.ptr, model.data, model.size, + model.decode_attn_a_offset, kDecodeAttnGroupDim, kDecodeAttnRank, + 0u, kDecodeAttnGroups, heads_gpu.ptr); + if (disabled_only_rc != 0) { + std::fprintf(stderr, + "grouped attention-A ENABLE+DISABLE: expected rc=0 got=%d FAIL\n", + disabled_only_rc); + ok = false; + } + ok = unchanged_after_rejected_call( + candidate_gpu.ptr, sentinel, + "grouped attention-A DISABLE dominates ENABLE") && ok; + + (void)setenv(kGroupedDecodeRequire, "1", 1); + const int disabled_rc = + ds4_gpu_attention_output_low_q4_K_slice_tensor( + candidate_gpu.ptr, model.data, model.size, + model.decode_attn_a_offset, kDecodeAttnGroupDim, kDecodeAttnRank, + 0u, kDecodeAttnGroups, heads_gpu.ptr); + if (disabled_rc != -1) { + std::fprintf(stderr, + "grouped attention-A DISABLE+REQUIRE: expected rc=-1 got=%d FAIL\n", + disabled_rc); + ok = false; + } + ok = unchanged_after_rejected_call( + candidate_gpu.ptr, sentinel, + "grouped attention-A DISABLE dominates REQUIRE") && ok; + + (void)unsetenv(kGroupedDecodeDisable); + const int invalid_rc = ds4_gpu_attention_output_low_q4_K_slice_tensor( + candidate_gpu.ptr, model.data, model.size, model.size - 16u, + kDecodeAttnGroupDim, kDecodeAttnRank, 0u, kDecodeAttnGroups, + heads_gpu.ptr); + if (invalid_rc != -1) { + std::fprintf(stderr, + "grouped attention-A REQUIRE range guard: expected rc=-1 got=%d FAIL\n", + invalid_rc); + ok = false; + } + ok = unchanged_after_rejected_call( + candidate_gpu.ptr, sentinel, + "grouped attention-A rejected range preserves output") && ok; + + const int candidate_rc = ds4_gpu_attention_output_low_q4_K_slice_tensor( + candidate_gpu.ptr, model.data, model.size, + model.decode_attn_a_offset, kDecodeAttnGroupDim, kDecodeAttnRank, + 0u, kDecodeAttnGroups, heads_gpu.ptr); + std::vector legacy_host(allocation_count); + std::vector candidate_host(allocation_count); + if (candidate_rc != 1 || !read_tensor(legacy_gpu.ptr, &legacy_host) || + !read_tensor(candidate_gpu.ptr, &candidate_host)) { + std::fprintf(stderr, + "grouped attention-A candidate dispatch/read rc=%d FAIL\n", + candidate_rc); + return false; + } + ok = output_guard_unchanged( + legacy_host, sentinel, logical_count, + "grouped attention-A legacy output canary") && ok; + ok = output_guard_unchanged( + candidate_host, sentinel, logical_count, + "grouped attention-A candidate output canary") && ok; + legacy_host.resize(logical_count); + candidate_host.resize(logical_count); + ok = bitwise_equal(candidate_host, legacy_host, + "grouped attention-A candidate vs 8 legacy calls") && ok; + + /* A non-zero weight-group origin consumes a compact input/output slice. + * Reuse groups 3 and 4 from the full fixture to verify both the weight + * skip and local grouped layout. */ + constexpr uint32_t subset_group0 = 3u; + constexpr uint32_t subset_group_cnt = 2u; + const size_t subset_logical_count = + (size_t)subset_group_cnt * kDecodeAttnRank; + const std::vector subset_sentinel = + sentinel_values(subset_logical_count + kOutputGuardFloats); + tensor_owner subset_heads(ds4_gpu_tensor_view( + heads_gpu.ptr, + (uint64_t)subset_group0 * kDecodeAttnGroupDim * sizeof(float), + (uint64_t)subset_group_cnt * kDecodeAttnGroupDim * sizeof(float))); + tensor_owner subset_legacy(subset_sentinel.size() * sizeof(float)); + tensor_owner subset_candidate(subset_sentinel.size() * sizeof(float)); + if (!subset_heads.ptr || !subset_legacy.ptr || !subset_candidate.ptr || + !write_tensor(subset_legacy.ptr, subset_sentinel) || + !write_tensor(subset_candidate.ptr, subset_sentinel)) { + std::fprintf(stderr, "grouped attention-A subset: setup FAIL\n"); + return false; + } + for (uint32_t i = 0; i < subset_group_cnt; i++) { + tensor_owner head_group(ds4_gpu_tensor_view( + subset_heads.ptr, + (uint64_t)i * kDecodeAttnGroupDim * sizeof(float), + (uint64_t)kDecodeAttnGroupDim * sizeof(float))); + tensor_owner low_group(ds4_gpu_tensor_view( + subset_legacy.ptr, + (uint64_t)i * kDecodeAttnRank * sizeof(float), + (uint64_t)kDecodeAttnRank * sizeof(float))); + if (!head_group.ptr || !low_group.ptr || + ds4_gpu_matmul_quant_tensor( + low_group.ptr, model.data, model.size, + model.decode_attn_a_offset + + (uint64_t)(subset_group0 + i) * group_weight_bytes, + kQ4Type, kDecodeAttnGroupDim, kDecodeAttnRank, + head_group.ptr, 1u) == 0) { + std::fprintf(stderr, + "grouped attention-A subset legacy group=%u FAIL\n", + subset_group0 + i); + return false; + } + } + const int subset_rc = ds4_gpu_attention_output_low_q4_K_slice_tensor( + subset_candidate.ptr, model.data, model.size, + model.decode_attn_a_offset, kDecodeAttnGroupDim, kDecodeAttnRank, + subset_group0, subset_group_cnt, subset_heads.ptr); + std::vector subset_legacy_host(subset_sentinel.size()); + std::vector subset_candidate_host(subset_sentinel.size()); + if (subset_rc != 1 || + !read_tensor(subset_legacy.ptr, &subset_legacy_host) || + !read_tensor(subset_candidate.ptr, &subset_candidate_host)) { + std::fprintf(stderr, + "grouped attention-A subset dispatch/read rc=%d FAIL\n", + subset_rc); + return false; + } + ok = output_guard_unchanged( + subset_legacy_host, subset_sentinel, subset_logical_count, + "grouped attention-A subset legacy canary") && ok; + ok = output_guard_unchanged( + subset_candidate_host, subset_sentinel, subset_logical_count, + "grouped attention-A subset candidate canary") && ok; + subset_legacy_host.resize(subset_logical_count); + subset_candidate_host.resize(subset_logical_count); + ok = bitwise_equal( + subset_candidate_host, subset_legacy_host, + "grouped attention-A subset group0=3 count=2 vs legacy") && ok; + std::fprintf(stderr, + "grouped attention-A decode groups=8 K=4096 rank=1024: " + "default=%d disabled=%d disabled_required=%d invalid=%d " + "candidate=%d subset=%d %s\n", + default_rc, disabled_only_rc, disabled_rc, invalid_rc, + candidate_rc, subset_rc, + ok ? "PASS" : "FAIL"); + return ok; +} + bool run_dense_guards(const aligned_model &model) { std::vector x; fill_activation(&x, 1u); @@ -1156,25 +1429,36 @@ int main(int argc, char **argv) { bool run_dense = true; bool run_pair = true; bool run_prefill = true; + bool run_grouped_decode = true; bool run_prefill_long = false; if (argc == 2 && std::strcmp(argv[1], "--dense") == 0) { run_pair = false; run_prefill = false; + run_grouped_decode = false; } else if (argc == 2 && std::strcmp(argv[1], "--pair") == 0) { run_dense = false; run_prefill = false; + run_grouped_decode = false; } else if (argc == 2 && std::strcmp(argv[1], "--prefill") == 0) { run_dense = false; run_pair = false; + run_grouped_decode = false; + } else if (argc == 2 && + std::strcmp(argv[1], "--grouped-decode") == 0) { + run_dense = false; + run_pair = false; + run_prefill = false; } else if (argc == 2 && std::strcmp(argv[1], "--prefill-long") == 0) { run_dense = false; run_pair = false; + run_grouped_decode = false; run_prefill_long = true; } else if (argc > 1 && !(argc == 2 && std::strcmp(argv[1], "--all") == 0)) { std::fprintf(stderr, - "usage: %s [--all|--dense|--pair|--prefill|--prefill-long]\n", + "usage: %s [--all|--dense|--pair|--grouped-decode|" + "--prefill|--prefill-long]\n", argv[0]); return 2; } @@ -1182,9 +1466,17 @@ int main(int argc, char **argv) { env_snapshot prefill_enable(kPrefillEnable); env_snapshot prefill_disable(kPrefillDisable); env_snapshot prefill_require(kPrefillRequire); + env_snapshot grouped_enable(kGroupedDecodeEnable); + env_snapshot grouped_disable(kGroupedDecodeDisable); + env_snapshot grouped_require(kGroupedDecodeRequire); + env_snapshot grouped_stats(kGroupedDecodeStats); (void)unsetenv(kPrefillEnable); (void)unsetenv(kPrefillDisable); (void)unsetenv(kPrefillRequire); + (void)unsetenv(kGroupedDecodeEnable); + (void)unsetenv(kGroupedDecodeDisable); + (void)unsetenv(kGroupedDecodeRequire); + (void)unsetenv(kGroupedDecodeStats); if (detect_rocm_device() <= 0) { const char *require_device = @@ -1238,10 +1530,16 @@ int main(int argc, char **argv) { ok = pair1_ok && pair3_ok && pair8_ok && pair_guard_ok && pair_opt_in_ok && ok; } + if (model_ready && run_grouped_decode) { + std::fprintf(stderr, + "ROCm Q4 grouped attention-A decode parity " + "(one grouped dispatch vs eight legacy calls):\n"); + ok = run_grouped_attention_decode_case(model) && ok; + } if (model_ready && run_prefill) { std::fprintf(stderr, "ROCm Q4 tiled prefill parity " - "(forced legacy vs ENABLE+REQUIRE):\n"); + "(forced DISABLE vs default+REQUIRE):\n"); const bool prefill9_ok = run_prefill_parity_case( model, 9u, model.weight0_offset, kM0, true, "prefill K=4096 M=65 n_tok=9"); From e3e0a7e52603a0a92b48210463a670a4d896a952 Mon Sep 17 00:00:00 2001 From: Giorgio Oppo Date: Thu, 20 Aug 2026 20:01:37 +0200 Subject: [PATCH 046/189] backend: add Metal Q4 prefill path and review fixes Add the token-tiled exact-N Q4 attention-output path and bitwise Metal oracle, promote the validated M1 IQ2 specialization, and avoid the intermediate mixed-split commit. Harden CUDA Q8-fold oracle cleanup, centralize standalone MMQ stubs, and document backend rollback and QA controls. --- ENVIRONMENT_VARIABLES.md | 93 ++++++ Makefile | 18 +- QA_BEFORE_RELEASES.md | 23 +- README.md | 21 +- cuda/mmq/ds4_mmq.cu | 89 ++++-- cuda/mmq/test/proto_gemm_dense_q8_d2r.cu | 10 - cuda/mmq/test/test_mmq_soa_tiles.cu | 11 - ds4_metal.m | 346 ++++++++++++++++++-- metal/moe.metal | 171 +++++++++- tests/test_metal_q4_attn_exactn.c | 382 +++++++++++++++++++++++ tests/test_mxfp4_cuda.cu | 10 - 11 files changed, 1075 insertions(+), 99 deletions(-) create mode 100644 ENVIRONMENT_VARIABLES.md create mode 100644 tests/test_metal_q4_attn_exactn.c diff --git a/ENVIRONMENT_VARIABLES.md b/ENVIRONMENT_VARIABLES.md new file mode 100644 index 000000000..d37cf54ea --- /dev/null +++ b/ENVIRONMENT_VARIABLES.md @@ -0,0 +1,93 @@ +# Performance and diagnostic environment variables + +Command-line options are the supported interface for normal inference. The +variables below are the curated, user-facing switches used to isolate optimized +paths, require test coverage, or collect diagnostics. They are not a promise +that every internal `getenv()` knob is a stable API. + +Most backend switches are cached on first use. Start a new process when +changing them. Unless a row says otherwise: + +- use the documented value `1`; many rollback switches are presence-based, so + setting them to `0` still enables the rollback and they must instead be unset; +- `DISABLE` or `NO` is the rollback switch and takes precedence; +- `REQUIRE` turns an eligible silent fallback into an error, so tests cannot + pass without exercising the intended path; +- `STATS`, `PROFILE`, and `ORACLE` are diagnostic and may perturb timing; +- benchmark controls and candidates in separate processes. + +## Metal + +| Variable | Default and purpose | +| --- | --- | +| `DS4_METAL_PREFILL_CHUNK=N` | Set the prefill cap when `--prefill-chunk` is absent; the CLI option takes precedence. This historical name is consumed by the shared graph planner rather than by a Metal kernel alone. | +| `DS4_METAL_NO_RESIDENCY=1` | Skip creation and residency requests for the model-view residency set. Diagnostic rollback for resident, non-streaming models. | +| `DS4_METAL_DISABLE_QUEUE_RESIDENCY_SET=1` | Still create, commit, and request the model residency set, but do not attach it to Metal command queues. This isolates queue-residency behavior without disabling the complete residency policy. | +| `DS4_METAL_STREAMING_EXPERT_NOCACHE=1` | Reopen the Metal SSD expert file with `F_NOCACHE` so streamed experts do not displace the dense working set from the page cache. Leave unset for cached `pread`. | +| `DS4_METAL_STREAMING_EXPERT_PREAD_SPLIT=N` | Split each expert read into 1–8 aligned requests. The automatic value is 1 below 64 configured cache experts and 4 at 64 or more. | +| `DS4_METAL_DISABLE_Q4_DENSE_PAIR=1` | Split the default Metal Q-A/KV Q4 pair back into two standalone projections. | +| `DS4_METAL_DISABLE_M1_IQ2_MID_ONLY=1` | Restore the canonical IQ2 address-table gate/up producer on the exact M1 SSD-streaming decode shape. The specialization is automatic by default. | +| `DS4_METAL_REQUIRE_M1_IQ2_MID_ONLY=1` | Fail closed when an otherwise eligible M1 IQ2 mid-only dispatch cannot use the specialization. | +| `DS4_METAL_ENABLE_M1_IQ2_MID_ONLY=1` | Compatibility alias for the former opt-in. It is harmless because the path is now automatic. | +| `DS4_METAL_DISABLE_IQ2_XXS_SSD_PREFILL_MM=1` | Restore sparse matvec for an eligible grouped IQ2 SSD-prefill chunk. | +| `DS4_METAL_REQUIRE_IQ2_XXS_SSD_PREFILL_MM=1` | Require grouped IQ2 SSD-prefill MM for eligible chunks and reject insufficient cache instead of silently falling back. | +| `DS4_METAL_ENABLE_STREAMING_PREFILL_EXPERT_READAHEAD=1` | Restore the historical `F_RDADVISE` plus parallel-`pread` sequence for cold-storage A/B tests. Normal grouped prefill skips the redundant hint. | + +The detailed Metal A/B contracts and expected oracle counters live in +[`QA_BEFORE_RELEASES.md`](QA_BEFORE_RELEASES.md). + +## CUDA Q4 and Q8 diagnostics + +| Variable | Default and purpose | +| --- | --- | +| `DS4_CUDA_DISABLE_Q4_DENSE_PAIR=1` | Split the Q-A/KV Q4 pair back into two standalone projections. | +| `DS4_CUDA_NO_Q4_GB10_FAST=1` | Umbrella rollback for the GB10-specific Q4 choices; it does not disable the older cross-CUDA dense pair. | +| `DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_BATCH=1` | Enable grouped attention-A for two-to-eight-token GB10 verifier batches. | +| `DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_BATCH=1` | Fail closed if that grouped batch path is unavailable. | +| `DS4_CUDA_Q4_GROUPED_ATTN_A_ORACLE=1` | Compare grouped attention-A with the canonical result and retain the canonical output. Disable graph capture for this diagnostic. | +| `DS4_CUDA_ENABLE_Q4_K1024_PERSISTENT=1` | Enable the experimental persistent-CTA kernel for the exact `32768x1024` Q4 shape. | +| `DS4_CUDA_NO_Q4_K1024_PERSISTENT=1` | Roll back the persistent K1024 experiment. | +| `DS4_CUDA_REQUIRE_Q4_K1024_PERSISTENT=1` | Require the K1024 candidate before enqueue instead of silently using canonical MMVQ. | +| `DS4_CUDA_ENABLE_Q8_FOLD=1` | Enable the experimental one-shot Q8_1 producer-to-consumer fold. | +| `DS4_CUDA_NO_Q8_FOLD=1` | Dominant rollback for the Q8_1 fold. | +| `DS4_CUDA_Q8_FOLD_ORACLE=1` | Compare fresh canonical Q8_1 bytes and consumer outputs. Use with `DS4_CUDA_DECODE_GRAPHS=0`; require nonzero calls and zero mismatches/skips. | + +## ROCm Q4 + +| Variable | Default and purpose | +| --- | --- | +| `DS4_ROCM_DISABLE_Q4_PREFILL_TILE8=1` | Restore the legacy Q4 prefill kernel. TILE8 is automatic for validated chunks of 9 through 4096 tokens. | +| `DS4_ROCM_REQUIRE_Q4_PREFILL_TILE8=1` | Fail closed when an eligible Q4 prefill call cannot use TILE8. | +| `DS4_ROCM_ENABLE_Q4_PREFILL_TILE8=1` | Legacy no-op accepted by existing scripts; TILE8 no longer requires an enable variable. | +| `DS4_ROCM_Q4_PREFILL_TILE8_STATS=1` | Report dense, pair, attention-batch, and token counters at process exit. | +| `DS4_ROCM_ENABLE_Q4_DENSE_PAIR=1` | Share one Q8_K activation quantization between the two Q4 dense projections. This pair remains opt-in. | +| `DS4_ROCM_DISABLE_Q4_DENSE_PAIR=1` | Dominant rollback for the ROCm Q4 dense pair. | +| `DS4_ROCM_ENABLE_Q4_GROUPED_ATTN_A=1` | Enable the two-launch grouped attention-A decode path. This path remains opt-in pending model A/B results. | +| `DS4_ROCM_DISABLE_Q4_GROUPED_ATTN_A=1` | Dominant rollback for grouped attention-A decode. | +| `DS4_ROCM_REQUIRE_Q4_GROUPED_ATTN_A=1` | Fail closed if grouped attention-A decode is disabled or ineligible. | +| `DS4_ROCM_Q4_GROUPED_ATTN_A_STATS=1` | Report grouped calls, dispatches, groups, fallbacks, and failures. | + +Run `make test-strix-rocm-q4-parity` and +`make test-strix-rocm-q4-prefill` on a `gfx1151` Strix Halo host before making +performance claims. The synthetic oracle proves layout and numerical parity; +it does not by itself prove that a complete Q4 model fits safely in GTT. + +## Historical `DS4_METAL_*` graph controls + +The shared graph implementation predates the CUDA backend and retained a few +`DS4_METAL_*` names. In a non-ROCm GPU build, these controls affect the shared +Metal/CUDA graph policy; ROCm explicitly ignores them: + +- `DS4_METAL_DISABLE_HC_FUSION` +- `DS4_METAL_DISABLE_HC_NORM_FUSION` +- `DS4_METAL_DISABLE_KV_FUSION` +- `DS4_METAL_DISABLE_QKV_NORM_FUSION` +- `DS4_METAL_DISABLE_QKV_PAIR_PROJ` +- `DS4_METAL_DISABLE_COMPRESSOR_PAIR_PROJ` +- `DS4_METAL_DISABLE_ATTN_OUT_HC_FUSION` +- `DS4_METAL_DISABLE_SHARED_DOWN_HC_FUSION` + +The prefix is historical rather than an indication that these particular +switches are always Metal-only. New backend-specific controls should use the +backend they actually configure (`DS4_CUDA_*` or `DS4_ROCM_*`) instead of +extending this legacy naming. diff --git a/Makefile b/Makefile index 461b31b10..73c6f77b5 100644 --- a/Makefile +++ b/Makefile @@ -68,7 +68,7 @@ DS4_LINK_LIBS ?= $(CUDA_LDLIBS) METAL_LDLIBS := $(LDLIBS) endif -.PHONY: all help clean test test-rocm test-glm53-kda-rocm test-metal-session-batch test-metal-session-batch-ssd test-metal-q4-streams test-metal-exactn-oracle test-metal-dspark-capture test-metal-iq2-midonly test-metal-iq2-ssd-grouped-mm test-metal-iq2-live-index test-mxfp4-cuda test-mxfp4-rocm test-mmq-parity-cuda test-rocm-q4-parity test-rocm-q4-dense test-rocm-q4-pair test-rocm-q4-prefill test-strix-rocm-q4-parity test-strix-rocm-q4-prefill test-strix-rocm-q4-prefill-long test-cuda-session-batch test-cuda-mixed-batch dspark-acceptance dspark-verify-depth rocm-dspark-acceptance rocm-dspark-verify-depth mtp-verify-depth cpu cuda cuda-spark cuda-generic cuda-regression strix-halo rocm +.PHONY: all help clean test test-rocm test-glm53-kda-rocm test-metal-session-batch test-metal-session-batch-ssd test-metal-q4-streams test-metal-q4-attn-exactn test-metal-exactn-oracle test-metal-dspark-capture test-metal-iq2-midonly test-metal-iq2-ssd-grouped-mm test-metal-iq2-live-index test-mxfp4-cuda test-mxfp4-rocm test-mmq-parity-cuda test-rocm-q4-parity test-rocm-q4-dense test-rocm-q4-pair test-rocm-q4-prefill test-strix-rocm-q4-parity test-strix-rocm-q4-prefill test-strix-rocm-q4-prefill-long test-cuda-session-batch test-cuda-mixed-batch dspark-acceptance dspark-verify-depth rocm-dspark-acceptance rocm-dspark-verify-depth mtp-verify-depth cpu cuda cuda-spark cuda-generic cuda-regression strix-halo rocm ifeq ($(UNAME_S),Darwin) .PHONY: metal-decode-schedule-bench metal-prefill-variant-bench check-mxfp4-half-lut test-mxfp4-metal @@ -82,6 +82,7 @@ help: @echo " make test Build and run tests" @echo " make test-metal-session-batch-ssd Exact-logit Metal SSD union control/candidate oracle" @echo " make test-metal-q4-streams Check resident Q4 Metal stream overlap" + @echo " make test-metal-q4-attn-exactn Bitwise/canary oracle for M1-M4 SSD-prefill Q4 attention output" @echo " make test-metal-dspark-capture Check fused DSpark HC capture bitwise" @echo " make test-metal-iq2-midonly Check M1 IQ2 addr mid-only output and sentinels" @echo " make test-metal-iq2-live-index Check IQ2 SSD live-cache index policy and fallback" @@ -159,6 +160,19 @@ test-metal-q4-streams: tests/test_metal_q4_streams env -u DS4_METAL_MODEL_UNTRACKED ./tests/test_metal_q4_streams DS4_METAL_MODEL_UNTRACKED=1 ./tests/test_metal_q4_streams +tests/test_metal_q4_attn_exactn.o: tests/test_metal_q4_attn_exactn.c ds4_gpu.h + $(CC) $(CFLAGS) -I. -c -o $@ $< + +tests/test_metal_q4_attn_exactn: tests/test_metal_q4_attn_exactn.o ds4_metal.o + $(CC) $(CFLAGS) -o $@ $^ $(METAL_LDLIBS) + +test-metal-q4-attn-exactn: tests/test_metal_q4_attn_exactn + env -u DS4_METAL_ENABLE_Q4_SSD_PREFILL_ATTN_OUT_EXACTN \ + -u DS4_METAL_DISABLE_Q4_SSD_PREFILL_ATTN_OUT_EXACTN \ + -u DS4_METAL_REQUIRE_Q4_SSD_PREFILL_ATTN_OUT_EXACTN \ + -u DS4_METAL_DISABLE_Q4_MV_CLASSIC \ + ./tests/test_metal_q4_attn_exactn + tests/test_metal_dspark_capture.o: tests/test_metal_dspark_capture.c ds4_gpu.h $(CC) $(CFLAGS) -I. -c -o $@ $< @@ -793,4 +807,4 @@ mxfp4-dot-test: tests/test_mxfp4_dot.c ./tests/test_mxfp4_dot clean: - rm -f ds4 ds4-server ds4-bench ds4-eval ds4-agent ds4_cpu ds4_native ds4_server_test ds4_test ds4_agent_test gguf-tools/quality-testing/score_official gguf-tools/quality-testing/score_official.o speed-bench/metal_decode_schedule_bench speed-bench/metal_prefill_variant_bench speed-bench/*.o tests/test_q4k_dot tests/test_mxfp4_dot tests/test_mxfp4_metal tests/test_mxfp4_rocm tests/test_mxfp4_cuda tests/test_rocm_q4_dense_pair tests/test_metal_session_batch tests/test_metal_q4_streams tests/test_metal_exactn_oracle tests/test_metal_dspark_capture tests/test_metal_iq2_midonly tests/test_metal_iq2_ssd_grouped_mm tests/test_metal_iq2_live_index tests/test_glm53_kda tests/test_glm53_kda_rocm tests/test_glm53_vision_engine tests/test_glm53_vision_prompt tests/test_gpu_xdev tests/test_gpu_model_cache tests/test_gpu_lookup_cache_strict tests/test_engine_mgpu_refusal tests/test_engine_mgpu_runtime tests/test_engine_correctness tests/test_sampling tests/test_cuda_session_batch tests/test_cuda_mixed_batch tests/*.o *.o tests/cuda_long_context_smoke tests/cuda_long_context_smoke.o + rm -f ds4 ds4-server ds4-bench ds4-eval ds4-agent ds4_cpu ds4_native ds4_server_test ds4_test ds4_agent_test gguf-tools/quality-testing/score_official gguf-tools/quality-testing/score_official.o speed-bench/metal_decode_schedule_bench speed-bench/metal_prefill_variant_bench speed-bench/*.o tests/test_q4k_dot tests/test_mxfp4_dot tests/test_mxfp4_metal tests/test_mxfp4_rocm tests/test_mxfp4_cuda tests/test_rocm_q4_dense_pair tests/test_metal_session_batch tests/test_metal_q4_streams tests/test_metal_q4_attn_exactn tests/test_metal_exactn_oracle tests/test_metal_dspark_capture tests/test_metal_iq2_midonly tests/test_metal_iq2_ssd_grouped_mm tests/test_metal_iq2_live_index tests/test_glm53_kda tests/test_glm53_kda_rocm tests/test_glm53_vision_engine tests/test_glm53_vision_prompt tests/test_gpu_xdev tests/test_gpu_model_cache tests/test_gpu_lookup_cache_strict tests/test_engine_mgpu_refusal tests/test_engine_mgpu_runtime tests/test_engine_correctness tests/test_sampling tests/test_cuda_session_batch tests/test_cuda_mixed_batch tests/*.o *.o tests/cuda_long_context_smoke tests/cuda_long_context_smoke.o diff --git a/QA_BEFORE_RELEASES.md b/QA_BEFORE_RELEASES.md index 7a39f794f..aed06495e 100644 --- a/QA_BEFORE_RELEASES.md +++ b/QA_BEFORE_RELEASES.md @@ -359,15 +359,16 @@ than a failure. `--dspark-strict` remains the byte-identical target-only mode. machine. Treat the FlashAttention memo rows as host-dispatch A/B tests: the selected specialization and output must remain identical, and any timing comparison must use repeated warm runs. -- For the M1 IQ2 address-table mid-only experiment, first run +- For the default M1 IQ2 address-table mid-only path, first run `make test-metal-iq2-midonly`. It must cover 12,288 full-shape top-6 mid words in both unmasked and complementary masked address-table modes with both mid mismatch counters at zero, no canonical unwritten rows, zero candidate gate/up writes, and zero guard mismatches. Then use the same greedy IQ2_XXS/Q2_K SSD-streaming model, prompt, cache state, and token count for - three decode runs: leave both switches unset for the canonical control, set - `DS4_METAL_REQUIRE_M1_IQ2_MID_ONLY=1` for the fail-closed candidate, and set both enable - and `DS4_METAL_DISABLE_M1_IQ2_MID_ONLY=1` for the kill-switch fallback. + three decode runs: leave all switches unset for the automatic candidate, + set `DS4_METAL_REQUIRE_M1_IQ2_MID_ONLY=1` for fail-closed coverage, and set + `DS4_METAL_DISABLE_M1_IQ2_MID_ONLY=1` for the canonical control and + kill-switch fallback. Enable the routed-MoE stage profiler on one candidate layer and require path `iq2_stream_addr_mid_only_4096x2048` or `iq2_stream_addr_mask_mid_only_4096x2048`; absence of both is failed model @@ -1151,6 +1152,20 @@ a substitute for CUDA or Metal release testing. - Do not use the mixed q2-q4 or Q4 Flash GGUFs for routine Strix Halo QA yet. They are dangerous on this machine for now because the ROCm path can hit system OOM instead of failing cleanly. +- When ROCm Q4 code changes, run `make test-strix-rocm-q4-parity` and + `make test-strix-rocm-q4-prefill` before attempting a model. The prefill + oracle must compare the TILE8 default with + `DS4_ROCM_DISABLE_Q4_PREFILL_TILE8=1` at K=256, 1024, and 4096 and at token + counts covering a partial tile and the 128-token production chunk. Require + bitwise dense, pair, Q4-attention-B, and Q8-attention-B parity with intact + canaries. A REQUIRE-plus-DISABLE arm must fail before modifying output. +- Keep ROCm grouped attention-A decode opt-in until a model A/B wins. Its + fail-closed test uses `DS4_ROCM_ENABLE_Q4_GROUPED_ATTN_A=1`, + `DS4_ROCM_REQUIRE_Q4_GROUPED_ATTN_A=1`, and + `DS4_ROCM_Q4_GROUPED_ATTN_A_STATS=1`; require dispatches and groups above + zero, with zero fallbacks/failures and bitwise equality to the per-group + reference. This synthetic coverage does not supersede the Q4-model OOM + warning above. - Run a short CLI prompt: `./ds4 -m gguf/DeepSeek-V4-Flash-IQ2XXS-w2Q2K-AProjQ8-SExpQ8-OutQ8-chat-v2-imatrix-0731.gguf --ctx 4096 --nothink -p "Reply with exactly: OK"`. - For DeepSeek Flash decode, confirm the default path uses prequantized Q8 diff --git a/README.md b/README.md index cad16b31a..00fe36a88 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,8 @@ next sections. guide for contributors. **Read this before sending a pull request**. - [QA_BEFORE_RELEASES.md](QA_BEFORE_RELEASES.md): the complete release test matrix, including the remote Metal, CUDA, and ROCm machines. +- [ENVIRONMENT_VARIABLES.md](ENVIRONMENT_VARIABLES.md): curated rollback, + fail-closed, and diagnostic environment switches for Metal, CUDA, and ROCm. - [gguf-tools/README.md](gguf-tools/README.md): offline GGUF generation, imatrix collection, quantization tooling, and quality checks. - [gguf-tools/imatrix/README.md](gguf-tools/imatrix/README.md): how the @@ -650,20 +652,21 @@ top-k implementation is now used instead and has no runtime re-enable switch. Use a previous binary only as a performance control, and require identical selected ids on tie-heavy inputs before comparing timing. -Apple M1 has an opt-in SSD-streaming decode experiment for the exact +Apple M1 defaults to a specialized SSD-streaming decode path for the exact IQ2_XXS/Q2_K routed-MoE shape with 256 experts, top-6 routing, and a -4096-to-2048 gate/up projection. Set -`DS4_METAL_ENABLE_M1_IQ2_MID_ONLY=1` to replace the IQ2 address-table -pair-SwiGLU producer, including complementary resident/missing cache masks. +4096-to-2048 gate/up projection. It replaces the IQ2 address-table pair-SwiGLU +producer, including complementary resident/missing cache masks. It preserves the canonical dot-product, reduction, clamp, activation, and route-weight order but writes `mid` directly instead of materializing the otherwise unused gate/up rows. Every other device, shape, streaming mode, unsupported mask/accumulate mode, or unavailable -pipeline keeps the canonical producer; `DS4_METAL_DISABLE_M1_IQ2_MID_ONLY=1` takes precedence as -the kill switch. For fail-closed model coverage, -`DS4_METAL_REQUIRE_M1_IQ2_MID_ONLY=1` implies the enable gate and rejects an -ineligible supported address-table dispatch; the kill switch still takes -precedence. `make test-metal-iq2-midonly` compares all 12,288 top-6 +pipeline keeps the canonical producer. Set +`DS4_METAL_DISABLE_M1_IQ2_MID_ONLY=1` to restore the canonical producer. +For fail-closed model coverage, `DS4_METAL_REQUIRE_M1_IQ2_MID_ONLY=1` rejects +an ineligible supported address-table dispatch; the kill switch still takes +precedence. The former `DS4_METAL_ENABLE_M1_IQ2_MID_ONLY=1` opt-in is accepted +as a harmless compatibility setting because the path is now automatic. +`make test-metal-iq2-midonly` compares all 12,288 top-6 output words bitwise at full shape for both unmasked and complementary masked address tables, verifies that the candidates leave gate/up sentinels untouched, and checks output guards. The routed-MoE stage profiler reports diff --git a/cuda/mmq/ds4_mmq.cu b/cuda/mmq/ds4_mmq.cu index 878b2ddc1..a6ee25af8 100644 --- a/cuda/mmq/ds4_mmq.cu +++ b/cuda/mmq/ds4_mmq.cu @@ -381,6 +381,19 @@ static uint64_t g_q8_fold_oracle_aligned_iq2_calls; static uint64_t g_q8_fold_oracle_skips; static int g_q8_fold_oracle_report_registered; +static cudaError_t ds4_mmq_q8_fold_oracle_free( + void *ptr, const char *label, cudaError_t prior_err) { + if (!ptr) return prior_err; + const cudaError_t free_err = cudaFree(ptr); + if (free_err != cudaSuccess) { + fprintf(stderr, + "ds4: CUDA Q8_1 fold oracle cudaFree(%s) failed: %s\n", + label, cudaGetErrorString(free_err)); + if (prior_err == cudaSuccess) return free_err; + } + return prior_err; +} + static void ds4_mmq_q8_fold_oracle_report(void) { fprintf(stderr, "ds4: CUDA Q8_1 fold oracle: byte_calls=%llu " @@ -463,7 +476,9 @@ static bool ds4_mmq_q8_fold_oracle_bytes( cudaMemcpyDeviceToHost) == cudaSuccess; if (!setup_ok) { (void)cudaGetLastError(); - (void)cudaFree(fresh); + cudaError_t cleanup_err = ds4_mmq_q8_fold_oracle_free( + fresh, "byte-fresh", cudaSuccess); + if (cleanup_err != cudaSuccess) (void)cudaGetLastError(); free(host); g_q8_fold_oracle_skips++; return false; @@ -476,14 +491,22 @@ static bool ds4_mmq_q8_fold_oracle_bytes( cudaMemcpyDeviceToDevice, stream) != cudaSuccess || cudaStreamSynchronize(stream) != cudaSuccess) { (void)cudaGetLastError(); - (void)cudaFree(fresh); + cudaError_t cleanup_err = ds4_mmq_q8_fold_oracle_free( + fresh, "byte-fresh", cudaSuccess); + if (cleanup_err != cudaSuccess) (void)cudaGetLastError(); free(host); g_q8_fold_oracle_skips++; return false; } } - (void)cudaFree(fresh); + cudaError_t cleanup_err = ds4_mmq_q8_fold_oracle_free( + fresh, "byte-fresh", cudaSuccess); free(host); + if (cleanup_err != cudaSuccess) { + (void)cudaGetLastError(); + g_q8_fold_oracle_skips++; + return false; + } return true; } @@ -4909,9 +4932,14 @@ int ds4_mmq_moe_gate_up_mid_vec_impl( fresh && reference && mismatch_device; if (!allocated) { (void)cudaGetLastError(); - if (fresh) (void)cudaFree(fresh); - if (reference) (void)cudaFree(reference); - if (mismatch_device) (void)cudaFree(mismatch_device); + cudaError_t cleanup_err = cudaSuccess; + cleanup_err = ds4_mmq_q8_fold_oracle_free( + fresh, "raw-moe-fresh", cleanup_err); + cleanup_err = ds4_mmq_q8_fold_oracle_free( + reference, "raw-moe-reference", cleanup_err); + cleanup_err = ds4_mmq_q8_fold_oracle_free( + mismatch_device, "raw-moe-mismatch", cleanup_err); + if (cleanup_err != cudaSuccess) (void)cudaGetLastError(); g_q8_fold_oracle_skips++; } else { cudaError_t oracle_err = cudaMemsetAsync( @@ -4964,9 +4992,12 @@ int ds4_mmq_moe_gate_up_mid_vec_impl( if (oracle_err == cudaSuccess) { oracle_err = cudaStreamSynchronize(stream); } - (void)cudaFree(fresh); - (void)cudaFree(reference); - (void)cudaFree(mismatch_device); + oracle_err = ds4_mmq_q8_fold_oracle_free( + fresh, "raw-moe-fresh", oracle_err); + oracle_err = ds4_mmq_q8_fold_oracle_free( + reference, "raw-moe-reference", oracle_err); + oracle_err = ds4_mmq_q8_fold_oracle_free( + mismatch_device, "raw-moe-mismatch", oracle_err); if (oracle_err != cudaSuccess) { (void)cudaGetLastError(); g_q8_fold_oracle_skips++; @@ -5530,9 +5561,14 @@ static int q8_fold_q8_aligned_output_oracle( cudaMalloc((void **)&mismatch_device, sizeof(uint32_t)) != cudaSuccess || !fresh || !reference || !mismatch_device) { (void)cudaGetLastError(); - if (fresh) (void)cudaFree(fresh); - if (reference) (void)cudaFree(reference); - if (mismatch_device) (void)cudaFree(mismatch_device); + cudaError_t cleanup_err = cudaSuccess; + cleanup_err = ds4_mmq_q8_fold_oracle_free( + fresh, "aligned-q8-fresh", cleanup_err); + cleanup_err = ds4_mmq_q8_fold_oracle_free( + reference, "aligned-q8-reference", cleanup_err); + cleanup_err = ds4_mmq_q8_fold_oracle_free( + mismatch_device, "aligned-q8-mismatch", cleanup_err); + if (cleanup_err != cudaSuccess) (void)cudaGetLastError(); g_q8_fold_oracle_skips++; return 0; } @@ -5572,9 +5608,12 @@ static int q8_fold_q8_aligned_output_oracle( } if (err == cudaSuccess) err = cudaStreamSynchronize(stream); - (void)cudaFree(fresh); - (void)cudaFree(reference); - (void)cudaFree(mismatch_device); + err = ds4_mmq_q8_fold_oracle_free( + fresh, "aligned-q8-fresh", err); + err = ds4_mmq_q8_fold_oracle_free( + reference, "aligned-q8-reference", err); + err = ds4_mmq_q8_fold_oracle_free( + mismatch_device, "aligned-q8-mismatch", err); if (err != cudaSuccess) { (void)cudaGetLastError(); g_q8_fold_oracle_skips++; @@ -6275,9 +6314,14 @@ extern "C" int ds4_mmq_iq2_xxs_aligned_moe_gate_up_mid_vec( fresh && reference && mismatch_device; if (!allocated) { (void)cudaGetLastError(); - if (fresh) (void)cudaFree(fresh); - if (reference) (void)cudaFree(reference); - if (mismatch_device) (void)cudaFree(mismatch_device); + cudaError_t cleanup_err = cudaSuccess; + cleanup_err = ds4_mmq_q8_fold_oracle_free( + fresh, "aligned-iq2-fresh", cleanup_err); + cleanup_err = ds4_mmq_q8_fold_oracle_free( + reference, "aligned-iq2-reference", cleanup_err); + cleanup_err = ds4_mmq_q8_fold_oracle_free( + mismatch_device, "aligned-iq2-mismatch", cleanup_err); + if (cleanup_err != cudaSuccess) (void)cudaGetLastError(); g_q8_fold_oracle_skips++; } else { cudaError_t oracle_err = cudaMemsetAsync( @@ -6323,9 +6367,12 @@ extern "C" int ds4_mmq_iq2_xxs_aligned_moe_gate_up_mid_vec( if (oracle_err == cudaSuccess) { oracle_err = cudaStreamSynchronize(stream); } - (void)cudaFree(fresh); - (void)cudaFree(reference); - (void)cudaFree(mismatch_device); + oracle_err = ds4_mmq_q8_fold_oracle_free( + fresh, "aligned-iq2-fresh", oracle_err); + oracle_err = ds4_mmq_q8_fold_oracle_free( + reference, "aligned-iq2-reference", oracle_err); + oracle_err = ds4_mmq_q8_fold_oracle_free( + mismatch_device, "aligned-iq2-mismatch", oracle_err); if (oracle_err != cudaSuccess) { (void)cudaGetLastError(); g_q8_fold_oracle_skips++; diff --git a/cuda/mmq/test/proto_gemm_dense_q8_d2r.cu b/cuda/mmq/test/proto_gemm_dense_q8_d2r.cu index 0fec3eb0f..8dc8d4275 100644 --- a/cuda/mmq/test/proto_gemm_dense_q8_d2r.cu +++ b/cuda/mmq/test/proto_gemm_dense_q8_d2r.cu @@ -42,16 +42,6 @@ #include "ds4_mmq.h" #include "quantize.cuh" -extern "C" int ds4_cuda_q8_fold_take_q81( - const void *src, uint64_t in_dim, cudaStream_t stream, - const void **q81) { - (void)src; - (void)in_dim; - (void)stream; - if (q81) *q81 = nullptr; - return 0; -} - #include #include diff --git a/cuda/mmq/test/test_mmq_soa_tiles.cu b/cuda/mmq/test/test_mmq_soa_tiles.cu index da7385932..36ba8f3dc 100644 --- a/cuda/mmq/test/test_mmq_soa_tiles.cu +++ b/cuda/mmq/test/test_mmq_soa_tiles.cu @@ -36,17 +36,6 @@ #include #endif -// libds4mmq.a references this ds4_cuda.cu symbol from the q8-fold vec paths -// (C3 Inc4); the entries under test never reach it, so a "no fold available" -// stub satisfies the link. -extern "C" int ds4_cuda_q8_fold_take_q81( - const void *src, uint64_t in_dim, cudaStream_t stream, - const void **q81) { - (void)src; (void)in_dim; (void)stream; - if (q81) *q81 = nullptr; - return 0; -} - #include #include #include diff --git a/ds4_metal.m b/ds4_metal.m index 801165a78..c7fa4d4a2 100644 --- a/ds4_metal.m +++ b/ds4_metal.m @@ -5426,6 +5426,31 @@ static int ds4_gpu_encode_sum_rows_f32( int16_t r3; } ds4_gpu_q8_0_matvec_args; +typedef struct { + uint32_t in_dim; + uint32_t out_rows; + uint32_t n_groups; + uint32_t n_tokens; + uint64_t weight_row_bytes; + uint64_t weight_group_bytes; + uint64_t input_group_bytes; + uint64_t input_token_bytes; + uint64_t output_group_bytes; + uint64_t output_token_bytes; +} ds4_gpu_q4_attn_exactn_args; + +typedef char ds4_gpu_q4_attn_exactn_args_must_be_64_bytes[ + sizeof(ds4_gpu_q4_attn_exactn_args) == 64 ? 1 : -1]; + +static bool ds4_gpu_u64_mul_checked( + uint64_t a, + uint64_t b, + uint64_t *result) { + if (!result || (a != 0 && b > UINT64_MAX / a)) return false; + *result = a * b; + return true; +} + typedef struct { int32_t ne00; int32_t ne02; @@ -14561,11 +14586,9 @@ static void ds4_gpu_stream_expert_cache_note_decode_token(void) { ds4_gpu_stream_expert_cache_maybe_decay_route_hotness(); } -static int ds4_gpu_m1_iq2_mid_only_requested(void) { +static int ds4_gpu_m1_iq2_mid_only_enabled(void) { return ds4_gpu_device_is_m1_apple_silicon() && - ds4_gpu_env_bool("DS4_METAL_DISABLE_M1_IQ2_MID_ONLY") != 1 && - (ds4_gpu_env_bool("DS4_METAL_ENABLE_M1_IQ2_MID_ONLY") == 1 || - ds4_gpu_env_bool("DS4_METAL_REQUIRE_M1_IQ2_MID_ONLY") == 1); + ds4_gpu_env_bool("DS4_METAL_DISABLE_M1_IQ2_MID_ONLY") != 1; } static int ds4_gpu_stream_compact_addr_requested(void) { @@ -14578,7 +14601,6 @@ static int ds4_gpu_stream_compact_addr_requested(void) { static int ds4_gpu_stream_expert_addr_table_requested(void) { return g_ssd_streaming_mode && (getenv("DS4_METAL_ENABLE_STREAMING_EXPERT_ADDR_TABLE") != NULL || - ds4_gpu_m1_iq2_mid_only_requested() || getenv("DS4_METAL_ENABLE_STREAMING_EXPERT_HIT_VALIDATOR") != NULL || getenv("DS4_METAL_ENABLE_STREAMING_EXPERT_MASKED_ADDR") != NULL || g_stream_prefill_batch_selected_addr_building || @@ -14592,7 +14614,6 @@ static int ds4_gpu_stream_expert_addr_table_requested(void) { static int ds4_gpu_stream_expert_addr_table_kernel_requested(void) { return g_ssd_streaming_mode && (getenv("DS4_METAL_ENABLE_STREAMING_EXPERT_ADDR_TABLE") != NULL || - ds4_gpu_m1_iq2_mid_only_requested() || getenv("DS4_METAL_ENABLE_STREAMING_EXPERT_HIT_VALIDATOR") != NULL || getenv("DS4_METAL_ENABLE_STREAMING_EXPERT_MASKED_ADDR") != NULL || ds4_gpu_stream_expert_split_ready()) && @@ -28005,6 +28026,268 @@ int ds4_gpu_attention_output_q8_batch_tensor( } } +static int ds4_gpu_attention_output_q4_K_ssd_prefill_exactn_tensor( + ds4_gpu_tensor *out, + ds4_gpu_tensor *low, + const void *model_map, + uint64_t model_size, + uint64_t out_a_offset, + uint64_t out_b_offset, + uint32_t out_b_type, + uint64_t group_dim, + uint64_t rank, + uint32_t n_groups, + uint64_t out_dim, + const ds4_gpu_tensor *heads, + uint32_t n_tokens) { + const bool scope = n_tokens >= 6u && n_tokens <= 31u; + const bool require = scope && + ds4_gpu_env_bool("DS4_METAL_REQUIRE_Q4_SSD_PREFILL_ATTN_OUT_EXACTN") == 1; + const int failure_rc = require ? -1 : 0; + if (!scope) return 0; + if (!g_initialized && !ds4_gpu_init()) return failure_rc; + + const bool disabled = + ds4_gpu_env_bool("DS4_METAL_DISABLE_Q4_SSD_PREFILL_ATTN_OUT_EXACTN") == 1; + const bool enabled = require || + ds4_gpu_env_bool("DS4_METAL_ENABLE_Q4_SSD_PREFILL_ATTN_OUT_EXACTN") == 1; + const bool classic_q4 = getenv("DS4_METAL_DISABLE_Q4_MV_CLASSIC") == NULL; + const bool platform_ok = + g_ssd_streaming_mode && + !g_quality_mode && + ds4_gpu_device_is_pre_m5_apple_silicon(); + if (!enabled || disabled || !classic_q4 || !platform_ok || + out_b_type != DS4_METAL_TENSOR_Q4_K) { + if (require) { + fprintf(stderr, + "ds4: required Metal Q4 SSD-prefill attention exact-N " + "path is ineligible (rows=%u type=%u ssd=%u quality=%u " + "pre_m5=%u disabled=%u classic=%u)\n", + n_tokens, + out_b_type, + g_ssd_streaming_mode ? 1u : 0u, + g_quality_mode ? 1u : 0u, + ds4_gpu_device_is_pre_m5_apple_silicon() ? 1u : 0u, + disabled ? 1u : 0u, + classic_q4 ? 1u : 0u); + } + return failure_rc; + } + + if (!out || !low || !heads || !model_map || + group_dim == 0 || rank == 0 || n_groups == 0 || out_dim == 0 || + group_dim > UINT32_MAX || rank > UINT32_MAX || out_dim > UINT32_MAX) { + return failure_rc; + } + + @autoreleasepool { + uint64_t low_dim = 0; + if (!ds4_gpu_u64_mul_checked((uint64_t)n_groups, rank, &low_dim)) { + return failure_rc; + } + if ((group_dim % 256u) != 0 || (low_dim % 256u) != 0 || + low_dim == 0 || low_dim > UINT32_MAX) { + if (require) { + fprintf(stderr, + "ds4: required Metal Q4 SSD-prefill attention exact-N " + "path received unaligned dimensions\n"); + } + return failure_rc; + } + + uint64_t row_a_bytes = 0; + uint64_t row_b_bytes = 0; + if (!ds4_gpu_quant_row_bytes(DS4_METAL_TENSOR_Q4_K, + (uint32_t)group_dim, + &row_a_bytes) || + !ds4_gpu_quant_row_bytes(DS4_METAL_TENSOR_Q4_K, + (uint32_t)low_dim, + &row_b_bytes)) { + return failure_rc; + } + + uint64_t group_a_bytes = 0; + uint64_t out_a_bytes = 0; + uint64_t out_b_bytes = 0; + uint64_t heads_group_bytes = 0; + uint64_t heads_row_bytes = 0; + uint64_t heads_bytes = 0; + uint64_t low_row_bytes = 0; + uint64_t low_bytes = 0; + uint64_t out_row_bytes = 0; + uint64_t out_bytes = 0; + uint64_t rank_bytes = 0; + uint64_t stage_a_bytes = 0; + uint64_t stage_b_bytes = 0; + if (!ds4_gpu_u64_mul_checked(rank, row_a_bytes, &group_a_bytes) || + !ds4_gpu_u64_mul_checked((uint64_t)n_groups, + group_a_bytes, + &out_a_bytes) || + !ds4_gpu_u64_mul_checked(out_dim, row_b_bytes, &out_b_bytes) || + !ds4_gpu_u64_mul_checked(group_dim, + sizeof(float), + &heads_group_bytes) || + !ds4_gpu_u64_mul_checked((uint64_t)n_groups, + heads_group_bytes, + &heads_row_bytes) || + !ds4_gpu_u64_mul_checked((uint64_t)n_tokens, + heads_row_bytes, + &heads_bytes) || + !ds4_gpu_u64_mul_checked(low_dim, + sizeof(float), + &low_row_bytes) || + !ds4_gpu_u64_mul_checked((uint64_t)n_tokens, + low_row_bytes, + &low_bytes) || + !ds4_gpu_u64_mul_checked(out_dim, + sizeof(float), + &out_row_bytes) || + !ds4_gpu_u64_mul_checked((uint64_t)n_tokens, + out_row_bytes, + &out_bytes) || + !ds4_gpu_u64_mul_checked(rank, + sizeof(float), + &rank_bytes) || + !ds4_gpu_u64_mul_checked(2u, + row_a_bytes, + &stage_a_bytes) || + !ds4_gpu_u64_mul_checked(2u, + row_b_bytes, + &stage_b_bytes)) { + if (require) { + fprintf(stderr, + "ds4: required Metal Q4 SSD-prefill attention exact-N " + "shape overflows byte strides\n"); + } + return failure_rc; + } + if (out_a_offset > model_size || out_a_bytes > model_size - out_a_offset || + out_b_offset > model_size || out_b_bytes > model_size - out_b_offset) { + if (require) { + fprintf(stderr, + "ds4: required Metal Q4 SSD-prefill attention exact-N " + "weights are outside the mapped model\n"); + } + return failure_rc; + } + + id heads_buf = ds4_gpu_tensor_buffer(heads); + id low_buf = ds4_gpu_tensor_buffer(low); + id out_buf = ds4_gpu_tensor_buffer(out); + if (!heads_buf || !low_buf || !out_buf || + ds4_gpu_tensor_bytes(heads) < heads_bytes || + ds4_gpu_tensor_bytes(low) < low_bytes || + ds4_gpu_tensor_bytes(out) < out_bytes) { + if (require) { + fprintf(stderr, + "ds4: required Metal Q4 SSD-prefill attention exact-N " + "path received undersized tensors\n"); + } + return failure_rc; + } + + id pipeline = ds4_gpu_get_pipeline( + "kernel_dsv4_attn_out_q4_K_ssd_prefill_exactn_f32"); + const uint64_t max_stage_bytes = + stage_a_bytes > stage_b_bytes ? stage_a_bytes : stage_b_bytes; + if (!pipeline || pipeline.threadExecutionWidth != 32u || + pipeline.maxTotalThreadsPerThreadgroup < 512u || + max_stage_bytes > (uint64_t)[g_device maxThreadgroupMemoryLength] || + max_stage_bytes > (uint64_t)NSUIntegerMax) { + if (require) { + fprintf(stderr, + "ds4: required Metal Q4 SSD-prefill attention exact-N " + "pipeline is unavailable (threads=%lu width=%lu " + "stage=%llu max_stage=%lu)\n", + (unsigned long)(pipeline ? + pipeline.maxTotalThreadsPerThreadgroup : 0u), + (unsigned long)(pipeline ? + pipeline.threadExecutionWidth : 0u), + (unsigned long long)max_stage_bytes, + (unsigned long)[g_device maxThreadgroupMemoryLength]); + } + return failure_rc; + } + + uint64_t out_a_inner = 0; + uint64_t out_b_inner = 0; + id out_a_buf = ds4_gpu_wrap_model_range( + model_map, model_size, out_a_offset, out_a_bytes, &out_a_inner); + id out_b_buf = ds4_gpu_wrap_model_range( + model_map, model_size, out_b_offset, out_b_bytes, &out_b_inner); + if (!out_a_buf || !out_b_buf) return failure_rc; + + const ds4_gpu_q4_attn_exactn_args args_a = { + .in_dim = (uint32_t)group_dim, + .out_rows = (uint32_t)rank, + .n_groups = n_groups, + .n_tokens = n_tokens, + .weight_row_bytes = row_a_bytes, + .weight_group_bytes = group_a_bytes, + .input_group_bytes = heads_group_bytes, + .input_token_bytes = heads_row_bytes, + .output_group_bytes = rank_bytes, + .output_token_bytes = low_row_bytes, + }; + const ds4_gpu_q4_attn_exactn_args args_b = { + .in_dim = (uint32_t)low_dim, + .out_rows = (uint32_t)out_dim, + .n_groups = 1u, + .n_tokens = n_tokens, + .weight_row_bytes = row_b_bytes, + .weight_group_bytes = out_b_bytes, + .input_group_bytes = 0, + .input_token_bytes = low_row_bytes, + .output_group_bytes = 0, + .output_token_bytes = out_row_bytes, + }; + + const bool had_batch = g_batch_cb != nil; + if (!had_batch && ds4_gpu_begin_commands() == 0) return failure_rc; + + int owned = 0; + id cb = ds4_gpu_command_buffer(&owned); + if (!cb || owned) { + if (!had_batch) (void)ds4_gpu_end_commands(); + return failure_rc; + } + id enc = ds4_gpu_compute_encoder(cb); + if (!enc) { + if (!had_batch) (void)ds4_gpu_end_commands(); + return failure_rc; + } + + [enc setComputePipelineState:pipeline]; + [enc setBytes:&args_a length:sizeof(args_a) atIndex:0]; + [enc setBuffer:out_a_buf offset:(NSUInteger)out_a_inner atIndex:1]; + [enc setBuffer:heads_buf offset:ds4_gpu_tensor_offset(heads) atIndex:2]; + [enc setBuffer:low_buf offset:ds4_gpu_tensor_offset(low) atIndex:3]; + [enc setThreadgroupMemoryLength:(NSUInteger)stage_a_bytes atIndex:0]; + [enc dispatchThreadgroups:MTLSizeMake(((NSUInteger)rank + 1u) / 2u, + ((NSUInteger)n_tokens + 15u) / 16u, + (NSUInteger)n_groups) + threadsPerThreadgroup:MTLSizeMake(32u, 16u, 1u)]; + + [enc setBytes:&args_b length:sizeof(args_b) atIndex:0]; + [enc setBuffer:out_b_buf offset:(NSUInteger)out_b_inner atIndex:1]; + [enc setBuffer:low_buf offset:ds4_gpu_tensor_offset(low) atIndex:2]; + [enc setBuffer:out_buf offset:ds4_gpu_tensor_offset(out) atIndex:3]; + [enc setThreadgroupMemoryLength:(NSUInteger)stage_b_bytes atIndex:0]; + [enc dispatchThreadgroups:MTLSizeMake(((NSUInteger)out_dim + 1u) / 2u, + ((NSUInteger)n_tokens + 15u) / 16u, + 1u) + threadsPerThreadgroup:MTLSizeMake(32u, 16u, 1u)]; + ds4_gpu_end_compute_encoder(cb, enc); + + if (!had_batch && ds4_gpu_end_commands() == 0) { + // Work was already submitted: never let the caller replay the + // row fallback into the same output after a command-buffer error. + return -1; + } + return 1; + } +} + int ds4_gpu_attention_output_q4_K_batch_tensor( ds4_gpu_tensor *out, ds4_gpu_tensor *low, @@ -28021,6 +28304,25 @@ int ds4_gpu_attention_output_q4_K_batch_tensor( uint64_t out_dim, const ds4_gpu_tensor *heads, uint32_t n_tokens) { + if (n_tokens >= 6u && n_tokens <= 31u) { + const int exactn_rc = + ds4_gpu_attention_output_q4_K_ssd_prefill_exactn_tensor( + out, + low, + model_map, + model_size, + out_a_offset, + out_b_offset, + out_b_type, + group_dim, + rank, + n_groups, + out_dim, + heads, + n_tokens); + if (exactn_rc != 0) return exactn_rc; + } + const bool tiny_scope = n_tokens >= 2u && n_tokens <= 5u; const bool require_tiny = tiny_scope && @@ -42140,7 +42442,7 @@ int ds4_gpu_routed_moe_one_tensor( (n_expert == 6 || (n_expert == 8 && g_tp_split_world == 2)) && n_tokens == 1 && down_sum6_pipeline != nil; - /* Experimental M1 SSD-streaming producer. The eventual dispatch is + /* Default M1 SSD-streaming specialization. The eventual dispatch is * additionally required to be an address-table path supported by the * unmasked or masked mid-only pipeline, where the following Q2 sum * consumes only `mid`. Presence of the disable switch always wins, @@ -42151,7 +42453,7 @@ int ds4_gpu_routed_moe_one_tensor( const bool m1_iq2_mid_only_required = ds4_gpu_env_bool("DS4_METAL_REQUIRE_M1_IQ2_MID_ONLY") == 1; const bool m1_iq2_addr_mid_only_candidate = - ds4_gpu_m1_iq2_mid_only_requested() && + ds4_gpu_m1_iq2_mid_only_enabled() && g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_mid_only_4096x2048_pipeline != nil && !force_resident && g_ssd_streaming_mode && gate_type == DS4_METAL_TENSOR_IQ2_XXS && @@ -43261,7 +43563,8 @@ int ds4_gpu_routed_moe_one_tensor( } else if (use_stream_expert_cache) { use_stream_expert_addr_table = ((use_iq2_selected_slots && - ds4_gpu_stream_expert_addr_table_kernel_requested() && + (ds4_gpu_stream_expert_addr_table_kernel_requested() || + m1_iq2_addr_mid_only_candidate) && g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_pipeline != nil && g_moe_mul_mv_addr_q2_k_sum6_pipeline != nil) || use_iq2_stream_addr_table) && @@ -43920,13 +44223,6 @@ int ds4_gpu_routed_moe_one_tensor( gate_smem, 2, false); - if (ok) { - ok = ds4_gpu_flush_commands(); - if (ok) { - cb = ds4_gpu_command_buffer(&owned); - if (!cb) ok = 0; - } - } const double stream_split_resident_ms = stream_split_timing ? ds4_gpu_now_ms() - stream_split_t0 : 0.0; if (stream_split_timing) stream_split_t0 = ds4_gpu_now_ms(); @@ -44008,19 +44304,13 @@ int ds4_gpu_routed_moe_one_tensor( } } if (ok) { - /* - * The resident stage was submitted before the - * CPU read of missing experts so I/O can overlap - * with GPU work. The missing stage reuses the same - * gate/up/mid scratch buffers, so it must not - * execute until the resident command buffer has - * finished. The down/sum pass is issued once after - * all six mid slots exist; this keeps the final - * accumulation order stable regardless of the - * resident/missing split. - */ + /* Resident and missing kernels share scratch but + * are encoded serially in the current command + * buffer. Drain older pending buffers to advance + * cache epochs without committing this layer's + * resident stage separately. */ ok = ds4_gpu_wait_pending_command_buffers( - "streaming expert split resident"); + "streaming expert split prior pending"); if (stream_split_timing) { const double now_ms = ds4_gpu_now_ms(); stream_split_missing_wait_ms = diff --git a/metal/moe.metal b/metal/moe.metal index c14df610f..7424c1210 100644 --- a/metal/moe.metal +++ b/metal/moe.metal @@ -2699,6 +2699,23 @@ struct ds4_metal_args_mul_mm_id { int32_t tp_expert_base; }; +// Exact-N Q4_K attention-output projection used only by the opt-in SSD +// prefill path. One threadgroup stages a pair of packed weight rows once +// and sixteen independent SIMDgroups consume that pair for sixteen tokens. +// Keep this layout in lock-step with ds4_gpu_q4_attn_exactn_args. +struct ds4_metal_args_q4_attn_exactn { + uint32_t in_dim; + uint32_t out_rows; + uint32_t n_groups; + uint32_t n_tokens; + uint64_t weight_row_bytes; + uint64_t weight_group_bytes; + uint64_t input_group_bytes; + uint64_t input_token_bytes; + uint64_t output_group_bytes; + uint64_t output_token_bytes; +}; + template void kernel_mul_mv_q2_K_f32_impl( args_t args, @@ -2900,6 +2917,153 @@ void kernel_mul_mv_q4_K_f32_impl( (void)shmem; } +// This is the classic Q4_K matvec inner loop with only the weight address +// space changed from device to threadgroup. Its lane-to-K mapping, scalar +// operation order and simd_sum reduction are deliberately kept identical to +// kernel_mul_mv_q4_K_f32_impl: the exact-N oracle requires bitwise equality. +template +void kernel_mul_mv_q4_K_staged_exactn_impl( + uint32_t in_dim, + uint64_t weight_row_bytes, + threadgroup const char *src0, + device const char *src1, + device char *dst, + uint32_t valid_rows, + ushort tiisg) { + constexpr uint16_t kmask1 = 0x3f3f; + constexpr uint16_t kmask2 = 0x0f0f; + constexpr uint16_t kmask3 = 0xc0c0; + + const short ix = tiisg / 8; + const short it = tiisg % 8; + const short iq = it / 4; + const short ir = it % 4; + const int nb = in_dim / QK_K; + + threadgroup const block_q4_K *x = + (threadgroup const block_q4_K *)src0; + device const float *y = (device const float *)src1; + + float yl[16]; + float yh[16]; + float sumf[nr0] = {0.f}; + + device const float *y4 = y + ix * QK_K + 64 * iq + 8 * ir; + + uint16_t sc16[4]; + thread const uint8_t *sc8 = (thread const uint8_t *)sc16; + + for (int ib = ix; ib < nb; ib += 4) { + float4 sumy = {0.f, 0.f, 0.f, 0.f}; + + for (short i = 0; i < 8; ++i) { + yl[i + 0] = y4[i + 0]; sumy[0] += yl[i + 0]; + yl[i + 8] = y4[i + 32]; sumy[1] += yl[i + 8]; + yh[i + 0] = y4[i + 128]; sumy[2] += yh[i + 0]; + yh[i + 8] = y4[i + 160]; sumy[3] += yh[i + 8]; + } + + threadgroup const uint16_t *sc = + (threadgroup const uint16_t *)x[ib].scales + iq; + threadgroup const uint16_t *q1 = + (threadgroup const uint16_t *)x[ib].qs + 16 * iq + 4 * ir; + threadgroup const half *dh = &x[ib].d; + + for (short row = 0; row < nr0 && row < valid_rows; row++) { + sc16[0] = sc[0] & kmask1; + sc16[1] = sc[2] & kmask1; + sc16[2] = ((sc[4] >> 0) & kmask2) | ((sc[0] & kmask3) >> 2); + sc16[3] = ((sc[4] >> 4) & kmask2) | ((sc[2] & kmask3) >> 2); + + threadgroup const uint16_t *q2 = q1 + 32; + + float4 acc1 = {0.f, 0.f, 0.f, 0.f}; + float4 acc2 = {0.f, 0.f, 0.f, 0.f}; + + FOR_UNROLL (short i = 0; i < 4; ++i) { + acc1[0] += yl[2 * i + 0] * (q1[i] & 0x000F); + acc1[1] += yl[2 * i + 1] * (q1[i] & 0x0F00); + acc1[2] += yl[2 * i + 8] * (q1[i] & 0x00F0); + acc1[3] += yl[2 * i + 9] * (q1[i] & 0xF000); + acc2[0] += yh[2 * i + 0] * (q2[i] & 0x000F); + acc2[1] += yh[2 * i + 1] * (q2[i] & 0x0F00); + acc2[2] += yh[2 * i + 8] * (q2[i] & 0x00F0); + acc2[3] += yh[2 * i + 9] * (q2[i] & 0xF000); + } + + sumf[row] += dh[0] * ((acc1[0] + 1.f / 256.f * acc1[1]) * sc8[0] + + (acc1[2] + 1.f / 256.f * acc1[3]) * sc8[1] * 1.f / 16.f + + (acc2[0] + 1.f / 256.f * acc2[1]) * sc8[4] + + (acc2[2] + 1.f / 256.f * acc2[3]) * sc8[5] * 1.f / 16.f) - + dh[1] * (sumy[0] * sc8[2] + sumy[1] * sc8[3] + sumy[2] * sc8[6] + sumy[3] * sc8[7]); + + q1 += weight_row_bytes / 2; + sc += weight_row_bytes / 2; + dh += weight_row_bytes / 2; + } + + y4 += 4 * QK_K; + } + + device float *dst_f32 = (device float *)dst; + for (int row = 0; row < nr0 && row < valid_rows; ++row) { + float sum_all = simd_sum(sumf[row]); + if (tiisg == 0) { + dst_f32[row] = sum_all; + } + } +} + +// Grid: x = output-row pairs, y = 16-token tiles, z = independent groups. +// All 512 threads join the raw packed-row load and barrier. Afterwards each +// SIMDgroup owns one token and follows the classic Q4_K arithmetic above. +kernel void kernel_dsv4_attn_out_q4_K_ssd_prefill_exactn_f32( + constant ds4_metal_args_q4_attn_exactn &args, + device const char *weights, + device const char *input, + device char *output, + threadgroup char *staged [[threadgroup(0)]], + uint3 tgpig [[threadgroup_position_in_grid]], + ushort tiitg [[thread_index_in_threadgroup]], + ushort tiisg [[thread_index_in_simdgroup]], + ushort sgitg [[simdgroup_index_in_threadgroup]]) { + const uint32_t first_row = tgpig.x * 2u; + const uint32_t valid_rows = min(2u, args.out_rows - first_row); + const uint64_t group_base = (uint64_t)tgpig.z * args.weight_group_bytes; + + const uint64_t row0_base = + group_base + (uint64_t)first_row * args.weight_row_bytes; + for (uint64_t i = tiitg; i < args.weight_row_bytes; i += 32u * 16u) { + staged[i] = weights[row0_base + i]; + } + const uint64_t row1_base = row0_base + args.weight_row_bytes; + for (uint64_t i = tiitg; i < args.weight_row_bytes; i += 32u * 16u) { + staged[args.weight_row_bytes + i] = + valid_rows == 2u ? weights[row1_base + i] : 0; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + + const uint32_t token = tgpig.y * 16u + sgitg; + if (token >= args.n_tokens) return; + + device const char *token_input = + input + (uint64_t)token * args.input_token_bytes + + (uint64_t)tgpig.z * args.input_group_bytes; + device char *token_output = + output + (uint64_t)token * args.output_token_bytes + + (uint64_t)tgpig.z * args.output_group_bytes + + (uint64_t)first_row * sizeof(float); + + kernel_mul_mv_q4_K_staged_exactn_impl<2>( + args.in_dim, + args.weight_row_bytes, + staged, + token_input, + token_output, + valid_rows, + tiisg); +} + template void kernel_mul_mv_mxfp4_f32_impl( args_t args, @@ -8234,12 +8398,11 @@ kernel void kernel_mul_mm_id_map0( threadgroup_barrier(mem_flags::mem_threadgroup); uint32_t route_base = 0; - for (ushort i = 0; i < ide; i++) { - route_base += route_counts[i]; - } uint32_t tile_base = 0; for (ushort i = 0; i < ide; i++) { - tile_base += (route_counts[i] + 31u) / 32u; + const uint32_t count = route_counts[i]; + route_base += count; + tile_base += (count + 31u) / 32u; } device uint32_t * tpe_u32 = (device uint32_t *) (htpe); diff --git a/tests/test_metal_q4_attn_exactn.c b/tests/test_metal_q4_attn_exactn.c new file mode 100644 index 000000000..0b3b86e30 --- /dev/null +++ b/tests/test_metal_q4_attn_exactn.c @@ -0,0 +1,382 @@ +#define _DARWIN_C_SOURCE + +/* Synthetic, GGUF-free bitwise oracle for the M1--M4 SSD-prefill Q4_K + * attention-output token-tiled kernel. */ + +#include "ds4_gpu.h" + +#include +#include +#include +#include +#include +#include + +bool ds4_log_is_tty(FILE *fp) { + (void)fp; + return false; +} + +#ifdef __APPLE__ + +enum { + Q4_K_TYPE = 12u, + QK_K = 256u, + /* Multiblock geometry: attention-A spans 16 Q4_K blocks per row and + * output-B spans four, exercising every ix lane and repeated ib steps. */ + GROUP_DIM = 4096u, + RANK = 512u, + N_GROUPS = 2u, + LOW_DIM = N_GROUPS * RANK, + OUT_DIM = 67u, + MAX_ROWS = 31u, + ALLOC_ROWS = 32u, +}; + +typedef struct { + uint16_t d; + uint16_t dmin; + uint8_t scales[12]; + uint8_t qs[QK_K / 2u]; +} block_q4_K; + +static const char *k_enable = + "DS4_METAL_ENABLE_Q4_SSD_PREFILL_ATTN_OUT_EXACTN"; +static const char *k_disable = + "DS4_METAL_DISABLE_Q4_SSD_PREFILL_ATTN_OUT_EXACTN"; +static const char *k_require = + "DS4_METAL_REQUIRE_Q4_SSD_PREFILL_ATTN_OUT_EXACTN"; +static const char *k_disable_classic = "DS4_METAL_DISABLE_Q4_MV_CLASSIC"; + +static void fail(const char *what) { + fprintf(stderr, "Metal Q4 SSD-prefill exact-N oracle FAIL: %s\n", what); + exit(1); +} + +#define CHECK(expr, what) do { if (!(expr)) fail(what); } while (0) + +static uint64_t align_up(uint64_t value, uint64_t alignment) { + return (value + alignment - 1u) & ~(alignment - 1u); +} + +static void pack_scales(uint8_t packed[12], + const uint8_t scale[8], + const uint8_t minimum[8]) { + memset(packed, 0, 12u); + for (uint32_t group = 0; group < 4u; group++) { + packed[group] = scale[group] & 63u; + packed[group + 4u] = minimum[group] & 63u; + } + for (uint32_t group = 4u; group < 8u; group++) { + packed[group + 4u] = (scale[group] & 15u) | + ((minimum[group] & 15u) << 4u); + packed[group - 4u] |= (scale[group] >> 4u) << 6u; + packed[group] |= (minimum[group] >> 4u) << 6u; + } +} + +static void fill_q4_matrix(void *raw, + uint32_t in_dim, + uint32_t rows, + uint32_t salt) { + CHECK(sizeof(block_q4_K) == 144u, "unexpected Q4_K block size"); + CHECK((in_dim % QK_K) == 0u, "unaligned Q4_K fixture"); + const uint32_t blocks_per_row = in_dim / QK_K; + block_q4_K *matrix = raw; + for (uint32_t row = 0; row < rows; row++) { + for (uint32_t block = 0; block < blocks_per_row; block++) { + block_q4_K *b = matrix + + (uint64_t)row * blocks_per_row + block; + const uint32_t key = + salt + row * 1009u + block * 313u + + (row ^ (block * 17u)); + uint8_t scale[8]; + uint8_t minimum[8]; + for (uint32_t group = 0; group < 8u; group++) { + scale[group] = (uint8_t)(1u + (key + group * 7u) % 31u); + minimum[group] = + (uint8_t)((key / 3u + group * 5u) % 17u); + } + pack_scales(b->scales, scale, minimum); + for (uint32_t i = 0; i < QK_K / 2u; i++) { + b->qs[i] = + (uint8_t)(key + i * 37u + (i >> 2u) * 11u); + } + /* Exact binary scales: 2^-5 and 2^-7. */ + b->d = 0x2800u; + b->dmin = 0x2000u; + } + } +} + +static void poison(float *values, uint64_t count, uint32_t base) { + for (uint64_t i = 0; i < count; i++) { + const uint32_t bits = base + (uint32_t)(i & 0xffffu); + memcpy(&values[i], &bits, sizeof(bits)); + } +} + +static uint64_t count_bit_mismatches(const float *reference, + const float *actual, + uint64_t count, + uint64_t *first) { + uint64_t mismatches = 0; + *first = UINT64_MAX; + for (uint64_t i = 0; i < count; i++) { + if (memcmp(&reference[i], &actual[i], sizeof(float)) != 0) { + if (*first == UINT64_MAX) *first = i; + mismatches++; + } + } + return mismatches; +} + +static uint64_t count_poison_mismatches(const float *actual, + uint64_t begin, + uint64_t end, + uint32_t base) { + uint64_t mismatches = 0; + for (uint64_t i = begin; i < end; i++) { + uint32_t bits = 0; + memcpy(&bits, &actual[i], sizeof(bits)); + if (bits != base + (uint32_t)(i & 0xffffu)) mismatches++; + } + return mismatches; +} + +int main(void) { + static const uint32_t exact_rows[] = {6u, 8u, 9u, 16u, 21u, 30u, 31u}; + const uint64_t page = (uint64_t)getpagesize(); + const uint64_t row_a_bytes = (GROUP_DIM / QK_K) * sizeof(block_q4_K); + const uint64_t out_a_bytes = + (uint64_t)N_GROUPS * RANK * row_a_bytes; + const uint64_t out_b_offset = align_up(out_a_bytes, page); + const uint64_t row_b_bytes = (LOW_DIM / QK_K) * sizeof(block_q4_K); + const uint64_t out_b_bytes = (uint64_t)OUT_DIM * row_b_bytes; + const uint64_t model_bytes = + align_up(out_b_offset + out_b_bytes, page); + const uint64_t heads_row_bytes = + (uint64_t)N_GROUPS * GROUP_DIM * sizeof(float); + const uint64_t low_row_bytes = (uint64_t)LOW_DIM * sizeof(float); + const uint64_t out_row_bytes = (uint64_t)OUT_DIM * sizeof(float); + const uint64_t heads_count = (uint64_t)ALLOC_ROWS * N_GROUPS * GROUP_DIM; + const uint64_t low_count = (uint64_t)ALLOC_ROWS * LOW_DIM; + const uint64_t out_count = (uint64_t)ALLOC_ROWS * OUT_DIM; + + CHECK(unsetenv(k_enable) == 0, "clear enable env"); + CHECK(unsetenv(k_disable) == 0, "clear disable env"); + CHECK(unsetenv(k_require) == 0, "clear require env"); + CHECK(unsetenv(k_disable_classic) == 0, "clear classic kill env"); + + CHECK(ds4_gpu_init() != 0, "Metal init"); + if (!ds4_gpu_device_is_pre_m5_apple_silicon()) { + fprintf(stderr, + "Metal Q4 SSD-prefill exact-N oracle SKIP: requires Apple M1--M4\n"); + ds4_gpu_cleanup(); + return 0; + } + + void *model = NULL; + CHECK(posix_memalign(&model, (size_t)page, (size_t)model_bytes) == 0, + "model allocation"); + memset(model, 0, (size_t)model_bytes); + fill_q4_matrix(model, GROUP_DIM, N_GROUPS * RANK, 211u); + fill_q4_matrix((uint8_t *)model + out_b_offset, + LOW_DIM, OUT_DIM, 307u); + + float *heads_host = malloc((size_t)heads_count * sizeof(float)); + float *reference_low_host = calloc((size_t)low_count, sizeof(float)); + float *reference_out_host = calloc((size_t)out_count, sizeof(float)); + float *candidate_low_host = malloc((size_t)low_count * sizeof(float)); + float *candidate_out_host = malloc((size_t)out_count * sizeof(float)); + CHECK(heads_host && reference_low_host && reference_out_host && + candidate_low_host && candidate_out_host, "host tensors"); + + for (uint32_t row = 0; row < ALLOC_ROWS; row++) { + for (uint32_t i = 0; i < N_GROUPS * GROUP_DIM; i++) { + const uint32_t key = + i * 41u + row * 271u + ((i >> 2u) ^ (row * 19u)); + heads_host[(uint64_t)row * N_GROUPS * GROUP_DIM + i] = + (float)((int)(key % 257u) - 128) / 137.0f; + } + } + + ds4_gpu_tensor *heads = + ds4_gpu_tensor_alloc(heads_count * sizeof(float)); + ds4_gpu_tensor *reference_low = + ds4_gpu_tensor_alloc(low_count * sizeof(float)); + ds4_gpu_tensor *reference_out = + ds4_gpu_tensor_alloc(out_count * sizeof(float)); + ds4_gpu_tensor *candidate_low = + ds4_gpu_tensor_alloc(low_count * sizeof(float)); + ds4_gpu_tensor *candidate_out = + ds4_gpu_tensor_alloc(out_count * sizeof(float)); + CHECK(heads && reference_low && reference_out && candidate_low && + candidate_out, "Metal tensors"); + CHECK(ds4_gpu_tensor_write(heads, 0, heads_host, + heads_count * sizeof(float)) != 0, + "heads upload"); + CHECK(ds4_gpu_tensor_write(reference_low, 0, reference_low_host, + low_count * sizeof(float)) != 0, + "reference low clear"); + CHECK(ds4_gpu_tensor_write(reference_out, 0, reference_out_host, + out_count * sizeof(float)) != 0, + "reference out clear"); + CHECK(ds4_gpu_set_model_map(model, model_bytes) != 0, "model map"); + ds4_gpu_set_quality(false); + ds4_gpu_set_ssd_streaming(false); + + /* Canonical oracle: one row at a time through the established classic + * Q4_K attention-A slice and output matvec entry points. */ + for (uint32_t row = 0; row < MAX_ROWS; row++) { + ds4_gpu_tensor *heads_row = ds4_gpu_tensor_view( + heads, (uint64_t)row * heads_row_bytes, heads_row_bytes); + ds4_gpu_tensor *low_row = ds4_gpu_tensor_view( + reference_low, (uint64_t)row * low_row_bytes, low_row_bytes); + ds4_gpu_tensor *out_row = ds4_gpu_tensor_view( + reference_out, (uint64_t)row * out_row_bytes, out_row_bytes); + CHECK(heads_row && low_row && out_row, "reference views"); + CHECK(ds4_gpu_attention_output_low_q4_K_slice_tensor( + low_row, model, model_bytes, 0, + GROUP_DIM, RANK, 0, N_GROUPS, heads_row) != 0, + "reference low projection"); + CHECK(ds4_gpu_matmul_quant_tensor( + out_row, model, model_bytes, out_b_offset, Q4_K_TYPE, + LOW_DIM, OUT_DIM, low_row, 1u) != 0, + "reference output projection"); + ds4_gpu_tensor_free(out_row); + ds4_gpu_tensor_free(low_row); + ds4_gpu_tensor_free(heads_row); + } + CHECK(ds4_gpu_tensor_read(reference_low, 0, reference_low_host, + low_count * sizeof(float)) != 0, + "reference low read"); + CHECK(ds4_gpu_tensor_read(reference_out, 0, reference_out_host, + out_count * sizeof(float)) != 0, + "reference out read"); + + ds4_gpu_set_ssd_streaming(true); + + /* Default-off is observable through the production wrapper. */ + CHECK(ds4_gpu_attention_output_q4_K_batch_tensor( + candidate_out, candidate_low, NULL, NULL, model, model_bytes, + 0, out_b_offset, Q4_K_TYPE, GROUP_DIM, RANK, + N_GROUPS, OUT_DIM, heads, 6u) == 0, + "default-off gate"); + CHECK(setenv(k_enable, "1", 1) == 0, "enable candidate"); + + for (uint32_t case_i = 0; + case_i < sizeof(exact_rows) / sizeof(exact_rows[0]); + case_i++) { + const uint32_t n_rows = exact_rows[case_i]; + poison(candidate_low_host, low_count, 0x7fc10000u); + poison(candidate_out_host, out_count, 0x7fc20000u); + CHECK(ds4_gpu_tensor_write(candidate_low, 0, candidate_low_host, + low_count * sizeof(float)) != 0, + "candidate low poison"); + CHECK(ds4_gpu_tensor_write(candidate_out, 0, candidate_out_host, + out_count * sizeof(float)) != 0, + "candidate out poison"); + + /* Exercise the production wrapper delegation, not only the direct + * test entry point. Scratch arguments are unused by this path. */ + CHECK(ds4_gpu_attention_output_q4_K_batch_tensor( + candidate_out, candidate_low, NULL, NULL, + model, model_bytes, 0, out_b_offset, Q4_K_TYPE, + GROUP_DIM, RANK, N_GROUPS, OUT_DIM, heads, n_rows) == 1, + "candidate dispatch"); + CHECK(ds4_gpu_tensor_read(candidate_low, 0, candidate_low_host, + low_count * sizeof(float)) != 0, + "candidate low read"); + CHECK(ds4_gpu_tensor_read(candidate_out, 0, candidate_out_host, + out_count * sizeof(float)) != 0, + "candidate out read"); + + uint64_t first_low = UINT64_MAX; + uint64_t first_out = UINT64_MAX; + const uint64_t compared_low = (uint64_t)n_rows * LOW_DIM; + const uint64_t compared_out = (uint64_t)n_rows * OUT_DIM; + const uint64_t low_mismatch = count_bit_mismatches( + reference_low_host, candidate_low_host, + compared_low, &first_low); + const uint64_t out_mismatch = count_bit_mismatches( + reference_out_host, candidate_out_host, + compared_out, &first_out); + const uint64_t low_canary = count_poison_mismatches( + candidate_low_host, compared_low, low_count, 0x7fc10000u); + const uint64_t out_canary = count_poison_mismatches( + candidate_out_host, compared_out, out_count, 0x7fc20000u); + fprintf(stderr, + "Metal Q4 SSD-prefill exact-N=%u low=%llu/%llu " + "out=%llu/%llu low_canary=%llu out_canary=%llu\n", + n_rows, + (unsigned long long)low_mismatch, + (unsigned long long)compared_low, + (unsigned long long)out_mismatch, + (unsigned long long)compared_out, + (unsigned long long)low_canary, + (unsigned long long)out_canary); + if (low_mismatch != 0) { + fprintf(stderr, " first low mismatch index=%llu\n", + (unsigned long long)first_low); + } + if (out_mismatch != 0) { + fprintf(stderr, " first out mismatch index=%llu\n", + (unsigned long long)first_out); + } + CHECK(low_mismatch == 0, "low projection bitwise mismatch"); + CHECK(out_mismatch == 0, "output projection bitwise mismatch"); + CHECK(low_canary == 0, "low tail canary"); + CHECK(out_canary == 0, "output tail canary"); + } + + /* REQUIRE implies enable. Both explicit kill switches win and must + * return -1 instead of allowing a false-green row fallback. */ + CHECK(unsetenv(k_enable) == 0, "clear enable before REQUIRE"); + CHECK(setenv(k_require, "1", 1) == 0, "set REQUIRE"); + CHECK(ds4_gpu_attention_output_q4_K_batch_tensor( + candidate_out, candidate_low, NULL, NULL, model, model_bytes, + 0, out_b_offset, Q4_K_TYPE, GROUP_DIM, RANK, + N_GROUPS, OUT_DIM, heads, 21u) == 1, + "REQUIRE implies enable"); + CHECK(setenv(k_disable, "1", 1) == 0, "set exact-N disable"); + CHECK(ds4_gpu_attention_output_q4_K_batch_tensor( + candidate_out, candidate_low, NULL, NULL, model, model_bytes, + 0, out_b_offset, Q4_K_TYPE, GROUP_DIM, RANK, + N_GROUPS, OUT_DIM, heads, 21u) == -1, + "disable wins over REQUIRE"); + CHECK(unsetenv(k_disable) == 0, "clear exact-N disable"); + CHECK(setenv(k_disable_classic, "1", 1) == 0, "set classic disable"); + CHECK(ds4_gpu_attention_output_q4_K_batch_tensor( + candidate_out, candidate_low, NULL, NULL, model, model_bytes, + 0, out_b_offset, Q4_K_TYPE, GROUP_DIM, RANK, + N_GROUPS, OUT_DIM, heads, 21u) == -1, + "classic kill wins over REQUIRE"); + + ds4_gpu_set_ssd_streaming(false); + ds4_gpu_tensor_free(candidate_out); + ds4_gpu_tensor_free(candidate_low); + ds4_gpu_tensor_free(reference_out); + ds4_gpu_tensor_free(reference_low); + ds4_gpu_tensor_free(heads); + ds4_gpu_cleanup(); + free(candidate_out_host); + free(candidate_low_host); + free(reference_out_host); + free(reference_low_host); + free(heads_host); + free(model); + fprintf(stderr, + "Metal Q4 SSD-prefill exact-N oracle PASS rows=6,8,9,16,21,30,31 " + "bitwise=1 canary=1 gates=1\n"); + return 0; +} + +#else + +int main(void) { + fprintf(stderr, "Metal Q4 SSD-prefill exact-N oracle SKIP: non-Apple host\n"); + return 0; +} + +#endif diff --git a/tests/test_mxfp4_cuda.cu b/tests/test_mxfp4_cuda.cu index be81a5442..048faf4e9 100644 --- a/tests/test_mxfp4_cuda.cu +++ b/tests/test_mxfp4_cuda.cu @@ -10,16 +10,6 @@ #include #include -extern "C" int ds4_cuda_q8_fold_take_q81( - const void *src, uint64_t in_dim, cudaStream_t stream, - const void **q81) { - (void)src; - (void)in_dim; - (void)stream; - if (q81) *q81 = nullptr; - return 0; -} - namespace { constexpr int QK = 32; From 7e802d6b4638fb05612a192a41e09458c117eb97 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Sat, 22 Aug 2026 15:42:13 +0200 Subject: [PATCH 047/189] Update ENVIRONMENT_VARIABLES.md Co-authored-by: Frank --- ENVIRONMENT_VARIABLES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ENVIRONMENT_VARIABLES.md b/ENVIRONMENT_VARIABLES.md index d37cf54ea..416811edc 100644 --- a/ENVIRONMENT_VARIABLES.md +++ b/ENVIRONMENT_VARIABLES.md @@ -38,7 +38,7 @@ The detailed Metal A/B contracts and expected oracle counters live in ## CUDA Q4 and Q8 diagnostics -| Variable | Default and purpose | +| Variable | Default behavior and purpose | | --- | --- | | `DS4_CUDA_DISABLE_Q4_DENSE_PAIR=1` | Split the Q-A/KV Q4 pair back into two standalone projections. | | `DS4_CUDA_NO_Q4_GB10_FAST=1` | Umbrella rollback for the GB10-specific Q4 choices; it does not disable the older cross-CUDA dense pair. | From f94e1cfe47dbe25e3c897d39315d341fff2a559c Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Sat, 22 Aug 2026 15:42:37 +0200 Subject: [PATCH 048/189] Update ENVIRONMENT_VARIABLES.md Co-authored-by: Frank --- ENVIRONMENT_VARIABLES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ENVIRONMENT_VARIABLES.md b/ENVIRONMENT_VARIABLES.md index 416811edc..d54da6f9f 100644 --- a/ENVIRONMENT_VARIABLES.md +++ b/ENVIRONMENT_VARIABLES.md @@ -54,7 +54,7 @@ The detailed Metal A/B contracts and expected oracle counters live in ## ROCm Q4 -| Variable | Default and purpose | +| Variable | Default behavior and purpose | | --- | --- | | `DS4_ROCM_DISABLE_Q4_PREFILL_TILE8=1` | Restore the legacy Q4 prefill kernel. TILE8 is automatic for validated chunks of 9 through 4096 tokens. | | `DS4_ROCM_REQUIRE_Q4_PREFILL_TILE8=1` | Fail closed when an eligible Q4 prefill call cannot use TILE8. | From b91bb77268da877802485e74f58757dc17ff7779 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Sat, 22 Aug 2026 15:42:53 +0200 Subject: [PATCH 049/189] Update ENVIRONMENT_VARIABLES.md Co-authored-by: Frank --- ENVIRONMENT_VARIABLES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ENVIRONMENT_VARIABLES.md b/ENVIRONMENT_VARIABLES.md index d54da6f9f..070f0d204 100644 --- a/ENVIRONMENT_VARIABLES.md +++ b/ENVIRONMENT_VARIABLES.md @@ -18,7 +18,7 @@ changing them. Unless a row says otherwise: ## Metal -| Variable | Default and purpose | +| Variable | Default behavior and purpose | | --- | --- | | `DS4_METAL_PREFILL_CHUNK=N` | Set the prefill cap when `--prefill-chunk` is absent; the CLI option takes precedence. This historical name is consumed by the shared graph planner rather than by a Metal kernel alone. | | `DS4_METAL_NO_RESIDENCY=1` | Skip creation and residency requests for the model-view residency set. Diagnostic rollback for resident, non-streaming models. | From 7bd98a922602ebcb6a7317830768a758a1a157cb Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Sat, 22 Aug 2026 15:43:06 +0200 Subject: [PATCH 050/189] Update metal/moe.metal Co-authored-by: Frank --- metal/moe.metal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metal/moe.metal b/metal/moe.metal index 7424c1210..e70249539 100644 --- a/metal/moe.metal +++ b/metal/moe.metal @@ -2956,7 +2956,7 @@ void kernel_mul_mv_q4_K_staged_exactn_impl( for (int ib = ix; ib < nb; ib += 4) { float4 sumy = {0.f, 0.f, 0.f, 0.f}; - for (short i = 0; i < 8; ++i) { + FOR_UNROLL (short i = 0; i < 8; ++i) { yl[i + 0] = y4[i + 0]; sumy[0] += yl[i + 0]; yl[i + 8] = y4[i + 32]; sumy[1] += yl[i + 8]; yh[i + 0] = y4[i + 128]; sumy[2] += yh[i + 0]; From 68a8ef9626dd2c80613f75d154c7b4fe51f68472 Mon Sep 17 00:00:00 2001 From: Giorgio Oppo Date: Sat, 22 Aug 2026 19:50:54 +0200 Subject: [PATCH 051/189] docs: document all environment variables --- CONTRIBUTING.md | 9 + ENVIRONMENT_VARIABLES.md | 1338 ++++++++++++++++++++- Makefile | 7 +- README.md | 5 +- scripts/environment_variables.tsv | 1195 ++++++++++++++++++ scripts/generate_environment_variables.py | 584 +++++++++ 6 files changed, 3121 insertions(+), 17 deletions(-) create mode 100644 scripts/environment_variables.tsv create mode 100644 scripts/generate_environment_variables.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f9bb07168..1571397ff 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -25,6 +25,15 @@ The C test runner is `ds4_test`. Running it without arguments is equivalent to make test ``` +If a change adds, renames, removes, or changes the parsing/default/effect of an +environment variable, update `scripts/environment_variables.tsv`, regenerate +the complete reference, and verify that it is current: + +```sh +python3 scripts/generate_environment_variables.py +make check-environment-docs +``` + Useful narrower checks: ```sh diff --git a/ENVIRONMENT_VARIABLES.md b/ENVIRONMENT_VARIABLES.md index 070f0d204..35b0d9261 100644 --- a/ENVIRONMENT_VARIABLES.md +++ b/ENVIRONMENT_VARIABLES.md @@ -1,9 +1,15 @@ -# Performance and diagnostic environment variables +# Environment variables Command-line options are the supported interface for normal inference. The -variables below are the curated, user-facing switches used to isolate optimized -paths, require test coverage, or collect diagnostics. They are not a promise -that every internal `getenv()` knob is a stable API. +first part of this document is the curated, user-facing reference for switches +used to isolate optimized paths, require test coverage, or collect diagnostics. +The generated inventory below it lists every environment name consumed by the +runtime, tests, and maintained tools. Inclusion in that complete inventory does +not make an internal tuning knob a stable API. + +Run `make check-environment-docs` after adding, renaming, or removing an +environment variable. Regenerate the inventory with +`python3 scripts/generate_environment_variables.py`. Most backend switches are cached on first use. Start a new process when changing them. Unless a row says otherwise: @@ -28,7 +34,7 @@ changing them. Unless a row says otherwise: | `DS4_METAL_DISABLE_Q4_DENSE_PAIR=1` | Split the default Metal Q-A/KV Q4 pair back into two standalone projections. | | `DS4_METAL_DISABLE_M1_IQ2_MID_ONLY=1` | Restore the canonical IQ2 address-table gate/up producer on the exact M1 SSD-streaming decode shape. The specialization is automatic by default. | | `DS4_METAL_REQUIRE_M1_IQ2_MID_ONLY=1` | Fail closed when an otherwise eligible M1 IQ2 mid-only dispatch cannot use the specialization. | -| `DS4_METAL_ENABLE_M1_IQ2_MID_ONLY=1` | Compatibility alias for the former opt-in. It is harmless because the path is now automatic. | +| `DS4_METAL_ENABLE_M1_IQ2_MID_ONLY=1` | Legacy spelling retained for migration notes only. The runtime does not read it; the path is automatic and this setting is ignored. | | `DS4_METAL_DISABLE_IQ2_XXS_SSD_PREFILL_MM=1` | Restore sparse matvec for an eligible grouped IQ2 SSD-prefill chunk. | | `DS4_METAL_REQUIRE_IQ2_XXS_SSD_PREFILL_MM=1` | Require grouped IQ2 SSD-prefill MM for eligible chunks and reject insufficient cache instead of silently falling back. | | `DS4_METAL_ENABLE_STREAMING_PREFILL_EXPERT_READAHEAD=1` | Restore the historical `F_RDADVISE` plus parallel-`pread` sequence for cold-storage A/B tests. Normal grouped prefill skips the redundant hint. | @@ -40,6 +46,7 @@ The detailed Metal A/B contracts and expected oracle counters live in | Variable | Default behavior and purpose | | --- | --- | +| `DS4_CUDA_DECODE_GRAPHS=0` | Disable CUDA decode graph capture. Unset, `1`, `on`, `yes`, or `true` enables capture; `0`, `off`, `no`, or `false` disables it. Oracle modes may also suppress capture. | | `DS4_CUDA_DISABLE_Q4_DENSE_PAIR=1` | Split the Q-A/KV Q4 pair back into two standalone projections. | | `DS4_CUDA_NO_Q4_GB10_FAST=1` | Umbrella rollback for the GB10-specific Q4 choices; it does not disable the older cross-CUDA dense pair. | | `DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_BATCH=1` | Enable grouped attention-A for two-to-eight-token GB10 verifier batches. | @@ -58,7 +65,7 @@ The detailed Metal A/B contracts and expected oracle counters live in | --- | --- | | `DS4_ROCM_DISABLE_Q4_PREFILL_TILE8=1` | Restore the legacy Q4 prefill kernel. TILE8 is automatic for validated chunks of 9 through 4096 tokens. | | `DS4_ROCM_REQUIRE_Q4_PREFILL_TILE8=1` | Fail closed when an eligible Q4 prefill call cannot use TILE8. | -| `DS4_ROCM_ENABLE_Q4_PREFILL_TILE8=1` | Legacy no-op accepted by existing scripts; TILE8 no longer requires an enable variable. | +| `DS4_ROCM_ENABLE_Q4_PREFILL_TILE8=1` | Legacy spelling retained for migration notes only. The runtime does not read it; TILE8 is automatic and this setting is ignored. | | `DS4_ROCM_Q4_PREFILL_TILE8_STATS=1` | Report dense, pair, attention-batch, and token counters at process exit. | | `DS4_ROCM_ENABLE_Q4_DENSE_PAIR=1` | Share one Q8_K activation quantization between the two Q4 dense projections. This pair remains opt-in. | | `DS4_ROCM_DISABLE_Q4_DENSE_PAIR=1` | Dominant rollback for the ROCm Q4 dense pair. | @@ -78,16 +85,1319 @@ The shared graph implementation predates the CUDA backend and retained a few `DS4_METAL_*` names. In a non-ROCm GPU build, these controls affect the shared Metal/CUDA graph policy; ROCm explicitly ignores them: -- `DS4_METAL_DISABLE_HC_FUSION` -- `DS4_METAL_DISABLE_HC_NORM_FUSION` -- `DS4_METAL_DISABLE_KV_FUSION` -- `DS4_METAL_DISABLE_QKV_NORM_FUSION` -- `DS4_METAL_DISABLE_QKV_PAIR_PROJ` -- `DS4_METAL_DISABLE_COMPRESSOR_PAIR_PROJ` -- `DS4_METAL_DISABLE_ATTN_OUT_HC_FUSION` -- `DS4_METAL_DISABLE_SHARED_DOWN_HC_FUSION` +- `DS4_METAL_DISABLE_HC_FUSION=1` +- `DS4_METAL_DISABLE_HC_NORM_FUSION=1` +- `DS4_METAL_DISABLE_KV_FUSION=1` +- `DS4_METAL_DISABLE_QKV_NORM_FUSION=1` +- `DS4_METAL_DISABLE_QKV_PAIR_PROJ=1` +- `DS4_METAL_DISABLE_COMPRESSOR_PAIR_PROJ=1` +- `DS4_METAL_DISABLE_ATTN_OUT_HC_FUSION=1` +- `DS4_METAL_DISABLE_SHARED_DOWN_HC_FUSION=1` The prefix is historical rather than an indication that these particular switches are always Metal-only. New backend-specific controls should use the backend they actually configure (`DS4_CUDA_*` or `DS4_ROCM_*`) instead of extending this legacy naming. + + +## Complete implementation inventory + +This section is generated by `scripts/generate_environment_variables.py`; do not edit it by hand. +Human-reviewed value/default and purpose metadata lives in +`scripts/environment_variables.tsv`; the generator verifies it against the source tree. +It lists every `DS4_*` string consumed by production C/C++/Objective-C/CUDA/ROCm +sources, including names passed indirectly through helper functions, macros, and +source-specification arrays. Unless a variable appears in the user-facing reference +above, it is an unstable internal diagnostic or tuning interface. The linked source +remains normative for exact eligibility +gates, bounds, and architecture-specific defaults. + +Inventory totals: **1057 `DS4_*` runtime variables** and +**6 external runtime variables**. +The auxiliary inventories contain **112 test/test-fixture entries** +and **19 tool/wrapper entries**. + +
+Metal (436) + +| Variable | Accepted value and default | Effect | Source | +| --- | --- | --- | --- | +| `DS4_METAL_ARGSORT_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the argsort Metal kernel source file loaded at runtime. | [ds4_metal.m:4942](ds4_metal.m#L4942) | +| `DS4_METAL_ATTN_OUT_STAGE_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for attn out stage. | [ds4.c:64961](ds4.c#L64961) | +| `DS4_METAL_BIN_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the binary operations Metal kernel source file loaded at runtime. | [ds4_metal.m:4951](ds4_metal.m#L4951) | +| `DS4_METAL_COMPRESSOR_PAIR_NR4` | presence control; unset: off/default; any value including 0 enables | Selects the NR4 compressor-pair variant. | [ds4_metal.m:2650](ds4_metal.m#L2650) | +| `DS4_METAL_CONCAT_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the concatenation Metal kernel source file loaded at runtime. | [ds4_metal.m:4944](ds4_metal.m#L4944) | +| `DS4_METAL_CPY_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the copy Metal kernel source file loaded at runtime. | [ds4_metal.m:4943](ds4_metal.m#L4943) | +| `DS4_METAL_DECODE_INDEXER_SPARSE_THRESHOLD` | integer in {64,128,256,512,1024,2048,4096}; default 1024; invalid restores default | Sets the compressed-row crossover from dense to sparse indexed attention. | [ds4.c:20179](ds4.c#L20179) | +| `DS4_METAL_DECODE_STAGE_PROFILE` | unset: off; 1/true/yes/on/all enables all layers; a layer index selects one; 0/false/no/off disables | Prints timing/profile diagnostics for decode stage. | [ds4.c:17395](ds4.c#L17395) | +| `DS4_METAL_DECODE_STAGE_PROFILE_LAYER` | single unsigned layer index; unset/empty: all layers enabled by the parent profile; invalid matches no layer | Restricts the corresponding shared graph stage profiler to one layer. | [ds4.c:28947](ds4.c#L28947) | +| `DS4_METAL_DENSE_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the dense matmul Metal kernel source file loaded at runtime. | [ds4_metal.m:4935](ds4_metal.m#L4935) | +| `DS4_METAL_DISABLE_AFFINE_ROPE_PAIR` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables affine RoPE pair. | [ds4_metal.m:25169](ds4_metal.m#L25169) | +| `DS4_METAL_DISABLE_ATTN_OUT_HC_FUSION` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Disables attn out HC fusion. | [ds4.c:20304](ds4.c#L20304) | +| `DS4_METAL_DISABLE_ATTN_OUT_IDS_CACHE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables attn out ids cache. | [ds4_metal.m:27769](ds4_metal.m#L27769) | +| `DS4_METAL_DISABLE_ATTN_OUT_LOW_DIRECT` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables attn out low direct. | [ds4_metal.m:27758](ds4_metal.m#L27758) | +| `DS4_METAL_DISABLE_BATCH_HC_NORM_FUSION` | nonempty value other than exact 0 disables; unset/empty/0 leaves the default enabled path | Dominant rollback for batched HC norm fusion. | [ds4.c:20284](ds4.c#L20284) | +| `DS4_METAL_DISABLE_COMPRESSOR_APE_ADD` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables compressor APE add. | [ds4_metal.m:25391](ds4_metal.m#L25391) | +| `DS4_METAL_DISABLE_COMPRESSOR_EXACT_POOL_RATIO4` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables compressor exact pool ratio4. | [ds4_metal.m:26377](ds4_metal.m#L26377) | +| `DS4_METAL_DISABLE_COMPRESSOR_PAIR_PROJ` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Disables compressor pair proj. | [ds4_metal.m:23246](ds4_metal.m#L23246) | +| `DS4_METAL_DISABLE_COMPRESSOR_QUAD_STORE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables compressor quad store. | [ds4.c:23361](ds4.c#L23361) | +| `DS4_METAL_DISABLE_COMPRESSOR_RATIO4_DIRECT_POOL` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables compressor ratio4 direct pool. | [ds4_metal.m:26243](ds4_metal.m#L26243) | +| `DS4_METAL_DISABLE_COMPRESSOR_RATIO4_PACK_FUSION` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables compressor ratio4 pack fusion. | [ds4_metal.m:26197](ds4_metal.m#L26197) | +| `DS4_METAL_DISABLE_COMPRESSOR_STORE_ONE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables compressor store one. | [ds4_metal.m:23247](ds4_metal.m#L23247) | +| `DS4_METAL_DISABLE_CONTIG_F16_F16_COPY` | value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables | Disables contig F16 F16 copy. | [ds4_metal.m:29538](ds4_metal.m#L29538) | +| `DS4_METAL_DISABLE_CONTIG_F32_F16_COPY` | value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables | Disables contig F32 F16 copy. | [ds4_metal.m:29314](ds4_metal.m#L29314) | +| `DS4_METAL_DISABLE_DECODE_NORM_EXACT_VIEWS` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables decode norm exact views. | [ds4_metal.m:36173](ds4_metal.m#L36173) | +| `DS4_METAL_DISABLE_DECODE_ROUTER_BIAS_EXACT_VIEWS` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables decode router bias exact views. | [ds4_metal.m:39363](ds4_metal.m#L39363) | +| `DS4_METAL_DISABLE_DSPARK_CAPTURE_FUSED_LAST` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Disables DSpark capture fused last. | [ds4.c:28188](ds4.c#L28188) | +| `DS4_METAL_DISABLE_DSPARK_EXACTN_BATCH_HEAD` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Disables DSpark exactn batch head. | [ds4.c:37538](ds4.c#L37538) | +| `DS4_METAL_DISABLE_EXACT_ROWS_PERSISTENT_CACHE` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Disables exact rows persistent cache. | [ds4_metal.m:12998](ds4_metal.m#L12998) | +| `DS4_METAL_DISABLE_GATHERED_KV_PAD_FUSION` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables gathered KV pad fusion. | [ds4_metal.m:29682](ds4_metal.m#L29682) | +| `DS4_METAL_DISABLE_GATHERED_KV_STAGE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables gathered KV stage. | [ds4_metal.m:29653](ds4_metal.m#L29653) | +| `DS4_METAL_DISABLE_GLM_DECODE_KV_GROUP4` | value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables | Disables GLM decode KV group4. | [ds4_metal.m:36708](ds4_metal.m#L36708) | +| `DS4_METAL_DISABLE_GLM_QKLOW_SG` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables GLM qklow sg. | [ds4_metal.m:37831](ds4_metal.m#L37831) | +| `DS4_METAL_DISABLE_GLM_STREAMING_EXPERT_EARLY_LOAD` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables GLM streaming expert early load. | [ds4_metal.m:18056](ds4_metal.m#L18056) | +| `DS4_METAL_DISABLE_GLM_STREAMING_EXPERT_SPLIT` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables GLM streaming expert split. | [ds4_metal.m:39762](ds4_metal.m#L39762) | +| `DS4_METAL_DISABLE_GLM_STREAMING_PREFILL_FULL_LAYER` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables GLM streaming prefill full layer. | [ds4.c:42520](ds4.c#L42520) | +| `DS4_METAL_DISABLE_GLM_STREAMING_PREFILL_FULL_LAYER_PREPARE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables GLM streaming prefill full layer prepare. | [ds4.c:42538](ds4.c#L42538) | +| `DS4_METAL_DISABLE_GLM_STREAMING_PREFILL_SELECTED_ASYNC_LOAD` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables GLM streaming prefill selected async load. | [ds4.c:46231](ds4.c#L46231) | +| `DS4_METAL_DISABLE_GLM_STREAMING_SELECTED_ASYNC_LOAD` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables GLM streaming selected async load. | [ds4.c:44139](ds4.c#L44139) | +| `DS4_METAL_DISABLE_HC_FUSION` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Disables HC fusion. | [ds4.c:20233](ds4.c#L20233) | +| `DS4_METAL_DISABLE_HC_NORM_FUSION` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Disables HC norm fusion. | [ds4.c:20277](ds4.c#L20277) | +| `DS4_METAL_DISABLE_HC_PRODUCER_PRE_NORM_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables HC producer pre norm fuse. | [ds4_metal.m:46507](ds4_metal.m#L46507) | +| `DS4_METAL_DISABLE_HC_RMS_SCALE_PROJ` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables HC RMS scale proj. | [ds4_metal.m:24146](ds4_metal.m#L24146) | +| `DS4_METAL_DISABLE_HOT_PIPELINE_STATICS` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables hot pipeline statics. | [ds4_metal.m:2633](ds4_metal.m#L2633) | +| `DS4_METAL_DISABLE_INPLACE_ROPE_PAIR` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables inplace RoPE pair. | [ds4_metal.m:25168](ds4_metal.m#L25168) | +| `DS4_METAL_DISABLE_IQ2_SELECTED_EXPERT_VIEWS` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables IQ2 selected expert views. | [ds4.c:21069](ds4.c#L21069) | +| `DS4_METAL_DISABLE_IQ2_SELECTED_SHARED_OVERLAP` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables IQ2 selected shared overlap. | [ds4.c:20823](ds4.c#L20823) | +| `DS4_METAL_DISABLE_IQ2_STREAM_ADDR_TABLE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables IQ2 stream address table. | [ds4_metal.m:42368](ds4_metal.m#L42368) | +| `DS4_METAL_DISABLE_IQ2_XXS_SSD_PREFILL_MM` | value-aware boolean; default off; true disables and dominates ENABLE; false leaves automatic policy | Rolls grouped IQ2_XXS/Q2_K SSD-prefill MM back to sparse matvec. | [ds4_metal.m:44750](ds4_metal.m#L44750) | +| `DS4_METAL_DISABLE_KV_FUSION` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Disables KV fusion. | [ds4.c:20257](ds4.c#L20257) | +| `DS4_METAL_DISABLE_M1_IQ2_MID_ONLY` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables M1 IQ2 mid only. | [ds4_metal.m:14590](ds4_metal.m#L14590) | +| `DS4_METAL_DISABLE_M3_COMPRESSOR_EXACT_POOL_RATIO4` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables M3 compressor exact pool ratio4. | [ds4_metal.m:26379](ds4_metal.m#L26379) | +| `DS4_METAL_DISABLE_M3_COMPRESSOR_PAIR_STATE_STORE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables M3 compressor pair state store. | [ds4_metal.m:23245](ds4_metal.m#L23245) | +| `DS4_METAL_DISABLE_M3_GATHERED_KV_STAGE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables M3 gathered KV stage. | [ds4_metal.m:29654](ds4_metal.m#L29654) | +| `DS4_METAL_DISABLE_M5_COMPRESSOR_EXACT_POOL_RATIO4` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables M5 compressor exact pool ratio4. | [ds4_metal.m:26382](ds4_metal.m#L26382) | +| `DS4_METAL_DISABLE_M5_COMP_FINALIZE_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables M5 comp finalize fuse. | [ds4.c:23474](ds4.c#L23474) | +| `DS4_METAL_DISABLE_M5_FLASH_ATTN_PACKED32_REDUCE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables M5 flash attn packed32 reduce. | [ds4_metal.m:31490](ds4_metal.m#L31490) | +| `DS4_METAL_DISABLE_M5_HC_NORM_MIX_CLUSTER2` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables M5 HC norm mix cluster2. | [ds4_metal.m:46462](ds4_metal.m#L46462) | +| `DS4_METAL_DISABLE_M5_HC_PRODUCER_PRE_NORM_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables M5 HC producer pre norm fuse. | [ds4_metal.m:46514](ds4_metal.m#L46514) | +| `DS4_METAL_DISABLE_M5_IQ2_PAIR_PACK2` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables M5 IQ2 pair pack2. | [ds4_metal.m:41986](ds4_metal.m#L41986) | +| `DS4_METAL_DISABLE_M5_PACKED_ZERO_MASK` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables M5 packed zero mask. | [ds4_metal.m:31446](ds4_metal.m#L31446) | +| `DS4_METAL_DISABLE_M5_PARALLEL_FULL_FFN` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables M5 parallel full FFN. | [ds4.c:22534](ds4.c#L22534) | +| `DS4_METAL_DISABLE_M5_PERSISTENT_ZERO_ATTN_MASK` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables M5 persistent zero attn mask. | [ds4_metal.m:31444](ds4_metal.m#L31444) | +| `DS4_METAL_DISABLE_M5_Q8_HC_VEC` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables M5 Q8 HC vec. | [ds4_metal.m:47546](ds4_metal.m#L47546) | +| `DS4_METAL_DISABLE_M5_QKV_PAIR_COMPRESSOR_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables M5 QKV pair compressor fuse. | [ds4.c:22870](ds4.c#L22870) | +| `DS4_METAL_DISABLE_M5_QKV_PAIR_QUAD_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables M5 QKV pair quad fuse. | [ds4.c:22866](ds4.c#L22866) | +| `DS4_METAL_DISABLE_M5_ROUTER_PROJECT_SELECT_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables M5 router project select fuse. | [ds4.c:24589](ds4.c#L24589) | +| `DS4_METAL_DISABLE_METAL4` | value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables | Disables metal4. | [ds4_metal.m:2994](ds4_metal.m#L2994) | +| `DS4_METAL_DISABLE_MOE_MM_ID_PAIR_SWIGLU` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables MoE MM ID pair SwiGLU. | [ds4_metal.m:44934](ds4_metal.m#L44934) | +| `DS4_METAL_DISABLE_MOE_MM_ID_USE_RESOURCES` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables MoE MM ID use resources. | [ds4_metal.m:35135](ds4_metal.m#L35135) | +| `DS4_METAL_DISABLE_MXFP4_SELECTED_EXPERT_VIEWS` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables MXFP4 selected expert views. | [ds4.c:21000](ds4.c#L21000) | +| `DS4_METAL_DISABLE_PERSISTENT_ZERO_ATTN_MASK` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables persistent zero attn mask. | [ds4_metal.m:31451](ds4_metal.m#L31451) | +| `DS4_METAL_DISABLE_PRE_M5_ATTN_INV_ROPE_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 attn inv RoPE fuse. | [ds4.c:22465](ds4.c#L22465) | +| `DS4_METAL_DISABLE_PRE_M5_COMPRESSOR_EXACT_POOL_RATIO4` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 compressor exact pool ratio4. | [ds4_metal.m:26381](ds4_metal.m#L26381) | +| `DS4_METAL_DISABLE_PRE_M5_COMPRESSOR_EXACT_REDUCTION_FUSION` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 compressor exact reduction fusion. | [ds4_metal.m:25905](ds4_metal.m#L25905) | +| `DS4_METAL_DISABLE_PRE_M5_COMPRESSOR_QUAD_STORE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 compressor quad store. | [ds4.c:23362](ds4.c#L23362) | +| `DS4_METAL_DISABLE_PRE_M5_COMPRESSOR_RATIO4_DECODE_PACK_FUSION` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 compressor ratio4 decode pack fusion. | [ds4_metal.m:26212](ds4_metal.m#L26212) | +| `DS4_METAL_DISABLE_PRE_M5_COMP_FINALIZE_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 comp finalize fuse. | [ds4.c:23473](ds4.c#L23473) | +| `DS4_METAL_DISABLE_PRE_M5_DECODE_EARLY_PIPELINE_FAST_LOOKUP` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 decode early pipeline fast lookup. | [ds4.c:27830](ds4.c#L27830) | +| `DS4_METAL_DISABLE_PRE_M5_DECODE_EARLY_SECOND_SPLIT12` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 decode early second split12. | [ds4.c:27914](ds4.c#L27914) | +| `DS4_METAL_DISABLE_PRE_M5_DECODE_EARLY_SPLIT3` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 decode early split3. | [ds4.c:27788](ds4.c#L27788) | +| `DS4_METAL_DISABLE_PRE_M5_DECODE_EARLY_SPLIT5` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 decode early split5. | [ds4.c:27800](ds4.c#L27800) | +| `DS4_METAL_DISABLE_PRE_M5_DECODE_PIPELINE_FAST_LOOKUP` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 decode pipeline fast lookup. | [ds4.c:27842](ds4.c#L27842) | +| `DS4_METAL_DISABLE_PRE_M5_DECODE_PORTS` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 decode ports. | [ds4.c:22440](ds4.c#L22440) | +| `DS4_METAL_DISABLE_PRE_M5_DECODE_RAW_ZERO_ATTN_MASK` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 decode raw zero attn mask. | [ds4_metal.m:29893](ds4_metal.m#L29893) | +| `DS4_METAL_DISABLE_PRE_M5_DECODE_SECOND_SPLIT16` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 decode second split16. | [ds4.c:27922](ds4.c#L27922) | +| `DS4_METAL_DISABLE_PRE_M5_FLASH_ATTN_BATCHED_MEMO` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 flash attn batched memo. | [ds4_metal.m:3751](ds4_metal.m#L3751) | +| `DS4_METAL_DISABLE_PRE_M5_FLASH_ATTN_PACKED32_REDUCE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 flash attn packed32 reduce. | [ds4_metal.m:31433](ds4_metal.m#L31433) | +| `DS4_METAL_DISABLE_PRE_M5_FLASH_ATTN_PAD_BLK_MEMO` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 flash attn pad blk memo. | [ds4_metal.m:3609](ds4_metal.m#L3609) | +| `DS4_METAL_DISABLE_PRE_M5_HC_NORM_MIX_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 HC norm mix fuse. | [ds4.c:22694](ds4.c#L22694) | +| `DS4_METAL_DISABLE_PRE_M5_HC_PRODUCER_PRE_NORM_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 HC producer pre norm fuse. | [ds4_metal.m:46512](ds4_metal.m#L46512) | +| `DS4_METAL_DISABLE_PRE_M5_HEAD_RMS_ROPE_PIPELINE_STATIC` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 head RMS RoPE pipeline static. | [ds4_metal.m:10025](ds4_metal.m#L10025) | +| `DS4_METAL_DISABLE_PRE_M5_KV_ROPE_FP8_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 KV RoPE fp8 fuse. | [ds4.c:23282](ds4.c#L23282) | +| `DS4_METAL_DISABLE_PRE_M5_MXFP4_MM_ID_PAIR_HALF_SCALE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 MXFP4 MM ID pair half scale. | [ds4_metal.m:45095](ds4_metal.m#L45095) | +| `DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_DECODE_FIXED_ROUTE_PAIR` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 MXFP4 MoE decode fixed route pair. | [ds4_metal.m:41914](ds4_metal.m#L41914) | +| `DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_DECODE_FIXED_ROUTE_SUM6` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 MXFP4 MoE decode fixed route sum6. | [ds4_metal.m:41927](ds4_metal.m#L41927) | +| `DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_DECODE_NSG1` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 MXFP4 MoE decode nsg1. | [ds4_metal.m:41685](ds4_metal.m#L41685) | +| `DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_DECODE_STATIC_TRIP` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 MXFP4 MoE decode static trip. | [ds4_metal.m:41953](ds4_metal.m#L41953) | +| `DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_DECODE_SUM6_FULL_ROWS` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 MXFP4 MoE decode sum6 full rows. | [ds4_metal.m:41940](ds4_metal.m#L41940) | +| `DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_DECODE_TG_MULTIPLE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 MXFP4 MoE decode tg multiple. | [ds4_metal.m:41902](ds4_metal.m#L41902) | +| `DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_MM_ID_DOWN_HALF_LUT` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 MXFP4 MoE MM ID down half lut. | [ds4_metal.m:45013](ds4_metal.m#L45013) | +| `DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_MM_ID_DOWN_TAIL_SIMDGROUP_CULL` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 MXFP4 MoE MM ID down tail simdgroup cull. | [ds4_metal.m:44996](ds4_metal.m#L44996) | +| `DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_MM_ID_MAP_SCATTER` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 MXFP4 MoE MM ID map scatter. | [ds4_metal.m:44963](ds4_metal.m#L44963) | +| `DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_MM_ID_PAIR_SWIGLU_COMPACT_TILE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 MXFP4 MoE MM ID pair SwiGLU compact tile. | [ds4_metal.m:44947](ds4_metal.m#L44947) | +| `DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_MM_ID_PAIR_TAIL_SIMDGROUP_CULL` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 MXFP4 MoE MM ID pair tail simdgroup cull. | [ds4_metal.m:44984](ds4_metal.m#L44984) | +| `DS4_METAL_DISABLE_PRE_M5_PARALLEL_FULL_FFN` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 parallel full FFN. | [ds4.c:22533](ds4.c#L22533) | +| `DS4_METAL_DISABLE_PRE_M5_Q2_DECODE_SPLIT2_32` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 q2 decode split2 32. | [ds4.c:27703](ds4.c#L27703) | +| `DS4_METAL_DISABLE_PRE_M5_QKV_NORM_KV_STORE_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 QKV norm KV store fuse. | [ds4.c:23141](ds4.c#L23141) | +| `DS4_METAL_DISABLE_PRE_M5_QKV_PAIR_COMPRESSOR_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 QKV pair compressor fuse. | [ds4.c:22869](ds4.c#L22869) | +| `DS4_METAL_DISABLE_PRE_M5_QKV_PAIR_QUAD_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 QKV pair quad fuse. | [ds4.c:22865](ds4.c#L22865) | +| `DS4_METAL_DISABLE_PRE_M5_ROUTER_SHARED_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 router shared fuse. | [ds4.c:24582](ds4.c#L24582) | +| `DS4_METAL_DISABLE_PRE_M5_ROUTER_SIMD_FINALIZE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 router simd finalize. | [ds4_metal.m:35793](ds4_metal.m#L35793) | +| `DS4_METAL_DISABLE_PRE_M5_ROUTER_SIMD_WEIGHTS_FUSION` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 router simd weights fusion. | [ds4_metal.m:35802](ds4_metal.m#L35802) | +| `DS4_METAL_DISABLE_PRE_M5_ROUTER_TRANSFORM_FINALIZE_FUSION` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 router transform finalize fusion. | [ds4_metal.m:35806](ds4_metal.m#L35806) | +| `DS4_METAL_DISABLE_PRO_Q4_EXPERT_ADDRESS_AUTO` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pro Q4 expert address auto. | [ds4_metal.m:19708](ds4_metal.m#L19708) | +| `DS4_METAL_DISABLE_PRO_Q4_EXPERT_TABLE_AUTO` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pro Q4 expert table auto. | [ds4.c:20867](ds4.c#L20867) | +| `DS4_METAL_DISABLE_PRO_Q4_EXPERT_TABLE_PRELOAD` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pro Q4 expert table preload. | [ds4.c:58590](ds4.c#L58590) | +| `DS4_METAL_DISABLE_Q4_ATTN_OUT_HC_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 attn out HC fuse. | [ds4_metal.m:47590](ds4_metal.m#L47590) | +| `DS4_METAL_DISABLE_Q4_ATTN_OUT_TINY_BATCH` | value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables | Disables Q4 attn out tiny batch. | [ds4_metal.m:28372](ds4_metal.m#L28372) | +| `DS4_METAL_DISABLE_Q4_BATCH_EXPERT_TABLE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 batch expert table. | [ds4_metal.m:44849](ds4_metal.m#L44849) | +| `DS4_METAL_DISABLE_Q4_DENSE_PAIR` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 dense pair. | [ds4_metal.m:21988](ds4_metal.m#L21988) | +| `DS4_METAL_DISABLE_Q4_EXACT_BOUNDARY` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 exact boundary. | [ds4_metal.m:42205](ds4_metal.m#L42205) | +| `DS4_METAL_DISABLE_Q4_EXACT_TENSOR_ID` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 exact tensor ID. | [ds4_metal.m:42185](ds4_metal.m#L42185) | +| `DS4_METAL_DISABLE_Q4_EXPERT_ADDRESS_TABLE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 expert address table. | [ds4_metal.m:19709](ds4_metal.m#L19709) | +| `DS4_METAL_DISABLE_Q4_EXPERT_TABLE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 expert table. | [ds4.c:20868](ds4.c#L20868) | +| `DS4_METAL_DISABLE_Q4_GATHER_SLOTS` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 gather slots. | [ds4_metal.m:42287](ds4_metal.m#L42287) | +| `DS4_METAL_DISABLE_Q4_GROUP24_EXPERT_TABLE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 group24 expert table. | [ds4_metal.m:42167](ds4_metal.m#L42167) | +| `DS4_METAL_DISABLE_Q4_GROUP6_EXPERT_TABLE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 group6 expert table. | [ds4_metal.m:42133](ds4_metal.m#L42133) | +| `DS4_METAL_DISABLE_Q4_GROUP8_EXPERT_TABLE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 group8 expert table. | [ds4_metal.m:42150](ds4_metal.m#L42150) | +| `DS4_METAL_DISABLE_Q4_GROUPED_BOUNDARY` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 grouped boundary. | [ds4_metal.m:42115](ds4_metal.m#L42115) | +| `DS4_METAL_DISABLE_Q4_GROUPED_EXPERTS` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 grouped experts. | [ds4_metal.m:42096](ds4_metal.m#L42096) | +| `DS4_METAL_DISABLE_Q4_MV_CLASSIC` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 MV classic. | [ds4_metal.m:21569](ds4_metal.m#L21569) | +| `DS4_METAL_DISABLE_Q4_QKV_COMPRESSOR_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 QKV compressor fuse. | [ds4_metal.m:22081](ds4_metal.m#L22081) | +| `DS4_METAL_DISABLE_Q4_SELECTED_EXPERT_VIEWS` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 selected expert views. | [ds4.c:21064](ds4.c#L21064) | +| `DS4_METAL_DISABLE_Q4_SSD_PREFILL_ATTN_OUT_EXACTN` | value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables | Disables Q4 SSD prefill attn out exactn. | [ds4_metal.m:28071](ds4_metal.m#L28071) | +| `DS4_METAL_DISABLE_Q4_SSD_SESSION_UNION` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Disables Q4 SSD session union. | [ds4.c:64970](ds4.c#L64970) | +| `DS4_METAL_DISABLE_Q4_STREAM_OVERLAP` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Disables Q4 stream overlap. | [ds4.c:64892](ds4.c#L64892) | +| `DS4_METAL_DISABLE_Q4_TABLE_BOUNDARY` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 table boundary. | [ds4_metal.m:42284](ds4_metal.m#L42284) | +| `DS4_METAL_DISABLE_Q8_DECODE_EXACT_VIEWS` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q8 decode exact views. | [ds4_metal.m:12849](ds4_metal.m#L12849) | +| `DS4_METAL_DISABLE_QKV_NORM_FUSION` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Disables QKV norm fusion. | [ds4.c:20262](ds4.c#L20262) | +| `DS4_METAL_DISABLE_QKV_PAIR_PROJ` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Disables QKV pair proj. | [ds4.c:20267](ds4.c#L20267) | +| `DS4_METAL_DISABLE_QUEUE_RESIDENCY_SET` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables queue residency set. | [ds4_metal.m:2123](ds4_metal.m#L2123) | +| `DS4_METAL_DISABLE_ROUTED_PAIR_SWIGLU_FUSION` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables routed pair SwiGLU fusion. | [ds4.c:18422](ds4.c#L18422) | +| `DS4_METAL_DISABLE_ROUTER_SELECT_FUSION` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables router select fusion. | [ds4_metal.m:35782](ds4_metal.m#L35782) | +| `DS4_METAL_DISABLE_ROUTER_WEIGHTS_BATCH_FUSION` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables router weights batch fusion. | [ds4_metal.m:36034](ds4_metal.m#L36034) | +| `DS4_METAL_DISABLE_SHARED_DOWN_HC_FUSION` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Disables shared down HC fusion. | [ds4.c:20299](ds4.c#L20299) | +| `DS4_METAL_DISABLE_SHARED_GATE_UP_SWIGLU_FUSION` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables shared gate up SwiGLU fusion. | [ds4.c:17394](ds4.c#L17394) | +| `DS4_METAL_DISABLE_SHARED_KV_PAD` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables shared KV pad. | [ds4_metal.m:31481](ds4_metal.m#L31481) | +| `DS4_METAL_DISABLE_SHARED_ROPE_COEFF` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables shared RoPE coeff. | [ds4_metal.m:6340](ds4_metal.m#L6340) | +| `DS4_METAL_DISABLE_STREAMING_COLD_DECODE_PREFILL` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming cold decode prefill. | [ds4.c:31817](ds4.c#L31817) | +| `DS4_METAL_DISABLE_STREAMING_COMPACT_ADDR` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming compact address. | [ds4_metal.m:14596](ds4_metal.m#L14596) | +| `DS4_METAL_DISABLE_STREAMING_DECODE_PREFILL` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming decode prefill. | [ds4.c:31766](ds4.c#L31766) | +| `DS4_METAL_DISABLE_STREAMING_EXPERT_ADDR_TABLE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming expert address table. | [ds4.c:18420](ds4.c#L18420) | +| `DS4_METAL_DISABLE_STREAMING_EXPERT_COMBINED_BUFFER` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming expert combined buffer. | [ds4_metal.m:14021](ds4_metal.m#L14021) | +| `DS4_METAL_DISABLE_STREAMING_EXPERT_EARLY_LOAD` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming expert early load. | [ds4_metal.m:17230](ds4_metal.m#L17230) | +| `DS4_METAL_DISABLE_STREAMING_EXPERT_EVICT_DONTNEED` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming expert evict dontneed. | [ds4_metal.m:14393](ds4_metal.m#L14393) | +| `DS4_METAL_DISABLE_STREAMING_EXPERT_HIT_VALIDATOR` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming expert hit validator. | [ds4_metal.m:14632](ds4_metal.m#L14632) | +| `DS4_METAL_DISABLE_STREAMING_EXPERT_HOTLIST` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming expert hotlist. | [ds4.c:21117](ds4.c#L21117) | +| `DS4_METAL_DISABLE_STREAMING_EXPERT_LIVE_INDEX` | value-aware boolean; default off; true disables and dominates ENABLE | Disables dense live-entry index and uses authoritative cache matrix. | [ds4_metal.m:15441](ds4_metal.m#L15441) | +| `DS4_METAL_DISABLE_STREAMING_EXPERT_MASKED_ADDR` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming expert masked address. | [ds4_metal.m:14626](ds4_metal.m#L14626) | +| `DS4_METAL_DISABLE_STREAMING_EXPERT_READAHEAD` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming expert readahead. | [ds4_metal.m:13194](ds4_metal.m#L13194) | +| `DS4_METAL_DISABLE_STREAMING_EXPERT_SLABS` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming expert slabs. | [ds4_metal.m:14026](ds4_metal.m#L14026) | +| `DS4_METAL_DISABLE_STREAMING_EXPERT_TIMING_SUMMARY` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming expert timing summary. | [ds4_metal.m:13060](ds4_metal.m#L13060) | +| `DS4_METAL_DISABLE_STREAMING_FULL_EXPERT_ADDR_TABLE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming full expert address table. | [ds4_metal.m:14714](ds4_metal.m#L14714) | +| `DS4_METAL_DISABLE_STREAMING_IQ2_CPU_ROUTER` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming IQ2 CPU router. | [ds4.c:20766](ds4.c#L20766) | +| `DS4_METAL_DISABLE_STREAMING_LAYER_BATCH` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming layer batch. | [ds4.c:18066](ds4.c#L18066) | +| `DS4_METAL_DISABLE_STREAMING_MADVISE_WILLNEED` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming madvise willneed. | [ds4.c:18039](ds4.c#L18039) | +| `DS4_METAL_DISABLE_STREAMING_PREFILL_BATCH_SELECTED_ADDR` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming prefill batch selected address. | [ds4.c:18418](ds4.c#L18418) | +| `DS4_METAL_DISABLE_STREAMING_PREFILL_LAYER_MADVISE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming prefill layer madvise. | [ds4.c:18293](ds4.c#L18293) | +| `DS4_METAL_DISABLE_STREAMING_PREFILL_LAYER_PAGEIN` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming prefill layer pagein. | [ds4.c:18261](ds4.c#L18261) | +| `DS4_METAL_DISABLE_STREAMING_PREFILL_LAYER_PAGEIN_OVERLAP` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming prefill layer pagein overlap. | [ds4.c:19148](ds4.c#L19148) | +| `DS4_METAL_DISABLE_STREAMING_PREFILL_LAYER_PREAD` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming prefill layer pread. | [ds4.c:18281](ds4.c#L18281) | +| `DS4_METAL_DISABLE_STREAMING_PREFILL_LAYER_PREPARE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming prefill layer prepare. | [ds4.c:18273](ds4.c#L18273) | +| `DS4_METAL_DISABLE_STREAMING_PREFILL_LAYER_PREPARE_OVERLAP` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming prefill layer prepare overlap. | [ds4.c:19146](ds4.c#L19146) | +| `DS4_METAL_DISABLE_STREAMING_PREFILL_LAYER_READAHEAD` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming prefill layer readahead. | [ds4.c:18271](ds4.c#L18271) | +| `DS4_METAL_DISABLE_STREAMING_PREFILL_SELECTED_ASYNC_LOAD` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming prefill selected async load. | [ds4.c:46234](ds4.c#L46234) | +| `DS4_METAL_DISABLE_STREAMING_PREFILL_SELECTED_MADVISE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming prefill selected madvise. | [ds4.c:18251](ds4.c#L18251) | +| `DS4_METAL_DISABLE_STREAMING_PREFILL_SELECTED_PAGEIN` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming prefill selected pagein. | [ds4.c:18241](ds4.c#L18241) | +| `DS4_METAL_DISABLE_STREAMING_PREFILL_SELECTED_PROFILE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming prefill selected profile. | [ds4.c:18735](ds4.c#L18735) | +| `DS4_METAL_DISABLE_STREAMING_PREFILL_SELECTED_READAHEAD` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming prefill selected readahead. | [ds4.c:19787](ds4.c#L19787) | +| `DS4_METAL_DISABLE_STREAMING_PREFILL_SELECTED_READAHEAD_SHARED` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming prefill selected readahead shared. | [ds4.c:19797](ds4.c#L19797) | +| `DS4_METAL_DISABLE_STREAMING_READAHEAD` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming readahead. | [ds4.c:18032](ds4.c#L18032) | +| `DS4_METAL_DISABLE_STREAMING_SELECTED_ASYNC_EARLY_COMMIT` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming selected async early commit. | [ds4.c:20846](ds4.c#L20846) | +| `DS4_METAL_DISABLE_STREAMING_SELECTED_ASYNC_LOAD` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming selected async load. | [ds4.c:20830](ds4.c#L20830) | +| `DS4_METAL_DISABLE_STREAMING_SELECTED_READAHEAD_SHARED_DELAY` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming selected readahead shared delay. | [ds4.c:21547](ds4.c#L21547) | +| `DS4_METAL_DISABLE_STREAMING_SELECTED_SHARED_OVERLAP` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming selected shared overlap. | [ds4.c:20822](ds4.c#L20822) | +| `DS4_METAL_DISABLE_STREAMING_STATIC_DECODE_MAP` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming static decode map. | [ds4.c:18044](ds4.c#L18044) | +| `DS4_METAL_DISABLE_STREAMING_STATIC_MAP_STATE_CACHE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming static map state cache. | [ds4.c:18057](ds4.c#L18057) | +| `DS4_METAL_DISABLE_SUPPORT_Q8_DECODE_EXACT_VIEWS` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables support Q8 decode exact views. | [ds4_metal.m:12856](ds4_metal.m#L12856) | +| `DS4_METAL_DISABLE_TINY_PAIR_SWIGLU_FUSION` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables tiny pair SwiGLU fusion. | [ds4_metal.m:44886](ds4_metal.m#L44886) | +| `DS4_METAL_DISABLE_TOKEN_EMBED_EXACT_VIEW` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables token embed exact view. | [ds4_metal.m:11905](ds4_metal.m#L11905) | +| `DS4_METAL_DISABLE_ZERO_PREFIX_PREFILL_MASK_CACHE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables zero prefix prefill mask cache. | [ds4_metal.m:2403](ds4_metal.m#L2403) | +| `DS4_METAL_DSPARK_ACCEPTANCE_ONLY_VERIFY` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Verifies only the draft rows still needed for acceptance after the base target logit. | [ds4.c:52309](ds4.c#L52309) | +| `DS4_METAL_DSPARK_DEVICE_PROPOSER` | boolean true values enable; false values/unset disable; NO_DEVICE_PROPOSER presence dominates | Keeps DSpark Q8 confidence/Markov proposal work on Metal and reads one compact result. | [ds4.c:34694](ds4.c#L34694) | +| `DS4_METAL_DSPARK_EXACT2` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Enables the resident single-GPU Metal exact-2 verifier. | [ds4.c:52100](ds4.c#L52100) | +| `DS4_METAL_DSPARK_EXACTN` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Enables the single-GPU Metal exact-N verifier. | [ds4.c:52121](ds4.c#L52121) | +| `DS4_METAL_DSPARK_EXACTN_BATCH_HEAD` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Batches the output head across exact-N verifier rows. | [ds4.c:37536](ds4.c#L37536) | +| `DS4_METAL_DSPARK_EXACTN_UNION` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Loads the union of experts for exact-N verifier rows once. | [ds4.c:52142](ds4.c#L52142) | +| `DS4_METAL_DSPARK_EXACT_ROWS_ASYNC_TAILS` | presence control; unset: off/default; any value including 0 enables | Runs exact-row routed tails asynchronously after union routing. | [ds4.c:37742](ds4.c#L37742) | +| `DS4_METAL_DSPARK_EXACT_ROWS_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for DSpark exact rows. | [ds4.c:37740](ds4.c#L37740) | +| `DS4_METAL_DSPARK_HEADLESS_REPLAY` | unset/empty: enabled; exact 0 disables; every other nonempty value enables | Skips output heads for accepted intermediate DSpark replay tokens. | [ds4.c:52289](ds4.c#L52289) | +| `DS4_METAL_DSPARK_NO_DEVICE_PROPOSER` | presence rollback; unset: automatic/default path; any value including 0 disables | Dominant presence-based rollback for the Metal DSpark device proposer. | [ds4.c:34696](ds4.c#L34696) | +| `DS4_METAL_DSPARK_PIN_MAIN_PROJ` | nonempty value other than exact 0 enables; unset/empty/0 disables | mlock-pins only DSpark stage-0 main_norm/main_proj in Metal SSD streaming. | [ds4.c:39253](ds4.c#L39253) | +| `DS4_METAL_DSPARK_PROPOSER_BLOCK_MAX` | uint32; unset: automatic cache/verifier cap; 0 or invalid: native width; positive: clamped to DSpark/native maximum | Caps rows proposed by single-device Metal DSpark. | [ds4.c:52253](ds4.c#L52253) | +| `DS4_METAL_DSPARK_SAFE_EXPERT_COUNT` | exact 1 enables; unset or any other value disables | Caps an explicit expert-count cache request to the safe Metal working-set budget for DSpark SSD streaming. | [ds4.c:4758](ds4.c#L4758) | +| `DS4_METAL_DSV4_HC_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the DeepSeek hidden-context Metal kernel source file loaded at runtime. | [ds4_metal.m:4937](ds4_metal.m#L4937) | +| `DS4_METAL_DSV4_KV_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the DeepSeek KV Metal kernel source file loaded at runtime. | [ds4_metal.m:4939](ds4_metal.m#L4939) | +| `DS4_METAL_DSV4_MISC_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the DeepSeek miscellaneous Metal kernel source file loaded at runtime. | [ds4_metal.m:4941](ds4_metal.m#L4941) | +| `DS4_METAL_DSV4_ROPE_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the DeepSeek RoPE Metal kernel source file loaded at runtime. | [ds4_metal.m:4940](ds4_metal.m#L4940) | +| `DS4_METAL_DUMP_PREFILL_LOGITS` | file path; unset/empty: no dump | Writes final GPU prefill logits as f32 binary. | [ds4.c:51157](ds4.c#L51157) | +| `DS4_METAL_ENABLE_BATCH_HC_NORM_FUSION` | legacy value-aware alias; default enabled; exact 0 disables; unset/empty/every other value enables unless DISABLE is active | Legacy control for the now-default batched HC norm fusion. | [ds4.c:20289](ds4.c#L20289) | +| `DS4_METAL_ENABLE_COMPRESSOR_EXACT_POOL_RATIO4` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables compressor exact pool ratio4. | [ds4_metal.m:26383](ds4_metal.m#L26383) | +| `DS4_METAL_ENABLE_COMPRESSOR_PAIR_STATE_STORE` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables compressor pair state store. | [ds4_metal.m:23241](ds4_metal.m#L23241) | +| `DS4_METAL_ENABLE_COMPRESSOR_QUAD_STORE` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables compressor quad store. | [ds4.c:23354](ds4.c#L23354) | +| `DS4_METAL_ENABLE_DSPARK_CAPTURE_FUSED_LAST` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Enables DSpark capture fused last. | [ds4.c:28186](ds4.c#L28186) | +| `DS4_METAL_ENABLE_GATHERED_KV_STAGE` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables gathered KV stage. | [ds4_metal.m:29651](ds4_metal.m#L29651) | +| `DS4_METAL_ENABLE_GLM_STREAMING_SELECTED_ASYNC_LOAD` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables GLM streaming selected async load. | [ds4.c:44145](ds4.c#L44145) | +| `DS4_METAL_ENABLE_HC_NORM_MIX_FUSE` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables HC norm mix fuse. | [ds4.c:22697](ds4.c#L22697) | +| `DS4_METAL_ENABLE_HC_PRODUCER_PRE_NORM_FUSE` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables HC producer pre norm fuse. | [ds4_metal.m:46518](ds4_metal.m#L46518) | +| `DS4_METAL_ENABLE_IQ2_SELECTED_ASYNC_EARLY_COMMIT` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables IQ2 selected async early commit. | [ds4.c:20845](ds4.c#L20845) | +| `DS4_METAL_ENABLE_IQ2_XXS_SSD_PREFILL_MM` | value-aware boolean; default automatic/on for eligible shape; explicit 0 turns request off unless REQUIRE=1 | Overrides automatic IQ2_XXS/Q2_K grouped address-MM selection for SSD prefill. | [ds4_metal.m:44746](ds4_metal.m#L44746) | +| `DS4_METAL_ENABLE_PRO_Q4_EXPERT_ADDRESS_AUTO` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables pro Q4 expert address auto. | [ds4.c:20809](ds4.c#L20809) | +| `DS4_METAL_ENABLE_PRO_Q4_EXPERT_TABLE_AUTO` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables pro Q4 expert table auto. | [ds4.c:20808](ds4.c#L20808) | +| `DS4_METAL_ENABLE_PRO_Q4_SELECTED_EXPERT_VIEWS` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables pro Q4 selected expert views. | [ds4.c:20805](ds4.c#L20805) | +| `DS4_METAL_ENABLE_Q4_ATTN_OUT_TINY_BATCH` | value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables | Enables Q4 attn out tiny batch. | [ds4_metal.m:28381](ds4_metal.m#L28381) | +| `DS4_METAL_ENABLE_Q4_BATCH_EXPERT_TABLE` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables Q4 batch expert table. | [ds4_metal.m:44835](ds4_metal.m#L44835) | +| `DS4_METAL_ENABLE_Q4_EXACT_TENSOR_ID` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables Q4 exact tensor ID. | [ds4_metal.m:42184](ds4_metal.m#L42184) | +| `DS4_METAL_ENABLE_Q4_EXPERT_ADDRESS_TABLE` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables Q4 expert address table. | [ds4.c:20807](ds4.c#L20807) | +| `DS4_METAL_ENABLE_Q4_EXPERT_TABLE` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables Q4 expert table. | [ds4.c:20806](ds4.c#L20806) | +| `DS4_METAL_ENABLE_Q4_GATHER_SLOTS` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables Q4 gather slots. | [ds4_metal.m:42286](ds4_metal.m#L42286) | +| `DS4_METAL_ENABLE_Q4_GROUP24_EXPERT_TABLE` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables Q4 group24 expert table. | [ds4_metal.m:42166](ds4_metal.m#L42166) | +| `DS4_METAL_ENABLE_Q4_GROUP6_EXPERT_TABLE` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables Q4 group6 expert table. | [ds4_metal.m:42132](ds4_metal.m#L42132) | +| `DS4_METAL_ENABLE_Q4_GROUP8_EXPERT_TABLE` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables Q4 group8 expert table. | [ds4_metal.m:42149](ds4_metal.m#L42149) | +| `DS4_METAL_ENABLE_Q4_GROUPED_EXPERTS` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables Q4 grouped experts. | [ds4_metal.m:42095](ds4_metal.m#L42095) | +| `DS4_METAL_ENABLE_Q4_QKV_COMPRESSOR_FUSE` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables Q4 QKV compressor fuse. | [ds4.c:22967](ds4.c#L22967) | +| `DS4_METAL_ENABLE_Q4_SELECTED_EXPERT_VIEWS` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables Q4 selected expert views. | [ds4.c:20804](ds4.c#L20804) | +| `DS4_METAL_ENABLE_Q4_SSD_PREFILL_ATTN_OUT_EXACTN` | value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables | Enables Q4 SSD prefill attn out exactn. | [ds4_metal.m:28073](ds4_metal.m#L28073) | +| `DS4_METAL_ENABLE_Q4_SSD_SESSION_UNION` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Enables Q4 SSD session union. | [ds4.c:64981](ds4.c#L64981) | +| `DS4_METAL_ENABLE_Q4_STREAM_OVERLAP` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Enables Q4 stream overlap. | [ds4.c:64890](ds4.c#L64890) | +| `DS4_METAL_ENABLE_Q8_DECODE_EXACT_VIEWS` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables Q8 decode exact views. | [ds4_metal.m:12863](ds4_metal.m#L12863) | +| `DS4_METAL_ENABLE_Q8_QKV_COMPRESSOR_FUSE` | nonempty boolean; unset/empty/exact 0: no streamed/union opt-in; other values enable; eligible resident full-decode remains automatic | Extends the automatic resident Q8 QKV/compressor compound fusion to SSD streaming or exact-N union scope. | [ds4.c:22854](ds4.c#L22854) | +| `DS4_METAL_ENABLE_STREAMING_COMPACT_ADDR` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming compact address. | [ds4_metal.m:14595](ds4_metal.m#L14595) | +| `DS4_METAL_ENABLE_STREAMING_EXPERT_ADDR_TABLE` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming expert address table. | [ds4_metal.m:14602](ds4_metal.m#L14602) | +| `DS4_METAL_ENABLE_STREAMING_EXPERT_EVICT_DONTNEED` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming expert evict dontneed. | [ds4_metal.m:14392](ds4_metal.m#L14392) | +| `DS4_METAL_ENABLE_STREAMING_EXPERT_HIT_VALIDATOR` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming expert hit validator. | [ds4_metal.m:14603](ds4_metal.m#L14603) | +| `DS4_METAL_ENABLE_STREAMING_EXPERT_LIVE_INDEX` | value-aware boolean; default automatic/on for validated IQ2 cache shape; explicit 0 disables | Overrides automatic dense live-entry index selection. | [ds4_metal.m:15440](ds4_metal.m#L15440) | +| `DS4_METAL_ENABLE_STREAMING_EXPERT_MASKED_ADDR` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming expert masked address. | [ds4_metal.m:14604](ds4_metal.m#L14604) | +| `DS4_METAL_ENABLE_STREAMING_FULL_EXPERT_ADDR_TABLE` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming full expert address table. | [ds4_metal.m:14713](ds4_metal.m#L14713) | +| `DS4_METAL_ENABLE_STREAMING_IQ2_CPU_ROUTER` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming IQ2 CPU router. | [ds4.c:20765](ds4.c#L20765) | +| `DS4_METAL_ENABLE_STREAMING_MADVISE_WILLNEED` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming madvise willneed. | [ds4.c:18037](ds4.c#L18037) | +| `DS4_METAL_ENABLE_STREAMING_PREFILL_BATCH_SELECTED_ADDR` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming prefill batch selected address. | [ds4_metal.m:14607](ds4_metal.m#L14607) | +| `DS4_METAL_ENABLE_STREAMING_PREFILL_CACHE_SEED` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming prefill cache seed. | [ds4.c:21086](ds4.c#L21086) | +| `DS4_METAL_ENABLE_STREAMING_PREFILL_EXPERT_READAHEAD` | value-aware boolean; for batches <32 readahead is automatic; for batches >=32 unset/false disables and true enables; global READHEAD rollback and F_NOCACHE still dominate | Restores F_RDADVISE immediately before parallel pread for large SSD-prefill batches. | [ds4_metal.m:13210](ds4_metal.m#L13210) | +| `DS4_METAL_ENABLE_STREAMING_PREFILL_LAYER_PAGEIN` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming prefill layer pagein. | [ds4.c:18259](ds4.c#L18259) | +| `DS4_METAL_ENABLE_STREAMING_PREFILL_LAYER_READAHEAD` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming prefill layer readahead. | [ds4.c:18269](ds4.c#L18269) | +| `DS4_METAL_ENABLE_STREAMING_PREFILL_SELECTED_MADVISE` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming prefill selected madvise. | [ds4.c:18249](ds4.c#L18249) | +| `DS4_METAL_ENABLE_STREAMING_PREFILL_SELECTED_PAGEIN` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming prefill selected pagein. | [ds4.c:18239](ds4.c#L18239) | +| `DS4_METAL_ENABLE_STREAMING_PREFILL_SELECTED_READAHEAD` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming prefill selected readahead. | [ds4.c:19783](ds4.c#L19783) | +| `DS4_METAL_ENABLE_STREAMING_PREFILL_SELECTED_READAHEAD_SHARED` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming prefill selected readahead shared. | [ds4.c:19785](ds4.c#L19785) | +| `DS4_METAL_ENABLE_STREAMING_READAHEAD` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming readahead. | [ds4.c:18030](ds4.c#L18030) | +| `DS4_METAL_ENABLE_STREAMING_SELECTED_READAHEAD_SHARED_DELAY` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming selected readahead shared delay. | [ds4.c:21546](ds4.c#L21546) | +| `DS4_METAL_ENABLE_STREAMING_STATIC_DECODE_MAP` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming static decode map. | [ds4.c:18049](ds4.c#L18049) | +| `DS4_METAL_ENABLE_TOKEN_EMBED_EXACT_VIEW` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables token embed exact view. | [ds4_metal.m:12217](ds4_metal.m#L12217) | +| `DS4_METAL_EXACT_VIEW_CACHE_GIB` | unsigned GiB; default 64; 0 disables size-triggered eviction; MIB overrides it | Sets the cached exact-model-view eviction threshold. | [ds4_metal.m:1332](ds4_metal.m#L1332) | +| `DS4_METAL_EXACT_VIEW_CACHE_MIB` | unsigned MiB; unset: inherit GIB/default; 0 disables size-triggered eviction; overrides GIB | Sets the cached exact-model-view eviction threshold with MiB precision. | [ds4_metal.m:1341](ds4_metal.m#L1341) | +| `DS4_METAL_EXACT_VIEW_CACHE_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for exact view cache. | [ds4_metal.m:1376](ds4_metal.m#L1376) | +| `DS4_METAL_FLASH_ATTN_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the FlashAttention Metal kernel source file loaded at runtime. | [ds4_metal.m:4934](ds4_metal.m#L4934) | +| `DS4_METAL_FLASH_ATTN_STAGE_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for flash attn stage. | [ds4.c:64962](ds4.c#L64962) | +| `DS4_METAL_FLASH_ATTN_STAGE_PROFILE_FILTER` | substring; unset/empty: all profiled modes/stages | Filters FlashAttention stage-profile output by mode or stage substring. | [ds4_metal.m:11177](ds4_metal.m#L11177) | +| `DS4_METAL_GET_ROWS_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the get-rows Metal kernel source file loaded at runtime. | [ds4_metal.m:4945](ds4_metal.m#L4945) | +| `DS4_METAL_GLM_DISABLE_STREAMING_EXPERT_CACHE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming expert cache for GLM. | [ds4_metal.m:39633](ds4_metal.m#L39633) | +| `DS4_METAL_GLM_DISABLE_STREAMING_GROUPED_ADDR_PREFILL` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming grouped address prefill for GLM. | [ds4_metal.m:41047](ds4_metal.m#L41047) | +| `DS4_METAL_GLM_DISABLE_STREAMING_SEED_BEFORE_PREFILL` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming seed before prefill for GLM. | [ds4.c:50884](ds4.c#L50884) | +| `DS4_METAL_GLM_DISABLE_STREAMING_TOKEN_PREFILL` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming token prefill for GLM. | [ds4.c:49514](ds4.c#L49514) | +| `DS4_METAL_GLM_MOE_ONE_STAGE_PROFILE` | unset: off; 1/true/yes/on/all enables all layers; accepts layer lists/ranges; 0/false/no/off disables | Prints timing/profile diagnostics for GLM MoE one stage. | [ds4_metal.m:39930](ds4_metal.m#L39930) | +| `DS4_METAL_GLM_MOE_ONE_STAGE_PROFILE_LAYER` | layer index/list/ranges or all; unset: all layers selected by profiler | Restricts GLM one-stage MoE profiling to selected layers. | [ds4_metal.m:39931](ds4_metal.m#L39931) | +| `DS4_METAL_GLM_MOE_STAGE_PROFILE_FILTER` | substring; unset/empty: all profiled stages | Filters GLM MoE stage-profile output. | [ds4_metal.m:39934](ds4_metal.m#L39934) | +| `DS4_METAL_GLM_QKLOW_DEBUG` | presence diagnostic; unset: off; any value including 0 enables | Enables debug diagnostics for GLM qklow. | [ds4_metal.m:37834](ds4_metal.m#L37834) | +| `DS4_METAL_GLM_STREAMING_ASYNC_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for GLM streaming async. | [ds4.c:44172](ds4.c#L44172) | +| `DS4_METAL_GLM_STREAMING_DECODE_FULL_LAYER_MAP` | presence control; unset: off/default; any value including 0 enables | Maps complete GLM layers during SSD-streaming decode instead of decode-only spans. | [ds4.c:42426](ds4.c#L42426) | +| `DS4_METAL_GLM_STREAMING_DECODE_SYNC_EACH_LAYER` | boolean text; Metal runtime always synchronizes and does not consult it; legacy fallback name read only in ROCm builds | Controls per-layer GLM streaming decode synchronization only as a legacy ROCm fallback alias. | [ds4.c:49590](ds4.c#L49590) | +| `DS4_METAL_GLM_STREAMING_PREFILL_FULL_LAYER` | presence control; unset: off/default; any value including 0 enables | Forces full-layer GLM SSD prefill regardless of the token crossover. | [ds4_metal.m:14708](ds4_metal.m#L14708) | +| `DS4_METAL_GLM_STREAMING_PREFILL_FULL_LAYER_MIN_TOKENS` | positive uint32; default 64 on Metal, 1024 when used as ROCm fallback; 0/invalid restores default | Sets the token crossover for GLM full-layer SSD prefill. | [ds4.c:42503](ds4.c#L42503) | +| `DS4_METAL_GLM_STREAMING_PREFILL_SYNC_EACH_LAYER` | boolean text; Metal runtime always synchronizes and does not consult it; legacy fallback name read only in ROCm builds | Controls per-layer GLM streaming prefill synchronization only as a legacy ROCm fallback alias. | [ds4.c:42206](ds4.c#L42206) | +| `DS4_METAL_GLM_STREAMING_TOKEN_PREFILL_MAX` | uint32; default 64 on Metal, 0 when used as ROCm fallback; 0 disables; invalid restores default | Sets largest GLM SSD prefill handled token-major by the decode graph. | [ds4.c:49493](ds4.c#L49493) | +| `DS4_METAL_GLU_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the GLU Metal kernel source file loaded at runtime. | [ds4_metal.m:4949](ds4_metal.m#L4949) | +| `DS4_METAL_GPU_BATCH_EMBED_MIN` | uint32 token threshold; default 512; invalid restores default | Sets the batch size at which prompt embedding moves from CPU upload to Metal kernels. | [ds4.c:28694](ds4.c#L28694) | +| `DS4_METAL_GPU_BUSY_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for GPU busy. | [ds4_metal.m:1292](ds4_metal.m#L1292) | +| `DS4_METAL_GRAPH_DUMP_LAYER` | unsigned layer index or all; unset: every layer | Restricts graph tensor dumps to one layer. | [ds4.c:16690](ds4.c#L16690) | +| `DS4_METAL_GRAPH_DUMP_LOGITS` | file path; unset/empty: no graph-logit dump | Writes Metal graph-test logits as f32 binary. | [ds4.c:38890](ds4.c#L38890) | +| `DS4_METAL_GRAPH_DUMP_NAME` | substring; unset/empty: every tensor name | Restricts graph tensor dumps by tensor-name substring. | [ds4.c:16686](ds4.c#L16686) | +| `DS4_METAL_GRAPH_DUMP_POS` | unsigned token position; unset: every position | Restricts graph tensor dumps to one token position. | [ds4.c:16697](ds4.c#L16697) | +| `DS4_METAL_GRAPH_DUMP_PREFIX` | path/prefix; unset/empty: tensor dumping disabled | Enables graph tensor dumps and supplies the filename prefix. | [ds4.c:64958](ds4.c#L64958) | +| `DS4_METAL_GRAPH_DUMP_TRACE` | presence diagnostic; unset: off; any value including 0 enables | Emits trace diagnostics for graph dump. | [ds4.c:16727](ds4.c#L16727) | +| `DS4_METAL_GRAPH_OUTPUT_ROW` | zero-based row smaller than current batch; default final row; invalid restores final row | Chooses which prefill output row is projected to logits. | [ds4.c:35656](ds4.c#L35656) | +| `DS4_METAL_GRAPH_PREFILL_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for graph prefill. | [ds4.c:35333](ds4.c#L35333) | +| `DS4_METAL_GRAPH_PREFILL_SPLIT_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for graph prefill split. | [ds4.c:66014](ds4.c#L66014) | +| `DS4_METAL_GRAPH_PROMPT_TOKENS` | integer 1..prompt length; default full prompt | Limits prompt length used by the Metal graph parity test. | [ds4.c:38833](ds4.c#L38833) | +| `DS4_METAL_GRAPH_RAW_CAP` | positive rows; default from SWA window+prefill; clamped to [raw_window,min(ctx,8192)] | Overrides raw sliding-window KV ring capacity. | [ds4.c:38200](ds4.c#L38200) | +| `DS4_METAL_GRAPH_TEACHER_FORCE` | presence control; unset: off/default; any value including 0 enables | Feeds CPU reference state back into the first-token graph trace at each layer. | [ds4.c:27569](ds4.c#L27569) | +| `DS4_METAL_GRAPH_TOKEN_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for graph token. | [ds4.c:32192](ds4.c#L32192) | +| `DS4_METAL_GRAPH_TOKEN_SECOND_SPLIT_LAYERS` | integer 0..layer count; default 0 plus eligible automatic pre-M5 schedules; explicit value wins | Overrides second command-buffer split layer for token decode. | [ds4.c:27856](ds4.c#L27856) | +| `DS4_METAL_GRAPH_TOKEN_SPLIT_LAYERS` | integer 0..layer count; default 4 on Apple and 0 elsewhere, with eligible pre-M5 adaptive override | Overrides first command-buffer split layer for token decode. | [ds4.c:27742](ds4.c#L27742) | +| `DS4_METAL_GRAPH_TRACE_CACHE` | presence control; unset: off/default; any value including 0 enables | Prints raw KV cache parity diagnostics in the graph prompt test. | [ds4.c:38902](ds4.c#L38902) | +| `DS4_METAL_GRAPH_TRACE_COMP` | presence control; unset: off/default; any value including 0 enables | Prints compressed-cache parity diagnostics in the graph prompt test. | [ds4.c:38903](ds4.c#L38903) | +| `DS4_METAL_GRAPH_TRACE_LAYERS` | presence control; unset: off/default; any value including 0 enables | Enables per-layer first-token CPU/GPU graph tracing. | [ds4.c:27566](ds4.c#L27566) | +| `DS4_METAL_GRAPH_TRACE_STAGE_LAYER` | signed layer index; unset gives -1/no stage-layer selection | Selects the layer used by first-token stage tracing. | [ds4.c:27570](ds4.c#L27570) | +| `DS4_METAL_HC_NORM_FUSION_CHECK` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Compares fused HC normalization against the reference result. | [ds4.c:20348](ds4.c#L20348) | +| `DS4_METAL_HC_NORM_FUSION_CHECK_TOL` | positive finite float; default 2e-4; invalid/nonpositive restores default | Sets the numerical tolerance for the HC norm-fusion oracle. | [ds4.c:20357](ds4.c#L20357) | +| `DS4_METAL_HC_STABLE` | boolean empty/1/true/yes/on vs 0/false/no/off; default on | Compiles stable hidden-context drift arithmetic into the Metal library. | [ds4_metal.m:7046](ds4_metal.m#L7046) | +| `DS4_METAL_INDEXER_STAGE_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for indexer stage. | [ds4.c:17396](ds4.c#L17396) | +| `DS4_METAL_IQ2_XXS_SSD_PREFILL_MM_STATS` | value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables | Collects/prints statistics for IQ2 XXS SSD prefill MM. | [ds4_metal.m:6979](ds4_metal.m#L6979) | +| `DS4_METAL_KV_RAW_F32` | boolean empty/1/true/yes/on vs 0/false/no/off; default off | Compiles raw KV storage as F32 for drift diagnosis. | [ds4_metal.m:7048](ds4_metal.m#L7048) | +| `DS4_METAL_LAYER_STAGE_PROFILE` | unset: off; 1/true/yes/on/all enables all layers; a layer index selects one; 0/false/no/off disables | Prints timing/profile diagnostics for layer stage. | [ds4.c:64960](ds4.c#L64960) | +| `DS4_METAL_LAYER_STAGE_PROFILE_LAYER` | single unsigned layer index; unset/empty: all layers enabled by the parent profile; invalid matches no layer | Restricts the corresponding shared graph stage profiler to one layer. | [ds4.c:28936](ds4.c#L28936) | +| `DS4_METAL_MATH_SAFE` | boolean empty/1/true/yes/on vs 0/false/no/off; default off | Compiles Metal shaders with strict/safe IEEE math instead of fast math. | [ds4_metal.m:7050](ds4_metal.m#L7050) | +| `DS4_METAL_MEMORY_REPORT` | presence control; unset: off/default; any value including 0 enables | Prints Metal allocation/cache/residency memory reports. | [ds4.c:38857](ds4.c#L38857) | +| `DS4_METAL_MODEL_UNTRACKED` | presence control; unset: off/default; any value including 0 enables | Creates mapped model buffers with untracked Metal hazard tracking. | [ds4_metal.m:1546](ds4_metal.m#L1546) | +| `DS4_METAL_MODEL_VIEW_MAX_GIB` | positive integer GiB; default device maximum (128-GiB cap for already-split span maps); cannot exceed device maximum | Caps each no-copy mapped Metal model view. | [ds4_metal.m:2215](ds4_metal.m#L2215) | +| `DS4_METAL_MODEL_WARMUP_STRIDE_KB` | integer 1..1048576 KiB, at least one page; unset inherits MB/default; overrides STRIDE_MB | Sets the model-view warmup touch stride with KiB precision. | [ds4_metal.m:3056](ds4_metal.m#L3056) | +| `DS4_METAL_MODEL_WARMUP_STRIDE_MB` | integer 1..1024 MiB; default 1 MiB; STRIDE_KB overrides | Sets the model-view warmup touch stride. | [ds4_metal.m:3048](ds4_metal.m#L3048) | +| `DS4_METAL_MOE_MM_ID_USE_RESOURCES` | presence control; unset: off/default; any value including 0 enables | Declares MM-ID MoE resource usage explicitly on the command encoder. | [ds4_metal.m:35134](ds4_metal.m#L35134) | +| `DS4_METAL_MOE_ONE_STAGE_PROFILE` | unset: off; 1/true/yes/on/all enables all layers; accepts layer lists/ranges; 0/false/no/off disables | Prints timing/profile diagnostics for MoE one stage. | [ds4.c:22541](ds4.c#L22541) | +| `DS4_METAL_MOE_ONE_STAGE_PROFILE_LAYER` | layer index/list/ranges or all; unset: all profiler-selected layers | Restricts one-stage MoE profiling to selected layers. | [ds4_metal.m:43402](ds4_metal.m#L43402) | +| `DS4_METAL_MOE_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the MoE Metal kernel source file loaded at runtime. | [ds4_metal.m:4936](ds4_metal.m#L4936) | +| `DS4_METAL_MOE_STAGE_PROFILE` | unset: off; 1/true/yes/on/all enables all layers; accepts layer lists/ranges; 0/false/no/off disables | Prints timing/profile diagnostics for MoE stage. | [ds4.c:64964](ds4.c#L64964) | +| `DS4_METAL_MOE_STAGE_PROFILE_FILTER` | substring; unset/empty: all profiled stages | Filters MoE stage-profile output. | [ds4_metal.m:43404](ds4_metal.m#L43404) | +| `DS4_METAL_MOE_STAGE_PROFILE_LAYER` | layer index/list/ranges or all; unset: all profiler-selected layers | Restricts batched MoE stage profiling to selected layers. | [ds4_metal.m:45312](ds4_metal.m#L45312) | +| `DS4_METAL_MOE_WRITE_CLAMPED_ACT` | presence control; unset: off/default; any value including 0 enables | Makes routed MoE write the clamped activation diagnostic. | [ds4.c:18421](ds4.c#L18421) | +| `DS4_METAL_NORM_RSQRT_DISABLE` | boolean empty/1/true/yes/on vs 0/false/no/off; default on | Compiles unified normalization-rsqrt arithmetic into the Metal library. | [ds4_metal.m:7047](ds4_metal.m#L7047) | +| `DS4_METAL_NORM_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the normalization Metal kernel source file loaded at runtime. | [ds4_metal.m:4950](ds4_metal.m#L4950) | +| `DS4_METAL_NO_MODEL_WARMUP` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables model warmup. | [ds4_metal.m:2308](ds4_metal.m#L2308) | +| `DS4_METAL_NO_PREFILL_KERNEL_WARMUP` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables prefill kernel warmup. | [ds4.c:28792](ds4.c#L28792) | +| `DS4_METAL_NO_RESIDENCY` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables residency. | [ds4_metal.m:2091](ds4_metal.m#L2091) | +| `DS4_METAL_OUTPUT_STAGE_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for output stage. | [ds4.c:17397](ds4.c#L17397) | +| `DS4_METAL_PREFILL_CHUNK` | positive token count used only when CLI chunk is absent; default full prompt, or 4096 for long non-PRO and 8192 for long PRO prompts; <=0 keeps automatic/full prompt | Provides the historical environment fallback for prefill chunk size. | [ds4.c:12629](ds4.c#L12629) | +| `DS4_METAL_PRO_Q4_CPU_ROUTER` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Uses the CPU router for PRO Q4 selected-expert decode. | [ds4.c:20761](ds4.c#L20761) | +| `DS4_METAL_PRO_Q4_CPU_ROUTER_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for pro Q4 CPU router. | [ds4.c:21471](ds4.c#L21471) | +| `DS4_METAL_Q4_ADDR_USE_RESOURCES` | presence control; unset: off/default; any value including 0 enables | Declares Q4 address-table resources explicitly on encoders. | [ds4_metal.m:20257](ds4_metal.m#L20257) | +| `DS4_METAL_Q4_EXPERT_GROUP_SIZE` | positive uint32; default 32; clamped to total expert count | Sets experts processed per grouped Q4 dispatch. | [ds4_metal.m:34054](ds4_metal.m#L34054) | +| `DS4_METAL_Q4_EXPERT_TABLE_GROUP_SIZE` | integer 2..total experts; default/invalid 1 (ungrouped) | Sets grouped exact-view width while building Q4 expert tables. | [ds4_metal.m:19602](ds4_metal.m#L19602) | +| `DS4_METAL_Q4_EXPERT_TABLE_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for Q4 expert table. | [ds4_metal.m:20101](ds4_metal.m#L20101) | +| `DS4_METAL_Q4_GROUP24_BASE_VIEWS` | presence control; unset: off/default; any value including 0 enables | Uses broad base model views for Q4 group-24 instead of exact views. | [ds4_metal.m:42171](ds4_metal.m#L42171) | +| `DS4_METAL_Q4_GROUP24_EXACT_VIEWS` | presence control; unset: off/default; any value including 0 enables | Uses exact mapped views for Q4 group-24 experts. | [ds4_metal.m:42170](ds4_metal.m#L42170) | +| `DS4_METAL_Q4_GROUPED_CACHE_VIEWS` | presence control; unset: off/default; any value including 0 enables | Caches exact Q4 grouped expert views. | [ds4_metal.m:42117](ds4_metal.m#L42117) | +| `DS4_METAL_Q4_PRO_MAP_GROUPS` | positive divisor of 384 in 1..384; default/invalid 1 | Splits each 384-expert PRO Q4 tensor into this many mapped views. | [ds4.c:6153](ds4.c#L6153) | +| `DS4_METAL_Q4_SELECTED_EXACT_VIEWS` | presence control; unset: off/default; any value including 0 enables | Forces exact/cached views for selected Q4 experts instead of base views. | [ds4_metal.m:42494](ds4_metal.m#L42494) | +| `DS4_METAL_Q4_SELECTED_OVERLAP_SHARED` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Overlaps selected Q4 expert preparation with the shared expert. | [ds4.c:20789](ds4.c#L20789) | +| `DS4_METAL_Q4_SELECTED_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for Q4 selected. | [ds4.c:37754](ds4.c#L37754) | +| `DS4_METAL_Q4_SELECTED_PROFILE_LAYER` | single nonnegative layer index; unset: every layer | Restricts legacy Q4 selected-expert profiling to one layer. | [ds4_metal.m:42475](ds4_metal.m#L42475) | +| `DS4_METAL_Q4_SELECTED_SHARED_EVENT` | presence control; unset: off/default; any value including 0 enables | Coordinates selected Q4 work with a shared Metal event. | [ds4_metal.m:42490](ds4_metal.m#L42490) | +| `DS4_METAL_Q4_SELECTED_TRANSIENT_VIEWS` | presence control; unset: off/default; any value including 0 enables | Uses transient exact views for selected Q4 experts. | [ds4_metal.m:42498](ds4_metal.m#L42498) | +| `DS4_METAL_Q4_SELECTED_USE_BASE_VIEWS` | presence control; unset: off/default; any value including 0 enables | Uses broad base model views for selected Q4 experts. | [ds4_metal.m:42493](ds4_metal.m#L42493) | +| `DS4_METAL_Q4_TABLE_BIND_ANCHORS` | presence control; unset: off/default; any value including 0 enables | Binds anchor buffers alongside the Q4 expert address table. | [ds4_metal.m:19714](ds4_metal.m#L19714) | +| `DS4_METAL_Q4_TABLE_MODEL_RESIDENCY_SET` | presence control; unset: off/default; any value including 0 enables | Adds Q4 expert table allocations to the model residency set. | [ds4_metal.m:19655](ds4_metal.m#L19655) | +| `DS4_METAL_Q4_TABLE_PER_TENSOR_RESIDENCY_SET` | presence control; unset: off/default; any value including 0 enables | Builds separate residency sets per Q4 expert tensor. | [ds4_metal.m:19758](ds4_metal.m#L19758) | +| `DS4_METAL_Q4_TABLE_QUEUE_RESIDENCY_SET` | presence control; unset: off/default; any value including 0 enables | Attaches Q4 expert table residency sets to command queues. | [ds4_metal.m:19613](ds4_metal.m#L19613) | +| `DS4_METAL_Q4_TABLE_RESIDENCY_SET` | presence control; unset: off/default; any value including 0 enables | Enables Q4 expert table residency-set handling. | [ds4_metal.m:19757](ds4_metal.m#L19757) | +| `DS4_METAL_Q4_TABLE_USE_RESOURCES` | presence control; unset: off/default; any value including 0 enables | Declares Q4 table resources explicitly on encoders. | [ds4_metal.m:20256](ds4_metal.m#L20256) | +| `DS4_METAL_Q8_DECODE_EXACT_VIEW_MAX_MIB` | integer 1..4096 MiB; default 1024; above max clamps, below min/invalid restores default | Caps weight ranges eligible for Q8 exact model views. | [ds4_metal.m:12874](ds4_metal.m#L12874) | +| `DS4_METAL_Q8_MV_EXT_MAX_TOKENS` | integer 2..128; default 16; above max clamps, below min/invalid restores default | Sets largest batch handled by extended Q8 matvec. | [ds4_metal.m:21124](ds4_metal.m#L21124) | +| `DS4_METAL_Q8_MV_NSG` | integer 1..8 simdgroups; default 4, or 2 with TP world=2; above max clamps, below min/invalid restores default | Overrides simdgroups per Q8 matvec threadgroup. | [ds4.c:22557](ds4.c#L22557) | +| `DS4_METAL_Q8_PREFILL_PROFILE` | value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables | Prints timing/profile diagnostics for Q8 prefill. | [ds4_metal.m:21278](ds4_metal.m#L21278) | +| `DS4_METAL_Q8_PREFILL_PROFILE_FILTER` | substring matched against generated operation label; unset/empty: all eligible calls | Filters Q8 prefill profiling. | [ds4_metal.m:21293](ds4_metal.m#L21293) | +| `DS4_METAL_Q_STAGE_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for q stage. | [ds4.c:29097](ds4.c#L29097) | +| `DS4_METAL_REPEAT_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the repeat Metal kernel source file loaded at runtime. | [ds4_metal.m:4948](ds4_metal.m#L4948) | +| `DS4_METAL_REQUIRE_COMPRESSOR_EXACT_POOL_RATIO4` | presence strict check; unset: fallback allowed; any value including 0 requires the path | Requires compressor exact pool ratio4 and makes eligible fallback fail closed. | [ds4_metal.m:26385](ds4_metal.m#L26385) | +| `DS4_METAL_REQUIRE_EXACT_ROWS_PERSISTENT_CACHE` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Requires exact rows persistent cache and makes eligible fallback fail closed. | [ds4_metal.m:13000](ds4_metal.m#L13000) | +| `DS4_METAL_REQUIRE_GATHERED_KV_STAGE` | presence strict check; unset: fallback allowed; any value including 0 requires the path | Requires gathered KV stage and makes eligible fallback fail closed. | [ds4_metal.m:29656](ds4_metal.m#L29656) | +| `DS4_METAL_REQUIRE_IQ2_XXS_SSD_PREFILL_MM` | value-aware boolean; default implicit fail-closed only with complete selected-address domain; explicit 1 is strict, 0 permits fallback | Makes eligible IQ2_XXS/Q2_K grouped SSD-prefill MM fail closed. | [ds4_metal.m:44748](ds4_metal.m#L44748) | +| `DS4_METAL_REQUIRE_M1_IQ2_MID_ONLY` | presence strict check; unset: fallback allowed; any value including 0 requires the path | Requires M1 IQ2 mid only and makes eligible fallback fail closed. | [ds4_metal.m:42040](ds4_metal.m#L42040) | +| `DS4_METAL_REQUIRE_OUTPUT_HC_WEIGHTS4` | presence strict check; unset: fallback allowed; any value including 0 requires the path | Requires output HC weights4 and makes eligible fallback fail closed. | [ds4_metal.m:46755](ds4_metal.m#L46755) | +| `DS4_METAL_REQUIRE_Q4_ATTN_OUT_TINY_BATCH` | value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables | Requires Q4 attn out tiny batch and makes eligible fallback fail closed. | [ds4_metal.m:28349](ds4_metal.m#L28349) | +| `DS4_METAL_REQUIRE_Q4_SSD_PREFILL_ATTN_OUT_EXACTN` | value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables | Requires Q4 SSD prefill attn out exactn and makes eligible fallback fail closed. | [ds4_metal.m:28065](ds4_metal.m#L28065) | +| `DS4_METAL_REQUIRE_Q4_SSD_SESSION_UNION` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Requires Q4 SSD session union and makes eligible fallback fail closed. | [ds4.c:64974](ds4.c#L64974) | +| `DS4_METAL_REQUIRE_Q8_QKV_COMPRESSOR_FUSE` | nonempty boolean; unset/empty/exact 0: fallback allowed; other values require and imply the streamed/union enable | Requires eligible Q8 QKV/compressor compound fusion and fails closed. | [ds4.c:22850](ds4.c#L22850) | +| `DS4_METAL_RESUME_PREFILL_MIN` | integer token threshold; default 4; <=0 disables resume-prefill | Sets the minimum shared-prefix suffix that uses batched resume-prefill. | [ds4.c:38229](ds4.c#L38229) | +| `DS4_METAL_ROPE_EXP2_LOG2` | boolean empty/1/true/yes/on vs 0/false/no/off; default off | Compiles the exp2/log2 RoPE drift variant. | [ds4_metal.m:7049](ds4_metal.m#L7049) | +| `DS4_METAL_SELECTED_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for selected. | [ds4.c:37753](ds4.c#L37753) | +| `DS4_METAL_SELECTED_PROFILE_LAYER` | single nonnegative layer index; unset: every layer | Restricts selected-expert profiling to one layer. | [ds4_metal.m:42473](ds4_metal.m#L42473) | +| `DS4_METAL_SESSION_BATCH_LOG` | presence diagnostic; unset: off; any value including 0 enables | Logs session batch decisions. | [ds4.c:65978](ds4.c#L65978) | +| `DS4_METAL_SESSION_BATCH_QKV` | default enabled; exact 0 disables; every other value/unset leaves enabled | Controls native batched QKV work for multi-session decode. | [ds4.c:65292](ds4.c#L65292) | +| `DS4_METAL_SESSION_BATCH_SHARED` | default enabled; exact 0 disables; every other value/unset leaves enabled | Controls native batched shared-expert work for multi-session decode. | [ds4.c:65242](ds4.c#L65242) | +| `DS4_METAL_SET_ROWS_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the set-rows Metal kernel source file loaded at runtime. | [ds4_metal.m:4952](ds4_metal.m#L4952) | +| `DS4_METAL_SOFTMAX_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the softmax Metal kernel source file loaded at runtime. | [ds4_metal.m:4947](ds4_metal.m#L4947) | +| `DS4_METAL_STREAMING_DECODE_PREFILL_MAX` | integer token maximum; default 64 for wide Flash Q4/MXFP4, 18 for other PRO/Flash, 0 otherwise; <=0 disables | Sets maximum SSD-streaming micro-prefill width that reuses decode. | [ds4.c:31772](ds4.c#L31772) | +| `DS4_METAL_STREAMING_EXPERT_AUTO_PRELOAD_CAP` | uint32 expert cap; default 4096; 0 means unlimited; invalid restores default | Caps automatic streaming-expert hotlist preload. | [ds4.c:21282](ds4.c#L21282) | +| `DS4_METAL_STREAMING_EXPERT_BUFFER_MLOCK_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming expert buffer mlock. | [ds4_metal.m:13995](ds4_metal.m#L13995) | +| `DS4_METAL_STREAMING_EXPERT_EARLY_LOAD_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming expert early load. | [ds4_metal.m:17057](ds4_metal.m#L17057) | +| `DS4_METAL_STREAMING_EXPERT_EVICT_DONTNEED_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming expert evict dontneed. | [ds4_metal.m:14434](ds4_metal.m#L14434) | +| `DS4_METAL_STREAMING_EXPERT_HOTLIST` | hotlist file path; unset/empty: built-in model hotlist | Loads the streaming-expert preload order from a file. | [ds4.c:21322](ds4.c#L21322) | +| `DS4_METAL_STREAMING_EXPERT_HOTLIST_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming expert hotlist. | [ds4.c:32055](ds4.c#L32055) | +| `DS4_METAL_STREAMING_EXPERT_LAYER_STATS` | presence diagnostic; unset: off; any value including 0 enables | Collects/prints statistics for streaming expert layer. | [ds4_metal.m:4637](ds4_metal.m#L4637) | +| `DS4_METAL_STREAMING_EXPERT_LAYER_STATS_DELTA` | presence control; unset: off/default; any value including 0 enables | Prints delta statistics for streaming expert layer. | [ds4_metal.m:4674](ds4_metal.m#L4674) | +| `DS4_METAL_STREAMING_EXPERT_NOCACHE` | nonempty value whose first character is not 0 enables; unset/empty/0 disables | Uses a reopened F_NOCACHE descriptor for SSD expert preads. | [ds4_metal.m:12600](ds4_metal.m#L12600) | +| `DS4_METAL_STREAMING_EXPERT_PREAD_POOL` | default enabled; exact 0 disables; every other value/unset keeps enabled | Controls reuse of persistent expert-pread worker threads. | [ds4_metal.m:13431](ds4_metal.m#L13431) | +| `DS4_METAL_STREAMING_EXPERT_PREAD_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming expert pread. | [ds4_metal.m:17003](ds4_metal.m#L17003) | +| `DS4_METAL_STREAMING_EXPERT_PREAD_SPLIT` | integer clamped 1..8; unset: automatic 1 below 64 cache experts, 4 at 64+ | Sets aligned requests per expert pread. | [ds4_metal.m:13699](ds4_metal.m#L13699) | +| `DS4_METAL_STREAMING_EXPERT_PREAD_THREADS` | unsigned integer clamped 1..18; default 9; invalid restores 9 | Sets expert-pread worker limit. | [ds4_metal.m:13335](ds4_metal.m#L13335) | +| `DS4_METAL_STREAMING_EXPERT_PROFILE_SUMMARY` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming expert. | [ds4_metal.m:13059](ds4_metal.m#L13059) | +| `DS4_METAL_STREAMING_EXPERT_SLAB_MB` | positive unsigned MiB; default 4096; 0/invalid restores default | Sets target allocation size for streaming-expert slabs. | [ds4_metal.m:14037](ds4_metal.m#L14037) | +| `DS4_METAL_STREAMING_EXPERT_SPLIT_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming expert split. | [ds4_metal.m:43776](ds4_metal.m#L43776) | +| `DS4_METAL_STREAMING_EXPERT_TIMING_SUMMARY` | presence control; unset: off/default; any value including 0 enables | Prints timing/profile diagnostics for streaming expert. | [ds4_metal.m:13058](ds4_metal.m#L13058) | +| `DS4_METAL_STREAMING_IQ2_CPU_ROUTER_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming IQ2 CPU router. | [ds4.c:21472](ds4.c#L21472) | +| `DS4_METAL_STREAMING_MAP_TRACE` | nonempty value other than exact 0 enables; unset/empty/0 disables | Emits SSD model-map decisions. | [ds4_metal.m:4848](ds4_metal.m#L4848) | +| `DS4_METAL_STREAMING_PREFILL_BATCH_SELECTED_ADDR_MAX` | integer token maximum; default 800 for 384 experts, 760 for 256, 0 otherwise; <=0 disables automatic selection | Sets automatic maximum batch width for selected-address SSD prefill. | [ds4_metal.m:14637](ds4_metal.m#L14637) | +| `DS4_METAL_STREAMING_PREFILL_BATCH_SELECTED_ADDR_MIN` | integer token minimum; default 2 for 256/384 experts, 0 otherwise; <=0 disables automatic selection | Sets automatic minimum batch width for selected-address SSD prefill. | [ds4_metal.m:14654](ds4_metal.m#L14654) | +| `DS4_METAL_STREAMING_PREFILL_BATCH_SELECTED_ADDR_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming prefill batch selected address. | [ds4_metal.m:18491](ds4_metal.m#L18491) | +| `DS4_METAL_STREAMING_PREFILL_CACHE_SEED_K` | uint32 seed rows; default 1; 0 disables; above 64 clamps to 64 | Sets how many prefill routing rows seed the decode expert cache. | [ds4.c:21095](ds4.c#L21095) | +| `DS4_METAL_STREAMING_PREFILL_CACHE_SEED_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming prefill cache seed. | [ds4_metal.m:17897](ds4_metal.m#L17897) | +| `DS4_METAL_STREAMING_PREFILL_LAYER_MADVISE_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming prefill layer madvise. | [ds4.c:19437](ds4.c#L19437) | +| `DS4_METAL_STREAMING_PREFILL_LAYER_PAGEIN_NO_OVERLAP` | presence rollback; unset: automatic/default path; any value including 0 disables | Prevents full-layer page-in preparation from overlapping compute. | [ds4.c:19144](ds4.c#L19144) | +| `DS4_METAL_STREAMING_PREFILL_LAYER_PAGEIN_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming prefill layer pagein. | [ds4.c:19433](ds4.c#L19433) | +| `DS4_METAL_STREAMING_PREFILL_LAYER_PAGEIN_THREADS` | integer 1..16; default 8; invalid/0 becomes 1; PREPARE_THREADS takes precedence | Sets worker count for full-layer page-in preparation. | [ds4.c:19102](ds4.c#L19102) | +| `DS4_METAL_STREAMING_PREFILL_LAYER_PREAD_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming prefill layer pread. | [ds4.c:19435](ds4.c#L19435) | +| `DS4_METAL_STREAMING_PREFILL_LAYER_PREPARE_AHEAD` | integer 1..4 layers; default 1; invalid/0 becomes 1 | Sets number of future layers prepared concurrently. | [ds4.c:19156](ds4.c#L19156) | +| `DS4_METAL_STREAMING_PREFILL_LAYER_PREPARE_NO_OVERLAP` | presence rollback; unset: automatic/default path; any value including 0 disables | Prevents generic full-layer preparation from overlapping compute. | [ds4.c:19142](ds4.c#L19142) | +| `DS4_METAL_STREAMING_PREFILL_LAYER_PREPARE_THREADS` | integer 1..16; default 8; invalid/0 becomes 1; preferred over PAGEIN_THREADS | Sets worker count for full-layer preparation. | [ds4.c:19098](ds4.c#L19098) | +| `DS4_METAL_STREAMING_PREFILL_LAYER_READAHEAD_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming prefill layer readahead. | [ds4.c:19439](ds4.c#L19439) | +| `DS4_METAL_STREAMING_PREFILL_SELECTED_MADVISE_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming prefill selected madvise. | [ds4.c:19183](ds4.c#L19183) | +| `DS4_METAL_STREAMING_PREFILL_SELECTED_MADVISE_THREADS` | integer 1..16; default inherits layer prepare threads; invalid/0 becomes 1; PREPARE_THREADS preferred | Sets worker count for selected-expert madvise preparation. | [ds4.c:19120](ds4.c#L19120) | +| `DS4_METAL_STREAMING_PREFILL_SELECTED_PAGEIN_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming prefill selected pagein. | [ds4.c:19181](ds4.c#L19181) | +| `DS4_METAL_STREAMING_PREFILL_SELECTED_PREPARE_GAP` | integer 0..8 layers; default 0; above 8 clamps; invalid restores 0 | Sets lookahead gap for selected-expert preparation. | [ds4.c:19132](ds4.c#L19132) | +| `DS4_METAL_STREAMING_PREFILL_SELECTED_PREPARE_THREADS` | integer 1..16 for madvise preparation; default inherits layer prepare threads; invalid/0 becomes 1 | Sets worker count for selected-expert preparation. | [ds4.c:19116](ds4.c#L19116) | +| `DS4_METAL_STREAMING_PREFILL_SELECTED_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming prefill selected. | [ds4.c:18733](ds4.c#L18733) | +| `DS4_METAL_STREAMING_PREFILL_SELECTED_READAHEAD_GAP` | integer 0..8 layers; default 0; above 8 clamps; invalid restores 0 | Sets lookahead gap for selected-expert readahead. | [ds4.c:19805](ds4.c#L19805) | +| `DS4_METAL_STREAMING_PREFILL_SELECTED_READAHEAD_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming prefill selected readahead. | [ds4.c:19890](ds4.c#L19890) | +| `DS4_METAL_STREAMING_SELECTED_READAHEAD_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming selected readahead. | [ds4.c:21564](ds4.c#L21564) | +| `DS4_METAL_SUM_ROWS_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the sum-rows Metal kernel source file loaded at runtime. | [ds4_metal.m:4946](ds4_metal.m#L4946) | +| `DS4_METAL_TEST_POISON_COMPRESSOR_EXACT_REDUCTION_SCRATCH` | internal test presence flag; unset: off; any value including 0 poisons scratch before the exact reduction | Validates that compressor exact-reduction kernels overwrite all scratch state. | [ds4_metal.m:25923](ds4_metal.m#L25923) | +| `DS4_METAL_TP_SESSION_BATCH` | default enabled; exact 0 disables; every other value/unset leaves enabled | Controls batched session evaluation with Metal TP. | [ds4.c:65120](ds4.c#L65120) | +| `DS4_METAL_TRACE_ALLOCS` | presence diagnostic; unset: off; any value including 0 enables | Emits trace diagnostics for allocs. | [ds4_metal.m:4056](ds4_metal.m#L4056) | +| `DS4_METAL_TRACE_M5_FLASH_ATTN_PACKED32_REDUCE` | presence diagnostic; unset: off; any value including 0 enables | Emits trace diagnostics for M5 flash attn packed32 reduce. | [ds4_metal.m:31506](ds4_metal.m#L31506) | +| `DS4_METAL_UNARY_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the unary operations Metal kernel source file loaded at runtime. | [ds4_metal.m:4938](ds4_metal.m#L4938) | +| `DS4_METAL_UNRETAINED_COMMAND_BUFFERS` | presence control; unset: off/default; any value including 0 enables | Creates Metal command buffers with unretained references. | [ds4_metal.m:1314](ds4_metal.m#L1314) | +| `DS4_METAL_USE_QUEUE_RESIDENCY_SET` | presence control; unset: off/default; any value including 0 enables | Allows queue-residency state to trigger Q4 expert address/table paths. | [ds4_metal.m:42246](ds4_metal.m#L42246) | + +
+ +
+CUDA (331) + +| Variable | Accepted value and default | Effect | Source | +| --- | --- | --- | --- | +| `DS4_CUDA_ATTENTION_OUTPUT_A_CUBLAS_MIN` | integer tokens; default 2; accepted range 2..4095, otherwise 2 | Set the token-count threshold for using cuBLAS on attention output-A. | [ds4_cuda.cu:23925](ds4_cuda.cu#L23925) | +| `DS4_CUDA_ATTENTION_OUTPUT_PRELOAD` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Allow attention-output Q8 weights to be preloaded into the selective F16 cache. | [ds4_cuda.cu:2402](ds4_cuda.cu#L2402) | +| `DS4_CUDA_ATTN_OUTPUT_PROFILE` | presence diagnostic flag; default off; any defined value including 0 enables | Measure and print CUDA attention-output stage timings. | [ds4_cuda.cu:23910](ds4_cuda.cu#L23910) | +| `DS4_CUDA_ATTN_Q_B_F32_CACHE` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Enable an F32-derived-weight cache for attention Q-B weights. | [ds4_cuda.cu:2414](ds4_cuda.cu#L2414) | +| `DS4_CUDA_BUILD_ARTIFACTS` | boolean-ish, default on for eligible derived artifacts; only exact 0 disables | Control construction of eligible CUDA derived/repacked weight artifacts. | [ds4_cuda.cu:8440](ds4_cuda.cu#L8440) | +| `DS4_CUDA_COPY_MODEL` | nonempty-string opt-in (but mere presence also suppresses prefetch); default off; value 0 is nonempty and requests a full copy | Copy the complete mapped model image into device memory. | [ds4_cuda.cu:2740](ds4_cuda.cu#L2740) | +| `DS4_CUDA_COPY_MODEL_CHUNKED` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Use range-by-range model prefetch/copy preparation instead of the normal bulk preparation. | [ds4_cuda.cu:37720](ds4_cuda.cu#L37720) | +| `DS4_CUDA_DECODE_GRAPHS` | boolean, default on; any value starting with 0, or exact off/no/false in listed case variants, disables; oracle flags force off; effective only on one GPU | Control CUDA Graph capture and replay for decode. | [ds4_cuda.cu:1468](ds4_cuda.cu#L1468) | +| `DS4_CUDA_DECODE_GRAPH_LOG` | presence diagnostic flag; default off; any defined value including 0 enables | Log CUDA decode-graph cache misses, capture failures, and lifecycle events. | [ds4_cuda.cu:1580](ds4_cuda.cu#L1580) | +| `DS4_CUDA_DECODE_HEADS8_ONLINE` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Force the eight-head online CUDA decode-attention kernel when eligible. | [ds4_cuda.cu:371](ds4_cuda.cu#L371) | +| `DS4_CUDA_DECODE_SCORE4` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Select four score lanes in the CUDA decode-attention fallback kernel. | [ds4_cuda.cu:372](ds4_cuda.cu#L372) | +| `DS4_CUDA_DECODE_SCORE8` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Select eight score lanes in the CUDA decode-attention fallback kernel. | [ds4_cuda.cu:373](ds4_cuda.cu#L373) | +| `DS4_CUDA_DIRECT_MODEL` | mixed presence/nonempty flag, default off; any defined value bypasses host caching, while backend direct lookup requires nonempty; value 0 therefore still changes behavior | Use the mapped model directly and bypass selective CUDA weight caching. | [ds4.c:3058](ds4.c#L3058); [ds4_cuda.cu:1250](ds4_cuda.cu#L1250) | +| `DS4_CUDA_DISABLE_DSPARK_EXACTN` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Disable the CUDA DSpark exactn optimization. | [ds4.c:52080](ds4.c#L52080) | +| `DS4_CUDA_DISABLE_DSPARK_EXACTN_BATCH_HEAD` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Disable the CUDA DSpark exactn batch head optimization. | [ds4.c:37095](ds4.c#L37095) | +| `DS4_CUDA_DISABLE_DSPARK_EXACTN_GRAPHS` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Disable the CUDA DSpark exactn graphs optimization. | [ds4.c:37061](ds4.c#L37061) | +| `DS4_CUDA_DISABLE_DSPARK_NONCAUSAL_ONLINE` | value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on | Disable the noncausal online-attention DSpark experiment. | [ds4_cuda.cu:21691](ds4_cuda.cu#L21691) | +| `DS4_CUDA_DISABLE_HC_NORM_MIX_FUSE` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable fused HC RMSNorm-plus-mix. | [ds4_cuda.cu:20905](ds4_cuda.cu#L20905) | +| `DS4_CUDA_DISABLE_HC_SPLIT_NORM_FUSED` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the fused HC split/weighted-sum/norm kernel. | [ds4_cuda.cu:31785](ds4_cuda.cu#L31785) | +| `DS4_CUDA_DISABLE_IQ2_XXS_SSD_PREFILL_MMQ` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Disable the CUDA IQ2 XXS SSD prefill MMQ optimization/path. | [ds4_cuda.cu:4627](ds4_cuda.cu#L4627) | +| `DS4_CUDA_DISABLE_Q4_ATTN_OUT_HC_FUSE` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable fused Q4 attention-output/HC expansion. | [ds4_cuda.cu:37337](ds4_cuda.cu#L37337) | +| `DS4_CUDA_DISABLE_Q4_DENSE_PAIR` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q4 dense pair CUDA Q4 optimization. | [ds4_cuda.cu:20402](ds4_cuda.cu#L20402) | +| `DS4_CUDA_DISABLE_Q8_HC_EXPAND_FUSED` | false-like-aware flag, default off; 0/false/no/off is off, other nonempty values request split; force-fused wins | Request the split Q8 shared-down/HC path when safe. | [ds4_cuda.cu:2086](ds4_cuda.cu#L2086) | +| `DS4_CUDA_DISABLE_QKV_RMS_FUSED` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA QKV RMS fused optimization/path. | [ds4_cuda.cu:369](ds4_cuda.cu#L369); [ds4.c:17255](ds4.c#L17255) | +| `DS4_CUDA_DISABLE_SHARED_GATE_UP_PAIR` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA shared gate up pair optimization/path. | [ds4_cuda.cu:24293](ds4_cuda.cu#L24293) | +| `DS4_CUDA_DISABLE_STREAMING_EXPERT_PERSISTENT_CACHE` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Disable streaming expert persistent cache in CUDA SSD streaming. | [ds4_cuda.cu:4113](ds4_cuda.cu#L4113) | +| `DS4_CUDA_DISABLE_STREAMING_PREFILL_BATCH_SELECTED_ADDR` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable streaming prefill batch selected addr in CUDA SSD streaming. | [ds4.c:18419](ds4.c#L18419) | +| `DS4_CUDA_DISABLE_STREAMING_PREFILL_BATCH_SELECTED_LOAD` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable streaming prefill batch selected load in CUDA SSD streaming. | [ds4.c:21744](ds4.c#L21744) | +| `DS4_CUDA_DISABLE_STREAMING_SELECTED_BATCH_IO` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Disable streaming selected batch I/O in CUDA SSD streaming. | [ds4_cuda.cu:4734](ds4_cuda.cu#L4734) | +| `DS4_CUDA_DISABLE_STREAMING_SELECTED_EVENT_PIPELINE` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Disable streaming selected event pipeline in CUDA SSD streaming. | [ds4_cuda.cu:4866](ds4_cuda.cu#L4866) | +| `DS4_CUDA_DISABLE_STREAMING_SELECTED_SHARED_OVERLAP` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable streaming selected shared overlap in CUDA SSD streaming. | [ds4.c:20796](ds4.c#L20796) | +| `DS4_CUDA_DSPARK_DEVICE_PROPOSER` | value-aware opt-in, default off; 0/off/no/false (lowercase only) disable; other nonempty enables unless rollback set | Enable the CUDA-resident DSpark proposer. | [ds4.c:34699](ds4.c#L34699); [ds4_cuda.cu:19035](ds4_cuda.cu#L19035) | +| `DS4_CUDA_DSPARK_EXACT2` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Enable the exact two-draft CUDA DSpark support path. | [ds4.c:52057](ds4.c#L52057) | +| `DS4_CUDA_DSPARK_EXACTN` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Enable the exact multi-draft CUDA DSpark support path. | [ds4.c:52078](ds4.c#L52078) | +| `DS4_CUDA_DSPARK_EXACTN_BATCH_HEAD` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Enable the batched output-head stage for exact-N DSpark verification. | [ds4.c:37093](ds4.c#L37093) | +| `DS4_CUDA_DSPARK_EXACTN_GRAPHS` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Enable CUDA Graph capture for stable exact-N DSpark islands. | [ds4.c:37059](ds4.c#L37059) | +| `DS4_CUDA_DSPARK_NO_DEVICE_PROPOSER` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Disable the CUDA-resident DSpark proposer. | [ds4.c:34709](ds4.c#L34709); [ds4_cuda.cu:19040](ds4_cuda.cu#L19040) | +| `DS4_CUDA_DSPARK_NO_PADDED_HEAD` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Disable the padded CUDA output-head optimization used by DSpark. | [ds4.c:34057](ds4.c#L34057) | +| `DS4_CUDA_DSPARK_NO_Q_NORM_ROPE_FUSION` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Disable fused Q RMSNorm plus RoPE in DSpark support stages. | [ds4.c:33433](ds4.c#L33433) | +| `DS4_CUDA_DSPARK_PROPOSER_BLOCK_MAX` | integer 0..UINT32_MAX; 0/invalid keeps native size; unset uses auto caps for exact-N/exact2; positive values cap the block and are limited by DS4_DSPARK_MAX_BLOCK_SIZE | Cap the CUDA DSpark proposal block length. | [ds4.c:52164](ds4.c#L52164) | +| `DS4_CUDA_DSPARK_TINY_ALIGNED_VEC` | value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on | Use aligned routed-MoE vector kernels for tiny DSpark batches. | [ds4_cuda.cu:29523](ds4_cuda.cu#L29523) | +| `DS4_CUDA_ENABLE_DSPARK_NONCAUSAL_ONLINE` | value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on | Enable the small-batch noncausal online-attention DSpark experiment. | [ds4_cuda.cu:21690](ds4_cuda.cu#L21690) | +| `DS4_CUDA_ENABLE_HC_NORM_MIX_FUSE` | nonempty opt-in, default off; only exact 0 disables; the F32/F16 activation mode follows the selected standalone matmul path; disable/serial/alternate flags can veto | Enable and select the fused HC RMSNorm-plus-mix one-token implementation. | [ds4_cuda.cu:20902](ds4_cuda.cu#L20902) | +| `DS4_CUDA_ENABLE_IQ2_XXS_SSD_PREFILL_MMQ` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Enable the CUDA IQ2 XXS SSD prefill MMQ experimental path. | [ds4_cuda.cu:4625](ds4_cuda.cu#L4625) | +| `DS4_CUDA_ENABLE_Q4_ATTN_OUT_HC_FUSE` | value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on | Opt in to the fused Q4 attention-output/HC expansion path. | [ds4_cuda.cu:37355](ds4_cuda.cu#L37355) | +| `DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_BATCH` | value-aware opt-in, default off; nonempty value other than exact 0 enables; rollback wins | Enable flattened grouped attention-A MMQ for two-to-eight-token GB10 batches. | [cuda/mmq/ds4_mmq.cu:4096](cuda/mmq/ds4_mmq.cu#L4096) | +| `DS4_CUDA_ENABLE_Q4_K1024_PERSISTENT` | presence flag, default off; any defined value including 0 requests the path; rollback wins | Enable the GB10 persistent-CTA kernel for M=32768, N=1, K=1024 Q4. | [cuda/mmq/ds4_mmq.cu:3698](cuda/mmq/ds4_mmq.cu#L3698) | +| `DS4_CUDA_ENABLE_Q8_FOLD` | strict flag, default off; only exact value 1 enables; overridden by DS4_CUDA_NO_Q8_FOLD | Enable one-shot producer-to-consumer reuse of freshly quantized Q8_1 data. | [ds4_cuda.cu:785](ds4_cuda.cu#L785) | +| `DS4_CUDA_ENABLE_STREAMING_EXPERT_PERSISTENT_CACHE` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Enable streaming expert persistent cache in CUDA SSD streaming. | [ds4_cuda.cu:4111](ds4_cuda.cu#L4111) | +| `DS4_CUDA_ENABLE_STREAMING_SELECTED_BATCH_IO` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Enable streaming selected batch I/O in CUDA SSD streaming. | [ds4_cuda.cu:4732](ds4_cuda.cu#L4732) | +| `DS4_CUDA_ENABLE_STREAMING_SELECTED_EVENT_PIPELINE` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Enable streaming selected event pipeline in CUDA SSD streaming. | [ds4_cuda.cu:4864](ds4_cuda.cu#L4864) | +| `DS4_CUDA_END_STREAM_SYNC` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Synchronize only CUDA stream 0 at command-batch end instead of synchronizing the whole device. | [ds4_cuda.cu:376](ds4_cuda.cu#L376) | +| `DS4_CUDA_EXACT_SCORE_SPLIT_CHUNK` | integer scores/chunk; default 512; clamped 1..8192 | Tune exact score split chunk for exact score-split CUDA decode attention. | [ds4_cuda.cu:13669](ds4_cuda.cu#L13669) | +| `DS4_CUDA_EXACT_SCORE_SPLIT_DECODE` | value-aware boolean, default on; exact 0 disables; a nonzero explicit setting also takes precedence over split-KV selection | Control the exact score-split decode-attention implementation. | [ds4_cuda.cu:13631](ds4_cuda.cu#L13631) | +| `DS4_CUDA_EXACT_SCORE_SPLIT_DIM2` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Tune exact score split dim2 for exact score-split CUDA decode attention. | [ds4_cuda.cu:387](ds4_cuda.cu#L387) | +| `DS4_CUDA_EXACT_SCORE_SPLIT_FUSE_INV_ROPE` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Tune exact score split fuse inv rope for exact score-split CUDA decode attention. | [ds4_cuda.cu:390](ds4_cuda.cu#L390) | +| `DS4_CUDA_EXACT_SCORE_SPLIT_GRAPH` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Tune exact score split graph for exact score-split CUDA decode attention. | [ds4_cuda.cu:379](ds4_cuda.cu#L379) | +| `DS4_CUDA_EXACT_SCORE_SPLIT_LDG` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Tune exact score split ldg for exact score-split CUDA decode attention. | [ds4_cuda.cu:381](ds4_cuda.cu#L381) | +| `DS4_CUDA_EXACT_SCORE_SPLIT_MIN_SCORE` | integer score count; default 1; clamped 0..8192 | Set the minimum visible-score count for exact score-split decode. | [ds4_cuda.cu:13665](ds4_cuda.cu#L13665) | +| `DS4_CUDA_EXACT_SCORE_SPLIT_S` | integer exact split count; unset/invalid = automatic; valid value clamped 1..16 | Tune exact score split s for exact score-split CUDA decode attention. | [ds4_cuda.cu:13679](ds4_cuda.cu#L13679) | +| `DS4_CUDA_EXACT_SCORE_SPLIT_S_FLOOR` | integer split count; default 6; clamped 1..16 | Tune exact score split s floor for exact score-split CUDA decode attention. | [ds4_cuda.cu:13672](ds4_cuda.cu#L13672) | +| `DS4_CUDA_EXACT_SCORE_SPLIT_S_MAX` | integer split count; default 16; clamped 1..16 | Tune exact score split s max for exact score-split CUDA decode attention. | [ds4_cuda.cu:13675](ds4_cuda.cu#L13675) | +| `DS4_CUDA_EXACT_SCORE_SPLIT_VEC4` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Tune exact score split vec4 for exact score-split CUDA decode attention. | [ds4_cuda.cu:383](ds4_cuda.cu#L383) | +| `DS4_CUDA_EXACT_SCORE_SPLIT_VEC4_PLAIN` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Tune exact score split vec4 plain for exact score-split CUDA decode attention. | [ds4_cuda.cu:385](ds4_cuda.cu#L385) | +| `DS4_CUDA_F16_CUBLAS_ONE` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Control the F16 cuBLAS one CUDA F16 matmul path. | [ds4_cuda.cu:20852](ds4_cuda.cu#L20852) | +| `DS4_CUDA_F16_SMALL_BATCH` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Control the F16 small batch CUDA F16 matmul path. | [ds4_cuda.cu:20837](ds4_cuda.cu#L20837) | +| `DS4_CUDA_F16_SMALL_OUT` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Control the F16 small out CUDA F16 matmul path. | [ds4_cuda.cu:20819](ds4_cuda.cu#L20819) | +| `DS4_CUDA_GLM_VERIFY_NO_Q8_TOK2` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Control or tune the CUDA glm verify no Q8 tok2 path. | [ds4_cuda.cu:19824](ds4_cuda.cu#L19824) | +| `DS4_CUDA_GREEDY_SPLITKV` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Enable greedy split-KV fast attention. | [ds4.c:17062](ds4.c#L17062) | +| `DS4_CUDA_GREEDY_SPLITKV_FALLBACK_LOG` | presence diagnostic flag; default off; any defined value including 0 enables | Control greedy splitkv fallback log in CUDA greedy fast decode. | [ds4.c:55473](ds4.c#L55473) | +| `DS4_CUDA_GREEDY_SPLITKV_MARGIN` | nonnegative finite float; default 0.25; invalid value warns and uses 0.25; 0 disables margin fallback | Control greedy splitkv margin in CUDA greedy fast decode. | [ds4.c:17140](ds4.c#L17140) | +| `DS4_CUDA_GREEDY_SPLITKV_MAX_SEGMENT` | integer 0..INT32_MAX; default/invalid 0 (segment cap disabled) | Control greedy splitkv max segment in CUDA greedy fast decode. | [ds4.c:17216](ds4.c#L17216) | +| `DS4_CUDA_GREEDY_SPLITKV_PAIR_REPLAY` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Replay greedy split-KV tokens in pairs. | [ds4.c:17190](ds4.c#L17190) | +| `DS4_CUDA_GREEDY_SPLITKV_TOP2` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Use top-2 output margins with greedy split-KV. | [ds4.c:17172](ds4.c#L17172) | +| `DS4_CUDA_GREEDY_SPLITKV_TRACE` | presence diagnostic flag; default off; any defined value including 0 enables | Control greedy splitkv trace in CUDA greedy fast decode. | [ds4.c:55508](ds4.c#L55508) | +| `DS4_CUDA_GREEDY_SPLITKV_TRUST_REPLAY` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Trust replayed greedy split-KV results without the normal confirmation policy. | [ds4.c:17180](ds4.c#L17180) | +| `DS4_CUDA_GREEDY_SPLIT_TOP1` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Enable split top-1 selection in greedy CUDA decode. | [ds4.c:17034](ds4.c#L17034) | +| `DS4_CUDA_GREEDY_TOP1` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control greedy top1 in CUDA greedy fast decode. | [ds4.c:55936](ds4.c#L55936) | +| `DS4_CUDA_GREEDY_VEC4` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Enable greedy vec4 fast attention. | [ds4.c:17072](ds4.c#L17072) | +| `DS4_CUDA_GREEDY_VEC4_FALLBACK_LOG` | presence diagnostic flag; default off; any defined value including 0 enables | Control greedy vec4 fallback log in CUDA greedy fast decode. | [ds4.c:55474](ds4.c#L55474) | +| `DS4_CUDA_GREEDY_VEC4_MARGIN` | nonnegative finite float; default 0.25; invalid value warns and uses 0.25; 0 disables margin fallback | Control greedy vec4 margin in CUDA greedy fast decode. | [ds4.c:17110](ds4.c#L17110) | +| `DS4_CUDA_GREEDY_VEC4_MAX_SEGMENT` | integer 0..INT32_MAX; default/invalid 0 (segment cap disabled) | Control greedy vec4 max segment in CUDA greedy fast decode. | [ds4.c:17224](ds4.c#L17224) | +| `DS4_CUDA_GREEDY_VEC4_TRACE` | presence diagnostic flag; default off; any defined value including 0 enables | Control greedy vec4 trace in CUDA greedy fast decode. | [ds4.c:55537](ds4.c#L55537) | +| `DS4_CUDA_INDEXED_TWOPASS` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Force the two-pass indexed-attention path instead of the fused heads8 online kernel. | [ds4_cuda.cu:23587](ds4_cuda.cu#L23587) | +| `DS4_CUDA_IQ2_XXS_SSD_PREFILL_MMQ_STATS` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Print CUDA IQ2 XXS SSD prefill MMQ counters. | [ds4_cuda.cu:4633](ds4_cuda.cu#L4633) | +| `DS4_CUDA_KEEP_MODEL_PAGES` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Keep source model pages resident instead of advising the OS to discard copied pages. | [ds4_cuda.cu:2840](ds4_cuda.cu#L2840) | +| `DS4_CUDA_MIXED_PREFILL_DECODE` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control native mixed prefill/decode scheduling. | [ds4.c:70511](ds4.c#L70511) | +| `DS4_CUDA_MIXED_ROUTED_MAX_PREFILL` | integer rows 0..UINT32_MAX; default/invalid 512 | Set the maximum prefill rows admitted to the mixed routed-MoE path. | [ds4.c:70000](ds4.c#L70000) | +| `DS4_CUDA_MIXED_ROUTED_SCATTER` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Use scattered row handling in mixed routed-MoE execution. | [ds4.c:69461](ds4.c#L69461) | +| `DS4_CUDA_MMQ` | boolean-ish, default on; any value beginning with 0 disables; quality mode and multi-GPU disable normal MMQ tier (MXFP4 path differs) | Control the vendored CUDA MMQ prefill tier. | [ds4_cuda.cu:1722](ds4_cuda.cu#L1722) | +| `DS4_CUDA_MMQ_Q81_PERSISTENT` | strict boolean, default off; accepts 1/on/true/yes and 0/off/false/no in listed lower/upper-case forms; unknown values are off | Reuse a persistent Q8_1 MMQ scratch arena on supported GB10 devices. | [cuda/mmq/ds4_mmq.cu:144](cuda/mmq/ds4_mmq.cu#L144) | +| `DS4_CUDA_MMQ_X_MAX` | integer >=8; rounded down to multiple of 8 and only lowers the hardware base; invalid/unset = hardware base | Cap the MMQ X tile-width selector for architecture tuning. | [cuda/mmq/mmq.cuh:127](cuda/mmq/mmq.cuh#L127) | +| `DS4_CUDA_MODEL_COPY_CHUNK_MB` | positive integer MiB; default 64; clamped 16..4096 | Set the chunk size used for CUDA model copying. | [ds4_cuda.cu:2827](ds4_cuda.cu#L2827) | +| `DS4_CUDA_MODEL_COPY_VERBOSE` | presence diagnostic flag; default off; any defined value including 0 enables | Print periodic progress while copying the model to device memory. | [ds4_cuda.cu:6537](ds4_cuda.cu#L6537) | +| `DS4_CUDA_MODEL_PREFETCH_SYNC` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Synchronize after each CUDA model prefetch range for diagnostics. | [ds4_cuda.cu:2808](ds4_cuda.cu#L2808) | +| `DS4_CUDA_MOE_ATOMIC_DOWN` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Select or tune the atomic down variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30086](ds4_cuda.cu#L30086) | +| `DS4_CUDA_MOE_DECODE_GRAPH` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Select or tune the decode graph variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:391](ds4_cuda.cu#L391) | +| `DS4_CUDA_MOE_DIRECT_MIDQ` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Select or tune the direct midq variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30140](ds4_cuda.cu#L30140) | +| `DS4_CUDA_MOE_DOWN_ROW1024` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Select or tune the down row1024 variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30109](ds4_cuda.cu#L30109) | +| `DS4_CUDA_MOE_DOWN_ROW128` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Select or tune the down row128 variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30128](ds4_cuda.cu#L30128) | +| `DS4_CUDA_MOE_DOWN_ROW2048` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Select or tune the down row2048 variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30110](ds4_cuda.cu#L30110) | +| `DS4_CUDA_MOE_DOWN_ROW256` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Select or tune the down row256 variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30127](ds4_cuda.cu#L30127) | +| `DS4_CUDA_MOE_DOWN_ROW512` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Select or tune the down row512 variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30108](ds4_cuda.cu#L30108) | +| `DS4_CUDA_MOE_DOWN_ROW64` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Select or tune the down row64 variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30129](ds4_cuda.cu#L30129) | +| `DS4_CUDA_MOE_GATE_ROW1024` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Select or tune the gate row1024 variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30120](ds4_cuda.cu#L30120) | +| `DS4_CUDA_MOE_GATE_ROW128` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Select or tune the gate row128 variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30093](ds4_cuda.cu#L30093) | +| `DS4_CUDA_MOE_GATE_ROW2048` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Select or tune the gate row2048 variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30091](ds4_cuda.cu#L30091) | +| `DS4_CUDA_MOE_GATE_ROW256` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Select or tune the gate row256 variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30092](ds4_cuda.cu#L30092) | +| `DS4_CUDA_MOE_MIDQ_SIDECAR` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Select or tune the midq sidecar variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30169](ds4_cuda.cu#L30169) | +| `DS4_CUDA_MOE_NO_ATOMIC_DOWN` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Disable the atomic down variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30087](ds4_cuda.cu#L30087) | +| `DS4_CUDA_MOE_NO_DECODE_LUT_GATE` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Disable the decode lut gate variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30117](ds4_cuda.cu#L30117) | +| `DS4_CUDA_MOE_NO_DIRECT_DOWN_SUM6` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Disable the direct down sum6 variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30137](ds4_cuda.cu#L30137) | +| `DS4_CUDA_MOE_NO_DIRECT_MIDQ` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Disable the direct midq variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30141](ds4_cuda.cu#L30141) | +| `DS4_CUDA_MOE_NO_DOWN_ROW128` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Disable the down row128 variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30133](ds4_cuda.cu#L30133) | +| `DS4_CUDA_MOE_NO_DOWN_ROW2048` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Disable the down row2048 variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30131](ds4_cuda.cu#L30131) | +| `DS4_CUDA_MOE_NO_DOWN_ROW256` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Disable the down row256 variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30132](ds4_cuda.cu#L30132) | +| `DS4_CUDA_MOE_NO_DOWN_ROW64` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Disable the down row64 variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30134](ds4_cuda.cu#L30134) | +| `DS4_CUDA_MOE_NO_DOWN_TILE16` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Disable the down tile16 variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30102](ds4_cuda.cu#L30102) | +| `DS4_CUDA_MOE_NO_EXPERT_TILES` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Disable the expert tiles variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30062](ds4_cuda.cu#L30062) | +| `DS4_CUDA_MOE_NO_GATE_ROW128` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Disable the gate row128 variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30097](ds4_cuda.cu#L30097) | +| `DS4_CUDA_MOE_NO_GATE_ROW2048` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Disable the gate row2048 variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30095](ds4_cuda.cu#L30095) | +| `DS4_CUDA_MOE_NO_GATE_ROW256` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Disable the gate row256 variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30096](ds4_cuda.cu#L30096) | +| `DS4_CUDA_MOE_NO_IQ2_ALIGNED` | value-aware kill switch, default off; nonempty value other than exact 0 disables aligned IQ2 path | Disable the IQ2 aligned variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:1011](ds4_cuda.cu#L1011) | +| `DS4_CUDA_MOE_NO_MIDQ_SIDECAR` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Disable the midq sidecar variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30170](ds4_cuda.cu#L30170) | +| `DS4_CUDA_MOE_NO_OWNED_SPARSE_BUFFERS` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Disable the owned sparse buffers variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30089](ds4_cuda.cu#L30089) | +| `DS4_CUDA_MOE_NO_P2` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Disable the p2 variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30084](ds4_cuda.cu#L30084) | +| `DS4_CUDA_MOE_NO_Q2K_ALIGNED` | value-aware kill switch, default off; nonempty value other than exact 0 disables aligned Q2_K path | Disable the q2k aligned variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:1016](ds4_cuda.cu#L1016) | +| `DS4_CUDA_MOE_NO_Q4_DOWN_ROWSPAN` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Disable the Q4 down rowspan variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30114](ds4_cuda.cu#L30114) | +| `DS4_CUDA_MOE_NO_Q4_DOWN_SLOT3` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Disable the Q4 down slot3 variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30164](ds4_cuda.cu#L30164) | +| `DS4_CUDA_MOE_NO_Q4_GATE_H16` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Disable the Q4 gate H16 variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30149](ds4_cuda.cu#L30149) | +| `DS4_CUDA_MOE_NO_Q4_GATE_H16R8` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Disable the Q4 gate H16R8 variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30145](ds4_cuda.cu#L30145) | +| `DS4_CUDA_MOE_NO_Q4_GATE_W32` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Disable the Q4 gate W32 variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30157](ds4_cuda.cu#L30157) | +| `DS4_CUDA_MOE_NO_Q4_GATE_W32R16` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Disable the Q4 gate W32R16 variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30153](ds4_cuda.cu#L30153) | +| `DS4_CUDA_MOE_NO_Q4_GATE_W32_NOAUX` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Disable the Q4 gate W32 no-aux variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30160](ds4_cuda.cu#L30160) | +| `DS4_CUDA_MOE_NO_Q4_MMA` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Disable the Q4 MMA variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:312](ds4_cuda.cu#L312) | +| `DS4_CUDA_MOE_NO_Q4_MMA_TILE16` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Disable the Q4 MMA tile16 variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30100](ds4_cuda.cu#L30100) | +| `DS4_CUDA_MOE_NO_Q4_SORTED` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Disable the Q4 sorted variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30061](ds4_cuda.cu#L30061) | +| `DS4_CUDA_MOE_NO_SMALL_SORTED_PREP` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Disable the small sorted prep variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30106](ds4_cuda.cu#L30106) | +| `DS4_CUDA_MOE_PROFILE` | presence diagnostic flag; default off; any defined value including 0 enables | Measure and print routed-MoE CUDA kernel-stage timings. | [ds4_cuda.cu:30045](ds4_cuda.cu#L30045) | +| `DS4_CUDA_MOE_Q4_DOWN_SLOT3` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Select or tune the Q4 down slot3 variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30163](ds4_cuda.cu#L30163) | +| `DS4_CUDA_MOE_Q4_GATE_H16` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Select or tune the Q4 gate H16 variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30148](ds4_cuda.cu#L30148) | +| `DS4_CUDA_MOE_Q4_GATE_H16R8` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Select or tune the Q4 gate H16R8 variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30144](ds4_cuda.cu#L30144) | +| `DS4_CUDA_MOE_Q4_GATE_W32R16` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Select or tune the Q4 gate W32R16 variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30152](ds4_cuda.cu#L30152) | +| `DS4_CUDA_MOE_TILE4` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Select or tune the tile4 variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30063](ds4_cuda.cu#L30063) | +| `DS4_CUDA_MOE_TILE8` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Select or tune the tile8 variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30079](ds4_cuda.cu#L30079) | +| `DS4_CUDA_MOE_WRITE_GATE_UP` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Select or tune the write gate up variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30081](ds4_cuda.cu#L30081) | +| `DS4_CUDA_NO_ATTENTION_OUTPUT_F16_CACHE` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the attention output F16 cache CUDA F16 path. | [ds4_cuda.cu:2364](ds4_cuda.cu#L2364) | +| `DS4_CUDA_NO_ATTN_A_TOK2` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA attn a tok2 optimization/path. | [ds4_cuda.cu:24024](ds4_cuda.cu#L24024) | +| `DS4_CUDA_NO_ATTN_Q_B_F16_CACHE` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the attn q b F16 cache CUDA F16 path. | [ds4_cuda.cu:2367](ds4_cuda.cu#L2367) | +| `DS4_CUDA_NO_COMPRESSOR_PREFILL_BATCH` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA compressor prefill batch optimization/path. | [ds4.c:29717](ds4.c#L29717) | +| `DS4_CUDA_NO_CUBLAS_ATTENTION` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA cuBLAS attention optimization/path. | [ds4_cuda.cu:23067](ds4_cuda.cu#L23067) | +| `DS4_CUDA_NO_CUBLAS_ATTENTION_OUTPUT_A` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA cuBLAS attention output a optimization/path. | [ds4_cuda.cu:23934](ds4_cuda.cu#L23934) | +| `DS4_CUDA_NO_DECODE_VALUE512` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the 512-thread CUDA decode value/finalize specialization. | [ds4_cuda.cu:374](ds4_cuda.cu#L374) | +| `DS4_CUDA_NO_DERIVED_WEIGHTS` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA derived weights optimization/path. | [ds4_cuda.cu:1034](ds4_cuda.cu#L1034) | +| `DS4_CUDA_NO_DIRECT_IO` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA direct I/O optimization/path. | [cuda/mmq/ds4_repack.cu:68](cuda/mmq/ds4_repack.cu#L68) | +| `DS4_CUDA_NO_DIRECT_Q2_PREFILL` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA direct q2 prefill optimization/path. | [ds4_cuda.cu:395](ds4_cuda.cu#L395) | +| `DS4_CUDA_NO_EXACT_SCORE_SPLIT_DECODE` | value-aware kill switch, default off; exact 0 is off, other nonempty values disable | Disable exact score split decode for exact score-split CUDA decode attention. | [ds4_cuda.cu:13629](ds4_cuda.cu#L13629); [ds4.c:67920](ds4.c#L67920) | +| `DS4_CUDA_NO_EXACT_SCORE_SPLIT_DIM2` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable exact score split dim2 for exact score-split CUDA decode attention. | [ds4_cuda.cu:388](ds4_cuda.cu#L388) | +| `DS4_CUDA_NO_F16_CUBLAS_BATCH` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the F16 cuBLAS batch CUDA F16 path. | [ds4_cuda.cu:20854](ds4_cuda.cu#L20854) | +| `DS4_CUDA_NO_F16_CUBLAS_ONE` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the F16 cuBLAS one CUDA F16 path. | [ds4_cuda.cu:20851](ds4_cuda.cu#L20851) | +| `DS4_CUDA_NO_F16_PAIR_COMPRESSOR_STORE` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the F16 pair compressor store CUDA F16 path. | [ds4_cuda.cu:399](ds4_cuda.cu#L399) | +| `DS4_CUDA_NO_F16_PAIR_COMPRESSOR_TRANSPOSE` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the F16 pair compressor transpose CUDA F16 path. | [ds4_cuda.cu:21343](ds4_cuda.cu#L21343) | +| `DS4_CUDA_NO_F16_PAIR_COMPRESSOR_TRANSPOSE_PREFETCH8` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the F16 pair compressor transpose prefetch8 CUDA F16 path. | [ds4_cuda.cu:21349](ds4_cuda.cu#L21349) | +| `DS4_CUDA_NO_F16_PAIR_MATMUL` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the F16 pair matmul CUDA F16 path. | [ds4_cuda.cu:21130](ds4_cuda.cu#L21130) | +| `DS4_CUDA_NO_F16_SMALL_BATCH` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the F16 small batch CUDA F16 path. | [ds4_cuda.cu:20838](ds4_cuda.cu#L20838) | +| `DS4_CUDA_NO_F16_SMALL_OUT` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the F16 small out CUDA F16 path. | [ds4_cuda.cu:20821](ds4_cuda.cu#L20821) | +| `DS4_CUDA_NO_FD_CACHE` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA fd cache optimization/path. | [ds4_cuda.cu:1274](ds4_cuda.cu#L1274) | +| `DS4_CUDA_NO_GREEDY_SPLITKV` | value-aware kill switch; default off; nonempty value other than exact 0 disables | Disable greedy splitkv in CUDA greedy fast decode. | [ds4.c:17060](ds4.c#L17060) | +| `DS4_CUDA_NO_GREEDY_SPLITKV_FALLBACK` | value-aware kill switch; default off; nonempty value other than exact 0 disables margin fallback | Disable greedy splitkv fallback in CUDA greedy fast decode. | [ds4.c:17160](ds4.c#L17160) | +| `DS4_CUDA_NO_GREEDY_SPLITKV_PAIR_REPLAY` | value-aware kill switch; default off; nonempty value other than exact 0 disables | Disable greedy splitkv pair replay in CUDA greedy fast decode. | [ds4.c:17188](ds4.c#L17188) | +| `DS4_CUDA_NO_GREEDY_SPLITKV_TOP2` | value-aware kill switch; default off; nonempty value other than exact 0 disables | Disable greedy splitkv top2 in CUDA greedy fast decode. | [ds4.c:17170](ds4.c#L17170) | +| `DS4_CUDA_NO_GREEDY_SPLIT_TOP1` | value-aware kill switch; default off; nonempty value other than exact 0 disables | Disable greedy split top1 in CUDA greedy fast decode. | [ds4.c:17032](ds4.c#L17032) | +| `DS4_CUDA_NO_GREEDY_VEC4` | value-aware kill switch; default off; nonempty value other than exact 0 disables | Disable greedy vec4 in CUDA greedy fast decode. | [ds4.c:17070](ds4.c#L17070) | +| `DS4_CUDA_NO_GREEDY_VEC4_FALLBACK` | value-aware kill switch; default off; nonempty value other than exact 0 disables margin fallback | Disable greedy vec4 fallback in CUDA greedy fast decode. | [ds4.c:17130](ds4.c#L17130) | +| `DS4_CUDA_NO_HC_SPLIT_NORM_SPLIT4096` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the split partial-reduction specialization for one-row, 4096-wide HC normalization. | [ds4_cuda.cu:31826](ds4_cuda.cu#L31826) | +| `DS4_CUDA_NO_INDEXED_HEADS8` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA indexed heads8 optimization/path. | [ds4_cuda.cu:23586](ds4_cuda.cu#L23586) | +| `DS4_CUDA_NO_INDEXED_TOPK_SORT` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA indexed topk sort optimization/path. | [ds4_cuda.cu:23577](ds4_cuda.cu#L23577) | +| `DS4_CUDA_NO_INDEXER_DIRECT_ONE` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the indexer direct one CUDA indexer kernel/path. | [ds4_cuda.cu:18881](ds4_cuda.cu#L18881) | +| `DS4_CUDA_NO_INDEXER_MXF4` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the indexer MXF4 CUDA indexer kernel/path. | [ds4_cuda.cu:17774](ds4_cuda.cu#L17774) | +| `DS4_CUDA_NO_INDEXER_WMMA` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the indexer WMMA CUDA indexer kernel/path. | [ds4_cuda.cu:18891](ds4_cuda.cu#L18891) | +| `DS4_CUDA_NO_INDEXER_WMMA128` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the indexer wmma128 CUDA indexer kernel/path. | [ds4_cuda.cu:18892](ds4_cuda.cu#L18892) | +| `DS4_CUDA_NO_INDEXER_WMMA32` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the indexer wmma32 CUDA indexer kernel/path. | [ds4_cuda.cu:18910](ds4_cuda.cu#L18910) | +| `DS4_CUDA_NO_INDEXER_WMMA64` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the indexer wmma64 CUDA indexer kernel/path. | [ds4_cuda.cu:18901](ds4_cuda.cu#L18901) | +| `DS4_CUDA_NO_IQ2_XXS_SSD_PREFILL_MMQ` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Disable the CUDA IQ2 XXS SSD prefill MMQ optimization/path. | [ds4_cuda.cu:4629](ds4_cuda.cu#L4629) | +| `DS4_CUDA_NO_MODEL_COPY` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA model copy optimization/path. | [ds4_cuda.cu:6472](ds4_cuda.cu#L6472) | +| `DS4_CUDA_NO_MODEL_PREFETCH` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA model prefetch optimization/path. | [ds4_cuda.cu:2739](ds4_cuda.cu#L2739) | +| `DS4_CUDA_NO_MOE_DEDUP` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA MoE dedup optimization/path. | [cuda/mmq/ds4_mmq.cu:5963](cuda/mmq/ds4_mmq.cu#L5963) | +| `DS4_CUDA_NO_ORDERED_F16_MATMUL` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the ordered F16 matmul CUDA F16 path. | [ds4_cuda.cu:20811](ds4_cuda.cu#L20811) | +| `DS4_CUDA_NO_PARALLEL_ROUTER_SELECT` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA parallel router select optimization/path. | [ds4_cuda.cu:24491](ds4_cuda.cu#L24491) | +| `DS4_CUDA_NO_Q4_DENSE_SCRATCH` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q4 dense scratch CUDA Q4 optimization. | [cuda/mmq/ds4_mmq.cu:3779](cuda/mmq/ds4_mmq.cu#L3779) | +| `DS4_CUDA_NO_Q4_GB10_FAST` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the GB10-specific Q4 fast-path family. | [cuda/mmq/ds4_mmq.cu:3701](cuda/mmq/ds4_mmq.cu#L3701) | +| `DS4_CUDA_NO_Q4_GROUPED_ATTN_A` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q4 grouped attn a CUDA Q4 optimization. | [cuda/mmq/ds4_mmq.cu:4088](cuda/mmq/ds4_mmq.cu#L4088) | +| `DS4_CUDA_NO_Q4_GROUPED_ATTN_A_BATCH` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q4 grouped attn a batch CUDA Q4 optimization. | [cuda/mmq/ds4_mmq.cu:4098](cuda/mmq/ds4_mmq.cu#L4098) | +| `DS4_CUDA_NO_Q4_K1024_PERSISTENT` | presence kill switch, default off; any defined value including 0 disables | Disable the Q4 K1024 persistent CUDA Q4 optimization. | [cuda/mmq/ds4_mmq.cu:3700](cuda/mmq/ds4_mmq.cu#L3700) | +| `DS4_CUDA_NO_Q8_ALIGNED_DENSE_SCRATCH` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q8 aligned dense scratch CUDA Q8 optimization. | [cuda/mmq/ds4_mmq.cu:5369](cuda/mmq/ds4_mmq.cu#L5369) | +| `DS4_CUDA_NO_Q8_ALIGNED_PERSISTENT` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q8 aligned persistent CUDA Q8 optimization. | [cuda/mmq/ds4_mmq.cu:5201](cuda/mmq/ds4_mmq.cu#L5201) | +| `DS4_CUDA_NO_Q8_BATCH_EXACT_TOK2` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q8 batch exact tok2 CUDA Q8 optimization. | [ds4_cuda.cu:19852](ds4_cuda.cu#L19852) | +| `DS4_CUDA_NO_Q8_BATCH_TOK4` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q8 batch tok4 CUDA Q8 optimization. | [ds4_cuda.cu:19805](ds4_cuda.cu#L19805) | +| `DS4_CUDA_NO_Q8_BATCH_TOK8` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q8 batch tok8 CUDA Q8 optimization. | [ds4_cuda.cu:19788](ds4_cuda.cu#L19788) | +| `DS4_CUDA_NO_Q8_BATCH_WARP` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q8 batch warp CUDA Q8 optimization. | [ds4_cuda.cu:19787](ds4_cuda.cu#L19787) | +| `DS4_CUDA_NO_Q8_DP4A` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q8 DP4A CUDA Q8 optimization. | [ds4_cuda.cu:2391](ds4_cuda.cu#L2391) | +| `DS4_CUDA_NO_Q8_F16_CACHE` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q8 F16 cache CUDA Q8 optimization. | [ds4_cuda.cu:2356](ds4_cuda.cu#L2356) | +| `DS4_CUDA_NO_Q8_F32_CACHE` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q8 F32 cache CUDA Q8 optimization. | [ds4_cuda.cu:2411](ds4_cuda.cu#L2411) | +| `DS4_CUDA_NO_Q8_FOLD` | value-aware kill switch, default off; nonempty value other than exact 0 disables and wins over enable | Disable the Q8_1 producer-to-consumer fold. | [ds4_cuda.cu:786](ds4_cuda.cu#L786) | +| `DS4_CUDA_NO_Q8_FUSED_ALIGNED` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q8 fused aligned CUDA Q8 optimization. | [ds4_cuda.cu:20141](ds4_cuda.cu#L20141) | +| `DS4_CUDA_NO_Q8_MMA` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q8 MMA CUDA Q8 optimization. | [ds4_cuda.cu:10781](ds4_cuda.cu#L10781) | +| `DS4_CUDA_NO_Q8_PAIR_BATCH_EXACT` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q8 pair batch exact CUDA Q8 optimization. | [ds4_cuda.cu:20302](ds4_cuda.cu#L20302) | +| `DS4_CUDA_NO_Q8_PAIR_BATCH_EXACT_TOK2` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q8 pair batch exact tok2 CUDA Q8 optimization. | [ds4_cuda.cu:20305](ds4_cuda.cu#L20305) | +| `DS4_CUDA_NO_QKV_KV_ROPE_FUSE` | value-aware kill switch; default off; nonempty value other than exact 0 disables | Disable the CUDA QKV KV rope fuse optimization/path. | [ds4.c:17253](ds4.c#L17253) | +| `DS4_CUDA_NO_QKV_PAIR` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA QKV pair optimization/path. | [ds4.c:17389](ds4.c#L17389) | +| `DS4_CUDA_NO_SCORE_TILE` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA score tile optimization/path. | [ds4_cuda.cu:13734](ds4_cuda.cu#L13734); [ds4.c:67932](ds4.c#L67932) | +| `DS4_CUDA_NO_SETDEVICE_CACHE` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the cached current-tier shortcut and call cudaSetDevice for every tier selection. | [ds4_cuda.cu:377](ds4_cuda.cu#L377) | +| `DS4_CUDA_NO_SPLITKV_DECODE` | value-aware kill switch, default off; exact 0/empty is off, other nonempty values disable | Disable splitkv decode in CUDA split-KV attention/speculation. | [ds4_cuda.cu:2211](ds4_cuda.cu#L2211) | +| `DS4_CUDA_NO_SPLITKV_SPEC` | value-aware kill switch; default off; nonempty value other than exact 0 disables | Disable splitkv spec in CUDA split-KV attention/speculation. | [ds4.c:17080](ds4.c#L17080) | +| `DS4_CUDA_NO_SPLITKV_SPEC_BATCH_VERIFY` | value-aware kill switch; default off; nonempty value other than exact 0 disables | Disable splitkv spec batch verify in CUDA split-KV attention/speculation. | [ds4.c:17100](ds4.c#L17100) | +| `DS4_CUDA_NO_SPLITKV_SPEC_TOPONLY_ROW0` | value-aware kill switch; default off; nonempty value other than exact 0 disables | Disable splitkv spec toponly row0 in CUDA split-KV attention/speculation. | [ds4.c:17090](ds4.c#L17090) | +| `DS4_CUDA_NO_STREAMING_EXPERT_PERSISTENT_CACHE` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Disable streaming expert persistent cache in CUDA SSD streaming. | [ds4_cuda.cu:4115](ds4_cuda.cu#L4115) | +| `DS4_CUDA_NO_STREAMING_SELECTED_BATCH_IO` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Disable streaming selected batch I/O in CUDA SSD streaming. | [ds4_cuda.cu:4736](ds4_cuda.cu#L4736) | +| `DS4_CUDA_NO_STREAMING_SELECTED_EVENT_PIPELINE` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Disable streaming selected event pipeline in CUDA SSD streaming. | [ds4_cuda.cu:4868](ds4_cuda.cu#L4868) | +| `DS4_CUDA_NO_TF32` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Use default cuBLAS math instead of TF32 tensor operations. | [ds4_cuda.cu:6657](ds4_cuda.cu#L6657) | +| `DS4_CUDA_NO_TOP1` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the dedicated CUDA indexer top-1 kernel. | [ds4_cuda.cu:375](ds4_cuda.cu#L375) | +| `DS4_CUDA_NO_TOPK1024` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the topk1024 CUDA indexer kernel/path. | [ds4_cuda.cu:19290](ds4_cuda.cu#L19290) | +| `DS4_CUDA_NO_TOPK2048` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the topk2048 CUDA indexer kernel/path. | [ds4_cuda.cu:19297](ds4_cuda.cu#L19297) | +| `DS4_CUDA_NO_TOPK2048_WIDE` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the topk2048 wide CUDA indexer kernel/path. | [ds4_cuda.cu:19212](ds4_cuda.cu#L19212) | +| `DS4_CUDA_NO_TOPK8192` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the topk8192 CUDA indexer kernel/path. | [ds4_cuda.cu:19335](ds4_cuda.cu#L19335) | +| `DS4_CUDA_NO_TOPK_CHUNKED` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the topk chunked CUDA indexer kernel/path. | [ds4_cuda.cu:19374](ds4_cuda.cu#L19374) | +| `DS4_CUDA_NO_TOPK_STREAM` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the topk stream CUDA indexer kernel/path. | [ds4_cuda.cu:19366](ds4_cuda.cu#L19366) | +| `DS4_CUDA_NO_TP_ATTN_OUT_HC_FUSE` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA TP attn out HC fuse optimization/path. | [ds4.c:17392](ds4.c#L17392) | +| `DS4_CUDA_NO_VERIFY_DECODE2_SPLIT_TOP1` | value-aware kill switch; default off; nonempty value other than exact 0 disables | Disable the CUDA verify decode2 split top1 optimization/path. | [ds4.c:17050](ds4.c#L17050) | +| `DS4_CUDA_NO_WARP_ROUTER_SELECT` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA warp router select optimization/path. | [ds4_cuda.cu:24490](ds4_cuda.cu#L24490) | +| `DS4_CUDA_NO_WINDOW_ATTENTION` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA window attention optimization/path. | [ds4_cuda.cu:23050](ds4_cuda.cu#L23050) | +| `DS4_CUDA_NSYS_PREFILL_START_POS` | nonempty-string flag, default off; any nonempty value enables MMQ NVTX ranges (the value is not parsed as a position) | Enable MMQ NVTX annotations intended for Nsight Systems prefill capture. | [cuda/mmq/ds4_mmq.cu:48](cuda/mmq/ds4_mmq.cu#L48) | +| `DS4_CUDA_NVTX` | strict flag, default off; only exact value 1 enables (a nonempty NSYS variable also enables ranges) | Enable NVTX ranges around MMQ work. | [cuda/mmq/ds4_mmq.cu:47](cuda/mmq/ds4_mmq.cu#L47) | +| `DS4_CUDA_OUTPUT_FUSED_TOP1` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Fuse output projection with top-1 selection in greedy decode. | [ds4.c:17042](ds4.c#L17042) | +| `DS4_CUDA_PREFILL_PIPELINE` | boolean, default follows CUDA TP decode; nonempty exact 0 disables, any other nonempty value enables | Control the CUDA multi-tier prefill pipeline. | [ds4.c:17281](ds4.c#L17281) | +| `DS4_CUDA_PREFILL_PIPELINE_MB` | positive integer rows; default/invalid 512 | Set prefill-pipeline microbatch rows. | [ds4.c:17296](ds4.c#L17296) | +| `DS4_CUDA_PREFILL_PIPELINE_Q8_CACHE` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Keep selective Q8 caches enabled while running the prefill pipeline. | [ds4.c:17291](ds4.c#L17291) | +| `DS4_CUDA_PREFILL_PIPELINE_SEQUENTIAL` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Execute prefill pipeline stages sequentially for diagnosis. | [ds4.c:35335](ds4.c#L35335) | +| `DS4_CUDA_PREFILL_PIPELINE_SYNC_BOUNDARY` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Synchronize CUDA at every prefill pipeline tier boundary. | [ds4.c:35382](ds4.c#L35382) | +| `DS4_CUDA_Q4_ATTN_OUT_HC_ORACLE` | value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on | Compare fused Q4 attention-output/HC expansion with the canonical path and retain canonical output. | [ds4_cuda.cu:1475](ds4_cuda.cu#L1475) | +| `DS4_CUDA_Q4_ATTN_OUT_HC_Q8K_EXPERIMENT` | value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on | Enable the experimental Q8_K-based Q4 attention-output/HC fusion. | [ds4_cuda.cu:37357](ds4_cuda.cu#L37357) | +| `DS4_CUDA_Q4_GROUPED_ATTN_A_ORACLE` | value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on | Compare grouped attention-A against the canonical per-group result. | [ds4_cuda.cu:1477](ds4_cuda.cu#L1477) | +| `DS4_CUDA_Q4_K1024_PERSISTENT_ORACLE` | value-aware flag, default off; nonempty value other than exact 0 enables and implies candidate admission | Bitwise-compare the exact-shape persistent Q4 K1024 kernel with canonical MMVQ and retain canonical output. | [cuda/mmq/ds4_mmq.cu:3534](cuda/mmq/ds4_mmq.cu#L3534) | +| `DS4_CUDA_Q4_K1024_PERSISTENT_STATS` | value-aware flag, default off; nonempty value other than exact 0 enables | Print exact-shape persistent Q4 K1024 dispatch counters at exit. | [cuda/mmq/ds4_mmq.cu:3533](cuda/mmq/ds4_mmq.cu#L3533) | +| `DS4_CUDA_Q8_F16_ALL` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Control the Q8 F16 all CUDA quantized-matmul/cache optimization. | [ds4_cuda.cu:2358](ds4_cuda.cu#L2358) | +| `DS4_CUDA_Q8_F16_CACHE_MB` | unsigned integer MiB, full-string parse; default unlimited; 0 disables this cache | Limit the selective Q8-to-F16 derived-weight cache. | [ds4_cuda.cu:2218](ds4_cuda.cu#L2218) | +| `DS4_CUDA_Q8_F16_CACHE_RESERVE_MB` | unsigned integer MiB, full-string parse; default is VRAM-dependent (>=112 GiB: 512; >=40 GiB: max(768,1%); smaller: max(4096,5%)) | Reserve free VRAM when growing the selective Q8-to-F16 cache. | [ds4_cuda.cu:2224](ds4_cuda.cu#L2224) | +| `DS4_CUDA_Q8_F32_ALL` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Control the Q8 F32 all CUDA quantized-matmul/cache optimization. | [ds4_cuda.cu:2412](ds4_cuda.cu#L2412) | +| `DS4_CUDA_Q8_F32_LARGE` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Control the Q8 F32 large CUDA quantized-matmul/cache optimization. | [ds4_cuda.cu:2416](ds4_cuda.cu#L2416) | +| `DS4_CUDA_Q8_F32_PRELOAD` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Control the Q8 F32 preload CUDA quantized-matmul/cache optimization. | [ds4_cuda.cu:8597](ds4_cuda.cu#L8597) | +| `DS4_CUDA_Q8_FOLD_ORACLE` | strict flag, default off; only exact value 1 enables | Compare folded Q8_1 bytes and consumer outputs against canonical work while retaining canonical results. | [cuda/mmq/ds4_mmq.cu:383](cuda/mmq/ds4_mmq.cu#L383) | +| `DS4_CUDA_Q8_HC_EXPAND_FUSED` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, other nonempty values force fused | Force the fused Q8 shared-down/HC expansion path. | [ds4_cuda.cu:2084](ds4_cuda.cu#L2084) | +| `DS4_CUDA_Q8_HC_EXPAND_STATS` | false-like-aware flag, default off; 0/false/no/off is off, other nonempty values print report | Print Q8 shared-down/HC policy and dispatch counters at exit. | [ds4_cuda.cu:2090](ds4_cuda.cu#L2090) | +| `DS4_CUDA_Q8_NO_ALIGNED` | value-aware kill switch, default off; nonempty value other than exact 0 disables aligned Q8 kernels | Disable aligned Q8 CUDA matmul kernels. | [ds4_cuda.cu:1021](ds4_cuda.cu#L1021) | +| `DS4_CUDA_Q8_PAIR_BATCH` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Control the Q8 pair batch CUDA quantized-matmul/cache optimization. | [ds4_cuda.cu:20167](ds4_cuda.cu#L20167) | +| `DS4_CUDA_QKV_KV_ROPE_FUSE` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control or tune the CUDA QKV KV rope fuse path. | [ds4.c:17256](ds4.c#L17256) | +| `DS4_CUDA_Q_NORM_ROPE_FUSE` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control or tune the CUDA q norm rope fuse path. | [ds4.c:17245](ds4.c#L17245) | +| `DS4_CUDA_REQUIRE_IQ2_XXS_SSD_PREFILL_MMQ` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Require the CUDA IQ2 XXS SSD prefill MMQ path; fail closed when unavailable. | [ds4_cuda.cu:4631](ds4_cuda.cu#L4631) | +| `DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_BATCH` | value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on | Fail if grouped batched attention-A cannot be used. | [ds4_cuda.cu:40178](ds4_cuda.cu#L40178) | +| `DS4_CUDA_REQUIRE_Q4_K1024_PERSISTENT` | presence flag, default off; any defined value including 0 makes ineligible candidate fail closed | Fail when the exact Q4 K1024 persistent candidate is unavailable instead of using MMVQ. | [cuda/mmq/ds4_mmq.cu:3722](cuda/mmq/ds4_mmq.cu#L3722) | +| `DS4_CUDA_REQUIRE_STREAMING_EXPERT_PERSISTENT_CACHE` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Require streaming expert persistent cache in CUDA SSD streaming; fail closed when unavailable. | [ds4_cuda.cu:4117](ds4_cuda.cu#L4117) | +| `DS4_CUDA_REQUIRE_STREAMING_SELECTED_BATCH_IO` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Require streaming selected batch I/O in CUDA SSD streaming; fail closed when unavailable. | [ds4_cuda.cu:4738](ds4_cuda.cu#L4738) | +| `DS4_CUDA_REQUIRE_STREAMING_SELECTED_EVENT_PIPELINE` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Require streaming selected event pipeline in CUDA SSD streaming; fail closed when unavailable. | [ds4_cuda.cu:4870](ds4_cuda.cu#L4870) | +| `DS4_CUDA_SERIAL_F16_MATMUL` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Control the serial F16 matmul CUDA F16 matmul path. | [ds4_cuda.cu:20801](ds4_cuda.cu#L20801) | +| `DS4_CUDA_SERIAL_ROUTER` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Control or tune the CUDA serial router path. | [ds4_cuda.cu:20806](ds4_cuda.cu#L20806) | +| `DS4_CUDA_SESSION_BATCH_ATTN_ALIAS` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control the grouped multi-session CUDA attn alias stage. | [ds4.c:69712](ds4.c#L69712) | +| `DS4_CUDA_SESSION_BATCH_ATTN_CORE` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control the grouped multi-session CUDA attn core stage. | [ds4.c:69715](ds4.c#L69715) | +| `DS4_CUDA_SESSION_BATCH_ATTN_POST` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control the grouped multi-session CUDA attn post stage. | [ds4.c:69727](ds4.c#L69727) | +| `DS4_CUDA_SESSION_BATCH_ATTN_PRE` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control the grouped multi-session CUDA attn pre stage. | [ds4.c:69708](ds4.c#L69708) | +| `DS4_CUDA_SESSION_BATCH_FFN_PRE` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control the grouped multi-session CUDA ffn pre stage. | [ds4.c:69704](ds4.c#L69704) | +| `DS4_CUDA_SESSION_BATCH_INTERLEAVE` | boolean, default on; unset/empty/nonzero enables pipeline interleaving; exact 0 disables | Control the grouped multi-session CUDA interleave stage. | [ds4.c:70388](ds4.c#L70388) | +| `DS4_CUDA_SESSION_BATCH_KV_STORE` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control the grouped multi-session CUDA KV store stage. | [ds4.c:69723](ds4.c#L69723) | +| `DS4_CUDA_SESSION_BATCH_MOE` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control the grouped multi-session CUDA MoE stage. | [ds4.c:69696](ds4.c#L69696) | +| `DS4_CUDA_SESSION_BATCH_MOE_COMBINE_ROWS` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control the grouped multi-session CUDA MoE combine rows stage. | [ds4.c:69241](ds4.c#L69241) | +| `DS4_CUDA_SESSION_BATCH_QKV` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control the grouped multi-session CUDA QKV stage. | [ds4.c:69719](ds4.c#L69719) | +| `DS4_CUDA_SESSION_BATCH_SHARED` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control the grouped multi-session CUDA shared stage. | [ds4.c:69700](ds4.c#L69700) | +| `DS4_CUDA_SPLITKV_CHUNK` | integer scores/chunk; default 512; clamped 1..512 | Control splitkv chunk in CUDA split-KV attention/speculation. | [ds4_cuda.cu:22568](ds4_cuda.cu#L22568) | +| `DS4_CUDA_SPLITKV_DECODE` | value-aware boolean, default off; exact 0/empty is off, other nonempty values enable; mere presence also excludes one session-batch path | Enable split-KV decode attention. | [ds4_cuda.cu:2213](ds4_cuda.cu#L2213); [ds4.c:67921](ds4.c#L67921) | +| `DS4_CUDA_SPLITKV_GLOBAL_SOFTMAX` | value-aware opt-in, default off; exact 0/empty is off, other nonempty values enable | Use the global-softmax variant of split-KV attention. | [ds4_cuda.cu:22589](ds4_cuda.cu#L22589) | +| `DS4_CUDA_SPLITKV_MIN_SCORE` | integer score count 0..UINT32_MAX; default 0 when explicitly enabled, otherwise 512; CUDA kernel clamps to 0..8192 | Set the minimum visible-score count for split-KV attention. | [ds4_cuda.cu:22557](ds4_cuda.cu#L22557); [ds4.c:17229](ds4.c#L17229) | +| `DS4_CUDA_SPLITKV_S` | integer exact split count; unset/invalid = automatic; valid value clamped 1..16 | Control splitkv s in CUDA split-KV attention/speculation. | [ds4_cuda.cu:22578](ds4_cuda.cu#L22578) | +| `DS4_CUDA_SPLITKV_SPEC` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Enable split-KV speculative decoding. | [ds4.c:17082](ds4.c#L17082) | +| `DS4_CUDA_SPLITKV_SPEC_BATCH_VERIFY` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Use batched verification for split-KV speculation. | [ds4.c:17102](ds4.c#L17102) | +| `DS4_CUDA_SPLITKV_SPEC_LOG` | presence diagnostic flag; default off; any defined value including 0 enables | Log split-KV speculative-decode admission and fallback decisions. | [ds4.c:55622](ds4.c#L55622) | +| `DS4_CUDA_SPLITKV_SPEC_TIMING` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Print timing for split-KV speculative-decode stages. | [ds4.c:55661](ds4.c#L55661) | +| `DS4_CUDA_SPLITKV_SPEC_TOPONLY_ROW0` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Compute only the top result for row zero in split-KV speculation. | [ds4.c:17092](ds4.c#L17092) | +| `DS4_CUDA_SPLITKV_S_FLOOR` | integer split count; default 4; clamped 1..16 | Control splitkv s floor in CUDA split-KV attention/speculation. | [ds4_cuda.cu:22571](ds4_cuda.cu#L22571) | +| `DS4_CUDA_SPLITKV_S_MAX` | integer split count; default 16; clamped 1..16 | Control splitkv s max in CUDA split-KV attention/speculation. | [ds4_cuda.cu:22574](ds4_cuda.cu#L22574) | +| `DS4_CUDA_STREAMING_EXPERT_CACHE_PROFILE` | presence diagnostic flag; default off; any defined value including 0 enables | Profile CUDA SSD-streaming streaming expert cache. | [ds4.c:21676](ds4.c#L21676) | +| `DS4_CUDA_STREAMING_EXPERT_PERSISTENT_CACHE_ORACLE` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Run the diagnostic oracle for CUDA SSD-streaming streaming expert persistent cache. | [ds4_cuda.cu:4121](ds4_cuda.cu#L4121) | +| `DS4_CUDA_STREAMING_EXPERT_PERSISTENT_CACHE_STATS` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Print counters for CUDA SSD-streaming streaming expert persistent cache. | [ds4_cuda.cu:4119](ds4_cuda.cu#L4119) | +| `DS4_CUDA_STREAMING_PREFILL_BATCH_SELECTED_PROFILE` | presence diagnostic flag; default off; any defined value including 0 enables | Profile CUDA SSD-streaming streaming prefill batch selected. | [ds4.c:21759](ds4.c#L21759) | +| `DS4_CUDA_STREAMING_SELECTED_BATCH_IO_ORACLE` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Run the diagnostic oracle for CUDA SSD-streaming streaming selected batch I/O. | [ds4_cuda.cu:4740](ds4_cuda.cu#L4740) | +| `DS4_CUDA_STREAMING_SELECTED_BATCH_IO_PROFILE` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Profile CUDA SSD-streaming streaming selected batch I/O. | [ds4_cuda.cu:5711](ds4_cuda.cu#L5711) | +| `DS4_CUDA_STREAMING_SELECTED_EVENT_PIPELINE_ORACLE` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Run the diagnostic oracle for CUDA SSD-streaming streaming selected event pipeline. | [ds4_cuda.cu:4872](ds4_cuda.cu#L4872) | +| `DS4_CUDA_STREAMING_SELECTED_EVENT_PIPELINE_STATS` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Print counters for CUDA SSD-streaming streaming selected event pipeline. | [ds4_cuda.cu:4874](ds4_cuda.cu#L4874) | +| `DS4_CUDA_STRICT_WEIGHT_CACHE` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Fail a weight lookup when cache allocation fails instead of falling back to mapped model memory. | [ds4_cuda.cu:6388](ds4_cuda.cu#L6388) | +| `DS4_CUDA_SYNC_XDEV` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Synchronize cross-device CUDA copies for debugging and error localization. | [ds4_cuda.cu:363](ds4_cuda.cu#L363) | +| `DS4_CUDA_TP_ATTN` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel attn execution. | [ds4.c:16862](ds4.c#L16862) | +| `DS4_CUDA_TP_ATTN_CACHE_DUP` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel attn cache dup execution. | [ds4.c:16886](ds4.c#L16886) | +| `DS4_CUDA_TP_ATTN_HEADS` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel attn heads execution. | [ds4.c:16878](ds4.c#L16878) | +| `DS4_CUDA_TP_ATTN_OUT_HC_FUSE` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Control CUDA tensor/expert-parallel attn out HC fuse execution. | [ds4.c:17391](ds4.c#L17391) | +| `DS4_CUDA_TP_ATTN_PEER_READ` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel attn peer read execution. | [ds4.c:16870](ds4.c#L16870) | +| `DS4_CUDA_TP_EP_BALANCED_SHARED_MID` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel EP balanced shared mid execution. | [ds4.c:16943](ds4.c#L16943) | +| `DS4_CUDA_TP_EP_DELAY_REDUCE` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel EP delay reduce execution. | [ds4.c:16918](ds4.c#L16918) | +| `DS4_CUDA_TP_EP_DIRECT_RETURN` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel EP direct return execution. | [ds4.c:16910](ds4.c#L16910) | +| `DS4_CUDA_TP_EP_DUAL_PREQUANT` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel EP dual prequant execution. | [ds4.c:16952](ds4.c#L16952) | +| `DS4_CUDA_TP_EP_FUSED_HC_REDUCE` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel EP fused HC reduce execution. | [ds4.c:16926](ds4.c#L16926) | +| `DS4_CUDA_TP_EP_FUSED_SHARED_MID` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel EP fused shared mid execution. | [ds4.c:16934](ds4.c#L16934) | +| `DS4_CUDA_TP_EP_PACK_EXACT` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel EP pack exact execution. | [ds4.c:16902](ds4.c#L16902) | +| `DS4_CUDA_TP_MOE` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel MoE execution. | [ds4.c:16894](ds4.c#L16894) | +| `DS4_CUDA_TP_MOE_COPY3` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel MoE copy3 execution. | [ds4.c:16976](ds4.c#L16976) | +| `DS4_CUDA_TP_MOE_DELAY_REDUCE` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel MoE delay reduce execution. | [ds4.c:16960](ds4.c#L16960) | +| `DS4_CUDA_TP_MOE_PACK` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel MoE pack execution. | [ds4.c:16968](ds4.c#L16968) | +| `DS4_CUDA_TP_MOE_PEER_READ` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel MoE peer read execution. | [ds4.c:16984](ds4.c#L16984) | +| `DS4_CUDA_TP_MOE_PEER_ROUTER` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel MoE peer router execution. | [ds4.c:16992](ds4.c#L16992) | +| `DS4_CUDA_TP_OUTPUT` | boolean, default on; empty/unset/nonzero enables, exact 0 disables | Control CUDA tensor/expert-parallel output execution. | [ds4.c:51523](ds4.c#L51523) | +| `DS4_CUDA_TP_OUTPUT_WAYS` | integer 2..DS4_MAX_GPUS (16); default 8; invalid value falls back to 2; capped by available GPUs | Set the number of GPU ways used to shard CUDA tensor-parallel output projection. | [ds4.c:58](ds4.c#L58); [ds4.c:51530](ds4.c#L51530) | +| `DS4_CUDA_TP_PREFILL_ATTN_OUTPUT` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel prefill attn output execution. | [ds4.c:17272](ds4.c#L17272) | +| `DS4_CUDA_TP_PREFILL_FFN` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel prefill ffn execution. | [ds4.c:17264](ds4.c#L17264) | +| `DS4_CUDA_TP_Q` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel q execution. | [ds4.c:17016](ds4.c#L17016) | +| `DS4_CUDA_TP_SHARED` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel shared execution. | [ds4.c:17000](ds4.c#L17000) | +| `DS4_CUDA_TP_SHARED_FOLD` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel shared fold execution. | [ds4.c:17008](ds4.c#L17008) | +| `DS4_CUDA_VERIFY_DECODE2_SPLIT_TOP1` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Enable the split top-1 path for two-row verification decode. | [ds4.c:17052](ds4.c#L17052) | +| `DS4_CUDA_WEIGHT_ARENA_CHUNK_MB` | positive integer MiB; default 1792; clamped 256..8192 and raised/aligned when one allocation needs more | Set the CUDA selective-weight arena allocation chunk. | [ds4_cuda.cu:6311](ds4_cuda.cu#L6311) | +| `DS4_CUDA_WEIGHT_CACHE` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Force selective CUDA weight caching instead of direct mapped access. | [ds4_cuda.cu:1246](ds4_cuda.cu#L1246) | +| `DS4_CUDA_WEIGHT_CACHE_LIMIT_GB` | unsigned integer GiB; default/0 = unlimited; parser accepts a numeric prefix even with trailing text | Limit total CUDA selective-weight cache allocation. | [ds4_cuda.cu:6299](ds4_cuda.cu#L6299) | +| `DS4_CUDA_WEIGHT_CACHE_VERBOSE` | presence diagnostic flag; default off; any defined value including 0 enables | Print CUDA weight mapping, caching, and preload diagnostics. | [ds4_cuda.cu:1297](ds4_cuda.cu#L1297) | +| `DS4_CUDA_WEIGHT_PRELOAD` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Request proactive CUDA weight caching/preloading. | [ds4_cuda.cu:1247](ds4_cuda.cu#L1247) | +| `DS4_CUDA_WEIGHT_PRELOAD_SPAN_MB` | positive integer MiB; default 1024; clamped 64..4096 | Set the maximum span size used by CUDA weight preload. | [ds4.c:2880](ds4.c#L2880) | +| `DS4_CUDA_WINDOW_ATTENTION` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Control or tune the CUDA window attention path. | [ds4_cuda.cu:23051](ds4_cuda.cu#L23051) | + +
+ +
+ROCm (134) + +| Variable | Accepted value and default | Effect | Source | +| --- | --- | --- | --- | +| `DS4_ROCM_DECODE_STAGE_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm decode stage profile. | [ds4.c:18071](ds4.c#L18071) | +| `DS4_ROCM_DECODE_STAGE_PROFILE_LAYER` | layer filter subordinate to DS4_ROCM_DECODE_STAGE_PROFILE; unset or whitespace-only: all layers allowed by the parent flag; otherwise the whitespace-trimmed value must be a complete base-10 strtoul result <= UINT32_MAX equal to the current layer; invalid values match none | Restricts the ROCm decode stage profiler to one layer; it does not enable profiling by itself. | [ds4.c:28943](ds4.c#L28943) | +| `DS4_ROCM_DISABLE_GLM_STREAMING_PREFILL_FULL_LAYER` | integer selector/tuning value; unset or invalid uses internal automatic/default value | Disable/roll back rocm disable glm streaming prefill full layer. | [ds4.c:42519](ds4.c#L42519) | +| `DS4_ROCM_DISABLE_GLM_STREAMING_PREFILL_FULL_LAYER_PREPARE` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable glm streaming prefill full layer prepare. | [ds4.c:42537](ds4.c#L42537) | +| `DS4_ROCM_DISABLE_GLM_STREAMING_PREFILL_SELECTED_ASYNC_LOAD` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable glm streaming prefill selected async load. | [ds4.c:46230](ds4.c#L46230) | +| `DS4_ROCM_DISABLE_GLM_STREAMING_SELECTED_ASYNC_LOAD` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable glm streaming selected async load. | [ds4.c:44138](ds4.c#L44138) | +| `DS4_ROCM_DISABLE_IQ2_SELECTED_EXPERT_VIEWS` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable iq2 selected expert views. | [ds4.c:20906](ds4.c#L20906) | +| `DS4_ROCM_DISABLE_IQ2_STREAM_ADDR_TABLE` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable iq2 stream addr table. | [ds4.c:6425](ds4.c#L6425) | +| `DS4_ROCM_DISABLE_Q4_DENSE_PAIR` | presence rollback; unset leaves opt-in policy unchanged | Disable/roll back rocm disable q4 dense pair. | [rocm/ds4_rocm_q4.cuh:299](rocm/ds4_rocm_q4.cuh#L299) | +| `DS4_ROCM_DISABLE_Q4_GROUPED_ATTN_A` | presence rollback; DISABLE wins over enable/require | Disable/roll back rocm disable q4 grouped attn a. | [rocm/ds4_rocm_q4.cuh:573](rocm/ds4_rocm_q4.cuh#L573) | +| `DS4_ROCM_DISABLE_Q4_PREFILL_TILE8` | presence rollback; TILE8 is default for 9..4096 tokens | Disable/roll back rocm disable q4 prefill tile8. | [rocm/ds4_rocm_q4.cuh:312](rocm/ds4_rocm_q4.cuh#L312) | +| `DS4_ROCM_DISABLE_Q4_SELECTED_EXPERT_VIEWS` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable q4 selected expert views. | [ds4.c:20962](ds4.c#L20962) | +| `DS4_ROCM_DISABLE_RESIDENT_IQ2_SORTED` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable resident iq2 sorted. | [rocm/ds4_rocm_moe_launch.cuh:747](rocm/ds4_rocm_moe_launch.cuh#L747) | +| `DS4_ROCM_DISABLE_ROUTED_PAIR_SWIGLU_FUSION` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable routed pair swiglu fusion. | [ds4.c:18355](ds4.c#L18355) | +| `DS4_ROCM_DISABLE_STREAMING_COLD_DECODE_PREFILL` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming cold decode prefill. | [ds4.c:31816](ds4.c#L31816) | +| `DS4_ROCM_DISABLE_STREAMING_DECODE_PREFILL` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming decode prefill. | [ds4.c:31765](ds4.c#L31765) | +| `DS4_ROCM_DISABLE_STREAMING_EXPERT_ADDR_TABLE` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming expert addr table. | [ds4.c:18351](ds4.c#L18351) | +| `DS4_ROCM_DISABLE_STREAMING_EXPERT_HOTLIST` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming expert hotlist. | [ds4.c:21116](ds4.c#L21116) | +| `DS4_ROCM_DISABLE_STREAMING_FULL_EXPERT_ADDR_TABLE` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming full expert addr table. | [ds4.c:18069](ds4.c#L18069) | +| `DS4_ROCM_DISABLE_STREAMING_LAYER_BATCH` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming layer batch. | [ds4.c:18065](ds4.c#L18065) | +| `DS4_ROCM_DISABLE_STREAMING_MADVISE_WILLNEED` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming madvise willneed. | [ds4.c:18038](ds4.c#L18038) | +| `DS4_ROCM_DISABLE_STREAMING_PREFILL_BATCH_SELECTED_ADDR` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill batch selected addr. | [ds4.c:18349](ds4.c#L18349) | +| `DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_MADVISE` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill layer madvise. | [ds4.c:18292](ds4.c#L18292) | +| `DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_PAGEIN` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill layer pagein. | [ds4.c:18260](ds4.c#L18260) | +| `DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_PAGEIN_OVERLAP` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill layer pagein overlap. | [ds4.c:19147](ds4.c#L19147) | +| `DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_PREAD` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill layer pread. | [ds4.c:18280](ds4.c#L18280) | +| `DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_PREPARE` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill layer prepare. | [ds4.c:18272](ds4.c#L18272) | +| `DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_PREPARE_OVERLAP` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill layer prepare overlap. | [ds4.c:19145](ds4.c#L19145) | +| `DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_READAHEAD` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill layer readahead. | [ds4.c:18270](ds4.c#L18270) | +| `DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_ASYNC_LOAD` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill selected async load. | [ds4.c:46233](ds4.c#L46233) | +| `DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_MADVISE` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill selected madvise. | [ds4.c:18250](ds4.c#L18250) | +| `DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_PAGEIN` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill selected pagein. | [ds4.c:18240](ds4.c#L18240) | +| `DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_PROFILE` | presence rollback flag; unset keeps automatic/default path | Collect timing/profile diagnostics for rocm disable streaming prefill selected profile. | [ds4.c:18734](ds4.c#L18734) | +| `DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_READAHEAD` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill selected readahead. | [ds4.c:19786](ds4.c#L19786) | +| `DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_READAHEAD_SHARED` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill selected readahead shared. | [ds4.c:19796](ds4.c#L19796) | +| `DS4_ROCM_DISABLE_STREAMING_READAHEAD` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming readahead. | [ds4.c:18031](ds4.c#L18031) | +| `DS4_ROCM_DISABLE_STREAMING_SELECTED_ASYNC_LOAD` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming selected async load. | [ds4.c:44136](ds4.c#L44136) | +| `DS4_ROCM_DISABLE_STREAMING_SPLIT_SELECTED` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming split selected. | [rocm/ds4_rocm_moe_launch.cuh:661](rocm/ds4_rocm_moe_launch.cuh#L661) | +| `DS4_ROCM_DISABLE_STREAMING_STATIC_DECODE_MAP` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming static decode map. | [ds4.c:18043](ds4.c#L18043) | +| `DS4_ROCM_DISABLE_STREAMING_STATIC_MAP_STATE_CACHE` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming static map state cache. | [ds4.c:18056](ds4.c#L18056) | +| `DS4_ROCM_DSV4_PREQUANT_DECODE` | sampled once; unset: enabled; present empty or exact 0: disabled; every other present value: enabled; quality mode and GLM models force it off regardless | ROCm DeepSeek-V4 decode: quantizes one-token F32 activations to Q8 once and selects the prequantized Q8_0/DP4A projection kernels instead of the full-F32 activation paths. | [rocm/ds4_rocm_runtime.cuh:4775](rocm/ds4_rocm_runtime.cuh#L4775) | +| `DS4_ROCM_ENABLE_Q4_DENSE_PAIR` | presence opt-in; unset=off; DISABLE takes precedence | Enable rocm enable q4 dense pair. | [rocm/ds4_rocm_q4.cuh:298](rocm/ds4_rocm_q4.cuh#L298) | +| `DS4_ROCM_ENABLE_Q4_GROUPED_ATTN_A` | presence opt-in; unset=off unless REQUIRE; DISABLE wins | Enable rocm enable q4 grouped attn a. | [rocm/ds4_rocm_q4.cuh:577](rocm/ds4_rocm_q4.cuh#L577) | +| `DS4_ROCM_ENABLE_STREAMING_FULL_EXPERT_ADDR_TABLE` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming full expert addr table. | [ds4.c:18067](ds4.c#L18067) | +| `DS4_ROCM_ENABLE_STREAMING_MADVISE_WILLNEED` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming madvise willneed. | [ds4.c:18036](ds4.c#L18036) | +| `DS4_ROCM_ENABLE_STREAMING_PREFILL_BATCH_SELECTED_ADDR` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming prefill batch selected addr. | [ds4.c:18396](ds4.c#L18396) | +| `DS4_ROCM_ENABLE_STREAMING_PREFILL_CACHE_SEED` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming prefill cache seed. | [ds4.c:21085](ds4.c#L21085) | +| `DS4_ROCM_ENABLE_STREAMING_PREFILL_LAYER_PAGEIN` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming prefill layer pagein. | [ds4.c:18258](ds4.c#L18258) | +| `DS4_ROCM_ENABLE_STREAMING_PREFILL_LAYER_READAHEAD` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming prefill layer readahead. | [ds4.c:18268](ds4.c#L18268) | +| `DS4_ROCM_ENABLE_STREAMING_PREFILL_SELECTED_MADVISE` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming prefill selected madvise. | [ds4.c:18248](ds4.c#L18248) | +| `DS4_ROCM_ENABLE_STREAMING_PREFILL_SELECTED_PAGEIN` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming prefill selected pagein. | [ds4.c:18238](ds4.c#L18238) | +| `DS4_ROCM_ENABLE_STREAMING_PREFILL_SELECTED_READAHEAD` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming prefill selected readahead. | [ds4.c:19782](ds4.c#L19782) | +| `DS4_ROCM_ENABLE_STREAMING_PREFILL_SELECTED_READAHEAD_SHARED` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming prefill selected readahead shared. | [ds4.c:19784](ds4.c#L19784) | +| `DS4_ROCM_ENABLE_STREAMING_READAHEAD` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming readahead. | [ds4.c:18029](ds4.c#L18029) | +| `DS4_ROCM_ENABLE_STREAMING_STATIC_DECODE_MAP` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming static decode map. | [ds4.c:18048](ds4.c#L18048) | +| `DS4_ROCM_GLM_CAUSAL_ATTN_GEMM` | Enabled by default when unset. Exact "0" or an empty value disables; every other nonempty value enables (including false/off/no), because cuda_env_present only tests nonempty and != "0". Eligibility still requires causal_range && !has_selected; a failed GEMM helper falls through to the scalar attention kernel. | Use FP16 BLAS GEMMs for dense causal GLM indexed prefill; =0 is the correctness/performance rollback to the scalar attention kernel. | [rocm/ds4_rocm_glm.cuh:3283](rocm/ds4_rocm_glm.cuh#L3283) | +| `DS4_ROCM_GLM_DISABLE_STREAMING_EXPERT_CACHE` | Pure presence flag: any defined value, including empty or "0", disables. Unset leaves automatic GLM streaming expert-cache eligibility enabled for supported model/quant/quality/SSD configurations. On ROCm builds DS4_METAL_GLM_DISABLE_STREAMING_EXPERT_CACHE is an accepted fallback alias. | Disable selected/resident streamed-expert cache paths and force generic/full-layer expert handling for GLM SSD streaming. | [ds4.c:21024](ds4.c#L21024) | +| `DS4_ROCM_GLM_DISABLE_STREAMING_SEED_BEFORE_PREFILL` | Pure presence flag: any defined value, including empty or "0", disables. Unset seeds before prefill whenever SSD streaming is active. On ROCm builds DS4_METAL_GLM_DISABLE_STREAMING_SEED_BEFORE_PREFILL is an accepted fallback alias. | Skip the pre-prefill hotlist seed of the streaming expert cache in both one-shot GLM generation and session setup. | [ds4.c:50883](ds4.c#L50883) | +| `DS4_ROCM_GLM_DISABLE_STREAMING_TOKEN_PREFILL` | Pure presence flag: any defined value, including empty or "0", disables. Unset leaves the token-major path eligible only for SSD streaming, non-quality mode, a nonempty batch fitting full attention, and n_tokens <= the configured nonzero maximum. DS4_METAL_GLM_DISABLE_STREAMING_TOKEN_PREFILL and generic DS4_GLM_DISABLE_STREAMING_TOKEN_PREFILL are also accepted presence aliases. | Roll back GLM SSD-streaming token-major prefill to the normal prefill implementation. | [ds4.c:49513](ds4.c#L49513) | +| `DS4_ROCM_GLM_GROUPED_QK_LOW` | sampled once; unset: enabled; present empty or exact 0: disabled; every other present value: enabled | Selects the grouped shared-input ROCm kernel for eligible multi-token GLM qk-lowrank projection; disabling uses the per-head/per-token projection kernel. | [rocm/ds4_rocm_runtime.cuh:4800](rocm/ds4_rocm_runtime.cuh#L4800) | +| `DS4_ROCM_GLM_GROUPED_VALUE_PROJECT` | sampled once; unset: enabled; present empty or exact 0: disabled; every other present value: enabled | Selects the grouped shared-input ROCm kernel for eligible multi-token GLM value projection; disabling uses the non-grouped batch projection path. | [rocm/ds4_rocm_runtime.cuh:4791](rocm/ds4_rocm_runtime.cuh#L4791) | +| `DS4_ROCM_GLM_LAYER_SLICE_TOKEN_DECODE` | Opt-in truthy parser; unset, empty, "0", false, off, or no (case-insensitive words) are false, every other nonempty value is true. Default off and compiled only for ROCm. | Allow a one-token, pos>0 GLM layer-slice with inter-node input/output hidden buffers to use the optimized resident token graph; without it only the no-hidden-buffer case takes that shortcut. | [ds4.c:43383](ds4.c#L43383) | +| `DS4_ROCM_GLM_SELECTED_ATTN_GEMM` | Enabled by default when unset. Exact "0" or an empty value disables; every other nonempty value enables (including false/off/no). Eligibility still requires !causal_range && has_selected; failure/ineligibility falls through to the scalar attention kernel. | Gather per-token selected cache rows into FP16 matrices and use strided-batched BLAS GEMMs for GLM selected indexed prefill; =0 forces the scalar path. | [rocm/ds4_rocm_glm.cuh:3239](rocm/ds4_rocm_glm.cuh#L3239) | +| `DS4_ROCM_GLM_SELECTED_ATTN_HEAD_TILE` | Unsigned integer read and cached once; valid values are exactly 1,2,4,8,16,32,64. Unset/empty defaults to 16. A nonnumeric, partially parsed, overflowed, or unsupported value prints a warning and uses 16; the effective tile is min(requested,n_head). | Set how many attention heads each selected-attention GEMM workspace tile processes. | [rocm/ds4_rocm_glm.cuh:2509](rocm/ds4_rocm_glm.cuh#L2509) | +| `DS4_ROCM_GLM_SELECTED_ATTN_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm glm selected attn profile. | [rocm/ds4_rocm_glm.cuh:2534](rocm/ds4_rocm_glm.cuh#L2534) | +| `DS4_ROCM_GLM_STREAMING_ASYNC_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm glm streaming async profile. | [ds4.c:44171](ds4.c#L44171) | +| `DS4_ROCM_GLM_STREAMING_DECODE_FULL_LAYER_MAP` | Pure presence flag: any defined value, including empty or "0", forces full mapping. Unset uses automatic mapping: resident layers map fully, eligible expert-cache layers use decode-only/expert mapping, otherwise full mapping. DS4_METAL_GLM_STREAMING_DECODE_FULL_LAYER_MAP and generic DS4_GLM_STREAMING_DECODE_FULL_LAYER_MAP are also accepted presence aliases. | Force every GLM SSD-streaming decode layer through full-layer mapping, bypassing the selected-expert/decode mapping optimization. | [ds4.c:42425](ds4.c#L42425) | +| `DS4_ROCM_GLM_STREAMING_DECODE_SYNC_EACH_LAYER` | ROCm-only primary value; nonempty takes priority over DS4_METAL_GLM_STREAMING_DECODE_SYNC_EACH_LAYER and the generic DS4_GLM_STREAMING_DECODE_SYNC_EACH_LAYER fallback; empty acts as unset; truthy unless exact 0 or case-insensitive false/off/no; with all aliases unset: false | For non-static GLM SSD decode on ROCm, opts into a full command/device synchronization after token mapping and every layer; default keeps ordered work queued across layer mappings, and static-map decode bypasses it. | [ds4.c:49589](ds4.c#L49589) | +| `DS4_ROCM_GLM_STREAMING_GROW_CACHE_AFTER_PREFILL` | Enabled by default when absent. If defined, only a truthy nonempty value enables; empty, "0", false, off, or no disable. Growth also requires SSD streaming plus nonzero base cache and prefill-headroom budgets, and occurs only if the recomputed expert count exceeds the current count. | After successful ROCm GLM prefill, add the released prefill headroom to the dynamic streaming expert-cache byte budget. | [ds4.c:50924](ds4.c#L50924) | +| `DS4_ROCM_GLM_STREAMING_PREFILL_FULL_LAYER` | presence force-on; any presence including empty or 0 enables; unset falls back to the Metal alias and then the automatic token threshold (1024 by default on ROCm); DS4_ROCM_DISABLE_GLM_STREAMING_PREFILL_FULL_LAYER dominates | Forces GLM SSD prefill into full-layer mapping/cache mode even below the automatic large-batch threshold. | [ds4.c:42523](ds4.c#L42523) | +| `DS4_ROCM_GLM_STREAMING_PREFILL_FULL_LAYER_MIN_TOKENS` | Positive uint32 threshold parsed with strtoul; ROCm default is 1024. Missing/empty, no leading number, errno/overflow, zero, or >UINT32_MAX returns 1024. The parser does not require end-of-string, so trailing junk after a valid leading number is accepted. A nonempty ROCm value takes precedence; otherwise DS4_METAL_GLM_STREAMING_PREFILL_FULL_LAYER_MIN_TOKENS is a fallback alias. | Set the automatic token-count crossover for ROCm GLM SSD prefill to load/use full resident expert layers when the layer supports that mode. | [ds4.c:42502](ds4.c#L42502) | +| `DS4_ROCM_GLM_STREAMING_PREFILL_SYNC_EACH_LAYER` | ROCm-only primary value; nonempty takes priority over DS4_METAL_GLM_STREAMING_PREFILL_SYNC_EACH_LAYER and the generic DS4_GLM_STREAMING_PREFILL_SYNC_EACH_LAYER fallback; empty acts as unset; truthy unless exact 0 or case-insensitive false/off/no; with all aliases unset: false for compact prefill; full-layer prefill always returns true | For compact GLM SSD prefill on ROCm, opts into a full command/device synchronization at every layer boundary; default preserves queued work across mappings, while full-layer cache mode always synchronizes. | [ds4.c:42205](ds4.c#L42205) | +| `DS4_ROCM_GLM_STREAMING_TOKEN_PREFILL_MAX` | primary nonempty value, then the Metal alias, then generic DS4_GLM_STREAMING_TOKEN_PREFILL_MAX; parsed by strtoul without requiring full-string consumption; 0 is valid and disables; no digits, ERANGE, or > UINT32_MAX uses the ROCm default 0 | Sets the largest non-quality GLM SSD-prefill chunk eligible for token-major/decode-style execution; ROCm defaults to canonical indexed batch prefill (0 disables token-major mode). | [ds4.c:49492](ds4.c#L49492) | +| `DS4_ROCM_GLM_VALUE_PROJECT_WAVE_DECODE` | Enabled by default when unset. Exact "0" or empty disables; every other nonempty value enables (including false/off/no). It applies only when n_tokens==1; =0 or multi-token input uses the generic per-head batch kernel. | Select the validated wave-per-output-row ROCm Q8 GLM value-projection kernel for one-token decode; =0 is the generic-kernel rollback. | [rocm/ds4_rocm_glm.cuh:2019](rocm/ds4_rocm_glm.cuh#L2019) | +| `DS4_ROCM_GRAPH_DUMP_LAYER` | unsigned layer or all; unset=all layers | Filter ROCm graph dumps by layer. | [ds4.c:16689](ds4.c#L16689) | +| `DS4_ROCM_GRAPH_DUMP_NAME` | nonempty substring filter; unset=all tensor names | Filter ROCm graph dumps by tensor/stage name. | [ds4.c:16685](ds4.c#L16685) | +| `DS4_ROCM_GRAPH_DUMP_NONINVASIVE` | truthy value under the shared parser; unset lets dumping select conservative kernels | Keep production ROCm kernel selection while graph dumping. | [rocm/ds4_rocm_runtime.cuh:4822](rocm/ds4_rocm_runtime.cuh#L4822) | +| `DS4_ROCM_GRAPH_DUMP_POS` | unsigned token position; unset=all positions | Filter ROCm graph dumps by position. | [ds4.c:16696](ds4.c#L16696) | +| `DS4_ROCM_GRAPH_DUMP_PREFIX` | nonempty output path prefix; unset=off | Enable ROCm intermediate graph/tensor dumps. | [ds4_cuda.cu:397](ds4_cuda.cu#L397) | +| `DS4_ROCM_GRAPH_DUMP_TRACE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Emit trace diagnostics for rocm graph dump trace. | [ds4.c:16726](ds4.c#L16726) | +| `DS4_ROCM_GRAPH_OUTPUT_ROW` | Nonempty string with a leading strtoul-parsable unsigned value < n_tokens selects that zero-based row. Default, empty, unparsable, or out-of-range selects n_tokens-1. Trailing characters are accepted because full consumption/errno are not checked. A nonempty ROCm value takes precedence; otherwise DS4_METAL_GRAPH_OUTPUT_ROW is a fallback alias. | Choose which prefill hidden-state row is sent through the output head to produce logits, primarily for graph/correctness diagnostics. | [ds4.c:35655](ds4.c#L35655) | +| `DS4_ROCM_GRAPH_PREFILL_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm graph prefill profile. | [ds4.c:31848](ds4.c#L31848) | +| `DS4_ROCM_GRAPH_PREFILL_SPLIT_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm graph prefill split profile. | [ds4.c:35578](ds4.c#L35578) | +| `DS4_ROCM_GRAPH_TOKEN_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm graph token profile. | [ds4.c:31532](ds4.c#L31532) | +| `DS4_ROCM_INDEXER_STAGE_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm indexer stage profile. | [ds4.c:29092](ds4.c#L29092) | +| `DS4_ROCM_LAYER_STAGE_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm layer stage profile. | [ds4.c:28931](ds4.c#L28931) | +| `DS4_ROCM_LAYER_STAGE_PROFILE_LAYER` | layer filter subordinate to DS4_ROCM_LAYER_STAGE_PROFILE; unset or whitespace-only: all layers allowed by the parent flag; otherwise the whitespace-trimmed value must be a complete base-10 strtoul result <= UINT32_MAX equal to the current layer; invalid values match none | Restricts the ROCm layer/prefill stage profiler to one layer; it does not enable profiling by itself. | [ds4.c:28932](ds4.c#L28932) | +| `DS4_ROCM_MOE_DECODE_DOWN_RPB` | sampled once; nonempty value is parsed by strtoul (a numeric prefix is sufficient), cast to uint32_t, and accepted only if 1/2/4/8/16/32; unset/empty/invalid inherits DS4_ROCM_MOE_DECODE_RPB, with defaults quality=8, non-quality SSD=2, resident=1 | Sets output rows (warps) per block for ROCm Q2_K routed-MoE decode down-projection kernels; threads per block are value * 32. | [rocm/ds4_rocm_runtime.cuh:4842](rocm/ds4_rocm_runtime.cuh#L4842) | +| `DS4_ROCM_MOE_DECODE_GATE_RPB` | sampled once; nonempty value is parsed by strtoul (a numeric prefix is sufficient), cast to uint32_t, and accepted only if 1/2/4/8/16/32; unset/empty/invalid defaults to 1 in non-quality SSD mode when DS4_ROCM_MOE_DECODE_RPB is unset/empty, otherwise inherits the resolved base RPB | Sets output rows (warps) per block for ROCm Q2_K routed-MoE decode gate/up kernels; threads per block are value * 32. | [rocm/ds4_rocm_runtime.cuh:4836](rocm/ds4_rocm_runtime.cuh#L4836) | +| `DS4_ROCM_MOE_DECODE_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm moe decode profile. | [rocm/ds4_rocm_moe_launch.cuh:82](rocm/ds4_rocm_moe_launch.cuh#L82) | +| `DS4_ROCM_MOE_DECODE_RPB` | sampled once; nonempty value is parsed by strtoul (a numeric prefix is sufficient), cast to uint32_t, and accepted only if 1/2/4/8/16/32; unset/empty/invalid default: quality=8, non-quality SSD=2, resident=1 | Sets the base ROCm Q2_K decode-MoE rows-per-block value inherited by gate/up and down controls, except the automatic SSD gate specialization defaults to 1 when this variable is unset/empty. | [rocm/ds4_rocm_runtime.cuh:4832](rocm/ds4_rocm_runtime.cuh#L4832) | +| `DS4_ROCM_MOE_WRITE_CLAMPED_ACT` | Pure presence sentinel: any defined value, including empty or "0", is active; DS4_METAL_MOE_WRITE_CLAMPED_ACT is an accepted fallback alias. On ROCm the variable is only consumed as a path-admission veto: it disables selected-expert cache/address-table, selected-slot and CPU-router/fused optimized paths. No ROCm call site parses a clamp amount or directly enables a write-clamped kernel. | Force shared graph selection away from optimizations incompatible with the clamped-intermediate MoE diagnostic; on ROCm this is a compatibility/rollback gate, not itself a clamped-write implementation. | [ds4.c:18353](ds4.c#L18353) | +| `DS4_ROCM_Q4_GROUPED_ATTN_A_STATS` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Print counters for rocm q4 grouped attn a stats. | [rocm/ds4_rocm_q4.cuh:344](rocm/ds4_rocm_q4.cuh#L344) | +| `DS4_ROCM_Q4_PREFILL_TILE8_STATS` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Print counters for rocm q4 prefill tile8 stats. | [rocm/ds4_rocm_q4.cuh:378](rocm/ds4_rocm_q4.cuh#L378) | +| `DS4_ROCM_Q8_DECODE_SHAREDX_64K` | sampled once; unset: enabled; present empty or exact 0: disabled; every other present value: enabled; effective only for one-token non-prequant Q8_0 matmul with 8192 < in_dim <= 16384 | Allows the ROCm shared-input Q8 decode kernel to use up to 64 KiB dynamic LDS for wide inputs; an unsupported/failed LDS launch automatically falls back to the regular kernel. | [rocm/ds4_rocm_runtime.cuh:4805](rocm/ds4_rocm_runtime.cuh#L4805) | +| `DS4_ROCM_Q_STAGE_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm q stage profile. | [ds4.c:29096](ds4.c#L29096) | +| `DS4_ROCM_REQUIRE_Q4_GROUPED_ATTN_A` | presence fail-closed assertion; also requests candidate unless disabled | Require rocm require q4 grouped attn a and fail instead of silently falling back. | [rocm/ds4_rocm_q4.cuh:575](rocm/ds4_rocm_q4.cuh#L575) | +| `DS4_ROCM_REQUIRE_Q4_PREFILL_TILE8` | presence fail-closed assertion for eligible TILE8 calls | Require rocm require q4 prefill tile8 and fail instead of silently falling back. | [rocm/ds4_rocm_q4.cuh:316](rocm/ds4_rocm_q4.cuh#L316) | +| `DS4_ROCM_STREAMING_DECODE_PREFILL_MAX` | primary nonempty value over the Metal alias; parsed by strtol when it has a numeric prefix (trailing text is accepted); <= 0 disables, values > UINT32_MAX clamp, no numeric prefix uses automatic default: 64 for Flash with uniform Q4_K/MXFP4 experts, 18 for other Pro/Flash, otherwise 0; the disable flag dominates | Sets the largest short, non-quality SSD-streaming prefill batch routed through the decode-style path instead of canonical layer-major prefill. | [ds4.c:31771](ds4.c#L31771) | +| `DS4_ROCM_STREAMING_EXPERT_AUTO_PRELOAD_CAP` | primary nonempty value over the Metal alias; strict full-string strtoul; valid values > UINT32_MAX clamp, invalid uses 4096, and 0 means no cap (not disabled); when CLI preload is auto/0, unset defaults to cap 4096 except ROCm GLM52, where absent/empty disables automatic preload entirely | Caps the number of hot experts synchronously seeded into the SSD-streaming expert cache in automatic preload mode; an explicit CLI preload count bypasses this cap, and setting this variable opts ROCm GLM52 back into auto preload. | [ds4.c:21281](ds4.c#L21281) | +| `DS4_ROCM_STREAMING_EXPERT_CACHE_VERBOSE` | presence flag; unset=off | Print verbose ROCm streaming expert-cache seed/load diagnostics. | [rocm/ds4_rocm_runtime.cuh:2904](rocm/ds4_rocm_runtime.cuh#L2904) | +| `DS4_ROCM_STREAMING_EXPERT_HOTLIST` | Nonempty filesystem path; a nonempty ROCm value takes precedence, otherwise DS4_METAL_STREAMING_EXPERT_HOTLIST is a fallback. The file contains whitespace-separated layer expert hits rows; blank/comment lines are ignored, zero-hit rows skipped, malformed/open/read errors fail seeding. Unset/empty uses the built-in Pro/Flash/GLM52 hotlist. Effective only when non-cold SSD hotlist seeding is enabled and cache/preload budget is nonzero. | Select a custom ranked expert hotlist used to preseed the streaming resident expert cache before decode. | [ds4.c:21321](ds4.c#L21321) | +| `DS4_ROCM_STREAMING_EXPERT_HOTLIST_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm streaming expert hotlist profile. | [ds4.c:32054](ds4.c#L32054) | +| `DS4_ROCM_STREAMING_MAP_TRACE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Emit trace diagnostics for rocm streaming map trace. | [ds4.c:42453](ds4.c#L42453) | +| `DS4_ROCM_STREAMING_PREFILL_BATCH_SELECTED_ADDR_MAX` | primary nonempty value over the Metal alias; strtol accepts a numeric prefix; <= 0 returns 0, > UINT32_MAX clamps, invalid uses ROCm default UINT32_MAX for Pro/Flash/GLM52 and 0 otherwise | Sets the inclusive upper token-count bound for automatically using selected-expert address-table kernels during eligible non-quality SSD batch prefill; 0 disables automatic selection. | [ds4.c:18298](ds4.c#L18298) | +| `DS4_ROCM_STREAMING_PREFILL_BATCH_SELECTED_ADDR_MIN` | primary nonempty value over the Metal alias; strtol accepts a numeric prefix; <= 0 returns 0, > UINT32_MAX clamps, invalid uses ROCm default 2 for Pro/Flash/GLM52 and 0 otherwise | Sets the inclusive lower token-count bound for automatically using selected-expert address-table kernels during eligible non-quality SSD batch prefill (the path independently requires more than one token). | [ds4.c:18321](ds4.c#L18321) | +| `DS4_ROCM_STREAMING_PREFILL_CACHE_SEED_K` | primary nonempty value over the Metal alias; strict full-string strtoul; unset/empty/invalid: 1; 0 disables; positive values clamp to 64; ignored unless SSD streaming and DS4_ROCM_ENABLE_STREAMING_PREFILL_CACHE_SEED (or Metal alias) is present | Chooses how many trailing token router selections per layer are captured from prefill and used to seed the streaming expert cache afterward. | [ds4.c:21094](ds4.c#L21094) | +| `DS4_ROCM_STREAMING_PREFILL_CACHE_SEED_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm streaming prefill cache seed profile. | [ds4.c:31963](ds4.c#L31963) | +| `DS4_ROCM_STREAMING_PREFILL_LAYER_MADVISE_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm streaming prefill layer madvise profile. | [ds4.c:19436](ds4.c#L19436) | +| `DS4_ROCM_STREAMING_PREFILL_LAYER_PAGEIN_NO_OVERLAP` | Pure presence flag: any defined value, including empty or "0", disables overlap. Default overlap is enabled only if this, PREPARE_NO_OVERLAP, DISABLE_*_PREPARE_OVERLAP, and DISABLE_*_PAGEIN_OVERLAP are all absent. The corresponding DS4_METAL name is an accepted fallback alias. In current code PAGEIN_NO_OVERLAP and PREPARE_NO_OVERLAP are exact synonyms. | Serialize SSD-streaming prefill layer page-in/preparation instead of overlapping preparation of upcoming layers. | [ds4.c:19143](ds4.c#L19143) | +| `DS4_ROCM_STREAMING_PREFILL_LAYER_PAGEIN_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm streaming prefill layer pagein profile. | [ds4.c:19432](ds4.c#L19432) | +| `DS4_ROCM_STREAMING_PREFILL_LAYER_PAGEIN_THREADS` | legacy fallback read only when DS4_ROCM_STREAMING_PREFILL_LAYER_PREPARE_THREADS and its Metal alias are absent/empty; strict full-string strtoul; unset/empty across both names: 8; invalid or 0: 1; values > 16 clamp to 16 | Sets worker count for full-layer SSD-prefill preparation (page touch, pread, readahead, or madvise) when the canonical PREPARE_THREADS control is not set. | [ds4.c:19101](ds4.c#L19101) | +| `DS4_ROCM_STREAMING_PREFILL_LAYER_PREAD_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm streaming prefill layer pread profile. | [ds4.c:19434](ds4.c#L19434) | +| `DS4_ROCM_STREAMING_PREFILL_LAYER_PREPARE_AHEAD` | primary nonempty value over the Metal alias; strict full-string strtoul; unset/empty: 1; invalid or 0: 1; values > 4 clamp to 4 | Sets how many future layer-preparation jobs may be queued concurrently while SSD-prefill preparation overlap is enabled. | [ds4.c:19155](ds4.c#L19155) | +| `DS4_ROCM_STREAMING_PREFILL_LAYER_PREPARE_NO_OVERLAP` | Pure presence flag: any defined value, including empty or "0", disables overlap. Default overlap is enabled only if this, PAGEIN_NO_OVERLAP, DISABLE_*_PREPARE_OVERLAP, and DISABLE_*_PAGEIN_OVERLAP are all absent. The corresponding DS4_METAL name is an accepted fallback alias. In current code PREPARE_NO_OVERLAP and PAGEIN_NO_OVERLAP are exact synonyms. | Serialize SSD-streaming prefill layer preparation/page-in instead of overlapping preparation of upcoming layers. | [ds4.c:19141](ds4.c#L19141) | +| `DS4_ROCM_STREAMING_PREFILL_LAYER_PREPARE_THREADS` | primary nonempty value over the Metal alias; strict full-string strtoul; unset/empty falls back to LAYER_PAGEIN_THREADS, then default 8; invalid or 0: 1; values > 16 clamp to 16 | Sets worker count used to split full-layer SSD-prefill page-touch, pread, readahead, or madvise preparation ranges. | [ds4.c:19097](ds4.c#L19097) | +| `DS4_ROCM_STREAMING_PREFILL_LAYER_READAHEAD_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm streaming prefill layer readahead profile. | [ds4.c:19438](ds4.c#L19438) | +| `DS4_ROCM_STREAMING_PREFILL_SELECTED_MADVISE_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm streaming prefill selected madvise profile. | [ds4.c:19182](ds4.c#L19182) | +| `DS4_ROCM_STREAMING_PREFILL_SELECTED_MADVISE_THREADS` | legacy fallback read only for selected-expert madvise preparation when DS4_ROCM_STREAMING_PREFILL_SELECTED_PREPARE_THREADS and its Metal alias are absent/empty; strict full-string strtoul; if all selected controls are unset it inherits layer preparation threads (default 8); invalid or 0: 1; values > 16 clamp to 16 | Sets worker count for selected-expert madvise preparation under its legacy name; non-madvise selected page-in always uses one worker. | [ds4.c:19119](ds4.c#L19119) | +| `DS4_ROCM_STREAMING_PREFILL_SELECTED_PAGEIN_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm streaming prefill selected pagein profile. | [ds4.c:19180](ds4.c#L19180) | +| `DS4_ROCM_STREAMING_PREFILL_SELECTED_PREPARE_GAP` | primary nonempty value over the Metal alias; strict full-string strtoul; unset/empty/invalid: 0; values > 8 clamp to 8 | For selected-expert madvise preparation, merges selected expert runs separated by at most this many unselected expert IDs, trading broader hints for fewer ranges. | [ds4.c:19131](ds4.c#L19131) | +| `DS4_ROCM_STREAMING_PREFILL_SELECTED_PREPARE_THREADS` | primary nonempty value over the Metal alias; strict full-string strtoul; for selected-expert madvise, unset/empty falls back to SELECTED_MADVISE_THREADS then layer preparation threads (default 8); invalid or 0: 1; values > 16 clamp to 16; non-madvise selected page-in ignores it and uses 1 | Sets worker count for selected-expert madvise preparation using the canonical control name. | [ds4.c:19115](ds4.c#L19115) | +| `DS4_ROCM_STREAMING_PREFILL_SELECTED_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm streaming prefill selected profile. | [ds4.c:18732](ds4.c#L18732) | +| `DS4_ROCM_STREAMING_PREFILL_SELECTED_READAHEAD_GAP` | primary nonempty value over the Metal alias; strict full-string strtoul; unset/empty/invalid: 0; values > 8 clamp to 8 | For selected-expert file readahead, merges selected expert runs separated by at most this many unselected expert IDs, reducing readahead calls at the cost of hinting extra weights. | [ds4.c:19804](ds4.c#L19804) | +| `DS4_ROCM_STREAMING_PREFILL_SELECTED_READAHEAD_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm streaming prefill selected readahead profile. | [ds4.c:19889](ds4.c#L19889) | +| `DS4_ROCM_STREAM_CACHE_LAYER_STATS` | presence flag; unset=off | Collect per-layer ROCm streaming cache statistics; also enables aggregate stats. | [rocm/ds4_rocm_runtime.cuh:390](rocm/ds4_rocm_runtime.cuh#L390) | +| `DS4_ROCM_STREAM_CACHE_STATS` | presence flag; unset=off unless layer stats are enabled | Collect aggregate ROCm streaming cache statistics. | [rocm/ds4_rocm_runtime.cuh:398](rocm/ds4_rocm_runtime.cuh#L398) | +| `DS4_ROCM_STREAM_EVICT_PAST_LAYERS_FIRST` | nonempty and not 0 enables; unset/empty/0=off | Prefer evicting cached experts from already-processed layers. | [rocm/ds4_rocm_runtime.cuh:406](rocm/ds4_rocm_runtime.cuh#L406) | +| `DS4_ROCM_STREAM_FREE_RESERVE_GB` | integer 2..64 GiB; default 16 | Reserve unified-memory headroom while growing the ROCm expert cache. | [rocm/ds4_rocm_runtime.cuh:1526](rocm/ds4_rocm_runtime.cuh#L1526) | +| `DS4_ROCM_STREAM_MODEL_CACHE_GB` | positive GiB integer; unset/invalid uses automatic streaming model cache limit | Cap cached streaming model spans. | [rocm/ds4_rocm_runtime.cuh:5464](rocm/ds4_rocm_runtime.cuh#L5464) | +| `DS4_ROCM_STREAM_NO_DIRECT` | nonempty and not 0 disables direct reads; unset/empty/0 keeps direct I/O eligible | Force the buffered ROCm SSD-streaming read path. | [rocm/ds4_rocm_runtime.cuh:1932](rocm/ds4_rocm_runtime.cuh#L1932) | +| `DS4_ROCM_STREAM_Q8_F16_CACHE_GB` | non-negative GiB integer; unset/invalid uses automatic Q8-F16 cache limit | Cap converted Q8-to-F16 weights in SSD mode. | [rocm/ds4_rocm_runtime.cuh:4859](rocm/ds4_rocm_runtime.cuh#L4859) | +| `DS4_ROCM_STREAM_READ_PROFILE` | nonempty and not 0 enables; unset/empty/0=off | Print ROCm SSD-streaming read/locality statistics at exit. | [rocm/ds4_rocm_runtime.cuh:1918](rocm/ds4_rocm_runtime.cuh#L1918) | +| `DS4_ROCM_STREAM_READ_WORKERS` | integer; default DS4_ROCM_STREAM_READ_DEFAULT_WORKERS; 0 coerces to 1; capped at compile-time max | Set parallel ROCm SSD read/upload workers. | [rocm/ds4_rocm_runtime.cuh:2077](rocm/ds4_rocm_runtime.cuh#L2077) | + +
+ +
+GLM shared (41) + +| Variable | Accepted value and default | Effect | Source | +| --- | --- | --- | --- | +| `DS4_GLM_ABLATE_COMBINE` | presence ablation; unset: exchange the local TP partial with the peer and add both halves; any presence including empty or 0 skips the exchange | Metal two-rank TP timing probe: doubles the local routed-MoE or split-attention partial instead of combining with the peer, deliberately producing invalid output; both ranks must set it or their exchange gates desynchronize. | [ds4.c:43991](ds4.c#L43991) | +| `DS4_GLM_ATTN_NO_LORA_VEC2` | presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it | Disable vectorized two-row LoRA accumulation in CUDA GLM indexed attention. | [ds4_cuda.cu:34176](ds4_cuda.cu#L34176) | +| `DS4_GLM_ATTN_NO_SCORE_VEC2` | presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it | Disable vectorized two-row score computation in CUDA GLM indexed attention. | [ds4_cuda.cu:34165](ds4_cuda.cu#L34165) | +| `DS4_GLM_ATTN_NO_STAGED_DECODE` | presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it | Disable staged CUDA GLM indexed-decode attention for large selected sets. | [ds4_cuda.cu:34214](ds4_cuda.cu#L34214) | +| `DS4_GLM_DECODE_ABLATE` | cached substring list; default empty mask; recognized tokens are attn_out, attn_core, qpath, indexer, routed, shared, qklow; unknown text has no effect; matching stages are skipped and output is invalid | Skip selected GLM decode stages for timing attribution; generated output is invalid. | [ds4.c:44238](ds4.c#L44238) | +| `DS4_GLM_DECODE_FLUSH_INTERVAL` | integer layers via atoi; default 4 for indexed decode and 32 otherwise; <=0/nonnumeric disables periodic flush; capped to layer count and forced to 0 for deferred completion | Set how often non-streaming GLM decode command work is flushed between layers. | [ds4.c:49628](ds4.c#L49628) | +| `DS4_GLM_DISABLE_FLASH_PREFILL` | presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it | Disable GLM Flash Attention prefill. | [ds4.c:44897](ds4.c#L44897) | +| `DS4_GLM_DISABLE_STREAMING_TOKEN_PREFILL` | presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it; either backend-specific ROCm/Metal alias also disables | Disable token-major GLM SSD-streaming prefill. | [ds4.c:49512](ds4.c#L49512) | +| `DS4_GLM_FENCE_TRACE` | presence flag; unset is off; any defined value, including empty or 0, is on; normal eligibility still applies | Log fenced CUDA tier switches used by GLM multi-GPU execution. | [ds4_cuda.cu:7984](ds4_cuda.cu#L7984) | +| `DS4_GLM_GEMM_TRACE` | presence flag; unset is off; any defined value, including empty or 0, is on; normal eligibility still applies | Measure and print CUDA GLM dequantization and cuBLAS GEMM timings. | [ds4_cuda.cu:19706](ds4_cuda.cu#L19706) | +| `DS4_GLM_HIDDEN_DUMP` | nonempty filesystem path/prefix; unset/empty disables; writes final hidden row or per-layer files selected by DS4_GLM_HIDDEN_DUMP_LAYER | Dump GLM hidden-state rows for correctness isolation. | [ds4.c:38333](ds4.c#L38333) | +| `DS4_GLM_HIDDEN_DUMP_LAYER` | selector; unset/empty = -1 (no per-layer dump, final hidden still dumped when path set); all = every layer; otherwise atoi result selects a layer, so invalid text selects layer 0 | Choose which GLM layer hidden states are dumped. | [ds4.c:38353](ds4.c#L38353) | +| `DS4_GLM_KV_DUMP` | nonempty filesystem prefix; unset/empty disables; writes layer-0 lora and rope compact-cache files after sync | Dump layer-0 compact GLM KV cache data after prompt synchronization. | [ds4.c:62941](ds4.c#L62941) | +| `DS4_GLM_LOGIT_DUMP` | nonempty filesystem path; unset/empty disables; dumps the first post-prefill logits vector once per process | Dump the first post-prefill GLM logits vector. | [ds4.c:38411](ds4.c#L38411) | +| `DS4_GLM_MEMORY_GUARD` | guard is on by default; exact 0 or case-insensitive false/off/no disables it; any other value and unset keep it enabled | Control the pre-allocation GLM host/GPU memory safety guard. | [ds4.c:41799](ds4.c#L41799) | +| `DS4_GLM_MEMORY_GUARD_FRACTION` | floating-point fraction; default 0.99; parsed numeric prefix is accepted, invalid/nonfinite falls back, values clamp to 0.50..1.00 | Set the fraction of detected memory usable by the GLM memory guard. | [ds4.c:41838](ds4.c#L41838) | +| `DS4_GLM_MEMORY_GUARD_REPORT` | nonempty diagnostic flag; unset/empty is off; any nonempty value including 0 prints successful-admission accounting (refusals always report) | Print successful GLM memory-guard budget accounting. | [ds4.c:41878](ds4.c#L41878) | +| `DS4_GLM_MEMORY_GUARD_RESERVE_GB` | floating-point GiB; dynamic default (normally 32, 24 on near-full 480..640 GiB hosts, possibly lower for resident ROCm slices); numeric prefixes accepted; invalid falls back; clamp 0..1024 | Set fixed headroom subtracted by the GLM memory guard. | [ds4.c:41856](ds4.c#L41856) | +| `DS4_GLM_MOE_EXPERT_MAJOR` | presence selector; unset: off; any presence including empty or 0 requests the path only for n_tokens >= 16; the automatic tile-8 path takes precedence when enabled (normally n_tokens >= 128) | CUDA only: groups selected token/expert pairs by expert and uses expert-major Q2_K routed-MoE gate/up/down kernels to reuse expert weights; otherwise the normal token-major path is used. | [ds4_cuda.cu:35973](ds4_cuda.cu#L35973) | +| `DS4_GLM_MOE_NO_DOWN_TILE8_EXACT` | presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it | Disable the exact tile-8 CUDA GLM routed-MoE down projection. | [ds4_cuda.cu:36028](ds4_cuda.cu#L36028) | +| `DS4_GLM_MOE_NO_EXPERT_TILE8` | presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it | Disable automatic expert tile-8 CUDA GLM routed-MoE batching. | [ds4_cuda.cu:35971](ds4_cuda.cu#L35971) | +| `DS4_GLM_MOE_NO_LOCAL_BATCH_IO` | presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it | Disable device-local batch scratch I/O for large CUDA GLM MoE batches. | [ds4_cuda.cu:35942](ds4_cuda.cu#L35942) | +| `DS4_GLM_MOE_SCALAR` | presence rollback; unset: use optimized warp kernels; any presence including empty or 0 selects scalar baseline kernels where an earlier expert-tile/expert-major path does not return; the special two-token MTP gate/up kernel still takes precedence | CUDA only: forces the baseline scalar Q2_K routed-MoE gate/up and down implementations for A/B or correctness testing (for two-token MTP, only the down half is forced). | [ds4_cuda.cu:36154](ds4_cuda.cu#L36154) | +| `DS4_GLM_MOE_SCRATCH_TIER0` | presence placement override; unset: allocate xq/midq quantization scratch on the current logical tier; any presence including empty or 0 allocates it on logical tier 0 | CUDA only: pins the routed-MoE xq_scratch and midq_scratch allocations to GPU tier 0 for multi-tier placement experiments; other MoE scratch remains on the current tier. | [ds4_cuda.cu:35918](ds4_cuda.cu#L35918) | +| `DS4_GLM_MTP_NO_ATTN_TOK2` | presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it | Disable the exact two-token CUDA GLM MTP attention kernel. | [ds4_cuda.cu:34168](ds4_cuda.cu#L34168) | +| `DS4_GLM_MTP_NO_MOE_TOK2` | presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it | Disable the exact two-token CUDA GLM MTP routed-MoE kernel. | [ds4_cuda.cu:36140](ds4_cuda.cu#L36140) | +| `DS4_GLM_MTP_NO_SHARED_TOK2` | presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it | Disable the exact two-token CUDA GLM MTP shared-FFN kernel. | [ds4_cuda.cu:37920](ds4_cuda.cu#L37920) | +| `DS4_GLM_MTP_PROBE` | presence flag; unset is off; any defined value, including empty or 0, is on; its second call site also rejects the normal batching path | Run the GLM next-N/MTP acceptance quality-and-timing probe without changing output and force probe-compatible scheduling. | [ds4.c:64769](ds4.c#L64769) | +| `DS4_GLM_PREFILL_TRUNC` | nonempty value parsed by atoi (leading whitespace/sign accepted and trailing junk ignored); effective only when the resulting int is > 0 and < the current prompt length; unset/empty or a result <= 0 or >= prompt length leaves the prompt unchanged | GPU GLM debug hook: truncates the prompt before prefill/checkpoint handling so dumped prefill logits can be aligned with a CPU first-token reference. | [ds4.c:63080](ds4.c#L63080) | +| `DS4_GLM_RESUME_PREFILL_MIN` | integer suffix tokens, non-ROCm builds only; default 4; parsed numeric prefixes accepted; <=0 maps to UINT32_MAX and effectively disables batched resume; ROCm build ignores it and stays at 4 | Set the suffix-length crossover from token decode to batched resumed prefill. | [ds4.c:38244](ds4.c#L38244) | +| `DS4_GLM_ROUTER_SCALAR` | presence rollback; unset: use the 256-thread parallel router when n_expert <= 256 (the scalar path is already automatic above 256); any presence including empty or 0 forces the scalar path | CUDA only: selects the one-active-thread-per-token sigmoid/top-k router kernel instead of the parallel shared-memory reduction, for A/B or correctness testing. | [ds4_cuda.cu:36390](ds4_cuda.cu#L36390) | +| `DS4_GLM_SHARED_SPLIT` | presence rollback; unset: use the fused one-token shared-expert Q8_0 gate+up+SwiGLU kernel when its shape/buffers are eligible; any presence including empty or 0 skips that fused one-token path; the earlier two-token MTP-specialized path is unaffected | CUDA only: forces shared-expert gate and up through two separate Q8_0 matmuls followed by a separate SwiGLU operation for one-token decode. | [ds4_cuda.cu:37946](ds4_cuda.cu#L37946) | +| `DS4_GLM_STREAMING_DECODE_FULL_LAYER_MAP` | presence compatibility alias; unset: automatic mapping; any presence including empty or 0 independently forces full-layer mapping, equivalent to the backend-specific DS4_ROCM_GLM_STREAMING_DECODE_FULL_LAYER_MAP or DS4_METAL_GLM_STREAMING_DECODE_FULL_LAYER_MAP control | Backend-neutral alias for supported GLM SSD streaming (Metal/ROCm): maps every tensor in each decode layer instead of using the decode-only map that can omit routed experts served by the expert cache; layers that already require a full map are unchanged. | [ds4.c:42427](ds4.c#L42427) | +| `DS4_GLM_STREAMING_DECODE_SYNC_EACH_LAYER` | ROCm-only third-priority legacy value: a nonempty DS4_ROCM_GLM_STREAMING_DECODE_SYNC_EACH_LAYER wins, otherwise a nonempty DS4_METAL_GLM_STREAMING_DECODE_SYNC_EACH_LAYER wins, otherwise this name is read; nonempty values are true except exact 0 or case-insensitive false/off/no; unset/empty: false; non-ROCm builds always return true and ignore this name | On ROCm non-static GLM SSD decode, opts into ending/synchronizing commands after token mapping and after every layer; the default keeps ordered work alive across layer mappings. Static-map decode bypasses this control. | [ds4.c:49591](ds4.c#L49591) | +| `DS4_GLM_STREAMING_PREFILL_SYNC_EACH_LAYER` | ROCm-only third-priority legacy value: a nonempty DS4_ROCM_GLM_STREAMING_PREFILL_SYNC_EACH_LAYER wins, otherwise a nonempty DS4_METAL_GLM_STREAMING_PREFILL_SYNC_EACH_LAYER wins, otherwise this name is read; nonempty values are true except exact 0 or case-insensitive false/off/no; unset/empty: false for compact prefill; full-layer prefill and non-ROCm builds always synchronize and ignore this name | On ROCm compact GLM SSD prefill, opts into ending/synchronizing commands at every layer boundary; the default carries ordered work across mappings, while full-layer expert-cache prefill always retains the boundary. | [ds4.c:42207](ds4.c#L42207) | +| `DS4_GLM_STREAMING_TOKEN_PREFILL_MAX` | unsigned token limit; backend-specific ROCm/Metal variable takes precedence, generic is fallback; default 0 on ROCm and 64 otherwise; invalid/overflow falls back, numeric prefixes accepted; 0 disables token-major streaming prefill | Set the largest SSD-streaming prefill handled by the token-major decode-like path. | [ds4.c:49494](ds4.c#L49494) | +| `DS4_GLM_SYNC_TRACE` | presence flag; unset is off; any defined value, including empty or 0, is on; normal eligibility still applies | Log GLM checkpoint/resume and dense-versus-indexed prefill decisions. | [ds4.c:63178](ds4.c#L63178) | +| `DS4_GLM_TP_DEBUG` | presence flag; unset is off; any defined value, including empty or 0, is on; normal eligibility still applies | Print CUDA/GLM tensor-parallel dispatch, gate, selected-ID, and failure diagnostics. | [ds4.c:43893](ds4.c#L43893) | +| `DS4_GLM_TP_EXACT_PREFILL_MAX` | integer suffix limit via atoi cast to uint32; default 64; nonnumeric becomes 0; negative values wrap to a very large unsigned limit | Set the maximum two-way TP suffix that uses exact token-by-token prefill. | [ds4.c:63120](ds4.c#L63120) | +| `DS4_GLM_TP_HEAD_SPLIT_MIN` | cached integer token threshold via atoi; default 64; negative values clamp to 0, nonnumeric becomes 0; 0 admits all otherwise-eligible batches | Set the minimum batch size for GLM tensor-parallel output-head splitting. | [ds4.c:38323](ds4.c#L38323) | +| `DS4_GLM_VALUE_NO_TILE16` | presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it | Disable the CUDA GLM 16-token tiled value-projection kernel. | [ds4_cuda.cu:36806](ds4_cuda.cu#L36806) | + +
+ +
+Distributed (15) + +| Variable | Accepted value and default | Effect | Source | +| --- | --- | --- | --- | +| `DS4_DIST_CONNECT_BIND_HOST` | non-empty string; unset/empty means no local bind constraint | Bind outgoing distributed connections to a local host/address. | [ds4_distributed.c:1332](ds4_distributed.c#L1332) | +| `DS4_DIST_CONNECT_BIND_IF` | non-empty string; unset/empty means no local bind constraint | Bind outgoing distributed connections to a network interface. | [ds4_distributed.c:1334](ds4_distributed.c#L1334) | +| `DS4_DIST_CONNECT_TRACE` | presence flag; unset=off, any set value=on | Emit trace diagnostics for dist connect trace. | [ds4_distributed.c:1112](ds4_distributed.c#L1112) | +| `DS4_DIST_DECODE_PROFILE` | presence flag; unset=off, any set value=on | Collect timing/profile diagnostics for dist decode profile. | [ds4_distributed.c:753](ds4_distributed.c#L753) | +| `DS4_DIST_DISABLE_PREFILL_ACK_ONLY` | presence flag; unset keeps optimized/default behavior, any set value disables it | Disable/roll back dist disable prefill ack only. | [ds4_distributed.c:3692](ds4_distributed.c#L3692) | +| `DS4_DIST_DISABLE_PREFILL_PIPELINE` | presence flag; unset keeps optimized/default behavior, any set value disables it | Disable/roll back dist disable prefill pipeline. | [ds4_distributed.c:3427](ds4_distributed.c#L3427) | +| `DS4_DIST_DISABLE_WORKER_PREFETCH` | presence flag; unset keeps optimized/default behavior, any set value disables it | Disable/roll back dist disable worker prefetch. | [ds4_distributed.c:7878](ds4_distributed.c#L7878) | +| `DS4_DIST_PREFILL_CHUNK` | positive integer; unset/0 uses session prefill capacity; explicit value may not exceed capacity | Set distributed prefill chunk size. | [ds4_distributed.c:3455](ds4_distributed.c#L3455) | +| `DS4_DIST_PREFILL_SEND_DEPTH` | integer 1..8; default 2; capped to chunk count | Set coordinator prefill sender queue depth. | [ds4_distributed.c:470](ds4_distributed.c#L470) | +| `DS4_DIST_PREFILL_WINDOW` | positive integer <=64; auto default remote stages+2 clamped 2..8 and chunk count | Set maximum distributed prefill chunks in flight. | [ds4_distributed.c:3485](ds4_distributed.c#L3485) | +| `DS4_DIST_SOCKET_BUFFER_MB` | integer 0..512 MiB; default 128; 0 disables socket buffer override | Set TCP send/receive buffer sizes. | [ds4_distributed.c:712](ds4_distributed.c#L712) | +| `DS4_DIST_SOCKET_RECV_TIMEOUT_SEC` | Nonempty base-10 integer parsed completely; valid range 1..3600 seconds. Unset, empty, partially parsed, or out-of-range values install no SO_RCVTIMEO at all. | Optionally bound blocking receives on distributed TCP sockets; the default deliberately permits indefinitely idle control connections during separate KV transfers. | [ds4_distributed.c:1049](ds4_distributed.c#L1049) | +| `DS4_DIST_SOCKET_TIMEOUT_SEC` | Nonempty base-10 integer parsed completely; valid range 1..3600 seconds. Default 60 seconds for unset, empty, partially parsed, or out-of-range values. | Set SO_SNDTIMEO on distributed TCP sockets so blocked coordinator/worker sends eventually fail. | [ds4_distributed.c:1032](ds4_distributed.c#L1032) | +| `DS4_DIST_WORKER_FORWARD_WINDOW` | integer 1..64; default 4 | Set worker forward-results window. | [ds4_distributed.c:740](ds4_distributed.c#L740) | +| `DS4_DIST_WORKER_PREFETCH_DEPTH` | integer 1..8; default 2 | Set worker input-prefetch queue depth. | [ds4_distributed.c:726](ds4_distributed.c#L726) | + +
+ +
+DSpark shared (24) + +| Variable | Accepted value and default | Effect | Source | +| --- | --- | --- | --- | +| `DS4_DSPARK_CACHE_RESERVE_GB` | integer GiB via atoi; default 4.5 GiB; values 1..32 replace it, all other values fall back; decimal/trailing text is truncated/accepted by atoi | Reserve VRAM on DSpark support-cache tiers before packing support-model tensors. | [ds4.c:59580](ds4.c#L59580) | +| `DS4_DSPARK_DISABLE_FINAL_OUTPUT_ALIAS` | presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it | Disable aliasing the final DSpark stage output to the next-stage buffer and use an explicit copy. | [ds4.c:33594](ds4.c#L33594) | +| `DS4_DSPARK_DISABLE_FUSED_CPU_MARKOV_ARGMAX` | cached value-aware kill switch; default off; nonempty value other than exact 0 disables; false/off also disable because only 0 is recognized as false | Disable the fused CPU Markov-bias plus argmax implementation. | [ds4.c:34297](ds4.c#L34297) | +| `DS4_DSPARK_DISABLE_REUSE_CONFIDENCE0_MARKOV` | cached value-aware kill switch; default off; nonempty value other than exact 0 disables; false/off also disable because only 0 is recognized as false | Disable reuse of the first confidence score during Markov proposal. | [ds4.c:34306](ds4.c#L34306) | +| `DS4_DSPARK_DISABLE_VERIFY_SELECTED_PROFILE` | presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it | Override and disable the selected-expert verifier profiler. | [ds4.c:36470](ds4.c#L36470) | +| `DS4_DSPARK_EXEC_TIER` | integer tier via atoi; default is placement/TP free-VRAM heuristic; valid 0..n_gpus-1 overrides; invalid numeric range falls back, but nonnumeric text becomes tier 0 | Choose the GPU tier that executes and primarily caches the DSpark support model. | [ds4.c:59553](ds4.c#L59553) | +| `DS4_DSPARK_FAKE_ARGMAX_PROPOSAL` | nonempty boolean; unset/empty or exact 0: off; every other nonempty value enables, but only while DSpark itself is enabled | If the real DSpark proposer produced no draft, installs a one-token fallback proposal equal to the argmax of the current target logits; debug/test mode also selects the non-fused stage-0 setup path. | [ds4.c:64088](ds4.c#L64088) | +| `DS4_DSPARK_LOW_MEMORY_PREFILL_CHUNK` | unsigned integer rows; default 128; 0 disables the low-memory policy; invalid/overflow falls back, numeric prefixes are accepted; only consulted for Metal SSD+DSpark on <=24 GiB hosts without an explicit chunk | Set the automatic low-memory Metal prefill chunk for SSD-streamed DSpark. | [ds4.c:60533](ds4.c#L60533) | +| `DS4_DSPARK_NO_GPU_MARKOV` | presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it | Disable GPU Markov bias/argmax and the fully device-resident proposal path. | [ds4.c:34521](ds4.c#L34521) | +| `DS4_DSPARK_NO_MARKOV` | cached value-aware kill switch; default off; nonempty value other than exact 0 disables Markov bias; false/off also disable | Disable Markov bias in DSpark proposal generation. | [ds4.c:34288](ds4.c#L34288) | +| `DS4_DSPARK_PROBE` | nonempty-string diagnostic; unset/empty is off, any nonempty value including 0 is on | Log DSpark proposal/probe diagnostics. | [ds4.c:64085](ds4.c#L64085) | +| `DS4_DSPARK_PROP_PROFILE` | presence flag; unset is off; any defined value, including empty or 0, is on; normal eligibility still applies | Print fine-grained timings for DSpark proposal setup. | [ds4.c:32771](ds4.c#L32771) | +| `DS4_DSPARK_SPEC_LOG` | presence flag; unset is off; any defined value, including empty or 0, is on; normal eligibility still applies | Log speculative proposal, verification, acceptance, and fallback decisions. | [ds4.c:66406](ds4.c#L66406) | +| `DS4_DSPARK_SSD_VERIFY_BLOCK_MAX` | unsigned integer rows; default/fallback 0 means automatic policy; numeric prefixes accepted; used both as verifier cap and as an exact-2 proposer-policy discriminator | Cap speculative rows verified from SSD and influence exact-2 proposal sizing. | [ds4.c:52190](ds4.c#L52190) | +| `DS4_DSPARK_STAGE_PROFILE` | presence flag; unset is off; any defined value, including empty or 0, is on; DS4_DSPARK_STAGE_PROFILE_STAGE must also match | Profile DSpark support stages with command-boundary timings. | [ds4.c:33226](ds4.c#L33226) | +| `DS4_DSPARK_STAGE_PROFILE_STAGE` | selector subordinate to DS4_DSPARK_STAGE_PROFILE; unset/empty: match every stage; otherwise strtoul base 10 must consume the whole value, fit uint32_t, and equal the current stage; invalid/out-of-range values match no stage | Restricts DSpark stage-boundary timing output to one stage; it does not enable profiling by itself. | [ds4.c:33227](ds4.c#L33227) | +| `DS4_DSPARK_STATS` | value-aware flag; default off; nonempty value other than exact 0 enables; false/off are treated as enabled | Collect and print aggregate DSpark runtime statistics. | [ds4.c:61417](ds4.c#L61417) | +| `DS4_DSPARK_VERIFY_CACHE` | presence diagnostic; unset: off; any presence including empty or 0 enables on each support-cache installation | CUDA only: copies every installed nonempty DSpark/support-cache range back to the host, byte-compares it with its source, and logs each mismatch plus a bad-count summary without changing the install result. | [ds4_cuda.cu:8257](ds4_cuda.cu#L8257) | +| `DS4_DSPARK_VERIFY_HEAD_NO_TP` | presence rollback; unset: allow eligible CUDA output tensor parallelism; any presence including empty or 0 removes the TP path from eligibility | CUDA only: forces the DSpark speculative batched vocabulary head away from output-TP for correctness isolation; under CUDA TP+EP the attempt fails instead of using unavailable full output weights. | [ds4.c:26468](ds4.c#L26468) | +| `DS4_DSPARK_VERIFY_NONCAUSAL` | presence diagnostic sampled once after the first successfully submitted CUDA noncausal-attention kernel; unset: verify 0 calls; any presence including empty or 0: verify that call and the next 2 | CUDA only: synchronizes and reads back Q/KV/output, computes the DSpark noncausal attention CPU reference, and logs max absolute/relative error; it reports only and does not fail the operation. | [ds4_cuda.cu:21736](ds4_cuda.cu#L21736) | +| `DS4_DSPARK_VERIFY_PROFILE` | cached presence diagnostic; unset is off; any defined value including empty/0 profiles only the first eligible verifier invocation | Profile one full DSpark target-verifier invocation layer by layer. | [ds4.c:36567](ds4.c#L36567) | +| `DS4_DSPARK_VERIFY_SELECTED_PROFILE` | presence flag; unset is off; any defined value, including empty or 0, enables unless DS4_DSPARK_DISABLE_VERIFY_SELECTED_PROFILE is also present (disable wins) | Profile selected-expert streaming inside the DSpark verifier. | [ds4.c:36469](ds4.c#L36469) | +| `DS4_DSPARK_VERIFY_SPLIT_HEAD` | nonempty boolean with inverted default; unset/empty or exact 0: fused head; every other nonempty value: split head | Runs the DSpark suffix verifier output head and top-1 reduction in a separate GPU command section after the layer loop, for timing/correctness isolation; default keeps them fused into the layer command section. | [ds4.c:36535](ds4.c#L36535) | +| `DS4_DSPARK_VERIFY_TOPS_CHECK` | presence flag; unset is off; any defined value, including empty or 0, is on; normal eligibility still applies | Read back verifier logits and compare GPU top IDs with CPU argmax. | [ds4.c:36683](ds4.c#L36683) | + +
+ +
+General and shared (76) + +| Variable | Accepted value and default | Effect | Source | +| --- | --- | --- | --- | +| `DS4_BATCHED_FFN` | Pure presence flag: any defined value, including empty or "0", enables. Unset leaves the default shared-expert-batched FFN path (or its configured fallback). It is read only by CPU layer-major prefill and takes precedence over shared-batch and token-parallel FFN choices. | Run the complete CPU prefill FFN in chunks through layer_ffn_batch instead of the default shared-expert-only batched path. | [ds4.c:14238](ds4.c#L14238) | +| `DS4_BATCHED_ROPE_MAX` | Nonempty value parsed by strtol without full-string validation; integers 0..65536 are accepted, otherwise default 4096. Zero disables batched RoPE for every nonempty prompt. Effective only when prefix batch attention is selected and DS4_NO_BATCHED_ROPE is absent. | Set the largest CPU prefix-prefill token batch that applies RoPE and inverse RoPE with the batched kernels. | [ds4.c:13738](ds4.c#L13738) | +| `DS4_BENCH_DISABLE_SNAPSHOT` | presence flag; unset allows snapshots for eligible frontiers | Disable benchmark state snapshots. | [ds4_bench.c:709](ds4_bench.c#L709) | +| `DS4_BENCH_FORCE_SNAPSHOT` | presence flag; unset obeys the normal size/eligibility checks | Force benchmark state snapshots despite the normal limit. | [ds4_bench.c:712](ds4_bench.c#L712) | +| `DS4_BENCH_SNAPSHOT_MAX_BYTES` | unsigned bytes or unlimited/inf; default DS4_BENCH_DEFAULT_SNAPSHOT_MAX_BYTES | Limit session snapshot size during benchmark sweeps. | [ds4_bench.c:70](ds4_bench.c#L70) | +| `DS4_CHROME` | executable path; unset auto-detects Chrome/Chromium via standard paths and PATH | Select the browser executable used by web tooling. | [ds4_web.c:1014](ds4_web.c#L1014) | +| `DS4_CLI_FORCE_SESSION` | Pure presence flag: any defined value, including empty or "0", forces the session path. Unset uses the session path only for distributed coordinators, TP leaders, temperature>0, or MTP depth>1; otherwise the CLI calls direct argmax generation. | Force ordinary CLI generation through run_sampled_generation/session APIs so single-node validation follows the same stateful path as TP/distributed runs. | [ds4_cli.c:1226](ds4_cli.c#L1226) | +| `DS4_CPU_DISABLE_UNROLLED_ARGMAX` | presence rollback flag; unset keeps optimized/default path | Disable/roll back cpu disable unrolled argmax. | [ds4.c:40885](ds4.c#L40885) | +| `DS4_CPU_DUMP_LOGITS` | filesystem path; unset disables read/write | Dump or select diagnostic data for cpu dump logits. | [ds4.c:38896](ds4.c#L38896) | +| `DS4_CPU_DUMP_PREFILL_LOGITS` | filesystem path; unset disables read/write | Dump or select diagnostic data for cpu dump prefill logits. | [ds4.c:41416](ds4.c#L41416) | +| `DS4_DECODE_PROFILE_DETAIL` | presence flag; unset=off | Print per-stage timing for the single-token CPU FFN path. | [ds4.c:12036](ds4.c#L12036) | +| `DS4_EXPERT_HOTLIST` | nonempty filesystem path; unset=off; currently Metal-only | Load an expert hotlist for Metal expert profiling/streaming. | [ds4.c:60179](ds4.c#L60179) | +| `DS4_EXPERT_PROFILE` | presence diagnostic flag; unset=off | Collect timing/profile diagnostics for expert profile. | [ds4.c:60177](ds4.c#L60177) | +| `DS4_FORCE_CUDA_PEER` | presence flag read once at CUDA init; unset uses automatic transfer selection; any defined value including 0 enables | Force cross-device transfers through cudaMemcpyPeerAsync for diagnostics. | [ds4_cuda.cu:364](ds4_cuda.cu#L364) | +| `DS4_FORCE_HOST_BOUNCE` | presence flag read once at CUDA init; unset uses automatic transfer selection; any defined value including 0 enables | Force cross-device transfers through pinned host bounce buffers for diagnostics. | [ds4_cuda.cu:365](ds4_cuda.cu#L365) | +| `DS4_LOCK_FILE` | path string; default /tmp/ds4.lock | Override the single-instance lock file. | [ds4.c:51809](ds4.c#L51809) | +| `DS4_MMID_CASE1` | boolean-ish cached flag; default on; a value starting with 0 disables | Disable the single-expert MM-IDs specialized fast path for comparison. | [cuda/mmq/mmid.cu:290](cuda/mmq/mmid.cu#L290) | +| `DS4_MMID_LARGE` | boolean-ish cached flag; default on; a value starting with 0 disables | Control the large-N global-memory MM-IDs path used beyond shared-memory capacity. | [cuda/mmq/mmid.cu:245](cuda/mmq/mmid.cu#L245) | +| `DS4_MMQ_D2R` | boolean-ish cached flag; default on; a value starting with 0 disables | Control the direct-to-register Q2_K MoE down path. | [cuda/mmq/ds4_mmq.cu:505](cuda/mmq/ds4_mmq.cu#L505) | +| `DS4_MMQ_D2R_IQ2` | boolean-ish cached flag; default on; a value starting with 0 disables | Control the direct-to-register IQ2 MoE gate/up path. | [cuda/mmq/ds4_mmq.cu:514](cuda/mmq/ds4_mmq.cu#L514) | +| `DS4_MMQ_D2R_MIN_COLS` | positive integer; default 1024; invalid or nonpositive input restores the default | Set the minimum output-column count for the MMQ direct-to-register path. | [cuda/mmq/ds4_mmq.cu:609](cuda/mmq/ds4_mmq.cu#L609) | +| `DS4_MMQ_D2R_STATS` | exact 1 enables; unset or every other value disables; cached and synchronizes the stream | Print partial-tile fill telemetry for the direct-to-register MMQ kernels. | [cuda/mmq/ds4_mmq_d2r.cu:33](cuda/mmq/ds4_mmq_d2r.cu#L33) | +| `DS4_MMQ_DENSE_D2R` | boolean-ish flag; default on; exact 0 disables | Control the eligible aligned-Q8 dense prefill direct-to-register path. | [ds4_cuda.cu:19567](ds4_cuda.cu#L19567) | +| `DS4_MMQ_NO_YIND` | presence rollback; unset keeps Y-indirect staging; any defined value including 0 disables it | Restore slot-gathered MoE gate/up activation quantization. | [cuda/mmq/ds4_mmq.cu:589](cuda/mmq/ds4_mmq.cu#L589) | +| `DS4_MMQ_OUT_MEMSET` | exact 1 enables; unset or every other value disables; cached | Restore blanket MMQ output-buffer zeroing for diagnostics. | [cuda/mmq/ds4_mmq.cu:532](cuda/mmq/ds4_mmq.cu#L532) | +| `DS4_MMQ_YBUF_MEMSET` | unset or 0 disables; 1 zero-fills; a value starting with p or P poison-fills with 0xFF | Control MMQ Q8_1 activation-staging initialization and its poison oracle. | [cuda/mmq/ds4_mmq.cu:558](cuda/mmq/ds4_mmq.cu#L558) | +| `DS4_MMQ_YIND_VERIFY` | presence diagnostic; unset is off; any defined value including 0 enables | Byte-compare Y-indirect and slot-gathered MoE activation buffers. | [cuda/mmq/ds4_mmq.cu:600](cuda/mmq/ds4_mmq.cu#L600) | +| `DS4_MOE_RECORD_SELECTED_HOTLIST` | nonempty output path; unset=off | Record per-layer selected-expert hit counts to a Metal hotlist file. | [ds4_metal.m:1740](ds4_metal.m#L1740) | +| `DS4_MOE_RECORD_SELECTED_HOTLIST_FRESH` | presence flag; only relevant with HOTLIST; overrides MERGE | Start the selected-expert hotlist from empty state. | [ds4_metal.m:1641](ds4_metal.m#L1641) | +| `DS4_MOE_RECORD_SELECTED_HOTLIST_MERGE` | presence flag; active only when FRESH is absent | Merge an existing selected-expert hotlist before recording. | [ds4_metal.m:1640](ds4_metal.m#L1640) | +| `DS4_MOE_RECORD_SELECTED_IDS` | nonempty output path; unset=off | Record routed-MoE six-expert selections; also disables incompatible optimized paths. | [ds4.c:65100](ds4.c#L65100) | +| `DS4_MOE_REPLAY_SELECTED_IDS` | nonempty input path; unset=off | Replay routed-MoE six-expert selections; also disables incompatible optimized paths. | [ds4.c:25135](ds4.c#L25135) | +| `DS4_MTP_BATCH_VERIFY` | Pure presence flag: any defined value, including empty or "0", suppresses the exact two-row decode verifier. Unset selects exact decode-2 when draft_n==2 and either strict mode is active or the build is ROCm; other cases already use the generic verifier. | Diagnostic rollback from the exact Q8/one-token-equivalent MTP decode-2 verifier to the generic microbatch verifier. | [ds4.c:70842](ds4.c#L70842) | +| `DS4_MTP_CAPTURE_PREFIX1` | Pure presence flag. In the generic verifier with exactly two drafts it enables prefix-1 state capture under strict mode; non-strict mode already captures prefix-1 without the variable. Unset under strict mode instead snapshots and replays a partial acceptance. | Let a one-of-two MTP partial acceptance commit the verifier's captured prefix directly, avoiding an exact one-token replay. | [ds4.c:70948](ds4.c#L70948) | +| `DS4_MTP_CONF_LOG` | Pure presence flag; default off. It forces materialization of full draft logits, computes the top-2 margin, and after a successful generic microbatch verification prints drafted/committed counts, top candidates, margin, target-next and draft-next. Exact decode-2 success does not emit that generic log line. | Inspect MTP draft confidence and compare the recursive draft token with the target verifier result. | [ds4.c:70710](ds4.c#L70710) | +| `DS4_MTP_EXACT_REPLAY` | Pure presence flag; default off. In the generic microbatch verifier it forces a pre-verifier frontier snapshot; after verification the snapshot is restored and every accepted draft is decoded sequentially to rebuild exact final state/logits. | Validate MTP acceptance while committing through the normal one-token decode path rather than retaining batched-verifier state. | [ds4.c:70949](ds4.c#L70949) | +| `DS4_MTP_FORCE_SNAPSHOT` | Pure presence flag; default off. It forces a speculative-frontier snapshot before the generic verifier regardless of draft count or prefix-capture mode; it does not by itself force restoration or replay after a successful full acceptance. | Measure/debug snapshot behavior and guarantee a restorable pre-verifier frontier for generic MTP verification. | [ds4.c:70953](ds4.c#L70953) | +| `DS4_MTP_FULL_LOGITS` | Pure presence flag; default off. When set, legacy and recursive MTP draft calls write the full vocabulary logits to s->mtp_logits; unset permits the faster top-token-only output unless confidence/margin logic independently needs logits. | Force full MTP draft-logit materialization for correctness comparison, inspection, or downstream confidence calculations. | [ds4.c:64070](ds4.c#L64070) | +| `DS4_MTP_MIN_MARGIN` | non-negative float; default engine --mtp-margin value | Set confidence margin threshold for speculative MTP verification. | [ds4.c:70703](ds4.c#L70703) | +| `DS4_MTP_PROBE` | Pure presence flag; default off. For legacy MTP it prepares drafts even when configured depth<=1, compares the previous draft with the next committed token, and prints cumulative hit counts/failures; generated output is unchanged. | Measure legacy MTP next-token draft accuracy without enabling speculative acceptance. | [ds4.c:64800](ds4.c#L64800) | +| `DS4_MTP_SPEC_DISABLE` | Pure presence flag: any defined value, including empty or "0", disables MTP speculative argmax in CLI/chat/server loops. Unset permits it for greedy temperature<=0 generation with draft depth>1; unrelated split-KV speculation can still be independently requested. | Fall back from MTP multi-token speculative evaluation to normal one-token session evaluation. | [ds4_cli.c:580](ds4_cli.c#L580) | +| `DS4_MTP_SPEC_LOG` | Pure presence flag; default off. It only emits diagnostics for first-draft misses, exact/generic verifier failures and sequential fallback misses/acceptance outcomes; it does not select a verifier. | Trace why MTP drafts were accepted, partially accepted, rejected, or sent to sequential fallback. | [ds4.c:70726](ds4.c#L70726) | +| `DS4_MTP_STRICT` | Pure presence flag; engine quality mode also enables strictness automatically. Strict mode skips the non-strict low-margin shortcut, selects exact decode-2 for two drafts unless DS4_MTP_BATCH_VERIFY is set, and disables default prefix-1 capture unless explicitly restored. | Force the exact/quality-oriented MTP verification policy on otherwise non-quality runs. | [ds4.c:70701](ds4.c#L70701) | +| `DS4_MTP_TIMING` | Pure presence flag; default off. When set, timestamps and prints draft, snapshot, verifier, prefix/replay and total durations for the path taken; algorithm selection is otherwise unchanged. | Profile end-to-end MTP speculative decoding and separate draft, verification and state-commit costs. | [ds4.c:70709](ds4.c#L70709) | +| `DS4_NO_BATCHED_ATTN` | presence rollback flag; unset keeps default/optimized path | Disable/roll back no batched attn. | [ds4.c:14237](ds4.c#L14237) | +| `DS4_NO_BATCHED_ROPE` | presence rollback flag; unset keeps default/optimized path | Disable/roll back no batched rope. | [ds4.c:13745](ds4.c#L13745) | +| `DS4_NO_PARALLEL_ATTN_ROWS` | presence rollback flag; unset keeps default/optimized path | Disable/roll back no parallel attn rows. | [ds4.c:13731](ds4.c#L13731) | +| `DS4_NO_ROUTED_TOKEN_PARALLEL` | presence rollback flag; unset keeps default/optimized path | Disable/roll back no routed token parallel. | [ds4.c:12415](ds4.c#L12415) | +| `DS4_NO_SHARED_BATCH_FFN` | presence rollback flag; unset keeps default/optimized path | Disable/roll back no shared batch ffn. | [ds4.c:14240](ds4.c#L14240) | +| `DS4_ORACLE_LOGITS` | filesystem path; unset disables read/write | Load reference logits for graph correctness comparison. | [ds4.c:38866](ds4.c#L38866) | +| `DS4_PARALLEL_ATTN_ROWS` | Pure presence opt-in; any defined value enables the preference, but DS4_NO_PARALLEL_ATTN_ROWS overrides it. The path is eligible only for prefix prefill with cache n_raw==0 and pos0==0; unset uses per-token attention rows. | Batch/parallelize CPU prefix attention-row evaluation after cache/index preparation. | [ds4.c:13728](ds4.c#L13728) | +| `DS4_PARALLEL_FFN` | Pure presence opt-in. It is effective only in CPU prefill when batched attention is enabled, DS4_BATCHED_FFN is absent, and the default shared-batch path has been disabled with DS4_NO_SHARED_BATCH_FFN; otherwise higher-priority paths win. | Run independent prompt-token FFNs through layer_ffn_tokens_parallel as the fallback after disabling shared batching. | [ds4.c:14239](ds4.c#L14239) | +| `DS4_PREFILL_BATCH` | Nonempty value parsed by strtol without full-string validation; accepted range 1..4095, default 128 for unset/invalid/out-of-range values. It is used only when DS4_BATCHED_FFN selects full batched CPU FFN. | Set the token chunk size for layer_ffn_batch during CPU layer-major prefill. | [ds4.c:14241](ds4.c#L14241) | +| `DS4_PREFILL_PROFILE_DETAIL` | presence flag; unset=off | Print detailed per-stage CPU prefill timing. | [ds4.c:12382](ds4.c#L12382) | +| `DS4_PREFILL_PROFILE_TOKEN` | presence flag; effective within detailed prefill profiling | Print token-loop substage timings during CPU prefill. | [ds4.c:13957](ds4.c#L13957) | +| `DS4_Q8_FOLD_SELFTEST` | positive call budget; unset/empty disables; a nonempty value parsing to 1 or less selects 512 calls | Byte-check folded Q8_1 activations against a fresh quantization; synchronizes eager streams. | [cuda/mmq/ds4_mmq.cu:5744](cuda/mmq/ds4_mmq.cu#L5744) | +| `DS4_ROUTED_TOKEN_PARALLEL` | Pure presence flag that forces token-parallel routed MoE, even if DS4_NO_ROUTED_TOKEN_PARALLEL is also set. When unset, token parallelism is automatic for n_tok>=64 unless the NO flag is present; smaller batches use per-token routed MoE. | Choose token-parallel CPU routed-expert evaluation inside the default shared-batch FFN prefill path. | [ds4.c:12414](ds4.c#L12414) | +| `DS4_SERVER_BATCH_LOG` | Pure presence flag read once when the decode worker starts; default off. Any defined value, including empty or "0", logs one record per coalesced decode batch with count, elapsed milliseconds and ok/error status. | Observe server-side decode coalescing size, latency and result without changing batching behavior. | [ds4_server.c:11090](ds4_server.c#L11090) | +| `DS4_SERVER_DECODE_COALESCE_US` | integer 0..100000 microseconds; default 2000; 0 disables wait | Control server micro-batch coalescing delay. | [ds4_server.c:11069](ds4_server.c#L11069) | +| `DS4_SSD_AUTO_CACHE_PCT` | integer 50..95; default 80 | Choose the RAM percentage used by automatic SSD expert-cache planning. | [ds4_ssd.c:81](ds4_ssd.c#L81) | +| `DS4_TEST_METAL_EXACTN_ORACLE` | presence flag compiled only with DS4_TEST_HOOKS; unset is off; any defined value enables | Force allocation of the Metal exact-N verifier/oracle workspace in test builds. | [ds4.c:61801](ds4.c#L61801) | +| `DS4_THREADS` | positive integer; default min(online CPUs,12), capped by DS4_MAX_THREADS; CLI thread request overrides env | Set CPU worker-pool size. | [ds4.c:1874](ds4.c#L1874) | +| `DS4_TOKEN_TIMING` | Pure presence flag; default off. Any defined value times and prints each CPU token decode evaluation; sampling, emission and callbacks are outside the measured interval. | Report per-token CPU model-evaluation latency during direct argmax generation. | [ds4.c:41429](ds4.c#L41429) | +| `DS4_TP_ABLATE` | comma/list string matched for hcpre,router,kv,compidx; unset=no ablation; must match on both ranks | Skip named TP encode chains for timing; output is semantically wrong. | [ds4.c:22355](ds4.c#L22355) | +| `DS4_TP_EVENT_GATES` | presence flag; unset uses lower-latency slab flag gates when available | Fall back to Metal shared-event arrival gates. | [ds4_metal.m:10769](ds4_metal.m#L10769) | +| `DS4_TP_GATE_PROFILE` | presence diagnostic flag; unset=off | Collect timing/profile diagnostics for tp gate profile. | [ds4_metal.m:10661](ds4_metal.m#L10661) | +| `DS4_TP_GATE_TRACE` | presence diagnostic flag; unset=off | Emit trace diagnostics for tp gate trace. | [ds4_tp.c:911](ds4_tp.c#L911) | +| `DS4_TP_KEEPALIVE_ITERS` | atoi unsigned iteration count; default 1200000 | Tune work per Metal TP keep-alive dispatch. | [ds4_metal.m:10625](ds4_metal.m#L10625) | +| `DS4_TP_KEEPALIVE_TGS` | integer 1..2048; invalid/out of range uses 1 | Tune threadgroups per Metal TP keep-alive dispatch. | [ds4_metal.m:10611](ds4_metal.m#L10611) | +| `DS4_TP_NO_KEEPALIVE` | presence flag; unset starts Metal TP keep-alive | Disable the Metal TP GPU keep-alive worker. | [ds4_metal.m:10797](ds4_metal.m#L10797) | +| `DS4_TP_PREFILL_SPLIT_MIN` | atoi token threshold; default 32; values below 2 clamp to 2 | Set when TP prefill row-splits the replicated shared expert. | [ds4.c:29019](ds4.c#L29019) | +| `DS4_TP_SUBGATE_PIPELINE` | nonempty integer; nonzero enables; default off; must match on both ranks | Enable TP prefill sub-chunk gate pipelining. | [ds4.c:29033](ds4.c#L29033) | +| `DS4_TP_TIMEOUT_SEC` | atoi seconds stored unsigned; default DS4_TP_DEFAULT_TIMEOUT_SEC | Override TP control/data socket operation timeout. | [ds4_tp.c:1329](ds4_tp.c#L1329) | +| `DS4_TRACE_TOP` | presence flag; unset=off | Print top-logit/token trace data during CPU generation. | [ds4.c:41398](ds4.c#L41398) | +| `DS4_WS_REPACK_HASH` | exact 1 enables; unset or every other value disables unless overridden by CLI; cached | Print a per-artifact FNV-1a hash for workspace repack identity checks. | [cuda/mmq/ds4_repack.cu:530](cuda/mmq/ds4_repack.cu#L530) | +| `DS4_WS_REPACK_THREADS` | positive integer; default min(6, hardware threads), capped at 16 and the job count | Set the CPU worker count for CUDA workspace artifact repacking. | [cuda/mmq/ds4_repack.cu:539](cuda/mmq/ds4_repack.cu#L539) | + +
+ +### External runtime environment + +These names are not owned by the `DS4_*` namespace but are read directly by +the binaries or vendored runtime code. + +| Variable | Accepted value and default | Effect | Source | +| --- | --- | --- | --- | +| `GGML_CUDA_DISABLE_GRAPHS` | Pure presence flag cached on first is_enabled() call; any defined value, including empty or "0", disables. Unset permits CUDA graphs unless the GPU architecture independently disables them; relevant only when compiled with USE_CUDA_GRAPH. | Disable vendored GGML CUDA graph capture/replay and execute through the ordinary CUDA path. | [cuda/mmq/common.cuh:1208](cuda/mmq/common.cuh#L1208) | +| `HOME` | Filesystem directory string. In ds4-agent, unset or empty falls back to "." for the default cache and history roots; the web helper applies the same fallback for its browser profile. A nonempty value roots .ds4/kvcache, .ds4_agent_history and .ds4/browser. | Choose the user's persistent ds4-agent cache, line-history and Chrome-profile base directory. | [ds4_agent.c:4023](ds4_agent.c#L4023) | +| `LINENOISE_ASSUME_TTY` | Pure presence test flag: any defined value, including empty or "0", treats non-TTY input as interactive while skipping real termios raw-mode setup/restoration. Unset follows isatty and uses plain blocking line input for pipes. | Exercise the live linenoise/agent editor, prompt and status layout in automated pipe-based tests without a real terminal. | [linenoise.c:593](linenoise.c#L593) | +| `LINENOISE_COLS` | If defined, its value is returned directly through atoi with no validation: empty/nonnumeric becomes 0 and signed values are accepted. If unset, linenoise uses TIOCGWINSZ, then a cursor-position query, then fallback width 80. | Force a deterministic terminal column count for linenoise wrapping/layout tests. | [linenoise.c:684](linenoise.c#L684) | +| `PATH` | Colon-separated executable search directories consulted only after DS4_CHROME, macOS app paths and fixed Chrome/Chromium paths fail. The first executable google-chrome, google-chrome-stable, chromium or chromium-browser wins; unset/empty/no match falls back to the literal command google-chrome, which execlp may search again. | Locate a Chrome/Chromium executable for the ds4 web/CDP tool. | [ds4_web.c:992](ds4_web.c#L992) | +| `TERM` | Case-insensitive terminal name. Values dumb, cons25, or emacs select linenoise's simple prompt plus blocking line reader; unset, empty, or any other value selects the normal interactive editor when stdin is a TTY. | Avoid ANSI/raw interactive editing on terminal types known not to support the required escape sequences. | [linenoise.c:559](linenoise.c#L559) | + +## Test and fixture environment inputs + +These entries are consumed by repository test binaries or fixture scripts. Some +production runtime controls are repeated here because a maintained fixture exposes +them as part of its own test contract. + +### Test binaries and cleanup hooks + +| Variable | Accepted value and default | Effect | Source | +| --- | --- | --- | --- | +| `DS4_CUDA_TOPK_REGRESSION_SEC` | positive floating-point seconds; default 2.0; invalid or nonpositive input restores the default | Set the CUDA large-top-k elapsed-time regression limit. | [tests/cuda_long_context_smoke.c:72](tests/cuda_long_context_smoke.c#L72) | +| `DS4_METAL_MOE_TILE_MAX` | cleanup-only historical spelling; no production consumer exists, so setting it has no runtime effect | Clears a legacy Metal MoE tile override while preparing the test environment. | [tests/ds4_test.c:7236](tests/ds4_test.c#L7236) | +| `DS4_ROCM_ENABLE_Q4_PREFILL_TILE8` | cleanup-only legacy spelling; no runtime or test reader; setting it has no effect | Remove a stale opt-in name while preparing ROCm Q4 test cases; TILE8 is automatic. | [tests/test_rocm_q4_dense_pair.cpp:58](tests/test_rocm_q4_dense_pair.cpp#L58) | +| `DS4_TEST_ALLOW_FALLBACK` | presence flag; unset: native mixed path required; any defined value including empty or 0 permits exactly the serialized-fallback counter outcome | Lets the CUDA mixed prefill/decode oracle accept serialized fallback while retaining bit-exact logit comparisons. | [tests/test_cuda_mixed_batch.c:204](tests/test_cuda_mixed_batch.c#L204) | +| `DS4_TEST_BACKEND` | exact cpu selects CPU; every other value, including unset/empty, selects Metal on Apple and CUDA elsewhere | Chooses the backend used by model-backed tests in tests/ds4_test.c. | [tests/ds4_test.c:91](tests/ds4_test.c#L91) | +| `DS4_TEST_BATCH_ONLY` | presence flag; unset: run batched and isolated-control phases; any defined value including empty or 0 stops after the batched archive/hash phase | Runs only the CUDA session-batch phase and skips replay against isolated control sessions. | [tests/test_cuda_session_batch.c:288](tests/test_cuda_session_batch.c#L288) | +| `DS4_TEST_CONTEXT` | CUDA session/mixed fixtures default to 1024 and require 1024..65536; mixed-batch uses strict full decimal parsing, while session-batch uses atoi and therefore accepts numeric prefixes | Sets the context and placement hint for the CUDA session-batch and mixed prefill/decode oracles. | [tests/test_cuda_session_batch.c:125](tests/test_cuda_session_batch.c#L125) | +| `DS4_TEST_DSPARK` | nonempty DSpark support-GGUF path; unset/empty skips the DSpark verify-depth test | Loads the DSpark support model for teacher-forced verification of committed speculative tokens. | [tests/ds4_test.c:8283](tests/ds4_test.c#L8283) | +| `DS4_TEST_GPU_DEVICES` | GPU device-list string parsed with the normal auto-VRAM parser; unset/empty defaults to 0,2,4,6,1,3,5,7; parse failure is fatal | Selects and orders the CUDA TP/EP devices used by the mixed prefill/decode oracle. | [tests/test_cuda_mixed_batch.c:122](tests/test_cuda_mixed_batch.c#L122) | +| `DS4_TEST_LOCAL_GOLDEN_FILE` | nonempty readable fixture path; unset/empty defaults to tests/test-vectors/flash-0731/local-golden.vec | Selects the local-golden vector file used for model-logit regression checks. | [tests/ds4_test.c:7221](tests/ds4_test.c#L7221) | +| `DS4_TEST_LOGPROB_AUTO_METAL` | presence flag; unset forces DS4_METAL_DISABLE_METAL4=1; any presence including empty or 0 removes that rollback and permits automatic Metal selection | Runs official log-probability vectors with automatic Metal-path selection instead of the fixed pre-Metal4 baseline. | [tests/ds4_test.c:6955](tests/ds4_test.c#L6955) | +| `DS4_TEST_LONG_PROMPT` | nonempty readable prompt-file path; unset/empty defaults to tests/long_context_story_prompt.txt | Selects the rendered story prompt for the long-context fact-recall test. | [tests/ds4_test.c:6690](tests/ds4_test.c#L6690) | +| `DS4_TEST_LONG_WORDS` | atoi integer; unset/empty/nonnumeric defaults to 0; valid range is 0..DS4_TEST_CONTEXT-128 and numeric prefixes are accepted | Adds repeated words to alternating CUDA session-batch prompts to exercise long-prefill rows. | [tests/test_cuda_session_batch.c:135](tests/test_cuda_session_batch.c#L135) | +| `DS4_TEST_METAL_EXACTN_BATCH_HEAD` | nonempty value other than exact 0 enables; unset/empty/0 disables; false/off also enable | Enables the Metal exact-N batch-head path and requires its attempt/use counters for every eligible oracle case. | [tests/test_metal_exactn_oracle.c:401](tests/test_metal_exactn_oracle.c#L401) | +| `DS4_TEST_METAL_EXACTN_ORACLE` | presence flag compiled only with DS4_TEST_HOOKS; absent from normal production builds | Force allocation of the exact-N Metal verifier/oracle workspace in tests. | [ds4.c:61801](ds4.c#L61801) | +| `DS4_TEST_MIXED_INITIAL` | integer 128..context-1; default 128 | Set the initial prefill length for the CUDA mixed-batch oracle. | [tests/test_cuda_mixed_batch.c:111](tests/test_cuda_mixed_batch.c#L111) | +| `DS4_TEST_MIXED_QUANTUM` | integer 1..context-1; default 128 | Set the number of prompt tokens added per CUDA mixed-batch round. | [tests/test_cuda_mixed_batch.c:113](tests/test_cuda_mixed_batch.c#L113) | +| `DS4_TEST_MIXED_ROUNDS` | integer 1..64; default 3 | Set the number of CUDA mixed-batch oracle rounds. | [tests/test_cuda_mixed_batch.c:115](tests/test_cuda_mixed_batch.c#L115) | +| `DS4_TEST_MODEL` | nonempty GGUF path; tests/ds4_test.c defaults to ds4flash.gguf, while standalone model-backed CUDA/Metal fixtures generally require a supplied path and fail or skip when absent | Selects the target model shared by model-backed test binaries. | [tests/ds4_test.c:14](tests/ds4_test.c#L14) | +| `DS4_TEST_MPP_EQ_CASE` | comma-separated substring filter; unset/empty runs all cases; tokens are whitespace-trimmed and the filter is truncated to 255 bytes | Restricts Metal tensor-equivalence vectors to IDs containing at least one requested substring. | [tests/ds4_test.c:7525](tests/ds4_test.c#L7525) | +| `DS4_TEST_MTP` | nonempty MTP support-GGUF path; unset/empty loads no MTP head; only the fast test engine uses it, with draft depth 4 | Enables the legacy MTP verify-depth regression; the test self-skips without this model. | [tests/ds4_test.c:104](tests/ds4_test.c#L104) | +| `DS4_TEST_Q4_STREAM_ITERS` | integer 1..10000; default 2 | Set measured iterations for the Metal Q4 stream oracle. | [tests/test_metal_q4_streams.c:734](tests/test_metal_q4_streams.c#L734) | +| `DS4_TEST_Q4_STREAM_SOAK` | integer 1..100000; default 8 | Set bounded overlap-soak iterations for the Metal Q4 stream oracle. | [tests/test_metal_q4_streams.c:736](tests/test_metal_q4_streams.c#L736) | +| `DS4_TEST_Q4_STREAM_TIMING` | presence flag; unset: correctness/leak checks only; any defined value including empty or 0 also runs timing pairs | Adds FIFO-versus-overlap and native-versus-overlap timing measurements to the Metal Q4 stream oracle. | [tests/test_metal_q4_streams.c:737](tests/test_metal_q4_streams.c#L737) | +| `DS4_TEST_Q4_STREAM_TIMING_BLOCKS` | integer 5..MAX_TIMING_BLOCKS; default 5 | Set the timing block count for the Metal Q4 stream oracle. | [tests/test_metal_q4_streams.c:739](tests/test_metal_q4_streams.c#L739) | +| `DS4_TEST_Q4_STREAM_TIMING_ITERS` | integer 1..10000; default 20 | Set timing iterations for the Metal Q4 stream oracle. | [tests/test_metal_q4_streams.c:741](tests/test_metal_q4_streams.c#L741) | +| `DS4_TEST_Q4_STREAM_WARMUP` | integer 1..64; default 1 | Set warmup iterations for the Metal Q4 stream oracle. | [tests/test_metal_q4_streams.c:732](tests/test_metal_q4_streams.c#L732) | +| `DS4_TEST_REQUIRE_MODEL` | nonempty value other than exact 0 requires a readable model; unset/empty/0 permits a skip; false/off count as required | Turns a missing Metal exact-N oracle model from a developer skip into a release-gate failure. | [tests/test_metal_exactn_oracle.c:390](tests/test_metal_exactn_oracle.c#L390) | +| `DS4_TEST_REQUIRE_ROCM_DEVICE` | nonempty value other than exact 0 requires a visible ROCm device; unset/empty/0 returns the fixture skip code; false/off count as required | Turns absence of a ROCm device from a skip into failure for the ROCm Q4 oracle. | [tests/test_rocm_q4_dense_pair.cpp:1483](tests/test_rocm_q4_dense_pair.cpp#L1483) | +| `DS4_TEST_SERVER_PREFILL` | presence flag; unset: normal prefill; any defined value including empty or 0 installs a no-op display-progress callback | Exercises the progress-split prefill path used by ds4-server in CUDA session-batch and control sessions. | [tests/test_cuda_session_batch.c:110](tests/test_cuda_session_batch.c#L110) | +| `DS4_TEST_SESSION_BATCH_ARM` | arbitrary nonempty label; unset/empty defaults to unspecified; it is logged only and does not alter execution | Labels the Metal session-batch experiment arm in setup diagnostics. | [tests/test_metal_session_batch.c:153](tests/test_metal_session_batch.c#L153) | +| `DS4_TEST_SESSION_BATCH_TIMING` | nonempty boolean; unset/empty/exact 0 disables; every other value enables | Print timing data from the Metal session-batch oracle. | [tests/test_metal_session_batch.c:150](tests/test_metal_session_batch.c#L150) | +| `DS4_TEST_SESSION_COUNT` | fixture-specific integer: CUDA session-batch defaults 8 and accepts atoi 2..16; CUDA mixed-batch defaults 8 with strict 3..16; Metal session-batch defaults 2 with strict 2..16 | Sets the number of simultaneous sessions exercised by the model-backed batch oracles. | [tests/test_cuda_session_batch.c:113](tests/test_cuda_session_batch.c#L113) | +| `DS4_TEST_SSD_CACHE_EXPERTS` | strict unsigned integer 30..UINT32_MAX; unset/empty defaults to 30; read only when Metal session-batch SSD streaming is enabled | Sizes the Metal session-batch routed-expert cache used to exercise SSD union policies for N=2..5. | [tests/test_metal_session_batch.c:88](tests/test_metal_session_batch.c#L88) | +| `DS4_TEST_SSD_STREAMING` | nonempty value other than exact 0 enables; unset/empty/0 disables; false/off also enable | Runs model-backed test engines through SSD streaming; the Metal session-batch fixture also uses cold mode and shared prefill workspace. | [tests/ds4_test.c:109](tests/ds4_test.c#L109) | +| `DS4_TEST_SSD_STREAMING_CACHE_EXPERTS` | strtoul decimal prefix; unset/empty/nonnumeric becomes 0, values above UINT32_MAX (including a parsed negative) saturate, and trailing text is accepted | Sets the routed-expert cache count on SSD-streaming engines created by tests/ds4_test.c. | [tests/ds4_test.c:112](tests/ds4_test.c#L112) | +| `DS4_TEST_SSD_STREAMING_CACHE_GB` | strtoull decimal GiB prefix; unset/empty/nonnumeric/zero becomes 0, byte overflow (including a parsed negative) saturates to UINT64_MAX, and trailing text is accepted | Sets the routed-expert cache byte budget on SSD-streaming engines created by tests/ds4_test.c. | [tests/ds4_test.c:114](tests/ds4_test.c#L114) | +| `DS4_TEST_SSD_STREAMING_COLD` | nonempty boolean; unset/empty/exact 0 disables; every other value enables | Run test engines in cold SSD-streaming mode and skip hot-expert preload. | [tests/ds4_test.c:110](tests/ds4_test.c#L110) | +| `DS4_TEST_SSD_STREAMING_PRELOAD_EXPERTS` | unsigned integer; default 0; numeric prefixes are accepted and overflow clamps to UINT32_MAX | Set the number of SSD-streaming experts preloaded by test engines. | [tests/ds4_test.c:116](tests/ds4_test.c#L116) | +| `DS4_TEST_SSD_UNION_POLICY_SWITCH` | nonempty boolean; unset/empty/exact 0 disables; every other value enables | Exercise an SSD session-union policy transition in the Metal session-batch oracle. | [tests/test_metal_session_batch.c:152](tests/test_metal_session_batch.c#L152) | +| `DS4_TEST_TP_DISCONNECT` | presence flag effective only in leader mode; unset: normal test; any defined value including empty or 0 enters the disconnect oracle | Waits for the TP worker to disconnect, then requires the next batch to fail and every session checkpoint to be invalidated. | [tests/test_metal_session_batch.c:268](tests/test_metal_session_batch.c#L268) | +| `DS4_TEST_TP_LEADER_HOST` | nonempty host string required in worker mode; no default; ignored outside worker mode | Sets the TP leader address contacted by the Metal session-batch worker. | [tests/test_metal_session_batch.c:212](tests/test_metal_session_batch.c#L212) | +| `DS4_TEST_TP_LISTEN_HOST` | nonempty host string; unset/empty defaults to 0.0.0.0; used only in leader mode | Sets the TP listen address for the Metal session-batch leader. | [tests/test_metal_session_batch.c:204](tests/test_metal_session_batch.c#L204) | +| `DS4_TEST_TP_MODE` | unset/empty: no TP; exact leader or worker selects that role; every other nonempty value fails; incompatible with SSD-streaming mode | Selects standalone, TP-leader, or TP-worker execution for the Metal session-batch oracle. | [tests/test_metal_session_batch.c:168](tests/test_metal_session_batch.c#L168) | +| `DS4_TEST_TP_PORT` | strict full decimal integer 1..65535; unset/empty defaults to 19452 | Sets the listen/connect port shared by Metal session-batch TP leader and worker. | [tests/test_metal_session_batch.c:63](tests/test_metal_session_batch.c#L63) | +| `DS4_TEST_TP_TRANSPORT` | unset/empty/auto selects automatic transport; exact tcp or rdma selects that transport; other values fail | Chooses the TP transport for Metal session-batch leader/worker tests. | [tests/test_metal_session_batch.c:52](tests/test_metal_session_batch.c#L52) | +| `DS4_TEST_VECTOR_FILE` | nonempty readable vector path; unset/empty defaults to tests/test-vectors/flash-0731/official.vec | Selects the official fixture used by log-probability and Metal tensor-equivalence tests. | [tests/ds4_test.c:6942](tests/ds4_test.c#L6942) | +| `PROTO_Q8_DEBUG` | presence diagnostic; unset: summary only; any defined value including empty or 0 prints detailed error structure after a Q8 parity failure | Dumps bad-element tile and row/column histograms for the CUDA Q8 prototype when parity fails. | [cuda/mmq/test/proto_gemm_dense_q8_d2r.cu:647](cuda/mmq/test/proto_gemm_dense_q8_d2r.cu#L647) | + +### Test fixture scripts + +| Variable | Accepted value and default | Effect | Source | +| --- | --- | --- | --- | +| `DEEPSEEK_API_KEY` | secret string; required | Authenticate official test-vector fetch. | [tests/test-vectors/fetch_official_vectors.py:236](tests/test-vectors/fetch_official_vectors.py#L236) | +| `DS4_BIN` | executable path; unset/empty defaults to ./ds4; the Q4 matrix requires it executable, the DSpark fixture skips if missing, and the GLM smoke lets command failure fail the test | Selects the ds4 binary launched by model-backed shell test fixtures. | [tests/cuda_q4_gb10_fast_matrix.sh:47](tests/cuda_q4_gb10_fast_matrix.sh#L47) | +| `DS4_CUDA_DISABLE_DSPARK_EXACTN` | arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4 | Records and passes through the CUDA exact-N rollback used by the acceptance run. | [tests/dspark_acceptance_fixture.sh:253](tests/dspark_acceptance_fixture.sh#L253) | +| `DS4_CUDA_DISABLE_DSPARK_EXACTN_BATCH_HEAD` | arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4 | Records and passes through the CUDA exact-N batch-head rollback. | [tests/dspark_acceptance_fixture.sh:255](tests/dspark_acceptance_fixture.sh#L255) | +| `DS4_CUDA_DISABLE_DSPARK_EXACTN_GRAPHS` | arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4 | Records and passes through the CUDA exact-N graph rollback. | [tests/dspark_acceptance_fixture.sh:257](tests/dspark_acceptance_fixture.sh#L257) | +| `DS4_CUDA_DISABLE_DSPARK_NONCAUSAL_ONLINE` | arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4 | Records and passes through the CUDA noncausal-online-attention rollback. | [tests/dspark_acceptance_fixture.sh:266](tests/dspark_acceptance_fixture.sh#L266) | +| `DS4_CUDA_DSPARK_DEVICE_PROPOSER` | arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4 | Records and passes through the CUDA device-proposer opt-in. | [tests/dspark_acceptance_fixture.sh:258](tests/dspark_acceptance_fixture.sh#L258) | +| `DS4_CUDA_DSPARK_EXACT2` | arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4 | Records and passes through the CUDA exact-2 verifier override. | [tests/dspark_acceptance_fixture.sh:250](tests/dspark_acceptance_fixture.sh#L250) | +| `DS4_CUDA_DSPARK_EXACTN` | arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4 | Records and passes through the CUDA exact-N verifier opt-in. | [tests/dspark_acceptance_fixture.sh:252](tests/dspark_acceptance_fixture.sh#L252) | +| `DS4_CUDA_DSPARK_EXACTN_BATCH_HEAD` | arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4 | Records and passes through the CUDA exact-N batch-head opt-in. | [tests/dspark_acceptance_fixture.sh:254](tests/dspark_acceptance_fixture.sh#L254) | +| `DS4_CUDA_DSPARK_EXACTN_GRAPHS` | arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4 | Records and passes through the CUDA exact-N graph opt-in. | [tests/dspark_acceptance_fixture.sh:256](tests/dspark_acceptance_fixture.sh#L256) | +| `DS4_CUDA_DSPARK_NO_DEVICE_PROPOSER` | arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4 | Records and passes through the CUDA device-proposer rollback. | [tests/dspark_acceptance_fixture.sh:259](tests/dspark_acceptance_fixture.sh#L259) | +| `DS4_CUDA_DSPARK_PROPOSER_BLOCK_MAX` | arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4 | Records and passes through the CUDA proposer block-size cap. | [tests/dspark_acceptance_fixture.sh:269](tests/dspark_acceptance_fixture.sh#L269) | +| `DS4_CUDA_ENABLE_DSPARK_NONCAUSAL_ONLINE` | arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4 | Records and passes through the CUDA noncausal-online-attention opt-in. | [tests/dspark_acceptance_fixture.sh:265](tests/dspark_acceptance_fixture.sh#L265) | +| `DS4_CUDA_Q4_MATRIX_CTX` | nonempty decimal digits other than exact 0; unset/empty defaults to 4096; no upper bound; leading-zero zero strings such as 00 pass the script's guard | Sets --ctx for every Q4 GB10 smoke and score_official matrix arm. | [tests/cuda_q4_gb10_fast_matrix.sh:49](tests/cuda_q4_gb10_fast_matrix.sh#L49) | +| `DS4_CUDA_Q4_MATRIX_DECODE_GRAPHS` | exact default, 0, or 1; unset/empty defaults to default; other values fail | Leaves decode graphs automatic, forces them off, or forces them on with capture logging for every matrix arm. | [tests/cuda_q4_gb10_fast_matrix.sh:56](tests/cuda_q4_gb10_fast_matrix.sh#L56) | +| `DS4_CUDA_Q4_MATRIX_PROMPT` | prompt string; unset/empty defaults to Write a complete Python quicksort function with comments. | Sets the deterministic smoke prompt whose log-probability output is compared across Q4 fast-path arms. | [tests/cuda_q4_gb10_fast_matrix.sh:52](tests/cuda_q4_gb10_fast_matrix.sh#L52) | +| `DS4_CUDA_Q4_MATRIX_SCORER` | executable path; unset/empty defaults to gguf-tools/quality-testing/score_official; a missing/nonexecutable path fails | Selects the scorer used to produce quality TSVs for each non-oracle Q4 matrix arm. | [tests/cuda_q4_gb10_fast_matrix.sh:48](tests/cuda_q4_gb10_fast_matrix.sh#L48) | +| `DS4_CUDA_Q4_MATRIX_SKIP_PARITY` | exact 0 or 1; unset/empty defaults to 0; other values fail | When 1, skips the synthetic MMQ parity prerequisite and marks the resulting QA run incomplete. | [tests/cuda_q4_gb10_fast_matrix.sh:57](tests/cuda_q4_gb10_fast_matrix.sh#L57) | +| `DS4_CUDA_Q4_MATRIX_SSD_CACHE` | unset/empty by default; required and passed verbatim as --ssd-streaming-cache-experts when streaming=1; must remain empty when streaming=0; no further validation | Sets the SSD expert cache as a count or NGB value for every streamed matrix arm. | [tests/cuda_q4_gb10_fast_matrix.sh:54](tests/cuda_q4_gb10_fast_matrix.sh#L54) | +| `DS4_CUDA_Q4_MATRIX_SSD_PRELOAD` | unset/empty omits preload; a nonempty value is passed verbatim as --ssd-streaming-preload-experts and is allowed only when streaming=1 | Sets optional expert preload for streamed Q4 smoke and scoring arms. | [tests/cuda_q4_gb10_fast_matrix.sh:55](tests/cuda_q4_gb10_fast_matrix.sh#L55) | +| `DS4_CUDA_Q4_MATRIX_SSD_STREAMING` | exact 0 or 1; unset/empty defaults to 0; other values fail | Runs all Q4 matrix arms resident or with SSD streaming and enforces matching cache/preload arguments. | [tests/cuda_q4_gb10_fast_matrix.sh:53](tests/cuda_q4_gb10_fast_matrix.sh#L53) | +| `DS4_CUDA_Q4_MATRIX_TOKENS` | nonempty decimal digits other than exact 0; unset/empty defaults to 32; no upper bound; leading-zero zero strings such as 00 pass the script's guard | Sets the continuation length for each Q4 GB10 smoke arm. | [tests/cuda_q4_gb10_fast_matrix.sh:50](tests/cuda_q4_gb10_fast_matrix.sh#L50) | +| `DS4_CUDA_Q4_MATRIX_TOP_K` | nonempty decimal digits other than exact 0 and numerically <=128; unset/empty defaults to 128; leading-zero zero strings such as 00 pass the guard | Sets --logprobs-top-k for the byte-comparable Q4 GB10 smoke dumps. | [tests/cuda_q4_gb10_fast_matrix.sh:51](tests/cuda_q4_gb10_fast_matrix.sh#L51) | +| `DS4_DSPARK_FIXTURE_BACKEND` | unset/empty defaults to auto; exact auto, metal, cuda, or rocm accepted; every other value fails | Chooses the explicit backend flag for baseline and DSpark acceptance runs; auto passes none. | [tests/dspark_acceptance_fixture.sh:14](tests/dspark_acceptance_fixture.sh#L14) | +| `DS4_DSPARK_FIXTURE_CONFIDENCE` | unset/empty omits the option and uses the runtime default; otherwise passed verbatim to --dspark-confidence without fixture-side validation | Overrides the DSpark confidence threshold for acceptance runs and records it in metadata. | [tests/dspark_acceptance_fixture.sh:13](tests/dspark_acceptance_fixture.sh#L13) | +| `DS4_DSPARK_FIXTURE_C_ADD_MIN_ACCEPTED` | decimal digits including 0; unset/empty defaults to 8; non-digits fail | Sets the minimum accepted-draft count for the c_add case when the proposal-quality guard is active. | [tests/dspark_acceptance_fixture.sh:12](tests/dspark_acceptance_fixture.sh#L12) | +| `DS4_DSPARK_FIXTURE_REQUIRE_ACTIVE` | exact 0 or 1; unset/empty defaults to 1; other values fail | When 1, requires aggregate proposed and accepted-draft counts to both be nonzero. | [tests/dspark_acceptance_fixture.sh:17](tests/dspark_acceptance_fixture.sh#L17) | +| `DS4_DSPARK_FIXTURE_REQUIRE_CUDA_DEVICE_PROPOSER` | exact 0 or 1; unset/empty defaults to 0; other values fail; 1 also forces byte-identical output checking | Requires CUDA device-proposer attempts to equal uses, with nonzero use and zero fallback or policy mismatch. | [tests/dspark_acceptance_fixture.sh:22](tests/dspark_acceptance_fixture.sh#L22) | +| `DS4_DSPARK_FIXTURE_REQUIRE_CUDA_EXACTN` | exact 0 or 1; unset/empty defaults to 0; other values fail; 1 also forces byte-identical output checking | Requires at least one CUDA exact-N attempt and zero exact-N error fallbacks. | [tests/dspark_acceptance_fixture.sh:19](tests/dspark_acceptance_fixture.sh#L19) | +| `DS4_DSPARK_FIXTURE_REQUIRE_CUDA_EXACTN_BATCH_HEAD` | exact 0 or 1; unset/empty defaults to 0; other values fail; 1 implies REQUIRE_CUDA_EXACTN and identical output | Requires nonzero CUDA exact-N batch-head attempts/uses and zero batch-head fallbacks. | [tests/dspark_acceptance_fixture.sh:20](tests/dspark_acceptance_fixture.sh#L20) | +| `DS4_DSPARK_FIXTURE_REQUIRE_CUDA_EXACTN_GRAPHS` | exact 0 or 1; unset/empty defaults to 0; other values fail; 1 implies REQUIRE_CUDA_EXACTN and identical output | Requires CUDA exact-N graph attempts, uses, captures, and replays, with zero no-slot or graph failures. | [tests/dspark_acceptance_fixture.sh:21](tests/dspark_acceptance_fixture.sh#L21) | +| `DS4_DSPARK_FIXTURE_REQUIRE_DIRECT_COMMIT` | exact 0 or 1; unset/empty defaults to 0; other values fail | Requires at least one direct verifier-state commit; with REQUIRE_PARTIAL it also requires a direct partial commit. | [tests/dspark_acceptance_fixture.sh:9](tests/dspark_acceptance_fixture.sh#L9) | +| `DS4_DSPARK_FIXTURE_REQUIRE_EXACT2` | exact 0 or 1; unset/empty defaults to 0; other values fail; 1 also forces byte-identical output checking | Requires at least one exact-2 attempt and zero exact-2 fallbacks. | [tests/dspark_acceptance_fixture.sh:18](tests/dspark_acceptance_fixture.sh#L18) | +| `DS4_DSPARK_FIXTURE_REQUIRE_IDENTICAL` | exact 0 or 1; unset/empty defaults to 0; other values fail; several path-specific requirements force it to 1 | When 1, fails any byte difference between baseline and DSpark stdout; otherwise mismatches are reported but allowed. | [tests/dspark_acceptance_fixture.sh:10](tests/dspark_acceptance_fixture.sh#L10) | +| `DS4_DSPARK_FIXTURE_REQUIRE_METAL_DEVICE_PROPOSER` | exact 0 or 1; unset/empty defaults to 0; other values fail; 1 also forces byte-identical output checking | Requires Metal device-proposer attempts to equal uses, with nonzero use and zero fallback or policy mismatch. | [tests/dspark_acceptance_fixture.sh:25](tests/dspark_acceptance_fixture.sh#L25) | +| `DS4_DSPARK_FIXTURE_REQUIRE_METAL_EXACTN_BATCH_HEAD` | exact 0 or 1; unset/empty defaults to 0; other values fail; 1 also forces byte-identical output checking | Requires nonzero Metal exact-N batch-head attempts/uses and zero batch-head fallbacks. | [tests/dspark_acceptance_fixture.sh:23](tests/dspark_acceptance_fixture.sh#L23) | +| `DS4_DSPARK_FIXTURE_REQUIRE_METAL_EXACTN_PARTIAL` | exact 0 or 1; unset/empty defaults to 0; other values fail; 1 also forces byte-identical output checking | Requires a Metal exact-N partial replay, matching verify-skip count, and zero union error fallbacks. | [tests/dspark_acceptance_fixture.sh:24](tests/dspark_acceptance_fixture.sh#L24) | +| `DS4_DSPARK_FIXTURE_REQUIRE_PARTIAL` | unset/empty defaults to 0; exact 0 disables; any other value enables, with no 0/1 validation | Requires at least one partial-accept case and, together with REQUIRE_DIRECT_COMMIT, one direct partial commit. | [tests/dspark_acceptance_fixture.sh:8](tests/dspark_acceptance_fixture.sh#L8) | +| `DS4_DSPARK_FIXTURE_REQUIRE_PROPOSAL_QUALITY` | unset/empty/auto selects auto; 0/false/no/off disables; 1/true/yes/on enables; values are lowercase and other strings fail; auto enables only without partial mode/confidence override and with tokens >=32 | Controls the c_add minimum-accepted-drafts quality guard. | [tests/dspark_acceptance_fixture.sh:11](tests/dspark_acceptance_fixture.sh#L11) | +| `DS4_DSPARK_FIXTURE_SSD_STREAMING` | exact 0 or 1; unset/empty defaults to 0; other values fail | Adds --ssd-streaming to both baseline and DSpark runs when enabled. | [tests/dspark_acceptance_fixture.sh:15](tests/dspark_acceptance_fixture.sh#L15) | +| `DS4_DSPARK_FIXTURE_SSD_STREAMING_CACHE_EXPERTS` | unset/empty omits the cache option; otherwise decimal digits including 0 are required, and a value is legal only with SSD streaming enabled | Passes an explicit --ssd-streaming-cache-experts value to both acceptance-run variants. | [tests/dspark_acceptance_fixture.sh:16](tests/dspark_acceptance_fixture.sh#L16) | +| `DS4_DSPARK_FIXTURE_TOKENS` | token-count argument; unset/empty defaults to 32; the fixture does not validate it before passing --tokens, and auto quality treats nonnumeric or <32 as ineligible | Sets generated-token count for each baseline and DSpark acceptance case. | [tests/dspark_acceptance_fixture.sh:7](tests/dspark_acceptance_fixture.sh#L7) | +| `DS4_DSPARK_MODEL` | target-model path; unset/empty falls back to DS4_TEST_MODEL, then ./ds4flash.gguf; a missing file causes a successful skip | Selects the target GGUF compared in baseline and DSpark acceptance runs. | [tests/dspark_acceptance_fixture.sh:5](tests/dspark_acceptance_fixture.sh#L5) | +| `DS4_DSPARK_SSD_VERIFY_BLOCK_MAX` | unsigned integer rows; default/fallback 0 means automatic policy; numeric prefixes accepted; used both as verifier cap and as an exact-2 proposer-policy discriminator | Cap speculative rows verified from SSD and influence exact-2 proposal sizing. | [tests/dspark_acceptance_fixture.sh:271](tests/dspark_acceptance_fixture.sh#L271) | +| `DS4_DSPARK_SUPPORT` | support-model path; unset/empty defaults to gguf/DeepSeek-V4-Flash-DSpark-support-0731.gguf; a missing file causes a successful skip | Selects the DSpark support GGUF passed through --mtp. | [tests/dspark_acceptance_fixture.sh:6](tests/dspark_acceptance_fixture.sh#L6) | +| `DS4_DSPARK_VERIFY_NONCAUSAL` | presence diagnostic sampled once after the first successfully submitted CUDA noncausal-attention kernel; unset: verify 0 calls; any presence including empty or 0: verify that call and the next 2 | CUDA only: synchronizes and reads back Q/KV/output, computes the DSpark noncausal attention CPU reference, and logs max absolute/relative error; it reports only and does not fail the operation. | [tests/dspark_acceptance_fixture.sh:267](tests/dspark_acceptance_fixture.sh#L267) | +| `DS4_GLM_BACKEND` | exact metal, cuda, or cpu; unset/empty defaults to metal; other values fail | Selects the backend flag used by the GLM long-context continuation smoke test. | [tests/glm_long_context_smoke.sh:29](tests/glm_long_context_smoke.sh#L29) | +| `DS4_GLM_EXTRA_ARGS` | unset/empty adds no arguments; otherwise intentionally unquoted and therefore shell field-split and pathname-expanded | Adds backend/device options to the ds4 invocation used for every GLM long-context case. | [tests/glm_long_context_smoke.sh:91](tests/glm_long_context_smoke.sh#L91) | +| `DS4_GLM_LONG_CONTEXT_CTX` | context argument; unset/empty defaults to 100000; forwarded to --ctx without script-side validation | Sets the context size for GLM long-context smoke invocations. | [tests/glm_long_context_smoke.sh:26](tests/glm_long_context_smoke.sh#L26) | +| `DS4_GLM_LONG_CONTEXT_GEN` | generation-count argument; unset/empty defaults to 32; forwarded to -n without script-side validation | Sets the number of continuation tokens checked by each GLM long-context case. | [tests/glm_long_context_smoke.sh:28](tests/glm_long_context_smoke.sh#L28) | +| `DS4_GLM_LONG_CONTEXT_REPEATS` | whitespace-separated list of prompt-padding counts; unset/empty defaults to the single count 130; each item must work as a shell integer | Chooses one or more audit-block counts used to construct long GLM prompts. | [tests/glm_long_context_smoke.sh:27](tests/glm_long_context_smoke.sh#L27) | +| `DS4_GLM_MODEL` | model path used only when no nonempty positional MODEL is supplied; unset/empty defaults to models/GLM-5.2-UD-Q4_K_XL.gguf | Selects the GLM-5.2 GGUF used by the long-context continuation smoke test. | [tests/glm_long_context_smoke.sh:25](tests/glm_long_context_smoke.sh#L25) | +| `DS4_METAL_DISABLE_DSPARK_EXACTN_BATCH_HEAD` | arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4 | Records and passes through the Metal exact-N batch-head rollback. | [tests/dspark_acceptance_fixture.sh:262](tests/dspark_acceptance_fixture.sh#L262) | +| `DS4_METAL_DSPARK_DEVICE_PROPOSER` | arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4 | Records and passes through the Metal device-proposer opt-in. | [tests/dspark_acceptance_fixture.sh:263](tests/dspark_acceptance_fixture.sh#L263) | +| `DS4_METAL_DSPARK_EXACT2` | arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4 | Records and passes through the Metal exact-2 verifier override. | [tests/dspark_acceptance_fixture.sh:251](tests/dspark_acceptance_fixture.sh#L251) | +| `DS4_METAL_DSPARK_EXACTN_BATCH_HEAD` | arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4 | Records and passes through the Metal exact-N batch-head opt-in. | [tests/dspark_acceptance_fixture.sh:261](tests/dspark_acceptance_fixture.sh#L261) | +| `DS4_METAL_DSPARK_EXACTN_UNION` | arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4 | Records and passes through the Metal exact-N union-verifier opt-in. | [tests/dspark_acceptance_fixture.sh:260](tests/dspark_acceptance_fixture.sh#L260) | +| `DS4_METAL_DSPARK_EXACT_ROWS_ASYNC_TAILS` | arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4 | Records and passes through the Metal exact-row asynchronous-tail override. | [tests/dspark_acceptance_fixture.sh:268](tests/dspark_acceptance_fixture.sh#L268) | +| `DS4_METAL_DSPARK_NO_DEVICE_PROPOSER` | arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4 | Records and passes through the Metal device-proposer rollback. | [tests/dspark_acceptance_fixture.sh:264](tests/dspark_acceptance_fixture.sh#L264) | +| `DS4_METAL_DSPARK_PROPOSER_BLOCK_MAX` | arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4 | Records and passes through the Metal proposer block-size cap. | [tests/dspark_acceptance_fixture.sh:270](tests/dspark_acceptance_fixture.sh#L270) | +| `DS4_TEST_MODEL` | fallback target-model path used only when DS4_DSPARK_MODEL is unset/empty; unset/empty then defaults to ./ds4flash.gguf | Provides the shared test-model fallback for the DSpark acceptance fixture. | [tests/dspark_acceptance_fixture.sh:5](tests/dspark_acceptance_fixture.sh#L5) | +| `OPENROUTER_API_KEY` | secret string; required | Authenticate OpenRouter GLM test-vector fetch. | [tests/test-vectors/fetch_openrouter_glm_vectors.py:355](tests/test-vectors/fetch_openrouter_glm_vectors.py#L355) | +| `TMPDIR` | temporary-directory base path; unset/empty defaults to /tmp | Chooses the parent directory for auto-created Q4 matrix, DSpark fixture, and GLM smoke work directories. | [tests/cuda_q4_gb10_fast_matrix.sh:108](tests/cuda_q4_gb10_fast_matrix.sh#L108) | + +## Tool and wrapper environment inputs + +These variables configure maintained download, service-wrapper, and offline tooling. +A tool that accepts a variable name dynamically (for example `--api-key-env`) may read +the caller-selected name in addition to the literal defaults listed here. + +| Variable | Accepted value and default | Effect | Source | +| --- | --- | --- | --- | +| `` | dynamic environment-variable name; no fixed identifier; overrides endpoint-derived key name | Allow a caller-selected credential environment variable. | [gguf-tools/quality-testing/collect_official.py:243](gguf-tools/quality-testing/collect_official.py#L243) | +| `DEEPSEEK_API_KEY` | secret string; default credential for non-OpenRouter endpoint; required unless --api-key-env selects another name | Authenticate official DeepSeek continuation collection. | [gguf-tools/quality-testing/collect_official.py:242](gguf-tools/quality-testing/collect_official.py#L242) | +| `DS4_BATCHED_SESSIONS` | unset/empty defaults to 16; otherwise passed verbatim to --batched-session; the wrapper does not validate it | Sets the maximum batched-session count for the managed CUDA tensor-parallel server. | [run-nvidia-tp-server.sh:14](run-nvidia-tp-server.sh#L14) | +| `DS4_CTX` | unset/empty defaults to 100000; otherwise passed verbatim to --ctx; the wrapper does not validate it | Sets the managed server context size. | [run-nvidia-tp-server.sh:9](run-nvidia-tp-server.sh#L9) | +| `DS4_GGUF_DIR` | path; default repository gguf/ directory | Choose the model download directory. | [download_model.sh:23](download_model.sh#L23) | +| `DS4_KV_DIR` | directory path; unset/empty defaults to /data/ds4-kv | Sets --kv-disk-dir for the managed server disk-backed KV cache. | [run-nvidia-tp-server.sh:12](run-nvidia-tp-server.sh#L12) | +| `DS4_KV_SPACE_MB` | unset/empty defaults to 8192; otherwise passed verbatim to --kv-disk-space-mb; the wrapper does not validate it | Sets the managed server disk-KV capacity in MiB. | [run-nvidia-tp-server.sh:13](run-nvidia-tp-server.sh#L13) | +| `DS4_LOCK_FILE` | lock-file path; unset/empty defaults to /tmp/ds4.lock | Selects the PID/instance lock inspected by start, stop, restart, and status; an explicit environment value is inherited by ds4-server. | [run-nvidia-tp-server.sh:15](run-nvidia-tp-server.sh#L15) | +| `DS4_MODEL` | model path; unset/empty defaults to /home/antirez/models/deepseek-v4-gguf/DeepSeek-V4-Flash-MXFP4Experts-F16HC-F16Compressor-F16Indexer-Q8Attn-Q8Shared-Q8Out-chat-v2-mxfp4-0731.gguf; unreadable paths fail | Sets the target GGUF passed to the managed CUDA tensor-parallel server. | [run-nvidia-tp-server.sh:8](run-nvidia-tp-server.sh#L8) | +| `DS4_SERVER_HOST` | host string; unset/empty defaults to 0.0.0.0 | Sets the HTTP listen address passed to the managed server. | [run-nvidia-tp-server.sh:10](run-nvidia-tp-server.sh#L10) | +| `DS4_SERVER_LOG` | log-file path; unset/empty defaults to /tmp/ds4-server.log | Receives detached-server stdout/stderr and supplies the readiness probe and failure tail. | [run-nvidia-tp-server.sh:16](run-nvidia-tp-server.sh#L16) | +| `DS4_SERVER_PORT` | unset/empty defaults to 8000; otherwise passed verbatim to --port; the wrapper does not validate it | Sets the HTTP listen port passed to the managed server. | [run-nvidia-tp-server.sh:11](run-nvidia-tp-server.sh#L11) | +| `DS4_START_TIMEOUT` | positive decimal integer with no leading zero; unset/empty defaults to 180; invalid values fail when starting | Sets how many seconds detached startup waits for the lock owner and listening log marker. | [run-nvidia-tp-server.sh:17](run-nvidia-tp-server.sh#L17) | +| `DS4_STOP_TIMEOUT` | positive decimal integer with no leading zero; unset/empty defaults to 120; invalid values fail when stopping | Sets how many seconds graceful stop waits after SIGTERM before failing. | [run-nvidia-tp-server.sh:18](run-nvidia-tp-server.sh#L18) | +| `FLATTEN_DOWNLOADS` | exact integer 1 enables; unset or 0 preserves normal shard paths | Move downloaded Hugging Face files from nested cache paths into the requested output directory. | [download_model.sh:244](download_model.sh#L244) | +| `FORCE_HF_DOWNLOAD` | exact integer 1 enables; unset or 0 uses the available downloader automatically | Force download_model.sh to use hf download instead of curl when available. | [download_model.sh:216](download_model.sh#L216) | +| `HF_TOKEN` | secret string; unset tries cached Hugging Face token or unauthenticated download | Authenticate Hugging Face downloads. | [download_model.sh:28](download_model.sh#L28) | +| `HOME` | Filesystem directory string. In ds4-agent, unset or empty falls back to "." for the default cache and history roots; the web helper applies the same fallback for its browser profile. A nonempty value roots .ds4/kvcache, .ds4_agent_history and .ds4/browser. | Choose the user's persistent ds4-agent cache, line-history and Chrome-profile base directory. | [ds4_agent.c:4023](ds4_agent.c#L4023) | +| `OPENROUTER_API_KEY` | secret string; default credential when endpoint contains openrouter.ai; required unless --api-key-env selects another name | Authenticate OpenRouter continuation collection. | [gguf-tools/quality-testing/collect_official.py:242](gguf-tools/quality-testing/collect_official.py#L242) | + + diff --git a/Makefile b/Makefile index 73c6f77b5..2fbb9a9af 100644 --- a/Makefile +++ b/Makefile @@ -68,7 +68,7 @@ DS4_LINK_LIBS ?= $(CUDA_LDLIBS) METAL_LDLIBS := $(LDLIBS) endif -.PHONY: all help clean test test-rocm test-glm53-kda-rocm test-metal-session-batch test-metal-session-batch-ssd test-metal-q4-streams test-metal-q4-attn-exactn test-metal-exactn-oracle test-metal-dspark-capture test-metal-iq2-midonly test-metal-iq2-ssd-grouped-mm test-metal-iq2-live-index test-mxfp4-cuda test-mxfp4-rocm test-mmq-parity-cuda test-rocm-q4-parity test-rocm-q4-dense test-rocm-q4-pair test-rocm-q4-prefill test-strix-rocm-q4-parity test-strix-rocm-q4-prefill test-strix-rocm-q4-prefill-long test-cuda-session-batch test-cuda-mixed-batch dspark-acceptance dspark-verify-depth rocm-dspark-acceptance rocm-dspark-verify-depth mtp-verify-depth cpu cuda cuda-spark cuda-generic cuda-regression strix-halo rocm +.PHONY: all help clean test check-environment-docs test-rocm test-glm53-kda-rocm test-metal-session-batch test-metal-session-batch-ssd test-metal-q4-streams test-metal-q4-attn-exactn test-metal-exactn-oracle test-metal-dspark-capture test-metal-iq2-midonly test-metal-iq2-ssd-grouped-mm test-metal-iq2-live-index test-mxfp4-cuda test-mxfp4-rocm test-mmq-parity-cuda test-rocm-q4-parity test-rocm-q4-dense test-rocm-q4-pair test-rocm-q4-prefill test-strix-rocm-q4-parity test-strix-rocm-q4-prefill test-strix-rocm-q4-prefill-long test-cuda-session-batch test-cuda-mixed-batch dspark-acceptance dspark-verify-depth rocm-dspark-acceptance rocm-dspark-verify-depth mtp-verify-depth cpu cuda cuda-spark cuda-generic cuda-regression strix-halo rocm ifeq ($(UNAME_S),Darwin) .PHONY: metal-decode-schedule-bench metal-prefill-variant-bench check-mxfp4-half-lut test-mxfp4-metal @@ -80,6 +80,7 @@ help: @echo " make Build Metal ./ds4, ./ds4-server, ./ds4-bench, ./ds4-eval, and ./ds4-agent" @echo " make cpu Build CPU-only ./ds4, ./ds4-server, ./ds4-bench, ./ds4-eval, and ./ds4-agent" @echo " make test Build and run tests" + @echo " make check-environment-docs Verify the generated environment-variable inventory" @echo " make test-metal-session-batch-ssd Exact-logit Metal SSD union control/candidate oracle" @echo " make test-metal-q4-streams Check resident Q4 Metal stream overlap" @echo " make test-metal-q4-attn-exactn Bitwise/canary oracle for M1-M4 SSD-prefill Q4 attention output" @@ -280,6 +281,7 @@ help: @echo " make rocm-dspark-verify-depth Build ROCm and run the DSpark verifier invariant" @echo " make cpu Build CPU-only ./ds4, ./ds4-server, ./ds4-bench, ./ds4-eval, and ./ds4-agent" @echo " make test Build and run tests" + @echo " make check-environment-docs Verify the generated environment-variable inventory" @echo " make dspark-verify-depth Run DSpark speculative verification smoke if support GGUF is present" @echo " make mtp-verify-depth Run legacy MTP speculative verification smoke if MTP GGUF is present" @echo " make clean Remove build outputs" @@ -406,6 +408,9 @@ test-mmq-parity-cuda: cuda/mmq/test/test_mmq_parity ./cuda/mmq/test/test_mmq_parity endif +check-environment-docs: + python3 scripts/generate_environment_variables.py --check + ds4.o: ds4.c ds4.h ds4_ssd.h ds4_distributed.h ds4_gpu.h $(CC) $(CFLAGS) -c -o $@ ds4.c diff --git a/README.md b/README.md index 00fe36a88..f65c7ed77 100644 --- a/README.md +++ b/README.md @@ -80,8 +80,9 @@ next sections. guide for contributors. **Read this before sending a pull request**. - [QA_BEFORE_RELEASES.md](QA_BEFORE_RELEASES.md): the complete release test matrix, including the remote Metal, CUDA, and ROCm machines. -- [ENVIRONMENT_VARIABLES.md](ENVIRONMENT_VARIABLES.md): curated rollback, - fail-closed, and diagnostic environment switches for Metal, CUDA, and ROCm. +- [ENVIRONMENT_VARIABLES.md](ENVIRONMENT_VARIABLES.md): complete runtime, + test, and tooling environment-variable inventory, with a curated quick + reference for supported rollback, fail-closed, and diagnostic switches. - [gguf-tools/README.md](gguf-tools/README.md): offline GGUF generation, imatrix collection, quantization tooling, and quality checks. - [gguf-tools/imatrix/README.md](gguf-tools/imatrix/README.md): how the diff --git a/scripts/environment_variables.tsv b/scripts/environment_variables.tsv new file mode 100644 index 000000000..81de9aa80 --- /dev/null +++ b/scripts/environment_variables.tsv @@ -0,0 +1,1195 @@ +SCOPE NAME VALUE_DEFAULT_SEMANTICS PURPOSE SOURCE +external/system GGML_CUDA_DISABLE_GRAPHS Pure presence flag cached on first is_enabled() call; any defined value, including empty or "0", disables. Unset permits CUDA graphs unless the GPU architecture independently disables them; relevant only when compiled with USE_CUDA_GRAPH. Disable vendored GGML CUDA graph capture/replay and execute through the ordinary CUDA path. cuda/mmq/common.cuh:1208 +external/system HOME Filesystem directory string. In ds4-agent, unset or empty falls back to "." for the default cache and history roots; the web helper applies the same fallback for its browser profile. A nonempty value roots .ds4/kvcache, .ds4_agent_history and .ds4/browser. Choose the user's persistent ds4-agent cache, line-history and Chrome-profile base directory. ds4_agent.c:4023 +external/system LINENOISE_ASSUME_TTY Pure presence test flag: any defined value, including empty or "0", treats non-TTY input as interactive while skipping real termios raw-mode setup/restoration. Unset follows isatty and uses plain blocking line input for pipes. Exercise the live linenoise/agent editor, prompt and status layout in automated pipe-based tests without a real terminal. linenoise.c:593 +external/system LINENOISE_COLS If defined, its value is returned directly through atoi with no validation: empty/nonnumeric becomes 0 and signed values are accepted. If unset, linenoise uses TIOCGWINSZ, then a cursor-position query, then fallback width 80. Force a deterministic terminal column count for linenoise wrapping/layout tests. linenoise.c:684 +external/system PATH Colon-separated executable search directories consulted only after DS4_CHROME, macOS app paths and fixed Chrome/Chromium paths fail. The first executable google-chrome, google-chrome-stable, chromium or chromium-browser wins; unset/empty/no match falls back to the literal command google-chrome, which execlp may search again. Locate a Chrome/Chromium executable for the ds4 web/CDP tool. ds4_web.c:992 +external/system TERM Case-insensitive terminal name. Values dumb, cons25, or emacs select linenoise's simple prompt plus blocking line reader; unset, empty, or any other value selects the normal interactive editor when stdin is a TTY. Avoid ANSI/raw interactive editing on terminal types known not to support the required escape sequences. linenoise.c:559 +runtime/bench DS4_BENCH_DISABLE_SNAPSHOT presence flag; unset allows snapshots for eligible frontiers Disable benchmark state snapshots. ds4_bench.c:709 +runtime/bench DS4_BENCH_FORCE_SNAPSHOT presence flag; unset obeys the normal size/eligibility checks Force benchmark state snapshots despite the normal limit. ds4_bench.c:712 +runtime/bench DS4_BENCH_SNAPSHOT_MAX_BYTES unsigned bytes or unlimited/inf; default DS4_BENCH_DEFAULT_SNAPSHOT_MAX_BYTES Limit session snapshot size during benchmark sweeps. ds4_bench.c:70 +runtime/cli DS4_CLI_FORCE_SESSION Pure presence flag: any defined value, including empty or "0", forces the session path. Unset uses the session path only for distributed coordinators, TP leaders, temperature>0, or MTP depth>1; otherwise the CLI calls direct argmax generation. Force ordinary CLI generation through run_sampled_generation/session APIs so single-node validation follows the same stateful path as TP/distributed runs. ds4_cli.c:1226 +runtime/core DS4_BATCHED_FFN Pure presence flag: any defined value, including empty or "0", enables. Unset leaves the default shared-expert-batched FFN path (or its configured fallback). It is read only by CPU layer-major prefill and takes precedence over shared-batch and token-parallel FFN choices. Run the complete CPU prefill FFN in chunks through layer_ffn_batch instead of the default shared-expert-only batched path. ds4.c:14238 +runtime/core DS4_BATCHED_ROPE_MAX Nonempty value parsed by strtol without full-string validation; integers 0..65536 are accepted, otherwise default 4096. Zero disables batched RoPE for every nonempty prompt. Effective only when prefix batch attention is selected and DS4_NO_BATCHED_ROPE is absent. Set the largest CPU prefix-prefill token batch that applies RoPE and inverse RoPE with the batched kernels. ds4.c:13738 +runtime/core DS4_DECODE_PROFILE_DETAIL presence flag; unset=off Print per-stage timing for the single-token CPU FFN path. ds4.c:12036 +runtime/core DS4_EXPERT_HOTLIST nonempty filesystem path; unset=off; currently Metal-only Load an expert hotlist for Metal expert profiling/streaming. ds4.c:60179 +runtime/core DS4_EXPERT_PROFILE presence diagnostic flag; unset=off Collect timing/profile diagnostics for expert profile. ds4.c:60177 +runtime/core DS4_LOCK_FILE path string; default /tmp/ds4.lock Override the single-instance lock file. ds4.c:51809 +runtime/core DS4_NO_BATCHED_ATTN presence rollback flag; unset keeps default/optimized path Disable/roll back no batched attn. ds4.c:14237 +runtime/core DS4_NO_BATCHED_ROPE presence rollback flag; unset keeps default/optimized path Disable/roll back no batched rope. ds4.c:13745 +runtime/core DS4_NO_PARALLEL_ATTN_ROWS presence rollback flag; unset keeps default/optimized path Disable/roll back no parallel attn rows. ds4.c:13731 +runtime/core DS4_NO_ROUTED_TOKEN_PARALLEL presence rollback flag; unset keeps default/optimized path Disable/roll back no routed token parallel. ds4.c:12415 +runtime/core DS4_NO_SHARED_BATCH_FFN presence rollback flag; unset keeps default/optimized path Disable/roll back no shared batch ffn. ds4.c:14240 +runtime/core DS4_ORACLE_LOGITS filesystem path; unset disables read/write Load reference logits for graph correctness comparison. ds4.c:38866 +runtime/core DS4_PARALLEL_ATTN_ROWS Pure presence opt-in; any defined value enables the preference, but DS4_NO_PARALLEL_ATTN_ROWS overrides it. The path is eligible only for prefix prefill with cache n_raw==0 and pos0==0; unset uses per-token attention rows. Batch/parallelize CPU prefix attention-row evaluation after cache/index preparation. ds4.c:13728 +runtime/core DS4_PARALLEL_FFN Pure presence opt-in. It is effective only in CPU prefill when batched attention is enabled, DS4_BATCHED_FFN is absent, and the default shared-batch path has been disabled with DS4_NO_SHARED_BATCH_FFN; otherwise higher-priority paths win. Run independent prompt-token FFNs through layer_ffn_tokens_parallel as the fallback after disabling shared batching. ds4.c:14239 +runtime/core DS4_PREFILL_BATCH Nonempty value parsed by strtol without full-string validation; accepted range 1..4095, default 128 for unset/invalid/out-of-range values. It is used only when DS4_BATCHED_FFN selects full batched CPU FFN. Set the token chunk size for layer_ffn_batch during CPU layer-major prefill. ds4.c:14241 +runtime/core DS4_PREFILL_PROFILE_DETAIL presence flag; unset=off Print detailed per-stage CPU prefill timing. ds4.c:12382 +runtime/core DS4_PREFILL_PROFILE_TOKEN presence flag; effective within detailed prefill profiling Print token-loop substage timings during CPU prefill. ds4.c:13957 +runtime/core DS4_ROUTED_TOKEN_PARALLEL Pure presence flag that forces token-parallel routed MoE, even if DS4_NO_ROUTED_TOKEN_PARALLEL is also set. When unset, token parallelism is automatic for n_tok>=64 unless the NO flag is present; smaller batches use per-token routed MoE. Choose token-parallel CPU routed-expert evaluation inside the default shared-batch FFN prefill path. ds4.c:12414 +runtime/core DS4_THREADS positive integer; default min(online CPUs,12), capped by DS4_MAX_THREADS; CLI thread request overrides env Set CPU worker-pool size. ds4.c:1874 +runtime/core DS4_TOKEN_TIMING Pure presence flag; default off. Any defined value times and prints each CPU token decode evaluation; sampling, emission and callbacks are outside the measured interval. Report per-token CPU model-evaluation latency during direct argmax generation. ds4.c:41429 +runtime/core DS4_TRACE_TOP presence flag; unset=off Print top-logit/token trace data during CPU generation. ds4.c:41398 +runtime/cpu DS4_CPU_DISABLE_UNROLLED_ARGMAX presence rollback flag; unset keeps optimized/default path Disable/roll back cpu disable unrolled argmax. ds4.c:40885 +runtime/cpu DS4_CPU_DUMP_LOGITS filesystem path; unset disables read/write Dump or select diagnostic data for cpu dump logits. ds4.c:38896 +runtime/cpu DS4_CPU_DUMP_PREFILL_LOGITS filesystem path; unset disables read/write Dump or select diagnostic data for cpu dump prefill logits. ds4.c:41416 +runtime/cuda DS4_CUDA_ATTENTION_OUTPUT_A_CUBLAS_MIN integer tokens; default 2; accepted range 2..4095, otherwise 2 Set the token-count threshold for using cuBLAS on attention output-A. ds4_cuda.cu:23925 +runtime/cuda DS4_CUDA_ATTENTION_OUTPUT_PRELOAD presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Allow attention-output Q8 weights to be preloaded into the selective F16 cache. ds4_cuda.cu:2402 +runtime/cuda DS4_CUDA_ATTN_OUTPUT_PROFILE presence diagnostic flag; default off; any defined value including 0 enables Measure and print CUDA attention-output stage timings. ds4_cuda.cu:23910 +runtime/cuda DS4_CUDA_ATTN_Q_B_F32_CACHE presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Enable an F32-derived-weight cache for attention Q-B weights. ds4_cuda.cu:2414 +runtime/cuda DS4_CUDA_BUILD_ARTIFACTS boolean-ish, default on for eligible derived artifacts; only exact 0 disables Control construction of eligible CUDA derived/repacked weight artifacts. ds4_cuda.cu:8440 +runtime/cuda DS4_CUDA_COPY_MODEL nonempty-string opt-in (but mere presence also suppresses prefetch); default off; value 0 is nonempty and requests a full copy Copy the complete mapped model image into device memory. ds4_cuda.cu:2740 +runtime/cuda DS4_CUDA_COPY_MODEL_CHUNKED presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Use range-by-range model prefetch/copy preparation instead of the normal bulk preparation. ds4_cuda.cu:37720 +runtime/cuda DS4_CUDA_DECODE_GRAPHS boolean, default on; any value starting with 0, or exact off/no/false in listed case variants, disables; oracle flags force off; effective only on one GPU Control CUDA Graph capture and replay for decode. ds4_cuda.cu:1468 +runtime/cuda DS4_CUDA_DECODE_GRAPH_LOG presence diagnostic flag; default off; any defined value including 0 enables Log CUDA decode-graph cache misses, capture failures, and lifecycle events. ds4_cuda.cu:1580 +runtime/cuda DS4_CUDA_DECODE_HEADS8_ONLINE presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Force the eight-head online CUDA decode-attention kernel when eligible. ds4_cuda.cu:371 +runtime/cuda DS4_CUDA_DECODE_SCORE4 presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Select four score lanes in the CUDA decode-attention fallback kernel. ds4_cuda.cu:372 +runtime/cuda DS4_CUDA_DECODE_SCORE8 presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Select eight score lanes in the CUDA decode-attention fallback kernel. ds4_cuda.cu:373 +runtime/cuda DS4_CUDA_DIRECT_MODEL mixed presence/nonempty flag, default off; any defined value bypasses host caching, while backend direct lookup requires nonempty; value 0 therefore still changes behavior Use the mapped model directly and bypass selective CUDA weight caching. ds4.c:3058; ds4_cuda.cu:1250 +runtime/cuda DS4_CUDA_DISABLE_DSPARK_EXACTN value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Disable the CUDA DSpark exactn optimization. ds4.c:52080 +runtime/cuda DS4_CUDA_DISABLE_DSPARK_EXACTN_BATCH_HEAD value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Disable the CUDA DSpark exactn batch head optimization. ds4.c:37095 +runtime/cuda DS4_CUDA_DISABLE_DSPARK_EXACTN_GRAPHS value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Disable the CUDA DSpark exactn graphs optimization. ds4.c:37061 +runtime/cuda DS4_CUDA_DISABLE_DSPARK_NONCAUSAL_ONLINE value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on Disable the noncausal online-attention DSpark experiment. ds4_cuda.cu:21691 +runtime/cuda DS4_CUDA_DISABLE_HC_NORM_MIX_FUSE presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable fused HC RMSNorm-plus-mix. ds4_cuda.cu:20905 +runtime/cuda DS4_CUDA_DISABLE_HC_SPLIT_NORM_FUSED presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the fused HC split/weighted-sum/norm kernel. ds4_cuda.cu:31785 +runtime/cuda DS4_CUDA_DISABLE_IQ2_XXS_SSD_PREFILL_MMQ false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Disable the CUDA IQ2 XXS SSD prefill MMQ optimization/path. ds4_cuda.cu:4627 +runtime/cuda DS4_CUDA_DISABLE_Q4_ATTN_OUT_HC_FUSE presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable fused Q4 attention-output/HC expansion. ds4_cuda.cu:37337 +runtime/cuda DS4_CUDA_DISABLE_Q4_DENSE_PAIR presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q4 dense pair CUDA Q4 optimization. ds4_cuda.cu:20402 +runtime/cuda DS4_CUDA_DISABLE_Q8_HC_EXPAND_FUSED false-like-aware flag, default off; 0/false/no/off is off, other nonempty values request split; force-fused wins Request the split Q8 shared-down/HC path when safe. ds4_cuda.cu:2086 +runtime/cuda DS4_CUDA_DISABLE_QKV_RMS_FUSED presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the CUDA QKV RMS fused optimization/path. ds4_cuda.cu:369; ds4.c:17255 +runtime/cuda DS4_CUDA_DISABLE_SHARED_GATE_UP_PAIR presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the CUDA shared gate up pair optimization/path. ds4_cuda.cu:24293 +runtime/cuda DS4_CUDA_DISABLE_STREAMING_EXPERT_PERSISTENT_CACHE false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Disable streaming expert persistent cache in CUDA SSD streaming. ds4_cuda.cu:4113 +runtime/cuda DS4_CUDA_DISABLE_STREAMING_PREFILL_BATCH_SELECTED_ADDR presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable streaming prefill batch selected addr in CUDA SSD streaming. ds4.c:18419 +runtime/cuda DS4_CUDA_DISABLE_STREAMING_PREFILL_BATCH_SELECTED_LOAD presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable streaming prefill batch selected load in CUDA SSD streaming. ds4.c:21744 +runtime/cuda DS4_CUDA_DISABLE_STREAMING_SELECTED_BATCH_IO false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Disable streaming selected batch I/O in CUDA SSD streaming. ds4_cuda.cu:4734 +runtime/cuda DS4_CUDA_DISABLE_STREAMING_SELECTED_EVENT_PIPELINE false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Disable streaming selected event pipeline in CUDA SSD streaming. ds4_cuda.cu:4866 +runtime/cuda DS4_CUDA_DISABLE_STREAMING_SELECTED_SHARED_OVERLAP presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable streaming selected shared overlap in CUDA SSD streaming. ds4.c:20796 +runtime/cuda DS4_CUDA_DSPARK_DEVICE_PROPOSER value-aware opt-in, default off; 0/off/no/false (lowercase only) disable; other nonempty enables unless rollback set Enable the CUDA-resident DSpark proposer. ds4.c:34699; ds4_cuda.cu:19035 +runtime/cuda DS4_CUDA_DSPARK_EXACT2 value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Enable the exact two-draft CUDA DSpark support path. ds4.c:52057 +runtime/cuda DS4_CUDA_DSPARK_EXACTN value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Enable the exact multi-draft CUDA DSpark support path. ds4.c:52078 +runtime/cuda DS4_CUDA_DSPARK_EXACTN_BATCH_HEAD value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Enable the batched output-head stage for exact-N DSpark verification. ds4.c:37093 +runtime/cuda DS4_CUDA_DSPARK_EXACTN_GRAPHS value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Enable CUDA Graph capture for stable exact-N DSpark islands. ds4.c:37059 +runtime/cuda DS4_CUDA_DSPARK_NO_DEVICE_PROPOSER presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Disable the CUDA-resident DSpark proposer. ds4.c:34709; ds4_cuda.cu:19040 +runtime/cuda DS4_CUDA_DSPARK_NO_PADDED_HEAD presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Disable the padded CUDA output-head optimization used by DSpark. ds4.c:34057 +runtime/cuda DS4_CUDA_DSPARK_NO_Q_NORM_ROPE_FUSION presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Disable fused Q RMSNorm plus RoPE in DSpark support stages. ds4.c:33433 +runtime/cuda DS4_CUDA_DSPARK_PROPOSER_BLOCK_MAX integer 0..UINT32_MAX; 0/invalid keeps native size; unset uses auto caps for exact-N/exact2; positive values cap the block and are limited by DS4_DSPARK_MAX_BLOCK_SIZE Cap the CUDA DSpark proposal block length. ds4.c:52164 +runtime/cuda DS4_CUDA_DSPARK_TINY_ALIGNED_VEC value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on Use aligned routed-MoE vector kernels for tiny DSpark batches. ds4_cuda.cu:29523 +runtime/cuda DS4_CUDA_ENABLE_DSPARK_NONCAUSAL_ONLINE value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on Enable the small-batch noncausal online-attention DSpark experiment. ds4_cuda.cu:21690 +runtime/cuda DS4_CUDA_ENABLE_HC_NORM_MIX_FUSE nonempty opt-in, default off; only exact 0 disables; the F32/F16 activation mode follows the selected standalone matmul path; disable/serial/alternate flags can veto Enable and select the fused HC RMSNorm-plus-mix one-token implementation. ds4_cuda.cu:20902 +runtime/cuda DS4_CUDA_ENABLE_IQ2_XXS_SSD_PREFILL_MMQ false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Enable the CUDA IQ2 XXS SSD prefill MMQ experimental path. ds4_cuda.cu:4625 +runtime/cuda DS4_CUDA_ENABLE_Q4_ATTN_OUT_HC_FUSE value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on Opt in to the fused Q4 attention-output/HC expansion path. ds4_cuda.cu:37355 +runtime/cuda DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_BATCH value-aware opt-in, default off; nonempty value other than exact 0 enables; rollback wins Enable flattened grouped attention-A MMQ for two-to-eight-token GB10 batches. cuda/mmq/ds4_mmq.cu:4096 +runtime/cuda DS4_CUDA_ENABLE_Q4_K1024_PERSISTENT presence flag, default off; any defined value including 0 requests the path; rollback wins Enable the GB10 persistent-CTA kernel for M=32768, N=1, K=1024 Q4. cuda/mmq/ds4_mmq.cu:3698 +runtime/cuda DS4_CUDA_ENABLE_Q8_FOLD strict flag, default off; only exact value 1 enables; overridden by DS4_CUDA_NO_Q8_FOLD Enable one-shot producer-to-consumer reuse of freshly quantized Q8_1 data. ds4_cuda.cu:785 +runtime/cuda DS4_CUDA_ENABLE_STREAMING_EXPERT_PERSISTENT_CACHE false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Enable streaming expert persistent cache in CUDA SSD streaming. ds4_cuda.cu:4111 +runtime/cuda DS4_CUDA_ENABLE_STREAMING_SELECTED_BATCH_IO false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Enable streaming selected batch I/O in CUDA SSD streaming. ds4_cuda.cu:4732 +runtime/cuda DS4_CUDA_ENABLE_STREAMING_SELECTED_EVENT_PIPELINE false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Enable streaming selected event pipeline in CUDA SSD streaming. ds4_cuda.cu:4864 +runtime/cuda DS4_CUDA_END_STREAM_SYNC presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Synchronize only CUDA stream 0 at command-batch end instead of synchronizing the whole device. ds4_cuda.cu:376 +runtime/cuda DS4_CUDA_EXACT_SCORE_SPLIT_CHUNK integer scores/chunk; default 512; clamped 1..8192 Tune exact score split chunk for exact score-split CUDA decode attention. ds4_cuda.cu:13669 +runtime/cuda DS4_CUDA_EXACT_SCORE_SPLIT_DECODE value-aware boolean, default on; exact 0 disables; a nonzero explicit setting also takes precedence over split-KV selection Control the exact score-split decode-attention implementation. ds4_cuda.cu:13631 +runtime/cuda DS4_CUDA_EXACT_SCORE_SPLIT_DIM2 presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Tune exact score split dim2 for exact score-split CUDA decode attention. ds4_cuda.cu:387 +runtime/cuda DS4_CUDA_EXACT_SCORE_SPLIT_FUSE_INV_ROPE presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Tune exact score split fuse inv rope for exact score-split CUDA decode attention. ds4_cuda.cu:390 +runtime/cuda DS4_CUDA_EXACT_SCORE_SPLIT_GRAPH presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Tune exact score split graph for exact score-split CUDA decode attention. ds4_cuda.cu:379 +runtime/cuda DS4_CUDA_EXACT_SCORE_SPLIT_LDG presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Tune exact score split ldg for exact score-split CUDA decode attention. ds4_cuda.cu:381 +runtime/cuda DS4_CUDA_EXACT_SCORE_SPLIT_MIN_SCORE integer score count; default 1; clamped 0..8192 Set the minimum visible-score count for exact score-split decode. ds4_cuda.cu:13665 +runtime/cuda DS4_CUDA_EXACT_SCORE_SPLIT_S integer exact split count; unset/invalid = automatic; valid value clamped 1..16 Tune exact score split s for exact score-split CUDA decode attention. ds4_cuda.cu:13679 +runtime/cuda DS4_CUDA_EXACT_SCORE_SPLIT_S_FLOOR integer split count; default 6; clamped 1..16 Tune exact score split s floor for exact score-split CUDA decode attention. ds4_cuda.cu:13672 +runtime/cuda DS4_CUDA_EXACT_SCORE_SPLIT_S_MAX integer split count; default 16; clamped 1..16 Tune exact score split s max for exact score-split CUDA decode attention. ds4_cuda.cu:13675 +runtime/cuda DS4_CUDA_EXACT_SCORE_SPLIT_VEC4 presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Tune exact score split vec4 for exact score-split CUDA decode attention. ds4_cuda.cu:383 +runtime/cuda DS4_CUDA_EXACT_SCORE_SPLIT_VEC4_PLAIN presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Tune exact score split vec4 plain for exact score-split CUDA decode attention. ds4_cuda.cu:385 +runtime/cuda DS4_CUDA_F16_CUBLAS_ONE presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Control the F16 cuBLAS one CUDA F16 matmul path. ds4_cuda.cu:20852 +runtime/cuda DS4_CUDA_F16_SMALL_BATCH presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Control the F16 small batch CUDA F16 matmul path. ds4_cuda.cu:20837 +runtime/cuda DS4_CUDA_F16_SMALL_OUT presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Control the F16 small out CUDA F16 matmul path. ds4_cuda.cu:20819 +runtime/cuda DS4_CUDA_GLM_VERIFY_NO_Q8_TOK2 presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Control or tune the CUDA glm verify no Q8 tok2 path. ds4_cuda.cu:19824 +runtime/cuda DS4_CUDA_GREEDY_SPLITKV value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Enable greedy split-KV fast attention. ds4.c:17062 +runtime/cuda DS4_CUDA_GREEDY_SPLITKV_FALLBACK_LOG presence diagnostic flag; default off; any defined value including 0 enables Control greedy splitkv fallback log in CUDA greedy fast decode. ds4.c:55473 +runtime/cuda DS4_CUDA_GREEDY_SPLITKV_MARGIN nonnegative finite float; default 0.25; invalid value warns and uses 0.25; 0 disables margin fallback Control greedy splitkv margin in CUDA greedy fast decode. ds4.c:17140 +runtime/cuda DS4_CUDA_GREEDY_SPLITKV_MAX_SEGMENT integer 0..INT32_MAX; default/invalid 0 (segment cap disabled) Control greedy splitkv max segment in CUDA greedy fast decode. ds4.c:17216 +runtime/cuda DS4_CUDA_GREEDY_SPLITKV_PAIR_REPLAY value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Replay greedy split-KV tokens in pairs. ds4.c:17190 +runtime/cuda DS4_CUDA_GREEDY_SPLITKV_TOP2 value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Use top-2 output margins with greedy split-KV. ds4.c:17172 +runtime/cuda DS4_CUDA_GREEDY_SPLITKV_TRACE presence diagnostic flag; default off; any defined value including 0 enables Control greedy splitkv trace in CUDA greedy fast decode. ds4.c:55508 +runtime/cuda DS4_CUDA_GREEDY_SPLITKV_TRUST_REPLAY value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Trust replayed greedy split-KV results without the normal confirmation policy. ds4.c:17180 +runtime/cuda DS4_CUDA_GREEDY_SPLIT_TOP1 value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Enable split top-1 selection in greedy CUDA decode. ds4.c:17034 +runtime/cuda DS4_CUDA_GREEDY_TOP1 value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control greedy top1 in CUDA greedy fast decode. ds4.c:55936 +runtime/cuda DS4_CUDA_GREEDY_VEC4 value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Enable greedy vec4 fast attention. ds4.c:17072 +runtime/cuda DS4_CUDA_GREEDY_VEC4_FALLBACK_LOG presence diagnostic flag; default off; any defined value including 0 enables Control greedy vec4 fallback log in CUDA greedy fast decode. ds4.c:55474 +runtime/cuda DS4_CUDA_GREEDY_VEC4_MARGIN nonnegative finite float; default 0.25; invalid value warns and uses 0.25; 0 disables margin fallback Control greedy vec4 margin in CUDA greedy fast decode. ds4.c:17110 +runtime/cuda DS4_CUDA_GREEDY_VEC4_MAX_SEGMENT integer 0..INT32_MAX; default/invalid 0 (segment cap disabled) Control greedy vec4 max segment in CUDA greedy fast decode. ds4.c:17224 +runtime/cuda DS4_CUDA_GREEDY_VEC4_TRACE presence diagnostic flag; default off; any defined value including 0 enables Control greedy vec4 trace in CUDA greedy fast decode. ds4.c:55537 +runtime/cuda DS4_CUDA_INDEXED_TWOPASS presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Force the two-pass indexed-attention path instead of the fused heads8 online kernel. ds4_cuda.cu:23587 +runtime/cuda DS4_CUDA_IQ2_XXS_SSD_PREFILL_MMQ_STATS false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Print CUDA IQ2 XXS SSD prefill MMQ counters. ds4_cuda.cu:4633 +runtime/cuda DS4_CUDA_KEEP_MODEL_PAGES presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Keep source model pages resident instead of advising the OS to discard copied pages. ds4_cuda.cu:2840 +runtime/cuda DS4_CUDA_MIXED_PREFILL_DECODE value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control native mixed prefill/decode scheduling. ds4.c:70511 +runtime/cuda DS4_CUDA_MIXED_ROUTED_MAX_PREFILL integer rows 0..UINT32_MAX; default/invalid 512 Set the maximum prefill rows admitted to the mixed routed-MoE path. ds4.c:70000 +runtime/cuda DS4_CUDA_MIXED_ROUTED_SCATTER value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Use scattered row handling in mixed routed-MoE execution. ds4.c:69461 +runtime/cuda DS4_CUDA_MMQ boolean-ish, default on; any value beginning with 0 disables; quality mode and multi-GPU disable normal MMQ tier (MXFP4 path differs) Control the vendored CUDA MMQ prefill tier. ds4_cuda.cu:1722 +runtime/cuda DS4_CUDA_MMQ_Q81_PERSISTENT strict boolean, default off; accepts 1/on/true/yes and 0/off/false/no in listed lower/upper-case forms; unknown values are off Reuse a persistent Q8_1 MMQ scratch arena on supported GB10 devices. cuda/mmq/ds4_mmq.cu:144 +runtime/cuda DS4_CUDA_MMQ_X_MAX integer >=8; rounded down to multiple of 8 and only lowers the hardware base; invalid/unset = hardware base Cap the MMQ X tile-width selector for architecture tuning. cuda/mmq/mmq.cuh:127 +runtime/cuda DS4_CUDA_MODEL_COPY_CHUNK_MB positive integer MiB; default 64; clamped 16..4096 Set the chunk size used for CUDA model copying. ds4_cuda.cu:2827 +runtime/cuda DS4_CUDA_MODEL_COPY_VERBOSE presence diagnostic flag; default off; any defined value including 0 enables Print periodic progress while copying the model to device memory. ds4_cuda.cu:6537 +runtime/cuda DS4_CUDA_MODEL_PREFETCH_SYNC presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Synchronize after each CUDA model prefetch range for diagnostics. ds4_cuda.cu:2808 +runtime/cuda DS4_CUDA_MOE_ATOMIC_DOWN presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Select or tune the atomic down variant in CUDA routed-MoE dispatch. ds4_cuda.cu:30086 +runtime/cuda DS4_CUDA_MOE_DECODE_GRAPH presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Select or tune the decode graph variant in CUDA routed-MoE dispatch. ds4_cuda.cu:391 +runtime/cuda DS4_CUDA_MOE_DIRECT_MIDQ presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Select or tune the direct midq variant in CUDA routed-MoE dispatch. ds4_cuda.cu:30140 +runtime/cuda DS4_CUDA_MOE_DOWN_ROW1024 presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Select or tune the down row1024 variant in CUDA routed-MoE dispatch. ds4_cuda.cu:30109 +runtime/cuda DS4_CUDA_MOE_DOWN_ROW128 presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Select or tune the down row128 variant in CUDA routed-MoE dispatch. ds4_cuda.cu:30128 +runtime/cuda DS4_CUDA_MOE_DOWN_ROW2048 presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Select or tune the down row2048 variant in CUDA routed-MoE dispatch. ds4_cuda.cu:30110 +runtime/cuda DS4_CUDA_MOE_DOWN_ROW256 presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Select or tune the down row256 variant in CUDA routed-MoE dispatch. ds4_cuda.cu:30127 +runtime/cuda DS4_CUDA_MOE_DOWN_ROW512 presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Select or tune the down row512 variant in CUDA routed-MoE dispatch. ds4_cuda.cu:30108 +runtime/cuda DS4_CUDA_MOE_DOWN_ROW64 presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Select or tune the down row64 variant in CUDA routed-MoE dispatch. ds4_cuda.cu:30129 +runtime/cuda DS4_CUDA_MOE_GATE_ROW1024 presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Select or tune the gate row1024 variant in CUDA routed-MoE dispatch. ds4_cuda.cu:30120 +runtime/cuda DS4_CUDA_MOE_GATE_ROW128 presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Select or tune the gate row128 variant in CUDA routed-MoE dispatch. ds4_cuda.cu:30093 +runtime/cuda DS4_CUDA_MOE_GATE_ROW2048 presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Select or tune the gate row2048 variant in CUDA routed-MoE dispatch. ds4_cuda.cu:30091 +runtime/cuda DS4_CUDA_MOE_GATE_ROW256 presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Select or tune the gate row256 variant in CUDA routed-MoE dispatch. ds4_cuda.cu:30092 +runtime/cuda DS4_CUDA_MOE_MIDQ_SIDECAR presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Select or tune the midq sidecar variant in CUDA routed-MoE dispatch. ds4_cuda.cu:30169 +runtime/cuda DS4_CUDA_MOE_NO_ATOMIC_DOWN presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Disable the atomic down variant in CUDA routed-MoE dispatch. ds4_cuda.cu:30087 +runtime/cuda DS4_CUDA_MOE_NO_DECODE_LUT_GATE presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Disable the decode lut gate variant in CUDA routed-MoE dispatch. ds4_cuda.cu:30117 +runtime/cuda DS4_CUDA_MOE_NO_DIRECT_DOWN_SUM6 presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Disable the direct down sum6 variant in CUDA routed-MoE dispatch. ds4_cuda.cu:30137 +runtime/cuda DS4_CUDA_MOE_NO_DIRECT_MIDQ presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Disable the direct midq variant in CUDA routed-MoE dispatch. ds4_cuda.cu:30141 +runtime/cuda DS4_CUDA_MOE_NO_DOWN_ROW128 presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Disable the down row128 variant in CUDA routed-MoE dispatch. ds4_cuda.cu:30133 +runtime/cuda DS4_CUDA_MOE_NO_DOWN_ROW2048 presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Disable the down row2048 variant in CUDA routed-MoE dispatch. ds4_cuda.cu:30131 +runtime/cuda DS4_CUDA_MOE_NO_DOWN_ROW256 presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Disable the down row256 variant in CUDA routed-MoE dispatch. ds4_cuda.cu:30132 +runtime/cuda DS4_CUDA_MOE_NO_DOWN_ROW64 presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Disable the down row64 variant in CUDA routed-MoE dispatch. ds4_cuda.cu:30134 +runtime/cuda DS4_CUDA_MOE_NO_DOWN_TILE16 presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Disable the down tile16 variant in CUDA routed-MoE dispatch. ds4_cuda.cu:30102 +runtime/cuda DS4_CUDA_MOE_NO_EXPERT_TILES presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Disable the expert tiles variant in CUDA routed-MoE dispatch. ds4_cuda.cu:30062 +runtime/cuda DS4_CUDA_MOE_NO_GATE_ROW128 presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Disable the gate row128 variant in CUDA routed-MoE dispatch. ds4_cuda.cu:30097 +runtime/cuda DS4_CUDA_MOE_NO_GATE_ROW2048 presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Disable the gate row2048 variant in CUDA routed-MoE dispatch. ds4_cuda.cu:30095 +runtime/cuda DS4_CUDA_MOE_NO_GATE_ROW256 presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Disable the gate row256 variant in CUDA routed-MoE dispatch. ds4_cuda.cu:30096 +runtime/cuda DS4_CUDA_MOE_NO_IQ2_ALIGNED value-aware kill switch, default off; nonempty value other than exact 0 disables aligned IQ2 path Disable the IQ2 aligned variant in CUDA routed-MoE dispatch. ds4_cuda.cu:1011 +runtime/cuda DS4_CUDA_MOE_NO_MIDQ_SIDECAR presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Disable the midq sidecar variant in CUDA routed-MoE dispatch. ds4_cuda.cu:30170 +runtime/cuda DS4_CUDA_MOE_NO_OWNED_SPARSE_BUFFERS presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Disable the owned sparse buffers variant in CUDA routed-MoE dispatch. ds4_cuda.cu:30089 +runtime/cuda DS4_CUDA_MOE_NO_P2 presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Disable the p2 variant in CUDA routed-MoE dispatch. ds4_cuda.cu:30084 +runtime/cuda DS4_CUDA_MOE_NO_Q2K_ALIGNED value-aware kill switch, default off; nonempty value other than exact 0 disables aligned Q2_K path Disable the q2k aligned variant in CUDA routed-MoE dispatch. ds4_cuda.cu:1016 +runtime/cuda DS4_CUDA_MOE_NO_Q4_DOWN_ROWSPAN presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Disable the Q4 down rowspan variant in CUDA routed-MoE dispatch. ds4_cuda.cu:30114 +runtime/cuda DS4_CUDA_MOE_NO_Q4_DOWN_SLOT3 presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Disable the Q4 down slot3 variant in CUDA routed-MoE dispatch. ds4_cuda.cu:30164 +runtime/cuda DS4_CUDA_MOE_NO_Q4_GATE_H16 presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Disable the Q4 gate H16 variant in CUDA routed-MoE dispatch. ds4_cuda.cu:30149 +runtime/cuda DS4_CUDA_MOE_NO_Q4_GATE_H16R8 presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Disable the Q4 gate H16R8 variant in CUDA routed-MoE dispatch. ds4_cuda.cu:30145 +runtime/cuda DS4_CUDA_MOE_NO_Q4_GATE_W32 presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Disable the Q4 gate W32 variant in CUDA routed-MoE dispatch. ds4_cuda.cu:30157 +runtime/cuda DS4_CUDA_MOE_NO_Q4_GATE_W32R16 presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Disable the Q4 gate W32R16 variant in CUDA routed-MoE dispatch. ds4_cuda.cu:30153 +runtime/cuda DS4_CUDA_MOE_NO_Q4_GATE_W32_NOAUX presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Disable the Q4 gate W32 no-aux variant in CUDA routed-MoE dispatch. ds4_cuda.cu:30160 +runtime/cuda DS4_CUDA_MOE_NO_Q4_MMA presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Disable the Q4 MMA variant in CUDA routed-MoE dispatch. ds4_cuda.cu:312 +runtime/cuda DS4_CUDA_MOE_NO_Q4_MMA_TILE16 presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Disable the Q4 MMA tile16 variant in CUDA routed-MoE dispatch. ds4_cuda.cu:30100 +runtime/cuda DS4_CUDA_MOE_NO_Q4_SORTED presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Disable the Q4 sorted variant in CUDA routed-MoE dispatch. ds4_cuda.cu:30061 +runtime/cuda DS4_CUDA_MOE_NO_SMALL_SORTED_PREP presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Disable the small sorted prep variant in CUDA routed-MoE dispatch. ds4_cuda.cu:30106 +runtime/cuda DS4_CUDA_MOE_PROFILE presence diagnostic flag; default off; any defined value including 0 enables Measure and print routed-MoE CUDA kernel-stage timings. ds4_cuda.cu:30045 +runtime/cuda DS4_CUDA_MOE_Q4_DOWN_SLOT3 presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Select or tune the Q4 down slot3 variant in CUDA routed-MoE dispatch. ds4_cuda.cu:30163 +runtime/cuda DS4_CUDA_MOE_Q4_GATE_H16 presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Select or tune the Q4 gate H16 variant in CUDA routed-MoE dispatch. ds4_cuda.cu:30148 +runtime/cuda DS4_CUDA_MOE_Q4_GATE_H16R8 presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Select or tune the Q4 gate H16R8 variant in CUDA routed-MoE dispatch. ds4_cuda.cu:30144 +runtime/cuda DS4_CUDA_MOE_Q4_GATE_W32R16 presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Select or tune the Q4 gate W32R16 variant in CUDA routed-MoE dispatch. ds4_cuda.cu:30152 +runtime/cuda DS4_CUDA_MOE_TILE4 presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Select or tune the tile4 variant in CUDA routed-MoE dispatch. ds4_cuda.cu:30063 +runtime/cuda DS4_CUDA_MOE_TILE8 presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Select or tune the tile8 variant in CUDA routed-MoE dispatch. ds4_cuda.cu:30079 +runtime/cuda DS4_CUDA_MOE_WRITE_GATE_UP presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Select or tune the write gate up variant in CUDA routed-MoE dispatch. ds4_cuda.cu:30081 +runtime/cuda DS4_CUDA_NO_ATTENTION_OUTPUT_F16_CACHE presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the attention output F16 cache CUDA F16 path. ds4_cuda.cu:2364 +runtime/cuda DS4_CUDA_NO_ATTN_A_TOK2 presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the CUDA attn a tok2 optimization/path. ds4_cuda.cu:24024 +runtime/cuda DS4_CUDA_NO_ATTN_Q_B_F16_CACHE presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the attn q b F16 cache CUDA F16 path. ds4_cuda.cu:2367 +runtime/cuda DS4_CUDA_NO_COMPRESSOR_PREFILL_BATCH presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the CUDA compressor prefill batch optimization/path. ds4.c:29717 +runtime/cuda DS4_CUDA_NO_CUBLAS_ATTENTION presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the CUDA cuBLAS attention optimization/path. ds4_cuda.cu:23067 +runtime/cuda DS4_CUDA_NO_CUBLAS_ATTENTION_OUTPUT_A presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the CUDA cuBLAS attention output a optimization/path. ds4_cuda.cu:23934 +runtime/cuda DS4_CUDA_NO_DECODE_VALUE512 presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the 512-thread CUDA decode value/finalize specialization. ds4_cuda.cu:374 +runtime/cuda DS4_CUDA_NO_DERIVED_WEIGHTS presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the CUDA derived weights optimization/path. ds4_cuda.cu:1034 +runtime/cuda DS4_CUDA_NO_DIRECT_IO presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the CUDA direct I/O optimization/path. cuda/mmq/ds4_repack.cu:68 +runtime/cuda DS4_CUDA_NO_DIRECT_Q2_PREFILL presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the CUDA direct q2 prefill optimization/path. ds4_cuda.cu:395 +runtime/cuda DS4_CUDA_NO_EXACT_SCORE_SPLIT_DECODE value-aware kill switch, default off; exact 0 is off, other nonempty values disable Disable exact score split decode for exact score-split CUDA decode attention. ds4_cuda.cu:13629; ds4.c:67920 +runtime/cuda DS4_CUDA_NO_EXACT_SCORE_SPLIT_DIM2 presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable exact score split dim2 for exact score-split CUDA decode attention. ds4_cuda.cu:388 +runtime/cuda DS4_CUDA_NO_F16_CUBLAS_BATCH presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the F16 cuBLAS batch CUDA F16 path. ds4_cuda.cu:20854 +runtime/cuda DS4_CUDA_NO_F16_CUBLAS_ONE presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the F16 cuBLAS one CUDA F16 path. ds4_cuda.cu:20851 +runtime/cuda DS4_CUDA_NO_F16_PAIR_COMPRESSOR_STORE presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the F16 pair compressor store CUDA F16 path. ds4_cuda.cu:399 +runtime/cuda DS4_CUDA_NO_F16_PAIR_COMPRESSOR_TRANSPOSE presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the F16 pair compressor transpose CUDA F16 path. ds4_cuda.cu:21343 +runtime/cuda DS4_CUDA_NO_F16_PAIR_COMPRESSOR_TRANSPOSE_PREFETCH8 presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the F16 pair compressor transpose prefetch8 CUDA F16 path. ds4_cuda.cu:21349 +runtime/cuda DS4_CUDA_NO_F16_PAIR_MATMUL presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the F16 pair matmul CUDA F16 path. ds4_cuda.cu:21130 +runtime/cuda DS4_CUDA_NO_F16_SMALL_BATCH presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the F16 small batch CUDA F16 path. ds4_cuda.cu:20838 +runtime/cuda DS4_CUDA_NO_F16_SMALL_OUT presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the F16 small out CUDA F16 path. ds4_cuda.cu:20821 +runtime/cuda DS4_CUDA_NO_FD_CACHE presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the CUDA fd cache optimization/path. ds4_cuda.cu:1274 +runtime/cuda DS4_CUDA_NO_GREEDY_SPLITKV value-aware kill switch; default off; nonempty value other than exact 0 disables Disable greedy splitkv in CUDA greedy fast decode. ds4.c:17060 +runtime/cuda DS4_CUDA_NO_GREEDY_SPLITKV_FALLBACK value-aware kill switch; default off; nonempty value other than exact 0 disables margin fallback Disable greedy splitkv fallback in CUDA greedy fast decode. ds4.c:17160 +runtime/cuda DS4_CUDA_NO_GREEDY_SPLITKV_PAIR_REPLAY value-aware kill switch; default off; nonempty value other than exact 0 disables Disable greedy splitkv pair replay in CUDA greedy fast decode. ds4.c:17188 +runtime/cuda DS4_CUDA_NO_GREEDY_SPLITKV_TOP2 value-aware kill switch; default off; nonempty value other than exact 0 disables Disable greedy splitkv top2 in CUDA greedy fast decode. ds4.c:17170 +runtime/cuda DS4_CUDA_NO_GREEDY_SPLIT_TOP1 value-aware kill switch; default off; nonempty value other than exact 0 disables Disable greedy split top1 in CUDA greedy fast decode. ds4.c:17032 +runtime/cuda DS4_CUDA_NO_GREEDY_VEC4 value-aware kill switch; default off; nonempty value other than exact 0 disables Disable greedy vec4 in CUDA greedy fast decode. ds4.c:17070 +runtime/cuda DS4_CUDA_NO_GREEDY_VEC4_FALLBACK value-aware kill switch; default off; nonempty value other than exact 0 disables margin fallback Disable greedy vec4 fallback in CUDA greedy fast decode. ds4.c:17130 +runtime/cuda DS4_CUDA_NO_HC_SPLIT_NORM_SPLIT4096 presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the split partial-reduction specialization for one-row, 4096-wide HC normalization. ds4_cuda.cu:31826 +runtime/cuda DS4_CUDA_NO_INDEXED_HEADS8 presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the CUDA indexed heads8 optimization/path. ds4_cuda.cu:23586 +runtime/cuda DS4_CUDA_NO_INDEXED_TOPK_SORT presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the CUDA indexed topk sort optimization/path. ds4_cuda.cu:23577 +runtime/cuda DS4_CUDA_NO_INDEXER_DIRECT_ONE presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the indexer direct one CUDA indexer kernel/path. ds4_cuda.cu:18881 +runtime/cuda DS4_CUDA_NO_INDEXER_MXF4 presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the indexer MXF4 CUDA indexer kernel/path. ds4_cuda.cu:17774 +runtime/cuda DS4_CUDA_NO_INDEXER_WMMA presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the indexer WMMA CUDA indexer kernel/path. ds4_cuda.cu:18891 +runtime/cuda DS4_CUDA_NO_INDEXER_WMMA128 presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the indexer wmma128 CUDA indexer kernel/path. ds4_cuda.cu:18892 +runtime/cuda DS4_CUDA_NO_INDEXER_WMMA32 presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the indexer wmma32 CUDA indexer kernel/path. ds4_cuda.cu:18910 +runtime/cuda DS4_CUDA_NO_INDEXER_WMMA64 presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the indexer wmma64 CUDA indexer kernel/path. ds4_cuda.cu:18901 +runtime/cuda DS4_CUDA_NO_IQ2_XXS_SSD_PREFILL_MMQ false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Disable the CUDA IQ2 XXS SSD prefill MMQ optimization/path. ds4_cuda.cu:4629 +runtime/cuda DS4_CUDA_NO_MODEL_COPY presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the CUDA model copy optimization/path. ds4_cuda.cu:6472 +runtime/cuda DS4_CUDA_NO_MODEL_PREFETCH presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the CUDA model prefetch optimization/path. ds4_cuda.cu:2739 +runtime/cuda DS4_CUDA_NO_MOE_DEDUP presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the CUDA MoE dedup optimization/path. cuda/mmq/ds4_mmq.cu:5963 +runtime/cuda DS4_CUDA_NO_ORDERED_F16_MATMUL presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the ordered F16 matmul CUDA F16 path. ds4_cuda.cu:20811 +runtime/cuda DS4_CUDA_NO_PARALLEL_ROUTER_SELECT presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the CUDA parallel router select optimization/path. ds4_cuda.cu:24491 +runtime/cuda DS4_CUDA_NO_Q4_DENSE_SCRATCH presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q4 dense scratch CUDA Q4 optimization. cuda/mmq/ds4_mmq.cu:3779 +runtime/cuda DS4_CUDA_NO_Q4_GB10_FAST presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the GB10-specific Q4 fast-path family. cuda/mmq/ds4_mmq.cu:3701 +runtime/cuda DS4_CUDA_NO_Q4_GROUPED_ATTN_A presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q4 grouped attn a CUDA Q4 optimization. cuda/mmq/ds4_mmq.cu:4088 +runtime/cuda DS4_CUDA_NO_Q4_GROUPED_ATTN_A_BATCH presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q4 grouped attn a batch CUDA Q4 optimization. cuda/mmq/ds4_mmq.cu:4098 +runtime/cuda DS4_CUDA_NO_Q4_K1024_PERSISTENT presence kill switch, default off; any defined value including 0 disables Disable the Q4 K1024 persistent CUDA Q4 optimization. cuda/mmq/ds4_mmq.cu:3700 +runtime/cuda DS4_CUDA_NO_Q8_ALIGNED_DENSE_SCRATCH presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q8 aligned dense scratch CUDA Q8 optimization. cuda/mmq/ds4_mmq.cu:5369 +runtime/cuda DS4_CUDA_NO_Q8_ALIGNED_PERSISTENT presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q8 aligned persistent CUDA Q8 optimization. cuda/mmq/ds4_mmq.cu:5201 +runtime/cuda DS4_CUDA_NO_Q8_BATCH_EXACT_TOK2 presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q8 batch exact tok2 CUDA Q8 optimization. ds4_cuda.cu:19852 +runtime/cuda DS4_CUDA_NO_Q8_BATCH_TOK4 presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q8 batch tok4 CUDA Q8 optimization. ds4_cuda.cu:19805 +runtime/cuda DS4_CUDA_NO_Q8_BATCH_TOK8 presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q8 batch tok8 CUDA Q8 optimization. ds4_cuda.cu:19788 +runtime/cuda DS4_CUDA_NO_Q8_BATCH_WARP presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q8 batch warp CUDA Q8 optimization. ds4_cuda.cu:19787 +runtime/cuda DS4_CUDA_NO_Q8_DP4A presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q8 DP4A CUDA Q8 optimization. ds4_cuda.cu:2391 +runtime/cuda DS4_CUDA_NO_Q8_F16_CACHE presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q8 F16 cache CUDA Q8 optimization. ds4_cuda.cu:2356 +runtime/cuda DS4_CUDA_NO_Q8_F32_CACHE presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q8 F32 cache CUDA Q8 optimization. ds4_cuda.cu:2411 +runtime/cuda DS4_CUDA_NO_Q8_FOLD value-aware kill switch, default off; nonempty value other than exact 0 disables and wins over enable Disable the Q8_1 producer-to-consumer fold. ds4_cuda.cu:786 +runtime/cuda DS4_CUDA_NO_Q8_FUSED_ALIGNED presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q8 fused aligned CUDA Q8 optimization. ds4_cuda.cu:20141 +runtime/cuda DS4_CUDA_NO_Q8_MMA presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q8 MMA CUDA Q8 optimization. ds4_cuda.cu:10781 +runtime/cuda DS4_CUDA_NO_Q8_PAIR_BATCH_EXACT presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q8 pair batch exact CUDA Q8 optimization. ds4_cuda.cu:20302 +runtime/cuda DS4_CUDA_NO_Q8_PAIR_BATCH_EXACT_TOK2 presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q8 pair batch exact tok2 CUDA Q8 optimization. ds4_cuda.cu:20305 +runtime/cuda DS4_CUDA_NO_QKV_KV_ROPE_FUSE value-aware kill switch; default off; nonempty value other than exact 0 disables Disable the CUDA QKV KV rope fuse optimization/path. ds4.c:17253 +runtime/cuda DS4_CUDA_NO_QKV_PAIR presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the CUDA QKV pair optimization/path. ds4.c:17389 +runtime/cuda DS4_CUDA_NO_SCORE_TILE presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the CUDA score tile optimization/path. ds4_cuda.cu:13734; ds4.c:67932 +runtime/cuda DS4_CUDA_NO_SETDEVICE_CACHE presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the cached current-tier shortcut and call cudaSetDevice for every tier selection. ds4_cuda.cu:377 +runtime/cuda DS4_CUDA_NO_SPLITKV_DECODE value-aware kill switch, default off; exact 0/empty is off, other nonempty values disable Disable splitkv decode in CUDA split-KV attention/speculation. ds4_cuda.cu:2211 +runtime/cuda DS4_CUDA_NO_SPLITKV_SPEC value-aware kill switch; default off; nonempty value other than exact 0 disables Disable splitkv spec in CUDA split-KV attention/speculation. ds4.c:17080 +runtime/cuda DS4_CUDA_NO_SPLITKV_SPEC_BATCH_VERIFY value-aware kill switch; default off; nonempty value other than exact 0 disables Disable splitkv spec batch verify in CUDA split-KV attention/speculation. ds4.c:17100 +runtime/cuda DS4_CUDA_NO_SPLITKV_SPEC_TOPONLY_ROW0 value-aware kill switch; default off; nonempty value other than exact 0 disables Disable splitkv spec toponly row0 in CUDA split-KV attention/speculation. ds4.c:17090 +runtime/cuda DS4_CUDA_NO_STREAMING_EXPERT_PERSISTENT_CACHE false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Disable streaming expert persistent cache in CUDA SSD streaming. ds4_cuda.cu:4115 +runtime/cuda DS4_CUDA_NO_STREAMING_SELECTED_BATCH_IO false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Disable streaming selected batch I/O in CUDA SSD streaming. ds4_cuda.cu:4736 +runtime/cuda DS4_CUDA_NO_STREAMING_SELECTED_EVENT_PIPELINE false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Disable streaming selected event pipeline in CUDA SSD streaming. ds4_cuda.cu:4868 +runtime/cuda DS4_CUDA_NO_TF32 presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Use default cuBLAS math instead of TF32 tensor operations. ds4_cuda.cu:6657 +runtime/cuda DS4_CUDA_NO_TOP1 presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the dedicated CUDA indexer top-1 kernel. ds4_cuda.cu:375 +runtime/cuda DS4_CUDA_NO_TOPK1024 presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the topk1024 CUDA indexer kernel/path. ds4_cuda.cu:19290 +runtime/cuda DS4_CUDA_NO_TOPK2048 presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the topk2048 CUDA indexer kernel/path. ds4_cuda.cu:19297 +runtime/cuda DS4_CUDA_NO_TOPK2048_WIDE presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the topk2048 wide CUDA indexer kernel/path. ds4_cuda.cu:19212 +runtime/cuda DS4_CUDA_NO_TOPK8192 presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the topk8192 CUDA indexer kernel/path. ds4_cuda.cu:19335 +runtime/cuda DS4_CUDA_NO_TOPK_CHUNKED presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the topk chunked CUDA indexer kernel/path. ds4_cuda.cu:19374 +runtime/cuda DS4_CUDA_NO_TOPK_STREAM presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the topk stream CUDA indexer kernel/path. ds4_cuda.cu:19366 +runtime/cuda DS4_CUDA_NO_TP_ATTN_OUT_HC_FUSE presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the CUDA TP attn out HC fuse optimization/path. ds4.c:17392 +runtime/cuda DS4_CUDA_NO_VERIFY_DECODE2_SPLIT_TOP1 value-aware kill switch; default off; nonempty value other than exact 0 disables Disable the CUDA verify decode2 split top1 optimization/path. ds4.c:17050 +runtime/cuda DS4_CUDA_NO_WARP_ROUTER_SELECT presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the CUDA warp router select optimization/path. ds4_cuda.cu:24490 +runtime/cuda DS4_CUDA_NO_WINDOW_ATTENTION presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the CUDA window attention optimization/path. ds4_cuda.cu:23050 +runtime/cuda DS4_CUDA_NSYS_PREFILL_START_POS nonempty-string flag, default off; any nonempty value enables MMQ NVTX ranges (the value is not parsed as a position) Enable MMQ NVTX annotations intended for Nsight Systems prefill capture. cuda/mmq/ds4_mmq.cu:48 +runtime/cuda DS4_CUDA_NVTX strict flag, default off; only exact value 1 enables (a nonempty NSYS variable also enables ranges) Enable NVTX ranges around MMQ work. cuda/mmq/ds4_mmq.cu:47 +runtime/cuda DS4_CUDA_OUTPUT_FUSED_TOP1 value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Fuse output projection with top-1 selection in greedy decode. ds4.c:17042 +runtime/cuda DS4_CUDA_PREFILL_PIPELINE boolean, default follows CUDA TP decode; nonempty exact 0 disables, any other nonempty value enables Control the CUDA multi-tier prefill pipeline. ds4.c:17281 +runtime/cuda DS4_CUDA_PREFILL_PIPELINE_MB positive integer rows; default/invalid 512 Set prefill-pipeline microbatch rows. ds4.c:17296 +runtime/cuda DS4_CUDA_PREFILL_PIPELINE_Q8_CACHE value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Keep selective Q8 caches enabled while running the prefill pipeline. ds4.c:17291 +runtime/cuda DS4_CUDA_PREFILL_PIPELINE_SEQUENTIAL presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Execute prefill pipeline stages sequentially for diagnosis. ds4.c:35335 +runtime/cuda DS4_CUDA_PREFILL_PIPELINE_SYNC_BOUNDARY presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Synchronize CUDA at every prefill pipeline tier boundary. ds4.c:35382 +runtime/cuda DS4_CUDA_Q4_ATTN_OUT_HC_ORACLE value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on Compare fused Q4 attention-output/HC expansion with the canonical path and retain canonical output. ds4_cuda.cu:1475 +runtime/cuda DS4_CUDA_Q4_ATTN_OUT_HC_Q8K_EXPERIMENT value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on Enable the experimental Q8_K-based Q4 attention-output/HC fusion. ds4_cuda.cu:37357 +runtime/cuda DS4_CUDA_Q4_GROUPED_ATTN_A_ORACLE value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on Compare grouped attention-A against the canonical per-group result. ds4_cuda.cu:1477 +runtime/cuda DS4_CUDA_Q4_K1024_PERSISTENT_ORACLE value-aware flag, default off; nonempty value other than exact 0 enables and implies candidate admission Bitwise-compare the exact-shape persistent Q4 K1024 kernel with canonical MMVQ and retain canonical output. cuda/mmq/ds4_mmq.cu:3534 +runtime/cuda DS4_CUDA_Q4_K1024_PERSISTENT_STATS value-aware flag, default off; nonempty value other than exact 0 enables Print exact-shape persistent Q4 K1024 dispatch counters at exit. cuda/mmq/ds4_mmq.cu:3533 +runtime/cuda DS4_CUDA_Q8_F16_ALL presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Control the Q8 F16 all CUDA quantized-matmul/cache optimization. ds4_cuda.cu:2358 +runtime/cuda DS4_CUDA_Q8_F16_CACHE_MB unsigned integer MiB, full-string parse; default unlimited; 0 disables this cache Limit the selective Q8-to-F16 derived-weight cache. ds4_cuda.cu:2218 +runtime/cuda DS4_CUDA_Q8_F16_CACHE_RESERVE_MB unsigned integer MiB, full-string parse; default is VRAM-dependent (>=112 GiB: 512; >=40 GiB: max(768,1%); smaller: max(4096,5%)) Reserve free VRAM when growing the selective Q8-to-F16 cache. ds4_cuda.cu:2224 +runtime/cuda DS4_CUDA_Q8_F32_ALL presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Control the Q8 F32 all CUDA quantized-matmul/cache optimization. ds4_cuda.cu:2412 +runtime/cuda DS4_CUDA_Q8_F32_LARGE presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Control the Q8 F32 large CUDA quantized-matmul/cache optimization. ds4_cuda.cu:2416 +runtime/cuda DS4_CUDA_Q8_F32_PRELOAD presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Control the Q8 F32 preload CUDA quantized-matmul/cache optimization. ds4_cuda.cu:8597 +runtime/cuda DS4_CUDA_Q8_FOLD_ORACLE strict flag, default off; only exact value 1 enables Compare folded Q8_1 bytes and consumer outputs against canonical work while retaining canonical results. cuda/mmq/ds4_mmq.cu:383 +runtime/cuda DS4_CUDA_Q8_HC_EXPAND_FUSED false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, other nonempty values force fused Force the fused Q8 shared-down/HC expansion path. ds4_cuda.cu:2084 +runtime/cuda DS4_CUDA_Q8_HC_EXPAND_STATS false-like-aware flag, default off; 0/false/no/off is off, other nonempty values print report Print Q8 shared-down/HC policy and dispatch counters at exit. ds4_cuda.cu:2090 +runtime/cuda DS4_CUDA_Q8_NO_ALIGNED value-aware kill switch, default off; nonempty value other than exact 0 disables aligned Q8 kernels Disable aligned Q8 CUDA matmul kernels. ds4_cuda.cu:1021 +runtime/cuda DS4_CUDA_Q8_PAIR_BATCH presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Control the Q8 pair batch CUDA quantized-matmul/cache optimization. ds4_cuda.cu:20167 +runtime/cuda DS4_CUDA_QKV_KV_ROPE_FUSE value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control or tune the CUDA QKV KV rope fuse path. ds4.c:17256 +runtime/cuda DS4_CUDA_Q_NORM_ROPE_FUSE value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control or tune the CUDA q norm rope fuse path. ds4.c:17245 +runtime/cuda DS4_CUDA_REQUIRE_IQ2_XXS_SSD_PREFILL_MMQ false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Require the CUDA IQ2 XXS SSD prefill MMQ path; fail closed when unavailable. ds4_cuda.cu:4631 +runtime/cuda DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_BATCH value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on Fail if grouped batched attention-A cannot be used. ds4_cuda.cu:40178 +runtime/cuda DS4_CUDA_REQUIRE_Q4_K1024_PERSISTENT presence flag, default off; any defined value including 0 makes ineligible candidate fail closed Fail when the exact Q4 K1024 persistent candidate is unavailable instead of using MMVQ. cuda/mmq/ds4_mmq.cu:3722 +runtime/cuda DS4_CUDA_REQUIRE_STREAMING_EXPERT_PERSISTENT_CACHE false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Require streaming expert persistent cache in CUDA SSD streaming; fail closed when unavailable. ds4_cuda.cu:4117 +runtime/cuda DS4_CUDA_REQUIRE_STREAMING_SELECTED_BATCH_IO false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Require streaming selected batch I/O in CUDA SSD streaming; fail closed when unavailable. ds4_cuda.cu:4738 +runtime/cuda DS4_CUDA_REQUIRE_STREAMING_SELECTED_EVENT_PIPELINE false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Require streaming selected event pipeline in CUDA SSD streaming; fail closed when unavailable. ds4_cuda.cu:4870 +runtime/cuda DS4_CUDA_SERIAL_F16_MATMUL presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Control the serial F16 matmul CUDA F16 matmul path. ds4_cuda.cu:20801 +runtime/cuda DS4_CUDA_SERIAL_ROUTER presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Control or tune the CUDA serial router path. ds4_cuda.cu:20806 +runtime/cuda DS4_CUDA_SESSION_BATCH_ATTN_ALIAS value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control the grouped multi-session CUDA attn alias stage. ds4.c:69712 +runtime/cuda DS4_CUDA_SESSION_BATCH_ATTN_CORE value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control the grouped multi-session CUDA attn core stage. ds4.c:69715 +runtime/cuda DS4_CUDA_SESSION_BATCH_ATTN_POST value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control the grouped multi-session CUDA attn post stage. ds4.c:69727 +runtime/cuda DS4_CUDA_SESSION_BATCH_ATTN_PRE value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control the grouped multi-session CUDA attn pre stage. ds4.c:69708 +runtime/cuda DS4_CUDA_SESSION_BATCH_FFN_PRE value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control the grouped multi-session CUDA ffn pre stage. ds4.c:69704 +runtime/cuda DS4_CUDA_SESSION_BATCH_INTERLEAVE boolean, default on; unset/empty/nonzero enables pipeline interleaving; exact 0 disables Control the grouped multi-session CUDA interleave stage. ds4.c:70388 +runtime/cuda DS4_CUDA_SESSION_BATCH_KV_STORE value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control the grouped multi-session CUDA KV store stage. ds4.c:69723 +runtime/cuda DS4_CUDA_SESSION_BATCH_MOE value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control the grouped multi-session CUDA MoE stage. ds4.c:69696 +runtime/cuda DS4_CUDA_SESSION_BATCH_MOE_COMBINE_ROWS value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control the grouped multi-session CUDA MoE combine rows stage. ds4.c:69241 +runtime/cuda DS4_CUDA_SESSION_BATCH_QKV value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control the grouped multi-session CUDA QKV stage. ds4.c:69719 +runtime/cuda DS4_CUDA_SESSION_BATCH_SHARED value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control the grouped multi-session CUDA shared stage. ds4.c:69700 +runtime/cuda DS4_CUDA_SPLITKV_CHUNK integer scores/chunk; default 512; clamped 1..512 Control splitkv chunk in CUDA split-KV attention/speculation. ds4_cuda.cu:22568 +runtime/cuda DS4_CUDA_SPLITKV_DECODE value-aware boolean, default off; exact 0/empty is off, other nonempty values enable; mere presence also excludes one session-batch path Enable split-KV decode attention. ds4_cuda.cu:2213; ds4.c:67921 +runtime/cuda DS4_CUDA_SPLITKV_GLOBAL_SOFTMAX value-aware opt-in, default off; exact 0/empty is off, other nonempty values enable Use the global-softmax variant of split-KV attention. ds4_cuda.cu:22589 +runtime/cuda DS4_CUDA_SPLITKV_MIN_SCORE integer score count 0..UINT32_MAX; default 0 when explicitly enabled, otherwise 512; CUDA kernel clamps to 0..8192 Set the minimum visible-score count for split-KV attention. ds4_cuda.cu:22557; ds4.c:17229 +runtime/cuda DS4_CUDA_SPLITKV_S integer exact split count; unset/invalid = automatic; valid value clamped 1..16 Control splitkv s in CUDA split-KV attention/speculation. ds4_cuda.cu:22578 +runtime/cuda DS4_CUDA_SPLITKV_SPEC value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Enable split-KV speculative decoding. ds4.c:17082 +runtime/cuda DS4_CUDA_SPLITKV_SPEC_BATCH_VERIFY value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Use batched verification for split-KV speculation. ds4.c:17102 +runtime/cuda DS4_CUDA_SPLITKV_SPEC_LOG presence diagnostic flag; default off; any defined value including 0 enables Log split-KV speculative-decode admission and fallback decisions. ds4.c:55622 +runtime/cuda DS4_CUDA_SPLITKV_SPEC_TIMING presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Print timing for split-KV speculative-decode stages. ds4.c:55661 +runtime/cuda DS4_CUDA_SPLITKV_SPEC_TOPONLY_ROW0 value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Compute only the top result for row zero in split-KV speculation. ds4.c:17092 +runtime/cuda DS4_CUDA_SPLITKV_S_FLOOR integer split count; default 4; clamped 1..16 Control splitkv s floor in CUDA split-KV attention/speculation. ds4_cuda.cu:22571 +runtime/cuda DS4_CUDA_SPLITKV_S_MAX integer split count; default 16; clamped 1..16 Control splitkv s max in CUDA split-KV attention/speculation. ds4_cuda.cu:22574 +runtime/cuda DS4_CUDA_STREAMING_EXPERT_CACHE_PROFILE presence diagnostic flag; default off; any defined value including 0 enables Profile CUDA SSD-streaming streaming expert cache. ds4.c:21676 +runtime/cuda DS4_CUDA_STREAMING_EXPERT_PERSISTENT_CACHE_ORACLE false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Run the diagnostic oracle for CUDA SSD-streaming streaming expert persistent cache. ds4_cuda.cu:4121 +runtime/cuda DS4_CUDA_STREAMING_EXPERT_PERSISTENT_CACHE_STATS false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Print counters for CUDA SSD-streaming streaming expert persistent cache. ds4_cuda.cu:4119 +runtime/cuda DS4_CUDA_STREAMING_PREFILL_BATCH_SELECTED_PROFILE presence diagnostic flag; default off; any defined value including 0 enables Profile CUDA SSD-streaming streaming prefill batch selected. ds4.c:21759 +runtime/cuda DS4_CUDA_STREAMING_SELECTED_BATCH_IO_ORACLE false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Run the diagnostic oracle for CUDA SSD-streaming streaming selected batch I/O. ds4_cuda.cu:4740 +runtime/cuda DS4_CUDA_STREAMING_SELECTED_BATCH_IO_PROFILE false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Profile CUDA SSD-streaming streaming selected batch I/O. ds4_cuda.cu:5711 +runtime/cuda DS4_CUDA_STREAMING_SELECTED_EVENT_PIPELINE_ORACLE false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Run the diagnostic oracle for CUDA SSD-streaming streaming selected event pipeline. ds4_cuda.cu:4872 +runtime/cuda DS4_CUDA_STREAMING_SELECTED_EVENT_PIPELINE_STATS false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Print counters for CUDA SSD-streaming streaming selected event pipeline. ds4_cuda.cu:4874 +runtime/cuda DS4_CUDA_STRICT_WEIGHT_CACHE presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Fail a weight lookup when cache allocation fails instead of falling back to mapped model memory. ds4_cuda.cu:6388 +runtime/cuda DS4_CUDA_SYNC_XDEV presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Synchronize cross-device CUDA copies for debugging and error localization. ds4_cuda.cu:363 +runtime/cuda DS4_CUDA_TP_ATTN value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control CUDA tensor/expert-parallel attn execution. ds4.c:16862 +runtime/cuda DS4_CUDA_TP_ATTN_CACHE_DUP value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Control CUDA tensor/expert-parallel attn cache dup execution. ds4.c:16886 +runtime/cuda DS4_CUDA_TP_ATTN_HEADS value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Control CUDA tensor/expert-parallel attn heads execution. ds4.c:16878 +runtime/cuda DS4_CUDA_TP_ATTN_OUT_HC_FUSE presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Control CUDA tensor/expert-parallel attn out HC fuse execution. ds4.c:17391 +runtime/cuda DS4_CUDA_TP_ATTN_PEER_READ value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control CUDA tensor/expert-parallel attn peer read execution. ds4.c:16870 +runtime/cuda DS4_CUDA_TP_EP_BALANCED_SHARED_MID value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control CUDA tensor/expert-parallel EP balanced shared mid execution. ds4.c:16943 +runtime/cuda DS4_CUDA_TP_EP_DELAY_REDUCE value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control CUDA tensor/expert-parallel EP delay reduce execution. ds4.c:16918 +runtime/cuda DS4_CUDA_TP_EP_DIRECT_RETURN value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control CUDA tensor/expert-parallel EP direct return execution. ds4.c:16910 +runtime/cuda DS4_CUDA_TP_EP_DUAL_PREQUANT value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control CUDA tensor/expert-parallel EP dual prequant execution. ds4.c:16952 +runtime/cuda DS4_CUDA_TP_EP_FUSED_HC_REDUCE value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control CUDA tensor/expert-parallel EP fused HC reduce execution. ds4.c:16926 +runtime/cuda DS4_CUDA_TP_EP_FUSED_SHARED_MID value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control CUDA tensor/expert-parallel EP fused shared mid execution. ds4.c:16934 +runtime/cuda DS4_CUDA_TP_EP_PACK_EXACT value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control CUDA tensor/expert-parallel EP pack exact execution. ds4.c:16902 +runtime/cuda DS4_CUDA_TP_MOE value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control CUDA tensor/expert-parallel MoE execution. ds4.c:16894 +runtime/cuda DS4_CUDA_TP_MOE_COPY3 value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Control CUDA tensor/expert-parallel MoE copy3 execution. ds4.c:16976 +runtime/cuda DS4_CUDA_TP_MOE_DELAY_REDUCE value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control CUDA tensor/expert-parallel MoE delay reduce execution. ds4.c:16960 +runtime/cuda DS4_CUDA_TP_MOE_PACK value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Control CUDA tensor/expert-parallel MoE pack execution. ds4.c:16968 +runtime/cuda DS4_CUDA_TP_MOE_PEER_READ value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Control CUDA tensor/expert-parallel MoE peer read execution. ds4.c:16984 +runtime/cuda DS4_CUDA_TP_MOE_PEER_ROUTER value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Control CUDA tensor/expert-parallel MoE peer router execution. ds4.c:16992 +runtime/cuda DS4_CUDA_TP_OUTPUT boolean, default on; empty/unset/nonzero enables, exact 0 disables Control CUDA tensor/expert-parallel output execution. ds4.c:51523 +runtime/cuda DS4_CUDA_TP_OUTPUT_WAYS integer 2..DS4_MAX_GPUS (16); default 8; invalid value falls back to 2; capped by available GPUs Set the number of GPU ways used to shard CUDA tensor-parallel output projection. ds4.c:58; ds4.c:51530 +runtime/cuda DS4_CUDA_TP_PREFILL_ATTN_OUTPUT value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control CUDA tensor/expert-parallel prefill attn output execution. ds4.c:17272 +runtime/cuda DS4_CUDA_TP_PREFILL_FFN value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control CUDA tensor/expert-parallel prefill ffn execution. ds4.c:17264 +runtime/cuda DS4_CUDA_TP_Q value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Control CUDA tensor/expert-parallel q execution. ds4.c:17016 +runtime/cuda DS4_CUDA_TP_SHARED value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Control CUDA tensor/expert-parallel shared execution. ds4.c:17000 +runtime/cuda DS4_CUDA_TP_SHARED_FOLD value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control CUDA tensor/expert-parallel shared fold execution. ds4.c:17008 +runtime/cuda DS4_CUDA_VERIFY_DECODE2_SPLIT_TOP1 value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Enable the split top-1 path for two-row verification decode. ds4.c:17052 +runtime/cuda DS4_CUDA_WEIGHT_ARENA_CHUNK_MB positive integer MiB; default 1792; clamped 256..8192 and raised/aligned when one allocation needs more Set the CUDA selective-weight arena allocation chunk. ds4_cuda.cu:6311 +runtime/cuda DS4_CUDA_WEIGHT_CACHE presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Force selective CUDA weight caching instead of direct mapped access. ds4_cuda.cu:1246 +runtime/cuda DS4_CUDA_WEIGHT_CACHE_LIMIT_GB unsigned integer GiB; default/0 = unlimited; parser accepts a numeric prefix even with trailing text Limit total CUDA selective-weight cache allocation. ds4_cuda.cu:6299 +runtime/cuda DS4_CUDA_WEIGHT_CACHE_VERBOSE presence diagnostic flag; default off; any defined value including 0 enables Print CUDA weight mapping, caching, and preload diagnostics. ds4_cuda.cu:1297 +runtime/cuda DS4_CUDA_WEIGHT_PRELOAD presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Request proactive CUDA weight caching/preloading. ds4_cuda.cu:1247 +runtime/cuda DS4_CUDA_WEIGHT_PRELOAD_SPAN_MB positive integer MiB; default 1024; clamped 64..4096 Set the maximum span size used by CUDA weight preload. ds4.c:2880 +runtime/cuda DS4_CUDA_WINDOW_ATTENTION presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Control or tune the CUDA window attention path. ds4_cuda.cu:23051 +runtime/cuda-mmq DS4_MMID_CASE1 boolean-ish cached flag; default on; a value starting with 0 disables Disable the single-expert MM-IDs specialized fast path for comparison. cuda/mmq/mmid.cu:290 +runtime/cuda-mmq DS4_MMID_LARGE boolean-ish cached flag; default on; a value starting with 0 disables Control the large-N global-memory MM-IDs path used beyond shared-memory capacity. cuda/mmq/mmid.cu:245 +runtime/cuda-mmq DS4_MMQ_D2R boolean-ish cached flag; default on; a value starting with 0 disables Control the direct-to-register Q2_K MoE down path. cuda/mmq/ds4_mmq.cu:505 +runtime/cuda-mmq DS4_MMQ_D2R_IQ2 boolean-ish cached flag; default on; a value starting with 0 disables Control the direct-to-register IQ2 MoE gate/up path. cuda/mmq/ds4_mmq.cu:514 +runtime/cuda-mmq DS4_MMQ_D2R_MIN_COLS positive integer; default 1024; invalid or nonpositive input restores the default Set the minimum output-column count for the MMQ direct-to-register path. cuda/mmq/ds4_mmq.cu:609 +runtime/cuda-mmq DS4_MMQ_D2R_STATS exact 1 enables; unset or every other value disables; cached and synchronizes the stream Print partial-tile fill telemetry for the direct-to-register MMQ kernels. cuda/mmq/ds4_mmq_d2r.cu:33 +runtime/cuda-mmq DS4_MMQ_DENSE_D2R boolean-ish flag; default on; exact 0 disables Control the eligible aligned-Q8 dense prefill direct-to-register path. ds4_cuda.cu:19567 +runtime/cuda-mmq DS4_MMQ_NO_YIND presence rollback; unset keeps Y-indirect staging; any defined value including 0 disables it Restore slot-gathered MoE gate/up activation quantization. cuda/mmq/ds4_mmq.cu:589 +runtime/cuda-mmq DS4_MMQ_OUT_MEMSET exact 1 enables; unset or every other value disables; cached Restore blanket MMQ output-buffer zeroing for diagnostics. cuda/mmq/ds4_mmq.cu:532 +runtime/cuda-mmq DS4_MMQ_YBUF_MEMSET unset or 0 disables; 1 zero-fills; a value starting with p or P poison-fills with 0xFF Control MMQ Q8_1 activation-staging initialization and its poison oracle. cuda/mmq/ds4_mmq.cu:558 +runtime/cuda-mmq DS4_MMQ_YIND_VERIFY presence diagnostic; unset is off; any defined value including 0 enables Byte-compare Y-indirect and slot-gathered MoE activation buffers. cuda/mmq/ds4_mmq.cu:600 +runtime/cuda-mmq DS4_Q8_FOLD_SELFTEST positive call budget; unset/empty disables; a nonempty value parsing to 1 or less selects 512 calls Byte-check folded Q8_1 activations against a fresh quantization; synchronizes eager streams. cuda/mmq/ds4_mmq.cu:5744 +runtime/cuda-shared DS4_FORCE_CUDA_PEER presence flag read once at CUDA init; unset uses automatic transfer selection; any defined value including 0 enables Force cross-device transfers through cudaMemcpyPeerAsync for diagnostics. ds4_cuda.cu:364 +runtime/cuda-shared DS4_FORCE_HOST_BOUNCE presence flag read once at CUDA init; unset uses automatic transfer selection; any defined value including 0 enables Force cross-device transfers through pinned host bounce buffers for diagnostics. ds4_cuda.cu:365 +runtime/cuda-tools DS4_WS_REPACK_HASH exact 1 enables; unset or every other value disables unless overridden by CLI; cached Print a per-artifact FNV-1a hash for workspace repack identity checks. cuda/mmq/ds4_repack.cu:530 +runtime/cuda-tools DS4_WS_REPACK_THREADS positive integer; default min(6, hardware threads), capped at 16 and the job count Set the CPU worker count for CUDA workspace artifact repacking. cuda/mmq/ds4_repack.cu:539 +runtime/distributed DS4_DIST_CONNECT_BIND_HOST non-empty string; unset/empty means no local bind constraint Bind outgoing distributed connections to a local host/address. ds4_distributed.c:1332 +runtime/distributed DS4_DIST_CONNECT_BIND_IF non-empty string; unset/empty means no local bind constraint Bind outgoing distributed connections to a network interface. ds4_distributed.c:1334 +runtime/distributed DS4_DIST_CONNECT_TRACE presence flag; unset=off, any set value=on Emit trace diagnostics for dist connect trace. ds4_distributed.c:1112 +runtime/distributed DS4_DIST_DECODE_PROFILE presence flag; unset=off, any set value=on Collect timing/profile diagnostics for dist decode profile. ds4_distributed.c:753 +runtime/distributed DS4_DIST_DISABLE_PREFILL_ACK_ONLY presence flag; unset keeps optimized/default behavior, any set value disables it Disable/roll back dist disable prefill ack only. ds4_distributed.c:3692 +runtime/distributed DS4_DIST_DISABLE_PREFILL_PIPELINE presence flag; unset keeps optimized/default behavior, any set value disables it Disable/roll back dist disable prefill pipeline. ds4_distributed.c:3427 +runtime/distributed DS4_DIST_DISABLE_WORKER_PREFETCH presence flag; unset keeps optimized/default behavior, any set value disables it Disable/roll back dist disable worker prefetch. ds4_distributed.c:7878 +runtime/distributed DS4_DIST_PREFILL_CHUNK positive integer; unset/0 uses session prefill capacity; explicit value may not exceed capacity Set distributed prefill chunk size. ds4_distributed.c:3455 +runtime/distributed DS4_DIST_PREFILL_SEND_DEPTH integer 1..8; default 2; capped to chunk count Set coordinator prefill sender queue depth. ds4_distributed.c:470 +runtime/distributed DS4_DIST_PREFILL_WINDOW positive integer <=64; auto default remote stages+2 clamped 2..8 and chunk count Set maximum distributed prefill chunks in flight. ds4_distributed.c:3485 +runtime/distributed DS4_DIST_SOCKET_BUFFER_MB integer 0..512 MiB; default 128; 0 disables socket buffer override Set TCP send/receive buffer sizes. ds4_distributed.c:712 +runtime/distributed DS4_DIST_SOCKET_RECV_TIMEOUT_SEC Nonempty base-10 integer parsed completely; valid range 1..3600 seconds. Unset, empty, partially parsed, or out-of-range values install no SO_RCVTIMEO at all. Optionally bound blocking receives on distributed TCP sockets; the default deliberately permits indefinitely idle control connections during separate KV transfers. ds4_distributed.c:1049 +runtime/distributed DS4_DIST_SOCKET_TIMEOUT_SEC Nonempty base-10 integer parsed completely; valid range 1..3600 seconds. Default 60 seconds for unset, empty, partially parsed, or out-of-range values. Set SO_SNDTIMEO on distributed TCP sockets so blocked coordinator/worker sends eventually fail. ds4_distributed.c:1032 +runtime/distributed DS4_DIST_WORKER_FORWARD_WINDOW integer 1..64; default 4 Set worker forward-results window. ds4_distributed.c:740 +runtime/distributed DS4_DIST_WORKER_PREFETCH_DEPTH integer 1..8; default 2 Set worker input-prefetch queue depth. ds4_distributed.c:726 +runtime/dspark DS4_DSPARK_CACHE_RESERVE_GB integer GiB via atoi; default 4.5 GiB; values 1..32 replace it, all other values fall back; decimal/trailing text is truncated/accepted by atoi Reserve VRAM on DSpark support-cache tiers before packing support-model tensors. ds4.c:59580 +runtime/dspark DS4_DSPARK_DISABLE_FINAL_OUTPUT_ALIAS presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it Disable aliasing the final DSpark stage output to the next-stage buffer and use an explicit copy. ds4.c:33594 +runtime/dspark DS4_DSPARK_DISABLE_FUSED_CPU_MARKOV_ARGMAX cached value-aware kill switch; default off; nonempty value other than exact 0 disables; false/off also disable because only 0 is recognized as false Disable the fused CPU Markov-bias plus argmax implementation. ds4.c:34297 +runtime/dspark DS4_DSPARK_DISABLE_REUSE_CONFIDENCE0_MARKOV cached value-aware kill switch; default off; nonempty value other than exact 0 disables; false/off also disable because only 0 is recognized as false Disable reuse of the first confidence score during Markov proposal. ds4.c:34306 +runtime/dspark DS4_DSPARK_DISABLE_VERIFY_SELECTED_PROFILE presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it Override and disable the selected-expert verifier profiler. ds4.c:36470 +runtime/dspark DS4_DSPARK_EXEC_TIER integer tier via atoi; default is placement/TP free-VRAM heuristic; valid 0..n_gpus-1 overrides; invalid numeric range falls back, but nonnumeric text becomes tier 0 Choose the GPU tier that executes and primarily caches the DSpark support model. ds4.c:59553 +runtime/dspark DS4_DSPARK_FAKE_ARGMAX_PROPOSAL nonempty boolean; unset/empty or exact 0: off; every other nonempty value enables, but only while DSpark itself is enabled If the real DSpark proposer produced no draft, installs a one-token fallback proposal equal to the argmax of the current target logits; debug/test mode also selects the non-fused stage-0 setup path. ds4.c:64088 +runtime/dspark DS4_DSPARK_LOW_MEMORY_PREFILL_CHUNK unsigned integer rows; default 128; 0 disables the low-memory policy; invalid/overflow falls back, numeric prefixes are accepted; only consulted for Metal SSD+DSpark on <=24 GiB hosts without an explicit chunk Set the automatic low-memory Metal prefill chunk for SSD-streamed DSpark. ds4.c:60533 +runtime/dspark DS4_DSPARK_NO_GPU_MARKOV presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it Disable GPU Markov bias/argmax and the fully device-resident proposal path. ds4.c:34521 +runtime/dspark DS4_DSPARK_NO_MARKOV cached value-aware kill switch; default off; nonempty value other than exact 0 disables Markov bias; false/off also disable Disable Markov bias in DSpark proposal generation. ds4.c:34288 +runtime/dspark DS4_DSPARK_PROBE nonempty-string diagnostic; unset/empty is off, any nonempty value including 0 is on Log DSpark proposal/probe diagnostics. ds4.c:64085 +runtime/dspark DS4_DSPARK_PROP_PROFILE presence flag; unset is off; any defined value, including empty or 0, is on; normal eligibility still applies Print fine-grained timings for DSpark proposal setup. ds4.c:32771 +runtime/dspark DS4_DSPARK_SPEC_LOG presence flag; unset is off; any defined value, including empty or 0, is on; normal eligibility still applies Log speculative proposal, verification, acceptance, and fallback decisions. ds4.c:66406 +runtime/dspark DS4_DSPARK_SSD_VERIFY_BLOCK_MAX unsigned integer rows; default/fallback 0 means automatic policy; numeric prefixes accepted; used both as verifier cap and as an exact-2 proposer-policy discriminator Cap speculative rows verified from SSD and influence exact-2 proposal sizing. ds4.c:52190 +runtime/dspark DS4_DSPARK_STAGE_PROFILE presence flag; unset is off; any defined value, including empty or 0, is on; DS4_DSPARK_STAGE_PROFILE_STAGE must also match Profile DSpark support stages with command-boundary timings. ds4.c:33226 +runtime/dspark DS4_DSPARK_STAGE_PROFILE_STAGE selector subordinate to DS4_DSPARK_STAGE_PROFILE; unset/empty: match every stage; otherwise strtoul base 10 must consume the whole value, fit uint32_t, and equal the current stage; invalid/out-of-range values match no stage Restricts DSpark stage-boundary timing output to one stage; it does not enable profiling by itself. ds4.c:33227 +runtime/dspark DS4_DSPARK_STATS value-aware flag; default off; nonempty value other than exact 0 enables; false/off are treated as enabled Collect and print aggregate DSpark runtime statistics. ds4.c:61417 +runtime/dspark DS4_DSPARK_VERIFY_CACHE presence diagnostic; unset: off; any presence including empty or 0 enables on each support-cache installation CUDA only: copies every installed nonempty DSpark/support-cache range back to the host, byte-compares it with its source, and logs each mismatch plus a bad-count summary without changing the install result. ds4_cuda.cu:8257 +runtime/dspark DS4_DSPARK_VERIFY_HEAD_NO_TP presence rollback; unset: allow eligible CUDA output tensor parallelism; any presence including empty or 0 removes the TP path from eligibility CUDA only: forces the DSpark speculative batched vocabulary head away from output-TP for correctness isolation; under CUDA TP+EP the attempt fails instead of using unavailable full output weights. ds4.c:26468 +runtime/dspark DS4_DSPARK_VERIFY_NONCAUSAL presence diagnostic sampled once after the first successfully submitted CUDA noncausal-attention kernel; unset: verify 0 calls; any presence including empty or 0: verify that call and the next 2 CUDA only: synchronizes and reads back Q/KV/output, computes the DSpark noncausal attention CPU reference, and logs max absolute/relative error; it reports only and does not fail the operation. ds4_cuda.cu:21736 +runtime/dspark DS4_DSPARK_VERIFY_PROFILE cached presence diagnostic; unset is off; any defined value including empty/0 profiles only the first eligible verifier invocation Profile one full DSpark target-verifier invocation layer by layer. ds4.c:36567 +runtime/dspark DS4_DSPARK_VERIFY_SELECTED_PROFILE presence flag; unset is off; any defined value, including empty or 0, enables unless DS4_DSPARK_DISABLE_VERIFY_SELECTED_PROFILE is also present (disable wins) Profile selected-expert streaming inside the DSpark verifier. ds4.c:36469 +runtime/dspark DS4_DSPARK_VERIFY_SPLIT_HEAD nonempty boolean with inverted default; unset/empty or exact 0: fused head; every other nonempty value: split head Runs the DSpark suffix verifier output head and top-1 reduction in a separate GPU command section after the layer loop, for timing/correctness isolation; default keeps them fused into the layer command section. ds4.c:36535 +runtime/dspark DS4_DSPARK_VERIFY_TOPS_CHECK presence flag; unset is off; any defined value, including empty or 0, is on; normal eligibility still applies Read back verifier logits and compare GPU top IDs with CPU argmax. ds4.c:36683 +runtime/glm DS4_GLM_ABLATE_COMBINE presence ablation; unset: exchange the local TP partial with the peer and add both halves; any presence including empty or 0 skips the exchange Metal two-rank TP timing probe: doubles the local routed-MoE or split-attention partial instead of combining with the peer, deliberately producing invalid output; both ranks must set it or their exchange gates desynchronize. ds4.c:43991 +runtime/glm DS4_GLM_ATTN_NO_LORA_VEC2 presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it Disable vectorized two-row LoRA accumulation in CUDA GLM indexed attention. ds4_cuda.cu:34176 +runtime/glm DS4_GLM_ATTN_NO_SCORE_VEC2 presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it Disable vectorized two-row score computation in CUDA GLM indexed attention. ds4_cuda.cu:34165 +runtime/glm DS4_GLM_ATTN_NO_STAGED_DECODE presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it Disable staged CUDA GLM indexed-decode attention for large selected sets. ds4_cuda.cu:34214 +runtime/glm DS4_GLM_DECODE_ABLATE cached substring list; default empty mask; recognized tokens are attn_out, attn_core, qpath, indexer, routed, shared, qklow; unknown text has no effect; matching stages are skipped and output is invalid Skip selected GLM decode stages for timing attribution; generated output is invalid. ds4.c:44238 +runtime/glm DS4_GLM_DECODE_FLUSH_INTERVAL integer layers via atoi; default 4 for indexed decode and 32 otherwise; <=0/nonnumeric disables periodic flush; capped to layer count and forced to 0 for deferred completion Set how often non-streaming GLM decode command work is flushed between layers. ds4.c:49628 +runtime/glm DS4_GLM_DISABLE_FLASH_PREFILL presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it Disable GLM Flash Attention prefill. ds4.c:44897 +runtime/glm DS4_GLM_DISABLE_STREAMING_TOKEN_PREFILL presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it; either backend-specific ROCm/Metal alias also disables Disable token-major GLM SSD-streaming prefill. ds4.c:49512 +runtime/glm DS4_GLM_FENCE_TRACE presence flag; unset is off; any defined value, including empty or 0, is on; normal eligibility still applies Log fenced CUDA tier switches used by GLM multi-GPU execution. ds4_cuda.cu:7984 +runtime/glm DS4_GLM_GEMM_TRACE presence flag; unset is off; any defined value, including empty or 0, is on; normal eligibility still applies Measure and print CUDA GLM dequantization and cuBLAS GEMM timings. ds4_cuda.cu:19706 +runtime/glm DS4_GLM_HIDDEN_DUMP nonempty filesystem path/prefix; unset/empty disables; writes final hidden row or per-layer files selected by DS4_GLM_HIDDEN_DUMP_LAYER Dump GLM hidden-state rows for correctness isolation. ds4.c:38333 +runtime/glm DS4_GLM_HIDDEN_DUMP_LAYER selector; unset/empty = -1 (no per-layer dump, final hidden still dumped when path set); all = every layer; otherwise atoi result selects a layer, so invalid text selects layer 0 Choose which GLM layer hidden states are dumped. ds4.c:38353 +runtime/glm DS4_GLM_KV_DUMP nonempty filesystem prefix; unset/empty disables; writes layer-0 lora and rope compact-cache files after sync Dump layer-0 compact GLM KV cache data after prompt synchronization. ds4.c:62941 +runtime/glm DS4_GLM_LOGIT_DUMP nonempty filesystem path; unset/empty disables; dumps the first post-prefill logits vector once per process Dump the first post-prefill GLM logits vector. ds4.c:38411 +runtime/glm DS4_GLM_MEMORY_GUARD guard is on by default; exact 0 or case-insensitive false/off/no disables it; any other value and unset keep it enabled Control the pre-allocation GLM host/GPU memory safety guard. ds4.c:41799 +runtime/glm DS4_GLM_MEMORY_GUARD_FRACTION floating-point fraction; default 0.99; parsed numeric prefix is accepted, invalid/nonfinite falls back, values clamp to 0.50..1.00 Set the fraction of detected memory usable by the GLM memory guard. ds4.c:41838 +runtime/glm DS4_GLM_MEMORY_GUARD_REPORT nonempty diagnostic flag; unset/empty is off; any nonempty value including 0 prints successful-admission accounting (refusals always report) Print successful GLM memory-guard budget accounting. ds4.c:41878 +runtime/glm DS4_GLM_MEMORY_GUARD_RESERVE_GB floating-point GiB; dynamic default (normally 32, 24 on near-full 480..640 GiB hosts, possibly lower for resident ROCm slices); numeric prefixes accepted; invalid falls back; clamp 0..1024 Set fixed headroom subtracted by the GLM memory guard. ds4.c:41856 +runtime/glm DS4_GLM_MOE_EXPERT_MAJOR presence selector; unset: off; any presence including empty or 0 requests the path only for n_tokens >= 16; the automatic tile-8 path takes precedence when enabled (normally n_tokens >= 128) CUDA only: groups selected token/expert pairs by expert and uses expert-major Q2_K routed-MoE gate/up/down kernels to reuse expert weights; otherwise the normal token-major path is used. ds4_cuda.cu:35973 +runtime/glm DS4_GLM_MOE_NO_DOWN_TILE8_EXACT presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it Disable the exact tile-8 CUDA GLM routed-MoE down projection. ds4_cuda.cu:36028 +runtime/glm DS4_GLM_MOE_NO_EXPERT_TILE8 presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it Disable automatic expert tile-8 CUDA GLM routed-MoE batching. ds4_cuda.cu:35971 +runtime/glm DS4_GLM_MOE_NO_LOCAL_BATCH_IO presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it Disable device-local batch scratch I/O for large CUDA GLM MoE batches. ds4_cuda.cu:35942 +runtime/glm DS4_GLM_MOE_SCALAR presence rollback; unset: use optimized warp kernels; any presence including empty or 0 selects scalar baseline kernels where an earlier expert-tile/expert-major path does not return; the special two-token MTP gate/up kernel still takes precedence CUDA only: forces the baseline scalar Q2_K routed-MoE gate/up and down implementations for A/B or correctness testing (for two-token MTP, only the down half is forced). ds4_cuda.cu:36154 +runtime/glm DS4_GLM_MOE_SCRATCH_TIER0 presence placement override; unset: allocate xq/midq quantization scratch on the current logical tier; any presence including empty or 0 allocates it on logical tier 0 CUDA only: pins the routed-MoE xq_scratch and midq_scratch allocations to GPU tier 0 for multi-tier placement experiments; other MoE scratch remains on the current tier. ds4_cuda.cu:35918 +runtime/glm DS4_GLM_MTP_NO_ATTN_TOK2 presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it Disable the exact two-token CUDA GLM MTP attention kernel. ds4_cuda.cu:34168 +runtime/glm DS4_GLM_MTP_NO_MOE_TOK2 presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it Disable the exact two-token CUDA GLM MTP routed-MoE kernel. ds4_cuda.cu:36140 +runtime/glm DS4_GLM_MTP_NO_SHARED_TOK2 presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it Disable the exact two-token CUDA GLM MTP shared-FFN kernel. ds4_cuda.cu:37920 +runtime/glm DS4_GLM_MTP_PROBE presence flag; unset is off; any defined value, including empty or 0, is on; its second call site also rejects the normal batching path Run the GLM next-N/MTP acceptance quality-and-timing probe without changing output and force probe-compatible scheduling. ds4.c:64769 +runtime/glm DS4_GLM_PREFILL_TRUNC nonempty value parsed by atoi (leading whitespace/sign accepted and trailing junk ignored); effective only when the resulting int is > 0 and < the current prompt length; unset/empty or a result <= 0 or >= prompt length leaves the prompt unchanged GPU GLM debug hook: truncates the prompt before prefill/checkpoint handling so dumped prefill logits can be aligned with a CPU first-token reference. ds4.c:63080 +runtime/glm DS4_GLM_RESUME_PREFILL_MIN integer suffix tokens, non-ROCm builds only; default 4; parsed numeric prefixes accepted; <=0 maps to UINT32_MAX and effectively disables batched resume; ROCm build ignores it and stays at 4 Set the suffix-length crossover from token decode to batched resumed prefill. ds4.c:38244 +runtime/glm DS4_GLM_ROUTER_SCALAR presence rollback; unset: use the 256-thread parallel router when n_expert <= 256 (the scalar path is already automatic above 256); any presence including empty or 0 forces the scalar path CUDA only: selects the one-active-thread-per-token sigmoid/top-k router kernel instead of the parallel shared-memory reduction, for A/B or correctness testing. ds4_cuda.cu:36390 +runtime/glm DS4_GLM_SHARED_SPLIT presence rollback; unset: use the fused one-token shared-expert Q8_0 gate+up+SwiGLU kernel when its shape/buffers are eligible; any presence including empty or 0 skips that fused one-token path; the earlier two-token MTP-specialized path is unaffected CUDA only: forces shared-expert gate and up through two separate Q8_0 matmuls followed by a separate SwiGLU operation for one-token decode. ds4_cuda.cu:37946 +runtime/glm DS4_GLM_STREAMING_DECODE_FULL_LAYER_MAP presence compatibility alias; unset: automatic mapping; any presence including empty or 0 independently forces full-layer mapping, equivalent to the backend-specific DS4_ROCM_GLM_STREAMING_DECODE_FULL_LAYER_MAP or DS4_METAL_GLM_STREAMING_DECODE_FULL_LAYER_MAP control Backend-neutral alias for supported GLM SSD streaming (Metal/ROCm): maps every tensor in each decode layer instead of using the decode-only map that can omit routed experts served by the expert cache; layers that already require a full map are unchanged. ds4.c:42427 +runtime/glm DS4_GLM_STREAMING_DECODE_SYNC_EACH_LAYER ROCm-only third-priority legacy value: a nonempty DS4_ROCM_GLM_STREAMING_DECODE_SYNC_EACH_LAYER wins, otherwise a nonempty DS4_METAL_GLM_STREAMING_DECODE_SYNC_EACH_LAYER wins, otherwise this name is read; nonempty values are true except exact 0 or case-insensitive false/off/no; unset/empty: false; non-ROCm builds always return true and ignore this name On ROCm non-static GLM SSD decode, opts into ending/synchronizing commands after token mapping and after every layer; the default keeps ordered work alive across layer mappings. Static-map decode bypasses this control. ds4.c:49591 +runtime/glm DS4_GLM_STREAMING_PREFILL_SYNC_EACH_LAYER ROCm-only third-priority legacy value: a nonempty DS4_ROCM_GLM_STREAMING_PREFILL_SYNC_EACH_LAYER wins, otherwise a nonempty DS4_METAL_GLM_STREAMING_PREFILL_SYNC_EACH_LAYER wins, otherwise this name is read; nonempty values are true except exact 0 or case-insensitive false/off/no; unset/empty: false for compact prefill; full-layer prefill and non-ROCm builds always synchronize and ignore this name On ROCm compact GLM SSD prefill, opts into ending/synchronizing commands at every layer boundary; the default carries ordered work across mappings, while full-layer expert-cache prefill always retains the boundary. ds4.c:42207 +runtime/glm DS4_GLM_STREAMING_TOKEN_PREFILL_MAX unsigned token limit; backend-specific ROCm/Metal variable takes precedence, generic is fallback; default 0 on ROCm and 64 otherwise; invalid/overflow falls back, numeric prefixes accepted; 0 disables token-major streaming prefill Set the largest SSD-streaming prefill handled by the token-major decode-like path. ds4.c:49494 +runtime/glm DS4_GLM_SYNC_TRACE presence flag; unset is off; any defined value, including empty or 0, is on; normal eligibility still applies Log GLM checkpoint/resume and dense-versus-indexed prefill decisions. ds4.c:63178 +runtime/glm DS4_GLM_TP_DEBUG presence flag; unset is off; any defined value, including empty or 0, is on; normal eligibility still applies Print CUDA/GLM tensor-parallel dispatch, gate, selected-ID, and failure diagnostics. ds4.c:43893 +runtime/glm DS4_GLM_TP_EXACT_PREFILL_MAX integer suffix limit via atoi cast to uint32; default 64; nonnumeric becomes 0; negative values wrap to a very large unsigned limit Set the maximum two-way TP suffix that uses exact token-by-token prefill. ds4.c:63120 +runtime/glm DS4_GLM_TP_HEAD_SPLIT_MIN cached integer token threshold via atoi; default 64; negative values clamp to 0, nonnumeric becomes 0; 0 admits all otherwise-eligible batches Set the minimum batch size for GLM tensor-parallel output-head splitting. ds4.c:38323 +runtime/glm DS4_GLM_VALUE_NO_TILE16 presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it Disable the CUDA GLM 16-token tiled value-projection kernel. ds4_cuda.cu:36806 +runtime/metal DS4_METAL_ARGSORT_SOURCE file path; unset/empty: use the in-tree Metal source file Overrides the argsort Metal kernel source file loaded at runtime. ds4_metal.m:4942 +runtime/metal DS4_METAL_ATTN_OUT_STAGE_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for attn out stage. ds4.c:64961 +runtime/metal DS4_METAL_BIN_SOURCE file path; unset/empty: use the in-tree Metal source file Overrides the binary operations Metal kernel source file loaded at runtime. ds4_metal.m:4951 +runtime/metal DS4_METAL_COMPRESSOR_PAIR_NR4 presence control; unset: off/default; any value including 0 enables Selects the NR4 compressor-pair variant. ds4_metal.m:2650 +runtime/metal DS4_METAL_CONCAT_SOURCE file path; unset/empty: use the in-tree Metal source file Overrides the concatenation Metal kernel source file loaded at runtime. ds4_metal.m:4944 +runtime/metal DS4_METAL_CPY_SOURCE file path; unset/empty: use the in-tree Metal source file Overrides the copy Metal kernel source file loaded at runtime. ds4_metal.m:4943 +runtime/metal DS4_METAL_DECODE_INDEXER_SPARSE_THRESHOLD integer in {64,128,256,512,1024,2048,4096}; default 1024; invalid restores default Sets the compressed-row crossover from dense to sparse indexed attention. ds4.c:20179 +runtime/metal DS4_METAL_DECODE_STAGE_PROFILE unset: off; 1/true/yes/on/all enables all layers; a layer index selects one; 0/false/no/off disables Prints timing/profile diagnostics for decode stage. ds4.c:17395 +runtime/metal DS4_METAL_DECODE_STAGE_PROFILE_LAYER single unsigned layer index; unset/empty: all layers enabled by the parent profile; invalid matches no layer Restricts the corresponding shared graph stage profiler to one layer. ds4.c:28947 +runtime/metal DS4_METAL_DENSE_SOURCE file path; unset/empty: use the in-tree Metal source file Overrides the dense matmul Metal kernel source file loaded at runtime. ds4_metal.m:4935 +runtime/metal DS4_METAL_DISABLE_AFFINE_ROPE_PAIR presence rollback; unset: automatic/default path; any value including 0 disables Disables affine RoPE pair. ds4_metal.m:25169 +runtime/metal DS4_METAL_DISABLE_ATTN_OUT_HC_FUSION nonempty boolean; unset/empty or exact 0: off; every other value: on Disables attn out HC fusion. ds4.c:20304 +runtime/metal DS4_METAL_DISABLE_ATTN_OUT_IDS_CACHE presence rollback; unset: automatic/default path; any value including 0 disables Disables attn out ids cache. ds4_metal.m:27769 +runtime/metal DS4_METAL_DISABLE_ATTN_OUT_LOW_DIRECT presence rollback; unset: automatic/default path; any value including 0 disables Disables attn out low direct. ds4_metal.m:27758 +runtime/metal DS4_METAL_DISABLE_BATCH_HC_NORM_FUSION nonempty value other than exact 0 disables; unset/empty/0 leaves the default enabled path Dominant rollback for batched HC norm fusion. ds4.c:20284 +runtime/metal DS4_METAL_DISABLE_COMPRESSOR_APE_ADD presence rollback; unset: automatic/default path; any value including 0 disables Disables compressor APE add. ds4_metal.m:25391 +runtime/metal DS4_METAL_DISABLE_COMPRESSOR_EXACT_POOL_RATIO4 presence rollback; unset: automatic/default path; any value including 0 disables Disables compressor exact pool ratio4. ds4_metal.m:26377 +runtime/metal DS4_METAL_DISABLE_COMPRESSOR_PAIR_PROJ nonempty boolean; unset/empty or exact 0: off; every other value: on Disables compressor pair proj. ds4_metal.m:23246 +runtime/metal DS4_METAL_DISABLE_COMPRESSOR_QUAD_STORE presence rollback; unset: automatic/default path; any value including 0 disables Disables compressor quad store. ds4.c:23361 +runtime/metal DS4_METAL_DISABLE_COMPRESSOR_RATIO4_DIRECT_POOL presence rollback; unset: automatic/default path; any value including 0 disables Disables compressor ratio4 direct pool. ds4_metal.m:26243 +runtime/metal DS4_METAL_DISABLE_COMPRESSOR_RATIO4_PACK_FUSION presence rollback; unset: automatic/default path; any value including 0 disables Disables compressor ratio4 pack fusion. ds4_metal.m:26197 +runtime/metal DS4_METAL_DISABLE_COMPRESSOR_STORE_ONE presence rollback; unset: automatic/default path; any value including 0 disables Disables compressor store one. ds4_metal.m:23247 +runtime/metal DS4_METAL_DISABLE_CONTIG_F16_F16_COPY value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Disables contig F16 F16 copy. ds4_metal.m:29538 +runtime/metal DS4_METAL_DISABLE_CONTIG_F32_F16_COPY value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Disables contig F32 F16 copy. ds4_metal.m:29314 +runtime/metal DS4_METAL_DISABLE_DECODE_NORM_EXACT_VIEWS presence rollback; unset: automatic/default path; any value including 0 disables Disables decode norm exact views. ds4_metal.m:36173 +runtime/metal DS4_METAL_DISABLE_DECODE_ROUTER_BIAS_EXACT_VIEWS presence rollback; unset: automatic/default path; any value including 0 disables Disables decode router bias exact views. ds4_metal.m:39363 +runtime/metal DS4_METAL_DISABLE_DSPARK_CAPTURE_FUSED_LAST nonempty boolean; unset/empty or exact 0: off; every other value: on Disables DSpark capture fused last. ds4.c:28188 +runtime/metal DS4_METAL_DISABLE_DSPARK_EXACTN_BATCH_HEAD nonempty boolean; unset/empty or exact 0: off; every other value: on Disables DSpark exactn batch head. ds4.c:37538 +runtime/metal DS4_METAL_DISABLE_EXACT_ROWS_PERSISTENT_CACHE nonempty boolean; unset/empty or exact 0: off; every other value: on Disables exact rows persistent cache. ds4_metal.m:12998 +runtime/metal DS4_METAL_DISABLE_GATHERED_KV_PAD_FUSION presence rollback; unset: automatic/default path; any value including 0 disables Disables gathered KV pad fusion. ds4_metal.m:29682 +runtime/metal DS4_METAL_DISABLE_GATHERED_KV_STAGE presence rollback; unset: automatic/default path; any value including 0 disables Disables gathered KV stage. ds4_metal.m:29653 +runtime/metal DS4_METAL_DISABLE_GLM_DECODE_KV_GROUP4 value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Disables GLM decode KV group4. ds4_metal.m:36708 +runtime/metal DS4_METAL_DISABLE_GLM_QKLOW_SG presence rollback; unset: automatic/default path; any value including 0 disables Disables GLM qklow sg. ds4_metal.m:37831 +runtime/metal DS4_METAL_DISABLE_GLM_STREAMING_EXPERT_EARLY_LOAD presence rollback; unset: automatic/default path; any value including 0 disables Disables GLM streaming expert early load. ds4_metal.m:18056 +runtime/metal DS4_METAL_DISABLE_GLM_STREAMING_EXPERT_SPLIT presence rollback; unset: automatic/default path; any value including 0 disables Disables GLM streaming expert split. ds4_metal.m:39762 +runtime/metal DS4_METAL_DISABLE_GLM_STREAMING_PREFILL_FULL_LAYER presence rollback; unset: automatic/default path; any value including 0 disables Disables GLM streaming prefill full layer. ds4.c:42520 +runtime/metal DS4_METAL_DISABLE_GLM_STREAMING_PREFILL_FULL_LAYER_PREPARE presence rollback; unset: automatic/default path; any value including 0 disables Disables GLM streaming prefill full layer prepare. ds4.c:42538 +runtime/metal DS4_METAL_DISABLE_GLM_STREAMING_PREFILL_SELECTED_ASYNC_LOAD presence rollback; unset: automatic/default path; any value including 0 disables Disables GLM streaming prefill selected async load. ds4.c:46231 +runtime/metal DS4_METAL_DISABLE_GLM_STREAMING_SELECTED_ASYNC_LOAD presence rollback; unset: automatic/default path; any value including 0 disables Disables GLM streaming selected async load. ds4.c:44139 +runtime/metal DS4_METAL_DISABLE_HC_FUSION nonempty boolean; unset/empty or exact 0: off; every other value: on Disables HC fusion. ds4.c:20233 +runtime/metal DS4_METAL_DISABLE_HC_NORM_FUSION nonempty boolean; unset/empty or exact 0: off; every other value: on Disables HC norm fusion. ds4.c:20277 +runtime/metal DS4_METAL_DISABLE_HC_PRODUCER_PRE_NORM_FUSE presence rollback; unset: automatic/default path; any value including 0 disables Disables HC producer pre norm fuse. ds4_metal.m:46507 +runtime/metal DS4_METAL_DISABLE_HC_RMS_SCALE_PROJ presence rollback; unset: automatic/default path; any value including 0 disables Disables HC RMS scale proj. ds4_metal.m:24146 +runtime/metal DS4_METAL_DISABLE_HOT_PIPELINE_STATICS presence rollback; unset: automatic/default path; any value including 0 disables Disables hot pipeline statics. ds4_metal.m:2633 +runtime/metal DS4_METAL_DISABLE_INPLACE_ROPE_PAIR presence rollback; unset: automatic/default path; any value including 0 disables Disables inplace RoPE pair. ds4_metal.m:25168 +runtime/metal DS4_METAL_DISABLE_IQ2_SELECTED_EXPERT_VIEWS presence rollback; unset: automatic/default path; any value including 0 disables Disables IQ2 selected expert views. ds4.c:21069 +runtime/metal DS4_METAL_DISABLE_IQ2_SELECTED_SHARED_OVERLAP presence rollback; unset: automatic/default path; any value including 0 disables Disables IQ2 selected shared overlap. ds4.c:20823 +runtime/metal DS4_METAL_DISABLE_IQ2_STREAM_ADDR_TABLE presence rollback; unset: automatic/default path; any value including 0 disables Disables IQ2 stream address table. ds4_metal.m:42368 +runtime/metal DS4_METAL_DISABLE_IQ2_XXS_SSD_PREFILL_MM value-aware boolean; default off; true disables and dominates ENABLE; false leaves automatic policy Rolls grouped IQ2_XXS/Q2_K SSD-prefill MM back to sparse matvec. ds4_metal.m:44750 +runtime/metal DS4_METAL_DISABLE_KV_FUSION nonempty boolean; unset/empty or exact 0: off; every other value: on Disables KV fusion. ds4.c:20257 +runtime/metal DS4_METAL_DISABLE_M1_IQ2_MID_ONLY presence rollback; unset: automatic/default path; any value including 0 disables Disables M1 IQ2 mid only. ds4_metal.m:14590 +runtime/metal DS4_METAL_DISABLE_M3_COMPRESSOR_EXACT_POOL_RATIO4 presence rollback; unset: automatic/default path; any value including 0 disables Disables M3 compressor exact pool ratio4. ds4_metal.m:26379 +runtime/metal DS4_METAL_DISABLE_M3_COMPRESSOR_PAIR_STATE_STORE presence rollback; unset: automatic/default path; any value including 0 disables Disables M3 compressor pair state store. ds4_metal.m:23245 +runtime/metal DS4_METAL_DISABLE_M3_GATHERED_KV_STAGE presence rollback; unset: automatic/default path; any value including 0 disables Disables M3 gathered KV stage. ds4_metal.m:29654 +runtime/metal DS4_METAL_DISABLE_M5_COMPRESSOR_EXACT_POOL_RATIO4 presence rollback; unset: automatic/default path; any value including 0 disables Disables M5 compressor exact pool ratio4. ds4_metal.m:26382 +runtime/metal DS4_METAL_DISABLE_M5_COMP_FINALIZE_FUSE presence rollback; unset: automatic/default path; any value including 0 disables Disables M5 comp finalize fuse. ds4.c:23474 +runtime/metal DS4_METAL_DISABLE_M5_FLASH_ATTN_PACKED32_REDUCE presence rollback; unset: automatic/default path; any value including 0 disables Disables M5 flash attn packed32 reduce. ds4_metal.m:31490 +runtime/metal DS4_METAL_DISABLE_M5_HC_NORM_MIX_CLUSTER2 presence rollback; unset: automatic/default path; any value including 0 disables Disables M5 HC norm mix cluster2. ds4_metal.m:46462 +runtime/metal DS4_METAL_DISABLE_M5_HC_PRODUCER_PRE_NORM_FUSE presence rollback; unset: automatic/default path; any value including 0 disables Disables M5 HC producer pre norm fuse. ds4_metal.m:46514 +runtime/metal DS4_METAL_DISABLE_M5_IQ2_PAIR_PACK2 presence rollback; unset: automatic/default path; any value including 0 disables Disables M5 IQ2 pair pack2. ds4_metal.m:41986 +runtime/metal DS4_METAL_DISABLE_M5_PACKED_ZERO_MASK presence rollback; unset: automatic/default path; any value including 0 disables Disables M5 packed zero mask. ds4_metal.m:31446 +runtime/metal DS4_METAL_DISABLE_M5_PARALLEL_FULL_FFN presence rollback; unset: automatic/default path; any value including 0 disables Disables M5 parallel full FFN. ds4.c:22534 +runtime/metal DS4_METAL_DISABLE_M5_PERSISTENT_ZERO_ATTN_MASK presence rollback; unset: automatic/default path; any value including 0 disables Disables M5 persistent zero attn mask. ds4_metal.m:31444 +runtime/metal DS4_METAL_DISABLE_M5_Q8_HC_VEC presence rollback; unset: automatic/default path; any value including 0 disables Disables M5 Q8 HC vec. ds4_metal.m:47546 +runtime/metal DS4_METAL_DISABLE_M5_QKV_PAIR_COMPRESSOR_FUSE presence rollback; unset: automatic/default path; any value including 0 disables Disables M5 QKV pair compressor fuse. ds4.c:22870 +runtime/metal DS4_METAL_DISABLE_M5_QKV_PAIR_QUAD_FUSE presence rollback; unset: automatic/default path; any value including 0 disables Disables M5 QKV pair quad fuse. ds4.c:22866 +runtime/metal DS4_METAL_DISABLE_M5_ROUTER_PROJECT_SELECT_FUSE presence rollback; unset: automatic/default path; any value including 0 disables Disables M5 router project select fuse. ds4.c:24589 +runtime/metal DS4_METAL_DISABLE_METAL4 value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Disables metal4. ds4_metal.m:2994 +runtime/metal DS4_METAL_DISABLE_MOE_MM_ID_PAIR_SWIGLU presence rollback; unset: automatic/default path; any value including 0 disables Disables MoE MM ID pair SwiGLU. ds4_metal.m:44934 +runtime/metal DS4_METAL_DISABLE_MOE_MM_ID_USE_RESOURCES presence rollback; unset: automatic/default path; any value including 0 disables Disables MoE MM ID use resources. ds4_metal.m:35135 +runtime/metal DS4_METAL_DISABLE_MXFP4_SELECTED_EXPERT_VIEWS presence rollback; unset: automatic/default path; any value including 0 disables Disables MXFP4 selected expert views. ds4.c:21000 +runtime/metal DS4_METAL_DISABLE_PERSISTENT_ZERO_ATTN_MASK presence rollback; unset: automatic/default path; any value including 0 disables Disables persistent zero attn mask. ds4_metal.m:31451 +runtime/metal DS4_METAL_DISABLE_PRE_M5_ATTN_INV_ROPE_FUSE presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 attn inv RoPE fuse. ds4.c:22465 +runtime/metal DS4_METAL_DISABLE_PRE_M5_COMPRESSOR_EXACT_POOL_RATIO4 presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 compressor exact pool ratio4. ds4_metal.m:26381 +runtime/metal DS4_METAL_DISABLE_PRE_M5_COMPRESSOR_EXACT_REDUCTION_FUSION presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 compressor exact reduction fusion. ds4_metal.m:25905 +runtime/metal DS4_METAL_DISABLE_PRE_M5_COMPRESSOR_QUAD_STORE presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 compressor quad store. ds4.c:23362 +runtime/metal DS4_METAL_DISABLE_PRE_M5_COMPRESSOR_RATIO4_DECODE_PACK_FUSION presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 compressor ratio4 decode pack fusion. ds4_metal.m:26212 +runtime/metal DS4_METAL_DISABLE_PRE_M5_COMP_FINALIZE_FUSE presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 comp finalize fuse. ds4.c:23473 +runtime/metal DS4_METAL_DISABLE_PRE_M5_DECODE_EARLY_PIPELINE_FAST_LOOKUP presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 decode early pipeline fast lookup. ds4.c:27830 +runtime/metal DS4_METAL_DISABLE_PRE_M5_DECODE_EARLY_SECOND_SPLIT12 presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 decode early second split12. ds4.c:27914 +runtime/metal DS4_METAL_DISABLE_PRE_M5_DECODE_EARLY_SPLIT3 presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 decode early split3. ds4.c:27788 +runtime/metal DS4_METAL_DISABLE_PRE_M5_DECODE_EARLY_SPLIT5 presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 decode early split5. ds4.c:27800 +runtime/metal DS4_METAL_DISABLE_PRE_M5_DECODE_PIPELINE_FAST_LOOKUP presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 decode pipeline fast lookup. ds4.c:27842 +runtime/metal DS4_METAL_DISABLE_PRE_M5_DECODE_PORTS presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 decode ports. ds4.c:22440 +runtime/metal DS4_METAL_DISABLE_PRE_M5_DECODE_RAW_ZERO_ATTN_MASK presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 decode raw zero attn mask. ds4_metal.m:29893 +runtime/metal DS4_METAL_DISABLE_PRE_M5_DECODE_SECOND_SPLIT16 presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 decode second split16. ds4.c:27922 +runtime/metal DS4_METAL_DISABLE_PRE_M5_FLASH_ATTN_BATCHED_MEMO presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 flash attn batched memo. ds4_metal.m:3751 +runtime/metal DS4_METAL_DISABLE_PRE_M5_FLASH_ATTN_PACKED32_REDUCE presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 flash attn packed32 reduce. ds4_metal.m:31433 +runtime/metal DS4_METAL_DISABLE_PRE_M5_FLASH_ATTN_PAD_BLK_MEMO presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 flash attn pad blk memo. ds4_metal.m:3609 +runtime/metal DS4_METAL_DISABLE_PRE_M5_HC_NORM_MIX_FUSE presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 HC norm mix fuse. ds4.c:22694 +runtime/metal DS4_METAL_DISABLE_PRE_M5_HC_PRODUCER_PRE_NORM_FUSE presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 HC producer pre norm fuse. ds4_metal.m:46512 +runtime/metal DS4_METAL_DISABLE_PRE_M5_HEAD_RMS_ROPE_PIPELINE_STATIC presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 head RMS RoPE pipeline static. ds4_metal.m:10025 +runtime/metal DS4_METAL_DISABLE_PRE_M5_KV_ROPE_FP8_FUSE presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 KV RoPE fp8 fuse. ds4.c:23282 +runtime/metal DS4_METAL_DISABLE_PRE_M5_MXFP4_MM_ID_PAIR_HALF_SCALE presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 MXFP4 MM ID pair half scale. ds4_metal.m:45095 +runtime/metal DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_DECODE_FIXED_ROUTE_PAIR presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 MXFP4 MoE decode fixed route pair. ds4_metal.m:41914 +runtime/metal DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_DECODE_FIXED_ROUTE_SUM6 presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 MXFP4 MoE decode fixed route sum6. ds4_metal.m:41927 +runtime/metal DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_DECODE_NSG1 presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 MXFP4 MoE decode nsg1. ds4_metal.m:41685 +runtime/metal DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_DECODE_STATIC_TRIP presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 MXFP4 MoE decode static trip. ds4_metal.m:41953 +runtime/metal DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_DECODE_SUM6_FULL_ROWS presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 MXFP4 MoE decode sum6 full rows. ds4_metal.m:41940 +runtime/metal DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_DECODE_TG_MULTIPLE presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 MXFP4 MoE decode tg multiple. ds4_metal.m:41902 +runtime/metal DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_MM_ID_DOWN_HALF_LUT presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 MXFP4 MoE MM ID down half lut. ds4_metal.m:45013 +runtime/metal DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_MM_ID_DOWN_TAIL_SIMDGROUP_CULL presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 MXFP4 MoE MM ID down tail simdgroup cull. ds4_metal.m:44996 +runtime/metal DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_MM_ID_MAP_SCATTER presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 MXFP4 MoE MM ID map scatter. ds4_metal.m:44963 +runtime/metal DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_MM_ID_PAIR_SWIGLU_COMPACT_TILE presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 MXFP4 MoE MM ID pair SwiGLU compact tile. ds4_metal.m:44947 +runtime/metal DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_MM_ID_PAIR_TAIL_SIMDGROUP_CULL presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 MXFP4 MoE MM ID pair tail simdgroup cull. ds4_metal.m:44984 +runtime/metal DS4_METAL_DISABLE_PRE_M5_PARALLEL_FULL_FFN presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 parallel full FFN. ds4.c:22533 +runtime/metal DS4_METAL_DISABLE_PRE_M5_Q2_DECODE_SPLIT2_32 presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 q2 decode split2 32. ds4.c:27703 +runtime/metal DS4_METAL_DISABLE_PRE_M5_QKV_NORM_KV_STORE_FUSE presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 QKV norm KV store fuse. ds4.c:23141 +runtime/metal DS4_METAL_DISABLE_PRE_M5_QKV_PAIR_COMPRESSOR_FUSE presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 QKV pair compressor fuse. ds4.c:22869 +runtime/metal DS4_METAL_DISABLE_PRE_M5_QKV_PAIR_QUAD_FUSE presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 QKV pair quad fuse. ds4.c:22865 +runtime/metal DS4_METAL_DISABLE_PRE_M5_ROUTER_SHARED_FUSE presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 router shared fuse. ds4.c:24582 +runtime/metal DS4_METAL_DISABLE_PRE_M5_ROUTER_SIMD_FINALIZE presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 router simd finalize. ds4_metal.m:35793 +runtime/metal DS4_METAL_DISABLE_PRE_M5_ROUTER_SIMD_WEIGHTS_FUSION presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 router simd weights fusion. ds4_metal.m:35802 +runtime/metal DS4_METAL_DISABLE_PRE_M5_ROUTER_TRANSFORM_FINALIZE_FUSION presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 router transform finalize fusion. ds4_metal.m:35806 +runtime/metal DS4_METAL_DISABLE_PRO_Q4_EXPERT_ADDRESS_AUTO presence rollback; unset: automatic/default path; any value including 0 disables Disables pro Q4 expert address auto. ds4_metal.m:19708 +runtime/metal DS4_METAL_DISABLE_PRO_Q4_EXPERT_TABLE_AUTO presence rollback; unset: automatic/default path; any value including 0 disables Disables pro Q4 expert table auto. ds4.c:20867 +runtime/metal DS4_METAL_DISABLE_PRO_Q4_EXPERT_TABLE_PRELOAD presence rollback; unset: automatic/default path; any value including 0 disables Disables pro Q4 expert table preload. ds4.c:58590 +runtime/metal DS4_METAL_DISABLE_Q4_ATTN_OUT_HC_FUSE presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 attn out HC fuse. ds4_metal.m:47590 +runtime/metal DS4_METAL_DISABLE_Q4_ATTN_OUT_TINY_BATCH value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Disables Q4 attn out tiny batch. ds4_metal.m:28372 +runtime/metal DS4_METAL_DISABLE_Q4_BATCH_EXPERT_TABLE presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 batch expert table. ds4_metal.m:44849 +runtime/metal DS4_METAL_DISABLE_Q4_DENSE_PAIR presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 dense pair. ds4_metal.m:21988 +runtime/metal DS4_METAL_DISABLE_Q4_EXACT_BOUNDARY presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 exact boundary. ds4_metal.m:42205 +runtime/metal DS4_METAL_DISABLE_Q4_EXACT_TENSOR_ID presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 exact tensor ID. ds4_metal.m:42185 +runtime/metal DS4_METAL_DISABLE_Q4_EXPERT_ADDRESS_TABLE presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 expert address table. ds4_metal.m:19709 +runtime/metal DS4_METAL_DISABLE_Q4_EXPERT_TABLE presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 expert table. ds4.c:20868 +runtime/metal DS4_METAL_DISABLE_Q4_GATHER_SLOTS presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 gather slots. ds4_metal.m:42287 +runtime/metal DS4_METAL_DISABLE_Q4_GROUP24_EXPERT_TABLE presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 group24 expert table. ds4_metal.m:42167 +runtime/metal DS4_METAL_DISABLE_Q4_GROUP6_EXPERT_TABLE presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 group6 expert table. ds4_metal.m:42133 +runtime/metal DS4_METAL_DISABLE_Q4_GROUP8_EXPERT_TABLE presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 group8 expert table. ds4_metal.m:42150 +runtime/metal DS4_METAL_DISABLE_Q4_GROUPED_BOUNDARY presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 grouped boundary. ds4_metal.m:42115 +runtime/metal DS4_METAL_DISABLE_Q4_GROUPED_EXPERTS presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 grouped experts. ds4_metal.m:42096 +runtime/metal DS4_METAL_DISABLE_Q4_MV_CLASSIC presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 MV classic. ds4_metal.m:21569 +runtime/metal DS4_METAL_DISABLE_Q4_QKV_COMPRESSOR_FUSE presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 QKV compressor fuse. ds4_metal.m:22081 +runtime/metal DS4_METAL_DISABLE_Q4_SELECTED_EXPERT_VIEWS presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 selected expert views. ds4.c:21064 +runtime/metal DS4_METAL_DISABLE_Q4_SSD_PREFILL_ATTN_OUT_EXACTN value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Disables Q4 SSD prefill attn out exactn. ds4_metal.m:28071 +runtime/metal DS4_METAL_DISABLE_Q4_SSD_SESSION_UNION nonempty boolean; unset/empty or exact 0: off; every other value: on Disables Q4 SSD session union. ds4.c:64970 +runtime/metal DS4_METAL_DISABLE_Q4_STREAM_OVERLAP nonempty boolean; unset/empty or exact 0: off; every other value: on Disables Q4 stream overlap. ds4.c:64892 +runtime/metal DS4_METAL_DISABLE_Q4_TABLE_BOUNDARY presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 table boundary. ds4_metal.m:42284 +runtime/metal DS4_METAL_DISABLE_Q8_DECODE_EXACT_VIEWS presence rollback; unset: automatic/default path; any value including 0 disables Disables Q8 decode exact views. ds4_metal.m:12849 +runtime/metal DS4_METAL_DISABLE_QKV_NORM_FUSION nonempty boolean; unset/empty or exact 0: off; every other value: on Disables QKV norm fusion. ds4.c:20262 +runtime/metal DS4_METAL_DISABLE_QKV_PAIR_PROJ nonempty boolean; unset/empty or exact 0: off; every other value: on Disables QKV pair proj. ds4.c:20267 +runtime/metal DS4_METAL_DISABLE_QUEUE_RESIDENCY_SET presence rollback; unset: automatic/default path; any value including 0 disables Disables queue residency set. ds4_metal.m:2123 +runtime/metal DS4_METAL_DISABLE_ROUTED_PAIR_SWIGLU_FUSION presence rollback; unset: automatic/default path; any value including 0 disables Disables routed pair SwiGLU fusion. ds4.c:18422 +runtime/metal DS4_METAL_DISABLE_ROUTER_SELECT_FUSION presence rollback; unset: automatic/default path; any value including 0 disables Disables router select fusion. ds4_metal.m:35782 +runtime/metal DS4_METAL_DISABLE_ROUTER_WEIGHTS_BATCH_FUSION presence rollback; unset: automatic/default path; any value including 0 disables Disables router weights batch fusion. ds4_metal.m:36034 +runtime/metal DS4_METAL_DISABLE_SHARED_DOWN_HC_FUSION nonempty boolean; unset/empty or exact 0: off; every other value: on Disables shared down HC fusion. ds4.c:20299 +runtime/metal DS4_METAL_DISABLE_SHARED_GATE_UP_SWIGLU_FUSION presence rollback; unset: automatic/default path; any value including 0 disables Disables shared gate up SwiGLU fusion. ds4.c:17394 +runtime/metal DS4_METAL_DISABLE_SHARED_KV_PAD presence rollback; unset: automatic/default path; any value including 0 disables Disables shared KV pad. ds4_metal.m:31481 +runtime/metal DS4_METAL_DISABLE_SHARED_ROPE_COEFF presence rollback; unset: automatic/default path; any value including 0 disables Disables shared RoPE coeff. ds4_metal.m:6340 +runtime/metal DS4_METAL_DISABLE_STREAMING_COLD_DECODE_PREFILL presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming cold decode prefill. ds4.c:31817 +runtime/metal DS4_METAL_DISABLE_STREAMING_COMPACT_ADDR presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming compact address. ds4_metal.m:14596 +runtime/metal DS4_METAL_DISABLE_STREAMING_DECODE_PREFILL presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming decode prefill. ds4.c:31766 +runtime/metal DS4_METAL_DISABLE_STREAMING_EXPERT_ADDR_TABLE presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming expert address table. ds4.c:18420 +runtime/metal DS4_METAL_DISABLE_STREAMING_EXPERT_COMBINED_BUFFER presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming expert combined buffer. ds4_metal.m:14021 +runtime/metal DS4_METAL_DISABLE_STREAMING_EXPERT_EARLY_LOAD presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming expert early load. ds4_metal.m:17230 +runtime/metal DS4_METAL_DISABLE_STREAMING_EXPERT_EVICT_DONTNEED presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming expert evict dontneed. ds4_metal.m:14393 +runtime/metal DS4_METAL_DISABLE_STREAMING_EXPERT_HIT_VALIDATOR presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming expert hit validator. ds4_metal.m:14632 +runtime/metal DS4_METAL_DISABLE_STREAMING_EXPERT_HOTLIST presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming expert hotlist. ds4.c:21117 +runtime/metal DS4_METAL_DISABLE_STREAMING_EXPERT_LIVE_INDEX value-aware boolean; default off; true disables and dominates ENABLE Disables dense live-entry index and uses authoritative cache matrix. ds4_metal.m:15441 +runtime/metal DS4_METAL_DISABLE_STREAMING_EXPERT_MASKED_ADDR presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming expert masked address. ds4_metal.m:14626 +runtime/metal DS4_METAL_DISABLE_STREAMING_EXPERT_READAHEAD presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming expert readahead. ds4_metal.m:13194 +runtime/metal DS4_METAL_DISABLE_STREAMING_EXPERT_SLABS presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming expert slabs. ds4_metal.m:14026 +runtime/metal DS4_METAL_DISABLE_STREAMING_EXPERT_TIMING_SUMMARY presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming expert timing summary. ds4_metal.m:13060 +runtime/metal DS4_METAL_DISABLE_STREAMING_FULL_EXPERT_ADDR_TABLE presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming full expert address table. ds4_metal.m:14714 +runtime/metal DS4_METAL_DISABLE_STREAMING_IQ2_CPU_ROUTER presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming IQ2 CPU router. ds4.c:20766 +runtime/metal DS4_METAL_DISABLE_STREAMING_LAYER_BATCH presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming layer batch. ds4.c:18066 +runtime/metal DS4_METAL_DISABLE_STREAMING_MADVISE_WILLNEED presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming madvise willneed. ds4.c:18039 +runtime/metal DS4_METAL_DISABLE_STREAMING_PREFILL_BATCH_SELECTED_ADDR presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming prefill batch selected address. ds4.c:18418 +runtime/metal DS4_METAL_DISABLE_STREAMING_PREFILL_LAYER_MADVISE presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming prefill layer madvise. ds4.c:18293 +runtime/metal DS4_METAL_DISABLE_STREAMING_PREFILL_LAYER_PAGEIN presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming prefill layer pagein. ds4.c:18261 +runtime/metal DS4_METAL_DISABLE_STREAMING_PREFILL_LAYER_PAGEIN_OVERLAP presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming prefill layer pagein overlap. ds4.c:19148 +runtime/metal DS4_METAL_DISABLE_STREAMING_PREFILL_LAYER_PREAD presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming prefill layer pread. ds4.c:18281 +runtime/metal DS4_METAL_DISABLE_STREAMING_PREFILL_LAYER_PREPARE presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming prefill layer prepare. ds4.c:18273 +runtime/metal DS4_METAL_DISABLE_STREAMING_PREFILL_LAYER_PREPARE_OVERLAP presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming prefill layer prepare overlap. ds4.c:19146 +runtime/metal DS4_METAL_DISABLE_STREAMING_PREFILL_LAYER_READAHEAD presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming prefill layer readahead. ds4.c:18271 +runtime/metal DS4_METAL_DISABLE_STREAMING_PREFILL_SELECTED_ASYNC_LOAD presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming prefill selected async load. ds4.c:46234 +runtime/metal DS4_METAL_DISABLE_STREAMING_PREFILL_SELECTED_MADVISE presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming prefill selected madvise. ds4.c:18251 +runtime/metal DS4_METAL_DISABLE_STREAMING_PREFILL_SELECTED_PAGEIN presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming prefill selected pagein. ds4.c:18241 +runtime/metal DS4_METAL_DISABLE_STREAMING_PREFILL_SELECTED_PROFILE presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming prefill selected profile. ds4.c:18735 +runtime/metal DS4_METAL_DISABLE_STREAMING_PREFILL_SELECTED_READAHEAD presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming prefill selected readahead. ds4.c:19787 +runtime/metal DS4_METAL_DISABLE_STREAMING_PREFILL_SELECTED_READAHEAD_SHARED presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming prefill selected readahead shared. ds4.c:19797 +runtime/metal DS4_METAL_DISABLE_STREAMING_READAHEAD presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming readahead. ds4.c:18032 +runtime/metal DS4_METAL_DISABLE_STREAMING_SELECTED_ASYNC_EARLY_COMMIT presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming selected async early commit. ds4.c:20846 +runtime/metal DS4_METAL_DISABLE_STREAMING_SELECTED_ASYNC_LOAD presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming selected async load. ds4.c:20830 +runtime/metal DS4_METAL_DISABLE_STREAMING_SELECTED_READAHEAD_SHARED_DELAY presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming selected readahead shared delay. ds4.c:21547 +runtime/metal DS4_METAL_DISABLE_STREAMING_SELECTED_SHARED_OVERLAP presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming selected shared overlap. ds4.c:20822 +runtime/metal DS4_METAL_DISABLE_STREAMING_STATIC_DECODE_MAP presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming static decode map. ds4.c:18044 +runtime/metal DS4_METAL_DISABLE_STREAMING_STATIC_MAP_STATE_CACHE presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming static map state cache. ds4.c:18057 +runtime/metal DS4_METAL_DISABLE_SUPPORT_Q8_DECODE_EXACT_VIEWS presence rollback; unset: automatic/default path; any value including 0 disables Disables support Q8 decode exact views. ds4_metal.m:12856 +runtime/metal DS4_METAL_DISABLE_TINY_PAIR_SWIGLU_FUSION presence rollback; unset: automatic/default path; any value including 0 disables Disables tiny pair SwiGLU fusion. ds4_metal.m:44886 +runtime/metal DS4_METAL_DISABLE_TOKEN_EMBED_EXACT_VIEW presence rollback; unset: automatic/default path; any value including 0 disables Disables token embed exact view. ds4_metal.m:11905 +runtime/metal DS4_METAL_DISABLE_ZERO_PREFIX_PREFILL_MASK_CACHE presence rollback; unset: automatic/default path; any value including 0 disables Disables zero prefix prefill mask cache. ds4_metal.m:2403 +runtime/metal DS4_METAL_DSPARK_ACCEPTANCE_ONLY_VERIFY nonempty boolean; unset/empty or exact 0: off; every other value: on Verifies only the draft rows still needed for acceptance after the base target logit. ds4.c:52309 +runtime/metal DS4_METAL_DSPARK_DEVICE_PROPOSER boolean true values enable; false values/unset disable; NO_DEVICE_PROPOSER presence dominates Keeps DSpark Q8 confidence/Markov proposal work on Metal and reads one compact result. ds4.c:34694 +runtime/metal DS4_METAL_DSPARK_EXACT2 nonempty boolean; unset/empty or exact 0: off; every other value: on Enables the resident single-GPU Metal exact-2 verifier. ds4.c:52100 +runtime/metal DS4_METAL_DSPARK_EXACTN nonempty boolean; unset/empty or exact 0: off; every other value: on Enables the single-GPU Metal exact-N verifier. ds4.c:52121 +runtime/metal DS4_METAL_DSPARK_EXACTN_BATCH_HEAD nonempty boolean; unset/empty or exact 0: off; every other value: on Batches the output head across exact-N verifier rows. ds4.c:37536 +runtime/metal DS4_METAL_DSPARK_EXACTN_UNION nonempty boolean; unset/empty or exact 0: off; every other value: on Loads the union of experts for exact-N verifier rows once. ds4.c:52142 +runtime/metal DS4_METAL_DSPARK_EXACT_ROWS_ASYNC_TAILS presence control; unset: off/default; any value including 0 enables Runs exact-row routed tails asynchronously after union routing. ds4.c:37742 +runtime/metal DS4_METAL_DSPARK_EXACT_ROWS_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for DSpark exact rows. ds4.c:37740 +runtime/metal DS4_METAL_DSPARK_HEADLESS_REPLAY unset/empty: enabled; exact 0 disables; every other nonempty value enables Skips output heads for accepted intermediate DSpark replay tokens. ds4.c:52289 +runtime/metal DS4_METAL_DSPARK_NO_DEVICE_PROPOSER presence rollback; unset: automatic/default path; any value including 0 disables Dominant presence-based rollback for the Metal DSpark device proposer. ds4.c:34696 +runtime/metal DS4_METAL_DSPARK_PIN_MAIN_PROJ nonempty value other than exact 0 enables; unset/empty/0 disables mlock-pins only DSpark stage-0 main_norm/main_proj in Metal SSD streaming. ds4.c:39253 +runtime/metal DS4_METAL_DSPARK_PROPOSER_BLOCK_MAX uint32; unset: automatic cache/verifier cap; 0 or invalid: native width; positive: clamped to DSpark/native maximum Caps rows proposed by single-device Metal DSpark. ds4.c:52253 +runtime/metal DS4_METAL_DSPARK_SAFE_EXPERT_COUNT exact 1 enables; unset or any other value disables Caps an explicit expert-count cache request to the safe Metal working-set budget for DSpark SSD streaming. ds4.c:4758 +runtime/metal DS4_METAL_DSV4_HC_SOURCE file path; unset/empty: use the in-tree Metal source file Overrides the DeepSeek hidden-context Metal kernel source file loaded at runtime. ds4_metal.m:4937 +runtime/metal DS4_METAL_DSV4_KV_SOURCE file path; unset/empty: use the in-tree Metal source file Overrides the DeepSeek KV Metal kernel source file loaded at runtime. ds4_metal.m:4939 +runtime/metal DS4_METAL_DSV4_MISC_SOURCE file path; unset/empty: use the in-tree Metal source file Overrides the DeepSeek miscellaneous Metal kernel source file loaded at runtime. ds4_metal.m:4941 +runtime/metal DS4_METAL_DSV4_ROPE_SOURCE file path; unset/empty: use the in-tree Metal source file Overrides the DeepSeek RoPE Metal kernel source file loaded at runtime. ds4_metal.m:4940 +runtime/metal DS4_METAL_DUMP_PREFILL_LOGITS file path; unset/empty: no dump Writes final GPU prefill logits as f32 binary. ds4.c:51157 +runtime/metal DS4_METAL_ENABLE_BATCH_HC_NORM_FUSION legacy value-aware alias; default enabled; exact 0 disables; unset/empty/every other value enables unless DISABLE is active Legacy control for the now-default batched HC norm fusion. ds4.c:20289 +runtime/metal DS4_METAL_ENABLE_COMPRESSOR_EXACT_POOL_RATIO4 presence opt-in; unset: off/automatic; any value including 0 enables Enables compressor exact pool ratio4. ds4_metal.m:26383 +runtime/metal DS4_METAL_ENABLE_COMPRESSOR_PAIR_STATE_STORE presence opt-in; unset: off/automatic; any value including 0 enables Enables compressor pair state store. ds4_metal.m:23241 +runtime/metal DS4_METAL_ENABLE_COMPRESSOR_QUAD_STORE presence opt-in; unset: off/automatic; any value including 0 enables Enables compressor quad store. ds4.c:23354 +runtime/metal DS4_METAL_ENABLE_DSPARK_CAPTURE_FUSED_LAST nonempty boolean; unset/empty or exact 0: off; every other value: on Enables DSpark capture fused last. ds4.c:28186 +runtime/metal DS4_METAL_ENABLE_GATHERED_KV_STAGE presence opt-in; unset: off/automatic; any value including 0 enables Enables gathered KV stage. ds4_metal.m:29651 +runtime/metal DS4_METAL_ENABLE_GLM_STREAMING_SELECTED_ASYNC_LOAD presence opt-in; unset: off/automatic; any value including 0 enables Enables GLM streaming selected async load. ds4.c:44145 +runtime/metal DS4_METAL_ENABLE_HC_NORM_MIX_FUSE presence opt-in; unset: off/automatic; any value including 0 enables Enables HC norm mix fuse. ds4.c:22697 +runtime/metal DS4_METAL_ENABLE_HC_PRODUCER_PRE_NORM_FUSE presence opt-in; unset: off/automatic; any value including 0 enables Enables HC producer pre norm fuse. ds4_metal.m:46518 +runtime/metal DS4_METAL_ENABLE_IQ2_SELECTED_ASYNC_EARLY_COMMIT presence opt-in; unset: off/automatic; any value including 0 enables Enables IQ2 selected async early commit. ds4.c:20845 +runtime/metal DS4_METAL_ENABLE_IQ2_XXS_SSD_PREFILL_MM value-aware boolean; default automatic/on for eligible shape; explicit 0 turns request off unless REQUIRE=1 Overrides automatic IQ2_XXS/Q2_K grouped address-MM selection for SSD prefill. ds4_metal.m:44746 +runtime/metal DS4_METAL_ENABLE_PRO_Q4_EXPERT_ADDRESS_AUTO presence opt-in; unset: off/automatic; any value including 0 enables Enables pro Q4 expert address auto. ds4.c:20809 +runtime/metal DS4_METAL_ENABLE_PRO_Q4_EXPERT_TABLE_AUTO presence opt-in; unset: off/automatic; any value including 0 enables Enables pro Q4 expert table auto. ds4.c:20808 +runtime/metal DS4_METAL_ENABLE_PRO_Q4_SELECTED_EXPERT_VIEWS presence opt-in; unset: off/automatic; any value including 0 enables Enables pro Q4 selected expert views. ds4.c:20805 +runtime/metal DS4_METAL_ENABLE_Q4_ATTN_OUT_TINY_BATCH value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Enables Q4 attn out tiny batch. ds4_metal.m:28381 +runtime/metal DS4_METAL_ENABLE_Q4_BATCH_EXPERT_TABLE presence opt-in; unset: off/automatic; any value including 0 enables Enables Q4 batch expert table. ds4_metal.m:44835 +runtime/metal DS4_METAL_ENABLE_Q4_EXACT_TENSOR_ID presence opt-in; unset: off/automatic; any value including 0 enables Enables Q4 exact tensor ID. ds4_metal.m:42184 +runtime/metal DS4_METAL_ENABLE_Q4_EXPERT_ADDRESS_TABLE presence opt-in; unset: off/automatic; any value including 0 enables Enables Q4 expert address table. ds4.c:20807 +runtime/metal DS4_METAL_ENABLE_Q4_EXPERT_TABLE presence opt-in; unset: off/automatic; any value including 0 enables Enables Q4 expert table. ds4.c:20806 +runtime/metal DS4_METAL_ENABLE_Q4_GATHER_SLOTS presence opt-in; unset: off/automatic; any value including 0 enables Enables Q4 gather slots. ds4_metal.m:42286 +runtime/metal DS4_METAL_ENABLE_Q4_GROUP24_EXPERT_TABLE presence opt-in; unset: off/automatic; any value including 0 enables Enables Q4 group24 expert table. ds4_metal.m:42166 +runtime/metal DS4_METAL_ENABLE_Q4_GROUP6_EXPERT_TABLE presence opt-in; unset: off/automatic; any value including 0 enables Enables Q4 group6 expert table. ds4_metal.m:42132 +runtime/metal DS4_METAL_ENABLE_Q4_GROUP8_EXPERT_TABLE presence opt-in; unset: off/automatic; any value including 0 enables Enables Q4 group8 expert table. ds4_metal.m:42149 +runtime/metal DS4_METAL_ENABLE_Q4_GROUPED_EXPERTS presence opt-in; unset: off/automatic; any value including 0 enables Enables Q4 grouped experts. ds4_metal.m:42095 +runtime/metal DS4_METAL_ENABLE_Q4_QKV_COMPRESSOR_FUSE presence opt-in; unset: off/automatic; any value including 0 enables Enables Q4 QKV compressor fuse. ds4.c:22967 +runtime/metal DS4_METAL_ENABLE_Q4_SELECTED_EXPERT_VIEWS presence opt-in; unset: off/automatic; any value including 0 enables Enables Q4 selected expert views. ds4.c:20804 +runtime/metal DS4_METAL_ENABLE_Q4_SSD_PREFILL_ATTN_OUT_EXACTN value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Enables Q4 SSD prefill attn out exactn. ds4_metal.m:28073 +runtime/metal DS4_METAL_ENABLE_Q4_SSD_SESSION_UNION nonempty boolean; unset/empty or exact 0: off; every other value: on Enables Q4 SSD session union. ds4.c:64981 +runtime/metal DS4_METAL_ENABLE_Q4_STREAM_OVERLAP nonempty boolean; unset/empty or exact 0: off; every other value: on Enables Q4 stream overlap. ds4.c:64890 +runtime/metal DS4_METAL_ENABLE_Q8_DECODE_EXACT_VIEWS presence opt-in; unset: off/automatic; any value including 0 enables Enables Q8 decode exact views. ds4_metal.m:12863 +runtime/metal DS4_METAL_ENABLE_Q8_QKV_COMPRESSOR_FUSE nonempty boolean; unset/empty/exact 0: no streamed/union opt-in; other values enable; eligible resident full-decode remains automatic Extends the automatic resident Q8 QKV/compressor compound fusion to SSD streaming or exact-N union scope. ds4.c:22854 +runtime/metal DS4_METAL_ENABLE_STREAMING_COMPACT_ADDR presence opt-in; unset: off/automatic; any value including 0 enables Enables streaming compact address. ds4_metal.m:14595 +runtime/metal DS4_METAL_ENABLE_STREAMING_EXPERT_ADDR_TABLE presence opt-in; unset: off/automatic; any value including 0 enables Enables streaming expert address table. ds4_metal.m:14602 +runtime/metal DS4_METAL_ENABLE_STREAMING_EXPERT_EVICT_DONTNEED presence opt-in; unset: off/automatic; any value including 0 enables Enables streaming expert evict dontneed. ds4_metal.m:14392 +runtime/metal DS4_METAL_ENABLE_STREAMING_EXPERT_HIT_VALIDATOR presence opt-in; unset: off/automatic; any value including 0 enables Enables streaming expert hit validator. ds4_metal.m:14603 +runtime/metal DS4_METAL_ENABLE_STREAMING_EXPERT_LIVE_INDEX value-aware boolean; default automatic/on for validated IQ2 cache shape; explicit 0 disables Overrides automatic dense live-entry index selection. ds4_metal.m:15440 +runtime/metal DS4_METAL_ENABLE_STREAMING_EXPERT_MASKED_ADDR presence opt-in; unset: off/automatic; any value including 0 enables Enables streaming expert masked address. ds4_metal.m:14604 +runtime/metal DS4_METAL_ENABLE_STREAMING_FULL_EXPERT_ADDR_TABLE presence opt-in; unset: off/automatic; any value including 0 enables Enables streaming full expert address table. ds4_metal.m:14713 +runtime/metal DS4_METAL_ENABLE_STREAMING_IQ2_CPU_ROUTER presence opt-in; unset: off/automatic; any value including 0 enables Enables streaming IQ2 CPU router. ds4.c:20765 +runtime/metal DS4_METAL_ENABLE_STREAMING_MADVISE_WILLNEED presence opt-in; unset: off/automatic; any value including 0 enables Enables streaming madvise willneed. ds4.c:18037 +runtime/metal DS4_METAL_ENABLE_STREAMING_PREFILL_BATCH_SELECTED_ADDR presence opt-in; unset: off/automatic; any value including 0 enables Enables streaming prefill batch selected address. ds4_metal.m:14607 +runtime/metal DS4_METAL_ENABLE_STREAMING_PREFILL_CACHE_SEED presence opt-in; unset: off/automatic; any value including 0 enables Enables streaming prefill cache seed. ds4.c:21086 +runtime/metal DS4_METAL_ENABLE_STREAMING_PREFILL_EXPERT_READAHEAD value-aware boolean; for batches <32 readahead is automatic; for batches >=32 unset/false disables and true enables; global READHEAD rollback and F_NOCACHE still dominate Restores F_RDADVISE immediately before parallel pread for large SSD-prefill batches. ds4_metal.m:13210 +runtime/metal DS4_METAL_ENABLE_STREAMING_PREFILL_LAYER_PAGEIN presence opt-in; unset: off/automatic; any value including 0 enables Enables streaming prefill layer pagein. ds4.c:18259 +runtime/metal DS4_METAL_ENABLE_STREAMING_PREFILL_LAYER_READAHEAD presence opt-in; unset: off/automatic; any value including 0 enables Enables streaming prefill layer readahead. ds4.c:18269 +runtime/metal DS4_METAL_ENABLE_STREAMING_PREFILL_SELECTED_MADVISE presence opt-in; unset: off/automatic; any value including 0 enables Enables streaming prefill selected madvise. ds4.c:18249 +runtime/metal DS4_METAL_ENABLE_STREAMING_PREFILL_SELECTED_PAGEIN presence opt-in; unset: off/automatic; any value including 0 enables Enables streaming prefill selected pagein. ds4.c:18239 +runtime/metal DS4_METAL_ENABLE_STREAMING_PREFILL_SELECTED_READAHEAD presence opt-in; unset: off/automatic; any value including 0 enables Enables streaming prefill selected readahead. ds4.c:19783 +runtime/metal DS4_METAL_ENABLE_STREAMING_PREFILL_SELECTED_READAHEAD_SHARED presence opt-in; unset: off/automatic; any value including 0 enables Enables streaming prefill selected readahead shared. ds4.c:19785 +runtime/metal DS4_METAL_ENABLE_STREAMING_READAHEAD presence opt-in; unset: off/automatic; any value including 0 enables Enables streaming readahead. ds4.c:18030 +runtime/metal DS4_METAL_ENABLE_STREAMING_SELECTED_READAHEAD_SHARED_DELAY presence opt-in; unset: off/automatic; any value including 0 enables Enables streaming selected readahead shared delay. ds4.c:21546 +runtime/metal DS4_METAL_ENABLE_STREAMING_STATIC_DECODE_MAP presence opt-in; unset: off/automatic; any value including 0 enables Enables streaming static decode map. ds4.c:18049 +runtime/metal DS4_METAL_ENABLE_TOKEN_EMBED_EXACT_VIEW presence opt-in; unset: off/automatic; any value including 0 enables Enables token embed exact view. ds4_metal.m:12217 +runtime/metal DS4_METAL_EXACT_VIEW_CACHE_GIB unsigned GiB; default 64; 0 disables size-triggered eviction; MIB overrides it Sets the cached exact-model-view eviction threshold. ds4_metal.m:1332 +runtime/metal DS4_METAL_EXACT_VIEW_CACHE_MIB unsigned MiB; unset: inherit GIB/default; 0 disables size-triggered eviction; overrides GIB Sets the cached exact-model-view eviction threshold with MiB precision. ds4_metal.m:1341 +runtime/metal DS4_METAL_EXACT_VIEW_CACHE_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for exact view cache. ds4_metal.m:1376 +runtime/metal DS4_METAL_FLASH_ATTN_SOURCE file path; unset/empty: use the in-tree Metal source file Overrides the FlashAttention Metal kernel source file loaded at runtime. ds4_metal.m:4934 +runtime/metal DS4_METAL_FLASH_ATTN_STAGE_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for flash attn stage. ds4.c:64962 +runtime/metal DS4_METAL_FLASH_ATTN_STAGE_PROFILE_FILTER substring; unset/empty: all profiled modes/stages Filters FlashAttention stage-profile output by mode or stage substring. ds4_metal.m:11177 +runtime/metal DS4_METAL_GET_ROWS_SOURCE file path; unset/empty: use the in-tree Metal source file Overrides the get-rows Metal kernel source file loaded at runtime. ds4_metal.m:4945 +runtime/metal DS4_METAL_GLM_DISABLE_STREAMING_EXPERT_CACHE presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming expert cache for GLM. ds4_metal.m:39633 +runtime/metal DS4_METAL_GLM_DISABLE_STREAMING_GROUPED_ADDR_PREFILL presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming grouped address prefill for GLM. ds4_metal.m:41047 +runtime/metal DS4_METAL_GLM_DISABLE_STREAMING_SEED_BEFORE_PREFILL presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming seed before prefill for GLM. ds4.c:50884 +runtime/metal DS4_METAL_GLM_DISABLE_STREAMING_TOKEN_PREFILL presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming token prefill for GLM. ds4.c:49514 +runtime/metal DS4_METAL_GLM_MOE_ONE_STAGE_PROFILE unset: off; 1/true/yes/on/all enables all layers; accepts layer lists/ranges; 0/false/no/off disables Prints timing/profile diagnostics for GLM MoE one stage. ds4_metal.m:39930 +runtime/metal DS4_METAL_GLM_MOE_ONE_STAGE_PROFILE_LAYER layer index/list/ranges or all; unset: all layers selected by profiler Restricts GLM one-stage MoE profiling to selected layers. ds4_metal.m:39931 +runtime/metal DS4_METAL_GLM_MOE_STAGE_PROFILE_FILTER substring; unset/empty: all profiled stages Filters GLM MoE stage-profile output. ds4_metal.m:39934 +runtime/metal DS4_METAL_GLM_QKLOW_DEBUG presence diagnostic; unset: off; any value including 0 enables Enables debug diagnostics for GLM qklow. ds4_metal.m:37834 +runtime/metal DS4_METAL_GLM_STREAMING_ASYNC_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for GLM streaming async. ds4.c:44172 +runtime/metal DS4_METAL_GLM_STREAMING_DECODE_FULL_LAYER_MAP presence control; unset: off/default; any value including 0 enables Maps complete GLM layers during SSD-streaming decode instead of decode-only spans. ds4.c:42426 +runtime/metal DS4_METAL_GLM_STREAMING_DECODE_SYNC_EACH_LAYER boolean text; Metal runtime always synchronizes and does not consult it; legacy fallback name read only in ROCm builds Controls per-layer GLM streaming decode synchronization only as a legacy ROCm fallback alias. ds4.c:49590 +runtime/metal DS4_METAL_GLM_STREAMING_PREFILL_FULL_LAYER presence control; unset: off/default; any value including 0 enables Forces full-layer GLM SSD prefill regardless of the token crossover. ds4_metal.m:14708 +runtime/metal DS4_METAL_GLM_STREAMING_PREFILL_FULL_LAYER_MIN_TOKENS positive uint32; default 64 on Metal, 1024 when used as ROCm fallback; 0/invalid restores default Sets the token crossover for GLM full-layer SSD prefill. ds4.c:42503 +runtime/metal DS4_METAL_GLM_STREAMING_PREFILL_SYNC_EACH_LAYER boolean text; Metal runtime always synchronizes and does not consult it; legacy fallback name read only in ROCm builds Controls per-layer GLM streaming prefill synchronization only as a legacy ROCm fallback alias. ds4.c:42206 +runtime/metal DS4_METAL_GLM_STREAMING_TOKEN_PREFILL_MAX uint32; default 64 on Metal, 0 when used as ROCm fallback; 0 disables; invalid restores default Sets largest GLM SSD prefill handled token-major by the decode graph. ds4.c:49493 +runtime/metal DS4_METAL_GLU_SOURCE file path; unset/empty: use the in-tree Metal source file Overrides the GLU Metal kernel source file loaded at runtime. ds4_metal.m:4949 +runtime/metal DS4_METAL_GPU_BATCH_EMBED_MIN uint32 token threshold; default 512; invalid restores default Sets the batch size at which prompt embedding moves from CPU upload to Metal kernels. ds4.c:28694 +runtime/metal DS4_METAL_GPU_BUSY_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for GPU busy. ds4_metal.m:1292 +runtime/metal DS4_METAL_GRAPH_DUMP_LAYER unsigned layer index or all; unset: every layer Restricts graph tensor dumps to one layer. ds4.c:16690 +runtime/metal DS4_METAL_GRAPH_DUMP_LOGITS file path; unset/empty: no graph-logit dump Writes Metal graph-test logits as f32 binary. ds4.c:38890 +runtime/metal DS4_METAL_GRAPH_DUMP_NAME substring; unset/empty: every tensor name Restricts graph tensor dumps by tensor-name substring. ds4.c:16686 +runtime/metal DS4_METAL_GRAPH_DUMP_POS unsigned token position; unset: every position Restricts graph tensor dumps to one token position. ds4.c:16697 +runtime/metal DS4_METAL_GRAPH_DUMP_PREFIX path/prefix; unset/empty: tensor dumping disabled Enables graph tensor dumps and supplies the filename prefix. ds4.c:64958 +runtime/metal DS4_METAL_GRAPH_DUMP_TRACE presence diagnostic; unset: off; any value including 0 enables Emits trace diagnostics for graph dump. ds4.c:16727 +runtime/metal DS4_METAL_GRAPH_OUTPUT_ROW zero-based row smaller than current batch; default final row; invalid restores final row Chooses which prefill output row is projected to logits. ds4.c:35656 +runtime/metal DS4_METAL_GRAPH_PREFILL_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for graph prefill. ds4.c:35333 +runtime/metal DS4_METAL_GRAPH_PREFILL_SPLIT_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for graph prefill split. ds4.c:66014 +runtime/metal DS4_METAL_GRAPH_PROMPT_TOKENS integer 1..prompt length; default full prompt Limits prompt length used by the Metal graph parity test. ds4.c:38833 +runtime/metal DS4_METAL_GRAPH_RAW_CAP positive rows; default from SWA window+prefill; clamped to [raw_window,min(ctx,8192)] Overrides raw sliding-window KV ring capacity. ds4.c:38200 +runtime/metal DS4_METAL_GRAPH_TEACHER_FORCE presence control; unset: off/default; any value including 0 enables Feeds CPU reference state back into the first-token graph trace at each layer. ds4.c:27569 +runtime/metal DS4_METAL_GRAPH_TOKEN_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for graph token. ds4.c:32192 +runtime/metal DS4_METAL_GRAPH_TOKEN_SECOND_SPLIT_LAYERS integer 0..layer count; default 0 plus eligible automatic pre-M5 schedules; explicit value wins Overrides second command-buffer split layer for token decode. ds4.c:27856 +runtime/metal DS4_METAL_GRAPH_TOKEN_SPLIT_LAYERS integer 0..layer count; default 4 on Apple and 0 elsewhere, with eligible pre-M5 adaptive override Overrides first command-buffer split layer for token decode. ds4.c:27742 +runtime/metal DS4_METAL_GRAPH_TRACE_CACHE presence control; unset: off/default; any value including 0 enables Prints raw KV cache parity diagnostics in the graph prompt test. ds4.c:38902 +runtime/metal DS4_METAL_GRAPH_TRACE_COMP presence control; unset: off/default; any value including 0 enables Prints compressed-cache parity diagnostics in the graph prompt test. ds4.c:38903 +runtime/metal DS4_METAL_GRAPH_TRACE_LAYERS presence control; unset: off/default; any value including 0 enables Enables per-layer first-token CPU/GPU graph tracing. ds4.c:27566 +runtime/metal DS4_METAL_GRAPH_TRACE_STAGE_LAYER signed layer index; unset gives -1/no stage-layer selection Selects the layer used by first-token stage tracing. ds4.c:27570 +runtime/metal DS4_METAL_HC_NORM_FUSION_CHECK nonempty boolean; unset/empty or exact 0: off; every other value: on Compares fused HC normalization against the reference result. ds4.c:20348 +runtime/metal DS4_METAL_HC_NORM_FUSION_CHECK_TOL positive finite float; default 2e-4; invalid/nonpositive restores default Sets the numerical tolerance for the HC norm-fusion oracle. ds4.c:20357 +runtime/metal DS4_METAL_HC_STABLE boolean empty/1/true/yes/on vs 0/false/no/off; default on Compiles stable hidden-context drift arithmetic into the Metal library. ds4_metal.m:7046 +runtime/metal DS4_METAL_INDEXER_STAGE_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for indexer stage. ds4.c:17396 +runtime/metal DS4_METAL_IQ2_XXS_SSD_PREFILL_MM_STATS value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Collects/prints statistics for IQ2 XXS SSD prefill MM. ds4_metal.m:6979 +runtime/metal DS4_METAL_KV_RAW_F32 boolean empty/1/true/yes/on vs 0/false/no/off; default off Compiles raw KV storage as F32 for drift diagnosis. ds4_metal.m:7048 +runtime/metal DS4_METAL_LAYER_STAGE_PROFILE unset: off; 1/true/yes/on/all enables all layers; a layer index selects one; 0/false/no/off disables Prints timing/profile diagnostics for layer stage. ds4.c:64960 +runtime/metal DS4_METAL_LAYER_STAGE_PROFILE_LAYER single unsigned layer index; unset/empty: all layers enabled by the parent profile; invalid matches no layer Restricts the corresponding shared graph stage profiler to one layer. ds4.c:28936 +runtime/metal DS4_METAL_MATH_SAFE boolean empty/1/true/yes/on vs 0/false/no/off; default off Compiles Metal shaders with strict/safe IEEE math instead of fast math. ds4_metal.m:7050 +runtime/metal DS4_METAL_MEMORY_REPORT presence control; unset: off/default; any value including 0 enables Prints Metal allocation/cache/residency memory reports. ds4.c:38857 +runtime/metal DS4_METAL_MODEL_UNTRACKED presence control; unset: off/default; any value including 0 enables Creates mapped model buffers with untracked Metal hazard tracking. ds4_metal.m:1546 +runtime/metal DS4_METAL_MODEL_VIEW_MAX_GIB positive integer GiB; default device maximum (128-GiB cap for already-split span maps); cannot exceed device maximum Caps each no-copy mapped Metal model view. ds4_metal.m:2215 +runtime/metal DS4_METAL_MODEL_WARMUP_STRIDE_KB integer 1..1048576 KiB, at least one page; unset inherits MB/default; overrides STRIDE_MB Sets the model-view warmup touch stride with KiB precision. ds4_metal.m:3056 +runtime/metal DS4_METAL_MODEL_WARMUP_STRIDE_MB integer 1..1024 MiB; default 1 MiB; STRIDE_KB overrides Sets the model-view warmup touch stride. ds4_metal.m:3048 +runtime/metal DS4_METAL_MOE_MM_ID_USE_RESOURCES presence control; unset: off/default; any value including 0 enables Declares MM-ID MoE resource usage explicitly on the command encoder. ds4_metal.m:35134 +runtime/metal DS4_METAL_MOE_ONE_STAGE_PROFILE unset: off; 1/true/yes/on/all enables all layers; accepts layer lists/ranges; 0/false/no/off disables Prints timing/profile diagnostics for MoE one stage. ds4.c:22541 +runtime/metal DS4_METAL_MOE_ONE_STAGE_PROFILE_LAYER layer index/list/ranges or all; unset: all profiler-selected layers Restricts one-stage MoE profiling to selected layers. ds4_metal.m:43402 +runtime/metal DS4_METAL_MOE_SOURCE file path; unset/empty: use the in-tree Metal source file Overrides the MoE Metal kernel source file loaded at runtime. ds4_metal.m:4936 +runtime/metal DS4_METAL_MOE_STAGE_PROFILE unset: off; 1/true/yes/on/all enables all layers; accepts layer lists/ranges; 0/false/no/off disables Prints timing/profile diagnostics for MoE stage. ds4.c:64964 +runtime/metal DS4_METAL_MOE_STAGE_PROFILE_FILTER substring; unset/empty: all profiled stages Filters MoE stage-profile output. ds4_metal.m:43404 +runtime/metal DS4_METAL_MOE_STAGE_PROFILE_LAYER layer index/list/ranges or all; unset: all profiler-selected layers Restricts batched MoE stage profiling to selected layers. ds4_metal.m:45312 +runtime/metal DS4_METAL_MOE_WRITE_CLAMPED_ACT presence control; unset: off/default; any value including 0 enables Makes routed MoE write the clamped activation diagnostic. ds4.c:18421 +runtime/metal DS4_METAL_NORM_RSQRT_DISABLE boolean empty/1/true/yes/on vs 0/false/no/off; default on Compiles unified normalization-rsqrt arithmetic into the Metal library. ds4_metal.m:7047 +runtime/metal DS4_METAL_NORM_SOURCE file path; unset/empty: use the in-tree Metal source file Overrides the normalization Metal kernel source file loaded at runtime. ds4_metal.m:4950 +runtime/metal DS4_METAL_NO_MODEL_WARMUP presence rollback; unset: automatic/default path; any value including 0 disables Disables model warmup. ds4_metal.m:2308 +runtime/metal DS4_METAL_NO_PREFILL_KERNEL_WARMUP presence rollback; unset: automatic/default path; any value including 0 disables Disables prefill kernel warmup. ds4.c:28792 +runtime/metal DS4_METAL_NO_RESIDENCY presence rollback; unset: automatic/default path; any value including 0 disables Disables residency. ds4_metal.m:2091 +runtime/metal DS4_METAL_OUTPUT_STAGE_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for output stage. ds4.c:17397 +runtime/metal DS4_METAL_PREFILL_CHUNK positive token count used only when CLI chunk is absent; default full prompt, or 4096 for long non-PRO and 8192 for long PRO prompts; <=0 keeps automatic/full prompt Provides the historical environment fallback for prefill chunk size. ds4.c:12629 +runtime/metal DS4_METAL_PRO_Q4_CPU_ROUTER nonempty boolean; unset/empty or exact 0: off; every other value: on Uses the CPU router for PRO Q4 selected-expert decode. ds4.c:20761 +runtime/metal DS4_METAL_PRO_Q4_CPU_ROUTER_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for pro Q4 CPU router. ds4.c:21471 +runtime/metal DS4_METAL_Q4_ADDR_USE_RESOURCES presence control; unset: off/default; any value including 0 enables Declares Q4 address-table resources explicitly on encoders. ds4_metal.m:20257 +runtime/metal DS4_METAL_Q4_EXPERT_GROUP_SIZE positive uint32; default 32; clamped to total expert count Sets experts processed per grouped Q4 dispatch. ds4_metal.m:34054 +runtime/metal DS4_METAL_Q4_EXPERT_TABLE_GROUP_SIZE integer 2..total experts; default/invalid 1 (ungrouped) Sets grouped exact-view width while building Q4 expert tables. ds4_metal.m:19602 +runtime/metal DS4_METAL_Q4_EXPERT_TABLE_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for Q4 expert table. ds4_metal.m:20101 +runtime/metal DS4_METAL_Q4_GROUP24_BASE_VIEWS presence control; unset: off/default; any value including 0 enables Uses broad base model views for Q4 group-24 instead of exact views. ds4_metal.m:42171 +runtime/metal DS4_METAL_Q4_GROUP24_EXACT_VIEWS presence control; unset: off/default; any value including 0 enables Uses exact mapped views for Q4 group-24 experts. ds4_metal.m:42170 +runtime/metal DS4_METAL_Q4_GROUPED_CACHE_VIEWS presence control; unset: off/default; any value including 0 enables Caches exact Q4 grouped expert views. ds4_metal.m:42117 +runtime/metal DS4_METAL_Q4_PRO_MAP_GROUPS positive divisor of 384 in 1..384; default/invalid 1 Splits each 384-expert PRO Q4 tensor into this many mapped views. ds4.c:6153 +runtime/metal DS4_METAL_Q4_SELECTED_EXACT_VIEWS presence control; unset: off/default; any value including 0 enables Forces exact/cached views for selected Q4 experts instead of base views. ds4_metal.m:42494 +runtime/metal DS4_METAL_Q4_SELECTED_OVERLAP_SHARED nonempty boolean; unset/empty or exact 0: off; every other value: on Overlaps selected Q4 expert preparation with the shared expert. ds4.c:20789 +runtime/metal DS4_METAL_Q4_SELECTED_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for Q4 selected. ds4.c:37754 +runtime/metal DS4_METAL_Q4_SELECTED_PROFILE_LAYER single nonnegative layer index; unset: every layer Restricts legacy Q4 selected-expert profiling to one layer. ds4_metal.m:42475 +runtime/metal DS4_METAL_Q4_SELECTED_SHARED_EVENT presence control; unset: off/default; any value including 0 enables Coordinates selected Q4 work with a shared Metal event. ds4_metal.m:42490 +runtime/metal DS4_METAL_Q4_SELECTED_TRANSIENT_VIEWS presence control; unset: off/default; any value including 0 enables Uses transient exact views for selected Q4 experts. ds4_metal.m:42498 +runtime/metal DS4_METAL_Q4_SELECTED_USE_BASE_VIEWS presence control; unset: off/default; any value including 0 enables Uses broad base model views for selected Q4 experts. ds4_metal.m:42493 +runtime/metal DS4_METAL_Q4_TABLE_BIND_ANCHORS presence control; unset: off/default; any value including 0 enables Binds anchor buffers alongside the Q4 expert address table. ds4_metal.m:19714 +runtime/metal DS4_METAL_Q4_TABLE_MODEL_RESIDENCY_SET presence control; unset: off/default; any value including 0 enables Adds Q4 expert table allocations to the model residency set. ds4_metal.m:19655 +runtime/metal DS4_METAL_Q4_TABLE_PER_TENSOR_RESIDENCY_SET presence control; unset: off/default; any value including 0 enables Builds separate residency sets per Q4 expert tensor. ds4_metal.m:19758 +runtime/metal DS4_METAL_Q4_TABLE_QUEUE_RESIDENCY_SET presence control; unset: off/default; any value including 0 enables Attaches Q4 expert table residency sets to command queues. ds4_metal.m:19613 +runtime/metal DS4_METAL_Q4_TABLE_RESIDENCY_SET presence control; unset: off/default; any value including 0 enables Enables Q4 expert table residency-set handling. ds4_metal.m:19757 +runtime/metal DS4_METAL_Q4_TABLE_USE_RESOURCES presence control; unset: off/default; any value including 0 enables Declares Q4 table resources explicitly on encoders. ds4_metal.m:20256 +runtime/metal DS4_METAL_Q8_DECODE_EXACT_VIEW_MAX_MIB integer 1..4096 MiB; default 1024; above max clamps, below min/invalid restores default Caps weight ranges eligible for Q8 exact model views. ds4_metal.m:12874 +runtime/metal DS4_METAL_Q8_MV_EXT_MAX_TOKENS integer 2..128; default 16; above max clamps, below min/invalid restores default Sets largest batch handled by extended Q8 matvec. ds4_metal.m:21124 +runtime/metal DS4_METAL_Q8_MV_NSG integer 1..8 simdgroups; default 4, or 2 with TP world=2; above max clamps, below min/invalid restores default Overrides simdgroups per Q8 matvec threadgroup. ds4.c:22557 +runtime/metal DS4_METAL_Q8_PREFILL_PROFILE value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Prints timing/profile diagnostics for Q8 prefill. ds4_metal.m:21278 +runtime/metal DS4_METAL_Q8_PREFILL_PROFILE_FILTER substring matched against generated operation label; unset/empty: all eligible calls Filters Q8 prefill profiling. ds4_metal.m:21293 +runtime/metal DS4_METAL_Q_STAGE_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for q stage. ds4.c:29097 +runtime/metal DS4_METAL_REPEAT_SOURCE file path; unset/empty: use the in-tree Metal source file Overrides the repeat Metal kernel source file loaded at runtime. ds4_metal.m:4948 +runtime/metal DS4_METAL_REQUIRE_COMPRESSOR_EXACT_POOL_RATIO4 presence strict check; unset: fallback allowed; any value including 0 requires the path Requires compressor exact pool ratio4 and makes eligible fallback fail closed. ds4_metal.m:26385 +runtime/metal DS4_METAL_REQUIRE_EXACT_ROWS_PERSISTENT_CACHE nonempty boolean; unset/empty or exact 0: off; every other value: on Requires exact rows persistent cache and makes eligible fallback fail closed. ds4_metal.m:13000 +runtime/metal DS4_METAL_REQUIRE_GATHERED_KV_STAGE presence strict check; unset: fallback allowed; any value including 0 requires the path Requires gathered KV stage and makes eligible fallback fail closed. ds4_metal.m:29656 +runtime/metal DS4_METAL_REQUIRE_IQ2_XXS_SSD_PREFILL_MM value-aware boolean; default implicit fail-closed only with complete selected-address domain; explicit 1 is strict, 0 permits fallback Makes eligible IQ2_XXS/Q2_K grouped SSD-prefill MM fail closed. ds4_metal.m:44748 +runtime/metal DS4_METAL_REQUIRE_M1_IQ2_MID_ONLY presence strict check; unset: fallback allowed; any value including 0 requires the path Requires M1 IQ2 mid only and makes eligible fallback fail closed. ds4_metal.m:42040 +runtime/metal DS4_METAL_REQUIRE_OUTPUT_HC_WEIGHTS4 presence strict check; unset: fallback allowed; any value including 0 requires the path Requires output HC weights4 and makes eligible fallback fail closed. ds4_metal.m:46755 +runtime/metal DS4_METAL_REQUIRE_Q4_ATTN_OUT_TINY_BATCH value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Requires Q4 attn out tiny batch and makes eligible fallback fail closed. ds4_metal.m:28349 +runtime/metal DS4_METAL_REQUIRE_Q4_SSD_PREFILL_ATTN_OUT_EXACTN value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Requires Q4 SSD prefill attn out exactn and makes eligible fallback fail closed. ds4_metal.m:28065 +runtime/metal DS4_METAL_REQUIRE_Q4_SSD_SESSION_UNION nonempty boolean; unset/empty or exact 0: off; every other value: on Requires Q4 SSD session union and makes eligible fallback fail closed. ds4.c:64974 +runtime/metal DS4_METAL_REQUIRE_Q8_QKV_COMPRESSOR_FUSE nonempty boolean; unset/empty/exact 0: fallback allowed; other values require and imply the streamed/union enable Requires eligible Q8 QKV/compressor compound fusion and fails closed. ds4.c:22850 +runtime/metal DS4_METAL_RESUME_PREFILL_MIN integer token threshold; default 4; <=0 disables resume-prefill Sets the minimum shared-prefix suffix that uses batched resume-prefill. ds4.c:38229 +runtime/metal DS4_METAL_ROPE_EXP2_LOG2 boolean empty/1/true/yes/on vs 0/false/no/off; default off Compiles the exp2/log2 RoPE drift variant. ds4_metal.m:7049 +runtime/metal DS4_METAL_SELECTED_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for selected. ds4.c:37753 +runtime/metal DS4_METAL_SELECTED_PROFILE_LAYER single nonnegative layer index; unset: every layer Restricts selected-expert profiling to one layer. ds4_metal.m:42473 +runtime/metal DS4_METAL_SESSION_BATCH_LOG presence diagnostic; unset: off; any value including 0 enables Logs session batch decisions. ds4.c:65978 +runtime/metal DS4_METAL_SESSION_BATCH_QKV default enabled; exact 0 disables; every other value/unset leaves enabled Controls native batched QKV work for multi-session decode. ds4.c:65292 +runtime/metal DS4_METAL_SESSION_BATCH_SHARED default enabled; exact 0 disables; every other value/unset leaves enabled Controls native batched shared-expert work for multi-session decode. ds4.c:65242 +runtime/metal DS4_METAL_SET_ROWS_SOURCE file path; unset/empty: use the in-tree Metal source file Overrides the set-rows Metal kernel source file loaded at runtime. ds4_metal.m:4952 +runtime/metal DS4_METAL_SOFTMAX_SOURCE file path; unset/empty: use the in-tree Metal source file Overrides the softmax Metal kernel source file loaded at runtime. ds4_metal.m:4947 +runtime/metal DS4_METAL_STREAMING_DECODE_PREFILL_MAX integer token maximum; default 64 for wide Flash Q4/MXFP4, 18 for other PRO/Flash, 0 otherwise; <=0 disables Sets maximum SSD-streaming micro-prefill width that reuses decode. ds4.c:31772 +runtime/metal DS4_METAL_STREAMING_EXPERT_AUTO_PRELOAD_CAP uint32 expert cap; default 4096; 0 means unlimited; invalid restores default Caps automatic streaming-expert hotlist preload. ds4.c:21282 +runtime/metal DS4_METAL_STREAMING_EXPERT_BUFFER_MLOCK_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for streaming expert buffer mlock. ds4_metal.m:13995 +runtime/metal DS4_METAL_STREAMING_EXPERT_EARLY_LOAD_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for streaming expert early load. ds4_metal.m:17057 +runtime/metal DS4_METAL_STREAMING_EXPERT_EVICT_DONTNEED_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for streaming expert evict dontneed. ds4_metal.m:14434 +runtime/metal DS4_METAL_STREAMING_EXPERT_HOTLIST hotlist file path; unset/empty: built-in model hotlist Loads the streaming-expert preload order from a file. ds4.c:21322 +runtime/metal DS4_METAL_STREAMING_EXPERT_HOTLIST_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for streaming expert hotlist. ds4.c:32055 +runtime/metal DS4_METAL_STREAMING_EXPERT_LAYER_STATS presence diagnostic; unset: off; any value including 0 enables Collects/prints statistics for streaming expert layer. ds4_metal.m:4637 +runtime/metal DS4_METAL_STREAMING_EXPERT_LAYER_STATS_DELTA presence control; unset: off/default; any value including 0 enables Prints delta statistics for streaming expert layer. ds4_metal.m:4674 +runtime/metal DS4_METAL_STREAMING_EXPERT_NOCACHE nonempty value whose first character is not 0 enables; unset/empty/0 disables Uses a reopened F_NOCACHE descriptor for SSD expert preads. ds4_metal.m:12600 +runtime/metal DS4_METAL_STREAMING_EXPERT_PREAD_POOL default enabled; exact 0 disables; every other value/unset keeps enabled Controls reuse of persistent expert-pread worker threads. ds4_metal.m:13431 +runtime/metal DS4_METAL_STREAMING_EXPERT_PREAD_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for streaming expert pread. ds4_metal.m:17003 +runtime/metal DS4_METAL_STREAMING_EXPERT_PREAD_SPLIT integer clamped 1..8; unset: automatic 1 below 64 cache experts, 4 at 64+ Sets aligned requests per expert pread. ds4_metal.m:13699 +runtime/metal DS4_METAL_STREAMING_EXPERT_PREAD_THREADS unsigned integer clamped 1..18; default 9; invalid restores 9 Sets expert-pread worker limit. ds4_metal.m:13335 +runtime/metal DS4_METAL_STREAMING_EXPERT_PROFILE_SUMMARY presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for streaming expert. ds4_metal.m:13059 +runtime/metal DS4_METAL_STREAMING_EXPERT_SLAB_MB positive unsigned MiB; default 4096; 0/invalid restores default Sets target allocation size for streaming-expert slabs. ds4_metal.m:14037 +runtime/metal DS4_METAL_STREAMING_EXPERT_SPLIT_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for streaming expert split. ds4_metal.m:43776 +runtime/metal DS4_METAL_STREAMING_EXPERT_TIMING_SUMMARY presence control; unset: off/default; any value including 0 enables Prints timing/profile diagnostics for streaming expert. ds4_metal.m:13058 +runtime/metal DS4_METAL_STREAMING_IQ2_CPU_ROUTER_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for streaming IQ2 CPU router. ds4.c:21472 +runtime/metal DS4_METAL_STREAMING_MAP_TRACE nonempty value other than exact 0 enables; unset/empty/0 disables Emits SSD model-map decisions. ds4_metal.m:4848 +runtime/metal DS4_METAL_STREAMING_PREFILL_BATCH_SELECTED_ADDR_MAX integer token maximum; default 800 for 384 experts, 760 for 256, 0 otherwise; <=0 disables automatic selection Sets automatic maximum batch width for selected-address SSD prefill. ds4_metal.m:14637 +runtime/metal DS4_METAL_STREAMING_PREFILL_BATCH_SELECTED_ADDR_MIN integer token minimum; default 2 for 256/384 experts, 0 otherwise; <=0 disables automatic selection Sets automatic minimum batch width for selected-address SSD prefill. ds4_metal.m:14654 +runtime/metal DS4_METAL_STREAMING_PREFILL_BATCH_SELECTED_ADDR_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for streaming prefill batch selected address. ds4_metal.m:18491 +runtime/metal DS4_METAL_STREAMING_PREFILL_CACHE_SEED_K uint32 seed rows; default 1; 0 disables; above 64 clamps to 64 Sets how many prefill routing rows seed the decode expert cache. ds4.c:21095 +runtime/metal DS4_METAL_STREAMING_PREFILL_CACHE_SEED_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for streaming prefill cache seed. ds4_metal.m:17897 +runtime/metal DS4_METAL_STREAMING_PREFILL_LAYER_MADVISE_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for streaming prefill layer madvise. ds4.c:19437 +runtime/metal DS4_METAL_STREAMING_PREFILL_LAYER_PAGEIN_NO_OVERLAP presence rollback; unset: automatic/default path; any value including 0 disables Prevents full-layer page-in preparation from overlapping compute. ds4.c:19144 +runtime/metal DS4_METAL_STREAMING_PREFILL_LAYER_PAGEIN_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for streaming prefill layer pagein. ds4.c:19433 +runtime/metal DS4_METAL_STREAMING_PREFILL_LAYER_PAGEIN_THREADS integer 1..16; default 8; invalid/0 becomes 1; PREPARE_THREADS takes precedence Sets worker count for full-layer page-in preparation. ds4.c:19102 +runtime/metal DS4_METAL_STREAMING_PREFILL_LAYER_PREAD_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for streaming prefill layer pread. ds4.c:19435 +runtime/metal DS4_METAL_STREAMING_PREFILL_LAYER_PREPARE_AHEAD integer 1..4 layers; default 1; invalid/0 becomes 1 Sets number of future layers prepared concurrently. ds4.c:19156 +runtime/metal DS4_METAL_STREAMING_PREFILL_LAYER_PREPARE_NO_OVERLAP presence rollback; unset: automatic/default path; any value including 0 disables Prevents generic full-layer preparation from overlapping compute. ds4.c:19142 +runtime/metal DS4_METAL_STREAMING_PREFILL_LAYER_PREPARE_THREADS integer 1..16; default 8; invalid/0 becomes 1; preferred over PAGEIN_THREADS Sets worker count for full-layer preparation. ds4.c:19098 +runtime/metal DS4_METAL_STREAMING_PREFILL_LAYER_READAHEAD_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for streaming prefill layer readahead. ds4.c:19439 +runtime/metal DS4_METAL_STREAMING_PREFILL_SELECTED_MADVISE_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for streaming prefill selected madvise. ds4.c:19183 +runtime/metal DS4_METAL_STREAMING_PREFILL_SELECTED_MADVISE_THREADS integer 1..16; default inherits layer prepare threads; invalid/0 becomes 1; PREPARE_THREADS preferred Sets worker count for selected-expert madvise preparation. ds4.c:19120 +runtime/metal DS4_METAL_STREAMING_PREFILL_SELECTED_PAGEIN_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for streaming prefill selected pagein. ds4.c:19181 +runtime/metal DS4_METAL_STREAMING_PREFILL_SELECTED_PREPARE_GAP integer 0..8 layers; default 0; above 8 clamps; invalid restores 0 Sets lookahead gap for selected-expert preparation. ds4.c:19132 +runtime/metal DS4_METAL_STREAMING_PREFILL_SELECTED_PREPARE_THREADS integer 1..16 for madvise preparation; default inherits layer prepare threads; invalid/0 becomes 1 Sets worker count for selected-expert preparation. ds4.c:19116 +runtime/metal DS4_METAL_STREAMING_PREFILL_SELECTED_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for streaming prefill selected. ds4.c:18733 +runtime/metal DS4_METAL_STREAMING_PREFILL_SELECTED_READAHEAD_GAP integer 0..8 layers; default 0; above 8 clamps; invalid restores 0 Sets lookahead gap for selected-expert readahead. ds4.c:19805 +runtime/metal DS4_METAL_STREAMING_PREFILL_SELECTED_READAHEAD_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for streaming prefill selected readahead. ds4.c:19890 +runtime/metal DS4_METAL_STREAMING_SELECTED_READAHEAD_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for streaming selected readahead. ds4.c:21564 +runtime/metal DS4_METAL_SUM_ROWS_SOURCE file path; unset/empty: use the in-tree Metal source file Overrides the sum-rows Metal kernel source file loaded at runtime. ds4_metal.m:4946 +runtime/metal DS4_METAL_TEST_POISON_COMPRESSOR_EXACT_REDUCTION_SCRATCH internal test presence flag; unset: off; any value including 0 poisons scratch before the exact reduction Validates that compressor exact-reduction kernels overwrite all scratch state. ds4_metal.m:25923 +runtime/metal DS4_METAL_TP_SESSION_BATCH default enabled; exact 0 disables; every other value/unset leaves enabled Controls batched session evaluation with Metal TP. ds4.c:65120 +runtime/metal DS4_METAL_TRACE_ALLOCS presence diagnostic; unset: off; any value including 0 enables Emits trace diagnostics for allocs. ds4_metal.m:4056 +runtime/metal DS4_METAL_TRACE_M5_FLASH_ATTN_PACKED32_REDUCE presence diagnostic; unset: off; any value including 0 enables Emits trace diagnostics for M5 flash attn packed32 reduce. ds4_metal.m:31506 +runtime/metal DS4_METAL_UNARY_SOURCE file path; unset/empty: use the in-tree Metal source file Overrides the unary operations Metal kernel source file loaded at runtime. ds4_metal.m:4938 +runtime/metal DS4_METAL_UNRETAINED_COMMAND_BUFFERS presence control; unset: off/default; any value including 0 enables Creates Metal command buffers with unretained references. ds4_metal.m:1314 +runtime/metal DS4_METAL_USE_QUEUE_RESIDENCY_SET presence control; unset: off/default; any value including 0 enables Allows queue-residency state to trigger Q4 expert address/table paths. ds4_metal.m:42246 +runtime/moe-debug DS4_MOE_RECORD_SELECTED_HOTLIST nonempty output path; unset=off Record per-layer selected-expert hit counts to a Metal hotlist file. ds4_metal.m:1740 +runtime/moe-debug DS4_MOE_RECORD_SELECTED_HOTLIST_FRESH presence flag; only relevant with HOTLIST; overrides MERGE Start the selected-expert hotlist from empty state. ds4_metal.m:1641 +runtime/moe-debug DS4_MOE_RECORD_SELECTED_HOTLIST_MERGE presence flag; active only when FRESH is absent Merge an existing selected-expert hotlist before recording. ds4_metal.m:1640 +runtime/moe-debug DS4_MOE_RECORD_SELECTED_IDS nonempty output path; unset=off Record routed-MoE six-expert selections; also disables incompatible optimized paths. ds4.c:65100 +runtime/moe-debug DS4_MOE_REPLAY_SELECTED_IDS nonempty input path; unset=off Replay routed-MoE six-expert selections; also disables incompatible optimized paths. ds4.c:25135 +runtime/mtp DS4_MTP_BATCH_VERIFY Pure presence flag: any defined value, including empty or "0", suppresses the exact two-row decode verifier. Unset selects exact decode-2 when draft_n==2 and either strict mode is active or the build is ROCm; other cases already use the generic verifier. Diagnostic rollback from the exact Q8/one-token-equivalent MTP decode-2 verifier to the generic microbatch verifier. ds4.c:70842 +runtime/mtp DS4_MTP_CAPTURE_PREFIX1 Pure presence flag. In the generic verifier with exactly two drafts it enables prefix-1 state capture under strict mode; non-strict mode already captures prefix-1 without the variable. Unset under strict mode instead snapshots and replays a partial acceptance. Let a one-of-two MTP partial acceptance commit the verifier's captured prefix directly, avoiding an exact one-token replay. ds4.c:70948 +runtime/mtp DS4_MTP_CONF_LOG Pure presence flag; default off. It forces materialization of full draft logits, computes the top-2 margin, and after a successful generic microbatch verification prints drafted/committed counts, top candidates, margin, target-next and draft-next. Exact decode-2 success does not emit that generic log line. Inspect MTP draft confidence and compare the recursive draft token with the target verifier result. ds4.c:70710 +runtime/mtp DS4_MTP_EXACT_REPLAY Pure presence flag; default off. In the generic microbatch verifier it forces a pre-verifier frontier snapshot; after verification the snapshot is restored and every accepted draft is decoded sequentially to rebuild exact final state/logits. Validate MTP acceptance while committing through the normal one-token decode path rather than retaining batched-verifier state. ds4.c:70949 +runtime/mtp DS4_MTP_FORCE_SNAPSHOT Pure presence flag; default off. It forces a speculative-frontier snapshot before the generic verifier regardless of draft count or prefix-capture mode; it does not by itself force restoration or replay after a successful full acceptance. Measure/debug snapshot behavior and guarantee a restorable pre-verifier frontier for generic MTP verification. ds4.c:70953 +runtime/mtp DS4_MTP_FULL_LOGITS Pure presence flag; default off. When set, legacy and recursive MTP draft calls write the full vocabulary logits to s->mtp_logits; unset permits the faster top-token-only output unless confidence/margin logic independently needs logits. Force full MTP draft-logit materialization for correctness comparison, inspection, or downstream confidence calculations. ds4.c:64070 +runtime/mtp DS4_MTP_MIN_MARGIN non-negative float; default engine --mtp-margin value Set confidence margin threshold for speculative MTP verification. ds4.c:70703 +runtime/mtp DS4_MTP_PROBE Pure presence flag; default off. For legacy MTP it prepares drafts even when configured depth<=1, compares the previous draft with the next committed token, and prints cumulative hit counts/failures; generated output is unchanged. Measure legacy MTP next-token draft accuracy without enabling speculative acceptance. ds4.c:64800 +runtime/mtp DS4_MTP_SPEC_DISABLE Pure presence flag: any defined value, including empty or "0", disables MTP speculative argmax in CLI/chat/server loops. Unset permits it for greedy temperature<=0 generation with draft depth>1; unrelated split-KV speculation can still be independently requested. Fall back from MTP multi-token speculative evaluation to normal one-token session evaluation. ds4_cli.c:580 +runtime/mtp DS4_MTP_SPEC_LOG Pure presence flag; default off. It only emits diagnostics for first-draft misses, exact/generic verifier failures and sequential fallback misses/acceptance outcomes; it does not select a verifier. Trace why MTP drafts were accepted, partially accepted, rejected, or sent to sequential fallback. ds4.c:70726 +runtime/mtp DS4_MTP_STRICT Pure presence flag; engine quality mode also enables strictness automatically. Strict mode skips the non-strict low-margin shortcut, selects exact decode-2 for two drafts unless DS4_MTP_BATCH_VERIFY is set, and disables default prefix-1 capture unless explicitly restored. Force the exact/quality-oriented MTP verification policy on otherwise non-quality runs. ds4.c:70701 +runtime/mtp DS4_MTP_TIMING Pure presence flag; default off. When set, timestamps and prints draft, snapshot, verifier, prefix/replay and total durations for the path taken; algorithm selection is otherwise unchanged. Profile end-to-end MTP speculative decoding and separate draft, verification and state-commit costs. ds4.c:70709 +runtime/rocm DS4_ROCM_DECODE_STAGE_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm decode stage profile. ds4.c:18071 +runtime/rocm DS4_ROCM_DECODE_STAGE_PROFILE_LAYER layer filter subordinate to DS4_ROCM_DECODE_STAGE_PROFILE; unset or whitespace-only: all layers allowed by the parent flag; otherwise the whitespace-trimmed value must be a complete base-10 strtoul result <= UINT32_MAX equal to the current layer; invalid values match none Restricts the ROCm decode stage profiler to one layer; it does not enable profiling by itself. ds4.c:28943 +runtime/rocm DS4_ROCM_DISABLE_GLM_STREAMING_PREFILL_FULL_LAYER integer selector/tuning value; unset or invalid uses internal automatic/default value Disable/roll back rocm disable glm streaming prefill full layer. ds4.c:42519 +runtime/rocm DS4_ROCM_DISABLE_GLM_STREAMING_PREFILL_FULL_LAYER_PREPARE presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable glm streaming prefill full layer prepare. ds4.c:42537 +runtime/rocm DS4_ROCM_DISABLE_GLM_STREAMING_PREFILL_SELECTED_ASYNC_LOAD presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable glm streaming prefill selected async load. ds4.c:46230 +runtime/rocm DS4_ROCM_DISABLE_GLM_STREAMING_SELECTED_ASYNC_LOAD presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable glm streaming selected async load. ds4.c:44138 +runtime/rocm DS4_ROCM_DISABLE_IQ2_SELECTED_EXPERT_VIEWS presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable iq2 selected expert views. ds4.c:20906 +runtime/rocm DS4_ROCM_DISABLE_IQ2_STREAM_ADDR_TABLE presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable iq2 stream addr table. ds4.c:6425 +runtime/rocm DS4_ROCM_DISABLE_Q4_DENSE_PAIR presence rollback; unset leaves opt-in policy unchanged Disable/roll back rocm disable q4 dense pair. rocm/ds4_rocm_q4.cuh:299 +runtime/rocm DS4_ROCM_DISABLE_Q4_GROUPED_ATTN_A presence rollback; DISABLE wins over enable/require Disable/roll back rocm disable q4 grouped attn a. rocm/ds4_rocm_q4.cuh:573 +runtime/rocm DS4_ROCM_DISABLE_Q4_PREFILL_TILE8 presence rollback; TILE8 is default for 9..4096 tokens Disable/roll back rocm disable q4 prefill tile8. rocm/ds4_rocm_q4.cuh:312 +runtime/rocm DS4_ROCM_DISABLE_Q4_SELECTED_EXPERT_VIEWS presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable q4 selected expert views. ds4.c:20962 +runtime/rocm DS4_ROCM_DISABLE_RESIDENT_IQ2_SORTED presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable resident iq2 sorted. rocm/ds4_rocm_moe_launch.cuh:747 +runtime/rocm DS4_ROCM_DISABLE_ROUTED_PAIR_SWIGLU_FUSION presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable routed pair swiglu fusion. ds4.c:18355 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_COLD_DECODE_PREFILL presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming cold decode prefill. ds4.c:31816 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_DECODE_PREFILL presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming decode prefill. ds4.c:31765 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_EXPERT_ADDR_TABLE presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming expert addr table. ds4.c:18351 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_EXPERT_HOTLIST presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming expert hotlist. ds4.c:21116 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_FULL_EXPERT_ADDR_TABLE presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming full expert addr table. ds4.c:18069 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_LAYER_BATCH presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming layer batch. ds4.c:18065 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_MADVISE_WILLNEED presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming madvise willneed. ds4.c:18038 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_BATCH_SELECTED_ADDR presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming prefill batch selected addr. ds4.c:18349 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_MADVISE presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming prefill layer madvise. ds4.c:18292 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_PAGEIN presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming prefill layer pagein. ds4.c:18260 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_PAGEIN_OVERLAP presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming prefill layer pagein overlap. ds4.c:19147 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_PREAD presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming prefill layer pread. ds4.c:18280 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_PREPARE presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming prefill layer prepare. ds4.c:18272 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_PREPARE_OVERLAP presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming prefill layer prepare overlap. ds4.c:19145 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_READAHEAD presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming prefill layer readahead. ds4.c:18270 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_ASYNC_LOAD presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming prefill selected async load. ds4.c:46233 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_MADVISE presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming prefill selected madvise. ds4.c:18250 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_PAGEIN presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming prefill selected pagein. ds4.c:18240 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_PROFILE presence rollback flag; unset keeps automatic/default path Collect timing/profile diagnostics for rocm disable streaming prefill selected profile. ds4.c:18734 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_READAHEAD presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming prefill selected readahead. ds4.c:19786 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_READAHEAD_SHARED presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming prefill selected readahead shared. ds4.c:19796 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_READAHEAD presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming readahead. ds4.c:18031 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_SELECTED_ASYNC_LOAD presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming selected async load. ds4.c:44136 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_SPLIT_SELECTED presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming split selected. rocm/ds4_rocm_moe_launch.cuh:661 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_STATIC_DECODE_MAP presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming static decode map. ds4.c:18043 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_STATIC_MAP_STATE_CACHE presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming static map state cache. ds4.c:18056 +runtime/rocm DS4_ROCM_DSV4_PREQUANT_DECODE sampled once; unset: enabled; present empty or exact 0: disabled; every other present value: enabled; quality mode and GLM models force it off regardless ROCm DeepSeek-V4 decode: quantizes one-token F32 activations to Q8 once and selects the prequantized Q8_0/DP4A projection kernels instead of the full-F32 activation paths. rocm/ds4_rocm_runtime.cuh:4775 +runtime/rocm DS4_ROCM_ENABLE_Q4_DENSE_PAIR presence opt-in; unset=off; DISABLE takes precedence Enable rocm enable q4 dense pair. rocm/ds4_rocm_q4.cuh:298 +runtime/rocm DS4_ROCM_ENABLE_Q4_GROUPED_ATTN_A presence opt-in; unset=off unless REQUIRE; DISABLE wins Enable rocm enable q4 grouped attn a. rocm/ds4_rocm_q4.cuh:577 +runtime/rocm DS4_ROCM_ENABLE_STREAMING_FULL_EXPERT_ADDR_TABLE presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming full expert addr table. ds4.c:18067 +runtime/rocm DS4_ROCM_ENABLE_STREAMING_MADVISE_WILLNEED presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming madvise willneed. ds4.c:18036 +runtime/rocm DS4_ROCM_ENABLE_STREAMING_PREFILL_BATCH_SELECTED_ADDR presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming prefill batch selected addr. ds4.c:18396 +runtime/rocm DS4_ROCM_ENABLE_STREAMING_PREFILL_CACHE_SEED presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming prefill cache seed. ds4.c:21085 +runtime/rocm DS4_ROCM_ENABLE_STREAMING_PREFILL_LAYER_PAGEIN presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming prefill layer pagein. ds4.c:18258 +runtime/rocm DS4_ROCM_ENABLE_STREAMING_PREFILL_LAYER_READAHEAD presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming prefill layer readahead. ds4.c:18268 +runtime/rocm DS4_ROCM_ENABLE_STREAMING_PREFILL_SELECTED_MADVISE presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming prefill selected madvise. ds4.c:18248 +runtime/rocm DS4_ROCM_ENABLE_STREAMING_PREFILL_SELECTED_PAGEIN presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming prefill selected pagein. ds4.c:18238 +runtime/rocm DS4_ROCM_ENABLE_STREAMING_PREFILL_SELECTED_READAHEAD presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming prefill selected readahead. ds4.c:19782 +runtime/rocm DS4_ROCM_ENABLE_STREAMING_PREFILL_SELECTED_READAHEAD_SHARED presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming prefill selected readahead shared. ds4.c:19784 +runtime/rocm DS4_ROCM_ENABLE_STREAMING_READAHEAD presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming readahead. ds4.c:18029 +runtime/rocm DS4_ROCM_ENABLE_STREAMING_STATIC_DECODE_MAP presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming static decode map. ds4.c:18048 +runtime/rocm DS4_ROCM_GLM_CAUSAL_ATTN_GEMM Enabled by default when unset. Exact "0" or an empty value disables; every other nonempty value enables (including false/off/no), because cuda_env_present only tests nonempty and != "0". Eligibility still requires causal_range && !has_selected; a failed GEMM helper falls through to the scalar attention kernel. Use FP16 BLAS GEMMs for dense causal GLM indexed prefill; =0 is the correctness/performance rollback to the scalar attention kernel. rocm/ds4_rocm_glm.cuh:3283 +runtime/rocm DS4_ROCM_GLM_DISABLE_STREAMING_EXPERT_CACHE Pure presence flag: any defined value, including empty or "0", disables. Unset leaves automatic GLM streaming expert-cache eligibility enabled for supported model/quant/quality/SSD configurations. On ROCm builds DS4_METAL_GLM_DISABLE_STREAMING_EXPERT_CACHE is an accepted fallback alias. Disable selected/resident streamed-expert cache paths and force generic/full-layer expert handling for GLM SSD streaming. ds4.c:21024 +runtime/rocm DS4_ROCM_GLM_DISABLE_STREAMING_SEED_BEFORE_PREFILL Pure presence flag: any defined value, including empty or "0", disables. Unset seeds before prefill whenever SSD streaming is active. On ROCm builds DS4_METAL_GLM_DISABLE_STREAMING_SEED_BEFORE_PREFILL is an accepted fallback alias. Skip the pre-prefill hotlist seed of the streaming expert cache in both one-shot GLM generation and session setup. ds4.c:50883 +runtime/rocm DS4_ROCM_GLM_DISABLE_STREAMING_TOKEN_PREFILL Pure presence flag: any defined value, including empty or "0", disables. Unset leaves the token-major path eligible only for SSD streaming, non-quality mode, a nonempty batch fitting full attention, and n_tokens <= the configured nonzero maximum. DS4_METAL_GLM_DISABLE_STREAMING_TOKEN_PREFILL and generic DS4_GLM_DISABLE_STREAMING_TOKEN_PREFILL are also accepted presence aliases. Roll back GLM SSD-streaming token-major prefill to the normal prefill implementation. ds4.c:49513 +runtime/rocm DS4_ROCM_GLM_GROUPED_QK_LOW sampled once; unset: enabled; present empty or exact 0: disabled; every other present value: enabled Selects the grouped shared-input ROCm kernel for eligible multi-token GLM qk-lowrank projection; disabling uses the per-head/per-token projection kernel. rocm/ds4_rocm_runtime.cuh:4800 +runtime/rocm DS4_ROCM_GLM_GROUPED_VALUE_PROJECT sampled once; unset: enabled; present empty or exact 0: disabled; every other present value: enabled Selects the grouped shared-input ROCm kernel for eligible multi-token GLM value projection; disabling uses the non-grouped batch projection path. rocm/ds4_rocm_runtime.cuh:4791 +runtime/rocm DS4_ROCM_GLM_LAYER_SLICE_TOKEN_DECODE Opt-in truthy parser; unset, empty, "0", false, off, or no (case-insensitive words) are false, every other nonempty value is true. Default off and compiled only for ROCm. Allow a one-token, pos>0 GLM layer-slice with inter-node input/output hidden buffers to use the optimized resident token graph; without it only the no-hidden-buffer case takes that shortcut. ds4.c:43383 +runtime/rocm DS4_ROCM_GLM_SELECTED_ATTN_GEMM Enabled by default when unset. Exact "0" or an empty value disables; every other nonempty value enables (including false/off/no). Eligibility still requires !causal_range && has_selected; failure/ineligibility falls through to the scalar attention kernel. Gather per-token selected cache rows into FP16 matrices and use strided-batched BLAS GEMMs for GLM selected indexed prefill; =0 forces the scalar path. rocm/ds4_rocm_glm.cuh:3239 +runtime/rocm DS4_ROCM_GLM_SELECTED_ATTN_HEAD_TILE Unsigned integer read and cached once; valid values are exactly 1,2,4,8,16,32,64. Unset/empty defaults to 16. A nonnumeric, partially parsed, overflowed, or unsupported value prints a warning and uses 16; the effective tile is min(requested,n_head). Set how many attention heads each selected-attention GEMM workspace tile processes. rocm/ds4_rocm_glm.cuh:2509 +runtime/rocm DS4_ROCM_GLM_SELECTED_ATTN_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm glm selected attn profile. rocm/ds4_rocm_glm.cuh:2534 +runtime/rocm DS4_ROCM_GLM_STREAMING_ASYNC_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm glm streaming async profile. ds4.c:44171 +runtime/rocm DS4_ROCM_GLM_STREAMING_DECODE_FULL_LAYER_MAP Pure presence flag: any defined value, including empty or "0", forces full mapping. Unset uses automatic mapping: resident layers map fully, eligible expert-cache layers use decode-only/expert mapping, otherwise full mapping. DS4_METAL_GLM_STREAMING_DECODE_FULL_LAYER_MAP and generic DS4_GLM_STREAMING_DECODE_FULL_LAYER_MAP are also accepted presence aliases. Force every GLM SSD-streaming decode layer through full-layer mapping, bypassing the selected-expert/decode mapping optimization. ds4.c:42425 +runtime/rocm DS4_ROCM_GLM_STREAMING_DECODE_SYNC_EACH_LAYER ROCm-only primary value; nonempty takes priority over DS4_METAL_GLM_STREAMING_DECODE_SYNC_EACH_LAYER and the generic DS4_GLM_STREAMING_DECODE_SYNC_EACH_LAYER fallback; empty acts as unset; truthy unless exact 0 or case-insensitive false/off/no; with all aliases unset: false For non-static GLM SSD decode on ROCm, opts into a full command/device synchronization after token mapping and every layer; default keeps ordered work queued across layer mappings, and static-map decode bypasses it. ds4.c:49589 +runtime/rocm DS4_ROCM_GLM_STREAMING_GROW_CACHE_AFTER_PREFILL Enabled by default when absent. If defined, only a truthy nonempty value enables; empty, "0", false, off, or no disable. Growth also requires SSD streaming plus nonzero base cache and prefill-headroom budgets, and occurs only if the recomputed expert count exceeds the current count. After successful ROCm GLM prefill, add the released prefill headroom to the dynamic streaming expert-cache byte budget. ds4.c:50924 +runtime/rocm DS4_ROCM_GLM_STREAMING_PREFILL_FULL_LAYER presence force-on; any presence including empty or 0 enables; unset falls back to the Metal alias and then the automatic token threshold (1024 by default on ROCm); DS4_ROCM_DISABLE_GLM_STREAMING_PREFILL_FULL_LAYER dominates Forces GLM SSD prefill into full-layer mapping/cache mode even below the automatic large-batch threshold. ds4.c:42523 +runtime/rocm DS4_ROCM_GLM_STREAMING_PREFILL_FULL_LAYER_MIN_TOKENS Positive uint32 threshold parsed with strtoul; ROCm default is 1024. Missing/empty, no leading number, errno/overflow, zero, or >UINT32_MAX returns 1024. The parser does not require end-of-string, so trailing junk after a valid leading number is accepted. A nonempty ROCm value takes precedence; otherwise DS4_METAL_GLM_STREAMING_PREFILL_FULL_LAYER_MIN_TOKENS is a fallback alias. Set the automatic token-count crossover for ROCm GLM SSD prefill to load/use full resident expert layers when the layer supports that mode. ds4.c:42502 +runtime/rocm DS4_ROCM_GLM_STREAMING_PREFILL_SYNC_EACH_LAYER ROCm-only primary value; nonempty takes priority over DS4_METAL_GLM_STREAMING_PREFILL_SYNC_EACH_LAYER and the generic DS4_GLM_STREAMING_PREFILL_SYNC_EACH_LAYER fallback; empty acts as unset; truthy unless exact 0 or case-insensitive false/off/no; with all aliases unset: false for compact prefill; full-layer prefill always returns true For compact GLM SSD prefill on ROCm, opts into a full command/device synchronization at every layer boundary; default preserves queued work across mappings, while full-layer cache mode always synchronizes. ds4.c:42205 +runtime/rocm DS4_ROCM_GLM_STREAMING_TOKEN_PREFILL_MAX primary nonempty value, then the Metal alias, then generic DS4_GLM_STREAMING_TOKEN_PREFILL_MAX; parsed by strtoul without requiring full-string consumption; 0 is valid and disables; no digits, ERANGE, or > UINT32_MAX uses the ROCm default 0 Sets the largest non-quality GLM SSD-prefill chunk eligible for token-major/decode-style execution; ROCm defaults to canonical indexed batch prefill (0 disables token-major mode). ds4.c:49492 +runtime/rocm DS4_ROCM_GLM_VALUE_PROJECT_WAVE_DECODE Enabled by default when unset. Exact "0" or empty disables; every other nonempty value enables (including false/off/no). It applies only when n_tokens==1; =0 or multi-token input uses the generic per-head batch kernel. Select the validated wave-per-output-row ROCm Q8 GLM value-projection kernel for one-token decode; =0 is the generic-kernel rollback. rocm/ds4_rocm_glm.cuh:2019 +runtime/rocm DS4_ROCM_GRAPH_DUMP_LAYER unsigned layer or all; unset=all layers Filter ROCm graph dumps by layer. ds4.c:16689 +runtime/rocm DS4_ROCM_GRAPH_DUMP_NAME nonempty substring filter; unset=all tensor names Filter ROCm graph dumps by tensor/stage name. ds4.c:16685 +runtime/rocm DS4_ROCM_GRAPH_DUMP_NONINVASIVE truthy value under the shared parser; unset lets dumping select conservative kernels Keep production ROCm kernel selection while graph dumping. rocm/ds4_rocm_runtime.cuh:4822 +runtime/rocm DS4_ROCM_GRAPH_DUMP_POS unsigned token position; unset=all positions Filter ROCm graph dumps by position. ds4.c:16696 +runtime/rocm DS4_ROCM_GRAPH_DUMP_PREFIX nonempty output path prefix; unset=off Enable ROCm intermediate graph/tensor dumps. ds4_cuda.cu:397 +runtime/rocm DS4_ROCM_GRAPH_DUMP_TRACE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Emit trace diagnostics for rocm graph dump trace. ds4.c:16726 +runtime/rocm DS4_ROCM_GRAPH_OUTPUT_ROW Nonempty string with a leading strtoul-parsable unsigned value < n_tokens selects that zero-based row. Default, empty, unparsable, or out-of-range selects n_tokens-1. Trailing characters are accepted because full consumption/errno are not checked. A nonempty ROCm value takes precedence; otherwise DS4_METAL_GRAPH_OUTPUT_ROW is a fallback alias. Choose which prefill hidden-state row is sent through the output head to produce logits, primarily for graph/correctness diagnostics. ds4.c:35655 +runtime/rocm DS4_ROCM_GRAPH_PREFILL_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm graph prefill profile. ds4.c:31848 +runtime/rocm DS4_ROCM_GRAPH_PREFILL_SPLIT_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm graph prefill split profile. ds4.c:35578 +runtime/rocm DS4_ROCM_GRAPH_TOKEN_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm graph token profile. ds4.c:31532 +runtime/rocm DS4_ROCM_INDEXER_STAGE_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm indexer stage profile. ds4.c:29092 +runtime/rocm DS4_ROCM_LAYER_STAGE_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm layer stage profile. ds4.c:28931 +runtime/rocm DS4_ROCM_LAYER_STAGE_PROFILE_LAYER layer filter subordinate to DS4_ROCM_LAYER_STAGE_PROFILE; unset or whitespace-only: all layers allowed by the parent flag; otherwise the whitespace-trimmed value must be a complete base-10 strtoul result <= UINT32_MAX equal to the current layer; invalid values match none Restricts the ROCm layer/prefill stage profiler to one layer; it does not enable profiling by itself. ds4.c:28932 +runtime/rocm DS4_ROCM_MOE_DECODE_DOWN_RPB sampled once; nonempty value is parsed by strtoul (a numeric prefix is sufficient), cast to uint32_t, and accepted only if 1/2/4/8/16/32; unset/empty/invalid inherits DS4_ROCM_MOE_DECODE_RPB, with defaults quality=8, non-quality SSD=2, resident=1 Sets output rows (warps) per block for ROCm Q2_K routed-MoE decode down-projection kernels; threads per block are value * 32. rocm/ds4_rocm_runtime.cuh:4842 +runtime/rocm DS4_ROCM_MOE_DECODE_GATE_RPB sampled once; nonempty value is parsed by strtoul (a numeric prefix is sufficient), cast to uint32_t, and accepted only if 1/2/4/8/16/32; unset/empty/invalid defaults to 1 in non-quality SSD mode when DS4_ROCM_MOE_DECODE_RPB is unset/empty, otherwise inherits the resolved base RPB Sets output rows (warps) per block for ROCm Q2_K routed-MoE decode gate/up kernels; threads per block are value * 32. rocm/ds4_rocm_runtime.cuh:4836 +runtime/rocm DS4_ROCM_MOE_DECODE_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm moe decode profile. rocm/ds4_rocm_moe_launch.cuh:82 +runtime/rocm DS4_ROCM_MOE_DECODE_RPB sampled once; nonempty value is parsed by strtoul (a numeric prefix is sufficient), cast to uint32_t, and accepted only if 1/2/4/8/16/32; unset/empty/invalid default: quality=8, non-quality SSD=2, resident=1 Sets the base ROCm Q2_K decode-MoE rows-per-block value inherited by gate/up and down controls, except the automatic SSD gate specialization defaults to 1 when this variable is unset/empty. rocm/ds4_rocm_runtime.cuh:4832 +runtime/rocm DS4_ROCM_MOE_WRITE_CLAMPED_ACT Pure presence sentinel: any defined value, including empty or "0", is active; DS4_METAL_MOE_WRITE_CLAMPED_ACT is an accepted fallback alias. On ROCm the variable is only consumed as a path-admission veto: it disables selected-expert cache/address-table, selected-slot and CPU-router/fused optimized paths. No ROCm call site parses a clamp amount or directly enables a write-clamped kernel. Force shared graph selection away from optimizations incompatible with the clamped-intermediate MoE diagnostic; on ROCm this is a compatibility/rollback gate, not itself a clamped-write implementation. ds4.c:18353 +runtime/rocm DS4_ROCM_Q4_GROUPED_ATTN_A_STATS presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Print counters for rocm q4 grouped attn a stats. rocm/ds4_rocm_q4.cuh:344 +runtime/rocm DS4_ROCM_Q4_PREFILL_TILE8_STATS presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Print counters for rocm q4 prefill tile8 stats. rocm/ds4_rocm_q4.cuh:378 +runtime/rocm DS4_ROCM_Q8_DECODE_SHAREDX_64K sampled once; unset: enabled; present empty or exact 0: disabled; every other present value: enabled; effective only for one-token non-prequant Q8_0 matmul with 8192 < in_dim <= 16384 Allows the ROCm shared-input Q8 decode kernel to use up to 64 KiB dynamic LDS for wide inputs; an unsupported/failed LDS launch automatically falls back to the regular kernel. rocm/ds4_rocm_runtime.cuh:4805 +runtime/rocm DS4_ROCM_Q_STAGE_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm q stage profile. ds4.c:29096 +runtime/rocm DS4_ROCM_REQUIRE_Q4_GROUPED_ATTN_A presence fail-closed assertion; also requests candidate unless disabled Require rocm require q4 grouped attn a and fail instead of silently falling back. rocm/ds4_rocm_q4.cuh:575 +runtime/rocm DS4_ROCM_REQUIRE_Q4_PREFILL_TILE8 presence fail-closed assertion for eligible TILE8 calls Require rocm require q4 prefill tile8 and fail instead of silently falling back. rocm/ds4_rocm_q4.cuh:316 +runtime/rocm DS4_ROCM_STREAMING_DECODE_PREFILL_MAX primary nonempty value over the Metal alias; parsed by strtol when it has a numeric prefix (trailing text is accepted); <= 0 disables, values > UINT32_MAX clamp, no numeric prefix uses automatic default: 64 for Flash with uniform Q4_K/MXFP4 experts, 18 for other Pro/Flash, otherwise 0; the disable flag dominates Sets the largest short, non-quality SSD-streaming prefill batch routed through the decode-style path instead of canonical layer-major prefill. ds4.c:31771 +runtime/rocm DS4_ROCM_STREAMING_EXPERT_AUTO_PRELOAD_CAP primary nonempty value over the Metal alias; strict full-string strtoul; valid values > UINT32_MAX clamp, invalid uses 4096, and 0 means no cap (not disabled); when CLI preload is auto/0, unset defaults to cap 4096 except ROCm GLM52, where absent/empty disables automatic preload entirely Caps the number of hot experts synchronously seeded into the SSD-streaming expert cache in automatic preload mode; an explicit CLI preload count bypasses this cap, and setting this variable opts ROCm GLM52 back into auto preload. ds4.c:21281 +runtime/rocm DS4_ROCM_STREAMING_EXPERT_CACHE_VERBOSE presence flag; unset=off Print verbose ROCm streaming expert-cache seed/load diagnostics. rocm/ds4_rocm_runtime.cuh:2904 +runtime/rocm DS4_ROCM_STREAMING_EXPERT_HOTLIST Nonempty filesystem path; a nonempty ROCm value takes precedence, otherwise DS4_METAL_STREAMING_EXPERT_HOTLIST is a fallback. The file contains whitespace-separated layer expert hits rows; blank/comment lines are ignored, zero-hit rows skipped, malformed/open/read errors fail seeding. Unset/empty uses the built-in Pro/Flash/GLM52 hotlist. Effective only when non-cold SSD hotlist seeding is enabled and cache/preload budget is nonzero. Select a custom ranked expert hotlist used to preseed the streaming resident expert cache before decode. ds4.c:21321 +runtime/rocm DS4_ROCM_STREAMING_EXPERT_HOTLIST_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm streaming expert hotlist profile. ds4.c:32054 +runtime/rocm DS4_ROCM_STREAMING_MAP_TRACE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Emit trace diagnostics for rocm streaming map trace. ds4.c:42453 +runtime/rocm DS4_ROCM_STREAMING_PREFILL_BATCH_SELECTED_ADDR_MAX primary nonempty value over the Metal alias; strtol accepts a numeric prefix; <= 0 returns 0, > UINT32_MAX clamps, invalid uses ROCm default UINT32_MAX for Pro/Flash/GLM52 and 0 otherwise Sets the inclusive upper token-count bound for automatically using selected-expert address-table kernels during eligible non-quality SSD batch prefill; 0 disables automatic selection. ds4.c:18298 +runtime/rocm DS4_ROCM_STREAMING_PREFILL_BATCH_SELECTED_ADDR_MIN primary nonempty value over the Metal alias; strtol accepts a numeric prefix; <= 0 returns 0, > UINT32_MAX clamps, invalid uses ROCm default 2 for Pro/Flash/GLM52 and 0 otherwise Sets the inclusive lower token-count bound for automatically using selected-expert address-table kernels during eligible non-quality SSD batch prefill (the path independently requires more than one token). ds4.c:18321 +runtime/rocm DS4_ROCM_STREAMING_PREFILL_CACHE_SEED_K primary nonempty value over the Metal alias; strict full-string strtoul; unset/empty/invalid: 1; 0 disables; positive values clamp to 64; ignored unless SSD streaming and DS4_ROCM_ENABLE_STREAMING_PREFILL_CACHE_SEED (or Metal alias) is present Chooses how many trailing token router selections per layer are captured from prefill and used to seed the streaming expert cache afterward. ds4.c:21094 +runtime/rocm DS4_ROCM_STREAMING_PREFILL_CACHE_SEED_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm streaming prefill cache seed profile. ds4.c:31963 +runtime/rocm DS4_ROCM_STREAMING_PREFILL_LAYER_MADVISE_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm streaming prefill layer madvise profile. ds4.c:19436 +runtime/rocm DS4_ROCM_STREAMING_PREFILL_LAYER_PAGEIN_NO_OVERLAP Pure presence flag: any defined value, including empty or "0", disables overlap. Default overlap is enabled only if this, PREPARE_NO_OVERLAP, DISABLE_*_PREPARE_OVERLAP, and DISABLE_*_PAGEIN_OVERLAP are all absent. The corresponding DS4_METAL name is an accepted fallback alias. In current code PAGEIN_NO_OVERLAP and PREPARE_NO_OVERLAP are exact synonyms. Serialize SSD-streaming prefill layer page-in/preparation instead of overlapping preparation of upcoming layers. ds4.c:19143 +runtime/rocm DS4_ROCM_STREAMING_PREFILL_LAYER_PAGEIN_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm streaming prefill layer pagein profile. ds4.c:19432 +runtime/rocm DS4_ROCM_STREAMING_PREFILL_LAYER_PAGEIN_THREADS legacy fallback read only when DS4_ROCM_STREAMING_PREFILL_LAYER_PREPARE_THREADS and its Metal alias are absent/empty; strict full-string strtoul; unset/empty across both names: 8; invalid or 0: 1; values > 16 clamp to 16 Sets worker count for full-layer SSD-prefill preparation (page touch, pread, readahead, or madvise) when the canonical PREPARE_THREADS control is not set. ds4.c:19101 +runtime/rocm DS4_ROCM_STREAMING_PREFILL_LAYER_PREAD_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm streaming prefill layer pread profile. ds4.c:19434 +runtime/rocm DS4_ROCM_STREAMING_PREFILL_LAYER_PREPARE_AHEAD primary nonempty value over the Metal alias; strict full-string strtoul; unset/empty: 1; invalid or 0: 1; values > 4 clamp to 4 Sets how many future layer-preparation jobs may be queued concurrently while SSD-prefill preparation overlap is enabled. ds4.c:19155 +runtime/rocm DS4_ROCM_STREAMING_PREFILL_LAYER_PREPARE_NO_OVERLAP Pure presence flag: any defined value, including empty or "0", disables overlap. Default overlap is enabled only if this, PAGEIN_NO_OVERLAP, DISABLE_*_PREPARE_OVERLAP, and DISABLE_*_PAGEIN_OVERLAP are all absent. The corresponding DS4_METAL name is an accepted fallback alias. In current code PREPARE_NO_OVERLAP and PAGEIN_NO_OVERLAP are exact synonyms. Serialize SSD-streaming prefill layer preparation/page-in instead of overlapping preparation of upcoming layers. ds4.c:19141 +runtime/rocm DS4_ROCM_STREAMING_PREFILL_LAYER_PREPARE_THREADS primary nonempty value over the Metal alias; strict full-string strtoul; unset/empty falls back to LAYER_PAGEIN_THREADS, then default 8; invalid or 0: 1; values > 16 clamp to 16 Sets worker count used to split full-layer SSD-prefill page-touch, pread, readahead, or madvise preparation ranges. ds4.c:19097 +runtime/rocm DS4_ROCM_STREAMING_PREFILL_LAYER_READAHEAD_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm streaming prefill layer readahead profile. ds4.c:19438 +runtime/rocm DS4_ROCM_STREAMING_PREFILL_SELECTED_MADVISE_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm streaming prefill selected madvise profile. ds4.c:19182 +runtime/rocm DS4_ROCM_STREAMING_PREFILL_SELECTED_MADVISE_THREADS legacy fallback read only for selected-expert madvise preparation when DS4_ROCM_STREAMING_PREFILL_SELECTED_PREPARE_THREADS and its Metal alias are absent/empty; strict full-string strtoul; if all selected controls are unset it inherits layer preparation threads (default 8); invalid or 0: 1; values > 16 clamp to 16 Sets worker count for selected-expert madvise preparation under its legacy name; non-madvise selected page-in always uses one worker. ds4.c:19119 +runtime/rocm DS4_ROCM_STREAMING_PREFILL_SELECTED_PAGEIN_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm streaming prefill selected pagein profile. ds4.c:19180 +runtime/rocm DS4_ROCM_STREAMING_PREFILL_SELECTED_PREPARE_GAP primary nonempty value over the Metal alias; strict full-string strtoul; unset/empty/invalid: 0; values > 8 clamp to 8 For selected-expert madvise preparation, merges selected expert runs separated by at most this many unselected expert IDs, trading broader hints for fewer ranges. ds4.c:19131 +runtime/rocm DS4_ROCM_STREAMING_PREFILL_SELECTED_PREPARE_THREADS primary nonempty value over the Metal alias; strict full-string strtoul; for selected-expert madvise, unset/empty falls back to SELECTED_MADVISE_THREADS then layer preparation threads (default 8); invalid or 0: 1; values > 16 clamp to 16; non-madvise selected page-in ignores it and uses 1 Sets worker count for selected-expert madvise preparation using the canonical control name. ds4.c:19115 +runtime/rocm DS4_ROCM_STREAMING_PREFILL_SELECTED_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm streaming prefill selected profile. ds4.c:18732 +runtime/rocm DS4_ROCM_STREAMING_PREFILL_SELECTED_READAHEAD_GAP primary nonempty value over the Metal alias; strict full-string strtoul; unset/empty/invalid: 0; values > 8 clamp to 8 For selected-expert file readahead, merges selected expert runs separated by at most this many unselected expert IDs, reducing readahead calls at the cost of hinting extra weights. ds4.c:19804 +runtime/rocm DS4_ROCM_STREAMING_PREFILL_SELECTED_READAHEAD_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm streaming prefill selected readahead profile. ds4.c:19889 +runtime/rocm DS4_ROCM_STREAM_CACHE_LAYER_STATS presence flag; unset=off Collect per-layer ROCm streaming cache statistics; also enables aggregate stats. rocm/ds4_rocm_runtime.cuh:390 +runtime/rocm DS4_ROCM_STREAM_CACHE_STATS presence flag; unset=off unless layer stats are enabled Collect aggregate ROCm streaming cache statistics. rocm/ds4_rocm_runtime.cuh:398 +runtime/rocm DS4_ROCM_STREAM_EVICT_PAST_LAYERS_FIRST nonempty and not 0 enables; unset/empty/0=off Prefer evicting cached experts from already-processed layers. rocm/ds4_rocm_runtime.cuh:406 +runtime/rocm DS4_ROCM_STREAM_FREE_RESERVE_GB integer 2..64 GiB; default 16 Reserve unified-memory headroom while growing the ROCm expert cache. rocm/ds4_rocm_runtime.cuh:1526 +runtime/rocm DS4_ROCM_STREAM_MODEL_CACHE_GB positive GiB integer; unset/invalid uses automatic streaming model cache limit Cap cached streaming model spans. rocm/ds4_rocm_runtime.cuh:5464 +runtime/rocm DS4_ROCM_STREAM_NO_DIRECT nonempty and not 0 disables direct reads; unset/empty/0 keeps direct I/O eligible Force the buffered ROCm SSD-streaming read path. rocm/ds4_rocm_runtime.cuh:1932 +runtime/rocm DS4_ROCM_STREAM_Q8_F16_CACHE_GB non-negative GiB integer; unset/invalid uses automatic Q8-F16 cache limit Cap converted Q8-to-F16 weights in SSD mode. rocm/ds4_rocm_runtime.cuh:4859 +runtime/rocm DS4_ROCM_STREAM_READ_PROFILE nonempty and not 0 enables; unset/empty/0=off Print ROCm SSD-streaming read/locality statistics at exit. rocm/ds4_rocm_runtime.cuh:1918 +runtime/rocm DS4_ROCM_STREAM_READ_WORKERS integer; default DS4_ROCM_STREAM_READ_DEFAULT_WORKERS; 0 coerces to 1; capped at compile-time max Set parallel ROCm SSD read/upload workers. rocm/ds4_rocm_runtime.cuh:2077 +runtime/server DS4_SERVER_BATCH_LOG Pure presence flag read once when the decode worker starts; default off. Any defined value, including empty or "0", logs one record per coalesced decode batch with count, elapsed milliseconds and ok/error status. Observe server-side decode coalescing size, latency and result without changing batching behavior. ds4_server.c:11090 +runtime/server DS4_SERVER_DECODE_COALESCE_US integer 0..100000 microseconds; default 2000; 0 disables wait Control server micro-batch coalescing delay. ds4_server.c:11069 +runtime/ssd DS4_SSD_AUTO_CACHE_PCT integer 50..95; default 80 Choose the RAM percentage used by automatic SSD expert-cache planning. ds4_ssd.c:81 +runtime/test-hook DS4_TEST_METAL_EXACTN_ORACLE presence flag compiled only with DS4_TEST_HOOKS; unset is off; any defined value enables Force allocation of the Metal exact-N verifier/oracle workspace in test builds. ds4.c:61801 +runtime/tp DS4_TP_ABLATE comma/list string matched for hcpre,router,kv,compidx; unset=no ablation; must match on both ranks Skip named TP encode chains for timing; output is semantically wrong. ds4.c:22355 +runtime/tp DS4_TP_EVENT_GATES presence flag; unset uses lower-latency slab flag gates when available Fall back to Metal shared-event arrival gates. ds4_metal.m:10769 +runtime/tp DS4_TP_GATE_PROFILE presence diagnostic flag; unset=off Collect timing/profile diagnostics for tp gate profile. ds4_metal.m:10661 +runtime/tp DS4_TP_GATE_TRACE presence diagnostic flag; unset=off Emit trace diagnostics for tp gate trace. ds4_tp.c:911 +runtime/tp DS4_TP_KEEPALIVE_ITERS atoi unsigned iteration count; default 1200000 Tune work per Metal TP keep-alive dispatch. ds4_metal.m:10625 +runtime/tp DS4_TP_KEEPALIVE_TGS integer 1..2048; invalid/out of range uses 1 Tune threadgroups per Metal TP keep-alive dispatch. ds4_metal.m:10611 +runtime/tp DS4_TP_NO_KEEPALIVE presence flag; unset starts Metal TP keep-alive Disable the Metal TP GPU keep-alive worker. ds4_metal.m:10797 +runtime/tp DS4_TP_PREFILL_SPLIT_MIN atoi token threshold; default 32; values below 2 clamp to 2 Set when TP prefill row-splits the replicated shared expert. ds4.c:29019 +runtime/tp DS4_TP_SUBGATE_PIPELINE nonempty integer; nonzero enables; default off; must match on both ranks Enable TP prefill sub-chunk gate pipelining. ds4.c:29033 +runtime/tp DS4_TP_TIMEOUT_SEC atoi seconds stored unsigned; default DS4_TP_DEFAULT_TIMEOUT_SEC Override TP control/data socket operation timeout. ds4_tp.c:1329 +runtime/web DS4_CHROME executable path; unset auto-detects Chrome/Chromium via standard paths and PATH Select the browser executable used by web tooling. ds4_web.c:1014 +script/downloader DS4_GGUF_DIR path; default repository gguf/ directory Choose the model download directory. download_model.sh:23 +script/downloader FLATTEN_DOWNLOADS exact integer 1 enables; unset or 0 preserves normal shard paths Move downloaded Hugging Face files from nested cache paths into the requested output directory. download_model.sh:244 +script/downloader FORCE_HF_DOWNLOAD exact integer 1 enables; unset or 0 uses the available downloader automatically Force download_model.sh to use hf download instead of curl when available. download_model.sh:216 +script/downloader HF_TOKEN secret string; unset tries cached Hugging Face token or unauthenticated download Authenticate Hugging Face downloads. download_model.sh:28 +script/downloader HOME Filesystem directory string. In ds4-agent, unset or empty falls back to "." for the default cache and history roots; the web helper applies the same fallback for its browser profile. A nonempty value roots .ds4/kvcache, .ds4_agent_history and .ds4/browser. Choose the user's persistent ds4-agent cache, line-history and Chrome-profile base directory. ds4_agent.c:4023 +script/quality-collector dynamic environment-variable name; no fixed identifier; overrides endpoint-derived key name Allow a caller-selected credential environment variable. gguf-tools/quality-testing/collect_official.py:243 +script/quality-collector DEEPSEEK_API_KEY secret string; default credential for non-OpenRouter endpoint; required unless --api-key-env selects another name Authenticate official DeepSeek continuation collection. gguf-tools/quality-testing/collect_official.py:242 +script/quality-collector OPENROUTER_API_KEY secret string; default credential when endpoint contains openrouter.ai; required unless --api-key-env selects another name Authenticate OpenRouter continuation collection. gguf-tools/quality-testing/collect_official.py:242 +script/server-wrapper DS4_BATCHED_SESSIONS unset/empty defaults to 16; otherwise passed verbatim to --batched-session; the wrapper does not validate it Sets the maximum batched-session count for the managed CUDA tensor-parallel server. run-nvidia-tp-server.sh:14 +script/server-wrapper DS4_CTX unset/empty defaults to 100000; otherwise passed verbatim to --ctx; the wrapper does not validate it Sets the managed server context size. run-nvidia-tp-server.sh:9 +script/server-wrapper DS4_KV_DIR directory path; unset/empty defaults to /data/ds4-kv Sets --kv-disk-dir for the managed server disk-backed KV cache. run-nvidia-tp-server.sh:12 +script/server-wrapper DS4_KV_SPACE_MB unset/empty defaults to 8192; otherwise passed verbatim to --kv-disk-space-mb; the wrapper does not validate it Sets the managed server disk-KV capacity in MiB. run-nvidia-tp-server.sh:13 +script/server-wrapper DS4_LOCK_FILE lock-file path; unset/empty defaults to /tmp/ds4.lock Selects the PID/instance lock inspected by start, stop, restart, and status; an explicit environment value is inherited by ds4-server. run-nvidia-tp-server.sh:15 +script/server-wrapper DS4_MODEL model path; unset/empty defaults to /home/antirez/models/deepseek-v4-gguf/DeepSeek-V4-Flash-MXFP4Experts-F16HC-F16Compressor-F16Indexer-Q8Attn-Q8Shared-Q8Out-chat-v2-mxfp4-0731.gguf; unreadable paths fail Sets the target GGUF passed to the managed CUDA tensor-parallel server. run-nvidia-tp-server.sh:8 +script/server-wrapper DS4_SERVER_HOST host string; unset/empty defaults to 0.0.0.0 Sets the HTTP listen address passed to the managed server. run-nvidia-tp-server.sh:10 +script/server-wrapper DS4_SERVER_LOG log-file path; unset/empty defaults to /tmp/ds4-server.log Receives detached-server stdout/stderr and supplies the readiness probe and failure tail. run-nvidia-tp-server.sh:16 +script/server-wrapper DS4_SERVER_PORT unset/empty defaults to 8000; otherwise passed verbatim to --port; the wrapper does not validate it Sets the HTTP listen port passed to the managed server. run-nvidia-tp-server.sh:11 +script/server-wrapper DS4_START_TIMEOUT positive decimal integer with no leading zero; unset/empty defaults to 180; invalid values fail when starting Sets how many seconds detached startup waits for the lock owner and listening log marker. run-nvidia-tp-server.sh:17 +script/server-wrapper DS4_STOP_TIMEOUT positive decimal integer with no leading zero; unset/empty defaults to 120; invalid values fail when stopping Sets how many seconds graceful stop waits after SIGTERM before failing. run-nvidia-tp-server.sh:18 +test-only DS4_CUDA_TOPK_REGRESSION_SEC positive floating-point seconds; default 2.0; invalid or nonpositive input restores the default Set the CUDA large-top-k elapsed-time regression limit. tests/cuda_long_context_smoke.c:72 +test-only DS4_METAL_MOE_TILE_MAX cleanup-only historical spelling; no production consumer exists, so setting it has no runtime effect Clears a legacy Metal MoE tile override while preparing the test environment. tests/ds4_test.c:7236 +test-only DS4_ROCM_ENABLE_Q4_PREFILL_TILE8 cleanup-only legacy spelling; no runtime or test reader; setting it has no effect Remove a stale opt-in name while preparing ROCm Q4 test cases; TILE8 is automatic. tests/test_rocm_q4_dense_pair.cpp:58 +test-only DS4_TEST_ALLOW_FALLBACK presence flag; unset: native mixed path required; any defined value including empty or 0 permits exactly the serialized-fallback counter outcome Lets the CUDA mixed prefill/decode oracle accept serialized fallback while retaining bit-exact logit comparisons. tests/test_cuda_mixed_batch.c:204 +test-only DS4_TEST_BACKEND exact cpu selects CPU; every other value, including unset/empty, selects Metal on Apple and CUDA elsewhere Chooses the backend used by model-backed tests in tests/ds4_test.c. tests/ds4_test.c:91 +test-only DS4_TEST_BATCH_ONLY presence flag; unset: run batched and isolated-control phases; any defined value including empty or 0 stops after the batched archive/hash phase Runs only the CUDA session-batch phase and skips replay against isolated control sessions. tests/test_cuda_session_batch.c:288 +test-only DS4_TEST_CONTEXT CUDA session/mixed fixtures default to 1024 and require 1024..65536; mixed-batch uses strict full decimal parsing, while session-batch uses atoi and therefore accepts numeric prefixes Sets the context and placement hint for the CUDA session-batch and mixed prefill/decode oracles. tests/test_cuda_session_batch.c:125 +test-only DS4_TEST_DSPARK nonempty DSpark support-GGUF path; unset/empty skips the DSpark verify-depth test Loads the DSpark support model for teacher-forced verification of committed speculative tokens. tests/ds4_test.c:8283 +test-only DS4_TEST_GPU_DEVICES GPU device-list string parsed with the normal auto-VRAM parser; unset/empty defaults to 0,2,4,6,1,3,5,7; parse failure is fatal Selects and orders the CUDA TP/EP devices used by the mixed prefill/decode oracle. tests/test_cuda_mixed_batch.c:122 +test-only DS4_TEST_LOCAL_GOLDEN_FILE nonempty readable fixture path; unset/empty defaults to tests/test-vectors/flash-0731/local-golden.vec Selects the local-golden vector file used for model-logit regression checks. tests/ds4_test.c:7221 +test-only DS4_TEST_LOGPROB_AUTO_METAL presence flag; unset forces DS4_METAL_DISABLE_METAL4=1; any presence including empty or 0 removes that rollback and permits automatic Metal selection Runs official log-probability vectors with automatic Metal-path selection instead of the fixed pre-Metal4 baseline. tests/ds4_test.c:6955 +test-only DS4_TEST_LONG_PROMPT nonempty readable prompt-file path; unset/empty defaults to tests/long_context_story_prompt.txt Selects the rendered story prompt for the long-context fact-recall test. tests/ds4_test.c:6690 +test-only DS4_TEST_LONG_WORDS atoi integer; unset/empty/nonnumeric defaults to 0; valid range is 0..DS4_TEST_CONTEXT-128 and numeric prefixes are accepted Adds repeated words to alternating CUDA session-batch prompts to exercise long-prefill rows. tests/test_cuda_session_batch.c:135 +test-only DS4_TEST_METAL_EXACTN_BATCH_HEAD nonempty value other than exact 0 enables; unset/empty/0 disables; false/off also enable Enables the Metal exact-N batch-head path and requires its attempt/use counters for every eligible oracle case. tests/test_metal_exactn_oracle.c:401 +test-only DS4_TEST_METAL_EXACTN_ORACLE presence flag compiled only with DS4_TEST_HOOKS; absent from normal production builds Force allocation of the exact-N Metal verifier/oracle workspace in tests. ds4.c:61801 +test-only DS4_TEST_MIXED_INITIAL integer 128..context-1; default 128 Set the initial prefill length for the CUDA mixed-batch oracle. tests/test_cuda_mixed_batch.c:111 +test-only DS4_TEST_MIXED_QUANTUM integer 1..context-1; default 128 Set the number of prompt tokens added per CUDA mixed-batch round. tests/test_cuda_mixed_batch.c:113 +test-only DS4_TEST_MIXED_ROUNDS integer 1..64; default 3 Set the number of CUDA mixed-batch oracle rounds. tests/test_cuda_mixed_batch.c:115 +test-only DS4_TEST_MODEL nonempty GGUF path; tests/ds4_test.c defaults to ds4flash.gguf, while standalone model-backed CUDA/Metal fixtures generally require a supplied path and fail or skip when absent Selects the target model shared by model-backed test binaries. tests/ds4_test.c:14 +test-only DS4_TEST_MPP_EQ_CASE comma-separated substring filter; unset/empty runs all cases; tokens are whitespace-trimmed and the filter is truncated to 255 bytes Restricts Metal tensor-equivalence vectors to IDs containing at least one requested substring. tests/ds4_test.c:7525 +test-only DS4_TEST_MTP nonempty MTP support-GGUF path; unset/empty loads no MTP head; only the fast test engine uses it, with draft depth 4 Enables the legacy MTP verify-depth regression; the test self-skips without this model. tests/ds4_test.c:104 +test-only DS4_TEST_Q4_STREAM_ITERS integer 1..10000; default 2 Set measured iterations for the Metal Q4 stream oracle. tests/test_metal_q4_streams.c:734 +test-only DS4_TEST_Q4_STREAM_SOAK integer 1..100000; default 8 Set bounded overlap-soak iterations for the Metal Q4 stream oracle. tests/test_metal_q4_streams.c:736 +test-only DS4_TEST_Q4_STREAM_TIMING presence flag; unset: correctness/leak checks only; any defined value including empty or 0 also runs timing pairs Adds FIFO-versus-overlap and native-versus-overlap timing measurements to the Metal Q4 stream oracle. tests/test_metal_q4_streams.c:737 +test-only DS4_TEST_Q4_STREAM_TIMING_BLOCKS integer 5..MAX_TIMING_BLOCKS; default 5 Set the timing block count for the Metal Q4 stream oracle. tests/test_metal_q4_streams.c:739 +test-only DS4_TEST_Q4_STREAM_TIMING_ITERS integer 1..10000; default 20 Set timing iterations for the Metal Q4 stream oracle. tests/test_metal_q4_streams.c:741 +test-only DS4_TEST_Q4_STREAM_WARMUP integer 1..64; default 1 Set warmup iterations for the Metal Q4 stream oracle. tests/test_metal_q4_streams.c:732 +test-only DS4_TEST_REQUIRE_MODEL nonempty value other than exact 0 requires a readable model; unset/empty/0 permits a skip; false/off count as required Turns a missing Metal exact-N oracle model from a developer skip into a release-gate failure. tests/test_metal_exactn_oracle.c:390 +test-only DS4_TEST_REQUIRE_ROCM_DEVICE nonempty value other than exact 0 requires a visible ROCm device; unset/empty/0 returns the fixture skip code; false/off count as required Turns absence of a ROCm device from a skip into failure for the ROCm Q4 oracle. tests/test_rocm_q4_dense_pair.cpp:1483 +test-only DS4_TEST_SERVER_PREFILL presence flag; unset: normal prefill; any defined value including empty or 0 installs a no-op display-progress callback Exercises the progress-split prefill path used by ds4-server in CUDA session-batch and control sessions. tests/test_cuda_session_batch.c:110 +test-only DS4_TEST_SESSION_BATCH_ARM arbitrary nonempty label; unset/empty defaults to unspecified; it is logged only and does not alter execution Labels the Metal session-batch experiment arm in setup diagnostics. tests/test_metal_session_batch.c:153 +test-only DS4_TEST_SESSION_BATCH_TIMING nonempty boolean; unset/empty/exact 0 disables; every other value enables Print timing data from the Metal session-batch oracle. tests/test_metal_session_batch.c:150 +test-only DS4_TEST_SESSION_COUNT fixture-specific integer: CUDA session-batch defaults 8 and accepts atoi 2..16; CUDA mixed-batch defaults 8 with strict 3..16; Metal session-batch defaults 2 with strict 2..16 Sets the number of simultaneous sessions exercised by the model-backed batch oracles. tests/test_cuda_session_batch.c:113 +test-only DS4_TEST_SSD_CACHE_EXPERTS strict unsigned integer 30..UINT32_MAX; unset/empty defaults to 30; read only when Metal session-batch SSD streaming is enabled Sizes the Metal session-batch routed-expert cache used to exercise SSD union policies for N=2..5. tests/test_metal_session_batch.c:88 +test-only DS4_TEST_SSD_STREAMING nonempty value other than exact 0 enables; unset/empty/0 disables; false/off also enable Runs model-backed test engines through SSD streaming; the Metal session-batch fixture also uses cold mode and shared prefill workspace. tests/ds4_test.c:109 +test-only DS4_TEST_SSD_STREAMING_CACHE_EXPERTS strtoul decimal prefix; unset/empty/nonnumeric becomes 0, values above UINT32_MAX (including a parsed negative) saturate, and trailing text is accepted Sets the routed-expert cache count on SSD-streaming engines created by tests/ds4_test.c. tests/ds4_test.c:112 +test-only DS4_TEST_SSD_STREAMING_CACHE_GB strtoull decimal GiB prefix; unset/empty/nonnumeric/zero becomes 0, byte overflow (including a parsed negative) saturates to UINT64_MAX, and trailing text is accepted Sets the routed-expert cache byte budget on SSD-streaming engines created by tests/ds4_test.c. tests/ds4_test.c:114 +test-only DS4_TEST_SSD_STREAMING_COLD nonempty boolean; unset/empty/exact 0 disables; every other value enables Run test engines in cold SSD-streaming mode and skip hot-expert preload. tests/ds4_test.c:110 +test-only DS4_TEST_SSD_STREAMING_PRELOAD_EXPERTS unsigned integer; default 0; numeric prefixes are accepted and overflow clamps to UINT32_MAX Set the number of SSD-streaming experts preloaded by test engines. tests/ds4_test.c:116 +test-only DS4_TEST_SSD_UNION_POLICY_SWITCH nonempty boolean; unset/empty/exact 0 disables; every other value enables Exercise an SSD session-union policy transition in the Metal session-batch oracle. tests/test_metal_session_batch.c:152 +test-only DS4_TEST_TP_DISCONNECT presence flag effective only in leader mode; unset: normal test; any defined value including empty or 0 enters the disconnect oracle Waits for the TP worker to disconnect, then requires the next batch to fail and every session checkpoint to be invalidated. tests/test_metal_session_batch.c:268 +test-only DS4_TEST_TP_LEADER_HOST nonempty host string required in worker mode; no default; ignored outside worker mode Sets the TP leader address contacted by the Metal session-batch worker. tests/test_metal_session_batch.c:212 +test-only DS4_TEST_TP_LISTEN_HOST nonempty host string; unset/empty defaults to 0.0.0.0; used only in leader mode Sets the TP listen address for the Metal session-batch leader. tests/test_metal_session_batch.c:204 +test-only DS4_TEST_TP_MODE unset/empty: no TP; exact leader or worker selects that role; every other nonempty value fails; incompatible with SSD-streaming mode Selects standalone, TP-leader, or TP-worker execution for the Metal session-batch oracle. tests/test_metal_session_batch.c:168 +test-only DS4_TEST_TP_PORT strict full decimal integer 1..65535; unset/empty defaults to 19452 Sets the listen/connect port shared by Metal session-batch TP leader and worker. tests/test_metal_session_batch.c:63 +test-only DS4_TEST_TP_TRANSPORT unset/empty/auto selects automatic transport; exact tcp or rdma selects that transport; other values fail Chooses the TP transport for Metal session-batch leader/worker tests. tests/test_metal_session_batch.c:52 +test-only DS4_TEST_VECTOR_FILE nonempty readable vector path; unset/empty defaults to tests/test-vectors/flash-0731/official.vec Selects the official fixture used by log-probability and Metal tensor-equivalence tests. tests/ds4_test.c:6942 +test-only PROTO_Q8_DEBUG presence diagnostic; unset: summary only; any defined value including empty or 0 prints detailed error structure after a Q8 parity failure Dumps bad-element tile and row/column histograms for the CUDA Q8 prototype when parity fails. cuda/mmq/test/proto_gemm_dense_q8_d2r.cu:647 +test-script DEEPSEEK_API_KEY secret string; required Authenticate official test-vector fetch. tests/test-vectors/fetch_official_vectors.py:236 +test-script DS4_BIN executable path; unset/empty defaults to ./ds4; the Q4 matrix requires it executable, the DSpark fixture skips if missing, and the GLM smoke lets command failure fail the test Selects the ds4 binary launched by model-backed shell test fixtures. tests/cuda_q4_gb10_fast_matrix.sh:47 +test-script DS4_CUDA_DISABLE_DSPARK_EXACTN arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4 Records and passes through the CUDA exact-N rollback used by the acceptance run. tests/dspark_acceptance_fixture.sh:253 +test-script DS4_CUDA_DISABLE_DSPARK_EXACTN_BATCH_HEAD arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4 Records and passes through the CUDA exact-N batch-head rollback. tests/dspark_acceptance_fixture.sh:255 +test-script DS4_CUDA_DISABLE_DSPARK_EXACTN_GRAPHS arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4 Records and passes through the CUDA exact-N graph rollback. tests/dspark_acceptance_fixture.sh:257 +test-script DS4_CUDA_DISABLE_DSPARK_NONCAUSAL_ONLINE arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4 Records and passes through the CUDA noncausal-online-attention rollback. tests/dspark_acceptance_fixture.sh:266 +test-script DS4_CUDA_DSPARK_DEVICE_PROPOSER arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4 Records and passes through the CUDA device-proposer opt-in. tests/dspark_acceptance_fixture.sh:258 +test-script DS4_CUDA_DSPARK_EXACT2 arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4 Records and passes through the CUDA exact-2 verifier override. tests/dspark_acceptance_fixture.sh:250 +test-script DS4_CUDA_DSPARK_EXACTN arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4 Records and passes through the CUDA exact-N verifier opt-in. tests/dspark_acceptance_fixture.sh:252 +test-script DS4_CUDA_DSPARK_EXACTN_BATCH_HEAD arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4 Records and passes through the CUDA exact-N batch-head opt-in. tests/dspark_acceptance_fixture.sh:254 +test-script DS4_CUDA_DSPARK_EXACTN_GRAPHS arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4 Records and passes through the CUDA exact-N graph opt-in. tests/dspark_acceptance_fixture.sh:256 +test-script DS4_CUDA_DSPARK_NO_DEVICE_PROPOSER arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4 Records and passes through the CUDA device-proposer rollback. tests/dspark_acceptance_fixture.sh:259 +test-script DS4_CUDA_DSPARK_PROPOSER_BLOCK_MAX arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4 Records and passes through the CUDA proposer block-size cap. tests/dspark_acceptance_fixture.sh:269 +test-script DS4_CUDA_ENABLE_DSPARK_NONCAUSAL_ONLINE arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4 Records and passes through the CUDA noncausal-online-attention opt-in. tests/dspark_acceptance_fixture.sh:265 +test-script DS4_CUDA_Q4_MATRIX_CTX nonempty decimal digits other than exact 0; unset/empty defaults to 4096; no upper bound; leading-zero zero strings such as 00 pass the script's guard Sets --ctx for every Q4 GB10 smoke and score_official matrix arm. tests/cuda_q4_gb10_fast_matrix.sh:49 +test-script DS4_CUDA_Q4_MATRIX_DECODE_GRAPHS exact default, 0, or 1; unset/empty defaults to default; other values fail Leaves decode graphs automatic, forces them off, or forces them on with capture logging for every matrix arm. tests/cuda_q4_gb10_fast_matrix.sh:56 +test-script DS4_CUDA_Q4_MATRIX_PROMPT prompt string; unset/empty defaults to Write a complete Python quicksort function with comments. Sets the deterministic smoke prompt whose log-probability output is compared across Q4 fast-path arms. tests/cuda_q4_gb10_fast_matrix.sh:52 +test-script DS4_CUDA_Q4_MATRIX_SCORER executable path; unset/empty defaults to gguf-tools/quality-testing/score_official; a missing/nonexecutable path fails Selects the scorer used to produce quality TSVs for each non-oracle Q4 matrix arm. tests/cuda_q4_gb10_fast_matrix.sh:48 +test-script DS4_CUDA_Q4_MATRIX_SKIP_PARITY exact 0 or 1; unset/empty defaults to 0; other values fail When 1, skips the synthetic MMQ parity prerequisite and marks the resulting QA run incomplete. tests/cuda_q4_gb10_fast_matrix.sh:57 +test-script DS4_CUDA_Q4_MATRIX_SSD_CACHE unset/empty by default; required and passed verbatim as --ssd-streaming-cache-experts when streaming=1; must remain empty when streaming=0; no further validation Sets the SSD expert cache as a count or NGB value for every streamed matrix arm. tests/cuda_q4_gb10_fast_matrix.sh:54 +test-script DS4_CUDA_Q4_MATRIX_SSD_PRELOAD unset/empty omits preload; a nonempty value is passed verbatim as --ssd-streaming-preload-experts and is allowed only when streaming=1 Sets optional expert preload for streamed Q4 smoke and scoring arms. tests/cuda_q4_gb10_fast_matrix.sh:55 +test-script DS4_CUDA_Q4_MATRIX_SSD_STREAMING exact 0 or 1; unset/empty defaults to 0; other values fail Runs all Q4 matrix arms resident or with SSD streaming and enforces matching cache/preload arguments. tests/cuda_q4_gb10_fast_matrix.sh:53 +test-script DS4_CUDA_Q4_MATRIX_TOKENS nonempty decimal digits other than exact 0; unset/empty defaults to 32; no upper bound; leading-zero zero strings such as 00 pass the script's guard Sets the continuation length for each Q4 GB10 smoke arm. tests/cuda_q4_gb10_fast_matrix.sh:50 +test-script DS4_CUDA_Q4_MATRIX_TOP_K nonempty decimal digits other than exact 0 and numerically <=128; unset/empty defaults to 128; leading-zero zero strings such as 00 pass the guard Sets --logprobs-top-k for the byte-comparable Q4 GB10 smoke dumps. tests/cuda_q4_gb10_fast_matrix.sh:51 +test-script DS4_DSPARK_FIXTURE_BACKEND unset/empty defaults to auto; exact auto, metal, cuda, or rocm accepted; every other value fails Chooses the explicit backend flag for baseline and DSpark acceptance runs; auto passes none. tests/dspark_acceptance_fixture.sh:14 +test-script DS4_DSPARK_FIXTURE_CONFIDENCE unset/empty omits the option and uses the runtime default; otherwise passed verbatim to --dspark-confidence without fixture-side validation Overrides the DSpark confidence threshold for acceptance runs and records it in metadata. tests/dspark_acceptance_fixture.sh:13 +test-script DS4_DSPARK_FIXTURE_C_ADD_MIN_ACCEPTED decimal digits including 0; unset/empty defaults to 8; non-digits fail Sets the minimum accepted-draft count for the c_add case when the proposal-quality guard is active. tests/dspark_acceptance_fixture.sh:12 +test-script DS4_DSPARK_FIXTURE_REQUIRE_ACTIVE exact 0 or 1; unset/empty defaults to 1; other values fail When 1, requires aggregate proposed and accepted-draft counts to both be nonzero. tests/dspark_acceptance_fixture.sh:17 +test-script DS4_DSPARK_FIXTURE_REQUIRE_CUDA_DEVICE_PROPOSER exact 0 or 1; unset/empty defaults to 0; other values fail; 1 also forces byte-identical output checking Requires CUDA device-proposer attempts to equal uses, with nonzero use and zero fallback or policy mismatch. tests/dspark_acceptance_fixture.sh:22 +test-script DS4_DSPARK_FIXTURE_REQUIRE_CUDA_EXACTN exact 0 or 1; unset/empty defaults to 0; other values fail; 1 also forces byte-identical output checking Requires at least one CUDA exact-N attempt and zero exact-N error fallbacks. tests/dspark_acceptance_fixture.sh:19 +test-script DS4_DSPARK_FIXTURE_REQUIRE_CUDA_EXACTN_BATCH_HEAD exact 0 or 1; unset/empty defaults to 0; other values fail; 1 implies REQUIRE_CUDA_EXACTN and identical output Requires nonzero CUDA exact-N batch-head attempts/uses and zero batch-head fallbacks. tests/dspark_acceptance_fixture.sh:20 +test-script DS4_DSPARK_FIXTURE_REQUIRE_CUDA_EXACTN_GRAPHS exact 0 or 1; unset/empty defaults to 0; other values fail; 1 implies REQUIRE_CUDA_EXACTN and identical output Requires CUDA exact-N graph attempts, uses, captures, and replays, with zero no-slot or graph failures. tests/dspark_acceptance_fixture.sh:21 +test-script DS4_DSPARK_FIXTURE_REQUIRE_DIRECT_COMMIT exact 0 or 1; unset/empty defaults to 0; other values fail Requires at least one direct verifier-state commit; with REQUIRE_PARTIAL it also requires a direct partial commit. tests/dspark_acceptance_fixture.sh:9 +test-script DS4_DSPARK_FIXTURE_REQUIRE_EXACT2 exact 0 or 1; unset/empty defaults to 0; other values fail; 1 also forces byte-identical output checking Requires at least one exact-2 attempt and zero exact-2 fallbacks. tests/dspark_acceptance_fixture.sh:18 +test-script DS4_DSPARK_FIXTURE_REQUIRE_IDENTICAL exact 0 or 1; unset/empty defaults to 0; other values fail; several path-specific requirements force it to 1 When 1, fails any byte difference between baseline and DSpark stdout; otherwise mismatches are reported but allowed. tests/dspark_acceptance_fixture.sh:10 +test-script DS4_DSPARK_FIXTURE_REQUIRE_METAL_DEVICE_PROPOSER exact 0 or 1; unset/empty defaults to 0; other values fail; 1 also forces byte-identical output checking Requires Metal device-proposer attempts to equal uses, with nonzero use and zero fallback or policy mismatch. tests/dspark_acceptance_fixture.sh:25 +test-script DS4_DSPARK_FIXTURE_REQUIRE_METAL_EXACTN_BATCH_HEAD exact 0 or 1; unset/empty defaults to 0; other values fail; 1 also forces byte-identical output checking Requires nonzero Metal exact-N batch-head attempts/uses and zero batch-head fallbacks. tests/dspark_acceptance_fixture.sh:23 +test-script DS4_DSPARK_FIXTURE_REQUIRE_METAL_EXACTN_PARTIAL exact 0 or 1; unset/empty defaults to 0; other values fail; 1 also forces byte-identical output checking Requires a Metal exact-N partial replay, matching verify-skip count, and zero union error fallbacks. tests/dspark_acceptance_fixture.sh:24 +test-script DS4_DSPARK_FIXTURE_REQUIRE_PARTIAL unset/empty defaults to 0; exact 0 disables; any other value enables, with no 0/1 validation Requires at least one partial-accept case and, together with REQUIRE_DIRECT_COMMIT, one direct partial commit. tests/dspark_acceptance_fixture.sh:8 +test-script DS4_DSPARK_FIXTURE_REQUIRE_PROPOSAL_QUALITY unset/empty/auto selects auto; 0/false/no/off disables; 1/true/yes/on enables; values are lowercase and other strings fail; auto enables only without partial mode/confidence override and with tokens >=32 Controls the c_add minimum-accepted-drafts quality guard. tests/dspark_acceptance_fixture.sh:11 +test-script DS4_DSPARK_FIXTURE_SSD_STREAMING exact 0 or 1; unset/empty defaults to 0; other values fail Adds --ssd-streaming to both baseline and DSpark runs when enabled. tests/dspark_acceptance_fixture.sh:15 +test-script DS4_DSPARK_FIXTURE_SSD_STREAMING_CACHE_EXPERTS unset/empty omits the cache option; otherwise decimal digits including 0 are required, and a value is legal only with SSD streaming enabled Passes an explicit --ssd-streaming-cache-experts value to both acceptance-run variants. tests/dspark_acceptance_fixture.sh:16 +test-script DS4_DSPARK_FIXTURE_TOKENS token-count argument; unset/empty defaults to 32; the fixture does not validate it before passing --tokens, and auto quality treats nonnumeric or <32 as ineligible Sets generated-token count for each baseline and DSpark acceptance case. tests/dspark_acceptance_fixture.sh:7 +test-script DS4_DSPARK_MODEL target-model path; unset/empty falls back to DS4_TEST_MODEL, then ./ds4flash.gguf; a missing file causes a successful skip Selects the target GGUF compared in baseline and DSpark acceptance runs. tests/dspark_acceptance_fixture.sh:5 +test-script DS4_DSPARK_SSD_VERIFY_BLOCK_MAX unsigned integer rows; default/fallback 0 means automatic policy; numeric prefixes accepted; used both as verifier cap and as an exact-2 proposer-policy discriminator Cap speculative rows verified from SSD and influence exact-2 proposal sizing. tests/dspark_acceptance_fixture.sh:271 +test-script DS4_DSPARK_SUPPORT support-model path; unset/empty defaults to gguf/DeepSeek-V4-Flash-DSpark-support-0731.gguf; a missing file causes a successful skip Selects the DSpark support GGUF passed through --mtp. tests/dspark_acceptance_fixture.sh:6 +test-script DS4_DSPARK_VERIFY_NONCAUSAL presence diagnostic sampled once after the first successfully submitted CUDA noncausal-attention kernel; unset: verify 0 calls; any presence including empty or 0: verify that call and the next 2 CUDA only: synchronizes and reads back Q/KV/output, computes the DSpark noncausal attention CPU reference, and logs max absolute/relative error; it reports only and does not fail the operation. tests/dspark_acceptance_fixture.sh:267 +test-script DS4_GLM_BACKEND exact metal, cuda, or cpu; unset/empty defaults to metal; other values fail Selects the backend flag used by the GLM long-context continuation smoke test. tests/glm_long_context_smoke.sh:29 +test-script DS4_GLM_EXTRA_ARGS unset/empty adds no arguments; otherwise intentionally unquoted and therefore shell field-split and pathname-expanded Adds backend/device options to the ds4 invocation used for every GLM long-context case. tests/glm_long_context_smoke.sh:91 +test-script DS4_GLM_LONG_CONTEXT_CTX context argument; unset/empty defaults to 100000; forwarded to --ctx without script-side validation Sets the context size for GLM long-context smoke invocations. tests/glm_long_context_smoke.sh:26 +test-script DS4_GLM_LONG_CONTEXT_GEN generation-count argument; unset/empty defaults to 32; forwarded to -n without script-side validation Sets the number of continuation tokens checked by each GLM long-context case. tests/glm_long_context_smoke.sh:28 +test-script DS4_GLM_LONG_CONTEXT_REPEATS whitespace-separated list of prompt-padding counts; unset/empty defaults to the single count 130; each item must work as a shell integer Chooses one or more audit-block counts used to construct long GLM prompts. tests/glm_long_context_smoke.sh:27 +test-script DS4_GLM_MODEL model path used only when no nonempty positional MODEL is supplied; unset/empty defaults to models/GLM-5.2-UD-Q4_K_XL.gguf Selects the GLM-5.2 GGUF used by the long-context continuation smoke test. tests/glm_long_context_smoke.sh:25 +test-script DS4_METAL_DISABLE_DSPARK_EXACTN_BATCH_HEAD arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4 Records and passes through the Metal exact-N batch-head rollback. tests/dspark_acceptance_fixture.sh:262 +test-script DS4_METAL_DSPARK_DEVICE_PROPOSER arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4 Records and passes through the Metal device-proposer opt-in. tests/dspark_acceptance_fixture.sh:263 +test-script DS4_METAL_DSPARK_EXACT2 arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4 Records and passes through the Metal exact-2 verifier override. tests/dspark_acceptance_fixture.sh:251 +test-script DS4_METAL_DSPARK_EXACTN_BATCH_HEAD arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4 Records and passes through the Metal exact-N batch-head opt-in. tests/dspark_acceptance_fixture.sh:261 +test-script DS4_METAL_DSPARK_EXACTN_UNION arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4 Records and passes through the Metal exact-N union-verifier opt-in. tests/dspark_acceptance_fixture.sh:260 +test-script DS4_METAL_DSPARK_EXACT_ROWS_ASYNC_TAILS arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4 Records and passes through the Metal exact-row asynchronous-tail override. tests/dspark_acceptance_fixture.sh:268 +test-script DS4_METAL_DSPARK_NO_DEVICE_PROPOSER arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4 Records and passes through the Metal device-proposer rollback. tests/dspark_acceptance_fixture.sh:264 +test-script DS4_METAL_DSPARK_PROPOSER_BLOCK_MAX arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4 Records and passes through the Metal proposer block-size cap. tests/dspark_acceptance_fixture.sh:270 +test-script DS4_TEST_MODEL fallback target-model path used only when DS4_DSPARK_MODEL is unset/empty; unset/empty then defaults to ./ds4flash.gguf Provides the shared test-model fallback for the DSpark acceptance fixture. tests/dspark_acceptance_fixture.sh:5 +test-script OPENROUTER_API_KEY secret string; required Authenticate OpenRouter GLM test-vector fetch. tests/test-vectors/fetch_openrouter_glm_vectors.py:355 +test-script TMPDIR temporary-directory base path; unset/empty defaults to /tmp Chooses the parent directory for auto-created Q4 matrix, DSpark fixture, and GLM smoke work directories. tests/cuda_q4_gb10_fast_matrix.sh:108 diff --git a/scripts/generate_environment_variables.py b/scripts/generate_environment_variables.py new file mode 100644 index 000000000..85185f569 --- /dev/null +++ b/scripts/generate_environment_variables.py @@ -0,0 +1,584 @@ +#!/usr/bin/env python3 +"""Generate and verify the complete environment-variable reference.""" + +from __future__ import annotations + +import argparse +import bisect +import csv +import os +import re +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Dict, Iterable, List, Mapping, MutableMapping, Sequence, Set, Tuple + + +ROOT = Path(__file__).resolve().parents[1] +DOCUMENT = ROOT / "ENVIRONMENT_VARIABLES.md" +METADATA = ROOT / "scripts" / "environment_variables.tsv" +START_MARKER = "" +END_MARKER = "" + +SOURCE_SUFFIXES = {".c", ".cc", ".cpp", ".cu", ".cuh", ".h", ".inc", ".m", ".mm"} +PRODUCTION_EXCLUDED_DIRS = { + ".git", + "dir-steering", + "tests", +} + +# These uppercase DS4 strings are protocol markers, compile-time identifiers, +# or display names rather than environment-variable names. Treat every other +# DS4_* string literal in production C-family sources as an environment input. +NON_ENV_DS4_STRINGS = { + "DS4_CUDA", + "DS4_IMATRIX_PROMPT", + "DS4_MAX_GPUS", + "DS4_METAL_HAS_TENSOR", + "DS4_SORT_ORDER_ASC", + "DS4_SORT_ORDER_DESC", +} +TEST_NON_ENV_DS4_STRINGS = {"DS4_N_LAYER"} + +GROUPS = ( + ("Metal", "DS4_METAL_"), + ("CUDA", "DS4_CUDA_"), + ("ROCm", "DS4_ROCM_"), + ("GLM shared", "DS4_GLM_"), + ("Distributed", "DS4_DIST_"), + ("DSpark shared", "DS4_DSPARK_"), +) + +DS4_TOKEN_RE = re.compile(r"\bDS4_[A-Z][A-Z0-9_]*\b") +C_STRING_RE = re.compile(r'(?:u8|u|U|L)?"(?:\\.|[^"\\])*"') +DIRECT_GETENV_RE = re.compile( + r'(?:std::)?getenv\s*\(\s*"([A-Z][A-Z0-9_]*)"\s*\)', re.MULTILINE +) +PYTHON_ENV_RE = re.compile( + r'os\.(?:environ\.get|getenv)\s*\(\s*["\']([A-Z][A-Z0-9_]*)["\']' +) +PYTHON_API_KEY_RE = re.compile(r'["\']([A-Z][A-Z0-9_]*_API_KEY)["\']') +SHELL_DEFAULT_RE = re.compile(r"\$\{([A-Z][A-Z0-9_]*):-([^}\n]*)\}") +SHELL_ENV_RE = re.compile(r"\$\{([A-Z][A-Z0-9_]*)(?::?[-+?=])") +SHELL_DEFAULT_OVERRIDES = { + "DS4_DSPARK_MODEL": "${DS4_TEST_MODEL:-./ds4flash.gguf}", +} +PLACEHOLDER_METADATA_PHRASES = ( + "ambiguous:", + "call-site specific", + "configure or diagnose ", + "configure or require test fixture", + "configure test script", + "configure the managed nvidia", + "test-script flag/value", + "string/path test input", + "integer test parameter", + "test/script input string or numeric value", + "semantics owned by os/linenoise/vendor helper", +) + + +@dataclass(frozen=True) +class Occurrence: + path: Path + line: int + line_text: str + context: str + + +@dataclass(frozen=True) +class VariableDoc: + scope: str + name: str + value_default: str + purpose: str + source: str + + +def relative(path: Path) -> Path: + return path.resolve().relative_to(ROOT) + + +def source_files() -> List[Path]: + files: List[Path] = [] + for directory, dirnames, filenames in os.walk(ROOT): + directory_path = Path(directory) + rel_dir = relative(directory_path) + if directory_path == ROOT: + dirnames[:] = [name for name in dirnames if name not in PRODUCTION_EXCLUDED_DIRS] + elif rel_dir.parts[:2] == ("cuda", "mmq"): + dirnames[:] = [name for name in dirnames if name != "test"] + for filename in filenames: + path = directory_path / filename + if path.suffix.lower() in SOURCE_SUFFIXES: + files.append(path) + return sorted(files, key=lambda item: str(relative(item))) + + +def test_files() -> List[Path]: + files = [ + path + for path in (ROOT / "tests").rglob("*") + if path.is_file() and path.suffix.lower() in SOURCE_SUFFIXES | {".py", ".sh"} + ] + mmq_tests = ROOT / "cuda" / "mmq" / "test" + if mmq_tests.exists(): + files.extend( + path + for path in mmq_tests.rglob("*") + if path.is_file() and path.suffix.lower() in SOURCE_SUFFIXES | {".py", ".sh"} + ) + return sorted(set(files), key=lambda item: str(relative(item))) + + +def add_line_occurrence( + result: MutableMapping[str, List[Occurrence]], + name: str, + path: Path, + lines: Sequence[str], + line_index: int, +) -> None: + first = max(0, line_index - 5) + last = min(len(lines), line_index + 6) + result.setdefault(name, []).append( + Occurrence( + relative(path), + line_index + 1, + lines[line_index].strip(), + "\n".join(lines[first:last]), + ) + ) + + +def scan_ds4_string_literals(files: Iterable[Path]) -> Dict[str, List[Occurrence]]: + result: Dict[str, List[Occurrence]] = {} + for path in files: + lines = path.read_text(encoding="utf-8", errors="replace").splitlines() + for line_index, line in enumerate(lines): + for literal in C_STRING_RE.finditer(line): + for token in DS4_TOKEN_RE.finditer(literal.group(0)): + add_line_occurrence(result, token.group(0), path, lines, line_index) + return result + + +def scan_direct_getenv(files: Iterable[Path]) -> Dict[str, List[Occurrence]]: + result: Dict[str, List[Occurrence]] = {} + for path in files: + text = path.read_text(encoding="utf-8", errors="replace") + lines = text.splitlines() + line_starts = [0] + line_starts.extend(match.end() for match in re.finditer("\n", text)) + + matches = list(DIRECT_GETENV_RE.finditer(text)) + if path.suffix == ".py": + matches.extend(PYTHON_ENV_RE.finditer(text)) + matches.extend(PYTHON_API_KEY_RE.finditer(text)) + for match in sorted(matches, key=lambda item: item.start()): + line_index = bisect.bisect_right(line_starts, match.start()) - 1 + add_line_occurrence(result, match.group(1), path, lines, line_index) + return result + + +def scan_shell_inputs(files: Iterable[Path]) -> Tuple[Dict[str, List[Occurrence]], Dict[str, str]]: + result: Dict[str, List[Occurrence]] = {} + defaults: Dict[str, str] = {} + for path in files: + if path.suffix != ".sh": + continue + lines = path.read_text(encoding="utf-8", errors="replace").splitlines() + for line_index, line in enumerate(lines): + for match in SHELL_ENV_RE.finditer(line): + add_line_occurrence(result, match.group(1), path, lines, line_index) + for match in SHELL_DEFAULT_RE.finditer(line): + defaults.setdefault(match.group(1), match.group(2).strip()) + for name, default in SHELL_DEFAULT_OVERRIDES.items(): + if name in result: + defaults[name] = default + return result, defaults + + +def load_metadata() -> List[VariableDoc]: + try: + handle = METADATA.open("r", encoding="utf-8", newline="") + except OSError as error: + raise ValueError(f"cannot read {METADATA.relative_to(ROOT)}: {error}") from error + with handle: + reader = csv.reader(handle, delimiter="\t") + rows = list(reader) + expected_header = ["SCOPE", "NAME", "VALUE_DEFAULT_SEMANTICS", "PURPOSE", "SOURCE"] + if not rows or rows[0] != expected_header: + raise ValueError(f"{METADATA.relative_to(ROOT)} has an invalid header") + + result: List[VariableDoc] = [] + seen: Set[Tuple[str, str]] = set() + for line, row in enumerate(rows[1:], 2): + if len(row) != 5 or any(not field.strip() for field in row): + raise ValueError( + f"{METADATA.relative_to(ROOT)}:{line}: expected five nonempty TSV fields" + ) + item = VariableDoc(*(field.strip() for field in row)) + searchable = f"{item.value_default}\n{item.purpose}".lower() + placeholder = next( + (phrase for phrase in PLACEHOLDER_METADATA_PHRASES if phrase in searchable), + None, + ) + if placeholder: + raise ValueError( + f"{METADATA.relative_to(ROOT)}:{line}: placeholder metadata " + f"{placeholder!r} remains for {item.name}" + ) + key = (item.scope, item.name) + if key in seen: + raise ValueError( + f"{METADATA.relative_to(ROOT)}:{line}: duplicate scope/name {key!r}" + ) + seen.add(key) + result.append(item) + + expected_order = sorted(result, key=lambda item: (item.scope, item.name)) + if result != expected_order: + raise ValueError( + f"{METADATA.relative_to(ROOT)} must be sorted by SCOPE and NAME" + ) + + source_cache: Dict[Path, List[str]] = {} + for item in result: + found_name = item.name.startswith("<") + for reference in (part.strip() for part in item.source.split(";")): + match = re.fullmatch(r"(.+):(\d+)", reference) + if not match: + raise ValueError( + f"{METADATA.relative_to(ROOT)}: invalid source reference {reference!r}" + ) + rel_path, line_text = match.groups() + path = ROOT / rel_path + if path not in source_cache: + try: + source_cache[path] = path.read_text( + encoding="utf-8", errors="replace" + ).splitlines() + except OSError as error: + raise ValueError(f"cannot read metadata source {rel_path}: {error}") from error + lines = source_cache[path] + line = int(line_text) + if line < 1 or line > len(lines): + raise ValueError(f"metadata source is outside {rel_path}: {line}") + first = max(0, line - 6) + last = min(len(lines), line + 5) + if item.name in "\n".join(lines[first:last]): + found_name = True + if not found_name: + raise ValueError( + f"metadata source for {item.scope}/{item.name} does not mention the name" + ) + return result + + +def metadata_source_links(source: str) -> str: + links: List[str] = [] + for value in (part.strip() for part in source.split(";")): + match = re.fullmatch(r"(.+):(\d+)", value) + if not match: + links.append(f"`{value}`") + continue + path, line = match.groups() + links.append(f"[{path}:{line}]({path}#L{line})") + return "; ".join(links) + + +def metadata_table(entries: Sequence[VariableDoc]) -> List[str]: + lines = [ + "| Variable | Accepted value and default | Effect | Source |", + "| --- | --- | --- | --- |", + ] + for item in sorted(entries, key=lambda entry: entry.name): + lines.append( + "| `{}` | {} | {} | {} |".format( + item.name, + escape_cell(item.value_default), + escape_cell(item.purpose), + metadata_source_links(item.source), + ) + ) + return lines + + +def group_for(name: str) -> str: + for title, prefix in GROUPS: + if name.startswith(prefix): + return title + return "General and shared" + + +def escape_cell(value: str) -> str: + return value.replace("|", "\\|").replace("\n", " ") + + +def merge_occurrences( + *mappings: Mapping[str, Sequence[Occurrence]], +) -> Dict[str, List[Occurrence]]: + result: Dict[str, List[Occurrence]] = {} + for mapping in mappings: + for name, occurrences in mapping.items(): + result.setdefault(name, []).extend(occurrences) + return result + + +def auxiliary_inputs( + runtime_names: Set[str], external_runtime_names: Set[str] +) -> Tuple[Dict[str, List[Occurrence]], Dict[str, str], Dict[str, List[Occurrence]], Dict[str, str]]: + tests = test_files() + test_direct = scan_direct_getenv(tests) + test_literals = scan_ds4_string_literals( + path for path in tests if path.suffix.lower() in SOURCE_SUFFIXES + ) + test_shell, test_defaults = scan_shell_inputs(tests) + test_all = merge_occurrences(test_direct, test_literals, test_shell) + test_only = { + name: occurrences + for name, occurrences in test_all.items() + if name not in runtime_names and name not in external_runtime_names + and name not in NON_ENV_DS4_STRINGS + and name not in TEST_NON_ENV_DS4_STRINGS + and (name.startswith("DS4_") or name in {"DEEPSEEK_API_KEY", "OPENROUTER_API_KEY", "PROTO_Q8_DEBUG", "TMPDIR"}) + } + + tool_files: List[Path] = [] + for directory, dirnames, filenames in os.walk(ROOT): + directory_path = Path(directory) + rel_dir = relative(directory_path) + dirnames[:] = [ + name + for name in dirnames + if name not in {".git", "__pycache__", "out", "dataset", "tests"} + and not name.endswith(".dSYM") + ] + if rel_dir.parts[:2] == ("cuda", "mmq"): + dirnames[:] = [name for name in dirnames if name != "test"] + for filename in filenames: + path = directory_path / filename + if path.suffix not in {".py", ".sh"}: + continue + if path.resolve() == Path(__file__).resolve(): + continue + tool_files.append(path) + tool_direct = scan_direct_getenv(tool_files) + tool_shell, tool_defaults = scan_shell_inputs(tool_files) + tool_all = { + name: occurrences + for name, occurrences in merge_occurrences(tool_direct, tool_shell).items() + if name.startswith("DS4_") + or name in {"DEEPSEEK_API_KEY", "FLATTEN_DOWNLOADS", "FORCE_HF_DOWNLOAD", "HF_TOKEN", "OPENROUTER_API_KEY"} + } + return test_only, test_defaults, tool_all, tool_defaults + + +def generated_block() -> Tuple[str, int, int, int, int]: + production = source_files() + literal_occurrences = scan_ds4_string_literals(production) + unexpected_allowlist = NON_ENV_DS4_STRINGS - set(literal_occurrences) + if unexpected_allowlist: + names = ", ".join(sorted(unexpected_allowlist)) + raise ValueError(f"stale NON_ENV_DS4_STRINGS entries: {names}") + + runtime_names = set(literal_occurrences) - NON_ENV_DS4_STRINGS + direct = scan_direct_getenv(production) + external_names = {name for name in direct if not name.startswith("DS4_")} + + metadata = load_metadata() + runtime_docs: Dict[str, VariableDoc] = {} + external_docs: Dict[str, VariableDoc] = {} + for item in metadata: + target: Dict[str, VariableDoc] + if item.scope.startswith("runtime/"): + target = runtime_docs + elif item.scope == "external/system": + target = external_docs + else: + continue + if item.name in target: + raise ValueError(f"duplicate documented runtime name: {item.name}") + target[item.name] = item + + missing_runtime = runtime_names - set(runtime_docs) + stale_runtime = set(runtime_docs) - runtime_names + missing_external = external_names - set(external_docs) + stale_external = set(external_docs) - external_names + if missing_runtime or stale_runtime or missing_external or stale_external: + details: List[str] = [] + if missing_runtime: + details.append("undocumented runtime: " + ", ".join(sorted(missing_runtime))) + if stale_runtime: + details.append("stale runtime metadata: " + ", ".join(sorted(stale_runtime))) + if missing_external: + details.append("undocumented external runtime: " + ", ".join(sorted(missing_external))) + if stale_external: + details.append("stale external metadata: " + ", ".join(sorted(stale_external))) + raise ValueError("; ".join(details)) + + test_binary = [item for item in metadata if item.scope == "test-only"] + test_scripts = [item for item in metadata if item.scope == "test-script"] + tools = [item for item in metadata if item.scope.startswith("script/")] + + scanned_tests, _test_defaults, scanned_tools, _tool_defaults = auxiliary_inputs( + runtime_names, external_names + ) + documented_test_names = { + item.name for item in test_binary + test_scripts + } - runtime_names - external_names + if set(scanned_tests) != documented_test_names: + missing = set(scanned_tests) - documented_test_names + stale = documented_test_names - set(scanned_tests) + details = [] + if missing: + details.append("undocumented test input: " + ", ".join(sorted(missing))) + if stale: + details.append("stale test metadata: " + ", ".join(sorted(stale))) + raise ValueError("; ".join(details)) + + documented_tool_names = {item.name for item in tools} + allowed_dynamic_tool_names = {"", "HOME"} + if set(scanned_tools) != documented_tool_names - allowed_dynamic_tool_names: + missing = set(scanned_tools) - documented_tool_names + stale = documented_tool_names - allowed_dynamic_tool_names - set(scanned_tools) + details = [] + if missing: + details.append("undocumented tool input: " + ", ".join(sorted(missing))) + if stale: + details.append("stale tool metadata: " + ", ".join(sorted(stale))) + raise ValueError("; ".join(details)) + + lines = [ + START_MARKER, + "## Complete implementation inventory", + "", + "This section is generated by `scripts/generate_environment_variables.py`; do not edit it by hand.", + "Human-reviewed value/default and purpose metadata lives in", + "`scripts/environment_variables.tsv`; the generator verifies it against the source tree.", + "It lists every `DS4_*` string consumed by production C/C++/Objective-C/CUDA/ROCm", + "sources, including names passed indirectly through helper functions, macros, and", + "source-specification arrays. Unless a variable appears in the user-facing reference", + "above, it is an unstable internal diagnostic or tuning interface. The linked source", + "remains normative for exact eligibility", + "gates, bounds, and architecture-specific defaults.", + "", + f"Inventory totals: **{len(runtime_names)} `DS4_*` runtime variables** and", + f"**{len(external_names)} external runtime variables**.", + f"The auxiliary inventories contain **{len(test_binary) + len(test_scripts)} test/test-fixture entries**", + f"and **{len(tools)} tool/wrapper entries**.", + "", + ] + + ordered_groups = [title for title, _prefix in GROUPS] + ["General and shared"] + for title in ordered_groups: + entries = [item for name, item in runtime_docs.items() if group_for(name) == title] + if not entries: + continue + lines.extend( + [ + "
", + f"{title} ({len(entries)})", + "", + *metadata_table(entries), + "", + "
", + "", + ] + ) + + if external_names: + lines.extend( + [ + "### External runtime environment", + "", + "These names are not owned by the `DS4_*` namespace but are read directly by", + "the binaries or vendored runtime code.", + "", + *metadata_table(list(external_docs.values())), + "", + ] + ) + + lines.extend( + [ + "## Test and fixture environment inputs", + "", + "These entries are consumed by repository test binaries or fixture scripts. Some", + "production runtime controls are repeated here because a maintained fixture exposes", + "them as part of its own test contract.", + "", + "### Test binaries and cleanup hooks", + "", + *metadata_table(test_binary), + "", + "### Test fixture scripts", + "", + *metadata_table(test_scripts), + "", + "## Tool and wrapper environment inputs", + "", + "These variables configure maintained download, service-wrapper, and offline tooling.", + "A tool that accepts a variable name dynamically (for example `--api-key-env`) may read", + "the caller-selected name in addition to the literal defaults listed here.", + "", + *metadata_table(tools), + "", + END_MARKER, + ] + ) + return ( + "\n".join(lines), + len(runtime_names), + len(external_names), + len(test_binary) + len(test_scripts), + len(tools), + ) + + +def replace_generated_block(document: str, block: str) -> str: + start = document.find(START_MARKER) + end = document.find(END_MARKER) + if start < 0 or end < 0 or end < start: + raise ValueError("environment inventory markers are missing from ENVIRONMENT_VARIABLES.md") + end += len(END_MARKER) + return document[:start] + block + document[end:] + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + mode = parser.add_mutually_exclusive_group() + mode.add_argument("--check", action="store_true", help="verify the checked-in reference") + mode.add_argument("--emit", action="store_true", help="print the generated inventory block") + args = parser.parse_args() + + try: + block, runtime_count, external_count, test_count, tool_count = generated_block() + if args.emit: + print(block) + return 0 + document = DOCUMENT.read_text(encoding="utf-8") + expected = replace_generated_block(document, block) + except (OSError, ValueError) as error: + print(f"environment-variable documentation: {error}", file=sys.stderr) + return 1 + + summary = ( + f"{DOCUMENT.relative_to(ROOT)}: {runtime_count} DS4 runtime, " + f"{external_count} external runtime, {test_count} test/test-fixture entries, " + f"{tool_count} tool/wrapper entries" + ) + if args.check: + if document != expected: + print(f"{summary}; generated inventory is stale", file=sys.stderr) + print("run: python3 scripts/generate_environment_variables.py", file=sys.stderr) + return 1 + print(f"{summary}; generated inventory is current") + return 0 + + DOCUMENT.write_text(expected, encoding="utf-8") + print(f"updated {summary}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 8dd4bf566f5558a22dcc8d0f6b3fe56ac2e57aec Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Sun, 23 Aug 2026 08:05:40 +0200 Subject: [PATCH 052/189] Update CONTRIBUTING.md Co-authored-by: Frank --- CONTRIBUTING.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1571397ff..f915b9262 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -30,8 +30,7 @@ environment variable, update `scripts/environment_variables.tsv`, regenerate the complete reference, and verify that it is current: ```sh -python3 scripts/generate_environment_variables.py -make check-environment-docs +make environment-docs ``` Useful narrower checks: From c6f02bfc0a5cf2d4a94a17a3aa1653ec2180b93c Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Sun, 23 Aug 2026 08:06:05 +0200 Subject: [PATCH 053/189] Update Makefile Co-authored-by: Frank --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 2fbb9a9af..915a177fc 100644 --- a/Makefile +++ b/Makefile @@ -408,7 +408,8 @@ test-mmq-parity-cuda: cuda/mmq/test/test_mmq_parity ./cuda/mmq/test/test_mmq_parity endif -check-environment-docs: +environment-docs: + python3 scripts/generate_environment_variables.py python3 scripts/generate_environment_variables.py --check ds4.o: ds4.c ds4.h ds4_ssd.h ds4_distributed.h ds4_gpu.h From c3b29fd897e2d4ec87967ce0219bf50f5f03940c Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Sun, 23 Aug 2026 08:06:17 +0200 Subject: [PATCH 054/189] Update Makefile Co-authored-by: Frank --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 915a177fc..35b29298a 100644 --- a/Makefile +++ b/Makefile @@ -80,7 +80,7 @@ help: @echo " make Build Metal ./ds4, ./ds4-server, ./ds4-bench, ./ds4-eval, and ./ds4-agent" @echo " make cpu Build CPU-only ./ds4, ./ds4-server, ./ds4-bench, ./ds4-eval, and ./ds4-agent" @echo " make test Build and run tests" - @echo " make check-environment-docs Verify the generated environment-variable inventory" + @echo " make environment-docs Generate and verify the environment variable inventory" @echo " make test-metal-session-batch-ssd Exact-logit Metal SSD union control/candidate oracle" @echo " make test-metal-q4-streams Check resident Q4 Metal stream overlap" @echo " make test-metal-q4-attn-exactn Bitwise/canary oracle for M1-M4 SSD-prefill Q4 attention output" From d9d1c8120985e984554cb8ffcb114da35d6824ae Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Sun, 23 Aug 2026 08:06:31 +0200 Subject: [PATCH 055/189] Update Makefile Co-authored-by: Frank --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 35b29298a..2a0c6f934 100644 --- a/Makefile +++ b/Makefile @@ -68,7 +68,7 @@ DS4_LINK_LIBS ?= $(CUDA_LDLIBS) METAL_LDLIBS := $(LDLIBS) endif -.PHONY: all help clean test check-environment-docs test-rocm test-glm53-kda-rocm test-metal-session-batch test-metal-session-batch-ssd test-metal-q4-streams test-metal-q4-attn-exactn test-metal-exactn-oracle test-metal-dspark-capture test-metal-iq2-midonly test-metal-iq2-ssd-grouped-mm test-metal-iq2-live-index test-mxfp4-cuda test-mxfp4-rocm test-mmq-parity-cuda test-rocm-q4-parity test-rocm-q4-dense test-rocm-q4-pair test-rocm-q4-prefill test-strix-rocm-q4-parity test-strix-rocm-q4-prefill test-strix-rocm-q4-prefill-long test-cuda-session-batch test-cuda-mixed-batch dspark-acceptance dspark-verify-depth rocm-dspark-acceptance rocm-dspark-verify-depth mtp-verify-depth cpu cuda cuda-spark cuda-generic cuda-regression strix-halo rocm +.PHONY: all help clean test environment-docs test-rocm test-glm53-kda-rocm test-metal-session-batch test-metal-session-batch-ssd test-metal-q4-streams test-metal-q4-attn-exactn test-metal-exactn-oracle test-metal-dspark-capture test-metal-iq2-midonly test-metal-iq2-ssd-grouped-mm test-metal-iq2-live-index test-mxfp4-cuda test-mxfp4-rocm test-mmq-parity-cuda test-rocm-q4-parity test-rocm-q4-dense test-rocm-q4-pair test-rocm-q4-prefill test-strix-rocm-q4-parity test-strix-rocm-q4-prefill test-strix-rocm-q4-prefill-long test-cuda-session-batch test-cuda-mixed-batch dspark-acceptance dspark-verify-depth rocm-dspark-acceptance rocm-dspark-verify-depth mtp-verify-depth cpu cuda cuda-spark cuda-generic cuda-regression strix-halo rocm ifeq ($(UNAME_S),Darwin) .PHONY: metal-decode-schedule-bench metal-prefill-variant-bench check-mxfp4-half-lut test-mxfp4-metal From af132474557d1cab8a6f6f48c27916df8c412960 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Sun, 23 Aug 2026 08:06:44 +0200 Subject: [PATCH 056/189] Update Makefile Co-authored-by: Frank --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 2a0c6f934..f9cb75b27 100644 --- a/Makefile +++ b/Makefile @@ -281,7 +281,7 @@ help: @echo " make rocm-dspark-verify-depth Build ROCm and run the DSpark verifier invariant" @echo " make cpu Build CPU-only ./ds4, ./ds4-server, ./ds4-bench, ./ds4-eval, and ./ds4-agent" @echo " make test Build and run tests" - @echo " make check-environment-docs Verify the generated environment-variable inventory" + @echo " make environment-docs Generate and verify the environment variable inventory" @echo " make dspark-verify-depth Run DSpark speculative verification smoke if support GGUF is present" @echo " make mtp-verify-depth Run legacy MTP speculative verification smoke if MTP GGUF is present" @echo " make clean Remove build outputs" From d545a1f75bbc1a16080dc9373def1291ce32d0b5 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Sun, 23 Aug 2026 08:07:07 +0200 Subject: [PATCH 057/189] Update metal/moe.metal Co-authored-by: Frank --- metal/moe.metal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metal/moe.metal b/metal/moe.metal index e70249539..1f1c72c61 100644 --- a/metal/moe.metal +++ b/metal/moe.metal @@ -3496,7 +3496,7 @@ void kernel_mul_mv_iq2_xxs_pair_swiglu_mid_only_4096x2048_impl( float sg = 0; float su = 0; - for (short l = 0; l < 4; ++l) { + FOR_UNROLL (short l = 0; l < 4; ++l) { const threadgroup uint8_t *gridg = (const threadgroup uint8_t *)(svalues + aux8g[l]); const threadgroup uint8_t *gridu = From 1ab8a2f587ee34d60a9fdce72f5fc6418cf948f2 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Sun, 23 Aug 2026 08:07:25 +0200 Subject: [PATCH 058/189] Update metal/moe.metal Co-authored-by: Frank --- metal/moe.metal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metal/moe.metal b/metal/moe.metal index 1f1c72c61..fc24d4507 100644 --- a/metal/moe.metal +++ b/metal/moe.metal @@ -3473,7 +3473,7 @@ void kernel_mul_mv_iq2_xxs_pair_swiglu_mid_only_4096x2048_impl( const int ix = tiisg; device const float *y4 = y + 32 * ix; for (int ib32 = ix; ib32 < nb32; ib32 += 32) { - for (short i = 0; i < 32; ++i) { + FOR_UNROLL (short i = 0; i < 32; ++i) { yl[i] = y4[i]; } From 28623a34b205ee91dbf78dce21a6c06526bd81dd Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Sun, 23 Aug 2026 08:07:41 +0200 Subject: [PATCH 059/189] Update metal/moe.metal Co-authored-by: Frank --- metal/moe.metal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metal/moe.metal b/metal/moe.metal index fc24d4507..446e1c282 100644 --- a/metal/moe.metal +++ b/metal/moe.metal @@ -3503,7 +3503,7 @@ void kernel_mul_mv_iq2_xxs_pair_swiglu_mid_only_4096x2048_impl( (const threadgroup uint8_t *)(svalues + aux8u[l]); const uint8_t signg = ssigns[(aux32g >> 7 * l) & 127]; const uint8_t signu = ssigns[(aux32u >> 7 * l) & 127]; - for (short j = 0; j < 8; ++j) { + FOR_UNROLL (short j = 0; j < 8; ++j) { const float v = yl[8 * l + j]; sg += v * gridg[j] * (signg & ds4_metal_kmask_iq2xs[j] ? -1.f : 1.f); From 0d774a9ce577130d168843529e9f35775b954062 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Sun, 23 Aug 2026 08:08:24 +0200 Subject: [PATCH 060/189] Update metal/moe.metal Co-authored-by: Frank --- metal/moe.metal | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/metal/moe.metal b/metal/moe.metal index 446e1c282..419eba139 100644 --- a/metal/moe.metal +++ b/metal/moe.metal @@ -3461,12 +3461,15 @@ void kernel_mul_mv_iq2_xxs_pair_swiglu_mid_only_4096x2048_impl( threadgroup uint64_t *svalues = (threadgroup uint64_t *)(shmem); threadgroup uint8_t *ssigns = (threadgroup uint8_t *)(svalues + 256); { - int nval = 4; - int pos = (32 * sgitg + tiisg) * nval; - for (int i = 0; i < nval; ++i) svalues[pos + i] = ds4_metal_iq2xxs_grid[pos + i]; - nval = 2; - pos = (32 * sgitg + tiisg) * nval; - for (int i = 0; i < nval; ++i) ssigns[pos + i] = ds4_metal_ksigns_iq2xs[pos + i]; + int base = (32 * sgitg + tiisg); + int pos_a = base << 2; + int pos_b = base << 1; + svalues[pos_a] = ds4_metal_iq2xxs_grid[pos_a]; + svalues[pos_a + 1] = ds4_metal_iq2xxs_grid[pos_a + 1]; + svalues[pos_a + 2] = ds4_metal_iq2xxs_grid[pos_a + 2]; + svalues[pos_a + 3] = ds4_metal_iq2xxs_grid[pos_a + 3]; + ssigns[pos_b] = ds4_metal_ksigns_iq2xs[pos_b]; + ssigns[pos_b + 1] = ds4_metal_ksigns_iq2xs[pos_b + 1]; threadgroup_barrier(mem_flags::mem_threadgroup); } From 03b97c0149defdf96911741aade58ac9c0f70fa9 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Sun, 23 Aug 2026 08:51:09 +0200 Subject: [PATCH 061/189] Update cuda/mmq/test/test_mmq_parity.cu Co-authored-by: Frank --- cuda/mmq/test/test_mmq_parity.cu | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/cuda/mmq/test/test_mmq_parity.cu b/cuda/mmq/test/test_mmq_parity.cu index 120ed016f..00ba0e3e8 100644 --- a/cuda/mmq/test/test_mmq_parity.cu +++ b/cuda/mmq/test/test_mmq_parity.cu @@ -1090,14 +1090,15 @@ bool run_iq2_xxs_q2_K_fused_raw_parity( cudaMemcpyAsync(down_canary.data(), d_down_got, down_canary.size(), cudaMemcpyDeviceToHost, stream); cudaError_t sync_err = cudaStreamSynchronize(stream); - const auto canary_intact = [](const std::vector & bytes) { + const auto is_canary_intact = [](const std::vector & bytes) { return std::all_of(bytes.begin(), bytes.end(), [](uint8_t value) { return value == 0xA5; }); }; + const bool na_ok = rc_na == DS4_MMQ_NOT_APPLICABLE && - sync_err == cudaSuccess && canary_intact(gate_canary) && - canary_intact(up_canary) && canary_intact(mid_canary) && - canary_intact(down_canary); + sync_err == cudaSuccess && is_canary_intact(gate_canary) && + is_canary_intact(up_canary) && is_canary_intact(mid_canary) && + is_canary_intact(down_canary); cudaMemsetAsync(d_gate_ref, 0, mid_count * sizeof(float), stream); cudaMemsetAsync(d_up_ref, 0, mid_count * sizeof(float), stream); From de2efc9c303af8dbd11e93b8a9971c63ce593670 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Sun, 23 Aug 2026 08:53:15 +0200 Subject: [PATCH 062/189] Update cuda/mmq/ds4_mmq.cu Co-authored-by: Frank --- cuda/mmq/ds4_mmq.cu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cuda/mmq/ds4_mmq.cu b/cuda/mmq/ds4_mmq.cu index a6ee25af8..5aaf53054 100644 --- a/cuda/mmq/ds4_mmq.cu +++ b/cuda/mmq/ds4_mmq.cu @@ -446,7 +446,7 @@ static bool ds4_mmq_q8_fold_oracle_bytes( cudaStreamCaptureStatus capture = cudaStreamCaptureStatusNone; if (cudaStreamIsCapturing(stream, &capture) != cudaSuccess || capture != cudaStreamCaptureStatusNone) { - (void)cudaGetLastError(); + fprintf(stderr, "Unable to capture CUDA stream\nERROR: %s", cudaGetErrorString(cudaGetLastError())); g_q8_fold_oracle_skips++; return false; } From 0e34a59f70f5ec2c86a05035a141a106bd489ace Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Sun, 23 Aug 2026 14:46:14 +0200 Subject: [PATCH 063/189] Update cuda/mmq/ds4_mmq.cu Co-authored-by: Frank --- cuda/mmq/ds4_mmq.cu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cuda/mmq/ds4_mmq.cu b/cuda/mmq/ds4_mmq.cu index 5aaf53054..cacb44499 100644 --- a/cuda/mmq/ds4_mmq.cu +++ b/cuda/mmq/ds4_mmq.cu @@ -459,7 +459,7 @@ static bool ds4_mmq_q8_fold_oracle_bytes( char *host = (char *)malloc(bytes * 2u); if (!host || cudaMalloc((void **)&fresh, bytes) != cudaSuccess || !fresh) { free(host); - (void)cudaGetLastError(); + fprintf(stderr, "Unable to allocate tensor \"fresh\"\nERROR: %s", cudaGetErrorString(cudaGetLastError())); g_q8_fold_oracle_skips++; return false; } From 86b1f610f4f1f9a861deb680378f898957d7a963 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Sun, 23 Aug 2026 09:52:28 +0200 Subject: [PATCH 064/189] perf(metal): specialize fixed-shape Q8 attention output --- ds4_gpu.h | 1 + ds4_metal.m | 28 +++++++++++++++++-- metal/moe.metal | 74 +++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 101 insertions(+), 2 deletions(-) diff --git a/ds4_gpu.h b/ds4_gpu.h index 1331176dc..0ba72b23b 100644 --- a/ds4_gpu.h +++ b/ds4_gpu.h @@ -280,6 +280,7 @@ enum { DS4_GPU_TEST_HC_RMS_SCALE_PROJ = 1u << 6, DS4_GPU_TEST_STREAMING_LIVE_INDEX_FAILURE = 1u << 7, DS4_GPU_TEST_IQ2_SSD_GROUPED_PIPELINE_FAILURE = 1u << 8, + DS4_GPU_TEST_ATTN_OUT_LOW_Q8_STATIC = 1u << 9, }; void ds4_gpu_test_set_flags(uint32_t flags); void ds4_gpu_release_zero_prefix_prefill_mask_cache(void); diff --git a/ds4_metal.m b/ds4_metal.m index c7fa4d4a2..6bc652401 100644 --- a/ds4_metal.m +++ b/ds4_metal.m @@ -204,6 +204,7 @@ static id g_dsv4_indexed_attention_heads16_dual_pipeline; static id g_dsv4_indexed_attention_heads8_split_pipeline; static id g_dsv4_indexed_attention_heads8_split_reduce_pipeline; +static bool g_attn_out_low_q8_static_unavailable; static id g_dsv4_softplus_sqrt_pipeline; static id g_dsv4_router_finalize_one_pipeline; static id g_dsv4_router_finalize_one_simd_pipeline; @@ -11422,6 +11423,7 @@ void ds4_gpu_cleanup(void) { g_dsv4_indexed_attention_heads16_dual_pipeline = nil; g_dsv4_indexed_attention_heads8_split_pipeline = nil; g_dsv4_indexed_attention_heads8_split_reduce_pipeline = nil; + g_attn_out_low_q8_static_unavailable = false; g_dsv4_softplus_sqrt_pipeline = nil; g_dsv4_router_finalize_one_pipeline = nil; g_dsv4_router_finalize_one_simd_pipeline = nil; @@ -27954,8 +27956,30 @@ int ds4_gpu_attention_output_q8_batch_tensor( .nb1 = (uint64_t)rank * sizeof(float), .nr0 = 2, }; - id pipeline = - ds4_gpu_get_mul_mv_pipeline("kernel_dsv4_attn_out_low_q8_0_f32", 4); + const bool force_flash_decode_static_for_test = + (g_test_flags & DS4_GPU_TEST_ATTN_OUT_LOW_Q8_STATIC) != 0u; + const bool use_pre_m5_flash_decode_static = + (ds4_gpu_device_is_pre_m5_apple_silicon() || + force_flash_decode_static_for_test) && + (!g_attn_out_low_q8_static_unavailable || + force_flash_decode_static_for_test) && + getenv("DS4_METAL_DISABLE_PRE_M5_DECODE_PORTS") == NULL && + getenv("DS4_METAL_DISABLE_PRE_M5_ATTN_OUT_LOW_Q8_STATIC") == NULL && + group_dim == 4096u && rank == 1024u && n_groups == 8u && + low_dim == 8192u && row_a_bytes == 4352u && + out_a_bytes == 35651584u; + id pipeline = ds4_gpu_get_mul_mv_pipeline( + use_pre_m5_flash_decode_static ? + "kernel_dsv4_attn_out_low_q8_0_flash_decode_static_f32" : + "kernel_dsv4_attn_out_low_q8_0_f32", + 4); + if (!pipeline && use_pre_m5_flash_decode_static) { + g_attn_out_low_q8_static_unavailable = true; + if (!force_flash_decode_static_for_test) { + pipeline = ds4_gpu_get_mul_mv_pipeline( + "kernel_dsv4_attn_out_low_q8_0_f32", 4); + } + } ok = ds4_gpu_encode_attn_out_low_q8_direct(cb, pipeline, &args, diff --git a/metal/moe.metal b/metal/moe.metal index 419eba139..ac4269c11 100644 --- a/metal/moe.metal +++ b/metal/moe.metal @@ -3982,6 +3982,80 @@ kernel void kernel_dsv4_attn_out_low_q8_0_f32( sgitg); } +#define DS4_ATTN_OUT_LOW_Q8_STATIC_K 4096 +#define DS4_ATTN_OUT_LOW_Q8_STATIC_ROWS 1024 +#define DS4_ATTN_OUT_LOW_Q8_STATIC_BLOCKS 128 +#define DS4_ATTN_OUT_LOW_Q8_STATIC_ROW_BYTES 4352 +#define DS4_ATTN_OUT_LOW_Q8_STATIC_GROUP_BYTES 4456448 + +static inline void ds4_attn_out_low_q8_static_impl( + device const char * src0s, + device const char * src1, + device char * dst, + threadgroup char * shmem, + uint3 tgpig, + ushort tiisg, + ushort sgitg) { + constexpr short NR0 = N_R0_Q8_0; + constexpr short NW = N_SIMDWIDTH; + constexpr short NQ = 8; + constexpr short NSG = 4; + + const uint group = tgpig.z; + const int r0 = (int)tgpig.x * NR0; + device const char *src0 = src0s + + (uint64_t)group * DS4_ATTN_OUT_LOW_Q8_STATIC_GROUP_BYTES; + device const float *y = (device const float *)(src1 + + (uint64_t)group * DS4_ATTN_OUT_LOW_Q8_STATIC_K * sizeof(float)); + device float *out = (device float *)(dst + + (uint64_t)group * DS4_ATTN_OUT_LOW_Q8_STATIC_ROWS * sizeof(float)); + + device const block_q8_0 *ax[NR0]; + FOR_UNROLL (short row = 0; row < NR0; ++row) { + ax[row] = (device const block_q8_0 *)(src0 + + (uint64_t)(r0 + row) * DS4_ATTN_OUT_LOW_Q8_STATIC_ROW_BYTES); + } + + float sumf[NR0] = { 0.0f }; + const short ix = tiisg / (NW / NQ); + const short il = tiisg % (NW / NQ); + const int ib0 = sgitg * NQ + ix; + device const float *yb = y + ib0 * QK8_0 + il * NQ; + float yl[NQ]; + + for (int ib = ib0; ib < DS4_ATTN_OUT_LOW_Q8_STATIC_BLOCKS; + ib += NSG * NQ) { + for (short i = 0; i < NQ; ++i) yl[i] = yb[i]; + for (short row = 0; row < NR0; ++row) { + device const int8_t *qs = ax[row][ib].qs + il * NQ; + float sumq = 0.0f; + FOR_UNROLL (short i = 0; i < NQ; ++i) sumq += qs[i] * yl[i]; + sumf[row] += sumq * ax[row][ib].d; + } + yb += NSG * NQ * QK8_0; + } + + helper_mv_reduce_and_write( + out, sumf, r0, DS4_ATTN_OUT_LOW_Q8_STATIC_ROWS, + tiisg, sgitg, shmem); +} + +kernel void kernel_dsv4_attn_out_low_q8_0_flash_decode_static_f32( + constant ds4_metal_args_mul_mv_id & args, + device const char * src0s, + device const char * src1, + device char * dst, + threadgroup char * shmem [[threadgroup(0)]], + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiitg[[thread_index_in_threadgroup]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]]) { + ds4_attn_out_low_q8_static_impl( + src0s, src1, dst, shmem, tgpig, tiisg, sgitg); + (void)args; + (void)tiitg; +} + kernel void kernel_dsv4_attn_out_low_q4_K_f32( constant ds4_metal_args_mul_mv_id & args, device const char * src0s, From 2262f13521c23867fb84ec0680b0c322ab0db6be Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Sun, 23 Aug 2026 09:52:56 +0200 Subject: [PATCH 065/189] perf(metal): use gathered attention for raw-only decode --- ds4_metal.m | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/ds4_metal.m b/ds4_metal.m index 6bc652401..7cb347c28 100644 --- a/ds4_metal.m +++ b/ds4_metal.m @@ -29739,9 +29739,9 @@ static int ds4_gpu_encode_flash_kv_stage_f16( bool shared_pad, bool *did_fuse_pad) { if (did_fuse_pad) *did_fuse_pad = false; - if (!cb || !raw || !comp || !dst || raw_cap == 0 || + if (!cb || !raw || !dst || raw_cap == 0 || raw_start >= raw_cap || n_raw == 0 || n_raw > raw_cap || - n_comp == 0 || head_dim == 0) { + (n_comp != 0 && !comp) || head_dim == 0) { return 0; } @@ -29828,6 +29828,9 @@ static int ds4_gpu_encode_flash_kv_stage_f16( dst_offset)) { return 0; } + if (n_comp == 0) { + return 1; + } return ds4_gpu_encode_copy_to_f16_1d( cb, comp, @@ -31536,7 +31539,9 @@ static int ds4_gpu_encode_flash_attention_gathered_heads( ds4_gpu_ported_m5_decode_feature_enabled( "DS4_METAL_DISABLE_PRE_M5_FLASH_ATTN_PACKED32_REDUCE", NULL) && - !g_quality_mode && use_mask == 0u && comp_kv_f16 != 0u && n_comp != 0u && + !g_quality_mode && use_mask == 0u && comp_kv_f16 != 0u && + (n_comp != 0u || + getenv("DS4_METAL_DISABLE_DECODE_RAW_PACKED32") == NULL) && n_head == 64u && head_dim == 512u && nsg == 1u && nwg == 32u && n_keys <= 1024u && g_decode_attn_rope_fuse != 0 && g_decode_attn_rope_args.head_dim == 512 && @@ -33086,7 +33091,12 @@ int ds4_gpu_attention_decode_heads_tensor( id sinks_buf = ds4_gpu_wrap_model_range(model_map, model_size, sinks_offset, sink_bytes, &sinks_inner); if (!sinks_buf) return 0; - if (n_comp == 0) { + /* Raw-only layers historically used a separate five-dispatch path. + * The gathered path handles n_comp == 0 with the same packed + * attention kernel and reduction topology, but fewer dispatches. */ + if (n_comp == 0 && + (use_mask != 0 || + getenv("DS4_METAL_DISABLE_DECODE_RAW_GATHERED_ATTN") != NULL)) { int owned = 0; id cb = ds4_gpu_command_buffer(&owned); if (!cb) return 0; From 15985cb46a36e5eb0d331e7f4d88fdbe396623cc Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Sun, 23 Aug 2026 09:53:23 +0200 Subject: [PATCH 066/189] bench: support SSD-streamed Metal decode comparisons --- speed-bench/README.md | 4 +++- speed-bench/metal_decode_schedule_bench.c | 13 ++++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/speed-bench/README.md b/speed-bench/README.md index 6d4188e72..3390bd8fb 100644 --- a/speed-bench/README.md +++ b/speed-bench/README.md @@ -42,7 +42,9 @@ The harness prefills two sessions and alternates both variant order and variant-to-session assignment. It aborts unless every full-vocabulary logit row is bit-identical and, with `--include-selection`, both variants select the same non-EOS token. Use `--candidate-env NAME` to measure a rollback control, -or `--help` to compare explicit split schedules. +or `--help` to compare explicit split schedules. Pass `--ssd-streaming` for a +model larger than RAM; the harness then skips full-weight warmup while keeping +both variants in the same engine and expert cache. To compare the default pre-M5 ratio-4 compressor pack/transpose fusion with the legacy decode path, including token selection, use: diff --git a/speed-bench/metal_decode_schedule_bench.c b/speed-bench/metal_decode_schedule_bench.c index 5866b49db..a7152bc1c 100644 --- a/speed-bench/metal_decode_schedule_bench.c +++ b/speed-bench/metal_decode_schedule_bench.c @@ -34,6 +34,7 @@ typedef struct { int warmup; int measured; bool include_selection; + bool ssd_streaming; decode_schedule control; decode_schedule candidate; } bench_config; @@ -53,6 +54,7 @@ static void usage(FILE *fp, const char *argv0) { " --candidate-first N candidate first split (default: 1; control with --candidate-env)\n" " --candidate-second N candidate second split (default: 32; control with --candidate-env)\n" " --candidate-env NAME unset NAME for control, set NAME=1 for candidate\n" + " --ssd-streaming use the SSD-backed model path instead of full residency\n" " --include-selection include one non-EOS argmax in each timed step\n", argv0); } @@ -92,6 +94,7 @@ static bench_config parse_options(int argc, char **argv) { .warmup = DEFAULT_WARMUP, .measured = DEFAULT_MEASURED, .include_selection = false, + .ssd_streaming = false, .control = {.first = 2, .second = 32}, .candidate = {.first = 1, .second = 32}, }; @@ -111,6 +114,8 @@ static bench_config parse_options(int argc, char **argv) { cfg.candidate_env = need_arg(&i, argc, argv, arg); } else if (!strcmp(arg, "--include-selection")) { cfg.include_selection = true; + } else if (!strcmp(arg, "--ssd-streaming")) { + cfg.ssd_streaming = true; } else if (!strcmp(arg, "--prefix-tokens")) { cfg.prefix_tokens = parse_int_arg(need_arg(&i, argc, argv, arg), arg, 1); @@ -372,7 +377,8 @@ int main(int argc, char **argv) { .backend = DS4_BACKEND_METAL, .context_size = cfg.ctx, .power_percent = 100, - .warm_weights = true, + .warm_weights = !cfg.ssd_streaming, + .ssd_streaming = cfg.ssd_streaming, }; ds4_engine *engine = NULL; ds4_session *sessions[VARIANT_COUNT] = {0}; @@ -438,7 +444,7 @@ int main(int argc, char **argv) { fprintf(stderr, "metal-decode-schedule-bench: model=%s prompt=%s prefix=%d " "ctx=%d warmup=%d measured=%d control=%d/%d candidate=%d/%d " - "candidate_env=%s include_selection=%s\n", + "candidate_env=%s include_selection=%s ssd_streaming=%s\n", cfg.model_path, cfg.prompt_path, cfg.prefix_tokens, @@ -450,7 +456,8 @@ int main(int argc, char **argv) { cfg.candidate.first, cfg.candidate.second, cfg.candidate_env ? cfg.candidate_env : "(none)", - cfg.include_selection ? "yes" : "no"); + cfg.include_selection ? "yes" : "no", + cfg.ssd_streaming ? "yes" : "no"); const int eos = ds4_token_eos(engine); const int total_steps = cfg.warmup + cfg.measured; From 758a31fb1e599c7c1dd685b5028292d5a5e51331 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Sun, 23 Aug 2026 09:53:50 +0200 Subject: [PATCH 067/189] docs: record M1 raw-gathered decode results --- speed-bench/m1_max_q4_raw_gathered_ab.md | 47 ++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 speed-bench/m1_max_q4_raw_gathered_ab.md diff --git a/speed-bench/m1_max_q4_raw_gathered_ab.md b/speed-bench/m1_max_q4_raw_gathered_ab.md new file mode 100644 index 000000000..21e9f34c3 --- /dev/null +++ b/speed-bench/m1_max_q4_raw_gathered_ab.md @@ -0,0 +1,47 @@ +# M1 Max Q4 raw-gathered attention A/B + +Date: 2026-08-22 + +Hardware: Apple M1 Max, 32 GiB RAM. Backend: Metal with SSD streaming. +Model: `DeepSeek-V4-Flash-IQ2XXS-w2Q2K-AProjQ4-SExpQ8-OutQ8-chat-v2-imatrix-0731.gguf`. + +The control is the new default raw-gathered path. The candidate sets +`DS4_METAL_DISABLE_DECODE_RAW_GATHERED_ATTN=1` and restores the legacy raw-only +attention path. Both use the same decode split schedule. + +## Correctness + +- 16-step greedy top-20 logprob dumps are byte-identical. +- Both files have SHA-256 + `7ee7b8a119f8f93c4ea91fb27867eb4563037e34fc8c61ca795a0953ace61738`. +- Each alternating run compared 145 full-vocabulary frontiers: 18,745,600 + float logits and 144 non-EOS selections were bit-identical. + +## Alternating same-process results + +| Prefix | Variant | Steady tokens/s | Delta | +|---:|---|---:|---:| +| 128 | raw-gathered | 6.7360 | +1.52% | +| 128 | legacy raw | 6.6350 | baseline | +| 2048 | raw-gathered | 5.4325 | +0.66% | +| 2048 | legacy raw | 5.3967 | baseline | + +Command shape: + +```sh +speed-bench/metal_decode_schedule_bench \ + -m /path/to/model.gguf \ + --prompt-file speed-bench/promessi_sposi.txt \ + --prefix-tokens 128 --ctx 300 \ + --warmup 16 --tokens 128 \ + --candidate-env DS4_METAL_DISABLE_DECODE_RAW_GATHERED_ATTN \ + --include-selection --ssd-streaming +``` + +Forcing `DS4_METAL_ENABLE_GATHERED_KV_STAGE=1` on the M1 was neutral: +6.4763 tokens/s versus 6.4769 tokens/s control (-0.01%), with exact logits. +It therefore remains automatic only on its existing device policy. + +The packed32 raw extension is retained for eligible devices, but the M1 does +not arm the inverse-RoPE fusion required by that kernel, so it was not selected +in these measurements. From 60770f709d304ac5e6eb235b6ba3cfdde3ed3119 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Sun, 23 Aug 2026 09:54:57 +0200 Subject: [PATCH 068/189] perf(metal): prune unused prefill indexer queries --- ds4.c | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/ds4.c b/ds4.c index 4f571679a..0cefb207a 100644 --- a/ds4.c +++ b/ds4.c @@ -30658,6 +30658,24 @@ static bool metal_graph_encode_layer_attention_batch( } DS4_METAL_PROFILE_ATTN_STAGE("compressor"); + const bool topk_prefill_needed = + ratio == 4 && n_comp > DS4_N_INDEXER_TOP_K; +#if defined(__APPLE__) + /* Before the compressed cache grows past top-k, zero-prefix prefill + * consumes every compressed row and never reads the transient indexer + * query or its per-head weights. This is also true for the Metal SSD + * layer-major path: the current layer is mapped while these dispatches + * would run, and skipping them changes no persistent cache state. */ + const bool prune_unused_indexer_query = + ratio == 4 && zero_prefix && n_tokens >= 32u && + !topk_prefill_needed && !g->quality && + g->placement == NULL && g->tp_world < 2u && + ds4_gpu_device_is_pre_m5_apple_silicon() && + getenv("DS4_METAL_DISABLE_PRE_M5_BATCH_INDEXER_QUERY_PRUNE") == NULL; +#else + const bool prune_unused_indexer_query = false; +#endif + if (ok && ratio == 4) { const uint32_t index_width = coff * DS4_N_INDEXER_HEAD_DIM; if (!layer->indexer_compressor_kv || !layer->indexer_compressor_gate || @@ -30694,14 +30712,14 @@ static bool metal_graph_encode_layer_attention_batch( (uint64_t)index_width * n_tokens, il, pos0); - if (ok) ok = metal_graph_matmul_plain_tensor(metal_graph_batch_indexer_q(g), + if (ok && !prune_unused_indexer_query) ok = metal_graph_matmul_plain_tensor(metal_graph_batch_indexer_q(g), model, layer->indexer_attn_q_b, q_rank, (uint64_t)DS4_N_INDEXER_HEAD * DS4_N_INDEXER_HEAD_DIM, metal_graph_batch_qr_norm(g), n_tokens); - if (ok) ok = ds4_gpu_rope_tail_tensor(metal_graph_batch_indexer_q(g), + if (ok && !prune_unused_indexer_query) ok = ds4_gpu_rope_tail_tensor(metal_graph_batch_indexer_q(g), n_tokens, DS4_N_INDEXER_HEAD, DS4_N_INDEXER_HEAD_DIM, @@ -30715,10 +30733,10 @@ static bool metal_graph_encode_layer_attention_batch( attn_factor, DS4_ROPE_YARN_BETA_FAST, DS4_ROPE_YARN_BETA_SLOW) != 0; - if (ok) ok = ds4_gpu_dsv4_indexer_qat_tensor(metal_graph_batch_indexer_q(g), + if (ok && !prune_unused_indexer_query) ok = ds4_gpu_dsv4_indexer_qat_tensor(metal_graph_batch_indexer_q(g), n_tokens * DS4_N_INDEXER_HEAD, DS4_N_INDEXER_HEAD_DIM) != 0; - if (ok) ok = ds4_gpu_matmul_f16_tensor(metal_graph_batch_indexer_weights(g), + if (ok && !prune_unused_indexer_query) ok = ds4_gpu_matmul_f16_tensor(metal_graph_batch_indexer_weights(g), model->map, model->size, layer->indexer_proj->abs_offset, @@ -31093,7 +31111,6 @@ static bool metal_graph_encode_layer_attention_batch( if (ok) batch_attention_done = true; } - const bool topk_prefill_needed = ratio == 4 && n_comp > DS4_N_INDEXER_TOP_K; if (ok && zero_prefix && topk_prefill_needed && n_comp != 0) { const float index_scale = 1.0f / sqrtf((float)(DS4_N_INDEXER_HEAD_DIM * DS4_N_INDEXER_HEAD)); double index_stage_t0 = 0.0; From 39e43d9680a5f8ec509b519e75446e543db68d20 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Sun, 23 Aug 2026 09:55:27 +0200 Subject: [PATCH 069/189] perf(cpu): accelerate batched Q4_K matmul --- ds4.c | 197 ++++++++++++++++++++++++++++++++++++++++--- tests/test_q4k_dot.c | 83 +++++++++++++++++- 2 files changed, 267 insertions(+), 13 deletions(-) diff --git a/ds4.c b/ds4.c index 0cefb207a..5d5e676ce 100644 --- a/ds4.c +++ b/ds4.c @@ -3715,6 +3715,129 @@ static void ds4_vec_dot_q4_K_q8_K(int n, float *s, const block_q4_K *x, const bl #endif } +/* Evaluate two activation rows against one Q4_K weight row. Each token keeps + * its own integer and floating-point accumulation order, while the packed + * Q4 nibbles and scale/min metadata are decoded only once. */ +static void ds4_vec_dot_q4_K_q8_K_2( + int n, + float *s0, + float *s1, + const block_q4_K *x, + const block_q8_K *y0, + const block_q8_K *y1) { + const int nb = n / QK_K; + +#if defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD) + const int32x4_t zero = vdupq_n_s32(0); + float sumf0 = 0.0f; + float sumf1 = 0.0f; + + for (int i = 0; i < nb; i++) { + const float xd = f16_to_f32(x[i].d); + const float xmin = f16_to_f32(x[i].dmin); + const float d0 = y0[i].d * xd; + const float d1 = y1[i].d * xd; + const float dm0 = -y0[i].d * xmin; + const float dm1 = -y1[i].d * xmin; + const uint8_t *qs = x[i].qs; + const uint8_t *sc = x[i].scales; + const int8_t *q80 = y0[i].qs; + const int8_t *q81 = y1[i].qs; + + int32_t summs0 = 0; + int32_t summs1 = 0; + for (int j = 0; j < QK_K / 32; j++) { + uint8_t sc_val, m_val; + q4_k_get_scale_min(j, sc, &sc_val, &m_val); + const int32_t gsum0 = (int32_t)y0[i].bsums[j * 2] + + (int32_t)y0[i].bsums[j * 2 + 1]; + const int32_t gsum1 = (int32_t)y1[i].bsums[j * 2] + + (int32_t)y1[i].bsums[j * 2 + 1]; + summs0 += m_val * gsum0; + summs1 += m_val * gsum1; + } + + int isum0 = 0; + int isum1 = 0; + for (int j = 0; j < QK_K / 32; j++) { + uint8_t sc_val, m_val; + q4_k_get_scale_min(j, sc, &sc_val, &m_val); + const int byte_off = (j >> 1) * 32; + const int shift = (j & 1) * 4; + const int8x16x2_t q8v0 = vld1q_s8_x2(q80 + j * 32); + const int8x16x2_t q8v1 = vld1q_s8_x2(q81 + j * 32); + uint8_t q4_u[32]; + if (shift == 0) { + for (int l = 0; l < 32; l++) q4_u[l] = qs[byte_off + l] & 0xF; + } else { + for (int l = 0; l < 32; l++) q4_u[l] = qs[byte_off + l] >> 4; + } + const int8x16_t q4a = vreinterpretq_s8_u8(vld1q_u8(q4_u)); + const int8x16_t q4b = vreinterpretq_s8_u8(vld1q_u8(q4_u + 16)); + isum0 += vaddvq_s32(vdotq_s32(zero, q4a, q8v0.val[0])) * sc_val; + isum0 += vaddvq_s32(vdotq_s32(zero, q4b, q8v0.val[1])) * sc_val; + isum1 += vaddvq_s32(vdotq_s32(zero, q4a, q8v1.val[0])) * sc_val; + isum1 += vaddvq_s32(vdotq_s32(zero, q4b, q8v1.val[1])) * sc_val; + } + + sumf0 += d0 * (float)isum0 + dm0 * (float)summs0; + sumf1 += d1 * (float)isum1 + dm1 * (float)summs1; + } + + *s0 = sumf0; + *s1 = sumf1; +#else + float sumf0 = 0.0f; + float sumf1 = 0.0f; + + for (int i = 0; i < nb; i++) { + const float xd = f16_to_f32(x[i].d); + const float xmin = f16_to_f32(x[i].dmin); + const float d0 = y0[i].d * xd; + const float d1 = y1[i].d * xd; + const float dm0 = -y0[i].d * xmin; + const float dm1 = -y1[i].d * xmin; + const uint8_t *qs = x[i].qs; + const uint8_t *sc = x[i].scales; + const int8_t *q80 = y0[i].qs; + const int8_t *q81 = y1[i].qs; + + int summs0 = 0; + int summs1 = 0; + for (int j = 0; j < QK_K / 32; j++) { + uint8_t sc_val, m_val; + q4_k_get_scale_min(j, sc, &sc_val, &m_val); + const int32_t gsum0 = (int32_t)y0[i].bsums[j * 2] + + (int32_t)y0[i].bsums[j * 2 + 1]; + const int32_t gsum1 = (int32_t)y1[i].bsums[j * 2] + + (int32_t)y1[i].bsums[j * 2 + 1]; + summs0 += m_val * gsum0; + summs1 += m_val * gsum1; + } + + int isum0 = 0; + int isum1 = 0; + for (int j = 0; j < QK_K / 32; j++) { + uint8_t sc_val, m_val; + q4_k_get_scale_min(j, sc, &sc_val, &m_val); + const int byte_off = (j >> 1) * 32; + const int shift = (j & 1) * 4; + for (int l = 0; l < 32; l++) { + const int q4 = (qs[byte_off + l] >> shift) & 0xF; + isum0 += q4 * (int)q80[j * 32 + l] * sc_val; + isum1 += q4 * (int)q81[j * 32 + l] * sc_val; + } + } + + sumf0 += d0 * (float)isum0 + dm0 * (float)summs0; + sumf1 += d1 * (float)isum1 + dm1 * (float)summs1; + } + + *s0 = sumf0; + *s1 = sumf1; +#endif +} + static void ds4_vec_dot_q5_K_q8_K(int n, float *s, const block_q5_K *x, const block_q8_K *y) { const int nb = n / QK_K; float sumf = 0.0f; @@ -8687,12 +8810,60 @@ typedef struct { uint64_t blocks; } matmul_q4_K_batch_ctx; +typedef struct { + const float *x; + block_q8_K *xq; + uint64_t row_dim; + uint64_t blocks; +} quantize_q8_K_rows_ctx; + +static void quantize_q8_K_rows_worker( + void *vctx, uint64_t row0, uint64_t row1) { + quantize_q8_K_rows_ctx *ctx = vctx; + for (uint64_t row = row0; row < row1; row++) { + ds4_quantize_row_q8_K(ctx->x + row * ctx->row_dim, + ctx->xq + row * ctx->blocks, + (int64_t)ctx->row_dim); + } +} + +static void quantize_q8_K_rows( + const float *x, + block_q8_K *xq, + uint64_t n_rows, + uint64_t row_dim) { + quantize_q8_K_rows_ctx ctx = { + .x = x, + .xq = xq, + .row_dim = row_dim, + .blocks = row_dim / QK_K, + }; + /* A pool round-trip costs more than quantizing a handful of rows. Match + * the existing F16 matvec crossover and parallelize only once the batch + * contains at least 256K activation elements. */ + const bool work_overflow = row_dim != 0 && n_rows > UINT64_MAX / row_dim; + const uint64_t work = work_overflow ? UINT64_MAX : n_rows * row_dim; + const uint64_t min_parallel_rows = work >= 262144u ? 1u : 512u; + ds4_parallel_for_min_rows(n_rows, quantize_q8_K_rows_worker, &ctx, + min_parallel_rows); +} + static void matmul_q4_K_batch_worker(void *vctx, uint64_t r0, uint64_t r1) { matmul_q4_K_batch_ctx *ctx = vctx; for (uint64_t r = r0; r < r1; r++) { const block_q4_K *row = (const block_q4_K *) (ctx->data + r * ctx->blocks * sizeof(block_q4_K)); - for (uint64_t t = 0; t < ctx->n_tok; t++) { + uint64_t t = 0; + for (; t + 1 < ctx->n_tok; t += 2) { + ds4_vec_dot_q4_K_q8_K_2( + (int)ctx->in_dim, + &ctx->out[t * ctx->out_dim + r], + &ctx->out[(t + 1) * ctx->out_dim + r], + row, + ctx->xq + t * ctx->blocks, + ctx->xq + (t + 1) * ctx->blocks); + } + for (; t < ctx->n_tok; t++) { ds4_vec_dot_q4_K_q8_K((int)ctx->in_dim, &ctx->out[t * ctx->out_dim + r], row, @@ -8711,9 +8882,7 @@ static void matmul_q4_K_batch( const uint64_t in_dim = w->dim[0]; const uint64_t blocks = in_dim / QK_K; block_q8_K *xq = xmalloc((size_t)n_tok * blocks * sizeof(block_q8_K)); - for (uint64_t t = 0; t < n_tok; t++) { - ds4_quantize_row_q8_K(x + t * in_dim, xq + t * blocks, (int64_t)in_dim); - } + quantize_q8_K_rows(x, xq, n_tok, in_dim); matmul_q4_K_batch_ctx ctx = { .out = out, .data = tensor_data(m, w), @@ -8744,7 +8913,17 @@ static void matmul_q4_K_grouped_batch_worker(void *vctx, uint64_t r0, uint64_t r const uint64_t group = idx / ctx->rank; const block_q4_K *row = (const block_q4_K *) (ctx->data + idx * ctx->blocks * sizeof(block_q4_K)); - for (uint64_t t = 0; t < ctx->n_tok; t++) { + uint64_t t = 0; + for (; t + 1 < ctx->n_tok; t += 2) { + ds4_vec_dot_q4_K_q8_K_2( + (int)ctx->group_dim, + &ctx->out[t * ctx->n_groups * ctx->rank + idx], + &ctx->out[(t + 1) * ctx->n_groups * ctx->rank + idx], + row, + ctx->xq + (t * ctx->n_groups + group) * ctx->blocks, + ctx->xq + ((t + 1) * ctx->n_groups + group) * ctx->blocks); + } + for (; t < ctx->n_tok; t++) { ds4_vec_dot_q4_K_q8_K((int)ctx->group_dim, &ctx->out[t * ctx->n_groups * ctx->rank + idx], row, @@ -8765,13 +8944,7 @@ static void matmul_q4_K_grouped_batch( matvec_q4_K_grouped_expect(w, n_groups, group_dim, rank); const uint64_t blocks = group_dim / QK_K; block_q8_K *xq = xmalloc((size_t)n_tok * n_groups * blocks * sizeof(block_q8_K)); - for (uint64_t t = 0; t < n_tok; t++) { - for (uint32_t g = 0; g < n_groups; g++) { - ds4_quantize_row_q8_K(x + (t * n_groups + g) * group_dim, - xq + (t * n_groups + g) * blocks, - (int64_t)group_dim); - } - } + quantize_q8_K_rows(x, xq, n_tok * n_groups, group_dim); matmul_q4_K_grouped_batch_ctx ctx = { .out = out, .data = tensor_data(m, w), diff --git a/tests/test_q4k_dot.c b/tests/test_q4k_dot.c index 4903ca7fd..8e363d6f3 100644 --- a/tests/test_q4k_dot.c +++ b/tests/test_q4k_dot.c @@ -94,6 +94,52 @@ static void vec_dot_q4_K_q8_K(int n, float *s, const block_q4_K *x, const block_ *s = sumf; } +/* Scalar oracle for the production two-token traversal. */ +static void vec_dot_q4_K_q8_K_2( + int n, float *s0, float *s1, const block_q4_K *x, + const block_q8_K *y0, const block_q8_K *y1) { + const int nb = n / QK_K; + float sumf0 = 0.0f; + float sumf1 = 0.0f; + for (int i = 0; i < nb; i++) { + const float xd = f16_to_f32(x[i].d); + const float xmin = f16_to_f32(x[i].dmin); + const float d0 = y0[i].d * xd; + const float d1 = y1[i].d * xd; + const float dm0 = -y0[i].d * xmin; + const float dm1 = -y1[i].d * xmin; + int summs0 = 0; + int summs1 = 0; + for (int j = 0; j < QK_K / 32; j++) { + uint8_t sc_val, m_val; + q4_k_get_scale_min(j, x[i].scales, &sc_val, &m_val); + const int32_t gsum0 = (int32_t)y0[i].bsums[j * 2] + + (int32_t)y0[i].bsums[j * 2 + 1]; + const int32_t gsum1 = (int32_t)y1[i].bsums[j * 2] + + (int32_t)y1[i].bsums[j * 2 + 1]; + summs0 += m_val * gsum0; + summs1 += m_val * gsum1; + } + int isum0 = 0; + int isum1 = 0; + for (int j = 0; j < QK_K / 32; j++) { + uint8_t sc_val, m_val; + q4_k_get_scale_min(j, x[i].scales, &sc_val, &m_val); + const int byte_off = (j >> 1) * 32; + const int shift = (j & 1) * 4; + for (int l = 0; l < 32; l++) { + const int q4 = (x[i].qs[byte_off + l] >> shift) & 0xF; + isum0 += q4 * (int)y0[i].qs[j * 32 + l] * sc_val; + isum1 += q4 * (int)y1[i].qs[j * 32 + l] * sc_val; + } + } + sumf0 += d0 * (float)isum0 + dm0 * (float)summs0; + sumf1 += d1 * (float)isum1 + dm1 * (float)summs1; + } + *s0 = sumf0; + *s1 = sumf1; +} + /* Reference: fully dequantize Q4_K to float, then dot with Q8_K's dequantized values. */ static float ref_dot(const block_q4_K *bx, const block_q8_K *by) { float x[QK_K]; @@ -204,6 +250,40 @@ static int test_dot_reference(void) { return ok ? 0 : 1; } +static int test_dot_pair_bitwise(void) { + static const int widths[] = {256, 1024, 4096, 7168, 16384}; + block_q4_K bx[16384 / QK_K]; + block_q8_K by0[16384 / QK_K]; + block_q8_K by1[16384 / QK_K]; + int ok = 1; + + for (size_t wi = 0; wi < sizeof(widths) / sizeof(widths[0]); wi++) { + const int nb = widths[wi] / QK_K; + for (uint32_t seed = 1; seed <= 16; seed++) { + for (int b = 0; b < nb; b++) { + fill_q4_K(&bx[b], seed * 101u + (uint32_t)b); + fill_q8_K(&by0[b], seed * 211u + (uint32_t)b); + fill_q8_K(&by1[b], seed * 307u + (uint32_t)b); + } + float ref0 = 0.0f, ref1 = 0.0f; + float got0 = 0.0f, got1 = 0.0f; + vec_dot_q4_K_q8_K(widths[wi], &ref0, bx, by0); + vec_dot_q4_K_q8_K(widths[wi], &ref1, bx, by1); + vec_dot_q4_K_q8_K_2(widths[wi], &got0, &got1, bx, by0, by1); + if (memcmp(&got0, &ref0, sizeof(got0)) != 0 || + memcmp(&got1, &ref1, sizeof(got1)) != 0) { + printf(" width=%d seed=%u: pair=(%g,%g) single=(%g,%g)\n", + widths[wi], seed, got0, got1, ref0, ref1); + ok = 0; + } + } + } + + printf(" two-token dot vs two singles (bitwise): %s\n", + ok ? "PASS" : "FAIL"); + return ok ? 0 : 1; +} + /* Test with a hand-crafted known block. */ static int test_dot_known(void) { block_q4_K bx; @@ -250,7 +330,8 @@ int main(void) { failures += test_scale_extraction(); failures += test_dot_known(); failures += test_dot_reference(); + failures += test_dot_pair_bitwise(); - printf("\n%d/%d tests passed\n", 4 - failures, 4); + printf("\n%d/%d tests passed\n", 5 - failures, 5); return failures ? 1 : 0; } From 072cb047831419357b42008ca73a73407b61cb96 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Sun, 23 Aug 2026 09:55:49 +0200 Subject: [PATCH 070/189] perf(cuda): share Q8_1 activations across Q4_K prefill pairs --- cuda/mmq/ds4_mmq.cu | 184 +++++++++++++++++++++++++++++++ cuda/mmq/ds4_mmq.h | 17 +++ cuda/mmq/test/test_mmq_parity.cu | 104 +++++++++++++++++ ds4_cuda.cu | 44 ++++++-- ds4_gpu.h | 5 +- 5 files changed, 340 insertions(+), 14 deletions(-) diff --git a/cuda/mmq/ds4_mmq.cu b/cuda/mmq/ds4_mmq.cu index cacb44499..5fadf813c 100644 --- a/cuda/mmq/ds4_mmq.cu +++ b/cuda/mmq/ds4_mmq.cu @@ -1108,6 +1108,182 @@ int ds4_mmq_dense_impl( return 0; } +/* Batched Q4_K pair for the prefill tier. The two ordinary dense calls + * differ only in their weight/output rows; their Q8_1 MMQ activation is + * byte-identical. Keep that activation alive across both established MMQ + * launches so Q-A and KV pay the quantize/tail-clear prelude once. */ +int ds4_mmq_q4_K_dense_pair_impl( + const void * W0, + const void * W1, + const float * X_f32, + float * out0_f32, + float * out1_f32, + int M0, + int M1, + int N, + int K, + cudaStream_t stream) { + const char *tag = "ds4_mmq_q4_K_dense_pair"; + if (!W0 || !W1 || !X_f32 || !out0_f32 || !out1_f32) { + fprintf(stderr, "%s: null pointer\n", tag); + return -1; + } + if (M0 <= 0 || M1 <= 0 || N <= 0 || K <= 0 || K % 256 != 0) { + fprintf(stderr, "%s: bad shape M0=%d M1=%d N=%d K=%d\n", + tag, M0, M1, N, K); + return -1; + } + if ((size_t)M0 > SIZE_MAX / (size_t)N / sizeof(float) || + (size_t)M1 > SIZE_MAX / (size_t)N / sizeof(float)) { + fprintf(stderr, "%s: output size overflow\n", tag); + return -1; + } + const size_t out0_bytes = (size_t)M0 * (size_t)N * sizeof(float); + const size_t out1_bytes = (size_t)M1 * (size_t)N * sizeof(float); + const uintptr_t out0_addr = (uintptr_t)out0_f32; + const uintptr_t out1_addr = (uintptr_t)out1_f32; + const bool outputs_overlap = out0_addr <= out1_addr + ? (size_t)(out1_addr - out0_addr) < out0_bytes + : (size_t)(out0_addr - out1_addr) < out1_bytes; + if (outputs_overlap) { + fprintf(stderr, "%s: output ranges overlap\n", tag); + return -1; + } + + const int dev = ggml_cuda_get_device(); + const int cc = ggml_cuda_info().devices[dev].cc; + if (!ds4_mmq_k_tile_supported(tag, K, cc)) return -1; + + ggml_backend_cuda_context *ctx = get_ctx_for_device(dev); + if (!ctx) { + fprintf(stderr, "%s: failed to get cuda context for device %d\n", + tag, dev); + return -1; + } + ds4_pool_set_stream(stream); + + const int64_t ne10_padded = GGML_PAD((int64_t)K, MATRIX_ROW_PADDING); + const size_t blocks_per_col = + (size_t)ne10_padded / (4u * (size_t)QK8_1); + const size_t bytes_per_col = + blocks_per_col * sizeof(block_q8_1_mmq); + const size_t slack_blocks = (size_t)get_mmq_x_max_host(cc); + if ((size_t)N > SIZE_MAX / bytes_per_col || + slack_blocks > SIZE_MAX / sizeof(block_q8_1_mmq)) { + fprintf(stderr, "%s: activation scratch size overflow\n", tag); + return -1; + } + const size_t payload_bytes = (size_t)N * bytes_per_col; + const size_t slack_bytes = slack_blocks * sizeof(block_q8_1_mmq); + if (payload_bytes > SIZE_MAX - slack_bytes) { + fprintf(stderr, "%s: activation scratch size overflow\n", tag); + return -1; + } + const size_t nbytes_q8_1 = payload_bytes + slack_bytes; + + ggml_cuda_pool_alloc src1_q8_1(ctx->pool(), nbytes_q8_1); + ybuf_memset(src1_q8_1.get(), nbytes_q8_1, stream); + quantize_mmq_q8_1_cuda( + X_f32, /*ids=*/nullptr, (void *)src1_q8_1.get(), + GGML_TYPE_Q4_K, /*ne00=*/K, /*s11=*/(int64_t)K, + /*s12=*/0, /*s13=*/0, + /*ne0=*/ne10_padded, /*ne1=*/(int64_t)N, + /*ne2=*/1, /*ne3=*/1, stream); + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) { + fprintf(stderr, "%s: quantize failed: %s\n", + tag, cudaGetErrorString(err)); + return -2; + } + + const int64_t stride_row_x = (int64_t)K / QK_K; + const int64_t stride_channel_y = + (int64_t)(payload_bytes / sizeof(int)); + const bool use_stream_k = + (GGML_CUDA_CC_IS_NVIDIA(cc) && + ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_VOLTA) || + GGML_CUDA_CC_IS_CDNA(cc); + + if (out_memset_enabled()) { + cudaMemsetAsync(out0_f32, 0, out0_bytes, stream); + } + const mmq_args args0 = { + /*x=*/(const char *)W0, + /*type_x=*/GGML_TYPE_Q4_K, + /*y=*/(const int *)src1_q8_1.get(), + /*ids_dst=*/nullptr, + /*expert_bounds=*/nullptr, + /*dst=*/out0_f32, + /*ncols_x=*/(int64_t)K, + /*nrows_x=*/(int64_t)M0, + /*ncols_dst=*/(int64_t)N, + /*stride_row_x=*/stride_row_x, + /*ncols_y=*/(int64_t)N, + /*nrows_dst=*/(int64_t)M0, + /*nchannels_x=*/1, + /*nchannels_y=*/1, + /*stride_channel_x=*/0, + /*stride_channel_y=*/stride_channel_y, + /*stride_channel_dst=*/0, + /*nsamples_x=*/1, + /*nsamples_y=*/1, + /*stride_sample_x=*/0, + /*stride_sample_y=*/stride_channel_y, + /*stride_sample_dst=*/0, + /*use_stream_k=*/use_stream_k, + /*ncols_max=*/(int64_t)N, + }; + mul_mat_q_case(*ctx, args0, stream); + err = cudaGetLastError(); + if (err != cudaSuccess) { + fprintf(stderr, "%s: first mul_mat_q_case launch failed: %s\n", + tag, cudaGetErrorString(err)); + return -3; + } + ds4_mmq_sanitize_f32( + out0_f32, (uint64_t)M0 * (uint64_t)N, stream); + + if (out_memset_enabled()) { + cudaMemsetAsync(out1_f32, 0, out1_bytes, stream); + } + const mmq_args args1 = { + /*x=*/(const char *)W1, + /*type_x=*/GGML_TYPE_Q4_K, + /*y=*/(const int *)src1_q8_1.get(), + /*ids_dst=*/nullptr, + /*expert_bounds=*/nullptr, + /*dst=*/out1_f32, + /*ncols_x=*/(int64_t)K, + /*nrows_x=*/(int64_t)M1, + /*ncols_dst=*/(int64_t)N, + /*stride_row_x=*/stride_row_x, + /*ncols_y=*/(int64_t)N, + /*nrows_dst=*/(int64_t)M1, + /*nchannels_x=*/1, + /*nchannels_y=*/1, + /*stride_channel_x=*/0, + /*stride_channel_y=*/stride_channel_y, + /*stride_channel_dst=*/0, + /*nsamples_x=*/1, + /*nsamples_y=*/1, + /*stride_sample_x=*/0, + /*stride_sample_y=*/stride_channel_y, + /*stride_sample_dst=*/0, + /*use_stream_k=*/use_stream_k, + /*ncols_max=*/(int64_t)N, + }; + mul_mat_q_case(*ctx, args1, stream); + err = cudaGetLastError(); + if (err != cudaSuccess) { + fprintf(stderr, "%s: second mul_mat_q_case launch failed: %s\n", + tag, cudaGetErrorString(err)); + return -4; + } + ds4_mmq_sanitize_f32( + out1_f32, (uint64_t)M1 * (uint64_t)N, stream); + return 0; +} + } // anonymous namespace extern "C" int ds4_mmq_q8_0_dense( @@ -1336,6 +1512,14 @@ extern "C" int ds4_mmq_q4_K_dense( return ds4_mmq_dense_impl("ds4_mmq_q4_K_dense", W, X, out, M, N, K, stream); } +extern "C" int ds4_mmq_q4_K_dense_pair( + const void * W0, const void * W1, const float * X, + float * out0, float * out1, + int M0, int M1, int N, int K, cudaStream_t stream) { + return ds4_mmq_q4_K_dense_pair_impl( + W0, W1, X, out0, out1, M0, M1, N, K, stream); +} + extern "C" int ds4_mmq_mxfp4_dense( const void * W, const float * X, float * out, int M, int N, int K, cudaStream_t stream) { diff --git a/cuda/mmq/ds4_mmq.h b/cuda/mmq/ds4_mmq.h index 6756b57bb..9c3a45e0c 100644 --- a/cuda/mmq/ds4_mmq.h +++ b/cuda/mmq/ds4_mmq.h @@ -176,6 +176,23 @@ int ds4_mmq_q4_K_dense( int K, cudaStream_t stream); +// Two dense Q4_K MMQ projections that share one token-tiled Q8_1 +// activation buffer. This is the prefill sibling of +// ds4_mmq_q4_K_dense_pair_vec: N is not limited to the MMVQ batch ceiling, +// M0 and M1 may differ, and each leg preserves ds4_mmq_q4_K_dense's +// reduction and output layout. The two output ranges must be disjoint. +int ds4_mmq_q4_K_dense_pair( + const void * W0_q4_K, + const void * W1_q4_K, + const float * X_f32, + float * out0_f32, + float * out1_f32, + int M0, + int M1, + int N, + int K, + cudaStream_t stream); + int ds4_mmq_mxfp4_dense( const void * W_mxfp4, const float * X_f32, diff --git a/cuda/mmq/test/test_mmq_parity.cu b/cuda/mmq/test/test_mmq_parity.cu index 00ba0e3e8..2f6fe3a96 100644 --- a/cuda/mmq/test/test_mmq_parity.cu +++ b/cuda/mmq/test/test_mmq_parity.cu @@ -496,6 +496,100 @@ bool run_q4_K(int M, int N, int K, uint32_t seed, float abs_scale = 0.20f) { return ok; } +// Prefill dense-pair verifier. The candidate shares only the canonical +// token-tiled Q8_1 activation; both weight legs still run the ordinary Q4_K +// MMQ kernel, so their outputs must match two independent dense calls bitwise. +bool run_q4_K_dense_pair_parity( + int M0, int M1, int N, int K, uint32_t seed) { + fprintf(stderr, + "=== Q4_K/DENSE_PAIR M0=%d M1=%d N=%d K=%d seed=%u ===\n", + M0, M1, N, K, seed); + + std::mt19937 rng(seed); + std::normal_distribution nd(0.0f, 1.0f); + const int blocks_per_row = K / QK_K_LOCAL; + std::vector W0((size_t)M0 * blocks_per_row); + std::vector W1((size_t)M1 * blocks_per_row); + for (auto &blk : W0) generate_random_block_q4_K(&blk, rng); + for (auto &blk : W1) generate_random_block_q4_K(&blk, rng); + std::vector X((size_t)N * K); + for (auto &v : X) v = nd(rng); + + cudaStream_t stream; + cudaStreamCreate(&stream); + void *dW0 = nullptr; + void *dW1 = nullptr; + float *dX = nullptr; + float *dRef0 = nullptr; + float *dRef1 = nullptr; + float *dGot0 = nullptr; + float *dGot1 = nullptr; + cudaMalloc(&dW0, W0.size() * sizeof(block_q4_K)); + cudaMalloc(&dW1, W1.size() * sizeof(block_q4_K)); + cudaMalloc(&dX, X.size() * sizeof(float)); + cudaMalloc(&dRef0, (size_t)M0 * N * sizeof(float)); + cudaMalloc(&dRef1, (size_t)M1 * N * sizeof(float)); + cudaMalloc(&dGot0, (size_t)M0 * N * sizeof(float)); + cudaMalloc(&dGot1, (size_t)M1 * N * sizeof(float)); + cudaMemcpyAsync(dW0, W0.data(), W0.size() * sizeof(block_q4_K), + cudaMemcpyHostToDevice, stream); + cudaMemcpyAsync(dW1, W1.data(), W1.size() * sizeof(block_q4_K), + cudaMemcpyHostToDevice, stream); + cudaMemcpyAsync(dX, X.data(), X.size() * sizeof(float), + cudaMemcpyHostToDevice, stream); + cudaMemsetAsync(dRef0, 0xa5, (size_t)M0 * N * sizeof(float), stream); + cudaMemsetAsync(dRef1, 0xa5, (size_t)M1 * N * sizeof(float), stream); + cudaMemsetAsync(dGot0, 0x5a, (size_t)M0 * N * sizeof(float), stream); + cudaMemsetAsync(dGot1, 0x5a, (size_t)M1 * N * sizeof(float), stream); + + const int rc0 = ds4_mmq_q4_K_dense( + dW0, dX, dRef0, M0, N, K, stream); + const int rc1 = ds4_mmq_q4_K_dense( + dW1, dX, dRef1, M1, N, K, stream); + const int rcp = ds4_mmq_q4_K_dense_pair( + dW0, dW1, dX, dGot0, dGot1, M0, M1, N, K, stream); + + std::vector ref0((size_t)M0 * N); + std::vector ref1((size_t)M1 * N); + std::vector got0((size_t)M0 * N); + std::vector got1((size_t)M1 * N); + cudaMemcpyAsync(ref0.data(), dRef0, ref0.size() * sizeof(float), + cudaMemcpyDeviceToHost, stream); + cudaMemcpyAsync(ref1.data(), dRef1, ref1.size() * sizeof(float), + cudaMemcpyDeviceToHost, stream); + cudaMemcpyAsync(got0.data(), dGot0, got0.size() * sizeof(float), + cudaMemcpyDeviceToHost, stream); + cudaMemcpyAsync(got1.data(), dGot1, got1.size() * sizeof(float), + cudaMemcpyDeviceToHost, stream); + const cudaError_t sync_err = cudaStreamSynchronize(stream); + + size_t bad0 = 0; + size_t bad1 = 0; + for (size_t i = 0; i < ref0.size(); i++) { + if (std::memcmp(&ref0[i], &got0[i], sizeof(float)) != 0) bad0++; + } + for (size_t i = 0; i < ref1.size(); i++) { + if (std::memcmp(&ref1[i], &got1[i], sizeof(float)) != 0) bad1++; + } + + cudaFree(dW0); + cudaFree(dW1); + cudaFree(dX); + cudaFree(dRef0); + cudaFree(dRef1); + cudaFree(dGot0); + cudaFree(dGot1); + cudaStreamDestroy(stream); + + const bool ok = rc0 == 0 && rc1 == 0 && rcp == 0 && + sync_err == cudaSuccess && bad0 == 0 && bad1 == 0; + fprintf(stderr, + "pair rc=%d/%d/%d sync=%s mismatches=%zu/%zu: %s\n\n", + rc0, rc1, rcp, cudaGetErrorString(sync_err), bad0, bad1, + ok ? "PASS" : "FAIL"); + return ok; +} + // IQ2_XXS internally accumulates in int8 via SIMD intrinsics // (__vsub4 / __vcmpne4 in vec_dot_iq2_xxs_q8_1) and applies the scale // post-accumulation, while the CPU reference does per-element float @@ -1946,6 +2040,16 @@ int main(int argc, char ** argv) { all_ok &= run_q4_K(/*M=*/128, /*N=*/8, /*K=*/512, 0xC4FE2); all_ok &= run_q4_K(/*M=*/256, /*N=*/1, /*K=*/2048, 0xC4FE3); all_ok &= run_q4_K(/*M=*/2048, /*N=*/16, /*K=*/4096, 0xC4FE4); + // Prefill Q-A/KV pair: cover the MMVQ/MMQ boundary, token-tile tails, + // asymmetric output dimensions, and a full-width token tile. + all_ok &= run_q4_K_dense_pair_parity( + /*M0=*/257, /*M1=*/65, /*N=*/9, /*K=*/768, 0xC4FE50); + all_ok &= run_q4_K_dense_pair_parity( + /*M0=*/128, /*M1=*/73, /*N=*/32, /*K=*/4096, 0xC4FE51); + all_ok &= run_q4_K_dense_pair_parity( + /*M0=*/65, /*M1=*/129, /*N=*/129, /*K=*/1024, 0xC4FE52); + all_ok &= run_q4_K_dense_pair_parity( + /*M0=*/96, /*M1=*/33, /*N=*/128, /*K=*/4096, 0xC4FE53); // MoE (_id) path. Small expert counts + small shapes for fast verification. // Per-token-distinct routing with top_k=2 or 6. diff --git a/ds4_cuda.cu b/ds4_cuda.cu index ce6a9cdf4..53379abf8 100644 --- a/ds4_cuda.cu +++ b/ds4_cuda.cu @@ -39130,7 +39130,8 @@ static int cuda_matmul_q4_K_pair_tensor_impl( uint64_t out1_dim, const ds4_gpu_tensor *x, uint64_t n_tok) { - if (!out0 || !out1 || !x || !model_map || n_tok == 0u || n_tok > 8u || + if (!out0 || !out1 || !x || !model_map || n_tok == 0u || + n_tok > INT_MAX || in_dim == 0u || (in_dim % CUDA_QK_K) != 0u || in_dim > INT_MAX || out0_dim == 0u || out1_dim == 0u || out0_dim > INT_MAX || out1_dim > INT_MAX) { @@ -39166,6 +39167,12 @@ static int cuda_matmul_q4_K_pair_tensor_impl( out1->bytes < out1_bytes) { return 0; } + const uintptr_t out0_addr = (uintptr_t)out0->ptr; + const uintptr_t out1_addr = (uintptr_t)out1->ptr; + const bool outputs_overlap = out0_addr <= out1_addr + ? (uint64_t)(out1_addr - out0_addr) < out0_bytes + : (uint64_t)(out0_addr - out1_addr) < out1_bytes; + if (outputs_overlap) return 0; const int logical_tier = ds4_tensor_device_idx(out0); if (logical_tier < 0 || logical_tier >= g_n_gpus || @@ -39184,26 +39191,39 @@ static int cuda_matmul_q4_K_pair_tensor_impl( const int gb10_canonical = cuda_q4_gb10_fast_path_enabled( logical_tier, "DS4_CUDA_DISABLE_Q4_DENSE_PAIR"); if (cuda_use_mmq()) { - /* One Q8_1 activation allocation/quantization is shared by both - * canonical MMVQ legs. On GB10 a rejected launch fails closed to - * ds4.c's independent projections; the Q8_K pair fallback below is - * intentionally retained only for the pre-existing non-GB10 path. */ - const int rc = ds4_mmq_q4_K_dense_pair_vec( - w0, w1, (const float *)x->ptr, - (float *)out0->ptr, (float *)out1->ptr, - (int)out0_dim, (int)out1_dim, (int)n_tok, (int)in_dim, - cuda_decode_stream()); + /* Share one canonical Q8_1 activation across both projections: + * MMVQ covers decode/speculative widths and token-tiled MMQ covers + * prefill. On GB10 a rejected launch fails closed to ds4.c's + * independent projections; the Q8_K fallback below remains the + * established non-GB10 rollback. */ + const int rc = n_tok <= 8u + ? ds4_mmq_q4_K_dense_pair_vec( + w0, w1, (const float *)x->ptr, + (float *)out0->ptr, (float *)out1->ptr, + (int)out0_dim, (int)out1_dim, + (int)n_tok, (int)in_dim, cuda_decode_stream()) + : ds4_mmq_q4_K_dense_pair( + w0, w1, (const float *)x->ptr, + (float *)out0->ptr, (float *)out1->ptr, + (int)out0_dim, (int)out1_dim, + (int)n_tok, (int)in_dim, cuda_decode_stream()); if (rc == 0) return 1; fprintf(stderr, - "ds4: Q4_K MMVQ pair returned %d " + "ds4: Q4_K %s pair returned %d " "(in=%llu out0=%llu out1=%llu n_tok=%llu); falling back\n", - rc, (unsigned long long)in_dim, + n_tok <= 8u ? "MMVQ" : "MMQ", rc, + (unsigned long long)in_dim, (unsigned long long)out0_dim, (unsigned long long)out1_dim, (unsigned long long)n_tok); if (gb10_canonical) return 0; } + /* The Q8_K pair is the established decode/microbatch rollback only. + * For prefill, preserve DS4_CUDA_MMQ=0 and MMQ rejection semantics by + * returning control to the caller's two independent dense projections. */ + if (n_tok > 8u) return 0; + if (n_tok > UINT64_MAX / blocks || n_tok * blocks > UINT64_MAX / sizeof(cuda_block_q8_K)) { return 0; diff --git a/ds4_gpu.h b/ds4_gpu.h index 0ba72b23b..5cde39204 100644 --- a/ds4_gpu.h +++ b/ds4_gpu.h @@ -1005,8 +1005,9 @@ int ds4_gpu_matmul_q4_K_pair_decode_tensor( uint64_t out_dim, const ds4_gpu_tensor *x); -/* Optional small-token dense Q4_K pair. Backends without a beneficial paired - * kernel return zero and request separate fallback matmuls from the graph. */ +/* Optional dense Q4_K pair for decode, microbatch, and prefill. Backends + * without a beneficial paired kernel return zero and request separate + * fallback matmuls from the graph. */ int ds4_gpu_matmul_q4_K_pair_tensor( ds4_gpu_tensor *out0, ds4_gpu_tensor *out1, From b82b4346b9f9694a25926704f3a2ad8f3bf48729 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Sun, 23 Aug 2026 09:56:17 +0200 Subject: [PATCH 071/189] perf(rocm): fuse paired Q4_K projection launches --- rocm/ds4_rocm_q4.cuh | 209 ++++++++++++++++++++++++------ tests/test_rocm_q4_dense_pair.cpp | 175 +++++++++++++++++++------ 2 files changed, 303 insertions(+), 81 deletions(-) diff --git a/rocm/ds4_rocm_q4.cuh b/rocm/ds4_rocm_q4.cuh index de28e792b..cf6639307 100644 --- a/rocm/ds4_rocm_q4.cuh +++ b/rocm/ds4_rocm_q4.cuh @@ -36,6 +36,43 @@ __global__ static void rocm_matmul_q4_K_dense_kernel( if (lane == 0u) out[(uint64_t)tok * out_dim + row] = acc; } +/* Latency-oriented pair variant of the canonical dense kernel. Concatenating + * the two row-tile domains keeps exactly the same per-row dot and reduction + * order while sharing one launch between Q-A and KV. */ +__global__ static void rocm_matmul_q4_K_dense_pair_kernel( + float *out0, + float *out1, + const char *w0, + const char *w1, + const cuda_block_q8_K *xq, + uint64_t row_bytes, + uint32_t xq_blocks, + uint32_t out0_dim, + uint32_t out1_dim, + uint32_t n_tok) { + const uint32_t lane = threadIdx.x & 7u; + const uint32_t row_lane = threadIdx.x >> 3u; + const uint32_t out0_tiles = (out0_dim - 1u) / 32u + 1u; + const bool second = blockIdx.x >= out0_tiles; + const uint32_t row_tile = second ? blockIdx.x - out0_tiles : blockIdx.x; + const uint32_t row = row_tile * 32u + row_lane; + const uint32_t tok = blockIdx.y; + const uint32_t out_dim = second ? out1_dim : out0_dim; + if (tok >= n_tok || row >= out_dim) return; + + float *const out = second ? out1 : out0; + const char *const w_base = second ? w1 : w0; + const cuda_block_q8_K *xqb = xq + (uint64_t)tok * xq_blocks; + const cuda_block_q4_K *wr = reinterpret_cast( + w_base + (uint64_t)row * row_bytes); + float acc = 0.0f; + for (uint32_t b = lane; b < xq_blocks; b += 8u) { + acc += dev_dot_q4_K_q8_K_block(wr + b, xqb + b); + } + acc = quarter_warp_sum_f32(acc, lane); + if (lane == 0u) out[(uint64_t)tok * out_dim + row] = acc; +} + /* One independent activation row and Q4_K matrix per output group. This is * the canonical dense decode walk with only a group grid dimension added. */ __global__ static void rocm_matmul_q4_K_dense_grouped_decode_kernel( @@ -245,6 +282,96 @@ __global__ static void rocm_matmul_q4_K_prefill_tile8_strided_kernel( } } +/* Two independent dense projections over the same activation tile. The + * row-tile ranges are concatenated in grid.x, so Q/KV prefill shares both + * the Q8_K quantization and a single launch without padding the smaller + * projection up to the larger one's row count. Each workgroup still handles + * only one weight matrix: this preserves the standalone TILE8 block walk and + * its accumulation order while removing the second host launch. */ +__global__ static void rocm_matmul_q4_K_prefill_tile8_pair_kernel( + float *out0, + float *out1, + const char *w0, + const char *w1, + const cuda_block_q8_K *xq, + uint64_t row_bytes, + uint32_t xq_blocks, + uint32_t out0_dim, + uint32_t out1_dim, + uint32_t n_tok) { + __shared__ cuda_block_q8_K sxq[ROCM_Q4_PREFILL_TOKEN_TILE] + [ROCM_Q4_PREFILL_KBLOCK_TILE]; + + const uint32_t tid = threadIdx.x; + const uint32_t lane = tid & 7u; + const uint32_t row_lane = tid >> 3u; + const uint32_t out0_tiles = (out0_dim - 1u) / 32u + 1u; + const bool second = blockIdx.x >= out0_tiles; + const uint32_t row_tile = second ? blockIdx.x - out0_tiles : blockIdx.x; + const uint32_t row = row_tile * 32u + row_lane; + const uint32_t tok0 = blockIdx.y * ROCM_Q4_PREFILL_TOKEN_TILE; + const uint32_t out_dim = second ? out1_dim : out0_dim; + float *const out = second ? out1 : out0; + const char *const w_base = second ? w1 : w0; + const uint32_t nt = n_tok - tok0 < ROCM_Q4_PREFILL_TOKEN_TILE + ? n_tok - tok0 : ROCM_Q4_PREFILL_TOKEN_TILE; + const bool row_valid = row < out_dim; + const cuda_block_q4_K *wr = row_valid + ? reinterpret_cast( + w_base + (uint64_t)row * row_bytes) + : NULL; + float acc[ROCM_Q4_PREFILL_TOKEN_TILE] = { + 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, + }; + + for (uint32_t b0 = 0u; b0 < xq_blocks; + b0 += ROCM_Q4_PREFILL_KBLOCK_TILE) { + const uint32_t nb = xq_blocks - b0 < ROCM_Q4_PREFILL_KBLOCK_TILE + ? xq_blocks - b0 + : ROCM_Q4_PREFILL_KBLOCK_TILE; + const uint32_t tile_words = nt * ROCM_Q4_PREFILL_KBLOCK_TILE * + ROCM_Q4_Q8K_WORDS; + uint32_t *const sxq_words = reinterpret_cast(sxq); + for (uint32_t i = tid; i < tile_words; i += blockDim.x) { + const uint32_t block_slot = i / ROCM_Q4_Q8K_WORDS; + const uint32_t word = i - block_slot * ROCM_Q4_Q8K_WORDS; + const uint32_t p = block_slot >> 3u; + const uint32_t bb = block_slot & 7u; + if (bb < nb) { + const uint64_t src_block = + (uint64_t)(tok0 + p) * xq_blocks + b0 + bb; + const uint32_t *const src_words = + reinterpret_cast(xq + src_block); + sxq_words[i] = src_words[word]; + } + } + __syncthreads(); + + if (row_valid && lane < nb) { + rocm_dot_q4_K_q8_K_block8_reuse_weights( + wr + b0 + lane, + sxq[0] + lane, sxq[1] + lane, + sxq[2] + lane, sxq[3] + lane, + sxq[4] + lane, sxq[5] + lane, + sxq[6] + lane, sxq[7] + lane, + nt, acc); + } + __syncthreads(); + } + + if (row_valid) { + #pragma unroll + for (uint32_t p = 0u; p < ROCM_Q4_PREFILL_TOKEN_TILE; p++) { + if (p < nt) { + const float v = quarter_warp_sum_f32(acc[p], lane); + if (lane == 0u) { + out[(uint64_t)(tok0 + p) * out_dim + row] = v; + } + } + } + } +} + static int rocm_q4_K_dense_validate( const ds4_gpu_tensor *out, const void *model_map, @@ -294,6 +421,15 @@ static cuda_block_q8_K *rocm_q4_K_prequant_alloc( return reinterpret_cast(cuda_tmp_alloc(bytes, what)); } +static int rocm_q4_K_byte_ranges_overlap( + const void *ptr0, uint64_t bytes0, + const void *ptr1, uint64_t bytes1) { + const uintptr_t p0 = reinterpret_cast(ptr0); + const uintptr_t p1 = reinterpret_cast(ptr1); + return p0 <= p1 ? (uint64_t)(p1 - p0) < bytes0 + : (uint64_t)(p0 - p1) < bytes1; +} + static int rocm_q4_K_dense_pair_requested(void) { return getenv("DS4_ROCM_ENABLE_Q4_DENSE_PAIR") != NULL && getenv("DS4_ROCM_DISABLE_Q4_DENSE_PAIR") == NULL; @@ -477,8 +613,8 @@ extern "C" int ds4_gpu_matmul_q4_K_pair_tensor( } /* Decode keeps its original, separately gated pair path. Prefill uses - * the common tile8 gate and shares one canonical Q8_K quantization across - * the two projections, then issues one tiled launch per weight matrix. */ + * the common tile8 gate and shares one canonical Q8_K quantization and + * one tiled launch across the two projections. */ const int decode_pair = n_tok <= 8u && rocm_q4_K_dense_pair_requested(); if ((!prefill_pair && !decode_pair) || @@ -499,6 +635,14 @@ extern "C" int ds4_gpu_matmul_q4_K_pair_tensor( blocks0 != blocks1 || row_bytes0 != row_bytes1) { return 0; } + uint64_t out0_bytes = 0; + uint64_t out1_bytes = 0; + if (!cuda_u64_mul3_checked(n_tok, out0_dim, sizeof(float), &out0_bytes) || + !cuda_u64_mul3_checked(n_tok, out1_dim, sizeof(float), &out1_bytes) || + rocm_q4_K_byte_ranges_overlap(out0->ptr, out0_bytes, + out1->ptr, out1_bytes)) { + return 0; + } const char *w0 = cuda_model_range_ptr(model_map, weight0_offset, weight0_bytes, "q4_K dense pair0"); @@ -518,50 +662,33 @@ extern "C" int ds4_gpu_matmul_q4_K_pair_tensor( } if (prefill_pair) { - const dim3 grid0((unsigned)((out0_dim - 1u) / 32u + 1u), - (unsigned)((n_tok - 1u) / - ROCM_Q4_PREFILL_TOKEN_TILE + 1u), - 1u); - rocm_matmul_q4_K_prefill_tile8_strided_kernel<<>>( - reinterpret_cast(out0->ptr), w0, xq, row_bytes0, - (uint32_t)blocks0, (uint32_t)out0_dim, (uint32_t)n_tok, - blocks0, out0_dim); - if (!cuda_ok(cudaGetLastError(), - "q4_K dense prefill pair0 tile8 launch")) { - return 0; - } - - const dim3 grid1((unsigned)((out1_dim - 1u) / 32u + 1u), - (unsigned)((n_tok - 1u) / - ROCM_Q4_PREFILL_TOKEN_TILE + 1u), - 1u); - rocm_matmul_q4_K_prefill_tile8_strided_kernel<<>>( - reinterpret_cast(out1->ptr), w1, xq, row_bytes1, - (uint32_t)blocks1, (uint32_t)out1_dim, (uint32_t)n_tok, - blocks1, out1_dim); + const uint64_t out0_tiles = (out0_dim - 1u) / 32u + 1u; + const uint64_t out1_tiles = (out1_dim - 1u) / 32u + 1u; + const dim3 grid((unsigned)(out0_tiles + out1_tiles), + (unsigned)((n_tok - 1u) / + ROCM_Q4_PREFILL_TOKEN_TILE + 1u), + 1u); + rocm_matmul_q4_K_prefill_tile8_pair_kernel<<>>( + reinterpret_cast(out0->ptr), + reinterpret_cast(out1->ptr), w0, w1, xq, + row_bytes0, (uint32_t)blocks0, (uint32_t)out0_dim, + (uint32_t)out1_dim, (uint32_t)n_tok); const int ok = cuda_ok(cudaGetLastError(), - "q4_K dense prefill pair1 tile8 launch"); + "q4_K dense prefill pair tile8 launch"); if (ok) rocm_q4_K_prefill_tile8_note(0u, 1u, 0u, n_tok); return ok; } - // Use the exact standalone kernel twice. This preserves its block walk and - // reduction path while still eliminating the second Q8_K quantization. - const dim3 grid0((unsigned)((out0_dim - 1u) / 32u + 1u), - (unsigned)n_tok, 1u); - rocm_matmul_q4_K_dense_kernel<<>>( - reinterpret_cast(out0->ptr), w0, xq, row_bytes0, - (uint32_t)blocks0, (uint32_t)out0_dim, (uint32_t)n_tok); - if (!cuda_ok(cudaGetLastError(), "q4_K dense pair0 matmul launch")) { - return 0; - } - - const dim3 grid1((unsigned)((out1_dim - 1u) / 32u + 1u), - (unsigned)n_tok, 1u); - rocm_matmul_q4_K_dense_kernel<<>>( - reinterpret_cast(out1->ptr), w1, xq, row_bytes1, - (uint32_t)blocks1, (uint32_t)out1_dim, (uint32_t)n_tok); - return cuda_ok(cudaGetLastError(), "q4_K dense pair1 matmul launch"); + const uint64_t out0_tiles = (out0_dim - 1u) / 32u + 1u; + const uint64_t out1_tiles = (out1_dim - 1u) / 32u + 1u; + const dim3 grid((unsigned)(out0_tiles + out1_tiles), + (unsigned)n_tok, 1u); + rocm_matmul_q4_K_dense_pair_kernel<<>>( + reinterpret_cast(out0->ptr), + reinterpret_cast(out1->ptr), w0, w1, xq, + row_bytes0, (uint32_t)blocks0, (uint32_t)out0_dim, + (uint32_t)out1_dim, (uint32_t)n_tok); + return cuda_ok(cudaGetLastError(), "q4_K dense pair matmul launch"); } extern "C" int ds4_gpu_attention_output_low_q4_K_slice_tensor( diff --git a/tests/test_rocm_q4_dense_pair.cpp b/tests/test_rocm_q4_dense_pair.cpp index bbc0df1e6..bd4f58128 100644 --- a/tests/test_rocm_q4_dense_pair.cpp +++ b/tests/test_rocm_q4_dense_pair.cpp @@ -115,6 +115,7 @@ struct aligned_model { uint64_t attn_b_offset = 0; uint64_t attn_b_q8_offset = 0; uint64_t tail_k1024_offset = 0; + uint64_t tail_k1024_pair_offset = 0; ~aligned_model() { std::free(data); } @@ -279,7 +280,8 @@ bool make_model(aligned_model *model) { (uint64_t)kAttnOutDim * attn_b_row_bytes; const uint64_t tail_row_bytes = (kTailK / kQkK) * sizeof(block_q4_K_test); - const uint64_t tail_bytes = (uint64_t)kM0 * tail_row_bytes; + const uint64_t tail0_bytes = (uint64_t)kM0 * tail_row_bytes; + const uint64_t tail1_bytes = (uint64_t)kM1 * tail_row_bytes; const uint64_t attn_b_q8_row_bytes = (kAttnLowDim / 32u) * sizeof(block_q8_0_test); const uint64_t attn_b_q8_bytes = @@ -294,8 +296,10 @@ bool make_model(aligned_model *model) { model->decode_attn_a_offset + decode_attn_a_bytes, page); model->tail_k1024_offset = round_up( model->attn_b_offset + attn_b_bytes, page); + model->tail_k1024_pair_offset = round_up( + model->tail_k1024_offset + tail0_bytes, page); model->attn_b_q8_offset = round_up( - model->tail_k1024_offset + tail_bytes, page); + model->tail_k1024_pair_offset + tail1_bytes, page); model->size = round_up( model->attn_b_q8_offset + attn_b_q8_bytes, page); void *storage = nullptr; @@ -326,6 +330,9 @@ bool make_model(aligned_model *model) { fill_q4_rows(reinterpret_cast( model->data + model->tail_k1024_offset), kM0, kTailK, 0x13198a2eu); + fill_q4_rows(reinterpret_cast( + model->data + model->tail_k1024_pair_offset), + kM1, kTailK, 0xa4093822u); fill_q8_0_rows(reinterpret_cast( model->data + model->attn_b_q8_offset), kAttnOutDim, kAttnLowDim, 0x03707344u); @@ -551,34 +558,60 @@ bool run_dense_case(const aligned_model &model, uint32_t n_tokens, return close_to_cpu(got, cpu, label); } +bool output_guard_unchanged(const std::vector &values, + const std::vector &sentinel, + size_t logical_count, + const char *label); + bool run_pair_case(const aligned_model &model, uint32_t n_tokens, - const char *label) { + const char *label, bool reverse_outputs = false, + uint32_t in_dim = kK) { + const uint32_t out0_dim = reverse_outputs ? kM1 : kM0; + const uint32_t out1_dim = reverse_outputs ? kM0 : kM1; + const uint64_t base0_offset = in_dim == kTailK + ? model.tail_k1024_offset + : model.weight0_offset; + const uint64_t base1_offset = in_dim == kTailK + ? model.tail_k1024_pair_offset + : model.weight1_offset; + const uint64_t weight0_offset = reverse_outputs + ? base1_offset : base0_offset; + const uint64_t weight1_offset = reverse_outputs + ? base0_offset : base1_offset; + const size_t count0 = (size_t)n_tokens * out0_dim; + const size_t count1 = (size_t)n_tokens * out1_dim; + const std::vector sentinel0 = + sentinel_values(count0 + kOutputGuardFloats); + const std::vector sentinel1 = + sentinel_values(count1 + kOutputGuardFloats); std::vector x; - fill_activation(&x, n_tokens); + fill_activation(&x, n_tokens, in_dim); tensor_owner x_gpu(x.size() * sizeof(float)); - tensor_owner dense0((uint64_t)n_tokens * kM0 * sizeof(float)); - tensor_owner dense1((uint64_t)n_tokens * kM1 * sizeof(float)); - tensor_owner pair0((uint64_t)n_tokens * kM0 * sizeof(float)); - tensor_owner pair1((uint64_t)n_tokens * kM1 * sizeof(float)); + tensor_owner dense0(count0 * sizeof(float)); + tensor_owner dense1(count1 * sizeof(float)); + tensor_owner pair0(sentinel0.size() * sizeof(float)); + tensor_owner pair1(sentinel1.size() * sizeof(float)); if (!x_gpu.ptr || !dense0.ptr || !dense1.ptr || !pair0.ptr || !pair1.ptr || - !write_tensor(x_gpu.ptr, x)) { + !write_tensor(x_gpu.ptr, x) || + !write_tensor(pair0.ptr, sentinel0) || + !write_tensor(pair1.ptr, sentinel1)) { std::fprintf(stderr, "%s: tensor allocation/write FAIL\n", label); return false; } const int dense_rc0 = ds4_gpu_matmul_quant_tensor( - dense0.ptr, model.data, model.size, model.weight0_offset, kQ4Type, - kK, kM0, x_gpu.ptr, n_tokens); + dense0.ptr, model.data, model.size, weight0_offset, kQ4Type, + in_dim, out0_dim, x_gpu.ptr, n_tokens); const int dense_rc1 = ds4_gpu_matmul_quant_tensor( - dense1.ptr, model.data, model.size, model.weight1_offset, kQ4Type, - kK, kM1, x_gpu.ptr, n_tokens); + dense1.ptr, model.data, model.size, weight1_offset, kQ4Type, + in_dim, out1_dim, x_gpu.ptr, n_tokens); const int pair_rc = ds4_gpu_matmul_q4_K_pair_tensor( pair0.ptr, pair1.ptr, model.data, model.size, - model.weight0_offset, model.weight1_offset, - kK, kM0, kM1, x_gpu.ptr, n_tokens); - std::vector dense0_host((uint64_t)n_tokens * kM0); - std::vector dense1_host((uint64_t)n_tokens * kM1); - std::vector pair0_host(dense0_host.size()); - std::vector pair1_host(dense1_host.size()); + weight0_offset, weight1_offset, + in_dim, out0_dim, out1_dim, x_gpu.ptr, n_tokens); + std::vector dense0_host(count0); + std::vector dense1_host(count1); + std::vector pair0_host(sentinel0.size()); + std::vector pair1_host(sentinel1.size()); if (dense_rc0 == 0 || dense_rc1 == 0 || pair_rc == 0 || !read_tensor(dense0.ptr, &dense0_host) || !read_tensor(dense1.ptr, &dense1_host) || @@ -590,11 +623,17 @@ bool run_pair_case(const aligned_model &model, uint32_t n_tokens, return false; } const std::vector cpu0 = dense_reference( - model.data + model.weight0_offset, x, kM0, n_tokens); + model.data + weight0_offset, x, out0_dim, n_tokens, in_dim); const std::vector cpu1 = dense_reference( - model.data + model.weight1_offset, x, kM1, n_tokens); + model.data + weight1_offset, x, out1_dim, n_tokens, in_dim); bool ok = close_to_cpu(dense0_host, cpu0, "pair control dense0 vs CPU"); ok = close_to_cpu(dense1_host, cpu1, "pair control dense1 vs CPU") && ok; + ok = output_guard_unchanged(pair0_host, sentinel0, count0, + "pair0 output canary") && ok; + ok = output_guard_unchanged(pair1_host, sentinel1, count1, + "pair1 output canary") && ok; + pair0_host.resize(count0); + pair1_host.resize(count1); ok = bitwise_equal(pair0_host, dense0_host, "pair0 vs standalone dense0") && ok; ok = bitwise_equal(pair1_host, dense1_host, "pair1 vs standalone dense1") && ok; std::fprintf(stderr, "%s: %s\n", label, ok ? "PASS" : "FAIL"); @@ -776,12 +815,24 @@ bool run_prefill_gate_guards(const aligned_model &model) { return ok; } -bool run_prefill_pair_case(const aligned_model &model) { - constexpr uint32_t n_tokens = 128u; +bool run_prefill_pair_case(const aligned_model &model, uint32_t n_tokens, + bool reverse_outputs, uint32_t in_dim = kK) { + const uint32_t out0_dim = reverse_outputs ? kM1 : kM0; + const uint32_t out1_dim = reverse_outputs ? kM0 : kM1; + const uint64_t base0_offset = in_dim == kTailK + ? model.tail_k1024_offset + : model.weight0_offset; + const uint64_t base1_offset = in_dim == kTailK + ? model.tail_k1024_pair_offset + : model.weight1_offset; + const uint64_t weight0_offset = reverse_outputs + ? base1_offset : base0_offset; + const uint64_t weight1_offset = reverse_outputs + ? base0_offset : base1_offset; std::vector x; - fill_activation(&x, n_tokens); - const size_t count0 = (size_t)n_tokens * kM0; - const size_t count1 = (size_t)n_tokens * kM1; + fill_activation(&x, n_tokens, in_dim); + const size_t count0 = (size_t)n_tokens * out0_dim; + const size_t count1 = (size_t)n_tokens * out1_dim; const std::vector sentinel0 = sentinel_values(count0 + kOutputGuardFloats); const std::vector sentinel1 = @@ -798,7 +849,8 @@ bool run_prefill_pair_case(const aligned_model &model) { !write_tensor(legacy1.ptr, sentinel1) || !write_tensor(pair0.ptr, sentinel0) || !write_tensor(pair1.ptr, sentinel1)) { - std::fprintf(stderr, "prefill pair n_tok=128: setup FAIL\n"); + std::fprintf(stderr, "prefill pair n_tok=%u reverse=%d: setup FAIL\n", + n_tokens, reverse_outputs ? 1 : 0); return false; } @@ -812,11 +864,11 @@ bool run_prefill_pair_case(const aligned_model &model) { (void)setenv(kPrefillDisable, "1", 1); (void)unsetenv(kPrefillRequire); const int legacy_rc0 = ds4_gpu_matmul_quant_tensor( - legacy0.ptr, model.data, model.size, model.weight0_offset, kQ4Type, - kK, kM0, x_gpu.ptr, n_tokens); + legacy0.ptr, model.data, model.size, weight0_offset, kQ4Type, + in_dim, out0_dim, x_gpu.ptr, n_tokens); const int legacy_rc1 = ds4_gpu_matmul_quant_tensor( - legacy1.ptr, model.data, model.size, model.weight1_offset, kQ4Type, - kK, kM1, x_gpu.ptr, n_tokens); + legacy1.ptr, model.data, model.size, weight1_offset, kQ4Type, + in_dim, out1_dim, x_gpu.ptr, n_tokens); // The prefill pair is a distinct path: it must not depend on the legacy // decode-pair opt-in, whose <=8-token behavior is tested separately. @@ -827,8 +879,8 @@ bool run_prefill_pair_case(const aligned_model &model) { (void)unsetenv("DS4_ROCM_DISABLE_Q4_DENSE_PAIR"); const int pair_rc = ds4_gpu_matmul_q4_K_pair_tensor( pair0.ptr, pair1.ptr, model.data, model.size, - model.weight0_offset, model.weight1_offset, - kK, kM0, kM1, x_gpu.ptr, n_tokens); + weight0_offset, weight1_offset, + in_dim, out0_dim, out1_dim, x_gpu.ptr, n_tokens); std::vector legacy0_host(sentinel0.size()); std::vector legacy1_host(sentinel1.size()); @@ -840,8 +892,9 @@ bool run_prefill_pair_case(const aligned_model &model) { !read_tensor(pair0.ptr, &pair0_host) || !read_tensor(pair1.ptr, &pair1_host)) { std::fprintf(stderr, - "prefill pair n_tok=128: dispatch/read legacy=(%d,%d) " - "pair=%d FAIL\n", + "prefill pair n_tok=%u reverse=%d: dispatch/read " + "legacy=(%d,%d) pair=%d FAIL\n", + n_tokens, reverse_outputs ? 1 : 0, legacy_rc0, legacy_rc1, pair_rc); return false; } @@ -860,8 +913,9 @@ bool run_prefill_pair_case(const aligned_model &model) { ok = bitwise_equal(pair1_host, legacy1_host, "prefill pair1 vs forced legacy dense1") && ok; std::fprintf(stderr, - "prefill pair K=4096 M=(65,33) n_tok=128 " + "prefill pair K=%u M=(%u,%u) n_tok=%u " "legacy=(%d,%d) pair=%d %s\n", + in_dim, out0_dim, out1_dim, n_tokens, legacy_rc0, legacy_rc1, pair_rc, ok ? "PASS" : "FAIL"); return ok; } @@ -1343,6 +1397,12 @@ bool run_pair_guards(const aligned_model &model) { std::fprintf(stderr, "pair guards: setup FAIL\n"); return false; } + env_snapshot prefill_enable(kPrefillEnable); + env_snapshot prefill_disable(kPrefillDisable); + env_snapshot prefill_require(kPrefillRequire); + (void)unsetenv(kPrefillEnable); + (void)setenv(kPrefillDisable, "1", 1); + (void)unsetenv(kPrefillRequire); const int rc = ds4_gpu_matmul_q4_K_pair_tensor( out0.ptr, out1.ptr, model.data, model.size, model.weight0_offset, model.weight1_offset, @@ -1355,6 +1415,32 @@ bool run_pair_guards(const aligned_model &model) { out0.ptr, sentinel0, "pair n_tok=9 preserves out0") && ok; ok = unchanged_after_rejected_call( out1.ptr, sentinel1, "pair n_tok=9 preserves out1") && ok; + + const size_t shared_count = (size_t)kM0 + kM1; + const std::vector shared_sentinel = sentinel_values(shared_count); + tensor_owner shared(shared_count * sizeof(float)); + tensor_owner overlap0(ds4_gpu_tensor_view( + shared.ptr, 0u, (uint64_t)kM0 * sizeof(float))); + tensor_owner overlap1(ds4_gpu_tensor_view( + shared.ptr, (uint64_t)(kM0 - 1u) * sizeof(float), + (uint64_t)kM1 * sizeof(float))); + if (!shared.ptr || !overlap0.ptr || !overlap1.ptr || + !write_tensor(shared.ptr, shared_sentinel)) { + std::fprintf(stderr, "pair overlap guard: setup FAIL\n"); + return false; + } + const int overlap_rc = ds4_gpu_matmul_q4_K_pair_tensor( + overlap0.ptr, overlap1.ptr, model.data, model.size, + model.weight0_offset, model.weight1_offset, + kK, kM0, kM1, x_gpu.ptr, 1u); + ok = overlap_rc == 0 && unchanged_after_rejected_call( + shared.ptr, shared_sentinel, + "pair partial-overlap guard preserves storage") && ok; + if (overlap_rc != 0) { + std::fprintf(stderr, + "pair partial-overlap guard: expected rc=0 got=%d FAIL\n", + overlap_rc); + } return ok; } @@ -1523,11 +1609,14 @@ int main(int argc, char **argv) { (void)setenv("DS4_ROCM_ENABLE_Q4_DENSE_PAIR", "1", 1); (void)unsetenv("DS4_ROCM_DISABLE_Q4_DENSE_PAIR"); const bool pair1_ok = run_pair_case(model, 1u, "pair n_tok=1"); - const bool pair3_ok = run_pair_case(model, 3u, "pair n_tok=3"); + const bool pair1_tail_ok = run_pair_case( + model, 1u, "pair K=1024 n_tok=1", false, kTailK); + const bool pair3_ok = run_pair_case( + model, 3u, "pair n_tok=3 reverse M=(33,65)", true); const bool pair8_ok = run_pair_case(model, 8u, "pair n_tok=8"); const bool pair_guard_ok = run_pair_guards(model); const bool pair_opt_in_ok = run_pair_opt_in_guards(model); - ok = pair1_ok && pair3_ok && pair8_ok && pair_guard_ok && + ok = pair1_ok && pair1_tail_ok && pair3_ok && pair8_ok && pair_guard_ok && pair_opt_in_ok && ok; } if (model_ready && run_grouped_decode) { @@ -1561,7 +1650,12 @@ int main(int argc, char **argv) { const bool prefill_single128_ok = run_prefill_parity_case( model, 128u, model.attn_b_offset, kAttnOutDim, true, "prefill K=256 M=65 n_tok=128 (K-tail nb=1)", kAttnLowDim); - const bool prefill_pair_ok = run_prefill_pair_case(model); + const bool prefill_pair9_ok = + run_prefill_pair_case(model, 9u, false, kTailK); + const bool prefill_pair30_reverse_ok = + run_prefill_pair_case(model, 30u, true); + const bool prefill_pair128_ok = + run_prefill_pair_case(model, 128u, false); const bool attention9_ok = run_attention_prefill_case( model, 9u, "attention prefill groups=8 K=4096 rank=32 M=65 n_tok=9"); @@ -1586,7 +1680,8 @@ int main(int argc, char **argv) { ok = prefill9_ok && prefill30_ok && prefill128_ok && prefill_tail9_ok && prefill_tail128_ok && prefill_single9_ok && - prefill_single128_ok && prefill_pair_ok && attention9_ok && + prefill_single128_ok && prefill_pair9_ok && + prefill_pair30_reverse_ok && prefill_pair128_ok && attention9_ok && attention30_ok && attention128_ok && attention_q8_9_ok && attention_q8_30_ok && gate_ok && ok; if (run_prefill_long) { From dd9bfae063d86cea76ce76b46b7ac16c8fadf6a7 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Sun, 23 Aug 2026 14:42:24 +0200 Subject: [PATCH 072/189] docs: analyze Q4_K complexity and optimization opportunities --- ANALISI_COMPLESSITA_PR621.md | 339 +++++++++++++++++++++++++++++++++++ 1 file changed, 339 insertions(+) create mode 100644 ANALISI_COMPLESSITA_PR621.md diff --git a/ANALISI_COMPLESSITA_PR621.md b/ANALISI_COMPLESSITA_PR621.md new file mode 100644 index 000000000..54ae78b22 --- /dev/null +++ b/ANALISI_COMPLESSITA_PR621.md @@ -0,0 +1,339 @@ +# Analisi di complessità e ristrutturazioni prestazionali — PR #621 + +Data: 23 agosto 2026 + +Base analizzata: `84cc882` + +HEAD dello snapshot originario: `167eb107f0f15aca922335718bfc7c69952aa815` + +## Stato delle implementazioni successive + +L'analisi delle funzioni e dei costi è stata redatta sullo snapshot indicato +sopra. Le ristrutturazioni seguenti sono state successivamente implementate +nel worktree e non sono più proposte future: + +- CPU: dot Q4_K su coppie di token con decode dei pesi condiviso e + quantizzazione Q8_K batch parallela (`6e0527aa`); +- CUDA: coppia MMQ Q4_K per il prefill con una sola quantizzazione Q8_1 + condivisa (`b4a106ec`); +- ROCm: singolo launch pair per decode e TILE8 prefill (`a02008f6`). + +Le sezioni seguenti mantengono la fotografia completa dell'analisi originaria, +ma annotano questi punti come completati dove compaiono tra le priorità. + +## Perimetro + +La PR contiene 43.301 inserimenti, 5.161 rimozioni e 49 file modificati. Non tutto il diff appartiene al supporto AProjQ4: include DSpark, SSD streaming, tooling, test, documentazione e ottimizzazioni di altri quanti. Questa analisi copre **tutte le funzioni produttive direttamente coinvolte nel percorso Q4_K delle proiezioni dense di attenzione**, incluse le funzioni di dispatch e gli helper che ne determinano il costo. I test, il generatore GGUF e le funzioni puramente diagnostiche sono classificati a parte: il loro costo non incide sull’inferenza. + +La complessità asintotica non cambia tra implementazioni corrette di una proiezione densa: il lavoro matematico minimo resta proporzionale agli elementi della matrice. Le ottimizzazioni utili riducono soprattutto traffico di memoria, quantizzazioni ripetute, lanci kernel e materiale intermedio. + +## Simboli usati + +| Simbolo | Significato | +|---|---| +| `L` | numero di layer | +| `T` | token nel batch/prefill chunk; `T=1` nel decode ordinario | +| `K` | dimensione di input della proiezione | +| `M` | dimensione di output della proiezione | +| `M0`,`M1` | output di due proiezioni accoppiate | +| `G` | gruppi/head group elaborati | +| `R` | rank per gruppo | +| `Q=256` | elementi per superblocco Q4_K/Q8_K | +| `B=K/Q` | superblocchi per riga | +| `S` | esperti selezionati per token | + +Per una matrice Q4_K `[M,K]`: + +- operazioni: `Θ(T·M·K)`; +- lettura pesi senza riuso tra token: `Θ(T·M·K)` elementi logici; +- lettura pesi con tiling su `τ` token: circa `Θ((T/τ)·M·K)` blocchi fisici, a parità di lavoro aritmetico; +- quantizzazione attivazioni F32→Q8_K: `Θ(T·K)` tempo e `Θ(T·K)` memoria temporanea, con costante compressa dal formato a blocchi. + +## 1. CPU reference path (`ds4.c`) + +Il backend CPU è dichiaratamente reference/debug; ottimizzarlo non accelera Metal/CUDA/ROCm. È comunque utile come baseline algoritmica. + +| Funzione | Tempo | Memoria extra | Osservazioni | +|---|---:|---:|---| +| `q4_k_get_scale_min` | `Θ(1)` | `Θ(1)` | Decodifica scale/min di un sottoblocco. Inlinabile. | +| `ds4_vec_dot_q4_K_q8_K` | `Θ(K)` | `Θ(1)` | Un dot di una riga. Due passaggi per blocco: min correction e prodotto Q4×Q8. | +| `ds4_vec_dot_q4_K_f32` | `Θ(K)` | `Θ(1)` | Fallback diretto F32; evita quantizzazione ma moltiplica per valori dequantizzati. | +| `dense_q4_K_expect` | `Θ(1)` | `Θ(1)` | Validazione forma/tipo. | +| `matvec_q4_K_dense_worker` | `Θ((r1-r0)·K)` | `Θ(1)` | Una riga per iterazione; pesi letti una volta per chiamata. | +| `matvec_q4_K_prequant` | `Θ(M·K)` | `Θ(1)` locale | Dispatch parallelo usando un `xq` già pronto. | +| `matvec_q4_K` | `Θ(K + M·K)` | `Θ(K)` | Alloca e quantizza l’input a ogni chiamata. | +| `matvec_q4_K_decode_scratch` | `Θ(K + M·K)` | `Θ(1)` per chiamata | Riusa lo scratch; elimina malloc/free, non la quantizzazione. | +| `matvec_q4_K_grouped_expect` | `Θ(1)` | `Θ(1)` | Validazione. | +| `matvec_q4_K_grouped_worker` | `Θ((r1-r0)·K)` | `Θ(1)` | Totale `Θ(G·R·K)`. | +| `matvec_q4_K_grouped_rows_prequant` | `Θ(G·R·K)` | `Θ(1)` locale | Input Q8_K già pronto. | +| `matvec_q4_K_grouped_rows` | `Θ(G·K + G·R·K)` | `Θ(G·K)` | Quantizza ogni gruppo e poi proietta. | +| `matvec_q4_K_grouped_rows_decode_scratch` | `Θ(G·K + G·R·K)` | `Θ(1)` per chiamata | Scratch persistente. | +| `matmul_q4_K_batch_worker` | `Θ((r1-r0)·T·K)` | `Θ(1)` | Elabora coppie di token condividendo decode Q4_K; coda singola per `T` dispari. | +| `matmul_q4_K_batch` | `Θ(T·K + T·M·K)` | `Θ(T·K)` | Prefill CPU; quantizzazione per riga parallela oltre la soglia di lavoro. | +| `matmul_q4_K_grouped_batch_worker` | `Θ((r1-r0)·T·K)` | `Θ(1)` | Totale `Θ(T·G·R·K)`, con lo stesso microkernel a due token. | +| `matmul_q4_K_grouped_batch` | `Θ(T·G·K + T·G·R·K)` | `Θ(T·G·K)` | Analogo grouped. | +| `matvec_any`, `matvec_any_decode_scratch` | `Θ(K+M·K)` sul ramo Q4 | dipende dal ramo | Dispatcher `Θ(1)` più callee. | +| `layer_q_projection_*`, `layer_kv_projection_*` | costo della proiezione | nessuna propria rilevante | Wrapper; sul ramo Q4 ereditano `Θ(M·K)`. | +| `layer_grouped_out_*` | `Θ(G·R·K)` o `Θ(T·G·R·K)` | come callee | Wrapper output attention. | + +### Ristrutturazioni CPU + +1. **Un solo passaggio NEON nel dot Q4×Q8.** L’implementazione ARM costruisce `q4_u[32]` sullo stack e percorre ogni superblocco due volte. Decodificare min/scale e nibble direttamente in registri NEON elimina lo scratch locale e una parte dei load/store. Complessità invariata `Θ(K)`, costante più bassa. +2. **Tiling 2D per `matmul_q4_K_batch_worker`.** Il loop `row → token` rilegge l’attivazione Q8 per ogni riga e non esprime un microkernel. Un blocco `MR×NT` mantiene più accumulatori e riusa blocchi di peso/attivazione: ancora `Θ(T·M·K)`, ma migliore cache e SIMD. +3. **Quantizzazione fusionata con RMSNorm.** Se l’output F32 della norm serve solo alla proiezione Q4, produrre contemporaneamente Q8_K elimina una lettura e una scrittura F32: risparmio `Θ(T·K)` di traffico. + +## 2. CUDA — primitive e kernel Q4_K (`ds4_cuda.cu`) + +### Primitive per superblocco + +| Funzione | Tempo per chiamata | Memoria | Nota | +|---|---:|---:|---| +| `dev_q4_K_get_scale_min` | `Θ(1)` | registri | Helper inlinabile. | +| `dev_dot_q4_K_q8_K_block` | `Θ(Q)` | registri | Un superblocco; costante perché `Q=256`, ma linearizzato come `Θ(Q)`. | +| `dev_dot_q4_K_q8_K_block_vec` | `Θ(Q)` | più registri | Nove load da 16 B; riduce istruzioni di load e migliora coalescenza. | +| `dev_dot_q4_K_q8_K_block8` | `Θ(8·Q)` | registri elevati | Riusa un blocco peso su fino a 8 token. È il nucleo concettuale del prefill tiled. | +| `quarter_warp_sum_f32` | `Θ(log 8)` = `Θ(1)` | registri | Riduzione intra-warp. | + +### Dense projection e wrapper + +| Funzione | Tempo | Memoria extra | Impatto | +|---|---:|---:|---| +| `matmul_q4_K_dense_kernel` | `Θ(T·M·K)` globale | `Θ(1)` per thread | Kernel bandwidth-oriented; ogni token rilegge tutti i pesi. Buono per `T=1`, pessimo per prefill grande. | +| `matmul_q4_K_dense_pair_kernel` | `Θ(T·(M0+M1)·K)` | due accumulatori | Condivide `xq`, non i pesi. Riduce un lancio/quantizzazione, non il termine dominante. | +| `cuda_matmul_q4_K_tensor` | `Θ(T·K + T·M·K)` | `Θ(T·K)` | Usa MMQ tiled quando accettato; fallback al kernel sopra. | +| `cuda_matmul_q4_K_pair_tensor_impl` | `Θ(T·K + T·(M0+M1)·K)` | `Θ(T·K)` | Quantizza una volta per due matrici. | +| `ds4_gpu_matmul_q4_K_pair_tensor` | come impl | `Θ(1)` propria | Wrapper. | +| `matmul_q4_K_kslice_kernel` | `Θ(M·Kslice)` | `Θ(1)` | Proiezione su slice K; decode/TP. | +| `cuda_matmul_q4_K_kslice_tensor` | `Θ(Kslice + M·Kslice)` | `Θ(Kslice)` | Quantizza la slice. | +| `ds4_gpu_attention_output_q4_K_batch_tensor` | `Θ(T·G·R·K + T·M·G·R)` circa | temporanei batch | Proietta A per gruppo e B globale; prova batch MMQ, poi fallback. | +| `ds4_gpu_attention_output_low_q4_K_slice_tensor` | `Θ(Gsel·R·K)` | `Θ(Gsel·K)` | Decode grouped attention-A. | + +### HC expand e fusioni + +| Funzione | Tempo | Memoria/traffico | Nota | +|---|---:|---:|---| +| `matmul_q4_K_hc_expand4_kernel` | `Θ(M·K + 4M)` | evita una rilettura di `block_out` | Fonde proiezione B e HC postprocess; ottima direzione per decode. | +| `q4_K_hc_expand4_rows_kernel` | `Θ(4M)` | legge/scrive intermedi | Esegue solo epilogo dopo MMQ canonico. | +| `q4_K_attn_hc_bitwise_compare_kernel` | `Θ(M)` | contatori diagnostici | Non deve essere nel release path. | +| `cuda_q4_K_hc_expand_canonical` | `Θ(M·K+4M)` | materiale `block_out` | MMQ + epilogo separato; due lanci. | +| `cuda_q4_K_hc_expand_q8k_launch` | `Θ(K+M·K+4M)` | Q8_K scratch | Fallback/fused. | +| `ds4_gpu_matmul_q4_K_hc_expand_available` | `Θ(1)` | `Θ(1)` | Query; dovrebbe essere precalcolata nel piano layer. | +| `ds4_gpu_matmul_q4_K_hc_expand_tensor` | `Θ(K+M·K+4M)` | scratch/oracle se attivo | Dispatcher con variabili diagnostiche. | + +## 3. CUDA MMQ (`cuda/mmq/ds4_mmq.cu`) + +Le funzioni MMQ sono il percorso che ha eliminato la regressione iniziale di prefill (~16×). La distinzione fondamentale è: + +- decode/vector: `Θ(M·K)`, ottimizzato per latenza e bandwidth; +- prefill/matrix: `Θ(T·M·K)`, con pesi riusati su tile di token e tensor-core/microkernel quando possibile. + +| Funzione | Complessità | Ruolo | +|---|---:|---| +| `ds4_mmq_dense_impl` | `Θ(T·M·K)` | Core generico tiled. | +| `ds4_mmq_q4_K_dense` | `Θ(T·M·K)` | Wrapper Q4_K prefill/MMQ. | +| `ds4_mmq_dense_vec_impl` | `Θ(M·K)` | Core decode vettoriale. | +| `ds4_mmq_q4_K_dense_vec` | `Θ(M·K)` | Wrapper decode Q4_K. | +| `ds4_mmq_dense_pair_vec_impl` | `Θ((M0+M1)·K)` | Due proiezioni, Q8 input condiviso. | +| `ds4_mmq_q4_K_dense_pair_vec` | stesso | Wrapper. | +| `ds4_mmq_q4_K_dense_pair` | `Θ(T·(M0+M1)·K)` | Prefill: due MMQ condividono una sola attivazione Q8_1 e il relativo scratch. | +| `ds4_mmq_q4_K_grouped_batch_vec_impl` | `Θ(T·G·R·K)` | Prefill grouped. | +| `ds4_mmq_q4_K_grouped_batch_vec` | stesso | Wrapper. | +| `ds4_mmq_q4_K_grouped_vec` | `Θ(G·R·K)` | Decode grouped. | +| `q4_K_k1024_bitwise_compare_kernel` | `Θ(M)` | Diagnostica. | +| `q4_k1024_env_flag`, report/counters | `Θ(1)` | Controllo/diagnostica; fuori dall’hot loop ideale. | +| `iq2_aligned_quantize_xn` | `Θ(T·K)` | Quantizzazione attivazioni per altri percorsi MMQ; può condividere scratch. | +| `q8_0_aligned_dense_vec_*` | `Θ(M·K)` | Q8, non AProjQ4 puro; incide sui tensori SExpQ8/OutQ8 della stessa rete. | + +### Osservazione critica + +La PR contiene più livelli di fallback che ripetono decisioni a runtime: capability, shape, env flag, disponibilità MMQ, path required/disabled. Ogni check è `Θ(1)`, ma viene ripetuto per proiezione e per layer. Il costo GPU domina sulle matrici grandi, tuttavia in decode a token singolo la latenza di launch/dispatch è visibile. Un **execution plan immutabile per layer**, costruito al load, può sostituire questa foresta di branch con puntatori a funzione e parametri già validati. + +## 4. ROCm (`rocm/ds4_rocm_q4.cuh`) + +| Funzione | Tempo | Memoria | Valutazione | +|---|---:|---:|---| +| `rocm_matmul_q4_K_dense_kernel` | `Θ(T·M·K)` | registri | Legacy: rilegge pesi per token. Decode. | +| `rocm_matmul_q4_K_dense_grouped_decode_kernel` | `Θ(G·R·K)` | registri | Decode grouped. | +| `rocm_matmul_q4_K_prefill_tile8_strided_kernel` | `Θ(T·M·K)` | LDS/register tile | Riusa pesi su 8 token; evidenza empirica +125–135% prefill. | +| `rocm_q4_K_dense_validate` | `Θ(1)` | `Θ(1)` | Validazione range/shape. | +| `rocm_q4_K_prequant_alloc` | `Θ(1)` logico | `Θ(T·K)` | Allocazione da scratch temporaneo. | +| `rocm_q4_K_dense_pair_requested` | `Θ(1)` | `Θ(1)` | Tre `getenv` nel path; da precalcolare. | +| `rocm_q4_K_prefill_tile8_scope/requested/required` | `Θ(1)` | `Θ(1)` | Policy. | +| report/note/result helpers | `Θ(1)` | `Θ(1)` | Diagnostica; atomiche/contatori host solo se abilitati. | +| `ds4_rocm_matmul_q4_K_tensor` | `Θ(T·K + T·M·K)` | `Θ(T·K)` | Quantizza, poi Tile8 per `T>8`, legacy altrimenti. | +| `ds4_gpu_matmul_q4_K_pair_tensor` | `Θ(T·K + T·(M0+M1)·K)` | `Θ(T·K)` | Condivide quantizzazione e usa un singolo launch pair sia in decode sia nel prefill Tile8. | +| `ds4_gpu_attention_output_low_q4_K_slice_tensor` | `Θ(G·R·K)` | `Θ(G·K)` | Grouped decode. | +| `rocm_q4_K_prefill_tile8_quant_launch` | `Θ(T·G·K + T·G·R·K)` | `Θ(T·G·K)` | Quant + grouped tile8. | +| `ds4_gpu_attention_output_q4_K_batch_tensor` | `Θ(T·G·R·K + T·M·G·R)` | temporanei | Due stadi A/B; Tile8 solo per `T>8`. | + +### Ristrutturazioni ROCm ad alto valore + +1. **Completato — pair Tile8 in un solo kernel.** Le due matrici condividono `xq` e un unico launch con domini di row-tile concatenati in `grid.x`. Il costo resta `Θ(T·(M0+M1)·K)` e l'ordine di accumulo per riga resta quello del kernel standalone. +2. **Token tile adattivo `{4,8,16}`.** La soglia fissa `T>8` lascia i microbatch 2–8 sul kernel che rilegge i pesi. Autotuning per shape/arch può usare tile4 per spec decode e tile16 quando LDS/occupancy lo consentono. +3. **Quantizzazione prodotta dal kernel precedente.** `q8_K_quantize_kernel` è sempre un lancio separato. Fusione RMSNorm→Q8_K o doppia uscita F32+Q8_K elimina `Θ(T·K)` traffico e un launch. + +## 5. Metal (`ds4_metal.m`, `metal/moe.metal`, `metal/dsv4_misc.metal`) + +### Funzioni host principali + +| Funzione | Tempo host | Lavoro GPU | Nota | +|---|---:|---:|---| +| `ds4_gpu_matmul_q4_K_pair_tensor` | `Θ(1)` encode | `Θ(T·(M0+M1)·K)` | Pair small-batch; condivide input e command buffer. | +| `ds4_gpu_q4_K_pair_quad_compressor_store_tensor` | `Θ(1)` encode | `Θ((M0+M1)·K + compressor)` | Fonde Q/KV pair con compress/store; riduce intermedi e launch. | +| `ds4_gpu_attention_output_q4_K_ssd_prefill_exactn_tensor` | `Θ(1)` encode | `Θ(T·G·R·K)` sulle righe esatte | Ottimizzazione SSD prefill, vincolata agli exact rows. | +| `ds4_gpu_attention_output_q4_K_batch_tensor` | `Θ(1)` + branch | `Θ(T·G·R·K + T·M·G·R)` | Sceglie classic MV, MM tiled e exactn. | +| `ds4_gpu_attention_output_low_q4_K_slice_tensor` | `Θ(1)` encode | `Θ(G·R·K)` | Decode low projection. | +| `ds4_gpu_matmul_q4_K_hc_expand_available` | `Θ(1)` | nessuno | Query pipeline. | +| `ds4_gpu_matmul_q4_K_hc_expand_tensor` | `Θ(1)` encode | `Θ(M·K+4M)` | Fused decode tail. | + +### Kernel Metal Q4_K direttamente rilevanti + +| Funzione/kernel | Complessità globale | Caratteristica | +|---|---:|---| +| `ds4_glm_q4_K_value`, `glm_q4_K_scale_min` | `Θ(1)` | Decodifica elemento/scale. | +| `kernel_mul_mv_q4_K_f32_impl` | `Θ(M·K)` | Core matvec Q4. | +| `kernel_mul_mv_q4_K_dense_f32` | `Θ(T·M·K)` | Entry dense small batch. | +| `kernel_mul_mv_q4_K_dense_pair_f32` | `Θ(T·(M0+M1)·K)` | Pair. | +| `kernel_dsv4_q4_K_qkv_pair_quad_compressor_store` | `Θ((M0+M1)·K + compressor)` | Fusione verticale molto utile nel decode. | +| `kernel_dsv4_q4_K_hc_expand4` | `Θ(M·K+4M)` | Fusione output-B + HC. | +| `kernel_dsv4_attn_out_low_q4_K_f32` | `Θ(G·R·K)` | Low projection. | +| `kernel_mul_mv_q4_K_staged_exactn_impl` | `Θ(Eexact·M·K)` | Solo righe richieste; riduce lavoro se `Eexact << T`. | +| `kernel_dsv4_attn_out_q4_K_ssd_prefill_exactn_f32` | `Θ(Eexact·G·R·K)` | Prefill SSD specializzato. | +| `glm_q4_K_dot_row_tg_f32`, `glm_q4_K_dot_row_lane_f32`, `glm_quant_dot_row_*` | `Θ(K)` per riga | Varianti di riduzione/layout. | +| `kernel_glm_q4_K_pair_swiglu*` e varianti addr/mapped | `Θ(S·M·K)` | MoE Q4, non proiezione dense AProjQ4 ma incide sui modelli SExpQ4. | +| `kernel_glm_q4_K_down*` | `Θ(S·M·K)` | Down MoE. | + +### Ristrutturazioni Metal + +1. **Piano di encoding per layer.** Spostare selezione pipeline, numero simdgroup, controllo tipi/env e dimensioni dal token loop al caricamento. Il runtime esegue una lista di encoder prevalidati. Non cambia Big-O, riduce CPU latency e branch. +2. **Unificare fusioni verticali in una pipeline AProjQ4.** Il codice ha già due fusioni corrette: Q/KV pair→compressor e output-B→HC expand. Il passo successivo è evitare la materializzazione F32 tra RMSNorm e pair quantizzato, mantenendo un buffer Q8_K/f16 compatto condiviso. +3. **Tile prefill determinato dalla shape.** Per `T` medio, classic matvec e MM general-purpose possono entrambi essere subottimali. Serve un microbenchmark al load per scegliere `mul_mv_ext`, `mul_mm direct RHS` e tile N per ogni shape AProjQ4. +4. **Non applicare `FOR_UNROLL` indiscriminato.** Le review inline lo suggeriscono, ma l’unroll può aumentare pressione registri e ridurre occupancy. Usarlo solo se il metallib disassembly/benchmark mostra meno cicli senza spill. + +## 6. Costo end-to-end per layer + +Le cinque proiezioni dense AProjQ4 dominano come: + +```text +Q_a : Θ(T·Kqa·Mqa) +Q_b : Θ(T·Kqb·Mqb) +KV : Θ(T·Kkv·Mkv) +Out_a : Θ(T·G·Koa·R) +Out_b : Θ(T·Kob·Mob) +``` + +Il costo layer rimane la somma dei cinque termini. Sul decode (`T=1`) l’operazione è generalmente **memory-bandwidth bound**: ogni token deve rileggere le matrici dense. Sul prefill (`T≫1`) diventa possibile riusare i pesi su più token e usare microkernel/GEMM; questa è la ragione per cui il passaggio CUDA da fallback matvec a MMQ ha recuperato circa 15,5× e TILE8 ROCm ha guadagnato 125–135%. + +## 7. Ristrutturazioni prioritarie + +### P0 — prefill: garantire un vero path tiled per ogni backend e shape + +**Problema:** il fallback `token × matvec` ha la stessa complessità `Θ(T·M·K)` ma traffico pesi circa `T` volte maggiore. È la causa già osservata del prefill ~16× più lento. + +**Intervento:** creare una sola API semantica: + +```c +q4_dense_run(plan, out[], weights[], input, T) +``` + +Il `plan` sceglie al load: + +- decode vector per `T=1`; +- microbatch tile4/8 per `2≤T≤8/16`; +- MMQ/GEMM tiled per prefill; +- pair/multi-projection quando gli input sono identici. + +**Guadagno atteso:** massimo sul prefill; nessun cambiamento numerico se si preserva l’ordine di accumulo dove richiesto. Il beneficio non è teorico: è già dimostrato da CUDA MMQ e ROCm TILE8. + +### P0 — prefill e decode: eliminare la quantizzazione ridondante + +**Problema:** F32→Q8_K è `Θ(T·K)` e spesso è un lancio separato. Pair Q_a/KV la condivide già, ma la rappresentazione quantizzata viene prodotta dopo che RMSNorm ha scritto F32 in memoria. + +**Intervento:** RMSNorm con doppia uscita o uscita Q8_K nativa per le proiezioni Q4; conservare F32 solo se un consumer reale la richiede. Il piano layer dichiara la lifetime del Q8_K. + +**Guadagno atteso:** riduzione di un kernel, una lettura F32 e una scrittura Q8_K per gruppo di proiezioni; più importante per decode latency che per il termine `M·K`. + +### P1 — decode: execution plan immutabile + +**Problema:** capability/env/shape/type checks e fallback sono ripetuti dentro il percorso per token. Il codice è difficile da verificare e impedisce al compilatore/CPU di avere un percorso lineare. + +**Intervento:** al model load costruire per ogni layer: + +- puntatore encoder/kernel; +- tile e geometry; +- offset/row bytes già validati; +- scratch richiesto; +- fusioni disponibili; +- fallback definitivo. + +Il token loop non chiama `getenv`, non rivalida i range e non cerca pipeline. + +**Guadagno atteso:** piccolo ma sistematico sul decode; grande vantaggio di manutenibilità e minore rischio di false fallback. + +### P1 completata — ROCm: pair Tile8 in un lancio + +Il path pair ora condivide la quantizzazione ed elabora entrambe le matrici in +un solo launch, concatenando i due domini di tile. La complessità resta +invariata; viene eliminato un launch per layer senza modificare l'ordine delle +riduzioni. + +### P1 — Metal: command-buffer fusion e buffer intermedi + +Completare la catena di fusioni già iniziata: + +```text +RMSNorm → Q8_K → {Q_a, KV} → compressor/store +attention heads → Out_a → Out_b → HC expand +``` + +Non è necessario fondere tutto in un solo mega-kernel: bastano due o tre kernel verticali con lifetime esplicite e nessuna round-trip F32 non necessaria. + +### P2 — decode: cache dequantizzata selettiva, solo con budget + +Una cache FP16 delle sole cinque matrici dense può abilitare GEMV/GEMM più semplici ma aumenta i byte per peso di circa 3–4× rispetto a Q4_K. Sul decode bandwidth-bound può essere peggiore. Va considerata solo: + +- su macchine con ampia memoria residua; +- se il backend dispone di una primitive tensor-core realmente più rapida; +- dopo benchmark A/B che includa il costo di warmup e pressione cache. + +Non è una raccomandazione di default. + +## 8. Proposte da evitare + +- **Ridurre Big-O con pruning non validato.** Saltare righe/pesi cambia il modello; non è una ristrutturazione semantics-preserving. +- **Fondere matrici concatenandole permanentemente nel GGUF senza misure.** Q_a e KV hanno output diversi; la concatenazione semplifica un launch ma può peggiorare locality e streaming. +- **Cache FP16 globale dei pesi Q4.** Aumenta memoria e bandwidth; contraddice il motivo di AProjQ4. +- **Flag permanenti per ogni variante.** Il file AGENT.md richiede un solo release path. Usare flag solo per diagnosi/rollback, poi scegliere il vincitore. +- **Unroll aggressivo non misurato.** Può ridurre occupancy e peggiorare sia prefill sia decode. + +## 9. Piano di benchmark per validare le ristrutturazioni + +Per ogni proposta misurare separatamente: + +1. kernel time e numero di launch per layer; +2. byte letti/scritti e cache hit, se il profiler lo espone; +3. prefill a `T={1,2,4,8,16,128,512,2048,4096}`; +4. decode a contesti 2k/4k/8k e almeno 128 token; +5. `--decode-consistency` e `score_official`; +6. memoria di picco e startup spans; +7. AProjQ8 control per impedire regressioni del percorso esistente. + +Ordine consigliato degli esperimenti: + +1. execution plan senza cambiare kernel; +2. RMSNorm→Q8_K fusion; +3. misurazione hardware del ROCm pair Tile8 a singolo launch (implementato); +4. tile adattivo per microbatch; +5. Metal vertical fusion; +6. solo infine cache dequantizzata selettiva. + +## 10. Conclusione + +Non esiste una ristrutturazione semantics-preserving che trasformi la proiezione densa da `Θ(T·M·K)` a un ordine inferiore. I guadagni reali vengono da quattro leve: + +1. **riuso dei pesi tra token nel prefill**; +2. **riuso della quantizzazione tra proiezioni**; +3. **fusione degli epiloghi e rimozione degli intermedi**; +4. **riduzione di launch e dispatch nel decode**. + +La priorità più solida è rendere il percorso tiled obbligatorio per tutte le shape di prefill, poi fondere RMSNorm→Q8_K e costruire un execution plan per layer. Queste modifiche attaccano costi già visibili nel codice e già confermati dai benchmark della PR, senza introdurre approssimazioni sul modello. From 51f9b9ef1cb34d45c9bf15fb85c1aeb36570a0de Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Sun, 23 Aug 2026 14:43:00 +0200 Subject: [PATCH 073/189] docs: add detailed PR 621 review summary --- Sintesi_dettagliata_PR_621.docx | Bin 0 -> 58995 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 Sintesi_dettagliata_PR_621.docx diff --git a/Sintesi_dettagliata_PR_621.docx b/Sintesi_dettagliata_PR_621.docx new file mode 100644 index 0000000000000000000000000000000000000000..053c900807b6a7584b3e1885f0db95844bcca1b7 GIT binary patch literal 58995 zcmY&;Q*DFX!o8HWM^LH&2CEn;uyVru82uj1)o>a0ueVQbTz{L_A3 z5Gm~X19k#~AikRlDnua*KMyw_MaW_EB6i>QS%&9#%j@$bcTA{sl=5F4VlI~JciykW ztftt>YIovp;T=#0{)s6oQ5v_)@15Bw1&fqvG$Yb zZ6Q*bmjg;BQvZDs+?dqfU=vWWLc1OE){fnqLFERJV>OO`<1?!GJF_hNQh{LqT!R6z@$}3daJYH`T3{HugQv&>%zdExaTm`9NP1acZcaY zrq+xZJIXr!q|&8}Yypz0O@A6zWT!?s!ej2E^D@Bhgu74>s@W0Oa`^8CR3v=#RX8pu zM*xJfoS8@uCAJHm{;(QR|HSV5d(ylF%C*Q}qXoW47iS!2bi%t|yEZIxS*(743ytrT zYA*b<-jYT&d_zPDB})a|pwOIdNqeEnX9l*>e3OV)b26-9M5fc^N#*Zan1OPPf@D>NlG!Rkhui$VFz{Tka;w@sb; z8_+2*U4cX%DtjTCHW|MjNe~51$)DNH)Cl=$wor$(sT!~5gldFE^{H(o5z9i#eZew0XyD06)dzXZ|P5hMdX$BJV+lPr@B!7aGE#qmd>^9OB~UWl1!<4O`ExH1uObH z%-~-z)~8-;HOH;^LXXX9vtR%B1tG+Q^tJzEz7RMF2=c!RGO;&SaI$xBW-zvQGW`$l zvlA!m78udOo_RyrXd8(3b@GH2$uAx6X}iv~bn;W7#psVRvVO@&DA3yJP` zz>yf_F1(Um-Qkzaaqe2)!QO00{n{n(z-Q$QBOJXHy$IncbT)heb+n4c9OBgr-EaR* zxFyfpJ@mHc>S`(@Kls#EDS{VBV77tF&P=0!_Up&`gsM8aX?Cir-r{THM)EZByf=-% zFAJGLHOX11qEEaQj4C*8!Zz1NjU25{9TPhPDj>?y)rOFD9rb6%s}~{ZU|iDG6>7%4 zi5rb3Z}FiVoaJd|T-Fc;x0<|5I^`UEWSccr71GXM^%D<)>>40 zmd)_7N$fwKX{ID2Ac}(-ezE{2JS#0JS0ltReiGs^8Fb%FS^%uzyj>m}Oiq#W%x$$X zc;`t3&+%y;?&jfN{=gn!7#Hmb`prIyJ5*+0QLckvL^Yi^5-K;8t4G|$T*mim2#w@% zv4MYzP)3jaB}uk2wC{gF&eaIqaExzzYAol8N0fO&1ibMGuY3SSsnfJn0D!&H!P)7m3=w!r>GqQA9e zI^KD8<%2tOhvf8YYlOj1y@av(9E+a74{Wp9PEF>YQr{-!3D3e%^|p~uNd2=eB~UCc z(cP8F@WFt81C#lBkMq6wctP;r#F%kF?*UT*0r?yO`=0{fZtrBm@UIHE+WzZ*|6#^u zx0LhxN(w!(|wL>+g8o zNY>wu51Y!3ro5)acl(0kh1WSf{Qh$fISsVOihYBeDBFD9$gA_${?GoOn~!El??a-Z zT){M=t4FST{?8^^$75!l*{{Y-yPDR-pRCh+hFf?0Qat#G#SCl9j@V6fgRCdDq%pn; zrZ=uUyd8V1sJ`t|KE0Jc+w{LIvBIEgO;vRFukeNVhghe+c1qa|m8n^F zwb@ZfcAT*d^|B@BPsQrNKSmD4_}}_iet2i6P1mkJY$oXs)}BIKJHH2VTcT9(e7~Uo z+{fn$N-djio1vTZH}|Ay{Nw& znDE=2dKlpah0J)#ges(zdnF9v2|_oQOpy13`fG^<#qq+EPf$wrHI8yMI0ny;UYG6= zb%-gyLq>1}wld3-)8zl`etG8cDDwSD9D4^JH{}B()1G|b7xVl` z<>NOb@{-eC)3~WfV18ZUK5;$2o$y`b@8Hw3EMU+HWpixe9+!Rj=~4F##Y?IVRk7ar zX&@!g3nS*a>YN_FSNlkM1ZUoQzUhZD()meRDu&sh(nTej(%9=tch5plpbvjn`-JN( zKfCHg=YBk`$Lao5AeqQHwYp@9{e7A|`1GvbJIcOFbG!!yjG#n~KIV#$kMD!0yq6!K zFzmYFx)&<*Kdt;RAkfEg%mIT$#BhgsjRT0ww>y{j^VH7AJUZj!JdEG|Bc_+lkz_B% z_{kW{uv?ff+vr1q_)7|q`B>z0Oxv@>Aj1E-LAjk5cdClky8&sd-~Fc8U7&m#3vXUY zy1qKHDsLL#AG^b$qsJG!@AVc5?AgrWVNp-LTa@=T^zV@E*RD|?Yv`=$xL8!9lu-1v z=D!G&e2jXL(9tE9ThPSkn|siT0&&VYI&+@aa_J+SzW2CGpoyS-xp}=z#~B+2y%dLb ze;e@H?K)ADuz=K{tZ1@P(k4^9)5_L+-K&C@0n0f4{iL3vpR?(@vCeDkW`944m`i#K zhG-$r@if1F{={qE-96FPEf9nnANAwfzH@I+>;8_}&WbEYp!=6OaU@In$mx`Qg0i~L zJtrn^xyLdhv-wnbWp}dn%_70?5^H>&zcK5nj@*g2Qg=++=2O9yb;+Mu$4vV>72^sz zj_~21=}C)lA!TG;coOL^WUH)s!oQZ@k&{)fQaw7e z&9P19F+sTvnfsAW5Y zwKJF@2^)H0ha>&KGfYhDE&M5lf5nWoo4xl7QD!I^Z8=)WBwxq*+RFY5t}T*Qo@-)= zG*TJAM)m5@v+yQ!B(;Hh!2HXV;lLp&WuH3PKFdo>??7Z4qC^~SJc_-#7*H{{$Wk-WzS!ORNo ziuZuXywu!4{il&ql13N9TkDo%PY#j~$p$}`KBe1c`K!W(#F!-`-YcuG_ zjGP-e0_*hb9hi)*%1uBVG4m?95lU&|pkgp_SgTSxy5V#(-*XAbYQpy&!HlNf0atI( zzRBpFBP^+qViNq7EbIz6{;L6KAAgw#Eah+i3DJkx{YOzRvaRorYlM`D1%mIp-hTK$ zTKz96ToLbSig{B6_Z9%2A~+Gz**>1JdsZ_M`axoJhgH&;;DJ(UUSK5S0qt)2r2WpV zDgDXZ`ulnbsQj=uG$^(&`8cr*{E2N^qb#n+!6(q1w`ARO?@`!-5?k^ zQDUU*ix_<^{}kT+j2mU6PbYLg*E>j@hcoS+W&LZV=7Aj5g3xHezz;Fr2}IBY@H#{X zm@Eo#!it|aO!XiVQSMFT73|^TF7fdaf`6#~nh^WHerxC)NF{)$7g3!N6J`3+zgG33Jj1ngexsg)d+VY|uoEzYpYcs;cP6bhAonM2 zG3nP+*XvlR%B*#`;%WE^3{;>yuI9N0aR2U*22 zoGIth`fCZC(YTB^tOdekB)|?(-|~eo4aQNi{z$@HB#+eXHGm2ziXMm(C zH2hou)FHQNI}O~c^qBV1NMq^5&=)Y8h!0-1$J%m*{mewTTesj0+fOf9WOB@QbXM>n zvG0E_IKlWbMVX-VkJaBv1tne~<@`wf4*tvYD-cq4FhHJVwMDDN()ur`Fm7a5tTtp8 zNEiaQBgZIZ+Wn^==R^7BDH2-6LWzeH6zcK{I*!iHS_W=4o; zFb_}CV`GCr!2B30+?%kP^lhJNRqK$wKnJV@lwh-oXPePf8`K@16sjgEKP8w==UIgH z>7!m$Bpw+&eom4!CXujjroZ4PIH>|CHjkG z1)f%+%Ojn;V21G(dD9XJLT4&XJ#?V+WQ%s|ISaE4rGjGl5 zQ1wsUNHF6QgCP%fjQ(IZ$Y+id0Y1ii{Z{f>qtleZA(&JNv(`|brWm18(#da39{>u4 z`$n~()pbSSIAI{o4i&Da(0!pBNdI-`es{t()VB0S)N=S#$B-Iv2X(dR3+@WwMfFGr zX6COtcsGfgM`H7>%!_UO92h5?a5PRQ+Y=+!5wUd4_aloc2a zB4XuVN!So3lq!FxO_e0t#m?Jc3Iu3l$Yuw;hys}*aWgH2!{mQ72o7~q>&J{1!Z^{4 zC$q#I(RgCk)gv?&N)n0fppCb0g61bG2FoF+qft7cns8zzHi0?05p#6PXRzsCGHE$j#=CAcb^`9r}poOIq)op2B()@G;15@ABy}wK*aq*0@Sb{JnKf-Ql&HY*$60`rKna9TUGP#oKE#`u;CDyOOsQ9YE3+p*)6BgIEf&HeHGECiP zACX*&(rloTIq{m&Qd{e>v^KXgM3g$bVRX)HM5p3TvRJxZ8Go|0h z1a5iH9sOy3Nb92cLATwPWUPXQ%&3CR3oZQ8V)Vbna?T!o{DV5@UR%YsS`m$$0 zzR8uL6*nbUEtw>+UWKFR$=~!9RjH6MIN+z;A*~Law)9 ztaKt7p9BD=b-*#gER*#lm@KT_nbjjs*r7di=xmu0PF)4~qC{@ZO7jPp4}z-Il3ccExoD`%e4vh~>+|Xx}7G-4(_7@<5k<8{@Lb+E9i z=ntzef)+YrtDEOW_R|wb%wSFeRrbaVOIKyxzAj_JSZKC1Z|$3I?5pEb zvW4&rWNO z`@BO(`03l#w%Q%G)ZpUZXsaa9^tfJs-_58Ey72dSTYt8~YI_##3+3Y)%pO0d80yr7 zo{rlZ*Y4JTHUo9KV;V2QUeVp`ssRLgFkkcfJ#4lsJlMJS{Y&Z1K{}VVTU6#6WJ;pF z9peUch6FthwZ~05GYjBFqkW#!RpL-w?!?P--)EO)i;dcDGnDQNC3tMvVeiQ;F4eLG zJ~Hl)&!DKodz_xfKVQ!isEL|`q8dheMuJ#(t+75GGLz+RfxP!?4D<<8ciP7aQS zM%>HxrF5Fs-y_5mc5yC4RLWk7=UU1&Pfu4B>_UsWFIRZ6sPEm)u~Ri$;u)LbBP5s_ zSxQrSp67fz?R-A}3QPTLe~7M*^?34nckt*BxcK7M;dVtwPc90#F10k8o{7?qIX`jZ z8nVmr%L4P~e}8obzRK36G@9qoKe&fcomtbX? zq<-v3?djL`oy^s>JF`hjDBdF0R7)G$LdsH>>=vskjomy0*mM{#JQ?Db+k#1cl|eTB zmuO{0HTDxWz$A2jh77@cDvHtP`@Z)-(n?hs6$umKefo|mmfV!HSJlKX>fZ6wgII$+ z9;AFic6!GIqQFYE@O2}JsBepfuMl5iYmhEkL}nL$me>IdkpSXc@aXQWam1Rza=xe- zr{McsU{x)#@AAno|I4$Z1CIVhvN_h}Pn%}Cpgk@Kh_KhDVs3}`#a z#mCR`Kdw)T6K{wR57I;!B&|ex`VGZF7N3IOV$ABL`-XNnkX_5ozq-2S**+o|Y068% z$tNaF_cWoefjotF14uj+6TSbfSl8u*lxcmd6M??n?ws{fwF7bNPDce99N&RdFQo}; zlgGXCqCN9uK!d)`C~&yi*6sV(zL1r{qBwUy2qw#;h`EC>xiko8A#hX*BYve8gwFp} zzt4aWROCZE@q=L|4v{SkDAx`=RX8wZaEfTVu}DU0cv(6Hr2^fIiheSP@~}yKqeNah zt|6V(u(^TYRi24!Bqk>8S6Fhic@b(~KoN>{YU>1BEY8tUp*x?%b)90k#M0Xk1?AY~ zw@{zzG}o56!Iu*50Gl1(ArB;5=ioF ziB8-D1AG@jGi-5U24dxmg{xOZ9j5U9f+_j6-~zD`I-x7^gdk%h z8Dmw<=?>Ts`)kJ>Pi7X9HrNRkSN%_6xTqsv!@BQZ9A!tWEP$*ah{K;f2xa1HBvj;OajkhL8 z;owO48@hfT&@b=O2gF)y;)V{jcJG;kglV(JOp(ydM;{{o`Gs384S@iGo%NwyL5Cv! zCtx}d-BVe%K2#iqpQ0BhIiHXw5DcdXr@u5?HOS?eZ37)lUsyPljJ&HWdocL#ax8}; zJx8RdW@GyB7r-w@EDdS1^h9U))Pqua>R3-EZ%@P(IsE-P9 zU6(z-j4-wPURpbs2QRc343LyCQoQ_j3UB5IVsIb4)D>B~#*}oig3%PhF;A=XXzfpMzz!3}%VEI8~1T|z|$RZI#=lEvY z#4Wt~?xo>rBjNcdVm+>Yfy5&Tz(5ngXE2TWV5YPCiQ!wP}v6C%>Q* zI1zv4@K}Q4_q?wwMj7stDC=d6O*cnQ)@a#Krt^8w}(aHiQAWEC5!9dF>5%TIU)%MwDSI+=vLke@c7qWIBpJ+Vg@Wk8GMs( z{yb(|@zNoNw#1BI+?&^4E3HY`{ko)9P-a44N5Trn1p>cnk~{Nq(Tlt;-o=89a9M06 zc_f*LJWeEeOlIRH6il6%us{~nphy8mGraaKb)?c%g;`V$L5o@JbLryYh)sr;5 zwcaZ;mAUf-MTC9~puQYN?Fw~gUUAU5d6e2XT$fg7-y0Xp`ylSone3G93@Kxw^}tq{ zQ=h_D=*gWh?H~%i7&IFqS9sUpV_w1ghC{1I1j_u5ofE>Aen8>$c}7>?a&E*j@z|z& z@?zJ29Jf?o1-g|IfzTsF*DLZR+4|pCDH^p+p9yw9nDUua8Nb{ENqkmI(;ZhwU2aU- zu%J*`YaX4ALAXV=?#SpamZ|4nFyVG%K|G6-GNeX>RnW>je^vPqM2j_Hjh*j2Ky92! z0s~=l^7lF}%jSLHTJ){PXkh(Le>wSB-PKU}gfZbgDB(!b+_%J%s0!$AMn^*0Eqwa60>eiVJ8;s556mafJaHGY z*~K21=d!I5!&V}*uZc#EnK3ux}()-@C zxn4Z){(9Y_U$2XX1QJGCuDz~0(=ClpTn3lnkG+seQMz_HJGnHu!&3x@L5K{U$~9(N zW#ddbE-r8NV8HXuu?frh5yWa$6+9d`wXP`Q9<8-ZM$JK|RZkWLNktB@m8?Qwp6C6L ztht|s`i?nd{i4cYQc~Q7leI-R;FkHE#wPHFTw7zW7K>QU_2~5dEEZhZOr}BjvL&p_ z307?gU|7plX4`OF>g>$w^oC6Q0eGw3G?-viuc7^Dt?w^YB~-Sa`mhcrtHrlw*hk!E zrxa)?_O^s|VT9bo-@7y_IB%uQQ}@A;xyL0TICQa86{!^O?ZcPJl18Y(4*Qs$M%YI_ zlJ?Cpa1tqHqgurq9v9I&8CcXytmP!L3AIk|I`dvhO~?iNPDbu{DwZUTC`4T#=>W%O!~ z;y8(_B2Oj?nugo3FH?-xjinwoVsVsBf^XfD7hQI0CjQc2&&Own)-<*Grp7MJs>8eS z%UwfW;gGOT-?OQapC-r0{IoB}iuk_IiP%K*{y{ zDShk6E~odGy@QLr$J^DhUBlpFqfuYR;M*Q>CTc~ndM=$uKT1yFO;|FL6l7$T70#Js z9zireGWLd=;)gTjVRko@mfb-x;DxokzeeLL_%*pYN{3~4@UiCu^W^{BDkySmYOt~=tInpoU*rn5M^o{4{cAem#K18C7J6~ii z0jPanS|GI>_$YtRaqV=2>%g#K%S(e8up&kYdAw zK>MmAsKGFraQ)zYNuTKkyHR%z?dIe?3U+olqS!K!*vN+Ac+}2eQFSlvmjUK^Shy_H z5aRtY3qu!;h{mK3XJk4}PAjlAz#&HD1B1={Luth-URLz0&r*Z^K`|>jSxtNglQX(5_3%v1{0X{h$I*kq3!-}vVu(JGW+-bePi(D@-fuNyL0?7nSwXM zvTv!^C>ri*-cDx7BX6i$PJf$SZZ7+gVbbD+{W1IqHvQkG9-UoXa>4Le-^iPA<69vG z0gX9Ig;^0mELPO&_#rHYB(FKZM;hE+rKGOfEI&DZOv?;5H;Bj(uN1;2)el14)HLtS z-Yk50&O%*PPLeR-!~w;XyuPSyHl!{Ct;Gdq_0JnaE86U;C_o>Utwl$CEq!uFNYNC< zh_SX1MlJT9O!3}){`0U`UI_18^23~^ET)h(%vWsJ`+wb~kv*jN;!9tVeTM}?vgDc! zp}p$ejx06kITICX9aGWT0EmPNh966L1C}+rur~W9xlbGMh2z{+yZd1K<|@q*NL4Xc z9P?aEj$r&jk`j3y0wF$e`EzJqMNc;6b#LYM*+kJSLhsTEmdycTDF-8^HUC8Sg- zoaw*8n&ux|8?q_RqP&?{pW%VhSg%%gO|ZDxIlE9kARJYQrNhR7Ob(0uLW&M92bN2m z;&LjCDttL$<2g@+%L>vq;n23coijUGm5$hxf z^!6!C`cW{2rIqA;^G-4fCRQ)Bi++EW6C_>u_c!|9-u>eDlXdz5-;)EPZ zLQ<5t%bJKULv{0JSOOFz2J~5oMf{?Dz2{~L@tg6>E`Y;T-rlo59At8F7qJ>MA7F>E zE4~E?l-D$9*gkQr5Z*A?v=}Ik_K)(a-2uEZ2d>9x)ww$rT04wN$eEIag88=5-^3%6 zGt;0Aj-nz9sWz?p}`*D5{OTqY3Y{Tmo0bL+zqS5b!w>5@z%_cDGu3xtp z#pcqD5>pV0Zv-xIrR5raDnBhnu3-)+uf~ox6h_k>C#V}$LhmFXe!NgdCCk$|`W@i# zsXK8WhNs{}OOjerzkNorh~m%3Fvb9RP&{$_#z?%Lq*<%7Fm9=2bY{drd2p(1))v5c zm;<0ypk?H>h8ziACyW%*{aDvXtox1)HB3DoJFBwlf^wBxKy;Up<&Jsytn0sAXmurn z!GwO%SW*RWg7a;ti6si+s-Q}ynQB~-pL@=a5tBBPV9%D7a&2bwfVt08T`Me0H?8Dg zw+2r{7pwNzB<*1xB+ixw*4PA-&NR;Xvn!9Oxza^ zu15OIJ65#^DJ5F1WJb)E2z6$hF-arQ-Qq5MCo?|?B?Yxp_S?WZ@Zw{spp@zshlq z=BOJk8lC@%(+&nw?B<>pO#@5p4)AJ)pm5Fh1! zN$9*bN@}~zY|r54s@yBBqQ|h>R$L7TmbFl^t^ahu@P^M;LRakeev2poT)od8>Spi1C`h5EsB%T^avTr)l%p z+~~zN&Sc2itZ@&ow>L)tvOJ|2rb$Xter1ZcT6(^mJvZK${&U;D{T@dbvpe6jt-bV* zWv)mbxSzFfhQE4@v0H3$beBIz-cSOX(ExSp?kjU!uS-Pdxw)6Nc!rK^Tk@se=EoN= zCg#@1hgh5o#Rb6owHv+Vg7rP<+kHTb=iZ+ErmX$r75|5&`Z?*6IgIrW=3JohW@ zc#5X79LlQoMtDMM9dClM|DcS?c3pYxDdTLBWcKO0xqkrLhJt<|VDk@zjBwx>m@VJ2 zak^ne0GKqtU%-gL3UgqqQy5v3A-w(IHZSas2QRwsyI^*hVJFc#Wp%NmkXVa|1`1<( z2@uuBB<#WGkTtTel#RRzO=9_-^|IAmn*r7d8~piu?qf;)M)J?xZ-J?qeehC z2{dnjd2}jjeg74i*L-`dk1U5M8r=6T#bT9_(`kUyE|Xl6Cvo32p6xxIt|gtCCGQzg5*!+15bAHTLGl#Ht>n| z@#~??Q3wZ^2p4wQ{V{WYf{mxV>?p&hm`7r%-EdQ1i1)+KnQvn$u0I#~dSy)EZE>}z z(^8ToP&YqD&0F$IrQ>~i=yHxtFTo7q!SI?ycyXi9dcuF~(aafrI%XY3pS|x{+^1ZD zLKbs&=F&_rCL1mra1dQfhtvlC;Cy*8n0-%T7LQQ|YM$-h5)Wpaw;Cks#udw_9%kGp z4;P9fSLIOmoej9~o^?hlsmWBZ@-~`l%9Re&&gsGyD>@nmXM^YKnk7fFE)aqu)XnFb zvHpoVjc56b#HOM*;S&vB*`K)WGu%>;Y3&z0GGh_WnU99=Ad051{^TdsKffQA1sk|Z z^e=*SiK?(kFc)6ko_x&CJAVQr z+XGB+A}Nt=YG1fWOR#EYUtQ0bl`eW#j>*HUgNBaah}KE&62^XCu<~pN*p#04+@%Q} zI_}<$&*YrC!7o=a9X{gc-o0@F@%x<;{z45Bl_*FcNys}RrKy_$?=PHC(HbLE%8D=p zNVv$k3$zlHHpE#!i&&k^*TAtacBszYgpnVSSjsD!)PKJ&_z9g#BcpW~FkuC(_77|P z3_YK-8He^pab`rzm=fF(9XOH|K$*W1Ibtn8XiVy~b)}rB**(H$xrC3A&O()xNMJS0 zC0>He)Q>1(HcMGLx>_aI8LVLlP8-Gzb|Dz+<5X|@$z+k#RX^V1Un7wQF4fF8@#rzz z$Vb8W)1T4dGJn1qtZ*4!u6D%RuXLHsYrwu9oK=$?w+zUuPuunfu6Hm-#L2g@C?93M zr8>u$hmBm?nJR)AiUH1-NOmBe$IWLzkIUx#<0I`dH@k~}bKUfqLcoJtp2c~zlEBZT zR2$0tCNjEd$1k(cMNcWpNoT#r*`mJ*sIVZSotO966?hC?dtNUDpTqYew2Jq+VMGak zf1U;)76+OAhI0Khzd(cBZ*XzBsbQ(|48qHsF#MiEm#=r@9ZYmWuc*1$R9aM4w)JQ% z8mK;hFz~gyfKPu?f^MaH>_s9bHIBx0kN2crENRXkcZe70tuIDsH&gbP)oK-YQA&1Xw=kF|j!Hlp5JVHFNVoW%M{6is zEh;4hY2PR8Db=dGNx#x5Hm<_Py@^^?XFKp%zle`8m(caMT>K~;j6Uv5-??N4s15Bw z-N>n>R=IG|fmb_U-3Yovixu~x3dG^7Qo}jzNgr`cYgVPY|B0$8I`aymPEBrsX`>s* zCI*vy+cq3@PGxSoy}xc?qCfztm9KFiuVpwObkWEmM>A71iy5jfBOR-|%*4$WJCKn* z(u$uk1y7pEkTlvIJ3mpyq>LG;B613nAy`1H0;MVpVjWpZ%>0rT4Nq!@J%B?tDRr<1 zZdA@hU}UWd3`RhPK|YyH>j{r2rj$L7B&dl=8B5H-<{NSC?B<@Y?nEp( zFSDY8vkxHWR+~9n0}#C}AN<5i5f9FZ zXL67W7#pn`((9vReS9v5g`9ms2s52e5Aw9yf%y%%(Dec+d>)kThAw&{G_n6iq6&q# zV-2=XOSYXhhf7(yx$unLU(lNOj+?>z58j{s`-N4JddLw)7ztWagszrFdBA za25j#qoAmDn$bL}EzOmcA*V|?83y{ftrEfVhr@)# z;2K|xe7q7?xjz`^aG44m0=4^wS~*vu3J#vCwNg0mkQFgt$#(s0i3^b_7u&mv`2`#) zM;MvrH0W6H*q=I}rj~0_E(odk$)$#F9_zb{O<6~WR8>*a5?>$~vm??meSS{-+G z?CIiaLTfaH=u_X_16c`sg1GoCc_#a^si$?zpxm~yH{*ki;g1d@&J(P`zPqq440!kn z7T?9RviMngICJo zvyPe{bCUf%m`FR02}5(Z#{5dojU=Mu6L%OS_9QDjb^c6_;q=nXHuFe;HLHq-2evq( zNtkIGXlJ4ixoLgXQP`p*-Jq*gmT&XqVSGOq?5@OM^G)f}QFDV6~>OUsNkQ)G+Jx)bLQ8`v158rft?W~O_D=$_;KkLnH9 zQpGCEgZS0h`{Wu|`!ZCr#&-dH>7@}A72fJ1W))sY=(x!ha6?ag%dk3&!!*KwI0m^< zi>f1cH1bg7Mdjq8p;>O^rbSWb(;etn4KCL&^=BNS_gdF*giCRFu&~T98Mg(gqGY{q z%nIjSU6zDPp**mvb(JiSvBLJN=cx?{VA%= zL3(0ze*UX$1RCEy=Uv~~k*=T1CiG?@gzU?^JykANoh9p)jx%j+xv^(-p=$gfcTo&d zCMmw&6-iPMJe}EyvnWw&1V6_g!eWwfd_+`fSlZRXKP3lu7?UczAvsF@YRPB1b6eb| z*DW9YSNo*Fw}3S|Ng> zuB%mUe8p6LS0}TqB()H|2QOoBD4^DwmN z5Q~~ku!$B!$zSVg)}G&(#X&B^z&UBR@QCL_eDOw?T_ zF1E)>oM9tqe8lwa+&8SVEM%A%Vak!9b^8}G%>k=k&ZbbQ=A^TMc|=8Pv~C<1g#L{TnedE-vxy}Z zKxa&Exy2v~?&ezQQ-*cA=&HCwpZ1SwrK6mBv_89Se9n zg9J?GF;7+LNo3bR0b(iQ3$-gM+wziJA+AtPuexFvibexn=gvUtrx#HJ_r{p0)do@A7$n%iFU^LuBN1&;r<)4okyQT^`l%1h^>nzY2`K924B&V zaR#hILteR?F{^w>=n|A9tGR^FpV>Zf>q=QSwq0_K0iLk(gQ=q{Vs_|y)0e*3a3LU! zuy(|*mOEmuUh(tC5GX8Lt8&cSRD`(vHZdk7)(1BNrLr2G31em`W6nHr#k@o2ltWfE zBOyl~)L@7=Pne8%i$z=n#U7Y5Wg|&6VnC6wHdRukQPQkYa4_x@RQdCZebQpGLi|^$ zy0EJTX+@h7=fH)lPcoham7JkfsH-kliQsLreMQj3i>h%BDF^j-4zym$I-5dzECD^0 zH(|6=>ooLvODme22d~+(0h|Y3@S!dBMU4Vqpiumfx7xQ4@4i!7_0Rr-TCtRnNv(SOi?{Yg-6HnTE_1;&V(bB z5Kq0H!o-}r>%gdZA`{{jc=ZrC>T!#dQYq zb;ct#QljMT$;C=Ts&!7eKQ;uO)Y8z09B6;N_jl!Gl~W(tDtaBI#wMA)AvuTH$XUFp zMlV3JEiMI=9jtwROXZ$RwWY_Z?6QVfllJ!`;6z4{H}98~XRQ{qs7TWznXYT7!njck zSF6mW+&v+;lhBBh(2ngQLyK9BWfvYCF7)C}K(sHHH_kc(754X4nN#Y^eni#$FsD%H zgv>d82+D_55MxTzK$TMirM7`tZ&MC=A2()@Lv$Zg=(@+^q$7W7$-c^d{PA||J#s87 zUrTo>5#ybARJhA=@^A8xY@Sot|4Q@M<%nYdXt7OWNR|54wnEPksu%bbmdGGo<;n-j zI={@HhUt%!72TQpXlx18tZ60aGc0KNP1)gKLG(p6M@rJ)R4IG#aVn-(ebU7v!56F` z;)50IMC(e?kKJd&>YJ^&-iw+-Y?s(uz*?**+=q|{@YA&<~-6X zsNcOa>{wxKJNN+|OdM+e=4j*FX6%B)?doz^(D-&AS6G&8*g4GoG{W|oW=@+sfZ_q# zVa4&5UMY)C2rs5hUWm>=XAAiTZC!Uzga6ubuE2e{Y3X73xoVlYYYyt+g*#+fWtlr4 z-PE_z3{%>I%&0j=n3S@D!TbvB zLZuYiRSZn%Jm}}fRkX7Tq5N($0kWj)Yy=}JGKD)Ls#CUQVN<3bkj@RHK-EAR#SOyY zba&hb?&A8su2Jbcv8bQr3zS!x%*cNye!_R12@N;`jzt|8igdy|AaY_$N*LWx>1ZSi z)|+E$P8ZwK=F(>(SK|zECz<4mf6=@lqY>u8=z^xqqFHLC%pOo@$v1h?<&>C6i7Kh4 zO_kj+Kh$QAJmJ*-6c;~bN0CfqryHjtM#rX#VJsHJ!KU*nLs6`eo&89gL%RW$klf^u zV?o-vW-FEKT+3NyoJ!ec(`bgVXc<8cRqM+oC8zrArI?ZC-L}z;0mofXxQ3P-(dZ+L zo6Yy>^CRfO8o}j>`*K+3pG6RNR&2bkUlNZ4X1kFq|BUs3tBl25TtNX;>}iGUIz<`G zTzMMgFQyM1IqPH>!;mHwI=QqK;X%(o`RYD4C8dY%zSWvLSN2fB-yWZq=9E`X~mg{`w4rCyU9!l0?YJrZexr9UH#P zFpHTZ;hAKw!3WGwryKo?y_y$?E0$LI%}W3&v(zg_s@JXBj$K9;>npYb4H95 zpWCO%+93N;{n{B~uPm zLxA$GIep8Nt0G2f=XFN&NT($hi^K6J$J7s~&rS%U#Oy}*RGS`m8MhnjiKKFofGl%_NzaB{c&rr?c|vr>jneKN?1+If69d z;riA6e2U5-hFqc>vZj~=j5~9AE3~Jga5)gC89k2`?&f>L%xR*Gw3G$#>W0qn*Efh$ zNe2#4$F8qu9;_=pwf@&4_|ACdigH#RxAOzIwYL+eGuM5ZiO{?vE z$eH;`1}1=3b6uG$A#vRK?<7@ncAY8vgjKea%2|m*Kk~;gm}a6=TKkup*Q0lw&Ty`r zql$979fN$q(Xj9VyHFA!RVfwIQrro*!raJ-?-;Uz(u^)k?30sZnEWQ*{SSrwS3;2S z?J1cRF)}%XhecBkrG*J*yk=`Xrgrue%E6>e#(3I?;$2|aO4Qy$wcE5M`XtY}3JZ&`FaC}Yc;LvOrUyuz`(=s@#Og%g3zzebmqq=D&Fg>O(knHPgO z7T|ED#VO6_;@ELz(9h-HnA;og{)UoL-2f{3ot+6e;k5^%R6var%}VKaQ< z=Bx^_s7Mj0p~$j>U}~m(6S&+1P(>oPm)Q$xuU~=yKaUQoAr7eFt4>-sDkNvD@MKH&bb3LJX@ZP!eK-c@V7u&#>jTx5*g--60T#3gMQa*e&ijhv|2 zxi|WG?aw;BT2`(-KY-%v-p8|ssF~|3bC(HImAYAI#KCW-(E%aZ=hA8|#?70&1&I`; z1hQVLgU;7XksngEG55?ytspMxs_TKFf(P?gq%5wz9n8Hwl?+vrCEa+ArGAI8Lt2X{ zmfKJ#x|HAVfXn9Ur4s8+q%_PT4W?WoWut^pZA<}acq-T5WTq9SoZX$XtqU7~3R0Fd_uxGzE*8<2A6=jzZ0LAC6t2}79Ce`(So3}U zr;$|7ODKptuom|ze%Bn&p@ruN{7Ol>BuID5YP}F}MdR^{g+FcRL$bl%P0Y(CoGUqh zShsZ*$@}+M6ES@~AF#kFJH{FpFgjc8?V%*(G8sN?Z6e!m>x_DEb%c|&N=a)@bf|v2 zJf;~>Zbe-k+R;unoM#uDX+nz~1s0>Aw~j1i2tWGP7^a;m5*@c%(zcyY0)=vGt8rfg zb0CWh_H2)@AipAoH&pq{sWdrzX|v}4=VYLfTjWaV&OPq32!@1sz+7q`$YsIk50 z^Hro-B3B?XU1+6|1I0shVFJYUmC0{Yrgl`PJ5{2JNvi5Fp^Biix{N{dWYv}E@MIhl z6a=Cs2Z8je{xNnHcbG}KLvwFiYDou5LL*;IwXc&gJcVkX<$z2HiyMI>CG`TqHX&Bb zLbM*JtFV(j87Gf>@uJwk1^{RnNq4)3w5U1zY(+JSL8)U>QGU$|fV^VNe!;1!At9>( z6Qyacp$i>6B;@>k+iq&^F6tz<+KZ@-&w!_1(<4y;bXg?S}+Dy^~~9|&eCI*e##n5 z#3{&I&8K;~lAlaHW_J%t8@3bhRPhHtrSur|{YkcoG3ZG#ufDkqV9Qd|x<*4thgScK zEHS9OW~Ie)iI?>f1}nUsRj*U{w0yU=VoZglt4+&yQ^)!aZldPsdqZdUW+l^@4NXQ& zW_ZbP0{Qap&cK%sytCegh$v0*A_&|YD|N_G`;KkP`P5i@{G_+RrQi!62#TSZNXGJZKxn$3( zju_E-e~!&Pjt3~cb{e_`FP!)BHAkVMNS!e_d58{|I(DSC;FzZx_9>aTlX?bTPSJ_2 zD@@Zz2w_r_C%G2ywW>xzn{|q)rw*S__wc-wBAxu`gBFVJg^x7c168_h5d6rWA9^By zwgqE{x|akyXx*UCPPQ+z8E}~9cqrT!yC#(?U*H)(caG$gsuKAY1I`M&)ac5maW6f7 zGQZNV6(2Z~j&~&z!!vN{<{Zp-hgKwx=Z9yTL=c^XKIoX9hZ`(SDDh0F;f;NLf1e_r zQ->>wcIEk2h2^oF@2A5kFD&7bN2@v5n)?$7N3BAkb#DiOi!OzkZJmjpOUratGtt|% z(@jSui#NZk+3AeIWy_{NBY|B8F_liXOv1s?w8rPR6$j)ni_8*h7Uvs!gacemg;3KZ zi33rVgzRx404j&(vY-+@UELb?rhq{NU7AH&6e2AN;x`-~fsuBaa$myTz^TJE_%SNV zzC{Y;%7j!=2mDerH=J)`KWny89g*A&-&a#bdn&#S1d=pW3a{67X#mr>-i5*6A!f#N zLiJRiIOBEVSK(u#eBbuF@1TWjU68z9|wT>6xKHDSQQ0u4v3Cxhs^> zAbUN)O6%pvY}O$PCDhn>Y{VqP;BikSJn|f`{Lng(z`H#zA(2qpkkc( zb*nYhUm3*7xs|1}C{q4&ECc~?ARuK<`Zm%A{rn6I9Nm3UQU0lOFWHaK3wV$@j-cg3 z{xxM>0=M{VIa<^7v^FJ{)0fuO7q}9VLUuF>lqkk@4swYJK(nZP;T965C7sBRVIKHA zSY%l-`K3(SZW|ccP7V|{79(ObK3=~iq(ReCI(QO@(ev1R8HCm2 zNi=^1(ftQi0mBkw0Rv1eRUW0|xX9oPLRYc^zkZqT7PHCsenwX7yRQeDKs~rI`h$54 zhWITDOV-L5(Z&KP!j>9!TcgHhE>4lCX3Hnnc_Bc`zZahbcK3=1 zygI5}3k{KQ`SFXX5}YwpBy(3>Bl7@l%te!GiG_Nu^;hPb0R&=@Rs7LL@QQB!^;k>l zz=@RCzj;>xjWN*0E_2^dXunB=#bPw~&#S|A#K;+hmdJ7V9~#{jhk<6i$b4;0fs1a1 zX^(a_y2_PGUqG6t_hjEJh8K*bzVUv8 zC5!V2%l!_rVK^OI>(TSboRFU(1|+N(>1(^(qV4rQ`8wYTvA-cL!g(haps4?R1CQOv zl=(U)Ry&grAnQuph@0ApOEconJ5xTTEx#BpTLxy7x^gz2!3*#RZlhl&f0hNL5a-j} ziurv#%0^-$TE!C0+Ed+BM7WD!2kF|Nj+bGPCqnt!uj=Qz`N{Vgc~RTY!(DhA`S22` zM-Rp*nEMH9;=YM6A-nVtNbXr-nVwCh)xw0s-c8sZ5KYYKy$x5}$vcoh-mdBceDv~g zM-*!xfV*<;3~}fZ;4BT@C@ka{d(g~DQ^AdDuF?5Roy;K;!SvOdd8&_i9|}z3nmLik zIE``PrdkvK0LTFrCF2(k!-H1JqKzo8eWV8l%U87zzoO}_8>GZ<+<9f)*lx!dXb&}m z-xhV2Rxa1QrPtbJA~Yq4^7;MwnxfY25VUs!vy_LZ5#ssn7|@`c!r7^x&|wbHvExRfI?GM``Y&pRmN(r+>mnNHb0tho0>DUqT>s1_R=b zQ-?KZH7*m1flUlj>=K4B8S3HmRpQly!$Is{8R1Bv^;z#gZjS`-d@a*VQu7?~(skIU zB@-C+MMkJC#DEbzkrE!3*q->hY3qo`Q62?UQ?dIo_@f|h-%iRFYNfnPZ<7@*N;M7I zt`FO6y1B2{x&W^jAN~4NFs*~8XPMc)^msu0x<~BU9p4p19i;6DKSf?Kt^^Wf9_%$q zJcBzXhiQ$;N*>k4YdG2lI|3}ula<45V^ZjW#1FZ~vZH#KZ9I(qUf-Y|r{TM9 z+^x|{`JR-2Wj@sJO5%kO{cJB!vaJ~lA3(~0C2XJ=*ap&WB7_-k)l^z*`f$G5k86W{ zd8A1AE_&-YaS1$wPt-iiy~(e)fG58eN>B$1S2=A+YP^_RFC2F(;5r~G>>GeBKjVNA zN{EfBrU4N#L@d0uo-4&u&|E1kYcw)v3KB>?4R2eA`qaQ_Q?j?P%zWwn?bj;2F>He-!dv*&8}@~+kJqde`^LK^GNBtfnIi=68Ay~_BtRM*XBBHGd`aA1ak5W+gnED zhZ+;=ie|AR?)L2#z+`5#ThICqDf-l)q53@;%(G1!daDANMu)e<7pX{58jzw^snsHw zjX0mbx+5Q&8e5x~k9g3JpGT8uuwXI57HH~9Z(H9r#Irau&wEf zT_rSPM1|`!#q7~r_Iz^<1#Q7%@2sMNTEnc&HS4xdZ!gL%!-gugs7*5co&j&XaTRcu z`ke`N+HGGM;hn8Q#KmOze7V%fMfxoS>ze-)?lmcIFJcprAepRluU2LBCT5 z1t(Ikqr5}3h~O6E?dMqijnxEjP;yV=zy+_MeYLr>rB&k=`9Us)4HC4BrOU8usg;$4 z%~F0LYo_1uhn7y68;tQOl~NU00X^PzEbi?nL2kNoI(NoTS3sOwmajF<3q-;{g$t1? zz-`ho@u3z|LTXf)F${k$cLyPahS*wg}ozgiet_+vvQQ4gy~~sX(a;s;~9KbNs!mdLj z$Kr0}7xVWG!%`v?(ymyZdImYZ(@#n^ZX?Q3dg`T_65-62>3_|~4#>m$dDB6qG#JXA z+j&a5QRx@G7!d;lC5TWyEnYG<1Co%=kJ~l~i0bGO_K8_Pm5MDV{1%x{%rQ<>&mXWcMogw`J?$~Ky&pZ6u;8(9nfm&;)*Ithge-*MXn%B_dkf4plv>m9tEH zbBnJiRF1=dm$%_iIQ-l%0tag71|&Y?eL)g&QC6tLujzv1^gPHeXw#cW44W~?HCYD6B^!nLpGkm`MfGbvKy+OXfDCCX8+dd66eFQ1N z<8#*+8Y8Jy_-G6SdfTI6fda!{ptMCF?VQs&UIKZVcYFe3g!;Wp5o@10|MXkyP|~y+ zl16k~S-VEzqP_bD)_Ao@f)uZy1qzero)$=L$1c$hqlk~goD+=4hTXflJEKQ(M8eLQ zlw8D6+OX~kPS%7Kp4!u?d9sdBn+G*}Vc{1tiVrfQQyA~kS~ z#tw5ZzIl>+N*38xmsjJ2qL{^i?Bjb7#0`2OXN7`91~9$lX~0Af-|mX?5x8@*uLG+e z3qj22026nBppXdjjX&~6**!^rXjoJ2+D z5>{TTLEZL!Al9}_tbjyl%vv=aBT5_1Rn{Ds(xr!Yk2v;90s^(U+m%<^Rkjhy9b#O^ zAQ+nkf{qE1zLq)=M4&E2KbW<7rS(vQowGzQk_;2;1;onTfBONcHrGC(dH%$%!isu@ zrJ|k;!2}3`Tam{Yh)TRkM)b>cn)BFc9-%_yID4V4^0j^BF~9FJ0wN1$cF_n~c}58I zd+6d3K%^;IT`8krAQz_!im2ImPI@B&swa{bxn@;okTG_({yfIixDIpA9xoh{+0?OM z=Ycm~p7S=3!KVKWVhWXtv~`E*aAU`VbMm>!9Xs#W`-e&vgM*LK5@;$-!;4lk`JdLO zD_p;Cw$JyU&90u`cZaJQ0;(;k*u9yKHVIJ6~NCi$zyg6$2vB!qd}K`>%a1Y4~1Lg zTqU`3h;xfO@(&q;^#Fy4~}N8V`A{A%df7%M(F z;P)WK2jvAoj*;w@py;x!{ zdu-oyn>;=4U96YXyZm2U>26NmFqOibcEz;6{0(lEr1Gy@4;~rrqk* zGq73|LUo<^`VsWuZX;fR2}F@Wy4dHQr%$VmFvRe0m*R4jo~GEJGA<77u|rkY!1mk_ z#UU_ngV(GY;Aa4dW4pNr>%JA%hKALHaooFlg1aY&3}|{Q7S*X@AiL9dr%h zBR2snm+Qb{cpcQC7mvF%yXaij6<8be{`}XidTbghQCT}T5ld7Ag}6xUiKJd9X_i>F zD#Bi$^YXd~YIIBdD;K$#8cP0?6L2;e6_I4_@RUU@(9_Krdcu=34B?>x*bP}$Fd49oz^4w}AuDI_CkbZ+2pPT|dh>7(1{9pu3`2E!LCblHiv zS;f!AJ6P3SO5H0V3CG%Y7kyD-nSVbtV9SxI)N|9$u?u#9ponh z(@M}>k$dmnH zOL-WRp5@uQ9pu^r11YC?6)6QoB`KoDE<~{A#PyaZEq-%PO4B#tIC9391KO)e;00)z zzi|fpU_TU~Lbg6W|Fd9d1(213_g5uZe=q<5oUclZ*UpjrQX8fOWtG1W$pS-E(^8SNN5_gzftTju$QcQTmd46xsORpI_j zuJVab#w0sHzG~T#I7l|` zZeEYKDH|PKV)y|2&v7%}+?t%B0|0ne0R@2nXWSf}+^mcp|0?sUt(~yfT<6u(V_>24 zz8@*9n7BViMjB>xPC0Df*?iR?TCkx|8Fwg|UU{MGBN0A-4NTxCngBAgStZ$-7)bU8 zeZ=fHr2F~%ZT$CT&)S^Z4{pzAi}#e7??2WC4xbyjJbzy=(D8xmPIb3)=`MzpJp6WD z_I^H%xp=AQ`MmJ?{d$mbcM$XO=!yROx83hUTFAi=WhdVo+ZkyeeQ$8HsHt;(k zIngC452Wu$CwBwmuU}P-&z3Uw#?M~sJ=QONr10TY&Wv=)1#^T&I=tVTx2(7i|5_V= zv#ycUCEx>Vd^=r+3TwO5wb>aSi(F#ZsW+A z9g{=r1Bp-8R^VECeYB_Wu1oj!){*jyZOe-nNA7I+<8i=?efncd@D~p*+velktF;e@ z*eCagtK{#rcdFk#+pevdSKXcsytXbKIYMPS0CXL8mA2?1cbB~HMjgDmXh$2IZ+SUd z5xhRp-6QV@hn}NaI+xilPw;WmcR4#DpW0V^>zbU$TC2Mk9_-sAaIlMak@Vx88_5Be&o=#N-jQmZQ*+|TjZ{mK5+-TlOtO>x^Zcj)?#dT z-}gVeSGq;BvAx~Z#8_VlU;B7pA32E0f4{6e5QoskUttoxNTM!VDw-w zL8b3O)Ti!A!s=JR@07wbDB$<2;2V_l4oZ84a(IT0+dDdUpHE%5{i2i9QGSbvpPT=k zzfauML(wLDK_`hP|JBrI>7!6@`}OXy^8Kd??q|n+Mp!NUduPXa%hw`<%GA^NuPV2D zuZ_PwHF|irL>w(8B!5ESZ{g7yv~a3+EX1VD-CVb+MjDre-RSD%EFEpM>BQ!r`b7Iq zZScuEzr0{L- zv8Tf^+_q@KV^Z(eDs}@07Q#=oPj4=rQ@^sJk=-6Ht>%!sM_K9gDhO=HyR5*d1u|Yp z0Sl(wK!T<+c|inESrqhqUXCx+3{p~%GMN-4f4f5&!1o|}TPSgXR5sAU++P_$_0xyH zhB(3l30Ufh2qDPbpuaA3n^AWdkOHTp9Xb#?w`j3J37w!IJ0!M%0_>xjzUxF^uf9Qy zYVt#A{OL^Y1U34nlOI9$77Zcr@AI!jD-?DGuj5;MsWbO!2>!vArTP;p_!a8^Cp3@L z7Et{Q_di1ars9uvzpY01Px&7N*^Tj?poIi~RG59mq5Y-e-zEMVR2lh|JE7~Jv;Iw( zZA;RZ9#BFzXmJ6l&94mnPl-VOc3({de^iU}xGh2V{Wg8o&#a!-xONW83=36Y zchB@ymY#I*uS+gUF3dfaAu0>yQBryH0)>Az{T$!BY2m61+F%_P2o$-%d+IRkoH&e` zU7jm99{tqV@apKIX`!lgr3>?Uj$flifyK7zv&`V-23Fs8jzRjX2Jrc`e6gUWP8IZ9#s?3lmM%^H2PJDC%j?i zeBfjE!66cuaFaLrH&=4z3SROl>BbBypGlN~O?jwt4^NWvjb|55gu7Nq_jL`hl++H> zN3u>&bfdctIiE+1(QA}jeE{K=F%0V4rEBv<#dEzkH~D9dH=U%{4o~o>Cr>utI@cxj zsA<|&Ceeh;+t|8`SXlk+jb!mS)+DA1c680TOlj}~SzFhywHF7^Fx=+jFw7!U8M}|; zOYJ;RE)vaeHWh2TQ7AYS^Xh>m-s-kc(ZkRK8NAwb2*9$~RGr zU{*TWNay-f)3T~HQBJi^Yd=1S;#-Lrk|x1YQN7n}bbTNW_zFLtd6_?62(QRv*5rog zSkq4puf&hw+3dUb)<_*vuRc3^!jMuqW}}AAK9#K$qIvgi6ed4zy&@c(I5zXCnZcBDv-$&bXn{M>JYkn>?z_!KyJV{+-DCf@!-nkjUQD#qcqO7#jV{B`Wwr`W z=MZ&GWQy`Z<32+=|FZ-Bqceh&oQ6TmY{)qevl#l<>yDt3mzFr(;ePI2Xy33XL^t!% z>nzS(%HhX#Q5tV|U}|(N4$XImgHBO{Md=9IkrUTJ`<%Mex9h7nBTGmf*KGii_yxq4w?OWdMT70-M zCXrM4F)8CWL7#7}zEk+ygg#90Eg{=yZ`Y5vtvsZ3bZs@*dnWkC&bk@C6c!q+>c8!z zxo8+{C_YM8c@}h{n?HjuR5$2&Z(2?wKq8l{ZYsi0&HZYqy~;7$?kzhK^mq?t&Ov4S zMtYe zhd?ZMS3Qj~R|50*AiH_>`{+9;O?VK}7|@qw6sQ$_r-Cz_O;u}EKit?zrL9WWVYKgi zGU7>)+Dx2707x_-a8QikdI6^9)$e>lZ(`@t-e@00`{nTlKx|$VpmIn9VFIWA^MTLh z?Uw6xEyVaVhxB)|@w8%GI8?9Ej>cv9iPqk2hj)mdJkI)hTm$D+k1~_^rre87ktvdb z8}Vvw(#i`F(e9NOP=qK=BVwf#uCe6`90BC{ZoY)_Z73@ggdWIBhz?%P@6Zeu^c=R= z?bV^SON9p747iYbGsij%52l*-So%p*_J@&49>DM4%JaOGJffmqq5RU@Iw3AU;!acg z>aug)yRe(fY+YB=Saev%`1FG|Ycu_H)yzh)$0C+CbIQG6u+9+6m&#Z$o?F^3y@D~A zC{YdfB}IO>Q8k7g5Wic9M8yzLh4KY4n#%TimhIB4n9p_3ULQT|I68$cc3rU2M~P0U zO;$++JcH$IG_ULNmIuOAaeHti3{L6f2&a2I-k2@U4$y5qzR|2*UByqWstiQo27lgc zTvU)=E=S_A>)Uo4lk1|okhAQwR?X4^^}aqi~&PeM}-n z+BJ(s_;E`XH{vP`782IOvM6ddvIZ!h`gDvU&K*zWh8A)QYm8*uRKFlZ4qRx_oRg}7 z9AZ@gIo7wOiO73^l%~Gh#4LSU8%yOEqLL@6W)i5|&w^92_T)*;y4Vz5XpyY7dDUvK zY2K00a8xz!qm0JSWosh&cw`o0nkJcK5~kTNf-|`^h{()l-${B*#+`Fq1@N$}lMRuH zS2(Zl)fz3xIN(agV?alcNxzj{vU$>-L0rzYafspt_Y>PqsDW??pX0dcHvizj#mvZ# zA9p|;()*6)8*h>X>Q$-PGrt?K!7K00T$c0%g9Hg%P^eGYu^!6@ie7LYi~3!~t2Sdb z%#Md2*peK`gR7RsaNhLDWnF-M6W8tIyO-HnAH5W7g%k3^DDzP)%O)Y}dsR78ORQVZ z0a>g8xY+_XqiFT&FVP00uaV1yH&7PA-#aavZ`0t|k+jB~p2*tl&~$u1`G78WuP1cp z3~BSh2VxF3Q7sr9*)b#2&QTv(KehR}^u0~DwZ30jAiVr_IU@ie3Co$=)K+EUUhunW zK~$w^vPLg9%V(41S{b@Ztaa1mtbjLi2MDBj@rgv6@hEqM6o|WCClzTsQdX+`qQt#~ zXDW~q>52kMJgv`P&S}gcJQ6KL@;$LNKY!u-NpJ593xlxA{6DQo{rtKUoZ@uKX{xX#IOc}qPO@v&8f-;H=tXKZ(=>RRrTNltX{=E?qz)9Ic&eSRjQ zI9V2Gc5-8XZm7&WY~ed;WRlO~w`3AKPooG%GFZH^PP+|j*3jWgEog0|E@(5vk397V zv_;3Cu$#$YbRJwzAaXyn3N*}bd!A*NUeMBJ*wH-T@qM27p6pr?Ccf7)g!I064~Omx z#If@W#P$oscE}f*4`OHZ?K~Bf{G~!j#I_VlQP{RJ>lNmHXtTD4!#$(jaa^YkId+5m zo>iANC4u8gpbxUsMNQC_bMgjg)0cG^dW7g+G@10?1diR_Yb%4kFD$ErQ|&&u>`RX- zZ(2s!=JK$UW%MSQA5#wPGURa=$rA8AD|Xvt|1J*&yWbL=)UnJkhfXi{^r0)Ddf?_r zec)y!Ai2)=uXjx3O!l^asX}JBXJtU_TG>lu(yd3H=5oCRbazaSSQtC_Pn>I2v4!z} zX{vh{iPh~@$z}qN+N4J<4BgaXxIat}-Sk}3j$&_WdFq@8zDJWjsva0^q~-o!)%YE8 z>DUVy+fU*!x(lhqGQ02LykndMc)S-tNLTReVD;2QUzSQ*nD%u4;-|!Zmk)?r^wae# z1i8t{FUDWW*y7EJw$(2-nvU0MN0HS6%`j;h8!xtF;$N7%xwzR_(1vmpPld1$BDxkUpBP<63}8cB)MyR(%G8`XKojG2O8OGjdiIB!jPD`xTA` zBP81OS9s}vh0p&h{P<6J0G?o1xfkob%`EV(yT!+x*t}+~MYr?MSOq%JPWfKs(;^OG z_!JAYhxm(Fs}g^Ax2~xAQuNK`Q281QAEd4fY`@NI4YCKs2eis`+ZP6-c_~0TlWqG=q6jii`er$8ZQuA&Q-O+Kxgs#nG z%EZCJrFja4BI3_<%o)Qhq4cGphO=tri)l*Y0^iq5SW*bJNCoT^&{<{c%bri<7*WEK zG5fL-qb1ga(wb_~nXR?f71`Dm%L^xV@)&x*cJnh+v=?>0511y^?{l^+ui@@JDwLZF zJuD-oxkTC5t?>+!ANpm1`VLNx5pL~)EgCchql+2JvjrvtC~z+?0!zCXs81Bnt1w^| zWmQQRB;oYg7ogof*c?90{Ey|u2fVjYvSryryh~aqF`lx>IhOelBbIzQG(i1pQ%Z=n zMvfMIRP>ak5ER_iMv00Yo|NvXI2-FkxwJb~st;!M$Q-DEC+DD0A@A?GmWWZ|z9^p7 zV8B`Q`Cn8}jNaQTcpV-lzsrk7`2WR&82b+oR}{G>NMO#8kVtR*uQKeO-Y-kJUzQXk zOTZOo2KJ8+(EyzxZJWx)62Kee5#{S3N)5T?*}q;5yx|WS6{=sGYl4uhNvz5i6YRZq zv$aKK^06~|sLT-&yoi=4%NgcaRzryMmV`~QD1hwa%BFs~f+sH|(i=T=VvqZ8wvca% z|H*bVukioImQSz;x;c@vAv&2;kkwUg{^h7Zsl23aisd^9NzUBhQ91?C?WDyd8sLs8 z%U2RRK{F>UD)CVlk>#DVR%G6snwe2~V4`ij<@ja#HR~WX0)`(I5#^i6bAxaAclbs4 zVXu5J65>qyN~{rMt;o%}()=S2==?p&o1gp|zO&E(#oIbbJ}^JFfq4MMPo3;I%Af#h z`w{86KQo{LB8K}qIW_&u9HrAdD=MIe&OJR_dz%QqcE4uTDIrR>GRS%H|C-Hw0%X_j zy(k-EQ)ot;%m0VL8=wIC?fBN{;qmg^=?!&N>&blmu-CEg806?T*R`bl+k@^UI=@H6 z@+bzZr`XpRpt?$G93W}6sED8PYF1G144m|*{7<>+{R5m0_ea{vspWn7C&b&r}%3Q*|X=wV1pB9(8k}PdQjp>i&omCKa0Cb}A zdnN!Tbu2^TTM`>rT}|{QmjlTMBqZv&^RhDwBTzz%2Fmu`MN~SLM zyylGg*&Cxvj;1sgT%p&V>40^iVWtEj40T+{+W5T!vqUL3 ziZzJ8kiQrq6sH#j2oczT;nqL&=Y7ymc_SC(G#NkN{yym!;*Q`6gOM3nJ0}B%*X-wI zw1}Cr2>}{)>6c?Jya+#hUoq<9MA)M~tGfMpC!EG_AVi=}+a&yO`yi6O7zhz@=)%;+ zbkxB_4H+X=P#`F>$s;)&jQ^%_q`X-*@x<9I{*P}L_%Gjb*Mw+FI=Wn%QQRR}u;hRF zHdA2ahco&*z*y-kw(h?n zvclR@`JF5@YfzynbyVPYr)^==}&k+W2!u|lu3OIJ@39$xANtEq8f$fY{T2IN?Rbyfo=%^koG6S}f>H=h~lQGB` zWPMAHH$u})TMA5lCQH5vv&AxHymuuXS_!oXb3{5iXB#1yEX%n};xPS)ka=)9807e? zSyM7TwG!3}QzjF6{CSWgiV4%b6Y0oGxam*-BeoI3(Y)N#SY|_WTW8UxRyI4iz3gtL zmy`rjLX#hCcbKCAW)6FaJ(*C<_ub5>(&VrHN$&ud-OSe$oixWqJW#zdd{F+(gg@=w zMI>RPtGzDggB*2{a+!1N33lP~0vl@kH)!|f*`Lo%Tn?c1z*4Tw147aAM%1}oAY znMF8&2>cKYCc(W1?U|##Fb|uUyIGD!z4NFzIe_F!Z0t|kT3a|B$$%b##y$8o8CNr4 zr$(%|>(Q4;sbexyGsxYX(F?;5Y~b-wGXHJyR15N;x}XQyvSEUk-0NYDMBNue8sx3r zrq;MasKZ6whcuuc_J>;PFKVPOYH%s7ttYaHw#`M@WWB$rU5v)RsJU4Grk*85kAVAM zsBd(3K{hAoXl-S$6zJ6a-773=1J@eK;ia~5Js^?^YG_8UUZ;NUg5d2o4Ioi7;3Pw+ zw{F!IjxS>}QZvY3xz-Neq6>JN554hNfJxEYs=y}Oy>agALw1${_aSMiDK2BAM~I;R zFVSYTB-ft9R5mse|3yx(rr0%)+TFikjT+=cp9KT>I&ZT-ewniL!&Xlpp7$cMTFCT> zPf1%{qA~kZE9-74aXZl|2XtJ3q3ebwxxN)1n^|0hYuLj5zQ(+{L51n}jThM*2J*h-7y29bA=o-|3I~WPcGV|iZIiFH%sSlzlB z&q^;ekGGd&mLX0j)8kTN+2WK*$S!w3y)B+=oP1k6k)6Z(jxfWFAX9OPYgYCIN0Pn> zoK80VF6DiBR$hPsaSzkck}tq4v-mzFR6=H+ZnTapGZ$J6xjd=-t3`IMu%tY@y>o}T z>Ww!$eXj8kRhSu8c&^-|OyjgmIjs1>w-}OH2!#6is&s(Js$@9V;}_OH9-VB{wf-Hf zhfT)iPrfo_=Qyt{E4gGWwf@dmdrM={re^xzCLPuOn2c`tV{&ut|I1|XT;t_ir={#% z`X7_}>5qTr`)fN4ukv|ydyTlNCP>5fqBFXz_EK~@8{U|{<6g6j{B-4oMf~Nxvw~0@ zmFf&VtW-nt&Ey%TDzCqx62^zmlmep15HX-8!-rBg4JKCvG5F<|zkY=~uE`CN;o^1$ zzY+D5&#gYWLq#fO^CGj7hkE{N)}bkGkOv%!0>SG$0!RQ7ih&^N3mq>EdFv16k36p- z_lWVWmpix?Xl@*B&Uib_Lp{X|@aq$Q4b<{xhDX=i5rd>d@x%ySJpRozq4Ot#+ z`nL5o1YKgkw|R{!)1-||2IeC|CqNufA2b37Kqmkqa6h!2K)kDf&b0iszampM-X3V` zFoKR=&&4gbZ5`5MJ?#z9X;t|Ss108+w~p_VAyKEiO+WesniQOjU8+93#RGUOkcAR- zEbs{sB*K5A@n8FEd=LEx&3%PYEt;c(R9$D0RENJyg{8CC8YX@i`g({U5PhtM(6l1% zsCK}+AI4+)71S5y&jDxT>wvNR3R+9B@&G;SAMB5w*i8QPUtT>FdIvL}&s%h;>CT)! zfDZpCyo7uBQg|g#-YLI%#i|=!oLT!LO&@+$A#Kk)h9VhMhwJ9&hAMJ^JM@_9I@UeYE*fBw+z()iGV z&6i*RI*l!tvP2*~rr;}4AJ80o?v(6R3wnf2WN0EXXksSV);?KsR^_V&!%jrC-sj?* zw|T4}Dx(xd$(Ya3Vkga2>8_R7EG1h~Mtf`lvR;M0=$Xdu4%=4XLPx>14Vs6G%fT!6 z-m6SKRvm|@6(G#)-rU?r?uP-v*PHnmD9c`A&T6aStaf|w{s+!w&MnZjJ0L1Ox)I`T zG0&LiY*tZ%?hpfX+Y>oIyO}o zK6T4#pEGTvxDCQNcS2529^wue1FiQRL==>PZ^xcAP4}V|Sl=6$ne5#EMco^rmMXKd zgyhf^pq2D+NsJibZb07~neKX}H#=>S%gB3!jYh@P`#-a!{TF6{`H zLK}Mn1^8n{FTq2f(CP3|R%W#xx7KeAoDJbZ+@0mByP{vczxVDW_NxCUl_C06j>owo zI#bn;k@t2!asI5AvV4^5Ry%NWt+Qgrp3@JnYiqANm(Xm5#&5RkgI&;zpbF%5eggLr zR%Q@ENZ6Rkxkf|DGKxfo@PV?16Czaa;w+UK2Ad-GQ>`DJJ2-#$RK zi{-e>WeXSLSohVlm|PRZvNO>g@ly}eE!fFzj?41N&55aj* zUl5A#`$v&qFmf4lx;Jgd64@}0N(6Jtf_U$7N^8*N%itH5mCu=+Ulp8*f5|ctZ=GiT zda79uIo-1W?Ku@%wR#X*E!`IT<`3n~L-*|LfRaw@y7iu0;*Z~7sKahYfe);FJ-75G zlXeTX_pW@+)U+Lv4gBx`MERVDl>y~lEXy~Bn|jYV81Gr($hy_zk6#{Bf%7X_-1kTD^&i32*k{fz z3@@rjR^2|Je9HLWsh(BvN3AOGN2UJ@*sZT7At2@-VEJ~Z{b=-G!vLu7_YdAm#hrcZ ze5mizG|zcQ{9awYP~N3O#a~2oG3D6iE=D~60W354^%oDL$I90Q{yD*ak`X!e=DpeV zPcB5U8{W%i|9@z{++n*RdG zqq}wwxCQiaUS>jhyMz9n89mM)xU8S{W!oinPZ^8{utP6p+ol$M@I2tBXQ0a0LM*Ib zj}q78*12Vbh0 z_jKMAGj&!%1ChT^Vauf7yGT+k(8N5fxQU~F8a4*3f@2J_P~JeR&upCH%$g1i-5BisRgtH(*Fk0Pcsas^+*KlzoXr2x0VIs_{d5Jw2KQO z4*iG$hhi(0767oKo`u`C24$zr2Yu9jWPi)(^ zZB00_ZQGgH=ESxpw(W^Mu`_Z0JJ0i+^PTg4_}9JCYo)t-UwiLrR9AP^g$Nx3o68_# zpSA5yu+zHNZWOs;yk&ae9JmLMV|+42r@fDf5xD15_dfytr~&{Aa-jYKaPya#yAFOEnm70d0EP4RSor_UU`P=F5QOEksl7d6kwtsi=hQ2N5_d+d?Voh61^w=PtGQFEp`eFhL_{>_dE-JN{qbdk#>_f9_EJv(IU z0!7fBp?nP<#}FuHs)KFNar1fJJ1KmT5UE0k;g=8OEC`+LCqy(bX~P4DJM}AZV3(qP z_CB<4N9bTz!UR+Ss$V>m zM;gw@p>yH<-G|$)=C*HpgjFHY+0fO{MGjNQLX)!X^F=tl>42`{xIyM{Iu^P*T>(nzd zXvkumiLF%r6Y0(7Gtyy-1{O07&0Y1FoFGC{GTQU}Aw-jA8HSAp5Dh?jJ4tS>jIgB<(bmx$(*RdUQD1N?8i4ygKIT z`|~2nu#F4N)v?=-kjE;u24)us6DBgU(#o&o(htSBJgR5M198JIqOV`Q9Pr33oT zri>{BU}wKneMdzuF{5Y;C?MtuAoeQHipEbfVnEA6$Ms>x=wzXf4@|LSVIjyN3KH37 zFefaVuszOhCpEqprox$G(E>V}FzaoTH)%#2>K)C@@o9oM!pYS_C&plOwj962!J3$a zOii^3QB*EOveOzPs=bvCjIo-UbYZjtd!dZA?4IG#dO8`lBlC zJF0Z|T=s@&x2KPw%Bfs8PPRLBtX(~~ke$<(?#kJ5a9fdElO-k%hAm~7yL9`Mom)Y8 zR@+!jtcx5JG%RsLI5ASRP#E^qDtm>}ojRk+>LcXZ2(3QMNBCx`j!S`}tm0M=r&kI| zhZj>ouz`Q@1J4|aEMb};V}O2OqZr>BEGHhYGbqgFTrQZ=wYCikwW{;tgfO_fj9eg0 zkUtDwqZDbWpkhECh!FS_sBRDvZLOYEt>f2JMWWh{q88NdMe=-Mv5FR2RsTO8B&CdS zr8B;l)%ijpjE1l216|WY`NBJ)j*9t>Z0Vm6I(dPoFPoyA>Mqb|uVRc3D{xUB6>`EM z5joPDr9gJ2qKXEXiurHkTp~d*Dt=v#q#5gM2xu@INkK40{83hrMJ^;TyxYiF_k ziv>OJKP*6|-8ienHQUPPn_zMTj#%wYHmSK)ABQde41_^lmna(Re_<}Ql%D_gCn@Yl zsjo0fa;FHPpfX$A_ZrEl52#Hx{VzY)Q5i5OJ^3AO;kkkdn)2i~3&`Uvs06NAH#3EE zCB&>=8F9q?z~2yOWPst}2sbHFVHWH{qq>T^uo7y44T8w)0)DyXU(ED6Hfuedk+XFU zLQV=ah(q^kLKH$C zANb2Q{~|6>KyxZZW#{DrT-p3RolO|V32>uir!}8UfP`;{*H6PxGyF`W_^~GPVkR{2lvB?)tbSyOr0s zP418(g|o$5djqjT#X3ahNYw2YLTD-^Wpfm&(sR@k1S(d)vm9Rl1+yh6s>kN`gkyUH zR%P=Y17ySw>t|FcKEw`h37BKQ?2m{4F=|8KBXFLJ4G74o1_B8CzrX%hw{SKWGc`3a za{hZDfR9dUfq4D-%M)7D$)G|`3J}rgxX%;{$dE4J!_)MKNB)Y61)YrQgcE7%7Qj#d z!CZfokyy!*;B((JDnYEcHWsI0gz(3Azjt?!t{0!SjVkujKU;7g?2m`GZN4kK)l0cs zez%u;dv+9C>*o)V_j|h@4Z2+)htmfanIBJ=AJ6ZnZtZWm)8iXDH!t@W?KL%smzTK@ zFUyykoojr&bb9FR-JO;A&6yjA^Y33b|^C<8FE!Yr}3+`kb7v-OJ~P25oC( zLx+*~mysK9Wo@bOdR7dPVYeSH5J6Hme=L10_8wx`YsOt1Z{|&3>Q^ZIYy~K~SA0FZ z-qc0bX#1zh_2dGPCXl>K~gFDUuo z`Y;@KN*vA&&J1fwJ@E8v?Q~l^aB}R=yuYxx_dMrVT}nt)I1;BO;JmHX(+kV_l^3=) zxV-OI4S%!#zIQFbFg?suW9z`VjTztgeVoDFmdNhbn?m!-C;2qP`p8%9bgkQt!>(p| zNzJnAt@Yi}f<_ZU^cQ!u(!;E)CCBZK#dU^kUTWB-?`y}VH3mU$B9FA^E4|qMo8QOs zhE6>%pDljhLCN6St#2wm7w+17Js%zqJ~sjHhs!I^X62vu_L?m^SHz0Nn$b71s_se@ z5rK)N_C+5@)3%j!v-U+|FJ^8$nQ4I{3}(Nl+BGu-$co{`xAv_24ZYE?-3&QfTRc;f zWlz<2`?rQ~Pu`Ag?+;r9^?%a2?wFXK-hJx;VaPM1QvZ z^V3}e&(`?bqFZ*U+4xbVV!8sAb!tW6>+;Medzmx4|u*BORX#?qsL!&nWT3 zPAdi(2CcwbqsL>t_iL?!d`Gu{h62L-%dlQ^#U#7t`qq|0Nl6ev3PS~fT5Rn2{maYF zQ{VmkiNn)JPRgm@WeQ*z(FYE};YZnE*yNh0pLd=NB;O@|fdGC=&GC<{Z}qx`{jW~X z6-Rdz?NN0iICd4Mba;rUvIGe49}OEjZ3Sbpd~_Jc)jk}RP3&Zihm1HFcEeMnPDW#4K}SRIk_8xF_&f>Pz2poa<~!1;FkLKKzkmPEFP7->GXvWc+ftRf%cm0mn_=q+2!x& zx6<0fSL=anV8ecB@Kmm|xT|5?Jk2qkd0x_DcJbAOmg9)Q^$}m(_Lp9ieHEQi8&|G& z@TDmt;&+iTP?N<)jh>{FdRqMmQy+9j$p%GOOj3EEsw32%pENc z=o<+L$cD@xFU_mViN;~VKOY@B^{U;*dH56Q!M!HW3qgVuYKverNLIALE0L}1L6%V6 zw0&hn*Q))60^ihouzSU6^`q@24!3<&d6>hf%JI_sQO{#04B}iRY=^>7YQ?u@p*2X> zK{zXsZGk5rH9?lboBcgy2DE{~g!<_G`>}V>dkmuOVYjcw%9?=?R8#>QsD1%95S{}z z!~r(cinoH>{~H2#391F?Ed!^UZ1=O!zxMtt(4fIAv#TfpH-$T7VFOf&pM`|HbtmO20r(=#cFI#h?Mjto|w1V`mGy%_!FN ziL(L12{-qN^PdI&LvyWoBj6TGya_0V0x0$=4Z+H>Lh2de;r1rxCFFqUK%4xvMk=ub z$k%(pk4}KDW&9p=_>>M+ZF-toKV=vGPQT&Farlpvzjh!llUGITbj9ROQK^6(T$I(8 z{>RblMcBbX_E_JaFHtyB0`B9!o35O$&llWn-XBL+b0=00Y*oX19Iz#B-Z|m7zXuR3&n=i-o)sW5Y#t}Ky_;9;z0xo-}@c2;+TdN1U@_dZOTa))V%@uF4 zk;awpZSD!&*M+~t9P#!b(7y5@xpIIRN5{$v!=X}(_imR+PvSm+Pd9B zj0CEd#&sUH@B2q%#}zVd=6p8y{io*Ll2X|&?C%e4a(zXg<}asHdD_;6FG$ALm#el? zeb#D(pC#Zlt6+J{Ng!7yS&5* zliT&=+`ydY!a9mF=FiPWl{eu90kbHviz^N&QHNJhAmQaBa@UozFe5zu1;6_B z0+D5#w>&;rs`V#>x#9>U;MCY}@%qRU}A zL>#Wv1ZMf=dR4Dk!%*e!>x$d^T^4WUU3bp~_>b~3U*Gn)vpw__db{=#dEe_s;9VS7 zy`NrRW5xJ04}5R6-sbnl>R%fLblGub%jsWRS1&mjaOrFj?H(3b(__9~cPR|JZ@2k+ zJ+-dq7{(Vl^o!W0B-HVe+vekor&sMCO>!Nntybks-%epPfxnW&o)|PLa4B|<6SQzC zvYgiT?A}~8%hv1MCL*&4e*sUWe) z^O@p0PH^%K$-W>92?e*_Lph8h?}895LMb_q$JD4WLEgJd2MM^Ew=QAfm~P#wtmWwi z4fD>oD>n3B&g0&exQ)F_cNVtqrQ~!aIblv2b>B`cyLoug0_Jsc$LfPdJzc2t(B{o` zkG{73hA`CcP|vsN{Ows(eq+h~xV3}W*D<~YgLd;QD>aw12=;Jfwpm{nSeL)^s`YG8_Ok&$c=cA8ehiVK9m*#X^ROf{Bxx;>RK(ShEvEz!28 zG0ox$Aw_Y9-t+Z%UaipL4Rd-I4125}AOR&yI~AYTg!=TB$SRk!Z6`lw-|a7xE(Ws( z15W5Cm-_sE&)95JpD{i&7lk=G#nk-0iZ@r*gLWUebTy^-TaLsG<%g4`|fldRp1dqc-8I4wcmc=oFx9f=~Uxj zy>qXOKYqMg(eF-aNOE-iAu9E|H^YSd;`-oc$>6h-*2w*p8>lSVd2&M4p{i)%4!ske zKn_CdPwTVBB@;5Mqx^Bvl*!_DYtFRBC9~eO+uM3W$IWbd#)?0-E;0Kfv8h|Buy~Ws zW{c~5x(4g0)yGOD2l!PPze@1gY(kgsO4qbv-@V;UihhVFU}!967o@Z|SdJ0YnEQ_L zsBC4Ye96whzt_H)EvNE3nH?(W(29-GKArWLyR&>#LjmVOSAr zUVixYguQ3lrC7pVIAzMyWO<^;z1TR)uuuj$wk25eZggF{krcVfaD0pcy?djA7pNqs zcY#6R*5<;>W&iH*E8v)=%hq+YSzyXqC0QLIY}MZBzG%;Se|EVt{|yS+OC4`Iwdd;HF$Z-r+VG#e+H~(h61m2okp?oBnfVXj83El5gxuoyzcJ$rj1~}` zSu^$o?bqL)9sZ0CZ(I(y;w1hNM*6bnDH}D)!~{W}Awxn06)wd`NWSl%b_a?SYGQ&U zI@E}drQ&ET`w{@>m55u5ck#3v_mmNx5;iYIT#|WRe_0BF;aTtc%|x{hu@g(9(vdky zJ>K9+Vo+o-Hlk(=x%4|F8D7l_PAUx0#CvNN*%@*&16kW0 z!XWsrDnp)d8jm{a4EcS;EUNa9QDnTKX)9C}5^OjSugy7BX_vavHB`>DZ}JT7o_U-eHahz~{^{eDKzR_GqDz&%-QS5xoi#3zHu6#=zM9A{_)hR4= zf1U~=^L)=y+a2R1PQGqv63ci!%X>GIM2MeUTh%If`Jmk5FsoRpDtVC`>(>FI==p2$ z+!U3MvB6o_rHSvvTA5#Iix>5<=0lEI@j~r0$TsYwO9s0C>(J0!;P}(F97xVYj9POK zF7CCTO&0{tBe&#Dxo<-KGu*@ZHu7Ro+@Q zZZBZsf6@^~WE1*Wxgrgo&Q`shiW#jUZi@w>?WA_vcUgIOwGpxqLk=GhIrJA{mNTo| zIz79%YT#$UJCZg94Jhck_gfNV5PGOx&#$)`JkGLb)WCa=z^@=*o5H&LLDG0<_<(5# z6t`MuW1&GY-<#JjE~ON|K{lrN+ka!up{Kpt@n~B?Z}Yx6Dp|Z0+LrpziNhm6I9%2A zJfkFPcQ6=E?pZTH@?Ek1=p+110EV`8u(m4ITm8r2A|_QWdG8{Wg(SVE^Y#MHEKk$( z0}aZ!zbHUP1V+=QQ27jjwjkNw%D=an32M1`xsuztpEClc^5@9n_PPcy9N2}N1t1rn zmrDF$A* z&2?LMWNYO>M+6P<+JkN7ScOr_cD5s4&P9~|!Lcm&!|y_lH|JS*lwOmTn5F^N36N3; z;sTz_xJgN-j;A(CtFIFnVr__UJyZAggfpt6Hm$Y-P!pSgnoz!&RH)eK8Y|(-QvYLA zZ?P05^=R9Y+M65O#@o=*+m>Tj!ipmGXOZn^GX8GA-=NZatNY`PN4tkibG8?I*!Fs0 zYw9h?uX+o{CTTX7?*p#UGb6$)2(ASi7h zqI1t{<4XCO0$FuHo$xU(*I< z)Pe{tZ&6EE;61HUBK;E9lG0gut-7^4KOHB!Td#uXF}731c3g#al`>rgV^;-jC&wSw z5(BpZe*{#b>%3E?d0b_HfXNN{>tP8fDkUKQr0Q$2#I&asYTN8il?j6tuT$IO9SbSL z&$9)QUoP%ea-LS;9S16decD!i+KCEM)g(La{Zx}>7&`7&YMxeUprKO%&7EvO1t$8X zo>s}Ln+==TV&!h`R;%OK3Ip`l#PwJfnP2Yn#~qv&@XDolRr=_nLOm#7=x2O=l+2N68quzM;ldso9>A z5r}qGQg&7TE#Aa>0E+HbH_&@7g+X?lJG}QlVC^QQ?ckkyX1HdWTOjls;ozw3&M(&% zjgnJcIcqMmOui)caEWlgr_5gyU^A#5&CI{rVP}NNSmemGMRT0K(c5_asnvSRK-heF zNgecKUn4MuMw%N+h1SVBSBpFw{q0Q4_mQLvn6jeZHkS*x5?_6g$YR~#MA`Ru{wnkj z%}6u}s{DsURqcGXJDc4fW8jBX;>m+C$D-xeE7v0{&*$1gzwe1qV~_C(FM|6dZ-t&O z4XLyRX}u;^eq{CNw5t{i7WC)OFbD zK!BwpER>AVdB;83{JB4;VpOKS&r~rGt*~B^2gZeZr;|58RqTfgy+b@EaPApuDpVl? z0uo_i96+6)f#0M2xiYp~HHm*N&jhDs)e zOgsWeasMMU3DxB3_$M3`Jw9bbhV~;8ajnjm+LCe|i}0STdRHy*A=h$xV$qphRPjaK z3H@LwBw`@ES_isg)o`Gm88|qmd)5dM4r2`GQG#>%@c#^n1pKd%rz>WFkS=F`h2(w+ zU8pnrCu9}`60u--4}x4zj`CO$5r~r|l@%K18r6L?l2D{d8iT0*H0ltXtpNmysQxuc z5W-4(iZit+kiI!YXrAYZgc2t}ZF!09g}IwBm|z}y#yn+M9mu{-C=`(6i%>O>==OCe zRQ)rbFxXHq)MI4l_=J7@lF4C<i z6Cl|Tq#6>z%28*q8%}51UF5?fe2d&_J4|ik!&LMLJ@&{5dqU>%$%07R0Dii z5GYE`6tGl;@*TR7AyHg5&!bHXtPeu8A?X@RFdT>ogohRQ0;|Aluw18M%nBJQeWuOY z0uA$NnLQYan>mEb3R#0CgegcntrrSe<8>qt^r4z88wME&Gb4bgG>3bZc8bEdJ7WxR zH2i1PENX!PwcQgg;}KtZh=B16gOP_q?OreZk)Aogcm=!0qM!RY@rrJtXXqNq%78|U z#6j5VZjb#x`ACa^8In8GYiogvzc;}X1;)r4bRh-ssbI3ShG0b@1q{Z*aqC}d&6Obm z`{!d}8lG6-itBQ=epZ<8|GC2C@rWOXZyfF)ZUFthY3r|mRspB3liSbV7sLTHV3766 za7eg$Prmwb?=-uX-LE7nzHhB5;10b8ap4eSIEgUuW0{HHzl>=ozB~wy;r9U@h5%FZ zUiJs{`{(u|6TH(PybKTar%ta;YWIcRrmT52C^&Mnb~LhfwX$}0NREYt$Up%mbiL9G z;3l}#ODDlWLt*@yG#qvI8nr+*(n9=w9w%9qt}V6giSTXs+hP516k4zB3bod;h;VdK z_Gs1NK(6k?p7Y^!&1-K3$6N&gWJ7O9p<@w5x9(`xgj$WUqG5%4J;lMsxK^+q;zt+q zhpw+QYu89)?F4K045r9thdVP|b!;4beDA#7(ERZF+2nfg@Yc^^f}aMsVq~zo50u`% zx1Pwle!u!-(w1}IBoTGHnRGvR@gesT3{~Y1Ajq1IY0?u{Z<67ptskWy3Ojr?Bd?x! z?m2ev#}&R0VWgv{q#t5n8~tAMX2f>4uGvyDv%Grd>hPVI+?d*~d&V8t*IlMRHb9H|ka(@fp$yJa(k!elFNm?kbF_t$SS7)ZI+o4%aym>aE z>pKlw-U?r45Fo?WGuF~D&CoNC%D0;*gc4UjqLkVC=28q%liPp#x3J}{uH{R>8BDEt zRWGq~pScWRI*uPTU@G*T_WY}@qH<ODnXcrna_K}Q5|EXQyzoYzICK%cw%}AnkUoE(Lo;($E ze;Z~N7S{bJIzj62P3(BTHs*)odw^goU(YLK_aSpqCb%!dOnNF*7|dCBdrLsyxd-44 zoTlcYL^Ckc(Xvc3unx_%2*>w%yU4n~0bEPV>`c$RdS3hUTvg*?)7Na^r(_b@`JOP> zX`(A22CV1)wmNPO^C%zG+iDziL&HH?CZ{)=rRsx#7Q}qOt$H?!2AtCPf6;S-Z~&{@ z``>!({IFz5n&D4RNwt38-Mvxo!*f1(Dtpwq1LP z@N3FY0{6iJm1|ZhfmR`h9}#-L>@9gy`6mUnAE;-MNXlSjp|7KT!H`;#1Xhox_r(KJ z#WVC2m`d36D|z9`D*T{cIz3BYfc;k7Z6Vjp_};cOzR;TojA`DeyW{lM#Hp4n+vt-6 zgi4twD2Ml@-2PZmtu^>tN++Bn);{xy$=kP`K}`KvX=q`_28?2)3Bt71wpSM*ADR!$;igEFgc+S)eX~4sNRt!Dse~Depm4_6Kd=<_lgx+I^;YvMYv5*Sy|6@CJ~5u# z`XZTn@X_i+%uN|&i86#(8=Rg_feN{ac7Eeogqt3Q%OFjj07Edt&%#kMO0(|KC|oF} zP=uKSMEv**;~xpu}b^jK?lm|TvN7I3Cx>mq^f;4#+LiI_@ zmEI4}UzLIsx03{Zb^_M{(8l2F5iBOk00tNTZrYV*?n>nb;PV+Df|V%4AY{2vJgTH? z_<`&dWP&K;`!4DaEUdr9IR0P7n5(nCSTt2orKjr(DU-9jXle1BXyDg#U15}K<@V&# zSY?BDPHJgm{o;D|H)5u0LeQy}O{JmVSaWaKyqx2z^U|pDm21CydmG;s$UH}EFm{l# zvq)$BShZ~$$%&F5p1V~z4w=y7KYK{;?>%u?jL0p8HyyyC6ERSBtP8Eg#mEr1hjrmp@Ud43SMluKwI(3NAZWqXM?=kcvSJOK$gJih~*Kq~u2E!ALLMK=Q5IWRNVc zBkMfqCh&$GgjG>jthj$;IwhKvlEHrl?%=+F5|`;I--_m7@sH$5Qp?Y9>6{y7RYl+f zx^L##2Xw}@Q*<==zy}A`1W6c4pDJ#sneqNs;Q<_Ms%stIFQxkWe)Fp6s{D@*mAoT> zj(4Yj>3APquV*Iye~@tbp_%13c>k=JqO9xlmla>qKCO6A{ujhUDZq*^{|_sM4E93? z_aTnL>Jx7)y-7XB#;;eg2bV3o4B%S$e1Tl6;@)U8EoU+&EV-nk8f|Raz}mZ~HZD>1 zCRX3rw(8{BB@e$Qm~-b~^V$JhTFM)|d2C}cip;--}jA;wew9_wz!($C#hqGz< zFUBfyXjdvxK?auTa{orM!lVvhLsR`px5O&&Z>~ya!}*mBpmh+2@KVtP4%J4I;6$2^ z(I8>~G00lfOX11Fb%AQGI+e&c%%6%Hfhq_JY@0O-lt*AxI-O^TmKeLb>xv)LO9nI0 zLxBz9A%nG0tT?F$xRTIz36}&biOS?Du18z2nl6n;*`!Z}DtAQ9J*KepiFMUx#;POP z3`HOS!K2aj?~N8F(XCXa84b%*5x57iL#6fQ7RPKO*(MP|2e1*Knn;)I7ua1WtuloH zrim!4f6Y3@RBrm3@~wc9b^u#GI3)rwaWF+yUa}5`)j%kmA^ZR~UC+sIX(ruLDJtTS zDg&N#I9rr%OYx@T|00WxYW3cw!>6VmAYDU9_0DRmlX;x=V+VfD&-&|;_Z6pG5GN#K znSM+MQ(3CY{^aNY`au<%+D+)$+fr~Re>orGQ~&R}0TeHFV8?cKWfuv&7GGqh&G zPtjSQbADNgR_i$}Gw+~OwSv_dc@pfBnqN>G8NINe z>2N-3D1~CET*iv82 z_6QN4DAM3XmG(%=R0dBXj6KRFmEz%4nY^&%@M_#~qhuTjxECYyRCVCcoRpKhRN=;w zES2gg#L%E=VU)!*Bl9AjE2R9XO|L}6(a;VtOMk?xGDVr~rBCzqTIWT4Ucac-mPJNY z#3DQ?^CDKqB})yrr;ZX6CS+{yi77?Uk|#zL#mItCRFy_^P`{GaQc9wk&;?STOq;-$ zr}ZLYeN{~}%8L+J9`wKrZQ%L?)(}7qxDvROYh*|BDWFl+*9B3VG89m8VtlsLy3f8c z1S*4nQ=O1XE>vNnR8f%}{BP7k=>J9~Lf^W3|3*uqXKP{MSk4StX zyjzLfF7-rC@>EwE%vxxVpoTEn{Hj_<@u&L7PqA4yG_t?=?tymkAN%FiQMe9Y7h-7d#mG z#%tBzm&i(~{q~eQR2w~nP{tN{G0h+y&n4+B9ix=?!Dsgg%^( zv;}R^ZAq2XdeVPH8lcZ$ORlf$6<)Q5U?8D#3|Upb?^5}f_*_jOeq1{OtK-RxxTwL8 zOM_f|Im=B({Wc2(Oo&4uAfrr8^PBlK{TO#_!Wqeua3P6eA)2B;kJT%wT$N<*r>dh# zz=KIW6)Y=5D!=B6zj>E2KPZw7cl_IwIoenv9?{-?P;^PohlNwZWc$|9){fx*=|9k2r&hkG zUL-sAsu~WIG?jvV0l_LKdkPR;sZ!v{+NvEOMo6jlqFSl(_M*R1MIutQ zZ-;#E1_eQ2OC8b_wqOMZI;`L(T}1N<4h4ZU>&BSsvmc*!bY2)nA%Iu^vVZ&0R#$`BPy>kgMa={no5M}U;)QU)IYr#f1>Rez+&g@c8%OAmCBrij4nvpLyK-%(3f5P-Xe` z1A>hdHY}8(!!rMg3zRk)Yj{M48-vSEDRo $DmXJxKhE!0Y!R=9IZuDWX4{6`fO*llqUwa znuZIphCN&Z%Mw^wl@m(}(vgjVCO1u5UFd`c%=i}3s>C8rNH9#V!eVg-~k#1(3KNC@|)eIuM7XgQoejj4NK4>3mYbw_ys>l$)^ z0Ld?pHSjSA_&SUdVI%I12(O})cNcNnEL@;bw)!w=w#VpY3qc^Zd@h{P4B4+ItI2GN z_}%|82%b~y75Ex*o+;uH5aoS+<=UFpDD~2~Z6jO&5J=$fK)l~mrmi84zX=1%hVcv# z{KWRVDxjQD*WqMXc$y;$7x2#|de;YQ%YYDKG&Bn#lLU%<1l>hUYjB;^{Nz>(ToNuo z_^!__j_e0iG(#W^+iyQa|82mr9q@lOvLB<7Z`m*bZ@{AnJE#BWo}Z;>iXKHU!xGK= zzLoL~2}eJIK4R@6(j`^^UV}(S_<6kYmP?cLWdZ0rE#+|i6_0%z_0tR4Vai>^Jk(!A zfEm6SaqR(~{f88ifX9BQza0~N&9AtIq{tY2N^(C6;r{pv_P1l0Le(Am?70mB^mx~| zrPR?-PUzQ=6PjL!w4cOQ{8c_BG2#^fWGnx!`Cx>RK;{V^`fn}X0gpkk|5h^lC<5s8 zlFH+pz5)mq5P9u|azu;$e1z-v@x}jIglzCx8(db?pK`7taqI!Lv5Q!21dyW$^M-_Q z*pQ&$u3^iui`c`qi)cE*003-V@qMyrT5AYGI9_eaSJcaP)Rkk2AgNZB`6k8#FA zZ})f4$}|b;_Y2W%)_bp;hA%cvZY4{Y;?`?QhhsY@FHX4gv@UIn2k$W_WDX`FhrqSM zYlvW)#(%kLbpJLNrbfCvK&jEeb78P*bhhw;HtDNitvAG{>sD{z0Q zUTqJfC8oG$_v4-NX6=vwe|Mal3*9zlzf$vn(!=g3qrQ4&Xa$@)R&_nCedR_N-gLCj zgIB733fA6=h1u%8#C1|O7}$m5^!fpNEl^mS3@CK!YEFQva|WJ#$!fqn;FMx7@!-dV z=WU)pGY-0JZSl-Dk`w#$A@(EM&H;j&a z$&>Op-sgVWS%(%#U; zV0`i=T&J^r7NPUjA@dU|iWvXUS^d7Tf)}%L>FrUOjVbRGmVKynCO6FD%h+RaQ9Zv7 zfxgY^W{IGHPI)lW&e08#@D; z5U=_Gkvb)9@U#H5Il(@QW-~&x?8AhUEI`g_ArpDdN&AdUmkbZ-)E)paxRN#oo@?i` zXh;(pkgP*ruF3e*X~;7Th%a#V8SSU6Edk)1lRDUvoRbf5g{AY}8ju2=c(P=$RBLAZ zdd_ZL(i=uo#arxCM)X%l^STZyaU-gF>EQC$2@4^OnkDj-`;&&U>)y8}RZh-KPID?P zy1megI5m}p63Q|6u5H8|j#@@Ad4bgZS}SjN1oM|u_&iH6xbj2`GYsMnwf=pmyn#jD z5Ob1jKfcx<1RLA9j~5J5XbLrr*}$=LWS7U%%D7V@R2hhvL2RjS?My>r6{NFR$e!cD zbTBpI^-L~A4`v8ky#9LF5s=qCco9V;rT2pJrp$QX^CpBT^6iwRV)*lV;OKBe&-hX0 zZS$&zDi(h-W+>(mT;GGKn$Q=*ry>P&-{n}P@|%?ST!xa2cv6JsI9&~8NPa?|0-`C{ zPECX^K+7;!Yj>jaDJ^>z!}GfZ0__%pCgxlpzaRhl55qZ#!@Tqnq3UdfDJ`II+eOl} zi6i7`meqpH3BW~Kd_5NIU;yr*u(0D>x}&g?3Bs7vPG1fc1##6lCBqux9yPvi5l((<#C7BFd5mInhIi7T`0 z8KfT+wJ9ldl;p){-S^SATB<3-!4LKwf2kf6G8TsTynDBb9TdV(2yw*+ed7U#{IueL z^p1pMyNuU7){z+zaEx;|chK1a*%=|D1rZ#ptQl;8L|eoebEhEx*!gNOsO)ST?IE z%PKUBvlNq-gB_JyXE+ZE6}3ntuwU12=#ytj^v;==pGyJFMR7zxbCKB-iWTvoP;X2M zb;x&ObNRL@FE0y0j~0~Bsj;<$xMa^gRn*jYRq{kbS9 z9OJAg3M62GW^-oy5&OflS)vDPtrg`>3+z$r!0DGrgt(YQxTvVAwE&bBG(!HSBr1xF zq%JS&jD>X|TId4}Sa?uUyeZ4?Qa%u~tB-bn2OK3DTsyi+XZg*F@;`rMfxV9q)I#$$ zpb-8y3-shffHMc8|8~X^1u`F_)+xUpGX;K33U|VcrP;C1la!LehKfR<%Jl`iZQuU; zPtgRqEST@XYQzEKmQrOZMk}Jp?cC0A*Mi0IJWi{bd@Mf8-nKbzn(m>v2VGJ>)V+bu zyG)N2Z%Ve595$gIQd8qtj;vN^~BXFMT6i7<5+GCa7p53l<`JdI=hY?6=%fa zMshwO(0M|_(m(6nr`@xXs3=Tkp{*2_KBGG>F0!6H<6?b^0*?6H@vS3uVs;jYB_*Hd zm4E>caILkDkPTm;Rpw34z1K0gsgOgVPLJ z%vX6sRA~*gY({jdQ2w|mNh-Ss2Nolc;~|7YnqsCln+qcGiPz9<7Q&6S^-GmW?B|!l zye#mqb;(_GlZ1cukTfu?_7Ji0qX0)%j&~)Q#G|gJ_7DRw#2!M#q|JWTW{Zgz=%Yln z5#Z9!#mP6Oh{^LUyRAlLrdL%=)7D1=aESZe*>mc;@l7Vta5M^sB#8CnT`UsnX%mR$ zP&ABpglk8-Y9)`8P@5Y@W@6X<-wbI^dwYrDk43FC_g8Yc=@s+>#G2>Se`@aY!u4Z) ze5(r(19XFcplha1Q;ri;-CzZX0gmR#Gx*4)Njw9d$gJt3pak`9O+*BA3rPyCabgKT zH+X{z9Ajd`L%LY=EC9(E39>(7+DyY~5CXmIIvz^HSZ6n%HnLEX7Ytpq6^XWf!|DA57KtNot z8x#oW2pR|o`QHPsOq`u9Y|Z`}XVs*(;rNRawQH+HzjenHy4gW?kLsATc2kbr8Zbow zP`wY3%^gPqT)x5<$y&r{ifP*U8QRU$RCkxtz_0>C7QAa7e6k@w3`5{7kSG-mHT&)C za*6e;0V`ulcmyMrK0miVme0@k^F4kxeLBl@U{56Sg9Go@Cbgo(?^G>4-`nC~zyhtx zxauCuIKIf=IaLi6?MG;6DT<#PpnwZ8pRJEP)h-dAe~aRxYhrb@$(EvCw$c**vqyKw z<)EEdL~Di6$Bs;ajd@5IvZ`#-!SSxwNz{5T?|O+=w>NQm-oG8?_(xg22K}h;2remw z3^NF0AYCCYg5Ul%s9<>o1GAl`&LQN`U?%H%0-lE=9^Er9g#6h)3!GuTJvuG}74A@E z!qh!F>Y1F3;A|dG=m;D$Q;8>w6DTtNbR7j23I{xj%M(Ec0xbJ{KAIdi{eUc54%|wa zPKG4INfYvW1f?k^j&jYpXF$WF4;$lYXI%y%D_@z93HRbiqeX}+}skj zmIQqAMj3nD+qEDeHgFNAPl`0Exh3e5Z zKE{#Sp)B7Z9HourA?3RjxE==MD_s!6Z~N1dktIQaaN(!vPq~3?7%5EUYN}Rcm!ctw zPZho7!^@F0^~~>AV$!EygSOdoCX4avaDS2XvdZ`ElO}+TIy=BXGX!aRd%=9KPB>n? zL1m6kI6E6gmPHS27JPax$FF^=4~%y8*E=I&EzdBJZI;L3n^x2hxTm`Q?D zv*nS`63@@IE4y+BZe`fU96jv|+H-)#l*b&m|Hg;ZcoTu$6zt+;O7nxGb4y+%H)+?h z)$jf*v4Y@7nMswFbIo_lxn@Wp>m2O_JTF za0HV58{!xPk5(7qkOM0?BBS1qh*(sjK|y+>$0Dl)TIt~p3RNZdui^Pd{4A-cx9_TGs_b3 zbG~UZmc7V-t{C=1av5Kj`y*#lq&K<&!G3PKi2^?ExyX^B3h;p`Ds5LP0NKPZF(F7*i0B6PvI+M_(H>wpcm;s4xizPtte-)H-8 ztpgPxKvJw`z#r~!GefBD=AA7V6I_OE7!6TE@jykR zXPm(#ov#(@mV-Yd2!}@x5F3^;*GnBCeH-K&#fr+q#aq$9oy9>mJfW}+7RcU^|KUJf zs%wE({aQ_lKNo%7Ldh8;-N!Q40F1NbaUmpuOgA|;8{548gYGxvkrIef$MA2vX($uF z^qY4A5#Jnw|3Koxj;wErRFR4QPE8&7;THu0sHcP9@;7 zTYJ8^IKab077?Lay&tXF+<^bLV=r8JPDUU=Krb*rKq&um%+$`-S=GSM+T=4M)kaFM z?Jq{$p-Z0#%G4}r>%=BhWa^>N#&HB3FQ6~DwdiZ{=Ynl>FH=0$v5e=nQtfv(ah~Ti z6Qsa_P*~0)lAuIfAJb2UA9rtfS}kt0_2i<|)UB62*!Hx2piG6{`#c2Co#UZKRMAgJ z5`kZ~(vc6=n;Ep`O(|Q|QrfaB$YH&`DqYr{bW z2sNVE@MgkK(j$Rd>z!~s!Xu$Xu;bK;B{6rnJGYWanPUPgn4%ZQY&^r)R7VVG#RQX> zwM~@{hZzs7aDJa-GGR;RiU>f9fylG4+b7)B4eN+67Q}Jjo9jGs7*~} zuydIfzxxhB5K9-S0_Giz5G+uz-(SnyH)=m!?ZuQ4yYpnHHYYM=&8Q;HGFL(}Di6Ar z&eWGmt)TrK5fhbB3cOO;J4HRyvf~PVK2c_;9=K3arxOsubsVH^J5&s(lFR&~vc=f4 z^9$VVTSwfJZUDDCFz>C1>HgO%kdp2DUuRG*o}KVD)n(r}cw75VizC!Mp;>Z`JY1M? z(!|spv~ET@={)jp?LkUMMzz&CTn>7^nPl1K3?3ivtj+f$o?hRU3^V9`C}FTa%qLj5 zb5LXaWcV_dJ7|p9AGaBA;9}GVFE|6VY!VIq0fxla!qm?BgWTe)&}mV0_}oY~g?TNl zdA6<841)PcH_R`?cPFNgrq*t~iSAr}TUog8uhTc=C{o`m$8Z=uc;YrJZc+rAb+=XZ z?e}|AZuwJGhAV4#?Z818ChIN_e7;Y)yry=MpO$#aRg0JVT;u%rA(O!xa8U05YwN1xqS)H- zN|#FuC?MV4ppr_LA|aj9T@vC-cXvuj?+UVXhoqD=?ji^XERE9ft={`xulu|6$INeL z=Xqw|nRnjv#+h^GnX&sAPVf7-n>^W@LWI%YI1hJKy*$!ied2SIZt*SfsBatU7j)G- zeftaA8IWmGVH?W6Y(E1rYQTfwQ25{|HjF|#>Q?cH4cSu~xWnk%AEK`eH4!A?HWIG`4FAo+RG1{pN5U&EZ`836?<-@0o<=>6YjQsvE!nK;x0hDbCMJ} zGYNIY{8CP}PKHZbR48Y|iW~xf^xSLRnxVoV@=$W@>^_M=-W zhew7;S#N4%_66mk_SuggLs_=A<(Wd5d&ZENZS9F>)}^t!;zyjPWIqBbCX|%o(Rzh2 z6Fr7v++d1HDQ{CX-orOMZX;MS^+Vt0YJB1lMcmOQReK=!iS3=5b>V_zrp;}t%;%D7 z*9X4uor-K(YzzZ0Tm4DMp&Qbl=t%eqR-W^#sMORx8HW^3DrS4RGQ^hh-m)yfzWZc4 z4N(ijC7Z=Izx{@|mKltXODo{dB-4P`*^@Y#{qgWyvL0xG0K35PCAIW3^VsROm#1GB z(ndQg6nKpvoUKlM?Ss%>8e+8LUb>Q8t)25;1)Q;;AWprVbN!)!{-ln@S7%%GChdkG zBBdS#B=2&6c@E($d3_kh`l#V1R+!RffM0mfy>-BrrUr*eE%WUf=6rODWhRgAyYB09 zm^^_$f-5?AROw$o0s$vzT=E9Y4Z<7veM;G(h?)q+zSPi$Ro~m@QBNlFD`x@q$&^Y9 zY8U%xjk8R+v#(v+H7lJsf_;;u*KYHCE9M!VQ9L05au&nphpW|JD6oX zR#!l~-|+_BwVEFL1>5j?)@)L2?yW%{Ipz23(=W&Ji(I6I z(!DW+=W)N}Fe&}sw0=pJnhK;4{(h_=ahJ?sGGcmaa4jUi;P9o^a%yr*Bgf?AG;6W^ z_JM8L;|NL(#W>}!>Aou`+a?>tF-2XfIcF;w8mWrxYVU8`vc4xFSt;dH!#us8J?a*% z>y}MR+%zWB_blyQK(SX{VJTl(+SqYzK~)Red{>}zDJ*hY^p?kRP*5EdUMj+6+Wp2c zW`#x};0IdcUEM0;#OlR5ht7#{yDiMV;{>O3Pxpcn^ePAaRQscBFzEzVbd*{@EL_Ym zrH*I+=G$_S%|Yf0sJ88t{-l1&v3At@0MU|-kZc{FypK5t)KkcUquGOjXu&fFosWsM z4!ziO^Ra|P`Vjgv&#Mm0*JYyu+Pr&fXsH3a&ZX{4vWjRFBY5hZ?eK>lbJ5|e8h)K` z4#r&uhN@}tfcD_4`R?ze-2(wz?7#OQlVw8*3at{WpST*AtIvi)Td`f?s ztOKY?XGaIfUhYZBotJ$OPh(vkbx>K+R^9r-Z>lF^wj>Z-*EheGo&aSe`=L3apUG)Wn9s59CgLzZ|3*of*f^}?-P~@PATBT`95=t zwg%evEXpu~a^&O~iVDn*K9y~)T7FZknH}S|cNCwd3R{w<64Xtzh<)afv0BFQ9Jt8D zYs!k1!5C}F#za7y^T?bxw9ceaBdvBEEiYAf?Gv-bUa&t0Fm2Cvt9S1rZ@{?tYrg5b zd)G@DT)1v=VwVu36YM`5zFd&RObN2V!$Nwmq{yv>r@M!h+J5 z++Fbew&m6D9v`kr9}zKPSTa(yBpUYg2Az&6cls zM1nRa>q1`r-xbSM%%$6XUCVNmnFGp=NiG^6$kMwGx$j&vYS;bMJ zD;+g$)v{(Csmqy5v_Bsxv{_Yc2`R8SQgF&Y1pg_rrkk^i`>&^(6JPBV#7FkXyRGYB zd4T^7h7DP`B7og~O4*3!1&G;!ap7T1`IVoVgs?@QH`rrm>K?V!YOs@OF%Pdr*S@MxSUfai@sePT>t zZf9L3T}Q}q+2~oG#$zaVyDKqWt@_!ve+V4B5>W;34ZjAqL?W2Jo1WZ<^ZNF z;lKzocafbSgOWu#t~Hh6>91GEY3nRo$4_wQNJvJ;8eU1;UN2q6jh+)POkk35KuXy3 zB<5N7XXh0&q8;wVh=z9|$6nRjONOjC4wrq@<)mix8||I&qWNN}8}Wr-S*uqt!~t8Y z;!WX&fC%2#g9*=UO_+s_YYQZtHN5OgBtBko6S;i|5oBRS-jzjj8IwhiR!CQKeCxxs z&b!B*u3A(1{EXTiHVR4^!w9!S*{J-7;VO~C^n$d`?B)WTqqo9K1I1ND#l$zKlhrUq znU%IfHAI>aGeCaA)VRFtuQGzz1~?(s7$camHj#L)O1RE5T%6)BPmVqoiIaQ7 z)EA^M7g-cUKF4lHwCWg^lU@LIm&P}yC%k&^JDU~&iDDEq>_1={5@{zqX(`E5L>NO^ z8x)H{)fY|0t)ku|DJn03>H2lCW84WR`1AA!Zi<>2g%m7oStFmA z)D)?*eLFHVUW&%W=KxKGlcUO5I4m;=4!EXd8THk89!Ik?k$2*sJ?@o!yjNJW&xr4bE z1>bmku~@3DV|sJX)x|svujQh)&!OfBmt*s0mg;XYy|J{f?+~;qDVVd49!Qs`w10V; z25wV&G4s6(*S>XZX{U8h)%amd>z?c)cBhf9(T9}~zel`SPC4Wf^R1+9oe@MrUT6dz zmYZt&wQqV;iI_$?E)REKU0nlf??#x1izSum2**^P-P-qmgRd^I{?PX9IhtTuUN!mC zNTjzL;_A)v_4C^HaF#~Fod0}`>#oL_#jOX6A0-Z;Mxy69ZCETOt!q`TSZj}k5bM&J z{tp(Zn)jjF))zQnpei^t#wbTTspIAF`7F7~3tj(3nwJva%45B*+wj%o3GUok0%G6L zG&VebVYKf7Jy?>GFyazS%DHm#{p!#NdH2S`+7wK6YqI0M;1(F`Xs7Rpcx|R*XZW4W zO)kFfmfN!(1n%~x`BYeTrgDPA;vsXa*ZJ^k{68($OjdE*4ANS)U;+R{NQ?DrR&g6E zGfOMCUzVykS;c9Ak0|u`0cJi=Z4EP3)g7D!bVix^t-z)S0J(9JJBf1fwc5w#uwzS_ z2W*%(<2m+2cdx%n6%zA80yvanKTF(b3P*NrB?VI@%j!P5xLnv?W?&i7GJ1lWNwFW% zd}4RFQ!{{=AR)6)t6(T^74=CJ({X4-?JKJyncc&q`wSQ9flpeBe<)L`^4lpdMwx1==^+xM&Mvn#vLE z4s(p!2~T2n(25`QJ< zZ+r!Dd!hCtp$S(vC+yr|$&@8iEY7RsE^F$&IA}##r-@Sa1$e@ABdBMy;|cS&Zmh@& zUk13b-l+bNbwDZUZW&hrZ+7Zud8OvVpb%}&^7;Cx0X9UInLqygz5A*(r0L;R`=E#q z*e(??vE6h6rdiNs19B3;naj{~a-H`Y-D>DC=biS%V}!Q+)dkDF{0H8F$%aI#kIuhE z9aZj!JA3?x62$1#Sqgii)4Q26xg;39uEjQG zQ#O}#!%0ltG^{+&tt|A82b7UGpIk{@$pL*lFAbEQxFayuxjt8Zjd%G-N6|Fgl_v$P z7>XCu@Kk5E$qixM=|aFtDpHUxtO?R` zEZ_HrIj{=GnksJ5(!E`y&6pSN|4Oe;^W1Vp>d|exsCW%b(=eajM1Ad1_`+tzv%veL zjeWce_U1>^rbbpN5^38GINUBDgwA1!slkMM-=0`JRK#~k6GQkoQT-OA%k z%Pe=?7ps!AFTNc{s@pV3)BipJLMk`HA#4rS!bz5xI{6HLw-vtH@dFz@h1^cnoQSDd zzL)3RZg8^EGBuj_agIu=-zD==dK)*j<`Ki1d|f25wWF&081uaHl-NojQDsc;ih{3U zZ*9F`{Af^bG(HB742>j=D0s*ejWszClb?XWx8+M1Kp*2fp3T0Xw>0`Z){Hn=%phr` zua%cnPD}*>__40ooln7vxA>F%TfW8P38az5%km|hu1E^RkrhJYhpEyLiu!qAPcOO= zj&Rebg6X_|Pf>2o#Qu%)|zJt?1(5Q{TMSN#v8X(v3d zm%hJr)bxCRbzprG(1xAYdNB|jDAk$QQ$Z2QR}V^j%m@!)CS+$1TDje{pV}ERDmfXF z=%HJ+eiJ;Z{=$Ye7@CsCk^fS^%TChnYVNZI+yQQPwV%mDIM3`vS|4-{S{T?~mc%Bl z&qAz8t{fpf`CaDLiaWb;ue~9YcOa?0SO_TKMo=%`JYT1#1EqbsWvOD1pe2>tO{-A$ zb*cw@@WdGva4+G|9@XyV7yGg%{qJY_&&s5qNKgk)&1_xIItYZWkemxQ*MW9c8)U6N zks~5Iin7W%Cs#KI+2g*IWCga#`p<*sJ$LRrrDG~fL}Y`vB${+Lwpe;uU(?}p3p}u? z%HOjaH`z^6^Lc*gphX-|j*;|WJ(vIE3Lb`Pq>4a9AXB(npf%Rm?i=5SCLSD%qAZ8f zT`as9>F4;l&#+1BR*69~%%I8)TmxWk1fwy=dgf|uxLUPfo>?0pzU9e9S^@YJ6IyqlfcWM*KFz%twU8&=BVoZN zbQydGf(Fp>`&l_`*UUPT&jCfirn&21mb0~s#z{?!uW6m^$LQ+s1V!v%S>Q#nyG0Z0 zF?LgBtkY6PH6E1HU71>anodF zBF-cwq5gUIS~70xHjFVlY(#AsLb?!E^!#}Yb;<+VgXHYyw<9fn(_4XEewDp-i4y{T zKAvjZ%giUfFxpOo-9t$7ilUbL(B^a)ZNSPn6hZ6f|MhK2Zot~yx7=6x?nBdQ#sO9f z6_D1`>4kf6pL+Xm;5bS6;Q;*Qxzy#>P`AksYH{Dn;}?q?&LjTku;WYXS7NW0mLcs% zXP0qO*9D&Z9XFFDZH#6yLYadZef252MYLWg?0t3f?2f#(VN?+sH)@cSXg2rG$L}~E z_jJUaakEFREQ8egNKHaRMT?^u+BJWzWrZHXyeR?Ppmq{tv<^OXSycvlyd!)Yh;`fM z&FlYMT?>XEbxCQe+(08k|5+--0SG>%Fyf(D0+b?~{z2@&QV_@=ek~y6F#y0o4p|to za`G^F>f>VNZuB!!LnZKGQIS4Do(f0Sl<;$WvsH zf#J_W{r?xxjWZX33tHw%W?e;Uq&AJ2X5x#@?ZaY)P=ka z?I{3o*@uR5@pEJ$;+GSQr0{e^%Kl&Q3N(ku0mzF7B2T9N|CNzj&-#m-+BF9Hk_S%79vw{LSFr_cy~IB}!C0Y6a;x9&7M#JZe1& zl>#*v_nYE+>_0Pes064 Date: Sun, 23 Aug 2026 14:45:27 +0200 Subject: [PATCH 074/189] docs: refresh environment variable inventory --- ENVIRONMENT_VARIABLES.md | 1538 +++++++++++++++-------------- scripts/environment_variables.tsv | 1534 ++++++++++++++-------------- 2 files changed, 1540 insertions(+), 1532 deletions(-) diff --git a/ENVIRONMENT_VARIABLES.md b/ENVIRONMENT_VARIABLES.md index 35b0d9261..2563997f7 100644 --- a/ENVIRONMENT_VARIABLES.md +++ b/ENVIRONMENT_VARIABLES.md @@ -112,452 +112,456 @@ above, it is an unstable internal diagnostic or tuning interface. The linked sou remains normative for exact eligibility gates, bounds, and architecture-specific defaults. -Inventory totals: **1057 `DS4_*` runtime variables** and +Inventory totals: **1061 `DS4_*` runtime variables** and **6 external runtime variables**. The auxiliary inventories contain **112 test/test-fixture entries** and **19 tool/wrapper entries**.
-Metal (436) +Metal (440) | Variable | Accepted value and default | Effect | Source | | --- | --- | --- | --- | -| `DS4_METAL_ARGSORT_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the argsort Metal kernel source file loaded at runtime. | [ds4_metal.m:4942](ds4_metal.m#L4942) | -| `DS4_METAL_ATTN_OUT_STAGE_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for attn out stage. | [ds4.c:64961](ds4.c#L64961) | -| `DS4_METAL_BIN_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the binary operations Metal kernel source file loaded at runtime. | [ds4_metal.m:4951](ds4_metal.m#L4951) | -| `DS4_METAL_COMPRESSOR_PAIR_NR4` | presence control; unset: off/default; any value including 0 enables | Selects the NR4 compressor-pair variant. | [ds4_metal.m:2650](ds4_metal.m#L2650) | -| `DS4_METAL_CONCAT_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the concatenation Metal kernel source file loaded at runtime. | [ds4_metal.m:4944](ds4_metal.m#L4944) | -| `DS4_METAL_CPY_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the copy Metal kernel source file loaded at runtime. | [ds4_metal.m:4943](ds4_metal.m#L4943) | -| `DS4_METAL_DECODE_INDEXER_SPARSE_THRESHOLD` | integer in {64,128,256,512,1024,2048,4096}; default 1024; invalid restores default | Sets the compressed-row crossover from dense to sparse indexed attention. | [ds4.c:20179](ds4.c#L20179) | -| `DS4_METAL_DECODE_STAGE_PROFILE` | unset: off; 1/true/yes/on/all enables all layers; a layer index selects one; 0/false/no/off disables | Prints timing/profile diagnostics for decode stage. | [ds4.c:17395](ds4.c#L17395) | -| `DS4_METAL_DECODE_STAGE_PROFILE_LAYER` | single unsigned layer index; unset/empty: all layers enabled by the parent profile; invalid matches no layer | Restricts the corresponding shared graph stage profiler to one layer. | [ds4.c:28947](ds4.c#L28947) | -| `DS4_METAL_DENSE_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the dense matmul Metal kernel source file loaded at runtime. | [ds4_metal.m:4935](ds4_metal.m#L4935) | -| `DS4_METAL_DISABLE_AFFINE_ROPE_PAIR` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables affine RoPE pair. | [ds4_metal.m:25169](ds4_metal.m#L25169) | -| `DS4_METAL_DISABLE_ATTN_OUT_HC_FUSION` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Disables attn out HC fusion. | [ds4.c:20304](ds4.c#L20304) | -| `DS4_METAL_DISABLE_ATTN_OUT_IDS_CACHE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables attn out ids cache. | [ds4_metal.m:27769](ds4_metal.m#L27769) | -| `DS4_METAL_DISABLE_ATTN_OUT_LOW_DIRECT` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables attn out low direct. | [ds4_metal.m:27758](ds4_metal.m#L27758) | -| `DS4_METAL_DISABLE_BATCH_HC_NORM_FUSION` | nonempty value other than exact 0 disables; unset/empty/0 leaves the default enabled path | Dominant rollback for batched HC norm fusion. | [ds4.c:20284](ds4.c#L20284) | -| `DS4_METAL_DISABLE_COMPRESSOR_APE_ADD` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables compressor APE add. | [ds4_metal.m:25391](ds4_metal.m#L25391) | -| `DS4_METAL_DISABLE_COMPRESSOR_EXACT_POOL_RATIO4` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables compressor exact pool ratio4. | [ds4_metal.m:26377](ds4_metal.m#L26377) | -| `DS4_METAL_DISABLE_COMPRESSOR_PAIR_PROJ` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Disables compressor pair proj. | [ds4_metal.m:23246](ds4_metal.m#L23246) | -| `DS4_METAL_DISABLE_COMPRESSOR_QUAD_STORE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables compressor quad store. | [ds4.c:23361](ds4.c#L23361) | -| `DS4_METAL_DISABLE_COMPRESSOR_RATIO4_DIRECT_POOL` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables compressor ratio4 direct pool. | [ds4_metal.m:26243](ds4_metal.m#L26243) | -| `DS4_METAL_DISABLE_COMPRESSOR_RATIO4_PACK_FUSION` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables compressor ratio4 pack fusion. | [ds4_metal.m:26197](ds4_metal.m#L26197) | -| `DS4_METAL_DISABLE_COMPRESSOR_STORE_ONE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables compressor store one. | [ds4_metal.m:23247](ds4_metal.m#L23247) | -| `DS4_METAL_DISABLE_CONTIG_F16_F16_COPY` | value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables | Disables contig F16 F16 copy. | [ds4_metal.m:29538](ds4_metal.m#L29538) | -| `DS4_METAL_DISABLE_CONTIG_F32_F16_COPY` | value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables | Disables contig F32 F16 copy. | [ds4_metal.m:29314](ds4_metal.m#L29314) | -| `DS4_METAL_DISABLE_DECODE_NORM_EXACT_VIEWS` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables decode norm exact views. | [ds4_metal.m:36173](ds4_metal.m#L36173) | -| `DS4_METAL_DISABLE_DECODE_ROUTER_BIAS_EXACT_VIEWS` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables decode router bias exact views. | [ds4_metal.m:39363](ds4_metal.m#L39363) | -| `DS4_METAL_DISABLE_DSPARK_CAPTURE_FUSED_LAST` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Disables DSpark capture fused last. | [ds4.c:28188](ds4.c#L28188) | -| `DS4_METAL_DISABLE_DSPARK_EXACTN_BATCH_HEAD` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Disables DSpark exactn batch head. | [ds4.c:37538](ds4.c#L37538) | -| `DS4_METAL_DISABLE_EXACT_ROWS_PERSISTENT_CACHE` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Disables exact rows persistent cache. | [ds4_metal.m:12998](ds4_metal.m#L12998) | -| `DS4_METAL_DISABLE_GATHERED_KV_PAD_FUSION` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables gathered KV pad fusion. | [ds4_metal.m:29682](ds4_metal.m#L29682) | -| `DS4_METAL_DISABLE_GATHERED_KV_STAGE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables gathered KV stage. | [ds4_metal.m:29653](ds4_metal.m#L29653) | -| `DS4_METAL_DISABLE_GLM_DECODE_KV_GROUP4` | value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables | Disables GLM decode KV group4. | [ds4_metal.m:36708](ds4_metal.m#L36708) | -| `DS4_METAL_DISABLE_GLM_QKLOW_SG` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables GLM qklow sg. | [ds4_metal.m:37831](ds4_metal.m#L37831) | -| `DS4_METAL_DISABLE_GLM_STREAMING_EXPERT_EARLY_LOAD` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables GLM streaming expert early load. | [ds4_metal.m:18056](ds4_metal.m#L18056) | -| `DS4_METAL_DISABLE_GLM_STREAMING_EXPERT_SPLIT` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables GLM streaming expert split. | [ds4_metal.m:39762](ds4_metal.m#L39762) | -| `DS4_METAL_DISABLE_GLM_STREAMING_PREFILL_FULL_LAYER` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables GLM streaming prefill full layer. | [ds4.c:42520](ds4.c#L42520) | -| `DS4_METAL_DISABLE_GLM_STREAMING_PREFILL_FULL_LAYER_PREPARE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables GLM streaming prefill full layer prepare. | [ds4.c:42538](ds4.c#L42538) | -| `DS4_METAL_DISABLE_GLM_STREAMING_PREFILL_SELECTED_ASYNC_LOAD` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables GLM streaming prefill selected async load. | [ds4.c:46231](ds4.c#L46231) | -| `DS4_METAL_DISABLE_GLM_STREAMING_SELECTED_ASYNC_LOAD` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables GLM streaming selected async load. | [ds4.c:44139](ds4.c#L44139) | -| `DS4_METAL_DISABLE_HC_FUSION` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Disables HC fusion. | [ds4.c:20233](ds4.c#L20233) | -| `DS4_METAL_DISABLE_HC_NORM_FUSION` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Disables HC norm fusion. | [ds4.c:20277](ds4.c#L20277) | -| `DS4_METAL_DISABLE_HC_PRODUCER_PRE_NORM_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables HC producer pre norm fuse. | [ds4_metal.m:46507](ds4_metal.m#L46507) | -| `DS4_METAL_DISABLE_HC_RMS_SCALE_PROJ` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables HC RMS scale proj. | [ds4_metal.m:24146](ds4_metal.m#L24146) | -| `DS4_METAL_DISABLE_HOT_PIPELINE_STATICS` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables hot pipeline statics. | [ds4_metal.m:2633](ds4_metal.m#L2633) | -| `DS4_METAL_DISABLE_INPLACE_ROPE_PAIR` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables inplace RoPE pair. | [ds4_metal.m:25168](ds4_metal.m#L25168) | -| `DS4_METAL_DISABLE_IQ2_SELECTED_EXPERT_VIEWS` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables IQ2 selected expert views. | [ds4.c:21069](ds4.c#L21069) | -| `DS4_METAL_DISABLE_IQ2_SELECTED_SHARED_OVERLAP` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables IQ2 selected shared overlap. | [ds4.c:20823](ds4.c#L20823) | -| `DS4_METAL_DISABLE_IQ2_STREAM_ADDR_TABLE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables IQ2 stream address table. | [ds4_metal.m:42368](ds4_metal.m#L42368) | -| `DS4_METAL_DISABLE_IQ2_XXS_SSD_PREFILL_MM` | value-aware boolean; default off; true disables and dominates ENABLE; false leaves automatic policy | Rolls grouped IQ2_XXS/Q2_K SSD-prefill MM back to sparse matvec. | [ds4_metal.m:44750](ds4_metal.m#L44750) | -| `DS4_METAL_DISABLE_KV_FUSION` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Disables KV fusion. | [ds4.c:20257](ds4.c#L20257) | -| `DS4_METAL_DISABLE_M1_IQ2_MID_ONLY` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables M1 IQ2 mid only. | [ds4_metal.m:14590](ds4_metal.m#L14590) | -| `DS4_METAL_DISABLE_M3_COMPRESSOR_EXACT_POOL_RATIO4` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables M3 compressor exact pool ratio4. | [ds4_metal.m:26379](ds4_metal.m#L26379) | -| `DS4_METAL_DISABLE_M3_COMPRESSOR_PAIR_STATE_STORE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables M3 compressor pair state store. | [ds4_metal.m:23245](ds4_metal.m#L23245) | -| `DS4_METAL_DISABLE_M3_GATHERED_KV_STAGE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables M3 gathered KV stage. | [ds4_metal.m:29654](ds4_metal.m#L29654) | -| `DS4_METAL_DISABLE_M5_COMPRESSOR_EXACT_POOL_RATIO4` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables M5 compressor exact pool ratio4. | [ds4_metal.m:26382](ds4_metal.m#L26382) | -| `DS4_METAL_DISABLE_M5_COMP_FINALIZE_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables M5 comp finalize fuse. | [ds4.c:23474](ds4.c#L23474) | -| `DS4_METAL_DISABLE_M5_FLASH_ATTN_PACKED32_REDUCE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables M5 flash attn packed32 reduce. | [ds4_metal.m:31490](ds4_metal.m#L31490) | -| `DS4_METAL_DISABLE_M5_HC_NORM_MIX_CLUSTER2` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables M5 HC norm mix cluster2. | [ds4_metal.m:46462](ds4_metal.m#L46462) | -| `DS4_METAL_DISABLE_M5_HC_PRODUCER_PRE_NORM_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables M5 HC producer pre norm fuse. | [ds4_metal.m:46514](ds4_metal.m#L46514) | -| `DS4_METAL_DISABLE_M5_IQ2_PAIR_PACK2` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables M5 IQ2 pair pack2. | [ds4_metal.m:41986](ds4_metal.m#L41986) | -| `DS4_METAL_DISABLE_M5_PACKED_ZERO_MASK` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables M5 packed zero mask. | [ds4_metal.m:31446](ds4_metal.m#L31446) | -| `DS4_METAL_DISABLE_M5_PARALLEL_FULL_FFN` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables M5 parallel full FFN. | [ds4.c:22534](ds4.c#L22534) | -| `DS4_METAL_DISABLE_M5_PERSISTENT_ZERO_ATTN_MASK` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables M5 persistent zero attn mask. | [ds4_metal.m:31444](ds4_metal.m#L31444) | -| `DS4_METAL_DISABLE_M5_Q8_HC_VEC` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables M5 Q8 HC vec. | [ds4_metal.m:47546](ds4_metal.m#L47546) | -| `DS4_METAL_DISABLE_M5_QKV_PAIR_COMPRESSOR_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables M5 QKV pair compressor fuse. | [ds4.c:22870](ds4.c#L22870) | -| `DS4_METAL_DISABLE_M5_QKV_PAIR_QUAD_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables M5 QKV pair quad fuse. | [ds4.c:22866](ds4.c#L22866) | -| `DS4_METAL_DISABLE_M5_ROUTER_PROJECT_SELECT_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables M5 router project select fuse. | [ds4.c:24589](ds4.c#L24589) | -| `DS4_METAL_DISABLE_METAL4` | value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables | Disables metal4. | [ds4_metal.m:2994](ds4_metal.m#L2994) | -| `DS4_METAL_DISABLE_MOE_MM_ID_PAIR_SWIGLU` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables MoE MM ID pair SwiGLU. | [ds4_metal.m:44934](ds4_metal.m#L44934) | -| `DS4_METAL_DISABLE_MOE_MM_ID_USE_RESOURCES` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables MoE MM ID use resources. | [ds4_metal.m:35135](ds4_metal.m#L35135) | -| `DS4_METAL_DISABLE_MXFP4_SELECTED_EXPERT_VIEWS` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables MXFP4 selected expert views. | [ds4.c:21000](ds4.c#L21000) | -| `DS4_METAL_DISABLE_PERSISTENT_ZERO_ATTN_MASK` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables persistent zero attn mask. | [ds4_metal.m:31451](ds4_metal.m#L31451) | -| `DS4_METAL_DISABLE_PRE_M5_ATTN_INV_ROPE_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 attn inv RoPE fuse. | [ds4.c:22465](ds4.c#L22465) | -| `DS4_METAL_DISABLE_PRE_M5_COMPRESSOR_EXACT_POOL_RATIO4` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 compressor exact pool ratio4. | [ds4_metal.m:26381](ds4_metal.m#L26381) | -| `DS4_METAL_DISABLE_PRE_M5_COMPRESSOR_EXACT_REDUCTION_FUSION` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 compressor exact reduction fusion. | [ds4_metal.m:25905](ds4_metal.m#L25905) | -| `DS4_METAL_DISABLE_PRE_M5_COMPRESSOR_QUAD_STORE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 compressor quad store. | [ds4.c:23362](ds4.c#L23362) | -| `DS4_METAL_DISABLE_PRE_M5_COMPRESSOR_RATIO4_DECODE_PACK_FUSION` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 compressor ratio4 decode pack fusion. | [ds4_metal.m:26212](ds4_metal.m#L26212) | -| `DS4_METAL_DISABLE_PRE_M5_COMP_FINALIZE_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 comp finalize fuse. | [ds4.c:23473](ds4.c#L23473) | -| `DS4_METAL_DISABLE_PRE_M5_DECODE_EARLY_PIPELINE_FAST_LOOKUP` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 decode early pipeline fast lookup. | [ds4.c:27830](ds4.c#L27830) | -| `DS4_METAL_DISABLE_PRE_M5_DECODE_EARLY_SECOND_SPLIT12` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 decode early second split12. | [ds4.c:27914](ds4.c#L27914) | -| `DS4_METAL_DISABLE_PRE_M5_DECODE_EARLY_SPLIT3` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 decode early split3. | [ds4.c:27788](ds4.c#L27788) | -| `DS4_METAL_DISABLE_PRE_M5_DECODE_EARLY_SPLIT5` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 decode early split5. | [ds4.c:27800](ds4.c#L27800) | -| `DS4_METAL_DISABLE_PRE_M5_DECODE_PIPELINE_FAST_LOOKUP` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 decode pipeline fast lookup. | [ds4.c:27842](ds4.c#L27842) | -| `DS4_METAL_DISABLE_PRE_M5_DECODE_PORTS` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 decode ports. | [ds4.c:22440](ds4.c#L22440) | -| `DS4_METAL_DISABLE_PRE_M5_DECODE_RAW_ZERO_ATTN_MASK` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 decode raw zero attn mask. | [ds4_metal.m:29893](ds4_metal.m#L29893) | -| `DS4_METAL_DISABLE_PRE_M5_DECODE_SECOND_SPLIT16` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 decode second split16. | [ds4.c:27922](ds4.c#L27922) | -| `DS4_METAL_DISABLE_PRE_M5_FLASH_ATTN_BATCHED_MEMO` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 flash attn batched memo. | [ds4_metal.m:3751](ds4_metal.m#L3751) | -| `DS4_METAL_DISABLE_PRE_M5_FLASH_ATTN_PACKED32_REDUCE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 flash attn packed32 reduce. | [ds4_metal.m:31433](ds4_metal.m#L31433) | -| `DS4_METAL_DISABLE_PRE_M5_FLASH_ATTN_PAD_BLK_MEMO` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 flash attn pad blk memo. | [ds4_metal.m:3609](ds4_metal.m#L3609) | -| `DS4_METAL_DISABLE_PRE_M5_HC_NORM_MIX_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 HC norm mix fuse. | [ds4.c:22694](ds4.c#L22694) | -| `DS4_METAL_DISABLE_PRE_M5_HC_PRODUCER_PRE_NORM_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 HC producer pre norm fuse. | [ds4_metal.m:46512](ds4_metal.m#L46512) | -| `DS4_METAL_DISABLE_PRE_M5_HEAD_RMS_ROPE_PIPELINE_STATIC` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 head RMS RoPE pipeline static. | [ds4_metal.m:10025](ds4_metal.m#L10025) | -| `DS4_METAL_DISABLE_PRE_M5_KV_ROPE_FP8_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 KV RoPE fp8 fuse. | [ds4.c:23282](ds4.c#L23282) | -| `DS4_METAL_DISABLE_PRE_M5_MXFP4_MM_ID_PAIR_HALF_SCALE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 MXFP4 MM ID pair half scale. | [ds4_metal.m:45095](ds4_metal.m#L45095) | -| `DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_DECODE_FIXED_ROUTE_PAIR` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 MXFP4 MoE decode fixed route pair. | [ds4_metal.m:41914](ds4_metal.m#L41914) | -| `DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_DECODE_FIXED_ROUTE_SUM6` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 MXFP4 MoE decode fixed route sum6. | [ds4_metal.m:41927](ds4_metal.m#L41927) | -| `DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_DECODE_NSG1` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 MXFP4 MoE decode nsg1. | [ds4_metal.m:41685](ds4_metal.m#L41685) | -| `DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_DECODE_STATIC_TRIP` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 MXFP4 MoE decode static trip. | [ds4_metal.m:41953](ds4_metal.m#L41953) | -| `DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_DECODE_SUM6_FULL_ROWS` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 MXFP4 MoE decode sum6 full rows. | [ds4_metal.m:41940](ds4_metal.m#L41940) | -| `DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_DECODE_TG_MULTIPLE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 MXFP4 MoE decode tg multiple. | [ds4_metal.m:41902](ds4_metal.m#L41902) | -| `DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_MM_ID_DOWN_HALF_LUT` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 MXFP4 MoE MM ID down half lut. | [ds4_metal.m:45013](ds4_metal.m#L45013) | -| `DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_MM_ID_DOWN_TAIL_SIMDGROUP_CULL` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 MXFP4 MoE MM ID down tail simdgroup cull. | [ds4_metal.m:44996](ds4_metal.m#L44996) | -| `DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_MM_ID_MAP_SCATTER` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 MXFP4 MoE MM ID map scatter. | [ds4_metal.m:44963](ds4_metal.m#L44963) | -| `DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_MM_ID_PAIR_SWIGLU_COMPACT_TILE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 MXFP4 MoE MM ID pair SwiGLU compact tile. | [ds4_metal.m:44947](ds4_metal.m#L44947) | -| `DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_MM_ID_PAIR_TAIL_SIMDGROUP_CULL` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 MXFP4 MoE MM ID pair tail simdgroup cull. | [ds4_metal.m:44984](ds4_metal.m#L44984) | -| `DS4_METAL_DISABLE_PRE_M5_PARALLEL_FULL_FFN` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 parallel full FFN. | [ds4.c:22533](ds4.c#L22533) | -| `DS4_METAL_DISABLE_PRE_M5_Q2_DECODE_SPLIT2_32` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 q2 decode split2 32. | [ds4.c:27703](ds4.c#L27703) | -| `DS4_METAL_DISABLE_PRE_M5_QKV_NORM_KV_STORE_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 QKV norm KV store fuse. | [ds4.c:23141](ds4.c#L23141) | -| `DS4_METAL_DISABLE_PRE_M5_QKV_PAIR_COMPRESSOR_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 QKV pair compressor fuse. | [ds4.c:22869](ds4.c#L22869) | -| `DS4_METAL_DISABLE_PRE_M5_QKV_PAIR_QUAD_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 QKV pair quad fuse. | [ds4.c:22865](ds4.c#L22865) | -| `DS4_METAL_DISABLE_PRE_M5_ROUTER_SHARED_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 router shared fuse. | [ds4.c:24582](ds4.c#L24582) | -| `DS4_METAL_DISABLE_PRE_M5_ROUTER_SIMD_FINALIZE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 router simd finalize. | [ds4_metal.m:35793](ds4_metal.m#L35793) | -| `DS4_METAL_DISABLE_PRE_M5_ROUTER_SIMD_WEIGHTS_FUSION` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 router simd weights fusion. | [ds4_metal.m:35802](ds4_metal.m#L35802) | -| `DS4_METAL_DISABLE_PRE_M5_ROUTER_TRANSFORM_FINALIZE_FUSION` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 router transform finalize fusion. | [ds4_metal.m:35806](ds4_metal.m#L35806) | -| `DS4_METAL_DISABLE_PRO_Q4_EXPERT_ADDRESS_AUTO` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pro Q4 expert address auto. | [ds4_metal.m:19708](ds4_metal.m#L19708) | -| `DS4_METAL_DISABLE_PRO_Q4_EXPERT_TABLE_AUTO` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pro Q4 expert table auto. | [ds4.c:20867](ds4.c#L20867) | -| `DS4_METAL_DISABLE_PRO_Q4_EXPERT_TABLE_PRELOAD` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pro Q4 expert table preload. | [ds4.c:58590](ds4.c#L58590) | -| `DS4_METAL_DISABLE_Q4_ATTN_OUT_HC_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 attn out HC fuse. | [ds4_metal.m:47590](ds4_metal.m#L47590) | -| `DS4_METAL_DISABLE_Q4_ATTN_OUT_TINY_BATCH` | value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables | Disables Q4 attn out tiny batch. | [ds4_metal.m:28372](ds4_metal.m#L28372) | -| `DS4_METAL_DISABLE_Q4_BATCH_EXPERT_TABLE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 batch expert table. | [ds4_metal.m:44849](ds4_metal.m#L44849) | -| `DS4_METAL_DISABLE_Q4_DENSE_PAIR` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 dense pair. | [ds4_metal.m:21988](ds4_metal.m#L21988) | -| `DS4_METAL_DISABLE_Q4_EXACT_BOUNDARY` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 exact boundary. | [ds4_metal.m:42205](ds4_metal.m#L42205) | -| `DS4_METAL_DISABLE_Q4_EXACT_TENSOR_ID` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 exact tensor ID. | [ds4_metal.m:42185](ds4_metal.m#L42185) | -| `DS4_METAL_DISABLE_Q4_EXPERT_ADDRESS_TABLE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 expert address table. | [ds4_metal.m:19709](ds4_metal.m#L19709) | -| `DS4_METAL_DISABLE_Q4_EXPERT_TABLE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 expert table. | [ds4.c:20868](ds4.c#L20868) | -| `DS4_METAL_DISABLE_Q4_GATHER_SLOTS` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 gather slots. | [ds4_metal.m:42287](ds4_metal.m#L42287) | -| `DS4_METAL_DISABLE_Q4_GROUP24_EXPERT_TABLE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 group24 expert table. | [ds4_metal.m:42167](ds4_metal.m#L42167) | -| `DS4_METAL_DISABLE_Q4_GROUP6_EXPERT_TABLE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 group6 expert table. | [ds4_metal.m:42133](ds4_metal.m#L42133) | -| `DS4_METAL_DISABLE_Q4_GROUP8_EXPERT_TABLE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 group8 expert table. | [ds4_metal.m:42150](ds4_metal.m#L42150) | -| `DS4_METAL_DISABLE_Q4_GROUPED_BOUNDARY` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 grouped boundary. | [ds4_metal.m:42115](ds4_metal.m#L42115) | -| `DS4_METAL_DISABLE_Q4_GROUPED_EXPERTS` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 grouped experts. | [ds4_metal.m:42096](ds4_metal.m#L42096) | -| `DS4_METAL_DISABLE_Q4_MV_CLASSIC` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 MV classic. | [ds4_metal.m:21569](ds4_metal.m#L21569) | -| `DS4_METAL_DISABLE_Q4_QKV_COMPRESSOR_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 QKV compressor fuse. | [ds4_metal.m:22081](ds4_metal.m#L22081) | -| `DS4_METAL_DISABLE_Q4_SELECTED_EXPERT_VIEWS` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 selected expert views. | [ds4.c:21064](ds4.c#L21064) | -| `DS4_METAL_DISABLE_Q4_SSD_PREFILL_ATTN_OUT_EXACTN` | value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables | Disables Q4 SSD prefill attn out exactn. | [ds4_metal.m:28071](ds4_metal.m#L28071) | -| `DS4_METAL_DISABLE_Q4_SSD_SESSION_UNION` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Disables Q4 SSD session union. | [ds4.c:64970](ds4.c#L64970) | -| `DS4_METAL_DISABLE_Q4_STREAM_OVERLAP` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Disables Q4 stream overlap. | [ds4.c:64892](ds4.c#L64892) | -| `DS4_METAL_DISABLE_Q4_TABLE_BOUNDARY` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 table boundary. | [ds4_metal.m:42284](ds4_metal.m#L42284) | -| `DS4_METAL_DISABLE_Q8_DECODE_EXACT_VIEWS` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q8 decode exact views. | [ds4_metal.m:12849](ds4_metal.m#L12849) | -| `DS4_METAL_DISABLE_QKV_NORM_FUSION` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Disables QKV norm fusion. | [ds4.c:20262](ds4.c#L20262) | -| `DS4_METAL_DISABLE_QKV_PAIR_PROJ` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Disables QKV pair proj. | [ds4.c:20267](ds4.c#L20267) | -| `DS4_METAL_DISABLE_QUEUE_RESIDENCY_SET` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables queue residency set. | [ds4_metal.m:2123](ds4_metal.m#L2123) | -| `DS4_METAL_DISABLE_ROUTED_PAIR_SWIGLU_FUSION` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables routed pair SwiGLU fusion. | [ds4.c:18422](ds4.c#L18422) | -| `DS4_METAL_DISABLE_ROUTER_SELECT_FUSION` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables router select fusion. | [ds4_metal.m:35782](ds4_metal.m#L35782) | -| `DS4_METAL_DISABLE_ROUTER_WEIGHTS_BATCH_FUSION` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables router weights batch fusion. | [ds4_metal.m:36034](ds4_metal.m#L36034) | -| `DS4_METAL_DISABLE_SHARED_DOWN_HC_FUSION` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Disables shared down HC fusion. | [ds4.c:20299](ds4.c#L20299) | -| `DS4_METAL_DISABLE_SHARED_GATE_UP_SWIGLU_FUSION` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables shared gate up SwiGLU fusion. | [ds4.c:17394](ds4.c#L17394) | -| `DS4_METAL_DISABLE_SHARED_KV_PAD` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables shared KV pad. | [ds4_metal.m:31481](ds4_metal.m#L31481) | -| `DS4_METAL_DISABLE_SHARED_ROPE_COEFF` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables shared RoPE coeff. | [ds4_metal.m:6340](ds4_metal.m#L6340) | -| `DS4_METAL_DISABLE_STREAMING_COLD_DECODE_PREFILL` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming cold decode prefill. | [ds4.c:31817](ds4.c#L31817) | -| `DS4_METAL_DISABLE_STREAMING_COMPACT_ADDR` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming compact address. | [ds4_metal.m:14596](ds4_metal.m#L14596) | -| `DS4_METAL_DISABLE_STREAMING_DECODE_PREFILL` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming decode prefill. | [ds4.c:31766](ds4.c#L31766) | -| `DS4_METAL_DISABLE_STREAMING_EXPERT_ADDR_TABLE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming expert address table. | [ds4.c:18420](ds4.c#L18420) | -| `DS4_METAL_DISABLE_STREAMING_EXPERT_COMBINED_BUFFER` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming expert combined buffer. | [ds4_metal.m:14021](ds4_metal.m#L14021) | -| `DS4_METAL_DISABLE_STREAMING_EXPERT_EARLY_LOAD` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming expert early load. | [ds4_metal.m:17230](ds4_metal.m#L17230) | -| `DS4_METAL_DISABLE_STREAMING_EXPERT_EVICT_DONTNEED` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming expert evict dontneed. | [ds4_metal.m:14393](ds4_metal.m#L14393) | -| `DS4_METAL_DISABLE_STREAMING_EXPERT_HIT_VALIDATOR` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming expert hit validator. | [ds4_metal.m:14632](ds4_metal.m#L14632) | -| `DS4_METAL_DISABLE_STREAMING_EXPERT_HOTLIST` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming expert hotlist. | [ds4.c:21117](ds4.c#L21117) | -| `DS4_METAL_DISABLE_STREAMING_EXPERT_LIVE_INDEX` | value-aware boolean; default off; true disables and dominates ENABLE | Disables dense live-entry index and uses authoritative cache matrix. | [ds4_metal.m:15441](ds4_metal.m#L15441) | -| `DS4_METAL_DISABLE_STREAMING_EXPERT_MASKED_ADDR` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming expert masked address. | [ds4_metal.m:14626](ds4_metal.m#L14626) | -| `DS4_METAL_DISABLE_STREAMING_EXPERT_READAHEAD` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming expert readahead. | [ds4_metal.m:13194](ds4_metal.m#L13194) | -| `DS4_METAL_DISABLE_STREAMING_EXPERT_SLABS` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming expert slabs. | [ds4_metal.m:14026](ds4_metal.m#L14026) | -| `DS4_METAL_DISABLE_STREAMING_EXPERT_TIMING_SUMMARY` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming expert timing summary. | [ds4_metal.m:13060](ds4_metal.m#L13060) | -| `DS4_METAL_DISABLE_STREAMING_FULL_EXPERT_ADDR_TABLE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming full expert address table. | [ds4_metal.m:14714](ds4_metal.m#L14714) | -| `DS4_METAL_DISABLE_STREAMING_IQ2_CPU_ROUTER` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming IQ2 CPU router. | [ds4.c:20766](ds4.c#L20766) | -| `DS4_METAL_DISABLE_STREAMING_LAYER_BATCH` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming layer batch. | [ds4.c:18066](ds4.c#L18066) | -| `DS4_METAL_DISABLE_STREAMING_MADVISE_WILLNEED` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming madvise willneed. | [ds4.c:18039](ds4.c#L18039) | -| `DS4_METAL_DISABLE_STREAMING_PREFILL_BATCH_SELECTED_ADDR` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming prefill batch selected address. | [ds4.c:18418](ds4.c#L18418) | -| `DS4_METAL_DISABLE_STREAMING_PREFILL_LAYER_MADVISE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming prefill layer madvise. | [ds4.c:18293](ds4.c#L18293) | -| `DS4_METAL_DISABLE_STREAMING_PREFILL_LAYER_PAGEIN` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming prefill layer pagein. | [ds4.c:18261](ds4.c#L18261) | -| `DS4_METAL_DISABLE_STREAMING_PREFILL_LAYER_PAGEIN_OVERLAP` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming prefill layer pagein overlap. | [ds4.c:19148](ds4.c#L19148) | -| `DS4_METAL_DISABLE_STREAMING_PREFILL_LAYER_PREAD` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming prefill layer pread. | [ds4.c:18281](ds4.c#L18281) | -| `DS4_METAL_DISABLE_STREAMING_PREFILL_LAYER_PREPARE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming prefill layer prepare. | [ds4.c:18273](ds4.c#L18273) | -| `DS4_METAL_DISABLE_STREAMING_PREFILL_LAYER_PREPARE_OVERLAP` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming prefill layer prepare overlap. | [ds4.c:19146](ds4.c#L19146) | -| `DS4_METAL_DISABLE_STREAMING_PREFILL_LAYER_READAHEAD` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming prefill layer readahead. | [ds4.c:18271](ds4.c#L18271) | -| `DS4_METAL_DISABLE_STREAMING_PREFILL_SELECTED_ASYNC_LOAD` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming prefill selected async load. | [ds4.c:46234](ds4.c#L46234) | -| `DS4_METAL_DISABLE_STREAMING_PREFILL_SELECTED_MADVISE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming prefill selected madvise. | [ds4.c:18251](ds4.c#L18251) | -| `DS4_METAL_DISABLE_STREAMING_PREFILL_SELECTED_PAGEIN` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming prefill selected pagein. | [ds4.c:18241](ds4.c#L18241) | -| `DS4_METAL_DISABLE_STREAMING_PREFILL_SELECTED_PROFILE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming prefill selected profile. | [ds4.c:18735](ds4.c#L18735) | -| `DS4_METAL_DISABLE_STREAMING_PREFILL_SELECTED_READAHEAD` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming prefill selected readahead. | [ds4.c:19787](ds4.c#L19787) | -| `DS4_METAL_DISABLE_STREAMING_PREFILL_SELECTED_READAHEAD_SHARED` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming prefill selected readahead shared. | [ds4.c:19797](ds4.c#L19797) | -| `DS4_METAL_DISABLE_STREAMING_READAHEAD` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming readahead. | [ds4.c:18032](ds4.c#L18032) | -| `DS4_METAL_DISABLE_STREAMING_SELECTED_ASYNC_EARLY_COMMIT` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming selected async early commit. | [ds4.c:20846](ds4.c#L20846) | -| `DS4_METAL_DISABLE_STREAMING_SELECTED_ASYNC_LOAD` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming selected async load. | [ds4.c:20830](ds4.c#L20830) | -| `DS4_METAL_DISABLE_STREAMING_SELECTED_READAHEAD_SHARED_DELAY` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming selected readahead shared delay. | [ds4.c:21547](ds4.c#L21547) | -| `DS4_METAL_DISABLE_STREAMING_SELECTED_SHARED_OVERLAP` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming selected shared overlap. | [ds4.c:20822](ds4.c#L20822) | -| `DS4_METAL_DISABLE_STREAMING_STATIC_DECODE_MAP` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming static decode map. | [ds4.c:18044](ds4.c#L18044) | -| `DS4_METAL_DISABLE_STREAMING_STATIC_MAP_STATE_CACHE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming static map state cache. | [ds4.c:18057](ds4.c#L18057) | -| `DS4_METAL_DISABLE_SUPPORT_Q8_DECODE_EXACT_VIEWS` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables support Q8 decode exact views. | [ds4_metal.m:12856](ds4_metal.m#L12856) | -| `DS4_METAL_DISABLE_TINY_PAIR_SWIGLU_FUSION` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables tiny pair SwiGLU fusion. | [ds4_metal.m:44886](ds4_metal.m#L44886) | -| `DS4_METAL_DISABLE_TOKEN_EMBED_EXACT_VIEW` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables token embed exact view. | [ds4_metal.m:11905](ds4_metal.m#L11905) | -| `DS4_METAL_DISABLE_ZERO_PREFIX_PREFILL_MASK_CACHE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables zero prefix prefill mask cache. | [ds4_metal.m:2403](ds4_metal.m#L2403) | -| `DS4_METAL_DSPARK_ACCEPTANCE_ONLY_VERIFY` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Verifies only the draft rows still needed for acceptance after the base target logit. | [ds4.c:52309](ds4.c#L52309) | -| `DS4_METAL_DSPARK_DEVICE_PROPOSER` | boolean true values enable; false values/unset disable; NO_DEVICE_PROPOSER presence dominates | Keeps DSpark Q8 confidence/Markov proposal work on Metal and reads one compact result. | [ds4.c:34694](ds4.c#L34694) | -| `DS4_METAL_DSPARK_EXACT2` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Enables the resident single-GPU Metal exact-2 verifier. | [ds4.c:52100](ds4.c#L52100) | -| `DS4_METAL_DSPARK_EXACTN` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Enables the single-GPU Metal exact-N verifier. | [ds4.c:52121](ds4.c#L52121) | -| `DS4_METAL_DSPARK_EXACTN_BATCH_HEAD` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Batches the output head across exact-N verifier rows. | [ds4.c:37536](ds4.c#L37536) | -| `DS4_METAL_DSPARK_EXACTN_UNION` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Loads the union of experts for exact-N verifier rows once. | [ds4.c:52142](ds4.c#L52142) | -| `DS4_METAL_DSPARK_EXACT_ROWS_ASYNC_TAILS` | presence control; unset: off/default; any value including 0 enables | Runs exact-row routed tails asynchronously after union routing. | [ds4.c:37742](ds4.c#L37742) | -| `DS4_METAL_DSPARK_EXACT_ROWS_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for DSpark exact rows. | [ds4.c:37740](ds4.c#L37740) | -| `DS4_METAL_DSPARK_HEADLESS_REPLAY` | unset/empty: enabled; exact 0 disables; every other nonempty value enables | Skips output heads for accepted intermediate DSpark replay tokens. | [ds4.c:52289](ds4.c#L52289) | -| `DS4_METAL_DSPARK_NO_DEVICE_PROPOSER` | presence rollback; unset: automatic/default path; any value including 0 disables | Dominant presence-based rollback for the Metal DSpark device proposer. | [ds4.c:34696](ds4.c#L34696) | -| `DS4_METAL_DSPARK_PIN_MAIN_PROJ` | nonempty value other than exact 0 enables; unset/empty/0 disables | mlock-pins only DSpark stage-0 main_norm/main_proj in Metal SSD streaming. | [ds4.c:39253](ds4.c#L39253) | -| `DS4_METAL_DSPARK_PROPOSER_BLOCK_MAX` | uint32; unset: automatic cache/verifier cap; 0 or invalid: native width; positive: clamped to DSpark/native maximum | Caps rows proposed by single-device Metal DSpark. | [ds4.c:52253](ds4.c#L52253) | -| `DS4_METAL_DSPARK_SAFE_EXPERT_COUNT` | exact 1 enables; unset or any other value disables | Caps an explicit expert-count cache request to the safe Metal working-set budget for DSpark SSD streaming. | [ds4.c:4758](ds4.c#L4758) | -| `DS4_METAL_DSV4_HC_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the DeepSeek hidden-context Metal kernel source file loaded at runtime. | [ds4_metal.m:4937](ds4_metal.m#L4937) | -| `DS4_METAL_DSV4_KV_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the DeepSeek KV Metal kernel source file loaded at runtime. | [ds4_metal.m:4939](ds4_metal.m#L4939) | -| `DS4_METAL_DSV4_MISC_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the DeepSeek miscellaneous Metal kernel source file loaded at runtime. | [ds4_metal.m:4941](ds4_metal.m#L4941) | -| `DS4_METAL_DSV4_ROPE_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the DeepSeek RoPE Metal kernel source file loaded at runtime. | [ds4_metal.m:4940](ds4_metal.m#L4940) | -| `DS4_METAL_DUMP_PREFILL_LOGITS` | file path; unset/empty: no dump | Writes final GPU prefill logits as f32 binary. | [ds4.c:51157](ds4.c#L51157) | -| `DS4_METAL_ENABLE_BATCH_HC_NORM_FUSION` | legacy value-aware alias; default enabled; exact 0 disables; unset/empty/every other value enables unless DISABLE is active | Legacy control for the now-default batched HC norm fusion. | [ds4.c:20289](ds4.c#L20289) | -| `DS4_METAL_ENABLE_COMPRESSOR_EXACT_POOL_RATIO4` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables compressor exact pool ratio4. | [ds4_metal.m:26383](ds4_metal.m#L26383) | -| `DS4_METAL_ENABLE_COMPRESSOR_PAIR_STATE_STORE` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables compressor pair state store. | [ds4_metal.m:23241](ds4_metal.m#L23241) | -| `DS4_METAL_ENABLE_COMPRESSOR_QUAD_STORE` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables compressor quad store. | [ds4.c:23354](ds4.c#L23354) | -| `DS4_METAL_ENABLE_DSPARK_CAPTURE_FUSED_LAST` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Enables DSpark capture fused last. | [ds4.c:28186](ds4.c#L28186) | -| `DS4_METAL_ENABLE_GATHERED_KV_STAGE` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables gathered KV stage. | [ds4_metal.m:29651](ds4_metal.m#L29651) | -| `DS4_METAL_ENABLE_GLM_STREAMING_SELECTED_ASYNC_LOAD` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables GLM streaming selected async load. | [ds4.c:44145](ds4.c#L44145) | -| `DS4_METAL_ENABLE_HC_NORM_MIX_FUSE` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables HC norm mix fuse. | [ds4.c:22697](ds4.c#L22697) | -| `DS4_METAL_ENABLE_HC_PRODUCER_PRE_NORM_FUSE` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables HC producer pre norm fuse. | [ds4_metal.m:46518](ds4_metal.m#L46518) | -| `DS4_METAL_ENABLE_IQ2_SELECTED_ASYNC_EARLY_COMMIT` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables IQ2 selected async early commit. | [ds4.c:20845](ds4.c#L20845) | -| `DS4_METAL_ENABLE_IQ2_XXS_SSD_PREFILL_MM` | value-aware boolean; default automatic/on for eligible shape; explicit 0 turns request off unless REQUIRE=1 | Overrides automatic IQ2_XXS/Q2_K grouped address-MM selection for SSD prefill. | [ds4_metal.m:44746](ds4_metal.m#L44746) | -| `DS4_METAL_ENABLE_PRO_Q4_EXPERT_ADDRESS_AUTO` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables pro Q4 expert address auto. | [ds4.c:20809](ds4.c#L20809) | -| `DS4_METAL_ENABLE_PRO_Q4_EXPERT_TABLE_AUTO` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables pro Q4 expert table auto. | [ds4.c:20808](ds4.c#L20808) | -| `DS4_METAL_ENABLE_PRO_Q4_SELECTED_EXPERT_VIEWS` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables pro Q4 selected expert views. | [ds4.c:20805](ds4.c#L20805) | -| `DS4_METAL_ENABLE_Q4_ATTN_OUT_TINY_BATCH` | value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables | Enables Q4 attn out tiny batch. | [ds4_metal.m:28381](ds4_metal.m#L28381) | -| `DS4_METAL_ENABLE_Q4_BATCH_EXPERT_TABLE` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables Q4 batch expert table. | [ds4_metal.m:44835](ds4_metal.m#L44835) | -| `DS4_METAL_ENABLE_Q4_EXACT_TENSOR_ID` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables Q4 exact tensor ID. | [ds4_metal.m:42184](ds4_metal.m#L42184) | -| `DS4_METAL_ENABLE_Q4_EXPERT_ADDRESS_TABLE` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables Q4 expert address table. | [ds4.c:20807](ds4.c#L20807) | -| `DS4_METAL_ENABLE_Q4_EXPERT_TABLE` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables Q4 expert table. | [ds4.c:20806](ds4.c#L20806) | -| `DS4_METAL_ENABLE_Q4_GATHER_SLOTS` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables Q4 gather slots. | [ds4_metal.m:42286](ds4_metal.m#L42286) | -| `DS4_METAL_ENABLE_Q4_GROUP24_EXPERT_TABLE` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables Q4 group24 expert table. | [ds4_metal.m:42166](ds4_metal.m#L42166) | -| `DS4_METAL_ENABLE_Q4_GROUP6_EXPERT_TABLE` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables Q4 group6 expert table. | [ds4_metal.m:42132](ds4_metal.m#L42132) | -| `DS4_METAL_ENABLE_Q4_GROUP8_EXPERT_TABLE` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables Q4 group8 expert table. | [ds4_metal.m:42149](ds4_metal.m#L42149) | -| `DS4_METAL_ENABLE_Q4_GROUPED_EXPERTS` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables Q4 grouped experts. | [ds4_metal.m:42095](ds4_metal.m#L42095) | -| `DS4_METAL_ENABLE_Q4_QKV_COMPRESSOR_FUSE` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables Q4 QKV compressor fuse. | [ds4.c:22967](ds4.c#L22967) | -| `DS4_METAL_ENABLE_Q4_SELECTED_EXPERT_VIEWS` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables Q4 selected expert views. | [ds4.c:20804](ds4.c#L20804) | -| `DS4_METAL_ENABLE_Q4_SSD_PREFILL_ATTN_OUT_EXACTN` | value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables | Enables Q4 SSD prefill attn out exactn. | [ds4_metal.m:28073](ds4_metal.m#L28073) | -| `DS4_METAL_ENABLE_Q4_SSD_SESSION_UNION` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Enables Q4 SSD session union. | [ds4.c:64981](ds4.c#L64981) | -| `DS4_METAL_ENABLE_Q4_STREAM_OVERLAP` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Enables Q4 stream overlap. | [ds4.c:64890](ds4.c#L64890) | -| `DS4_METAL_ENABLE_Q8_DECODE_EXACT_VIEWS` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables Q8 decode exact views. | [ds4_metal.m:12863](ds4_metal.m#L12863) | -| `DS4_METAL_ENABLE_Q8_QKV_COMPRESSOR_FUSE` | nonempty boolean; unset/empty/exact 0: no streamed/union opt-in; other values enable; eligible resident full-decode remains automatic | Extends the automatic resident Q8 QKV/compressor compound fusion to SSD streaming or exact-N union scope. | [ds4.c:22854](ds4.c#L22854) | -| `DS4_METAL_ENABLE_STREAMING_COMPACT_ADDR` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming compact address. | [ds4_metal.m:14595](ds4_metal.m#L14595) | -| `DS4_METAL_ENABLE_STREAMING_EXPERT_ADDR_TABLE` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming expert address table. | [ds4_metal.m:14602](ds4_metal.m#L14602) | -| `DS4_METAL_ENABLE_STREAMING_EXPERT_EVICT_DONTNEED` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming expert evict dontneed. | [ds4_metal.m:14392](ds4_metal.m#L14392) | -| `DS4_METAL_ENABLE_STREAMING_EXPERT_HIT_VALIDATOR` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming expert hit validator. | [ds4_metal.m:14603](ds4_metal.m#L14603) | -| `DS4_METAL_ENABLE_STREAMING_EXPERT_LIVE_INDEX` | value-aware boolean; default automatic/on for validated IQ2 cache shape; explicit 0 disables | Overrides automatic dense live-entry index selection. | [ds4_metal.m:15440](ds4_metal.m#L15440) | -| `DS4_METAL_ENABLE_STREAMING_EXPERT_MASKED_ADDR` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming expert masked address. | [ds4_metal.m:14604](ds4_metal.m#L14604) | -| `DS4_METAL_ENABLE_STREAMING_FULL_EXPERT_ADDR_TABLE` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming full expert address table. | [ds4_metal.m:14713](ds4_metal.m#L14713) | -| `DS4_METAL_ENABLE_STREAMING_IQ2_CPU_ROUTER` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming IQ2 CPU router. | [ds4.c:20765](ds4.c#L20765) | -| `DS4_METAL_ENABLE_STREAMING_MADVISE_WILLNEED` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming madvise willneed. | [ds4.c:18037](ds4.c#L18037) | -| `DS4_METAL_ENABLE_STREAMING_PREFILL_BATCH_SELECTED_ADDR` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming prefill batch selected address. | [ds4_metal.m:14607](ds4_metal.m#L14607) | -| `DS4_METAL_ENABLE_STREAMING_PREFILL_CACHE_SEED` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming prefill cache seed. | [ds4.c:21086](ds4.c#L21086) | -| `DS4_METAL_ENABLE_STREAMING_PREFILL_EXPERT_READAHEAD` | value-aware boolean; for batches <32 readahead is automatic; for batches >=32 unset/false disables and true enables; global READHEAD rollback and F_NOCACHE still dominate | Restores F_RDADVISE immediately before parallel pread for large SSD-prefill batches. | [ds4_metal.m:13210](ds4_metal.m#L13210) | -| `DS4_METAL_ENABLE_STREAMING_PREFILL_LAYER_PAGEIN` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming prefill layer pagein. | [ds4.c:18259](ds4.c#L18259) | -| `DS4_METAL_ENABLE_STREAMING_PREFILL_LAYER_READAHEAD` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming prefill layer readahead. | [ds4.c:18269](ds4.c#L18269) | -| `DS4_METAL_ENABLE_STREAMING_PREFILL_SELECTED_MADVISE` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming prefill selected madvise. | [ds4.c:18249](ds4.c#L18249) | -| `DS4_METAL_ENABLE_STREAMING_PREFILL_SELECTED_PAGEIN` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming prefill selected pagein. | [ds4.c:18239](ds4.c#L18239) | -| `DS4_METAL_ENABLE_STREAMING_PREFILL_SELECTED_READAHEAD` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming prefill selected readahead. | [ds4.c:19783](ds4.c#L19783) | -| `DS4_METAL_ENABLE_STREAMING_PREFILL_SELECTED_READAHEAD_SHARED` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming prefill selected readahead shared. | [ds4.c:19785](ds4.c#L19785) | -| `DS4_METAL_ENABLE_STREAMING_READAHEAD` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming readahead. | [ds4.c:18030](ds4.c#L18030) | -| `DS4_METAL_ENABLE_STREAMING_SELECTED_READAHEAD_SHARED_DELAY` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming selected readahead shared delay. | [ds4.c:21546](ds4.c#L21546) | -| `DS4_METAL_ENABLE_STREAMING_STATIC_DECODE_MAP` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming static decode map. | [ds4.c:18049](ds4.c#L18049) | -| `DS4_METAL_ENABLE_TOKEN_EMBED_EXACT_VIEW` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables token embed exact view. | [ds4_metal.m:12217](ds4_metal.m#L12217) | -| `DS4_METAL_EXACT_VIEW_CACHE_GIB` | unsigned GiB; default 64; 0 disables size-triggered eviction; MIB overrides it | Sets the cached exact-model-view eviction threshold. | [ds4_metal.m:1332](ds4_metal.m#L1332) | -| `DS4_METAL_EXACT_VIEW_CACHE_MIB` | unsigned MiB; unset: inherit GIB/default; 0 disables size-triggered eviction; overrides GIB | Sets the cached exact-model-view eviction threshold with MiB precision. | [ds4_metal.m:1341](ds4_metal.m#L1341) | -| `DS4_METAL_EXACT_VIEW_CACHE_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for exact view cache. | [ds4_metal.m:1376](ds4_metal.m#L1376) | -| `DS4_METAL_FLASH_ATTN_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the FlashAttention Metal kernel source file loaded at runtime. | [ds4_metal.m:4934](ds4_metal.m#L4934) | -| `DS4_METAL_FLASH_ATTN_STAGE_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for flash attn stage. | [ds4.c:64962](ds4.c#L64962) | -| `DS4_METAL_FLASH_ATTN_STAGE_PROFILE_FILTER` | substring; unset/empty: all profiled modes/stages | Filters FlashAttention stage-profile output by mode or stage substring. | [ds4_metal.m:11177](ds4_metal.m#L11177) | -| `DS4_METAL_GET_ROWS_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the get-rows Metal kernel source file loaded at runtime. | [ds4_metal.m:4945](ds4_metal.m#L4945) | -| `DS4_METAL_GLM_DISABLE_STREAMING_EXPERT_CACHE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming expert cache for GLM. | [ds4_metal.m:39633](ds4_metal.m#L39633) | -| `DS4_METAL_GLM_DISABLE_STREAMING_GROUPED_ADDR_PREFILL` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming grouped address prefill for GLM. | [ds4_metal.m:41047](ds4_metal.m#L41047) | -| `DS4_METAL_GLM_DISABLE_STREAMING_SEED_BEFORE_PREFILL` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming seed before prefill for GLM. | [ds4.c:50884](ds4.c#L50884) | -| `DS4_METAL_GLM_DISABLE_STREAMING_TOKEN_PREFILL` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming token prefill for GLM. | [ds4.c:49514](ds4.c#L49514) | -| `DS4_METAL_GLM_MOE_ONE_STAGE_PROFILE` | unset: off; 1/true/yes/on/all enables all layers; accepts layer lists/ranges; 0/false/no/off disables | Prints timing/profile diagnostics for GLM MoE one stage. | [ds4_metal.m:39930](ds4_metal.m#L39930) | -| `DS4_METAL_GLM_MOE_ONE_STAGE_PROFILE_LAYER` | layer index/list/ranges or all; unset: all layers selected by profiler | Restricts GLM one-stage MoE profiling to selected layers. | [ds4_metal.m:39931](ds4_metal.m#L39931) | -| `DS4_METAL_GLM_MOE_STAGE_PROFILE_FILTER` | substring; unset/empty: all profiled stages | Filters GLM MoE stage-profile output. | [ds4_metal.m:39934](ds4_metal.m#L39934) | -| `DS4_METAL_GLM_QKLOW_DEBUG` | presence diagnostic; unset: off; any value including 0 enables | Enables debug diagnostics for GLM qklow. | [ds4_metal.m:37834](ds4_metal.m#L37834) | -| `DS4_METAL_GLM_STREAMING_ASYNC_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for GLM streaming async. | [ds4.c:44172](ds4.c#L44172) | -| `DS4_METAL_GLM_STREAMING_DECODE_FULL_LAYER_MAP` | presence control; unset: off/default; any value including 0 enables | Maps complete GLM layers during SSD-streaming decode instead of decode-only spans. | [ds4.c:42426](ds4.c#L42426) | -| `DS4_METAL_GLM_STREAMING_DECODE_SYNC_EACH_LAYER` | boolean text; Metal runtime always synchronizes and does not consult it; legacy fallback name read only in ROCm builds | Controls per-layer GLM streaming decode synchronization only as a legacy ROCm fallback alias. | [ds4.c:49590](ds4.c#L49590) | -| `DS4_METAL_GLM_STREAMING_PREFILL_FULL_LAYER` | presence control; unset: off/default; any value including 0 enables | Forces full-layer GLM SSD prefill regardless of the token crossover. | [ds4_metal.m:14708](ds4_metal.m#L14708) | -| `DS4_METAL_GLM_STREAMING_PREFILL_FULL_LAYER_MIN_TOKENS` | positive uint32; default 64 on Metal, 1024 when used as ROCm fallback; 0/invalid restores default | Sets the token crossover for GLM full-layer SSD prefill. | [ds4.c:42503](ds4.c#L42503) | -| `DS4_METAL_GLM_STREAMING_PREFILL_SYNC_EACH_LAYER` | boolean text; Metal runtime always synchronizes and does not consult it; legacy fallback name read only in ROCm builds | Controls per-layer GLM streaming prefill synchronization only as a legacy ROCm fallback alias. | [ds4.c:42206](ds4.c#L42206) | -| `DS4_METAL_GLM_STREAMING_TOKEN_PREFILL_MAX` | uint32; default 64 on Metal, 0 when used as ROCm fallback; 0 disables; invalid restores default | Sets largest GLM SSD prefill handled token-major by the decode graph. | [ds4.c:49493](ds4.c#L49493) | -| `DS4_METAL_GLU_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the GLU Metal kernel source file loaded at runtime. | [ds4_metal.m:4949](ds4_metal.m#L4949) | -| `DS4_METAL_GPU_BATCH_EMBED_MIN` | uint32 token threshold; default 512; invalid restores default | Sets the batch size at which prompt embedding moves from CPU upload to Metal kernels. | [ds4.c:28694](ds4.c#L28694) | -| `DS4_METAL_GPU_BUSY_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for GPU busy. | [ds4_metal.m:1292](ds4_metal.m#L1292) | -| `DS4_METAL_GRAPH_DUMP_LAYER` | unsigned layer index or all; unset: every layer | Restricts graph tensor dumps to one layer. | [ds4.c:16690](ds4.c#L16690) | -| `DS4_METAL_GRAPH_DUMP_LOGITS` | file path; unset/empty: no graph-logit dump | Writes Metal graph-test logits as f32 binary. | [ds4.c:38890](ds4.c#L38890) | -| `DS4_METAL_GRAPH_DUMP_NAME` | substring; unset/empty: every tensor name | Restricts graph tensor dumps by tensor-name substring. | [ds4.c:16686](ds4.c#L16686) | -| `DS4_METAL_GRAPH_DUMP_POS` | unsigned token position; unset: every position | Restricts graph tensor dumps to one token position. | [ds4.c:16697](ds4.c#L16697) | -| `DS4_METAL_GRAPH_DUMP_PREFIX` | path/prefix; unset/empty: tensor dumping disabled | Enables graph tensor dumps and supplies the filename prefix. | [ds4.c:64958](ds4.c#L64958) | -| `DS4_METAL_GRAPH_DUMP_TRACE` | presence diagnostic; unset: off; any value including 0 enables | Emits trace diagnostics for graph dump. | [ds4.c:16727](ds4.c#L16727) | -| `DS4_METAL_GRAPH_OUTPUT_ROW` | zero-based row smaller than current batch; default final row; invalid restores final row | Chooses which prefill output row is projected to logits. | [ds4.c:35656](ds4.c#L35656) | -| `DS4_METAL_GRAPH_PREFILL_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for graph prefill. | [ds4.c:35333](ds4.c#L35333) | -| `DS4_METAL_GRAPH_PREFILL_SPLIT_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for graph prefill split. | [ds4.c:66014](ds4.c#L66014) | -| `DS4_METAL_GRAPH_PROMPT_TOKENS` | integer 1..prompt length; default full prompt | Limits prompt length used by the Metal graph parity test. | [ds4.c:38833](ds4.c#L38833) | -| `DS4_METAL_GRAPH_RAW_CAP` | positive rows; default from SWA window+prefill; clamped to [raw_window,min(ctx,8192)] | Overrides raw sliding-window KV ring capacity. | [ds4.c:38200](ds4.c#L38200) | -| `DS4_METAL_GRAPH_TEACHER_FORCE` | presence control; unset: off/default; any value including 0 enables | Feeds CPU reference state back into the first-token graph trace at each layer. | [ds4.c:27569](ds4.c#L27569) | -| `DS4_METAL_GRAPH_TOKEN_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for graph token. | [ds4.c:32192](ds4.c#L32192) | -| `DS4_METAL_GRAPH_TOKEN_SECOND_SPLIT_LAYERS` | integer 0..layer count; default 0 plus eligible automatic pre-M5 schedules; explicit value wins | Overrides second command-buffer split layer for token decode. | [ds4.c:27856](ds4.c#L27856) | -| `DS4_METAL_GRAPH_TOKEN_SPLIT_LAYERS` | integer 0..layer count; default 4 on Apple and 0 elsewhere, with eligible pre-M5 adaptive override | Overrides first command-buffer split layer for token decode. | [ds4.c:27742](ds4.c#L27742) | -| `DS4_METAL_GRAPH_TRACE_CACHE` | presence control; unset: off/default; any value including 0 enables | Prints raw KV cache parity diagnostics in the graph prompt test. | [ds4.c:38902](ds4.c#L38902) | -| `DS4_METAL_GRAPH_TRACE_COMP` | presence control; unset: off/default; any value including 0 enables | Prints compressed-cache parity diagnostics in the graph prompt test. | [ds4.c:38903](ds4.c#L38903) | -| `DS4_METAL_GRAPH_TRACE_LAYERS` | presence control; unset: off/default; any value including 0 enables | Enables per-layer first-token CPU/GPU graph tracing. | [ds4.c:27566](ds4.c#L27566) | -| `DS4_METAL_GRAPH_TRACE_STAGE_LAYER` | signed layer index; unset gives -1/no stage-layer selection | Selects the layer used by first-token stage tracing. | [ds4.c:27570](ds4.c#L27570) | -| `DS4_METAL_HC_NORM_FUSION_CHECK` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Compares fused HC normalization against the reference result. | [ds4.c:20348](ds4.c#L20348) | -| `DS4_METAL_HC_NORM_FUSION_CHECK_TOL` | positive finite float; default 2e-4; invalid/nonpositive restores default | Sets the numerical tolerance for the HC norm-fusion oracle. | [ds4.c:20357](ds4.c#L20357) | -| `DS4_METAL_HC_STABLE` | boolean empty/1/true/yes/on vs 0/false/no/off; default on | Compiles stable hidden-context drift arithmetic into the Metal library. | [ds4_metal.m:7046](ds4_metal.m#L7046) | -| `DS4_METAL_INDEXER_STAGE_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for indexer stage. | [ds4.c:17396](ds4.c#L17396) | -| `DS4_METAL_IQ2_XXS_SSD_PREFILL_MM_STATS` | value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables | Collects/prints statistics for IQ2 XXS SSD prefill MM. | [ds4_metal.m:6979](ds4_metal.m#L6979) | -| `DS4_METAL_KV_RAW_F32` | boolean empty/1/true/yes/on vs 0/false/no/off; default off | Compiles raw KV storage as F32 for drift diagnosis. | [ds4_metal.m:7048](ds4_metal.m#L7048) | -| `DS4_METAL_LAYER_STAGE_PROFILE` | unset: off; 1/true/yes/on/all enables all layers; a layer index selects one; 0/false/no/off disables | Prints timing/profile diagnostics for layer stage. | [ds4.c:64960](ds4.c#L64960) | -| `DS4_METAL_LAYER_STAGE_PROFILE_LAYER` | single unsigned layer index; unset/empty: all layers enabled by the parent profile; invalid matches no layer | Restricts the corresponding shared graph stage profiler to one layer. | [ds4.c:28936](ds4.c#L28936) | -| `DS4_METAL_MATH_SAFE` | boolean empty/1/true/yes/on vs 0/false/no/off; default off | Compiles Metal shaders with strict/safe IEEE math instead of fast math. | [ds4_metal.m:7050](ds4_metal.m#L7050) | -| `DS4_METAL_MEMORY_REPORT` | presence control; unset: off/default; any value including 0 enables | Prints Metal allocation/cache/residency memory reports. | [ds4.c:38857](ds4.c#L38857) | -| `DS4_METAL_MODEL_UNTRACKED` | presence control; unset: off/default; any value including 0 enables | Creates mapped model buffers with untracked Metal hazard tracking. | [ds4_metal.m:1546](ds4_metal.m#L1546) | -| `DS4_METAL_MODEL_VIEW_MAX_GIB` | positive integer GiB; default device maximum (128-GiB cap for already-split span maps); cannot exceed device maximum | Caps each no-copy mapped Metal model view. | [ds4_metal.m:2215](ds4_metal.m#L2215) | -| `DS4_METAL_MODEL_WARMUP_STRIDE_KB` | integer 1..1048576 KiB, at least one page; unset inherits MB/default; overrides STRIDE_MB | Sets the model-view warmup touch stride with KiB precision. | [ds4_metal.m:3056](ds4_metal.m#L3056) | -| `DS4_METAL_MODEL_WARMUP_STRIDE_MB` | integer 1..1024 MiB; default 1 MiB; STRIDE_KB overrides | Sets the model-view warmup touch stride. | [ds4_metal.m:3048](ds4_metal.m#L3048) | -| `DS4_METAL_MOE_MM_ID_USE_RESOURCES` | presence control; unset: off/default; any value including 0 enables | Declares MM-ID MoE resource usage explicitly on the command encoder. | [ds4_metal.m:35134](ds4_metal.m#L35134) | -| `DS4_METAL_MOE_ONE_STAGE_PROFILE` | unset: off; 1/true/yes/on/all enables all layers; accepts layer lists/ranges; 0/false/no/off disables | Prints timing/profile diagnostics for MoE one stage. | [ds4.c:22541](ds4.c#L22541) | -| `DS4_METAL_MOE_ONE_STAGE_PROFILE_LAYER` | layer index/list/ranges or all; unset: all profiler-selected layers | Restricts one-stage MoE profiling to selected layers. | [ds4_metal.m:43402](ds4_metal.m#L43402) | -| `DS4_METAL_MOE_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the MoE Metal kernel source file loaded at runtime. | [ds4_metal.m:4936](ds4_metal.m#L4936) | -| `DS4_METAL_MOE_STAGE_PROFILE` | unset: off; 1/true/yes/on/all enables all layers; accepts layer lists/ranges; 0/false/no/off disables | Prints timing/profile diagnostics for MoE stage. | [ds4.c:64964](ds4.c#L64964) | -| `DS4_METAL_MOE_STAGE_PROFILE_FILTER` | substring; unset/empty: all profiled stages | Filters MoE stage-profile output. | [ds4_metal.m:43404](ds4_metal.m#L43404) | -| `DS4_METAL_MOE_STAGE_PROFILE_LAYER` | layer index/list/ranges or all; unset: all profiler-selected layers | Restricts batched MoE stage profiling to selected layers. | [ds4_metal.m:45312](ds4_metal.m#L45312) | -| `DS4_METAL_MOE_WRITE_CLAMPED_ACT` | presence control; unset: off/default; any value including 0 enables | Makes routed MoE write the clamped activation diagnostic. | [ds4.c:18421](ds4.c#L18421) | -| `DS4_METAL_NORM_RSQRT_DISABLE` | boolean empty/1/true/yes/on vs 0/false/no/off; default on | Compiles unified normalization-rsqrt arithmetic into the Metal library. | [ds4_metal.m:7047](ds4_metal.m#L7047) | -| `DS4_METAL_NORM_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the normalization Metal kernel source file loaded at runtime. | [ds4_metal.m:4950](ds4_metal.m#L4950) | -| `DS4_METAL_NO_MODEL_WARMUP` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables model warmup. | [ds4_metal.m:2308](ds4_metal.m#L2308) | -| `DS4_METAL_NO_PREFILL_KERNEL_WARMUP` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables prefill kernel warmup. | [ds4.c:28792](ds4.c#L28792) | -| `DS4_METAL_NO_RESIDENCY` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables residency. | [ds4_metal.m:2091](ds4_metal.m#L2091) | -| `DS4_METAL_OUTPUT_STAGE_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for output stage. | [ds4.c:17397](ds4.c#L17397) | -| `DS4_METAL_PREFILL_CHUNK` | positive token count used only when CLI chunk is absent; default full prompt, or 4096 for long non-PRO and 8192 for long PRO prompts; <=0 keeps automatic/full prompt | Provides the historical environment fallback for prefill chunk size. | [ds4.c:12629](ds4.c#L12629) | -| `DS4_METAL_PRO_Q4_CPU_ROUTER` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Uses the CPU router for PRO Q4 selected-expert decode. | [ds4.c:20761](ds4.c#L20761) | -| `DS4_METAL_PRO_Q4_CPU_ROUTER_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for pro Q4 CPU router. | [ds4.c:21471](ds4.c#L21471) | -| `DS4_METAL_Q4_ADDR_USE_RESOURCES` | presence control; unset: off/default; any value including 0 enables | Declares Q4 address-table resources explicitly on encoders. | [ds4_metal.m:20257](ds4_metal.m#L20257) | -| `DS4_METAL_Q4_EXPERT_GROUP_SIZE` | positive uint32; default 32; clamped to total expert count | Sets experts processed per grouped Q4 dispatch. | [ds4_metal.m:34054](ds4_metal.m#L34054) | -| `DS4_METAL_Q4_EXPERT_TABLE_GROUP_SIZE` | integer 2..total experts; default/invalid 1 (ungrouped) | Sets grouped exact-view width while building Q4 expert tables. | [ds4_metal.m:19602](ds4_metal.m#L19602) | -| `DS4_METAL_Q4_EXPERT_TABLE_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for Q4 expert table. | [ds4_metal.m:20101](ds4_metal.m#L20101) | -| `DS4_METAL_Q4_GROUP24_BASE_VIEWS` | presence control; unset: off/default; any value including 0 enables | Uses broad base model views for Q4 group-24 instead of exact views. | [ds4_metal.m:42171](ds4_metal.m#L42171) | -| `DS4_METAL_Q4_GROUP24_EXACT_VIEWS` | presence control; unset: off/default; any value including 0 enables | Uses exact mapped views for Q4 group-24 experts. | [ds4_metal.m:42170](ds4_metal.m#L42170) | -| `DS4_METAL_Q4_GROUPED_CACHE_VIEWS` | presence control; unset: off/default; any value including 0 enables | Caches exact Q4 grouped expert views. | [ds4_metal.m:42117](ds4_metal.m#L42117) | -| `DS4_METAL_Q4_PRO_MAP_GROUPS` | positive divisor of 384 in 1..384; default/invalid 1 | Splits each 384-expert PRO Q4 tensor into this many mapped views. | [ds4.c:6153](ds4.c#L6153) | -| `DS4_METAL_Q4_SELECTED_EXACT_VIEWS` | presence control; unset: off/default; any value including 0 enables | Forces exact/cached views for selected Q4 experts instead of base views. | [ds4_metal.m:42494](ds4_metal.m#L42494) | -| `DS4_METAL_Q4_SELECTED_OVERLAP_SHARED` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Overlaps selected Q4 expert preparation with the shared expert. | [ds4.c:20789](ds4.c#L20789) | -| `DS4_METAL_Q4_SELECTED_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for Q4 selected. | [ds4.c:37754](ds4.c#L37754) | -| `DS4_METAL_Q4_SELECTED_PROFILE_LAYER` | single nonnegative layer index; unset: every layer | Restricts legacy Q4 selected-expert profiling to one layer. | [ds4_metal.m:42475](ds4_metal.m#L42475) | -| `DS4_METAL_Q4_SELECTED_SHARED_EVENT` | presence control; unset: off/default; any value including 0 enables | Coordinates selected Q4 work with a shared Metal event. | [ds4_metal.m:42490](ds4_metal.m#L42490) | -| `DS4_METAL_Q4_SELECTED_TRANSIENT_VIEWS` | presence control; unset: off/default; any value including 0 enables | Uses transient exact views for selected Q4 experts. | [ds4_metal.m:42498](ds4_metal.m#L42498) | -| `DS4_METAL_Q4_SELECTED_USE_BASE_VIEWS` | presence control; unset: off/default; any value including 0 enables | Uses broad base model views for selected Q4 experts. | [ds4_metal.m:42493](ds4_metal.m#L42493) | -| `DS4_METAL_Q4_TABLE_BIND_ANCHORS` | presence control; unset: off/default; any value including 0 enables | Binds anchor buffers alongside the Q4 expert address table. | [ds4_metal.m:19714](ds4_metal.m#L19714) | -| `DS4_METAL_Q4_TABLE_MODEL_RESIDENCY_SET` | presence control; unset: off/default; any value including 0 enables | Adds Q4 expert table allocations to the model residency set. | [ds4_metal.m:19655](ds4_metal.m#L19655) | -| `DS4_METAL_Q4_TABLE_PER_TENSOR_RESIDENCY_SET` | presence control; unset: off/default; any value including 0 enables | Builds separate residency sets per Q4 expert tensor. | [ds4_metal.m:19758](ds4_metal.m#L19758) | -| `DS4_METAL_Q4_TABLE_QUEUE_RESIDENCY_SET` | presence control; unset: off/default; any value including 0 enables | Attaches Q4 expert table residency sets to command queues. | [ds4_metal.m:19613](ds4_metal.m#L19613) | -| `DS4_METAL_Q4_TABLE_RESIDENCY_SET` | presence control; unset: off/default; any value including 0 enables | Enables Q4 expert table residency-set handling. | [ds4_metal.m:19757](ds4_metal.m#L19757) | -| `DS4_METAL_Q4_TABLE_USE_RESOURCES` | presence control; unset: off/default; any value including 0 enables | Declares Q4 table resources explicitly on encoders. | [ds4_metal.m:20256](ds4_metal.m#L20256) | -| `DS4_METAL_Q8_DECODE_EXACT_VIEW_MAX_MIB` | integer 1..4096 MiB; default 1024; above max clamps, below min/invalid restores default | Caps weight ranges eligible for Q8 exact model views. | [ds4_metal.m:12874](ds4_metal.m#L12874) | -| `DS4_METAL_Q8_MV_EXT_MAX_TOKENS` | integer 2..128; default 16; above max clamps, below min/invalid restores default | Sets largest batch handled by extended Q8 matvec. | [ds4_metal.m:21124](ds4_metal.m#L21124) | -| `DS4_METAL_Q8_MV_NSG` | integer 1..8 simdgroups; default 4, or 2 with TP world=2; above max clamps, below min/invalid restores default | Overrides simdgroups per Q8 matvec threadgroup. | [ds4.c:22557](ds4.c#L22557) | -| `DS4_METAL_Q8_PREFILL_PROFILE` | value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables | Prints timing/profile diagnostics for Q8 prefill. | [ds4_metal.m:21278](ds4_metal.m#L21278) | -| `DS4_METAL_Q8_PREFILL_PROFILE_FILTER` | substring matched against generated operation label; unset/empty: all eligible calls | Filters Q8 prefill profiling. | [ds4_metal.m:21293](ds4_metal.m#L21293) | -| `DS4_METAL_Q_STAGE_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for q stage. | [ds4.c:29097](ds4.c#L29097) | -| `DS4_METAL_REPEAT_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the repeat Metal kernel source file loaded at runtime. | [ds4_metal.m:4948](ds4_metal.m#L4948) | -| `DS4_METAL_REQUIRE_COMPRESSOR_EXACT_POOL_RATIO4` | presence strict check; unset: fallback allowed; any value including 0 requires the path | Requires compressor exact pool ratio4 and makes eligible fallback fail closed. | [ds4_metal.m:26385](ds4_metal.m#L26385) | -| `DS4_METAL_REQUIRE_EXACT_ROWS_PERSISTENT_CACHE` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Requires exact rows persistent cache and makes eligible fallback fail closed. | [ds4_metal.m:13000](ds4_metal.m#L13000) | -| `DS4_METAL_REQUIRE_GATHERED_KV_STAGE` | presence strict check; unset: fallback allowed; any value including 0 requires the path | Requires gathered KV stage and makes eligible fallback fail closed. | [ds4_metal.m:29656](ds4_metal.m#L29656) | -| `DS4_METAL_REQUIRE_IQ2_XXS_SSD_PREFILL_MM` | value-aware boolean; default implicit fail-closed only with complete selected-address domain; explicit 1 is strict, 0 permits fallback | Makes eligible IQ2_XXS/Q2_K grouped SSD-prefill MM fail closed. | [ds4_metal.m:44748](ds4_metal.m#L44748) | -| `DS4_METAL_REQUIRE_M1_IQ2_MID_ONLY` | presence strict check; unset: fallback allowed; any value including 0 requires the path | Requires M1 IQ2 mid only and makes eligible fallback fail closed. | [ds4_metal.m:42040](ds4_metal.m#L42040) | -| `DS4_METAL_REQUIRE_OUTPUT_HC_WEIGHTS4` | presence strict check; unset: fallback allowed; any value including 0 requires the path | Requires output HC weights4 and makes eligible fallback fail closed. | [ds4_metal.m:46755](ds4_metal.m#L46755) | -| `DS4_METAL_REQUIRE_Q4_ATTN_OUT_TINY_BATCH` | value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables | Requires Q4 attn out tiny batch and makes eligible fallback fail closed. | [ds4_metal.m:28349](ds4_metal.m#L28349) | -| `DS4_METAL_REQUIRE_Q4_SSD_PREFILL_ATTN_OUT_EXACTN` | value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables | Requires Q4 SSD prefill attn out exactn and makes eligible fallback fail closed. | [ds4_metal.m:28065](ds4_metal.m#L28065) | -| `DS4_METAL_REQUIRE_Q4_SSD_SESSION_UNION` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Requires Q4 SSD session union and makes eligible fallback fail closed. | [ds4.c:64974](ds4.c#L64974) | -| `DS4_METAL_REQUIRE_Q8_QKV_COMPRESSOR_FUSE` | nonempty boolean; unset/empty/exact 0: fallback allowed; other values require and imply the streamed/union enable | Requires eligible Q8 QKV/compressor compound fusion and fails closed. | [ds4.c:22850](ds4.c#L22850) | -| `DS4_METAL_RESUME_PREFILL_MIN` | integer token threshold; default 4; <=0 disables resume-prefill | Sets the minimum shared-prefix suffix that uses batched resume-prefill. | [ds4.c:38229](ds4.c#L38229) | -| `DS4_METAL_ROPE_EXP2_LOG2` | boolean empty/1/true/yes/on vs 0/false/no/off; default off | Compiles the exp2/log2 RoPE drift variant. | [ds4_metal.m:7049](ds4_metal.m#L7049) | -| `DS4_METAL_SELECTED_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for selected. | [ds4.c:37753](ds4.c#L37753) | -| `DS4_METAL_SELECTED_PROFILE_LAYER` | single nonnegative layer index; unset: every layer | Restricts selected-expert profiling to one layer. | [ds4_metal.m:42473](ds4_metal.m#L42473) | -| `DS4_METAL_SESSION_BATCH_LOG` | presence diagnostic; unset: off; any value including 0 enables | Logs session batch decisions. | [ds4.c:65978](ds4.c#L65978) | -| `DS4_METAL_SESSION_BATCH_QKV` | default enabled; exact 0 disables; every other value/unset leaves enabled | Controls native batched QKV work for multi-session decode. | [ds4.c:65292](ds4.c#L65292) | -| `DS4_METAL_SESSION_BATCH_SHARED` | default enabled; exact 0 disables; every other value/unset leaves enabled | Controls native batched shared-expert work for multi-session decode. | [ds4.c:65242](ds4.c#L65242) | -| `DS4_METAL_SET_ROWS_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the set-rows Metal kernel source file loaded at runtime. | [ds4_metal.m:4952](ds4_metal.m#L4952) | -| `DS4_METAL_SOFTMAX_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the softmax Metal kernel source file loaded at runtime. | [ds4_metal.m:4947](ds4_metal.m#L4947) | -| `DS4_METAL_STREAMING_DECODE_PREFILL_MAX` | integer token maximum; default 64 for wide Flash Q4/MXFP4, 18 for other PRO/Flash, 0 otherwise; <=0 disables | Sets maximum SSD-streaming micro-prefill width that reuses decode. | [ds4.c:31772](ds4.c#L31772) | -| `DS4_METAL_STREAMING_EXPERT_AUTO_PRELOAD_CAP` | uint32 expert cap; default 4096; 0 means unlimited; invalid restores default | Caps automatic streaming-expert hotlist preload. | [ds4.c:21282](ds4.c#L21282) | -| `DS4_METAL_STREAMING_EXPERT_BUFFER_MLOCK_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming expert buffer mlock. | [ds4_metal.m:13995](ds4_metal.m#L13995) | -| `DS4_METAL_STREAMING_EXPERT_EARLY_LOAD_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming expert early load. | [ds4_metal.m:17057](ds4_metal.m#L17057) | -| `DS4_METAL_STREAMING_EXPERT_EVICT_DONTNEED_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming expert evict dontneed. | [ds4_metal.m:14434](ds4_metal.m#L14434) | -| `DS4_METAL_STREAMING_EXPERT_HOTLIST` | hotlist file path; unset/empty: built-in model hotlist | Loads the streaming-expert preload order from a file. | [ds4.c:21322](ds4.c#L21322) | -| `DS4_METAL_STREAMING_EXPERT_HOTLIST_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming expert hotlist. | [ds4.c:32055](ds4.c#L32055) | -| `DS4_METAL_STREAMING_EXPERT_LAYER_STATS` | presence diagnostic; unset: off; any value including 0 enables | Collects/prints statistics for streaming expert layer. | [ds4_metal.m:4637](ds4_metal.m#L4637) | -| `DS4_METAL_STREAMING_EXPERT_LAYER_STATS_DELTA` | presence control; unset: off/default; any value including 0 enables | Prints delta statistics for streaming expert layer. | [ds4_metal.m:4674](ds4_metal.m#L4674) | -| `DS4_METAL_STREAMING_EXPERT_NOCACHE` | nonempty value whose first character is not 0 enables; unset/empty/0 disables | Uses a reopened F_NOCACHE descriptor for SSD expert preads. | [ds4_metal.m:12600](ds4_metal.m#L12600) | -| `DS4_METAL_STREAMING_EXPERT_PREAD_POOL` | default enabled; exact 0 disables; every other value/unset keeps enabled | Controls reuse of persistent expert-pread worker threads. | [ds4_metal.m:13431](ds4_metal.m#L13431) | -| `DS4_METAL_STREAMING_EXPERT_PREAD_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming expert pread. | [ds4_metal.m:17003](ds4_metal.m#L17003) | -| `DS4_METAL_STREAMING_EXPERT_PREAD_SPLIT` | integer clamped 1..8; unset: automatic 1 below 64 cache experts, 4 at 64+ | Sets aligned requests per expert pread. | [ds4_metal.m:13699](ds4_metal.m#L13699) | -| `DS4_METAL_STREAMING_EXPERT_PREAD_THREADS` | unsigned integer clamped 1..18; default 9; invalid restores 9 | Sets expert-pread worker limit. | [ds4_metal.m:13335](ds4_metal.m#L13335) | -| `DS4_METAL_STREAMING_EXPERT_PROFILE_SUMMARY` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming expert. | [ds4_metal.m:13059](ds4_metal.m#L13059) | -| `DS4_METAL_STREAMING_EXPERT_SLAB_MB` | positive unsigned MiB; default 4096; 0/invalid restores default | Sets target allocation size for streaming-expert slabs. | [ds4_metal.m:14037](ds4_metal.m#L14037) | -| `DS4_METAL_STREAMING_EXPERT_SPLIT_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming expert split. | [ds4_metal.m:43776](ds4_metal.m#L43776) | -| `DS4_METAL_STREAMING_EXPERT_TIMING_SUMMARY` | presence control; unset: off/default; any value including 0 enables | Prints timing/profile diagnostics for streaming expert. | [ds4_metal.m:13058](ds4_metal.m#L13058) | -| `DS4_METAL_STREAMING_IQ2_CPU_ROUTER_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming IQ2 CPU router. | [ds4.c:21472](ds4.c#L21472) | -| `DS4_METAL_STREAMING_MAP_TRACE` | nonempty value other than exact 0 enables; unset/empty/0 disables | Emits SSD model-map decisions. | [ds4_metal.m:4848](ds4_metal.m#L4848) | -| `DS4_METAL_STREAMING_PREFILL_BATCH_SELECTED_ADDR_MAX` | integer token maximum; default 800 for 384 experts, 760 for 256, 0 otherwise; <=0 disables automatic selection | Sets automatic maximum batch width for selected-address SSD prefill. | [ds4_metal.m:14637](ds4_metal.m#L14637) | -| `DS4_METAL_STREAMING_PREFILL_BATCH_SELECTED_ADDR_MIN` | integer token minimum; default 2 for 256/384 experts, 0 otherwise; <=0 disables automatic selection | Sets automatic minimum batch width for selected-address SSD prefill. | [ds4_metal.m:14654](ds4_metal.m#L14654) | -| `DS4_METAL_STREAMING_PREFILL_BATCH_SELECTED_ADDR_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming prefill batch selected address. | [ds4_metal.m:18491](ds4_metal.m#L18491) | -| `DS4_METAL_STREAMING_PREFILL_CACHE_SEED_K` | uint32 seed rows; default 1; 0 disables; above 64 clamps to 64 | Sets how many prefill routing rows seed the decode expert cache. | [ds4.c:21095](ds4.c#L21095) | -| `DS4_METAL_STREAMING_PREFILL_CACHE_SEED_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming prefill cache seed. | [ds4_metal.m:17897](ds4_metal.m#L17897) | -| `DS4_METAL_STREAMING_PREFILL_LAYER_MADVISE_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming prefill layer madvise. | [ds4.c:19437](ds4.c#L19437) | -| `DS4_METAL_STREAMING_PREFILL_LAYER_PAGEIN_NO_OVERLAP` | presence rollback; unset: automatic/default path; any value including 0 disables | Prevents full-layer page-in preparation from overlapping compute. | [ds4.c:19144](ds4.c#L19144) | -| `DS4_METAL_STREAMING_PREFILL_LAYER_PAGEIN_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming prefill layer pagein. | [ds4.c:19433](ds4.c#L19433) | -| `DS4_METAL_STREAMING_PREFILL_LAYER_PAGEIN_THREADS` | integer 1..16; default 8; invalid/0 becomes 1; PREPARE_THREADS takes precedence | Sets worker count for full-layer page-in preparation. | [ds4.c:19102](ds4.c#L19102) | -| `DS4_METAL_STREAMING_PREFILL_LAYER_PREAD_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming prefill layer pread. | [ds4.c:19435](ds4.c#L19435) | -| `DS4_METAL_STREAMING_PREFILL_LAYER_PREPARE_AHEAD` | integer 1..4 layers; default 1; invalid/0 becomes 1 | Sets number of future layers prepared concurrently. | [ds4.c:19156](ds4.c#L19156) | -| `DS4_METAL_STREAMING_PREFILL_LAYER_PREPARE_NO_OVERLAP` | presence rollback; unset: automatic/default path; any value including 0 disables | Prevents generic full-layer preparation from overlapping compute. | [ds4.c:19142](ds4.c#L19142) | -| `DS4_METAL_STREAMING_PREFILL_LAYER_PREPARE_THREADS` | integer 1..16; default 8; invalid/0 becomes 1; preferred over PAGEIN_THREADS | Sets worker count for full-layer preparation. | [ds4.c:19098](ds4.c#L19098) | -| `DS4_METAL_STREAMING_PREFILL_LAYER_READAHEAD_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming prefill layer readahead. | [ds4.c:19439](ds4.c#L19439) | -| `DS4_METAL_STREAMING_PREFILL_SELECTED_MADVISE_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming prefill selected madvise. | [ds4.c:19183](ds4.c#L19183) | -| `DS4_METAL_STREAMING_PREFILL_SELECTED_MADVISE_THREADS` | integer 1..16; default inherits layer prepare threads; invalid/0 becomes 1; PREPARE_THREADS preferred | Sets worker count for selected-expert madvise preparation. | [ds4.c:19120](ds4.c#L19120) | -| `DS4_METAL_STREAMING_PREFILL_SELECTED_PAGEIN_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming prefill selected pagein. | [ds4.c:19181](ds4.c#L19181) | -| `DS4_METAL_STREAMING_PREFILL_SELECTED_PREPARE_GAP` | integer 0..8 layers; default 0; above 8 clamps; invalid restores 0 | Sets lookahead gap for selected-expert preparation. | [ds4.c:19132](ds4.c#L19132) | -| `DS4_METAL_STREAMING_PREFILL_SELECTED_PREPARE_THREADS` | integer 1..16 for madvise preparation; default inherits layer prepare threads; invalid/0 becomes 1 | Sets worker count for selected-expert preparation. | [ds4.c:19116](ds4.c#L19116) | -| `DS4_METAL_STREAMING_PREFILL_SELECTED_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming prefill selected. | [ds4.c:18733](ds4.c#L18733) | -| `DS4_METAL_STREAMING_PREFILL_SELECTED_READAHEAD_GAP` | integer 0..8 layers; default 0; above 8 clamps; invalid restores 0 | Sets lookahead gap for selected-expert readahead. | [ds4.c:19805](ds4.c#L19805) | -| `DS4_METAL_STREAMING_PREFILL_SELECTED_READAHEAD_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming prefill selected readahead. | [ds4.c:19890](ds4.c#L19890) | -| `DS4_METAL_STREAMING_SELECTED_READAHEAD_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming selected readahead. | [ds4.c:21564](ds4.c#L21564) | -| `DS4_METAL_SUM_ROWS_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the sum-rows Metal kernel source file loaded at runtime. | [ds4_metal.m:4946](ds4_metal.m#L4946) | -| `DS4_METAL_TEST_POISON_COMPRESSOR_EXACT_REDUCTION_SCRATCH` | internal test presence flag; unset: off; any value including 0 poisons scratch before the exact reduction | Validates that compressor exact-reduction kernels overwrite all scratch state. | [ds4_metal.m:25923](ds4_metal.m#L25923) | -| `DS4_METAL_TP_SESSION_BATCH` | default enabled; exact 0 disables; every other value/unset leaves enabled | Controls batched session evaluation with Metal TP. | [ds4.c:65120](ds4.c#L65120) | -| `DS4_METAL_TRACE_ALLOCS` | presence diagnostic; unset: off; any value including 0 enables | Emits trace diagnostics for allocs. | [ds4_metal.m:4056](ds4_metal.m#L4056) | -| `DS4_METAL_TRACE_M5_FLASH_ATTN_PACKED32_REDUCE` | presence diagnostic; unset: off; any value including 0 enables | Emits trace diagnostics for M5 flash attn packed32 reduce. | [ds4_metal.m:31506](ds4_metal.m#L31506) | -| `DS4_METAL_UNARY_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the unary operations Metal kernel source file loaded at runtime. | [ds4_metal.m:4938](ds4_metal.m#L4938) | -| `DS4_METAL_UNRETAINED_COMMAND_BUFFERS` | presence control; unset: off/default; any value including 0 enables | Creates Metal command buffers with unretained references. | [ds4_metal.m:1314](ds4_metal.m#L1314) | -| `DS4_METAL_USE_QUEUE_RESIDENCY_SET` | presence control; unset: off/default; any value including 0 enables | Allows queue-residency state to trigger Q4 expert address/table paths. | [ds4_metal.m:42246](ds4_metal.m#L42246) | +| `DS4_METAL_ARGSORT_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the argsort Metal kernel source file loaded at runtime. | [ds4_metal.m:4943](ds4_metal.m#L4943) | +| `DS4_METAL_ATTN_OUT_STAGE_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for attn out stage. | [ds4.c:65151](ds4.c#L65151) | +| `DS4_METAL_BIN_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the binary operations Metal kernel source file loaded at runtime. | [ds4_metal.m:4952](ds4_metal.m#L4952) | +| `DS4_METAL_COMPRESSOR_PAIR_NR4` | presence control; unset: off/default; any value including 0 enables | Selects the NR4 compressor-pair variant. | [ds4_metal.m:2651](ds4_metal.m#L2651) | +| `DS4_METAL_CONCAT_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the concatenation Metal kernel source file loaded at runtime. | [ds4_metal.m:4945](ds4_metal.m#L4945) | +| `DS4_METAL_CPY_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the copy Metal kernel source file loaded at runtime. | [ds4_metal.m:4944](ds4_metal.m#L4944) | +| `DS4_METAL_DECODE_INDEXER_SPARSE_THRESHOLD` | integer in {64,128,256,512,1024,2048,4096}; default 1024; invalid restores default | Sets the compressed-row crossover from dense to sparse indexed attention. | [ds4.c:20352](ds4.c#L20352) | +| `DS4_METAL_DECODE_STAGE_PROFILE` | unset: off; 1/true/yes/on/all enables all layers; a layer index selects one; 0/false/no/off disables | Prints timing/profile diagnostics for decode stage. | [ds4.c:17568](ds4.c#L17568) | +| `DS4_METAL_DECODE_STAGE_PROFILE_LAYER` | single unsigned layer index; unset/empty: all layers enabled by the parent profile; invalid matches no layer | Restricts the corresponding shared graph stage profiler to one layer. | [ds4.c:29120](ds4.c#L29120) | +| `DS4_METAL_DENSE_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the dense matmul Metal kernel source file loaded at runtime. | [ds4_metal.m:4936](ds4_metal.m#L4936) | +| `DS4_METAL_DISABLE_AFFINE_ROPE_PAIR` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables affine RoPE pair. | [ds4_metal.m:25171](ds4_metal.m#L25171) | +| `DS4_METAL_DISABLE_ATTN_OUT_HC_FUSION` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Disables attn out HC fusion. | [ds4.c:20477](ds4.c#L20477) | +| `DS4_METAL_DISABLE_ATTN_OUT_IDS_CACHE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables attn out ids cache. | [ds4_metal.m:27771](ds4_metal.m#L27771) | +| `DS4_METAL_DISABLE_ATTN_OUT_LOW_DIRECT` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables attn out low direct. | [ds4_metal.m:27760](ds4_metal.m#L27760) | +| `DS4_METAL_DISABLE_BATCH_HC_NORM_FUSION` | nonempty value other than exact 0 disables; unset/empty/0 leaves the default enabled path | Dominant rollback for batched HC norm fusion. | [ds4.c:20457](ds4.c#L20457) | +| `DS4_METAL_DISABLE_COMPRESSOR_APE_ADD` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables compressor APE add. | [ds4_metal.m:25393](ds4_metal.m#L25393) | +| `DS4_METAL_DISABLE_COMPRESSOR_EXACT_POOL_RATIO4` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables compressor exact pool ratio4. | [ds4_metal.m:26379](ds4_metal.m#L26379) | +| `DS4_METAL_DISABLE_COMPRESSOR_PAIR_PROJ` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Disables compressor pair proj. | [ds4_metal.m:23248](ds4_metal.m#L23248) | +| `DS4_METAL_DISABLE_COMPRESSOR_QUAD_STORE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables compressor quad store. | [ds4.c:23534](ds4.c#L23534) | +| `DS4_METAL_DISABLE_COMPRESSOR_RATIO4_DIRECT_POOL` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables compressor ratio4 direct pool. | [ds4_metal.m:26245](ds4_metal.m#L26245) | +| `DS4_METAL_DISABLE_COMPRESSOR_RATIO4_PACK_FUSION` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables compressor ratio4 pack fusion. | [ds4_metal.m:26199](ds4_metal.m#L26199) | +| `DS4_METAL_DISABLE_COMPRESSOR_STORE_ONE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables compressor store one. | [ds4_metal.m:23249](ds4_metal.m#L23249) | +| `DS4_METAL_DISABLE_CONTIG_F16_F16_COPY` | value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables | Disables contig F16 F16 copy. | [ds4_metal.m:29562](ds4_metal.m#L29562) | +| `DS4_METAL_DISABLE_CONTIG_F32_F16_COPY` | value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables | Disables contig F32 F16 copy. | [ds4_metal.m:29338](ds4_metal.m#L29338) | +| `DS4_METAL_DISABLE_DECODE_NORM_EXACT_VIEWS` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables decode norm exact views. | [ds4_metal.m:36207](ds4_metal.m#L36207) | +| `DS4_METAL_DISABLE_DECODE_RAW_GATHERED_ATTN` | presence rollback; unset: raw-only decode uses gathered attention; any value including 0 restores the legacy raw path | Restores the separate raw-only attention path instead of gathered staging and attention. | [ds4_metal.m:32968](ds4_metal.m#L32968) | +| `DS4_METAL_DISABLE_DECODE_RAW_PACKED32` | presence rollback; unset: raw-only gathered attention may use packed32; any value including 0 disables it for raw-only layers | Disables the packed32 reduce kernel for raw-only gathered attention while leaving compressed layers unchanged. | [ds4_metal.m:31464](ds4_metal.m#L31464) | +| `DS4_METAL_DISABLE_DECODE_ROUTER_BIAS_EXACT_VIEWS` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables decode router bias exact views. | [ds4_metal.m:39397](ds4_metal.m#L39397) | +| `DS4_METAL_DISABLE_DSPARK_CAPTURE_FUSED_LAST` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Disables DSpark capture fused last. | [ds4.c:28361](ds4.c#L28361) | +| `DS4_METAL_DISABLE_DSPARK_EXACTN_BATCH_HEAD` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Disables DSpark exactn batch head. | [ds4.c:37728](ds4.c#L37728) | +| `DS4_METAL_DISABLE_EXACT_ROWS_PERSISTENT_CACHE` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Disables exact rows persistent cache. | [ds4_metal.m:13000](ds4_metal.m#L13000) | +| `DS4_METAL_DISABLE_GATHERED_KV_PAD_FUSION` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables gathered KV pad fusion. | [ds4_metal.m:29706](ds4_metal.m#L29706) | +| `DS4_METAL_DISABLE_GATHERED_KV_STAGE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables gathered KV stage. | [ds4_metal.m:29677](ds4_metal.m#L29677) | +| `DS4_METAL_DISABLE_GLM_DECODE_KV_GROUP4` | value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables | Disables GLM decode KV group4. | [ds4_metal.m:36742](ds4_metal.m#L36742) | +| `DS4_METAL_DISABLE_GLM_QKLOW_SG` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables GLM qklow sg. | [ds4_metal.m:37865](ds4_metal.m#L37865) | +| `DS4_METAL_DISABLE_GLM_STREAMING_EXPERT_EARLY_LOAD` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables GLM streaming expert early load. | [ds4_metal.m:18058](ds4_metal.m#L18058) | +| `DS4_METAL_DISABLE_GLM_STREAMING_EXPERT_SPLIT` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables GLM streaming expert split. | [ds4_metal.m:39796](ds4_metal.m#L39796) | +| `DS4_METAL_DISABLE_GLM_STREAMING_PREFILL_FULL_LAYER` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables GLM streaming prefill full layer. | [ds4.c:42710](ds4.c#L42710) | +| `DS4_METAL_DISABLE_GLM_STREAMING_PREFILL_FULL_LAYER_PREPARE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables GLM streaming prefill full layer prepare. | [ds4.c:42728](ds4.c#L42728) | +| `DS4_METAL_DISABLE_GLM_STREAMING_PREFILL_SELECTED_ASYNC_LOAD` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables GLM streaming prefill selected async load. | [ds4.c:46421](ds4.c#L46421) | +| `DS4_METAL_DISABLE_GLM_STREAMING_SELECTED_ASYNC_LOAD` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables GLM streaming selected async load. | [ds4.c:44329](ds4.c#L44329) | +| `DS4_METAL_DISABLE_HC_FUSION` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Disables HC fusion. | [ds4.c:20406](ds4.c#L20406) | +| `DS4_METAL_DISABLE_HC_NORM_FUSION` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Disables HC norm fusion. | [ds4.c:20450](ds4.c#L20450) | +| `DS4_METAL_DISABLE_HC_PRODUCER_PRE_NORM_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables HC producer pre norm fuse. | [ds4_metal.m:46541](ds4_metal.m#L46541) | +| `DS4_METAL_DISABLE_HC_RMS_SCALE_PROJ` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables HC RMS scale proj. | [ds4_metal.m:24148](ds4_metal.m#L24148) | +| `DS4_METAL_DISABLE_HOT_PIPELINE_STATICS` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables hot pipeline statics. | [ds4_metal.m:2634](ds4_metal.m#L2634) | +| `DS4_METAL_DISABLE_INPLACE_ROPE_PAIR` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables inplace RoPE pair. | [ds4_metal.m:25170](ds4_metal.m#L25170) | +| `DS4_METAL_DISABLE_IQ2_SELECTED_EXPERT_VIEWS` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables IQ2 selected expert views. | [ds4.c:21080](ds4.c#L21080) | +| `DS4_METAL_DISABLE_IQ2_SELECTED_SHARED_OVERLAP` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables IQ2 selected shared overlap. | [ds4.c:20996](ds4.c#L20996) | +| `DS4_METAL_DISABLE_IQ2_STREAM_ADDR_TABLE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables IQ2 stream address table. | [ds4_metal.m:42402](ds4_metal.m#L42402) | +| `DS4_METAL_DISABLE_IQ2_XXS_SSD_PREFILL_MM` | value-aware boolean; default off; true disables and dominates ENABLE; false leaves automatic policy | Rolls grouped IQ2_XXS/Q2_K SSD-prefill MM back to sparse matvec. | [ds4_metal.m:44784](ds4_metal.m#L44784) | +| `DS4_METAL_DISABLE_KV_FUSION` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Disables KV fusion. | [ds4.c:20430](ds4.c#L20430) | +| `DS4_METAL_DISABLE_M1_IQ2_MID_ONLY` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables M1 IQ2 mid only. | [ds4_metal.m:14592](ds4_metal.m#L14592) | +| `DS4_METAL_DISABLE_M3_COMPRESSOR_EXACT_POOL_RATIO4` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables M3 compressor exact pool ratio4. | [ds4_metal.m:26381](ds4_metal.m#L26381) | +| `DS4_METAL_DISABLE_M3_COMPRESSOR_PAIR_STATE_STORE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables M3 compressor pair state store. | [ds4_metal.m:23247](ds4_metal.m#L23247) | +| `DS4_METAL_DISABLE_M3_GATHERED_KV_STAGE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables M3 gathered KV stage. | [ds4_metal.m:29678](ds4_metal.m#L29678) | +| `DS4_METAL_DISABLE_M5_COMPRESSOR_EXACT_POOL_RATIO4` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables M5 compressor exact pool ratio4. | [ds4_metal.m:26384](ds4_metal.m#L26384) | +| `DS4_METAL_DISABLE_M5_COMP_FINALIZE_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables M5 comp finalize fuse. | [ds4.c:23647](ds4.c#L23647) | +| `DS4_METAL_DISABLE_M5_FLASH_ATTN_PACKED32_REDUCE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables M5 flash attn packed32 reduce. | [ds4_metal.m:31519](ds4_metal.m#L31519) | +| `DS4_METAL_DISABLE_M5_HC_NORM_MIX_CLUSTER2` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables M5 HC norm mix cluster2. | [ds4_metal.m:46496](ds4_metal.m#L46496) | +| `DS4_METAL_DISABLE_M5_HC_PRODUCER_PRE_NORM_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables M5 HC producer pre norm fuse. | [ds4_metal.m:46548](ds4_metal.m#L46548) | +| `DS4_METAL_DISABLE_M5_IQ2_PAIR_PACK2` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables M5 IQ2 pair pack2. | [ds4_metal.m:42020](ds4_metal.m#L42020) | +| `DS4_METAL_DISABLE_M5_PACKED_ZERO_MASK` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables M5 packed zero mask. | [ds4_metal.m:31475](ds4_metal.m#L31475) | +| `DS4_METAL_DISABLE_M5_PARALLEL_FULL_FFN` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables M5 parallel full FFN. | [ds4.c:22707](ds4.c#L22707) | +| `DS4_METAL_DISABLE_M5_PERSISTENT_ZERO_ATTN_MASK` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables M5 persistent zero attn mask. | [ds4_metal.m:31473](ds4_metal.m#L31473) | +| `DS4_METAL_DISABLE_M5_Q8_HC_VEC` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables M5 Q8 HC vec. | [ds4_metal.m:47580](ds4_metal.m#L47580) | +| `DS4_METAL_DISABLE_M5_QKV_PAIR_COMPRESSOR_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables M5 QKV pair compressor fuse. | [ds4.c:23043](ds4.c#L23043) | +| `DS4_METAL_DISABLE_M5_QKV_PAIR_QUAD_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables M5 QKV pair quad fuse. | [ds4.c:23039](ds4.c#L23039) | +| `DS4_METAL_DISABLE_M5_ROUTER_PROJECT_SELECT_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables M5 router project select fuse. | [ds4.c:24762](ds4.c#L24762) | +| `DS4_METAL_DISABLE_METAL4` | value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables | Disables metal4. | [ds4_metal.m:2995](ds4_metal.m#L2995) | +| `DS4_METAL_DISABLE_MOE_MM_ID_PAIR_SWIGLU` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables MoE MM ID pair SwiGLU. | [ds4_metal.m:44968](ds4_metal.m#L44968) | +| `DS4_METAL_DISABLE_MOE_MM_ID_USE_RESOURCES` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables MoE MM ID use resources. | [ds4_metal.m:35169](ds4_metal.m#L35169) | +| `DS4_METAL_DISABLE_MXFP4_SELECTED_EXPERT_VIEWS` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables MXFP4 selected expert views. | [ds4.c:21173](ds4.c#L21173) | +| `DS4_METAL_DISABLE_PERSISTENT_ZERO_ATTN_MASK` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables persistent zero attn mask. | [ds4_metal.m:31480](ds4_metal.m#L31480) | +| `DS4_METAL_DISABLE_PRE_M5_ATTN_INV_ROPE_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 attn inv RoPE fuse. | [ds4.c:22638](ds4.c#L22638) | +| `DS4_METAL_DISABLE_PRE_M5_ATTN_OUT_LOW_Q8_STATIC` | presence rollback; unset: exact fixed-shape kernel is automatic on eligible pre-M5 Flash decode; any value including 0 disables | Restores the generic Q8 attention-output low projection kernel. | [ds4_metal.m:27987](ds4_metal.m#L27987) | +| `DS4_METAL_DISABLE_PRE_M5_BATCH_INDEXER_QUERY_PRUNE` | presence rollback; unset: unused zero-prefix indexer queries are pruned before compressed rows exceed top-k; any value including 0 disables | Restores transient indexer query and weight dispatches during eligible pre-M5 prefill. | [ds4.c:30086](ds4.c#L30086) | +| `DS4_METAL_DISABLE_PRE_M5_COMPRESSOR_EXACT_POOL_RATIO4` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 compressor exact pool ratio4. | [ds4_metal.m:26383](ds4_metal.m#L26383) | +| `DS4_METAL_DISABLE_PRE_M5_COMPRESSOR_EXACT_REDUCTION_FUSION` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 compressor exact reduction fusion. | [ds4_metal.m:25907](ds4_metal.m#L25907) | +| `DS4_METAL_DISABLE_PRE_M5_COMPRESSOR_QUAD_STORE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 compressor quad store. | [ds4.c:23535](ds4.c#L23535) | +| `DS4_METAL_DISABLE_PRE_M5_COMPRESSOR_RATIO4_DECODE_PACK_FUSION` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 compressor ratio4 decode pack fusion. | [ds4_metal.m:26214](ds4_metal.m#L26214) | +| `DS4_METAL_DISABLE_PRE_M5_COMP_FINALIZE_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 comp finalize fuse. | [ds4.c:23646](ds4.c#L23646) | +| `DS4_METAL_DISABLE_PRE_M5_DECODE_EARLY_PIPELINE_FAST_LOOKUP` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 decode early pipeline fast lookup. | [ds4.c:28003](ds4.c#L28003) | +| `DS4_METAL_DISABLE_PRE_M5_DECODE_EARLY_SECOND_SPLIT12` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 decode early second split12. | [ds4.c:28087](ds4.c#L28087) | +| `DS4_METAL_DISABLE_PRE_M5_DECODE_EARLY_SPLIT3` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 decode early split3. | [ds4.c:27961](ds4.c#L27961) | +| `DS4_METAL_DISABLE_PRE_M5_DECODE_EARLY_SPLIT5` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 decode early split5. | [ds4.c:27973](ds4.c#L27973) | +| `DS4_METAL_DISABLE_PRE_M5_DECODE_PIPELINE_FAST_LOOKUP` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 decode pipeline fast lookup. | [ds4.c:28015](ds4.c#L28015) | +| `DS4_METAL_DISABLE_PRE_M5_DECODE_PORTS` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 decode ports. | [ds4.c:22613](ds4.c#L22613) | +| `DS4_METAL_DISABLE_PRE_M5_DECODE_RAW_ZERO_ATTN_MASK` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 decode raw zero attn mask. | [ds4_metal.m:29920](ds4_metal.m#L29920) | +| `DS4_METAL_DISABLE_PRE_M5_DECODE_SECOND_SPLIT16` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 decode second split16. | [ds4.c:28095](ds4.c#L28095) | +| `DS4_METAL_DISABLE_PRE_M5_FLASH_ATTN_BATCHED_MEMO` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 flash attn batched memo. | [ds4_metal.m:3752](ds4_metal.m#L3752) | +| `DS4_METAL_DISABLE_PRE_M5_FLASH_ATTN_PACKED32_REDUCE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 flash attn packed32 reduce. | [ds4_metal.m:31460](ds4_metal.m#L31460) | +| `DS4_METAL_DISABLE_PRE_M5_FLASH_ATTN_PAD_BLK_MEMO` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 flash attn pad blk memo. | [ds4_metal.m:3610](ds4_metal.m#L3610) | +| `DS4_METAL_DISABLE_PRE_M5_HC_NORM_MIX_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 HC norm mix fuse. | [ds4.c:22867](ds4.c#L22867) | +| `DS4_METAL_DISABLE_PRE_M5_HC_PRODUCER_PRE_NORM_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 HC producer pre norm fuse. | [ds4_metal.m:46546](ds4_metal.m#L46546) | +| `DS4_METAL_DISABLE_PRE_M5_HEAD_RMS_ROPE_PIPELINE_STATIC` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 head RMS RoPE pipeline static. | [ds4_metal.m:10026](ds4_metal.m#L10026) | +| `DS4_METAL_DISABLE_PRE_M5_KV_ROPE_FP8_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 KV RoPE fp8 fuse. | [ds4.c:23455](ds4.c#L23455) | +| `DS4_METAL_DISABLE_PRE_M5_MXFP4_MM_ID_PAIR_HALF_SCALE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 MXFP4 MM ID pair half scale. | [ds4_metal.m:45129](ds4_metal.m#L45129) | +| `DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_DECODE_FIXED_ROUTE_PAIR` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 MXFP4 MoE decode fixed route pair. | [ds4_metal.m:41948](ds4_metal.m#L41948) | +| `DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_DECODE_FIXED_ROUTE_SUM6` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 MXFP4 MoE decode fixed route sum6. | [ds4_metal.m:41961](ds4_metal.m#L41961) | +| `DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_DECODE_NSG1` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 MXFP4 MoE decode nsg1. | [ds4_metal.m:41719](ds4_metal.m#L41719) | +| `DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_DECODE_STATIC_TRIP` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 MXFP4 MoE decode static trip. | [ds4_metal.m:41987](ds4_metal.m#L41987) | +| `DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_DECODE_SUM6_FULL_ROWS` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 MXFP4 MoE decode sum6 full rows. | [ds4_metal.m:41974](ds4_metal.m#L41974) | +| `DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_DECODE_TG_MULTIPLE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 MXFP4 MoE decode tg multiple. | [ds4_metal.m:41936](ds4_metal.m#L41936) | +| `DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_MM_ID_DOWN_HALF_LUT` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 MXFP4 MoE MM ID down half lut. | [ds4_metal.m:45047](ds4_metal.m#L45047) | +| `DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_MM_ID_DOWN_TAIL_SIMDGROUP_CULL` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 MXFP4 MoE MM ID down tail simdgroup cull. | [ds4_metal.m:45030](ds4_metal.m#L45030) | +| `DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_MM_ID_MAP_SCATTER` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 MXFP4 MoE MM ID map scatter. | [ds4_metal.m:44997](ds4_metal.m#L44997) | +| `DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_MM_ID_PAIR_SWIGLU_COMPACT_TILE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 MXFP4 MoE MM ID pair SwiGLU compact tile. | [ds4_metal.m:44981](ds4_metal.m#L44981) | +| `DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_MM_ID_PAIR_TAIL_SIMDGROUP_CULL` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 MXFP4 MoE MM ID pair tail simdgroup cull. | [ds4_metal.m:45018](ds4_metal.m#L45018) | +| `DS4_METAL_DISABLE_PRE_M5_PARALLEL_FULL_FFN` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 parallel full FFN. | [ds4.c:22706](ds4.c#L22706) | +| `DS4_METAL_DISABLE_PRE_M5_Q2_DECODE_SPLIT2_32` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 q2 decode split2 32. | [ds4.c:27876](ds4.c#L27876) | +| `DS4_METAL_DISABLE_PRE_M5_QKV_NORM_KV_STORE_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 QKV norm KV store fuse. | [ds4.c:23314](ds4.c#L23314) | +| `DS4_METAL_DISABLE_PRE_M5_QKV_PAIR_COMPRESSOR_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 QKV pair compressor fuse. | [ds4.c:23042](ds4.c#L23042) | +| `DS4_METAL_DISABLE_PRE_M5_QKV_PAIR_QUAD_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 QKV pair quad fuse. | [ds4.c:23038](ds4.c#L23038) | +| `DS4_METAL_DISABLE_PRE_M5_ROUTER_SHARED_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 router shared fuse. | [ds4.c:24755](ds4.c#L24755) | +| `DS4_METAL_DISABLE_PRE_M5_ROUTER_SIMD_FINALIZE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 router simd finalize. | [ds4_metal.m:35827](ds4_metal.m#L35827) | +| `DS4_METAL_DISABLE_PRE_M5_ROUTER_SIMD_WEIGHTS_FUSION` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 router simd weights fusion. | [ds4_metal.m:35836](ds4_metal.m#L35836) | +| `DS4_METAL_DISABLE_PRE_M5_ROUTER_TRANSFORM_FINALIZE_FUSION` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 router transform finalize fusion. | [ds4_metal.m:35840](ds4_metal.m#L35840) | +| `DS4_METAL_DISABLE_PRO_Q4_EXPERT_ADDRESS_AUTO` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pro Q4 expert address auto. | [ds4_metal.m:19710](ds4_metal.m#L19710) | +| `DS4_METAL_DISABLE_PRO_Q4_EXPERT_TABLE_AUTO` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pro Q4 expert table auto. | [ds4.c:21040](ds4.c#L21040) | +| `DS4_METAL_DISABLE_PRO_Q4_EXPERT_TABLE_PRELOAD` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pro Q4 expert table preload. | [ds4.c:58780](ds4.c#L58780) | +| `DS4_METAL_DISABLE_Q4_ATTN_OUT_HC_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 attn out HC fuse. | [ds4_metal.m:47624](ds4_metal.m#L47624) | +| `DS4_METAL_DISABLE_Q4_ATTN_OUT_TINY_BATCH` | value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables | Disables Q4 attn out tiny batch. | [ds4_metal.m:28396](ds4_metal.m#L28396) | +| `DS4_METAL_DISABLE_Q4_BATCH_EXPERT_TABLE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 batch expert table. | [ds4_metal.m:44883](ds4_metal.m#L44883) | +| `DS4_METAL_DISABLE_Q4_DENSE_PAIR` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 dense pair. | [ds4_metal.m:21990](ds4_metal.m#L21990) | +| `DS4_METAL_DISABLE_Q4_EXACT_BOUNDARY` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 exact boundary. | [ds4_metal.m:42239](ds4_metal.m#L42239) | +| `DS4_METAL_DISABLE_Q4_EXACT_TENSOR_ID` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 exact tensor ID. | [ds4_metal.m:42219](ds4_metal.m#L42219) | +| `DS4_METAL_DISABLE_Q4_EXPERT_ADDRESS_TABLE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 expert address table. | [ds4_metal.m:19711](ds4_metal.m#L19711) | +| `DS4_METAL_DISABLE_Q4_EXPERT_TABLE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 expert table. | [ds4.c:21041](ds4.c#L21041) | +| `DS4_METAL_DISABLE_Q4_GATHER_SLOTS` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 gather slots. | [ds4_metal.m:42321](ds4_metal.m#L42321) | +| `DS4_METAL_DISABLE_Q4_GROUP24_EXPERT_TABLE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 group24 expert table. | [ds4_metal.m:42201](ds4_metal.m#L42201) | +| `DS4_METAL_DISABLE_Q4_GROUP6_EXPERT_TABLE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 group6 expert table. | [ds4_metal.m:42167](ds4_metal.m#L42167) | +| `DS4_METAL_DISABLE_Q4_GROUP8_EXPERT_TABLE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 group8 expert table. | [ds4_metal.m:42184](ds4_metal.m#L42184) | +| `DS4_METAL_DISABLE_Q4_GROUPED_BOUNDARY` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 grouped boundary. | [ds4_metal.m:42149](ds4_metal.m#L42149) | +| `DS4_METAL_DISABLE_Q4_GROUPED_EXPERTS` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 grouped experts. | [ds4_metal.m:42130](ds4_metal.m#L42130) | +| `DS4_METAL_DISABLE_Q4_MV_CLASSIC` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 MV classic. | [ds4_metal.m:21571](ds4_metal.m#L21571) | +| `DS4_METAL_DISABLE_Q4_QKV_COMPRESSOR_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 QKV compressor fuse. | [ds4_metal.m:22083](ds4_metal.m#L22083) | +| `DS4_METAL_DISABLE_Q4_SELECTED_EXPERT_VIEWS` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 selected expert views. | [ds4.c:21136](ds4.c#L21136) | +| `DS4_METAL_DISABLE_Q4_SSD_PREFILL_ATTN_OUT_EXACTN` | value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables | Disables Q4 SSD prefill attn out exactn. | [ds4_metal.m:28095](ds4_metal.m#L28095) | +| `DS4_METAL_DISABLE_Q4_SSD_SESSION_UNION` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Disables Q4 SSD session union. | [ds4.c:65160](ds4.c#L65160) | +| `DS4_METAL_DISABLE_Q4_STREAM_OVERLAP` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Disables Q4 stream overlap. | [ds4.c:65082](ds4.c#L65082) | +| `DS4_METAL_DISABLE_Q4_TABLE_BOUNDARY` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 table boundary. | [ds4_metal.m:42318](ds4_metal.m#L42318) | +| `DS4_METAL_DISABLE_Q8_DECODE_EXACT_VIEWS` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q8 decode exact views. | [ds4_metal.m:12851](ds4_metal.m#L12851) | +| `DS4_METAL_DISABLE_QKV_NORM_FUSION` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Disables QKV norm fusion. | [ds4.c:20435](ds4.c#L20435) | +| `DS4_METAL_DISABLE_QKV_PAIR_PROJ` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Disables QKV pair proj. | [ds4.c:20440](ds4.c#L20440) | +| `DS4_METAL_DISABLE_QUEUE_RESIDENCY_SET` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables queue residency set. | [ds4_metal.m:2124](ds4_metal.m#L2124) | +| `DS4_METAL_DISABLE_ROUTED_PAIR_SWIGLU_FUSION` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables routed pair SwiGLU fusion. | [ds4.c:18529](ds4.c#L18529) | +| `DS4_METAL_DISABLE_ROUTER_SELECT_FUSION` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables router select fusion. | [ds4_metal.m:35816](ds4_metal.m#L35816) | +| `DS4_METAL_DISABLE_ROUTER_WEIGHTS_BATCH_FUSION` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables router weights batch fusion. | [ds4_metal.m:36068](ds4_metal.m#L36068) | +| `DS4_METAL_DISABLE_SHARED_DOWN_HC_FUSION` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Disables shared down HC fusion. | [ds4.c:20472](ds4.c#L20472) | +| `DS4_METAL_DISABLE_SHARED_GATE_UP_SWIGLU_FUSION` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables shared gate up SwiGLU fusion. | [ds4.c:17567](ds4.c#L17567) | +| `DS4_METAL_DISABLE_SHARED_KV_PAD` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables shared KV pad. | [ds4_metal.m:31510](ds4_metal.m#L31510) | +| `DS4_METAL_DISABLE_SHARED_ROPE_COEFF` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables shared RoPE coeff. | [ds4_metal.m:6341](ds4_metal.m#L6341) | +| `DS4_METAL_DISABLE_STREAMING_COLD_DECODE_PREFILL` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming cold decode prefill. | [ds4.c:32007](ds4.c#L32007) | +| `DS4_METAL_DISABLE_STREAMING_COMPACT_ADDR` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming compact address. | [ds4_metal.m:14598](ds4_metal.m#L14598) | +| `DS4_METAL_DISABLE_STREAMING_DECODE_PREFILL` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming decode prefill. | [ds4.c:31956](ds4.c#L31956) | +| `DS4_METAL_DISABLE_STREAMING_EXPERT_ADDR_TABLE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming expert address table. | [ds4.c:18525](ds4.c#L18525) | +| `DS4_METAL_DISABLE_STREAMING_EXPERT_COMBINED_BUFFER` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming expert combined buffer. | [ds4_metal.m:14023](ds4_metal.m#L14023) | +| `DS4_METAL_DISABLE_STREAMING_EXPERT_EARLY_LOAD` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming expert early load. | [ds4_metal.m:17232](ds4_metal.m#L17232) | +| `DS4_METAL_DISABLE_STREAMING_EXPERT_EVICT_DONTNEED` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming expert evict dontneed. | [ds4_metal.m:14395](ds4_metal.m#L14395) | +| `DS4_METAL_DISABLE_STREAMING_EXPERT_HIT_VALIDATOR` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming expert hit validator. | [ds4_metal.m:14634](ds4_metal.m#L14634) | +| `DS4_METAL_DISABLE_STREAMING_EXPERT_HOTLIST` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming expert hotlist. | [ds4.c:21290](ds4.c#L21290) | +| `DS4_METAL_DISABLE_STREAMING_EXPERT_LIVE_INDEX` | value-aware boolean; default off; true disables and dominates ENABLE | Disables dense live-entry index and uses authoritative cache matrix. | [ds4_metal.m:15443](ds4_metal.m#L15443) | +| `DS4_METAL_DISABLE_STREAMING_EXPERT_MASKED_ADDR` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming expert masked address. | [ds4_metal.m:14628](ds4_metal.m#L14628) | +| `DS4_METAL_DISABLE_STREAMING_EXPERT_READAHEAD` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming expert readahead. | [ds4_metal.m:13196](ds4_metal.m#L13196) | +| `DS4_METAL_DISABLE_STREAMING_EXPERT_SLABS` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming expert slabs. | [ds4_metal.m:14028](ds4_metal.m#L14028) | +| `DS4_METAL_DISABLE_STREAMING_EXPERT_TIMING_SUMMARY` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming expert timing summary. | [ds4_metal.m:13062](ds4_metal.m#L13062) | +| `DS4_METAL_DISABLE_STREAMING_FULL_EXPERT_ADDR_TABLE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming full expert address table. | [ds4_metal.m:14716](ds4_metal.m#L14716) | +| `DS4_METAL_DISABLE_STREAMING_IQ2_CPU_ROUTER` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming IQ2 CPU router. | [ds4.c:20939](ds4.c#L20939) | +| `DS4_METAL_DISABLE_STREAMING_LAYER_BATCH` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming layer batch. | [ds4.c:18239](ds4.c#L18239) | +| `DS4_METAL_DISABLE_STREAMING_MADVISE_WILLNEED` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming madvise willneed. | [ds4.c:18212](ds4.c#L18212) | +| `DS4_METAL_DISABLE_STREAMING_PREFILL_BATCH_SELECTED_ADDR` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming prefill batch selected address. | [ds4.c:18523](ds4.c#L18523) | +| `DS4_METAL_DISABLE_STREAMING_PREFILL_LAYER_MADVISE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming prefill layer madvise. | [ds4.c:18466](ds4.c#L18466) | +| `DS4_METAL_DISABLE_STREAMING_PREFILL_LAYER_PAGEIN` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming prefill layer pagein. | [ds4.c:18434](ds4.c#L18434) | +| `DS4_METAL_DISABLE_STREAMING_PREFILL_LAYER_PAGEIN_OVERLAP` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming prefill layer pagein overlap. | [ds4.c:19321](ds4.c#L19321) | +| `DS4_METAL_DISABLE_STREAMING_PREFILL_LAYER_PREAD` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming prefill layer pread. | [ds4.c:18454](ds4.c#L18454) | +| `DS4_METAL_DISABLE_STREAMING_PREFILL_LAYER_PREPARE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming prefill layer prepare. | [ds4.c:18446](ds4.c#L18446) | +| `DS4_METAL_DISABLE_STREAMING_PREFILL_LAYER_PREPARE_OVERLAP` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming prefill layer prepare overlap. | [ds4.c:19319](ds4.c#L19319) | +| `DS4_METAL_DISABLE_STREAMING_PREFILL_LAYER_READAHEAD` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming prefill layer readahead. | [ds4.c:18444](ds4.c#L18444) | +| `DS4_METAL_DISABLE_STREAMING_PREFILL_SELECTED_ASYNC_LOAD` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming prefill selected async load. | [ds4.c:46424](ds4.c#L46424) | +| `DS4_METAL_DISABLE_STREAMING_PREFILL_SELECTED_MADVISE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming prefill selected madvise. | [ds4.c:18424](ds4.c#L18424) | +| `DS4_METAL_DISABLE_STREAMING_PREFILL_SELECTED_PAGEIN` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming prefill selected pagein. | [ds4.c:18414](ds4.c#L18414) | +| `DS4_METAL_DISABLE_STREAMING_PREFILL_SELECTED_PROFILE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming prefill selected profile. | [ds4.c:18908](ds4.c#L18908) | +| `DS4_METAL_DISABLE_STREAMING_PREFILL_SELECTED_READAHEAD` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming prefill selected readahead. | [ds4.c:19960](ds4.c#L19960) | +| `DS4_METAL_DISABLE_STREAMING_PREFILL_SELECTED_READAHEAD_SHARED` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming prefill selected readahead shared. | [ds4.c:19970](ds4.c#L19970) | +| `DS4_METAL_DISABLE_STREAMING_READAHEAD` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming readahead. | [ds4.c:18205](ds4.c#L18205) | +| `DS4_METAL_DISABLE_STREAMING_SELECTED_ASYNC_EARLY_COMMIT` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming selected async early commit. | [ds4.c:21019](ds4.c#L21019) | +| `DS4_METAL_DISABLE_STREAMING_SELECTED_ASYNC_LOAD` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming selected async load. | [ds4.c:21003](ds4.c#L21003) | +| `DS4_METAL_DISABLE_STREAMING_SELECTED_READAHEAD_SHARED_DELAY` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming selected readahead shared delay. | [ds4.c:21720](ds4.c#L21720) | +| `DS4_METAL_DISABLE_STREAMING_SELECTED_SHARED_OVERLAP` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming selected shared overlap. | [ds4.c:20995](ds4.c#L20995) | +| `DS4_METAL_DISABLE_STREAMING_STATIC_DECODE_MAP` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming static decode map. | [ds4.c:18217](ds4.c#L18217) | +| `DS4_METAL_DISABLE_STREAMING_STATIC_MAP_STATE_CACHE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming static map state cache. | [ds4.c:18230](ds4.c#L18230) | +| `DS4_METAL_DISABLE_SUPPORT_Q8_DECODE_EXACT_VIEWS` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables support Q8 decode exact views. | [ds4_metal.m:12858](ds4_metal.m#L12858) | +| `DS4_METAL_DISABLE_TINY_PAIR_SWIGLU_FUSION` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables tiny pair SwiGLU fusion. | [ds4_metal.m:44920](ds4_metal.m#L44920) | +| `DS4_METAL_DISABLE_TOKEN_EMBED_EXACT_VIEW` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables token embed exact view. | [ds4_metal.m:11907](ds4_metal.m#L11907) | +| `DS4_METAL_DISABLE_ZERO_PREFIX_PREFILL_MASK_CACHE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables zero prefix prefill mask cache. | [ds4_metal.m:2404](ds4_metal.m#L2404) | +| `DS4_METAL_DSPARK_ACCEPTANCE_ONLY_VERIFY` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Verifies only the draft rows still needed for acceptance after the base target logit. | [ds4.c:52499](ds4.c#L52499) | +| `DS4_METAL_DSPARK_DEVICE_PROPOSER` | boolean true values enable; false values/unset disable; NO_DEVICE_PROPOSER presence dominates | Keeps DSpark Q8 confidence/Markov proposal work on Metal and reads one compact result. | [ds4.c:34884](ds4.c#L34884) | +| `DS4_METAL_DSPARK_EXACT2` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Enables the resident single-GPU Metal exact-2 verifier. | [ds4.c:52290](ds4.c#L52290) | +| `DS4_METAL_DSPARK_EXACTN` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Enables the single-GPU Metal exact-N verifier. | [ds4.c:52311](ds4.c#L52311) | +| `DS4_METAL_DSPARK_EXACTN_BATCH_HEAD` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Batches the output head across exact-N verifier rows. | [ds4.c:37726](ds4.c#L37726) | +| `DS4_METAL_DSPARK_EXACTN_UNION` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Loads the union of experts for exact-N verifier rows once. | [ds4.c:52332](ds4.c#L52332) | +| `DS4_METAL_DSPARK_EXACT_ROWS_ASYNC_TAILS` | presence control; unset: off/default; any value including 0 enables | Runs exact-row routed tails asynchronously after union routing. | [ds4.c:37932](ds4.c#L37932) | +| `DS4_METAL_DSPARK_EXACT_ROWS_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for DSpark exact rows. | [ds4.c:37930](ds4.c#L37930) | +| `DS4_METAL_DSPARK_HEADLESS_REPLAY` | unset/empty: enabled; exact 0 disables; every other nonempty value enables | Skips output heads for accepted intermediate DSpark replay tokens. | [ds4.c:52479](ds4.c#L52479) | +| `DS4_METAL_DSPARK_NO_DEVICE_PROPOSER` | presence rollback; unset: automatic/default path; any value including 0 disables | Dominant presence-based rollback for the Metal DSpark device proposer. | [ds4.c:34886](ds4.c#L34886) | +| `DS4_METAL_DSPARK_PIN_MAIN_PROJ` | nonempty value other than exact 0 enables; unset/empty/0 disables | mlock-pins only DSpark stage-0 main_norm/main_proj in Metal SSD streaming. | [ds4.c:39443](ds4.c#L39443) | +| `DS4_METAL_DSPARK_PROPOSER_BLOCK_MAX` | uint32; unset: automatic cache/verifier cap; 0 or invalid: native width; positive: clamped to DSpark/native maximum | Caps rows proposed by single-device Metal DSpark. | [ds4.c:52430](ds4.c#L52430) | +| `DS4_METAL_DSPARK_SAFE_EXPERT_COUNT` | exact 1 enables; unset or any other value disables | Caps an explicit expert-count cache request to the safe Metal working-set budget for DSpark SSD streaming. | [ds4.c:4880](ds4.c#L4880) | +| `DS4_METAL_DSV4_HC_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the DeepSeek hidden-context Metal kernel source file loaded at runtime. | [ds4_metal.m:4938](ds4_metal.m#L4938) | +| `DS4_METAL_DSV4_KV_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the DeepSeek KV Metal kernel source file loaded at runtime. | [ds4_metal.m:4940](ds4_metal.m#L4940) | +| `DS4_METAL_DSV4_MISC_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the DeepSeek miscellaneous Metal kernel source file loaded at runtime. | [ds4_metal.m:4942](ds4_metal.m#L4942) | +| `DS4_METAL_DSV4_ROPE_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the DeepSeek RoPE Metal kernel source file loaded at runtime. | [ds4_metal.m:4941](ds4_metal.m#L4941) | +| `DS4_METAL_DUMP_PREFILL_LOGITS` | file path; unset/empty: no dump | Writes final GPU prefill logits as f32 binary. | [ds4.c:51347](ds4.c#L51347) | +| `DS4_METAL_ENABLE_BATCH_HC_NORM_FUSION` | legacy value-aware alias; default enabled; exact 0 disables; unset/empty/every other value enables unless DISABLE is active | Legacy control for the now-default batched HC norm fusion. | [ds4.c:20462](ds4.c#L20462) | +| `DS4_METAL_ENABLE_COMPRESSOR_EXACT_POOL_RATIO4` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables compressor exact pool ratio4. | [ds4_metal.m:26385](ds4_metal.m#L26385) | +| `DS4_METAL_ENABLE_COMPRESSOR_PAIR_STATE_STORE` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables compressor pair state store. | [ds4_metal.m:23243](ds4_metal.m#L23243) | +| `DS4_METAL_ENABLE_COMPRESSOR_QUAD_STORE` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables compressor quad store. | [ds4.c:23527](ds4.c#L23527) | +| `DS4_METAL_ENABLE_DSPARK_CAPTURE_FUSED_LAST` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Enables DSpark capture fused last. | [ds4.c:28359](ds4.c#L28359) | +| `DS4_METAL_ENABLE_GATHERED_KV_STAGE` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables gathered KV stage. | [ds4_metal.m:29675](ds4_metal.m#L29675) | +| `DS4_METAL_ENABLE_GLM_STREAMING_SELECTED_ASYNC_LOAD` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables GLM streaming selected async load. | [ds4.c:44335](ds4.c#L44335) | +| `DS4_METAL_ENABLE_HC_NORM_MIX_FUSE` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables HC norm mix fuse. | [ds4.c:22870](ds4.c#L22870) | +| `DS4_METAL_ENABLE_HC_PRODUCER_PRE_NORM_FUSE` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables HC producer pre norm fuse. | [ds4_metal.m:46552](ds4_metal.m#L46552) | +| `DS4_METAL_ENABLE_IQ2_SELECTED_ASYNC_EARLY_COMMIT` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables IQ2 selected async early commit. | [ds4.c:21018](ds4.c#L21018) | +| `DS4_METAL_ENABLE_IQ2_XXS_SSD_PREFILL_MM` | value-aware boolean; default automatic/on for eligible shape; explicit 0 turns request off unless REQUIRE=1 | Overrides automatic IQ2_XXS/Q2_K grouped address-MM selection for SSD prefill. | [ds4_metal.m:44780](ds4_metal.m#L44780) | +| `DS4_METAL_ENABLE_PRO_Q4_EXPERT_ADDRESS_AUTO` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables pro Q4 expert address auto. | [ds4.c:20982](ds4.c#L20982) | +| `DS4_METAL_ENABLE_PRO_Q4_EXPERT_TABLE_AUTO` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables pro Q4 expert table auto. | [ds4.c:20981](ds4.c#L20981) | +| `DS4_METAL_ENABLE_PRO_Q4_SELECTED_EXPERT_VIEWS` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables pro Q4 selected expert views. | [ds4.c:20978](ds4.c#L20978) | +| `DS4_METAL_ENABLE_Q4_ATTN_OUT_TINY_BATCH` | value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables | Enables Q4 attn out tiny batch. | [ds4_metal.m:28405](ds4_metal.m#L28405) | +| `DS4_METAL_ENABLE_Q4_BATCH_EXPERT_TABLE` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables Q4 batch expert table. | [ds4_metal.m:44869](ds4_metal.m#L44869) | +| `DS4_METAL_ENABLE_Q4_EXACT_TENSOR_ID` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables Q4 exact tensor ID. | [ds4_metal.m:42218](ds4_metal.m#L42218) | +| `DS4_METAL_ENABLE_Q4_EXPERT_ADDRESS_TABLE` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables Q4 expert address table. | [ds4.c:20980](ds4.c#L20980) | +| `DS4_METAL_ENABLE_Q4_EXPERT_TABLE` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables Q4 expert table. | [ds4.c:20979](ds4.c#L20979) | +| `DS4_METAL_ENABLE_Q4_GATHER_SLOTS` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables Q4 gather slots. | [ds4_metal.m:42320](ds4_metal.m#L42320) | +| `DS4_METAL_ENABLE_Q4_GROUP24_EXPERT_TABLE` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables Q4 group24 expert table. | [ds4_metal.m:42200](ds4_metal.m#L42200) | +| `DS4_METAL_ENABLE_Q4_GROUP6_EXPERT_TABLE` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables Q4 group6 expert table. | [ds4_metal.m:42166](ds4_metal.m#L42166) | +| `DS4_METAL_ENABLE_Q4_GROUP8_EXPERT_TABLE` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables Q4 group8 expert table. | [ds4_metal.m:42183](ds4_metal.m#L42183) | +| `DS4_METAL_ENABLE_Q4_GROUPED_EXPERTS` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables Q4 grouped experts. | [ds4_metal.m:42129](ds4_metal.m#L42129) | +| `DS4_METAL_ENABLE_Q4_QKV_COMPRESSOR_FUSE` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables Q4 QKV compressor fuse. | [ds4.c:23140](ds4.c#L23140) | +| `DS4_METAL_ENABLE_Q4_SELECTED_EXPERT_VIEWS` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables Q4 selected expert views. | [ds4.c:20977](ds4.c#L20977) | +| `DS4_METAL_ENABLE_Q4_SSD_PREFILL_ATTN_OUT_EXACTN` | value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables | Enables Q4 SSD prefill attn out exactn. | [ds4_metal.m:28097](ds4_metal.m#L28097) | +| `DS4_METAL_ENABLE_Q4_SSD_SESSION_UNION` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Enables Q4 SSD session union. | [ds4.c:65171](ds4.c#L65171) | +| `DS4_METAL_ENABLE_Q4_STREAM_OVERLAP` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Enables Q4 stream overlap. | [ds4.c:65080](ds4.c#L65080) | +| `DS4_METAL_ENABLE_Q8_DECODE_EXACT_VIEWS` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables Q8 decode exact views. | [ds4_metal.m:12865](ds4_metal.m#L12865) | +| `DS4_METAL_ENABLE_Q8_QKV_COMPRESSOR_FUSE` | nonempty boolean; unset/empty/exact 0: no streamed/union opt-in; other values enable; eligible resident full-decode remains automatic | Extends the automatic resident Q8 QKV/compressor compound fusion to SSD streaming or exact-N union scope. | [ds4.c:23027](ds4.c#L23027) | +| `DS4_METAL_ENABLE_STREAMING_COMPACT_ADDR` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming compact address. | [ds4_metal.m:14597](ds4_metal.m#L14597) | +| `DS4_METAL_ENABLE_STREAMING_EXPERT_ADDR_TABLE` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming expert address table. | [ds4_metal.m:14604](ds4_metal.m#L14604) | +| `DS4_METAL_ENABLE_STREAMING_EXPERT_EVICT_DONTNEED` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming expert evict dontneed. | [ds4_metal.m:14394](ds4_metal.m#L14394) | +| `DS4_METAL_ENABLE_STREAMING_EXPERT_HIT_VALIDATOR` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming expert hit validator. | [ds4_metal.m:14605](ds4_metal.m#L14605) | +| `DS4_METAL_ENABLE_STREAMING_EXPERT_LIVE_INDEX` | value-aware boolean; default automatic/on for validated IQ2 cache shape; explicit 0 disables | Overrides automatic dense live-entry index selection. | [ds4_metal.m:15442](ds4_metal.m#L15442) | +| `DS4_METAL_ENABLE_STREAMING_EXPERT_MASKED_ADDR` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming expert masked address. | [ds4_metal.m:14606](ds4_metal.m#L14606) | +| `DS4_METAL_ENABLE_STREAMING_FULL_EXPERT_ADDR_TABLE` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming full expert address table. | [ds4_metal.m:14715](ds4_metal.m#L14715) | +| `DS4_METAL_ENABLE_STREAMING_IQ2_CPU_ROUTER` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming IQ2 CPU router. | [ds4.c:20938](ds4.c#L20938) | +| `DS4_METAL_ENABLE_STREAMING_MADVISE_WILLNEED` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming madvise willneed. | [ds4.c:18210](ds4.c#L18210) | +| `DS4_METAL_ENABLE_STREAMING_PREFILL_BATCH_SELECTED_ADDR` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming prefill batch selected address. | [ds4_metal.m:14609](ds4_metal.m#L14609) | +| `DS4_METAL_ENABLE_STREAMING_PREFILL_CACHE_SEED` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming prefill cache seed. | [ds4.c:21259](ds4.c#L21259) | +| `DS4_METAL_ENABLE_STREAMING_PREFILL_EXPERT_READAHEAD` | value-aware boolean; for batches <32 readahead is automatic; for batches >=32 unset/false disables and true enables; global READHEAD rollback and F_NOCACHE still dominate | Restores F_RDADVISE immediately before parallel pread for large SSD-prefill batches. | [ds4_metal.m:13212](ds4_metal.m#L13212) | +| `DS4_METAL_ENABLE_STREAMING_PREFILL_LAYER_PAGEIN` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming prefill layer pagein. | [ds4.c:18432](ds4.c#L18432) | +| `DS4_METAL_ENABLE_STREAMING_PREFILL_LAYER_READAHEAD` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming prefill layer readahead. | [ds4.c:18442](ds4.c#L18442) | +| `DS4_METAL_ENABLE_STREAMING_PREFILL_SELECTED_MADVISE` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming prefill selected madvise. | [ds4.c:18422](ds4.c#L18422) | +| `DS4_METAL_ENABLE_STREAMING_PREFILL_SELECTED_PAGEIN` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming prefill selected pagein. | [ds4.c:18412](ds4.c#L18412) | +| `DS4_METAL_ENABLE_STREAMING_PREFILL_SELECTED_READAHEAD` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming prefill selected readahead. | [ds4.c:19956](ds4.c#L19956) | +| `DS4_METAL_ENABLE_STREAMING_PREFILL_SELECTED_READAHEAD_SHARED` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming prefill selected readahead shared. | [ds4.c:19958](ds4.c#L19958) | +| `DS4_METAL_ENABLE_STREAMING_READAHEAD` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming readahead. | [ds4.c:18203](ds4.c#L18203) | +| `DS4_METAL_ENABLE_STREAMING_SELECTED_READAHEAD_SHARED_DELAY` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming selected readahead shared delay. | [ds4.c:21719](ds4.c#L21719) | +| `DS4_METAL_ENABLE_STREAMING_STATIC_DECODE_MAP` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming static decode map. | [ds4.c:18222](ds4.c#L18222) | +| `DS4_METAL_ENABLE_TOKEN_EMBED_EXACT_VIEW` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables token embed exact view. | [ds4_metal.m:12219](ds4_metal.m#L12219) | +| `DS4_METAL_EXACT_VIEW_CACHE_GIB` | unsigned GiB; default 64; 0 disables size-triggered eviction; MIB overrides it | Sets the cached exact-model-view eviction threshold. | [ds4_metal.m:1333](ds4_metal.m#L1333) | +| `DS4_METAL_EXACT_VIEW_CACHE_MIB` | unsigned MiB; unset: inherit GIB/default; 0 disables size-triggered eviction; overrides GIB | Sets the cached exact-model-view eviction threshold with MiB precision. | [ds4_metal.m:1342](ds4_metal.m#L1342) | +| `DS4_METAL_EXACT_VIEW_CACHE_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for exact view cache. | [ds4_metal.m:1377](ds4_metal.m#L1377) | +| `DS4_METAL_FLASH_ATTN_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the FlashAttention Metal kernel source file loaded at runtime. | [ds4_metal.m:4935](ds4_metal.m#L4935) | +| `DS4_METAL_FLASH_ATTN_STAGE_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for flash attn stage. | [ds4.c:65152](ds4.c#L65152) | +| `DS4_METAL_FLASH_ATTN_STAGE_PROFILE_FILTER` | substring; unset/empty: all profiled modes/stages | Filters FlashAttention stage-profile output by mode or stage substring. | [ds4_metal.m:11178](ds4_metal.m#L11178) | +| `DS4_METAL_GET_ROWS_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the get-rows Metal kernel source file loaded at runtime. | [ds4_metal.m:4946](ds4_metal.m#L4946) | +| `DS4_METAL_GLM_DISABLE_STREAMING_EXPERT_CACHE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming expert cache for GLM. | [ds4_metal.m:39667](ds4_metal.m#L39667) | +| `DS4_METAL_GLM_DISABLE_STREAMING_GROUPED_ADDR_PREFILL` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming grouped address prefill for GLM. | [ds4_metal.m:41081](ds4_metal.m#L41081) | +| `DS4_METAL_GLM_DISABLE_STREAMING_SEED_BEFORE_PREFILL` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming seed before prefill for GLM. | [ds4.c:51074](ds4.c#L51074) | +| `DS4_METAL_GLM_DISABLE_STREAMING_TOKEN_PREFILL` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming token prefill for GLM. | [ds4.c:49704](ds4.c#L49704) | +| `DS4_METAL_GLM_MOE_ONE_STAGE_PROFILE` | unset: off; 1/true/yes/on/all enables all layers; accepts layer lists/ranges; 0/false/no/off disables | Prints timing/profile diagnostics for GLM MoE one stage. | [ds4_metal.m:39964](ds4_metal.m#L39964) | +| `DS4_METAL_GLM_MOE_ONE_STAGE_PROFILE_LAYER` | layer index/list/ranges or all; unset: all layers selected by profiler | Restricts GLM one-stage MoE profiling to selected layers. | [ds4_metal.m:39965](ds4_metal.m#L39965) | +| `DS4_METAL_GLM_MOE_STAGE_PROFILE_FILTER` | substring; unset/empty: all profiled stages | Filters GLM MoE stage-profile output. | [ds4_metal.m:39968](ds4_metal.m#L39968) | +| `DS4_METAL_GLM_QKLOW_DEBUG` | presence diagnostic; unset: off; any value including 0 enables | Enables debug diagnostics for GLM qklow. | [ds4_metal.m:37868](ds4_metal.m#L37868) | +| `DS4_METAL_GLM_STREAMING_ASYNC_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for GLM streaming async. | [ds4.c:44362](ds4.c#L44362) | +| `DS4_METAL_GLM_STREAMING_DECODE_FULL_LAYER_MAP` | presence control; unset: off/default; any value including 0 enables | Maps complete GLM layers during SSD-streaming decode instead of decode-only spans. | [ds4.c:42616](ds4.c#L42616) | +| `DS4_METAL_GLM_STREAMING_DECODE_SYNC_EACH_LAYER` | boolean text; Metal runtime always synchronizes and does not consult it; legacy fallback name read only in ROCm builds | Controls per-layer GLM streaming decode synchronization only as a legacy ROCm fallback alias. | [ds4.c:49780](ds4.c#L49780) | +| `DS4_METAL_GLM_STREAMING_PREFILL_FULL_LAYER` | presence control; unset: off/default; any value including 0 enables | Forces full-layer GLM SSD prefill regardless of the token crossover. | [ds4_metal.m:14710](ds4_metal.m#L14710) | +| `DS4_METAL_GLM_STREAMING_PREFILL_FULL_LAYER_MIN_TOKENS` | positive uint32; default 64 on Metal, 1024 when used as ROCm fallback; 0/invalid restores default | Sets the token crossover for GLM full-layer SSD prefill. | [ds4.c:42693](ds4.c#L42693) | +| `DS4_METAL_GLM_STREAMING_PREFILL_SYNC_EACH_LAYER` | boolean text; Metal runtime always synchronizes and does not consult it; legacy fallback name read only in ROCm builds | Controls per-layer GLM streaming prefill synchronization only as a legacy ROCm fallback alias. | [ds4.c:42396](ds4.c#L42396) | +| `DS4_METAL_GLM_STREAMING_TOKEN_PREFILL_MAX` | uint32; default 64 on Metal, 0 when used as ROCm fallback; 0 disables; invalid restores default | Sets largest GLM SSD prefill handled token-major by the decode graph. | [ds4.c:49683](ds4.c#L49683) | +| `DS4_METAL_GLU_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the GLU Metal kernel source file loaded at runtime. | [ds4_metal.m:4950](ds4_metal.m#L4950) | +| `DS4_METAL_GPU_BATCH_EMBED_MIN` | uint32 token threshold; default 512; invalid restores default | Sets the batch size at which prompt embedding moves from CPU upload to Metal kernels. | [ds4.c:28867](ds4.c#L28867) | +| `DS4_METAL_GPU_BUSY_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for GPU busy. | [ds4_metal.m:1293](ds4_metal.m#L1293) | +| `DS4_METAL_GRAPH_DUMP_LAYER` | unsigned layer index or all; unset: every layer | Restricts graph tensor dumps to one layer. | [ds4.c:16863](ds4.c#L16863) | +| `DS4_METAL_GRAPH_DUMP_LOGITS` | file path; unset/empty: no graph-logit dump | Writes Metal graph-test logits as f32 binary. | [ds4.c:39080](ds4.c#L39080) | +| `DS4_METAL_GRAPH_DUMP_NAME` | substring; unset/empty: every tensor name | Restricts graph tensor dumps by tensor-name substring. | [ds4.c:16859](ds4.c#L16859) | +| `DS4_METAL_GRAPH_DUMP_POS` | unsigned token position; unset: every position | Restricts graph tensor dumps to one token position. | [ds4.c:16870](ds4.c#L16870) | +| `DS4_METAL_GRAPH_DUMP_PREFIX` | path/prefix; unset/empty: tensor dumping disabled | Enables graph tensor dumps and supplies the filename prefix. | [ds4.c:65148](ds4.c#L65148) | +| `DS4_METAL_GRAPH_DUMP_TRACE` | presence diagnostic; unset: off; any value including 0 enables | Emits trace diagnostics for graph dump. | [ds4.c:16900](ds4.c#L16900) | +| `DS4_METAL_GRAPH_OUTPUT_ROW` | zero-based row smaller than current batch; default final row; invalid restores final row | Chooses which prefill output row is projected to logits. | [ds4.c:35846](ds4.c#L35846) | +| `DS4_METAL_GRAPH_PREFILL_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for graph prefill. | [ds4.c:35523](ds4.c#L35523) | +| `DS4_METAL_GRAPH_PREFILL_SPLIT_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for graph prefill split. | [ds4.c:66204](ds4.c#L66204) | +| `DS4_METAL_GRAPH_PROMPT_TOKENS` | integer 1..prompt length; default full prompt | Limits prompt length used by the Metal graph parity test. | [ds4.c:39023](ds4.c#L39023) | +| `DS4_METAL_GRAPH_RAW_CAP` | positive rows; default from SWA window+prefill; clamped to [raw_window,min(ctx,8192)] | Overrides raw sliding-window KV ring capacity. | [ds4.c:38390](ds4.c#L38390) | +| `DS4_METAL_GRAPH_TEACHER_FORCE` | presence control; unset: off/default; any value including 0 enables | Feeds CPU reference state back into the first-token graph trace at each layer. | [ds4.c:27742](ds4.c#L27742) | +| `DS4_METAL_GRAPH_TOKEN_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for graph token. | [ds4.c:32382](ds4.c#L32382) | +| `DS4_METAL_GRAPH_TOKEN_SECOND_SPLIT_LAYERS` | integer 0..layer count; default 0 plus eligible automatic pre-M5 schedules; explicit value wins | Overrides second command-buffer split layer for token decode. | [ds4.c:28029](ds4.c#L28029) | +| `DS4_METAL_GRAPH_TOKEN_SPLIT_LAYERS` | integer 0..layer count; default 4 on Apple and 0 elsewhere, with eligible pre-M5 adaptive override | Overrides first command-buffer split layer for token decode. | [ds4.c:27915](ds4.c#L27915) | +| `DS4_METAL_GRAPH_TRACE_CACHE` | presence control; unset: off/default; any value including 0 enables | Prints raw KV cache parity diagnostics in the graph prompt test. | [ds4.c:39092](ds4.c#L39092) | +| `DS4_METAL_GRAPH_TRACE_COMP` | presence control; unset: off/default; any value including 0 enables | Prints compressed-cache parity diagnostics in the graph prompt test. | [ds4.c:39093](ds4.c#L39093) | +| `DS4_METAL_GRAPH_TRACE_LAYERS` | presence control; unset: off/default; any value including 0 enables | Enables per-layer first-token CPU/GPU graph tracing. | [ds4.c:27739](ds4.c#L27739) | +| `DS4_METAL_GRAPH_TRACE_STAGE_LAYER` | signed layer index; unset gives -1/no stage-layer selection | Selects the layer used by first-token stage tracing. | [ds4.c:27743](ds4.c#L27743) | +| `DS4_METAL_HC_NORM_FUSION_CHECK` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Compares fused HC normalization against the reference result. | [ds4.c:20521](ds4.c#L20521) | +| `DS4_METAL_HC_NORM_FUSION_CHECK_TOL` | positive finite float; default 2e-4; invalid/nonpositive restores default | Sets the numerical tolerance for the HC norm-fusion oracle. | [ds4.c:20530](ds4.c#L20530) | +| `DS4_METAL_HC_STABLE` | boolean empty/1/true/yes/on vs 0/false/no/off; default on | Compiles stable hidden-context drift arithmetic into the Metal library. | [ds4_metal.m:7047](ds4_metal.m#L7047) | +| `DS4_METAL_INDEXER_STAGE_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for indexer stage. | [ds4.c:17569](ds4.c#L17569) | +| `DS4_METAL_IQ2_XXS_SSD_PREFILL_MM_STATS` | value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables | Collects/prints statistics for IQ2 XXS SSD prefill MM. | [ds4_metal.m:6980](ds4_metal.m#L6980) | +| `DS4_METAL_KV_RAW_F32` | boolean empty/1/true/yes/on vs 0/false/no/off; default off | Compiles raw KV storage as F32 for drift diagnosis. | [ds4_metal.m:7049](ds4_metal.m#L7049) | +| `DS4_METAL_LAYER_STAGE_PROFILE` | unset: off; 1/true/yes/on/all enables all layers; a layer index selects one; 0/false/no/off disables | Prints timing/profile diagnostics for layer stage. | [ds4.c:65150](ds4.c#L65150) | +| `DS4_METAL_LAYER_STAGE_PROFILE_LAYER` | single unsigned layer index; unset/empty: all layers enabled by the parent profile; invalid matches no layer | Restricts the corresponding shared graph stage profiler to one layer. | [ds4.c:29109](ds4.c#L29109) | +| `DS4_METAL_MATH_SAFE` | boolean empty/1/true/yes/on vs 0/false/no/off; default off | Compiles Metal shaders with strict/safe IEEE math instead of fast math. | [ds4_metal.m:7051](ds4_metal.m#L7051) | +| `DS4_METAL_MEMORY_REPORT` | presence control; unset: off/default; any value including 0 enables | Prints Metal allocation/cache/residency memory reports. | [ds4.c:39047](ds4.c#L39047) | +| `DS4_METAL_MODEL_UNTRACKED` | presence control; unset: off/default; any value including 0 enables | Creates mapped model buffers with untracked Metal hazard tracking. | [ds4_metal.m:1547](ds4_metal.m#L1547) | +| `DS4_METAL_MODEL_VIEW_MAX_GIB` | positive integer GiB; default device maximum (128-GiB cap for already-split span maps); cannot exceed device maximum | Caps each no-copy mapped Metal model view. | [ds4_metal.m:2216](ds4_metal.m#L2216) | +| `DS4_METAL_MODEL_WARMUP_STRIDE_KB` | integer 1..1048576 KiB, at least one page; unset inherits MB/default; overrides STRIDE_MB | Sets the model-view warmup touch stride with KiB precision. | [ds4_metal.m:3057](ds4_metal.m#L3057) | +| `DS4_METAL_MODEL_WARMUP_STRIDE_MB` | integer 1..1024 MiB; default 1 MiB; STRIDE_KB overrides | Sets the model-view warmup touch stride. | [ds4_metal.m:3049](ds4_metal.m#L3049) | +| `DS4_METAL_MOE_MM_ID_USE_RESOURCES` | presence control; unset: off/default; any value including 0 enables | Declares MM-ID MoE resource usage explicitly on the command encoder. | [ds4_metal.m:35168](ds4_metal.m#L35168) | +| `DS4_METAL_MOE_ONE_STAGE_PROFILE` | unset: off; 1/true/yes/on/all enables all layers; accepts layer lists/ranges; 0/false/no/off disables | Prints timing/profile diagnostics for MoE one stage. | [ds4.c:22714](ds4.c#L22714) | +| `DS4_METAL_MOE_ONE_STAGE_PROFILE_LAYER` | layer index/list/ranges or all; unset: all profiler-selected layers | Restricts one-stage MoE profiling to selected layers. | [ds4_metal.m:43436](ds4_metal.m#L43436) | +| `DS4_METAL_MOE_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the MoE Metal kernel source file loaded at runtime. | [ds4_metal.m:4937](ds4_metal.m#L4937) | +| `DS4_METAL_MOE_STAGE_PROFILE` | unset: off; 1/true/yes/on/all enables all layers; accepts layer lists/ranges; 0/false/no/off disables | Prints timing/profile diagnostics for MoE stage. | [ds4.c:65154](ds4.c#L65154) | +| `DS4_METAL_MOE_STAGE_PROFILE_FILTER` | substring; unset/empty: all profiled stages | Filters MoE stage-profile output. | [ds4_metal.m:43438](ds4_metal.m#L43438) | +| `DS4_METAL_MOE_STAGE_PROFILE_LAYER` | layer index/list/ranges or all; unset: all profiler-selected layers | Restricts batched MoE stage profiling to selected layers. | [ds4_metal.m:45346](ds4_metal.m#L45346) | +| `DS4_METAL_MOE_WRITE_CLAMPED_ACT` | presence control; unset: off/default; any value including 0 enables | Makes routed MoE write the clamped activation diagnostic. | [ds4.c:18527](ds4.c#L18527) | +| `DS4_METAL_NORM_RSQRT_DISABLE` | boolean empty/1/true/yes/on vs 0/false/no/off; default on | Compiles unified normalization-rsqrt arithmetic into the Metal library. | [ds4_metal.m:7048](ds4_metal.m#L7048) | +| `DS4_METAL_NORM_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the normalization Metal kernel source file loaded at runtime. | [ds4_metal.m:4951](ds4_metal.m#L4951) | +| `DS4_METAL_NO_MODEL_WARMUP` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables model warmup. | [ds4_metal.m:2309](ds4_metal.m#L2309) | +| `DS4_METAL_NO_PREFILL_KERNEL_WARMUP` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables prefill kernel warmup. | [ds4.c:28965](ds4.c#L28965) | +| `DS4_METAL_NO_RESIDENCY` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables residency. | [ds4_metal.m:2092](ds4_metal.m#L2092) | +| `DS4_METAL_OUTPUT_STAGE_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for output stage. | [ds4.c:17570](ds4.c#L17570) | +| `DS4_METAL_PREFILL_CHUNK` | positive token count used only when CLI chunk is absent; default full prompt, or 4096 for long non-PRO and 8192 for long PRO prompts; <=0 keeps automatic/full prompt | Provides the historical environment fallback for prefill chunk size. | [ds4.c:12802](ds4.c#L12802) | +| `DS4_METAL_PRO_Q4_CPU_ROUTER` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Uses the CPU router for PRO Q4 selected-expert decode. | [ds4.c:20934](ds4.c#L20934) | +| `DS4_METAL_PRO_Q4_CPU_ROUTER_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for pro Q4 CPU router. | [ds4.c:21644](ds4.c#L21644) | +| `DS4_METAL_Q4_ADDR_USE_RESOURCES` | presence control; unset: off/default; any value including 0 enables | Declares Q4 address-table resources explicitly on encoders. | [ds4_metal.m:20259](ds4_metal.m#L20259) | +| `DS4_METAL_Q4_EXPERT_GROUP_SIZE` | positive uint32; default 32; clamped to total expert count | Sets experts processed per grouped Q4 dispatch. | [ds4_metal.m:34088](ds4_metal.m#L34088) | +| `DS4_METAL_Q4_EXPERT_TABLE_GROUP_SIZE` | integer 2..total experts; default/invalid 1 (ungrouped) | Sets grouped exact-view width while building Q4 expert tables. | [ds4_metal.m:19604](ds4_metal.m#L19604) | +| `DS4_METAL_Q4_EXPERT_TABLE_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for Q4 expert table. | [ds4_metal.m:20103](ds4_metal.m#L20103) | +| `DS4_METAL_Q4_GROUP24_BASE_VIEWS` | presence control; unset: off/default; any value including 0 enables | Uses broad base model views for Q4 group-24 instead of exact views. | [ds4_metal.m:42205](ds4_metal.m#L42205) | +| `DS4_METAL_Q4_GROUP24_EXACT_VIEWS` | presence control; unset: off/default; any value including 0 enables | Uses exact mapped views for Q4 group-24 experts. | [ds4_metal.m:42204](ds4_metal.m#L42204) | +| `DS4_METAL_Q4_GROUPED_CACHE_VIEWS` | presence control; unset: off/default; any value including 0 enables | Caches exact Q4 grouped expert views. | [ds4_metal.m:42151](ds4_metal.m#L42151) | +| `DS4_METAL_Q4_PRO_MAP_GROUPS` | positive divisor of 384 in 1..384; default/invalid 1 | Splits each 384-expert PRO Q4 tensor into this many mapped views. | [ds4.c:6276](ds4.c#L6276) | +| `DS4_METAL_Q4_SELECTED_EXACT_VIEWS` | presence control; unset: off/default; any value including 0 enables | Forces exact/cached views for selected Q4 experts instead of base views. | [ds4_metal.m:42528](ds4_metal.m#L42528) | +| `DS4_METAL_Q4_SELECTED_OVERLAP_SHARED` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Overlaps selected Q4 expert preparation with the shared expert. | [ds4.c:20962](ds4.c#L20962) | +| `DS4_METAL_Q4_SELECTED_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for Q4 selected. | [ds4.c:37944](ds4.c#L37944) | +| `DS4_METAL_Q4_SELECTED_PROFILE_LAYER` | single nonnegative layer index; unset: every layer | Restricts legacy Q4 selected-expert profiling to one layer. | [ds4_metal.m:42509](ds4_metal.m#L42509) | +| `DS4_METAL_Q4_SELECTED_SHARED_EVENT` | presence control; unset: off/default; any value including 0 enables | Coordinates selected Q4 work with a shared Metal event. | [ds4_metal.m:42524](ds4_metal.m#L42524) | +| `DS4_METAL_Q4_SELECTED_TRANSIENT_VIEWS` | presence control; unset: off/default; any value including 0 enables | Uses transient exact views for selected Q4 experts. | [ds4_metal.m:42532](ds4_metal.m#L42532) | +| `DS4_METAL_Q4_SELECTED_USE_BASE_VIEWS` | presence control; unset: off/default; any value including 0 enables | Uses broad base model views for selected Q4 experts. | [ds4_metal.m:42527](ds4_metal.m#L42527) | +| `DS4_METAL_Q4_TABLE_BIND_ANCHORS` | presence control; unset: off/default; any value including 0 enables | Binds anchor buffers alongside the Q4 expert address table. | [ds4_metal.m:19716](ds4_metal.m#L19716) | +| `DS4_METAL_Q4_TABLE_MODEL_RESIDENCY_SET` | presence control; unset: off/default; any value including 0 enables | Adds Q4 expert table allocations to the model residency set. | [ds4_metal.m:19657](ds4_metal.m#L19657) | +| `DS4_METAL_Q4_TABLE_PER_TENSOR_RESIDENCY_SET` | presence control; unset: off/default; any value including 0 enables | Builds separate residency sets per Q4 expert tensor. | [ds4_metal.m:19760](ds4_metal.m#L19760) | +| `DS4_METAL_Q4_TABLE_QUEUE_RESIDENCY_SET` | presence control; unset: off/default; any value including 0 enables | Attaches Q4 expert table residency sets to command queues. | [ds4_metal.m:19615](ds4_metal.m#L19615) | +| `DS4_METAL_Q4_TABLE_RESIDENCY_SET` | presence control; unset: off/default; any value including 0 enables | Enables Q4 expert table residency-set handling. | [ds4_metal.m:19759](ds4_metal.m#L19759) | +| `DS4_METAL_Q4_TABLE_USE_RESOURCES` | presence control; unset: off/default; any value including 0 enables | Declares Q4 table resources explicitly on encoders. | [ds4_metal.m:20258](ds4_metal.m#L20258) | +| `DS4_METAL_Q8_DECODE_EXACT_VIEW_MAX_MIB` | integer 1..4096 MiB; default 1024; above max clamps, below min/invalid restores default | Caps weight ranges eligible for Q8 exact model views. | [ds4_metal.m:12876](ds4_metal.m#L12876) | +| `DS4_METAL_Q8_MV_EXT_MAX_TOKENS` | integer 2..128; default 16; above max clamps, below min/invalid restores default | Sets largest batch handled by extended Q8 matvec. | [ds4_metal.m:21126](ds4_metal.m#L21126) | +| `DS4_METAL_Q8_MV_NSG` | integer 1..8 simdgroups; default 4, or 2 with TP world=2; above max clamps, below min/invalid restores default | Overrides simdgroups per Q8 matvec threadgroup. | [ds4.c:22730](ds4.c#L22730) | +| `DS4_METAL_Q8_PREFILL_PROFILE` | value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables | Prints timing/profile diagnostics for Q8 prefill. | [ds4_metal.m:21280](ds4_metal.m#L21280) | +| `DS4_METAL_Q8_PREFILL_PROFILE_FILTER` | substring matched against generated operation label; unset/empty: all eligible calls | Filters Q8 prefill profiling. | [ds4_metal.m:21295](ds4_metal.m#L21295) | +| `DS4_METAL_Q_STAGE_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for q stage. | [ds4.c:29270](ds4.c#L29270) | +| `DS4_METAL_REPEAT_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the repeat Metal kernel source file loaded at runtime. | [ds4_metal.m:4949](ds4_metal.m#L4949) | +| `DS4_METAL_REQUIRE_COMPRESSOR_EXACT_POOL_RATIO4` | presence strict check; unset: fallback allowed; any value including 0 requires the path | Requires compressor exact pool ratio4 and makes eligible fallback fail closed. | [ds4_metal.m:26387](ds4_metal.m#L26387) | +| `DS4_METAL_REQUIRE_EXACT_ROWS_PERSISTENT_CACHE` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Requires exact rows persistent cache and makes eligible fallback fail closed. | [ds4_metal.m:13002](ds4_metal.m#L13002) | +| `DS4_METAL_REQUIRE_GATHERED_KV_STAGE` | presence strict check; unset: fallback allowed; any value including 0 requires the path | Requires gathered KV stage and makes eligible fallback fail closed. | [ds4_metal.m:29680](ds4_metal.m#L29680) | +| `DS4_METAL_REQUIRE_IQ2_XXS_SSD_PREFILL_MM` | value-aware boolean; default implicit fail-closed only with complete selected-address domain; explicit 1 is strict, 0 permits fallback | Makes eligible IQ2_XXS/Q2_K grouped SSD-prefill MM fail closed. | [ds4_metal.m:44782](ds4_metal.m#L44782) | +| `DS4_METAL_REQUIRE_M1_IQ2_MID_ONLY` | presence strict check; unset: fallback allowed; any value including 0 requires the path | Requires M1 IQ2 mid only and makes eligible fallback fail closed. | [ds4_metal.m:42074](ds4_metal.m#L42074) | +| `DS4_METAL_REQUIRE_OUTPUT_HC_WEIGHTS4` | presence strict check; unset: fallback allowed; any value including 0 requires the path | Requires output HC weights4 and makes eligible fallback fail closed. | [ds4_metal.m:46789](ds4_metal.m#L46789) | +| `DS4_METAL_REQUIRE_Q4_ATTN_OUT_TINY_BATCH` | value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables | Requires Q4 attn out tiny batch and makes eligible fallback fail closed. | [ds4_metal.m:28373](ds4_metal.m#L28373) | +| `DS4_METAL_REQUIRE_Q4_SSD_PREFILL_ATTN_OUT_EXACTN` | value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables | Requires Q4 SSD prefill attn out exactn and makes eligible fallback fail closed. | [ds4_metal.m:28089](ds4_metal.m#L28089) | +| `DS4_METAL_REQUIRE_Q4_SSD_SESSION_UNION` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Requires Q4 SSD session union and makes eligible fallback fail closed. | [ds4.c:65164](ds4.c#L65164) | +| `DS4_METAL_REQUIRE_Q8_QKV_COMPRESSOR_FUSE` | nonempty boolean; unset/empty/exact 0: fallback allowed; other values require and imply the streamed/union enable | Requires eligible Q8 QKV/compressor compound fusion and fails closed. | [ds4.c:23023](ds4.c#L23023) | +| `DS4_METAL_RESUME_PREFILL_MIN` | integer token threshold; default 4; <=0 disables resume-prefill | Sets the minimum shared-prefix suffix that uses batched resume-prefill. | [ds4.c:38419](ds4.c#L38419) | +| `DS4_METAL_ROPE_EXP2_LOG2` | boolean empty/1/true/yes/on vs 0/false/no/off; default off | Compiles the exp2/log2 RoPE drift variant. | [ds4_metal.m:7050](ds4_metal.m#L7050) | +| `DS4_METAL_SELECTED_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for selected. | [ds4.c:37943](ds4.c#L37943) | +| `DS4_METAL_SELECTED_PROFILE_LAYER` | single nonnegative layer index; unset: every layer | Restricts selected-expert profiling to one layer. | [ds4_metal.m:42507](ds4_metal.m#L42507) | +| `DS4_METAL_SESSION_BATCH_LOG` | presence diagnostic; unset: off; any value including 0 enables | Logs session batch decisions. | [ds4.c:66168](ds4.c#L66168) | +| `DS4_METAL_SESSION_BATCH_QKV` | default enabled; exact 0 disables; every other value/unset leaves enabled | Controls native batched QKV work for multi-session decode. | [ds4.c:65482](ds4.c#L65482) | +| `DS4_METAL_SESSION_BATCH_SHARED` | default enabled; exact 0 disables; every other value/unset leaves enabled | Controls native batched shared-expert work for multi-session decode. | [ds4.c:65432](ds4.c#L65432) | +| `DS4_METAL_SET_ROWS_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the set-rows Metal kernel source file loaded at runtime. | [ds4_metal.m:4953](ds4_metal.m#L4953) | +| `DS4_METAL_SOFTMAX_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the softmax Metal kernel source file loaded at runtime. | [ds4_metal.m:4948](ds4_metal.m#L4948) | +| `DS4_METAL_STREAMING_DECODE_PREFILL_MAX` | integer token maximum; default 64 for wide Flash Q4/MXFP4, 18 for other PRO/Flash, 0 otherwise; <=0 disables | Sets maximum SSD-streaming micro-prefill width that reuses decode. | [ds4.c:31962](ds4.c#L31962) | +| `DS4_METAL_STREAMING_EXPERT_AUTO_PRELOAD_CAP` | uint32 expert cap; default 4096; 0 means unlimited; invalid restores default | Caps automatic streaming-expert hotlist preload. | [ds4.c:21455](ds4.c#L21455) | +| `DS4_METAL_STREAMING_EXPERT_BUFFER_MLOCK_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming expert buffer mlock. | [ds4_metal.m:13997](ds4_metal.m#L13997) | +| `DS4_METAL_STREAMING_EXPERT_EARLY_LOAD_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming expert early load. | [ds4_metal.m:17059](ds4_metal.m#L17059) | +| `DS4_METAL_STREAMING_EXPERT_EVICT_DONTNEED_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming expert evict dontneed. | [ds4_metal.m:14436](ds4_metal.m#L14436) | +| `DS4_METAL_STREAMING_EXPERT_HOTLIST` | hotlist file path; unset/empty: built-in model hotlist | Loads the streaming-expert preload order from a file. | [ds4.c:21495](ds4.c#L21495) | +| `DS4_METAL_STREAMING_EXPERT_HOTLIST_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming expert hotlist. | [ds4.c:32245](ds4.c#L32245) | +| `DS4_METAL_STREAMING_EXPERT_LAYER_STATS` | presence diagnostic; unset: off; any value including 0 enables | Collects/prints statistics for streaming expert layer. | [ds4_metal.m:4638](ds4_metal.m#L4638) | +| `DS4_METAL_STREAMING_EXPERT_LAYER_STATS_DELTA` | presence control; unset: off/default; any value including 0 enables | Prints delta statistics for streaming expert layer. | [ds4_metal.m:4675](ds4_metal.m#L4675) | +| `DS4_METAL_STREAMING_EXPERT_NOCACHE` | nonempty value whose first character is not 0 enables; unset/empty/0 disables | Uses a reopened F_NOCACHE descriptor for SSD expert preads. | [ds4_metal.m:12602](ds4_metal.m#L12602) | +| `DS4_METAL_STREAMING_EXPERT_PREAD_POOL` | default enabled; exact 0 disables; every other value/unset keeps enabled | Controls reuse of persistent expert-pread worker threads. | [ds4_metal.m:13433](ds4_metal.m#L13433) | +| `DS4_METAL_STREAMING_EXPERT_PREAD_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming expert pread. | [ds4_metal.m:17005](ds4_metal.m#L17005) | +| `DS4_METAL_STREAMING_EXPERT_PREAD_SPLIT` | integer clamped 1..8; unset: automatic 1 below 64 cache experts, 4 at 64+ | Sets aligned requests per expert pread. | [ds4_metal.m:13701](ds4_metal.m#L13701) | +| `DS4_METAL_STREAMING_EXPERT_PREAD_THREADS` | unsigned integer clamped 1..18; default 9; invalid restores 9 | Sets expert-pread worker limit. | [ds4_metal.m:13337](ds4_metal.m#L13337) | +| `DS4_METAL_STREAMING_EXPERT_PROFILE_SUMMARY` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming expert. | [ds4_metal.m:13061](ds4_metal.m#L13061) | +| `DS4_METAL_STREAMING_EXPERT_SLAB_MB` | positive unsigned MiB; default 4096; 0/invalid restores default | Sets target allocation size for streaming-expert slabs. | [ds4_metal.m:14039](ds4_metal.m#L14039) | +| `DS4_METAL_STREAMING_EXPERT_SPLIT_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming expert split. | [ds4_metal.m:43810](ds4_metal.m#L43810) | +| `DS4_METAL_STREAMING_EXPERT_TIMING_SUMMARY` | presence control; unset: off/default; any value including 0 enables | Prints timing/profile diagnostics for streaming expert. | [ds4_metal.m:13060](ds4_metal.m#L13060) | +| `DS4_METAL_STREAMING_IQ2_CPU_ROUTER_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming IQ2 CPU router. | [ds4.c:21645](ds4.c#L21645) | +| `DS4_METAL_STREAMING_MAP_TRACE` | nonempty value other than exact 0 enables; unset/empty/0 disables | Emits SSD model-map decisions. | [ds4_metal.m:4849](ds4_metal.m#L4849) | +| `DS4_METAL_STREAMING_PREFILL_BATCH_SELECTED_ADDR_MAX` | integer token maximum; default 800 for 384 experts, 760 for 256, 0 otherwise; <=0 disables automatic selection | Sets automatic maximum batch width for selected-address SSD prefill. | [ds4_metal.m:14639](ds4_metal.m#L14639) | +| `DS4_METAL_STREAMING_PREFILL_BATCH_SELECTED_ADDR_MIN` | integer token minimum; default 2 for 256/384 experts, 0 otherwise; <=0 disables automatic selection | Sets automatic minimum batch width for selected-address SSD prefill. | [ds4_metal.m:14656](ds4_metal.m#L14656) | +| `DS4_METAL_STREAMING_PREFILL_BATCH_SELECTED_ADDR_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming prefill batch selected address. | [ds4_metal.m:18493](ds4_metal.m#L18493) | +| `DS4_METAL_STREAMING_PREFILL_CACHE_SEED_K` | uint32 seed rows; default 1; 0 disables; above 64 clamps to 64 | Sets how many prefill routing rows seed the decode expert cache. | [ds4.c:21268](ds4.c#L21268) | +| `DS4_METAL_STREAMING_PREFILL_CACHE_SEED_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming prefill cache seed. | [ds4_metal.m:17899](ds4_metal.m#L17899) | +| `DS4_METAL_STREAMING_PREFILL_LAYER_MADVISE_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming prefill layer madvise. | [ds4.c:19610](ds4.c#L19610) | +| `DS4_METAL_STREAMING_PREFILL_LAYER_PAGEIN_NO_OVERLAP` | presence rollback; unset: automatic/default path; any value including 0 disables | Prevents full-layer page-in preparation from overlapping compute. | [ds4.c:19317](ds4.c#L19317) | +| `DS4_METAL_STREAMING_PREFILL_LAYER_PAGEIN_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming prefill layer pagein. | [ds4.c:19606](ds4.c#L19606) | +| `DS4_METAL_STREAMING_PREFILL_LAYER_PAGEIN_THREADS` | integer 1..16; default 8; invalid/0 becomes 1; PREPARE_THREADS takes precedence | Sets worker count for full-layer page-in preparation. | [ds4.c:19275](ds4.c#L19275) | +| `DS4_METAL_STREAMING_PREFILL_LAYER_PREAD_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming prefill layer pread. | [ds4.c:19608](ds4.c#L19608) | +| `DS4_METAL_STREAMING_PREFILL_LAYER_PREPARE_AHEAD` | integer 1..4 layers; default 1; invalid/0 becomes 1 | Sets number of future layers prepared concurrently. | [ds4.c:19329](ds4.c#L19329) | +| `DS4_METAL_STREAMING_PREFILL_LAYER_PREPARE_NO_OVERLAP` | presence rollback; unset: automatic/default path; any value including 0 disables | Prevents generic full-layer preparation from overlapping compute. | [ds4.c:19315](ds4.c#L19315) | +| `DS4_METAL_STREAMING_PREFILL_LAYER_PREPARE_THREADS` | integer 1..16; default 8; invalid/0 becomes 1; preferred over PAGEIN_THREADS | Sets worker count for full-layer preparation. | [ds4.c:19271](ds4.c#L19271) | +| `DS4_METAL_STREAMING_PREFILL_LAYER_READAHEAD_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming prefill layer readahead. | [ds4.c:19612](ds4.c#L19612) | +| `DS4_METAL_STREAMING_PREFILL_SELECTED_MADVISE_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming prefill selected madvise. | [ds4.c:19356](ds4.c#L19356) | +| `DS4_METAL_STREAMING_PREFILL_SELECTED_MADVISE_THREADS` | integer 1..16; default inherits layer prepare threads; invalid/0 becomes 1; PREPARE_THREADS preferred | Sets worker count for selected-expert madvise preparation. | [ds4.c:19293](ds4.c#L19293) | +| `DS4_METAL_STREAMING_PREFILL_SELECTED_PAGEIN_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming prefill selected pagein. | [ds4.c:19354](ds4.c#L19354) | +| `DS4_METAL_STREAMING_PREFILL_SELECTED_PREPARE_GAP` | integer 0..8 layers; default 0; above 8 clamps; invalid restores 0 | Sets lookahead gap for selected-expert preparation. | [ds4.c:19305](ds4.c#L19305) | +| `DS4_METAL_STREAMING_PREFILL_SELECTED_PREPARE_THREADS` | integer 1..16 for madvise preparation; default inherits layer prepare threads; invalid/0 becomes 1 | Sets worker count for selected-expert preparation. | [ds4.c:19289](ds4.c#L19289) | +| `DS4_METAL_STREAMING_PREFILL_SELECTED_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming prefill selected. | [ds4.c:18906](ds4.c#L18906) | +| `DS4_METAL_STREAMING_PREFILL_SELECTED_READAHEAD_GAP` | integer 0..8 layers; default 0; above 8 clamps; invalid restores 0 | Sets lookahead gap for selected-expert readahead. | [ds4.c:19978](ds4.c#L19978) | +| `DS4_METAL_STREAMING_PREFILL_SELECTED_READAHEAD_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming prefill selected readahead. | [ds4.c:20063](ds4.c#L20063) | +| `DS4_METAL_STREAMING_SELECTED_READAHEAD_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming selected readahead. | [ds4.c:21737](ds4.c#L21737) | +| `DS4_METAL_SUM_ROWS_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the sum-rows Metal kernel source file loaded at runtime. | [ds4_metal.m:4947](ds4_metal.m#L4947) | +| `DS4_METAL_TEST_POISON_COMPRESSOR_EXACT_REDUCTION_SCRATCH` | internal test presence flag; unset: off; any value including 0 poisons scratch before the exact reduction | Validates that compressor exact-reduction kernels overwrite all scratch state. | [ds4_metal.m:25925](ds4_metal.m#L25925) | +| `DS4_METAL_TP_SESSION_BATCH` | default enabled; exact 0 disables; every other value/unset leaves enabled | Controls batched session evaluation with Metal TP. | [ds4.c:65310](ds4.c#L65310) | +| `DS4_METAL_TRACE_ALLOCS` | presence diagnostic; unset: off; any value including 0 enables | Emits trace diagnostics for allocs. | [ds4_metal.m:4057](ds4_metal.m#L4057) | +| `DS4_METAL_TRACE_M5_FLASH_ATTN_PACKED32_REDUCE` | presence diagnostic; unset: off; any value including 0 enables | Emits trace diagnostics for M5 flash attn packed32 reduce. | [ds4_metal.m:31535](ds4_metal.m#L31535) | +| `DS4_METAL_UNARY_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the unary operations Metal kernel source file loaded at runtime. | [ds4_metal.m:4939](ds4_metal.m#L4939) | +| `DS4_METAL_UNRETAINED_COMMAND_BUFFERS` | presence control; unset: off/default; any value including 0 enables | Creates Metal command buffers with unretained references. | [ds4_metal.m:1315](ds4_metal.m#L1315) | +| `DS4_METAL_USE_QUEUE_RESIDENCY_SET` | presence control; unset: off/default; any value including 0 enables | Allows queue-residency state to trigger Q4 expert address/table paths. | [ds4_metal.m:42280](ds4_metal.m#L42280) |
@@ -572,47 +576,47 @@ and **19 tool/wrapper entries**. | `DS4_CUDA_ATTN_Q_B_F32_CACHE` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Enable an F32-derived-weight cache for attention Q-B weights. | [ds4_cuda.cu:2414](ds4_cuda.cu#L2414) | | `DS4_CUDA_BUILD_ARTIFACTS` | boolean-ish, default on for eligible derived artifacts; only exact 0 disables | Control construction of eligible CUDA derived/repacked weight artifacts. | [ds4_cuda.cu:8440](ds4_cuda.cu#L8440) | | `DS4_CUDA_COPY_MODEL` | nonempty-string opt-in (but mere presence also suppresses prefetch); default off; value 0 is nonempty and requests a full copy | Copy the complete mapped model image into device memory. | [ds4_cuda.cu:2740](ds4_cuda.cu#L2740) | -| `DS4_CUDA_COPY_MODEL_CHUNKED` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Use range-by-range model prefetch/copy preparation instead of the normal bulk preparation. | [ds4_cuda.cu:37720](ds4_cuda.cu#L37720) | +| `DS4_CUDA_COPY_MODEL_CHUNKED` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Use range-by-range model prefetch/copy preparation instead of the normal bulk preparation. | [ds4_cuda.cu:37740](ds4_cuda.cu#L37740) | | `DS4_CUDA_DECODE_GRAPHS` | boolean, default on; any value starting with 0, or exact off/no/false in listed case variants, disables; oracle flags force off; effective only on one GPU | Control CUDA Graph capture and replay for decode. | [ds4_cuda.cu:1468](ds4_cuda.cu#L1468) | | `DS4_CUDA_DECODE_GRAPH_LOG` | presence diagnostic flag; default off; any defined value including 0 enables | Log CUDA decode-graph cache misses, capture failures, and lifecycle events. | [ds4_cuda.cu:1580](ds4_cuda.cu#L1580) | | `DS4_CUDA_DECODE_HEADS8_ONLINE` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Force the eight-head online CUDA decode-attention kernel when eligible. | [ds4_cuda.cu:371](ds4_cuda.cu#L371) | | `DS4_CUDA_DECODE_SCORE4` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Select four score lanes in the CUDA decode-attention fallback kernel. | [ds4_cuda.cu:372](ds4_cuda.cu#L372) | | `DS4_CUDA_DECODE_SCORE8` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Select eight score lanes in the CUDA decode-attention fallback kernel. | [ds4_cuda.cu:373](ds4_cuda.cu#L373) | | `DS4_CUDA_DIRECT_MODEL` | mixed presence/nonempty flag, default off; any defined value bypasses host caching, while backend direct lookup requires nonempty; value 0 therefore still changes behavior | Use the mapped model directly and bypass selective CUDA weight caching. | [ds4.c:3058](ds4.c#L3058); [ds4_cuda.cu:1250](ds4_cuda.cu#L1250) | -| `DS4_CUDA_DISABLE_DSPARK_EXACTN` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Disable the CUDA DSpark exactn optimization. | [ds4.c:52080](ds4.c#L52080) | -| `DS4_CUDA_DISABLE_DSPARK_EXACTN_BATCH_HEAD` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Disable the CUDA DSpark exactn batch head optimization. | [ds4.c:37095](ds4.c#L37095) | -| `DS4_CUDA_DISABLE_DSPARK_EXACTN_GRAPHS` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Disable the CUDA DSpark exactn graphs optimization. | [ds4.c:37061](ds4.c#L37061) | +| `DS4_CUDA_DISABLE_DSPARK_EXACTN` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Disable the CUDA DSpark exactn optimization. | [ds4.c:52270](ds4.c#L52270) | +| `DS4_CUDA_DISABLE_DSPARK_EXACTN_BATCH_HEAD` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Disable the CUDA DSpark exactn batch head optimization. | [ds4.c:37285](ds4.c#L37285) | +| `DS4_CUDA_DISABLE_DSPARK_EXACTN_GRAPHS` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Disable the CUDA DSpark exactn graphs optimization. | [ds4.c:37251](ds4.c#L37251) | | `DS4_CUDA_DISABLE_DSPARK_NONCAUSAL_ONLINE` | value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on | Disable the noncausal online-attention DSpark experiment. | [ds4_cuda.cu:21691](ds4_cuda.cu#L21691) | | `DS4_CUDA_DISABLE_HC_NORM_MIX_FUSE` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable fused HC RMSNorm-plus-mix. | [ds4_cuda.cu:20905](ds4_cuda.cu#L20905) | | `DS4_CUDA_DISABLE_HC_SPLIT_NORM_FUSED` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the fused HC split/weighted-sum/norm kernel. | [ds4_cuda.cu:31785](ds4_cuda.cu#L31785) | | `DS4_CUDA_DISABLE_IQ2_XXS_SSD_PREFILL_MMQ` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Disable the CUDA IQ2 XXS SSD prefill MMQ optimization/path. | [ds4_cuda.cu:4627](ds4_cuda.cu#L4627) | -| `DS4_CUDA_DISABLE_Q4_ATTN_OUT_HC_FUSE` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable fused Q4 attention-output/HC expansion. | [ds4_cuda.cu:37337](ds4_cuda.cu#L37337) | +| `DS4_CUDA_DISABLE_Q4_ATTN_OUT_HC_FUSE` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable fused Q4 attention-output/HC expansion. | [ds4_cuda.cu:37357](ds4_cuda.cu#L37357) | | `DS4_CUDA_DISABLE_Q4_DENSE_PAIR` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q4 dense pair CUDA Q4 optimization. | [ds4_cuda.cu:20402](ds4_cuda.cu#L20402) | | `DS4_CUDA_DISABLE_Q8_HC_EXPAND_FUSED` | false-like-aware flag, default off; 0/false/no/off is off, other nonempty values request split; force-fused wins | Request the split Q8 shared-down/HC path when safe. | [ds4_cuda.cu:2086](ds4_cuda.cu#L2086) | -| `DS4_CUDA_DISABLE_QKV_RMS_FUSED` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA QKV RMS fused optimization/path. | [ds4_cuda.cu:369](ds4_cuda.cu#L369); [ds4.c:17255](ds4.c#L17255) | +| `DS4_CUDA_DISABLE_QKV_RMS_FUSED` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA QKV RMS fused optimization/path. | [ds4_cuda.cu:369](ds4_cuda.cu#L369); [ds4.c:17428](ds4.c#L17428) | | `DS4_CUDA_DISABLE_SHARED_GATE_UP_PAIR` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA shared gate up pair optimization/path. | [ds4_cuda.cu:24293](ds4_cuda.cu#L24293) | | `DS4_CUDA_DISABLE_STREAMING_EXPERT_PERSISTENT_CACHE` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Disable streaming expert persistent cache in CUDA SSD streaming. | [ds4_cuda.cu:4113](ds4_cuda.cu#L4113) | -| `DS4_CUDA_DISABLE_STREAMING_PREFILL_BATCH_SELECTED_ADDR` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable streaming prefill batch selected addr in CUDA SSD streaming. | [ds4.c:18419](ds4.c#L18419) | -| `DS4_CUDA_DISABLE_STREAMING_PREFILL_BATCH_SELECTED_LOAD` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable streaming prefill batch selected load in CUDA SSD streaming. | [ds4.c:21744](ds4.c#L21744) | +| `DS4_CUDA_DISABLE_STREAMING_PREFILL_BATCH_SELECTED_ADDR` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable streaming prefill batch selected addr in CUDA SSD streaming. | [ds4.c:18592](ds4.c#L18592) | +| `DS4_CUDA_DISABLE_STREAMING_PREFILL_BATCH_SELECTED_LOAD` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable streaming prefill batch selected load in CUDA SSD streaming. | [ds4.c:21917](ds4.c#L21917) | | `DS4_CUDA_DISABLE_STREAMING_SELECTED_BATCH_IO` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Disable streaming selected batch I/O in CUDA SSD streaming. | [ds4_cuda.cu:4734](ds4_cuda.cu#L4734) | | `DS4_CUDA_DISABLE_STREAMING_SELECTED_EVENT_PIPELINE` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Disable streaming selected event pipeline in CUDA SSD streaming. | [ds4_cuda.cu:4866](ds4_cuda.cu#L4866) | -| `DS4_CUDA_DISABLE_STREAMING_SELECTED_SHARED_OVERLAP` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable streaming selected shared overlap in CUDA SSD streaming. | [ds4.c:20796](ds4.c#L20796) | -| `DS4_CUDA_DSPARK_DEVICE_PROPOSER` | value-aware opt-in, default off; 0/off/no/false (lowercase only) disable; other nonempty enables unless rollback set | Enable the CUDA-resident DSpark proposer. | [ds4.c:34699](ds4.c#L34699); [ds4_cuda.cu:19035](ds4_cuda.cu#L19035) | -| `DS4_CUDA_DSPARK_EXACT2` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Enable the exact two-draft CUDA DSpark support path. | [ds4.c:52057](ds4.c#L52057) | -| `DS4_CUDA_DSPARK_EXACTN` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Enable the exact multi-draft CUDA DSpark support path. | [ds4.c:52078](ds4.c#L52078) | -| `DS4_CUDA_DSPARK_EXACTN_BATCH_HEAD` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Enable the batched output-head stage for exact-N DSpark verification. | [ds4.c:37093](ds4.c#L37093) | -| `DS4_CUDA_DSPARK_EXACTN_GRAPHS` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Enable CUDA Graph capture for stable exact-N DSpark islands. | [ds4.c:37059](ds4.c#L37059) | -| `DS4_CUDA_DSPARK_NO_DEVICE_PROPOSER` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Disable the CUDA-resident DSpark proposer. | [ds4.c:34709](ds4.c#L34709); [ds4_cuda.cu:19040](ds4_cuda.cu#L19040) | -| `DS4_CUDA_DSPARK_NO_PADDED_HEAD` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Disable the padded CUDA output-head optimization used by DSpark. | [ds4.c:34057](ds4.c#L34057) | -| `DS4_CUDA_DSPARK_NO_Q_NORM_ROPE_FUSION` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Disable fused Q RMSNorm plus RoPE in DSpark support stages. | [ds4.c:33433](ds4.c#L33433) | -| `DS4_CUDA_DSPARK_PROPOSER_BLOCK_MAX` | integer 0..UINT32_MAX; 0/invalid keeps native size; unset uses auto caps for exact-N/exact2; positive values cap the block and are limited by DS4_DSPARK_MAX_BLOCK_SIZE | Cap the CUDA DSpark proposal block length. | [ds4.c:52164](ds4.c#L52164) | +| `DS4_CUDA_DISABLE_STREAMING_SELECTED_SHARED_OVERLAP` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable streaming selected shared overlap in CUDA SSD streaming. | [ds4.c:20969](ds4.c#L20969) | +| `DS4_CUDA_DSPARK_DEVICE_PROPOSER` | value-aware opt-in, default off; 0/off/no/false (lowercase only) disable; other nonempty enables unless rollback set | Enable the CUDA-resident DSpark proposer. | [ds4.c:34889](ds4.c#L34889); [ds4_cuda.cu:19035](ds4_cuda.cu#L19035) | +| `DS4_CUDA_DSPARK_EXACT2` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Enable the exact two-draft CUDA DSpark support path. | [ds4.c:52247](ds4.c#L52247) | +| `DS4_CUDA_DSPARK_EXACTN` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Enable the exact multi-draft CUDA DSpark support path. | [ds4.c:52268](ds4.c#L52268) | +| `DS4_CUDA_DSPARK_EXACTN_BATCH_HEAD` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Enable the batched output-head stage for exact-N DSpark verification. | [ds4.c:37283](ds4.c#L37283) | +| `DS4_CUDA_DSPARK_EXACTN_GRAPHS` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Enable CUDA Graph capture for stable exact-N DSpark islands. | [ds4.c:37249](ds4.c#L37249) | +| `DS4_CUDA_DSPARK_NO_DEVICE_PROPOSER` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Disable the CUDA-resident DSpark proposer. | [ds4.c:34899](ds4.c#L34899); [ds4_cuda.cu:19040](ds4_cuda.cu#L19040) | +| `DS4_CUDA_DSPARK_NO_PADDED_HEAD` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Disable the padded CUDA output-head optimization used by DSpark. | [ds4.c:34247](ds4.c#L34247) | +| `DS4_CUDA_DSPARK_NO_Q_NORM_ROPE_FUSION` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Disable fused Q RMSNorm plus RoPE in DSpark support stages. | [ds4.c:33623](ds4.c#L33623) | +| `DS4_CUDA_DSPARK_PROPOSER_BLOCK_MAX` | integer 0..UINT32_MAX; 0/invalid keeps native size; unset uses auto caps for exact-N/exact2; positive values cap the block and are limited by DS4_DSPARK_MAX_BLOCK_SIZE | Cap the CUDA DSpark proposal block length. | [ds4.c:52354](ds4.c#L52354) | | `DS4_CUDA_DSPARK_TINY_ALIGNED_VEC` | value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on | Use aligned routed-MoE vector kernels for tiny DSpark batches. | [ds4_cuda.cu:29523](ds4_cuda.cu#L29523) | | `DS4_CUDA_ENABLE_DSPARK_NONCAUSAL_ONLINE` | value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on | Enable the small-batch noncausal online-attention DSpark experiment. | [ds4_cuda.cu:21690](ds4_cuda.cu#L21690) | | `DS4_CUDA_ENABLE_HC_NORM_MIX_FUSE` | nonempty opt-in, default off; only exact 0 disables; the F32/F16 activation mode follows the selected standalone matmul path; disable/serial/alternate flags can veto | Enable and select the fused HC RMSNorm-plus-mix one-token implementation. | [ds4_cuda.cu:20902](ds4_cuda.cu#L20902) | | `DS4_CUDA_ENABLE_IQ2_XXS_SSD_PREFILL_MMQ` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Enable the CUDA IQ2 XXS SSD prefill MMQ experimental path. | [ds4_cuda.cu:4625](ds4_cuda.cu#L4625) | -| `DS4_CUDA_ENABLE_Q4_ATTN_OUT_HC_FUSE` | value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on | Opt in to the fused Q4 attention-output/HC expansion path. | [ds4_cuda.cu:37355](ds4_cuda.cu#L37355) | -| `DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_BATCH` | value-aware opt-in, default off; nonempty value other than exact 0 enables; rollback wins | Enable flattened grouped attention-A MMQ for two-to-eight-token GB10 batches. | [cuda/mmq/ds4_mmq.cu:4096](cuda/mmq/ds4_mmq.cu#L4096) | -| `DS4_CUDA_ENABLE_Q4_K1024_PERSISTENT` | presence flag, default off; any defined value including 0 requests the path; rollback wins | Enable the GB10 persistent-CTA kernel for M=32768, N=1, K=1024 Q4. | [cuda/mmq/ds4_mmq.cu:3698](cuda/mmq/ds4_mmq.cu#L3698) | +| `DS4_CUDA_ENABLE_Q4_ATTN_OUT_HC_FUSE` | value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on | Opt in to the fused Q4 attention-output/HC expansion path. | [ds4_cuda.cu:37375](ds4_cuda.cu#L37375) | +| `DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_BATCH` | value-aware opt-in, default off; nonempty value other than exact 0 enables; rollback wins | Enable flattened grouped attention-A MMQ for two-to-eight-token GB10 batches. | [cuda/mmq/ds4_mmq.cu:4280](cuda/mmq/ds4_mmq.cu#L4280) | +| `DS4_CUDA_ENABLE_Q4_K1024_PERSISTENT` | presence flag, default off; any defined value including 0 requests the path; rollback wins | Enable the GB10 persistent-CTA kernel for M=32768, N=1, K=1024 Q4. | [cuda/mmq/ds4_mmq.cu:3882](cuda/mmq/ds4_mmq.cu#L3882) | | `DS4_CUDA_ENABLE_Q8_FOLD` | strict flag, default off; only exact value 1 enables; overridden by DS4_CUDA_NO_Q8_FOLD | Enable one-shot producer-to-consumer reuse of freshly quantized Q8_1 data. | [ds4_cuda.cu:785](ds4_cuda.cu#L785) | | `DS4_CUDA_ENABLE_STREAMING_EXPERT_PERSISTENT_CACHE` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Enable streaming expert persistent cache in CUDA SSD streaming. | [ds4_cuda.cu:4111](ds4_cuda.cu#L4111) | | `DS4_CUDA_ENABLE_STREAMING_SELECTED_BATCH_IO` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Enable streaming selected batch I/O in CUDA SSD streaming. | [ds4_cuda.cu:4732](ds4_cuda.cu#L4732) | @@ -634,27 +638,27 @@ and **19 tool/wrapper entries**. | `DS4_CUDA_F16_SMALL_BATCH` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Control the F16 small batch CUDA F16 matmul path. | [ds4_cuda.cu:20837](ds4_cuda.cu#L20837) | | `DS4_CUDA_F16_SMALL_OUT` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Control the F16 small out CUDA F16 matmul path. | [ds4_cuda.cu:20819](ds4_cuda.cu#L20819) | | `DS4_CUDA_GLM_VERIFY_NO_Q8_TOK2` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Control or tune the CUDA glm verify no Q8 tok2 path. | [ds4_cuda.cu:19824](ds4_cuda.cu#L19824) | -| `DS4_CUDA_GREEDY_SPLITKV` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Enable greedy split-KV fast attention. | [ds4.c:17062](ds4.c#L17062) | -| `DS4_CUDA_GREEDY_SPLITKV_FALLBACK_LOG` | presence diagnostic flag; default off; any defined value including 0 enables | Control greedy splitkv fallback log in CUDA greedy fast decode. | [ds4.c:55473](ds4.c#L55473) | -| `DS4_CUDA_GREEDY_SPLITKV_MARGIN` | nonnegative finite float; default 0.25; invalid value warns and uses 0.25; 0 disables margin fallback | Control greedy splitkv margin in CUDA greedy fast decode. | [ds4.c:17140](ds4.c#L17140) | -| `DS4_CUDA_GREEDY_SPLITKV_MAX_SEGMENT` | integer 0..INT32_MAX; default/invalid 0 (segment cap disabled) | Control greedy splitkv max segment in CUDA greedy fast decode. | [ds4.c:17216](ds4.c#L17216) | -| `DS4_CUDA_GREEDY_SPLITKV_PAIR_REPLAY` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Replay greedy split-KV tokens in pairs. | [ds4.c:17190](ds4.c#L17190) | -| `DS4_CUDA_GREEDY_SPLITKV_TOP2` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Use top-2 output margins with greedy split-KV. | [ds4.c:17172](ds4.c#L17172) | -| `DS4_CUDA_GREEDY_SPLITKV_TRACE` | presence diagnostic flag; default off; any defined value including 0 enables | Control greedy splitkv trace in CUDA greedy fast decode. | [ds4.c:55508](ds4.c#L55508) | -| `DS4_CUDA_GREEDY_SPLITKV_TRUST_REPLAY` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Trust replayed greedy split-KV results without the normal confirmation policy. | [ds4.c:17180](ds4.c#L17180) | -| `DS4_CUDA_GREEDY_SPLIT_TOP1` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Enable split top-1 selection in greedy CUDA decode. | [ds4.c:17034](ds4.c#L17034) | -| `DS4_CUDA_GREEDY_TOP1` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control greedy top1 in CUDA greedy fast decode. | [ds4.c:55936](ds4.c#L55936) | -| `DS4_CUDA_GREEDY_VEC4` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Enable greedy vec4 fast attention. | [ds4.c:17072](ds4.c#L17072) | -| `DS4_CUDA_GREEDY_VEC4_FALLBACK_LOG` | presence diagnostic flag; default off; any defined value including 0 enables | Control greedy vec4 fallback log in CUDA greedy fast decode. | [ds4.c:55474](ds4.c#L55474) | -| `DS4_CUDA_GREEDY_VEC4_MARGIN` | nonnegative finite float; default 0.25; invalid value warns and uses 0.25; 0 disables margin fallback | Control greedy vec4 margin in CUDA greedy fast decode. | [ds4.c:17110](ds4.c#L17110) | -| `DS4_CUDA_GREEDY_VEC4_MAX_SEGMENT` | integer 0..INT32_MAX; default/invalid 0 (segment cap disabled) | Control greedy vec4 max segment in CUDA greedy fast decode. | [ds4.c:17224](ds4.c#L17224) | -| `DS4_CUDA_GREEDY_VEC4_TRACE` | presence diagnostic flag; default off; any defined value including 0 enables | Control greedy vec4 trace in CUDA greedy fast decode. | [ds4.c:55537](ds4.c#L55537) | +| `DS4_CUDA_GREEDY_SPLITKV` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Enable greedy split-KV fast attention. | [ds4.c:17235](ds4.c#L17235) | +| `DS4_CUDA_GREEDY_SPLITKV_FALLBACK_LOG` | presence diagnostic flag; default off; any defined value including 0 enables | Control greedy splitkv fallback log in CUDA greedy fast decode. | [ds4.c:55663](ds4.c#L55663) | +| `DS4_CUDA_GREEDY_SPLITKV_MARGIN` | nonnegative finite float; default 0.25; invalid value warns and uses 0.25; 0 disables margin fallback | Control greedy splitkv margin in CUDA greedy fast decode. | [ds4.c:17313](ds4.c#L17313) | +| `DS4_CUDA_GREEDY_SPLITKV_MAX_SEGMENT` | integer 0..INT32_MAX; default/invalid 0 (segment cap disabled) | Control greedy splitkv max segment in CUDA greedy fast decode. | [ds4.c:17389](ds4.c#L17389) | +| `DS4_CUDA_GREEDY_SPLITKV_PAIR_REPLAY` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Replay greedy split-KV tokens in pairs. | [ds4.c:17363](ds4.c#L17363) | +| `DS4_CUDA_GREEDY_SPLITKV_TOP2` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Use top-2 output margins with greedy split-KV. | [ds4.c:17345](ds4.c#L17345) | +| `DS4_CUDA_GREEDY_SPLITKV_TRACE` | presence diagnostic flag; default off; any defined value including 0 enables | Control greedy splitkv trace in CUDA greedy fast decode. | [ds4.c:55698](ds4.c#L55698) | +| `DS4_CUDA_GREEDY_SPLITKV_TRUST_REPLAY` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Trust replayed greedy split-KV results without the normal confirmation policy. | [ds4.c:17353](ds4.c#L17353) | +| `DS4_CUDA_GREEDY_SPLIT_TOP1` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Enable split top-1 selection in greedy CUDA decode. | [ds4.c:17207](ds4.c#L17207) | +| `DS4_CUDA_GREEDY_TOP1` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control greedy top1 in CUDA greedy fast decode. | [ds4.c:56126](ds4.c#L56126) | +| `DS4_CUDA_GREEDY_VEC4` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Enable greedy vec4 fast attention. | [ds4.c:17245](ds4.c#L17245) | +| `DS4_CUDA_GREEDY_VEC4_FALLBACK_LOG` | presence diagnostic flag; default off; any defined value including 0 enables | Control greedy vec4 fallback log in CUDA greedy fast decode. | [ds4.c:55664](ds4.c#L55664) | +| `DS4_CUDA_GREEDY_VEC4_MARGIN` | nonnegative finite float; default 0.25; invalid value warns and uses 0.25; 0 disables margin fallback | Control greedy vec4 margin in CUDA greedy fast decode. | [ds4.c:17283](ds4.c#L17283) | +| `DS4_CUDA_GREEDY_VEC4_MAX_SEGMENT` | integer 0..INT32_MAX; default/invalid 0 (segment cap disabled) | Control greedy vec4 max segment in CUDA greedy fast decode. | [ds4.c:17397](ds4.c#L17397) | +| `DS4_CUDA_GREEDY_VEC4_TRACE` | presence diagnostic flag; default off; any defined value including 0 enables | Control greedy vec4 trace in CUDA greedy fast decode. | [ds4.c:55727](ds4.c#L55727) | | `DS4_CUDA_INDEXED_TWOPASS` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Force the two-pass indexed-attention path instead of the fused heads8 online kernel. | [ds4_cuda.cu:23587](ds4_cuda.cu#L23587) | | `DS4_CUDA_IQ2_XXS_SSD_PREFILL_MMQ_STATS` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Print CUDA IQ2 XXS SSD prefill MMQ counters. | [ds4_cuda.cu:4633](ds4_cuda.cu#L4633) | | `DS4_CUDA_KEEP_MODEL_PAGES` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Keep source model pages resident instead of advising the OS to discard copied pages. | [ds4_cuda.cu:2840](ds4_cuda.cu#L2840) | -| `DS4_CUDA_MIXED_PREFILL_DECODE` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control native mixed prefill/decode scheduling. | [ds4.c:70511](ds4.c#L70511) | -| `DS4_CUDA_MIXED_ROUTED_MAX_PREFILL` | integer rows 0..UINT32_MAX; default/invalid 512 | Set the maximum prefill rows admitted to the mixed routed-MoE path. | [ds4.c:70000](ds4.c#L70000) | -| `DS4_CUDA_MIXED_ROUTED_SCATTER` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Use scattered row handling in mixed routed-MoE execution. | [ds4.c:69461](ds4.c#L69461) | +| `DS4_CUDA_MIXED_PREFILL_DECODE` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control native mixed prefill/decode scheduling. | [ds4.c:70701](ds4.c#L70701) | +| `DS4_CUDA_MIXED_ROUTED_MAX_PREFILL` | integer rows 0..UINT32_MAX; default/invalid 512 | Set the maximum prefill rows admitted to the mixed routed-MoE path. | [ds4.c:70190](ds4.c#L70190) | +| `DS4_CUDA_MIXED_ROUTED_SCATTER` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Use scattered row handling in mixed routed-MoE execution. | [ds4.c:69651](ds4.c#L69651) | | `DS4_CUDA_MMQ` | boolean-ish, default on; any value beginning with 0 disables; quality mode and multi-GPU disable normal MMQ tier (MXFP4 path differs) | Control the vendored CUDA MMQ prefill tier. | [ds4_cuda.cu:1722](ds4_cuda.cu#L1722) | | `DS4_CUDA_MMQ_Q81_PERSISTENT` | strict boolean, default off; accepts 1/on/true/yes and 0/off/false/no in listed lower/upper-case forms; unknown values are off | Reuse a persistent Q8_1 MMQ scratch arena on supported GB10 devices. | [cuda/mmq/ds4_mmq.cu:144](cuda/mmq/ds4_mmq.cu#L144) | | `DS4_CUDA_MMQ_X_MAX` | integer >=8; rounded down to multiple of 8 and only lowers the hardware base; invalid/unset = hardware base | Cap the MMQ X tile-width selector for architecture tuning. | [cuda/mmq/mmq.cuh:127](cuda/mmq/mmq.cuh#L127) | @@ -715,14 +719,14 @@ and **19 tool/wrapper entries**. | `DS4_CUDA_NO_ATTENTION_OUTPUT_F16_CACHE` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the attention output F16 cache CUDA F16 path. | [ds4_cuda.cu:2364](ds4_cuda.cu#L2364) | | `DS4_CUDA_NO_ATTN_A_TOK2` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA attn a tok2 optimization/path. | [ds4_cuda.cu:24024](ds4_cuda.cu#L24024) | | `DS4_CUDA_NO_ATTN_Q_B_F16_CACHE` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the attn q b F16 cache CUDA F16 path. | [ds4_cuda.cu:2367](ds4_cuda.cu#L2367) | -| `DS4_CUDA_NO_COMPRESSOR_PREFILL_BATCH` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA compressor prefill batch optimization/path. | [ds4.c:29717](ds4.c#L29717) | +| `DS4_CUDA_NO_COMPRESSOR_PREFILL_BATCH` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA compressor prefill batch optimization/path. | [ds4.c:29890](ds4.c#L29890) | | `DS4_CUDA_NO_CUBLAS_ATTENTION` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA cuBLAS attention optimization/path. | [ds4_cuda.cu:23067](ds4_cuda.cu#L23067) | | `DS4_CUDA_NO_CUBLAS_ATTENTION_OUTPUT_A` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA cuBLAS attention output a optimization/path. | [ds4_cuda.cu:23934](ds4_cuda.cu#L23934) | | `DS4_CUDA_NO_DECODE_VALUE512` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the 512-thread CUDA decode value/finalize specialization. | [ds4_cuda.cu:374](ds4_cuda.cu#L374) | | `DS4_CUDA_NO_DERIVED_WEIGHTS` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA derived weights optimization/path. | [ds4_cuda.cu:1034](ds4_cuda.cu#L1034) | | `DS4_CUDA_NO_DIRECT_IO` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA direct I/O optimization/path. | [cuda/mmq/ds4_repack.cu:68](cuda/mmq/ds4_repack.cu#L68) | | `DS4_CUDA_NO_DIRECT_Q2_PREFILL` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA direct q2 prefill optimization/path. | [ds4_cuda.cu:395](ds4_cuda.cu#L395) | -| `DS4_CUDA_NO_EXACT_SCORE_SPLIT_DECODE` | value-aware kill switch, default off; exact 0 is off, other nonempty values disable | Disable exact score split decode for exact score-split CUDA decode attention. | [ds4_cuda.cu:13629](ds4_cuda.cu#L13629); [ds4.c:67920](ds4.c#L67920) | +| `DS4_CUDA_NO_EXACT_SCORE_SPLIT_DECODE` | value-aware kill switch, default off; exact 0 is off, other nonempty values disable | Disable exact score split decode for exact score-split CUDA decode attention. | [ds4_cuda.cu:13629](ds4_cuda.cu#L13629); [ds4.c:68110](ds4.c#L68110) | | `DS4_CUDA_NO_EXACT_SCORE_SPLIT_DIM2` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable exact score split dim2 for exact score-split CUDA decode attention. | [ds4_cuda.cu:388](ds4_cuda.cu#L388) | | `DS4_CUDA_NO_F16_CUBLAS_BATCH` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the F16 cuBLAS batch CUDA F16 path. | [ds4_cuda.cu:20854](ds4_cuda.cu#L20854) | | `DS4_CUDA_NO_F16_CUBLAS_ONE` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the F16 cuBLAS one CUDA F16 path. | [ds4_cuda.cu:20851](ds4_cuda.cu#L20851) | @@ -733,13 +737,13 @@ and **19 tool/wrapper entries**. | `DS4_CUDA_NO_F16_SMALL_BATCH` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the F16 small batch CUDA F16 path. | [ds4_cuda.cu:20838](ds4_cuda.cu#L20838) | | `DS4_CUDA_NO_F16_SMALL_OUT` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the F16 small out CUDA F16 path. | [ds4_cuda.cu:20821](ds4_cuda.cu#L20821) | | `DS4_CUDA_NO_FD_CACHE` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA fd cache optimization/path. | [ds4_cuda.cu:1274](ds4_cuda.cu#L1274) | -| `DS4_CUDA_NO_GREEDY_SPLITKV` | value-aware kill switch; default off; nonempty value other than exact 0 disables | Disable greedy splitkv in CUDA greedy fast decode. | [ds4.c:17060](ds4.c#L17060) | -| `DS4_CUDA_NO_GREEDY_SPLITKV_FALLBACK` | value-aware kill switch; default off; nonempty value other than exact 0 disables margin fallback | Disable greedy splitkv fallback in CUDA greedy fast decode. | [ds4.c:17160](ds4.c#L17160) | -| `DS4_CUDA_NO_GREEDY_SPLITKV_PAIR_REPLAY` | value-aware kill switch; default off; nonempty value other than exact 0 disables | Disable greedy splitkv pair replay in CUDA greedy fast decode. | [ds4.c:17188](ds4.c#L17188) | -| `DS4_CUDA_NO_GREEDY_SPLITKV_TOP2` | value-aware kill switch; default off; nonempty value other than exact 0 disables | Disable greedy splitkv top2 in CUDA greedy fast decode. | [ds4.c:17170](ds4.c#L17170) | -| `DS4_CUDA_NO_GREEDY_SPLIT_TOP1` | value-aware kill switch; default off; nonempty value other than exact 0 disables | Disable greedy split top1 in CUDA greedy fast decode. | [ds4.c:17032](ds4.c#L17032) | -| `DS4_CUDA_NO_GREEDY_VEC4` | value-aware kill switch; default off; nonempty value other than exact 0 disables | Disable greedy vec4 in CUDA greedy fast decode. | [ds4.c:17070](ds4.c#L17070) | -| `DS4_CUDA_NO_GREEDY_VEC4_FALLBACK` | value-aware kill switch; default off; nonempty value other than exact 0 disables margin fallback | Disable greedy vec4 fallback in CUDA greedy fast decode. | [ds4.c:17130](ds4.c#L17130) | +| `DS4_CUDA_NO_GREEDY_SPLITKV` | value-aware kill switch; default off; nonempty value other than exact 0 disables | Disable greedy splitkv in CUDA greedy fast decode. | [ds4.c:17233](ds4.c#L17233) | +| `DS4_CUDA_NO_GREEDY_SPLITKV_FALLBACK` | value-aware kill switch; default off; nonempty value other than exact 0 disables margin fallback | Disable greedy splitkv fallback in CUDA greedy fast decode. | [ds4.c:17333](ds4.c#L17333) | +| `DS4_CUDA_NO_GREEDY_SPLITKV_PAIR_REPLAY` | value-aware kill switch; default off; nonempty value other than exact 0 disables | Disable greedy splitkv pair replay in CUDA greedy fast decode. | [ds4.c:17361](ds4.c#L17361) | +| `DS4_CUDA_NO_GREEDY_SPLITKV_TOP2` | value-aware kill switch; default off; nonempty value other than exact 0 disables | Disable greedy splitkv top2 in CUDA greedy fast decode. | [ds4.c:17343](ds4.c#L17343) | +| `DS4_CUDA_NO_GREEDY_SPLIT_TOP1` | value-aware kill switch; default off; nonempty value other than exact 0 disables | Disable greedy split top1 in CUDA greedy fast decode. | [ds4.c:17205](ds4.c#L17205) | +| `DS4_CUDA_NO_GREEDY_VEC4` | value-aware kill switch; default off; nonempty value other than exact 0 disables | Disable greedy vec4 in CUDA greedy fast decode. | [ds4.c:17243](ds4.c#L17243) | +| `DS4_CUDA_NO_GREEDY_VEC4_FALLBACK` | value-aware kill switch; default off; nonempty value other than exact 0 disables margin fallback | Disable greedy vec4 fallback in CUDA greedy fast decode. | [ds4.c:17303](ds4.c#L17303) | | `DS4_CUDA_NO_HC_SPLIT_NORM_SPLIT4096` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the split partial-reduction specialization for one-row, 4096-wide HC normalization. | [ds4_cuda.cu:31826](ds4_cuda.cu#L31826) | | `DS4_CUDA_NO_INDEXED_HEADS8` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA indexed heads8 optimization/path. | [ds4_cuda.cu:23586](ds4_cuda.cu#L23586) | | `DS4_CUDA_NO_INDEXED_TOPK_SORT` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA indexed topk sort optimization/path. | [ds4_cuda.cu:23577](ds4_cuda.cu#L23577) | @@ -752,16 +756,16 @@ and **19 tool/wrapper entries**. | `DS4_CUDA_NO_IQ2_XXS_SSD_PREFILL_MMQ` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Disable the CUDA IQ2 XXS SSD prefill MMQ optimization/path. | [ds4_cuda.cu:4629](ds4_cuda.cu#L4629) | | `DS4_CUDA_NO_MODEL_COPY` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA model copy optimization/path. | [ds4_cuda.cu:6472](ds4_cuda.cu#L6472) | | `DS4_CUDA_NO_MODEL_PREFETCH` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA model prefetch optimization/path. | [ds4_cuda.cu:2739](ds4_cuda.cu#L2739) | -| `DS4_CUDA_NO_MOE_DEDUP` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA MoE dedup optimization/path. | [cuda/mmq/ds4_mmq.cu:5963](cuda/mmq/ds4_mmq.cu#L5963) | +| `DS4_CUDA_NO_MOE_DEDUP` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA MoE dedup optimization/path. | [cuda/mmq/ds4_mmq.cu:6145](cuda/mmq/ds4_mmq.cu#L6145) | | `DS4_CUDA_NO_ORDERED_F16_MATMUL` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the ordered F16 matmul CUDA F16 path. | [ds4_cuda.cu:20811](ds4_cuda.cu#L20811) | | `DS4_CUDA_NO_PARALLEL_ROUTER_SELECT` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA parallel router select optimization/path. | [ds4_cuda.cu:24491](ds4_cuda.cu#L24491) | -| `DS4_CUDA_NO_Q4_DENSE_SCRATCH` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q4 dense scratch CUDA Q4 optimization. | [cuda/mmq/ds4_mmq.cu:3779](cuda/mmq/ds4_mmq.cu#L3779) | -| `DS4_CUDA_NO_Q4_GB10_FAST` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the GB10-specific Q4 fast-path family. | [cuda/mmq/ds4_mmq.cu:3701](cuda/mmq/ds4_mmq.cu#L3701) | -| `DS4_CUDA_NO_Q4_GROUPED_ATTN_A` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q4 grouped attn a CUDA Q4 optimization. | [cuda/mmq/ds4_mmq.cu:4088](cuda/mmq/ds4_mmq.cu#L4088) | -| `DS4_CUDA_NO_Q4_GROUPED_ATTN_A_BATCH` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q4 grouped attn a batch CUDA Q4 optimization. | [cuda/mmq/ds4_mmq.cu:4098](cuda/mmq/ds4_mmq.cu#L4098) | -| `DS4_CUDA_NO_Q4_K1024_PERSISTENT` | presence kill switch, default off; any defined value including 0 disables | Disable the Q4 K1024 persistent CUDA Q4 optimization. | [cuda/mmq/ds4_mmq.cu:3700](cuda/mmq/ds4_mmq.cu#L3700) | -| `DS4_CUDA_NO_Q8_ALIGNED_DENSE_SCRATCH` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q8 aligned dense scratch CUDA Q8 optimization. | [cuda/mmq/ds4_mmq.cu:5369](cuda/mmq/ds4_mmq.cu#L5369) | -| `DS4_CUDA_NO_Q8_ALIGNED_PERSISTENT` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q8 aligned persistent CUDA Q8 optimization. | [cuda/mmq/ds4_mmq.cu:5201](cuda/mmq/ds4_mmq.cu#L5201) | +| `DS4_CUDA_NO_Q4_DENSE_SCRATCH` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q4 dense scratch CUDA Q4 optimization. | [cuda/mmq/ds4_mmq.cu:3963](cuda/mmq/ds4_mmq.cu#L3963) | +| `DS4_CUDA_NO_Q4_GB10_FAST` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the GB10-specific Q4 fast-path family. | [cuda/mmq/ds4_mmq.cu:3885](cuda/mmq/ds4_mmq.cu#L3885) | +| `DS4_CUDA_NO_Q4_GROUPED_ATTN_A` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q4 grouped attn a CUDA Q4 optimization. | [cuda/mmq/ds4_mmq.cu:4272](cuda/mmq/ds4_mmq.cu#L4272) | +| `DS4_CUDA_NO_Q4_GROUPED_ATTN_A_BATCH` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q4 grouped attn a batch CUDA Q4 optimization. | [cuda/mmq/ds4_mmq.cu:4282](cuda/mmq/ds4_mmq.cu#L4282) | +| `DS4_CUDA_NO_Q4_K1024_PERSISTENT` | presence kill switch, default off; any defined value including 0 disables | Disable the Q4 K1024 persistent CUDA Q4 optimization. | [cuda/mmq/ds4_mmq.cu:3884](cuda/mmq/ds4_mmq.cu#L3884) | +| `DS4_CUDA_NO_Q8_ALIGNED_DENSE_SCRATCH` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q8 aligned dense scratch CUDA Q8 optimization. | [cuda/mmq/ds4_mmq.cu:5553](cuda/mmq/ds4_mmq.cu#L5553) | +| `DS4_CUDA_NO_Q8_ALIGNED_PERSISTENT` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q8 aligned persistent CUDA Q8 optimization. | [cuda/mmq/ds4_mmq.cu:5385](cuda/mmq/ds4_mmq.cu#L5385) | | `DS4_CUDA_NO_Q8_BATCH_EXACT_TOK2` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q8 batch exact tok2 CUDA Q8 optimization. | [ds4_cuda.cu:19852](ds4_cuda.cu#L19852) | | `DS4_CUDA_NO_Q8_BATCH_TOK4` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q8 batch tok4 CUDA Q8 optimization. | [ds4_cuda.cu:19805](ds4_cuda.cu#L19805) | | `DS4_CUDA_NO_Q8_BATCH_TOK8` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q8 batch tok8 CUDA Q8 optimization. | [ds4_cuda.cu:19788](ds4_cuda.cu#L19788) | @@ -774,14 +778,14 @@ and **19 tool/wrapper entries**. | `DS4_CUDA_NO_Q8_MMA` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q8 MMA CUDA Q8 optimization. | [ds4_cuda.cu:10781](ds4_cuda.cu#L10781) | | `DS4_CUDA_NO_Q8_PAIR_BATCH_EXACT` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q8 pair batch exact CUDA Q8 optimization. | [ds4_cuda.cu:20302](ds4_cuda.cu#L20302) | | `DS4_CUDA_NO_Q8_PAIR_BATCH_EXACT_TOK2` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q8 pair batch exact tok2 CUDA Q8 optimization. | [ds4_cuda.cu:20305](ds4_cuda.cu#L20305) | -| `DS4_CUDA_NO_QKV_KV_ROPE_FUSE` | value-aware kill switch; default off; nonempty value other than exact 0 disables | Disable the CUDA QKV KV rope fuse optimization/path. | [ds4.c:17253](ds4.c#L17253) | -| `DS4_CUDA_NO_QKV_PAIR` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA QKV pair optimization/path. | [ds4.c:17389](ds4.c#L17389) | -| `DS4_CUDA_NO_SCORE_TILE` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA score tile optimization/path. | [ds4_cuda.cu:13734](ds4_cuda.cu#L13734); [ds4.c:67932](ds4.c#L67932) | +| `DS4_CUDA_NO_QKV_KV_ROPE_FUSE` | value-aware kill switch; default off; nonempty value other than exact 0 disables | Disable the CUDA QKV KV rope fuse optimization/path. | [ds4.c:17426](ds4.c#L17426) | +| `DS4_CUDA_NO_QKV_PAIR` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA QKV pair optimization/path. | [ds4.c:17562](ds4.c#L17562) | +| `DS4_CUDA_NO_SCORE_TILE` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA score tile optimization/path. | [ds4_cuda.cu:13734](ds4_cuda.cu#L13734); [ds4.c:68122](ds4.c#L68122) | | `DS4_CUDA_NO_SETDEVICE_CACHE` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the cached current-tier shortcut and call cudaSetDevice for every tier selection. | [ds4_cuda.cu:377](ds4_cuda.cu#L377) | | `DS4_CUDA_NO_SPLITKV_DECODE` | value-aware kill switch, default off; exact 0/empty is off, other nonempty values disable | Disable splitkv decode in CUDA split-KV attention/speculation. | [ds4_cuda.cu:2211](ds4_cuda.cu#L2211) | -| `DS4_CUDA_NO_SPLITKV_SPEC` | value-aware kill switch; default off; nonempty value other than exact 0 disables | Disable splitkv spec in CUDA split-KV attention/speculation. | [ds4.c:17080](ds4.c#L17080) | -| `DS4_CUDA_NO_SPLITKV_SPEC_BATCH_VERIFY` | value-aware kill switch; default off; nonempty value other than exact 0 disables | Disable splitkv spec batch verify in CUDA split-KV attention/speculation. | [ds4.c:17100](ds4.c#L17100) | -| `DS4_CUDA_NO_SPLITKV_SPEC_TOPONLY_ROW0` | value-aware kill switch; default off; nonempty value other than exact 0 disables | Disable splitkv spec toponly row0 in CUDA split-KV attention/speculation. | [ds4.c:17090](ds4.c#L17090) | +| `DS4_CUDA_NO_SPLITKV_SPEC` | value-aware kill switch; default off; nonempty value other than exact 0 disables | Disable splitkv spec in CUDA split-KV attention/speculation. | [ds4.c:17253](ds4.c#L17253) | +| `DS4_CUDA_NO_SPLITKV_SPEC_BATCH_VERIFY` | value-aware kill switch; default off; nonempty value other than exact 0 disables | Disable splitkv spec batch verify in CUDA split-KV attention/speculation. | [ds4.c:17273](ds4.c#L17273) | +| `DS4_CUDA_NO_SPLITKV_SPEC_TOPONLY_ROW0` | value-aware kill switch; default off; nonempty value other than exact 0 disables | Disable splitkv spec toponly row0 in CUDA split-KV attention/speculation. | [ds4.c:17263](ds4.c#L17263) | | `DS4_CUDA_NO_STREAMING_EXPERT_PERSISTENT_CACHE` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Disable streaming expert persistent cache in CUDA SSD streaming. | [ds4_cuda.cu:4115](ds4_cuda.cu#L4115) | | `DS4_CUDA_NO_STREAMING_SELECTED_BATCH_IO` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Disable streaming selected batch I/O in CUDA SSD streaming. | [ds4_cuda.cu:4736](ds4_cuda.cu#L4736) | | `DS4_CUDA_NO_STREAMING_SELECTED_EVENT_PIPELINE` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Disable streaming selected event pipeline in CUDA SSD streaming. | [ds4_cuda.cu:4868](ds4_cuda.cu#L4868) | @@ -793,23 +797,23 @@ and **19 tool/wrapper entries**. | `DS4_CUDA_NO_TOPK8192` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the topk8192 CUDA indexer kernel/path. | [ds4_cuda.cu:19335](ds4_cuda.cu#L19335) | | `DS4_CUDA_NO_TOPK_CHUNKED` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the topk chunked CUDA indexer kernel/path. | [ds4_cuda.cu:19374](ds4_cuda.cu#L19374) | | `DS4_CUDA_NO_TOPK_STREAM` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the topk stream CUDA indexer kernel/path. | [ds4_cuda.cu:19366](ds4_cuda.cu#L19366) | -| `DS4_CUDA_NO_TP_ATTN_OUT_HC_FUSE` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA TP attn out HC fuse optimization/path. | [ds4.c:17392](ds4.c#L17392) | -| `DS4_CUDA_NO_VERIFY_DECODE2_SPLIT_TOP1` | value-aware kill switch; default off; nonempty value other than exact 0 disables | Disable the CUDA verify decode2 split top1 optimization/path. | [ds4.c:17050](ds4.c#L17050) | +| `DS4_CUDA_NO_TP_ATTN_OUT_HC_FUSE` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA TP attn out HC fuse optimization/path. | [ds4.c:17565](ds4.c#L17565) | +| `DS4_CUDA_NO_VERIFY_DECODE2_SPLIT_TOP1` | value-aware kill switch; default off; nonempty value other than exact 0 disables | Disable the CUDA verify decode2 split top1 optimization/path. | [ds4.c:17223](ds4.c#L17223) | | `DS4_CUDA_NO_WARP_ROUTER_SELECT` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA warp router select optimization/path. | [ds4_cuda.cu:24490](ds4_cuda.cu#L24490) | | `DS4_CUDA_NO_WINDOW_ATTENTION` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA window attention optimization/path. | [ds4_cuda.cu:23050](ds4_cuda.cu#L23050) | | `DS4_CUDA_NSYS_PREFILL_START_POS` | nonempty-string flag, default off; any nonempty value enables MMQ NVTX ranges (the value is not parsed as a position) | Enable MMQ NVTX annotations intended for Nsight Systems prefill capture. | [cuda/mmq/ds4_mmq.cu:48](cuda/mmq/ds4_mmq.cu#L48) | | `DS4_CUDA_NVTX` | strict flag, default off; only exact value 1 enables (a nonempty NSYS variable also enables ranges) | Enable NVTX ranges around MMQ work. | [cuda/mmq/ds4_mmq.cu:47](cuda/mmq/ds4_mmq.cu#L47) | -| `DS4_CUDA_OUTPUT_FUSED_TOP1` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Fuse output projection with top-1 selection in greedy decode. | [ds4.c:17042](ds4.c#L17042) | -| `DS4_CUDA_PREFILL_PIPELINE` | boolean, default follows CUDA TP decode; nonempty exact 0 disables, any other nonempty value enables | Control the CUDA multi-tier prefill pipeline. | [ds4.c:17281](ds4.c#L17281) | -| `DS4_CUDA_PREFILL_PIPELINE_MB` | positive integer rows; default/invalid 512 | Set prefill-pipeline microbatch rows. | [ds4.c:17296](ds4.c#L17296) | -| `DS4_CUDA_PREFILL_PIPELINE_Q8_CACHE` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Keep selective Q8 caches enabled while running the prefill pipeline. | [ds4.c:17291](ds4.c#L17291) | -| `DS4_CUDA_PREFILL_PIPELINE_SEQUENTIAL` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Execute prefill pipeline stages sequentially for diagnosis. | [ds4.c:35335](ds4.c#L35335) | -| `DS4_CUDA_PREFILL_PIPELINE_SYNC_BOUNDARY` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Synchronize CUDA at every prefill pipeline tier boundary. | [ds4.c:35382](ds4.c#L35382) | +| `DS4_CUDA_OUTPUT_FUSED_TOP1` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Fuse output projection with top-1 selection in greedy decode. | [ds4.c:17215](ds4.c#L17215) | +| `DS4_CUDA_PREFILL_PIPELINE` | boolean, default follows CUDA TP decode; nonempty exact 0 disables, any other nonempty value enables | Control the CUDA multi-tier prefill pipeline. | [ds4.c:17454](ds4.c#L17454) | +| `DS4_CUDA_PREFILL_PIPELINE_MB` | positive integer rows; default/invalid 512 | Set prefill-pipeline microbatch rows. | [ds4.c:17469](ds4.c#L17469) | +| `DS4_CUDA_PREFILL_PIPELINE_Q8_CACHE` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Keep selective Q8 caches enabled while running the prefill pipeline. | [ds4.c:17464](ds4.c#L17464) | +| `DS4_CUDA_PREFILL_PIPELINE_SEQUENTIAL` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Execute prefill pipeline stages sequentially for diagnosis. | [ds4.c:35525](ds4.c#L35525) | +| `DS4_CUDA_PREFILL_PIPELINE_SYNC_BOUNDARY` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Synchronize CUDA at every prefill pipeline tier boundary. | [ds4.c:35572](ds4.c#L35572) | | `DS4_CUDA_Q4_ATTN_OUT_HC_ORACLE` | value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on | Compare fused Q4 attention-output/HC expansion with the canonical path and retain canonical output. | [ds4_cuda.cu:1475](ds4_cuda.cu#L1475) | -| `DS4_CUDA_Q4_ATTN_OUT_HC_Q8K_EXPERIMENT` | value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on | Enable the experimental Q8_K-based Q4 attention-output/HC fusion. | [ds4_cuda.cu:37357](ds4_cuda.cu#L37357) | +| `DS4_CUDA_Q4_ATTN_OUT_HC_Q8K_EXPERIMENT` | value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on | Enable the experimental Q8_K-based Q4 attention-output/HC fusion. | [ds4_cuda.cu:37377](ds4_cuda.cu#L37377) | | `DS4_CUDA_Q4_GROUPED_ATTN_A_ORACLE` | value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on | Compare grouped attention-A against the canonical per-group result. | [ds4_cuda.cu:1477](ds4_cuda.cu#L1477) | -| `DS4_CUDA_Q4_K1024_PERSISTENT_ORACLE` | value-aware flag, default off; nonempty value other than exact 0 enables and implies candidate admission | Bitwise-compare the exact-shape persistent Q4 K1024 kernel with canonical MMVQ and retain canonical output. | [cuda/mmq/ds4_mmq.cu:3534](cuda/mmq/ds4_mmq.cu#L3534) | -| `DS4_CUDA_Q4_K1024_PERSISTENT_STATS` | value-aware flag, default off; nonempty value other than exact 0 enables | Print exact-shape persistent Q4 K1024 dispatch counters at exit. | [cuda/mmq/ds4_mmq.cu:3533](cuda/mmq/ds4_mmq.cu#L3533) | +| `DS4_CUDA_Q4_K1024_PERSISTENT_ORACLE` | value-aware flag, default off; nonempty value other than exact 0 enables and implies candidate admission | Bitwise-compare the exact-shape persistent Q4 K1024 kernel with canonical MMVQ and retain canonical output. | [cuda/mmq/ds4_mmq.cu:3718](cuda/mmq/ds4_mmq.cu#L3718) | +| `DS4_CUDA_Q4_K1024_PERSISTENT_STATS` | value-aware flag, default off; nonempty value other than exact 0 enables | Print exact-shape persistent Q4 K1024 dispatch counters at exit. | [cuda/mmq/ds4_mmq.cu:3717](cuda/mmq/ds4_mmq.cu#L3717) | | `DS4_CUDA_Q8_F16_ALL` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Control the Q8 F16 all CUDA quantized-matmul/cache optimization. | [ds4_cuda.cu:2358](ds4_cuda.cu#L2358) | | `DS4_CUDA_Q8_F16_CACHE_MB` | unsigned integer MiB, full-string parse; default unlimited; 0 disables this cache | Limit the selective Q8-to-F16 derived-weight cache. | [ds4_cuda.cu:2218](ds4_cuda.cu#L2218) | | `DS4_CUDA_Q8_F16_CACHE_RESERVE_MB` | unsigned integer MiB, full-string parse; default is VRAM-dependent (>=112 GiB: 512; >=40 GiB: max(768,1%); smaller: max(4096,5%)) | Reserve free VRAM when growing the selective Q8-to-F16 cache. | [ds4_cuda.cu:2224](ds4_cuda.cu#L2224) | @@ -821,75 +825,75 @@ and **19 tool/wrapper entries**. | `DS4_CUDA_Q8_HC_EXPAND_STATS` | false-like-aware flag, default off; 0/false/no/off is off, other nonempty values print report | Print Q8 shared-down/HC policy and dispatch counters at exit. | [ds4_cuda.cu:2090](ds4_cuda.cu#L2090) | | `DS4_CUDA_Q8_NO_ALIGNED` | value-aware kill switch, default off; nonempty value other than exact 0 disables aligned Q8 kernels | Disable aligned Q8 CUDA matmul kernels. | [ds4_cuda.cu:1021](ds4_cuda.cu#L1021) | | `DS4_CUDA_Q8_PAIR_BATCH` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Control the Q8 pair batch CUDA quantized-matmul/cache optimization. | [ds4_cuda.cu:20167](ds4_cuda.cu#L20167) | -| `DS4_CUDA_QKV_KV_ROPE_FUSE` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control or tune the CUDA QKV KV rope fuse path. | [ds4.c:17256](ds4.c#L17256) | -| `DS4_CUDA_Q_NORM_ROPE_FUSE` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control or tune the CUDA q norm rope fuse path. | [ds4.c:17245](ds4.c#L17245) | +| `DS4_CUDA_QKV_KV_ROPE_FUSE` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control or tune the CUDA QKV KV rope fuse path. | [ds4.c:17429](ds4.c#L17429) | +| `DS4_CUDA_Q_NORM_ROPE_FUSE` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control or tune the CUDA q norm rope fuse path. | [ds4.c:17418](ds4.c#L17418) | | `DS4_CUDA_REQUIRE_IQ2_XXS_SSD_PREFILL_MMQ` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Require the CUDA IQ2 XXS SSD prefill MMQ path; fail closed when unavailable. | [ds4_cuda.cu:4631](ds4_cuda.cu#L4631) | -| `DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_BATCH` | value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on | Fail if grouped batched attention-A cannot be used. | [ds4_cuda.cu:40178](ds4_cuda.cu#L40178) | -| `DS4_CUDA_REQUIRE_Q4_K1024_PERSISTENT` | presence flag, default off; any defined value including 0 makes ineligible candidate fail closed | Fail when the exact Q4 K1024 persistent candidate is unavailable instead of using MMVQ. | [cuda/mmq/ds4_mmq.cu:3722](cuda/mmq/ds4_mmq.cu#L3722) | +| `DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_BATCH` | value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on | Fail if grouped batched attention-A cannot be used. | [ds4_cuda.cu:40198](ds4_cuda.cu#L40198) | +| `DS4_CUDA_REQUIRE_Q4_K1024_PERSISTENT` | presence flag, default off; any defined value including 0 makes ineligible candidate fail closed | Fail when the exact Q4 K1024 persistent candidate is unavailable instead of using MMVQ. | [cuda/mmq/ds4_mmq.cu:3906](cuda/mmq/ds4_mmq.cu#L3906) | | `DS4_CUDA_REQUIRE_STREAMING_EXPERT_PERSISTENT_CACHE` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Require streaming expert persistent cache in CUDA SSD streaming; fail closed when unavailable. | [ds4_cuda.cu:4117](ds4_cuda.cu#L4117) | | `DS4_CUDA_REQUIRE_STREAMING_SELECTED_BATCH_IO` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Require streaming selected batch I/O in CUDA SSD streaming; fail closed when unavailable. | [ds4_cuda.cu:4738](ds4_cuda.cu#L4738) | | `DS4_CUDA_REQUIRE_STREAMING_SELECTED_EVENT_PIPELINE` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Require streaming selected event pipeline in CUDA SSD streaming; fail closed when unavailable. | [ds4_cuda.cu:4870](ds4_cuda.cu#L4870) | | `DS4_CUDA_SERIAL_F16_MATMUL` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Control the serial F16 matmul CUDA F16 matmul path. | [ds4_cuda.cu:20801](ds4_cuda.cu#L20801) | | `DS4_CUDA_SERIAL_ROUTER` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Control or tune the CUDA serial router path. | [ds4_cuda.cu:20806](ds4_cuda.cu#L20806) | -| `DS4_CUDA_SESSION_BATCH_ATTN_ALIAS` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control the grouped multi-session CUDA attn alias stage. | [ds4.c:69712](ds4.c#L69712) | -| `DS4_CUDA_SESSION_BATCH_ATTN_CORE` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control the grouped multi-session CUDA attn core stage. | [ds4.c:69715](ds4.c#L69715) | -| `DS4_CUDA_SESSION_BATCH_ATTN_POST` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control the grouped multi-session CUDA attn post stage. | [ds4.c:69727](ds4.c#L69727) | -| `DS4_CUDA_SESSION_BATCH_ATTN_PRE` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control the grouped multi-session CUDA attn pre stage. | [ds4.c:69708](ds4.c#L69708) | -| `DS4_CUDA_SESSION_BATCH_FFN_PRE` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control the grouped multi-session CUDA ffn pre stage. | [ds4.c:69704](ds4.c#L69704) | -| `DS4_CUDA_SESSION_BATCH_INTERLEAVE` | boolean, default on; unset/empty/nonzero enables pipeline interleaving; exact 0 disables | Control the grouped multi-session CUDA interleave stage. | [ds4.c:70388](ds4.c#L70388) | -| `DS4_CUDA_SESSION_BATCH_KV_STORE` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control the grouped multi-session CUDA KV store stage. | [ds4.c:69723](ds4.c#L69723) | -| `DS4_CUDA_SESSION_BATCH_MOE` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control the grouped multi-session CUDA MoE stage. | [ds4.c:69696](ds4.c#L69696) | -| `DS4_CUDA_SESSION_BATCH_MOE_COMBINE_ROWS` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control the grouped multi-session CUDA MoE combine rows stage. | [ds4.c:69241](ds4.c#L69241) | -| `DS4_CUDA_SESSION_BATCH_QKV` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control the grouped multi-session CUDA QKV stage. | [ds4.c:69719](ds4.c#L69719) | -| `DS4_CUDA_SESSION_BATCH_SHARED` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control the grouped multi-session CUDA shared stage. | [ds4.c:69700](ds4.c#L69700) | +| `DS4_CUDA_SESSION_BATCH_ATTN_ALIAS` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control the grouped multi-session CUDA attn alias stage. | [ds4.c:69902](ds4.c#L69902) | +| `DS4_CUDA_SESSION_BATCH_ATTN_CORE` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control the grouped multi-session CUDA attn core stage. | [ds4.c:69905](ds4.c#L69905) | +| `DS4_CUDA_SESSION_BATCH_ATTN_POST` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control the grouped multi-session CUDA attn post stage. | [ds4.c:69917](ds4.c#L69917) | +| `DS4_CUDA_SESSION_BATCH_ATTN_PRE` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control the grouped multi-session CUDA attn pre stage. | [ds4.c:69898](ds4.c#L69898) | +| `DS4_CUDA_SESSION_BATCH_FFN_PRE` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control the grouped multi-session CUDA ffn pre stage. | [ds4.c:69894](ds4.c#L69894) | +| `DS4_CUDA_SESSION_BATCH_INTERLEAVE` | boolean, default on; unset/empty/nonzero enables pipeline interleaving; exact 0 disables | Control the grouped multi-session CUDA interleave stage. | [ds4.c:70578](ds4.c#L70578) | +| `DS4_CUDA_SESSION_BATCH_KV_STORE` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control the grouped multi-session CUDA KV store stage. | [ds4.c:69913](ds4.c#L69913) | +| `DS4_CUDA_SESSION_BATCH_MOE` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control the grouped multi-session CUDA MoE stage. | [ds4.c:69649](ds4.c#L69649) | +| `DS4_CUDA_SESSION_BATCH_MOE_COMBINE_ROWS` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control the grouped multi-session CUDA MoE combine rows stage. | [ds4.c:69431](ds4.c#L69431) | +| `DS4_CUDA_SESSION_BATCH_QKV` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control the grouped multi-session CUDA QKV stage. | [ds4.c:69909](ds4.c#L69909) | +| `DS4_CUDA_SESSION_BATCH_SHARED` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control the grouped multi-session CUDA shared stage. | [ds4.c:69890](ds4.c#L69890) | | `DS4_CUDA_SPLITKV_CHUNK` | integer scores/chunk; default 512; clamped 1..512 | Control splitkv chunk in CUDA split-KV attention/speculation. | [ds4_cuda.cu:22568](ds4_cuda.cu#L22568) | -| `DS4_CUDA_SPLITKV_DECODE` | value-aware boolean, default off; exact 0/empty is off, other nonempty values enable; mere presence also excludes one session-batch path | Enable split-KV decode attention. | [ds4_cuda.cu:2213](ds4_cuda.cu#L2213); [ds4.c:67921](ds4.c#L67921) | +| `DS4_CUDA_SPLITKV_DECODE` | value-aware boolean, default off; exact 0/empty is off, other nonempty values enable; mere presence also excludes one session-batch path | Enable split-KV decode attention. | [ds4_cuda.cu:2213](ds4_cuda.cu#L2213); [ds4.c:68111](ds4.c#L68111) | | `DS4_CUDA_SPLITKV_GLOBAL_SOFTMAX` | value-aware opt-in, default off; exact 0/empty is off, other nonempty values enable | Use the global-softmax variant of split-KV attention. | [ds4_cuda.cu:22589](ds4_cuda.cu#L22589) | -| `DS4_CUDA_SPLITKV_MIN_SCORE` | integer score count 0..UINT32_MAX; default 0 when explicitly enabled, otherwise 512; CUDA kernel clamps to 0..8192 | Set the minimum visible-score count for split-KV attention. | [ds4_cuda.cu:22557](ds4_cuda.cu#L22557); [ds4.c:17229](ds4.c#L17229) | +| `DS4_CUDA_SPLITKV_MIN_SCORE` | integer score count 0..UINT32_MAX; default 0 when explicitly enabled, otherwise 512; CUDA kernel clamps to 0..8192 | Set the minimum visible-score count for split-KV attention. | [ds4_cuda.cu:22557](ds4_cuda.cu#L22557); [ds4.c:17402](ds4.c#L17402) | | `DS4_CUDA_SPLITKV_S` | integer exact split count; unset/invalid = automatic; valid value clamped 1..16 | Control splitkv s in CUDA split-KV attention/speculation. | [ds4_cuda.cu:22578](ds4_cuda.cu#L22578) | -| `DS4_CUDA_SPLITKV_SPEC` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Enable split-KV speculative decoding. | [ds4.c:17082](ds4.c#L17082) | -| `DS4_CUDA_SPLITKV_SPEC_BATCH_VERIFY` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Use batched verification for split-KV speculation. | [ds4.c:17102](ds4.c#L17102) | -| `DS4_CUDA_SPLITKV_SPEC_LOG` | presence diagnostic flag; default off; any defined value including 0 enables | Log split-KV speculative-decode admission and fallback decisions. | [ds4.c:55622](ds4.c#L55622) | -| `DS4_CUDA_SPLITKV_SPEC_TIMING` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Print timing for split-KV speculative-decode stages. | [ds4.c:55661](ds4.c#L55661) | -| `DS4_CUDA_SPLITKV_SPEC_TOPONLY_ROW0` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Compute only the top result for row zero in split-KV speculation. | [ds4.c:17092](ds4.c#L17092) | +| `DS4_CUDA_SPLITKV_SPEC` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Enable split-KV speculative decoding. | [ds4.c:17255](ds4.c#L17255) | +| `DS4_CUDA_SPLITKV_SPEC_BATCH_VERIFY` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Use batched verification for split-KV speculation. | [ds4.c:17275](ds4.c#L17275) | +| `DS4_CUDA_SPLITKV_SPEC_LOG` | presence diagnostic flag; default off; any defined value including 0 enables | Log split-KV speculative-decode admission and fallback decisions. | [ds4.c:55812](ds4.c#L55812) | +| `DS4_CUDA_SPLITKV_SPEC_TIMING` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Print timing for split-KV speculative-decode stages. | [ds4.c:55851](ds4.c#L55851) | +| `DS4_CUDA_SPLITKV_SPEC_TOPONLY_ROW0` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Compute only the top result for row zero in split-KV speculation. | [ds4.c:17265](ds4.c#L17265) | | `DS4_CUDA_SPLITKV_S_FLOOR` | integer split count; default 4; clamped 1..16 | Control splitkv s floor in CUDA split-KV attention/speculation. | [ds4_cuda.cu:22571](ds4_cuda.cu#L22571) | | `DS4_CUDA_SPLITKV_S_MAX` | integer split count; default 16; clamped 1..16 | Control splitkv s max in CUDA split-KV attention/speculation. | [ds4_cuda.cu:22574](ds4_cuda.cu#L22574) | -| `DS4_CUDA_STREAMING_EXPERT_CACHE_PROFILE` | presence diagnostic flag; default off; any defined value including 0 enables | Profile CUDA SSD-streaming streaming expert cache. | [ds4.c:21676](ds4.c#L21676) | +| `DS4_CUDA_STREAMING_EXPERT_CACHE_PROFILE` | presence diagnostic flag; default off; any defined value including 0 enables | Profile CUDA SSD-streaming streaming expert cache. | [ds4.c:21849](ds4.c#L21849) | | `DS4_CUDA_STREAMING_EXPERT_PERSISTENT_CACHE_ORACLE` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Run the diagnostic oracle for CUDA SSD-streaming streaming expert persistent cache. | [ds4_cuda.cu:4121](ds4_cuda.cu#L4121) | | `DS4_CUDA_STREAMING_EXPERT_PERSISTENT_CACHE_STATS` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Print counters for CUDA SSD-streaming streaming expert persistent cache. | [ds4_cuda.cu:4119](ds4_cuda.cu#L4119) | -| `DS4_CUDA_STREAMING_PREFILL_BATCH_SELECTED_PROFILE` | presence diagnostic flag; default off; any defined value including 0 enables | Profile CUDA SSD-streaming streaming prefill batch selected. | [ds4.c:21759](ds4.c#L21759) | +| `DS4_CUDA_STREAMING_PREFILL_BATCH_SELECTED_PROFILE` | presence diagnostic flag; default off; any defined value including 0 enables | Profile CUDA SSD-streaming streaming prefill batch selected. | [ds4.c:21932](ds4.c#L21932) | | `DS4_CUDA_STREAMING_SELECTED_BATCH_IO_ORACLE` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Run the diagnostic oracle for CUDA SSD-streaming streaming selected batch I/O. | [ds4_cuda.cu:4740](ds4_cuda.cu#L4740) | | `DS4_CUDA_STREAMING_SELECTED_BATCH_IO_PROFILE` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Profile CUDA SSD-streaming streaming selected batch I/O. | [ds4_cuda.cu:5711](ds4_cuda.cu#L5711) | | `DS4_CUDA_STREAMING_SELECTED_EVENT_PIPELINE_ORACLE` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Run the diagnostic oracle for CUDA SSD-streaming streaming selected event pipeline. | [ds4_cuda.cu:4872](ds4_cuda.cu#L4872) | | `DS4_CUDA_STREAMING_SELECTED_EVENT_PIPELINE_STATS` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Print counters for CUDA SSD-streaming streaming selected event pipeline. | [ds4_cuda.cu:4874](ds4_cuda.cu#L4874) | | `DS4_CUDA_STRICT_WEIGHT_CACHE` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Fail a weight lookup when cache allocation fails instead of falling back to mapped model memory. | [ds4_cuda.cu:6388](ds4_cuda.cu#L6388) | | `DS4_CUDA_SYNC_XDEV` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Synchronize cross-device CUDA copies for debugging and error localization. | [ds4_cuda.cu:363](ds4_cuda.cu#L363) | -| `DS4_CUDA_TP_ATTN` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel attn execution. | [ds4.c:16862](ds4.c#L16862) | -| `DS4_CUDA_TP_ATTN_CACHE_DUP` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel attn cache dup execution. | [ds4.c:16886](ds4.c#L16886) | -| `DS4_CUDA_TP_ATTN_HEADS` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel attn heads execution. | [ds4.c:16878](ds4.c#L16878) | -| `DS4_CUDA_TP_ATTN_OUT_HC_FUSE` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Control CUDA tensor/expert-parallel attn out HC fuse execution. | [ds4.c:17391](ds4.c#L17391) | -| `DS4_CUDA_TP_ATTN_PEER_READ` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel attn peer read execution. | [ds4.c:16870](ds4.c#L16870) | -| `DS4_CUDA_TP_EP_BALANCED_SHARED_MID` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel EP balanced shared mid execution. | [ds4.c:16943](ds4.c#L16943) | -| `DS4_CUDA_TP_EP_DELAY_REDUCE` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel EP delay reduce execution. | [ds4.c:16918](ds4.c#L16918) | -| `DS4_CUDA_TP_EP_DIRECT_RETURN` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel EP direct return execution. | [ds4.c:16910](ds4.c#L16910) | -| `DS4_CUDA_TP_EP_DUAL_PREQUANT` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel EP dual prequant execution. | [ds4.c:16952](ds4.c#L16952) | -| `DS4_CUDA_TP_EP_FUSED_HC_REDUCE` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel EP fused HC reduce execution. | [ds4.c:16926](ds4.c#L16926) | -| `DS4_CUDA_TP_EP_FUSED_SHARED_MID` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel EP fused shared mid execution. | [ds4.c:16934](ds4.c#L16934) | -| `DS4_CUDA_TP_EP_PACK_EXACT` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel EP pack exact execution. | [ds4.c:16902](ds4.c#L16902) | -| `DS4_CUDA_TP_MOE` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel MoE execution. | [ds4.c:16894](ds4.c#L16894) | -| `DS4_CUDA_TP_MOE_COPY3` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel MoE copy3 execution. | [ds4.c:16976](ds4.c#L16976) | -| `DS4_CUDA_TP_MOE_DELAY_REDUCE` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel MoE delay reduce execution. | [ds4.c:16960](ds4.c#L16960) | -| `DS4_CUDA_TP_MOE_PACK` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel MoE pack execution. | [ds4.c:16968](ds4.c#L16968) | -| `DS4_CUDA_TP_MOE_PEER_READ` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel MoE peer read execution. | [ds4.c:16984](ds4.c#L16984) | -| `DS4_CUDA_TP_MOE_PEER_ROUTER` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel MoE peer router execution. | [ds4.c:16992](ds4.c#L16992) | -| `DS4_CUDA_TP_OUTPUT` | boolean, default on; empty/unset/nonzero enables, exact 0 disables | Control CUDA tensor/expert-parallel output execution. | [ds4.c:51523](ds4.c#L51523) | -| `DS4_CUDA_TP_OUTPUT_WAYS` | integer 2..DS4_MAX_GPUS (16); default 8; invalid value falls back to 2; capped by available GPUs | Set the number of GPU ways used to shard CUDA tensor-parallel output projection. | [ds4.c:58](ds4.c#L58); [ds4.c:51530](ds4.c#L51530) | -| `DS4_CUDA_TP_PREFILL_ATTN_OUTPUT` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel prefill attn output execution. | [ds4.c:17272](ds4.c#L17272) | -| `DS4_CUDA_TP_PREFILL_FFN` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel prefill ffn execution. | [ds4.c:17264](ds4.c#L17264) | -| `DS4_CUDA_TP_Q` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel q execution. | [ds4.c:17016](ds4.c#L17016) | -| `DS4_CUDA_TP_SHARED` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel shared execution. | [ds4.c:17000](ds4.c#L17000) | -| `DS4_CUDA_TP_SHARED_FOLD` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel shared fold execution. | [ds4.c:17008](ds4.c#L17008) | -| `DS4_CUDA_VERIFY_DECODE2_SPLIT_TOP1` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Enable the split top-1 path for two-row verification decode. | [ds4.c:17052](ds4.c#L17052) | +| `DS4_CUDA_TP_ATTN` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel attn execution. | [ds4.c:17035](ds4.c#L17035) | +| `DS4_CUDA_TP_ATTN_CACHE_DUP` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel attn cache dup execution. | [ds4.c:17059](ds4.c#L17059) | +| `DS4_CUDA_TP_ATTN_HEADS` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel attn heads execution. | [ds4.c:17051](ds4.c#L17051) | +| `DS4_CUDA_TP_ATTN_OUT_HC_FUSE` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Control CUDA tensor/expert-parallel attn out HC fuse execution. | [ds4.c:17564](ds4.c#L17564) | +| `DS4_CUDA_TP_ATTN_PEER_READ` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel attn peer read execution. | [ds4.c:17043](ds4.c#L17043) | +| `DS4_CUDA_TP_EP_BALANCED_SHARED_MID` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel EP balanced shared mid execution. | [ds4.c:17116](ds4.c#L17116) | +| `DS4_CUDA_TP_EP_DELAY_REDUCE` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel EP delay reduce execution. | [ds4.c:17091](ds4.c#L17091) | +| `DS4_CUDA_TP_EP_DIRECT_RETURN` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel EP direct return execution. | [ds4.c:17083](ds4.c#L17083) | +| `DS4_CUDA_TP_EP_DUAL_PREQUANT` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel EP dual prequant execution. | [ds4.c:17125](ds4.c#L17125) | +| `DS4_CUDA_TP_EP_FUSED_HC_REDUCE` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel EP fused HC reduce execution. | [ds4.c:17099](ds4.c#L17099) | +| `DS4_CUDA_TP_EP_FUSED_SHARED_MID` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel EP fused shared mid execution. | [ds4.c:17107](ds4.c#L17107) | +| `DS4_CUDA_TP_EP_PACK_EXACT` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel EP pack exact execution. | [ds4.c:17075](ds4.c#L17075) | +| `DS4_CUDA_TP_MOE` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel MoE execution. | [ds4.c:17067](ds4.c#L17067) | +| `DS4_CUDA_TP_MOE_COPY3` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel MoE copy3 execution. | [ds4.c:17149](ds4.c#L17149) | +| `DS4_CUDA_TP_MOE_DELAY_REDUCE` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel MoE delay reduce execution. | [ds4.c:17133](ds4.c#L17133) | +| `DS4_CUDA_TP_MOE_PACK` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel MoE pack execution. | [ds4.c:17141](ds4.c#L17141) | +| `DS4_CUDA_TP_MOE_PEER_READ` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel MoE peer read execution. | [ds4.c:17157](ds4.c#L17157) | +| `DS4_CUDA_TP_MOE_PEER_ROUTER` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel MoE peer router execution. | [ds4.c:17165](ds4.c#L17165) | +| `DS4_CUDA_TP_OUTPUT` | boolean, default on; empty/unset/nonzero enables, exact 0 disables | Control CUDA tensor/expert-parallel output execution. | [ds4.c:51713](ds4.c#L51713) | +| `DS4_CUDA_TP_OUTPUT_WAYS` | integer 2..DS4_MAX_GPUS (16); default 8; invalid value falls back to 2; capped by available GPUs | Set the number of GPU ways used to shard CUDA tensor-parallel output projection. | [ds4.c:58](ds4.c#L58); [ds4.c:51720](ds4.c#L51720) | +| `DS4_CUDA_TP_PREFILL_ATTN_OUTPUT` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel prefill attn output execution. | [ds4.c:17445](ds4.c#L17445) | +| `DS4_CUDA_TP_PREFILL_FFN` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel prefill ffn execution. | [ds4.c:17437](ds4.c#L17437) | +| `DS4_CUDA_TP_Q` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel q execution. | [ds4.c:17189](ds4.c#L17189) | +| `DS4_CUDA_TP_SHARED` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel shared execution. | [ds4.c:17173](ds4.c#L17173) | +| `DS4_CUDA_TP_SHARED_FOLD` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel shared fold execution. | [ds4.c:17181](ds4.c#L17181) | +| `DS4_CUDA_VERIFY_DECODE2_SPLIT_TOP1` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Enable the split top-1 path for two-row verification decode. | [ds4.c:17225](ds4.c#L17225) | | `DS4_CUDA_WEIGHT_ARENA_CHUNK_MB` | positive integer MiB; default 1792; clamped 256..8192 and raised/aligned when one allocation needs more | Set the CUDA selective-weight arena allocation chunk. | [ds4_cuda.cu:6311](ds4_cuda.cu#L6311) | | `DS4_CUDA_WEIGHT_CACHE` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Force selective CUDA weight caching instead of direct mapped access. | [ds4_cuda.cu:1246](ds4_cuda.cu#L1246) | | `DS4_CUDA_WEIGHT_CACHE_LIMIT_GB` | unsigned integer GiB; default/0 = unlimited; parser accepts a numeric prefix even with trailing text | Limit total CUDA selective-weight cache allocation. | [ds4_cuda.cu:6299](ds4_cuda.cu#L6299) | @@ -905,131 +909,131 @@ and **19 tool/wrapper entries**. | Variable | Accepted value and default | Effect | Source | | --- | --- | --- | --- | -| `DS4_ROCM_DECODE_STAGE_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm decode stage profile. | [ds4.c:18071](ds4.c#L18071) | -| `DS4_ROCM_DECODE_STAGE_PROFILE_LAYER` | layer filter subordinate to DS4_ROCM_DECODE_STAGE_PROFILE; unset or whitespace-only: all layers allowed by the parent flag; otherwise the whitespace-trimmed value must be a complete base-10 strtoul result <= UINT32_MAX equal to the current layer; invalid values match none | Restricts the ROCm decode stage profiler to one layer; it does not enable profiling by itself. | [ds4.c:28943](ds4.c#L28943) | -| `DS4_ROCM_DISABLE_GLM_STREAMING_PREFILL_FULL_LAYER` | integer selector/tuning value; unset or invalid uses internal automatic/default value | Disable/roll back rocm disable glm streaming prefill full layer. | [ds4.c:42519](ds4.c#L42519) | -| `DS4_ROCM_DISABLE_GLM_STREAMING_PREFILL_FULL_LAYER_PREPARE` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable glm streaming prefill full layer prepare. | [ds4.c:42537](ds4.c#L42537) | -| `DS4_ROCM_DISABLE_GLM_STREAMING_PREFILL_SELECTED_ASYNC_LOAD` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable glm streaming prefill selected async load. | [ds4.c:46230](ds4.c#L46230) | -| `DS4_ROCM_DISABLE_GLM_STREAMING_SELECTED_ASYNC_LOAD` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable glm streaming selected async load. | [ds4.c:44138](ds4.c#L44138) | -| `DS4_ROCM_DISABLE_IQ2_SELECTED_EXPERT_VIEWS` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable iq2 selected expert views. | [ds4.c:20906](ds4.c#L20906) | -| `DS4_ROCM_DISABLE_IQ2_STREAM_ADDR_TABLE` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable iq2 stream addr table. | [ds4.c:6425](ds4.c#L6425) | -| `DS4_ROCM_DISABLE_Q4_DENSE_PAIR` | presence rollback; unset leaves opt-in policy unchanged | Disable/roll back rocm disable q4 dense pair. | [rocm/ds4_rocm_q4.cuh:299](rocm/ds4_rocm_q4.cuh#L299) | -| `DS4_ROCM_DISABLE_Q4_GROUPED_ATTN_A` | presence rollback; DISABLE wins over enable/require | Disable/roll back rocm disable q4 grouped attn a. | [rocm/ds4_rocm_q4.cuh:573](rocm/ds4_rocm_q4.cuh#L573) | -| `DS4_ROCM_DISABLE_Q4_PREFILL_TILE8` | presence rollback; TILE8 is default for 9..4096 tokens | Disable/roll back rocm disable q4 prefill tile8. | [rocm/ds4_rocm_q4.cuh:312](rocm/ds4_rocm_q4.cuh#L312) | -| `DS4_ROCM_DISABLE_Q4_SELECTED_EXPERT_VIEWS` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable q4 selected expert views. | [ds4.c:20962](ds4.c#L20962) | +| `DS4_ROCM_DECODE_STAGE_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm decode stage profile. | [ds4.c:18244](ds4.c#L18244) | +| `DS4_ROCM_DECODE_STAGE_PROFILE_LAYER` | layer filter subordinate to DS4_ROCM_DECODE_STAGE_PROFILE; unset or whitespace-only: all layers allowed by the parent flag; otherwise the whitespace-trimmed value must be a complete base-10 strtoul result <= UINT32_MAX equal to the current layer; invalid values match none | Restricts the ROCm decode stage profiler to one layer; it does not enable profiling by itself. | [ds4.c:29116](ds4.c#L29116) | +| `DS4_ROCM_DISABLE_GLM_STREAMING_PREFILL_FULL_LAYER` | integer selector/tuning value; unset or invalid uses internal automatic/default value | Disable/roll back rocm disable glm streaming prefill full layer. | [ds4.c:42709](ds4.c#L42709) | +| `DS4_ROCM_DISABLE_GLM_STREAMING_PREFILL_FULL_LAYER_PREPARE` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable glm streaming prefill full layer prepare. | [ds4.c:42727](ds4.c#L42727) | +| `DS4_ROCM_DISABLE_GLM_STREAMING_PREFILL_SELECTED_ASYNC_LOAD` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable glm streaming prefill selected async load. | [ds4.c:46420](ds4.c#L46420) | +| `DS4_ROCM_DISABLE_GLM_STREAMING_SELECTED_ASYNC_LOAD` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable glm streaming selected async load. | [ds4.c:44328](ds4.c#L44328) | +| `DS4_ROCM_DISABLE_IQ2_SELECTED_EXPERT_VIEWS` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable iq2 selected expert views. | [ds4.c:21079](ds4.c#L21079) | +| `DS4_ROCM_DISABLE_IQ2_STREAM_ADDR_TABLE` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable iq2 stream addr table. | [ds4.c:6548](ds4.c#L6548) | +| `DS4_ROCM_DISABLE_Q4_DENSE_PAIR` | presence rollback; unset leaves opt-in policy unchanged | Disable/roll back rocm disable q4 dense pair. | [rocm/ds4_rocm_q4.cuh:435](rocm/ds4_rocm_q4.cuh#L435) | +| `DS4_ROCM_DISABLE_Q4_GROUPED_ATTN_A` | presence rollback; DISABLE wins over enable/require | Disable/roll back rocm disable q4 grouped attn a. | [rocm/ds4_rocm_q4.cuh:700](rocm/ds4_rocm_q4.cuh#L700) | +| `DS4_ROCM_DISABLE_Q4_PREFILL_TILE8` | presence rollback; TILE8 is default for 9..4096 tokens | Disable/roll back rocm disable q4 prefill tile8. | [rocm/ds4_rocm_q4.cuh:448](rocm/ds4_rocm_q4.cuh#L448) | +| `DS4_ROCM_DISABLE_Q4_SELECTED_EXPERT_VIEWS` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable q4 selected expert views. | [ds4.c:21135](ds4.c#L21135) | | `DS4_ROCM_DISABLE_RESIDENT_IQ2_SORTED` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable resident iq2 sorted. | [rocm/ds4_rocm_moe_launch.cuh:747](rocm/ds4_rocm_moe_launch.cuh#L747) | -| `DS4_ROCM_DISABLE_ROUTED_PAIR_SWIGLU_FUSION` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable routed pair swiglu fusion. | [ds4.c:18355](ds4.c#L18355) | -| `DS4_ROCM_DISABLE_STREAMING_COLD_DECODE_PREFILL` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming cold decode prefill. | [ds4.c:31816](ds4.c#L31816) | -| `DS4_ROCM_DISABLE_STREAMING_DECODE_PREFILL` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming decode prefill. | [ds4.c:31765](ds4.c#L31765) | -| `DS4_ROCM_DISABLE_STREAMING_EXPERT_ADDR_TABLE` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming expert addr table. | [ds4.c:18351](ds4.c#L18351) | -| `DS4_ROCM_DISABLE_STREAMING_EXPERT_HOTLIST` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming expert hotlist. | [ds4.c:21116](ds4.c#L21116) | -| `DS4_ROCM_DISABLE_STREAMING_FULL_EXPERT_ADDR_TABLE` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming full expert addr table. | [ds4.c:18069](ds4.c#L18069) | -| `DS4_ROCM_DISABLE_STREAMING_LAYER_BATCH` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming layer batch. | [ds4.c:18065](ds4.c#L18065) | -| `DS4_ROCM_DISABLE_STREAMING_MADVISE_WILLNEED` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming madvise willneed. | [ds4.c:18038](ds4.c#L18038) | -| `DS4_ROCM_DISABLE_STREAMING_PREFILL_BATCH_SELECTED_ADDR` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill batch selected addr. | [ds4.c:18349](ds4.c#L18349) | -| `DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_MADVISE` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill layer madvise. | [ds4.c:18292](ds4.c#L18292) | -| `DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_PAGEIN` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill layer pagein. | [ds4.c:18260](ds4.c#L18260) | -| `DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_PAGEIN_OVERLAP` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill layer pagein overlap. | [ds4.c:19147](ds4.c#L19147) | -| `DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_PREAD` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill layer pread. | [ds4.c:18280](ds4.c#L18280) | -| `DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_PREPARE` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill layer prepare. | [ds4.c:18272](ds4.c#L18272) | -| `DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_PREPARE_OVERLAP` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill layer prepare overlap. | [ds4.c:19145](ds4.c#L19145) | -| `DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_READAHEAD` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill layer readahead. | [ds4.c:18270](ds4.c#L18270) | -| `DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_ASYNC_LOAD` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill selected async load. | [ds4.c:46233](ds4.c#L46233) | -| `DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_MADVISE` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill selected madvise. | [ds4.c:18250](ds4.c#L18250) | -| `DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_PAGEIN` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill selected pagein. | [ds4.c:18240](ds4.c#L18240) | -| `DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_PROFILE` | presence rollback flag; unset keeps automatic/default path | Collect timing/profile diagnostics for rocm disable streaming prefill selected profile. | [ds4.c:18734](ds4.c#L18734) | -| `DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_READAHEAD` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill selected readahead. | [ds4.c:19786](ds4.c#L19786) | -| `DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_READAHEAD_SHARED` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill selected readahead shared. | [ds4.c:19796](ds4.c#L19796) | -| `DS4_ROCM_DISABLE_STREAMING_READAHEAD` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming readahead. | [ds4.c:18031](ds4.c#L18031) | -| `DS4_ROCM_DISABLE_STREAMING_SELECTED_ASYNC_LOAD` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming selected async load. | [ds4.c:44136](ds4.c#L44136) | +| `DS4_ROCM_DISABLE_ROUTED_PAIR_SWIGLU_FUSION` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable routed pair swiglu fusion. | [ds4.c:18528](ds4.c#L18528) | +| `DS4_ROCM_DISABLE_STREAMING_COLD_DECODE_PREFILL` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming cold decode prefill. | [ds4.c:32006](ds4.c#L32006) | +| `DS4_ROCM_DISABLE_STREAMING_DECODE_PREFILL` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming decode prefill. | [ds4.c:31955](ds4.c#L31955) | +| `DS4_ROCM_DISABLE_STREAMING_EXPERT_ADDR_TABLE` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming expert addr table. | [ds4.c:18524](ds4.c#L18524) | +| `DS4_ROCM_DISABLE_STREAMING_EXPERT_HOTLIST` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming expert hotlist. | [ds4.c:21289](ds4.c#L21289) | +| `DS4_ROCM_DISABLE_STREAMING_FULL_EXPERT_ADDR_TABLE` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming full expert addr table. | [ds4.c:18242](ds4.c#L18242) | +| `DS4_ROCM_DISABLE_STREAMING_LAYER_BATCH` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming layer batch. | [ds4.c:18238](ds4.c#L18238) | +| `DS4_ROCM_DISABLE_STREAMING_MADVISE_WILLNEED` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming madvise willneed. | [ds4.c:18211](ds4.c#L18211) | +| `DS4_ROCM_DISABLE_STREAMING_PREFILL_BATCH_SELECTED_ADDR` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill batch selected addr. | [ds4.c:18522](ds4.c#L18522) | +| `DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_MADVISE` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill layer madvise. | [ds4.c:18465](ds4.c#L18465) | +| `DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_PAGEIN` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill layer pagein. | [ds4.c:18433](ds4.c#L18433) | +| `DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_PAGEIN_OVERLAP` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill layer pagein overlap. | [ds4.c:19320](ds4.c#L19320) | +| `DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_PREAD` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill layer pread. | [ds4.c:18453](ds4.c#L18453) | +| `DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_PREPARE` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill layer prepare. | [ds4.c:18445](ds4.c#L18445) | +| `DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_PREPARE_OVERLAP` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill layer prepare overlap. | [ds4.c:19318](ds4.c#L19318) | +| `DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_READAHEAD` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill layer readahead. | [ds4.c:18443](ds4.c#L18443) | +| `DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_ASYNC_LOAD` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill selected async load. | [ds4.c:46423](ds4.c#L46423) | +| `DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_MADVISE` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill selected madvise. | [ds4.c:18423](ds4.c#L18423) | +| `DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_PAGEIN` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill selected pagein. | [ds4.c:18413](ds4.c#L18413) | +| `DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_PROFILE` | presence rollback flag; unset keeps automatic/default path | Collect timing/profile diagnostics for rocm disable streaming prefill selected profile. | [ds4.c:18907](ds4.c#L18907) | +| `DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_READAHEAD` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill selected readahead. | [ds4.c:19959](ds4.c#L19959) | +| `DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_READAHEAD_SHARED` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill selected readahead shared. | [ds4.c:19969](ds4.c#L19969) | +| `DS4_ROCM_DISABLE_STREAMING_READAHEAD` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming readahead. | [ds4.c:18204](ds4.c#L18204) | +| `DS4_ROCM_DISABLE_STREAMING_SELECTED_ASYNC_LOAD` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming selected async load. | [ds4.c:44326](ds4.c#L44326) | | `DS4_ROCM_DISABLE_STREAMING_SPLIT_SELECTED` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming split selected. | [rocm/ds4_rocm_moe_launch.cuh:661](rocm/ds4_rocm_moe_launch.cuh#L661) | -| `DS4_ROCM_DISABLE_STREAMING_STATIC_DECODE_MAP` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming static decode map. | [ds4.c:18043](ds4.c#L18043) | -| `DS4_ROCM_DISABLE_STREAMING_STATIC_MAP_STATE_CACHE` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming static map state cache. | [ds4.c:18056](ds4.c#L18056) | +| `DS4_ROCM_DISABLE_STREAMING_STATIC_DECODE_MAP` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming static decode map. | [ds4.c:18216](ds4.c#L18216) | +| `DS4_ROCM_DISABLE_STREAMING_STATIC_MAP_STATE_CACHE` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming static map state cache. | [ds4.c:18229](ds4.c#L18229) | | `DS4_ROCM_DSV4_PREQUANT_DECODE` | sampled once; unset: enabled; present empty or exact 0: disabled; every other present value: enabled; quality mode and GLM models force it off regardless | ROCm DeepSeek-V4 decode: quantizes one-token F32 activations to Q8 once and selects the prequantized Q8_0/DP4A projection kernels instead of the full-F32 activation paths. | [rocm/ds4_rocm_runtime.cuh:4775](rocm/ds4_rocm_runtime.cuh#L4775) | -| `DS4_ROCM_ENABLE_Q4_DENSE_PAIR` | presence opt-in; unset=off; DISABLE takes precedence | Enable rocm enable q4 dense pair. | [rocm/ds4_rocm_q4.cuh:298](rocm/ds4_rocm_q4.cuh#L298) | -| `DS4_ROCM_ENABLE_Q4_GROUPED_ATTN_A` | presence opt-in; unset=off unless REQUIRE; DISABLE wins | Enable rocm enable q4 grouped attn a. | [rocm/ds4_rocm_q4.cuh:577](rocm/ds4_rocm_q4.cuh#L577) | -| `DS4_ROCM_ENABLE_STREAMING_FULL_EXPERT_ADDR_TABLE` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming full expert addr table. | [ds4.c:18067](ds4.c#L18067) | -| `DS4_ROCM_ENABLE_STREAMING_MADVISE_WILLNEED` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming madvise willneed. | [ds4.c:18036](ds4.c#L18036) | -| `DS4_ROCM_ENABLE_STREAMING_PREFILL_BATCH_SELECTED_ADDR` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming prefill batch selected addr. | [ds4.c:18396](ds4.c#L18396) | -| `DS4_ROCM_ENABLE_STREAMING_PREFILL_CACHE_SEED` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming prefill cache seed. | [ds4.c:21085](ds4.c#L21085) | -| `DS4_ROCM_ENABLE_STREAMING_PREFILL_LAYER_PAGEIN` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming prefill layer pagein. | [ds4.c:18258](ds4.c#L18258) | -| `DS4_ROCM_ENABLE_STREAMING_PREFILL_LAYER_READAHEAD` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming prefill layer readahead. | [ds4.c:18268](ds4.c#L18268) | -| `DS4_ROCM_ENABLE_STREAMING_PREFILL_SELECTED_MADVISE` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming prefill selected madvise. | [ds4.c:18248](ds4.c#L18248) | -| `DS4_ROCM_ENABLE_STREAMING_PREFILL_SELECTED_PAGEIN` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming prefill selected pagein. | [ds4.c:18238](ds4.c#L18238) | -| `DS4_ROCM_ENABLE_STREAMING_PREFILL_SELECTED_READAHEAD` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming prefill selected readahead. | [ds4.c:19782](ds4.c#L19782) | -| `DS4_ROCM_ENABLE_STREAMING_PREFILL_SELECTED_READAHEAD_SHARED` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming prefill selected readahead shared. | [ds4.c:19784](ds4.c#L19784) | -| `DS4_ROCM_ENABLE_STREAMING_READAHEAD` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming readahead. | [ds4.c:18029](ds4.c#L18029) | -| `DS4_ROCM_ENABLE_STREAMING_STATIC_DECODE_MAP` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming static decode map. | [ds4.c:18048](ds4.c#L18048) | +| `DS4_ROCM_ENABLE_Q4_DENSE_PAIR` | presence opt-in; unset=off; DISABLE takes precedence | Enable rocm enable q4 dense pair. | [rocm/ds4_rocm_q4.cuh:434](rocm/ds4_rocm_q4.cuh#L434) | +| `DS4_ROCM_ENABLE_Q4_GROUPED_ATTN_A` | presence opt-in; unset=off unless REQUIRE; DISABLE wins | Enable rocm enable q4 grouped attn a. | [rocm/ds4_rocm_q4.cuh:704](rocm/ds4_rocm_q4.cuh#L704) | +| `DS4_ROCM_ENABLE_STREAMING_FULL_EXPERT_ADDR_TABLE` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming full expert addr table. | [ds4.c:18240](ds4.c#L18240) | +| `DS4_ROCM_ENABLE_STREAMING_MADVISE_WILLNEED` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming madvise willneed. | [ds4.c:18209](ds4.c#L18209) | +| `DS4_ROCM_ENABLE_STREAMING_PREFILL_BATCH_SELECTED_ADDR` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming prefill batch selected addr. | [ds4.c:18569](ds4.c#L18569) | +| `DS4_ROCM_ENABLE_STREAMING_PREFILL_CACHE_SEED` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming prefill cache seed. | [ds4.c:21258](ds4.c#L21258) | +| `DS4_ROCM_ENABLE_STREAMING_PREFILL_LAYER_PAGEIN` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming prefill layer pagein. | [ds4.c:18431](ds4.c#L18431) | +| `DS4_ROCM_ENABLE_STREAMING_PREFILL_LAYER_READAHEAD` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming prefill layer readahead. | [ds4.c:18441](ds4.c#L18441) | +| `DS4_ROCM_ENABLE_STREAMING_PREFILL_SELECTED_MADVISE` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming prefill selected madvise. | [ds4.c:18421](ds4.c#L18421) | +| `DS4_ROCM_ENABLE_STREAMING_PREFILL_SELECTED_PAGEIN` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming prefill selected pagein. | [ds4.c:18411](ds4.c#L18411) | +| `DS4_ROCM_ENABLE_STREAMING_PREFILL_SELECTED_READAHEAD` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming prefill selected readahead. | [ds4.c:19955](ds4.c#L19955) | +| `DS4_ROCM_ENABLE_STREAMING_PREFILL_SELECTED_READAHEAD_SHARED` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming prefill selected readahead shared. | [ds4.c:19957](ds4.c#L19957) | +| `DS4_ROCM_ENABLE_STREAMING_READAHEAD` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming readahead. | [ds4.c:18202](ds4.c#L18202) | +| `DS4_ROCM_ENABLE_STREAMING_STATIC_DECODE_MAP` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming static decode map. | [ds4.c:18221](ds4.c#L18221) | | `DS4_ROCM_GLM_CAUSAL_ATTN_GEMM` | Enabled by default when unset. Exact "0" or an empty value disables; every other nonempty value enables (including false/off/no), because cuda_env_present only tests nonempty and != "0". Eligibility still requires causal_range && !has_selected; a failed GEMM helper falls through to the scalar attention kernel. | Use FP16 BLAS GEMMs for dense causal GLM indexed prefill; =0 is the correctness/performance rollback to the scalar attention kernel. | [rocm/ds4_rocm_glm.cuh:3283](rocm/ds4_rocm_glm.cuh#L3283) | -| `DS4_ROCM_GLM_DISABLE_STREAMING_EXPERT_CACHE` | Pure presence flag: any defined value, including empty or "0", disables. Unset leaves automatic GLM streaming expert-cache eligibility enabled for supported model/quant/quality/SSD configurations. On ROCm builds DS4_METAL_GLM_DISABLE_STREAMING_EXPERT_CACHE is an accepted fallback alias. | Disable selected/resident streamed-expert cache paths and force generic/full-layer expert handling for GLM SSD streaming. | [ds4.c:21024](ds4.c#L21024) | -| `DS4_ROCM_GLM_DISABLE_STREAMING_SEED_BEFORE_PREFILL` | Pure presence flag: any defined value, including empty or "0", disables. Unset seeds before prefill whenever SSD streaming is active. On ROCm builds DS4_METAL_GLM_DISABLE_STREAMING_SEED_BEFORE_PREFILL is an accepted fallback alias. | Skip the pre-prefill hotlist seed of the streaming expert cache in both one-shot GLM generation and session setup. | [ds4.c:50883](ds4.c#L50883) | -| `DS4_ROCM_GLM_DISABLE_STREAMING_TOKEN_PREFILL` | Pure presence flag: any defined value, including empty or "0", disables. Unset leaves the token-major path eligible only for SSD streaming, non-quality mode, a nonempty batch fitting full attention, and n_tokens <= the configured nonzero maximum. DS4_METAL_GLM_DISABLE_STREAMING_TOKEN_PREFILL and generic DS4_GLM_DISABLE_STREAMING_TOKEN_PREFILL are also accepted presence aliases. | Roll back GLM SSD-streaming token-major prefill to the normal prefill implementation. | [ds4.c:49513](ds4.c#L49513) | +| `DS4_ROCM_GLM_DISABLE_STREAMING_EXPERT_CACHE` | Pure presence flag: any defined value, including empty or "0", disables. Unset leaves automatic GLM streaming expert-cache eligibility enabled for supported model/quant/quality/SSD configurations. On ROCm builds DS4_METAL_GLM_DISABLE_STREAMING_EXPERT_CACHE is an accepted fallback alias. | Disable selected/resident streamed-expert cache paths and force generic/full-layer expert handling for GLM SSD streaming. | [ds4.c:21197](ds4.c#L21197) | +| `DS4_ROCM_GLM_DISABLE_STREAMING_SEED_BEFORE_PREFILL` | Pure presence flag: any defined value, including empty or "0", disables. Unset seeds before prefill whenever SSD streaming is active. On ROCm builds DS4_METAL_GLM_DISABLE_STREAMING_SEED_BEFORE_PREFILL is an accepted fallback alias. | Skip the pre-prefill hotlist seed of the streaming expert cache in both one-shot GLM generation and session setup. | [ds4.c:51073](ds4.c#L51073) | +| `DS4_ROCM_GLM_DISABLE_STREAMING_TOKEN_PREFILL` | Pure presence flag: any defined value, including empty or "0", disables. Unset leaves the token-major path eligible only for SSD streaming, non-quality mode, a nonempty batch fitting full attention, and n_tokens <= the configured nonzero maximum. DS4_METAL_GLM_DISABLE_STREAMING_TOKEN_PREFILL and generic DS4_GLM_DISABLE_STREAMING_TOKEN_PREFILL are also accepted presence aliases. | Roll back GLM SSD-streaming token-major prefill to the normal prefill implementation. | [ds4.c:49703](ds4.c#L49703) | | `DS4_ROCM_GLM_GROUPED_QK_LOW` | sampled once; unset: enabled; present empty or exact 0: disabled; every other present value: enabled | Selects the grouped shared-input ROCm kernel for eligible multi-token GLM qk-lowrank projection; disabling uses the per-head/per-token projection kernel. | [rocm/ds4_rocm_runtime.cuh:4800](rocm/ds4_rocm_runtime.cuh#L4800) | | `DS4_ROCM_GLM_GROUPED_VALUE_PROJECT` | sampled once; unset: enabled; present empty or exact 0: disabled; every other present value: enabled | Selects the grouped shared-input ROCm kernel for eligible multi-token GLM value projection; disabling uses the non-grouped batch projection path. | [rocm/ds4_rocm_runtime.cuh:4791](rocm/ds4_rocm_runtime.cuh#L4791) | -| `DS4_ROCM_GLM_LAYER_SLICE_TOKEN_DECODE` | Opt-in truthy parser; unset, empty, "0", false, off, or no (case-insensitive words) are false, every other nonempty value is true. Default off and compiled only for ROCm. | Allow a one-token, pos>0 GLM layer-slice with inter-node input/output hidden buffers to use the optimized resident token graph; without it only the no-hidden-buffer case takes that shortcut. | [ds4.c:43383](ds4.c#L43383) | +| `DS4_ROCM_GLM_LAYER_SLICE_TOKEN_DECODE` | Opt-in truthy parser; unset, empty, "0", false, off, or no (case-insensitive words) are false, every other nonempty value is true. Default off and compiled only for ROCm. | Allow a one-token, pos>0 GLM layer-slice with inter-node input/output hidden buffers to use the optimized resident token graph; without it only the no-hidden-buffer case takes that shortcut. | [ds4.c:43573](ds4.c#L43573) | | `DS4_ROCM_GLM_SELECTED_ATTN_GEMM` | Enabled by default when unset. Exact "0" or an empty value disables; every other nonempty value enables (including false/off/no). Eligibility still requires !causal_range && has_selected; failure/ineligibility falls through to the scalar attention kernel. | Gather per-token selected cache rows into FP16 matrices and use strided-batched BLAS GEMMs for GLM selected indexed prefill; =0 forces the scalar path. | [rocm/ds4_rocm_glm.cuh:3239](rocm/ds4_rocm_glm.cuh#L3239) | | `DS4_ROCM_GLM_SELECTED_ATTN_HEAD_TILE` | Unsigned integer read and cached once; valid values are exactly 1,2,4,8,16,32,64. Unset/empty defaults to 16. A nonnumeric, partially parsed, overflowed, or unsupported value prints a warning and uses 16; the effective tile is min(requested,n_head). | Set how many attention heads each selected-attention GEMM workspace tile processes. | [rocm/ds4_rocm_glm.cuh:2509](rocm/ds4_rocm_glm.cuh#L2509) | | `DS4_ROCM_GLM_SELECTED_ATTN_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm glm selected attn profile. | [rocm/ds4_rocm_glm.cuh:2534](rocm/ds4_rocm_glm.cuh#L2534) | -| `DS4_ROCM_GLM_STREAMING_ASYNC_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm glm streaming async profile. | [ds4.c:44171](ds4.c#L44171) | -| `DS4_ROCM_GLM_STREAMING_DECODE_FULL_LAYER_MAP` | Pure presence flag: any defined value, including empty or "0", forces full mapping. Unset uses automatic mapping: resident layers map fully, eligible expert-cache layers use decode-only/expert mapping, otherwise full mapping. DS4_METAL_GLM_STREAMING_DECODE_FULL_LAYER_MAP and generic DS4_GLM_STREAMING_DECODE_FULL_LAYER_MAP are also accepted presence aliases. | Force every GLM SSD-streaming decode layer through full-layer mapping, bypassing the selected-expert/decode mapping optimization. | [ds4.c:42425](ds4.c#L42425) | -| `DS4_ROCM_GLM_STREAMING_DECODE_SYNC_EACH_LAYER` | ROCm-only primary value; nonempty takes priority over DS4_METAL_GLM_STREAMING_DECODE_SYNC_EACH_LAYER and the generic DS4_GLM_STREAMING_DECODE_SYNC_EACH_LAYER fallback; empty acts as unset; truthy unless exact 0 or case-insensitive false/off/no; with all aliases unset: false | For non-static GLM SSD decode on ROCm, opts into a full command/device synchronization after token mapping and every layer; default keeps ordered work queued across layer mappings, and static-map decode bypasses it. | [ds4.c:49589](ds4.c#L49589) | -| `DS4_ROCM_GLM_STREAMING_GROW_CACHE_AFTER_PREFILL` | Enabled by default when absent. If defined, only a truthy nonempty value enables; empty, "0", false, off, or no disable. Growth also requires SSD streaming plus nonzero base cache and prefill-headroom budgets, and occurs only if the recomputed expert count exceeds the current count. | After successful ROCm GLM prefill, add the released prefill headroom to the dynamic streaming expert-cache byte budget. | [ds4.c:50924](ds4.c#L50924) | -| `DS4_ROCM_GLM_STREAMING_PREFILL_FULL_LAYER` | presence force-on; any presence including empty or 0 enables; unset falls back to the Metal alias and then the automatic token threshold (1024 by default on ROCm); DS4_ROCM_DISABLE_GLM_STREAMING_PREFILL_FULL_LAYER dominates | Forces GLM SSD prefill into full-layer mapping/cache mode even below the automatic large-batch threshold. | [ds4.c:42523](ds4.c#L42523) | -| `DS4_ROCM_GLM_STREAMING_PREFILL_FULL_LAYER_MIN_TOKENS` | Positive uint32 threshold parsed with strtoul; ROCm default is 1024. Missing/empty, no leading number, errno/overflow, zero, or >UINT32_MAX returns 1024. The parser does not require end-of-string, so trailing junk after a valid leading number is accepted. A nonempty ROCm value takes precedence; otherwise DS4_METAL_GLM_STREAMING_PREFILL_FULL_LAYER_MIN_TOKENS is a fallback alias. | Set the automatic token-count crossover for ROCm GLM SSD prefill to load/use full resident expert layers when the layer supports that mode. | [ds4.c:42502](ds4.c#L42502) | -| `DS4_ROCM_GLM_STREAMING_PREFILL_SYNC_EACH_LAYER` | ROCm-only primary value; nonempty takes priority over DS4_METAL_GLM_STREAMING_PREFILL_SYNC_EACH_LAYER and the generic DS4_GLM_STREAMING_PREFILL_SYNC_EACH_LAYER fallback; empty acts as unset; truthy unless exact 0 or case-insensitive false/off/no; with all aliases unset: false for compact prefill; full-layer prefill always returns true | For compact GLM SSD prefill on ROCm, opts into a full command/device synchronization at every layer boundary; default preserves queued work across mappings, while full-layer cache mode always synchronizes. | [ds4.c:42205](ds4.c#L42205) | -| `DS4_ROCM_GLM_STREAMING_TOKEN_PREFILL_MAX` | primary nonempty value, then the Metal alias, then generic DS4_GLM_STREAMING_TOKEN_PREFILL_MAX; parsed by strtoul without requiring full-string consumption; 0 is valid and disables; no digits, ERANGE, or > UINT32_MAX uses the ROCm default 0 | Sets the largest non-quality GLM SSD-prefill chunk eligible for token-major/decode-style execution; ROCm defaults to canonical indexed batch prefill (0 disables token-major mode). | [ds4.c:49492](ds4.c#L49492) | +| `DS4_ROCM_GLM_STREAMING_ASYNC_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm glm streaming async profile. | [ds4.c:44361](ds4.c#L44361) | +| `DS4_ROCM_GLM_STREAMING_DECODE_FULL_LAYER_MAP` | Pure presence flag: any defined value, including empty or "0", forces full mapping. Unset uses automatic mapping: resident layers map fully, eligible expert-cache layers use decode-only/expert mapping, otherwise full mapping. DS4_METAL_GLM_STREAMING_DECODE_FULL_LAYER_MAP and generic DS4_GLM_STREAMING_DECODE_FULL_LAYER_MAP are also accepted presence aliases. | Force every GLM SSD-streaming decode layer through full-layer mapping, bypassing the selected-expert/decode mapping optimization. | [ds4.c:42615](ds4.c#L42615) | +| `DS4_ROCM_GLM_STREAMING_DECODE_SYNC_EACH_LAYER` | ROCm-only primary value; nonempty takes priority over DS4_METAL_GLM_STREAMING_DECODE_SYNC_EACH_LAYER and the generic DS4_GLM_STREAMING_DECODE_SYNC_EACH_LAYER fallback; empty acts as unset; truthy unless exact 0 or case-insensitive false/off/no; with all aliases unset: false | For non-static GLM SSD decode on ROCm, opts into a full command/device synchronization after token mapping and every layer; default keeps ordered work queued across layer mappings, and static-map decode bypasses it. | [ds4.c:49779](ds4.c#L49779) | +| `DS4_ROCM_GLM_STREAMING_GROW_CACHE_AFTER_PREFILL` | Enabled by default when absent. If defined, only a truthy nonempty value enables; empty, "0", false, off, or no disable. Growth also requires SSD streaming plus nonzero base cache and prefill-headroom budgets, and occurs only if the recomputed expert count exceeds the current count. | After successful ROCm GLM prefill, add the released prefill headroom to the dynamic streaming expert-cache byte budget. | [ds4.c:51111](ds4.c#L51111) | +| `DS4_ROCM_GLM_STREAMING_PREFILL_FULL_LAYER` | presence force-on; any presence including empty or 0 enables; unset falls back to the Metal alias and then the automatic token threshold (1024 by default on ROCm); DS4_ROCM_DISABLE_GLM_STREAMING_PREFILL_FULL_LAYER dominates | Forces GLM SSD prefill into full-layer mapping/cache mode even below the automatic large-batch threshold. | [ds4.c:42692](ds4.c#L42692) | +| `DS4_ROCM_GLM_STREAMING_PREFILL_FULL_LAYER_MIN_TOKENS` | Positive uint32 threshold parsed with strtoul; ROCm default is 1024. Missing/empty, no leading number, errno/overflow, zero, or >UINT32_MAX returns 1024. The parser does not require end-of-string, so trailing junk after a valid leading number is accepted. A nonempty ROCm value takes precedence; otherwise DS4_METAL_GLM_STREAMING_PREFILL_FULL_LAYER_MIN_TOKENS is a fallback alias. | Set the automatic token-count crossover for ROCm GLM SSD prefill to load/use full resident expert layers when the layer supports that mode. | [ds4.c:42692](ds4.c#L42692) | +| `DS4_ROCM_GLM_STREAMING_PREFILL_SYNC_EACH_LAYER` | ROCm-only primary value; nonempty takes priority over DS4_METAL_GLM_STREAMING_PREFILL_SYNC_EACH_LAYER and the generic DS4_GLM_STREAMING_PREFILL_SYNC_EACH_LAYER fallback; empty acts as unset; truthy unless exact 0 or case-insensitive false/off/no; with all aliases unset: false for compact prefill; full-layer prefill always returns true | For compact GLM SSD prefill on ROCm, opts into a full command/device synchronization at every layer boundary; default preserves queued work across mappings, while full-layer cache mode always synchronizes. | [ds4.c:42395](ds4.c#L42395) | +| `DS4_ROCM_GLM_STREAMING_TOKEN_PREFILL_MAX` | primary nonempty value, then the Metal alias, then generic DS4_GLM_STREAMING_TOKEN_PREFILL_MAX; parsed by strtoul without requiring full-string consumption; 0 is valid and disables; no digits, ERANGE, or > UINT32_MAX uses the ROCm default 0 | Sets the largest non-quality GLM SSD-prefill chunk eligible for token-major/decode-style execution; ROCm defaults to canonical indexed batch prefill (0 disables token-major mode). | [ds4.c:49682](ds4.c#L49682) | | `DS4_ROCM_GLM_VALUE_PROJECT_WAVE_DECODE` | Enabled by default when unset. Exact "0" or empty disables; every other nonempty value enables (including false/off/no). It applies only when n_tokens==1; =0 or multi-token input uses the generic per-head batch kernel. | Select the validated wave-per-output-row ROCm Q8 GLM value-projection kernel for one-token decode; =0 is the generic-kernel rollback. | [rocm/ds4_rocm_glm.cuh:2019](rocm/ds4_rocm_glm.cuh#L2019) | -| `DS4_ROCM_GRAPH_DUMP_LAYER` | unsigned layer or all; unset=all layers | Filter ROCm graph dumps by layer. | [ds4.c:16689](ds4.c#L16689) | -| `DS4_ROCM_GRAPH_DUMP_NAME` | nonempty substring filter; unset=all tensor names | Filter ROCm graph dumps by tensor/stage name. | [ds4.c:16685](ds4.c#L16685) | +| `DS4_ROCM_GRAPH_DUMP_LAYER` | unsigned layer or all; unset=all layers | Filter ROCm graph dumps by layer. | [ds4.c:16862](ds4.c#L16862) | +| `DS4_ROCM_GRAPH_DUMP_NAME` | nonempty substring filter; unset=all tensor names | Filter ROCm graph dumps by tensor/stage name. | [ds4.c:16858](ds4.c#L16858) | | `DS4_ROCM_GRAPH_DUMP_NONINVASIVE` | truthy value under the shared parser; unset lets dumping select conservative kernels | Keep production ROCm kernel selection while graph dumping. | [rocm/ds4_rocm_runtime.cuh:4822](rocm/ds4_rocm_runtime.cuh#L4822) | -| `DS4_ROCM_GRAPH_DUMP_POS` | unsigned token position; unset=all positions | Filter ROCm graph dumps by position. | [ds4.c:16696](ds4.c#L16696) | +| `DS4_ROCM_GRAPH_DUMP_POS` | unsigned token position; unset=all positions | Filter ROCm graph dumps by position. | [ds4.c:16869](ds4.c#L16869) | | `DS4_ROCM_GRAPH_DUMP_PREFIX` | nonempty output path prefix; unset=off | Enable ROCm intermediate graph/tensor dumps. | [ds4_cuda.cu:397](ds4_cuda.cu#L397) | -| `DS4_ROCM_GRAPH_DUMP_TRACE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Emit trace diagnostics for rocm graph dump trace. | [ds4.c:16726](ds4.c#L16726) | -| `DS4_ROCM_GRAPH_OUTPUT_ROW` | Nonempty string with a leading strtoul-parsable unsigned value < n_tokens selects that zero-based row. Default, empty, unparsable, or out-of-range selects n_tokens-1. Trailing characters are accepted because full consumption/errno are not checked. A nonempty ROCm value takes precedence; otherwise DS4_METAL_GRAPH_OUTPUT_ROW is a fallback alias. | Choose which prefill hidden-state row is sent through the output head to produce logits, primarily for graph/correctness diagnostics. | [ds4.c:35655](ds4.c#L35655) | -| `DS4_ROCM_GRAPH_PREFILL_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm graph prefill profile. | [ds4.c:31848](ds4.c#L31848) | -| `DS4_ROCM_GRAPH_PREFILL_SPLIT_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm graph prefill split profile. | [ds4.c:35578](ds4.c#L35578) | -| `DS4_ROCM_GRAPH_TOKEN_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm graph token profile. | [ds4.c:31532](ds4.c#L31532) | -| `DS4_ROCM_INDEXER_STAGE_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm indexer stage profile. | [ds4.c:29092](ds4.c#L29092) | -| `DS4_ROCM_LAYER_STAGE_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm layer stage profile. | [ds4.c:28931](ds4.c#L28931) | -| `DS4_ROCM_LAYER_STAGE_PROFILE_LAYER` | layer filter subordinate to DS4_ROCM_LAYER_STAGE_PROFILE; unset or whitespace-only: all layers allowed by the parent flag; otherwise the whitespace-trimmed value must be a complete base-10 strtoul result <= UINT32_MAX equal to the current layer; invalid values match none | Restricts the ROCm layer/prefill stage profiler to one layer; it does not enable profiling by itself. | [ds4.c:28932](ds4.c#L28932) | +| `DS4_ROCM_GRAPH_DUMP_TRACE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Emit trace diagnostics for rocm graph dump trace. | [ds4.c:16899](ds4.c#L16899) | +| `DS4_ROCM_GRAPH_OUTPUT_ROW` | Nonempty string with a leading strtoul-parsable unsigned value < n_tokens selects that zero-based row. Default, empty, unparsable, or out-of-range selects n_tokens-1. Trailing characters are accepted because full consumption/errno are not checked. A nonempty ROCm value takes precedence; otherwise DS4_METAL_GRAPH_OUTPUT_ROW is a fallback alias. | Choose which prefill hidden-state row is sent through the output head to produce logits, primarily for graph/correctness diagnostics. | [ds4.c:35845](ds4.c#L35845) | +| `DS4_ROCM_GRAPH_PREFILL_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm graph prefill profile. | [ds4.c:32038](ds4.c#L32038) | +| `DS4_ROCM_GRAPH_PREFILL_SPLIT_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm graph prefill split profile. | [ds4.c:35768](ds4.c#L35768) | +| `DS4_ROCM_GRAPH_TOKEN_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm graph token profile. | [ds4.c:31722](ds4.c#L31722) | +| `DS4_ROCM_INDEXER_STAGE_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm indexer stage profile. | [ds4.c:29265](ds4.c#L29265) | +| `DS4_ROCM_LAYER_STAGE_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm layer stage profile. | [ds4.c:29104](ds4.c#L29104) | +| `DS4_ROCM_LAYER_STAGE_PROFILE_LAYER` | layer filter subordinate to DS4_ROCM_LAYER_STAGE_PROFILE; unset or whitespace-only: all layers allowed by the parent flag; otherwise the whitespace-trimmed value must be a complete base-10 strtoul result <= UINT32_MAX equal to the current layer; invalid values match none | Restricts the ROCm layer/prefill stage profiler to one layer; it does not enable profiling by itself. | [ds4.c:29105](ds4.c#L29105) | | `DS4_ROCM_MOE_DECODE_DOWN_RPB` | sampled once; nonempty value is parsed by strtoul (a numeric prefix is sufficient), cast to uint32_t, and accepted only if 1/2/4/8/16/32; unset/empty/invalid inherits DS4_ROCM_MOE_DECODE_RPB, with defaults quality=8, non-quality SSD=2, resident=1 | Sets output rows (warps) per block for ROCm Q2_K routed-MoE decode down-projection kernels; threads per block are value * 32. | [rocm/ds4_rocm_runtime.cuh:4842](rocm/ds4_rocm_runtime.cuh#L4842) | | `DS4_ROCM_MOE_DECODE_GATE_RPB` | sampled once; nonempty value is parsed by strtoul (a numeric prefix is sufficient), cast to uint32_t, and accepted only if 1/2/4/8/16/32; unset/empty/invalid defaults to 1 in non-quality SSD mode when DS4_ROCM_MOE_DECODE_RPB is unset/empty, otherwise inherits the resolved base RPB | Sets output rows (warps) per block for ROCm Q2_K routed-MoE decode gate/up kernels; threads per block are value * 32. | [rocm/ds4_rocm_runtime.cuh:4836](rocm/ds4_rocm_runtime.cuh#L4836) | | `DS4_ROCM_MOE_DECODE_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm moe decode profile. | [rocm/ds4_rocm_moe_launch.cuh:82](rocm/ds4_rocm_moe_launch.cuh#L82) | | `DS4_ROCM_MOE_DECODE_RPB` | sampled once; nonempty value is parsed by strtoul (a numeric prefix is sufficient), cast to uint32_t, and accepted only if 1/2/4/8/16/32; unset/empty/invalid default: quality=8, non-quality SSD=2, resident=1 | Sets the base ROCm Q2_K decode-MoE rows-per-block value inherited by gate/up and down controls, except the automatic SSD gate specialization defaults to 1 when this variable is unset/empty. | [rocm/ds4_rocm_runtime.cuh:4832](rocm/ds4_rocm_runtime.cuh#L4832) | -| `DS4_ROCM_MOE_WRITE_CLAMPED_ACT` | Pure presence sentinel: any defined value, including empty or "0", is active; DS4_METAL_MOE_WRITE_CLAMPED_ACT is an accepted fallback alias. On ROCm the variable is only consumed as a path-admission veto: it disables selected-expert cache/address-table, selected-slot and CPU-router/fused optimized paths. No ROCm call site parses a clamp amount or directly enables a write-clamped kernel. | Force shared graph selection away from optimizations incompatible with the clamped-intermediate MoE diagnostic; on ROCm this is a compatibility/rollback gate, not itself a clamped-write implementation. | [ds4.c:18353](ds4.c#L18353) | -| `DS4_ROCM_Q4_GROUPED_ATTN_A_STATS` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Print counters for rocm q4 grouped attn a stats. | [rocm/ds4_rocm_q4.cuh:344](rocm/ds4_rocm_q4.cuh#L344) | -| `DS4_ROCM_Q4_PREFILL_TILE8_STATS` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Print counters for rocm q4 prefill tile8 stats. | [rocm/ds4_rocm_q4.cuh:378](rocm/ds4_rocm_q4.cuh#L378) | +| `DS4_ROCM_MOE_WRITE_CLAMPED_ACT` | Pure presence sentinel: any defined value, including empty or "0", is active; DS4_METAL_MOE_WRITE_CLAMPED_ACT is an accepted fallback alias. On ROCm the variable is only consumed as a path-admission veto: it disables selected-expert cache/address-table, selected-slot and CPU-router/fused optimized paths. No ROCm call site parses a clamp amount or directly enables a write-clamped kernel. | Force shared graph selection away from optimizations incompatible with the clamped-intermediate MoE diagnostic; on ROCm this is a compatibility/rollback gate, not itself a clamped-write implementation. | [ds4.c:18526](ds4.c#L18526) | +| `DS4_ROCM_Q4_GROUPED_ATTN_A_STATS` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Print counters for rocm q4 grouped attn a stats. | [rocm/ds4_rocm_q4.cuh:480](rocm/ds4_rocm_q4.cuh#L480) | +| `DS4_ROCM_Q4_PREFILL_TILE8_STATS` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Print counters for rocm q4 prefill tile8 stats. | [rocm/ds4_rocm_q4.cuh:514](rocm/ds4_rocm_q4.cuh#L514) | | `DS4_ROCM_Q8_DECODE_SHAREDX_64K` | sampled once; unset: enabled; present empty or exact 0: disabled; every other present value: enabled; effective only for one-token non-prequant Q8_0 matmul with 8192 < in_dim <= 16384 | Allows the ROCm shared-input Q8 decode kernel to use up to 64 KiB dynamic LDS for wide inputs; an unsupported/failed LDS launch automatically falls back to the regular kernel. | [rocm/ds4_rocm_runtime.cuh:4805](rocm/ds4_rocm_runtime.cuh#L4805) | -| `DS4_ROCM_Q_STAGE_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm q stage profile. | [ds4.c:29096](ds4.c#L29096) | -| `DS4_ROCM_REQUIRE_Q4_GROUPED_ATTN_A` | presence fail-closed assertion; also requests candidate unless disabled | Require rocm require q4 grouped attn a and fail instead of silently falling back. | [rocm/ds4_rocm_q4.cuh:575](rocm/ds4_rocm_q4.cuh#L575) | -| `DS4_ROCM_REQUIRE_Q4_PREFILL_TILE8` | presence fail-closed assertion for eligible TILE8 calls | Require rocm require q4 prefill tile8 and fail instead of silently falling back. | [rocm/ds4_rocm_q4.cuh:316](rocm/ds4_rocm_q4.cuh#L316) | -| `DS4_ROCM_STREAMING_DECODE_PREFILL_MAX` | primary nonempty value over the Metal alias; parsed by strtol when it has a numeric prefix (trailing text is accepted); <= 0 disables, values > UINT32_MAX clamp, no numeric prefix uses automatic default: 64 for Flash with uniform Q4_K/MXFP4 experts, 18 for other Pro/Flash, otherwise 0; the disable flag dominates | Sets the largest short, non-quality SSD-streaming prefill batch routed through the decode-style path instead of canonical layer-major prefill. | [ds4.c:31771](ds4.c#L31771) | -| `DS4_ROCM_STREAMING_EXPERT_AUTO_PRELOAD_CAP` | primary nonempty value over the Metal alias; strict full-string strtoul; valid values > UINT32_MAX clamp, invalid uses 4096, and 0 means no cap (not disabled); when CLI preload is auto/0, unset defaults to cap 4096 except ROCm GLM52, where absent/empty disables automatic preload entirely | Caps the number of hot experts synchronously seeded into the SSD-streaming expert cache in automatic preload mode; an explicit CLI preload count bypasses this cap, and setting this variable opts ROCm GLM52 back into auto preload. | [ds4.c:21281](ds4.c#L21281) | +| `DS4_ROCM_Q_STAGE_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm q stage profile. | [ds4.c:29269](ds4.c#L29269) | +| `DS4_ROCM_REQUIRE_Q4_GROUPED_ATTN_A` | presence fail-closed assertion; also requests candidate unless disabled | Require rocm require q4 grouped attn a and fail instead of silently falling back. | [rocm/ds4_rocm_q4.cuh:702](rocm/ds4_rocm_q4.cuh#L702) | +| `DS4_ROCM_REQUIRE_Q4_PREFILL_TILE8` | presence fail-closed assertion for eligible TILE8 calls | Require rocm require q4 prefill tile8 and fail instead of silently falling back. | [rocm/ds4_rocm_q4.cuh:452](rocm/ds4_rocm_q4.cuh#L452) | +| `DS4_ROCM_STREAMING_DECODE_PREFILL_MAX` | primary nonempty value over the Metal alias; parsed by strtol when it has a numeric prefix (trailing text is accepted); <= 0 disables, values > UINT32_MAX clamp, no numeric prefix uses automatic default: 64 for Flash with uniform Q4_K/MXFP4 experts, 18 for other Pro/Flash, otherwise 0; the disable flag dominates | Sets the largest short, non-quality SSD-streaming prefill batch routed through the decode-style path instead of canonical layer-major prefill. | [ds4.c:31961](ds4.c#L31961) | +| `DS4_ROCM_STREAMING_EXPERT_AUTO_PRELOAD_CAP` | primary nonempty value over the Metal alias; strict full-string strtoul; valid values > UINT32_MAX clamp, invalid uses 4096, and 0 means no cap (not disabled); when CLI preload is auto/0, unset defaults to cap 4096 except ROCm GLM52, where absent/empty disables automatic preload entirely | Caps the number of hot experts synchronously seeded into the SSD-streaming expert cache in automatic preload mode; an explicit CLI preload count bypasses this cap, and setting this variable opts ROCm GLM52 back into auto preload. | [ds4.c:21454](ds4.c#L21454) | | `DS4_ROCM_STREAMING_EXPERT_CACHE_VERBOSE` | presence flag; unset=off | Print verbose ROCm streaming expert-cache seed/load diagnostics. | [rocm/ds4_rocm_runtime.cuh:2904](rocm/ds4_rocm_runtime.cuh#L2904) | -| `DS4_ROCM_STREAMING_EXPERT_HOTLIST` | Nonempty filesystem path; a nonempty ROCm value takes precedence, otherwise DS4_METAL_STREAMING_EXPERT_HOTLIST is a fallback. The file contains whitespace-separated layer expert hits rows; blank/comment lines are ignored, zero-hit rows skipped, malformed/open/read errors fail seeding. Unset/empty uses the built-in Pro/Flash/GLM52 hotlist. Effective only when non-cold SSD hotlist seeding is enabled and cache/preload budget is nonzero. | Select a custom ranked expert hotlist used to preseed the streaming resident expert cache before decode. | [ds4.c:21321](ds4.c#L21321) | -| `DS4_ROCM_STREAMING_EXPERT_HOTLIST_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm streaming expert hotlist profile. | [ds4.c:32054](ds4.c#L32054) | -| `DS4_ROCM_STREAMING_MAP_TRACE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Emit trace diagnostics for rocm streaming map trace. | [ds4.c:42453](ds4.c#L42453) | -| `DS4_ROCM_STREAMING_PREFILL_BATCH_SELECTED_ADDR_MAX` | primary nonempty value over the Metal alias; strtol accepts a numeric prefix; <= 0 returns 0, > UINT32_MAX clamps, invalid uses ROCm default UINT32_MAX for Pro/Flash/GLM52 and 0 otherwise | Sets the inclusive upper token-count bound for automatically using selected-expert address-table kernels during eligible non-quality SSD batch prefill; 0 disables automatic selection. | [ds4.c:18298](ds4.c#L18298) | -| `DS4_ROCM_STREAMING_PREFILL_BATCH_SELECTED_ADDR_MIN` | primary nonempty value over the Metal alias; strtol accepts a numeric prefix; <= 0 returns 0, > UINT32_MAX clamps, invalid uses ROCm default 2 for Pro/Flash/GLM52 and 0 otherwise | Sets the inclusive lower token-count bound for automatically using selected-expert address-table kernels during eligible non-quality SSD batch prefill (the path independently requires more than one token). | [ds4.c:18321](ds4.c#L18321) | -| `DS4_ROCM_STREAMING_PREFILL_CACHE_SEED_K` | primary nonempty value over the Metal alias; strict full-string strtoul; unset/empty/invalid: 1; 0 disables; positive values clamp to 64; ignored unless SSD streaming and DS4_ROCM_ENABLE_STREAMING_PREFILL_CACHE_SEED (or Metal alias) is present | Chooses how many trailing token router selections per layer are captured from prefill and used to seed the streaming expert cache afterward. | [ds4.c:21094](ds4.c#L21094) | -| `DS4_ROCM_STREAMING_PREFILL_CACHE_SEED_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm streaming prefill cache seed profile. | [ds4.c:31963](ds4.c#L31963) | -| `DS4_ROCM_STREAMING_PREFILL_LAYER_MADVISE_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm streaming prefill layer madvise profile. | [ds4.c:19436](ds4.c#L19436) | -| `DS4_ROCM_STREAMING_PREFILL_LAYER_PAGEIN_NO_OVERLAP` | Pure presence flag: any defined value, including empty or "0", disables overlap. Default overlap is enabled only if this, PREPARE_NO_OVERLAP, DISABLE_*_PREPARE_OVERLAP, and DISABLE_*_PAGEIN_OVERLAP are all absent. The corresponding DS4_METAL name is an accepted fallback alias. In current code PAGEIN_NO_OVERLAP and PREPARE_NO_OVERLAP are exact synonyms. | Serialize SSD-streaming prefill layer page-in/preparation instead of overlapping preparation of upcoming layers. | [ds4.c:19143](ds4.c#L19143) | -| `DS4_ROCM_STREAMING_PREFILL_LAYER_PAGEIN_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm streaming prefill layer pagein profile. | [ds4.c:19432](ds4.c#L19432) | -| `DS4_ROCM_STREAMING_PREFILL_LAYER_PAGEIN_THREADS` | legacy fallback read only when DS4_ROCM_STREAMING_PREFILL_LAYER_PREPARE_THREADS and its Metal alias are absent/empty; strict full-string strtoul; unset/empty across both names: 8; invalid or 0: 1; values > 16 clamp to 16 | Sets worker count for full-layer SSD-prefill preparation (page touch, pread, readahead, or madvise) when the canonical PREPARE_THREADS control is not set. | [ds4.c:19101](ds4.c#L19101) | -| `DS4_ROCM_STREAMING_PREFILL_LAYER_PREAD_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm streaming prefill layer pread profile. | [ds4.c:19434](ds4.c#L19434) | -| `DS4_ROCM_STREAMING_PREFILL_LAYER_PREPARE_AHEAD` | primary nonempty value over the Metal alias; strict full-string strtoul; unset/empty: 1; invalid or 0: 1; values > 4 clamp to 4 | Sets how many future layer-preparation jobs may be queued concurrently while SSD-prefill preparation overlap is enabled. | [ds4.c:19155](ds4.c#L19155) | -| `DS4_ROCM_STREAMING_PREFILL_LAYER_PREPARE_NO_OVERLAP` | Pure presence flag: any defined value, including empty or "0", disables overlap. Default overlap is enabled only if this, PAGEIN_NO_OVERLAP, DISABLE_*_PREPARE_OVERLAP, and DISABLE_*_PAGEIN_OVERLAP are all absent. The corresponding DS4_METAL name is an accepted fallback alias. In current code PREPARE_NO_OVERLAP and PAGEIN_NO_OVERLAP are exact synonyms. | Serialize SSD-streaming prefill layer preparation/page-in instead of overlapping preparation of upcoming layers. | [ds4.c:19141](ds4.c#L19141) | -| `DS4_ROCM_STREAMING_PREFILL_LAYER_PREPARE_THREADS` | primary nonempty value over the Metal alias; strict full-string strtoul; unset/empty falls back to LAYER_PAGEIN_THREADS, then default 8; invalid or 0: 1; values > 16 clamp to 16 | Sets worker count used to split full-layer SSD-prefill page-touch, pread, readahead, or madvise preparation ranges. | [ds4.c:19097](ds4.c#L19097) | -| `DS4_ROCM_STREAMING_PREFILL_LAYER_READAHEAD_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm streaming prefill layer readahead profile. | [ds4.c:19438](ds4.c#L19438) | -| `DS4_ROCM_STREAMING_PREFILL_SELECTED_MADVISE_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm streaming prefill selected madvise profile. | [ds4.c:19182](ds4.c#L19182) | -| `DS4_ROCM_STREAMING_PREFILL_SELECTED_MADVISE_THREADS` | legacy fallback read only for selected-expert madvise preparation when DS4_ROCM_STREAMING_PREFILL_SELECTED_PREPARE_THREADS and its Metal alias are absent/empty; strict full-string strtoul; if all selected controls are unset it inherits layer preparation threads (default 8); invalid or 0: 1; values > 16 clamp to 16 | Sets worker count for selected-expert madvise preparation under its legacy name; non-madvise selected page-in always uses one worker. | [ds4.c:19119](ds4.c#L19119) | -| `DS4_ROCM_STREAMING_PREFILL_SELECTED_PAGEIN_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm streaming prefill selected pagein profile. | [ds4.c:19180](ds4.c#L19180) | -| `DS4_ROCM_STREAMING_PREFILL_SELECTED_PREPARE_GAP` | primary nonempty value over the Metal alias; strict full-string strtoul; unset/empty/invalid: 0; values > 8 clamp to 8 | For selected-expert madvise preparation, merges selected expert runs separated by at most this many unselected expert IDs, trading broader hints for fewer ranges. | [ds4.c:19131](ds4.c#L19131) | -| `DS4_ROCM_STREAMING_PREFILL_SELECTED_PREPARE_THREADS` | primary nonempty value over the Metal alias; strict full-string strtoul; for selected-expert madvise, unset/empty falls back to SELECTED_MADVISE_THREADS then layer preparation threads (default 8); invalid or 0: 1; values > 16 clamp to 16; non-madvise selected page-in ignores it and uses 1 | Sets worker count for selected-expert madvise preparation using the canonical control name. | [ds4.c:19115](ds4.c#L19115) | -| `DS4_ROCM_STREAMING_PREFILL_SELECTED_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm streaming prefill selected profile. | [ds4.c:18732](ds4.c#L18732) | -| `DS4_ROCM_STREAMING_PREFILL_SELECTED_READAHEAD_GAP` | primary nonempty value over the Metal alias; strict full-string strtoul; unset/empty/invalid: 0; values > 8 clamp to 8 | For selected-expert file readahead, merges selected expert runs separated by at most this many unselected expert IDs, reducing readahead calls at the cost of hinting extra weights. | [ds4.c:19804](ds4.c#L19804) | -| `DS4_ROCM_STREAMING_PREFILL_SELECTED_READAHEAD_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm streaming prefill selected readahead profile. | [ds4.c:19889](ds4.c#L19889) | +| `DS4_ROCM_STREAMING_EXPERT_HOTLIST` | Nonempty filesystem path; a nonempty ROCm value takes precedence, otherwise DS4_METAL_STREAMING_EXPERT_HOTLIST is a fallback. The file contains whitespace-separated layer expert hits rows; blank/comment lines are ignored, zero-hit rows skipped, malformed/open/read errors fail seeding. Unset/empty uses the built-in Pro/Flash/GLM52 hotlist. Effective only when non-cold SSD hotlist seeding is enabled and cache/preload budget is nonzero. | Select a custom ranked expert hotlist used to preseed the streaming resident expert cache before decode. | [ds4.c:21494](ds4.c#L21494) | +| `DS4_ROCM_STREAMING_EXPERT_HOTLIST_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm streaming expert hotlist profile. | [ds4.c:32244](ds4.c#L32244) | +| `DS4_ROCM_STREAMING_MAP_TRACE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Emit trace diagnostics for rocm streaming map trace. | [ds4.c:42643](ds4.c#L42643) | +| `DS4_ROCM_STREAMING_PREFILL_BATCH_SELECTED_ADDR_MAX` | primary nonempty value over the Metal alias; strtol accepts a numeric prefix; <= 0 returns 0, > UINT32_MAX clamps, invalid uses ROCm default UINT32_MAX for Pro/Flash/GLM52 and 0 otherwise | Sets the inclusive upper token-count bound for automatically using selected-expert address-table kernels during eligible non-quality SSD batch prefill; 0 disables automatic selection. | [ds4.c:18471](ds4.c#L18471) | +| `DS4_ROCM_STREAMING_PREFILL_BATCH_SELECTED_ADDR_MIN` | primary nonempty value over the Metal alias; strtol accepts a numeric prefix; <= 0 returns 0, > UINT32_MAX clamps, invalid uses ROCm default 2 for Pro/Flash/GLM52 and 0 otherwise | Sets the inclusive lower token-count bound for automatically using selected-expert address-table kernels during eligible non-quality SSD batch prefill (the path independently requires more than one token). | [ds4.c:18494](ds4.c#L18494) | +| `DS4_ROCM_STREAMING_PREFILL_CACHE_SEED_K` | primary nonempty value over the Metal alias; strict full-string strtoul; unset/empty/invalid: 1; 0 disables; positive values clamp to 64; ignored unless SSD streaming and DS4_ROCM_ENABLE_STREAMING_PREFILL_CACHE_SEED (or Metal alias) is present | Chooses how many trailing token router selections per layer are captured from prefill and used to seed the streaming expert cache afterward. | [ds4.c:21267](ds4.c#L21267) | +| `DS4_ROCM_STREAMING_PREFILL_CACHE_SEED_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm streaming prefill cache seed profile. | [ds4.c:32153](ds4.c#L32153) | +| `DS4_ROCM_STREAMING_PREFILL_LAYER_MADVISE_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm streaming prefill layer madvise profile. | [ds4.c:19609](ds4.c#L19609) | +| `DS4_ROCM_STREAMING_PREFILL_LAYER_PAGEIN_NO_OVERLAP` | Pure presence flag: any defined value, including empty or "0", disables overlap. Default overlap is enabled only if this, PREPARE_NO_OVERLAP, DISABLE_*_PREPARE_OVERLAP, and DISABLE_*_PAGEIN_OVERLAP are all absent. The corresponding DS4_METAL name is an accepted fallback alias. In current code PAGEIN_NO_OVERLAP and PREPARE_NO_OVERLAP are exact synonyms. | Serialize SSD-streaming prefill layer page-in/preparation instead of overlapping preparation of upcoming layers. | [ds4.c:19316](ds4.c#L19316) | +| `DS4_ROCM_STREAMING_PREFILL_LAYER_PAGEIN_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm streaming prefill layer pagein profile. | [ds4.c:19605](ds4.c#L19605) | +| `DS4_ROCM_STREAMING_PREFILL_LAYER_PAGEIN_THREADS` | legacy fallback read only when DS4_ROCM_STREAMING_PREFILL_LAYER_PREPARE_THREADS and its Metal alias are absent/empty; strict full-string strtoul; unset/empty across both names: 8; invalid or 0: 1; values > 16 clamp to 16 | Sets worker count for full-layer SSD-prefill preparation (page touch, pread, readahead, or madvise) when the canonical PREPARE_THREADS control is not set. | [ds4.c:19274](ds4.c#L19274) | +| `DS4_ROCM_STREAMING_PREFILL_LAYER_PREAD_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm streaming prefill layer pread profile. | [ds4.c:19607](ds4.c#L19607) | +| `DS4_ROCM_STREAMING_PREFILL_LAYER_PREPARE_AHEAD` | primary nonempty value over the Metal alias; strict full-string strtoul; unset/empty: 1; invalid or 0: 1; values > 4 clamp to 4 | Sets how many future layer-preparation jobs may be queued concurrently while SSD-prefill preparation overlap is enabled. | [ds4.c:19328](ds4.c#L19328) | +| `DS4_ROCM_STREAMING_PREFILL_LAYER_PREPARE_NO_OVERLAP` | Pure presence flag: any defined value, including empty or "0", disables overlap. Default overlap is enabled only if this, PAGEIN_NO_OVERLAP, DISABLE_*_PREPARE_OVERLAP, and DISABLE_*_PAGEIN_OVERLAP are all absent. The corresponding DS4_METAL name is an accepted fallback alias. In current code PREPARE_NO_OVERLAP and PAGEIN_NO_OVERLAP are exact synonyms. | Serialize SSD-streaming prefill layer preparation/page-in instead of overlapping preparation of upcoming layers. | [ds4.c:19314](ds4.c#L19314) | +| `DS4_ROCM_STREAMING_PREFILL_LAYER_PREPARE_THREADS` | primary nonempty value over the Metal alias; strict full-string strtoul; unset/empty falls back to LAYER_PAGEIN_THREADS, then default 8; invalid or 0: 1; values > 16 clamp to 16 | Sets worker count used to split full-layer SSD-prefill page-touch, pread, readahead, or madvise preparation ranges. | [ds4.c:19270](ds4.c#L19270) | +| `DS4_ROCM_STREAMING_PREFILL_LAYER_READAHEAD_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm streaming prefill layer readahead profile. | [ds4.c:19611](ds4.c#L19611) | +| `DS4_ROCM_STREAMING_PREFILL_SELECTED_MADVISE_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm streaming prefill selected madvise profile. | [ds4.c:19355](ds4.c#L19355) | +| `DS4_ROCM_STREAMING_PREFILL_SELECTED_MADVISE_THREADS` | legacy fallback read only for selected-expert madvise preparation when DS4_ROCM_STREAMING_PREFILL_SELECTED_PREPARE_THREADS and its Metal alias are absent/empty; strict full-string strtoul; if all selected controls are unset it inherits layer preparation threads (default 8); invalid or 0: 1; values > 16 clamp to 16 | Sets worker count for selected-expert madvise preparation under its legacy name; non-madvise selected page-in always uses one worker. | [ds4.c:19292](ds4.c#L19292) | +| `DS4_ROCM_STREAMING_PREFILL_SELECTED_PAGEIN_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm streaming prefill selected pagein profile. | [ds4.c:19353](ds4.c#L19353) | +| `DS4_ROCM_STREAMING_PREFILL_SELECTED_PREPARE_GAP` | primary nonempty value over the Metal alias; strict full-string strtoul; unset/empty/invalid: 0; values > 8 clamp to 8 | For selected-expert madvise preparation, merges selected expert runs separated by at most this many unselected expert IDs, trading broader hints for fewer ranges. | [ds4.c:19304](ds4.c#L19304) | +| `DS4_ROCM_STREAMING_PREFILL_SELECTED_PREPARE_THREADS` | primary nonempty value over the Metal alias; strict full-string strtoul; for selected-expert madvise, unset/empty falls back to SELECTED_MADVISE_THREADS then layer preparation threads (default 8); invalid or 0: 1; values > 16 clamp to 16; non-madvise selected page-in ignores it and uses 1 | Sets worker count for selected-expert madvise preparation using the canonical control name. | [ds4.c:19288](ds4.c#L19288) | +| `DS4_ROCM_STREAMING_PREFILL_SELECTED_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm streaming prefill selected profile. | [ds4.c:18905](ds4.c#L18905) | +| `DS4_ROCM_STREAMING_PREFILL_SELECTED_READAHEAD_GAP` | primary nonempty value over the Metal alias; strict full-string strtoul; unset/empty/invalid: 0; values > 8 clamp to 8 | For selected-expert file readahead, merges selected expert runs separated by at most this many unselected expert IDs, reducing readahead calls at the cost of hinting extra weights. | [ds4.c:19977](ds4.c#L19977) | +| `DS4_ROCM_STREAMING_PREFILL_SELECTED_READAHEAD_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm streaming prefill selected readahead profile. | [ds4.c:20062](ds4.c#L20062) | | `DS4_ROCM_STREAM_CACHE_LAYER_STATS` | presence flag; unset=off | Collect per-layer ROCm streaming cache statistics; also enables aggregate stats. | [rocm/ds4_rocm_runtime.cuh:390](rocm/ds4_rocm_runtime.cuh#L390) | | `DS4_ROCM_STREAM_CACHE_STATS` | presence flag; unset=off unless layer stats are enabled | Collect aggregate ROCm streaming cache statistics. | [rocm/ds4_rocm_runtime.cuh:398](rocm/ds4_rocm_runtime.cuh#L398) | | `DS4_ROCM_STREAM_EVICT_PAST_LAYERS_FIRST` | nonempty and not 0 enables; unset/empty/0=off | Prefer evicting cached experts from already-processed layers. | [rocm/ds4_rocm_runtime.cuh:406](rocm/ds4_rocm_runtime.cuh#L406) | @@ -1047,24 +1051,24 @@ and **19 tool/wrapper entries**. | Variable | Accepted value and default | Effect | Source | | --- | --- | --- | --- | -| `DS4_GLM_ABLATE_COMBINE` | presence ablation; unset: exchange the local TP partial with the peer and add both halves; any presence including empty or 0 skips the exchange | Metal two-rank TP timing probe: doubles the local routed-MoE or split-attention partial instead of combining with the peer, deliberately producing invalid output; both ranks must set it or their exchange gates desynchronize. | [ds4.c:43991](ds4.c#L43991) | +| `DS4_GLM_ABLATE_COMBINE` | presence ablation; unset: exchange the local TP partial with the peer and add both halves; any presence including empty or 0 skips the exchange | Metal two-rank TP timing probe: doubles the local routed-MoE or split-attention partial instead of combining with the peer, deliberately producing invalid output; both ranks must set it or their exchange gates desynchronize. | [ds4.c:44181](ds4.c#L44181) | | `DS4_GLM_ATTN_NO_LORA_VEC2` | presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it | Disable vectorized two-row LoRA accumulation in CUDA GLM indexed attention. | [ds4_cuda.cu:34176](ds4_cuda.cu#L34176) | | `DS4_GLM_ATTN_NO_SCORE_VEC2` | presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it | Disable vectorized two-row score computation in CUDA GLM indexed attention. | [ds4_cuda.cu:34165](ds4_cuda.cu#L34165) | | `DS4_GLM_ATTN_NO_STAGED_DECODE` | presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it | Disable staged CUDA GLM indexed-decode attention for large selected sets. | [ds4_cuda.cu:34214](ds4_cuda.cu#L34214) | -| `DS4_GLM_DECODE_ABLATE` | cached substring list; default empty mask; recognized tokens are attn_out, attn_core, qpath, indexer, routed, shared, qklow; unknown text has no effect; matching stages are skipped and output is invalid | Skip selected GLM decode stages for timing attribution; generated output is invalid. | [ds4.c:44238](ds4.c#L44238) | -| `DS4_GLM_DECODE_FLUSH_INTERVAL` | integer layers via atoi; default 4 for indexed decode and 32 otherwise; <=0/nonnumeric disables periodic flush; capped to layer count and forced to 0 for deferred completion | Set how often non-streaming GLM decode command work is flushed between layers. | [ds4.c:49628](ds4.c#L49628) | -| `DS4_GLM_DISABLE_FLASH_PREFILL` | presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it | Disable GLM Flash Attention prefill. | [ds4.c:44897](ds4.c#L44897) | -| `DS4_GLM_DISABLE_STREAMING_TOKEN_PREFILL` | presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it; either backend-specific ROCm/Metal alias also disables | Disable token-major GLM SSD-streaming prefill. | [ds4.c:49512](ds4.c#L49512) | +| `DS4_GLM_DECODE_ABLATE` | cached substring list; default empty mask; recognized tokens are attn_out, attn_core, qpath, indexer, routed, shared, qklow; unknown text has no effect; matching stages are skipped and output is invalid | Skip selected GLM decode stages for timing attribution; generated output is invalid. | [ds4.c:44411](ds4.c#L44411) | +| `DS4_GLM_DECODE_FLUSH_INTERVAL` | integer layers via atoi; default 4 for indexed decode and 32 otherwise; <=0/nonnumeric disables periodic flush; capped to layer count and forced to 0 for deferred completion | Set how often non-streaming GLM decode command work is flushed between layers. | [ds4.c:49818](ds4.c#L49818) | +| `DS4_GLM_DISABLE_FLASH_PREFILL` | presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it | Disable GLM Flash Attention prefill. | [ds4.c:45087](ds4.c#L45087) | +| `DS4_GLM_DISABLE_STREAMING_TOKEN_PREFILL` | presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it; either backend-specific ROCm/Metal alias also disables | Disable token-major GLM SSD-streaming prefill. | [ds4.c:49702](ds4.c#L49702) | | `DS4_GLM_FENCE_TRACE` | presence flag; unset is off; any defined value, including empty or 0, is on; normal eligibility still applies | Log fenced CUDA tier switches used by GLM multi-GPU execution. | [ds4_cuda.cu:7984](ds4_cuda.cu#L7984) | | `DS4_GLM_GEMM_TRACE` | presence flag; unset is off; any defined value, including empty or 0, is on; normal eligibility still applies | Measure and print CUDA GLM dequantization and cuBLAS GEMM timings. | [ds4_cuda.cu:19706](ds4_cuda.cu#L19706) | -| `DS4_GLM_HIDDEN_DUMP` | nonempty filesystem path/prefix; unset/empty disables; writes final hidden row or per-layer files selected by DS4_GLM_HIDDEN_DUMP_LAYER | Dump GLM hidden-state rows for correctness isolation. | [ds4.c:38333](ds4.c#L38333) | -| `DS4_GLM_HIDDEN_DUMP_LAYER` | selector; unset/empty = -1 (no per-layer dump, final hidden still dumped when path set); all = every layer; otherwise atoi result selects a layer, so invalid text selects layer 0 | Choose which GLM layer hidden states are dumped. | [ds4.c:38353](ds4.c#L38353) | -| `DS4_GLM_KV_DUMP` | nonempty filesystem prefix; unset/empty disables; writes layer-0 lora and rope compact-cache files after sync | Dump layer-0 compact GLM KV cache data after prompt synchronization. | [ds4.c:62941](ds4.c#L62941) | -| `DS4_GLM_LOGIT_DUMP` | nonempty filesystem path; unset/empty disables; dumps the first post-prefill logits vector once per process | Dump the first post-prefill GLM logits vector. | [ds4.c:38411](ds4.c#L38411) | -| `DS4_GLM_MEMORY_GUARD` | guard is on by default; exact 0 or case-insensitive false/off/no disables it; any other value and unset keep it enabled | Control the pre-allocation GLM host/GPU memory safety guard. | [ds4.c:41799](ds4.c#L41799) | -| `DS4_GLM_MEMORY_GUARD_FRACTION` | floating-point fraction; default 0.99; parsed numeric prefix is accepted, invalid/nonfinite falls back, values clamp to 0.50..1.00 | Set the fraction of detected memory usable by the GLM memory guard. | [ds4.c:41838](ds4.c#L41838) | -| `DS4_GLM_MEMORY_GUARD_REPORT` | nonempty diagnostic flag; unset/empty is off; any nonempty value including 0 prints successful-admission accounting (refusals always report) | Print successful GLM memory-guard budget accounting. | [ds4.c:41878](ds4.c#L41878) | -| `DS4_GLM_MEMORY_GUARD_RESERVE_GB` | floating-point GiB; dynamic default (normally 32, 24 on near-full 480..640 GiB hosts, possibly lower for resident ROCm slices); numeric prefixes accepted; invalid falls back; clamp 0..1024 | Set fixed headroom subtracted by the GLM memory guard. | [ds4.c:41856](ds4.c#L41856) | +| `DS4_GLM_HIDDEN_DUMP` | nonempty filesystem path/prefix; unset/empty disables; writes final hidden row or per-layer files selected by DS4_GLM_HIDDEN_DUMP_LAYER | Dump GLM hidden-state rows for correctness isolation. | [ds4.c:38523](ds4.c#L38523) | +| `DS4_GLM_HIDDEN_DUMP_LAYER` | selector; unset/empty = -1 (no per-layer dump, final hidden still dumped when path set); all = every layer; otherwise atoi result selects a layer, so invalid text selects layer 0 | Choose which GLM layer hidden states are dumped. | [ds4.c:38543](ds4.c#L38543) | +| `DS4_GLM_KV_DUMP` | nonempty filesystem prefix; unset/empty disables; writes layer-0 lora and rope compact-cache files after sync | Dump layer-0 compact GLM KV cache data after prompt synchronization. | [ds4.c:63131](ds4.c#L63131) | +| `DS4_GLM_LOGIT_DUMP` | nonempty filesystem path; unset/empty disables; dumps the first post-prefill logits vector once per process | Dump the first post-prefill GLM logits vector. | [ds4.c:38601](ds4.c#L38601) | +| `DS4_GLM_MEMORY_GUARD` | guard is on by default; exact 0 or case-insensitive false/off/no disables it; any other value and unset keep it enabled | Control the pre-allocation GLM host/GPU memory safety guard. | [ds4.c:41989](ds4.c#L41989) | +| `DS4_GLM_MEMORY_GUARD_FRACTION` | floating-point fraction; default 0.99; parsed numeric prefix is accepted, invalid/nonfinite falls back, values clamp to 0.50..1.00 | Set the fraction of detected memory usable by the GLM memory guard. | [ds4.c:42028](ds4.c#L42028) | +| `DS4_GLM_MEMORY_GUARD_REPORT` | nonempty diagnostic flag; unset/empty is off; any nonempty value including 0 prints successful-admission accounting (refusals always report) | Print successful GLM memory-guard budget accounting. | [ds4.c:42068](ds4.c#L42068) | +| `DS4_GLM_MEMORY_GUARD_RESERVE_GB` | floating-point GiB; dynamic default (normally 32, 24 on near-full 480..640 GiB hosts, possibly lower for resident ROCm slices); numeric prefixes accepted; invalid falls back; clamp 0..1024 | Set fixed headroom subtracted by the GLM memory guard. | [ds4.c:42046](ds4.c#L42046) | | `DS4_GLM_MOE_EXPERT_MAJOR` | presence selector; unset: off; any presence including empty or 0 requests the path only for n_tokens >= 16; the automatic tile-8 path takes precedence when enabled (normally n_tokens >= 128) | CUDA only: groups selected token/expert pairs by expert and uses expert-major Q2_K routed-MoE gate/up/down kernels to reuse expert weights; otherwise the normal token-major path is used. | [ds4_cuda.cu:35973](ds4_cuda.cu#L35973) | | `DS4_GLM_MOE_NO_DOWN_TILE8_EXACT` | presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it | Disable the exact tile-8 CUDA GLM routed-MoE down projection. | [ds4_cuda.cu:36028](ds4_cuda.cu#L36028) | | `DS4_GLM_MOE_NO_EXPERT_TILE8` | presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it | Disable automatic expert tile-8 CUDA GLM routed-MoE batching. | [ds4_cuda.cu:35971](ds4_cuda.cu#L35971) | @@ -1073,20 +1077,20 @@ and **19 tool/wrapper entries**. | `DS4_GLM_MOE_SCRATCH_TIER0` | presence placement override; unset: allocate xq/midq quantization scratch on the current logical tier; any presence including empty or 0 allocates it on logical tier 0 | CUDA only: pins the routed-MoE xq_scratch and midq_scratch allocations to GPU tier 0 for multi-tier placement experiments; other MoE scratch remains on the current tier. | [ds4_cuda.cu:35918](ds4_cuda.cu#L35918) | | `DS4_GLM_MTP_NO_ATTN_TOK2` | presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it | Disable the exact two-token CUDA GLM MTP attention kernel. | [ds4_cuda.cu:34168](ds4_cuda.cu#L34168) | | `DS4_GLM_MTP_NO_MOE_TOK2` | presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it | Disable the exact two-token CUDA GLM MTP routed-MoE kernel. | [ds4_cuda.cu:36140](ds4_cuda.cu#L36140) | -| `DS4_GLM_MTP_NO_SHARED_TOK2` | presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it | Disable the exact two-token CUDA GLM MTP shared-FFN kernel. | [ds4_cuda.cu:37920](ds4_cuda.cu#L37920) | -| `DS4_GLM_MTP_PROBE` | presence flag; unset is off; any defined value, including empty or 0, is on; its second call site also rejects the normal batching path | Run the GLM next-N/MTP acceptance quality-and-timing probe without changing output and force probe-compatible scheduling. | [ds4.c:64769](ds4.c#L64769) | -| `DS4_GLM_PREFILL_TRUNC` | nonempty value parsed by atoi (leading whitespace/sign accepted and trailing junk ignored); effective only when the resulting int is > 0 and < the current prompt length; unset/empty or a result <= 0 or >= prompt length leaves the prompt unchanged | GPU GLM debug hook: truncates the prompt before prefill/checkpoint handling so dumped prefill logits can be aligned with a CPU first-token reference. | [ds4.c:63080](ds4.c#L63080) | -| `DS4_GLM_RESUME_PREFILL_MIN` | integer suffix tokens, non-ROCm builds only; default 4; parsed numeric prefixes accepted; <=0 maps to UINT32_MAX and effectively disables batched resume; ROCm build ignores it and stays at 4 | Set the suffix-length crossover from token decode to batched resumed prefill. | [ds4.c:38244](ds4.c#L38244) | +| `DS4_GLM_MTP_NO_SHARED_TOK2` | presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it | Disable the exact two-token CUDA GLM MTP shared-FFN kernel. | [ds4_cuda.cu:37940](ds4_cuda.cu#L37940) | +| `DS4_GLM_MTP_PROBE` | presence flag; unset is off; any defined value, including empty or 0, is on; its second call site also rejects the normal batching path | Run the GLM next-N/MTP acceptance quality-and-timing probe without changing output and force probe-compatible scheduling. | [ds4.c:64959](ds4.c#L64959) | +| `DS4_GLM_PREFILL_TRUNC` | nonempty value parsed by atoi (leading whitespace/sign accepted and trailing junk ignored); effective only when the resulting int is > 0 and < the current prompt length; unset/empty or a result <= 0 or >= prompt length leaves the prompt unchanged | GPU GLM debug hook: truncates the prompt before prefill/checkpoint handling so dumped prefill logits can be aligned with a CPU first-token reference. | [ds4.c:63270](ds4.c#L63270) | +| `DS4_GLM_RESUME_PREFILL_MIN` | integer suffix tokens, non-ROCm builds only; default 4; parsed numeric prefixes accepted; <=0 maps to UINT32_MAX and effectively disables batched resume; ROCm build ignores it and stays at 4 | Set the suffix-length crossover from token decode to batched resumed prefill. | [ds4.c:38434](ds4.c#L38434) | | `DS4_GLM_ROUTER_SCALAR` | presence rollback; unset: use the 256-thread parallel router when n_expert <= 256 (the scalar path is already automatic above 256); any presence including empty or 0 forces the scalar path | CUDA only: selects the one-active-thread-per-token sigmoid/top-k router kernel instead of the parallel shared-memory reduction, for A/B or correctness testing. | [ds4_cuda.cu:36390](ds4_cuda.cu#L36390) | -| `DS4_GLM_SHARED_SPLIT` | presence rollback; unset: use the fused one-token shared-expert Q8_0 gate+up+SwiGLU kernel when its shape/buffers are eligible; any presence including empty or 0 skips that fused one-token path; the earlier two-token MTP-specialized path is unaffected | CUDA only: forces shared-expert gate and up through two separate Q8_0 matmuls followed by a separate SwiGLU operation for one-token decode. | [ds4_cuda.cu:37946](ds4_cuda.cu#L37946) | -| `DS4_GLM_STREAMING_DECODE_FULL_LAYER_MAP` | presence compatibility alias; unset: automatic mapping; any presence including empty or 0 independently forces full-layer mapping, equivalent to the backend-specific DS4_ROCM_GLM_STREAMING_DECODE_FULL_LAYER_MAP or DS4_METAL_GLM_STREAMING_DECODE_FULL_LAYER_MAP control | Backend-neutral alias for supported GLM SSD streaming (Metal/ROCm): maps every tensor in each decode layer instead of using the decode-only map that can omit routed experts served by the expert cache; layers that already require a full map are unchanged. | [ds4.c:42427](ds4.c#L42427) | -| `DS4_GLM_STREAMING_DECODE_SYNC_EACH_LAYER` | ROCm-only third-priority legacy value: a nonempty DS4_ROCM_GLM_STREAMING_DECODE_SYNC_EACH_LAYER wins, otherwise a nonempty DS4_METAL_GLM_STREAMING_DECODE_SYNC_EACH_LAYER wins, otherwise this name is read; nonempty values are true except exact 0 or case-insensitive false/off/no; unset/empty: false; non-ROCm builds always return true and ignore this name | On ROCm non-static GLM SSD decode, opts into ending/synchronizing commands after token mapping and after every layer; the default keeps ordered work alive across layer mappings. Static-map decode bypasses this control. | [ds4.c:49591](ds4.c#L49591) | -| `DS4_GLM_STREAMING_PREFILL_SYNC_EACH_LAYER` | ROCm-only third-priority legacy value: a nonempty DS4_ROCM_GLM_STREAMING_PREFILL_SYNC_EACH_LAYER wins, otherwise a nonempty DS4_METAL_GLM_STREAMING_PREFILL_SYNC_EACH_LAYER wins, otherwise this name is read; nonempty values are true except exact 0 or case-insensitive false/off/no; unset/empty: false for compact prefill; full-layer prefill and non-ROCm builds always synchronize and ignore this name | On ROCm compact GLM SSD prefill, opts into ending/synchronizing commands at every layer boundary; the default carries ordered work across mappings, while full-layer expert-cache prefill always retains the boundary. | [ds4.c:42207](ds4.c#L42207) | -| `DS4_GLM_STREAMING_TOKEN_PREFILL_MAX` | unsigned token limit; backend-specific ROCm/Metal variable takes precedence, generic is fallback; default 0 on ROCm and 64 otherwise; invalid/overflow falls back, numeric prefixes accepted; 0 disables token-major streaming prefill | Set the largest SSD-streaming prefill handled by the token-major decode-like path. | [ds4.c:49494](ds4.c#L49494) | -| `DS4_GLM_SYNC_TRACE` | presence flag; unset is off; any defined value, including empty or 0, is on; normal eligibility still applies | Log GLM checkpoint/resume and dense-versus-indexed prefill decisions. | [ds4.c:63178](ds4.c#L63178) | -| `DS4_GLM_TP_DEBUG` | presence flag; unset is off; any defined value, including empty or 0, is on; normal eligibility still applies | Print CUDA/GLM tensor-parallel dispatch, gate, selected-ID, and failure diagnostics. | [ds4.c:43893](ds4.c#L43893) | -| `DS4_GLM_TP_EXACT_PREFILL_MAX` | integer suffix limit via atoi cast to uint32; default 64; nonnumeric becomes 0; negative values wrap to a very large unsigned limit | Set the maximum two-way TP suffix that uses exact token-by-token prefill. | [ds4.c:63120](ds4.c#L63120) | -| `DS4_GLM_TP_HEAD_SPLIT_MIN` | cached integer token threshold via atoi; default 64; negative values clamp to 0, nonnumeric becomes 0; 0 admits all otherwise-eligible batches | Set the minimum batch size for GLM tensor-parallel output-head splitting. | [ds4.c:38323](ds4.c#L38323) | +| `DS4_GLM_SHARED_SPLIT` | presence rollback; unset: use the fused one-token shared-expert Q8_0 gate+up+SwiGLU kernel when its shape/buffers are eligible; any presence including empty or 0 skips that fused one-token path; the earlier two-token MTP-specialized path is unaffected | CUDA only: forces shared-expert gate and up through two separate Q8_0 matmuls followed by a separate SwiGLU operation for one-token decode. | [ds4_cuda.cu:37966](ds4_cuda.cu#L37966) | +| `DS4_GLM_STREAMING_DECODE_FULL_LAYER_MAP` | presence compatibility alias; unset: automatic mapping; any presence including empty or 0 independently forces full-layer mapping, equivalent to the backend-specific DS4_ROCM_GLM_STREAMING_DECODE_FULL_LAYER_MAP or DS4_METAL_GLM_STREAMING_DECODE_FULL_LAYER_MAP control | Backend-neutral alias for supported GLM SSD streaming (Metal/ROCm): maps every tensor in each decode layer instead of using the decode-only map that can omit routed experts served by the expert cache; layers that already require a full map are unchanged. | [ds4.c:42617](ds4.c#L42617) | +| `DS4_GLM_STREAMING_DECODE_SYNC_EACH_LAYER` | ROCm-only third-priority legacy value: a nonempty DS4_ROCM_GLM_STREAMING_DECODE_SYNC_EACH_LAYER wins, otherwise a nonempty DS4_METAL_GLM_STREAMING_DECODE_SYNC_EACH_LAYER wins, otherwise this name is read; nonempty values are true except exact 0 or case-insensitive false/off/no; unset/empty: false; non-ROCm builds always return true and ignore this name | On ROCm non-static GLM SSD decode, opts into ending/synchronizing commands after token mapping and after every layer; the default keeps ordered work alive across layer mappings. Static-map decode bypasses this control. | [ds4.c:49781](ds4.c#L49781) | +| `DS4_GLM_STREAMING_PREFILL_SYNC_EACH_LAYER` | ROCm-only third-priority legacy value: a nonempty DS4_ROCM_GLM_STREAMING_PREFILL_SYNC_EACH_LAYER wins, otherwise a nonempty DS4_METAL_GLM_STREAMING_PREFILL_SYNC_EACH_LAYER wins, otherwise this name is read; nonempty values are true except exact 0 or case-insensitive false/off/no; unset/empty: false for compact prefill; full-layer prefill and non-ROCm builds always synchronize and ignore this name | On ROCm compact GLM SSD prefill, opts into ending/synchronizing commands at every layer boundary; the default carries ordered work across mappings, while full-layer expert-cache prefill always retains the boundary. | [ds4.c:42397](ds4.c#L42397) | +| `DS4_GLM_STREAMING_TOKEN_PREFILL_MAX` | unsigned token limit; backend-specific ROCm/Metal variable takes precedence, generic is fallback; default 0 on ROCm and 64 otherwise; invalid/overflow falls back, numeric prefixes accepted; 0 disables token-major streaming prefill | Set the largest SSD-streaming prefill handled by the token-major decode-like path. | [ds4.c:49684](ds4.c#L49684) | +| `DS4_GLM_SYNC_TRACE` | presence flag; unset is off; any defined value, including empty or 0, is on; normal eligibility still applies | Log GLM checkpoint/resume and dense-versus-indexed prefill decisions. | [ds4.c:63368](ds4.c#L63368) | +| `DS4_GLM_TP_DEBUG` | presence flag; unset is off; any defined value, including empty or 0, is on; normal eligibility still applies | Print CUDA/GLM tensor-parallel dispatch, gate, selected-ID, and failure diagnostics. | [ds4.c:44083](ds4.c#L44083) | +| `DS4_GLM_TP_EXACT_PREFILL_MAX` | integer suffix limit via atoi cast to uint32; default 64; nonnumeric becomes 0; negative values wrap to a very large unsigned limit | Set the maximum two-way TP suffix that uses exact token-by-token prefill. | [ds4.c:63310](ds4.c#L63310) | +| `DS4_GLM_TP_HEAD_SPLIT_MIN` | cached integer token threshold via atoi; default 64; negative values clamp to 0, nonnumeric becomes 0; 0 admits all otherwise-eligible batches | Set the minimum batch size for GLM tensor-parallel output-head splitting. | [ds4.c:38513](ds4.c#L38513) | | `DS4_GLM_VALUE_NO_TILE16` | presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it | Disable the CUDA GLM 16-token tiled value-projection kernel. | [ds4_cuda.cu:36806](ds4_cuda.cu#L36806) | @@ -1119,30 +1123,30 @@ and **19 tool/wrapper entries**. | Variable | Accepted value and default | Effect | Source | | --- | --- | --- | --- | -| `DS4_DSPARK_CACHE_RESERVE_GB` | integer GiB via atoi; default 4.5 GiB; values 1..32 replace it, all other values fall back; decimal/trailing text is truncated/accepted by atoi | Reserve VRAM on DSpark support-cache tiers before packing support-model tensors. | [ds4.c:59580](ds4.c#L59580) | -| `DS4_DSPARK_DISABLE_FINAL_OUTPUT_ALIAS` | presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it | Disable aliasing the final DSpark stage output to the next-stage buffer and use an explicit copy. | [ds4.c:33594](ds4.c#L33594) | -| `DS4_DSPARK_DISABLE_FUSED_CPU_MARKOV_ARGMAX` | cached value-aware kill switch; default off; nonempty value other than exact 0 disables; false/off also disable because only 0 is recognized as false | Disable the fused CPU Markov-bias plus argmax implementation. | [ds4.c:34297](ds4.c#L34297) | -| `DS4_DSPARK_DISABLE_REUSE_CONFIDENCE0_MARKOV` | cached value-aware kill switch; default off; nonempty value other than exact 0 disables; false/off also disable because only 0 is recognized as false | Disable reuse of the first confidence score during Markov proposal. | [ds4.c:34306](ds4.c#L34306) | -| `DS4_DSPARK_DISABLE_VERIFY_SELECTED_PROFILE` | presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it | Override and disable the selected-expert verifier profiler. | [ds4.c:36470](ds4.c#L36470) | -| `DS4_DSPARK_EXEC_TIER` | integer tier via atoi; default is placement/TP free-VRAM heuristic; valid 0..n_gpus-1 overrides; invalid numeric range falls back, but nonnumeric text becomes tier 0 | Choose the GPU tier that executes and primarily caches the DSpark support model. | [ds4.c:59553](ds4.c#L59553) | -| `DS4_DSPARK_FAKE_ARGMAX_PROPOSAL` | nonempty boolean; unset/empty or exact 0: off; every other nonempty value enables, but only while DSpark itself is enabled | If the real DSpark proposer produced no draft, installs a one-token fallback proposal equal to the argmax of the current target logits; debug/test mode also selects the non-fused stage-0 setup path. | [ds4.c:64088](ds4.c#L64088) | -| `DS4_DSPARK_LOW_MEMORY_PREFILL_CHUNK` | unsigned integer rows; default 128; 0 disables the low-memory policy; invalid/overflow falls back, numeric prefixes are accepted; only consulted for Metal SSD+DSpark on <=24 GiB hosts without an explicit chunk | Set the automatic low-memory Metal prefill chunk for SSD-streamed DSpark. | [ds4.c:60533](ds4.c#L60533) | -| `DS4_DSPARK_NO_GPU_MARKOV` | presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it | Disable GPU Markov bias/argmax and the fully device-resident proposal path. | [ds4.c:34521](ds4.c#L34521) | -| `DS4_DSPARK_NO_MARKOV` | cached value-aware kill switch; default off; nonempty value other than exact 0 disables Markov bias; false/off also disable | Disable Markov bias in DSpark proposal generation. | [ds4.c:34288](ds4.c#L34288) | -| `DS4_DSPARK_PROBE` | nonempty-string diagnostic; unset/empty is off, any nonempty value including 0 is on | Log DSpark proposal/probe diagnostics. | [ds4.c:64085](ds4.c#L64085) | -| `DS4_DSPARK_PROP_PROFILE` | presence flag; unset is off; any defined value, including empty or 0, is on; normal eligibility still applies | Print fine-grained timings for DSpark proposal setup. | [ds4.c:32771](ds4.c#L32771) | -| `DS4_DSPARK_SPEC_LOG` | presence flag; unset is off; any defined value, including empty or 0, is on; normal eligibility still applies | Log speculative proposal, verification, acceptance, and fallback decisions. | [ds4.c:66406](ds4.c#L66406) | -| `DS4_DSPARK_SSD_VERIFY_BLOCK_MAX` | unsigned integer rows; default/fallback 0 means automatic policy; numeric prefixes accepted; used both as verifier cap and as an exact-2 proposer-policy discriminator | Cap speculative rows verified from SSD and influence exact-2 proposal sizing. | [ds4.c:52190](ds4.c#L52190) | -| `DS4_DSPARK_STAGE_PROFILE` | presence flag; unset is off; any defined value, including empty or 0, is on; DS4_DSPARK_STAGE_PROFILE_STAGE must also match | Profile DSpark support stages with command-boundary timings. | [ds4.c:33226](ds4.c#L33226) | -| `DS4_DSPARK_STAGE_PROFILE_STAGE` | selector subordinate to DS4_DSPARK_STAGE_PROFILE; unset/empty: match every stage; otherwise strtoul base 10 must consume the whole value, fit uint32_t, and equal the current stage; invalid/out-of-range values match no stage | Restricts DSpark stage-boundary timing output to one stage; it does not enable profiling by itself. | [ds4.c:33227](ds4.c#L33227) | -| `DS4_DSPARK_STATS` | value-aware flag; default off; nonempty value other than exact 0 enables; false/off are treated as enabled | Collect and print aggregate DSpark runtime statistics. | [ds4.c:61417](ds4.c#L61417) | +| `DS4_DSPARK_CACHE_RESERVE_GB` | integer GiB via atoi; default 4.5 GiB; values 1..32 replace it, all other values fall back; decimal/trailing text is truncated/accepted by atoi | Reserve VRAM on DSpark support-cache tiers before packing support-model tensors. | [ds4.c:59770](ds4.c#L59770) | +| `DS4_DSPARK_DISABLE_FINAL_OUTPUT_ALIAS` | presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it | Disable aliasing the final DSpark stage output to the next-stage buffer and use an explicit copy. | [ds4.c:33784](ds4.c#L33784) | +| `DS4_DSPARK_DISABLE_FUSED_CPU_MARKOV_ARGMAX` | cached value-aware kill switch; default off; nonempty value other than exact 0 disables; false/off also disable because only 0 is recognized as false | Disable the fused CPU Markov-bias plus argmax implementation. | [ds4.c:34487](ds4.c#L34487) | +| `DS4_DSPARK_DISABLE_REUSE_CONFIDENCE0_MARKOV` | cached value-aware kill switch; default off; nonempty value other than exact 0 disables; false/off also disable because only 0 is recognized as false | Disable reuse of the first confidence score during Markov proposal. | [ds4.c:34496](ds4.c#L34496) | +| `DS4_DSPARK_DISABLE_VERIFY_SELECTED_PROFILE` | presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it | Override and disable the selected-expert verifier profiler. | [ds4.c:36660](ds4.c#L36660) | +| `DS4_DSPARK_EXEC_TIER` | integer tier via atoi; default is placement/TP free-VRAM heuristic; valid 0..n_gpus-1 overrides; invalid numeric range falls back, but nonnumeric text becomes tier 0 | Choose the GPU tier that executes and primarily caches the DSpark support model. | [ds4.c:59743](ds4.c#L59743) | +| `DS4_DSPARK_FAKE_ARGMAX_PROPOSAL` | nonempty boolean; unset/empty or exact 0: off; every other nonempty value enables, but only while DSpark itself is enabled | If the real DSpark proposer produced no draft, installs a one-token fallback proposal equal to the argmax of the current target logits; debug/test mode also selects the non-fused stage-0 setup path. | [ds4.c:64278](ds4.c#L64278) | +| `DS4_DSPARK_LOW_MEMORY_PREFILL_CHUNK` | unsigned integer rows; default 128; 0 disables the low-memory policy; invalid/overflow falls back, numeric prefixes are accepted; only consulted for Metal SSD+DSpark on <=24 GiB hosts without an explicit chunk | Set the automatic low-memory Metal prefill chunk for SSD-streamed DSpark. | [ds4.c:60714](ds4.c#L60714) | +| `DS4_DSPARK_NO_GPU_MARKOV` | presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it | Disable GPU Markov bias/argmax and the fully device-resident proposal path. | [ds4.c:34711](ds4.c#L34711) | +| `DS4_DSPARK_NO_MARKOV` | cached value-aware kill switch; default off; nonempty value other than exact 0 disables Markov bias; false/off also disable | Disable Markov bias in DSpark proposal generation. | [ds4.c:34478](ds4.c#L34478) | +| `DS4_DSPARK_PROBE` | nonempty-string diagnostic; unset/empty is off, any nonempty value including 0 is on | Log DSpark proposal/probe diagnostics. | [ds4.c:64275](ds4.c#L64275) | +| `DS4_DSPARK_PROP_PROFILE` | presence flag; unset is off; any defined value, including empty or 0, is on; normal eligibility still applies | Print fine-grained timings for DSpark proposal setup. | [ds4.c:32959](ds4.c#L32959) | +| `DS4_DSPARK_SPEC_LOG` | presence flag; unset is off; any defined value, including empty or 0, is on; normal eligibility still applies | Log speculative proposal, verification, acceptance, and fallback decisions. | [ds4.c:66596](ds4.c#L66596) | +| `DS4_DSPARK_SSD_VERIFY_BLOCK_MAX` | unsigned integer rows; default/fallback 0 means automatic policy; numeric prefixes accepted; used both as verifier cap and as an exact-2 proposer-policy discriminator | Cap speculative rows verified from SSD and influence exact-2 proposal sizing. | [ds4.c:52380](ds4.c#L52380) | +| `DS4_DSPARK_STAGE_PROFILE` | presence flag; unset is off; any defined value, including empty or 0, is on; DS4_DSPARK_STAGE_PROFILE_STAGE must also match | Profile DSpark support stages with command-boundary timings. | [ds4.c:33416](ds4.c#L33416) | +| `DS4_DSPARK_STAGE_PROFILE_STAGE` | selector subordinate to DS4_DSPARK_STAGE_PROFILE; unset/empty: match every stage; otherwise strtoul base 10 must consume the whole value, fit uint32_t, and equal the current stage; invalid/out-of-range values match no stage | Restricts DSpark stage-boundary timing output to one stage; it does not enable profiling by itself. | [ds4.c:33417](ds4.c#L33417) | +| `DS4_DSPARK_STATS` | value-aware flag; default off; nonempty value other than exact 0 enables; false/off are treated as enabled | Collect and print aggregate DSpark runtime statistics. | [ds4.c:61607](ds4.c#L61607) | | `DS4_DSPARK_VERIFY_CACHE` | presence diagnostic; unset: off; any presence including empty or 0 enables on each support-cache installation | CUDA only: copies every installed nonempty DSpark/support-cache range back to the host, byte-compares it with its source, and logs each mismatch plus a bad-count summary without changing the install result. | [ds4_cuda.cu:8257](ds4_cuda.cu#L8257) | -| `DS4_DSPARK_VERIFY_HEAD_NO_TP` | presence rollback; unset: allow eligible CUDA output tensor parallelism; any presence including empty or 0 removes the TP path from eligibility | CUDA only: forces the DSpark speculative batched vocabulary head away from output-TP for correctness isolation; under CUDA TP+EP the attempt fails instead of using unavailable full output weights. | [ds4.c:26468](ds4.c#L26468) | +| `DS4_DSPARK_VERIFY_HEAD_NO_TP` | presence rollback; unset: allow eligible CUDA output tensor parallelism; any presence including empty or 0 removes the TP path from eligibility | CUDA only: forces the DSpark speculative batched vocabulary head away from output-TP for correctness isolation; under CUDA TP+EP the attempt fails instead of using unavailable full output weights. | [ds4.c:26641](ds4.c#L26641) | | `DS4_DSPARK_VERIFY_NONCAUSAL` | presence diagnostic sampled once after the first successfully submitted CUDA noncausal-attention kernel; unset: verify 0 calls; any presence including empty or 0: verify that call and the next 2 | CUDA only: synchronizes and reads back Q/KV/output, computes the DSpark noncausal attention CPU reference, and logs max absolute/relative error; it reports only and does not fail the operation. | [ds4_cuda.cu:21736](ds4_cuda.cu#L21736) | -| `DS4_DSPARK_VERIFY_PROFILE` | cached presence diagnostic; unset is off; any defined value including empty/0 profiles only the first eligible verifier invocation | Profile one full DSpark target-verifier invocation layer by layer. | [ds4.c:36567](ds4.c#L36567) | -| `DS4_DSPARK_VERIFY_SELECTED_PROFILE` | presence flag; unset is off; any defined value, including empty or 0, enables unless DS4_DSPARK_DISABLE_VERIFY_SELECTED_PROFILE is also present (disable wins) | Profile selected-expert streaming inside the DSpark verifier. | [ds4.c:36469](ds4.c#L36469) | -| `DS4_DSPARK_VERIFY_SPLIT_HEAD` | nonempty boolean with inverted default; unset/empty or exact 0: fused head; every other nonempty value: split head | Runs the DSpark suffix verifier output head and top-1 reduction in a separate GPU command section after the layer loop, for timing/correctness isolation; default keeps them fused into the layer command section. | [ds4.c:36535](ds4.c#L36535) | -| `DS4_DSPARK_VERIFY_TOPS_CHECK` | presence flag; unset is off; any defined value, including empty or 0, is on; normal eligibility still applies | Read back verifier logits and compare GPU top IDs with CPU argmax. | [ds4.c:36683](ds4.c#L36683) | +| `DS4_DSPARK_VERIFY_PROFILE` | cached presence diagnostic; unset is off; any defined value including empty/0 profiles only the first eligible verifier invocation | Profile one full DSpark target-verifier invocation layer by layer. | [ds4.c:36757](ds4.c#L36757) | +| `DS4_DSPARK_VERIFY_SELECTED_PROFILE` | presence flag; unset is off; any defined value, including empty or 0, enables unless DS4_DSPARK_DISABLE_VERIFY_SELECTED_PROFILE is also present (disable wins) | Profile selected-expert streaming inside the DSpark verifier. | [ds4.c:36659](ds4.c#L36659) | +| `DS4_DSPARK_VERIFY_SPLIT_HEAD` | nonempty boolean with inverted default; unset/empty or exact 0: fused head; every other nonempty value: split head | Runs the DSpark suffix verifier output head and top-1 reduction in a separate GPU command section after the layer loop, for timing/correctness isolation; default keeps them fused into the layer command section. | [ds4.c:36725](ds4.c#L36725) | +| `DS4_DSPARK_VERIFY_TOPS_CHECK` | presence flag; unset is off; any defined value, including empty or 0, is on; normal eligibility still applies | Read back verifier logits and compare GPU top IDs with CPU argmax. | [ds4.c:36873](ds4.c#L36873) | @@ -1151,22 +1155,22 @@ and **19 tool/wrapper entries**. | Variable | Accepted value and default | Effect | Source | | --- | --- | --- | --- | -| `DS4_BATCHED_FFN` | Pure presence flag: any defined value, including empty or "0", enables. Unset leaves the default shared-expert-batched FFN path (or its configured fallback). It is read only by CPU layer-major prefill and takes precedence over shared-batch and token-parallel FFN choices. | Run the complete CPU prefill FFN in chunks through layer_ffn_batch instead of the default shared-expert-only batched path. | [ds4.c:14238](ds4.c#L14238) | -| `DS4_BATCHED_ROPE_MAX` | Nonempty value parsed by strtol without full-string validation; integers 0..65536 are accepted, otherwise default 4096. Zero disables batched RoPE for every nonempty prompt. Effective only when prefix batch attention is selected and DS4_NO_BATCHED_ROPE is absent. | Set the largest CPU prefix-prefill token batch that applies RoPE and inverse RoPE with the batched kernels. | [ds4.c:13738](ds4.c#L13738) | +| `DS4_BATCHED_FFN` | Pure presence flag: any defined value, including empty or "0", enables. Unset leaves the default shared-expert-batched FFN path (or its configured fallback). It is read only by CPU layer-major prefill and takes precedence over shared-batch and token-parallel FFN choices. | Run the complete CPU prefill FFN in chunks through layer_ffn_batch instead of the default shared-expert-only batched path. | [ds4.c:14411](ds4.c#L14411) | +| `DS4_BATCHED_ROPE_MAX` | Nonempty value parsed by strtol without full-string validation; integers 0..65536 are accepted, otherwise default 4096. Zero disables batched RoPE for every nonempty prompt. Effective only when prefix batch attention is selected and DS4_NO_BATCHED_ROPE is absent. | Set the largest CPU prefix-prefill token batch that applies RoPE and inverse RoPE with the batched kernels. | [ds4.c:13911](ds4.c#L13911) | | `DS4_BENCH_DISABLE_SNAPSHOT` | presence flag; unset allows snapshots for eligible frontiers | Disable benchmark state snapshots. | [ds4_bench.c:709](ds4_bench.c#L709) | | `DS4_BENCH_FORCE_SNAPSHOT` | presence flag; unset obeys the normal size/eligibility checks | Force benchmark state snapshots despite the normal limit. | [ds4_bench.c:712](ds4_bench.c#L712) | | `DS4_BENCH_SNAPSHOT_MAX_BYTES` | unsigned bytes or unlimited/inf; default DS4_BENCH_DEFAULT_SNAPSHOT_MAX_BYTES | Limit session snapshot size during benchmark sweeps. | [ds4_bench.c:70](ds4_bench.c#L70) | | `DS4_CHROME` | executable path; unset auto-detects Chrome/Chromium via standard paths and PATH | Select the browser executable used by web tooling. | [ds4_web.c:1014](ds4_web.c#L1014) | | `DS4_CLI_FORCE_SESSION` | Pure presence flag: any defined value, including empty or "0", forces the session path. Unset uses the session path only for distributed coordinators, TP leaders, temperature>0, or MTP depth>1; otherwise the CLI calls direct argmax generation. | Force ordinary CLI generation through run_sampled_generation/session APIs so single-node validation follows the same stateful path as TP/distributed runs. | [ds4_cli.c:1226](ds4_cli.c#L1226) | -| `DS4_CPU_DISABLE_UNROLLED_ARGMAX` | presence rollback flag; unset keeps optimized/default path | Disable/roll back cpu disable unrolled argmax. | [ds4.c:40885](ds4.c#L40885) | -| `DS4_CPU_DUMP_LOGITS` | filesystem path; unset disables read/write | Dump or select diagnostic data for cpu dump logits. | [ds4.c:38896](ds4.c#L38896) | -| `DS4_CPU_DUMP_PREFILL_LOGITS` | filesystem path; unset disables read/write | Dump or select diagnostic data for cpu dump prefill logits. | [ds4.c:41416](ds4.c#L41416) | -| `DS4_DECODE_PROFILE_DETAIL` | presence flag; unset=off | Print per-stage timing for the single-token CPU FFN path. | [ds4.c:12036](ds4.c#L12036) | -| `DS4_EXPERT_HOTLIST` | nonempty filesystem path; unset=off; currently Metal-only | Load an expert hotlist for Metal expert profiling/streaming. | [ds4.c:60179](ds4.c#L60179) | -| `DS4_EXPERT_PROFILE` | presence diagnostic flag; unset=off | Collect timing/profile diagnostics for expert profile. | [ds4.c:60177](ds4.c#L60177) | +| `DS4_CPU_DISABLE_UNROLLED_ARGMAX` | presence rollback flag; unset keeps optimized/default path | Disable/roll back cpu disable unrolled argmax. | [ds4.c:41075](ds4.c#L41075) | +| `DS4_CPU_DUMP_LOGITS` | filesystem path; unset disables read/write | Dump or select diagnostic data for cpu dump logits. | [ds4.c:39086](ds4.c#L39086) | +| `DS4_CPU_DUMP_PREFILL_LOGITS` | filesystem path; unset disables read/write | Dump or select diagnostic data for cpu dump prefill logits. | [ds4.c:41606](ds4.c#L41606) | +| `DS4_DECODE_PROFILE_DETAIL` | presence flag; unset=off | Print per-stage timing for the single-token CPU FFN path. | [ds4.c:12209](ds4.c#L12209) | +| `DS4_EXPERT_HOTLIST` | nonempty filesystem path; unset=off; currently Metal-only | Load an expert hotlist for Metal expert profiling/streaming. | [ds4.c:60369](ds4.c#L60369) | +| `DS4_EXPERT_PROFILE` | presence diagnostic flag; unset=off | Collect timing/profile diagnostics for expert profile. | [ds4.c:60367](ds4.c#L60367) | | `DS4_FORCE_CUDA_PEER` | presence flag read once at CUDA init; unset uses automatic transfer selection; any defined value including 0 enables | Force cross-device transfers through cudaMemcpyPeerAsync for diagnostics. | [ds4_cuda.cu:364](ds4_cuda.cu#L364) | | `DS4_FORCE_HOST_BOUNCE` | presence flag read once at CUDA init; unset uses automatic transfer selection; any defined value including 0 enables | Force cross-device transfers through pinned host bounce buffers for diagnostics. | [ds4_cuda.cu:365](ds4_cuda.cu#L365) | -| `DS4_LOCK_FILE` | path string; default /tmp/ds4.lock | Override the single-instance lock file. | [ds4.c:51809](ds4.c#L51809) | +| `DS4_LOCK_FILE` | path string; default /tmp/ds4.lock | Override the single-instance lock file. | [ds4.c:51999](ds4.c#L51999) | | `DS4_MMID_CASE1` | boolean-ish cached flag; default on; a value starting with 0 disables | Disable the single-expert MM-IDs specialized fast path for comparison. | [cuda/mmq/mmid.cu:290](cuda/mmq/mmid.cu#L290) | | `DS4_MMID_LARGE` | boolean-ish cached flag; default on; a value starting with 0 disables | Control the large-N global-memory MM-IDs path used beyond shared-memory capacity. | [cuda/mmq/mmid.cu:245](cuda/mmq/mmid.cu#L245) | | `DS4_MMQ_D2R` | boolean-ish cached flag; default on; a value starting with 0 disables | Control the direct-to-register Q2_K MoE down path. | [cuda/mmq/ds4_mmq.cu:505](cuda/mmq/ds4_mmq.cu#L505) | @@ -1178,53 +1182,53 @@ and **19 tool/wrapper entries**. | `DS4_MMQ_OUT_MEMSET` | exact 1 enables; unset or every other value disables; cached | Restore blanket MMQ output-buffer zeroing for diagnostics. | [cuda/mmq/ds4_mmq.cu:532](cuda/mmq/ds4_mmq.cu#L532) | | `DS4_MMQ_YBUF_MEMSET` | unset or 0 disables; 1 zero-fills; a value starting with p or P poison-fills with 0xFF | Control MMQ Q8_1 activation-staging initialization and its poison oracle. | [cuda/mmq/ds4_mmq.cu:558](cuda/mmq/ds4_mmq.cu#L558) | | `DS4_MMQ_YIND_VERIFY` | presence diagnostic; unset is off; any defined value including 0 enables | Byte-compare Y-indirect and slot-gathered MoE activation buffers. | [cuda/mmq/ds4_mmq.cu:600](cuda/mmq/ds4_mmq.cu#L600) | -| `DS4_MOE_RECORD_SELECTED_HOTLIST` | nonempty output path; unset=off | Record per-layer selected-expert hit counts to a Metal hotlist file. | [ds4_metal.m:1740](ds4_metal.m#L1740) | -| `DS4_MOE_RECORD_SELECTED_HOTLIST_FRESH` | presence flag; only relevant with HOTLIST; overrides MERGE | Start the selected-expert hotlist from empty state. | [ds4_metal.m:1641](ds4_metal.m#L1641) | -| `DS4_MOE_RECORD_SELECTED_HOTLIST_MERGE` | presence flag; active only when FRESH is absent | Merge an existing selected-expert hotlist before recording. | [ds4_metal.m:1640](ds4_metal.m#L1640) | -| `DS4_MOE_RECORD_SELECTED_IDS` | nonempty output path; unset=off | Record routed-MoE six-expert selections; also disables incompatible optimized paths. | [ds4.c:65100](ds4.c#L65100) | -| `DS4_MOE_REPLAY_SELECTED_IDS` | nonempty input path; unset=off | Replay routed-MoE six-expert selections; also disables incompatible optimized paths. | [ds4.c:25135](ds4.c#L25135) | -| `DS4_MTP_BATCH_VERIFY` | Pure presence flag: any defined value, including empty or "0", suppresses the exact two-row decode verifier. Unset selects exact decode-2 when draft_n==2 and either strict mode is active or the build is ROCm; other cases already use the generic verifier. | Diagnostic rollback from the exact Q8/one-token-equivalent MTP decode-2 verifier to the generic microbatch verifier. | [ds4.c:70842](ds4.c#L70842) | -| `DS4_MTP_CAPTURE_PREFIX1` | Pure presence flag. In the generic verifier with exactly two drafts it enables prefix-1 state capture under strict mode; non-strict mode already captures prefix-1 without the variable. Unset under strict mode instead snapshots and replays a partial acceptance. | Let a one-of-two MTP partial acceptance commit the verifier's captured prefix directly, avoiding an exact one-token replay. | [ds4.c:70948](ds4.c#L70948) | -| `DS4_MTP_CONF_LOG` | Pure presence flag; default off. It forces materialization of full draft logits, computes the top-2 margin, and after a successful generic microbatch verification prints drafted/committed counts, top candidates, margin, target-next and draft-next. Exact decode-2 success does not emit that generic log line. | Inspect MTP draft confidence and compare the recursive draft token with the target verifier result. | [ds4.c:70710](ds4.c#L70710) | -| `DS4_MTP_EXACT_REPLAY` | Pure presence flag; default off. In the generic microbatch verifier it forces a pre-verifier frontier snapshot; after verification the snapshot is restored and every accepted draft is decoded sequentially to rebuild exact final state/logits. | Validate MTP acceptance while committing through the normal one-token decode path rather than retaining batched-verifier state. | [ds4.c:70949](ds4.c#L70949) | -| `DS4_MTP_FORCE_SNAPSHOT` | Pure presence flag; default off. It forces a speculative-frontier snapshot before the generic verifier regardless of draft count or prefix-capture mode; it does not by itself force restoration or replay after a successful full acceptance. | Measure/debug snapshot behavior and guarantee a restorable pre-verifier frontier for generic MTP verification. | [ds4.c:70953](ds4.c#L70953) | -| `DS4_MTP_FULL_LOGITS` | Pure presence flag; default off. When set, legacy and recursive MTP draft calls write the full vocabulary logits to s->mtp_logits; unset permits the faster top-token-only output unless confidence/margin logic independently needs logits. | Force full MTP draft-logit materialization for correctness comparison, inspection, or downstream confidence calculations. | [ds4.c:64070](ds4.c#L64070) | -| `DS4_MTP_MIN_MARGIN` | non-negative float; default engine --mtp-margin value | Set confidence margin threshold for speculative MTP verification. | [ds4.c:70703](ds4.c#L70703) | -| `DS4_MTP_PROBE` | Pure presence flag; default off. For legacy MTP it prepares drafts even when configured depth<=1, compares the previous draft with the next committed token, and prints cumulative hit counts/failures; generated output is unchanged. | Measure legacy MTP next-token draft accuracy without enabling speculative acceptance. | [ds4.c:64800](ds4.c#L64800) | +| `DS4_MOE_RECORD_SELECTED_HOTLIST` | nonempty output path; unset=off | Record per-layer selected-expert hit counts to a Metal hotlist file. | [ds4_metal.m:1741](ds4_metal.m#L1741) | +| `DS4_MOE_RECORD_SELECTED_HOTLIST_FRESH` | presence flag; only relevant with HOTLIST; overrides MERGE | Start the selected-expert hotlist from empty state. | [ds4_metal.m:1642](ds4_metal.m#L1642) | +| `DS4_MOE_RECORD_SELECTED_HOTLIST_MERGE` | presence flag; active only when FRESH is absent | Merge an existing selected-expert hotlist before recording. | [ds4_metal.m:1641](ds4_metal.m#L1641) | +| `DS4_MOE_RECORD_SELECTED_IDS` | nonempty output path; unset=off | Record routed-MoE six-expert selections; also disables incompatible optimized paths. | [ds4.c:65290](ds4.c#L65290) | +| `DS4_MOE_REPLAY_SELECTED_IDS` | nonempty input path; unset=off | Replay routed-MoE six-expert selections; also disables incompatible optimized paths. | [ds4.c:25308](ds4.c#L25308) | +| `DS4_MTP_BATCH_VERIFY` | Pure presence flag: any defined value, including empty or "0", suppresses the exact two-row decode verifier. Unset selects exact decode-2 when draft_n==2 and either strict mode is active or the build is ROCm; other cases already use the generic verifier. | Diagnostic rollback from the exact Q8/one-token-equivalent MTP decode-2 verifier to the generic microbatch verifier. | [ds4.c:71021](ds4.c#L71021) | +| `DS4_MTP_CAPTURE_PREFIX1` | Pure presence flag. In the generic verifier with exactly two drafts it enables prefix-1 state capture under strict mode; non-strict mode already captures prefix-1 without the variable. Unset under strict mode instead snapshots and replays a partial acceptance. | Let a one-of-two MTP partial acceptance commit the verifier's captured prefix directly, avoiding an exact one-token replay. | [ds4.c:71134](ds4.c#L71134) | +| `DS4_MTP_CONF_LOG` | Pure presence flag; default off. It forces materialization of full draft logits, computes the top-2 margin, and after a successful generic microbatch verification prints drafted/committed counts, top candidates, margin, target-next and draft-next. Exact decode-2 success does not emit that generic log line. | Inspect MTP draft confidence and compare the recursive draft token with the target verifier result. | [ds4.c:70900](ds4.c#L70900) | +| `DS4_MTP_EXACT_REPLAY` | Pure presence flag; default off. In the generic microbatch verifier it forces a pre-verifier frontier snapshot; after verification the snapshot is restored and every accepted draft is decoded sequentially to rebuild exact final state/logits. | Validate MTP acceptance while committing through the normal one-token decode path rather than retaining batched-verifier state. | [ds4.c:71139](ds4.c#L71139) | +| `DS4_MTP_FORCE_SNAPSHOT` | Pure presence flag; default off. It forces a speculative-frontier snapshot before the generic verifier regardless of draft count or prefix-capture mode; it does not by itself force restoration or replay after a successful full acceptance. | Measure/debug snapshot behavior and guarantee a restorable pre-verifier frontier for generic MTP verification. | [ds4.c:71143](ds4.c#L71143) | +| `DS4_MTP_FULL_LOGITS` | Pure presence flag; default off. When set, legacy and recursive MTP draft calls write the full vocabulary logits to s->mtp_logits; unset permits the faster top-token-only output unless confidence/margin logic independently needs logits. | Force full MTP draft-logit materialization for correctness comparison, inspection, or downstream confidence calculations. | [ds4.c:64260](ds4.c#L64260) | +| `DS4_MTP_MIN_MARGIN` | non-negative float; default engine --mtp-margin value | Set confidence margin threshold for speculative MTP verification. | [ds4.c:70893](ds4.c#L70893) | +| `DS4_MTP_PROBE` | Pure presence flag; default off. For legacy MTP it prepares drafts even when configured depth<=1, compares the previous draft with the next committed token, and prints cumulative hit counts/failures; generated output is unchanged. | Measure legacy MTP next-token draft accuracy without enabling speculative acceptance. | [ds4.c:64990](ds4.c#L64990) | | `DS4_MTP_SPEC_DISABLE` | Pure presence flag: any defined value, including empty or "0", disables MTP speculative argmax in CLI/chat/server loops. Unset permits it for greedy temperature<=0 generation with draft depth>1; unrelated split-KV speculation can still be independently requested. | Fall back from MTP multi-token speculative evaluation to normal one-token session evaluation. | [ds4_cli.c:580](ds4_cli.c#L580) | -| `DS4_MTP_SPEC_LOG` | Pure presence flag; default off. It only emits diagnostics for first-draft misses, exact/generic verifier failures and sequential fallback misses/acceptance outcomes; it does not select a verifier. | Trace why MTP drafts were accepted, partially accepted, rejected, or sent to sequential fallback. | [ds4.c:70726](ds4.c#L70726) | -| `DS4_MTP_STRICT` | Pure presence flag; engine quality mode also enables strictness automatically. Strict mode skips the non-strict low-margin shortcut, selects exact decode-2 for two drafts unless DS4_MTP_BATCH_VERIFY is set, and disables default prefix-1 capture unless explicitly restored. | Force the exact/quality-oriented MTP verification policy on otherwise non-quality runs. | [ds4.c:70701](ds4.c#L70701) | -| `DS4_MTP_TIMING` | Pure presence flag; default off. When set, timestamps and prints draft, snapshot, verifier, prefix/replay and total durations for the path taken; algorithm selection is otherwise unchanged. | Profile end-to-end MTP speculative decoding and separate draft, verification and state-commit costs. | [ds4.c:70709](ds4.c#L70709) | -| `DS4_NO_BATCHED_ATTN` | presence rollback flag; unset keeps default/optimized path | Disable/roll back no batched attn. | [ds4.c:14237](ds4.c#L14237) | -| `DS4_NO_BATCHED_ROPE` | presence rollback flag; unset keeps default/optimized path | Disable/roll back no batched rope. | [ds4.c:13745](ds4.c#L13745) | -| `DS4_NO_PARALLEL_ATTN_ROWS` | presence rollback flag; unset keeps default/optimized path | Disable/roll back no parallel attn rows. | [ds4.c:13731](ds4.c#L13731) | -| `DS4_NO_ROUTED_TOKEN_PARALLEL` | presence rollback flag; unset keeps default/optimized path | Disable/roll back no routed token parallel. | [ds4.c:12415](ds4.c#L12415) | -| `DS4_NO_SHARED_BATCH_FFN` | presence rollback flag; unset keeps default/optimized path | Disable/roll back no shared batch ffn. | [ds4.c:14240](ds4.c#L14240) | -| `DS4_ORACLE_LOGITS` | filesystem path; unset disables read/write | Load reference logits for graph correctness comparison. | [ds4.c:38866](ds4.c#L38866) | -| `DS4_PARALLEL_ATTN_ROWS` | Pure presence opt-in; any defined value enables the preference, but DS4_NO_PARALLEL_ATTN_ROWS overrides it. The path is eligible only for prefix prefill with cache n_raw==0 and pos0==0; unset uses per-token attention rows. | Batch/parallelize CPU prefix attention-row evaluation after cache/index preparation. | [ds4.c:13728](ds4.c#L13728) | -| `DS4_PARALLEL_FFN` | Pure presence opt-in. It is effective only in CPU prefill when batched attention is enabled, DS4_BATCHED_FFN is absent, and the default shared-batch path has been disabled with DS4_NO_SHARED_BATCH_FFN; otherwise higher-priority paths win. | Run independent prompt-token FFNs through layer_ffn_tokens_parallel as the fallback after disabling shared batching. | [ds4.c:14239](ds4.c#L14239) | -| `DS4_PREFILL_BATCH` | Nonempty value parsed by strtol without full-string validation; accepted range 1..4095, default 128 for unset/invalid/out-of-range values. It is used only when DS4_BATCHED_FFN selects full batched CPU FFN. | Set the token chunk size for layer_ffn_batch during CPU layer-major prefill. | [ds4.c:14241](ds4.c#L14241) | -| `DS4_PREFILL_PROFILE_DETAIL` | presence flag; unset=off | Print detailed per-stage CPU prefill timing. | [ds4.c:12382](ds4.c#L12382) | -| `DS4_PREFILL_PROFILE_TOKEN` | presence flag; effective within detailed prefill profiling | Print token-loop substage timings during CPU prefill. | [ds4.c:13957](ds4.c#L13957) | -| `DS4_Q8_FOLD_SELFTEST` | positive call budget; unset/empty disables; a nonempty value parsing to 1 or less selects 512 calls | Byte-check folded Q8_1 activations against a fresh quantization; synchronizes eager streams. | [cuda/mmq/ds4_mmq.cu:5744](cuda/mmq/ds4_mmq.cu#L5744) | -| `DS4_ROUTED_TOKEN_PARALLEL` | Pure presence flag that forces token-parallel routed MoE, even if DS4_NO_ROUTED_TOKEN_PARALLEL is also set. When unset, token parallelism is automatic for n_tok>=64 unless the NO flag is present; smaller batches use per-token routed MoE. | Choose token-parallel CPU routed-expert evaluation inside the default shared-batch FFN prefill path. | [ds4.c:12414](ds4.c#L12414) | +| `DS4_MTP_SPEC_LOG` | Pure presence flag; default off. It only emits diagnostics for first-draft misses, exact/generic verifier failures and sequential fallback misses/acceptance outcomes; it does not select a verifier. | Trace why MTP drafts were accepted, partially accepted, rejected, or sent to sequential fallback. | [ds4.c:70916](ds4.c#L70916) | +| `DS4_MTP_STRICT` | Pure presence flag; engine quality mode also enables strictness automatically. Strict mode skips the non-strict low-margin shortcut, selects exact decode-2 for two drafts unless DS4_MTP_BATCH_VERIFY is set, and disables default prefix-1 capture unless explicitly restored. | Force the exact/quality-oriented MTP verification policy on otherwise non-quality runs. | [ds4.c:70891](ds4.c#L70891) | +| `DS4_MTP_TIMING` | Pure presence flag; default off. When set, timestamps and prints draft, snapshot, verifier, prefix/replay and total durations for the path taken; algorithm selection is otherwise unchanged. | Profile end-to-end MTP speculative decoding and separate draft, verification and state-commit costs. | [ds4.c:70899](ds4.c#L70899) | +| `DS4_NO_BATCHED_ATTN` | presence rollback flag; unset keeps default/optimized path | Disable/roll back no batched attn. | [ds4.c:14410](ds4.c#L14410) | +| `DS4_NO_BATCHED_ROPE` | presence rollback flag; unset keeps default/optimized path | Disable/roll back no batched rope. | [ds4.c:13918](ds4.c#L13918) | +| `DS4_NO_PARALLEL_ATTN_ROWS` | presence rollback flag; unset keeps default/optimized path | Disable/roll back no parallel attn rows. | [ds4.c:13904](ds4.c#L13904) | +| `DS4_NO_ROUTED_TOKEN_PARALLEL` | presence rollback flag; unset keeps default/optimized path | Disable/roll back no routed token parallel. | [ds4.c:12588](ds4.c#L12588) | +| `DS4_NO_SHARED_BATCH_FFN` | presence rollback flag; unset keeps default/optimized path | Disable/roll back no shared batch ffn. | [ds4.c:14413](ds4.c#L14413) | +| `DS4_ORACLE_LOGITS` | filesystem path; unset disables read/write | Load reference logits for graph correctness comparison. | [ds4.c:39056](ds4.c#L39056) | +| `DS4_PARALLEL_ATTN_ROWS` | Pure presence opt-in; any defined value enables the preference, but DS4_NO_PARALLEL_ATTN_ROWS overrides it. The path is eligible only for prefix prefill with cache n_raw==0 and pos0==0; unset uses per-token attention rows. | Batch/parallelize CPU prefix attention-row evaluation after cache/index preparation. | [ds4.c:13901](ds4.c#L13901) | +| `DS4_PARALLEL_FFN` | Pure presence opt-in. It is effective only in CPU prefill when batched attention is enabled, DS4_BATCHED_FFN is absent, and the default shared-batch path has been disabled with DS4_NO_SHARED_BATCH_FFN; otherwise higher-priority paths win. | Run independent prompt-token FFNs through layer_ffn_tokens_parallel as the fallback after disabling shared batching. | [ds4.c:14412](ds4.c#L14412) | +| `DS4_PREFILL_BATCH` | Nonempty value parsed by strtol without full-string validation; accepted range 1..4095, default 128 for unset/invalid/out-of-range values. It is used only when DS4_BATCHED_FFN selects full batched CPU FFN. | Set the token chunk size for layer_ffn_batch during CPU layer-major prefill. | [ds4.c:14414](ds4.c#L14414) | +| `DS4_PREFILL_PROFILE_DETAIL` | presence flag; unset=off | Print detailed per-stage CPU prefill timing. | [ds4.c:12555](ds4.c#L12555) | +| `DS4_PREFILL_PROFILE_TOKEN` | presence flag; effective within detailed prefill profiling | Print token-loop substage timings during CPU prefill. | [ds4.c:14130](ds4.c#L14130) | +| `DS4_Q8_FOLD_SELFTEST` | positive call budget; unset/empty disables; a nonempty value parsing to 1 or less selects 512 calls | Byte-check folded Q8_1 activations against a fresh quantization; synchronizes eager streams. | [cuda/mmq/ds4_mmq.cu:5921](cuda/mmq/ds4_mmq.cu#L5921) | +| `DS4_ROUTED_TOKEN_PARALLEL` | Pure presence flag that forces token-parallel routed MoE, even if DS4_NO_ROUTED_TOKEN_PARALLEL is also set. When unset, token parallelism is automatic for n_tok>=64 unless the NO flag is present; smaller batches use per-token routed MoE. | Choose token-parallel CPU routed-expert evaluation inside the default shared-batch FFN prefill path. | [ds4.c:12587](ds4.c#L12587) | | `DS4_SERVER_BATCH_LOG` | Pure presence flag read once when the decode worker starts; default off. Any defined value, including empty or "0", logs one record per coalesced decode batch with count, elapsed milliseconds and ok/error status. | Observe server-side decode coalescing size, latency and result without changing batching behavior. | [ds4_server.c:11090](ds4_server.c#L11090) | | `DS4_SERVER_DECODE_COALESCE_US` | integer 0..100000 microseconds; default 2000; 0 disables wait | Control server micro-batch coalescing delay. | [ds4_server.c:11069](ds4_server.c#L11069) | | `DS4_SSD_AUTO_CACHE_PCT` | integer 50..95; default 80 | Choose the RAM percentage used by automatic SSD expert-cache planning. | [ds4_ssd.c:81](ds4_ssd.c#L81) | -| `DS4_TEST_METAL_EXACTN_ORACLE` | presence flag compiled only with DS4_TEST_HOOKS; unset is off; any defined value enables | Force allocation of the Metal exact-N verifier/oracle workspace in test builds. | [ds4.c:61801](ds4.c#L61801) | +| `DS4_TEST_METAL_EXACTN_ORACLE` | presence flag compiled only with DS4_TEST_HOOKS; unset is off; any defined value enables | Force allocation of the Metal exact-N verifier/oracle workspace in test builds. | [ds4.c:61991](ds4.c#L61991) | | `DS4_THREADS` | positive integer; default min(online CPUs,12), capped by DS4_MAX_THREADS; CLI thread request overrides env | Set CPU worker-pool size. | [ds4.c:1874](ds4.c#L1874) | -| `DS4_TOKEN_TIMING` | Pure presence flag; default off. Any defined value times and prints each CPU token decode evaluation; sampling, emission and callbacks are outside the measured interval. | Report per-token CPU model-evaluation latency during direct argmax generation. | [ds4.c:41429](ds4.c#L41429) | -| `DS4_TP_ABLATE` | comma/list string matched for hcpre,router,kv,compidx; unset=no ablation; must match on both ranks | Skip named TP encode chains for timing; output is semantically wrong. | [ds4.c:22355](ds4.c#L22355) | -| `DS4_TP_EVENT_GATES` | presence flag; unset uses lower-latency slab flag gates when available | Fall back to Metal shared-event arrival gates. | [ds4_metal.m:10769](ds4_metal.m#L10769) | -| `DS4_TP_GATE_PROFILE` | presence diagnostic flag; unset=off | Collect timing/profile diagnostics for tp gate profile. | [ds4_metal.m:10661](ds4_metal.m#L10661) | +| `DS4_TOKEN_TIMING` | Pure presence flag; default off. Any defined value times and prints each CPU token decode evaluation; sampling, emission and callbacks are outside the measured interval. | Report per-token CPU model-evaluation latency during direct argmax generation. | [ds4.c:41619](ds4.c#L41619) | +| `DS4_TP_ABLATE` | comma/list string matched for hcpre,router,kv,compidx; unset=no ablation; must match on both ranks | Skip named TP encode chains for timing; output is semantically wrong. | [ds4.c:22520](ds4.c#L22520) | +| `DS4_TP_EVENT_GATES` | presence flag; unset uses lower-latency slab flag gates when available | Fall back to Metal shared-event arrival gates. | [ds4_metal.m:10768](ds4_metal.m#L10768) | +| `DS4_TP_GATE_PROFILE` | presence diagnostic flag; unset=off | Collect timing/profile diagnostics for tp gate profile. | [ds4_metal.m:10662](ds4_metal.m#L10662) | | `DS4_TP_GATE_TRACE` | presence diagnostic flag; unset=off | Emit trace diagnostics for tp gate trace. | [ds4_tp.c:911](ds4_tp.c#L911) | -| `DS4_TP_KEEPALIVE_ITERS` | atoi unsigned iteration count; default 1200000 | Tune work per Metal TP keep-alive dispatch. | [ds4_metal.m:10625](ds4_metal.m#L10625) | -| `DS4_TP_KEEPALIVE_TGS` | integer 1..2048; invalid/out of range uses 1 | Tune threadgroups per Metal TP keep-alive dispatch. | [ds4_metal.m:10611](ds4_metal.m#L10611) | -| `DS4_TP_NO_KEEPALIVE` | presence flag; unset starts Metal TP keep-alive | Disable the Metal TP GPU keep-alive worker. | [ds4_metal.m:10797](ds4_metal.m#L10797) | -| `DS4_TP_PREFILL_SPLIT_MIN` | atoi token threshold; default 32; values below 2 clamp to 2 | Set when TP prefill row-splits the replicated shared expert. | [ds4.c:29019](ds4.c#L29019) | -| `DS4_TP_SUBGATE_PIPELINE` | nonempty integer; nonzero enables; default off; must match on both ranks | Enable TP prefill sub-chunk gate pipelining. | [ds4.c:29033](ds4.c#L29033) | +| `DS4_TP_KEEPALIVE_ITERS` | atoi unsigned iteration count; default 1200000 | Tune work per Metal TP keep-alive dispatch. | [ds4_metal.m:10626](ds4_metal.m#L10626) | +| `DS4_TP_KEEPALIVE_TGS` | integer 1..2048; invalid/out of range uses 1 | Tune threadgroups per Metal TP keep-alive dispatch. | [ds4_metal.m:10612](ds4_metal.m#L10612) | +| `DS4_TP_NO_KEEPALIVE` | presence flag; unset starts Metal TP keep-alive | Disable the Metal TP GPU keep-alive worker. | [ds4_metal.m:10798](ds4_metal.m#L10798) | +| `DS4_TP_PREFILL_SPLIT_MIN` | atoi token threshold; default 32; values below 2 clamp to 2 | Set when TP prefill row-splits the replicated shared expert. | [ds4.c:29192](ds4.c#L29192) | +| `DS4_TP_SUBGATE_PIPELINE` | nonempty integer; nonzero enables; default off; must match on both ranks | Enable TP prefill sub-chunk gate pipelining. | [ds4.c:29206](ds4.c#L29206) | | `DS4_TP_TIMEOUT_SEC` | atoi seconds stored unsigned; default DS4_TP_DEFAULT_TIMEOUT_SEC | Override TP control/data socket operation timeout. | [ds4_tp.c:1329](ds4_tp.c#L1329) | -| `DS4_TRACE_TOP` | presence flag; unset=off | Print top-logit/token trace data during CPU generation. | [ds4.c:41398](ds4.c#L41398) | +| `DS4_TRACE_TOP` | presence flag; unset=off | Print top-logit/token trace data during CPU generation. | [ds4.c:41588](ds4.c#L41588) | | `DS4_WS_REPACK_HASH` | exact 1 enables; unset or every other value disables unless overridden by CLI; cached | Print a per-artifact FNV-1a hash for workspace repack identity checks. | [cuda/mmq/ds4_repack.cu:530](cuda/mmq/ds4_repack.cu#L530) | | `DS4_WS_REPACK_THREADS` | positive integer; default min(6, hardware threads), capped at 16 and the job count | Set the CPU worker count for CUDA workspace artifact repacking. | [cuda/mmq/ds4_repack.cu:539](cuda/mmq/ds4_repack.cu#L539) | @@ -1268,7 +1272,7 @@ them as part of its own test contract. | `DS4_TEST_LONG_PROMPT` | nonempty readable prompt-file path; unset/empty defaults to tests/long_context_story_prompt.txt | Selects the rendered story prompt for the long-context fact-recall test. | [tests/ds4_test.c:6690](tests/ds4_test.c#L6690) | | `DS4_TEST_LONG_WORDS` | atoi integer; unset/empty/nonnumeric defaults to 0; valid range is 0..DS4_TEST_CONTEXT-128 and numeric prefixes are accepted | Adds repeated words to alternating CUDA session-batch prompts to exercise long-prefill rows. | [tests/test_cuda_session_batch.c:135](tests/test_cuda_session_batch.c#L135) | | `DS4_TEST_METAL_EXACTN_BATCH_HEAD` | nonempty value other than exact 0 enables; unset/empty/0 disables; false/off also enable | Enables the Metal exact-N batch-head path and requires its attempt/use counters for every eligible oracle case. | [tests/test_metal_exactn_oracle.c:401](tests/test_metal_exactn_oracle.c#L401) | -| `DS4_TEST_METAL_EXACTN_ORACLE` | presence flag compiled only with DS4_TEST_HOOKS; absent from normal production builds | Force allocation of the exact-N Metal verifier/oracle workspace in tests. | [ds4.c:61801](ds4.c#L61801) | +| `DS4_TEST_METAL_EXACTN_ORACLE` | presence flag compiled only with DS4_TEST_HOOKS; absent from normal production builds | Force allocation of the exact-N Metal verifier/oracle workspace in tests. | [ds4.c:61991](ds4.c#L61991) | | `DS4_TEST_MIXED_INITIAL` | integer 128..context-1; default 128 | Set the initial prefill length for the CUDA mixed-batch oracle. | [tests/test_cuda_mixed_batch.c:111](tests/test_cuda_mixed_batch.c#L111) | | `DS4_TEST_MIXED_QUANTUM` | integer 1..context-1; default 128 | Set the number of prompt tokens added per CUDA mixed-batch round. | [tests/test_cuda_mixed_batch.c:113](tests/test_cuda_mixed_batch.c#L113) | | `DS4_TEST_MIXED_ROUNDS` | integer 1..64; default 3 | Set the number of CUDA mixed-batch oracle rounds. | [tests/test_cuda_mixed_batch.c:115](tests/test_cuda_mixed_batch.c#L115) | @@ -1282,7 +1286,7 @@ them as part of its own test contract. | `DS4_TEST_Q4_STREAM_TIMING_ITERS` | integer 1..10000; default 20 | Set timing iterations for the Metal Q4 stream oracle. | [tests/test_metal_q4_streams.c:741](tests/test_metal_q4_streams.c#L741) | | `DS4_TEST_Q4_STREAM_WARMUP` | integer 1..64; default 1 | Set warmup iterations for the Metal Q4 stream oracle. | [tests/test_metal_q4_streams.c:732](tests/test_metal_q4_streams.c#L732) | | `DS4_TEST_REQUIRE_MODEL` | nonempty value other than exact 0 requires a readable model; unset/empty/0 permits a skip; false/off count as required | Turns a missing Metal exact-N oracle model from a developer skip into a release-gate failure. | [tests/test_metal_exactn_oracle.c:390](tests/test_metal_exactn_oracle.c#L390) | -| `DS4_TEST_REQUIRE_ROCM_DEVICE` | nonempty value other than exact 0 requires a visible ROCm device; unset/empty/0 returns the fixture skip code; false/off count as required | Turns absence of a ROCm device from a skip into failure for the ROCm Q4 oracle. | [tests/test_rocm_q4_dense_pair.cpp:1483](tests/test_rocm_q4_dense_pair.cpp#L1483) | +| `DS4_TEST_REQUIRE_ROCM_DEVICE` | nonempty value other than exact 0 requires a visible ROCm device; unset/empty/0 returns the fixture skip code; false/off count as required | Turns absence of a ROCm device from a skip into failure for the ROCm Q4 oracle. | [tests/test_rocm_q4_dense_pair.cpp:1569](tests/test_rocm_q4_dense_pair.cpp#L1569) | | `DS4_TEST_SERVER_PREFILL` | presence flag; unset: normal prefill; any defined value including empty or 0 installs a no-op display-progress callback | Exercises the progress-split prefill path used by ds4-server in CUDA session-batch and control sessions. | [tests/test_cuda_session_batch.c:110](tests/test_cuda_session_batch.c#L110) | | `DS4_TEST_SESSION_BATCH_ARM` | arbitrary nonempty label; unset/empty defaults to unspecified; it is logged only and does not alter execution | Labels the Metal session-batch experiment arm in setup diagnostics. | [tests/test_metal_session_batch.c:153](tests/test_metal_session_batch.c#L153) | | `DS4_TEST_SESSION_BATCH_TIMING` | nonempty boolean; unset/empty/exact 0 disables; every other value enables | Print timing data from the Metal session-batch oracle. | [tests/test_metal_session_batch.c:150](tests/test_metal_session_batch.c#L150) | diff --git a/scripts/environment_variables.tsv b/scripts/environment_variables.tsv index 81de9aa80..36675eaf6 100644 --- a/scripts/environment_variables.tsv +++ b/scripts/environment_variables.tsv @@ -9,77 +9,77 @@ runtime/bench DS4_BENCH_DISABLE_SNAPSHOT presence flag; unset allows snapshots f runtime/bench DS4_BENCH_FORCE_SNAPSHOT presence flag; unset obeys the normal size/eligibility checks Force benchmark state snapshots despite the normal limit. ds4_bench.c:712 runtime/bench DS4_BENCH_SNAPSHOT_MAX_BYTES unsigned bytes or unlimited/inf; default DS4_BENCH_DEFAULT_SNAPSHOT_MAX_BYTES Limit session snapshot size during benchmark sweeps. ds4_bench.c:70 runtime/cli DS4_CLI_FORCE_SESSION Pure presence flag: any defined value, including empty or "0", forces the session path. Unset uses the session path only for distributed coordinators, TP leaders, temperature>0, or MTP depth>1; otherwise the CLI calls direct argmax generation. Force ordinary CLI generation through run_sampled_generation/session APIs so single-node validation follows the same stateful path as TP/distributed runs. ds4_cli.c:1226 -runtime/core DS4_BATCHED_FFN Pure presence flag: any defined value, including empty or "0", enables. Unset leaves the default shared-expert-batched FFN path (or its configured fallback). It is read only by CPU layer-major prefill and takes precedence over shared-batch and token-parallel FFN choices. Run the complete CPU prefill FFN in chunks through layer_ffn_batch instead of the default shared-expert-only batched path. ds4.c:14238 -runtime/core DS4_BATCHED_ROPE_MAX Nonempty value parsed by strtol without full-string validation; integers 0..65536 are accepted, otherwise default 4096. Zero disables batched RoPE for every nonempty prompt. Effective only when prefix batch attention is selected and DS4_NO_BATCHED_ROPE is absent. Set the largest CPU prefix-prefill token batch that applies RoPE and inverse RoPE with the batched kernels. ds4.c:13738 -runtime/core DS4_DECODE_PROFILE_DETAIL presence flag; unset=off Print per-stage timing for the single-token CPU FFN path. ds4.c:12036 -runtime/core DS4_EXPERT_HOTLIST nonempty filesystem path; unset=off; currently Metal-only Load an expert hotlist for Metal expert profiling/streaming. ds4.c:60179 -runtime/core DS4_EXPERT_PROFILE presence diagnostic flag; unset=off Collect timing/profile diagnostics for expert profile. ds4.c:60177 -runtime/core DS4_LOCK_FILE path string; default /tmp/ds4.lock Override the single-instance lock file. ds4.c:51809 -runtime/core DS4_NO_BATCHED_ATTN presence rollback flag; unset keeps default/optimized path Disable/roll back no batched attn. ds4.c:14237 -runtime/core DS4_NO_BATCHED_ROPE presence rollback flag; unset keeps default/optimized path Disable/roll back no batched rope. ds4.c:13745 -runtime/core DS4_NO_PARALLEL_ATTN_ROWS presence rollback flag; unset keeps default/optimized path Disable/roll back no parallel attn rows. ds4.c:13731 -runtime/core DS4_NO_ROUTED_TOKEN_PARALLEL presence rollback flag; unset keeps default/optimized path Disable/roll back no routed token parallel. ds4.c:12415 -runtime/core DS4_NO_SHARED_BATCH_FFN presence rollback flag; unset keeps default/optimized path Disable/roll back no shared batch ffn. ds4.c:14240 -runtime/core DS4_ORACLE_LOGITS filesystem path; unset disables read/write Load reference logits for graph correctness comparison. ds4.c:38866 -runtime/core DS4_PARALLEL_ATTN_ROWS Pure presence opt-in; any defined value enables the preference, but DS4_NO_PARALLEL_ATTN_ROWS overrides it. The path is eligible only for prefix prefill with cache n_raw==0 and pos0==0; unset uses per-token attention rows. Batch/parallelize CPU prefix attention-row evaluation after cache/index preparation. ds4.c:13728 -runtime/core DS4_PARALLEL_FFN Pure presence opt-in. It is effective only in CPU prefill when batched attention is enabled, DS4_BATCHED_FFN is absent, and the default shared-batch path has been disabled with DS4_NO_SHARED_BATCH_FFN; otherwise higher-priority paths win. Run independent prompt-token FFNs through layer_ffn_tokens_parallel as the fallback after disabling shared batching. ds4.c:14239 -runtime/core DS4_PREFILL_BATCH Nonempty value parsed by strtol without full-string validation; accepted range 1..4095, default 128 for unset/invalid/out-of-range values. It is used only when DS4_BATCHED_FFN selects full batched CPU FFN. Set the token chunk size for layer_ffn_batch during CPU layer-major prefill. ds4.c:14241 -runtime/core DS4_PREFILL_PROFILE_DETAIL presence flag; unset=off Print detailed per-stage CPU prefill timing. ds4.c:12382 -runtime/core DS4_PREFILL_PROFILE_TOKEN presence flag; effective within detailed prefill profiling Print token-loop substage timings during CPU prefill. ds4.c:13957 -runtime/core DS4_ROUTED_TOKEN_PARALLEL Pure presence flag that forces token-parallel routed MoE, even if DS4_NO_ROUTED_TOKEN_PARALLEL is also set. When unset, token parallelism is automatic for n_tok>=64 unless the NO flag is present; smaller batches use per-token routed MoE. Choose token-parallel CPU routed-expert evaluation inside the default shared-batch FFN prefill path. ds4.c:12414 +runtime/core DS4_BATCHED_FFN Pure presence flag: any defined value, including empty or "0", enables. Unset leaves the default shared-expert-batched FFN path (or its configured fallback). It is read only by CPU layer-major prefill and takes precedence over shared-batch and token-parallel FFN choices. Run the complete CPU prefill FFN in chunks through layer_ffn_batch instead of the default shared-expert-only batched path. ds4.c:14411 +runtime/core DS4_BATCHED_ROPE_MAX Nonempty value parsed by strtol without full-string validation; integers 0..65536 are accepted, otherwise default 4096. Zero disables batched RoPE for every nonempty prompt. Effective only when prefix batch attention is selected and DS4_NO_BATCHED_ROPE is absent. Set the largest CPU prefix-prefill token batch that applies RoPE and inverse RoPE with the batched kernels. ds4.c:13911 +runtime/core DS4_DECODE_PROFILE_DETAIL presence flag; unset=off Print per-stage timing for the single-token CPU FFN path. ds4.c:12209 +runtime/core DS4_EXPERT_HOTLIST nonempty filesystem path; unset=off; currently Metal-only Load an expert hotlist for Metal expert profiling/streaming. ds4.c:60369 +runtime/core DS4_EXPERT_PROFILE presence diagnostic flag; unset=off Collect timing/profile diagnostics for expert profile. ds4.c:60367 +runtime/core DS4_LOCK_FILE path string; default /tmp/ds4.lock Override the single-instance lock file. ds4.c:51999 +runtime/core DS4_NO_BATCHED_ATTN presence rollback flag; unset keeps default/optimized path Disable/roll back no batched attn. ds4.c:14410 +runtime/core DS4_NO_BATCHED_ROPE presence rollback flag; unset keeps default/optimized path Disable/roll back no batched rope. ds4.c:13918 +runtime/core DS4_NO_PARALLEL_ATTN_ROWS presence rollback flag; unset keeps default/optimized path Disable/roll back no parallel attn rows. ds4.c:13904 +runtime/core DS4_NO_ROUTED_TOKEN_PARALLEL presence rollback flag; unset keeps default/optimized path Disable/roll back no routed token parallel. ds4.c:12588 +runtime/core DS4_NO_SHARED_BATCH_FFN presence rollback flag; unset keeps default/optimized path Disable/roll back no shared batch ffn. ds4.c:14413 +runtime/core DS4_ORACLE_LOGITS filesystem path; unset disables read/write Load reference logits for graph correctness comparison. ds4.c:39056 +runtime/core DS4_PARALLEL_ATTN_ROWS Pure presence opt-in; any defined value enables the preference, but DS4_NO_PARALLEL_ATTN_ROWS overrides it. The path is eligible only for prefix prefill with cache n_raw==0 and pos0==0; unset uses per-token attention rows. Batch/parallelize CPU prefix attention-row evaluation after cache/index preparation. ds4.c:13901 +runtime/core DS4_PARALLEL_FFN Pure presence opt-in. It is effective only in CPU prefill when batched attention is enabled, DS4_BATCHED_FFN is absent, and the default shared-batch path has been disabled with DS4_NO_SHARED_BATCH_FFN; otherwise higher-priority paths win. Run independent prompt-token FFNs through layer_ffn_tokens_parallel as the fallback after disabling shared batching. ds4.c:14412 +runtime/core DS4_PREFILL_BATCH Nonempty value parsed by strtol without full-string validation; accepted range 1..4095, default 128 for unset/invalid/out-of-range values. It is used only when DS4_BATCHED_FFN selects full batched CPU FFN. Set the token chunk size for layer_ffn_batch during CPU layer-major prefill. ds4.c:14414 +runtime/core DS4_PREFILL_PROFILE_DETAIL presence flag; unset=off Print detailed per-stage CPU prefill timing. ds4.c:12555 +runtime/core DS4_PREFILL_PROFILE_TOKEN presence flag; effective within detailed prefill profiling Print token-loop substage timings during CPU prefill. ds4.c:14130 +runtime/core DS4_ROUTED_TOKEN_PARALLEL Pure presence flag that forces token-parallel routed MoE, even if DS4_NO_ROUTED_TOKEN_PARALLEL is also set. When unset, token parallelism is automatic for n_tok>=64 unless the NO flag is present; smaller batches use per-token routed MoE. Choose token-parallel CPU routed-expert evaluation inside the default shared-batch FFN prefill path. ds4.c:12587 runtime/core DS4_THREADS positive integer; default min(online CPUs,12), capped by DS4_MAX_THREADS; CLI thread request overrides env Set CPU worker-pool size. ds4.c:1874 -runtime/core DS4_TOKEN_TIMING Pure presence flag; default off. Any defined value times and prints each CPU token decode evaluation; sampling, emission and callbacks are outside the measured interval. Report per-token CPU model-evaluation latency during direct argmax generation. ds4.c:41429 -runtime/core DS4_TRACE_TOP presence flag; unset=off Print top-logit/token trace data during CPU generation. ds4.c:41398 -runtime/cpu DS4_CPU_DISABLE_UNROLLED_ARGMAX presence rollback flag; unset keeps optimized/default path Disable/roll back cpu disable unrolled argmax. ds4.c:40885 -runtime/cpu DS4_CPU_DUMP_LOGITS filesystem path; unset disables read/write Dump or select diagnostic data for cpu dump logits. ds4.c:38896 -runtime/cpu DS4_CPU_DUMP_PREFILL_LOGITS filesystem path; unset disables read/write Dump or select diagnostic data for cpu dump prefill logits. ds4.c:41416 +runtime/core DS4_TOKEN_TIMING Pure presence flag; default off. Any defined value times and prints each CPU token decode evaluation; sampling, emission and callbacks are outside the measured interval. Report per-token CPU model-evaluation latency during direct argmax generation. ds4.c:41619 +runtime/core DS4_TRACE_TOP presence flag; unset=off Print top-logit/token trace data during CPU generation. ds4.c:41588 +runtime/cpu DS4_CPU_DISABLE_UNROLLED_ARGMAX presence rollback flag; unset keeps optimized/default path Disable/roll back cpu disable unrolled argmax. ds4.c:41075 +runtime/cpu DS4_CPU_DUMP_LOGITS filesystem path; unset disables read/write Dump or select diagnostic data for cpu dump logits. ds4.c:39086 +runtime/cpu DS4_CPU_DUMP_PREFILL_LOGITS filesystem path; unset disables read/write Dump or select diagnostic data for cpu dump prefill logits. ds4.c:41606 runtime/cuda DS4_CUDA_ATTENTION_OUTPUT_A_CUBLAS_MIN integer tokens; default 2; accepted range 2..4095, otherwise 2 Set the token-count threshold for using cuBLAS on attention output-A. ds4_cuda.cu:23925 runtime/cuda DS4_CUDA_ATTENTION_OUTPUT_PRELOAD presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Allow attention-output Q8 weights to be preloaded into the selective F16 cache. ds4_cuda.cu:2402 runtime/cuda DS4_CUDA_ATTN_OUTPUT_PROFILE presence diagnostic flag; default off; any defined value including 0 enables Measure and print CUDA attention-output stage timings. ds4_cuda.cu:23910 runtime/cuda DS4_CUDA_ATTN_Q_B_F32_CACHE presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Enable an F32-derived-weight cache for attention Q-B weights. ds4_cuda.cu:2414 runtime/cuda DS4_CUDA_BUILD_ARTIFACTS boolean-ish, default on for eligible derived artifacts; only exact 0 disables Control construction of eligible CUDA derived/repacked weight artifacts. ds4_cuda.cu:8440 runtime/cuda DS4_CUDA_COPY_MODEL nonempty-string opt-in (but mere presence also suppresses prefetch); default off; value 0 is nonempty and requests a full copy Copy the complete mapped model image into device memory. ds4_cuda.cu:2740 -runtime/cuda DS4_CUDA_COPY_MODEL_CHUNKED presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Use range-by-range model prefetch/copy preparation instead of the normal bulk preparation. ds4_cuda.cu:37720 +runtime/cuda DS4_CUDA_COPY_MODEL_CHUNKED presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Use range-by-range model prefetch/copy preparation instead of the normal bulk preparation. ds4_cuda.cu:37740 runtime/cuda DS4_CUDA_DECODE_GRAPHS boolean, default on; any value starting with 0, or exact off/no/false in listed case variants, disables; oracle flags force off; effective only on one GPU Control CUDA Graph capture and replay for decode. ds4_cuda.cu:1468 runtime/cuda DS4_CUDA_DECODE_GRAPH_LOG presence diagnostic flag; default off; any defined value including 0 enables Log CUDA decode-graph cache misses, capture failures, and lifecycle events. ds4_cuda.cu:1580 runtime/cuda DS4_CUDA_DECODE_HEADS8_ONLINE presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Force the eight-head online CUDA decode-attention kernel when eligible. ds4_cuda.cu:371 runtime/cuda DS4_CUDA_DECODE_SCORE4 presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Select four score lanes in the CUDA decode-attention fallback kernel. ds4_cuda.cu:372 runtime/cuda DS4_CUDA_DECODE_SCORE8 presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Select eight score lanes in the CUDA decode-attention fallback kernel. ds4_cuda.cu:373 runtime/cuda DS4_CUDA_DIRECT_MODEL mixed presence/nonempty flag, default off; any defined value bypasses host caching, while backend direct lookup requires nonempty; value 0 therefore still changes behavior Use the mapped model directly and bypass selective CUDA weight caching. ds4.c:3058; ds4_cuda.cu:1250 -runtime/cuda DS4_CUDA_DISABLE_DSPARK_EXACTN value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Disable the CUDA DSpark exactn optimization. ds4.c:52080 -runtime/cuda DS4_CUDA_DISABLE_DSPARK_EXACTN_BATCH_HEAD value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Disable the CUDA DSpark exactn batch head optimization. ds4.c:37095 -runtime/cuda DS4_CUDA_DISABLE_DSPARK_EXACTN_GRAPHS value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Disable the CUDA DSpark exactn graphs optimization. ds4.c:37061 +runtime/cuda DS4_CUDA_DISABLE_DSPARK_EXACTN value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Disable the CUDA DSpark exactn optimization. ds4.c:52270 +runtime/cuda DS4_CUDA_DISABLE_DSPARK_EXACTN_BATCH_HEAD value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Disable the CUDA DSpark exactn batch head optimization. ds4.c:37285 +runtime/cuda DS4_CUDA_DISABLE_DSPARK_EXACTN_GRAPHS value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Disable the CUDA DSpark exactn graphs optimization. ds4.c:37251 runtime/cuda DS4_CUDA_DISABLE_DSPARK_NONCAUSAL_ONLINE value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on Disable the noncausal online-attention DSpark experiment. ds4_cuda.cu:21691 runtime/cuda DS4_CUDA_DISABLE_HC_NORM_MIX_FUSE presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable fused HC RMSNorm-plus-mix. ds4_cuda.cu:20905 runtime/cuda DS4_CUDA_DISABLE_HC_SPLIT_NORM_FUSED presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the fused HC split/weighted-sum/norm kernel. ds4_cuda.cu:31785 runtime/cuda DS4_CUDA_DISABLE_IQ2_XXS_SSD_PREFILL_MMQ false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Disable the CUDA IQ2 XXS SSD prefill MMQ optimization/path. ds4_cuda.cu:4627 -runtime/cuda DS4_CUDA_DISABLE_Q4_ATTN_OUT_HC_FUSE presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable fused Q4 attention-output/HC expansion. ds4_cuda.cu:37337 +runtime/cuda DS4_CUDA_DISABLE_Q4_ATTN_OUT_HC_FUSE presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable fused Q4 attention-output/HC expansion. ds4_cuda.cu:37357 runtime/cuda DS4_CUDA_DISABLE_Q4_DENSE_PAIR presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q4 dense pair CUDA Q4 optimization. ds4_cuda.cu:20402 runtime/cuda DS4_CUDA_DISABLE_Q8_HC_EXPAND_FUSED false-like-aware flag, default off; 0/false/no/off is off, other nonempty values request split; force-fused wins Request the split Q8 shared-down/HC path when safe. ds4_cuda.cu:2086 -runtime/cuda DS4_CUDA_DISABLE_QKV_RMS_FUSED presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the CUDA QKV RMS fused optimization/path. ds4_cuda.cu:369; ds4.c:17255 +runtime/cuda DS4_CUDA_DISABLE_QKV_RMS_FUSED presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the CUDA QKV RMS fused optimization/path. ds4_cuda.cu:369; ds4.c:17428 runtime/cuda DS4_CUDA_DISABLE_SHARED_GATE_UP_PAIR presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the CUDA shared gate up pair optimization/path. ds4_cuda.cu:24293 runtime/cuda DS4_CUDA_DISABLE_STREAMING_EXPERT_PERSISTENT_CACHE false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Disable streaming expert persistent cache in CUDA SSD streaming. ds4_cuda.cu:4113 -runtime/cuda DS4_CUDA_DISABLE_STREAMING_PREFILL_BATCH_SELECTED_ADDR presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable streaming prefill batch selected addr in CUDA SSD streaming. ds4.c:18419 -runtime/cuda DS4_CUDA_DISABLE_STREAMING_PREFILL_BATCH_SELECTED_LOAD presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable streaming prefill batch selected load in CUDA SSD streaming. ds4.c:21744 +runtime/cuda DS4_CUDA_DISABLE_STREAMING_PREFILL_BATCH_SELECTED_ADDR presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable streaming prefill batch selected addr in CUDA SSD streaming. ds4.c:18592 +runtime/cuda DS4_CUDA_DISABLE_STREAMING_PREFILL_BATCH_SELECTED_LOAD presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable streaming prefill batch selected load in CUDA SSD streaming. ds4.c:21917 runtime/cuda DS4_CUDA_DISABLE_STREAMING_SELECTED_BATCH_IO false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Disable streaming selected batch I/O in CUDA SSD streaming. ds4_cuda.cu:4734 runtime/cuda DS4_CUDA_DISABLE_STREAMING_SELECTED_EVENT_PIPELINE false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Disable streaming selected event pipeline in CUDA SSD streaming. ds4_cuda.cu:4866 -runtime/cuda DS4_CUDA_DISABLE_STREAMING_SELECTED_SHARED_OVERLAP presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable streaming selected shared overlap in CUDA SSD streaming. ds4.c:20796 -runtime/cuda DS4_CUDA_DSPARK_DEVICE_PROPOSER value-aware opt-in, default off; 0/off/no/false (lowercase only) disable; other nonempty enables unless rollback set Enable the CUDA-resident DSpark proposer. ds4.c:34699; ds4_cuda.cu:19035 -runtime/cuda DS4_CUDA_DSPARK_EXACT2 value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Enable the exact two-draft CUDA DSpark support path. ds4.c:52057 -runtime/cuda DS4_CUDA_DSPARK_EXACTN value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Enable the exact multi-draft CUDA DSpark support path. ds4.c:52078 -runtime/cuda DS4_CUDA_DSPARK_EXACTN_BATCH_HEAD value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Enable the batched output-head stage for exact-N DSpark verification. ds4.c:37093 -runtime/cuda DS4_CUDA_DSPARK_EXACTN_GRAPHS value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Enable CUDA Graph capture for stable exact-N DSpark islands. ds4.c:37059 -runtime/cuda DS4_CUDA_DSPARK_NO_DEVICE_PROPOSER presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Disable the CUDA-resident DSpark proposer. ds4.c:34709; ds4_cuda.cu:19040 -runtime/cuda DS4_CUDA_DSPARK_NO_PADDED_HEAD presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Disable the padded CUDA output-head optimization used by DSpark. ds4.c:34057 -runtime/cuda DS4_CUDA_DSPARK_NO_Q_NORM_ROPE_FUSION presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Disable fused Q RMSNorm plus RoPE in DSpark support stages. ds4.c:33433 -runtime/cuda DS4_CUDA_DSPARK_PROPOSER_BLOCK_MAX integer 0..UINT32_MAX; 0/invalid keeps native size; unset uses auto caps for exact-N/exact2; positive values cap the block and are limited by DS4_DSPARK_MAX_BLOCK_SIZE Cap the CUDA DSpark proposal block length. ds4.c:52164 +runtime/cuda DS4_CUDA_DISABLE_STREAMING_SELECTED_SHARED_OVERLAP presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable streaming selected shared overlap in CUDA SSD streaming. ds4.c:20969 +runtime/cuda DS4_CUDA_DSPARK_DEVICE_PROPOSER value-aware opt-in, default off; 0/off/no/false (lowercase only) disable; other nonempty enables unless rollback set Enable the CUDA-resident DSpark proposer. ds4.c:34889; ds4_cuda.cu:19035 +runtime/cuda DS4_CUDA_DSPARK_EXACT2 value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Enable the exact two-draft CUDA DSpark support path. ds4.c:52247 +runtime/cuda DS4_CUDA_DSPARK_EXACTN value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Enable the exact multi-draft CUDA DSpark support path. ds4.c:52268 +runtime/cuda DS4_CUDA_DSPARK_EXACTN_BATCH_HEAD value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Enable the batched output-head stage for exact-N DSpark verification. ds4.c:37283 +runtime/cuda DS4_CUDA_DSPARK_EXACTN_GRAPHS value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Enable CUDA Graph capture for stable exact-N DSpark islands. ds4.c:37249 +runtime/cuda DS4_CUDA_DSPARK_NO_DEVICE_PROPOSER presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Disable the CUDA-resident DSpark proposer. ds4.c:34899; ds4_cuda.cu:19040 +runtime/cuda DS4_CUDA_DSPARK_NO_PADDED_HEAD presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Disable the padded CUDA output-head optimization used by DSpark. ds4.c:34247 +runtime/cuda DS4_CUDA_DSPARK_NO_Q_NORM_ROPE_FUSION presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Disable fused Q RMSNorm plus RoPE in DSpark support stages. ds4.c:33623 +runtime/cuda DS4_CUDA_DSPARK_PROPOSER_BLOCK_MAX integer 0..UINT32_MAX; 0/invalid keeps native size; unset uses auto caps for exact-N/exact2; positive values cap the block and are limited by DS4_DSPARK_MAX_BLOCK_SIZE Cap the CUDA DSpark proposal block length. ds4.c:52354 runtime/cuda DS4_CUDA_DSPARK_TINY_ALIGNED_VEC value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on Use aligned routed-MoE vector kernels for tiny DSpark batches. ds4_cuda.cu:29523 runtime/cuda DS4_CUDA_ENABLE_DSPARK_NONCAUSAL_ONLINE value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on Enable the small-batch noncausal online-attention DSpark experiment. ds4_cuda.cu:21690 runtime/cuda DS4_CUDA_ENABLE_HC_NORM_MIX_FUSE nonempty opt-in, default off; only exact 0 disables; the F32/F16 activation mode follows the selected standalone matmul path; disable/serial/alternate flags can veto Enable and select the fused HC RMSNorm-plus-mix one-token implementation. ds4_cuda.cu:20902 runtime/cuda DS4_CUDA_ENABLE_IQ2_XXS_SSD_PREFILL_MMQ false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Enable the CUDA IQ2 XXS SSD prefill MMQ experimental path. ds4_cuda.cu:4625 -runtime/cuda DS4_CUDA_ENABLE_Q4_ATTN_OUT_HC_FUSE value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on Opt in to the fused Q4 attention-output/HC expansion path. ds4_cuda.cu:37355 -runtime/cuda DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_BATCH value-aware opt-in, default off; nonempty value other than exact 0 enables; rollback wins Enable flattened grouped attention-A MMQ for two-to-eight-token GB10 batches. cuda/mmq/ds4_mmq.cu:4096 -runtime/cuda DS4_CUDA_ENABLE_Q4_K1024_PERSISTENT presence flag, default off; any defined value including 0 requests the path; rollback wins Enable the GB10 persistent-CTA kernel for M=32768, N=1, K=1024 Q4. cuda/mmq/ds4_mmq.cu:3698 +runtime/cuda DS4_CUDA_ENABLE_Q4_ATTN_OUT_HC_FUSE value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on Opt in to the fused Q4 attention-output/HC expansion path. ds4_cuda.cu:37375 +runtime/cuda DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_BATCH value-aware opt-in, default off; nonempty value other than exact 0 enables; rollback wins Enable flattened grouped attention-A MMQ for two-to-eight-token GB10 batches. cuda/mmq/ds4_mmq.cu:4280 +runtime/cuda DS4_CUDA_ENABLE_Q4_K1024_PERSISTENT presence flag, default off; any defined value including 0 requests the path; rollback wins Enable the GB10 persistent-CTA kernel for M=32768, N=1, K=1024 Q4. cuda/mmq/ds4_mmq.cu:3882 runtime/cuda DS4_CUDA_ENABLE_Q8_FOLD strict flag, default off; only exact value 1 enables; overridden by DS4_CUDA_NO_Q8_FOLD Enable one-shot producer-to-consumer reuse of freshly quantized Q8_1 data. ds4_cuda.cu:785 runtime/cuda DS4_CUDA_ENABLE_STREAMING_EXPERT_PERSISTENT_CACHE false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Enable streaming expert persistent cache in CUDA SSD streaming. ds4_cuda.cu:4111 runtime/cuda DS4_CUDA_ENABLE_STREAMING_SELECTED_BATCH_IO false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Enable streaming selected batch I/O in CUDA SSD streaming. ds4_cuda.cu:4732 @@ -101,27 +101,27 @@ runtime/cuda DS4_CUDA_F16_CUBLAS_ONE presence flag; unset does not force the pat runtime/cuda DS4_CUDA_F16_SMALL_BATCH presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Control the F16 small batch CUDA F16 matmul path. ds4_cuda.cu:20837 runtime/cuda DS4_CUDA_F16_SMALL_OUT presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Control the F16 small out CUDA F16 matmul path. ds4_cuda.cu:20819 runtime/cuda DS4_CUDA_GLM_VERIFY_NO_Q8_TOK2 presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Control or tune the CUDA glm verify no Q8 tok2 path. ds4_cuda.cu:19824 -runtime/cuda DS4_CUDA_GREEDY_SPLITKV value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Enable greedy split-KV fast attention. ds4.c:17062 -runtime/cuda DS4_CUDA_GREEDY_SPLITKV_FALLBACK_LOG presence diagnostic flag; default off; any defined value including 0 enables Control greedy splitkv fallback log in CUDA greedy fast decode. ds4.c:55473 -runtime/cuda DS4_CUDA_GREEDY_SPLITKV_MARGIN nonnegative finite float; default 0.25; invalid value warns and uses 0.25; 0 disables margin fallback Control greedy splitkv margin in CUDA greedy fast decode. ds4.c:17140 -runtime/cuda DS4_CUDA_GREEDY_SPLITKV_MAX_SEGMENT integer 0..INT32_MAX; default/invalid 0 (segment cap disabled) Control greedy splitkv max segment in CUDA greedy fast decode. ds4.c:17216 -runtime/cuda DS4_CUDA_GREEDY_SPLITKV_PAIR_REPLAY value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Replay greedy split-KV tokens in pairs. ds4.c:17190 -runtime/cuda DS4_CUDA_GREEDY_SPLITKV_TOP2 value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Use top-2 output margins with greedy split-KV. ds4.c:17172 -runtime/cuda DS4_CUDA_GREEDY_SPLITKV_TRACE presence diagnostic flag; default off; any defined value including 0 enables Control greedy splitkv trace in CUDA greedy fast decode. ds4.c:55508 -runtime/cuda DS4_CUDA_GREEDY_SPLITKV_TRUST_REPLAY value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Trust replayed greedy split-KV results without the normal confirmation policy. ds4.c:17180 -runtime/cuda DS4_CUDA_GREEDY_SPLIT_TOP1 value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Enable split top-1 selection in greedy CUDA decode. ds4.c:17034 -runtime/cuda DS4_CUDA_GREEDY_TOP1 value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control greedy top1 in CUDA greedy fast decode. ds4.c:55936 -runtime/cuda DS4_CUDA_GREEDY_VEC4 value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Enable greedy vec4 fast attention. ds4.c:17072 -runtime/cuda DS4_CUDA_GREEDY_VEC4_FALLBACK_LOG presence diagnostic flag; default off; any defined value including 0 enables Control greedy vec4 fallback log in CUDA greedy fast decode. ds4.c:55474 -runtime/cuda DS4_CUDA_GREEDY_VEC4_MARGIN nonnegative finite float; default 0.25; invalid value warns and uses 0.25; 0 disables margin fallback Control greedy vec4 margin in CUDA greedy fast decode. ds4.c:17110 -runtime/cuda DS4_CUDA_GREEDY_VEC4_MAX_SEGMENT integer 0..INT32_MAX; default/invalid 0 (segment cap disabled) Control greedy vec4 max segment in CUDA greedy fast decode. ds4.c:17224 -runtime/cuda DS4_CUDA_GREEDY_VEC4_TRACE presence diagnostic flag; default off; any defined value including 0 enables Control greedy vec4 trace in CUDA greedy fast decode. ds4.c:55537 +runtime/cuda DS4_CUDA_GREEDY_SPLITKV value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Enable greedy split-KV fast attention. ds4.c:17235 +runtime/cuda DS4_CUDA_GREEDY_SPLITKV_FALLBACK_LOG presence diagnostic flag; default off; any defined value including 0 enables Control greedy splitkv fallback log in CUDA greedy fast decode. ds4.c:55663 +runtime/cuda DS4_CUDA_GREEDY_SPLITKV_MARGIN nonnegative finite float; default 0.25; invalid value warns and uses 0.25; 0 disables margin fallback Control greedy splitkv margin in CUDA greedy fast decode. ds4.c:17313 +runtime/cuda DS4_CUDA_GREEDY_SPLITKV_MAX_SEGMENT integer 0..INT32_MAX; default/invalid 0 (segment cap disabled) Control greedy splitkv max segment in CUDA greedy fast decode. ds4.c:17389 +runtime/cuda DS4_CUDA_GREEDY_SPLITKV_PAIR_REPLAY value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Replay greedy split-KV tokens in pairs. ds4.c:17363 +runtime/cuda DS4_CUDA_GREEDY_SPLITKV_TOP2 value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Use top-2 output margins with greedy split-KV. ds4.c:17345 +runtime/cuda DS4_CUDA_GREEDY_SPLITKV_TRACE presence diagnostic flag; default off; any defined value including 0 enables Control greedy splitkv trace in CUDA greedy fast decode. ds4.c:55698 +runtime/cuda DS4_CUDA_GREEDY_SPLITKV_TRUST_REPLAY value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Trust replayed greedy split-KV results without the normal confirmation policy. ds4.c:17353 +runtime/cuda DS4_CUDA_GREEDY_SPLIT_TOP1 value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Enable split top-1 selection in greedy CUDA decode. ds4.c:17207 +runtime/cuda DS4_CUDA_GREEDY_TOP1 value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control greedy top1 in CUDA greedy fast decode. ds4.c:56126 +runtime/cuda DS4_CUDA_GREEDY_VEC4 value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Enable greedy vec4 fast attention. ds4.c:17245 +runtime/cuda DS4_CUDA_GREEDY_VEC4_FALLBACK_LOG presence diagnostic flag; default off; any defined value including 0 enables Control greedy vec4 fallback log in CUDA greedy fast decode. ds4.c:55664 +runtime/cuda DS4_CUDA_GREEDY_VEC4_MARGIN nonnegative finite float; default 0.25; invalid value warns and uses 0.25; 0 disables margin fallback Control greedy vec4 margin in CUDA greedy fast decode. ds4.c:17283 +runtime/cuda DS4_CUDA_GREEDY_VEC4_MAX_SEGMENT integer 0..INT32_MAX; default/invalid 0 (segment cap disabled) Control greedy vec4 max segment in CUDA greedy fast decode. ds4.c:17397 +runtime/cuda DS4_CUDA_GREEDY_VEC4_TRACE presence diagnostic flag; default off; any defined value including 0 enables Control greedy vec4 trace in CUDA greedy fast decode. ds4.c:55727 runtime/cuda DS4_CUDA_INDEXED_TWOPASS presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Force the two-pass indexed-attention path instead of the fused heads8 online kernel. ds4_cuda.cu:23587 runtime/cuda DS4_CUDA_IQ2_XXS_SSD_PREFILL_MMQ_STATS false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Print CUDA IQ2 XXS SSD prefill MMQ counters. ds4_cuda.cu:4633 runtime/cuda DS4_CUDA_KEEP_MODEL_PAGES presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Keep source model pages resident instead of advising the OS to discard copied pages. ds4_cuda.cu:2840 -runtime/cuda DS4_CUDA_MIXED_PREFILL_DECODE value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control native mixed prefill/decode scheduling. ds4.c:70511 -runtime/cuda DS4_CUDA_MIXED_ROUTED_MAX_PREFILL integer rows 0..UINT32_MAX; default/invalid 512 Set the maximum prefill rows admitted to the mixed routed-MoE path. ds4.c:70000 -runtime/cuda DS4_CUDA_MIXED_ROUTED_SCATTER value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Use scattered row handling in mixed routed-MoE execution. ds4.c:69461 +runtime/cuda DS4_CUDA_MIXED_PREFILL_DECODE value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control native mixed prefill/decode scheduling. ds4.c:70701 +runtime/cuda DS4_CUDA_MIXED_ROUTED_MAX_PREFILL integer rows 0..UINT32_MAX; default/invalid 512 Set the maximum prefill rows admitted to the mixed routed-MoE path. ds4.c:70190 +runtime/cuda DS4_CUDA_MIXED_ROUTED_SCATTER value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Use scattered row handling in mixed routed-MoE execution. ds4.c:69651 runtime/cuda DS4_CUDA_MMQ boolean-ish, default on; any value beginning with 0 disables; quality mode and multi-GPU disable normal MMQ tier (MXFP4 path differs) Control the vendored CUDA MMQ prefill tier. ds4_cuda.cu:1722 runtime/cuda DS4_CUDA_MMQ_Q81_PERSISTENT strict boolean, default off; accepts 1/on/true/yes and 0/off/false/no in listed lower/upper-case forms; unknown values are off Reuse a persistent Q8_1 MMQ scratch arena on supported GB10 devices. cuda/mmq/ds4_mmq.cu:144 runtime/cuda DS4_CUDA_MMQ_X_MAX integer >=8; rounded down to multiple of 8 and only lowers the hardware base; invalid/unset = hardware base Cap the MMQ X tile-width selector for architecture tuning. cuda/mmq/mmq.cuh:127 @@ -182,14 +182,14 @@ runtime/cuda DS4_CUDA_MOE_WRITE_GATE_UP presence flag; unset does not force the runtime/cuda DS4_CUDA_NO_ATTENTION_OUTPUT_F16_CACHE presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the attention output F16 cache CUDA F16 path. ds4_cuda.cu:2364 runtime/cuda DS4_CUDA_NO_ATTN_A_TOK2 presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the CUDA attn a tok2 optimization/path. ds4_cuda.cu:24024 runtime/cuda DS4_CUDA_NO_ATTN_Q_B_F16_CACHE presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the attn q b F16 cache CUDA F16 path. ds4_cuda.cu:2367 -runtime/cuda DS4_CUDA_NO_COMPRESSOR_PREFILL_BATCH presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the CUDA compressor prefill batch optimization/path. ds4.c:29717 +runtime/cuda DS4_CUDA_NO_COMPRESSOR_PREFILL_BATCH presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the CUDA compressor prefill batch optimization/path. ds4.c:29890 runtime/cuda DS4_CUDA_NO_CUBLAS_ATTENTION presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the CUDA cuBLAS attention optimization/path. ds4_cuda.cu:23067 runtime/cuda DS4_CUDA_NO_CUBLAS_ATTENTION_OUTPUT_A presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the CUDA cuBLAS attention output a optimization/path. ds4_cuda.cu:23934 runtime/cuda DS4_CUDA_NO_DECODE_VALUE512 presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the 512-thread CUDA decode value/finalize specialization. ds4_cuda.cu:374 runtime/cuda DS4_CUDA_NO_DERIVED_WEIGHTS presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the CUDA derived weights optimization/path. ds4_cuda.cu:1034 runtime/cuda DS4_CUDA_NO_DIRECT_IO presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the CUDA direct I/O optimization/path. cuda/mmq/ds4_repack.cu:68 runtime/cuda DS4_CUDA_NO_DIRECT_Q2_PREFILL presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the CUDA direct q2 prefill optimization/path. ds4_cuda.cu:395 -runtime/cuda DS4_CUDA_NO_EXACT_SCORE_SPLIT_DECODE value-aware kill switch, default off; exact 0 is off, other nonempty values disable Disable exact score split decode for exact score-split CUDA decode attention. ds4_cuda.cu:13629; ds4.c:67920 +runtime/cuda DS4_CUDA_NO_EXACT_SCORE_SPLIT_DECODE value-aware kill switch, default off; exact 0 is off, other nonempty values disable Disable exact score split decode for exact score-split CUDA decode attention. ds4_cuda.cu:13629; ds4.c:68110 runtime/cuda DS4_CUDA_NO_EXACT_SCORE_SPLIT_DIM2 presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable exact score split dim2 for exact score-split CUDA decode attention. ds4_cuda.cu:388 runtime/cuda DS4_CUDA_NO_F16_CUBLAS_BATCH presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the F16 cuBLAS batch CUDA F16 path. ds4_cuda.cu:20854 runtime/cuda DS4_CUDA_NO_F16_CUBLAS_ONE presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the F16 cuBLAS one CUDA F16 path. ds4_cuda.cu:20851 @@ -200,13 +200,13 @@ runtime/cuda DS4_CUDA_NO_F16_PAIR_MATMUL presence kill switch; default unset (el runtime/cuda DS4_CUDA_NO_F16_SMALL_BATCH presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the F16 small batch CUDA F16 path. ds4_cuda.cu:20838 runtime/cuda DS4_CUDA_NO_F16_SMALL_OUT presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the F16 small out CUDA F16 path. ds4_cuda.cu:20821 runtime/cuda DS4_CUDA_NO_FD_CACHE presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the CUDA fd cache optimization/path. ds4_cuda.cu:1274 -runtime/cuda DS4_CUDA_NO_GREEDY_SPLITKV value-aware kill switch; default off; nonempty value other than exact 0 disables Disable greedy splitkv in CUDA greedy fast decode. ds4.c:17060 -runtime/cuda DS4_CUDA_NO_GREEDY_SPLITKV_FALLBACK value-aware kill switch; default off; nonempty value other than exact 0 disables margin fallback Disable greedy splitkv fallback in CUDA greedy fast decode. ds4.c:17160 -runtime/cuda DS4_CUDA_NO_GREEDY_SPLITKV_PAIR_REPLAY value-aware kill switch; default off; nonempty value other than exact 0 disables Disable greedy splitkv pair replay in CUDA greedy fast decode. ds4.c:17188 -runtime/cuda DS4_CUDA_NO_GREEDY_SPLITKV_TOP2 value-aware kill switch; default off; nonempty value other than exact 0 disables Disable greedy splitkv top2 in CUDA greedy fast decode. ds4.c:17170 -runtime/cuda DS4_CUDA_NO_GREEDY_SPLIT_TOP1 value-aware kill switch; default off; nonempty value other than exact 0 disables Disable greedy split top1 in CUDA greedy fast decode. ds4.c:17032 -runtime/cuda DS4_CUDA_NO_GREEDY_VEC4 value-aware kill switch; default off; nonempty value other than exact 0 disables Disable greedy vec4 in CUDA greedy fast decode. ds4.c:17070 -runtime/cuda DS4_CUDA_NO_GREEDY_VEC4_FALLBACK value-aware kill switch; default off; nonempty value other than exact 0 disables margin fallback Disable greedy vec4 fallback in CUDA greedy fast decode. ds4.c:17130 +runtime/cuda DS4_CUDA_NO_GREEDY_SPLITKV value-aware kill switch; default off; nonempty value other than exact 0 disables Disable greedy splitkv in CUDA greedy fast decode. ds4.c:17233 +runtime/cuda DS4_CUDA_NO_GREEDY_SPLITKV_FALLBACK value-aware kill switch; default off; nonempty value other than exact 0 disables margin fallback Disable greedy splitkv fallback in CUDA greedy fast decode. ds4.c:17333 +runtime/cuda DS4_CUDA_NO_GREEDY_SPLITKV_PAIR_REPLAY value-aware kill switch; default off; nonempty value other than exact 0 disables Disable greedy splitkv pair replay in CUDA greedy fast decode. ds4.c:17361 +runtime/cuda DS4_CUDA_NO_GREEDY_SPLITKV_TOP2 value-aware kill switch; default off; nonempty value other than exact 0 disables Disable greedy splitkv top2 in CUDA greedy fast decode. ds4.c:17343 +runtime/cuda DS4_CUDA_NO_GREEDY_SPLIT_TOP1 value-aware kill switch; default off; nonempty value other than exact 0 disables Disable greedy split top1 in CUDA greedy fast decode. ds4.c:17205 +runtime/cuda DS4_CUDA_NO_GREEDY_VEC4 value-aware kill switch; default off; nonempty value other than exact 0 disables Disable greedy vec4 in CUDA greedy fast decode. ds4.c:17243 +runtime/cuda DS4_CUDA_NO_GREEDY_VEC4_FALLBACK value-aware kill switch; default off; nonempty value other than exact 0 disables margin fallback Disable greedy vec4 fallback in CUDA greedy fast decode. ds4.c:17303 runtime/cuda DS4_CUDA_NO_HC_SPLIT_NORM_SPLIT4096 presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the split partial-reduction specialization for one-row, 4096-wide HC normalization. ds4_cuda.cu:31826 runtime/cuda DS4_CUDA_NO_INDEXED_HEADS8 presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the CUDA indexed heads8 optimization/path. ds4_cuda.cu:23586 runtime/cuda DS4_CUDA_NO_INDEXED_TOPK_SORT presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the CUDA indexed topk sort optimization/path. ds4_cuda.cu:23577 @@ -219,16 +219,16 @@ runtime/cuda DS4_CUDA_NO_INDEXER_WMMA64 presence kill switch; default unset (eli runtime/cuda DS4_CUDA_NO_IQ2_XXS_SSD_PREFILL_MMQ false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Disable the CUDA IQ2 XXS SSD prefill MMQ optimization/path. ds4_cuda.cu:4629 runtime/cuda DS4_CUDA_NO_MODEL_COPY presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the CUDA model copy optimization/path. ds4_cuda.cu:6472 runtime/cuda DS4_CUDA_NO_MODEL_PREFETCH presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the CUDA model prefetch optimization/path. ds4_cuda.cu:2739 -runtime/cuda DS4_CUDA_NO_MOE_DEDUP presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the CUDA MoE dedup optimization/path. cuda/mmq/ds4_mmq.cu:5963 +runtime/cuda DS4_CUDA_NO_MOE_DEDUP presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the CUDA MoE dedup optimization/path. cuda/mmq/ds4_mmq.cu:6145 runtime/cuda DS4_CUDA_NO_ORDERED_F16_MATMUL presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the ordered F16 matmul CUDA F16 path. ds4_cuda.cu:20811 runtime/cuda DS4_CUDA_NO_PARALLEL_ROUTER_SELECT presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the CUDA parallel router select optimization/path. ds4_cuda.cu:24491 -runtime/cuda DS4_CUDA_NO_Q4_DENSE_SCRATCH presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q4 dense scratch CUDA Q4 optimization. cuda/mmq/ds4_mmq.cu:3779 -runtime/cuda DS4_CUDA_NO_Q4_GB10_FAST presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the GB10-specific Q4 fast-path family. cuda/mmq/ds4_mmq.cu:3701 -runtime/cuda DS4_CUDA_NO_Q4_GROUPED_ATTN_A presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q4 grouped attn a CUDA Q4 optimization. cuda/mmq/ds4_mmq.cu:4088 -runtime/cuda DS4_CUDA_NO_Q4_GROUPED_ATTN_A_BATCH presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q4 grouped attn a batch CUDA Q4 optimization. cuda/mmq/ds4_mmq.cu:4098 -runtime/cuda DS4_CUDA_NO_Q4_K1024_PERSISTENT presence kill switch, default off; any defined value including 0 disables Disable the Q4 K1024 persistent CUDA Q4 optimization. cuda/mmq/ds4_mmq.cu:3700 -runtime/cuda DS4_CUDA_NO_Q8_ALIGNED_DENSE_SCRATCH presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q8 aligned dense scratch CUDA Q8 optimization. cuda/mmq/ds4_mmq.cu:5369 -runtime/cuda DS4_CUDA_NO_Q8_ALIGNED_PERSISTENT presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q8 aligned persistent CUDA Q8 optimization. cuda/mmq/ds4_mmq.cu:5201 +runtime/cuda DS4_CUDA_NO_Q4_DENSE_SCRATCH presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q4 dense scratch CUDA Q4 optimization. cuda/mmq/ds4_mmq.cu:3963 +runtime/cuda DS4_CUDA_NO_Q4_GB10_FAST presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the GB10-specific Q4 fast-path family. cuda/mmq/ds4_mmq.cu:3885 +runtime/cuda DS4_CUDA_NO_Q4_GROUPED_ATTN_A presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q4 grouped attn a CUDA Q4 optimization. cuda/mmq/ds4_mmq.cu:4272 +runtime/cuda DS4_CUDA_NO_Q4_GROUPED_ATTN_A_BATCH presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q4 grouped attn a batch CUDA Q4 optimization. cuda/mmq/ds4_mmq.cu:4282 +runtime/cuda DS4_CUDA_NO_Q4_K1024_PERSISTENT presence kill switch, default off; any defined value including 0 disables Disable the Q4 K1024 persistent CUDA Q4 optimization. cuda/mmq/ds4_mmq.cu:3884 +runtime/cuda DS4_CUDA_NO_Q8_ALIGNED_DENSE_SCRATCH presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q8 aligned dense scratch CUDA Q8 optimization. cuda/mmq/ds4_mmq.cu:5553 +runtime/cuda DS4_CUDA_NO_Q8_ALIGNED_PERSISTENT presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q8 aligned persistent CUDA Q8 optimization. cuda/mmq/ds4_mmq.cu:5385 runtime/cuda DS4_CUDA_NO_Q8_BATCH_EXACT_TOK2 presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q8 batch exact tok2 CUDA Q8 optimization. ds4_cuda.cu:19852 runtime/cuda DS4_CUDA_NO_Q8_BATCH_TOK4 presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q8 batch tok4 CUDA Q8 optimization. ds4_cuda.cu:19805 runtime/cuda DS4_CUDA_NO_Q8_BATCH_TOK8 presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q8 batch tok8 CUDA Q8 optimization. ds4_cuda.cu:19788 @@ -241,14 +241,14 @@ runtime/cuda DS4_CUDA_NO_Q8_FUSED_ALIGNED presence kill switch; default unset (e runtime/cuda DS4_CUDA_NO_Q8_MMA presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q8 MMA CUDA Q8 optimization. ds4_cuda.cu:10781 runtime/cuda DS4_CUDA_NO_Q8_PAIR_BATCH_EXACT presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q8 pair batch exact CUDA Q8 optimization. ds4_cuda.cu:20302 runtime/cuda DS4_CUDA_NO_Q8_PAIR_BATCH_EXACT_TOK2 presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q8 pair batch exact tok2 CUDA Q8 optimization. ds4_cuda.cu:20305 -runtime/cuda DS4_CUDA_NO_QKV_KV_ROPE_FUSE value-aware kill switch; default off; nonempty value other than exact 0 disables Disable the CUDA QKV KV rope fuse optimization/path. ds4.c:17253 -runtime/cuda DS4_CUDA_NO_QKV_PAIR presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the CUDA QKV pair optimization/path. ds4.c:17389 -runtime/cuda DS4_CUDA_NO_SCORE_TILE presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the CUDA score tile optimization/path. ds4_cuda.cu:13734; ds4.c:67932 +runtime/cuda DS4_CUDA_NO_QKV_KV_ROPE_FUSE value-aware kill switch; default off; nonempty value other than exact 0 disables Disable the CUDA QKV KV rope fuse optimization/path. ds4.c:17426 +runtime/cuda DS4_CUDA_NO_QKV_PAIR presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the CUDA QKV pair optimization/path. ds4.c:17562 +runtime/cuda DS4_CUDA_NO_SCORE_TILE presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the CUDA score tile optimization/path. ds4_cuda.cu:13734; ds4.c:68122 runtime/cuda DS4_CUDA_NO_SETDEVICE_CACHE presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the cached current-tier shortcut and call cudaSetDevice for every tier selection. ds4_cuda.cu:377 runtime/cuda DS4_CUDA_NO_SPLITKV_DECODE value-aware kill switch, default off; exact 0/empty is off, other nonempty values disable Disable splitkv decode in CUDA split-KV attention/speculation. ds4_cuda.cu:2211 -runtime/cuda DS4_CUDA_NO_SPLITKV_SPEC value-aware kill switch; default off; nonempty value other than exact 0 disables Disable splitkv spec in CUDA split-KV attention/speculation. ds4.c:17080 -runtime/cuda DS4_CUDA_NO_SPLITKV_SPEC_BATCH_VERIFY value-aware kill switch; default off; nonempty value other than exact 0 disables Disable splitkv spec batch verify in CUDA split-KV attention/speculation. ds4.c:17100 -runtime/cuda DS4_CUDA_NO_SPLITKV_SPEC_TOPONLY_ROW0 value-aware kill switch; default off; nonempty value other than exact 0 disables Disable splitkv spec toponly row0 in CUDA split-KV attention/speculation. ds4.c:17090 +runtime/cuda DS4_CUDA_NO_SPLITKV_SPEC value-aware kill switch; default off; nonempty value other than exact 0 disables Disable splitkv spec in CUDA split-KV attention/speculation. ds4.c:17253 +runtime/cuda DS4_CUDA_NO_SPLITKV_SPEC_BATCH_VERIFY value-aware kill switch; default off; nonempty value other than exact 0 disables Disable splitkv spec batch verify in CUDA split-KV attention/speculation. ds4.c:17273 +runtime/cuda DS4_CUDA_NO_SPLITKV_SPEC_TOPONLY_ROW0 value-aware kill switch; default off; nonempty value other than exact 0 disables Disable splitkv spec toponly row0 in CUDA split-KV attention/speculation. ds4.c:17263 runtime/cuda DS4_CUDA_NO_STREAMING_EXPERT_PERSISTENT_CACHE false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Disable streaming expert persistent cache in CUDA SSD streaming. ds4_cuda.cu:4115 runtime/cuda DS4_CUDA_NO_STREAMING_SELECTED_BATCH_IO false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Disable streaming selected batch I/O in CUDA SSD streaming. ds4_cuda.cu:4736 runtime/cuda DS4_CUDA_NO_STREAMING_SELECTED_EVENT_PIPELINE false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Disable streaming selected event pipeline in CUDA SSD streaming. ds4_cuda.cu:4868 @@ -260,23 +260,23 @@ runtime/cuda DS4_CUDA_NO_TOPK2048_WIDE presence kill switch; default unset (elig runtime/cuda DS4_CUDA_NO_TOPK8192 presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the topk8192 CUDA indexer kernel/path. ds4_cuda.cu:19335 runtime/cuda DS4_CUDA_NO_TOPK_CHUNKED presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the topk chunked CUDA indexer kernel/path. ds4_cuda.cu:19374 runtime/cuda DS4_CUDA_NO_TOPK_STREAM presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the topk stream CUDA indexer kernel/path. ds4_cuda.cu:19366 -runtime/cuda DS4_CUDA_NO_TP_ATTN_OUT_HC_FUSE presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the CUDA TP attn out HC fuse optimization/path. ds4.c:17392 -runtime/cuda DS4_CUDA_NO_VERIFY_DECODE2_SPLIT_TOP1 value-aware kill switch; default off; nonempty value other than exact 0 disables Disable the CUDA verify decode2 split top1 optimization/path. ds4.c:17050 +runtime/cuda DS4_CUDA_NO_TP_ATTN_OUT_HC_FUSE presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the CUDA TP attn out HC fuse optimization/path. ds4.c:17565 +runtime/cuda DS4_CUDA_NO_VERIFY_DECODE2_SPLIT_TOP1 value-aware kill switch; default off; nonempty value other than exact 0 disables Disable the CUDA verify decode2 split top1 optimization/path. ds4.c:17223 runtime/cuda DS4_CUDA_NO_WARP_ROUTER_SELECT presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the CUDA warp router select optimization/path. ds4_cuda.cu:24490 runtime/cuda DS4_CUDA_NO_WINDOW_ATTENTION presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the CUDA window attention optimization/path. ds4_cuda.cu:23050 runtime/cuda DS4_CUDA_NSYS_PREFILL_START_POS nonempty-string flag, default off; any nonempty value enables MMQ NVTX ranges (the value is not parsed as a position) Enable MMQ NVTX annotations intended for Nsight Systems prefill capture. cuda/mmq/ds4_mmq.cu:48 runtime/cuda DS4_CUDA_NVTX strict flag, default off; only exact value 1 enables (a nonempty NSYS variable also enables ranges) Enable NVTX ranges around MMQ work. cuda/mmq/ds4_mmq.cu:47 -runtime/cuda DS4_CUDA_OUTPUT_FUSED_TOP1 value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Fuse output projection with top-1 selection in greedy decode. ds4.c:17042 -runtime/cuda DS4_CUDA_PREFILL_PIPELINE boolean, default follows CUDA TP decode; nonempty exact 0 disables, any other nonempty value enables Control the CUDA multi-tier prefill pipeline. ds4.c:17281 -runtime/cuda DS4_CUDA_PREFILL_PIPELINE_MB positive integer rows; default/invalid 512 Set prefill-pipeline microbatch rows. ds4.c:17296 -runtime/cuda DS4_CUDA_PREFILL_PIPELINE_Q8_CACHE value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Keep selective Q8 caches enabled while running the prefill pipeline. ds4.c:17291 -runtime/cuda DS4_CUDA_PREFILL_PIPELINE_SEQUENTIAL presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Execute prefill pipeline stages sequentially for diagnosis. ds4.c:35335 -runtime/cuda DS4_CUDA_PREFILL_PIPELINE_SYNC_BOUNDARY presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Synchronize CUDA at every prefill pipeline tier boundary. ds4.c:35382 +runtime/cuda DS4_CUDA_OUTPUT_FUSED_TOP1 value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Fuse output projection with top-1 selection in greedy decode. ds4.c:17215 +runtime/cuda DS4_CUDA_PREFILL_PIPELINE boolean, default follows CUDA TP decode; nonempty exact 0 disables, any other nonempty value enables Control the CUDA multi-tier prefill pipeline. ds4.c:17454 +runtime/cuda DS4_CUDA_PREFILL_PIPELINE_MB positive integer rows; default/invalid 512 Set prefill-pipeline microbatch rows. ds4.c:17469 +runtime/cuda DS4_CUDA_PREFILL_PIPELINE_Q8_CACHE value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Keep selective Q8 caches enabled while running the prefill pipeline. ds4.c:17464 +runtime/cuda DS4_CUDA_PREFILL_PIPELINE_SEQUENTIAL presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Execute prefill pipeline stages sequentially for diagnosis. ds4.c:35525 +runtime/cuda DS4_CUDA_PREFILL_PIPELINE_SYNC_BOUNDARY presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Synchronize CUDA at every prefill pipeline tier boundary. ds4.c:35572 runtime/cuda DS4_CUDA_Q4_ATTN_OUT_HC_ORACLE value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on Compare fused Q4 attention-output/HC expansion with the canonical path and retain canonical output. ds4_cuda.cu:1475 -runtime/cuda DS4_CUDA_Q4_ATTN_OUT_HC_Q8K_EXPERIMENT value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on Enable the experimental Q8_K-based Q4 attention-output/HC fusion. ds4_cuda.cu:37357 +runtime/cuda DS4_CUDA_Q4_ATTN_OUT_HC_Q8K_EXPERIMENT value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on Enable the experimental Q8_K-based Q4 attention-output/HC fusion. ds4_cuda.cu:37377 runtime/cuda DS4_CUDA_Q4_GROUPED_ATTN_A_ORACLE value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on Compare grouped attention-A against the canonical per-group result. ds4_cuda.cu:1477 -runtime/cuda DS4_CUDA_Q4_K1024_PERSISTENT_ORACLE value-aware flag, default off; nonempty value other than exact 0 enables and implies candidate admission Bitwise-compare the exact-shape persistent Q4 K1024 kernel with canonical MMVQ and retain canonical output. cuda/mmq/ds4_mmq.cu:3534 -runtime/cuda DS4_CUDA_Q4_K1024_PERSISTENT_STATS value-aware flag, default off; nonempty value other than exact 0 enables Print exact-shape persistent Q4 K1024 dispatch counters at exit. cuda/mmq/ds4_mmq.cu:3533 +runtime/cuda DS4_CUDA_Q4_K1024_PERSISTENT_ORACLE value-aware flag, default off; nonempty value other than exact 0 enables and implies candidate admission Bitwise-compare the exact-shape persistent Q4 K1024 kernel with canonical MMVQ and retain canonical output. cuda/mmq/ds4_mmq.cu:3718 +runtime/cuda DS4_CUDA_Q4_K1024_PERSISTENT_STATS value-aware flag, default off; nonempty value other than exact 0 enables Print exact-shape persistent Q4 K1024 dispatch counters at exit. cuda/mmq/ds4_mmq.cu:3717 runtime/cuda DS4_CUDA_Q8_F16_ALL presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Control the Q8 F16 all CUDA quantized-matmul/cache optimization. ds4_cuda.cu:2358 runtime/cuda DS4_CUDA_Q8_F16_CACHE_MB unsigned integer MiB, full-string parse; default unlimited; 0 disables this cache Limit the selective Q8-to-F16 derived-weight cache. ds4_cuda.cu:2218 runtime/cuda DS4_CUDA_Q8_F16_CACHE_RESERVE_MB unsigned integer MiB, full-string parse; default is VRAM-dependent (>=112 GiB: 512; >=40 GiB: max(768,1%); smaller: max(4096,5%)) Reserve free VRAM when growing the selective Q8-to-F16 cache. ds4_cuda.cu:2224 @@ -288,75 +288,75 @@ runtime/cuda DS4_CUDA_Q8_HC_EXPAND_FUSED false-like-aware flag, default off; 0/f runtime/cuda DS4_CUDA_Q8_HC_EXPAND_STATS false-like-aware flag, default off; 0/false/no/off is off, other nonempty values print report Print Q8 shared-down/HC policy and dispatch counters at exit. ds4_cuda.cu:2090 runtime/cuda DS4_CUDA_Q8_NO_ALIGNED value-aware kill switch, default off; nonempty value other than exact 0 disables aligned Q8 kernels Disable aligned Q8 CUDA matmul kernels. ds4_cuda.cu:1021 runtime/cuda DS4_CUDA_Q8_PAIR_BATCH presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Control the Q8 pair batch CUDA quantized-matmul/cache optimization. ds4_cuda.cu:20167 -runtime/cuda DS4_CUDA_QKV_KV_ROPE_FUSE value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control or tune the CUDA QKV KV rope fuse path. ds4.c:17256 -runtime/cuda DS4_CUDA_Q_NORM_ROPE_FUSE value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control or tune the CUDA q norm rope fuse path. ds4.c:17245 +runtime/cuda DS4_CUDA_QKV_KV_ROPE_FUSE value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control or tune the CUDA QKV KV rope fuse path. ds4.c:17429 +runtime/cuda DS4_CUDA_Q_NORM_ROPE_FUSE value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control or tune the CUDA q norm rope fuse path. ds4.c:17418 runtime/cuda DS4_CUDA_REQUIRE_IQ2_XXS_SSD_PREFILL_MMQ false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Require the CUDA IQ2 XXS SSD prefill MMQ path; fail closed when unavailable. ds4_cuda.cu:4631 -runtime/cuda DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_BATCH value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on Fail if grouped batched attention-A cannot be used. ds4_cuda.cu:40178 -runtime/cuda DS4_CUDA_REQUIRE_Q4_K1024_PERSISTENT presence flag, default off; any defined value including 0 makes ineligible candidate fail closed Fail when the exact Q4 K1024 persistent candidate is unavailable instead of using MMVQ. cuda/mmq/ds4_mmq.cu:3722 +runtime/cuda DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_BATCH value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on Fail if grouped batched attention-A cannot be used. ds4_cuda.cu:40198 +runtime/cuda DS4_CUDA_REQUIRE_Q4_K1024_PERSISTENT presence flag, default off; any defined value including 0 makes ineligible candidate fail closed Fail when the exact Q4 K1024 persistent candidate is unavailable instead of using MMVQ. cuda/mmq/ds4_mmq.cu:3906 runtime/cuda DS4_CUDA_REQUIRE_STREAMING_EXPERT_PERSISTENT_CACHE false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Require streaming expert persistent cache in CUDA SSD streaming; fail closed when unavailable. ds4_cuda.cu:4117 runtime/cuda DS4_CUDA_REQUIRE_STREAMING_SELECTED_BATCH_IO false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Require streaming selected batch I/O in CUDA SSD streaming; fail closed when unavailable. ds4_cuda.cu:4738 runtime/cuda DS4_CUDA_REQUIRE_STREAMING_SELECTED_EVENT_PIPELINE false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Require streaming selected event pipeline in CUDA SSD streaming; fail closed when unavailable. ds4_cuda.cu:4870 runtime/cuda DS4_CUDA_SERIAL_F16_MATMUL presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Control the serial F16 matmul CUDA F16 matmul path. ds4_cuda.cu:20801 runtime/cuda DS4_CUDA_SERIAL_ROUTER presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Control or tune the CUDA serial router path. ds4_cuda.cu:20806 -runtime/cuda DS4_CUDA_SESSION_BATCH_ATTN_ALIAS value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control the grouped multi-session CUDA attn alias stage. ds4.c:69712 -runtime/cuda DS4_CUDA_SESSION_BATCH_ATTN_CORE value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control the grouped multi-session CUDA attn core stage. ds4.c:69715 -runtime/cuda DS4_CUDA_SESSION_BATCH_ATTN_POST value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control the grouped multi-session CUDA attn post stage. ds4.c:69727 -runtime/cuda DS4_CUDA_SESSION_BATCH_ATTN_PRE value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control the grouped multi-session CUDA attn pre stage. ds4.c:69708 -runtime/cuda DS4_CUDA_SESSION_BATCH_FFN_PRE value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control the grouped multi-session CUDA ffn pre stage. ds4.c:69704 -runtime/cuda DS4_CUDA_SESSION_BATCH_INTERLEAVE boolean, default on; unset/empty/nonzero enables pipeline interleaving; exact 0 disables Control the grouped multi-session CUDA interleave stage. ds4.c:70388 -runtime/cuda DS4_CUDA_SESSION_BATCH_KV_STORE value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control the grouped multi-session CUDA KV store stage. ds4.c:69723 -runtime/cuda DS4_CUDA_SESSION_BATCH_MOE value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control the grouped multi-session CUDA MoE stage. ds4.c:69696 -runtime/cuda DS4_CUDA_SESSION_BATCH_MOE_COMBINE_ROWS value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control the grouped multi-session CUDA MoE combine rows stage. ds4.c:69241 -runtime/cuda DS4_CUDA_SESSION_BATCH_QKV value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control the grouped multi-session CUDA QKV stage. ds4.c:69719 -runtime/cuda DS4_CUDA_SESSION_BATCH_SHARED value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control the grouped multi-session CUDA shared stage. ds4.c:69700 +runtime/cuda DS4_CUDA_SESSION_BATCH_ATTN_ALIAS value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control the grouped multi-session CUDA attn alias stage. ds4.c:69902 +runtime/cuda DS4_CUDA_SESSION_BATCH_ATTN_CORE value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control the grouped multi-session CUDA attn core stage. ds4.c:69905 +runtime/cuda DS4_CUDA_SESSION_BATCH_ATTN_POST value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control the grouped multi-session CUDA attn post stage. ds4.c:69917 +runtime/cuda DS4_CUDA_SESSION_BATCH_ATTN_PRE value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control the grouped multi-session CUDA attn pre stage. ds4.c:69898 +runtime/cuda DS4_CUDA_SESSION_BATCH_FFN_PRE value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control the grouped multi-session CUDA ffn pre stage. ds4.c:69894 +runtime/cuda DS4_CUDA_SESSION_BATCH_INTERLEAVE boolean, default on; unset/empty/nonzero enables pipeline interleaving; exact 0 disables Control the grouped multi-session CUDA interleave stage. ds4.c:70578 +runtime/cuda DS4_CUDA_SESSION_BATCH_KV_STORE value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control the grouped multi-session CUDA KV store stage. ds4.c:69913 +runtime/cuda DS4_CUDA_SESSION_BATCH_MOE value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control the grouped multi-session CUDA MoE stage. ds4.c:69649 +runtime/cuda DS4_CUDA_SESSION_BATCH_MOE_COMBINE_ROWS value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control the grouped multi-session CUDA MoE combine rows stage. ds4.c:69431 +runtime/cuda DS4_CUDA_SESSION_BATCH_QKV value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control the grouped multi-session CUDA QKV stage. ds4.c:69909 +runtime/cuda DS4_CUDA_SESSION_BATCH_SHARED value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control the grouped multi-session CUDA shared stage. ds4.c:69890 runtime/cuda DS4_CUDA_SPLITKV_CHUNK integer scores/chunk; default 512; clamped 1..512 Control splitkv chunk in CUDA split-KV attention/speculation. ds4_cuda.cu:22568 -runtime/cuda DS4_CUDA_SPLITKV_DECODE value-aware boolean, default off; exact 0/empty is off, other nonempty values enable; mere presence also excludes one session-batch path Enable split-KV decode attention. ds4_cuda.cu:2213; ds4.c:67921 +runtime/cuda DS4_CUDA_SPLITKV_DECODE value-aware boolean, default off; exact 0/empty is off, other nonempty values enable; mere presence also excludes one session-batch path Enable split-KV decode attention. ds4_cuda.cu:2213; ds4.c:68111 runtime/cuda DS4_CUDA_SPLITKV_GLOBAL_SOFTMAX value-aware opt-in, default off; exact 0/empty is off, other nonempty values enable Use the global-softmax variant of split-KV attention. ds4_cuda.cu:22589 -runtime/cuda DS4_CUDA_SPLITKV_MIN_SCORE integer score count 0..UINT32_MAX; default 0 when explicitly enabled, otherwise 512; CUDA kernel clamps to 0..8192 Set the minimum visible-score count for split-KV attention. ds4_cuda.cu:22557; ds4.c:17229 +runtime/cuda DS4_CUDA_SPLITKV_MIN_SCORE integer score count 0..UINT32_MAX; default 0 when explicitly enabled, otherwise 512; CUDA kernel clamps to 0..8192 Set the minimum visible-score count for split-KV attention. ds4_cuda.cu:22557; ds4.c:17402 runtime/cuda DS4_CUDA_SPLITKV_S integer exact split count; unset/invalid = automatic; valid value clamped 1..16 Control splitkv s in CUDA split-KV attention/speculation. ds4_cuda.cu:22578 -runtime/cuda DS4_CUDA_SPLITKV_SPEC value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Enable split-KV speculative decoding. ds4.c:17082 -runtime/cuda DS4_CUDA_SPLITKV_SPEC_BATCH_VERIFY value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Use batched verification for split-KV speculation. ds4.c:17102 -runtime/cuda DS4_CUDA_SPLITKV_SPEC_LOG presence diagnostic flag; default off; any defined value including 0 enables Log split-KV speculative-decode admission and fallback decisions. ds4.c:55622 -runtime/cuda DS4_CUDA_SPLITKV_SPEC_TIMING presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Print timing for split-KV speculative-decode stages. ds4.c:55661 -runtime/cuda DS4_CUDA_SPLITKV_SPEC_TOPONLY_ROW0 value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Compute only the top result for row zero in split-KV speculation. ds4.c:17092 +runtime/cuda DS4_CUDA_SPLITKV_SPEC value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Enable split-KV speculative decoding. ds4.c:17255 +runtime/cuda DS4_CUDA_SPLITKV_SPEC_BATCH_VERIFY value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Use batched verification for split-KV speculation. ds4.c:17275 +runtime/cuda DS4_CUDA_SPLITKV_SPEC_LOG presence diagnostic flag; default off; any defined value including 0 enables Log split-KV speculative-decode admission and fallback decisions. ds4.c:55812 +runtime/cuda DS4_CUDA_SPLITKV_SPEC_TIMING presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Print timing for split-KV speculative-decode stages. ds4.c:55851 +runtime/cuda DS4_CUDA_SPLITKV_SPEC_TOPONLY_ROW0 value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Compute only the top result for row zero in split-KV speculation. ds4.c:17265 runtime/cuda DS4_CUDA_SPLITKV_S_FLOOR integer split count; default 4; clamped 1..16 Control splitkv s floor in CUDA split-KV attention/speculation. ds4_cuda.cu:22571 runtime/cuda DS4_CUDA_SPLITKV_S_MAX integer split count; default 16; clamped 1..16 Control splitkv s max in CUDA split-KV attention/speculation. ds4_cuda.cu:22574 -runtime/cuda DS4_CUDA_STREAMING_EXPERT_CACHE_PROFILE presence diagnostic flag; default off; any defined value including 0 enables Profile CUDA SSD-streaming streaming expert cache. ds4.c:21676 +runtime/cuda DS4_CUDA_STREAMING_EXPERT_CACHE_PROFILE presence diagnostic flag; default off; any defined value including 0 enables Profile CUDA SSD-streaming streaming expert cache. ds4.c:21849 runtime/cuda DS4_CUDA_STREAMING_EXPERT_PERSISTENT_CACHE_ORACLE false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Run the diagnostic oracle for CUDA SSD-streaming streaming expert persistent cache. ds4_cuda.cu:4121 runtime/cuda DS4_CUDA_STREAMING_EXPERT_PERSISTENT_CACHE_STATS false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Print counters for CUDA SSD-streaming streaming expert persistent cache. ds4_cuda.cu:4119 -runtime/cuda DS4_CUDA_STREAMING_PREFILL_BATCH_SELECTED_PROFILE presence diagnostic flag; default off; any defined value including 0 enables Profile CUDA SSD-streaming streaming prefill batch selected. ds4.c:21759 +runtime/cuda DS4_CUDA_STREAMING_PREFILL_BATCH_SELECTED_PROFILE presence diagnostic flag; default off; any defined value including 0 enables Profile CUDA SSD-streaming streaming prefill batch selected. ds4.c:21932 runtime/cuda DS4_CUDA_STREAMING_SELECTED_BATCH_IO_ORACLE false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Run the diagnostic oracle for CUDA SSD-streaming streaming selected batch I/O. ds4_cuda.cu:4740 runtime/cuda DS4_CUDA_STREAMING_SELECTED_BATCH_IO_PROFILE false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Profile CUDA SSD-streaming streaming selected batch I/O. ds4_cuda.cu:5711 runtime/cuda DS4_CUDA_STREAMING_SELECTED_EVENT_PIPELINE_ORACLE false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Run the diagnostic oracle for CUDA SSD-streaming streaming selected event pipeline. ds4_cuda.cu:4872 runtime/cuda DS4_CUDA_STREAMING_SELECTED_EVENT_PIPELINE_STATS false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Print counters for CUDA SSD-streaming streaming selected event pipeline. ds4_cuda.cu:4874 runtime/cuda DS4_CUDA_STRICT_WEIGHT_CACHE presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Fail a weight lookup when cache allocation fails instead of falling back to mapped model memory. ds4_cuda.cu:6388 runtime/cuda DS4_CUDA_SYNC_XDEV presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Synchronize cross-device CUDA copies for debugging and error localization. ds4_cuda.cu:363 -runtime/cuda DS4_CUDA_TP_ATTN value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control CUDA tensor/expert-parallel attn execution. ds4.c:16862 -runtime/cuda DS4_CUDA_TP_ATTN_CACHE_DUP value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Control CUDA tensor/expert-parallel attn cache dup execution. ds4.c:16886 -runtime/cuda DS4_CUDA_TP_ATTN_HEADS value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Control CUDA tensor/expert-parallel attn heads execution. ds4.c:16878 -runtime/cuda DS4_CUDA_TP_ATTN_OUT_HC_FUSE presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Control CUDA tensor/expert-parallel attn out HC fuse execution. ds4.c:17391 -runtime/cuda DS4_CUDA_TP_ATTN_PEER_READ value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control CUDA tensor/expert-parallel attn peer read execution. ds4.c:16870 -runtime/cuda DS4_CUDA_TP_EP_BALANCED_SHARED_MID value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control CUDA tensor/expert-parallel EP balanced shared mid execution. ds4.c:16943 -runtime/cuda DS4_CUDA_TP_EP_DELAY_REDUCE value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control CUDA tensor/expert-parallel EP delay reduce execution. ds4.c:16918 -runtime/cuda DS4_CUDA_TP_EP_DIRECT_RETURN value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control CUDA tensor/expert-parallel EP direct return execution. ds4.c:16910 -runtime/cuda DS4_CUDA_TP_EP_DUAL_PREQUANT value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control CUDA tensor/expert-parallel EP dual prequant execution. ds4.c:16952 -runtime/cuda DS4_CUDA_TP_EP_FUSED_HC_REDUCE value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control CUDA tensor/expert-parallel EP fused HC reduce execution. ds4.c:16926 -runtime/cuda DS4_CUDA_TP_EP_FUSED_SHARED_MID value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control CUDA tensor/expert-parallel EP fused shared mid execution. ds4.c:16934 -runtime/cuda DS4_CUDA_TP_EP_PACK_EXACT value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control CUDA tensor/expert-parallel EP pack exact execution. ds4.c:16902 -runtime/cuda DS4_CUDA_TP_MOE value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control CUDA tensor/expert-parallel MoE execution. ds4.c:16894 -runtime/cuda DS4_CUDA_TP_MOE_COPY3 value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Control CUDA tensor/expert-parallel MoE copy3 execution. ds4.c:16976 -runtime/cuda DS4_CUDA_TP_MOE_DELAY_REDUCE value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control CUDA tensor/expert-parallel MoE delay reduce execution. ds4.c:16960 -runtime/cuda DS4_CUDA_TP_MOE_PACK value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Control CUDA tensor/expert-parallel MoE pack execution. ds4.c:16968 -runtime/cuda DS4_CUDA_TP_MOE_PEER_READ value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Control CUDA tensor/expert-parallel MoE peer read execution. ds4.c:16984 -runtime/cuda DS4_CUDA_TP_MOE_PEER_ROUTER value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Control CUDA tensor/expert-parallel MoE peer router execution. ds4.c:16992 -runtime/cuda DS4_CUDA_TP_OUTPUT boolean, default on; empty/unset/nonzero enables, exact 0 disables Control CUDA tensor/expert-parallel output execution. ds4.c:51523 -runtime/cuda DS4_CUDA_TP_OUTPUT_WAYS integer 2..DS4_MAX_GPUS (16); default 8; invalid value falls back to 2; capped by available GPUs Set the number of GPU ways used to shard CUDA tensor-parallel output projection. ds4.c:58; ds4.c:51530 -runtime/cuda DS4_CUDA_TP_PREFILL_ATTN_OUTPUT value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control CUDA tensor/expert-parallel prefill attn output execution. ds4.c:17272 -runtime/cuda DS4_CUDA_TP_PREFILL_FFN value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control CUDA tensor/expert-parallel prefill ffn execution. ds4.c:17264 -runtime/cuda DS4_CUDA_TP_Q value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Control CUDA tensor/expert-parallel q execution. ds4.c:17016 -runtime/cuda DS4_CUDA_TP_SHARED value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Control CUDA tensor/expert-parallel shared execution. ds4.c:17000 -runtime/cuda DS4_CUDA_TP_SHARED_FOLD value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control CUDA tensor/expert-parallel shared fold execution. ds4.c:17008 -runtime/cuda DS4_CUDA_VERIFY_DECODE2_SPLIT_TOP1 value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Enable the split top-1 path for two-row verification decode. ds4.c:17052 +runtime/cuda DS4_CUDA_TP_ATTN value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control CUDA tensor/expert-parallel attn execution. ds4.c:17035 +runtime/cuda DS4_CUDA_TP_ATTN_CACHE_DUP value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Control CUDA tensor/expert-parallel attn cache dup execution. ds4.c:17059 +runtime/cuda DS4_CUDA_TP_ATTN_HEADS value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Control CUDA tensor/expert-parallel attn heads execution. ds4.c:17051 +runtime/cuda DS4_CUDA_TP_ATTN_OUT_HC_FUSE presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Control CUDA tensor/expert-parallel attn out HC fuse execution. ds4.c:17564 +runtime/cuda DS4_CUDA_TP_ATTN_PEER_READ value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control CUDA tensor/expert-parallel attn peer read execution. ds4.c:17043 +runtime/cuda DS4_CUDA_TP_EP_BALANCED_SHARED_MID value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control CUDA tensor/expert-parallel EP balanced shared mid execution. ds4.c:17116 +runtime/cuda DS4_CUDA_TP_EP_DELAY_REDUCE value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control CUDA tensor/expert-parallel EP delay reduce execution. ds4.c:17091 +runtime/cuda DS4_CUDA_TP_EP_DIRECT_RETURN value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control CUDA tensor/expert-parallel EP direct return execution. ds4.c:17083 +runtime/cuda DS4_CUDA_TP_EP_DUAL_PREQUANT value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control CUDA tensor/expert-parallel EP dual prequant execution. ds4.c:17125 +runtime/cuda DS4_CUDA_TP_EP_FUSED_HC_REDUCE value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control CUDA tensor/expert-parallel EP fused HC reduce execution. ds4.c:17099 +runtime/cuda DS4_CUDA_TP_EP_FUSED_SHARED_MID value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control CUDA tensor/expert-parallel EP fused shared mid execution. ds4.c:17107 +runtime/cuda DS4_CUDA_TP_EP_PACK_EXACT value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control CUDA tensor/expert-parallel EP pack exact execution. ds4.c:17075 +runtime/cuda DS4_CUDA_TP_MOE value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control CUDA tensor/expert-parallel MoE execution. ds4.c:17067 +runtime/cuda DS4_CUDA_TP_MOE_COPY3 value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Control CUDA tensor/expert-parallel MoE copy3 execution. ds4.c:17149 +runtime/cuda DS4_CUDA_TP_MOE_DELAY_REDUCE value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control CUDA tensor/expert-parallel MoE delay reduce execution. ds4.c:17133 +runtime/cuda DS4_CUDA_TP_MOE_PACK value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Control CUDA tensor/expert-parallel MoE pack execution. ds4.c:17141 +runtime/cuda DS4_CUDA_TP_MOE_PEER_READ value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Control CUDA tensor/expert-parallel MoE peer read execution. ds4.c:17157 +runtime/cuda DS4_CUDA_TP_MOE_PEER_ROUTER value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Control CUDA tensor/expert-parallel MoE peer router execution. ds4.c:17165 +runtime/cuda DS4_CUDA_TP_OUTPUT boolean, default on; empty/unset/nonzero enables, exact 0 disables Control CUDA tensor/expert-parallel output execution. ds4.c:51713 +runtime/cuda DS4_CUDA_TP_OUTPUT_WAYS integer 2..DS4_MAX_GPUS (16); default 8; invalid value falls back to 2; capped by available GPUs Set the number of GPU ways used to shard CUDA tensor-parallel output projection. ds4.c:58; ds4.c:51720 +runtime/cuda DS4_CUDA_TP_PREFILL_ATTN_OUTPUT value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control CUDA tensor/expert-parallel prefill attn output execution. ds4.c:17445 +runtime/cuda DS4_CUDA_TP_PREFILL_FFN value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control CUDA tensor/expert-parallel prefill ffn execution. ds4.c:17437 +runtime/cuda DS4_CUDA_TP_Q value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Control CUDA tensor/expert-parallel q execution. ds4.c:17189 +runtime/cuda DS4_CUDA_TP_SHARED value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Control CUDA tensor/expert-parallel shared execution. ds4.c:17173 +runtime/cuda DS4_CUDA_TP_SHARED_FOLD value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control CUDA tensor/expert-parallel shared fold execution. ds4.c:17181 +runtime/cuda DS4_CUDA_VERIFY_DECODE2_SPLIT_TOP1 value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Enable the split top-1 path for two-row verification decode. ds4.c:17225 runtime/cuda DS4_CUDA_WEIGHT_ARENA_CHUNK_MB positive integer MiB; default 1792; clamped 256..8192 and raised/aligned when one allocation needs more Set the CUDA selective-weight arena allocation chunk. ds4_cuda.cu:6311 runtime/cuda DS4_CUDA_WEIGHT_CACHE presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Force selective CUDA weight caching instead of direct mapped access. ds4_cuda.cu:1246 runtime/cuda DS4_CUDA_WEIGHT_CACHE_LIMIT_GB unsigned integer GiB; default/0 = unlimited; parser accepts a numeric prefix even with trailing text Limit total CUDA selective-weight cache allocation. ds4_cuda.cu:6299 @@ -375,7 +375,7 @@ runtime/cuda-mmq DS4_MMQ_NO_YIND presence rollback; unset keeps Y-indirect stagi runtime/cuda-mmq DS4_MMQ_OUT_MEMSET exact 1 enables; unset or every other value disables; cached Restore blanket MMQ output-buffer zeroing for diagnostics. cuda/mmq/ds4_mmq.cu:532 runtime/cuda-mmq DS4_MMQ_YBUF_MEMSET unset or 0 disables; 1 zero-fills; a value starting with p or P poison-fills with 0xFF Control MMQ Q8_1 activation-staging initialization and its poison oracle. cuda/mmq/ds4_mmq.cu:558 runtime/cuda-mmq DS4_MMQ_YIND_VERIFY presence diagnostic; unset is off; any defined value including 0 enables Byte-compare Y-indirect and slot-gathered MoE activation buffers. cuda/mmq/ds4_mmq.cu:600 -runtime/cuda-mmq DS4_Q8_FOLD_SELFTEST positive call budget; unset/empty disables; a nonempty value parsing to 1 or less selects 512 calls Byte-check folded Q8_1 activations against a fresh quantization; synchronizes eager streams. cuda/mmq/ds4_mmq.cu:5744 +runtime/cuda-mmq DS4_Q8_FOLD_SELFTEST positive call budget; unset/empty disables; a nonempty value parsing to 1 or less selects 512 calls Byte-check folded Q8_1 activations against a fresh quantization; synchronizes eager streams. cuda/mmq/ds4_mmq.cu:5921 runtime/cuda-shared DS4_FORCE_CUDA_PEER presence flag read once at CUDA init; unset uses automatic transfer selection; any defined value including 0 enables Force cross-device transfers through cudaMemcpyPeerAsync for diagnostics. ds4_cuda.cu:364 runtime/cuda-shared DS4_FORCE_HOST_BOUNCE presence flag read once at CUDA init; unset uses automatic transfer selection; any defined value including 0 enables Force cross-device transfers through pinned host bounce buffers for diagnostics. ds4_cuda.cu:365 runtime/cuda-tools DS4_WS_REPACK_HASH exact 1 enables; unset or every other value disables unless overridden by CLI; cached Print a per-artifact FNV-1a hash for workspace repack identity checks. cuda/mmq/ds4_repack.cu:530 @@ -395,48 +395,48 @@ runtime/distributed DS4_DIST_SOCKET_RECV_TIMEOUT_SEC Nonempty base-10 integer pa runtime/distributed DS4_DIST_SOCKET_TIMEOUT_SEC Nonempty base-10 integer parsed completely; valid range 1..3600 seconds. Default 60 seconds for unset, empty, partially parsed, or out-of-range values. Set SO_SNDTIMEO on distributed TCP sockets so blocked coordinator/worker sends eventually fail. ds4_distributed.c:1032 runtime/distributed DS4_DIST_WORKER_FORWARD_WINDOW integer 1..64; default 4 Set worker forward-results window. ds4_distributed.c:740 runtime/distributed DS4_DIST_WORKER_PREFETCH_DEPTH integer 1..8; default 2 Set worker input-prefetch queue depth. ds4_distributed.c:726 -runtime/dspark DS4_DSPARK_CACHE_RESERVE_GB integer GiB via atoi; default 4.5 GiB; values 1..32 replace it, all other values fall back; decimal/trailing text is truncated/accepted by atoi Reserve VRAM on DSpark support-cache tiers before packing support-model tensors. ds4.c:59580 -runtime/dspark DS4_DSPARK_DISABLE_FINAL_OUTPUT_ALIAS presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it Disable aliasing the final DSpark stage output to the next-stage buffer and use an explicit copy. ds4.c:33594 -runtime/dspark DS4_DSPARK_DISABLE_FUSED_CPU_MARKOV_ARGMAX cached value-aware kill switch; default off; nonempty value other than exact 0 disables; false/off also disable because only 0 is recognized as false Disable the fused CPU Markov-bias plus argmax implementation. ds4.c:34297 -runtime/dspark DS4_DSPARK_DISABLE_REUSE_CONFIDENCE0_MARKOV cached value-aware kill switch; default off; nonempty value other than exact 0 disables; false/off also disable because only 0 is recognized as false Disable reuse of the first confidence score during Markov proposal. ds4.c:34306 -runtime/dspark DS4_DSPARK_DISABLE_VERIFY_SELECTED_PROFILE presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it Override and disable the selected-expert verifier profiler. ds4.c:36470 -runtime/dspark DS4_DSPARK_EXEC_TIER integer tier via atoi; default is placement/TP free-VRAM heuristic; valid 0..n_gpus-1 overrides; invalid numeric range falls back, but nonnumeric text becomes tier 0 Choose the GPU tier that executes and primarily caches the DSpark support model. ds4.c:59553 -runtime/dspark DS4_DSPARK_FAKE_ARGMAX_PROPOSAL nonempty boolean; unset/empty or exact 0: off; every other nonempty value enables, but only while DSpark itself is enabled If the real DSpark proposer produced no draft, installs a one-token fallback proposal equal to the argmax of the current target logits; debug/test mode also selects the non-fused stage-0 setup path. ds4.c:64088 -runtime/dspark DS4_DSPARK_LOW_MEMORY_PREFILL_CHUNK unsigned integer rows; default 128; 0 disables the low-memory policy; invalid/overflow falls back, numeric prefixes are accepted; only consulted for Metal SSD+DSpark on <=24 GiB hosts without an explicit chunk Set the automatic low-memory Metal prefill chunk for SSD-streamed DSpark. ds4.c:60533 -runtime/dspark DS4_DSPARK_NO_GPU_MARKOV presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it Disable GPU Markov bias/argmax and the fully device-resident proposal path. ds4.c:34521 -runtime/dspark DS4_DSPARK_NO_MARKOV cached value-aware kill switch; default off; nonempty value other than exact 0 disables Markov bias; false/off also disable Disable Markov bias in DSpark proposal generation. ds4.c:34288 -runtime/dspark DS4_DSPARK_PROBE nonempty-string diagnostic; unset/empty is off, any nonempty value including 0 is on Log DSpark proposal/probe diagnostics. ds4.c:64085 -runtime/dspark DS4_DSPARK_PROP_PROFILE presence flag; unset is off; any defined value, including empty or 0, is on; normal eligibility still applies Print fine-grained timings for DSpark proposal setup. ds4.c:32771 -runtime/dspark DS4_DSPARK_SPEC_LOG presence flag; unset is off; any defined value, including empty or 0, is on; normal eligibility still applies Log speculative proposal, verification, acceptance, and fallback decisions. ds4.c:66406 -runtime/dspark DS4_DSPARK_SSD_VERIFY_BLOCK_MAX unsigned integer rows; default/fallback 0 means automatic policy; numeric prefixes accepted; used both as verifier cap and as an exact-2 proposer-policy discriminator Cap speculative rows verified from SSD and influence exact-2 proposal sizing. ds4.c:52190 -runtime/dspark DS4_DSPARK_STAGE_PROFILE presence flag; unset is off; any defined value, including empty or 0, is on; DS4_DSPARK_STAGE_PROFILE_STAGE must also match Profile DSpark support stages with command-boundary timings. ds4.c:33226 -runtime/dspark DS4_DSPARK_STAGE_PROFILE_STAGE selector subordinate to DS4_DSPARK_STAGE_PROFILE; unset/empty: match every stage; otherwise strtoul base 10 must consume the whole value, fit uint32_t, and equal the current stage; invalid/out-of-range values match no stage Restricts DSpark stage-boundary timing output to one stage; it does not enable profiling by itself. ds4.c:33227 -runtime/dspark DS4_DSPARK_STATS value-aware flag; default off; nonempty value other than exact 0 enables; false/off are treated as enabled Collect and print aggregate DSpark runtime statistics. ds4.c:61417 +runtime/dspark DS4_DSPARK_CACHE_RESERVE_GB integer GiB via atoi; default 4.5 GiB; values 1..32 replace it, all other values fall back; decimal/trailing text is truncated/accepted by atoi Reserve VRAM on DSpark support-cache tiers before packing support-model tensors. ds4.c:59770 +runtime/dspark DS4_DSPARK_DISABLE_FINAL_OUTPUT_ALIAS presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it Disable aliasing the final DSpark stage output to the next-stage buffer and use an explicit copy. ds4.c:33784 +runtime/dspark DS4_DSPARK_DISABLE_FUSED_CPU_MARKOV_ARGMAX cached value-aware kill switch; default off; nonempty value other than exact 0 disables; false/off also disable because only 0 is recognized as false Disable the fused CPU Markov-bias plus argmax implementation. ds4.c:34487 +runtime/dspark DS4_DSPARK_DISABLE_REUSE_CONFIDENCE0_MARKOV cached value-aware kill switch; default off; nonempty value other than exact 0 disables; false/off also disable because only 0 is recognized as false Disable reuse of the first confidence score during Markov proposal. ds4.c:34496 +runtime/dspark DS4_DSPARK_DISABLE_VERIFY_SELECTED_PROFILE presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it Override and disable the selected-expert verifier profiler. ds4.c:36660 +runtime/dspark DS4_DSPARK_EXEC_TIER integer tier via atoi; default is placement/TP free-VRAM heuristic; valid 0..n_gpus-1 overrides; invalid numeric range falls back, but nonnumeric text becomes tier 0 Choose the GPU tier that executes and primarily caches the DSpark support model. ds4.c:59743 +runtime/dspark DS4_DSPARK_FAKE_ARGMAX_PROPOSAL nonempty boolean; unset/empty or exact 0: off; every other nonempty value enables, but only while DSpark itself is enabled If the real DSpark proposer produced no draft, installs a one-token fallback proposal equal to the argmax of the current target logits; debug/test mode also selects the non-fused stage-0 setup path. ds4.c:64278 +runtime/dspark DS4_DSPARK_LOW_MEMORY_PREFILL_CHUNK unsigned integer rows; default 128; 0 disables the low-memory policy; invalid/overflow falls back, numeric prefixes are accepted; only consulted for Metal SSD+DSpark on <=24 GiB hosts without an explicit chunk Set the automatic low-memory Metal prefill chunk for SSD-streamed DSpark. ds4.c:60714 +runtime/dspark DS4_DSPARK_NO_GPU_MARKOV presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it Disable GPU Markov bias/argmax and the fully device-resident proposal path. ds4.c:34711 +runtime/dspark DS4_DSPARK_NO_MARKOV cached value-aware kill switch; default off; nonempty value other than exact 0 disables Markov bias; false/off also disable Disable Markov bias in DSpark proposal generation. ds4.c:34478 +runtime/dspark DS4_DSPARK_PROBE nonempty-string diagnostic; unset/empty is off, any nonempty value including 0 is on Log DSpark proposal/probe diagnostics. ds4.c:64275 +runtime/dspark DS4_DSPARK_PROP_PROFILE presence flag; unset is off; any defined value, including empty or 0, is on; normal eligibility still applies Print fine-grained timings for DSpark proposal setup. ds4.c:32959 +runtime/dspark DS4_DSPARK_SPEC_LOG presence flag; unset is off; any defined value, including empty or 0, is on; normal eligibility still applies Log speculative proposal, verification, acceptance, and fallback decisions. ds4.c:66596 +runtime/dspark DS4_DSPARK_SSD_VERIFY_BLOCK_MAX unsigned integer rows; default/fallback 0 means automatic policy; numeric prefixes accepted; used both as verifier cap and as an exact-2 proposer-policy discriminator Cap speculative rows verified from SSD and influence exact-2 proposal sizing. ds4.c:52380 +runtime/dspark DS4_DSPARK_STAGE_PROFILE presence flag; unset is off; any defined value, including empty or 0, is on; DS4_DSPARK_STAGE_PROFILE_STAGE must also match Profile DSpark support stages with command-boundary timings. ds4.c:33416 +runtime/dspark DS4_DSPARK_STAGE_PROFILE_STAGE selector subordinate to DS4_DSPARK_STAGE_PROFILE; unset/empty: match every stage; otherwise strtoul base 10 must consume the whole value, fit uint32_t, and equal the current stage; invalid/out-of-range values match no stage Restricts DSpark stage-boundary timing output to one stage; it does not enable profiling by itself. ds4.c:33417 +runtime/dspark DS4_DSPARK_STATS value-aware flag; default off; nonempty value other than exact 0 enables; false/off are treated as enabled Collect and print aggregate DSpark runtime statistics. ds4.c:61607 runtime/dspark DS4_DSPARK_VERIFY_CACHE presence diagnostic; unset: off; any presence including empty or 0 enables on each support-cache installation CUDA only: copies every installed nonempty DSpark/support-cache range back to the host, byte-compares it with its source, and logs each mismatch plus a bad-count summary without changing the install result. ds4_cuda.cu:8257 -runtime/dspark DS4_DSPARK_VERIFY_HEAD_NO_TP presence rollback; unset: allow eligible CUDA output tensor parallelism; any presence including empty or 0 removes the TP path from eligibility CUDA only: forces the DSpark speculative batched vocabulary head away from output-TP for correctness isolation; under CUDA TP+EP the attempt fails instead of using unavailable full output weights. ds4.c:26468 +runtime/dspark DS4_DSPARK_VERIFY_HEAD_NO_TP presence rollback; unset: allow eligible CUDA output tensor parallelism; any presence including empty or 0 removes the TP path from eligibility CUDA only: forces the DSpark speculative batched vocabulary head away from output-TP for correctness isolation; under CUDA TP+EP the attempt fails instead of using unavailable full output weights. ds4.c:26641 runtime/dspark DS4_DSPARK_VERIFY_NONCAUSAL presence diagnostic sampled once after the first successfully submitted CUDA noncausal-attention kernel; unset: verify 0 calls; any presence including empty or 0: verify that call and the next 2 CUDA only: synchronizes and reads back Q/KV/output, computes the DSpark noncausal attention CPU reference, and logs max absolute/relative error; it reports only and does not fail the operation. ds4_cuda.cu:21736 -runtime/dspark DS4_DSPARK_VERIFY_PROFILE cached presence diagnostic; unset is off; any defined value including empty/0 profiles only the first eligible verifier invocation Profile one full DSpark target-verifier invocation layer by layer. ds4.c:36567 -runtime/dspark DS4_DSPARK_VERIFY_SELECTED_PROFILE presence flag; unset is off; any defined value, including empty or 0, enables unless DS4_DSPARK_DISABLE_VERIFY_SELECTED_PROFILE is also present (disable wins) Profile selected-expert streaming inside the DSpark verifier. ds4.c:36469 -runtime/dspark DS4_DSPARK_VERIFY_SPLIT_HEAD nonempty boolean with inverted default; unset/empty or exact 0: fused head; every other nonempty value: split head Runs the DSpark suffix verifier output head and top-1 reduction in a separate GPU command section after the layer loop, for timing/correctness isolation; default keeps them fused into the layer command section. ds4.c:36535 -runtime/dspark DS4_DSPARK_VERIFY_TOPS_CHECK presence flag; unset is off; any defined value, including empty or 0, is on; normal eligibility still applies Read back verifier logits and compare GPU top IDs with CPU argmax. ds4.c:36683 -runtime/glm DS4_GLM_ABLATE_COMBINE presence ablation; unset: exchange the local TP partial with the peer and add both halves; any presence including empty or 0 skips the exchange Metal two-rank TP timing probe: doubles the local routed-MoE or split-attention partial instead of combining with the peer, deliberately producing invalid output; both ranks must set it or their exchange gates desynchronize. ds4.c:43991 +runtime/dspark DS4_DSPARK_VERIFY_PROFILE cached presence diagnostic; unset is off; any defined value including empty/0 profiles only the first eligible verifier invocation Profile one full DSpark target-verifier invocation layer by layer. ds4.c:36757 +runtime/dspark DS4_DSPARK_VERIFY_SELECTED_PROFILE presence flag; unset is off; any defined value, including empty or 0, enables unless DS4_DSPARK_DISABLE_VERIFY_SELECTED_PROFILE is also present (disable wins) Profile selected-expert streaming inside the DSpark verifier. ds4.c:36659 +runtime/dspark DS4_DSPARK_VERIFY_SPLIT_HEAD nonempty boolean with inverted default; unset/empty or exact 0: fused head; every other nonempty value: split head Runs the DSpark suffix verifier output head and top-1 reduction in a separate GPU command section after the layer loop, for timing/correctness isolation; default keeps them fused into the layer command section. ds4.c:36725 +runtime/dspark DS4_DSPARK_VERIFY_TOPS_CHECK presence flag; unset is off; any defined value, including empty or 0, is on; normal eligibility still applies Read back verifier logits and compare GPU top IDs with CPU argmax. ds4.c:36873 +runtime/glm DS4_GLM_ABLATE_COMBINE presence ablation; unset: exchange the local TP partial with the peer and add both halves; any presence including empty or 0 skips the exchange Metal two-rank TP timing probe: doubles the local routed-MoE or split-attention partial instead of combining with the peer, deliberately producing invalid output; both ranks must set it or their exchange gates desynchronize. ds4.c:44181 runtime/glm DS4_GLM_ATTN_NO_LORA_VEC2 presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it Disable vectorized two-row LoRA accumulation in CUDA GLM indexed attention. ds4_cuda.cu:34176 runtime/glm DS4_GLM_ATTN_NO_SCORE_VEC2 presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it Disable vectorized two-row score computation in CUDA GLM indexed attention. ds4_cuda.cu:34165 runtime/glm DS4_GLM_ATTN_NO_STAGED_DECODE presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it Disable staged CUDA GLM indexed-decode attention for large selected sets. ds4_cuda.cu:34214 -runtime/glm DS4_GLM_DECODE_ABLATE cached substring list; default empty mask; recognized tokens are attn_out, attn_core, qpath, indexer, routed, shared, qklow; unknown text has no effect; matching stages are skipped and output is invalid Skip selected GLM decode stages for timing attribution; generated output is invalid. ds4.c:44238 -runtime/glm DS4_GLM_DECODE_FLUSH_INTERVAL integer layers via atoi; default 4 for indexed decode and 32 otherwise; <=0/nonnumeric disables periodic flush; capped to layer count and forced to 0 for deferred completion Set how often non-streaming GLM decode command work is flushed between layers. ds4.c:49628 -runtime/glm DS4_GLM_DISABLE_FLASH_PREFILL presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it Disable GLM Flash Attention prefill. ds4.c:44897 -runtime/glm DS4_GLM_DISABLE_STREAMING_TOKEN_PREFILL presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it; either backend-specific ROCm/Metal alias also disables Disable token-major GLM SSD-streaming prefill. ds4.c:49512 +runtime/glm DS4_GLM_DECODE_ABLATE cached substring list; default empty mask; recognized tokens are attn_out, attn_core, qpath, indexer, routed, shared, qklow; unknown text has no effect; matching stages are skipped and output is invalid Skip selected GLM decode stages for timing attribution; generated output is invalid. ds4.c:44411 +runtime/glm DS4_GLM_DECODE_FLUSH_INTERVAL integer layers via atoi; default 4 for indexed decode and 32 otherwise; <=0/nonnumeric disables periodic flush; capped to layer count and forced to 0 for deferred completion Set how often non-streaming GLM decode command work is flushed between layers. ds4.c:49818 +runtime/glm DS4_GLM_DISABLE_FLASH_PREFILL presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it Disable GLM Flash Attention prefill. ds4.c:45087 +runtime/glm DS4_GLM_DISABLE_STREAMING_TOKEN_PREFILL presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it; either backend-specific ROCm/Metal alias also disables Disable token-major GLM SSD-streaming prefill. ds4.c:49702 runtime/glm DS4_GLM_FENCE_TRACE presence flag; unset is off; any defined value, including empty or 0, is on; normal eligibility still applies Log fenced CUDA tier switches used by GLM multi-GPU execution. ds4_cuda.cu:7984 runtime/glm DS4_GLM_GEMM_TRACE presence flag; unset is off; any defined value, including empty or 0, is on; normal eligibility still applies Measure and print CUDA GLM dequantization and cuBLAS GEMM timings. ds4_cuda.cu:19706 -runtime/glm DS4_GLM_HIDDEN_DUMP nonempty filesystem path/prefix; unset/empty disables; writes final hidden row or per-layer files selected by DS4_GLM_HIDDEN_DUMP_LAYER Dump GLM hidden-state rows for correctness isolation. ds4.c:38333 -runtime/glm DS4_GLM_HIDDEN_DUMP_LAYER selector; unset/empty = -1 (no per-layer dump, final hidden still dumped when path set); all = every layer; otherwise atoi result selects a layer, so invalid text selects layer 0 Choose which GLM layer hidden states are dumped. ds4.c:38353 -runtime/glm DS4_GLM_KV_DUMP nonempty filesystem prefix; unset/empty disables; writes layer-0 lora and rope compact-cache files after sync Dump layer-0 compact GLM KV cache data after prompt synchronization. ds4.c:62941 -runtime/glm DS4_GLM_LOGIT_DUMP nonempty filesystem path; unset/empty disables; dumps the first post-prefill logits vector once per process Dump the first post-prefill GLM logits vector. ds4.c:38411 -runtime/glm DS4_GLM_MEMORY_GUARD guard is on by default; exact 0 or case-insensitive false/off/no disables it; any other value and unset keep it enabled Control the pre-allocation GLM host/GPU memory safety guard. ds4.c:41799 -runtime/glm DS4_GLM_MEMORY_GUARD_FRACTION floating-point fraction; default 0.99; parsed numeric prefix is accepted, invalid/nonfinite falls back, values clamp to 0.50..1.00 Set the fraction of detected memory usable by the GLM memory guard. ds4.c:41838 -runtime/glm DS4_GLM_MEMORY_GUARD_REPORT nonempty diagnostic flag; unset/empty is off; any nonempty value including 0 prints successful-admission accounting (refusals always report) Print successful GLM memory-guard budget accounting. ds4.c:41878 -runtime/glm DS4_GLM_MEMORY_GUARD_RESERVE_GB floating-point GiB; dynamic default (normally 32, 24 on near-full 480..640 GiB hosts, possibly lower for resident ROCm slices); numeric prefixes accepted; invalid falls back; clamp 0..1024 Set fixed headroom subtracted by the GLM memory guard. ds4.c:41856 +runtime/glm DS4_GLM_HIDDEN_DUMP nonempty filesystem path/prefix; unset/empty disables; writes final hidden row or per-layer files selected by DS4_GLM_HIDDEN_DUMP_LAYER Dump GLM hidden-state rows for correctness isolation. ds4.c:38523 +runtime/glm DS4_GLM_HIDDEN_DUMP_LAYER selector; unset/empty = -1 (no per-layer dump, final hidden still dumped when path set); all = every layer; otherwise atoi result selects a layer, so invalid text selects layer 0 Choose which GLM layer hidden states are dumped. ds4.c:38543 +runtime/glm DS4_GLM_KV_DUMP nonempty filesystem prefix; unset/empty disables; writes layer-0 lora and rope compact-cache files after sync Dump layer-0 compact GLM KV cache data after prompt synchronization. ds4.c:63131 +runtime/glm DS4_GLM_LOGIT_DUMP nonempty filesystem path; unset/empty disables; dumps the first post-prefill logits vector once per process Dump the first post-prefill GLM logits vector. ds4.c:38601 +runtime/glm DS4_GLM_MEMORY_GUARD guard is on by default; exact 0 or case-insensitive false/off/no disables it; any other value and unset keep it enabled Control the pre-allocation GLM host/GPU memory safety guard. ds4.c:41989 +runtime/glm DS4_GLM_MEMORY_GUARD_FRACTION floating-point fraction; default 0.99; parsed numeric prefix is accepted, invalid/nonfinite falls back, values clamp to 0.50..1.00 Set the fraction of detected memory usable by the GLM memory guard. ds4.c:42028 +runtime/glm DS4_GLM_MEMORY_GUARD_REPORT nonempty diagnostic flag; unset/empty is off; any nonempty value including 0 prints successful-admission accounting (refusals always report) Print successful GLM memory-guard budget accounting. ds4.c:42068 +runtime/glm DS4_GLM_MEMORY_GUARD_RESERVE_GB floating-point GiB; dynamic default (normally 32, 24 on near-full 480..640 GiB hosts, possibly lower for resident ROCm slices); numeric prefixes accepted; invalid falls back; clamp 0..1024 Set fixed headroom subtracted by the GLM memory guard. ds4.c:42046 runtime/glm DS4_GLM_MOE_EXPERT_MAJOR presence selector; unset: off; any presence including empty or 0 requests the path only for n_tokens >= 16; the automatic tile-8 path takes precedence when enabled (normally n_tokens >= 128) CUDA only: groups selected token/expert pairs by expert and uses expert-major Q2_K routed-MoE gate/up/down kernels to reuse expert weights; otherwise the normal token-major path is used. ds4_cuda.cu:35973 runtime/glm DS4_GLM_MOE_NO_DOWN_TILE8_EXACT presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it Disable the exact tile-8 CUDA GLM routed-MoE down projection. ds4_cuda.cu:36028 runtime/glm DS4_GLM_MOE_NO_EXPERT_TILE8 presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it Disable automatic expert tile-8 CUDA GLM routed-MoE batching. ds4_cuda.cu:35971 @@ -445,599 +445,603 @@ runtime/glm DS4_GLM_MOE_SCALAR presence rollback; unset: use optimized warp kern runtime/glm DS4_GLM_MOE_SCRATCH_TIER0 presence placement override; unset: allocate xq/midq quantization scratch on the current logical tier; any presence including empty or 0 allocates it on logical tier 0 CUDA only: pins the routed-MoE xq_scratch and midq_scratch allocations to GPU tier 0 for multi-tier placement experiments; other MoE scratch remains on the current tier. ds4_cuda.cu:35918 runtime/glm DS4_GLM_MTP_NO_ATTN_TOK2 presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it Disable the exact two-token CUDA GLM MTP attention kernel. ds4_cuda.cu:34168 runtime/glm DS4_GLM_MTP_NO_MOE_TOK2 presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it Disable the exact two-token CUDA GLM MTP routed-MoE kernel. ds4_cuda.cu:36140 -runtime/glm DS4_GLM_MTP_NO_SHARED_TOK2 presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it Disable the exact two-token CUDA GLM MTP shared-FFN kernel. ds4_cuda.cu:37920 -runtime/glm DS4_GLM_MTP_PROBE presence flag; unset is off; any defined value, including empty or 0, is on; its second call site also rejects the normal batching path Run the GLM next-N/MTP acceptance quality-and-timing probe without changing output and force probe-compatible scheduling. ds4.c:64769 -runtime/glm DS4_GLM_PREFILL_TRUNC nonempty value parsed by atoi (leading whitespace/sign accepted and trailing junk ignored); effective only when the resulting int is > 0 and < the current prompt length; unset/empty or a result <= 0 or >= prompt length leaves the prompt unchanged GPU GLM debug hook: truncates the prompt before prefill/checkpoint handling so dumped prefill logits can be aligned with a CPU first-token reference. ds4.c:63080 -runtime/glm DS4_GLM_RESUME_PREFILL_MIN integer suffix tokens, non-ROCm builds only; default 4; parsed numeric prefixes accepted; <=0 maps to UINT32_MAX and effectively disables batched resume; ROCm build ignores it and stays at 4 Set the suffix-length crossover from token decode to batched resumed prefill. ds4.c:38244 +runtime/glm DS4_GLM_MTP_NO_SHARED_TOK2 presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it Disable the exact two-token CUDA GLM MTP shared-FFN kernel. ds4_cuda.cu:37940 +runtime/glm DS4_GLM_MTP_PROBE presence flag; unset is off; any defined value, including empty or 0, is on; its second call site also rejects the normal batching path Run the GLM next-N/MTP acceptance quality-and-timing probe without changing output and force probe-compatible scheduling. ds4.c:64959 +runtime/glm DS4_GLM_PREFILL_TRUNC nonempty value parsed by atoi (leading whitespace/sign accepted and trailing junk ignored); effective only when the resulting int is > 0 and < the current prompt length; unset/empty or a result <= 0 or >= prompt length leaves the prompt unchanged GPU GLM debug hook: truncates the prompt before prefill/checkpoint handling so dumped prefill logits can be aligned with a CPU first-token reference. ds4.c:63270 +runtime/glm DS4_GLM_RESUME_PREFILL_MIN integer suffix tokens, non-ROCm builds only; default 4; parsed numeric prefixes accepted; <=0 maps to UINT32_MAX and effectively disables batched resume; ROCm build ignores it and stays at 4 Set the suffix-length crossover from token decode to batched resumed prefill. ds4.c:38434 runtime/glm DS4_GLM_ROUTER_SCALAR presence rollback; unset: use the 256-thread parallel router when n_expert <= 256 (the scalar path is already automatic above 256); any presence including empty or 0 forces the scalar path CUDA only: selects the one-active-thread-per-token sigmoid/top-k router kernel instead of the parallel shared-memory reduction, for A/B or correctness testing. ds4_cuda.cu:36390 -runtime/glm DS4_GLM_SHARED_SPLIT presence rollback; unset: use the fused one-token shared-expert Q8_0 gate+up+SwiGLU kernel when its shape/buffers are eligible; any presence including empty or 0 skips that fused one-token path; the earlier two-token MTP-specialized path is unaffected CUDA only: forces shared-expert gate and up through two separate Q8_0 matmuls followed by a separate SwiGLU operation for one-token decode. ds4_cuda.cu:37946 -runtime/glm DS4_GLM_STREAMING_DECODE_FULL_LAYER_MAP presence compatibility alias; unset: automatic mapping; any presence including empty or 0 independently forces full-layer mapping, equivalent to the backend-specific DS4_ROCM_GLM_STREAMING_DECODE_FULL_LAYER_MAP or DS4_METAL_GLM_STREAMING_DECODE_FULL_LAYER_MAP control Backend-neutral alias for supported GLM SSD streaming (Metal/ROCm): maps every tensor in each decode layer instead of using the decode-only map that can omit routed experts served by the expert cache; layers that already require a full map are unchanged. ds4.c:42427 -runtime/glm DS4_GLM_STREAMING_DECODE_SYNC_EACH_LAYER ROCm-only third-priority legacy value: a nonempty DS4_ROCM_GLM_STREAMING_DECODE_SYNC_EACH_LAYER wins, otherwise a nonempty DS4_METAL_GLM_STREAMING_DECODE_SYNC_EACH_LAYER wins, otherwise this name is read; nonempty values are true except exact 0 or case-insensitive false/off/no; unset/empty: false; non-ROCm builds always return true and ignore this name On ROCm non-static GLM SSD decode, opts into ending/synchronizing commands after token mapping and after every layer; the default keeps ordered work alive across layer mappings. Static-map decode bypasses this control. ds4.c:49591 -runtime/glm DS4_GLM_STREAMING_PREFILL_SYNC_EACH_LAYER ROCm-only third-priority legacy value: a nonempty DS4_ROCM_GLM_STREAMING_PREFILL_SYNC_EACH_LAYER wins, otherwise a nonempty DS4_METAL_GLM_STREAMING_PREFILL_SYNC_EACH_LAYER wins, otherwise this name is read; nonempty values are true except exact 0 or case-insensitive false/off/no; unset/empty: false for compact prefill; full-layer prefill and non-ROCm builds always synchronize and ignore this name On ROCm compact GLM SSD prefill, opts into ending/synchronizing commands at every layer boundary; the default carries ordered work across mappings, while full-layer expert-cache prefill always retains the boundary. ds4.c:42207 -runtime/glm DS4_GLM_STREAMING_TOKEN_PREFILL_MAX unsigned token limit; backend-specific ROCm/Metal variable takes precedence, generic is fallback; default 0 on ROCm and 64 otherwise; invalid/overflow falls back, numeric prefixes accepted; 0 disables token-major streaming prefill Set the largest SSD-streaming prefill handled by the token-major decode-like path. ds4.c:49494 -runtime/glm DS4_GLM_SYNC_TRACE presence flag; unset is off; any defined value, including empty or 0, is on; normal eligibility still applies Log GLM checkpoint/resume and dense-versus-indexed prefill decisions. ds4.c:63178 -runtime/glm DS4_GLM_TP_DEBUG presence flag; unset is off; any defined value, including empty or 0, is on; normal eligibility still applies Print CUDA/GLM tensor-parallel dispatch, gate, selected-ID, and failure diagnostics. ds4.c:43893 -runtime/glm DS4_GLM_TP_EXACT_PREFILL_MAX integer suffix limit via atoi cast to uint32; default 64; nonnumeric becomes 0; negative values wrap to a very large unsigned limit Set the maximum two-way TP suffix that uses exact token-by-token prefill. ds4.c:63120 -runtime/glm DS4_GLM_TP_HEAD_SPLIT_MIN cached integer token threshold via atoi; default 64; negative values clamp to 0, nonnumeric becomes 0; 0 admits all otherwise-eligible batches Set the minimum batch size for GLM tensor-parallel output-head splitting. ds4.c:38323 +runtime/glm DS4_GLM_SHARED_SPLIT presence rollback; unset: use the fused one-token shared-expert Q8_0 gate+up+SwiGLU kernel when its shape/buffers are eligible; any presence including empty or 0 skips that fused one-token path; the earlier two-token MTP-specialized path is unaffected CUDA only: forces shared-expert gate and up through two separate Q8_0 matmuls followed by a separate SwiGLU operation for one-token decode. ds4_cuda.cu:37966 +runtime/glm DS4_GLM_STREAMING_DECODE_FULL_LAYER_MAP presence compatibility alias; unset: automatic mapping; any presence including empty or 0 independently forces full-layer mapping, equivalent to the backend-specific DS4_ROCM_GLM_STREAMING_DECODE_FULL_LAYER_MAP or DS4_METAL_GLM_STREAMING_DECODE_FULL_LAYER_MAP control Backend-neutral alias for supported GLM SSD streaming (Metal/ROCm): maps every tensor in each decode layer instead of using the decode-only map that can omit routed experts served by the expert cache; layers that already require a full map are unchanged. ds4.c:42617 +runtime/glm DS4_GLM_STREAMING_DECODE_SYNC_EACH_LAYER ROCm-only third-priority legacy value: a nonempty DS4_ROCM_GLM_STREAMING_DECODE_SYNC_EACH_LAYER wins, otherwise a nonempty DS4_METAL_GLM_STREAMING_DECODE_SYNC_EACH_LAYER wins, otherwise this name is read; nonempty values are true except exact 0 or case-insensitive false/off/no; unset/empty: false; non-ROCm builds always return true and ignore this name On ROCm non-static GLM SSD decode, opts into ending/synchronizing commands after token mapping and after every layer; the default keeps ordered work alive across layer mappings. Static-map decode bypasses this control. ds4.c:49781 +runtime/glm DS4_GLM_STREAMING_PREFILL_SYNC_EACH_LAYER ROCm-only third-priority legacy value: a nonempty DS4_ROCM_GLM_STREAMING_PREFILL_SYNC_EACH_LAYER wins, otherwise a nonempty DS4_METAL_GLM_STREAMING_PREFILL_SYNC_EACH_LAYER wins, otherwise this name is read; nonempty values are true except exact 0 or case-insensitive false/off/no; unset/empty: false for compact prefill; full-layer prefill and non-ROCm builds always synchronize and ignore this name On ROCm compact GLM SSD prefill, opts into ending/synchronizing commands at every layer boundary; the default carries ordered work across mappings, while full-layer expert-cache prefill always retains the boundary. ds4.c:42397 +runtime/glm DS4_GLM_STREAMING_TOKEN_PREFILL_MAX unsigned token limit; backend-specific ROCm/Metal variable takes precedence, generic is fallback; default 0 on ROCm and 64 otherwise; invalid/overflow falls back, numeric prefixes accepted; 0 disables token-major streaming prefill Set the largest SSD-streaming prefill handled by the token-major decode-like path. ds4.c:49684 +runtime/glm DS4_GLM_SYNC_TRACE presence flag; unset is off; any defined value, including empty or 0, is on; normal eligibility still applies Log GLM checkpoint/resume and dense-versus-indexed prefill decisions. ds4.c:63368 +runtime/glm DS4_GLM_TP_DEBUG presence flag; unset is off; any defined value, including empty or 0, is on; normal eligibility still applies Print CUDA/GLM tensor-parallel dispatch, gate, selected-ID, and failure diagnostics. ds4.c:44083 +runtime/glm DS4_GLM_TP_EXACT_PREFILL_MAX integer suffix limit via atoi cast to uint32; default 64; nonnumeric becomes 0; negative values wrap to a very large unsigned limit Set the maximum two-way TP suffix that uses exact token-by-token prefill. ds4.c:63310 +runtime/glm DS4_GLM_TP_HEAD_SPLIT_MIN cached integer token threshold via atoi; default 64; negative values clamp to 0, nonnumeric becomes 0; 0 admits all otherwise-eligible batches Set the minimum batch size for GLM tensor-parallel output-head splitting. ds4.c:38513 runtime/glm DS4_GLM_VALUE_NO_TILE16 presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it Disable the CUDA GLM 16-token tiled value-projection kernel. ds4_cuda.cu:36806 -runtime/metal DS4_METAL_ARGSORT_SOURCE file path; unset/empty: use the in-tree Metal source file Overrides the argsort Metal kernel source file loaded at runtime. ds4_metal.m:4942 -runtime/metal DS4_METAL_ATTN_OUT_STAGE_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for attn out stage. ds4.c:64961 -runtime/metal DS4_METAL_BIN_SOURCE file path; unset/empty: use the in-tree Metal source file Overrides the binary operations Metal kernel source file loaded at runtime. ds4_metal.m:4951 -runtime/metal DS4_METAL_COMPRESSOR_PAIR_NR4 presence control; unset: off/default; any value including 0 enables Selects the NR4 compressor-pair variant. ds4_metal.m:2650 -runtime/metal DS4_METAL_CONCAT_SOURCE file path; unset/empty: use the in-tree Metal source file Overrides the concatenation Metal kernel source file loaded at runtime. ds4_metal.m:4944 -runtime/metal DS4_METAL_CPY_SOURCE file path; unset/empty: use the in-tree Metal source file Overrides the copy Metal kernel source file loaded at runtime. ds4_metal.m:4943 -runtime/metal DS4_METAL_DECODE_INDEXER_SPARSE_THRESHOLD integer in {64,128,256,512,1024,2048,4096}; default 1024; invalid restores default Sets the compressed-row crossover from dense to sparse indexed attention. ds4.c:20179 -runtime/metal DS4_METAL_DECODE_STAGE_PROFILE unset: off; 1/true/yes/on/all enables all layers; a layer index selects one; 0/false/no/off disables Prints timing/profile diagnostics for decode stage. ds4.c:17395 -runtime/metal DS4_METAL_DECODE_STAGE_PROFILE_LAYER single unsigned layer index; unset/empty: all layers enabled by the parent profile; invalid matches no layer Restricts the corresponding shared graph stage profiler to one layer. ds4.c:28947 -runtime/metal DS4_METAL_DENSE_SOURCE file path; unset/empty: use the in-tree Metal source file Overrides the dense matmul Metal kernel source file loaded at runtime. ds4_metal.m:4935 -runtime/metal DS4_METAL_DISABLE_AFFINE_ROPE_PAIR presence rollback; unset: automatic/default path; any value including 0 disables Disables affine RoPE pair. ds4_metal.m:25169 -runtime/metal DS4_METAL_DISABLE_ATTN_OUT_HC_FUSION nonempty boolean; unset/empty or exact 0: off; every other value: on Disables attn out HC fusion. ds4.c:20304 -runtime/metal DS4_METAL_DISABLE_ATTN_OUT_IDS_CACHE presence rollback; unset: automatic/default path; any value including 0 disables Disables attn out ids cache. ds4_metal.m:27769 -runtime/metal DS4_METAL_DISABLE_ATTN_OUT_LOW_DIRECT presence rollback; unset: automatic/default path; any value including 0 disables Disables attn out low direct. ds4_metal.m:27758 -runtime/metal DS4_METAL_DISABLE_BATCH_HC_NORM_FUSION nonempty value other than exact 0 disables; unset/empty/0 leaves the default enabled path Dominant rollback for batched HC norm fusion. ds4.c:20284 -runtime/metal DS4_METAL_DISABLE_COMPRESSOR_APE_ADD presence rollback; unset: automatic/default path; any value including 0 disables Disables compressor APE add. ds4_metal.m:25391 -runtime/metal DS4_METAL_DISABLE_COMPRESSOR_EXACT_POOL_RATIO4 presence rollback; unset: automatic/default path; any value including 0 disables Disables compressor exact pool ratio4. ds4_metal.m:26377 -runtime/metal DS4_METAL_DISABLE_COMPRESSOR_PAIR_PROJ nonempty boolean; unset/empty or exact 0: off; every other value: on Disables compressor pair proj. ds4_metal.m:23246 -runtime/metal DS4_METAL_DISABLE_COMPRESSOR_QUAD_STORE presence rollback; unset: automatic/default path; any value including 0 disables Disables compressor quad store. ds4.c:23361 -runtime/metal DS4_METAL_DISABLE_COMPRESSOR_RATIO4_DIRECT_POOL presence rollback; unset: automatic/default path; any value including 0 disables Disables compressor ratio4 direct pool. ds4_metal.m:26243 -runtime/metal DS4_METAL_DISABLE_COMPRESSOR_RATIO4_PACK_FUSION presence rollback; unset: automatic/default path; any value including 0 disables Disables compressor ratio4 pack fusion. ds4_metal.m:26197 -runtime/metal DS4_METAL_DISABLE_COMPRESSOR_STORE_ONE presence rollback; unset: automatic/default path; any value including 0 disables Disables compressor store one. ds4_metal.m:23247 -runtime/metal DS4_METAL_DISABLE_CONTIG_F16_F16_COPY value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Disables contig F16 F16 copy. ds4_metal.m:29538 -runtime/metal DS4_METAL_DISABLE_CONTIG_F32_F16_COPY value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Disables contig F32 F16 copy. ds4_metal.m:29314 -runtime/metal DS4_METAL_DISABLE_DECODE_NORM_EXACT_VIEWS presence rollback; unset: automatic/default path; any value including 0 disables Disables decode norm exact views. ds4_metal.m:36173 -runtime/metal DS4_METAL_DISABLE_DECODE_ROUTER_BIAS_EXACT_VIEWS presence rollback; unset: automatic/default path; any value including 0 disables Disables decode router bias exact views. ds4_metal.m:39363 -runtime/metal DS4_METAL_DISABLE_DSPARK_CAPTURE_FUSED_LAST nonempty boolean; unset/empty or exact 0: off; every other value: on Disables DSpark capture fused last. ds4.c:28188 -runtime/metal DS4_METAL_DISABLE_DSPARK_EXACTN_BATCH_HEAD nonempty boolean; unset/empty or exact 0: off; every other value: on Disables DSpark exactn batch head. ds4.c:37538 -runtime/metal DS4_METAL_DISABLE_EXACT_ROWS_PERSISTENT_CACHE nonempty boolean; unset/empty or exact 0: off; every other value: on Disables exact rows persistent cache. ds4_metal.m:12998 -runtime/metal DS4_METAL_DISABLE_GATHERED_KV_PAD_FUSION presence rollback; unset: automatic/default path; any value including 0 disables Disables gathered KV pad fusion. ds4_metal.m:29682 -runtime/metal DS4_METAL_DISABLE_GATHERED_KV_STAGE presence rollback; unset: automatic/default path; any value including 0 disables Disables gathered KV stage. ds4_metal.m:29653 -runtime/metal DS4_METAL_DISABLE_GLM_DECODE_KV_GROUP4 value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Disables GLM decode KV group4. ds4_metal.m:36708 -runtime/metal DS4_METAL_DISABLE_GLM_QKLOW_SG presence rollback; unset: automatic/default path; any value including 0 disables Disables GLM qklow sg. ds4_metal.m:37831 -runtime/metal DS4_METAL_DISABLE_GLM_STREAMING_EXPERT_EARLY_LOAD presence rollback; unset: automatic/default path; any value including 0 disables Disables GLM streaming expert early load. ds4_metal.m:18056 -runtime/metal DS4_METAL_DISABLE_GLM_STREAMING_EXPERT_SPLIT presence rollback; unset: automatic/default path; any value including 0 disables Disables GLM streaming expert split. ds4_metal.m:39762 -runtime/metal DS4_METAL_DISABLE_GLM_STREAMING_PREFILL_FULL_LAYER presence rollback; unset: automatic/default path; any value including 0 disables Disables GLM streaming prefill full layer. ds4.c:42520 -runtime/metal DS4_METAL_DISABLE_GLM_STREAMING_PREFILL_FULL_LAYER_PREPARE presence rollback; unset: automatic/default path; any value including 0 disables Disables GLM streaming prefill full layer prepare. ds4.c:42538 -runtime/metal DS4_METAL_DISABLE_GLM_STREAMING_PREFILL_SELECTED_ASYNC_LOAD presence rollback; unset: automatic/default path; any value including 0 disables Disables GLM streaming prefill selected async load. ds4.c:46231 -runtime/metal DS4_METAL_DISABLE_GLM_STREAMING_SELECTED_ASYNC_LOAD presence rollback; unset: automatic/default path; any value including 0 disables Disables GLM streaming selected async load. ds4.c:44139 -runtime/metal DS4_METAL_DISABLE_HC_FUSION nonempty boolean; unset/empty or exact 0: off; every other value: on Disables HC fusion. ds4.c:20233 -runtime/metal DS4_METAL_DISABLE_HC_NORM_FUSION nonempty boolean; unset/empty or exact 0: off; every other value: on Disables HC norm fusion. ds4.c:20277 -runtime/metal DS4_METAL_DISABLE_HC_PRODUCER_PRE_NORM_FUSE presence rollback; unset: automatic/default path; any value including 0 disables Disables HC producer pre norm fuse. ds4_metal.m:46507 -runtime/metal DS4_METAL_DISABLE_HC_RMS_SCALE_PROJ presence rollback; unset: automatic/default path; any value including 0 disables Disables HC RMS scale proj. ds4_metal.m:24146 -runtime/metal DS4_METAL_DISABLE_HOT_PIPELINE_STATICS presence rollback; unset: automatic/default path; any value including 0 disables Disables hot pipeline statics. ds4_metal.m:2633 -runtime/metal DS4_METAL_DISABLE_INPLACE_ROPE_PAIR presence rollback; unset: automatic/default path; any value including 0 disables Disables inplace RoPE pair. ds4_metal.m:25168 -runtime/metal DS4_METAL_DISABLE_IQ2_SELECTED_EXPERT_VIEWS presence rollback; unset: automatic/default path; any value including 0 disables Disables IQ2 selected expert views. ds4.c:21069 -runtime/metal DS4_METAL_DISABLE_IQ2_SELECTED_SHARED_OVERLAP presence rollback; unset: automatic/default path; any value including 0 disables Disables IQ2 selected shared overlap. ds4.c:20823 -runtime/metal DS4_METAL_DISABLE_IQ2_STREAM_ADDR_TABLE presence rollback; unset: automatic/default path; any value including 0 disables Disables IQ2 stream address table. ds4_metal.m:42368 -runtime/metal DS4_METAL_DISABLE_IQ2_XXS_SSD_PREFILL_MM value-aware boolean; default off; true disables and dominates ENABLE; false leaves automatic policy Rolls grouped IQ2_XXS/Q2_K SSD-prefill MM back to sparse matvec. ds4_metal.m:44750 -runtime/metal DS4_METAL_DISABLE_KV_FUSION nonempty boolean; unset/empty or exact 0: off; every other value: on Disables KV fusion. ds4.c:20257 -runtime/metal DS4_METAL_DISABLE_M1_IQ2_MID_ONLY presence rollback; unset: automatic/default path; any value including 0 disables Disables M1 IQ2 mid only. ds4_metal.m:14590 -runtime/metal DS4_METAL_DISABLE_M3_COMPRESSOR_EXACT_POOL_RATIO4 presence rollback; unset: automatic/default path; any value including 0 disables Disables M3 compressor exact pool ratio4. ds4_metal.m:26379 -runtime/metal DS4_METAL_DISABLE_M3_COMPRESSOR_PAIR_STATE_STORE presence rollback; unset: automatic/default path; any value including 0 disables Disables M3 compressor pair state store. ds4_metal.m:23245 -runtime/metal DS4_METAL_DISABLE_M3_GATHERED_KV_STAGE presence rollback; unset: automatic/default path; any value including 0 disables Disables M3 gathered KV stage. ds4_metal.m:29654 -runtime/metal DS4_METAL_DISABLE_M5_COMPRESSOR_EXACT_POOL_RATIO4 presence rollback; unset: automatic/default path; any value including 0 disables Disables M5 compressor exact pool ratio4. ds4_metal.m:26382 -runtime/metal DS4_METAL_DISABLE_M5_COMP_FINALIZE_FUSE presence rollback; unset: automatic/default path; any value including 0 disables Disables M5 comp finalize fuse. ds4.c:23474 -runtime/metal DS4_METAL_DISABLE_M5_FLASH_ATTN_PACKED32_REDUCE presence rollback; unset: automatic/default path; any value including 0 disables Disables M5 flash attn packed32 reduce. ds4_metal.m:31490 -runtime/metal DS4_METAL_DISABLE_M5_HC_NORM_MIX_CLUSTER2 presence rollback; unset: automatic/default path; any value including 0 disables Disables M5 HC norm mix cluster2. ds4_metal.m:46462 -runtime/metal DS4_METAL_DISABLE_M5_HC_PRODUCER_PRE_NORM_FUSE presence rollback; unset: automatic/default path; any value including 0 disables Disables M5 HC producer pre norm fuse. ds4_metal.m:46514 -runtime/metal DS4_METAL_DISABLE_M5_IQ2_PAIR_PACK2 presence rollback; unset: automatic/default path; any value including 0 disables Disables M5 IQ2 pair pack2. ds4_metal.m:41986 -runtime/metal DS4_METAL_DISABLE_M5_PACKED_ZERO_MASK presence rollback; unset: automatic/default path; any value including 0 disables Disables M5 packed zero mask. ds4_metal.m:31446 -runtime/metal DS4_METAL_DISABLE_M5_PARALLEL_FULL_FFN presence rollback; unset: automatic/default path; any value including 0 disables Disables M5 parallel full FFN. ds4.c:22534 -runtime/metal DS4_METAL_DISABLE_M5_PERSISTENT_ZERO_ATTN_MASK presence rollback; unset: automatic/default path; any value including 0 disables Disables M5 persistent zero attn mask. ds4_metal.m:31444 -runtime/metal DS4_METAL_DISABLE_M5_Q8_HC_VEC presence rollback; unset: automatic/default path; any value including 0 disables Disables M5 Q8 HC vec. ds4_metal.m:47546 -runtime/metal DS4_METAL_DISABLE_M5_QKV_PAIR_COMPRESSOR_FUSE presence rollback; unset: automatic/default path; any value including 0 disables Disables M5 QKV pair compressor fuse. ds4.c:22870 -runtime/metal DS4_METAL_DISABLE_M5_QKV_PAIR_QUAD_FUSE presence rollback; unset: automatic/default path; any value including 0 disables Disables M5 QKV pair quad fuse. ds4.c:22866 -runtime/metal DS4_METAL_DISABLE_M5_ROUTER_PROJECT_SELECT_FUSE presence rollback; unset: automatic/default path; any value including 0 disables Disables M5 router project select fuse. ds4.c:24589 -runtime/metal DS4_METAL_DISABLE_METAL4 value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Disables metal4. ds4_metal.m:2994 -runtime/metal DS4_METAL_DISABLE_MOE_MM_ID_PAIR_SWIGLU presence rollback; unset: automatic/default path; any value including 0 disables Disables MoE MM ID pair SwiGLU. ds4_metal.m:44934 -runtime/metal DS4_METAL_DISABLE_MOE_MM_ID_USE_RESOURCES presence rollback; unset: automatic/default path; any value including 0 disables Disables MoE MM ID use resources. ds4_metal.m:35135 -runtime/metal DS4_METAL_DISABLE_MXFP4_SELECTED_EXPERT_VIEWS presence rollback; unset: automatic/default path; any value including 0 disables Disables MXFP4 selected expert views. ds4.c:21000 -runtime/metal DS4_METAL_DISABLE_PERSISTENT_ZERO_ATTN_MASK presence rollback; unset: automatic/default path; any value including 0 disables Disables persistent zero attn mask. ds4_metal.m:31451 -runtime/metal DS4_METAL_DISABLE_PRE_M5_ATTN_INV_ROPE_FUSE presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 attn inv RoPE fuse. ds4.c:22465 -runtime/metal DS4_METAL_DISABLE_PRE_M5_COMPRESSOR_EXACT_POOL_RATIO4 presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 compressor exact pool ratio4. ds4_metal.m:26381 -runtime/metal DS4_METAL_DISABLE_PRE_M5_COMPRESSOR_EXACT_REDUCTION_FUSION presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 compressor exact reduction fusion. ds4_metal.m:25905 -runtime/metal DS4_METAL_DISABLE_PRE_M5_COMPRESSOR_QUAD_STORE presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 compressor quad store. ds4.c:23362 -runtime/metal DS4_METAL_DISABLE_PRE_M5_COMPRESSOR_RATIO4_DECODE_PACK_FUSION presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 compressor ratio4 decode pack fusion. ds4_metal.m:26212 -runtime/metal DS4_METAL_DISABLE_PRE_M5_COMP_FINALIZE_FUSE presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 comp finalize fuse. ds4.c:23473 -runtime/metal DS4_METAL_DISABLE_PRE_M5_DECODE_EARLY_PIPELINE_FAST_LOOKUP presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 decode early pipeline fast lookup. ds4.c:27830 -runtime/metal DS4_METAL_DISABLE_PRE_M5_DECODE_EARLY_SECOND_SPLIT12 presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 decode early second split12. ds4.c:27914 -runtime/metal DS4_METAL_DISABLE_PRE_M5_DECODE_EARLY_SPLIT3 presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 decode early split3. ds4.c:27788 -runtime/metal DS4_METAL_DISABLE_PRE_M5_DECODE_EARLY_SPLIT5 presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 decode early split5. ds4.c:27800 -runtime/metal DS4_METAL_DISABLE_PRE_M5_DECODE_PIPELINE_FAST_LOOKUP presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 decode pipeline fast lookup. ds4.c:27842 -runtime/metal DS4_METAL_DISABLE_PRE_M5_DECODE_PORTS presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 decode ports. ds4.c:22440 -runtime/metal DS4_METAL_DISABLE_PRE_M5_DECODE_RAW_ZERO_ATTN_MASK presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 decode raw zero attn mask. ds4_metal.m:29893 -runtime/metal DS4_METAL_DISABLE_PRE_M5_DECODE_SECOND_SPLIT16 presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 decode second split16. ds4.c:27922 -runtime/metal DS4_METAL_DISABLE_PRE_M5_FLASH_ATTN_BATCHED_MEMO presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 flash attn batched memo. ds4_metal.m:3751 -runtime/metal DS4_METAL_DISABLE_PRE_M5_FLASH_ATTN_PACKED32_REDUCE presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 flash attn packed32 reduce. ds4_metal.m:31433 -runtime/metal DS4_METAL_DISABLE_PRE_M5_FLASH_ATTN_PAD_BLK_MEMO presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 flash attn pad blk memo. ds4_metal.m:3609 -runtime/metal DS4_METAL_DISABLE_PRE_M5_HC_NORM_MIX_FUSE presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 HC norm mix fuse. ds4.c:22694 -runtime/metal DS4_METAL_DISABLE_PRE_M5_HC_PRODUCER_PRE_NORM_FUSE presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 HC producer pre norm fuse. ds4_metal.m:46512 -runtime/metal DS4_METAL_DISABLE_PRE_M5_HEAD_RMS_ROPE_PIPELINE_STATIC presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 head RMS RoPE pipeline static. ds4_metal.m:10025 -runtime/metal DS4_METAL_DISABLE_PRE_M5_KV_ROPE_FP8_FUSE presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 KV RoPE fp8 fuse. ds4.c:23282 -runtime/metal DS4_METAL_DISABLE_PRE_M5_MXFP4_MM_ID_PAIR_HALF_SCALE presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 MXFP4 MM ID pair half scale. ds4_metal.m:45095 -runtime/metal DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_DECODE_FIXED_ROUTE_PAIR presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 MXFP4 MoE decode fixed route pair. ds4_metal.m:41914 -runtime/metal DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_DECODE_FIXED_ROUTE_SUM6 presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 MXFP4 MoE decode fixed route sum6. ds4_metal.m:41927 -runtime/metal DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_DECODE_NSG1 presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 MXFP4 MoE decode nsg1. ds4_metal.m:41685 -runtime/metal DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_DECODE_STATIC_TRIP presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 MXFP4 MoE decode static trip. ds4_metal.m:41953 -runtime/metal DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_DECODE_SUM6_FULL_ROWS presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 MXFP4 MoE decode sum6 full rows. ds4_metal.m:41940 -runtime/metal DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_DECODE_TG_MULTIPLE presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 MXFP4 MoE decode tg multiple. ds4_metal.m:41902 -runtime/metal DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_MM_ID_DOWN_HALF_LUT presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 MXFP4 MoE MM ID down half lut. ds4_metal.m:45013 -runtime/metal DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_MM_ID_DOWN_TAIL_SIMDGROUP_CULL presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 MXFP4 MoE MM ID down tail simdgroup cull. ds4_metal.m:44996 -runtime/metal DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_MM_ID_MAP_SCATTER presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 MXFP4 MoE MM ID map scatter. ds4_metal.m:44963 -runtime/metal DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_MM_ID_PAIR_SWIGLU_COMPACT_TILE presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 MXFP4 MoE MM ID pair SwiGLU compact tile. ds4_metal.m:44947 -runtime/metal DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_MM_ID_PAIR_TAIL_SIMDGROUP_CULL presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 MXFP4 MoE MM ID pair tail simdgroup cull. ds4_metal.m:44984 -runtime/metal DS4_METAL_DISABLE_PRE_M5_PARALLEL_FULL_FFN presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 parallel full FFN. ds4.c:22533 -runtime/metal DS4_METAL_DISABLE_PRE_M5_Q2_DECODE_SPLIT2_32 presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 q2 decode split2 32. ds4.c:27703 -runtime/metal DS4_METAL_DISABLE_PRE_M5_QKV_NORM_KV_STORE_FUSE presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 QKV norm KV store fuse. ds4.c:23141 -runtime/metal DS4_METAL_DISABLE_PRE_M5_QKV_PAIR_COMPRESSOR_FUSE presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 QKV pair compressor fuse. ds4.c:22869 -runtime/metal DS4_METAL_DISABLE_PRE_M5_QKV_PAIR_QUAD_FUSE presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 QKV pair quad fuse. ds4.c:22865 -runtime/metal DS4_METAL_DISABLE_PRE_M5_ROUTER_SHARED_FUSE presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 router shared fuse. ds4.c:24582 -runtime/metal DS4_METAL_DISABLE_PRE_M5_ROUTER_SIMD_FINALIZE presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 router simd finalize. ds4_metal.m:35793 -runtime/metal DS4_METAL_DISABLE_PRE_M5_ROUTER_SIMD_WEIGHTS_FUSION presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 router simd weights fusion. ds4_metal.m:35802 -runtime/metal DS4_METAL_DISABLE_PRE_M5_ROUTER_TRANSFORM_FINALIZE_FUSION presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 router transform finalize fusion. ds4_metal.m:35806 -runtime/metal DS4_METAL_DISABLE_PRO_Q4_EXPERT_ADDRESS_AUTO presence rollback; unset: automatic/default path; any value including 0 disables Disables pro Q4 expert address auto. ds4_metal.m:19708 -runtime/metal DS4_METAL_DISABLE_PRO_Q4_EXPERT_TABLE_AUTO presence rollback; unset: automatic/default path; any value including 0 disables Disables pro Q4 expert table auto. ds4.c:20867 -runtime/metal DS4_METAL_DISABLE_PRO_Q4_EXPERT_TABLE_PRELOAD presence rollback; unset: automatic/default path; any value including 0 disables Disables pro Q4 expert table preload. ds4.c:58590 -runtime/metal DS4_METAL_DISABLE_Q4_ATTN_OUT_HC_FUSE presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 attn out HC fuse. ds4_metal.m:47590 -runtime/metal DS4_METAL_DISABLE_Q4_ATTN_OUT_TINY_BATCH value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Disables Q4 attn out tiny batch. ds4_metal.m:28372 -runtime/metal DS4_METAL_DISABLE_Q4_BATCH_EXPERT_TABLE presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 batch expert table. ds4_metal.m:44849 -runtime/metal DS4_METAL_DISABLE_Q4_DENSE_PAIR presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 dense pair. ds4_metal.m:21988 -runtime/metal DS4_METAL_DISABLE_Q4_EXACT_BOUNDARY presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 exact boundary. ds4_metal.m:42205 -runtime/metal DS4_METAL_DISABLE_Q4_EXACT_TENSOR_ID presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 exact tensor ID. ds4_metal.m:42185 -runtime/metal DS4_METAL_DISABLE_Q4_EXPERT_ADDRESS_TABLE presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 expert address table. ds4_metal.m:19709 -runtime/metal DS4_METAL_DISABLE_Q4_EXPERT_TABLE presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 expert table. ds4.c:20868 -runtime/metal DS4_METAL_DISABLE_Q4_GATHER_SLOTS presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 gather slots. ds4_metal.m:42287 -runtime/metal DS4_METAL_DISABLE_Q4_GROUP24_EXPERT_TABLE presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 group24 expert table. ds4_metal.m:42167 -runtime/metal DS4_METAL_DISABLE_Q4_GROUP6_EXPERT_TABLE presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 group6 expert table. ds4_metal.m:42133 -runtime/metal DS4_METAL_DISABLE_Q4_GROUP8_EXPERT_TABLE presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 group8 expert table. ds4_metal.m:42150 -runtime/metal DS4_METAL_DISABLE_Q4_GROUPED_BOUNDARY presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 grouped boundary. ds4_metal.m:42115 -runtime/metal DS4_METAL_DISABLE_Q4_GROUPED_EXPERTS presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 grouped experts. ds4_metal.m:42096 -runtime/metal DS4_METAL_DISABLE_Q4_MV_CLASSIC presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 MV classic. ds4_metal.m:21569 -runtime/metal DS4_METAL_DISABLE_Q4_QKV_COMPRESSOR_FUSE presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 QKV compressor fuse. ds4_metal.m:22081 -runtime/metal DS4_METAL_DISABLE_Q4_SELECTED_EXPERT_VIEWS presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 selected expert views. ds4.c:21064 -runtime/metal DS4_METAL_DISABLE_Q4_SSD_PREFILL_ATTN_OUT_EXACTN value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Disables Q4 SSD prefill attn out exactn. ds4_metal.m:28071 -runtime/metal DS4_METAL_DISABLE_Q4_SSD_SESSION_UNION nonempty boolean; unset/empty or exact 0: off; every other value: on Disables Q4 SSD session union. ds4.c:64970 -runtime/metal DS4_METAL_DISABLE_Q4_STREAM_OVERLAP nonempty boolean; unset/empty or exact 0: off; every other value: on Disables Q4 stream overlap. ds4.c:64892 -runtime/metal DS4_METAL_DISABLE_Q4_TABLE_BOUNDARY presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 table boundary. ds4_metal.m:42284 -runtime/metal DS4_METAL_DISABLE_Q8_DECODE_EXACT_VIEWS presence rollback; unset: automatic/default path; any value including 0 disables Disables Q8 decode exact views. ds4_metal.m:12849 -runtime/metal DS4_METAL_DISABLE_QKV_NORM_FUSION nonempty boolean; unset/empty or exact 0: off; every other value: on Disables QKV norm fusion. ds4.c:20262 -runtime/metal DS4_METAL_DISABLE_QKV_PAIR_PROJ nonempty boolean; unset/empty or exact 0: off; every other value: on Disables QKV pair proj. ds4.c:20267 -runtime/metal DS4_METAL_DISABLE_QUEUE_RESIDENCY_SET presence rollback; unset: automatic/default path; any value including 0 disables Disables queue residency set. ds4_metal.m:2123 -runtime/metal DS4_METAL_DISABLE_ROUTED_PAIR_SWIGLU_FUSION presence rollback; unset: automatic/default path; any value including 0 disables Disables routed pair SwiGLU fusion. ds4.c:18422 -runtime/metal DS4_METAL_DISABLE_ROUTER_SELECT_FUSION presence rollback; unset: automatic/default path; any value including 0 disables Disables router select fusion. ds4_metal.m:35782 -runtime/metal DS4_METAL_DISABLE_ROUTER_WEIGHTS_BATCH_FUSION presence rollback; unset: automatic/default path; any value including 0 disables Disables router weights batch fusion. ds4_metal.m:36034 -runtime/metal DS4_METAL_DISABLE_SHARED_DOWN_HC_FUSION nonempty boolean; unset/empty or exact 0: off; every other value: on Disables shared down HC fusion. ds4.c:20299 -runtime/metal DS4_METAL_DISABLE_SHARED_GATE_UP_SWIGLU_FUSION presence rollback; unset: automatic/default path; any value including 0 disables Disables shared gate up SwiGLU fusion. ds4.c:17394 -runtime/metal DS4_METAL_DISABLE_SHARED_KV_PAD presence rollback; unset: automatic/default path; any value including 0 disables Disables shared KV pad. ds4_metal.m:31481 -runtime/metal DS4_METAL_DISABLE_SHARED_ROPE_COEFF presence rollback; unset: automatic/default path; any value including 0 disables Disables shared RoPE coeff. ds4_metal.m:6340 -runtime/metal DS4_METAL_DISABLE_STREAMING_COLD_DECODE_PREFILL presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming cold decode prefill. ds4.c:31817 -runtime/metal DS4_METAL_DISABLE_STREAMING_COMPACT_ADDR presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming compact address. ds4_metal.m:14596 -runtime/metal DS4_METAL_DISABLE_STREAMING_DECODE_PREFILL presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming decode prefill. ds4.c:31766 -runtime/metal DS4_METAL_DISABLE_STREAMING_EXPERT_ADDR_TABLE presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming expert address table. ds4.c:18420 -runtime/metal DS4_METAL_DISABLE_STREAMING_EXPERT_COMBINED_BUFFER presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming expert combined buffer. ds4_metal.m:14021 -runtime/metal DS4_METAL_DISABLE_STREAMING_EXPERT_EARLY_LOAD presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming expert early load. ds4_metal.m:17230 -runtime/metal DS4_METAL_DISABLE_STREAMING_EXPERT_EVICT_DONTNEED presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming expert evict dontneed. ds4_metal.m:14393 -runtime/metal DS4_METAL_DISABLE_STREAMING_EXPERT_HIT_VALIDATOR presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming expert hit validator. ds4_metal.m:14632 -runtime/metal DS4_METAL_DISABLE_STREAMING_EXPERT_HOTLIST presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming expert hotlist. ds4.c:21117 -runtime/metal DS4_METAL_DISABLE_STREAMING_EXPERT_LIVE_INDEX value-aware boolean; default off; true disables and dominates ENABLE Disables dense live-entry index and uses authoritative cache matrix. ds4_metal.m:15441 -runtime/metal DS4_METAL_DISABLE_STREAMING_EXPERT_MASKED_ADDR presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming expert masked address. ds4_metal.m:14626 -runtime/metal DS4_METAL_DISABLE_STREAMING_EXPERT_READAHEAD presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming expert readahead. ds4_metal.m:13194 -runtime/metal DS4_METAL_DISABLE_STREAMING_EXPERT_SLABS presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming expert slabs. ds4_metal.m:14026 -runtime/metal DS4_METAL_DISABLE_STREAMING_EXPERT_TIMING_SUMMARY presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming expert timing summary. ds4_metal.m:13060 -runtime/metal DS4_METAL_DISABLE_STREAMING_FULL_EXPERT_ADDR_TABLE presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming full expert address table. ds4_metal.m:14714 -runtime/metal DS4_METAL_DISABLE_STREAMING_IQ2_CPU_ROUTER presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming IQ2 CPU router. ds4.c:20766 -runtime/metal DS4_METAL_DISABLE_STREAMING_LAYER_BATCH presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming layer batch. ds4.c:18066 -runtime/metal DS4_METAL_DISABLE_STREAMING_MADVISE_WILLNEED presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming madvise willneed. ds4.c:18039 -runtime/metal DS4_METAL_DISABLE_STREAMING_PREFILL_BATCH_SELECTED_ADDR presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming prefill batch selected address. ds4.c:18418 -runtime/metal DS4_METAL_DISABLE_STREAMING_PREFILL_LAYER_MADVISE presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming prefill layer madvise. ds4.c:18293 -runtime/metal DS4_METAL_DISABLE_STREAMING_PREFILL_LAYER_PAGEIN presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming prefill layer pagein. ds4.c:18261 -runtime/metal DS4_METAL_DISABLE_STREAMING_PREFILL_LAYER_PAGEIN_OVERLAP presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming prefill layer pagein overlap. ds4.c:19148 -runtime/metal DS4_METAL_DISABLE_STREAMING_PREFILL_LAYER_PREAD presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming prefill layer pread. ds4.c:18281 -runtime/metal DS4_METAL_DISABLE_STREAMING_PREFILL_LAYER_PREPARE presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming prefill layer prepare. ds4.c:18273 -runtime/metal DS4_METAL_DISABLE_STREAMING_PREFILL_LAYER_PREPARE_OVERLAP presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming prefill layer prepare overlap. ds4.c:19146 -runtime/metal DS4_METAL_DISABLE_STREAMING_PREFILL_LAYER_READAHEAD presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming prefill layer readahead. ds4.c:18271 -runtime/metal DS4_METAL_DISABLE_STREAMING_PREFILL_SELECTED_ASYNC_LOAD presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming prefill selected async load. ds4.c:46234 -runtime/metal DS4_METAL_DISABLE_STREAMING_PREFILL_SELECTED_MADVISE presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming prefill selected madvise. ds4.c:18251 -runtime/metal DS4_METAL_DISABLE_STREAMING_PREFILL_SELECTED_PAGEIN presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming prefill selected pagein. ds4.c:18241 -runtime/metal DS4_METAL_DISABLE_STREAMING_PREFILL_SELECTED_PROFILE presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming prefill selected profile. ds4.c:18735 -runtime/metal DS4_METAL_DISABLE_STREAMING_PREFILL_SELECTED_READAHEAD presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming prefill selected readahead. ds4.c:19787 -runtime/metal DS4_METAL_DISABLE_STREAMING_PREFILL_SELECTED_READAHEAD_SHARED presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming prefill selected readahead shared. ds4.c:19797 -runtime/metal DS4_METAL_DISABLE_STREAMING_READAHEAD presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming readahead. ds4.c:18032 -runtime/metal DS4_METAL_DISABLE_STREAMING_SELECTED_ASYNC_EARLY_COMMIT presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming selected async early commit. ds4.c:20846 -runtime/metal DS4_METAL_DISABLE_STREAMING_SELECTED_ASYNC_LOAD presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming selected async load. ds4.c:20830 -runtime/metal DS4_METAL_DISABLE_STREAMING_SELECTED_READAHEAD_SHARED_DELAY presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming selected readahead shared delay. ds4.c:21547 -runtime/metal DS4_METAL_DISABLE_STREAMING_SELECTED_SHARED_OVERLAP presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming selected shared overlap. ds4.c:20822 -runtime/metal DS4_METAL_DISABLE_STREAMING_STATIC_DECODE_MAP presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming static decode map. ds4.c:18044 -runtime/metal DS4_METAL_DISABLE_STREAMING_STATIC_MAP_STATE_CACHE presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming static map state cache. ds4.c:18057 -runtime/metal DS4_METAL_DISABLE_SUPPORT_Q8_DECODE_EXACT_VIEWS presence rollback; unset: automatic/default path; any value including 0 disables Disables support Q8 decode exact views. ds4_metal.m:12856 -runtime/metal DS4_METAL_DISABLE_TINY_PAIR_SWIGLU_FUSION presence rollback; unset: automatic/default path; any value including 0 disables Disables tiny pair SwiGLU fusion. ds4_metal.m:44886 -runtime/metal DS4_METAL_DISABLE_TOKEN_EMBED_EXACT_VIEW presence rollback; unset: automatic/default path; any value including 0 disables Disables token embed exact view. ds4_metal.m:11905 -runtime/metal DS4_METAL_DISABLE_ZERO_PREFIX_PREFILL_MASK_CACHE presence rollback; unset: automatic/default path; any value including 0 disables Disables zero prefix prefill mask cache. ds4_metal.m:2403 -runtime/metal DS4_METAL_DSPARK_ACCEPTANCE_ONLY_VERIFY nonempty boolean; unset/empty or exact 0: off; every other value: on Verifies only the draft rows still needed for acceptance after the base target logit. ds4.c:52309 -runtime/metal DS4_METAL_DSPARK_DEVICE_PROPOSER boolean true values enable; false values/unset disable; NO_DEVICE_PROPOSER presence dominates Keeps DSpark Q8 confidence/Markov proposal work on Metal and reads one compact result. ds4.c:34694 -runtime/metal DS4_METAL_DSPARK_EXACT2 nonempty boolean; unset/empty or exact 0: off; every other value: on Enables the resident single-GPU Metal exact-2 verifier. ds4.c:52100 -runtime/metal DS4_METAL_DSPARK_EXACTN nonempty boolean; unset/empty or exact 0: off; every other value: on Enables the single-GPU Metal exact-N verifier. ds4.c:52121 -runtime/metal DS4_METAL_DSPARK_EXACTN_BATCH_HEAD nonempty boolean; unset/empty or exact 0: off; every other value: on Batches the output head across exact-N verifier rows. ds4.c:37536 -runtime/metal DS4_METAL_DSPARK_EXACTN_UNION nonempty boolean; unset/empty or exact 0: off; every other value: on Loads the union of experts for exact-N verifier rows once. ds4.c:52142 -runtime/metal DS4_METAL_DSPARK_EXACT_ROWS_ASYNC_TAILS presence control; unset: off/default; any value including 0 enables Runs exact-row routed tails asynchronously after union routing. ds4.c:37742 -runtime/metal DS4_METAL_DSPARK_EXACT_ROWS_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for DSpark exact rows. ds4.c:37740 -runtime/metal DS4_METAL_DSPARK_HEADLESS_REPLAY unset/empty: enabled; exact 0 disables; every other nonempty value enables Skips output heads for accepted intermediate DSpark replay tokens. ds4.c:52289 -runtime/metal DS4_METAL_DSPARK_NO_DEVICE_PROPOSER presence rollback; unset: automatic/default path; any value including 0 disables Dominant presence-based rollback for the Metal DSpark device proposer. ds4.c:34696 -runtime/metal DS4_METAL_DSPARK_PIN_MAIN_PROJ nonempty value other than exact 0 enables; unset/empty/0 disables mlock-pins only DSpark stage-0 main_norm/main_proj in Metal SSD streaming. ds4.c:39253 -runtime/metal DS4_METAL_DSPARK_PROPOSER_BLOCK_MAX uint32; unset: automatic cache/verifier cap; 0 or invalid: native width; positive: clamped to DSpark/native maximum Caps rows proposed by single-device Metal DSpark. ds4.c:52253 -runtime/metal DS4_METAL_DSPARK_SAFE_EXPERT_COUNT exact 1 enables; unset or any other value disables Caps an explicit expert-count cache request to the safe Metal working-set budget for DSpark SSD streaming. ds4.c:4758 -runtime/metal DS4_METAL_DSV4_HC_SOURCE file path; unset/empty: use the in-tree Metal source file Overrides the DeepSeek hidden-context Metal kernel source file loaded at runtime. ds4_metal.m:4937 -runtime/metal DS4_METAL_DSV4_KV_SOURCE file path; unset/empty: use the in-tree Metal source file Overrides the DeepSeek KV Metal kernel source file loaded at runtime. ds4_metal.m:4939 -runtime/metal DS4_METAL_DSV4_MISC_SOURCE file path; unset/empty: use the in-tree Metal source file Overrides the DeepSeek miscellaneous Metal kernel source file loaded at runtime. ds4_metal.m:4941 -runtime/metal DS4_METAL_DSV4_ROPE_SOURCE file path; unset/empty: use the in-tree Metal source file Overrides the DeepSeek RoPE Metal kernel source file loaded at runtime. ds4_metal.m:4940 -runtime/metal DS4_METAL_DUMP_PREFILL_LOGITS file path; unset/empty: no dump Writes final GPU prefill logits as f32 binary. ds4.c:51157 -runtime/metal DS4_METAL_ENABLE_BATCH_HC_NORM_FUSION legacy value-aware alias; default enabled; exact 0 disables; unset/empty/every other value enables unless DISABLE is active Legacy control for the now-default batched HC norm fusion. ds4.c:20289 -runtime/metal DS4_METAL_ENABLE_COMPRESSOR_EXACT_POOL_RATIO4 presence opt-in; unset: off/automatic; any value including 0 enables Enables compressor exact pool ratio4. ds4_metal.m:26383 -runtime/metal DS4_METAL_ENABLE_COMPRESSOR_PAIR_STATE_STORE presence opt-in; unset: off/automatic; any value including 0 enables Enables compressor pair state store. ds4_metal.m:23241 -runtime/metal DS4_METAL_ENABLE_COMPRESSOR_QUAD_STORE presence opt-in; unset: off/automatic; any value including 0 enables Enables compressor quad store. ds4.c:23354 -runtime/metal DS4_METAL_ENABLE_DSPARK_CAPTURE_FUSED_LAST nonempty boolean; unset/empty or exact 0: off; every other value: on Enables DSpark capture fused last. ds4.c:28186 -runtime/metal DS4_METAL_ENABLE_GATHERED_KV_STAGE presence opt-in; unset: off/automatic; any value including 0 enables Enables gathered KV stage. ds4_metal.m:29651 -runtime/metal DS4_METAL_ENABLE_GLM_STREAMING_SELECTED_ASYNC_LOAD presence opt-in; unset: off/automatic; any value including 0 enables Enables GLM streaming selected async load. ds4.c:44145 -runtime/metal DS4_METAL_ENABLE_HC_NORM_MIX_FUSE presence opt-in; unset: off/automatic; any value including 0 enables Enables HC norm mix fuse. ds4.c:22697 -runtime/metal DS4_METAL_ENABLE_HC_PRODUCER_PRE_NORM_FUSE presence opt-in; unset: off/automatic; any value including 0 enables Enables HC producer pre norm fuse. ds4_metal.m:46518 -runtime/metal DS4_METAL_ENABLE_IQ2_SELECTED_ASYNC_EARLY_COMMIT presence opt-in; unset: off/automatic; any value including 0 enables Enables IQ2 selected async early commit. ds4.c:20845 -runtime/metal DS4_METAL_ENABLE_IQ2_XXS_SSD_PREFILL_MM value-aware boolean; default automatic/on for eligible shape; explicit 0 turns request off unless REQUIRE=1 Overrides automatic IQ2_XXS/Q2_K grouped address-MM selection for SSD prefill. ds4_metal.m:44746 -runtime/metal DS4_METAL_ENABLE_PRO_Q4_EXPERT_ADDRESS_AUTO presence opt-in; unset: off/automatic; any value including 0 enables Enables pro Q4 expert address auto. ds4.c:20809 -runtime/metal DS4_METAL_ENABLE_PRO_Q4_EXPERT_TABLE_AUTO presence opt-in; unset: off/automatic; any value including 0 enables Enables pro Q4 expert table auto. ds4.c:20808 -runtime/metal DS4_METAL_ENABLE_PRO_Q4_SELECTED_EXPERT_VIEWS presence opt-in; unset: off/automatic; any value including 0 enables Enables pro Q4 selected expert views. ds4.c:20805 -runtime/metal DS4_METAL_ENABLE_Q4_ATTN_OUT_TINY_BATCH value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Enables Q4 attn out tiny batch. ds4_metal.m:28381 -runtime/metal DS4_METAL_ENABLE_Q4_BATCH_EXPERT_TABLE presence opt-in; unset: off/automatic; any value including 0 enables Enables Q4 batch expert table. ds4_metal.m:44835 -runtime/metal DS4_METAL_ENABLE_Q4_EXACT_TENSOR_ID presence opt-in; unset: off/automatic; any value including 0 enables Enables Q4 exact tensor ID. ds4_metal.m:42184 -runtime/metal DS4_METAL_ENABLE_Q4_EXPERT_ADDRESS_TABLE presence opt-in; unset: off/automatic; any value including 0 enables Enables Q4 expert address table. ds4.c:20807 -runtime/metal DS4_METAL_ENABLE_Q4_EXPERT_TABLE presence opt-in; unset: off/automatic; any value including 0 enables Enables Q4 expert table. ds4.c:20806 -runtime/metal DS4_METAL_ENABLE_Q4_GATHER_SLOTS presence opt-in; unset: off/automatic; any value including 0 enables Enables Q4 gather slots. ds4_metal.m:42286 -runtime/metal DS4_METAL_ENABLE_Q4_GROUP24_EXPERT_TABLE presence opt-in; unset: off/automatic; any value including 0 enables Enables Q4 group24 expert table. ds4_metal.m:42166 -runtime/metal DS4_METAL_ENABLE_Q4_GROUP6_EXPERT_TABLE presence opt-in; unset: off/automatic; any value including 0 enables Enables Q4 group6 expert table. ds4_metal.m:42132 -runtime/metal DS4_METAL_ENABLE_Q4_GROUP8_EXPERT_TABLE presence opt-in; unset: off/automatic; any value including 0 enables Enables Q4 group8 expert table. ds4_metal.m:42149 -runtime/metal DS4_METAL_ENABLE_Q4_GROUPED_EXPERTS presence opt-in; unset: off/automatic; any value including 0 enables Enables Q4 grouped experts. ds4_metal.m:42095 -runtime/metal DS4_METAL_ENABLE_Q4_QKV_COMPRESSOR_FUSE presence opt-in; unset: off/automatic; any value including 0 enables Enables Q4 QKV compressor fuse. ds4.c:22967 -runtime/metal DS4_METAL_ENABLE_Q4_SELECTED_EXPERT_VIEWS presence opt-in; unset: off/automatic; any value including 0 enables Enables Q4 selected expert views. ds4.c:20804 -runtime/metal DS4_METAL_ENABLE_Q4_SSD_PREFILL_ATTN_OUT_EXACTN value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Enables Q4 SSD prefill attn out exactn. ds4_metal.m:28073 -runtime/metal DS4_METAL_ENABLE_Q4_SSD_SESSION_UNION nonempty boolean; unset/empty or exact 0: off; every other value: on Enables Q4 SSD session union. ds4.c:64981 -runtime/metal DS4_METAL_ENABLE_Q4_STREAM_OVERLAP nonempty boolean; unset/empty or exact 0: off; every other value: on Enables Q4 stream overlap. ds4.c:64890 -runtime/metal DS4_METAL_ENABLE_Q8_DECODE_EXACT_VIEWS presence opt-in; unset: off/automatic; any value including 0 enables Enables Q8 decode exact views. ds4_metal.m:12863 -runtime/metal DS4_METAL_ENABLE_Q8_QKV_COMPRESSOR_FUSE nonempty boolean; unset/empty/exact 0: no streamed/union opt-in; other values enable; eligible resident full-decode remains automatic Extends the automatic resident Q8 QKV/compressor compound fusion to SSD streaming or exact-N union scope. ds4.c:22854 -runtime/metal DS4_METAL_ENABLE_STREAMING_COMPACT_ADDR presence opt-in; unset: off/automatic; any value including 0 enables Enables streaming compact address. ds4_metal.m:14595 -runtime/metal DS4_METAL_ENABLE_STREAMING_EXPERT_ADDR_TABLE presence opt-in; unset: off/automatic; any value including 0 enables Enables streaming expert address table. ds4_metal.m:14602 -runtime/metal DS4_METAL_ENABLE_STREAMING_EXPERT_EVICT_DONTNEED presence opt-in; unset: off/automatic; any value including 0 enables Enables streaming expert evict dontneed. ds4_metal.m:14392 -runtime/metal DS4_METAL_ENABLE_STREAMING_EXPERT_HIT_VALIDATOR presence opt-in; unset: off/automatic; any value including 0 enables Enables streaming expert hit validator. ds4_metal.m:14603 -runtime/metal DS4_METAL_ENABLE_STREAMING_EXPERT_LIVE_INDEX value-aware boolean; default automatic/on for validated IQ2 cache shape; explicit 0 disables Overrides automatic dense live-entry index selection. ds4_metal.m:15440 -runtime/metal DS4_METAL_ENABLE_STREAMING_EXPERT_MASKED_ADDR presence opt-in; unset: off/automatic; any value including 0 enables Enables streaming expert masked address. ds4_metal.m:14604 -runtime/metal DS4_METAL_ENABLE_STREAMING_FULL_EXPERT_ADDR_TABLE presence opt-in; unset: off/automatic; any value including 0 enables Enables streaming full expert address table. ds4_metal.m:14713 -runtime/metal DS4_METAL_ENABLE_STREAMING_IQ2_CPU_ROUTER presence opt-in; unset: off/automatic; any value including 0 enables Enables streaming IQ2 CPU router. ds4.c:20765 -runtime/metal DS4_METAL_ENABLE_STREAMING_MADVISE_WILLNEED presence opt-in; unset: off/automatic; any value including 0 enables Enables streaming madvise willneed. ds4.c:18037 -runtime/metal DS4_METAL_ENABLE_STREAMING_PREFILL_BATCH_SELECTED_ADDR presence opt-in; unset: off/automatic; any value including 0 enables Enables streaming prefill batch selected address. ds4_metal.m:14607 -runtime/metal DS4_METAL_ENABLE_STREAMING_PREFILL_CACHE_SEED presence opt-in; unset: off/automatic; any value including 0 enables Enables streaming prefill cache seed. ds4.c:21086 -runtime/metal DS4_METAL_ENABLE_STREAMING_PREFILL_EXPERT_READAHEAD value-aware boolean; for batches <32 readahead is automatic; for batches >=32 unset/false disables and true enables; global READHEAD rollback and F_NOCACHE still dominate Restores F_RDADVISE immediately before parallel pread for large SSD-prefill batches. ds4_metal.m:13210 -runtime/metal DS4_METAL_ENABLE_STREAMING_PREFILL_LAYER_PAGEIN presence opt-in; unset: off/automatic; any value including 0 enables Enables streaming prefill layer pagein. ds4.c:18259 -runtime/metal DS4_METAL_ENABLE_STREAMING_PREFILL_LAYER_READAHEAD presence opt-in; unset: off/automatic; any value including 0 enables Enables streaming prefill layer readahead. ds4.c:18269 -runtime/metal DS4_METAL_ENABLE_STREAMING_PREFILL_SELECTED_MADVISE presence opt-in; unset: off/automatic; any value including 0 enables Enables streaming prefill selected madvise. ds4.c:18249 -runtime/metal DS4_METAL_ENABLE_STREAMING_PREFILL_SELECTED_PAGEIN presence opt-in; unset: off/automatic; any value including 0 enables Enables streaming prefill selected pagein. ds4.c:18239 -runtime/metal DS4_METAL_ENABLE_STREAMING_PREFILL_SELECTED_READAHEAD presence opt-in; unset: off/automatic; any value including 0 enables Enables streaming prefill selected readahead. ds4.c:19783 -runtime/metal DS4_METAL_ENABLE_STREAMING_PREFILL_SELECTED_READAHEAD_SHARED presence opt-in; unset: off/automatic; any value including 0 enables Enables streaming prefill selected readahead shared. ds4.c:19785 -runtime/metal DS4_METAL_ENABLE_STREAMING_READAHEAD presence opt-in; unset: off/automatic; any value including 0 enables Enables streaming readahead. ds4.c:18030 -runtime/metal DS4_METAL_ENABLE_STREAMING_SELECTED_READAHEAD_SHARED_DELAY presence opt-in; unset: off/automatic; any value including 0 enables Enables streaming selected readahead shared delay. ds4.c:21546 -runtime/metal DS4_METAL_ENABLE_STREAMING_STATIC_DECODE_MAP presence opt-in; unset: off/automatic; any value including 0 enables Enables streaming static decode map. ds4.c:18049 -runtime/metal DS4_METAL_ENABLE_TOKEN_EMBED_EXACT_VIEW presence opt-in; unset: off/automatic; any value including 0 enables Enables token embed exact view. ds4_metal.m:12217 -runtime/metal DS4_METAL_EXACT_VIEW_CACHE_GIB unsigned GiB; default 64; 0 disables size-triggered eviction; MIB overrides it Sets the cached exact-model-view eviction threshold. ds4_metal.m:1332 -runtime/metal DS4_METAL_EXACT_VIEW_CACHE_MIB unsigned MiB; unset: inherit GIB/default; 0 disables size-triggered eviction; overrides GIB Sets the cached exact-model-view eviction threshold with MiB precision. ds4_metal.m:1341 -runtime/metal DS4_METAL_EXACT_VIEW_CACHE_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for exact view cache. ds4_metal.m:1376 -runtime/metal DS4_METAL_FLASH_ATTN_SOURCE file path; unset/empty: use the in-tree Metal source file Overrides the FlashAttention Metal kernel source file loaded at runtime. ds4_metal.m:4934 -runtime/metal DS4_METAL_FLASH_ATTN_STAGE_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for flash attn stage. ds4.c:64962 -runtime/metal DS4_METAL_FLASH_ATTN_STAGE_PROFILE_FILTER substring; unset/empty: all profiled modes/stages Filters FlashAttention stage-profile output by mode or stage substring. ds4_metal.m:11177 -runtime/metal DS4_METAL_GET_ROWS_SOURCE file path; unset/empty: use the in-tree Metal source file Overrides the get-rows Metal kernel source file loaded at runtime. ds4_metal.m:4945 -runtime/metal DS4_METAL_GLM_DISABLE_STREAMING_EXPERT_CACHE presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming expert cache for GLM. ds4_metal.m:39633 -runtime/metal DS4_METAL_GLM_DISABLE_STREAMING_GROUPED_ADDR_PREFILL presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming grouped address prefill for GLM. ds4_metal.m:41047 -runtime/metal DS4_METAL_GLM_DISABLE_STREAMING_SEED_BEFORE_PREFILL presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming seed before prefill for GLM. ds4.c:50884 -runtime/metal DS4_METAL_GLM_DISABLE_STREAMING_TOKEN_PREFILL presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming token prefill for GLM. ds4.c:49514 -runtime/metal DS4_METAL_GLM_MOE_ONE_STAGE_PROFILE unset: off; 1/true/yes/on/all enables all layers; accepts layer lists/ranges; 0/false/no/off disables Prints timing/profile diagnostics for GLM MoE one stage. ds4_metal.m:39930 -runtime/metal DS4_METAL_GLM_MOE_ONE_STAGE_PROFILE_LAYER layer index/list/ranges or all; unset: all layers selected by profiler Restricts GLM one-stage MoE profiling to selected layers. ds4_metal.m:39931 -runtime/metal DS4_METAL_GLM_MOE_STAGE_PROFILE_FILTER substring; unset/empty: all profiled stages Filters GLM MoE stage-profile output. ds4_metal.m:39934 -runtime/metal DS4_METAL_GLM_QKLOW_DEBUG presence diagnostic; unset: off; any value including 0 enables Enables debug diagnostics for GLM qklow. ds4_metal.m:37834 -runtime/metal DS4_METAL_GLM_STREAMING_ASYNC_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for GLM streaming async. ds4.c:44172 -runtime/metal DS4_METAL_GLM_STREAMING_DECODE_FULL_LAYER_MAP presence control; unset: off/default; any value including 0 enables Maps complete GLM layers during SSD-streaming decode instead of decode-only spans. ds4.c:42426 -runtime/metal DS4_METAL_GLM_STREAMING_DECODE_SYNC_EACH_LAYER boolean text; Metal runtime always synchronizes and does not consult it; legacy fallback name read only in ROCm builds Controls per-layer GLM streaming decode synchronization only as a legacy ROCm fallback alias. ds4.c:49590 -runtime/metal DS4_METAL_GLM_STREAMING_PREFILL_FULL_LAYER presence control; unset: off/default; any value including 0 enables Forces full-layer GLM SSD prefill regardless of the token crossover. ds4_metal.m:14708 -runtime/metal DS4_METAL_GLM_STREAMING_PREFILL_FULL_LAYER_MIN_TOKENS positive uint32; default 64 on Metal, 1024 when used as ROCm fallback; 0/invalid restores default Sets the token crossover for GLM full-layer SSD prefill. ds4.c:42503 -runtime/metal DS4_METAL_GLM_STREAMING_PREFILL_SYNC_EACH_LAYER boolean text; Metal runtime always synchronizes and does not consult it; legacy fallback name read only in ROCm builds Controls per-layer GLM streaming prefill synchronization only as a legacy ROCm fallback alias. ds4.c:42206 -runtime/metal DS4_METAL_GLM_STREAMING_TOKEN_PREFILL_MAX uint32; default 64 on Metal, 0 when used as ROCm fallback; 0 disables; invalid restores default Sets largest GLM SSD prefill handled token-major by the decode graph. ds4.c:49493 -runtime/metal DS4_METAL_GLU_SOURCE file path; unset/empty: use the in-tree Metal source file Overrides the GLU Metal kernel source file loaded at runtime. ds4_metal.m:4949 -runtime/metal DS4_METAL_GPU_BATCH_EMBED_MIN uint32 token threshold; default 512; invalid restores default Sets the batch size at which prompt embedding moves from CPU upload to Metal kernels. ds4.c:28694 -runtime/metal DS4_METAL_GPU_BUSY_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for GPU busy. ds4_metal.m:1292 -runtime/metal DS4_METAL_GRAPH_DUMP_LAYER unsigned layer index or all; unset: every layer Restricts graph tensor dumps to one layer. ds4.c:16690 -runtime/metal DS4_METAL_GRAPH_DUMP_LOGITS file path; unset/empty: no graph-logit dump Writes Metal graph-test logits as f32 binary. ds4.c:38890 -runtime/metal DS4_METAL_GRAPH_DUMP_NAME substring; unset/empty: every tensor name Restricts graph tensor dumps by tensor-name substring. ds4.c:16686 -runtime/metal DS4_METAL_GRAPH_DUMP_POS unsigned token position; unset: every position Restricts graph tensor dumps to one token position. ds4.c:16697 -runtime/metal DS4_METAL_GRAPH_DUMP_PREFIX path/prefix; unset/empty: tensor dumping disabled Enables graph tensor dumps and supplies the filename prefix. ds4.c:64958 -runtime/metal DS4_METAL_GRAPH_DUMP_TRACE presence diagnostic; unset: off; any value including 0 enables Emits trace diagnostics for graph dump. ds4.c:16727 -runtime/metal DS4_METAL_GRAPH_OUTPUT_ROW zero-based row smaller than current batch; default final row; invalid restores final row Chooses which prefill output row is projected to logits. ds4.c:35656 -runtime/metal DS4_METAL_GRAPH_PREFILL_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for graph prefill. ds4.c:35333 -runtime/metal DS4_METAL_GRAPH_PREFILL_SPLIT_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for graph prefill split. ds4.c:66014 -runtime/metal DS4_METAL_GRAPH_PROMPT_TOKENS integer 1..prompt length; default full prompt Limits prompt length used by the Metal graph parity test. ds4.c:38833 -runtime/metal DS4_METAL_GRAPH_RAW_CAP positive rows; default from SWA window+prefill; clamped to [raw_window,min(ctx,8192)] Overrides raw sliding-window KV ring capacity. ds4.c:38200 -runtime/metal DS4_METAL_GRAPH_TEACHER_FORCE presence control; unset: off/default; any value including 0 enables Feeds CPU reference state back into the first-token graph trace at each layer. ds4.c:27569 -runtime/metal DS4_METAL_GRAPH_TOKEN_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for graph token. ds4.c:32192 -runtime/metal DS4_METAL_GRAPH_TOKEN_SECOND_SPLIT_LAYERS integer 0..layer count; default 0 plus eligible automatic pre-M5 schedules; explicit value wins Overrides second command-buffer split layer for token decode. ds4.c:27856 -runtime/metal DS4_METAL_GRAPH_TOKEN_SPLIT_LAYERS integer 0..layer count; default 4 on Apple and 0 elsewhere, with eligible pre-M5 adaptive override Overrides first command-buffer split layer for token decode. ds4.c:27742 -runtime/metal DS4_METAL_GRAPH_TRACE_CACHE presence control; unset: off/default; any value including 0 enables Prints raw KV cache parity diagnostics in the graph prompt test. ds4.c:38902 -runtime/metal DS4_METAL_GRAPH_TRACE_COMP presence control; unset: off/default; any value including 0 enables Prints compressed-cache parity diagnostics in the graph prompt test. ds4.c:38903 -runtime/metal DS4_METAL_GRAPH_TRACE_LAYERS presence control; unset: off/default; any value including 0 enables Enables per-layer first-token CPU/GPU graph tracing. ds4.c:27566 -runtime/metal DS4_METAL_GRAPH_TRACE_STAGE_LAYER signed layer index; unset gives -1/no stage-layer selection Selects the layer used by first-token stage tracing. ds4.c:27570 -runtime/metal DS4_METAL_HC_NORM_FUSION_CHECK nonempty boolean; unset/empty or exact 0: off; every other value: on Compares fused HC normalization against the reference result. ds4.c:20348 -runtime/metal DS4_METAL_HC_NORM_FUSION_CHECK_TOL positive finite float; default 2e-4; invalid/nonpositive restores default Sets the numerical tolerance for the HC norm-fusion oracle. ds4.c:20357 -runtime/metal DS4_METAL_HC_STABLE boolean empty/1/true/yes/on vs 0/false/no/off; default on Compiles stable hidden-context drift arithmetic into the Metal library. ds4_metal.m:7046 -runtime/metal DS4_METAL_INDEXER_STAGE_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for indexer stage. ds4.c:17396 -runtime/metal DS4_METAL_IQ2_XXS_SSD_PREFILL_MM_STATS value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Collects/prints statistics for IQ2 XXS SSD prefill MM. ds4_metal.m:6979 -runtime/metal DS4_METAL_KV_RAW_F32 boolean empty/1/true/yes/on vs 0/false/no/off; default off Compiles raw KV storage as F32 for drift diagnosis. ds4_metal.m:7048 -runtime/metal DS4_METAL_LAYER_STAGE_PROFILE unset: off; 1/true/yes/on/all enables all layers; a layer index selects one; 0/false/no/off disables Prints timing/profile diagnostics for layer stage. ds4.c:64960 -runtime/metal DS4_METAL_LAYER_STAGE_PROFILE_LAYER single unsigned layer index; unset/empty: all layers enabled by the parent profile; invalid matches no layer Restricts the corresponding shared graph stage profiler to one layer. ds4.c:28936 -runtime/metal DS4_METAL_MATH_SAFE boolean empty/1/true/yes/on vs 0/false/no/off; default off Compiles Metal shaders with strict/safe IEEE math instead of fast math. ds4_metal.m:7050 -runtime/metal DS4_METAL_MEMORY_REPORT presence control; unset: off/default; any value including 0 enables Prints Metal allocation/cache/residency memory reports. ds4.c:38857 -runtime/metal DS4_METAL_MODEL_UNTRACKED presence control; unset: off/default; any value including 0 enables Creates mapped model buffers with untracked Metal hazard tracking. ds4_metal.m:1546 -runtime/metal DS4_METAL_MODEL_VIEW_MAX_GIB positive integer GiB; default device maximum (128-GiB cap for already-split span maps); cannot exceed device maximum Caps each no-copy mapped Metal model view. ds4_metal.m:2215 -runtime/metal DS4_METAL_MODEL_WARMUP_STRIDE_KB integer 1..1048576 KiB, at least one page; unset inherits MB/default; overrides STRIDE_MB Sets the model-view warmup touch stride with KiB precision. ds4_metal.m:3056 -runtime/metal DS4_METAL_MODEL_WARMUP_STRIDE_MB integer 1..1024 MiB; default 1 MiB; STRIDE_KB overrides Sets the model-view warmup touch stride. ds4_metal.m:3048 -runtime/metal DS4_METAL_MOE_MM_ID_USE_RESOURCES presence control; unset: off/default; any value including 0 enables Declares MM-ID MoE resource usage explicitly on the command encoder. ds4_metal.m:35134 -runtime/metal DS4_METAL_MOE_ONE_STAGE_PROFILE unset: off; 1/true/yes/on/all enables all layers; accepts layer lists/ranges; 0/false/no/off disables Prints timing/profile diagnostics for MoE one stage. ds4.c:22541 -runtime/metal DS4_METAL_MOE_ONE_STAGE_PROFILE_LAYER layer index/list/ranges or all; unset: all profiler-selected layers Restricts one-stage MoE profiling to selected layers. ds4_metal.m:43402 -runtime/metal DS4_METAL_MOE_SOURCE file path; unset/empty: use the in-tree Metal source file Overrides the MoE Metal kernel source file loaded at runtime. ds4_metal.m:4936 -runtime/metal DS4_METAL_MOE_STAGE_PROFILE unset: off; 1/true/yes/on/all enables all layers; accepts layer lists/ranges; 0/false/no/off disables Prints timing/profile diagnostics for MoE stage. ds4.c:64964 -runtime/metal DS4_METAL_MOE_STAGE_PROFILE_FILTER substring; unset/empty: all profiled stages Filters MoE stage-profile output. ds4_metal.m:43404 -runtime/metal DS4_METAL_MOE_STAGE_PROFILE_LAYER layer index/list/ranges or all; unset: all profiler-selected layers Restricts batched MoE stage profiling to selected layers. ds4_metal.m:45312 -runtime/metal DS4_METAL_MOE_WRITE_CLAMPED_ACT presence control; unset: off/default; any value including 0 enables Makes routed MoE write the clamped activation diagnostic. ds4.c:18421 -runtime/metal DS4_METAL_NORM_RSQRT_DISABLE boolean empty/1/true/yes/on vs 0/false/no/off; default on Compiles unified normalization-rsqrt arithmetic into the Metal library. ds4_metal.m:7047 -runtime/metal DS4_METAL_NORM_SOURCE file path; unset/empty: use the in-tree Metal source file Overrides the normalization Metal kernel source file loaded at runtime. ds4_metal.m:4950 -runtime/metal DS4_METAL_NO_MODEL_WARMUP presence rollback; unset: automatic/default path; any value including 0 disables Disables model warmup. ds4_metal.m:2308 -runtime/metal DS4_METAL_NO_PREFILL_KERNEL_WARMUP presence rollback; unset: automatic/default path; any value including 0 disables Disables prefill kernel warmup. ds4.c:28792 -runtime/metal DS4_METAL_NO_RESIDENCY presence rollback; unset: automatic/default path; any value including 0 disables Disables residency. ds4_metal.m:2091 -runtime/metal DS4_METAL_OUTPUT_STAGE_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for output stage. ds4.c:17397 -runtime/metal DS4_METAL_PREFILL_CHUNK positive token count used only when CLI chunk is absent; default full prompt, or 4096 for long non-PRO and 8192 for long PRO prompts; <=0 keeps automatic/full prompt Provides the historical environment fallback for prefill chunk size. ds4.c:12629 -runtime/metal DS4_METAL_PRO_Q4_CPU_ROUTER nonempty boolean; unset/empty or exact 0: off; every other value: on Uses the CPU router for PRO Q4 selected-expert decode. ds4.c:20761 -runtime/metal DS4_METAL_PRO_Q4_CPU_ROUTER_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for pro Q4 CPU router. ds4.c:21471 -runtime/metal DS4_METAL_Q4_ADDR_USE_RESOURCES presence control; unset: off/default; any value including 0 enables Declares Q4 address-table resources explicitly on encoders. ds4_metal.m:20257 -runtime/metal DS4_METAL_Q4_EXPERT_GROUP_SIZE positive uint32; default 32; clamped to total expert count Sets experts processed per grouped Q4 dispatch. ds4_metal.m:34054 -runtime/metal DS4_METAL_Q4_EXPERT_TABLE_GROUP_SIZE integer 2..total experts; default/invalid 1 (ungrouped) Sets grouped exact-view width while building Q4 expert tables. ds4_metal.m:19602 -runtime/metal DS4_METAL_Q4_EXPERT_TABLE_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for Q4 expert table. ds4_metal.m:20101 -runtime/metal DS4_METAL_Q4_GROUP24_BASE_VIEWS presence control; unset: off/default; any value including 0 enables Uses broad base model views for Q4 group-24 instead of exact views. ds4_metal.m:42171 -runtime/metal DS4_METAL_Q4_GROUP24_EXACT_VIEWS presence control; unset: off/default; any value including 0 enables Uses exact mapped views for Q4 group-24 experts. ds4_metal.m:42170 -runtime/metal DS4_METAL_Q4_GROUPED_CACHE_VIEWS presence control; unset: off/default; any value including 0 enables Caches exact Q4 grouped expert views. ds4_metal.m:42117 -runtime/metal DS4_METAL_Q4_PRO_MAP_GROUPS positive divisor of 384 in 1..384; default/invalid 1 Splits each 384-expert PRO Q4 tensor into this many mapped views. ds4.c:6153 -runtime/metal DS4_METAL_Q4_SELECTED_EXACT_VIEWS presence control; unset: off/default; any value including 0 enables Forces exact/cached views for selected Q4 experts instead of base views. ds4_metal.m:42494 -runtime/metal DS4_METAL_Q4_SELECTED_OVERLAP_SHARED nonempty boolean; unset/empty or exact 0: off; every other value: on Overlaps selected Q4 expert preparation with the shared expert. ds4.c:20789 -runtime/metal DS4_METAL_Q4_SELECTED_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for Q4 selected. ds4.c:37754 -runtime/metal DS4_METAL_Q4_SELECTED_PROFILE_LAYER single nonnegative layer index; unset: every layer Restricts legacy Q4 selected-expert profiling to one layer. ds4_metal.m:42475 -runtime/metal DS4_METAL_Q4_SELECTED_SHARED_EVENT presence control; unset: off/default; any value including 0 enables Coordinates selected Q4 work with a shared Metal event. ds4_metal.m:42490 -runtime/metal DS4_METAL_Q4_SELECTED_TRANSIENT_VIEWS presence control; unset: off/default; any value including 0 enables Uses transient exact views for selected Q4 experts. ds4_metal.m:42498 -runtime/metal DS4_METAL_Q4_SELECTED_USE_BASE_VIEWS presence control; unset: off/default; any value including 0 enables Uses broad base model views for selected Q4 experts. ds4_metal.m:42493 -runtime/metal DS4_METAL_Q4_TABLE_BIND_ANCHORS presence control; unset: off/default; any value including 0 enables Binds anchor buffers alongside the Q4 expert address table. ds4_metal.m:19714 -runtime/metal DS4_METAL_Q4_TABLE_MODEL_RESIDENCY_SET presence control; unset: off/default; any value including 0 enables Adds Q4 expert table allocations to the model residency set. ds4_metal.m:19655 -runtime/metal DS4_METAL_Q4_TABLE_PER_TENSOR_RESIDENCY_SET presence control; unset: off/default; any value including 0 enables Builds separate residency sets per Q4 expert tensor. ds4_metal.m:19758 -runtime/metal DS4_METAL_Q4_TABLE_QUEUE_RESIDENCY_SET presence control; unset: off/default; any value including 0 enables Attaches Q4 expert table residency sets to command queues. ds4_metal.m:19613 -runtime/metal DS4_METAL_Q4_TABLE_RESIDENCY_SET presence control; unset: off/default; any value including 0 enables Enables Q4 expert table residency-set handling. ds4_metal.m:19757 -runtime/metal DS4_METAL_Q4_TABLE_USE_RESOURCES presence control; unset: off/default; any value including 0 enables Declares Q4 table resources explicitly on encoders. ds4_metal.m:20256 -runtime/metal DS4_METAL_Q8_DECODE_EXACT_VIEW_MAX_MIB integer 1..4096 MiB; default 1024; above max clamps, below min/invalid restores default Caps weight ranges eligible for Q8 exact model views. ds4_metal.m:12874 -runtime/metal DS4_METAL_Q8_MV_EXT_MAX_TOKENS integer 2..128; default 16; above max clamps, below min/invalid restores default Sets largest batch handled by extended Q8 matvec. ds4_metal.m:21124 -runtime/metal DS4_METAL_Q8_MV_NSG integer 1..8 simdgroups; default 4, or 2 with TP world=2; above max clamps, below min/invalid restores default Overrides simdgroups per Q8 matvec threadgroup. ds4.c:22557 -runtime/metal DS4_METAL_Q8_PREFILL_PROFILE value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Prints timing/profile diagnostics for Q8 prefill. ds4_metal.m:21278 -runtime/metal DS4_METAL_Q8_PREFILL_PROFILE_FILTER substring matched against generated operation label; unset/empty: all eligible calls Filters Q8 prefill profiling. ds4_metal.m:21293 -runtime/metal DS4_METAL_Q_STAGE_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for q stage. ds4.c:29097 -runtime/metal DS4_METAL_REPEAT_SOURCE file path; unset/empty: use the in-tree Metal source file Overrides the repeat Metal kernel source file loaded at runtime. ds4_metal.m:4948 -runtime/metal DS4_METAL_REQUIRE_COMPRESSOR_EXACT_POOL_RATIO4 presence strict check; unset: fallback allowed; any value including 0 requires the path Requires compressor exact pool ratio4 and makes eligible fallback fail closed. ds4_metal.m:26385 -runtime/metal DS4_METAL_REQUIRE_EXACT_ROWS_PERSISTENT_CACHE nonempty boolean; unset/empty or exact 0: off; every other value: on Requires exact rows persistent cache and makes eligible fallback fail closed. ds4_metal.m:13000 -runtime/metal DS4_METAL_REQUIRE_GATHERED_KV_STAGE presence strict check; unset: fallback allowed; any value including 0 requires the path Requires gathered KV stage and makes eligible fallback fail closed. ds4_metal.m:29656 -runtime/metal DS4_METAL_REQUIRE_IQ2_XXS_SSD_PREFILL_MM value-aware boolean; default implicit fail-closed only with complete selected-address domain; explicit 1 is strict, 0 permits fallback Makes eligible IQ2_XXS/Q2_K grouped SSD-prefill MM fail closed. ds4_metal.m:44748 -runtime/metal DS4_METAL_REQUIRE_M1_IQ2_MID_ONLY presence strict check; unset: fallback allowed; any value including 0 requires the path Requires M1 IQ2 mid only and makes eligible fallback fail closed. ds4_metal.m:42040 -runtime/metal DS4_METAL_REQUIRE_OUTPUT_HC_WEIGHTS4 presence strict check; unset: fallback allowed; any value including 0 requires the path Requires output HC weights4 and makes eligible fallback fail closed. ds4_metal.m:46755 -runtime/metal DS4_METAL_REQUIRE_Q4_ATTN_OUT_TINY_BATCH value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Requires Q4 attn out tiny batch and makes eligible fallback fail closed. ds4_metal.m:28349 -runtime/metal DS4_METAL_REQUIRE_Q4_SSD_PREFILL_ATTN_OUT_EXACTN value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Requires Q4 SSD prefill attn out exactn and makes eligible fallback fail closed. ds4_metal.m:28065 -runtime/metal DS4_METAL_REQUIRE_Q4_SSD_SESSION_UNION nonempty boolean; unset/empty or exact 0: off; every other value: on Requires Q4 SSD session union and makes eligible fallback fail closed. ds4.c:64974 -runtime/metal DS4_METAL_REQUIRE_Q8_QKV_COMPRESSOR_FUSE nonempty boolean; unset/empty/exact 0: fallback allowed; other values require and imply the streamed/union enable Requires eligible Q8 QKV/compressor compound fusion and fails closed. ds4.c:22850 -runtime/metal DS4_METAL_RESUME_PREFILL_MIN integer token threshold; default 4; <=0 disables resume-prefill Sets the minimum shared-prefix suffix that uses batched resume-prefill. ds4.c:38229 -runtime/metal DS4_METAL_ROPE_EXP2_LOG2 boolean empty/1/true/yes/on vs 0/false/no/off; default off Compiles the exp2/log2 RoPE drift variant. ds4_metal.m:7049 -runtime/metal DS4_METAL_SELECTED_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for selected. ds4.c:37753 -runtime/metal DS4_METAL_SELECTED_PROFILE_LAYER single nonnegative layer index; unset: every layer Restricts selected-expert profiling to one layer. ds4_metal.m:42473 -runtime/metal DS4_METAL_SESSION_BATCH_LOG presence diagnostic; unset: off; any value including 0 enables Logs session batch decisions. ds4.c:65978 -runtime/metal DS4_METAL_SESSION_BATCH_QKV default enabled; exact 0 disables; every other value/unset leaves enabled Controls native batched QKV work for multi-session decode. ds4.c:65292 -runtime/metal DS4_METAL_SESSION_BATCH_SHARED default enabled; exact 0 disables; every other value/unset leaves enabled Controls native batched shared-expert work for multi-session decode. ds4.c:65242 -runtime/metal DS4_METAL_SET_ROWS_SOURCE file path; unset/empty: use the in-tree Metal source file Overrides the set-rows Metal kernel source file loaded at runtime. ds4_metal.m:4952 -runtime/metal DS4_METAL_SOFTMAX_SOURCE file path; unset/empty: use the in-tree Metal source file Overrides the softmax Metal kernel source file loaded at runtime. ds4_metal.m:4947 -runtime/metal DS4_METAL_STREAMING_DECODE_PREFILL_MAX integer token maximum; default 64 for wide Flash Q4/MXFP4, 18 for other PRO/Flash, 0 otherwise; <=0 disables Sets maximum SSD-streaming micro-prefill width that reuses decode. ds4.c:31772 -runtime/metal DS4_METAL_STREAMING_EXPERT_AUTO_PRELOAD_CAP uint32 expert cap; default 4096; 0 means unlimited; invalid restores default Caps automatic streaming-expert hotlist preload. ds4.c:21282 -runtime/metal DS4_METAL_STREAMING_EXPERT_BUFFER_MLOCK_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for streaming expert buffer mlock. ds4_metal.m:13995 -runtime/metal DS4_METAL_STREAMING_EXPERT_EARLY_LOAD_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for streaming expert early load. ds4_metal.m:17057 -runtime/metal DS4_METAL_STREAMING_EXPERT_EVICT_DONTNEED_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for streaming expert evict dontneed. ds4_metal.m:14434 -runtime/metal DS4_METAL_STREAMING_EXPERT_HOTLIST hotlist file path; unset/empty: built-in model hotlist Loads the streaming-expert preload order from a file. ds4.c:21322 -runtime/metal DS4_METAL_STREAMING_EXPERT_HOTLIST_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for streaming expert hotlist. ds4.c:32055 -runtime/metal DS4_METAL_STREAMING_EXPERT_LAYER_STATS presence diagnostic; unset: off; any value including 0 enables Collects/prints statistics for streaming expert layer. ds4_metal.m:4637 -runtime/metal DS4_METAL_STREAMING_EXPERT_LAYER_STATS_DELTA presence control; unset: off/default; any value including 0 enables Prints delta statistics for streaming expert layer. ds4_metal.m:4674 -runtime/metal DS4_METAL_STREAMING_EXPERT_NOCACHE nonempty value whose first character is not 0 enables; unset/empty/0 disables Uses a reopened F_NOCACHE descriptor for SSD expert preads. ds4_metal.m:12600 -runtime/metal DS4_METAL_STREAMING_EXPERT_PREAD_POOL default enabled; exact 0 disables; every other value/unset keeps enabled Controls reuse of persistent expert-pread worker threads. ds4_metal.m:13431 -runtime/metal DS4_METAL_STREAMING_EXPERT_PREAD_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for streaming expert pread. ds4_metal.m:17003 -runtime/metal DS4_METAL_STREAMING_EXPERT_PREAD_SPLIT integer clamped 1..8; unset: automatic 1 below 64 cache experts, 4 at 64+ Sets aligned requests per expert pread. ds4_metal.m:13699 -runtime/metal DS4_METAL_STREAMING_EXPERT_PREAD_THREADS unsigned integer clamped 1..18; default 9; invalid restores 9 Sets expert-pread worker limit. ds4_metal.m:13335 -runtime/metal DS4_METAL_STREAMING_EXPERT_PROFILE_SUMMARY presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for streaming expert. ds4_metal.m:13059 -runtime/metal DS4_METAL_STREAMING_EXPERT_SLAB_MB positive unsigned MiB; default 4096; 0/invalid restores default Sets target allocation size for streaming-expert slabs. ds4_metal.m:14037 -runtime/metal DS4_METAL_STREAMING_EXPERT_SPLIT_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for streaming expert split. ds4_metal.m:43776 -runtime/metal DS4_METAL_STREAMING_EXPERT_TIMING_SUMMARY presence control; unset: off/default; any value including 0 enables Prints timing/profile diagnostics for streaming expert. ds4_metal.m:13058 -runtime/metal DS4_METAL_STREAMING_IQ2_CPU_ROUTER_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for streaming IQ2 CPU router. ds4.c:21472 -runtime/metal DS4_METAL_STREAMING_MAP_TRACE nonempty value other than exact 0 enables; unset/empty/0 disables Emits SSD model-map decisions. ds4_metal.m:4848 -runtime/metal DS4_METAL_STREAMING_PREFILL_BATCH_SELECTED_ADDR_MAX integer token maximum; default 800 for 384 experts, 760 for 256, 0 otherwise; <=0 disables automatic selection Sets automatic maximum batch width for selected-address SSD prefill. ds4_metal.m:14637 -runtime/metal DS4_METAL_STREAMING_PREFILL_BATCH_SELECTED_ADDR_MIN integer token minimum; default 2 for 256/384 experts, 0 otherwise; <=0 disables automatic selection Sets automatic minimum batch width for selected-address SSD prefill. ds4_metal.m:14654 -runtime/metal DS4_METAL_STREAMING_PREFILL_BATCH_SELECTED_ADDR_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for streaming prefill batch selected address. ds4_metal.m:18491 -runtime/metal DS4_METAL_STREAMING_PREFILL_CACHE_SEED_K uint32 seed rows; default 1; 0 disables; above 64 clamps to 64 Sets how many prefill routing rows seed the decode expert cache. ds4.c:21095 -runtime/metal DS4_METAL_STREAMING_PREFILL_CACHE_SEED_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for streaming prefill cache seed. ds4_metal.m:17897 -runtime/metal DS4_METAL_STREAMING_PREFILL_LAYER_MADVISE_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for streaming prefill layer madvise. ds4.c:19437 -runtime/metal DS4_METAL_STREAMING_PREFILL_LAYER_PAGEIN_NO_OVERLAP presence rollback; unset: automatic/default path; any value including 0 disables Prevents full-layer page-in preparation from overlapping compute. ds4.c:19144 -runtime/metal DS4_METAL_STREAMING_PREFILL_LAYER_PAGEIN_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for streaming prefill layer pagein. ds4.c:19433 -runtime/metal DS4_METAL_STREAMING_PREFILL_LAYER_PAGEIN_THREADS integer 1..16; default 8; invalid/0 becomes 1; PREPARE_THREADS takes precedence Sets worker count for full-layer page-in preparation. ds4.c:19102 -runtime/metal DS4_METAL_STREAMING_PREFILL_LAYER_PREAD_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for streaming prefill layer pread. ds4.c:19435 -runtime/metal DS4_METAL_STREAMING_PREFILL_LAYER_PREPARE_AHEAD integer 1..4 layers; default 1; invalid/0 becomes 1 Sets number of future layers prepared concurrently. ds4.c:19156 -runtime/metal DS4_METAL_STREAMING_PREFILL_LAYER_PREPARE_NO_OVERLAP presence rollback; unset: automatic/default path; any value including 0 disables Prevents generic full-layer preparation from overlapping compute. ds4.c:19142 -runtime/metal DS4_METAL_STREAMING_PREFILL_LAYER_PREPARE_THREADS integer 1..16; default 8; invalid/0 becomes 1; preferred over PAGEIN_THREADS Sets worker count for full-layer preparation. ds4.c:19098 -runtime/metal DS4_METAL_STREAMING_PREFILL_LAYER_READAHEAD_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for streaming prefill layer readahead. ds4.c:19439 -runtime/metal DS4_METAL_STREAMING_PREFILL_SELECTED_MADVISE_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for streaming prefill selected madvise. ds4.c:19183 -runtime/metal DS4_METAL_STREAMING_PREFILL_SELECTED_MADVISE_THREADS integer 1..16; default inherits layer prepare threads; invalid/0 becomes 1; PREPARE_THREADS preferred Sets worker count for selected-expert madvise preparation. ds4.c:19120 -runtime/metal DS4_METAL_STREAMING_PREFILL_SELECTED_PAGEIN_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for streaming prefill selected pagein. ds4.c:19181 -runtime/metal DS4_METAL_STREAMING_PREFILL_SELECTED_PREPARE_GAP integer 0..8 layers; default 0; above 8 clamps; invalid restores 0 Sets lookahead gap for selected-expert preparation. ds4.c:19132 -runtime/metal DS4_METAL_STREAMING_PREFILL_SELECTED_PREPARE_THREADS integer 1..16 for madvise preparation; default inherits layer prepare threads; invalid/0 becomes 1 Sets worker count for selected-expert preparation. ds4.c:19116 -runtime/metal DS4_METAL_STREAMING_PREFILL_SELECTED_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for streaming prefill selected. ds4.c:18733 -runtime/metal DS4_METAL_STREAMING_PREFILL_SELECTED_READAHEAD_GAP integer 0..8 layers; default 0; above 8 clamps; invalid restores 0 Sets lookahead gap for selected-expert readahead. ds4.c:19805 -runtime/metal DS4_METAL_STREAMING_PREFILL_SELECTED_READAHEAD_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for streaming prefill selected readahead. ds4.c:19890 -runtime/metal DS4_METAL_STREAMING_SELECTED_READAHEAD_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for streaming selected readahead. ds4.c:21564 -runtime/metal DS4_METAL_SUM_ROWS_SOURCE file path; unset/empty: use the in-tree Metal source file Overrides the sum-rows Metal kernel source file loaded at runtime. ds4_metal.m:4946 -runtime/metal DS4_METAL_TEST_POISON_COMPRESSOR_EXACT_REDUCTION_SCRATCH internal test presence flag; unset: off; any value including 0 poisons scratch before the exact reduction Validates that compressor exact-reduction kernels overwrite all scratch state. ds4_metal.m:25923 -runtime/metal DS4_METAL_TP_SESSION_BATCH default enabled; exact 0 disables; every other value/unset leaves enabled Controls batched session evaluation with Metal TP. ds4.c:65120 -runtime/metal DS4_METAL_TRACE_ALLOCS presence diagnostic; unset: off; any value including 0 enables Emits trace diagnostics for allocs. ds4_metal.m:4056 -runtime/metal DS4_METAL_TRACE_M5_FLASH_ATTN_PACKED32_REDUCE presence diagnostic; unset: off; any value including 0 enables Emits trace diagnostics for M5 flash attn packed32 reduce. ds4_metal.m:31506 -runtime/metal DS4_METAL_UNARY_SOURCE file path; unset/empty: use the in-tree Metal source file Overrides the unary operations Metal kernel source file loaded at runtime. ds4_metal.m:4938 -runtime/metal DS4_METAL_UNRETAINED_COMMAND_BUFFERS presence control; unset: off/default; any value including 0 enables Creates Metal command buffers with unretained references. ds4_metal.m:1314 -runtime/metal DS4_METAL_USE_QUEUE_RESIDENCY_SET presence control; unset: off/default; any value including 0 enables Allows queue-residency state to trigger Q4 expert address/table paths. ds4_metal.m:42246 -runtime/moe-debug DS4_MOE_RECORD_SELECTED_HOTLIST nonempty output path; unset=off Record per-layer selected-expert hit counts to a Metal hotlist file. ds4_metal.m:1740 -runtime/moe-debug DS4_MOE_RECORD_SELECTED_HOTLIST_FRESH presence flag; only relevant with HOTLIST; overrides MERGE Start the selected-expert hotlist from empty state. ds4_metal.m:1641 -runtime/moe-debug DS4_MOE_RECORD_SELECTED_HOTLIST_MERGE presence flag; active only when FRESH is absent Merge an existing selected-expert hotlist before recording. ds4_metal.m:1640 -runtime/moe-debug DS4_MOE_RECORD_SELECTED_IDS nonempty output path; unset=off Record routed-MoE six-expert selections; also disables incompatible optimized paths. ds4.c:65100 -runtime/moe-debug DS4_MOE_REPLAY_SELECTED_IDS nonempty input path; unset=off Replay routed-MoE six-expert selections; also disables incompatible optimized paths. ds4.c:25135 -runtime/mtp DS4_MTP_BATCH_VERIFY Pure presence flag: any defined value, including empty or "0", suppresses the exact two-row decode verifier. Unset selects exact decode-2 when draft_n==2 and either strict mode is active or the build is ROCm; other cases already use the generic verifier. Diagnostic rollback from the exact Q8/one-token-equivalent MTP decode-2 verifier to the generic microbatch verifier. ds4.c:70842 -runtime/mtp DS4_MTP_CAPTURE_PREFIX1 Pure presence flag. In the generic verifier with exactly two drafts it enables prefix-1 state capture under strict mode; non-strict mode already captures prefix-1 without the variable. Unset under strict mode instead snapshots and replays a partial acceptance. Let a one-of-two MTP partial acceptance commit the verifier's captured prefix directly, avoiding an exact one-token replay. ds4.c:70948 -runtime/mtp DS4_MTP_CONF_LOG Pure presence flag; default off. It forces materialization of full draft logits, computes the top-2 margin, and after a successful generic microbatch verification prints drafted/committed counts, top candidates, margin, target-next and draft-next. Exact decode-2 success does not emit that generic log line. Inspect MTP draft confidence and compare the recursive draft token with the target verifier result. ds4.c:70710 -runtime/mtp DS4_MTP_EXACT_REPLAY Pure presence flag; default off. In the generic microbatch verifier it forces a pre-verifier frontier snapshot; after verification the snapshot is restored and every accepted draft is decoded sequentially to rebuild exact final state/logits. Validate MTP acceptance while committing through the normal one-token decode path rather than retaining batched-verifier state. ds4.c:70949 -runtime/mtp DS4_MTP_FORCE_SNAPSHOT Pure presence flag; default off. It forces a speculative-frontier snapshot before the generic verifier regardless of draft count or prefix-capture mode; it does not by itself force restoration or replay after a successful full acceptance. Measure/debug snapshot behavior and guarantee a restorable pre-verifier frontier for generic MTP verification. ds4.c:70953 -runtime/mtp DS4_MTP_FULL_LOGITS Pure presence flag; default off. When set, legacy and recursive MTP draft calls write the full vocabulary logits to s->mtp_logits; unset permits the faster top-token-only output unless confidence/margin logic independently needs logits. Force full MTP draft-logit materialization for correctness comparison, inspection, or downstream confidence calculations. ds4.c:64070 -runtime/mtp DS4_MTP_MIN_MARGIN non-negative float; default engine --mtp-margin value Set confidence margin threshold for speculative MTP verification. ds4.c:70703 -runtime/mtp DS4_MTP_PROBE Pure presence flag; default off. For legacy MTP it prepares drafts even when configured depth<=1, compares the previous draft with the next committed token, and prints cumulative hit counts/failures; generated output is unchanged. Measure legacy MTP next-token draft accuracy without enabling speculative acceptance. ds4.c:64800 +runtime/metal DS4_METAL_ARGSORT_SOURCE file path; unset/empty: use the in-tree Metal source file Overrides the argsort Metal kernel source file loaded at runtime. ds4_metal.m:4943 +runtime/metal DS4_METAL_ATTN_OUT_STAGE_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for attn out stage. ds4.c:65151 +runtime/metal DS4_METAL_BIN_SOURCE file path; unset/empty: use the in-tree Metal source file Overrides the binary operations Metal kernel source file loaded at runtime. ds4_metal.m:4952 +runtime/metal DS4_METAL_COMPRESSOR_PAIR_NR4 presence control; unset: off/default; any value including 0 enables Selects the NR4 compressor-pair variant. ds4_metal.m:2651 +runtime/metal DS4_METAL_CONCAT_SOURCE file path; unset/empty: use the in-tree Metal source file Overrides the concatenation Metal kernel source file loaded at runtime. ds4_metal.m:4945 +runtime/metal DS4_METAL_CPY_SOURCE file path; unset/empty: use the in-tree Metal source file Overrides the copy Metal kernel source file loaded at runtime. ds4_metal.m:4944 +runtime/metal DS4_METAL_DECODE_INDEXER_SPARSE_THRESHOLD integer in {64,128,256,512,1024,2048,4096}; default 1024; invalid restores default Sets the compressed-row crossover from dense to sparse indexed attention. ds4.c:20352 +runtime/metal DS4_METAL_DECODE_STAGE_PROFILE unset: off; 1/true/yes/on/all enables all layers; a layer index selects one; 0/false/no/off disables Prints timing/profile diagnostics for decode stage. ds4.c:17568 +runtime/metal DS4_METAL_DECODE_STAGE_PROFILE_LAYER single unsigned layer index; unset/empty: all layers enabled by the parent profile; invalid matches no layer Restricts the corresponding shared graph stage profiler to one layer. ds4.c:29120 +runtime/metal DS4_METAL_DENSE_SOURCE file path; unset/empty: use the in-tree Metal source file Overrides the dense matmul Metal kernel source file loaded at runtime. ds4_metal.m:4936 +runtime/metal DS4_METAL_DISABLE_AFFINE_ROPE_PAIR presence rollback; unset: automatic/default path; any value including 0 disables Disables affine RoPE pair. ds4_metal.m:25171 +runtime/metal DS4_METAL_DISABLE_ATTN_OUT_HC_FUSION nonempty boolean; unset/empty or exact 0: off; every other value: on Disables attn out HC fusion. ds4.c:20477 +runtime/metal DS4_METAL_DISABLE_ATTN_OUT_IDS_CACHE presence rollback; unset: automatic/default path; any value including 0 disables Disables attn out ids cache. ds4_metal.m:27771 +runtime/metal DS4_METAL_DISABLE_ATTN_OUT_LOW_DIRECT presence rollback; unset: automatic/default path; any value including 0 disables Disables attn out low direct. ds4_metal.m:27760 +runtime/metal DS4_METAL_DISABLE_BATCH_HC_NORM_FUSION nonempty value other than exact 0 disables; unset/empty/0 leaves the default enabled path Dominant rollback for batched HC norm fusion. ds4.c:20457 +runtime/metal DS4_METAL_DISABLE_COMPRESSOR_APE_ADD presence rollback; unset: automatic/default path; any value including 0 disables Disables compressor APE add. ds4_metal.m:25393 +runtime/metal DS4_METAL_DISABLE_COMPRESSOR_EXACT_POOL_RATIO4 presence rollback; unset: automatic/default path; any value including 0 disables Disables compressor exact pool ratio4. ds4_metal.m:26379 +runtime/metal DS4_METAL_DISABLE_COMPRESSOR_PAIR_PROJ nonempty boolean; unset/empty or exact 0: off; every other value: on Disables compressor pair proj. ds4_metal.m:23248 +runtime/metal DS4_METAL_DISABLE_COMPRESSOR_QUAD_STORE presence rollback; unset: automatic/default path; any value including 0 disables Disables compressor quad store. ds4.c:23534 +runtime/metal DS4_METAL_DISABLE_COMPRESSOR_RATIO4_DIRECT_POOL presence rollback; unset: automatic/default path; any value including 0 disables Disables compressor ratio4 direct pool. ds4_metal.m:26245 +runtime/metal DS4_METAL_DISABLE_COMPRESSOR_RATIO4_PACK_FUSION presence rollback; unset: automatic/default path; any value including 0 disables Disables compressor ratio4 pack fusion. ds4_metal.m:26199 +runtime/metal DS4_METAL_DISABLE_COMPRESSOR_STORE_ONE presence rollback; unset: automatic/default path; any value including 0 disables Disables compressor store one. ds4_metal.m:23249 +runtime/metal DS4_METAL_DISABLE_CONTIG_F16_F16_COPY value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Disables contig F16 F16 copy. ds4_metal.m:29562 +runtime/metal DS4_METAL_DISABLE_CONTIG_F32_F16_COPY value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Disables contig F32 F16 copy. ds4_metal.m:29338 +runtime/metal DS4_METAL_DISABLE_DECODE_NORM_EXACT_VIEWS presence rollback; unset: automatic/default path; any value including 0 disables Disables decode norm exact views. ds4_metal.m:36207 +runtime/metal DS4_METAL_DISABLE_DECODE_RAW_GATHERED_ATTN presence rollback; unset: raw-only decode uses gathered attention; any value including 0 restores the legacy raw path Restores the separate raw-only attention path instead of gathered staging and attention. ds4_metal.m:32968 +runtime/metal DS4_METAL_DISABLE_DECODE_RAW_PACKED32 presence rollback; unset: raw-only gathered attention may use packed32; any value including 0 disables it for raw-only layers Disables the packed32 reduce kernel for raw-only gathered attention while leaving compressed layers unchanged. ds4_metal.m:31464 +runtime/metal DS4_METAL_DISABLE_DECODE_ROUTER_BIAS_EXACT_VIEWS presence rollback; unset: automatic/default path; any value including 0 disables Disables decode router bias exact views. ds4_metal.m:39397 +runtime/metal DS4_METAL_DISABLE_DSPARK_CAPTURE_FUSED_LAST nonempty boolean; unset/empty or exact 0: off; every other value: on Disables DSpark capture fused last. ds4.c:28361 +runtime/metal DS4_METAL_DISABLE_DSPARK_EXACTN_BATCH_HEAD nonempty boolean; unset/empty or exact 0: off; every other value: on Disables DSpark exactn batch head. ds4.c:37728 +runtime/metal DS4_METAL_DISABLE_EXACT_ROWS_PERSISTENT_CACHE nonempty boolean; unset/empty or exact 0: off; every other value: on Disables exact rows persistent cache. ds4_metal.m:13000 +runtime/metal DS4_METAL_DISABLE_GATHERED_KV_PAD_FUSION presence rollback; unset: automatic/default path; any value including 0 disables Disables gathered KV pad fusion. ds4_metal.m:29706 +runtime/metal DS4_METAL_DISABLE_GATHERED_KV_STAGE presence rollback; unset: automatic/default path; any value including 0 disables Disables gathered KV stage. ds4_metal.m:29677 +runtime/metal DS4_METAL_DISABLE_GLM_DECODE_KV_GROUP4 value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Disables GLM decode KV group4. ds4_metal.m:36742 +runtime/metal DS4_METAL_DISABLE_GLM_QKLOW_SG presence rollback; unset: automatic/default path; any value including 0 disables Disables GLM qklow sg. ds4_metal.m:37865 +runtime/metal DS4_METAL_DISABLE_GLM_STREAMING_EXPERT_EARLY_LOAD presence rollback; unset: automatic/default path; any value including 0 disables Disables GLM streaming expert early load. ds4_metal.m:18058 +runtime/metal DS4_METAL_DISABLE_GLM_STREAMING_EXPERT_SPLIT presence rollback; unset: automatic/default path; any value including 0 disables Disables GLM streaming expert split. ds4_metal.m:39796 +runtime/metal DS4_METAL_DISABLE_GLM_STREAMING_PREFILL_FULL_LAYER presence rollback; unset: automatic/default path; any value including 0 disables Disables GLM streaming prefill full layer. ds4.c:42710 +runtime/metal DS4_METAL_DISABLE_GLM_STREAMING_PREFILL_FULL_LAYER_PREPARE presence rollback; unset: automatic/default path; any value including 0 disables Disables GLM streaming prefill full layer prepare. ds4.c:42728 +runtime/metal DS4_METAL_DISABLE_GLM_STREAMING_PREFILL_SELECTED_ASYNC_LOAD presence rollback; unset: automatic/default path; any value including 0 disables Disables GLM streaming prefill selected async load. ds4.c:46421 +runtime/metal DS4_METAL_DISABLE_GLM_STREAMING_SELECTED_ASYNC_LOAD presence rollback; unset: automatic/default path; any value including 0 disables Disables GLM streaming selected async load. ds4.c:44329 +runtime/metal DS4_METAL_DISABLE_HC_FUSION nonempty boolean; unset/empty or exact 0: off; every other value: on Disables HC fusion. ds4.c:20406 +runtime/metal DS4_METAL_DISABLE_HC_NORM_FUSION nonempty boolean; unset/empty or exact 0: off; every other value: on Disables HC norm fusion. ds4.c:20450 +runtime/metal DS4_METAL_DISABLE_HC_PRODUCER_PRE_NORM_FUSE presence rollback; unset: automatic/default path; any value including 0 disables Disables HC producer pre norm fuse. ds4_metal.m:46541 +runtime/metal DS4_METAL_DISABLE_HC_RMS_SCALE_PROJ presence rollback; unset: automatic/default path; any value including 0 disables Disables HC RMS scale proj. ds4_metal.m:24148 +runtime/metal DS4_METAL_DISABLE_HOT_PIPELINE_STATICS presence rollback; unset: automatic/default path; any value including 0 disables Disables hot pipeline statics. ds4_metal.m:2634 +runtime/metal DS4_METAL_DISABLE_INPLACE_ROPE_PAIR presence rollback; unset: automatic/default path; any value including 0 disables Disables inplace RoPE pair. ds4_metal.m:25170 +runtime/metal DS4_METAL_DISABLE_IQ2_SELECTED_EXPERT_VIEWS presence rollback; unset: automatic/default path; any value including 0 disables Disables IQ2 selected expert views. ds4.c:21080 +runtime/metal DS4_METAL_DISABLE_IQ2_SELECTED_SHARED_OVERLAP presence rollback; unset: automatic/default path; any value including 0 disables Disables IQ2 selected shared overlap. ds4.c:20996 +runtime/metal DS4_METAL_DISABLE_IQ2_STREAM_ADDR_TABLE presence rollback; unset: automatic/default path; any value including 0 disables Disables IQ2 stream address table. ds4_metal.m:42402 +runtime/metal DS4_METAL_DISABLE_IQ2_XXS_SSD_PREFILL_MM value-aware boolean; default off; true disables and dominates ENABLE; false leaves automatic policy Rolls grouped IQ2_XXS/Q2_K SSD-prefill MM back to sparse matvec. ds4_metal.m:44784 +runtime/metal DS4_METAL_DISABLE_KV_FUSION nonempty boolean; unset/empty or exact 0: off; every other value: on Disables KV fusion. ds4.c:20430 +runtime/metal DS4_METAL_DISABLE_M1_IQ2_MID_ONLY presence rollback; unset: automatic/default path; any value including 0 disables Disables M1 IQ2 mid only. ds4_metal.m:14592 +runtime/metal DS4_METAL_DISABLE_M3_COMPRESSOR_EXACT_POOL_RATIO4 presence rollback; unset: automatic/default path; any value including 0 disables Disables M3 compressor exact pool ratio4. ds4_metal.m:26381 +runtime/metal DS4_METAL_DISABLE_M3_COMPRESSOR_PAIR_STATE_STORE presence rollback; unset: automatic/default path; any value including 0 disables Disables M3 compressor pair state store. ds4_metal.m:23247 +runtime/metal DS4_METAL_DISABLE_M3_GATHERED_KV_STAGE presence rollback; unset: automatic/default path; any value including 0 disables Disables M3 gathered KV stage. ds4_metal.m:29678 +runtime/metal DS4_METAL_DISABLE_M5_COMPRESSOR_EXACT_POOL_RATIO4 presence rollback; unset: automatic/default path; any value including 0 disables Disables M5 compressor exact pool ratio4. ds4_metal.m:26384 +runtime/metal DS4_METAL_DISABLE_M5_COMP_FINALIZE_FUSE presence rollback; unset: automatic/default path; any value including 0 disables Disables M5 comp finalize fuse. ds4.c:23647 +runtime/metal DS4_METAL_DISABLE_M5_FLASH_ATTN_PACKED32_REDUCE presence rollback; unset: automatic/default path; any value including 0 disables Disables M5 flash attn packed32 reduce. ds4_metal.m:31519 +runtime/metal DS4_METAL_DISABLE_M5_HC_NORM_MIX_CLUSTER2 presence rollback; unset: automatic/default path; any value including 0 disables Disables M5 HC norm mix cluster2. ds4_metal.m:46496 +runtime/metal DS4_METAL_DISABLE_M5_HC_PRODUCER_PRE_NORM_FUSE presence rollback; unset: automatic/default path; any value including 0 disables Disables M5 HC producer pre norm fuse. ds4_metal.m:46548 +runtime/metal DS4_METAL_DISABLE_M5_IQ2_PAIR_PACK2 presence rollback; unset: automatic/default path; any value including 0 disables Disables M5 IQ2 pair pack2. ds4_metal.m:42020 +runtime/metal DS4_METAL_DISABLE_M5_PACKED_ZERO_MASK presence rollback; unset: automatic/default path; any value including 0 disables Disables M5 packed zero mask. ds4_metal.m:31475 +runtime/metal DS4_METAL_DISABLE_M5_PARALLEL_FULL_FFN presence rollback; unset: automatic/default path; any value including 0 disables Disables M5 parallel full FFN. ds4.c:22707 +runtime/metal DS4_METAL_DISABLE_M5_PERSISTENT_ZERO_ATTN_MASK presence rollback; unset: automatic/default path; any value including 0 disables Disables M5 persistent zero attn mask. ds4_metal.m:31473 +runtime/metal DS4_METAL_DISABLE_M5_Q8_HC_VEC presence rollback; unset: automatic/default path; any value including 0 disables Disables M5 Q8 HC vec. ds4_metal.m:47580 +runtime/metal DS4_METAL_DISABLE_M5_QKV_PAIR_COMPRESSOR_FUSE presence rollback; unset: automatic/default path; any value including 0 disables Disables M5 QKV pair compressor fuse. ds4.c:23043 +runtime/metal DS4_METAL_DISABLE_M5_QKV_PAIR_QUAD_FUSE presence rollback; unset: automatic/default path; any value including 0 disables Disables M5 QKV pair quad fuse. ds4.c:23039 +runtime/metal DS4_METAL_DISABLE_M5_ROUTER_PROJECT_SELECT_FUSE presence rollback; unset: automatic/default path; any value including 0 disables Disables M5 router project select fuse. ds4.c:24762 +runtime/metal DS4_METAL_DISABLE_METAL4 value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Disables metal4. ds4_metal.m:2995 +runtime/metal DS4_METAL_DISABLE_MOE_MM_ID_PAIR_SWIGLU presence rollback; unset: automatic/default path; any value including 0 disables Disables MoE MM ID pair SwiGLU. ds4_metal.m:44968 +runtime/metal DS4_METAL_DISABLE_MOE_MM_ID_USE_RESOURCES presence rollback; unset: automatic/default path; any value including 0 disables Disables MoE MM ID use resources. ds4_metal.m:35169 +runtime/metal DS4_METAL_DISABLE_MXFP4_SELECTED_EXPERT_VIEWS presence rollback; unset: automatic/default path; any value including 0 disables Disables MXFP4 selected expert views. ds4.c:21173 +runtime/metal DS4_METAL_DISABLE_PERSISTENT_ZERO_ATTN_MASK presence rollback; unset: automatic/default path; any value including 0 disables Disables persistent zero attn mask. ds4_metal.m:31480 +runtime/metal DS4_METAL_DISABLE_PRE_M5_ATTN_INV_ROPE_FUSE presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 attn inv RoPE fuse. ds4.c:22638 +runtime/metal DS4_METAL_DISABLE_PRE_M5_ATTN_OUT_LOW_Q8_STATIC presence rollback; unset: exact fixed-shape kernel is automatic on eligible pre-M5 Flash decode; any value including 0 disables Restores the generic Q8 attention-output low projection kernel. ds4_metal.m:27987 +runtime/metal DS4_METAL_DISABLE_PRE_M5_BATCH_INDEXER_QUERY_PRUNE presence rollback; unset: unused zero-prefix indexer queries are pruned before compressed rows exceed top-k; any value including 0 disables Restores transient indexer query and weight dispatches during eligible pre-M5 prefill. ds4.c:30086 +runtime/metal DS4_METAL_DISABLE_PRE_M5_COMPRESSOR_EXACT_POOL_RATIO4 presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 compressor exact pool ratio4. ds4_metal.m:26383 +runtime/metal DS4_METAL_DISABLE_PRE_M5_COMPRESSOR_EXACT_REDUCTION_FUSION presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 compressor exact reduction fusion. ds4_metal.m:25907 +runtime/metal DS4_METAL_DISABLE_PRE_M5_COMPRESSOR_QUAD_STORE presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 compressor quad store. ds4.c:23535 +runtime/metal DS4_METAL_DISABLE_PRE_M5_COMPRESSOR_RATIO4_DECODE_PACK_FUSION presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 compressor ratio4 decode pack fusion. ds4_metal.m:26214 +runtime/metal DS4_METAL_DISABLE_PRE_M5_COMP_FINALIZE_FUSE presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 comp finalize fuse. ds4.c:23646 +runtime/metal DS4_METAL_DISABLE_PRE_M5_DECODE_EARLY_PIPELINE_FAST_LOOKUP presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 decode early pipeline fast lookup. ds4.c:28003 +runtime/metal DS4_METAL_DISABLE_PRE_M5_DECODE_EARLY_SECOND_SPLIT12 presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 decode early second split12. ds4.c:28087 +runtime/metal DS4_METAL_DISABLE_PRE_M5_DECODE_EARLY_SPLIT3 presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 decode early split3. ds4.c:27961 +runtime/metal DS4_METAL_DISABLE_PRE_M5_DECODE_EARLY_SPLIT5 presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 decode early split5. ds4.c:27973 +runtime/metal DS4_METAL_DISABLE_PRE_M5_DECODE_PIPELINE_FAST_LOOKUP presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 decode pipeline fast lookup. ds4.c:28015 +runtime/metal DS4_METAL_DISABLE_PRE_M5_DECODE_PORTS presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 decode ports. ds4.c:22613 +runtime/metal DS4_METAL_DISABLE_PRE_M5_DECODE_RAW_ZERO_ATTN_MASK presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 decode raw zero attn mask. ds4_metal.m:29920 +runtime/metal DS4_METAL_DISABLE_PRE_M5_DECODE_SECOND_SPLIT16 presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 decode second split16. ds4.c:28095 +runtime/metal DS4_METAL_DISABLE_PRE_M5_FLASH_ATTN_BATCHED_MEMO presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 flash attn batched memo. ds4_metal.m:3752 +runtime/metal DS4_METAL_DISABLE_PRE_M5_FLASH_ATTN_PACKED32_REDUCE presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 flash attn packed32 reduce. ds4_metal.m:31460 +runtime/metal DS4_METAL_DISABLE_PRE_M5_FLASH_ATTN_PAD_BLK_MEMO presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 flash attn pad blk memo. ds4_metal.m:3610 +runtime/metal DS4_METAL_DISABLE_PRE_M5_HC_NORM_MIX_FUSE presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 HC norm mix fuse. ds4.c:22867 +runtime/metal DS4_METAL_DISABLE_PRE_M5_HC_PRODUCER_PRE_NORM_FUSE presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 HC producer pre norm fuse. ds4_metal.m:46546 +runtime/metal DS4_METAL_DISABLE_PRE_M5_HEAD_RMS_ROPE_PIPELINE_STATIC presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 head RMS RoPE pipeline static. ds4_metal.m:10026 +runtime/metal DS4_METAL_DISABLE_PRE_M5_KV_ROPE_FP8_FUSE presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 KV RoPE fp8 fuse. ds4.c:23455 +runtime/metal DS4_METAL_DISABLE_PRE_M5_MXFP4_MM_ID_PAIR_HALF_SCALE presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 MXFP4 MM ID pair half scale. ds4_metal.m:45129 +runtime/metal DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_DECODE_FIXED_ROUTE_PAIR presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 MXFP4 MoE decode fixed route pair. ds4_metal.m:41948 +runtime/metal DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_DECODE_FIXED_ROUTE_SUM6 presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 MXFP4 MoE decode fixed route sum6. ds4_metal.m:41961 +runtime/metal DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_DECODE_NSG1 presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 MXFP4 MoE decode nsg1. ds4_metal.m:41719 +runtime/metal DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_DECODE_STATIC_TRIP presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 MXFP4 MoE decode static trip. ds4_metal.m:41987 +runtime/metal DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_DECODE_SUM6_FULL_ROWS presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 MXFP4 MoE decode sum6 full rows. ds4_metal.m:41974 +runtime/metal DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_DECODE_TG_MULTIPLE presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 MXFP4 MoE decode tg multiple. ds4_metal.m:41936 +runtime/metal DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_MM_ID_DOWN_HALF_LUT presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 MXFP4 MoE MM ID down half lut. ds4_metal.m:45047 +runtime/metal DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_MM_ID_DOWN_TAIL_SIMDGROUP_CULL presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 MXFP4 MoE MM ID down tail simdgroup cull. ds4_metal.m:45030 +runtime/metal DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_MM_ID_MAP_SCATTER presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 MXFP4 MoE MM ID map scatter. ds4_metal.m:44997 +runtime/metal DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_MM_ID_PAIR_SWIGLU_COMPACT_TILE presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 MXFP4 MoE MM ID pair SwiGLU compact tile. ds4_metal.m:44981 +runtime/metal DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_MM_ID_PAIR_TAIL_SIMDGROUP_CULL presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 MXFP4 MoE MM ID pair tail simdgroup cull. ds4_metal.m:45018 +runtime/metal DS4_METAL_DISABLE_PRE_M5_PARALLEL_FULL_FFN presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 parallel full FFN. ds4.c:22706 +runtime/metal DS4_METAL_DISABLE_PRE_M5_Q2_DECODE_SPLIT2_32 presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 q2 decode split2 32. ds4.c:27876 +runtime/metal DS4_METAL_DISABLE_PRE_M5_QKV_NORM_KV_STORE_FUSE presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 QKV norm KV store fuse. ds4.c:23314 +runtime/metal DS4_METAL_DISABLE_PRE_M5_QKV_PAIR_COMPRESSOR_FUSE presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 QKV pair compressor fuse. ds4.c:23042 +runtime/metal DS4_METAL_DISABLE_PRE_M5_QKV_PAIR_QUAD_FUSE presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 QKV pair quad fuse. ds4.c:23038 +runtime/metal DS4_METAL_DISABLE_PRE_M5_ROUTER_SHARED_FUSE presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 router shared fuse. ds4.c:24755 +runtime/metal DS4_METAL_DISABLE_PRE_M5_ROUTER_SIMD_FINALIZE presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 router simd finalize. ds4_metal.m:35827 +runtime/metal DS4_METAL_DISABLE_PRE_M5_ROUTER_SIMD_WEIGHTS_FUSION presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 router simd weights fusion. ds4_metal.m:35836 +runtime/metal DS4_METAL_DISABLE_PRE_M5_ROUTER_TRANSFORM_FINALIZE_FUSION presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 router transform finalize fusion. ds4_metal.m:35840 +runtime/metal DS4_METAL_DISABLE_PRO_Q4_EXPERT_ADDRESS_AUTO presence rollback; unset: automatic/default path; any value including 0 disables Disables pro Q4 expert address auto. ds4_metal.m:19710 +runtime/metal DS4_METAL_DISABLE_PRO_Q4_EXPERT_TABLE_AUTO presence rollback; unset: automatic/default path; any value including 0 disables Disables pro Q4 expert table auto. ds4.c:21040 +runtime/metal DS4_METAL_DISABLE_PRO_Q4_EXPERT_TABLE_PRELOAD presence rollback; unset: automatic/default path; any value including 0 disables Disables pro Q4 expert table preload. ds4.c:58780 +runtime/metal DS4_METAL_DISABLE_Q4_ATTN_OUT_HC_FUSE presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 attn out HC fuse. ds4_metal.m:47624 +runtime/metal DS4_METAL_DISABLE_Q4_ATTN_OUT_TINY_BATCH value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Disables Q4 attn out tiny batch. ds4_metal.m:28396 +runtime/metal DS4_METAL_DISABLE_Q4_BATCH_EXPERT_TABLE presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 batch expert table. ds4_metal.m:44883 +runtime/metal DS4_METAL_DISABLE_Q4_DENSE_PAIR presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 dense pair. ds4_metal.m:21990 +runtime/metal DS4_METAL_DISABLE_Q4_EXACT_BOUNDARY presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 exact boundary. ds4_metal.m:42239 +runtime/metal DS4_METAL_DISABLE_Q4_EXACT_TENSOR_ID presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 exact tensor ID. ds4_metal.m:42219 +runtime/metal DS4_METAL_DISABLE_Q4_EXPERT_ADDRESS_TABLE presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 expert address table. ds4_metal.m:19711 +runtime/metal DS4_METAL_DISABLE_Q4_EXPERT_TABLE presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 expert table. ds4.c:21041 +runtime/metal DS4_METAL_DISABLE_Q4_GATHER_SLOTS presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 gather slots. ds4_metal.m:42321 +runtime/metal DS4_METAL_DISABLE_Q4_GROUP24_EXPERT_TABLE presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 group24 expert table. ds4_metal.m:42201 +runtime/metal DS4_METAL_DISABLE_Q4_GROUP6_EXPERT_TABLE presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 group6 expert table. ds4_metal.m:42167 +runtime/metal DS4_METAL_DISABLE_Q4_GROUP8_EXPERT_TABLE presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 group8 expert table. ds4_metal.m:42184 +runtime/metal DS4_METAL_DISABLE_Q4_GROUPED_BOUNDARY presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 grouped boundary. ds4_metal.m:42149 +runtime/metal DS4_METAL_DISABLE_Q4_GROUPED_EXPERTS presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 grouped experts. ds4_metal.m:42130 +runtime/metal DS4_METAL_DISABLE_Q4_MV_CLASSIC presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 MV classic. ds4_metal.m:21571 +runtime/metal DS4_METAL_DISABLE_Q4_QKV_COMPRESSOR_FUSE presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 QKV compressor fuse. ds4_metal.m:22083 +runtime/metal DS4_METAL_DISABLE_Q4_SELECTED_EXPERT_VIEWS presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 selected expert views. ds4.c:21136 +runtime/metal DS4_METAL_DISABLE_Q4_SSD_PREFILL_ATTN_OUT_EXACTN value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Disables Q4 SSD prefill attn out exactn. ds4_metal.m:28095 +runtime/metal DS4_METAL_DISABLE_Q4_SSD_SESSION_UNION nonempty boolean; unset/empty or exact 0: off; every other value: on Disables Q4 SSD session union. ds4.c:65160 +runtime/metal DS4_METAL_DISABLE_Q4_STREAM_OVERLAP nonempty boolean; unset/empty or exact 0: off; every other value: on Disables Q4 stream overlap. ds4.c:65082 +runtime/metal DS4_METAL_DISABLE_Q4_TABLE_BOUNDARY presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 table boundary. ds4_metal.m:42318 +runtime/metal DS4_METAL_DISABLE_Q8_DECODE_EXACT_VIEWS presence rollback; unset: automatic/default path; any value including 0 disables Disables Q8 decode exact views. ds4_metal.m:12851 +runtime/metal DS4_METAL_DISABLE_QKV_NORM_FUSION nonempty boolean; unset/empty or exact 0: off; every other value: on Disables QKV norm fusion. ds4.c:20435 +runtime/metal DS4_METAL_DISABLE_QKV_PAIR_PROJ nonempty boolean; unset/empty or exact 0: off; every other value: on Disables QKV pair proj. ds4.c:20440 +runtime/metal DS4_METAL_DISABLE_QUEUE_RESIDENCY_SET presence rollback; unset: automatic/default path; any value including 0 disables Disables queue residency set. ds4_metal.m:2124 +runtime/metal DS4_METAL_DISABLE_ROUTED_PAIR_SWIGLU_FUSION presence rollback; unset: automatic/default path; any value including 0 disables Disables routed pair SwiGLU fusion. ds4.c:18529 +runtime/metal DS4_METAL_DISABLE_ROUTER_SELECT_FUSION presence rollback; unset: automatic/default path; any value including 0 disables Disables router select fusion. ds4_metal.m:35816 +runtime/metal DS4_METAL_DISABLE_ROUTER_WEIGHTS_BATCH_FUSION presence rollback; unset: automatic/default path; any value including 0 disables Disables router weights batch fusion. ds4_metal.m:36068 +runtime/metal DS4_METAL_DISABLE_SHARED_DOWN_HC_FUSION nonempty boolean; unset/empty or exact 0: off; every other value: on Disables shared down HC fusion. ds4.c:20472 +runtime/metal DS4_METAL_DISABLE_SHARED_GATE_UP_SWIGLU_FUSION presence rollback; unset: automatic/default path; any value including 0 disables Disables shared gate up SwiGLU fusion. ds4.c:17567 +runtime/metal DS4_METAL_DISABLE_SHARED_KV_PAD presence rollback; unset: automatic/default path; any value including 0 disables Disables shared KV pad. ds4_metal.m:31510 +runtime/metal DS4_METAL_DISABLE_SHARED_ROPE_COEFF presence rollback; unset: automatic/default path; any value including 0 disables Disables shared RoPE coeff. ds4_metal.m:6341 +runtime/metal DS4_METAL_DISABLE_STREAMING_COLD_DECODE_PREFILL presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming cold decode prefill. ds4.c:32007 +runtime/metal DS4_METAL_DISABLE_STREAMING_COMPACT_ADDR presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming compact address. ds4_metal.m:14598 +runtime/metal DS4_METAL_DISABLE_STREAMING_DECODE_PREFILL presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming decode prefill. ds4.c:31956 +runtime/metal DS4_METAL_DISABLE_STREAMING_EXPERT_ADDR_TABLE presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming expert address table. ds4.c:18525 +runtime/metal DS4_METAL_DISABLE_STREAMING_EXPERT_COMBINED_BUFFER presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming expert combined buffer. ds4_metal.m:14023 +runtime/metal DS4_METAL_DISABLE_STREAMING_EXPERT_EARLY_LOAD presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming expert early load. ds4_metal.m:17232 +runtime/metal DS4_METAL_DISABLE_STREAMING_EXPERT_EVICT_DONTNEED presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming expert evict dontneed. ds4_metal.m:14395 +runtime/metal DS4_METAL_DISABLE_STREAMING_EXPERT_HIT_VALIDATOR presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming expert hit validator. ds4_metal.m:14634 +runtime/metal DS4_METAL_DISABLE_STREAMING_EXPERT_HOTLIST presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming expert hotlist. ds4.c:21290 +runtime/metal DS4_METAL_DISABLE_STREAMING_EXPERT_LIVE_INDEX value-aware boolean; default off; true disables and dominates ENABLE Disables dense live-entry index and uses authoritative cache matrix. ds4_metal.m:15443 +runtime/metal DS4_METAL_DISABLE_STREAMING_EXPERT_MASKED_ADDR presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming expert masked address. ds4_metal.m:14628 +runtime/metal DS4_METAL_DISABLE_STREAMING_EXPERT_READAHEAD presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming expert readahead. ds4_metal.m:13196 +runtime/metal DS4_METAL_DISABLE_STREAMING_EXPERT_SLABS presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming expert slabs. ds4_metal.m:14028 +runtime/metal DS4_METAL_DISABLE_STREAMING_EXPERT_TIMING_SUMMARY presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming expert timing summary. ds4_metal.m:13062 +runtime/metal DS4_METAL_DISABLE_STREAMING_FULL_EXPERT_ADDR_TABLE presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming full expert address table. ds4_metal.m:14716 +runtime/metal DS4_METAL_DISABLE_STREAMING_IQ2_CPU_ROUTER presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming IQ2 CPU router. ds4.c:20939 +runtime/metal DS4_METAL_DISABLE_STREAMING_LAYER_BATCH presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming layer batch. ds4.c:18239 +runtime/metal DS4_METAL_DISABLE_STREAMING_MADVISE_WILLNEED presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming madvise willneed. ds4.c:18212 +runtime/metal DS4_METAL_DISABLE_STREAMING_PREFILL_BATCH_SELECTED_ADDR presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming prefill batch selected address. ds4.c:18523 +runtime/metal DS4_METAL_DISABLE_STREAMING_PREFILL_LAYER_MADVISE presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming prefill layer madvise. ds4.c:18466 +runtime/metal DS4_METAL_DISABLE_STREAMING_PREFILL_LAYER_PAGEIN presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming prefill layer pagein. ds4.c:18434 +runtime/metal DS4_METAL_DISABLE_STREAMING_PREFILL_LAYER_PAGEIN_OVERLAP presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming prefill layer pagein overlap. ds4.c:19321 +runtime/metal DS4_METAL_DISABLE_STREAMING_PREFILL_LAYER_PREAD presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming prefill layer pread. ds4.c:18454 +runtime/metal DS4_METAL_DISABLE_STREAMING_PREFILL_LAYER_PREPARE presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming prefill layer prepare. ds4.c:18446 +runtime/metal DS4_METAL_DISABLE_STREAMING_PREFILL_LAYER_PREPARE_OVERLAP presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming prefill layer prepare overlap. ds4.c:19319 +runtime/metal DS4_METAL_DISABLE_STREAMING_PREFILL_LAYER_READAHEAD presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming prefill layer readahead. ds4.c:18444 +runtime/metal DS4_METAL_DISABLE_STREAMING_PREFILL_SELECTED_ASYNC_LOAD presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming prefill selected async load. ds4.c:46424 +runtime/metal DS4_METAL_DISABLE_STREAMING_PREFILL_SELECTED_MADVISE presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming prefill selected madvise. ds4.c:18424 +runtime/metal DS4_METAL_DISABLE_STREAMING_PREFILL_SELECTED_PAGEIN presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming prefill selected pagein. ds4.c:18414 +runtime/metal DS4_METAL_DISABLE_STREAMING_PREFILL_SELECTED_PROFILE presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming prefill selected profile. ds4.c:18908 +runtime/metal DS4_METAL_DISABLE_STREAMING_PREFILL_SELECTED_READAHEAD presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming prefill selected readahead. ds4.c:19960 +runtime/metal DS4_METAL_DISABLE_STREAMING_PREFILL_SELECTED_READAHEAD_SHARED presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming prefill selected readahead shared. ds4.c:19970 +runtime/metal DS4_METAL_DISABLE_STREAMING_READAHEAD presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming readahead. ds4.c:18205 +runtime/metal DS4_METAL_DISABLE_STREAMING_SELECTED_ASYNC_EARLY_COMMIT presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming selected async early commit. ds4.c:21019 +runtime/metal DS4_METAL_DISABLE_STREAMING_SELECTED_ASYNC_LOAD presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming selected async load. ds4.c:21003 +runtime/metal DS4_METAL_DISABLE_STREAMING_SELECTED_READAHEAD_SHARED_DELAY presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming selected readahead shared delay. ds4.c:21720 +runtime/metal DS4_METAL_DISABLE_STREAMING_SELECTED_SHARED_OVERLAP presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming selected shared overlap. ds4.c:20995 +runtime/metal DS4_METAL_DISABLE_STREAMING_STATIC_DECODE_MAP presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming static decode map. ds4.c:18217 +runtime/metal DS4_METAL_DISABLE_STREAMING_STATIC_MAP_STATE_CACHE presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming static map state cache. ds4.c:18230 +runtime/metal DS4_METAL_DISABLE_SUPPORT_Q8_DECODE_EXACT_VIEWS presence rollback; unset: automatic/default path; any value including 0 disables Disables support Q8 decode exact views. ds4_metal.m:12858 +runtime/metal DS4_METAL_DISABLE_TINY_PAIR_SWIGLU_FUSION presence rollback; unset: automatic/default path; any value including 0 disables Disables tiny pair SwiGLU fusion. ds4_metal.m:44920 +runtime/metal DS4_METAL_DISABLE_TOKEN_EMBED_EXACT_VIEW presence rollback; unset: automatic/default path; any value including 0 disables Disables token embed exact view. ds4_metal.m:11907 +runtime/metal DS4_METAL_DISABLE_ZERO_PREFIX_PREFILL_MASK_CACHE presence rollback; unset: automatic/default path; any value including 0 disables Disables zero prefix prefill mask cache. ds4_metal.m:2404 +runtime/metal DS4_METAL_DSPARK_ACCEPTANCE_ONLY_VERIFY nonempty boolean; unset/empty or exact 0: off; every other value: on Verifies only the draft rows still needed for acceptance after the base target logit. ds4.c:52499 +runtime/metal DS4_METAL_DSPARK_DEVICE_PROPOSER boolean true values enable; false values/unset disable; NO_DEVICE_PROPOSER presence dominates Keeps DSpark Q8 confidence/Markov proposal work on Metal and reads one compact result. ds4.c:34884 +runtime/metal DS4_METAL_DSPARK_EXACT2 nonempty boolean; unset/empty or exact 0: off; every other value: on Enables the resident single-GPU Metal exact-2 verifier. ds4.c:52290 +runtime/metal DS4_METAL_DSPARK_EXACTN nonempty boolean; unset/empty or exact 0: off; every other value: on Enables the single-GPU Metal exact-N verifier. ds4.c:52311 +runtime/metal DS4_METAL_DSPARK_EXACTN_BATCH_HEAD nonempty boolean; unset/empty or exact 0: off; every other value: on Batches the output head across exact-N verifier rows. ds4.c:37726 +runtime/metal DS4_METAL_DSPARK_EXACTN_UNION nonempty boolean; unset/empty or exact 0: off; every other value: on Loads the union of experts for exact-N verifier rows once. ds4.c:52332 +runtime/metal DS4_METAL_DSPARK_EXACT_ROWS_ASYNC_TAILS presence control; unset: off/default; any value including 0 enables Runs exact-row routed tails asynchronously after union routing. ds4.c:37932 +runtime/metal DS4_METAL_DSPARK_EXACT_ROWS_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for DSpark exact rows. ds4.c:37930 +runtime/metal DS4_METAL_DSPARK_HEADLESS_REPLAY unset/empty: enabled; exact 0 disables; every other nonempty value enables Skips output heads for accepted intermediate DSpark replay tokens. ds4.c:52479 +runtime/metal DS4_METAL_DSPARK_NO_DEVICE_PROPOSER presence rollback; unset: automatic/default path; any value including 0 disables Dominant presence-based rollback for the Metal DSpark device proposer. ds4.c:34886 +runtime/metal DS4_METAL_DSPARK_PIN_MAIN_PROJ nonempty value other than exact 0 enables; unset/empty/0 disables mlock-pins only DSpark stage-0 main_norm/main_proj in Metal SSD streaming. ds4.c:39443 +runtime/metal DS4_METAL_DSPARK_PROPOSER_BLOCK_MAX uint32; unset: automatic cache/verifier cap; 0 or invalid: native width; positive: clamped to DSpark/native maximum Caps rows proposed by single-device Metal DSpark. ds4.c:52430 +runtime/metal DS4_METAL_DSPARK_SAFE_EXPERT_COUNT exact 1 enables; unset or any other value disables Caps an explicit expert-count cache request to the safe Metal working-set budget for DSpark SSD streaming. ds4.c:4880 +runtime/metal DS4_METAL_DSV4_HC_SOURCE file path; unset/empty: use the in-tree Metal source file Overrides the DeepSeek hidden-context Metal kernel source file loaded at runtime. ds4_metal.m:4938 +runtime/metal DS4_METAL_DSV4_KV_SOURCE file path; unset/empty: use the in-tree Metal source file Overrides the DeepSeek KV Metal kernel source file loaded at runtime. ds4_metal.m:4940 +runtime/metal DS4_METAL_DSV4_MISC_SOURCE file path; unset/empty: use the in-tree Metal source file Overrides the DeepSeek miscellaneous Metal kernel source file loaded at runtime. ds4_metal.m:4942 +runtime/metal DS4_METAL_DSV4_ROPE_SOURCE file path; unset/empty: use the in-tree Metal source file Overrides the DeepSeek RoPE Metal kernel source file loaded at runtime. ds4_metal.m:4941 +runtime/metal DS4_METAL_DUMP_PREFILL_LOGITS file path; unset/empty: no dump Writes final GPU prefill logits as f32 binary. ds4.c:51347 +runtime/metal DS4_METAL_ENABLE_BATCH_HC_NORM_FUSION legacy value-aware alias; default enabled; exact 0 disables; unset/empty/every other value enables unless DISABLE is active Legacy control for the now-default batched HC norm fusion. ds4.c:20462 +runtime/metal DS4_METAL_ENABLE_COMPRESSOR_EXACT_POOL_RATIO4 presence opt-in; unset: off/automatic; any value including 0 enables Enables compressor exact pool ratio4. ds4_metal.m:26385 +runtime/metal DS4_METAL_ENABLE_COMPRESSOR_PAIR_STATE_STORE presence opt-in; unset: off/automatic; any value including 0 enables Enables compressor pair state store. ds4_metal.m:23243 +runtime/metal DS4_METAL_ENABLE_COMPRESSOR_QUAD_STORE presence opt-in; unset: off/automatic; any value including 0 enables Enables compressor quad store. ds4.c:23527 +runtime/metal DS4_METAL_ENABLE_DSPARK_CAPTURE_FUSED_LAST nonempty boolean; unset/empty or exact 0: off; every other value: on Enables DSpark capture fused last. ds4.c:28359 +runtime/metal DS4_METAL_ENABLE_GATHERED_KV_STAGE presence opt-in; unset: off/automatic; any value including 0 enables Enables gathered KV stage. ds4_metal.m:29675 +runtime/metal DS4_METAL_ENABLE_GLM_STREAMING_SELECTED_ASYNC_LOAD presence opt-in; unset: off/automatic; any value including 0 enables Enables GLM streaming selected async load. ds4.c:44335 +runtime/metal DS4_METAL_ENABLE_HC_NORM_MIX_FUSE presence opt-in; unset: off/automatic; any value including 0 enables Enables HC norm mix fuse. ds4.c:22870 +runtime/metal DS4_METAL_ENABLE_HC_PRODUCER_PRE_NORM_FUSE presence opt-in; unset: off/automatic; any value including 0 enables Enables HC producer pre norm fuse. ds4_metal.m:46552 +runtime/metal DS4_METAL_ENABLE_IQ2_SELECTED_ASYNC_EARLY_COMMIT presence opt-in; unset: off/automatic; any value including 0 enables Enables IQ2 selected async early commit. ds4.c:21018 +runtime/metal DS4_METAL_ENABLE_IQ2_XXS_SSD_PREFILL_MM value-aware boolean; default automatic/on for eligible shape; explicit 0 turns request off unless REQUIRE=1 Overrides automatic IQ2_XXS/Q2_K grouped address-MM selection for SSD prefill. ds4_metal.m:44780 +runtime/metal DS4_METAL_ENABLE_PRO_Q4_EXPERT_ADDRESS_AUTO presence opt-in; unset: off/automatic; any value including 0 enables Enables pro Q4 expert address auto. ds4.c:20982 +runtime/metal DS4_METAL_ENABLE_PRO_Q4_EXPERT_TABLE_AUTO presence opt-in; unset: off/automatic; any value including 0 enables Enables pro Q4 expert table auto. ds4.c:20981 +runtime/metal DS4_METAL_ENABLE_PRO_Q4_SELECTED_EXPERT_VIEWS presence opt-in; unset: off/automatic; any value including 0 enables Enables pro Q4 selected expert views. ds4.c:20978 +runtime/metal DS4_METAL_ENABLE_Q4_ATTN_OUT_TINY_BATCH value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Enables Q4 attn out tiny batch. ds4_metal.m:28405 +runtime/metal DS4_METAL_ENABLE_Q4_BATCH_EXPERT_TABLE presence opt-in; unset: off/automatic; any value including 0 enables Enables Q4 batch expert table. ds4_metal.m:44869 +runtime/metal DS4_METAL_ENABLE_Q4_EXACT_TENSOR_ID presence opt-in; unset: off/automatic; any value including 0 enables Enables Q4 exact tensor ID. ds4_metal.m:42218 +runtime/metal DS4_METAL_ENABLE_Q4_EXPERT_ADDRESS_TABLE presence opt-in; unset: off/automatic; any value including 0 enables Enables Q4 expert address table. ds4.c:20980 +runtime/metal DS4_METAL_ENABLE_Q4_EXPERT_TABLE presence opt-in; unset: off/automatic; any value including 0 enables Enables Q4 expert table. ds4.c:20979 +runtime/metal DS4_METAL_ENABLE_Q4_GATHER_SLOTS presence opt-in; unset: off/automatic; any value including 0 enables Enables Q4 gather slots. ds4_metal.m:42320 +runtime/metal DS4_METAL_ENABLE_Q4_GROUP24_EXPERT_TABLE presence opt-in; unset: off/automatic; any value including 0 enables Enables Q4 group24 expert table. ds4_metal.m:42200 +runtime/metal DS4_METAL_ENABLE_Q4_GROUP6_EXPERT_TABLE presence opt-in; unset: off/automatic; any value including 0 enables Enables Q4 group6 expert table. ds4_metal.m:42166 +runtime/metal DS4_METAL_ENABLE_Q4_GROUP8_EXPERT_TABLE presence opt-in; unset: off/automatic; any value including 0 enables Enables Q4 group8 expert table. ds4_metal.m:42183 +runtime/metal DS4_METAL_ENABLE_Q4_GROUPED_EXPERTS presence opt-in; unset: off/automatic; any value including 0 enables Enables Q4 grouped experts. ds4_metal.m:42129 +runtime/metal DS4_METAL_ENABLE_Q4_QKV_COMPRESSOR_FUSE presence opt-in; unset: off/automatic; any value including 0 enables Enables Q4 QKV compressor fuse. ds4.c:23140 +runtime/metal DS4_METAL_ENABLE_Q4_SELECTED_EXPERT_VIEWS presence opt-in; unset: off/automatic; any value including 0 enables Enables Q4 selected expert views. ds4.c:20977 +runtime/metal DS4_METAL_ENABLE_Q4_SSD_PREFILL_ATTN_OUT_EXACTN value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Enables Q4 SSD prefill attn out exactn. ds4_metal.m:28097 +runtime/metal DS4_METAL_ENABLE_Q4_SSD_SESSION_UNION nonempty boolean; unset/empty or exact 0: off; every other value: on Enables Q4 SSD session union. ds4.c:65171 +runtime/metal DS4_METAL_ENABLE_Q4_STREAM_OVERLAP nonempty boolean; unset/empty or exact 0: off; every other value: on Enables Q4 stream overlap. ds4.c:65080 +runtime/metal DS4_METAL_ENABLE_Q8_DECODE_EXACT_VIEWS presence opt-in; unset: off/automatic; any value including 0 enables Enables Q8 decode exact views. ds4_metal.m:12865 +runtime/metal DS4_METAL_ENABLE_Q8_QKV_COMPRESSOR_FUSE nonempty boolean; unset/empty/exact 0: no streamed/union opt-in; other values enable; eligible resident full-decode remains automatic Extends the automatic resident Q8 QKV/compressor compound fusion to SSD streaming or exact-N union scope. ds4.c:23027 +runtime/metal DS4_METAL_ENABLE_STREAMING_COMPACT_ADDR presence opt-in; unset: off/automatic; any value including 0 enables Enables streaming compact address. ds4_metal.m:14597 +runtime/metal DS4_METAL_ENABLE_STREAMING_EXPERT_ADDR_TABLE presence opt-in; unset: off/automatic; any value including 0 enables Enables streaming expert address table. ds4_metal.m:14604 +runtime/metal DS4_METAL_ENABLE_STREAMING_EXPERT_EVICT_DONTNEED presence opt-in; unset: off/automatic; any value including 0 enables Enables streaming expert evict dontneed. ds4_metal.m:14394 +runtime/metal DS4_METAL_ENABLE_STREAMING_EXPERT_HIT_VALIDATOR presence opt-in; unset: off/automatic; any value including 0 enables Enables streaming expert hit validator. ds4_metal.m:14605 +runtime/metal DS4_METAL_ENABLE_STREAMING_EXPERT_LIVE_INDEX value-aware boolean; default automatic/on for validated IQ2 cache shape; explicit 0 disables Overrides automatic dense live-entry index selection. ds4_metal.m:15442 +runtime/metal DS4_METAL_ENABLE_STREAMING_EXPERT_MASKED_ADDR presence opt-in; unset: off/automatic; any value including 0 enables Enables streaming expert masked address. ds4_metal.m:14606 +runtime/metal DS4_METAL_ENABLE_STREAMING_FULL_EXPERT_ADDR_TABLE presence opt-in; unset: off/automatic; any value including 0 enables Enables streaming full expert address table. ds4_metal.m:14715 +runtime/metal DS4_METAL_ENABLE_STREAMING_IQ2_CPU_ROUTER presence opt-in; unset: off/automatic; any value including 0 enables Enables streaming IQ2 CPU router. ds4.c:20938 +runtime/metal DS4_METAL_ENABLE_STREAMING_MADVISE_WILLNEED presence opt-in; unset: off/automatic; any value including 0 enables Enables streaming madvise willneed. ds4.c:18210 +runtime/metal DS4_METAL_ENABLE_STREAMING_PREFILL_BATCH_SELECTED_ADDR presence opt-in; unset: off/automatic; any value including 0 enables Enables streaming prefill batch selected address. ds4_metal.m:14609 +runtime/metal DS4_METAL_ENABLE_STREAMING_PREFILL_CACHE_SEED presence opt-in; unset: off/automatic; any value including 0 enables Enables streaming prefill cache seed. ds4.c:21259 +runtime/metal DS4_METAL_ENABLE_STREAMING_PREFILL_EXPERT_READAHEAD value-aware boolean; for batches <32 readahead is automatic; for batches >=32 unset/false disables and true enables; global READHEAD rollback and F_NOCACHE still dominate Restores F_RDADVISE immediately before parallel pread for large SSD-prefill batches. ds4_metal.m:13212 +runtime/metal DS4_METAL_ENABLE_STREAMING_PREFILL_LAYER_PAGEIN presence opt-in; unset: off/automatic; any value including 0 enables Enables streaming prefill layer pagein. ds4.c:18432 +runtime/metal DS4_METAL_ENABLE_STREAMING_PREFILL_LAYER_READAHEAD presence opt-in; unset: off/automatic; any value including 0 enables Enables streaming prefill layer readahead. ds4.c:18442 +runtime/metal DS4_METAL_ENABLE_STREAMING_PREFILL_SELECTED_MADVISE presence opt-in; unset: off/automatic; any value including 0 enables Enables streaming prefill selected madvise. ds4.c:18422 +runtime/metal DS4_METAL_ENABLE_STREAMING_PREFILL_SELECTED_PAGEIN presence opt-in; unset: off/automatic; any value including 0 enables Enables streaming prefill selected pagein. ds4.c:18412 +runtime/metal DS4_METAL_ENABLE_STREAMING_PREFILL_SELECTED_READAHEAD presence opt-in; unset: off/automatic; any value including 0 enables Enables streaming prefill selected readahead. ds4.c:19956 +runtime/metal DS4_METAL_ENABLE_STREAMING_PREFILL_SELECTED_READAHEAD_SHARED presence opt-in; unset: off/automatic; any value including 0 enables Enables streaming prefill selected readahead shared. ds4.c:19958 +runtime/metal DS4_METAL_ENABLE_STREAMING_READAHEAD presence opt-in; unset: off/automatic; any value including 0 enables Enables streaming readahead. ds4.c:18203 +runtime/metal DS4_METAL_ENABLE_STREAMING_SELECTED_READAHEAD_SHARED_DELAY presence opt-in; unset: off/automatic; any value including 0 enables Enables streaming selected readahead shared delay. ds4.c:21719 +runtime/metal DS4_METAL_ENABLE_STREAMING_STATIC_DECODE_MAP presence opt-in; unset: off/automatic; any value including 0 enables Enables streaming static decode map. ds4.c:18222 +runtime/metal DS4_METAL_ENABLE_TOKEN_EMBED_EXACT_VIEW presence opt-in; unset: off/automatic; any value including 0 enables Enables token embed exact view. ds4_metal.m:12219 +runtime/metal DS4_METAL_EXACT_VIEW_CACHE_GIB unsigned GiB; default 64; 0 disables size-triggered eviction; MIB overrides it Sets the cached exact-model-view eviction threshold. ds4_metal.m:1333 +runtime/metal DS4_METAL_EXACT_VIEW_CACHE_MIB unsigned MiB; unset: inherit GIB/default; 0 disables size-triggered eviction; overrides GIB Sets the cached exact-model-view eviction threshold with MiB precision. ds4_metal.m:1342 +runtime/metal DS4_METAL_EXACT_VIEW_CACHE_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for exact view cache. ds4_metal.m:1377 +runtime/metal DS4_METAL_FLASH_ATTN_SOURCE file path; unset/empty: use the in-tree Metal source file Overrides the FlashAttention Metal kernel source file loaded at runtime. ds4_metal.m:4935 +runtime/metal DS4_METAL_FLASH_ATTN_STAGE_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for flash attn stage. ds4.c:65152 +runtime/metal DS4_METAL_FLASH_ATTN_STAGE_PROFILE_FILTER substring; unset/empty: all profiled modes/stages Filters FlashAttention stage-profile output by mode or stage substring. ds4_metal.m:11178 +runtime/metal DS4_METAL_GET_ROWS_SOURCE file path; unset/empty: use the in-tree Metal source file Overrides the get-rows Metal kernel source file loaded at runtime. ds4_metal.m:4946 +runtime/metal DS4_METAL_GLM_DISABLE_STREAMING_EXPERT_CACHE presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming expert cache for GLM. ds4_metal.m:39667 +runtime/metal DS4_METAL_GLM_DISABLE_STREAMING_GROUPED_ADDR_PREFILL presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming grouped address prefill for GLM. ds4_metal.m:41081 +runtime/metal DS4_METAL_GLM_DISABLE_STREAMING_SEED_BEFORE_PREFILL presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming seed before prefill for GLM. ds4.c:51074 +runtime/metal DS4_METAL_GLM_DISABLE_STREAMING_TOKEN_PREFILL presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming token prefill for GLM. ds4.c:49704 +runtime/metal DS4_METAL_GLM_MOE_ONE_STAGE_PROFILE unset: off; 1/true/yes/on/all enables all layers; accepts layer lists/ranges; 0/false/no/off disables Prints timing/profile diagnostics for GLM MoE one stage. ds4_metal.m:39964 +runtime/metal DS4_METAL_GLM_MOE_ONE_STAGE_PROFILE_LAYER layer index/list/ranges or all; unset: all layers selected by profiler Restricts GLM one-stage MoE profiling to selected layers. ds4_metal.m:39965 +runtime/metal DS4_METAL_GLM_MOE_STAGE_PROFILE_FILTER substring; unset/empty: all profiled stages Filters GLM MoE stage-profile output. ds4_metal.m:39968 +runtime/metal DS4_METAL_GLM_QKLOW_DEBUG presence diagnostic; unset: off; any value including 0 enables Enables debug diagnostics for GLM qklow. ds4_metal.m:37868 +runtime/metal DS4_METAL_GLM_STREAMING_ASYNC_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for GLM streaming async. ds4.c:44362 +runtime/metal DS4_METAL_GLM_STREAMING_DECODE_FULL_LAYER_MAP presence control; unset: off/default; any value including 0 enables Maps complete GLM layers during SSD-streaming decode instead of decode-only spans. ds4.c:42616 +runtime/metal DS4_METAL_GLM_STREAMING_DECODE_SYNC_EACH_LAYER boolean text; Metal runtime always synchronizes and does not consult it; legacy fallback name read only in ROCm builds Controls per-layer GLM streaming decode synchronization only as a legacy ROCm fallback alias. ds4.c:49780 +runtime/metal DS4_METAL_GLM_STREAMING_PREFILL_FULL_LAYER presence control; unset: off/default; any value including 0 enables Forces full-layer GLM SSD prefill regardless of the token crossover. ds4_metal.m:14710 +runtime/metal DS4_METAL_GLM_STREAMING_PREFILL_FULL_LAYER_MIN_TOKENS positive uint32; default 64 on Metal, 1024 when used as ROCm fallback; 0/invalid restores default Sets the token crossover for GLM full-layer SSD prefill. ds4.c:42693 +runtime/metal DS4_METAL_GLM_STREAMING_PREFILL_SYNC_EACH_LAYER boolean text; Metal runtime always synchronizes and does not consult it; legacy fallback name read only in ROCm builds Controls per-layer GLM streaming prefill synchronization only as a legacy ROCm fallback alias. ds4.c:42396 +runtime/metal DS4_METAL_GLM_STREAMING_TOKEN_PREFILL_MAX uint32; default 64 on Metal, 0 when used as ROCm fallback; 0 disables; invalid restores default Sets largest GLM SSD prefill handled token-major by the decode graph. ds4.c:49683 +runtime/metal DS4_METAL_GLU_SOURCE file path; unset/empty: use the in-tree Metal source file Overrides the GLU Metal kernel source file loaded at runtime. ds4_metal.m:4950 +runtime/metal DS4_METAL_GPU_BATCH_EMBED_MIN uint32 token threshold; default 512; invalid restores default Sets the batch size at which prompt embedding moves from CPU upload to Metal kernels. ds4.c:28867 +runtime/metal DS4_METAL_GPU_BUSY_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for GPU busy. ds4_metal.m:1293 +runtime/metal DS4_METAL_GRAPH_DUMP_LAYER unsigned layer index or all; unset: every layer Restricts graph tensor dumps to one layer. ds4.c:16863 +runtime/metal DS4_METAL_GRAPH_DUMP_LOGITS file path; unset/empty: no graph-logit dump Writes Metal graph-test logits as f32 binary. ds4.c:39080 +runtime/metal DS4_METAL_GRAPH_DUMP_NAME substring; unset/empty: every tensor name Restricts graph tensor dumps by tensor-name substring. ds4.c:16859 +runtime/metal DS4_METAL_GRAPH_DUMP_POS unsigned token position; unset: every position Restricts graph tensor dumps to one token position. ds4.c:16870 +runtime/metal DS4_METAL_GRAPH_DUMP_PREFIX path/prefix; unset/empty: tensor dumping disabled Enables graph tensor dumps and supplies the filename prefix. ds4.c:65148 +runtime/metal DS4_METAL_GRAPH_DUMP_TRACE presence diagnostic; unset: off; any value including 0 enables Emits trace diagnostics for graph dump. ds4.c:16900 +runtime/metal DS4_METAL_GRAPH_OUTPUT_ROW zero-based row smaller than current batch; default final row; invalid restores final row Chooses which prefill output row is projected to logits. ds4.c:35846 +runtime/metal DS4_METAL_GRAPH_PREFILL_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for graph prefill. ds4.c:35523 +runtime/metal DS4_METAL_GRAPH_PREFILL_SPLIT_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for graph prefill split. ds4.c:66204 +runtime/metal DS4_METAL_GRAPH_PROMPT_TOKENS integer 1..prompt length; default full prompt Limits prompt length used by the Metal graph parity test. ds4.c:39023 +runtime/metal DS4_METAL_GRAPH_RAW_CAP positive rows; default from SWA window+prefill; clamped to [raw_window,min(ctx,8192)] Overrides raw sliding-window KV ring capacity. ds4.c:38390 +runtime/metal DS4_METAL_GRAPH_TEACHER_FORCE presence control; unset: off/default; any value including 0 enables Feeds CPU reference state back into the first-token graph trace at each layer. ds4.c:27742 +runtime/metal DS4_METAL_GRAPH_TOKEN_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for graph token. ds4.c:32382 +runtime/metal DS4_METAL_GRAPH_TOKEN_SECOND_SPLIT_LAYERS integer 0..layer count; default 0 plus eligible automatic pre-M5 schedules; explicit value wins Overrides second command-buffer split layer for token decode. ds4.c:28029 +runtime/metal DS4_METAL_GRAPH_TOKEN_SPLIT_LAYERS integer 0..layer count; default 4 on Apple and 0 elsewhere, with eligible pre-M5 adaptive override Overrides first command-buffer split layer for token decode. ds4.c:27915 +runtime/metal DS4_METAL_GRAPH_TRACE_CACHE presence control; unset: off/default; any value including 0 enables Prints raw KV cache parity diagnostics in the graph prompt test. ds4.c:39092 +runtime/metal DS4_METAL_GRAPH_TRACE_COMP presence control; unset: off/default; any value including 0 enables Prints compressed-cache parity diagnostics in the graph prompt test. ds4.c:39093 +runtime/metal DS4_METAL_GRAPH_TRACE_LAYERS presence control; unset: off/default; any value including 0 enables Enables per-layer first-token CPU/GPU graph tracing. ds4.c:27739 +runtime/metal DS4_METAL_GRAPH_TRACE_STAGE_LAYER signed layer index; unset gives -1/no stage-layer selection Selects the layer used by first-token stage tracing. ds4.c:27743 +runtime/metal DS4_METAL_HC_NORM_FUSION_CHECK nonempty boolean; unset/empty or exact 0: off; every other value: on Compares fused HC normalization against the reference result. ds4.c:20521 +runtime/metal DS4_METAL_HC_NORM_FUSION_CHECK_TOL positive finite float; default 2e-4; invalid/nonpositive restores default Sets the numerical tolerance for the HC norm-fusion oracle. ds4.c:20530 +runtime/metal DS4_METAL_HC_STABLE boolean empty/1/true/yes/on vs 0/false/no/off; default on Compiles stable hidden-context drift arithmetic into the Metal library. ds4_metal.m:7047 +runtime/metal DS4_METAL_INDEXER_STAGE_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for indexer stage. ds4.c:17569 +runtime/metal DS4_METAL_IQ2_XXS_SSD_PREFILL_MM_STATS value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Collects/prints statistics for IQ2 XXS SSD prefill MM. ds4_metal.m:6980 +runtime/metal DS4_METAL_KV_RAW_F32 boolean empty/1/true/yes/on vs 0/false/no/off; default off Compiles raw KV storage as F32 for drift diagnosis. ds4_metal.m:7049 +runtime/metal DS4_METAL_LAYER_STAGE_PROFILE unset: off; 1/true/yes/on/all enables all layers; a layer index selects one; 0/false/no/off disables Prints timing/profile diagnostics for layer stage. ds4.c:65150 +runtime/metal DS4_METAL_LAYER_STAGE_PROFILE_LAYER single unsigned layer index; unset/empty: all layers enabled by the parent profile; invalid matches no layer Restricts the corresponding shared graph stage profiler to one layer. ds4.c:29109 +runtime/metal DS4_METAL_MATH_SAFE boolean empty/1/true/yes/on vs 0/false/no/off; default off Compiles Metal shaders with strict/safe IEEE math instead of fast math. ds4_metal.m:7051 +runtime/metal DS4_METAL_MEMORY_REPORT presence control; unset: off/default; any value including 0 enables Prints Metal allocation/cache/residency memory reports. ds4.c:39047 +runtime/metal DS4_METAL_MODEL_UNTRACKED presence control; unset: off/default; any value including 0 enables Creates mapped model buffers with untracked Metal hazard tracking. ds4_metal.m:1547 +runtime/metal DS4_METAL_MODEL_VIEW_MAX_GIB positive integer GiB; default device maximum (128-GiB cap for already-split span maps); cannot exceed device maximum Caps each no-copy mapped Metal model view. ds4_metal.m:2216 +runtime/metal DS4_METAL_MODEL_WARMUP_STRIDE_KB integer 1..1048576 KiB, at least one page; unset inherits MB/default; overrides STRIDE_MB Sets the model-view warmup touch stride with KiB precision. ds4_metal.m:3057 +runtime/metal DS4_METAL_MODEL_WARMUP_STRIDE_MB integer 1..1024 MiB; default 1 MiB; STRIDE_KB overrides Sets the model-view warmup touch stride. ds4_metal.m:3049 +runtime/metal DS4_METAL_MOE_MM_ID_USE_RESOURCES presence control; unset: off/default; any value including 0 enables Declares MM-ID MoE resource usage explicitly on the command encoder. ds4_metal.m:35168 +runtime/metal DS4_METAL_MOE_ONE_STAGE_PROFILE unset: off; 1/true/yes/on/all enables all layers; accepts layer lists/ranges; 0/false/no/off disables Prints timing/profile diagnostics for MoE one stage. ds4.c:22714 +runtime/metal DS4_METAL_MOE_ONE_STAGE_PROFILE_LAYER layer index/list/ranges or all; unset: all profiler-selected layers Restricts one-stage MoE profiling to selected layers. ds4_metal.m:43436 +runtime/metal DS4_METAL_MOE_SOURCE file path; unset/empty: use the in-tree Metal source file Overrides the MoE Metal kernel source file loaded at runtime. ds4_metal.m:4937 +runtime/metal DS4_METAL_MOE_STAGE_PROFILE unset: off; 1/true/yes/on/all enables all layers; accepts layer lists/ranges; 0/false/no/off disables Prints timing/profile diagnostics for MoE stage. ds4.c:65154 +runtime/metal DS4_METAL_MOE_STAGE_PROFILE_FILTER substring; unset/empty: all profiled stages Filters MoE stage-profile output. ds4_metal.m:43438 +runtime/metal DS4_METAL_MOE_STAGE_PROFILE_LAYER layer index/list/ranges or all; unset: all profiler-selected layers Restricts batched MoE stage profiling to selected layers. ds4_metal.m:45346 +runtime/metal DS4_METAL_MOE_WRITE_CLAMPED_ACT presence control; unset: off/default; any value including 0 enables Makes routed MoE write the clamped activation diagnostic. ds4.c:18527 +runtime/metal DS4_METAL_NORM_RSQRT_DISABLE boolean empty/1/true/yes/on vs 0/false/no/off; default on Compiles unified normalization-rsqrt arithmetic into the Metal library. ds4_metal.m:7048 +runtime/metal DS4_METAL_NORM_SOURCE file path; unset/empty: use the in-tree Metal source file Overrides the normalization Metal kernel source file loaded at runtime. ds4_metal.m:4951 +runtime/metal DS4_METAL_NO_MODEL_WARMUP presence rollback; unset: automatic/default path; any value including 0 disables Disables model warmup. ds4_metal.m:2309 +runtime/metal DS4_METAL_NO_PREFILL_KERNEL_WARMUP presence rollback; unset: automatic/default path; any value including 0 disables Disables prefill kernel warmup. ds4.c:28965 +runtime/metal DS4_METAL_NO_RESIDENCY presence rollback; unset: automatic/default path; any value including 0 disables Disables residency. ds4_metal.m:2092 +runtime/metal DS4_METAL_OUTPUT_STAGE_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for output stage. ds4.c:17570 +runtime/metal DS4_METAL_PREFILL_CHUNK positive token count used only when CLI chunk is absent; default full prompt, or 4096 for long non-PRO and 8192 for long PRO prompts; <=0 keeps automatic/full prompt Provides the historical environment fallback for prefill chunk size. ds4.c:12802 +runtime/metal DS4_METAL_PRO_Q4_CPU_ROUTER nonempty boolean; unset/empty or exact 0: off; every other value: on Uses the CPU router for PRO Q4 selected-expert decode. ds4.c:20934 +runtime/metal DS4_METAL_PRO_Q4_CPU_ROUTER_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for pro Q4 CPU router. ds4.c:21644 +runtime/metal DS4_METAL_Q4_ADDR_USE_RESOURCES presence control; unset: off/default; any value including 0 enables Declares Q4 address-table resources explicitly on encoders. ds4_metal.m:20259 +runtime/metal DS4_METAL_Q4_EXPERT_GROUP_SIZE positive uint32; default 32; clamped to total expert count Sets experts processed per grouped Q4 dispatch. ds4_metal.m:34088 +runtime/metal DS4_METAL_Q4_EXPERT_TABLE_GROUP_SIZE integer 2..total experts; default/invalid 1 (ungrouped) Sets grouped exact-view width while building Q4 expert tables. ds4_metal.m:19604 +runtime/metal DS4_METAL_Q4_EXPERT_TABLE_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for Q4 expert table. ds4_metal.m:20103 +runtime/metal DS4_METAL_Q4_GROUP24_BASE_VIEWS presence control; unset: off/default; any value including 0 enables Uses broad base model views for Q4 group-24 instead of exact views. ds4_metal.m:42205 +runtime/metal DS4_METAL_Q4_GROUP24_EXACT_VIEWS presence control; unset: off/default; any value including 0 enables Uses exact mapped views for Q4 group-24 experts. ds4_metal.m:42204 +runtime/metal DS4_METAL_Q4_GROUPED_CACHE_VIEWS presence control; unset: off/default; any value including 0 enables Caches exact Q4 grouped expert views. ds4_metal.m:42151 +runtime/metal DS4_METAL_Q4_PRO_MAP_GROUPS positive divisor of 384 in 1..384; default/invalid 1 Splits each 384-expert PRO Q4 tensor into this many mapped views. ds4.c:6276 +runtime/metal DS4_METAL_Q4_SELECTED_EXACT_VIEWS presence control; unset: off/default; any value including 0 enables Forces exact/cached views for selected Q4 experts instead of base views. ds4_metal.m:42528 +runtime/metal DS4_METAL_Q4_SELECTED_OVERLAP_SHARED nonempty boolean; unset/empty or exact 0: off; every other value: on Overlaps selected Q4 expert preparation with the shared expert. ds4.c:20962 +runtime/metal DS4_METAL_Q4_SELECTED_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for Q4 selected. ds4.c:37944 +runtime/metal DS4_METAL_Q4_SELECTED_PROFILE_LAYER single nonnegative layer index; unset: every layer Restricts legacy Q4 selected-expert profiling to one layer. ds4_metal.m:42509 +runtime/metal DS4_METAL_Q4_SELECTED_SHARED_EVENT presence control; unset: off/default; any value including 0 enables Coordinates selected Q4 work with a shared Metal event. ds4_metal.m:42524 +runtime/metal DS4_METAL_Q4_SELECTED_TRANSIENT_VIEWS presence control; unset: off/default; any value including 0 enables Uses transient exact views for selected Q4 experts. ds4_metal.m:42532 +runtime/metal DS4_METAL_Q4_SELECTED_USE_BASE_VIEWS presence control; unset: off/default; any value including 0 enables Uses broad base model views for selected Q4 experts. ds4_metal.m:42527 +runtime/metal DS4_METAL_Q4_TABLE_BIND_ANCHORS presence control; unset: off/default; any value including 0 enables Binds anchor buffers alongside the Q4 expert address table. ds4_metal.m:19716 +runtime/metal DS4_METAL_Q4_TABLE_MODEL_RESIDENCY_SET presence control; unset: off/default; any value including 0 enables Adds Q4 expert table allocations to the model residency set. ds4_metal.m:19657 +runtime/metal DS4_METAL_Q4_TABLE_PER_TENSOR_RESIDENCY_SET presence control; unset: off/default; any value including 0 enables Builds separate residency sets per Q4 expert tensor. ds4_metal.m:19760 +runtime/metal DS4_METAL_Q4_TABLE_QUEUE_RESIDENCY_SET presence control; unset: off/default; any value including 0 enables Attaches Q4 expert table residency sets to command queues. ds4_metal.m:19615 +runtime/metal DS4_METAL_Q4_TABLE_RESIDENCY_SET presence control; unset: off/default; any value including 0 enables Enables Q4 expert table residency-set handling. ds4_metal.m:19759 +runtime/metal DS4_METAL_Q4_TABLE_USE_RESOURCES presence control; unset: off/default; any value including 0 enables Declares Q4 table resources explicitly on encoders. ds4_metal.m:20258 +runtime/metal DS4_METAL_Q8_DECODE_EXACT_VIEW_MAX_MIB integer 1..4096 MiB; default 1024; above max clamps, below min/invalid restores default Caps weight ranges eligible for Q8 exact model views. ds4_metal.m:12876 +runtime/metal DS4_METAL_Q8_MV_EXT_MAX_TOKENS integer 2..128; default 16; above max clamps, below min/invalid restores default Sets largest batch handled by extended Q8 matvec. ds4_metal.m:21126 +runtime/metal DS4_METAL_Q8_MV_NSG integer 1..8 simdgroups; default 4, or 2 with TP world=2; above max clamps, below min/invalid restores default Overrides simdgroups per Q8 matvec threadgroup. ds4.c:22730 +runtime/metal DS4_METAL_Q8_PREFILL_PROFILE value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Prints timing/profile diagnostics for Q8 prefill. ds4_metal.m:21280 +runtime/metal DS4_METAL_Q8_PREFILL_PROFILE_FILTER substring matched against generated operation label; unset/empty: all eligible calls Filters Q8 prefill profiling. ds4_metal.m:21295 +runtime/metal DS4_METAL_Q_STAGE_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for q stage. ds4.c:29270 +runtime/metal DS4_METAL_REPEAT_SOURCE file path; unset/empty: use the in-tree Metal source file Overrides the repeat Metal kernel source file loaded at runtime. ds4_metal.m:4949 +runtime/metal DS4_METAL_REQUIRE_COMPRESSOR_EXACT_POOL_RATIO4 presence strict check; unset: fallback allowed; any value including 0 requires the path Requires compressor exact pool ratio4 and makes eligible fallback fail closed. ds4_metal.m:26387 +runtime/metal DS4_METAL_REQUIRE_EXACT_ROWS_PERSISTENT_CACHE nonempty boolean; unset/empty or exact 0: off; every other value: on Requires exact rows persistent cache and makes eligible fallback fail closed. ds4_metal.m:13002 +runtime/metal DS4_METAL_REQUIRE_GATHERED_KV_STAGE presence strict check; unset: fallback allowed; any value including 0 requires the path Requires gathered KV stage and makes eligible fallback fail closed. ds4_metal.m:29680 +runtime/metal DS4_METAL_REQUIRE_IQ2_XXS_SSD_PREFILL_MM value-aware boolean; default implicit fail-closed only with complete selected-address domain; explicit 1 is strict, 0 permits fallback Makes eligible IQ2_XXS/Q2_K grouped SSD-prefill MM fail closed. ds4_metal.m:44782 +runtime/metal DS4_METAL_REQUIRE_M1_IQ2_MID_ONLY presence strict check; unset: fallback allowed; any value including 0 requires the path Requires M1 IQ2 mid only and makes eligible fallback fail closed. ds4_metal.m:42074 +runtime/metal DS4_METAL_REQUIRE_OUTPUT_HC_WEIGHTS4 presence strict check; unset: fallback allowed; any value including 0 requires the path Requires output HC weights4 and makes eligible fallback fail closed. ds4_metal.m:46789 +runtime/metal DS4_METAL_REQUIRE_Q4_ATTN_OUT_TINY_BATCH value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Requires Q4 attn out tiny batch and makes eligible fallback fail closed. ds4_metal.m:28373 +runtime/metal DS4_METAL_REQUIRE_Q4_SSD_PREFILL_ATTN_OUT_EXACTN value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Requires Q4 SSD prefill attn out exactn and makes eligible fallback fail closed. ds4_metal.m:28089 +runtime/metal DS4_METAL_REQUIRE_Q4_SSD_SESSION_UNION nonempty boolean; unset/empty or exact 0: off; every other value: on Requires Q4 SSD session union and makes eligible fallback fail closed. ds4.c:65164 +runtime/metal DS4_METAL_REQUIRE_Q8_QKV_COMPRESSOR_FUSE nonempty boolean; unset/empty/exact 0: fallback allowed; other values require and imply the streamed/union enable Requires eligible Q8 QKV/compressor compound fusion and fails closed. ds4.c:23023 +runtime/metal DS4_METAL_RESUME_PREFILL_MIN integer token threshold; default 4; <=0 disables resume-prefill Sets the minimum shared-prefix suffix that uses batched resume-prefill. ds4.c:38419 +runtime/metal DS4_METAL_ROPE_EXP2_LOG2 boolean empty/1/true/yes/on vs 0/false/no/off; default off Compiles the exp2/log2 RoPE drift variant. ds4_metal.m:7050 +runtime/metal DS4_METAL_SELECTED_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for selected. ds4.c:37943 +runtime/metal DS4_METAL_SELECTED_PROFILE_LAYER single nonnegative layer index; unset: every layer Restricts selected-expert profiling to one layer. ds4_metal.m:42507 +runtime/metal DS4_METAL_SESSION_BATCH_LOG presence diagnostic; unset: off; any value including 0 enables Logs session batch decisions. ds4.c:66168 +runtime/metal DS4_METAL_SESSION_BATCH_QKV default enabled; exact 0 disables; every other value/unset leaves enabled Controls native batched QKV work for multi-session decode. ds4.c:65482 +runtime/metal DS4_METAL_SESSION_BATCH_SHARED default enabled; exact 0 disables; every other value/unset leaves enabled Controls native batched shared-expert work for multi-session decode. ds4.c:65432 +runtime/metal DS4_METAL_SET_ROWS_SOURCE file path; unset/empty: use the in-tree Metal source file Overrides the set-rows Metal kernel source file loaded at runtime. ds4_metal.m:4953 +runtime/metal DS4_METAL_SOFTMAX_SOURCE file path; unset/empty: use the in-tree Metal source file Overrides the softmax Metal kernel source file loaded at runtime. ds4_metal.m:4948 +runtime/metal DS4_METAL_STREAMING_DECODE_PREFILL_MAX integer token maximum; default 64 for wide Flash Q4/MXFP4, 18 for other PRO/Flash, 0 otherwise; <=0 disables Sets maximum SSD-streaming micro-prefill width that reuses decode. ds4.c:31962 +runtime/metal DS4_METAL_STREAMING_EXPERT_AUTO_PRELOAD_CAP uint32 expert cap; default 4096; 0 means unlimited; invalid restores default Caps automatic streaming-expert hotlist preload. ds4.c:21455 +runtime/metal DS4_METAL_STREAMING_EXPERT_BUFFER_MLOCK_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for streaming expert buffer mlock. ds4_metal.m:13997 +runtime/metal DS4_METAL_STREAMING_EXPERT_EARLY_LOAD_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for streaming expert early load. ds4_metal.m:17059 +runtime/metal DS4_METAL_STREAMING_EXPERT_EVICT_DONTNEED_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for streaming expert evict dontneed. ds4_metal.m:14436 +runtime/metal DS4_METAL_STREAMING_EXPERT_HOTLIST hotlist file path; unset/empty: built-in model hotlist Loads the streaming-expert preload order from a file. ds4.c:21495 +runtime/metal DS4_METAL_STREAMING_EXPERT_HOTLIST_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for streaming expert hotlist. ds4.c:32245 +runtime/metal DS4_METAL_STREAMING_EXPERT_LAYER_STATS presence diagnostic; unset: off; any value including 0 enables Collects/prints statistics for streaming expert layer. ds4_metal.m:4638 +runtime/metal DS4_METAL_STREAMING_EXPERT_LAYER_STATS_DELTA presence control; unset: off/default; any value including 0 enables Prints delta statistics for streaming expert layer. ds4_metal.m:4675 +runtime/metal DS4_METAL_STREAMING_EXPERT_NOCACHE nonempty value whose first character is not 0 enables; unset/empty/0 disables Uses a reopened F_NOCACHE descriptor for SSD expert preads. ds4_metal.m:12602 +runtime/metal DS4_METAL_STREAMING_EXPERT_PREAD_POOL default enabled; exact 0 disables; every other value/unset keeps enabled Controls reuse of persistent expert-pread worker threads. ds4_metal.m:13433 +runtime/metal DS4_METAL_STREAMING_EXPERT_PREAD_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for streaming expert pread. ds4_metal.m:17005 +runtime/metal DS4_METAL_STREAMING_EXPERT_PREAD_SPLIT integer clamped 1..8; unset: automatic 1 below 64 cache experts, 4 at 64+ Sets aligned requests per expert pread. ds4_metal.m:13701 +runtime/metal DS4_METAL_STREAMING_EXPERT_PREAD_THREADS unsigned integer clamped 1..18; default 9; invalid restores 9 Sets expert-pread worker limit. ds4_metal.m:13337 +runtime/metal DS4_METAL_STREAMING_EXPERT_PROFILE_SUMMARY presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for streaming expert. ds4_metal.m:13061 +runtime/metal DS4_METAL_STREAMING_EXPERT_SLAB_MB positive unsigned MiB; default 4096; 0/invalid restores default Sets target allocation size for streaming-expert slabs. ds4_metal.m:14039 +runtime/metal DS4_METAL_STREAMING_EXPERT_SPLIT_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for streaming expert split. ds4_metal.m:43810 +runtime/metal DS4_METAL_STREAMING_EXPERT_TIMING_SUMMARY presence control; unset: off/default; any value including 0 enables Prints timing/profile diagnostics for streaming expert. ds4_metal.m:13060 +runtime/metal DS4_METAL_STREAMING_IQ2_CPU_ROUTER_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for streaming IQ2 CPU router. ds4.c:21645 +runtime/metal DS4_METAL_STREAMING_MAP_TRACE nonempty value other than exact 0 enables; unset/empty/0 disables Emits SSD model-map decisions. ds4_metal.m:4849 +runtime/metal DS4_METAL_STREAMING_PREFILL_BATCH_SELECTED_ADDR_MAX integer token maximum; default 800 for 384 experts, 760 for 256, 0 otherwise; <=0 disables automatic selection Sets automatic maximum batch width for selected-address SSD prefill. ds4_metal.m:14639 +runtime/metal DS4_METAL_STREAMING_PREFILL_BATCH_SELECTED_ADDR_MIN integer token minimum; default 2 for 256/384 experts, 0 otherwise; <=0 disables automatic selection Sets automatic minimum batch width for selected-address SSD prefill. ds4_metal.m:14656 +runtime/metal DS4_METAL_STREAMING_PREFILL_BATCH_SELECTED_ADDR_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for streaming prefill batch selected address. ds4_metal.m:18493 +runtime/metal DS4_METAL_STREAMING_PREFILL_CACHE_SEED_K uint32 seed rows; default 1; 0 disables; above 64 clamps to 64 Sets how many prefill routing rows seed the decode expert cache. ds4.c:21268 +runtime/metal DS4_METAL_STREAMING_PREFILL_CACHE_SEED_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for streaming prefill cache seed. ds4_metal.m:17899 +runtime/metal DS4_METAL_STREAMING_PREFILL_LAYER_MADVISE_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for streaming prefill layer madvise. ds4.c:19610 +runtime/metal DS4_METAL_STREAMING_PREFILL_LAYER_PAGEIN_NO_OVERLAP presence rollback; unset: automatic/default path; any value including 0 disables Prevents full-layer page-in preparation from overlapping compute. ds4.c:19317 +runtime/metal DS4_METAL_STREAMING_PREFILL_LAYER_PAGEIN_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for streaming prefill layer pagein. ds4.c:19606 +runtime/metal DS4_METAL_STREAMING_PREFILL_LAYER_PAGEIN_THREADS integer 1..16; default 8; invalid/0 becomes 1; PREPARE_THREADS takes precedence Sets worker count for full-layer page-in preparation. ds4.c:19275 +runtime/metal DS4_METAL_STREAMING_PREFILL_LAYER_PREAD_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for streaming prefill layer pread. ds4.c:19608 +runtime/metal DS4_METAL_STREAMING_PREFILL_LAYER_PREPARE_AHEAD integer 1..4 layers; default 1; invalid/0 becomes 1 Sets number of future layers prepared concurrently. ds4.c:19329 +runtime/metal DS4_METAL_STREAMING_PREFILL_LAYER_PREPARE_NO_OVERLAP presence rollback; unset: automatic/default path; any value including 0 disables Prevents generic full-layer preparation from overlapping compute. ds4.c:19315 +runtime/metal DS4_METAL_STREAMING_PREFILL_LAYER_PREPARE_THREADS integer 1..16; default 8; invalid/0 becomes 1; preferred over PAGEIN_THREADS Sets worker count for full-layer preparation. ds4.c:19271 +runtime/metal DS4_METAL_STREAMING_PREFILL_LAYER_READAHEAD_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for streaming prefill layer readahead. ds4.c:19612 +runtime/metal DS4_METAL_STREAMING_PREFILL_SELECTED_MADVISE_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for streaming prefill selected madvise. ds4.c:19356 +runtime/metal DS4_METAL_STREAMING_PREFILL_SELECTED_MADVISE_THREADS integer 1..16; default inherits layer prepare threads; invalid/0 becomes 1; PREPARE_THREADS preferred Sets worker count for selected-expert madvise preparation. ds4.c:19293 +runtime/metal DS4_METAL_STREAMING_PREFILL_SELECTED_PAGEIN_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for streaming prefill selected pagein. ds4.c:19354 +runtime/metal DS4_METAL_STREAMING_PREFILL_SELECTED_PREPARE_GAP integer 0..8 layers; default 0; above 8 clamps; invalid restores 0 Sets lookahead gap for selected-expert preparation. ds4.c:19305 +runtime/metal DS4_METAL_STREAMING_PREFILL_SELECTED_PREPARE_THREADS integer 1..16 for madvise preparation; default inherits layer prepare threads; invalid/0 becomes 1 Sets worker count for selected-expert preparation. ds4.c:19289 +runtime/metal DS4_METAL_STREAMING_PREFILL_SELECTED_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for streaming prefill selected. ds4.c:18906 +runtime/metal DS4_METAL_STREAMING_PREFILL_SELECTED_READAHEAD_GAP integer 0..8 layers; default 0; above 8 clamps; invalid restores 0 Sets lookahead gap for selected-expert readahead. ds4.c:19978 +runtime/metal DS4_METAL_STREAMING_PREFILL_SELECTED_READAHEAD_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for streaming prefill selected readahead. ds4.c:20063 +runtime/metal DS4_METAL_STREAMING_SELECTED_READAHEAD_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for streaming selected readahead. ds4.c:21737 +runtime/metal DS4_METAL_SUM_ROWS_SOURCE file path; unset/empty: use the in-tree Metal source file Overrides the sum-rows Metal kernel source file loaded at runtime. ds4_metal.m:4947 +runtime/metal DS4_METAL_TEST_POISON_COMPRESSOR_EXACT_REDUCTION_SCRATCH internal test presence flag; unset: off; any value including 0 poisons scratch before the exact reduction Validates that compressor exact-reduction kernels overwrite all scratch state. ds4_metal.m:25925 +runtime/metal DS4_METAL_TP_SESSION_BATCH default enabled; exact 0 disables; every other value/unset leaves enabled Controls batched session evaluation with Metal TP. ds4.c:65310 +runtime/metal DS4_METAL_TRACE_ALLOCS presence diagnostic; unset: off; any value including 0 enables Emits trace diagnostics for allocs. ds4_metal.m:4057 +runtime/metal DS4_METAL_TRACE_M5_FLASH_ATTN_PACKED32_REDUCE presence diagnostic; unset: off; any value including 0 enables Emits trace diagnostics for M5 flash attn packed32 reduce. ds4_metal.m:31535 +runtime/metal DS4_METAL_UNARY_SOURCE file path; unset/empty: use the in-tree Metal source file Overrides the unary operations Metal kernel source file loaded at runtime. ds4_metal.m:4939 +runtime/metal DS4_METAL_UNRETAINED_COMMAND_BUFFERS presence control; unset: off/default; any value including 0 enables Creates Metal command buffers with unretained references. ds4_metal.m:1315 +runtime/metal DS4_METAL_USE_QUEUE_RESIDENCY_SET presence control; unset: off/default; any value including 0 enables Allows queue-residency state to trigger Q4 expert address/table paths. ds4_metal.m:42280 +runtime/moe-debug DS4_MOE_RECORD_SELECTED_HOTLIST nonempty output path; unset=off Record per-layer selected-expert hit counts to a Metal hotlist file. ds4_metal.m:1741 +runtime/moe-debug DS4_MOE_RECORD_SELECTED_HOTLIST_FRESH presence flag; only relevant with HOTLIST; overrides MERGE Start the selected-expert hotlist from empty state. ds4_metal.m:1642 +runtime/moe-debug DS4_MOE_RECORD_SELECTED_HOTLIST_MERGE presence flag; active only when FRESH is absent Merge an existing selected-expert hotlist before recording. ds4_metal.m:1641 +runtime/moe-debug DS4_MOE_RECORD_SELECTED_IDS nonempty output path; unset=off Record routed-MoE six-expert selections; also disables incompatible optimized paths. ds4.c:65290 +runtime/moe-debug DS4_MOE_REPLAY_SELECTED_IDS nonempty input path; unset=off Replay routed-MoE six-expert selections; also disables incompatible optimized paths. ds4.c:25308 +runtime/mtp DS4_MTP_BATCH_VERIFY Pure presence flag: any defined value, including empty or "0", suppresses the exact two-row decode verifier. Unset selects exact decode-2 when draft_n==2 and either strict mode is active or the build is ROCm; other cases already use the generic verifier. Diagnostic rollback from the exact Q8/one-token-equivalent MTP decode-2 verifier to the generic microbatch verifier. ds4.c:71021 +runtime/mtp DS4_MTP_CAPTURE_PREFIX1 Pure presence flag. In the generic verifier with exactly two drafts it enables prefix-1 state capture under strict mode; non-strict mode already captures prefix-1 without the variable. Unset under strict mode instead snapshots and replays a partial acceptance. Let a one-of-two MTP partial acceptance commit the verifier's captured prefix directly, avoiding an exact one-token replay. ds4.c:71134 +runtime/mtp DS4_MTP_CONF_LOG Pure presence flag; default off. It forces materialization of full draft logits, computes the top-2 margin, and after a successful generic microbatch verification prints drafted/committed counts, top candidates, margin, target-next and draft-next. Exact decode-2 success does not emit that generic log line. Inspect MTP draft confidence and compare the recursive draft token with the target verifier result. ds4.c:70900 +runtime/mtp DS4_MTP_EXACT_REPLAY Pure presence flag; default off. In the generic microbatch verifier it forces a pre-verifier frontier snapshot; after verification the snapshot is restored and every accepted draft is decoded sequentially to rebuild exact final state/logits. Validate MTP acceptance while committing through the normal one-token decode path rather than retaining batched-verifier state. ds4.c:71139 +runtime/mtp DS4_MTP_FORCE_SNAPSHOT Pure presence flag; default off. It forces a speculative-frontier snapshot before the generic verifier regardless of draft count or prefix-capture mode; it does not by itself force restoration or replay after a successful full acceptance. Measure/debug snapshot behavior and guarantee a restorable pre-verifier frontier for generic MTP verification. ds4.c:71143 +runtime/mtp DS4_MTP_FULL_LOGITS Pure presence flag; default off. When set, legacy and recursive MTP draft calls write the full vocabulary logits to s->mtp_logits; unset permits the faster top-token-only output unless confidence/margin logic independently needs logits. Force full MTP draft-logit materialization for correctness comparison, inspection, or downstream confidence calculations. ds4.c:64260 +runtime/mtp DS4_MTP_MIN_MARGIN non-negative float; default engine --mtp-margin value Set confidence margin threshold for speculative MTP verification. ds4.c:70893 +runtime/mtp DS4_MTP_PROBE Pure presence flag; default off. For legacy MTP it prepares drafts even when configured depth<=1, compares the previous draft with the next committed token, and prints cumulative hit counts/failures; generated output is unchanged. Measure legacy MTP next-token draft accuracy without enabling speculative acceptance. ds4.c:64990 runtime/mtp DS4_MTP_SPEC_DISABLE Pure presence flag: any defined value, including empty or "0", disables MTP speculative argmax in CLI/chat/server loops. Unset permits it for greedy temperature<=0 generation with draft depth>1; unrelated split-KV speculation can still be independently requested. Fall back from MTP multi-token speculative evaluation to normal one-token session evaluation. ds4_cli.c:580 -runtime/mtp DS4_MTP_SPEC_LOG Pure presence flag; default off. It only emits diagnostics for first-draft misses, exact/generic verifier failures and sequential fallback misses/acceptance outcomes; it does not select a verifier. Trace why MTP drafts were accepted, partially accepted, rejected, or sent to sequential fallback. ds4.c:70726 -runtime/mtp DS4_MTP_STRICT Pure presence flag; engine quality mode also enables strictness automatically. Strict mode skips the non-strict low-margin shortcut, selects exact decode-2 for two drafts unless DS4_MTP_BATCH_VERIFY is set, and disables default prefix-1 capture unless explicitly restored. Force the exact/quality-oriented MTP verification policy on otherwise non-quality runs. ds4.c:70701 -runtime/mtp DS4_MTP_TIMING Pure presence flag; default off. When set, timestamps and prints draft, snapshot, verifier, prefix/replay and total durations for the path taken; algorithm selection is otherwise unchanged. Profile end-to-end MTP speculative decoding and separate draft, verification and state-commit costs. ds4.c:70709 -runtime/rocm DS4_ROCM_DECODE_STAGE_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm decode stage profile. ds4.c:18071 -runtime/rocm DS4_ROCM_DECODE_STAGE_PROFILE_LAYER layer filter subordinate to DS4_ROCM_DECODE_STAGE_PROFILE; unset or whitespace-only: all layers allowed by the parent flag; otherwise the whitespace-trimmed value must be a complete base-10 strtoul result <= UINT32_MAX equal to the current layer; invalid values match none Restricts the ROCm decode stage profiler to one layer; it does not enable profiling by itself. ds4.c:28943 -runtime/rocm DS4_ROCM_DISABLE_GLM_STREAMING_PREFILL_FULL_LAYER integer selector/tuning value; unset or invalid uses internal automatic/default value Disable/roll back rocm disable glm streaming prefill full layer. ds4.c:42519 -runtime/rocm DS4_ROCM_DISABLE_GLM_STREAMING_PREFILL_FULL_LAYER_PREPARE presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable glm streaming prefill full layer prepare. ds4.c:42537 -runtime/rocm DS4_ROCM_DISABLE_GLM_STREAMING_PREFILL_SELECTED_ASYNC_LOAD presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable glm streaming prefill selected async load. ds4.c:46230 -runtime/rocm DS4_ROCM_DISABLE_GLM_STREAMING_SELECTED_ASYNC_LOAD presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable glm streaming selected async load. ds4.c:44138 -runtime/rocm DS4_ROCM_DISABLE_IQ2_SELECTED_EXPERT_VIEWS presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable iq2 selected expert views. ds4.c:20906 -runtime/rocm DS4_ROCM_DISABLE_IQ2_STREAM_ADDR_TABLE presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable iq2 stream addr table. ds4.c:6425 -runtime/rocm DS4_ROCM_DISABLE_Q4_DENSE_PAIR presence rollback; unset leaves opt-in policy unchanged Disable/roll back rocm disable q4 dense pair. rocm/ds4_rocm_q4.cuh:299 -runtime/rocm DS4_ROCM_DISABLE_Q4_GROUPED_ATTN_A presence rollback; DISABLE wins over enable/require Disable/roll back rocm disable q4 grouped attn a. rocm/ds4_rocm_q4.cuh:573 -runtime/rocm DS4_ROCM_DISABLE_Q4_PREFILL_TILE8 presence rollback; TILE8 is default for 9..4096 tokens Disable/roll back rocm disable q4 prefill tile8. rocm/ds4_rocm_q4.cuh:312 -runtime/rocm DS4_ROCM_DISABLE_Q4_SELECTED_EXPERT_VIEWS presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable q4 selected expert views. ds4.c:20962 +runtime/mtp DS4_MTP_SPEC_LOG Pure presence flag; default off. It only emits diagnostics for first-draft misses, exact/generic verifier failures and sequential fallback misses/acceptance outcomes; it does not select a verifier. Trace why MTP drafts were accepted, partially accepted, rejected, or sent to sequential fallback. ds4.c:70916 +runtime/mtp DS4_MTP_STRICT Pure presence flag; engine quality mode also enables strictness automatically. Strict mode skips the non-strict low-margin shortcut, selects exact decode-2 for two drafts unless DS4_MTP_BATCH_VERIFY is set, and disables default prefix-1 capture unless explicitly restored. Force the exact/quality-oriented MTP verification policy on otherwise non-quality runs. ds4.c:70891 +runtime/mtp DS4_MTP_TIMING Pure presence flag; default off. When set, timestamps and prints draft, snapshot, verifier, prefix/replay and total durations for the path taken; algorithm selection is otherwise unchanged. Profile end-to-end MTP speculative decoding and separate draft, verification and state-commit costs. ds4.c:70899 +runtime/rocm DS4_ROCM_DECODE_STAGE_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm decode stage profile. ds4.c:18244 +runtime/rocm DS4_ROCM_DECODE_STAGE_PROFILE_LAYER layer filter subordinate to DS4_ROCM_DECODE_STAGE_PROFILE; unset or whitespace-only: all layers allowed by the parent flag; otherwise the whitespace-trimmed value must be a complete base-10 strtoul result <= UINT32_MAX equal to the current layer; invalid values match none Restricts the ROCm decode stage profiler to one layer; it does not enable profiling by itself. ds4.c:29116 +runtime/rocm DS4_ROCM_DISABLE_GLM_STREAMING_PREFILL_FULL_LAYER integer selector/tuning value; unset or invalid uses internal automatic/default value Disable/roll back rocm disable glm streaming prefill full layer. ds4.c:42709 +runtime/rocm DS4_ROCM_DISABLE_GLM_STREAMING_PREFILL_FULL_LAYER_PREPARE presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable glm streaming prefill full layer prepare. ds4.c:42727 +runtime/rocm DS4_ROCM_DISABLE_GLM_STREAMING_PREFILL_SELECTED_ASYNC_LOAD presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable glm streaming prefill selected async load. ds4.c:46420 +runtime/rocm DS4_ROCM_DISABLE_GLM_STREAMING_SELECTED_ASYNC_LOAD presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable glm streaming selected async load. ds4.c:44328 +runtime/rocm DS4_ROCM_DISABLE_IQ2_SELECTED_EXPERT_VIEWS presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable iq2 selected expert views. ds4.c:21079 +runtime/rocm DS4_ROCM_DISABLE_IQ2_STREAM_ADDR_TABLE presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable iq2 stream addr table. ds4.c:6548 +runtime/rocm DS4_ROCM_DISABLE_Q4_DENSE_PAIR presence rollback; unset leaves opt-in policy unchanged Disable/roll back rocm disable q4 dense pair. rocm/ds4_rocm_q4.cuh:435 +runtime/rocm DS4_ROCM_DISABLE_Q4_GROUPED_ATTN_A presence rollback; DISABLE wins over enable/require Disable/roll back rocm disable q4 grouped attn a. rocm/ds4_rocm_q4.cuh:700 +runtime/rocm DS4_ROCM_DISABLE_Q4_PREFILL_TILE8 presence rollback; TILE8 is default for 9..4096 tokens Disable/roll back rocm disable q4 prefill tile8. rocm/ds4_rocm_q4.cuh:448 +runtime/rocm DS4_ROCM_DISABLE_Q4_SELECTED_EXPERT_VIEWS presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable q4 selected expert views. ds4.c:21135 runtime/rocm DS4_ROCM_DISABLE_RESIDENT_IQ2_SORTED presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable resident iq2 sorted. rocm/ds4_rocm_moe_launch.cuh:747 -runtime/rocm DS4_ROCM_DISABLE_ROUTED_PAIR_SWIGLU_FUSION presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable routed pair swiglu fusion. ds4.c:18355 -runtime/rocm DS4_ROCM_DISABLE_STREAMING_COLD_DECODE_PREFILL presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming cold decode prefill. ds4.c:31816 -runtime/rocm DS4_ROCM_DISABLE_STREAMING_DECODE_PREFILL presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming decode prefill. ds4.c:31765 -runtime/rocm DS4_ROCM_DISABLE_STREAMING_EXPERT_ADDR_TABLE presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming expert addr table. ds4.c:18351 -runtime/rocm DS4_ROCM_DISABLE_STREAMING_EXPERT_HOTLIST presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming expert hotlist. ds4.c:21116 -runtime/rocm DS4_ROCM_DISABLE_STREAMING_FULL_EXPERT_ADDR_TABLE presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming full expert addr table. ds4.c:18069 -runtime/rocm DS4_ROCM_DISABLE_STREAMING_LAYER_BATCH presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming layer batch. ds4.c:18065 -runtime/rocm DS4_ROCM_DISABLE_STREAMING_MADVISE_WILLNEED presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming madvise willneed. ds4.c:18038 -runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_BATCH_SELECTED_ADDR presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming prefill batch selected addr. ds4.c:18349 -runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_MADVISE presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming prefill layer madvise. ds4.c:18292 -runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_PAGEIN presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming prefill layer pagein. ds4.c:18260 -runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_PAGEIN_OVERLAP presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming prefill layer pagein overlap. ds4.c:19147 -runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_PREAD presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming prefill layer pread. ds4.c:18280 -runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_PREPARE presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming prefill layer prepare. ds4.c:18272 -runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_PREPARE_OVERLAP presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming prefill layer prepare overlap. ds4.c:19145 -runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_READAHEAD presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming prefill layer readahead. ds4.c:18270 -runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_ASYNC_LOAD presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming prefill selected async load. ds4.c:46233 -runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_MADVISE presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming prefill selected madvise. ds4.c:18250 -runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_PAGEIN presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming prefill selected pagein. ds4.c:18240 -runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_PROFILE presence rollback flag; unset keeps automatic/default path Collect timing/profile diagnostics for rocm disable streaming prefill selected profile. ds4.c:18734 -runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_READAHEAD presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming prefill selected readahead. ds4.c:19786 -runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_READAHEAD_SHARED presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming prefill selected readahead shared. ds4.c:19796 -runtime/rocm DS4_ROCM_DISABLE_STREAMING_READAHEAD presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming readahead. ds4.c:18031 -runtime/rocm DS4_ROCM_DISABLE_STREAMING_SELECTED_ASYNC_LOAD presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming selected async load. ds4.c:44136 +runtime/rocm DS4_ROCM_DISABLE_ROUTED_PAIR_SWIGLU_FUSION presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable routed pair swiglu fusion. ds4.c:18528 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_COLD_DECODE_PREFILL presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming cold decode prefill. ds4.c:32006 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_DECODE_PREFILL presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming decode prefill. ds4.c:31955 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_EXPERT_ADDR_TABLE presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming expert addr table. ds4.c:18524 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_EXPERT_HOTLIST presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming expert hotlist. ds4.c:21289 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_FULL_EXPERT_ADDR_TABLE presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming full expert addr table. ds4.c:18242 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_LAYER_BATCH presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming layer batch. ds4.c:18238 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_MADVISE_WILLNEED presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming madvise willneed. ds4.c:18211 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_BATCH_SELECTED_ADDR presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming prefill batch selected addr. ds4.c:18522 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_MADVISE presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming prefill layer madvise. ds4.c:18465 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_PAGEIN presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming prefill layer pagein. ds4.c:18433 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_PAGEIN_OVERLAP presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming prefill layer pagein overlap. ds4.c:19320 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_PREAD presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming prefill layer pread. ds4.c:18453 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_PREPARE presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming prefill layer prepare. ds4.c:18445 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_PREPARE_OVERLAP presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming prefill layer prepare overlap. ds4.c:19318 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_READAHEAD presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming prefill layer readahead. ds4.c:18443 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_ASYNC_LOAD presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming prefill selected async load. ds4.c:46423 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_MADVISE presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming prefill selected madvise. ds4.c:18423 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_PAGEIN presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming prefill selected pagein. ds4.c:18413 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_PROFILE presence rollback flag; unset keeps automatic/default path Collect timing/profile diagnostics for rocm disable streaming prefill selected profile. ds4.c:18907 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_READAHEAD presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming prefill selected readahead. ds4.c:19959 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_READAHEAD_SHARED presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming prefill selected readahead shared. ds4.c:19969 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_READAHEAD presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming readahead. ds4.c:18204 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_SELECTED_ASYNC_LOAD presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming selected async load. ds4.c:44326 runtime/rocm DS4_ROCM_DISABLE_STREAMING_SPLIT_SELECTED presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming split selected. rocm/ds4_rocm_moe_launch.cuh:661 -runtime/rocm DS4_ROCM_DISABLE_STREAMING_STATIC_DECODE_MAP presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming static decode map. ds4.c:18043 -runtime/rocm DS4_ROCM_DISABLE_STREAMING_STATIC_MAP_STATE_CACHE presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming static map state cache. ds4.c:18056 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_STATIC_DECODE_MAP presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming static decode map. ds4.c:18216 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_STATIC_MAP_STATE_CACHE presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming static map state cache. ds4.c:18229 runtime/rocm DS4_ROCM_DSV4_PREQUANT_DECODE sampled once; unset: enabled; present empty or exact 0: disabled; every other present value: enabled; quality mode and GLM models force it off regardless ROCm DeepSeek-V4 decode: quantizes one-token F32 activations to Q8 once and selects the prequantized Q8_0/DP4A projection kernels instead of the full-F32 activation paths. rocm/ds4_rocm_runtime.cuh:4775 -runtime/rocm DS4_ROCM_ENABLE_Q4_DENSE_PAIR presence opt-in; unset=off; DISABLE takes precedence Enable rocm enable q4 dense pair. rocm/ds4_rocm_q4.cuh:298 -runtime/rocm DS4_ROCM_ENABLE_Q4_GROUPED_ATTN_A presence opt-in; unset=off unless REQUIRE; DISABLE wins Enable rocm enable q4 grouped attn a. rocm/ds4_rocm_q4.cuh:577 -runtime/rocm DS4_ROCM_ENABLE_STREAMING_FULL_EXPERT_ADDR_TABLE presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming full expert addr table. ds4.c:18067 -runtime/rocm DS4_ROCM_ENABLE_STREAMING_MADVISE_WILLNEED presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming madvise willneed. ds4.c:18036 -runtime/rocm DS4_ROCM_ENABLE_STREAMING_PREFILL_BATCH_SELECTED_ADDR presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming prefill batch selected addr. ds4.c:18396 -runtime/rocm DS4_ROCM_ENABLE_STREAMING_PREFILL_CACHE_SEED presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming prefill cache seed. ds4.c:21085 -runtime/rocm DS4_ROCM_ENABLE_STREAMING_PREFILL_LAYER_PAGEIN presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming prefill layer pagein. ds4.c:18258 -runtime/rocm DS4_ROCM_ENABLE_STREAMING_PREFILL_LAYER_READAHEAD presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming prefill layer readahead. ds4.c:18268 -runtime/rocm DS4_ROCM_ENABLE_STREAMING_PREFILL_SELECTED_MADVISE presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming prefill selected madvise. ds4.c:18248 -runtime/rocm DS4_ROCM_ENABLE_STREAMING_PREFILL_SELECTED_PAGEIN presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming prefill selected pagein. ds4.c:18238 -runtime/rocm DS4_ROCM_ENABLE_STREAMING_PREFILL_SELECTED_READAHEAD presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming prefill selected readahead. ds4.c:19782 -runtime/rocm DS4_ROCM_ENABLE_STREAMING_PREFILL_SELECTED_READAHEAD_SHARED presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming prefill selected readahead shared. ds4.c:19784 -runtime/rocm DS4_ROCM_ENABLE_STREAMING_READAHEAD presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming readahead. ds4.c:18029 -runtime/rocm DS4_ROCM_ENABLE_STREAMING_STATIC_DECODE_MAP presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming static decode map. ds4.c:18048 +runtime/rocm DS4_ROCM_ENABLE_Q4_DENSE_PAIR presence opt-in; unset=off; DISABLE takes precedence Enable rocm enable q4 dense pair. rocm/ds4_rocm_q4.cuh:434 +runtime/rocm DS4_ROCM_ENABLE_Q4_GROUPED_ATTN_A presence opt-in; unset=off unless REQUIRE; DISABLE wins Enable rocm enable q4 grouped attn a. rocm/ds4_rocm_q4.cuh:704 +runtime/rocm DS4_ROCM_ENABLE_STREAMING_FULL_EXPERT_ADDR_TABLE presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming full expert addr table. ds4.c:18240 +runtime/rocm DS4_ROCM_ENABLE_STREAMING_MADVISE_WILLNEED presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming madvise willneed. ds4.c:18209 +runtime/rocm DS4_ROCM_ENABLE_STREAMING_PREFILL_BATCH_SELECTED_ADDR presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming prefill batch selected addr. ds4.c:18569 +runtime/rocm DS4_ROCM_ENABLE_STREAMING_PREFILL_CACHE_SEED presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming prefill cache seed. ds4.c:21258 +runtime/rocm DS4_ROCM_ENABLE_STREAMING_PREFILL_LAYER_PAGEIN presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming prefill layer pagein. ds4.c:18431 +runtime/rocm DS4_ROCM_ENABLE_STREAMING_PREFILL_LAYER_READAHEAD presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming prefill layer readahead. ds4.c:18441 +runtime/rocm DS4_ROCM_ENABLE_STREAMING_PREFILL_SELECTED_MADVISE presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming prefill selected madvise. ds4.c:18421 +runtime/rocm DS4_ROCM_ENABLE_STREAMING_PREFILL_SELECTED_PAGEIN presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming prefill selected pagein. ds4.c:18411 +runtime/rocm DS4_ROCM_ENABLE_STREAMING_PREFILL_SELECTED_READAHEAD presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming prefill selected readahead. ds4.c:19955 +runtime/rocm DS4_ROCM_ENABLE_STREAMING_PREFILL_SELECTED_READAHEAD_SHARED presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming prefill selected readahead shared. ds4.c:19957 +runtime/rocm DS4_ROCM_ENABLE_STREAMING_READAHEAD presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming readahead. ds4.c:18202 +runtime/rocm DS4_ROCM_ENABLE_STREAMING_STATIC_DECODE_MAP presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming static decode map. ds4.c:18221 runtime/rocm DS4_ROCM_GLM_CAUSAL_ATTN_GEMM Enabled by default when unset. Exact "0" or an empty value disables; every other nonempty value enables (including false/off/no), because cuda_env_present only tests nonempty and != "0". Eligibility still requires causal_range && !has_selected; a failed GEMM helper falls through to the scalar attention kernel. Use FP16 BLAS GEMMs for dense causal GLM indexed prefill; =0 is the correctness/performance rollback to the scalar attention kernel. rocm/ds4_rocm_glm.cuh:3283 -runtime/rocm DS4_ROCM_GLM_DISABLE_STREAMING_EXPERT_CACHE Pure presence flag: any defined value, including empty or "0", disables. Unset leaves automatic GLM streaming expert-cache eligibility enabled for supported model/quant/quality/SSD configurations. On ROCm builds DS4_METAL_GLM_DISABLE_STREAMING_EXPERT_CACHE is an accepted fallback alias. Disable selected/resident streamed-expert cache paths and force generic/full-layer expert handling for GLM SSD streaming. ds4.c:21024 -runtime/rocm DS4_ROCM_GLM_DISABLE_STREAMING_SEED_BEFORE_PREFILL Pure presence flag: any defined value, including empty or "0", disables. Unset seeds before prefill whenever SSD streaming is active. On ROCm builds DS4_METAL_GLM_DISABLE_STREAMING_SEED_BEFORE_PREFILL is an accepted fallback alias. Skip the pre-prefill hotlist seed of the streaming expert cache in both one-shot GLM generation and session setup. ds4.c:50883 -runtime/rocm DS4_ROCM_GLM_DISABLE_STREAMING_TOKEN_PREFILL Pure presence flag: any defined value, including empty or "0", disables. Unset leaves the token-major path eligible only for SSD streaming, non-quality mode, a nonempty batch fitting full attention, and n_tokens <= the configured nonzero maximum. DS4_METAL_GLM_DISABLE_STREAMING_TOKEN_PREFILL and generic DS4_GLM_DISABLE_STREAMING_TOKEN_PREFILL are also accepted presence aliases. Roll back GLM SSD-streaming token-major prefill to the normal prefill implementation. ds4.c:49513 +runtime/rocm DS4_ROCM_GLM_DISABLE_STREAMING_EXPERT_CACHE Pure presence flag: any defined value, including empty or "0", disables. Unset leaves automatic GLM streaming expert-cache eligibility enabled for supported model/quant/quality/SSD configurations. On ROCm builds DS4_METAL_GLM_DISABLE_STREAMING_EXPERT_CACHE is an accepted fallback alias. Disable selected/resident streamed-expert cache paths and force generic/full-layer expert handling for GLM SSD streaming. ds4.c:21197 +runtime/rocm DS4_ROCM_GLM_DISABLE_STREAMING_SEED_BEFORE_PREFILL Pure presence flag: any defined value, including empty or "0", disables. Unset seeds before prefill whenever SSD streaming is active. On ROCm builds DS4_METAL_GLM_DISABLE_STREAMING_SEED_BEFORE_PREFILL is an accepted fallback alias. Skip the pre-prefill hotlist seed of the streaming expert cache in both one-shot GLM generation and session setup. ds4.c:51073 +runtime/rocm DS4_ROCM_GLM_DISABLE_STREAMING_TOKEN_PREFILL Pure presence flag: any defined value, including empty or "0", disables. Unset leaves the token-major path eligible only for SSD streaming, non-quality mode, a nonempty batch fitting full attention, and n_tokens <= the configured nonzero maximum. DS4_METAL_GLM_DISABLE_STREAMING_TOKEN_PREFILL and generic DS4_GLM_DISABLE_STREAMING_TOKEN_PREFILL are also accepted presence aliases. Roll back GLM SSD-streaming token-major prefill to the normal prefill implementation. ds4.c:49703 runtime/rocm DS4_ROCM_GLM_GROUPED_QK_LOW sampled once; unset: enabled; present empty or exact 0: disabled; every other present value: enabled Selects the grouped shared-input ROCm kernel for eligible multi-token GLM qk-lowrank projection; disabling uses the per-head/per-token projection kernel. rocm/ds4_rocm_runtime.cuh:4800 runtime/rocm DS4_ROCM_GLM_GROUPED_VALUE_PROJECT sampled once; unset: enabled; present empty or exact 0: disabled; every other present value: enabled Selects the grouped shared-input ROCm kernel for eligible multi-token GLM value projection; disabling uses the non-grouped batch projection path. rocm/ds4_rocm_runtime.cuh:4791 -runtime/rocm DS4_ROCM_GLM_LAYER_SLICE_TOKEN_DECODE Opt-in truthy parser; unset, empty, "0", false, off, or no (case-insensitive words) are false, every other nonempty value is true. Default off and compiled only for ROCm. Allow a one-token, pos>0 GLM layer-slice with inter-node input/output hidden buffers to use the optimized resident token graph; without it only the no-hidden-buffer case takes that shortcut. ds4.c:43383 +runtime/rocm DS4_ROCM_GLM_LAYER_SLICE_TOKEN_DECODE Opt-in truthy parser; unset, empty, "0", false, off, or no (case-insensitive words) are false, every other nonempty value is true. Default off and compiled only for ROCm. Allow a one-token, pos>0 GLM layer-slice with inter-node input/output hidden buffers to use the optimized resident token graph; without it only the no-hidden-buffer case takes that shortcut. ds4.c:43573 runtime/rocm DS4_ROCM_GLM_SELECTED_ATTN_GEMM Enabled by default when unset. Exact "0" or an empty value disables; every other nonempty value enables (including false/off/no). Eligibility still requires !causal_range && has_selected; failure/ineligibility falls through to the scalar attention kernel. Gather per-token selected cache rows into FP16 matrices and use strided-batched BLAS GEMMs for GLM selected indexed prefill; =0 forces the scalar path. rocm/ds4_rocm_glm.cuh:3239 runtime/rocm DS4_ROCM_GLM_SELECTED_ATTN_HEAD_TILE Unsigned integer read and cached once; valid values are exactly 1,2,4,8,16,32,64. Unset/empty defaults to 16. A nonnumeric, partially parsed, overflowed, or unsupported value prints a warning and uses 16; the effective tile is min(requested,n_head). Set how many attention heads each selected-attention GEMM workspace tile processes. rocm/ds4_rocm_glm.cuh:2509 runtime/rocm DS4_ROCM_GLM_SELECTED_ATTN_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm glm selected attn profile. rocm/ds4_rocm_glm.cuh:2534 -runtime/rocm DS4_ROCM_GLM_STREAMING_ASYNC_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm glm streaming async profile. ds4.c:44171 -runtime/rocm DS4_ROCM_GLM_STREAMING_DECODE_FULL_LAYER_MAP Pure presence flag: any defined value, including empty or "0", forces full mapping. Unset uses automatic mapping: resident layers map fully, eligible expert-cache layers use decode-only/expert mapping, otherwise full mapping. DS4_METAL_GLM_STREAMING_DECODE_FULL_LAYER_MAP and generic DS4_GLM_STREAMING_DECODE_FULL_LAYER_MAP are also accepted presence aliases. Force every GLM SSD-streaming decode layer through full-layer mapping, bypassing the selected-expert/decode mapping optimization. ds4.c:42425 -runtime/rocm DS4_ROCM_GLM_STREAMING_DECODE_SYNC_EACH_LAYER ROCm-only primary value; nonempty takes priority over DS4_METAL_GLM_STREAMING_DECODE_SYNC_EACH_LAYER and the generic DS4_GLM_STREAMING_DECODE_SYNC_EACH_LAYER fallback; empty acts as unset; truthy unless exact 0 or case-insensitive false/off/no; with all aliases unset: false For non-static GLM SSD decode on ROCm, opts into a full command/device synchronization after token mapping and every layer; default keeps ordered work queued across layer mappings, and static-map decode bypasses it. ds4.c:49589 -runtime/rocm DS4_ROCM_GLM_STREAMING_GROW_CACHE_AFTER_PREFILL Enabled by default when absent. If defined, only a truthy nonempty value enables; empty, "0", false, off, or no disable. Growth also requires SSD streaming plus nonzero base cache and prefill-headroom budgets, and occurs only if the recomputed expert count exceeds the current count. After successful ROCm GLM prefill, add the released prefill headroom to the dynamic streaming expert-cache byte budget. ds4.c:50924 -runtime/rocm DS4_ROCM_GLM_STREAMING_PREFILL_FULL_LAYER presence force-on; any presence including empty or 0 enables; unset falls back to the Metal alias and then the automatic token threshold (1024 by default on ROCm); DS4_ROCM_DISABLE_GLM_STREAMING_PREFILL_FULL_LAYER dominates Forces GLM SSD prefill into full-layer mapping/cache mode even below the automatic large-batch threshold. ds4.c:42523 -runtime/rocm DS4_ROCM_GLM_STREAMING_PREFILL_FULL_LAYER_MIN_TOKENS Positive uint32 threshold parsed with strtoul; ROCm default is 1024. Missing/empty, no leading number, errno/overflow, zero, or >UINT32_MAX returns 1024. The parser does not require end-of-string, so trailing junk after a valid leading number is accepted. A nonempty ROCm value takes precedence; otherwise DS4_METAL_GLM_STREAMING_PREFILL_FULL_LAYER_MIN_TOKENS is a fallback alias. Set the automatic token-count crossover for ROCm GLM SSD prefill to load/use full resident expert layers when the layer supports that mode. ds4.c:42502 -runtime/rocm DS4_ROCM_GLM_STREAMING_PREFILL_SYNC_EACH_LAYER ROCm-only primary value; nonempty takes priority over DS4_METAL_GLM_STREAMING_PREFILL_SYNC_EACH_LAYER and the generic DS4_GLM_STREAMING_PREFILL_SYNC_EACH_LAYER fallback; empty acts as unset; truthy unless exact 0 or case-insensitive false/off/no; with all aliases unset: false for compact prefill; full-layer prefill always returns true For compact GLM SSD prefill on ROCm, opts into a full command/device synchronization at every layer boundary; default preserves queued work across mappings, while full-layer cache mode always synchronizes. ds4.c:42205 -runtime/rocm DS4_ROCM_GLM_STREAMING_TOKEN_PREFILL_MAX primary nonempty value, then the Metal alias, then generic DS4_GLM_STREAMING_TOKEN_PREFILL_MAX; parsed by strtoul without requiring full-string consumption; 0 is valid and disables; no digits, ERANGE, or > UINT32_MAX uses the ROCm default 0 Sets the largest non-quality GLM SSD-prefill chunk eligible for token-major/decode-style execution; ROCm defaults to canonical indexed batch prefill (0 disables token-major mode). ds4.c:49492 +runtime/rocm DS4_ROCM_GLM_STREAMING_ASYNC_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm glm streaming async profile. ds4.c:44361 +runtime/rocm DS4_ROCM_GLM_STREAMING_DECODE_FULL_LAYER_MAP Pure presence flag: any defined value, including empty or "0", forces full mapping. Unset uses automatic mapping: resident layers map fully, eligible expert-cache layers use decode-only/expert mapping, otherwise full mapping. DS4_METAL_GLM_STREAMING_DECODE_FULL_LAYER_MAP and generic DS4_GLM_STREAMING_DECODE_FULL_LAYER_MAP are also accepted presence aliases. Force every GLM SSD-streaming decode layer through full-layer mapping, bypassing the selected-expert/decode mapping optimization. ds4.c:42615 +runtime/rocm DS4_ROCM_GLM_STREAMING_DECODE_SYNC_EACH_LAYER ROCm-only primary value; nonempty takes priority over DS4_METAL_GLM_STREAMING_DECODE_SYNC_EACH_LAYER and the generic DS4_GLM_STREAMING_DECODE_SYNC_EACH_LAYER fallback; empty acts as unset; truthy unless exact 0 or case-insensitive false/off/no; with all aliases unset: false For non-static GLM SSD decode on ROCm, opts into a full command/device synchronization after token mapping and every layer; default keeps ordered work queued across layer mappings, and static-map decode bypasses it. ds4.c:49779 +runtime/rocm DS4_ROCM_GLM_STREAMING_GROW_CACHE_AFTER_PREFILL Enabled by default when absent. If defined, only a truthy nonempty value enables; empty, "0", false, off, or no disable. Growth also requires SSD streaming plus nonzero base cache and prefill-headroom budgets, and occurs only if the recomputed expert count exceeds the current count. After successful ROCm GLM prefill, add the released prefill headroom to the dynamic streaming expert-cache byte budget. ds4.c:51111 +runtime/rocm DS4_ROCM_GLM_STREAMING_PREFILL_FULL_LAYER presence force-on; any presence including empty or 0 enables; unset falls back to the Metal alias and then the automatic token threshold (1024 by default on ROCm); DS4_ROCM_DISABLE_GLM_STREAMING_PREFILL_FULL_LAYER dominates Forces GLM SSD prefill into full-layer mapping/cache mode even below the automatic large-batch threshold. ds4.c:42692 +runtime/rocm DS4_ROCM_GLM_STREAMING_PREFILL_FULL_LAYER_MIN_TOKENS Positive uint32 threshold parsed with strtoul; ROCm default is 1024. Missing/empty, no leading number, errno/overflow, zero, or >UINT32_MAX returns 1024. The parser does not require end-of-string, so trailing junk after a valid leading number is accepted. A nonempty ROCm value takes precedence; otherwise DS4_METAL_GLM_STREAMING_PREFILL_FULL_LAYER_MIN_TOKENS is a fallback alias. Set the automatic token-count crossover for ROCm GLM SSD prefill to load/use full resident expert layers when the layer supports that mode. ds4.c:42692 +runtime/rocm DS4_ROCM_GLM_STREAMING_PREFILL_SYNC_EACH_LAYER ROCm-only primary value; nonempty takes priority over DS4_METAL_GLM_STREAMING_PREFILL_SYNC_EACH_LAYER and the generic DS4_GLM_STREAMING_PREFILL_SYNC_EACH_LAYER fallback; empty acts as unset; truthy unless exact 0 or case-insensitive false/off/no; with all aliases unset: false for compact prefill; full-layer prefill always returns true For compact GLM SSD prefill on ROCm, opts into a full command/device synchronization at every layer boundary; default preserves queued work across mappings, while full-layer cache mode always synchronizes. ds4.c:42395 +runtime/rocm DS4_ROCM_GLM_STREAMING_TOKEN_PREFILL_MAX primary nonempty value, then the Metal alias, then generic DS4_GLM_STREAMING_TOKEN_PREFILL_MAX; parsed by strtoul without requiring full-string consumption; 0 is valid and disables; no digits, ERANGE, or > UINT32_MAX uses the ROCm default 0 Sets the largest non-quality GLM SSD-prefill chunk eligible for token-major/decode-style execution; ROCm defaults to canonical indexed batch prefill (0 disables token-major mode). ds4.c:49682 runtime/rocm DS4_ROCM_GLM_VALUE_PROJECT_WAVE_DECODE Enabled by default when unset. Exact "0" or empty disables; every other nonempty value enables (including false/off/no). It applies only when n_tokens==1; =0 or multi-token input uses the generic per-head batch kernel. Select the validated wave-per-output-row ROCm Q8 GLM value-projection kernel for one-token decode; =0 is the generic-kernel rollback. rocm/ds4_rocm_glm.cuh:2019 -runtime/rocm DS4_ROCM_GRAPH_DUMP_LAYER unsigned layer or all; unset=all layers Filter ROCm graph dumps by layer. ds4.c:16689 -runtime/rocm DS4_ROCM_GRAPH_DUMP_NAME nonempty substring filter; unset=all tensor names Filter ROCm graph dumps by tensor/stage name. ds4.c:16685 +runtime/rocm DS4_ROCM_GRAPH_DUMP_LAYER unsigned layer or all; unset=all layers Filter ROCm graph dumps by layer. ds4.c:16862 +runtime/rocm DS4_ROCM_GRAPH_DUMP_NAME nonempty substring filter; unset=all tensor names Filter ROCm graph dumps by tensor/stage name. ds4.c:16858 runtime/rocm DS4_ROCM_GRAPH_DUMP_NONINVASIVE truthy value under the shared parser; unset lets dumping select conservative kernels Keep production ROCm kernel selection while graph dumping. rocm/ds4_rocm_runtime.cuh:4822 -runtime/rocm DS4_ROCM_GRAPH_DUMP_POS unsigned token position; unset=all positions Filter ROCm graph dumps by position. ds4.c:16696 +runtime/rocm DS4_ROCM_GRAPH_DUMP_POS unsigned token position; unset=all positions Filter ROCm graph dumps by position. ds4.c:16869 runtime/rocm DS4_ROCM_GRAPH_DUMP_PREFIX nonempty output path prefix; unset=off Enable ROCm intermediate graph/tensor dumps. ds4_cuda.cu:397 -runtime/rocm DS4_ROCM_GRAPH_DUMP_TRACE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Emit trace diagnostics for rocm graph dump trace. ds4.c:16726 -runtime/rocm DS4_ROCM_GRAPH_OUTPUT_ROW Nonempty string with a leading strtoul-parsable unsigned value < n_tokens selects that zero-based row. Default, empty, unparsable, or out-of-range selects n_tokens-1. Trailing characters are accepted because full consumption/errno are not checked. A nonempty ROCm value takes precedence; otherwise DS4_METAL_GRAPH_OUTPUT_ROW is a fallback alias. Choose which prefill hidden-state row is sent through the output head to produce logits, primarily for graph/correctness diagnostics. ds4.c:35655 -runtime/rocm DS4_ROCM_GRAPH_PREFILL_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm graph prefill profile. ds4.c:31848 -runtime/rocm DS4_ROCM_GRAPH_PREFILL_SPLIT_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm graph prefill split profile. ds4.c:35578 -runtime/rocm DS4_ROCM_GRAPH_TOKEN_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm graph token profile. ds4.c:31532 -runtime/rocm DS4_ROCM_INDEXER_STAGE_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm indexer stage profile. ds4.c:29092 -runtime/rocm DS4_ROCM_LAYER_STAGE_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm layer stage profile. ds4.c:28931 -runtime/rocm DS4_ROCM_LAYER_STAGE_PROFILE_LAYER layer filter subordinate to DS4_ROCM_LAYER_STAGE_PROFILE; unset or whitespace-only: all layers allowed by the parent flag; otherwise the whitespace-trimmed value must be a complete base-10 strtoul result <= UINT32_MAX equal to the current layer; invalid values match none Restricts the ROCm layer/prefill stage profiler to one layer; it does not enable profiling by itself. ds4.c:28932 +runtime/rocm DS4_ROCM_GRAPH_DUMP_TRACE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Emit trace diagnostics for rocm graph dump trace. ds4.c:16899 +runtime/rocm DS4_ROCM_GRAPH_OUTPUT_ROW Nonempty string with a leading strtoul-parsable unsigned value < n_tokens selects that zero-based row. Default, empty, unparsable, or out-of-range selects n_tokens-1. Trailing characters are accepted because full consumption/errno are not checked. A nonempty ROCm value takes precedence; otherwise DS4_METAL_GRAPH_OUTPUT_ROW is a fallback alias. Choose which prefill hidden-state row is sent through the output head to produce logits, primarily for graph/correctness diagnostics. ds4.c:35845 +runtime/rocm DS4_ROCM_GRAPH_PREFILL_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm graph prefill profile. ds4.c:32038 +runtime/rocm DS4_ROCM_GRAPH_PREFILL_SPLIT_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm graph prefill split profile. ds4.c:35768 +runtime/rocm DS4_ROCM_GRAPH_TOKEN_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm graph token profile. ds4.c:31722 +runtime/rocm DS4_ROCM_INDEXER_STAGE_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm indexer stage profile. ds4.c:29265 +runtime/rocm DS4_ROCM_LAYER_STAGE_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm layer stage profile. ds4.c:29104 +runtime/rocm DS4_ROCM_LAYER_STAGE_PROFILE_LAYER layer filter subordinate to DS4_ROCM_LAYER_STAGE_PROFILE; unset or whitespace-only: all layers allowed by the parent flag; otherwise the whitespace-trimmed value must be a complete base-10 strtoul result <= UINT32_MAX equal to the current layer; invalid values match none Restricts the ROCm layer/prefill stage profiler to one layer; it does not enable profiling by itself. ds4.c:29105 runtime/rocm DS4_ROCM_MOE_DECODE_DOWN_RPB sampled once; nonempty value is parsed by strtoul (a numeric prefix is sufficient), cast to uint32_t, and accepted only if 1/2/4/8/16/32; unset/empty/invalid inherits DS4_ROCM_MOE_DECODE_RPB, with defaults quality=8, non-quality SSD=2, resident=1 Sets output rows (warps) per block for ROCm Q2_K routed-MoE decode down-projection kernels; threads per block are value * 32. rocm/ds4_rocm_runtime.cuh:4842 runtime/rocm DS4_ROCM_MOE_DECODE_GATE_RPB sampled once; nonempty value is parsed by strtoul (a numeric prefix is sufficient), cast to uint32_t, and accepted only if 1/2/4/8/16/32; unset/empty/invalid defaults to 1 in non-quality SSD mode when DS4_ROCM_MOE_DECODE_RPB is unset/empty, otherwise inherits the resolved base RPB Sets output rows (warps) per block for ROCm Q2_K routed-MoE decode gate/up kernels; threads per block are value * 32. rocm/ds4_rocm_runtime.cuh:4836 runtime/rocm DS4_ROCM_MOE_DECODE_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm moe decode profile. rocm/ds4_rocm_moe_launch.cuh:82 runtime/rocm DS4_ROCM_MOE_DECODE_RPB sampled once; nonempty value is parsed by strtoul (a numeric prefix is sufficient), cast to uint32_t, and accepted only if 1/2/4/8/16/32; unset/empty/invalid default: quality=8, non-quality SSD=2, resident=1 Sets the base ROCm Q2_K decode-MoE rows-per-block value inherited by gate/up and down controls, except the automatic SSD gate specialization defaults to 1 when this variable is unset/empty. rocm/ds4_rocm_runtime.cuh:4832 -runtime/rocm DS4_ROCM_MOE_WRITE_CLAMPED_ACT Pure presence sentinel: any defined value, including empty or "0", is active; DS4_METAL_MOE_WRITE_CLAMPED_ACT is an accepted fallback alias. On ROCm the variable is only consumed as a path-admission veto: it disables selected-expert cache/address-table, selected-slot and CPU-router/fused optimized paths. No ROCm call site parses a clamp amount or directly enables a write-clamped kernel. Force shared graph selection away from optimizations incompatible with the clamped-intermediate MoE diagnostic; on ROCm this is a compatibility/rollback gate, not itself a clamped-write implementation. ds4.c:18353 -runtime/rocm DS4_ROCM_Q4_GROUPED_ATTN_A_STATS presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Print counters for rocm q4 grouped attn a stats. rocm/ds4_rocm_q4.cuh:344 -runtime/rocm DS4_ROCM_Q4_PREFILL_TILE8_STATS presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Print counters for rocm q4 prefill tile8 stats. rocm/ds4_rocm_q4.cuh:378 +runtime/rocm DS4_ROCM_MOE_WRITE_CLAMPED_ACT Pure presence sentinel: any defined value, including empty or "0", is active; DS4_METAL_MOE_WRITE_CLAMPED_ACT is an accepted fallback alias. On ROCm the variable is only consumed as a path-admission veto: it disables selected-expert cache/address-table, selected-slot and CPU-router/fused optimized paths. No ROCm call site parses a clamp amount or directly enables a write-clamped kernel. Force shared graph selection away from optimizations incompatible with the clamped-intermediate MoE diagnostic; on ROCm this is a compatibility/rollback gate, not itself a clamped-write implementation. ds4.c:18526 +runtime/rocm DS4_ROCM_Q4_GROUPED_ATTN_A_STATS presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Print counters for rocm q4 grouped attn a stats. rocm/ds4_rocm_q4.cuh:480 +runtime/rocm DS4_ROCM_Q4_PREFILL_TILE8_STATS presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Print counters for rocm q4 prefill tile8 stats. rocm/ds4_rocm_q4.cuh:514 runtime/rocm DS4_ROCM_Q8_DECODE_SHAREDX_64K sampled once; unset: enabled; present empty or exact 0: disabled; every other present value: enabled; effective only for one-token non-prequant Q8_0 matmul with 8192 < in_dim <= 16384 Allows the ROCm shared-input Q8 decode kernel to use up to 64 KiB dynamic LDS for wide inputs; an unsupported/failed LDS launch automatically falls back to the regular kernel. rocm/ds4_rocm_runtime.cuh:4805 -runtime/rocm DS4_ROCM_Q_STAGE_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm q stage profile. ds4.c:29096 -runtime/rocm DS4_ROCM_REQUIRE_Q4_GROUPED_ATTN_A presence fail-closed assertion; also requests candidate unless disabled Require rocm require q4 grouped attn a and fail instead of silently falling back. rocm/ds4_rocm_q4.cuh:575 -runtime/rocm DS4_ROCM_REQUIRE_Q4_PREFILL_TILE8 presence fail-closed assertion for eligible TILE8 calls Require rocm require q4 prefill tile8 and fail instead of silently falling back. rocm/ds4_rocm_q4.cuh:316 -runtime/rocm DS4_ROCM_STREAMING_DECODE_PREFILL_MAX primary nonempty value over the Metal alias; parsed by strtol when it has a numeric prefix (trailing text is accepted); <= 0 disables, values > UINT32_MAX clamp, no numeric prefix uses automatic default: 64 for Flash with uniform Q4_K/MXFP4 experts, 18 for other Pro/Flash, otherwise 0; the disable flag dominates Sets the largest short, non-quality SSD-streaming prefill batch routed through the decode-style path instead of canonical layer-major prefill. ds4.c:31771 -runtime/rocm DS4_ROCM_STREAMING_EXPERT_AUTO_PRELOAD_CAP primary nonempty value over the Metal alias; strict full-string strtoul; valid values > UINT32_MAX clamp, invalid uses 4096, and 0 means no cap (not disabled); when CLI preload is auto/0, unset defaults to cap 4096 except ROCm GLM52, where absent/empty disables automatic preload entirely Caps the number of hot experts synchronously seeded into the SSD-streaming expert cache in automatic preload mode; an explicit CLI preload count bypasses this cap, and setting this variable opts ROCm GLM52 back into auto preload. ds4.c:21281 +runtime/rocm DS4_ROCM_Q_STAGE_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm q stage profile. ds4.c:29269 +runtime/rocm DS4_ROCM_REQUIRE_Q4_GROUPED_ATTN_A presence fail-closed assertion; also requests candidate unless disabled Require rocm require q4 grouped attn a and fail instead of silently falling back. rocm/ds4_rocm_q4.cuh:702 +runtime/rocm DS4_ROCM_REQUIRE_Q4_PREFILL_TILE8 presence fail-closed assertion for eligible TILE8 calls Require rocm require q4 prefill tile8 and fail instead of silently falling back. rocm/ds4_rocm_q4.cuh:452 +runtime/rocm DS4_ROCM_STREAMING_DECODE_PREFILL_MAX primary nonempty value over the Metal alias; parsed by strtol when it has a numeric prefix (trailing text is accepted); <= 0 disables, values > UINT32_MAX clamp, no numeric prefix uses automatic default: 64 for Flash with uniform Q4_K/MXFP4 experts, 18 for other Pro/Flash, otherwise 0; the disable flag dominates Sets the largest short, non-quality SSD-streaming prefill batch routed through the decode-style path instead of canonical layer-major prefill. ds4.c:31961 +runtime/rocm DS4_ROCM_STREAMING_EXPERT_AUTO_PRELOAD_CAP primary nonempty value over the Metal alias; strict full-string strtoul; valid values > UINT32_MAX clamp, invalid uses 4096, and 0 means no cap (not disabled); when CLI preload is auto/0, unset defaults to cap 4096 except ROCm GLM52, where absent/empty disables automatic preload entirely Caps the number of hot experts synchronously seeded into the SSD-streaming expert cache in automatic preload mode; an explicit CLI preload count bypasses this cap, and setting this variable opts ROCm GLM52 back into auto preload. ds4.c:21454 runtime/rocm DS4_ROCM_STREAMING_EXPERT_CACHE_VERBOSE presence flag; unset=off Print verbose ROCm streaming expert-cache seed/load diagnostics. rocm/ds4_rocm_runtime.cuh:2904 -runtime/rocm DS4_ROCM_STREAMING_EXPERT_HOTLIST Nonempty filesystem path; a nonempty ROCm value takes precedence, otherwise DS4_METAL_STREAMING_EXPERT_HOTLIST is a fallback. The file contains whitespace-separated layer expert hits rows; blank/comment lines are ignored, zero-hit rows skipped, malformed/open/read errors fail seeding. Unset/empty uses the built-in Pro/Flash/GLM52 hotlist. Effective only when non-cold SSD hotlist seeding is enabled and cache/preload budget is nonzero. Select a custom ranked expert hotlist used to preseed the streaming resident expert cache before decode. ds4.c:21321 -runtime/rocm DS4_ROCM_STREAMING_EXPERT_HOTLIST_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm streaming expert hotlist profile. ds4.c:32054 -runtime/rocm DS4_ROCM_STREAMING_MAP_TRACE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Emit trace diagnostics for rocm streaming map trace. ds4.c:42453 -runtime/rocm DS4_ROCM_STREAMING_PREFILL_BATCH_SELECTED_ADDR_MAX primary nonempty value over the Metal alias; strtol accepts a numeric prefix; <= 0 returns 0, > UINT32_MAX clamps, invalid uses ROCm default UINT32_MAX for Pro/Flash/GLM52 and 0 otherwise Sets the inclusive upper token-count bound for automatically using selected-expert address-table kernels during eligible non-quality SSD batch prefill; 0 disables automatic selection. ds4.c:18298 -runtime/rocm DS4_ROCM_STREAMING_PREFILL_BATCH_SELECTED_ADDR_MIN primary nonempty value over the Metal alias; strtol accepts a numeric prefix; <= 0 returns 0, > UINT32_MAX clamps, invalid uses ROCm default 2 for Pro/Flash/GLM52 and 0 otherwise Sets the inclusive lower token-count bound for automatically using selected-expert address-table kernels during eligible non-quality SSD batch prefill (the path independently requires more than one token). ds4.c:18321 -runtime/rocm DS4_ROCM_STREAMING_PREFILL_CACHE_SEED_K primary nonempty value over the Metal alias; strict full-string strtoul; unset/empty/invalid: 1; 0 disables; positive values clamp to 64; ignored unless SSD streaming and DS4_ROCM_ENABLE_STREAMING_PREFILL_CACHE_SEED (or Metal alias) is present Chooses how many trailing token router selections per layer are captured from prefill and used to seed the streaming expert cache afterward. ds4.c:21094 -runtime/rocm DS4_ROCM_STREAMING_PREFILL_CACHE_SEED_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm streaming prefill cache seed profile. ds4.c:31963 -runtime/rocm DS4_ROCM_STREAMING_PREFILL_LAYER_MADVISE_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm streaming prefill layer madvise profile. ds4.c:19436 -runtime/rocm DS4_ROCM_STREAMING_PREFILL_LAYER_PAGEIN_NO_OVERLAP Pure presence flag: any defined value, including empty or "0", disables overlap. Default overlap is enabled only if this, PREPARE_NO_OVERLAP, DISABLE_*_PREPARE_OVERLAP, and DISABLE_*_PAGEIN_OVERLAP are all absent. The corresponding DS4_METAL name is an accepted fallback alias. In current code PAGEIN_NO_OVERLAP and PREPARE_NO_OVERLAP are exact synonyms. Serialize SSD-streaming prefill layer page-in/preparation instead of overlapping preparation of upcoming layers. ds4.c:19143 -runtime/rocm DS4_ROCM_STREAMING_PREFILL_LAYER_PAGEIN_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm streaming prefill layer pagein profile. ds4.c:19432 -runtime/rocm DS4_ROCM_STREAMING_PREFILL_LAYER_PAGEIN_THREADS legacy fallback read only when DS4_ROCM_STREAMING_PREFILL_LAYER_PREPARE_THREADS and its Metal alias are absent/empty; strict full-string strtoul; unset/empty across both names: 8; invalid or 0: 1; values > 16 clamp to 16 Sets worker count for full-layer SSD-prefill preparation (page touch, pread, readahead, or madvise) when the canonical PREPARE_THREADS control is not set. ds4.c:19101 -runtime/rocm DS4_ROCM_STREAMING_PREFILL_LAYER_PREAD_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm streaming prefill layer pread profile. ds4.c:19434 -runtime/rocm DS4_ROCM_STREAMING_PREFILL_LAYER_PREPARE_AHEAD primary nonempty value over the Metal alias; strict full-string strtoul; unset/empty: 1; invalid or 0: 1; values > 4 clamp to 4 Sets how many future layer-preparation jobs may be queued concurrently while SSD-prefill preparation overlap is enabled. ds4.c:19155 -runtime/rocm DS4_ROCM_STREAMING_PREFILL_LAYER_PREPARE_NO_OVERLAP Pure presence flag: any defined value, including empty or "0", disables overlap. Default overlap is enabled only if this, PAGEIN_NO_OVERLAP, DISABLE_*_PREPARE_OVERLAP, and DISABLE_*_PAGEIN_OVERLAP are all absent. The corresponding DS4_METAL name is an accepted fallback alias. In current code PREPARE_NO_OVERLAP and PAGEIN_NO_OVERLAP are exact synonyms. Serialize SSD-streaming prefill layer preparation/page-in instead of overlapping preparation of upcoming layers. ds4.c:19141 -runtime/rocm DS4_ROCM_STREAMING_PREFILL_LAYER_PREPARE_THREADS primary nonempty value over the Metal alias; strict full-string strtoul; unset/empty falls back to LAYER_PAGEIN_THREADS, then default 8; invalid or 0: 1; values > 16 clamp to 16 Sets worker count used to split full-layer SSD-prefill page-touch, pread, readahead, or madvise preparation ranges. ds4.c:19097 -runtime/rocm DS4_ROCM_STREAMING_PREFILL_LAYER_READAHEAD_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm streaming prefill layer readahead profile. ds4.c:19438 -runtime/rocm DS4_ROCM_STREAMING_PREFILL_SELECTED_MADVISE_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm streaming prefill selected madvise profile. ds4.c:19182 -runtime/rocm DS4_ROCM_STREAMING_PREFILL_SELECTED_MADVISE_THREADS legacy fallback read only for selected-expert madvise preparation when DS4_ROCM_STREAMING_PREFILL_SELECTED_PREPARE_THREADS and its Metal alias are absent/empty; strict full-string strtoul; if all selected controls are unset it inherits layer preparation threads (default 8); invalid or 0: 1; values > 16 clamp to 16 Sets worker count for selected-expert madvise preparation under its legacy name; non-madvise selected page-in always uses one worker. ds4.c:19119 -runtime/rocm DS4_ROCM_STREAMING_PREFILL_SELECTED_PAGEIN_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm streaming prefill selected pagein profile. ds4.c:19180 -runtime/rocm DS4_ROCM_STREAMING_PREFILL_SELECTED_PREPARE_GAP primary nonempty value over the Metal alias; strict full-string strtoul; unset/empty/invalid: 0; values > 8 clamp to 8 For selected-expert madvise preparation, merges selected expert runs separated by at most this many unselected expert IDs, trading broader hints for fewer ranges. ds4.c:19131 -runtime/rocm DS4_ROCM_STREAMING_PREFILL_SELECTED_PREPARE_THREADS primary nonempty value over the Metal alias; strict full-string strtoul; for selected-expert madvise, unset/empty falls back to SELECTED_MADVISE_THREADS then layer preparation threads (default 8); invalid or 0: 1; values > 16 clamp to 16; non-madvise selected page-in ignores it and uses 1 Sets worker count for selected-expert madvise preparation using the canonical control name. ds4.c:19115 -runtime/rocm DS4_ROCM_STREAMING_PREFILL_SELECTED_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm streaming prefill selected profile. ds4.c:18732 -runtime/rocm DS4_ROCM_STREAMING_PREFILL_SELECTED_READAHEAD_GAP primary nonempty value over the Metal alias; strict full-string strtoul; unset/empty/invalid: 0; values > 8 clamp to 8 For selected-expert file readahead, merges selected expert runs separated by at most this many unselected expert IDs, reducing readahead calls at the cost of hinting extra weights. ds4.c:19804 -runtime/rocm DS4_ROCM_STREAMING_PREFILL_SELECTED_READAHEAD_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm streaming prefill selected readahead profile. ds4.c:19889 +runtime/rocm DS4_ROCM_STREAMING_EXPERT_HOTLIST Nonempty filesystem path; a nonempty ROCm value takes precedence, otherwise DS4_METAL_STREAMING_EXPERT_HOTLIST is a fallback. The file contains whitespace-separated layer expert hits rows; blank/comment lines are ignored, zero-hit rows skipped, malformed/open/read errors fail seeding. Unset/empty uses the built-in Pro/Flash/GLM52 hotlist. Effective only when non-cold SSD hotlist seeding is enabled and cache/preload budget is nonzero. Select a custom ranked expert hotlist used to preseed the streaming resident expert cache before decode. ds4.c:21494 +runtime/rocm DS4_ROCM_STREAMING_EXPERT_HOTLIST_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm streaming expert hotlist profile. ds4.c:32244 +runtime/rocm DS4_ROCM_STREAMING_MAP_TRACE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Emit trace diagnostics for rocm streaming map trace. ds4.c:42643 +runtime/rocm DS4_ROCM_STREAMING_PREFILL_BATCH_SELECTED_ADDR_MAX primary nonempty value over the Metal alias; strtol accepts a numeric prefix; <= 0 returns 0, > UINT32_MAX clamps, invalid uses ROCm default UINT32_MAX for Pro/Flash/GLM52 and 0 otherwise Sets the inclusive upper token-count bound for automatically using selected-expert address-table kernels during eligible non-quality SSD batch prefill; 0 disables automatic selection. ds4.c:18471 +runtime/rocm DS4_ROCM_STREAMING_PREFILL_BATCH_SELECTED_ADDR_MIN primary nonempty value over the Metal alias; strtol accepts a numeric prefix; <= 0 returns 0, > UINT32_MAX clamps, invalid uses ROCm default 2 for Pro/Flash/GLM52 and 0 otherwise Sets the inclusive lower token-count bound for automatically using selected-expert address-table kernels during eligible non-quality SSD batch prefill (the path independently requires more than one token). ds4.c:18494 +runtime/rocm DS4_ROCM_STREAMING_PREFILL_CACHE_SEED_K primary nonempty value over the Metal alias; strict full-string strtoul; unset/empty/invalid: 1; 0 disables; positive values clamp to 64; ignored unless SSD streaming and DS4_ROCM_ENABLE_STREAMING_PREFILL_CACHE_SEED (or Metal alias) is present Chooses how many trailing token router selections per layer are captured from prefill and used to seed the streaming expert cache afterward. ds4.c:21267 +runtime/rocm DS4_ROCM_STREAMING_PREFILL_CACHE_SEED_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm streaming prefill cache seed profile. ds4.c:32153 +runtime/rocm DS4_ROCM_STREAMING_PREFILL_LAYER_MADVISE_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm streaming prefill layer madvise profile. ds4.c:19609 +runtime/rocm DS4_ROCM_STREAMING_PREFILL_LAYER_PAGEIN_NO_OVERLAP Pure presence flag: any defined value, including empty or "0", disables overlap. Default overlap is enabled only if this, PREPARE_NO_OVERLAP, DISABLE_*_PREPARE_OVERLAP, and DISABLE_*_PAGEIN_OVERLAP are all absent. The corresponding DS4_METAL name is an accepted fallback alias. In current code PAGEIN_NO_OVERLAP and PREPARE_NO_OVERLAP are exact synonyms. Serialize SSD-streaming prefill layer page-in/preparation instead of overlapping preparation of upcoming layers. ds4.c:19316 +runtime/rocm DS4_ROCM_STREAMING_PREFILL_LAYER_PAGEIN_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm streaming prefill layer pagein profile. ds4.c:19605 +runtime/rocm DS4_ROCM_STREAMING_PREFILL_LAYER_PAGEIN_THREADS legacy fallback read only when DS4_ROCM_STREAMING_PREFILL_LAYER_PREPARE_THREADS and its Metal alias are absent/empty; strict full-string strtoul; unset/empty across both names: 8; invalid or 0: 1; values > 16 clamp to 16 Sets worker count for full-layer SSD-prefill preparation (page touch, pread, readahead, or madvise) when the canonical PREPARE_THREADS control is not set. ds4.c:19274 +runtime/rocm DS4_ROCM_STREAMING_PREFILL_LAYER_PREAD_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm streaming prefill layer pread profile. ds4.c:19607 +runtime/rocm DS4_ROCM_STREAMING_PREFILL_LAYER_PREPARE_AHEAD primary nonempty value over the Metal alias; strict full-string strtoul; unset/empty: 1; invalid or 0: 1; values > 4 clamp to 4 Sets how many future layer-preparation jobs may be queued concurrently while SSD-prefill preparation overlap is enabled. ds4.c:19328 +runtime/rocm DS4_ROCM_STREAMING_PREFILL_LAYER_PREPARE_NO_OVERLAP Pure presence flag: any defined value, including empty or "0", disables overlap. Default overlap is enabled only if this, PAGEIN_NO_OVERLAP, DISABLE_*_PREPARE_OVERLAP, and DISABLE_*_PAGEIN_OVERLAP are all absent. The corresponding DS4_METAL name is an accepted fallback alias. In current code PREPARE_NO_OVERLAP and PAGEIN_NO_OVERLAP are exact synonyms. Serialize SSD-streaming prefill layer preparation/page-in instead of overlapping preparation of upcoming layers. ds4.c:19314 +runtime/rocm DS4_ROCM_STREAMING_PREFILL_LAYER_PREPARE_THREADS primary nonempty value over the Metal alias; strict full-string strtoul; unset/empty falls back to LAYER_PAGEIN_THREADS, then default 8; invalid or 0: 1; values > 16 clamp to 16 Sets worker count used to split full-layer SSD-prefill page-touch, pread, readahead, or madvise preparation ranges. ds4.c:19270 +runtime/rocm DS4_ROCM_STREAMING_PREFILL_LAYER_READAHEAD_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm streaming prefill layer readahead profile. ds4.c:19611 +runtime/rocm DS4_ROCM_STREAMING_PREFILL_SELECTED_MADVISE_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm streaming prefill selected madvise profile. ds4.c:19355 +runtime/rocm DS4_ROCM_STREAMING_PREFILL_SELECTED_MADVISE_THREADS legacy fallback read only for selected-expert madvise preparation when DS4_ROCM_STREAMING_PREFILL_SELECTED_PREPARE_THREADS and its Metal alias are absent/empty; strict full-string strtoul; if all selected controls are unset it inherits layer preparation threads (default 8); invalid or 0: 1; values > 16 clamp to 16 Sets worker count for selected-expert madvise preparation under its legacy name; non-madvise selected page-in always uses one worker. ds4.c:19292 +runtime/rocm DS4_ROCM_STREAMING_PREFILL_SELECTED_PAGEIN_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm streaming prefill selected pagein profile. ds4.c:19353 +runtime/rocm DS4_ROCM_STREAMING_PREFILL_SELECTED_PREPARE_GAP primary nonempty value over the Metal alias; strict full-string strtoul; unset/empty/invalid: 0; values > 8 clamp to 8 For selected-expert madvise preparation, merges selected expert runs separated by at most this many unselected expert IDs, trading broader hints for fewer ranges. ds4.c:19304 +runtime/rocm DS4_ROCM_STREAMING_PREFILL_SELECTED_PREPARE_THREADS primary nonempty value over the Metal alias; strict full-string strtoul; for selected-expert madvise, unset/empty falls back to SELECTED_MADVISE_THREADS then layer preparation threads (default 8); invalid or 0: 1; values > 16 clamp to 16; non-madvise selected page-in ignores it and uses 1 Sets worker count for selected-expert madvise preparation using the canonical control name. ds4.c:19288 +runtime/rocm DS4_ROCM_STREAMING_PREFILL_SELECTED_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm streaming prefill selected profile. ds4.c:18905 +runtime/rocm DS4_ROCM_STREAMING_PREFILL_SELECTED_READAHEAD_GAP primary nonempty value over the Metal alias; strict full-string strtoul; unset/empty/invalid: 0; values > 8 clamp to 8 For selected-expert file readahead, merges selected expert runs separated by at most this many unselected expert IDs, reducing readahead calls at the cost of hinting extra weights. ds4.c:19977 +runtime/rocm DS4_ROCM_STREAMING_PREFILL_SELECTED_READAHEAD_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm streaming prefill selected readahead profile. ds4.c:20062 runtime/rocm DS4_ROCM_STREAM_CACHE_LAYER_STATS presence flag; unset=off Collect per-layer ROCm streaming cache statistics; also enables aggregate stats. rocm/ds4_rocm_runtime.cuh:390 runtime/rocm DS4_ROCM_STREAM_CACHE_STATS presence flag; unset=off unless layer stats are enabled Collect aggregate ROCm streaming cache statistics. rocm/ds4_rocm_runtime.cuh:398 runtime/rocm DS4_ROCM_STREAM_EVICT_PAST_LAYERS_FIRST nonempty and not 0 enables; unset/empty/0=off Prefer evicting cached experts from already-processed layers. rocm/ds4_rocm_runtime.cuh:406 @@ -1050,16 +1054,16 @@ runtime/rocm DS4_ROCM_STREAM_READ_WORKERS integer; default DS4_ROCM_STREAM_READ_ runtime/server DS4_SERVER_BATCH_LOG Pure presence flag read once when the decode worker starts; default off. Any defined value, including empty or "0", logs one record per coalesced decode batch with count, elapsed milliseconds and ok/error status. Observe server-side decode coalescing size, latency and result without changing batching behavior. ds4_server.c:11090 runtime/server DS4_SERVER_DECODE_COALESCE_US integer 0..100000 microseconds; default 2000; 0 disables wait Control server micro-batch coalescing delay. ds4_server.c:11069 runtime/ssd DS4_SSD_AUTO_CACHE_PCT integer 50..95; default 80 Choose the RAM percentage used by automatic SSD expert-cache planning. ds4_ssd.c:81 -runtime/test-hook DS4_TEST_METAL_EXACTN_ORACLE presence flag compiled only with DS4_TEST_HOOKS; unset is off; any defined value enables Force allocation of the Metal exact-N verifier/oracle workspace in test builds. ds4.c:61801 -runtime/tp DS4_TP_ABLATE comma/list string matched for hcpre,router,kv,compidx; unset=no ablation; must match on both ranks Skip named TP encode chains for timing; output is semantically wrong. ds4.c:22355 -runtime/tp DS4_TP_EVENT_GATES presence flag; unset uses lower-latency slab flag gates when available Fall back to Metal shared-event arrival gates. ds4_metal.m:10769 -runtime/tp DS4_TP_GATE_PROFILE presence diagnostic flag; unset=off Collect timing/profile diagnostics for tp gate profile. ds4_metal.m:10661 +runtime/test-hook DS4_TEST_METAL_EXACTN_ORACLE presence flag compiled only with DS4_TEST_HOOKS; unset is off; any defined value enables Force allocation of the Metal exact-N verifier/oracle workspace in test builds. ds4.c:61991 +runtime/tp DS4_TP_ABLATE comma/list string matched for hcpre,router,kv,compidx; unset=no ablation; must match on both ranks Skip named TP encode chains for timing; output is semantically wrong. ds4.c:22520 +runtime/tp DS4_TP_EVENT_GATES presence flag; unset uses lower-latency slab flag gates when available Fall back to Metal shared-event arrival gates. ds4_metal.m:10768 +runtime/tp DS4_TP_GATE_PROFILE presence diagnostic flag; unset=off Collect timing/profile diagnostics for tp gate profile. ds4_metal.m:10662 runtime/tp DS4_TP_GATE_TRACE presence diagnostic flag; unset=off Emit trace diagnostics for tp gate trace. ds4_tp.c:911 -runtime/tp DS4_TP_KEEPALIVE_ITERS atoi unsigned iteration count; default 1200000 Tune work per Metal TP keep-alive dispatch. ds4_metal.m:10625 -runtime/tp DS4_TP_KEEPALIVE_TGS integer 1..2048; invalid/out of range uses 1 Tune threadgroups per Metal TP keep-alive dispatch. ds4_metal.m:10611 -runtime/tp DS4_TP_NO_KEEPALIVE presence flag; unset starts Metal TP keep-alive Disable the Metal TP GPU keep-alive worker. ds4_metal.m:10797 -runtime/tp DS4_TP_PREFILL_SPLIT_MIN atoi token threshold; default 32; values below 2 clamp to 2 Set when TP prefill row-splits the replicated shared expert. ds4.c:29019 -runtime/tp DS4_TP_SUBGATE_PIPELINE nonempty integer; nonzero enables; default off; must match on both ranks Enable TP prefill sub-chunk gate pipelining. ds4.c:29033 +runtime/tp DS4_TP_KEEPALIVE_ITERS atoi unsigned iteration count; default 1200000 Tune work per Metal TP keep-alive dispatch. ds4_metal.m:10626 +runtime/tp DS4_TP_KEEPALIVE_TGS integer 1..2048; invalid/out of range uses 1 Tune threadgroups per Metal TP keep-alive dispatch. ds4_metal.m:10612 +runtime/tp DS4_TP_NO_KEEPALIVE presence flag; unset starts Metal TP keep-alive Disable the Metal TP GPU keep-alive worker. ds4_metal.m:10798 +runtime/tp DS4_TP_PREFILL_SPLIT_MIN atoi token threshold; default 32; values below 2 clamp to 2 Set when TP prefill row-splits the replicated shared expert. ds4.c:29192 +runtime/tp DS4_TP_SUBGATE_PIPELINE nonempty integer; nonzero enables; default off; must match on both ranks Enable TP prefill sub-chunk gate pipelining. ds4.c:29206 runtime/tp DS4_TP_TIMEOUT_SEC atoi seconds stored unsigned; default DS4_TP_DEFAULT_TIMEOUT_SEC Override TP control/data socket operation timeout. ds4_tp.c:1329 runtime/web DS4_CHROME executable path; unset auto-detects Chrome/Chromium via standard paths and PATH Select the browser executable used by web tooling. ds4_web.c:1014 script/downloader DS4_GGUF_DIR path; default repository gguf/ directory Choose the model download directory. download_model.sh:23 @@ -1095,7 +1099,7 @@ test-only DS4_TEST_LOGPROB_AUTO_METAL presence flag; unset forces DS4_METAL_DISA test-only DS4_TEST_LONG_PROMPT nonempty readable prompt-file path; unset/empty defaults to tests/long_context_story_prompt.txt Selects the rendered story prompt for the long-context fact-recall test. tests/ds4_test.c:6690 test-only DS4_TEST_LONG_WORDS atoi integer; unset/empty/nonnumeric defaults to 0; valid range is 0..DS4_TEST_CONTEXT-128 and numeric prefixes are accepted Adds repeated words to alternating CUDA session-batch prompts to exercise long-prefill rows. tests/test_cuda_session_batch.c:135 test-only DS4_TEST_METAL_EXACTN_BATCH_HEAD nonempty value other than exact 0 enables; unset/empty/0 disables; false/off also enable Enables the Metal exact-N batch-head path and requires its attempt/use counters for every eligible oracle case. tests/test_metal_exactn_oracle.c:401 -test-only DS4_TEST_METAL_EXACTN_ORACLE presence flag compiled only with DS4_TEST_HOOKS; absent from normal production builds Force allocation of the exact-N Metal verifier/oracle workspace in tests. ds4.c:61801 +test-only DS4_TEST_METAL_EXACTN_ORACLE presence flag compiled only with DS4_TEST_HOOKS; absent from normal production builds Force allocation of the exact-N Metal verifier/oracle workspace in tests. ds4.c:61991 test-only DS4_TEST_MIXED_INITIAL integer 128..context-1; default 128 Set the initial prefill length for the CUDA mixed-batch oracle. tests/test_cuda_mixed_batch.c:111 test-only DS4_TEST_MIXED_QUANTUM integer 1..context-1; default 128 Set the number of prompt tokens added per CUDA mixed-batch round. tests/test_cuda_mixed_batch.c:113 test-only DS4_TEST_MIXED_ROUNDS integer 1..64; default 3 Set the number of CUDA mixed-batch oracle rounds. tests/test_cuda_mixed_batch.c:115 @@ -1109,7 +1113,7 @@ test-only DS4_TEST_Q4_STREAM_TIMING_BLOCKS integer 5..MAX_TIMING_BLOCKS; default test-only DS4_TEST_Q4_STREAM_TIMING_ITERS integer 1..10000; default 20 Set timing iterations for the Metal Q4 stream oracle. tests/test_metal_q4_streams.c:741 test-only DS4_TEST_Q4_STREAM_WARMUP integer 1..64; default 1 Set warmup iterations for the Metal Q4 stream oracle. tests/test_metal_q4_streams.c:732 test-only DS4_TEST_REQUIRE_MODEL nonempty value other than exact 0 requires a readable model; unset/empty/0 permits a skip; false/off count as required Turns a missing Metal exact-N oracle model from a developer skip into a release-gate failure. tests/test_metal_exactn_oracle.c:390 -test-only DS4_TEST_REQUIRE_ROCM_DEVICE nonempty value other than exact 0 requires a visible ROCm device; unset/empty/0 returns the fixture skip code; false/off count as required Turns absence of a ROCm device from a skip into failure for the ROCm Q4 oracle. tests/test_rocm_q4_dense_pair.cpp:1483 +test-only DS4_TEST_REQUIRE_ROCM_DEVICE nonempty value other than exact 0 requires a visible ROCm device; unset/empty/0 returns the fixture skip code; false/off count as required Turns absence of a ROCm device from a skip into failure for the ROCm Q4 oracle. tests/test_rocm_q4_dense_pair.cpp:1569 test-only DS4_TEST_SERVER_PREFILL presence flag; unset: normal prefill; any defined value including empty or 0 installs a no-op display-progress callback Exercises the progress-split prefill path used by ds4-server in CUDA session-batch and control sessions. tests/test_cuda_session_batch.c:110 test-only DS4_TEST_SESSION_BATCH_ARM arbitrary nonempty label; unset/empty defaults to unspecified; it is logged only and does not alter execution Labels the Metal session-batch experiment arm in setup diagnostics. tests/test_metal_session_batch.c:153 test-only DS4_TEST_SESSION_BATCH_TIMING nonempty boolean; unset/empty/exact 0 disables; every other value enables Print timing data from the Metal session-batch oracle. tests/test_metal_session_batch.c:150 From 5589ac13d785ab26eebd2fe3e5bf8e38a6507b52 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Sun, 23 Aug 2026 15:14:45 +0200 Subject: [PATCH 075/189] fix(cuda): harden MMQ review edge cases --- ENVIRONMENT_VARIABLES.md | 50 ++++++++++---------- cuda/mmq/ds4_mmq.cu | 76 +++++++++++++++++++++++-------- scripts/environment_variables.tsv | 50 ++++++++++---------- 3 files changed, 106 insertions(+), 70 deletions(-) diff --git a/ENVIRONMENT_VARIABLES.md b/ENVIRONMENT_VARIABLES.md index 2563997f7..e808b2f5b 100644 --- a/ENVIRONMENT_VARIABLES.md +++ b/ENVIRONMENT_VARIABLES.md @@ -615,8 +615,8 @@ and **19 tool/wrapper entries**. | `DS4_CUDA_ENABLE_HC_NORM_MIX_FUSE` | nonempty opt-in, default off; only exact 0 disables; the F32/F16 activation mode follows the selected standalone matmul path; disable/serial/alternate flags can veto | Enable and select the fused HC RMSNorm-plus-mix one-token implementation. | [ds4_cuda.cu:20902](ds4_cuda.cu#L20902) | | `DS4_CUDA_ENABLE_IQ2_XXS_SSD_PREFILL_MMQ` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Enable the CUDA IQ2 XXS SSD prefill MMQ experimental path. | [ds4_cuda.cu:4625](ds4_cuda.cu#L4625) | | `DS4_CUDA_ENABLE_Q4_ATTN_OUT_HC_FUSE` | value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on | Opt in to the fused Q4 attention-output/HC expansion path. | [ds4_cuda.cu:37375](ds4_cuda.cu#L37375) | -| `DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_BATCH` | value-aware opt-in, default off; nonempty value other than exact 0 enables; rollback wins | Enable flattened grouped attention-A MMQ for two-to-eight-token GB10 batches. | [cuda/mmq/ds4_mmq.cu:4280](cuda/mmq/ds4_mmq.cu#L4280) | -| `DS4_CUDA_ENABLE_Q4_K1024_PERSISTENT` | presence flag, default off; any defined value including 0 requests the path; rollback wins | Enable the GB10 persistent-CTA kernel for M=32768, N=1, K=1024 Q4. | [cuda/mmq/ds4_mmq.cu:3882](cuda/mmq/ds4_mmq.cu#L3882) | +| `DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_BATCH` | value-aware opt-in, default off; nonempty value other than exact 0 enables; rollback wins | Enable flattened grouped attention-A MMQ for two-to-eight-token GB10 batches. | [cuda/mmq/ds4_mmq.cu:4313](cuda/mmq/ds4_mmq.cu#L4313) | +| `DS4_CUDA_ENABLE_Q4_K1024_PERSISTENT` | presence flag, default off; any defined value including 0 requests the path; rollback wins | Enable the GB10 persistent-CTA kernel for M=32768, N=1, K=1024 Q4. | [cuda/mmq/ds4_mmq.cu:3915](cuda/mmq/ds4_mmq.cu#L3915) | | `DS4_CUDA_ENABLE_Q8_FOLD` | strict flag, default off; only exact value 1 enables; overridden by DS4_CUDA_NO_Q8_FOLD | Enable one-shot producer-to-consumer reuse of freshly quantized Q8_1 data. | [ds4_cuda.cu:785](ds4_cuda.cu#L785) | | `DS4_CUDA_ENABLE_STREAMING_EXPERT_PERSISTENT_CACHE` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Enable streaming expert persistent cache in CUDA SSD streaming. | [ds4_cuda.cu:4111](ds4_cuda.cu#L4111) | | `DS4_CUDA_ENABLE_STREAMING_SELECTED_BATCH_IO` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Enable streaming selected batch I/O in CUDA SSD streaming. | [ds4_cuda.cu:4732](ds4_cuda.cu#L4732) | @@ -660,7 +660,7 @@ and **19 tool/wrapper entries**. | `DS4_CUDA_MIXED_ROUTED_MAX_PREFILL` | integer rows 0..UINT32_MAX; default/invalid 512 | Set the maximum prefill rows admitted to the mixed routed-MoE path. | [ds4.c:70190](ds4.c#L70190) | | `DS4_CUDA_MIXED_ROUTED_SCATTER` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Use scattered row handling in mixed routed-MoE execution. | [ds4.c:69651](ds4.c#L69651) | | `DS4_CUDA_MMQ` | boolean-ish, default on; any value beginning with 0 disables; quality mode and multi-GPU disable normal MMQ tier (MXFP4 path differs) | Control the vendored CUDA MMQ prefill tier. | [ds4_cuda.cu:1722](ds4_cuda.cu#L1722) | -| `DS4_CUDA_MMQ_Q81_PERSISTENT` | strict boolean, default off; accepts 1/on/true/yes and 0/off/false/no in listed lower/upper-case forms; unknown values are off | Reuse a persistent Q8_1 MMQ scratch arena on supported GB10 devices. | [cuda/mmq/ds4_mmq.cu:144](cuda/mmq/ds4_mmq.cu#L144) | +| `DS4_CUDA_MMQ_Q81_PERSISTENT` | strict boolean, default off; accepts 1/on/true/yes and 0/off/false/no in listed lower/upper-case forms; unknown values are off | Reuse a persistent Q8_1 MMQ scratch arena on supported GB10 devices. | [cuda/mmq/ds4_mmq.cu:152](cuda/mmq/ds4_mmq.cu#L152) | | `DS4_CUDA_MMQ_X_MAX` | integer >=8; rounded down to multiple of 8 and only lowers the hardware base; invalid/unset = hardware base | Cap the MMQ X tile-width selector for architecture tuning. | [cuda/mmq/mmq.cuh:127](cuda/mmq/mmq.cuh#L127) | | `DS4_CUDA_MODEL_COPY_CHUNK_MB` | positive integer MiB; default 64; clamped 16..4096 | Set the chunk size used for CUDA model copying. | [ds4_cuda.cu:2827](ds4_cuda.cu#L2827) | | `DS4_CUDA_MODEL_COPY_VERBOSE` | presence diagnostic flag; default off; any defined value including 0 enables | Print periodic progress while copying the model to device memory. | [ds4_cuda.cu:6537](ds4_cuda.cu#L6537) | @@ -756,16 +756,16 @@ and **19 tool/wrapper entries**. | `DS4_CUDA_NO_IQ2_XXS_SSD_PREFILL_MMQ` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Disable the CUDA IQ2 XXS SSD prefill MMQ optimization/path. | [ds4_cuda.cu:4629](ds4_cuda.cu#L4629) | | `DS4_CUDA_NO_MODEL_COPY` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA model copy optimization/path. | [ds4_cuda.cu:6472](ds4_cuda.cu#L6472) | | `DS4_CUDA_NO_MODEL_PREFETCH` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA model prefetch optimization/path. | [ds4_cuda.cu:2739](ds4_cuda.cu#L2739) | -| `DS4_CUDA_NO_MOE_DEDUP` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA MoE dedup optimization/path. | [cuda/mmq/ds4_mmq.cu:6145](cuda/mmq/ds4_mmq.cu#L6145) | +| `DS4_CUDA_NO_MOE_DEDUP` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA MoE dedup optimization/path. | [cuda/mmq/ds4_mmq.cu:6182](cuda/mmq/ds4_mmq.cu#L6182) | | `DS4_CUDA_NO_ORDERED_F16_MATMUL` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the ordered F16 matmul CUDA F16 path. | [ds4_cuda.cu:20811](ds4_cuda.cu#L20811) | | `DS4_CUDA_NO_PARALLEL_ROUTER_SELECT` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA parallel router select optimization/path. | [ds4_cuda.cu:24491](ds4_cuda.cu#L24491) | -| `DS4_CUDA_NO_Q4_DENSE_SCRATCH` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q4 dense scratch CUDA Q4 optimization. | [cuda/mmq/ds4_mmq.cu:3963](cuda/mmq/ds4_mmq.cu#L3963) | -| `DS4_CUDA_NO_Q4_GB10_FAST` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the GB10-specific Q4 fast-path family. | [cuda/mmq/ds4_mmq.cu:3885](cuda/mmq/ds4_mmq.cu#L3885) | -| `DS4_CUDA_NO_Q4_GROUPED_ATTN_A` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q4 grouped attn a CUDA Q4 optimization. | [cuda/mmq/ds4_mmq.cu:4272](cuda/mmq/ds4_mmq.cu#L4272) | -| `DS4_CUDA_NO_Q4_GROUPED_ATTN_A_BATCH` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q4 grouped attn a batch CUDA Q4 optimization. | [cuda/mmq/ds4_mmq.cu:4282](cuda/mmq/ds4_mmq.cu#L4282) | -| `DS4_CUDA_NO_Q4_K1024_PERSISTENT` | presence kill switch, default off; any defined value including 0 disables | Disable the Q4 K1024 persistent CUDA Q4 optimization. | [cuda/mmq/ds4_mmq.cu:3884](cuda/mmq/ds4_mmq.cu#L3884) | -| `DS4_CUDA_NO_Q8_ALIGNED_DENSE_SCRATCH` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q8 aligned dense scratch CUDA Q8 optimization. | [cuda/mmq/ds4_mmq.cu:5553](cuda/mmq/ds4_mmq.cu#L5553) | -| `DS4_CUDA_NO_Q8_ALIGNED_PERSISTENT` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q8 aligned persistent CUDA Q8 optimization. | [cuda/mmq/ds4_mmq.cu:5385](cuda/mmq/ds4_mmq.cu#L5385) | +| `DS4_CUDA_NO_Q4_DENSE_SCRATCH` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q4 dense scratch CUDA Q4 optimization. | [cuda/mmq/ds4_mmq.cu:3996](cuda/mmq/ds4_mmq.cu#L3996) | +| `DS4_CUDA_NO_Q4_GB10_FAST` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the GB10-specific Q4 fast-path family. | [cuda/mmq/ds4_mmq.cu:3918](cuda/mmq/ds4_mmq.cu#L3918) | +| `DS4_CUDA_NO_Q4_GROUPED_ATTN_A` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q4 grouped attn a CUDA Q4 optimization. | [cuda/mmq/ds4_mmq.cu:4305](cuda/mmq/ds4_mmq.cu#L4305) | +| `DS4_CUDA_NO_Q4_GROUPED_ATTN_A_BATCH` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q4 grouped attn a batch CUDA Q4 optimization. | [cuda/mmq/ds4_mmq.cu:4315](cuda/mmq/ds4_mmq.cu#L4315) | +| `DS4_CUDA_NO_Q4_K1024_PERSISTENT` | presence kill switch, default off; any defined value including 0 disables | Disable the Q4 K1024 persistent CUDA Q4 optimization. | [cuda/mmq/ds4_mmq.cu:3917](cuda/mmq/ds4_mmq.cu#L3917) | +| `DS4_CUDA_NO_Q8_ALIGNED_DENSE_SCRATCH` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q8 aligned dense scratch CUDA Q8 optimization. | [cuda/mmq/ds4_mmq.cu:5588](cuda/mmq/ds4_mmq.cu#L5588) | +| `DS4_CUDA_NO_Q8_ALIGNED_PERSISTENT` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q8 aligned persistent CUDA Q8 optimization. | [cuda/mmq/ds4_mmq.cu:5420](cuda/mmq/ds4_mmq.cu#L5420) | | `DS4_CUDA_NO_Q8_BATCH_EXACT_TOK2` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q8 batch exact tok2 CUDA Q8 optimization. | [ds4_cuda.cu:19852](ds4_cuda.cu#L19852) | | `DS4_CUDA_NO_Q8_BATCH_TOK4` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q8 batch tok4 CUDA Q8 optimization. | [ds4_cuda.cu:19805](ds4_cuda.cu#L19805) | | `DS4_CUDA_NO_Q8_BATCH_TOK8` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q8 batch tok8 CUDA Q8 optimization. | [ds4_cuda.cu:19788](ds4_cuda.cu#L19788) | @@ -801,8 +801,8 @@ and **19 tool/wrapper entries**. | `DS4_CUDA_NO_VERIFY_DECODE2_SPLIT_TOP1` | value-aware kill switch; default off; nonempty value other than exact 0 disables | Disable the CUDA verify decode2 split top1 optimization/path. | [ds4.c:17223](ds4.c#L17223) | | `DS4_CUDA_NO_WARP_ROUTER_SELECT` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA warp router select optimization/path. | [ds4_cuda.cu:24490](ds4_cuda.cu#L24490) | | `DS4_CUDA_NO_WINDOW_ATTENTION` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA window attention optimization/path. | [ds4_cuda.cu:23050](ds4_cuda.cu#L23050) | -| `DS4_CUDA_NSYS_PREFILL_START_POS` | nonempty-string flag, default off; any nonempty value enables MMQ NVTX ranges (the value is not parsed as a position) | Enable MMQ NVTX annotations intended for Nsight Systems prefill capture. | [cuda/mmq/ds4_mmq.cu:48](cuda/mmq/ds4_mmq.cu#L48) | -| `DS4_CUDA_NVTX` | strict flag, default off; only exact value 1 enables (a nonempty NSYS variable also enables ranges) | Enable NVTX ranges around MMQ work. | [cuda/mmq/ds4_mmq.cu:47](cuda/mmq/ds4_mmq.cu#L47) | +| `DS4_CUDA_NSYS_PREFILL_START_POS` | nonempty-string flag, default off; any nonempty value enables MMQ NVTX ranges (the value is not parsed as a position) | Enable MMQ NVTX annotations intended for Nsight Systems prefill capture. | [cuda/mmq/ds4_mmq.cu:49](cuda/mmq/ds4_mmq.cu#L49) | +| `DS4_CUDA_NVTX` | strict flag, default off; only exact value 1 enables (a nonempty NSYS variable also enables ranges) | Enable NVTX ranges around MMQ work. | [cuda/mmq/ds4_mmq.cu:48](cuda/mmq/ds4_mmq.cu#L48) | | `DS4_CUDA_OUTPUT_FUSED_TOP1` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Fuse output projection with top-1 selection in greedy decode. | [ds4.c:17215](ds4.c#L17215) | | `DS4_CUDA_PREFILL_PIPELINE` | boolean, default follows CUDA TP decode; nonempty exact 0 disables, any other nonempty value enables | Control the CUDA multi-tier prefill pipeline. | [ds4.c:17454](ds4.c#L17454) | | `DS4_CUDA_PREFILL_PIPELINE_MB` | positive integer rows; default/invalid 512 | Set prefill-pipeline microbatch rows. | [ds4.c:17469](ds4.c#L17469) | @@ -812,15 +812,15 @@ and **19 tool/wrapper entries**. | `DS4_CUDA_Q4_ATTN_OUT_HC_ORACLE` | value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on | Compare fused Q4 attention-output/HC expansion with the canonical path and retain canonical output. | [ds4_cuda.cu:1475](ds4_cuda.cu#L1475) | | `DS4_CUDA_Q4_ATTN_OUT_HC_Q8K_EXPERIMENT` | value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on | Enable the experimental Q8_K-based Q4 attention-output/HC fusion. | [ds4_cuda.cu:37377](ds4_cuda.cu#L37377) | | `DS4_CUDA_Q4_GROUPED_ATTN_A_ORACLE` | value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on | Compare grouped attention-A against the canonical per-group result. | [ds4_cuda.cu:1477](ds4_cuda.cu#L1477) | -| `DS4_CUDA_Q4_K1024_PERSISTENT_ORACLE` | value-aware flag, default off; nonempty value other than exact 0 enables and implies candidate admission | Bitwise-compare the exact-shape persistent Q4 K1024 kernel with canonical MMVQ and retain canonical output. | [cuda/mmq/ds4_mmq.cu:3718](cuda/mmq/ds4_mmq.cu#L3718) | -| `DS4_CUDA_Q4_K1024_PERSISTENT_STATS` | value-aware flag, default off; nonempty value other than exact 0 enables | Print exact-shape persistent Q4 K1024 dispatch counters at exit. | [cuda/mmq/ds4_mmq.cu:3717](cuda/mmq/ds4_mmq.cu#L3717) | +| `DS4_CUDA_Q4_K1024_PERSISTENT_ORACLE` | value-aware flag, default off; nonempty value other than exact 0 enables and implies candidate admission | Bitwise-compare the exact-shape persistent Q4 K1024 kernel with canonical MMVQ and retain canonical output. | [cuda/mmq/ds4_mmq.cu:3748](cuda/mmq/ds4_mmq.cu#L3748) | +| `DS4_CUDA_Q4_K1024_PERSISTENT_STATS` | value-aware flag, default off; nonempty value other than exact 0 enables | Print exact-shape persistent Q4 K1024 dispatch counters at exit. | [cuda/mmq/ds4_mmq.cu:3747](cuda/mmq/ds4_mmq.cu#L3747) | | `DS4_CUDA_Q8_F16_ALL` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Control the Q8 F16 all CUDA quantized-matmul/cache optimization. | [ds4_cuda.cu:2358](ds4_cuda.cu#L2358) | | `DS4_CUDA_Q8_F16_CACHE_MB` | unsigned integer MiB, full-string parse; default unlimited; 0 disables this cache | Limit the selective Q8-to-F16 derived-weight cache. | [ds4_cuda.cu:2218](ds4_cuda.cu#L2218) | | `DS4_CUDA_Q8_F16_CACHE_RESERVE_MB` | unsigned integer MiB, full-string parse; default is VRAM-dependent (>=112 GiB: 512; >=40 GiB: max(768,1%); smaller: max(4096,5%)) | Reserve free VRAM when growing the selective Q8-to-F16 cache. | [ds4_cuda.cu:2224](ds4_cuda.cu#L2224) | | `DS4_CUDA_Q8_F32_ALL` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Control the Q8 F32 all CUDA quantized-matmul/cache optimization. | [ds4_cuda.cu:2412](ds4_cuda.cu#L2412) | | `DS4_CUDA_Q8_F32_LARGE` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Control the Q8 F32 large CUDA quantized-matmul/cache optimization. | [ds4_cuda.cu:2416](ds4_cuda.cu#L2416) | | `DS4_CUDA_Q8_F32_PRELOAD` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Control the Q8 F32 preload CUDA quantized-matmul/cache optimization. | [ds4_cuda.cu:8597](ds4_cuda.cu#L8597) | -| `DS4_CUDA_Q8_FOLD_ORACLE` | strict flag, default off; only exact value 1 enables | Compare folded Q8_1 bytes and consumer outputs against canonical work while retaining canonical results. | [cuda/mmq/ds4_mmq.cu:383](cuda/mmq/ds4_mmq.cu#L383) | +| `DS4_CUDA_Q8_FOLD_ORACLE` | strict flag, default off; only exact value 1 enables | Compare folded Q8_1 bytes and consumer outputs against canonical work while retaining canonical results. | [cuda/mmq/ds4_mmq.cu:391](cuda/mmq/ds4_mmq.cu#L391) | | `DS4_CUDA_Q8_HC_EXPAND_FUSED` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, other nonempty values force fused | Force the fused Q8 shared-down/HC expansion path. | [ds4_cuda.cu:2084](ds4_cuda.cu#L2084) | | `DS4_CUDA_Q8_HC_EXPAND_STATS` | false-like-aware flag, default off; 0/false/no/off is off, other nonempty values print report | Print Q8 shared-down/HC policy and dispatch counters at exit. | [ds4_cuda.cu:2090](ds4_cuda.cu#L2090) | | `DS4_CUDA_Q8_NO_ALIGNED` | value-aware kill switch, default off; nonempty value other than exact 0 disables aligned Q8 kernels | Disable aligned Q8 CUDA matmul kernels. | [ds4_cuda.cu:1021](ds4_cuda.cu#L1021) | @@ -829,7 +829,7 @@ and **19 tool/wrapper entries**. | `DS4_CUDA_Q_NORM_ROPE_FUSE` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control or tune the CUDA q norm rope fuse path. | [ds4.c:17418](ds4.c#L17418) | | `DS4_CUDA_REQUIRE_IQ2_XXS_SSD_PREFILL_MMQ` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Require the CUDA IQ2 XXS SSD prefill MMQ path; fail closed when unavailable. | [ds4_cuda.cu:4631](ds4_cuda.cu#L4631) | | `DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_BATCH` | value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on | Fail if grouped batched attention-A cannot be used. | [ds4_cuda.cu:40198](ds4_cuda.cu#L40198) | -| `DS4_CUDA_REQUIRE_Q4_K1024_PERSISTENT` | presence flag, default off; any defined value including 0 makes ineligible candidate fail closed | Fail when the exact Q4 K1024 persistent candidate is unavailable instead of using MMVQ. | [cuda/mmq/ds4_mmq.cu:3906](cuda/mmq/ds4_mmq.cu#L3906) | +| `DS4_CUDA_REQUIRE_Q4_K1024_PERSISTENT` | presence flag, default off; any defined value including 0 makes ineligible candidate fail closed | Fail when the exact Q4 K1024 persistent candidate is unavailable instead of using MMVQ. | [cuda/mmq/ds4_mmq.cu:3939](cuda/mmq/ds4_mmq.cu#L3939) | | `DS4_CUDA_REQUIRE_STREAMING_EXPERT_PERSISTENT_CACHE` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Require streaming expert persistent cache in CUDA SSD streaming; fail closed when unavailable. | [ds4_cuda.cu:4117](ds4_cuda.cu#L4117) | | `DS4_CUDA_REQUIRE_STREAMING_SELECTED_BATCH_IO` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Require streaming selected batch I/O in CUDA SSD streaming; fail closed when unavailable. | [ds4_cuda.cu:4738](ds4_cuda.cu#L4738) | | `DS4_CUDA_REQUIRE_STREAMING_SELECTED_EVENT_PIPELINE` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Require streaming selected event pipeline in CUDA SSD streaming; fail closed when unavailable. | [ds4_cuda.cu:4870](ds4_cuda.cu#L4870) | @@ -1173,15 +1173,15 @@ and **19 tool/wrapper entries**. | `DS4_LOCK_FILE` | path string; default /tmp/ds4.lock | Override the single-instance lock file. | [ds4.c:51999](ds4.c#L51999) | | `DS4_MMID_CASE1` | boolean-ish cached flag; default on; a value starting with 0 disables | Disable the single-expert MM-IDs specialized fast path for comparison. | [cuda/mmq/mmid.cu:290](cuda/mmq/mmid.cu#L290) | | `DS4_MMID_LARGE` | boolean-ish cached flag; default on; a value starting with 0 disables | Control the large-N global-memory MM-IDs path used beyond shared-memory capacity. | [cuda/mmq/mmid.cu:245](cuda/mmq/mmid.cu#L245) | -| `DS4_MMQ_D2R` | boolean-ish cached flag; default on; a value starting with 0 disables | Control the direct-to-register Q2_K MoE down path. | [cuda/mmq/ds4_mmq.cu:505](cuda/mmq/ds4_mmq.cu#L505) | -| `DS4_MMQ_D2R_IQ2` | boolean-ish cached flag; default on; a value starting with 0 disables | Control the direct-to-register IQ2 MoE gate/up path. | [cuda/mmq/ds4_mmq.cu:514](cuda/mmq/ds4_mmq.cu#L514) | -| `DS4_MMQ_D2R_MIN_COLS` | positive integer; default 1024; invalid or nonpositive input restores the default | Set the minimum output-column count for the MMQ direct-to-register path. | [cuda/mmq/ds4_mmq.cu:609](cuda/mmq/ds4_mmq.cu#L609) | +| `DS4_MMQ_D2R` | boolean-ish cached flag; default on; a value starting with 0 disables | Control the direct-to-register Q2_K MoE down path. | [cuda/mmq/ds4_mmq.cu:535](cuda/mmq/ds4_mmq.cu#L535) | +| `DS4_MMQ_D2R_IQ2` | boolean-ish cached flag; default on; a value starting with 0 disables | Control the direct-to-register IQ2 MoE gate/up path. | [cuda/mmq/ds4_mmq.cu:544](cuda/mmq/ds4_mmq.cu#L544) | +| `DS4_MMQ_D2R_MIN_COLS` | positive integer; default 1024; invalid or nonpositive input restores the default | Set the minimum output-column count for the MMQ direct-to-register path. | [cuda/mmq/ds4_mmq.cu:639](cuda/mmq/ds4_mmq.cu#L639) | | `DS4_MMQ_D2R_STATS` | exact 1 enables; unset or every other value disables; cached and synchronizes the stream | Print partial-tile fill telemetry for the direct-to-register MMQ kernels. | [cuda/mmq/ds4_mmq_d2r.cu:33](cuda/mmq/ds4_mmq_d2r.cu#L33) | | `DS4_MMQ_DENSE_D2R` | boolean-ish flag; default on; exact 0 disables | Control the eligible aligned-Q8 dense prefill direct-to-register path. | [ds4_cuda.cu:19567](ds4_cuda.cu#L19567) | -| `DS4_MMQ_NO_YIND` | presence rollback; unset keeps Y-indirect staging; any defined value including 0 disables it | Restore slot-gathered MoE gate/up activation quantization. | [cuda/mmq/ds4_mmq.cu:589](cuda/mmq/ds4_mmq.cu#L589) | -| `DS4_MMQ_OUT_MEMSET` | exact 1 enables; unset or every other value disables; cached | Restore blanket MMQ output-buffer zeroing for diagnostics. | [cuda/mmq/ds4_mmq.cu:532](cuda/mmq/ds4_mmq.cu#L532) | -| `DS4_MMQ_YBUF_MEMSET` | unset or 0 disables; 1 zero-fills; a value starting with p or P poison-fills with 0xFF | Control MMQ Q8_1 activation-staging initialization and its poison oracle. | [cuda/mmq/ds4_mmq.cu:558](cuda/mmq/ds4_mmq.cu#L558) | -| `DS4_MMQ_YIND_VERIFY` | presence diagnostic; unset is off; any defined value including 0 enables | Byte-compare Y-indirect and slot-gathered MoE activation buffers. | [cuda/mmq/ds4_mmq.cu:600](cuda/mmq/ds4_mmq.cu#L600) | +| `DS4_MMQ_NO_YIND` | presence rollback; unset keeps Y-indirect staging; any defined value including 0 disables it | Restore slot-gathered MoE gate/up activation quantization. | [cuda/mmq/ds4_mmq.cu:619](cuda/mmq/ds4_mmq.cu#L619) | +| `DS4_MMQ_OUT_MEMSET` | exact 1 enables; unset or every other value disables; cached | Restore blanket MMQ output-buffer zeroing for diagnostics. | [cuda/mmq/ds4_mmq.cu:562](cuda/mmq/ds4_mmq.cu#L562) | +| `DS4_MMQ_YBUF_MEMSET` | unset or 0 disables; 1 zero-fills; a value starting with p or P poison-fills with 0xFF | Control MMQ Q8_1 activation-staging initialization and its poison oracle. | [cuda/mmq/ds4_mmq.cu:588](cuda/mmq/ds4_mmq.cu#L588) | +| `DS4_MMQ_YIND_VERIFY` | presence diagnostic; unset is off; any defined value including 0 enables | Byte-compare Y-indirect and slot-gathered MoE activation buffers. | [cuda/mmq/ds4_mmq.cu:630](cuda/mmq/ds4_mmq.cu#L630) | | `DS4_MOE_RECORD_SELECTED_HOTLIST` | nonempty output path; unset=off | Record per-layer selected-expert hit counts to a Metal hotlist file. | [ds4_metal.m:1741](ds4_metal.m#L1741) | | `DS4_MOE_RECORD_SELECTED_HOTLIST_FRESH` | presence flag; only relevant with HOTLIST; overrides MERGE | Start the selected-expert hotlist from empty state. | [ds4_metal.m:1642](ds4_metal.m#L1642) | | `DS4_MOE_RECORD_SELECTED_HOTLIST_MERGE` | presence flag; active only when FRESH is absent | Merge an existing selected-expert hotlist before recording. | [ds4_metal.m:1641](ds4_metal.m#L1641) | @@ -1210,7 +1210,7 @@ and **19 tool/wrapper entries**. | `DS4_PREFILL_BATCH` | Nonempty value parsed by strtol without full-string validation; accepted range 1..4095, default 128 for unset/invalid/out-of-range values. It is used only when DS4_BATCHED_FFN selects full batched CPU FFN. | Set the token chunk size for layer_ffn_batch during CPU layer-major prefill. | [ds4.c:14414](ds4.c#L14414) | | `DS4_PREFILL_PROFILE_DETAIL` | presence flag; unset=off | Print detailed per-stage CPU prefill timing. | [ds4.c:12555](ds4.c#L12555) | | `DS4_PREFILL_PROFILE_TOKEN` | presence flag; effective within detailed prefill profiling | Print token-loop substage timings during CPU prefill. | [ds4.c:14130](ds4.c#L14130) | -| `DS4_Q8_FOLD_SELFTEST` | positive call budget; unset/empty disables; a nonempty value parsing to 1 or less selects 512 calls | Byte-check folded Q8_1 activations against a fresh quantization; synchronizes eager streams. | [cuda/mmq/ds4_mmq.cu:5921](cuda/mmq/ds4_mmq.cu#L5921) | +| `DS4_Q8_FOLD_SELFTEST` | positive call budget; unset/empty disables; a nonempty value parsing to 1 or less selects 512 calls | Byte-check folded Q8_1 activations against a fresh quantization; synchronizes eager streams. | [cuda/mmq/ds4_mmq.cu:5963](cuda/mmq/ds4_mmq.cu#L5963) | | `DS4_ROUTED_TOKEN_PARALLEL` | Pure presence flag that forces token-parallel routed MoE, even if DS4_NO_ROUTED_TOKEN_PARALLEL is also set. When unset, token parallelism is automatic for n_tok>=64 unless the NO flag is present; smaller batches use per-token routed MoE. | Choose token-parallel CPU routed-expert evaluation inside the default shared-batch FFN prefill path. | [ds4.c:12587](ds4.c#L12587) | | `DS4_SERVER_BATCH_LOG` | Pure presence flag read once when the decode worker starts; default off. Any defined value, including empty or "0", logs one record per coalesced decode batch with count, elapsed milliseconds and ok/error status. | Observe server-side decode coalescing size, latency and result without changing batching behavior. | [ds4_server.c:11090](ds4_server.c#L11090) | | `DS4_SERVER_DECODE_COALESCE_US` | integer 0..100000 microseconds; default 2000; 0 disables wait | Control server micro-batch coalescing delay. | [ds4_server.c:11069](ds4_server.c#L11069) | diff --git a/cuda/mmq/ds4_mmq.cu b/cuda/mmq/ds4_mmq.cu index 5fadf813c..e7bd1096e 100644 --- a/cuda/mmq/ds4_mmq.cu +++ b/cuda/mmq/ds4_mmq.cu @@ -26,6 +26,7 @@ #include "mmid.cuh" #include "ds4_mmq_d2r.cuh" +#include #include #include #include @@ -117,7 +118,7 @@ static constexpr bool g_q81_scratch_enabled = false; static bool g_q81_grouped_enabled = false; static bool g_q81_scratch_poisoned = false; static int g_q81_scratch_device = -1; -static std::mutex g_q81_scratch_mutex; +static std::mutex g_q81_state_mutex; // Process-global state, not per-tensor. static uint64_t g_q81_grouped_candidates; static uint64_t g_q81_grouped_uses; @@ -152,7 +153,15 @@ struct mmq_pair_map_scratch { }; static mmq_pair_map_scratch g_mmq_pair_maps[GGML_CUDA_MAX_DEVICES] = {}; -static int g_gb10_optimizations = 0; + +// Backend init/teardown owns transitions, but dispatch admission reads this +// flag without taking the Q8_1 arena mutex. Keep those reads race-free while +// retaining the mutex below for arena ownership and cleanup serialization. +static std::atomic g_gb10_optimizations{false}; + +static bool gb10_optimizations_enabled() { + return g_gb10_optimizations.load(std::memory_order_relaxed); +} static constexpr size_t DS4_MMQ_Q81_ARENA_MIN_BYTES = 4u * 1024u * 1024u; @@ -178,7 +187,7 @@ static bool q81_persistent_requested() { } static bool q81_is_gb10_owner(int device) { - if (!g_gb10_optimizations || device < 0 || + if (!gb10_optimizations_enabled() || device < 0 || device >= ggml_cuda_info().device_count) { return false; } @@ -348,7 +357,7 @@ extern "C" int ds4_mmq_q81_persistent_preflight_for_test( char *arena = nullptr; { std::unique_lock lease( - g_q81_scratch_mutex, std::defer_lock); + g_q81_state_mutex, std::defer_lock); arena = q81_grouped_persistent_acquire( device, (cudaStream_t)0, required, &lease); } @@ -444,9 +453,18 @@ static bool ds4_mmq_q8_fold_oracle_bytes( return false; } cudaStreamCaptureStatus capture = cudaStreamCaptureStatusNone; - if (cudaStreamIsCapturing(stream, &capture) != cudaSuccess || - capture != cudaStreamCaptureStatusNone) { - fprintf(stderr, "Unable to capture CUDA stream\nERROR: %s", cudaGetErrorString(cudaGetLastError())); + const cudaError_t capture_err = cudaStreamIsCapturing(stream, &capture); + if (capture_err != cudaSuccess) { + fprintf(stderr, + "ds4_mmq: Q8 fold oracle stream-capture query failed: %s\n", + cudaGetErrorString(capture_err)); + (void)cudaGetLastError(); + g_q8_fold_oracle_skips++; + return false; + } + if (capture != cudaStreamCaptureStatusNone) { + fprintf(stderr, + "ds4_mmq: Q8 fold oracle skipped during stream capture\n"); g_q8_fold_oracle_skips++; return false; } @@ -457,7 +475,20 @@ static bool ds4_mmq_q8_fold_oracle_bytes( } char *fresh = nullptr; char *host = (char *)malloc(bytes * 2u); - if (!host || cudaMalloc((void **)&fresh, bytes) != cudaSuccess || !fresh) { + if (!host) { + fprintf(stderr, + "ds4_mmq: Q8 fold oracle host allocation (%zu B) failed\n", + bytes * 2u); + g_q8_fold_oracle_skips++; + return false; + } + const cudaError_t fresh_alloc_err = cudaMalloc((void **)&fresh, bytes); + if (fresh_alloc_err != cudaSuccess || !fresh) { + fprintf(stderr, + "ds4_mmq: Q8 fold oracle device allocation (%zu B) failed: " + "%s%s\n", + bytes, cudaGetErrorString(fresh_alloc_err), + fresh_alloc_err == cudaSuccess ? " (null pointer)" : ""); free(host); fprintf(stderr, "Unable to allocate tensor \"fresh\"\nERROR: %s", cudaGetErrorString(cudaGetLastError())); g_q8_fold_oracle_skips++; @@ -695,7 +726,7 @@ extern "C" int ds4_mmq_init(int device) { // knows its exact maximum input/down staging requirement and resolves the // arena during preflight, before it submits any device operation. { - std::lock_guard lock(g_q81_scratch_mutex); + std::lock_guard lock(g_q81_state_mutex); const bool requested = q81_persistent_requested(); g_q81_grouped_enabled = requested && q81_is_gb10_owner(device) && g_q81_scratch_ptr && !g_q81_scratch_poisoned && @@ -705,7 +736,7 @@ extern "C" int ds4_mmq_init(int device) { } extern "C" int ds4_mmq_q81_persistent_cleanup(void) { - std::lock_guard lock(g_q81_scratch_mutex); + std::lock_guard lock(g_q81_state_mutex); g_q81_grouped_enabled = false; if (!g_q81_scratch_ptr && !g_q81_unpublished_replacement_ptr) { g_q81_scratch_bytes = 0; @@ -779,7 +810,7 @@ extern "C" void ds4_mmq_q81_persistent_counters( uint64_t *candidates, uint64_t *uses, uint64_t *hits, uint64_t *pool_fallbacks, uint64_t *allocations, uint64_t *resizes, size_t *arena_bytes, size_t *high_water) { - std::lock_guard lock(g_q81_scratch_mutex); + std::lock_guard lock(g_q81_state_mutex); if (candidates) *candidates = g_q81_grouped_candidates; if (uses) *uses = g_q81_grouped_uses; if (hits) *hits = g_q81_grouped_hits; @@ -791,7 +822,7 @@ extern "C" void ds4_mmq_q81_persistent_counters( } extern "C" void ds4_mmq_q81_persistent_report(void) { - std::lock_guard lock(g_q81_scratch_mutex); + std::lock_guard lock(g_q81_state_mutex); fprintf(stderr, "ds4: CUDA MMQ grouped Q8_1 persistent: candidates=%llu " "uses=%llu hits=%llu pool_fallbacks=%llu allocations=%llu " @@ -2058,7 +2089,7 @@ int ds4_mmq_moe_pair_impl( size_t grouped_down_q8_bytes = 0; size_t grouped_q81_required = 0; std::unique_lock grouped_q81_lease( - g_q81_scratch_mutex, std::defer_lock); + g_q81_state_mutex, std::defer_lock); ggml_cuda_pool_alloc grouped_q81_pool; char *grouped_q81_scratch = nullptr; if (grouped_raw_q81 && q81_persistent_requested()) { @@ -3947,6 +3978,9 @@ q4_K_dense_vec_k1024_persistent_kernel( const uint32_t group_tid = warp_in_group * 32u + lane; const uint64_t row_tiles = ((uint64_t)(uint32_t)M + 7u) / 8u; + /* tile, row_tiles, and gridDim.x are block-uniform, and this loop has no + * divergent exit. Every thread therefore reaches both barriers below on + * every iteration; unrolling is unrelated to their correctness. */ for (uint64_t tile = blockIdx.x; tile < row_tiles; tile += gridDim.x) { const uint32_t row0 = (uint32_t)(tile * 8u) + group * k_rows_per_group; @@ -4052,7 +4086,7 @@ int ds4_mmq_dense_vec_impl( if (q4_k1024_exact) { g_q4_k1024_persistent_candidates++; } - if (q4_k1024_exact && g_gb10_optimizations && + if (q4_k1024_exact && gb10_optimizations_enabled() && (enable || q4_k1024_oracle) && !disable && (((uintptr_t)W & 15u) == 0u)) { const uint64_t row_tiles = ((uint64_t)(uint32_t)M + 7u) / 8u; @@ -4125,7 +4159,7 @@ int ds4_mmq_dense_vec_impl( ggml_cuda_pool_alloc src1_q8_1; char *x8 = nullptr; if constexpr (type == GGML_TYPE_Q4_K) { - if (g_gb10_optimizations && + if (gb10_optimizations_enabled() && getenv("DS4_CUDA_NO_Q4_GB10_FAST") == nullptr && getenv("DS4_CUDA_NO_Q4_DENSE_SCRATCH") == nullptr) { x8 = (char *)ds4_mmq_aligned_q81_scratch(dev, nbytes_q8_1); @@ -4310,7 +4344,7 @@ int ds4_mmq_dense_pair_vec_impl( ggml_cuda_pool_alloc src1_q8_1; char *x8 = nullptr; if constexpr (type == GGML_TYPE_Q4_K) { - if (g_gb10_optimizations && + if (gb10_optimizations_enabled() && getenv("DS4_CUDA_NO_Q4_GB10_FAST") == nullptr && getenv("DS4_CUDA_NO_Q4_DENSE_SCRATCH") == nullptr) { x8 = (char *)ds4_mmq_aligned_q81_scratch(dev, nbytes_q8_1); @@ -4430,7 +4464,7 @@ static int ds4_mmq_q4_K_grouped_batch_vec_impl( fprintf(stderr, "%s: null pointer\n", tag); return -1; } - if (!g_gb10_optimizations || + if (!gb10_optimizations_enabled() || getenv("DS4_CUDA_NO_Q4_GB10_FAST") != nullptr || getenv("DS4_CUDA_NO_Q4_GROUPED_ATTN_A") != nullptr || M <= 0 || K <= 0 || n_tokens <= 0 || n_tokens > 8 || @@ -5444,8 +5478,10 @@ extern "C" int ds4_mmq_iq2_xxs_aligned_derepack( extern "C" void ds4_mmq_set_gb10_optimizations(int enabled) { { - std::lock_guard lock(g_q81_scratch_mutex); - g_gb10_optimizations = enabled != 0; + // Serialize the transition with any persistent Q8_1 host lease. The + // atomic also covers GB10 admission reads outside this arena lock. + std::lock_guard lock(g_q81_state_mutex); + g_gb10_optimizations.store(enabled != 0, std::memory_order_relaxed); } if (!enabled) { // Backend teardown/reinit already funnels through this setter. Keep @@ -5658,7 +5694,7 @@ static cudaError_t q8_0_aligned_dense_vec_launch( cudaStream_t stream) { switch (N) { case 1: - if (g_gb10_optimizations && + if (gb10_optimizations_enabled() && getenv("DS4_CUDA_NO_Q8_ALIGNED_PERSISTENT") == NULL && (K == 1024 || K == 4096) && M >= 32768) { const uint64_t row_blocks = ((uint64_t)(unsigned)M + 7u) / 8u; diff --git a/scripts/environment_variables.tsv b/scripts/environment_variables.tsv index 36675eaf6..fe9580a00 100644 --- a/scripts/environment_variables.tsv +++ b/scripts/environment_variables.tsv @@ -78,8 +78,8 @@ runtime/cuda DS4_CUDA_ENABLE_DSPARK_NONCAUSAL_ONLINE value-aware flag, default o runtime/cuda DS4_CUDA_ENABLE_HC_NORM_MIX_FUSE nonempty opt-in, default off; only exact 0 disables; the F32/F16 activation mode follows the selected standalone matmul path; disable/serial/alternate flags can veto Enable and select the fused HC RMSNorm-plus-mix one-token implementation. ds4_cuda.cu:20902 runtime/cuda DS4_CUDA_ENABLE_IQ2_XXS_SSD_PREFILL_MMQ false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Enable the CUDA IQ2 XXS SSD prefill MMQ experimental path. ds4_cuda.cu:4625 runtime/cuda DS4_CUDA_ENABLE_Q4_ATTN_OUT_HC_FUSE value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on Opt in to the fused Q4 attention-output/HC expansion path. ds4_cuda.cu:37375 -runtime/cuda DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_BATCH value-aware opt-in, default off; nonempty value other than exact 0 enables; rollback wins Enable flattened grouped attention-A MMQ for two-to-eight-token GB10 batches. cuda/mmq/ds4_mmq.cu:4280 -runtime/cuda DS4_CUDA_ENABLE_Q4_K1024_PERSISTENT presence flag, default off; any defined value including 0 requests the path; rollback wins Enable the GB10 persistent-CTA kernel for M=32768, N=1, K=1024 Q4. cuda/mmq/ds4_mmq.cu:3882 +runtime/cuda DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_BATCH value-aware opt-in, default off; nonempty value other than exact 0 enables; rollback wins Enable flattened grouped attention-A MMQ for two-to-eight-token GB10 batches. cuda/mmq/ds4_mmq.cu:4313 +runtime/cuda DS4_CUDA_ENABLE_Q4_K1024_PERSISTENT presence flag, default off; any defined value including 0 requests the path; rollback wins Enable the GB10 persistent-CTA kernel for M=32768, N=1, K=1024 Q4. cuda/mmq/ds4_mmq.cu:3915 runtime/cuda DS4_CUDA_ENABLE_Q8_FOLD strict flag, default off; only exact value 1 enables; overridden by DS4_CUDA_NO_Q8_FOLD Enable one-shot producer-to-consumer reuse of freshly quantized Q8_1 data. ds4_cuda.cu:785 runtime/cuda DS4_CUDA_ENABLE_STREAMING_EXPERT_PERSISTENT_CACHE false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Enable streaming expert persistent cache in CUDA SSD streaming. ds4_cuda.cu:4111 runtime/cuda DS4_CUDA_ENABLE_STREAMING_SELECTED_BATCH_IO false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Enable streaming selected batch I/O in CUDA SSD streaming. ds4_cuda.cu:4732 @@ -123,7 +123,7 @@ runtime/cuda DS4_CUDA_MIXED_PREFILL_DECODE value-aware boolean; default on; unse runtime/cuda DS4_CUDA_MIXED_ROUTED_MAX_PREFILL integer rows 0..UINT32_MAX; default/invalid 512 Set the maximum prefill rows admitted to the mixed routed-MoE path. ds4.c:70190 runtime/cuda DS4_CUDA_MIXED_ROUTED_SCATTER value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Use scattered row handling in mixed routed-MoE execution. ds4.c:69651 runtime/cuda DS4_CUDA_MMQ boolean-ish, default on; any value beginning with 0 disables; quality mode and multi-GPU disable normal MMQ tier (MXFP4 path differs) Control the vendored CUDA MMQ prefill tier. ds4_cuda.cu:1722 -runtime/cuda DS4_CUDA_MMQ_Q81_PERSISTENT strict boolean, default off; accepts 1/on/true/yes and 0/off/false/no in listed lower/upper-case forms; unknown values are off Reuse a persistent Q8_1 MMQ scratch arena on supported GB10 devices. cuda/mmq/ds4_mmq.cu:144 +runtime/cuda DS4_CUDA_MMQ_Q81_PERSISTENT strict boolean, default off; accepts 1/on/true/yes and 0/off/false/no in listed lower/upper-case forms; unknown values are off Reuse a persistent Q8_1 MMQ scratch arena on supported GB10 devices. cuda/mmq/ds4_mmq.cu:152 runtime/cuda DS4_CUDA_MMQ_X_MAX integer >=8; rounded down to multiple of 8 and only lowers the hardware base; invalid/unset = hardware base Cap the MMQ X tile-width selector for architecture tuning. cuda/mmq/mmq.cuh:127 runtime/cuda DS4_CUDA_MODEL_COPY_CHUNK_MB positive integer MiB; default 64; clamped 16..4096 Set the chunk size used for CUDA model copying. ds4_cuda.cu:2827 runtime/cuda DS4_CUDA_MODEL_COPY_VERBOSE presence diagnostic flag; default off; any defined value including 0 enables Print periodic progress while copying the model to device memory. ds4_cuda.cu:6537 @@ -219,16 +219,16 @@ runtime/cuda DS4_CUDA_NO_INDEXER_WMMA64 presence kill switch; default unset (eli runtime/cuda DS4_CUDA_NO_IQ2_XXS_SSD_PREFILL_MMQ false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Disable the CUDA IQ2 XXS SSD prefill MMQ optimization/path. ds4_cuda.cu:4629 runtime/cuda DS4_CUDA_NO_MODEL_COPY presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the CUDA model copy optimization/path. ds4_cuda.cu:6472 runtime/cuda DS4_CUDA_NO_MODEL_PREFETCH presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the CUDA model prefetch optimization/path. ds4_cuda.cu:2739 -runtime/cuda DS4_CUDA_NO_MOE_DEDUP presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the CUDA MoE dedup optimization/path. cuda/mmq/ds4_mmq.cu:6145 +runtime/cuda DS4_CUDA_NO_MOE_DEDUP presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the CUDA MoE dedup optimization/path. cuda/mmq/ds4_mmq.cu:6182 runtime/cuda DS4_CUDA_NO_ORDERED_F16_MATMUL presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the ordered F16 matmul CUDA F16 path. ds4_cuda.cu:20811 runtime/cuda DS4_CUDA_NO_PARALLEL_ROUTER_SELECT presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the CUDA parallel router select optimization/path. ds4_cuda.cu:24491 -runtime/cuda DS4_CUDA_NO_Q4_DENSE_SCRATCH presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q4 dense scratch CUDA Q4 optimization. cuda/mmq/ds4_mmq.cu:3963 -runtime/cuda DS4_CUDA_NO_Q4_GB10_FAST presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the GB10-specific Q4 fast-path family. cuda/mmq/ds4_mmq.cu:3885 -runtime/cuda DS4_CUDA_NO_Q4_GROUPED_ATTN_A presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q4 grouped attn a CUDA Q4 optimization. cuda/mmq/ds4_mmq.cu:4272 -runtime/cuda DS4_CUDA_NO_Q4_GROUPED_ATTN_A_BATCH presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q4 grouped attn a batch CUDA Q4 optimization. cuda/mmq/ds4_mmq.cu:4282 -runtime/cuda DS4_CUDA_NO_Q4_K1024_PERSISTENT presence kill switch, default off; any defined value including 0 disables Disable the Q4 K1024 persistent CUDA Q4 optimization. cuda/mmq/ds4_mmq.cu:3884 -runtime/cuda DS4_CUDA_NO_Q8_ALIGNED_DENSE_SCRATCH presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q8 aligned dense scratch CUDA Q8 optimization. cuda/mmq/ds4_mmq.cu:5553 -runtime/cuda DS4_CUDA_NO_Q8_ALIGNED_PERSISTENT presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q8 aligned persistent CUDA Q8 optimization. cuda/mmq/ds4_mmq.cu:5385 +runtime/cuda DS4_CUDA_NO_Q4_DENSE_SCRATCH presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q4 dense scratch CUDA Q4 optimization. cuda/mmq/ds4_mmq.cu:3996 +runtime/cuda DS4_CUDA_NO_Q4_GB10_FAST presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the GB10-specific Q4 fast-path family. cuda/mmq/ds4_mmq.cu:3918 +runtime/cuda DS4_CUDA_NO_Q4_GROUPED_ATTN_A presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q4 grouped attn a CUDA Q4 optimization. cuda/mmq/ds4_mmq.cu:4305 +runtime/cuda DS4_CUDA_NO_Q4_GROUPED_ATTN_A_BATCH presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q4 grouped attn a batch CUDA Q4 optimization. cuda/mmq/ds4_mmq.cu:4315 +runtime/cuda DS4_CUDA_NO_Q4_K1024_PERSISTENT presence kill switch, default off; any defined value including 0 disables Disable the Q4 K1024 persistent CUDA Q4 optimization. cuda/mmq/ds4_mmq.cu:3917 +runtime/cuda DS4_CUDA_NO_Q8_ALIGNED_DENSE_SCRATCH presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q8 aligned dense scratch CUDA Q8 optimization. cuda/mmq/ds4_mmq.cu:5588 +runtime/cuda DS4_CUDA_NO_Q8_ALIGNED_PERSISTENT presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q8 aligned persistent CUDA Q8 optimization. cuda/mmq/ds4_mmq.cu:5420 runtime/cuda DS4_CUDA_NO_Q8_BATCH_EXACT_TOK2 presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q8 batch exact tok2 CUDA Q8 optimization. ds4_cuda.cu:19852 runtime/cuda DS4_CUDA_NO_Q8_BATCH_TOK4 presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q8 batch tok4 CUDA Q8 optimization. ds4_cuda.cu:19805 runtime/cuda DS4_CUDA_NO_Q8_BATCH_TOK8 presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q8 batch tok8 CUDA Q8 optimization. ds4_cuda.cu:19788 @@ -264,8 +264,8 @@ runtime/cuda DS4_CUDA_NO_TP_ATTN_OUT_HC_FUSE presence kill switch; default unset runtime/cuda DS4_CUDA_NO_VERIFY_DECODE2_SPLIT_TOP1 value-aware kill switch; default off; nonempty value other than exact 0 disables Disable the CUDA verify decode2 split top1 optimization/path. ds4.c:17223 runtime/cuda DS4_CUDA_NO_WARP_ROUTER_SELECT presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the CUDA warp router select optimization/path. ds4_cuda.cu:24490 runtime/cuda DS4_CUDA_NO_WINDOW_ATTENTION presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the CUDA window attention optimization/path. ds4_cuda.cu:23050 -runtime/cuda DS4_CUDA_NSYS_PREFILL_START_POS nonempty-string flag, default off; any nonempty value enables MMQ NVTX ranges (the value is not parsed as a position) Enable MMQ NVTX annotations intended for Nsight Systems prefill capture. cuda/mmq/ds4_mmq.cu:48 -runtime/cuda DS4_CUDA_NVTX strict flag, default off; only exact value 1 enables (a nonempty NSYS variable also enables ranges) Enable NVTX ranges around MMQ work. cuda/mmq/ds4_mmq.cu:47 +runtime/cuda DS4_CUDA_NSYS_PREFILL_START_POS nonempty-string flag, default off; any nonempty value enables MMQ NVTX ranges (the value is not parsed as a position) Enable MMQ NVTX annotations intended for Nsight Systems prefill capture. cuda/mmq/ds4_mmq.cu:49 +runtime/cuda DS4_CUDA_NVTX strict flag, default off; only exact value 1 enables (a nonempty NSYS variable also enables ranges) Enable NVTX ranges around MMQ work. cuda/mmq/ds4_mmq.cu:48 runtime/cuda DS4_CUDA_OUTPUT_FUSED_TOP1 value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Fuse output projection with top-1 selection in greedy decode. ds4.c:17215 runtime/cuda DS4_CUDA_PREFILL_PIPELINE boolean, default follows CUDA TP decode; nonempty exact 0 disables, any other nonempty value enables Control the CUDA multi-tier prefill pipeline. ds4.c:17454 runtime/cuda DS4_CUDA_PREFILL_PIPELINE_MB positive integer rows; default/invalid 512 Set prefill-pipeline microbatch rows. ds4.c:17469 @@ -275,15 +275,15 @@ runtime/cuda DS4_CUDA_PREFILL_PIPELINE_SYNC_BOUNDARY presence flag; unset does n runtime/cuda DS4_CUDA_Q4_ATTN_OUT_HC_ORACLE value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on Compare fused Q4 attention-output/HC expansion with the canonical path and retain canonical output. ds4_cuda.cu:1475 runtime/cuda DS4_CUDA_Q4_ATTN_OUT_HC_Q8K_EXPERIMENT value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on Enable the experimental Q8_K-based Q4 attention-output/HC fusion. ds4_cuda.cu:37377 runtime/cuda DS4_CUDA_Q4_GROUPED_ATTN_A_ORACLE value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on Compare grouped attention-A against the canonical per-group result. ds4_cuda.cu:1477 -runtime/cuda DS4_CUDA_Q4_K1024_PERSISTENT_ORACLE value-aware flag, default off; nonempty value other than exact 0 enables and implies candidate admission Bitwise-compare the exact-shape persistent Q4 K1024 kernel with canonical MMVQ and retain canonical output. cuda/mmq/ds4_mmq.cu:3718 -runtime/cuda DS4_CUDA_Q4_K1024_PERSISTENT_STATS value-aware flag, default off; nonempty value other than exact 0 enables Print exact-shape persistent Q4 K1024 dispatch counters at exit. cuda/mmq/ds4_mmq.cu:3717 +runtime/cuda DS4_CUDA_Q4_K1024_PERSISTENT_ORACLE value-aware flag, default off; nonempty value other than exact 0 enables and implies candidate admission Bitwise-compare the exact-shape persistent Q4 K1024 kernel with canonical MMVQ and retain canonical output. cuda/mmq/ds4_mmq.cu:3748 +runtime/cuda DS4_CUDA_Q4_K1024_PERSISTENT_STATS value-aware flag, default off; nonempty value other than exact 0 enables Print exact-shape persistent Q4 K1024 dispatch counters at exit. cuda/mmq/ds4_mmq.cu:3747 runtime/cuda DS4_CUDA_Q8_F16_ALL presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Control the Q8 F16 all CUDA quantized-matmul/cache optimization. ds4_cuda.cu:2358 runtime/cuda DS4_CUDA_Q8_F16_CACHE_MB unsigned integer MiB, full-string parse; default unlimited; 0 disables this cache Limit the selective Q8-to-F16 derived-weight cache. ds4_cuda.cu:2218 runtime/cuda DS4_CUDA_Q8_F16_CACHE_RESERVE_MB unsigned integer MiB, full-string parse; default is VRAM-dependent (>=112 GiB: 512; >=40 GiB: max(768,1%); smaller: max(4096,5%)) Reserve free VRAM when growing the selective Q8-to-F16 cache. ds4_cuda.cu:2224 runtime/cuda DS4_CUDA_Q8_F32_ALL presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Control the Q8 F32 all CUDA quantized-matmul/cache optimization. ds4_cuda.cu:2412 runtime/cuda DS4_CUDA_Q8_F32_LARGE presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Control the Q8 F32 large CUDA quantized-matmul/cache optimization. ds4_cuda.cu:2416 runtime/cuda DS4_CUDA_Q8_F32_PRELOAD presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Control the Q8 F32 preload CUDA quantized-matmul/cache optimization. ds4_cuda.cu:8597 -runtime/cuda DS4_CUDA_Q8_FOLD_ORACLE strict flag, default off; only exact value 1 enables Compare folded Q8_1 bytes and consumer outputs against canonical work while retaining canonical results. cuda/mmq/ds4_mmq.cu:383 +runtime/cuda DS4_CUDA_Q8_FOLD_ORACLE strict flag, default off; only exact value 1 enables Compare folded Q8_1 bytes and consumer outputs against canonical work while retaining canonical results. cuda/mmq/ds4_mmq.cu:391 runtime/cuda DS4_CUDA_Q8_HC_EXPAND_FUSED false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, other nonempty values force fused Force the fused Q8 shared-down/HC expansion path. ds4_cuda.cu:2084 runtime/cuda DS4_CUDA_Q8_HC_EXPAND_STATS false-like-aware flag, default off; 0/false/no/off is off, other nonempty values print report Print Q8 shared-down/HC policy and dispatch counters at exit. ds4_cuda.cu:2090 runtime/cuda DS4_CUDA_Q8_NO_ALIGNED value-aware kill switch, default off; nonempty value other than exact 0 disables aligned Q8 kernels Disable aligned Q8 CUDA matmul kernels. ds4_cuda.cu:1021 @@ -292,7 +292,7 @@ runtime/cuda DS4_CUDA_QKV_KV_ROPE_FUSE value-aware boolean; default on; unset/em runtime/cuda DS4_CUDA_Q_NORM_ROPE_FUSE value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control or tune the CUDA q norm rope fuse path. ds4.c:17418 runtime/cuda DS4_CUDA_REQUIRE_IQ2_XXS_SSD_PREFILL_MMQ false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Require the CUDA IQ2 XXS SSD prefill MMQ path; fail closed when unavailable. ds4_cuda.cu:4631 runtime/cuda DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_BATCH value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on Fail if grouped batched attention-A cannot be used. ds4_cuda.cu:40198 -runtime/cuda DS4_CUDA_REQUIRE_Q4_K1024_PERSISTENT presence flag, default off; any defined value including 0 makes ineligible candidate fail closed Fail when the exact Q4 K1024 persistent candidate is unavailable instead of using MMVQ. cuda/mmq/ds4_mmq.cu:3906 +runtime/cuda DS4_CUDA_REQUIRE_Q4_K1024_PERSISTENT presence flag, default off; any defined value including 0 makes ineligible candidate fail closed Fail when the exact Q4 K1024 persistent candidate is unavailable instead of using MMVQ. cuda/mmq/ds4_mmq.cu:3939 runtime/cuda DS4_CUDA_REQUIRE_STREAMING_EXPERT_PERSISTENT_CACHE false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Require streaming expert persistent cache in CUDA SSD streaming; fail closed when unavailable. ds4_cuda.cu:4117 runtime/cuda DS4_CUDA_REQUIRE_STREAMING_SELECTED_BATCH_IO false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Require streaming selected batch I/O in CUDA SSD streaming; fail closed when unavailable. ds4_cuda.cu:4738 runtime/cuda DS4_CUDA_REQUIRE_STREAMING_SELECTED_EVENT_PIPELINE false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Require streaming selected event pipeline in CUDA SSD streaming; fail closed when unavailable. ds4_cuda.cu:4870 @@ -366,16 +366,16 @@ runtime/cuda DS4_CUDA_WEIGHT_PRELOAD_SPAN_MB positive integer MiB; default 1024; runtime/cuda DS4_CUDA_WINDOW_ATTENTION presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Control or tune the CUDA window attention path. ds4_cuda.cu:23051 runtime/cuda-mmq DS4_MMID_CASE1 boolean-ish cached flag; default on; a value starting with 0 disables Disable the single-expert MM-IDs specialized fast path for comparison. cuda/mmq/mmid.cu:290 runtime/cuda-mmq DS4_MMID_LARGE boolean-ish cached flag; default on; a value starting with 0 disables Control the large-N global-memory MM-IDs path used beyond shared-memory capacity. cuda/mmq/mmid.cu:245 -runtime/cuda-mmq DS4_MMQ_D2R boolean-ish cached flag; default on; a value starting with 0 disables Control the direct-to-register Q2_K MoE down path. cuda/mmq/ds4_mmq.cu:505 -runtime/cuda-mmq DS4_MMQ_D2R_IQ2 boolean-ish cached flag; default on; a value starting with 0 disables Control the direct-to-register IQ2 MoE gate/up path. cuda/mmq/ds4_mmq.cu:514 -runtime/cuda-mmq DS4_MMQ_D2R_MIN_COLS positive integer; default 1024; invalid or nonpositive input restores the default Set the minimum output-column count for the MMQ direct-to-register path. cuda/mmq/ds4_mmq.cu:609 +runtime/cuda-mmq DS4_MMQ_D2R boolean-ish cached flag; default on; a value starting with 0 disables Control the direct-to-register Q2_K MoE down path. cuda/mmq/ds4_mmq.cu:535 +runtime/cuda-mmq DS4_MMQ_D2R_IQ2 boolean-ish cached flag; default on; a value starting with 0 disables Control the direct-to-register IQ2 MoE gate/up path. cuda/mmq/ds4_mmq.cu:544 +runtime/cuda-mmq DS4_MMQ_D2R_MIN_COLS positive integer; default 1024; invalid or nonpositive input restores the default Set the minimum output-column count for the MMQ direct-to-register path. cuda/mmq/ds4_mmq.cu:639 runtime/cuda-mmq DS4_MMQ_D2R_STATS exact 1 enables; unset or every other value disables; cached and synchronizes the stream Print partial-tile fill telemetry for the direct-to-register MMQ kernels. cuda/mmq/ds4_mmq_d2r.cu:33 runtime/cuda-mmq DS4_MMQ_DENSE_D2R boolean-ish flag; default on; exact 0 disables Control the eligible aligned-Q8 dense prefill direct-to-register path. ds4_cuda.cu:19567 -runtime/cuda-mmq DS4_MMQ_NO_YIND presence rollback; unset keeps Y-indirect staging; any defined value including 0 disables it Restore slot-gathered MoE gate/up activation quantization. cuda/mmq/ds4_mmq.cu:589 -runtime/cuda-mmq DS4_MMQ_OUT_MEMSET exact 1 enables; unset or every other value disables; cached Restore blanket MMQ output-buffer zeroing for diagnostics. cuda/mmq/ds4_mmq.cu:532 -runtime/cuda-mmq DS4_MMQ_YBUF_MEMSET unset or 0 disables; 1 zero-fills; a value starting with p or P poison-fills with 0xFF Control MMQ Q8_1 activation-staging initialization and its poison oracle. cuda/mmq/ds4_mmq.cu:558 -runtime/cuda-mmq DS4_MMQ_YIND_VERIFY presence diagnostic; unset is off; any defined value including 0 enables Byte-compare Y-indirect and slot-gathered MoE activation buffers. cuda/mmq/ds4_mmq.cu:600 -runtime/cuda-mmq DS4_Q8_FOLD_SELFTEST positive call budget; unset/empty disables; a nonempty value parsing to 1 or less selects 512 calls Byte-check folded Q8_1 activations against a fresh quantization; synchronizes eager streams. cuda/mmq/ds4_mmq.cu:5921 +runtime/cuda-mmq DS4_MMQ_NO_YIND presence rollback; unset keeps Y-indirect staging; any defined value including 0 disables it Restore slot-gathered MoE gate/up activation quantization. cuda/mmq/ds4_mmq.cu:619 +runtime/cuda-mmq DS4_MMQ_OUT_MEMSET exact 1 enables; unset or every other value disables; cached Restore blanket MMQ output-buffer zeroing for diagnostics. cuda/mmq/ds4_mmq.cu:562 +runtime/cuda-mmq DS4_MMQ_YBUF_MEMSET unset or 0 disables; 1 zero-fills; a value starting with p or P poison-fills with 0xFF Control MMQ Q8_1 activation-staging initialization and its poison oracle. cuda/mmq/ds4_mmq.cu:588 +runtime/cuda-mmq DS4_MMQ_YIND_VERIFY presence diagnostic; unset is off; any defined value including 0 enables Byte-compare Y-indirect and slot-gathered MoE activation buffers. cuda/mmq/ds4_mmq.cu:630 +runtime/cuda-mmq DS4_Q8_FOLD_SELFTEST positive call budget; unset/empty disables; a nonempty value parsing to 1 or less selects 512 calls Byte-check folded Q8_1 activations against a fresh quantization; synchronizes eager streams. cuda/mmq/ds4_mmq.cu:5963 runtime/cuda-shared DS4_FORCE_CUDA_PEER presence flag read once at CUDA init; unset uses automatic transfer selection; any defined value including 0 enables Force cross-device transfers through cudaMemcpyPeerAsync for diagnostics. ds4_cuda.cu:364 runtime/cuda-shared DS4_FORCE_HOST_BOUNCE presence flag read once at CUDA init; unset uses automatic transfer selection; any defined value including 0 enables Force cross-device transfers through pinned host bounce buffers for diagnostics. ds4_cuda.cu:365 runtime/cuda-tools DS4_WS_REPACK_HASH exact 1 enables; unset or every other value disables unless overridden by CLI; cached Print a per-artifact FNV-1a hash for workspace repack identity checks. cuda/mmq/ds4_repack.cu:530 From f762dbabde6819a2d08ad708982668c36d81d0a3 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Sun, 23 Aug 2026 15:15:16 +0200 Subject: [PATCH 076/189] test(cuda): harden persistent Q8 parity setup --- cuda/mmq/test/test_mmq_parity.cu | 249 +++++++++++++++++++++++-------- 1 file changed, 190 insertions(+), 59 deletions(-) diff --git a/cuda/mmq/test/test_mmq_parity.cu b/cuda/mmq/test/test_mmq_parity.cu index 2f6fe3a96..6621e52b1 100644 --- a/cuda/mmq/test/test_mmq_parity.cu +++ b/cuda/mmq/test/test_mmq_parity.cu @@ -40,18 +40,70 @@ #include #include +#include #include #include #include #include #include #include +#include #include namespace { constexpr int QK_K_LOCAL = 256; +class scoped_env_override { +public: + explicit scoped_env_override(const char *name) : name_(name) { + const char *value = std::getenv(name_); + if (value) { + had_original_ = true; + original_value_ = value; + } + } + + ~scoped_env_override() { + (void)restore(); + } + + bool set(const char *value) { + if (setenv(name_, value, 1) != 0) { + const int saved_errno = errno; + fprintf(stderr, "setenv(%s=%s) failed: %s\n", + name_, value, std::strerror(saved_errno)); + return false; + } + active_ = true; + return true; + } + + bool restore() { + if (!active_) return true; + const int rc = had_original_ + ? setenv(name_, original_value_.c_str(), 1) + : unsetenv(name_); + if (rc != 0) { + const int saved_errno = errno; + fprintf(stderr, "restoring %s failed: %s\n", + name_, std::strerror(saved_errno)); + return false; + } + active_ = false; + return true; + } + + scoped_env_override(const scoped_env_override &) = delete; + scoped_env_override &operator=(const scoped_env_override &) = delete; + +private: + const char *name_; + std::string original_value_; + bool had_original_ = false; + bool active_ = false; +}; + // -------------------------------------------------------------------------- // Half-precision conversion (standalone, no CUDA host fp16 needed). // -------------------------------------------------------------------------- @@ -959,28 +1011,6 @@ bool run_iq2_xxs_q2_K_fused_raw_parity( persistent_q81 ? "/PERSISTENT_Q81" : "", n_tokens, compact_experts, n_expert_used, seed); - int initial_arena_cleanup = 0; - int q81_lazy_init_rc = 0; - uint64_t q81_init_allocations0 = 0, q81_init_resizes0 = 0; - uint64_t q81_init_allocations1 = 0, q81_init_resizes1 = 0; - size_t q81_init_arena0 = 0, q81_init_arena1 = 0; - if (persistent_q81) { - initial_arena_cleanup = ds4_mmq_q81_persistent_cleanup(); - ds4_mmq_set_gb10_optimizations(1); - ds4_mmq_q81_persistent_counters( - nullptr, nullptr, nullptr, nullptr, - &q81_init_allocations0, &q81_init_resizes0, - &q81_init_arena0, nullptr); - setenv("DS4_CUDA_MMQ_Q81_PERSISTENT", "1", 1); - q81_lazy_init_rc = ds4_mmq_init(persistent_device); - ds4_mmq_q81_persistent_counters( - nullptr, nullptr, nullptr, nullptr, - &q81_init_allocations1, &q81_init_resizes1, - &q81_init_arena1, nullptr); - // A real `=0` dispatch below is the value-aware opt-out oracle. - setenv("DS4_CUDA_MMQ_Q81_PERSISTENT", "0", 1); - } - std::mt19937 rng(seed); std::normal_distribution activation(0.0f, 0.05f); const size_t iq2_blocks_per_expert = @@ -1070,32 +1100,31 @@ bool run_iq2_xxs_q2_K_fused_raw_parity( float *d_gate_global = nullptr, *d_up_global = nullptr, *d_mid_global = nullptr, *d_down_global = nullptr; - bool allocated = (persistent_q81 || - cudaStreamCreate(&stream) == cudaSuccess) && - cudaMalloc(&d_gate_w, gate_compact.size() * sizeof(block_iq2_xxs)) == cudaSuccess && - cudaMalloc(&d_up_w, up_compact.size() * sizeof(block_iq2_xxs)) == cudaSuccess && - cudaMalloc(&d_down_w, down_compact.size() * sizeof(block_q2_K)) == cudaSuccess && - cudaMalloc(&d_gate_global_w, gate_global.size() * sizeof(block_iq2_xxs)) == cudaSuccess && - cudaMalloc(&d_up_global_w, up_global.size() * sizeof(block_iq2_xxs)) == cudaSuccess && - cudaMalloc(&d_down_global_w, down_global.size() * sizeof(block_q2_K)) == cudaSuccess && - cudaMalloc(&d_x, X.size() * sizeof(float)) == cudaSuccess && - cudaMalloc(&d_ids, remapped_ids.size() * sizeof(int32_t)) == cudaSuccess && - cudaMalloc(&d_global_ids, global_ids.size() * sizeof(int32_t)) == cudaSuccess && - cudaMalloc(&d_router, router_weights.size() * sizeof(float)) == cudaSuccess && - cudaMalloc(&d_gate_ref, mid_count * sizeof(float)) == cudaSuccess && - cudaMalloc(&d_up_ref, mid_count * sizeof(float)) == cudaSuccess && - cudaMalloc(&d_mid_ref, mid_count * sizeof(float)) == cudaSuccess && - cudaMalloc(&d_down_ref, down_count * sizeof(float)) == cudaSuccess && - cudaMalloc(&d_gate_got, mid_count * sizeof(float)) == cudaSuccess && - cudaMalloc(&d_up_got, mid_count * sizeof(float)) == cudaSuccess && - cudaMalloc(&d_mid_got, mid_count * sizeof(float)) == cudaSuccess && - cudaMalloc(&d_down_got, down_count * sizeof(float)) == cudaSuccess && - cudaMalloc(&d_gate_global, mid_count * sizeof(float)) == cudaSuccess && - cudaMalloc(&d_up_global, mid_count * sizeof(float)) == cudaSuccess && - cudaMalloc(&d_mid_global, mid_count * sizeof(float)) == cudaSuccess && - cudaMalloc(&d_down_global, down_count * sizeof(float)) == cudaSuccess; + scoped_env_override q81_env("DS4_CUDA_MMQ_Q81_PERSISTENT"); + int initial_arena_cleanup = 0; + int final_arena_cleanup = 0; + int q81_lazy_init_rc = 0; + uint64_t q81_init_allocations0 = 0, q81_init_resizes0 = 0; + uint64_t q81_init_allocations1 = 0, q81_init_resizes1 = 0; + size_t q81_init_arena0 = 0, q81_init_arena1 = 0; + bool q81_env_restore_ok = true; + bool persistent_active = false; + + auto teardown_persistent = [&]() { + if (!persistent_active) return; + // Disable acquisition before retiring the owned arena, then restore + // the caller's exact environment (including an originally absent key). + q81_env_restore_ok = q81_env.set("0") && q81_env_restore_ok; + // Record the cleanup API result explicitly. The setter repeats an + // idempotent cleanup while returning the runner-owned flag to false. + final_arena_cleanup = ds4_mmq_q81_persistent_cleanup(); + ds4_mmq_set_gb10_optimizations(0); + q81_env_restore_ok = q81_env.restore() && q81_env_restore_ok; + persistent_active = false; + }; auto cleanup = [&]() { + teardown_persistent(); if (d_down_global) cudaFree(d_down_global); if (d_mid_global) cudaFree(d_mid_global); if (d_up_global) cudaFree(d_up_global); @@ -1119,13 +1148,109 @@ bool run_iq2_xxs_q2_K_fused_raw_parity( if (d_up_w) cudaFree(d_up_w); if (d_gate_w) cudaFree(d_gate_w); if (stream) cudaStreamDestroy(stream); - if (persistent_q81) { - unsetenv("DS4_CUDA_MMQ_Q81_PERSISTENT"); - ds4_mmq_set_gb10_optimizations(0); + }; + + if (persistent_q81) { + if (!q81_env.set("1")) return false; + initial_arena_cleanup = ds4_mmq_q81_persistent_cleanup(); + ds4_mmq_set_gb10_optimizations(1); + persistent_active = true; + ds4_mmq_q81_persistent_counters( + nullptr, nullptr, nullptr, nullptr, + &q81_init_allocations0, &q81_init_resizes0, + &q81_init_arena0, nullptr); + q81_lazy_init_rc = ds4_mmq_init(persistent_device); + ds4_mmq_q81_persistent_counters( + nullptr, nullptr, nullptr, nullptr, + &q81_init_allocations1, &q81_init_resizes1, + &q81_init_arena1, nullptr); + // A real `=0` dispatch below is the value-aware opt-out oracle. + if (!q81_env.set("0")) { + cleanup(); + return false; + } + } + + bool setup_ok = true; + cudaError_t setup_err = cudaSuccess; + const char *setup_step = nullptr; + size_t setup_bytes = 0; + bool setup_null_pointer = false; + if (!persistent_q81) { + setup_err = cudaStreamCreate(&stream); + if (setup_err != cudaSuccess) { + setup_ok = false; + setup_step = "cudaStreamCreate"; + } + } + // Persistent Q8_1 admission deliberately requires the legacy default + // stream, represented by the null handle initialized above. + auto try_alloc = [&](void **ptr, size_t bytes, const char *label) { + if (!setup_ok) return; + setup_err = cudaMalloc(ptr, bytes); + if (setup_err != cudaSuccess || !*ptr) { + setup_ok = false; + setup_step = label; + setup_bytes = bytes; + setup_null_pointer = setup_err == cudaSuccess && !*ptr; } }; - if (!allocated) { - fprintf(stderr, "fused raw parity allocation failed\nFAIL\n\n"); + try_alloc(&d_gate_w, + gate_compact.size() * sizeof(block_iq2_xxs), "gate weights"); + try_alloc(&d_up_w, + up_compact.size() * sizeof(block_iq2_xxs), "up weights"); + try_alloc(&d_down_w, + down_compact.size() * sizeof(block_q2_K), "down weights"); + try_alloc(&d_gate_global_w, + gate_global.size() * sizeof(block_iq2_xxs), "global gate weights"); + try_alloc(&d_up_global_w, + up_global.size() * sizeof(block_iq2_xxs), "global up weights"); + try_alloc(&d_down_global_w, + down_global.size() * sizeof(block_q2_K), "global down weights"); + try_alloc((void **)&d_x, X.size() * sizeof(float), "activations"); + try_alloc((void **)&d_ids, + remapped_ids.size() * sizeof(int32_t), "remapped ids"); + try_alloc((void **)&d_global_ids, + global_ids.size() * sizeof(int32_t), "global ids"); + try_alloc((void **)&d_router, + router_weights.size() * sizeof(float), "router weights"); + try_alloc((void **)&d_gate_ref, + mid_count * sizeof(float), "reference gate output"); + try_alloc((void **)&d_up_ref, + mid_count * sizeof(float), "reference up output"); + try_alloc((void **)&d_mid_ref, + mid_count * sizeof(float), "reference mid output"); + try_alloc((void **)&d_down_ref, + down_count * sizeof(float), "reference down output"); + try_alloc((void **)&d_gate_got, + mid_count * sizeof(float), "candidate gate output"); + try_alloc((void **)&d_up_got, + mid_count * sizeof(float), "candidate up output"); + try_alloc((void **)&d_mid_got, + mid_count * sizeof(float), "candidate mid output"); + try_alloc((void **)&d_down_got, + down_count * sizeof(float), "candidate down output"); + try_alloc((void **)&d_gate_global, + mid_count * sizeof(float), "global gate output"); + try_alloc((void **)&d_up_global, + mid_count * sizeof(float), "global up output"); + try_alloc((void **)&d_mid_global, + mid_count * sizeof(float), "global mid output"); + try_alloc((void **)&d_down_global, + down_count * sizeof(float), "global down output"); + + if (!setup_ok) { + if (setup_bytes != 0) { + fprintf(stderr, + "fused raw parity %s allocation (%zu B) failed: %s%s\n", + setup_step, setup_bytes, cudaGetErrorString(setup_err), + setup_null_pointer ? " (null pointer)" : ""); + } else { + fprintf(stderr, "fused raw parity %s failed: %s\n", + setup_step, cudaGetErrorString(setup_err)); + } + (void)cudaGetLastError(); + fprintf(stderr, "FAIL\n\n"); cleanup(); return false; } @@ -1243,7 +1368,10 @@ bool run_iq2_xxs_q2_K_fused_raw_parity( &q81_candidates_off, &q81_uses_off, &q81_hits_off, &q81_fallbacks_off, &q81_allocations_off, &q81_resizes_off, &q81_arena_off, &q81_high_water_off); - setenv("DS4_CUDA_MMQ_Q81_PERSISTENT", "1", 1); + if (!q81_env.set("1")) { + cleanup(); + return false; + } } // Run the same fused path against the original global expert table and // unremapped ids. Bitwise equality with the compact result validates the @@ -1365,11 +1493,8 @@ bool run_iq2_xxs_q2_K_fused_raw_parity( (unsigned long long)(q81_resizes1 - q81_resizes_off), q81_arena1, q81_high_water1); } - int final_arena_cleanup = 0; if (persistent_q81) { - unsetenv("DS4_CUDA_MMQ_Q81_PERSISTENT"); - final_arena_cleanup = ds4_mmq_q81_persistent_cleanup(); - ds4_mmq_set_gb10_optimizations(0); + teardown_persistent(); } const auto mismatches = [](const std::vector & a, @@ -1392,7 +1517,7 @@ bool run_iq2_xxs_q2_K_fused_raw_parity( rc_down == 0 && rc_fused == 0 && rc_global == 0 && rc_global_reuse == 0 && rc_q81_grow == 0 && initial_arena_cleanup == 0 && - final_arena_cleanup == 0 && q81_counters_ok && + final_arena_cleanup == 0 && q81_env_restore_ok && q81_counters_ok && sync_err == cudaSuccess && gate_bad == 0 && up_bad == 0 && mid_bad == 0 && down_bad == 0 && gate_remap_bad == 0 && up_remap_bad == 0 && mid_remap_bad == 0 && down_remap_bad == 0; @@ -1412,6 +1537,12 @@ bool run_iq2_xxs_q2_K_fused_raw_parity( return ok; } +bool run_iq2_xxs_q2_K_fused_raw_persistent_gb10_parity( + int n_tokens, uint32_t seed) { + return run_iq2_xxs_q2_K_fused_raw_parity( + n_tokens, seed, /*persistent_q81=*/true); +} + bool run_q4_K_moe(int M, int K, int nt, int ne, int nu, uint32_t seed) { auto fn = [](block_q4_K * blk, float * out, int n_experts, int M, int K, int blocks_per_expert, @@ -2120,8 +2251,8 @@ int main(int argc, char ** argv) { all_ok &= run_iq2_xxs_q2_K_fused_raw_parity(/*nt=*/8, 0xC2F008); all_ok &= run_iq2_xxs_q2_K_fused_raw_parity(/*nt=*/32, 0xC2F020); all_ok &= run_iq2_xxs_q2_K_fused_raw_parity(/*nt=*/128, 0xC2F080); - all_ok &= run_iq2_xxs_q2_K_fused_raw_parity( - /*nt=*/32, 0xC2F021, /*persistent_q81=*/true); + all_ok &= run_iq2_xxs_q2_K_fused_raw_persistent_gb10_parity( + /*nt=*/32, 0xC2F021); // Step 6 - mmvq vector matmul tests. // From fdfd7dff46e36728486a4d7ca7848b7854a6d0d6 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Sun, 23 Aug 2026 15:31:42 +0200 Subject: [PATCH 077/189] fix(cuda): reconcile Q8 oracle allocation diagnostics --- cuda/mmq/ds4_mmq.cu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cuda/mmq/ds4_mmq.cu b/cuda/mmq/ds4_mmq.cu index e7bd1096e..9eab07858 100644 --- a/cuda/mmq/ds4_mmq.cu +++ b/cuda/mmq/ds4_mmq.cu @@ -490,7 +490,7 @@ static bool ds4_mmq_q8_fold_oracle_bytes( bytes, cudaGetErrorString(fresh_alloc_err), fresh_alloc_err == cudaSuccess ? " (null pointer)" : ""); free(host); - fprintf(stderr, "Unable to allocate tensor \"fresh\"\nERROR: %s", cudaGetErrorString(cudaGetLastError())); + (void)cudaGetLastError(); g_q8_fold_oracle_skips++; return false; } From dee75d4f36446b1c7a6223690e59b481a82915f9 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Sun, 23 Aug 2026 15:31:55 +0200 Subject: [PATCH 078/189] build: preserve targets across ROCm merge --- Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index f9cb75b27..7b2cda6b8 100644 --- a/Makefile +++ b/Makefile @@ -68,7 +68,7 @@ DS4_LINK_LIBS ?= $(CUDA_LDLIBS) METAL_LDLIBS := $(LDLIBS) endif -.PHONY: all help clean test environment-docs test-rocm test-glm53-kda-rocm test-metal-session-batch test-metal-session-batch-ssd test-metal-q4-streams test-metal-q4-attn-exactn test-metal-exactn-oracle test-metal-dspark-capture test-metal-iq2-midonly test-metal-iq2-ssd-grouped-mm test-metal-iq2-live-index test-mxfp4-cuda test-mxfp4-rocm test-mmq-parity-cuda test-rocm-q4-parity test-rocm-q4-dense test-rocm-q4-pair test-rocm-q4-prefill test-strix-rocm-q4-parity test-strix-rocm-q4-prefill test-strix-rocm-q4-prefill-long test-cuda-session-batch test-cuda-mixed-batch dspark-acceptance dspark-verify-depth rocm-dspark-acceptance rocm-dspark-verify-depth mtp-verify-depth cpu cuda cuda-spark cuda-generic cuda-regression strix-halo rocm +.PHONY: all help clean test environment-docs test-rocm test-glm53-kda-rocm test-metal-session-batch test-metal-session-batch-ssd test-metal-q4-streams test-metal-q4-attn-exactn test-metal-exactn-oracle test-metal-dspark-capture test-metal-iq2-midonly test-metal-iq2-ssd-grouped-mm test-metal-iq2-live-index test-mxfp4-metal test-mxfp4-cuda test-mxfp4-rocm test-mmq-parity-cuda test-rocm-q4-parity test-rocm-q4-dense test-rocm-q4-pair test-rocm-q4-prefill test-strix-rocm-q4-parity test-strix-rocm-q4-prefill test-strix-rocm-q4-prefill-long test-cuda-session-batch test-cuda-mixed-batch dspark-acceptance dspark-verify-depth rocm-dspark-acceptance rocm-dspark-verify-depth mtp-verify-depth cpu cuda cuda-spark cuda-generic cuda-regression strix-halo rocm ifeq ($(UNAME_S),Darwin) .PHONY: metal-decode-schedule-bench metal-prefill-variant-bench check-mxfp4-half-lut test-mxfp4-metal @@ -275,10 +275,10 @@ help: @echo " make test-strix-rocm-q4-parity Require a visible gfx1151 device and run the Q4 tests" @echo " make strix-halo Build ROCm for Strix Halo / gfx1151" @echo " make rocm Alias for make strix-halo" - @echo " make test-mxfp4-rocm Build and run the synthetic ROCm MXFP4 MoE test" - @echo " make test-rocm Core regression suite on ROCm-only hosts" @echo " make rocm-dspark-acceptance Build ROCm and run the DSpark acceptance fixture" @echo " make rocm-dspark-verify-depth Build ROCm and run the DSpark verifier invariant" + @echo " make test-mxfp4-rocm Build and run the synthetic ROCm MXFP4 MoE test" + @echo " make test-rocm Core regression suite on ROCm-only hosts" @echo " make cpu Build CPU-only ./ds4, ./ds4-server, ./ds4-bench, ./ds4-eval, and ./ds4-agent" @echo " make test Build and run tests" @echo " make environment-docs Generate and verify the environment variable inventory" @@ -813,4 +813,4 @@ mxfp4-dot-test: tests/test_mxfp4_dot.c ./tests/test_mxfp4_dot clean: - rm -f ds4 ds4-server ds4-bench ds4-eval ds4-agent ds4_cpu ds4_native ds4_server_test ds4_test ds4_agent_test gguf-tools/quality-testing/score_official gguf-tools/quality-testing/score_official.o speed-bench/metal_decode_schedule_bench speed-bench/metal_prefill_variant_bench speed-bench/*.o tests/test_q4k_dot tests/test_mxfp4_dot tests/test_mxfp4_metal tests/test_mxfp4_rocm tests/test_mxfp4_cuda tests/test_rocm_q4_dense_pair tests/test_metal_session_batch tests/test_metal_q4_streams tests/test_metal_q4_attn_exactn tests/test_metal_exactn_oracle tests/test_metal_dspark_capture tests/test_metal_iq2_midonly tests/test_metal_iq2_ssd_grouped_mm tests/test_metal_iq2_live_index tests/test_glm53_kda tests/test_glm53_kda_rocm tests/test_glm53_vision_engine tests/test_glm53_vision_prompt tests/test_gpu_xdev tests/test_gpu_model_cache tests/test_gpu_lookup_cache_strict tests/test_engine_mgpu_refusal tests/test_engine_mgpu_runtime tests/test_engine_correctness tests/test_sampling tests/test_cuda_session_batch tests/test_cuda_mixed_batch tests/*.o *.o tests/cuda_long_context_smoke tests/cuda_long_context_smoke.o + rm -f ds4 ds4-server ds4-bench ds4-eval ds4-agent ds4_cpu ds4_native ds4_server_test ds4_test ds4_agent_test gguf-tools/quality-testing/score_official gguf-tools/quality-testing/score_official.o speed-bench/metal_decode_schedule_bench speed-bench/metal_prefill_variant_bench speed-bench/*.o tests/test_q4k_dot tests/test_mxfp4_dot tests/test_mxfp4_metal tests/test_mxfp4_rocm tests/bench_mxfp4_rocm tests/test_mxfp4_cuda tests/test_rocm_q4_dense_pair tests/test_metal_session_batch tests/test_metal_q4_streams tests/test_metal_q4_attn_exactn tests/test_metal_exactn_oracle tests/test_metal_dspark_capture tests/test_metal_iq2_midonly tests/test_metal_iq2_ssd_grouped_mm tests/test_metal_iq2_live_index tests/test_glm53_kda tests/test_glm53_kda_rocm tests/test_glm53_vision_engine tests/test_glm53_vision_prompt tests/test_gpu_xdev tests/test_gpu_model_cache tests/test_gpu_lookup_cache_strict tests/test_engine_mgpu_refusal tests/test_engine_mgpu_runtime tests/test_engine_correctness tests/test_sampling tests/test_cuda_session_batch tests/test_cuda_mixed_batch tests/*.o *.o tests/cuda_long_context_smoke tests/cuda_long_context_smoke.o From 5532f5d9fb895106793b5d5e81d88f6230713783 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Sun, 23 Aug 2026 15:32:05 +0200 Subject: [PATCH 079/189] docs: document ROCm MXFP4 environment controls --- ENVIRONMENT_VARIABLES.md | 10 ++++++++-- scripts/environment_variables.tsv | 6 ++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/ENVIRONMENT_VARIABLES.md b/ENVIRONMENT_VARIABLES.md index e808b2f5b..db71de60d 100644 --- a/ENVIRONMENT_VARIABLES.md +++ b/ENVIRONMENT_VARIABLES.md @@ -112,7 +112,7 @@ above, it is an unstable internal diagnostic or tuning interface. The linked sou remains normative for exact eligibility gates, bounds, and architecture-specific defaults. -Inventory totals: **1061 `DS4_*` runtime variables** and +Inventory totals: **1067 `DS4_*` runtime variables** and **6 external runtime variables**. The auxiliary inventories contain **112 test/test-fixture entries** and **19 tool/wrapper entries**. @@ -905,7 +905,7 @@ and **19 tool/wrapper entries**.
-ROCm (134) +ROCm (140) | Variable | Accepted value and default | Effect | Source | | --- | --- | --- | --- | @@ -950,6 +950,10 @@ and **19 tool/wrapper entries**. | `DS4_ROCM_DISABLE_STREAMING_STATIC_DECODE_MAP` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming static decode map. | [ds4.c:18216](ds4.c#L18216) | | `DS4_ROCM_DISABLE_STREAMING_STATIC_MAP_STATE_CACHE` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming static map state cache. | [ds4.c:18229](ds4.c#L18229) | | `DS4_ROCM_DSV4_PREQUANT_DECODE` | sampled once; unset: enabled; present empty or exact 0: disabled; every other present value: enabled; quality mode and GLM models force it off regardless | ROCm DeepSeek-V4 decode: quantizes one-token F32 activations to Q8 once and selects the prequantized Q8_0/DP4A projection kernels instead of the full-F32 activation paths. | [rocm/ds4_rocm_runtime.cuh:4775](rocm/ds4_rocm_runtime.cuh#L4775) | +| `DS4_ROCM_ENABLE_MXFP4_LDSB` | presence opt-in; unset=off; any defined value including empty or 0 enables the candidate when the MXFP4 path, sorted expert tiles, token count >= 128, LDS-size limit, and dimension-alignment gates all pass | Select the ROCm MXFP4 prefill gate/up kernel that stages eight gate and eight up weight rows in LDS and reuses them across expert tiles of up to 128 tokens. | [rocm/ds4_rocm_moe_launch.cuh:782](rocm/ds4_rocm_moe_launch.cuh#L782) | +| `DS4_ROCM_ENABLE_MXFP4_ROW64` | presence opt-in; unset=off; any defined value including empty or 0 enables the candidate when the MXFP4 sorted-tile path has at least 8 tokens and the TILE32, LDSB, and TILE4 candidates are not selected | Select the ROCm MXFP4 gate/up tile8 occupancy variant with 64 row slots and 512 threads per block. | [rocm/ds4_rocm_moe_launch.cuh:798](rocm/ds4_rocm_moe_launch.cuh#L798) | +| `DS4_ROCM_ENABLE_MXFP4_TILE32` | presence opt-in; unset=off; any defined value including empty or 0 enables the candidate when the MXFP4 sorted-tile path has at least 32 tokens and the expert intermediate dimension is divisible by 32 | Select the ROCm MXFP4 gate/up tile32 kernel, reusing each loaded expert-weight chunk across as many as 32 tokens. | [rocm/ds4_rocm_moe_launch.cuh:786](rocm/ds4_rocm_moe_launch.cuh#L786) | +| `DS4_ROCM_ENABLE_MXFP4_TILE4` | presence opt-in; unset=off; any defined value including empty or 0 enables the candidate when the MXFP4 sorted-tile path has at least 5 tokens and neither TILE32 nor LDSB is selected | Select the ROCm MXFP4 gate/up tile4 occupancy variant, reducing staged-activation LDS per block. | [rocm/ds4_rocm_moe_launch.cuh:794](rocm/ds4_rocm_moe_launch.cuh#L794) | | `DS4_ROCM_ENABLE_Q4_DENSE_PAIR` | presence opt-in; unset=off; DISABLE takes precedence | Enable rocm enable q4 dense pair. | [rocm/ds4_rocm_q4.cuh:434](rocm/ds4_rocm_q4.cuh#L434) | | `DS4_ROCM_ENABLE_Q4_GROUPED_ATTN_A` | presence opt-in; unset=off unless REQUIRE; DISABLE wins | Enable rocm enable q4 grouped attn a. | [rocm/ds4_rocm_q4.cuh:704](rocm/ds4_rocm_q4.cuh#L704) | | `DS4_ROCM_ENABLE_STREAMING_FULL_EXPERT_ADDR_TABLE` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming full expert addr table. | [ds4.c:18240](ds4.c#L18240) | @@ -1000,7 +1004,9 @@ and **19 tool/wrapper entries**. | `DS4_ROCM_MOE_DECODE_GATE_RPB` | sampled once; nonempty value is parsed by strtoul (a numeric prefix is sufficient), cast to uint32_t, and accepted only if 1/2/4/8/16/32; unset/empty/invalid defaults to 1 in non-quality SSD mode when DS4_ROCM_MOE_DECODE_RPB is unset/empty, otherwise inherits the resolved base RPB | Sets output rows (warps) per block for ROCm Q2_K routed-MoE decode gate/up kernels; threads per block are value * 32. | [rocm/ds4_rocm_runtime.cuh:4836](rocm/ds4_rocm_runtime.cuh#L4836) | | `DS4_ROCM_MOE_DECODE_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm moe decode profile. | [rocm/ds4_rocm_moe_launch.cuh:82](rocm/ds4_rocm_moe_launch.cuh#L82) | | `DS4_ROCM_MOE_DECODE_RPB` | sampled once; nonempty value is parsed by strtoul (a numeric prefix is sufficient), cast to uint32_t, and accepted only if 1/2/4/8/16/32; unset/empty/invalid default: quality=8, non-quality SSD=2, resident=1 | Sets the base ROCm Q2_K decode-MoE rows-per-block value inherited by gate/up and down controls, except the automatic SSD gate specialization defaults to 1 when this variable is unset/empty. | [rocm/ds4_rocm_runtime.cuh:4832](rocm/ds4_rocm_runtime.cuh#L4832) | +| `DS4_ROCM_MOE_PATH_DEBUG` | presence diagnostic; unset=off; any defined value including empty or 0 enables it | Print ROCm routed-MoE path selection, sorted-tile scratch state, and MXFP4 gate/up launch diagnostics to stderr. | [rocm/ds4_rocm_moe_launch.cuh:831](rocm/ds4_rocm_moe_launch.cuh#L831) | | `DS4_ROCM_MOE_WRITE_CLAMPED_ACT` | Pure presence sentinel: any defined value, including empty or "0", is active; DS4_METAL_MOE_WRITE_CLAMPED_ACT is an accepted fallback alias. On ROCm the variable is only consumed as a path-admission veto: it disables selected-expert cache/address-table, selected-slot and CPU-router/fused optimized paths. No ROCm call site parses a clamp amount or directly enables a write-clamped kernel. | Force shared graph selection away from optimizations incompatible with the clamped-intermediate MoE diagnostic; on ROCm this is a compatibility/rollback gate, not itself a clamped-write implementation. | [ds4.c:18526](ds4.c#L18526) | +| `DS4_ROCM_MXFP4_DOWN_RGROUP` | nonempty value is parsed by strtol and a numeric prefix is sufficient; integers 1..8 are accepted; unset, empty, invalid, or out-of-range values use 1 | Set how many 32-row output blocks each ROCm MXFP4 tiled down-projection block computes, reducing the first launch-grid dimension as the value increases. | [rocm/ds4_rocm_moe_launch.cuh:801](rocm/ds4_rocm_moe_launch.cuh#L801) | | `DS4_ROCM_Q4_GROUPED_ATTN_A_STATS` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Print counters for rocm q4 grouped attn a stats. | [rocm/ds4_rocm_q4.cuh:480](rocm/ds4_rocm_q4.cuh#L480) | | `DS4_ROCM_Q4_PREFILL_TILE8_STATS` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Print counters for rocm q4 prefill tile8 stats. | [rocm/ds4_rocm_q4.cuh:514](rocm/ds4_rocm_q4.cuh#L514) | | `DS4_ROCM_Q8_DECODE_SHAREDX_64K` | sampled once; unset: enabled; present empty or exact 0: disabled; every other present value: enabled; effective only for one-token non-prequant Q8_0 matmul with 8192 < in_dim <= 16384 | Allows the ROCm shared-input Q8 decode kernel to use up to 64 KiB dynamic LDS for wide inputs; an unsupported/failed LDS launch automatically falls back to the regular kernel. | [rocm/ds4_rocm_runtime.cuh:4805](rocm/ds4_rocm_runtime.cuh#L4805) | diff --git a/scripts/environment_variables.tsv b/scripts/environment_variables.tsv index fe9580a00..b4b68d609 100644 --- a/scripts/environment_variables.tsv +++ b/scripts/environment_variables.tsv @@ -958,6 +958,10 @@ runtime/rocm DS4_ROCM_DISABLE_STREAMING_SPLIT_SELECTED presence rollback flag; u runtime/rocm DS4_ROCM_DISABLE_STREAMING_STATIC_DECODE_MAP presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming static decode map. ds4.c:18216 runtime/rocm DS4_ROCM_DISABLE_STREAMING_STATIC_MAP_STATE_CACHE presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming static map state cache. ds4.c:18229 runtime/rocm DS4_ROCM_DSV4_PREQUANT_DECODE sampled once; unset: enabled; present empty or exact 0: disabled; every other present value: enabled; quality mode and GLM models force it off regardless ROCm DeepSeek-V4 decode: quantizes one-token F32 activations to Q8 once and selects the prequantized Q8_0/DP4A projection kernels instead of the full-F32 activation paths. rocm/ds4_rocm_runtime.cuh:4775 +runtime/rocm DS4_ROCM_ENABLE_MXFP4_LDSB presence opt-in; unset=off; any defined value including empty or 0 enables the candidate when the MXFP4 path, sorted expert tiles, token count >= 128, LDS-size limit, and dimension-alignment gates all pass Select the ROCm MXFP4 prefill gate/up kernel that stages eight gate and eight up weight rows in LDS and reuses them across expert tiles of up to 128 tokens. rocm/ds4_rocm_moe_launch.cuh:782 +runtime/rocm DS4_ROCM_ENABLE_MXFP4_ROW64 presence opt-in; unset=off; any defined value including empty or 0 enables the candidate when the MXFP4 sorted-tile path has at least 8 tokens and the TILE32, LDSB, and TILE4 candidates are not selected Select the ROCm MXFP4 gate/up tile8 occupancy variant with 64 row slots and 512 threads per block. rocm/ds4_rocm_moe_launch.cuh:798 +runtime/rocm DS4_ROCM_ENABLE_MXFP4_TILE32 presence opt-in; unset=off; any defined value including empty or 0 enables the candidate when the MXFP4 sorted-tile path has at least 32 tokens and the expert intermediate dimension is divisible by 32 Select the ROCm MXFP4 gate/up tile32 kernel, reusing each loaded expert-weight chunk across as many as 32 tokens. rocm/ds4_rocm_moe_launch.cuh:786 +runtime/rocm DS4_ROCM_ENABLE_MXFP4_TILE4 presence opt-in; unset=off; any defined value including empty or 0 enables the candidate when the MXFP4 sorted-tile path has at least 5 tokens and neither TILE32 nor LDSB is selected Select the ROCm MXFP4 gate/up tile4 occupancy variant, reducing staged-activation LDS per block. rocm/ds4_rocm_moe_launch.cuh:794 runtime/rocm DS4_ROCM_ENABLE_Q4_DENSE_PAIR presence opt-in; unset=off; DISABLE takes precedence Enable rocm enable q4 dense pair. rocm/ds4_rocm_q4.cuh:434 runtime/rocm DS4_ROCM_ENABLE_Q4_GROUPED_ATTN_A presence opt-in; unset=off unless REQUIRE; DISABLE wins Enable rocm enable q4 grouped attn a. rocm/ds4_rocm_q4.cuh:704 runtime/rocm DS4_ROCM_ENABLE_STREAMING_FULL_EXPERT_ADDR_TABLE presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming full expert addr table. ds4.c:18240 @@ -1008,7 +1012,9 @@ runtime/rocm DS4_ROCM_MOE_DECODE_DOWN_RPB sampled once; nonempty value is parsed runtime/rocm DS4_ROCM_MOE_DECODE_GATE_RPB sampled once; nonempty value is parsed by strtoul (a numeric prefix is sufficient), cast to uint32_t, and accepted only if 1/2/4/8/16/32; unset/empty/invalid defaults to 1 in non-quality SSD mode when DS4_ROCM_MOE_DECODE_RPB is unset/empty, otherwise inherits the resolved base RPB Sets output rows (warps) per block for ROCm Q2_K routed-MoE decode gate/up kernels; threads per block are value * 32. rocm/ds4_rocm_runtime.cuh:4836 runtime/rocm DS4_ROCM_MOE_DECODE_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm moe decode profile. rocm/ds4_rocm_moe_launch.cuh:82 runtime/rocm DS4_ROCM_MOE_DECODE_RPB sampled once; nonempty value is parsed by strtoul (a numeric prefix is sufficient), cast to uint32_t, and accepted only if 1/2/4/8/16/32; unset/empty/invalid default: quality=8, non-quality SSD=2, resident=1 Sets the base ROCm Q2_K decode-MoE rows-per-block value inherited by gate/up and down controls, except the automatic SSD gate specialization defaults to 1 when this variable is unset/empty. rocm/ds4_rocm_runtime.cuh:4832 +runtime/rocm DS4_ROCM_MOE_PATH_DEBUG presence diagnostic; unset=off; any defined value including empty or 0 enables it Print ROCm routed-MoE path selection, sorted-tile scratch state, and MXFP4 gate/up launch diagnostics to stderr. rocm/ds4_rocm_moe_launch.cuh:831 runtime/rocm DS4_ROCM_MOE_WRITE_CLAMPED_ACT Pure presence sentinel: any defined value, including empty or "0", is active; DS4_METAL_MOE_WRITE_CLAMPED_ACT is an accepted fallback alias. On ROCm the variable is only consumed as a path-admission veto: it disables selected-expert cache/address-table, selected-slot and CPU-router/fused optimized paths. No ROCm call site parses a clamp amount or directly enables a write-clamped kernel. Force shared graph selection away from optimizations incompatible with the clamped-intermediate MoE diagnostic; on ROCm this is a compatibility/rollback gate, not itself a clamped-write implementation. ds4.c:18526 +runtime/rocm DS4_ROCM_MXFP4_DOWN_RGROUP nonempty value is parsed by strtol and a numeric prefix is sufficient; integers 1..8 are accepted; unset, empty, invalid, or out-of-range values use 1 Set how many 32-row output blocks each ROCm MXFP4 tiled down-projection block computes, reducing the first launch-grid dimension as the value increases. rocm/ds4_rocm_moe_launch.cuh:801 runtime/rocm DS4_ROCM_Q4_GROUPED_ATTN_A_STATS presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Print counters for rocm q4 grouped attn a stats. rocm/ds4_rocm_q4.cuh:480 runtime/rocm DS4_ROCM_Q4_PREFILL_TILE8_STATS presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Print counters for rocm q4 prefill tile8 stats. rocm/ds4_rocm_q4.cuh:514 runtime/rocm DS4_ROCM_Q8_DECODE_SHAREDX_64K sampled once; unset: enabled; present empty or exact 0: disabled; every other present value: enabled; effective only for one-token non-prequant Q8_0 matmul with 8192 < in_dim <= 16384 Allows the ROCm shared-input Q8 decode kernel to use up to 64 KiB dynamic LDS for wide inputs; an unsupported/failed LDS launch automatically falls back to the regular kernel. rocm/ds4_rocm_runtime.cuh:4805 From 8d84bda46dd8075287151a26cc9b530e97d942a9 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Mon, 24 Aug 2026 00:43:31 +0200 Subject: [PATCH 080/189] Update metal/moe.metal Co-authored-by: Frank --- metal/moe.metal | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/metal/moe.metal b/metal/moe.metal index ac4269c11..a5903df96 100644 --- a/metal/moe.metal +++ b/metal/moe.metal @@ -4023,10 +4023,10 @@ static inline void ds4_attn_out_low_q8_static_impl( device const float *yb = y + ib0 * QK8_0 + il * NQ; float yl[NQ]; - for (int ib = ib0; ib < DS4_ATTN_OUT_LOW_Q8_STATIC_BLOCKS; + FOR_UNROLL (int ib = ib0; ib < DS4_ATTN_OUT_LOW_Q8_STATIC_BLOCKS; ib += NSG * NQ) { - for (short i = 0; i < NQ; ++i) yl[i] = yb[i]; - for (short row = 0; row < NR0; ++row) { + FOR_UNROLL (short i = 0; i < NQ; ++i) yl[i] = yb[i]; + FOR_UNROLL (short row = 0; row < NR0; ++row) { device const int8_t *qs = ax[row][ib].qs + il * NQ; float sumq = 0.0f; FOR_UNROLL (short i = 0; i < NQ; ++i) sumq += qs[i] * yl[i]; From 59636dfd76b40d89072177054b262ab68092e7f3 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Mon, 24 Aug 2026 09:09:25 +0200 Subject: [PATCH 081/189] metal: fuse router finalize during SSD decode --- ds4_metal.m | 1 - 1 file changed, 1 deletion(-) diff --git a/ds4_metal.m b/ds4_metal.m index 7cb347c28..d3a7b5fa9 100644 --- a/ds4_metal.m +++ b/ds4_metal.m @@ -35966,7 +35966,6 @@ static int ds4_gpu_encode_router_select( (!pre_m5_device || getenv("DS4_METAL_DISABLE_PRE_M5_ROUTER_SIMD_WEIGHTS_FUSION") == NULL); const bool use_pre_m5_transform_finalize_fusion_default = - !g_ssd_streaming_mode && (pre_m5_device || ds4_gpu_device_name_contains("M5")) && getenv("DS4_METAL_DISABLE_PRE_M5_ROUTER_TRANSFORM_FINALIZE_FUSION") == NULL; const bool use_transform_finalize_fusion = From d24c0987dc76cc6553c62587cf03841690852daa Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Mon, 24 Aug 2026 09:10:04 +0200 Subject: [PATCH 082/189] chore: ignore local speed benchmark outputs --- .gitignore | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitignore b/.gitignore index 5e60e2bca..e21ca8074 100644 --- a/.gitignore +++ b/.gitignore @@ -40,3 +40,8 @@ __pycache__/ /misc/ .*.swp .DS_Store + +# Local speed benchmark outputs +/speed-bench/*.csv +/speed-bench/*_logprobs.json +/speed-bench/*_logits/ From fffc46937cdcd9583538e517e7567562e3ca52fe Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Mon, 24 Aug 2026 09:10:17 +0200 Subject: [PATCH 083/189] docs: remove detailed PR 621 summary --- Sintesi_dettagliata_PR_621.docx | Bin 58995 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 Sintesi_dettagliata_PR_621.docx diff --git a/Sintesi_dettagliata_PR_621.docx b/Sintesi_dettagliata_PR_621.docx deleted file mode 100644 index 053c900807b6a7584b3e1885f0db95844bcca1b7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 58995 zcmY&;Q*DFX!o8HWM^LH&2CEn;uyVru82uj1)o>a0ueVQbTz{L_A3 z5Gm~X19k#~AikRlDnua*KMyw_MaW_EB6i>QS%&9#%j@$bcTA{sl=5F4VlI~JciykW ztftt>YIovp;T=#0{)s6oQ5v_)@15Bw1&fqvG$Yb zZ6Q*bmjg;BQvZDs+?dqfU=vWWLc1OE){fnqLFERJV>OO`<1?!GJF_hNQh{LqT!R6z@$}3daJYH`T3{HugQv&>%zdExaTm`9NP1acZcaY zrq+xZJIXr!q|&8}Yypz0O@A6zWT!?s!ej2E^D@Bhgu74>s@W0Oa`^8CR3v=#RX8pu zM*xJfoS8@uCAJHm{;(QR|HSV5d(ylF%C*Q}qXoW47iS!2bi%t|yEZIxS*(743ytrT zYA*b<-jYT&d_zPDB})a|pwOIdNqeEnX9l*>e3OV)b26-9M5fc^N#*Zan1OPPf@D>NlG!Rkhui$VFz{Tka;w@sb; z8_+2*U4cX%DtjTCHW|MjNe~51$)DNH)Cl=$wor$(sT!~5gldFE^{H(o5z9i#eZew0XyD06)dzXZ|P5hMdX$BJV+lPr@B!7aGE#qmd>^9OB~UWl1!<4O`ExH1uObH z%-~-z)~8-;HOH;^LXXX9vtR%B1tG+Q^tJzEz7RMF2=c!RGO;&SaI$xBW-zvQGW`$l zvlA!m78udOo_RyrXd8(3b@GH2$uAx6X}iv~bn;W7#psVRvVO@&DA3yJP` zz>yf_F1(Um-Qkzaaqe2)!QO00{n{n(z-Q$QBOJXHy$IncbT)heb+n4c9OBgr-EaR* zxFyfpJ@mHc>S`(@Kls#EDS{VBV77tF&P=0!_Up&`gsM8aX?Cir-r{THM)EZByf=-% zFAJGLHOX11qEEaQj4C*8!Zz1NjU25{9TPhPDj>?y)rOFD9rb6%s}~{ZU|iDG6>7%4 zi5rb3Z}FiVoaJd|T-Fc;x0<|5I^`UEWSccr71GXM^%D<)>>40 zmd)_7N$fwKX{ID2Ac}(-ezE{2JS#0JS0ltReiGs^8Fb%FS^%uzyj>m}Oiq#W%x$$X zc;`t3&+%y;?&jfN{=gn!7#Hmb`prIyJ5*+0QLckvL^Yi^5-K;8t4G|$T*mim2#w@% zv4MYzP)3jaB}uk2wC{gF&eaIqaExzzYAol8N0fO&1ibMGuY3SSsnfJn0D!&H!P)7m3=w!r>GqQA9e zI^KD8<%2tOhvf8YYlOj1y@av(9E+a74{Wp9PEF>YQr{-!3D3e%^|p~uNd2=eB~UCc z(cP8F@WFt81C#lBkMq6wctP;r#F%kF?*UT*0r?yO`=0{fZtrBm@UIHE+WzZ*|6#^u zx0LhxN(w!(|wL>+g8o zNY>wu51Y!3ro5)acl(0kh1WSf{Qh$fISsVOihYBeDBFD9$gA_${?GoOn~!El??a-Z zT){M=t4FST{?8^^$75!l*{{Y-yPDR-pRCh+hFf?0Qat#G#SCl9j@V6fgRCdDq%pn; zrZ=uUyd8V1sJ`t|KE0Jc+w{LIvBIEgO;vRFukeNVhghe+c1qa|m8n^F zwb@ZfcAT*d^|B@BPsQrNKSmD4_}}_iet2i6P1mkJY$oXs)}BIKJHH2VTcT9(e7~Uo z+{fn$N-djio1vTZH}|Ay{Nw& znDE=2dKlpah0J)#ges(zdnF9v2|_oQOpy13`fG^<#qq+EPf$wrHI8yMI0ny;UYG6= zb%-gyLq>1}wld3-)8zl`etG8cDDwSD9D4^JH{}B()1G|b7xVl` z<>NOb@{-eC)3~WfV18ZUK5;$2o$y`b@8Hw3EMU+HWpixe9+!Rj=~4F##Y?IVRk7ar zX&@!g3nS*a>YN_FSNlkM1ZUoQzUhZD()meRDu&sh(nTej(%9=tch5plpbvjn`-JN( zKfCHg=YBk`$Lao5AeqQHwYp@9{e7A|`1GvbJIcOFbG!!yjG#n~KIV#$kMD!0yq6!K zFzmYFx)&<*Kdt;RAkfEg%mIT$#BhgsjRT0ww>y{j^VH7AJUZj!JdEG|Bc_+lkz_B% z_{kW{uv?ff+vr1q_)7|q`B>z0Oxv@>Aj1E-LAjk5cdClky8&sd-~Fc8U7&m#3vXUY zy1qKHDsLL#AG^b$qsJG!@AVc5?AgrWVNp-LTa@=T^zV@E*RD|?Yv`=$xL8!9lu-1v z=D!G&e2jXL(9tE9ThPSkn|siT0&&VYI&+@aa_J+SzW2CGpoyS-xp}=z#~B+2y%dLb ze;e@H?K)ADuz=K{tZ1@P(k4^9)5_L+-K&C@0n0f4{iL3vpR?(@vCeDkW`944m`i#K zhG-$r@if1F{={qE-96FPEf9nnANAwfzH@I+>;8_}&WbEYp!=6OaU@In$mx`Qg0i~L zJtrn^xyLdhv-wnbWp}dn%_70?5^H>&zcK5nj@*g2Qg=++=2O9yb;+Mu$4vV>72^sz zj_~21=}C)lA!TG;coOL^WUH)s!oQZ@k&{)fQaw7e z&9P19F+sTvnfsAW5Y zwKJF@2^)H0ha>&KGfYhDE&M5lf5nWoo4xl7QD!I^Z8=)WBwxq*+RFY5t}T*Qo@-)= zG*TJAM)m5@v+yQ!B(;Hh!2HXV;lLp&WuH3PKFdo>??7Z4qC^~SJc_-#7*H{{$Wk-WzS!ORNo ziuZuXywu!4{il&ql13N9TkDo%PY#j~$p$}`KBe1c`K!W(#F!-`-YcuG_ zjGP-e0_*hb9hi)*%1uBVG4m?95lU&|pkgp_SgTSxy5V#(-*XAbYQpy&!HlNf0atI( zzRBpFBP^+qViNq7EbIz6{;L6KAAgw#Eah+i3DJkx{YOzRvaRorYlM`D1%mIp-hTK$ zTKz96ToLbSig{B6_Z9%2A~+Gz**>1JdsZ_M`axoJhgH&;;DJ(UUSK5S0qt)2r2WpV zDgDXZ`ulnbsQj=uG$^(&`8cr*{E2N^qb#n+!6(q1w`ARO?@`!-5?k^ zQDUU*ix_<^{}kT+j2mU6PbYLg*E>j@hcoS+W&LZV=7Aj5g3xHezz;Fr2}IBY@H#{X zm@Eo#!it|aO!XiVQSMFT73|^TF7fdaf`6#~nh^WHerxC)NF{)$7g3!N6J`3+zgG33Jj1ngexsg)d+VY|uoEzYpYcs;cP6bhAonM2 zG3nP+*XvlR%B*#`;%WE^3{;>yuI9N0aR2U*22 zoGIth`fCZC(YTB^tOdekB)|?(-|~eo4aQNi{z$@HB#+eXHGm2ziXMm(C zH2hou)FHQNI}O~c^qBV1NMq^5&=)Y8h!0-1$J%m*{mewTTesj0+fOf9WOB@QbXM>n zvG0E_IKlWbMVX-VkJaBv1tne~<@`wf4*tvYD-cq4FhHJVwMDDN()ur`Fm7a5tTtp8 zNEiaQBgZIZ+Wn^==R^7BDH2-6LWzeH6zcK{I*!iHS_W=4o; zFb_}CV`GCr!2B30+?%kP^lhJNRqK$wKnJV@lwh-oXPePf8`K@16sjgEKP8w==UIgH z>7!m$Bpw+&eom4!CXujjroZ4PIH>|CHjkG z1)f%+%Ojn;V21G(dD9XJLT4&XJ#?V+WQ%s|ISaE4rGjGl5 zQ1wsUNHF6QgCP%fjQ(IZ$Y+id0Y1ii{Z{f>qtleZA(&JNv(`|brWm18(#da39{>u4 z`$n~()pbSSIAI{o4i&Da(0!pBNdI-`es{t()VB0S)N=S#$B-Iv2X(dR3+@WwMfFGr zX6COtcsGfgM`H7>%!_UO92h5?a5PRQ+Y=+!5wUd4_aloc2a zB4XuVN!So3lq!FxO_e0t#m?Jc3Iu3l$Yuw;hys}*aWgH2!{mQ72o7~q>&J{1!Z^{4 zC$q#I(RgCk)gv?&N)n0fppCb0g61bG2FoF+qft7cns8zzHi0?05p#6PXRzsCGHE$j#=CAcb^`9r}poOIq)op2B()@G;15@ABy}wK*aq*0@Sb{JnKf-Ql&HY*$60`rKna9TUGP#oKE#`u;CDyOOsQ9YE3+p*)6BgIEf&HeHGECiP zACX*&(rloTIq{m&Qd{e>v^KXgM3g$bVRX)HM5p3TvRJxZ8Go|0h z1a5iH9sOy3Nb92cLATwPWUPXQ%&3CR3oZQ8V)Vbna?T!o{DV5@UR%YsS`m$$0 zzR8uL6*nbUEtw>+UWKFR$=~!9RjH6MIN+z;A*~Law)9 ztaKt7p9BD=b-*#gER*#lm@KT_nbjjs*r7di=xmu0PF)4~qC{@ZO7jPp4}z-Il3ccExoD`%e4vh~>+|Xx}7G-4(_7@<5k<8{@Lb+E9i z=ntzef)+YrtDEOW_R|wb%wSFeRrbaVOIKyxzAj_JSZKC1Z|$3I?5pEb zvW4&rWNO z`@BO(`03l#w%Q%G)ZpUZXsaa9^tfJs-_58Ey72dSTYt8~YI_##3+3Y)%pO0d80yr7 zo{rlZ*Y4JTHUo9KV;V2QUeVp`ssRLgFkkcfJ#4lsJlMJS{Y&Z1K{}VVTU6#6WJ;pF z9peUch6FthwZ~05GYjBFqkW#!RpL-w?!?P--)EO)i;dcDGnDQNC3tMvVeiQ;F4eLG zJ~Hl)&!DKodz_xfKVQ!isEL|`q8dheMuJ#(t+75GGLz+RfxP!?4D<<8ciP7aQS zM%>HxrF5Fs-y_5mc5yC4RLWk7=UU1&Pfu4B>_UsWFIRZ6sPEm)u~Ri$;u)LbBP5s_ zSxQrSp67fz?R-A}3QPTLe~7M*^?34nckt*BxcK7M;dVtwPc90#F10k8o{7?qIX`jZ z8nVmr%L4P~e}8obzRK36G@9qoKe&fcomtbX? zq<-v3?djL`oy^s>JF`hjDBdF0R7)G$LdsH>>=vskjomy0*mM{#JQ?Db+k#1cl|eTB zmuO{0HTDxWz$A2jh77@cDvHtP`@Z)-(n?hs6$umKefo|mmfV!HSJlKX>fZ6wgII$+ z9;AFic6!GIqQFYE@O2}JsBepfuMl5iYmhEkL}nL$me>IdkpSXc@aXQWam1Rza=xe- zr{McsU{x)#@AAno|I4$Z1CIVhvN_h}Pn%}Cpgk@Kh_KhDVs3}`#a z#mCR`Kdw)T6K{wR57I;!B&|ex`VGZF7N3IOV$ABL`-XNnkX_5ozq-2S**+o|Y068% z$tNaF_cWoefjotF14uj+6TSbfSl8u*lxcmd6M??n?ws{fwF7bNPDce99N&RdFQo}; zlgGXCqCN9uK!d)`C~&yi*6sV(zL1r{qBwUy2qw#;h`EC>xiko8A#hX*BYve8gwFp} zzt4aWROCZE@q=L|4v{SkDAx`=RX8wZaEfTVu}DU0cv(6Hr2^fIiheSP@~}yKqeNah zt|6V(u(^TYRi24!Bqk>8S6Fhic@b(~KoN>{YU>1BEY8tUp*x?%b)90k#M0Xk1?AY~ zw@{zzG}o56!Iu*50Gl1(ArB;5=ioF ziB8-D1AG@jGi-5U24dxmg{xOZ9j5U9f+_j6-~zD`I-x7^gdk%h z8Dmw<=?>Ts`)kJ>Pi7X9HrNRkSN%_6xTqsv!@BQZ9A!tWEP$*ah{K;f2xa1HBvj;OajkhL8 z;owO48@hfT&@b=O2gF)y;)V{jcJG;kglV(JOp(ydM;{{o`Gs384S@iGo%NwyL5Cv! zCtx}d-BVe%K2#iqpQ0BhIiHXw5DcdXr@u5?HOS?eZ37)lUsyPljJ&HWdocL#ax8}; zJx8RdW@GyB7r-w@EDdS1^h9U))Pqua>R3-EZ%@P(IsE-P9 zU6(z-j4-wPURpbs2QRc343LyCQoQ_j3UB5IVsIb4)D>B~#*}oig3%PhF;A=XXzfpMzz!3}%VEI8~1T|z|$RZI#=lEvY z#4Wt~?xo>rBjNcdVm+>Yfy5&Tz(5ngXE2TWV5YPCiQ!wP}v6C%>Q* zI1zv4@K}Q4_q?wwMj7stDC=d6O*cnQ)@a#Krt^8w}(aHiQAWEC5!9dF>5%TIU)%MwDSI+=vLke@c7qWIBpJ+Vg@Wk8GMs( z{yb(|@zNoNw#1BI+?&^4E3HY`{ko)9P-a44N5Trn1p>cnk~{Nq(Tlt;-o=89a9M06 zc_f*LJWeEeOlIRH6il6%us{~nphy8mGraaKb)?c%g;`V$L5o@JbLryYh)sr;5 zwcaZ;mAUf-MTC9~puQYN?Fw~gUUAU5d6e2XT$fg7-y0Xp`ylSone3G93@Kxw^}tq{ zQ=h_D=*gWh?H~%i7&IFqS9sUpV_w1ghC{1I1j_u5ofE>Aen8>$c}7>?a&E*j@z|z& z@?zJ29Jf?o1-g|IfzTsF*DLZR+4|pCDH^p+p9yw9nDUua8Nb{ENqkmI(;ZhwU2aU- zu%J*`YaX4ALAXV=?#SpamZ|4nFyVG%K|G6-GNeX>RnW>je^vPqM2j_Hjh*j2Ky92! z0s~=l^7lF}%jSLHTJ){PXkh(Le>wSB-PKU}gfZbgDB(!b+_%J%s0!$AMn^*0Eqwa60>eiVJ8;s556mafJaHGY z*~K21=d!I5!&V}*uZc#EnK3ux}()-@C zxn4Z){(9Y_U$2XX1QJGCuDz~0(=ClpTn3lnkG+seQMz_HJGnHu!&3x@L5K{U$~9(N zW#ddbE-r8NV8HXuu?frh5yWa$6+9d`wXP`Q9<8-ZM$JK|RZkWLNktB@m8?Qwp6C6L ztht|s`i?nd{i4cYQc~Q7leI-R;FkHE#wPHFTw7zW7K>QU_2~5dEEZhZOr}BjvL&p_ z307?gU|7plX4`OF>g>$w^oC6Q0eGw3G?-viuc7^Dt?w^YB~-Sa`mhcrtHrlw*hk!E zrxa)?_O^s|VT9bo-@7y_IB%uQQ}@A;xyL0TICQa86{!^O?ZcPJl18Y(4*Qs$M%YI_ zlJ?Cpa1tqHqgurq9v9I&8CcXytmP!L3AIk|I`dvhO~?iNPDbu{DwZUTC`4T#=>W%O!~ z;y8(_B2Oj?nugo3FH?-xjinwoVsVsBf^XfD7hQI0CjQc2&&Own)-<*Grp7MJs>8eS z%UwfW;gGOT-?OQapC-r0{IoB}iuk_IiP%K*{y{ zDShk6E~odGy@QLr$J^DhUBlpFqfuYR;M*Q>CTc~ndM=$uKT1yFO;|FL6l7$T70#Js z9zireGWLd=;)gTjVRko@mfb-x;DxokzeeLL_%*pYN{3~4@UiCu^W^{BDkySmYOt~=tInpoU*rn5M^o{4{cAem#K18C7J6~ii z0jPanS|GI>_$YtRaqV=2>%g#K%S(e8up&kYdAw zK>MmAsKGFraQ)zYNuTKkyHR%z?dIe?3U+olqS!K!*vN+Ac+}2eQFSlvmjUK^Shy_H z5aRtY3qu!;h{mK3XJk4}PAjlAz#&HD1B1={Luth-URLz0&r*Z^K`|>jSxtNglQX(5_3%v1{0X{h$I*kq3!-}vVu(JGW+-bePi(D@-fuNyL0?7nSwXM zvTv!^C>ri*-cDx7BX6i$PJf$SZZ7+gVbbD+{W1IqHvQkG9-UoXa>4Le-^iPA<69vG z0gX9Ig;^0mELPO&_#rHYB(FKZM;hE+rKGOfEI&DZOv?;5H;Bj(uN1;2)el14)HLtS z-Yk50&O%*PPLeR-!~w;XyuPSyHl!{Ct;Gdq_0JnaE86U;C_o>Utwl$CEq!uFNYNC< zh_SX1MlJT9O!3}){`0U`UI_18^23~^ET)h(%vWsJ`+wb~kv*jN;!9tVeTM}?vgDc! zp}p$ejx06kITICX9aGWT0EmPNh966L1C}+rur~W9xlbGMh2z{+yZd1K<|@q*NL4Xc z9P?aEj$r&jk`j3y0wF$e`EzJqMNc;6b#LYM*+kJSLhsTEmdycTDF-8^HUC8Sg- zoaw*8n&ux|8?q_RqP&?{pW%VhSg%%gO|ZDxIlE9kARJYQrNhR7Ob(0uLW&M92bN2m z;&LjCDttL$<2g@+%L>vq;n23coijUGm5$hxf z^!6!C`cW{2rIqA;^G-4fCRQ)Bi++EW6C_>u_c!|9-u>eDlXdz5-;)EPZ zLQ<5t%bJKULv{0JSOOFz2J~5oMf{?Dz2{~L@tg6>E`Y;T-rlo59At8F7qJ>MA7F>E zE4~E?l-D$9*gkQr5Z*A?v=}Ik_K)(a-2uEZ2d>9x)ww$rT04wN$eEIag88=5-^3%6 zGt;0Aj-nz9sWz?p}`*D5{OTqY3Y{Tmo0bL+zqS5b!w>5@z%_cDGu3xtp z#pcqD5>pV0Zv-xIrR5raDnBhnu3-)+uf~ox6h_k>C#V}$LhmFXe!NgdCCk$|`W@i# zsXK8WhNs{}OOjerzkNorh~m%3Fvb9RP&{$_#z?%Lq*<%7Fm9=2bY{drd2p(1))v5c zm;<0ypk?H>h8ziACyW%*{aDvXtox1)HB3DoJFBwlf^wBxKy;Up<&Jsytn0sAXmurn z!GwO%SW*RWg7a;ti6si+s-Q}ynQB~-pL@=a5tBBPV9%D7a&2bwfVt08T`Me0H?8Dg zw+2r{7pwNzB<*1xB+ixw*4PA-&NR;Xvn!9Oxza^ zu15OIJ65#^DJ5F1WJb)E2z6$hF-arQ-Qq5MCo?|?B?Yxp_S?WZ@Zw{spp@zshlq z=BOJk8lC@%(+&nw?B<>pO#@5p4)AJ)pm5Fh1! zN$9*bN@}~zY|r54s@yBBqQ|h>R$L7TmbFl^t^ahu@P^M;LRakeev2poT)od8>Spi1C`h5EsB%T^avTr)l%p z+~~zN&Sc2itZ@&ow>L)tvOJ|2rb$Xter1ZcT6(^mJvZK${&U;D{T@dbvpe6jt-bV* zWv)mbxSzFfhQE4@v0H3$beBIz-cSOX(ExSp?kjU!uS-Pdxw)6Nc!rK^Tk@se=EoN= zCg#@1hgh5o#Rb6owHv+Vg7rP<+kHTb=iZ+ErmX$r75|5&`Z?*6IgIrW=3JohW@ zc#5X79LlQoMtDMM9dClM|DcS?c3pYxDdTLBWcKO0xqkrLhJt<|VDk@zjBwx>m@VJ2 zak^ne0GKqtU%-gL3UgqqQy5v3A-w(IHZSas2QRwsyI^*hVJFc#Wp%NmkXVa|1`1<( z2@uuBB<#WGkTtTel#RRzO=9_-^|IAmn*r7d8~piu?qf;)M)J?xZ-J?qeehC z2{dnjd2}jjeg74i*L-`dk1U5M8r=6T#bT9_(`kUyE|Xl6Cvo32p6xxIt|gtCCGQzg5*!+15bAHTLGl#Ht>n| z@#~??Q3wZ^2p4wQ{V{WYf{mxV>?p&hm`7r%-EdQ1i1)+KnQvn$u0I#~dSy)EZE>}z z(^8ToP&YqD&0F$IrQ>~i=yHxtFTo7q!SI?ycyXi9dcuF~(aafrI%XY3pS|x{+^1ZD zLKbs&=F&_rCL1mra1dQfhtvlC;Cy*8n0-%T7LQQ|YM$-h5)Wpaw;Cks#udw_9%kGp z4;P9fSLIOmoej9~o^?hlsmWBZ@-~`l%9Re&&gsGyD>@nmXM^YKnk7fFE)aqu)XnFb zvHpoVjc56b#HOM*;S&vB*`K)WGu%>;Y3&z0GGh_WnU99=Ad051{^TdsKffQA1sk|Z z^e=*SiK?(kFc)6ko_x&CJAVQr z+XGB+A}Nt=YG1fWOR#EYUtQ0bl`eW#j>*HUgNBaah}KE&62^XCu<~pN*p#04+@%Q} zI_}<$&*YrC!7o=a9X{gc-o0@F@%x<;{z45Bl_*FcNys}RrKy_$?=PHC(HbLE%8D=p zNVv$k3$zlHHpE#!i&&k^*TAtacBszYgpnVSSjsD!)PKJ&_z9g#BcpW~FkuC(_77|P z3_YK-8He^pab`rzm=fF(9XOH|K$*W1Ibtn8XiVy~b)}rB**(H$xrC3A&O()xNMJS0 zC0>He)Q>1(HcMGLx>_aI8LVLlP8-Gzb|Dz+<5X|@$z+k#RX^V1Un7wQF4fF8@#rzz z$Vb8W)1T4dGJn1qtZ*4!u6D%RuXLHsYrwu9oK=$?w+zUuPuunfu6Hm-#L2g@C?93M zr8>u$hmBm?nJR)AiUH1-NOmBe$IWLzkIUx#<0I`dH@k~}bKUfqLcoJtp2c~zlEBZT zR2$0tCNjEd$1k(cMNcWpNoT#r*`mJ*sIVZSotO966?hC?dtNUDpTqYew2Jq+VMGak zf1U;)76+OAhI0Khzd(cBZ*XzBsbQ(|48qHsF#MiEm#=r@9ZYmWuc*1$R9aM4w)JQ% z8mK;hFz~gyfKPu?f^MaH>_s9bHIBx0kN2crENRXkcZe70tuIDsH&gbP)oK-YQA&1Xw=kF|j!Hlp5JVHFNVoW%M{6is zEh;4hY2PR8Db=dGNx#x5Hm<_Py@^^?XFKp%zle`8m(caMT>K~;j6Uv5-??N4s15Bw z-N>n>R=IG|fmb_U-3Yovixu~x3dG^7Qo}jzNgr`cYgVPY|B0$8I`aymPEBrsX`>s* zCI*vy+cq3@PGxSoy}xc?qCfztm9KFiuVpwObkWEmM>A71iy5jfBOR-|%*4$WJCKn* z(u$uk1y7pEkTlvIJ3mpyq>LG;B613nAy`1H0;MVpVjWpZ%>0rT4Nq!@J%B?tDRr<1 zZdA@hU}UWd3`RhPK|YyH>j{r2rj$L7B&dl=8B5H-<{NSC?B<@Y?nEp( zFSDY8vkxHWR+~9n0}#C}AN<5i5f9FZ zXL67W7#pn`((9vReS9v5g`9ms2s52e5Aw9yf%y%%(Dec+d>)kThAw&{G_n6iq6&q# zV-2=XOSYXhhf7(yx$unLU(lNOj+?>z58j{s`-N4JddLw)7ztWagszrFdBA za25j#qoAmDn$bL}EzOmcA*V|?83y{ftrEfVhr@)# z;2K|xe7q7?xjz`^aG44m0=4^wS~*vu3J#vCwNg0mkQFgt$#(s0i3^b_7u&mv`2`#) zM;MvrH0W6H*q=I}rj~0_E(odk$)$#F9_zb{O<6~WR8>*a5?>$~vm??meSS{-+G z?CIiaLTfaH=u_X_16c`sg1GoCc_#a^si$?zpxm~yH{*ki;g1d@&J(P`zPqq440!kn z7T?9RviMngICJo zvyPe{bCUf%m`FR02}5(Z#{5dojU=Mu6L%OS_9QDjb^c6_;q=nXHuFe;HLHq-2evq( zNtkIGXlJ4ixoLgXQP`p*-Jq*gmT&XqVSGOq?5@OM^G)f}QFDV6~>OUsNkQ)G+Jx)bLQ8`v158rft?W~O_D=$_;KkLnH9 zQpGCEgZS0h`{Wu|`!ZCr#&-dH>7@}A72fJ1W))sY=(x!ha6?ag%dk3&!!*KwI0m^< zi>f1cH1bg7Mdjq8p;>O^rbSWb(;etn4KCL&^=BNS_gdF*giCRFu&~T98Mg(gqGY{q z%nIjSU6zDPp**mvb(JiSvBLJN=cx?{VA%= zL3(0ze*UX$1RCEy=Uv~~k*=T1CiG?@gzU?^JykANoh9p)jx%j+xv^(-p=$gfcTo&d zCMmw&6-iPMJe}EyvnWw&1V6_g!eWwfd_+`fSlZRXKP3lu7?UczAvsF@YRPB1b6eb| z*DW9YSNo*Fw}3S|Ng> zuB%mUe8p6LS0}TqB()H|2QOoBD4^DwmN z5Q~~ku!$B!$zSVg)}G&(#X&B^z&UBR@QCL_eDOw?T_ zF1E)>oM9tqe8lwa+&8SVEM%A%Vak!9b^8}G%>k=k&ZbbQ=A^TMc|=8Pv~C<1g#L{TnedE-vxy}Z zKxa&Exy2v~?&ezQQ-*cA=&HCwpZ1SwrK6mBv_89Se9n zg9J?GF;7+LNo3bR0b(iQ3$-gM+wziJA+AtPuexFvibexn=gvUtrx#HJ_r{p0)do@A7$n%iFU^LuBN1&;r<)4okyQT^`l%1h^>nzY2`K924B&V zaR#hILteR?F{^w>=n|A9tGR^FpV>Zf>q=QSwq0_K0iLk(gQ=q{Vs_|y)0e*3a3LU! zuy(|*mOEmuUh(tC5GX8Lt8&cSRD`(vHZdk7)(1BNrLr2G31em`W6nHr#k@o2ltWfE zBOyl~)L@7=Pne8%i$z=n#U7Y5Wg|&6VnC6wHdRukQPQkYa4_x@RQdCZebQpGLi|^$ zy0EJTX+@h7=fH)lPcoham7JkfsH-kliQsLreMQj3i>h%BDF^j-4zym$I-5dzECD^0 zH(|6=>ooLvODme22d~+(0h|Y3@S!dBMU4Vqpiumfx7xQ4@4i!7_0Rr-TCtRnNv(SOi?{Yg-6HnTE_1;&V(bB z5Kq0H!o-}r>%gdZA`{{jc=ZrC>T!#dQYq zb;ct#QljMT$;C=Ts&!7eKQ;uO)Y8z09B6;N_jl!Gl~W(tDtaBI#wMA)AvuTH$XUFp zMlV3JEiMI=9jtwROXZ$RwWY_Z?6QVfllJ!`;6z4{H}98~XRQ{qs7TWznXYT7!njck zSF6mW+&v+;lhBBh(2ngQLyK9BWfvYCF7)C}K(sHHH_kc(754X4nN#Y^eni#$FsD%H zgv>d82+D_55MxTzK$TMirM7`tZ&MC=A2()@Lv$Zg=(@+^q$7W7$-c^d{PA||J#s87 zUrTo>5#ybARJhA=@^A8xY@Sot|4Q@M<%nYdXt7OWNR|54wnEPksu%bbmdGGo<;n-j zI={@HhUt%!72TQpXlx18tZ60aGc0KNP1)gKLG(p6M@rJ)R4IG#aVn-(ebU7v!56F` z;)50IMC(e?kKJd&>YJ^&-iw+-Y?s(uz*?**+=q|{@YA&<~-6X zsNcOa>{wxKJNN+|OdM+e=4j*FX6%B)?doz^(D-&AS6G&8*g4GoG{W|oW=@+sfZ_q# zVa4&5UMY)C2rs5hUWm>=XAAiTZC!Uzga6ubuE2e{Y3X73xoVlYYYyt+g*#+fWtlr4 z-PE_z3{%>I%&0j=n3S@D!TbvB zLZuYiRSZn%Jm}}fRkX7Tq5N($0kWj)Yy=}JGKD)Ls#CUQVN<3bkj@RHK-EAR#SOyY zba&hb?&A8su2Jbcv8bQr3zS!x%*cNye!_R12@N;`jzt|8igdy|AaY_$N*LWx>1ZSi z)|+E$P8ZwK=F(>(SK|zECz<4mf6=@lqY>u8=z^xqqFHLC%pOo@$v1h?<&>C6i7Kh4 zO_kj+Kh$QAJmJ*-6c;~bN0CfqryHjtM#rX#VJsHJ!KU*nLs6`eo&89gL%RW$klf^u zV?o-vW-FEKT+3NyoJ!ec(`bgVXc<8cRqM+oC8zrArI?ZC-L}z;0mofXxQ3P-(dZ+L zo6Yy>^CRfO8o}j>`*K+3pG6RNR&2bkUlNZ4X1kFq|BUs3tBl25TtNX;>}iGUIz<`G zTzMMgFQyM1IqPH>!;mHwI=QqK;X%(o`RYD4C8dY%zSWvLSN2fB-yWZq=9E`X~mg{`w4rCyU9!l0?YJrZexr9UH#P zFpHTZ;hAKw!3WGwryKo?y_y$?E0$LI%}W3&v(zg_s@JXBj$K9;>npYb4H95 zpWCO%+93N;{n{B~uPm zLxA$GIep8Nt0G2f=XFN&NT($hi^K6J$J7s~&rS%U#Oy}*RGS`m8MhnjiKKFofGl%_NzaB{c&rr?c|vr>jneKN?1+If69d z;riA6e2U5-hFqc>vZj~=j5~9AE3~Jga5)gC89k2`?&f>L%xR*Gw3G$#>W0qn*Efh$ zNe2#4$F8qu9;_=pwf@&4_|ACdigH#RxAOzIwYL+eGuM5ZiO{?vE z$eH;`1}1=3b6uG$A#vRK?<7@ncAY8vgjKea%2|m*Kk~;gm}a6=TKkup*Q0lw&Ty`r zql$979fN$q(Xj9VyHFA!RVfwIQrro*!raJ-?-;Uz(u^)k?30sZnEWQ*{SSrwS3;2S z?J1cRF)}%XhecBkrG*J*yk=`Xrgrue%E6>e#(3I?;$2|aO4Qy$wcE5M`XtY}3JZ&`FaC}Yc;LvOrUyuz`(=s@#Og%g3zzebmqq=D&Fg>O(knHPgO z7T|ED#VO6_;@ELz(9h-HnA;og{)UoL-2f{3ot+6e;k5^%R6var%}VKaQ< z=Bx^_s7Mj0p~$j>U}~m(6S&+1P(>oPm)Q$xuU~=yKaUQoAr7eFt4>-sDkNvD@MKH&bb3LJX@ZP!eK-c@V7u&#>jTx5*g--60T#3gMQa*e&ijhv|2 zxi|WG?aw;BT2`(-KY-%v-p8|ssF~|3bC(HImAYAI#KCW-(E%aZ=hA8|#?70&1&I`; z1hQVLgU;7XksngEG55?ytspMxs_TKFf(P?gq%5wz9n8Hwl?+vrCEa+ArGAI8Lt2X{ zmfKJ#x|HAVfXn9Ur4s8+q%_PT4W?WoWut^pZA<}acq-T5WTq9SoZX$XtqU7~3R0Fd_uxGzE*8<2A6=jzZ0LAC6t2}79Ce`(So3}U zr;$|7ODKptuom|ze%Bn&p@ruN{7Ol>BuID5YP}F}MdR^{g+FcRL$bl%P0Y(CoGUqh zShsZ*$@}+M6ES@~AF#kFJH{FpFgjc8?V%*(G8sN?Z6e!m>x_DEb%c|&N=a)@bf|v2 zJf;~>Zbe-k+R;unoM#uDX+nz~1s0>Aw~j1i2tWGP7^a;m5*@c%(zcyY0)=vGt8rfg zb0CWh_H2)@AipAoH&pq{sWdrzX|v}4=VYLfTjWaV&OPq32!@1sz+7q`$YsIk50 z^Hro-B3B?XU1+6|1I0shVFJYUmC0{Yrgl`PJ5{2JNvi5Fp^Biix{N{dWYv}E@MIhl z6a=Cs2Z8je{xNnHcbG}KLvwFiYDou5LL*;IwXc&gJcVkX<$z2HiyMI>CG`TqHX&Bb zLbM*JtFV(j87Gf>@uJwk1^{RnNq4)3w5U1zY(+JSL8)U>QGU$|fV^VNe!;1!At9>( z6Qyacp$i>6B;@>k+iq&^F6tz<+KZ@-&w!_1(<4y;bXg?S}+Dy^~~9|&eCI*e##n5 z#3{&I&8K;~lAlaHW_J%t8@3bhRPhHtrSur|{YkcoG3ZG#ufDkqV9Qd|x<*4thgScK zEHS9OW~Ie)iI?>f1}nUsRj*U{w0yU=VoZglt4+&yQ^)!aZldPsdqZdUW+l^@4NXQ& zW_ZbP0{Qap&cK%sytCegh$v0*A_&|YD|N_G`;KkP`P5i@{G_+RrQi!62#TSZNXGJZKxn$3( zju_E-e~!&Pjt3~cb{e_`FP!)BHAkVMNS!e_d58{|I(DSC;FzZx_9>aTlX?bTPSJ_2 zD@@Zz2w_r_C%G2ywW>xzn{|q)rw*S__wc-wBAxu`gBFVJg^x7c168_h5d6rWA9^By zwgqE{x|akyXx*UCPPQ+z8E}~9cqrT!yC#(?U*H)(caG$gsuKAY1I`M&)ac5maW6f7 zGQZNV6(2Z~j&~&z!!vN{<{Zp-hgKwx=Z9yTL=c^XKIoX9hZ`(SDDh0F;f;NLf1e_r zQ->>wcIEk2h2^oF@2A5kFD&7bN2@v5n)?$7N3BAkb#DiOi!OzkZJmjpOUratGtt|% z(@jSui#NZk+3AeIWy_{NBY|B8F_liXOv1s?w8rPR6$j)ni_8*h7Uvs!gacemg;3KZ zi33rVgzRx404j&(vY-+@UELb?rhq{NU7AH&6e2AN;x`-~fsuBaa$myTz^TJE_%SNV zzC{Y;%7j!=2mDerH=J)`KWny89g*A&-&a#bdn&#S1d=pW3a{67X#mr>-i5*6A!f#N zLiJRiIOBEVSK(u#eBbuF@1TWjU68z9|wT>6xKHDSQQ0u4v3Cxhs^> zAbUN)O6%pvY}O$PCDhn>Y{VqP;BikSJn|f`{Lng(z`H#zA(2qpkkc( zb*nYhUm3*7xs|1}C{q4&ECc~?ARuK<`Zm%A{rn6I9Nm3UQU0lOFWHaK3wV$@j-cg3 z{xxM>0=M{VIa<^7v^FJ{)0fuO7q}9VLUuF>lqkk@4swYJK(nZP;T965C7sBRVIKHA zSY%l-`K3(SZW|ccP7V|{79(ObK3=~iq(ReCI(QO@(ev1R8HCm2 zNi=^1(ftQi0mBkw0Rv1eRUW0|xX9oPLRYc^zkZqT7PHCsenwX7yRQeDKs~rI`h$54 zhWITDOV-L5(Z&KP!j>9!TcgHhE>4lCX3Hnnc_Bc`zZahbcK3=1 zygI5}3k{KQ`SFXX5}YwpBy(3>Bl7@l%te!GiG_Nu^;hPb0R&=@Rs7LL@QQB!^;k>l zz=@RCzj;>xjWN*0E_2^dXunB=#bPw~&#S|A#K;+hmdJ7V9~#{jhk<6i$b4;0fs1a1 zX^(a_y2_PGUqG6t_hjEJh8K*bzVUv8 zC5!V2%l!_rVK^OI>(TSboRFU(1|+N(>1(^(qV4rQ`8wYTvA-cL!g(haps4?R1CQOv zl=(U)Ry&grAnQuph@0ApOEconJ5xTTEx#BpTLxy7x^gz2!3*#RZlhl&f0hNL5a-j} ziurv#%0^-$TE!C0+Ed+BM7WD!2kF|Nj+bGPCqnt!uj=Qz`N{Vgc~RTY!(DhA`S22` zM-Rp*nEMH9;=YM6A-nVtNbXr-nVwCh)xw0s-c8sZ5KYYKy$x5}$vcoh-mdBceDv~g zM-*!xfV*<;3~}fZ;4BT@C@ka{d(g~DQ^AdDuF?5Roy;K;!SvOdd8&_i9|}z3nmLik zIE``PrdkvK0LTFrCF2(k!-H1JqKzo8eWV8l%U87zzoO}_8>GZ<+<9f)*lx!dXb&}m z-xhV2Rxa1QrPtbJA~Yq4^7;MwnxfY25VUs!vy_LZ5#ssn7|@`c!r7^x&|wbHvExRfI?GM``Y&pRmN(r+>mnNHb0tho0>DUqT>s1_R=b zQ-?KZH7*m1flUlj>=K4B8S3HmRpQly!$Is{8R1Bv^;z#gZjS`-d@a*VQu7?~(skIU zB@-C+MMkJC#DEbzkrE!3*q->hY3qo`Q62?UQ?dIo_@f|h-%iRFYNfnPZ<7@*N;M7I zt`FO6y1B2{x&W^jAN~4NFs*~8XPMc)^msu0x<~BU9p4p19i;6DKSf?Kt^^Wf9_%$q zJcBzXhiQ$;N*>k4YdG2lI|3}ula<45V^ZjW#1FZ~vZH#KZ9I(qUf-Y|r{TM9 z+^x|{`JR-2Wj@sJO5%kO{cJB!vaJ~lA3(~0C2XJ=*ap&WB7_-k)l^z*`f$G5k86W{ zd8A1AE_&-YaS1$wPt-iiy~(e)fG58eN>B$1S2=A+YP^_RFC2F(;5r~G>>GeBKjVNA zN{EfBrU4N#L@d0uo-4&u&|E1kYcw)v3KB>?4R2eA`qaQ_Q?j?P%zWwn?bj;2F>He-!dv*&8}@~+kJqde`^LK^GNBtfnIi=68Ay~_BtRM*XBBHGd`aA1ak5W+gnED zhZ+;=ie|AR?)L2#z+`5#ThICqDf-l)q53@;%(G1!daDANMu)e<7pX{58jzw^snsHw zjX0mbx+5Q&8e5x~k9g3JpGT8uuwXI57HH~9Z(H9r#Irau&wEf zT_rSPM1|`!#q7~r_Iz^<1#Q7%@2sMNTEnc&HS4xdZ!gL%!-gugs7*5co&j&XaTRcu z`ke`N+HGGM;hn8Q#KmOze7V%fMfxoS>ze-)?lmcIFJcprAepRluU2LBCT5 z1t(Ikqr5}3h~O6E?dMqijnxEjP;yV=zy+_MeYLr>rB&k=`9Us)4HC4BrOU8usg;$4 z%~F0LYo_1uhn7y68;tQOl~NU00X^PzEbi?nL2kNoI(NoTS3sOwmajF<3q-;{g$t1? zz-`ho@u3z|LTXf)F${k$cLyPahS*wg}ozgiet_+vvQQ4gy~~sX(a;s;~9KbNs!mdLj z$Kr0}7xVWG!%`v?(ymyZdImYZ(@#n^ZX?Q3dg`T_65-62>3_|~4#>m$dDB6qG#JXA z+j&a5QRx@G7!d;lC5TWyEnYG<1Co%=kJ~l~i0bGO_K8_Pm5MDV{1%x{%rQ<>&mXWcMogw`J?$~Ky&pZ6u;8(9nfm&;)*Ithge-*MXn%B_dkf4plv>m9tEH zbBnJiRF1=dm$%_iIQ-l%0tag71|&Y?eL)g&QC6tLujzv1^gPHeXw#cW44W~?HCYD6B^!nLpGkm`MfGbvKy+OXfDCCX8+dd66eFQ1N z<8#*+8Y8Jy_-G6SdfTI6fda!{ptMCF?VQs&UIKZVcYFe3g!;Wp5o@10|MXkyP|~y+ zl16k~S-VEzqP_bD)_Ao@f)uZy1qzero)$=L$1c$hqlk~goD+=4hTXflJEKQ(M8eLQ zlw8D6+OX~kPS%7Kp4!u?d9sdBn+G*}Vc{1tiVrfQQyA~kS~ z#tw5ZzIl>+N*38xmsjJ2qL{^i?Bjb7#0`2OXN7`91~9$lX~0Af-|mX?5x8@*uLG+e z3qj22026nBppXdjjX&~6**!^rXjoJ2+D z5>{TTLEZL!Al9}_tbjyl%vv=aBT5_1Rn{Ds(xr!Yk2v;90s^(U+m%<^Rkjhy9b#O^ zAQ+nkf{qE1zLq)=M4&E2KbW<7rS(vQowGzQk_;2;1;onTfBONcHrGC(dH%$%!isu@ zrJ|k;!2}3`Tam{Yh)TRkM)b>cn)BFc9-%_yID4V4^0j^BF~9FJ0wN1$cF_n~c}58I zd+6d3K%^;IT`8krAQz_!im2ImPI@B&swa{bxn@;okTG_({yfIixDIpA9xoh{+0?OM z=Ycm~p7S=3!KVKWVhWXtv~`E*aAU`VbMm>!9Xs#W`-e&vgM*LK5@;$-!;4lk`JdLO zD_p;Cw$JyU&90u`cZaJQ0;(;k*u9yKHVIJ6~NCi$zyg6$2vB!qd}K`>%a1Y4~1Lg zTqU`3h;xfO@(&q;^#Fy4~}N8V`A{A%df7%M(F z;P)WK2jvAoj*;w@py;x!{ zdu-oyn>;=4U96YXyZm2U>26NmFqOibcEz;6{0(lEr1Gy@4;~rrqk* zGq73|LUo<^`VsWuZX;fR2}F@Wy4dHQr%$VmFvRe0m*R4jo~GEJGA<77u|rkY!1mk_ z#UU_ngV(GY;Aa4dW4pNr>%JA%hKALHaooFlg1aY&3}|{Q7S*X@AiL9dr%h zBR2snm+Qb{cpcQC7mvF%yXaij6<8be{`}XidTbghQCT}T5ld7Ag}6xUiKJd9X_i>F zD#Bi$^YXd~YIIBdD;K$#8cP0?6L2;e6_I4_@RUU@(9_Krdcu=34B?>x*bP}$Fd49oz^4w}AuDI_CkbZ+2pPT|dh>7(1{9pu3`2E!LCblHiv zS;f!AJ6P3SO5H0V3CG%Y7kyD-nSVbtV9SxI)N|9$u?u#9ponh z(@M}>k$dmnH zOL-WRp5@uQ9pu^r11YC?6)6QoB`KoDE<~{A#PyaZEq-%PO4B#tIC9391KO)e;00)z zzi|fpU_TU~Lbg6W|Fd9d1(213_g5uZe=q<5oUclZ*UpjrQX8fOWtG1W$pS-E(^8SNN5_gzftTju$QcQTmd46xsORpI_j zuJVab#w0sHzG~T#I7l|` zZeEYKDH|PKV)y|2&v7%}+?t%B0|0ne0R@2nXWSf}+^mcp|0?sUt(~yfT<6u(V_>24 zz8@*9n7BViMjB>xPC0Df*?iR?TCkx|8Fwg|UU{MGBN0A-4NTxCngBAgStZ$-7)bU8 zeZ=fHr2F~%ZT$CT&)S^Z4{pzAi}#e7??2WC4xbyjJbzy=(D8xmPIb3)=`MzpJp6WD z_I^H%xp=AQ`MmJ?{d$mbcM$XO=!yROx83hUTFAi=WhdVo+ZkyeeQ$8HsHt;(k zIngC452Wu$CwBwmuU}P-&z3Uw#?M~sJ=QONr10TY&Wv=)1#^T&I=tVTx2(7i|5_V= zv#ycUCEx>Vd^=r+3TwO5wb>aSi(F#ZsW+A z9g{=r1Bp-8R^VECeYB_Wu1oj!){*jyZOe-nNA7I+<8i=?efncd@D~p*+velktF;e@ z*eCagtK{#rcdFk#+pevdSKXcsytXbKIYMPS0CXL8mA2?1cbB~HMjgDmXh$2IZ+SUd z5xhRp-6QV@hn}NaI+xilPw;WmcR4#DpW0V^>zbU$TC2Mk9_-sAaIlMak@Vx88_5Be&o=#N-jQmZQ*+|TjZ{mK5+-TlOtO>x^Zcj)?#dT z-}gVeSGq;BvAx~Z#8_VlU;B7pA32E0f4{6e5QoskUttoxNTM!VDw-w zL8b3O)Ti!A!s=JR@07wbDB$<2;2V_l4oZ84a(IT0+dDdUpHE%5{i2i9QGSbvpPT=k zzfauML(wLDK_`hP|JBrI>7!6@`}OXy^8Kd??q|n+Mp!NUduPXa%hw`<%GA^NuPV2D zuZ_PwHF|irL>w(8B!5ESZ{g7yv~a3+EX1VD-CVb+MjDre-RSD%EFEpM>BQ!r`b7Iq zZScuEzr0{L- zv8Tf^+_q@KV^Z(eDs}@07Q#=oPj4=rQ@^sJk=-6Ht>%!sM_K9gDhO=HyR5*d1u|Yp z0Sl(wK!T<+c|inESrqhqUXCx+3{p~%GMN-4f4f5&!1o|}TPSgXR5sAU++P_$_0xyH zhB(3l30Ufh2qDPbpuaA3n^AWdkOHTp9Xb#?w`j3J37w!IJ0!M%0_>xjzUxF^uf9Qy zYVt#A{OL^Y1U34nlOI9$77Zcr@AI!jD-?DGuj5;MsWbO!2>!vArTP;p_!a8^Cp3@L z7Et{Q_di1ars9uvzpY01Px&7N*^Tj?poIi~RG59mq5Y-e-zEMVR2lh|JE7~Jv;Iw( zZA;RZ9#BFzXmJ6l&94mnPl-VOc3({de^iU}xGh2V{Wg8o&#a!-xONW83=36Y zchB@ymY#I*uS+gUF3dfaAu0>yQBryH0)>Az{T$!BY2m61+F%_P2o$-%d+IRkoH&e` zU7jm99{tqV@apKIX`!lgr3>?Uj$flifyK7zv&`V-23Fs8jzRjX2Jrc`e6gUWP8IZ9#s?3lmM%^H2PJDC%j?i zeBfjE!66cuaFaLrH&=4z3SROl>BbBypGlN~O?jwt4^NWvjb|55gu7Nq_jL`hl++H> zN3u>&bfdctIiE+1(QA}jeE{K=F%0V4rEBv<#dEzkH~D9dH=U%{4o~o>Cr>utI@cxj zsA<|&Ceeh;+t|8`SXlk+jb!mS)+DA1c680TOlj}~SzFhywHF7^Fx=+jFw7!U8M}|; zOYJ;RE)vaeHWh2TQ7AYS^Xh>m-s-kc(ZkRK8NAwb2*9$~RGr zU{*TWNay-f)3T~HQBJi^Yd=1S;#-Lrk|x1YQN7n}bbTNW_zFLtd6_?62(QRv*5rog zSkq4puf&hw+3dUb)<_*vuRc3^!jMuqW}}AAK9#K$qIvgi6ed4zy&@c(I5zXCnZcBDv-$&bXn{M>JYkn>?z_!KyJV{+-DCf@!-nkjUQD#qcqO7#jV{B`Wwr`W z=MZ&GWQy`Z<32+=|FZ-Bqceh&oQ6TmY{)qevl#l<>yDt3mzFr(;ePI2Xy33XL^t!% z>nzS(%HhX#Q5tV|U}|(N4$XImgHBO{Md=9IkrUTJ`<%Mex9h7nBTGmf*KGii_yxq4w?OWdMT70-M zCXrM4F)8CWL7#7}zEk+ygg#90Eg{=yZ`Y5vtvsZ3bZs@*dnWkC&bk@C6c!q+>c8!z zxo8+{C_YM8c@}h{n?HjuR5$2&Z(2?wKq8l{ZYsi0&HZYqy~;7$?kzhK^mq?t&Ov4S zMtYe zhd?ZMS3Qj~R|50*AiH_>`{+9;O?VK}7|@qw6sQ$_r-Cz_O;u}EKit?zrL9WWVYKgi zGU7>)+Dx2707x_-a8QikdI6^9)$e>lZ(`@t-e@00`{nTlKx|$VpmIn9VFIWA^MTLh z?Uw6xEyVaVhxB)|@w8%GI8?9Ej>cv9iPqk2hj)mdJkI)hTm$D+k1~_^rre87ktvdb z8}Vvw(#i`F(e9NOP=qK=BVwf#uCe6`90BC{ZoY)_Z73@ggdWIBhz?%P@6Zeu^c=R= z?bV^SON9p747iYbGsij%52l*-So%p*_J@&49>DM4%JaOGJffmqq5RU@Iw3AU;!acg z>aug)yRe(fY+YB=Saev%`1FG|Ycu_H)yzh)$0C+CbIQG6u+9+6m&#Z$o?F^3y@D~A zC{YdfB}IO>Q8k7g5Wic9M8yzLh4KY4n#%TimhIB4n9p_3ULQT|I68$cc3rU2M~P0U zO;$++JcH$IG_ULNmIuOAaeHti3{L6f2&a2I-k2@U4$y5qzR|2*UByqWstiQo27lgc zTvU)=E=S_A>)Uo4lk1|okhAQwR?X4^^}aqi~&PeM}-n z+BJ(s_;E`XH{vP`782IOvM6ddvIZ!h`gDvU&K*zWh8A)QYm8*uRKFlZ4qRx_oRg}7 z9AZ@gIo7wOiO73^l%~Gh#4LSU8%yOEqLL@6W)i5|&w^92_T)*;y4Vz5XpyY7dDUvK zY2K00a8xz!qm0JSWosh&cw`o0nkJcK5~kTNf-|`^h{()l-${B*#+`Fq1@N$}lMRuH zS2(Zl)fz3xIN(agV?alcNxzj{vU$>-L0rzYafspt_Y>PqsDW??pX0dcHvizj#mvZ# zA9p|;()*6)8*h>X>Q$-PGrt?K!7K00T$c0%g9Hg%P^eGYu^!6@ie7LYi~3!~t2Sdb z%#Md2*peK`gR7RsaNhLDWnF-M6W8tIyO-HnAH5W7g%k3^DDzP)%O)Y}dsR78ORQVZ z0a>g8xY+_XqiFT&FVP00uaV1yH&7PA-#aavZ`0t|k+jB~p2*tl&~$u1`G78WuP1cp z3~BSh2VxF3Q7sr9*)b#2&QTv(KehR}^u0~DwZ30jAiVr_IU@ie3Co$=)K+EUUhunW zK~$w^vPLg9%V(41S{b@Ztaa1mtbjLi2MDBj@rgv6@hEqM6o|WCClzTsQdX+`qQt#~ zXDW~q>52kMJgv`P&S}gcJQ6KL@;$LNKY!u-NpJ593xlxA{6DQo{rtKUoZ@uKX{xX#IOc}qPO@v&8f-;H=tXKZ(=>RRrTNltX{=E?qz)9Ic&eSRjQ zI9V2Gc5-8XZm7&WY~ed;WRlO~w`3AKPooG%GFZH^PP+|j*3jWgEog0|E@(5vk397V zv_;3Cu$#$YbRJwzAaXyn3N*}bd!A*NUeMBJ*wH-T@qM27p6pr?Ccf7)g!I064~Omx z#If@W#P$oscE}f*4`OHZ?K~Bf{G~!j#I_VlQP{RJ>lNmHXtTD4!#$(jaa^YkId+5m zo>iANC4u8gpbxUsMNQC_bMgjg)0cG^dW7g+G@10?1diR_Yb%4kFD$ErQ|&&u>`RX- zZ(2s!=JK$UW%MSQA5#wPGURa=$rA8AD|Xvt|1J*&yWbL=)UnJkhfXi{^r0)Ddf?_r zec)y!Ai2)=uXjx3O!l^asX}JBXJtU_TG>lu(yd3H=5oCRbazaSSQtC_Pn>I2v4!z} zX{vh{iPh~@$z}qN+N4J<4BgaXxIat}-Sk}3j$&_WdFq@8zDJWjsva0^q~-o!)%YE8 z>DUVy+fU*!x(lhqGQ02LykndMc)S-tNLTReVD;2QUzSQ*nD%u4;-|!Zmk)?r^wae# z1i8t{FUDWW*y7EJw$(2-nvU0MN0HS6%`j;h8!xtF;$N7%xwzR_(1vmpPld1$BDxkUpBP<63}8cB)MyR(%G8`XKojG2O8OGjdiIB!jPD`xTA` zBP81OS9s}vh0p&h{P<6J0G?o1xfkob%`EV(yT!+x*t}+~MYr?MSOq%JPWfKs(;^OG z_!JAYhxm(Fs}g^Ax2~xAQuNK`Q281QAEd4fY`@NI4YCKs2eis`+ZP6-c_~0TlWqG=q6jii`er$8ZQuA&Q-O+Kxgs#nG z%EZCJrFja4BI3_<%o)Qhq4cGphO=tri)l*Y0^iq5SW*bJNCoT^&{<{c%bri<7*WEK zG5fL-qb1ga(wb_~nXR?f71`Dm%L^xV@)&x*cJnh+v=?>0511y^?{l^+ui@@JDwLZF zJuD-oxkTC5t?>+!ANpm1`VLNx5pL~)EgCchql+2JvjrvtC~z+?0!zCXs81Bnt1w^| zWmQQRB;oYg7ogof*c?90{Ey|u2fVjYvSryryh~aqF`lx>IhOelBbIzQG(i1pQ%Z=n zMvfMIRP>ak5ER_iMv00Yo|NvXI2-FkxwJb~st;!M$Q-DEC+DD0A@A?GmWWZ|z9^p7 zV8B`Q`Cn8}jNaQTcpV-lzsrk7`2WR&82b+oR}{G>NMO#8kVtR*uQKeO-Y-kJUzQXk zOTZOo2KJ8+(EyzxZJWx)62Kee5#{S3N)5T?*}q;5yx|WS6{=sGYl4uhNvz5i6YRZq zv$aKK^06~|sLT-&yoi=4%NgcaRzryMmV`~QD1hwa%BFs~f+sH|(i=T=VvqZ8wvca% z|H*bVukioImQSz;x;c@vAv&2;kkwUg{^h7Zsl23aisd^9NzUBhQ91?C?WDyd8sLs8 z%U2RRK{F>UD)CVlk>#DVR%G6snwe2~V4`ij<@ja#HR~WX0)`(I5#^i6bAxaAclbs4 zVXu5J65>qyN~{rMt;o%}()=S2==?p&o1gp|zO&E(#oIbbJ}^JFfq4MMPo3;I%Af#h z`w{86KQo{LB8K}qIW_&u9HrAdD=MIe&OJR_dz%QqcE4uTDIrR>GRS%H|C-Hw0%X_j zy(k-EQ)ot;%m0VL8=wIC?fBN{;qmg^=?!&N>&blmu-CEg806?T*R`bl+k@^UI=@H6 z@+bzZr`XpRpt?$G93W}6sED8PYF1G144m|*{7<>+{R5m0_ea{vspWn7C&b&r}%3Q*|X=wV1pB9(8k}PdQjp>i&omCKa0Cb}A zdnN!Tbu2^TTM`>rT}|{QmjlTMBqZv&^RhDwBTzz%2Fmu`MN~SLM zyylGg*&Cxvj;1sgT%p&V>40^iVWtEj40T+{+W5T!vqUL3 ziZzJ8kiQrq6sH#j2oczT;nqL&=Y7ymc_SC(G#NkN{yym!;*Q`6gOM3nJ0}B%*X-wI zw1}Cr2>}{)>6c?Jya+#hUoq<9MA)M~tGfMpC!EG_AVi=}+a&yO`yi6O7zhz@=)%;+ zbkxB_4H+X=P#`F>$s;)&jQ^%_q`X-*@x<9I{*P}L_%Gjb*Mw+FI=Wn%QQRR}u;hRF zHdA2ahco&*z*y-kw(h?n zvclR@`JF5@YfzynbyVPYr)^==}&k+W2!u|lu3OIJ@39$xANtEq8f$fY{T2IN?Rbyfo=%^koG6S}f>H=h~lQGB` zWPMAHH$u})TMA5lCQH5vv&AxHymuuXS_!oXb3{5iXB#1yEX%n};xPS)ka=)9807e? zSyM7TwG!3}QzjF6{CSWgiV4%b6Y0oGxam*-BeoI3(Y)N#SY|_WTW8UxRyI4iz3gtL zmy`rjLX#hCcbKCAW)6FaJ(*C<_ub5>(&VrHN$&ud-OSe$oixWqJW#zdd{F+(gg@=w zMI>RPtGzDggB*2{a+!1N33lP~0vl@kH)!|f*`Lo%Tn?c1z*4Tw147aAM%1}oAY znMF8&2>cKYCc(W1?U|##Fb|uUyIGD!z4NFzIe_F!Z0t|kT3a|B$$%b##y$8o8CNr4 zr$(%|>(Q4;sbexyGsxYX(F?;5Y~b-wGXHJyR15N;x}XQyvSEUk-0NYDMBNue8sx3r zrq;MasKZ6whcuuc_J>;PFKVPOYH%s7ttYaHw#`M@WWB$rU5v)RsJU4Grk*85kAVAM zsBd(3K{hAoXl-S$6zJ6a-773=1J@eK;ia~5Js^?^YG_8UUZ;NUg5d2o4Ioi7;3Pw+ zw{F!IjxS>}QZvY3xz-Neq6>JN554hNfJxEYs=y}Oy>agALw1${_aSMiDK2BAM~I;R zFVSYTB-ft9R5mse|3yx(rr0%)+TFikjT+=cp9KT>I&ZT-ewniL!&Xlpp7$cMTFCT> zPf1%{qA~kZE9-74aXZl|2XtJ3q3ebwxxN)1n^|0hYuLj5zQ(+{L51n}jThM*2J*h-7y29bA=o-|3I~WPcGV|iZIiFH%sSlzlB z&q^;ekGGd&mLX0j)8kTN+2WK*$S!w3y)B+=oP1k6k)6Z(jxfWFAX9OPYgYCIN0Pn> zoK80VF6DiBR$hPsaSzkck}tq4v-mzFR6=H+ZnTapGZ$J6xjd=-t3`IMu%tY@y>o}T z>Ww!$eXj8kRhSu8c&^-|OyjgmIjs1>w-}OH2!#6is&s(Js$@9V;}_OH9-VB{wf-Hf zhfT)iPrfo_=Qyt{E4gGWwf@dmdrM={re^xzCLPuOn2c`tV{&ut|I1|XT;t_ir={#% z`X7_}>5qTr`)fN4ukv|ydyTlNCP>5fqBFXz_EK~@8{U|{<6g6j{B-4oMf~Nxvw~0@ zmFf&VtW-nt&Ey%TDzCqx62^zmlmep15HX-8!-rBg4JKCvG5F<|zkY=~uE`CN;o^1$ zzY+D5&#gYWLq#fO^CGj7hkE{N)}bkGkOv%!0>SG$0!RQ7ih&^N3mq>EdFv16k36p- z_lWVWmpix?Xl@*B&Uib_Lp{X|@aq$Q4b<{xhDX=i5rd>d@x%ySJpRozq4Ot#+ z`nL5o1YKgkw|R{!)1-||2IeC|CqNufA2b37Kqmkqa6h!2K)kDf&b0iszampM-X3V` zFoKR=&&4gbZ5`5MJ?#z9X;t|Ss108+w~p_VAyKEiO+WesniQOjU8+93#RGUOkcAR- zEbs{sB*K5A@n8FEd=LEx&3%PYEt;c(R9$D0RENJyg{8CC8YX@i`g({U5PhtM(6l1% zsCK}+AI4+)71S5y&jDxT>wvNR3R+9B@&G;SAMB5w*i8QPUtT>FdIvL}&s%h;>CT)! zfDZpCyo7uBQg|g#-YLI%#i|=!oLT!LO&@+$A#Kk)h9VhMhwJ9&hAMJ^JM@_9I@UeYE*fBw+z()iGV z&6i*RI*l!tvP2*~rr;}4AJ80o?v(6R3wnf2WN0EXXksSV);?KsR^_V&!%jrC-sj?* zw|T4}Dx(xd$(Ya3Vkga2>8_R7EG1h~Mtf`lvR;M0=$Xdu4%=4XLPx>14Vs6G%fT!6 z-m6SKRvm|@6(G#)-rU?r?uP-v*PHnmD9c`A&T6aStaf|w{s+!w&MnZjJ0L1Ox)I`T zG0&LiY*tZ%?hpfX+Y>oIyO}o zK6T4#pEGTvxDCQNcS2529^wue1FiQRL==>PZ^xcAP4}V|Sl=6$ne5#EMco^rmMXKd zgyhf^pq2D+NsJibZb07~neKX}H#=>S%gB3!jYh@P`#-a!{TF6{`H zLK}Mn1^8n{FTq2f(CP3|R%W#xx7KeAoDJbZ+@0mByP{vczxVDW_NxCUl_C06j>owo zI#bn;k@t2!asI5AvV4^5Ry%NWt+Qgrp3@JnYiqANm(Xm5#&5RkgI&;zpbF%5eggLr zR%Q@ENZ6Rkxkf|DGKxfo@PV?16Czaa;w+UK2Ad-GQ>`DJJ2-#$RK zi{-e>WeXSLSohVlm|PRZvNO>g@ly}eE!fFzj?41N&55aj* zUl5A#`$v&qFmf4lx;Jgd64@}0N(6Jtf_U$7N^8*N%itH5mCu=+Ulp8*f5|ctZ=GiT zda79uIo-1W?Ku@%wR#X*E!`IT<`3n~L-*|LfRaw@y7iu0;*Z~7sKahYfe);FJ-75G zlXeTX_pW@+)U+Lv4gBx`MERVDl>y~lEXy~Bn|jYV81Gr($hy_zk6#{Bf%7X_-1kTD^&i32*k{fz z3@@rjR^2|Je9HLWsh(BvN3AOGN2UJ@*sZT7At2@-VEJ~Z{b=-G!vLu7_YdAm#hrcZ ze5mizG|zcQ{9awYP~N3O#a~2oG3D6iE=D~60W354^%oDL$I90Q{yD*ak`X!e=DpeV zPcB5U8{W%i|9@z{++n*RdG zqq}wwxCQiaUS>jhyMz9n89mM)xU8S{W!oinPZ^8{utP6p+ol$M@I2tBXQ0a0LM*Ib zj}q78*12Vbh0 z_jKMAGj&!%1ChT^Vauf7yGT+k(8N5fxQU~F8a4*3f@2J_P~JeR&upCH%$g1i-5BisRgtH(*Fk0Pcsas^+*KlzoXr2x0VIs_{d5Jw2KQO z4*iG$hhi(0767oKo`u`C24$zr2Yu9jWPi)(^ zZB00_ZQGgH=ESxpw(W^Mu`_Z0JJ0i+^PTg4_}9JCYo)t-UwiLrR9AP^g$Nx3o68_# zpSA5yu+zHNZWOs;yk&ae9JmLMV|+42r@fDf5xD15_dfytr~&{Aa-jYKaPya#yAFOEnm70d0EP4RSor_UU`P=F5QOEksl7d6kwtsi=hQ2N5_d+d?Voh61^w=PtGQFEp`eFhL_{>_dE-JN{qbdk#>_f9_EJv(IU z0!7fBp?nP<#}FuHs)KFNar1fJJ1KmT5UE0k;g=8OEC`+LCqy(bX~P4DJM}AZV3(qP z_CB<4N9bTz!UR+Ss$V>m zM;gw@p>yH<-G|$)=C*HpgjFHY+0fO{MGjNQLX)!X^F=tl>42`{xIyM{Iu^P*T>(nzd zXvkumiLF%r6Y0(7Gtyy-1{O07&0Y1FoFGC{GTQU}Aw-jA8HSAp5Dh?jJ4tS>jIgB<(bmx$(*RdUQD1N?8i4ygKIT z`|~2nu#F4N)v?=-kjE;u24)us6DBgU(#o&o(htSBJgR5M198JIqOV`Q9Pr33oT zri>{BU}wKneMdzuF{5Y;C?MtuAoeQHipEbfVnEA6$Ms>x=wzXf4@|LSVIjyN3KH37 zFefaVuszOhCpEqprox$G(E>V}FzaoTH)%#2>K)C@@o9oM!pYS_C&plOwj962!J3$a zOii^3QB*EOveOzPs=bvCjIo-UbYZjtd!dZA?4IG#dO8`lBlC zJF0Z|T=s@&x2KPw%Bfs8PPRLBtX(~~ke$<(?#kJ5a9fdElO-k%hAm~7yL9`Mom)Y8 zR@+!jtcx5JG%RsLI5ASRP#E^qDtm>}ojRk+>LcXZ2(3QMNBCx`j!S`}tm0M=r&kI| zhZj>ouz`Q@1J4|aEMb};V}O2OqZr>BEGHhYGbqgFTrQZ=wYCikwW{;tgfO_fj9eg0 zkUtDwqZDbWpkhECh!FS_sBRDvZLOYEt>f2JMWWh{q88NdMe=-Mv5FR2RsTO8B&CdS zr8B;l)%ijpjE1l216|WY`NBJ)j*9t>Z0Vm6I(dPoFPoyA>Mqb|uVRc3D{xUB6>`EM z5joPDr9gJ2qKXEXiurHkTp~d*Dt=v#q#5gM2xu@INkK40{83hrMJ^;TyxYiF_k ziv>OJKP*6|-8ienHQUPPn_zMTj#%wYHmSK)ABQde41_^lmna(Re_<}Ql%D_gCn@Yl zsjo0fa;FHPpfX$A_ZrEl52#Hx{VzY)Q5i5OJ^3AO;kkkdn)2i~3&`Uvs06NAH#3EE zCB&>=8F9q?z~2yOWPst}2sbHFVHWH{qq>T^uo7y44T8w)0)DyXU(ED6Hfuedk+XFU zLQV=ah(q^kLKH$C zANb2Q{~|6>KyxZZW#{DrT-p3RolO|V32>uir!}8UfP`;{*H6PxGyF`W_^~GPVkR{2lvB?)tbSyOr0s zP418(g|o$5djqjT#X3ahNYw2YLTD-^Wpfm&(sR@k1S(d)vm9Rl1+yh6s>kN`gkyUH zR%P=Y17ySw>t|FcKEw`h37BKQ?2m{4F=|8KBXFLJ4G74o1_B8CzrX%hw{SKWGc`3a za{hZDfR9dUfq4D-%M)7D$)G|`3J}rgxX%;{$dE4J!_)MKNB)Y61)YrQgcE7%7Qj#d z!CZfokyy!*;B((JDnYEcHWsI0gz(3Azjt?!t{0!SjVkujKU;7g?2m`GZN4kK)l0cs zez%u;dv+9C>*o)V_j|h@4Z2+)htmfanIBJ=AJ6ZnZtZWm)8iXDH!t@W?KL%smzTK@ zFUyykoojr&bb9FR-JO;A&6yjA^Y33b|^C<8FE!Yr}3+`kb7v-OJ~P25oC( zLx+*~mysK9Wo@bOdR7dPVYeSH5J6Hme=L10_8wx`YsOt1Z{|&3>Q^ZIYy~K~SA0FZ z-qc0bX#1zh_2dGPCXl>K~gFDUuo z`Y;@KN*vA&&J1fwJ@E8v?Q~l^aB}R=yuYxx_dMrVT}nt)I1;BO;JmHX(+kV_l^3=) zxV-OI4S%!#zIQFbFg?suW9z`VjTztgeVoDFmdNhbn?m!-C;2qP`p8%9bgkQt!>(p| zNzJnAt@Yi}f<_ZU^cQ!u(!;E)CCBZK#dU^kUTWB-?`y}VH3mU$B9FA^E4|qMo8QOs zhE6>%pDljhLCN6St#2wm7w+17Js%zqJ~sjHhs!I^X62vu_L?m^SHz0Nn$b71s_se@ z5rK)N_C+5@)3%j!v-U+|FJ^8$nQ4I{3}(Nl+BGu-$co{`xAv_24ZYE?-3&QfTRc;f zWlz<2`?rQ~Pu`Ag?+;r9^?%a2?wFXK-hJx;VaPM1QvZ z^V3}e&(`?bqFZ*U+4xbVV!8sAb!tW6>+;Medzmx4|u*BORX#?qsL!&nWT3 zPAdi(2CcwbqsL>t_iL?!d`Gu{h62L-%dlQ^#U#7t`qq|0Nl6ev3PS~fT5Rn2{maYF zQ{VmkiNn)JPRgm@WeQ*z(FYE};YZnE*yNh0pLd=NB;O@|fdGC=&GC<{Z}qx`{jW~X z6-Rdz?NN0iICd4Mba;rUvIGe49}OEjZ3Sbpd~_Jc)jk}RP3&Zihm1HFcEeMnPDW#4K}SRIk_8xF_&f>Pz2poa<~!1;FkLKKzkmPEFP7->GXvWc+ftRf%cm0mn_=q+2!x& zx6<0fSL=anV8ecB@Kmm|xT|5?Jk2qkd0x_DcJbAOmg9)Q^$}m(_Lp9ieHEQi8&|G& z@TDmt;&+iTP?N<)jh>{FdRqMmQy+9j$p%GOOj3EEsw32%pENc z=o<+L$cD@xFU_mViN;~VKOY@B^{U;*dH56Q!M!HW3qgVuYKverNLIALE0L}1L6%V6 zw0&hn*Q))60^ihouzSU6^`q@24!3<&d6>hf%JI_sQO{#04B}iRY=^>7YQ?u@p*2X> zK{zXsZGk5rH9?lboBcgy2DE{~g!<_G`>}V>dkmuOVYjcw%9?=?R8#>QsD1%95S{}z z!~r(cinoH>{~H2#391F?Ed!^UZ1=O!zxMtt(4fIAv#TfpH-$T7VFOf&pM`|HbtmO20r(=#cFI#h?Mjto|w1V`mGy%_!FN ziL(L12{-qN^PdI&LvyWoBj6TGya_0V0x0$=4Z+H>Lh2de;r1rxCFFqUK%4xvMk=ub z$k%(pk4}KDW&9p=_>>M+ZF-toKV=vGPQT&Farlpvzjh!llUGITbj9ROQK^6(T$I(8 z{>RblMcBbX_E_JaFHtyB0`B9!o35O$&llWn-XBL+b0=00Y*oX19Iz#B-Z|m7zXuR3&n=i-o)sW5Y#t}Ky_;9;z0xo-}@c2;+TdN1U@_dZOTa))V%@uF4 zk;awpZSD!&*M+~t9P#!b(7y5@xpIIRN5{$v!=X}(_imR+PvSm+Pd9B zj0CEd#&sUH@B2q%#}zVd=6p8y{io*Ll2X|&?C%e4a(zXg<}asHdD_;6FG$ALm#el? zeb#D(pC#Zlt6+J{Ng!7yS&5* zliT&=+`ydY!a9mF=FiPWl{eu90kbHviz^N&QHNJhAmQaBa@UozFe5zu1;6_B z0+D5#w>&;rs`V#>x#9>U;MCY}@%qRU}A zL>#Wv1ZMf=dR4Dk!%*e!>x$d^T^4WUU3bp~_>b~3U*Gn)vpw__db{=#dEe_s;9VS7 zy`NrRW5xJ04}5R6-sbnl>R%fLblGub%jsWRS1&mjaOrFj?H(3b(__9~cPR|JZ@2k+ zJ+-dq7{(Vl^o!W0B-HVe+vekor&sMCO>!Nntybks-%epPfxnW&o)|PLa4B|<6SQzC zvYgiT?A}~8%hv1MCL*&4e*sUWe) z^O@p0PH^%K$-W>92?e*_Lph8h?}895LMb_q$JD4WLEgJd2MM^Ew=QAfm~P#wtmWwi z4fD>oD>n3B&g0&exQ)F_cNVtqrQ~!aIblv2b>B`cyLoug0_Jsc$LfPdJzc2t(B{o` zkG{73hA`CcP|vsN{Ows(eq+h~xV3}W*D<~YgLd;QD>aw12=;Jfwpm{nSeL)^s`YG8_Ok&$c=cA8ehiVK9m*#X^ROf{Bxx;>RK(ShEvEz!28 zG0ox$Aw_Y9-t+Z%UaipL4Rd-I4125}AOR&yI~AYTg!=TB$SRk!Z6`lw-|a7xE(Ws( z15W5Cm-_sE&)95JpD{i&7lk=G#nk-0iZ@r*gLWUebTy^-TaLsG<%g4`|fldRp1dqc-8I4wcmc=oFx9f=~Uxj zy>qXOKYqMg(eF-aNOE-iAu9E|H^YSd;`-oc$>6h-*2w*p8>lSVd2&M4p{i)%4!ske zKn_CdPwTVBB@;5Mqx^Bvl*!_DYtFRBC9~eO+uM3W$IWbd#)?0-E;0Kfv8h|Buy~Ws zW{c~5x(4g0)yGOD2l!PPze@1gY(kgsO4qbv-@V;UihhVFU}!967o@Z|SdJ0YnEQ_L zsBC4Ye96whzt_H)EvNE3nH?(W(29-GKArWLyR&>#LjmVOSAr zUVixYguQ3lrC7pVIAzMyWO<^;z1TR)uuuj$wk25eZggF{krcVfaD0pcy?djA7pNqs zcY#6R*5<;>W&iH*E8v)=%hq+YSzyXqC0QLIY}MZBzG%;Se|EVt{|yS+OC4`Iwdd;HF$Z-r+VG#e+H~(h61m2okp?oBnfVXj83El5gxuoyzcJ$rj1~}` zSu^$o?bqL)9sZ0CZ(I(y;w1hNM*6bnDH}D)!~{W}Awxn06)wd`NWSl%b_a?SYGQ&U zI@E}drQ&ET`w{@>m55u5ck#3v_mmNx5;iYIT#|WRe_0BF;aTtc%|x{hu@g(9(vdky zJ>K9+Vo+o-Hlk(=x%4|F8D7l_PAUx0#CvNN*%@*&16kW0 z!XWsrDnp)d8jm{a4EcS;EUNa9QDnTKX)9C}5^OjSugy7BX_vavHB`>DZ}JT7o_U-eHahz~{^{eDKzR_GqDz&%-QS5xoi#3zHu6#=zM9A{_)hR4= zf1U~=^L)=y+a2R1PQGqv63ci!%X>GIM2MeUTh%If`Jmk5FsoRpDtVC`>(>FI==p2$ z+!U3MvB6o_rHSvvTA5#Iix>5<=0lEI@j~r0$TsYwO9s0C>(J0!;P}(F97xVYj9POK zF7CCTO&0{tBe&#Dxo<-KGu*@ZHu7Ro+@Q zZZBZsf6@^~WE1*Wxgrgo&Q`shiW#jUZi@w>?WA_vcUgIOwGpxqLk=GhIrJA{mNTo| zIz79%YT#$UJCZg94Jhck_gfNV5PGOx&#$)`JkGLb)WCa=z^@=*o5H&LLDG0<_<(5# z6t`MuW1&GY-<#JjE~ON|K{lrN+ka!up{Kpt@n~B?Z}Yx6Dp|Z0+LrpziNhm6I9%2A zJfkFPcQ6=E?pZTH@?Ek1=p+110EV`8u(m4ITm8r2A|_QWdG8{Wg(SVE^Y#MHEKk$( z0}aZ!zbHUP1V+=QQ27jjwjkNw%D=an32M1`xsuztpEClc^5@9n_PPcy9N2}N1t1rn zmrDF$A* z&2?LMWNYO>M+6P<+JkN7ScOr_cD5s4&P9~|!Lcm&!|y_lH|JS*lwOmTn5F^N36N3; z;sTz_xJgN-j;A(CtFIFnVr__UJyZAggfpt6Hm$Y-P!pSgnoz!&RH)eK8Y|(-QvYLA zZ?P05^=R9Y+M65O#@o=*+m>Tj!ipmGXOZn^GX8GA-=NZatNY`PN4tkibG8?I*!Fs0 zYw9h?uX+o{CTTX7?*p#UGb6$)2(ASi7h zqI1t{<4XCO0$FuHo$xU(*I< z)Pe{tZ&6EE;61HUBK;E9lG0gut-7^4KOHB!Td#uXF}731c3g#al`>rgV^;-jC&wSw z5(BpZe*{#b>%3E?d0b_HfXNN{>tP8fDkUKQr0Q$2#I&asYTN8il?j6tuT$IO9SbSL z&$9)QUoP%ea-LS;9S16decD!i+KCEM)g(La{Zx}>7&`7&YMxeUprKO%&7EvO1t$8X zo>s}Ln+==TV&!h`R;%OK3Ip`l#PwJfnP2Yn#~qv&@XDolRr=_nLOm#7=x2O=l+2N68quzM;ldso9>A z5r}qGQg&7TE#Aa>0E+HbH_&@7g+X?lJG}QlVC^QQ?ckkyX1HdWTOjls;ozw3&M(&% zjgnJcIcqMmOui)caEWlgr_5gyU^A#5&CI{rVP}NNSmemGMRT0K(c5_asnvSRK-heF zNgecKUn4MuMw%N+h1SVBSBpFw{q0Q4_mQLvn6jeZHkS*x5?_6g$YR~#MA`Ru{wnkj z%}6u}s{DsURqcGXJDc4fW8jBX;>m+C$D-xeE7v0{&*$1gzwe1qV~_C(FM|6dZ-t&O z4XLyRX}u;^eq{CNw5t{i7WC)OFbD zK!BwpER>AVdB;83{JB4;VpOKS&r~rGt*~B^2gZeZr;|58RqTfgy+b@EaPApuDpVl? z0uo_i96+6)f#0M2xiYp~HHm*N&jhDs)e zOgsWeasMMU3DxB3_$M3`Jw9bbhV~;8ajnjm+LCe|i}0STdRHy*A=h$xV$qphRPjaK z3H@LwBw`@ES_isg)o`Gm88|qmd)5dM4r2`GQG#>%@c#^n1pKd%rz>WFkS=F`h2(w+ zU8pnrCu9}`60u--4}x4zj`CO$5r~r|l@%K18r6L?l2D{d8iT0*H0ltXtpNmysQxuc z5W-4(iZit+kiI!YXrAYZgc2t}ZF!09g}IwBm|z}y#yn+M9mu{-C=`(6i%>O>==OCe zRQ)rbFxXHq)MI4l_=J7@lF4C<i z6Cl|Tq#6>z%28*q8%}51UF5?fe2d&_J4|ik!&LMLJ@&{5dqU>%$%07R0Dii z5GYE`6tGl;@*TR7AyHg5&!bHXtPeu8A?X@RFdT>ogohRQ0;|Aluw18M%nBJQeWuOY z0uA$NnLQYan>mEb3R#0CgegcntrrSe<8>qt^r4z88wME&Gb4bgG>3bZc8bEdJ7WxR zH2i1PENX!PwcQgg;}KtZh=B16gOP_q?OreZk)Aogcm=!0qM!RY@rrJtXXqNq%78|U z#6j5VZjb#x`ACa^8In8GYiogvzc;}X1;)r4bRh-ssbI3ShG0b@1q{Z*aqC}d&6Obm z`{!d}8lG6-itBQ=epZ<8|GC2C@rWOXZyfF)ZUFthY3r|mRspB3liSbV7sLTHV3766 za7eg$Prmwb?=-uX-LE7nzHhB5;10b8ap4eSIEgUuW0{HHzl>=ozB~wy;r9U@h5%FZ zUiJs{`{(u|6TH(PybKTar%ta;YWIcRrmT52C^&Mnb~LhfwX$}0NREYt$Up%mbiL9G z;3l}#ODDlWLt*@yG#qvI8nr+*(n9=w9w%9qt}V6giSTXs+hP516k4zB3bod;h;VdK z_Gs1NK(6k?p7Y^!&1-K3$6N&gWJ7O9p<@w5x9(`xgj$WUqG5%4J;lMsxK^+q;zt+q zhpw+QYu89)?F4K045r9thdVP|b!;4beDA#7(ERZF+2nfg@Yc^^f}aMsVq~zo50u`% zx1Pwle!u!-(w1}IBoTGHnRGvR@gesT3{~Y1Ajq1IY0?u{Z<67ptskWy3Ojr?Bd?x! z?m2ev#}&R0VWgv{q#t5n8~tAMX2f>4uGvyDv%Grd>hPVI+?d*~d&V8t*IlMRHb9H|ka(@fp$yJa(k!elFNm?kbF_t$SS7)ZI+o4%aym>aE z>pKlw-U?r45Fo?WGuF~D&CoNC%D0;*gc4UjqLkVC=28q%liPp#x3J}{uH{R>8BDEt zRWGq~pScWRI*uPTU@G*T_WY}@qH<ODnXcrna_K}Q5|EXQyzoYzICK%cw%}AnkUoE(Lo;($E ze;Z~N7S{bJIzj62P3(BTHs*)odw^goU(YLK_aSpqCb%!dOnNF*7|dCBdrLsyxd-44 zoTlcYL^Ckc(Xvc3unx_%2*>w%yU4n~0bEPV>`c$RdS3hUTvg*?)7Na^r(_b@`JOP> zX`(A22CV1)wmNPO^C%zG+iDziL&HH?CZ{)=rRsx#7Q}qOt$H?!2AtCPf6;S-Z~&{@ z``>!({IFz5n&D4RNwt38-Mvxo!*f1(Dtpwq1LP z@N3FY0{6iJm1|ZhfmR`h9}#-L>@9gy`6mUnAE;-MNXlSjp|7KT!H`;#1Xhox_r(KJ z#WVC2m`d36D|z9`D*T{cIz3BYfc;k7Z6Vjp_};cOzR;TojA`DeyW{lM#Hp4n+vt-6 zgi4twD2Ml@-2PZmtu^>tN++Bn);{xy$=kP`K}`KvX=q`_28?2)3Bt71wpSM*ADR!$;igEFgc+S)eX~4sNRt!Dse~Depm4_6Kd=<_lgx+I^;YvMYv5*Sy|6@CJ~5u# z`XZTn@X_i+%uN|&i86#(8=Rg_feN{ac7Eeogqt3Q%OFjj07Edt&%#kMO0(|KC|oF} zP=uKSMEv**;~xpu}b^jK?lm|TvN7I3Cx>mq^f;4#+LiI_@ zmEI4}UzLIsx03{Zb^_M{(8l2F5iBOk00tNTZrYV*?n>nb;PV+Df|V%4AY{2vJgTH? z_<`&dWP&K;`!4DaEUdr9IR0P7n5(nCSTt2orKjr(DU-9jXle1BXyDg#U15}K<@V&# zSY?BDPHJgm{o;D|H)5u0LeQy}O{JmVSaWaKyqx2z^U|pDm21CydmG;s$UH}EFm{l# zvq)$BShZ~$$%&F5p1V~z4w=y7KYK{;?>%u?jL0p8HyyyC6ERSBtP8Eg#mEr1hjrmp@Ud43SMluKwI(3NAZWqXM?=kcvSJOK$gJih~*Kq~u2E!ALLMK=Q5IWRNVc zBkMfqCh&$GgjG>jthj$;IwhKvlEHrl?%=+F5|`;I--_m7@sH$5Qp?Y9>6{y7RYl+f zx^L##2Xw}@Q*<==zy}A`1W6c4pDJ#sneqNs;Q<_Ms%stIFQxkWe)Fp6s{D@*mAoT> zj(4Yj>3APquV*Iye~@tbp_%13c>k=JqO9xlmla>qKCO6A{ujhUDZq*^{|_sM4E93? z_aTnL>Jx7)y-7XB#;;eg2bV3o4B%S$e1Tl6;@)U8EoU+&EV-nk8f|Raz}mZ~HZD>1 zCRX3rw(8{BB@e$Qm~-b~^V$JhTFM)|d2C}cip;--}jA;wew9_wz!($C#hqGz< zFUBfyXjdvxK?auTa{orM!lVvhLsR`px5O&&Z>~ya!}*mBpmh+2@KVtP4%J4I;6$2^ z(I8>~G00lfOX11Fb%AQGI+e&c%%6%Hfhq_JY@0O-lt*AxI-O^TmKeLb>xv)LO9nI0 zLxBz9A%nG0tT?F$xRTIz36}&biOS?Du18z2nl6n;*`!Z}DtAQ9J*KepiFMUx#;POP z3`HOS!K2aj?~N8F(XCXa84b%*5x57iL#6fQ7RPKO*(MP|2e1*Knn;)I7ua1WtuloH zrim!4f6Y3@RBrm3@~wc9b^u#GI3)rwaWF+yUa}5`)j%kmA^ZR~UC+sIX(ruLDJtTS zDg&N#I9rr%OYx@T|00WxYW3cw!>6VmAYDU9_0DRmlX;x=V+VfD&-&|;_Z6pG5GN#K znSM+MQ(3CY{^aNY`au<%+D+)$+fr~Re>orGQ~&R}0TeHFV8?cKWfuv&7GGqh&G zPtjSQbADNgR_i$}Gw+~OwSv_dc@pfBnqN>G8NINe z>2N-3D1~CET*iv82 z_6QN4DAM3XmG(%=R0dBXj6KRFmEz%4nY^&%@M_#~qhuTjxECYyRCVCcoRpKhRN=;w zES2gg#L%E=VU)!*Bl9AjE2R9XO|L}6(a;VtOMk?xGDVr~rBCzqTIWT4Ucac-mPJNY z#3DQ?^CDKqB})yrr;ZX6CS+{yi77?Uk|#zL#mItCRFy_^P`{GaQc9wk&;?STOq;-$ zr}ZLYeN{~}%8L+J9`wKrZQ%L?)(}7qxDvROYh*|BDWFl+*9B3VG89m8VtlsLy3f8c z1S*4nQ=O1XE>vNnR8f%}{BP7k=>J9~Lf^W3|3*uqXKP{MSk4StX zyjzLfF7-rC@>EwE%vxxVpoTEn{Hj_<@u&L7PqA4yG_t?=?tymkAN%FiQMe9Y7h-7d#mG z#%tBzm&i(~{q~eQR2w~nP{tN{G0h+y&n4+B9ix=?!Dsgg%^( zv;}R^ZAq2XdeVPH8lcZ$ORlf$6<)Q5U?8D#3|Upb?^5}f_*_jOeq1{OtK-RxxTwL8 zOM_f|Im=B({Wc2(Oo&4uAfrr8^PBlK{TO#_!Wqeua3P6eA)2B;kJT%wT$N<*r>dh# zz=KIW6)Y=5D!=B6zj>E2KPZw7cl_IwIoenv9?{-?P;^PohlNwZWc$|9){fx*=|9k2r&hkG zUL-sAsu~WIG?jvV0l_LKdkPR;sZ!v{+NvEOMo6jlqFSl(_M*R1MIutQ zZ-;#E1_eQ2OC8b_wqOMZI;`L(T}1N<4h4ZU>&BSsvmc*!bY2)nA%Iu^vVZ&0R#$`BPy>kgMa={no5M}U;)QU)IYr#f1>Rez+&g@c8%OAmCBrij4nvpLyK-%(3f5P-Xe` z1A>hdHY}8(!!rMg3zRk)Yj{M48-vSEDRo $DmXJxKhE!0Y!R=9IZuDWX4{6`fO*llqUwa znuZIphCN&Z%Mw^wl@m(}(vgjVCO1u5UFd`c%=i}3s>C8rNH9#V!eVg-~k#1(3KNC@|)eIuM7XgQoejj4NK4>3mYbw_ys>l$)^ z0Ld?pHSjSA_&SUdVI%I12(O})cNcNnEL@;bw)!w=w#VpY3qc^Zd@h{P4B4+ItI2GN z_}%|82%b~y75Ex*o+;uH5aoS+<=UFpDD~2~Z6jO&5J=$fK)l~mrmi84zX=1%hVcv# z{KWRVDxjQD*WqMXc$y;$7x2#|de;YQ%YYDKG&Bn#lLU%<1l>hUYjB;^{Nz>(ToNuo z_^!__j_e0iG(#W^+iyQa|82mr9q@lOvLB<7Z`m*bZ@{AnJE#BWo}Z;>iXKHU!xGK= zzLoL~2}eJIK4R@6(j`^^UV}(S_<6kYmP?cLWdZ0rE#+|i6_0%z_0tR4Vai>^Jk(!A zfEm6SaqR(~{f88ifX9BQza0~N&9AtIq{tY2N^(C6;r{pv_P1l0Le(Am?70mB^mx~| zrPR?-PUzQ=6PjL!w4cOQ{8c_BG2#^fWGnx!`Cx>RK;{V^`fn}X0gpkk|5h^lC<5s8 zlFH+pz5)mq5P9u|azu;$e1z-v@x}jIglzCx8(db?pK`7taqI!Lv5Q!21dyW$^M-_Q z*pQ&$u3^iui`c`qi)cE*003-V@qMyrT5AYGI9_eaSJcaP)Rkk2AgNZB`6k8#FA zZ})f4$}|b;_Y2W%)_bp;hA%cvZY4{Y;?`?QhhsY@FHX4gv@UIn2k$W_WDX`FhrqSM zYlvW)#(%kLbpJLNrbfCvK&jEeb78P*bhhw;HtDNitvAG{>sD{z0Q zUTqJfC8oG$_v4-NX6=vwe|Mal3*9zlzf$vn(!=g3qrQ4&Xa$@)R&_nCedR_N-gLCj zgIB733fA6=h1u%8#C1|O7}$m5^!fpNEl^mS3@CK!YEFQva|WJ#$!fqn;FMx7@!-dV z=WU)pGY-0JZSl-Dk`w#$A@(EM&H;j&a z$&>Op-sgVWS%(%#U; zV0`i=T&J^r7NPUjA@dU|iWvXUS^d7Tf)}%L>FrUOjVbRGmVKynCO6FD%h+RaQ9Zv7 zfxgY^W{IGHPI)lW&e08#@D; z5U=_Gkvb)9@U#H5Il(@QW-~&x?8AhUEI`g_ArpDdN&AdUmkbZ-)E)paxRN#oo@?i` zXh;(pkgP*ruF3e*X~;7Th%a#V8SSU6Edk)1lRDUvoRbf5g{AY}8ju2=c(P=$RBLAZ zdd_ZL(i=uo#arxCM)X%l^STZyaU-gF>EQC$2@4^OnkDj-`;&&U>)y8}RZh-KPID?P zy1megI5m}p63Q|6u5H8|j#@@Ad4bgZS}SjN1oM|u_&iH6xbj2`GYsMnwf=pmyn#jD z5Ob1jKfcx<1RLA9j~5J5XbLrr*}$=LWS7U%%D7V@R2hhvL2RjS?My>r6{NFR$e!cD zbTBpI^-L~A4`v8ky#9LF5s=qCco9V;rT2pJrp$QX^CpBT^6iwRV)*lV;OKBe&-hX0 zZS$&zDi(h-W+>(mT;GGKn$Q=*ry>P&-{n}P@|%?ST!xa2cv6JsI9&~8NPa?|0-`C{ zPECX^K+7;!Yj>jaDJ^>z!}GfZ0__%pCgxlpzaRhl55qZ#!@Tqnq3UdfDJ`II+eOl} zi6i7`meqpH3BW~Kd_5NIU;yr*u(0D>x}&g?3Bs7vPG1fc1##6lCBqux9yPvi5l((<#C7BFd5mInhIi7T`0 z8KfT+wJ9ldl;p){-S^SATB<3-!4LKwf2kf6G8TsTynDBb9TdV(2yw*+ed7U#{IueL z^p1pMyNuU7){z+zaEx;|chK1a*%=|D1rZ#ptQl;8L|eoebEhEx*!gNOsO)ST?IE z%PKUBvlNq-gB_JyXE+ZE6}3ntuwU12=#ytj^v;==pGyJFMR7zxbCKB-iWTvoP;X2M zb;x&ObNRL@FE0y0j~0~Bsj;<$xMa^gRn*jYRq{kbS9 z9OJAg3M62GW^-oy5&OflS)vDPtrg`>3+z$r!0DGrgt(YQxTvVAwE&bBG(!HSBr1xF zq%JS&jD>X|TId4}Sa?uUyeZ4?Qa%u~tB-bn2OK3DTsyi+XZg*F@;`rMfxV9q)I#$$ zpb-8y3-shffHMc8|8~X^1u`F_)+xUpGX;K33U|VcrP;C1la!LehKfR<%Jl`iZQuU; zPtgRqEST@XYQzEKmQrOZMk}Jp?cC0A*Mi0IJWi{bd@Mf8-nKbzn(m>v2VGJ>)V+bu zyG)N2Z%Ve595$gIQd8qtj;vN^~BXFMT6i7<5+GCa7p53l<`JdI=hY?6=%fa zMshwO(0M|_(m(6nr`@xXs3=Tkp{*2_KBGG>F0!6H<6?b^0*?6H@vS3uVs;jYB_*Hd zm4E>caILkDkPTm;Rpw34z1K0gsgOgVPLJ z%vX6sRA~*gY({jdQ2w|mNh-Ss2Nolc;~|7YnqsCln+qcGiPz9<7Q&6S^-GmW?B|!l zye#mqb;(_GlZ1cukTfu?_7Ji0qX0)%j&~)Q#G|gJ_7DRw#2!M#q|JWTW{Zgz=%Yln z5#Z9!#mP6Oh{^LUyRAlLrdL%=)7D1=aESZe*>mc;@l7Vta5M^sB#8CnT`UsnX%mR$ zP&ABpglk8-Y9)`8P@5Y@W@6X<-wbI^dwYrDk43FC_g8Yc=@s+>#G2>Se`@aY!u4Z) ze5(r(19XFcplha1Q;ri;-CzZX0gmR#Gx*4)Njw9d$gJt3pak`9O+*BA3rPyCabgKT zH+X{z9Ajd`L%LY=EC9(E39>(7+DyY~5CXmIIvz^HSZ6n%HnLEX7Ytpq6^XWf!|DA57KtNot z8x#oW2pR|o`QHPsOq`u9Y|Z`}XVs*(;rNRawQH+HzjenHy4gW?kLsATc2kbr8Zbow zP`wY3%^gPqT)x5<$y&r{ifP*U8QRU$RCkxtz_0>C7QAa7e6k@w3`5{7kSG-mHT&)C za*6e;0V`ulcmyMrK0miVme0@k^F4kxeLBl@U{56Sg9Go@Cbgo(?^G>4-`nC~zyhtx zxauCuIKIf=IaLi6?MG;6DT<#PpnwZ8pRJEP)h-dAe~aRxYhrb@$(EvCw$c**vqyKw z<)EEdL~Di6$Bs;ajd@5IvZ`#-!SSxwNz{5T?|O+=w>NQm-oG8?_(xg22K}h;2remw z3^NF0AYCCYg5Ul%s9<>o1GAl`&LQN`U?%H%0-lE=9^Er9g#6h)3!GuTJvuG}74A@E z!qh!F>Y1F3;A|dG=m;D$Q;8>w6DTtNbR7j23I{xj%M(Ec0xbJ{KAIdi{eUc54%|wa zPKG4INfYvW1f?k^j&jYpXF$WF4;$lYXI%y%D_@z93HRbiqeX}+}skj zmIQqAMj3nD+qEDeHgFNAPl`0Exh3e5Z zKE{#Sp)B7Z9HourA?3RjxE==MD_s!6Z~N1dktIQaaN(!vPq~3?7%5EUYN}Rcm!ctw zPZho7!^@F0^~~>AV$!EygSOdoCX4avaDS2XvdZ`ElO}+TIy=BXGX!aRd%=9KPB>n? zL1m6kI6E6gmPHS27JPax$FF^=4~%y8*E=I&EzdBJZI;L3n^x2hxTm`Q?D zv*nS`63@@IE4y+BZe`fU96jv|+H-)#l*b&m|Hg;ZcoTu$6zt+;O7nxGb4y+%H)+?h z)$jf*v4Y@7nMswFbIo_lxn@Wp>m2O_JTF za0HV58{!xPk5(7qkOM0?BBS1qh*(sjK|y+>$0Dl)TIt~p3RNZdui^Pd{4A-cx9_TGs_b3 zbG~UZmc7V-t{C=1av5Kj`y*#lq&K<&!G3PKi2^?ExyX^B3h;p`Ds5LP0NKPZF(F7*i0B6PvI+M_(H>wpcm;s4xizPtte-)H-8 ztpgPxKvJw`z#r~!GefBD=AA7V6I_OE7!6TE@jykR zXPm(#ov#(@mV-Yd2!}@x5F3^;*GnBCeH-K&#fr+q#aq$9oy9>mJfW}+7RcU^|KUJf zs%wE({aQ_lKNo%7Ldh8;-N!Q40F1NbaUmpuOgA|;8{548gYGxvkrIef$MA2vX($uF z^qY4A5#Jnw|3Koxj;wErRFR4QPE8&7;THu0sHcP9@;7 zTYJ8^IKab077?Lay&tXF+<^bLV=r8JPDUU=Krb*rKq&um%+$`-S=GSM+T=4M)kaFM z?Jq{$p-Z0#%G4}r>%=BhWa^>N#&HB3FQ6~DwdiZ{=Ynl>FH=0$v5e=nQtfv(ah~Ti z6Qsa_P*~0)lAuIfAJb2UA9rtfS}kt0_2i<|)UB62*!Hx2piG6{`#c2Co#UZKRMAgJ z5`kZ~(vc6=n;Ep`O(|Q|QrfaB$YH&`DqYr{bW z2sNVE@MgkK(j$Rd>z!~s!Xu$Xu;bK;B{6rnJGYWanPUPgn4%ZQY&^r)R7VVG#RQX> zwM~@{hZzs7aDJa-GGR;RiU>f9fylG4+b7)B4eN+67Q}Jjo9jGs7*~} zuydIfzxxhB5K9-S0_Giz5G+uz-(SnyH)=m!?ZuQ4yYpnHHYYM=&8Q;HGFL(}Di6Ar z&eWGmt)TrK5fhbB3cOO;J4HRyvf~PVK2c_;9=K3arxOsubsVH^J5&s(lFR&~vc=f4 z^9$VVTSwfJZUDDCFz>C1>HgO%kdp2DUuRG*o}KVD)n(r}cw75VizC!Mp;>Z`JY1M? z(!|spv~ET@={)jp?LkUMMzz&CTn>7^nPl1K3?3ivtj+f$o?hRU3^V9`C}FTa%qLj5 zb5LXaWcV_dJ7|p9AGaBA;9}GVFE|6VY!VIq0fxla!qm?BgWTe)&}mV0_}oY~g?TNl zdA6<841)PcH_R`?cPFNgrq*t~iSAr}TUog8uhTc=C{o`m$8Z=uc;YrJZc+rAb+=XZ z?e}|AZuwJGhAV4#?Z818ChIN_e7;Y)yry=MpO$#aRg0JVT;u%rA(O!xa8U05YwN1xqS)H- zN|#FuC?MV4ppr_LA|aj9T@vC-cXvuj?+UVXhoqD=?ji^XERE9ft={`xulu|6$INeL z=Xqw|nRnjv#+h^GnX&sAPVf7-n>^W@LWI%YI1hJKy*$!ied2SIZt*SfsBatU7j)G- zeftaA8IWmGVH?W6Y(E1rYQTfwQ25{|HjF|#>Q?cH4cSu~xWnk%AEK`eH4!A?HWIG`4FAo+RG1{pN5U&EZ`836?<-@0o<=>6YjQsvE!nK;x0hDbCMJ} zGYNIY{8CP}PKHZbR48Y|iW~xf^xSLRnxVoV@=$W@>^_M=-W zhew7;S#N4%_66mk_SuggLs_=A<(Wd5d&ZENZS9F>)}^t!;zyjPWIqBbCX|%o(Rzh2 z6Fr7v++d1HDQ{CX-orOMZX;MS^+Vt0YJB1lMcmOQReK=!iS3=5b>V_zrp;}t%;%D7 z*9X4uor-K(YzzZ0Tm4DMp&Qbl=t%eqR-W^#sMORx8HW^3DrS4RGQ^hh-m)yfzWZc4 z4N(ijC7Z=Izx{@|mKltXODo{dB-4P`*^@Y#{qgWyvL0xG0K35PCAIW3^VsROm#1GB z(ndQg6nKpvoUKlM?Ss%>8e+8LUb>Q8t)25;1)Q;;AWprVbN!)!{-ln@S7%%GChdkG zBBdS#B=2&6c@E($d3_kh`l#V1R+!RffM0mfy>-BrrUr*eE%WUf=6rODWhRgAyYB09 zm^^_$f-5?AROw$o0s$vzT=E9Y4Z<7veM;G(h?)q+zSPi$Ro~m@QBNlFD`x@q$&^Y9 zY8U%xjk8R+v#(v+H7lJsf_;;u*KYHCE9M!VQ9L05au&nphpW|JD6oX zR#!l~-|+_BwVEFL1>5j?)@)L2?yW%{Ipz23(=W&Ji(I6I z(!DW+=W)N}Fe&}sw0=pJnhK;4{(h_=ahJ?sGGcmaa4jUi;P9o^a%yr*Bgf?AG;6W^ z_JM8L;|NL(#W>}!>Aou`+a?>tF-2XfIcF;w8mWrxYVU8`vc4xFSt;dH!#us8J?a*% z>y}MR+%zWB_blyQK(SX{VJTl(+SqYzK~)Red{>}zDJ*hY^p?kRP*5EdUMj+6+Wp2c zW`#x};0IdcUEM0;#OlR5ht7#{yDiMV;{>O3Pxpcn^ePAaRQscBFzEzVbd*{@EL_Ym zrH*I+=G$_S%|Yf0sJ88t{-l1&v3At@0MU|-kZc{FypK5t)KkcUquGOjXu&fFosWsM z4!ziO^Ra|P`Vjgv&#Mm0*JYyu+Pr&fXsH3a&ZX{4vWjRFBY5hZ?eK>lbJ5|e8h)K` z4#r&uhN@}tfcD_4`R?ze-2(wz?7#OQlVw8*3at{WpST*AtIvi)Td`f?s ztOKY?XGaIfUhYZBotJ$OPh(vkbx>K+R^9r-Z>lF^wj>Z-*EheGo&aSe`=L3apUG)Wn9s59CgLzZ|3*of*f^}?-P~@PATBT`95=t zwg%evEXpu~a^&O~iVDn*K9y~)T7FZknH}S|cNCwd3R{w<64Xtzh<)afv0BFQ9Jt8D zYs!k1!5C}F#za7y^T?bxw9ceaBdvBEEiYAf?Gv-bUa&t0Fm2Cvt9S1rZ@{?tYrg5b zd)G@DT)1v=VwVu36YM`5zFd&RObN2V!$Nwmq{yv>r@M!h+J5 z++Fbew&m6D9v`kr9}zKPSTa(yBpUYg2Az&6cls zM1nRa>q1`r-xbSM%%$6XUCVNmnFGp=NiG^6$kMwGx$j&vYS;bMJ zD;+g$)v{(Csmqy5v_Bsxv{_Yc2`R8SQgF&Y1pg_rrkk^i`>&^(6JPBV#7FkXyRGYB zd4T^7h7DP`B7og~O4*3!1&G;!ap7T1`IVoVgs?@QH`rrm>K?V!YOs@OF%Pdr*S@MxSUfai@sePT>t zZf9L3T}Q}q+2~oG#$zaVyDKqWt@_!ve+V4B5>W;34ZjAqL?W2Jo1WZ<^ZNF z;lKzocafbSgOWu#t~Hh6>91GEY3nRo$4_wQNJvJ;8eU1;UN2q6jh+)POkk35KuXy3 zB<5N7XXh0&q8;wVh=z9|$6nRjONOjC4wrq@<)mix8||I&qWNN}8}Wr-S*uqt!~t8Y z;!WX&fC%2#g9*=UO_+s_YYQZtHN5OgBtBko6S;i|5oBRS-jzjj8IwhiR!CQKeCxxs z&b!B*u3A(1{EXTiHVR4^!w9!S*{J-7;VO~C^n$d`?B)WTqqo9K1I1ND#l$zKlhrUq znU%IfHAI>aGeCaA)VRFtuQGzz1~?(s7$camHj#L)O1RE5T%6)BPmVqoiIaQ7 z)EA^M7g-cUKF4lHwCWg^lU@LIm&P}yC%k&^JDU~&iDDEq>_1={5@{zqX(`E5L>NO^ z8x)H{)fY|0t)ku|DJn03>H2lCW84WR`1AA!Zi<>2g%m7oStFmA z)D)?*eLFHVUW&%W=KxKGlcUO5I4m;=4!EXd8THk89!Ik?k$2*sJ?@o!yjNJW&xr4bE z1>bmku~@3DV|sJX)x|svujQh)&!OfBmt*s0mg;XYy|J{f?+~;qDVVd49!Qs`w10V; z25wV&G4s6(*S>XZX{U8h)%amd>z?c)cBhf9(T9}~zel`SPC4Wf^R1+9oe@MrUT6dz zmYZt&wQqV;iI_$?E)REKU0nlf??#x1izSum2**^P-P-qmgRd^I{?PX9IhtTuUN!mC zNTjzL;_A)v_4C^HaF#~Fod0}`>#oL_#jOX6A0-Z;Mxy69ZCETOt!q`TSZj}k5bM&J z{tp(Zn)jjF))zQnpei^t#wbTTspIAF`7F7~3tj(3nwJva%45B*+wj%o3GUok0%G6L zG&VebVYKf7Jy?>GFyazS%DHm#{p!#NdH2S`+7wK6YqI0M;1(F`Xs7Rpcx|R*XZW4W zO)kFfmfN!(1n%~x`BYeTrgDPA;vsXa*ZJ^k{68($OjdE*4ANS)U;+R{NQ?DrR&g6E zGfOMCUzVykS;c9Ak0|u`0cJi=Z4EP3)g7D!bVix^t-z)S0J(9JJBf1fwc5w#uwzS_ z2W*%(<2m+2cdx%n6%zA80yvanKTF(b3P*NrB?VI@%j!P5xLnv?W?&i7GJ1lWNwFW% zd}4RFQ!{{=AR)6)t6(T^74=CJ({X4-?JKJyncc&q`wSQ9flpeBe<)L`^4lpdMwx1==^+xM&Mvn#vLE z4s(p!2~T2n(25`QJ< zZ+r!Dd!hCtp$S(vC+yr|$&@8iEY7RsE^F$&IA}##r-@Sa1$e@ABdBMy;|cS&Zmh@& zUk13b-l+bNbwDZUZW&hrZ+7Zud8OvVpb%}&^7;Cx0X9UInLqygz5A*(r0L;R`=E#q z*e(??vE6h6rdiNs19B3;naj{~a-H`Y-D>DC=biS%V}!Q+)dkDF{0H8F$%aI#kIuhE z9aZj!JA3?x62$1#Sqgii)4Q26xg;39uEjQG zQ#O}#!%0ltG^{+&tt|A82b7UGpIk{@$pL*lFAbEQxFayuxjt8Zjd%G-N6|Fgl_v$P z7>XCu@Kk5E$qixM=|aFtDpHUxtO?R` zEZ_HrIj{=GnksJ5(!E`y&6pSN|4Oe;^W1Vp>d|exsCW%b(=eajM1Ad1_`+tzv%veL zjeWce_U1>^rbbpN5^38GINUBDgwA1!slkMM-=0`JRK#~k6GQkoQT-OA%k z%Pe=?7ps!AFTNc{s@pV3)BipJLMk`HA#4rS!bz5xI{6HLw-vtH@dFz@h1^cnoQSDd zzL)3RZg8^EGBuj_agIu=-zD==dK)*j<`Ki1d|f25wWF&081uaHl-NojQDsc;ih{3U zZ*9F`{Af^bG(HB742>j=D0s*ejWszClb?XWx8+M1Kp*2fp3T0Xw>0`Z){Hn=%phr` zua%cnPD}*>__40ooln7vxA>F%TfW8P38az5%km|hu1E^RkrhJYhpEyLiu!qAPcOO= zj&Rebg6X_|Pf>2o#Qu%)|zJt?1(5Q{TMSN#v8X(v3d zm%hJr)bxCRbzprG(1xAYdNB|jDAk$QQ$Z2QR}V^j%m@!)CS+$1TDje{pV}ERDmfXF z=%HJ+eiJ;Z{=$Ye7@CsCk^fS^%TChnYVNZI+yQQPwV%mDIM3`vS|4-{S{T?~mc%Bl z&qAz8t{fpf`CaDLiaWb;ue~9YcOa?0SO_TKMo=%`JYT1#1EqbsWvOD1pe2>tO{-A$ zb*cw@@WdGva4+G|9@XyV7yGg%{qJY_&&s5qNKgk)&1_xIItYZWkemxQ*MW9c8)U6N zks~5Iin7W%Cs#KI+2g*IWCga#`p<*sJ$LRrrDG~fL}Y`vB${+Lwpe;uU(?}p3p}u? z%HOjaH`z^6^Lc*gphX-|j*;|WJ(vIE3Lb`Pq>4a9AXB(npf%Rm?i=5SCLSD%qAZ8f zT`as9>F4;l&#+1BR*69~%%I8)TmxWk1fwy=dgf|uxLUPfo>?0pzU9e9S^@YJ6IyqlfcWM*KFz%twU8&=BVoZN zbQydGf(Fp>`&l_`*UUPT&jCfirn&21mb0~s#z{?!uW6m^$LQ+s1V!v%S>Q#nyG0Z0 zF?LgBtkY6PH6E1HU71>anodF zBF-cwq5gUIS~70xHjFVlY(#AsLb?!E^!#}Yb;<+VgXHYyw<9fn(_4XEewDp-i4y{T zKAvjZ%giUfFxpOo-9t$7ilUbL(B^a)ZNSPn6hZ6f|MhK2Zot~yx7=6x?nBdQ#sO9f z6_D1`>4kf6pL+Xm;5bS6;Q;*Qxzy#>P`AksYH{Dn;}?q?&LjTku;WYXS7NW0mLcs% zXP0qO*9D&Z9XFFDZH#6yLYadZef252MYLWg?0t3f?2f#(VN?+sH)@cSXg2rG$L}~E z_jJUaakEFREQ8egNKHaRMT?^u+BJWzWrZHXyeR?Ppmq{tv<^OXSycvlyd!)Yh;`fM z&FlYMT?>XEbxCQe+(08k|5+--0SG>%Fyf(D0+b?~{z2@&QV_@=ek~y6F#y0o4p|to za`G^F>f>VNZuB!!LnZKGQIS4Do(f0Sl<;$WvsH zf#J_W{r?xxjWZX33tHw%W?e;Uq&AJ2X5x#@?ZaY)P=ka z?I{3o*@uR5@pEJ$;+GSQr0{e^%Kl&Q3N(ku0mzF7B2T9N|CNzj&-#m-+BF9Hk_S%79vw{LSFr_cy~IB}!C0Y6a;x9&7M#JZe1& zl>#*v_nYE+>_0Pes064 Date: Mon, 24 Aug 2026 09:41:15 +0200 Subject: [PATCH 084/189] metal: fix batched Q8 static attention output --- ENVIRONMENT_VARIABLES.md | 14 +- metal/moe.metal | 9 +- scripts/environment_variables.tsv | 14 +- tests/ds4_test.c | 280 ++++++++++++++++++++++++++++++ 4 files changed, 300 insertions(+), 17 deletions(-) diff --git a/ENVIRONMENT_VARIABLES.md b/ENVIRONMENT_VARIABLES.md index db71de60d..029c12dbf 100644 --- a/ENVIRONMENT_VARIABLES.md +++ b/ENVIRONMENT_VARIABLES.md @@ -1265,17 +1265,17 @@ them as part of its own test contract. | Variable | Accepted value and default | Effect | Source | | --- | --- | --- | --- | | `DS4_CUDA_TOPK_REGRESSION_SEC` | positive floating-point seconds; default 2.0; invalid or nonpositive input restores the default | Set the CUDA large-top-k elapsed-time regression limit. | [tests/cuda_long_context_smoke.c:72](tests/cuda_long_context_smoke.c#L72) | -| `DS4_METAL_MOE_TILE_MAX` | cleanup-only historical spelling; no production consumer exists, so setting it has no runtime effect | Clears a legacy Metal MoE tile override while preparing the test environment. | [tests/ds4_test.c:7236](tests/ds4_test.c#L7236) | +| `DS4_METAL_MOE_TILE_MAX` | cleanup-only historical spelling; no production consumer exists, so setting it has no runtime effect | Clears a legacy Metal MoE tile override while preparing the test environment. | [tests/ds4_test.c:7516](tests/ds4_test.c#L7516) | | `DS4_ROCM_ENABLE_Q4_PREFILL_TILE8` | cleanup-only legacy spelling; no runtime or test reader; setting it has no effect | Remove a stale opt-in name while preparing ROCm Q4 test cases; TILE8 is automatic. | [tests/test_rocm_q4_dense_pair.cpp:58](tests/test_rocm_q4_dense_pair.cpp#L58) | | `DS4_TEST_ALLOW_FALLBACK` | presence flag; unset: native mixed path required; any defined value including empty or 0 permits exactly the serialized-fallback counter outcome | Lets the CUDA mixed prefill/decode oracle accept serialized fallback while retaining bit-exact logit comparisons. | [tests/test_cuda_mixed_batch.c:204](tests/test_cuda_mixed_batch.c#L204) | | `DS4_TEST_BACKEND` | exact cpu selects CPU; every other value, including unset/empty, selects Metal on Apple and CUDA elsewhere | Chooses the backend used by model-backed tests in tests/ds4_test.c. | [tests/ds4_test.c:91](tests/ds4_test.c#L91) | | `DS4_TEST_BATCH_ONLY` | presence flag; unset: run batched and isolated-control phases; any defined value including empty or 0 stops after the batched archive/hash phase | Runs only the CUDA session-batch phase and skips replay against isolated control sessions. | [tests/test_cuda_session_batch.c:288](tests/test_cuda_session_batch.c#L288) | | `DS4_TEST_CONTEXT` | CUDA session/mixed fixtures default to 1024 and require 1024..65536; mixed-batch uses strict full decimal parsing, while session-batch uses atoi and therefore accepts numeric prefixes | Sets the context and placement hint for the CUDA session-batch and mixed prefill/decode oracles. | [tests/test_cuda_session_batch.c:125](tests/test_cuda_session_batch.c#L125) | -| `DS4_TEST_DSPARK` | nonempty DSpark support-GGUF path; unset/empty skips the DSpark verify-depth test | Loads the DSpark support model for teacher-forced verification of committed speculative tokens. | [tests/ds4_test.c:8283](tests/ds4_test.c#L8283) | +| `DS4_TEST_DSPARK` | nonempty DSpark support-GGUF path; unset/empty skips the DSpark verify-depth test | Loads the DSpark support model for teacher-forced verification of committed speculative tokens. | [tests/ds4_test.c:8563](tests/ds4_test.c#L8563) | | `DS4_TEST_GPU_DEVICES` | GPU device-list string parsed with the normal auto-VRAM parser; unset/empty defaults to 0,2,4,6,1,3,5,7; parse failure is fatal | Selects and orders the CUDA TP/EP devices used by the mixed prefill/decode oracle. | [tests/test_cuda_mixed_batch.c:122](tests/test_cuda_mixed_batch.c#L122) | -| `DS4_TEST_LOCAL_GOLDEN_FILE` | nonempty readable fixture path; unset/empty defaults to tests/test-vectors/flash-0731/local-golden.vec | Selects the local-golden vector file used for model-logit regression checks. | [tests/ds4_test.c:7221](tests/ds4_test.c#L7221) | -| `DS4_TEST_LOGPROB_AUTO_METAL` | presence flag; unset forces DS4_METAL_DISABLE_METAL4=1; any presence including empty or 0 removes that rollback and permits automatic Metal selection | Runs official log-probability vectors with automatic Metal-path selection instead of the fixed pre-Metal4 baseline. | [tests/ds4_test.c:6955](tests/ds4_test.c#L6955) | -| `DS4_TEST_LONG_PROMPT` | nonempty readable prompt-file path; unset/empty defaults to tests/long_context_story_prompt.txt | Selects the rendered story prompt for the long-context fact-recall test. | [tests/ds4_test.c:6690](tests/ds4_test.c#L6690) | +| `DS4_TEST_LOCAL_GOLDEN_FILE` | nonempty readable fixture path; unset/empty defaults to tests/test-vectors/flash-0731/local-golden.vec | Selects the local-golden vector file used for model-logit regression checks. | [tests/ds4_test.c:7501](tests/ds4_test.c#L7501) | +| `DS4_TEST_LOGPROB_AUTO_METAL` | presence flag; unset forces DS4_METAL_DISABLE_METAL4=1; any presence including empty or 0 removes that rollback and permits automatic Metal selection | Runs official log-probability vectors with automatic Metal-path selection instead of the fixed pre-Metal4 baseline. | [tests/ds4_test.c:7235](tests/ds4_test.c#L7235) | +| `DS4_TEST_LONG_PROMPT` | nonempty readable prompt-file path; unset/empty defaults to tests/long_context_story_prompt.txt | Selects the rendered story prompt for the long-context fact-recall test. | [tests/ds4_test.c:6970](tests/ds4_test.c#L6970) | | `DS4_TEST_LONG_WORDS` | atoi integer; unset/empty/nonnumeric defaults to 0; valid range is 0..DS4_TEST_CONTEXT-128 and numeric prefixes are accepted | Adds repeated words to alternating CUDA session-batch prompts to exercise long-prefill rows. | [tests/test_cuda_session_batch.c:135](tests/test_cuda_session_batch.c#L135) | | `DS4_TEST_METAL_EXACTN_BATCH_HEAD` | nonempty value other than exact 0 enables; unset/empty/0 disables; false/off also enable | Enables the Metal exact-N batch-head path and requires its attempt/use counters for every eligible oracle case. | [tests/test_metal_exactn_oracle.c:401](tests/test_metal_exactn_oracle.c#L401) | | `DS4_TEST_METAL_EXACTN_ORACLE` | presence flag compiled only with DS4_TEST_HOOKS; absent from normal production builds | Force allocation of the exact-N Metal verifier/oracle workspace in tests. | [ds4.c:61991](ds4.c#L61991) | @@ -1283,7 +1283,7 @@ them as part of its own test contract. | `DS4_TEST_MIXED_QUANTUM` | integer 1..context-1; default 128 | Set the number of prompt tokens added per CUDA mixed-batch round. | [tests/test_cuda_mixed_batch.c:113](tests/test_cuda_mixed_batch.c#L113) | | `DS4_TEST_MIXED_ROUNDS` | integer 1..64; default 3 | Set the number of CUDA mixed-batch oracle rounds. | [tests/test_cuda_mixed_batch.c:115](tests/test_cuda_mixed_batch.c#L115) | | `DS4_TEST_MODEL` | nonempty GGUF path; tests/ds4_test.c defaults to ds4flash.gguf, while standalone model-backed CUDA/Metal fixtures generally require a supplied path and fail or skip when absent | Selects the target model shared by model-backed test binaries. | [tests/ds4_test.c:14](tests/ds4_test.c#L14) | -| `DS4_TEST_MPP_EQ_CASE` | comma-separated substring filter; unset/empty runs all cases; tokens are whitespace-trimmed and the filter is truncated to 255 bytes | Restricts Metal tensor-equivalence vectors to IDs containing at least one requested substring. | [tests/ds4_test.c:7525](tests/ds4_test.c#L7525) | +| `DS4_TEST_MPP_EQ_CASE` | comma-separated substring filter; unset/empty runs all cases; tokens are whitespace-trimmed and the filter is truncated to 255 bytes | Restricts Metal tensor-equivalence vectors to IDs containing at least one requested substring. | [tests/ds4_test.c:7805](tests/ds4_test.c#L7805) | | `DS4_TEST_MTP` | nonempty MTP support-GGUF path; unset/empty loads no MTP head; only the fast test engine uses it, with draft depth 4 | Enables the legacy MTP verify-depth regression; the test self-skips without this model. | [tests/ds4_test.c:104](tests/ds4_test.c#L104) | | `DS4_TEST_Q4_STREAM_ITERS` | integer 1..10000; default 2 | Set measured iterations for the Metal Q4 stream oracle. | [tests/test_metal_q4_streams.c:734](tests/test_metal_q4_streams.c#L734) | | `DS4_TEST_Q4_STREAM_SOAK` | integer 1..100000; default 8 | Set bounded overlap-soak iterations for the Metal Q4 stream oracle. | [tests/test_metal_q4_streams.c:736](tests/test_metal_q4_streams.c#L736) | @@ -1310,7 +1310,7 @@ them as part of its own test contract. | `DS4_TEST_TP_MODE` | unset/empty: no TP; exact leader or worker selects that role; every other nonempty value fails; incompatible with SSD-streaming mode | Selects standalone, TP-leader, or TP-worker execution for the Metal session-batch oracle. | [tests/test_metal_session_batch.c:168](tests/test_metal_session_batch.c#L168) | | `DS4_TEST_TP_PORT` | strict full decimal integer 1..65535; unset/empty defaults to 19452 | Sets the listen/connect port shared by Metal session-batch TP leader and worker. | [tests/test_metal_session_batch.c:63](tests/test_metal_session_batch.c#L63) | | `DS4_TEST_TP_TRANSPORT` | unset/empty/auto selects automatic transport; exact tcp or rdma selects that transport; other values fail | Chooses the TP transport for Metal session-batch leader/worker tests. | [tests/test_metal_session_batch.c:52](tests/test_metal_session_batch.c#L52) | -| `DS4_TEST_VECTOR_FILE` | nonempty readable vector path; unset/empty defaults to tests/test-vectors/flash-0731/official.vec | Selects the official fixture used by log-probability and Metal tensor-equivalence tests. | [tests/ds4_test.c:6942](tests/ds4_test.c#L6942) | +| `DS4_TEST_VECTOR_FILE` | nonempty readable vector path; unset/empty defaults to tests/test-vectors/flash-0731/official.vec | Selects the official fixture used by log-probability and Metal tensor-equivalence tests. | [tests/ds4_test.c:7222](tests/ds4_test.c#L7222) | | `PROTO_Q8_DEBUG` | presence diagnostic; unset: summary only; any defined value including empty or 0 prints detailed error structure after a Q8 parity failure | Dumps bad-element tile and row/column histograms for the CUDA Q8 prototype when parity fails. | [cuda/mmq/test/proto_gemm_dense_q8_d2r.cu:647](cuda/mmq/test/proto_gemm_dense_q8_d2r.cu#L647) | ### Test fixture scripts diff --git a/metal/moe.metal b/metal/moe.metal index a5903df96..4860d1929 100644 --- a/metal/moe.metal +++ b/metal/moe.metal @@ -3984,6 +3984,7 @@ kernel void kernel_dsv4_attn_out_low_q8_0_f32( #define DS4_ATTN_OUT_LOW_Q8_STATIC_K 4096 #define DS4_ATTN_OUT_LOW_Q8_STATIC_ROWS 1024 +#define DS4_ATTN_OUT_LOW_Q8_STATIC_GROUPS 8u #define DS4_ATTN_OUT_LOW_Q8_STATIC_BLOCKS 128 #define DS4_ATTN_OUT_LOW_Q8_STATIC_ROW_BYTES 4352 #define DS4_ATTN_OUT_LOW_Q8_STATIC_GROUP_BYTES 4456448 @@ -4001,14 +4002,16 @@ static inline void ds4_attn_out_low_q8_static_impl( constexpr short NQ = 8; constexpr short NSG = 4; - const uint group = tgpig.z; + // The z grid is flattened as pair = token * groups + group. + const uint pair = tgpig.z; + const uint group = pair % DS4_ATTN_OUT_LOW_Q8_STATIC_GROUPS; const int r0 = (int)tgpig.x * NR0; device const char *src0 = src0s + (uint64_t)group * DS4_ATTN_OUT_LOW_Q8_STATIC_GROUP_BYTES; device const float *y = (device const float *)(src1 + - (uint64_t)group * DS4_ATTN_OUT_LOW_Q8_STATIC_K * sizeof(float)); + (uint64_t)pair * DS4_ATTN_OUT_LOW_Q8_STATIC_K * sizeof(float)); device float *out = (device float *)(dst + - (uint64_t)group * DS4_ATTN_OUT_LOW_Q8_STATIC_ROWS * sizeof(float)); + (uint64_t)pair * DS4_ATTN_OUT_LOW_Q8_STATIC_ROWS * sizeof(float)); device const block_q8_0 *ax[NR0]; FOR_UNROLL (short row = 0; row < NR0; ++row) { diff --git a/scripts/environment_variables.tsv b/scripts/environment_variables.tsv index b4b68d609..a7335c852 100644 --- a/scripts/environment_variables.tsv +++ b/scripts/environment_variables.tsv @@ -1092,17 +1092,17 @@ script/server-wrapper DS4_SERVER_PORT unset/empty defaults to 8000; otherwise pa script/server-wrapper DS4_START_TIMEOUT positive decimal integer with no leading zero; unset/empty defaults to 180; invalid values fail when starting Sets how many seconds detached startup waits for the lock owner and listening log marker. run-nvidia-tp-server.sh:17 script/server-wrapper DS4_STOP_TIMEOUT positive decimal integer with no leading zero; unset/empty defaults to 120; invalid values fail when stopping Sets how many seconds graceful stop waits after SIGTERM before failing. run-nvidia-tp-server.sh:18 test-only DS4_CUDA_TOPK_REGRESSION_SEC positive floating-point seconds; default 2.0; invalid or nonpositive input restores the default Set the CUDA large-top-k elapsed-time regression limit. tests/cuda_long_context_smoke.c:72 -test-only DS4_METAL_MOE_TILE_MAX cleanup-only historical spelling; no production consumer exists, so setting it has no runtime effect Clears a legacy Metal MoE tile override while preparing the test environment. tests/ds4_test.c:7236 +test-only DS4_METAL_MOE_TILE_MAX cleanup-only historical spelling; no production consumer exists, so setting it has no runtime effect Clears a legacy Metal MoE tile override while preparing the test environment. tests/ds4_test.c:7516 test-only DS4_ROCM_ENABLE_Q4_PREFILL_TILE8 cleanup-only legacy spelling; no runtime or test reader; setting it has no effect Remove a stale opt-in name while preparing ROCm Q4 test cases; TILE8 is automatic. tests/test_rocm_q4_dense_pair.cpp:58 test-only DS4_TEST_ALLOW_FALLBACK presence flag; unset: native mixed path required; any defined value including empty or 0 permits exactly the serialized-fallback counter outcome Lets the CUDA mixed prefill/decode oracle accept serialized fallback while retaining bit-exact logit comparisons. tests/test_cuda_mixed_batch.c:204 test-only DS4_TEST_BACKEND exact cpu selects CPU; every other value, including unset/empty, selects Metal on Apple and CUDA elsewhere Chooses the backend used by model-backed tests in tests/ds4_test.c. tests/ds4_test.c:91 test-only DS4_TEST_BATCH_ONLY presence flag; unset: run batched and isolated-control phases; any defined value including empty or 0 stops after the batched archive/hash phase Runs only the CUDA session-batch phase and skips replay against isolated control sessions. tests/test_cuda_session_batch.c:288 test-only DS4_TEST_CONTEXT CUDA session/mixed fixtures default to 1024 and require 1024..65536; mixed-batch uses strict full decimal parsing, while session-batch uses atoi and therefore accepts numeric prefixes Sets the context and placement hint for the CUDA session-batch and mixed prefill/decode oracles. tests/test_cuda_session_batch.c:125 -test-only DS4_TEST_DSPARK nonempty DSpark support-GGUF path; unset/empty skips the DSpark verify-depth test Loads the DSpark support model for teacher-forced verification of committed speculative tokens. tests/ds4_test.c:8283 +test-only DS4_TEST_DSPARK nonempty DSpark support-GGUF path; unset/empty skips the DSpark verify-depth test Loads the DSpark support model for teacher-forced verification of committed speculative tokens. tests/ds4_test.c:8563 test-only DS4_TEST_GPU_DEVICES GPU device-list string parsed with the normal auto-VRAM parser; unset/empty defaults to 0,2,4,6,1,3,5,7; parse failure is fatal Selects and orders the CUDA TP/EP devices used by the mixed prefill/decode oracle. tests/test_cuda_mixed_batch.c:122 -test-only DS4_TEST_LOCAL_GOLDEN_FILE nonempty readable fixture path; unset/empty defaults to tests/test-vectors/flash-0731/local-golden.vec Selects the local-golden vector file used for model-logit regression checks. tests/ds4_test.c:7221 -test-only DS4_TEST_LOGPROB_AUTO_METAL presence flag; unset forces DS4_METAL_DISABLE_METAL4=1; any presence including empty or 0 removes that rollback and permits automatic Metal selection Runs official log-probability vectors with automatic Metal-path selection instead of the fixed pre-Metal4 baseline. tests/ds4_test.c:6955 -test-only DS4_TEST_LONG_PROMPT nonempty readable prompt-file path; unset/empty defaults to tests/long_context_story_prompt.txt Selects the rendered story prompt for the long-context fact-recall test. tests/ds4_test.c:6690 +test-only DS4_TEST_LOCAL_GOLDEN_FILE nonempty readable fixture path; unset/empty defaults to tests/test-vectors/flash-0731/local-golden.vec Selects the local-golden vector file used for model-logit regression checks. tests/ds4_test.c:7501 +test-only DS4_TEST_LOGPROB_AUTO_METAL presence flag; unset forces DS4_METAL_DISABLE_METAL4=1; any presence including empty or 0 removes that rollback and permits automatic Metal selection Runs official log-probability vectors with automatic Metal-path selection instead of the fixed pre-Metal4 baseline. tests/ds4_test.c:7235 +test-only DS4_TEST_LONG_PROMPT nonempty readable prompt-file path; unset/empty defaults to tests/long_context_story_prompt.txt Selects the rendered story prompt for the long-context fact-recall test. tests/ds4_test.c:6970 test-only DS4_TEST_LONG_WORDS atoi integer; unset/empty/nonnumeric defaults to 0; valid range is 0..DS4_TEST_CONTEXT-128 and numeric prefixes are accepted Adds repeated words to alternating CUDA session-batch prompts to exercise long-prefill rows. tests/test_cuda_session_batch.c:135 test-only DS4_TEST_METAL_EXACTN_BATCH_HEAD nonempty value other than exact 0 enables; unset/empty/0 disables; false/off also enable Enables the Metal exact-N batch-head path and requires its attempt/use counters for every eligible oracle case. tests/test_metal_exactn_oracle.c:401 test-only DS4_TEST_METAL_EXACTN_ORACLE presence flag compiled only with DS4_TEST_HOOKS; absent from normal production builds Force allocation of the exact-N Metal verifier/oracle workspace in tests. ds4.c:61991 @@ -1110,7 +1110,7 @@ test-only DS4_TEST_MIXED_INITIAL integer 128..context-1; default 128 Set the ini test-only DS4_TEST_MIXED_QUANTUM integer 1..context-1; default 128 Set the number of prompt tokens added per CUDA mixed-batch round. tests/test_cuda_mixed_batch.c:113 test-only DS4_TEST_MIXED_ROUNDS integer 1..64; default 3 Set the number of CUDA mixed-batch oracle rounds. tests/test_cuda_mixed_batch.c:115 test-only DS4_TEST_MODEL nonempty GGUF path; tests/ds4_test.c defaults to ds4flash.gguf, while standalone model-backed CUDA/Metal fixtures generally require a supplied path and fail or skip when absent Selects the target model shared by model-backed test binaries. tests/ds4_test.c:14 -test-only DS4_TEST_MPP_EQ_CASE comma-separated substring filter; unset/empty runs all cases; tokens are whitespace-trimmed and the filter is truncated to 255 bytes Restricts Metal tensor-equivalence vectors to IDs containing at least one requested substring. tests/ds4_test.c:7525 +test-only DS4_TEST_MPP_EQ_CASE comma-separated substring filter; unset/empty runs all cases; tokens are whitespace-trimmed and the filter is truncated to 255 bytes Restricts Metal tensor-equivalence vectors to IDs containing at least one requested substring. tests/ds4_test.c:7805 test-only DS4_TEST_MTP nonempty MTP support-GGUF path; unset/empty loads no MTP head; only the fast test engine uses it, with draft depth 4 Enables the legacy MTP verify-depth regression; the test self-skips without this model. tests/ds4_test.c:104 test-only DS4_TEST_Q4_STREAM_ITERS integer 1..10000; default 2 Set measured iterations for the Metal Q4 stream oracle. tests/test_metal_q4_streams.c:734 test-only DS4_TEST_Q4_STREAM_SOAK integer 1..100000; default 8 Set bounded overlap-soak iterations for the Metal Q4 stream oracle. tests/test_metal_q4_streams.c:736 @@ -1137,7 +1137,7 @@ test-only DS4_TEST_TP_LISTEN_HOST nonempty host string; unset/empty defaults to test-only DS4_TEST_TP_MODE unset/empty: no TP; exact leader or worker selects that role; every other nonempty value fails; incompatible with SSD-streaming mode Selects standalone, TP-leader, or TP-worker execution for the Metal session-batch oracle. tests/test_metal_session_batch.c:168 test-only DS4_TEST_TP_PORT strict full decimal integer 1..65535; unset/empty defaults to 19452 Sets the listen/connect port shared by Metal session-batch TP leader and worker. tests/test_metal_session_batch.c:63 test-only DS4_TEST_TP_TRANSPORT unset/empty/auto selects automatic transport; exact tcp or rdma selects that transport; other values fail Chooses the TP transport for Metal session-batch leader/worker tests. tests/test_metal_session_batch.c:52 -test-only DS4_TEST_VECTOR_FILE nonempty readable vector path; unset/empty defaults to tests/test-vectors/flash-0731/official.vec Selects the official fixture used by log-probability and Metal tensor-equivalence tests. tests/ds4_test.c:6942 +test-only DS4_TEST_VECTOR_FILE nonempty readable vector path; unset/empty defaults to tests/test-vectors/flash-0731/official.vec Selects the official fixture used by log-probability and Metal tensor-equivalence tests. tests/ds4_test.c:7222 test-only PROTO_Q8_DEBUG presence diagnostic; unset: summary only; any defined value including empty or 0 prints detailed error structure after a Q8 parity failure Dumps bad-element tile and row/column histograms for the CUDA Q8 prototype when parity fails. cuda/mmq/test/proto_gemm_dense_q8_d2r.cu:647 test-script DEEPSEEK_API_KEY secret string; required Authenticate official test-vector fetch. tests/test-vectors/fetch_official_vectors.py:236 test-script DS4_BIN executable path; unset/empty defaults to ./ds4; the Q4 matrix requires it executable, the DSpark fixture skips if missing, and the GLM smoke lets command failure fail the test Selects the ds4 binary launched by model-backed shell test fixtures. tests/cuda_q4_gb10_fast_matrix.sh:47 diff --git a/tests/ds4_test.c b/tests/ds4_test.c index 1ffac1c9e..7dd2cf8df 100644 --- a/tests/ds4_test.c +++ b/tests/ds4_test.c @@ -1165,6 +1165,285 @@ static void test_metal_q8_0_decode_rows_exact(void) { free(weights_raw); } +static void test_fill_q8_0_constant_weights(uint8_t *weights, + uint32_t in_dim, + uint32_t out_dim, + int8_t quant) { + const uint32_t blocks = in_dim / 32u; + const uint64_t row_bytes = (uint64_t)blocks * 34u; + const uint16_t scale_bits = test_float_to_f16(1.0f / 256.0f); + for (uint32_t row = 0; row < out_dim; row++) { + uint8_t *dst = weights + (uint64_t)row * row_bytes; + for (uint32_t block = 0; block < blocks; block++) { + memcpy(dst + (uint64_t)block * 34u, + &scale_bits, + sizeof(scale_bits)); + memset(dst + (uint64_t)block * 34u + 2u, + (unsigned char)quant, + 32u); + } + } +} + +static bool test_metal_q8_attention_output_static_batch_exact_case( + uint32_t n_tokens) { + const int failures_before = test_failures; + /* + * The production AProjQ8 static kernel receives a flattened z coordinate: + * + * pair = token * n_groups + group + * + * Only pair % n_groups may select Woa. Keep a second, sign-inverted Woa + * immediately after the real one so the old pair-as-group bug is a safe, + * deterministic wrong read for token 1 instead of an out-of-bounds Metal + * access. The public batch API also runs the small Q8 output projection; + * both its low intermediate and final output must match the generic direct + * kernel bit for bit. + */ + const uint32_t group_dim = 4096u; + const uint32_t rank = 1024u; + const uint32_t n_groups = 8u; + const uint32_t low_dim = n_groups * rank; + const uint32_t out_dim = 32u; + const uint32_t alloc_tokens = n_tokens + 1u; + const uint64_t page = (uint64_t)getpagesize(); + const uint64_t row_a_bytes = (uint64_t)(group_dim / 32u) * 34u; + const uint64_t group_a_bytes = (uint64_t)rank * row_a_bytes; + const uint64_t out_a_bytes = (uint64_t)n_groups * group_a_bytes; + const uint64_t shadow_a_offset = out_a_bytes; + const uint64_t out_b_offset = 2u * out_a_bytes; + const uint64_t row_b_bytes = (uint64_t)(low_dim / 32u) * 34u; + const uint64_t out_b_bytes = (uint64_t)out_dim * row_b_bytes; + const uint64_t model_bytes = + test_round_up_u64(out_b_offset + out_b_bytes, page); + const uint64_t heads_bytes = + (uint64_t)alloc_tokens * n_groups * group_dim * sizeof(float); + const uint64_t low_bytes = + (uint64_t)alloc_tokens * low_dim * sizeof(float); + const uint64_t out_bytes = + (uint64_t)alloc_tokens * out_dim * sizeof(float); + const uint64_t active_low_bytes = + (uint64_t)n_tokens * low_dim * sizeof(float); + const uint64_t active_out_bytes = + (uint64_t)n_tokens * out_dim * sizeof(float); + const char *disable_direct_env = + "DS4_METAL_DISABLE_ATTN_OUT_LOW_DIRECT"; + const char *disable_ports_env = + "DS4_METAL_DISABLE_PRE_M5_DECODE_PORTS"; + const char *disable_static_env = + "DS4_METAL_DISABLE_PRE_M5_ATTN_OUT_LOW_Q8_STATIC"; + + _Static_assert(4096u / 32u * 34u == 4352u, + "production AProjQ8 row size changed"); + _Static_assert(8u * 1024u * 4352u == 35651584u, + "production AProjQ8 Woa size changed"); + + char *saved_disable_direct = test_save_env(disable_direct_env); + char *saved_disable_ports = test_save_env(disable_ports_env); + char *saved_disable_static = test_save_env(disable_static_env); + void *model_raw = NULL; + float *heads_host = NULL; + float *reference_low_host = NULL; + float *candidate_low_host = NULL; + float *reference_out_host = NULL; + float *candidate_out_host = NULL; + ds4_gpu_tensor *heads = NULL; + ds4_gpu_tensor *reference_low = NULL; + ds4_gpu_tensor *candidate_low = NULL; + ds4_gpu_tensor *reference_out = NULL; + ds4_gpu_tensor *candidate_out = NULL; + ds4_gpu_tensor *group_tmp = NULL; + ds4_gpu_tensor *low_tmp = NULL; + + TEST_ASSERT(row_a_bytes == 4352u); + TEST_ASSERT(out_a_bytes == 35651584u); + TEST_ASSERT(posix_memalign( + &model_raw, (size_t)page, (size_t)model_bytes) == 0); + if (!model_raw) goto cleanup; + memset(model_raw, 0, (size_t)model_bytes); + for (uint32_t group = 0; group < n_groups; group++) { + test_fill_q8_0_constant_weights( + (uint8_t *)model_raw + (uint64_t)group * group_a_bytes, + group_dim, + rank, + (int8_t)(group + 1u)); + test_fill_q8_0_constant_weights( + (uint8_t *)model_raw + shadow_a_offset + + (uint64_t)group * group_a_bytes, + group_dim, + rank, + (int8_t)-(int8_t)(group + 1u)); + } + test_fill_q8_0_constant_weights( + (uint8_t *)model_raw + out_b_offset, + low_dim, + out_dim, + 1); + + heads_host = malloc((size_t)heads_bytes); + reference_low_host = malloc((size_t)low_bytes); + candidate_low_host = malloc((size_t)low_bytes); + reference_out_host = malloc((size_t)out_bytes); + candidate_out_host = malloc((size_t)out_bytes); + heads = ds4_gpu_tensor_alloc(heads_bytes); + reference_low = ds4_gpu_tensor_alloc(low_bytes); + candidate_low = ds4_gpu_tensor_alloc(low_bytes); + reference_out = ds4_gpu_tensor_alloc(out_bytes); + candidate_out = ds4_gpu_tensor_alloc(out_bytes); + group_tmp = ds4_gpu_tensor_alloc( + (uint64_t)n_tokens * group_dim * sizeof(float)); + low_tmp = ds4_gpu_tensor_alloc( + (uint64_t)n_tokens * rank * sizeof(float)); + TEST_ASSERT(heads_host && reference_low_host && candidate_low_host && + reference_out_host && candidate_out_host && heads && + reference_low && candidate_low && reference_out && + candidate_out && group_tmp && low_tmp); + if (!heads_host || !reference_low_host || !candidate_low_host || + !reference_out_host || !candidate_out_host || !heads || + !reference_low || !candidate_low || !reference_out || + !candidate_out || !group_tmp || !low_tmp) { + goto cleanup; + } + + for (uint64_t i = 0; i < heads_bytes / sizeof(float); i++) { + const uint32_t token = (uint32_t)(i / ((uint64_t)n_groups * group_dim)); + const uint32_t key = + (uint32_t)i * 17u + token * 131u + ((uint32_t)i >> 4u); + heads_host[i] = (float)(1u + key % 13u) / 128.0f; + } + memset(reference_low_host, 0xa5, (size_t)low_bytes); + memset(candidate_low_host, 0xa5, (size_t)low_bytes); + memset(reference_out_host, 0xa5, (size_t)out_bytes); + memset(candidate_out_host, 0xa5, (size_t)out_bytes); + TEST_ASSERT(ds4_gpu_tensor_write( + heads, 0, heads_host, heads_bytes) != 0); + TEST_ASSERT(ds4_gpu_tensor_write( + reference_low, 0, reference_low_host, low_bytes) != 0); + TEST_ASSERT(ds4_gpu_tensor_write( + candidate_low, 0, candidate_low_host, low_bytes) != 0); + TEST_ASSERT(ds4_gpu_tensor_write( + reference_out, 0, reference_out_host, out_bytes) != 0); + TEST_ASSERT(ds4_gpu_tensor_write( + candidate_out, 0, candidate_out_host, out_bytes) != 0); + TEST_ASSERT(ds4_gpu_set_model_map(model_raw, model_bytes) != 0); + ds4_gpu_set_quality(false); + + TEST_ASSERT(unsetenv(disable_direct_env) == 0); + TEST_ASSERT(unsetenv(disable_ports_env) == 0); + TEST_ASSERT(unsetenv(disable_static_env) == 0); + ds4_gpu_test_set_flags(DS4_GPU_TEST_ATTN_OUT_LOW_Q8_STATIC); + TEST_ASSERT(ds4_gpu_attention_output_q8_batch_tensor( + candidate_out, + candidate_low, + group_tmp, + low_tmp, + model_raw, + model_bytes, + 0, + out_b_offset, + group_dim, + rank, + n_groups, + out_dim, + heads, + n_tokens) == 1); + + ds4_gpu_test_set_flags(0); + TEST_ASSERT(setenv(disable_static_env, "1", 1) == 0); + TEST_ASSERT(ds4_gpu_attention_output_q8_batch_tensor( + reference_out, + reference_low, + group_tmp, + low_tmp, + model_raw, + model_bytes, + 0, + out_b_offset, + group_dim, + rank, + n_groups, + out_dim, + heads, + n_tokens) == 1); + + TEST_ASSERT(ds4_gpu_tensor_read( + reference_low, 0, reference_low_host, low_bytes) != 0); + TEST_ASSERT(ds4_gpu_tensor_read( + candidate_low, 0, candidate_low_host, low_bytes) != 0); + TEST_ASSERT(ds4_gpu_tensor_read( + reference_out, 0, reference_out_host, out_bytes) != 0); + TEST_ASSERT(ds4_gpu_tensor_read( + candidate_out, 0, candidate_out_host, out_bytes) != 0); + + const size_t low_count = (size_t)n_tokens * low_dim; + const size_t out_count = (size_t)n_tokens * out_dim; + const test_float_compare_stats low_stats = test_compare_float_bits( + reference_low_host, candidate_low_host, low_count); + const test_float_compare_stats out_stats = test_compare_float_bits( + reference_out_host, candidate_out_host, out_count); + size_t tail_byte_mismatch = 0; + for (uint64_t i = active_low_bytes; i < low_bytes; i++) { + if (((const uint8_t *)reference_low_host)[i] != 0xa5u) { + tail_byte_mismatch++; + } + if (((const uint8_t *)candidate_low_host)[i] != 0xa5u) { + tail_byte_mismatch++; + } + } + for (uint64_t i = active_out_bytes; i < out_bytes; i++) { + if (((const uint8_t *)reference_out_host)[i] != 0xa5u) { + tail_byte_mismatch++; + } + if (((const uint8_t *)candidate_out_host)[i] != 0xa5u) { + tail_byte_mismatch++; + } + } + fprintf(stderr, + "ds4-test: Metal Q8 static attention-output exact-%u " + "low=%zu/%zu max_ulp=%u max_abs=%g " + "out=%zu/%zu max_ulp=%u max_abs=%g tail_bytes=%zu\n", + n_tokens, + low_stats.mismatch_count, + low_count, + low_stats.max_ulp, + low_stats.max_abs, + out_stats.mismatch_count, + out_count, + out_stats.max_ulp, + out_stats.max_abs, + tail_byte_mismatch); + TEST_ASSERT(low_stats.mismatch_count == 0 && low_stats.max_ulp == 0); + TEST_ASSERT(out_stats.mismatch_count == 0 && out_stats.max_ulp == 0); + TEST_ASSERT(tail_byte_mismatch == 0); + +cleanup: + ds4_gpu_test_set_flags(0); + ds4_gpu_tensor_free(low_tmp); + ds4_gpu_tensor_free(group_tmp); + ds4_gpu_tensor_free(candidate_out); + ds4_gpu_tensor_free(reference_out); + ds4_gpu_tensor_free(candidate_low); + ds4_gpu_tensor_free(reference_low); + ds4_gpu_tensor_free(heads); + free(candidate_out_host); + free(reference_out_host); + free(candidate_low_host); + free(reference_low_host); + free(heads_host); + free(model_raw); + test_restore_env(disable_static_env, saved_disable_static); + test_restore_env(disable_ports_env, saved_disable_ports); + test_restore_env(disable_direct_env, saved_disable_direct); + return test_failures == failures_before; +} + +static void test_metal_q8_attention_output_static_batch_exact(void) { + /* The shadow Woa makes the old bug safe at N=2; stop there on failure. */ + if (test_metal_q8_attention_output_static_batch_exact_case(2u)) { + (void)test_metal_q8_attention_output_static_batch_exact_case(31u); + } +} + static void test_metal_q4_attention_output_tiny_batch_exact_case( uint32_t out_b_type) { /* @@ -6629,6 +6908,7 @@ static void test_metal_kernel_group(void) { test_metal_q8_0_decode_pair_exact(); #if defined(__APPLE__) test_metal_q8_0_decode_rows_exact(); + test_metal_q8_attention_output_static_batch_exact(); test_metal_q4_attention_output_tiny_batch_exact(); test_metal_dspark_device_proposer_q8(); test_metal_f16_compressor_pair_state_store_exact(); From ec2f27c4b4df5a71075549c82e3e788559f65ed8 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Mon, 24 Aug 2026 10:14:37 +0200 Subject: [PATCH 085/189] cuda: centralize Q8 fold registry bridge ABI --- ENVIRONMENT_VARIABLES.md | 44 +++++++++++++++---------------- Makefile | 2 +- cuda/mmq/ds4_ggml_stubs.cu | 1 + cuda/mmq/ds4_mmq.cu | 10 ------- cuda/mmq/ds4_mmq.h | 12 +++++++++ scripts/environment_variables.tsv | 44 +++++++++++++++---------------- 6 files changed, 58 insertions(+), 55 deletions(-) diff --git a/ENVIRONMENT_VARIABLES.md b/ENVIRONMENT_VARIABLES.md index 029c12dbf..4beac3ee0 100644 --- a/ENVIRONMENT_VARIABLES.md +++ b/ENVIRONMENT_VARIABLES.md @@ -615,8 +615,8 @@ and **19 tool/wrapper entries**. | `DS4_CUDA_ENABLE_HC_NORM_MIX_FUSE` | nonempty opt-in, default off; only exact 0 disables; the F32/F16 activation mode follows the selected standalone matmul path; disable/serial/alternate flags can veto | Enable and select the fused HC RMSNorm-plus-mix one-token implementation. | [ds4_cuda.cu:20902](ds4_cuda.cu#L20902) | | `DS4_CUDA_ENABLE_IQ2_XXS_SSD_PREFILL_MMQ` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Enable the CUDA IQ2 XXS SSD prefill MMQ experimental path. | [ds4_cuda.cu:4625](ds4_cuda.cu#L4625) | | `DS4_CUDA_ENABLE_Q4_ATTN_OUT_HC_FUSE` | value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on | Opt in to the fused Q4 attention-output/HC expansion path. | [ds4_cuda.cu:37375](ds4_cuda.cu#L37375) | -| `DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_BATCH` | value-aware opt-in, default off; nonempty value other than exact 0 enables; rollback wins | Enable flattened grouped attention-A MMQ for two-to-eight-token GB10 batches. | [cuda/mmq/ds4_mmq.cu:4313](cuda/mmq/ds4_mmq.cu#L4313) | -| `DS4_CUDA_ENABLE_Q4_K1024_PERSISTENT` | presence flag, default off; any defined value including 0 requests the path; rollback wins | Enable the GB10 persistent-CTA kernel for M=32768, N=1, K=1024 Q4. | [cuda/mmq/ds4_mmq.cu:3915](cuda/mmq/ds4_mmq.cu#L3915) | +| `DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_BATCH` | value-aware opt-in, default off; nonempty value other than exact 0 enables; rollback wins | Enable flattened grouped attention-A MMQ for two-to-eight-token GB10 batches. | [cuda/mmq/ds4_mmq.cu:4303](cuda/mmq/ds4_mmq.cu#L4303) | +| `DS4_CUDA_ENABLE_Q4_K1024_PERSISTENT` | presence flag, default off; any defined value including 0 requests the path; rollback wins | Enable the GB10 persistent-CTA kernel for M=32768, N=1, K=1024 Q4. | [cuda/mmq/ds4_mmq.cu:3905](cuda/mmq/ds4_mmq.cu#L3905) | | `DS4_CUDA_ENABLE_Q8_FOLD` | strict flag, default off; only exact value 1 enables; overridden by DS4_CUDA_NO_Q8_FOLD | Enable one-shot producer-to-consumer reuse of freshly quantized Q8_1 data. | [ds4_cuda.cu:785](ds4_cuda.cu#L785) | | `DS4_CUDA_ENABLE_STREAMING_EXPERT_PERSISTENT_CACHE` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Enable streaming expert persistent cache in CUDA SSD streaming. | [ds4_cuda.cu:4111](ds4_cuda.cu#L4111) | | `DS4_CUDA_ENABLE_STREAMING_SELECTED_BATCH_IO` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Enable streaming selected batch I/O in CUDA SSD streaming. | [ds4_cuda.cu:4732](ds4_cuda.cu#L4732) | @@ -756,16 +756,16 @@ and **19 tool/wrapper entries**. | `DS4_CUDA_NO_IQ2_XXS_SSD_PREFILL_MMQ` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Disable the CUDA IQ2 XXS SSD prefill MMQ optimization/path. | [ds4_cuda.cu:4629](ds4_cuda.cu#L4629) | | `DS4_CUDA_NO_MODEL_COPY` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA model copy optimization/path. | [ds4_cuda.cu:6472](ds4_cuda.cu#L6472) | | `DS4_CUDA_NO_MODEL_PREFETCH` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA model prefetch optimization/path. | [ds4_cuda.cu:2739](ds4_cuda.cu#L2739) | -| `DS4_CUDA_NO_MOE_DEDUP` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA MoE dedup optimization/path. | [cuda/mmq/ds4_mmq.cu:6182](cuda/mmq/ds4_mmq.cu#L6182) | +| `DS4_CUDA_NO_MOE_DEDUP` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA MoE dedup optimization/path. | [cuda/mmq/ds4_mmq.cu:6172](cuda/mmq/ds4_mmq.cu#L6172) | | `DS4_CUDA_NO_ORDERED_F16_MATMUL` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the ordered F16 matmul CUDA F16 path. | [ds4_cuda.cu:20811](ds4_cuda.cu#L20811) | | `DS4_CUDA_NO_PARALLEL_ROUTER_SELECT` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA parallel router select optimization/path. | [ds4_cuda.cu:24491](ds4_cuda.cu#L24491) | -| `DS4_CUDA_NO_Q4_DENSE_SCRATCH` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q4 dense scratch CUDA Q4 optimization. | [cuda/mmq/ds4_mmq.cu:3996](cuda/mmq/ds4_mmq.cu#L3996) | -| `DS4_CUDA_NO_Q4_GB10_FAST` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the GB10-specific Q4 fast-path family. | [cuda/mmq/ds4_mmq.cu:3918](cuda/mmq/ds4_mmq.cu#L3918) | -| `DS4_CUDA_NO_Q4_GROUPED_ATTN_A` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q4 grouped attn a CUDA Q4 optimization. | [cuda/mmq/ds4_mmq.cu:4305](cuda/mmq/ds4_mmq.cu#L4305) | -| `DS4_CUDA_NO_Q4_GROUPED_ATTN_A_BATCH` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q4 grouped attn a batch CUDA Q4 optimization. | [cuda/mmq/ds4_mmq.cu:4315](cuda/mmq/ds4_mmq.cu#L4315) | -| `DS4_CUDA_NO_Q4_K1024_PERSISTENT` | presence kill switch, default off; any defined value including 0 disables | Disable the Q4 K1024 persistent CUDA Q4 optimization. | [cuda/mmq/ds4_mmq.cu:3917](cuda/mmq/ds4_mmq.cu#L3917) | -| `DS4_CUDA_NO_Q8_ALIGNED_DENSE_SCRATCH` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q8 aligned dense scratch CUDA Q8 optimization. | [cuda/mmq/ds4_mmq.cu:5588](cuda/mmq/ds4_mmq.cu#L5588) | -| `DS4_CUDA_NO_Q8_ALIGNED_PERSISTENT` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q8 aligned persistent CUDA Q8 optimization. | [cuda/mmq/ds4_mmq.cu:5420](cuda/mmq/ds4_mmq.cu#L5420) | +| `DS4_CUDA_NO_Q4_DENSE_SCRATCH` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q4 dense scratch CUDA Q4 optimization. | [cuda/mmq/ds4_mmq.cu:3986](cuda/mmq/ds4_mmq.cu#L3986) | +| `DS4_CUDA_NO_Q4_GB10_FAST` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the GB10-specific Q4 fast-path family. | [cuda/mmq/ds4_mmq.cu:3908](cuda/mmq/ds4_mmq.cu#L3908) | +| `DS4_CUDA_NO_Q4_GROUPED_ATTN_A` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q4 grouped attn a CUDA Q4 optimization. | [cuda/mmq/ds4_mmq.cu:4295](cuda/mmq/ds4_mmq.cu#L4295) | +| `DS4_CUDA_NO_Q4_GROUPED_ATTN_A_BATCH` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q4 grouped attn a batch CUDA Q4 optimization. | [cuda/mmq/ds4_mmq.cu:4305](cuda/mmq/ds4_mmq.cu#L4305) | +| `DS4_CUDA_NO_Q4_K1024_PERSISTENT` | presence kill switch, default off; any defined value including 0 disables | Disable the Q4 K1024 persistent CUDA Q4 optimization. | [cuda/mmq/ds4_mmq.cu:3907](cuda/mmq/ds4_mmq.cu#L3907) | +| `DS4_CUDA_NO_Q8_ALIGNED_DENSE_SCRATCH` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q8 aligned dense scratch CUDA Q8 optimization. | [cuda/mmq/ds4_mmq.cu:5578](cuda/mmq/ds4_mmq.cu#L5578) | +| `DS4_CUDA_NO_Q8_ALIGNED_PERSISTENT` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q8 aligned persistent CUDA Q8 optimization. | [cuda/mmq/ds4_mmq.cu:5410](cuda/mmq/ds4_mmq.cu#L5410) | | `DS4_CUDA_NO_Q8_BATCH_EXACT_TOK2` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q8 batch exact tok2 CUDA Q8 optimization. | [ds4_cuda.cu:19852](ds4_cuda.cu#L19852) | | `DS4_CUDA_NO_Q8_BATCH_TOK4` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q8 batch tok4 CUDA Q8 optimization. | [ds4_cuda.cu:19805](ds4_cuda.cu#L19805) | | `DS4_CUDA_NO_Q8_BATCH_TOK8` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q8 batch tok8 CUDA Q8 optimization. | [ds4_cuda.cu:19788](ds4_cuda.cu#L19788) | @@ -812,15 +812,15 @@ and **19 tool/wrapper entries**. | `DS4_CUDA_Q4_ATTN_OUT_HC_ORACLE` | value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on | Compare fused Q4 attention-output/HC expansion with the canonical path and retain canonical output. | [ds4_cuda.cu:1475](ds4_cuda.cu#L1475) | | `DS4_CUDA_Q4_ATTN_OUT_HC_Q8K_EXPERIMENT` | value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on | Enable the experimental Q8_K-based Q4 attention-output/HC fusion. | [ds4_cuda.cu:37377](ds4_cuda.cu#L37377) | | `DS4_CUDA_Q4_GROUPED_ATTN_A_ORACLE` | value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on | Compare grouped attention-A against the canonical per-group result. | [ds4_cuda.cu:1477](ds4_cuda.cu#L1477) | -| `DS4_CUDA_Q4_K1024_PERSISTENT_ORACLE` | value-aware flag, default off; nonempty value other than exact 0 enables and implies candidate admission | Bitwise-compare the exact-shape persistent Q4 K1024 kernel with canonical MMVQ and retain canonical output. | [cuda/mmq/ds4_mmq.cu:3748](cuda/mmq/ds4_mmq.cu#L3748) | -| `DS4_CUDA_Q4_K1024_PERSISTENT_STATS` | value-aware flag, default off; nonempty value other than exact 0 enables | Print exact-shape persistent Q4 K1024 dispatch counters at exit. | [cuda/mmq/ds4_mmq.cu:3747](cuda/mmq/ds4_mmq.cu#L3747) | +| `DS4_CUDA_Q4_K1024_PERSISTENT_ORACLE` | value-aware flag, default off; nonempty value other than exact 0 enables and implies candidate admission | Bitwise-compare the exact-shape persistent Q4 K1024 kernel with canonical MMVQ and retain canonical output. | [cuda/mmq/ds4_mmq.cu:3738](cuda/mmq/ds4_mmq.cu#L3738) | +| `DS4_CUDA_Q4_K1024_PERSISTENT_STATS` | value-aware flag, default off; nonempty value other than exact 0 enables | Print exact-shape persistent Q4 K1024 dispatch counters at exit. | [cuda/mmq/ds4_mmq.cu:3737](cuda/mmq/ds4_mmq.cu#L3737) | | `DS4_CUDA_Q8_F16_ALL` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Control the Q8 F16 all CUDA quantized-matmul/cache optimization. | [ds4_cuda.cu:2358](ds4_cuda.cu#L2358) | | `DS4_CUDA_Q8_F16_CACHE_MB` | unsigned integer MiB, full-string parse; default unlimited; 0 disables this cache | Limit the selective Q8-to-F16 derived-weight cache. | [ds4_cuda.cu:2218](ds4_cuda.cu#L2218) | | `DS4_CUDA_Q8_F16_CACHE_RESERVE_MB` | unsigned integer MiB, full-string parse; default is VRAM-dependent (>=112 GiB: 512; >=40 GiB: max(768,1%); smaller: max(4096,5%)) | Reserve free VRAM when growing the selective Q8-to-F16 cache. | [ds4_cuda.cu:2224](ds4_cuda.cu#L2224) | | `DS4_CUDA_Q8_F32_ALL` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Control the Q8 F32 all CUDA quantized-matmul/cache optimization. | [ds4_cuda.cu:2412](ds4_cuda.cu#L2412) | | `DS4_CUDA_Q8_F32_LARGE` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Control the Q8 F32 large CUDA quantized-matmul/cache optimization. | [ds4_cuda.cu:2416](ds4_cuda.cu#L2416) | | `DS4_CUDA_Q8_F32_PRELOAD` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Control the Q8 F32 preload CUDA quantized-matmul/cache optimization. | [ds4_cuda.cu:8597](ds4_cuda.cu#L8597) | -| `DS4_CUDA_Q8_FOLD_ORACLE` | strict flag, default off; only exact value 1 enables | Compare folded Q8_1 bytes and consumer outputs against canonical work while retaining canonical results. | [cuda/mmq/ds4_mmq.cu:391](cuda/mmq/ds4_mmq.cu#L391) | +| `DS4_CUDA_Q8_FOLD_ORACLE` | strict flag, default off; only exact value 1 enables | Compare folded Q8_1 bytes and consumer outputs against canonical work while retaining canonical results. | [cuda/mmq/ds4_mmq.cu:381](cuda/mmq/ds4_mmq.cu#L381) | | `DS4_CUDA_Q8_HC_EXPAND_FUSED` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, other nonempty values force fused | Force the fused Q8 shared-down/HC expansion path. | [ds4_cuda.cu:2084](ds4_cuda.cu#L2084) | | `DS4_CUDA_Q8_HC_EXPAND_STATS` | false-like-aware flag, default off; 0/false/no/off is off, other nonempty values print report | Print Q8 shared-down/HC policy and dispatch counters at exit. | [ds4_cuda.cu:2090](ds4_cuda.cu#L2090) | | `DS4_CUDA_Q8_NO_ALIGNED` | value-aware kill switch, default off; nonempty value other than exact 0 disables aligned Q8 kernels | Disable aligned Q8 CUDA matmul kernels. | [ds4_cuda.cu:1021](ds4_cuda.cu#L1021) | @@ -829,7 +829,7 @@ and **19 tool/wrapper entries**. | `DS4_CUDA_Q_NORM_ROPE_FUSE` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control or tune the CUDA q norm rope fuse path. | [ds4.c:17418](ds4.c#L17418) | | `DS4_CUDA_REQUIRE_IQ2_XXS_SSD_PREFILL_MMQ` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Require the CUDA IQ2 XXS SSD prefill MMQ path; fail closed when unavailable. | [ds4_cuda.cu:4631](ds4_cuda.cu#L4631) | | `DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_BATCH` | value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on | Fail if grouped batched attention-A cannot be used. | [ds4_cuda.cu:40198](ds4_cuda.cu#L40198) | -| `DS4_CUDA_REQUIRE_Q4_K1024_PERSISTENT` | presence flag, default off; any defined value including 0 makes ineligible candidate fail closed | Fail when the exact Q4 K1024 persistent candidate is unavailable instead of using MMVQ. | [cuda/mmq/ds4_mmq.cu:3939](cuda/mmq/ds4_mmq.cu#L3939) | +| `DS4_CUDA_REQUIRE_Q4_K1024_PERSISTENT` | presence flag, default off; any defined value including 0 makes ineligible candidate fail closed | Fail when the exact Q4 K1024 persistent candidate is unavailable instead of using MMVQ. | [cuda/mmq/ds4_mmq.cu:3929](cuda/mmq/ds4_mmq.cu#L3929) | | `DS4_CUDA_REQUIRE_STREAMING_EXPERT_PERSISTENT_CACHE` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Require streaming expert persistent cache in CUDA SSD streaming; fail closed when unavailable. | [ds4_cuda.cu:4117](ds4_cuda.cu#L4117) | | `DS4_CUDA_REQUIRE_STREAMING_SELECTED_BATCH_IO` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Require streaming selected batch I/O in CUDA SSD streaming; fail closed when unavailable. | [ds4_cuda.cu:4738](ds4_cuda.cu#L4738) | | `DS4_CUDA_REQUIRE_STREAMING_SELECTED_EVENT_PIPELINE` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Require streaming selected event pipeline in CUDA SSD streaming; fail closed when unavailable. | [ds4_cuda.cu:4870](ds4_cuda.cu#L4870) | @@ -1179,15 +1179,15 @@ and **19 tool/wrapper entries**. | `DS4_LOCK_FILE` | path string; default /tmp/ds4.lock | Override the single-instance lock file. | [ds4.c:51999](ds4.c#L51999) | | `DS4_MMID_CASE1` | boolean-ish cached flag; default on; a value starting with 0 disables | Disable the single-expert MM-IDs specialized fast path for comparison. | [cuda/mmq/mmid.cu:290](cuda/mmq/mmid.cu#L290) | | `DS4_MMID_LARGE` | boolean-ish cached flag; default on; a value starting with 0 disables | Control the large-N global-memory MM-IDs path used beyond shared-memory capacity. | [cuda/mmq/mmid.cu:245](cuda/mmq/mmid.cu#L245) | -| `DS4_MMQ_D2R` | boolean-ish cached flag; default on; a value starting with 0 disables | Control the direct-to-register Q2_K MoE down path. | [cuda/mmq/ds4_mmq.cu:535](cuda/mmq/ds4_mmq.cu#L535) | -| `DS4_MMQ_D2R_IQ2` | boolean-ish cached flag; default on; a value starting with 0 disables | Control the direct-to-register IQ2 MoE gate/up path. | [cuda/mmq/ds4_mmq.cu:544](cuda/mmq/ds4_mmq.cu#L544) | -| `DS4_MMQ_D2R_MIN_COLS` | positive integer; default 1024; invalid or nonpositive input restores the default | Set the minimum output-column count for the MMQ direct-to-register path. | [cuda/mmq/ds4_mmq.cu:639](cuda/mmq/ds4_mmq.cu#L639) | +| `DS4_MMQ_D2R` | boolean-ish cached flag; default on; a value starting with 0 disables | Control the direct-to-register Q2_K MoE down path. | [cuda/mmq/ds4_mmq.cu:525](cuda/mmq/ds4_mmq.cu#L525) | +| `DS4_MMQ_D2R_IQ2` | boolean-ish cached flag; default on; a value starting with 0 disables | Control the direct-to-register IQ2 MoE gate/up path. | [cuda/mmq/ds4_mmq.cu:534](cuda/mmq/ds4_mmq.cu#L534) | +| `DS4_MMQ_D2R_MIN_COLS` | positive integer; default 1024; invalid or nonpositive input restores the default | Set the minimum output-column count for the MMQ direct-to-register path. | [cuda/mmq/ds4_mmq.cu:629](cuda/mmq/ds4_mmq.cu#L629) | | `DS4_MMQ_D2R_STATS` | exact 1 enables; unset or every other value disables; cached and synchronizes the stream | Print partial-tile fill telemetry for the direct-to-register MMQ kernels. | [cuda/mmq/ds4_mmq_d2r.cu:33](cuda/mmq/ds4_mmq_d2r.cu#L33) | | `DS4_MMQ_DENSE_D2R` | boolean-ish flag; default on; exact 0 disables | Control the eligible aligned-Q8 dense prefill direct-to-register path. | [ds4_cuda.cu:19567](ds4_cuda.cu#L19567) | -| `DS4_MMQ_NO_YIND` | presence rollback; unset keeps Y-indirect staging; any defined value including 0 disables it | Restore slot-gathered MoE gate/up activation quantization. | [cuda/mmq/ds4_mmq.cu:619](cuda/mmq/ds4_mmq.cu#L619) | -| `DS4_MMQ_OUT_MEMSET` | exact 1 enables; unset or every other value disables; cached | Restore blanket MMQ output-buffer zeroing for diagnostics. | [cuda/mmq/ds4_mmq.cu:562](cuda/mmq/ds4_mmq.cu#L562) | -| `DS4_MMQ_YBUF_MEMSET` | unset or 0 disables; 1 zero-fills; a value starting with p or P poison-fills with 0xFF | Control MMQ Q8_1 activation-staging initialization and its poison oracle. | [cuda/mmq/ds4_mmq.cu:588](cuda/mmq/ds4_mmq.cu#L588) | -| `DS4_MMQ_YIND_VERIFY` | presence diagnostic; unset is off; any defined value including 0 enables | Byte-compare Y-indirect and slot-gathered MoE activation buffers. | [cuda/mmq/ds4_mmq.cu:630](cuda/mmq/ds4_mmq.cu#L630) | +| `DS4_MMQ_NO_YIND` | presence rollback; unset keeps Y-indirect staging; any defined value including 0 disables it | Restore slot-gathered MoE gate/up activation quantization. | [cuda/mmq/ds4_mmq.cu:609](cuda/mmq/ds4_mmq.cu#L609) | +| `DS4_MMQ_OUT_MEMSET` | exact 1 enables; unset or every other value disables; cached | Restore blanket MMQ output-buffer zeroing for diagnostics. | [cuda/mmq/ds4_mmq.cu:552](cuda/mmq/ds4_mmq.cu#L552) | +| `DS4_MMQ_YBUF_MEMSET` | unset or 0 disables; 1 zero-fills; a value starting with p or P poison-fills with 0xFF | Control MMQ Q8_1 activation-staging initialization and its poison oracle. | [cuda/mmq/ds4_mmq.cu:578](cuda/mmq/ds4_mmq.cu#L578) | +| `DS4_MMQ_YIND_VERIFY` | presence diagnostic; unset is off; any defined value including 0 enables | Byte-compare Y-indirect and slot-gathered MoE activation buffers. | [cuda/mmq/ds4_mmq.cu:620](cuda/mmq/ds4_mmq.cu#L620) | | `DS4_MOE_RECORD_SELECTED_HOTLIST` | nonempty output path; unset=off | Record per-layer selected-expert hit counts to a Metal hotlist file. | [ds4_metal.m:1741](ds4_metal.m#L1741) | | `DS4_MOE_RECORD_SELECTED_HOTLIST_FRESH` | presence flag; only relevant with HOTLIST; overrides MERGE | Start the selected-expert hotlist from empty state. | [ds4_metal.m:1642](ds4_metal.m#L1642) | | `DS4_MOE_RECORD_SELECTED_HOTLIST_MERGE` | presence flag; active only when FRESH is absent | Merge an existing selected-expert hotlist before recording. | [ds4_metal.m:1641](ds4_metal.m#L1641) | @@ -1216,7 +1216,7 @@ and **19 tool/wrapper entries**. | `DS4_PREFILL_BATCH` | Nonempty value parsed by strtol without full-string validation; accepted range 1..4095, default 128 for unset/invalid/out-of-range values. It is used only when DS4_BATCHED_FFN selects full batched CPU FFN. | Set the token chunk size for layer_ffn_batch during CPU layer-major prefill. | [ds4.c:14414](ds4.c#L14414) | | `DS4_PREFILL_PROFILE_DETAIL` | presence flag; unset=off | Print detailed per-stage CPU prefill timing. | [ds4.c:12555](ds4.c#L12555) | | `DS4_PREFILL_PROFILE_TOKEN` | presence flag; effective within detailed prefill profiling | Print token-loop substage timings during CPU prefill. | [ds4.c:14130](ds4.c#L14130) | -| `DS4_Q8_FOLD_SELFTEST` | positive call budget; unset/empty disables; a nonempty value parsing to 1 or less selects 512 calls | Byte-check folded Q8_1 activations against a fresh quantization; synchronizes eager streams. | [cuda/mmq/ds4_mmq.cu:5963](cuda/mmq/ds4_mmq.cu#L5963) | +| `DS4_Q8_FOLD_SELFTEST` | positive call budget; unset/empty disables; a nonempty value parsing to 1 or less selects 512 calls | Byte-check folded Q8_1 activations against a fresh quantization; synchronizes eager streams. | [cuda/mmq/ds4_mmq.cu:5953](cuda/mmq/ds4_mmq.cu#L5953) | | `DS4_ROUTED_TOKEN_PARALLEL` | Pure presence flag that forces token-parallel routed MoE, even if DS4_NO_ROUTED_TOKEN_PARALLEL is also set. When unset, token parallelism is automatic for n_tok>=64 unless the NO flag is present; smaller batches use per-token routed MoE. | Choose token-parallel CPU routed-expert evaluation inside the default shared-batch FFN prefill path. | [ds4.c:12587](ds4.c#L12587) | | `DS4_SERVER_BATCH_LOG` | Pure presence flag read once when the decode worker starts; default off. Any defined value, including empty or "0", logs one record per coalesced decode batch with count, elapsed milliseconds and ok/error status. | Observe server-side decode coalescing size, latency and result without changing batching behavior. | [ds4_server.c:11090](ds4_server.c#L11090) | | `DS4_SERVER_DECODE_COALESCE_US` | integer 0..100000 microseconds; default 2000; 0 disables wait | Control server micro-batch coalescing delay. | [ds4_server.c:11069](ds4_server.c#L11069) | diff --git a/Makefile b/Makefile index 7b2cda6b8..03b3a72e1 100644 --- a/Makefile +++ b/Makefile @@ -543,7 +543,7 @@ ds4_cuda.o: ds4_cuda.cu ds4_gpu.h ds4_gpu_mgpu.h ds4_glm53_vision_gpu.cuh ds4_iq # Vendored mmq pieces (see cuda/mmq/VENDOR.md). ds4_mmq.cu transitively # pulls in mmq.cuh which has heavy template instantiation -- each piece # compiles in its own TU and links in. -cuda/mmq/ds4_ggml_stubs.o: cuda/mmq/ds4_ggml_stubs.cu cuda/mmq/ds4_ggml_stubs.h cuda/mmq/common.cuh +cuda/mmq/ds4_ggml_stubs.o: cuda/mmq/ds4_ggml_stubs.cu cuda/mmq/ds4_mmq.h cuda/mmq/ds4_ggml_stubs.h cuda/mmq/common.cuh $(NVCC) $(NVCCFLAGS) -std=c++17 $(MMQ_INCLUDES) -c -o $@ $< cuda/mmq/ds4_mmq.o: cuda/mmq/ds4_mmq.cu cuda/mmq/ds4_mmq.h cuda/mmq/ds4_mmq_d2r.cuh cuda/mmq/mmq.cuh cuda/mmq/common.cuh cuda/mmq/ds4_ggml_stubs.h cuda/mmq/quantize.cuh cuda/mmq/mmid.cuh cuda/mmq/vecdotq.cuh cuda/mmq/mma.cuh diff --git a/cuda/mmq/ds4_ggml_stubs.cu b/cuda/mmq/ds4_ggml_stubs.cu index 18f139bac..1a5df364e 100644 --- a/cuda/mmq/ds4_ggml_stubs.cu +++ b/cuda/mmq/ds4_ggml_stubs.cu @@ -7,6 +7,7 @@ // Phase 0: pool is plain cudaMallocAsync / cudaFreeAsync. Phase 4 swaps // in ds4's existing cuda_tmp_alloc slab allocator. +#include "ds4_mmq.h" #include "common.cuh" // pulls in ds4_ggml_stubs.h via redirect headers #if defined(GGML_USE_HIP) diff --git a/cuda/mmq/ds4_mmq.cu b/cuda/mmq/ds4_mmq.cu index 9eab07858..d2891c6de 100644 --- a/cuda/mmq/ds4_mmq.cu +++ b/cuda/mmq/ds4_mmq.cu @@ -370,16 +370,6 @@ extern "C" int ds4_mmq_q81_persistent_preflight_for_test( return arena ? 0 : -3; } -// M2-Inc2a: registry of producer-emitted q8_1 activations (ds4_cuda.cu). -// A hit returns canonical block_q8_1 codes for this exact activation -// pointer (bit-exact vs quantize_row_q8_1_cuda), letting the caller skip -// its quantize prelude. Only valid for single-token unpadded rows -// (ne10_padded == K); the registry itself guarantees freshness (slots are -// reset by the producing entry every layer and pops are one-shot). -extern "C" int ds4_cuda_q8_fold_take_q81(const void *src, uint64_t in_dim, - cudaStream_t stream, - const void **q81); - static uint64_t g_q8_fold_oracle_byte_calls; static uint64_t g_q8_fold_oracle_byte_mismatches; static uint64_t g_q8_fold_oracle_output_calls; diff --git a/cuda/mmq/ds4_mmq.h b/cuda/mmq/ds4_mmq.h index 9c3a45e0c..759ff8e52 100644 --- a/cuda/mmq/ds4_mmq.h +++ b/cuda/mmq/ds4_mmq.h @@ -34,6 +34,18 @@ extern "C" { int ds4_mmq_init(int device); void ds4_mmq_set_aligned_q81_scratch(void *ptr, size_t bytes); +// Producer-fold registry bridge implemented by the full CUDA runtime. +// A hit returns the canonical Q8_1 sidecar for this exact activation pointer +// and stream. It is valid only for single-token unpadded rows; registry slots +// are refreshed by each producer layer and consumed once. The standalone MMQ +// library provides a weak, fail-closed miss so its tests do not need to link +// ds4_cuda.cu; full ds4 overrides it with the stream-aware implementation. +int ds4_cuda_q8_fold_take_q81( + const void *src, + uint64_t in_dim, + cudaStream_t stream, + const void **q81); + // Opt-in grouped-MMQ Q8_1 arena controlled by // DS4_CUDA_MMQ_Q81_PERSISTENT. Unset and =0 keep the stream-pool path. // cleanup drains the owner device before freeing and is safe across reinit; diff --git a/scripts/environment_variables.tsv b/scripts/environment_variables.tsv index a7335c852..9e98ca524 100644 --- a/scripts/environment_variables.tsv +++ b/scripts/environment_variables.tsv @@ -78,8 +78,8 @@ runtime/cuda DS4_CUDA_ENABLE_DSPARK_NONCAUSAL_ONLINE value-aware flag, default o runtime/cuda DS4_CUDA_ENABLE_HC_NORM_MIX_FUSE nonempty opt-in, default off; only exact 0 disables; the F32/F16 activation mode follows the selected standalone matmul path; disable/serial/alternate flags can veto Enable and select the fused HC RMSNorm-plus-mix one-token implementation. ds4_cuda.cu:20902 runtime/cuda DS4_CUDA_ENABLE_IQ2_XXS_SSD_PREFILL_MMQ false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Enable the CUDA IQ2 XXS SSD prefill MMQ experimental path. ds4_cuda.cu:4625 runtime/cuda DS4_CUDA_ENABLE_Q4_ATTN_OUT_HC_FUSE value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on Opt in to the fused Q4 attention-output/HC expansion path. ds4_cuda.cu:37375 -runtime/cuda DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_BATCH value-aware opt-in, default off; nonempty value other than exact 0 enables; rollback wins Enable flattened grouped attention-A MMQ for two-to-eight-token GB10 batches. cuda/mmq/ds4_mmq.cu:4313 -runtime/cuda DS4_CUDA_ENABLE_Q4_K1024_PERSISTENT presence flag, default off; any defined value including 0 requests the path; rollback wins Enable the GB10 persistent-CTA kernel for M=32768, N=1, K=1024 Q4. cuda/mmq/ds4_mmq.cu:3915 +runtime/cuda DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_BATCH value-aware opt-in, default off; nonempty value other than exact 0 enables; rollback wins Enable flattened grouped attention-A MMQ for two-to-eight-token GB10 batches. cuda/mmq/ds4_mmq.cu:4303 +runtime/cuda DS4_CUDA_ENABLE_Q4_K1024_PERSISTENT presence flag, default off; any defined value including 0 requests the path; rollback wins Enable the GB10 persistent-CTA kernel for M=32768, N=1, K=1024 Q4. cuda/mmq/ds4_mmq.cu:3905 runtime/cuda DS4_CUDA_ENABLE_Q8_FOLD strict flag, default off; only exact value 1 enables; overridden by DS4_CUDA_NO_Q8_FOLD Enable one-shot producer-to-consumer reuse of freshly quantized Q8_1 data. ds4_cuda.cu:785 runtime/cuda DS4_CUDA_ENABLE_STREAMING_EXPERT_PERSISTENT_CACHE false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Enable streaming expert persistent cache in CUDA SSD streaming. ds4_cuda.cu:4111 runtime/cuda DS4_CUDA_ENABLE_STREAMING_SELECTED_BATCH_IO false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Enable streaming selected batch I/O in CUDA SSD streaming. ds4_cuda.cu:4732 @@ -219,16 +219,16 @@ runtime/cuda DS4_CUDA_NO_INDEXER_WMMA64 presence kill switch; default unset (eli runtime/cuda DS4_CUDA_NO_IQ2_XXS_SSD_PREFILL_MMQ false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Disable the CUDA IQ2 XXS SSD prefill MMQ optimization/path. ds4_cuda.cu:4629 runtime/cuda DS4_CUDA_NO_MODEL_COPY presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the CUDA model copy optimization/path. ds4_cuda.cu:6472 runtime/cuda DS4_CUDA_NO_MODEL_PREFETCH presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the CUDA model prefetch optimization/path. ds4_cuda.cu:2739 -runtime/cuda DS4_CUDA_NO_MOE_DEDUP presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the CUDA MoE dedup optimization/path. cuda/mmq/ds4_mmq.cu:6182 +runtime/cuda DS4_CUDA_NO_MOE_DEDUP presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the CUDA MoE dedup optimization/path. cuda/mmq/ds4_mmq.cu:6172 runtime/cuda DS4_CUDA_NO_ORDERED_F16_MATMUL presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the ordered F16 matmul CUDA F16 path. ds4_cuda.cu:20811 runtime/cuda DS4_CUDA_NO_PARALLEL_ROUTER_SELECT presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the CUDA parallel router select optimization/path. ds4_cuda.cu:24491 -runtime/cuda DS4_CUDA_NO_Q4_DENSE_SCRATCH presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q4 dense scratch CUDA Q4 optimization. cuda/mmq/ds4_mmq.cu:3996 -runtime/cuda DS4_CUDA_NO_Q4_GB10_FAST presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the GB10-specific Q4 fast-path family. cuda/mmq/ds4_mmq.cu:3918 -runtime/cuda DS4_CUDA_NO_Q4_GROUPED_ATTN_A presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q4 grouped attn a CUDA Q4 optimization. cuda/mmq/ds4_mmq.cu:4305 -runtime/cuda DS4_CUDA_NO_Q4_GROUPED_ATTN_A_BATCH presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q4 grouped attn a batch CUDA Q4 optimization. cuda/mmq/ds4_mmq.cu:4315 -runtime/cuda DS4_CUDA_NO_Q4_K1024_PERSISTENT presence kill switch, default off; any defined value including 0 disables Disable the Q4 K1024 persistent CUDA Q4 optimization. cuda/mmq/ds4_mmq.cu:3917 -runtime/cuda DS4_CUDA_NO_Q8_ALIGNED_DENSE_SCRATCH presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q8 aligned dense scratch CUDA Q8 optimization. cuda/mmq/ds4_mmq.cu:5588 -runtime/cuda DS4_CUDA_NO_Q8_ALIGNED_PERSISTENT presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q8 aligned persistent CUDA Q8 optimization. cuda/mmq/ds4_mmq.cu:5420 +runtime/cuda DS4_CUDA_NO_Q4_DENSE_SCRATCH presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q4 dense scratch CUDA Q4 optimization. cuda/mmq/ds4_mmq.cu:3986 +runtime/cuda DS4_CUDA_NO_Q4_GB10_FAST presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the GB10-specific Q4 fast-path family. cuda/mmq/ds4_mmq.cu:3908 +runtime/cuda DS4_CUDA_NO_Q4_GROUPED_ATTN_A presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q4 grouped attn a CUDA Q4 optimization. cuda/mmq/ds4_mmq.cu:4295 +runtime/cuda DS4_CUDA_NO_Q4_GROUPED_ATTN_A_BATCH presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q4 grouped attn a batch CUDA Q4 optimization. cuda/mmq/ds4_mmq.cu:4305 +runtime/cuda DS4_CUDA_NO_Q4_K1024_PERSISTENT presence kill switch, default off; any defined value including 0 disables Disable the Q4 K1024 persistent CUDA Q4 optimization. cuda/mmq/ds4_mmq.cu:3907 +runtime/cuda DS4_CUDA_NO_Q8_ALIGNED_DENSE_SCRATCH presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q8 aligned dense scratch CUDA Q8 optimization. cuda/mmq/ds4_mmq.cu:5578 +runtime/cuda DS4_CUDA_NO_Q8_ALIGNED_PERSISTENT presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q8 aligned persistent CUDA Q8 optimization. cuda/mmq/ds4_mmq.cu:5410 runtime/cuda DS4_CUDA_NO_Q8_BATCH_EXACT_TOK2 presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q8 batch exact tok2 CUDA Q8 optimization. ds4_cuda.cu:19852 runtime/cuda DS4_CUDA_NO_Q8_BATCH_TOK4 presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q8 batch tok4 CUDA Q8 optimization. ds4_cuda.cu:19805 runtime/cuda DS4_CUDA_NO_Q8_BATCH_TOK8 presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q8 batch tok8 CUDA Q8 optimization. ds4_cuda.cu:19788 @@ -275,15 +275,15 @@ runtime/cuda DS4_CUDA_PREFILL_PIPELINE_SYNC_BOUNDARY presence flag; unset does n runtime/cuda DS4_CUDA_Q4_ATTN_OUT_HC_ORACLE value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on Compare fused Q4 attention-output/HC expansion with the canonical path and retain canonical output. ds4_cuda.cu:1475 runtime/cuda DS4_CUDA_Q4_ATTN_OUT_HC_Q8K_EXPERIMENT value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on Enable the experimental Q8_K-based Q4 attention-output/HC fusion. ds4_cuda.cu:37377 runtime/cuda DS4_CUDA_Q4_GROUPED_ATTN_A_ORACLE value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on Compare grouped attention-A against the canonical per-group result. ds4_cuda.cu:1477 -runtime/cuda DS4_CUDA_Q4_K1024_PERSISTENT_ORACLE value-aware flag, default off; nonempty value other than exact 0 enables and implies candidate admission Bitwise-compare the exact-shape persistent Q4 K1024 kernel with canonical MMVQ and retain canonical output. cuda/mmq/ds4_mmq.cu:3748 -runtime/cuda DS4_CUDA_Q4_K1024_PERSISTENT_STATS value-aware flag, default off; nonempty value other than exact 0 enables Print exact-shape persistent Q4 K1024 dispatch counters at exit. cuda/mmq/ds4_mmq.cu:3747 +runtime/cuda DS4_CUDA_Q4_K1024_PERSISTENT_ORACLE value-aware flag, default off; nonempty value other than exact 0 enables and implies candidate admission Bitwise-compare the exact-shape persistent Q4 K1024 kernel with canonical MMVQ and retain canonical output. cuda/mmq/ds4_mmq.cu:3738 +runtime/cuda DS4_CUDA_Q4_K1024_PERSISTENT_STATS value-aware flag, default off; nonempty value other than exact 0 enables Print exact-shape persistent Q4 K1024 dispatch counters at exit. cuda/mmq/ds4_mmq.cu:3737 runtime/cuda DS4_CUDA_Q8_F16_ALL presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Control the Q8 F16 all CUDA quantized-matmul/cache optimization. ds4_cuda.cu:2358 runtime/cuda DS4_CUDA_Q8_F16_CACHE_MB unsigned integer MiB, full-string parse; default unlimited; 0 disables this cache Limit the selective Q8-to-F16 derived-weight cache. ds4_cuda.cu:2218 runtime/cuda DS4_CUDA_Q8_F16_CACHE_RESERVE_MB unsigned integer MiB, full-string parse; default is VRAM-dependent (>=112 GiB: 512; >=40 GiB: max(768,1%); smaller: max(4096,5%)) Reserve free VRAM when growing the selective Q8-to-F16 cache. ds4_cuda.cu:2224 runtime/cuda DS4_CUDA_Q8_F32_ALL presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Control the Q8 F32 all CUDA quantized-matmul/cache optimization. ds4_cuda.cu:2412 runtime/cuda DS4_CUDA_Q8_F32_LARGE presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Control the Q8 F32 large CUDA quantized-matmul/cache optimization. ds4_cuda.cu:2416 runtime/cuda DS4_CUDA_Q8_F32_PRELOAD presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Control the Q8 F32 preload CUDA quantized-matmul/cache optimization. ds4_cuda.cu:8597 -runtime/cuda DS4_CUDA_Q8_FOLD_ORACLE strict flag, default off; only exact value 1 enables Compare folded Q8_1 bytes and consumer outputs against canonical work while retaining canonical results. cuda/mmq/ds4_mmq.cu:391 +runtime/cuda DS4_CUDA_Q8_FOLD_ORACLE strict flag, default off; only exact value 1 enables Compare folded Q8_1 bytes and consumer outputs against canonical work while retaining canonical results. cuda/mmq/ds4_mmq.cu:381 runtime/cuda DS4_CUDA_Q8_HC_EXPAND_FUSED false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, other nonempty values force fused Force the fused Q8 shared-down/HC expansion path. ds4_cuda.cu:2084 runtime/cuda DS4_CUDA_Q8_HC_EXPAND_STATS false-like-aware flag, default off; 0/false/no/off is off, other nonempty values print report Print Q8 shared-down/HC policy and dispatch counters at exit. ds4_cuda.cu:2090 runtime/cuda DS4_CUDA_Q8_NO_ALIGNED value-aware kill switch, default off; nonempty value other than exact 0 disables aligned Q8 kernels Disable aligned Q8 CUDA matmul kernels. ds4_cuda.cu:1021 @@ -292,7 +292,7 @@ runtime/cuda DS4_CUDA_QKV_KV_ROPE_FUSE value-aware boolean; default on; unset/em runtime/cuda DS4_CUDA_Q_NORM_ROPE_FUSE value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control or tune the CUDA q norm rope fuse path. ds4.c:17418 runtime/cuda DS4_CUDA_REQUIRE_IQ2_XXS_SSD_PREFILL_MMQ false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Require the CUDA IQ2 XXS SSD prefill MMQ path; fail closed when unavailable. ds4_cuda.cu:4631 runtime/cuda DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_BATCH value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on Fail if grouped batched attention-A cannot be used. ds4_cuda.cu:40198 -runtime/cuda DS4_CUDA_REQUIRE_Q4_K1024_PERSISTENT presence flag, default off; any defined value including 0 makes ineligible candidate fail closed Fail when the exact Q4 K1024 persistent candidate is unavailable instead of using MMVQ. cuda/mmq/ds4_mmq.cu:3939 +runtime/cuda DS4_CUDA_REQUIRE_Q4_K1024_PERSISTENT presence flag, default off; any defined value including 0 makes ineligible candidate fail closed Fail when the exact Q4 K1024 persistent candidate is unavailable instead of using MMVQ. cuda/mmq/ds4_mmq.cu:3929 runtime/cuda DS4_CUDA_REQUIRE_STREAMING_EXPERT_PERSISTENT_CACHE false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Require streaming expert persistent cache in CUDA SSD streaming; fail closed when unavailable. ds4_cuda.cu:4117 runtime/cuda DS4_CUDA_REQUIRE_STREAMING_SELECTED_BATCH_IO false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Require streaming selected batch I/O in CUDA SSD streaming; fail closed when unavailable. ds4_cuda.cu:4738 runtime/cuda DS4_CUDA_REQUIRE_STREAMING_SELECTED_EVENT_PIPELINE false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Require streaming selected event pipeline in CUDA SSD streaming; fail closed when unavailable. ds4_cuda.cu:4870 @@ -366,16 +366,16 @@ runtime/cuda DS4_CUDA_WEIGHT_PRELOAD_SPAN_MB positive integer MiB; default 1024; runtime/cuda DS4_CUDA_WINDOW_ATTENTION presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Control or tune the CUDA window attention path. ds4_cuda.cu:23051 runtime/cuda-mmq DS4_MMID_CASE1 boolean-ish cached flag; default on; a value starting with 0 disables Disable the single-expert MM-IDs specialized fast path for comparison. cuda/mmq/mmid.cu:290 runtime/cuda-mmq DS4_MMID_LARGE boolean-ish cached flag; default on; a value starting with 0 disables Control the large-N global-memory MM-IDs path used beyond shared-memory capacity. cuda/mmq/mmid.cu:245 -runtime/cuda-mmq DS4_MMQ_D2R boolean-ish cached flag; default on; a value starting with 0 disables Control the direct-to-register Q2_K MoE down path. cuda/mmq/ds4_mmq.cu:535 -runtime/cuda-mmq DS4_MMQ_D2R_IQ2 boolean-ish cached flag; default on; a value starting with 0 disables Control the direct-to-register IQ2 MoE gate/up path. cuda/mmq/ds4_mmq.cu:544 -runtime/cuda-mmq DS4_MMQ_D2R_MIN_COLS positive integer; default 1024; invalid or nonpositive input restores the default Set the minimum output-column count for the MMQ direct-to-register path. cuda/mmq/ds4_mmq.cu:639 +runtime/cuda-mmq DS4_MMQ_D2R boolean-ish cached flag; default on; a value starting with 0 disables Control the direct-to-register Q2_K MoE down path. cuda/mmq/ds4_mmq.cu:525 +runtime/cuda-mmq DS4_MMQ_D2R_IQ2 boolean-ish cached flag; default on; a value starting with 0 disables Control the direct-to-register IQ2 MoE gate/up path. cuda/mmq/ds4_mmq.cu:534 +runtime/cuda-mmq DS4_MMQ_D2R_MIN_COLS positive integer; default 1024; invalid or nonpositive input restores the default Set the minimum output-column count for the MMQ direct-to-register path. cuda/mmq/ds4_mmq.cu:629 runtime/cuda-mmq DS4_MMQ_D2R_STATS exact 1 enables; unset or every other value disables; cached and synchronizes the stream Print partial-tile fill telemetry for the direct-to-register MMQ kernels. cuda/mmq/ds4_mmq_d2r.cu:33 runtime/cuda-mmq DS4_MMQ_DENSE_D2R boolean-ish flag; default on; exact 0 disables Control the eligible aligned-Q8 dense prefill direct-to-register path. ds4_cuda.cu:19567 -runtime/cuda-mmq DS4_MMQ_NO_YIND presence rollback; unset keeps Y-indirect staging; any defined value including 0 disables it Restore slot-gathered MoE gate/up activation quantization. cuda/mmq/ds4_mmq.cu:619 -runtime/cuda-mmq DS4_MMQ_OUT_MEMSET exact 1 enables; unset or every other value disables; cached Restore blanket MMQ output-buffer zeroing for diagnostics. cuda/mmq/ds4_mmq.cu:562 -runtime/cuda-mmq DS4_MMQ_YBUF_MEMSET unset or 0 disables; 1 zero-fills; a value starting with p or P poison-fills with 0xFF Control MMQ Q8_1 activation-staging initialization and its poison oracle. cuda/mmq/ds4_mmq.cu:588 -runtime/cuda-mmq DS4_MMQ_YIND_VERIFY presence diagnostic; unset is off; any defined value including 0 enables Byte-compare Y-indirect and slot-gathered MoE activation buffers. cuda/mmq/ds4_mmq.cu:630 -runtime/cuda-mmq DS4_Q8_FOLD_SELFTEST positive call budget; unset/empty disables; a nonempty value parsing to 1 or less selects 512 calls Byte-check folded Q8_1 activations against a fresh quantization; synchronizes eager streams. cuda/mmq/ds4_mmq.cu:5963 +runtime/cuda-mmq DS4_MMQ_NO_YIND presence rollback; unset keeps Y-indirect staging; any defined value including 0 disables it Restore slot-gathered MoE gate/up activation quantization. cuda/mmq/ds4_mmq.cu:609 +runtime/cuda-mmq DS4_MMQ_OUT_MEMSET exact 1 enables; unset or every other value disables; cached Restore blanket MMQ output-buffer zeroing for diagnostics. cuda/mmq/ds4_mmq.cu:552 +runtime/cuda-mmq DS4_MMQ_YBUF_MEMSET unset or 0 disables; 1 zero-fills; a value starting with p or P poison-fills with 0xFF Control MMQ Q8_1 activation-staging initialization and its poison oracle. cuda/mmq/ds4_mmq.cu:578 +runtime/cuda-mmq DS4_MMQ_YIND_VERIFY presence diagnostic; unset is off; any defined value including 0 enables Byte-compare Y-indirect and slot-gathered MoE activation buffers. cuda/mmq/ds4_mmq.cu:620 +runtime/cuda-mmq DS4_Q8_FOLD_SELFTEST positive call budget; unset/empty disables; a nonempty value parsing to 1 or less selects 512 calls Byte-check folded Q8_1 activations against a fresh quantization; synchronizes eager streams. cuda/mmq/ds4_mmq.cu:5953 runtime/cuda-shared DS4_FORCE_CUDA_PEER presence flag read once at CUDA init; unset uses automatic transfer selection; any defined value including 0 enables Force cross-device transfers through cudaMemcpyPeerAsync for diagnostics. ds4_cuda.cu:364 runtime/cuda-shared DS4_FORCE_HOST_BOUNCE presence flag read once at CUDA init; unset uses automatic transfer selection; any defined value including 0 enables Force cross-device transfers through pinned host bounce buffers for diagnostics. ds4_cuda.cu:365 runtime/cuda-tools DS4_WS_REPACK_HASH exact 1 enables; unset or every other value disables unless overridden by CLI; cached Print a per-artifact FNV-1a hash for workspace repack identity checks. cuda/mmq/ds4_repack.cu:530 From fcea9f400ca1c5125f1f9249bcef4e3b2dda006d Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Mon, 24 Aug 2026 14:09:52 +0200 Subject: [PATCH 086/189] bench: support SSD-streamed Metal A/B runs --- speed-bench/README.md | 20 ++++++++- speed-bench/metal_decode_schedule_bench.c | 39 +++++++++++++++++- speed-bench/metal_prefill_variant_bench.c | 49 +++++++++++++++++++++-- 3 files changed, 100 insertions(+), 8 deletions(-) diff --git a/speed-bench/README.md b/speed-bench/README.md index 3390bd8fb..ca0c997fd 100644 --- a/speed-bench/README.md +++ b/speed-bench/README.md @@ -44,7 +44,16 @@ row is bit-identical and, with `--include-selection`, both variants select the same non-EOS token. Use `--candidate-env NAME` to measure a rollback control, or `--help` to compare explicit split schedules. Pass `--ssd-streaming` for a model larger than RAM; the harness then skips full-weight warmup while keeping -both variants in the same engine and expert cache. +both variants in the same engine and expert cache. SSD runs can use +`--ssd-streaming-cold`, `--ssd-streaming-cache-experts N`, and +`--ssd-streaming-preload-experts N` to hold the cache policy constant. + +This paired harness is the exact-logit gate. Since its two sessions share the +expert cache, confirm SSD throughput separately with one process and engine per +variant before promoting a scheduling change. Environment variables consumed +while the engine opens, including the Metal streaming `F_NOCACHE` controls, +also require separate processes: `--candidate-env` changes them too late to +create a different model descriptor inside this harness. To compare the default pre-M5 ratio-4 compressor pack/transpose fusion with the legacy decode path, including token selection, use: @@ -81,4 +90,11 @@ both variants with at least 32 tokens, alternates control/candidate order in ABBA and BAAB blocks, poisons host logit buffers before copying, and aborts unless every final full-vocabulary logit row is bit-identical. Defaults are an 8192-token prefix, an automatically sized 8193-token context, and two repeats; -use `--help` to override them. +use `--help` to override them. SSD-prefill variants can add `--ssd-streaming`, +`--ssd-streaming-cold`, `--ssd-streaming-cache-experts N`, and +`--ssd-streaming-preload-experts N`. Since those paired runs intentionally +share one engine and expert cache, confirm any SSD throughput win separately +with one process and engine per variant before promotion. +Environment variables consumed while the engine opens, including the Metal +streaming `F_NOCACHE` controls, cannot be compared with `--candidate-env` in +this harness and likewise require separate processes. diff --git a/speed-bench/metal_decode_schedule_bench.c b/speed-bench/metal_decode_schedule_bench.c index a7152bc1c..3b3852c7e 100644 --- a/speed-bench/metal_decode_schedule_bench.c +++ b/speed-bench/metal_decode_schedule_bench.c @@ -33,8 +33,11 @@ typedef struct { int ctx; int warmup; int measured; + uint32_t ssd_cache_experts; + uint32_t ssd_preload_experts; bool include_selection; bool ssd_streaming; + bool ssd_streaming_cold; decode_schedule control; decode_schedule candidate; } bench_config; @@ -55,6 +58,11 @@ static void usage(FILE *fp, const char *argv0) { " --candidate-second N candidate second split (default: 32; control with --candidate-env)\n" " --candidate-env NAME unset NAME for control, set NAME=1 for candidate\n" " --ssd-streaming use the SSD-backed model path instead of full residency\n" + " --ssd-streaming-cold skip the default expert-cache preload\n" + " --ssd-streaming-cache-experts N\n" + " dynamic expert-cache entry count\n" + " --ssd-streaming-preload-experts N\n" + " popularity preload count\n" " --include-selection include one non-EOS argmax in each timed step\n", argv0); } @@ -93,8 +101,11 @@ static bench_config parse_options(int argc, char **argv) { .ctx = DEFAULT_CTX, .warmup = DEFAULT_WARMUP, .measured = DEFAULT_MEASURED, + .ssd_cache_experts = 0, + .ssd_preload_experts = 0, .include_selection = false, .ssd_streaming = false, + .ssd_streaming_cold = false, .control = {.first = 2, .second = 32}, .candidate = {.first = 1, .second = 32}, }; @@ -116,6 +127,14 @@ static bench_config parse_options(int argc, char **argv) { cfg.include_selection = true; } else if (!strcmp(arg, "--ssd-streaming")) { cfg.ssd_streaming = true; + } else if (!strcmp(arg, "--ssd-streaming-cold")) { + cfg.ssd_streaming_cold = true; + } else if (!strcmp(arg, "--ssd-streaming-cache-experts")) { + cfg.ssd_cache_experts = (uint32_t)parse_int_arg( + need_arg(&i, argc, argv, arg), arg, 1); + } else if (!strcmp(arg, "--ssd-streaming-preload-experts")) { + cfg.ssd_preload_experts = (uint32_t)parse_int_arg( + need_arg(&i, argc, argv, arg), arg, 1); } else if (!strcmp(arg, "--prefix-tokens")) { cfg.prefix_tokens = parse_int_arg(need_arg(&i, argc, argv, arg), arg, 1); @@ -164,6 +183,15 @@ static bench_config parse_options(int argc, char **argv) { } } + if (!cfg.ssd_streaming && + (cfg.ssd_streaming_cold || cfg.ssd_cache_experts != 0 || + cfg.ssd_preload_experts != 0)) { + fprintf(stderr, + "metal-decode-schedule-bench: SSD cache options require " + "--ssd-streaming\n"); + exit(2); + } + const int64_t needed = (int64_t)cfg.prefix_tokens + cfg.warmup + cfg.measured + 1; if (needed > cfg.ctx) { @@ -379,6 +407,9 @@ int main(int argc, char **argv) { .power_percent = 100, .warm_weights = !cfg.ssd_streaming, .ssd_streaming = cfg.ssd_streaming, + .ssd_streaming_cold = cfg.ssd_streaming_cold, + .ssd_streaming_cache_experts = cfg.ssd_cache_experts, + .ssd_streaming_preload_experts = cfg.ssd_preload_experts, }; ds4_engine *engine = NULL; ds4_session *sessions[VARIANT_COUNT] = {0}; @@ -444,7 +475,8 @@ int main(int argc, char **argv) { fprintf(stderr, "metal-decode-schedule-bench: model=%s prompt=%s prefix=%d " "ctx=%d warmup=%d measured=%d control=%d/%d candidate=%d/%d " - "candidate_env=%s include_selection=%s ssd_streaming=%s\n", + "candidate_env=%s include_selection=%s ssd_streaming=%s " + "ssd_cold=%s cache_experts=%u preload_experts=%u\n", cfg.model_path, cfg.prompt_path, cfg.prefix_tokens, @@ -457,7 +489,10 @@ int main(int argc, char **argv) { cfg.candidate.second, cfg.candidate_env ? cfg.candidate_env : "(none)", cfg.include_selection ? "yes" : "no", - cfg.ssd_streaming ? "yes" : "no"); + cfg.ssd_streaming ? "yes" : "no", + cfg.ssd_streaming_cold ? "yes" : "no", + cfg.ssd_cache_experts, + cfg.ssd_preload_experts); const int eos = ds4_token_eos(engine); const int total_steps = cfg.warmup + cfg.measured; diff --git a/speed-bench/metal_prefill_variant_bench.c b/speed-bench/metal_prefill_variant_bench.c index 82f218414..f2f1c8ed3 100644 --- a/speed-bench/metal_prefill_variant_bench.c +++ b/speed-bench/metal_prefill_variant_bench.c @@ -27,6 +27,10 @@ typedef struct { int warmup_tokens; int ctx; int repeats; + uint32_t ssd_cache_experts; + uint32_t ssd_preload_experts; + bool ssd_streaming; + bool ssd_streaming_cold; } bench_config; typedef struct { @@ -45,7 +49,13 @@ static void usage(FILE *fp, const char *argv0) { " --prefix-tokens N timed prefill length (default: 8192)\n" " --warmup-tokens N untimed tokens per variant (default: 32; min: 32)\n" " --ctx N session allocation (default: max lengths + 1)\n" - " --repeats N alternating ABBA/BAAB pairs (default: 2)\n", + " --repeats N alternating ABBA/BAAB pairs (default: 2)\n" + " --ssd-streaming use the SSD-backed model path\n" + " --ssd-streaming-cold skip the default expert-cache preload\n" + " --ssd-streaming-cache-experts N\n" + " dynamic expert-cache entry count\n" + " --ssd-streaming-preload-experts N\n" + " popularity preload count\n", argv0); } @@ -82,6 +92,10 @@ static bench_config parse_options(int argc, char **argv) { .warmup_tokens = DEFAULT_WARMUP_TOKENS, .ctx = 0, .repeats = DEFAULT_REPEATS, + .ssd_cache_experts = 0, + .ssd_preload_experts = 0, + .ssd_streaming = false, + .ssd_streaming_cold = false, }; for (int i = 1; i < argc; i++) { @@ -106,6 +120,16 @@ static bench_config parse_options(int argc, char **argv) { } else if (!strcmp(arg, "--repeats")) { cfg.repeats = parse_int_arg(need_arg(&i, argc, argv, arg), arg, 1); + } else if (!strcmp(arg, "--ssd-streaming")) { + cfg.ssd_streaming = true; + } else if (!strcmp(arg, "--ssd-streaming-cold")) { + cfg.ssd_streaming_cold = true; + } else if (!strcmp(arg, "--ssd-streaming-cache-experts")) { + cfg.ssd_cache_experts = (uint32_t)parse_int_arg( + need_arg(&i, argc, argv, arg), arg, 1); + } else if (!strcmp(arg, "--ssd-streaming-preload-experts")) { + cfg.ssd_preload_experts = (uint32_t)parse_int_arg( + need_arg(&i, argc, argv, arg), arg, 1); } else { fprintf(stderr, "%s: unknown option: %s\n", BENCH_NAME, arg); usage(stderr, argv[0]); @@ -118,6 +142,14 @@ static bench_config parse_options(int argc, char **argv) { fprintf(stderr, "%s: --candidate-env requires a valid name\n", BENCH_NAME); exit(2); } + if (!cfg.ssd_streaming && + (cfg.ssd_streaming_cold || cfg.ssd_cache_experts != 0 || + cfg.ssd_preload_experts != 0)) { + fprintf(stderr, + "%s: SSD cache options require --ssd-streaming\n", + BENCH_NAME); + exit(2); + } const int longest = cfg.prefix_tokens > cfg.warmup_tokens ? cfg.prefix_tokens @@ -293,7 +325,11 @@ int main(int argc, char **argv) { .context_size = cfg.ctx, .prefill_chunk = 4096, .power_percent = 100, - .warm_weights = true, + .warm_weights = !cfg.ssd_streaming, + .ssd_streaming = cfg.ssd_streaming, + .ssd_streaming_cold = cfg.ssd_streaming_cold, + .ssd_streaming_cache_experts = cfg.ssd_cache_experts, + .ssd_streaming_preload_experts = cfg.ssd_preload_experts, }; ds4_engine *engine = NULL; ds4_tokens tokens = {0}; @@ -333,7 +369,8 @@ int main(int argc, char **argv) { fprintf(stderr, "%s: model=%s prompt=%s prefix=%d warmup=%d ctx=%d repeats=%d " - "candidate_env=%s\n", + "candidate_env=%s ssd_streaming=%s ssd_cold=%s " + "cache_experts=%u preload_experts=%u\n", BENCH_NAME, cfg.model_path, cfg.prompt_path, @@ -341,7 +378,11 @@ int main(int argc, char **argv) { cfg.warmup_tokens, cfg.ctx, cfg.repeats, - cfg.candidate_env); + cfg.candidate_env, + cfg.ssd_streaming ? "yes" : "no", + cfg.ssd_streaming_cold ? "yes" : "no", + cfg.ssd_cache_experts, + cfg.ssd_preload_experts); for (int variant = 0; variant < VARIANT_COUNT; variant++) { err[0] = '\0'; From b8785133c273bc2f5bc6f9f39ba7724e21a6b191 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Mon, 24 Aug 2026 14:10:11 +0200 Subject: [PATCH 087/189] metal: speed up SSD-streamed IQ2 prefill --- ds4_metal.m | 209 ++++++++++++++++++--- metal/moe.metal | 48 +++-- tests/test_metal_iq2_ssd_grouped_mm.c | 255 ++++++++++++++++++++++++-- 3 files changed, 451 insertions(+), 61 deletions(-) diff --git a/ds4_metal.m b/ds4_metal.m index d3a7b5fa9..72c9a4ba9 100644 --- a/ds4_metal.m +++ b/ds4_metal.m @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -478,9 +479,13 @@ #define g_moe_q4_down_slots_bytes DS4_STREAM_SCRATCH(moe_q4_down_slots_bytes) #define g_attn_out_group_ids_bytes DS4_STREAM_SCRATCH(attn_out_group_ids_bytes) static int g_model_fd = -1; -/* Second model descriptor with F_NOCACHE for the streaming expert preads - * (DS4_METAL_STREAMING_EXPERT_NOCACHE); -1 = use the cached g_model_fd. */ +/* Second model descriptor with F_NOCACHE for all streaming expert preads or + * only the batched-prefill requests. Individual pread tasks carry the phase + * preference so decode never depends on mutable process-global phase state. */ static int g_model_fd_nocache = -1; +static int g_model_fd_nocache_all; +static int g_model_fd_nocache_prefill; +static int g_model_fd_nocache_prefill_auto; static const void *g_model_map_ptr; static uint64_t g_model_map_size; static uint64_t g_model_mapped_offset; @@ -520,6 +525,10 @@ static uint64_t g_stream_expert_cache_willneed_advise_bytes; static uint64_t g_stream_expert_cache_pread_bytes; static double g_stream_expert_cache_pread_ms; +static uint64_t g_stream_expert_pread_cached_calls; +static uint64_t g_stream_expert_pread_cached_bytes; +static uint64_t g_stream_expert_pread_nocache_calls; +static uint64_t g_stream_expert_pread_nocache_bytes; static uint64_t g_stream_expert_cache_buffer_allocs; static uint64_t g_stream_expert_cache_buffer_reuses; static uint64_t g_stream_expert_cache_decode_tokens; @@ -4584,6 +4593,22 @@ void ds4_gpu_print_memory_report(const char *label) { ds4_gpu_gib(g_stream_expert_cache_willneed_advise_bytes), ds4_gpu_gib(g_stream_expert_cache_pread_bytes), g_stream_expert_cache_pread_ms); + fprintf(stderr, + "ds4: streaming expert pread descriptors " + "cached_calls=%llu cached=%.2f GiB " + "nocache_calls=%llu nocache=%.2f GiB\n", + (unsigned long long)__atomic_load_n( + &g_stream_expert_pread_cached_calls, + __ATOMIC_RELAXED), + ds4_gpu_gib(__atomic_load_n( + &g_stream_expert_pread_cached_bytes, + __ATOMIC_RELAXED)), + (unsigned long long)__atomic_load_n( + &g_stream_expert_pread_nocache_calls, + __ATOMIC_RELAXED), + ds4_gpu_gib(__atomic_load_n( + &g_stream_expert_pread_nocache_bytes, + __ATOMIC_RELAXED))); } else { fprintf(stderr, "ds4: streaming expert cache budget=%llu experts entries=%u expert=%.2f MiB target=%.2f GiB live=%.2f GiB, hits=%llu misses=%llu hit_rate=%.3f wraps=%llu evictions=%llu buffer_allocs=%llu buffer_reuses=%llu\n", @@ -11539,6 +11564,9 @@ void ds4_gpu_cleanup(void) { close(g_model_fd_nocache); g_model_fd_nocache = -1; } + g_model_fd_nocache_all = 0; + g_model_fd_nocache_prefill = 0; + g_model_fd_nocache_prefill_auto = 0; g_model_map_ptr = NULL; g_model_map_size = 0; g_model_mapped_offset = 0; @@ -12614,43 +12642,95 @@ int ds4_gpu_prepare_support_model(const void *model_map, return ok; } -/* DS4_METAL_STREAMING_EXPERT_NOCACHE: serve the streaming expert preads from - * a second F_NOCACHE descriptor so the ~1 GB/token of expert churn stops - * evicting the mapped dense weights from the page cache. On tight-RAM - * machines the dense working set (attention projections, shared experts, - * routing) is re-read every token through the page cache: with the default - * cached preads the expert traffic keeps pushing it out and decode collapses - * to SSD fault speed. Opt-in: on machines where everything fits in RAM the - * cached preads are strictly better (second touch is free). */ +/* DS4_METAL_STREAMING_EXPERT_NOCACHE keeps its established all-phase + * behavior. DS4_METAL_STREAMING_EXPERT_PREFILL_NOCACHE uses the same second + * descriptor only for batched-prefill expert tasks, preserving the cached + * descriptor and readahead policy for steady decode. */ static int ds4_gpu_stream_expert_nocache_requested(void) { const char *env = getenv("DS4_METAL_STREAMING_EXPERT_NOCACHE"); return env != NULL && env[0] != '\0' && env[0] != '0'; } +static int ds4_gpu_stream_expert_prefill_nocache_auto(int fd) { + if (fd < 0 || !g_ssd_streaming_mode || g_glm_model_mode || + !ds4_gpu_device_is_pre_m5_apple_silicon()) { + return 0; + } + struct stat st; + if (fstat(fd, &st) != 0 || st.st_size <= 0) return 0; + const uint64_t working_set = ds4_gpu_recommended_working_set_size(); + if (working_set == 0 || working_set > UINT64_MAX / 2u) return 0; + return (uint64_t)st.st_size >= 2u * working_set; +} + +static int ds4_gpu_stream_expert_prefill_nocache_resolve(int fd, + int *automatic) { + if (automatic) *automatic = 0; + if (fd < 0 || !g_ssd_streaming_mode) return 0; + const int enable = ds4_gpu_env_bool( + "DS4_METAL_STREAMING_EXPERT_PREFILL_NOCACHE"); + const int disable = ds4_gpu_env_bool( + "DS4_METAL_DISABLE_STREAMING_EXPERT_PREFILL_NOCACHE"); + if (disable == 1 || enable == 0) return 0; + if (enable == 1) return 1; + const int auto_enabled = + ds4_gpu_stream_expert_prefill_nocache_auto(fd); + if (automatic) *automatic = auto_enabled; + return auto_enabled; +} + int ds4_gpu_set_model_fd(int fd) { g_model_fd = fd; if (g_model_fd_nocache >= 0) { close(g_model_fd_nocache); g_model_fd_nocache = -1; } - if (fd >= 0 && ds4_gpu_stream_expert_nocache_requested()) { + g_model_fd_nocache_prefill_auto = 0; + g_model_fd_nocache_all = + fd >= 0 && ds4_gpu_stream_expert_nocache_requested(); + g_model_fd_nocache_prefill = + !g_model_fd_nocache_all && + ds4_gpu_stream_expert_prefill_nocache_resolve( + fd, &g_model_fd_nocache_prefill_auto); + if (fd >= 0 && + (g_model_fd_nocache_all || g_model_fd_nocache_prefill)) { /* A dup() would share the file description (and its F_NOCACHE flag) * with the mmap-backed descriptor, so reopen the model by path. */ char path[1024] = {0}; int nfd = -1; + struct stat source_stat; + struct stat reopened_stat; + int same_file = 0; if (fcntl(fd, F_GETPATH, path) == 0) nfd = open(path, O_RDONLY); - if (nfd >= 0) { + if (nfd >= 0 && + fstat(fd, &source_stat) == 0 && + fstat(nfd, &reopened_stat) == 0 && + source_stat.st_dev == reopened_stat.st_dev && + source_stat.st_ino == reopened_stat.st_ino) { + same_file = 1; + } else if (nfd >= 0) { + errno = ESTALE; + } + if (same_file && fcntl(nfd, F_NOCACHE, 1) == 0) { (void)fcntl(nfd, F_SETFD, FD_CLOEXEC); - (void)fcntl(nfd, F_NOCACHE, 1); g_model_fd_nocache = nfd; fprintf(stderr, - "ds4: Metal streaming expert preads on a F_NOCACHE descriptor; " - "page cache reserved for the dense weights (readahead hints off)\n"); + "ds4: Metal streaming expert %s preads on a F_NOCACHE " + "descriptor; page cache reserved for dense weights\n", + g_model_fd_nocache_all ? "all-phase" : + g_model_fd_nocache_prefill_auto ? + "batched-prefill automatic" : + "batched-prefill"); } else { + const int saved_errno = errno; + if (nfd >= 0) close(nfd); + g_model_fd_nocache_all = 0; + g_model_fd_nocache_prefill = 0; + g_model_fd_nocache_prefill_auto = 0; fprintf(stderr, "ds4: WARNING: F_NOCACHE expert descriptor unavailable (%s); " "using cached preads\n", - strerror(errno)); + strerror(saved_errno)); } } return 1; @@ -13188,12 +13268,16 @@ static void ds4_gpu_stream_expert_timing_note_cache_class( g_stream_expert_timing_cache_missing_experts += missing; } +static int ds4_gpu_stream_prefill_nocache_for_tokens(uint32_t n_tokens) { + return g_model_fd_nocache_prefill && n_tokens >= 32u; +} + static int ds4_gpu_stream_expert_readahead_enabled(void) { - /* F_RDADVISE warms the PAGE CACHE: with the F_NOCACHE expert descriptor - * active those pages would never be consumed by the preads and only evict - * the dense weights — the exact pollution that mode exists to stop. */ + /* All-phase F_NOCACHE never consumes these page-cache hints. The + * prefill-only mode keeps them for decode and suppresses them explicitly + * in ds4_gpu_stream_prefill_expert_readahead_enabled(). */ return g_ssd_streaming_mode && - g_model_fd_nocache < 0 && + !g_model_fd_nocache_all && getenv("DS4_METAL_DISABLE_STREAMING_EXPERT_READAHEAD") == NULL; } @@ -13207,7 +13291,8 @@ static int ds4_gpu_stream_prefill_expert_readahead_enabled( * but require an explicit opt-in for this immediate-pread path so the old * policy remains available for cold-storage A/B tests. */ - if (!ds4_gpu_stream_expert_readahead_enabled()) return 0; + if (!ds4_gpu_stream_expert_readahead_enabled() || + ds4_gpu_stream_prefill_nocache_for_tokens(n_tokens)) return 0; if (n_tokens < 32u) return 1; return ds4_gpu_env_bool( "DS4_METAL_ENABLE_STREAMING_PREFILL_EXPERT_READAHEAD") > 0; @@ -13251,6 +13336,7 @@ static void ds4_gpu_stream_expert_readahead_range(uint64_t offset, uint64_t len) uint64_t offset; uint64_t len; uint8_t *dst; + int prefer_nocache; uint64_t read_bytes; double ms; int ok; @@ -13359,11 +13445,16 @@ static int ds4_gpu_stream_expert_pread_into( uint64_t offset, uint64_t len, uint8_t *dst, + int prefer_nocache, uint64_t *read_bytes, double *ms_out) { if (read_bytes) *read_bytes = 0; if (ms_out) *ms_out = 0.0; - const int fd = g_model_fd_nocache >= 0 ? g_model_fd_nocache : g_model_fd; + const int use_nocache = + g_model_fd_nocache >= 0 && + (g_model_fd_nocache_all || + (g_model_fd_nocache_prefill && prefer_nocache)); + const int fd = use_nocache ? g_model_fd_nocache : g_model_fd; if (fd < 0 || !dst || len == 0 || @@ -13389,6 +13480,21 @@ static int ds4_gpu_stream_expert_pread_into( pos += (uint64_t)nread; } const double dt = ds4_gpu_now_ms() - t0; + if (use_nocache) { + __atomic_add_fetch(&g_stream_expert_pread_nocache_calls, + 1, + __ATOMIC_RELAXED); + __atomic_add_fetch(&g_stream_expert_pread_nocache_bytes, + pos, + __ATOMIC_RELAXED); + } else { + __atomic_add_fetch(&g_stream_expert_pread_cached_calls, + 1, + __ATOMIC_RELAXED); + __atomic_add_fetch(&g_stream_expert_pread_cached_bytes, + pos, + __ATOMIC_RELAXED); + } if (read_bytes) *read_bytes = pos; if (ms_out) *ms_out = dt; if (!ok || pos != len) { @@ -13410,6 +13516,7 @@ static int ds4_gpu_stream_expert_pread_into( task->ok = ds4_gpu_stream_expert_pread_into(task->offset, task->len, task->dst, + task->prefer_nocache, &task->read_bytes, &task->ms); } @@ -13469,6 +13576,7 @@ static int ds4_gpu_stream_expert_pread_pool_enabled(void) { task->ok = ds4_gpu_stream_expert_pread_into(task->offset, task->len, task->dst, + task->prefer_nocache, &task->read_bytes, &task->ms); @@ -13763,6 +13871,7 @@ static uint32_t ds4_gpu_stream_expert_pread_expand_tasks_bounded( .offset = original[i].offset + off, .len = part, .dst = original[i].dst + off, + .prefer_nocache = original[i].prefer_nocache, .read_bytes = 0, .ms = 0.0, .ok = 0, @@ -13826,6 +13935,7 @@ static int ds4_gpu_stream_expert_pread_tasks( sub[w].offset = tasks[i].offset + off; sub[w].len = part; sub[w].dst = tasks[i].dst + off; + sub[w].prefer_nocache = tasks[i].prefer_nocache; sub[w].read_bytes = 0; sub[w].ms = 0.0; sub[w].ok = 0; @@ -15931,6 +16041,18 @@ static void ds4_gpu_stream_expert_cache_clear_all(int reset_stats) { g_stream_expert_cache_willneed_advise_bytes = 0; g_stream_expert_cache_pread_bytes = 0; g_stream_expert_cache_pread_ms = 0.0; + __atomic_store_n(&g_stream_expert_pread_cached_calls, + 0, + __ATOMIC_RELAXED); + __atomic_store_n(&g_stream_expert_pread_cached_bytes, + 0, + __ATOMIC_RELAXED); + __atomic_store_n(&g_stream_expert_pread_nocache_calls, + 0, + __ATOMIC_RELAXED); + __atomic_store_n(&g_stream_expert_pread_nocache_bytes, + 0, + __ATOMIC_RELAXED); g_stream_expert_cache_mlock_bytes = 0; g_stream_expert_cache_mlock_fail_bytes = 0; g_stream_expert_cache_mlock_failures = 0; @@ -18484,6 +18606,8 @@ static int ds4_gpu_stream_expert_cache_prepare_selected_batch( return 0; } if (n_tokens > UINT32_MAX / n_selected) return 0; + const int prefer_prefill_nocache = + ds4_gpu_stream_prefill_nocache_for_tokens(n_tokens); const uint64_t n_ids = (uint64_t)n_tokens * n_selected; if (n_ids > SIZE_MAX / sizeof(int32_t)) return 0; @@ -18779,16 +18903,19 @@ static int ds4_gpu_stream_expert_cache_prepare_selected_batch( .offset = unique_gate_offsets[u], .len = gate_expert_bytes, .dst = gate_dst, + .prefer_nocache = prefer_prefill_nocache, }; tasks[n_tasks++] = (ds4_gpu_stream_expert_pread_task) { .offset = unique_up_offsets[u], .len = gate_expert_bytes, .dst = up_dst, + .prefer_nocache = prefer_prefill_nocache, }; tasks[n_tasks++] = (ds4_gpu_stream_expert_pread_task) { .offset = unique_down_offsets[u], .len = down_expert_bytes, .dst = down_dst, + .prefer_nocache = prefer_prefill_nocache, }; if (load_timing) { ds4_gpu_stream_expert_timing_note_prepare_task( @@ -33771,6 +33898,20 @@ static int ds4_gpu_routed_mm_mpp_mask(void) { } } +static id +ds4_gpu_routed_mm_addr_tail_cull_pipeline(uint32_t type) { + switch (type) { + case DS4_METAL_TENSOR_IQ2_XXS: + return ds4_gpu_get_mul_mm_id_pipeline( + "kernel_mul_mm_id_addr_iq2_xxs_f32_tail_cull", false); + case DS4_METAL_TENSOR_Q2_K: + return ds4_gpu_get_mul_mm_id_pipeline( + "kernel_mul_mm_id_addr_q2_K_f32_tail_cull", false); + default: + return nil; + } +} + static id ds4_gpu_routed_mm_f16_rhs_pipeline(uint32_t type) { switch (type) { case DS4_METAL_TENSOR_Q8_0: @@ -45230,16 +45371,36 @@ int ds4_gpu_routed_moe_batch_tensor( request_iq2_batch_addr_mm && iq2_batch_addr_mm_candidate && use_iq2_batch_selected_addr; + /* Balanced full-model A/B on M1 Max is stable above the promotion + * gate at 512 tokens, while the shorter batches remain I/O-bound. + * An explicit ENABLE forces the specialization below the automatic + * threshold; ENABLE=0 and DISABLE=1 are rollback controls. */ + const int enable_iq2_batch_addr_mm_tail_cull = + ds4_gpu_env_bool( + "DS4_METAL_ENABLE_IQ2_XXS_SSD_PREFILL_MM_ADDR_TAIL_CULL"); + const int disable_iq2_batch_addr_mm_tail_cull = + ds4_gpu_env_bool( + "DS4_METAL_DISABLE_IQ2_XXS_SSD_PREFILL_MM_ADDR_TAIL_CULL"); + const bool iq2_batch_addr_mm_tail_cull = + disable_iq2_batch_addr_mm_tail_cull != 1 && + (enable_iq2_batch_addr_mm_tail_cull == 1 || + (enable_iq2_batch_addr_mm_tail_cull == -1 && + ds4_gpu_device_is_pre_m5_apple_silicon() && + n_tokens >= 512u)); id iq2_gate_addr_mm_pipeline = iq2_batch_addr_mm_policy && (g_test_flags & DS4_GPU_TEST_IQ2_SSD_GROUPED_PIPELINE_FAILURE) == 0u ? - ds4_gpu_routed_mm_addr_pipeline(gate_type) : nil; + (iq2_batch_addr_mm_tail_cull ? + ds4_gpu_routed_mm_addr_tail_cull_pipeline(gate_type) : + ds4_gpu_routed_mm_addr_pipeline(gate_type)) : nil; id iq2_down_addr_mm_pipeline = iq2_batch_addr_mm_policy && (g_test_flags & DS4_GPU_TEST_IQ2_SSD_GROUPED_PIPELINE_FAILURE) == 0u ? - ds4_gpu_routed_mm_addr_pipeline(down_type) : nil; + (iq2_batch_addr_mm_tail_cull ? + ds4_gpu_routed_mm_addr_tail_cull_pipeline(down_type) : + ds4_gpu_routed_mm_addr_pipeline(down_type)) : nil; const bool use_iq2_batch_addr_mm = iq2_batch_addr_mm_policy && iq2_gate_addr_mm_pipeline != nil && diff --git a/metal/moe.metal b/metal/moe.metal index 4860d1929..ac195458f 100644 --- a/metal/moe.metal +++ b/metal/moe.metal @@ -8908,7 +8908,7 @@ kernel void kernel_mul_mm_id( // Address-table variant used by SSD streaming. The routing ids remain the // model's original expert ids, but each expert's resident buffer is found via a // GPU-address table instead of a contiguous full-layer tensor. -template +template kernel void kernel_mul_mm_id_addr( constant ds4_metal_args_mul_mm_id & args, device const uint64_t * src0_addrs, @@ -8971,6 +8971,8 @@ kernel void kernel_mul_mm_id_addr( const short nr0 = (args.ne0 - r0 < NR0) ? (args.ne0 - r0) : NR0; const short nr1 = ( neh1 - r1 < NR1) ? ( neh1 - r1) : NR1; + const bool mma_active = + !CULL_TAIL_SIMDGROUPS || 16*(short)(sgitg/2) < nr1; const short lr0 = ((short)tiitg/NL0) < nr0 ? ((short)tiitg/NL0) : nr0 - 1; const short lr1 = ((short)tiitg/NL1) < nr1 ? ((short)tiitg/NL1) : nr1 - 1; @@ -9073,27 +9075,30 @@ kernel void kernel_mul_mm_id_addr( threadgroup const S0 * lsma = (sa + 4*64*(sgitg%2)); threadgroup const S1 * lsmb = (sb + 2*64*(sgitg/2)); - FOR_UNROLL (short ik = 0; ik < NK/8; ik++) { - simdgroup_barrier(mem_flags::mem_none); + if (mma_active) { + FOR_UNROLL (short ik = 0; ik < NK/8; ik++) { + simdgroup_barrier(mem_flags::mem_none); - FOR_UNROLL (short i = 0; i < 4; i++) { - simdgroup_load(ma[i], lsma + 64*i, 8, 0, false); - } + FOR_UNROLL (short i = 0; i < 4; i++) { + simdgroup_load(ma[i], lsma + 64*i, 8, 0, false); + } - simdgroup_barrier(mem_flags::mem_none); + simdgroup_barrier(mem_flags::mem_none); - FOR_UNROLL (short i = 0; i < 2; i++) { - simdgroup_load(mb[i], lsmb + 64*i, 8, 0, false); - } + FOR_UNROLL (short i = 0; i < 2; i++) { + simdgroup_load(mb[i], lsmb + 64*i, 8, 0, false); + } - simdgroup_barrier(mem_flags::mem_none); + simdgroup_barrier(mem_flags::mem_none); - FOR_UNROLL (short i = 0; i < 8; i++){ - simdgroup_multiply_accumulate(mc[i], mb[i/4], ma[i%4], mc[i]); - } + FOR_UNROLL (short i = 0; i < 8; i++){ + simdgroup_multiply_accumulate( + mc[i], mb[i/4], ma[i%4], mc[i]); + } - lsma += 8*64; - lsmb += 4*64; + lsma += 8*64; + lsmb += 4*64; + } } } @@ -9101,8 +9106,12 @@ kernel void kernel_mul_mm_id_addr( threadgroup float * temp_str = ((threadgroup float *) shmem) + 32*(sgitg&1) + (16*(sgitg >> 1))*NR0; - for (short i = 0; i < 8; i++) { - simdgroup_store(mc[i], temp_str + 8*(i%4) + 8*NR0*(i/4), NR0, 0, false); + if (mma_active) { + for (short i = 0; i < 8; i++) { + simdgroup_store(mc[i], + temp_str + 8*(i%4) + 8*NR0*(i/4), + NR0, 0, false); + } } threadgroup_barrier(mem_flags::mem_threadgroup); @@ -9592,6 +9601,7 @@ typedef decltype(kernel_mul_mm_id<32, half, half4x4, simdgroup_half8x8, half, ha typedef decltype(kernel_mul_mm_id<32, half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, block_mxfp4, 2, dequantize_mxfp4_half_lut, half, half4x4, half, half2x4>) mul_mm_id_mxfp4_f16_rhs_half_lut; typedef decltype(kernel_mul_mm_id<32, half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, block_mxfp4, 2, dequantize_mxfp4_half_lut, half, half4x4, half, half2x4, true>) mul_mm_id_mxfp4_f16_rhs_half_lut_tail_cull; typedef decltype(kernel_mul_mm_id_addr<32, half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, block_q2_K, QK_NL, dequantize_q2_K, float, float4x4, float, float2x4>) mul_mm_id_addr; +typedef decltype(kernel_mul_mm_id_addr<32, half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, block_q2_K, QK_NL, dequantize_q2_K, float, float4x4, float, float2x4, true>) mul_mm_id_addr_tail_cull; typedef decltype(kernel_mul_mm_id_addr<32, half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, block_q2_K, QK_NL, dequantize_q2_K, half, half4x4, half, half2x4>) mul_mm_id_addr_f16_rhs; // Host-visible batched MoE matmul variants for the DS4 quant formats. @@ -9615,6 +9625,8 @@ template [[host_name("kernel_mul_mm_id_mxfp4_f16_half_lut_tail_cull")]] kernel m template [[host_name("kernel_mul_mm_id_addr_q2_K_f32")]] kernel mul_mm_id_addr kernel_mul_mm_id_addr<32, half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, block_q2_K, QK_NL, dequantize_q2_K, float, float4x4, float, float2x4>; template [[host_name("kernel_mul_mm_id_addr_iq2_xxs_f32")]] kernel mul_mm_id_addr kernel_mul_mm_id_addr<32, half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, block_iq2_xxs, QK_NL, dequantize_iq2_xxs, float, float4x4, float, float2x4>; +template [[host_name("kernel_mul_mm_id_addr_q2_K_f32_tail_cull")]] kernel mul_mm_id_addr_tail_cull kernel_mul_mm_id_addr<32, half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, block_q2_K, QK_NL, dequantize_q2_K, float, float4x4, float, float2x4, true>; +template [[host_name("kernel_mul_mm_id_addr_iq2_xxs_f32_tail_cull")]] kernel mul_mm_id_addr_tail_cull kernel_mul_mm_id_addr<32, half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, block_iq2_xxs, QK_NL, dequantize_iq2_xxs, float, float4x4, float, float2x4, true>; template [[host_name("kernel_mul_mm_id_addr_q4_K_f32")]] kernel mul_mm_id_addr kernel_mul_mm_id_addr<32, half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, block_q4_K, QK_NL, dequantize_q4_K, float, float4x4, float, float2x4>; template [[host_name("kernel_mul_mm_id_addr_mxfp4_f32")]] kernel mul_mm_id_addr kernel_mul_mm_id_addr<32, half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, block_mxfp4, 2, dequantize_mxfp4, float, float4x4, float, float2x4>; template [[host_name("kernel_mul_mm_id_addr_q2_K_f16")]] kernel mul_mm_id_addr_f16_rhs kernel_mul_mm_id_addr<32, half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, block_q2_K, QK_NL, dequantize_q2_K, half, half4x4, half, half2x4>; diff --git a/tests/test_metal_iq2_ssd_grouped_mm.c b/tests/test_metal_iq2_ssd_grouped_mm.c index fc0df99f4..038c0bc3b 100644 --- a/tests/test_metal_iq2_ssd_grouped_mm.c +++ b/tests/test_metal_iq2_ssd_grouped_mm.c @@ -26,6 +26,10 @@ #define HIGH_EXPERT_ID 255u #define CLAMP 4.0f #define SENTINEL 1234567.0f +#define TAIL_CULL_ENV \ + "DS4_METAL_ENABLE_IQ2_XXS_SSD_PREFILL_MM_ADDR_TAIL_CULL" +#define TAIL_CULL_DISABLE_ENV \ + "DS4_METAL_DISABLE_IQ2_XXS_SSD_PREFILL_MM_ADDR_TAIL_CULL" /* Production Flash routed-expert geometry. Eight physical experts keep the * standalone fixture bounded while retaining the production top-6 routing, @@ -448,6 +452,180 @@ static int compare_results(const char *name, const run_result *candidate, return ok; } +static void clear_tail_cull_test_env(void) { + unsetenv(TAIL_CULL_ENV); + unsetenv(TAIL_CULL_DISABLE_ENV); + unsetenv("DS4_METAL_ENABLE_IQ2_XXS_SSD_PREFILL_MM"); + unsetenv("DS4_METAL_REQUIRE_IQ2_XXS_SSD_PREFILL_MM"); + unsetenv("DS4_METAL_DISABLE_IQ2_XXS_SSD_PREFILL_MM"); +} + +static void configure_tail_cull_test_env(bool enable_tail_cull) { + unsetenv(TAIL_CULL_DISABLE_ENV); + unsetenv("DS4_METAL_DISABLE_IQ2_XXS_SSD_PREFILL_MM"); + setenv("DS4_METAL_ENABLE_IQ2_XXS_SSD_PREFILL_MM", "1", 1); + setenv("DS4_METAL_REQUIRE_IQ2_XXS_SSD_PREFILL_MM", "1", 1); + if (enable_tail_cull) { + setenv(TAIL_CULL_ENV, "1", 1); + } else { + unsetenv(TAIL_CULL_ENV); + } +} + +static int compare_array_bit_exact(const char *case_name, + const char *tensor_name, + const float *candidate, + const float *control, + uint64_t count) { + uint32_t sentinel_bits = 0; + memcpy(&sentinel_bits, &(float){ SENTINEL }, sizeof(sentinel_bits)); + uint64_t mismatches = 0; + uint64_t nonfinite = 0; + uint64_t unwritten = 0; + uint64_t first_mismatch = UINT64_MAX; + uint32_t first_candidate = 0; + uint32_t first_control = 0; + for (uint64_t i = 0; i < count; i++) { + uint32_t candidate_bits = 0; + uint32_t control_bits = 0; + memcpy(&candidate_bits, candidate + i, sizeof(candidate_bits)); + memcpy(&control_bits, control + i, sizeof(control_bits)); + if (candidate_bits == sentinel_bits || control_bits == sentinel_bits) { + unwritten++; + } + if ((candidate_bits & 0x7f800000u) == 0x7f800000u || + (control_bits & 0x7f800000u) == 0x7f800000u) { + nonfinite++; + } + if (candidate_bits != control_bits) { + if (first_mismatch == UINT64_MAX) { + first_mismatch = i; + first_candidate = candidate_bits; + first_control = control_bits; + } + mismatches++; + } + } + const int pass = mismatches == 0 && nonfinite == 0 && unwritten == 0; + fprintf(stderr, + "IQ2_XXS SSD tail-cull %-20s %-4s %s count=%llu " + "bit_mismatches=%llu nonfinite=%llu unwritten=%llu", + case_name, tensor_name, pass ? "PASS" : "FAIL", + (unsigned long long)count, + (unsigned long long)mismatches, + (unsigned long long)nonfinite, + (unsigned long long)unwritten); + if (first_mismatch != UINT64_MAX) { + fprintf(stderr, " first=%llu candidate=0x%08x control=0x%08x", + (unsigned long long)first_mismatch, + first_candidate, first_control); + } + fputc('\n', stderr); + return pass; +} + +static int compare_results_bit_exact(const char *name, + const run_result *candidate, + const run_result *control) { + const int shape_ok = + candidate->pair_count == control->pair_count && + candidate->out_count == control->out_count && + candidate->guard_mismatches == 0 && + control->guard_mismatches == 0; + if (!shape_ok) { + fprintf(stderr, + "IQ2_XXS SSD tail-cull %-20s shape/guard FAIL\n", name); + return 0; + } + int ok = compare_array_bit_exact( + name, "gate", candidate->gate, control->gate, + candidate->pair_count); + ok = compare_array_bit_exact( + name, "up", candidate->up, control->up, + candidate->pair_count) && ok; + ok = compare_array_bit_exact( + name, "mid", candidate->mid, control->mid, + candidate->pair_count) && ok; + ok = compare_array_bit_exact( + name, "out", candidate->out, control->out, + candidate->out_count) && ok; + return ok; +} + +static int run_tail_cull_bit_exact( + const void *model, + uint64_t model_size, + uint64_t gate_offset, + uint64_t up_offset, + uint64_t down_offset, + uint64_t gate_expert_bytes, + uint64_t gate_row_bytes, + uint64_t down_expert_bytes, + uint64_t down_row_bytes, + uint32_t n_total_expert, + uint32_t cache_budget, + const float *x, + const int32_t *selected, + const float *weights) { + run_result control; + run_result candidate; + run_result repeat; + memset(&control, 0, sizeof(control)); + memset(&candidate, 0, sizeof(candidate)); + memset(&repeat, 0, sizeof(repeat)); + int ok = 0; + + clear_tail_cull_test_env(); + if (!result_alloc(&control, MAX_TOKENS) || + !result_alloc(&candidate, MAX_TOKENS) || + !result_alloc(&repeat, MAX_TOKENS)) { + goto cleanup; + } + + configure_tail_cull_test_env(false); + ds4_gpu_set_streaming_expert_cache_budget(cache_budget); + const int control_ok = run_once( + "tail-cull-control", &control, model, model_size, + gate_offset, up_offset, down_offset, gate_expert_bytes, + gate_row_bytes, down_expert_bytes, down_row_bytes, + n_total_expert, x, selected, weights, false); + + configure_tail_cull_test_env(true); + ds4_gpu_set_streaming_expert_cache_budget(cache_budget); + const int candidate_ok = run_once( + "tail-cull-candidate", &candidate, model, model_size, + gate_offset, up_offset, down_offset, gate_expert_bytes, + gate_row_bytes, down_expert_bytes, down_row_bytes, + n_total_expert, x, selected, weights, false); + + configure_tail_cull_test_env(true); + ds4_gpu_set_streaming_expert_cache_budget(cache_budget); + const int repeat_ok = run_once( + "tail-cull-repeat", &repeat, model, model_size, + gate_offset, up_offset, down_offset, gate_expert_bytes, + gate_row_bytes, down_expert_bytes, down_row_bytes, + n_total_expert, x, selected, weights, false); + + ok = control_ok && candidate_ok && repeat_ok; + if (ok) { + const int candidate_exact = compare_results_bit_exact( + "candidate-control", &candidate, &control); + const int repeat_exact = compare_results_bit_exact( + "repeat-control", &repeat, &control); + ok = candidate_exact && repeat_exact; + } + fprintf(stderr, + "IQ2_XXS/Q2_K Metal SSD address-MM tail-cull bit-exact %s\n", + ok ? "PASS" : "FAIL"); + +cleanup: + clear_tail_cull_test_env(); + result_free(&control); + result_free(&candidate); + result_free(&repeat); + return ok; +} + static int run_pair( const char *name, uint32_t tokens, @@ -1251,7 +1429,6 @@ static int run_256_expert_oracle( ok = 0; goto cleanup; } - uint32_t high_id_routes = 0; for (uint32_t token = 0; token < tokens; token++) { for (uint32_t k = 0; k < IN_DIM; k++) { const int32_t v = @@ -1261,26 +1438,64 @@ static int run_256_expert_oracle( } for (uint32_t slot = 0; slot < N_EXPERT; slot++) { const uint64_t route = (uint64_t)token * N_EXPERT + slot; - selected[route] = slot == 0u ? (int32_t)HIGH_EXPERT_ID : - (int32_t)(slot - 1u); + if (slot == 0u) { + /* Exactly 33 routes: one full tile plus a one-row tail. */ + selected[route] = (int32_t)HIGH_EXPERT_ID; + } else if (slot == 1u && token < 15u) { + selected[route] = 0; + } else if (slot == 2u && token == 0u) { + /* Duplicate expert zero within token zero. Together with the + * 15 routes above this makes the critical nr1 == 16 tile. */ + selected[route] = 0; + } else if (slot == 1u && token < 32u) { + /* Exactly 17 routes exercise the first non-culled row half. */ + selected[route] = 1; + } else { + /* Keep all remaining routes away from 0, 1, and 255 so the + * three boundary counts remain exact. */ + selected[route] = + (int32_t)(2u + (token * 17u + slot * 29u) % 252u); + } weights[route] = (float)(slot + 1u) / 21.0f; - if (selected[route] == (int32_t)HIGH_EXPERT_ID) high_id_routes++; + } + } + uint32_t routes_16 = 0; + uint32_t routes_17 = 0; + uint32_t high_id_routes = 0; + uint32_t duplicate_routes = 0; + for (uint32_t token = 0; token < tokens; token++) { + for (uint32_t slot = 0; slot < N_EXPERT; slot++) { + const int32_t id = selected[(uint64_t)token * N_EXPERT + slot]; + if (id == 0) routes_16++; + if (id == 1) routes_17++; + if (id == (int32_t)HIGH_EXPERT_ID) high_id_routes++; + for (uint32_t prior = 0; prior < slot; prior++) { + if (id == selected[(uint64_t)token * N_EXPERT + prior]) { + duplicate_routes++; + break; + } + } } } const uint32_t high_id_work_items = (high_id_routes + 31u) / 32u; - if (high_id_routes < 33u || high_id_work_items < 2u) { + if (routes_16 != 16u || routes_17 != 17u || high_id_routes != 33u || + duplicate_routes == 0u || high_id_work_items != 2u) { fprintf(stderr, - "IQ2_XXS SSD 256-expert route construction FAIL " - "id255_rows=%u work_items=%u\n", - high_id_routes, high_id_work_items); + "IQ2_XXS SSD tail-cull route construction FAIL " + "rows16=%u rows17=%u id255_rows=%u duplicates=%u " + "work_items=%u\n", + routes_16, routes_17, high_id_routes, duplicate_routes, + high_id_work_items); ok = 0; goto cleanup; } fprintf(stderr, - "IQ2_XXS SSD 256-expert address table prepared model_bytes=%llu " - "max_id=%u id255_rows=%u work_items=%u second_tile_r1=32\n", - (unsigned long long)model_size, HIGH_EXPERT_ID, - high_id_routes, high_id_work_items); + "IQ2_XXS SSD tail-cull routes PASS model_bytes=%llu " + "rows16=%u rows17=%u id255_rows=%u duplicates=%u " + "work_items=%u second_tile_r1=32\n", + (unsigned long long)model_size, routes_16, routes_17, + high_id_routes, duplicate_routes, + high_id_work_items); model_fd = mkstemp(tmp_path); if (model_fd < 0 || ftruncate(model_fd, (off_t)model_size) != 0) { @@ -1330,12 +1545,15 @@ static int run_256_expert_oracle( x, selected, weights); const int stats_ok = read_mm_stats(&after) && check_mm_stats_delta("id255-hot-33", &before, &after, tokens); - ok = pair_ok && stats_ok && ok; + const int tail_cull_ok = run_tail_cull_bit_exact( + model, model_size, gate_offset, up_offset, down_offset, + gate_expert_bytes, gate_row_bytes, down_expert_bytes, + down_row_bytes, N_TOTAL_EXPERT_256, N_TOTAL_EXPERT_256, + x, selected, weights); + ok = pair_ok && stats_ok && tail_cull_ok && ok; cleanup: - unsetenv("DS4_METAL_ENABLE_IQ2_XXS_SSD_PREFILL_MM"); - unsetenv("DS4_METAL_REQUIRE_IQ2_XXS_SSD_PREFILL_MM"); - unsetenv("DS4_METAL_DISABLE_IQ2_XXS_SSD_PREFILL_MM"); + clear_tail_cull_test_env(); if (backend_switched) { ds4_gpu_set_streaming_expert_cache_budget(N_TOTAL_EXPERT); ds4_gpu_set_streaming_expert_cache_expert_bytes( @@ -1456,6 +1674,7 @@ int main(void) { unsetenv("DS4_METAL_DISABLE_ROUTED_PAIR_SWIGLU_FUSION"); unsetenv("DS4_METAL_MOE_WRITE_CLAMPED_ACT"); unsetenv("DS4_METAL_GRAPH_DUMP_PREFIX"); + clear_tail_cull_test_env(); setenv("DS4_METAL_ENABLE_STREAMING_PREFILL_BATCH_SELECTED_ADDR", "1", 1); setenv("DS4_METAL_ENABLE_STREAMING_EXPERT_ADDR_TABLE", "1", 1); setenv("DS4_METAL_IQ2_XXS_SSD_PREFILL_MM_STATS", "1", 1); @@ -1578,9 +1797,7 @@ int main(void) { ok = full_ok && ok; } - unsetenv("DS4_METAL_ENABLE_IQ2_XXS_SSD_PREFILL_MM"); - unsetenv("DS4_METAL_REQUIRE_IQ2_XXS_SSD_PREFILL_MM"); - unsetenv("DS4_METAL_DISABLE_IQ2_XXS_SSD_PREFILL_MM"); + clear_tail_cull_test_env(); ds4_gpu_set_model_fd(-1); ds4_gpu_set_ssd_streaming(false); ds4_gpu_cleanup(); From 33f98659edaa38a0e57fd0e2775a3f3a772c2344 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Mon, 24 Aug 2026 15:50:09 +0200 Subject: [PATCH 088/189] Update metal/moe.metal Co-authored-by: Frank --- metal/moe.metal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metal/moe.metal b/metal/moe.metal index ac195458f..31bd27fef 100644 --- a/metal/moe.metal +++ b/metal/moe.metal @@ -9107,7 +9107,7 @@ kernel void kernel_mul_mm_id_addr( threadgroup float * temp_str = ((threadgroup float *) shmem) + 32*(sgitg&1) + (16*(sgitg >> 1))*NR0; if (mma_active) { - for (short i = 0; i < 8; i++) { + FOR_UNROLL (short i = 0; i < 8; i++) { simdgroup_store(mc[i], temp_str + 8*(i%4) + 8*NR0*(i/4), NR0, 0, false); From 19c1d2c323b04aa5734c34d3ee28509cd4f1d8d3 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Mon, 24 Aug 2026 15:50:21 +0200 Subject: [PATCH 089/189] Update metal/moe.metal Co-authored-by: Frank --- metal/moe.metal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metal/moe.metal b/metal/moe.metal index 31bd27fef..5e84a8e0a 100644 --- a/metal/moe.metal +++ b/metal/moe.metal @@ -9012,7 +9012,7 @@ kernel void kernel_mul_mm_id_addr( if (is_same::value && FC_mul_mm_bc_inp) { threadgroup_barrier(mem_flags::mem_threadgroup); - for (short i = 0; i < 16; i++) { + FOR_UNROLL (short i = 0; i < 16; i++) { const short sx = 2*il0 + i/8; const short sy = (tiitg/NL0)/8; From 77f41ec20a75d6e43f523b712b3e859177b1c7e8 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Mon, 24 Aug 2026 15:50:39 +0200 Subject: [PATCH 090/189] Update metal/moe.metal Co-authored-by: Frank --- metal/moe.metal | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/metal/moe.metal b/metal/moe.metal index 5e84a8e0a..ec66b3297 100644 --- a/metal/moe.metal +++ b/metal/moe.metal @@ -9136,6 +9136,12 @@ kernel void kernel_mul_mm_id_addr( i = (4*(nr0/4)) + tiisg; for (; i < nr0; i += 32) { *(D + i) = *(C + i); + FOR_UNROLL (int i = tiisg; i < nr0/4; i += 32) { + *(D4 + i) = *(C4 + i); + } + + FOR_UNROLL (int i = (4*(nr0/4)) + tiisg; i < nr0; i += 32) { + *(D + i) = *(C + i); } } } From 31a97a9737291bb2b23bfcc9147ee6f11187b4ba Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Mon, 24 Aug 2026 15:58:35 +0200 Subject: [PATCH 091/189] Update metal/moe.metal Co-authored-by: Frank --- metal/moe.metal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metal/moe.metal b/metal/moe.metal index ec66b3297..b33f585a4 100644 --- a/metal/moe.metal +++ b/metal/moe.metal @@ -9004,7 +9004,7 @@ kernel void kernel_mul_mm_id_addr( simdgroup_float8x8 mc[8]; - for (short i = 0; i < 8; i++){ + FOR_UNROLL (short i = 0; i < 8; i++) { mc[i] = make_filled_simdgroup_matrix(0.f); } From 0c8134206155ac1f9a43787d36fe017144a9f3c7 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Mon, 24 Aug 2026 15:58:50 +0200 Subject: [PATCH 092/189] Update metal/moe.metal Co-authored-by: Frank --- metal/moe.metal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metal/moe.metal b/metal/moe.metal index b33f585a4..7e4a1d207 100644 --- a/metal/moe.metal +++ b/metal/moe.metal @@ -9043,7 +9043,7 @@ kernel void kernel_mul_mm_id_addr( } if (FC_mul_mm_bc_inp) { - for (short i = 0; i < 8; ++i) { + FOR_UNROLL (short i = 0; i < 8; ++i) { const short sx = (tiitg%NL1); const short sy = (tiitg/NL1)/8; From e086800f0a1c0735b0c1fbd065aa7f6c2a444e38 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Mon, 24 Aug 2026 16:14:12 +0200 Subject: [PATCH 093/189] metal: fix grouped address-MM unroll integration --- metal/moe.metal | 8 -------- 1 file changed, 8 deletions(-) diff --git a/metal/moe.metal b/metal/moe.metal index 7e4a1d207..8b9200dd7 100644 --- a/metal/moe.metal +++ b/metal/moe.metal @@ -9128,14 +9128,6 @@ kernel void kernel_mul_mm_id_addr( threadgroup float * C = (threadgroup float *) shmem + j*NR0; threadgroup float4 * C4 = (threadgroup float4 *) C; - int i = tiisg; - for (; i < nr0/4; i += 32) { - *(D4 + i) = *(C4 + i); - } - - i = (4*(nr0/4)) + tiisg; - for (; i < nr0; i += 32) { - *(D + i) = *(C + i); FOR_UNROLL (int i = tiisg; i < nr0/4; i += 32) { *(D4 + i) = *(C4 + i); } From db6db1829329de26cb86c3c10d98c12b76073899 Mon Sep 17 00:00:00 2001 From: adamlawi Date: Mon, 24 Aug 2026 17:29:39 +0200 Subject: [PATCH 094/189] test(mmq): report mismatch magnitudes in the fused-raw parity case The compact-remap comparisons use memcmp, so a failure prints how many floats differ but not by how much. On a new architecture that makes triage guesswork: a last-bit rounding difference and a real divergence look identical. Add a diagnostics-only report next to each count: worst absolute and relative difference, ULP distance, how many differences are within one ULP, how many exceed 1e-3, and the value pair at the worst point. The pass/fail criterion is unchanged. On GB10 (sm_121) this separates the two: gate/up/mid stay under 1.1e-05 with about half the flagged elements within a single ULP, while the down leg has a tail of ~0.5% of elements above 1e-3. --- cuda/mmq/test/test_mmq_parity.cu | 53 ++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/cuda/mmq/test/test_mmq_parity.cu b/cuda/mmq/test/test_mmq_parity.cu index 6621e52b1..0f6e27ac5 100644 --- a/cuda/mmq/test/test_mmq_parity.cu +++ b/cuda/mmq/test/test_mmq_parity.cu @@ -1513,6 +1513,59 @@ bool run_iq2_xxs_q2_K_fused_raw_parity( const size_t up_remap_bad = mismatches(up_got, up_global_out); const size_t mid_remap_bad = mismatches(mid_got, mid_global_out); const size_t down_remap_bad = mismatches(down_got, down_global_out); + + // The comparisons above use memcmp, i.e. bit-exact equality with no + // tolerance. When a case fails, the counts alone cannot tell a last-bit + // rounding difference from a real divergence, which makes triage on a new + // architecture guesswork. Report the magnitude next to the count: worst + // absolute and relative difference, ULP distance, how many differences are + // within a single ULP, how many exceed 1e-3, and the value pair at the + // worst point. Diagnostics only - the pass/fail criterion is unchanged. + const auto diagnose = [](const char *tag, const std::vector & a, + const std::vector & b) { + double max_abs = 0.0, max_rel = 0.0; + float at_max_got = 0.0f, at_max_ref = 0.0f; + size_t bad = 0, first = (size_t)-1, ulp1 = 0, nonfinite = 0, big = 0; + long long max_ulp = 0; + for (size_t i = 0; i < a.size(); i++) { + if (std::memcmp(&a[i], &b[i], sizeof(float)) == 0) continue; + bad++; + if (first == (size_t)-1) first = i; + if (!std::isfinite(a[i]) || !std::isfinite(b[i])) nonfinite++; + const double d = std::fabs((double)a[i] - (double)b[i]); + const double den = std::fabs((double)b[i]); + if (d > max_abs) { max_abs = d; at_max_got = a[i]; at_max_ref = b[i]; } + if (den > 0.0 && d / den > max_rel) max_rel = d / den; + if (d > 1e-3) big++; + // Monotonic ordering of the float bit patterns, so the subtraction + // is a real ULP distance across the sign boundary as well. + int32_t ia = 0, ib = 0; + std::memcpy(&ia, &a[i], sizeof(int32_t)); + std::memcpy(&ib, &b[i], sizeof(int32_t)); + if (ia < 0) ia = (int32_t)0x80000000 - ia; + if (ib < 0) ib = (int32_t)0x80000000 - ib; + const long long u = std::llabs((long long)ia - (long long)ib); + if (u <= 1) ulp1++; + if (u > max_ulp) max_ulp = u; + } + if (bad == 0) { fprintf(stderr, " diag %-11s clean\n", tag); return; } + fprintf(stderr, + " diag %-11s bad=%zu/%zu (%.1f%%) max_abs=%.4g max_rel=%.4g " + "max_ulp=%lld within_1ulp=%zu nonfinite=%zu abs_gt_1e-3=%zu " + "at_max(got/ref)=%.9g/%.9g first=%zu got=%.9g ref=%.9g\n", + tag, bad, a.size(), 100.0 * (double)bad / (double)a.size(), + max_abs, max_rel, max_ulp, ulp1, nonfinite, big, + (double)at_max_got, (double)at_max_ref, + first, (double)a[first], (double)b[first]); + }; + diagnose("gate", gate_got, gate_ref); + diagnose("up", up_got, up_ref); + diagnose("mid", mid_got, mid_ref); + diagnose("down", down_got, down_ref); + diagnose("gate/remap", gate_got, gate_global_out); + diagnose("up/remap", up_got, up_global_out); + diagnose("mid/remap", mid_got, mid_global_out); + diagnose("down/remap", down_got, down_global_out); const bool ok = na_ok && rc_pair == 0 && swiglu_err == cudaSuccess && rc_down == 0 && rc_fused == 0 && rc_global == 0 && rc_global_reuse == 0 && rc_q81_grow == 0 && From 9aec14a6ce8f41a4cd854fd1cce6ced282800b65 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Mon, 24 Aug 2026 16:47:45 +0200 Subject: [PATCH 095/189] metal: unroll fixed simdgroup matrix loops --- metal/moe.metal | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/metal/moe.metal b/metal/moe.metal index 8b9200dd7..a0b6e2226 100644 --- a/metal/moe.metal +++ b/metal/moe.metal @@ -8772,7 +8772,7 @@ kernel void kernel_mul_mm_id( simdgroup_float8x8 mc[8]; - for (short i = 0; i < 8; i++){ + FOR_UNROLL (short i = 0; i < 8; i++) { mc[i] = make_filled_simdgroup_matrix(0.f); } @@ -8874,7 +8874,7 @@ kernel void kernel_mul_mm_id( threadgroup float * temp_str = ((threadgroup float *) shmem) + 32*(sgitg&1) + (16*(sgitg >> 1))*NR0; if (mma_active) { - for (short i = 0; i < 8; i++) { + FOR_UNROLL (short i = 0; i < 8; i++) { simdgroup_store(mc[i], temp_str + 8*(i%4) + 8*NR0*(i/4), NR0, 0, false); } } @@ -9234,7 +9234,7 @@ kernel void kernel_mul_mm_id_pair_swiglu_f16_impl( simdgroup_float8x8 mc_gate[8]; simdgroup_float8x8 mc_up[8]; - for (short i = 0; i < 8; i++) { + FOR_UNROLL (short i = 0; i < 8; i++) { mc_gate[i] = make_filled_simdgroup_matrix(0.f); mc_up[i] = make_filled_simdgroup_matrix(0.f); } @@ -9315,7 +9315,7 @@ kernel void kernel_mul_mm_id_pair_swiglu_f16_impl( temp_up + 32*(sgitg&1) + (16*(sgitg >> 1))*NR0; if (mma_active) { - for (short i = 0; i < 8; i++) { + FOR_UNROLL (short i = 0; i < 8; i++) { simdgroup_store(mc_gate[i], temp_gate_str + 8*(i%4) + 8*NR0*(i/4), NR0, 0, false); simdgroup_store(mc_up[i], temp_up_str + 8*(i%4) + 8*NR0*(i/4), NR0, 0, false); } @@ -9456,7 +9456,7 @@ kernel void kernel_mul_mm_id_pair_swiglu_f16_compact_tail_impl( simdgroup_float8x8 mc_gate[8]; simdgroup_float8x8 mc_up[8]; - for (short i = 0; i < 8; i++) { + FOR_UNROLL (short i = 0; i < 8; i++) { mc_gate[i] = make_filled_simdgroup_matrix(0.f); mc_up[i] = make_filled_simdgroup_matrix(0.f); } @@ -9540,7 +9540,7 @@ kernel void kernel_mul_mm_id_pair_swiglu_f16_compact_tail_impl( threadgroup float * temp_up_str = temp_up + 16*sgitg*NR0; if (mma_active) { - for (short i = 0; i < 8; i++) { + FOR_UNROLL (short i = 0; i < 8; i++) { simdgroup_store(mc_gate[i], temp_gate_str + 8*(i%4) + 8*NR0*(i/4), NR0, 0, false); simdgroup_store(mc_up[i], temp_up_str + 8*(i%4) + 8*NR0*(i/4), NR0, 0, false); } From 94a763dd377f2de0a786d0eb98464b74f56b14b5 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Tue, 25 Aug 2026 06:34:55 +0200 Subject: [PATCH 096/189] Update metal/moe.metal Co-authored-by: Frank --- metal/moe.metal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metal/moe.metal b/metal/moe.metal index a0b6e2226..91b145cd8 100644 --- a/metal/moe.metal +++ b/metal/moe.metal @@ -8780,7 +8780,7 @@ kernel void kernel_mul_mm_id( if (is_same::value && FC_mul_mm_bc_inp) { threadgroup_barrier(mem_flags::mem_threadgroup); - for (short i = 0; i < 16; i++) { + FOR_UNROLL (short i = 0; i < 16; i++) { const short sx = 2*il0 + i/8; const short sy = (tiitg/NL0)/8; From 654da4cb6a453c834f914aacb2be93ad6218d8c4 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Tue, 25 Aug 2026 06:35:05 +0200 Subject: [PATCH 097/189] Update metal/moe.metal Co-authored-by: Frank --- metal/moe.metal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metal/moe.metal b/metal/moe.metal index 91b145cd8..7eb7d755e 100644 --- a/metal/moe.metal +++ b/metal/moe.metal @@ -8811,7 +8811,7 @@ kernel void kernel_mul_mm_id( } if (FC_mul_mm_bc_inp) { - for (short i = 0; i < 8; ++i) { + FOR_UNROLL (short i = 0; i < 8; ++i) { const short sx = (tiitg%NL1); const short sy = (tiitg/NL1)/8; From 378f71810b3df1e37a083ee89a5ca4a15c98ef22 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Tue, 25 Aug 2026 06:35:16 +0200 Subject: [PATCH 098/189] Update metal/moe.metal Co-authored-by: Frank --- metal/moe.metal | 1 + 1 file changed, 1 insertion(+) diff --git a/metal/moe.metal b/metal/moe.metal index 7eb7d755e..a3a87cd14 100644 --- a/metal/moe.metal +++ b/metal/moe.metal @@ -8734,6 +8734,7 @@ kernel void kernel_mul_mm_id( device float * D = (device float *) dst + r0 + ide*args.ne0 + idt*args.ne1*args.ne0; + #pragma unroll(nr0) for (int i = tiisg; i < nr0; i += 32) { D[i] = 0.0f; } From 0442c937ce9ab3eebbb08f771da6b5267214fd04 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Tue, 25 Aug 2026 06:35:26 +0200 Subject: [PATCH 099/189] Update cuda/mmq/test/test_mmq_parity.cu Co-authored-by: Frank --- cuda/mmq/test/test_mmq_parity.cu | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cuda/mmq/test/test_mmq_parity.cu b/cuda/mmq/test/test_mmq_parity.cu index 0f6e27ac5..8b7468548 100644 --- a/cuda/mmq/test/test_mmq_parity.cu +++ b/cuda/mmq/test/test_mmq_parity.cu @@ -1548,7 +1548,10 @@ bool run_iq2_xxs_q2_K_fused_raw_parity( if (u <= 1) ulp1++; if (u > max_ulp) max_ulp = u; } - if (bad == 0) { fprintf(stderr, " diag %-11s clean\n", tag); return; } + if (bad == 0) { + fprintf(stderr, " diag %-11s clean\n", tag); + return; + } fprintf(stderr, " diag %-11s bad=%zu/%zu (%.1f%%) max_abs=%.4g max_rel=%.4g " "max_ulp=%lld within_1ulp=%zu nonfinite=%zu abs_gt_1e-3=%zu " From ea76e2a111d072e898b7dd2f5cb162ead436d604 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Tue, 25 Aug 2026 06:35:36 +0200 Subject: [PATCH 100/189] Update metal/moe.metal Co-authored-by: Frank --- metal/moe.metal | 1 + 1 file changed, 1 insertion(+) diff --git a/metal/moe.metal b/metal/moe.metal index a3a87cd14..9e9f50613 100644 --- a/metal/moe.metal +++ b/metal/moe.metal @@ -8726,6 +8726,7 @@ kernel void kernel_mul_mm_id( /* Unowned expert under the TP split: zero this tile's output rows so * the downstream swiglu/sum stages stay unchanged. Each (token,slot) * row belongs to exactly one expert, so nothing else writes them. */ + #pragma unroll(nr1) for (short j = sgitg; j < nr1; j += 4) { const int idj = ids_i32[route_base + r1 + j]; From a73fe1015f92636e8ca3a915370926a6215fc458 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Tue, 25 Aug 2026 06:35:46 +0200 Subject: [PATCH 101/189] Update metal/moe.metal Co-authored-by: Frank --- metal/moe.metal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metal/moe.metal b/metal/moe.metal index 9e9f50613..d0ab178cb 100644 --- a/metal/moe.metal +++ b/metal/moe.metal @@ -9566,7 +9566,7 @@ kernel void kernel_mul_mm_id_pair_swiglu_f16_compact_tail_impl( threadgroup float *Cu = temp_up + j*NR0; int i = tiisg; - for (; i < nr0; i += 32) { + FOR_UNROLL (int i = tiisg; i < nr0; i += 32) { float g = Cg[i]; float u = Cu[i]; if (c > 1.0e-6f) { From f2cdaf2ceaa8c3ccf9d5360d3b8b4d7313493c22 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Tue, 25 Aug 2026 06:36:16 +0200 Subject: [PATCH 102/189] Update metal/moe.metal Co-authored-by: Frank --- metal/moe.metal | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/metal/moe.metal b/metal/moe.metal index d0ab178cb..ba94d1e73 100644 --- a/metal/moe.metal +++ b/metal/moe.metal @@ -8904,6 +8904,13 @@ kernel void kernel_mul_mm_id( for (; i < nr0; i += 32) { *(D + i) = *(C + i); } + FOR_UNROLL (int i = tiisg; i < nr0/4; i += 32) { + *(D4 + i) = *(C4 + i); + } + + FOR_UNROLL (i = (4*(nr0/4)) + tiisg; i < nr0; i += 32) { + *(D + i) = *(C + i); + } } } From e80ba77da14670f4b628fd3743bfc6694726356b Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Tue, 25 Aug 2026 06:36:27 +0200 Subject: [PATCH 103/189] Update metal/moe.metal Co-authored-by: Frank --- metal/moe.metal | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/metal/moe.metal b/metal/moe.metal index ba94d1e73..0bddf0829 100644 --- a/metal/moe.metal +++ b/metal/moe.metal @@ -9550,8 +9550,9 @@ kernel void kernel_mul_mm_id_pair_swiglu_f16_compact_tail_impl( if (mma_active) { FOR_UNROLL (short i = 0; i < 8; i++) { - simdgroup_store(mc_gate[i], temp_gate_str + 8*(i%4) + 8*NR0*(i/4), NR0, 0, false); - simdgroup_store(mc_up[i], temp_up_str + 8*(i%4) + 8*NR0*(i/4), NR0, 0, false); + int si = (8 * (i & 3)) + (8 * NR0 * (i >> 2)); + simdgroup_store(mc_gate[i], temp_gate_str + si, NR0, 0, false); + simdgroup_store(mc_up[i], temp_up_str + si, NR0, 0, false); } } From 5ff3e2bd9027dc4ac7adc488e46d29c1f8d28091 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Tue, 25 Aug 2026 06:36:41 +0200 Subject: [PATCH 104/189] Update metal/moe.metal Co-authored-by: Frank --- metal/moe.metal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metal/moe.metal b/metal/moe.metal index 0bddf0829..3135441e1 100644 --- a/metal/moe.metal +++ b/metal/moe.metal @@ -9348,7 +9348,7 @@ kernel void kernel_mul_mm_id_pair_swiglu_f16_impl( threadgroup float *Cu = temp_up + j*NR0; int i = tiisg; - for (; i < nr0; i += 32) { + FOR_UNROLL (int i = tiisg; i < nr0; i += 32) { float g = Cg[i]; float u = Cu[i]; if (c > 1.0e-6f) { From 988f8d95d390b0dd723ac21422951077b41266d7 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Tue, 25 Aug 2026 06:36:55 +0200 Subject: [PATCH 105/189] Update metal/moe.metal Co-authored-by: Frank --- metal/moe.metal | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/metal/moe.metal b/metal/moe.metal index 3135441e1..a6c2ad978 100644 --- a/metal/moe.metal +++ b/metal/moe.metal @@ -9328,6 +9328,18 @@ kernel void kernel_mul_mm_id_pair_swiglu_f16_impl( simdgroup_store(mc_gate[i], temp_gate_str + 8*(i%4) + 8*NR0*(i/4), NR0, 0, false); simdgroup_store(mc_up[i], temp_up_str + 8*(i%4) + 8*NR0*(i/4), NR0, 0, false); } + int str_indice = 32*(sgitg&1) + (16*(sgitg >> 1))*NR0; + threadgroup float * temp_gate_str = + temp_gate + str_indice; + threadgroup float * temp_up_str = + temp_up + str_indice; + + if (mma_active) { + FOR_UNROLL (int i = 0; i < 8; i++) { + int si = (8 * (i & 3)) + (8 * NR0 * (i >> 2)); + simdgroup_store(mc_gate[i], temp_gate_str + si, NR0, 0, false); + simdgroup_store(mc_up[i], temp_up_str + si, NR0, 0, false); + } } threadgroup_barrier(mem_flags::mem_threadgroup); From 4f34ec7f7c9b7b423c57534838229365ce4e036e Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Tue, 25 Aug 2026 06:37:54 +0200 Subject: [PATCH 106/189] Update metal/moe.metal Co-authored-by: Frank --- metal/moe.metal | 1 + 1 file changed, 1 insertion(+) diff --git a/metal/moe.metal b/metal/moe.metal index a6c2ad978..e7d6e0789 100644 --- a/metal/moe.metal +++ b/metal/moe.metal @@ -8727,6 +8727,7 @@ kernel void kernel_mul_mm_id( * the downstream swiglu/sum stages stay unchanged. Each (token,slot) * row belongs to exactly one expert, so nothing else writes them. */ #pragma unroll(nr1) + #pragma unroll(nr1) for (short j = sgitg; j < nr1; j += 4) { const int idj = ids_i32[route_base + r1 + j]; From 00bf2d00f1eeef023bbf2306ba41282232fcc324 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Tue, 25 Aug 2026 06:53:37 +0200 Subject: [PATCH 107/189] metal: fix invalid loop unroll integration --- metal/moe.metal | 38 +++++++------------------------------- 1 file changed, 7 insertions(+), 31 deletions(-) diff --git a/metal/moe.metal b/metal/moe.metal index e7d6e0789..50d811d8d 100644 --- a/metal/moe.metal +++ b/metal/moe.metal @@ -8726,8 +8726,6 @@ kernel void kernel_mul_mm_id( /* Unowned expert under the TP split: zero this tile's output rows so * the downstream swiglu/sum stages stay unchanged. Each (token,slot) * row belongs to exactly one expert, so nothing else writes them. */ - #pragma unroll(nr1) - #pragma unroll(nr1) for (short j = sgitg; j < nr1; j += 4) { const int idj = ids_i32[route_base + r1 + j]; @@ -8736,7 +8734,6 @@ kernel void kernel_mul_mm_id( device float * D = (device float *) dst + r0 + ide*args.ne0 + idt*args.ne1*args.ne0; - #pragma unroll(nr0) for (int i = tiisg; i < nr0; i += 32) { D[i] = 0.0f; } @@ -8905,13 +8902,6 @@ kernel void kernel_mul_mm_id( for (; i < nr0; i += 32) { *(D + i) = *(C + i); } - FOR_UNROLL (int i = tiisg; i < nr0/4; i += 32) { - *(D4 + i) = *(C4 + i); - } - - FOR_UNROLL (i = (4*(nr0/4)) + tiisg; i < nr0; i += 32) { - *(D + i) = *(C + i); - } } } @@ -9319,25 +9309,13 @@ kernel void kernel_mul_mm_id_pair_swiglu_f16_impl( threadgroup float * temp_gate = (threadgroup float *) shmem; threadgroup float * temp_up = temp_gate + NR0*NR1; - threadgroup float * temp_gate_str = - temp_gate + 32*(sgitg&1) + (16*(sgitg >> 1))*NR0; - threadgroup float * temp_up_str = - temp_up + 32*(sgitg&1) + (16*(sgitg >> 1))*NR0; + const int str_index = 32*(sgitg&1) + (16*(sgitg >> 1))*NR0; + threadgroup float * temp_gate_str = temp_gate + str_index; + threadgroup float * temp_up_str = temp_up + str_index; if (mma_active) { FOR_UNROLL (short i = 0; i < 8; i++) { - simdgroup_store(mc_gate[i], temp_gate_str + 8*(i%4) + 8*NR0*(i/4), NR0, 0, false); - simdgroup_store(mc_up[i], temp_up_str + 8*(i%4) + 8*NR0*(i/4), NR0, 0, false); - } - int str_indice = 32*(sgitg&1) + (16*(sgitg >> 1))*NR0; - threadgroup float * temp_gate_str = - temp_gate + str_indice; - threadgroup float * temp_up_str = - temp_up + str_indice; - - if (mma_active) { - FOR_UNROLL (int i = 0; i < 8; i++) { - int si = (8 * (i & 3)) + (8 * NR0 * (i >> 2)); + const int si = (8 * (i & 3)) + (8 * NR0 * (i >> 2)); simdgroup_store(mc_gate[i], temp_gate_str + si, NR0, 0, false); simdgroup_store(mc_up[i], temp_up_str + si, NR0, 0, false); } @@ -9360,8 +9338,7 @@ kernel void kernel_mul_mm_id_pair_swiglu_f16_impl( threadgroup float *Cg = temp_gate + j*NR0; threadgroup float *Cu = temp_up + j*NR0; - int i = tiisg; - FOR_UNROLL (int i = tiisg; i < nr0; i += 32) { + for (int i = tiisg; i < nr0; i += 32) { float g = Cg[i]; float u = Cu[i]; if (c > 1.0e-6f) { @@ -9563,7 +9540,7 @@ kernel void kernel_mul_mm_id_pair_swiglu_f16_compact_tail_impl( if (mma_active) { FOR_UNROLL (short i = 0; i < 8; i++) { - int si = (8 * (i & 3)) + (8 * NR0 * (i >> 2)); + const int si = (8 * (i & 3)) + (8 * NR0 * (i >> 2)); simdgroup_store(mc_gate[i], temp_gate_str + si, NR0, 0, false); simdgroup_store(mc_up[i], temp_up_str + si, NR0, 0, false); } @@ -9586,8 +9563,7 @@ kernel void kernel_mul_mm_id_pair_swiglu_f16_compact_tail_impl( threadgroup float *Cg = temp_gate + j*NR0; threadgroup float *Cu = temp_up + j*NR0; - int i = tiisg; - FOR_UNROLL (int i = tiisg; i < nr0; i += 32) { + for (int i = tiisg; i < nr0; i += 32) { float g = Cg[i]; float u = Cu[i]; if (c > 1.0e-6f) { From a1e326f3676d46438b0f95a5a0454ec61c0797cd Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Tue, 25 Aug 2026 07:42:27 +0200 Subject: [PATCH 108/189] Update metal/moe.metal Co-authored-by: Frank --- metal/moe.metal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metal/moe.metal b/metal/moe.metal index 50d811d8d..0f775ed3f 100644 --- a/metal/moe.metal +++ b/metal/moe.metal @@ -9132,7 +9132,7 @@ kernel void kernel_mul_mm_id_addr( *(D4 + i) = *(C4 + i); } - FOR_UNROLL (int i = (4*(nr0/4)) + tiisg; i < nr0; i += 32) { + FOR_UNROLL (int i = nr0 + tiisg; i < nr0; i += 32) { *(D + i) = *(C + i); } } From 8e41f365d556e5a8053c48f2dd89d4bbda5823ba Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Tue, 25 Aug 2026 07:42:39 +0200 Subject: [PATCH 109/189] Update metal/moe.metal Co-authored-by: Frank --- metal/moe.metal | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/metal/moe.metal b/metal/moe.metal index 0f775ed3f..2a3024252 100644 --- a/metal/moe.metal +++ b/metal/moe.metal @@ -8901,6 +8901,12 @@ kernel void kernel_mul_mm_id( i = (4*(nr0/4)) + tiisg; for (; i < nr0; i += 32) { *(D + i) = *(C + i); + for (int i = tiisg; i < nr0/4; i += 32) { + *(D4 + i) = *(C4 + i); + } + + for (int i = tiisg + nr0; i < nr0; i += 32) { + *(D + i) = *(C + i); } } } From 39bbef31313253b894776785f115230c23654010 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Tue, 25 Aug 2026 10:52:57 +0200 Subject: [PATCH 110/189] Update metal/moe.metal Co-authored-by: Frank --- metal/moe.metal | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/metal/moe.metal b/metal/moe.metal index 2a3024252..56b57a3a5 100644 --- a/metal/moe.metal +++ b/metal/moe.metal @@ -8905,6 +8905,11 @@ kernel void kernel_mul_mm_id( *(D4 + i) = *(C4 + i); } + for (int i = tiisg + nr0; i < nr0; i += 32) { + for (int i = tiisg; i < nr0/4; i += 32) { + *(D4 + i) = *(C4 + i); + } + for (int i = tiisg + nr0; i < nr0; i += 32) { *(D + i) = *(C + i); } From 5dced7bc7076e8d1a6ac714cb2863d28ba25363a Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Tue, 25 Aug 2026 11:24:26 +0200 Subject: [PATCH 111/189] indexer: add Q4_K quantization and Metal support Allow direct F16-to-Q4_K requantization of indexer query projections, accept Q4_K at runtime, and add CPU/Metal production-shape regression coverage. Restore all as the default Make target after adding the quantizer helper target. --- .gitignore | 3 + Makefile | 27 +- ds4.c | 24 +- ds4.h | 1 + gguf-tools/README.md | 23 +- gguf-tools/deepseek4-quantize.c | 66 +++- tests/test_metal_indexer_q4.c | 434 ++++++++++++++++++++++++++ tests/test_metal_q4_streams.c | 11 + tests/test_quantizer_indexer_q4.c | 488 ++++++++++++++++++++++++++++++ 9 files changed, 1046 insertions(+), 31 deletions(-) create mode 100644 tests/test_metal_indexer_q4.c create mode 100644 tests/test_quantizer_indexer_q4.c diff --git a/.gitignore b/.gitignore index e21ca8074..4743e8cf6 100644 --- a/.gitignore +++ b/.gitignore @@ -25,12 +25,15 @@ /tests/cuda_long_context_smoke /tests/test_layer_pack /tests/test_metal_session_batch +/tests/test_metal_indexer_q4 +/tests/test_metal_q4_streams /tests/test_mxfp4_cuda /tests/test_mxfp4_dot /tests/test_mxfp4_metal /tests/test_q4k_dot /tests/test_sampling /tests/test_glm53_kda +/tests/test_quantizer_indexer_q4 *.o *.dSYM/ __pycache__/ diff --git a/Makefile b/Makefile index 03b3a72e1..d3c4df9a7 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,6 @@ CC ?= cc UNAME_S := $(shell uname -s) +.DEFAULT_GOAL := all ifeq ($(UNAME_S),Darwin) NATIVE_CPU_FLAG ?= -mcpu=native @@ -68,7 +69,16 @@ DS4_LINK_LIBS ?= $(CUDA_LDLIBS) METAL_LDLIBS := $(LDLIBS) endif -.PHONY: all help clean test environment-docs test-rocm test-glm53-kda-rocm test-metal-session-batch test-metal-session-batch-ssd test-metal-q4-streams test-metal-q4-attn-exactn test-metal-exactn-oracle test-metal-dspark-capture test-metal-iq2-midonly test-metal-iq2-ssd-grouped-mm test-metal-iq2-live-index test-mxfp4-metal test-mxfp4-cuda test-mxfp4-rocm test-mmq-parity-cuda test-rocm-q4-parity test-rocm-q4-dense test-rocm-q4-pair test-rocm-q4-prefill test-strix-rocm-q4-parity test-strix-rocm-q4-prefill test-strix-rocm-q4-prefill-long test-cuda-session-batch test-cuda-mixed-batch dspark-acceptance dspark-verify-depth rocm-dspark-acceptance rocm-dspark-verify-depth mtp-verify-depth cpu cuda cuda-spark cuda-generic cuda-regression strix-halo rocm +.PHONY: all help clean test environment-docs test-quantizer-indexer-q4 test-rocm test-glm53-kda-rocm test-metal-session-batch test-metal-session-batch-ssd test-metal-q4-streams test-metal-indexer-q4 test-metal-q4-attn-exactn test-metal-exactn-oracle test-metal-dspark-capture test-metal-iq2-midonly test-metal-iq2-ssd-grouped-mm test-metal-iq2-live-index test-mxfp4-metal test-mxfp4-cuda test-mxfp4-rocm test-mmq-parity-cuda test-rocm-q4-parity test-rocm-q4-dense test-rocm-q4-pair test-rocm-q4-prefill test-strix-rocm-q4-parity test-strix-rocm-q4-prefill test-strix-rocm-q4-prefill-long test-cuda-session-batch test-cuda-mixed-batch dspark-acceptance dspark-verify-depth rocm-dspark-acceptance rocm-dspark-verify-depth mtp-verify-depth cpu cuda cuda-spark cuda-generic cuda-regression strix-halo rocm + +gguf-tools/deepseek4-quantize: gguf-tools/deepseek4-quantize.c gguf-tools/quants.c gguf-tools/quants.h + $(MAKE) -C gguf-tools deepseek4-quantize + +tests/test_quantizer_indexer_q4: tests/test_quantizer_indexer_q4.c gguf-tools/quants.c gguf-tools/quants.h + $(CC) -O2 -Wall -Wextra -std=c99 -Igguf-tools -o $@ tests/test_quantizer_indexer_q4.c gguf-tools/quants.c $(LDLIBS) + +test-quantizer-indexer-q4: gguf-tools/deepseek4-quantize tests/test_quantizer_indexer_q4 + ./tests/test_quantizer_indexer_q4 ./gguf-tools/deepseek4-quantize ifeq ($(UNAME_S),Darwin) .PHONY: metal-decode-schedule-bench metal-prefill-variant-bench check-mxfp4-half-lut test-mxfp4-metal @@ -80,9 +90,11 @@ help: @echo " make Build Metal ./ds4, ./ds4-server, ./ds4-bench, ./ds4-eval, and ./ds4-agent" @echo " make cpu Build CPU-only ./ds4, ./ds4-server, ./ds4-bench, ./ds4-eval, and ./ds4-agent" @echo " make test Build and run tests" + @echo " make test-quantizer-indexer-q4 Check direct F16-to-Q4_K indexer conversion" @echo " make environment-docs Generate and verify the environment variable inventory" @echo " make test-metal-session-batch-ssd Exact-logit Metal SSD union control/candidate oracle" @echo " make test-metal-q4-streams Check resident Q4 Metal stream overlap" + @echo " make test-metal-indexer-q4 Check the production-shape Q4_K indexer projection" @echo " make test-metal-q4-attn-exactn Bitwise/canary oracle for M1-M4 SSD-prefill Q4 attention output" @echo " make test-metal-dspark-capture Check fused DSpark HC capture bitwise" @echo " make test-metal-iq2-midonly Check M1 IQ2 addr mid-only output and sentinels" @@ -161,6 +173,15 @@ test-metal-q4-streams: tests/test_metal_q4_streams env -u DS4_METAL_MODEL_UNTRACKED ./tests/test_metal_q4_streams DS4_METAL_MODEL_UNTRACKED=1 ./tests/test_metal_q4_streams +tests/test_metal_indexer_q4.o: tests/test_metal_indexer_q4.c ds4_gpu.h + $(CC) $(CFLAGS) -I. -c -o $@ $< + +tests/test_metal_indexer_q4: tests/test_metal_indexer_q4.o ds4_metal.o + $(CC) $(CFLAGS) -o $@ $^ $(METAL_LDLIBS) + +test-metal-indexer-q4: tests/test_metal_indexer_q4 + ./tests/test_metal_indexer_q4 + tests/test_metal_q4_attn_exactn.o: tests/test_metal_q4_attn_exactn.c ds4_gpu.h $(CC) $(CFLAGS) -I. -c -o $@ $< @@ -769,6 +790,7 @@ endif test: ds4_test ds4_agent_test ds4-eval q4k-dot-test mxfp4-dot-test \ tests/test_layer_pack tests/test_engine_mgpu_placement tests/test_gpu_args \ + tests/test_quantizer_indexer_q4 gguf-tools/deepseek4-quantize \ $(SAMPLING_TEST) ds4 ds4-server ds4-bench ds4-agent ./ds4-eval --self-test-extractors ./ds4_agent_test @@ -778,6 +800,7 @@ test: ds4_test ds4_agent_test ds4-eval q4k-dot-test mxfp4-dot-test \ ./tests/test_gpu_args ./tests/test_gpu_args_cli.sh ./tests/test_sampling + ./tests/test_quantizer_indexer_q4 ./gguf-tools/deepseek4-quantize dspark-acceptance: ds4 DS4_DSPARK_MODEL="$(DS4_DSPARK_MODEL)" \ @@ -813,4 +836,4 @@ mxfp4-dot-test: tests/test_mxfp4_dot.c ./tests/test_mxfp4_dot clean: - rm -f ds4 ds4-server ds4-bench ds4-eval ds4-agent ds4_cpu ds4_native ds4_server_test ds4_test ds4_agent_test gguf-tools/quality-testing/score_official gguf-tools/quality-testing/score_official.o speed-bench/metal_decode_schedule_bench speed-bench/metal_prefill_variant_bench speed-bench/*.o tests/test_q4k_dot tests/test_mxfp4_dot tests/test_mxfp4_metal tests/test_mxfp4_rocm tests/bench_mxfp4_rocm tests/test_mxfp4_cuda tests/test_rocm_q4_dense_pair tests/test_metal_session_batch tests/test_metal_q4_streams tests/test_metal_q4_attn_exactn tests/test_metal_exactn_oracle tests/test_metal_dspark_capture tests/test_metal_iq2_midonly tests/test_metal_iq2_ssd_grouped_mm tests/test_metal_iq2_live_index tests/test_glm53_kda tests/test_glm53_kda_rocm tests/test_glm53_vision_engine tests/test_glm53_vision_prompt tests/test_gpu_xdev tests/test_gpu_model_cache tests/test_gpu_lookup_cache_strict tests/test_engine_mgpu_refusal tests/test_engine_mgpu_runtime tests/test_engine_correctness tests/test_sampling tests/test_cuda_session_batch tests/test_cuda_mixed_batch tests/*.o *.o tests/cuda_long_context_smoke tests/cuda_long_context_smoke.o + rm -f ds4 ds4-server ds4-bench ds4-eval ds4-agent ds4_cpu ds4_native ds4_server_test ds4_test ds4_agent_test gguf-tools/quality-testing/score_official gguf-tools/quality-testing/score_official.o speed-bench/metal_decode_schedule_bench speed-bench/metal_prefill_variant_bench speed-bench/*.o tests/test_q4k_dot tests/test_mxfp4_dot tests/test_quantizer_indexer_q4 tests/test_mxfp4_metal tests/test_mxfp4_rocm tests/bench_mxfp4_rocm tests/test_mxfp4_cuda tests/test_rocm_q4_dense_pair tests/test_metal_session_batch tests/test_metal_q4_streams tests/test_metal_indexer_q4 tests/test_metal_q4_attn_exactn tests/test_metal_exactn_oracle tests/test_metal_dspark_capture tests/test_metal_iq2_midonly tests/test_metal_iq2_ssd_grouped_mm tests/test_metal_iq2_live_index tests/test_glm53_kda tests/test_glm53_kda_rocm tests/test_glm53_vision_engine tests/test_glm53_vision_prompt tests/test_gpu_xdev tests/test_gpu_model_cache tests/test_gpu_lookup_cache_strict tests/test_engine_mgpu_refusal tests/test_engine_mgpu_runtime tests/test_engine_correctness tests/test_sampling tests/test_cuda_session_batch tests/test_cuda_mixed_batch tests/*.o *.o tests/cuda_long_context_smoke tests/cuda_long_context_smoke.o diff --git a/ds4.c b/ds4.c index 5d5e676ce..97861f2fa 100644 --- a/ds4.c +++ b/ds4.c @@ -4638,20 +4638,28 @@ static void tensor_expect_plain_layout( tensor_expect_layout(t, t->type, ndim, d0, d1, d2); } -static bool tensor_type_is_f16_or_q8_0(uint32_t type) { - return type == DS4_TENSOR_F16 || type == DS4_TENSOR_Q8_0; +static bool tensor_type_is_indexer_q(uint32_t type) { + return type == DS4_TENSOR_F16 || + type == DS4_TENSOR_Q8_0 || + type == DS4_TENSOR_Q4_K; } -static void tensor_expect_f16_or_q8_0_layout( +#ifdef DS4_TEST_HOOKS +int ds4_test_indexer_q_type_supported(uint32_t type) { + return tensor_type_is_indexer_q(type) ? 1 : 0; +} +#endif + +static void tensor_expect_indexer_q_layout( const ds4_tensor *t, uint32_t ndim, uint64_t d0, uint64_t d1, uint64_t d2) { if (!t) ds4_die("internal error: missing tensor while validating layout"); - if (!tensor_type_is_f16_or_q8_0(t->type)) { + if (!tensor_type_is_indexer_q(t->type)) { fprintf(stderr, - "ds4: tensor %.*s has type %s, expected f16 or q8_0\n", + "ds4: tensor %.*s has type %s, expected f16, q8_0, or q4_K\n", (int)t->name.len, t->name.ptr, tensor_type_name(t->type)); @@ -5506,7 +5514,7 @@ static void weights_validate_layout( if (ratio == 4) { const uint64_t index_q_dim = (uint64_t)DS4_N_INDEXER_HEAD * DS4_N_INDEXER_HEAD_DIM; const uint64_t index_width = 2u * DS4_N_INDEXER_HEAD_DIM; - tensor_expect_f16_or_q8_0_layout(l->indexer_attn_q_b, 2, DS4_N_LORA_Q, index_q_dim, 0); + tensor_expect_indexer_q_layout(l->indexer_attn_q_b, 2, DS4_N_LORA_Q, index_q_dim, 0); tensor_expect_layout(l->indexer_proj, DS4_TENSOR_F16, 2, DS4_N_EMBD, DS4_N_INDEXER_HEAD, 0); tensor_expect_layout(l->indexer_compressor_ape, DS4_TENSOR_F16, 2, index_width, ratio, 0); tensor_expect_layout(l->indexer_compressor_kv, DS4_TENSOR_F16, 2, DS4_N_EMBD, index_width, 0); @@ -24581,10 +24589,10 @@ static bool metal_graph_encode_decode_layer_phase( g->layer_n_index_comp[il] > DS4_N_INDEXER_TOP_K) { const uint64_t indexer_q_dim = (uint64_t)DS4_N_INDEXER_HEAD * DS4_N_INDEXER_HEAD_DIM; if (!layer->indexer_attn_q_b || - !tensor_type_is_f16_or_q8_0(layer->indexer_attn_q_b->type) || + !tensor_type_is_indexer_q(layer->indexer_attn_q_b->type) || layer->indexer_attn_q_b->dim[0] != q_rank || layer->indexer_attn_q_b->dim[1] != indexer_q_dim) { - fprintf(stderr, "ds4: Metal graph indexer q projection expects F16 or Q8_0 weights\n"); + fprintf(stderr, "ds4: Metal graph indexer q projection expects F16, Q8_0, or Q4_K weights\n"); ok = false; } if (ok && (!layer->indexer_proj || diff --git a/ds4.h b/ds4.h index ba2195d71..879498d2a 100644 --- a/ds4.h +++ b/ds4.h @@ -466,6 +466,7 @@ int ds4_test_speculative_delta_sample(const float *target_logits, float *target_probs); int ds4_test_argmax_excluding_logits(const float *logits, uint32_t n_vocab, int excluded_id); +int ds4_test_indexer_q_type_supported(uint32_t type); uint64_t ds4_test_mixed_native_count(void); #if defined(__APPLE__) int ds4_test_q4_stream_overlap_policy( diff --git a/gguf-tools/README.md b/gguf-tools/README.md index 56330550f..e2d52f01a 100644 --- a/gguf-tools/README.md +++ b/gguf-tools/README.md @@ -66,14 +66,14 @@ family, plus enough free disk for the temporary output. Use `--dry-run` and `--compare-tensor` before starting a full write, and use `--overwrite` only when you really mean to replace an existing GGUF. -### Requantize A Q8 GGUF Directly +### Requantize a GGUF Directly Dense attention projections can be requantized directly from an existing GGUF -without the original Hugging Face safetensors. Direct requantization currently -supports `Q8_0 -> Q4_K`; tensors not selected by the policy are copied byte for -byte. The output path must differ from the source path. For a full run, write to -a temporary output name and rename it only after validation; an interrupted run -leaves a partial output file. +without the original Hugging Face safetensors. Direct requantization supports +`Q8_0 -> Q4_K` and `F16 -> Q4_K`; tensors not selected by the policy are copied +byte for byte. The output path must differ from the source path. For a full run, +write to a temporary output name and rename it only after validation; an +interrupted run leaves a partial output file. Validate the plan and all required imatrix entries first: @@ -97,6 +97,16 @@ gguf-tools/deepseek4-quantize \ --imatrix-strict ``` +Quantize only the sparse-attention indexer query projections while preserving +the F16 indexer compressors and weight projection: + +```sh +gguf-tools/deepseek4-quantize \ + --source-gguf /path/to/DeepSeek-V4-Flash-AProjQ4.gguf \ + --out /path/to/DeepSeek-V4-Flash-AProjQ4-IndexerQ4.gguf \ + --indexer-q q4_k +``` + Q2 routed experts with imatrix: ```sh @@ -134,6 +144,7 @@ You can override tensor families: --experts iq2_xxs --routed-w2 q2_k --attention-proj q8_0 +--indexer-q q4_k --shared q8_0 --output q8_0 ``` diff --git a/gguf-tools/deepseek4-quantize.c b/gguf-tools/deepseek4-quantize.c index aaabf84ef..a8ac8c2fb 100644 --- a/gguf-tools/deepseek4-quantize.c +++ b/gguf-tools/deepseek4-quantize.c @@ -1189,7 +1189,7 @@ typedef struct { typedef struct { ds4q_type routed_w1, routed_w2, routed_w3; - ds4q_type attention_proj, attention, shared, embedding, output, dense; + ds4q_type attention_proj, indexer_q, attention, shared, embedding, output, dense; type_override *overrides; int n_overrides; } quant_policy; @@ -1201,6 +1201,10 @@ static bool is_attention_projection(const char *name) { strstr(name, ".attn_output_b.weight"); } +static bool is_indexer_q_projection(const char *name) { + return strstr(name, ".indexer.attn_q_b.weight") != NULL; +} + static bool is_attention_tensor(const char *name) { return strstr(name, ".attn") || strstr(name, "attn_") || strstr(name, ".indexer") || strstr(name, "indexer_"); } @@ -1250,6 +1254,7 @@ static ds4q_type policy_type(const quant_policy *p, const char *name, const tens if (strcmp(name, "token_embd.weight") == 0 && p->embedding != DS4Q_TYPE_COUNT) return p->embedding; if (is_output_tensor(name) && p->output != DS4Q_TYPE_COUNT) return p->output; if (is_shared_expert(name) && p->shared != DS4Q_TYPE_COUNT) return p->shared; + if (is_indexer_q_projection(name) && p->indexer_q != DS4Q_TYPE_COUNT) return p->indexer_q; if (is_attention_projection(name) && p->attention_proj != DS4Q_TYPE_COUNT) return p->attention_proj; if (is_attention_tensor(name) && p->attention != DS4Q_TYPE_COUNT) return p->attention; if (p->dense != DS4Q_TYPE_COUNT) return p->dense; @@ -1988,23 +1993,46 @@ static void dequantize_q8_0_rows(const uint8_t *src, float *dst, } } +static void dequantize_f16_rows(const uint8_t *src, float *dst, + int64_t nrows, int64_t ncols) { + const size_t row_size = (size_t)ncols * sizeof(uint16_t); + for (int64_t r = 0; r < nrows; r++) { + const uint8_t *row = src + (size_t)r * row_size; + float *out = dst + (size_t)r * (size_t)ncols; + for (int64_t c = 0; c < ncols; c++) { + out[c] = ds4q_f16_to_f32(load_u16_le(row + (size_t)c * sizeof(uint16_t))); + } + } +} + +static bool direct_requant_supported(const tensor_meta *src, + const tensor_meta *dst) { + const int64_t block_size = ds4q_block_size(DS4Q_TYPE_Q4_K); + return dst->type == DS4Q_TYPE_Q4_K && + (src->type == DS4Q_TYPE_Q8_0 || src->type == DS4Q_TYPE_F16) && + block_size > 0 && src->ne[0] > 0 && src->ne[0] % block_size == 0; +} + static void validate_requant_plan(const gguf_file *src_g, const output_context *out_ctx, const imatrix_store *imatrix) { size_t changed = 0; + size_t changed_q8 = 0; + size_t changed_f16 = 0; for (uint64_t i = 0; i < out_ctx->n_tensors; i++) { const tensor_meta *src = &src_g->tensors[i]; const tensor_meta *dst = &out_ctx->tensors[i]; if (src->type == dst->type) continue; changed++; - if (src->type != DS4Q_TYPE_Q8_0 || dst->type != DS4Q_TYPE_Q4_K || - src->ne[0] % 256 != 0) { + if (!direct_requant_supported(src, dst)) { fprintf(stderr, "error: direct requantization unsupported for %s (%s -> %s)\n", src->name, ds4q_type_name(src->type), ds4q_type_name(dst->type)); exit(1); } + if (src->type == DS4Q_TYPE_Q8_0) changed_q8++; + else changed_f16++; const char *names[1] = { src->name }; (void)imatrix_find(imatrix, names, 1, src->ne[0], -1, 0); } @@ -2012,8 +2040,9 @@ static void validate_requant_plan(const gguf_file *src_g, die("direct requantization plan does not change any tensors"); } fprintf(stderr, - "validated direct GGUF requantization plan: %zu Q8_0 -> Q4_K tensors\n", - changed); + "validated direct GGUF requantization plan: %zu tensors -> Q4_K " + "(%zu Q8_0, %zu F16)\n", + changed, changed_q8, changed_f16); } static void write_requant_gguf(const gguf_file *src_g, const output_context *out_ctx, @@ -2054,8 +2083,7 @@ static void write_requant_gguf(const gguf_file *src_g, const output_context *out if (src->type == dst->type) { copy_bytes(fp, src_fp, src->size, src_g->path); } else { - if (src->type != DS4Q_TYPE_Q8_0 || dst->type != DS4Q_TYPE_Q4_K || - src->ne[0] % 256 != 0) { + if (!direct_requant_supported(src, dst)) { fprintf(stderr, "error: direct requantization unsupported for %s (%s -> %s)\n", src->name, ds4q_type_name(src->type), ds4q_type_name(dst->type)); exit(1); @@ -2063,10 +2091,10 @@ static void write_requant_gguf(const gguf_file *src_g, const output_context *out int64_t nrows = 1; for (int d = 1; d < src->n_dims; d++) nrows *= src->ne[d]; const int64_t ncols = src->ne[0]; - const size_t q8_row = ds4q_row_size(DS4Q_TYPE_Q8_0, ncols); + const size_t src_row = ds4q_row_size(src->type, ncols); const size_t q4_row = ds4q_row_size(DS4Q_TYPE_Q4_K, ncols); const int64_t batch_cap = 16; - uint8_t *q8 = xmalloc((size_t)batch_cap * q8_row); + uint8_t *encoded = xmalloc((size_t)batch_cap * src_row); float *f32 = xmalloc((size_t)batch_cap * (size_t)ncols * sizeof(float)); uint8_t *q4 = xmalloc((size_t)batch_cap * q4_row); const char *names[1] = { src->name }; @@ -2074,10 +2102,14 @@ static void write_requant_gguf(const gguf_file *src_g, const output_context *out ds4q_quantize_init(DS4Q_TYPE_Q4_K); for (int64_t row0 = 0; row0 < nrows; row0 += batch_cap) { const int64_t nr = nrows - row0 < batch_cap ? nrows - row0 : batch_cap; - if (fread(q8, q8_row, (size_t)nr, src_fp) != (size_t)nr) { - die_errno("read Q8_0 source tensor", src_g->path); + if (fread(encoded, src_row, (size_t)nr, src_fp) != (size_t)nr) { + die_errno("read source tensor", src_g->path); + } + if (src->type == DS4Q_TYPE_Q8_0) { + dequantize_q8_0_rows(encoded, f32, nr, ncols); + } else { + dequantize_f16_rows(encoded, f32, nr, ncols); } - dequantize_q8_0_rows(q8, f32, nr, ncols); const size_t wrote = ds4q_quantize_chunk( DS4Q_TYPE_Q4_K, f32, q4, 0, nr, ncols, imat); if (wrote != (size_t)nr * q4_row || @@ -2087,7 +2119,7 @@ static void write_requant_gguf(const gguf_file *src_g, const output_context *out } free(q4); free(f32); - free(q8); + free(encoded); } const size_t padded = ds4q_pad(dst->size, out_ctx->alignment); write_padding(fp, padded - dst->size); @@ -2855,7 +2887,7 @@ static void usage(const char *argv0) { printf("\nDeepSeek V4 Flash/Pro safetensors -> GGUF quantizer in plain C.\n\n"); printf("options:\n"); printf(" --hf DIR Hugging Face model directory with model.safetensors.index.json\n"); - printf(" --source-gguf FILE requantize directly from a GGUF (currently Q8_0 -> Q4_K)\n"); + printf(" --source-gguf FILE requantize Q8_0/F16 tensors directly to Q4_K\n"); printf(" --template FILE GGUF metadata/layout template required with --hf\n"); printf(" --out FILE output GGUF path\n"); printf(" --compare-gguf FILE reference GGUF for --compare-tensor; normal mode defaults to template\n"); @@ -2875,6 +2907,7 @@ static void usage(const char *argv0) { printf(" --routed-w2 TYPE routed down expert tensor type\n"); printf(" --routed-w3 TYPE routed up expert tensor type\n"); printf(" --attention-proj TYPE attn_q/kv/output projection type\n"); + printf(" --indexer-q TYPE indexer query projection type\n"); printf(" --attention TYPE other 2D attention/indexer/compressor type\n"); printf(" --shared TYPE shared expert tensor type\n"); printf(" --embedding TYPE token embedding type\n"); @@ -2975,7 +3008,8 @@ static void require_complete_gguf(const gguf_file *g) { static params parse_args(int argc, char **argv) { params p = {0}; p.policy.routed_w1 = p.policy.routed_w2 = p.policy.routed_w3 = DS4Q_TYPE_COUNT; - p.policy.attention_proj = p.policy.attention = p.policy.shared = DS4Q_TYPE_COUNT; + p.policy.attention_proj = p.policy.indexer_q = DS4Q_TYPE_COUNT; + p.policy.attention = p.policy.shared = DS4Q_TYPE_COUNT; p.policy.embedding = p.policy.output = p.policy.dense = DS4Q_TYPE_COUNT; p.n_experts = 0; p.n_threads = 8; @@ -3029,6 +3063,8 @@ static params parse_args(int argc, char **argv) { p.policy.routed_w3 = parse_type(need_value(argc, argv, &i, arg)); } else if (strcmp(arg, "--attention-proj") == 0 || strcmp(arg, "--attn-proj") == 0) { p.policy.attention_proj = parse_type(need_value(argc, argv, &i, arg)); + } else if (strcmp(arg, "--indexer-q") == 0) { + p.policy.indexer_q = parse_type(need_value(argc, argv, &i, arg)); } else if (strcmp(arg, "--attention") == 0) { p.policy.attention = parse_type(need_value(argc, argv, &i, arg)); } else if (strcmp(arg, "--shared") == 0) { diff --git a/tests/test_metal_indexer_q4.c b/tests/test_metal_indexer_q4.c new file mode 100644 index 000000000..759a039cd --- /dev/null +++ b/tests/test_metal_indexer_q4.c @@ -0,0 +1,434 @@ +#define _DARWIN_C_SOURCE + +/* GGUF-free production-shape oracle for the DeepSeek Flash Q4_K indexer + * query projection. The fixture deliberately uses the real 1024 -> 8192 + * geometry and checks the dispatch boundaries around the Metal matvec, + * generic matmul, and 32-token matrix paths. */ + +#include "ds4_gpu.h" + +#include +#include +#include +#include +#include +#include +#include + +bool ds4_log_is_tty(FILE *fp) { + (void)fp; + return false; +} + +#ifdef __APPLE__ + +enum { + Q4_K_TYPE = 12u, + QK_K = 256u, + INDEXER_IN_DIM = 1024u, + INDEXER_OUT_DIM = 8192u, + BLOCKS_PER_ROW = INDEXER_IN_DIM / QK_K, + GROUPS_PER_BLOCK = 8u, + GROUP_SIZE = 32u, + Q_PATTERNS = 16u, + MAX_TOKENS = 33u, + GUARD_FLOATS = 64u, +}; + +typedef struct { + uint16_t d; + uint16_t dmin; + uint8_t scales[12]; + uint8_t qs[QK_K / 2u]; +} block_q4_K; + +static const float k_abs_tolerance = 2.0e-3f; +static const float k_rel_tolerance = 3.0e-5f; +static const uint32_t k_poison_base = 0x7fc10000u; + +static void fail(const char *what) { + fprintf(stderr, "Metal indexer Q4_K oracle FAIL: %s\n", what); + exit(1); +} + +#define CHECK(expr, what) do { if (!(expr)) fail(what); } while (0) + +static uint64_t align_up(uint64_t value, uint64_t alignment) { + return (value + alignment - 1u) / alignment * alignment; +} + +static float f16_to_f32(uint16_t h) { + const uint32_t sign = (uint32_t)(h & 0x8000u) << 16u; + uint32_t exp = (h >> 10u) & 0x1fu; + uint32_t mant = h & 0x03ffu; + uint32_t bits; + + if (exp == 0u) { + if (mant == 0u) { + bits = sign; + } else { + exp = 1u; + while ((mant & 0x0400u) == 0u) { + mant <<= 1u; + exp--; + } + mant &= 0x03ffu; + bits = sign | ((exp + 127u - 15u) << 23u) | (mant << 13u); + } + } else if (exp == 31u) { + bits = sign | 0x7f800000u | (mant << 13u); + } else { + bits = sign | ((exp + 127u - 15u) << 23u) | (mant << 13u); + } + + float value; + memcpy(&value, &bits, sizeof(value)); + return value; +} + +static void q4_pack_scales(uint8_t packed[12], + const uint8_t scales[GROUPS_PER_BLOCK], + const uint8_t minima[GROUPS_PER_BLOCK]) { + memset(packed, 0, 12u); + for (uint32_t group = 0; group < 4u; group++) { + packed[group] = scales[group] & 63u; + packed[group + 4u] = minima[group] & 63u; + } + for (uint32_t group = 4u; group < GROUPS_PER_BLOCK; group++) { + packed[group + 4u] = (scales[group] & 15u) | + ((minima[group] & 15u) << 4u); + packed[group - 4u] |= (scales[group] >> 4u) << 6u; + packed[group] |= (minima[group] >> 4u) << 6u; + } +} + +static void q4_scale_min(const uint8_t packed[12], uint32_t group, + uint8_t *scale, uint8_t *minimum) { + if (group < 4u) { + *scale = packed[group] & 63u; + *minimum = packed[group + 4u] & 63u; + } else { + *scale = (packed[group + 4u] & 15u) | + ((packed[group - 4u] >> 6u) << 4u); + *minimum = (packed[group + 4u] >> 4u) | + ((packed[group] >> 6u) << 4u); + } +} + +static uint32_t q4_value(const block_q4_K *block, + uint32_t group, uint32_t lane) { + const uint32_t byte_offset = (group >> 1u) * GROUP_SIZE + lane; + const uint32_t shift = (group & 1u) * 4u; + return (block->qs[byte_offset] >> shift) & 15u; +} + +static void fill_q4_indexer(block_q4_K *matrix) { + CHECK(sizeof(block_q4_K) == 144u, "unexpected Q4_K block size"); + + for (uint32_t row = 0; row < INDEXER_OUT_DIM; row++) { + const uint32_t pattern = row & (Q_PATTERNS - 1u); + for (uint32_t block = 0; block < BLOCKS_PER_ROW; block++) { + block_q4_K *b = matrix + + (uint64_t)row * BLOCKS_PER_ROW + block; + uint8_t scales[GROUPS_PER_BLOCK]; + uint8_t minima[GROUPS_PER_BLOCK]; + + for (uint32_t group = 0; group < GROUPS_PER_BLOCK; group++) { + scales[group] = (uint8_t)(1u + + (row * 13u + block * 7u + group * 5u + 3u) % 31u); + minima[group] = (uint8_t)( + (row * 11u + block * 3u + group * 7u + 1u) % 17u); + } + q4_pack_scales(b->scales, scales, minima); + memset(b->qs, 0, sizeof(b->qs)); + + for (uint32_t group = 0; group < GROUPS_PER_BLOCK; group++) { + for (uint32_t lane = 0; lane < GROUP_SIZE; lane++) { + const uint32_t q = + (lane * 5u + pattern * 3u + block * 7u + + group * 11u) & 15u; + const uint32_t byte_offset = + (group >> 1u) * GROUP_SIZE + lane; + const uint32_t shift = (group & 1u) * 4u; + b->qs[byte_offset] |= (uint8_t)(q << shift); + } + } + + /* Exact binary scales: d=2^-8 and dmin=2^-9. */ + b->d = 0x1c00u; + b->dmin = 0x1800u; + } + } +} + +static void fill_inputs(float *inputs) { + for (uint32_t token = 0; token < MAX_TOKENS; token++) { + for (uint32_t col = 0; col < INDEXER_IN_DIM; col++) { + const uint32_t key = token * 131u + col * 17u + + ((col >> 3u) ^ (token * 29u)); + /* Multiples of 2^-6 are exactly representable by the half RHS + * used in the Metal matrix kernels. */ + inputs[(uint64_t)token * INDEXER_IN_DIM + col] = + (float)((int)(key % 129u) - 64) / 64.0f; + } + } +} + +static void build_cpu_dequant_oracle(const block_q4_K *matrix, + const float *inputs, + float *reference) { + float group_sum[MAX_TOKENS][BLOCKS_PER_ROW][GROUPS_PER_BLOCK]; + float q_dot[MAX_TOKENS][Q_PATTERNS] + [BLOCKS_PER_ROW][GROUPS_PER_BLOCK]; + + memset(group_sum, 0, sizeof(group_sum)); + memset(q_dot, 0, sizeof(q_dot)); + + /* q nibbles repeat every Q_PATTERNS rows, while scales/minima remain + * row-specific. Factoring these sums preserves the exact dequantized + * dot-product algebra and keeps the production-size oracle inexpensive. */ + for (uint32_t token = 0; token < MAX_TOKENS; token++) { + const float *x = inputs + (uint64_t)token * INDEXER_IN_DIM; + for (uint32_t block = 0; block < BLOCKS_PER_ROW; block++) { + for (uint32_t group = 0; group < GROUPS_PER_BLOCK; group++) { + const uint32_t col0 = block * QK_K + group * GROUP_SIZE; + float x_sum = 0.0f; + for (uint32_t lane = 0; lane < GROUP_SIZE; lane++) { + x_sum += x[col0 + lane]; + } + group_sum[token][block][group] = x_sum; + + for (uint32_t pattern = 0; pattern < Q_PATTERNS; pattern++) { + const block_q4_K *b = matrix + + (uint64_t)pattern * BLOCKS_PER_ROW + block; + float sum = 0.0f; + for (uint32_t lane = 0; lane < GROUP_SIZE; lane++) { + sum += (float)q4_value(b, group, lane) * + x[col0 + lane]; + } + q_dot[token][pattern][block][group] = sum; + } + } + } + } + + for (uint32_t token = 0; token < MAX_TOKENS; token++) { + for (uint32_t row = 0; row < INDEXER_OUT_DIM; row++) { + const uint32_t pattern = row & (Q_PATTERNS - 1u); + const block_q4_K *row_blocks = matrix + + (uint64_t)row * BLOCKS_PER_ROW; + float acc = 0.0f; + + for (uint32_t block = 0; block < BLOCKS_PER_ROW; block++) { + const block_q4_K *b = row_blocks + block; + const float d = f16_to_f32(b->d); + const float dmin = f16_to_f32(b->dmin); + for (uint32_t group = 0; group < GROUPS_PER_BLOCK; group++) { + uint8_t scale = 0; + uint8_t minimum = 0; + q4_scale_min(b->scales, group, &scale, &minimum); + acc += d * (float)scale * + q_dot[token][pattern][block][group] - + dmin * (float)minimum * + group_sum[token][block][group]; + } + } + reference[(uint64_t)token * INDEXER_OUT_DIM + row] = acc; + } + } +} + +static uint32_t poison_bits(uint64_t index) { + return k_poison_base + (uint32_t)(index & 0xffffu); +} + +static void poison(float *values, uint64_t count) { + for (uint64_t i = 0; i < count; i++) { + const uint32_t bits = poison_bits(i); + memcpy(&values[i], &bits, sizeof(bits)); + } +} + +static uint64_t count_canary_failures(const float *values, + uint64_t begin, uint64_t end, + uint64_t *first) { + uint64_t failures = 0; + *first = UINT64_MAX; + for (uint64_t i = begin; i < end; i++) { + uint32_t actual = 0; + memcpy(&actual, &values[i], sizeof(actual)); + if (actual != poison_bits(i)) { + if (*first == UINT64_MAX) *first = i; + failures++; + } + } + return failures; +} + +static bool compare_case(const float *actual, const float *reference, + uint32_t n_tokens) { + const uint64_t count = (uint64_t)n_tokens * INDEXER_OUT_DIM; + uint64_t raw_mismatches = 0; + uint64_t tolerance_failures = 0; + uint64_t worst = 0; + float max_abs = 0.0f; + float max_rel = 0.0f; + + for (uint64_t i = 0; i < count; i++) { + if (memcmp(&actual[i], &reference[i], sizeof(float)) != 0) { + raw_mismatches++; + } + const float diff = fabsf(actual[i] - reference[i]); + const float rel = diff / fmaxf(1.0f, fabsf(reference[i])); + const float limit = k_abs_tolerance + + k_rel_tolerance * fabsf(reference[i]); + if (diff > max_abs) { + max_abs = diff; + worst = i; + } + if (rel > max_rel) max_rel = rel; + if (!isfinite(actual[i]) || diff > limit) tolerance_failures++; + } + + fprintf(stderr, + "Metal indexer Q4_K n_tok=%u: raw=%llu/%llu " + "max_abs=%g max_rel=%g tol(abs=%g rel=%g) %s\n", + n_tokens, + (unsigned long long)raw_mismatches, + (unsigned long long)count, + max_abs, max_rel, k_abs_tolerance, k_rel_tolerance, + tolerance_failures == 0 ? "PASS" : "FAIL"); + if (tolerance_failures != 0) { + fprintf(stderr, + " worst token=%llu row=%llu gpu=%g cpu=%g delta=%g " + "failures=%llu\n", + (unsigned long long)(worst / INDEXER_OUT_DIM), + (unsigned long long)(worst % INDEXER_OUT_DIM), + actual[worst], reference[worst], + actual[worst] - reference[worst], + (unsigned long long)tolerance_failures); + } + return tolerance_failures == 0; +} + +int main(void) { + static const uint32_t token_cases[] = {1u, 8u, 9u, 31u, 32u, 33u}; + const uint64_t row_bytes = + (INDEXER_IN_DIM / QK_K) * sizeof(block_q4_K); + const uint64_t weight_bytes = (uint64_t)INDEXER_OUT_DIM * row_bytes; + const uint64_t page = (uint64_t)getpagesize(); + const uint64_t model_bytes = align_up(weight_bytes, page); + const uint64_t input_count = (uint64_t)MAX_TOKENS * INDEXER_IN_DIM; + const uint64_t max_output_count = + (uint64_t)MAX_TOKENS * INDEXER_OUT_DIM; + const uint64_t storage_count = + GUARD_FLOATS + max_output_count + GUARD_FLOATS; + + CHECK(Q4_K_TYPE == 12u, "Q4_K GGUF type must be 12"); + CHECK(row_bytes == 576u, "unexpected production Q4_K row size"); + CHECK(weight_bytes == 4718592u, + "unexpected production Q4_K indexer size"); + CHECK(unsetenv("DS4_METAL_DISABLE_Q4_MV_CLASSIC") == 0, + "clear classic Q4_K kill switch"); + CHECK(ds4_gpu_init() != 0, "Metal init"); + + void *model = NULL; + CHECK(posix_memalign(&model, (size_t)page, (size_t)model_bytes) == 0, + "page-aligned model allocation"); + memset(model, 0, (size_t)model_bytes); + fill_q4_indexer(model); + + float *inputs = malloc((size_t)input_count * sizeof(float)); + float *reference = malloc((size_t)max_output_count * sizeof(float)); + float *storage = malloc((size_t)storage_count * sizeof(float)); + CHECK(inputs && reference && storage, "host tensor allocation"); + fill_inputs(inputs); + build_cpu_dequant_oracle(model, inputs, reference); + + ds4_gpu_tensor *x = ds4_gpu_tensor_alloc(input_count * sizeof(float)); + ds4_gpu_tensor *out_base = + ds4_gpu_tensor_alloc(storage_count * sizeof(float)); + CHECK(x && out_base, "Metal tensor allocation"); + CHECK(ds4_gpu_tensor_write(x, 0, inputs, + input_count * sizeof(float)) != 0, + "input upload"); + CHECK(ds4_gpu_set_model_map(model, model_bytes) != 0, "model map"); + ds4_gpu_set_quality(false); + ds4_gpu_set_ssd_streaming(false); + + bool passed = true; + for (uint32_t case_i = 0; + case_i < sizeof(token_cases) / sizeof(token_cases[0]); + case_i++) { + const uint32_t n_tokens = token_cases[case_i]; + const uint64_t output_count = + (uint64_t)n_tokens * INDEXER_OUT_DIM; + const uint64_t output_bytes = output_count * sizeof(float); + + poison(storage, storage_count); + CHECK(ds4_gpu_tensor_write(out_base, 0, storage, + storage_count * sizeof(float)) != 0, + "output poison upload"); + ds4_gpu_tensor *out = ds4_gpu_tensor_view( + out_base, GUARD_FLOATS * sizeof(float), output_bytes); + CHECK(out != NULL, "exact-size output view"); + + /* This is the production entry point and Q4_K's GGUF type is 12. */ + CHECK(ds4_gpu_matmul_quant_tensor( + out, model, model_bytes, 0, Q4_K_TYPE, + INDEXER_IN_DIM, INDEXER_OUT_DIM, x, n_tokens) != 0, + "ds4_gpu_matmul_quant_tensor(type=12)"); + ds4_gpu_tensor_free(out); + + CHECK(ds4_gpu_tensor_read(out_base, 0, storage, + storage_count * sizeof(float)) != 0, + "output readback"); + + const bool values_ok = compare_case( + storage + GUARD_FLOATS, reference, n_tokens); + uint64_t first_prefix = UINT64_MAX; + uint64_t first_suffix = UINT64_MAX; + const uint64_t prefix_failures = count_canary_failures( + storage, 0, GUARD_FLOATS, &first_prefix); + const uint64_t suffix_begin = GUARD_FLOATS + output_count; + const uint64_t suffix_failures = count_canary_failures( + storage, suffix_begin, storage_count, &first_suffix); + const bool canary_ok = prefix_failures == 0 && suffix_failures == 0; + + fprintf(stderr, + "Metal indexer Q4_K n_tok=%u canary: prefix=%llu " + "suffix=%llu %s\n", + n_tokens, + (unsigned long long)prefix_failures, + (unsigned long long)suffix_failures, + canary_ok ? "PASS" : "FAIL"); + if (!canary_ok) { + fprintf(stderr, " first prefix=%llu first suffix=%llu\n", + (unsigned long long)first_prefix, + (unsigned long long)first_suffix); + } + if (!values_ok || !canary_ok) passed = false; + } + + ds4_gpu_tensor_free(out_base); + ds4_gpu_tensor_free(x); + ds4_gpu_cleanup(); + free(storage); + free(reference); + free(inputs); + free(model); + + fprintf(stderr, + "Metal indexer Q4_K production geometry 1024x8192: %s\n", + passed ? "PASS" : "FAIL"); + return passed ? 0 : 1; +} + +#else + +int main(void) { + fprintf(stderr, "Metal indexer Q4_K oracle SKIP: requires macOS\n"); + return 0; +} + +#endif diff --git a/tests/test_metal_q4_streams.c b/tests/test_metal_q4_streams.c index 25739f1eb..97c92dfa2 100644 --- a/tests/test_metal_q4_streams.c +++ b/tests/test_metal_q4_streams.c @@ -154,6 +154,16 @@ static void test_overlap_policy(void) { "count-bounds=1 resident-only=1 ssd-fallback=1 quality-fallback=1\n"); } +static void test_indexer_q_type_policy(void) { + if (!ds4_test_indexer_q_type_supported(1u) || + !ds4_test_indexer_q_type_supported(8u) || + !ds4_test_indexer_q_type_supported(Q4_K_TYPE) || + ds4_test_indexer_q_type_supported(0u) || + ds4_test_indexer_q_type_supported(2u)) { + fail("indexer query projection type policy"); + } +} + static uint64_t env_u64(const char *name, uint64_t fallback, uint64_t minimum, uint64_t maximum) { const char *value = getenv(name); @@ -741,6 +751,7 @@ int main(void) { "DS4_TEST_Q4_STREAM_TIMING_ITERS", 20u, 1u, 10000u); test_overlap_policy(); + test_indexer_q_type_policy(); fprintf(stderr, "Q4 stream oracle model_untracked=%s warmup=%llu iterations=%llu " diff --git a/tests/test_quantizer_indexer_q4.c b/tests/test_quantizer_indexer_q4.c new file mode 100644 index 000000000..f73593a6a --- /dev/null +++ b/tests/test_quantizer_indexer_q4.c @@ -0,0 +1,488 @@ +#define _DARWIN_C_SOURCE +#define _POSIX_C_SOURCE 200809L + +/* + * End-to-end regression for direct F16 -> Q4_K requantization of + * blk.*.indexer.attn_q_b.weight. + * + * The fixture is deliberately GGUF-library-free. It writes three tiny + * tensors, invokes the production CLI, then parses the output and compares + * the Q4_K payload with the public quantization facade. Seventeen rows cross + * the writer's 16-row conversion batch boundary. + */ + +#include "quants.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +enum { + GGUF_VERSION = 3, + GGUF_ALIGNMENT = 32, + FIXTURE_TENSORS = 3, + INDEXER_COLS = 256, + INDEXER_ROWS = 17, +}; + +static const char *const k_before_name = "test.sentinel_before"; +static const char *const k_indexer_name = + "blk.2.indexer.attn_q_b.weight"; +static const char *const k_after_name = "test.sentinel_after"; + +static uint8_t k_before_payload[28]; +static uint8_t k_after_payload[26]; + +typedef struct { + char *name; + uint32_t n_dims; + uint64_t dims[DS4Q_MAX_DIMS]; + uint32_t type; + uint64_t offset; + size_t size; +} tensor_info; + +typedef struct { + FILE *fp; + uint64_t data_offset; + tensor_info tensors[FIXTURE_TENSORS]; +} parsed_gguf; + +static int g_failures; + +static void fail(const char *what) { + fprintf(stderr, "test_quantizer_indexer_q4 FAIL: %s\n", what); + g_failures++; +} + +static uint64_t align_up(uint64_t value, uint64_t alignment) { + return (value + alignment - 1u) / alignment * alignment; +} + +static bool write_bytes(FILE *fp, const void *data, size_t size) { + return size == 0 || fwrite(data, 1, size, fp) == size; +} + +static bool write_u16_le(FILE *fp, uint16_t value) { + const uint8_t bytes[2] = { + (uint8_t)value, + (uint8_t)(value >> 8u), + }; + return write_bytes(fp, bytes, sizeof(bytes)); +} + +static bool write_u32_le(FILE *fp, uint32_t value) { + uint8_t bytes[4]; + for (uint32_t i = 0; i < 4u; i++) { + bytes[i] = (uint8_t)(value >> (8u * i)); + } + return write_bytes(fp, bytes, sizeof(bytes)); +} + +static bool write_u64_le(FILE *fp, uint64_t value) { + uint8_t bytes[8]; + for (uint32_t i = 0; i < 8u; i++) { + bytes[i] = (uint8_t)(value >> (8u * i)); + } + return write_bytes(fp, bytes, sizeof(bytes)); +} + +static bool write_string(FILE *fp, const char *value) { + const size_t len = strlen(value); + return write_u64_le(fp, len) && write_bytes(fp, value, len); +} + +static bool write_zeros(FILE *fp, uint64_t count) { + static const uint8_t zeros[256] = {0}; + while (count != 0) { + const size_t chunk = count < sizeof(zeros) ? (size_t)count : sizeof(zeros); + if (!write_bytes(fp, zeros, chunk)) return false; + count -= chunk; + } + return true; +} + +static uint16_t *make_indexer_f16(uint32_t cols, uint32_t rows) { + const size_t count = (size_t)cols * rows; + float *values = malloc(count * sizeof(*values)); + uint16_t *half = malloc(count * sizeof(*half)); + if (!values || !half) { + free(half); + free(values); + return NULL; + } + for (uint32_t row = 0; row < rows; row++) { + for (uint32_t col = 0; col < cols; col++) { + const int32_t raw = + (int32_t)((row * 67u + col * 29u + (row ^ col) * 3u) % 257u) - + 128; + values[(size_t)row * cols + col] = (float)raw / 32.0f; + } + } + ds4q_f32_to_f16_row(values, half, (int64_t)count); + free(values); + return half; +} + +static bool write_tensor_info(FILE *fp, const char *name, uint32_t n_dims, + uint64_t d0, uint64_t d1, uint32_t type, + uint64_t offset) { + return write_string(fp, name) && + write_u32_le(fp, n_dims) && + write_u64_le(fp, d0) && + (n_dims == 1u || write_u64_le(fp, d1)) && + write_u32_le(fp, type) && + write_u64_le(fp, offset); +} + +static bool write_fixture(const char *path, uint32_t cols, uint32_t rows, + const uint16_t *indexer_f16) { + const uint64_t before_offset = 0; + const uint64_t indexer_offset = + align_up(sizeof(k_before_payload), GGUF_ALIGNMENT); + const uint64_t indexer_bytes = (uint64_t)cols * rows * sizeof(uint16_t); + const uint64_t after_offset = + align_up(indexer_offset + indexer_bytes, GGUF_ALIGNMENT); + + FILE *fp = fopen(path, "wb"); + if (!fp) return false; + bool ok = write_bytes(fp, "GGUF", 4) && + write_u32_le(fp, GGUF_VERSION) && + write_u64_le(fp, FIXTURE_TENSORS) && + write_u64_le(fp, 0) && + write_tensor_info(fp, k_before_name, 1, + sizeof(k_before_payload) / sizeof(uint32_t), + 0, DS4Q_TYPE_F32, before_offset) && + write_tensor_info(fp, k_indexer_name, 2, cols, rows, + DS4Q_TYPE_F16, indexer_offset) && + write_tensor_info(fp, k_after_name, 1, + sizeof(k_after_payload) / sizeof(uint16_t), + 0, DS4Q_TYPE_F16, after_offset); + + const off_t metadata_end = ftello(fp); + if (metadata_end < 0) ok = false; + const uint64_t data_offset = metadata_end < 0 ? 0 : + align_up((uint64_t)metadata_end, GGUF_ALIGNMENT); + if (ok) ok = write_zeros(fp, data_offset - (uint64_t)metadata_end); + if (ok) ok = write_bytes(fp, k_before_payload, sizeof(k_before_payload)); + if (ok) ok = write_zeros(fp, indexer_offset - sizeof(k_before_payload)); + for (uint64_t i = 0; ok && i < (uint64_t)cols * rows; i++) { + ok = write_u16_le(fp, indexer_f16[i]); + } + if (ok) { + ok = write_zeros(fp, after_offset - indexer_offset - indexer_bytes) && + write_bytes(fp, k_after_payload, sizeof(k_after_payload)); + } + if (fclose(fp) != 0) ok = false; + return ok; +} + +static bool read_exact(FILE *fp, void *data, size_t size) { + return size == 0 || fread(data, 1, size, fp) == size; +} + +static bool read_u32_le(FILE *fp, uint32_t *value) { + uint8_t bytes[4]; + if (!read_exact(fp, bytes, sizeof(bytes))) return false; + *value = 0; + for (uint32_t i = 0; i < 4u; i++) { + *value |= (uint32_t)bytes[i] << (8u * i); + } + return true; +} + +static bool read_u64_le(FILE *fp, uint64_t *value) { + uint8_t bytes[8]; + if (!read_exact(fp, bytes, sizeof(bytes))) return false; + *value = 0; + for (uint32_t i = 0; i < 8u; i++) { + *value |= (uint64_t)bytes[i] << (8u * i); + } + return true; +} + +static char *read_string(FILE *fp) { + uint64_t len = 0; + if (!read_u64_le(fp, &len) || len > 4096u || len > SIZE_MAX - 1u) return NULL; + char *value = malloc((size_t)len + 1u); + if (!value) return NULL; + if (!read_exact(fp, value, (size_t)len)) { + free(value); + return NULL; + } + value[len] = '\0'; + return value; +} + +static size_t tensor_size(const tensor_info *tensor) { + if (tensor->n_dims == 0 || tensor->n_dims > DS4Q_MAX_DIMS || + tensor->dims[0] > INT64_MAX) { + return 0; + } + const size_t row = ds4q_row_size((ds4q_type)tensor->type, + (int64_t)tensor->dims[0]); + if (row == 0) return 0; + size_t size = row; + for (uint32_t d = 1; d < tensor->n_dims; d++) { + if (tensor->dims[d] > SIZE_MAX || + (tensor->dims[d] != 0 && size > SIZE_MAX / tensor->dims[d])) { + return 0; + } + size *= (size_t)tensor->dims[d]; + } + return size; +} + +static void close_parsed(parsed_gguf *gguf) { + if (gguf->fp) fclose(gguf->fp); + gguf->fp = NULL; + for (uint32_t i = 0; i < FIXTURE_TENSORS; i++) { + free(gguf->tensors[i].name); + gguf->tensors[i].name = NULL; + } +} + +static bool parse_output(const char *path, parsed_gguf *gguf) { + memset(gguf, 0, sizeof(*gguf)); + gguf->fp = fopen(path, "rb"); + if (!gguf->fp) return false; + + char magic[4]; + uint32_t version = 0; + uint64_t n_tensors = 0; + uint64_t n_kv = 0; + bool ok = read_exact(gguf->fp, magic, sizeof(magic)) && + memcmp(magic, "GGUF", sizeof(magic)) == 0 && + read_u32_le(gguf->fp, &version) && + read_u64_le(gguf->fp, &n_tensors) && + read_u64_le(gguf->fp, &n_kv) && + version == GGUF_VERSION && + n_tensors == FIXTURE_TENSORS && n_kv == 0; + + for (uint32_t i = 0; ok && i < FIXTURE_TENSORS; i++) { + tensor_info *tensor = &gguf->tensors[i]; + tensor->name = read_string(gguf->fp); + ok = tensor->name && read_u32_le(gguf->fp, &tensor->n_dims) && + tensor->n_dims >= 1u && tensor->n_dims <= DS4Q_MAX_DIMS; + for (uint32_t d = 0; ok && d < tensor->n_dims; d++) { + ok = read_u64_le(gguf->fp, &tensor->dims[d]); + } + ok = ok && read_u32_le(gguf->fp, &tensor->type) && + read_u64_le(gguf->fp, &tensor->offset); + if (ok) { + tensor->size = tensor_size(tensor); + ok = tensor->size != 0; + } + } + const off_t metadata_end = ok ? ftello(gguf->fp) : -1; + if (metadata_end < 0) ok = false; + if (ok) { + gguf->data_offset = align_up((uint64_t)metadata_end, GGUF_ALIGNMENT); + } else { + close_parsed(gguf); + } + return ok; +} + +static bool read_payload(const parsed_gguf *gguf, uint32_t index, + void *data, size_t size) { + if (!gguf->fp || index >= FIXTURE_TENSORS || + size != gguf->tensors[index].size || + gguf->data_offset > UINT64_MAX - gguf->tensors[index].offset) { + return false; + } + const uint64_t absolute = + gguf->data_offset + gguf->tensors[index].offset; + return absolute <= (uint64_t)INT64_MAX && + fseeko(gguf->fp, (off_t)absolute, SEEK_SET) == 0 && + read_exact(gguf->fp, data, size); +} + +static int run_quantizer(const char *tool, const char *source, + const char *output) { + const pid_t pid = fork(); + if (pid < 0) return -1; + if (pid == 0) { + execl(tool, tool, + "--source-gguf", source, + "--out", output, + "--indexer-q", "q4_k", + (char *)NULL); + _exit(127); + } + int status = 0; + while (waitpid(pid, &status, 0) < 0) { + if (errno != EINTR) return -1; + } + return WIFEXITED(status) ? WEXITSTATUS(status) : -1; +} + +static void verify_positive(const char *output, const uint16_t *half) { + parsed_gguf gguf; + if (!parse_output(output, &gguf)) { + fail("parse positive output"); + return; + } + + const tensor_info *before = &gguf.tensors[0]; + const tensor_info *indexer = &gguf.tensors[1]; + const tensor_info *after = &gguf.tensors[2]; + if (strcmp(before->name, k_before_name) != 0 || + before->type != DS4Q_TYPE_F32 || + before->size != sizeof(k_before_payload)) { + fail("before sentinel metadata"); + } + if (strcmp(indexer->name, k_indexer_name) != 0 || + indexer->type != DS4Q_TYPE_Q4_K || indexer->n_dims != 2u || + indexer->dims[0] != INDEXER_COLS || + indexer->dims[1] != INDEXER_ROWS || + indexer->size != INDEXER_ROWS * 144u) { + fail("indexer Q4_K metadata"); + } + if (strcmp(after->name, k_after_name) != 0 || + after->type != DS4Q_TYPE_F16 || + after->size != sizeof(k_after_payload)) { + fail("after sentinel metadata"); + } + + uint8_t before_actual[sizeof(k_before_payload)]; + uint8_t after_actual[sizeof(k_after_payload)]; + if (!read_payload(&gguf, 0, before_actual, sizeof(before_actual)) || + memcmp(before_actual, k_before_payload, sizeof(before_actual)) != 0) { + fail("before sentinel payload changed"); + } + if (!read_payload(&gguf, 2, after_actual, sizeof(after_actual)) || + memcmp(after_actual, k_after_payload, sizeof(after_actual)) != 0) { + fail("after sentinel payload changed"); + } + + const size_t count = (size_t)INDEXER_COLS * INDEXER_ROWS; + const size_t q4_size = INDEXER_ROWS * ds4q_row_size( + DS4Q_TYPE_Q4_K, INDEXER_COLS); + float *rounded = malloc(count * sizeof(*rounded)); + uint8_t *expected = malloc(q4_size); + uint8_t *actual = malloc(q4_size); + if (!rounded || !expected || !actual) { + fail("reference allocation"); + } else { + for (size_t i = 0; i < count; i++) { + rounded[i] = ds4q_f16_to_f32(half[i]); + } + ds4q_quantize_init(DS4Q_TYPE_Q4_K); + const size_t written = ds4q_quantize_chunk( + DS4Q_TYPE_Q4_K, rounded, expected, 0, + INDEXER_ROWS, INDEXER_COLS, NULL); + if (written != q4_size) { + fail("reference Q4_K size"); + } else if (!read_payload(&gguf, 1, actual, q4_size)) { + fail("read indexer payload"); + } else if (memcmp(actual, expected, q4_size) != 0) { + size_t first = 0; + while (first < q4_size && actual[first] == expected[first]) first++; + fprintf(stderr, + "test_quantizer_indexer_q4: first Q4 mismatch at %zu/%zu\n", + first, q4_size); + fail("indexer Q4_K payload mismatch"); + } + } + free(actual); + free(expected); + free(rounded); + close_parsed(&gguf); +} + +int main(int argc, char **argv) { + const char *tool = argc > 1 ? argv[1] : "./gguf-tools/deepseek4-quantize"; + if (argc > 2) { + fprintf(stderr, "usage: %s [deepseek4-quantize]\n", argv[0]); + return 2; + } + if (access(tool, X_OK) != 0) { + fprintf(stderr, "test_quantizer_indexer_q4: executable not found: %s\n", + tool); + return 2; + } + for (size_t i = 0; i < sizeof(k_before_payload); i++) { + k_before_payload[i] = (uint8_t)(0x31u + i * 7u); + } + for (size_t i = 0; i < sizeof(k_after_payload); i++) { + k_after_payload[i] = (uint8_t)(0xd3u - i * 5u); + } + + char tmpdir[] = "/tmp/ds4-indexer-q4.XXXXXX"; + if (!mkdtemp(tmpdir)) { + perror("mkdtemp"); + return 1; + } + char source[512]; + char output[512]; + char bad_source[512]; + char bad_output[512]; + if (snprintf(source, sizeof(source), "%s/source.gguf", tmpdir) >= + (int)sizeof(source) || + snprintf(output, sizeof(output), "%s/output.gguf", tmpdir) >= + (int)sizeof(output) || + snprintf(bad_source, sizeof(bad_source), "%s/bad-source.gguf", tmpdir) >= + (int)sizeof(bad_source) || + snprintf(bad_output, sizeof(bad_output), "%s/bad-output.gguf", tmpdir) >= + (int)sizeof(bad_output)) { + fail("temporary path too long"); + goto cleanup_dir; + } + + uint16_t *half = make_indexer_f16(INDEXER_COLS, INDEXER_ROWS); + if (!half || !write_fixture(source, INDEXER_COLS, INDEXER_ROWS, half)) { + fail("write positive fixture"); + } else { + const int status = run_quantizer(tool, source, output); + if (status != 0) { + fprintf(stderr, + "test_quantizer_indexer_q4: positive quantizer exit=%d\n", + status); + fail("positive quantizer invocation"); + } else { + verify_positive(output, half); + } + } + free(half); + + uint16_t *bad_half = make_indexer_f16(INDEXER_COLS - 1u, 2u); + if (!bad_half || + !write_fixture(bad_source, INDEXER_COLS - 1u, 2u, bad_half)) { + fail("write unaligned fixture"); + } else { + const int status = run_quantizer(tool, bad_source, bad_output); + if (status == 0) fail("unaligned width unexpectedly accepted"); + if (access(bad_output, F_OK) == 0) { + fail("unaligned conversion created output"); + } + } + free(bad_half); + + unlink(bad_output); + unlink(bad_source); + unlink(output); + unlink(source); +cleanup_dir: + if (rmdir(tmpdir) != 0 && errno != ENOENT) { + fail("remove temporary directory"); + } + + if (g_failures != 0) { + fprintf(stderr, "test_quantizer_indexer_q4: %d failure(s)\n", g_failures); + return 1; + } + fprintf(stderr, + "test_quantizer_indexer_q4 PASS f16_rows=17 q4_bytes=%zu " + "sentinels=2 unaligned_rejected=1\n", + (size_t)INDEXER_ROWS * ds4q_row_size(DS4Q_TYPE_Q4_K, INDEXER_COLS)); + return 0; +} From 71c9fed0976f1541599a7b04b3f81f312b83c735 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Tue, 25 Aug 2026 11:24:52 +0200 Subject: [PATCH 112/189] metal: centralize stable SiLU and repair MoE stores Load shared numerically stable sigmoid/SiLU helpers before all Metal kernels and cover the negative tail on device. Remove the duplicated routed-MM store loops introduced upstream and restore the scalar tail start for address-table SSD streaming. --- ds4_metal.m | 24 ++++-- metal/activations.metal | 35 +++++++++ metal/dense.metal | 6 +- metal/dsv4_misc.metal | 12 +-- metal/glu.metal | 4 +- metal/moe.metal | 53 +++++-------- metal/unary.metal | 4 +- tests/test_mxfp4_metal.c | 163 ++++++++++++++++++++++++++++++++++++++- 8 files changed, 242 insertions(+), 59 deletions(-) create mode 100644 metal/activations.metal diff --git a/ds4_metal.m b/ds4_metal.m index 72c9a4ba9..2044af825 100644 --- a/ds4_metal.m +++ b/ds4_metal.m @@ -4941,9 +4941,11 @@ static int ds4_gpu_model_map_log_enabled(void) { /* * Kernels are kept as separate files for review, then concatenated into one * Metal library. Environment overrides are still honored so a diagnostic - * run can swap one source file without changing the executable. + * run can swap one source file without changing the executable. A one-item + * entry is a fixed shared source; two-item entries are [override, source]. */ NSArray *> *required_sources = @[ + @[@"metal/activations.metal"], @[@"DS4_METAL_FLASH_ATTN_SOURCE", @"metal/flash_attn.metal"], @[@"DS4_METAL_DENSE_SOURCE", @"metal/dense.metal"], @[@"DS4_METAL_GLM53_BF16_SOURCE", @"metal/glm53_bf16.metal"], @@ -4970,13 +4972,16 @@ static int ds4_gpu_model_map_log_enabled(void) { NSMutableString *source = [NSMutableString stringWithString:base]; for (NSArray *spec in required_sources) { - const char *override_path = getenv([spec[0] UTF8String]); + NSString *override_name = spec.count > 1 ? spec[0] : nil; + NSString *source_path = spec.lastObject; + const char *override_path = + override_name ? getenv([override_name UTF8String]) : NULL; NSMutableArray *paths = [NSMutableArray array]; if (override_path && override_path[0]) { [paths addObject:[NSString stringWithUTF8String:override_path]]; } - [paths addObject:spec[1]]; - [paths addObject:[@"./" stringByAppendingString:spec[1]]]; + [paths addObject:source_path]; + [paths addObject:[@"./" stringByAppendingString:source_path]]; NSString *loaded = nil; NSString *loaded_path = nil; @@ -4997,9 +5002,14 @@ static int ds4_gpu_model_map_log_enabled(void) { } if (!loaded) { - fprintf(stderr, - "ds4: Metal source %s not found (set %s to override)\n", - [spec[1] UTF8String], [spec[0] UTF8String]); + if (override_name) { + fprintf(stderr, + "ds4: Metal source %s not found (set %s to override)\n", + [source_path UTF8String], [override_name UTF8String]); + } else { + fprintf(stderr, "ds4: Metal source %s not found\n", + [source_path UTF8String]); + } return nil; } [source appendFormat:@"\n// appended %@\n%@\n", loaded_path, loaded]; diff --git a/metal/activations.metal b/metal/activations.metal new file mode 100644 index 000000000..4e2bcf711 --- /dev/null +++ b/metal/activations.metal @@ -0,0 +1,35 @@ +// Shared activation functions. This source is loaded before every kernel file. + +// Keep finite activation behavior aligned with the stable CPU implementation. +// exp(-abs(x)) avoids overflow without a divergent branch on the sign. +static inline float ds4_sigmoid_stable(float x) { + const float e = exp(-fabs(x)); + const float numer = x >= 0.0f ? 1.0f : e; + return numer / (1.0f + e); +} + +static inline float4 ds4_sigmoid_stable(float4 x) { + const float4 e = exp(-fabs(x)); + const float4 numer = select(e, float4(1.0f), x >= 0.0f); + return numer / (1.0f + e); +} + +static inline float ds4_silu(float x) { + // Metal fast-math may flush exp(x) before x can lift a subnormal tail + // back into the normal range. Here 1 + exp(x) rounds to 1, so fold |x| + // into the exponent and produce the final magnitude without a subnormal + // intermediate. The precise calls also prevent unsafe fast-math rewrites. + if (x < -87.0f) { + return -precise::exp(x + precise::log(-x)); + } + return x * ds4_sigmoid_stable(x); +} + +static inline float4 ds4_silu(float4 x) { + float4 result = x * ds4_sigmoid_stable(x); + if (x.x < -87.0f) result.x = ds4_silu(x.x); + if (x.y < -87.0f) result.y = ds4_silu(x.y); + if (x.z < -87.0f) result.z = ds4_silu(x.z); + if (x.w < -87.0f) result.w = ds4_silu(x.w); + return result; +} diff --git a/metal/dense.metal b/metal/dense.metal index b56f50972..e5c0f1d5a 100644 --- a/metal/dense.metal +++ b/metal/dense.metal @@ -445,7 +445,7 @@ void kernel_dsv4_shared_gate_up_swiglu_q8_0_impl( g = min(g, clamp_value); u = clamp(u, -clamp_value, clamp_value); } - const float silu = g / (1.0f + exp(-g)); + const float silu = ds4_silu(g); mid_f32[out_row] = silu * u; } } @@ -647,7 +647,7 @@ kernel void kernel_dsv4_router_shared_gate_up_q8_0( g = min(g, clamp_value); u = clamp(u, -clamp_value, clamp_value); } - const float silu = g / (1.0f + exp(-g)); + const float silu = ds4_silu(g); mid_f32[out_row] = silu * u; } } @@ -1842,7 +1842,7 @@ void kernel_mul_mv_ext_q8_0_pair_swiglu_f32_impl( g = min(g, clamp_value); u = clamp(u, -clamp_value, clamp_value); } - const float silu = g / (1.0f + exp(-g)); + const float silu = ds4_silu(g); mid_f32[i01] = silu * u; } } diff --git a/metal/dsv4_misc.metal b/metal/dsv4_misc.metal index 4d7e8c60d..4d1536b7a 100644 --- a/metal/dsv4_misc.metal +++ b/metal/dsv4_misc.metal @@ -510,16 +510,6 @@ kernel void kernel_dsv4_router_weights_one( w[tid] = p[s[tid]] / sum * 1.5f; } -static inline float ds4_glm_router_sigmoid(float x) { - if (x >= 0.0f) { - const float e = exp(-x); - return 1.0f / (1.0f + e); - } else { - const float e = exp(x); - return e / (1.0f + e); - } -} - static inline bool ds4_glm_router_better( threadgroup const float *scores, int32_t a, @@ -4775,7 +4765,7 @@ kernel void kernel_glm_router_select_one( const uint n_expert = min(args.n_expert, 512u); const bool active = tid < n_expert; - const float p = active ? ds4_glm_router_sigmoid(token_logits[tid]) : 0.0f; + const float p = active ? ds4_sigmoid_stable(token_logits[tid]) : 0.0f; if (active) token_probs[tid] = p; sel_scores[tid] = active ? p + bias[tid] : -INFINITY; idx[tid] = (int32_t)tid; diff --git a/metal/glu.metal b/metal/glu.metal index fd5c6fd09..dde0c22fd 100644 --- a/metal/glu.metal +++ b/metal/glu.metal @@ -33,7 +33,7 @@ kernel void kernel_swiglu_f32( x1 = clamp(x1, -args.limit, args.limit); } - const float silu = x0 / (1.0f + exp(-x0)); + const float silu = ds4_silu(x0); dst_row[i0] = silu*x1*args.alpha; } @@ -58,6 +58,6 @@ kernel void kernel_swiglu_flat_f32( x1 = clamp(x1, -args.limit, args.limit); } - const float silu = x0 / (1.0f + exp(-x0)); + const float silu = ds4_silu(x0); dst_f32[i] = silu*x1*args.alpha; } diff --git a/metal/moe.metal b/metal/moe.metal index 56b57a3a5..654319a0d 100644 --- a/metal/moe.metal +++ b/metal/moe.metal @@ -442,7 +442,7 @@ static inline float ds4_glm_swiglu(float gate, float up, float limit) { gate = min(gate, limit); up = clamp(up, -limit, limit); } - return (gate / (1.0f + exp(-gate))) * up; + return ds4_silu(gate) * up; } @@ -506,7 +506,7 @@ kernel void kernel_dsv4_moe_swiglu_weight( up_row[i] = u; } } - const float silu = g / (1.0f + exp(-g)); + const float silu = ds4_silu(g); mid_row[i] = silu * u * route_weight; } } @@ -544,7 +544,7 @@ kernel void kernel_dsv4_moe_swiglu_weight_f16( up_row[i] = u; } } - const float silu = g / (1.0f + exp(-g)); + const float silu = ds4_silu(g); mid_row[i] = (half)(silu * u * route_weight); } } @@ -3541,7 +3541,7 @@ void kernel_mul_mv_iq2_xxs_pair_swiglu_mid_only_4096x2048_impl( g = min(g, c); u = clamp(u, -c, c); } - const float silu = g / (1.0f + exp(-g)); + const float silu = ds4_silu(g); mid_f32[first_row + row] = silu * u * route_weight; } } @@ -4330,7 +4330,7 @@ kernel void kernel_mul_mv_id_iq2_xxs_pair_swiglu_f32( } dst_gate_f32[out_row] = gate; dst_up_f32[out_row] = up; - const float silu = g / (1.0f + exp(-g)); + const float silu = ds4_silu(g); dst_mid_f32[out_row] = silu * u * route_weight; } @@ -4482,7 +4482,7 @@ kernel void kernel_mul_mv_id_iq2_xxs_pair_swiglu_pack2_overlap_f32( } dst_gate_f32[out_row] = gate; dst_up_f32[out_row] = up; - const float silu = g / (1.0f + exp(-g)); + const float silu = ds4_silu(g); dst_mid_f32[out_row] = silu * u * route_weight; } @@ -4578,7 +4578,7 @@ kernel void kernel_mul_mv_slots6_iq2_xxs_pair_swiglu_f32( g = min(g, c); u = clamp(u, -c, c); } - const float silu = g / (1.0f + exp(-g)); + const float silu = ds4_silu(g); mid_f32[out_row] = silu * u * route_weight; } } @@ -4668,7 +4668,7 @@ kernel void kernel_mul_mv_addr_iq2_xxs_pair_swiglu_f32( g = min(g, c); u = clamp(u, -c, c); } - const float silu = g / (1.0f + exp(-g)); + const float silu = ds4_silu(g); mid_f32[out_row] = silu * u * route_weight; } } @@ -4969,7 +4969,7 @@ kernel void kernel_mul_mv_addr_iq2_xxs_pair_swiglu_masked_f32( g = min(g, c); u = clamp(u, -c, c); } - const float silu = g / (1.0f + exp(-g)); + const float silu = ds4_silu(g); mid_f32[out_row] = silu * u * route_weight; } } @@ -5254,7 +5254,7 @@ kernel void kernel_mul_mv_id_q4_K_pair_swiglu_f32( } gate_f32[out_row] = gate; up_f32[out_row] = up; - const float silu = g / (1.0f + exp(-g)); + const float silu = ds4_silu(g); mid_f32[out_row] = silu * u * route_weight; } } @@ -5352,7 +5352,7 @@ void kernel_mul_mv_mxfp4_pair_swiglu_impl( } gate_f32[out_row] = gate; up_f32[out_row] = up; - mid_f32[out_row] = (g / (1.0f + exp(-g))) * u * route_weight; + mid_f32[out_row] = ds4_silu(g) * u * route_weight; } } } @@ -5533,7 +5533,7 @@ void kernel_mul_mv_mxfp4_pair_swiglu_static_impl( } gate_f32[out_row] = gate; up_f32[out_row] = up; - mid_f32[out_row] = (g / (1.0f + exp(-g))) * u * route_weight; + mid_f32[out_row] = ds4_silu(g) * u * route_weight; } } } @@ -5796,7 +5796,7 @@ kernel void kernel_mul_mv_table_q4_K_pair_swiglu_f32( } gate_f32[out_row] = gate; up_f32[out_row] = up; - const float silu = g / (1.0f + exp(-g)); + const float silu = ds4_silu(g); mid_f32[out_row] = silu * u * route_weight; } } @@ -5887,7 +5887,7 @@ kernel void kernel_mul_mv_addr_q4_K_pair_swiglu_f32( g = min(g, c); u = clamp(u, -c, c); } - const float silu = g / (1.0f + exp(-g)); + const float silu = ds4_silu(g); mid_f32[out_row] = silu * u * route_weight; } } @@ -6034,7 +6034,7 @@ kernel void kernel_mul_mv_slots6_q4_K_pair_swiglu_f32( g = min(g, c); u = clamp(u, -c, c); } - const float silu = g / (1.0f + exp(-g)); + const float silu = ds4_silu(g); mid_f32[out_row] = silu * u * route_weight; } } @@ -6206,7 +6206,7 @@ kernel void kernel_mul_mv_group6_q4_K_pair_swiglu_f32( g = min(g, c); u = clamp(u, -c, c); } - const float silu = g / (1.0f + exp(-g)); + const float silu = ds4_silu(g); mid_f32[out_row] = silu * u * route_weight; } } @@ -6330,7 +6330,7 @@ kernel void kernel_mul_mv_group8_q4_K_pair_swiglu_f32( g = min(g, c); u = clamp(u, -c, c); } - const float silu = g / (1.0f + exp(-g)); + const float silu = ds4_silu(g); mid_f32[out_row] = silu * u * route_weight; } } @@ -6513,7 +6513,7 @@ kernel void kernel_mul_mv_group_q4_K_pair_swiglu_f32( g = min(g, c); u = clamp(u, -c, c); } - const float silu = g / (1.0f + exp(-g)); + const float silu = ds4_silu(g); mid_f32[out_row] = silu * u * route_weight; } } @@ -8901,17 +8901,6 @@ kernel void kernel_mul_mm_id( i = (4*(nr0/4)) + tiisg; for (; i < nr0; i += 32) { *(D + i) = *(C + i); - for (int i = tiisg; i < nr0/4; i += 32) { - *(D4 + i) = *(C4 + i); - } - - for (int i = tiisg + nr0; i < nr0; i += 32) { - for (int i = tiisg; i < nr0/4; i += 32) { - *(D4 + i) = *(C4 + i); - } - - for (int i = tiisg + nr0; i < nr0; i += 32) { - *(D + i) = *(C + i); } } } @@ -9143,7 +9132,7 @@ kernel void kernel_mul_mm_id_addr( *(D4 + i) = *(C4 + i); } - FOR_UNROLL (int i = nr0 + tiisg; i < nr0; i += 32) { + FOR_UNROLL (int i = (4*(nr0/4)) + tiisg; i < nr0; i += 32) { *(D + i) = *(C + i); } } @@ -9356,7 +9345,7 @@ kernel void kernel_mul_mm_id_pair_swiglu_f16_impl( g = min(g, c); u = clamp(u, -c, c); } - const float silu = g / (1.0f + exp(-g)); + const float silu = ds4_silu(g); D[i] = (half)(silu * u * route_weight); } } @@ -9581,7 +9570,7 @@ kernel void kernel_mul_mm_id_pair_swiglu_f16_compact_tail_impl( g = min(g, c); u = clamp(u, -c, c); } - const float silu = g / (1.0f + exp(-g)); + const float silu = ds4_silu(g); D[i] = (half)(silu * u * route_weight); } } diff --git a/metal/unary.metal b/metal/unary.metal index 717fb350b..497b7b947 100644 --- a/metal/unary.metal +++ b/metal/unary.metal @@ -191,7 +191,7 @@ kernel void kernel_unary_impl( } if (FC_OP == OP_UNARY_NUM_SIGMOID) { - dst_ptr[i0] = (T) (1 / (1 + exp(-x))); + dst_ptr[i0] = (T) ds4_sigmoid_stable(x); } if (FC_OP == OP_UNARY_NUM_GELU) { @@ -207,7 +207,7 @@ kernel void kernel_unary_impl( } if (FC_OP == OP_UNARY_NUM_SILU) { - dst_ptr[i0] = (T) (x / (1 + exp(-x))); + dst_ptr[i0] = (T) ds4_silu(x); } if (FC_OP == OP_UNARY_NUM_ELU) { diff --git a/tests/test_mxfp4_metal.c b/tests/test_mxfp4_metal.c index bb1f37fed..80cb777f4 100644 --- a/tests/test_mxfp4_metal.c +++ b/tests/test_mxfp4_metal.c @@ -2,12 +2,15 @@ #include "ds4_gpu.h" +#include +#include #include #include #include #include #include #include +#include #include #define MXFP4_TYPE 39u @@ -43,6 +46,50 @@ static float e8m0_to_f32(uint8_t e) { return value; } +static uint32_t f32_bits(float value) { + uint32_t bits; + memcpy(&bits, &value, sizeof(bits)); + return bits; +} + +static int env_bool_enabled(const char *value) { + if (!value) return 0; + while (isspace((unsigned char)*value)) value++; + size_t len = strlen(value); + while (len > 0 && isspace((unsigned char)value[len - 1])) len--; + if (len == 0) return 1; + if ((len == 1 && value[0] == '0') || + (len == 5 && strncasecmp(value, "false", len) == 0) || + (len == 2 && strncasecmp(value, "no", len) == 0) || + (len == 3 && strncasecmp(value, "off", len) == 0)) { + return 0; + } + return 1; +} + +static float sigmoid_stable(float x) { + if (x >= 0.0f) { + const float e = expf(-x); + return 1.0f / (1.0f + e); + } + const float e = expf(x); + return e / (1.0f + e); +} + +static float silu_stable(float x) { + return x * sigmoid_stable(x); +} + +static float silu_reference_f64(float x) { + const double xd = (double)x; + if (xd >= 0.0) { + const double e = exp(-xd); + return (float)(xd / (1.0 + e)); + } + const double e = exp(xd); + return (float)(xd * (e / (1.0 + e))); +} + static float dot_mxfp4(const block_mxfp4 *row, const float *x) { float sum = 0.0f; for (uint32_t block = 0; block < DIM / QK_MXFP4; block++) { @@ -107,6 +154,116 @@ static int compare_values(const char *name, const float *actual, return max_abs <= tolerance; } +static int test_silu_stable_metal(void) { + const float gate[] = { + -100.0f, + -92.0f, + -89.0f, + nextafterf(-87.0f, -FLT_MAX), + -87.0f, + nextafterf(-87.0f, FLT_MAX), + -80.0f, + -10.0f, + -1.0f, + -0.0f, + 0.0f, + 1.0f, + 10.0f, + }; + float up[sizeof(gate) / sizeof(gate[0])]; + float actual[sizeof(gate) / sizeof(gate[0])]; + float expected[sizeof(gate) / sizeof(gate[0])]; + const uint32_t count = (uint32_t)(sizeof(gate) / sizeof(gate[0])); + const int require_ieee = env_bool_enabled(getenv("DS4_METAL_MATH_SAFE")); + + for (uint32_t i = 0; i < count; i++) { + up[i] = 1.0f; + expected[i] = silu_reference_f64(gate[i]); + } + + ds4_gpu_tensor *gate_tensor = ds4_gpu_tensor_alloc(sizeof(gate)); + ds4_gpu_tensor *up_tensor = ds4_gpu_tensor_alloc(sizeof(up)); + ds4_gpu_tensor *out_tensor = ds4_gpu_tensor_alloc(sizeof(actual)); + int ok = gate_tensor && up_tensor && out_tensor; + if (ok) ok = ds4_gpu_tensor_write(gate_tensor, 0, gate, sizeof(gate)); + if (ok) ok = ds4_gpu_tensor_write(up_tensor, 0, up, sizeof(up)); + if (ok) { + ok = ds4_gpu_swiglu_tensor(out_tensor, gate_tensor, up_tensor, + count, 0.0f, 1.0f); + } + if (ok) ok = ds4_gpu_tensor_read(out_tensor, 0, actual, sizeof(actual)); + + for (uint32_t i = 0; ok && i < count; i++) { + const uint32_t actual_bits = f32_bits(actual[i]); + const uint32_t expected_bits = f32_bits(expected[i]); + const uint32_t actual_abs_bits = actual_bits & 0x7fffffffu; + const float expected_abs = fabsf(expected[i]); + if ((actual_bits & 0x7f800000u) == 0x7f800000u) { + fprintf(stderr, + "MXFP4 Metal stable SiLU mismatch i=%u x=%g actual=%g expected=%g\n", + i, gate[i], actual[i], expected[i]); + ok = 0; + break; + } + if (expected_abs > 0.0f && expected_abs < FLT_MIN) { + // The final SiLU value itself is subnormal and may be flushed by + // Metal even in math-safe mode. Safe mode must retain its sign. + if (actual_abs_bits == 0u) { + const int sign_mismatch = + require_ieee && ((actual_bits ^ expected_bits) >> 31u) != 0u; + if (sign_mismatch) { + fprintf(stderr, + "MXFP4 Metal stable SiLU subnormal-zero sign mismatch " + "i=%u x=%g actual_bits=0x%08x expected_bits=0x%08x\n", + i, gate[i], actual_bits, expected_bits); + ok = 0; + } + continue; + } + + const double rel = fabs(((double)actual[i] - (double)expected[i]) / + (double)expected[i]); + if ((actual_bits >> 31u) == 0u || rel > 2.0e-2) { + fprintf(stderr, + "MXFP4 Metal stable SiLU subnormal mismatch " + "i=%u x=%g actual=%g expected=%g rel=%g strict=%d\n", + i, gate[i], actual[i], expected[i], rel, require_ieee); + ok = 0; + } + continue; + } + if (expected[i] == 0.0f) { + const int sign_mismatch = + require_ieee && ((actual_bits ^ expected_bits) >> 31u) != 0u; + if (actual_abs_bits != 0u || sign_mismatch) { + fprintf(stderr, + "MXFP4 Metal stable SiLU zero mismatch " + "i=%u x=%g actual_bits=0x%08x expected_bits=0x%08x strict=%d\n", + i, gate[i], actual_bits, expected_bits, require_ieee); + ok = 0; + } + continue; + } + + const double rel = fabs(((double)actual[i] - (double)expected[i]) / + (double)expected[i]); + const double rel_limit = expected_abs < 1.0e-20f ? 5.0e-4 : 5.0e-5; + if (rel > rel_limit) { + fprintf(stderr, + "MXFP4 Metal stable SiLU relative mismatch " + "i=%u x=%g actual=%g expected=%g rel=%g limit=%g\n", + i, gate[i], actual[i], expected[i], rel, rel_limit); + ok = 0; + } + } + + ds4_gpu_tensor_free(out_tensor); + ds4_gpu_tensor_free(up_tensor); + ds4_gpu_tensor_free(gate_tensor); + if (ok) fprintf(stderr, "MXFP4 Metal numerically stable SiLU tail PASS\n"); + return ok; +} + int main(void) { const uint64_t page = (uint64_t)getpagesize(); const uint64_t row_bytes = @@ -178,7 +335,7 @@ int main(void) { x); const float g = fminf(gate_ref[pair], 7.0f); const float u = fmaxf(-7.0f, fminf(up_ref[pair], 7.0f)); - mid_ref[pair] = (g / (1.0f + expf(-g))) * u * weights[slot]; + mid_ref[pair] = silu_stable(g) * u * weights[slot]; } } for (uint32_t row = 0; row < DIM; row++) { @@ -191,7 +348,9 @@ int main(void) { } } - int ok = ds4_gpu_init() && ds4_gpu_set_model_map(model, model_size); + int ok = ds4_gpu_init(); + if (ok) ok = test_silu_stable_metal(); + if (ok) ok = ds4_gpu_set_model_map(model, model_size); ok = ok && ds4_gpu_test_decode_pipeline_fast_lookup(); if (ok) { fprintf(stderr, From 9d3e719f35edffce6dbf97287a76e6f58666f14e Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Tue, 25 Aug 2026 17:17:28 +0200 Subject: [PATCH 113/189] metal: speed up IQ2_XXS MoE prefill Use an exact binary16 dequantization LUT and split the Metal MPP routed matmul into double-buffered 16-row tiles with tail culling. Preserve the packed route map integration from this branch. Adapted from antirez/ds4#864. Co-authored-by: cole --- ds4_metal.m | 15 +- metal/moe.metal | 379 ++++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 345 insertions(+), 49 deletions(-) diff --git a/ds4_metal.m b/ds4_metal.m index 2044af825..49be6eee8 100644 --- a/ds4_metal.m +++ b/ds4_metal.m @@ -46109,6 +46109,10 @@ int ds4_gpu_routed_moe_batch_tensor( false, q4_batch_table_queue_residency); } else if (use_mm_id) { + /* kernel_mul_mm_id_mpp double-buffers tiles (12 KiB). */ + const NSUInteger gate_up_tg = + (ds4_gpu_routed_mm_mpp_mask() != 0 && + gate_type == DS4_METAL_TENSOR_IQ2_XXS) ? 12288u : 8192u; /* * The routed pair ids are the same for gate, up, and down. Build * the expert-major work map once, then reuse it for all three @@ -46158,7 +46162,7 @@ int ds4_gpu_routed_moe_batch_tensor( ds4_gpu_tensor_offset(x), gatebuf, ds4_gpu_tensor_offset(gate), - 8192u); + gate_up_tg); DS4_METAL_PROFILE_MOE_STAGE("gate"); } if (ok && !use_mm_id_pair_swiglu) { @@ -46171,7 +46175,7 @@ int ds4_gpu_routed_moe_batch_tensor( ds4_gpu_tensor_offset(x), upbuf, ds4_gpu_tensor_offset(up), - 8192u); + gate_up_tg); DS4_METAL_PROFILE_MOE_STAGE("up"); } } else if (use_tiny_pair_swiglu) { @@ -46441,6 +46445,11 @@ int ds4_gpu_routed_moe_batch_tensor( down_smem, 2); } else if (use_mm_id) { + const NSUInteger down_tg = + ((ds4_gpu_routed_mm_mpp_mask() & 4) != 0 && + request_mid_f16 && + (down_type == DS4_METAL_TENSOR_Q2_K || + down_type == DS4_METAL_TENSOR_IQ2_XXS)) ? 12288u : 8192u; ok = ds4_gpu_encode_mul_mm_id_mapped_tile(cb, down_mm_pipeline, &down_mm_args, @@ -46450,7 +46459,7 @@ int ds4_gpu_routed_moe_batch_tensor( ds4_gpu_tensor_offset(mid), down_dst, down_dst_off, - 8192u); + down_tg); } else { ok = ds4_gpu_encode_mul_mv_id(cb, down_mv_pipeline, diff --git a/metal/moe.metal b/metal/moe.metal index 654319a0d..6b11e7431 100644 --- a/metal/moe.metal +++ b/metal/moe.metal @@ -375,6 +375,266 @@ static constant ulong ds4_metal_iq2xxs_grid[256] = { #define ksigns_iq2xs ds4_metal_ksigns_iq2xs #define iq2xxs_grid ds4_metal_iq2xxs_grid +// iq2xxs_grid entry (row) x byte: exact binary16 of 0.25 * grid byte. +static constant ushort ds4_metal_iq2xxs_half_values[256][8] = { + { 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u }, + { 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u }, + { 0x4640u, 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u }, + { 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u }, + { 0x4960u, 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u }, + { 0x4640u, 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u }, + { 0x4000u, 0x4640u, 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u }, + { 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u }, + { 0x4960u, 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u }, + { 0x4000u, 0x4960u, 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u }, + { 0x4960u, 0x4960u, 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u }, + { 0x4640u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4000u }, + { 0x4000u, 0x4640u, 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4000u }, + { 0x4000u, 0x4000u, 0x4640u, 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4000u }, + { 0x4000u, 0x4960u, 0x4640u, 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4000u }, + { 0x4640u, 0x4000u, 0x4960u, 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4000u }, + { 0x4000u, 0x4640u, 0x4960u, 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4000u }, + { 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4000u }, + { 0x4960u, 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4000u }, + { 0x4960u, 0x4960u, 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4000u }, + { 0x4960u, 0x4000u, 0x4960u, 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4000u }, + { 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4000u }, + { 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4000u }, + { 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4000u }, + { 0x4640u, 0x4640u, 0x4640u, 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4000u }, + { 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4640u, 0x4000u, 0x4000u, 0x4000u }, + { 0x4000u, 0x4640u, 0x4000u, 0x4960u, 0x4640u, 0x4000u, 0x4000u, 0x4000u }, + { 0x4000u, 0x4960u, 0x4640u, 0x4960u, 0x4640u, 0x4000u, 0x4000u, 0x4000u }, + { 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4000u }, + { 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4000u }, + { 0x4960u, 0x4000u, 0x4960u, 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4000u }, + { 0x4960u, 0x4000u, 0x4000u, 0x4960u, 0x4960u, 0x4000u, 0x4000u, 0x4000u }, + { 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4000u }, + { 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4000u }, + { 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4000u }, + { 0x4640u, 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4000u }, + { 0x4000u, 0x4640u, 0x4960u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4000u }, + { 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4640u, 0x4000u, 0x4000u }, + { 0x4960u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4640u, 0x4000u, 0x4000u }, + { 0x4000u, 0x4960u, 0x4000u, 0x4640u, 0x4000u, 0x4640u, 0x4000u, 0x4000u }, + { 0x4000u, 0x4000u, 0x4960u, 0x4640u, 0x4000u, 0x4640u, 0x4000u, 0x4000u }, + { 0x4640u, 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4640u, 0x4000u, 0x4000u }, + { 0x4000u, 0x4640u, 0x4000u, 0x4960u, 0x4000u, 0x4640u, 0x4000u, 0x4000u }, + { 0x4000u, 0x4000u, 0x4640u, 0x4960u, 0x4000u, 0x4640u, 0x4000u, 0x4000u }, + { 0x4000u, 0x4640u, 0x4960u, 0x4960u, 0x4000u, 0x4640u, 0x4000u, 0x4000u }, + { 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4640u, 0x4000u, 0x4000u }, + { 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4640u, 0x4000u, 0x4000u }, + { 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4640u, 0x4640u, 0x4000u, 0x4000u }, + { 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4640u, 0x4640u, 0x4000u, 0x4000u }, + { 0x4960u, 0x4640u, 0x4000u, 0x4640u, 0x4640u, 0x4640u, 0x4000u, 0x4000u }, + { 0x4640u, 0x4960u, 0x4960u, 0x4640u, 0x4640u, 0x4640u, 0x4000u, 0x4000u }, + { 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4640u, 0x4640u, 0x4000u, 0x4000u }, + { 0x4640u, 0x4000u, 0x4640u, 0x4960u, 0x4640u, 0x4640u, 0x4000u, 0x4000u }, + { 0x4640u, 0x4960u, 0x4000u, 0x4000u, 0x4960u, 0x4640u, 0x4000u, 0x4000u }, + { 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4960u, 0x4640u, 0x4000u, 0x4000u }, + { 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4960u, 0x4640u, 0x4000u, 0x4000u }, + { 0x4000u, 0x4640u, 0x4000u, 0x4960u, 0x4960u, 0x4640u, 0x4000u, 0x4000u }, + { 0x4000u, 0x4640u, 0x4960u, 0x4960u, 0x4960u, 0x4640u, 0x4000u, 0x4000u }, + { 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4000u }, + { 0x4640u, 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4000u }, + { 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4000u }, + { 0x4000u, 0x4640u, 0x4640u, 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4000u }, + { 0x4000u, 0x4960u, 0x4960u, 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4000u }, + { 0x4640u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4960u, 0x4000u, 0x4000u }, + { 0x4000u, 0x4640u, 0x4000u, 0x4640u, 0x4000u, 0x4960u, 0x4000u, 0x4000u }, + { 0x4000u, 0x4000u, 0x4640u, 0x4640u, 0x4000u, 0x4960u, 0x4000u, 0x4000u }, + { 0x4960u, 0x4000u, 0x4640u, 0x4640u, 0x4000u, 0x4960u, 0x4000u, 0x4000u }, + { 0x4000u, 0x4960u, 0x4000u, 0x4960u, 0x4000u, 0x4960u, 0x4000u, 0x4000u }, + { 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4640u, 0x4960u, 0x4000u, 0x4000u }, + { 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4640u, 0x4960u, 0x4000u, 0x4000u }, + { 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4960u, 0x4000u, 0x4000u }, + { 0x4000u, 0x4640u, 0x4640u, 0x4000u, 0x4960u, 0x4960u, 0x4000u, 0x4000u }, + { 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4000u }, + { 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4000u }, + { 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4000u }, + { 0x4640u, 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4000u }, + { 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4640u, 0x4000u }, + { 0x4000u, 0x4000u, 0x4960u, 0x4640u, 0x4000u, 0x4000u, 0x4640u, 0x4000u }, + { 0x4000u, 0x4640u, 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4640u, 0x4000u }, + { 0x4000u, 0x4000u, 0x4640u, 0x4960u, 0x4000u, 0x4000u, 0x4640u, 0x4000u }, + { 0x4640u, 0x4640u, 0x4640u, 0x4960u, 0x4000u, 0x4000u, 0x4640u, 0x4000u }, + { 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4640u, 0x4000u }, + { 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4640u, 0x4000u }, + { 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4640u, 0x4000u, 0x4640u, 0x4000u }, + { 0x4000u, 0x4000u, 0x4640u, 0x4640u, 0x4640u, 0x4000u, 0x4640u, 0x4000u }, + { 0x4960u, 0x4960u, 0x4640u, 0x4640u, 0x4640u, 0x4000u, 0x4640u, 0x4000u }, + { 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4640u, 0x4000u, 0x4640u, 0x4000u }, + { 0x4000u, 0x4640u, 0x4960u, 0x4000u, 0x4960u, 0x4000u, 0x4640u, 0x4000u }, + { 0x4640u, 0x4640u, 0x4000u, 0x4640u, 0x4960u, 0x4000u, 0x4640u, 0x4000u }, + { 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4640u, 0x4000u }, + { 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4640u, 0x4000u }, + { 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4640u, 0x4640u, 0x4000u }, + { 0x4640u, 0x4640u, 0x4960u, 0x4000u, 0x4000u, 0x4640u, 0x4640u, 0x4000u }, + { 0x4640u, 0x4960u, 0x4000u, 0x4640u, 0x4000u, 0x4640u, 0x4640u, 0x4000u }, + { 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4640u, 0x4640u, 0x4000u }, + { 0x4000u, 0x4960u, 0x4640u, 0x4000u, 0x4640u, 0x4640u, 0x4640u, 0x4000u }, + { 0x4960u, 0x4000u, 0x4960u, 0x4640u, 0x4640u, 0x4640u, 0x4640u, 0x4000u }, + { 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4640u, 0x4640u, 0x4000u }, + { 0x4960u, 0x4640u, 0x4640u, 0x4000u, 0x4960u, 0x4640u, 0x4640u, 0x4000u }, + { 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4640u, 0x4000u }, + { 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4640u, 0x4000u }, + { 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4960u, 0x4640u, 0x4000u }, + { 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4960u, 0x4640u, 0x4000u }, + { 0x4640u, 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4960u, 0x4640u, 0x4000u }, + { 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4960u, 0x4640u, 0x4000u }, + { 0x4640u, 0x4640u, 0x4000u, 0x4000u, 0x4640u, 0x4960u, 0x4640u, 0x4000u }, + { 0x4000u, 0x4000u, 0x4960u, 0x4960u, 0x4640u, 0x4960u, 0x4640u, 0x4000u }, + { 0x4640u, 0x4000u, 0x4640u, 0x4640u, 0x4960u, 0x4960u, 0x4640u, 0x4000u }, + { 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4000u }, + { 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4000u }, + { 0x4960u, 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4000u }, + { 0x4000u, 0x4640u, 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4960u, 0x4000u }, + { 0x4640u, 0x4000u, 0x4960u, 0x4640u, 0x4000u, 0x4000u, 0x4960u, 0x4000u }, + { 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4960u, 0x4000u }, + { 0x4960u, 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4960u, 0x4000u }, + { 0x4640u, 0x4960u, 0x4960u, 0x4000u, 0x4640u, 0x4000u, 0x4960u, 0x4000u }, + { 0x4000u, 0x4960u, 0x4000u, 0x4640u, 0x4640u, 0x4000u, 0x4960u, 0x4000u }, + { 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4960u, 0x4000u }, + { 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4960u, 0x4000u }, + { 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4960u, 0x4000u }, + { 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4960u, 0x4000u }, + { 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4640u, 0x4960u, 0x4000u }, + { 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4640u, 0x4960u, 0x4000u }, + { 0x4960u, 0x4640u, 0x4640u, 0x4640u, 0x4000u, 0x4640u, 0x4960u, 0x4000u }, + { 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4640u, 0x4960u, 0x4000u }, + { 0x4640u, 0x4000u, 0x4000u, 0x4640u, 0x4640u, 0x4640u, 0x4960u, 0x4000u }, + { 0x4000u, 0x4640u, 0x4960u, 0x4640u, 0x4640u, 0x4640u, 0x4960u, 0x4000u }, + { 0x4000u, 0x4000u, 0x4640u, 0x4960u, 0x4960u, 0x4640u, 0x4960u, 0x4000u }, + { 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4960u, 0x4000u }, + { 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4960u, 0x4960u, 0x4000u }, + { 0x4000u, 0x4640u, 0x4640u, 0x4960u, 0x4000u, 0x4960u, 0x4960u, 0x4000u }, + { 0x4000u, 0x4640u, 0x4000u, 0x4640u, 0x4960u, 0x4960u, 0x4960u, 0x4000u }, + { 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4640u }, + { 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4640u }, + { 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4640u }, + { 0x4000u, 0x4960u, 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4640u }, + { 0x4640u, 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4640u }, + { 0x4000u, 0x4640u, 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4640u }, + { 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4640u }, + { 0x4000u, 0x4960u, 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4640u }, + { 0x4960u, 0x4640u, 0x4640u, 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4640u }, + { 0x4000u, 0x4000u, 0x4960u, 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4640u }, + { 0x4640u, 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4640u }, + { 0x4000u, 0x4640u, 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4640u }, + { 0x4000u, 0x4000u, 0x4640u, 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4640u }, + { 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4640u }, + { 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4640u }, + { 0x4640u, 0x4000u, 0x4960u, 0x4640u, 0x4640u, 0x4000u, 0x4000u, 0x4640u }, + { 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4640u, 0x4000u, 0x4000u, 0x4640u }, + { 0x4640u, 0x4640u, 0x4000u, 0x4960u, 0x4640u, 0x4000u, 0x4000u, 0x4640u }, + { 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4640u }, + { 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4640u }, + { 0x4000u, 0x4960u, 0x4000u, 0x4640u, 0x4960u, 0x4000u, 0x4000u, 0x4640u }, + { 0x4960u, 0x4640u, 0x4640u, 0x4640u, 0x4960u, 0x4000u, 0x4000u, 0x4640u }, + { 0x4000u, 0x4960u, 0x4960u, 0x4640u, 0x4960u, 0x4000u, 0x4000u, 0x4640u }, + { 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4640u }, + { 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4640u }, + { 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4640u }, + { 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4640u, 0x4000u, 0x4640u }, + { 0x4640u, 0x4960u, 0x4640u, 0x4960u, 0x4000u, 0x4640u, 0x4000u, 0x4640u }, + { 0x4960u, 0x4000u, 0x4640u, 0x4000u, 0x4640u, 0x4640u, 0x4000u, 0x4640u }, + { 0x4000u, 0x4640u, 0x4960u, 0x4000u, 0x4640u, 0x4640u, 0x4000u, 0x4640u }, + { 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4640u, 0x4000u, 0x4640u }, + { 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4640u }, + { 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4640u }, + { 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4640u }, + { 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4960u, 0x4000u, 0x4640u }, + { 0x4640u, 0x4640u, 0x4000u, 0x4640u, 0x4000u, 0x4960u, 0x4000u, 0x4640u }, + { 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4960u, 0x4000u, 0x4640u }, + { 0x4000u, 0x4960u, 0x4640u, 0x4640u, 0x4640u, 0x4960u, 0x4000u, 0x4640u }, + { 0x4640u, 0x4000u, 0x4960u, 0x4640u, 0x4640u, 0x4960u, 0x4000u, 0x4640u }, + { 0x4960u, 0x4000u, 0x4000u, 0x4960u, 0x4640u, 0x4960u, 0x4000u, 0x4640u }, + { 0x4640u, 0x4640u, 0x4000u, 0x4640u, 0x4960u, 0x4960u, 0x4000u, 0x4640u }, + { 0x4000u, 0x4000u, 0x4640u, 0x4960u, 0x4960u, 0x4960u, 0x4000u, 0x4640u }, + { 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4640u }, + { 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4640u }, + { 0x4640u, 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4640u }, + { 0x4640u, 0x4960u, 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4640u }, + { 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4640u }, + { 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4640u, 0x4640u }, + { 0x4000u, 0x4960u, 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4640u, 0x4640u }, + { 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4640u, 0x4640u }, + { 0x4960u, 0x4000u, 0x4000u, 0x4640u, 0x4640u, 0x4000u, 0x4640u, 0x4640u }, + { 0x4000u, 0x4640u, 0x4960u, 0x4960u, 0x4640u, 0x4000u, 0x4640u, 0x4640u }, + { 0x4640u, 0x4000u, 0x4640u, 0x4960u, 0x4960u, 0x4000u, 0x4640u, 0x4640u }, + { 0x4000u, 0x4000u, 0x4640u, 0x4960u, 0x4000u, 0x4640u, 0x4640u, 0x4640u }, + { 0x4960u, 0x4000u, 0x4640u, 0x4960u, 0x4000u, 0x4640u, 0x4640u, 0x4640u }, + { 0x4960u, 0x4960u, 0x4000u, 0x4000u, 0x4640u, 0x4640u, 0x4640u, 0x4640u }, + { 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4640u, 0x4640u, 0x4640u }, + { 0x4000u, 0x4640u, 0x4640u, 0x4640u, 0x4960u, 0x4640u, 0x4640u, 0x4640u }, + { 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4640u, 0x4640u }, + { 0x4640u, 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4960u, 0x4640u, 0x4640u }, + { 0x4640u, 0x4960u, 0x4640u, 0x4000u, 0x4000u, 0x4960u, 0x4640u, 0x4640u }, + { 0x4000u, 0x4640u, 0x4960u, 0x4640u, 0x4000u, 0x4960u, 0x4640u, 0x4640u }, + { 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4640u, 0x4960u, 0x4640u, 0x4640u }, + { 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4960u, 0x4960u, 0x4640u, 0x4640u }, + { 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4640u }, + { 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4640u }, + { 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4960u, 0x4640u }, + { 0x4000u, 0x4960u, 0x4960u, 0x4640u, 0x4000u, 0x4000u, 0x4960u, 0x4640u }, + { 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4960u, 0x4640u }, + { 0x4640u, 0x4640u, 0x4640u, 0x4640u, 0x4640u, 0x4000u, 0x4960u, 0x4640u }, + { 0x4000u, 0x4960u, 0x4640u, 0x4000u, 0x4960u, 0x4000u, 0x4960u, 0x4640u }, + { 0x4000u, 0x4000u, 0x4960u, 0x4640u, 0x4960u, 0x4000u, 0x4960u, 0x4640u }, + { 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4960u, 0x4640u }, + { 0x4640u, 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4960u, 0x4640u }, + { 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4640u, 0x4640u, 0x4960u, 0x4640u }, + { 0x4960u, 0x4000u, 0x4640u, 0x4000u, 0x4640u, 0x4640u, 0x4960u, 0x4640u }, + { 0x4000u, 0x4640u, 0x4000u, 0x4960u, 0x4640u, 0x4640u, 0x4960u, 0x4640u }, + { 0x4960u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4960u, 0x4960u, 0x4640u }, + { 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4960u }, + { 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4960u }, + { 0x4960u, 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4960u }, + { 0x4640u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4960u }, + { 0x4960u, 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4960u }, + { 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4960u }, + { 0x4000u, 0x4960u, 0x4640u, 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4960u }, + { 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4640u, 0x4000u, 0x4000u, 0x4960u }, + { 0x4640u, 0x4000u, 0x4640u, 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4960u }, + { 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4960u }, + { 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4960u }, + { 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4960u }, + { 0x4640u, 0x4640u, 0x4640u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4960u }, + { 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4640u, 0x4000u, 0x4960u }, + { 0x4000u, 0x4000u, 0x4960u, 0x4640u, 0x4000u, 0x4640u, 0x4000u, 0x4960u }, + { 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4640u, 0x4000u, 0x4960u }, + { 0x4960u, 0x4640u, 0x4000u, 0x4640u, 0x4640u, 0x4640u, 0x4000u, 0x4960u }, + { 0x4000u, 0x4640u, 0x4640u, 0x4960u, 0x4640u, 0x4640u, 0x4000u, 0x4960u }, + { 0x4640u, 0x4960u, 0x4000u, 0x4000u, 0x4960u, 0x4640u, 0x4000u, 0x4960u }, + { 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4960u, 0x4640u, 0x4000u, 0x4960u }, + { 0x4000u, 0x4000u, 0x4960u, 0x4640u, 0x4960u, 0x4640u, 0x4000u, 0x4960u }, + { 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4960u }, + { 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4640u, 0x4960u, 0x4000u, 0x4960u }, + { 0x4640u, 0x4000u, 0x4640u, 0x4000u, 0x4960u, 0x4960u, 0x4000u, 0x4960u }, + { 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4960u }, + { 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4960u }, + { 0x4000u, 0x4640u, 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4960u }, + { 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4640u, 0x4960u }, + { 0x4640u, 0x4000u, 0x4960u, 0x4960u, 0x4000u, 0x4000u, 0x4640u, 0x4960u }, + { 0x4960u, 0x4640u, 0x4640u, 0x4000u, 0x4640u, 0x4000u, 0x4640u, 0x4960u }, + { 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4640u, 0x4000u, 0x4640u, 0x4960u }, + { 0x4640u, 0x4640u, 0x4000u, 0x4640u, 0x4960u, 0x4000u, 0x4640u, 0x4960u }, + { 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4640u, 0x4960u }, + { 0x4960u, 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4640u, 0x4640u, 0x4960u }, + { 0x4000u, 0x4640u, 0x4000u, 0x4640u, 0x4000u, 0x4640u, 0x4640u, 0x4960u }, + { 0x4640u, 0x4000u, 0x4640u, 0x4640u, 0x4640u, 0x4640u, 0x4640u, 0x4960u }, + { 0x4640u, 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4960u, 0x4640u, 0x4960u }, + { 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4640u, 0x4960u, 0x4640u, 0x4960u }, + { 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4960u }, + { 0x4000u, 0x4000u, 0x4640u, 0x4640u, 0x4000u, 0x4000u, 0x4960u, 0x4960u }, + { 0x4640u, 0x4640u, 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4960u, 0x4960u }, + { 0x4640u, 0x4960u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4960u, 0x4960u }, + { 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4960u, 0x4960u }, + { 0x4000u, 0x4960u, 0x4640u, 0x4000u, 0x4000u, 0x4640u, 0x4960u, 0x4960u }, + { 0x4000u, 0x4000u, 0x4640u, 0x4640u, 0x4000u, 0x4960u, 0x4960u, 0x4960u }, + { 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4640u, 0x4960u, 0x4960u, 0x4960u }, +}; + struct block_q2_K { uchar scales[QK_K/16]; uchar qs[QK_K/4]; @@ -2575,6 +2835,9 @@ void dequantize_q6_K(device const block_q6_K *xb, short il, thread type4x4 ®) } } +/* 0.25 * grid byte is exact in binary16, so the table replaces the u64 + * grid load, byte extract, and float conversion. Sign is a bit flip; + * the remaining rounding is the same f32->f16 store as before. */ template void dequantize_iq2_xxs(device const block_iq2_xxs * xb, short il, thread type4x4 & reg) { const float d = xb->d; @@ -2584,16 +2847,18 @@ void dequantize_iq2_xxs(device const block_iq2_xxs * xb, short il, thread type4x const uint32_t aux32_g = q2[0] | (q2[1] << 16); const uint32_t aux32_s = q2[2] | (q2[3] << 16); thread const uint8_t * aux8 = (thread const uint8_t *)&aux32_g; - const float dl = d * (0.5f + (aux32_s >> 28)) * 0.25f; - constant uint8_t * grid = (constant uint8_t *)(iq2xxs_grid + aux8[2*il+0]); + const float dl = d * (0.5f + (aux32_s >> 28)); + constant const ushort * values = ds4_metal_iq2xxs_half_values[aux8[2*il+0]]; uint8_t signs = ksigns_iq2xs[(aux32_s >> 14*il) & 127]; for (int i = 0; i < 8; ++i) { - reg[i/4][i%4] = dl * grid[i] * (signs & kmask_iq2xs[i] ? -1.f : 1.f); + const ushort bits = values[i] ^ (signs & kmask_iq2xs[i] ? 0x8000u : 0x0000u); + reg[i/4][i%4] = dl * (float)as_type(bits); } - grid = (constant uint8_t *)(iq2xxs_grid + aux8[2*il+1]); + values = ds4_metal_iq2xxs_half_values[aux8[2*il+1]]; signs = ksigns_iq2xs[(aux32_s >> (14*il+7)) & 127]; for (int i = 0; i < 8; ++i) { - reg[2+i/4][i%4] = dl * grid[i] * (signs & kmask_iq2xs[i] ? -1.f : 1.f); + const ushort bits = values[i] ^ (signs & kmask_iq2xs[i] ? 0x8000u : 0x0000u); + reg[2+i/4][i%4] = dl * (float)as_type(bits); } } @@ -9758,9 +10023,12 @@ kernel void kernel_attn_out_low_mpp_direct_rhs( } } -// Routed-expert grouped matmul on the Metal4 TensorOps/MPP pipeline. The -// barrier after mm.run prevents the next K iteration from replacing staged -// tiles while the cooperative matmul still reads them. +// Routed-expert grouped matmul on the Metal4 TensorOps/MPP pipeline. +// The 32 routed rows are two independent 16-row cooperative matmul2d +// ops (one per SIMDgroup pair). Staging is double-buffered so the K +// loop takes one barrier per step. A short final tile's empty row-half +// skips MMA, accumulator init, and store; staging and barriers stay +// unconditional. Threadgroup is sa[2]+sb[2] = 12 KiB. template kernel void kernel_mul_mm_id_mpp( constant ds4_metal_args_mul_mm_id & args, @@ -9776,7 +10044,7 @@ kernel void kernel_mul_mm_id_mpp( ushort tiisg[[thread_index_in_simdgroup]], ushort sgitg[[simdgroup_index_in_threadgroup]]) { threadgroup S0 * sa = (threadgroup S0 *)(shmem); - threadgroup S1 * sb = (threadgroup S1 *)(shmem + 4096); + threadgroup S1 * sb = (threadgroup S1 *)(shmem + 8192); threadgroup float *sc = (threadgroup float *)shmem; constexpr int NR0 = 64; @@ -9809,6 +10077,9 @@ kernel void kernel_mul_mm_id_mpp( const short nr0 = (args.ne0 - r0 < NR0) ? (args.ne0 - r0) : NR0; const short nr1 = ( neh1 - r1 < NR1) ? ( neh1 - r1) : NR1; + const short h = sgitg/2; + const bool mma_active = 16*h < nr1; + if (!ds4_tp_owns_expert(im, args.ne02, args.tp_rank, args.tp_world)) { for (short j = sgitg; j < nr1; j += 4) { const int idj = ids_i32[route_base + r1 + j]; @@ -9825,7 +10096,6 @@ kernel void kernel_mul_mm_id_mpp( const short lr1 = ((short)tiitg/NL1) < nr1 ? ((short)tiitg/NL1) : nr1 - 1; const short il0 = (tiitg % NL0); - short il = il0; const int id = ids_i32[route_base + r1 + lr1]; @@ -9837,51 +10107,58 @@ kernel void kernel_mul_mm_id_mpp( (uint64_t)(im - args.tp_expert_base)*args.nb02 + i13*args.nb03; const short offset1 = il0/nl; - device const block_q * x = (device const block_q *)(src0 + args.nb01*(r0 + lr0) + offset0) + offset1; + device const block_q * x_base = (device const block_q *)(src0 + args.nb01*(r0 + lr0) + offset0) + offset1; const short iy = 8*(tiitg % NL1); - device const T1 * y = (device const T1 *)(src1 + device const T1 * y_base = (device const T1 *)(src1 + args.nb13*i13 + args.nb12*i12 + args.nb11*i11 + args.nb10*iy); auto tA = tensor(sa, dextents(NK, NR0)); - auto tB = tensor(sb, dextents(NR1, NK)); + auto tA1 = tensor(sa + NR0*NK, dextents(NK, NR0)); + auto tBh = tensor(sb + h*16*NK, dextents(NK, 16)); + auto tBh1 = tensor(sb + NR1*NK + h*16*NK, dextents(NK, 16)); matmul2d< - matmul2d_descriptor(NR1, NR0, NK, false, true, false, + matmul2d_descriptor(16, NR0, NK, false, true, false, matmul2d_descriptor::mode::multiply_accumulate), - execution_simdgroups<4>> mm; + execution_simdgroups<2>> mm; - auto cT = mm.template get_destination_cooperative_tensor(); + auto cT = mm.template get_destination_cooperative_tensor(); - #pragma unroll - for (uint16_t i = 0; i < cT.get_capacity(); ++i) { - if (cT.is_valid_element(i)) { - cT[i] = 0.0f; + if (mma_active) { + #pragma unroll + for (uint16_t i = 0; i < cT.get_capacity(); ++i) { + if (cT.is_valid_element(i)) { + cT[i] = 0.0f; + } } } - for (int loop_k = 0; loop_k < args.ne00; loop_k += NK) { - if (is_same::value && FC_mul_mm_bc_inp) { - threadgroup_barrier(mem_flags::mem_threadgroup); + auto stage_tile = [&](const int loop_k, + threadgroup S0 * sa_buf, + threadgroup S1 * sb_buf) { + const int chunk = loop_k / 16 + il0; + const short il = (short)(chunk % nl); + + device const block_q * xb = x_base + (chunk / nl); + if (is_same::value && FC_mul_mm_bc_inp) { for (short i = 0; i < 16; i++) { const short sx = 2*il0 + i/8; const short sy = (tiitg/NL0)/8; const short lx = i%8; const short ly = (tiitg/NL0)%8; - *(sa + NK*(8*sy + ly) + 8*sx + lx) = - loop_k + 16*il + i < args.ne00 ? *((device T0 *) x + i) : 0; + *(sa_buf + NK*(8*sy + ly) + 8*sx + lx) = + loop_k + 16*il + i < args.ne00 ? *((device T0 *) xb + i) : 0; } } else { S0_4x4 temp_a; - dequantize_func(x, il, temp_a); - - threadgroup_barrier(mem_flags::mem_threadgroup); + dequantize_func(xb, il, temp_a); FOR_UNROLL (short i = 0; i < 16; i++) { const short sx = 2*il0 + i/8; @@ -9889,10 +10166,12 @@ kernel void kernel_mul_mm_id_mpp( const short lx = i%8; const short ly = (tiitg/NL0)%8; - *(sa + NK*(8*sy + ly) + 8*sx + lx) = temp_a[i/4][i%4]; + *(sa_buf + NK*(8*sy + ly) + 8*sx + lx) = temp_a[i/4][i%4]; } } + device const T1 * yb = y_base + loop_k; + if (FC_mul_mm_bc_inp) { for (short i = 0; i < 8; ++i) { const short sx = (tiitg%NL1); @@ -9900,36 +10179,46 @@ kernel void kernel_mul_mm_id_mpp( const short lx = i; const short ly = (tiitg/NL1)%8; - *(sb + NK*(8*sy + ly) + 8*sx + lx) = - loop_k + iy + i < args.ne00 ? (S1) *((device T1 *) y + i) : 0; + *(sb_buf + NK*(8*sy + ly) + 8*sx + lx) = + loop_k + iy + i < args.ne00 ? (S1) *((device T1 *) yb + i) : 0; } } else { const short sx = (tiitg%NL1); const short sy = (tiitg/NL1)/8; const short ly = (tiitg/NL1)%8; - *(threadgroup S1_2x4 *)(sb + NK*(8*sy + ly) + 8*sx) = - (S1_2x4)(*((device T1_2x4 *) y)); + *(threadgroup S1_2x4 *)(sb_buf + NK*(8*sy + ly) + 8*sx) = + (S1_2x4)(*((device T1_2x4 *) yb)); } + }; - il = (il + 2 < nl) ? il + 2 : il % 2; - x = (il < 2) ? x + (2 + nl - 1)/nl : x; - - y += NK; - - threadgroup_barrier(mem_flags::mem_threadgroup); + stage_tile(0, sa, sb); + threadgroup_barrier(mem_flags::mem_threadgroup); - auto sA = tA.slice(0, 0); - auto sB = tB.slice(0, 0); - mm.run(sB, sA, cT); + uint buf_sel = 0; + for (int loop_k = 0; loop_k < args.ne00; loop_k += NK) { + if (mma_active) { + auto sA = (buf_sel ? tA1 : tA).slice(0, 0); + auto sB = (buf_sel ? tBh1 : tBh).slice(0, 0); + mm.run(sB, sA, cT); + } + const int next_k = loop_k + NK; + if (next_k < args.ne00) { + buf_sel ^= 1u; + stage_tile(next_k, + buf_sel ? (sa + NR0*NK) : sa, + buf_sel ? (sb + NR1*NK) : sb); + } threadgroup_barrier(mem_flags::mem_threadgroup); } threadgroup_barrier(mem_flags::mem_threadgroup); - auto tC = tensor(sc, dextents(NR0, NR1)); - cT.store(tC); + if (mma_active) { + auto tC = tensor(sc + h*16*NR0, dextents(NR0, 16)); + cT.store(tC); + } threadgroup_barrier(mem_flags::mem_threadgroup); @@ -9957,8 +10246,6 @@ kernel void kernel_mul_mm_id_mpp( } } - - typedef decltype(kernel_mul_mm_id_mpp) mul_mm_id_mpp_t; typedef decltype(kernel_mul_mm_id_mpp) mul_mm_id_mpp_f16_rhs_t; From 24ea7bec62e871be33e9b680342f5034dc082705 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Tue, 25 Aug 2026 19:07:26 +0200 Subject: [PATCH 114/189] metal: share Q4_K scale metadata in exact-N prefill --- ENVIRONMENT_VARIABLES.md | 8 +- Makefile | 2 + ds4_metal.m | 81 ++++++++++++-- metal/moe.metal | 75 +++++++++++-- scripts/environment_variables.tsv | 2 + tests/test_metal_q4_attn_exactn.c | 179 +++++++++++++++++++----------- 6 files changed, 260 insertions(+), 87 deletions(-) diff --git a/ENVIRONMENT_VARIABLES.md b/ENVIRONMENT_VARIABLES.md index 4beac3ee0..a8a8c6f6c 100644 --- a/ENVIRONMENT_VARIABLES.md +++ b/ENVIRONMENT_VARIABLES.md @@ -37,6 +37,8 @@ changing them. Unless a row says otherwise: | `DS4_METAL_ENABLE_M1_IQ2_MID_ONLY=1` | Legacy spelling retained for migration notes only. The runtime does not read it; the path is automatic and this setting is ignored. | | `DS4_METAL_DISABLE_IQ2_XXS_SSD_PREFILL_MM=1` | Restore sparse matvec for an eligible grouped IQ2 SSD-prefill chunk. | | `DS4_METAL_REQUIRE_IQ2_XXS_SSD_PREFILL_MM=1` | Require grouped IQ2 SSD-prefill MM for eligible chunks and reject insufficient cache instead of silently falling back. | +| `DS4_METAL_DISABLE_Q4_SSD_PREFILL_ATTN_OUT_SCALE_META=1` | Restore per-SIMDgroup Q4_K scale/min unpacking inside the opt-in SSD-prefill attention-output exact-N path. Shared scale metadata is automatic when that path is enabled and the PSO and threadgroup memory are available. | +| `DS4_METAL_REQUIRE_Q4_SSD_PREFILL_ATTN_OUT_SCALE_META=1` | Enable the Q4_K SSD-prefill attention-output exact-N path and fail closed unless its shared scale/min metadata PSO is used. Intended for oracles and A/B tests. | | `DS4_METAL_ENABLE_STREAMING_PREFILL_EXPERT_READAHEAD=1` | Restore the historical `F_RDADVISE` plus parallel-`pread` sequence for cold-storage A/B tests. Normal grouped prefill skips the redundant hint. | The detailed Metal A/B contracts and expected oracle counters live in @@ -112,13 +114,13 @@ above, it is an unstable internal diagnostic or tuning interface. The linked sou remains normative for exact eligibility gates, bounds, and architecture-specific defaults. -Inventory totals: **1067 `DS4_*` runtime variables** and +Inventory totals: **1069 `DS4_*` runtime variables** and **6 external runtime variables**. The auxiliary inventories contain **112 test/test-fixture entries** and **19 tool/wrapper entries**.
-Metal (440) +Metal (442) | Variable | Accepted value and default | Effect | Source | | --- | --- | --- | --- | @@ -261,6 +263,7 @@ and **19 tool/wrapper entries**. | `DS4_METAL_DISABLE_Q4_QKV_COMPRESSOR_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 QKV compressor fuse. | [ds4_metal.m:22083](ds4_metal.m#L22083) | | `DS4_METAL_DISABLE_Q4_SELECTED_EXPERT_VIEWS` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 selected expert views. | [ds4.c:21136](ds4.c#L21136) | | `DS4_METAL_DISABLE_Q4_SSD_PREFILL_ATTN_OUT_EXACTN` | value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables | Disables Q4 SSD prefill attn out exactn. | [ds4_metal.m:28095](ds4_metal.m#L28095) | +| `DS4_METAL_DISABLE_Q4_SSD_PREFILL_ATTN_OUT_SCALE_META` | value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables | Disables shared scale/min metadata in the Q4 SSD prefill attention-output exact-N kernel. | [ds4_metal.m:28220](ds4_metal.m#L28220) | | `DS4_METAL_DISABLE_Q4_SSD_SESSION_UNION` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Disables Q4 SSD session union. | [ds4.c:65160](ds4.c#L65160) | | `DS4_METAL_DISABLE_Q4_STREAM_OVERLAP` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Disables Q4 stream overlap. | [ds4.c:65082](ds4.c#L65082) | | `DS4_METAL_DISABLE_Q4_TABLE_BOUNDARY` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 table boundary. | [ds4_metal.m:42318](ds4_metal.m#L42318) | @@ -500,6 +503,7 @@ and **19 tool/wrapper entries**. | `DS4_METAL_REQUIRE_OUTPUT_HC_WEIGHTS4` | presence strict check; unset: fallback allowed; any value including 0 requires the path | Requires output HC weights4 and makes eligible fallback fail closed. | [ds4_metal.m:46789](ds4_metal.m#L46789) | | `DS4_METAL_REQUIRE_Q4_ATTN_OUT_TINY_BATCH` | value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables | Requires Q4 attn out tiny batch and makes eligible fallback fail closed. | [ds4_metal.m:28373](ds4_metal.m#L28373) | | `DS4_METAL_REQUIRE_Q4_SSD_PREFILL_ATTN_OUT_EXACTN` | value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables | Requires Q4 SSD prefill attn out exactn and makes eligible fallback fail closed. | [ds4_metal.m:28089](ds4_metal.m#L28089) | +| `DS4_METAL_REQUIRE_Q4_SSD_PREFILL_ATTN_OUT_SCALE_META` | value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables | Requires shared scale/min metadata in the Q4 SSD prefill attention-output exact-N kernel and makes fallback fail closed. | [ds4_metal.m:28209](ds4_metal.m#L28209) | | `DS4_METAL_REQUIRE_Q4_SSD_SESSION_UNION` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Requires Q4 SSD session union and makes eligible fallback fail closed. | [ds4.c:65164](ds4.c#L65164) | | `DS4_METAL_REQUIRE_Q8_QKV_COMPRESSOR_FUSE` | nonempty boolean; unset/empty/exact 0: fallback allowed; other values require and imply the streamed/union enable | Requires eligible Q8 QKV/compressor compound fusion and fails closed. | [ds4.c:23023](ds4.c#L23023) | | `DS4_METAL_RESUME_PREFILL_MIN` | integer token threshold; default 4; <=0 disables resume-prefill | Sets the minimum shared-prefix suffix that uses batched resume-prefill. | [ds4.c:38419](ds4.c#L38419) | diff --git a/Makefile b/Makefile index d3c4df9a7..2fbdabc26 100644 --- a/Makefile +++ b/Makefile @@ -192,6 +192,8 @@ test-metal-q4-attn-exactn: tests/test_metal_q4_attn_exactn env -u DS4_METAL_ENABLE_Q4_SSD_PREFILL_ATTN_OUT_EXACTN \ -u DS4_METAL_DISABLE_Q4_SSD_PREFILL_ATTN_OUT_EXACTN \ -u DS4_METAL_REQUIRE_Q4_SSD_PREFILL_ATTN_OUT_EXACTN \ + -u DS4_METAL_DISABLE_Q4_SSD_PREFILL_ATTN_OUT_SCALE_META \ + -u DS4_METAL_REQUIRE_Q4_SSD_PREFILL_ATTN_OUT_SCALE_META \ -u DS4_METAL_DISABLE_Q4_MV_CLASSIC \ ./tests/test_metal_q4_attn_exactn diff --git a/ds4_metal.m b/ds4_metal.m index 49be6eee8..7d60569a9 100644 --- a/ds4_metal.m +++ b/ds4_metal.m @@ -28204,14 +28204,20 @@ static int ds4_gpu_attention_output_q4_K_ssd_prefill_exactn_tensor( const bool scope = n_tokens >= 6u && n_tokens <= 31u; const bool require = scope && ds4_gpu_env_bool("DS4_METAL_REQUIRE_Q4_SSD_PREFILL_ATTN_OUT_EXACTN") == 1; - const int failure_rc = require ? -1 : 0; + const bool require_scale_meta = scope && + ds4_gpu_env_bool( + "DS4_METAL_REQUIRE_Q4_SSD_PREFILL_ATTN_OUT_SCALE_META") == 1; + const int failure_rc = require || require_scale_meta ? -1 : 0; if (!scope) return 0; if (!g_initialized && !ds4_gpu_init()) return failure_rc; const bool disabled = ds4_gpu_env_bool("DS4_METAL_DISABLE_Q4_SSD_PREFILL_ATTN_OUT_EXACTN") == 1; - const bool enabled = require || + const bool enabled = require || require_scale_meta || ds4_gpu_env_bool("DS4_METAL_ENABLE_Q4_SSD_PREFILL_ATTN_OUT_EXACTN") == 1; + const bool scale_meta_disabled = + ds4_gpu_env_bool( + "DS4_METAL_DISABLE_Q4_SSD_PREFILL_ATTN_OUT_SCALE_META") == 1; const bool classic_q4 = getenv("DS4_METAL_DISABLE_Q4_MV_CLASSIC") == NULL; const bool platform_ok = g_ssd_streaming_mode && @@ -28219,7 +28225,7 @@ static int ds4_gpu_attention_output_q4_K_ssd_prefill_exactn_tensor( ds4_gpu_device_is_pre_m5_apple_silicon(); if (!enabled || disabled || !classic_q4 || !platform_ok || out_b_type != DS4_METAL_TENSOR_Q4_K) { - if (require) { + if (failure_rc < 0) { fprintf(stderr, "ds4: required Metal Q4 SSD-prefill attention exact-N " "path is ineligible (rows=%u type=%u ssd=%u quality=%u " @@ -28234,6 +28240,12 @@ static int ds4_gpu_attention_output_q4_K_ssd_prefill_exactn_tensor( } return failure_rc; } + if (require_scale_meta && scale_meta_disabled) { + fprintf(stderr, + "ds4: required Metal Q4 SSD-prefill attention exact-N " + "scale metadata is disabled\n"); + return -1; + } if (!out || !low || !heads || !model_map || group_dim == 0 || rank == 0 || n_groups == 0 || out_dim == 0 || @@ -28248,7 +28260,7 @@ static int ds4_gpu_attention_output_q4_K_ssd_prefill_exactn_tensor( } if ((group_dim % 256u) != 0 || (low_dim % 256u) != 0 || low_dim == 0 || low_dim > UINT32_MAX) { - if (require) { + if (failure_rc < 0) { fprintf(stderr, "ds4: required Metal Q4 SSD-prefill attention exact-N " "path received unaligned dimensions\n"); @@ -28280,6 +28292,10 @@ static int ds4_gpu_attention_output_q4_K_ssd_prefill_exactn_tensor( uint64_t rank_bytes = 0; uint64_t stage_a_bytes = 0; uint64_t stage_b_bytes = 0; + uint64_t scale_meta_a_bytes = 0; + uint64_t scale_meta_b_bytes = 0; + const uint64_t scale_meta_bytes_per_block_pair = + 2u * 2u * sizeof(uint64_t); /* two rows, two iq records */ if (!ds4_gpu_u64_mul_checked(rank, row_a_bytes, &group_a_bytes) || !ds4_gpu_u64_mul_checked((uint64_t)n_groups, group_a_bytes, @@ -28314,8 +28330,16 @@ static int ds4_gpu_attention_output_q4_K_ssd_prefill_exactn_tensor( &stage_a_bytes) || !ds4_gpu_u64_mul_checked(2u, row_b_bytes, - &stage_b_bytes)) { - if (require) { + &stage_b_bytes) || + !ds4_gpu_u64_mul_checked(group_dim / 256u, + scale_meta_bytes_per_block_pair, + &scale_meta_a_bytes) || + !ds4_gpu_u64_mul_checked(low_dim / 256u, + scale_meta_bytes_per_block_pair, + &scale_meta_b_bytes) || + scale_meta_a_bytes > UINT64_MAX - stage_a_bytes || + scale_meta_b_bytes > UINT64_MAX - stage_b_bytes) { + if (failure_rc < 0) { fprintf(stderr, "ds4: required Metal Q4 SSD-prefill attention exact-N " "shape overflows byte strides\n"); @@ -28324,7 +28348,7 @@ static int ds4_gpu_attention_output_q4_K_ssd_prefill_exactn_tensor( } if (out_a_offset > model_size || out_a_bytes > model_size - out_a_offset || out_b_offset > model_size || out_b_bytes > model_size - out_b_offset) { - if (require) { + if (failure_rc < 0) { fprintf(stderr, "ds4: required Metal Q4 SSD-prefill attention exact-N " "weights are outside the mapped model\n"); @@ -28339,7 +28363,7 @@ static int ds4_gpu_attention_output_q4_K_ssd_prefill_exactn_tensor( ds4_gpu_tensor_bytes(heads) < heads_bytes || ds4_gpu_tensor_bytes(low) < low_bytes || ds4_gpu_tensor_bytes(out) < out_bytes) { - if (require) { + if (failure_rc < 0) { fprintf(stderr, "ds4: required Metal Q4 SSD-prefill attention exact-N " "path received undersized tensors\n"); @@ -28347,15 +28371,50 @@ static int ds4_gpu_attention_output_q4_K_ssd_prefill_exactn_tensor( return failure_rc; } - id pipeline = ds4_gpu_get_pipeline( - "kernel_dsv4_attn_out_q4_K_ssd_prefill_exactn_f32"); + const uint64_t scale_meta_stage_a_bytes = + stage_a_bytes + scale_meta_a_bytes; + const uint64_t scale_meta_stage_b_bytes = + stage_b_bytes + scale_meta_b_bytes; + const uint64_t scale_meta_max_stage_bytes = + scale_meta_stage_a_bytes > scale_meta_stage_b_bytes ? + scale_meta_stage_a_bytes : scale_meta_stage_b_bytes; + bool scale_meta = + !scale_meta_disabled && + scale_meta_max_stage_bytes <= + (uint64_t)[g_device maxThreadgroupMemoryLength] && + scale_meta_max_stage_bytes <= (uint64_t)NSUIntegerMax; + id pipeline = nil; + if (scale_meta) { + pipeline = ds4_gpu_get_pipeline( + "kernel_dsv4_attn_out_q4_K_ssd_prefill_exactn_scale_meta_f32"); + if (!pipeline || pipeline.threadExecutionWidth != 32u || + pipeline.maxTotalThreadsPerThreadgroup < 512u) { + scale_meta = false; + } + } + if (!scale_meta && require_scale_meta) { + fprintf(stderr, + "ds4: required Metal Q4 SSD-prefill attention exact-N " + "scale metadata pipeline is unavailable " + "(stage=%llu max_stage=%lu)\n", + (unsigned long long)scale_meta_max_stage_bytes, + (unsigned long)[g_device maxThreadgroupMemoryLength]); + return -1; + } + if (!scale_meta) { + pipeline = ds4_gpu_get_pipeline( + "kernel_dsv4_attn_out_q4_K_ssd_prefill_exactn_f32"); + } else { + stage_a_bytes = scale_meta_stage_a_bytes; + stage_b_bytes = scale_meta_stage_b_bytes; + } const uint64_t max_stage_bytes = stage_a_bytes > stage_b_bytes ? stage_a_bytes : stage_b_bytes; if (!pipeline || pipeline.threadExecutionWidth != 32u || pipeline.maxTotalThreadsPerThreadgroup < 512u || max_stage_bytes > (uint64_t)[g_device maxThreadgroupMemoryLength] || max_stage_bytes > (uint64_t)NSUIntegerMax) { - if (require) { + if (failure_rc < 0) { fprintf(stderr, "ds4: required Metal Q4 SSD-prefill attention exact-N " "pipeline is unavailable (threads=%lu width=%lu " diff --git a/metal/moe.metal b/metal/moe.metal index 6b11e7431..840eebba1 100644 --- a/metal/moe.metal +++ b/metal/moe.metal @@ -3186,11 +3186,12 @@ void kernel_mul_mv_q4_K_f32_impl( // space changed from device to threadgroup. Its lane-to-K mapping, scalar // operation order and simd_sum reduction are deliberately kept identical to // kernel_mul_mv_q4_K_f32_impl: the exact-N oracle requires bitwise equality. -template +template void kernel_mul_mv_q4_K_staged_exactn_impl( uint32_t in_dim, uint64_t weight_row_bytes, threadgroup const char *src0, + threadgroup const ushort4 *preexpanded_scales, device const char *src1, device char *dst, uint32_t valid_rows, @@ -3216,7 +3217,6 @@ void kernel_mul_mv_q4_K_staged_exactn_impl( device const float *y4 = y + ix * QK_K + 64 * iq + 8 * ir; uint16_t sc16[4]; - thread const uint8_t *sc8 = (thread const uint8_t *)sc16; for (int ib = ix; ib < nb; ib += 4) { float4 sumy = {0.f, 0.f, 0.f, 0.f}; @@ -3235,10 +3235,24 @@ void kernel_mul_mv_q4_K_staged_exactn_impl( threadgroup const half *dh = &x[ib].d; for (short row = 0; row < nr0 && row < valid_rows; row++) { - sc16[0] = sc[0] & kmask1; - sc16[1] = sc[2] & kmask1; - sc16[2] = ((sc[4] >> 0) & kmask2) | ((sc[0] & kmask3) >> 2); - sc16[3] = ((sc[4] >> 4) & kmask2) | ((sc[2] & kmask3) >> 2); + if (use_preexpanded_scales) { + const uint32_t meta_index = + ((uint32_t)row * (uint32_t)nb + (uint32_t)ib) * 2u + + (uint32_t)iq; + const ushort4 expanded = preexpanded_scales[meta_index]; + sc16[0] = expanded[0]; + sc16[1] = expanded[1]; + sc16[2] = expanded[2]; + sc16[3] = expanded[3]; + } else { + sc16[0] = sc[0] & kmask1; + sc16[1] = sc[2] & kmask1; + sc16[2] = + ((sc[4] >> 0) & kmask2) | ((sc[0] & kmask3) >> 2); + sc16[3] = + ((sc[4] >> 4) & kmask2) | ((sc[2] & kmask3) >> 2); + } + thread const uint8_t *sc8 = (thread const uint8_t *)sc16; threadgroup const uint16_t *q2 = q1 + 32; @@ -3280,9 +3294,11 @@ void kernel_mul_mv_q4_K_staged_exactn_impl( } // Grid: x = output-row pairs, y = 16-token tiles, z = independent groups. -// All 512 threads join the raw packed-row load and barrier. Afterwards each -// SIMDgroup owns one token and follows the classic Q4_K arithmetic above. -kernel void kernel_dsv4_attn_out_q4_K_ssd_prefill_exactn_f32( +// All 512 threads join the raw packed-row load and barrier. The optimized +// specialization also expands the integer scale/min metadata before that same +// barrier so all token SIMDgroups can reuse it without changing FP arithmetic. +template +kernel void kernel_dsv4_attn_out_q4_K_ssd_prefill_exactn_impl( constant ds4_metal_args_q4_attn_exactn &args, device const char *weights, device const char *input, @@ -3306,6 +3322,34 @@ kernel void kernel_dsv4_attn_out_q4_K_ssd_prefill_exactn_f32( staged[args.weight_row_bytes + i] = valid_rows == 2u ? weights[row1_base + i] : 0; } + + const uint32_t blocks_per_row = args.in_dim / QK_K; + threadgroup ushort4 *scale_meta = + (threadgroup ushort4 *)(staged + 2u * args.weight_row_bytes); + if (preexpand_scales) { + const uint32_t meta_count = valid_rows * blocks_per_row * 2u; + for (uint32_t meta_index = tiitg; + meta_index < meta_count; + meta_index += 32u * 16u) { + const uint32_t iq = meta_index & 1u; + const uint32_t block_index = meta_index >> 1u; + const uint32_t row = block_index / blocks_per_row; + const uint32_t ib = block_index - row * blocks_per_row; + device const block_q4_K *xb = + (device const block_q4_K *)(weights + row0_base + + (uint64_t)row * args.weight_row_bytes) + ib; + device const uint16_t *sc = + (device const uint16_t *)xb->scales + iq; + const ushort4 expanded = ushort4( + sc[0] & 0x3f3fu, + sc[2] & 0x3f3fu, + ((sc[4] >> 0) & 0x0f0fu) | + ((sc[0] & 0xc0c0u) >> 2), + ((sc[4] >> 4) & 0x0f0fu) | + ((sc[2] & 0xc0c0u) >> 2)); + scale_meta[meta_index] = expanded; + } + } threadgroup_barrier(mem_flags::mem_threadgroup); const uint32_t token = tgpig.y * 16u + sgitg; @@ -3319,16 +3363,27 @@ kernel void kernel_dsv4_attn_out_q4_K_ssd_prefill_exactn_f32( (uint64_t)tgpig.z * args.output_group_bytes + (uint64_t)first_row * sizeof(float); - kernel_mul_mv_q4_K_staged_exactn_impl<2>( + kernel_mul_mv_q4_K_staged_exactn_impl<2, preexpand_scales>( args.in_dim, args.weight_row_bytes, staged, + scale_meta, token_input, token_output, valid_rows, tiisg); } +typedef decltype( + kernel_dsv4_attn_out_q4_K_ssd_prefill_exactn_impl) + q4_attn_ssd_prefill_exactn_t; +template [[host_name("kernel_dsv4_attn_out_q4_K_ssd_prefill_exactn_f32")]] +kernel q4_attn_ssd_prefill_exactn_t + kernel_dsv4_attn_out_q4_K_ssd_prefill_exactn_impl; +template [[host_name("kernel_dsv4_attn_out_q4_K_ssd_prefill_exactn_scale_meta_f32")]] +kernel q4_attn_ssd_prefill_exactn_t + kernel_dsv4_attn_out_q4_K_ssd_prefill_exactn_impl; + template void kernel_mul_mv_mxfp4_f32_impl( args_t args, diff --git a/scripts/environment_variables.tsv b/scripts/environment_variables.tsv index 9e98ca524..85eae5a16 100644 --- a/scripts/environment_variables.tsv +++ b/scripts/environment_variables.tsv @@ -599,6 +599,7 @@ runtime/metal DS4_METAL_DISABLE_Q4_MV_CLASSIC presence rollback; unset: automati runtime/metal DS4_METAL_DISABLE_Q4_QKV_COMPRESSOR_FUSE presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 QKV compressor fuse. ds4_metal.m:22083 runtime/metal DS4_METAL_DISABLE_Q4_SELECTED_EXPERT_VIEWS presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 selected expert views. ds4.c:21136 runtime/metal DS4_METAL_DISABLE_Q4_SSD_PREFILL_ATTN_OUT_EXACTN value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Disables Q4 SSD prefill attn out exactn. ds4_metal.m:28095 +runtime/metal DS4_METAL_DISABLE_Q4_SSD_PREFILL_ATTN_OUT_SCALE_META value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Disables shared scale/min metadata in the Q4 SSD prefill attention-output exact-N kernel. ds4_metal.m:28220 runtime/metal DS4_METAL_DISABLE_Q4_SSD_SESSION_UNION nonempty boolean; unset/empty or exact 0: off; every other value: on Disables Q4 SSD session union. ds4.c:65160 runtime/metal DS4_METAL_DISABLE_Q4_STREAM_OVERLAP nonempty boolean; unset/empty or exact 0: off; every other value: on Disables Q4 stream overlap. ds4.c:65082 runtime/metal DS4_METAL_DISABLE_Q4_TABLE_BOUNDARY presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 table boundary. ds4_metal.m:42318 @@ -838,6 +839,7 @@ runtime/metal DS4_METAL_REQUIRE_M1_IQ2_MID_ONLY presence strict check; unset: fa runtime/metal DS4_METAL_REQUIRE_OUTPUT_HC_WEIGHTS4 presence strict check; unset: fallback allowed; any value including 0 requires the path Requires output HC weights4 and makes eligible fallback fail closed. ds4_metal.m:46789 runtime/metal DS4_METAL_REQUIRE_Q4_ATTN_OUT_TINY_BATCH value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Requires Q4 attn out tiny batch and makes eligible fallback fail closed. ds4_metal.m:28373 runtime/metal DS4_METAL_REQUIRE_Q4_SSD_PREFILL_ATTN_OUT_EXACTN value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Requires Q4 SSD prefill attn out exactn and makes eligible fallback fail closed. ds4_metal.m:28089 +runtime/metal DS4_METAL_REQUIRE_Q4_SSD_PREFILL_ATTN_OUT_SCALE_META value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Requires shared scale/min metadata in the Q4 SSD prefill attention-output exact-N kernel and makes fallback fail closed. ds4_metal.m:28209 runtime/metal DS4_METAL_REQUIRE_Q4_SSD_SESSION_UNION nonempty boolean; unset/empty or exact 0: off; every other value: on Requires Q4 SSD session union and makes eligible fallback fail closed. ds4.c:65164 runtime/metal DS4_METAL_REQUIRE_Q8_QKV_COMPRESSOR_FUSE nonempty boolean; unset/empty/exact 0: fallback allowed; other values require and imply the streamed/union enable Requires eligible Q8 QKV/compressor compound fusion and fails closed. ds4.c:23023 runtime/metal DS4_METAL_RESUME_PREFILL_MIN integer token threshold; default 4; <=0 disables resume-prefill Sets the minimum shared-prefix suffix that uses batched resume-prefill. ds4.c:38419 diff --git a/tests/test_metal_q4_attn_exactn.c b/tests/test_metal_q4_attn_exactn.c index 0b3b86e30..b2e879637 100644 --- a/tests/test_metal_q4_attn_exactn.c +++ b/tests/test_metal_q4_attn_exactn.c @@ -46,6 +46,10 @@ static const char *k_disable = "DS4_METAL_DISABLE_Q4_SSD_PREFILL_ATTN_OUT_EXACTN"; static const char *k_require = "DS4_METAL_REQUIRE_Q4_SSD_PREFILL_ATTN_OUT_EXACTN"; +static const char *k_disable_scale_meta = + "DS4_METAL_DISABLE_Q4_SSD_PREFILL_ATTN_OUT_SCALE_META"; +static const char *k_require_scale_meta = + "DS4_METAL_REQUIRE_Q4_SSD_PREFILL_ATTN_OUT_SCALE_META"; static const char *k_disable_classic = "DS4_METAL_DISABLE_Q4_MV_CLASSIC"; static void fail(const char *what) { @@ -93,9 +97,9 @@ static void fill_q4_matrix(void *raw, uint8_t scale[8]; uint8_t minimum[8]; for (uint32_t group = 0; group < 8u; group++) { - scale[group] = (uint8_t)(1u + (key + group * 7u) % 31u); + scale[group] = (uint8_t)((key + group * 7u) % 64u); minimum[group] = - (uint8_t)((key / 3u + group * 5u) % 17u); + (uint8_t)((key / 3u + group * 5u) % 64u); } pack_scales(b->scales, scale, minimum); for (uint32_t i = 0; i < QK_K / 2u; i++) { @@ -166,6 +170,10 @@ int main(void) { CHECK(unsetenv(k_enable) == 0, "clear enable env"); CHECK(unsetenv(k_disable) == 0, "clear disable env"); CHECK(unsetenv(k_require) == 0, "clear require env"); + CHECK(unsetenv(k_disable_scale_meta) == 0, + "clear scale-meta disable env"); + CHECK(unsetenv(k_require_scale_meta) == 0, + "clear scale-meta require env"); CHECK(unsetenv(k_disable_classic) == 0, "clear classic kill env"); CHECK(ds4_gpu_init() != 0, "Metal init"); @@ -265,70 +273,113 @@ int main(void) { "default-off gate"); CHECK(setenv(k_enable, "1", 1) == 0, "enable candidate"); - for (uint32_t case_i = 0; - case_i < sizeof(exact_rows) / sizeof(exact_rows[0]); - case_i++) { - const uint32_t n_rows = exact_rows[case_i]; - poison(candidate_low_host, low_count, 0x7fc10000u); - poison(candidate_out_host, out_count, 0x7fc20000u); - CHECK(ds4_gpu_tensor_write(candidate_low, 0, candidate_low_host, - low_count * sizeof(float)) != 0, - "candidate low poison"); - CHECK(ds4_gpu_tensor_write(candidate_out, 0, candidate_out_host, - out_count * sizeof(float)) != 0, - "candidate out poison"); - - /* Exercise the production wrapper delegation, not only the direct - * test entry point. Scratch arguments are unused by this path. */ - CHECK(ds4_gpu_attention_output_q4_K_batch_tensor( - candidate_out, candidate_low, NULL, NULL, - model, model_bytes, 0, out_b_offset, Q4_K_TYPE, - GROUP_DIM, RANK, N_GROUPS, OUT_DIM, heads, n_rows) == 1, - "candidate dispatch"); - CHECK(ds4_gpu_tensor_read(candidate_low, 0, candidate_low_host, - low_count * sizeof(float)) != 0, - "candidate low read"); - CHECK(ds4_gpu_tensor_read(candidate_out, 0, candidate_out_host, - out_count * sizeof(float)) != 0, - "candidate out read"); - - uint64_t first_low = UINT64_MAX; - uint64_t first_out = UINT64_MAX; - const uint64_t compared_low = (uint64_t)n_rows * LOW_DIM; - const uint64_t compared_out = (uint64_t)n_rows * OUT_DIM; - const uint64_t low_mismatch = count_bit_mismatches( - reference_low_host, candidate_low_host, - compared_low, &first_low); - const uint64_t out_mismatch = count_bit_mismatches( - reference_out_host, candidate_out_host, - compared_out, &first_out); - const uint64_t low_canary = count_poison_mismatches( - candidate_low_host, compared_low, low_count, 0x7fc10000u); - const uint64_t out_canary = count_poison_mismatches( - candidate_out_host, compared_out, out_count, 0x7fc20000u); - fprintf(stderr, - "Metal Q4 SSD-prefill exact-N=%u low=%llu/%llu " - "out=%llu/%llu low_canary=%llu out_canary=%llu\n", - n_rows, - (unsigned long long)low_mismatch, - (unsigned long long)compared_low, - (unsigned long long)out_mismatch, - (unsigned long long)compared_out, - (unsigned long long)low_canary, - (unsigned long long)out_canary); - if (low_mismatch != 0) { - fprintf(stderr, " first low mismatch index=%llu\n", - (unsigned long long)first_low); + for (uint32_t scale_variant = 0; scale_variant < 2u; scale_variant++) { + const bool use_scale_meta = scale_variant == 0u; + if (use_scale_meta) { + CHECK(unsetenv(k_disable_scale_meta) == 0, + "enable shared scale metadata"); + CHECK(setenv(k_require_scale_meta, "1", 1) == 0, + "require shared scale metadata"); + } else { + CHECK(unsetenv(k_require_scale_meta) == 0, + "clear shared scale metadata requirement"); + CHECK(setenv(k_disable_scale_meta, "1", 1) == 0, + "select legacy scale unpack"); } - if (out_mismatch != 0) { - fprintf(stderr, " first out mismatch index=%llu\n", - (unsigned long long)first_out); + for (uint32_t case_i = 0; + case_i < sizeof(exact_rows) / sizeof(exact_rows[0]); + case_i++) { + const uint32_t n_rows = exact_rows[case_i]; + poison(candidate_low_host, low_count, 0x7fc10000u); + poison(candidate_out_host, out_count, 0x7fc20000u); + CHECK(ds4_gpu_tensor_write(candidate_low, 0, candidate_low_host, + low_count * sizeof(float)) != 0, + "candidate low poison"); + CHECK(ds4_gpu_tensor_write(candidate_out, 0, candidate_out_host, + out_count * sizeof(float)) != 0, + "candidate out poison"); + + /* Exercise the production wrapper delegation, not only the direct + * test entry point. Scratch arguments are unused by this path. */ + CHECK(ds4_gpu_attention_output_q4_K_batch_tensor( + candidate_out, candidate_low, NULL, NULL, + model, model_bytes, 0, out_b_offset, Q4_K_TYPE, + GROUP_DIM, RANK, N_GROUPS, OUT_DIM, heads, n_rows) == 1, + "candidate dispatch"); + CHECK(ds4_gpu_tensor_read(candidate_low, 0, candidate_low_host, + low_count * sizeof(float)) != 0, + "candidate low read"); + CHECK(ds4_gpu_tensor_read(candidate_out, 0, candidate_out_host, + out_count * sizeof(float)) != 0, + "candidate out read"); + + uint64_t first_low = UINT64_MAX; + uint64_t first_out = UINT64_MAX; + const uint64_t compared_low = (uint64_t)n_rows * LOW_DIM; + const uint64_t compared_out = (uint64_t)n_rows * OUT_DIM; + const uint64_t low_mismatch = count_bit_mismatches( + reference_low_host, candidate_low_host, + compared_low, &first_low); + const uint64_t out_mismatch = count_bit_mismatches( + reference_out_host, candidate_out_host, + compared_out, &first_out); + const uint64_t low_canary = count_poison_mismatches( + candidate_low_host, compared_low, low_count, 0x7fc10000u); + const uint64_t out_canary = count_poison_mismatches( + candidate_out_host, compared_out, out_count, 0x7fc20000u); + fprintf(stderr, + "Metal Q4 SSD-prefill exact-N=%u scale_meta=%s " + "low=%llu/%llu " + "out=%llu/%llu low_canary=%llu out_canary=%llu\n", + n_rows, + use_scale_meta ? "shared" : "legacy", + (unsigned long long)low_mismatch, + (unsigned long long)compared_low, + (unsigned long long)out_mismatch, + (unsigned long long)compared_out, + (unsigned long long)low_canary, + (unsigned long long)out_canary); + if (low_mismatch != 0) { + uint32_t reference_bits = 0; + uint32_t candidate_bits = 0; + memcpy(&reference_bits, &reference_low_host[first_low], + sizeof(reference_bits)); + memcpy(&candidate_bits, &candidate_low_host[first_low], + sizeof(candidate_bits)); + fprintf(stderr, + " first low mismatch index=%llu " + "reference=%a (0x%08x) candidate=%a (0x%08x)\n", + (unsigned long long)first_low, + reference_low_host[first_low], reference_bits, + candidate_low_host[first_low], candidate_bits); + } + if (out_mismatch != 0) { + fprintf(stderr, " first out mismatch index=%llu\n", + (unsigned long long)first_out); + } + CHECK(low_mismatch == 0, "low projection bitwise mismatch"); + CHECK(out_mismatch == 0, "output projection bitwise mismatch"); + CHECK(low_canary == 0, "low tail canary"); + CHECK(out_canary == 0, "output tail canary"); } - CHECK(low_mismatch == 0, "low projection bitwise mismatch"); - CHECK(out_mismatch == 0, "output projection bitwise mismatch"); - CHECK(low_canary == 0, "low tail canary"); - CHECK(out_canary == 0, "output tail canary"); } + CHECK(unsetenv(k_disable_scale_meta) == 0, + "restore shared scale metadata"); + CHECK(unsetenv(k_require_scale_meta) == 0, + "clear shared scale metadata requirement"); + CHECK(setenv(k_require_scale_meta, "1", 1) == 0, + "set scale-meta REQUIRE for kill-switch check"); + CHECK(setenv(k_disable_scale_meta, "1", 1) == 0, + "set scale-meta disable for kill-switch check"); + CHECK(ds4_gpu_attention_output_q4_K_batch_tensor( + candidate_out, candidate_low, NULL, NULL, model, model_bytes, + 0, out_b_offset, Q4_K_TYPE, GROUP_DIM, RANK, + N_GROUPS, OUT_DIM, heads, 21u) == -1, + "scale-meta disable wins over REQUIRE"); + CHECK(unsetenv(k_disable_scale_meta) == 0, + "clear scale-meta disable after kill-switch check"); + CHECK(unsetenv(k_require_scale_meta) == 0, + "clear scale-meta REQUIRE after kill-switch check"); /* REQUIRE implies enable. Both explicit kill switches win and must * return -1 instead of allowing a false-green row fallback. */ @@ -368,7 +419,7 @@ int main(void) { free(model); fprintf(stderr, "Metal Q4 SSD-prefill exact-N oracle PASS rows=6,8,9,16,21,30,31 " - "bitwise=1 canary=1 gates=1\n"); + "scale_meta=shared,legacy bitwise=1 canary=1 gates=1\n"); return 0; } From f639f48da1df9a49834b4cd3f933bc8c81e1d66f Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Wed, 26 Aug 2026 07:50:53 +0200 Subject: [PATCH 115/189] metal: optimize resident Q4 attention output --- ds4_metal.m | 231 ++++++++++++++++++++-- metal/dense.metal | 5 + scripts/environment_variables.tsv | 2 + tests/test_metal_q4_attn_exactn.c | 309 +++++++++++++++++++++++++++++- 4 files changed, 535 insertions(+), 12 deletions(-) diff --git a/ds4_metal.m b/ds4_metal.m index 7d60569a9..c3ed50789 100644 --- a/ds4_metal.m +++ b/ds4_metal.m @@ -28508,6 +28508,9 @@ static int ds4_gpu_attention_output_q4_K_ssd_prefill_exactn_tensor( } } +static bool ds4_gpu_tensor_prefixes_overlap(const ds4_gpu_tensor *a, uint64_t a_bytes, const ds4_gpu_tensor *b, uint64_t b_bytes); +static int ds4_gpu_encode_q4_K_f16_rhs_mm(id cb, id pipeline, id weights, NSUInteger weights_offset, const ds4_gpu_tensor *rhs_f16, ds4_gpu_tensor *out, uint64_t in_dim, uint64_t out_dim, uint64_t n_tok, uint64_t row_bytes, bool bc_out); + int ds4_gpu_attention_output_q4_K_batch_tensor( ds4_gpu_tensor *out, ds4_gpu_tensor *low, @@ -28524,7 +28527,12 @@ int ds4_gpu_attention_output_q4_K_batch_tensor( uint64_t out_dim, const ds4_gpu_tensor *heads, uint32_t n_tokens) { - if (n_tokens >= 6u && n_tokens <= 31u) { + const bool require_f16_rhs = + ds4_gpu_env_bool("DS4_METAL_REQUIRE_Q4_ATTN_OUT_B_F16_RHS") == 1; + const bool disable_f16_rhs = + ds4_gpu_env_bool("DS4_METAL_DISABLE_Q4_ATTN_OUT_B_F16_RHS") == 1; + + if (!require_f16_rhs && n_tokens >= 6u && n_tokens <= 31u) { const int exactn_rc = ds4_gpu_attention_output_q4_K_ssd_prefill_exactn_tensor( out, @@ -28547,7 +28555,7 @@ int ds4_gpu_attention_output_q4_K_batch_tensor( const bool require_tiny = tiny_scope && ds4_gpu_env_bool("DS4_METAL_REQUIRE_Q4_ATTN_OUT_TINY_BATCH") == 1; - const int failure_rc = require_tiny ? -1 : 0; + const int failure_rc = (require_tiny || require_f16_rhs) ? -1 : 0; if (!g_initialized && !ds4_gpu_init()) return failure_rc; if (!out || !low || !heads || !model_map || group_dim == 0 || rank == 0 || n_groups == 0 || out_dim == 0 || n_tokens == 0 || @@ -28587,10 +28595,13 @@ int ds4_gpu_attention_output_q4_K_batch_tensor( n_tokens, out_b_type, tiny_disabled ? 1u : 0u); return -1; } - if (n_tokens < 32u && !use_tiny_exact) return 0; + if (n_tokens < 32u && !use_tiny_exact) return failure_rc; @autoreleasepool { - const uint64_t low_dim = (uint64_t)n_groups * rank; + uint64_t low_dim = 0; + if (!ds4_gpu_u64_mul_checked((uint64_t)n_groups, rank, &low_dim)) { + return failure_rc; + } if ((group_dim % 256u) != 0 || (low_dim % 256u) != 0 || low_dim > UINT32_MAX) { return failure_rc; } @@ -28602,26 +28613,108 @@ int ds4_gpu_attention_output_q4_K_batch_tensor( return failure_rc; } - const uint64_t out_a_bytes = (uint64_t)n_groups * rank * row_a_bytes; - const uint64_t out_b_bytes = out_dim * row_b_bytes; + uint64_t out_a_bytes = 0; + uint64_t out_b_bytes = 0; + if (!ds4_gpu_u64_mul_checked(low_dim, row_a_bytes, &out_a_bytes) || + !ds4_gpu_u64_mul_checked(out_dim, row_b_bytes, &out_b_bytes)) { + return failure_rc; + } if (out_a_offset > model_size || out_a_bytes > model_size - out_a_offset || out_b_offset > model_size || out_b_bytes > model_size - out_b_offset) { fprintf(stderr, "ds4: Metal Q4 attention output batch weights are outside the mapped model\n"); return failure_rc; } - const uint64_t heads_bytes = (uint64_t)n_tokens * n_groups * group_dim * sizeof(float); - const uint64_t low_bytes = (uint64_t)n_tokens * low_dim * sizeof(float); - const uint64_t out_bytes = (uint64_t)n_tokens * out_dim * sizeof(float); + uint64_t heads_row_elements = 0; + uint64_t heads_elements = 0; + uint64_t heads_bytes = 0; + uint64_t low_elements = 0; + uint64_t low_bytes = 0; + uint64_t out_elements = 0; + uint64_t out_bytes = 0; + if (!ds4_gpu_u64_mul_checked( + (uint64_t)n_groups, group_dim, &heads_row_elements) || + !ds4_gpu_u64_mul_checked( + (uint64_t)n_tokens, heads_row_elements, &heads_elements) || + !ds4_gpu_u64_mul_checked( + heads_elements, sizeof(float), &heads_bytes) || + !ds4_gpu_u64_mul_checked( + (uint64_t)n_tokens, low_dim, &low_elements) || + !ds4_gpu_u64_mul_checked( + low_elements, sizeof(float), &low_bytes) || + !ds4_gpu_u64_mul_checked( + (uint64_t)n_tokens, out_dim, &out_elements) || + !ds4_gpu_u64_mul_checked( + out_elements, sizeof(float), &out_bytes)) { + return failure_rc; + } if (ds4_gpu_tensor_bytes(heads) < heads_bytes || ds4_gpu_tensor_bytes(low) < low_bytes || ds4_gpu_tensor_bytes(out) < out_bytes) { fprintf(stderr, "ds4: Metal Q4 attention output batch received undersized buffers\n"); return failure_rc; } - (void)group_tmp; (void)low_tmp; + const uint64_t f16_rhs_elements = low_elements; + uint64_t f16_rhs_bytes = 0; + const bool f16_rhs_size_valid = + f16_rhs_elements <= UINT32_MAX && + ds4_gpu_u64_mul_checked( + f16_rhs_elements, sizeof(uint16_t), &f16_rhs_bytes); + const bool f16_rhs_buffers_valid = + group_tmp != NULL && + ds4_gpu_tensor_bytes(group_tmp) >= f16_rhs_bytes && + ds4_gpu_tensor_buffer(group_tmp) != nil && + !ds4_gpu_tensor_prefixes_overlap( + group_tmp, f16_rhs_bytes, low, low_bytes) && + !ds4_gpu_tensor_prefixes_overlap( + group_tmp, f16_rhs_bytes, out, out_bytes) && + !ds4_gpu_tensor_prefixes_overlap( + group_tmp, f16_rhs_bytes, heads, heads_bytes); + /* Resident pre-M5 models benefit from materializing output-B's RHS + * once. Keep SSD streaming on the established F32 path: storage + * latency masks this saving, while the extra copy/cache footprint did + * not produce a stable whole-prefill win in that execution mode. */ + bool use_f16_rhs = + !g_ssd_streaming_mode && + !disable_f16_rhs && + ds4_gpu_device_is_pre_m5_apple_silicon() && + !g_batch_encoder_concurrent && + out_b_type == DS4_METAL_TENSOR_Q4_K && + (n_tokens % 32u) == 0 && + f16_rhs_size_valid && + f16_rhs_buffers_valid; + const bool f16_rhs_bc_out = (out_dim % 64u) != 0; + id f16_rhs_pipeline = nil; + id out_b_buf = nil; + uint64_t out_b_inner = 0; + if (use_f16_rhs) { + f16_rhs_pipeline = ds4_gpu_get_mul_mm_pipeline( + "kernel_mul_mm_q4_K_f16_rhs", false, f16_rhs_bc_out); + out_b_buf = ds4_gpu_wrap_model_range( + model_map, model_size, out_b_offset, out_b_bytes, + &out_b_inner); + if (!f16_rhs_pipeline || !out_b_buf || out_b_inner > NSUIntegerMax) { + use_f16_rhs = false; + } + } + if (require_f16_rhs && !use_f16_rhs) { + fprintf(stderr, + "ds4: required Metal Q4 attention output-B F16 RHS is " + "ineligible (rows=%u type=%u scratch=%llu/%llu " + "disabled=%u pre_m5=%u ssd=%u)\n", + n_tokens, + out_b_type, + (unsigned long long)(group_tmp + ? ds4_gpu_tensor_bytes(group_tmp) : 0u), + (unsigned long long)f16_rhs_bytes, + disable_f16_rhs ? 1u : 0u, + ds4_gpu_device_is_pre_m5_apple_silicon() ? 1u : 0u, + g_ssd_streaming_mode ? 1u : 0u); + return -1; + } + const uint64_t padded_n_tokens_u64 = ((uint64_t)n_tokens + DS4_METAL_ATTN_OUT_MPP_TILE_N - 1u) / DS4_METAL_ATTN_OUT_MPP_TILE_N * DS4_METAL_ATTN_OUT_MPP_TILE_N; @@ -28782,7 +28875,42 @@ int ds4_gpu_attention_output_q4_K_batch_tensor( } if (ok) { - if (use_tiny_exact && out_b_type == DS4_METAL_TENSOR_Q8_0) { + if (use_f16_rhs) { + const bool encoded_f16_rhs = + ds4_gpu_encode_cpy_f32_f16_1d( + cb, + ds4_gpu_tensor_buffer(low), + ds4_gpu_tensor_offset(low), + ds4_gpu_tensor_buffer(group_tmp), + ds4_gpu_tensor_offset(group_tmp), + (uint32_t)f16_rhs_elements) != 0 && + ds4_gpu_encode_q4_K_f16_rhs_mm( + cb, + f16_rhs_pipeline, + out_b_buf, + (NSUInteger)out_b_inner, + group_tmp, + out, + low_dim, + out_dim, + n_tokens, + row_b_bytes, + f16_rhs_bc_out) != 0; + if (!encoded_f16_rhs) { + ok = require_f16_rhs + ? false + : ds4_gpu_matmul_quant_tensor( + out, + model_map, + model_size, + out_b_offset, + out_b_type, + low_dim, + out_dim, + low, + n_tokens) != 0; + } + } else if (use_tiny_exact && out_b_type == DS4_METAL_TENSOR_Q8_0) { ok = ds4_gpu_matmul_q8_0_decode_rows_exact_tensor( out, model_map, @@ -29609,6 +29737,7 @@ static int ds4_gpu_encode_cpy_f32_f16_1d( id pipeline = use_contiguous ? g_cpy_contig_f32_f16_pipeline : g_cpy_f32_f16_pipeline; + if (!pipeline) return 0; const NSUInteger work_items = use_contiguous ? ((NSUInteger)n + 3u) / 4u : (NSUInteger)n; @@ -29616,6 +29745,7 @@ static int ds4_gpu_encode_cpy_f32_f16_1d( const NSUInteger groups = (work_items + nth - 1u) / nth; id enc = ds4_gpu_compute_encoder(cb); + if (!enc) return 0; [enc setComputePipelineState:pipeline]; if (use_contiguous) { [enc setBytes:&n length:sizeof(n) atIndex:0]; @@ -49388,3 +49518,82 @@ int ds4_gpu_glm53_kda_prefill( void ds4_gpu_set_glm_mtp_verify_mode(bool enabled) { (void)enabled; } + +static bool ds4_gpu_tensor_prefixes_overlap( + const ds4_gpu_tensor *a, + uint64_t a_bytes, + const ds4_gpu_tensor *b, + uint64_t b_bytes) { + if (!a || !b || a_bytes == 0 || b_bytes == 0 || + ds4_gpu_tensor_buffer(a) != ds4_gpu_tensor_buffer(b)) { + return false; + } + const uint64_t a_off = (uint64_t)ds4_gpu_tensor_offset(a); + const uint64_t b_off = (uint64_t)ds4_gpu_tensor_offset(b); + return a_off <= b_off ? b_off - a_off < a_bytes + : a_off - b_off < b_bytes; +} + +/* Encode the legacy Q4_K prefill matmul against an RHS that has already been + * rounded to F16. kernel_mul_mm_q4_K_f32 performs that same rounding while + * staging every 64-row output tile, so materializing it once can remove a + * large amount of repeated activation traffic without changing the MMA or + * accumulation schedule. This helper deliberately does not submit the + * command buffer: attention-A, the conversion, and output-B must remain in + * one ordered batch. */ +static int ds4_gpu_encode_q4_K_f16_rhs_mm( + id cb, + id pipeline, + id weights, + NSUInteger weights_offset, + const ds4_gpu_tensor *rhs_f16, + ds4_gpu_tensor *out, + uint64_t in_dim, + uint64_t out_dim, + uint64_t n_tok, + uint64_t row_bytes, + bool bc_out) { + if (!cb || !pipeline || !weights || !rhs_f16 || !out || + in_dim == 0 || out_dim == 0 || n_tok == 0 || + (in_dim % 256u) != 0 || (n_tok % 32u) != 0 || + in_dim > UINT32_MAX || out_dim > UINT32_MAX || n_tok > UINT32_MAX || + in_dim > UINT64_MAX / n_tok || + in_dim * n_tok > UINT64_MAX / sizeof(uint16_t) || + out_dim > UINT64_MAX / n_tok || + out_dim * n_tok > UINT64_MAX / sizeof(float)) { + return 0; + } + + const uint64_t rhs_bytes = in_dim * n_tok * sizeof(uint16_t); + const uint64_t out_bytes = out_dim * n_tok * sizeof(float); + id rhs_buf = ds4_gpu_tensor_buffer(rhs_f16); + id out_buf = ds4_gpu_tensor_buffer(out); + if (!rhs_buf || !out_buf || + ds4_gpu_tensor_bytes(rhs_f16) < rhs_bytes || + ds4_gpu_tensor_bytes(out) < out_bytes) { + return 0; + } + + ds4_gpu_mul_mm_args args = + ds4_gpu_make_mm_args(in_dim, out_dim, n_tok, row_bytes); + args.nb10 = sizeof(uint16_t); + args.nb11 = in_dim * sizeof(uint16_t); + args.nb12 = in_dim * n_tok * sizeof(uint16_t); + args.nb13 = args.nb12; + + id enc = ds4_gpu_compute_encoder(cb); + if (!enc) return 0; + [enc setComputePipelineState:pipeline]; + [enc setBytes:&args length:sizeof(args) atIndex:0]; + [enc setBuffer:weights offset:weights_offset atIndex:1]; + [enc setBuffer:rhs_buf offset:ds4_gpu_tensor_offset(rhs_f16) atIndex:2]; + [enc setBuffer:out_buf offset:ds4_gpu_tensor_offset(out) atIndex:3]; + [enc setThreadgroupMemoryLength:(bc_out ? 8192u : 6144u) atIndex:0]; + [enc dispatchThreadgroups: + MTLSizeMake((NSUInteger)n_tok / 32u, + ((NSUInteger)out_dim + 63u) / 64u, + 1) + threadsPerThreadgroup:MTLSizeMake(128, 1, 1)]; + ds4_gpu_end_compute_encoder(cb, enc); + return 1; +} diff --git a/metal/dense.metal b/metal/dense.metal index e5c0f1d5a..c9dde5615 100644 --- a/metal/dense.metal +++ b/metal/dense.metal @@ -2457,3 +2457,8 @@ template [[host_name("kernel_mul_mm_f16_f32")]] kernel mul_mm_t kernel_mul_mm; template [[host_name("kernel_mul_mm_q4_0_f32")]] kernel mul_mm_t kernel_mul_mm; template [[host_name("kernel_mul_mm_q4_K_f32")]] kernel mul_mm_t kernel_mul_mm; +// Q4_K output projection with a pre-materialized F16 RHS. The ordinary F32 +// variant performs this same F32-to-F16 conversion every time a 64-row weight +// tile revisits the activation matrix; this variant lets the host perform it +// once while preserving the MMA tile and accumulation order. +template [[host_name("kernel_mul_mm_q4_K_f16_rhs")]] kernel mul_mm_t kernel_mul_mm; diff --git a/scripts/environment_variables.tsv b/scripts/environment_variables.tsv index 85eae5a16..f206deb42 100644 --- a/scripts/environment_variables.tsv +++ b/scripts/environment_variables.tsv @@ -581,6 +581,7 @@ runtime/metal DS4_METAL_DISABLE_PRE_M5_ROUTER_TRANSFORM_FINALIZE_FUSION presence runtime/metal DS4_METAL_DISABLE_PRO_Q4_EXPERT_ADDRESS_AUTO presence rollback; unset: automatic/default path; any value including 0 disables Disables pro Q4 expert address auto. ds4_metal.m:19710 runtime/metal DS4_METAL_DISABLE_PRO_Q4_EXPERT_TABLE_AUTO presence rollback; unset: automatic/default path; any value including 0 disables Disables pro Q4 expert table auto. ds4.c:21040 runtime/metal DS4_METAL_DISABLE_PRO_Q4_EXPERT_TABLE_PRELOAD presence rollback; unset: automatic/default path; any value including 0 disables Disables pro Q4 expert table preload. ds4.c:58780 +runtime/metal DS4_METAL_DISABLE_Q4_ATTN_OUT_B_F16_RHS value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Disables resident pre-M5 Q4 attention output-B F16 RHS materialization and restores per-tile F32 staging. ds4_metal.m:28533 runtime/metal DS4_METAL_DISABLE_Q4_ATTN_OUT_HC_FUSE presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 attn out HC fuse. ds4_metal.m:47624 runtime/metal DS4_METAL_DISABLE_Q4_ATTN_OUT_TINY_BATCH value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Disables Q4 attn out tiny batch. ds4_metal.m:28396 runtime/metal DS4_METAL_DISABLE_Q4_BATCH_EXPERT_TABLE presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 batch expert table. ds4_metal.m:44883 @@ -837,6 +838,7 @@ runtime/metal DS4_METAL_REQUIRE_GATHERED_KV_STAGE presence strict check; unset: runtime/metal DS4_METAL_REQUIRE_IQ2_XXS_SSD_PREFILL_MM value-aware boolean; default implicit fail-closed only with complete selected-address domain; explicit 1 is strict, 0 permits fallback Makes eligible IQ2_XXS/Q2_K grouped SSD-prefill MM fail closed. ds4_metal.m:44782 runtime/metal DS4_METAL_REQUIRE_M1_IQ2_MID_ONLY presence strict check; unset: fallback allowed; any value including 0 requires the path Requires M1 IQ2 mid only and makes eligible fallback fail closed. ds4_metal.m:42074 runtime/metal DS4_METAL_REQUIRE_OUTPUT_HC_WEIGHTS4 presence strict check; unset: fallback allowed; any value including 0 requires the path Requires output HC weights4 and makes eligible fallback fail closed. ds4_metal.m:46789 +runtime/metal DS4_METAL_REQUIRE_Q4_ATTN_OUT_B_F16_RHS value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Requires the resident pre-M5 Q4 attention output-B F16 RHS path and makes an ineligible or disabled candidate fail closed. ds4_metal.m:28531 runtime/metal DS4_METAL_REQUIRE_Q4_ATTN_OUT_TINY_BATCH value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Requires Q4 attn out tiny batch and makes eligible fallback fail closed. ds4_metal.m:28373 runtime/metal DS4_METAL_REQUIRE_Q4_SSD_PREFILL_ATTN_OUT_EXACTN value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Requires Q4 SSD prefill attn out exactn and makes eligible fallback fail closed. ds4_metal.m:28089 runtime/metal DS4_METAL_REQUIRE_Q4_SSD_PREFILL_ATTN_OUT_SCALE_META value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Requires shared scale/min metadata in the Q4 SSD prefill attention-output exact-N kernel and makes fallback fail closed. ds4_metal.m:28209 diff --git a/tests/test_metal_q4_attn_exactn.c b/tests/test_metal_q4_attn_exactn.c index b2e879637..8cc2bb4f4 100644 --- a/tests/test_metal_q4_attn_exactn.c +++ b/tests/test_metal_q4_attn_exactn.c @@ -30,7 +30,7 @@ enum { LOW_DIM = N_GROUPS * RANK, OUT_DIM = 67u, MAX_ROWS = 31u, - ALLOC_ROWS = 32u, + ALLOC_ROWS = 33u, }; typedef struct { @@ -51,6 +51,10 @@ static const char *k_disable_scale_meta = static const char *k_require_scale_meta = "DS4_METAL_REQUIRE_Q4_SSD_PREFILL_ATTN_OUT_SCALE_META"; static const char *k_disable_classic = "DS4_METAL_DISABLE_Q4_MV_CLASSIC"; +static const char *k_disable_f16_rhs = + "DS4_METAL_DISABLE_Q4_ATTN_OUT_B_F16_RHS"; +static const char *k_require_f16_rhs = + "DS4_METAL_REQUIRE_Q4_ATTN_OUT_B_F16_RHS"; static void fail(const char *what) { fprintf(stderr, "Metal Q4 SSD-prefill exact-N oracle FAIL: %s\n", what); @@ -175,6 +179,8 @@ int main(void) { CHECK(unsetenv(k_require_scale_meta) == 0, "clear scale-meta require env"); CHECK(unsetenv(k_disable_classic) == 0, "clear classic kill env"); + CHECK(unsetenv(k_disable_f16_rhs) == 0, "clear F16 RHS disable env"); + CHECK(unsetenv(k_require_f16_rhs) == 0, "clear F16 RHS require env"); CHECK(ds4_gpu_init() != 0, "Metal init"); if (!ds4_gpu_device_is_pre_m5_apple_silicon()) { @@ -363,6 +369,307 @@ int main(void) { CHECK(out_canary == 0, "output tail canary"); } } + + /* At N=32 both output-B variants execute the same legacy M64xN32xK32 + * schedule. The candidate differs only by materializing the staging cast + * once into scratch, so the complete wrapper output must remain bitwise + * identical. Keep one extra output row and guards around the F16 scratch + * to catch either output or conversion overruns. */ + ds4_gpu_set_ssd_streaming(false); + ds4_gpu_set_quality(true); + enum { F16_RHS_ROWS = 32u, F16_RHS_GUARD = 64u }; + const uint64_t f16_rhs_payload = + (uint64_t)F16_RHS_ROWS * LOW_DIM; + const uint64_t f16_rhs_storage = + F16_RHS_GUARD + f16_rhs_payload + F16_RHS_GUARD; + uint16_t *f16_rhs_host = malloc( + (size_t)f16_rhs_storage * sizeof(uint16_t)); + ds4_gpu_tensor *f16_rhs_base = ds4_gpu_tensor_alloc( + f16_rhs_storage * sizeof(uint16_t)); + ds4_gpu_tensor *f16_rhs = ds4_gpu_tensor_view( + f16_rhs_base, + F16_RHS_GUARD * sizeof(uint16_t), + f16_rhs_payload * sizeof(uint16_t)); + CHECK(f16_rhs_host && f16_rhs_base && f16_rhs, + "F16 RHS scratch allocation"); + for (uint64_t i = 0; i < f16_rhs_storage; i++) { + f16_rhs_host[i] = + (uint16_t)(0x7e00u | (uint16_t)(i & 0x1ffu)); + } + CHECK(ds4_gpu_tensor_write( + f16_rhs_base, 0, f16_rhs_host, + f16_rhs_storage * sizeof(uint16_t)) != 0, + "F16 RHS scratch poison"); + + poison(reference_low_host, low_count, 0x7fc30000u); + poison(reference_out_host, out_count, 0x7fc40000u); + poison(candidate_low_host, low_count, 0x7fc30000u); + poison(candidate_out_host, out_count, 0x7fc40000u); + CHECK(ds4_gpu_tensor_write(reference_low, 0, reference_low_host, + low_count * sizeof(float)) != 0, + "F16 RHS baseline low poison"); + CHECK(ds4_gpu_tensor_write(reference_out, 0, reference_out_host, + out_count * sizeof(float)) != 0, + "F16 RHS baseline out poison"); + CHECK(ds4_gpu_tensor_write(candidate_low, 0, candidate_low_host, + low_count * sizeof(float)) != 0, + "F16 RHS candidate low poison"); + CHECK(ds4_gpu_tensor_write(candidate_out, 0, candidate_out_host, + out_count * sizeof(float)) != 0, + "F16 RHS candidate out poison"); + + CHECK(ds4_gpu_attention_output_q4_K_batch_tensor( + reference_out, reference_low, NULL, NULL, + model, model_bytes, 0, out_b_offset, Q4_K_TYPE, + GROUP_DIM, RANK, N_GROUPS, OUT_DIM, heads, + F16_RHS_ROWS) == 1, + "F16 RHS baseline dispatch"); + CHECK(ds4_gpu_attention_output_q4_K_batch_tensor( + candidate_out, candidate_low, f16_rhs, NULL, + model, model_bytes, 0, out_b_offset, Q4_K_TYPE, + GROUP_DIM, RANK, N_GROUPS, OUT_DIM, heads, + F16_RHS_ROWS) == 1, + "F16 RHS default-on candidate dispatch"); + CHECK(ds4_gpu_tensor_read(reference_low, 0, reference_low_host, + low_count * sizeof(float)) != 0, + "F16 RHS baseline low read"); + CHECK(ds4_gpu_tensor_read(reference_out, 0, reference_out_host, + out_count * sizeof(float)) != 0, + "F16 RHS baseline out read"); + CHECK(ds4_gpu_tensor_read(candidate_low, 0, candidate_low_host, + low_count * sizeof(float)) != 0, + "F16 RHS candidate low read"); + CHECK(ds4_gpu_tensor_read(candidate_out, 0, candidate_out_host, + out_count * sizeof(float)) != 0, + "F16 RHS candidate out read"); + CHECK(ds4_gpu_tensor_read( + f16_rhs_base, 0, f16_rhs_host, + f16_rhs_storage * sizeof(uint16_t)) != 0, + "F16 RHS scratch read"); + + uint64_t first_low = UINT64_MAX; + uint64_t first_out = UINT64_MAX; + const uint64_t f16_low_mismatch = count_bit_mismatches( + reference_low_host, candidate_low_host, + (uint64_t)F16_RHS_ROWS * LOW_DIM, &first_low); + const uint64_t f16_out_mismatch = count_bit_mismatches( + reference_out_host, candidate_out_host, + (uint64_t)F16_RHS_ROWS * OUT_DIM, &first_out); + const uint64_t baseline_low_tail = count_poison_mismatches( + reference_low_host, (uint64_t)F16_RHS_ROWS * LOW_DIM, + low_count, 0x7fc30000u); + const uint64_t candidate_low_tail = count_poison_mismatches( + candidate_low_host, (uint64_t)F16_RHS_ROWS * LOW_DIM, + low_count, 0x7fc30000u); + const uint64_t baseline_out_tail = count_poison_mismatches( + reference_out_host, (uint64_t)F16_RHS_ROWS * OUT_DIM, + out_count, 0x7fc40000u); + const uint64_t candidate_out_tail = count_poison_mismatches( + candidate_out_host, (uint64_t)F16_RHS_ROWS * OUT_DIM, + out_count, 0x7fc40000u); + uint64_t f16_guard_mismatch = 0; + uint64_t f16_payload_poison = 0; + for (uint64_t i = 0; i < F16_RHS_GUARD; i++) { + const uint16_t expected = + (uint16_t)(0x7e00u | (uint16_t)(i & 0x1ffu)); + if (f16_rhs_host[i] != expected) f16_guard_mismatch++; + } + for (uint64_t i = F16_RHS_GUARD; + i < F16_RHS_GUARD + f16_rhs_payload; i++) { + const uint16_t poison_value = + (uint16_t)(0x7e00u | (uint16_t)(i & 0x1ffu)); + if (f16_rhs_host[i] == poison_value) f16_payload_poison++; + } + for (uint64_t i = F16_RHS_GUARD + f16_rhs_payload; + i < f16_rhs_storage; i++) { + const uint16_t expected = + (uint16_t)(0x7e00u | (uint16_t)(i & 0x1ffu)); + if (f16_rhs_host[i] != expected) f16_guard_mismatch++; + } + fprintf(stderr, + "Metal Q4 attention output-B F16 RHS N=32 " + "low=%llu out=%llu low_tail=%llu/%llu " + "out_tail=%llu/%llu scratch_guard=%llu payload_poison=%llu\n", + (unsigned long long)f16_low_mismatch, + (unsigned long long)f16_out_mismatch, + (unsigned long long)baseline_low_tail, + (unsigned long long)candidate_low_tail, + (unsigned long long)baseline_out_tail, + (unsigned long long)candidate_out_tail, + (unsigned long long)f16_guard_mismatch, + (unsigned long long)f16_payload_poison); + CHECK(f16_low_mismatch == 0, "F16 RHS low bitwise mismatch"); + CHECK(f16_out_mismatch == 0, "F16 RHS output bitwise mismatch"); + CHECK(baseline_low_tail == 0 && candidate_low_tail == 0, + "F16 RHS low tail canary"); + CHECK(baseline_out_tail == 0 && candidate_out_tail == 0, + "F16 RHS output tail canary"); + CHECK(f16_guard_mismatch == 0, "F16 RHS scratch canary"); + CHECK(f16_payload_poison == 0, "F16 RHS default-on materialization"); + + /* Flash uses an output dimension divisible by 64, which selects the + * direct full-tile store and its smaller threadgroup allocation. Reuse + * the first 64 rows of the fixture so both legacy specializations remain + * covered without growing the model allocation. */ + enum { F16_RHS_FULL_TILE_OUT_DIM = 64u }; + const uint64_t f16_full_tile_count = + (uint64_t)F16_RHS_ROWS * F16_RHS_FULL_TILE_OUT_DIM; + poison(reference_out_host, out_count, 0x7fc50000u); + poison(candidate_out_host, out_count, 0x7fc50000u); + CHECK(ds4_gpu_tensor_write(reference_out, 0, reference_out_host, + out_count * sizeof(float)) != 0, + "F16 RHS full-tile baseline poison"); + CHECK(ds4_gpu_tensor_write(candidate_out, 0, candidate_out_host, + out_count * sizeof(float)) != 0, + "F16 RHS full-tile candidate poison"); + CHECK(ds4_gpu_attention_output_q4_K_batch_tensor( + reference_out, reference_low, NULL, NULL, + model, model_bytes, 0, out_b_offset, Q4_K_TYPE, + GROUP_DIM, RANK, N_GROUPS, F16_RHS_FULL_TILE_OUT_DIM, heads, + F16_RHS_ROWS) == 1, + "F16 RHS full-tile baseline dispatch"); + CHECK(ds4_gpu_attention_output_q4_K_batch_tensor( + candidate_out, candidate_low, f16_rhs, NULL, + model, model_bytes, 0, out_b_offset, Q4_K_TYPE, + GROUP_DIM, RANK, N_GROUPS, F16_RHS_FULL_TILE_OUT_DIM, heads, + F16_RHS_ROWS) == 1, + "F16 RHS full-tile candidate dispatch"); + CHECK(ds4_gpu_tensor_read(reference_out, 0, reference_out_host, + out_count * sizeof(float)) != 0, + "F16 RHS full-tile baseline read"); + CHECK(ds4_gpu_tensor_read(candidate_out, 0, candidate_out_host, + out_count * sizeof(float)) != 0, + "F16 RHS full-tile candidate read"); + uint64_t first_full_tile = UINT64_MAX; + const uint64_t f16_full_tile_mismatch = count_bit_mismatches( + reference_out_host, candidate_out_host, + f16_full_tile_count, &first_full_tile); + const uint64_t baseline_full_tile_tail = count_poison_mismatches( + reference_out_host, f16_full_tile_count, + out_count, 0x7fc50000u); + const uint64_t candidate_full_tile_tail = count_poison_mismatches( + candidate_out_host, f16_full_tile_count, + out_count, 0x7fc50000u); + fprintf(stderr, + "Metal Q4 attention output-B F16 RHS full tile N=32 " + "out=%llu tail=%llu/%llu\n", + (unsigned long long)f16_full_tile_mismatch, + (unsigned long long)baseline_full_tile_tail, + (unsigned long long)candidate_full_tile_tail); + CHECK(f16_full_tile_mismatch == 0, + "F16 RHS full-tile output bitwise mismatch"); + CHECK(baseline_full_tile_tail == 0 && candidate_full_tile_tail == 0, + "F16 RHS full-tile output tail canary"); + + /* Restore the boundary-specialized reference consumed by the SSD + * fallback comparison below. */ + poison(reference_out_host, out_count, 0x7fc40000u); + CHECK(ds4_gpu_tensor_write(reference_out, 0, reference_out_host, + out_count * sizeof(float)) != 0, + "F16 RHS boundary reference poison restore"); + CHECK(ds4_gpu_attention_output_q4_K_batch_tensor( + reference_out, reference_low, NULL, NULL, + model, model_bytes, 0, out_b_offset, Q4_K_TYPE, + GROUP_DIM, RANK, N_GROUPS, OUT_DIM, heads, + F16_RHS_ROWS) == 1, + "F16 RHS boundary reference restore dispatch"); + CHECK(ds4_gpu_tensor_read(reference_out, 0, reference_out_host, + out_count * sizeof(float)) != 0, + "F16 RHS boundary reference restore read"); + + CHECK(setenv(k_require_f16_rhs, "1", 1) == 0, + "require F16 RHS candidate"); + CHECK(ds4_gpu_attention_output_q4_K_batch_tensor( + candidate_out, candidate_low, f16_rhs, NULL, + model, model_bytes, 0, out_b_offset, Q4_K_TYPE, + GROUP_DIM, RANK, N_GROUPS, OUT_DIM, heads, + F16_RHS_ROWS) == 1, + "required F16 RHS candidate dispatch"); + CHECK(ds4_gpu_attention_output_q4_K_batch_tensor( + candidate_out, candidate_low, f16_rhs, NULL, + model, model_bytes, 0, out_b_offset, Q4_K_TYPE, + GROUP_DIM, RANK, N_GROUPS, OUT_DIM, heads, 31u) == -1, + "required F16 RHS rejects N below one MM tile"); + CHECK(setenv(k_disable_f16_rhs, "1", 1) == 0, + "disable required F16 RHS candidate"); + CHECK(ds4_gpu_attention_output_q4_K_batch_tensor( + candidate_out, candidate_low, f16_rhs, NULL, + model, model_bytes, 0, out_b_offset, Q4_K_TYPE, + GROUP_DIM, RANK, N_GROUPS, OUT_DIM, heads, + F16_RHS_ROWS) == -1, + "F16 RHS disable wins over REQUIRE"); + CHECK(unsetenv(k_disable_f16_rhs) == 0, + "clear F16 RHS disable env"); + ds4_gpu_set_ssd_streaming(true); + CHECK(ds4_gpu_attention_output_q4_K_batch_tensor( + candidate_out, candidate_low, f16_rhs, NULL, + model, model_bytes, 0, out_b_offset, Q4_K_TYPE, + GROUP_DIM, RANK, N_GROUPS, OUT_DIM, heads, + F16_RHS_ROWS) == -1, + "required F16 RHS rejects SSD streaming"); + CHECK(unsetenv(k_require_f16_rhs) == 0, + "clear F16 RHS require env"); + + /* In ordinary SSD mode the wrapper must use output-B's established F32 + * path, return success, and leave the F16 scratch completely untouched. */ + for (uint64_t i = 0; i < f16_rhs_storage; i++) { + f16_rhs_host[i] = + (uint16_t)(0x7e00u | (uint16_t)(i & 0x1ffu)); + } + poison(candidate_low_host, low_count, 0x7fc30000u); + poison(candidate_out_host, out_count, 0x7fc40000u); + CHECK(ds4_gpu_tensor_write( + f16_rhs_base, 0, f16_rhs_host, + f16_rhs_storage * sizeof(uint16_t)) != 0, + "SSD fallback scratch poison"); + CHECK(ds4_gpu_tensor_write(candidate_low, 0, candidate_low_host, + low_count * sizeof(float)) != 0, + "SSD fallback low poison"); + CHECK(ds4_gpu_tensor_write(candidate_out, 0, candidate_out_host, + out_count * sizeof(float)) != 0, + "SSD fallback out poison"); + CHECK(ds4_gpu_attention_output_q4_K_batch_tensor( + candidate_out, candidate_low, f16_rhs, NULL, + model, model_bytes, 0, out_b_offset, Q4_K_TYPE, + GROUP_DIM, RANK, N_GROUPS, OUT_DIM, heads, + F16_RHS_ROWS) == 1, + "SSD fallback F32 dispatch"); + CHECK(ds4_gpu_tensor_read(candidate_low, 0, candidate_low_host, + low_count * sizeof(float)) != 0, + "SSD fallback low read"); + CHECK(ds4_gpu_tensor_read(candidate_out, 0, candidate_out_host, + out_count * sizeof(float)) != 0, + "SSD fallback out read"); + CHECK(ds4_gpu_tensor_read( + f16_rhs_base, 0, f16_rhs_host, + f16_rhs_storage * sizeof(uint16_t)) != 0, + "SSD fallback scratch read"); + const uint64_t ssd_low_mismatch = count_bit_mismatches( + reference_low_host, candidate_low_host, + (uint64_t)F16_RHS_ROWS * LOW_DIM, &first_low); + const uint64_t ssd_out_mismatch = count_bit_mismatches( + reference_out_host, candidate_out_host, + (uint64_t)F16_RHS_ROWS * OUT_DIM, &first_out); + uint64_t ssd_scratch_mismatch = 0; + for (uint64_t i = 0; i < f16_rhs_storage; i++) { + const uint16_t expected = + (uint16_t)(0x7e00u | (uint16_t)(i & 0x1ffu)); + if (f16_rhs_host[i] != expected) ssd_scratch_mismatch++; + } + fprintf(stderr, + "Metal Q4 attention output-B SSD fallback N=32 " + "low=%llu out=%llu scratch=%llu\n", + (unsigned long long)ssd_low_mismatch, + (unsigned long long)ssd_out_mismatch, + (unsigned long long)ssd_scratch_mismatch); + CHECK(ssd_low_mismatch == 0, "SSD fallback low bitwise mismatch"); + CHECK(ssd_out_mismatch == 0, "SSD fallback output bitwise mismatch"); + CHECK(ssd_scratch_mismatch == 0, "SSD fallback touched F16 scratch"); + ds4_gpu_set_quality(false); + ds4_gpu_tensor_free(f16_rhs); + ds4_gpu_tensor_free(f16_rhs_base); + free(f16_rhs_host); + CHECK(unsetenv(k_disable_scale_meta) == 0, "restore shared scale metadata"); CHECK(unsetenv(k_require_scale_meta) == 0, From 0e80b90e51c96fd61abac6b3eeaaf0e9091e7fbe Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Wed, 26 Aug 2026 12:40:03 +0200 Subject: [PATCH 116/189] metal: accelerate resident Q4 attn_q_b prefill Prewarm Q4_K attention projection weights into bounded F16 sidecars on pre-M5 Metal, with prompt-aware admission, cache generations, and lifecycle eviction. Add bitwise/canary coverage, memory-safe test targets, and frontend preflight outside measured prefill timing. --- .gitignore | 1 + Makefile | 39 +- ds4.c | 412 ++++++++-- ds4.h | 6 + ds4_bench.c | 11 + ds4_cli.c | 15 +- ds4_cuda.cu | 3 +- ds4_gpu.h | 61 ++ ds4_metal.m | 1098 +++++++++++++++++++++++++- metal/dense.metal | 38 + rocm/ds4_rocm_norm_rope.cuh | 3 +- scripts/environment_variables.tsv | 13 +- tests/test_metal_q4_qb_f16_cache.c | 1181 ++++++++++++++++++++++++++++ 13 files changed, 2817 insertions(+), 64 deletions(-) create mode 100644 tests/test_metal_q4_qb_f16_cache.c diff --git a/.gitignore b/.gitignore index 4743e8cf6..4db2bbf6b 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,7 @@ /tests/test_layer_pack /tests/test_metal_session_batch /tests/test_metal_indexer_q4 +/tests/test_metal_q4_qb_f16_cache /tests/test_metal_q4_streams /tests/test_mxfp4_cuda /tests/test_mxfp4_dot diff --git a/Makefile b/Makefile index 2fbdabc26..87999f657 100644 --- a/Makefile +++ b/Makefile @@ -69,7 +69,7 @@ DS4_LINK_LIBS ?= $(CUDA_LDLIBS) METAL_LDLIBS := $(LDLIBS) endif -.PHONY: all help clean test environment-docs test-quantizer-indexer-q4 test-rocm test-glm53-kda-rocm test-metal-session-batch test-metal-session-batch-ssd test-metal-q4-streams test-metal-indexer-q4 test-metal-q4-attn-exactn test-metal-exactn-oracle test-metal-dspark-capture test-metal-iq2-midonly test-metal-iq2-ssd-grouped-mm test-metal-iq2-live-index test-mxfp4-metal test-mxfp4-cuda test-mxfp4-rocm test-mmq-parity-cuda test-rocm-q4-parity test-rocm-q4-dense test-rocm-q4-pair test-rocm-q4-prefill test-strix-rocm-q4-parity test-strix-rocm-q4-prefill test-strix-rocm-q4-prefill-long test-cuda-session-batch test-cuda-mixed-batch dspark-acceptance dspark-verify-depth rocm-dspark-acceptance rocm-dspark-verify-depth mtp-verify-depth cpu cuda cuda-spark cuda-generic cuda-regression strix-halo rocm +.PHONY: all help clean test test-ssd environment-docs test-quantizer-indexer-q4 test-rocm test-glm53-kda-rocm test-metal-session-batch test-metal-session-batch-ssd test-metal-q4-streams test-metal-indexer-q4 test-metal-q4-attn-exactn test-metal-q4-qb-f16-cache test-metal-q4-qb-f16-cache-timing test-metal-exactn-oracle test-metal-dspark-capture test-metal-iq2-midonly test-metal-iq2-ssd-grouped-mm test-metal-iq2-live-index test-mxfp4-metal test-mxfp4-cuda test-mxfp4-rocm test-mmq-parity-cuda test-rocm-q4-parity test-rocm-q4-dense test-rocm-q4-pair test-rocm-q4-prefill test-strix-rocm-q4-parity test-strix-rocm-q4-prefill test-strix-rocm-q4-prefill-long test-cuda-session-batch test-cuda-mixed-batch dspark-acceptance dspark-verify-depth rocm-dspark-acceptance rocm-dspark-verify-depth mtp-verify-depth cpu cuda cuda-spark cuda-generic cuda-regression strix-halo rocm gguf-tools/deepseek4-quantize: gguf-tools/deepseek4-quantize.c gguf-tools/quants.c gguf-tools/quants.h $(MAKE) -C gguf-tools deepseek4-quantize @@ -90,12 +90,15 @@ help: @echo " make Build Metal ./ds4, ./ds4-server, ./ds4-bench, ./ds4-eval, and ./ds4-agent" @echo " make cpu Build CPU-only ./ds4, ./ds4-server, ./ds4-bench, ./ds4-eval, and ./ds4-agent" @echo " make test Build and run tests" + @echo " make test-ssd Run the model suite with cold SSD streaming" @echo " make test-quantizer-indexer-q4 Check direct F16-to-Q4_K indexer conversion" @echo " make environment-docs Generate and verify the environment variable inventory" @echo " make test-metal-session-batch-ssd Exact-logit Metal SSD union control/candidate oracle" @echo " make test-metal-q4-streams Check resident Q4 Metal stream overlap" @echo " make test-metal-indexer-q4 Check the production-shape Q4_K indexer projection" @echo " make test-metal-q4-attn-exactn Bitwise/canary oracle for M1-M4 SSD-prefill Q4 attention output" + @echo " make test-metal-q4-qb-f16-cache Bitwise/canary oracle for the resident M1-M4 Q4 q_b F16 cache" + @echo " make test-metal-q4-qb-f16-cache-timing Time the Q4 q_b F16 cache at N=4096" @echo " make test-metal-dspark-capture Check fused DSpark HC capture bitwise" @echo " make test-metal-iq2-midonly Check M1 IQ2 addr mid-only output and sentinels" @echo " make test-metal-iq2-live-index Check IQ2 SSD live-cache index policy and fallback" @@ -197,6 +200,28 @@ test-metal-q4-attn-exactn: tests/test_metal_q4_attn_exactn -u DS4_METAL_DISABLE_Q4_MV_CLASSIC \ ./tests/test_metal_q4_attn_exactn +tests/test_metal_q4_qb_f16_cache.o: tests/test_metal_q4_qb_f16_cache.c ds4_gpu.h + $(CC) $(CFLAGS) -I. -c -o $@ $< + +tests/test_metal_q4_qb_f16_cache: tests/test_metal_q4_qb_f16_cache.o ds4_metal.o + $(CC) $(CFLAGS) -o $@ $^ $(METAL_LDLIBS) + +test-metal-q4-qb-f16-cache: tests/test_metal_q4_qb_f16_cache + env -u DS4_METAL_DISABLE_Q4_ATTN_Q_B_F16_CACHE \ + -u DS4_TEST_METAL_Q4_QB_F16_CACHE_TIMING \ + -u DS4_TEST_METAL_Q4_QB_F16_CACHE_TIMING_TOKENS \ + DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MIN_TOKENS=32 \ + DS4_METAL_REQUIRE_Q4_ATTN_Q_B_F16_CACHE=1 \ + ./tests/test_metal_q4_qb_f16_cache + +test-metal-q4-qb-f16-cache-timing: tests/test_metal_q4_qb_f16_cache + env -u DS4_METAL_DISABLE_Q4_ATTN_Q_B_F16_CACHE \ + DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MIN_TOKENS=32 \ + DS4_METAL_REQUIRE_Q4_ATTN_Q_B_F16_CACHE=1 \ + DS4_TEST_METAL_Q4_QB_F16_CACHE_TIMING=1 \ + DS4_TEST_METAL_Q4_QB_F16_CACHE_TIMING_TOKENS=4096 \ + ./tests/test_metal_q4_qb_f16_cache + tests/test_metal_dspark_capture.o: tests/test_metal_dspark_capture.c ds4_gpu.h $(CC) $(CFLAGS) -I. -c -o $@ $< @@ -304,6 +329,7 @@ help: @echo " make test-rocm Core regression suite on ROCm-only hosts" @echo " make cpu Build CPU-only ./ds4, ./ds4-server, ./ds4-bench, ./ds4-eval, and ./ds4-agent" @echo " make test Build and run tests" + @echo " make test-ssd Run the model suite with cold SSD streaming" @echo " make environment-docs Generate and verify the environment variable inventory" @echo " make dspark-verify-depth Run DSpark speculative verification smoke if support GGUF is present" @echo " make mtp-verify-depth Run legacy MTP speculative verification smoke if MTP GGUF is present" @@ -796,7 +822,9 @@ test: ds4_test ds4_agent_test ds4-eval q4k-dot-test mxfp4-dot-test \ $(SAMPLING_TEST) ds4 ds4-server ds4-bench ds4-agent ./ds4-eval --self-test-extractors ./ds4_agent_test - ./ds4_test + # Avoid adding the Q4 resident sidecar's 2.69 GiB to this broad suite. + # This does not enable SSD streaming; use `make test-ssd` for oversized models. + DS4_METAL_DISABLE_Q4_ATTN_Q_B_F16_CACHE=1 ./ds4_test ./tests/test_layer_pack ./tests/test_engine_mgpu_placement ./tests/test_gpu_args @@ -804,6 +832,11 @@ test: ds4_test ds4_agent_test ds4-eval q4k-dot-test mxfp4-dot-test \ ./tests/test_sampling ./tests/test_quantizer_indexer_q4 ./gguf-tools/deepseek4-quantize +test-ssd: + DS4_TEST_SSD_STREAMING=1 \ + DS4_TEST_SSD_STREAMING_COLD=1 \ + $(MAKE) test + dspark-acceptance: ds4 DS4_DSPARK_MODEL="$(DS4_DSPARK_MODEL)" \ DS4_DSPARK_SUPPORT="$(DS4_DSPARK_SUPPORT)" \ @@ -838,4 +871,4 @@ mxfp4-dot-test: tests/test_mxfp4_dot.c ./tests/test_mxfp4_dot clean: - rm -f ds4 ds4-server ds4-bench ds4-eval ds4-agent ds4_cpu ds4_native ds4_server_test ds4_test ds4_agent_test gguf-tools/quality-testing/score_official gguf-tools/quality-testing/score_official.o speed-bench/metal_decode_schedule_bench speed-bench/metal_prefill_variant_bench speed-bench/*.o tests/test_q4k_dot tests/test_mxfp4_dot tests/test_quantizer_indexer_q4 tests/test_mxfp4_metal tests/test_mxfp4_rocm tests/bench_mxfp4_rocm tests/test_mxfp4_cuda tests/test_rocm_q4_dense_pair tests/test_metal_session_batch tests/test_metal_q4_streams tests/test_metal_indexer_q4 tests/test_metal_q4_attn_exactn tests/test_metal_exactn_oracle tests/test_metal_dspark_capture tests/test_metal_iq2_midonly tests/test_metal_iq2_ssd_grouped_mm tests/test_metal_iq2_live_index tests/test_glm53_kda tests/test_glm53_kda_rocm tests/test_glm53_vision_engine tests/test_glm53_vision_prompt tests/test_gpu_xdev tests/test_gpu_model_cache tests/test_gpu_lookup_cache_strict tests/test_engine_mgpu_refusal tests/test_engine_mgpu_runtime tests/test_engine_correctness tests/test_sampling tests/test_cuda_session_batch tests/test_cuda_mixed_batch tests/*.o *.o tests/cuda_long_context_smoke tests/cuda_long_context_smoke.o + rm -f ds4 ds4-server ds4-bench ds4-eval ds4-agent ds4_cpu ds4_native ds4_server_test ds4_test ds4_agent_test gguf-tools/quality-testing/score_official gguf-tools/quality-testing/score_official.o speed-bench/metal_decode_schedule_bench speed-bench/metal_prefill_variant_bench speed-bench/*.o tests/test_q4k_dot tests/test_mxfp4_dot tests/test_quantizer_indexer_q4 tests/test_mxfp4_metal tests/test_mxfp4_rocm tests/bench_mxfp4_rocm tests/test_mxfp4_cuda tests/test_rocm_q4_dense_pair tests/test_metal_session_batch tests/test_metal_q4_streams tests/test_metal_indexer_q4 tests/test_metal_q4_attn_exactn tests/test_metal_q4_qb_f16_cache tests/test_metal_exactn_oracle tests/test_metal_dspark_capture tests/test_metal_iq2_midonly tests/test_metal_iq2_ssd_grouped_mm tests/test_metal_iq2_live_index tests/test_glm53_kda tests/test_glm53_kda_rocm tests/test_glm53_vision_engine tests/test_glm53_vision_prompt tests/test_gpu_xdev tests/test_gpu_model_cache tests/test_gpu_lookup_cache_strict tests/test_engine_mgpu_refusal tests/test_engine_mgpu_runtime tests/test_engine_correctness tests/test_sampling tests/test_cuda_session_batch tests/test_cuda_mixed_batch tests/*.o *.o tests/cuda_long_context_smoke tests/cuda_long_context_smoke.o diff --git a/ds4.c b/ds4.c index 97861f2fa..325b67a47 100644 --- a/ds4.c +++ b/ds4.c @@ -30267,29 +30267,43 @@ static bool metal_graph_encode_layer_attention_batch( ok = false; } bool q_b_f16_out = false; - if (ok && !q_path_debug && layer->attn_q_b->type == DS4_TENSOR_Q8_0) { - q_b_f16_out = ds4_gpu_attn_q_b_f16_head_rms_rope_tail_tensor(tp_q ? tp_q : metal_graph_batch_q(g), - tp_q_half ? tp_q_half : g->batch_q_half, - model->map, - model->size, - layer->attn_q_b->abs_offset, - q_rank, - q_dim, - tp_qr_norm ? tp_qr_norm : metal_graph_batch_qr_norm(g), - tp_rows, - DS4_N_HEAD, - DS4_N_HEAD_DIM, - DS4_N_ROT, - pos0 + tp_row0, - compressed ? (uint32_t)DS4_ROPE_ORIG_CTX : 0, - false, - freq_base, - freq_scale, - ext_factor, - attn_factor, - DS4_ROPE_YARN_BETA_FAST, - DS4_ROPE_YARN_BETA_SLOW, - DS4_RMS_EPS) != 0; +#if defined(__APPLE__) + const bool q_b_f16_weight = + layer->attn_q_b->type == DS4_TENSOR_Q8_0 || + (layer->attn_q_b->type == DS4_TENSOR_Q4_K && + tp_rows >= 32u); +#else + const bool q_b_f16_weight = + layer->attn_q_b->type == DS4_TENSOR_Q8_0; +#endif + if (ok && !q_path_debug && q_b_f16_weight) { + const int q_b_f16_rc = + ds4_gpu_attn_q_b_f16_head_rms_rope_tail_tensor( + tp_q ? tp_q : metal_graph_batch_q(g), + tp_q_half ? tp_q_half : g->batch_q_half, + model->map, + model->size, + layer->attn_q_b->abs_offset, + layer->attn_q_b->type, + q_rank, + q_dim, + tp_qr_norm ? tp_qr_norm : metal_graph_batch_qr_norm(g), + tp_rows, + DS4_N_HEAD, + DS4_N_HEAD_DIM, + DS4_N_ROT, + pos0 + tp_row0, + compressed ? (uint32_t)DS4_ROPE_ORIG_CTX : 0, + false, + freq_base, + freq_scale, + ext_factor, + attn_factor, + DS4_ROPE_YARN_BETA_FAST, + DS4_ROPE_YARN_BETA_SLOW, + DS4_RMS_EPS); + if (q_b_f16_rc < 0) ok = false; + q_b_f16_out = q_b_f16_rc > 0; } if (q_b_f16_out) { DS4_METAL_PROFILE_Q_STAGE("q_b"); @@ -37459,6 +37473,44 @@ static bool metal_graph_prefill_raw_swa( display_progress_ud); } +static uint32_t metal_graph_prefill_chunk_rows_at( + const ds4_gpu_graph *g, + uint32_t range_start, + uint32_t pos0, + uint32_t remaining) { + if (!g || remaining == 0u || g->prefill_cap == 0u) return 0u; + uint32_t cap = g->prefill_cap; + if (range_start != 0u && cap > g->raw_cap) cap = g->raw_cap; + if (cap == 0u) return 0u; + + if (range_start != 0u) { + const uint32_t mod = pos0 % g->prefill_cap; + if (mod != 0u) { + const uint32_t to_boundary = g->prefill_cap - mod; + if (to_boundary < cap) cap = to_boundary; + } + } + return remaining < cap ? remaining : cap; +} + +/* Maximum batch width the chunk executor below can emit. If the first chunk + * is smaller than cap it reaches an absolute prefill boundary, so the next + * chunk can use the full cap; if it is cap-sized, it is already the maximum. */ +static uint32_t metal_graph_prefill_max_chunk_rows( + const ds4_gpu_graph *g, + uint32_t start, + uint32_t n_tokens) { + const uint32_t first = metal_graph_prefill_chunk_rows_at( + g, start, start, n_tokens); + if (first == 0u || first >= n_tokens) return first; + + uint32_t cap = g->prefill_cap; + if (start != 0u && cap > g->raw_cap) cap = g->raw_cap; + const uint32_t remaining = n_tokens - first; + const uint32_t later = remaining < cap ? remaining : cap; + return first > later ? first : later; +} + /* Prefill a contiguous token range in fixed-size chunks. * * The common case starts at token zero, but server sessions also use this to @@ -37533,15 +37585,9 @@ static bool metal_graph_prefill_chunked_range( return true; } const uint32_t remaining = end - pos0; - uint32_t local_cap = chunk_cap; - if (start != 0 && g->prefill_cap != 0) { - const uint32_t mod = pos0 % g->prefill_cap; - if (mod != 0) { - const uint32_t to_boundary = g->prefill_cap - mod; - if (to_boundary < local_cap) local_cap = to_boundary; - } - } - const uint32_t chunk = remaining < local_cap ? remaining : local_cap; + const uint32_t chunk = metal_graph_prefill_chunk_rows_at( + g, start, pos0, remaining); + if (chunk == 0u) return false; const uint32_t chunk_end = pos0 + chunk; float *chunk_logits = (progress || chunk_end == end) ? logits : NULL; bool ok = metal_graph_prefill_layer_major(g, @@ -40578,6 +40624,7 @@ struct ds4_engine { * caller that doesn't set the option observe the prior behavior). */ int placement_ctx_hint; int placement_session_count_hint; + uint32_t live_session_count; }; #if defined(__APPLE__) && !defined(DS4_NO_GPU) @@ -55358,6 +55405,14 @@ static int generate_glm_metal_argmax( return 0; } +#if defined(__APPLE__) && !defined(DS4_NO_GPU) +static int ds4_prepare_metal_q4_attn_q_b_sidecars( + const ds4_model *model, + const ds4_weights *weights, + uint32_t max_batch_rows, + uint64_t working_set_reserve_bytes); +#endif + /* Metal generation entry point. The model runs as one local whole-graph * pipeline: graph prefill followed by graph decode steps. Streaming PRO may * use decode-style prefill for short prompts. */ @@ -55453,6 +55508,22 @@ static int generate_metal_graph_raw_swa( metal_graph_free(&g); return 1; } +#if defined(__APPLE__) && !defined(DS4_NO_GPU) + /* This frontend knows the real prompt width. Prepare only when that + * workload can use the resident sidecars, after graph allocation is + * visible to Metal but before warmup and the measured prefill window. */ + const uint32_t q4_sidecar_rows = + (uint32_t)prompt->len < prefill_cap ? + (uint32_t)prompt->len : prefill_cap; + if (ds4_prepare_metal_q4_attn_q_b_sidecars( + model, weights, q4_sidecar_rows, 0u) < 0) { + fprintf(stderr, + "ds4: required Metal Q4 attn_q_b F16 sidecar prewarm " + "could not be completed\n"); + metal_graph_free(&g); + return 1; + } +#endif const bool memory_report = getenv("DS4_METAL_MEMORY_REPORT") != NULL; if (memory_report) ds4_gpu_print_memory_report("after graph alloc"); @@ -56376,6 +56447,10 @@ struct ds4_session { void *cancel_ud; uint32_t prefill_cap; int ctx_size; + bool engine_session_counted; +#if defined(__APPLE__) && !defined(DS4_NO_GPU) + uint64_t metal_q4_attn_q_b_f16_sidecars_generation; +#endif bool checkpoint_valid; bool mtp_draft_valid; bool greedy_splitkv_anchor_valid; @@ -60707,20 +60782,28 @@ int ds4_engine_generate_argmax( ds4_session_free(s); return rc; } - return generate_metal_graph_raw_swa(model, vocab, weights, prompt, - n_predict, ctx_size, e->quality, - e->ssd_streaming, - e->ssd_streaming_cold, - e->ssd_streaming_preload_experts, - e->ssd_streaming_cache_bytes, - e->ssd_streaming_prefill_headroom_bytes, - e->power_percent, - e->prefill_chunk, - e->directional_steering_file, - e->directional_steering_attn_scale, - e->directional_steering_ffn_scale, - emit, done, emit_ud, - progress, progress_ud); + const int rc = generate_metal_graph_raw_swa( + model, vocab, weights, prompt, n_predict, ctx_size, e->quality, + e->ssd_streaming, e->ssd_streaming_cold, + e->ssd_streaming_preload_experts, + e->ssd_streaming_cache_bytes, + e->ssd_streaming_prefill_headroom_bytes, e->power_percent, + e->prefill_chunk, e->directional_steering_file, + e->directional_steering_attn_scale, + e->directional_steering_ffn_scale, emit, done, emit_ud, + progress, progress_ud); +#if defined(__APPLE__) + /* The legacy frontend owns a temporary graph rather than a session. + * Do not leave its 2.69 GiB resident sidecars pinned after return. */ + if (__atomic_load_n( + &e->live_session_count, __ATOMIC_RELAXED) == 0u && + !ds4_gpu_release_q4_attn_q_b_f16_sidecars()) { + fprintf(stderr, + "ds4: WARNING: could not release resident Metal Q4 " + "attn_q_b F16 sidecars after legacy generation\n"); + } +#endif + return rc; #else fprintf(stderr, "ds4: %s generation requested but this build has no graph backend support\n", ds4_backend_name(e->backend)); @@ -66981,6 +67064,99 @@ static int ds4_session_tp_register(ds4_session *s) { return 1; } +#if defined(__APPLE__) && !defined(DS4_NO_GPU) +static int ds4_prepare_metal_q4_attn_q_b_sidecars( + const ds4_model *model, + const ds4_weights *weights, + uint32_t max_batch_rows, + uint64_t working_set_reserve_bytes) { + if (!model || !weights) return 0; + ds4_gpu_q4_attn_q_b_f16_sidecar_desc descs[DS4_MAX_LAYER]; + uint32_t count = 0; + const uint64_t q_dim = (uint64_t)DS4_N_HEAD * DS4_N_HEAD_DIM; + for (uint32_t il = 0; il < (uint32_t)DS4_N_LAYER; il++) { + const ds4_tensor *tensor = weights->layer[il].attn_q_b; + if (!tensor || tensor->type != DS4_TENSOR_Q4_K || + tensor->ndim != 2u || tensor->dim[0] != DS4_N_LORA_Q || + tensor->dim[1] != q_dim) { + continue; + } + descs[count++] = (ds4_gpu_q4_attn_q_b_f16_sidecar_desc) { + .weight_offset = tensor->abs_offset, + .weight_bytes = tensor->bytes, + .in_dim = tensor->dim[0], + .out_dim = tensor->dim[1], + .weight_type = tensor->type, + .layer = il, + }; + } + if (count == 0u) return 1; + + uint64_t prepared_bytes = 0; + return ds4_gpu_prepare_q4_attn_q_b_f16_sidecars( + model->map, model->size, descs, count, max_batch_rows, + working_set_reserve_bytes, &prepared_bytes); +} + +static int ds4_session_prepare_metal_q4_attn_q_b_sidecars( + ds4_session *s, + uint32_t max_batch_rows, + bool reserve_future_sessions) { + if (!s || !s->engine || + s->engine->backend != DS4_BACKEND_METAL) { + return 1; + } + const uint64_t cache_generation = + ds4_gpu_q4_attn_q_b_f16_cache_generation(); + if (s->metal_q4_attn_q_b_f16_sidecars_generation == + cache_generation) { + return 1; + } + ds4_engine *e = s->engine; + + uint64_t future_session_bytes = 0; + const uint32_t session_count = engine_placement_session_count(e); + const uint32_t live_sessions = __atomic_load_n( + &e->live_session_count, __ATOMIC_RELAXED); + const uint32_t including_current = + live_sessions == UINT32_MAX ? UINT32_MAX : live_sessions + 1u; + const uint32_t remaining = + session_count > including_current ? + session_count - including_current : 0u; + if (reserve_future_sessions && remaining != 0u) { + const ds4_context_memory memory = + ds4_context_memory_estimate_with_prefill_mode( + e->backend, s->ctx_size, e->prefill_chunk, + e->ssd_streaming); + future_session_bytes = + memory.total_bytes > UINT64_MAX / remaining + ? UINT64_MAX + : memory.total_bytes * remaining; + } + + const int rc = ds4_prepare_metal_q4_attn_q_b_sidecars( + &e->model, &e->weights, max_batch_rows, future_session_bytes); + if (rc < 0) { + fprintf(stderr, + "ds4: required Metal Q4 attn_q_b F16 sidecar prewarm " + "could not be completed\n"); + return 0; + } + if (rc > 0) { + s->metal_q4_attn_q_b_f16_sidecars_generation = + ds4_gpu_q4_attn_q_b_f16_cache_generation(); + } + return 1; +} +#endif + +static void ds4_session_mark_engine_counted(ds4_session *s) { + if (!s || !s->engine || s->engine_session_counted) return; + __atomic_add_fetch( + &s->engine->live_session_count, 1u, __ATOMIC_RELAXED); + s->engine_session_counted = true; +} + int ds4_session_create(ds4_session **out, ds4_engine *e, int ctx_size) { if (!out || !e || ctx_size <= 0) return 1; if (e->backend == DS4_BACKEND_CPU) { @@ -67005,6 +67181,7 @@ int ds4_session_create(ds4_session **out, ds4_engine *e, int ctx_size) { ds4_session_free(s); return 1; } + ds4_session_mark_engine_counted(s); *out = s; return 0; } @@ -67131,6 +67308,7 @@ int ds4_session_create(ds4_session **out, ds4_engine *e, int ctx_size) { ds4_session_free(s); return 1; } + ds4_session_mark_engine_counted(s); *out = s; return 0; } @@ -67157,6 +67335,22 @@ int ds4_session_create(ds4_session **out, ds4_engine *e, int ctx_size) { e->shared_prefill_workspace_ready ? &e->shared_prefill_workspace : NULL; +#if defined(__APPLE__) && !defined(DS4_NO_GPU) + if (e->backend == DS4_BACKEND_METAL && + __atomic_load_n( + &e->live_session_count, __ATOMIC_RELAXED) != 0u) { + /* The graph estimator intentionally omits several large prefill + * workspaces. Evict conservatively before dynamic session growth; + * sessions created together at startup see an empty cache and pay + * nothing. Cache generations make existing sessions re-prewarm. */ + if (!ds4_gpu_make_room_for_q4_attn_q_b_f16_session()) { + fprintf(stderr, + "ds4: could not make room for the Metal session graph\n"); + free(s); + return 1; + } + } +#endif s->graph.dspark_exec_tier = e->multi_tier ? e->dspark_exec_tier : 0; if (!metal_graph_alloc_raw_cap(&s->graph, &e->weights, shape_layer, raw_cap, (uint32_t)ctx_size, s->prefill_cap, @@ -67248,6 +67442,23 @@ int ds4_session_create(ds4_session **out, ds4_engine *e, int ctx_size) { fprintf(stderr, "\n"); } } +#if defined(__APPLE__) && !defined(DS4_NO_GPU) + /* Remote workers do not necessarily enter the local sync preflight before + * their first mirrored/layer-slice batch, so keep eager preparation for + * TP/distributed sessions. Local sessions defer until the real prompt is + * known, avoiding a 2.69 GiB decode-only allocation. */ + if ((e->tp.active || + e->distributed.role != DS4_DISTRIBUTED_NONE) && + !ds4_session_prepare_metal_q4_attn_q_b_sidecars( + s, s->prefill_cap, true)) { + if (__atomic_load_n(&e->live_session_count, __ATOMIC_RELAXED) == 0u) { + (void)ds4_gpu_release_q4_attn_q_b_f16_sidecars(); + } + metal_graph_free(&s->graph); + free(s); + return 1; + } +#endif s->logits = xmalloc((size_t)DS4_N_VOCAB * sizeof(s->logits[0])); s->sample_probs = xmalloc((size_t)DS4_N_VOCAB * sizeof(s->sample_probs[0])); @@ -67284,6 +67495,12 @@ int ds4_session_create(ds4_session **out, ds4_engine *e, int ctx_size) { fprintf(stderr, "ds4: failed to create distributed coordinator session: %s\n", err[0] ? err : "unknown error"); +#if defined(__APPLE__) && !defined(DS4_NO_GPU) + if (__atomic_load_n( + &e->live_session_count, __ATOMIC_RELAXED) == 0u) { + (void)ds4_gpu_release_q4_attn_q_b_f16_sidecars(); + } +#endif metal_graph_free(&s->graph); free(s->logits); free(s->sample_probs); @@ -67299,6 +67516,7 @@ int ds4_session_create(ds4_session **out, ds4_engine *e, int ctx_size) { ds4_session_free(s); return 1; } + ds4_session_mark_engine_counted(s); *out = s; return 0; #endif @@ -67306,6 +67524,31 @@ int ds4_session_create(ds4_session **out, ds4_engine *e, int ctx_size) { void ds4_session_free(ds4_session *s) { if (!s) return; +#if defined(__APPLE__) && !defined(DS4_NO_GPU) + bool release_metal_q4_sidecars = false; +#endif + if (s->engine && s->engine_session_counted) { + const uint32_t remaining_sessions = __atomic_sub_fetch( + &s->engine->live_session_count, 1u, __ATOMIC_RELAXED); +#if defined(__APPLE__) && !defined(DS4_NO_GPU) + release_metal_q4_sidecars = + remaining_sessions == 0u && + s->engine->backend == DS4_BACKEND_METAL; +#else + (void)remaining_sessions; +#endif + s->engine_session_counted = false; + } +#if defined(__APPLE__) && !defined(DS4_NO_GPU) + else if (s->engine && s->engine->backend == DS4_BACKEND_METAL && + __atomic_load_n( + &s->engine->live_session_count, __ATOMIC_RELAXED) == 0u) { + /* An eager TP/distributed prewarm may have succeeded before session + * registration failed, so this uncounted session can still own the + * process-global sidecars. */ + release_metal_q4_sidecars = true; + } +#endif if (ds4_session_tp_leader(s) && s->tp_session_id != 0 && !ds4_tp_failed(s->engine->tp.ctx)) { char err[256] = ""; @@ -67328,6 +67571,14 @@ void ds4_session_free(ds4_session *s) { } #ifndef DS4_NO_GPU else { +#if defined(__APPLE__) + if (release_metal_q4_sidecars && + !ds4_gpu_release_q4_attn_q_b_f16_sidecars()) { + fprintf(stderr, + "ds4: WARNING: could not release resident Metal Q4 " + "attn_q_b F16 sidecars at session teardown\n"); + } +#endif if (ds4_session_is_glm(s)) { glm_graph_free(&s->glm_graph); } else { @@ -68636,6 +68887,67 @@ static bool ds4_session_store_vision_identities(ds4_session *s) { return true; } +/* Prepare one-shot resources for the actual work the next sync will emit, + * without changing checkpoint/KV state. Timed frontends may call this before + * starting their clock; ds4_session_sync() repeats it as an idempotent safety + * net for API users that do not. */ +int ds4_session_prepare_sync(ds4_session *s, + const ds4_tokens *prompt, + char *err, + size_t errlen) { + if (!s || !prompt) { + if (err && errlen) snprintf(err, errlen, "missing session or prompt"); + return 1; + } + if (prompt->len <= 0) { + if (err && errlen) snprintf(err, errlen, "empty prompt"); + return 1; + } + if (prompt->len >= s->ctx_size) { + if (err && errlen) { + snprintf(err, errlen, + "prompt length %d exceeds context %d " + "(one token of generation room is required)", + prompt->len, s->ctx_size); + } + return 1; + } + +#if defined(__APPLE__) && !defined(DS4_NO_GPU) + if (!s->engine || s->engine->backend != DS4_BACKEND_METAL || + ds4_session_is_cpu(s) || ds4_session_is_glm(s)) { + return 0; + } + + uint32_t start = 0u; + uint32_t rows = (uint32_t)prompt->len; + if (s->checkpoint_valid && + prompt->len >= s->checkpoint.len && + ds4_tokens_starts_with(prompt, &s->checkpoint)) { + start = (uint32_t)s->checkpoint.len; + rows = (uint32_t)(prompt->len - s->checkpoint.len); + if (rows < metal_graph_resume_prefill_min_tokens()) return 0; + } + if (rows == 0u) return 0; + + const uint32_t max_batch_rows = metal_graph_prefill_max_chunk_rows( + &s->graph, start, rows); + if (max_batch_rows == 0u) return 0; + if (!ds4_session_prepare_metal_q4_attn_q_b_sidecars( + s, max_batch_rows, false)) { + if (err && errlen) { + snprintf(err, errlen, + "required Metal Q4 attn_q_b F16 sidecar prewarm failed"); + } + return 1; + } +#else + (void)err; + (void)errlen; +#endif + return 0; +} + /* Under tensor parallelism the leader mirrors every public sync/eval to the * worker before doing the work itself, so both engines execute the same * graph sequence and the per-layer gates pair up. The worker acks a sync @@ -68646,6 +68958,7 @@ int ds4_session_sync(ds4_session *s, const ds4_tokens *prompt, char *err, size_t s, s->sync_images, s->sync_image_count)) { ds4_session_invalidate(s); } + if (ds4_session_prepare_sync(s, prompt, err, errlen) != 0) return 1; const bool mirror = ds4_session_tp_leader(s); if (mirror && prompt && prompt->len > 0) { if (s->sync_image_count > UINT32_MAX) { @@ -72683,6 +72996,13 @@ static int ds4_sessions_eval_batch_with_prefill_metal( const uint64_t hc_dim = (uint64_t)DS4_N_HC * DS4_N_EMBD; const bool mirror = e->tp.active && e->tp.rank == 0; + /* This mixed path bypasses ds4_session_sync(), so run the same one-shot + * preparation before it can publish a TP command or encode layer work. */ + if (ds4_session_prepare_sync( + prefill_session, prefill_prompt, err, errlen) != 0) { + return 1; + } + if (mirror) { ds4_tp_batch_item *wire = ds4_sessions_tp_batch_items(items, count); const bool sent = wire && diff --git a/ds4.h b/ds4.h index 879498d2a..8ba383e44 100644 --- a/ds4.h +++ b/ds4.h @@ -386,6 +386,12 @@ int ds4_engine_tp_bind(ds4_engine *e, struct ds4_tp *tp, char *err, size_t errle int ds4_session_create(ds4_session **out, ds4_engine *e, int ctx_size); void ds4_session_free(ds4_session *s); +/* Prepares one-shot resources for the next prompt without changing session + * state. Timed callers can invoke this before ds4_session_sync(). */ +int ds4_session_prepare_sync(ds4_session *s, + const ds4_tokens *prompt, + char *err, + size_t errlen); int ds4_session_power(ds4_session *s); int ds4_session_set_power(ds4_session *s, int power_percent); float ds4_session_directional_steering_ffn(ds4_session *s); diff --git a/ds4_bench.c b/ds4_bench.c index fca6e635b..4a0a13225 100644 --- a/ds4_bench.c +++ b/ds4_bench.c @@ -814,6 +814,17 @@ int main(int argc, char **argv) { .cap = frontier, }; + /* Keep one-shot resident-weight preparation outside the prefill TPS + * window. The preflight uses this frontier's real suffix/chunk width, + * so decode-only and sub-threshold runs allocate no sidecars. */ + if (ds4_session_prepare_sync( + session, &prefix, err, sizeof(err)) != 0) { + fprintf(stderr, + "ds4-bench: prefill preparation to %d failed: %s\n", + frontier, err); + rc = 1; + break; + } const double prefill_t0 = bench_now_sec(); #if defined(DS4_BENCH_HAVE_CUDA_PROFILER) const bool cuda_profile_prefill = diff --git a/ds4_cli.c b/ds4_cli.c index 163678d7e..f1f167066 100644 --- a/ds4_cli.c +++ b/ds4_cli.c @@ -529,11 +529,17 @@ static int run_sampled_generation(ds4_engine *engine, const cli_config *cfg, con ds4_session_free(session); return 1; } + char err[160]; + if (ds4_session_prepare_sync( + session, prompt, err, sizeof(err)) != 0) { + fprintf(stderr, "ds4: prompt preparation failed: %s\n", err); + ds4_session_free(session); + return 1; + } /* Pay the one-time first-submission GPU cost before the prefill timer * starts (matches the TP worker's startup warmup). */ ds4_session_gpu_warmup(session); - char err[160]; ds4_think_mode think_mode = cli_effective_think_mode(&cfg->gen); token_printer printer = { .engine = engine, @@ -1510,6 +1516,13 @@ static int run_chat_turn(ds4_engine *engine, cli_config *cfg, repl_chat *chat, .input_tokens = suffix, .use_color = ds4_log_is_tty(stderr), }; + if (ds4_session_prepare_sync( + chat->session, &chat->transcript, err, sizeof(err)) != 0) { + chat->transcript.len = rollback_len; + repl_chat_trim_images(chat, rollback_images); + fprintf(stderr, "ds4: prompt preparation failed: %s\n", err); + return 1; + } const double t_prefill0 = cli_now_sec(); ds4_session_set_progress(chat->session, cli_prefill_progress_cb, &progress); ds4_session_set_display_progress(chat->session, diff --git a/ds4_cuda.cu b/ds4_cuda.cu index 53379abf8..bcc6ca4e7 100644 --- a/ds4_cuda.cu +++ b/ds4_cuda.cu @@ -42144,13 +42144,14 @@ extern "C" int ds4_gpu_matmul_q8_0_f16_out_tensor( extern "C" int ds4_gpu_attn_q_b_f16_head_rms_rope_tail_tensor( ds4_gpu_tensor *out, ds4_gpu_tensor *q_half, const void *model_map, uint64_t model_size, uint64_t weight_offset, + uint32_t weight_type, uint64_t in_dim, uint64_t out_dim, const ds4_gpu_tensor *x, uint32_t n_tok, uint32_t n_head, uint32_t head_dim, uint32_t n_rot, uint32_t pos0, uint32_t n_ctx_orig, bool inverse, float freq_base, float freq_scale, float ext_factor, float attn_factor, float beta_fast, float beta_slow, float eps) { (void)out; (void)q_half; (void)model_map; (void)model_size; - (void)weight_offset; (void)in_dim; (void)out_dim; (void)x; + (void)weight_offset; (void)weight_type; (void)in_dim; (void)out_dim; (void)x; (void)n_tok; (void)n_head; (void)head_dim; (void)n_rot; (void)pos0; (void)n_ctx_orig; (void)inverse; (void)freq_base; (void)freq_scale; (void)ext_factor; (void)attn_factor; (void)beta_fast; (void)beta_slow; diff --git a/ds4_gpu.h b/ds4_gpu.h index 5cde39204..0c6231ccc 100644 --- a/ds4_gpu.h +++ b/ds4_gpu.h @@ -188,6 +188,12 @@ void ds4_gpu_set_glm_model(bool enabled); void ds4_gpu_set_ssd_streaming(bool enabled); void ds4_gpu_set_glm_streaming_prefill_full_layer(bool enabled); #ifdef __APPLE__ +/* Release resident Q4 attn_q_b F16 sidecars at a quiescent lifecycle point. + * Returns zero only when pending Metal work could not be synchronized safely. */ +int ds4_gpu_release_q4_attn_q_b_f16_sidecars(void); +uint64_t ds4_gpu_q4_attn_q_b_f16_cache_generation(void); +/* Evict resident sidecars before adding a graph to a live-session set. */ +int ds4_gpu_make_room_for_q4_attn_q_b_f16_session(void); int ds4_gpu_device_is_pre_m5_apple_silicon(void); int ds4_gpu_device_is_m5_apple_silicon(void); int ds4_gpu_set_decode_pipeline_fast_lookup(int enabled); @@ -257,6 +263,57 @@ int ds4_gpu_test_exact_rows_persistent_policy( int size_class_ok); void ds4_gpu_test_exact_rows_persistent_report( ds4_gpu_exact_rows_persistent_report *report); +typedef struct ds4_gpu_q4_attn_q_b_f16_cache_report { + uint64_t entries; + uint64_t bytes; + uint64_t lookups; + uint64_t hits; + uint64_t misses; + uint64_t builds; + uint64_t build_failures; + uint64_t candidate_calls; + uint64_t fallbacks; + uint64_t rejects; + uint64_t build_circuit_open; +} ds4_gpu_q4_attn_q_b_f16_cache_report; +/* Test observability for the resident Metal Q4_K attn_q_b F16 sidecar. */ +void ds4_gpu_test_q4_attn_q_b_f16_cache_report( + ds4_gpu_q4_attn_q_b_f16_cache_report *report); +void ds4_gpu_test_q4_attn_q_b_f16_cache_reset(void); +int ds4_gpu_test_q4_attn_q_b_f16_projection_tensor( + ds4_gpu_tensor *out, + const void *model_map, + uint64_t model_size, + uint64_t weight_offset, + uint64_t in_dim, + uint64_t out_dim, + const ds4_gpu_tensor *x, + uint32_t n_tok); +int ds4_gpu_test_q4_attn_q_b_f16_working_set_policy( + uint64_t recommended, + uint64_t allocated, + uint64_t additional); + +typedef struct ds4_gpu_q4_attn_q_b_f16_sidecar_desc { + uint64_t weight_offset; + uint64_t weight_bytes; + uint64_t in_dim; + uint64_t out_dim; + uint32_t weight_type; + uint32_t layer; +} ds4_gpu_q4_attn_q_b_f16_sidecar_desc; + +/* Prepare every missing resident Q4_K attn_q_b sidecar transactionally. + * Returns 1 when all descriptors are READY, 0 for a policy/safety skip, and + * -1 when strict mode requires a specialization that cannot be prepared. */ +int ds4_gpu_prepare_q4_attn_q_b_f16_sidecars( + const void *model_map, + uint64_t model_size, + const ds4_gpu_q4_attn_q_b_f16_sidecar_desc *descs, + uint32_t count, + uint32_t max_prefill_rows, + uint64_t working_set_reserve_bytes, + uint64_t *prepared_bytes); typedef struct ds4_gpu_stream_test_stats { uint64_t tensor_live_bytes; uint64_t transient_references; @@ -1512,12 +1569,16 @@ int ds4_gpu_head_rms_norm_rope_tail_tensor( float beta_slow, float eps); +/* Returns 1 when the backend-specific fused projection was encoded, 0 when + * the caller should use its generic projection path, and -1 when a required + * specialization could not be honored. */ int ds4_gpu_attn_q_b_f16_head_rms_rope_tail_tensor( ds4_gpu_tensor *out, ds4_gpu_tensor *q_half, const void *model_map, uint64_t model_size, uint64_t weight_offset, + uint32_t weight_type, uint64_t in_dim, uint64_t out_dim, const ds4_gpu_tensor *x, diff --git a/ds4_metal.m b/ds4_metal.m index c3ed50789..bedbd7f4e 100644 --- a/ds4_metal.m +++ b/ds4_metal.m @@ -509,6 +509,131 @@ static uint64_t g_model_buffer_cache_bytes; static uint64_t g_model_buffer_cache_evictions; static int g_model_buffer_cache_over_limit; +#define DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MAX_ENTRIES 80u +typedef struct { + const void *model_map; + uint64_t model_size; + uint64_t weight_offset; + uint64_t weight_bytes; + uint64_t in_dim; + uint64_t out_dim; + uint64_t f16_bytes; + id __strong buffer; + int valid; +} ds4_gpu_q4_attn_q_b_f16_cache_entry; +static ds4_gpu_q4_attn_q_b_f16_cache_entry + g_q4_attn_q_b_f16_cache[DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MAX_ENTRIES]; +static pthread_mutex_t g_q4_attn_q_b_f16_cache_mu = PTHREAD_MUTEX_INITIALIZER; +static pthread_mutex_t g_q4_attn_q_b_f16_build_mu = PTHREAD_MUTEX_INITIALIZER; +static uint64_t g_q4_attn_q_b_f16_cache_bytes; +static uint64_t g_q4_attn_q_b_f16_cache_lookups; +static uint64_t g_q4_attn_q_b_f16_cache_hits; +static uint64_t g_q4_attn_q_b_f16_cache_misses; +static uint64_t g_q4_attn_q_b_f16_cache_builds; +static uint64_t g_q4_attn_q_b_f16_cache_build_failures; +static uint64_t g_q4_attn_q_b_f16_cache_candidate_calls; +static uint64_t g_q4_attn_q_b_f16_cache_fallbacks; +static uint64_t g_q4_attn_q_b_f16_cache_rejects; +static uint32_t g_q4_attn_q_b_f16_cache_entries; +static uint64_t g_q4_attn_q_b_f16_cache_generation = 1u; +typedef enum { + DS4_Q4_ATTN_Q_B_F16_CIRCUIT_CLOSED = 0, + DS4_Q4_ATTN_Q_B_F16_CIRCUIT_HARD = 1, + DS4_Q4_ATTN_Q_B_F16_CIRCUIT_PRESSURE = 2, +} ds4_gpu_q4_attn_q_b_f16_circuit_state; +static ds4_gpu_q4_attn_q_b_f16_circuit_state + g_q4_attn_q_b_f16_build_circuit_state; +static int g_initialized; + +static void ds4_gpu_q4_attn_q_b_f16_cache_clear(int reset_stats) { + pthread_mutex_lock(&g_q4_attn_q_b_f16_build_mu); + pthread_mutex_lock(&g_q4_attn_q_b_f16_cache_mu); + for (uint32_t i = 0; + i < DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MAX_ENTRIES; + i++) { + ds4_gpu_q4_attn_q_b_f16_cache_entry *entry = + &g_q4_attn_q_b_f16_cache[i]; + entry->buffer = nil; + entry->model_map = NULL; + entry->model_size = 0; + entry->weight_offset = 0; + entry->weight_bytes = 0; + entry->in_dim = 0; + entry->out_dim = 0; + entry->f16_bytes = 0; + entry->valid = 0; + } + g_q4_attn_q_b_f16_cache_bytes = 0; + g_q4_attn_q_b_f16_cache_entries = 0; + g_q4_attn_q_b_f16_cache_generation++; + if (g_q4_attn_q_b_f16_cache_generation == 0u) { + g_q4_attn_q_b_f16_cache_generation = 1u; + } + g_q4_attn_q_b_f16_build_circuit_state = + DS4_Q4_ATTN_Q_B_F16_CIRCUIT_CLOSED; + if (reset_stats) { + g_q4_attn_q_b_f16_cache_lookups = 0; + g_q4_attn_q_b_f16_cache_hits = 0; + g_q4_attn_q_b_f16_cache_misses = 0; + g_q4_attn_q_b_f16_cache_builds = 0; + g_q4_attn_q_b_f16_cache_build_failures = 0; + g_q4_attn_q_b_f16_cache_candidate_calls = 0; + g_q4_attn_q_b_f16_cache_fallbacks = 0; + g_q4_attn_q_b_f16_cache_rejects = 0; + } + pthread_mutex_unlock(&g_q4_attn_q_b_f16_cache_mu); + pthread_mutex_unlock(&g_q4_attn_q_b_f16_build_mu); +} + +void ds4_gpu_test_q4_attn_q_b_f16_cache_report( + ds4_gpu_q4_attn_q_b_f16_cache_report *report) { + if (!report) return; + pthread_mutex_lock(&g_q4_attn_q_b_f16_cache_mu); + *report = (ds4_gpu_q4_attn_q_b_f16_cache_report) { + .entries = g_q4_attn_q_b_f16_cache_entries, + .bytes = g_q4_attn_q_b_f16_cache_bytes, + .lookups = g_q4_attn_q_b_f16_cache_lookups, + .hits = g_q4_attn_q_b_f16_cache_hits, + .misses = g_q4_attn_q_b_f16_cache_misses, + .builds = g_q4_attn_q_b_f16_cache_builds, + .build_failures = g_q4_attn_q_b_f16_cache_build_failures, + .candidate_calls = g_q4_attn_q_b_f16_cache_candidate_calls, + .fallbacks = g_q4_attn_q_b_f16_cache_fallbacks, + .rejects = g_q4_attn_q_b_f16_cache_rejects, + .build_circuit_open = (uint64_t)( + g_q4_attn_q_b_f16_build_circuit_state != + DS4_Q4_ATTN_Q_B_F16_CIRCUIT_CLOSED), + }; + pthread_mutex_unlock(&g_q4_attn_q_b_f16_cache_mu); +} + +void ds4_gpu_test_q4_attn_q_b_f16_cache_reset(void) { + if (g_initialized) (void)ds4_gpu_synchronize(); + ds4_gpu_q4_attn_q_b_f16_cache_clear(1); +} + +uint64_t ds4_gpu_q4_attn_q_b_f16_cache_generation(void) { + pthread_mutex_lock(&g_q4_attn_q_b_f16_cache_mu); + const uint64_t generation = g_q4_attn_q_b_f16_cache_generation; + pthread_mutex_unlock(&g_q4_attn_q_b_f16_cache_mu); + return generation; +} + +int ds4_gpu_release_q4_attn_q_b_f16_sidecars(void) { + int has_entries = 0; + pthread_mutex_lock(&g_q4_attn_q_b_f16_cache_mu); + has_entries = g_q4_attn_q_b_f16_cache_entries != 0u; + pthread_mutex_unlock(&g_q4_attn_q_b_f16_cache_mu); + + /* Callers use this only at session/mode lifecycle boundaries. Finish + * every command that may still reference an unretained Metal buffer + * before dropping the cache's final strong references. Always clear, + * even with zero entries: a rejected build may have left the circuit + * breaker open and must not poison the next resident session. */ + if (has_entries && g_initialized && !ds4_gpu_synchronize()) return 0; + ds4_gpu_q4_attn_q_b_f16_cache_clear(0); + return 1; +} static uint64_t g_stream_expert_cache_bytes; static uint64_t g_stream_expert_cache_expert_bytes; static uint64_t g_stream_expert_cache_gate_class_bytes; @@ -685,7 +810,6 @@ static int ds4_gpu_stream_expert_cache_on_service_thread(void) { static uint32_t g_glm_flash_attn_mask_tokens; static uint32_t g_glm_flash_attn_mask_cache_len; static int g_glm_flash_attn_mask_valid; -static int g_initialized; static int g_quality_mode; static int g_mpp_invalid_env_reported; #define DS4_METAL_MAX_ROUTED_EXPERT_USED 8 @@ -4508,6 +4632,10 @@ void ds4_gpu_print_memory_report(const char *label) { const uint64_t tensor_live_snap = g_tensor_alloc_live_bytes; const uint64_t tensor_peak_snap = g_tensor_alloc_peak_bytes; pthread_mutex_unlock(&g_tensor_mu); + pthread_mutex_lock(&g_q4_attn_q_b_f16_cache_mu); + const uint64_t q4_q_b_f16_cache_snap = + g_q4_attn_q_b_f16_cache_bytes; + pthread_mutex_unlock(&g_q4_attn_q_b_f16_cache_mu); uint64_t tracked_live = tensor_live_snap; if (tracked_live > UINT64_MAX - g_stream_expert_cache_bytes) { @@ -4515,18 +4643,25 @@ void ds4_gpu_print_memory_report(const char *label) { } else { tracked_live += g_stream_expert_cache_bytes; } + if (tracked_live > UINT64_MAX - q4_q_b_f16_cache_snap) { + tracked_live = UINT64_MAX; + } else { + tracked_live += q4_q_b_f16_cache_snap; + } const bool color = ds4_log_is_tty(stderr); const char *green = color ? "\x1b[32m" : ""; const char *bright_green = color ? "\x1b[1;32m" : ""; const char *reset = color ? "\x1b[0m" : ""; fprintf(stderr, - "%sds4: Metal memory%s%s: runtime %.2f GiB + streaming experts %.2f GiB = %s%.2f GiB tracked live%s\n", + "%sds4: Metal memory%s%s: runtime %.2f GiB + streaming experts %.2f GiB " + "+ Q4 attn_q_b F16 %.2f GiB = %s%.2f GiB tracked live%s\n", green, label && label[0] ? " " : "", label && label[0] ? label : "", ds4_gpu_gib(tensor_live_snap), ds4_gpu_gib(g_stream_expert_cache_bytes), + ds4_gpu_gib(q4_q_b_f16_cache_snap), bright_green, ds4_gpu_gib(tracked_live), reset); @@ -4814,6 +4949,12 @@ void ds4_gpu_set_glm_model(bool enabled) { void ds4_gpu_set_ssd_streaming(bool enabled) { g_ssd_streaming_mode = enabled ? 1 : 0; + if (g_ssd_streaming_mode && + !ds4_gpu_release_q4_attn_q_b_f16_sidecars()) { + fprintf(stderr, + "ds4: WARNING: could not release resident Q4 attn_q_b " + "F16 sidecars while enabling SSD streaming\n"); + } ds4_gpu_stream_expert_cache_clear_all(1); if (g_ssd_streaming_mode) { fprintf(stderr, @@ -11308,6 +11449,28 @@ void ds4_gpu_cleanup(void) { } g_selected_readback_event = nil; g_selected_readback_event_value = 0; + if (getenv("DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_PROFILE") != NULL && + (g_q4_attn_q_b_f16_cache_candidate_calls != 0 || + g_q4_attn_q_b_f16_cache_builds != 0 || + g_q4_attn_q_b_f16_cache_build_failures != 0 || + g_q4_attn_q_b_f16_build_circuit_state != + DS4_Q4_ATTN_Q_B_F16_CIRCUIT_CLOSED)) { + fprintf(stderr, + "ds4: Metal Q4 attn_q_b F16 cache entries=%u bytes=%.2f GiB " + "calls=%llu hits=%llu misses=%llu builds=%llu failures=%llu " + "fallbacks=%llu rejects=%llu circuit_state=%d\n", + g_q4_attn_q_b_f16_cache_entries, + ds4_gpu_gib(g_q4_attn_q_b_f16_cache_bytes), + (unsigned long long)g_q4_attn_q_b_f16_cache_candidate_calls, + (unsigned long long)g_q4_attn_q_b_f16_cache_hits, + (unsigned long long)g_q4_attn_q_b_f16_cache_misses, + (unsigned long long)g_q4_attn_q_b_f16_cache_builds, + (unsigned long long)g_q4_attn_q_b_f16_cache_build_failures, + (unsigned long long)g_q4_attn_q_b_f16_cache_fallbacks, + (unsigned long long)g_q4_attn_q_b_f16_cache_rejects, + (int)g_q4_attn_q_b_f16_build_circuit_state); + } + ds4_gpu_q4_attn_q_b_f16_cache_clear(1); ds4_gpu_stream_expert_pread_pool_shutdown(); ds4_gpu_stream_expert_cache_clear_all(1); ds4_gpu_stream_expert_cache_live_release(); @@ -24959,12 +25122,915 @@ int ds4_gpu_head_rms_norm_rope_tail_tensor( return 1; } +static int ds4_gpu_q4_attn_q_b_f16_cache_key_equal( + const ds4_gpu_q4_attn_q_b_f16_cache_entry *entry, + const void *model_map, + uint64_t model_size, + uint64_t weight_offset, + uint64_t weight_bytes, + uint64_t in_dim, + uint64_t out_dim) { + return entry->valid && + entry->model_map == model_map && + entry->model_size == model_size && + entry->weight_offset == weight_offset && + entry->weight_bytes == weight_bytes && + entry->in_dim == in_dim && + entry->out_dim == out_dim; +} + +static int ds4_gpu_q4_attn_q_b_f16_cache_lookup_impl( + const void *model_map, + uint64_t model_size, + uint64_t weight_offset, + uint64_t weight_bytes, + uint64_t in_dim, + uint64_t out_dim, + id __strong *buffer, + int count_stats) { + int found = 0; + pthread_mutex_lock(&g_q4_attn_q_b_f16_cache_mu); + if (count_stats) g_q4_attn_q_b_f16_cache_lookups++; + for (uint32_t i = 0; + i < DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MAX_ENTRIES; + i++) { + ds4_gpu_q4_attn_q_b_f16_cache_entry *entry = + &g_q4_attn_q_b_f16_cache[i]; + if (!ds4_gpu_q4_attn_q_b_f16_cache_key_equal( + entry, model_map, model_size, weight_offset, weight_bytes, + in_dim, out_dim)) { + continue; + } + *buffer = entry->buffer; + if (count_stats) g_q4_attn_q_b_f16_cache_hits++; + found = 1; + break; + } + if (count_stats && !found) g_q4_attn_q_b_f16_cache_misses++; + pthread_mutex_unlock(&g_q4_attn_q_b_f16_cache_mu); + return found; +} + +static int ds4_gpu_q4_attn_q_b_f16_cache_lookup( + const void *model_map, + uint64_t model_size, + uint64_t weight_offset, + uint64_t weight_bytes, + uint64_t in_dim, + uint64_t out_dim, + id __strong *buffer) { + return ds4_gpu_q4_attn_q_b_f16_cache_lookup_impl( + model_map, model_size, weight_offset, weight_bytes, in_dim, + out_dim, buffer, 1); +} + +static int ds4_gpu_q4_attn_q_b_f16_cache_peek( + const void *model_map, + uint64_t model_size, + uint64_t weight_offset, + uint64_t weight_bytes, + uint64_t in_dim, + uint64_t out_dim, + id __strong *buffer) { + return ds4_gpu_q4_attn_q_b_f16_cache_lookup_impl( + model_map, model_size, weight_offset, weight_bytes, in_dim, + out_dim, buffer, 0); +} + +static int ds4_gpu_q4_attn_q_b_f16_cache_insert( + const void *model_map, + uint64_t model_size, + uint64_t weight_offset, + uint64_t weight_bytes, + uint64_t in_dim, + uint64_t out_dim, + uint64_t f16_bytes, + uint64_t budget_bytes, + id buffer) { + int inserted = 0; + pthread_mutex_lock(&g_q4_attn_q_b_f16_cache_mu); + + /* Normal graph encoding is serialized. Recheck nevertheless so an + * unusual concurrent caller cannot publish a duplicate key. */ + for (uint32_t i = 0; + i < DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MAX_ENTRIES; + i++) { + if (ds4_gpu_q4_attn_q_b_f16_cache_key_equal( + &g_q4_attn_q_b_f16_cache[i], model_map, model_size, + weight_offset, weight_bytes, in_dim, out_dim)) { + pthread_mutex_unlock(&g_q4_attn_q_b_f16_cache_mu); + return -1; + } + } + + if (f16_bytes <= budget_bytes && + g_q4_attn_q_b_f16_cache_bytes <= budget_bytes - f16_bytes) { + for (uint32_t i = 0; + i < DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MAX_ENTRIES; + i++) { + ds4_gpu_q4_attn_q_b_f16_cache_entry *entry = + &g_q4_attn_q_b_f16_cache[i]; + if (entry->valid) continue; + entry->model_map = model_map; + entry->model_size = model_size; + entry->weight_offset = weight_offset; + entry->weight_bytes = weight_bytes; + entry->in_dim = in_dim; + entry->out_dim = out_dim; + entry->f16_bytes = f16_bytes; + entry->buffer = buffer; + entry->valid = 1; + const int first_entry = + g_q4_attn_q_b_f16_cache_entries == 0u; + g_q4_attn_q_b_f16_cache_bytes += f16_bytes; + g_q4_attn_q_b_f16_cache_entries++; + g_q4_attn_q_b_f16_cache_builds++; + inserted = first_entry ? 2 : 1; + break; + } + } + pthread_mutex_unlock(&g_q4_attn_q_b_f16_cache_mu); + return inserted; +} + +static int ds4_gpu_q4_attn_q_b_f16_cache_insert_batch( + const void *model_map, + uint64_t model_size, + const ds4_gpu_q4_attn_q_b_f16_sidecar_desc *descs, + const uint32_t *miss_indices, + id __strong *buffers, + uint32_t miss_count, + uint64_t budget_bytes) { + uint32_t free_slots[DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MAX_ENTRIES]; + uint64_t new_bytes = 0; + uint32_t free_count = 0; + int inserted = 0; + + if (!descs || !miss_indices || !buffers || miss_count == 0u || + miss_count > DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MAX_ENTRIES) { + return 0; + } + + pthread_mutex_lock(&g_q4_attn_q_b_f16_cache_mu); + for (uint32_t i = 0; + i < DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MAX_ENTRIES; + i++) { + if (!g_q4_attn_q_b_f16_cache[i].valid) { + free_slots[free_count++] = i; + } + } + + for (uint32_t mi = 0; mi < miss_count; mi++) { + const ds4_gpu_q4_attn_q_b_f16_sidecar_desc *desc = + &descs[miss_indices[mi]]; + const uint64_t f16_bytes = + desc->in_dim * desc->out_dim * sizeof(uint16_t); + if (!buffers[mi] || UINT64_MAX - new_bytes < f16_bytes) { + goto done; + } + new_bytes += f16_bytes; + + for (uint32_t i = 0; + i < DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MAX_ENTRIES; + i++) { + if (ds4_gpu_q4_attn_q_b_f16_cache_key_equal( + &g_q4_attn_q_b_f16_cache[i], model_map, model_size, + desc->weight_offset, desc->weight_bytes, + desc->in_dim, desc->out_dim)) { + goto done; + } + } + for (uint32_t mj = 0; mj < mi; mj++) { + const ds4_gpu_q4_attn_q_b_f16_sidecar_desc *prior = + &descs[miss_indices[mj]]; + if (prior->weight_offset == desc->weight_offset && + prior->weight_bytes == desc->weight_bytes && + prior->in_dim == desc->in_dim && + prior->out_dim == desc->out_dim) { + goto done; + } + } + } + + if (free_count < miss_count || new_bytes > budget_bytes || + g_q4_attn_q_b_f16_cache_bytes > budget_bytes - new_bytes) { + goto done; + } + + const int first_entry = g_q4_attn_q_b_f16_cache_entries == 0u; + for (uint32_t mi = 0; mi < miss_count; mi++) { + const ds4_gpu_q4_attn_q_b_f16_sidecar_desc *desc = + &descs[miss_indices[mi]]; + ds4_gpu_q4_attn_q_b_f16_cache_entry *entry = + &g_q4_attn_q_b_f16_cache[free_slots[mi]]; + entry->model_map = model_map; + entry->model_size = model_size; + entry->weight_offset = desc->weight_offset; + entry->weight_bytes = desc->weight_bytes; + entry->in_dim = desc->in_dim; + entry->out_dim = desc->out_dim; + entry->f16_bytes = + desc->in_dim * desc->out_dim * sizeof(uint16_t); + entry->buffer = buffers[mi]; + entry->valid = 1; + } + g_q4_attn_q_b_f16_cache_bytes += new_bytes; + g_q4_attn_q_b_f16_cache_entries += miss_count; + g_q4_attn_q_b_f16_cache_builds += miss_count; + inserted = first_entry ? 2 : 1; + +done: + pthread_mutex_unlock(&g_q4_attn_q_b_f16_cache_mu); + return inserted; +} + +static int ds4_gpu_q4_attn_q_b_f16_cache_has_room( + uint64_t f16_bytes, + uint64_t budget_bytes) { + int has_room = 0; + pthread_mutex_lock(&g_q4_attn_q_b_f16_cache_mu); + has_room = + g_q4_attn_q_b_f16_cache_entries < + DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MAX_ENTRIES && + f16_bytes <= budget_bytes && + g_q4_attn_q_b_f16_cache_bytes <= budget_bytes - f16_bytes; + pthread_mutex_unlock(&g_q4_attn_q_b_f16_cache_mu); + return has_room; +} + +static int ds4_gpu_q4_attn_q_b_f16_cache_has_room_batch( + uint32_t new_entries, + uint64_t new_bytes, + uint64_t budget_bytes) { + int has_room = 0; + pthread_mutex_lock(&g_q4_attn_q_b_f16_cache_mu); + has_room = + new_entries <= + DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MAX_ENTRIES - + g_q4_attn_q_b_f16_cache_entries && + new_bytes <= budget_bytes && + g_q4_attn_q_b_f16_cache_bytes <= budget_bytes - new_bytes; + pthread_mutex_unlock(&g_q4_attn_q_b_f16_cache_mu); + return has_room; +} + +static int ds4_gpu_q4_attn_q_b_f16_builds_suppressed(void) { + int suppressed = 0; + pthread_mutex_lock(&g_q4_attn_q_b_f16_cache_mu); + suppressed = + g_q4_attn_q_b_f16_build_circuit_state != + DS4_Q4_ATTN_Q_B_F16_CIRCUIT_CLOSED; + pthread_mutex_unlock(&g_q4_attn_q_b_f16_cache_mu); + return suppressed; +} + +static void ds4_gpu_q4_attn_q_b_f16_suppress_builds( + ds4_gpu_q4_attn_q_b_f16_circuit_state state) { + if (state == DS4_Q4_ATTN_Q_B_F16_CIRCUIT_CLOSED) return; + pthread_mutex_lock(&g_q4_attn_q_b_f16_cache_mu); + if (state == DS4_Q4_ATTN_Q_B_F16_CIRCUIT_HARD || + g_q4_attn_q_b_f16_build_circuit_state == + DS4_Q4_ATTN_Q_B_F16_CIRCUIT_CLOSED) { + g_q4_attn_q_b_f16_build_circuit_state = state; + } + pthread_mutex_unlock(&g_q4_attn_q_b_f16_cache_mu); +} + +/* A working-set rejection describes the allocations present at one instant, + * unlike a cache-budget or build failure. A later session can have a smaller + * graph or follow cleanup of a failed strict session, so let its explicit + * prewarm re-evaluate pressure. Lazy per-layer misses stay suppressed to + * avoid retry storms. Called only while the cold-build mutex is held. */ +static int ds4_gpu_q4_attn_q_b_f16_prewarm_hard_suppressed(void) { + int hard = 0; + pthread_mutex_lock(&g_q4_attn_q_b_f16_cache_mu); + hard = g_q4_attn_q_b_f16_build_circuit_state == + DS4_Q4_ATTN_Q_B_F16_CIRCUIT_HARD; + if (g_q4_attn_q_b_f16_build_circuit_state == + DS4_Q4_ATTN_Q_B_F16_CIRCUIT_PRESSURE) { + g_q4_attn_q_b_f16_build_circuit_state = + DS4_Q4_ATTN_Q_B_F16_CIRCUIT_CLOSED; + } + pthread_mutex_unlock(&g_q4_attn_q_b_f16_cache_mu); + return hard; +} + +static int ds4_gpu_q4_attn_q_b_f16_working_set_policy( + uint64_t recommended, + uint64_t allocated, + uint64_t additional_bytes) { + if (recommended == 0u) return 0; + + const uint64_t reserve = recommended / 8u; + const uint64_t limit = recommended - reserve; + return allocated <= limit && additional_bytes <= limit - allocated; +} + +int ds4_gpu_test_q4_attn_q_b_f16_working_set_policy( + uint64_t recommended, + uint64_t allocated, + uint64_t additional) { + return ds4_gpu_q4_attn_q_b_f16_working_set_policy( + recommended, allocated, additional); +} + +/* Keep one eighth of Metal's recommended working set free for graph + * activations, KV growth, command-buffer transients, and the OS. Metal + * accounts both mmap-backed no-copy model buffers and private sidecars in + * currentAllocatedSize, so this conservative gate covers the combined Metal + * resource footprint rather than just the logical sidecar budget. */ +static int ds4_gpu_q4_attn_q_b_f16_working_set_has_room( + uint64_t additional_bytes) { + if (!g_device || additional_bytes > NSUIntegerMax) return 0; + return ds4_gpu_q4_attn_q_b_f16_working_set_policy( + (uint64_t)[g_device recommendedMaxWorkingSetSize], + (uint64_t)[g_device currentAllocatedSize], + additional_bytes); +} + +int ds4_gpu_make_room_for_q4_attn_q_b_f16_session(void) { + pthread_mutex_lock(&g_q4_attn_q_b_f16_cache_mu); + const uint32_t entries = g_q4_attn_q_b_f16_cache_entries; + const uint64_t bytes = g_q4_attn_q_b_f16_cache_bytes; + pthread_mutex_unlock(&g_q4_attn_q_b_f16_cache_mu); + if (entries == 0u) return 1; + + fprintf(stderr, + "ds4: evicting %.2f GiB of resident Q4 attn_q_b F16 " + "sidecars before allocating another live session\n", + ds4_gpu_gib(bytes)); + return ds4_gpu_release_q4_attn_q_b_f16_sidecars(); +} + +static int ds4_gpu_q4_attn_q_b_f16_fallback(int required, + int rejected, + int build_failure) { + pthread_mutex_lock(&g_q4_attn_q_b_f16_cache_mu); + g_q4_attn_q_b_f16_cache_fallbacks++; + if (rejected) g_q4_attn_q_b_f16_cache_rejects++; + if (build_failure) g_q4_attn_q_b_f16_cache_build_failures++; + pthread_mutex_unlock(&g_q4_attn_q_b_f16_cache_mu); + return required ? -1 : 0; +} + +/* Called only while the cold-build mutex is held. Existing READY entries + * remain usable; only future misses are suppressed. Hard failures persist + * until cache reset, while transient working-set pressure may be re-evaluated + * by the next explicit session prewarm. */ +static int ds4_gpu_q4_attn_q_b_f16_abort_cold_build( + int required, + int rejected, + int build_failure, + ds4_gpu_q4_attn_q_b_f16_circuit_state circuit_state) { + ds4_gpu_q4_attn_q_b_f16_suppress_builds(circuit_state); + pthread_mutex_unlock(&g_q4_attn_q_b_f16_build_mu); + return ds4_gpu_q4_attn_q_b_f16_fallback( + required, rejected, build_failure); +} + +static int ds4_gpu_q4_attn_q_b_f16_abort_prewarm( + int required, + int rejected, + int build_failure, + ds4_gpu_q4_attn_q_b_f16_circuit_state circuit_state) { + ds4_gpu_q4_attn_q_b_f16_suppress_builds(circuit_state); + pthread_mutex_unlock(&g_q4_attn_q_b_f16_build_mu); + pthread_mutex_lock(&g_q4_attn_q_b_f16_cache_mu); + if (rejected) g_q4_attn_q_b_f16_cache_rejects++; + if (build_failure) g_q4_attn_q_b_f16_cache_build_failures++; + pthread_mutex_unlock(&g_q4_attn_q_b_f16_cache_mu); + return required ? -1 : 0; +} + +int ds4_gpu_prepare_q4_attn_q_b_f16_sidecars( + const void *model_map, + uint64_t model_size, + const ds4_gpu_q4_attn_q_b_f16_sidecar_desc *descs, + uint32_t count, + uint32_t max_prefill_rows, + uint64_t working_set_reserve_bytes, + uint64_t *prepared_bytes) { + if (prepared_bytes) *prepared_bytes = 0; + if (!model_map || !descs || count == 0u || + count > DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MAX_ENTRIES || + max_prefill_rows < 32u) { + return 0; + } + if (!g_initialized && !ds4_gpu_init()) return 0; + + const int required = + ds4_gpu_env_bool("DS4_METAL_REQUIRE_Q4_ATTN_Q_B_F16_CACHE") == 1; + const int disabled = + ds4_gpu_env_bool("DS4_METAL_DISABLE_Q4_ATTN_Q_B_F16_CACHE") == 1; + const uint64_t min_tokens = ds4_gpu_env_u64( + "DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MIN_TOKENS", 512u, 32u, + UINT32_MAX); + + /* A session whose largest possible chunk is below the configured + * threshold can never be a cache candidate, even in strict mode. */ + if (max_prefill_rows < min_tokens) return 0; + if (disabled || g_ssd_streaming_mode || g_quality_mode || + !ds4_gpu_device_is_pre_m5_apple_silicon() || + ds4_gpu_mpp_available()) { + return required ? -1 : 0; + } + + uint64_t total_f16_bytes = 0; + for (uint32_t i = 0; i < count; i++) { + const ds4_gpu_q4_attn_q_b_f16_sidecar_desc *desc = &descs[i]; + if (desc->weight_type != DS4_METAL_TENSOR_Q4_K || + desc->in_dim != 1024u || desc->out_dim != 32768u || + desc->in_dim > UINT64_MAX / desc->out_dim / + sizeof(uint16_t)) { + return required ? -1 : 0; + } + const uint64_t blocks_per_row = desc->in_dim / 256u; + const uint64_t expected_weight_bytes = + desc->out_dim * blocks_per_row * 144u; + const uint64_t f16_bytes = + desc->in_dim * desc->out_dim * sizeof(uint16_t); + if (desc->weight_bytes != expected_weight_bytes || + desc->weight_offset > model_size || + desc->weight_bytes > model_size - desc->weight_offset || + UINT64_MAX - total_f16_bytes < f16_bytes) { + return required ? -1 : 0; + } + total_f16_bytes += f16_bytes; + } + + const uint64_t budget_mib = ds4_gpu_env_u64( + "DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MB", 3072u, 1u, 65536u); + const uint64_t budget_bytes = budget_mib * 1024u * 1024u; + const double build_t0 = ds4_gpu_now_ms(); + + pthread_mutex_lock(&g_q4_attn_q_b_f16_build_mu); + + uint32_t miss_indices[DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MAX_ENTRIES]; + uint32_t miss_count = 0; + uint64_t missing_bytes = 0; + for (uint32_t i = 0; i < count; i++) { + id hit_buffer = nil; + const ds4_gpu_q4_attn_q_b_f16_sidecar_desc *desc = &descs[i]; + if (ds4_gpu_q4_attn_q_b_f16_cache_peek( + model_map, model_size, desc->weight_offset, + desc->weight_bytes, desc->in_dim, desc->out_dim, + &hit_buffer)) { + continue; + } + miss_indices[miss_count++] = i; + missing_bytes += + desc->in_dim * desc->out_dim * sizeof(uint16_t); + } + if (miss_count == 0u) { + pthread_mutex_unlock(&g_q4_attn_q_b_f16_build_mu); + return 1; + } + + if (ds4_gpu_q4_attn_q_b_f16_prewarm_hard_suppressed()) { + return ds4_gpu_q4_attn_q_b_f16_abort_prewarm( + required, 1, 0, DS4_Q4_ATTN_Q_B_F16_CIRCUIT_CLOSED); + } + if (!ds4_gpu_q4_attn_q_b_f16_cache_has_room_batch( + miss_count, missing_bytes, budget_bytes)) { + fprintf(stderr, + "ds4: Metal Q4 attn_q_b F16 prewarm skipped: " + "%u missing sidecars need %.2f GiB, cache budget is " + "%llu MiB\n", + miss_count, + (double)missing_bytes / 1073741824.0, + (unsigned long long)budget_mib); + return ds4_gpu_q4_attn_q_b_f16_abort_prewarm( + required, 1, 0, DS4_Q4_ATTN_Q_B_F16_CIRCUIT_HARD); + } + if (UINT64_MAX - missing_bytes < working_set_reserve_bytes || + !ds4_gpu_q4_attn_q_b_f16_working_set_has_room( + missing_bytes + working_set_reserve_bytes)) { + const uint64_t allocated = + (uint64_t)[g_device currentAllocatedSize]; + const uint64_t recommended = + (uint64_t)[g_device recommendedMaxWorkingSetSize]; + fprintf(stderr, + "ds4: Metal Q4 attn_q_b F16 prewarm skipped for " + "working-set headroom: allocated %.2f GiB + sidecars " + "%.2f GiB + future sessions %.2f GiB, 7/8 safety " + "limit %.2f GiB\n", + (double)allocated / 1073741824.0, + (double)missing_bytes / 1073741824.0, + (double)working_set_reserve_bytes / 1073741824.0, + (double)(recommended - recommended / 8u) / + 1073741824.0); + return ds4_gpu_q4_attn_q_b_f16_abort_prewarm( + required, 1, 0, DS4_Q4_ATTN_Q_B_F16_CIRCUIT_PRESSURE); + } + + id dequant_pipeline = + ds4_gpu_get_pipeline("kernel_dequantize_q4_K_f16"); + /* Resolve both consumers before publishing any READY entry. Exact + * multiples of 32 and boundary batches use distinct function-constant + * specializations; compiling them here keeps first-prefill PSO cost out + * of the measured path and makes REQUIRE genuinely fail at preparation. */ + id mm_aligned_pipeline = + ds4_gpu_get_mul_mm_pipeline( + "kernel_mul_mm_f16_f32", false, false); + id mm_boundary_pipeline = + ds4_gpu_get_mul_mm_pipeline( + "kernel_mul_mm_f16_f32", false, true); + id build_cb = + [ds4_gpu_active_queue() commandBuffer]; + if (!dequant_pipeline || !mm_aligned_pipeline || + !mm_boundary_pipeline || !build_cb) { + return ds4_gpu_q4_attn_q_b_f16_abort_prewarm( + required, 0, 1, DS4_Q4_ATTN_Q_B_F16_CIRCUIT_HARD); + } + build_cb.label = @"ds4 Q4 attn_q_b F16 sidecar prewarm"; + + __strong id + sidecars[DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MAX_ENTRIES] = {nil}; + __strong id + weights[DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MAX_ENTRIES] = {nil}; + uint64_t weight_inner[DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MAX_ENTRIES] = {0}; + + for (uint32_t mi = 0; mi < miss_count; mi++) { + const ds4_gpu_q4_attn_q_b_f16_sidecar_desc *desc = + &descs[miss_indices[mi]]; + const uint64_t f16_bytes = + desc->in_dim * desc->out_dim * sizeof(uint16_t); + weights[mi] = ds4_gpu_wrap_model_range( + model_map, model_size, desc->weight_offset, + desc->weight_bytes, &weight_inner[mi]); + sidecars[mi] = [g_device + newBufferWithLength:(NSUInteger)f16_bytes + options:MTLResourceStorageModePrivate]; + if (!weights[mi] || !sidecars[mi]) { + return ds4_gpu_q4_attn_q_b_f16_abort_prewarm( + required, 0, 1, DS4_Q4_ATTN_Q_B_F16_CIRCUIT_HARD); + } + sidecars[mi].label = [NSString stringWithFormat: + @"ds4_attn_q_b_q4_f16_cache_layer_%u", desc->layer]; + } + + id enc = [build_cb computeCommandEncoder]; + if (!enc) { + return ds4_gpu_q4_attn_q_b_f16_abort_prewarm( + required, 0, 1, DS4_Q4_ATTN_Q_B_F16_CIRCUIT_HARD); + } + [enc setComputePipelineState:dequant_pipeline]; + for (uint32_t mi = 0; mi < miss_count; mi++) { + const ds4_gpu_q4_attn_q_b_f16_sidecar_desc *desc = + &descs[miss_indices[mi]]; + const uint32_t chunks_per_row = (uint32_t)(desc->in_dim / 16u); + const uint32_t row_count = (uint32_t)desc->out_dim; + [enc setBuffer:weights[mi] + offset:(NSUInteger)weight_inner[mi] + atIndex:0]; + [enc setBuffer:sidecars[mi] offset:0 atIndex:1]; + [enc setBytes:&chunks_per_row + length:sizeof(chunks_per_row) + atIndex:2]; + [enc setBytes:&row_count length:sizeof(row_count) atIndex:3]; + [enc dispatchThreadgroups:MTLSizeMake( + ((NSUInteger)chunks_per_row + 63u) / 64u, + (NSUInteger)row_count, + 1) + threadsPerThreadgroup:MTLSizeMake(64, 1, 1)]; + } + [enc endEncoding]; + [build_cb commit]; + if (!ds4_gpu_wait_command_buffer( + build_cb, "Q4 attn_q_b F16 sidecar prewarm")) { + return ds4_gpu_q4_attn_q_b_f16_abort_prewarm( + required, 0, 1, DS4_Q4_ATTN_Q_B_F16_CIRCUIT_HARD); + } + + const int insert_rc = ds4_gpu_q4_attn_q_b_f16_cache_insert_batch( + model_map, model_size, descs, miss_indices, sidecars, + miss_count, budget_bytes); + if (insert_rc <= 0) { + return ds4_gpu_q4_attn_q_b_f16_abort_prewarm( + required, insert_rc == 0, insert_rc < 0, + DS4_Q4_ATTN_Q_B_F16_CIRCUIT_HARD); + } + pthread_mutex_unlock(&g_q4_attn_q_b_f16_build_mu); + + if (prepared_bytes) *prepared_bytes = missing_bytes; + fprintf(stderr, + "ds4: Metal prewarmed %u resident Q4 attn_q_b F16 " + "sidecars (%.2f GiB) in %.3f ms; cache budget %llu MiB\n", + miss_count, + (double)missing_bytes / 1073741824.0, + ds4_gpu_now_ms() - build_t0, + (unsigned long long)budget_mib); + return 1; +} + +static int ds4_gpu_attn_q_b_f16_head_rms_rope_tail_tensor_impl( + ds4_gpu_tensor *out, + ds4_gpu_tensor *q_half, + const void *model_map, + uint64_t model_size, + uint64_t weight_offset, + uint32_t weight_type, + uint64_t in_dim, + uint64_t out_dim, + const ds4_gpu_tensor *x, + uint32_t n_tok, + uint32_t n_head, + uint32_t head_dim, + uint32_t n_rot, + uint32_t pos0, + uint32_t n_ctx_orig, + bool inverse, + float freq_base, + float freq_scale, + float ext_factor, + float attn_factor, + float beta_fast, + float beta_slow, + float eps, + int apply_norm_rope) { + (void)q_half; + if (weight_type != DS4_METAL_TENSOR_Q4_K) return 0; + /* This path is prefill-only. Keep decode and tiny batches off the + * environment parser, cache mutex, and pipeline lookup entirely. */ + if (n_tok < 32u) return 0; + if (!g_initialized && !ds4_gpu_init()) return 0; + + const int required = + ds4_gpu_env_bool("DS4_METAL_REQUIRE_Q4_ATTN_Q_B_F16_CACHE") == 1; + const int disabled = + ds4_gpu_env_bool("DS4_METAL_DISABLE_Q4_ATTN_Q_B_F16_CACHE") == 1; + const uint64_t min_tokens = ds4_gpu_env_u64( + "DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MIN_TOKENS", 512u, 32u, + UINT32_MAX); + + /* REQUIRE applies only to actual cache candidates. In particular, a + * short final prefill chunk below the configured threshold must retain + * the native Q4 fallback just like decode and tiny batches. */ + if ((uint64_t)n_tok < min_tokens) return 0; + + pthread_mutex_lock(&g_q4_attn_q_b_f16_cache_mu); + g_q4_attn_q_b_f16_cache_candidate_calls++; + pthread_mutex_unlock(&g_q4_attn_q_b_f16_cache_mu); + + /* The initial production arm targets DeepSeek-V4 Flash q_b exactly. + * Larger variants need more than 10 GiB of sidecar storage and require a + * separate memory/performance admission study. */ + if (disabled || g_ssd_streaming_mode || g_quality_mode || + !ds4_gpu_device_is_pre_m5_apple_silicon() || + ds4_gpu_mpp_available() || + in_dim != 1024u || out_dim != 32768u || + n_head == 0u || head_dim == 0u || + out_dim != (uint64_t)n_head * head_dim || + n_rot > head_dim || (n_rot & 1u) != 0u || + n_tok > (uint32_t)INT32_MAX || + pos0 > (uint32_t)INT32_MAX - n_tok) { + return ds4_gpu_q4_attn_q_b_f16_fallback(required, 1, 0); + } + + if (!out || !x || !model_map || + n_tok > UINT64_MAX / in_dim / sizeof(float) || + n_tok > UINT64_MAX / out_dim / sizeof(float) || + ds4_gpu_tensor_bytes(x) < + (uint64_t)n_tok * in_dim * sizeof(float) || + ds4_gpu_tensor_bytes(out) < + (uint64_t)n_tok * out_dim * sizeof(float)) { + return ds4_gpu_q4_attn_q_b_f16_fallback(required, 1, 0); + } + + const uint64_t blocks_per_row = in_dim / 256u; + const uint64_t row_bytes = blocks_per_row * 144u; + if (out_dim > UINT64_MAX / row_bytes) { + return ds4_gpu_q4_attn_q_b_f16_fallback(required, 1, 0); + } + const uint64_t weight_bytes = out_dim * row_bytes; + if (weight_offset > model_size || + weight_bytes > model_size - weight_offset || + out_dim > UINT64_MAX / in_dim / sizeof(uint16_t)) { + return ds4_gpu_q4_attn_q_b_f16_fallback(required, 1, 0); + } + const uint64_t f16_bytes = out_dim * in_dim * sizeof(uint16_t); + const uint64_t budget_mib = ds4_gpu_env_u64( + "DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MB", 3072u, 1u, 65536u); + const uint64_t budget_bytes = budget_mib * 1024u * 1024u; + + @autoreleasepool { + id f16_buffer = nil; + int hit = ds4_gpu_q4_attn_q_b_f16_cache_lookup( + model_map, model_size, weight_offset, weight_bytes, in_dim, + out_dim, &f16_buffer); + + /* Resolve every resource used by the hot matmul before starting a + * cold build. A failure can then take the native-Q4 fallback without + * abandoning an owned command buffer or publishing partial state. */ + id xbuf = ds4_gpu_tensor_buffer(x); + id outbuf = ds4_gpu_tensor_buffer(out); + const bool mm_bc_out = (n_tok & 31u) != 0u; + id mm_pipeline = + ds4_gpu_get_mul_mm_pipeline( + "kernel_mul_mm_f16_f32", false, mm_bc_out); + if (!xbuf || !outbuf || !mm_pipeline) { + return ds4_gpu_q4_attn_q_b_f16_fallback(required, 0, 1); + } + + int build_lock_held = 0; + if (!hit) { + /* Serialize cold builders. Recheck after taking the lock so + * another stream's completed build becomes a hit instead of a + * duplicate 64 MiB allocation and dequant dispatch. */ + pthread_mutex_lock(&g_q4_attn_q_b_f16_build_mu); + build_lock_held = 1; + f16_buffer = nil; + hit = ds4_gpu_q4_attn_q_b_f16_cache_peek( + model_map, model_size, weight_offset, weight_bytes, in_dim, + out_dim, &f16_buffer); + } + + if (!hit) { + if (ds4_gpu_q4_attn_q_b_f16_builds_suppressed()) { + return ds4_gpu_q4_attn_q_b_f16_abort_cold_build( + required, 1, 0, + DS4_Q4_ATTN_Q_B_F16_CIRCUIT_CLOSED); + } + if (!ds4_gpu_q4_attn_q_b_f16_cache_has_room( + f16_bytes, budget_bytes)) { + return ds4_gpu_q4_attn_q_b_f16_abort_cold_build( + required, 1, 0, + DS4_Q4_ATTN_Q_B_F16_CIRCUIT_HARD); + } + if (!ds4_gpu_q4_attn_q_b_f16_working_set_has_room( + f16_bytes)) { + return ds4_gpu_q4_attn_q_b_f16_abort_cold_build( + required, 1, 0, + DS4_Q4_ATTN_Q_B_F16_CIRCUIT_PRESSURE); + } + + id dequant_pipeline = + ds4_gpu_get_pipeline("kernel_dequantize_q4_K_f16"); + if (!dequant_pipeline) { + return ds4_gpu_q4_attn_q_b_f16_abort_cold_build( + required, 0, 1, + DS4_Q4_ATTN_Q_B_F16_CIRCUIT_HARD); + } + + uint64_t weight_inner = 0; + id weight_buffer = ds4_gpu_wrap_model_range( + model_map, model_size, weight_offset, weight_bytes, + &weight_inner); + if (!weight_buffer || f16_bytes > NSUIntegerMax) { + return ds4_gpu_q4_attn_q_b_f16_abort_cold_build( + required, 0, 1, + DS4_Q4_ATTN_Q_B_F16_CIRCUIT_HARD); + } + + f16_buffer = [g_device + newBufferWithLength:(NSUInteger)f16_bytes + options:MTLResourceStorageModePrivate]; + if (!f16_buffer) { + return ds4_gpu_q4_attn_q_b_f16_abort_cold_build( + required, 0, 1, + DS4_Q4_ATTN_Q_B_F16_CIRCUIT_HARD); + } + f16_buffer.label = @"ds4_attn_q_b_q4_f16_cache"; + + /* The sidecar build is deliberately isolated from g_batch_cb. + * It has no activation dependency, and a retained command buffer + * lets us verify completion before making the entry visible to + * any stream. Cache entries are therefore READY by definition. */ + id build_cb = + [ds4_gpu_active_queue() commandBuffer]; + if (!build_cb) { + return ds4_gpu_q4_attn_q_b_f16_abort_cold_build( + required, 0, 1, + DS4_Q4_ATTN_Q_B_F16_CIRCUIT_HARD); + } + build_cb.label = @"ds4 Q4 attn_q_b F16 sidecar build"; + + const uint32_t chunks_per_row = (uint32_t)(in_dim / 16u); + const uint32_t row_count = (uint32_t)out_dim; + id enc = + [build_cb computeCommandEncoder]; + if (!enc) { + return ds4_gpu_q4_attn_q_b_f16_abort_cold_build( + required, 0, 1, + DS4_Q4_ATTN_Q_B_F16_CIRCUIT_HARD); + } + [enc setComputePipelineState:dequant_pipeline]; + [enc setBuffer:weight_buffer + offset:(NSUInteger)weight_inner + atIndex:0]; + [enc setBuffer:f16_buffer offset:0 atIndex:1]; + [enc setBytes:&chunks_per_row + length:sizeof(chunks_per_row) + atIndex:2]; + [enc setBytes:&row_count length:sizeof(row_count) atIndex:3]; + [enc dispatchThreadgroups:MTLSizeMake( + ((NSUInteger)chunks_per_row + 63u) / 64u, + (NSUInteger)row_count, + 1) + threadsPerThreadgroup:MTLSizeMake(64, 1, 1)]; + [enc endEncoding]; + [build_cb commit]; + if (!ds4_gpu_wait_command_buffer( + build_cb, "Q4 attn_q_b F16 sidecar build")) { + return ds4_gpu_q4_attn_q_b_f16_abort_cold_build( + required, 0, 1, + DS4_Q4_ATTN_Q_B_F16_CIRCUIT_HARD); + } + + const int insert_rc = ds4_gpu_q4_attn_q_b_f16_cache_insert( + model_map, model_size, weight_offset, weight_bytes, + in_dim, out_dim, f16_bytes, budget_bytes, f16_buffer); + if (insert_rc == 0) { + /* Another key may have consumed the last budget slot while + * this build was in flight. Nothing has touched out yet, so + * the native Q4 path remains a clean fallback. */ + return ds4_gpu_q4_attn_q_b_f16_abort_cold_build( + required, 1, 0, + DS4_Q4_ATTN_Q_B_F16_CIRCUIT_HARD); + } + + if (insert_rc < 0) { + /* A concurrent builder won. Consume the published winner + * rather than reporting a false build failure. */ + f16_buffer = nil; + hit = ds4_gpu_q4_attn_q_b_f16_cache_lookup( + model_map, model_size, weight_offset, weight_bytes, + in_dim, out_dim, &f16_buffer); + if (!hit || !f16_buffer) { + return ds4_gpu_q4_attn_q_b_f16_abort_cold_build( + required, 0, 1, + DS4_Q4_ATTN_Q_B_F16_CIRCUIT_HARD); + } + } + + if (insert_rc == 2) { + fprintf(stderr, + "ds4: Metal resident Q4 attn_q_b F16 cache enabled " + "(budget %llu MiB, min batch %llu tokens)\n", + (unsigned long long)budget_mib, + (unsigned long long)min_tokens); + } + } + + if (build_lock_held) { + pthread_mutex_unlock(&g_q4_attn_q_b_f16_build_mu); + } + + if (!f16_buffer) { + return ds4_gpu_q4_attn_q_b_f16_fallback(required, 0, 1); + } + + int owned = 0; + id cb = ds4_gpu_command_buffer(&owned); + if (!cb) { + return ds4_gpu_q4_attn_q_b_f16_fallback(required, 0, 1); + } + + ds4_gpu_mul_mm_args mm_args = ds4_gpu_make_mm_args( + in_dim, out_dim, n_tok, in_dim * sizeof(uint16_t)); + id enc = ds4_gpu_compute_encoder(cb); + if (!enc) { + (void)ds4_gpu_finish_command_buffer( + cb, owned, "Q4 attn_q_b cached F16 encoder allocation"); + return ds4_gpu_q4_attn_q_b_f16_fallback(required, 0, 1); + } + [enc setComputePipelineState:mm_pipeline]; + [enc setBytes:&mm_args length:sizeof(mm_args) atIndex:0]; + [enc setBuffer:f16_buffer offset:0 atIndex:1]; + [enc setBuffer:xbuf offset:ds4_gpu_tensor_offset(x) atIndex:2]; + [enc setBuffer:outbuf offset:ds4_gpu_tensor_offset(out) atIndex:3]; + [enc setThreadgroupMemoryLength:(mm_bc_out ? 8192u : 6144u) + atIndex:0]; + [enc dispatchThreadgroups:MTLSizeMake( + ((NSUInteger)n_tok + 31u) / 32u, + (NSUInteger)out_dim / 64u, + 1) + threadsPerThreadgroup:MTLSizeMake(128, 1, 1)]; + ds4_gpu_end_compute_encoder(cb, enc); + + if (!ds4_gpu_finish_command_buffer( + cb, owned, "Q4 attn_q_b cached F16 matmul")) { + return ds4_gpu_q4_attn_q_b_f16_fallback(required, 0, 1); + } + } + + if (!apply_norm_rope) return 1; + if (!ds4_gpu_head_rms_norm_rope_tail_tensor( + out, n_tok, n_head, head_dim, n_rot, pos0, n_ctx_orig, + inverse, freq_base, freq_scale, ext_factor, attn_factor, + beta_fast, beta_slow, eps)) { + return ds4_gpu_q4_attn_q_b_f16_fallback(required, 0, 1); + } + return 1; +} + int ds4_gpu_attn_q_b_f16_head_rms_rope_tail_tensor( ds4_gpu_tensor *out, ds4_gpu_tensor *q_half, const void *model_map, uint64_t model_size, uint64_t weight_offset, + uint32_t weight_type, uint64_t in_dim, uint64_t out_dim, const ds4_gpu_tensor *x, @@ -24982,13 +26048,27 @@ int ds4_gpu_attn_q_b_f16_head_rms_rope_tail_tensor( float beta_fast, float beta_slow, float eps) { - (void)out; (void)q_half; (void)model_map; (void)model_size; - (void)weight_offset; (void)in_dim; (void)out_dim; (void)x; - (void)n_tok; (void)n_head; (void)head_dim; (void)n_rot; (void)pos0; - (void)n_ctx_orig; (void)inverse; (void)freq_base; (void)freq_scale; - (void)ext_factor; (void)attn_factor; (void)beta_fast; (void)beta_slow; - (void)eps; - return 0; + return ds4_gpu_attn_q_b_f16_head_rms_rope_tail_tensor_impl( + out, q_half, model_map, model_size, weight_offset, weight_type, + in_dim, out_dim, x, n_tok, n_head, head_dim, n_rot, pos0, + n_ctx_orig, inverse, freq_base, freq_scale, ext_factor, + attn_factor, beta_fast, beta_slow, eps, 1); +} + +int ds4_gpu_test_q4_attn_q_b_f16_projection_tensor( + ds4_gpu_tensor *out, + const void *model_map, + uint64_t model_size, + uint64_t weight_offset, + uint64_t in_dim, + uint64_t out_dim, + const ds4_gpu_tensor *x, + uint32_t n_tok) { + return ds4_gpu_attn_q_b_f16_head_rms_rope_tail_tensor_impl( + out, NULL, model_map, model_size, weight_offset, + DS4_METAL_TENSOR_Q4_K, in_dim, out_dim, x, n_tok, + 64u, 512u, 0u, 0u, 0u, false, + 10000.0f, 1.0f, 0.0f, 1.0f, 32.0f, 1.0f, 1.0e-6f, 0); } int ds4_gpu_dsv4_fp8_kv_quantize_tensor( diff --git a/metal/dense.metal b/metal/dense.metal index c9dde5615..ecb58729b 100644 --- a/metal/dense.metal +++ b/metal/dense.metal @@ -1539,6 +1539,44 @@ void dequantize_dense_q4_K(device const ds4_dense_block_q4_K *xb, short il, thre reg = (type4x4)reg_f; } +/* + * One-shot resident Q4_K -> F16 materialization used by the attn_q_b + * prefill cache. Keeping the production dequantizer here is intentional: + * kernel_mul_mm_q4_K_f32 computes the same float4x4 values and rounds them + * while storing into its half threadgroup tile. Instantiating the helper + * with half4x4 performs that same single rounding here, so the cached matrix + * contains exactly the half values the native Q4 kernel would otherwise + * rebuild for every 64x32 output tile. + * + * Each thread expands one consecutive 16-value chunk. Four explicit half4 + * stores preserve the half4x4 layout without relying on device-address-space + * matrix stores, which are not accepted by every supported Metal compiler. + */ +kernel void kernel_dequantize_q4_K_f16( + device const ds4_dense_block_q4_K *src [[buffer(0)]], + device half4 *dst [[buffer(1)]], + constant uint &chunks_per_row [[buffer(2)]], + constant uint &row_count [[buffer(3)]], + uint2 gid [[thread_position_in_grid]]) { + if (gid.x >= chunks_per_row || gid.y >= row_count) return; + + const uint block = gid.x >> 4; + const short il = (short)(gid.x & 15u); + const uint blocks_per_row = chunks_per_row >> 4; + device const ds4_dense_block_q4_K *xb = + src + (uint64_t)gid.y * blocks_per_row + block; + + half4x4 values; + dequantize_dense_q4_K(xb, il, values); + + const uint64_t out4 = + ((uint64_t)gid.y * chunks_per_row + gid.x) * 4u; + dst[out4 + 0u] = values[0]; + dst[out4 + 1u] = values[1]; + dst[out4 + 2u] = values[2]; + dst[out4 + 3u] = values[3]; +} + /* * Bit-identical twin of dequantize_q8_0 for the MPP staging loop: same * half(float(qs[i]) * d) per element, but the 16 consecutive int8 lanes are diff --git a/rocm/ds4_rocm_norm_rope.cuh b/rocm/ds4_rocm_norm_rope.cuh index b3f543b2f..87fdc53f7 100644 --- a/rocm/ds4_rocm_norm_rope.cuh +++ b/rocm/ds4_rocm_norm_rope.cuh @@ -535,6 +535,7 @@ extern "C" int ds4_gpu_attn_q_b_f16_head_rms_rope_tail_tensor( const void *model_map, uint64_t model_size, uint64_t weight_offset, + uint32_t weight_type, uint64_t in_dim, uint64_t out_dim, const ds4_gpu_tensor *x, @@ -552,7 +553,7 @@ extern "C" int ds4_gpu_attn_q_b_f16_head_rms_rope_tail_tensor( float beta_fast, float beta_slow, float eps) { - if (!g_cublas_ready || !out || !q_half || !x || !model_map || n_tok == 0 || + if (weight_type != 8u || !g_cublas_ready || !out || !q_half || !x || !model_map || n_tok == 0 || n_rot > head_dim || (n_rot & 1u) || out_dim != (uint64_t)n_head * head_dim || x->bytes < (uint64_t)n_tok * in_dim * sizeof(float) || out->bytes < (uint64_t)n_tok * out_dim * sizeof(float) || diff --git a/scripts/environment_variables.tsv b/scripts/environment_variables.tsv index f206deb42..5496bc2fa 100644 --- a/scripts/environment_variables.tsv +++ b/scripts/environment_variables.tsv @@ -5,8 +5,8 @@ external/system LINENOISE_ASSUME_TTY Pure presence test flag: any defined value, external/system LINENOISE_COLS If defined, its value is returned directly through atoi with no validation: empty/nonnumeric becomes 0 and signed values are accepted. If unset, linenoise uses TIOCGWINSZ, then a cursor-position query, then fallback width 80. Force a deterministic terminal column count for linenoise wrapping/layout tests. linenoise.c:684 external/system PATH Colon-separated executable search directories consulted only after DS4_CHROME, macOS app paths and fixed Chrome/Chromium paths fail. The first executable google-chrome, google-chrome-stable, chromium or chromium-browser wins; unset/empty/no match falls back to the literal command google-chrome, which execlp may search again. Locate a Chrome/Chromium executable for the ds4 web/CDP tool. ds4_web.c:992 external/system TERM Case-insensitive terminal name. Values dumb, cons25, or emacs select linenoise's simple prompt plus blocking line reader; unset, empty, or any other value selects the normal interactive editor when stdin is a TTY. Avoid ANSI/raw interactive editing on terminal types known not to support the required escape sequences. linenoise.c:559 -runtime/bench DS4_BENCH_DISABLE_SNAPSHOT presence flag; unset allows snapshots for eligible frontiers Disable benchmark state snapshots. ds4_bench.c:709 -runtime/bench DS4_BENCH_FORCE_SNAPSHOT presence flag; unset obeys the normal size/eligibility checks Force benchmark state snapshots despite the normal limit. ds4_bench.c:712 +runtime/bench DS4_BENCH_DISABLE_SNAPSHOT presence flag; unset allows snapshots for eligible frontiers Disable benchmark state snapshots. ds4_bench.c:720 +runtime/bench DS4_BENCH_FORCE_SNAPSHOT presence flag; unset obeys the normal size/eligibility checks Force benchmark state snapshots despite the normal limit. ds4_bench.c:723 runtime/bench DS4_BENCH_SNAPSHOT_MAX_BYTES unsigned bytes or unlimited/inf; default DS4_BENCH_DEFAULT_SNAPSHOT_MAX_BYTES Limit session snapshot size during benchmark sweeps. ds4_bench.c:70 runtime/cli DS4_CLI_FORCE_SESSION Pure presence flag: any defined value, including empty or "0", forces the session path. Unset uses the session path only for distributed coordinators, TP leaders, temperature>0, or MTP depth>1; otherwise the CLI calls direct argmax generation. Force ordinary CLI generation through run_sampled_generation/session APIs so single-node validation follows the same stateful path as TP/distributed runs. ds4_cli.c:1226 runtime/core DS4_BATCHED_FFN Pure presence flag: any defined value, including empty or "0", enables. Unset leaves the default shared-expert-batched FFN path (or its configured fallback). It is read only by CPU layer-major prefill and takes precedence over shared-batch and token-parallel FFN choices. Run the complete CPU prefill FFN in chunks through layer_ffn_batch instead of the default shared-expert-only batched path. ds4.c:14411 @@ -584,6 +584,7 @@ runtime/metal DS4_METAL_DISABLE_PRO_Q4_EXPERT_TABLE_PRELOAD presence rollback; u runtime/metal DS4_METAL_DISABLE_Q4_ATTN_OUT_B_F16_RHS value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Disables resident pre-M5 Q4 attention output-B F16 RHS materialization and restores per-tile F32 staging. ds4_metal.m:28533 runtime/metal DS4_METAL_DISABLE_Q4_ATTN_OUT_HC_FUSE presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 attn out HC fuse. ds4_metal.m:47624 runtime/metal DS4_METAL_DISABLE_Q4_ATTN_OUT_TINY_BATCH value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Disables Q4 attn out tiny batch. ds4_metal.m:28396 +runtime/metal DS4_METAL_DISABLE_Q4_ATTN_Q_B_F16_CACHE value-aware boolean; unset/0: automatic path remains enabled; empty/1/true/yes/on disables Disables the resident pre-M5 Q4_K attn_q_b F16 weight sidecar and restores native per-tile Q4 dequantization. ds4_metal.m:25485; ds4_metal.m:25722 runtime/metal DS4_METAL_DISABLE_Q4_BATCH_EXPERT_TABLE presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 batch expert table. ds4_metal.m:44883 runtime/metal DS4_METAL_DISABLE_Q4_DENSE_PAIR presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 dense pair. ds4_metal.m:21990 runtime/metal DS4_METAL_DISABLE_Q4_EXACT_BOUNDARY presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 exact boundary. ds4_metal.m:42239 @@ -805,6 +806,9 @@ runtime/metal DS4_METAL_PREFILL_CHUNK positive token count used only when CLI ch runtime/metal DS4_METAL_PRO_Q4_CPU_ROUTER nonempty boolean; unset/empty or exact 0: off; every other value: on Uses the CPU router for PRO Q4 selected-expert decode. ds4.c:20934 runtime/metal DS4_METAL_PRO_Q4_CPU_ROUTER_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for pro Q4 CPU router. ds4.c:21644 runtime/metal DS4_METAL_Q4_ADDR_USE_RESOURCES presence control; unset: off/default; any value including 0 enables Declares Q4 address-table resources explicitly on encoders. ds4_metal.m:20259 +runtime/metal DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MB integer 1..65536 MiB; default 3072; above max clamps, below min/invalid restores default Caps copied F16 sidecar storage for resident Q4_K attn_q_b weights. ds4_metal.m:25523; ds4_metal.m:25774 +runtime/metal DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MIN_TOKENS integer 32..UINT32_MAX; default 512; below min/invalid restores default Sets the minimum resident prefill batch that may build or use the Q4_K attn_q_b F16 sidecar; smaller tails remain non-candidates. ds4_metal.m:25487; ds4_metal.m:25724 +runtime/metal DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints resident Q4_K attn_q_b F16 cache counters and circuit state at Metal cleanup. ds4_metal.m:11399 runtime/metal DS4_METAL_Q4_EXPERT_GROUP_SIZE positive uint32; default 32; clamped to total expert count Sets experts processed per grouped Q4 dispatch. ds4_metal.m:34088 runtime/metal DS4_METAL_Q4_EXPERT_TABLE_GROUP_SIZE integer 2..total experts; default/invalid 1 (ungrouped) Sets grouped exact-view width while building Q4 expert tables. ds4_metal.m:19604 runtime/metal DS4_METAL_Q4_EXPERT_TABLE_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for Q4 expert table. ds4_metal.m:20103 @@ -840,6 +844,7 @@ runtime/metal DS4_METAL_REQUIRE_M1_IQ2_MID_ONLY presence strict check; unset: fa runtime/metal DS4_METAL_REQUIRE_OUTPUT_HC_WEIGHTS4 presence strict check; unset: fallback allowed; any value including 0 requires the path Requires output HC weights4 and makes eligible fallback fail closed. ds4_metal.m:46789 runtime/metal DS4_METAL_REQUIRE_Q4_ATTN_OUT_B_F16_RHS value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Requires the resident pre-M5 Q4 attention output-B F16 RHS path and makes an ineligible or disabled candidate fail closed. ds4_metal.m:28531 runtime/metal DS4_METAL_REQUIRE_Q4_ATTN_OUT_TINY_BATCH value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Requires Q4 attn out tiny batch and makes eligible fallback fail closed. ds4_metal.m:28373 +runtime/metal DS4_METAL_REQUIRE_Q4_ATTN_Q_B_F16_CACHE value-aware boolean; unset/0: fallback allowed; empty/1/true/yes/on requires Requires the sidecar for resident pre-M5 Q4_K Flash attn_q_b batches at or above the configured minimum; decode and below-min batches remain non-candidates. ds4_metal.m:25483; ds4_metal.m:25720 runtime/metal DS4_METAL_REQUIRE_Q4_SSD_PREFILL_ATTN_OUT_EXACTN value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Requires Q4 SSD prefill attn out exactn and makes eligible fallback fail closed. ds4_metal.m:28089 runtime/metal DS4_METAL_REQUIRE_Q4_SSD_PREFILL_ATTN_OUT_SCALE_META value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Requires shared scale/min metadata in the Q4 SSD prefill attention-output exact-N kernel and makes fallback fail closed. ds4_metal.m:28209 runtime/metal DS4_METAL_REQUIRE_Q4_SSD_SESSION_UNION nonempty boolean; unset/empty or exact 0: off; every other value: on Requires Q4 SSD session union and makes eligible fallback fail closed. ds4.c:65164 @@ -1109,7 +1114,9 @@ test-only DS4_TEST_LOGPROB_AUTO_METAL presence flag; unset forces DS4_METAL_DISA test-only DS4_TEST_LONG_PROMPT nonempty readable prompt-file path; unset/empty defaults to tests/long_context_story_prompt.txt Selects the rendered story prompt for the long-context fact-recall test. tests/ds4_test.c:6970 test-only DS4_TEST_LONG_WORDS atoi integer; unset/empty/nonnumeric defaults to 0; valid range is 0..DS4_TEST_CONTEXT-128 and numeric prefixes are accepted Adds repeated words to alternating CUDA session-batch prompts to exercise long-prefill rows. tests/test_cuda_session_batch.c:135 test-only DS4_TEST_METAL_EXACTN_BATCH_HEAD nonempty value other than exact 0 enables; unset/empty/0 disables; false/off also enable Enables the Metal exact-N batch-head path and requires its attempt/use counters for every eligible oracle case. tests/test_metal_exactn_oracle.c:401 -test-only DS4_TEST_METAL_EXACTN_ORACLE presence flag compiled only with DS4_TEST_HOOKS; absent from normal production builds Force allocation of the exact-N Metal verifier/oracle workspace in tests. ds4.c:61991 +test-only DS4_TEST_METAL_EXACTN_ORACLE presence flag compiled only with DS4_TEST_HOOKS; absent from normal production builds Force allocation of the exact-N Metal verifier/oracle workspace in tests. ds4.c:67329 +test-only DS4_TEST_METAL_Q4_QB_F16_CACHE_TIMING presence flag; unset runs correctness only; any defined value including empty or 0 also runs resident cold/warm timing Enables the production-shape timing phase for the Metal Q4_K attn_q_b F16 sidecar oracle. tests/test_metal_q4_qb_f16_cache.c:59 +test-only DS4_TEST_METAL_Q4_QB_F16_CACHE_TIMING_TOKENS strict decimal integer in 32..4096; default 4096 Sets the resident synthetic token count for the Metal Q4_K attn_q_b F16 sidecar timing phase. tests/test_metal_q4_qb_f16_cache.c:978 test-only DS4_TEST_MIXED_INITIAL integer 128..context-1; default 128 Set the initial prefill length for the CUDA mixed-batch oracle. tests/test_cuda_mixed_batch.c:111 test-only DS4_TEST_MIXED_QUANTUM integer 1..context-1; default 128 Set the number of prompt tokens added per CUDA mixed-batch round. tests/test_cuda_mixed_batch.c:113 test-only DS4_TEST_MIXED_ROUNDS integer 1..64; default 3 Set the number of CUDA mixed-batch oracle rounds. tests/test_cuda_mixed_batch.c:115 diff --git a/tests/test_metal_q4_qb_f16_cache.c b/tests/test_metal_q4_qb_f16_cache.c new file mode 100644 index 000000000..c6d66d577 --- /dev/null +++ b/tests/test_metal_q4_qb_f16_cache.c @@ -0,0 +1,1181 @@ +#define _DARWIN_C_SOURCE + +/* GGUF-free production-shape oracle for the resident pre-M5 Metal Q4_K + * attn_q_b F16 weight sidecar. The candidate is compared bit-for-bit with + * the established Q4_K matmul followed by the exact same head norm/RoPE + * entry point used by the production fallback. */ + +#include "ds4_gpu.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +bool ds4_log_is_tty(FILE *fp) { + (void)fp; + return false; +} + +#ifdef __APPLE__ + +enum { + Q4_K_TYPE = 12u, + QK_K = 256u, + IN_DIM = 1024u, + OUT_DIM = 32768u, + N_HEAD = 64u, + HEAD_DIM = 512u, + N_ROT = 64u, + MAX_TOKENS = 64u, + BLOCKS_PER_ROW = IN_DIM / QK_K, + GROUPS_PER_BLOCK = 8u, + GROUP_SIZE = 32u, + GUARD_FLOATS = 256u, + GUARD_HALFS = 256u, + TIMING_SAMPLES = 5u, +}; + +typedef struct { + uint16_t d; + uint16_t dmin; + uint8_t scales[12]; + uint8_t qs[QK_K / 2u]; +} block_q4_K; + +static const char *k_disable = + "DS4_METAL_DISABLE_Q4_ATTN_Q_B_F16_CACHE"; +static const char *k_require = + "DS4_METAL_REQUIRE_Q4_ATTN_Q_B_F16_CACHE"; +static const char *k_min_tokens = + "DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MIN_TOKENS"; +static const char *k_cache_mb = + "DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MB"; +static const char *k_timing = + "DS4_TEST_METAL_Q4_QB_F16_CACHE_TIMING"; + +static const uint32_t k_reference_poison = 0x7fc10000u; +static const uint32_t k_candidate_poison = 0x7fc30000u; + +static void fail(const char *what) { + fprintf(stderr, "Metal Q4 attn_q_b F16 cache oracle FAIL: %s\n", what); + exit(1); +} + +#define CHECK(expr, what) do { if (!(expr)) fail(what); } while (0) + +static uint64_t align_up(uint64_t value, uint64_t alignment) { + return (value + alignment - 1u) / alignment * alignment; +} + +static void pack_scales(uint8_t packed[12], + const uint8_t scales[GROUPS_PER_BLOCK], + const uint8_t minima[GROUPS_PER_BLOCK]) { + memset(packed, 0, 12u); + for (uint32_t group = 0; group < 4u; group++) { + packed[group] = scales[group] & 63u; + packed[group + 4u] = minima[group] & 63u; + } + for (uint32_t group = 4u; group < GROUPS_PER_BLOCK; group++) { + packed[group + 4u] = (scales[group] & 15u) | + ((minima[group] & 15u) << 4u); + packed[group - 4u] |= (scales[group] >> 4u) << 6u; + packed[group] |= (minima[group] >> 4u) << 6u; + } +} + +static void fill_q4_matrix(block_q4_K *matrix) { + CHECK(sizeof(block_q4_K) == 144u, "unexpected Q4_K block size"); + + for (uint32_t row = 0; row < OUT_DIM; row++) { + for (uint32_t block = 0; block < BLOCKS_PER_ROW; block++) { + block_q4_K *b = matrix + + (uint64_t)row * BLOCKS_PER_ROW + block; + const uint32_t key = + row * 1009u + block * 313u + + (row ^ (block * 17u)) + 29u; + uint8_t scales[GROUPS_PER_BLOCK]; + uint8_t minima[GROUPS_PER_BLOCK]; + + for (uint32_t group = 0; group < GROUPS_PER_BLOCK; group++) { + scales[group] = + (uint8_t)((key + group * 37u) & 63u); + minima[group] = + (uint8_t)((key / 3u + group * 29u) & 63u); + } + pack_scales(b->scales, scales, minima); + for (uint32_t i = 0; i < QK_K / 2u; i++) { + b->qs[i] = (uint8_t)( + key + i * 37u + (i >> 2u) * 11u); + } + + /* Exercise non-power-of-two half scales and all packed 6-bit + * scale/minimum lanes, including the high bits of groups 4--7. */ + b->d = (uint16_t)(0x1801u + (key & 0x01ffu)); + b->dmin = (uint16_t)(0x1403u + ((key >> 3u) & 0x01ffu)); + } + } +} + +static uint32_t poison_bits(uint32_t base, uint64_t index) { + return base + (uint32_t)(index & 0xffffu); +} + +static void poison_f32(float *values, uint64_t count, uint32_t base) { + for (uint64_t i = 0; i < count; i++) { + const uint32_t bits = poison_bits(base, i); + memcpy(&values[i], &bits, sizeof(bits)); + } +} + +static uint64_t count_poison_f32_mismatches(const float *values, + uint64_t begin, + uint64_t end, + uint32_t base) { + uint64_t mismatches = 0; + for (uint64_t i = begin; i < end; i++) { + uint32_t bits = 0; + memcpy(&bits, &values[i], sizeof(bits)); + if (bits != poison_bits(base, i)) mismatches++; + } + return mismatches; +} + +static uint16_t half_poison_bits(uint64_t index) { + return (uint16_t)(0x7e00u | (uint16_t)(index & 0x01ffu)); +} + +static void poison_f16(uint16_t *values, uint64_t count) { + for (uint64_t i = 0; i < count; i++) { + values[i] = half_poison_bits(i); + } +} + +static uint64_t count_poison_f16_mismatches(const uint16_t *values, + uint64_t count) { + uint64_t mismatches = 0; + for (uint64_t i = 0; i < count; i++) { + if (values[i] != half_poison_bits(i)) mismatches++; + } + return mismatches; +} + +static void fill_inputs(float *values) { + for (uint32_t token = 0; token < MAX_TOKENS; token++) { + for (uint32_t col = 0; col < IN_DIM; col++) { + const uint32_t key = token * 131u + col * 17u + + ((col >> 3u) ^ (token * 29u)); + /* The Metal MM kernels consume a half RHS. */ + values[(uint64_t)token * IN_DIM + col] = + (float)((int)(key % 129u) - 64) / 64.0f; + } + } +} + +static uint64_t count_bit_mismatches(const float *reference, + const float *candidate, + uint64_t count, + uint64_t *first) { + uint64_t mismatches = 0; + *first = UINT64_MAX; + for (uint64_t i = 0; i < count; i++) { + if (memcmp(&reference[i], &candidate[i], sizeof(float)) != 0) { + if (*first == UINT64_MAX) *first = i; + mismatches++; + } + } + return mismatches; +} + +static int run_reference(ds4_gpu_tensor *out, + const void *model, + uint64_t model_bytes, + const ds4_gpu_tensor *x, + uint32_t n_tok) { + if (!ds4_gpu_matmul_quant_tensor( + out, model, model_bytes, 0, Q4_K_TYPE, + IN_DIM, OUT_DIM, x, n_tok)) { + return 0; + } + return ds4_gpu_head_rms_norm_rope_tail_tensor( + out, n_tok, N_HEAD, HEAD_DIM, N_ROT, + 17u, 0u, false, + 10000.0f, 1.0f, 0.0f, 1.0f, 32.0f, 1.0f, 1.0e-6f); +} + +static int run_candidate(ds4_gpu_tensor *out, + ds4_gpu_tensor *q_half, + const void *model, + uint64_t model_bytes, + const ds4_gpu_tensor *x, + uint32_t n_tok) { + return ds4_gpu_attn_q_b_f16_head_rms_rope_tail_tensor( + out, q_half, model, model_bytes, 0, Q4_K_TYPE, + IN_DIM, OUT_DIM, x, n_tok, N_HEAD, HEAD_DIM, N_ROT, + 17u, 0u, false, + 10000.0f, 1.0f, 0.0f, 1.0f, 32.0f, 1.0f, 1.0e-6f); +} + +static void check_cache_storage_unchanged( + const ds4_gpu_q4_attn_q_b_f16_cache_report *before, + const ds4_gpu_q4_attn_q_b_f16_cache_report *after, + const char *what) { + if (after->entries != before->entries || + after->bytes != before->bytes || + after->lookups != before->lookups || + after->hits != before->hits || + after->misses != before->misses || + after->builds != before->builds || + after->build_failures != before->build_failures || + after->build_circuit_open != before->build_circuit_open) { + fail(what); + } +} + +static double monotonic_ms(void) { + struct timespec ts; + CHECK(clock_gettime(CLOCK_MONOTONIC, &ts) == 0, "monotonic clock"); + return (double)ts.tv_sec * 1000.0 + (double)ts.tv_nsec / 1.0e6; +} + +static int compare_double(const void *lhs, const void *rhs) { + const double a = *(const double *)lhs; + const double b = *(const double *)rhs; + return (a > b) - (a < b); +} + +int main(void) { + static const uint32_t token_cases[] = {32u, 33u, 64u}; + const uint64_t page = (uint64_t)getpagesize(); + const uint64_t row_bytes = + (uint64_t)BLOCKS_PER_ROW * sizeof(block_q4_K); + const uint64_t weight_bytes = (uint64_t)OUT_DIM * row_bytes; + const uint64_t model_bytes = align_up(2u * weight_bytes, page); + const uint64_t support_model_bytes = align_up(weight_bytes, page); + const uint64_t f16_cache_bytes = + (uint64_t)OUT_DIM * IN_DIM * sizeof(uint16_t); + const uint64_t input_count = (uint64_t)MAX_TOKENS * IN_DIM; + const uint64_t input_storage_count = + GUARD_FLOATS + input_count + GUARD_FLOATS; + const uint64_t max_output_count = (uint64_t)MAX_TOKENS * OUT_DIM; + const uint64_t output_storage_count = + GUARD_FLOATS + max_output_count + GUARD_FLOATS; + const uint64_t q_half_storage_count = + GUARD_HALFS + max_output_count + GUARD_HALFS; + + CHECK(Q4_K_TYPE == 12u, "Q4_K GGUF type must be 12"); + CHECK(row_bytes == 576u, "unexpected Q4_K row size"); + CHECK(weight_bytes == 18u * 1024u * 1024u, + "unexpected production q_b Q4_K size"); + CHECK(f16_cache_bytes == 64u * 1024u * 1024u, + "unexpected production q_b F16 sidecar size"); + CHECK(ds4_gpu_test_q4_attn_q_b_f16_working_set_policy( + 0u, 0u, 0u) == 0, + "unknown working set must reject"); + CHECK(ds4_gpu_test_q4_attn_q_b_f16_working_set_policy( + 800u, 600u, 100u) == 1, + "working-set equality boundary"); + CHECK(ds4_gpu_test_q4_attn_q_b_f16_working_set_policy( + 800u, 600u, 101u) == 0, + "working-set one-byte overflow"); + CHECK(ds4_gpu_test_q4_attn_q_b_f16_working_set_policy( + 800u, 701u, 0u) == 0, + "allocated working set beyond safety limit"); + CHECK(ds4_gpu_test_q4_attn_q_b_f16_working_set_policy( + UINT64_MAX, 0u, UINT64_MAX) == 0, + "working-set overflow-sized request"); + + CHECK(unsetenv(k_disable) == 0, "clear cache disable env"); + CHECK(unsetenv(k_require) == 0, "clear cache require env"); + CHECK(unsetenv(k_min_tokens) == 0, "clear cache minimum env"); + CHECK(unsetenv(k_cache_mb) == 0, "clear cache budget env"); + CHECK(setenv(k_min_tokens, "32", 1) == 0, + "set 32-token cache minimum"); + CHECK(setenv(k_require, "1", 1) == 0, + "require Q4 attn_q_b F16 cache"); + + CHECK(ds4_gpu_init() != 0, "Metal init"); + if (!ds4_gpu_device_is_pre_m5_apple_silicon()) { + fprintf(stderr, + "Metal Q4 attn_q_b F16 cache oracle SKIP: " + "requires Apple M1--M4\n"); + ds4_gpu_cleanup(); + return 0; + } + + void *model = NULL; + CHECK(posix_memalign(&model, (size_t)page, (size_t)model_bytes) == 0, + "page-aligned model allocation"); + memset(model, 0, (size_t)model_bytes); + fill_q4_matrix(model); + fill_q4_matrix((block_q4_K *)((uint8_t *)model + weight_bytes)); + ((block_q4_K *)((uint8_t *)model + weight_bytes))[0].d ^= 0x001fu; + + void *support_model = NULL; + CHECK(posix_memalign(&support_model, (size_t)page, + (size_t)support_model_bytes) == 0, + "page-aligned support model allocation"); + memset(support_model, 0, (size_t)support_model_bytes); + fill_q4_matrix(support_model); + ((block_q4_K *)support_model)[0].dmin ^= 0x003du; + + float *input_host = malloc( + (size_t)input_storage_count * sizeof(float)); + float *input_readback = malloc( + (size_t)input_storage_count * sizeof(float)); + float *reference_host = malloc( + (size_t)output_storage_count * sizeof(float)); + float *candidate_host = malloc( + (size_t)output_storage_count * sizeof(float)); + uint16_t *q_half_host = malloc( + (size_t)q_half_storage_count * sizeof(uint16_t)); + CHECK(input_host && input_readback && reference_host && + candidate_host && q_half_host, "host tensor allocation"); + + poison_f32(input_host, input_storage_count, 0x7fc50000u); + fill_inputs(input_host + GUARD_FLOATS); + poison_f16(q_half_host, q_half_storage_count); + + ds4_gpu_tensor *x_base = ds4_gpu_tensor_alloc( + input_storage_count * sizeof(float)); + ds4_gpu_tensor *reference_base = ds4_gpu_tensor_alloc( + output_storage_count * sizeof(float)); + ds4_gpu_tensor *candidate_base = ds4_gpu_tensor_alloc( + output_storage_count * sizeof(float)); + ds4_gpu_tensor *q_half_base = ds4_gpu_tensor_alloc( + q_half_storage_count * sizeof(uint16_t)); + CHECK(x_base && reference_base && candidate_base && q_half_base, + "Metal tensor allocation"); + + ds4_gpu_tensor *x = ds4_gpu_tensor_view( + x_base, GUARD_FLOATS * sizeof(float), + input_count * sizeof(float)); + CHECK(x != NULL, "input tensor view"); + CHECK(ds4_gpu_tensor_write( + x_base, 0, input_host, + input_storage_count * sizeof(float)) != 0, + "input upload"); + CHECK(ds4_gpu_tensor_write( + q_half_base, 0, q_half_host, + q_half_storage_count * sizeof(uint16_t)) != 0, + "q_half poison upload"); + + /* Residency must be selected before installing the synthetic model. */ + ds4_gpu_set_quality(false); + ds4_gpu_set_ssd_streaming(false); + CHECK(ds4_gpu_set_model_map(model, model_bytes) != 0, + "resident model map"); + CHECK(ds4_gpu_prepare_support_model( + support_model, support_model_bytes, 0, + support_model_bytes, 0) != 0, + "resident support model map"); + ds4_gpu_test_q4_attn_q_b_f16_cache_reset(); + + /* REQUIRE applies only at or above MIN_TOKENS. A short tail must remain + * a non-candidate rather than failing after a successful session prewarm. */ + { + CHECK(setenv(k_min_tokens, "512", 1) == 0, + "set short-tail cache minimum"); + ds4_gpu_tensor *short_out = ds4_gpu_tensor_view( + candidate_base, GUARD_FLOATS * sizeof(float), + (uint64_t)32u * OUT_DIM * sizeof(float)); + ds4_gpu_tensor *short_half = ds4_gpu_tensor_view( + q_half_base, GUARD_HALFS * sizeof(uint16_t), + (uint64_t)32u * OUT_DIM * sizeof(uint16_t)); + CHECK(short_out && short_half, "short-tail tensor views"); + CHECK(run_candidate(short_out, short_half, model, model_bytes, + x, 32u) == 0, + "below-min REQUIRE batch must remain a non-candidate"); + ds4_gpu_tensor_free(short_half); + ds4_gpu_tensor_free(short_out); + + ds4_gpu_q4_attn_q_b_f16_cache_report short_report; + ds4_gpu_test_q4_attn_q_b_f16_cache_report(&short_report); + CHECK(short_report.candidate_calls == 0u && + short_report.lookups == 0u && + short_report.fallbacks == 0u && + short_report.rejects == 0u && + short_report.build_circuit_open == 0u, + "below-min REQUIRE candidate accounting"); + CHECK(setenv(k_min_tokens, "32", 1) == 0, + "restore 32-token cache minimum"); + } + + /* A stable admission failure opens the build circuit. Raising the + * logical budget alone must not trigger repeated allocations; an + * explicit cache reset is required before builds may resume. */ + { + const uint64_t gate_output_count = (uint64_t)32u * OUT_DIM; + const uint64_t gate_output_bytes = + gate_output_count * sizeof(float); + const uint64_t gate_half_bytes = + gate_output_count * sizeof(uint16_t); + CHECK(setenv(k_cache_mb, "63", 1) == 0, + "set undersized sidecar budget"); + poison_f32(candidate_host, output_storage_count, + k_candidate_poison); + poison_f16(q_half_host, q_half_storage_count); + CHECK(ds4_gpu_tensor_write( + candidate_base, 0, candidate_host, + output_storage_count * sizeof(float)) != 0, + "budget output poison upload"); + CHECK(ds4_gpu_tensor_write( + q_half_base, 0, q_half_host, + q_half_storage_count * sizeof(uint16_t)) != 0, + "budget q_half poison upload"); + + ds4_gpu_tensor *budget_out = ds4_gpu_tensor_view( + candidate_base, GUARD_FLOATS * sizeof(float), + gate_output_bytes); + ds4_gpu_tensor *budget_half = ds4_gpu_tensor_view( + q_half_base, GUARD_HALFS * sizeof(uint16_t), + gate_half_bytes); + CHECK(budget_out && budget_half, "budget tensor views"); + CHECK(run_candidate(budget_out, budget_half, model, model_bytes, + x, 32u) == -1, + "undersized budget must fail required candidate"); + ds4_gpu_tensor_free(budget_half); + ds4_gpu_tensor_free(budget_out); + + ds4_gpu_q4_attn_q_b_f16_cache_report budget_report; + ds4_gpu_test_q4_attn_q_b_f16_cache_report(&budget_report); + CHECK(budget_report.entries == 0u && budget_report.bytes == 0u && + budget_report.builds == 0u && + budget_report.build_failures == 0u && + budget_report.rejects == 1u && + budget_report.build_circuit_open == 1u, + "undersized budget circuit accounting"); + + CHECK(setenv(k_cache_mb, "3072", 1) == 0, + "raise sidecar budget without reset"); + budget_out = ds4_gpu_tensor_view( + candidate_base, GUARD_FLOATS * sizeof(float), + gate_output_bytes); + budget_half = ds4_gpu_tensor_view( + q_half_base, GUARD_HALFS * sizeof(uint16_t), + gate_half_bytes); + CHECK(budget_out && budget_half, + "open-circuit tensor views"); + CHECK(run_candidate(budget_out, budget_half, model, model_bytes, + x, 32u) == -1, + "open circuit must suppress budget-only retry"); + ds4_gpu_tensor_free(budget_half); + ds4_gpu_tensor_free(budget_out); + ds4_gpu_test_q4_attn_q_b_f16_cache_report(&budget_report); + CHECK(budget_report.entries == 0u && budget_report.builds == 0u && + budget_report.build_failures == 0u && + budget_report.rejects == 2u && + budget_report.build_circuit_open == 1u, + "open circuit retry accounting"); + + CHECK(ds4_gpu_tensor_read( + candidate_base, 0, candidate_host, + output_storage_count * sizeof(float)) != 0, + "budget output readback"); + CHECK(ds4_gpu_tensor_read( + q_half_base, 0, q_half_host, + q_half_storage_count * sizeof(uint16_t)) != 0, + "budget q_half readback"); + CHECK(count_poison_f32_mismatches( + candidate_host, 0, output_storage_count, + k_candidate_poison) == 0, + "budget rejection touched output"); + CHECK(count_poison_f16_mismatches( + q_half_host, q_half_storage_count) == 0, + "budget rejection touched q_half"); + + CHECK(ds4_gpu_release_q4_attn_q_b_f16_sidecars() != 0, + "production lifecycle release after open circuit"); + ds4_gpu_test_q4_attn_q_b_f16_cache_report(&budget_report); + CHECK(budget_report.entries == 0u && budget_report.bytes == 0u && + budget_report.build_circuit_open == 0u, + "lifecycle release did not close empty build circuit"); + ds4_gpu_test_q4_attn_q_b_f16_cache_reset(); + CHECK(unsetenv(k_cache_mb) == 0, + "clear sidecar budget override"); + } + + /* Session creation uses the batch-prewarm API before prefill timing. + * Exercise its transactional publication independently from lazy use. */ + { + const ds4_gpu_q4_attn_q_b_f16_sidecar_desc descs[2] = { + { + .weight_offset = 0, + .weight_bytes = weight_bytes, + .in_dim = IN_DIM, + .out_dim = OUT_DIM, + .weight_type = Q4_K_TYPE, + .layer = 0, + }, + { + .weight_offset = weight_bytes, + .weight_bytes = weight_bytes, + .in_dim = IN_DIM, + .out_dim = OUT_DIM, + .weight_type = Q4_K_TYPE, + .layer = 1, + }, + }; + const ds4_gpu_q4_attn_q_b_f16_sidecar_desc support_desc = { + .weight_offset = 0, + .weight_bytes = weight_bytes, + .in_dim = IN_DIM, + .out_dim = OUT_DIM, + .weight_type = Q4_K_TYPE, + .layer = 0, + }; + uint64_t prepared_bytes = 0; + CHECK(setenv(k_cache_mb, "127", 1) == 0, + "set undersized transactional prewarm budget"); + CHECK(ds4_gpu_prepare_q4_attn_q_b_f16_sidecars( + model, model_bytes, descs, 2u, 64u, 0u, + &prepared_bytes) == -1, + "transactional prewarm budget rejection"); + ds4_gpu_q4_attn_q_b_f16_cache_report prewarm_report; + ds4_gpu_test_q4_attn_q_b_f16_cache_report(&prewarm_report); + CHECK(prepared_bytes == 0u && prewarm_report.entries == 0u && + prewarm_report.bytes == 0u && + prewarm_report.builds == 0u && + prewarm_report.build_circuit_open == 1u, + "transactional prewarm published a partial cache"); + ds4_gpu_test_q4_attn_q_b_f16_cache_reset(); + CHECK(unsetenv(k_cache_mb) == 0, + "clear transactional prewarm budget"); + + CHECK(ds4_gpu_prepare_q4_attn_q_b_f16_sidecars( + model, model_bytes, descs, 2u, 64u, 0u, + &prepared_bytes) == 1, + "transactional two-offset prewarm"); + ds4_gpu_test_q4_attn_q_b_f16_cache_report(&prewarm_report); + CHECK(prepared_bytes == 2u * f16_cache_bytes && + prewarm_report.entries == 2u && + prewarm_report.bytes == 2u * f16_cache_bytes && + prewarm_report.builds == 2u && + prewarm_report.build_circuit_open == 0u, + "two-offset prewarm publication accounting"); + + prepared_bytes = 0; + CHECK(ds4_gpu_prepare_q4_attn_q_b_f16_sidecars( + support_model, support_model_bytes, &support_desc, + 1u, 64u, 0u, &prepared_bytes) == 1, + "same-offset second-model prewarm"); + ds4_gpu_test_q4_attn_q_b_f16_cache_report(&prewarm_report); + CHECK(prepared_bytes == f16_cache_bytes && + prewarm_report.entries == 3u && + prewarm_report.bytes == 3u * f16_cache_bytes && + prewarm_report.builds == 3u && + prewarm_report.build_circuit_open == 0u, + "model-map cache-key isolation"); + const uint64_t generation_before_eviction = + ds4_gpu_q4_attn_q_b_f16_cache_generation(); + CHECK(ds4_gpu_make_room_for_q4_attn_q_b_f16_session() != 0, + "session admission cache eviction"); + CHECK(ds4_gpu_q4_attn_q_b_f16_cache_generation() != + generation_before_eviction, + "session admission did not advance cache generation"); + ds4_gpu_test_q4_attn_q_b_f16_cache_report(&prewarm_report); + CHECK(prewarm_report.entries == 0u && + prewarm_report.bytes == 0u && + prewarm_report.build_circuit_open == 0u, + "session admission retained resident sidecars"); + ds4_gpu_test_q4_attn_q_b_f16_cache_reset(); + } + + for (uint32_t case_i = 0; + case_i < sizeof(token_cases) / sizeof(token_cases[0]); + case_i++) { + const uint32_t n_tok = token_cases[case_i]; + const uint64_t output_count = (uint64_t)n_tok * OUT_DIM; + const uint64_t output_bytes = output_count * sizeof(float); + const uint64_t q_half_count = (uint64_t)n_tok * OUT_DIM; + const uint64_t q_half_bytes = q_half_count * sizeof(uint16_t); + + poison_f32(reference_host, output_storage_count, + k_reference_poison); + poison_f32(candidate_host, output_storage_count, + k_candidate_poison); + poison_f16(q_half_host, q_half_storage_count); + CHECK(ds4_gpu_tensor_write( + reference_base, 0, reference_host, + output_storage_count * sizeof(float)) != 0, + "reference poison upload"); + CHECK(ds4_gpu_tensor_write( + candidate_base, 0, candidate_host, + output_storage_count * sizeof(float)) != 0, + "candidate poison upload"); + CHECK(ds4_gpu_tensor_write( + q_half_base, 0, q_half_host, + q_half_storage_count * sizeof(uint16_t)) != 0, + "q_half poison refresh"); + + ds4_gpu_tensor *reference = ds4_gpu_tensor_view( + reference_base, GUARD_FLOATS * sizeof(float), output_bytes); + ds4_gpu_tensor *candidate = ds4_gpu_tensor_view( + candidate_base, GUARD_FLOATS * sizeof(float), output_bytes); + ds4_gpu_tensor *q_half = ds4_gpu_tensor_view( + q_half_base, GUARD_HALFS * sizeof(uint16_t), q_half_bytes); + CHECK(reference && candidate && q_half, "case tensor views"); + + CHECK(run_reference(reference, model, model_bytes, x, n_tok) == 1, + "baseline Q4 projection plus head norm/RoPE"); + CHECK(run_candidate( + candidate, q_half, model, model_bytes, x, n_tok) == 1, + "required cached-F16 candidate"); + + ds4_gpu_tensor_free(q_half); + ds4_gpu_tensor_free(candidate); + ds4_gpu_tensor_free(reference); + + CHECK(ds4_gpu_tensor_read( + reference_base, 0, reference_host, + output_storage_count * sizeof(float)) != 0, + "reference readback"); + CHECK(ds4_gpu_tensor_read( + candidate_base, 0, candidate_host, + output_storage_count * sizeof(float)) != 0, + "candidate readback"); + CHECK(ds4_gpu_tensor_read( + q_half_base, 0, q_half_host, + q_half_storage_count * sizeof(uint16_t)) != 0, + "q_half readback"); + + uint64_t first = UINT64_MAX; + const uint64_t bit_mismatches = count_bit_mismatches( + reference_host + GUARD_FLOATS, + candidate_host + GUARD_FLOATS, + output_count, &first); + const uint64_t reference_prefix = count_poison_f32_mismatches( + reference_host, 0, GUARD_FLOATS, k_reference_poison); + const uint64_t reference_suffix = count_poison_f32_mismatches( + reference_host, GUARD_FLOATS + output_count, + output_storage_count, k_reference_poison); + const uint64_t candidate_prefix = count_poison_f32_mismatches( + candidate_host, 0, GUARD_FLOATS, k_candidate_poison); + const uint64_t candidate_suffix = count_poison_f32_mismatches( + candidate_host, GUARD_FLOATS + output_count, + output_storage_count, k_candidate_poison); + const uint64_t q_half_mismatches = + count_poison_f16_mismatches( + q_half_host, q_half_storage_count); + + fprintf(stderr, + "Metal Q4 attn_q_b F16 cache N=%u bitwise=%llu " + "ref_guard=%llu/%llu candidate_guard=%llu/%llu " + "q_half=%llu\n", + n_tok, + (unsigned long long)bit_mismatches, + (unsigned long long)reference_prefix, + (unsigned long long)reference_suffix, + (unsigned long long)candidate_prefix, + (unsigned long long)candidate_suffix, + (unsigned long long)q_half_mismatches); + if (first != UINT64_MAX) { + fprintf(stderr, + " first bitwise mismatch token=%llu row=%llu\n", + (unsigned long long)(first / OUT_DIM), + (unsigned long long)(first % OUT_DIM)); + } + CHECK(bit_mismatches == 0, "candidate bitwise mismatch"); + CHECK(reference_prefix == 0 && reference_suffix == 0, + "reference output canary"); + CHECK(candidate_prefix == 0 && candidate_suffix == 0, + "candidate output canary"); + CHECK(q_half_mismatches == 0, + "Q4 candidate unexpectedly touched q_half scratch"); + + ds4_gpu_q4_attn_q_b_f16_cache_report report; + ds4_gpu_test_q4_attn_q_b_f16_cache_report(&report); + CHECK(report.entries == 1u, "one cache entry"); + CHECK(report.bytes == f16_cache_bytes, "64 MiB cache entry"); + CHECK(report.lookups == (uint64_t)case_i + 1u, + "cache lookup count"); + CHECK(report.hits == (uint64_t)case_i, + "cache hit count"); + CHECK(report.misses == 1u, "single cold cache miss"); + CHECK(report.builds == 1u, "single cache build"); + CHECK(report.build_failures == 0u, "no cache build failure"); + CHECK(report.candidate_calls == (uint64_t)case_i + 1u, + "candidate call count"); + CHECK(report.fallbacks == 0u && report.rejects == 0u, + "no correctness-case fallback"); + } + + /* Compare the raw projection before RMSNorm/RoPE so normalization cannot + * hide a uniform scale or dequantization error. */ + { + const uint32_t n_tok = 33u; + const uint64_t output_count = (uint64_t)n_tok * OUT_DIM; + const uint64_t output_bytes = output_count * sizeof(float); + poison_f32(reference_host, output_storage_count, + k_reference_poison); + poison_f32(candidate_host, output_storage_count, + k_candidate_poison); + CHECK(ds4_gpu_tensor_write( + reference_base, 0, reference_host, + output_storage_count * sizeof(float)) != 0, + "raw reference poison upload"); + CHECK(ds4_gpu_tensor_write( + candidate_base, 0, candidate_host, + output_storage_count * sizeof(float)) != 0, + "raw candidate poison upload"); + ds4_gpu_tensor *reference = ds4_gpu_tensor_view( + reference_base, GUARD_FLOATS * sizeof(float), output_bytes); + ds4_gpu_tensor *candidate = ds4_gpu_tensor_view( + candidate_base, GUARD_FLOATS * sizeof(float), output_bytes); + CHECK(reference && candidate, "raw projection tensor views"); + CHECK(ds4_gpu_matmul_quant_tensor( + reference, model, model_bytes, 0, Q4_K_TYPE, + IN_DIM, OUT_DIM, x, n_tok) == 1, + "raw native Q4 projection"); + CHECK(ds4_gpu_test_q4_attn_q_b_f16_projection_tensor( + candidate, model, model_bytes, 0, + IN_DIM, OUT_DIM, x, n_tok) == 1, + "raw cached-F16 projection"); + ds4_gpu_tensor_free(candidate); + ds4_gpu_tensor_free(reference); + CHECK(ds4_gpu_tensor_read( + reference_base, 0, reference_host, + output_storage_count * sizeof(float)) != 0, + "raw reference readback"); + CHECK(ds4_gpu_tensor_read( + candidate_base, 0, candidate_host, + output_storage_count * sizeof(float)) != 0, + "raw candidate readback"); + uint64_t first = UINT64_MAX; + CHECK(count_bit_mismatches( + reference_host + GUARD_FLOATS, + candidate_host + GUARD_FLOATS, + output_count, &first) == 0, + "raw projection bitwise mismatch"); + CHECK(count_poison_f32_mismatches( + reference_host, 0, GUARD_FLOATS, + k_reference_poison) == 0 && + count_poison_f32_mismatches( + reference_host, GUARD_FLOATS + output_count, + output_storage_count, k_reference_poison) == 0, + "raw reference output canary"); + CHECK(count_poison_f32_mismatches( + candidate_host, 0, GUARD_FLOATS, + k_candidate_poison) == 0 && + count_poison_f32_mismatches( + candidate_host, GUARD_FLOATS + output_count, + output_storage_count, k_candidate_poison) == 0, + "raw candidate output canary"); + fprintf(stderr, + "Metal Q4 attn_q_b F16 cache raw projection N=33: PASS\n"); + } + + /* Exercise the production command-batch lifecycle from a cold cache. + * The sidecar build must complete before publication even though the + * projection and norm/RoPE remain encoded in the caller's batch. A + * second batch then proves that the published entry is a real hot hit. */ + { + const uint32_t n_tok = 64u; + const uint64_t output_count = (uint64_t)n_tok * OUT_DIM; + const uint64_t output_bytes = output_count * sizeof(float); + const uint64_t q_half_bytes = + output_count * sizeof(uint16_t); + + ds4_gpu_test_q4_attn_q_b_f16_cache_reset(); + poison_f32(reference_host, output_storage_count, + k_reference_poison); + poison_f32(candidate_host, output_storage_count, + k_candidate_poison); + CHECK(ds4_gpu_tensor_write( + reference_base, 0, reference_host, + output_storage_count * sizeof(float)) != 0, + "batch reference poison upload"); + CHECK(ds4_gpu_tensor_write( + candidate_base, 0, candidate_host, + output_storage_count * sizeof(float)) != 0, + "batch candidate poison upload"); + + ds4_gpu_tensor *reference = ds4_gpu_tensor_view( + reference_base, GUARD_FLOATS * sizeof(float), output_bytes); + ds4_gpu_tensor *candidate = ds4_gpu_tensor_view( + candidate_base, GUARD_FLOATS * sizeof(float), output_bytes); + ds4_gpu_tensor *q_half = ds4_gpu_tensor_view( + q_half_base, GUARD_HALFS * sizeof(uint16_t), q_half_bytes); + CHECK(reference && candidate && q_half, + "cold batch tensor views"); + CHECK(run_reference(reference, model, model_bytes, x, n_tok) == 1, + "cold batch reference"); + CHECK(ds4_gpu_begin_commands() != 0, + "begin cold candidate batch"); + CHECK(run_candidate(candidate, q_half, model, model_bytes, + x, n_tok) == 1, + "encode cold batched candidate"); + CHECK(ds4_gpu_end_commands() != 0, + "finish cold candidate batch"); + ds4_gpu_tensor_free(q_half); + ds4_gpu_tensor_free(candidate); + ds4_gpu_tensor_free(reference); + + CHECK(ds4_gpu_tensor_read( + reference_base, 0, reference_host, + output_storage_count * sizeof(float)) != 0, + "cold batch reference readback"); + CHECK(ds4_gpu_tensor_read( + candidate_base, 0, candidate_host, + output_storage_count * sizeof(float)) != 0, + "cold batch candidate readback"); + uint64_t first = UINT64_MAX; + CHECK(count_bit_mismatches( + reference_host + GUARD_FLOATS, + candidate_host + GUARD_FLOATS, + output_count, &first) == 0, + "cold batched candidate bitwise mismatch"); + + ds4_gpu_q4_attn_q_b_f16_cache_report report; + ds4_gpu_test_q4_attn_q_b_f16_cache_report(&report); + CHECK(report.entries == 1u && report.builds == 1u && + report.misses == 1u && report.hits == 0u, + "cold batch cache publication"); + + poison_f32(candidate_host, output_storage_count, + k_candidate_poison); + CHECK(ds4_gpu_tensor_write( + candidate_base, 0, candidate_host, + output_storage_count * sizeof(float)) != 0, + "hot batch candidate poison upload"); + candidate = ds4_gpu_tensor_view( + candidate_base, GUARD_FLOATS * sizeof(float), output_bytes); + q_half = ds4_gpu_tensor_view( + q_half_base, GUARD_HALFS * sizeof(uint16_t), q_half_bytes); + CHECK(candidate && q_half, "hot batch tensor views"); + CHECK(ds4_gpu_begin_commands() != 0, + "begin hot candidate batch"); + CHECK(run_candidate(candidate, q_half, model, model_bytes, + x, n_tok) == 1, + "encode hot batched candidate"); + CHECK(ds4_gpu_end_commands() != 0, + "finish hot candidate batch"); + ds4_gpu_tensor_free(q_half); + ds4_gpu_tensor_free(candidate); + CHECK(ds4_gpu_tensor_read( + candidate_base, 0, candidate_host, + output_storage_count * sizeof(float)) != 0, + "hot batch candidate readback"); + first = UINT64_MAX; + CHECK(count_bit_mismatches( + reference_host + GUARD_FLOATS, + candidate_host + GUARD_FLOATS, + output_count, &first) == 0, + "hot batched candidate bitwise mismatch"); + ds4_gpu_test_q4_attn_q_b_f16_cache_report(&report); + CHECK(report.entries == 1u && report.builds == 1u && + report.misses == 1u && report.hits == 1u && + report.lookups == 2u, + "hot batch cache hit"); + fprintf(stderr, + "Metal Q4 attn_q_b F16 cache cold/hot command batch: PASS\n"); + } + + /* DISABLE wins over REQUIRE, must reject before cache lookup, and must + * leave the entire output allocation untouched. */ + ds4_gpu_q4_attn_q_b_f16_cache_report gate_before; + ds4_gpu_q4_attn_q_b_f16_cache_report gate_after; + ds4_gpu_test_q4_attn_q_b_f16_cache_report(&gate_before); + poison_f32(candidate_host, output_storage_count, k_candidate_poison); + CHECK(ds4_gpu_tensor_write( + candidate_base, 0, candidate_host, + output_storage_count * sizeof(float)) != 0, + "DISABLE output poison"); + CHECK(setenv(k_disable, "1", 1) == 0, "set cache disable env"); + ds4_gpu_tensor *gate_out = ds4_gpu_tensor_view( + candidate_base, GUARD_FLOATS * sizeof(float), + (uint64_t)32u * OUT_DIM * sizeof(float)); + ds4_gpu_tensor *gate_half = ds4_gpu_tensor_view( + q_half_base, GUARD_HALFS * sizeof(uint16_t), + (uint64_t)32u * OUT_DIM * sizeof(uint16_t)); + CHECK(gate_out && gate_half, "DISABLE tensor views"); + CHECK(run_candidate( + gate_out, gate_half, model, model_bytes, x, 32u) == -1, + "DISABLE must win over REQUIRE"); + ds4_gpu_tensor_free(gate_half); + ds4_gpu_tensor_free(gate_out); + CHECK(ds4_gpu_tensor_read( + candidate_base, 0, candidate_host, + output_storage_count * sizeof(float)) != 0, + "DISABLE output readback"); + CHECK(count_poison_f32_mismatches( + candidate_host, 0, output_storage_count, + k_candidate_poison) == 0, + "DISABLE touched output"); + ds4_gpu_test_q4_attn_q_b_f16_cache_report(&gate_after); + check_cache_storage_unchanged( + &gate_before, &gate_after, "DISABLE touched cache storage/state"); + CHECK(gate_after.candidate_calls == gate_before.candidate_calls + 1u && + gate_after.fallbacks == gate_before.fallbacks + 1u && + gate_after.rejects == gate_before.rejects + 1u, + "DISABLE accounting"); + CHECK(unsetenv(k_disable) == 0, "clear cache disable env"); + + /* Entering SSD mode must drop resident-only sidecars, then reject the + * optimization without rebuilding them. */ + gate_before = gate_after; + poison_f32(candidate_host, output_storage_count, k_candidate_poison); + CHECK(ds4_gpu_tensor_write( + candidate_base, 0, candidate_host, + output_storage_count * sizeof(float)) != 0, + "SSD output poison"); + ds4_gpu_set_ssd_streaming(true); + ds4_gpu_test_q4_attn_q_b_f16_cache_report(&gate_after); + CHECK(gate_after.entries == 0u && gate_after.bytes == 0u, + "enabling SSD mode retained resident sidecars"); + CHECK(gate_after.build_circuit_open == 0u, + "enabling SSD mode retained the build circuit state"); + CHECK(gate_after.candidate_calls == gate_before.candidate_calls && + gate_after.fallbacks == gate_before.fallbacks && + gate_after.rejects == gate_before.rejects, + "SSD transition reset cache accounting"); + gate_before = gate_after; + gate_out = ds4_gpu_tensor_view( + candidate_base, GUARD_FLOATS * sizeof(float), + (uint64_t)32u * OUT_DIM * sizeof(float)); + gate_half = ds4_gpu_tensor_view( + q_half_base, GUARD_HALFS * sizeof(uint16_t), + (uint64_t)32u * OUT_DIM * sizeof(uint16_t)); + CHECK(gate_out && gate_half, "SSD tensor views"); + CHECK(run_candidate( + gate_out, gate_half, model, model_bytes, x, 32u) == -1, + "SSD mode must reject required resident cache"); + ds4_gpu_tensor_free(gate_half); + ds4_gpu_tensor_free(gate_out); + CHECK(ds4_gpu_tensor_read( + candidate_base, 0, candidate_host, + output_storage_count * sizeof(float)) != 0, + "SSD output readback"); + CHECK(count_poison_f32_mismatches( + candidate_host, 0, output_storage_count, + k_candidate_poison) == 0, + "SSD rejection touched output"); + ds4_gpu_test_q4_attn_q_b_f16_cache_report(&gate_after); + CHECK(gate_after.entries == 0u && gate_after.bytes == 0u && + gate_after.build_circuit_open == 0u, + "SSD rejection rebuilt resident cache"); + CHECK(gate_after.candidate_calls == gate_before.candidate_calls + 1u && + gate_after.fallbacks == gate_before.fallbacks + 1u && + gate_after.rejects == gate_before.rejects + 1u, + "SSD rejection accounting"); + ds4_gpu_set_ssd_streaming(false); + + /* The input, including both guards, is immutable across every path. */ + CHECK(ds4_gpu_tensor_read( + x_base, 0, input_readback, + input_storage_count * sizeof(float)) != 0, + "input readback"); + CHECK(memcmp(input_host, input_readback, + (size_t)input_storage_count * sizeof(float)) == 0, + "input payload/canary modified"); + + if (getenv(k_timing) != NULL) { + const char *timing_tokens_env = getenv( + "DS4_TEST_METAL_Q4_QB_F16_CACHE_TIMING_TOKENS"); + char *timing_tokens_end = NULL; + errno = 0; + const unsigned long timing_tokens_value = timing_tokens_env + ? strtoul(timing_tokens_env, &timing_tokens_end, 10) + : 4096ul; + CHECK(!timing_tokens_env || + (timing_tokens_env[0] >= '0' && + timing_tokens_env[0] <= '9' && + errno == 0 && + timing_tokens_end != timing_tokens_env && + *timing_tokens_end == '\0'), + "invalid timing token count"); + CHECK(timing_tokens_value >= 32ul && + timing_tokens_value <= 4096ul, + "timing tokens must be in [32, 4096]"); + const uint32_t timing_tokens = (uint32_t)timing_tokens_value; + const uint64_t timing_input_count = + (uint64_t)timing_tokens * IN_DIM; + const uint64_t timing_output_count = + (uint64_t)timing_tokens * OUT_DIM; + double reference_ms[TIMING_SAMPLES]; + double candidate_ms[TIMING_SAMPLES]; + float *timing_input = malloc( + (size_t)timing_input_count * sizeof(float)); + CHECK(timing_input != NULL, "timing input host allocation"); + for (uint32_t token = 0; token < timing_tokens; token++) { + for (uint32_t col = 0; col < IN_DIM; col++) { + const uint32_t key = token * 131u + col * 17u + + ((col >> 3u) ^ (token * 29u)); + timing_input[(uint64_t)token * IN_DIM + col] = + (float)((int)(key % 129u) - 64) / 64.0f; + } + } + + /* Timing allocations are independent from the guarded correctness + * tensors. A single output is sufficient because each projection + * overwrites it completely before the in-place norm/RoPE stage. */ + ds4_gpu_tensor *timing_x = ds4_gpu_tensor_alloc( + timing_input_count * sizeof(float)); + ds4_gpu_tensor *timing_out = ds4_gpu_tensor_alloc( + timing_output_count * sizeof(float)); + ds4_gpu_tensor *timing_q_half = ds4_gpu_tensor_alloc( + timing_output_count * sizeof(uint16_t)); + CHECK(timing_x && timing_out && timing_q_half, + "timing Metal tensor allocation"); + CHECK(ds4_gpu_tensor_write( + timing_x, 0, timing_input, + timing_input_count * sizeof(float)) != 0, + "timing input upload"); + free(timing_input); + + ds4_gpu_test_q4_attn_q_b_f16_cache_reset(); + CHECK(ds4_gpu_synchronize() != 0, "pre-cold synchronize"); + const double cold_t0 = monotonic_ms(); + CHECK(run_candidate(timing_out, timing_q_half, model, model_bytes, + timing_x, timing_tokens) == 1, + "timing cold candidate"); + CHECK(ds4_gpu_synchronize() != 0, "cold synchronize"); + const double cold_ms = monotonic_ms() - cold_t0; + + CHECK(run_reference(timing_out, model, model_bytes, timing_x, + timing_tokens) == 1, + "timing reference warmup"); + CHECK(run_candidate(timing_out, timing_q_half, model, model_bytes, + timing_x, timing_tokens) == 1, + "timing candidate warmup"); + + for (uint32_t i = 0; i < TIMING_SAMPLES; i++) { + if ((i & 1u) == 0u) { + double t0 = monotonic_ms(); + CHECK(run_reference(timing_out, model, model_bytes, + timing_x, timing_tokens) == 1, + "timing reference"); + reference_ms[i] = monotonic_ms() - t0; + t0 = monotonic_ms(); + CHECK(run_candidate(timing_out, timing_q_half, model, + model_bytes, timing_x, + timing_tokens) == 1, + "timing candidate"); + candidate_ms[i] = monotonic_ms() - t0; + } else { + double t0 = monotonic_ms(); + CHECK(run_candidate(timing_out, timing_q_half, model, + model_bytes, timing_x, + timing_tokens) == 1, + "timing candidate"); + candidate_ms[i] = monotonic_ms() - t0; + t0 = monotonic_ms(); + CHECK(run_reference(timing_out, model, model_bytes, + timing_x, timing_tokens) == 1, + "timing reference"); + reference_ms[i] = monotonic_ms() - t0; + } + } + qsort(reference_ms, TIMING_SAMPLES, sizeof(double), compare_double); + qsort(candidate_ms, TIMING_SAMPLES, sizeof(double), compare_double); + const double reference_median = reference_ms[TIMING_SAMPLES / 2u]; + const double candidate_median = candidate_ms[TIMING_SAMPLES / 2u]; + fprintf(stderr, + "Metal Q4 attn_q_b F16 cache timing N=%u: " + "cold=%.3f ms reference=%.3f ms steady=%.3f ms " + "speedup=%.3fx\n", + timing_tokens, cold_ms, reference_median, candidate_median, + reference_median / candidate_median); + + /* Verify the exact timing geometry after sampling so readback and the + * second output allocation cannot perturb the measured resident path. + * Chunked reads bound host memory even for N=4096. */ + ds4_gpu_tensor *timing_reference = ds4_gpu_tensor_alloc( + timing_output_count * sizeof(float)); + CHECK(timing_reference != NULL, + "timing verification output allocation"); + CHECK(run_reference(timing_reference, model, model_bytes, timing_x, + timing_tokens) == 1, + "timing verification reference"); + CHECK(run_candidate(timing_out, timing_q_half, model, model_bytes, + timing_x, timing_tokens) == 1, + "timing verification candidate"); + + const uint64_t verify_bytes = + timing_output_count * sizeof(float); + const uint64_t verify_chunk_bytes = 4u * 1024u * 1024u; + float *verify_reference = malloc((size_t)verify_chunk_bytes); + float *verify_candidate = malloc((size_t)verify_chunk_bytes); + CHECK(verify_reference && verify_candidate, + "timing verification host chunks"); + uint64_t verify_mismatches = 0; + uint64_t verify_first = UINT64_MAX; + for (uint64_t offset = 0; offset < verify_bytes; + offset += verify_chunk_bytes) { + const uint64_t chunk_bytes = + verify_bytes - offset < verify_chunk_bytes + ? verify_bytes - offset + : verify_chunk_bytes; + CHECK(ds4_gpu_tensor_read(timing_reference, offset, + verify_reference, chunk_bytes) != 0, + "timing verification reference readback"); + CHECK(ds4_gpu_tensor_read(timing_out, offset, + verify_candidate, chunk_bytes) != 0, + "timing verification candidate readback"); + uint64_t chunk_first = UINT64_MAX; + verify_mismatches += count_bit_mismatches( + verify_reference, verify_candidate, + chunk_bytes / sizeof(float), &chunk_first); + if (verify_first == UINT64_MAX && chunk_first != UINT64_MAX) { + verify_first = offset / sizeof(float) + chunk_first; + } + } + fprintf(stderr, + "Metal Q4 attn_q_b F16 cache timing oracle N=%u " + "bitwise=%llu\n", + timing_tokens, + (unsigned long long)verify_mismatches); + if (verify_first != UINT64_MAX) { + fprintf(stderr, + " first timing mismatch token=%llu row=%llu\n", + (unsigned long long)(verify_first / OUT_DIM), + (unsigned long long)(verify_first % OUT_DIM)); + } + CHECK(verify_mismatches == 0, + "timing geometry candidate bitwise mismatch"); + free(verify_candidate); + free(verify_reference); + ds4_gpu_tensor_free(timing_reference); + + ds4_gpu_tensor_free(timing_q_half); + ds4_gpu_tensor_free(timing_out); + ds4_gpu_tensor_free(timing_x); + } + + ds4_gpu_tensor_free(x); + ds4_gpu_tensor_free(q_half_base); + ds4_gpu_tensor_free(candidate_base); + ds4_gpu_tensor_free(reference_base); + ds4_gpu_tensor_free(x_base); + ds4_gpu_cleanup(); + + free(q_half_host); + free(candidate_host); + free(reference_host); + free(input_readback); + free(input_host); + free(support_model); + free(model); + + CHECK(unsetenv(k_require) == 0, "clear cache require env at exit"); + CHECK(unsetenv(k_min_tokens) == 0, + "clear cache minimum env at exit"); + fprintf(stderr, + "Metal Q4 attn_q_b F16 cache production geometry " + "1024x32768 N=32/33/64: PASS\n"); + return 0; +} + +#else + +int main(void) { + fprintf(stderr, + "Metal Q4 attn_q_b F16 cache oracle SKIP: requires macOS\n"); + return 0; +} + +#endif From 3f59fcc4a80b4706305a90d74716f4530d1b1132 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Wed, 26 Aug 2026 13:33:34 +0200 Subject: [PATCH 117/189] cuda, rocm: accelerate resident Q4 attn_q_b prefill --- ds4.c | 105 ++-- ds4_cuda.cu | 896 ++++++++++++++++++++++++++- ds4_gpu.h | 50 +- ds4_rocm.cu | 2 + rocm/ds4_rocm_current_api_compat.cuh | 3 + rocm/ds4_rocm_norm_rope.cuh | 156 +++++ rocm/ds4_rocm_q4_qb_sidecar.cuh | 787 +++++++++++++++++++++++ rocm/ds4_rocm_runtime.cuh | 6 + scripts/environment_variables.tsv | 12 +- 9 files changed, 1925 insertions(+), 92 deletions(-) create mode 100644 rocm/ds4_rocm_q4_qb_sidecar.cuh diff --git a/ds4.c b/ds4.c index 325b67a47..0a4bc5f65 100644 --- a/ds4.c +++ b/ds4.c @@ -30267,15 +30267,10 @@ static bool metal_graph_encode_layer_attention_batch( ok = false; } bool q_b_f16_out = false; -#if defined(__APPLE__) const bool q_b_f16_weight = layer->attn_q_b->type == DS4_TENSOR_Q8_0 || (layer->attn_q_b->type == DS4_TENSOR_Q4_K && tp_rows >= 32u); -#else - const bool q_b_f16_weight = - layer->attn_q_b->type == DS4_TENSOR_Q8_0; -#endif if (ok && !q_path_debug && q_b_f16_weight) { const int q_b_f16_rc = ds4_gpu_attn_q_b_f16_head_rms_rope_tail_tensor( @@ -55405,13 +55400,11 @@ static int generate_glm_metal_argmax( return 0; } -#if defined(__APPLE__) && !defined(DS4_NO_GPU) -static int ds4_prepare_metal_q4_attn_q_b_sidecars( +static int ds4_prepare_q4_attn_q_b_sidecars( const ds4_model *model, const ds4_weights *weights, uint32_t max_batch_rows, uint64_t working_set_reserve_bytes); -#endif /* Metal generation entry point. The model runs as one local whole-graph * pipeline: graph prefill followed by graph decode steps. Streaming PRO may @@ -55508,22 +55501,21 @@ static int generate_metal_graph_raw_swa( metal_graph_free(&g); return 1; } -#if defined(__APPLE__) && !defined(DS4_NO_GPU) /* This frontend knows the real prompt width. Prepare only when that * workload can use the resident sidecars, after graph allocation is - * visible to Metal but before warmup and the measured prefill window. */ + * visible to the backend but before warmup and the measured prefill + * window. */ const uint32_t q4_sidecar_rows = (uint32_t)prompt->len < prefill_cap ? (uint32_t)prompt->len : prefill_cap; - if (ds4_prepare_metal_q4_attn_q_b_sidecars( + if (ds4_prepare_q4_attn_q_b_sidecars( model, weights, q4_sidecar_rows, 0u) < 0) { fprintf(stderr, - "ds4: required Metal Q4 attn_q_b F16 sidecar prewarm " + "ds4: required GPU Q4 attn_q_b F16 sidecar prewarm " "could not be completed\n"); metal_graph_free(&g); return 1; } -#endif const bool memory_report = getenv("DS4_METAL_MEMORY_REPORT") != NULL; if (memory_report) ds4_gpu_print_memory_report("after graph alloc"); @@ -56448,8 +56440,8 @@ struct ds4_session { uint32_t prefill_cap; int ctx_size; bool engine_session_counted; -#if defined(__APPLE__) && !defined(DS4_NO_GPU) - uint64_t metal_q4_attn_q_b_f16_sidecars_generation; +#ifndef DS4_NO_GPU + uint64_t q4_attn_q_b_f16_sidecars_generation; #endif bool checkpoint_valid; bool mtp_draft_valid; @@ -60792,17 +60784,16 @@ int ds4_engine_generate_argmax( e->directional_steering_attn_scale, e->directional_steering_ffn_scale, emit, done, emit_ud, progress, progress_ud); -#if defined(__APPLE__) /* The legacy frontend owns a temporary graph rather than a session. * Do not leave its 2.69 GiB resident sidecars pinned after return. */ if (__atomic_load_n( &e->live_session_count, __ATOMIC_RELAXED) == 0u && !ds4_gpu_release_q4_attn_q_b_f16_sidecars()) { fprintf(stderr, - "ds4: WARNING: could not release resident Metal Q4 " - "attn_q_b F16 sidecars after legacy generation\n"); + "ds4: WARNING: could not release resident %s Q4 " + "attn_q_b F16 sidecars after legacy generation\n", + ds4_backend_name(e->backend)); } -#endif return rc; #else fprintf(stderr, "ds4: %s generation requested but this build has no graph backend support\n", @@ -67064,8 +67055,8 @@ static int ds4_session_tp_register(ds4_session *s) { return 1; } -#if defined(__APPLE__) && !defined(DS4_NO_GPU) -static int ds4_prepare_metal_q4_attn_q_b_sidecars( +#ifndef DS4_NO_GPU +static int ds4_prepare_q4_attn_q_b_sidecars( const ds4_model *model, const ds4_weights *weights, uint32_t max_batch_rows, @@ -67098,17 +67089,18 @@ static int ds4_prepare_metal_q4_attn_q_b_sidecars( working_set_reserve_bytes, &prepared_bytes); } -static int ds4_session_prepare_metal_q4_attn_q_b_sidecars( +static int ds4_session_prepare_q4_attn_q_b_sidecars( ds4_session *s, uint32_t max_batch_rows, bool reserve_future_sessions) { if (!s || !s->engine || - s->engine->backend != DS4_BACKEND_METAL) { + !ds4_backend_uses_graph(s->engine->backend) || + ds4_session_is_cpu(s) || ds4_session_is_glm(s)) { return 1; } const uint64_t cache_generation = ds4_gpu_q4_attn_q_b_f16_cache_generation(); - if (s->metal_q4_attn_q_b_f16_sidecars_generation == + if (s->q4_attn_q_b_f16_sidecars_generation == cache_generation) { return 1; } @@ -67118,8 +67110,13 @@ static int ds4_session_prepare_metal_q4_attn_q_b_sidecars( const uint32_t session_count = engine_placement_session_count(e); const uint32_t live_sessions = __atomic_load_n( &e->live_session_count, __ATOMIC_RELAXED); - const uint32_t including_current = - live_sessions == UINT32_MAX ? UINT32_MAX : live_sessions + 1u; + uint32_t including_current = live_sessions; + /* Eager preparation runs before session registration, while the ordinary + * prompt-aware preflight runs after it. Count the current session only + * in the former case instead of under-reserving one future graph. */ + if (!s->engine_session_counted && including_current != UINT32_MAX) { + including_current++; + } const uint32_t remaining = session_count > including_current ? session_count - including_current : 0u; @@ -67134,16 +67131,17 @@ static int ds4_session_prepare_metal_q4_attn_q_b_sidecars( : memory.total_bytes * remaining; } - const int rc = ds4_prepare_metal_q4_attn_q_b_sidecars( + const int rc = ds4_prepare_q4_attn_q_b_sidecars( &e->model, &e->weights, max_batch_rows, future_session_bytes); if (rc < 0) { fprintf(stderr, - "ds4: required Metal Q4 attn_q_b F16 sidecar prewarm " - "could not be completed\n"); + "ds4: required %s Q4 attn_q_b F16 sidecar prewarm " + "could not be completed\n", + ds4_backend_name(e->backend)); return 0; } if (rc > 0) { - s->metal_q4_attn_q_b_f16_sidecars_generation = + s->q4_attn_q_b_f16_sidecars_generation = ds4_gpu_q4_attn_q_b_f16_cache_generation(); } return 1; @@ -67335,8 +67333,7 @@ int ds4_session_create(ds4_session **out, ds4_engine *e, int ctx_size) { e->shared_prefill_workspace_ready ? &e->shared_prefill_workspace : NULL; -#if defined(__APPLE__) && !defined(DS4_NO_GPU) - if (e->backend == DS4_BACKEND_METAL && + if (ds4_backend_uses_graph(e->backend) && __atomic_load_n( &e->live_session_count, __ATOMIC_RELAXED) != 0u) { /* The graph estimator intentionally omits several large prefill @@ -67345,12 +67342,12 @@ int ds4_session_create(ds4_session **out, ds4_engine *e, int ctx_size) { * nothing. Cache generations make existing sessions re-prewarm. */ if (!ds4_gpu_make_room_for_q4_attn_q_b_f16_session()) { fprintf(stderr, - "ds4: could not make room for the Metal session graph\n"); + "ds4: could not make room for the %s session graph\n", + ds4_backend_name(e->backend)); free(s); return 1; } } -#endif s->graph.dspark_exec_tier = e->multi_tier ? e->dspark_exec_tier : 0; if (!metal_graph_alloc_raw_cap(&s->graph, &e->weights, shape_layer, raw_cap, (uint32_t)ctx_size, s->prefill_cap, @@ -67442,14 +67439,13 @@ int ds4_session_create(ds4_session **out, ds4_engine *e, int ctx_size) { fprintf(stderr, "\n"); } } -#if defined(__APPLE__) && !defined(DS4_NO_GPU) /* Remote workers do not necessarily enter the local sync preflight before * their first mirrored/layer-slice batch, so keep eager preparation for * TP/distributed sessions. Local sessions defer until the real prompt is * known, avoiding a 2.69 GiB decode-only allocation. */ if ((e->tp.active || e->distributed.role != DS4_DISTRIBUTED_NONE) && - !ds4_session_prepare_metal_q4_attn_q_b_sidecars( + !ds4_session_prepare_q4_attn_q_b_sidecars( s, s->prefill_cap, true)) { if (__atomic_load_n(&e->live_session_count, __ATOMIC_RELAXED) == 0u) { (void)ds4_gpu_release_q4_attn_q_b_f16_sidecars(); @@ -67458,7 +67454,6 @@ int ds4_session_create(ds4_session **out, ds4_engine *e, int ctx_size) { free(s); return 1; } -#endif s->logits = xmalloc((size_t)DS4_N_VOCAB * sizeof(s->logits[0])); s->sample_probs = xmalloc((size_t)DS4_N_VOCAB * sizeof(s->sample_probs[0])); @@ -67495,12 +67490,10 @@ int ds4_session_create(ds4_session **out, ds4_engine *e, int ctx_size) { fprintf(stderr, "ds4: failed to create distributed coordinator session: %s\n", err[0] ? err : "unknown error"); -#if defined(__APPLE__) && !defined(DS4_NO_GPU) if (__atomic_load_n( &e->live_session_count, __ATOMIC_RELAXED) == 0u) { (void)ds4_gpu_release_q4_attn_q_b_f16_sidecars(); } -#endif metal_graph_free(&s->graph); free(s->logits); free(s->sample_probs); @@ -67524,29 +67517,29 @@ int ds4_session_create(ds4_session **out, ds4_engine *e, int ctx_size) { void ds4_session_free(ds4_session *s) { if (!s) return; -#if defined(__APPLE__) && !defined(DS4_NO_GPU) - bool release_metal_q4_sidecars = false; +#ifndef DS4_NO_GPU + bool release_gpu_q4_sidecars = false; #endif if (s->engine && s->engine_session_counted) { const uint32_t remaining_sessions = __atomic_sub_fetch( &s->engine->live_session_count, 1u, __ATOMIC_RELAXED); -#if defined(__APPLE__) && !defined(DS4_NO_GPU) - release_metal_q4_sidecars = +#ifndef DS4_NO_GPU + release_gpu_q4_sidecars = remaining_sessions == 0u && - s->engine->backend == DS4_BACKEND_METAL; + ds4_backend_uses_graph(s->engine->backend); #else (void)remaining_sessions; #endif s->engine_session_counted = false; } -#if defined(__APPLE__) && !defined(DS4_NO_GPU) - else if (s->engine && s->engine->backend == DS4_BACKEND_METAL && +#ifndef DS4_NO_GPU + else if (s->engine && ds4_backend_uses_graph(s->engine->backend) && __atomic_load_n( &s->engine->live_session_count, __ATOMIC_RELAXED) == 0u) { /* An eager TP/distributed prewarm may have succeeded before session * registration failed, so this uncounted session can still own the * process-global sidecars. */ - release_metal_q4_sidecars = true; + release_gpu_q4_sidecars = true; } #endif if (ds4_session_tp_leader(s) && s->tp_session_id != 0 && @@ -67571,14 +67564,13 @@ void ds4_session_free(ds4_session *s) { } #ifndef DS4_NO_GPU else { -#if defined(__APPLE__) - if (release_metal_q4_sidecars && + if (release_gpu_q4_sidecars && !ds4_gpu_release_q4_attn_q_b_f16_sidecars()) { fprintf(stderr, - "ds4: WARNING: could not release resident Metal Q4 " - "attn_q_b F16 sidecars at session teardown\n"); + "ds4: WARNING: could not release resident %s Q4 " + "attn_q_b F16 sidecars at session teardown\n", + ds4_backend_name(s->engine->backend)); } -#endif if (ds4_session_is_glm(s)) { glm_graph_free(&s->glm_graph); } else { @@ -68913,8 +68905,8 @@ int ds4_session_prepare_sync(ds4_session *s, return 1; } -#if defined(__APPLE__) && !defined(DS4_NO_GPU) - if (!s->engine || s->engine->backend != DS4_BACKEND_METAL || +#ifndef DS4_NO_GPU + if (!s->engine || !ds4_backend_uses_graph(s->engine->backend) || ds4_session_is_cpu(s) || ds4_session_is_glm(s)) { return 0; } @@ -68933,11 +68925,12 @@ int ds4_session_prepare_sync(ds4_session *s, const uint32_t max_batch_rows = metal_graph_prefill_max_chunk_rows( &s->graph, start, rows); if (max_batch_rows == 0u) return 0; - if (!ds4_session_prepare_metal_q4_attn_q_b_sidecars( + if (!ds4_session_prepare_q4_attn_q_b_sidecars( s, max_batch_rows, false)) { if (err && errlen) { snprintf(err, errlen, - "required Metal Q4 attn_q_b F16 sidecar prewarm failed"); + "required %s Q4 attn_q_b F16 sidecar prewarm failed", + ds4_backend_name(s->engine->backend)); } return 1; } diff --git a/ds4_cuda.cu b/ds4_cuda.cu index bcc6ca4e7..749c15930 100644 --- a/ds4_cuda.cu +++ b/ds4_cuda.cu @@ -476,6 +476,33 @@ struct cuda_q8_f32_range { int device_id; /* physical CUDA device id; 0 in single-tier */ }; +/* Resident DeepSeek-V4 Flash attn_q_b acceleration. Keep this cache + * separate from the older, opportunistic Q8 cache: Q4 preparation is an + * explicit all-or-nothing session preflight, while Q8 entries may be built + * lazily by unrelated projections. */ +struct cuda_q4_attn_q_b_f16_range { + const void *host_base; + uint64_t model_size; + uint64_t offset; + uint64_t weight_bytes; + uint64_t in_dim; + uint64_t out_dim; + uint64_t f16_bytes; + __half *device_ptr; + int device_id; +}; + +/* ds4_cuda.cu historically does not include ds4_gpu.h. Mirror the public + * descriptor exactly so the extern "C" preparation entry retains that ABI. */ +typedef struct ds4_gpu_q4_attn_q_b_f16_sidecar_desc { + uint64_t weight_offset; + uint64_t weight_bytes; + uint64_t in_dim; + uint64_t out_dim; + uint32_t weight_type; + uint32_t layer; +} ds4_gpu_q4_attn_q_b_f16_sidecar_desc; + /* Decode-only exact compressor layout. Each lane retains its original * contiguous 128-element accumulation chunk, while the 32 lanes' weights at * a given iteration are interleaved for one coalesced transaction. */ @@ -514,6 +541,18 @@ static std::vector g_q8_f16_ranges; static std::unordered_map g_q8_f16_by_offset; static std::vector g_q8_f32_ranges; static std::unordered_map g_q8_f32_by_offset; +static std::vector + g_q4_attn_q_b_f16_ranges; +static std::mutex g_q4_attn_q_b_f16_cache_mutex; +static std::mutex g_q4_attn_q_b_f16_build_mutex; +static uint64_t g_q4_attn_q_b_f16_bytes; +static uint64_t g_q4_attn_q_b_f16_generation = 1u; +static int g_q4_attn_q_b_f16_hard_disabled; +static int g_q4_attn_q_b_f16_dispatch_disabled; +static int g_q4_attn_q_b_f16_pending_evict; +/* Support-model residency means two independent GGUF mappings coexist. + * Keep the large q_b expansion disabled in that conservative mode. */ +static int g_q4_attn_q_b_f16_multi_model_active; static std::vector g_f16_pair_chunk32_ranges; static int g_f16_pair_chunk32_disabled_after_oom; static std::vector g_derived_ranges; @@ -994,6 +1033,14 @@ __global__ static void dequant_q8_0_to_f32_kernel( uint64_t in_dim, uint64_t out_dim, uint64_t blocks); +__global__ static void dequant_q4_K_to_f16_kernel( + __half *out, + const cuda_block_q4_K *w, + uint64_t in_dim, + uint64_t out_dim, + uint64_t blocks); + +extern "C" int ds4_gpu_release_q4_attn_q_b_f16_sidecars(void); static int cuda_aligned_iq2_enabled(void) { const char *s = getenv("DS4_CUDA_MOE_NO_IQ2_ALIGNED"); @@ -2399,6 +2446,513 @@ static int cuda_q8_f16_preload_allowed(const char *label, uint64_t in_dim, uint6 return cuda_q8_f16_cache_allowed(label, in_dim, out_dim); } +enum { + CUDA_Q4_ATTN_Q_B_TYPE = 12u, + CUDA_Q4_ATTN_Q_B_IN_DIM = 1024u, + CUDA_Q4_ATTN_Q_B_OUT_DIM = 32768u, + CUDA_Q4_ATTN_Q_B_MAX_ENTRIES = 80u, +}; + +static uint64_t cuda_q4_attn_q_b_f16_cache_limit_bytes(void) { + int present = 0; + const uint64_t parsed = cuda_parse_mib_env( + "DS4_CUDA_Q4_ATTN_Q_B_F16_CACHE_MB", &present); + return present ? parsed : 3072ull * 1048576ull; +} + +static uint32_t cuda_q4_attn_q_b_f16_min_tokens(void) { + int present = 0; + return cuda_parse_u32_env_clamped( + "DS4_CUDA_Q4_ATTN_Q_B_F16_CACHE_MIN_TOKENS", + 512u, 32u, UINT32_MAX, &present); +} + +static int cuda_q4_attn_q_b_f16_requested(void) { + return cuda_env_value_enabled( + getenv("DS4_CUDA_ENABLE_Q4_ATTN_Q_B_F16_CACHE")) || + cuda_env_value_enabled( + getenv("DS4_CUDA_REQUIRE_Q4_ATTN_Q_B_F16_CACHE")); +} + +static int cuda_q4_attn_q_b_f16_required(void) { + return cuda_env_value_enabled( + getenv("DS4_CUDA_REQUIRE_Q4_ATTN_Q_B_F16_CACHE")); +} + +static int cuda_q4_attn_q_b_f16_disabled(void) { + return cuda_env_value_enabled( + getenv("DS4_CUDA_DISABLE_Q4_ATTN_Q_B_F16_CACHE")); +} + +/* Match the existing CUDA weight-cache safety floor without coupling this + * cache to the Q8-specific reserve environment variable. The explicit + * future-session reserve supplied by ds4.c is added independently. */ +static uint64_t cuda_q4_attn_q_b_f16_reserve_bytes(uint64_t total_bytes) { + if (total_bytes >= 112ull * 1024ull * 1024ull * 1024ull) { + return 512ull * 1048576ull; + } + if (total_bytes >= 40ull * 1024ull * 1024ull * 1024ull) { + const uint64_t min_reserve = 768ull * 1048576ull; + const uint64_t pct_reserve = total_bytes / 100u; + return pct_reserve > min_reserve ? pct_reserve : min_reserve; + } + const uint64_t min_reserve = 4096ull * 1048576ull; + const uint64_t pct_reserve = total_bytes / 20u; + return pct_reserve > min_reserve ? pct_reserve : min_reserve; +} + +static int cuda_q4_attn_q_b_f16_key_equal( + const cuda_q4_attn_q_b_f16_range &entry, + const void *model_map, + uint64_t model_size, + uint64_t weight_offset, + uint64_t weight_bytes, + uint64_t in_dim, + uint64_t out_dim, + int device_id) { + return entry.host_base == model_map && + entry.model_size == model_size && + entry.offset == weight_offset && + entry.weight_bytes == weight_bytes && + entry.in_dim == in_dim && + entry.out_dim == out_dim && + entry.device_id == device_id; +} + +/* Caller holds g_q4_attn_q_b_f16_cache_mutex. */ +static const __half *cuda_q4_attn_q_b_f16_cache_lookup_locked( + const void *model_map, + uint64_t model_size, + uint64_t weight_offset, + uint64_t weight_bytes, + uint64_t in_dim, + uint64_t out_dim, + int device_id) { + for (const cuda_q4_attn_q_b_f16_range &entry : + g_q4_attn_q_b_f16_ranges) { + if (cuda_q4_attn_q_b_f16_key_equal( + entry, model_map, model_size, weight_offset, weight_bytes, + in_dim, out_dim, device_id)) { + return entry.device_ptr; + } + } + return NULL; +} + +static int cuda_q4_attn_q_b_f16_multi_model_policy_active(void) { + std::lock_guard lock(g_q4_attn_q_b_f16_cache_mutex); + return g_q4_attn_q_b_f16_multi_model_active; +} + +static void cuda_q4_attn_q_b_f16_set_multi_model_policy(int active) { + std::lock_guard lock(g_q4_attn_q_b_f16_cache_mutex); + g_q4_attn_q_b_f16_multi_model_active = active ? 1 : 0; +} + +extern "C" uint64_t ds4_gpu_q4_attn_q_b_f16_cache_generation(void) { + std::lock_guard lock(g_q4_attn_q_b_f16_cache_mutex); + return g_q4_attn_q_b_f16_generation; +} + +/* Release implementation for callers that already own the build mutex. + * Maintaining the global build->cache lock order lets make_room cover its + * check and eviction atomically without recursively taking build_mutex. */ +static int cuda_q4_attn_q_b_f16_release_under_build_lock(void) { + std::lock_guard cache_lock(g_q4_attn_q_b_f16_cache_mutex); + + int previous_device = -1; + (void)cudaGetDevice(&previous_device); + int synchronized_device = INT_MIN; + for (const cuda_q4_attn_q_b_f16_range &entry : + g_q4_attn_q_b_f16_ranges) { + if (!entry.device_ptr || entry.device_id == synchronized_device) { + continue; + } + if (cudaSetDevice(entry.device_id) != cudaSuccess || + cudaDeviceSynchronize() != cudaSuccess) { + (void)cudaGetLastError(); + g_q4_attn_q_b_f16_hard_disabled = 1; + g_q4_attn_q_b_f16_dispatch_disabled = 1; + g_q4_attn_q_b_f16_pending_evict = 1; + if (previous_device >= 0) (void)cudaSetDevice(previous_device); + return 0; + } + synchronized_device = entry.device_id; + } + + int ok = 1; + size_t keep = 0; + uint64_t remaining_bytes = 0; + const size_t original_count = g_q4_attn_q_b_f16_ranges.size(); + for (size_t i = 0; i < original_count; i++) { + const cuda_q4_attn_q_b_f16_range entry = + g_q4_attn_q_b_f16_ranges[i]; + int freed = entry.device_ptr == NULL; + if (!freed && cudaSetDevice(entry.device_id) == cudaSuccess && + cudaFree(entry.device_ptr) == cudaSuccess) { + freed = 1; + } else if (!freed) { + (void)cudaGetLastError(); + ok = 0; + } + if (!freed) { + if (keep != i) g_q4_attn_q_b_f16_ranges[keep] = entry; + keep++; + remaining_bytes += entry.f16_bytes; + } + } + if (previous_device >= 0) (void)cudaSetDevice(previous_device); + g_q4_attn_q_b_f16_ranges.resize(keep); + g_q4_attn_q_b_f16_bytes = remaining_bytes; + if (!ok) { + /* Successful frees are removed immediately, so a later retry cannot + * double-free them. Failed allocations retain their full key and + * byte metadata until the retry succeeds. */ + g_q4_attn_q_b_f16_hard_disabled = 1; + g_q4_attn_q_b_f16_dispatch_disabled = 1; + g_q4_attn_q_b_f16_pending_evict = 1; + if (keep != original_count) { + if (++g_q4_attn_q_b_f16_generation == 0u) { + g_q4_attn_q_b_f16_generation = 1u; + } + } + return 0; + } + g_q4_attn_q_b_f16_hard_disabled = 0; + g_q4_attn_q_b_f16_dispatch_disabled = 0; + g_q4_attn_q_b_f16_pending_evict = 0; + if (++g_q4_attn_q_b_f16_generation == 0u) { + g_q4_attn_q_b_f16_generation = 1u; + } + return ok; +} + +extern "C" int ds4_gpu_release_q4_attn_q_b_f16_sidecars(void) { + std::lock_guard build_lock(g_q4_attn_q_b_f16_build_mutex); + return cuda_q4_attn_q_b_f16_release_under_build_lock(); +} + +/* Called after a fast-path launch/runtime failure while cache_use_lock is + * held. Block future dispatch/build attempts, drop the use lock, then + * serialize with cold builders and synchronously evict. Re-arm the circuit + * breaker after eviction so this session cannot rebuild and repeat the same + * failure; an explicit lifecycle release/make_room resets it later. */ +static void cuda_q4_attn_q_b_f16_runtime_failure_evict( + std::unique_lock *cache_use_lock) { + g_q4_attn_q_b_f16_hard_disabled = 1; + g_q4_attn_q_b_f16_dispatch_disabled = 1; + g_q4_attn_q_b_f16_pending_evict = 1; + cache_use_lock->unlock(); + + std::lock_guard build_lock( + g_q4_attn_q_b_f16_build_mutex); + const int released = + cuda_q4_attn_q_b_f16_release_under_build_lock(); + { + std::lock_guard cache_lock( + g_q4_attn_q_b_f16_cache_mutex); + g_q4_attn_q_b_f16_hard_disabled = 1; + g_q4_attn_q_b_f16_dispatch_disabled = 1; + g_q4_attn_q_b_f16_pending_evict = released ? 0 : 1; + } + if (!released) { + fprintf(stderr, + "ds4: CUDA Q4 attn_q_b sidecar eviction deferred until " + "the next successful synchronization point\n"); + } +} + +static int cuda_q4_attn_q_b_f16_consume_pending_evict(void) { + int pending = 0; + { + std::lock_guard cache_lock( + g_q4_attn_q_b_f16_cache_mutex); + pending = g_q4_attn_q_b_f16_pending_evict; + } + if (!pending) return 1; + std::lock_guard build_lock( + g_q4_attn_q_b_f16_build_mutex); + const int released = + cuda_q4_attn_q_b_f16_release_under_build_lock(); + { + std::lock_guard cache_lock( + g_q4_attn_q_b_f16_cache_mutex); + g_q4_attn_q_b_f16_hard_disabled = 1; + g_q4_attn_q_b_f16_dispatch_disabled = 1; + g_q4_attn_q_b_f16_pending_evict = released ? 0 : 1; + } + return released; +} + +extern "C" int ds4_gpu_make_room_for_q4_attn_q_b_f16_session(void) { + std::lock_guard build_lock(g_q4_attn_q_b_f16_build_mutex); + uint64_t bytes = 0; + int needs_reset = 0; + { + std::lock_guard lock(g_q4_attn_q_b_f16_cache_mutex); + bytes = g_q4_attn_q_b_f16_bytes; + needs_reset = bytes != 0u || g_q4_attn_q_b_f16_hard_disabled || + g_q4_attn_q_b_f16_dispatch_disabled || + g_q4_attn_q_b_f16_pending_evict; + } + if (!needs_reset) return 1; + if (bytes != 0u) { + fprintf(stderr, + "ds4: evicting %.2f GiB of CUDA Q4 attn_q_b F16 sidecars " + "before allocating another live session\n", + (double)bytes / 1073741824.0); + } + return cuda_q4_attn_q_b_f16_release_under_build_lock(); +} + +extern "C" int ds4_gpu_prepare_q4_attn_q_b_f16_sidecars( + const void *model_map, + uint64_t model_size, + const ds4_gpu_q4_attn_q_b_f16_sidecar_desc *descs, + uint32_t count, + uint32_t max_prefill_rows, + uint64_t working_set_reserve_bytes, + uint64_t *prepared_bytes) { + if (prepared_bytes) *prepared_bytes = 0; + const int required = cuda_q4_attn_q_b_f16_required(); + if (!cuda_q4_attn_q_b_f16_requested() || + max_prefill_rows < cuda_q4_attn_q_b_f16_min_tokens()) { + return 0; + } + if (cuda_q4_attn_q_b_f16_disabled() || g_ssd_streaming_mode || + g_quality_mode || g_n_gpus != 1 || !g_cublas_ready || + !g_gpu[0].cublas_ready || !g_gpu[0].cublas || + cuda_q4_attn_q_b_f16_multi_model_policy_active() || + !model_map || !descs || count == 0u || + count > CUDA_Q4_ATTN_Q_B_MAX_ENTRIES) { + return required ? -1 : 0; + } + + const uint64_t expected_weight_bytes = + (uint64_t)CUDA_Q4_ATTN_Q_B_OUT_DIM * + (CUDA_Q4_ATTN_Q_B_IN_DIM / CUDA_QK_K) * + sizeof(cuda_block_q4_K); + const uint64_t one_f16_bytes = + (uint64_t)CUDA_Q4_ATTN_Q_B_IN_DIM * + CUDA_Q4_ATTN_Q_B_OUT_DIM * sizeof(__half); + for (uint32_t i = 0; i < count; i++) { + const ds4_gpu_q4_attn_q_b_f16_sidecar_desc &desc = descs[i]; + if (desc.weight_type != CUDA_Q4_ATTN_Q_B_TYPE || + desc.in_dim != CUDA_Q4_ATTN_Q_B_IN_DIM || + desc.out_dim != CUDA_Q4_ATTN_Q_B_OUT_DIM || + desc.weight_bytes != expected_weight_bytes || + desc.weight_offset > model_size || + desc.weight_bytes > model_size - desc.weight_offset) { + return required ? -1 : 0; + } + } + + std::lock_guard build_lock(g_q4_attn_q_b_f16_build_mutex); + uint32_t miss_indices[CUDA_Q4_ATTN_Q_B_MAX_ENTRIES]; + uint32_t miss_count = 0; + uint64_t missing_bytes = 0; + { + std::lock_guard cache_lock( + g_q4_attn_q_b_f16_cache_mutex); + if (g_q4_attn_q_b_f16_multi_model_active) { + return required ? -1 : 0; + } + /* One cache generation belongs to one mmap identity. Do not let a + * second live model consume the remaining budget or poison READY + * entries if its cold build fails; the session lifecycle calls + * make_room/release before changing ownership. */ + if (!g_q4_attn_q_b_f16_ranges.empty() && + (g_q4_attn_q_b_f16_ranges[0].host_base != model_map || + g_q4_attn_q_b_f16_ranges[0].model_size != model_size)) { + return required ? -1 : 0; + } + for (uint32_t i = 0; i < count; i++) { + int found = 0; + for (const cuda_q4_attn_q_b_f16_range &entry : + g_q4_attn_q_b_f16_ranges) { + if (cuda_q4_attn_q_b_f16_key_equal( + entry, model_map, model_size, + descs[i].weight_offset, descs[i].weight_bytes, + descs[i].in_dim, descs[i].out_dim, + g_gpu[0].device_id)) { + found = 1; + break; + } + } + /* Shared/aliased tensors need only one sidecar. Earlier + * descriptors in this same transaction are future READY keys, + * even though publication intentionally happens after every + * allocation and dequantization succeeds. */ + for (uint32_t j = 0; !found && j < i; j++) { + if (descs[j].weight_offset == descs[i].weight_offset && + descs[j].weight_bytes == descs[i].weight_bytes && + descs[j].in_dim == descs[i].in_dim && + descs[j].out_dim == descs[i].out_dim) { + found = 1; + } + } + if (!found) { + miss_indices[miss_count++] = i; + if (UINT64_MAX - missing_bytes < one_f16_bytes) { + return required ? -1 : 0; + } + missing_bytes += one_f16_bytes; + } + } + if (miss_count != 0u) { + if (g_q4_attn_q_b_f16_hard_disabled) { + return required ? -1 : 0; + } + const uint64_t limit = + cuda_q4_attn_q_b_f16_cache_limit_bytes(); + if (g_q4_attn_q_b_f16_ranges.size() > + CUDA_Q4_ATTN_Q_B_MAX_ENTRIES - miss_count || + g_q4_attn_q_b_f16_bytes > limit || + missing_bytes > limit - g_q4_attn_q_b_f16_bytes) { + return required ? -1 : 0; + } + } + } + if (miss_count == 0u) return 1; + + int previous_device = -1; + if (cudaGetDevice(&previous_device) != cudaSuccess) { + (void)cudaGetLastError(); + return required ? -1 : 0; + } + if (cudaSetDevice(g_gpu[0].device_id) != cudaSuccess) { + (void)cudaGetLastError(); + if (previous_device >= 0) (void)cudaSetDevice(previous_device); + return required ? -1 : 0; + } + + const cuda_block_q4_K *sources[CUDA_Q4_ATTN_Q_B_MAX_ENTRIES] = {}; + __half *sidecars[CUDA_Q4_ATTN_Q_B_MAX_ENTRIES] = {}; + int build_failed = 0; + for (uint32_t mi = 0; mi < miss_count; mi++) { + const ds4_gpu_q4_attn_q_b_f16_sidecar_desc &desc = + descs[miss_indices[mi]]; + const char *source = cuda_model_range_ptr( + model_map, desc.weight_offset, desc.weight_bytes, + "Q4 attn_q_b sidecar source"); + /* cuda_model_range_ptr is the backend's GPU-addressability contract. + * Its result may be device memory, managed/HMM memory, or a mapped + * host pointer on coherent systems; all are valid for this one-shot + * prewarm kernel. */ + if (!source) { + build_failed = 1; + break; + } + sources[mi] = reinterpret_cast(source); + } + + size_t free_bytes = 0; + size_t total_bytes = 0; + if (!build_failed && cudaMemGetInfo(&free_bytes, &total_bytes) != cudaSuccess) { + (void)cudaGetLastError(); + build_failed = 1; + } + if (!build_failed) { + const uint64_t reserve = + cuda_q4_attn_q_b_f16_reserve_bytes((uint64_t)total_bytes); + uint64_t required_free = missing_bytes; + if (UINT64_MAX - required_free < reserve) { + build_failed = 1; + } else { + required_free += reserve; + } + if (!build_failed && + UINT64_MAX - required_free < working_set_reserve_bytes) { + build_failed = 1; + } else if (!build_failed) { + required_free += working_set_reserve_bytes; + if (required_free > (uint64_t)free_bytes) { + if (getenv("DS4_CUDA_WEIGHT_CACHE_VERBOSE") != NULL || required) { + fprintf(stderr, + "ds4: CUDA Q4 attn_q_b F16 prewarm skipped: " + "need %.2f GiB sidecars + %.2f GiB reserve + " + "%.2f GiB future sessions, only %.2f GiB free\n", + (double)missing_bytes / 1073741824.0, + (double)reserve / 1073741824.0, + (double)working_set_reserve_bytes / 1073741824.0, + (double)free_bytes / 1073741824.0); + } + if (previous_device >= 0) (void)cudaSetDevice(previous_device); + return required ? -1 : 0; + } + } + } + + if (!build_failed) { + for (uint32_t mi = 0; mi < miss_count; mi++) { + if (cudaMalloc((void **)&sidecars[mi], (size_t)one_f16_bytes) != + cudaSuccess) { + (void)cudaGetLastError(); + build_failed = 1; + break; + } + } + } + if (!build_failed) { + const uint64_t chunks = + (uint64_t)CUDA_Q4_ATTN_Q_B_IN_DIM * + CUDA_Q4_ATTN_Q_B_OUT_DIM / 16u; + for (uint32_t mi = 0; mi < miss_count; mi++) { + dequant_q4_K_to_f16_kernel<<< + (unsigned)((chunks + 255u) / 256u), 256>>>( + sidecars[mi], sources[mi], + CUDA_Q4_ATTN_Q_B_IN_DIM, + CUDA_Q4_ATTN_Q_B_OUT_DIM, + CUDA_Q4_ATTN_Q_B_IN_DIM / CUDA_QK_K); + if (cudaGetLastError() != cudaSuccess) { + build_failed = 1; + break; + } + } + if (cudaDeviceSynchronize() != cudaSuccess) { + (void)cudaGetLastError(); + build_failed = 1; + } + } + + if (build_failed) { + (void)cudaDeviceSynchronize(); + for (uint32_t mi = 0; mi < miss_count; mi++) { + if (sidecars[mi]) (void)cudaFree(sidecars[mi]); + } + { + std::lock_guard cache_lock( + g_q4_attn_q_b_f16_cache_mutex); + g_q4_attn_q_b_f16_hard_disabled = 1; + } + if (previous_device >= 0) (void)cudaSetDevice(previous_device); + return required ? -1 : 0; + } + + { + std::lock_guard cache_lock( + g_q4_attn_q_b_f16_cache_mutex); + for (uint32_t mi = 0; mi < miss_count; mi++) { + const ds4_gpu_q4_attn_q_b_f16_sidecar_desc &desc = + descs[miss_indices[mi]]; + g_q4_attn_q_b_f16_ranges.push_back({ + model_map, model_size, desc.weight_offset, + desc.weight_bytes, desc.in_dim, desc.out_dim, + one_f16_bytes, sidecars[mi], g_gpu[0].device_id, + }); + } + g_q4_attn_q_b_f16_bytes += missing_bytes; + } + if (previous_device >= 0) (void)cudaSetDevice(previous_device); + if (prepared_bytes) *prepared_bytes = missing_bytes; + fprintf(stderr, + "ds4: CUDA prewarmed %u resident Q4 attn_q_b F16 sidecars " + "(%.2f GiB, min batch %u tokens)\n", + miss_count, (double)missing_bytes / 1073741824.0, + cuda_q4_attn_q_b_f16_min_tokens()); + return 1; +} + static int cuda_q8_f32_cache_allowed(const char *label, uint64_t in_dim, uint64_t out_dim) { if (g_q8_cache_suppressed) return 0; if (getenv("DS4_CUDA_NO_Q8_F32_CACHE") != NULL) return 0; @@ -6814,6 +7368,11 @@ extern "C" int ds4_gpu_init(void) { extern "C" void ds4_gpu_cleanup(void) { g_stream_expert_persistent_runtime_ready = 0; (void)cudaDeviceSynchronize(); + /* The resident q_b GEMMs may still reference cache-owned allocations. + * Retire them while the tier contexts and physical-device mapping are + * intact; release performs its own quiescence check. */ + (void)ds4_gpu_release_q4_attn_q_b_f16_sidecars(); + cuda_q4_attn_q_b_f16_set_multi_model_policy(0); cuda_decode_graphs_shutdown(); cuda_q8_fold_release_all(); g_current_logical_tier = -1; @@ -7735,20 +8294,31 @@ extern "C" int ds4_gpu_pack_slot_rows_f32_tensor( } extern "C" int ds4_gpu_begin_commands(void) { return 1; } -extern "C" int ds4_gpu_flush_commands(void) { return cuda_ok(cudaDeviceSynchronize(), "flush"); } +extern "C" int ds4_gpu_flush_commands(void) { + if (!cuda_ok(cudaDeviceSynchronize(), "flush")) return 0; + return cuda_q4_attn_q_b_f16_consume_pending_evict(); +} extern "C" int ds4_gpu_end_commands(void) { + int ok = 0; if (g_cuda_end_stream_sync) { - return cuda_ok(cudaStreamSynchronize(0), "end commands stream"); + ok = cuda_ok(cudaStreamSynchronize(0), "end commands stream"); + } else { + ok = cuda_ok(cudaDeviceSynchronize(), "end commands"); } - return cuda_ok(cudaDeviceSynchronize(), "end commands"); + return ok && cuda_q4_attn_q_b_f16_consume_pending_evict(); +} +extern "C" int ds4_gpu_synchronize(void) { + if (!cuda_ok(cudaDeviceSynchronize(), "synchronize")) return 0; + return cuda_q4_attn_q_b_f16_consume_pending_evict(); } -extern "C" int ds4_gpu_synchronize(void) { return cuda_ok(cudaDeviceSynchronize(), "synchronize"); } extern "C" int ds4_gpu_set_model_map(const void *model_map, uint64_t model_size) { if (!model_map || model_size == 0) return 0; if (g_model_host_base == model_map && g_model_registered_size == model_size) return 1; cuda_q8_fold_invalidate_all(); if (!cuda_stream_expert_storage_release(1)) return 0; + if (!ds4_gpu_release_q4_attn_q_b_f16_sidecars()) return 0; + cuda_q4_attn_q_b_f16_set_multi_model_policy(0); cuda_f16_pair_chunk32_release_all(); cuda_model_range_release_all(); cuda_q8_f16_cache_release_all(); @@ -7926,6 +8496,18 @@ extern "C" int ds4_gpu_prepare_support_model( "ds4: CUDA support model fd does not match its mmap\n"); return 0; } + /* A resident support GGUF creates a genuine multi-model working set. + * Serialize the mode transition with sidecar builders, retire any + * existing single-model sidecars, then publish the conservative policy + * before allocating the support payload. */ + { + std::lock_guard build_lock( + g_q4_attn_q_b_f16_build_mutex); + if (!cuda_q4_attn_q_b_f16_release_under_build_lock()) return 0; + std::lock_guard cache_lock( + g_q4_attn_q_b_f16_cache_mutex); + g_q4_attn_q_b_f16_multi_model_active = 1; + } /* Keep the target mmap active and install the support payload as one * host-base-keyed device range. Dynamic target SSD remaps then cannot * unregister or reinterpret the secondary GGUF. The caller has selected @@ -7952,6 +8534,8 @@ extern "C" int ds4_gpu_register_model_map_no_copy(const void *model_map, uint64_ cuda_q8_fold_invalidate_all(); if (!cuda_stream_expert_storage_release(1)) return 0; + if (!ds4_gpu_release_q4_attn_q_b_f16_sidecars()) return 0; + cuda_q4_attn_q_b_f16_set_multi_model_policy(0); cuda_f16_pair_chunk32_release_all(); cuda_model_range_release_all(); cuda_q8_f16_cache_release_all(); @@ -8684,6 +9268,12 @@ extern "C" void ds4_gpu_print_memory_report(const char *label) { } extern "C" void ds4_gpu_set_quality(bool quality) { + if (quality && !g_quality_mode && + !ds4_gpu_release_q4_attn_q_b_f16_sidecars()) { + fprintf(stderr, + "ds4: CUDA could not safely release Q4 attn_q_b F16 " + "sidecars while enabling quality mode\n"); + } g_quality_mode = quality ? 1 : 0; const cublasMath_t math_mode = (g_quality_mode || getenv("DS4_CUDA_NO_TF32") != NULL) @@ -11465,6 +12055,96 @@ __global__ static void head_rms_norm_rope_tail_kernel( } } +/* Fused epilogue for the resident Q4 attn_q_b GEMM. cuBLAS writes the + * projection once as F16; this kernel converts directly to the canonical + * F32 graph tensor while applying the same per-head RMS normalization and + * RoPE tail as head_rms_norm_rope_tail_kernel. */ +__global__ static void head_rms_norm_rope_tail_from_half_kernel( + float *out, + const __half *x, + uint32_t n_tok, + uint32_t n_head, + uint32_t head_dim, + uint32_t n_rot, + uint32_t pos0, + uint32_t n_ctx_orig, + int inverse, + float freq_base, + float freq_scale, + float ext_factor, + float attn_factor, + float beta_fast, + float beta_slow, + float eps) { + const uint32_t row = blockIdx.x; + if (row >= n_tok * n_head) return; + const uint32_t t = row / n_head; + const __half *xr = x + (uint64_t)row * head_dim; + float *orow = out + (uint64_t)row * head_dim; + float sum = 0.0f; + for (uint32_t i = threadIdx.x; i < head_dim; i += blockDim.x) { + const float v = __half2float(xr[i]); + sum += v * v; + } + __shared__ float partial[256]; + partial[threadIdx.x] = sum; + __syncthreads(); + for (uint32_t stride = blockDim.x >> 1; stride > 0; stride >>= 1) { + if (threadIdx.x < stride) { + partial[threadIdx.x] += partial[threadIdx.x + stride]; + } + __syncthreads(); + } + + const float scale = rsqrtf(partial[0] / (float)head_dim + eps); + const uint32_t n_nope = head_dim - n_rot; + for (uint32_t i = threadIdx.x; i < n_nope; i += blockDim.x) { + orow[i] = __half2float(xr[i]) * scale; + } + + float corr0 = 0.0f, corr1 = 0.0f; + if (ext_factor != 0.0f) { + const float denom = 2.0f * logf(freq_base); + corr0 = floorf((float)n_rot * + logf((float)n_ctx_orig / + (beta_fast * 2.0f * (float)M_PI)) / + denom); + corr1 = ceilf((float)n_rot * + logf((float)n_ctx_orig / + (beta_slow * 2.0f * (float)M_PI)) / + denom); + corr0 = fmaxf(0.0f, corr0); + corr1 = fminf((float)(n_rot - 1u), corr1); + } + const __half *tail = xr + n_nope; + float *out_tail = orow + n_nope; + for (uint32_t pair = threadIdx.x; + pair < n_rot / 2u; + pair += blockDim.x) { + const uint32_t i = pair * 2u; + const float theta_extrap = + (float)(pos0 + t) * + powf(freq_base, -((float)i) / (float)n_rot); + const float theta_interp = freq_scale * theta_extrap; + float theta = theta_interp; + float mscale = attn_factor; + if (ext_factor != 0.0f) { + const float ramp_mix = + rope_yarn_ramp_dev(corr0, corr1, (int)i) * ext_factor; + theta = theta_interp * (1.0f - ramp_mix) + + theta_extrap * ramp_mix; + mscale *= 1.0f + 0.1f * logf(1.0f / freq_scale); + } + const float c = cosf(theta) * mscale; + float s = sinf(theta) * mscale; + if (inverse) s = -s; + const float x0 = __half2float(tail[i]) * scale; + const float x1 = __half2float(tail[i + 1u]) * scale; + out_tail[i] = x0 * c - x1 * s; + out_tail[i + 1u] = x0 * s + x1 * c; + } +} + __device__ static float rope_yarn_ramp_dev(float low, float high, int i0) { float y = ((float)(i0 / 2) - low) / fmaxf(0.001f, high - low); return 1.0f - fminf(1.0f, fmaxf(0.0f, y)); @@ -25024,6 +25704,48 @@ __device__ static void dev_q4_K_get_scale_min( } } +/* Expand one contiguous 16-value chunk per thread. This mirrors the Q4_K + * production dequantization algebra and performs exactly one float-to-half + * rounding when publishing the resident matrix. The source pointer may be + * device, managed/HMM, or CUDA-mapped host memory. */ +__global__ static void dequant_q4_K_to_f16_kernel( + __half *out, + const cuda_block_q4_K *w, + uint64_t in_dim, + uint64_t out_dim, + uint64_t blocks) { + const uint64_t chunk = + (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + const uint64_t chunks_per_row = in_dim / 16u; + const uint64_t total_chunks = out_dim * chunks_per_row; + if (chunk >= total_chunks) return; + + const uint64_t row = chunk / chunks_per_row; + const uint64_t col0 = (chunk - row * chunks_per_row) * 16u; + const uint64_t block_in_row = col0 / CUDA_QK_K; + const uint32_t within0 = (uint32_t)(col0 % CUDA_QK_K); + const cuda_block_q4_K *block = w + row * blocks + block_in_row; + const float d = dev_f16_to_f32(block->d); + const float dmin = dev_f16_to_f32(block->dmin); + +#pragma unroll + for (uint32_t k = 0; k < 16u; k++) { + const uint32_t within = within0 + k; + const uint32_t group = within >> 5u; + uint8_t scale_code, min_code; + dev_q4_K_get_scale_min( + group, block->scales, &scale_code, &min_code); + const uint8_t packed = + block->qs[(group >> 1u) * 32u + (within & 31u)]; + const uint8_t q = (group & 1u) ? (packed >> 4u) + : (packed & 15u); + const float value = + (d * (float)scale_code) * (float)q - + dmin * (float)min_code; + out[row * in_dim + col0 + k] = __float2half_rn(value); + } +} + __device__ __forceinline__ static int32_t dev_dot_q4_32(const uint8_t *qs, const int8_t *q8, int shift) { int32_t sum = 0; #pragma unroll @@ -41933,6 +42655,12 @@ extern "C" void ds4_gpu_set_glm_model(bool enabled) { } extern "C" void ds4_gpu_set_ssd_streaming(bool enabled) { + if (enabled && !g_ssd_streaming_mode && + !ds4_gpu_release_q4_attn_q_b_f16_sidecars()) { + fprintf(stderr, + "ds4: CUDA could not safely release Q4 attn_q_b F16 " + "sidecars while enabling SSD streaming\n"); + } cuda_stream_selected_writer_guard writer; g_ssd_streaming_mode = enabled ? 1 : 0; if (!g_ssd_streaming_mode) { @@ -42150,13 +42878,159 @@ extern "C" int ds4_gpu_attn_q_b_f16_head_rms_rope_tail_tensor( uint32_t n_rot, uint32_t pos0, uint32_t n_ctx_orig, bool inverse, float freq_base, float freq_scale, float ext_factor, float attn_factor, float beta_fast, float beta_slow, float eps) { - (void)out; (void)q_half; (void)model_map; (void)model_size; - (void)weight_offset; (void)weight_type; (void)in_dim; (void)out_dim; (void)x; - (void)n_tok; (void)n_head; (void)head_dim; (void)n_rot; (void)pos0; - (void)n_ctx_orig; (void)inverse; (void)freq_base; (void)freq_scale; - (void)ext_factor; (void)attn_factor; (void)beta_fast; (void)beta_slow; - (void)eps; - return 0; + /* The pre-existing CUDA Q8 specialization was a stub. Preserve that + * fallback behavior and arm only the explicit resident-Q4 experiment. */ + if (weight_type != CUDA_Q4_ATTN_Q_B_TYPE) return 0; + if (n_tok < 32u || + n_tok < cuda_q4_attn_q_b_f16_min_tokens() || + !cuda_q4_attn_q_b_f16_requested()) { + return 0; + } + const int required = cuda_q4_attn_q_b_f16_required(); + const int fallback = required ? -1 : 0; + + if (cuda_q4_attn_q_b_f16_disabled() || g_ssd_streaming_mode || + g_quality_mode || g_n_gpus != 1 || !g_cublas_ready || + !g_gpu[0].cublas_ready || !g_gpu[0].cublas || + g_decode_graph_capturing || !out || !q_half || !x || !model_map || + !out->ptr || !q_half->ptr || !x->ptr || n_head == 0u || + head_dim == 0u || in_dim != CUDA_Q4_ATTN_Q_B_IN_DIM || + out_dim != CUDA_Q4_ATTN_Q_B_OUT_DIM || + out_dim != (uint64_t)n_head * head_dim || n_rot > head_dim || + (n_rot & 1u) != 0u || + ds4_tensor_device_idx(out) != 0 || + ds4_tensor_device_idx(q_half) != 0 || + ds4_tensor_device_idx(x) != 0 || + n_tok > (uint32_t)INT_MAX || + (uint64_t)n_tok * in_dim > (uint64_t)UINT32_MAX * 256u || + (uint64_t)n_tok * n_head > UINT32_MAX || + pos0 > UINT32_MAX - (n_tok - 1u)) { + return fallback; + } + + const uint64_t x_bytes = + (uint64_t)n_tok * in_dim * sizeof(float); + const uint64_t out_bytes = + (uint64_t)n_tok * out_dim * sizeof(float); + const uint64_t q_half_bytes = + (uint64_t)n_tok * out_dim * sizeof(__half); + if (x->bytes < x_bytes || out->bytes < out_bytes || + q_half->bytes < q_half_bytes) { + return fallback; + } + + const uint64_t blocks = in_dim / CUDA_QK_K; + const uint64_t row_bytes = blocks * sizeof(cuda_block_q4_K); + if (out_dim > UINT64_MAX / row_bytes || weight_offset > model_size) { + return fallback; + } + const uint64_t weight_bytes = out_dim * row_bytes; + if (weight_bytes > model_size - weight_offset) return fallback; + + /* Keep the cache lock until the final consumer is enqueued. A lifecycle + * release can then acquire the lock, synchronize all work queued before + * it, and free the sidecars without a lookup/free/launch race. */ + std::unique_lock cache_use_lock( + g_q4_attn_q_b_f16_cache_mutex); + if (g_q4_attn_q_b_f16_dispatch_disabled || + g_q4_attn_q_b_f16_multi_model_active) { + return fallback; + } + const __half *w_f16 = cuda_q4_attn_q_b_f16_cache_lookup_locked( + model_map, model_size, weight_offset, weight_bytes, + in_dim, out_dim, g_gpu[0].device_id); + if (!w_f16) return fallback; + + int previous_device = -1; + if (cudaGetDevice(&previous_device) != cudaSuccess) { + (void)cudaGetLastError(); + return fallback; + } + if (cudaSetDevice(g_gpu[0].device_id) != cudaSuccess) { + (void)cudaGetLastError(); + if (previous_device >= 0) (void)cudaSetDevice(previous_device); + return fallback; + } + + const cudaStream_t stream = cuda_decode_stream(); + const cublasHandle_t handle = cuda_cublas_for_tier(0); + cudaStream_t handle_stream = NULL; + if (cublasGetStream(handle, &handle_stream) != CUBLAS_STATUS_SUCCESS || + handle_stream != stream) { + /* Never mutate the shared handle here. A non-default stream means + * another subsystem owns its ordering; the native Q4 fallback is + * safer than racing the activation conversion. */ + if (previous_device >= 0) (void)cudaSetDevice(previous_device); + return fallback; + } + + const uint64_t xh_count = (uint64_t)n_tok * in_dim; + __half *xh = (__half *)cuda_tmp_alloc_on( + 0, xh_count * sizeof(__half), + "Q4 attn_q_b cached F16 activations"); + if (!xh) { + cuda_q4_attn_q_b_f16_runtime_failure_evict(&cache_use_lock); + if (previous_device >= 0) (void)cudaSetDevice(previous_device); + return fallback; + } + + /* Prefill is deliberately excluded from decode-graph capture above, so + * both kernels and the tier-0 cuBLAS handle use the legacy default + * stream. Avoid retargeting the shared handle on this hot per-layer + * path, which could race other launchers and adds measurable overhead. */ + f32_to_f16_kernel<<< + (unsigned)((xh_count + 255u) / 256u), 256, 0, stream>>>( + xh, (const float *)x->ptr, xh_count); + cudaError_t launch_err = cudaGetLastError(); + if (launch_err != cudaSuccess) { + fprintf(stderr, + "ds4: CUDA Q4 attn_q_b activation conversion failed: %s\n", + cudaGetErrorString(launch_err)); + cuda_q4_attn_q_b_f16_runtime_failure_evict(&cache_use_lock); + if (previous_device >= 0) (void)cudaSetDevice(previous_device); + return fallback; + } + + const float alpha = 1.0f; + const float beta = 0.0f; + const cublasStatus_t status = cublasGemmEx( + handle, + CUBLAS_OP_T, CUBLAS_OP_N, + (int)out_dim, (int)n_tok, (int)in_dim, + &alpha, + w_f16, CUDA_R_16F, (int)in_dim, + xh, CUDA_R_16F, (int)in_dim, + &beta, + q_half->ptr, CUDA_R_16F, (int)out_dim, + CUBLAS_COMPUTE_32F, + CUBLAS_GEMM_DEFAULT); + if (status != CUBLAS_STATUS_SUCCESS) { + fprintf(stderr, + "ds4: CUDA Q4 attn_q_b cached F16 GEMM failed: status %d\n", + (int)status); + cuda_q4_attn_q_b_f16_runtime_failure_evict(&cache_use_lock); + if (previous_device >= 0) (void)cudaSetDevice(previous_device); + return fallback; + } + + head_rms_norm_rope_tail_from_half_kernel<<< + n_tok * n_head, 256, 0, stream>>>( + (float *)out->ptr, (const __half *)q_half->ptr, + n_tok, n_head, head_dim, n_rot, pos0, n_ctx_orig, + inverse ? 1 : 0, freq_base, freq_scale, ext_factor, + attn_factor, beta_fast, beta_slow, eps); + launch_err = cudaGetLastError(); + if (launch_err != cudaSuccess) { + fprintf(stderr, + "ds4: CUDA Q4 attn_q_b F16 RMS/RoPE epilogue failed: %s\n", + cudaGetErrorString(launch_err)); + cuda_q4_attn_q_b_f16_runtime_failure_evict(&cache_use_lock); + if (previous_device >= 0) (void)cudaSetDevice(previous_device); + return fallback; + } + + if (previous_device >= 0) (void)cudaSetDevice(previous_device); + return 1; } extern "C" int ds4_gpu_attention_prefill_raw_heads_range_tensor( diff --git a/ds4_gpu.h b/ds4_gpu.h index 0c6231ccc..b8707a652 100644 --- a/ds4_gpu.h +++ b/ds4_gpu.h @@ -187,13 +187,36 @@ void ds4_gpu_set_quality(bool quality); void ds4_gpu_set_glm_model(bool enabled); void ds4_gpu_set_ssd_streaming(bool enabled); void ds4_gpu_set_glm_streaming_prefill_full_layer(bool enabled); -#ifdef __APPLE__ -/* Release resident Q4 attn_q_b F16 sidecars at a quiescent lifecycle point. - * Returns zero only when pending Metal work could not be synchronized safely. */ + +typedef struct ds4_gpu_q4_attn_q_b_f16_sidecar_desc { + uint64_t weight_offset; + uint64_t weight_bytes; + uint64_t in_dim; + uint64_t out_dim; + uint32_t weight_type; + uint32_t layer; +} ds4_gpu_q4_attn_q_b_f16_sidecar_desc; + +/* Backend-neutral lifecycle for optional resident Q4_K attn_q_b F16 + * sidecars. Each graph backend owns its storage and admission policy. + * Prepare returns 1 when all descriptors are READY, 0 for a policy/safety + * skip, and -1 when strict mode requires an unavailable specialization. */ +int ds4_gpu_prepare_q4_attn_q_b_f16_sidecars( + const void *model_map, + uint64_t model_size, + const ds4_gpu_q4_attn_q_b_f16_sidecar_desc *descs, + uint32_t count, + uint32_t max_prefill_rows, + uint64_t working_set_reserve_bytes, + uint64_t *prepared_bytes); +/* Release sidecars at a quiescent backend lifecycle point. Returns zero + * only when pending GPU work could not be synchronized safely. */ int ds4_gpu_release_q4_attn_q_b_f16_sidecars(void); uint64_t ds4_gpu_q4_attn_q_b_f16_cache_generation(void); /* Evict resident sidecars before adding a graph to a live-session set. */ int ds4_gpu_make_room_for_q4_attn_q_b_f16_session(void); + +#ifdef __APPLE__ int ds4_gpu_device_is_pre_m5_apple_silicon(void); int ds4_gpu_device_is_m5_apple_silicon(void); int ds4_gpu_set_decode_pipeline_fast_lookup(int enabled); @@ -293,27 +316,6 @@ int ds4_gpu_test_q4_attn_q_b_f16_working_set_policy( uint64_t recommended, uint64_t allocated, uint64_t additional); - -typedef struct ds4_gpu_q4_attn_q_b_f16_sidecar_desc { - uint64_t weight_offset; - uint64_t weight_bytes; - uint64_t in_dim; - uint64_t out_dim; - uint32_t weight_type; - uint32_t layer; -} ds4_gpu_q4_attn_q_b_f16_sidecar_desc; - -/* Prepare every missing resident Q4_K attn_q_b sidecar transactionally. - * Returns 1 when all descriptors are READY, 0 for a policy/safety skip, and - * -1 when strict mode requires a specialization that cannot be prepared. */ -int ds4_gpu_prepare_q4_attn_q_b_f16_sidecars( - const void *model_map, - uint64_t model_size, - const ds4_gpu_q4_attn_q_b_f16_sidecar_desc *descs, - uint32_t count, - uint32_t max_prefill_rows, - uint64_t working_set_reserve_bytes, - uint64_t *prepared_bytes); typedef struct ds4_gpu_stream_test_stats { uint64_t tensor_live_bytes; uint64_t transient_references; diff --git a/ds4_rocm.cu b/ds4_rocm.cu index 996776611..652b43c30 100644 --- a/ds4_rocm.cu +++ b/ds4_rocm.cu @@ -131,6 +131,8 @@ extern "C" int ds4_gpu_dspark_gfx1151_fast_path(void) { return ds4_rocm_is_gfx1151(); } +#include "rocm/ds4_rocm_q4_qb_sidecar.cuh" + #include "rocm/ds4_rocm_q8.cuh" #include "rocm/ds4_rocm_norm_rope.cuh" diff --git a/rocm/ds4_rocm_current_api_compat.cuh b/rocm/ds4_rocm_current_api_compat.cuh index d24668ce7..5cf63a986 100644 --- a/rocm/ds4_rocm_current_api_compat.cuh +++ b/rocm/ds4_rocm_current_api_compat.cuh @@ -117,6 +117,9 @@ extern "C" int ds4_gpu_preload_q4_expert_tables( } extern "C" void ds4_gpu_set_ssd_streaming(bool enabled) { + if (enabled && !g_ssd_streaming_mode) { + (void)ds4_gpu_release_q4_attn_q_b_f16_sidecars(); + } g_ssd_streaming_mode = enabled ? 1 : 0; cuda_model_range_release_all(); cuda_q8_f16_cache_release_all(); diff --git a/rocm/ds4_rocm_norm_rope.cuh b/rocm/ds4_rocm_norm_rope.cuh index 87fdc53f7..057255e5c 100644 --- a/rocm/ds4_rocm_norm_rope.cuh +++ b/rocm/ds4_rocm_norm_rope.cuh @@ -529,6 +529,155 @@ extern "C" int ds4_gpu_head_rms_norm_rope_tail_tensor(ds4_gpu_tensor *x, uint32_ head_rms_norm_rope_tail_kernel<<<(uint32_t)rows64, 256>>>((float *)x->ptr, n_tok, n_head, head_dim, n_rot, pos0, n_ctx_orig, inverse ? 1 : 0, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow, eps); return cuda_ok(cudaGetLastError(), "head_rms_norm_rope_tail launch"); } + +static int rocm_q4_attn_q_b_f16_head_rms_rope_tail_tensor( + ds4_gpu_tensor *out, + ds4_gpu_tensor *q_half, + const void *model_map, + uint64_t model_size, + uint64_t weight_offset, + uint64_t in_dim, + uint64_t out_dim, + const ds4_gpu_tensor *x, + uint32_t n_tok, + uint32_t n_head, + uint32_t head_dim, + uint32_t n_rot, + uint32_t pos0, + uint32_t n_ctx_orig, + bool inverse, + float freq_base, + float freq_scale, + float ext_factor, + float attn_factor, + float beta_fast, + float beta_slow, + float eps) { + /* Decode and tiny/final chunks retain the native Q4_K path without even + * taking the cache mutex. REQUIRE applies only to configured candidates. */ + if (n_tok < 32u) return 0; + const int required = rocm_q4_attn_q_b_f16_required(); + if (!rocm_q4_attn_q_b_f16_enabled() && !required) return 0; + if ((uint64_t)n_tok < rocm_q4_attn_q_b_f16_min_tokens()) return 0; + rocm_q4_attn_q_b_f16_note_candidate(); + + uint64_t x_elems = 0; + uint64_t out_elems = 0; + uint64_t x_bytes = 0; + uint64_t out_bytes = 0; + uint64_t head_rows = 0; + if (!rocm_q4_attn_q_b_f16_policy_allowed() || + rocm_q4_attn_q_b_f16_circuit_open() || + !g_cublas_ready || !out || !q_half || !x || !model_map || + model_map != g_model_host_base || + model_size != g_model_registered_size || + in_dim != DS4_ROCM_Q4_ATTN_Q_B_IN_DIM || + out_dim != DS4_ROCM_Q4_ATTN_Q_B_OUT_DIM || + n_head == 0u || head_dim == 0u || + out_dim != (uint64_t)n_head * head_dim || + n_rot > head_dim || (n_rot & 1u) != 0u || + n_tok > (uint32_t)INT_MAX || + pos0 > (uint32_t)INT_MAX - n_tok || + !cuda_u64_mul_checked(n_tok, in_dim, &x_elems) || + !cuda_u64_mul_checked(n_tok, out_dim, &out_elems) || + !cuda_u64_mul_checked(n_tok, n_head, &head_rows) || + head_rows > UINT32_MAX || + (x_elems + 255u) / 256u > UINT32_MAX || + !cuda_u64_mul_checked(x_elems, sizeof(float), &x_bytes) || + !cuda_u64_mul_checked(out_elems, sizeof(float), &out_bytes) || + x->bytes < x_bytes || out->bytes < out_bytes || + q_half->bytes < out_elems * sizeof(__half)) { + return rocm_q4_attn_q_b_f16_fallback(required, 1, 0); + } + + const uint64_t blocks_per_row = in_dim / CUDA_QK_K; + uint64_t row_bytes = 0; + uint64_t weight_bytes = 0; + if (!cuda_u64_mul_checked(blocks_per_row, + sizeof(cuda_block_q4_K), &row_bytes) || + !cuda_u64_mul_checked(out_dim, row_bytes, &weight_bytes) || + !cuda_model_range_fits(model_size, weight_offset, weight_bytes)) { + return rocm_q4_attn_q_b_f16_fallback(required, 1, 0); + } + + const __half *w_f16 = rocm_q4_attn_q_b_f16_acquire( + model_map, model_size, weight_offset, weight_bytes, in_dim, out_dim, + DS4_ROCM_Q4_K_TYPE); + if (!w_f16) { + return rocm_q4_attn_q_b_f16_fallback(required, 0, 0); + } + + /* Keep the cache mutex through GEMM submission. A concurrent lifecycle + * release cannot synchronize and free the sidecar between lookup and the + * first command that references it. */ + __half *xh = (__half *)cuda_tmp_alloc( + x_elems * sizeof(__half), "Q4 attn q_b F16 activations"); + if (!xh) { + rocm_q4_attn_q_b_f16_release_acquired(); + return rocm_q4_attn_q_b_f16_fallback(required, 0, 1); + } + f32_to_f16_kernel<<<(x_elems + 255u) / 256u, 256>>>( + xh, (const float *)x->ptr, x_elems); + cudaError_t launch_err = cudaGetLastError(); + if (launch_err != cudaSuccess) { + fprintf(stderr, + DS4_GPU_LOG_PREFIX + "Q4 attn q_b F16 activation conversion failed: %s\n", + cudaGetErrorString(launch_err)); + (void)cudaGetLastError(); + rocm_q4_attn_q_b_f16_release_acquired(); + return rocm_q4_attn_q_b_f16_fallback(required, 0, 1); + } + + const float alpha = 1.0f; + const float beta = 0.0f; + const cublasStatus_t st = cublasGemmEx( + g_cublas, + CUBLAS_OP_T, + CUBLAS_OP_N, + (int)out_dim, + (int)n_tok, + (int)in_dim, + &alpha, + w_f16, + CUDA_R_16F, + (int)in_dim, + xh, + CUDA_R_16F, + (int)in_dim, + &beta, + q_half->ptr, + CUDA_R_16F, + (int)out_dim, + CUBLAS_COMPUTE_32F, + CUBLAS_GEMM_DEFAULT); + rocm_q4_attn_q_b_f16_release_acquired(); + if (st != CUBLAS_STATUS_SUCCESS) { + fprintf(stderr, + DS4_GPU_LOG_PREFIX + DS4_GPU_BLAS_NAME + " cached Q4 attn q_b F16-out matmul failed: status %d\n", + (int)st); + return rocm_q4_attn_q_b_f16_fallback(required, 0, 1); + } + + head_rms_norm_rope_tail_from_half_kernel<<<(uint32_t)head_rows, 256>>>( + (float *)out->ptr, (const __half *)q_half->ptr, + n_tok, n_head, head_dim, n_rot, pos0, n_ctx_orig, + inverse ? 1 : 0, freq_base, freq_scale, ext_factor, attn_factor, + beta_fast, beta_slow, eps); + launch_err = cudaGetLastError(); + if (launch_err != cudaSuccess) { + fprintf(stderr, + DS4_GPU_LOG_PREFIX + "cached Q4 attn q_b head RMS/RoPE launch failed: %s\n", + cudaGetErrorString(launch_err)); + (void)cudaGetLastError(); + return rocm_q4_attn_q_b_f16_fallback(required, 0, 1); + } + return 1; +} + extern "C" int ds4_gpu_attn_q_b_f16_head_rms_rope_tail_tensor( ds4_gpu_tensor *out, ds4_gpu_tensor *q_half, @@ -553,6 +702,13 @@ extern "C" int ds4_gpu_attn_q_b_f16_head_rms_rope_tail_tensor( float beta_fast, float beta_slow, float eps) { + if (weight_type == DS4_ROCM_Q4_K_TYPE) { + return rocm_q4_attn_q_b_f16_head_rms_rope_tail_tensor( + out, q_half, model_map, model_size, weight_offset, + in_dim, out_dim, x, n_tok, n_head, head_dim, n_rot, pos0, + n_ctx_orig, inverse, freq_base, freq_scale, ext_factor, + attn_factor, beta_fast, beta_slow, eps); + } if (weight_type != 8u || !g_cublas_ready || !out || !q_half || !x || !model_map || n_tok == 0 || n_rot > head_dim || (n_rot & 1u) || out_dim != (uint64_t)n_head * head_dim || x->bytes < (uint64_t)n_tok * in_dim * sizeof(float) || diff --git a/rocm/ds4_rocm_q4_qb_sidecar.cuh b/rocm/ds4_rocm_q4_qb_sidecar.cuh new file mode 100644 index 000000000..d0ec793ae --- /dev/null +++ b/rocm/ds4_rocm_q4_qb_sidecar.cuh @@ -0,0 +1,787 @@ +// Experimental resident Q4_K attn_q_b -> F16 sidecars for ROCm prefill. +// +// The native Q4_K/Q8_K TILE8 path remains the unconditional fallback. This +// cache is deliberately resident-only and opt-in: expanding every production +// attn_q_b matrix costs 64 MiB per layer, so admission happens once, before +// prefill, with both an explicit cache budget and device-memory headroom. + +enum { + DS4_ROCM_Q4_ATTN_Q_B_F16_CACHE_MAX_ENTRIES = 80u, + DS4_ROCM_Q4_K_TYPE = 12u, + DS4_ROCM_Q4_ATTN_Q_B_IN_DIM = 1024u, + DS4_ROCM_Q4_ATTN_Q_B_OUT_DIM = 32768u, +}; + +struct rocm_q4_attn_q_b_f16_cache_entry { + const void *model_map; + uint64_t model_size; + uint64_t weight_offset; + uint64_t weight_bytes; + uint64_t in_dim; + uint64_t out_dim; + uint32_t weight_type; + __half *device_ptr; + uint64_t f16_bytes; + int valid; +}; + +struct rocm_q4_attn_q_b_f16_arena { + __half *device_ptr; + uint64_t bytes; +}; + +static rocm_q4_attn_q_b_f16_cache_entry + g_rocm_q4_attn_q_b_f16_entries[ + DS4_ROCM_Q4_ATTN_Q_B_F16_CACHE_MAX_ENTRIES]; +static rocm_q4_attn_q_b_f16_arena + g_rocm_q4_attn_q_b_f16_arenas[ + DS4_ROCM_Q4_ATTN_Q_B_F16_CACHE_MAX_ENTRIES]; +static uint32_t g_rocm_q4_attn_q_b_f16_entry_count; +static uint32_t g_rocm_q4_attn_q_b_f16_arena_count; +static uint64_t g_rocm_q4_attn_q_b_f16_bytes; +static uint64_t g_rocm_q4_attn_q_b_f16_generation = 1u; +static uint64_t g_rocm_q4_attn_q_b_f16_lookups; +static uint64_t g_rocm_q4_attn_q_b_f16_hits; +static uint64_t g_rocm_q4_attn_q_b_f16_misses; +static uint64_t g_rocm_q4_attn_q_b_f16_builds; +static uint64_t g_rocm_q4_attn_q_b_f16_build_failures; +static uint64_t g_rocm_q4_attn_q_b_f16_candidate_calls; +static uint64_t g_rocm_q4_attn_q_b_f16_fallbacks; +static uint64_t g_rocm_q4_attn_q_b_f16_rejects; +static int g_rocm_q4_attn_q_b_f16_hard_failure; +static int g_rocm_q4_attn_q_b_f16_pending_evict; +static pthread_mutex_t g_rocm_q4_attn_q_b_f16_cache_mu = + PTHREAD_MUTEX_INITIALIZER; +static pthread_mutex_t g_rocm_q4_attn_q_b_f16_build_mu = + PTHREAD_MUTEX_INITIALIZER; + +static int rocm_q4_attn_q_b_env_value_eq( + const char *value, size_t n, const char *literal) { + const size_t literal_n = strlen(literal); + if (n != literal_n) return 0; + for (size_t i = 0; i < n; i++) { + if (tolower((unsigned char)value[i]) != + tolower((unsigned char)literal[i])) { + return 0; + } + } + return 1; +} + +/* Return -1 when unset, 0 for an explicit false value, and 1 otherwise. + * In particular, VAR=0 / false / no / off must not enable an experimental + * path merely because the variable exists. */ +static int rocm_q4_attn_q_b_env_bool(const char *name) { + const char *value = getenv(name); + if (!value) return -1; + while (isspace((unsigned char)*value)) value++; + size_t n = strlen(value); + while (n != 0u && isspace((unsigned char)value[n - 1u])) n--; + if (n == 0u) return 1; + if (rocm_q4_attn_q_b_env_value_eq(value, n, "1") || + rocm_q4_attn_q_b_env_value_eq(value, n, "true") || + rocm_q4_attn_q_b_env_value_eq(value, n, "yes") || + rocm_q4_attn_q_b_env_value_eq(value, n, "on")) { + return 1; + } + if (rocm_q4_attn_q_b_env_value_eq(value, n, "0") || + rocm_q4_attn_q_b_env_value_eq(value, n, "false") || + rocm_q4_attn_q_b_env_value_eq(value, n, "no") || + rocm_q4_attn_q_b_env_value_eq(value, n, "off")) { + return 0; + } + /* Preserve the project's traditional presence-enables behavior for + * unknown non-empty values while still handling conventional booleans. */ + return 1; +} + +static uint64_t rocm_q4_attn_q_b_env_u64( + const char *name, + uint64_t fallback, + uint64_t min_value, + uint64_t max_value) { + const char *value = getenv(name); + if (!value) return fallback; + while (isspace((unsigned char)*value)) value++; + if (!*value) return fallback; + + errno = 0; + char *end = NULL; + const unsigned long long parsed = strtoull(value, &end, 10); + if (end == value || errno == ERANGE) return fallback; + while (isspace((unsigned char)*end)) end++; + if (*end != '\0' || parsed < min_value) return fallback; + const uint64_t result = (uint64_t)parsed; + return result > max_value ? max_value : result; +} + +static int rocm_q4_attn_q_b_f16_enabled(void) { + return rocm_q4_attn_q_b_env_bool( + "DS4_ROCM_ENABLE_Q4_ATTN_Q_B_F16_CACHE") == 1; +} + +static int rocm_q4_attn_q_b_f16_required(void) { + return rocm_q4_attn_q_b_env_bool( + "DS4_ROCM_REQUIRE_Q4_ATTN_Q_B_F16_CACHE") == 1; +} + +static int rocm_q4_attn_q_b_f16_disabled(void) { + return rocm_q4_attn_q_b_env_bool( + "DS4_ROCM_DISABLE_Q4_ATTN_Q_B_F16_CACHE") == 1; +} + +static uint64_t rocm_q4_attn_q_b_f16_min_tokens(void) { + return rocm_q4_attn_q_b_env_u64( + "DS4_ROCM_Q4_ATTN_Q_B_F16_CACHE_MIN_TOKENS", + 512u, 32u, UINT32_MAX); +} + +static uint64_t rocm_q4_attn_q_b_f16_budget_bytes(void) { + const uint64_t budget_mib = rocm_q4_attn_q_b_env_u64( + "DS4_ROCM_Q4_ATTN_Q_B_F16_CACHE_MB", + 3072u, 1u, 65536u); + return budget_mib * 1048576u; +} + +static int rocm_q4_attn_q_b_f16_policy_allowed(void) { + return (rocm_q4_attn_q_b_f16_enabled() || + rocm_q4_attn_q_b_f16_required()) && + !rocm_q4_attn_q_b_f16_disabled() && + !g_ssd_streaming_mode && + !g_quality_mode && + !g_q8_f16_disabled_for_multi_model; +} + +static int rocm_q4_attn_q_b_f16_key_equal( + const rocm_q4_attn_q_b_f16_cache_entry *entry, + const void *model_map, + uint64_t model_size, + uint64_t weight_offset, + uint64_t weight_bytes, + uint64_t in_dim, + uint64_t out_dim, + uint32_t weight_type) { + return entry->valid && + entry->model_map == model_map && + entry->model_size == model_size && + entry->weight_offset == weight_offset && + entry->weight_bytes == weight_bytes && + entry->in_dim == in_dim && + entry->out_dim == out_dim && + entry->weight_type == weight_type; +} + +static rocm_q4_attn_q_b_f16_cache_entry * +rocm_q4_attn_q_b_f16_find_locked( + const void *model_map, + uint64_t model_size, + uint64_t weight_offset, + uint64_t weight_bytes, + uint64_t in_dim, + uint64_t out_dim, + uint32_t weight_type) { + for (uint32_t i = 0; + i < DS4_ROCM_Q4_ATTN_Q_B_F16_CACHE_MAX_ENTRIES; + i++) { + rocm_q4_attn_q_b_f16_cache_entry *entry = + &g_rocm_q4_attn_q_b_f16_entries[i]; + if (rocm_q4_attn_q_b_f16_key_equal( + entry, model_map, model_size, weight_offset, weight_bytes, + in_dim, out_dim, weight_type)) { + return entry; + } + } + return NULL; +} + +/* On success the cache mutex remains held until the caller has enqueued the + * GEMM that consumes the returned pointer. Release holds the same mutex + * across device synchronization and frees, closing the lookup/free race. */ +static const __half *rocm_q4_attn_q_b_f16_acquire( + const void *model_map, + uint64_t model_size, + uint64_t weight_offset, + uint64_t weight_bytes, + uint64_t in_dim, + uint64_t out_dim, + uint32_t weight_type) { + pthread_mutex_lock(&g_rocm_q4_attn_q_b_f16_cache_mu); + g_rocm_q4_attn_q_b_f16_lookups++; + /* Recheck the circuit while holding the same mutex that protects the + * entries. A concurrent runtime failure may open it after the caller's + * cheap policy check but before this lookup; refusing here also keeps a + * partially freed arena unreachable while pending eviction is retried. */ + if (g_rocm_q4_attn_q_b_f16_hard_failure) { + g_rocm_q4_attn_q_b_f16_misses++; + pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_cache_mu); + return NULL; + } + rocm_q4_attn_q_b_f16_cache_entry *entry = + rocm_q4_attn_q_b_f16_find_locked( + model_map, model_size, weight_offset, weight_bytes, + in_dim, out_dim, weight_type); + if (!entry) { + g_rocm_q4_attn_q_b_f16_misses++; + pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_cache_mu); + return NULL; + } + g_rocm_q4_attn_q_b_f16_hits++; + return entry->device_ptr; +} + +static void rocm_q4_attn_q_b_f16_release_acquired(void) { + pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_cache_mu); +} + +static void rocm_q4_attn_q_b_f16_note_candidate(void) { + pthread_mutex_lock(&g_rocm_q4_attn_q_b_f16_cache_mu); + g_rocm_q4_attn_q_b_f16_candidate_calls++; + pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_cache_mu); +} + +static int rocm_q4_attn_q_b_f16_try_runtime_evict(void); + +static int rocm_q4_attn_q_b_f16_fallback( + int required, int rejected, int build_failure) { + pthread_mutex_lock(&g_rocm_q4_attn_q_b_f16_cache_mu); + g_rocm_q4_attn_q_b_f16_fallbacks++; + if (rejected) g_rocm_q4_attn_q_b_f16_rejects++; + if (build_failure) { + g_rocm_q4_attn_q_b_f16_build_failures++; + /* A submission/launch failure is backend-wide for this optional + * specialization. Fail closed for later layers instead of retrying + * the same hipBLAS or kernel error dozens of times per prefill. */ + g_rocm_q4_attn_q_b_f16_hard_failure = 1; + g_rocm_q4_attn_q_b_f16_pending_evict = 1; + } + pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_cache_mu); + /* All runtime callers reach this helper after releasing the cache mutex + * used to pin the sidecar through GEMM submission, and none holds the + * build mutex. Try the synchronized eviction now; if HIP cannot reach a + * safe point, pending_evict keeps the cache disabled until a later + * lifecycle/prewarm boundary can retry. */ + if (build_failure) (void)rocm_q4_attn_q_b_f16_try_runtime_evict(); + return required ? -1 : 0; +} + +static int rocm_q4_attn_q_b_f16_circuit_open(void) { + pthread_mutex_lock(&g_rocm_q4_attn_q_b_f16_cache_mu); + const int open = g_rocm_q4_attn_q_b_f16_hard_failure; + pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_cache_mu); + return open; +} + +__device__ __forceinline__ static void +rocm_q4_attn_q_b_get_scale_min( + uint32_t group, + const uint8_t *scales, + uint8_t *scale, + uint8_t *minimum) { + if (group < 4u) { + *scale = scales[group] & 63u; + *minimum = scales[group + 4u] & 63u; + } else { + *scale = (scales[group + 4u] & 0x0fu) | + ((scales[group - 4u] >> 6u) << 4u); + *minimum = (scales[group + 4u] >> 4u) | + ((scales[group] >> 6u) << 4u); + } +} + +/* One block expands one canonical 256-value GGUF Q4_K block. The output is + * row-major [out_dim, in_dim], the same storage consumed as W^T by hipBLAS. */ +__global__ static void rocm_dequant_q4_K_attn_q_b_f16_kernel( + __half *dst, + const cuda_block_q4_K *src, + uint64_t block_count) { + const uint64_t block = (uint64_t)blockIdx.x; + const uint32_t i = threadIdx.x; + if (block >= block_count || i >= CUDA_QK_K) return; + + const cuda_block_q4_K *xb = src + block; + const uint32_t group = i >> 5u; + const uint32_t within_group = i & 31u; + const uint32_t byte_offset = (group >> 1u) * 32u + within_group; + const uint32_t shift = (group & 1u) * 4u; + const uint32_t q = (xb->qs[byte_offset] >> shift) & 0x0fu; + uint8_t scale = 0; + uint8_t minimum = 0; + rocm_q4_attn_q_b_get_scale_min( + group, xb->scales, &scale, &minimum); + const float d = __half2float( + __ushort_as_half((unsigned short)xb->d)); + const float dmin = __half2float( + __ushort_as_half((unsigned short)xb->dmin)); + dst[block * CUDA_QK_K + i] = + __float2half(d * (float)scale * (float)q - + dmin * (float)minimum); +} + +static int rocm_q4_attn_q_b_f16_desc_valid( + const ds4_gpu_q4_attn_q_b_f16_sidecar_desc *desc, + uint64_t model_size, + uint64_t *f16_bytes) { + if (!desc || desc->weight_type != DS4_ROCM_Q4_K_TYPE || + desc->in_dim != DS4_ROCM_Q4_ATTN_Q_B_IN_DIM || + desc->out_dim != DS4_ROCM_Q4_ATTN_Q_B_OUT_DIM) { + return 0; + } + uint64_t row_bytes = 0; + uint64_t expected_weight_bytes = 0; + uint64_t elems = 0; + uint64_t expanded_bytes = 0; + if (!cuda_u64_mul_checked(desc->in_dim / CUDA_QK_K, + sizeof(cuda_block_q4_K), &row_bytes) || + !cuda_u64_mul_checked(desc->out_dim, row_bytes, + &expected_weight_bytes) || + !cuda_u64_mul_checked(desc->in_dim, desc->out_dim, &elems) || + !cuda_u64_mul_checked(elems, sizeof(__half), &expanded_bytes)) { + return 0; + } + if (desc->weight_bytes != expected_weight_bytes || + !cuda_model_range_fits(model_size, desc->weight_offset, + desc->weight_bytes)) { + return 0; + } + if (f16_bytes) *f16_bytes = expanded_bytes; + return 1; +} + +static int rocm_q4_attn_q_b_f16_memory_has_room( + uint64_t sidecar_bytes, + uint64_t working_set_reserve_bytes, + uint64_t *free_bytes_out, + uint64_t *total_bytes_out, + uint64_t *reserve_bytes_out) { + size_t free_b = 0; + size_t total_b = 0; + const cudaError_t err = cudaMemGetInfo(&free_b, &total_b); + if (err != cudaSuccess || total_b == 0u) { + (void)cudaGetLastError(); + return 0; + } + const uint64_t free_bytes = (uint64_t)free_b; + const uint64_t total_bytes = (uint64_t)total_b; + const uint64_t reserve_bytes = + cuda_q8_f16_cache_reserve_bytes(total_bytes); + uint64_t required_free = 0; + if (!cuda_u64_add_checked(sidecar_bytes, reserve_bytes, + &required_free) || + !cuda_u64_add_checked(required_free, working_set_reserve_bytes, + &required_free)) { + return 0; + } + if (free_bytes_out) *free_bytes_out = free_bytes; + if (total_bytes_out) *total_bytes_out = total_bytes; + if (reserve_bytes_out) *reserve_bytes_out = reserve_bytes; + return required_free <= free_bytes; +} + +static void rocm_q4_attn_q_b_f16_clear_locked( + int reset_stats, int reset_circuit) { + for (uint32_t i = 0; + i < DS4_ROCM_Q4_ATTN_Q_B_F16_CACHE_MAX_ENTRIES; + i++) { + g_rocm_q4_attn_q_b_f16_entries[i] = {}; + g_rocm_q4_attn_q_b_f16_arenas[i] = {}; + } + g_rocm_q4_attn_q_b_f16_entry_count = 0; + g_rocm_q4_attn_q_b_f16_arena_count = 0; + g_rocm_q4_attn_q_b_f16_bytes = 0; + if (reset_circuit) g_rocm_q4_attn_q_b_f16_hard_failure = 0; + g_rocm_q4_attn_q_b_f16_pending_evict = 0; + g_rocm_q4_attn_q_b_f16_generation++; + if (g_rocm_q4_attn_q_b_f16_generation == 0u) { + g_rocm_q4_attn_q_b_f16_generation = 1u; + } + if (reset_stats) { + g_rocm_q4_attn_q_b_f16_lookups = 0; + g_rocm_q4_attn_q_b_f16_hits = 0; + g_rocm_q4_attn_q_b_f16_misses = 0; + g_rocm_q4_attn_q_b_f16_builds = 0; + g_rocm_q4_attn_q_b_f16_build_failures = 0; + g_rocm_q4_attn_q_b_f16_candidate_calls = 0; + g_rocm_q4_attn_q_b_f16_fallbacks = 0; + g_rocm_q4_attn_q_b_f16_rejects = 0; + } +} + +/* The caller owns build_mu. Keeping this separate lets make_room serialize + * against an unpublished build without recursively locking through the public + * release API. cache_mu remains held across synchronization and frees, so a + * successful lookup cannot race the last reference to its arena. */ +static int rocm_q4_attn_q_b_f16_release_with_build_lock( + int reset_circuit) { + pthread_mutex_lock(&g_rocm_q4_attn_q_b_f16_cache_mu); + + if (g_rocm_q4_attn_q_b_f16_arena_count != 0u) { + const cudaError_t sync_err = cudaDeviceSynchronize(); + if (sync_err != cudaSuccess) { + fprintf(stderr, + DS4_GPU_LOG_PREFIX + "Q4 attn_q_b F16 cache release sync failed: %s\n", + cudaGetErrorString(sync_err)); + (void)cudaGetLastError(); + g_rocm_q4_attn_q_b_f16_hard_failure = 1; + g_rocm_q4_attn_q_b_f16_pending_evict = 1; + pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_cache_mu); + return 0; + } + } + int ok = 1; + for (uint32_t i = 0; i < g_rocm_q4_attn_q_b_f16_arena_count; i++) { + if (g_rocm_q4_attn_q_b_f16_arenas[i].device_ptr) { + const cudaError_t free_err = + cudaFree(g_rocm_q4_attn_q_b_f16_arenas[i].device_ptr); + if (free_err != cudaSuccess) { + fprintf(stderr, + DS4_GPU_LOG_PREFIX + "Q4 attn_q_b F16 cache free failed: %s\n", + cudaGetErrorString(free_err)); + (void)cudaGetLastError(); + ok = 0; + } else { + /* A later retry must not double-free arenas already released + * before another arena reported an error. */ + g_rocm_q4_attn_q_b_f16_arenas[i].device_ptr = NULL; + } + } + } + if (!ok) { + g_rocm_q4_attn_q_b_f16_hard_failure = 1; + g_rocm_q4_attn_q_b_f16_pending_evict = 1; + pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_cache_mu); + return 0; + } + rocm_q4_attn_q_b_f16_clear_locked(0, reset_circuit); + pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_cache_mu); + return 1; +} + +static int rocm_q4_attn_q_b_f16_try_runtime_evict(void) { + pthread_mutex_lock(&g_rocm_q4_attn_q_b_f16_build_mu); + const int ok = rocm_q4_attn_q_b_f16_release_with_build_lock(0); + pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_build_mu); + return ok; +} + +extern "C" int ds4_gpu_release_q4_attn_q_b_f16_sidecars(void) { + pthread_mutex_lock(&g_rocm_q4_attn_q_b_f16_build_mu); + const int ok = rocm_q4_attn_q_b_f16_release_with_build_lock(1); + pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_build_mu); + return ok; +} + +extern "C" uint64_t ds4_gpu_q4_attn_q_b_f16_cache_generation(void) { + pthread_mutex_lock(&g_rocm_q4_attn_q_b_f16_cache_mu); + const uint64_t generation = g_rocm_q4_attn_q_b_f16_generation; + pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_cache_mu); + return generation; +} + +extern "C" int ds4_gpu_make_room_for_q4_attn_q_b_f16_session(void) { + pthread_mutex_lock(&g_rocm_q4_attn_q_b_f16_build_mu); + pthread_mutex_lock(&g_rocm_q4_attn_q_b_f16_cache_mu); + const uint32_t entries = g_rocm_q4_attn_q_b_f16_entry_count; + const uint64_t bytes = g_rocm_q4_attn_q_b_f16_bytes; + const int needs_reset = + entries != 0u || g_rocm_q4_attn_q_b_f16_arena_count != 0u || + g_rocm_q4_attn_q_b_f16_hard_failure || + g_rocm_q4_attn_q_b_f16_pending_evict; + pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_cache_mu); + if (!needs_reset) { + pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_build_mu); + return 1; + } + if (entries != 0u) { + fprintf(stderr, + DS4_GPU_LOG_PREFIX + "evicting %.2f GiB of resident Q4 attn_q_b F16 sidecars " + "before allocating another live session\n", + (double)bytes / 1073741824.0); + } + const int ok = rocm_q4_attn_q_b_f16_release_with_build_lock(1); + pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_build_mu); + return ok; +} + +extern "C" int ds4_gpu_prepare_q4_attn_q_b_f16_sidecars( + const void *model_map, + uint64_t model_size, + const ds4_gpu_q4_attn_q_b_f16_sidecar_desc *descs, + uint32_t count, + uint32_t max_prefill_rows, + uint64_t working_set_reserve_bytes, + uint64_t *prepared_bytes) { + if (prepared_bytes) *prepared_bytes = 0; + if (!model_map || !descs || count == 0u || + count > DS4_ROCM_Q4_ATTN_Q_B_F16_CACHE_MAX_ENTRIES || + max_prefill_rows < 32u) { + return 0; + } + + const int required = rocm_q4_attn_q_b_f16_required(); + const uint64_t min_tokens = rocm_q4_attn_q_b_f16_min_tokens(); + if ((uint64_t)max_prefill_rows < min_tokens) return 0; + if (!rocm_q4_attn_q_b_f16_policy_allowed() || !g_cublas_ready || + model_map != g_model_host_base || + model_size != g_model_registered_size) { + return required ? -1 : 0; + } + + uint64_t desc_f16_bytes[ + DS4_ROCM_Q4_ATTN_Q_B_F16_CACHE_MAX_ENTRIES] = {0}; + for (uint32_t i = 0; i < count; i++) { + if (!rocm_q4_attn_q_b_f16_desc_valid( + &descs[i], model_size, &desc_f16_bytes[i])) { + return required ? -1 : 0; + } + } + + pthread_mutex_lock(&g_rocm_q4_attn_q_b_f16_build_mu); + pthread_mutex_lock(&g_rocm_q4_attn_q_b_f16_cache_mu); + const int pending_evict = g_rocm_q4_attn_q_b_f16_pending_evict; + pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_cache_mu); + /* Explicit prewarm is a quiescent session boundary. Retry an eviction + * that could not synchronize at the original runtime error, while + * preserving the hard circuit so this session cannot rebuild and repeat + * the failed specialization. */ + if (pending_evict && + !rocm_q4_attn_q_b_f16_release_with_build_lock(0)) { + pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_build_mu); + return required ? -1 : 0; + } + if (!rocm_q4_attn_q_b_f16_policy_allowed() || !g_cublas_ready || + model_map != g_model_host_base || + model_size != g_model_registered_size) { + pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_build_mu); + return required ? -1 : 0; + } + + uint32_t miss_indices[ + DS4_ROCM_Q4_ATTN_Q_B_F16_CACHE_MAX_ENTRIES] = {0}; + uint32_t free_slots[ + DS4_ROCM_Q4_ATTN_Q_B_F16_CACHE_MAX_ENTRIES] = {0}; + uint32_t miss_count = 0; + uint32_t free_count = 0; + uint64_t missing_bytes = 0; + uint64_t cached_bytes = 0; + int hard_failure = 0; + pthread_mutex_lock(&g_rocm_q4_attn_q_b_f16_cache_mu); + cached_bytes = g_rocm_q4_attn_q_b_f16_bytes; + hard_failure = g_rocm_q4_attn_q_b_f16_hard_failure; + for (uint32_t i = 0; + i < DS4_ROCM_Q4_ATTN_Q_B_F16_CACHE_MAX_ENTRIES; + i++) { + if (!g_rocm_q4_attn_q_b_f16_entries[i].valid) { + free_slots[free_count++] = i; + } + } + for (uint32_t i = 0; i < count; i++) { + if (rocm_q4_attn_q_b_f16_find_locked( + model_map, model_size, descs[i].weight_offset, + descs[i].weight_bytes, descs[i].in_dim, + descs[i].out_dim, descs[i].weight_type)) { + continue; + } + int duplicate_miss = 0; + for (uint32_t mi = 0; mi < miss_count; mi++) { + const ds4_gpu_q4_attn_q_b_f16_sidecar_desc *prior = + &descs[miss_indices[mi]]; + if (prior->weight_offset == descs[i].weight_offset && + prior->weight_bytes == descs[i].weight_bytes && + prior->in_dim == descs[i].in_dim && + prior->out_dim == descs[i].out_dim && + prior->weight_type == descs[i].weight_type) { + duplicate_miss = 1; + break; + } + } + if (duplicate_miss) continue; + if (!cuda_u64_add_checked(missing_bytes, desc_f16_bytes[i], + &missing_bytes)) { + pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_cache_mu); + pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_build_mu); + return required ? -1 : 0; + } + miss_indices[miss_count++] = i; + } + const int entry_room = + miss_count <= free_count && + g_rocm_q4_attn_q_b_f16_arena_count < + DS4_ROCM_Q4_ATTN_Q_B_F16_CACHE_MAX_ENTRIES; + pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_cache_mu); + + if (hard_failure) { + pthread_mutex_lock(&g_rocm_q4_attn_q_b_f16_cache_mu); + g_rocm_q4_attn_q_b_f16_rejects++; + pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_cache_mu); + pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_build_mu); + return required ? -1 : 0; + } + if (miss_count == 0u) { + pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_build_mu); + return 1; + } + const uint64_t budget_bytes = rocm_q4_attn_q_b_f16_budget_bytes(); + if (!entry_room || + missing_bytes > (uint64_t)SIZE_MAX || + missing_bytes > budget_bytes || + cached_bytes > budget_bytes - missing_bytes) { + pthread_mutex_lock(&g_rocm_q4_attn_q_b_f16_cache_mu); + g_rocm_q4_attn_q_b_f16_rejects++; + pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_cache_mu); + pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_build_mu); + return required ? -1 : 0; + } + + const char *weight_ptrs[ + DS4_ROCM_Q4_ATTN_Q_B_F16_CACHE_MAX_ENTRIES] = {NULL}; + for (uint32_t mi = 0; mi < miss_count; mi++) { + const ds4_gpu_q4_attn_q_b_f16_sidecar_desc *desc = + &descs[miss_indices[mi]]; + weight_ptrs[mi] = cuda_model_range_ptr( + model_map, desc->weight_offset, desc->weight_bytes, + "Q4 attn_q_b sidecar source"); + if (!weight_ptrs[mi]) { + pthread_mutex_lock(&g_rocm_q4_attn_q_b_f16_cache_mu); + g_rocm_q4_attn_q_b_f16_build_failures++; + g_rocm_q4_attn_q_b_f16_hard_failure = 1; + pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_cache_mu); + pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_build_mu); + return required ? -1 : 0; + } + } + + uint64_t free_bytes = 0; + uint64_t total_bytes = 0; + uint64_t reserve_bytes = 0; + if (!rocm_q4_attn_q_b_f16_memory_has_room( + missing_bytes, working_set_reserve_bytes, + &free_bytes, &total_bytes, &reserve_bytes)) { + fprintf(stderr, + DS4_GPU_LOG_PREFIX + "Q4 attn_q_b F16 prewarm skipped for device headroom: " + "sidecars %.2f GiB + future sessions %.2f GiB + reserve " + "%.2f GiB, free %.2f GiB of %.2f GiB\n", + (double)missing_bytes / 1073741824.0, + (double)working_set_reserve_bytes / 1073741824.0, + (double)reserve_bytes / 1073741824.0, + (double)free_bytes / 1073741824.0, + (double)total_bytes / 1073741824.0); + pthread_mutex_lock(&g_rocm_q4_attn_q_b_f16_cache_mu); + g_rocm_q4_attn_q_b_f16_rejects++; + pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_cache_mu); + pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_build_mu); + return required ? -1 : 0; + } + + void *arena_raw = NULL; + cudaError_t err = cudaMalloc(&arena_raw, (size_t)missing_bytes); + if (err != cudaSuccess || !arena_raw) { + fprintf(stderr, + DS4_GPU_LOG_PREFIX + "Q4 attn_q_b F16 sidecar allocation failed (%.2f GiB): %s\n", + (double)missing_bytes / 1073741824.0, + cudaGetErrorString(err)); + (void)cudaGetLastError(); + pthread_mutex_lock(&g_rocm_q4_attn_q_b_f16_cache_mu); + g_rocm_q4_attn_q_b_f16_build_failures++; + g_rocm_q4_attn_q_b_f16_hard_failure = 1; + pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_cache_mu); + pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_build_mu); + return required ? -1 : 0; + } + + __half *arena = (__half *)arena_raw; + __half *sidecar_ptrs[ + DS4_ROCM_Q4_ATTN_Q_B_F16_CACHE_MAX_ENTRIES] = {NULL}; + uint64_t arena_offset = 0; + (void)cudaGetLastError(); + int launch_ok = 1; + for (uint32_t mi = 0; mi < miss_count; mi++) { + const ds4_gpu_q4_attn_q_b_f16_sidecar_desc *desc = + &descs[miss_indices[mi]]; + sidecar_ptrs[mi] = (__half *)((char *)arena + arena_offset); + const uint64_t block_count = + (desc->in_dim / CUDA_QK_K) * desc->out_dim; + rocm_dequant_q4_K_attn_q_b_f16_kernel<<< + (uint32_t)block_count, CUDA_QK_K>>>( + sidecar_ptrs[mi], + (const cuda_block_q4_K *)weight_ptrs[mi], + block_count); + err = cudaGetLastError(); + if (err != cudaSuccess) { + fprintf(stderr, + DS4_GPU_LOG_PREFIX + "Q4 attn_q_b F16 dequant launch failed at layer %u: %s\n", + desc->layer, cudaGetErrorString(err)); + launch_ok = 0; + break; + } + arena_offset += desc_f16_bytes[miss_indices[mi]]; + } + if (launch_ok) { + err = cudaDeviceSynchronize(); + if (err != cudaSuccess) { + fprintf(stderr, + DS4_GPU_LOG_PREFIX + "Q4 attn_q_b F16 dequant synchronization failed: %s\n", + cudaGetErrorString(err)); + (void)cudaGetLastError(); + launch_ok = 0; + } + } else { + (void)cudaDeviceSynchronize(); + } + if (!launch_ok) { + (void)cudaFree(arena); + pthread_mutex_lock(&g_rocm_q4_attn_q_b_f16_cache_mu); + g_rocm_q4_attn_q_b_f16_build_failures++; + g_rocm_q4_attn_q_b_f16_hard_failure = 1; + pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_cache_mu); + pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_build_mu); + return required ? -1 : 0; + } + + /* Publish only after every dequantization completed. Until this point no + * lookup can observe any part of the new batch. */ + pthread_mutex_lock(&g_rocm_q4_attn_q_b_f16_cache_mu); + const uint32_t arena_slot = g_rocm_q4_attn_q_b_f16_arena_count++; + g_rocm_q4_attn_q_b_f16_arenas[arena_slot] = {arena, missing_bytes}; + uint32_t published = 0; + for (uint32_t mi = 0; mi < miss_count; mi++) { + const uint32_t slot = free_slots[mi]; + const ds4_gpu_q4_attn_q_b_f16_sidecar_desc *desc = + &descs[miss_indices[mi]]; + g_rocm_q4_attn_q_b_f16_entries[slot] = { + model_map, + model_size, + desc->weight_offset, + desc->weight_bytes, + desc->in_dim, + desc->out_dim, + desc->weight_type, + sidecar_ptrs[mi], + desc_f16_bytes[miss_indices[mi]], + 1, + }; + published++; + } + g_rocm_q4_attn_q_b_f16_entry_count += published; + g_rocm_q4_attn_q_b_f16_bytes += missing_bytes; + g_rocm_q4_attn_q_b_f16_builds += published; + pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_cache_mu); + pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_build_mu); + + if (prepared_bytes) *prepared_bytes = missing_bytes; + fprintf(stderr, + DS4_GPU_LOG_PREFIX + "prewarmed %u resident Q4 attn_q_b F16 sidecars " + "(%.2f GiB; cache budget %llu MiB; min batch %llu tokens)\n", + published, + (double)missing_bytes / 1073741824.0, + (unsigned long long)(budget_bytes / 1048576u), + (unsigned long long)min_tokens); + return 1; +} diff --git a/rocm/ds4_rocm_runtime.cuh b/rocm/ds4_rocm_runtime.cuh index b1075d10e..dfb8bb009 100644 --- a/rocm/ds4_rocm_runtime.cuh +++ b/rocm/ds4_rocm_runtime.cuh @@ -5882,6 +5882,7 @@ extern "C" int ds4_gpu_init(void) { extern "C" void ds4_gpu_cleanup(void) { (void)cudaDeviceSynchronize(); + (void)ds4_gpu_release_q4_attn_q_b_f16_sidecars(); cuda_stream_cache_stats_print("cleanup"); cuda_shared_gate_up_async_cleanup(); #ifdef __HIP_PLATFORM_AMD__ @@ -6140,6 +6141,7 @@ extern "C" int ds4_gpu_set_model_map(const void *model_map, uint64_t model_size) const int multi_model = g_model_host_base != NULL && (g_model_host_base != model_map || g_model_registered_size != model_size); + if (!ds4_gpu_release_q4_attn_q_b_f16_sidecars()) return 0; cuda_model_range_release_all(); cuda_q8_f16_cache_release_all(); g_q8_f16_disabled_after_oom = 0; @@ -6259,6 +6261,7 @@ extern "C" int ds4_gpu_prepare_support_model( const uint64_t saved_registered_size = g_model_registered_size; const int saved_device_owned = g_model_device_owned; + if (!ds4_gpu_release_q4_attn_q_b_f16_sidecars()) return 0; cuda_q8_f16_cache_release_all(); g_q8_f16_disabled_for_multi_model = 1; const int ok = cuda_model_copy_chunked(model_map, @@ -6476,6 +6479,9 @@ extern "C" void ds4_gpu_print_memory_report(const char *label) { extern "C" void ds4_gpu_set_quality(bool quality) { const int new_quality_mode = quality ? 1 : 0; + if (new_quality_mode && !g_quality_mode) { + (void)ds4_gpu_release_q4_attn_q_b_f16_sidecars(); + } if (g_quality_mode != new_quality_mode) { g_rocm_cfg.initialized = 0; } diff --git a/scripts/environment_variables.tsv b/scripts/environment_variables.tsv index 5496bc2fa..8d3c9cf1a 100644 --- a/scripts/environment_variables.tsv +++ b/scripts/environment_variables.tsv @@ -54,6 +54,7 @@ runtime/cuda DS4_CUDA_DISABLE_HC_NORM_MIX_FUSE presence kill switch; default uns runtime/cuda DS4_CUDA_DISABLE_HC_SPLIT_NORM_FUSED presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the fused HC split/weighted-sum/norm kernel. ds4_cuda.cu:31785 runtime/cuda DS4_CUDA_DISABLE_IQ2_XXS_SSD_PREFILL_MMQ false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Disable the CUDA IQ2 XXS SSD prefill MMQ optimization/path. ds4_cuda.cu:4627 runtime/cuda DS4_CUDA_DISABLE_Q4_ATTN_OUT_HC_FUSE presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable fused Q4 attention-output/HC expansion. ds4_cuda.cu:37357 +runtime/cuda DS4_CUDA_DISABLE_Q4_ATTN_Q_B_F16_CACHE value-aware rollback; unset/empty/0/false/no/off keeps the experiment available, any other nonempty value disables it Disable the resident Q4_K attn_q_b-to-F16 prefill cache even when ENABLE or REQUIRE is set. ds4_cuda.cu:2491 runtime/cuda DS4_CUDA_DISABLE_Q4_DENSE_PAIR presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q4 dense pair CUDA Q4 optimization. ds4_cuda.cu:20402 runtime/cuda DS4_CUDA_DISABLE_Q8_HC_EXPAND_FUSED false-like-aware flag, default off; 0/false/no/off is off, other nonempty values request split; force-fused wins Request the split Q8 shared-down/HC path when safe. ds4_cuda.cu:2086 runtime/cuda DS4_CUDA_DISABLE_QKV_RMS_FUSED presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the CUDA QKV RMS fused optimization/path. ds4_cuda.cu:369; ds4.c:17428 @@ -78,6 +79,7 @@ runtime/cuda DS4_CUDA_ENABLE_DSPARK_NONCAUSAL_ONLINE value-aware flag, default o runtime/cuda DS4_CUDA_ENABLE_HC_NORM_MIX_FUSE nonempty opt-in, default off; only exact 0 disables; the F32/F16 activation mode follows the selected standalone matmul path; disable/serial/alternate flags can veto Enable and select the fused HC RMSNorm-plus-mix one-token implementation. ds4_cuda.cu:20902 runtime/cuda DS4_CUDA_ENABLE_IQ2_XXS_SSD_PREFILL_MMQ false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Enable the CUDA IQ2 XXS SSD prefill MMQ experimental path. ds4_cuda.cu:4625 runtime/cuda DS4_CUDA_ENABLE_Q4_ATTN_OUT_HC_FUSE value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on Opt in to the fused Q4 attention-output/HC expansion path. ds4_cuda.cu:37375 +runtime/cuda DS4_CUDA_ENABLE_Q4_ATTN_Q_B_F16_CACHE value-aware opt-in, default off; unset/empty/0/false/no/off is off, any other nonempty value enables unless DISABLE wins Prewarm and use resident F16 sidecars for eligible single-GPU Q4_K attn_q_b prefills. ds4_cuda.cu:2479 runtime/cuda DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_BATCH value-aware opt-in, default off; nonempty value other than exact 0 enables; rollback wins Enable flattened grouped attention-A MMQ for two-to-eight-token GB10 batches. cuda/mmq/ds4_mmq.cu:4303 runtime/cuda DS4_CUDA_ENABLE_Q4_K1024_PERSISTENT presence flag, default off; any defined value including 0 requests the path; rollback wins Enable the GB10 persistent-CTA kernel for M=32768, N=1, K=1024 Q4. cuda/mmq/ds4_mmq.cu:3905 runtime/cuda DS4_CUDA_ENABLE_Q8_FOLD strict flag, default off; only exact value 1 enables; overridden by DS4_CUDA_NO_Q8_FOLD Enable one-shot producer-to-consumer reuse of freshly quantized Q8_1 data. ds4_cuda.cu:785 @@ -274,6 +276,8 @@ runtime/cuda DS4_CUDA_PREFILL_PIPELINE_SEQUENTIAL presence flag; unset does not runtime/cuda DS4_CUDA_PREFILL_PIPELINE_SYNC_BOUNDARY presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Synchronize CUDA at every prefill pipeline tier boundary. ds4.c:35572 runtime/cuda DS4_CUDA_Q4_ATTN_OUT_HC_ORACLE value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on Compare fused Q4 attention-output/HC expansion with the canonical path and retain canonical output. ds4_cuda.cu:1475 runtime/cuda DS4_CUDA_Q4_ATTN_OUT_HC_Q8K_EXPERIMENT value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on Enable the experimental Q8_K-based Q4 attention-output/HC fusion. ds4_cuda.cu:37377 +runtime/cuda DS4_CUDA_Q4_ATTN_Q_B_F16_CACHE_MB unsigned integer MiB with a full-string parse; default 3072; 0 prevents admission; invalid text restores the default; overflow saturates Cap device memory used by resident Q4_K attn_q_b F16 sidecars. ds4_cuda.cu:2466 +runtime/cuda DS4_CUDA_Q4_ATTN_Q_B_F16_CACHE_MIN_TOKENS integer token count; default 512; valid values clamp to 32..UINT32_MAX; empty or invalid text restores 512 Set the minimum prefill batch eligible to prepare or use the CUDA Q4 attn_q_b F16 sidecars. ds4_cuda.cu:2473 runtime/cuda DS4_CUDA_Q4_GROUPED_ATTN_A_ORACLE value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on Compare grouped attention-A against the canonical per-group result. ds4_cuda.cu:1477 runtime/cuda DS4_CUDA_Q4_K1024_PERSISTENT_ORACLE value-aware flag, default off; nonempty value other than exact 0 enables and implies candidate admission Bitwise-compare the exact-shape persistent Q4 K1024 kernel with canonical MMVQ and retain canonical output. cuda/mmq/ds4_mmq.cu:3738 runtime/cuda DS4_CUDA_Q4_K1024_PERSISTENT_STATS value-aware flag, default off; nonempty value other than exact 0 enables Print exact-shape persistent Q4 K1024 dispatch counters at exit. cuda/mmq/ds4_mmq.cu:3737 @@ -291,6 +295,7 @@ runtime/cuda DS4_CUDA_Q8_PAIR_BATCH presence flag; unset does not force the path runtime/cuda DS4_CUDA_QKV_KV_ROPE_FUSE value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control or tune the CUDA QKV KV rope fuse path. ds4.c:17429 runtime/cuda DS4_CUDA_Q_NORM_ROPE_FUSE value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control or tune the CUDA q norm rope fuse path. ds4.c:17418 runtime/cuda DS4_CUDA_REQUIRE_IQ2_XXS_SSD_PREFILL_MMQ false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Require the CUDA IQ2 XXS SSD prefill MMQ path; fail closed when unavailable. ds4_cuda.cu:4631 +runtime/cuda DS4_CUDA_REQUIRE_Q4_ATTN_Q_B_F16_CACHE value-aware strict opt-in, default off; unset/empty/0/false/no/off is off, any other nonempty value requires eligible batches to use the cache; DISABLE wins Fail an eligible CUDA prefill instead of falling back when the resident Q4_K attn_q_b F16 specialization cannot be prepared or dispatched. ds4_cuda.cu:2481; ds4_cuda.cu:2486 runtime/cuda DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_BATCH value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on Fail if grouped batched attention-A cannot be used. ds4_cuda.cu:40198 runtime/cuda DS4_CUDA_REQUIRE_Q4_K1024_PERSISTENT presence flag, default off; any defined value including 0 makes ineligible candidate fail closed Fail when the exact Q4 K1024 persistent candidate is unavailable instead of using MMVQ. cuda/mmq/ds4_mmq.cu:3929 runtime/cuda DS4_CUDA_REQUIRE_STREAMING_EXPERT_PERSISTENT_CACHE false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Require streaming expert persistent cache in CUDA SSD streaming; fail closed when unavailable. ds4_cuda.cu:4117 @@ -934,6 +939,7 @@ runtime/rocm DS4_ROCM_DISABLE_GLM_STREAMING_PREFILL_SELECTED_ASYNC_LOAD presence runtime/rocm DS4_ROCM_DISABLE_GLM_STREAMING_SELECTED_ASYNC_LOAD presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable glm streaming selected async load. ds4.c:44328 runtime/rocm DS4_ROCM_DISABLE_IQ2_SELECTED_EXPERT_VIEWS presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable iq2 selected expert views. ds4.c:21079 runtime/rocm DS4_ROCM_DISABLE_IQ2_STREAM_ADDR_TABLE presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable iq2 stream addr table. ds4.c:6548 +runtime/rocm DS4_ROCM_DISABLE_Q4_ATTN_Q_B_F16_CACHE value-aware rollback; unset/0/false/no/off keeps the experiment available, empty or any other value disables it Disable the resident ROCm Q4_K attn_q_b-to-F16 prefill cache even when ENABLE or REQUIRE is set. rocm/ds4_rocm_q4_qb_sidecar.cuh:130 runtime/rocm DS4_ROCM_DISABLE_Q4_DENSE_PAIR presence rollback; unset leaves opt-in policy unchanged Disable/roll back rocm disable q4 dense pair. rocm/ds4_rocm_q4.cuh:435 runtime/rocm DS4_ROCM_DISABLE_Q4_GROUPED_ATTN_A presence rollback; DISABLE wins over enable/require Disable/roll back rocm disable q4 grouped attn a. rocm/ds4_rocm_q4.cuh:700 runtime/rocm DS4_ROCM_DISABLE_Q4_PREFILL_TILE8 presence rollback; TILE8 is default for 9..4096 tokens Disable/roll back rocm disable q4 prefill tile8. rocm/ds4_rocm_q4.cuh:448 @@ -971,6 +977,7 @@ runtime/rocm DS4_ROCM_ENABLE_MXFP4_LDSB presence opt-in; unset=off; any defined runtime/rocm DS4_ROCM_ENABLE_MXFP4_ROW64 presence opt-in; unset=off; any defined value including empty or 0 enables the candidate when the MXFP4 sorted-tile path has at least 8 tokens and the TILE32, LDSB, and TILE4 candidates are not selected Select the ROCm MXFP4 gate/up tile8 occupancy variant with 64 row slots and 512 threads per block. rocm/ds4_rocm_moe_launch.cuh:798 runtime/rocm DS4_ROCM_ENABLE_MXFP4_TILE32 presence opt-in; unset=off; any defined value including empty or 0 enables the candidate when the MXFP4 sorted-tile path has at least 32 tokens and the expert intermediate dimension is divisible by 32 Select the ROCm MXFP4 gate/up tile32 kernel, reusing each loaded expert-weight chunk across as many as 32 tokens. rocm/ds4_rocm_moe_launch.cuh:786 runtime/rocm DS4_ROCM_ENABLE_MXFP4_TILE4 presence opt-in; unset=off; any defined value including empty or 0 enables the candidate when the MXFP4 sorted-tile path has at least 5 tokens and neither TILE32 nor LDSB is selected Select the ROCm MXFP4 gate/up tile4 occupancy variant, reducing staged-activation LDS per block. rocm/ds4_rocm_moe_launch.cuh:794 +runtime/rocm DS4_ROCM_ENABLE_Q4_ATTN_Q_B_F16_CACHE value-aware opt-in, default off; unset/0/false/no/off is off, empty or any other value enables unless DISABLE wins Prewarm and use resident F16 sidecars for eligible ROCm Q4_K attn_q_b prefills. rocm/ds4_rocm_q4_qb_sidecar.cuh:120 runtime/rocm DS4_ROCM_ENABLE_Q4_DENSE_PAIR presence opt-in; unset=off; DISABLE takes precedence Enable rocm enable q4 dense pair. rocm/ds4_rocm_q4.cuh:434 runtime/rocm DS4_ROCM_ENABLE_Q4_GROUPED_ATTN_A presence opt-in; unset=off unless REQUIRE; DISABLE wins Enable rocm enable q4 grouped attn a. rocm/ds4_rocm_q4.cuh:704 runtime/rocm DS4_ROCM_ENABLE_STREAMING_FULL_EXPERT_ADDR_TABLE presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming full expert addr table. ds4.c:18240 @@ -1024,10 +1031,13 @@ runtime/rocm DS4_ROCM_MOE_DECODE_RPB sampled once; nonempty value is parsed by s runtime/rocm DS4_ROCM_MOE_PATH_DEBUG presence diagnostic; unset=off; any defined value including empty or 0 enables it Print ROCm routed-MoE path selection, sorted-tile scratch state, and MXFP4 gate/up launch diagnostics to stderr. rocm/ds4_rocm_moe_launch.cuh:831 runtime/rocm DS4_ROCM_MOE_WRITE_CLAMPED_ACT Pure presence sentinel: any defined value, including empty or "0", is active; DS4_METAL_MOE_WRITE_CLAMPED_ACT is an accepted fallback alias. On ROCm the variable is only consumed as a path-admission veto: it disables selected-expert cache/address-table, selected-slot and CPU-router/fused optimized paths. No ROCm call site parses a clamp amount or directly enables a write-clamped kernel. Force shared graph selection away from optimizations incompatible with the clamped-intermediate MoE diagnostic; on ROCm this is a compatibility/rollback gate, not itself a clamped-write implementation. ds4.c:18526 runtime/rocm DS4_ROCM_MXFP4_DOWN_RGROUP nonempty value is parsed by strtol and a numeric prefix is sufficient; integers 1..8 are accepted; unset, empty, invalid, or out-of-range values use 1 Set how many 32-row output blocks each ROCm MXFP4 tiled down-projection block computes, reducing the first launch-grid dimension as the value increases. rocm/ds4_rocm_moe_launch.cuh:801 +runtime/rocm DS4_ROCM_Q4_ATTN_Q_B_F16_CACHE_MB integer MiB with a full-string parse; default 3072; accepted range 1..65536, values above clamp and invalid or smaller values restore the default Cap device memory used by resident ROCm Q4_K attn_q_b F16 sidecars. rocm/ds4_rocm_q4_qb_sidecar.cuh:141 +runtime/rocm DS4_ROCM_Q4_ATTN_Q_B_F16_CACHE_MIN_TOKENS integer token count with a full-string parse; default 512; accepted range 32..UINT32_MAX, values above clamp and invalid or smaller values restore the default Set the minimum prefill batch eligible to prepare or use the ROCm Q4 attn_q_b F16 sidecars. rocm/ds4_rocm_q4_qb_sidecar.cuh:135 runtime/rocm DS4_ROCM_Q4_GROUPED_ATTN_A_STATS presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Print counters for rocm q4 grouped attn a stats. rocm/ds4_rocm_q4.cuh:480 runtime/rocm DS4_ROCM_Q4_PREFILL_TILE8_STATS presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Print counters for rocm q4 prefill tile8 stats. rocm/ds4_rocm_q4.cuh:514 runtime/rocm DS4_ROCM_Q8_DECODE_SHAREDX_64K sampled once; unset: enabled; present empty or exact 0: disabled; every other present value: enabled; effective only for one-token non-prequant Q8_0 matmul with 8192 < in_dim <= 16384 Allows the ROCm shared-input Q8 decode kernel to use up to 64 KiB dynamic LDS for wide inputs; an unsupported/failed LDS launch automatically falls back to the regular kernel. rocm/ds4_rocm_runtime.cuh:4805 -runtime/rocm DS4_ROCM_Q_STAGE_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm q stage profile. ds4.c:29269 +runtime/rocm DS4_ROCM_Q_STAGE_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm q stage profile. ds4.c:30038 +runtime/rocm DS4_ROCM_REQUIRE_Q4_ATTN_Q_B_F16_CACHE value-aware strict opt-in, default off; unset/0/false/no/off is off, empty or any other value requires eligible batches to use the cache; DISABLE wins Fail an eligible ROCm prefill instead of falling back when the resident Q4_K attn_q_b F16 specialization cannot be prepared or dispatched. rocm/ds4_rocm_q4_qb_sidecar.cuh:125 runtime/rocm DS4_ROCM_REQUIRE_Q4_GROUPED_ATTN_A presence fail-closed assertion; also requests candidate unless disabled Require rocm require q4 grouped attn a and fail instead of silently falling back. rocm/ds4_rocm_q4.cuh:702 runtime/rocm DS4_ROCM_REQUIRE_Q4_PREFILL_TILE8 presence fail-closed assertion for eligible TILE8 calls Require rocm require q4 prefill tile8 and fail instead of silently falling back. rocm/ds4_rocm_q4.cuh:452 runtime/rocm DS4_ROCM_STREAMING_DECODE_PREFILL_MAX primary nonempty value over the Metal alias; parsed by strtol when it has a numeric prefix (trailing text is accepted); <= 0 disables, values > UINT32_MAX clamp, no numeric prefix uses automatic default: 64 for Flash with uniform Q4_K/MXFP4 experts, 18 for other Pro/Flash, otherwise 0; the disable flag dominates Sets the largest short, non-quality SSD-streaming prefill batch routed through the decode-style path instead of canonical layer-major prefill. ds4.c:31961 From bd92e16eb6cea9f68a90145b10992ad98f1879d1 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Wed, 26 Aug 2026 18:07:56 +0200 Subject: [PATCH 118/189] metal: add transient F16 q_b prefill acceleration --- ds4.c | 38 +- ds4_gpu.h | 46 +- ds4_metal.m | 1086 ++++++++++++++++++++++++++++++++++++++++++--- metal/dense.metal | 5 + 4 files changed, 1097 insertions(+), 78 deletions(-) diff --git a/ds4.c b/ds4.c index 0a4bc5f65..2fd8ba7e3 100644 --- a/ds4.c +++ b/ds4.c @@ -55422,6 +55422,7 @@ static int generate_metal_graph_raw_swa( uint32_t ssd_streaming_preload_experts, uint64_t ssd_streaming_cache_bytes, uint64_t ssd_streaming_prefill_headroom_bytes, + uint64_t q4_sidecar_working_set_reserve_bytes, int power_percent, uint32_t prefill_chunk, const char * directional_steering_file, @@ -55501,17 +55502,18 @@ static int generate_metal_graph_raw_swa( metal_graph_free(&g); return 1; } - /* This frontend knows the real prompt width. Prepare only when that - * workload can use the resident sidecars, after graph allocation is - * visible to the backend but before warmup and the measured prefill - * window. */ + /* This frontend knows the real prompt width. Let the backend prepare + * either its reusable transient scratch or explicit resident sidecars + * after graph allocation is visible, but before warmup and the measured + * prefill window. */ const uint32_t q4_sidecar_rows = (uint32_t)prompt->len < prefill_cap ? (uint32_t)prompt->len : prefill_cap; if (ds4_prepare_q4_attn_q_b_sidecars( - model, weights, q4_sidecar_rows, 0u) < 0) { + model, weights, q4_sidecar_rows, + q4_sidecar_working_set_reserve_bytes) < 0) { fprintf(stderr, - "ds4: required GPU Q4 attn_q_b F16 sidecar prewarm " + "ds4: required GPU Q4 attn_q_b F16 acceleration preflight " "could not be completed\n"); metal_graph_free(&g); return 1; @@ -56442,6 +56444,7 @@ struct ds4_session { bool engine_session_counted; #ifndef DS4_NO_GPU uint64_t q4_attn_q_b_f16_sidecars_generation; + uint32_t q4_attn_q_b_f16_prepared_rows; #endif bool checkpoint_valid; bool mtp_draft_valid; @@ -60779,13 +60782,17 @@ int ds4_engine_generate_argmax( e->ssd_streaming, e->ssd_streaming_cold, e->ssd_streaming_preload_experts, e->ssd_streaming_cache_bytes, - e->ssd_streaming_prefill_headroom_bytes, e->power_percent, + e->ssd_streaming_prefill_headroom_bytes, + ds4_engine_streaming_transient_guard_bytes(e), + e->power_percent, e->prefill_chunk, e->directional_steering_file, e->directional_steering_attn_scale, e->directional_steering_ffn_scale, emit, done, emit_ud, progress, progress_ud); /* The legacy frontend owns a temporary graph rather than a session. - * Do not leave its 2.69 GiB resident sidecars pinned after return. */ + * Do not leave explicitly requested resident sidecars pinned after + * return. Each backend may either retain reusable scratch for its + * lifecycle or release it at this quiescent boundary. */ if (__atomic_load_n( &e->live_session_count, __ATOMIC_RELAXED) == 0u && !ds4_gpu_release_q4_attn_q_b_f16_sidecars()) { @@ -67101,7 +67108,8 @@ static int ds4_session_prepare_q4_attn_q_b_sidecars( const uint64_t cache_generation = ds4_gpu_q4_attn_q_b_f16_cache_generation(); if (s->q4_attn_q_b_f16_sidecars_generation == - cache_generation) { + cache_generation && + max_batch_rows <= s->q4_attn_q_b_f16_prepared_rows) { return 1; } ds4_engine *e = s->engine; @@ -67131,11 +67139,16 @@ static int ds4_session_prepare_q4_attn_q_b_sidecars( : memory.total_bytes * remaining; } + const uint64_t streaming_reserve_bytes = + ds4_engine_streaming_transient_guard_bytes(e); + const uint64_t working_set_reserve_bytes = + ds4_add_sat_u64(future_session_bytes, streaming_reserve_bytes); const int rc = ds4_prepare_q4_attn_q_b_sidecars( - &e->model, &e->weights, max_batch_rows, future_session_bytes); + &e->model, &e->weights, max_batch_rows, + working_set_reserve_bytes); if (rc < 0) { fprintf(stderr, - "ds4: required %s Q4 attn_q_b F16 sidecar prewarm " + "ds4: required %s Q4 attn_q_b F16 acceleration preflight " "could not be completed\n", ds4_backend_name(e->backend)); return 0; @@ -67143,6 +67156,7 @@ static int ds4_session_prepare_q4_attn_q_b_sidecars( if (rc > 0) { s->q4_attn_q_b_f16_sidecars_generation = ds4_gpu_q4_attn_q_b_f16_cache_generation(); + s->q4_attn_q_b_f16_prepared_rows = max_batch_rows; } return 1; } @@ -68929,7 +68943,7 @@ int ds4_session_prepare_sync(ds4_session *s, s, max_batch_rows, false)) { if (err && errlen) { snprintf(err, errlen, - "required %s Q4 attn_q_b F16 sidecar prewarm failed", + "required %s Q4 attn_q_b F16 acceleration preflight failed", ds4_backend_name(s->engine->backend)); } return 1; diff --git a/ds4_gpu.h b/ds4_gpu.h index b8707a652..c33541792 100644 --- a/ds4_gpu.h +++ b/ds4_gpu.h @@ -197,10 +197,11 @@ typedef struct ds4_gpu_q4_attn_q_b_f16_sidecar_desc { uint32_t layer; } ds4_gpu_q4_attn_q_b_f16_sidecar_desc; -/* Backend-neutral lifecycle for optional resident Q4_K attn_q_b F16 - * sidecars. Each graph backend owns its storage and admission policy. - * Prepare returns 1 when all descriptors are READY, 0 for a policy/safety - * skip, and -1 when strict mode requires an unavailable specialization. */ +/* Backend-neutral prefill preflight for optional Q4_K attn_q_b F16 + * acceleration. A backend may prepare resident sidecars or reusable + * transient scratch/pipelines according to its policy. Prepare returns 1 + * when the selected path is ready, 0 for a policy/safety skip, and -1 when + * strict mode requires an unavailable specialization. */ int ds4_gpu_prepare_q4_attn_q_b_f16_sidecars( const void *model_map, uint64_t model_size, @@ -312,6 +313,33 @@ int ds4_gpu_test_q4_attn_q_b_f16_projection_tensor( uint64_t out_dim, const ds4_gpu_tensor *x, uint32_t n_tok); +typedef enum ds4_gpu_test_q4_qb_mm_arm { + DS4_GPU_TEST_Q4_QB_MM_Q4_F32 = 0, + DS4_GPU_TEST_Q4_QB_MM_Q4_F16 = 1, + DS4_GPU_TEST_Q4_QB_MM_F16_F32 = 2, + DS4_GPU_TEST_Q4_QB_MM_F16_F16 = 3, + DS4_GPU_TEST_Q4_QB_MM_Q4_TRANSIENT_F16_F16 = 4, + DS4_GPU_TEST_Q4_QB_MM_ARM_COUNT = 5, +} ds4_gpu_test_q4_qb_mm_arm; +/* Runtime capability probe for test-only matmul arms. */ +int ds4_gpu_test_q4_attn_q_b_mm_arm_supported( + ds4_gpu_test_q4_qb_mm_arm arm); +/* Strict projection-only resident benchmark hook. F16-weight arms require + * a READY sidecar; F16-RHS arms optionally include the production copy. The + * transient arm rebuilds its F16 weight matrix for every benchmark/oracle + * projection and may consume either a prepacked or freshly copied F16 RHS. */ +int ds4_gpu_test_q4_attn_q_b_mm_variant_tensor( + ds4_gpu_tensor *out_f32, + ds4_gpu_tensor *rhs_f16, + const void *model_map, + uint64_t model_size, + uint64_t weight_offset, + uint64_t in_dim, + uint64_t out_dim, + const ds4_gpu_tensor *x_f32, + uint32_t n_tok, + ds4_gpu_test_q4_qb_mm_arm arm, + bool materialize_rhs); int ds4_gpu_test_q4_attn_q_b_f16_working_set_policy( uint64_t recommended, uint64_t allocated, @@ -1571,9 +1599,13 @@ int ds4_gpu_head_rms_norm_rope_tail_tensor( float beta_slow, float eps); -/* Returns 1 when the backend-specific fused projection was encoded, 0 when - * the caller should use its generic projection path, and -1 when a required - * specialization could not be honored. */ +/* Returns 1 when the backend-specific fused projection and its consumers were + * accepted/encoded successfully, 0 only while the caller may safely use its + * generic projection path, and -1 when a required specialization could not be + * honored or work failed after an output writer was encoded (so replay is + * unsafe). Completion is established by the enclosing backend synchronization. + * q_half is optional: backends whose graph owns F16 projection staging may use + * it, while CUDA/ROCm can emit F32 directly into out. */ int ds4_gpu_attn_q_b_f16_head_rms_rope_tail_tensor( ds4_gpu_tensor *out, ds4_gpu_tensor *q_half, diff --git a/ds4_metal.m b/ds4_metal.m index bedbd7f4e..2926f47c6 100644 --- a/ds4_metal.m +++ b/ds4_metal.m @@ -536,6 +536,10 @@ static uint64_t g_q4_attn_q_b_f16_cache_rejects; static uint32_t g_q4_attn_q_b_f16_cache_entries; static uint64_t g_q4_attn_q_b_f16_cache_generation = 1u; +/* A failed resident->SSD release must not leave an old resident sidecar + * reachable through the SSD opt-in path. The lifecycle setter closes this + * latch only after a successful synchronized release. */ +static int g_q4_attn_q_b_f16_ssd_admission_blocked; typedef enum { DS4_Q4_ATTN_Q_B_F16_CIRCUIT_CLOSED = 0, DS4_Q4_ATTN_Q_B_F16_CIRCUIT_HARD = 1, @@ -543,6 +547,19 @@ } ds4_gpu_q4_attn_q_b_f16_circuit_state; static ds4_gpu_q4_attn_q_b_f16_circuit_state g_q4_attn_q_b_f16_build_circuit_state; +/* Layer-agnostic, stream-local F16 expansion scratch. Each serial Metal + * stream rewrites its own buffer from the current Q4_K source immediately + * before consuming it. Independent streams therefore never alias this + * transient state, while consecutive layers reuse only 64 MiB per stream. */ +static id __strong + g_q4_qb_transient_f16_scratch[DS4_GPU_MAX_STREAMS]; +static NSUInteger + g_q4_qb_transient_f16_scratch_capacity[DS4_GPU_MAX_STREAMS]; +/* Protect only allocation/publication and cleanup. Encoding never retains + * this mutex; command ordering and the one-in-flight-batch-per-stream rule + * protect reuse after a buffer has been captured strongly by the caller. */ +static pthread_mutex_t g_q4_qb_transient_f16_scratch_mu = + PTHREAD_MUTEX_INITIALIZER; static int g_initialized; static void ds4_gpu_q4_attn_q_b_f16_cache_clear(int reset_stats) { @@ -608,8 +625,16 @@ void ds4_gpu_test_q4_attn_q_b_f16_cache_report( } void ds4_gpu_test_q4_attn_q_b_f16_cache_reset(void) { - if (g_initialized) (void)ds4_gpu_synchronize(); + if (g_initialized && !ds4_gpu_synchronize()) { + __atomic_store_n(&g_q4_attn_q_b_f16_ssd_admission_blocked, + 1, + __ATOMIC_RELEASE); + return; + } ds4_gpu_q4_attn_q_b_f16_cache_clear(1); + __atomic_store_n(&g_q4_attn_q_b_f16_ssd_admission_blocked, + 0, + __ATOMIC_RELEASE); } uint64_t ds4_gpu_q4_attn_q_b_f16_cache_generation(void) { @@ -632,6 +657,9 @@ int ds4_gpu_release_q4_attn_q_b_f16_sidecars(void) { * breaker open and must not poison the next resident session. */ if (has_entries && g_initialized && !ds4_gpu_synchronize()) return 0; ds4_gpu_q4_attn_q_b_f16_cache_clear(0); + __atomic_store_n(&g_q4_attn_q_b_f16_ssd_admission_blocked, + 0, + __ATOMIC_RELEASE); return 1; } static uint64_t g_stream_expert_cache_bytes; @@ -1029,6 +1057,7 @@ static void ds4_gpu_stream_expert_exact_rows_clear(void) { static double g_stream_expert_cache_mlock_ms; static int g_stream_expert_cache_mlock_warned; static uint64_t g_stream_expert_cache_slab_slot_bytes; +static uint64_t g_stream_expert_cache_slab_allocated_bytes; static uint64_t g_stream_expert_cache_cb_seq; static uint64_t g_stream_expert_cache_done_seq; static uint64_t g_stream_expert_cache_batch_seq; @@ -1041,6 +1070,19 @@ static void ds4_gpu_stream_expert_exact_rows_clear(void) { static id g_stream_selected_id_buffers[DS4_METAL_STREAM_EXPERT_CACHE_MAX_LAYER]; static id g_stream_expert_validate_status_buffer; +static void ds4_gpu_atomic_u64_add_sat(uint64_t *value, uint64_t add) { + uint64_t current = __atomic_load_n(value, __ATOMIC_RELAXED); + for (;;) { + const uint64_t next = + add > UINT64_MAX - current ? UINT64_MAX : current + add; + if (__atomic_compare_exchange_n(value, ¤t, next, 0, + __ATOMIC_RELEASE, + __ATOMIC_RELAXED)) { + return; + } + } +} + @interface DS4MetalTensor : NSObject @property(nonatomic, strong) id buffer; @property(nonatomic, assign) uint64_t offset; @@ -4948,15 +4990,34 @@ void ds4_gpu_set_glm_model(bool enabled) { } void ds4_gpu_set_ssd_streaming(bool enabled) { + const int was_ssd_streaming = g_ssd_streaming_mode; g_ssd_streaming_mode = enabled ? 1 : 0; + /* A real resident->streaming transition must re-evaluate sidecar + * admission against the streaming reserve. Reasserting an already active + * SSD mode is configuration plumbing, not a lifecycle boundary, and must + * not evict a successfully admitted hybrid cache. */ if (g_ssd_streaming_mode && - !ds4_gpu_release_q4_attn_q_b_f16_sidecars()) { - fprintf(stderr, - "ds4: WARNING: could not release resident Q4 attn_q_b " - "F16 sidecars while enabling SSD streaming\n"); + (!was_ssd_streaming || + __atomic_load_n(&g_q4_attn_q_b_f16_ssd_admission_blocked, + __ATOMIC_ACQUIRE))) { + __atomic_store_n(&g_q4_attn_q_b_f16_ssd_admission_blocked, + 1, + __ATOMIC_RELEASE); + if (!ds4_gpu_release_q4_attn_q_b_f16_sidecars()) { + fprintf(stderr, + "ds4: WARNING: could not release resident Q4 attn_q_b " + "F16 sidecars while enabling SSD streaming; the hybrid " + "cache remains blocked\n"); + } + } else if (!g_ssd_streaming_mode) { + __atomic_store_n(&g_q4_attn_q_b_f16_ssd_admission_blocked, + 0, + __ATOMIC_RELEASE); } - ds4_gpu_stream_expert_cache_clear_all(1); - if (g_ssd_streaming_mode) { + if (g_ssd_streaming_mode != was_ssd_streaming) { + ds4_gpu_stream_expert_cache_clear_all(1); + } + if (g_ssd_streaming_mode && !was_ssd_streaming) { fprintf(stderr, "ds4: Metal SSD streaming mode enabled; full model residency and warmup are skipped\n"); } @@ -11471,6 +11532,12 @@ void ds4_gpu_cleanup(void) { (int)g_q4_attn_q_b_f16_build_circuit_state); } ds4_gpu_q4_attn_q_b_f16_cache_clear(1); + pthread_mutex_lock(&g_q4_qb_transient_f16_scratch_mu); + for (int si = 0; si < DS4_GPU_MAX_STREAMS; si++) { + g_q4_qb_transient_f16_scratch[si] = nil; + g_q4_qb_transient_f16_scratch_capacity[si] = 0u; + } + pthread_mutex_unlock(&g_q4_qb_transient_f16_scratch_mu); ds4_gpu_stream_expert_pread_pool_shutdown(); ds4_gpu_stream_expert_cache_clear_all(1); ds4_gpu_stream_expert_cache_live_release(); @@ -14638,6 +14705,9 @@ static int ds4_gpu_stream_expert_alloc_slab_slot( slab = g_stream_expert_cache_slab_count++; g_stream_expert_cache_slabs[slab] = slab_buffer; + ds4_gpu_atomic_u64_add_sat( + &g_stream_expert_cache_slab_allocated_bytes, + (uint64_t)[slab_buffer length]); g_stream_expert_cache_slab_start_slot[slab] = g_stream_expert_cache_slab_total_slots; g_stream_expert_cache_slab_slot_count[slab] = slots; @@ -16199,6 +16269,9 @@ static void ds4_gpu_stream_expert_cache_clear_all(int reset_stats) { } g_stream_expert_cache_slab_count = 0; g_stream_expert_cache_slab_total_slots = 0; + __atomic_store_n(&g_stream_expert_cache_slab_allocated_bytes, + 0, + __ATOMIC_RELEASE); g_stream_expert_cache_free_slot_count = 0; g_stream_expert_cache_slab_slot_bytes = 0; memset(g_stream_expert_cache_slab_slot_locked, @@ -25102,6 +25175,14 @@ int ds4_gpu_head_rms_norm_rope_tail_tensor( if (!cb) return 0; id enc = ds4_gpu_compute_encoder(cb); + if (!enc) { + if (owned) { + (void)ds4_gpu_finish_command_buffer( + cb, owned, + "fused head norm/RoPE encoder allocation"); + } + return 0; + } [enc setComputePipelineState:pipeline]; [enc setBytes:&args length:sizeof(args) atIndex:0]; [enc setBuffer:xbuf offset:ds4_gpu_tensor_offset(x) atIndex:1]; @@ -25384,6 +25465,81 @@ static int ds4_gpu_q4_attn_q_b_f16_builds_suppressed(void) { return suppressed; } +/* SSD streaming normally rejects this resident expansion: 43 production + * attn_q_b matrices occupy about 2.69 GiB as F16. Keep the hybrid mode + * explicit until hardware A/B data shows that its compute saving outweighs + * the smaller unified-memory budget left for streamed experts. The regular + * cache DISABLE flag is checked separately and always wins. */ +static int ds4_gpu_q4_attn_q_b_f16_streaming_allowed(void) { + if (!g_ssd_streaming_mode) return 1; + return + __atomic_load_n(&g_q4_attn_q_b_f16_ssd_admission_blocked, + __ATOMIC_ACQUIRE) == 0 && + ds4_gpu_env_bool( + "DS4_METAL_ENABLE_Q4_ATTN_Q_B_F16_CACHE_WITH_SSD_STREAMING") == 1; +} + +/* currentAllocatedSize already contains streaming expert buffers that have + * been materialized by an earlier session. Credit those bytes against only + * the dynamic-cache portion of the host's future working-set reserve; graph, + * full-layer and prefill headroom must remain reserved. Slabs are counted by + * their full allocation rather than by occupied slots because Metal accounts + * the entire MTLBuffer in currentAllocatedSize. Deliberately do not credit + * standalone fallback buffers: unlike slabs, ordinary cache rotation may + * release them between this snapshot and admission. Double-counting that + * uncommon path is conservative; stale credit would not be. */ +static uint64_t ds4_gpu_stream_expert_cache_materialized_bytes(void) { + return __atomic_load_n( + &g_stream_expert_cache_slab_allocated_bytes, + __ATOMIC_ACQUIRE); +} + +static uint64_t ds4_gpu_q4_attn_q_b_f16_effective_working_set_reserve( + uint64_t planned_reserve, + uint64_t *materialized_credit) { + if (materialized_credit) *materialized_credit = 0; + if (!g_ssd_streaming_mode || planned_reserve == 0u) { + return planned_reserve; + } + + const uint32_t budget = + ds4_gpu_stream_expert_cache_configured_budget(); + if (budget == 0u || g_stream_expert_cache_expert_bytes == 0u) { + return planned_reserve; + } + const uint64_t dynamic_target = + budget > UINT64_MAX / g_stream_expert_cache_expert_bytes + ? UINT64_MAX + : (uint64_t)budget * g_stream_expert_cache_expert_bytes; + uint64_t credit = ds4_gpu_stream_expert_cache_materialized_bytes(); + if (credit > dynamic_target) credit = dynamic_target; + if (credit > planned_reserve) credit = planned_reserve; + if (materialized_credit) *materialized_credit = credit; + return planned_reserve - credit; +} + +/* The initial SSD model map intentionally covers token/static spans only. + * Wrap each dense q_b source as a short-lived exact no-copy view while its + * sidecar is built; retaining those ~18 MiB views in the general exact-view + * cache would pin another ~0.77 GiB of model mappings for no steady-state + * benefit. The caller keeps the returned owned buffer alive through command + * completion. */ +static id ds4_gpu_q4_attn_q_b_f16_wrap_source( + const void *model_map, + uint64_t model_size, + uint64_t weight_offset, + uint64_t weight_bytes, + uint64_t *inner_offset) { + if (g_ssd_streaming_mode) { + return ds4_gpu_wrap_model_exact_range_owned( + model_map, model_size, weight_offset, weight_bytes, + inner_offset); + } + return ds4_gpu_wrap_model_range( + model_map, model_size, weight_offset, weight_bytes, + inner_offset); +} + static void ds4_gpu_q4_attn_q_b_f16_suppress_builds( ds4_gpu_q4_attn_q_b_f16_circuit_state state) { if (state == DS4_Q4_ATTN_Q_B_F16_CIRCUIT_CLOSED) return; @@ -25502,6 +25658,120 @@ static int ds4_gpu_q4_attn_q_b_f16_abort_prewarm( return required ? -1 : 0; } +static int ds4_gpu_q4_qb_transient_f16_scratch_ensure_locked( + int stream, + uint64_t required_bytes); + +/* Resolve the complete transient path and reserve its one stream-local + * expansion buffer before prefill timing starts. Per-layer Q4->F16 work is + * intentionally not performed here: unlike PSO creation and allocation, that + * dequantization is a real recurring cost of the optimized projection. */ +static int ds4_gpu_prepare_q4_attn_q_b_transient_f16( + const void *model_map, + uint64_t model_size, + const ds4_gpu_q4_attn_q_b_f16_sidecar_desc *descs, + uint32_t count, + uint32_t max_prefill_rows, + uint64_t working_set_reserve_bytes, + uint64_t *prepared_bytes) { + const double prepare_t0 = ds4_gpu_now_ms(); + const uint64_t min_tokens = ds4_gpu_env_u64( + "DS4_METAL_Q4_ATTN_Q_B_TRANSIENT_F16_MIN_TOKENS", + 4096u, 32u, UINT32_MAX); + if (max_prefill_rows < min_tokens || + g_ssd_streaming_mode || g_quality_mode || + g_batch_encoder_concurrent || + !ds4_gpu_device_is_pre_m5_apple_silicon() || + ds4_gpu_mpp_available() || + g_ds4_stream < 0 || g_ds4_stream >= DS4_GPU_MAX_STREAMS) { + return 0; + } + + uint64_t f16_bytes = 0u; + for (uint32_t i = 0; i < count; i++) { + const ds4_gpu_q4_attn_q_b_f16_sidecar_desc *desc = &descs[i]; + if (desc->weight_type != DS4_METAL_TENSOR_Q4_K || + desc->in_dim != 1024u || desc->out_dim != 32768u) { + return 0; + } + const uint64_t row_bytes = + (desc->in_dim / 256u) * 144u; + const uint64_t expected_weight_bytes = + desc->out_dim * row_bytes; + const uint64_t desc_f16_bytes = + desc->in_dim * desc->out_dim * sizeof(uint16_t); + if (desc->weight_bytes != expected_weight_bytes || + desc->weight_offset > model_size || + desc->weight_bytes > model_size - desc->weight_offset || + (f16_bytes != 0u && f16_bytes != desc_f16_bytes)) { + return 0; + } + f16_bytes = desc_f16_bytes; + } + if (!model_map || f16_bytes == 0u || f16_bytes > NSUIntegerMax) { + return 0; + } + + id dequant_pipeline = + ds4_gpu_get_pipeline("kernel_dequantize_q4_K_f16"); + id mm_aligned_pipeline = + ds4_gpu_get_mul_mm_pipeline( + "kernel_mul_mm_f16_f16_rhs", false, false); + id mm_boundary_pipeline = + ds4_gpu_get_mul_mm_pipeline( + "kernel_mul_mm_f16_f16_rhs", false, true); + const int use_contiguous_copy = + ds4_gpu_env_bool( + "DS4_METAL_DISABLE_CONTIG_F32_F16_COPY") <= 0; + id copy_pipeline = use_contiguous_copy + ? g_cpy_contig_f32_f16_pipeline + : g_cpy_f32_f16_pipeline; + if (!dequant_pipeline || !mm_aligned_pipeline || + !mm_boundary_pipeline || !copy_pipeline || + dequant_pipeline.maxTotalThreadsPerThreadgroup < 64u || + mm_aligned_pipeline.threadExecutionWidth != 32u || + mm_aligned_pipeline.maxTotalThreadsPerThreadgroup < 128u || + mm_boundary_pipeline.threadExecutionWidth != 32u || + mm_boundary_pipeline.maxTotalThreadsPerThreadgroup < 128u) { + return 0; + } + + const int stream = g_ds4_stream; + bool allocated = false; + pthread_mutex_lock(&g_q4_qb_transient_f16_scratch_mu); + const bool needs_allocation = + !g_q4_qb_transient_f16_scratch[stream] || + g_q4_qb_transient_f16_scratch_capacity[stream] < + (NSUInteger)f16_bytes; + uint64_t admission_bytes = f16_bytes; + if (UINT64_MAX - admission_bytes < working_set_reserve_bytes) { + admission_bytes = UINT64_MAX; + } else { + admission_bytes += working_set_reserve_bytes; + } + if ((needs_allocation && + !ds4_gpu_q4_attn_q_b_f16_working_set_has_room( + admission_bytes)) || + !ds4_gpu_q4_qb_transient_f16_scratch_ensure_locked( + stream, f16_bytes)) { + pthread_mutex_unlock(&g_q4_qb_transient_f16_scratch_mu); + return 0; + } + allocated = needs_allocation; + pthread_mutex_unlock(&g_q4_qb_transient_f16_scratch_mu); + + if (allocated) { + if (prepared_bytes) *prepared_bytes = f16_bytes; + fprintf(stderr, + "ds4: Metal prepared %.2f MiB stream-local Q4 attn_q_b " + "transient F16 scratch and pipelines before prefill " + "in %.3f ms\n", + (double)f16_bytes / 1048576.0, + ds4_gpu_now_ms() - prepare_t0); + } + return 1; +} + int ds4_gpu_prepare_q4_attn_q_b_f16_sidecars( const void *model_map, uint64_t model_size, @@ -25520,8 +25790,26 @@ int ds4_gpu_prepare_q4_attn_q_b_f16_sidecars( const int required = ds4_gpu_env_bool("DS4_METAL_REQUIRE_Q4_ATTN_Q_B_F16_CACHE") == 1; + const int transient_disabled = + ds4_gpu_env_bool( + "DS4_METAL_DISABLE_Q4_ATTN_Q_B_TRANSIENT_F16") == 1; + const int ssd_sidecar_opt_in = + g_ssd_streaming_mode && + ds4_gpu_env_bool( + "DS4_METAL_ENABLE_Q4_ATTN_Q_B_F16_CACHE_WITH_SSD_STREAMING") == 1; + /* The 64 MiB transient production path is the resident pre-M5 default. + * Do not prewarm 43 persistent layer sidecars (~2.69 GiB) unless the + * operator explicitly selects the legacy path, makes it mandatory, or + * opts into the established SSD-streaming hybrid. */ + if (!required && !transient_disabled && !ssd_sidecar_opt_in) { + return ds4_gpu_prepare_q4_attn_q_b_transient_f16( + model_map, model_size, descs, count, max_prefill_rows, + working_set_reserve_bytes, prepared_bytes); + } const int disabled = ds4_gpu_env_bool("DS4_METAL_DISABLE_Q4_ATTN_Q_B_F16_CACHE") == 1; + const int disable_f16_rhs = + ds4_gpu_env_bool("DS4_METAL_DISABLE_Q4_ATTN_Q_B_F16_RHS") == 1; const uint64_t min_tokens = ds4_gpu_env_u64( "DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MIN_TOKENS", 512u, 32u, UINT32_MAX); @@ -25529,7 +25817,9 @@ int ds4_gpu_prepare_q4_attn_q_b_f16_sidecars( /* A session whose largest possible chunk is below the configured * threshold can never be a cache candidate, even in strict mode. */ if (max_prefill_rows < min_tokens) return 0; - if (disabled || g_ssd_streaming_mode || g_quality_mode || + if (disabled || + !ds4_gpu_q4_attn_q_b_f16_streaming_allowed() || + g_quality_mode || !ds4_gpu_device_is_pre_m5_apple_silicon() || ds4_gpu_mpp_available()) { return required ? -1 : 0; @@ -25568,6 +25858,7 @@ int ds4_gpu_prepare_q4_attn_q_b_f16_sidecars( uint32_t miss_indices[DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MAX_ENTRIES]; uint32_t miss_count = 0; uint64_t missing_bytes = 0; + uint64_t missing_source_bytes = 0; for (uint32_t i = 0; i < count; i++) { id hit_buffer = nil; const ds4_gpu_q4_attn_q_b_f16_sidecar_desc *desc = &descs[i]; @@ -25580,6 +25871,11 @@ int ds4_gpu_prepare_q4_attn_q_b_f16_sidecars( miss_indices[miss_count++] = i; missing_bytes += desc->in_dim * desc->out_dim * sizeof(uint16_t); + if (UINT64_MAX - missing_source_bytes < desc->weight_bytes) { + return ds4_gpu_q4_attn_q_b_f16_abort_prewarm( + required, 1, 0, DS4_Q4_ATTN_Q_B_F16_CIRCUIT_HARD); + } + missing_source_bytes += desc->weight_bytes; } if (miss_count == 0u) { pthread_mutex_unlock(&g_q4_attn_q_b_f16_build_mu); @@ -25602,9 +25898,21 @@ int ds4_gpu_prepare_q4_attn_q_b_f16_sidecars( return ds4_gpu_q4_attn_q_b_f16_abort_prewarm( required, 1, 0, DS4_Q4_ATTN_Q_B_F16_CIRCUIT_HARD); } - if (UINT64_MAX - missing_bytes < working_set_reserve_bytes || + const uint64_t temporary_source_reserve_bytes = + g_ssd_streaming_mode ? missing_source_bytes : 0u; + uint64_t materialized_streaming_credit_bytes = 0; + const uint64_t effective_working_set_reserve_bytes = + ds4_gpu_q4_attn_q_b_f16_effective_working_set_reserve( + working_set_reserve_bytes, + &materialized_streaming_credit_bytes); + if (UINT64_MAX - effective_working_set_reserve_bytes < + temporary_source_reserve_bytes || + UINT64_MAX - missing_bytes < + effective_working_set_reserve_bytes + + temporary_source_reserve_bytes || !ds4_gpu_q4_attn_q_b_f16_working_set_has_room( - missing_bytes + working_set_reserve_bytes)) { + missing_bytes + effective_working_set_reserve_bytes + + temporary_source_reserve_bytes)) { const uint64_t allocated = (uint64_t)[g_device currentAllocatedSize]; const uint64_t recommended = @@ -25612,11 +25920,17 @@ int ds4_gpu_prepare_q4_attn_q_b_f16_sidecars( fprintf(stderr, "ds4: Metal Q4 attn_q_b F16 prewarm skipped for " "working-set headroom: allocated %.2f GiB + sidecars " - "%.2f GiB + future sessions %.2f GiB, 7/8 safety " - "limit %.2f GiB\n", + "%.2f GiB + reserved working set %.2f GiB (planned %.2f, " + "%.2f already materialized) + temporary " + "SSD sources %.2f GiB, 7/8 safety limit %.2f GiB\n", (double)allocated / 1073741824.0, (double)missing_bytes / 1073741824.0, + (double)effective_working_set_reserve_bytes / + 1073741824.0, (double)working_set_reserve_bytes / 1073741824.0, + (double)materialized_streaming_credit_bytes / + 1073741824.0, + (double)temporary_source_reserve_bytes / 1073741824.0, (double)(recommended - recommended / 8u) / 1073741824.0); return ds4_gpu_q4_attn_q_b_f16_abort_prewarm( @@ -25635,6 +25949,14 @@ int ds4_gpu_prepare_q4_attn_q_b_f16_sidecars( id mm_boundary_pipeline = ds4_gpu_get_mul_mm_pipeline( "kernel_mul_mm_f16_f32", false, true); + /* The compact-RHS specialization is nested under the sidecar and must + * never make the established F16-weight/F32-RHS cache unavailable. */ + if (!disable_f16_rhs && !g_ssd_streaming_mode) { + (void)ds4_gpu_get_mul_mm_pipeline( + "kernel_mul_mm_f16_f16_rhs", false, false); + (void)ds4_gpu_get_mul_mm_pipeline( + "kernel_mul_mm_f16_f16_rhs", false, true); + } id build_cb = [ds4_gpu_active_queue() commandBuffer]; if (!dequant_pipeline || !mm_aligned_pipeline || @@ -25655,7 +25977,7 @@ int ds4_gpu_prepare_q4_attn_q_b_f16_sidecars( &descs[miss_indices[mi]]; const uint64_t f16_bytes = desc->in_dim * desc->out_dim * sizeof(uint16_t); - weights[mi] = ds4_gpu_wrap_model_range( + weights[mi] = ds4_gpu_q4_attn_q_b_f16_wrap_source( model_map, model_size, desc->weight_offset, desc->weight_bytes, &weight_inner[mi]); sidecars[mi] = [g_device @@ -25713,16 +26035,56 @@ int ds4_gpu_prepare_q4_attn_q_b_f16_sidecars( pthread_mutex_unlock(&g_q4_attn_q_b_f16_build_mu); if (prepared_bytes) *prepared_bytes = missing_bytes; + const uint64_t allocated_after_prewarm = + (uint64_t)[g_device currentAllocatedSize]; + const uint64_t recommended_working_set = + (uint64_t)[g_device recommendedMaxWorkingSetSize]; fprintf(stderr, - "ds4: Metal prewarmed %u resident Q4 attn_q_b F16 " - "sidecars (%.2f GiB) in %.3f ms; cache budget %llu MiB\n", + "ds4: Metal%s prewarmed %u resident Q4 attn_q_b F16 " + "sidecars (%.2f GiB) in %.3f ms; cache budget %llu MiB; " + "reserved working set %.2f GiB (planned %.2f, %.2f already " + "materialized); temporary SSD sources " + "%.2f GiB; Metal allocated %.2f / %.2f GiB recommended\n", + g_ssd_streaming_mode ? " SSD-streaming hybrid" : "", miss_count, (double)missing_bytes / 1073741824.0, ds4_gpu_now_ms() - build_t0, - (unsigned long long)budget_mib); + (unsigned long long)budget_mib, + (double)effective_working_set_reserve_bytes / 1073741824.0, + (double)working_set_reserve_bytes / 1073741824.0, + (double)materialized_streaming_credit_bytes / 1073741824.0, + (double)temporary_source_reserve_bytes / 1073741824.0, + (double)allocated_after_prewarm / 1073741824.0, + (double)recommended_working_set / 1073741824.0); return 1; } +static bool ds4_gpu_tensor_prefixes_overlap( + const ds4_gpu_tensor *a, + uint64_t a_bytes, + const ds4_gpu_tensor *b, + uint64_t b_bytes); +static int ds4_gpu_encode_f16_rhs_mm( + id cb, + id pipeline, + id weights, + NSUInteger weights_offset, + const ds4_gpu_tensor *rhs_f16, + ds4_gpu_tensor *out, + uint64_t in_dim, + uint64_t out_dim, + uint64_t n_tok, + uint64_t row_bytes, + bool bc_out); +static int ds4_gpu_encode_q4_K_transient_f16( + id cb, + id pipeline, + id q4_weights, + NSUInteger q4_weights_offset, + id f16_scratch, + uint64_t in_dim, + uint64_t out_dim); + static int ds4_gpu_attn_q_b_f16_head_rms_rope_tail_tensor_impl( ds4_gpu_tensor *out, ds4_gpu_tensor *q_half, @@ -25748,7 +26110,6 @@ static int ds4_gpu_attn_q_b_f16_head_rms_rope_tail_tensor_impl( float beta_slow, float eps, int apply_norm_rope) { - (void)q_half; if (weight_type != DS4_METAL_TENSOR_Q4_K) return 0; /* This path is prefill-only. Keep decode and tiny batches off the * environment parser, cache mutex, and pipeline lookup entirely. */ @@ -25759,6 +26120,11 @@ static int ds4_gpu_attn_q_b_f16_head_rms_rope_tail_tensor_impl( ds4_gpu_env_bool("DS4_METAL_REQUIRE_Q4_ATTN_Q_B_F16_CACHE") == 1; const int disabled = ds4_gpu_env_bool("DS4_METAL_DISABLE_Q4_ATTN_Q_B_F16_CACHE") == 1; + bool use_f16_rhs = + q_half != NULL && + !g_ssd_streaming_mode && + !g_batch_encoder_concurrent && + ds4_gpu_env_bool("DS4_METAL_DISABLE_Q4_ATTN_Q_B_F16_RHS") != 1; const uint64_t min_tokens = ds4_gpu_env_u64( "DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MIN_TOKENS", 512u, 32u, UINT32_MAX); @@ -25775,7 +26141,9 @@ static int ds4_gpu_attn_q_b_f16_head_rms_rope_tail_tensor_impl( /* The initial production arm targets DeepSeek-V4 Flash q_b exactly. * Larger variants need more than 10 GiB of sidecar storage and require a * separate memory/performance admission study. */ - if (disabled || g_ssd_streaming_mode || g_quality_mode || + if (disabled || + !ds4_gpu_q4_attn_q_b_f16_streaming_allowed() || + g_quality_mode || !ds4_gpu_device_is_pre_m5_apple_silicon() || ds4_gpu_mpp_available() || in_dim != 1024u || out_dim != 32768u || @@ -25797,6 +26165,26 @@ static int ds4_gpu_attn_q_b_f16_head_rms_rope_tail_tensor_impl( return ds4_gpu_q4_attn_q_b_f16_fallback(required, 1, 0); } + const uint64_t x_f32_bytes = + (uint64_t)n_tok * in_dim * sizeof(float); + const uint64_t x_f16_bytes = + (uint64_t)n_tok * in_dim * sizeof(uint16_t); + const uint64_t out_f32_bytes = + (uint64_t)n_tok * out_dim * sizeof(float); + if (use_f16_rhs && + ((uint64_t)n_tok * in_dim > UINT32_MAX || + ds4_gpu_tensor_bytes(q_half) < x_f16_bytes || + !ds4_gpu_tensor_buffer(q_half) || + ds4_gpu_tensor_prefixes_overlap( + q_half, x_f16_bytes, x, x_f32_bytes) || + ds4_gpu_tensor_prefixes_overlap( + q_half, x_f16_bytes, out, out_f32_bytes))) { + /* Compact F16 RHS staging is a nested optimization. An unusable + * scratch view must not disable the already-valid resident + * F16-weight/F32-RHS sidecar path (or make REQUIRE reject it). */ + use_f16_rhs = false; + } + const uint64_t blocks_per_row = in_dim / 256u; const uint64_t row_bytes = blocks_per_row * 144u; if (out_dim > UINT64_MAX / row_bytes) { @@ -25825,10 +26213,25 @@ static int ds4_gpu_attn_q_b_f16_head_rms_rope_tail_tensor_impl( id xbuf = ds4_gpu_tensor_buffer(x); id outbuf = ds4_gpu_tensor_buffer(out); const bool mm_bc_out = (n_tok & 31u) != 0u; - id mm_pipeline = + id mm_f32_pipeline = ds4_gpu_get_mul_mm_pipeline( "kernel_mul_mm_f16_f32", false, mm_bc_out); - if (!xbuf || !outbuf || !mm_pipeline) { + id mm_f16_pipeline = nil; + if (use_f16_rhs) { + mm_f16_pipeline = ds4_gpu_get_mul_mm_pipeline( + "kernel_mul_mm_f16_f16_rhs", false, mm_bc_out); + const int use_contiguous_copy = + ds4_gpu_env_bool( + "DS4_METAL_DISABLE_CONTIG_F32_F16_COPY") <= 0; + id copy_pipeline = + use_contiguous_copy + ? g_cpy_contig_f32_f16_pipeline + : g_cpy_f32_f16_pipeline; + if (!mm_f16_pipeline || !copy_pipeline) { + use_f16_rhs = false; + } + } + if (!xbuf || !outbuf || !mm_f32_pipeline) { return ds4_gpu_q4_attn_q_b_f16_fallback(required, 0, 1); } @@ -25857,8 +26260,11 @@ static int ds4_gpu_attn_q_b_f16_head_rms_rope_tail_tensor_impl( required, 1, 0, DS4_Q4_ATTN_Q_B_F16_CIRCUIT_HARD); } - if (!ds4_gpu_q4_attn_q_b_f16_working_set_has_room( - f16_bytes)) { + const uint64_t temporary_source_bytes = + g_ssd_streaming_mode ? weight_bytes : 0u; + if (UINT64_MAX - f16_bytes < temporary_source_bytes || + !ds4_gpu_q4_attn_q_b_f16_working_set_has_room( + f16_bytes + temporary_source_bytes)) { return ds4_gpu_q4_attn_q_b_f16_abort_cold_build( required, 1, 0, DS4_Q4_ATTN_Q_B_F16_CIRCUIT_PRESSURE); @@ -25873,7 +26279,8 @@ static int ds4_gpu_attn_q_b_f16_head_rms_rope_tail_tensor_impl( } uint64_t weight_inner = 0; - id weight_buffer = ds4_gpu_wrap_model_range( + id weight_buffer = + ds4_gpu_q4_attn_q_b_f16_wrap_source( model_map, model_size, weight_offset, weight_bytes, &weight_inner); if (!weight_buffer || f16_bytes > NSUIntegerMax) { @@ -25986,31 +26393,82 @@ static int ds4_gpu_attn_q_b_f16_head_rms_rope_tail_tensor_impl( return ds4_gpu_q4_attn_q_b_f16_fallback(required, 0, 1); } - ds4_gpu_mul_mm_args mm_args = ds4_gpu_make_mm_args( - in_dim, out_dim, n_tok, in_dim * sizeof(uint16_t)); - id enc = ds4_gpu_compute_encoder(cb); - if (!enc) { - (void)ds4_gpu_finish_command_buffer( - cb, owned, "Q4 attn_q_b cached F16 encoder allocation"); + bool mm_encoded = false; + if (use_f16_rhs) { + const bool rhs_encoded = + ds4_gpu_encode_cpy_f32_f16_1d( + cb, + xbuf, + ds4_gpu_tensor_offset(x), + ds4_gpu_tensor_buffer(q_half), + ds4_gpu_tensor_offset(q_half), + (uint32_t)((uint64_t)n_tok * in_dim)) != 0; + mm_encoded = + rhs_encoded && + ds4_gpu_encode_f16_rhs_mm( + cb, + mm_f16_pipeline, + f16_buffer, + 0, + q_half, + out, + in_dim, + out_dim, + n_tok, + in_dim * sizeof(uint16_t), + mm_bc_out) != 0; + if (!mm_encoded) { + /* Copy/encoder setup failed before any output writer was + * dispatched. Keep the sidecar useful by degrading inside + * this command buffer to its established F16/F32 consumer. */ + use_f16_rhs = false; + } + } + if (!use_f16_rhs) { + ds4_gpu_mul_mm_args mm_args = ds4_gpu_make_mm_args( + in_dim, out_dim, n_tok, in_dim * sizeof(uint16_t)); + id enc = + ds4_gpu_compute_encoder(cb); + if (enc) { + [enc setComputePipelineState:mm_f32_pipeline]; + [enc setBytes:&mm_args length:sizeof(mm_args) atIndex:0]; + [enc setBuffer:f16_buffer offset:0 atIndex:1]; + [enc setBuffer:xbuf + offset:ds4_gpu_tensor_offset(x) + atIndex:2]; + [enc setBuffer:outbuf + offset:ds4_gpu_tensor_offset(out) + atIndex:3]; + [enc setThreadgroupMemoryLength: + (mm_bc_out ? 8192u : 6144u) + atIndex:0]; + [enc dispatchThreadgroups:MTLSizeMake( + ((NSUInteger)n_tok + 31u) / 32u, + (NSUInteger)out_dim / 64u, + 1) + threadsPerThreadgroup:MTLSizeMake(128, 1, 1)]; + ds4_gpu_end_compute_encoder(cb, enc); + mm_encoded = true; + } + } + if (!mm_encoded) { + if (!ds4_gpu_finish_command_buffer( + cb, owned, + "Q4 attn_q_b cached F16 RHS encoder allocation")) { + return ds4_gpu_q4_attn_q_b_f16_fallback(1, 0, 1); + } return ds4_gpu_q4_attn_q_b_f16_fallback(required, 0, 1); } - [enc setComputePipelineState:mm_pipeline]; - [enc setBytes:&mm_args length:sizeof(mm_args) atIndex:0]; - [enc setBuffer:f16_buffer offset:0 atIndex:1]; - [enc setBuffer:xbuf offset:ds4_gpu_tensor_offset(x) atIndex:2]; - [enc setBuffer:outbuf offset:ds4_gpu_tensor_offset(out) atIndex:3]; - [enc setThreadgroupMemoryLength:(mm_bc_out ? 8192u : 6144u) - atIndex:0]; - [enc dispatchThreadgroups:MTLSizeMake( - ((NSUInteger)n_tok + 31u) / 32u, - (NSUInteger)out_dim / 64u, - 1) - threadsPerThreadgroup:MTLSizeMake(128, 1, 1)]; - ds4_gpu_end_compute_encoder(cb, enc); if (!ds4_gpu_finish_command_buffer( - cb, owned, "Q4 attn_q_b cached F16 matmul")) { - return ds4_gpu_q4_attn_q_b_f16_fallback(required, 0, 1); + cb, owned, + use_f16_rhs + ? "Q4 attn_q_b cached F16/F16 matmul" + : "Q4 attn_q_b cached F16/F32 matmul")) { + /* out has an encoded writer (and may already be committed). + * Returning the optional-path sentinel would replay native Q4 + * over partial/failed work, so this failure is always fatal. */ + return ds4_gpu_q4_attn_q_b_f16_fallback(1, 0, 1); } } @@ -26019,7 +26477,191 @@ static int ds4_gpu_attn_q_b_f16_head_rms_rope_tail_tensor_impl( out, n_tok, n_head, head_dim, n_rot, pos0, n_ctx_orig, inverse, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow, eps)) { - return ds4_gpu_q4_attn_q_b_f16_fallback(required, 0, 1); + /* The q_b matmul is already encoded. Do not authorize the caller to + * replay the native path after a downstream encode/submit failure. */ + return ds4_gpu_q4_attn_q_b_f16_fallback(1, 0, 1); + } + return 1; +} + +/* Resident pre-M5 production path: rebuild one Q4_K q_b matrix into a + * stream-local 64 MiB F16 buffer, consume it immediately with the compact + * F16 RHS, and reuse that allocation for the next layer. This removes the + * native kernel's per-output-tile dequantization without retaining a 64 MiB + * sidecar for every layer. */ +static int ds4_gpu_attn_q_b_transient_f16_head_rms_rope_tail_tensor( + ds4_gpu_tensor *out, + ds4_gpu_tensor *q_half, + const void *model_map, + uint64_t model_size, + uint64_t weight_offset, + uint32_t weight_type, + uint64_t in_dim, + uint64_t out_dim, + const ds4_gpu_tensor *x, + uint32_t n_tok, + uint32_t n_head, + uint32_t head_dim, + uint32_t n_rot, + uint32_t pos0, + uint32_t n_ctx_orig, + bool inverse, + float freq_base, + float freq_scale, + float ext_factor, + float attn_factor, + float beta_fast, + float beta_slow, + float eps) { + if (weight_type != DS4_METAL_TENSOR_Q4_K) return 0; + if (!g_initialized && !ds4_gpu_init()) return 0; + + const uint64_t min_tokens = ds4_gpu_env_u64( + "DS4_METAL_Q4_ATTN_Q_B_TRANSIENT_F16_MIN_TOKENS", + 4096u, 32u, UINT32_MAX); + if ((uint64_t)n_tok < min_tokens || + g_ssd_streaming_mode || g_quality_mode || + g_batch_encoder_concurrent || + !ds4_gpu_device_is_pre_m5_apple_silicon() || + ds4_gpu_mpp_available() || + in_dim != 1024u || out_dim != 32768u || + !out || !q_half || !x || !model_map || + n_head == 0u || head_dim == 0u || + out_dim != (uint64_t)n_head * head_dim || + n_rot > head_dim || (n_rot & 1u) != 0u || + n_tok > (uint32_t)INT32_MAX || + pos0 > (uint32_t)INT32_MAX - n_tok || + (uint64_t)n_tok * in_dim > UINT32_MAX) { + return 0; + } + + const uint64_t x_f32_bytes = + (uint64_t)n_tok * in_dim * sizeof(float); + const uint64_t x_f16_bytes = + (uint64_t)n_tok * in_dim * sizeof(uint16_t); + const uint64_t out_f32_bytes = + (uint64_t)n_tok * out_dim * sizeof(float); + id xbuf = ds4_gpu_tensor_buffer(x); + id qbuf = ds4_gpu_tensor_buffer(q_half); + id outbuf = ds4_gpu_tensor_buffer(out); + if (!xbuf || !qbuf || !outbuf || + ds4_gpu_tensor_bytes(x) < x_f32_bytes || + ds4_gpu_tensor_bytes(q_half) < x_f16_bytes || + ds4_gpu_tensor_bytes(out) < out_f32_bytes || + ds4_gpu_tensor_prefixes_overlap( + q_half, x_f16_bytes, x, x_f32_bytes) || + ds4_gpu_tensor_prefixes_overlap( + q_half, x_f16_bytes, out, out_f32_bytes)) { + return 0; + } + + const uint64_t row_bytes = (in_dim / 256u) * 144u; + const uint64_t weight_bytes = out_dim * row_bytes; + const uint64_t f16_bytes = + in_dim * out_dim * sizeof(uint16_t); + if (weight_offset > model_size || + weight_bytes > model_size - weight_offset || + f16_bytes > NSUIntegerMax || + g_ds4_stream < 0 || g_ds4_stream >= DS4_GPU_MAX_STREAMS) { + return 0; + } + + const bool mm_bc_out = (n_tok & 31u) != 0u; + id dequant_pipeline = + ds4_gpu_get_pipeline("kernel_dequantize_q4_K_f16"); + id mm_pipeline = + ds4_gpu_get_mul_mm_pipeline( + "kernel_mul_mm_f16_f16_rhs", false, mm_bc_out); + const int use_contiguous_copy = + ds4_gpu_env_bool( + "DS4_METAL_DISABLE_CONTIG_F32_F16_COPY") <= 0; + id copy_pipeline = use_contiguous_copy + ? g_cpy_contig_f32_f16_pipeline + : g_cpy_f32_f16_pipeline; + if (!dequant_pipeline || !mm_pipeline || !copy_pipeline || + dequant_pipeline.maxTotalThreadsPerThreadgroup < 64u || + mm_pipeline.threadExecutionWidth != 32u || + mm_pipeline.maxTotalThreadsPerThreadgroup < 128u) { + return 0; + } + + uint64_t weight_inner = 0u; + id weight_buffer = ds4_gpu_wrap_model_range( + model_map, model_size, weight_offset, weight_bytes, + &weight_inner); + if (!weight_buffer || weight_inner > NSUIntegerMax) return 0; + + const int stream = g_ds4_stream; + id scratch = nil; + pthread_mutex_lock(&g_q4_qb_transient_f16_scratch_mu); + const bool needs_allocation = + !g_q4_qb_transient_f16_scratch[stream] || + g_q4_qb_transient_f16_scratch_capacity[stream] < + (NSUInteger)f16_bytes; + if ((needs_allocation && + !ds4_gpu_q4_attn_q_b_f16_working_set_has_room(f16_bytes)) || + !ds4_gpu_q4_qb_transient_f16_scratch_ensure_locked( + stream, f16_bytes)) { + pthread_mutex_unlock(&g_q4_qb_transient_f16_scratch_mu); + return 0; + } + scratch = g_q4_qb_transient_f16_scratch[stream]; + pthread_mutex_unlock(&g_q4_qb_transient_f16_scratch_mu); + + int owned = 0; + id cb = ds4_gpu_command_buffer(&owned); + if (!cb) return 0; + + /* Copy may share the caller's current serial batch encoder. Close it + * before dequantization, then close the dequant encoder as well: each + * following layer can overwrite the same scratch only after the previous + * F16/F16 consumer has completed in command-buffer order. */ + if (!ds4_gpu_encode_cpy_f32_f16_1d( + cb, xbuf, ds4_gpu_tensor_offset(x), + qbuf, ds4_gpu_tensor_offset(q_half), + (uint32_t)((uint64_t)n_tok * in_dim))) { + if (owned) { + (void)ds4_gpu_finish_command_buffer( + cb, owned, "Q4 q_b transient F16 RHS encode failure"); + } + return 0; + } + if (!owned) ds4_gpu_close_batch_encoder(); + + if (!ds4_gpu_encode_q4_K_transient_f16( + cb, dequant_pipeline, weight_buffer, + (NSUInteger)weight_inner, scratch, in_dim, out_dim)) { + if (owned) { + (void)ds4_gpu_finish_command_buffer( + cb, owned, "Q4 q_b transient dequant encode failure"); + } + return 0; + } + if (!owned) ds4_gpu_close_batch_encoder(); + + /* A true return from the helper means the output-writing matmul dispatch + * has been encoded. From this point onward the native Q4 path must not + * replay the projection, even if submission or the downstream tail fails. */ + if (!ds4_gpu_encode_f16_rhs_mm( + cb, mm_pipeline, scratch, 0u, q_half, out, + in_dim, out_dim, n_tok, + in_dim * sizeof(uint16_t), mm_bc_out)) { + if (owned) { + (void)ds4_gpu_finish_command_buffer( + cb, owned, "Q4 q_b transient F16/F16 encode failure"); + } + return 0; + } + if (!ds4_gpu_finish_command_buffer( + cb, owned, "Q4 q_b transient F16/F16 matmul")) { + return -1; + } + + if (!ds4_gpu_head_rms_norm_rope_tail_tensor( + out, n_tok, n_head, head_dim, n_rot, pos0, n_ctx_orig, + inverse, freq_base, freq_scale, ext_factor, attn_factor, + beta_fast, beta_slow, eps)) { + return -1; } return 1; } @@ -26048,6 +26690,28 @@ int ds4_gpu_attn_q_b_f16_head_rms_rope_tail_tensor( float beta_fast, float beta_slow, float eps) { + const int required = + ds4_gpu_env_bool( + "DS4_METAL_REQUIRE_Q4_ATTN_Q_B_F16_CACHE") == 1; + const int transient_disabled = + ds4_gpu_env_bool( + "DS4_METAL_DISABLE_Q4_ATTN_Q_B_TRANSIENT_F16") == 1; + const int ssd_sidecar_opt_in = + g_ssd_streaming_mode && + ds4_gpu_env_bool( + "DS4_METAL_ENABLE_Q4_ATTN_Q_B_F16_CACHE_WITH_SSD_STREAMING") == 1; + if (!required && !transient_disabled && !ssd_sidecar_opt_in) { + /* Default mode never falls through to the multi-GiB sidecar cache. + * A non-candidate or any failure before the output writer returns the + * native-Q4 fallback sentinel directly. The explicit SSD hybrid + * remains on its persistent sidecar path because transient scratch is + * deliberately resident-only. */ + return ds4_gpu_attn_q_b_transient_f16_head_rms_rope_tail_tensor( + out, q_half, model_map, model_size, weight_offset, weight_type, + in_dim, out_dim, x, n_tok, n_head, head_dim, n_rot, pos0, + n_ctx_orig, inverse, freq_base, freq_scale, ext_factor, + attn_factor, beta_fast, beta_slow, eps); + } return ds4_gpu_attn_q_b_f16_head_rms_rope_tail_tensor_impl( out, q_half, model_map, model_size, weight_offset, weight_type, in_dim, out_dim, x, n_tok, n_head, head_dim, n_rot, pos0, @@ -26071,6 +26735,314 @@ int ds4_gpu_test_q4_attn_q_b_f16_projection_tensor( 10000.0f, 1.0f, 0.0f, 1.0f, 32.0f, 1.0f, 1.0e-6f, 0); } +/* Allocate or grow one stream's private transient buffer. The caller holds + * the metadata mutex only through publication and captures a strong reference + * before unlocking; GPU execution itself never serializes on this mutex. */ +static int ds4_gpu_q4_qb_transient_f16_scratch_ensure_locked( + int stream, + uint64_t required_bytes) { + if (!g_device || stream < 0 || stream >= DS4_GPU_MAX_STREAMS || + required_bytes == 0u || + required_bytes > NSUIntegerMax) { + return 0; + } + if (g_q4_qb_transient_f16_scratch[stream] && + g_q4_qb_transient_f16_scratch_capacity[stream] >= + (NSUInteger)required_bytes) { + return 1; + } + + id scratch = [g_device + newBufferWithLength:(NSUInteger)required_bytes + options:MTLResourceStorageModePrivate]; + if (!scratch) return 0; + scratch.label = [NSString stringWithFormat: + @"ds4 Q4 q_b transient F16 scratch stream %d", stream]; + g_q4_qb_transient_f16_scratch[stream] = scratch; + g_q4_qb_transient_f16_scratch_capacity[stream] = + (NSUInteger)required_bytes; + return 1; +} + +/* Expand Q4_K using the exact production sidecar kernel. Ending this encoder + * before opening the F16/F16 matmul encoder establishes the required Metal + * dependency in one command buffer without a CPU wait or an intermediate + * allocation. */ +static int ds4_gpu_encode_q4_K_transient_f16( + id cb, + id pipeline, + id q4_weights, + NSUInteger q4_weights_offset, + id f16_scratch, + uint64_t in_dim, + uint64_t out_dim) { + if (!cb || !pipeline || !q4_weights || !f16_scratch || + in_dim == 0u || out_dim == 0u || + (in_dim % 256u) != 0u || + in_dim > UINT32_MAX || out_dim > UINT32_MAX) { + return 0; + } + + const uint32_t chunks_per_row = (uint32_t)(in_dim / 16u); + const uint32_t row_count = (uint32_t)out_dim; + id enc = ds4_gpu_compute_encoder(cb); + if (!enc) return 0; + [enc setComputePipelineState:pipeline]; + [enc setBuffer:q4_weights offset:q4_weights_offset atIndex:0]; + [enc setBuffer:f16_scratch offset:0 atIndex:1]; + [enc setBytes:&chunks_per_row + length:sizeof(chunks_per_row) + atIndex:2]; + [enc setBytes:&row_count length:sizeof(row_count) atIndex:3]; + [enc dispatchThreadgroups:MTLSizeMake( + ((NSUInteger)chunks_per_row + 63u) / 64u, + (NSUInteger)row_count, + 1) + threadsPerThreadgroup:MTLSizeMake(64, 1, 1)]; + ds4_gpu_end_compute_encoder(cb, enc); + return 1; +} + +int ds4_gpu_test_q4_attn_q_b_mm_arm_supported( + ds4_gpu_test_q4_qb_mm_arm arm) { + if (!g_initialized && !ds4_gpu_init()) return 0; + if (arm != DS4_GPU_TEST_Q4_QB_MM_Q4_TRANSIENT_F16_F16 || + g_batch_cb || g_batch_encoder_concurrent) { + return 0; + } + id dequant_pipeline = + ds4_gpu_get_pipeline("kernel_dequantize_q4_K_f16"); + id mm_pipeline = + ds4_gpu_get_mul_mm_pipeline( + "kernel_mul_mm_f16_f16_rhs", false, false); + return dequant_pipeline && mm_pipeline && + dequant_pipeline.maxTotalThreadsPerThreadgroup >= 64u && + mm_pipeline.threadExecutionWidth == 32u && + mm_pipeline.maxTotalThreadsPerThreadgroup >= 128u; +} + +int ds4_gpu_test_q4_attn_q_b_mm_variant_tensor( + ds4_gpu_tensor *out_f32, + ds4_gpu_tensor *rhs_f16, + const void *model_map, + uint64_t model_size, + uint64_t weight_offset, + uint64_t in_dim, + uint64_t out_dim, + const ds4_gpu_tensor *x_f32, + uint32_t n_tok, + ds4_gpu_test_q4_qb_mm_arm arm, + bool materialize_rhs) { + if (!g_initialized && !ds4_gpu_init()) return 0; + if (!out_f32 || !x_f32 || !model_map || n_tok == 0u || + arm < DS4_GPU_TEST_Q4_QB_MM_Q4_F32 || + arm >= DS4_GPU_TEST_Q4_QB_MM_ARM_COUNT || + in_dim == 0u || out_dim == 0u || + in_dim > INT32_MAX || out_dim > INT32_MAX || + n_tok > (uint32_t)INT32_MAX || + (in_dim % 256u) != 0u || + n_tok > UINT32_MAX / in_dim || + out_dim > UINT64_MAX / n_tok / sizeof(float)) { + return 0; + } + + const bool weight_f16 = + arm == DS4_GPU_TEST_Q4_QB_MM_F16_F32 || + arm == DS4_GPU_TEST_Q4_QB_MM_F16_F16; + const bool rhs_is_f16 = + arm == DS4_GPU_TEST_Q4_QB_MM_Q4_F16 || + arm == DS4_GPU_TEST_Q4_QB_MM_F16_F16 || + arm == DS4_GPU_TEST_Q4_QB_MM_Q4_TRANSIENT_F16_F16; + const bool transient_f16 = + arm == DS4_GPU_TEST_Q4_QB_MM_Q4_TRANSIENT_F16_F16; + const uint64_t q4_row_bytes = (in_dim / 256u) * 144u; + if (out_dim > UINT64_MAX / q4_row_bytes) return 0; + const uint64_t q4_weight_bytes = out_dim * q4_row_bytes; + if (weight_offset > model_size || + q4_weight_bytes > model_size - weight_offset) { + return 0; + } + + const uint64_t x_f32_bytes = + (uint64_t)n_tok * in_dim * sizeof(float); + const uint64_t rhs_f16_bytes = + (uint64_t)n_tok * in_dim * sizeof(uint16_t); + const uint64_t out_f32_bytes = + (uint64_t)n_tok * out_dim * sizeof(float); + if (ds4_gpu_tensor_bytes(x_f32) < x_f32_bytes || + ds4_gpu_tensor_bytes(out_f32) < out_f32_bytes || + (rhs_is_f16 && + (!rhs_f16 || + ds4_gpu_tensor_bytes(rhs_f16) < rhs_f16_bytes || + ds4_gpu_tensor_prefixes_overlap( + rhs_f16, rhs_f16_bytes, x_f32, x_f32_bytes) || + ds4_gpu_tensor_prefixes_overlap( + rhs_f16, rhs_f16_bytes, out_f32, out_f32_bytes)))) { + return 0; + } + + @autoreleasepool { + id weight_buffer = nil; + uint64_t weight_inner = 0u; + uint64_t weight_row_bytes = q4_row_bytes; + if (weight_f16) { + if (!ds4_gpu_q4_attn_q_b_f16_cache_peek( + model_map, model_size, weight_offset, + q4_weight_bytes, in_dim, out_dim, + &weight_buffer) || !weight_buffer) { + return 0; + } + weight_row_bytes = in_dim * sizeof(uint16_t); + } else { + weight_buffer = ds4_gpu_wrap_model_range( + model_map, model_size, weight_offset, q4_weight_bytes, + &weight_inner); + if (!weight_buffer || weight_inner > NSUIntegerMax) return 0; + } + + const char *pipeline_name = NULL; + switch (arm) { + case DS4_GPU_TEST_Q4_QB_MM_Q4_F32: + pipeline_name = "kernel_mul_mm_q4_K_f32"; + break; + case DS4_GPU_TEST_Q4_QB_MM_Q4_F16: + pipeline_name = "kernel_mul_mm_q4_K_f16_rhs"; + break; + case DS4_GPU_TEST_Q4_QB_MM_F16_F32: + pipeline_name = "kernel_mul_mm_f16_f32"; + break; + case DS4_GPU_TEST_Q4_QB_MM_F16_F16: + pipeline_name = "kernel_mul_mm_f16_f16_rhs"; + break; + case DS4_GPU_TEST_Q4_QB_MM_Q4_TRANSIENT_F16_F16: + pipeline_name = "kernel_mul_mm_f16_f16_rhs"; + break; + default: + return 0; + } + const bool bc_out = + ((out_dim % 64u) != 0u || (n_tok % 32u) != 0u); + id pipeline = + ds4_gpu_get_mul_mm_pipeline(pipeline_name, false, bc_out); + id dequant_pipeline = transient_f16 + ? ds4_gpu_get_pipeline("kernel_dequantize_q4_K_f16") + : nil; + id x_buffer = ds4_gpu_tensor_buffer(x_f32); + id out_buffer = ds4_gpu_tensor_buffer(out_f32); + if (!pipeline || !x_buffer || !out_buffer || + (transient_f16 && + (!dequant_pipeline || + dequant_pipeline.maxTotalThreadsPerThreadgroup < 64u || + pipeline.threadExecutionWidth != 32u || + pipeline.maxTotalThreadsPerThreadgroup < 128u))) { + return 0; + } + + uint64_t transient_f16_bytes = 0u; + id transient_scratch = nil; + if (transient_f16) { + if (out_dim > UINT64_MAX / in_dim / + sizeof(uint16_t) || + g_batch_cb || g_batch_encoder_concurrent || + g_ds4_stream < 0 || + g_ds4_stream >= DS4_GPU_MAX_STREAMS) { + return 0; + } + transient_f16_bytes = + in_dim * out_dim * sizeof(uint16_t); + const int stream = g_ds4_stream; + pthread_mutex_lock(&g_q4_qb_transient_f16_scratch_mu); + /* Recheck after taking the lock: another test thread may have + * waited while a batch was opened. */ + if (g_batch_cb || g_batch_encoder_concurrent || + !ds4_gpu_q4_qb_transient_f16_scratch_ensure_locked( + stream, transient_f16_bytes)) { + pthread_mutex_unlock( + &g_q4_qb_transient_f16_scratch_mu); + return 0; + } + transient_scratch = + g_q4_qb_transient_f16_scratch[stream]; + pthread_mutex_unlock( + &g_q4_qb_transient_f16_scratch_mu); + } + + int owned = 0; + id cb = ds4_gpu_command_buffer(&owned); + if (!cb || (transient_f16 && !owned)) { + return 0; + } + + bool encoded = true; + if (rhs_is_f16 && materialize_rhs) { + encoded = ds4_gpu_encode_cpy_f32_f16_1d( + cb, + x_buffer, + ds4_gpu_tensor_offset(x_f32), + ds4_gpu_tensor_buffer(rhs_f16), + ds4_gpu_tensor_offset(rhs_f16), + (uint32_t)((uint64_t)n_tok * in_dim)) != 0; + } + id mm_weight_buffer = weight_buffer; + NSUInteger mm_weight_offset = (NSUInteger)weight_inner; + uint64_t mm_weight_row_bytes = weight_row_bytes; + if (encoded && transient_f16) { + encoded = ds4_gpu_encode_q4_K_transient_f16( + cb, dequant_pipeline, weight_buffer, + (NSUInteger)weight_inner, transient_scratch, + in_dim, out_dim) != 0; + if (encoded) { + mm_weight_buffer = transient_scratch; + mm_weight_offset = 0u; + mm_weight_row_bytes = + in_dim * sizeof(uint16_t); + } + } + if (encoded && rhs_is_f16) { + encoded = ds4_gpu_encode_f16_rhs_mm( + cb, pipeline, mm_weight_buffer, mm_weight_offset, + rhs_f16, out_f32, in_dim, out_dim, n_tok, + mm_weight_row_bytes, bc_out) != 0; + } else if (encoded) { + ds4_gpu_mul_mm_args args = ds4_gpu_make_mm_args( + in_dim, out_dim, n_tok, weight_row_bytes); + id enc = + ds4_gpu_compute_encoder(cb); + if (!enc) { + encoded = false; + } else { + [enc setComputePipelineState:pipeline]; + [enc setBytes:&args length:sizeof(args) atIndex:0]; + [enc setBuffer:weight_buffer + offset:(NSUInteger)weight_inner + atIndex:1]; + [enc setBuffer:x_buffer + offset:ds4_gpu_tensor_offset(x_f32) + atIndex:2]; + [enc setBuffer:out_buffer + offset:ds4_gpu_tensor_offset(out_f32) + atIndex:3]; + [enc setThreadgroupMemoryLength:(bc_out ? 8192u : 6144u) + atIndex:0]; + [enc dispatchThreadgroups:MTLSizeMake( + ((NSUInteger)n_tok + 31u) / 32u, + ((NSUInteger)out_dim + 63u) / 64u, + 1) + threadsPerThreadgroup:MTLSizeMake(128, 1, 1)]; + ds4_gpu_end_compute_encoder(cb, enc); + } + } + + if (!encoded) { + (void)ds4_gpu_finish_command_buffer( + cb, owned, "Q4 q_b four-way benchmark encode failure"); + return 0; + } + return ds4_gpu_finish_command_buffer( + cb, owned, "Q4 q_b four-way resident benchmark"); + } +} + int ds4_gpu_dsv4_fp8_kv_quantize_tensor( ds4_gpu_tensor *x, uint32_t n_tok, @@ -29588,9 +30560,6 @@ static int ds4_gpu_attention_output_q4_K_ssd_prefill_exactn_tensor( } } -static bool ds4_gpu_tensor_prefixes_overlap(const ds4_gpu_tensor *a, uint64_t a_bytes, const ds4_gpu_tensor *b, uint64_t b_bytes); -static int ds4_gpu_encode_q4_K_f16_rhs_mm(id cb, id pipeline, id weights, NSUInteger weights_offset, const ds4_gpu_tensor *rhs_f16, ds4_gpu_tensor *out, uint64_t in_dim, uint64_t out_dim, uint64_t n_tok, uint64_t row_bytes, bool bc_out); - int ds4_gpu_attention_output_q4_K_batch_tensor( ds4_gpu_tensor *out, ds4_gpu_tensor *low, @@ -29964,7 +30933,7 @@ int ds4_gpu_attention_output_q4_K_batch_tensor( ds4_gpu_tensor_buffer(group_tmp), ds4_gpu_tensor_offset(group_tmp), (uint32_t)f16_rhs_elements) != 0 && - ds4_gpu_encode_q4_K_f16_rhs_mm( + ds4_gpu_encode_f16_rhs_mm( cb, f16_rhs_pipeline, out_b_buf, @@ -50614,14 +51583,12 @@ static bool ds4_gpu_tensor_prefixes_overlap( : a_off - b_off < b_bytes; } -/* Encode the legacy Q4_K prefill matmul against an RHS that has already been - * rounded to F16. kernel_mul_mm_q4_K_f32 performs that same rounding while - * staging every 64-row output tile, so materializing it once can remove a - * large amount of repeated activation traffic without changing the MMA or +/* Encode a dense or quantized prefill matmul against an RHS that has already + * been rounded to F16. Materializing that RHS once avoids repeating the + * conversion for every 64-row output tile without changing the MMA or F32 * accumulation schedule. This helper deliberately does not submit the - * command buffer: attention-A, the conversion, and output-B must remain in - * one ordered batch. */ -static int ds4_gpu_encode_q4_K_f16_rhs_mm( + * command buffer so callers can keep conversion and consumers ordered. */ +static int ds4_gpu_encode_f16_rhs_mm( id cb, id pipeline, id weights, @@ -50635,8 +51602,9 @@ static int ds4_gpu_encode_q4_K_f16_rhs_mm( bool bc_out) { if (!cb || !pipeline || !weights || !rhs_f16 || !out || in_dim == 0 || out_dim == 0 || n_tok == 0 || - (in_dim % 256u) != 0 || (n_tok % 32u) != 0 || - in_dim > UINT32_MAX || out_dim > UINT32_MAX || n_tok > UINT32_MAX || + (in_dim % 256u) != 0 || + in_dim > INT32_MAX || out_dim > INT32_MAX || n_tok > INT32_MAX || + (row_bytes != 0 && out_dim > UINT64_MAX / row_bytes) || in_dim > UINT64_MAX / n_tok || in_dim * n_tok > UINT64_MAX / sizeof(uint16_t) || out_dim > UINT64_MAX / n_tok || @@ -50670,7 +51638,7 @@ static int ds4_gpu_encode_q4_K_f16_rhs_mm( [enc setBuffer:out_buf offset:ds4_gpu_tensor_offset(out) atIndex:3]; [enc setThreadgroupMemoryLength:(bc_out ? 8192u : 6144u) atIndex:0]; [enc dispatchThreadgroups: - MTLSizeMake((NSUInteger)n_tok / 32u, + MTLSizeMake(((NSUInteger)n_tok + 31u) / 32u, ((NSUInteger)out_dim + 63u) / 64u, 1) threadsPerThreadgroup:MTLSizeMake(128, 1, 1)]; diff --git a/metal/dense.metal b/metal/dense.metal index ecb58729b..490047181 100644 --- a/metal/dense.metal +++ b/metal/dense.metal @@ -2492,6 +2492,11 @@ typedef decltype(kernel_mul_mm; +// Resident Q4 sidecars and stream-local transient expansions can pair their +// F16 weights with an RHS rounded to F16 once by the producer. This retains +// the exact legacy simdgroup-MMA and FP32 accumulation schedule while avoiding +// the repeated F32 load/conversion performed once per 64-row output tile. +template [[host_name("kernel_mul_mm_f16_f16_rhs")]] kernel mul_mm_t kernel_mul_mm; template [[host_name("kernel_mul_mm_q8_0_f32")]] kernel mul_mm_t kernel_mul_mm; template [[host_name("kernel_mul_mm_q4_0_f32")]] kernel mul_mm_t kernel_mul_mm; template [[host_name("kernel_mul_mm_q4_K_f32")]] kernel mul_mm_t kernel_mul_mm; From 8dbfa398b34de613fa12345d1672d36a303d1da7 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Wed, 26 Aug 2026 18:08:14 +0200 Subject: [PATCH 119/189] cuda, rocm: add transient F16 q_b prefill acceleration --- ds4_cuda.cu | 550 ++++++++++++++++++++++++++++---- rocm/ds4_rocm_norm_rope.cuh | 224 +++++++++++-- rocm/ds4_rocm_q4_qb_sidecar.cuh | 433 +++++++++++++++++++++++-- 3 files changed, 1084 insertions(+), 123 deletions(-) diff --git a/ds4_cuda.cu b/ds4_cuda.cu index 749c15930..597e0ff03 100644 --- a/ds4_cuda.cu +++ b/ds4_cuda.cu @@ -550,6 +550,17 @@ static uint64_t g_q4_attn_q_b_f16_generation = 1u; static int g_q4_attn_q_b_f16_hard_disabled; static int g_q4_attn_q_b_f16_dispatch_disabled; static int g_q4_attn_q_b_f16_pending_evict; +/* The long-prefill transient path expands exactly one q_b matrix at a time. + * Keep its weight and activation staging in one dedicated allocation so it + * cannot alias the legacy global CUDA scratch used by unrelated projections. + * A single mutex covers both ownership and the complete enqueue sequence: the + * NULL stream orders GPU work, while the mutex prevents two host threads from + * interleaving dequantize -> GEMM sequences that reuse this storage. */ +static __half *g_q4_attn_q_b_transient_f16_scratch; +static uint64_t g_q4_attn_q_b_transient_f16_scratch_bytes; +static int g_q4_attn_q_b_transient_f16_scratch_device = -1; +static int g_q4_attn_q_b_transient_f16_runtime_disabled; +static std::mutex g_q4_attn_q_b_transient_f16_mutex; /* Support-model residency means two independent GGUF mappings coexist. * Keep the large q_b expansion disabled in that conservative mode. */ static int g_q4_attn_q_b_f16_multi_model_active; @@ -2484,6 +2495,64 @@ static int cuda_q4_attn_q_b_f16_disabled(void) { getenv("DS4_CUDA_DISABLE_Q4_ATTN_Q_B_F16_CACHE")); } +static uint32_t cuda_q4_attn_q_b_transient_f16_min_tokens(void) { + int present = 0; + return cuda_parse_u32_env_clamped( + "DS4_CUDA_Q4_ATTN_Q_B_TRANSIENT_F16_MIN_TOKENS", + 4096u, 32u, UINT32_MAX, &present); +} + +static int cuda_q4_attn_q_b_transient_f16_disabled(void) { + return cuda_env_value_enabled( + getenv("DS4_CUDA_DISABLE_Q4_ATTN_Q_B_TRANSIENT_F16")); +} + +static int cuda_q4_attn_q_b_transient_f16_scratch_size( + uint32_t max_rows, + uint64_t *weight_bytes, + uint64_t *activation_offset, + uint64_t *scratch_bytes) { + const uint64_t wh_bytes = + (uint64_t)CUDA_Q4_ATTN_Q_B_IN_DIM * + CUDA_Q4_ATTN_Q_B_OUT_DIM * sizeof(__half); + const uint64_t xh_off = (wh_bytes + 255u) & ~UINT64_C(255); + const uint64_t row_bytes = + (uint64_t)CUDA_Q4_ATTN_Q_B_IN_DIM * sizeof(__half); + if (max_rows == 0u || + (uint64_t)max_rows > (UINT64_MAX - xh_off) / row_bytes) { + return 0; + } + const uint64_t total = xh_off + (uint64_t)max_rows * row_bytes; + if (total > SIZE_MAX) return 0; + if (weight_bytes) *weight_bytes = wh_bytes; + if (activation_offset) *activation_offset = xh_off; + if (scratch_bytes) *scratch_bytes = total; + return 1; +} + +/* The automatic path is intentionally limited to physical device storage. + * cuda_model_range_ptr may also expose managed/HMM or mapped host memory; a + * per-layer dequantization from either source would fold migration/PCIe cost + * into prefill and could be much slower than the native Q4 kernel. */ +static int cuda_q4_attn_q_b_source_is_device_resident( + const void *ptr, + int expected_device) { + if (!ptr) return 0; + cudaPointerAttributes attr = {}; + const cudaError_t err = cudaPointerGetAttributes(&attr, ptr); + if (err != cudaSuccess) { + (void)cudaGetLastError(); + return 0; + } +#if CUDART_VERSION >= 10000 + return attr.type == cudaMemoryTypeDevice && + attr.device == expected_device; +#else + return attr.memoryType == cudaMemoryTypeDevice && + attr.device == expected_device; +#endif +} + /* Match the existing CUDA weight-cache safety floor without coupling this * cache to the Q8-specific reserve environment variable. The explicit * future-session reserve supplied by ds4.c is added independently. */ @@ -2501,6 +2570,165 @@ static uint64_t cuda_q4_attn_q_b_f16_reserve_bytes(uint64_t total_bytes) { return pct_reserve > min_reserve ? pct_reserve : min_reserve; } +static int cuda_q4_attn_q_b_transient_f16_scratch_release(void) { + std::lock_guard lock( + g_q4_attn_q_b_transient_f16_mutex); + if (!g_q4_attn_q_b_transient_f16_scratch) { + g_q4_attn_q_b_transient_f16_scratch_bytes = 0; + g_q4_attn_q_b_transient_f16_scratch_device = -1; + g_q4_attn_q_b_transient_f16_runtime_disabled = 0; + return 1; + } + + int previous_device = -1; + (void)cudaGetDevice(&previous_device); + const int owner = g_q4_attn_q_b_transient_f16_scratch_device; + if (owner < 0 || cudaSetDevice(owner) != cudaSuccess || + cudaDeviceSynchronize() != cudaSuccess) { + (void)cudaGetLastError(); + if (previous_device >= 0) (void)cudaSetDevice(previous_device); + return 0; + } + if (cudaFree(g_q4_attn_q_b_transient_f16_scratch) != cudaSuccess) { + (void)cudaGetLastError(); + if (previous_device >= 0) (void)cudaSetDevice(previous_device); + return 0; + } + g_q4_attn_q_b_transient_f16_scratch = NULL; + g_q4_attn_q_b_transient_f16_scratch_bytes = 0; + g_q4_attn_q_b_transient_f16_scratch_device = -1; + g_q4_attn_q_b_transient_f16_runtime_disabled = 0; + { + std::lock_guard cache_lock( + g_q4_attn_q_b_f16_cache_mutex); + if (++g_q4_attn_q_b_f16_generation == 0u) { + g_q4_attn_q_b_f16_generation = 1u; + } + } + if (previous_device >= 0) (void)cudaSetDevice(previous_device); + return 1; +} + +static int cuda_q4_attn_q_b_transient_f16_scratch_ensure( + uint32_t max_rows, + uint64_t working_set_reserve_bytes, + uint64_t *prepared_bytes) { + uint64_t scratch_bytes = 0; + if (!cuda_q4_attn_q_b_transient_f16_scratch_size( + max_rows, NULL, NULL, &scratch_bytes)) { + return 0; + } + + std::lock_guard lock( + g_q4_attn_q_b_transient_f16_mutex); + const int target_device = g_gpu[0].device_id; + if (g_q4_attn_q_b_transient_f16_scratch && + g_q4_attn_q_b_transient_f16_scratch_device == target_device && + g_q4_attn_q_b_transient_f16_scratch_bytes >= scratch_bytes) { + return 1; + } + /* A CUDA backend instance does not migrate its single-GPU arena between + * physical devices. Cleanup/reinit owns that transition; preserving the + * old allocation is safer than partially replacing it here. */ + if (g_q4_attn_q_b_transient_f16_scratch && + g_q4_attn_q_b_transient_f16_scratch_device != target_device) { + return 0; + } + + int previous_device = -1; + if (cudaGetDevice(&previous_device) != cudaSuccess || + cudaSetDevice(target_device) != cudaSuccess) { + (void)cudaGetLastError(); + if (previous_device >= 0) (void)cudaSetDevice(previous_device); + return 0; + } + + size_t free_bytes = 0; + size_t total_bytes = 0; + if (cudaMemGetInfo(&free_bytes, &total_bytes) != cudaSuccess) { + (void)cudaGetLastError(); + if (previous_device >= 0) (void)cudaSetDevice(previous_device); + return 0; + } + const uint64_t effective_free = (uint64_t)free_bytes; + const uint64_t reserve = + cuda_q4_attn_q_b_f16_reserve_bytes((uint64_t)total_bytes); + uint64_t required_free = scratch_bytes; + if (required_free > UINT64_MAX - reserve) { + if (previous_device >= 0) (void)cudaSetDevice(previous_device); + return 0; + } + required_free += reserve; + if (required_free > UINT64_MAX - working_set_reserve_bytes) { + if (previous_device >= 0) (void)cudaSetDevice(previous_device); + return 0; + } + required_free += working_set_reserve_bytes; + if (required_free > effective_free) { + if (getenv("DS4_CUDA_WEIGHT_CACHE_VERBOSE") != NULL) { + fprintf(stderr, + "ds4: CUDA Q4 attn_q_b transient F16 preflight skipped: " + "need %.2f MiB scratch + %.2f GiB reserve + %.2f GiB " + "future sessions, only %.2f GiB currently free\n", + (double)scratch_bytes / 1048576.0, + (double)reserve / 1073741824.0, + (double)working_set_reserve_bytes / 1073741824.0, + (double)effective_free / 1073741824.0); + } + if (previous_device >= 0) (void)cudaSetDevice(previous_device); + return 0; + } + + /* Allocate first so a failed grow cannot invalidate the capacity published + * by the current generation. The conservative admission above intentionally + * does not count the old arena as reclaimable. */ + __half *scratch = NULL; + if (cudaMalloc((void **)&scratch, (size_t)scratch_bytes) != cudaSuccess) { + (void)cudaGetLastError(); + if (previous_device >= 0) (void)cudaSetDevice(previous_device); + return 0; + } + + /* Any old arena can still be referenced by stream work queued by a previous + * command batch. Drain before replacing it; this synchronization happens in + * prompt-aware preflight, never in the timed layer loop. */ + if (cudaDeviceSynchronize() != cudaSuccess) { + (void)cudaGetLastError(); + (void)cudaFree(scratch); + if (previous_device >= 0) (void)cudaSetDevice(previous_device); + return 0; + } + if (g_q4_attn_q_b_transient_f16_scratch) { + if (cudaFree(g_q4_attn_q_b_transient_f16_scratch) != cudaSuccess) { + (void)cudaGetLastError(); + (void)cudaFree(scratch); + if (previous_device >= 0) (void)cudaSetDevice(previous_device); + return 0; + } + } + g_q4_attn_q_b_transient_f16_scratch = scratch; + g_q4_attn_q_b_transient_f16_scratch_bytes = scratch_bytes; + g_q4_attn_q_b_transient_f16_scratch_device = target_device; + g_q4_attn_q_b_transient_f16_runtime_disabled = 0; + /* Session preflight caches the backend generation. Capacity is part of + * readiness: after a grow, sessions prepared for a smaller prompt must + * revisit preflight before their next larger sync. */ + { + std::lock_guard cache_lock( + g_q4_attn_q_b_f16_cache_mutex); + if (++g_q4_attn_q_b_f16_generation == 0u) { + g_q4_attn_q_b_f16_generation = 1u; + } + } + if (prepared_bytes) *prepared_bytes = scratch_bytes; + if (previous_device >= 0) (void)cudaSetDevice(previous_device); + fprintf(stderr, + "ds4: CUDA prepared reusable Q4 attn_q_b transient F16 scratch " + "(%.2f MiB, max batch %u tokens)\n", + (double)scratch_bytes / 1048576.0, max_rows); + return 1; +} + static int cuda_q4_attn_q_b_f16_key_equal( const cuda_q4_attn_q_b_f16_range &entry, const void *model_map, @@ -2632,17 +2860,20 @@ extern "C" int ds4_gpu_release_q4_attn_q_b_f16_sidecars(void) { return cuda_q4_attn_q_b_f16_release_under_build_lock(); } -/* Called after a fast-path launch/runtime failure while cache_use_lock is - * held. Block future dispatch/build attempts, drop the use lock, then - * serialize with cold builders and synchronously evict. Re-arm the circuit - * breaker after eviction so this session cannot rebuild and repeat the same - * failure; an explicit lifecycle release/make_room resets it later. */ +/* Called after a persistent fast-path failure while both use locks are held. + * Block future dispatch/build attempts, drop cache then scratch ownership, and + * only then take build_mutex for synchronous eviction. This preserves the + * global build -> scratch -> cache lifecycle order. Re-arm the circuit breaker + * after eviction so this session cannot rebuild and repeat the same failure; + * an explicit lifecycle release/make_room resets it later. */ static void cuda_q4_attn_q_b_f16_runtime_failure_evict( - std::unique_lock *cache_use_lock) { + std::unique_lock *cache_use_lock, + std::unique_lock *scratch_use_lock) { g_q4_attn_q_b_f16_hard_disabled = 1; g_q4_attn_q_b_f16_dispatch_disabled = 1; g_q4_attn_q_b_f16_pending_evict = 1; cache_use_lock->unlock(); + scratch_use_lock->unlock(); std::lock_guard build_lock( g_q4_attn_q_b_f16_build_mutex); @@ -2686,6 +2917,9 @@ static int cuda_q4_attn_q_b_f16_consume_pending_evict(void) { extern "C" int ds4_gpu_make_room_for_q4_attn_q_b_f16_session(void) { std::lock_guard build_lock(g_q4_attn_q_b_f16_build_mutex); + if (!cuda_q4_attn_q_b_transient_f16_scratch_release()) { + return 0; + } uint64_t bytes = 0; int needs_reset = 0; { @@ -2715,17 +2949,45 @@ extern "C" int ds4_gpu_prepare_q4_attn_q_b_f16_sidecars( uint64_t *prepared_bytes) { if (prepared_bytes) *prepared_bytes = 0; const int required = cuda_q4_attn_q_b_f16_required(); - if (!cuda_q4_attn_q_b_f16_requested() || - max_prefill_rows < cuda_q4_attn_q_b_f16_min_tokens()) { - return 0; - } - if (cuda_q4_attn_q_b_f16_disabled() || g_ssd_streaming_mode || - g_quality_mode || g_n_gpus != 1 || !g_cublas_ready || + const int persistent_requested = + cuda_q4_attn_q_b_f16_requested(); + const int persistent_disabled = + cuda_q4_attn_q_b_f16_disabled(); + const int persistent_selected = + required || (persistent_requested && !persistent_disabled); + const uint32_t persistent_min_tokens = + cuda_q4_attn_q_b_f16_min_tokens(); + /* An explicit persistent selection owns its configured candidate domain. + * Below that domain, preserve native Q4 exactly like Metal and ROCm. + * A non-strict ENABLE+DISABLE pair cancels the selection and may still + * use the independent automatic transient policy. */ + if (persistent_selected && + max_prefill_rows < persistent_min_tokens) { + return 0; + } + const int persistent_candidate = + persistent_selected && + max_prefill_rows >= persistent_min_tokens; + /* REQUIRE is strict only for batches that are actually eligible for the + * persistent cache. A long transient-only batch below CACHE_MIN_TOKENS + * must retain the same native-Q4 fallback as Metal and ROCm. */ + const int strict = required && persistent_candidate; + const int transient_candidate = + !cuda_q4_attn_q_b_transient_f16_disabled() && + max_prefill_rows >= + cuda_q4_attn_q_b_transient_f16_min_tokens(); + if (!persistent_candidate && !transient_candidate) { + return 0; + } + if ((strict && persistent_disabled) || + g_ssd_streaming_mode || g_quality_mode || g_n_gpus != 1 || + !g_cublas_ready || !g_gpu[0].cublas_ready || !g_gpu[0].cublas || cuda_q4_attn_q_b_f16_multi_model_policy_active() || + g_decode_graph_capturing || !model_map || !descs || count == 0u || count > CUDA_Q4_ATTN_Q_B_MAX_ENTRIES) { - return required ? -1 : 0; + return strict ? -1 : 0; } const uint64_t expected_weight_bytes = @@ -2743,11 +3005,61 @@ extern "C" int ds4_gpu_prepare_q4_attn_q_b_f16_sidecars( desc.weight_bytes != expected_weight_bytes || desc.weight_offset > model_size || desc.weight_bytes > model_size - desc.weight_offset) { - return required ? -1 : 0; + return strict ? -1 : 0; } } - std::lock_guard build_lock(g_q4_attn_q_b_f16_build_mutex); + /* Serialize scratch capacity publication with sidecar construction and + * lifecycle eviction. The global order is build -> scratch -> cache. */ + std::lock_guard build_lock( + g_q4_attn_q_b_f16_build_mutex); + + /* Without an explicit resident-cache request, the default long-prefill + * specialization retains only one expanded matrix. Verify every source in + * the already-owned device image now; the automatic path never populates a + * range cache or registers host pages. DISABLE_CACHE rolls an optional + * persistent request over to this path; REQUIRE remains strict only for + * persistent-cache candidates. */ + const int use_persistent = + persistent_candidate && !persistent_disabled; + if (!use_persistent) { + if (!transient_candidate) return strict ? -1 : 0; + int previous_device = -1; + if (cudaGetDevice(&previous_device) != cudaSuccess || + cudaSetDevice(g_gpu[0].device_id) != cudaSuccess) { + (void)cudaGetLastError(); + if (previous_device >= 0) (void)cudaSetDevice(previous_device); + return strict ? -1 : 0; + } + int resident = + model_map == g_model_host_base && + g_model_device_owned && g_model_device_base; + for (uint32_t i = 0; i < count; i++) { + if (!resident) break; + const char *source = + g_model_device_base + descs[i].weight_offset; + if (!cuda_q4_attn_q_b_source_is_device_resident( + source, g_gpu[0].device_id)) { + resident = 0; + break; + } + } + if (previous_device >= 0) (void)cudaSetDevice(previous_device); + if (!resident || + !cuda_q4_attn_q_b_transient_f16_scratch_ensure( + max_prefill_rows, working_set_reserve_bytes, + prepared_bytes)) { + return strict ? -1 : 0; + } + return 1; + } + + /* The persistent path also stages its F16 RHS in the dedicated arena. */ + if (!cuda_q4_attn_q_b_transient_f16_scratch_ensure( + max_prefill_rows, working_set_reserve_bytes, prepared_bytes)) { + return strict ? -1 : 0; + } + uint32_t miss_indices[CUDA_Q4_ATTN_Q_B_MAX_ENTRIES]; uint32_t miss_count = 0; uint64_t missing_bytes = 0; @@ -2755,7 +3067,7 @@ extern "C" int ds4_gpu_prepare_q4_attn_q_b_f16_sidecars( std::lock_guard cache_lock( g_q4_attn_q_b_f16_cache_mutex); if (g_q4_attn_q_b_f16_multi_model_active) { - return required ? -1 : 0; + return strict ? -1 : 0; } /* One cache generation belongs to one mmap identity. Do not let a * second live model consume the remaining budget or poison READY @@ -2764,7 +3076,7 @@ extern "C" int ds4_gpu_prepare_q4_attn_q_b_f16_sidecars( if (!g_q4_attn_q_b_f16_ranges.empty() && (g_q4_attn_q_b_f16_ranges[0].host_base != model_map || g_q4_attn_q_b_f16_ranges[0].model_size != model_size)) { - return required ? -1 : 0; + return strict ? -1 : 0; } for (uint32_t i = 0; i < count; i++) { int found = 0; @@ -2794,14 +3106,14 @@ extern "C" int ds4_gpu_prepare_q4_attn_q_b_f16_sidecars( if (!found) { miss_indices[miss_count++] = i; if (UINT64_MAX - missing_bytes < one_f16_bytes) { - return required ? -1 : 0; + return strict ? -1 : 0; } missing_bytes += one_f16_bytes; } } if (miss_count != 0u) { if (g_q4_attn_q_b_f16_hard_disabled) { - return required ? -1 : 0; + return strict ? -1 : 0; } const uint64_t limit = cuda_q4_attn_q_b_f16_cache_limit_bytes(); @@ -2809,7 +3121,7 @@ extern "C" int ds4_gpu_prepare_q4_attn_q_b_f16_sidecars( CUDA_Q4_ATTN_Q_B_MAX_ENTRIES - miss_count || g_q4_attn_q_b_f16_bytes > limit || missing_bytes > limit - g_q4_attn_q_b_f16_bytes) { - return required ? -1 : 0; + return strict ? -1 : 0; } } } @@ -2818,12 +3130,12 @@ extern "C" int ds4_gpu_prepare_q4_attn_q_b_f16_sidecars( int previous_device = -1; if (cudaGetDevice(&previous_device) != cudaSuccess) { (void)cudaGetLastError(); - return required ? -1 : 0; + return strict ? -1 : 0; } if (cudaSetDevice(g_gpu[0].device_id) != cudaSuccess) { (void)cudaGetLastError(); if (previous_device >= 0) (void)cudaSetDevice(previous_device); - return required ? -1 : 0; + return strict ? -1 : 0; } const cuda_block_q4_K *sources[CUDA_Q4_ATTN_Q_B_MAX_ENTRIES] = {}; @@ -2867,7 +3179,7 @@ extern "C" int ds4_gpu_prepare_q4_attn_q_b_f16_sidecars( } else if (!build_failed) { required_free += working_set_reserve_bytes; if (required_free > (uint64_t)free_bytes) { - if (getenv("DS4_CUDA_WEIGHT_CACHE_VERBOSE") != NULL || required) { + if (getenv("DS4_CUDA_WEIGHT_CACHE_VERBOSE") != NULL || strict) { fprintf(stderr, "ds4: CUDA Q4 attn_q_b F16 prewarm skipped: " "need %.2f GiB sidecars + %.2f GiB reserve + " @@ -2878,7 +3190,7 @@ extern "C" int ds4_gpu_prepare_q4_attn_q_b_f16_sidecars( (double)free_bytes / 1073741824.0); } if (previous_device >= 0) (void)cudaSetDevice(previous_device); - return required ? -1 : 0; + return strict ? -1 : 0; } } } @@ -2926,7 +3238,7 @@ extern "C" int ds4_gpu_prepare_q4_attn_q_b_f16_sidecars( g_q4_attn_q_b_f16_hard_disabled = 1; } if (previous_device >= 0) (void)cudaSetDevice(previous_device); - return required ? -1 : 0; + return strict ? -1 : 0; } { @@ -7372,6 +7684,9 @@ extern "C" void ds4_gpu_cleanup(void) { * Retire them while the tier contexts and physical-device mapping are * intact; release performs its own quiescence check. */ (void)ds4_gpu_release_q4_attn_q_b_f16_sidecars(); + /* The transient arena is backend-owned rather than graph-owned. Release + * it only after quiescence and before tearing down the device contexts. */ + (void)cuda_q4_attn_q_b_transient_f16_scratch_release(); cuda_q4_attn_q_b_f16_set_multi_model_policy(0); cuda_decode_graphs_shutdown(); cuda_q8_fold_release_all(); @@ -42661,6 +42976,12 @@ extern "C" void ds4_gpu_set_ssd_streaming(bool enabled) { "ds4: CUDA could not safely release Q4 attn_q_b F16 " "sidecars while enabling SSD streaming\n"); } + if (enabled && !g_ssd_streaming_mode && + !cuda_q4_attn_q_b_transient_f16_scratch_release()) { + fprintf(stderr, + "ds4: CUDA could not safely release Q4 attn_q_b transient " + "F16 scratch while enabling SSD streaming\n"); + } cuda_stream_selected_writer_guard writer; g_ssd_streaming_mode = enabled ? 1 : 0; if (!g_ssd_streaming_mode) { @@ -42879,27 +43200,45 @@ extern "C" int ds4_gpu_attn_q_b_f16_head_rms_rope_tail_tensor( float freq_base, float freq_scale, float ext_factor, float attn_factor, float beta_fast, float beta_slow, float eps) { /* The pre-existing CUDA Q8 specialization was a stub. Preserve that - * fallback behavior and arm only the explicit resident-Q4 experiment. */ + * fallback behavior and arm only resident Q4_K. CUDA writes the GEMM + * directly to the canonical F32 graph tensor, so the Metal-only q_half + * workspace is deliberately optional here. */ + (void)q_half; if (weight_type != CUDA_Q4_ATTN_Q_B_TYPE) return 0; - if (n_tok < 32u || - n_tok < cuda_q4_attn_q_b_f16_min_tokens() || - !cuda_q4_attn_q_b_f16_requested()) { - return 0; - } + if (n_tok < 32u) return 0; const int required = cuda_q4_attn_q_b_f16_required(); - const int fallback = required ? -1 : 0; - - if (cuda_q4_attn_q_b_f16_disabled() || g_ssd_streaming_mode || - g_quality_mode || g_n_gpus != 1 || !g_cublas_ready || + const int persistent_requested = + cuda_q4_attn_q_b_f16_requested(); + const int persistent_disabled = + cuda_q4_attn_q_b_f16_disabled(); + const int persistent_selected = + required || (persistent_requested && !persistent_disabled); + const uint32_t persistent_min_tokens = + cuda_q4_attn_q_b_f16_min_tokens(); + if (persistent_selected && n_tok < persistent_min_tokens) return 0; + const int persistent_candidate = + persistent_selected && n_tok >= persistent_min_tokens; + const int strict = required && persistent_candidate; + const int use_persistent = + persistent_candidate && !persistent_disabled; + const int fallback = strict ? -1 : 0; + const int transient_candidate = + !cuda_q4_attn_q_b_transient_f16_disabled() && + n_tok >= cuda_q4_attn_q_b_transient_f16_min_tokens(); + if (!persistent_candidate && !transient_candidate) return 0; + + if ((strict && persistent_disabled) || + g_ssd_streaming_mode || g_quality_mode || g_n_gpus != 1 || + !g_cublas_ready || !g_gpu[0].cublas_ready || !g_gpu[0].cublas || - g_decode_graph_capturing || !out || !q_half || !x || !model_map || - !out->ptr || !q_half->ptr || !x->ptr || n_head == 0u || + cuda_q4_attn_q_b_f16_multi_model_policy_active() || + g_decode_graph_capturing || !out || !x || !model_map || + !out->ptr || !x->ptr || n_head == 0u || head_dim == 0u || in_dim != CUDA_Q4_ATTN_Q_B_IN_DIM || out_dim != CUDA_Q4_ATTN_Q_B_OUT_DIM || out_dim != (uint64_t)n_head * head_dim || n_rot > head_dim || (n_rot & 1u) != 0u || ds4_tensor_device_idx(out) != 0 || - ds4_tensor_device_idx(q_half) != 0 || ds4_tensor_device_idx(x) != 0 || n_tok > (uint32_t)INT_MAX || (uint64_t)n_tok * in_dim > (uint64_t)UINT32_MAX * 256u || @@ -42912,10 +43251,7 @@ extern "C" int ds4_gpu_attn_q_b_f16_head_rms_rope_tail_tensor( (uint64_t)n_tok * in_dim * sizeof(float); const uint64_t out_bytes = (uint64_t)n_tok * out_dim * sizeof(float); - const uint64_t q_half_bytes = - (uint64_t)n_tok * out_dim * sizeof(__half); - if (x->bytes < x_bytes || out->bytes < out_bytes || - q_half->bytes < q_half_bytes) { + if (x->bytes < x_bytes || out->bytes < out_bytes) { return fallback; } @@ -42927,19 +43263,57 @@ extern "C" int ds4_gpu_attn_q_b_f16_head_rms_rope_tail_tensor( const uint64_t weight_bytes = out_dim * row_bytes; if (weight_bytes > model_size - weight_offset) return fallback; - /* Keep the cache lock until the final consumer is enqueued. A lifecycle - * release can then acquire the lock, synchronize all work queued before - * it, and free the sidecars without a lookup/free/launch race. */ + uint64_t transient_weight_bytes = 0; + uint64_t xh_offset = 0; + uint64_t required_scratch_bytes = 0; + if (!cuda_q4_attn_q_b_transient_f16_scratch_size( + n_tok, &transient_weight_bytes, &xh_offset, + &required_scratch_bytes)) { + return fallback; + } + + /* Protect the shared W/X arena for the entire enqueue sequence. Stream + * ordering protects reuse after this function returns; host serialization + * protects the sequence itself from interleaving with another session. */ + std::unique_lock scratch_use_lock( + g_q4_attn_q_b_transient_f16_mutex); + if (!g_q4_attn_q_b_transient_f16_scratch || + g_q4_attn_q_b_transient_f16_scratch_device != g_gpu[0].device_id || + g_q4_attn_q_b_transient_f16_scratch_bytes < + required_scratch_bytes) { + return fallback; + } + __half *const scratch = + g_q4_attn_q_b_transient_f16_scratch; + __half *const xh = reinterpret_cast<__half *>( + reinterpret_cast(scratch) + xh_offset); + + /* Prefer an explicitly prepared persistent sidecar. Keep its cache lock + * until the epilogue is enqueued so lifecycle release cannot free the + * matrix between lookup and its final consumer. An optional cache miss + * may still use the default transient path. */ std::unique_lock cache_use_lock( g_q4_attn_q_b_f16_cache_mutex); - if (g_q4_attn_q_b_f16_dispatch_disabled || - g_q4_attn_q_b_f16_multi_model_active) { + const __half *w_f16 = NULL; + int using_persistent = 0; + if (g_q4_attn_q_b_f16_multi_model_active) return fallback; + if (use_persistent) { + if (!g_q4_attn_q_b_f16_dispatch_disabled) { + w_f16 = cuda_q4_attn_q_b_f16_cache_lookup_locked( + model_map, model_size, weight_offset, weight_bytes, + in_dim, out_dim, g_gpu[0].device_id); + } + if (w_f16) { + using_persistent = 1; + } else { + if (strict) return -1; + } + } + if (!using_persistent && + (!transient_candidate || + g_q4_attn_q_b_transient_f16_runtime_disabled)) { return fallback; } - const __half *w_f16 = cuda_q4_attn_q_b_f16_cache_lookup_locked( - model_map, model_size, weight_offset, weight_bytes, - in_dim, out_dim, g_gpu[0].device_id); - if (!w_f16) return fallback; int previous_device = -1; if (cudaGetDevice(&previous_device) != cudaSuccess) { @@ -42965,19 +43339,38 @@ extern "C" int ds4_gpu_attn_q_b_f16_head_rms_rope_tail_tensor( } const uint64_t xh_count = (uint64_t)n_tok * in_dim; - __half *xh = (__half *)cuda_tmp_alloc_on( - 0, xh_count * sizeof(__half), - "Q4 attn_q_b cached F16 activations"); - if (!xh) { - cuda_q4_attn_q_b_f16_runtime_failure_evict(&cache_use_lock); - if (previous_device >= 0) (void)cudaSetDevice(previous_device); - return fallback; + if (!using_persistent) { + const char *source = + model_map == g_model_host_base && + g_model_device_owned && g_model_device_base + ? g_model_device_base + weight_offset : NULL; + if (!cuda_q4_attn_q_b_source_is_device_resident( + source, g_gpu[0].device_id)) { + if (previous_device >= 0) (void)cudaSetDevice(previous_device); + return fallback; + } + w_f16 = scratch; + const uint64_t chunks = transient_weight_bytes / (16u * sizeof(__half)); + dequant_q4_K_to_f16_kernel<<< + (unsigned)((chunks + 255u) / 256u), 256, 0, stream>>>( + scratch, + reinterpret_cast(source), + in_dim, out_dim, blocks); + const cudaError_t dequant_err = cudaGetLastError(); + if (dequant_err != cudaSuccess) { + fprintf(stderr, + "ds4: CUDA Q4 attn_q_b transient dequantization failed: " + "%s\n", + cudaGetErrorString(dequant_err)); + g_q4_attn_q_b_transient_f16_runtime_disabled = 1; + if (previous_device >= 0) (void)cudaSetDevice(previous_device); + return fallback; + } } /* Prefill is deliberately excluded from decode-graph capture above, so - * both kernels and the tier-0 cuBLAS handle use the legacy default - * stream. Avoid retargeting the shared handle on this hot per-layer - * path, which could race other launchers and adds measurable overhead. */ + * dequantization, activation conversion, cuBLAS, and the epilogue all ride + * one stream. Avoid retargeting the shared handle on this hot path. */ f32_to_f16_kernel<<< (unsigned)((xh_count + 255u) / 256u), 256, 0, stream>>>( xh, (const float *)x->ptr, xh_count); @@ -42986,7 +43379,12 @@ extern "C" int ds4_gpu_attn_q_b_f16_head_rms_rope_tail_tensor( fprintf(stderr, "ds4: CUDA Q4 attn_q_b activation conversion failed: %s\n", cudaGetErrorString(launch_err)); - cuda_q4_attn_q_b_f16_runtime_failure_evict(&cache_use_lock); + if (using_persistent) { + cuda_q4_attn_q_b_f16_runtime_failure_evict( + &cache_use_lock, &scratch_use_lock); + } else { + g_q4_attn_q_b_transient_f16_runtime_disabled = 1; + } if (previous_device >= 0) (void)cudaSetDevice(previous_device); return fallback; } @@ -43001,21 +43399,28 @@ extern "C" int ds4_gpu_attn_q_b_f16_head_rms_rope_tail_tensor( w_f16, CUDA_R_16F, (int)in_dim, xh, CUDA_R_16F, (int)in_dim, &beta, - q_half->ptr, CUDA_R_16F, (int)out_dim, - CUBLAS_COMPUTE_32F, + out->ptr, CUDA_R_32F, (int)out_dim, + CUDA_R_32F, CUBLAS_GEMM_DEFAULT); if (status != CUBLAS_STATUS_SUCCESS) { fprintf(stderr, - "ds4: CUDA Q4 attn_q_b cached F16 GEMM failed: status %d\n", + "ds4: CUDA Q4 attn_q_b F16 GEMM failed: status %d\n", (int)status); - cuda_q4_attn_q_b_f16_runtime_failure_evict(&cache_use_lock); + if (using_persistent) { + cuda_q4_attn_q_b_f16_runtime_failure_evict( + &cache_use_lock, &scratch_use_lock); + } else { + g_q4_attn_q_b_transient_f16_runtime_disabled = 1; + } if (previous_device >= 0) (void)cudaSetDevice(previous_device); return fallback; } - head_rms_norm_rope_tail_from_half_kernel<<< + /* cuBLAS has accepted the first output writer. From this point onward a + * native-Q4 replay is unsafe even when the specialization was optional. */ + head_rms_norm_rope_tail_kernel<<< n_tok * n_head, 256, 0, stream>>>( - (float *)out->ptr, (const __half *)q_half->ptr, + (float *)out->ptr, n_tok, n_head, head_dim, n_rot, pos0, n_ctx_orig, inverse ? 1 : 0, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow, eps); @@ -43024,9 +43429,14 @@ extern "C" int ds4_gpu_attn_q_b_f16_head_rms_rope_tail_tensor( fprintf(stderr, "ds4: CUDA Q4 attn_q_b F16 RMS/RoPE epilogue failed: %s\n", cudaGetErrorString(launch_err)); - cuda_q4_attn_q_b_f16_runtime_failure_evict(&cache_use_lock); + if (using_persistent) { + cuda_q4_attn_q_b_f16_runtime_failure_evict( + &cache_use_lock, &scratch_use_lock); + } else { + g_q4_attn_q_b_transient_f16_runtime_disabled = 1; + } if (previous_device >= 0) (void)cudaSetDevice(previous_device); - return fallback; + return -1; } if (previous_device >= 0) (void)cudaSetDevice(previous_device); diff --git a/rocm/ds4_rocm_norm_rope.cuh b/rocm/ds4_rocm_norm_rope.cuh index 057255e5c..b68ec1a1e 100644 --- a/rocm/ds4_rocm_norm_rope.cuh +++ b/rocm/ds4_rocm_norm_rope.cuh @@ -553,6 +553,7 @@ static int rocm_q4_attn_q_b_f16_head_rms_rope_tail_tensor( float beta_fast, float beta_slow, float eps) { + (void)q_half; /* Decode and tiny/final chunks retain the native Q4_K path without even * taking the cache mutex. REQUIRE applies only to configured candidates. */ if (n_tok < 32u) return 0; @@ -568,7 +569,7 @@ static int rocm_q4_attn_q_b_f16_head_rms_rope_tail_tensor( uint64_t head_rows = 0; if (!rocm_q4_attn_q_b_f16_policy_allowed() || rocm_q4_attn_q_b_f16_circuit_open() || - !g_cublas_ready || !out || !q_half || !x || !model_map || + !g_cublas_ready || !out || !x || !model_map || model_map != g_model_host_base || model_size != g_model_registered_size || in_dim != DS4_ROCM_Q4_ATTN_Q_B_IN_DIM || @@ -585,8 +586,7 @@ static int rocm_q4_attn_q_b_f16_head_rms_rope_tail_tensor( (x_elems + 255u) / 256u > UINT32_MAX || !cuda_u64_mul_checked(x_elems, sizeof(float), &x_bytes) || !cuda_u64_mul_checked(out_elems, sizeof(float), &out_bytes) || - x->bytes < x_bytes || out->bytes < out_bytes || - q_half->bytes < out_elems * sizeof(__half)) { + x->bytes < x_bytes || out->bytes < out_bytes) { return rocm_q4_attn_q_b_f16_fallback(required, 1, 0); } @@ -600,22 +600,25 @@ static int rocm_q4_attn_q_b_f16_head_rms_rope_tail_tensor( return rocm_q4_attn_q_b_f16_fallback(required, 1, 0); } + __half *unused_weight_f16 = NULL; + __half *xh = NULL; + if (!rocm_q4_attn_q_b_transient_f16_acquire( + n_tok, &unused_weight_f16, &xh)) { + return rocm_q4_attn_q_b_f16_fallback(required, 0, 0); + } + (void)unused_weight_f16; + const __half *w_f16 = rocm_q4_attn_q_b_f16_acquire( model_map, model_size, weight_offset, weight_bytes, in_dim, out_dim, DS4_ROCM_Q4_K_TYPE); if (!w_f16) { + rocm_q4_attn_q_b_transient_f16_release_acquired(); return rocm_q4_attn_q_b_f16_fallback(required, 0, 0); } - /* Keep the cache mutex through GEMM submission. A concurrent lifecycle - * release cannot synchronize and free the sidecar between lookup and the - * first command that references it. */ - __half *xh = (__half *)cuda_tmp_alloc( - x_elems * sizeof(__half), "Q4 attn q_b F16 activations"); - if (!xh) { - rocm_q4_attn_q_b_f16_release_acquired(); - return rocm_q4_attn_q_b_f16_fallback(required, 0, 1); - } + /* Keep both the persistent weight pin and the dedicated X staging lock + * through the complete enqueue sequence. Lifecycle release takes the same + * locks before synchronizing, so it cannot miss the final F32 epilogue. */ f32_to_f16_kernel<<<(x_elems + 255u) / 256u, 256>>>( xh, (const float *)x->ptr, x_elems); cudaError_t launch_err = cudaGetLastError(); @@ -626,6 +629,7 @@ static int rocm_q4_attn_q_b_f16_head_rms_rope_tail_tensor( cudaGetErrorString(launch_err)); (void)cudaGetLastError(); rocm_q4_attn_q_b_f16_release_acquired(); + rocm_q4_attn_q_b_transient_f16_release_acquired(); return rocm_q4_attn_q_b_f16_fallback(required, 0, 1); } @@ -646,34 +650,194 @@ static int rocm_q4_attn_q_b_f16_head_rms_rope_tail_tensor( CUDA_R_16F, (int)in_dim, &beta, - q_half->ptr, - CUDA_R_16F, + out->ptr, + CUDA_R_32F, (int)out_dim, CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT); - rocm_q4_attn_q_b_f16_release_acquired(); if (st != CUBLAS_STATUS_SUCCESS) { fprintf(stderr, DS4_GPU_LOG_PREFIX DS4_GPU_BLAS_NAME - " cached Q4 attn q_b F16-out matmul failed: status %d\n", + " cached Q4 attn q_b F16/F16-to-F32 matmul failed: " + "status %d\n", (int)st); + rocm_q4_attn_q_b_f16_release_acquired(); + rocm_q4_attn_q_b_transient_f16_release_acquired(); return rocm_q4_attn_q_b_f16_fallback(required, 0, 1); } - head_rms_norm_rope_tail_from_half_kernel<<<(uint32_t)head_rows, 256>>>( - (float *)out->ptr, (const __half *)q_half->ptr, - n_tok, n_head, head_dim, n_rot, pos0, n_ctx_orig, - inverse ? 1 : 0, freq_base, freq_scale, ext_factor, attn_factor, - beta_fast, beta_slow, eps); + const int tail_ok = ds4_gpu_head_rms_norm_rope_tail_tensor( + out, n_tok, n_head, head_dim, n_rot, pos0, n_ctx_orig, + inverse, freq_base, freq_scale, ext_factor, attn_factor, + beta_fast, beta_slow, eps); + rocm_q4_attn_q_b_f16_release_acquired(); + rocm_q4_attn_q_b_transient_f16_release_acquired(); + if (!tail_ok) { + /* GEMM has already accepted the output writer. Never authorize the + * caller to replay native Q4 over an asynchronous/partial result. */ + return rocm_q4_attn_q_b_f16_fallback(1, 0, 1); + } + return 1; +} + +/* Resident default: expand only the current Q4_K q_b matrix into the shared + * 64 MiB W_F16 region, convert X into the adjacent preflighted region, consume + * both immediately with hipBLAS, and enqueue the F32 epilogue before allowing + * another host caller to reuse the allocation. */ +static int rocm_q4_attn_q_b_transient_f16_head_rms_rope_tail_tensor( + ds4_gpu_tensor *out, + ds4_gpu_tensor *q_half, + const void *model_map, + uint64_t model_size, + uint64_t weight_offset, + uint64_t in_dim, + uint64_t out_dim, + const ds4_gpu_tensor *x, + uint32_t n_tok, + uint32_t n_head, + uint32_t head_dim, + uint32_t n_rot, + uint32_t pos0, + uint32_t n_ctx_orig, + bool inverse, + float freq_base, + float freq_scale, + float ext_factor, + float attn_factor, + float beta_fast, + float beta_slow, + float eps) { + (void)q_half; + if ((uint64_t)n_tok < + rocm_q4_attn_q_b_transient_f16_min_tokens()) { + return 0; + } + rocm_q4_attn_q_b_f16_note_candidate(); + + uint64_t x_elems = 0; + uint64_t out_elems = 0; + uint64_t head_rows = 0; + uint64_t x_bytes = 0; + uint64_t out_bytes = 0; + if (!rocm_q4_attn_q_b_transient_f16_policy_allowed() || + rocm_q4_attn_q_b_f16_circuit_open() || + !g_cublas_ready || !out || !x || !model_map || + model_map != g_model_host_base || + model_size != g_model_registered_size || + in_dim != DS4_ROCM_Q4_ATTN_Q_B_IN_DIM || + out_dim != DS4_ROCM_Q4_ATTN_Q_B_OUT_DIM || + n_head == 0u || head_dim == 0u || + out_dim != (uint64_t)n_head * head_dim || + n_rot > head_dim || (n_rot & 1u) != 0u || + n_tok > (uint32_t)INT_MAX || + pos0 > (uint32_t)INT_MAX - n_tok || + !cuda_u64_mul_checked(n_tok, in_dim, &x_elems) || + !cuda_u64_mul_checked(n_tok, out_dim, &out_elems) || + !cuda_u64_mul_checked(n_tok, n_head, &head_rows) || + head_rows > UINT32_MAX || + (x_elems + 255u) / 256u > UINT32_MAX || + !cuda_u64_mul_checked(x_elems, sizeof(float), &x_bytes) || + !cuda_u64_mul_checked(out_elems, sizeof(float), &out_bytes) || + x->bytes < x_bytes || out->bytes < out_bytes) { + return rocm_q4_attn_q_b_f16_fallback(0, 1, 0); + } + + uint64_t row_bytes = 0; + uint64_t weight_bytes = 0; + const uint64_t blocks_per_row = in_dim / CUDA_QK_K; + if (!cuda_u64_mul_checked(blocks_per_row, + sizeof(cuda_block_q4_K), &row_bytes) || + !cuda_u64_mul_checked(out_dim, row_bytes, &weight_bytes) || + !cuda_model_range_fits(model_size, weight_offset, weight_bytes)) { + return rocm_q4_attn_q_b_f16_fallback(0, 1, 0); + } + + __half *w_f16 = NULL; + __half *x_f16 = NULL; + if (!rocm_q4_attn_q_b_transient_f16_acquire( + n_tok, &w_f16, &x_f16)) { + return rocm_q4_attn_q_b_f16_fallback(0, 0, 0); + } + + const char *w_q4 = rocm_q4_attn_q_b_device_resident_source( + model_map, weight_offset, weight_bytes); + if (!w_q4) { + rocm_q4_attn_q_b_transient_f16_release_acquired(); + return rocm_q4_attn_q_b_f16_fallback(0, 0, 1); + } + + const uint64_t total_chunks = out_dim * (in_dim / 16u); + rocm_dequant_q4_K_attn_q_b_f16_kernel<<< + (uint32_t)((total_chunks + 255u) / 256u), 256>>>( + w_f16, (const cuda_block_q4_K *)w_q4, + in_dim, out_dim, blocks_per_row); + cudaError_t launch_err = cudaGetLastError(); + if (launch_err != cudaSuccess) { + fprintf(stderr, + DS4_GPU_LOG_PREFIX + "Q4 attn_q_b transient dequant launch failed: %s\n", + cudaGetErrorString(launch_err)); + (void)cudaGetLastError(); + rocm_q4_attn_q_b_transient_f16_release_acquired(); + return rocm_q4_attn_q_b_f16_fallback(0, 0, 1); + } + + f32_to_f16_kernel<<<(x_elems + 255u) / 256u, 256>>>( + x_f16, (const float *)x->ptr, x_elems); launch_err = cudaGetLastError(); if (launch_err != cudaSuccess) { fprintf(stderr, DS4_GPU_LOG_PREFIX - "cached Q4 attn q_b head RMS/RoPE launch failed: %s\n", + "Q4 attn_q_b transient activation conversion failed: %s\n", cudaGetErrorString(launch_err)); (void)cudaGetLastError(); - return rocm_q4_attn_q_b_f16_fallback(required, 0, 1); + rocm_q4_attn_q_b_transient_f16_release_acquired(); + return rocm_q4_attn_q_b_f16_fallback(0, 0, 1); + } + + const float alpha = 1.0f; + const float beta = 0.0f; + const cublasStatus_t st = cublasGemmEx( + g_cublas, + CUBLAS_OP_T, + CUBLAS_OP_N, + (int)out_dim, + (int)n_tok, + (int)in_dim, + &alpha, + w_f16, + CUDA_R_16F, + (int)in_dim, + x_f16, + CUDA_R_16F, + (int)in_dim, + &beta, + out->ptr, + CUDA_R_32F, + (int)out_dim, + CUBLAS_COMPUTE_32F, + CUBLAS_GEMM_DEFAULT); + if (st != CUBLAS_STATUS_SUCCESS) { + fprintf(stderr, + DS4_GPU_LOG_PREFIX + DS4_GPU_BLAS_NAME + " transient Q4 attn q_b F16/F16-to-F32 matmul failed: " + "status %d\n", + (int)st); + rocm_q4_attn_q_b_transient_f16_release_acquired(); + return rocm_q4_attn_q_b_f16_fallback(0, 0, 1); + } + + const int tail_ok = ds4_gpu_head_rms_norm_rope_tail_tensor( + out, n_tok, n_head, head_dim, n_rot, pos0, n_ctx_orig, + inverse, freq_base, freq_scale, ext_factor, attn_factor, + beta_fast, beta_slow, eps); + rocm_q4_attn_q_b_transient_f16_release_acquired(); + if (!tail_ok) { + /* GEMM was accepted and may already be executing. Returning zero + * would make the graph replay native Q4 over an in-flight writer. */ + return rocm_q4_attn_q_b_f16_fallback(1, 0, 1); } return 1; } @@ -703,7 +867,19 @@ extern "C" int ds4_gpu_attn_q_b_f16_head_rms_rope_tail_tensor( float beta_slow, float eps) { if (weight_type == DS4_ROCM_Q4_K_TYPE) { - return rocm_q4_attn_q_b_f16_head_rms_rope_tail_tensor( + const int required = rocm_q4_attn_q_b_f16_required(); + const int persistent_requested = + required || + (rocm_q4_attn_q_b_f16_enabled() && + !rocm_q4_attn_q_b_f16_disabled()); + if (persistent_requested) { + return rocm_q4_attn_q_b_f16_head_rms_rope_tail_tensor( + out, q_half, model_map, model_size, weight_offset, + in_dim, out_dim, x, n_tok, n_head, head_dim, n_rot, pos0, + n_ctx_orig, inverse, freq_base, freq_scale, ext_factor, + attn_factor, beta_fast, beta_slow, eps); + } + return rocm_q4_attn_q_b_transient_f16_head_rms_rope_tail_tensor( out, q_half, model_map, model_size, weight_offset, in_dim, out_dim, x, n_tok, n_head, head_dim, n_rot, pos0, n_ctx_orig, inverse, freq_base, freq_scale, ext_factor, diff --git a/rocm/ds4_rocm_q4_qb_sidecar.cuh b/rocm/ds4_rocm_q4_qb_sidecar.cuh index d0ec793ae..9b3eb66ef 100644 --- a/rocm/ds4_rocm_q4_qb_sidecar.cuh +++ b/rocm/ds4_rocm_q4_qb_sidecar.cuh @@ -50,10 +50,20 @@ static uint64_t g_rocm_q4_attn_q_b_f16_fallbacks; static uint64_t g_rocm_q4_attn_q_b_f16_rejects; static int g_rocm_q4_attn_q_b_f16_hard_failure; static int g_rocm_q4_attn_q_b_f16_pending_evict; +/* The resident default rebuilds one layer at a time into this combined + * allocation. The first 64 MiB hold W_F16 and the suffix holds the largest + * preflighted X_F16 batch. ROCm currently submits graph work on stream 0, but + * keep the mutex through the complete dequant/copy/GEMM/epilogue enqueue + * sequence so two host callers cannot interleave reuse of either region. */ +static void *g_rocm_q4_attn_q_b_transient_f16_scratch; +static uint64_t g_rocm_q4_attn_q_b_transient_f16_scratch_bytes; +static uint64_t g_rocm_q4_attn_q_b_transient_f16_weight_bytes; static pthread_mutex_t g_rocm_q4_attn_q_b_f16_cache_mu = PTHREAD_MUTEX_INITIALIZER; static pthread_mutex_t g_rocm_q4_attn_q_b_f16_build_mu = PTHREAD_MUTEX_INITIALIZER; +static pthread_mutex_t g_rocm_q4_attn_q_b_transient_f16_mu = + PTHREAD_MUTEX_INITIALIZER; static int rocm_q4_attn_q_b_env_value_eq( const char *value, size_t n, const char *literal) { @@ -130,6 +140,17 @@ static int rocm_q4_attn_q_b_f16_disabled(void) { "DS4_ROCM_DISABLE_Q4_ATTN_Q_B_F16_CACHE") == 1; } +static int rocm_q4_attn_q_b_transient_f16_disabled(void) { + return rocm_q4_attn_q_b_env_bool( + "DS4_ROCM_DISABLE_Q4_ATTN_Q_B_TRANSIENT_F16") == 1; +} + +static uint64_t rocm_q4_attn_q_b_transient_f16_min_tokens(void) { + return rocm_q4_attn_q_b_env_u64( + "DS4_ROCM_Q4_ATTN_Q_B_TRANSIENT_F16_MIN_TOKENS", + 4096u, 32u, UINT32_MAX); +} + static uint64_t rocm_q4_attn_q_b_f16_min_tokens(void) { return rocm_q4_attn_q_b_env_u64( "DS4_ROCM_Q4_ATTN_Q_B_F16_CACHE_MIN_TOKENS", @@ -152,6 +173,53 @@ static int rocm_q4_attn_q_b_f16_policy_allowed(void) { !g_q8_f16_disabled_for_multi_model; } +static int rocm_q4_attn_q_b_transient_f16_policy_allowed(void) { + return !rocm_q4_attn_q_b_transient_f16_disabled() && + !g_ssd_streaming_mode && + !g_quality_mode && + !g_q8_f16_disabled_for_multi_model; +} + +/* Read-only lookup for the automatic path. Normal full-model ROCm loading + * may use either a contiguous device image or hipMalloc-backed range arenas. + * Accept both, but never mapped/registered host memory and never populate the + * range cache here: that would move I/O or page migration into prefill. Model + * cache construction is complete before session preflight begins. */ +static const char *rocm_q4_attn_q_b_device_resident_source( + const void *model_map, + uint64_t offset, + uint64_t bytes) { + const char *image = + cuda_model_image_range_ptr(model_map, offset, bytes); + if (image) return image; + if (!model_map || bytes == 0u || offset > UINT64_MAX - bytes) { + return NULL; + } + const uint64_t end = offset + bytes; + const auto exact = g_model_range_by_offset.find(offset); + if (exact != g_model_range_by_offset.end() && + exact->second < g_model_ranges.size()) { + const cuda_model_range &range = g_model_ranges[exact->second]; + if (range.host_base == model_map && !range.host_registered && + range.device_ptr && range.offset == offset && + bytes <= range.bytes) { + return range.device_ptr; + } + } + for (const cuda_model_range &range : g_model_ranges) { + if (range.host_base != model_map || range.host_registered || + !range.device_ptr || offset < range.offset || + range.offset > UINT64_MAX - range.bytes) { + continue; + } + const uint64_t range_end = range.offset + range.bytes; + if (end <= range_end) { + return range.device_ptr + (offset - range.offset); + } + } + return NULL; +} + static int rocm_q4_attn_q_b_f16_key_equal( const rocm_q4_attn_q_b_f16_cache_entry *entry, const void *model_map, @@ -288,33 +356,49 @@ rocm_q4_attn_q_b_get_scale_min( } } -/* One block expands one canonical 256-value GGUF Q4_K block. The output is - * row-major [out_dim, in_dim], the same storage consumed as W^T by hipBLAS. */ +/* Expand one contiguous 16-value chunk per thread. Compared with launching a + * 256-thread workgroup for every Q4_K block, this cuts the logical thread count + * by 16x while preserving the row-major [out_dim, in_dim] layout consumed as + * W^T by hipBLAS. */ __global__ static void rocm_dequant_q4_K_attn_q_b_f16_kernel( __half *dst, const cuda_block_q4_K *src, - uint64_t block_count) { - const uint64_t block = (uint64_t)blockIdx.x; - const uint32_t i = threadIdx.x; - if (block >= block_count || i >= CUDA_QK_K) return; - - const cuda_block_q4_K *xb = src + block; - const uint32_t group = i >> 5u; - const uint32_t within_group = i & 31u; - const uint32_t byte_offset = (group >> 1u) * 32u + within_group; - const uint32_t shift = (group & 1u) * 4u; - const uint32_t q = (xb->qs[byte_offset] >> shift) & 0x0fu; - uint8_t scale = 0; - uint8_t minimum = 0; - rocm_q4_attn_q_b_get_scale_min( - group, xb->scales, &scale, &minimum); + uint64_t in_dim, + uint64_t out_dim, + uint64_t blocks_per_row) { + const uint64_t chunk = + (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + const uint64_t chunks_per_row = in_dim / 16u; + const uint64_t total_chunks = out_dim * chunks_per_row; + if (chunk >= total_chunks) return; + + const uint64_t row = chunk / chunks_per_row; + const uint64_t col0 = (chunk - row * chunks_per_row) * 16u; + const uint64_t block_in_row = col0 / CUDA_QK_K; + const uint32_t within0 = (uint32_t)(col0 % CUDA_QK_K); + const cuda_block_q4_K *xb = + src + row * blocks_per_row + block_in_row; const float d = __half2float( __ushort_as_half((unsigned short)xb->d)); const float dmin = __half2float( __ushort_as_half((unsigned short)xb->dmin)); - dst[block * CUDA_QK_K + i] = - __float2half(d * (float)scale * (float)q - - dmin * (float)minimum); + +#pragma unroll + for (uint32_t k = 0; k < 16u; k++) { + const uint32_t within = within0 + k; + const uint32_t group = within >> 5u; + uint8_t scale = 0; + uint8_t minimum = 0; + rocm_q4_attn_q_b_get_scale_min( + group, xb->scales, &scale, &minimum); + const uint8_t packed = + xb->qs[(group >> 1u) * 32u + (within & 31u)]; + const uint32_t q = + (group & 1u) ? (packed >> 4u) : (packed & 0x0fu); + dst[row * in_dim + col0 + k] = + __float2half(d * (float)scale * (float)q - + dmin * (float)minimum); + } } static int rocm_q4_attn_q_b_f16_desc_valid( @@ -377,6 +461,133 @@ static int rocm_q4_attn_q_b_f16_memory_has_room( return required_free <= free_bytes; } +static int rocm_q4_attn_q_b_transient_f16_layout( + uint64_t rows, + uint64_t *weight_bytes_out, + uint64_t *x_bytes_out, + uint64_t *total_bytes_out) { + uint64_t weight_elems = 0; + uint64_t weight_bytes = 0; + uint64_t x_elems = 0; + uint64_t x_bytes = 0; + uint64_t total_bytes = 0; + if (rows == 0u || + !cuda_u64_mul_checked(DS4_ROCM_Q4_ATTN_Q_B_IN_DIM, + DS4_ROCM_Q4_ATTN_Q_B_OUT_DIM, + &weight_elems) || + !cuda_u64_mul_checked(weight_elems, sizeof(__half), + &weight_bytes) || + !cuda_u64_mul_checked(rows, DS4_ROCM_Q4_ATTN_Q_B_IN_DIM, + &x_elems) || + !cuda_u64_mul_checked(x_elems, sizeof(__half), &x_bytes) || + !cuda_u64_add_checked(weight_bytes, x_bytes, &total_bytes) || + total_bytes > (uint64_t)SIZE_MAX) { + return 0; + } + if (weight_bytes_out) *weight_bytes_out = weight_bytes; + if (x_bytes_out) *x_bytes_out = x_bytes; + if (total_bytes_out) *total_bytes_out = total_bytes; + return 1; +} + +/* Caller holds g_rocm_q4_attn_q_b_transient_f16_mu. Growth is a preflight + * operation. Allocate the replacement first, then synchronize and retire the + * old arena: an allocation failure must not silently destroy the capacity + * already advertised by the current cache generation. */ +static int rocm_q4_attn_q_b_transient_f16_ensure_locked( + uint64_t required_bytes, + uint64_t weight_bytes, + uint64_t working_set_reserve_bytes, + int *allocated_out) { + if (allocated_out) *allocated_out = 0; + if (required_bytes == 0u || weight_bytes == 0u || + required_bytes > (uint64_t)SIZE_MAX) { + return 0; + } + if (g_rocm_q4_attn_q_b_transient_f16_scratch && + g_rocm_q4_attn_q_b_transient_f16_weight_bytes == weight_bytes && + g_rocm_q4_attn_q_b_transient_f16_scratch_bytes >= required_bytes) { + return 1; + } + + if (!rocm_q4_attn_q_b_f16_memory_has_room( + required_bytes, working_set_reserve_bytes, + NULL, NULL, NULL)) { + return 0; + } + void *scratch = NULL; + cudaError_t err = cudaMalloc(&scratch, (size_t)required_bytes); + if (err != cudaSuccess || !scratch) { + fprintf(stderr, + DS4_GPU_LOG_PREFIX + "Q4 attn_q_b transient scratch allocation failed " + "(%.2f MiB): %s\n", + (double)required_bytes / 1048576.0, + cudaGetErrorString(err)); + (void)cudaGetLastError(); + return 0; + } + + if (g_rocm_q4_attn_q_b_transient_f16_scratch) { + err = cudaDeviceSynchronize(); + if (err != cudaSuccess) { + fprintf(stderr, + DS4_GPU_LOG_PREFIX + "Q4 attn_q_b transient scratch growth sync failed: %s\n", + cudaGetErrorString(err)); + (void)cudaGetLastError(); + (void)cudaFree(scratch); + return 0; + } + err = cudaFree(g_rocm_q4_attn_q_b_transient_f16_scratch); + if (err != cudaSuccess) { + fprintf(stderr, + DS4_GPU_LOG_PREFIX + "Q4 attn_q_b transient scratch free failed: %s\n", + cudaGetErrorString(err)); + (void)cudaGetLastError(); + (void)cudaFree(scratch); + return 0; + } + } + g_rocm_q4_attn_q_b_transient_f16_scratch = scratch; + g_rocm_q4_attn_q_b_transient_f16_scratch_bytes = required_bytes; + g_rocm_q4_attn_q_b_transient_f16_weight_bytes = weight_bytes; + if (allocated_out) *allocated_out = 1; + return 1; +} + +/* Success leaves the transient mutex held through the caller's complete GPU + * enqueue sequence. No allocation or synchronization is permitted here. */ +static int rocm_q4_attn_q_b_transient_f16_acquire( + uint64_t rows, + __half **weight_f16_out, + __half **x_f16_out) { + uint64_t weight_bytes = 0; + uint64_t total_bytes = 0; + if (!weight_f16_out || !x_f16_out || + !rocm_q4_attn_q_b_transient_f16_layout( + rows, &weight_bytes, NULL, &total_bytes)) { + return 0; + } + pthread_mutex_lock(&g_rocm_q4_attn_q_b_transient_f16_mu); + if (!g_rocm_q4_attn_q_b_transient_f16_scratch || + g_rocm_q4_attn_q_b_transient_f16_weight_bytes != weight_bytes || + g_rocm_q4_attn_q_b_transient_f16_scratch_bytes < total_bytes) { + pthread_mutex_unlock(&g_rocm_q4_attn_q_b_transient_f16_mu); + return 0; + } + *weight_f16_out = + (__half *)g_rocm_q4_attn_q_b_transient_f16_scratch; + *x_f16_out = (__half *)( + (char *)g_rocm_q4_attn_q_b_transient_f16_scratch + weight_bytes); + return 1; +} + +static void rocm_q4_attn_q_b_transient_f16_release_acquired(void) { + pthread_mutex_unlock(&g_rocm_q4_attn_q_b_transient_f16_mu); +} + static void rocm_q4_attn_q_b_f16_clear_locked( int reset_stats, int reset_circuit) { for (uint32_t i = 0; @@ -406,15 +617,16 @@ static void rocm_q4_attn_q_b_f16_clear_locked( } } -/* The caller owns build_mu. Keeping this separate lets make_room serialize - * against an unpublished build without recursively locking through the public - * release API. cache_mu remains held across synchronization and frees, so a - * successful lookup cannot race the last reference to its arena. */ +/* The caller owns build_mu. Keep both dispatch mutexes through synchronization + * and free: a persistent lookup cannot lose its arena, and a transient enqueue + * cannot lose the combined scratch while its final consumer is being queued. */ static int rocm_q4_attn_q_b_f16_release_with_build_lock( int reset_circuit) { + pthread_mutex_lock(&g_rocm_q4_attn_q_b_transient_f16_mu); pthread_mutex_lock(&g_rocm_q4_attn_q_b_f16_cache_mu); - if (g_rocm_q4_attn_q_b_f16_arena_count != 0u) { + if (g_rocm_q4_attn_q_b_f16_arena_count != 0u || + g_rocm_q4_attn_q_b_transient_f16_scratch) { const cudaError_t sync_err = cudaDeviceSynchronize(); if (sync_err != cudaSuccess) { fprintf(stderr, @@ -425,6 +637,7 @@ static int rocm_q4_attn_q_b_f16_release_with_build_lock( g_rocm_q4_attn_q_b_f16_hard_failure = 1; g_rocm_q4_attn_q_b_f16_pending_evict = 1; pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_cache_mu); + pthread_mutex_unlock(&g_rocm_q4_attn_q_b_transient_f16_mu); return 0; } } @@ -447,14 +660,32 @@ static int rocm_q4_attn_q_b_f16_release_with_build_lock( } } } + if (g_rocm_q4_attn_q_b_transient_f16_scratch) { + const cudaError_t free_err = + cudaFree(g_rocm_q4_attn_q_b_transient_f16_scratch); + if (free_err != cudaSuccess) { + fprintf(stderr, + DS4_GPU_LOG_PREFIX + "Q4 attn_q_b transient scratch free failed: %s\n", + cudaGetErrorString(free_err)); + (void)cudaGetLastError(); + ok = 0; + } else { + g_rocm_q4_attn_q_b_transient_f16_scratch = NULL; + g_rocm_q4_attn_q_b_transient_f16_scratch_bytes = 0; + g_rocm_q4_attn_q_b_transient_f16_weight_bytes = 0; + } + } if (!ok) { g_rocm_q4_attn_q_b_f16_hard_failure = 1; g_rocm_q4_attn_q_b_f16_pending_evict = 1; pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_cache_mu); + pthread_mutex_unlock(&g_rocm_q4_attn_q_b_transient_f16_mu); return 0; } rocm_q4_attn_q_b_f16_clear_locked(0, reset_circuit); pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_cache_mu); + pthread_mutex_unlock(&g_rocm_q4_attn_q_b_transient_f16_mu); return 1; } @@ -481,14 +712,17 @@ extern "C" uint64_t ds4_gpu_q4_attn_q_b_f16_cache_generation(void) { extern "C" int ds4_gpu_make_room_for_q4_attn_q_b_f16_session(void) { pthread_mutex_lock(&g_rocm_q4_attn_q_b_f16_build_mu); + pthread_mutex_lock(&g_rocm_q4_attn_q_b_transient_f16_mu); pthread_mutex_lock(&g_rocm_q4_attn_q_b_f16_cache_mu); const uint32_t entries = g_rocm_q4_attn_q_b_f16_entry_count; const uint64_t bytes = g_rocm_q4_attn_q_b_f16_bytes; const int needs_reset = entries != 0u || g_rocm_q4_attn_q_b_f16_arena_count != 0u || + g_rocm_q4_attn_q_b_transient_f16_scratch != NULL || g_rocm_q4_attn_q_b_f16_hard_failure || g_rocm_q4_attn_q_b_f16_pending_evict; pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_cache_mu); + pthread_mutex_unlock(&g_rocm_q4_attn_q_b_transient_f16_mu); if (!needs_reset) { pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_build_mu); return 1; @@ -505,6 +739,98 @@ extern "C" int ds4_gpu_make_room_for_q4_attn_q_b_f16_session(void) { return ok; } +static int rocm_q4_attn_q_b_prepare_transient_f16( + const void *model_map, + uint64_t model_size, + const ds4_gpu_q4_attn_q_b_f16_sidecar_desc *descs, + uint32_t count, + uint32_t max_prefill_rows, + uint64_t working_set_reserve_bytes, + uint64_t *prepared_bytes) { + const uint64_t min_tokens = + rocm_q4_attn_q_b_transient_f16_min_tokens(); + if ((uint64_t)max_prefill_rows < min_tokens || + !rocm_q4_attn_q_b_transient_f16_policy_allowed() || + rocm_q4_attn_q_b_f16_circuit_open() || + !g_cublas_ready || + model_map != g_model_host_base || + model_size != g_model_registered_size) { + return 0; + } + + /* Serialize prompt-aware preparation with cache construction, lifecycle + * release, and model-range teardown. The automatic path is deliberately + * stricter than the explicit persistent experiment: every q_b source must + * already belong to a device image or device-backed resident range, so + * preflight never registers host pages or populates the mutable cache. */ + pthread_mutex_lock(&g_rocm_q4_attn_q_b_f16_build_mu); + if (!rocm_q4_attn_q_b_transient_f16_policy_allowed() || + rocm_q4_attn_q_b_f16_circuit_open() || + !g_cublas_ready || + model_map != g_model_host_base || + model_size != g_model_registered_size) { + pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_build_mu); + return 0; + } + + uint64_t weight_f16_bytes = 0; + for (uint32_t i = 0; i < count; i++) { + uint64_t desc_f16_bytes = 0; + if (!rocm_q4_attn_q_b_f16_desc_valid( + &descs[i], model_size, &desc_f16_bytes) || + (weight_f16_bytes != 0u && + weight_f16_bytes != desc_f16_bytes)) { + pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_build_mu); + return 0; + } + weight_f16_bytes = desc_f16_bytes; + + if (!rocm_q4_attn_q_b_device_resident_source( + model_map, descs[i].weight_offset, + descs[i].weight_bytes)) { + pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_build_mu); + return 0; + } + } + + uint64_t layout_weight_bytes = 0; + uint64_t total_bytes = 0; + if (!rocm_q4_attn_q_b_transient_f16_layout( + max_prefill_rows, &layout_weight_bytes, NULL, &total_bytes) || + layout_weight_bytes != weight_f16_bytes) { + pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_build_mu); + return 0; + } + + int allocated = 0; + pthread_mutex_lock(&g_rocm_q4_attn_q_b_transient_f16_mu); + const int ready = rocm_q4_attn_q_b_transient_f16_ensure_locked( + total_bytes, layout_weight_bytes, working_set_reserve_bytes, + &allocated); + if (ready && allocated) { + pthread_mutex_lock(&g_rocm_q4_attn_q_b_f16_cache_mu); + if (++g_rocm_q4_attn_q_b_f16_generation == 0u) { + g_rocm_q4_attn_q_b_f16_generation = 1u; + } + pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_cache_mu); + } + pthread_mutex_unlock(&g_rocm_q4_attn_q_b_transient_f16_mu); + pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_build_mu); + if (!ready) return 0; + + if (allocated) { + if (prepared_bytes) *prepared_bytes = total_bytes; + fprintf(stderr, + DS4_GPU_LOG_PREFIX + "prepared %.2f MiB Q4 attn_q_b transient F16 scratch " + "for up to %u rows (min batch %llu tokens)\n", + (double)total_bytes / 1048576.0, + max_prefill_rows, + (unsigned long long)min_tokens); + } + return 1; +} + extern "C" int ds4_gpu_prepare_q4_attn_q_b_f16_sidecars( const void *model_map, uint64_t model_size, @@ -521,6 +847,18 @@ extern "C" int ds4_gpu_prepare_q4_attn_q_b_f16_sidecars( } const int required = rocm_q4_attn_q_b_f16_required(); + const int persistent_requested = + required || + (rocm_q4_attn_q_b_f16_enabled() && + !rocm_q4_attn_q_b_f16_disabled()); + /* ENABLE and REQUIRE deliberately select the multi-GiB persistent cache. + * DISABLE cancels a non-strict ENABLE back to transient, while REQUIRE + * still enters the persistent policy and reports DISABLE as a hard skip. */ + if (!persistent_requested) { + return rocm_q4_attn_q_b_prepare_transient_f16( + model_map, model_size, descs, count, max_prefill_rows, + working_set_reserve_bytes, prepared_bytes); + } const uint64_t min_tokens = rocm_q4_attn_q_b_f16_min_tokens(); if ((uint64_t)max_prefill_rows < min_tokens) return 0; if (!rocm_q4_attn_q_b_f16_policy_allowed() || !g_cublas_ready || @@ -539,6 +877,7 @@ extern "C" int ds4_gpu_prepare_q4_attn_q_b_f16_sidecars( } pthread_mutex_lock(&g_rocm_q4_attn_q_b_f16_build_mu); + pthread_mutex_lock(&g_rocm_q4_attn_q_b_f16_cache_mu); const int pending_evict = g_rocm_q4_attn_q_b_f16_pending_evict; pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_cache_mu); @@ -557,6 +896,41 @@ extern "C" int ds4_gpu_prepare_q4_attn_q_b_f16_sidecars( pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_build_mu); return required ? -1 : 0; } + if (rocm_q4_attn_q_b_f16_circuit_open()) { + pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_build_mu); + return required ? -1 : 0; + } + + /* The persistent sidecar still needs private X_F16 staging. Reuse the + * dedicated combined arena instead of the backend-global cuda_tmp buffer; + * dispatch holds transient_mu through conversion, GEMM, and epilogue. + * Keep the global lock order build -> transient -> cache. */ + uint64_t scratch_weight_bytes = 0; + uint64_t scratch_bytes = 0; + if (!rocm_q4_attn_q_b_transient_f16_layout( + max_prefill_rows, &scratch_weight_bytes, NULL, &scratch_bytes) || + scratch_weight_bytes != desc_f16_bytes[0]) { + pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_build_mu); + return required ? -1 : 0; + } + int scratch_allocated = 0; + pthread_mutex_lock(&g_rocm_q4_attn_q_b_transient_f16_mu); + const int scratch_ready = + rocm_q4_attn_q_b_transient_f16_ensure_locked( + scratch_bytes, scratch_weight_bytes, + working_set_reserve_bytes, &scratch_allocated); + if (scratch_ready && scratch_allocated) { + pthread_mutex_lock(&g_rocm_q4_attn_q_b_f16_cache_mu); + if (++g_rocm_q4_attn_q_b_f16_generation == 0u) { + g_rocm_q4_attn_q_b_f16_generation = 1u; + } + pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_cache_mu); + } + pthread_mutex_unlock(&g_rocm_q4_attn_q_b_transient_f16_mu); + if (!scratch_ready) { + pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_build_mu); + return required ? -1 : 0; + } uint32_t miss_indices[ DS4_ROCM_Q4_ATTN_Q_B_F16_CACHE_MAX_ENTRIES] = {0}; @@ -703,13 +1077,14 @@ extern "C" int ds4_gpu_prepare_q4_attn_q_b_f16_sidecars( const ds4_gpu_q4_attn_q_b_f16_sidecar_desc *desc = &descs[miss_indices[mi]]; sidecar_ptrs[mi] = (__half *)((char *)arena + arena_offset); - const uint64_t block_count = - (desc->in_dim / CUDA_QK_K) * desc->out_dim; + const uint64_t blocks_per_row = desc->in_dim / CUDA_QK_K; + const uint64_t total_chunks = + desc->out_dim * (desc->in_dim / 16u); rocm_dequant_q4_K_attn_q_b_f16_kernel<<< - (uint32_t)block_count, CUDA_QK_K>>>( + (uint32_t)((total_chunks + 255u) / 256u), 256>>>( sidecar_ptrs[mi], (const cuda_block_q4_K *)weight_ptrs[mi], - block_count); + desc->in_dim, desc->out_dim, blocks_per_row); err = cudaGetLastError(); if (err != cudaSuccess) { fprintf(stderr, From 49acc2ec06bd3105694e6876e10e81f4a30ffeae Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Wed, 26 Aug 2026 18:08:35 +0200 Subject: [PATCH 120/189] tests, docs: cover transient Q4 q_b prefill policies --- ENVIRONMENT_VARIABLES.md | 4 + Makefile | 12 +- scripts/environment_variables.tsv | 38 +- tests/test_metal_q4_qb_f16_cache.c | 1436 +++++++++++++++++++++++++--- 4 files changed, 1359 insertions(+), 131 deletions(-) diff --git a/ENVIRONMENT_VARIABLES.md b/ENVIRONMENT_VARIABLES.md index a8a8c6f6c..b04889e20 100644 --- a/ENVIRONMENT_VARIABLES.md +++ b/ENVIRONMENT_VARIABLES.md @@ -49,6 +49,8 @@ The detailed Metal A/B contracts and expected oracle counters live in | Variable | Default behavior and purpose | | --- | --- | | `DS4_CUDA_DECODE_GRAPHS=0` | Disable CUDA decode graph capture. Unset, `1`, `on`, `yes`, or `true` enables capture; `0`, `off`, `no`, or `false` disables it. Oracle modes may also suppress capture. | +| `DS4_CUDA_DISABLE_Q4_ATTN_Q_B_TRANSIENT_F16=1` | Restore native Q4_K `attn_q_b` for automatic long resident prefills. The transient path is otherwise eligible from 4096 tokens when the single-GPU model image is physically device-resident and retains only one reusable expanded matrix; explicit persistent-cache controls remain independent. | +| `DS4_CUDA_Q4_ATTN_Q_B_TRANSIENT_F16_MIN_TOKENS=N` | Override the automatic transient Q4_K-to-F16 crossover (default 4096 tokens) for single-GPU, device-image-resident, non-SSD, non-quality prefills. | | `DS4_CUDA_DISABLE_Q4_DENSE_PAIR=1` | Split the Q-A/KV Q4 pair back into two standalone projections. | | `DS4_CUDA_NO_Q4_GB10_FAST=1` | Umbrella rollback for the GB10-specific Q4 choices; it does not disable the older cross-CUDA dense pair. | | `DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_BATCH=1` | Enable grouped attention-A for two-to-eight-token GB10 verifier batches. | @@ -65,6 +67,8 @@ The detailed Metal A/B contracts and expected oracle counters live in | Variable | Default behavior and purpose | | --- | --- | +| `DS4_ROCM_DISABLE_Q4_ATTN_Q_B_TRANSIENT_F16=1` | Restore native Q4_K `attn_q_b` for automatic long resident prefills. The transient path is otherwise eligible from 4096 tokens when every `attn_q_b` source is already in a device image or device-backed resident range and retains only one reusable expanded matrix; explicit persistent-cache controls remain independent. | +| `DS4_ROCM_Q4_ATTN_Q_B_TRANSIENT_F16_MIN_TOKENS=N` | Override the automatic transient Q4_K-to-F16 crossover (default 4096 tokens) for device-resident, non-SSD, non-quality prefills. | | `DS4_ROCM_DISABLE_Q4_PREFILL_TILE8=1` | Restore the legacy Q4 prefill kernel. TILE8 is automatic for validated chunks of 9 through 4096 tokens. | | `DS4_ROCM_REQUIRE_Q4_PREFILL_TILE8=1` | Fail closed when an eligible Q4 prefill call cannot use TILE8. | | `DS4_ROCM_ENABLE_Q4_PREFILL_TILE8=1` | Legacy spelling retained for migration notes only. The runtime does not read it; TILE8 is automatic and this setting is ignored. | diff --git a/Makefile b/Makefile index 87999f657..cffc6b839 100644 --- a/Makefile +++ b/Makefile @@ -97,8 +97,8 @@ help: @echo " make test-metal-q4-streams Check resident Q4 Metal stream overlap" @echo " make test-metal-indexer-q4 Check the production-shape Q4_K indexer projection" @echo " make test-metal-q4-attn-exactn Bitwise/canary oracle for M1-M4 SSD-prefill Q4 attention output" - @echo " make test-metal-q4-qb-f16-cache Bitwise/canary oracle for the resident M1-M4 Q4 q_b F16 cache" - @echo " make test-metal-q4-qb-f16-cache-timing Time the Q4 q_b F16 cache at N=4096" + @echo " make test-metal-q4-qb-f16-cache Oracle for M1-M4 Q4 q_b sidecar and transient F16 paths" + @echo " make test-metal-q4-qb-f16-cache-timing Compare Q4 direct, sidecar, and transient production at N=4096" @echo " make test-metal-dspark-capture Check fused DSpark HC capture bitwise" @echo " make test-metal-iq2-midonly Check M1 IQ2 addr mid-only output and sentinels" @echo " make test-metal-iq2-live-index Check IQ2 SSD live-cache index policy and fallback" @@ -208,6 +208,10 @@ tests/test_metal_q4_qb_f16_cache: tests/test_metal_q4_qb_f16_cache.o ds4_metal.o test-metal-q4-qb-f16-cache: tests/test_metal_q4_qb_f16_cache env -u DS4_METAL_DISABLE_Q4_ATTN_Q_B_F16_CACHE \ + -u DS4_METAL_ENABLE_Q4_ATTN_Q_B_F16_CACHE_WITH_SSD_STREAMING \ + -u DS4_METAL_DISABLE_Q4_ATTN_Q_B_F16_RHS \ + -u DS4_METAL_DISABLE_Q4_ATTN_Q_B_TRANSIENT_F16 \ + -u DS4_METAL_Q4_ATTN_Q_B_TRANSIENT_F16_MIN_TOKENS \ -u DS4_TEST_METAL_Q4_QB_F16_CACHE_TIMING \ -u DS4_TEST_METAL_Q4_QB_F16_CACHE_TIMING_TOKENS \ DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MIN_TOKENS=32 \ @@ -216,6 +220,10 @@ test-metal-q4-qb-f16-cache: tests/test_metal_q4_qb_f16_cache test-metal-q4-qb-f16-cache-timing: tests/test_metal_q4_qb_f16_cache env -u DS4_METAL_DISABLE_Q4_ATTN_Q_B_F16_CACHE \ + -u DS4_METAL_ENABLE_Q4_ATTN_Q_B_F16_CACHE_WITH_SSD_STREAMING \ + -u DS4_METAL_DISABLE_Q4_ATTN_Q_B_F16_RHS \ + -u DS4_METAL_DISABLE_Q4_ATTN_Q_B_TRANSIENT_F16 \ + -u DS4_METAL_Q4_ATTN_Q_B_TRANSIENT_F16_MIN_TOKENS \ DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MIN_TOKENS=32 \ DS4_METAL_REQUIRE_Q4_ATTN_Q_B_F16_CACHE=1 \ DS4_TEST_METAL_Q4_QB_F16_CACHE_TIMING=1 \ diff --git a/scripts/environment_variables.tsv b/scripts/environment_variables.tsv index 8d3c9cf1a..b2bba9ea3 100644 --- a/scripts/environment_variables.tsv +++ b/scripts/environment_variables.tsv @@ -54,7 +54,8 @@ runtime/cuda DS4_CUDA_DISABLE_HC_NORM_MIX_FUSE presence kill switch; default uns runtime/cuda DS4_CUDA_DISABLE_HC_SPLIT_NORM_FUSED presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the fused HC split/weighted-sum/norm kernel. ds4_cuda.cu:31785 runtime/cuda DS4_CUDA_DISABLE_IQ2_XXS_SSD_PREFILL_MMQ false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Disable the CUDA IQ2 XXS SSD prefill MMQ optimization/path. ds4_cuda.cu:4627 runtime/cuda DS4_CUDA_DISABLE_Q4_ATTN_OUT_HC_FUSE presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable fused Q4 attention-output/HC expansion. ds4_cuda.cu:37357 -runtime/cuda DS4_CUDA_DISABLE_Q4_ATTN_Q_B_F16_CACHE value-aware rollback; unset/empty/0/false/no/off keeps the experiment available, any other nonempty value disables it Disable the resident Q4_K attn_q_b-to-F16 prefill cache even when ENABLE or REQUIRE is set. ds4_cuda.cu:2491 +runtime/cuda DS4_CUDA_DISABLE_Q4_ATTN_Q_B_F16_CACHE value-aware rollback; unset/empty/0/false/no/off keeps the experiment available, any other nonempty value disables it Disable the resident Q4_K attn_q_b-to-F16 prefill cache even when ENABLE or REQUIRE is set. ds4_cuda.cu:2502 +runtime/cuda DS4_CUDA_DISABLE_Q4_ATTN_Q_B_TRANSIENT_F16 value-aware rollback; unset/empty/0/false/no/off keeps the automatic transient path eligible, any other nonempty value disables it Disable per-layer transient Q4_K attn_q_b-to-F16 scratch for physically device-resident single-GPU model images; the existing resident-cache controls remain independent and otherwise eligible calls use native Q4. ds4_cuda.cu:2514 runtime/cuda DS4_CUDA_DISABLE_Q4_DENSE_PAIR presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q4 dense pair CUDA Q4 optimization. ds4_cuda.cu:20402 runtime/cuda DS4_CUDA_DISABLE_Q8_HC_EXPAND_FUSED false-like-aware flag, default off; 0/false/no/off is off, other nonempty values request split; force-fused wins Request the split Q8 shared-down/HC path when safe. ds4_cuda.cu:2086 runtime/cuda DS4_CUDA_DISABLE_QKV_RMS_FUSED presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the CUDA QKV RMS fused optimization/path. ds4_cuda.cu:369; ds4.c:17428 @@ -79,7 +80,7 @@ runtime/cuda DS4_CUDA_ENABLE_DSPARK_NONCAUSAL_ONLINE value-aware flag, default o runtime/cuda DS4_CUDA_ENABLE_HC_NORM_MIX_FUSE nonempty opt-in, default off; only exact 0 disables; the F32/F16 activation mode follows the selected standalone matmul path; disable/serial/alternate flags can veto Enable and select the fused HC RMSNorm-plus-mix one-token implementation. ds4_cuda.cu:20902 runtime/cuda DS4_CUDA_ENABLE_IQ2_XXS_SSD_PREFILL_MMQ false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Enable the CUDA IQ2 XXS SSD prefill MMQ experimental path. ds4_cuda.cu:4625 runtime/cuda DS4_CUDA_ENABLE_Q4_ATTN_OUT_HC_FUSE value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on Opt in to the fused Q4 attention-output/HC expansion path. ds4_cuda.cu:37375 -runtime/cuda DS4_CUDA_ENABLE_Q4_ATTN_Q_B_F16_CACHE value-aware opt-in, default off; unset/empty/0/false/no/off is off, any other nonempty value enables unless DISABLE wins Prewarm and use resident F16 sidecars for eligible single-GPU Q4_K attn_q_b prefills. ds4_cuda.cu:2479 +runtime/cuda DS4_CUDA_ENABLE_Q4_ATTN_Q_B_F16_CACHE value-aware persistent-cache opt-in, default off; unset/empty/0/false/no/off is off; DISABLE cancels an optional persistent request but leaves the automatic transient path independent; REQUIRE plus DISABLE fails closed Prewarm and use persistent resident F16 sidecars for eligible single-GPU Q4_K attn_q_b prefills. ds4_cuda.cu:2490 runtime/cuda DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_BATCH value-aware opt-in, default off; nonempty value other than exact 0 enables; rollback wins Enable flattened grouped attention-A MMQ for two-to-eight-token GB10 batches. cuda/mmq/ds4_mmq.cu:4303 runtime/cuda DS4_CUDA_ENABLE_Q4_K1024_PERSISTENT presence flag, default off; any defined value including 0 requests the path; rollback wins Enable the GB10 persistent-CTA kernel for M=32768, N=1, K=1024 Q4. cuda/mmq/ds4_mmq.cu:3905 runtime/cuda DS4_CUDA_ENABLE_Q8_FOLD strict flag, default off; only exact value 1 enables; overridden by DS4_CUDA_NO_Q8_FOLD Enable one-shot producer-to-consumer reuse of freshly quantized Q8_1 data. ds4_cuda.cu:785 @@ -276,8 +277,9 @@ runtime/cuda DS4_CUDA_PREFILL_PIPELINE_SEQUENTIAL presence flag; unset does not runtime/cuda DS4_CUDA_PREFILL_PIPELINE_SYNC_BOUNDARY presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Synchronize CUDA at every prefill pipeline tier boundary. ds4.c:35572 runtime/cuda DS4_CUDA_Q4_ATTN_OUT_HC_ORACLE value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on Compare fused Q4 attention-output/HC expansion with the canonical path and retain canonical output. ds4_cuda.cu:1475 runtime/cuda DS4_CUDA_Q4_ATTN_OUT_HC_Q8K_EXPERIMENT value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on Enable the experimental Q8_K-based Q4 attention-output/HC fusion. ds4_cuda.cu:37377 -runtime/cuda DS4_CUDA_Q4_ATTN_Q_B_F16_CACHE_MB unsigned integer MiB with a full-string parse; default 3072; 0 prevents admission; invalid text restores the default; overflow saturates Cap device memory used by resident Q4_K attn_q_b F16 sidecars. ds4_cuda.cu:2466 -runtime/cuda DS4_CUDA_Q4_ATTN_Q_B_F16_CACHE_MIN_TOKENS integer token count; default 512; valid values clamp to 32..UINT32_MAX; empty or invalid text restores 512 Set the minimum prefill batch eligible to prepare or use the CUDA Q4 attn_q_b F16 sidecars. ds4_cuda.cu:2473 +runtime/cuda DS4_CUDA_Q4_ATTN_Q_B_F16_CACHE_MB unsigned integer MiB with a full-string parse; default 3072; 0 prevents admission; invalid text restores the default; overflow saturates Cap device memory used by resident Q4_K attn_q_b F16 sidecars. ds4_cuda.cu:2477 +runtime/cuda DS4_CUDA_Q4_ATTN_Q_B_F16_CACHE_MIN_TOKENS integer token count; default 512; valid values clamp to 32..UINT32_MAX; empty or invalid text restores 512 Set the minimum prefill batch eligible to prepare or use the CUDA Q4 attn_q_b F16 sidecars. ds4_cuda.cu:2484 +runtime/cuda DS4_CUDA_Q4_ATTN_Q_B_TRANSIENT_F16_MIN_TOKENS full-string unsigned token count; default 4096; empty or invalid text restores 4096; parsed values clamp to 32..UINT32_MAX Set the minimum physically device-image-resident, single-GPU, non-SSD prefill batch eligible for per-layer transient CUDA Q4_K attn_q_b-to-F16 expansion. ds4_cuda.cu:2508 runtime/cuda DS4_CUDA_Q4_GROUPED_ATTN_A_ORACLE value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on Compare grouped attention-A against the canonical per-group result. ds4_cuda.cu:1477 runtime/cuda DS4_CUDA_Q4_K1024_PERSISTENT_ORACLE value-aware flag, default off; nonempty value other than exact 0 enables and implies candidate admission Bitwise-compare the exact-shape persistent Q4 K1024 kernel with canonical MMVQ and retain canonical output. cuda/mmq/ds4_mmq.cu:3738 runtime/cuda DS4_CUDA_Q4_K1024_PERSISTENT_STATS value-aware flag, default off; nonempty value other than exact 0 enables Print exact-shape persistent Q4 K1024 dispatch counters at exit. cuda/mmq/ds4_mmq.cu:3737 @@ -295,7 +297,7 @@ runtime/cuda DS4_CUDA_Q8_PAIR_BATCH presence flag; unset does not force the path runtime/cuda DS4_CUDA_QKV_KV_ROPE_FUSE value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control or tune the CUDA QKV KV rope fuse path. ds4.c:17429 runtime/cuda DS4_CUDA_Q_NORM_ROPE_FUSE value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables Control or tune the CUDA q norm rope fuse path. ds4.c:17418 runtime/cuda DS4_CUDA_REQUIRE_IQ2_XXS_SSD_PREFILL_MMQ false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Require the CUDA IQ2 XXS SSD prefill MMQ path; fail closed when unavailable. ds4_cuda.cu:4631 -runtime/cuda DS4_CUDA_REQUIRE_Q4_ATTN_Q_B_F16_CACHE value-aware strict opt-in, default off; unset/empty/0/false/no/off is off, any other nonempty value requires eligible batches to use the cache; DISABLE wins Fail an eligible CUDA prefill instead of falling back when the resident Q4_K attn_q_b F16 specialization cannot be prepared or dispatched. ds4_cuda.cu:2481; ds4_cuda.cu:2486 +runtime/cuda DS4_CUDA_REQUIRE_Q4_ATTN_Q_B_F16_CACHE value-aware strict opt-in, default off; unset/empty/0/false/no/off is off, any other nonempty value requires eligible batches to use the cache; DISABLE wins Fail an eligible CUDA prefill instead of falling back when the resident Q4_K attn_q_b F16 specialization cannot be prepared or dispatched. ds4_cuda.cu:2492; ds4_cuda.cu:2497 runtime/cuda DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_BATCH value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on Fail if grouped batched attention-A cannot be used. ds4_cuda.cu:40198 runtime/cuda DS4_CUDA_REQUIRE_Q4_K1024_PERSISTENT presence flag, default off; any defined value including 0 makes ineligible candidate fail closed Fail when the exact Q4 K1024 persistent candidate is unavailable instead of using MMVQ. cuda/mmq/ds4_mmq.cu:3929 runtime/cuda DS4_CUDA_REQUIRE_STREAMING_EXPERT_PERSISTENT_CACHE false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Require streaming expert persistent cache in CUDA SSD streaming; fail closed when unavailable. ds4_cuda.cu:4117 @@ -589,7 +591,9 @@ runtime/metal DS4_METAL_DISABLE_PRO_Q4_EXPERT_TABLE_PRELOAD presence rollback; u runtime/metal DS4_METAL_DISABLE_Q4_ATTN_OUT_B_F16_RHS value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Disables resident pre-M5 Q4 attention output-B F16 RHS materialization and restores per-tile F32 staging. ds4_metal.m:28533 runtime/metal DS4_METAL_DISABLE_Q4_ATTN_OUT_HC_FUSE presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 attn out HC fuse. ds4_metal.m:47624 runtime/metal DS4_METAL_DISABLE_Q4_ATTN_OUT_TINY_BATCH value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Disables Q4 attn out tiny batch. ds4_metal.m:28396 -runtime/metal DS4_METAL_DISABLE_Q4_ATTN_Q_B_F16_CACHE value-aware boolean; unset/0: automatic path remains enabled; empty/1/true/yes/on disables Disables the resident pre-M5 Q4_K attn_q_b F16 weight sidecar and restores native per-tile Q4 dequantization. ds4_metal.m:25485; ds4_metal.m:25722 +runtime/metal DS4_METAL_DISABLE_Q4_ATTN_Q_B_F16_CACHE value-aware boolean; unset/0: persistent sidecar remains eligible; empty/1/true/yes/on disables; REQUIRE plus DISABLE fails closed Disables only the persistent pre-M5 Q4_K attn_q_b F16 weight sidecar; unless REQUIRE is set, the default transient F16 path remains independently eligible, so also set DS4_METAL_DISABLE_Q4_ATTN_Q_B_TRANSIENT_F16=1 to force native per-tile Q4 dequantization. ds4_metal.m:25819; ds4_metal.m:26131 +runtime/metal DS4_METAL_DISABLE_Q4_ATTN_Q_B_F16_RHS value-aware boolean; unset/0/false/no/off keeps the resident path enabled; empty/1/true/yes/on disables Disables one-time F32-to-F16 RHS materialization for the resident pre-M5 Q4_K attn_q_b F16 sidecar and restores repeated per-tile F32 staging. ds4_metal.m:25821; ds4_metal.m:26136 +runtime/metal DS4_METAL_DISABLE_Q4_ATTN_Q_B_TRANSIENT_F16 value-aware rollback; unset/0/false/no/off keeps the automatic path enabled; empty/1/true/yes/on disables Disables per-layer transient Q4_K attn_q_b-to-F16 scratch for long resident, non-SSD prefill; an enabled/required and eligible persistent sidecar may still run, otherwise dispatch returns to native Q4. ds4_metal.m:25804; ds4_metal.m:26707 runtime/metal DS4_METAL_DISABLE_Q4_BATCH_EXPERT_TABLE presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 batch expert table. ds4_metal.m:44883 runtime/metal DS4_METAL_DISABLE_Q4_DENSE_PAIR presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 dense pair. ds4_metal.m:21990 runtime/metal DS4_METAL_DISABLE_Q4_EXACT_BOUNDARY presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 exact boundary. ds4_metal.m:42239 @@ -697,6 +701,7 @@ runtime/metal DS4_METAL_ENABLE_PRO_Q4_EXPERT_ADDRESS_AUTO presence opt-in; unset runtime/metal DS4_METAL_ENABLE_PRO_Q4_EXPERT_TABLE_AUTO presence opt-in; unset: off/automatic; any value including 0 enables Enables pro Q4 expert table auto. ds4.c:20981 runtime/metal DS4_METAL_ENABLE_PRO_Q4_SELECTED_EXPERT_VIEWS presence opt-in; unset: off/automatic; any value including 0 enables Enables pro Q4 selected expert views. ds4.c:20978 runtime/metal DS4_METAL_ENABLE_Q4_ATTN_OUT_TINY_BATCH value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Enables Q4 attn out tiny batch. ds4_metal.m:28405 +runtime/metal DS4_METAL_ENABLE_Q4_ATTN_Q_B_F16_CACHE_WITH_SSD_STREAMING value-aware opt-in, default off; empty/1/true/yes/on enables and 0/false/no/off disables; DS4_METAL_DISABLE_Q4_ATTN_Q_B_F16_CACHE wins Allow resident Q4_K attn_q_b F16 sidecars while Metal continues streaming routed experts from SSD; admission reserves the configured streaming cache and prefill headroom and enforces the 7/8 working-set safety limit. ds4_metal.m:25488; ds4_metal.m:25808; ds4_metal.m:26711 runtime/metal DS4_METAL_ENABLE_Q4_BATCH_EXPERT_TABLE presence opt-in; unset: off/automatic; any value including 0 enables Enables Q4 batch expert table. ds4_metal.m:44869 runtime/metal DS4_METAL_ENABLE_Q4_EXACT_TENSOR_ID presence opt-in; unset: off/automatic; any value including 0 enables Enables Q4 exact tensor ID. ds4_metal.m:42218 runtime/metal DS4_METAL_ENABLE_Q4_EXPERT_ADDRESS_TABLE presence opt-in; unset: off/automatic; any value including 0 enables Enables Q4 expert address table. ds4.c:20980 @@ -811,9 +816,10 @@ runtime/metal DS4_METAL_PREFILL_CHUNK positive token count used only when CLI ch runtime/metal DS4_METAL_PRO_Q4_CPU_ROUTER nonempty boolean; unset/empty or exact 0: off; every other value: on Uses the CPU router for PRO Q4 selected-expert decode. ds4.c:20934 runtime/metal DS4_METAL_PRO_Q4_CPU_ROUTER_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for pro Q4 CPU router. ds4.c:21644 runtime/metal DS4_METAL_Q4_ADDR_USE_RESOURCES presence control; unset: off/default; any value including 0 enables Declares Q4 address-table resources explicitly on encoders. ds4_metal.m:20259 -runtime/metal DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MB integer 1..65536 MiB; default 3072; above max clamps, below min/invalid restores default Caps copied F16 sidecar storage for resident Q4_K attn_q_b weights. ds4_metal.m:25523; ds4_metal.m:25774 -runtime/metal DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MIN_TOKENS integer 32..UINT32_MAX; default 512; below min/invalid restores default Sets the minimum resident prefill batch that may build or use the Q4_K attn_q_b F16 sidecar; smaller tails remain non-candidates. ds4_metal.m:25487; ds4_metal.m:25724 -runtime/metal DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints resident Q4_K attn_q_b F16 cache counters and circuit state at Metal cleanup. ds4_metal.m:11399 +runtime/metal DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MB integer 1..65536 MiB; default 3072; above max clamps, below min/invalid restores default Caps copied F16 sidecar storage for resident Q4_K attn_q_b weights. ds4_metal.m:25861; ds4_metal.m:26210 +runtime/metal DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MIN_TOKENS integer 32..UINT32_MAX; default 512; below min/invalid restores default Sets the minimum resident, or explicitly enabled SSD-hybrid, prefill batch that may build or use the Q4_K attn_q_b F16 sidecar; smaller tails remain non-candidates. ds4_metal.m:25823; ds4_metal.m:26138 +runtime/metal DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints resident Q4_K attn_q_b F16 cache counters and circuit state at Metal cleanup. ds4_metal.m:11494 +runtime/metal DS4_METAL_Q4_ATTN_Q_B_TRANSIENT_F16_MIN_TOKENS integer token count; default 4096; invalid or values below 32 restore 4096 Sets the minimum long-N resident, non-SSD prefill batch eligible for per-layer transient Q4_K attn_q_b-to-F16 scratch; smaller chunks and tails stay on native Q4. ds4_metal.m:25688; ds4_metal.m:26529 runtime/metal DS4_METAL_Q4_EXPERT_GROUP_SIZE positive uint32; default 32; clamped to total expert count Sets experts processed per grouped Q4 dispatch. ds4_metal.m:34088 runtime/metal DS4_METAL_Q4_EXPERT_TABLE_GROUP_SIZE integer 2..total experts; default/invalid 1 (ungrouped) Sets grouped exact-view width while building Q4 expert tables. ds4_metal.m:19604 runtime/metal DS4_METAL_Q4_EXPERT_TABLE_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for Q4 expert table. ds4_metal.m:20103 @@ -849,7 +855,7 @@ runtime/metal DS4_METAL_REQUIRE_M1_IQ2_MID_ONLY presence strict check; unset: fa runtime/metal DS4_METAL_REQUIRE_OUTPUT_HC_WEIGHTS4 presence strict check; unset: fallback allowed; any value including 0 requires the path Requires output HC weights4 and makes eligible fallback fail closed. ds4_metal.m:46789 runtime/metal DS4_METAL_REQUIRE_Q4_ATTN_OUT_B_F16_RHS value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Requires the resident pre-M5 Q4 attention output-B F16 RHS path and makes an ineligible or disabled candidate fail closed. ds4_metal.m:28531 runtime/metal DS4_METAL_REQUIRE_Q4_ATTN_OUT_TINY_BATCH value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Requires Q4 attn out tiny batch and makes eligible fallback fail closed. ds4_metal.m:28373 -runtime/metal DS4_METAL_REQUIRE_Q4_ATTN_Q_B_F16_CACHE value-aware boolean; unset/0: fallback allowed; empty/1/true/yes/on requires Requires the sidecar for resident pre-M5 Q4_K Flash attn_q_b batches at or above the configured minimum; decode and below-min batches remain non-candidates. ds4_metal.m:25483; ds4_metal.m:25720 +runtime/metal DS4_METAL_REQUIRE_Q4_ATTN_Q_B_F16_CACHE value-aware boolean; unset/0: fallback allowed; empty/1/true/yes/on requires; DISABLE fails closed Requires the sidecar for resident, or explicitly enabled SSD-hybrid, pre-M5 Q4_K Flash attn_q_b batches at or above the configured minimum; decode and below-min batches remain non-candidates. ds4_metal.m:25801; ds4_metal.m:26129; ds4_metal.m:26704 runtime/metal DS4_METAL_REQUIRE_Q4_SSD_PREFILL_ATTN_OUT_EXACTN value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Requires Q4 SSD prefill attn out exactn and makes eligible fallback fail closed. ds4_metal.m:28089 runtime/metal DS4_METAL_REQUIRE_Q4_SSD_PREFILL_ATTN_OUT_SCALE_META value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Requires shared scale/min metadata in the Q4 SSD prefill attention-output exact-N kernel and makes fallback fail closed. ds4_metal.m:28209 runtime/metal DS4_METAL_REQUIRE_Q4_SSD_SESSION_UNION nonempty boolean; unset/empty or exact 0: off; every other value: on Requires Q4 SSD session union and makes eligible fallback fail closed. ds4.c:65164 @@ -939,7 +945,8 @@ runtime/rocm DS4_ROCM_DISABLE_GLM_STREAMING_PREFILL_SELECTED_ASYNC_LOAD presence runtime/rocm DS4_ROCM_DISABLE_GLM_STREAMING_SELECTED_ASYNC_LOAD presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable glm streaming selected async load. ds4.c:44328 runtime/rocm DS4_ROCM_DISABLE_IQ2_SELECTED_EXPERT_VIEWS presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable iq2 selected expert views. ds4.c:21079 runtime/rocm DS4_ROCM_DISABLE_IQ2_STREAM_ADDR_TABLE presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable iq2 stream addr table. ds4.c:6548 -runtime/rocm DS4_ROCM_DISABLE_Q4_ATTN_Q_B_F16_CACHE value-aware rollback; unset/0/false/no/off keeps the experiment available, empty or any other value disables it Disable the resident ROCm Q4_K attn_q_b-to-F16 prefill cache even when ENABLE or REQUIRE is set. rocm/ds4_rocm_q4_qb_sidecar.cuh:130 +runtime/rocm DS4_ROCM_DISABLE_Q4_ATTN_Q_B_F16_CACHE value-aware rollback; unset/0/false/no/off keeps the experiment available, empty or any other value disables it Disable the resident ROCm Q4_K attn_q_b-to-F16 prefill cache even when ENABLE or REQUIRE is set. rocm/ds4_rocm_q4_qb_sidecar.cuh:140 +runtime/rocm DS4_ROCM_DISABLE_Q4_ATTN_Q_B_TRANSIENT_F16 value-aware rollback; unset/0/false/no/off keeps the automatic path eligible, empty or any other non-false value disables it Disable per-layer transient Q4_K attn_q_b-to-F16 scratch for device-image or device-range-resident weights; the existing resident-cache controls remain independent and otherwise eligible calls use native Q4. rocm/ds4_rocm_q4_qb_sidecar.cuh:145 runtime/rocm DS4_ROCM_DISABLE_Q4_DENSE_PAIR presence rollback; unset leaves opt-in policy unchanged Disable/roll back rocm disable q4 dense pair. rocm/ds4_rocm_q4.cuh:435 runtime/rocm DS4_ROCM_DISABLE_Q4_GROUPED_ATTN_A presence rollback; DISABLE wins over enable/require Disable/roll back rocm disable q4 grouped attn a. rocm/ds4_rocm_q4.cuh:700 runtime/rocm DS4_ROCM_DISABLE_Q4_PREFILL_TILE8 presence rollback; TILE8 is default for 9..4096 tokens Disable/roll back rocm disable q4 prefill tile8. rocm/ds4_rocm_q4.cuh:448 @@ -977,7 +984,7 @@ runtime/rocm DS4_ROCM_ENABLE_MXFP4_LDSB presence opt-in; unset=off; any defined runtime/rocm DS4_ROCM_ENABLE_MXFP4_ROW64 presence opt-in; unset=off; any defined value including empty or 0 enables the candidate when the MXFP4 sorted-tile path has at least 8 tokens and the TILE32, LDSB, and TILE4 candidates are not selected Select the ROCm MXFP4 gate/up tile8 occupancy variant with 64 row slots and 512 threads per block. rocm/ds4_rocm_moe_launch.cuh:798 runtime/rocm DS4_ROCM_ENABLE_MXFP4_TILE32 presence opt-in; unset=off; any defined value including empty or 0 enables the candidate when the MXFP4 sorted-tile path has at least 32 tokens and the expert intermediate dimension is divisible by 32 Select the ROCm MXFP4 gate/up tile32 kernel, reusing each loaded expert-weight chunk across as many as 32 tokens. rocm/ds4_rocm_moe_launch.cuh:786 runtime/rocm DS4_ROCM_ENABLE_MXFP4_TILE4 presence opt-in; unset=off; any defined value including empty or 0 enables the candidate when the MXFP4 sorted-tile path has at least 5 tokens and neither TILE32 nor LDSB is selected Select the ROCm MXFP4 gate/up tile4 occupancy variant, reducing staged-activation LDS per block. rocm/ds4_rocm_moe_launch.cuh:794 -runtime/rocm DS4_ROCM_ENABLE_Q4_ATTN_Q_B_F16_CACHE value-aware opt-in, default off; unset/0/false/no/off is off, empty or any other value enables unless DISABLE wins Prewarm and use resident F16 sidecars for eligible ROCm Q4_K attn_q_b prefills. rocm/ds4_rocm_q4_qb_sidecar.cuh:120 +runtime/rocm DS4_ROCM_ENABLE_Q4_ATTN_Q_B_F16_CACHE value-aware persistent-cache opt-in, default off; unset/0/false/no/off is off; DISABLE cancels an optional persistent request but leaves the automatic transient path independent; REQUIRE plus DISABLE fails closed Prewarm and use persistent resident F16 sidecars for eligible ROCm Q4_K attn_q_b prefills. rocm/ds4_rocm_q4_qb_sidecar.cuh:130 runtime/rocm DS4_ROCM_ENABLE_Q4_DENSE_PAIR presence opt-in; unset=off; DISABLE takes precedence Enable rocm enable q4 dense pair. rocm/ds4_rocm_q4.cuh:434 runtime/rocm DS4_ROCM_ENABLE_Q4_GROUPED_ATTN_A presence opt-in; unset=off unless REQUIRE; DISABLE wins Enable rocm enable q4 grouped attn a. rocm/ds4_rocm_q4.cuh:704 runtime/rocm DS4_ROCM_ENABLE_STREAMING_FULL_EXPERT_ADDR_TABLE presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming full expert addr table. ds4.c:18240 @@ -1031,13 +1038,14 @@ runtime/rocm DS4_ROCM_MOE_DECODE_RPB sampled once; nonempty value is parsed by s runtime/rocm DS4_ROCM_MOE_PATH_DEBUG presence diagnostic; unset=off; any defined value including empty or 0 enables it Print ROCm routed-MoE path selection, sorted-tile scratch state, and MXFP4 gate/up launch diagnostics to stderr. rocm/ds4_rocm_moe_launch.cuh:831 runtime/rocm DS4_ROCM_MOE_WRITE_CLAMPED_ACT Pure presence sentinel: any defined value, including empty or "0", is active; DS4_METAL_MOE_WRITE_CLAMPED_ACT is an accepted fallback alias. On ROCm the variable is only consumed as a path-admission veto: it disables selected-expert cache/address-table, selected-slot and CPU-router/fused optimized paths. No ROCm call site parses a clamp amount or directly enables a write-clamped kernel. Force shared graph selection away from optimizations incompatible with the clamped-intermediate MoE diagnostic; on ROCm this is a compatibility/rollback gate, not itself a clamped-write implementation. ds4.c:18526 runtime/rocm DS4_ROCM_MXFP4_DOWN_RGROUP nonempty value is parsed by strtol and a numeric prefix is sufficient; integers 1..8 are accepted; unset, empty, invalid, or out-of-range values use 1 Set how many 32-row output blocks each ROCm MXFP4 tiled down-projection block computes, reducing the first launch-grid dimension as the value increases. rocm/ds4_rocm_moe_launch.cuh:801 -runtime/rocm DS4_ROCM_Q4_ATTN_Q_B_F16_CACHE_MB integer MiB with a full-string parse; default 3072; accepted range 1..65536, values above clamp and invalid or smaller values restore the default Cap device memory used by resident ROCm Q4_K attn_q_b F16 sidecars. rocm/ds4_rocm_q4_qb_sidecar.cuh:141 -runtime/rocm DS4_ROCM_Q4_ATTN_Q_B_F16_CACHE_MIN_TOKENS integer token count with a full-string parse; default 512; accepted range 32..UINT32_MAX, values above clamp and invalid or smaller values restore the default Set the minimum prefill batch eligible to prepare or use the ROCm Q4 attn_q_b F16 sidecars. rocm/ds4_rocm_q4_qb_sidecar.cuh:135 +runtime/rocm DS4_ROCM_Q4_ATTN_Q_B_F16_CACHE_MB integer MiB with a full-string parse; default 3072; accepted range 1..65536, values above clamp and invalid or smaller values restore the default Cap device memory used by resident ROCm Q4_K attn_q_b F16 sidecars. rocm/ds4_rocm_q4_qb_sidecar.cuh:162 +runtime/rocm DS4_ROCM_Q4_ATTN_Q_B_F16_CACHE_MIN_TOKENS integer token count with a full-string parse; default 512; accepted range 32..UINT32_MAX, values above clamp and invalid or smaller values restore the default Set the minimum prefill batch eligible to prepare or use the ROCm Q4 attn_q_b F16 sidecars. rocm/ds4_rocm_q4_qb_sidecar.cuh:156 +runtime/rocm DS4_ROCM_Q4_ATTN_Q_B_TRANSIENT_F16_MIN_TOKENS full-string unsigned token count; default 4096; accepted range 32..UINT32_MAX; values above clamp and invalid or smaller values restore 4096 Set the minimum device-resident, non-SSD prefill batch eligible for per-layer transient ROCm Q4_K attn_q_b-to-F16 expansion. rocm/ds4_rocm_q4_qb_sidecar.cuh:150 runtime/rocm DS4_ROCM_Q4_GROUPED_ATTN_A_STATS presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Print counters for rocm q4 grouped attn a stats. rocm/ds4_rocm_q4.cuh:480 runtime/rocm DS4_ROCM_Q4_PREFILL_TILE8_STATS presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Print counters for rocm q4 prefill tile8 stats. rocm/ds4_rocm_q4.cuh:514 runtime/rocm DS4_ROCM_Q8_DECODE_SHAREDX_64K sampled once; unset: enabled; present empty or exact 0: disabled; every other present value: enabled; effective only for one-token non-prequant Q8_0 matmul with 8192 < in_dim <= 16384 Allows the ROCm shared-input Q8 decode kernel to use up to 64 KiB dynamic LDS for wide inputs; an unsupported/failed LDS launch automatically falls back to the regular kernel. rocm/ds4_rocm_runtime.cuh:4805 runtime/rocm DS4_ROCM_Q_STAGE_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm q stage profile. ds4.c:30038 -runtime/rocm DS4_ROCM_REQUIRE_Q4_ATTN_Q_B_F16_CACHE value-aware strict opt-in, default off; unset/0/false/no/off is off, empty or any other value requires eligible batches to use the cache; DISABLE wins Fail an eligible ROCm prefill instead of falling back when the resident Q4_K attn_q_b F16 specialization cannot be prepared or dispatched. rocm/ds4_rocm_q4_qb_sidecar.cuh:125 +runtime/rocm DS4_ROCM_REQUIRE_Q4_ATTN_Q_B_F16_CACHE value-aware strict opt-in, default off; unset/0/false/no/off is off, empty or any other value requires eligible batches to use the cache; DISABLE wins Fail an eligible ROCm prefill instead of falling back when the resident Q4_K attn_q_b F16 specialization cannot be prepared or dispatched. rocm/ds4_rocm_q4_qb_sidecar.cuh:135 runtime/rocm DS4_ROCM_REQUIRE_Q4_GROUPED_ATTN_A presence fail-closed assertion; also requests candidate unless disabled Require rocm require q4 grouped attn a and fail instead of silently falling back. rocm/ds4_rocm_q4.cuh:702 runtime/rocm DS4_ROCM_REQUIRE_Q4_PREFILL_TILE8 presence fail-closed assertion for eligible TILE8 calls Require rocm require q4 prefill tile8 and fail instead of silently falling back. rocm/ds4_rocm_q4.cuh:452 runtime/rocm DS4_ROCM_STREAMING_DECODE_PREFILL_MAX primary nonempty value over the Metal alias; parsed by strtol when it has a numeric prefix (trailing text is accepted); <= 0 disables, values > UINT32_MAX clamp, no numeric prefix uses automatic default: 64 for Flash with uniform Q4_K/MXFP4 experts, 18 for other Pro/Flash, otherwise 0; the disable flag dominates Sets the largest short, non-quality SSD-streaming prefill batch routed through the decode-style path instead of canonical layer-major prefill. ds4.c:31961 diff --git a/tests/test_metal_q4_qb_f16_cache.c b/tests/test_metal_q4_qb_f16_cache.c index c6d66d577..a036f17e2 100644 --- a/tests/test_metal_q4_qb_f16_cache.c +++ b/tests/test_metal_q4_qb_f16_cache.c @@ -8,6 +8,7 @@ #include "ds4_gpu.h" #include +#include #include #include #include @@ -37,7 +38,8 @@ enum { GROUP_SIZE = 32u, GUARD_FLOATS = 256u, GUARD_HALFS = 256u, - TIMING_SAMPLES = 5u, + TIMING_SAMPLES = 8u, + SSD_SOURCE_LEADING = 128u, }; typedef struct { @@ -49,10 +51,18 @@ typedef struct { static const char *k_disable = "DS4_METAL_DISABLE_Q4_ATTN_Q_B_F16_CACHE"; +static const char *k_enable_ssd_streaming = + "DS4_METAL_ENABLE_Q4_ATTN_Q_B_F16_CACHE_WITH_SSD_STREAMING"; +static const char *k_disable_f16_rhs = + "DS4_METAL_DISABLE_Q4_ATTN_Q_B_F16_RHS"; +static const char *k_disable_transient_f16 = + "DS4_METAL_DISABLE_Q4_ATTN_Q_B_TRANSIENT_F16"; static const char *k_require = "DS4_METAL_REQUIRE_Q4_ATTN_Q_B_F16_CACHE"; static const char *k_min_tokens = "DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MIN_TOKENS"; +static const char *k_transient_min_tokens = + "DS4_METAL_Q4_ATTN_Q_B_TRANSIENT_F16_MIN_TOKENS"; static const char *k_cache_mb = "DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MB"; static const char *k_timing = @@ -155,23 +165,31 @@ static void poison_f16(uint16_t *values, uint64_t count) { } } -static uint64_t count_poison_f16_mismatches(const uint16_t *values, - uint64_t count) { +static uint64_t count_poison_f16_mismatches_range( + const uint16_t *values, + uint64_t begin, + uint64_t end) { uint64_t mismatches = 0; - for (uint64_t i = 0; i < count; i++) { + for (uint64_t i = begin; i < end; i++) { if (values[i] != half_poison_bits(i)) mismatches++; } return mismatches; } +static uint64_t count_poison_f16_mismatches(const uint16_t *values, + uint64_t count) { + return count_poison_f16_mismatches_range(values, 0, count); +} + static void fill_inputs(float *values) { for (uint32_t token = 0; token < MAX_TOKENS; token++) { for (uint32_t col = 0; col < IN_DIM; col++) { const uint32_t key = token * 131u + col * 17u + ((col >> 3u) ^ (token * 29u)); - /* The Metal MM kernels consume a half RHS. */ + /* Exercise values that are not exactly representable as half; + * the F16-RHS path must match the legacy per-tile narrowing. */ values[(uint64_t)token * IN_DIM + col] = - (float)((int)(key % 129u) - 64) / 64.0f; + (float)((int)(key % 129u) - 64) / 509.0f; } } } @@ -191,13 +209,14 @@ static uint64_t count_bit_mismatches(const float *reference, return mismatches; } -static int run_reference(ds4_gpu_tensor *out, - const void *model, - uint64_t model_bytes, - const ds4_gpu_tensor *x, - uint32_t n_tok) { +static int run_reference_at(ds4_gpu_tensor *out, + const void *model, + uint64_t model_bytes, + uint64_t weight_offset, + const ds4_gpu_tensor *x, + uint32_t n_tok) { if (!ds4_gpu_matmul_quant_tensor( - out, model, model_bytes, 0, Q4_K_TYPE, + out, model, model_bytes, weight_offset, Q4_K_TYPE, IN_DIM, OUT_DIM, x, n_tok)) { return 0; } @@ -207,19 +226,105 @@ static int run_reference(ds4_gpu_tensor *out, 10000.0f, 1.0f, 0.0f, 1.0f, 32.0f, 1.0f, 1.0e-6f); } -static int run_candidate(ds4_gpu_tensor *out, - ds4_gpu_tensor *q_half, +static int run_reference(ds4_gpu_tensor *out, const void *model, uint64_t model_bytes, const ds4_gpu_tensor *x, uint32_t n_tok) { + return run_reference_at( + out, model, model_bytes, 0u, x, n_tok); +} + +static int run_candidate_at(ds4_gpu_tensor *out, + ds4_gpu_tensor *q_half, + const void *model, + uint64_t model_bytes, + uint64_t weight_offset, + const ds4_gpu_tensor *x, + uint32_t n_tok) { return ds4_gpu_attn_q_b_f16_head_rms_rope_tail_tensor( - out, q_half, model, model_bytes, 0, Q4_K_TYPE, + out, q_half, model, model_bytes, weight_offset, Q4_K_TYPE, IN_DIM, OUT_DIM, x, n_tok, N_HEAD, HEAD_DIM, N_ROT, 17u, 0u, false, 10000.0f, 1.0f, 0.0f, 1.0f, 32.0f, 1.0f, 1.0e-6f); } +static int run_candidate(ds4_gpu_tensor *out, + ds4_gpu_tensor *q_half, + const void *model, + uint64_t model_bytes, + const ds4_gpu_tensor *x, + uint32_t n_tok) { + return run_candidate_at( + out, q_half, model, model_bytes, 0u, x, n_tok); +} + +static const char *mm_arm_name(ds4_gpu_test_q4_qb_mm_arm arm) { + switch (arm) { + case DS4_GPU_TEST_Q4_QB_MM_Q4_F32: return "Q4/F32"; + case DS4_GPU_TEST_Q4_QB_MM_Q4_F16: return "Q4/F16"; + case DS4_GPU_TEST_Q4_QB_MM_F16_F32: return "F16/F32"; + case DS4_GPU_TEST_Q4_QB_MM_F16_F16: return "F16/F16"; + case DS4_GPU_TEST_Q4_QB_MM_Q4_TRANSIENT_F16_F16: + return "Q4 transient/F16/F16"; + default: return "invalid"; + } +} + +static bool mm_arm_uses_f16_rhs(ds4_gpu_test_q4_qb_mm_arm arm) { + switch (arm) { + case DS4_GPU_TEST_Q4_QB_MM_Q4_F16: + case DS4_GPU_TEST_Q4_QB_MM_F16_F16: + case DS4_GPU_TEST_Q4_QB_MM_Q4_TRANSIENT_F16_F16: + return true; + default: + return false; + } +} + +static bool mm_arm_is_transient(ds4_gpu_test_q4_qb_mm_arm arm) { + return arm == DS4_GPU_TEST_Q4_QB_MM_Q4_TRANSIENT_F16_F16; +} + +static bool mm_arm_is_prepacked_experiment( + ds4_gpu_test_q4_qb_mm_arm arm) { + return mm_arm_is_transient(arm); +} + +static int run_mm_arm_projection( + ds4_gpu_tensor *out, + ds4_gpu_tensor *rhs_f16, + const void *model, + uint64_t model_bytes, + const ds4_gpu_tensor *x, + uint32_t n_tok, + ds4_gpu_test_q4_qb_mm_arm arm, + bool materialize_rhs) { + return ds4_gpu_test_q4_attn_q_b_mm_variant_tensor( + out, rhs_f16, model, model_bytes, 0u, IN_DIM, OUT_DIM, + x, n_tok, arm, materialize_rhs); +} + +static int run_mm_arm_with_tail( + ds4_gpu_tensor *out, + ds4_gpu_tensor *rhs_f16, + const void *model, + uint64_t model_bytes, + const ds4_gpu_tensor *x, + uint32_t n_tok, + ds4_gpu_test_q4_qb_mm_arm arm, + bool materialize_rhs) { + if (!run_mm_arm_projection( + out, rhs_f16, model, model_bytes, x, n_tok, + arm, materialize_rhs)) { + return 0; + } + return ds4_gpu_head_rms_norm_rope_tail_tensor( + out, n_tok, N_HEAD, HEAD_DIM, N_ROT, + 17u, 0u, false, + 10000.0f, 1.0f, 0.0f, 1.0f, 32.0f, 1.0f, 1.0e-6f); +} + static void check_cache_storage_unchanged( const ds4_gpu_q4_attn_q_b_f16_cache_report *before, const ds4_gpu_q4_attn_q_b_f16_cache_report *after, @@ -248,13 +353,40 @@ static int compare_double(const void *lhs, const void *rhs) { return (a > b) - (a < b); } +static double timing_quantile(const double samples[TIMING_SAMPLES], + double q) { + double sorted[TIMING_SAMPLES]; + memcpy(sorted, samples, sizeof(sorted)); + qsort(sorted, TIMING_SAMPLES, sizeof(double), compare_double); + const double position = q * (double)(TIMING_SAMPLES - 1u); + const uint32_t lower = (uint32_t)position; + const uint32_t upper = lower + 1u < TIMING_SAMPLES ? lower + 1u : lower; + const double fraction = position - (double)lower; + return sorted[lower] + (sorted[upper] - sorted[lower]) * fraction; +} + +static double timing_paired_geomean_speedup( + const double baseline[TIMING_SAMPLES], + const double candidate[TIMING_SAMPLES]) { + double log_sum = 0.0; + for (uint32_t i = 0; i < TIMING_SAMPLES; i++) { + log_sum += log(baseline[i] / candidate[i]); + } + /* TIMING_SAMPLES is two complete Williams cycles. The geometric mean + * preserves their multiplicative position balancing. */ + return exp(log_sum / (double)TIMING_SAMPLES); +} + int main(void) { static const uint32_t token_cases[] = {32u, 33u, 64u}; const uint64_t page = (uint64_t)getpagesize(); const uint64_t row_bytes = (uint64_t)BLOCKS_PER_ROW * sizeof(block_q4_K); const uint64_t weight_bytes = (uint64_t)OUT_DIM * row_bytes; - const uint64_t model_bytes = align_up(2u * weight_bytes, page); + const uint64_t ssd_weight_offset = + 2u * weight_bytes + SSD_SOURCE_LEADING; + const uint64_t model_bytes = align_up( + ssd_weight_offset + weight_bytes, page); const uint64_t support_model_bytes = align_up(weight_bytes, page); const uint64_t f16_cache_bytes = (uint64_t)OUT_DIM * IN_DIM * sizeof(uint16_t); @@ -271,6 +403,8 @@ int main(void) { CHECK(row_bytes == 576u, "unexpected Q4_K row size"); CHECK(weight_bytes == 18u * 1024u * 1024u, "unexpected production q_b Q4_K size"); + CHECK(ssd_weight_offset % page == SSD_SOURCE_LEADING, + "SSD source offset must exercise a non-page-aligned exact view"); CHECK(f16_cache_bytes == 64u * 1024u * 1024u, "unexpected production q_b F16 sidecar size"); CHECK(ds4_gpu_test_q4_attn_q_b_f16_working_set_policy( @@ -290,8 +424,16 @@ int main(void) { "working-set overflow-sized request"); CHECK(unsetenv(k_disable) == 0, "clear cache disable env"); + CHECK(unsetenv(k_enable_ssd_streaming) == 0, + "clear SSD-streaming cache opt-in env"); + CHECK(unsetenv(k_disable_f16_rhs) == 0, + "clear compact F16 RHS disable env"); + CHECK(unsetenv(k_disable_transient_f16) == 0, + "clear transient F16 disable env"); CHECK(unsetenv(k_require) == 0, "clear cache require env"); CHECK(unsetenv(k_min_tokens) == 0, "clear cache minimum env"); + CHECK(unsetenv(k_transient_min_tokens) == 0, + "clear transient F16 minimum env"); CHECK(unsetenv(k_cache_mb) == 0, "clear cache budget env"); CHECK(setenv(k_min_tokens, "32", 1) == 0, "set 32-token cache minimum"); @@ -314,6 +456,8 @@ int main(void) { fill_q4_matrix(model); fill_q4_matrix((block_q4_K *)((uint8_t *)model + weight_bytes)); ((block_q4_K *)((uint8_t *)model + weight_bytes))[0].d ^= 0x001fu; + fill_q4_matrix((block_q4_K *)((uint8_t *)model + ssd_weight_offset)); + ((block_q4_K *)((uint8_t *)model + ssd_weight_offset))[0].d ^= 0x005bu; void *support_model = NULL; CHECK(posix_memalign(&support_model, (size_t)page, @@ -658,21 +802,34 @@ int main(void) { const uint64_t candidate_suffix = count_poison_f32_mismatches( candidate_host, GUARD_FLOATS + output_count, output_storage_count, k_candidate_poison); - const uint64_t q_half_mismatches = - count_poison_f16_mismatches( - q_half_host, q_half_storage_count); + const uint64_t q_half_rhs_count = (uint64_t)n_tok * IN_DIM; + const uint64_t q_half_written = + count_poison_f16_mismatches_range( + q_half_host, + GUARD_HALFS, + GUARD_HALFS + q_half_rhs_count); + const uint64_t q_half_prefix = + count_poison_f16_mismatches_range( + q_half_host, 0, GUARD_HALFS); + const uint64_t q_half_tail = + count_poison_f16_mismatches_range( + q_half_host, + GUARD_HALFS + q_half_rhs_count, + q_half_storage_count); fprintf(stderr, "Metal Q4 attn_q_b F16 cache N=%u bitwise=%llu " "ref_guard=%llu/%llu candidate_guard=%llu/%llu " - "q_half=%llu\n", + "q_half_written=%llu guard=%llu/%llu\n", n_tok, (unsigned long long)bit_mismatches, (unsigned long long)reference_prefix, (unsigned long long)reference_suffix, (unsigned long long)candidate_prefix, (unsigned long long)candidate_suffix, - (unsigned long long)q_half_mismatches); + (unsigned long long)q_half_written, + (unsigned long long)q_half_prefix, + (unsigned long long)q_half_tail); if (first != UINT64_MAX) { fprintf(stderr, " first bitwise mismatch token=%llu row=%llu\n", @@ -684,8 +841,10 @@ int main(void) { "reference output canary"); CHECK(candidate_prefix == 0 && candidate_suffix == 0, "candidate output canary"); - CHECK(q_half_mismatches == 0, - "Q4 candidate unexpectedly touched q_half scratch"); + CHECK(q_half_written == q_half_rhs_count, + "Q4 candidate did not materialize the complete F16 RHS"); + CHECK(q_half_prefix == 0 && q_half_tail == 0, + "Q4 candidate F16 RHS scratch canary"); ds4_gpu_q4_attn_q_b_f16_cache_report report; ds4_gpu_test_q4_attn_q_b_f16_cache_report(&report); @@ -704,6 +863,91 @@ int main(void) { "no correctness-case fallback"); } + /* Compact RHS staging is subordinate to the resident sidecar. Invalid + * scratch must use F16 weights with the original F32 RHS, even when the + * sidecar itself is required, rather than rejecting or replaying Q4. */ + { + const uint32_t n_tok = 33u; + const uint64_t output_count = (uint64_t)n_tok * OUT_DIM; + const uint64_t output_bytes = output_count * sizeof(float); + const uint64_t compact_rhs_bytes = + (uint64_t)n_tok * IN_DIM * sizeof(uint16_t); + + poison_f32(reference_host, output_storage_count, + k_reference_poison); + poison_f16(q_half_host, q_half_storage_count); + CHECK(ds4_gpu_tensor_write( + reference_base, 0, reference_host, + output_storage_count * sizeof(float)) != 0, + "nested fallback reference poison upload"); + CHECK(ds4_gpu_tensor_write( + q_half_base, 0, q_half_host, + q_half_storage_count * sizeof(uint16_t)) != 0, + "nested fallback scratch poison upload"); + + ds4_gpu_tensor *reference = ds4_gpu_tensor_view( + reference_base, GUARD_FLOATS * sizeof(float), output_bytes); + ds4_gpu_tensor *candidate = ds4_gpu_tensor_view( + candidate_base, GUARD_FLOATS * sizeof(float), output_bytes); + ds4_gpu_tensor *short_half = ds4_gpu_tensor_view( + q_half_base, GUARD_HALFS * sizeof(uint16_t), + compact_rhs_bytes - sizeof(uint16_t)); + CHECK(reference && candidate && short_half, + "nested fallback tensor views"); + CHECK(run_reference(reference, model, model_bytes, x, n_tok) == 1, + "nested fallback reference"); + CHECK(ds4_gpu_tensor_read( + reference_base, 0, reference_host, + output_storage_count * sizeof(float)) != 0, + "nested fallback reference readback"); + + ds4_gpu_tensor *scratch_cases[] = {x, short_half}; + const char *scratch_names[] = {"alias", "undersized"}; + for (uint32_t scratch_i = 0; scratch_i < 2u; scratch_i++) { + poison_f32(candidate_host, output_storage_count, + k_candidate_poison); + CHECK(ds4_gpu_tensor_write( + candidate_base, 0, candidate_host, + output_storage_count * sizeof(float)) != 0, + "nested fallback candidate poison upload"); + CHECK(run_candidate( + candidate, scratch_cases[scratch_i], + model, model_bytes, x, n_tok) == 1, + "nested F16/F32 fallback candidate"); + CHECK(ds4_gpu_tensor_read( + candidate_base, 0, candidate_host, + output_storage_count * sizeof(float)) != 0, + "nested fallback candidate readback"); + uint64_t first = UINT64_MAX; + CHECK(count_bit_mismatches( + reference_host + GUARD_FLOATS, + candidate_host + GUARD_FLOATS, + output_count, &first) == 0, + "nested F16/F32 fallback bitwise mismatch"); + CHECK(count_poison_f32_mismatches( + candidate_host, 0, GUARD_FLOATS, + k_candidate_poison) == 0 && + count_poison_f32_mismatches( + candidate_host, GUARD_FLOATS + output_count, + output_storage_count, k_candidate_poison) == 0, + "nested fallback output canary"); + fprintf(stderr, + "Metal Q4 attn_q_b compact RHS %s fallback: PASS\n", + scratch_names[scratch_i]); + } + CHECK(ds4_gpu_tensor_read( + q_half_base, 0, q_half_host, + q_half_storage_count * sizeof(uint16_t)) != 0, + "nested fallback scratch readback"); + CHECK(count_poison_f16_mismatches( + q_half_host, q_half_storage_count) == 0, + "undersized compact RHS fallback touched scratch"); + + ds4_gpu_tensor_free(short_half); + ds4_gpu_tensor_free(candidate); + ds4_gpu_tensor_free(reference); + } + /* Compare the raw projection before RMSNorm/RoPE so normalization cannot * hide a uniform scale or dequantization error. */ { @@ -767,6 +1011,103 @@ int main(void) { "raw candidate output canary"); fprintf(stderr, "Metal Q4 attn_q_b F16 cache raw projection N=33: PASS\n"); + + /* Keep the boundary raw oracle in the default (non-timing) target as + * well: the tail must not be able to mask a projection discrepancy. */ + static const ds4_gpu_test_q4_qb_mm_arm raw_arms[] = { + DS4_GPU_TEST_Q4_QB_MM_Q4_F16, + DS4_GPU_TEST_Q4_QB_MM_F16_F32, + DS4_GPU_TEST_Q4_QB_MM_F16_F16, + }; + ds4_gpu_tensor *raw_rhs_f16 = ds4_gpu_tensor_view( + q_half_base, GUARD_HALFS * sizeof(uint16_t), + (uint64_t)n_tok * IN_DIM * sizeof(uint16_t)); + CHECK(raw_rhs_f16 != NULL, "raw four-way F16 RHS view"); + const uint64_t raw_rhs_count = (uint64_t)n_tok * IN_DIM; + for (uint32_t arm_i = 0; + arm_i < sizeof(raw_arms) / sizeof(raw_arms[0]); + arm_i++) { + const bool rhs_is_f16 = + mm_arm_uses_f16_rhs(raw_arms[arm_i]); + const uint32_t rhs_modes = rhs_is_f16 ? 2u : 1u; + for (uint32_t rhs_mode = 0; rhs_mode < rhs_modes; + rhs_mode++) { + const bool materialize_rhs = rhs_mode == 0u; + if (rhs_is_f16) { + poison_f16(q_half_host, q_half_storage_count); + CHECK(ds4_gpu_tensor_write( + q_half_base, 0, q_half_host, + q_half_storage_count * sizeof(uint16_t)) != 0, + "raw four-way RHS poison upload"); + if (!materialize_rhs) { + CHECK(ds4_gpu_tensor_copy_f32_to_f16( + raw_rhs_f16, 0u, x, 0u, + raw_rhs_count) != 0, + "raw four-way explicit RHS prepack"); + } + } + + poison_f32(candidate_host, output_storage_count, + k_candidate_poison); + CHECK(ds4_gpu_tensor_write( + candidate_base, 0, candidate_host, + output_storage_count * sizeof(float)) != 0, + "raw four-way candidate poison upload"); + candidate = ds4_gpu_tensor_view( + candidate_base, GUARD_FLOATS * sizeof(float), + output_bytes); + CHECK(candidate != NULL, "raw four-way candidate view"); + CHECK(run_mm_arm_projection( + candidate, raw_rhs_f16, + model, model_bytes, x, n_tok, + raw_arms[arm_i], materialize_rhs) == 1, + "raw four-way projection"); + ds4_gpu_tensor_free(candidate); + CHECK(ds4_gpu_tensor_read( + candidate_base, 0, candidate_host, + output_storage_count * sizeof(float)) != 0, + "raw four-way candidate readback"); + first = UINT64_MAX; + CHECK(count_bit_mismatches( + reference_host + GUARD_FLOATS, + candidate_host + GUARD_FLOATS, + output_count, &first) == 0, + "raw four-way bitwise mismatch"); + CHECK(count_poison_f32_mismatches( + candidate_host, 0, GUARD_FLOATS, + k_candidate_poison) == 0 && + count_poison_f32_mismatches( + candidate_host, GUARD_FLOATS + output_count, + output_storage_count, k_candidate_poison) == 0, + "raw four-way candidate canary"); + if (rhs_is_f16) { + CHECK(ds4_gpu_tensor_read( + q_half_base, 0, q_half_host, + q_half_storage_count * sizeof(uint16_t)) != 0, + "raw four-way RHS readback"); + CHECK(count_poison_f16_mismatches_range( + q_half_host, GUARD_HALFS, + GUARD_HALFS + raw_rhs_count) == + raw_rhs_count, + "raw four-way RHS payload was not materialized"); + CHECK(count_poison_f16_mismatches_range( + q_half_host, 0, GUARD_HALFS) == 0 && + count_poison_f16_mismatches_range( + q_half_host, + GUARD_HALFS + raw_rhs_count, + q_half_storage_count) == 0, + "raw four-way RHS canary"); + } + fprintf(stderr, + "Metal Q4 attn_q_b raw N=33 %s %s: PASS\n", + mm_arm_name(raw_arms[arm_i]), + rhs_is_f16 + ? (materialize_rhs + ? "with-pack" : "prepacked") + : "control"); + } + } + ds4_gpu_tensor_free(raw_rhs_f16); } /* Exercise the production command-batch lifecycle from a cold cache. @@ -879,6 +1220,15 @@ int main(void) { * leave the entire output allocation untouched. */ ds4_gpu_q4_attn_q_b_f16_cache_report gate_before; ds4_gpu_q4_attn_q_b_f16_cache_report gate_after; + const ds4_gpu_q4_attn_q_b_f16_sidecar_desc ssd_desc = { + .weight_offset = ssd_weight_offset, + .weight_bytes = weight_bytes, + .in_dim = IN_DIM, + .out_dim = OUT_DIM, + .weight_type = Q4_K_TYPE, + .layer = 0, + }; + uint64_t ssd_prepared_bytes = 0; ds4_gpu_test_q4_attn_q_b_f16_cache_report(&gate_before); poison_f32(candidate_host, output_storage_count, k_candidate_poison); CHECK(ds4_gpu_tensor_write( @@ -915,9 +1265,34 @@ int main(void) { "DISABLE accounting"); CHECK(unsetenv(k_disable) == 0, "clear cache disable env"); - /* Entering SSD mode must drop resident-only sidecars, then reject the - * optimization without rebuilding them. */ + /* The SSD-streaming extension is opt-in: its default must continue to + * drop resident-only sidecars and reject the optimization without + * rebuilding them. */ gate_before = gate_after; + CHECK(gate_before.entries != 0u, + "default SSD transition lacks a resident sidecar to release"); + const uint64_t default_ssd_generation = + ds4_gpu_q4_attn_q_b_f16_cache_generation(); + const uint64_t ssd_output_count = (uint64_t)32u * OUT_DIM; + poison_f32(reference_host, output_storage_count, + k_reference_poison); + CHECK(ds4_gpu_tensor_write( + reference_base, 0, reference_host, + output_storage_count * sizeof(float)) != 0, + "SSD exact-view reference poison"); + ds4_gpu_tensor *ssd_reference = ds4_gpu_tensor_view( + reference_base, GUARD_FLOATS * sizeof(float), + ssd_output_count * sizeof(float)); + CHECK(ssd_reference != NULL, "SSD exact-view reference tensor view"); + CHECK(run_reference_at( + ssd_reference, model, model_bytes, ssd_weight_offset, + x, 32u) == 1, + "SSD exact-view native Q4 reference"); + ds4_gpu_tensor_free(ssd_reference); + CHECK(ds4_gpu_tensor_read( + reference_base, 0, reference_host, + output_storage_count * sizeof(float)) != 0, + "SSD exact-view reference readback"); poison_f32(candidate_host, output_storage_count, k_candidate_poison); CHECK(ds4_gpu_tensor_write( candidate_base, 0, candidate_host, @@ -927,12 +1302,27 @@ int main(void) { ds4_gpu_test_q4_attn_q_b_f16_cache_report(&gate_after); CHECK(gate_after.entries == 0u && gate_after.bytes == 0u, "enabling SSD mode retained resident sidecars"); + CHECK(ds4_gpu_q4_attn_q_b_f16_cache_generation() != + default_ssd_generation, + "default SSD transition did not advance cache generation"); CHECK(gate_after.build_circuit_open == 0u, "enabling SSD mode retained the build circuit state"); CHECK(gate_after.candidate_calls == gate_before.candidate_calls && gate_after.fallbacks == gate_before.fallbacks && gate_after.rejects == gate_before.rejects, "SSD transition reset cache accounting"); + + gate_before = gate_after; + CHECK(ds4_gpu_prepare_q4_attn_q_b_f16_sidecars( + model, model_bytes, &ssd_desc, 1u, 32u, 0u, + &ssd_prepared_bytes) == -1, + "REQUIRE alone must not enable SSD-streaming prewarm"); + CHECK(ssd_prepared_bytes == 0u, + "default SSD prewarm reported prepared bytes"); + ds4_gpu_test_q4_attn_q_b_f16_cache_report(&gate_after); + check_cache_storage_unchanged( + &gate_before, &gate_after, + "default SSD prewarm touched cache storage/state"); gate_before = gate_after; gate_out = ds4_gpu_tensor_view( candidate_base, GUARD_FLOATS * sizeof(float), @@ -941,8 +1331,9 @@ int main(void) { q_half_base, GUARD_HALFS * sizeof(uint16_t), (uint64_t)32u * OUT_DIM * sizeof(uint16_t)); CHECK(gate_out && gate_half, "SSD tensor views"); - CHECK(run_candidate( - gate_out, gate_half, model, model_bytes, x, 32u) == -1, + CHECK(run_candidate_at( + gate_out, gate_half, model, model_bytes, + ssd_weight_offset, x, 32u) == -1, "SSD mode must reject required resident cache"); ds4_gpu_tensor_free(gate_half); ds4_gpu_tensor_free(gate_out); @@ -962,8 +1353,475 @@ int main(void) { gate_after.fallbacks == gate_before.fallbacks + 1u && gate_after.rejects == gate_before.rejects + 1u, "SSD rejection accounting"); + + /* Explicit opt-in permits the production prewarm to build the sidecar and + * the prefill dispatch to hit it while experts remain SSD-streamed. + * REQUIRE is deliberately not the opt-in: it only makes failure strict. */ + CHECK(unsetenv(k_require) == 0, + "clear REQUIRE before standalone SSD opt-in"); + CHECK(setenv(k_enable_ssd_streaming, "1", 1) == 0, + "enable Q4 attn_q_b F16 cache with SSD streaming"); + gate_before = gate_after; + ssd_prepared_bytes = 0; + CHECK(ds4_gpu_prepare_q4_attn_q_b_f16_sidecars( + model, model_bytes, &ssd_desc, 1u, 32u, 0u, + &ssd_prepared_bytes) == 1, + "SSD opt-in prewarm build"); + ds4_gpu_test_q4_attn_q_b_f16_cache_report(&gate_after); + CHECK(ssd_prepared_bytes == f16_cache_bytes && + gate_after.entries == 1u && + gate_after.bytes == f16_cache_bytes && + gate_after.builds == gate_before.builds + 1u && + gate_after.misses == gate_before.misses && + gate_after.hits == gate_before.hits && + gate_after.lookups == gate_before.lookups && + gate_after.candidate_calls == gate_before.candidate_calls && + gate_after.fallbacks == gate_before.fallbacks && + gate_after.rejects == gate_before.rejects && + gate_after.build_circuit_open == 0u, + "SSD opt-in prewarm build accounting"); + + gate_before = gate_after; + poison_f32(candidate_host, output_storage_count, k_candidate_poison); + CHECK(ds4_gpu_tensor_write( + candidate_base, 0, candidate_host, + output_storage_count * sizeof(float)) != 0, + "SSD opt-in first-hit output poison"); + gate_out = ds4_gpu_tensor_view( + candidate_base, GUARD_FLOATS * sizeof(float), + (uint64_t)32u * OUT_DIM * sizeof(float)); + gate_half = ds4_gpu_tensor_view( + q_half_base, GUARD_HALFS * sizeof(uint16_t), + (uint64_t)32u * OUT_DIM * sizeof(uint16_t)); + CHECK(gate_out && gate_half, "SSD opt-in first-hit tensor views"); + CHECK(run_candidate_at( + gate_out, gate_half, model, model_bytes, + ssd_weight_offset, x, 32u) == 1, + "SSD opt-in first prefill cache hit"); + ds4_gpu_tensor_free(gate_half); + ds4_gpu_tensor_free(gate_out); + ds4_gpu_test_q4_attn_q_b_f16_cache_report(&gate_after); + CHECK(gate_after.entries == gate_before.entries && + gate_after.bytes == gate_before.bytes && + gate_after.builds == gate_before.builds && + gate_after.misses == gate_before.misses && + gate_after.hits == gate_before.hits + 1u && + gate_after.lookups == gate_before.lookups + 1u && + gate_after.candidate_calls == + gate_before.candidate_calls + 1u && + gate_after.fallbacks == gate_before.fallbacks && + gate_after.rejects == gate_before.rejects && + gate_after.build_circuit_open == 0u, + "SSD opt-in first-hit accounting"); + CHECK(ds4_gpu_tensor_read( + candidate_base, 0, candidate_host, + output_storage_count * sizeof(float)) != 0, + "SSD opt-in first-hit output readback"); + CHECK(count_poison_f32_mismatches( + candidate_host, GUARD_FLOATS, + GUARD_FLOATS + (uint64_t)32u * OUT_DIM, + k_candidate_poison) != 0u, + "SSD opt-in first hit did not write output"); + uint64_t ssd_first_mismatch = UINT64_MAX; + CHECK(count_bit_mismatches( + reference_host + GUARD_FLOATS, + candidate_host + GUARD_FLOATS, + ssd_output_count, + &ssd_first_mismatch) == 0u, + "SSD non-page-aligned exact-view candidate bitwise mismatch"); + + gate_before = gate_after; + gate_out = ds4_gpu_tensor_view( + candidate_base, GUARD_FLOATS * sizeof(float), + (uint64_t)32u * OUT_DIM * sizeof(float)); + gate_half = ds4_gpu_tensor_view( + q_half_base, GUARD_HALFS * sizeof(uint16_t), + (uint64_t)32u * OUT_DIM * sizeof(uint16_t)); + CHECK(gate_out && gate_half, "SSD opt-in hot tensor views"); + CHECK(run_candidate_at( + gate_out, gate_half, model, model_bytes, + ssd_weight_offset, x, 32u) == 1, + "SSD opt-in cache hit"); + ds4_gpu_tensor_free(gate_half); + ds4_gpu_tensor_free(gate_out); + ds4_gpu_test_q4_attn_q_b_f16_cache_report(&gate_after); + CHECK(gate_after.entries == gate_before.entries && + gate_after.bytes == gate_before.bytes && + gate_after.builds == gate_before.builds && + gate_after.misses == gate_before.misses && + gate_after.hits == gate_before.hits + 1u && + gate_after.lookups == gate_before.lookups + 1u && + gate_after.candidate_calls == + gate_before.candidate_calls + 1u && + gate_after.fallbacks == gate_before.fallbacks && + gate_after.rejects == gate_before.rejects, + "SSD opt-in hot-hit accounting"); + + /* DISABLE remains the highest-priority policy even when SSD opt-in and + * REQUIRE are both active, and it must not evict a ready sidecar. */ + CHECK(setenv(k_require, "1", 1) == 0, + "restore REQUIRE for SSD DISABLE priority test"); + CHECK(setenv(k_disable, "1", 1) == 0, + "disable SSD opt-in Q4 attn_q_b F16 cache"); + gate_before = gate_after; + ssd_prepared_bytes = 0; + CHECK(ds4_gpu_prepare_q4_attn_q_b_f16_sidecars( + model, model_bytes, &ssd_desc, 1u, 32u, 0u, + &ssd_prepared_bytes) == -1, + "DISABLE must win over SSD opt-in prewarm and REQUIRE"); + CHECK(ssd_prepared_bytes == 0u, + "disabled SSD opt-in prewarm reported prepared bytes"); + ds4_gpu_test_q4_attn_q_b_f16_cache_report(&gate_after); + check_cache_storage_unchanged( + &gate_before, &gate_after, + "SSD opt-in prewarm DISABLE touched cache storage/state"); + gate_before = gate_after; + gate_out = ds4_gpu_tensor_view( + candidate_base, GUARD_FLOATS * sizeof(float), + (uint64_t)32u * OUT_DIM * sizeof(float)); + gate_half = ds4_gpu_tensor_view( + q_half_base, GUARD_HALFS * sizeof(uint16_t), + (uint64_t)32u * OUT_DIM * sizeof(uint16_t)); + CHECK(gate_out && gate_half, "SSD opt-in DISABLE tensor views"); + CHECK(run_candidate_at( + gate_out, gate_half, model, model_bytes, + ssd_weight_offset, x, 32u) == -1, + "DISABLE must win over SSD opt-in and REQUIRE"); + ds4_gpu_tensor_free(gate_half); + ds4_gpu_tensor_free(gate_out); + ds4_gpu_test_q4_attn_q_b_f16_cache_report(&gate_after); + check_cache_storage_unchanged( + &gate_before, &gate_after, + "SSD opt-in DISABLE touched cache storage/state"); + CHECK(gate_after.candidate_calls == gate_before.candidate_calls + 1u && + gate_after.fallbacks == gate_before.fallbacks + 1u && + gate_after.rejects == gate_before.rejects + 1u, + "SSD opt-in DISABLE accounting"); + CHECK(unsetenv(k_disable) == 0, + "clear SSD opt-in cache disable env"); + + /* Reasserting SSD mode with opt-in preserves the sidecar. Explicit + * lifecycle release remains authoritative and advances the generation. */ + const uint64_t opt_in_ssd_generation = + ds4_gpu_q4_attn_q_b_f16_cache_generation(); + gate_before = gate_after; + ds4_gpu_set_ssd_streaming(true); + ds4_gpu_test_q4_attn_q_b_f16_cache_report(&gate_after); + check_cache_storage_unchanged( + &gate_before, &gate_after, + "SSD opt-in transition evicted ready sidecar"); + CHECK(ds4_gpu_q4_attn_q_b_f16_cache_generation() == + opt_in_ssd_generation, + "SSD opt-in transition advanced cache generation"); + + CHECK(ds4_gpu_release_q4_attn_q_b_f16_sidecars() != 0, + "SSD opt-in lifecycle release"); + ds4_gpu_test_q4_attn_q_b_f16_cache_report(&gate_after); + CHECK(gate_after.entries == 0u && gate_after.bytes == 0u && + gate_after.build_circuit_open == 0u, + "SSD opt-in lifecycle release retained cache state"); + CHECK(ds4_gpu_q4_attn_q_b_f16_cache_generation() != + opt_in_ssd_generation, + "SSD opt-in lifecycle release did not advance generation"); + fprintf(stderr, + "Metal Q4 attn_q_b F16 cache SSD opt-in policy/lifecycle: " + "PASS\n"); + + CHECK(unsetenv(k_enable_ssd_streaming) == 0, + "clear SSD-streaming cache opt-in env"); ds4_gpu_set_ssd_streaming(false); + /* Exercise the public production selector, not only its benchmark hook. + * Lower the threshold for this bounded oracle so aligned and boundary + * geometries fit in the guarded allocations. Default mode must use one + * transient scratch without publishing persistent sidecars, while SSD + * streaming must return the clean native-Q4 fallback sentinel before + * touching output. */ + ds4_gpu_test_q4_attn_q_b_f16_cache_reset(); + CHECK(unsetenv(k_require) == 0, + "clear REQUIRE for transient production oracle"); + const ds4_gpu_q4_attn_q_b_f16_sidecar_desc transient_desc = { + .weight_offset = 0u, + .weight_bytes = weight_bytes, + .in_dim = IN_DIM, + .out_dim = OUT_DIM, + .weight_type = Q4_K_TYPE, + .layer = 0u, + }; + uint64_t transient_prepared_bytes = 0u; + CHECK(ds4_gpu_prepare_q4_attn_q_b_f16_sidecars( + model, model_bytes, &transient_desc, 1u, 4096u, 0u, + &transient_prepared_bytes) == 1, + "transient production preflight"); + CHECK(transient_prepared_bytes == f16_cache_bytes, + "transient production preflight did not allocate one scratch"); + ds4_gpu_q4_attn_q_b_f16_cache_report transient_preflight_report; + ds4_gpu_test_q4_attn_q_b_f16_cache_report( + &transient_preflight_report); + CHECK(transient_preflight_report.entries == 0u && + transient_preflight_report.bytes == 0u && + transient_preflight_report.lookups == 0u && + transient_preflight_report.builds == 0u, + "transient production preflight published a sidecar"); + CHECK(setenv(k_transient_min_tokens, "32", 1) == 0, + "lower transient production threshold for oracle"); + for (uint32_t case_i = 0; + case_i < sizeof(token_cases) / sizeof(token_cases[0]); + case_i++) { + const uint32_t n_tok = token_cases[case_i]; + const uint64_t output_count = (uint64_t)n_tok * OUT_DIM; + const uint64_t output_bytes = output_count * sizeof(float); + const uint64_t q_half_count = (uint64_t)n_tok * IN_DIM; + const uint64_t q_half_bytes = q_half_count * sizeof(uint16_t); + + poison_f32(reference_host, output_storage_count, + k_reference_poison); + poison_f32(candidate_host, output_storage_count, + k_candidate_poison); + poison_f16(q_half_host, q_half_storage_count); + CHECK(ds4_gpu_tensor_write( + reference_base, 0, reference_host, + output_storage_count * sizeof(float)) != 0, + "transient production reference poison upload"); + CHECK(ds4_gpu_tensor_write( + candidate_base, 0, candidate_host, + output_storage_count * sizeof(float)) != 0, + "transient production candidate poison upload"); + CHECK(ds4_gpu_tensor_write( + q_half_base, 0, q_half_host, + q_half_storage_count * sizeof(uint16_t)) != 0, + "transient production q_half poison upload"); + + ds4_gpu_tensor *transient_reference = ds4_gpu_tensor_view( + reference_base, GUARD_FLOATS * sizeof(float), output_bytes); + ds4_gpu_tensor *transient_candidate = ds4_gpu_tensor_view( + candidate_base, GUARD_FLOATS * sizeof(float), output_bytes); + ds4_gpu_tensor *transient_half = ds4_gpu_tensor_view( + q_half_base, GUARD_HALFS * sizeof(uint16_t), q_half_bytes); + CHECK(transient_reference && transient_candidate && transient_half, + "transient production tensor views"); + CHECK(run_reference( + transient_reference, model, model_bytes, x, n_tok) == 1, + "transient production native Q4 reference"); + CHECK(run_candidate( + transient_candidate, transient_half, + model, model_bytes, x, n_tok) == 1, + "transient production public entry point"); + ds4_gpu_tensor_free(transient_half); + ds4_gpu_tensor_free(transient_candidate); + ds4_gpu_tensor_free(transient_reference); + + CHECK(ds4_gpu_tensor_read( + reference_base, 0, reference_host, + output_storage_count * sizeof(float)) != 0, + "transient production reference readback"); + CHECK(ds4_gpu_tensor_read( + candidate_base, 0, candidate_host, + output_storage_count * sizeof(float)) != 0, + "transient production candidate readback"); + CHECK(ds4_gpu_tensor_read( + q_half_base, 0, q_half_host, + q_half_storage_count * sizeof(uint16_t)) != 0, + "transient production q_half readback"); + uint64_t transient_first = UINT64_MAX; + CHECK(count_bit_mismatches( + reference_host + GUARD_FLOATS, + candidate_host + GUARD_FLOATS, + output_count, &transient_first) == 0u, + "transient production bitwise mismatch"); + CHECK(count_poison_f32_mismatches( + candidate_host, 0u, GUARD_FLOATS, + k_candidate_poison) == 0u && + count_poison_f32_mismatches( + candidate_host, GUARD_FLOATS + output_count, + output_storage_count, k_candidate_poison) == 0u, + "transient production touched output guards"); + CHECK(count_poison_f16_mismatches_range( + q_half_host, 0u, GUARD_HALFS) == 0u && + count_poison_f16_mismatches_range( + q_half_host, GUARD_HALFS, + GUARD_HALFS + q_half_count) == q_half_count && + count_poison_f16_mismatches_range( + q_half_host, GUARD_HALFS + q_half_count, + q_half_storage_count) == 0u, + "transient production q_half payload/canary mismatch"); + } + + /* Two different layer weights in one real command batch exercise the + * production encoder boundaries and prove that stream-local scratch is + * not overwritten before the preceding F16/F16 consumer and tail. */ + { + const uint32_t n_tok = 33u; + const uint64_t output_count = (uint64_t)n_tok * OUT_DIM; + const uint64_t output_bytes = output_count * sizeof(float); + const uint64_t q_half_count = (uint64_t)n_tok * IN_DIM; + const uint64_t q_half_bytes = + q_half_count * sizeof(uint16_t); + poison_f32(reference_host, output_storage_count, + k_reference_poison); + poison_f32(candidate_host, output_storage_count, + k_candidate_poison); + poison_f16(q_half_host, q_half_storage_count); + CHECK(ds4_gpu_tensor_write( + reference_base, 0, reference_host, + output_storage_count * sizeof(float)) != 0, + "transient batch second-output poison upload"); + CHECK(ds4_gpu_tensor_write( + candidate_base, 0, candidate_host, + output_storage_count * sizeof(float)) != 0, + "transient batch first-output poison upload"); + CHECK(ds4_gpu_tensor_write( + q_half_base, 0, q_half_host, + q_half_storage_count * sizeof(uint16_t)) != 0, + "transient batch q_half poison upload"); + + ds4_gpu_tensor *batch_first = ds4_gpu_tensor_view( + candidate_base, GUARD_FLOATS * sizeof(float), output_bytes); + ds4_gpu_tensor *batch_second = ds4_gpu_tensor_view( + reference_base, GUARD_FLOATS * sizeof(float), output_bytes); + ds4_gpu_tensor *batch_half = ds4_gpu_tensor_view( + q_half_base, GUARD_HALFS * sizeof(uint16_t), q_half_bytes); + CHECK(batch_first && batch_second && batch_half, + "transient production batch tensor views"); + CHECK(ds4_gpu_begin_commands() != 0, + "begin transient production command batch"); + CHECK(run_candidate_at( + batch_first, batch_half, model, model_bytes, 0u, + x, n_tok) == 1, + "encode first transient production batch layer"); + CHECK(run_candidate_at( + batch_second, batch_half, model, model_bytes, + weight_bytes, x, n_tok) == 1, + "encode second transient production batch layer"); + CHECK(ds4_gpu_end_commands() != 0, + "finish transient production command batch"); + ds4_gpu_tensor_free(batch_half); + ds4_gpu_tensor_free(batch_second); + ds4_gpu_tensor_free(batch_first); + + CHECK(ds4_gpu_tensor_read( + candidate_base, 0, candidate_host, + output_storage_count * sizeof(float)) != 0, + "transient batch first candidate readback"); + ds4_gpu_tensor *batch_reference = ds4_gpu_tensor_view( + candidate_base, GUARD_FLOATS * sizeof(float), output_bytes); + CHECK(batch_reference != NULL, + "transient batch first reference view"); + CHECK(run_reference_at( + batch_reference, model, model_bytes, 0u, + x, n_tok) == 1, + "transient batch first native reference"); + ds4_gpu_tensor_free(batch_reference); + CHECK(ds4_gpu_tensor_read( + candidate_base, 0, reference_host, + output_storage_count * sizeof(float)) != 0, + "transient batch first reference readback"); + uint64_t batch_first_mismatch = UINT64_MAX; + CHECK(count_bit_mismatches( + reference_host + GUARD_FLOATS, + candidate_host + GUARD_FLOATS, + output_count, &batch_first_mismatch) == 0u, + "transient batch first layer mismatch"); + + CHECK(ds4_gpu_tensor_read( + reference_base, 0, candidate_host, + output_storage_count * sizeof(float)) != 0, + "transient batch second candidate readback"); + batch_reference = ds4_gpu_tensor_view( + candidate_base, GUARD_FLOATS * sizeof(float), output_bytes); + CHECK(batch_reference != NULL, + "transient batch second reference view"); + CHECK(run_reference_at( + batch_reference, model, model_bytes, weight_bytes, + x, n_tok) == 1, + "transient batch second native reference"); + ds4_gpu_tensor_free(batch_reference); + CHECK(ds4_gpu_tensor_read( + candidate_base, 0, reference_host, + output_storage_count * sizeof(float)) != 0, + "transient batch second reference readback"); + uint64_t batch_second_mismatch = UINT64_MAX; + CHECK(count_bit_mismatches( + reference_host + GUARD_FLOATS, + candidate_host + GUARD_FLOATS, + output_count, &batch_second_mismatch) == 0u, + "transient batch second layer mismatch"); + + CHECK(ds4_gpu_tensor_read( + q_half_base, 0, q_half_host, + q_half_storage_count * sizeof(uint16_t)) != 0, + "transient batch q_half readback"); + CHECK(count_poison_f16_mismatches_range( + q_half_host, 0u, GUARD_HALFS) == 0u && + count_poison_f16_mismatches_range( + q_half_host, GUARD_HALFS, + GUARD_HALFS + q_half_count) == q_half_count && + count_poison_f16_mismatches_range( + q_half_host, GUARD_HALFS + q_half_count, + q_half_storage_count) == 0u, + "transient batch q_half payload/canary mismatch"); + fprintf(stderr, + "Metal Q4 attn_q_b transient F16 command-batch " + "two-layer scratch reuse N=33: PASS\n"); + } + + ds4_gpu_q4_attn_q_b_f16_cache_report transient_report; + ds4_gpu_test_q4_attn_q_b_f16_cache_report(&transient_report); + CHECK(transient_report.entries == 0u && + transient_report.bytes == 0u && + transient_report.lookups == 0u && + transient_report.builds == 0u, + "transient production published a persistent sidecar"); + + poison_f32(candidate_host, output_storage_count, k_candidate_poison); + poison_f16(q_half_host, q_half_storage_count); + CHECK(ds4_gpu_tensor_write( + candidate_base, 0, candidate_host, + output_storage_count * sizeof(float)) != 0, + "transient SSD fallback poison upload"); + CHECK(ds4_gpu_tensor_write( + q_half_base, 0, q_half_host, + q_half_storage_count * sizeof(uint16_t)) != 0, + "transient SSD fallback q_half poison upload"); + ds4_gpu_tensor *transient_ssd_out = ds4_gpu_tensor_view( + candidate_base, GUARD_FLOATS * sizeof(float), + (uint64_t)64u * OUT_DIM * sizeof(float)); + ds4_gpu_tensor *transient_ssd_half = ds4_gpu_tensor_view( + q_half_base, GUARD_HALFS * sizeof(uint16_t), + (uint64_t)64u * IN_DIM * sizeof(uint16_t)); + CHECK(transient_ssd_out && transient_ssd_half, + "transient SSD fallback tensor views"); + ds4_gpu_set_ssd_streaming(true); + CHECK(run_candidate( + transient_ssd_out, transient_ssd_half, + model, model_bytes, x, 64u) == 0, + "transient path must fall back under SSD streaming"); + ds4_gpu_tensor_free(transient_ssd_half); + ds4_gpu_tensor_free(transient_ssd_out); + CHECK(ds4_gpu_tensor_read( + candidate_base, 0, candidate_host, + output_storage_count * sizeof(float)) != 0, + "transient SSD fallback output readback"); + CHECK(count_poison_f32_mismatches( + candidate_host, 0u, output_storage_count, + k_candidate_poison) == 0u, + "transient SSD fallback touched output"); + CHECK(ds4_gpu_tensor_read( + q_half_base, 0, q_half_host, + q_half_storage_count * sizeof(uint16_t)) != 0, + "transient SSD fallback q_half readback"); + CHECK(count_poison_f16_mismatches( + q_half_host, q_half_storage_count) == 0u, + "transient SSD fallback touched q_half"); + ds4_gpu_set_ssd_streaming(false); + CHECK(setenv(k_require, "1", 1) == 0, + "restore REQUIRE after transient production oracle"); + CHECK(unsetenv(k_transient_min_tokens) == 0, + "restore transient production threshold"); + fprintf(stderr, + "Metal Q4 attn_q_b transient F16 production selector " + "N=32/33/64 and SSD fallback: PASS\n"); + /* The input, including both guards, is immutable across every path. */ CHECK(ds4_gpu_tensor_read( x_base, 0, input_readback, @@ -996,8 +1854,41 @@ int main(void) { (uint64_t)timing_tokens * IN_DIM; const uint64_t timing_output_count = (uint64_t)timing_tokens * OUT_DIM; - double reference_ms[TIMING_SAMPLES]; - double candidate_ms[TIMING_SAMPLES]; + static const ds4_gpu_test_q4_qb_mm_arm base_arms[] = { + DS4_GPU_TEST_Q4_QB_MM_Q4_F32, + DS4_GPU_TEST_Q4_QB_MM_Q4_F16, + DS4_GPU_TEST_Q4_QB_MM_F16_F32, + DS4_GPU_TEST_Q4_QB_MM_F16_F16, + }; + static const ds4_gpu_test_q4_qb_mm_arm transient_panel_arms[] = { + DS4_GPU_TEST_Q4_QB_MM_Q4_F16, + DS4_GPU_TEST_Q4_QB_MM_Q4_TRANSIENT_F16_F16, + DS4_GPU_TEST_Q4_QB_MM_F16_F16, + }; + static const ds4_gpu_test_q4_qb_mm_arm oracle_arms[] = { + DS4_GPU_TEST_Q4_QB_MM_Q4_F32, + DS4_GPU_TEST_Q4_QB_MM_Q4_F16, + DS4_GPU_TEST_Q4_QB_MM_F16_F32, + DS4_GPU_TEST_Q4_QB_MM_F16_F16, + DS4_GPU_TEST_Q4_QB_MM_Q4_TRANSIENT_F16_F16, + }; + static const uint8_t williams_order[4][4] = { + {0u, 1u, 3u, 2u}, + {1u, 2u, 0u, 3u}, + {2u, 3u, 1u, 0u}, + {3u, 0u, 2u, 1u}, + }; + static const uint8_t transient_order[2][3] = { + {0u, 1u, 2u}, + {2u, 1u, 0u}, + }; + double with_pack_ms[DS4_GPU_TEST_Q4_QB_MM_ARM_COUNT] + [TIMING_SAMPLES]; + double prepacked_ms[DS4_GPU_TEST_Q4_QB_MM_ARM_COUNT] + [TIMING_SAMPLES]; + double transient_panel_ms[3][TIMING_SAMPLES]; + double production_reference_ms[TIMING_SAMPLES]; + double production_transient_ms[TIMING_SAMPLES]; float *timing_input = malloc( (size_t)timing_input_count * sizeof(float)); CHECK(timing_input != NULL, "timing input host allocation"); @@ -1006,7 +1897,7 @@ int main(void) { const uint32_t key = token * 131u + col * 17u + ((col >> 3u) ^ (token * 29u)); timing_input[(uint64_t)token * IN_DIM + col] = - (float)((int)(key % 129u) - 64) / 64.0f; + (float)((int)(key % 129u) - 64) / 509.0f; } } @@ -1017,9 +1908,9 @@ int main(void) { timing_input_count * sizeof(float)); ds4_gpu_tensor *timing_out = ds4_gpu_tensor_alloc( timing_output_count * sizeof(float)); - ds4_gpu_tensor *timing_q_half = ds4_gpu_tensor_alloc( - timing_output_count * sizeof(uint16_t)); - CHECK(timing_x && timing_out && timing_q_half, + ds4_gpu_tensor *timing_rhs_f16 = ds4_gpu_tensor_alloc( + timing_input_count * sizeof(uint16_t)); + CHECK(timing_x && timing_out && timing_rhs_f16, "timing Metal tensor allocation"); CHECK(ds4_gpu_tensor_write( timing_x, 0, timing_input, @@ -1030,56 +1921,194 @@ int main(void) { ds4_gpu_test_q4_attn_q_b_f16_cache_reset(); CHECK(ds4_gpu_synchronize() != 0, "pre-cold synchronize"); const double cold_t0 = monotonic_ms(); - CHECK(run_candidate(timing_out, timing_q_half, model, model_bytes, + CHECK(run_candidate(timing_out, timing_rhs_f16, model, model_bytes, timing_x, timing_tokens) == 1, "timing cold candidate"); - CHECK(ds4_gpu_synchronize() != 0, "cold synchronize"); const double cold_ms = monotonic_ms() - cold_t0; + CHECK(ds4_gpu_synchronize() != 0, "post-cold synchronize"); + + CHECK(ds4_gpu_tensor_copy_f32_to_f16( + timing_rhs_f16, 0u, timing_x, 0u, + timing_input_count) != 0, + "timing prepacked F16 RHS"); + for (uint32_t arm_i = 0; + arm_i < sizeof(base_arms) / sizeof(base_arms[0]); + arm_i++) { + CHECK(run_mm_arm_projection( + timing_out, timing_rhs_f16, model, model_bytes, + timing_x, timing_tokens, base_arms[arm_i], true) == 1, + "four-way with-pack warmup"); + CHECK(run_mm_arm_projection( + timing_out, timing_rhs_f16, model, model_bytes, + timing_x, timing_tokens, base_arms[arm_i], false) == 1, + "four-way prepacked warmup"); + } - CHECK(run_reference(timing_out, model, model_bytes, timing_x, - timing_tokens) == 1, - "timing reference warmup"); - CHECK(run_candidate(timing_out, timing_q_half, model, model_bytes, - timing_x, timing_tokens) == 1, - "timing candidate warmup"); - - for (uint32_t i = 0; i < TIMING_SAMPLES; i++) { - if ((i & 1u) == 0u) { - double t0 = monotonic_ms(); - CHECK(run_reference(timing_out, model, model_bytes, - timing_x, timing_tokens) == 1, - "timing reference"); - reference_ms[i] = monotonic_ms() - t0; - t0 = monotonic_ms(); - CHECK(run_candidate(timing_out, timing_q_half, model, - model_bytes, timing_x, - timing_tokens) == 1, - "timing candidate"); - candidate_ms[i] = monotonic_ms() - t0; - } else { - double t0 = monotonic_ms(); - CHECK(run_candidate(timing_out, timing_q_half, model, - model_bytes, timing_x, - timing_tokens) == 1, - "timing candidate"); - candidate_ms[i] = monotonic_ms() - t0; - t0 = monotonic_ms(); - CHECK(run_reference(timing_out, model, model_bytes, - timing_x, timing_tokens) == 1, - "timing reference"); - reference_ms[i] = monotonic_ms() - t0; + for (uint32_t sample = 0; sample < TIMING_SAMPLES; sample++) { + const uint8_t *order = williams_order[sample & 3u]; + for (uint32_t pos = 0; pos < 4u; pos++) { + const uint32_t arm_i = order[pos]; + const ds4_gpu_test_q4_qb_mm_arm arm = base_arms[arm_i]; + if ((sample & 1u) == 0u) { + double t0 = monotonic_ms(); + CHECK(run_mm_arm_projection( + timing_out, timing_rhs_f16, + model, model_bytes, timing_x, timing_tokens, + arm, true) == 1, + "four-way with-pack timing"); + with_pack_ms[arm_i][sample] = monotonic_ms() - t0; + t0 = monotonic_ms(); + CHECK(run_mm_arm_projection( + timing_out, timing_rhs_f16, + model, model_bytes, timing_x, timing_tokens, + arm, false) == 1, + "four-way prepacked timing"); + prepacked_ms[arm_i][sample] = monotonic_ms() - t0; + } else { + double t0 = monotonic_ms(); + CHECK(run_mm_arm_projection( + timing_out, timing_rhs_f16, + model, model_bytes, timing_x, timing_tokens, + arm, false) == 1, + "four-way prepacked timing"); + prepacked_ms[arm_i][sample] = monotonic_ms() - t0; + t0 = monotonic_ms(); + CHECK(run_mm_arm_projection( + timing_out, timing_rhs_f16, + model, model_bytes, timing_x, timing_tokens, + arm, true) == 1, + "four-way with-pack timing"); + with_pack_ms[arm_i][sample] = monotonic_ms() - t0; + } } } - qsort(reference_ms, TIMING_SAMPLES, sizeof(double), compare_double); - qsort(candidate_ms, TIMING_SAMPLES, sizeof(double), compare_double); - const double reference_median = reference_ms[TIMING_SAMPLES / 2u]; - const double candidate_median = candidate_ms[TIMING_SAMPLES / 2u]; fprintf(stderr, - "Metal Q4 attn_q_b F16 cache timing N=%u: " - "cold=%.3f ms reference=%.3f ms steady=%.3f ms " - "speedup=%.3fx\n", - timing_tokens, cold_ms, reference_median, candidate_median, - reference_median / candidate_median); + "Metal Q4 attn_q_b four-way resident timing N=%u " + "cold-sidecar=%.3f ms\n", + timing_tokens, cold_ms); + for (uint32_t arm_i = 0; + arm_i < sizeof(base_arms) / sizeof(base_arms[0]); + arm_i++) { + const bool rhs_is_f16 = + mm_arm_uses_f16_rhs(base_arms[arm_i]); + const double with_pack = + timing_quantile(with_pack_ms[arm_i], 0.5); + const double with_pack_p25 = + timing_quantile(with_pack_ms[arm_i], 0.25); + const double with_pack_p75 = + timing_quantile(with_pack_ms[arm_i], 0.75); + const double prepacked = + timing_quantile(prepacked_ms[arm_i], 0.5); + const double prepacked_p25 = + timing_quantile(prepacked_ms[arm_i], 0.25); + const double prepacked_p75 = + timing_quantile(prepacked_ms[arm_i], 0.75); + const double with_pack_speedup = + timing_paired_geomean_speedup( + with_pack_ms[DS4_GPU_TEST_Q4_QB_MM_Q4_F32], + with_pack_ms[arm_i]); + const double prepacked_speedup = + timing_paired_geomean_speedup( + prepacked_ms[DS4_GPU_TEST_Q4_QB_MM_Q4_F32], + prepacked_ms[arm_i]); + fprintf(stderr, + " %-8s %s=%.3f ms [%.3f, %.3f] %.3fx paired-gmean " + "%s=%.3f ms [%.3f, %.3f] %.3fx paired-gmean\n", + mm_arm_name(base_arms[arm_i]), + rhs_is_f16 ? "with-pack" : "control-a", + with_pack, with_pack_p25, with_pack_p75, + with_pack_speedup, + rhs_is_f16 ? "prepacked" : "control-b", + prepacked, prepacked_p25, prepacked_p75, + prepacked_speedup); + } + + /* This arm deliberately rebuilds a transient F16 weight matrix for + * every projection, then dispatches the same F16/F16 multiply as the + * resident sidecar control. Keep it at the production-prefill + * N=4096 geometry and time only with the already-packed RHS. The two + * controls swap first/last position every sample while the transient + * arm remains between them, yielding eight directly paired samples. */ + if (timing_tokens != 4096u) { + fprintf(stderr, + "Metal Q4 attn_q_b transient prepacked timing N=%u: " + "SKIP (dedicated geometry is N=4096)\n", + timing_tokens); + } else if (!ds4_gpu_test_q4_attn_q_b_mm_arm_supported( + DS4_GPU_TEST_Q4_QB_MM_Q4_TRANSIENT_F16_F16)) { + fprintf(stderr, + "Metal Q4 attn_q_b transient prepacked timing N=4096: " + "SKIP (pipeline unsupported)\n"); + } else { + for (uint32_t arm_i = 0; + arm_i < sizeof(transient_panel_arms) / + sizeof(transient_panel_arms[0]); + arm_i++) { + CHECK(run_mm_arm_projection( + timing_out, timing_rhs_f16, model, model_bytes, + timing_x, timing_tokens, + transient_panel_arms[arm_i], false) == 1, + "transient panel prepacked warmup"); + } + + for (uint32_t sample = 0; sample < TIMING_SAMPLES; sample++) { + const uint8_t *order = transient_order[sample & 1u]; + for (uint32_t pos = 0; pos < 3u; pos++) { + const uint32_t arm_i = order[pos]; + const double t0 = monotonic_ms(); + CHECK(run_mm_arm_projection( + timing_out, timing_rhs_f16, + model, model_bytes, timing_x, timing_tokens, + transient_panel_arms[arm_i], false) == 1, + "transient panel prepacked timing"); + transient_panel_ms[arm_i][sample] = + monotonic_ms() - t0; + } + } + + const double legacy_median = + timing_quantile(transient_panel_ms[0], 0.5); + const double legacy_p25 = + timing_quantile(transient_panel_ms[0], 0.25); + const double legacy_p75 = + timing_quantile(transient_panel_ms[0], 0.75); + const double transient_median = + timing_quantile(transient_panel_ms[1], 0.5); + const double transient_p25 = + timing_quantile(transient_panel_ms[1], 0.25); + const double transient_p75 = + timing_quantile(transient_panel_ms[1], 0.75); + const double sidecar_median = + timing_quantile(transient_panel_ms[2], 0.5); + const double sidecar_p25 = + timing_quantile(transient_panel_ms[2], 0.25); + const double sidecar_p75 = + timing_quantile(transient_panel_ms[2], 0.75); + const double sidecar_vs_legacy = + timing_paired_geomean_speedup( + transient_panel_ms[0], transient_panel_ms[2]); + const double transient_vs_legacy = + timing_paired_geomean_speedup( + transient_panel_ms[0], transient_panel_ms[1]); + const double transient_vs_sidecar = + timing_paired_geomean_speedup( + transient_panel_ms[2], transient_panel_ms[1]); + fprintf(stderr, + "Metal Q4 attn_q_b transient prepacked timing N=4096\n" + " %-22s %.3f ms [%.3f, %.3f] control\n" + " %-22s %.3f ms [%.3f, %.3f] " + "%.3fx vs legacy paired-gmean\n" + " %-22s %.3f ms [%.3f, %.3f] " + "%.3fx vs legacy, %.3fx vs sidecar paired-gmean\n", + mm_arm_name(transient_panel_arms[0]), + legacy_median, legacy_p25, legacy_p75, + mm_arm_name(transient_panel_arms[2]), + sidecar_median, sidecar_p25, sidecar_p75, + sidecar_vs_legacy, + mm_arm_name(transient_panel_arms[1]), + transient_median, transient_p25, transient_p75, + transient_vs_legacy, transient_vs_sidecar); + } /* Verify the exact timing geometry after sampling so readback and the * second output allocation cannot perturb the measured resident path. @@ -1088,12 +2117,11 @@ int main(void) { timing_output_count * sizeof(float)); CHECK(timing_reference != NULL, "timing verification output allocation"); - CHECK(run_reference(timing_reference, model, model_bytes, timing_x, - timing_tokens) == 1, - "timing verification reference"); - CHECK(run_candidate(timing_out, timing_q_half, model, model_bytes, - timing_x, timing_tokens) == 1, - "timing verification candidate"); + CHECK(run_mm_arm_projection( + timing_reference, timing_rhs_f16, model, model_bytes, + timing_x, timing_tokens, + DS4_GPU_TEST_Q4_QB_MM_Q4_F32, true) == 1, + "timing raw verification reference"); const uint64_t verify_bytes = timing_output_count * sizeof(float); @@ -1102,46 +2130,222 @@ int main(void) { float *verify_candidate = malloc((size_t)verify_chunk_bytes); CHECK(verify_reference && verify_candidate, "timing verification host chunks"); - uint64_t verify_mismatches = 0; - uint64_t verify_first = UINT64_MAX; - for (uint64_t offset = 0; offset < verify_bytes; - offset += verify_chunk_bytes) { - const uint64_t chunk_bytes = - verify_bytes - offset < verify_chunk_bytes - ? verify_bytes - offset - : verify_chunk_bytes; - CHECK(ds4_gpu_tensor_read(timing_reference, offset, - verify_reference, chunk_bytes) != 0, - "timing verification reference readback"); - CHECK(ds4_gpu_tensor_read(timing_out, offset, - verify_candidate, chunk_bytes) != 0, - "timing verification candidate readback"); - uint64_t chunk_first = UINT64_MAX; - verify_mismatches += count_bit_mismatches( - verify_reference, verify_candidate, - chunk_bytes / sizeof(float), &chunk_first); - if (verify_first == UINT64_MAX && chunk_first != UINT64_MAX) { - verify_first = offset / sizeof(float) + chunk_first; + for (uint32_t pass = 0; pass < 2u; pass++) { + if (pass == 1u) { + CHECK(ds4_gpu_head_rms_norm_rope_tail_tensor( + timing_reference, timing_tokens, + N_HEAD, HEAD_DIM, N_ROT, + 17u, 0u, false, + 10000.0f, 1.0f, 0.0f, 1.0f, + 32.0f, 1.0f, 1.0e-6f) != 0, + "timing tail verification reference"); + } + for (uint32_t arm_i = 1; + arm_i < sizeof(oracle_arms) / sizeof(oracle_arms[0]); + arm_i++) { + const ds4_gpu_test_q4_qb_mm_arm arm = + oracle_arms[arm_i]; + if (mm_arm_is_transient(arm) && + timing_tokens != 4096u) { + continue; + } + const bool prepacked_experiment = + mm_arm_is_prepacked_experiment(arm); + if (prepacked_experiment && + !ds4_gpu_test_q4_attn_q_b_mm_arm_supported(arm)) { + continue; + } + const bool rhs_is_f16 = mm_arm_uses_f16_rhs(arm); + /* The transient arm uses only the prepacked RHS; the compact + * copy oracle already covers the same producer above. */ + const uint32_t rhs_modes = + prepacked_experiment ? 1u : (rhs_is_f16 ? 2u : 1u); + for (uint32_t rhs_mode = 0; rhs_mode < rhs_modes; + rhs_mode++) { + const bool materialize_rhs = + prepacked_experiment ? false : rhs_mode == 0u; + const int ok = pass == 0u + ? run_mm_arm_projection( + timing_out, timing_rhs_f16, + model, model_bytes, timing_x, timing_tokens, + arm, materialize_rhs) + : run_mm_arm_with_tail( + timing_out, timing_rhs_f16, + model, model_bytes, timing_x, timing_tokens, + arm, materialize_rhs); + CHECK(ok == 1, "timing projection verification arm"); + + uint64_t verify_mismatches = 0; + uint64_t verify_first = UINT64_MAX; + for (uint64_t offset = 0; offset < verify_bytes; + offset += verify_chunk_bytes) { + const uint64_t chunk_bytes = + verify_bytes - offset < verify_chunk_bytes + ? verify_bytes - offset + : verify_chunk_bytes; + CHECK(ds4_gpu_tensor_read( + timing_reference, offset, + verify_reference, chunk_bytes) != 0, + "timing verification reference readback"); + CHECK(ds4_gpu_tensor_read( + timing_out, offset, + verify_candidate, chunk_bytes) != 0, + "timing verification candidate readback"); + uint64_t chunk_first = UINT64_MAX; + verify_mismatches += count_bit_mismatches( + verify_reference, verify_candidate, + chunk_bytes / sizeof(float), &chunk_first); + if (verify_first == UINT64_MAX && + chunk_first != UINT64_MAX) { + verify_first = + offset / sizeof(float) + chunk_first; + } + } + fprintf(stderr, + "Metal Q4 attn_q_b projection oracle N=%u " + "%s %s %s bitwise=%llu\n", + timing_tokens, + pass == 0u ? "raw" : "tail", + mm_arm_name(arm), + rhs_is_f16 + ? (materialize_rhs + ? "with-pack" : "prepacked") + : "control", + (unsigned long long)verify_mismatches); + if (verify_first != UINT64_MAX) { + fprintf(stderr, + " first mismatch token=%llu row=%llu\n", + (unsigned long long)(verify_first / OUT_DIM), + (unsigned long long)(verify_first % OUT_DIM)); + } + CHECK(verify_mismatches == 0, + "timing projection bitwise mismatch"); + } } } - fprintf(stderr, - "Metal Q4 attn_q_b F16 cache timing oracle N=%u " - "bitwise=%llu\n", - timing_tokens, - (unsigned long long)verify_mismatches); - if (verify_first != UINT64_MAX) { + + /* Time the complete public production selector after the kernel-only + * panels: F32->F16 RHS copy, Q4->F16 transient expansion, F16/F16 MM, + * and head norm/RoPE are all included. The control is the native + * Q4/F32 projection with the identical tail. Alternate first place + * to keep command-order and thermal bias paired. */ + if (timing_tokens == 4096u) { + CHECK(ds4_gpu_release_q4_attn_q_b_f16_sidecars() != 0, + "release sidecar before production timing"); + ds4_gpu_test_q4_attn_q_b_f16_cache_reset(); + CHECK(unsetenv(k_require) == 0, + "clear REQUIRE for production timing"); + CHECK(unsetenv(k_transient_min_tokens) == 0, + "use default production transient threshold"); + + CHECK(run_reference( + timing_reference, model, model_bytes, + timing_x, timing_tokens) == 1, + "production timing native warmup"); + CHECK(run_candidate( + timing_out, timing_rhs_f16, model, model_bytes, + timing_x, timing_tokens) == 1, + "production timing transient warmup"); + + for (uint32_t sample = 0; sample < TIMING_SAMPLES; sample++) { + for (uint32_t pos = 0; pos < 2u; pos++) { + const bool run_transient = + ((sample & 1u) == 0u) ? pos == 1u : pos == 0u; + const double t0 = monotonic_ms(); + const int ok = run_transient + ? run_candidate( + timing_out, timing_rhs_f16, + model, model_bytes, timing_x, timing_tokens) + : run_reference( + timing_reference, model, model_bytes, + timing_x, timing_tokens); + CHECK(ok == 1, "production timing dispatch"); + const double elapsed = monotonic_ms() - t0; + if (run_transient) { + production_transient_ms[sample] = elapsed; + } else { + production_reference_ms[sample] = elapsed; + } + } + } + + const double production_reference_median = + timing_quantile(production_reference_ms, 0.5); + const double production_reference_p25 = + timing_quantile(production_reference_ms, 0.25); + const double production_reference_p75 = + timing_quantile(production_reference_ms, 0.75); + const double production_transient_median = + timing_quantile(production_transient_ms, 0.5); + const double production_transient_p25 = + timing_quantile(production_transient_ms, 0.25); + const double production_transient_p75 = + timing_quantile(production_transient_ms, 0.75); + const double production_speedup = + timing_paired_geomean_speedup( + production_reference_ms, production_transient_ms); fprintf(stderr, - " first timing mismatch token=%llu row=%llu\n", - (unsigned long long)(verify_first / OUT_DIM), - (unsigned long long)(verify_first % OUT_DIM)); + "Metal Q4 attn_q_b public production timing N=4096\n" + " native Q4/F32 + tail %.3f ms [%.3f, %.3f] control\n" + " transient full + tail %.3f ms [%.3f, %.3f] " + "%.3fx paired-gmean\n", + production_reference_median, + production_reference_p25, + production_reference_p75, + production_transient_median, + production_transient_p25, + production_transient_p75, + production_speedup); + + /* Re-run both arms immediately before readback, then prove that + * the complete public entry point is exact at production N. */ + CHECK(run_reference( + timing_reference, model, model_bytes, + timing_x, timing_tokens) == 1, + "production timing final native reference"); + CHECK(run_candidate( + timing_out, timing_rhs_f16, model, model_bytes, + timing_x, timing_tokens) == 1, + "production timing final transient candidate"); + uint64_t production_mismatches = 0u; + for (uint64_t offset = 0; offset < verify_bytes; + offset += verify_chunk_bytes) { + const uint64_t chunk_bytes = + verify_bytes - offset < verify_chunk_bytes + ? verify_bytes - offset + : verify_chunk_bytes; + CHECK(ds4_gpu_tensor_read( + timing_reference, offset, + verify_reference, chunk_bytes) != 0, + "production timing reference readback"); + CHECK(ds4_gpu_tensor_read( + timing_out, offset, + verify_candidate, chunk_bytes) != 0, + "production timing transient readback"); + uint64_t production_chunk_first = UINT64_MAX; + production_mismatches += count_bit_mismatches( + verify_reference, verify_candidate, + chunk_bytes / sizeof(float), + &production_chunk_first); + } + CHECK(production_mismatches == 0u, + "production timing public entry point mismatch"); + ds4_gpu_q4_attn_q_b_f16_cache_report production_report; + ds4_gpu_test_q4_attn_q_b_f16_cache_report( + &production_report); + CHECK(production_report.entries == 0u && + production_report.bytes == 0u && + production_report.lookups == 0u && + production_report.builds == 0u, + "production timing retained a sidecar"); + CHECK(setenv(k_require, "1", 1) == 0, + "restore REQUIRE after production timing"); } - CHECK(verify_mismatches == 0, - "timing geometry candidate bitwise mismatch"); free(verify_candidate); free(verify_reference); ds4_gpu_tensor_free(timing_reference); - ds4_gpu_tensor_free(timing_q_half); + ds4_gpu_tensor_free(timing_rhs_f16); ds4_gpu_tensor_free(timing_out); ds4_gpu_tensor_free(timing_x); } @@ -1164,6 +2368,10 @@ int main(void) { CHECK(unsetenv(k_require) == 0, "clear cache require env at exit"); CHECK(unsetenv(k_min_tokens) == 0, "clear cache minimum env at exit"); + CHECK(unsetenv(k_disable_transient_f16) == 0, + "clear transient F16 disable env at exit"); + CHECK(unsetenv(k_transient_min_tokens) == 0, + "clear transient F16 minimum env at exit"); fprintf(stderr, "Metal Q4 attn_q_b F16 cache production geometry " "1024x32768 N=32/33/64: PASS\n"); From 759605fcb699119f15c502e37c39caeb5e777f7a Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Wed, 26 Aug 2026 22:28:02 +0200 Subject: [PATCH 121/189] docs: remove obsolete PR621 complexity analysis --- ANALISI_COMPLESSITA_PR621.md | 339 ----------------------------------- 1 file changed, 339 deletions(-) delete mode 100644 ANALISI_COMPLESSITA_PR621.md diff --git a/ANALISI_COMPLESSITA_PR621.md b/ANALISI_COMPLESSITA_PR621.md deleted file mode 100644 index 54ae78b22..000000000 --- a/ANALISI_COMPLESSITA_PR621.md +++ /dev/null @@ -1,339 +0,0 @@ -# Analisi di complessità e ristrutturazioni prestazionali — PR #621 - -Data: 23 agosto 2026 - -Base analizzata: `84cc882` - -HEAD dello snapshot originario: `167eb107f0f15aca922335718bfc7c69952aa815` - -## Stato delle implementazioni successive - -L'analisi delle funzioni e dei costi è stata redatta sullo snapshot indicato -sopra. Le ristrutturazioni seguenti sono state successivamente implementate -nel worktree e non sono più proposte future: - -- CPU: dot Q4_K su coppie di token con decode dei pesi condiviso e - quantizzazione Q8_K batch parallela (`6e0527aa`); -- CUDA: coppia MMQ Q4_K per il prefill con una sola quantizzazione Q8_1 - condivisa (`b4a106ec`); -- ROCm: singolo launch pair per decode e TILE8 prefill (`a02008f6`). - -Le sezioni seguenti mantengono la fotografia completa dell'analisi originaria, -ma annotano questi punti come completati dove compaiono tra le priorità. - -## Perimetro - -La PR contiene 43.301 inserimenti, 5.161 rimozioni e 49 file modificati. Non tutto il diff appartiene al supporto AProjQ4: include DSpark, SSD streaming, tooling, test, documentazione e ottimizzazioni di altri quanti. Questa analisi copre **tutte le funzioni produttive direttamente coinvolte nel percorso Q4_K delle proiezioni dense di attenzione**, incluse le funzioni di dispatch e gli helper che ne determinano il costo. I test, il generatore GGUF e le funzioni puramente diagnostiche sono classificati a parte: il loro costo non incide sull’inferenza. - -La complessità asintotica non cambia tra implementazioni corrette di una proiezione densa: il lavoro matematico minimo resta proporzionale agli elementi della matrice. Le ottimizzazioni utili riducono soprattutto traffico di memoria, quantizzazioni ripetute, lanci kernel e materiale intermedio. - -## Simboli usati - -| Simbolo | Significato | -|---|---| -| `L` | numero di layer | -| `T` | token nel batch/prefill chunk; `T=1` nel decode ordinario | -| `K` | dimensione di input della proiezione | -| `M` | dimensione di output della proiezione | -| `M0`,`M1` | output di due proiezioni accoppiate | -| `G` | gruppi/head group elaborati | -| `R` | rank per gruppo | -| `Q=256` | elementi per superblocco Q4_K/Q8_K | -| `B=K/Q` | superblocchi per riga | -| `S` | esperti selezionati per token | - -Per una matrice Q4_K `[M,K]`: - -- operazioni: `Θ(T·M·K)`; -- lettura pesi senza riuso tra token: `Θ(T·M·K)` elementi logici; -- lettura pesi con tiling su `τ` token: circa `Θ((T/τ)·M·K)` blocchi fisici, a parità di lavoro aritmetico; -- quantizzazione attivazioni F32→Q8_K: `Θ(T·K)` tempo e `Θ(T·K)` memoria temporanea, con costante compressa dal formato a blocchi. - -## 1. CPU reference path (`ds4.c`) - -Il backend CPU è dichiaratamente reference/debug; ottimizzarlo non accelera Metal/CUDA/ROCm. È comunque utile come baseline algoritmica. - -| Funzione | Tempo | Memoria extra | Osservazioni | -|---|---:|---:|---| -| `q4_k_get_scale_min` | `Θ(1)` | `Θ(1)` | Decodifica scale/min di un sottoblocco. Inlinabile. | -| `ds4_vec_dot_q4_K_q8_K` | `Θ(K)` | `Θ(1)` | Un dot di una riga. Due passaggi per blocco: min correction e prodotto Q4×Q8. | -| `ds4_vec_dot_q4_K_f32` | `Θ(K)` | `Θ(1)` | Fallback diretto F32; evita quantizzazione ma moltiplica per valori dequantizzati. | -| `dense_q4_K_expect` | `Θ(1)` | `Θ(1)` | Validazione forma/tipo. | -| `matvec_q4_K_dense_worker` | `Θ((r1-r0)·K)` | `Θ(1)` | Una riga per iterazione; pesi letti una volta per chiamata. | -| `matvec_q4_K_prequant` | `Θ(M·K)` | `Θ(1)` locale | Dispatch parallelo usando un `xq` già pronto. | -| `matvec_q4_K` | `Θ(K + M·K)` | `Θ(K)` | Alloca e quantizza l’input a ogni chiamata. | -| `matvec_q4_K_decode_scratch` | `Θ(K + M·K)` | `Θ(1)` per chiamata | Riusa lo scratch; elimina malloc/free, non la quantizzazione. | -| `matvec_q4_K_grouped_expect` | `Θ(1)` | `Θ(1)` | Validazione. | -| `matvec_q4_K_grouped_worker` | `Θ((r1-r0)·K)` | `Θ(1)` | Totale `Θ(G·R·K)`. | -| `matvec_q4_K_grouped_rows_prequant` | `Θ(G·R·K)` | `Θ(1)` locale | Input Q8_K già pronto. | -| `matvec_q4_K_grouped_rows` | `Θ(G·K + G·R·K)` | `Θ(G·K)` | Quantizza ogni gruppo e poi proietta. | -| `matvec_q4_K_grouped_rows_decode_scratch` | `Θ(G·K + G·R·K)` | `Θ(1)` per chiamata | Scratch persistente. | -| `matmul_q4_K_batch_worker` | `Θ((r1-r0)·T·K)` | `Θ(1)` | Elabora coppie di token condividendo decode Q4_K; coda singola per `T` dispari. | -| `matmul_q4_K_batch` | `Θ(T·K + T·M·K)` | `Θ(T·K)` | Prefill CPU; quantizzazione per riga parallela oltre la soglia di lavoro. | -| `matmul_q4_K_grouped_batch_worker` | `Θ((r1-r0)·T·K)` | `Θ(1)` | Totale `Θ(T·G·R·K)`, con lo stesso microkernel a due token. | -| `matmul_q4_K_grouped_batch` | `Θ(T·G·K + T·G·R·K)` | `Θ(T·G·K)` | Analogo grouped. | -| `matvec_any`, `matvec_any_decode_scratch` | `Θ(K+M·K)` sul ramo Q4 | dipende dal ramo | Dispatcher `Θ(1)` più callee. | -| `layer_q_projection_*`, `layer_kv_projection_*` | costo della proiezione | nessuna propria rilevante | Wrapper; sul ramo Q4 ereditano `Θ(M·K)`. | -| `layer_grouped_out_*` | `Θ(G·R·K)` o `Θ(T·G·R·K)` | come callee | Wrapper output attention. | - -### Ristrutturazioni CPU - -1. **Un solo passaggio NEON nel dot Q4×Q8.** L’implementazione ARM costruisce `q4_u[32]` sullo stack e percorre ogni superblocco due volte. Decodificare min/scale e nibble direttamente in registri NEON elimina lo scratch locale e una parte dei load/store. Complessità invariata `Θ(K)`, costante più bassa. -2. **Tiling 2D per `matmul_q4_K_batch_worker`.** Il loop `row → token` rilegge l’attivazione Q8 per ogni riga e non esprime un microkernel. Un blocco `MR×NT` mantiene più accumulatori e riusa blocchi di peso/attivazione: ancora `Θ(T·M·K)`, ma migliore cache e SIMD. -3. **Quantizzazione fusionata con RMSNorm.** Se l’output F32 della norm serve solo alla proiezione Q4, produrre contemporaneamente Q8_K elimina una lettura e una scrittura F32: risparmio `Θ(T·K)` di traffico. - -## 2. CUDA — primitive e kernel Q4_K (`ds4_cuda.cu`) - -### Primitive per superblocco - -| Funzione | Tempo per chiamata | Memoria | Nota | -|---|---:|---:|---| -| `dev_q4_K_get_scale_min` | `Θ(1)` | registri | Helper inlinabile. | -| `dev_dot_q4_K_q8_K_block` | `Θ(Q)` | registri | Un superblocco; costante perché `Q=256`, ma linearizzato come `Θ(Q)`. | -| `dev_dot_q4_K_q8_K_block_vec` | `Θ(Q)` | più registri | Nove load da 16 B; riduce istruzioni di load e migliora coalescenza. | -| `dev_dot_q4_K_q8_K_block8` | `Θ(8·Q)` | registri elevati | Riusa un blocco peso su fino a 8 token. È il nucleo concettuale del prefill tiled. | -| `quarter_warp_sum_f32` | `Θ(log 8)` = `Θ(1)` | registri | Riduzione intra-warp. | - -### Dense projection e wrapper - -| Funzione | Tempo | Memoria extra | Impatto | -|---|---:|---:|---| -| `matmul_q4_K_dense_kernel` | `Θ(T·M·K)` globale | `Θ(1)` per thread | Kernel bandwidth-oriented; ogni token rilegge tutti i pesi. Buono per `T=1`, pessimo per prefill grande. | -| `matmul_q4_K_dense_pair_kernel` | `Θ(T·(M0+M1)·K)` | due accumulatori | Condivide `xq`, non i pesi. Riduce un lancio/quantizzazione, non il termine dominante. | -| `cuda_matmul_q4_K_tensor` | `Θ(T·K + T·M·K)` | `Θ(T·K)` | Usa MMQ tiled quando accettato; fallback al kernel sopra. | -| `cuda_matmul_q4_K_pair_tensor_impl` | `Θ(T·K + T·(M0+M1)·K)` | `Θ(T·K)` | Quantizza una volta per due matrici. | -| `ds4_gpu_matmul_q4_K_pair_tensor` | come impl | `Θ(1)` propria | Wrapper. | -| `matmul_q4_K_kslice_kernel` | `Θ(M·Kslice)` | `Θ(1)` | Proiezione su slice K; decode/TP. | -| `cuda_matmul_q4_K_kslice_tensor` | `Θ(Kslice + M·Kslice)` | `Θ(Kslice)` | Quantizza la slice. | -| `ds4_gpu_attention_output_q4_K_batch_tensor` | `Θ(T·G·R·K + T·M·G·R)` circa | temporanei batch | Proietta A per gruppo e B globale; prova batch MMQ, poi fallback. | -| `ds4_gpu_attention_output_low_q4_K_slice_tensor` | `Θ(Gsel·R·K)` | `Θ(Gsel·K)` | Decode grouped attention-A. | - -### HC expand e fusioni - -| Funzione | Tempo | Memoria/traffico | Nota | -|---|---:|---:|---| -| `matmul_q4_K_hc_expand4_kernel` | `Θ(M·K + 4M)` | evita una rilettura di `block_out` | Fonde proiezione B e HC postprocess; ottima direzione per decode. | -| `q4_K_hc_expand4_rows_kernel` | `Θ(4M)` | legge/scrive intermedi | Esegue solo epilogo dopo MMQ canonico. | -| `q4_K_attn_hc_bitwise_compare_kernel` | `Θ(M)` | contatori diagnostici | Non deve essere nel release path. | -| `cuda_q4_K_hc_expand_canonical` | `Θ(M·K+4M)` | materiale `block_out` | MMQ + epilogo separato; due lanci. | -| `cuda_q4_K_hc_expand_q8k_launch` | `Θ(K+M·K+4M)` | Q8_K scratch | Fallback/fused. | -| `ds4_gpu_matmul_q4_K_hc_expand_available` | `Θ(1)` | `Θ(1)` | Query; dovrebbe essere precalcolata nel piano layer. | -| `ds4_gpu_matmul_q4_K_hc_expand_tensor` | `Θ(K+M·K+4M)` | scratch/oracle se attivo | Dispatcher con variabili diagnostiche. | - -## 3. CUDA MMQ (`cuda/mmq/ds4_mmq.cu`) - -Le funzioni MMQ sono il percorso che ha eliminato la regressione iniziale di prefill (~16×). La distinzione fondamentale è: - -- decode/vector: `Θ(M·K)`, ottimizzato per latenza e bandwidth; -- prefill/matrix: `Θ(T·M·K)`, con pesi riusati su tile di token e tensor-core/microkernel quando possibile. - -| Funzione | Complessità | Ruolo | -|---|---:|---| -| `ds4_mmq_dense_impl` | `Θ(T·M·K)` | Core generico tiled. | -| `ds4_mmq_q4_K_dense` | `Θ(T·M·K)` | Wrapper Q4_K prefill/MMQ. | -| `ds4_mmq_dense_vec_impl` | `Θ(M·K)` | Core decode vettoriale. | -| `ds4_mmq_q4_K_dense_vec` | `Θ(M·K)` | Wrapper decode Q4_K. | -| `ds4_mmq_dense_pair_vec_impl` | `Θ((M0+M1)·K)` | Due proiezioni, Q8 input condiviso. | -| `ds4_mmq_q4_K_dense_pair_vec` | stesso | Wrapper. | -| `ds4_mmq_q4_K_dense_pair` | `Θ(T·(M0+M1)·K)` | Prefill: due MMQ condividono una sola attivazione Q8_1 e il relativo scratch. | -| `ds4_mmq_q4_K_grouped_batch_vec_impl` | `Θ(T·G·R·K)` | Prefill grouped. | -| `ds4_mmq_q4_K_grouped_batch_vec` | stesso | Wrapper. | -| `ds4_mmq_q4_K_grouped_vec` | `Θ(G·R·K)` | Decode grouped. | -| `q4_K_k1024_bitwise_compare_kernel` | `Θ(M)` | Diagnostica. | -| `q4_k1024_env_flag`, report/counters | `Θ(1)` | Controllo/diagnostica; fuori dall’hot loop ideale. | -| `iq2_aligned_quantize_xn` | `Θ(T·K)` | Quantizzazione attivazioni per altri percorsi MMQ; può condividere scratch. | -| `q8_0_aligned_dense_vec_*` | `Θ(M·K)` | Q8, non AProjQ4 puro; incide sui tensori SExpQ8/OutQ8 della stessa rete. | - -### Osservazione critica - -La PR contiene più livelli di fallback che ripetono decisioni a runtime: capability, shape, env flag, disponibilità MMQ, path required/disabled. Ogni check è `Θ(1)`, ma viene ripetuto per proiezione e per layer. Il costo GPU domina sulle matrici grandi, tuttavia in decode a token singolo la latenza di launch/dispatch è visibile. Un **execution plan immutabile per layer**, costruito al load, può sostituire questa foresta di branch con puntatori a funzione e parametri già validati. - -## 4. ROCm (`rocm/ds4_rocm_q4.cuh`) - -| Funzione | Tempo | Memoria | Valutazione | -|---|---:|---:|---| -| `rocm_matmul_q4_K_dense_kernel` | `Θ(T·M·K)` | registri | Legacy: rilegge pesi per token. Decode. | -| `rocm_matmul_q4_K_dense_grouped_decode_kernel` | `Θ(G·R·K)` | registri | Decode grouped. | -| `rocm_matmul_q4_K_prefill_tile8_strided_kernel` | `Θ(T·M·K)` | LDS/register tile | Riusa pesi su 8 token; evidenza empirica +125–135% prefill. | -| `rocm_q4_K_dense_validate` | `Θ(1)` | `Θ(1)` | Validazione range/shape. | -| `rocm_q4_K_prequant_alloc` | `Θ(1)` logico | `Θ(T·K)` | Allocazione da scratch temporaneo. | -| `rocm_q4_K_dense_pair_requested` | `Θ(1)` | `Θ(1)` | Tre `getenv` nel path; da precalcolare. | -| `rocm_q4_K_prefill_tile8_scope/requested/required` | `Θ(1)` | `Θ(1)` | Policy. | -| report/note/result helpers | `Θ(1)` | `Θ(1)` | Diagnostica; atomiche/contatori host solo se abilitati. | -| `ds4_rocm_matmul_q4_K_tensor` | `Θ(T·K + T·M·K)` | `Θ(T·K)` | Quantizza, poi Tile8 per `T>8`, legacy altrimenti. | -| `ds4_gpu_matmul_q4_K_pair_tensor` | `Θ(T·K + T·(M0+M1)·K)` | `Θ(T·K)` | Condivide quantizzazione e usa un singolo launch pair sia in decode sia nel prefill Tile8. | -| `ds4_gpu_attention_output_low_q4_K_slice_tensor` | `Θ(G·R·K)` | `Θ(G·K)` | Grouped decode. | -| `rocm_q4_K_prefill_tile8_quant_launch` | `Θ(T·G·K + T·G·R·K)` | `Θ(T·G·K)` | Quant + grouped tile8. | -| `ds4_gpu_attention_output_q4_K_batch_tensor` | `Θ(T·G·R·K + T·M·G·R)` | temporanei | Due stadi A/B; Tile8 solo per `T>8`. | - -### Ristrutturazioni ROCm ad alto valore - -1. **Completato — pair Tile8 in un solo kernel.** Le due matrici condividono `xq` e un unico launch con domini di row-tile concatenati in `grid.x`. Il costo resta `Θ(T·(M0+M1)·K)` e l'ordine di accumulo per riga resta quello del kernel standalone. -2. **Token tile adattivo `{4,8,16}`.** La soglia fissa `T>8` lascia i microbatch 2–8 sul kernel che rilegge i pesi. Autotuning per shape/arch può usare tile4 per spec decode e tile16 quando LDS/occupancy lo consentono. -3. **Quantizzazione prodotta dal kernel precedente.** `q8_K_quantize_kernel` è sempre un lancio separato. Fusione RMSNorm→Q8_K o doppia uscita F32+Q8_K elimina `Θ(T·K)` traffico e un launch. - -## 5. Metal (`ds4_metal.m`, `metal/moe.metal`, `metal/dsv4_misc.metal`) - -### Funzioni host principali - -| Funzione | Tempo host | Lavoro GPU | Nota | -|---|---:|---:|---| -| `ds4_gpu_matmul_q4_K_pair_tensor` | `Θ(1)` encode | `Θ(T·(M0+M1)·K)` | Pair small-batch; condivide input e command buffer. | -| `ds4_gpu_q4_K_pair_quad_compressor_store_tensor` | `Θ(1)` encode | `Θ((M0+M1)·K + compressor)` | Fonde Q/KV pair con compress/store; riduce intermedi e launch. | -| `ds4_gpu_attention_output_q4_K_ssd_prefill_exactn_tensor` | `Θ(1)` encode | `Θ(T·G·R·K)` sulle righe esatte | Ottimizzazione SSD prefill, vincolata agli exact rows. | -| `ds4_gpu_attention_output_q4_K_batch_tensor` | `Θ(1)` + branch | `Θ(T·G·R·K + T·M·G·R)` | Sceglie classic MV, MM tiled e exactn. | -| `ds4_gpu_attention_output_low_q4_K_slice_tensor` | `Θ(1)` encode | `Θ(G·R·K)` | Decode low projection. | -| `ds4_gpu_matmul_q4_K_hc_expand_available` | `Θ(1)` | nessuno | Query pipeline. | -| `ds4_gpu_matmul_q4_K_hc_expand_tensor` | `Θ(1)` encode | `Θ(M·K+4M)` | Fused decode tail. | - -### Kernel Metal Q4_K direttamente rilevanti - -| Funzione/kernel | Complessità globale | Caratteristica | -|---|---:|---| -| `ds4_glm_q4_K_value`, `glm_q4_K_scale_min` | `Θ(1)` | Decodifica elemento/scale. | -| `kernel_mul_mv_q4_K_f32_impl` | `Θ(M·K)` | Core matvec Q4. | -| `kernel_mul_mv_q4_K_dense_f32` | `Θ(T·M·K)` | Entry dense small batch. | -| `kernel_mul_mv_q4_K_dense_pair_f32` | `Θ(T·(M0+M1)·K)` | Pair. | -| `kernel_dsv4_q4_K_qkv_pair_quad_compressor_store` | `Θ((M0+M1)·K + compressor)` | Fusione verticale molto utile nel decode. | -| `kernel_dsv4_q4_K_hc_expand4` | `Θ(M·K+4M)` | Fusione output-B + HC. | -| `kernel_dsv4_attn_out_low_q4_K_f32` | `Θ(G·R·K)` | Low projection. | -| `kernel_mul_mv_q4_K_staged_exactn_impl` | `Θ(Eexact·M·K)` | Solo righe richieste; riduce lavoro se `Eexact << T`. | -| `kernel_dsv4_attn_out_q4_K_ssd_prefill_exactn_f32` | `Θ(Eexact·G·R·K)` | Prefill SSD specializzato. | -| `glm_q4_K_dot_row_tg_f32`, `glm_q4_K_dot_row_lane_f32`, `glm_quant_dot_row_*` | `Θ(K)` per riga | Varianti di riduzione/layout. | -| `kernel_glm_q4_K_pair_swiglu*` e varianti addr/mapped | `Θ(S·M·K)` | MoE Q4, non proiezione dense AProjQ4 ma incide sui modelli SExpQ4. | -| `kernel_glm_q4_K_down*` | `Θ(S·M·K)` | Down MoE. | - -### Ristrutturazioni Metal - -1. **Piano di encoding per layer.** Spostare selezione pipeline, numero simdgroup, controllo tipi/env e dimensioni dal token loop al caricamento. Il runtime esegue una lista di encoder prevalidati. Non cambia Big-O, riduce CPU latency e branch. -2. **Unificare fusioni verticali in una pipeline AProjQ4.** Il codice ha già due fusioni corrette: Q/KV pair→compressor e output-B→HC expand. Il passo successivo è evitare la materializzazione F32 tra RMSNorm e pair quantizzato, mantenendo un buffer Q8_K/f16 compatto condiviso. -3. **Tile prefill determinato dalla shape.** Per `T` medio, classic matvec e MM general-purpose possono entrambi essere subottimali. Serve un microbenchmark al load per scegliere `mul_mv_ext`, `mul_mm direct RHS` e tile N per ogni shape AProjQ4. -4. **Non applicare `FOR_UNROLL` indiscriminato.** Le review inline lo suggeriscono, ma l’unroll può aumentare pressione registri e ridurre occupancy. Usarlo solo se il metallib disassembly/benchmark mostra meno cicli senza spill. - -## 6. Costo end-to-end per layer - -Le cinque proiezioni dense AProjQ4 dominano come: - -```text -Q_a : Θ(T·Kqa·Mqa) -Q_b : Θ(T·Kqb·Mqb) -KV : Θ(T·Kkv·Mkv) -Out_a : Θ(T·G·Koa·R) -Out_b : Θ(T·Kob·Mob) -``` - -Il costo layer rimane la somma dei cinque termini. Sul decode (`T=1`) l’operazione è generalmente **memory-bandwidth bound**: ogni token deve rileggere le matrici dense. Sul prefill (`T≫1`) diventa possibile riusare i pesi su più token e usare microkernel/GEMM; questa è la ragione per cui il passaggio CUDA da fallback matvec a MMQ ha recuperato circa 15,5× e TILE8 ROCm ha guadagnato 125–135%. - -## 7. Ristrutturazioni prioritarie - -### P0 — prefill: garantire un vero path tiled per ogni backend e shape - -**Problema:** il fallback `token × matvec` ha la stessa complessità `Θ(T·M·K)` ma traffico pesi circa `T` volte maggiore. È la causa già osservata del prefill ~16× più lento. - -**Intervento:** creare una sola API semantica: - -```c -q4_dense_run(plan, out[], weights[], input, T) -``` - -Il `plan` sceglie al load: - -- decode vector per `T=1`; -- microbatch tile4/8 per `2≤T≤8/16`; -- MMQ/GEMM tiled per prefill; -- pair/multi-projection quando gli input sono identici. - -**Guadagno atteso:** massimo sul prefill; nessun cambiamento numerico se si preserva l’ordine di accumulo dove richiesto. Il beneficio non è teorico: è già dimostrato da CUDA MMQ e ROCm TILE8. - -### P0 — prefill e decode: eliminare la quantizzazione ridondante - -**Problema:** F32→Q8_K è `Θ(T·K)` e spesso è un lancio separato. Pair Q_a/KV la condivide già, ma la rappresentazione quantizzata viene prodotta dopo che RMSNorm ha scritto F32 in memoria. - -**Intervento:** RMSNorm con doppia uscita o uscita Q8_K nativa per le proiezioni Q4; conservare F32 solo se un consumer reale la richiede. Il piano layer dichiara la lifetime del Q8_K. - -**Guadagno atteso:** riduzione di un kernel, una lettura F32 e una scrittura Q8_K per gruppo di proiezioni; più importante per decode latency che per il termine `M·K`. - -### P1 — decode: execution plan immutabile - -**Problema:** capability/env/shape/type checks e fallback sono ripetuti dentro il percorso per token. Il codice è difficile da verificare e impedisce al compilatore/CPU di avere un percorso lineare. - -**Intervento:** al model load costruire per ogni layer: - -- puntatore encoder/kernel; -- tile e geometry; -- offset/row bytes già validati; -- scratch richiesto; -- fusioni disponibili; -- fallback definitivo. - -Il token loop non chiama `getenv`, non rivalida i range e non cerca pipeline. - -**Guadagno atteso:** piccolo ma sistematico sul decode; grande vantaggio di manutenibilità e minore rischio di false fallback. - -### P1 completata — ROCm: pair Tile8 in un lancio - -Il path pair ora condivide la quantizzazione ed elabora entrambe le matrici in -un solo launch, concatenando i due domini di tile. La complessità resta -invariata; viene eliminato un launch per layer senza modificare l'ordine delle -riduzioni. - -### P1 — Metal: command-buffer fusion e buffer intermedi - -Completare la catena di fusioni già iniziata: - -```text -RMSNorm → Q8_K → {Q_a, KV} → compressor/store -attention heads → Out_a → Out_b → HC expand -``` - -Non è necessario fondere tutto in un solo mega-kernel: bastano due o tre kernel verticali con lifetime esplicite e nessuna round-trip F32 non necessaria. - -### P2 — decode: cache dequantizzata selettiva, solo con budget - -Una cache FP16 delle sole cinque matrici dense può abilitare GEMV/GEMM più semplici ma aumenta i byte per peso di circa 3–4× rispetto a Q4_K. Sul decode bandwidth-bound può essere peggiore. Va considerata solo: - -- su macchine con ampia memoria residua; -- se il backend dispone di una primitive tensor-core realmente più rapida; -- dopo benchmark A/B che includa il costo di warmup e pressione cache. - -Non è una raccomandazione di default. - -## 8. Proposte da evitare - -- **Ridurre Big-O con pruning non validato.** Saltare righe/pesi cambia il modello; non è una ristrutturazione semantics-preserving. -- **Fondere matrici concatenandole permanentemente nel GGUF senza misure.** Q_a e KV hanno output diversi; la concatenazione semplifica un launch ma può peggiorare locality e streaming. -- **Cache FP16 globale dei pesi Q4.** Aumenta memoria e bandwidth; contraddice il motivo di AProjQ4. -- **Flag permanenti per ogni variante.** Il file AGENT.md richiede un solo release path. Usare flag solo per diagnosi/rollback, poi scegliere il vincitore. -- **Unroll aggressivo non misurato.** Può ridurre occupancy e peggiorare sia prefill sia decode. - -## 9. Piano di benchmark per validare le ristrutturazioni - -Per ogni proposta misurare separatamente: - -1. kernel time e numero di launch per layer; -2. byte letti/scritti e cache hit, se il profiler lo espone; -3. prefill a `T={1,2,4,8,16,128,512,2048,4096}`; -4. decode a contesti 2k/4k/8k e almeno 128 token; -5. `--decode-consistency` e `score_official`; -6. memoria di picco e startup spans; -7. AProjQ8 control per impedire regressioni del percorso esistente. - -Ordine consigliato degli esperimenti: - -1. execution plan senza cambiare kernel; -2. RMSNorm→Q8_K fusion; -3. misurazione hardware del ROCm pair Tile8 a singolo launch (implementato); -4. tile adattivo per microbatch; -5. Metal vertical fusion; -6. solo infine cache dequantizzata selettiva. - -## 10. Conclusione - -Non esiste una ristrutturazione semantics-preserving che trasformi la proiezione densa da `Θ(T·M·K)` a un ordine inferiore. I guadagni reali vengono da quattro leve: - -1. **riuso dei pesi tra token nel prefill**; -2. **riuso della quantizzazione tra proiezioni**; -3. **fusione degli epiloghi e rimozione degli intermedi**; -4. **riduzione di launch e dispatch nel decode**. - -La priorità più solida è rendere il percorso tiled obbligatorio per tutte le shape di prefill, poi fondere RMSNorm→Q8_K e costruire un execution plan per layer. Queste modifiche attaccano costi già visibili nel codice e già confermati dai benchmark della PR, senza introdurre approssimazioni sul modello. From a4902403b6754915e16bb6885b8cfe0654f6e946 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Thu, 27 Aug 2026 11:55:49 +0200 Subject: [PATCH 122/189] perf(gpu): optimize Q4 resident prefill across backends --- ds4.c | 232 +++++++++- ds4_cuda.cu | 124 ++++-- ds4_gpu.h | 54 +++ ds4_metal.m | 600 +++++++++++++++++++++++-- metal/dsv4_hc.metal | 574 ++++++++++++------------ rocm/ds4_rocm_moe.cuh | 10 +- rocm/ds4_rocm_norm_rope.cuh | 123 +++++- rocm/ds4_rocm_q4.cuh | 170 +++++++- rocm/ds4_rocm_q4_qb_sidecar.cuh | 68 ++- scripts/environment_variables.tsv | 41 +- tests/ds4_test.c | 699 ++++++++++++++++++++++++++++++ tests/test_rocm_q4_dense_pair.cpp | 255 ++++++++++- 12 files changed, 2537 insertions(+), 413 deletions(-) diff --git a/ds4.c b/ds4.c index 2fd8ba7e3..ae0757b07 100644 --- a/ds4.c +++ b/ds4.c @@ -30850,18 +30850,37 @@ static bool metal_graph_encode_layer_attention_batch( const bool topk_prefill_needed = ratio == 4 && n_comp > DS4_N_INDEXER_TOP_K; -#if defined(__APPLE__) +#if !defined(DS4_NO_GPU) + const bool indexer_query_prune_common = + ratio == 4 && zero_prefix && n_tokens >= 32u && + !topk_prefill_needed && !g->quality && + g->placement == NULL && g->tp_world < 2u; +#endif +#if defined(__APPLE__) && !defined(DS4_NO_GPU) /* Before the compressed cache grows past top-k, zero-prefix prefill * consumes every compressed row and never reads the transient indexer * query or its per-head weights. This is also true for the Metal SSD * layer-major path: the current layer is mapped while these dispatches * would run, and skipping them changes no persistent cache state. */ const bool prune_unused_indexer_query = - ratio == 4 && zero_prefix && n_tokens >= 32u && - !topk_prefill_needed && !g->quality && - g->placement == NULL && g->tp_world < 2u && + indexer_query_prune_common && ds4_gpu_device_is_pre_m5_apple_silicon() && getenv("DS4_METAL_DISABLE_PRE_M5_BATCH_INDEXER_QUERY_PRUNE") == NULL; +#elif defined(DS4_ROCM_BUILD) + /* The query projection is equally dead on resident ROCm: before the + * compressed cache exceeds top-k, attention consumes every compressed + * row and no later stage observes these transient query/weight tensors. + * Keep streaming out of scope because its layer-lifetime and overlap + * policy are intentionally independent from the resident fast path. */ + const bool prune_unused_indexer_query = + indexer_query_prune_common && !g->ssd_streaming && + getenv("DS4_ROCM_DISABLE_BATCH_INDEXER_QUERY_PRUNE") == NULL; +#elif !defined(DS4_NO_GPU) + /* CUDA has the same resident layer lifetime as ROCm here. Do not + * couple this graph-level pruning to any SSD streaming policy. */ + const bool prune_unused_indexer_query = + indexer_query_prune_common && !g->ssd_streaming && + getenv("DS4_CUDA_DISABLE_BATCH_INDEXER_QUERY_PRUNE") == NULL; #else const bool prune_unused_indexer_query = false; #endif @@ -31633,8 +31652,61 @@ static bool metal_graph_encode_layer_attention_batch( const bool attn_out_debug = metal_graph_debug_wants("attn_low", il, pos0) || metal_graph_debug_wants("attn_out", il, pos0); + bool attn_out_hc_fused = false; +#ifdef __APPLE__ + if (ok && !attn_out_debug && !tp_row_split_attn && + !metal_graph_directional_steering_attn_enabled(g)) { + int fused_rc = 0; + if (layer->attn_output_a->type == DS4_TENSOR_Q8_0 && + layer->attn_output_b->type == DS4_TENSOR_Q8_0) { + fused_rc = ds4_gpu_attention_output_q8_batch_hc_tensor( + metal_graph_batch_attn_out(g), + after_attn_hc_view, + metal_graph_batch_cur_hc(g), + hc_split_view, + metal_graph_batch_attn_low(g), + metal_graph_batch_group_tmp(g), + metal_graph_batch_low_tmp(g), + model->map, + model->size, + layer->attn_output_a->abs_offset, + layer->attn_output_b->abs_offset, + group_dim, + rank, + n_groups, + DS4_N_EMBD, + metal_graph_batch_heads(g), + n_tokens, + DS4_N_HC); + } else if (layer->attn_output_a->type == DS4_TENSOR_Q4_K && + layer->attn_output_b->type == DS4_TENSOR_Q4_K) { + fused_rc = ds4_gpu_attention_output_q4_K_batch_hc_tensor( + metal_graph_batch_attn_out(g), + after_attn_hc_view, + metal_graph_batch_cur_hc(g), + hc_split_view, + metal_graph_batch_attn_low(g), + metal_graph_batch_group_tmp(g), + metal_graph_batch_low_tmp(g), + model->map, + model->size, + layer->attn_output_a->abs_offset, + layer->attn_output_b->abs_offset, + layer->attn_output_b->type, + group_dim, + rank, + n_groups, + DS4_N_EMBD, + metal_graph_batch_heads(g), + n_tokens, + DS4_N_HC); + } + if (fused_rc < 0) ok = false; + attn_out_hc_fused = fused_rc > 0; + } +#endif bool attn_out_f16 = false; - if (ok && + if (ok && !attn_out_hc_fused && !attn_out_debug && !tp_row_split_attn && layer->attn_output_a->type == DS4_TENSOR_Q8_0 && @@ -31667,7 +31739,7 @@ static bool metal_graph_encode_layer_attention_batch( const bool tp_attn_pipeline = tp_row_split_attn && (n_tokens % 256u) == 0u && metal_graph_tp_subgate_pipeline(); - if (!attn_out_f16) { + if (!attn_out_f16 && !attn_out_hc_fused) { if (ok && tp_attn_pipeline) { /* Sub-chunk pipelined swap: the output projection runs in two * sub-halves of this rank's rows and each sub-half's row swap @@ -31772,10 +31844,13 @@ static bool metal_graph_encode_layer_attention_batch( } if (!ok) fprintf(stderr, "ds4: TP prefill attention row gate failed (layer %u)\n", il); } - if (ok && !attn_out_f16 && metal_graph_directional_steering_attn_enabled(g)) { + if (ok && !attn_out_f16 && !attn_out_hc_fused && + metal_graph_directional_steering_attn_enabled(g)) { ok = metal_graph_apply_directional_steering_attn(g, metal_graph_batch_attn_out(g), il, n_tokens); } - if (ok && attn_out_f16) { + if (ok && attn_out_hc_fused) { + /* The fused output-B epilogue already wrote after_attn_hc_view. */ + } else if (ok && attn_out_f16) { ok = ds4_gpu_hc_expand_split_half_tensor(after_attn_hc_view, g->batch_q_half, metal_graph_batch_cur_hc(g), @@ -36399,6 +36474,38 @@ static void gpu_graph_report_prefill_display_progress( (int)(start + (uint32_t)done), total); } +#ifdef __APPLE__ +typedef struct { + ds4_session_progress_fn display_progress; + void *ud; + int current; + int total; + volatile uint32_t completed; +} metal_graph_flush_progress_ctx; + +/* Metal completion threads only publish readiness. User callbacks can own + * sockets, terminal state, or non-atomic session fields, so they stay on the + * graph caller thread and are emitted in monotonically increasing order. */ +static void metal_graph_flush_progress_mark(void *vctx) { + metal_graph_flush_progress_ctx *ctx = vctx; + __atomic_store_n(&ctx->completed, 1u, __ATOMIC_RELEASE); +} + +static void metal_graph_flush_progress_report_ready( + metal_graph_flush_progress_ctx *ctx, + uint32_t submitted, + uint32_t *reported) { + if (!ctx || !reported) return; + while (*reported < submitted && + __atomic_load_n(&ctx[*reported].completed, __ATOMIC_ACQUIRE)) { + metal_graph_flush_progress_ctx *ready = &ctx[*reported]; + ready->display_progress( + ready->ud, "prefill_display", ready->current, ready->total); + (*reported)++; + } +} +#endif + typedef struct { int tier; uint32_t first_layer; @@ -36759,6 +36866,16 @@ static bool metal_graph_prefill_layer_major( */ const bool throttle = graph_power_throttle_enabled(g); const bool callback_split = display_progress != NULL && n_tokens >= 32; +#ifdef __APPLE__ + /* A display-only split must preserve layer boundaries for truthful UI + * progress, but it does not need a host-side drain after every layer. */ + const bool progress_flush = + callback_split && n_tokens <= 2048u && !g->ssd_streaming && + !split_profile && !throttle && imatrix == NULL && + getenv("DS4_METAL_DISABLE_PREFILL_FLUSH_PROGRESS") == NULL; +#else + const bool progress_flush = false; +#endif const bool split_commands = g->ssd_streaming || split_profile || throttle || callback_split || n_tokens > 2048 || imatrix != NULL; @@ -36984,7 +37101,23 @@ static bool metal_graph_prefill_layer_major( return false; } +#ifdef __APPLE__ + metal_graph_flush_progress_ctx flush_ctx[DS4_N_LAYER]; + memset(flush_ctx, 0, sizeof(flush_ctx)); + uint32_t flush_submitted = 0u; + uint32_t flush_reported = 0u; + if (progress_flush && ok) { + ok = ds4_gpu_begin_commands() != 0; + } +#endif + for (uint32_t il = 0; ok && il < DS4_N_LAYER; il++) { +#ifdef __APPLE__ + if (progress_flush) { + metal_graph_flush_progress_report_ready( + flush_ctx, flush_submitted, &flush_reported); + } +#endif double layer_elapsed = 0.0; const bool layer_selected_addr = batch_selected_addr && @@ -37186,7 +37319,7 @@ static bool metal_graph_prefill_layer_major( (t_ffn_done - t_ffn_encoded) * 1000.0); } else { const double t_chunk0 = (profile || throttle) ? now_sec() : 0.0; - ok = ds4_gpu_begin_commands() != 0; + if (!progress_flush) ok = ds4_gpu_begin_commands() != 0; if (ok) ok = metal_graph_encode_layer_batch(g, model, &weights->layer[il], @@ -37213,7 +37346,31 @@ static bool metal_graph_prefill_layer_major( } #endif const double t_encoded = (profile || throttle) ? now_sec() : 0.0; - if (ok) ok = ds4_gpu_end_commands() != 0; +#ifdef __APPLE__ + if (ok && progress_flush) { + uint64_t pdone = + (uint64_t)n_tokens * (il + 1u) / (uint32_t)DS4_N_LAYER; + if (il + 1u == (uint32_t)DS4_N_LAYER) pdone = n_tokens; + flush_ctx[il] = (metal_graph_flush_progress_ctx){ + display_progress, + display_progress_ud, + (int)(start + (uint32_t)pdone), + prompt->len, + 0u, + }; + ok = ds4_gpu_flush_commands_progress( + metal_graph_flush_progress_mark, + &flush_ctx[il]) != 0; + if (ok) { + flush_submitted = il + 1u; + metal_graph_flush_progress_report_ready( + flush_ctx, flush_submitted, &flush_reported); + } + } else +#endif + if (ok) { + ok = ds4_gpu_end_commands() != 0; + } const double t_done = (profile || throttle) ? now_sec() : 0.0; #ifdef DS4_ROCM_BUILD if (ok) { @@ -37285,12 +37442,14 @@ static bool metal_graph_prefill_layer_major( return false; } graph_power_note_prefill_layer(g, il, layer_elapsed); - gpu_graph_report_prefill_display_progress(display_progress, - display_progress_ud, - start, - n_tokens, - il + 1, - prompt->len); + if (!progress_flush) { + gpu_graph_report_prefill_display_progress(display_progress, + display_progress_ud, + start, + n_tokens, + il + 1, + prompt->len); + } if (show_progress) { fprintf(stderr, "ds4: gpu prefill layer %u/%u\r", il + 1, (uint32_t)DS4_N_LAYER); fflush(stderr); @@ -37310,6 +37469,18 @@ static bool metal_graph_prefill_layer_major( } return false; } +#ifdef __APPLE__ + /* Every flush opens the next batch. If no output head follows, close the + * final empty batch and wait here so completion contexts stay alive. */ + if (progress_flush && !logits) { + ok = ds4_gpu_end_commands() != 0; + if (ok) { + metal_graph_flush_progress_report_ready( + flush_ctx, flush_submitted, &flush_reported); + ok = flush_reported == flush_submitted; + } + } +#endif #ifdef __APPLE__ /* Zero-prefix masks are shared across the 43 per-layer command batches, * then become dead weight. Release them before the output head and later @@ -37334,9 +37505,15 @@ static bool metal_graph_prefill_layer_major( if (g->ssd_streaming) ds4_gpu_release_q8_f16_cache(); #endif if (!metal_graph_seed_streaming_expert_cache_from_hotlist(g, model, weights)) { +#ifdef __APPLE__ + if (progress_flush) (void)ds4_gpu_synchronize(); +#endif return false; } if (!metal_graph_seed_streaming_expert_cache_from_prefill(g, model, weights)) { +#ifdef __APPLE__ + if (progress_flush) (void)ds4_gpu_synchronize(); +#endif return false; } @@ -37379,15 +37556,34 @@ static bool metal_graph_prefill_layer_major( } if (ok && logits) { g->cur_hc_by_tier[g->active_tier] = last_hc; - ok = ds4_gpu_begin_commands() != 0; + if (!progress_flush || !ds4_gpu_commands_active()) { + ok = ds4_gpu_begin_commands() != 0; + } } if (ok && logits) ok = metal_graph_encode_output_head(g, model, weights, weights->output->dim[1]); const double t_head_encoded = profile ? now_sec() : 0.0; if (ok && logits) ok = ds4_gpu_end_commands() != 0; +#ifdef __APPLE__ + if (progress_flush && !ds4_gpu_commands_active()) { + metal_graph_flush_progress_report_ready( + flush_ctx, flush_submitted, &flush_reported); + if (ok && flush_reported != flush_submitted) ok = false; + } +#endif const double t_head_done = profile ? now_sec() : 0.0; g->cur_hc_by_tier[g->active_tier] = saved_cur; if (last_hc) ds4_gpu_tensor_free(last_hc); - if (!ok) return false; + if (!ok) { +#ifdef __APPLE__ + /* A failed head setup/encode can leave the empty post-flush batch + * open. Close it and join completion hooks before stack contexts go + * out of scope. */ + if (progress_flush && ds4_gpu_commands_active()) { + (void)ds4_gpu_end_commands(); + } +#endif + return false; + } const double t_before_read = profile ? now_sec() : 0.0; if (logits) { diff --git a/ds4_cuda.cu b/ds4_cuda.cu index 597e0ff03..2c7a70f79 100644 --- a/ds4_cuda.cu +++ b/ds4_cuda.cu @@ -551,11 +551,12 @@ static int g_q4_attn_q_b_f16_hard_disabled; static int g_q4_attn_q_b_f16_dispatch_disabled; static int g_q4_attn_q_b_f16_pending_evict; /* The long-prefill transient path expands exactly one q_b matrix at a time. - * Keep its weight and activation staging in one dedicated allocation so it - * cannot alias the legacy global CUDA scratch used by unrelated projections. - * A single mutex covers both ownership and the complete enqueue sequence: the - * NULL stream orders GPU work, while the mutex prevents two host threads from - * interleaving dequantize -> GEMM sequences that reuse this storage. */ + * Keep its weight and activation staging, plus the optional diagnostic F16 + * projection, in one dedicated allocation so it cannot alias the legacy + * global CUDA scratch used by unrelated projections. A single mutex covers + * both ownership and the complete enqueue sequence: the decode stream orders + * GPU work, while the mutex prevents two host threads from interleaving + * sequences that reuse this storage. */ static __half *g_q4_attn_q_b_transient_f16_scratch; static uint64_t g_q4_attn_q_b_transient_f16_scratch_bytes; static int g_q4_attn_q_b_transient_f16_scratch_device = -1; @@ -2507,25 +2508,55 @@ static int cuda_q4_attn_q_b_transient_f16_disabled(void) { getenv("DS4_CUDA_DISABLE_Q4_ATTN_Q_B_TRANSIENT_F16")); } +/* Diagnostic-only until CUDA logits/acceptance testing establishes the extra + * projection rounding as an acceptable release boundary. The established + * path keeps the cuBLAS output and RMS/RoPE input in F32. */ +static int cuda_q4_attn_q_b_f16_output_requested(void) { + return cuda_env_value_enabled( + getenv("DS4_CUDA_ENABLE_Q4_ATTN_Q_B_F16_OUTPUT")); +} + static int cuda_q4_attn_q_b_transient_f16_scratch_size( uint32_t max_rows, + int include_output, uint64_t *weight_bytes, uint64_t *activation_offset, + uint64_t *output_offset, uint64_t *scratch_bytes) { const uint64_t wh_bytes = (uint64_t)CUDA_Q4_ATTN_Q_B_IN_DIM * CUDA_Q4_ATTN_Q_B_OUT_DIM * sizeof(__half); const uint64_t xh_off = (wh_bytes + 255u) & ~UINT64_C(255); - const uint64_t row_bytes = + const uint64_t xh_row_bytes = (uint64_t)CUDA_Q4_ATTN_Q_B_IN_DIM * sizeof(__half); if (max_rows == 0u || - (uint64_t)max_rows > (UINT64_MAX - xh_off) / row_bytes) { + (uint64_t)max_rows > (UINT64_MAX - xh_off) / xh_row_bytes) { + return 0; + } + const uint64_t xh_end = + xh_off + (uint64_t)max_rows * xh_row_bytes; + if (!include_output) { + if (xh_end > SIZE_MAX) return 0; + if (weight_bytes) *weight_bytes = wh_bytes; + if (activation_offset) *activation_offset = xh_off; + if (output_offset) *output_offset = 0u; + if (scratch_bytes) *scratch_bytes = xh_end; + return 1; + } + if (xh_end > UINT64_MAX - 255u) return 0; + const uint64_t qh_off = (xh_end + 255u) & ~UINT64_C(255); + const uint64_t qh_row_bytes = + (uint64_t)CUDA_Q4_ATTN_Q_B_OUT_DIM * sizeof(__half); + if ((uint64_t)max_rows > + (UINT64_MAX - qh_off) / qh_row_bytes) { return 0; } - const uint64_t total = xh_off + (uint64_t)max_rows * row_bytes; + const uint64_t total = + qh_off + (uint64_t)max_rows * qh_row_bytes; if (total > SIZE_MAX) return 0; if (weight_bytes) *weight_bytes = wh_bytes; if (activation_offset) *activation_offset = xh_off; + if (output_offset) *output_offset = qh_off; if (scratch_bytes) *scratch_bytes = total; return 1; } @@ -2613,9 +2644,12 @@ static int cuda_q4_attn_q_b_transient_f16_scratch_ensure( uint32_t max_rows, uint64_t working_set_reserve_bytes, uint64_t *prepared_bytes) { + const int include_output = + cuda_q4_attn_q_b_f16_output_requested(); uint64_t scratch_bytes = 0; if (!cuda_q4_attn_q_b_transient_f16_scratch_size( - max_rows, NULL, NULL, &scratch_bytes)) { + max_rows, include_output, + NULL, NULL, NULL, &scratch_bytes)) { return 0; } @@ -2724,8 +2758,9 @@ static int cuda_q4_attn_q_b_transient_f16_scratch_ensure( if (previous_device >= 0) (void)cudaSetDevice(previous_device); fprintf(stderr, "ds4: CUDA prepared reusable Q4 attn_q_b transient F16 scratch " - "(%.2f MiB, max batch %u tokens)\n", - (double)scratch_bytes / 1048576.0, max_rows); + "(%.2f MiB, max batch %u tokens, output %s)\n", + (double)scratch_bytes / 1048576.0, max_rows, + include_output ? "F16 diagnostic" : "F32"); return 1; } @@ -43200,10 +43235,9 @@ extern "C" int ds4_gpu_attn_q_b_f16_head_rms_rope_tail_tensor( float freq_base, float freq_scale, float ext_factor, float attn_factor, float beta_fast, float beta_slow, float eps) { /* The pre-existing CUDA Q8 specialization was a stub. Preserve that - * fallback behavior and arm only resident Q4_K. CUDA writes the GEMM - * directly to the canonical F32 graph tensor, so the Metal-only q_half - * workspace is deliberately optional here. */ - (void)q_half; + * fallback behavior and arm only resident Q4_K. The release path keeps + * its established F32 projection boundary; an explicit diagnostic can + * stage an F16 result in the resident-only arena for CUDA validation. */ if (weight_type != CUDA_Q4_ATTN_Q_B_TYPE) return 0; if (n_tok < 32u) return 0; const int required = cuda_q4_attn_q_b_f16_required(); @@ -43254,6 +43288,25 @@ extern "C" int ds4_gpu_attn_q_b_f16_head_rms_rope_tail_tensor( if (x->bytes < x_bytes || out->bytes < out_bytes) { return fallback; } + const int use_f16_output = + cuda_q4_attn_q_b_f16_output_requested(); + const uint64_t q_half_bytes = + (uint64_t)n_tok * out_dim * sizeof(__half); + int use_graph_q_half = 0; + if (use_f16_output && q_half && q_half->ptr && + q_half->bytes >= q_half_bytes && + ds4_tensor_device_idx(q_half) == 0) { + const uintptr_t q_half_addr = (uintptr_t)q_half->ptr; + const uintptr_t out_addr = (uintptr_t)out->ptr; + const uintptr_t x_addr = (uintptr_t)x->ptr; + const int overlaps_out = q_half_addr <= out_addr + ? (uint64_t)(out_addr - q_half_addr) < q_half_bytes + : (uint64_t)(q_half_addr - out_addr) < out_bytes; + const int overlaps_x = q_half_addr <= x_addr + ? (uint64_t)(x_addr - q_half_addr) < q_half_bytes + : (uint64_t)(q_half_addr - x_addr) < x_bytes; + use_graph_q_half = !overlaps_out && !overlaps_x; + } const uint64_t blocks = in_dim / CUDA_QK_K; const uint64_t row_bytes = blocks * sizeof(cuda_block_q4_K); @@ -43265,16 +43318,19 @@ extern "C" int ds4_gpu_attn_q_b_f16_head_rms_rope_tail_tensor( uint64_t transient_weight_bytes = 0; uint64_t xh_offset = 0; + uint64_t qh_offset = 0; uint64_t required_scratch_bytes = 0; if (!cuda_q4_attn_q_b_transient_f16_scratch_size( - n_tok, &transient_weight_bytes, &xh_offset, + n_tok, use_f16_output, + &transient_weight_bytes, &xh_offset, &qh_offset, &required_scratch_bytes)) { return fallback; } - /* Protect the shared W/X arena for the entire enqueue sequence. Stream - * ordering protects reuse after this function returns; host serialization - * protects the sequence itself from interleaving with another session. */ + /* Protect the shared W/X/output arena for the entire enqueue sequence. + * Stream ordering protects reuse after this function returns; host + * serialization protects the sequence itself from interleaving with + * another session. */ std::unique_lock scratch_use_lock( g_q4_attn_q_b_transient_f16_mutex); if (!g_q4_attn_q_b_transient_f16_scratch || @@ -43287,6 +43343,12 @@ extern "C" int ds4_gpu_attn_q_b_f16_head_rms_rope_tail_tensor( g_q4_attn_q_b_transient_f16_scratch; __half *const xh = reinterpret_cast<__half *>( reinterpret_cast(scratch) + xh_offset); + __half *const scratch_qh = use_f16_output + ? reinterpret_cast<__half *>( + reinterpret_cast(scratch) + qh_offset) + : NULL; + __half *const qh = use_graph_q_half + ? (__half *)q_half->ptr : scratch_qh; /* Prefer an explicitly prepared persistent sidecar. Keep its cache lock * until the epilogue is enqueued so lifecycle release cannot free the @@ -43399,7 +43461,8 @@ extern "C" int ds4_gpu_attn_q_b_f16_head_rms_rope_tail_tensor( w_f16, CUDA_R_16F, (int)in_dim, xh, CUDA_R_16F, (int)in_dim, &beta, - out->ptr, CUDA_R_32F, (int)out_dim, + use_f16_output ? (void *)qh : out->ptr, + use_f16_output ? CUDA_R_16F : CUDA_R_32F, (int)out_dim, CUDA_R_32F, CUBLAS_GEMM_DEFAULT); if (status != CUBLAS_STATUS_SUCCESS) { @@ -43418,12 +43481,21 @@ extern "C" int ds4_gpu_attn_q_b_f16_head_rms_rope_tail_tensor( /* cuBLAS has accepted the first output writer. From this point onward a * native-Q4 replay is unsafe even when the specialization was optional. */ - head_rms_norm_rope_tail_kernel<<< - n_tok * n_head, 256, 0, stream>>>( - (float *)out->ptr, - n_tok, n_head, head_dim, n_rot, pos0, n_ctx_orig, - inverse ? 1 : 0, freq_base, freq_scale, ext_factor, - attn_factor, beta_fast, beta_slow, eps); + if (use_f16_output) { + head_rms_norm_rope_tail_from_half_kernel<<< + n_tok * n_head, 256, 0, stream>>>( + (float *)out->ptr, qh, + n_tok, n_head, head_dim, n_rot, pos0, n_ctx_orig, + inverse ? 1 : 0, freq_base, freq_scale, ext_factor, + attn_factor, beta_fast, beta_slow, eps); + } else { + head_rms_norm_rope_tail_kernel<<< + n_tok * n_head, 256, 0, stream>>>( + (float *)out->ptr, + n_tok, n_head, head_dim, n_rot, pos0, n_ctx_orig, + inverse ? 1 : 0, freq_base, freq_scale, ext_factor, + attn_factor, beta_fast, beta_slow, eps); + } launch_err = cudaGetLastError(); if (launch_err != cudaSuccess) { fprintf(stderr, diff --git a/ds4_gpu.h b/ds4_gpu.h index c33541792..af30002e4 100644 --- a/ds4_gpu.h +++ b/ds4_gpu.h @@ -83,6 +83,12 @@ int ds4_gpu_pack_slot_rows_f32_tensor( int ds4_gpu_begin_commands(void); int ds4_gpu_flush_encoder(void); int ds4_gpu_flush_commands(void); +#ifdef __APPLE__ +/* Commit the current Metal batch without draining. The completion hook runs + * on a Metal-owned thread and must only publish thread-safe readiness state; + * the next full command drain joins it before returning. */ +int ds4_gpu_flush_commands_progress(void (*report)(void *ctx), void *ctx); +#endif int ds4_gpu_commands_active(void); #ifdef __APPLE__ int ds4_gpu_parallel_ffn_finish(void); @@ -368,6 +374,8 @@ enum { DS4_GPU_TEST_STREAMING_LIVE_INDEX_FAILURE = 1u << 7, DS4_GPU_TEST_IQ2_SSD_GROUPED_PIPELINE_FAILURE = 1u << 8, DS4_GPU_TEST_ATTN_OUT_LOW_Q8_STATIC = 1u << 9, + DS4_GPU_TEST_BATCH_ATTN_OUT_Q8_HC_FUSION = 1u << 10, + DS4_GPU_TEST_BATCH_ATTN_OUT_Q4_HC_FUSION = 1u << 11, }; void ds4_gpu_test_set_flags(uint32_t flags); void ds4_gpu_release_zero_prefix_prefill_mask_cache(void); @@ -2710,6 +2718,30 @@ int ds4_gpu_attention_output_q8_batch_tensor( uint64_t out_dim, const ds4_gpu_tensor *heads, uint32_t n_tokens); +#ifdef __APPLE__ +/* Optional resident Metal output-B + HC4 epilogues. Return 1 when fused work + * was encoded, 0 before writing anything when ineligible, and -1 after an + * attempted-path failure (the caller must not replay the fallback then). */ +int ds4_gpu_attention_output_q8_batch_hc_tensor( + ds4_gpu_tensor *out, + ds4_gpu_tensor *out_hc, + const ds4_gpu_tensor *residual_hc, + const ds4_gpu_tensor *split, + ds4_gpu_tensor *low, + ds4_gpu_tensor *group_tmp, + ds4_gpu_tensor *low_tmp, + const void *model_map, + uint64_t model_size, + uint64_t out_a_offset, + uint64_t out_b_offset, + uint64_t group_dim, + uint64_t rank, + uint32_t n_groups, + uint64_t out_dim, + const ds4_gpu_tensor *heads, + uint32_t n_tokens, + uint32_t n_hc); +#endif /* Returns 1 when the batch path ran, 0 for the ordinary row fallback, and -1 * for a post-enqueue failure or a backend REQUIRE diagnostic. The caller * must not retry the row fallback after -1. */ @@ -2729,6 +2761,28 @@ int ds4_gpu_attention_output_q4_K_batch_tensor( uint64_t out_dim, const ds4_gpu_tensor *heads, uint32_t n_tokens); +#ifdef __APPLE__ +int ds4_gpu_attention_output_q4_K_batch_hc_tensor( + ds4_gpu_tensor *out, + ds4_gpu_tensor *out_hc, + const ds4_gpu_tensor *residual_hc, + const ds4_gpu_tensor *split, + ds4_gpu_tensor *low, + ds4_gpu_tensor *group_tmp, + ds4_gpu_tensor *low_tmp, + const void *model_map, + uint64_t model_size, + uint64_t out_a_offset, + uint64_t out_b_offset, + uint32_t out_b_type, + uint64_t group_dim, + uint64_t rank, + uint32_t n_groups, + uint64_t out_dim, + const ds4_gpu_tensor *heads, + uint32_t n_tokens, + uint32_t n_hc); +#endif int ds4_gpu_attention_output_q8_batch_f16_tensor( ds4_gpu_tensor *out_h, diff --git a/ds4_metal.m b/ds4_metal.m index 2926f47c6..bcdf95eeb 100644 --- a/ds4_metal.m +++ b/ds4_metal.m @@ -1,5 +1,6 @@ #import #import +#import #include #include @@ -68,6 +69,10 @@ static NSMutableArray> *g_pending_cbs_by_stream[DS4_GPU_MAX_STREAMS]; #define g_pending_cbs (g_pending_cbs_by_stream[g_ds4_stream]) +/* Completion handlers may still be running after waitUntilCompleted returns. + * Track the small progress hooks explicitly so stack-backed hook contexts can + * be joined before a command drain returns to the graph driver. */ +static dispatch_group_t g_progress_completion_group; static id g_selected_readback_event; static uint64_t g_selected_readback_event_value; static id g_set_rows_f32_i32_pipeline; @@ -279,6 +284,8 @@ static id g_glm_q6_k_down_f32_pipeline; static id g_dsv4_router_weights_batch_pipeline; static id g_dsv4_hc_expand4_pipeline; +static id g_dsv4_attn_out_q8_mm_hc_expand4_pipeline; +static id g_dsv4_attn_out_q4_mm_hc_expand4_pipeline; static NSMutableDictionary> *g_pipeline_cache; static uint64_t g_pipeline_cache_generation; @@ -1666,6 +1673,10 @@ static int ds4_gpu_wait_pending_command_buffers_for_stream( if (!ds4_gpu_wait_command_buffer(pending, label)) ok = 0; } [g_pending_cbs_by_stream[idx] removeAllObjects]; + if (g_progress_completion_group) { + dispatch_group_wait(g_progress_completion_group, + DISPATCH_TIME_FOREVER); + } ds4_gpu_stream_expert_cache_note_pending_completed(); if (!ok) ds4_gpu_invalidate_zero_prefix_prefill_block_maps(); return ok; @@ -7256,6 +7267,9 @@ int ds4_gpu_init(void) { if (g_pipeline_cache_generation == 0) g_pipeline_cache_generation++; g_dsv4_completion_cache = [NSCache new]; g_dsv4_completion_cache.countLimit = 256u; + if (!g_progress_completion_group) { + g_progress_completion_group = dispatch_group_create(); + } for (int si = 0; si < DS4_GPU_MAX_STREAMS; si++) { g_transient_buffers_by_stream[si] = [NSMutableArray array]; g_pending_cbs_by_stream[si] = [NSMutableArray array]; @@ -7263,6 +7277,7 @@ int ds4_gpu_init(void) { if (!g_model_buffer_cache || !g_q4_expert_table_cache || !g_q4_expert_layer_residency_cache || !g_pipeline_cache || !g_dsv4_completion_cache || + !g_progress_completion_group || !g_transient_buffers_by_stream[0] || !g_pending_cbs_by_stream[0]) { fprintf(stderr, "ds4: Metal bookkeeping allocation failed\n"); @@ -9506,6 +9521,12 @@ int ds4_gpu_init(void) { ds4_gpu_get_pipeline("kernel_dsv4_router_weights_batch"); g_dsv4_hc_expand4_pipeline = ds4_gpu_get_pipeline("kernel_dsv4_hc_expand4"); + g_dsv4_attn_out_q8_mm_hc_expand4_pipeline = + ds4_gpu_get_pipeline( + "kernel_dsv4_attn_out_q8_mm_hc_expand4_batch"); + g_dsv4_attn_out_q4_mm_hc_expand4_pipeline = + ds4_gpu_get_pipeline( + "kernel_dsv4_attn_out_q4_K_f16_rhs_mm_hc_expand4_batch"); if (!g_dsv4_indexer_score_one_direct_pipeline || !g_dsv4_compressor_store_one_pipeline || !g_dsv4_sort_i32_rows_asc_pipeline || @@ -10318,6 +10339,41 @@ int ds4_gpu_flush_commands(void) { return 1; } +/* Commit the current batch without draining the GPU. `report` is a backend + * completion hook, not a user callback: it must only publish thread-safe + * state. A later full drain joins this handler explicitly. */ +int ds4_gpu_flush_commands_progress(void (*report)(void *ctx), void *ctx) { + if (!g_initialized && !ds4_gpu_init()) return 0; + ds4_gpu_parallel_ffn_reset_state(YES); + if (!g_batch_cb || (report && !g_progress_completion_group)) return 0; + + ds4_gpu_close_batch_encoder(); + id cb = g_batch_cb; + g_batch_cb = nil; + g_batch_has_work = NO; + if (report) { + dispatch_group_enter(g_progress_completion_group); + [cb addCompletedHandler:^(id done_cb) { + (void)done_cb; + report(ctx); + dispatch_group_leave(g_progress_completion_group); + }]; + } + [cb commit]; + [g_pending_cbs addObject:cb]; + ds4_gpu_stream_expert_cache_note_batch_committed(); + + g_batch_cb = ds4_gpu_new_command_buffer(); + g_batch_has_work = NO; + if (g_batch_cb) ds4_gpu_stream_expert_cache_note_batch_created(); + if (!g_batch_cb) { + (void)ds4_gpu_wait_pending_command_buffers("command batch"); + [g_transient_buffers removeAllObjects]; + return 0; + } + return 1; +} + int ds4_gpu_commands_active(void) { return g_batch_cb != nil; } @@ -11763,6 +11819,8 @@ void ds4_gpu_cleanup(void) { g_glm_q6_k_down_f32_pipeline = nil; g_dsv4_router_weights_batch_pipeline = nil; g_dsv4_hc_expand4_pipeline = nil; + g_dsv4_attn_out_q8_mm_hc_expand4_pipeline = nil; + g_dsv4_attn_out_q4_mm_hc_expand4_pipeline = nil; ds4_gpu_clear_zero_prefix_prefill_mask_cache(); g_stream_expert_validate_status_buffer = nil; for (int si = 0; si < DS4_GPU_MAX_STREAMS; si++) { @@ -29875,7 +29933,169 @@ static int ds4_gpu_encode_fill_f32_rows( return 1; } -int ds4_gpu_attention_output_q8_batch_tensor( +typedef struct { + ds4_gpu_tensor *out_hc; + const ds4_gpu_tensor *residual_hc; + const ds4_gpu_tensor *split; + id __strong weight_buffer; + NSUInteger weight_offset; +} ds4_gpu_attn_out_hc_target; + +static int ds4_gpu_encode_attn_out_q8_mm_hc( + id cb, + const ds4_gpu_attn_out_hc_target *target, + const ds4_gpu_tensor *low, + uint32_t n_tokens) { + if (!cb || !target || !target->out_hc || !target->residual_hc || + !target->split || !target->weight_buffer || !low || n_tokens == 0u) { + return 0; + } + + id lowbuf = ds4_gpu_tensor_buffer(low); + id resbuf = ds4_gpu_tensor_buffer(target->residual_hc); + id splitbuf = ds4_gpu_tensor_buffer(target->split); + id outbuf = ds4_gpu_tensor_buffer(target->out_hc); + if (!lowbuf || !resbuf || !splitbuf || !outbuf || + !g_dsv4_attn_out_q8_mm_hc_expand4_pipeline) { + return 0; + } + + const uint32_t in_dim = 8192u; + const uint32_t out_dim = 4096u; + const uint32_t n_hc = 4u; + const uint64_t row_bytes = (uint64_t)(in_dim / 32u) * 34u; + ds4_gpu_mul_mm_args mm = + ds4_gpu_make_mm_args(in_dim, out_dim, n_tokens, row_bytes); + const uint64_t mix_hc = 2ull * n_hc + (uint64_t)n_hc * n_hc; + ds4_gpu_hc_expand_args hc = { + .n_embd = out_dim, + .n_hc = n_hc, + .n_tokens = (int64_t)n_tokens, + .nb_block0 = sizeof(float), + .nb_block1 = (uint64_t)out_dim * sizeof(float), + .nb_add0 = sizeof(float), + .nb_add1 = (uint64_t)out_dim * sizeof(float), + .nb_res0 = sizeof(float), + .nb_res1 = (uint64_t)out_dim * sizeof(float), + .nb_res2 = (uint64_t)n_hc * out_dim * sizeof(float), + .nb_post0 = sizeof(float), + .nb_post1 = mix_hc * sizeof(float), + .nb_comb0 = sizeof(float), + .nb_comb1 = (uint64_t)n_hc * sizeof(float), + .nb_comb2 = mix_hc * sizeof(float), + .nb0 = sizeof(float), + .nb1 = (uint64_t)out_dim * sizeof(float), + .nb2 = (uint64_t)n_hc * out_dim * sizeof(float), + .has_add = 0, + }; + + id enc = ds4_gpu_compute_encoder(cb); + if (!enc) return 0; + [enc setComputePipelineState:g_dsv4_attn_out_q8_mm_hc_expand4_pipeline]; + [enc setBytes:&mm length:sizeof(mm) atIndex:0]; + [enc setBuffer:target->weight_buffer offset:target->weight_offset atIndex:1]; + [enc setBuffer:lowbuf offset:ds4_gpu_tensor_offset(low) atIndex:2]; + [enc setBuffer:resbuf offset:ds4_gpu_tensor_offset(target->residual_hc) atIndex:3]; + [enc setBuffer:splitbuf + offset:ds4_gpu_tensor_offset(target->split) + + (NSUInteger)n_hc * sizeof(float) + atIndex:4]; + [enc setBuffer:splitbuf + offset:ds4_gpu_tensor_offset(target->split) + + (NSUInteger)(2u * n_hc) * sizeof(float) + atIndex:5]; + [enc setBuffer:outbuf offset:ds4_gpu_tensor_offset(target->out_hc) atIndex:6]; + [enc setBytes:&hc length:sizeof(hc) atIndex:7]; + [enc setThreadgroupMemoryLength:8192u atIndex:0]; + [enc dispatchThreadgroups:MTLSizeMake((NSUInteger)n_tokens / 32u, + (NSUInteger)out_dim / 64u, + 1u) + threadsPerThreadgroup:MTLSizeMake(128u, 1u, 1u)]; + ds4_gpu_end_compute_encoder(cb, enc); + return 1; +} + +static int ds4_gpu_encode_attn_out_q4_mm_hc( + id cb, + const ds4_gpu_attn_out_hc_target *target, + const ds4_gpu_tensor *rhs_f16, + uint32_t n_tokens) { + if (!cb || !target || !target->out_hc || !target->residual_hc || + !target->split || !target->weight_buffer || !rhs_f16 || + n_tokens == 0u || (n_tokens % 32u) != 0u) { + return 0; + } + + id rhsbuf = ds4_gpu_tensor_buffer(rhs_f16); + id resbuf = ds4_gpu_tensor_buffer(target->residual_hc); + id splitbuf = ds4_gpu_tensor_buffer(target->split); + id outbuf = ds4_gpu_tensor_buffer(target->out_hc); + if (!rhsbuf || !resbuf || !splitbuf || !outbuf || + !g_dsv4_attn_out_q4_mm_hc_expand4_pipeline) { + return 0; + } + + const uint32_t in_dim = 8192u; + const uint32_t out_dim = 4096u; + const uint32_t n_hc = 4u; + const uint64_t row_bytes = (uint64_t)(in_dim / 256u) * 144u; + ds4_gpu_mul_mm_args mm = + ds4_gpu_make_mm_args(in_dim, out_dim, n_tokens, row_bytes); + mm.nb10 = sizeof(uint16_t); + mm.nb11 = (uint64_t)in_dim * sizeof(uint16_t); + mm.nb12 = (uint64_t)in_dim * n_tokens * sizeof(uint16_t); + mm.nb13 = mm.nb12; + + const uint64_t mix_hc = 2ull * n_hc + (uint64_t)n_hc * n_hc; + ds4_gpu_hc_expand_args hc = { + .n_embd = out_dim, + .n_hc = n_hc, + .n_tokens = (int64_t)n_tokens, + .nb_block0 = sizeof(float), + .nb_block1 = (uint64_t)out_dim * sizeof(float), + .nb_add0 = sizeof(float), + .nb_add1 = (uint64_t)out_dim * sizeof(float), + .nb_res0 = sizeof(float), + .nb_res1 = (uint64_t)out_dim * sizeof(float), + .nb_res2 = (uint64_t)n_hc * out_dim * sizeof(float), + .nb_post0 = sizeof(float), + .nb_post1 = mix_hc * sizeof(float), + .nb_comb0 = sizeof(float), + .nb_comb1 = (uint64_t)n_hc * sizeof(float), + .nb_comb2 = mix_hc * sizeof(float), + .nb0 = sizeof(float), + .nb1 = (uint64_t)out_dim * sizeof(float), + .nb2 = (uint64_t)n_hc * out_dim * sizeof(float), + .has_add = 0, + }; + + id enc = ds4_gpu_compute_encoder(cb); + if (!enc) return 0; + [enc setComputePipelineState:g_dsv4_attn_out_q4_mm_hc_expand4_pipeline]; + [enc setBytes:&mm length:sizeof(mm) atIndex:0]; + [enc setBuffer:target->weight_buffer offset:target->weight_offset atIndex:1]; + [enc setBuffer:rhsbuf offset:ds4_gpu_tensor_offset(rhs_f16) atIndex:2]; + [enc setBuffer:resbuf offset:ds4_gpu_tensor_offset(target->residual_hc) atIndex:3]; + [enc setBuffer:splitbuf + offset:ds4_gpu_tensor_offset(target->split) + + (NSUInteger)n_hc * sizeof(float) + atIndex:4]; + [enc setBuffer:splitbuf + offset:ds4_gpu_tensor_offset(target->split) + + (NSUInteger)(2u * n_hc) * sizeof(float) + atIndex:5]; + [enc setBuffer:outbuf offset:ds4_gpu_tensor_offset(target->out_hc) atIndex:6]; + [enc setBytes:&hc length:sizeof(hc) atIndex:7]; + [enc setThreadgroupMemoryLength:8192u atIndex:0]; + [enc dispatchThreadgroups:MTLSizeMake((NSUInteger)n_tokens / 32u, + (NSUInteger)out_dim / 64u, + 1u) + threadsPerThreadgroup:MTLSizeMake(128u, 1u, 1u)]; + ds4_gpu_end_compute_encoder(cb, enc); + return 1; +} + +static int ds4_gpu_attention_output_q8_batch_impl( ds4_gpu_tensor *out, ds4_gpu_tensor *low, ds4_gpu_tensor *group_tmp, @@ -29889,7 +30109,8 @@ int ds4_gpu_attention_output_q8_batch_tensor( uint32_t n_groups, uint64_t out_dim, const ds4_gpu_tensor *heads, - uint32_t n_tokens) { + uint32_t n_tokens, + const ds4_gpu_attn_out_hc_target *hc_target) { if (!g_initialized && !ds4_gpu_init()) return 0; if (!out || !low || !group_tmp || !low_tmp || !heads || !model_map || group_dim == 0 || rank == 0 || n_groups == 0 || out_dim == 0 || n_tokens == 0 || @@ -30225,9 +30446,14 @@ int ds4_gpu_attention_output_q8_batch_tensor( DS4_METAL_PROFILE_ATTN_OUT_STAGE("low_proj"); if (ok) { - ok = ds4_gpu_matmul_q8_0_tensor(out, model_map, model_size, - out_b_offset, - low_dim, out_dim, low, n_tokens) != 0; + if (hc_target) { + ok = ds4_gpu_encode_attn_out_q8_mm_hc( + cb, hc_target, low, n_tokens) != 0; + } else { + ok = ds4_gpu_matmul_q8_0_tensor( + out, model_map, model_size, out_b_offset, + low_dim, out_dim, low, n_tokens) != 0; + } } DS4_METAL_PROFILE_ATTN_OUT_STAGE("out_proj"); @@ -30239,6 +30465,150 @@ int ds4_gpu_attention_output_q8_batch_tensor( } } +int ds4_gpu_attention_output_q8_batch_tensor( + ds4_gpu_tensor *out, + ds4_gpu_tensor *low, + ds4_gpu_tensor *group_tmp, + ds4_gpu_tensor *low_tmp, + const void *model_map, + uint64_t model_size, + uint64_t out_a_offset, + uint64_t out_b_offset, + uint64_t group_dim, + uint64_t rank, + uint32_t n_groups, + uint64_t out_dim, + const ds4_gpu_tensor *heads, + uint32_t n_tokens) { + return ds4_gpu_attention_output_q8_batch_impl( + out, low, group_tmp, low_tmp, model_map, model_size, + out_a_offset, out_b_offset, group_dim, rank, n_groups, out_dim, + heads, n_tokens, NULL); +} + +/* Return 0 before encoding when the exact resident fast path is unavailable, + * allowing the caller to run the established output-B + HC fallback. Once + * selected, an encode/submit failure returns -1 so work is never replayed over + * a partially written HC destination. */ +int ds4_gpu_attention_output_q8_batch_hc_tensor( + ds4_gpu_tensor *out, + ds4_gpu_tensor *out_hc, + const ds4_gpu_tensor *residual_hc, + const ds4_gpu_tensor *split, + ds4_gpu_tensor *low, + ds4_gpu_tensor *group_tmp, + ds4_gpu_tensor *low_tmp, + const void *model_map, + uint64_t model_size, + uint64_t out_a_offset, + uint64_t out_b_offset, + uint64_t group_dim, + uint64_t rank, + uint32_t n_groups, + uint64_t out_dim, + const ds4_gpu_tensor *heads, + uint32_t n_tokens, + uint32_t n_hc) { + if (!g_initialized && !ds4_gpu_init()) return -1; + const bool force = + (g_test_flags & DS4_GPU_TEST_BATCH_ATTN_OUT_Q8_HC_FUSION) != 0u; + const bool require = + getenv("DS4_METAL_REQUIRE_PRE_M5_BATCH_ATTN_OUT_HC_FUSION") != NULL; + if (group_dim != 4096u || rank != 1024u || n_groups != 8u || + out_dim != 4096u || n_hc != 4u || + n_tokens < 32u || (n_tokens < 512u && !force) || + n_tokens > 4096u || (n_tokens % 32u) != 0u) { + return require ? -1 : 0; + } + + const NSUInteger dynamic_bytes = 8192u; + const NSUInteger static_bytes = + g_dsv4_attn_out_q8_mm_hc_expand4_pipeline + ? g_dsv4_attn_out_q8_mm_hc_expand4_pipeline + .staticThreadgroupMemoryLength + : 0u; + const NSUInteger max_tg_bytes = + g_device ? g_device.maxThreadgroupMemoryLength : 0u; + const bool pipeline_ok = + g_dsv4_attn_out_q8_mm_hc_expand4_pipeline != nil && + g_dsv4_attn_out_q8_mm_hc_expand4_pipeline.threadExecutionWidth == 32u && + g_dsv4_attn_out_q8_mm_hc_expand4_pipeline + .maxTotalThreadsPerThreadgroup >= 128u && + static_bytes <= max_tg_bytes && + dynamic_bytes <= max_tg_bytes - static_bytes; + const uint64_t low_dim = (uint64_t)n_groups * rank; + const uint64_t row_a_bytes = group_dim / 32u * 34u; + const uint64_t row_b_bytes = low_dim / 32u * 34u; + const uint64_t out_a_bytes = (uint64_t)n_groups * rank * row_a_bytes; + const uint64_t out_b_bytes = out_dim * row_b_bytes; + const uint64_t heads_bytes = + (uint64_t)n_tokens * n_groups * group_dim * sizeof(float); + const uint64_t low_bytes = + (uint64_t)n_tokens * low_dim * sizeof(float); + const uint64_t out_bytes = + (uint64_t)n_tokens * out_dim * sizeof(float); + const uint64_t hc_bytes = + (uint64_t)n_tokens * n_hc * out_dim * sizeof(float); + const uint64_t split_bytes = + (uint64_t)n_tokens * (2ull * n_hc + (uint64_t)n_hc * n_hc) * + sizeof(float); + bool eligible = + out && out_hc && residual_hc && split && low && group_tmp && low_tmp && + heads && model_map && + (ds4_gpu_device_is_pre_m5_apple_silicon() || force) && + (!g_quality_mode || force) && + !g_ssd_streaming_mode && !ds4_gpu_tp_world_is_two() && + getenv("DS4_METAL_DISABLE_PRE_M5_BATCH_ATTN_OUT_HC_FUSION") == NULL && + getenv("DS4_METAL_ATTN_OUT_STAGE_PROFILE") == NULL && + getenv("DS4_METAL_Q8_PREFILL_PROFILE") == NULL && + pipeline_ok && low_dim == 8192u && + out_a_offset <= model_size && + out_a_bytes <= model_size - out_a_offset && + out_b_offset <= model_size && + out_b_bytes <= model_size - out_b_offset && + ds4_gpu_tensor_buffer(out) != nil && + ds4_gpu_tensor_buffer(out_hc) != nil && + ds4_gpu_tensor_buffer(residual_hc) != nil && + ds4_gpu_tensor_buffer(split) != nil && + ds4_gpu_tensor_buffer(low) != nil && + ds4_gpu_tensor_buffer(heads) != nil && + ds4_gpu_tensor_bytes(out) >= out_bytes && + ds4_gpu_tensor_bytes(out_hc) >= hc_bytes && + ds4_gpu_tensor_bytes(residual_hc) >= hc_bytes && + ds4_gpu_tensor_bytes(split) >= split_bytes && + ds4_gpu_tensor_bytes(low) >= low_bytes && + ds4_gpu_tensor_bytes(heads) >= heads_bytes; + + uint64_t out_b_inner = 0u; + id out_b_buf = nil; + if (eligible) { + out_b_buf = ds4_gpu_wrap_model_range( + model_map, model_size, out_b_offset, out_b_bytes, &out_b_inner); + eligible = out_b_buf != nil && out_b_inner <= NSUIntegerMax; + } + if (!eligible) { + if (require) { + fprintf(stderr, + "ds4: required Metal Q8 batch attention-output HC " + "fusion was not selected\n"); + return -1; + } + return 0; + } + + ds4_gpu_attn_out_hc_target target = { + .out_hc = out_hc, + .residual_hc = residual_hc, + .split = split, + .weight_buffer = out_b_buf, + .weight_offset = (NSUInteger)out_b_inner, + }; + return ds4_gpu_attention_output_q8_batch_impl( + out, low, group_tmp, low_tmp, model_map, model_size, + out_a_offset, out_b_offset, group_dim, rank, n_groups, out_dim, + heads, n_tokens, &target) ? 1 : -1; +} + static int ds4_gpu_attention_output_q4_K_ssd_prefill_exactn_tensor( ds4_gpu_tensor *out, ds4_gpu_tensor *low, @@ -30560,7 +30930,7 @@ static int ds4_gpu_attention_output_q4_K_ssd_prefill_exactn_tensor( } } -int ds4_gpu_attention_output_q4_K_batch_tensor( +static int ds4_gpu_attention_output_q4_K_batch_impl( ds4_gpu_tensor *out, ds4_gpu_tensor *low, ds4_gpu_tensor *group_tmp, @@ -30575,8 +30945,12 @@ int ds4_gpu_attention_output_q4_K_batch_tensor( uint32_t n_groups, uint64_t out_dim, const ds4_gpu_tensor *heads, - uint32_t n_tokens) { + uint32_t n_tokens, + const ds4_gpu_attn_out_hc_target *hc_target) { + const bool force_f16_rhs = + (g_test_flags & DS4_GPU_TEST_BATCH_ATTN_OUT_Q4_HC_FUSION) != 0u; const bool require_f16_rhs = + hc_target != NULL || ds4_gpu_env_bool("DS4_METAL_REQUIRE_Q4_ATTN_OUT_B_F16_RHS") == 1; const bool disable_f16_rhs = ds4_gpu_env_bool("DS4_METAL_DISABLE_Q4_ATTN_OUT_B_F16_RHS") == 1; @@ -30728,7 +31102,7 @@ int ds4_gpu_attention_output_q4_K_batch_tensor( bool use_f16_rhs = !g_ssd_streaming_mode && !disable_f16_rhs && - ds4_gpu_device_is_pre_m5_apple_silicon() && + (ds4_gpu_device_is_pre_m5_apple_silicon() || force_f16_rhs) && !g_batch_encoder_concurrent && out_b_type == DS4_METAL_TENSOR_Q4_K && (n_tokens % 32u) == 0 && @@ -30739,11 +31113,18 @@ int ds4_gpu_attention_output_q4_K_batch_tensor( id out_b_buf = nil; uint64_t out_b_inner = 0; if (use_f16_rhs) { - f16_rhs_pipeline = ds4_gpu_get_mul_mm_pipeline( - "kernel_mul_mm_q4_K_f16_rhs", false, f16_rhs_bc_out); - out_b_buf = ds4_gpu_wrap_model_range( - model_map, model_size, out_b_offset, out_b_bytes, - &out_b_inner); + f16_rhs_pipeline = hc_target + ? g_dsv4_attn_out_q4_mm_hc_expand4_pipeline + : ds4_gpu_get_mul_mm_pipeline( + "kernel_mul_mm_q4_K_f16_rhs", false, f16_rhs_bc_out); + if (hc_target) { + out_b_buf = hc_target->weight_buffer; + out_b_inner = hc_target->weight_offset; + } else { + out_b_buf = ds4_gpu_wrap_model_range( + model_map, model_size, out_b_offset, out_b_bytes, + &out_b_inner); + } if (!f16_rhs_pipeline || !out_b_buf || out_b_inner > NSUIntegerMax) { use_f16_rhs = false; } @@ -30926,25 +31307,28 @@ int ds4_gpu_attention_output_q4_K_batch_tensor( if (ok) { if (use_f16_rhs) { const bool encoded_f16_rhs = - ds4_gpu_encode_cpy_f32_f16_1d( + ds4_gpu_encode_cpy_f32_f16_1d( cb, ds4_gpu_tensor_buffer(low), ds4_gpu_tensor_offset(low), ds4_gpu_tensor_buffer(group_tmp), ds4_gpu_tensor_offset(group_tmp), (uint32_t)f16_rhs_elements) != 0 && - ds4_gpu_encode_f16_rhs_mm( - cb, - f16_rhs_pipeline, - out_b_buf, - (NSUInteger)out_b_inner, - group_tmp, - out, - low_dim, - out_dim, - n_tokens, - row_b_bytes, - f16_rhs_bc_out) != 0; + (hc_target + ? ds4_gpu_encode_attn_out_q4_mm_hc( + cb, hc_target, group_tmp, n_tokens) != 0 + : ds4_gpu_encode_f16_rhs_mm( + cb, + f16_rhs_pipeline, + out_b_buf, + (NSUInteger)out_b_inner, + group_tmp, + out, + low_dim, + out_dim, + n_tokens, + row_b_bytes, + f16_rhs_bc_out) != 0); if (!encoded_f16_rhs) { ok = require_f16_rhs ? false @@ -30999,6 +31383,170 @@ int ds4_gpu_attention_output_q4_K_batch_tensor( } } +int ds4_gpu_attention_output_q4_K_batch_tensor( + ds4_gpu_tensor *out, + ds4_gpu_tensor *low, + ds4_gpu_tensor *group_tmp, + ds4_gpu_tensor *low_tmp, + const void *model_map, + uint64_t model_size, + uint64_t out_a_offset, + uint64_t out_b_offset, + uint32_t out_b_type, + uint64_t group_dim, + uint64_t rank, + uint32_t n_groups, + uint64_t out_dim, + const ds4_gpu_tensor *heads, + uint32_t n_tokens) { + return ds4_gpu_attention_output_q4_K_batch_impl( + out, low, group_tmp, low_tmp, model_map, model_size, + out_a_offset, out_b_offset, out_b_type, group_dim, rank, + n_groups, out_dim, heads, n_tokens, NULL); +} + +/* Like the Q8 resident tail, return zero only while the established output-B + * plus standalone-HC fallback is still safe to run. Once selected, failures + * are fatal to the candidate because the batch may contain partial writes. */ +int ds4_gpu_attention_output_q4_K_batch_hc_tensor( + ds4_gpu_tensor *out, + ds4_gpu_tensor *out_hc, + const ds4_gpu_tensor *residual_hc, + const ds4_gpu_tensor *split, + ds4_gpu_tensor *low, + ds4_gpu_tensor *group_tmp, + ds4_gpu_tensor *low_tmp, + const void *model_map, + uint64_t model_size, + uint64_t out_a_offset, + uint64_t out_b_offset, + uint32_t out_b_type, + uint64_t group_dim, + uint64_t rank, + uint32_t n_groups, + uint64_t out_dim, + const ds4_gpu_tensor *heads, + uint32_t n_tokens, + uint32_t n_hc) { + if (!g_initialized && !ds4_gpu_init()) return -1; + const bool force = + (g_test_flags & DS4_GPU_TEST_BATCH_ATTN_OUT_Q4_HC_FUSION) != 0u; + const bool require = + getenv("DS4_METAL_REQUIRE_Q4_BATCH_ATTN_OUT_HC_FUSION") != NULL; + if (out_b_type != DS4_METAL_TENSOR_Q4_K || group_dim != 4096u || + rank != 1024u || n_groups != 8u || out_dim != 4096u || + n_hc != 4u || n_tokens < 32u || + (n_tokens < 512u && !force) || + n_tokens > 4096u || (n_tokens % 32u) != 0u) { + return require ? -1 : 0; + } + + const NSUInteger dynamic_bytes = 8192u; + const NSUInteger static_bytes = + g_dsv4_attn_out_q4_mm_hc_expand4_pipeline + ? g_dsv4_attn_out_q4_mm_hc_expand4_pipeline + .staticThreadgroupMemoryLength + : 0u; + const NSUInteger max_tg_bytes = + g_device ? g_device.maxThreadgroupMemoryLength : 0u; + const bool pipeline_ok = + g_dsv4_attn_out_q4_mm_hc_expand4_pipeline != nil && + g_dsv4_attn_out_q4_mm_hc_expand4_pipeline.threadExecutionWidth == 32u && + g_dsv4_attn_out_q4_mm_hc_expand4_pipeline + .maxTotalThreadsPerThreadgroup >= 128u && + static_bytes <= max_tg_bytes && + dynamic_bytes <= max_tg_bytes - static_bytes; + const uint64_t low_dim = (uint64_t)n_groups * rank; + const uint64_t row_a_bytes = group_dim / 256u * 144u; + const uint64_t row_b_bytes = low_dim / 256u * 144u; + const uint64_t out_a_bytes = low_dim * row_a_bytes; + const uint64_t out_b_bytes = out_dim * row_b_bytes; + const uint64_t heads_bytes = + (uint64_t)n_tokens * n_groups * group_dim * sizeof(float); + const uint64_t low_bytes = + (uint64_t)n_tokens * low_dim * sizeof(float); + const uint64_t rhs_f16_bytes = + (uint64_t)n_tokens * low_dim * sizeof(uint16_t); + const uint64_t out_bytes = + (uint64_t)n_tokens * out_dim * sizeof(float); + const uint64_t hc_bytes = + (uint64_t)n_tokens * n_hc * out_dim * sizeof(float); + const uint64_t split_bytes = + (uint64_t)n_tokens * (2ull * n_hc + (uint64_t)n_hc * n_hc) * + sizeof(float); + bool eligible = + out && out_hc && residual_hc && split && low && group_tmp && low_tmp && + heads && model_map && + (ds4_gpu_device_is_pre_m5_apple_silicon() || force) && + (!g_quality_mode || force) && + !g_ssd_streaming_mode && !ds4_gpu_tp_world_is_two() && + !g_batch_encoder_concurrent && + getenv("DS4_METAL_DISABLE_PRE_M5_BATCH_ATTN_OUT_HC_FUSION") == NULL && + getenv("DS4_METAL_DISABLE_Q4_ATTN_OUT_B_F16_RHS") == NULL && + getenv("DS4_METAL_ATTN_OUT_STAGE_PROFILE") == NULL && + pipeline_ok && low_dim == 8192u && + out_a_offset <= model_size && + out_a_bytes <= model_size - out_a_offset && + out_b_offset <= model_size && + out_b_bytes <= model_size - out_b_offset && + ds4_gpu_tensor_buffer(out) != nil && + ds4_gpu_tensor_buffer(out_hc) != nil && + ds4_gpu_tensor_buffer(residual_hc) != nil && + ds4_gpu_tensor_buffer(split) != nil && + ds4_gpu_tensor_buffer(low) != nil && + ds4_gpu_tensor_buffer(group_tmp) != nil && + ds4_gpu_tensor_buffer(heads) != nil && + ds4_gpu_tensor_bytes(out) >= out_bytes && + ds4_gpu_tensor_bytes(out_hc) >= hc_bytes && + ds4_gpu_tensor_bytes(residual_hc) >= hc_bytes && + ds4_gpu_tensor_bytes(split) >= split_bytes && + ds4_gpu_tensor_bytes(low) >= low_bytes && + ds4_gpu_tensor_bytes(group_tmp) >= rhs_f16_bytes && + ds4_gpu_tensor_bytes(heads) >= heads_bytes && + !ds4_gpu_tensor_prefixes_overlap( + group_tmp, rhs_f16_bytes, out, out_bytes) && + !ds4_gpu_tensor_prefixes_overlap( + group_tmp, rhs_f16_bytes, out_hc, hc_bytes) && + !ds4_gpu_tensor_prefixes_overlap( + group_tmp, rhs_f16_bytes, residual_hc, hc_bytes) && + !ds4_gpu_tensor_prefixes_overlap( + group_tmp, rhs_f16_bytes, split, split_bytes) && + !ds4_gpu_tensor_prefixes_overlap( + group_tmp, rhs_f16_bytes, low, low_bytes) && + !ds4_gpu_tensor_prefixes_overlap( + group_tmp, rhs_f16_bytes, heads, heads_bytes); + + uint64_t out_b_inner = 0u; + id out_b_buf = nil; + if (eligible) { + out_b_buf = ds4_gpu_wrap_model_range( + model_map, model_size, out_b_offset, out_b_bytes, &out_b_inner); + eligible = out_b_buf != nil && out_b_inner <= NSUIntegerMax; + } + if (!eligible) { + if (require) { + fprintf(stderr, + "ds4: required Metal Q4 batch attention-output HC " + "fusion was not selected\n"); + return -1; + } + return 0; + } + + ds4_gpu_attn_out_hc_target target = { + .out_hc = out_hc, + .residual_hc = residual_hc, + .split = split, + .weight_buffer = out_b_buf, + .weight_offset = (NSUInteger)out_b_inner, + }; + const int rc = ds4_gpu_attention_output_q4_K_batch_impl( + out, low, group_tmp, low_tmp, model_map, model_size, + out_a_offset, out_b_offset, out_b_type, group_dim, rank, + n_groups, out_dim, heads, n_tokens, &target); + return rc == 1 ? 1 : -1; +} + int ds4_gpu_attention_output_q8_batch_f16_tensor( ds4_gpu_tensor *out_h, ds4_gpu_tensor *low, diff --git a/metal/dsv4_hc.metal b/metal/dsv4_hc.metal index dfcefdded..a6922df94 100644 --- a/metal/dsv4_hc.metal +++ b/metal/dsv4_hc.metal @@ -692,6 +692,306 @@ kernel void kernel_dsv4_hc_expand4( } } +// Resident prefill attention-output tail for the exact legacy Q8_0 tile: +// materialize each 64x32 F32 result in threadgroup memory and immediately +// apply the HC=4 epilogue. This preserves the existing F32 boundary and HC +// statement order while removing the global attn_out round trip. +kernel void kernel_dsv4_attn_out_q8_mm_hc_expand4_batch( + constant ds4_metal_args_mul_mm & mm [[buffer(0)]], + device const char * weight [[buffer(1)]], + device const char * input [[buffer(2)]], + device const char * residual [[buffer(3)]], + device const char * post [[buffer(4)]], + device const char * comb [[buffer(5)]], + device char * dst [[buffer(6)]], + constant ds4_metal_args_dsv4_hc_expand & hc [[buffer(7)]], + threadgroup char * shmem [[threadgroup(0)]], + uint3 tgpig [[threadgroup_position_in_grid]], + ushort tiitg [[thread_index_in_threadgroup]], + ushort sgitg [[simdgroup_index_in_threadgroup]]) { + constexpr int NR0 = 64; + constexpr int NR1 = 32; + constexpr int NK = 32; + constexpr int NL0 = NK / 16; + constexpr int NL1 = NK / 8; + + if (hc.n_hc != 4 || mm.ne0 != hc.n_embd || mm.ne1 != hc.n_tokens || + mm.ne00 != 8192 || mm.ne0 != 4096 || (mm.ne1 & 31) != 0) { + return; + } + + threadgroup half * sa = (threadgroup half *)shmem; + threadgroup half * sb = (threadgroup half *)(shmem + 4096); + + const int im = tgpig.z; + const int r0 = tgpig.y * NR0; + const int r1 = tgpig.x * NR1; + const short lr0 = (short)tiitg / NL0; + const short lr1 = (short)tiitg / NL1; + const short il0 = tiitg % NL0; + short il = il0; + + const int i12 = im % mm.ne12; + const int i13 = im / mm.ne12; + const uint64_t offset0 = + (i12 / mm.r2) * mm.nb02 + (i13 / mm.r3) * mm.nb03; + const short offset1 = il0 / 2; + device const block_q8_0 * x = + (device const block_q8_0 *)(weight + mm.nb01 * (r0 + lr0) + offset0) + + offset1; + + const short iy = 8 * (tiitg % NL1); + device const float * y = (device const float *)(input + + mm.nb13 * i13 + mm.nb12 * i12 + mm.nb11 * (r1 + lr1) + mm.nb10 * iy); + + simdgroup_half8x8 ma[4]; + simdgroup_half8x8 mb[2]; + simdgroup_float8x8 mc[8]; + FOR_UNROLL (short i = 0; i < 8; ++i) { + mc[i] = make_filled_simdgroup_matrix(0.0f); + } + + for (int loop_k = 0; loop_k < mm.ne00; loop_k += NK) { + half4x4 temp_a; + dequantize_q8_0(x, il, temp_a); + + threadgroup_barrier(mem_flags::mem_threadgroup); + + FOR_UNROLL (short i = 0; i < 16; ++i) { + const short sx = 2 * il0 + i / 8; + const short sy = (tiitg / NL0) / 8; + const short lx = (tiitg / NL0) % 8; + const short ly = i % 8; + const short ib = 8 * sx + sy; + *(sa + 64 * ib + 8 * ly + lx) = temp_a[i / 4][i % 4]; + } + + const short sx = tiitg % NL1; + const short sy = (tiitg / NL1) / 8; + const short ly = (tiitg / NL1) % 8; + const short ib = 4 * sx + sy; + *(threadgroup half2x4 *)(sb + 64 * ib + 8 * ly) = + (half2x4)(*((device float2x4 *)y)); + + il = (il + 2 < 2) ? il + 2 : il % 2; + x = (il < 2) ? x + 1 : x; + y += NK; + + threadgroup_barrier(mem_flags::mem_threadgroup); + + threadgroup const half * lsma = sa + 4 * 64 * (sgitg % 2); + threadgroup const half * lsmb = sb + 2 * 64 * (sgitg / 2); + + FOR_UNROLL (short ik = 0; ik < NK / 8; ++ik) { + simdgroup_barrier(mem_flags::mem_none); + FOR_UNROLL (short i = 0; i < 4; ++i) { + simdgroup_load(ma[i], lsma + 64 * i, 8, 0, false); + } + simdgroup_barrier(mem_flags::mem_none); + FOR_UNROLL (short i = 0; i < 2; ++i) { + simdgroup_load(mb[i], lsmb + 64 * i, 8, 0, false); + } + simdgroup_barrier(mem_flags::mem_none); + FOR_UNROLL (short i = 0; i < 8; ++i) { + simdgroup_multiply_accumulate(mc[i], mb[i / 4], ma[i % 4], mc[i]); + } + lsma += 8 * 64; + lsmb += 4 * 64; + } + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + threadgroup float * tile = (threadgroup float *)shmem; + threadgroup float * tile_sg = + tile + 32 * (sgitg & 1) + 16 * (sgitg >> 1) * NR0; + FOR_UNROLL (short i = 0; i < 8; ++i) { + simdgroup_store(mc[i], + tile_sg + 8 * (i % 4) + 8 * NR0 * (i / 4), + NR0, 0, false); + } + threadgroup_barrier(mem_flags::mem_threadgroup); + + for (uint e = tiitg; e < (uint)(NR0 * NR1); e += 128u) { + const int64_t t = r1 + e / NR0; + const int64_t d = r0 + e % NR0; + const float block_v = tile[e]; + + const float rv0 = *((device const float *)(residual + + d * hc.nb_res0 + 0 * hc.nb_res1 + t * hc.nb_res2)); + const float rv1 = *((device const float *)(residual + + d * hc.nb_res0 + 1 * hc.nb_res1 + t * hc.nb_res2)); + const float rv2 = *((device const float *)(residual + + d * hc.nb_res0 + 2 * hc.nb_res1 + t * hc.nb_res2)); + const float rv3 = *((device const float *)(residual + + d * hc.nb_res0 + 3 * hc.nb_res1 + t * hc.nb_res2)); + + for (int64_t dst_hc = 0; dst_hc < 4; ++dst_hc) { + float acc = block_v * *((device const float *)(post + + dst_hc * hc.nb_post0 + t * hc.nb_post1)); + acc += *((device const float *)(comb + + dst_hc * hc.nb_comb0 + 0 * hc.nb_comb1 + t * hc.nb_comb2)) * rv0; + acc += *((device const float *)(comb + + dst_hc * hc.nb_comb0 + 1 * hc.nb_comb1 + t * hc.nb_comb2)) * rv1; + acc += *((device const float *)(comb + + dst_hc * hc.nb_comb0 + 2 * hc.nb_comb1 + t * hc.nb_comb2)) * rv2; + acc += *((device const float *)(comb + + dst_hc * hc.nb_comb0 + 3 * hc.nb_comb1 + t * hc.nb_comb2)) * rv3; + *((device float *)(dst + d * hc.nb0 + dst_hc * hc.nb1 + + t * hc.nb2)) = acc; + } + } +} + +// Q4_K counterpart of the resident output-B/HC tail. It consumes the same +// pre-materialized F16 RHS as kernel_mul_mm_q4_K_f16_rhs, keeps its Q4_K +// dequantization and FP32 MMA order, and applies the identical HC=4 epilogue +// directly from the threadgroup F32 tile. +kernel void kernel_dsv4_attn_out_q4_K_f16_rhs_mm_hc_expand4_batch( + constant ds4_metal_args_mul_mm & mm [[buffer(0)]], + device const char * weight [[buffer(1)]], + device const char * input [[buffer(2)]], + device const char * residual [[buffer(3)]], + device const char * post [[buffer(4)]], + device const char * comb [[buffer(5)]], + device char * dst [[buffer(6)]], + constant ds4_metal_args_dsv4_hc_expand & hc [[buffer(7)]], + threadgroup char * shmem [[threadgroup(0)]], + uint3 tgpig [[threadgroup_position_in_grid]], + ushort tiitg [[thread_index_in_threadgroup]], + ushort sgitg [[simdgroup_index_in_threadgroup]]) { + constexpr int NR0 = 64; + constexpr int NR1 = 32; + constexpr int NK = 32; + constexpr int NL0 = NK / 16; + constexpr int NL1 = NK / 8; + constexpr short NL = 16; + + if (hc.n_hc != 4 || mm.ne0 != hc.n_embd || mm.ne1 != hc.n_tokens || + mm.ne00 != 8192 || mm.ne0 != 4096 || (mm.ne1 & 31) != 0) { + return; + } + + threadgroup half * sa = (threadgroup half *)shmem; + threadgroup half * sb = (threadgroup half *)(shmem + 4096); + + const int im = tgpig.z; + const int r0 = tgpig.y * NR0; + const int r1 = tgpig.x * NR1; + const short lr0 = (short)tiitg / NL0; + const short lr1 = (short)tiitg / NL1; + const short il0 = tiitg % NL0; + short il = il0; + + const int i12 = im % mm.ne12; + const int i13 = im / mm.ne12; + const uint64_t offset0 = + (i12 / mm.r2) * mm.nb02 + (i13 / mm.r3) * mm.nb03; + device const ds4_dense_block_q4_K * x = + (device const ds4_dense_block_q4_K *)( + weight + mm.nb01 * (r0 + lr0) + offset0); + + const short iy = 8 * (tiitg % NL1); + device const half * y = (device const half *)(input + + mm.nb13 * i13 + mm.nb12 * i12 + mm.nb11 * (r1 + lr1) + mm.nb10 * iy); + + simdgroup_half8x8 ma[4]; + simdgroup_half8x8 mb[2]; + simdgroup_float8x8 mc[8]; + FOR_UNROLL (short i = 0; i < 8; ++i) { + mc[i] = make_filled_simdgroup_matrix(0.0f); + } + + for (int loop_k = 0; loop_k < mm.ne00; loop_k += NK) { + half4x4 temp_a; + dequantize_dense_q4_K(x, il, temp_a); + + threadgroup_barrier(mem_flags::mem_threadgroup); + + FOR_UNROLL (short i = 0; i < 16; ++i) { + const short sx = 2 * il0 + i / 8; + const short sy = (tiitg / NL0) / 8; + const short lx = (tiitg / NL0) % 8; + const short ly = i % 8; + const short ib = 8 * sx + sy; + *(sa + 64 * ib + 8 * ly + lx) = temp_a[i / 4][i % 4]; + } + + const short sx = tiitg % NL1; + const short sy = (tiitg / NL1) / 8; + const short ly = (tiitg / NL1) % 8; + const short ib = 4 * sx + sy; + *(threadgroup half2x4 *)(sb + 64 * ib + 8 * ly) = + *((device half2x4 *)y); + + il = (il + 2 < NL) ? il + 2 : il % 2; + x = (il < 2) ? x + 1 : x; + y += NK; + + threadgroup_barrier(mem_flags::mem_threadgroup); + + threadgroup const half * lsma = sa + 4 * 64 * (sgitg % 2); + threadgroup const half * lsmb = sb + 2 * 64 * (sgitg / 2); + + FOR_UNROLL (short ik = 0; ik < NK / 8; ++ik) { + simdgroup_barrier(mem_flags::mem_none); + FOR_UNROLL (short i = 0; i < 4; ++i) { + simdgroup_load(ma[i], lsma + 64 * i, 8, 0, false); + } + simdgroup_barrier(mem_flags::mem_none); + FOR_UNROLL (short i = 0; i < 2; ++i) { + simdgroup_load(mb[i], lsmb + 64 * i, 8, 0, false); + } + simdgroup_barrier(mem_flags::mem_none); + FOR_UNROLL (short i = 0; i < 8; ++i) { + simdgroup_multiply_accumulate(mc[i], mb[i / 4], ma[i % 4], mc[i]); + } + lsma += 8 * 64; + lsmb += 4 * 64; + } + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + threadgroup float * tile = (threadgroup float *)shmem; + threadgroup float * tile_sg = + tile + 32 * (sgitg & 1) + 16 * (sgitg >> 1) * NR0; + FOR_UNROLL (short i = 0; i < 8; ++i) { + simdgroup_store(mc[i], + tile_sg + 8 * (i % 4) + 8 * NR0 * (i / 4), + NR0, 0, false); + } + threadgroup_barrier(mem_flags::mem_threadgroup); + + for (uint e = tiitg; e < (uint)(NR0 * NR1); e += 128u) { + const int64_t t = r1 + e / NR0; + const int64_t d = r0 + e % NR0; + const float block_v = tile[e]; + + const float rv0 = *((device const float *)(residual + + d * hc.nb_res0 + 0 * hc.nb_res1 + t * hc.nb_res2)); + const float rv1 = *((device const float *)(residual + + d * hc.nb_res0 + 1 * hc.nb_res1 + t * hc.nb_res2)); + const float rv2 = *((device const float *)(residual + + d * hc.nb_res0 + 2 * hc.nb_res1 + t * hc.nb_res2)); + const float rv3 = *((device const float *)(residual + + d * hc.nb_res0 + 3 * hc.nb_res1 + t * hc.nb_res2)); + + for (int64_t dst_hc = 0; dst_hc < 4; ++dst_hc) { + float acc = block_v * *((device const float *)(post + + dst_hc * hc.nb_post0 + t * hc.nb_post1)); + acc += *((device const float *)(comb + + dst_hc * hc.nb_comb0 + 0 * hc.nb_comb1 + t * hc.nb_comb2)) * rv0; + acc += *((device const float *)(comb + + dst_hc * hc.nb_comb0 + 1 * hc.nb_comb1 + t * hc.nb_comb2)) * rv1; + acc += *((device const float *)(comb + + dst_hc * hc.nb_comb0 + 2 * hc.nb_comb1 + t * hc.nb_comb2)) * rv2; + acc += *((device const float *)(comb + + dst_hc * hc.nb_comb0 + 3 * hc.nb_comb1 + t * hc.nb_comb2)) * rv3; + *((device float *)(dst + d * hc.nb0 + dst_hc * hc.nb1 + + t * hc.nb2)) = acc; + } + } +} + // Decode-time FFN tail fusion: // // shared_out = shared_mid @ Wshared_down @@ -1572,277 +1872,3 @@ kernel void kernel_dsv4_hc_rms_norm_mix_f16_cluster2_pre_norm( thread_scope_device); atomic_store_explicit(completion, 0u, memory_order_relaxed); } - -/* The compound producer writes the 4x4 combination logits from four - * independent threadgroups. The final arriving group performs exactly the - * same Sinkhorn sequence used by the standalone HC split kernel. */ -static __attribute__((always_inline)) inline void -ds4_hc_comb_weights4_exact_continuation( - constant ds4_metal_args_dsv4_hc_split_weighted_sum_norm & args, - device volatile const float *mix, - device const float *scale, - device const float *base, - device float *out) { - const float epsv = args.eps; - const float comb_scale = scale[2]; - - float4 r0 = *((device volatile const float4 *)(mix + 8)) * comb_scale + - *((device const float4 *)(base + 8)); - float4 r1 = *((device volatile const float4 *)(mix + 12)) * comb_scale + - *((device const float4 *)(base + 12)); - float4 r2 = *((device volatile const float4 *)(mix + 16)) * comb_scale + - *((device const float4 *)(base + 16)); - float4 r3 = *((device volatile const float4 *)(mix + 20)) * comb_scale + - *((device const float4 *)(base + 20)); - - const float m0 = max(max(r0.x, r0.y), max(r0.z, r0.w)); - const float m1 = max(max(r1.x, r1.y), max(r1.z, r1.w)); - const float m2 = max(max(r2.x, r2.y), max(r2.z, r2.w)); - const float m3 = max(max(r3.x, r3.y), max(r3.z, r3.w)); - - r0 = exp(r0 - m0); - r1 = exp(r1 - m1); - r2 = exp(r2 - m2); - r3 = exp(r3 - m3); - - r0 = r0 * (1.0f / (r0.x + r0.y + r0.z + r0.w)) + epsv; - r1 = r1 * (1.0f / (r1.x + r1.y + r1.z + r1.w)) + epsv; - r2 = r2 * (1.0f / (r2.x + r2.y + r2.z + r2.w)) + epsv; - r3 = r3 * (1.0f / (r3.x + r3.y + r3.z + r3.w)) + epsv; - - float4 col_inv = 1.0f / (r0 + r1 + r2 + r3 + epsv); - r0 *= col_inv; - r1 *= col_inv; - r2 *= col_inv; - r3 *= col_inv; - - for (int iter = 1; iter < args.sinkhorn_iters; ++iter) { - r0 *= 1.0f / (r0.x + r0.y + r0.z + r0.w + epsv); - r1 *= 1.0f / (r1.x + r1.y + r1.z + r1.w + epsv); - r2 *= 1.0f / (r2.x + r2.y + r2.z + r2.w + epsv); - r3 *= 1.0f / (r3.x + r3.y + r3.z + r3.w + epsv); - - col_inv = 1.0f / (r0 + r1 + r2 + r3 + epsv); - r0 *= col_inv; - r1 *= col_inv; - r2 *= col_inv; - r3 *= col_inv; - } - - *((device float4 *)(out + 8)) = r0; - *((device float4 *)(out + 12)) = r1; - *((device float4 *)(out + 16)) = r2; - *((device float4 *)(out + 20)) = r3; -} - -/* Exact DS4 decode compound for the fixed 16384 -> 24 HC producer shape. - * Six 512-thread groups reproduce the original RMSNorm and F16 matvec trees. - * Group zero continues directly into pre weighting, HC collapse, and the next - * weighted RMSNorm; group one emits post weights; groups two through five - * publish the combination rows and the last arrival runs Sinkhorn. */ -kernel void kernel_dsv4_hc_rms_norm_mix_f16_cluster2_pre_norm( - constant ds4_metal_args_hc_norm_mix & args, - constant ds4_metal_args_dsv4_hc_split_weighted_sum_norm & split_args, - device const char * x, - device const char * weight, - device char * dst, - device const float * hc_scale, - device const float * hc_base, - device char * split, - device char * collapse_dst, - device const char * norm_weight, - device char * norm_dst, - device atomic_uint * completion, - threadgroup char * shmem [[threadgroup(0)]], - uint3 tgpig [[threadgroup_position_in_grid]], - ushort tiisg [[thread_index_in_simdgroup]], - ushort sgitg [[simdgroup_index_in_threadgroup]]) { - constexpr short NSG_CLUSTER = 8; - constexpr short NCLUSTER = 2; - constexpr short NSG_TOTAL = NSG_CLUSTER * NCLUSTER; - constexpr short NW = N_SIMDWIDTH; - constexpr short NR0 = 2; - constexpr short NB = 32; - constexpr short NF = 16; - constexpr short NF4 = NF/4; - constexpr uint VTHREADS = 1024u; - constexpr short VSLICES = VTHREADS/(NSG_TOTAL*NW); - - const uint n = (uint)args.n; - const uint n4 = n >> 2; - device const float4 *x4 = (device const float4 *)x; - threadgroup float *norm_shmem = (threadgroup float *)shmem; - threadgroup float *mv_shmem = norm_shmem + NW; - - for (short v = 0; v < VSLICES; ++v) { - const uint vt = (uint)(sgitg + NSG_TOTAL*v)*NW + tiisg; - float sumf = 0.0f; - for (uint i00 = vt; i00 < n4; i00 += VTHREADS) { - sumf += dot(x4[i00], x4[i00]); - } - sumf = simd_sum(sumf); - if (tiisg == 0) { - norm_shmem[sgitg + NSG_TOTAL*v] = sumf; - } - } - threadgroup_barrier(mem_flags::mem_threadgroup); - - float total = norm_shmem[tiisg]; - total = simd_sum(total); - const float mean = total/(float)args.n; - const float scale = 1.0f/sqrt(mean + args.eps); - - const short cluster = sgitg / NSG_CLUSTER; - const short local_sg = sgitg - cluster*NSG_CLUSTER; - const int nb = args.n/NB; - const int r0 = (int)tgpig.x*(NCLUSTER*NR0) + cluster*NR0; - - device const half4 *ax4[NR0]; - FOR_UNROLL(short row = 0; row < NR0; ++row) { - ax4[row] = (device const half4 *)( - weight + (uint64_t)(r0 + row)*(uint64_t)n*sizeof(half)); - } - - float sumf_mv[NR0] = { 0.0f }; - const short ix = tiisg/(NW/NF); - const short il = tiisg%(NW/NF); - const int ib0 = local_sg*NF + ix; - for (int ib = ib0; ib < nb; ib += NSG_CLUSTER*NF) { - float4 yl4[NF4]; - FOR_UNROLL(short i = 0; i < NF4; ++i) { - yl4[i] = x4[(ib*NB + il*NF)/4 + i]*scale; - } - FOR_UNROLL(short row = 0; row < NR0; ++row) { - device const half4 *xb4 = ax4[row] + (ib*NB + il*NF)/4; - float sumq = 0.0f; - FOR_UNROLL(short i = 0; i < NF4; ++i) { - sumq += dot(float4(xb4[i]), yl4[i]); - } - sumf_mv[row] += sumq; - } - } - - threadgroup float *cluster_shmem[NR0]; - FOR_UNROLL(short row = 0; row < NR0; ++row) { - cluster_shmem[row] = mv_shmem + ((uint)cluster*NR0 + row)*NW; - if (local_sg == 0) { - cluster_shmem[row][tiisg] = 0.0f; - } - sumf_mv[row] = simd_sum(sumf_mv[row]); - } - threadgroup_barrier(mem_flags::mem_threadgroup); - FOR_UNROLL(short row = 0; row < NR0; ++row) { - if (tiisg == 0) { - cluster_shmem[row][local_sg] = sumf_mv[row]; - } - } - threadgroup_barrier(mem_flags::mem_threadgroup); - - device volatile float *mixes_f32 = (device volatile float *)dst; - if (local_sg == 0) { - FOR_UNROLL(short row = 0; row < NR0; ++row) { - const float tot = simd_sum(cluster_shmem[row][tiisg]); - if (tiisg == 0 && r0 + row < args.out_dim) { - mixes_f32[r0 + row] = tot; - } - } - } - - threadgroup_barrier(mem_flags::mem_device_and_threadgroup); - const uint tid = (uint)sgitg*(uint)NW + (uint)tiisg; - threadgroup float *pre_shmem = norm_shmem + 32u + 4u*NW; - threadgroup float *sum_shmem = pre_shmem + 4; - - if (tgpig.x == 0) { - device float *out = (device float *)split; - if (tid == 0) { - const float4 pre_z = - *((device volatile const float4 *)mixes_f32)*hc_scale[0] + - *((device const float4 *)hc_base); - const float4 pre = - 1.0f/(1.0f + exp(-pre_z)) + split_args.eps; - *((device float4 *)out) = pre; - pre_shmem[0] = pre.x; - pre_shmem[1] = pre.y; - pre_shmem[2] = pre.z; - pre_shmem[3] = pre.w; - } - threadgroup_barrier(mem_flags::mem_threadgroup); - - const uint n4_collapse = uint(split_args.n_embd) >> 2; - const uint i0 = tid; - const uint i1 = tid + 512u; - device const float4 *x0 = - (device const float4 *)(x + 0*split_args.nb_x1); - device const float4 *x1 = - (device const float4 *)(x + 1*split_args.nb_x1); - device const float4 *x2 = - (device const float4 *)(x + 2*split_args.nb_x1); - device const float4 *x3 = - (device const float4 *)(x + 3*split_args.nb_x1); - - float4 v0 = 0.0f; - v0 += x0[i0]*pre_shmem[0]; - v0 += x1[i0]*pre_shmem[1]; - v0 += x2[i0]*pre_shmem[2]; - v0 += x3[i0]*pre_shmem[3]; - const float sum0 = simd_sum(dot(v0, v0)); - - float4 v1 = 0.0f; - if (i1 < n4_collapse) { - v1 += x0[i1]*pre_shmem[0]; - v1 += x1[i1]*pre_shmem[1]; - v1 += x2[i1]*pre_shmem[2]; - v1 += x3[i1]*pre_shmem[3]; - } - const float sum1 = simd_sum(dot(v1, v1)); - if (tiisg == 0) { - sum_shmem[sgitg] = sum0; - sum_shmem[sgitg + 16] = sum1; - } - threadgroup_barrier(mem_flags::mem_threadgroup); - - float sumf = sum_shmem[tiisg]; - sumf = simd_sum(sumf); - const float norm_arg = - sumf/float(split_args.n_embd) + split_args.norm_eps; - const float norm_scale = rsqrt(norm_arg); - device float4 *dst4 = (device float4 *)collapse_dst; - device const float4 *w4 = (device const float4 *)norm_weight; - device float4 *norm4 = (device float4 *)norm_dst; - dst4[i0] = v0; - norm4[i0] = (v0*norm_scale)*w4[i0]; - if (i1 < n4_collapse) { - dst4[i1] = v1; - norm4[i1] = (v1*norm_scale)*w4[i1]; - } - } else if (tgpig.x == 1 && tid == 0) { - device float *out = (device float *)split; - const float4 post_z = - *((device volatile const float4 *)(mixes_f32 + 4))*hc_scale[1] + - *((device const float4 *)(hc_base + 4)); - *((device float4 *)(out + 4)) = 2.0f/(1.0f + exp(-post_z)); - } - - atomic_thread_fence(mem_flags::mem_device, - memory_order_seq_cst, - thread_scope_device); - if (tgpig.x < 2 || tid != 0) { - return; - } - - const uint old = atomic_fetch_add_explicit( - completion, 1u, memory_order_relaxed); - if (old + 1u != 4u) { - return; - } - atomic_thread_fence(mem_flags::mem_device, - memory_order_seq_cst, - thread_scope_device); - ds4_hc_comb_weights4_exact_continuation( - split_args, mixes_f32, hc_scale, hc_base, (device float *)split); - atomic_thread_fence(mem_flags::mem_device, - memory_order_seq_cst, - thread_scope_device); - atomic_store_explicit(completion, 0u, memory_order_relaxed); -} diff --git a/rocm/ds4_rocm_moe.cuh b/rocm/ds4_rocm_moe.cuh index 57f20218f..6ed2d5cec 100644 --- a/rocm/ds4_rocm_moe.cuh +++ b/rocm/ds4_rocm_moe.cuh @@ -750,18 +750,20 @@ __device__ static void dev_dot_q2_K_q8_K_block8( } __device__ static float half_warp_sum_f32(float v, uint32_t lane16) { - uint32_t mask = 0xffffu << (threadIdx.x & 16u); + const uint32_t wave_lane = threadIdx.x & (warpSize - 1u); + const MASK_T mask = static_cast(0xffffu) << (wave_lane & ~15u); for (int offset = 8; offset > 0; offset >>= 1) { - v += __shfl_down_sync(static_cast(mask), v, offset, 16); + v += __shfl_down_sync(mask, v, offset, 16); } (void)lane16; return v; } __device__ static float quarter_warp_sum_f32(float v, uint32_t lane8) { - uint32_t mask = 0xffu << (threadIdx.x & 24u); + const uint32_t wave_lane = threadIdx.x & (warpSize - 1u); + const MASK_T mask = static_cast(0xffu) << (wave_lane & ~7u); for (int offset = 4; offset > 0; offset >>= 1) { - v += __shfl_down_sync(static_cast(mask), v, offset, 8); + v += __shfl_down_sync(mask, v, offset, 8); } (void)lane8; return v; diff --git a/rocm/ds4_rocm_norm_rope.cuh b/rocm/ds4_rocm_norm_rope.cuh index b68ec1a1e..0b3009207 100644 --- a/rocm/ds4_rocm_norm_rope.cuh +++ b/rocm/ds4_rocm_norm_rope.cuh @@ -530,6 +530,15 @@ extern "C" int ds4_gpu_head_rms_norm_rope_tail_tensor(ds4_gpu_tensor *x, uint32_ return cuda_ok(cudaGetLastError(), "head_rms_norm_rope_tail launch"); } +static int rocm_q4_attn_q_b_prefixes_overlap( + const void *a, uint64_t a_bytes, + const void *b, uint64_t b_bytes) { + const uintptr_t ap = reinterpret_cast(a); + const uintptr_t bp = reinterpret_cast(b); + return ap <= bp ? (uint64_t)(bp - ap) < a_bytes + : (uint64_t)(ap - bp) < b_bytes; +} + static int rocm_q4_attn_q_b_f16_head_rms_rope_tail_tensor( ds4_gpu_tensor *out, ds4_gpu_tensor *q_half, @@ -553,7 +562,6 @@ static int rocm_q4_attn_q_b_f16_head_rms_rope_tail_tensor( float beta_fast, float beta_slow, float eps) { - (void)q_half; /* Decode and tiny/final chunks retain the native Q4_K path without even * taking the cache mutex. REQUIRE applies only to configured candidates. */ if (n_tok < 32u) return 0; @@ -561,15 +569,18 @@ static int rocm_q4_attn_q_b_f16_head_rms_rope_tail_tensor( if (!rocm_q4_attn_q_b_f16_enabled() && !required) return 0; if ((uint64_t)n_tok < rocm_q4_attn_q_b_f16_min_tokens()) return 0; rocm_q4_attn_q_b_f16_note_candidate(); + const int f16_output = rocm_q4_attn_q_b_f16_output_enabled(); uint64_t x_elems = 0; uint64_t out_elems = 0; uint64_t x_bytes = 0; uint64_t out_bytes = 0; + uint64_t out_f16_bytes = 0; uint64_t head_rows = 0; if (!rocm_q4_attn_q_b_f16_policy_allowed() || rocm_q4_attn_q_b_f16_circuit_open() || - !g_cublas_ready || !out || !x || !model_map || + !g_cublas_ready || !out || !out->ptr || + !x || !x->ptr || !model_map || model_map != g_model_host_base || model_size != g_model_registered_size || in_dim != DS4_ROCM_Q4_ATTN_Q_B_IN_DIM || @@ -586,6 +597,7 @@ static int rocm_q4_attn_q_b_f16_head_rms_rope_tail_tensor( (x_elems + 255u) / 256u > UINT32_MAX || !cuda_u64_mul_checked(x_elems, sizeof(float), &x_bytes) || !cuda_u64_mul_checked(out_elems, sizeof(float), &out_bytes) || + !cuda_u64_mul_checked(out_elems, sizeof(__half), &out_f16_bytes) || x->bytes < x_bytes || out->bytes < out_bytes) { return rocm_q4_attn_q_b_f16_fallback(required, 1, 0); } @@ -602,11 +614,22 @@ static int rocm_q4_attn_q_b_f16_head_rms_rope_tail_tensor( __half *unused_weight_f16 = NULL; __half *xh = NULL; + __half *q_scratch = NULL; if (!rocm_q4_attn_q_b_transient_f16_acquire( - n_tok, &unused_weight_f16, &xh)) { + n_tok, f16_output, + &unused_weight_f16, &xh, &q_scratch)) { return rocm_q4_attn_q_b_f16_fallback(required, 0, 0); } (void)unused_weight_f16; + const int use_graph_q_half = + f16_output && q_half && q_half->ptr && + q_half->bytes >= out_f16_bytes && + !rocm_q4_attn_q_b_prefixes_overlap( + q_half->ptr, out_f16_bytes, out->ptr, out_bytes) && + !rocm_q4_attn_q_b_prefixes_overlap( + q_half->ptr, out_f16_bytes, x->ptr, x_bytes); + __half *const qh = use_graph_q_half + ? (__half *)q_half->ptr : q_scratch; const __half *w_f16 = rocm_q4_attn_q_b_f16_acquire( model_map, model_size, weight_offset, weight_bytes, in_dim, out_dim, @@ -616,9 +639,9 @@ static int rocm_q4_attn_q_b_f16_head_rms_rope_tail_tensor( return rocm_q4_attn_q_b_f16_fallback(required, 0, 0); } - /* Keep both the persistent weight pin and the dedicated X staging lock + /* Keep both the persistent weight pin and the dedicated X/Q staging lock * through the complete enqueue sequence. Lifecycle release takes the same - * locks before synchronizing, so it cannot miss the final F32 epilogue. */ + * locks before synchronizing, so it cannot miss the final epilogue. */ f32_to_f16_kernel<<<(x_elems + 255u) / 256u, 256>>>( xh, (const float *)x->ptr, x_elems); cudaError_t launch_err = cudaGetLastError(); @@ -635,6 +658,9 @@ static int rocm_q4_attn_q_b_f16_head_rms_rope_tail_tensor( const float alpha = 1.0f; const float beta = 0.0f; + /* The release path remains F32 by default. The explicit F16-output arm + * matches Q8's boundary and consumes either caller staging or ROCm-owned + * Q_F16 without materializing the large F32 Q. */ const cublasStatus_t st = cublasGemmEx( g_cublas, CUBLAS_OP_T, @@ -650,8 +676,8 @@ static int rocm_q4_attn_q_b_f16_head_rms_rope_tail_tensor( CUDA_R_16F, (int)in_dim, &beta, - out->ptr, - CUDA_R_32F, + f16_output ? (void *)qh : out->ptr, + f16_output ? CUDA_R_16F : CUDA_R_32F, (int)out_dim, CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT); @@ -659,21 +685,40 @@ static int rocm_q4_attn_q_b_f16_head_rms_rope_tail_tensor( fprintf(stderr, DS4_GPU_LOG_PREFIX DS4_GPU_BLAS_NAME - " cached Q4 attn q_b F16/F16-to-F32 matmul failed: " + " cached Q4 attn q_b F16/F16-to-%s matmul failed: " "status %d\n", + f16_output ? "F16" : "F32", (int)st); rocm_q4_attn_q_b_f16_release_acquired(); rocm_q4_attn_q_b_transient_f16_release_acquired(); return rocm_q4_attn_q_b_f16_fallback(required, 0, 1); } - const int tail_ok = ds4_gpu_head_rms_norm_rope_tail_tensor( + int tail_ok = 0; + if (f16_output) { + head_rms_norm_rope_tail_from_half_kernel<<<(uint32_t)head_rows, 256>>>( + (float *)out->ptr, qh, + n_tok, n_head, head_dim, n_rot, pos0, n_ctx_orig, + inverse ? 1 : 0, freq_base, freq_scale, ext_factor, + attn_factor, beta_fast, beta_slow, eps); + launch_err = cudaGetLastError(); + tail_ok = launch_err == cudaSuccess; + } else { + tail_ok = ds4_gpu_head_rms_norm_rope_tail_tensor( out, n_tok, n_head, head_dim, n_rot, pos0, n_ctx_orig, inverse, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow, eps); + } rocm_q4_attn_q_b_f16_release_acquired(); rocm_q4_attn_q_b_transient_f16_release_acquired(); if (!tail_ok) { + if (f16_output) { + fprintf(stderr, + DS4_GPU_LOG_PREFIX + "cached Q4 attn q_b F16-out epilogue launch failed: %s\n", + cudaGetErrorString(launch_err)); + (void)cudaGetLastError(); + } /* GEMM has already accepted the output writer. Never authorize the * caller to replay native Q4 over an asynchronous/partial result. */ return rocm_q4_attn_q_b_f16_fallback(1, 0, 1); @@ -682,9 +727,9 @@ static int rocm_q4_attn_q_b_f16_head_rms_rope_tail_tensor( } /* Resident default: expand only the current Q4_K q_b matrix into the shared - * 64 MiB W_F16 region, convert X into the adjacent preflighted region, consume - * both immediately with hipBLAS, and enqueue the F32 epilogue before allowing - * another host caller to reuse the allocation. */ + * 64 MiB W_F16 region, stage X_F16 beside it, and consume both inputs + * immediately with hipBLAS. The opt-in F16-output arm also stages Q_F16 before + * its fused epilogue. No caller may reuse the allocation between these steps. */ static int rocm_q4_attn_q_b_transient_f16_head_rms_rope_tail_tensor( ds4_gpu_tensor *out, ds4_gpu_tensor *q_half, @@ -708,21 +753,23 @@ static int rocm_q4_attn_q_b_transient_f16_head_rms_rope_tail_tensor( float beta_fast, float beta_slow, float eps) { - (void)q_half; if ((uint64_t)n_tok < rocm_q4_attn_q_b_transient_f16_min_tokens()) { return 0; } rocm_q4_attn_q_b_f16_note_candidate(); + const int f16_output = rocm_q4_attn_q_b_f16_output_enabled(); uint64_t x_elems = 0; uint64_t out_elems = 0; uint64_t head_rows = 0; uint64_t x_bytes = 0; uint64_t out_bytes = 0; + uint64_t out_f16_bytes = 0; if (!rocm_q4_attn_q_b_transient_f16_policy_allowed() || rocm_q4_attn_q_b_f16_circuit_open() || - !g_cublas_ready || !out || !x || !model_map || + !g_cublas_ready || !out || !out->ptr || + !x || !x->ptr || !model_map || model_map != g_model_host_base || model_size != g_model_registered_size || in_dim != DS4_ROCM_Q4_ATTN_Q_B_IN_DIM || @@ -739,6 +786,7 @@ static int rocm_q4_attn_q_b_transient_f16_head_rms_rope_tail_tensor( (x_elems + 255u) / 256u > UINT32_MAX || !cuda_u64_mul_checked(x_elems, sizeof(float), &x_bytes) || !cuda_u64_mul_checked(out_elems, sizeof(float), &out_bytes) || + !cuda_u64_mul_checked(out_elems, sizeof(__half), &out_f16_bytes) || x->bytes < x_bytes || out->bytes < out_bytes) { return rocm_q4_attn_q_b_f16_fallback(0, 1, 0); } @@ -755,10 +803,20 @@ static int rocm_q4_attn_q_b_transient_f16_head_rms_rope_tail_tensor( __half *w_f16 = NULL; __half *x_f16 = NULL; + __half *q_scratch = NULL; if (!rocm_q4_attn_q_b_transient_f16_acquire( - n_tok, &w_f16, &x_f16)) { + n_tok, f16_output, &w_f16, &x_f16, &q_scratch)) { return rocm_q4_attn_q_b_f16_fallback(0, 0, 0); } + const int use_graph_q_half = + f16_output && q_half && q_half->ptr && + q_half->bytes >= out_f16_bytes && + !rocm_q4_attn_q_b_prefixes_overlap( + q_half->ptr, out_f16_bytes, out->ptr, out_bytes) && + !rocm_q4_attn_q_b_prefixes_overlap( + q_half->ptr, out_f16_bytes, x->ptr, x_bytes); + __half *const qh = use_graph_q_half + ? (__half *)q_half->ptr : q_scratch; const char *w_q4 = rocm_q4_attn_q_b_device_resident_source( model_map, weight_offset, weight_bytes); @@ -798,6 +856,8 @@ static int rocm_q4_attn_q_b_transient_f16_head_rms_rope_tail_tensor( const float alpha = 1.0f; const float beta = 0.0f; + /* Keep the release path's F32 output by default. The explicit F16-output + * experiment uses the same boundary for cached and transient weights. */ const cublasStatus_t st = cublasGemmEx( g_cublas, CUBLAS_OP_T, @@ -813,8 +873,8 @@ static int rocm_q4_attn_q_b_transient_f16_head_rms_rope_tail_tensor( CUDA_R_16F, (int)in_dim, &beta, - out->ptr, - CUDA_R_32F, + f16_output ? (void *)qh : out->ptr, + f16_output ? CUDA_R_16F : CUDA_R_32F, (int)out_dim, CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT); @@ -822,19 +882,38 @@ static int rocm_q4_attn_q_b_transient_f16_head_rms_rope_tail_tensor( fprintf(stderr, DS4_GPU_LOG_PREFIX DS4_GPU_BLAS_NAME - " transient Q4 attn q_b F16/F16-to-F32 matmul failed: " + " transient Q4 attn q_b F16/F16-to-%s matmul failed: " "status %d\n", + f16_output ? "F16" : "F32", (int)st); rocm_q4_attn_q_b_transient_f16_release_acquired(); return rocm_q4_attn_q_b_f16_fallback(0, 0, 1); } - const int tail_ok = ds4_gpu_head_rms_norm_rope_tail_tensor( - out, n_tok, n_head, head_dim, n_rot, pos0, n_ctx_orig, - inverse, freq_base, freq_scale, ext_factor, attn_factor, - beta_fast, beta_slow, eps); + int tail_ok = 0; + if (f16_output) { + head_rms_norm_rope_tail_from_half_kernel<<<(uint32_t)head_rows, 256>>>( + (float *)out->ptr, qh, + n_tok, n_head, head_dim, n_rot, pos0, n_ctx_orig, + inverse ? 1 : 0, freq_base, freq_scale, ext_factor, + attn_factor, beta_fast, beta_slow, eps); + launch_err = cudaGetLastError(); + tail_ok = launch_err == cudaSuccess; + } else { + tail_ok = ds4_gpu_head_rms_norm_rope_tail_tensor( + out, n_tok, n_head, head_dim, n_rot, pos0, n_ctx_orig, + inverse, freq_base, freq_scale, ext_factor, attn_factor, + beta_fast, beta_slow, eps); + } rocm_q4_attn_q_b_transient_f16_release_acquired(); if (!tail_ok) { + if (f16_output) { + fprintf(stderr, + DS4_GPU_LOG_PREFIX + "transient Q4 attn q_b F16-out epilogue launch failed: %s\n", + cudaGetErrorString(launch_err)); + (void)cudaGetLastError(); + } /* GEMM was accepted and may already be executing. Returning zero * would make the graph replay native Q4 over an in-flight writer. */ return rocm_q4_attn_q_b_f16_fallback(1, 0, 1); diff --git a/rocm/ds4_rocm_q4.cuh b/rocm/ds4_rocm_q4.cuh index cf6639307..156c02f04 100644 --- a/rocm/ds4_rocm_q4.cuh +++ b/rocm/ds4_rocm_q4.cuh @@ -116,6 +116,8 @@ __global__ static void rocm_matmul_q4_K_dense_grouped_decode_kernel( enum { ROCM_Q4_PREFILL_TOKEN_TILE = 8u, ROCM_Q4_PREFILL_KBLOCK_TILE = 8u, + ROCM_Q4_PREFILL_K1024_KBLOCK_TILE = 4u, + ROCM_Q4_PREFILL_K1024_ROWS = 64u, ROCM_Q4_Q8K_WORDS = sizeof(cuda_block_q8_K) / sizeof(uint32_t), }; static_assert((sizeof(cuda_block_q8_K) % sizeof(uint32_t)) == 0u, @@ -197,6 +199,89 @@ rocm_dot_q4_K_q8_K_block8_reuse_weights( } } +/* K=1024 has exactly four Q8_K blocks. The generic TILE8 kernel leaves half + * of each eight-lane row group idle and still reserves LDS for eight blocks. + * Four-lane groups preserve the legacy block/reduction order while doubling + * the rows produced by a 256-thread workgroup and halving the LDS footprint + * to 8 tokens * 4 K blocks * 292 bytes = 9,344 bytes. */ +__device__ __forceinline__ static float +rocm_q4_K_lane4_sum_f32(float v) { + /* Build the active-lane mask relative to the physical wave. A 32-bit + * mask repeats lanes 0..31 for the upper half of an AMD wave64 and + * violates HIP's __shfl_down_sync contract even though width=4 keeps the + * data exchange inside the intended subgroup. */ + const uint32_t wave_lane = threadIdx.x & (warpSize - 1u); + const MASK_T mask = static_cast(0x0fu) << (wave_lane & ~3u); + v += __shfl_down_sync(mask, v, 2, 4); + v += __shfl_down_sync(mask, v, 1, 4); + return v; +} + +__global__ static void rocm_matmul_q4_K_prefill_k1024_tile4_kernel( + float *out, + const char *w_base, + const cuda_block_q8_K *xq, + uint64_t row_bytes, + uint32_t out_dim, + uint32_t n_tok) { + __shared__ cuda_block_q8_K sxq[ROCM_Q4_PREFILL_TOKEN_TILE] + [ROCM_Q4_PREFILL_K1024_KBLOCK_TILE]; + + const uint32_t tid = threadIdx.x; + const uint32_t lane = tid & 3u; + const uint32_t row_lane = tid >> 2u; + const uint32_t row = blockIdx.x * ROCM_Q4_PREFILL_K1024_ROWS + row_lane; + const uint32_t tok0 = blockIdx.y * ROCM_Q4_PREFILL_TOKEN_TILE; + const uint32_t nt = n_tok - tok0 < ROCM_Q4_PREFILL_TOKEN_TILE + ? n_tok - tok0 : ROCM_Q4_PREFILL_TOKEN_TILE; + const bool row_valid = row < out_dim; + const cuda_block_q4_K *wr = row_valid + ? reinterpret_cast( + w_base + (uint64_t)row * row_bytes) + : NULL; + float acc[ROCM_Q4_PREFILL_TOKEN_TILE] = { + 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, + }; + + /* The complete K dimension fits one LDS tile. Flatten the copy so + * neighboring threads read consecutive words across token/block rows. */ + const uint32_t tile_words = nt * ROCM_Q4_PREFILL_K1024_KBLOCK_TILE * + ROCM_Q4_Q8K_WORDS; + uint32_t *const sxq_words = reinterpret_cast(sxq); + for (uint32_t i = tid; i < tile_words; i += blockDim.x) { + const uint32_t block_slot = i / ROCM_Q4_Q8K_WORDS; + const uint32_t word = i - block_slot * ROCM_Q4_Q8K_WORDS; + const uint32_t p = block_slot >> 2u; + const uint32_t bb = block_slot & 3u; + const uint32_t *const src_words = + reinterpret_cast( + xq + (uint64_t)(tok0 + p) * + ROCM_Q4_PREFILL_K1024_KBLOCK_TILE + bb); + sxq_words[i] = src_words[word]; + } + __syncthreads(); + + if (row_valid) { + rocm_dot_q4_K_q8_K_block8_reuse_weights( + wr + lane, + sxq[0] + lane, sxq[1] + lane, + sxq[2] + lane, sxq[3] + lane, + sxq[4] + lane, sxq[5] + lane, + sxq[6] + lane, sxq[7] + lane, + nt, acc); + + #pragma unroll + for (uint32_t p = 0u; p < ROCM_Q4_PREFILL_TOKEN_TILE; p++) { + if (p < nt) { + const float v = rocm_q4_K_lane4_sum_f32(acc[p]); + if (lane == 0u) { + out[(uint64_t)(tok0 + p) * out_dim + row] = v; + } + } + } + } +} + __global__ static void rocm_matmul_q4_K_prefill_tile8_strided_kernel( float *out, const char *w_base, @@ -452,11 +537,27 @@ static int rocm_q4_K_prefill_tile8_required(void) { return getenv("DS4_ROCM_REQUIRE_Q4_PREFILL_TILE8") != NULL; } +static int rocm_q4_K_prefill_k1024_tile4_requested( + uint64_t blocks, + uint64_t out_dim) { + /* This is the production attn_q_b shape. Keep SSD streaming on the + * established TILE8 path: only a fully resident model can use the new + * specialization, and the dedicated switch provides a narrow rollback. */ + return !g_ssd_streaming_mode && + blocks == ROCM_Q4_PREFILL_K1024_KBLOCK_TILE && + out_dim == DS4_ROCM_Q4_ATTN_Q_B_OUT_DIM && + rocm_q4_attn_q_b_env_bool( + "DS4_ROCM_DISABLE_Q4_PREFILL_K1024_TILE4") != 1; +} + static uint64_t g_rocm_q4_prefill_tile8_dense_calls; static uint64_t g_rocm_q4_prefill_tile8_pair_calls; static uint64_t g_rocm_q4_prefill_tile8_attention_batch_calls; +static uint64_t g_rocm_q4_prefill_k1024_tile4_calls; static uint64_t g_rocm_q4_prefill_tile8_tokens; static int g_rocm_q4_prefill_tile8_report_registered; +static pthread_mutex_t g_rocm_q4_prefill_tile8_stats_mutex = + PTHREAD_MUTEX_INITIALIZER; static uint64_t g_rocm_q4_grouped_attn_a_calls; static uint64_t g_rocm_q4_grouped_attn_a_dispatches; @@ -464,20 +565,30 @@ static uint64_t g_rocm_q4_grouped_attn_a_groups; static uint64_t g_rocm_q4_grouped_attn_a_fallbacks; static uint64_t g_rocm_q4_grouped_attn_a_failures; static int g_rocm_q4_grouped_attn_a_report_registered; +static pthread_mutex_t g_rocm_q4_grouped_attn_a_stats_mutex = + PTHREAD_MUTEX_INITIALIZER; static void rocm_q4_K_grouped_attn_a_report(void) { + pthread_mutex_lock(&g_rocm_q4_grouped_attn_a_stats_mutex); + const uint64_t calls = g_rocm_q4_grouped_attn_a_calls; + const uint64_t dispatches = g_rocm_q4_grouped_attn_a_dispatches; + const uint64_t groups = g_rocm_q4_grouped_attn_a_groups; + const uint64_t fallbacks = g_rocm_q4_grouped_attn_a_fallbacks; + const uint64_t failures = g_rocm_q4_grouped_attn_a_failures; + pthread_mutex_unlock(&g_rocm_q4_grouped_attn_a_stats_mutex); fprintf(stderr, "ds4: ROCm Q4_K grouped attention-A decode stats: " "calls=%llu dispatches=%llu groups=%llu fallbacks=%llu failures=%llu\n", - (unsigned long long)g_rocm_q4_grouped_attn_a_calls, - (unsigned long long)g_rocm_q4_grouped_attn_a_dispatches, - (unsigned long long)g_rocm_q4_grouped_attn_a_groups, - (unsigned long long)g_rocm_q4_grouped_attn_a_fallbacks, - (unsigned long long)g_rocm_q4_grouped_attn_a_failures); + (unsigned long long)calls, + (unsigned long long)dispatches, + (unsigned long long)groups, + (unsigned long long)fallbacks, + (unsigned long long)failures); } static int rocm_q4_K_grouped_attn_a_result(int rc, uint32_t n_groups) { if (getenv("DS4_ROCM_Q4_GROUPED_ATTN_A_STATS") != NULL) { + pthread_mutex_lock(&g_rocm_q4_grouped_attn_a_stats_mutex); if (!g_rocm_q4_grouped_attn_a_report_registered) { g_rocm_q4_grouped_attn_a_report_registered = 1; (void)atexit(rocm_q4_K_grouped_attn_a_report); @@ -491,27 +602,40 @@ static int rocm_q4_K_grouped_attn_a_result(int rc, uint32_t n_groups) { } else { g_rocm_q4_grouped_attn_a_fallbacks++; } + pthread_mutex_unlock(&g_rocm_q4_grouped_attn_a_stats_mutex); } return rc; } static void rocm_q4_K_prefill_tile8_report(void) { + pthread_mutex_lock(&g_rocm_q4_prefill_tile8_stats_mutex); + const uint64_t dense_calls = g_rocm_q4_prefill_tile8_dense_calls; + const uint64_t pair_calls = g_rocm_q4_prefill_tile8_pair_calls; + const uint64_t attention_batch_calls = + g_rocm_q4_prefill_tile8_attention_batch_calls; + const uint64_t k1024_tile4_calls = + g_rocm_q4_prefill_k1024_tile4_calls; + const uint64_t tokens = g_rocm_q4_prefill_tile8_tokens; + pthread_mutex_unlock(&g_rocm_q4_prefill_tile8_stats_mutex); fprintf(stderr, "ds4: ROCm Q4_K prefill tile8 stats: " "dense_calls=%llu pair_calls=%llu attention_batch_calls=%llu " - "tokens=%llu\n", - (unsigned long long)g_rocm_q4_prefill_tile8_dense_calls, - (unsigned long long)g_rocm_q4_prefill_tile8_pair_calls, - (unsigned long long)g_rocm_q4_prefill_tile8_attention_batch_calls, - (unsigned long long)g_rocm_q4_prefill_tile8_tokens); + "k1024_tile4_calls=%llu tokens=%llu\n", + (unsigned long long)dense_calls, + (unsigned long long)pair_calls, + (unsigned long long)attention_batch_calls, + (unsigned long long)k1024_tile4_calls, + (unsigned long long)tokens); } static void rocm_q4_K_prefill_tile8_note( uint32_t dense_calls, uint32_t pair_calls, uint32_t attention_batch_calls, + uint32_t k1024_tile4_calls, uint64_t tokens) { if (getenv("DS4_ROCM_Q4_PREFILL_TILE8_STATS") == NULL) return; + pthread_mutex_lock(&g_rocm_q4_prefill_tile8_stats_mutex); if (!g_rocm_q4_prefill_tile8_report_registered) { g_rocm_q4_prefill_tile8_report_registered = 1; (void)atexit(rocm_q4_K_prefill_tile8_report); @@ -519,7 +643,9 @@ static void rocm_q4_K_prefill_tile8_note( g_rocm_q4_prefill_tile8_dense_calls += dense_calls; g_rocm_q4_prefill_tile8_pair_calls += pair_calls; g_rocm_q4_prefill_tile8_attention_batch_calls += attention_batch_calls; + g_rocm_q4_prefill_k1024_tile4_calls += k1024_tile4_calls; g_rocm_q4_prefill_tile8_tokens += tokens; + pthread_mutex_unlock(&g_rocm_q4_prefill_tile8_stats_mutex); } extern "C" int ds4_rocm_matmul_q4_K_tensor( @@ -565,6 +691,24 @@ extern "C" int ds4_rocm_matmul_q4_K_tensor( if (!cuda_ok(cudaGetLastError(), "q4_K dense quantize launch")) return 0; if (prefill_scope && prefill_tile8) { + if (rocm_q4_K_prefill_k1024_tile4_requested(blocks, out_dim)) { + const dim3 tiled_grid( + (unsigned)((out_dim - 1u) / + ROCM_Q4_PREFILL_K1024_ROWS + 1u), + (unsigned)((n_tok - 1u) / + ROCM_Q4_PREFILL_TOKEN_TILE + 1u), + 1u); + rocm_matmul_q4_K_prefill_k1024_tile4_kernel<<>>( + reinterpret_cast(out->ptr), wptr, xq, + row_bytes, (uint32_t)out_dim, (uint32_t)n_tok); + const int ok = cuda_ok( + cudaGetLastError(), + "q4_K dense prefill K1024 tile4 launch"); + if (ok) { + rocm_q4_K_prefill_tile8_note(1u, 0u, 0u, 1u, n_tok); + } + return ok; + } const dim3 tiled_grid((unsigned)((out_dim - 1u) / 32u + 1u), (unsigned)((n_tok - 1u) / ROCM_Q4_PREFILL_TOKEN_TILE + 1u), @@ -575,7 +719,7 @@ extern "C" int ds4_rocm_matmul_q4_K_tensor( blocks, out_dim); const int ok = cuda_ok(cudaGetLastError(), "q4_K dense prefill tile8 launch"); - if (ok) rocm_q4_K_prefill_tile8_note(1u, 0u, 0u, n_tok); + if (ok) rocm_q4_K_prefill_tile8_note(1u, 0u, 0u, 0u, n_tok); return ok; } @@ -675,7 +819,7 @@ extern "C" int ds4_gpu_matmul_q4_K_pair_tensor( (uint32_t)out1_dim, (uint32_t)n_tok); const int ok = cuda_ok(cudaGetLastError(), "q4_K dense prefill pair tile8 launch"); - if (ok) rocm_q4_K_prefill_tile8_note(0u, 1u, 0u, n_tok); + if (ok) rocm_q4_K_prefill_tile8_note(0u, 1u, 0u, 0u, n_tok); return ok; } @@ -948,6 +1092,6 @@ extern "C" int ds4_gpu_attention_output_q4_K_batch_tensor( } if (b_rc <= 0) return -1; - rocm_q4_K_prefill_tile8_note(0u, 0u, 1u, n_tokens); + rocm_q4_K_prefill_tile8_note(0u, 0u, 1u, 0u, n_tokens); return 1; } diff --git a/rocm/ds4_rocm_q4_qb_sidecar.cuh b/rocm/ds4_rocm_q4_qb_sidecar.cuh index 9b3eb66ef..aee8f4c7f 100644 --- a/rocm/ds4_rocm_q4_qb_sidecar.cuh +++ b/rocm/ds4_rocm_q4_qb_sidecar.cuh @@ -51,10 +51,12 @@ static uint64_t g_rocm_q4_attn_q_b_f16_rejects; static int g_rocm_q4_attn_q_b_f16_hard_failure; static int g_rocm_q4_attn_q_b_f16_pending_evict; /* The resident default rebuilds one layer at a time into this combined - * allocation. The first 64 MiB hold W_F16 and the suffix holds the largest - * preflighted X_F16 batch. ROCm currently submits graph work on stream 0, but - * keep the mutex through the complete dequant/copy/GEMM/epilogue enqueue - * sequence so two host callers cannot interleave reuse of either region. */ + * allocation. The first 64 MiB hold W_F16; the suffix holds the largest + * preflighted X_F16 batch and, only for the explicit F16-output experiment, + * Q_F16. ROCm currently submits graph work on stream 0, but keep the mutex + * through the complete + * dequant/copy/GEMM/epilogue enqueue sequence so two host callers cannot + * interleave reuse of any region. */ static void *g_rocm_q4_attn_q_b_transient_f16_scratch; static uint64_t g_rocm_q4_attn_q_b_transient_f16_scratch_bytes; static uint64_t g_rocm_q4_attn_q_b_transient_f16_weight_bytes; @@ -145,6 +147,16 @@ static int rocm_q4_attn_q_b_transient_f16_disabled(void) { "DS4_ROCM_DISABLE_Q4_ATTN_Q_B_TRANSIENT_F16") == 1; } +static int rocm_q4_attn_q_b_f16_output_enabled(void) { + const char *value = getenv( + "DS4_ROCM_ENABLE_Q4_ATTN_Q_B_F16_OUTPUT"); + if (!value) return 0; + while (isspace((unsigned char)*value)) value++; + if (!*value) return 0; + return rocm_q4_attn_q_b_env_bool( + "DS4_ROCM_ENABLE_Q4_ATTN_Q_B_F16_OUTPUT") == 1; +} + static uint64_t rocm_q4_attn_q_b_transient_f16_min_tokens(void) { return rocm_q4_attn_q_b_env_u64( "DS4_ROCM_Q4_ATTN_Q_B_TRANSIENT_F16_MIN_TOKENS", @@ -463,13 +475,17 @@ static int rocm_q4_attn_q_b_f16_memory_has_room( static int rocm_q4_attn_q_b_transient_f16_layout( uint64_t rows, + int include_output, uint64_t *weight_bytes_out, uint64_t *x_bytes_out, + uint64_t *q_bytes_out, uint64_t *total_bytes_out) { uint64_t weight_elems = 0; uint64_t weight_bytes = 0; uint64_t x_elems = 0; uint64_t x_bytes = 0; + uint64_t q_elems = 0; + uint64_t q_bytes = 0; uint64_t total_bytes = 0; if (rows == 0u || !cuda_u64_mul_checked(DS4_ROCM_Q4_ATTN_Q_B_IN_DIM, @@ -484,8 +500,17 @@ static int rocm_q4_attn_q_b_transient_f16_layout( total_bytes > (uint64_t)SIZE_MAX) { return 0; } + if (include_output && + (!cuda_u64_mul_checked(rows, DS4_ROCM_Q4_ATTN_Q_B_OUT_DIM, + &q_elems) || + !cuda_u64_mul_checked(q_elems, sizeof(__half), &q_bytes) || + !cuda_u64_add_checked(total_bytes, q_bytes, &total_bytes) || + total_bytes > (uint64_t)SIZE_MAX)) { + return 0; + } if (weight_bytes_out) *weight_bytes_out = weight_bytes; if (x_bytes_out) *x_bytes_out = x_bytes; + if (q_bytes_out) *q_bytes_out = q_bytes; if (total_bytes_out) *total_bytes_out = total_bytes; return 1; } @@ -561,13 +586,17 @@ static int rocm_q4_attn_q_b_transient_f16_ensure_locked( * enqueue sequence. No allocation or synchronization is permitted here. */ static int rocm_q4_attn_q_b_transient_f16_acquire( uint64_t rows, + int include_output, __half **weight_f16_out, - __half **x_f16_out) { + __half **x_f16_out, + __half **q_f16_out) { uint64_t weight_bytes = 0; + uint64_t x_bytes = 0; uint64_t total_bytes = 0; - if (!weight_f16_out || !x_f16_out || + if (!weight_f16_out || !x_f16_out || !q_f16_out || !rocm_q4_attn_q_b_transient_f16_layout( - rows, &weight_bytes, NULL, &total_bytes)) { + rows, include_output, &weight_bytes, &x_bytes, NULL, + &total_bytes)) { return 0; } pthread_mutex_lock(&g_rocm_q4_attn_q_b_transient_f16_mu); @@ -581,6 +610,9 @@ static int rocm_q4_attn_q_b_transient_f16_acquire( (__half *)g_rocm_q4_attn_q_b_transient_f16_scratch; *x_f16_out = (__half *)( (char *)g_rocm_q4_attn_q_b_transient_f16_scratch + weight_bytes); + *q_f16_out = (__half *)( + (char *)g_rocm_q4_attn_q_b_transient_f16_scratch + + weight_bytes + x_bytes); return 1; } @@ -795,8 +827,12 @@ static int rocm_q4_attn_q_b_prepare_transient_f16( uint64_t layout_weight_bytes = 0; uint64_t total_bytes = 0; + const int include_output = + rocm_q4_attn_q_b_f16_output_enabled(); if (!rocm_q4_attn_q_b_transient_f16_layout( - max_prefill_rows, &layout_weight_bytes, NULL, &total_bytes) || + max_prefill_rows, include_output, + &layout_weight_bytes, NULL, NULL, + &total_bytes) || layout_weight_bytes != weight_f16_bytes) { pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_build_mu); return 0; @@ -901,14 +937,20 @@ extern "C" int ds4_gpu_prepare_q4_attn_q_b_f16_sidecars( return required ? -1 : 0; } - /* The persistent sidecar still needs private X_F16 staging. Reuse the - * dedicated combined arena instead of the backend-global cuda_tmp buffer; - * dispatch holds transient_mu through conversion, GEMM, and epilogue. - * Keep the global lock order build -> transient -> cache. */ + /* The persistent sidecar still needs private X_F16 staging and, for the + * explicit output experiment, Q_F16. Reuse the dedicated combined arena + * instead of the backend-global cuda_tmp buffer; unlike Metal, the ROCm + * graph does not normally own a batch_q_half tensor. Dispatch holds + * transient_mu through conversion, GEMM, and epilogue. Keep the global + * lock order build -> transient -> cache. */ uint64_t scratch_weight_bytes = 0; uint64_t scratch_bytes = 0; + const int include_output = + rocm_q4_attn_q_b_f16_output_enabled(); if (!rocm_q4_attn_q_b_transient_f16_layout( - max_prefill_rows, &scratch_weight_bytes, NULL, &scratch_bytes) || + max_prefill_rows, include_output, + &scratch_weight_bytes, NULL, NULL, + &scratch_bytes) || scratch_weight_bytes != desc_f16_bytes[0]) { pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_build_mu); return required ? -1 : 0; diff --git a/scripts/environment_variables.tsv b/scripts/environment_variables.tsv index b2bba9ea3..854f7c66c 100644 --- a/scripts/environment_variables.tsv +++ b/scripts/environment_variables.tsv @@ -46,9 +46,10 @@ runtime/cuda DS4_CUDA_DECODE_HEADS8_ONLINE presence flag; unset does not force t runtime/cuda DS4_CUDA_DECODE_SCORE4 presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Select four score lanes in the CUDA decode-attention fallback kernel. ds4_cuda.cu:372 runtime/cuda DS4_CUDA_DECODE_SCORE8 presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Select eight score lanes in the CUDA decode-attention fallback kernel. ds4_cuda.cu:373 runtime/cuda DS4_CUDA_DIRECT_MODEL mixed presence/nonempty flag, default off; any defined value bypasses host caching, while backend direct lookup requires nonempty; value 0 therefore still changes behavior Use the mapped model directly and bypass selective CUDA weight caching. ds4.c:3058; ds4_cuda.cu:1250 -runtime/cuda DS4_CUDA_DISABLE_DSPARK_EXACTN value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Disable the CUDA DSpark exactn optimization. ds4.c:52270 -runtime/cuda DS4_CUDA_DISABLE_DSPARK_EXACTN_BATCH_HEAD value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Disable the CUDA DSpark exactn batch head optimization. ds4.c:37285 -runtime/cuda DS4_CUDA_DISABLE_DSPARK_EXACTN_GRAPHS value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Disable the CUDA DSpark exactn graphs optimization. ds4.c:37251 +runtime/cuda DS4_CUDA_DISABLE_BATCH_INDEXER_QUERY_PRUNE presence rollback; unset prunes unused zero-prefix indexer query work on eligible resident CUDA prefills; any value including 0 disables Restore transient indexer Q projection, RoPE, QAT, and weight projection before compressed rows exceed top-k. ds4.c:30137 +runtime/cuda DS4_CUDA_DISABLE_DSPARK_EXACTN value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Disable the CUDA DSpark exactn optimization. ds4.c:52534 +runtime/cuda DS4_CUDA_DISABLE_DSPARK_EXACTN_BATCH_HEAD value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Disable the CUDA DSpark exactn batch head optimization. ds4.c:37300 +runtime/cuda DS4_CUDA_DISABLE_DSPARK_EXACTN_GRAPHS value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables Disable the CUDA DSpark exactn graphs optimization. ds4.c:37266 runtime/cuda DS4_CUDA_DISABLE_DSPARK_NONCAUSAL_ONLINE value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on Disable the noncausal online-attention DSpark experiment. ds4_cuda.cu:21691 runtime/cuda DS4_CUDA_DISABLE_HC_NORM_MIX_FUSE presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable fused HC RMSNorm-plus-mix. ds4_cuda.cu:20905 runtime/cuda DS4_CUDA_DISABLE_HC_SPLIT_NORM_FUSED presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the fused HC split/weighted-sum/norm kernel. ds4_cuda.cu:31785 @@ -81,6 +82,7 @@ runtime/cuda DS4_CUDA_ENABLE_HC_NORM_MIX_FUSE nonempty opt-in, default off; only runtime/cuda DS4_CUDA_ENABLE_IQ2_XXS_SSD_PREFILL_MMQ false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Enable the CUDA IQ2 XXS SSD prefill MMQ experimental path. ds4_cuda.cu:4625 runtime/cuda DS4_CUDA_ENABLE_Q4_ATTN_OUT_HC_FUSE value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on Opt in to the fused Q4 attention-output/HC expansion path. ds4_cuda.cu:37375 runtime/cuda DS4_CUDA_ENABLE_Q4_ATTN_Q_B_F16_CACHE value-aware persistent-cache opt-in, default off; unset/empty/0/false/no/off is off; DISABLE cancels an optional persistent request but leaves the automatic transient path independent; REQUIRE plus DISABLE fails closed Prewarm and use persistent resident F16 sidecars for eligible single-GPU Q4_K attn_q_b prefills. ds4_cuda.cu:2490 +runtime/cuda DS4_CUDA_ENABLE_Q4_ATTN_Q_B_F16_OUTPUT value-aware experimental opt-in; unset/empty/0/false/no/off keeps the release F32 projection boundary; other nonempty values enable Write eligible resident Q4_K attn_q_b GEMM output in F16 and run the half-input norm/RoPE epilogue; SSD remains excluded. ds4_cuda.cu:2523 runtime/cuda DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_BATCH value-aware opt-in, default off; nonempty value other than exact 0 enables; rollback wins Enable flattened grouped attention-A MMQ for two-to-eight-token GB10 batches. cuda/mmq/ds4_mmq.cu:4303 runtime/cuda DS4_CUDA_ENABLE_Q4_K1024_PERSISTENT presence flag, default off; any defined value including 0 requests the path; rollback wins Enable the GB10 persistent-CTA kernel for M=32768, N=1, K=1024 Q4. cuda/mmq/ds4_mmq.cu:3905 runtime/cuda DS4_CUDA_ENABLE_Q8_FOLD strict flag, default off; only exact value 1 enables; overridden by DS4_CUDA_NO_Q8_FOLD Enable one-shot producer-to-consumer reuse of freshly quantized Q8_1 data. ds4_cuda.cu:785 @@ -541,9 +543,11 @@ runtime/metal DS4_METAL_DISABLE_MOE_MM_ID_PAIR_SWIGLU presence rollback; unset: runtime/metal DS4_METAL_DISABLE_MOE_MM_ID_USE_RESOURCES presence rollback; unset: automatic/default path; any value including 0 disables Disables MoE MM ID use resources. ds4_metal.m:35169 runtime/metal DS4_METAL_DISABLE_MXFP4_SELECTED_EXPERT_VIEWS presence rollback; unset: automatic/default path; any value including 0 disables Disables MXFP4 selected expert views. ds4.c:21173 runtime/metal DS4_METAL_DISABLE_PERSISTENT_ZERO_ATTN_MASK presence rollback; unset: automatic/default path; any value including 0 disables Disables persistent zero attn mask. ds4_metal.m:31480 -runtime/metal DS4_METAL_DISABLE_PRE_M5_ATTN_INV_ROPE_FUSE presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 attn inv RoPE fuse. ds4.c:22638 +runtime/metal DS4_METAL_DISABLE_PREFILL_FLUSH_PROGRESS presence rollback; unset commits eligible resident 32..2048-token per-layer progress batches without host drains; any value including 0 disables Restore synchronous per-layer drains when display progress is active. ds4.c:35924 +runtime/metal DS4_METAL_DISABLE_PRE_M5_ATTN_INV_ROPE_FUSE presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 attn inv RoPE fuse. ds4.c:22653 runtime/metal DS4_METAL_DISABLE_PRE_M5_ATTN_OUT_LOW_Q8_STATIC presence rollback; unset: exact fixed-shape kernel is automatic on eligible pre-M5 Flash decode; any value including 0 disables Restores the generic Q8 attention-output low projection kernel. ds4_metal.m:27987 -runtime/metal DS4_METAL_DISABLE_PRE_M5_BATCH_INDEXER_QUERY_PRUNE presence rollback; unset: unused zero-prefix indexer queries are pruned before compressed rows exceed top-k; any value including 0 disables Restores transient indexer query and weight dispatches during eligible pre-M5 prefill. ds4.c:30086 +runtime/metal DS4_METAL_DISABLE_PRE_M5_BATCH_ATTN_OUT_HC_FUSION presence rollback; unset enables the exact resident pre-M5 Q8_0 or Q4_K output-B-to-HC4 tail when all shape and safety gates pass; any value including 0 disables Restore the separate attention output-B materialization and HC expansion dispatches. ds4_metal.m:30581; ds4_metal.m:31504 +runtime/metal DS4_METAL_DISABLE_PRE_M5_BATCH_INDEXER_QUERY_PRUNE presence rollback; unset: unused zero-prefix indexer queries are pruned before compressed rows exceed top-k; any value including 0 disables Restores transient indexer query and weight dispatches during eligible pre-M5 prefill. ds4.c:30101 runtime/metal DS4_METAL_DISABLE_PRE_M5_COMPRESSOR_EXACT_POOL_RATIO4 presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 compressor exact pool ratio4. ds4_metal.m:26383 runtime/metal DS4_METAL_DISABLE_PRE_M5_COMPRESSOR_EXACT_REDUCTION_FUSION presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 compressor exact reduction fusion. ds4_metal.m:25907 runtime/metal DS4_METAL_DISABLE_PRE_M5_COMPRESSOR_QUAD_STORE presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 compressor quad store. ds4.c:23535 @@ -853,9 +857,11 @@ runtime/metal DS4_METAL_REQUIRE_GATHERED_KV_STAGE presence strict check; unset: runtime/metal DS4_METAL_REQUIRE_IQ2_XXS_SSD_PREFILL_MM value-aware boolean; default implicit fail-closed only with complete selected-address domain; explicit 1 is strict, 0 permits fallback Makes eligible IQ2_XXS/Q2_K grouped SSD-prefill MM fail closed. ds4_metal.m:44782 runtime/metal DS4_METAL_REQUIRE_M1_IQ2_MID_ONLY presence strict check; unset: fallback allowed; any value including 0 requires the path Requires M1 IQ2 mid only and makes eligible fallback fail closed. ds4_metal.m:42074 runtime/metal DS4_METAL_REQUIRE_OUTPUT_HC_WEIGHTS4 presence strict check; unset: fallback allowed; any value including 0 requires the path Requires output HC weights4 and makes eligible fallback fail closed. ds4_metal.m:46789 -runtime/metal DS4_METAL_REQUIRE_Q4_ATTN_OUT_B_F16_RHS value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Requires the resident pre-M5 Q4 attention output-B F16 RHS path and makes an ineligible or disabled candidate fail closed. ds4_metal.m:28531 +runtime/metal DS4_METAL_REQUIRE_PRE_M5_BATCH_ATTN_OUT_HC_FUSION presence strict check; unset permits fallback; any value including 0 requires the exact resident Q8_0 output-B-to-HC4 tail Fail closed instead of replaying the separate Q8 output-B and HC path when the fused candidate is ineligible or fails. ds4_metal.m:30536 +runtime/metal DS4_METAL_REQUIRE_Q4_ATTN_OUT_B_F16_RHS value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Requires the resident pre-M5 Q4 attention output-B F16 RHS path and makes an ineligible or disabled candidate fail closed. ds4_metal.m:28551 runtime/metal DS4_METAL_REQUIRE_Q4_ATTN_OUT_TINY_BATCH value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Requires Q4 attn out tiny batch and makes eligible fallback fail closed. ds4_metal.m:28373 runtime/metal DS4_METAL_REQUIRE_Q4_ATTN_Q_B_F16_CACHE value-aware boolean; unset/0: fallback allowed; empty/1/true/yes/on requires; DISABLE fails closed Requires the sidecar for resident, or explicitly enabled SSD-hybrid, pre-M5 Q4_K Flash attn_q_b batches at or above the configured minimum; decode and below-min batches remain non-candidates. ds4_metal.m:25801; ds4_metal.m:26129; ds4_metal.m:26704 +runtime/metal DS4_METAL_REQUIRE_Q4_BATCH_ATTN_OUT_HC_FUSION presence strict check; unset permits fallback; any value including 0 requires the exact resident Q4_K/F16-RHS output-B-to-HC4 tail Fail closed instead of replaying the separate Q4 output-B and HC path when the fused candidate is ineligible or fails. ds4_metal.m:31455 runtime/metal DS4_METAL_REQUIRE_Q4_SSD_PREFILL_ATTN_OUT_EXACTN value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Requires Q4 SSD prefill attn out exactn and makes eligible fallback fail closed. ds4_metal.m:28089 runtime/metal DS4_METAL_REQUIRE_Q4_SSD_PREFILL_ATTN_OUT_SCALE_META value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Requires shared scale/min metadata in the Q4 SSD prefill attention-output exact-N kernel and makes fallback fail closed. ds4_metal.m:28209 runtime/metal DS4_METAL_REQUIRE_Q4_SSD_SESSION_UNION nonempty boolean; unset/empty or exact 0: off; every other value: on Requires Q4 SSD session union and makes eligible fallback fail closed. ds4.c:65164 @@ -934,21 +940,23 @@ runtime/mtp DS4_MTP_FULL_LOGITS Pure presence flag; default off. When set, legac runtime/mtp DS4_MTP_MIN_MARGIN non-negative float; default engine --mtp-margin value Set confidence margin threshold for speculative MTP verification. ds4.c:70893 runtime/mtp DS4_MTP_PROBE Pure presence flag; default off. For legacy MTP it prepares drafts even when configured depth<=1, compares the previous draft with the next committed token, and prints cumulative hit counts/failures; generated output is unchanged. Measure legacy MTP next-token draft accuracy without enabling speculative acceptance. ds4.c:64990 runtime/mtp DS4_MTP_SPEC_DISABLE Pure presence flag: any defined value, including empty or "0", disables MTP speculative argmax in CLI/chat/server loops. Unset permits it for greedy temperature<=0 generation with draft depth>1; unrelated split-KV speculation can still be independently requested. Fall back from MTP multi-token speculative evaluation to normal one-token session evaluation. ds4_cli.c:580 -runtime/mtp DS4_MTP_SPEC_LOG Pure presence flag; default off. It only emits diagnostics for first-draft misses, exact/generic verifier failures and sequential fallback misses/acceptance outcomes; it does not select a verifier. Trace why MTP drafts were accepted, partially accepted, rejected, or sent to sequential fallback. ds4.c:70916 -runtime/mtp DS4_MTP_STRICT Pure presence flag; engine quality mode also enables strictness automatically. Strict mode skips the non-strict low-margin shortcut, selects exact decode-2 for two drafts unless DS4_MTP_BATCH_VERIFY is set, and disables default prefix-1 capture unless explicitly restored. Force the exact/quality-oriented MTP verification policy on otherwise non-quality runs. ds4.c:70891 -runtime/mtp DS4_MTP_TIMING Pure presence flag; default off. When set, timestamps and prints draft, snapshot, verifier, prefix/replay and total durations for the path taken; algorithm selection is otherwise unchanged. Profile end-to-end MTP speculative decoding and separate draft, verification and state-commit costs. ds4.c:70899 -runtime/rocm DS4_ROCM_DECODE_STAGE_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm decode stage profile. ds4.c:18244 -runtime/rocm DS4_ROCM_DECODE_STAGE_PROFILE_LAYER layer filter subordinate to DS4_ROCM_DECODE_STAGE_PROFILE; unset or whitespace-only: all layers allowed by the parent flag; otherwise the whitespace-trimmed value must be a complete base-10 strtoul result <= UINT32_MAX equal to the current layer; invalid values match none Restricts the ROCm decode stage profiler to one layer; it does not enable profiling by itself. ds4.c:29116 -runtime/rocm DS4_ROCM_DISABLE_GLM_STREAMING_PREFILL_FULL_LAYER integer selector/tuning value; unset or invalid uses internal automatic/default value Disable/roll back rocm disable glm streaming prefill full layer. ds4.c:42709 -runtime/rocm DS4_ROCM_DISABLE_GLM_STREAMING_PREFILL_FULL_LAYER_PREPARE presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable glm streaming prefill full layer prepare. ds4.c:42727 -runtime/rocm DS4_ROCM_DISABLE_GLM_STREAMING_PREFILL_SELECTED_ASYNC_LOAD presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable glm streaming prefill selected async load. ds4.c:46420 -runtime/rocm DS4_ROCM_DISABLE_GLM_STREAMING_SELECTED_ASYNC_LOAD presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable glm streaming selected async load. ds4.c:44328 -runtime/rocm DS4_ROCM_DISABLE_IQ2_SELECTED_EXPERT_VIEWS presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable iq2 selected expert views. ds4.c:21079 +runtime/mtp DS4_MTP_SPEC_LOG Pure presence flag; default off. It only emits diagnostics for first-draft misses, exact/generic verifier failures and sequential fallback misses/acceptance outcomes; it does not select a verifier. Trace why MTP drafts were accepted, partially accepted, rejected, or sent to sequential fallback. ds4.c:71542 +runtime/mtp DS4_MTP_STRICT Pure presence flag; engine quality mode also enables strictness automatically. Strict mode skips the non-strict low-margin shortcut, selects exact decode-2 for two drafts unless DS4_MTP_BATCH_VERIFY is set, and disables default prefix-1 capture unless explicitly restored. Force the exact/quality-oriented MTP verification policy on otherwise non-quality runs. ds4.c:71517 +runtime/mtp DS4_MTP_TIMING Pure presence flag; default off. When set, timestamps and prints draft, snapshot, verifier, prefix/replay and total durations for the path taken; algorithm selection is otherwise unchanged. Profile end-to-end MTP speculative decoding and separate draft, verification and state-commit costs. ds4.c:71525 +runtime/rocm DS4_ROCM_DECODE_STAGE_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm decode stage profile. ds4.c:18259 +runtime/rocm DS4_ROCM_DECODE_STAGE_PROFILE_LAYER layer filter subordinate to DS4_ROCM_DECODE_STAGE_PROFILE; unset or whitespace-only: all layers allowed by the parent flag; otherwise the whitespace-trimmed value must be a complete base-10 strtoul result <= UINT32_MAX equal to the current layer; invalid values match none Restricts the ROCm decode stage profiler to one layer; it does not enable profiling by itself. ds4.c:29131 +runtime/rocm DS4_ROCM_DISABLE_BATCH_INDEXER_QUERY_PRUNE presence rollback; unset prunes unused zero-prefix indexer query work on eligible resident ROCm prefills; any value including 0 disables Restore transient indexer Q projection, RoPE, QAT, and weight projection before compressed rows exceed top-k. ds4.c:30131 +runtime/rocm DS4_ROCM_DISABLE_GLM_STREAMING_PREFILL_FULL_LAYER integer selector/tuning value; unset or invalid uses internal automatic/default value Disable/roll back rocm disable glm streaming prefill full layer. ds4.c:42970 +runtime/rocm DS4_ROCM_DISABLE_GLM_STREAMING_PREFILL_FULL_LAYER_PREPARE presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable glm streaming prefill full layer prepare. ds4.c:42988 +runtime/rocm DS4_ROCM_DISABLE_GLM_STREAMING_PREFILL_SELECTED_ASYNC_LOAD presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable glm streaming prefill selected async load. ds4.c:46681 +runtime/rocm DS4_ROCM_DISABLE_GLM_STREAMING_SELECTED_ASYNC_LOAD presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable glm streaming selected async load. ds4.c:44589 +runtime/rocm DS4_ROCM_DISABLE_IQ2_SELECTED_EXPERT_VIEWS presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable iq2 selected expert views. ds4.c:21094 runtime/rocm DS4_ROCM_DISABLE_IQ2_STREAM_ADDR_TABLE presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable iq2 stream addr table. ds4.c:6548 runtime/rocm DS4_ROCM_DISABLE_Q4_ATTN_Q_B_F16_CACHE value-aware rollback; unset/0/false/no/off keeps the experiment available, empty or any other value disables it Disable the resident ROCm Q4_K attn_q_b-to-F16 prefill cache even when ENABLE or REQUIRE is set. rocm/ds4_rocm_q4_qb_sidecar.cuh:140 runtime/rocm DS4_ROCM_DISABLE_Q4_ATTN_Q_B_TRANSIENT_F16 value-aware rollback; unset/0/false/no/off keeps the automatic path eligible, empty or any other non-false value disables it Disable per-layer transient Q4_K attn_q_b-to-F16 scratch for device-image or device-range-resident weights; the existing resident-cache controls remain independent and otherwise eligible calls use native Q4. rocm/ds4_rocm_q4_qb_sidecar.cuh:145 runtime/rocm DS4_ROCM_DISABLE_Q4_DENSE_PAIR presence rollback; unset leaves opt-in policy unchanged Disable/roll back rocm disable q4 dense pair. rocm/ds4_rocm_q4.cuh:435 runtime/rocm DS4_ROCM_DISABLE_Q4_GROUPED_ATTN_A presence rollback; DISABLE wins over enable/require Disable/roll back rocm disable q4 grouped attn a. rocm/ds4_rocm_q4.cuh:700 +runtime/rocm DS4_ROCM_DISABLE_Q4_PREFILL_K1024_TILE4 value-aware rollback; unset/0/false/no/off keeps the exact K=1024 four-block kernel enabled; empty or any other value disables Restore the generic eight-block Q4_K tiled-prefill kernel for K=1024. rocm/ds4_rocm_q4.cuh:550 runtime/rocm DS4_ROCM_DISABLE_Q4_PREFILL_TILE8 presence rollback; TILE8 is default for 9..4096 tokens Disable/roll back rocm disable q4 prefill tile8. rocm/ds4_rocm_q4.cuh:448 runtime/rocm DS4_ROCM_DISABLE_Q4_SELECTED_EXPERT_VIEWS presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable q4 selected expert views. ds4.c:21135 runtime/rocm DS4_ROCM_DISABLE_RESIDENT_IQ2_SORTED presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable resident iq2 sorted. rocm/ds4_rocm_moe_launch.cuh:747 @@ -985,6 +993,7 @@ runtime/rocm DS4_ROCM_ENABLE_MXFP4_ROW64 presence opt-in; unset=off; any defined runtime/rocm DS4_ROCM_ENABLE_MXFP4_TILE32 presence opt-in; unset=off; any defined value including empty or 0 enables the candidate when the MXFP4 sorted-tile path has at least 32 tokens and the expert intermediate dimension is divisible by 32 Select the ROCm MXFP4 gate/up tile32 kernel, reusing each loaded expert-weight chunk across as many as 32 tokens. rocm/ds4_rocm_moe_launch.cuh:786 runtime/rocm DS4_ROCM_ENABLE_MXFP4_TILE4 presence opt-in; unset=off; any defined value including empty or 0 enables the candidate when the MXFP4 sorted-tile path has at least 5 tokens and neither TILE32 nor LDSB is selected Select the ROCm MXFP4 gate/up tile4 occupancy variant, reducing staged-activation LDS per block. rocm/ds4_rocm_moe_launch.cuh:794 runtime/rocm DS4_ROCM_ENABLE_Q4_ATTN_Q_B_F16_CACHE value-aware persistent-cache opt-in, default off; unset/0/false/no/off is off; DISABLE cancels an optional persistent request but leaves the automatic transient path independent; REQUIRE plus DISABLE fails closed Prewarm and use persistent resident F16 sidecars for eligible ROCm Q4_K attn_q_b prefills. rocm/ds4_rocm_q4_qb_sidecar.cuh:130 +runtime/rocm DS4_ROCM_ENABLE_Q4_ATTN_Q_B_F16_OUTPUT value-aware experimental opt-in; unset/empty/0/false/no/off keeps the release F32 projection boundary; other nonempty values enable Write eligible resident Q4_K attn_q_b GEMM output in F16 and run the half-input norm/RoPE epilogue; SSD remains excluded. rocm/ds4_rocm_q4_qb_sidecar.cuh:152 runtime/rocm DS4_ROCM_ENABLE_Q4_DENSE_PAIR presence opt-in; unset=off; DISABLE takes precedence Enable rocm enable q4 dense pair. rocm/ds4_rocm_q4.cuh:434 runtime/rocm DS4_ROCM_ENABLE_Q4_GROUPED_ATTN_A presence opt-in; unset=off unless REQUIRE; DISABLE wins Enable rocm enable q4 grouped attn a. rocm/ds4_rocm_q4.cuh:704 runtime/rocm DS4_ROCM_ENABLE_STREAMING_FULL_EXPERT_ADDR_TABLE presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming full expert addr table. ds4.c:18240 diff --git a/tests/ds4_test.c b/tests/ds4_test.c index 7dd2cf8df..63574d9c4 100644 --- a/tests/ds4_test.c +++ b/tests/ds4_test.c @@ -863,6 +863,185 @@ static void test_metal_store_raw_kv_batch_wrap(void) { ds4_gpu_tensor_free(raw); } +#if defined(__APPLE__) +typedef struct { + volatile uint32_t *sequence_count; + volatile uint32_t *sequence; + volatile uint32_t calls; + uint32_t marker; +} test_metal_progress_callback_ctx; + +static void test_metal_progress_callback(void *opaque) { + test_metal_progress_callback_ctx *ctx = opaque; + const uint32_t slot = __atomic_fetch_add( + ctx->sequence_count, 1u, __ATOMIC_ACQ_REL); + if (slot < 2u) { + __atomic_store_n( + ctx->sequence + slot, ctx->marker, __ATOMIC_RELEASE); + } + (void)__atomic_fetch_add(&ctx->calls, 1u, __ATOMIC_ACQ_REL); +} + +static void test_metal_flush_commands_progress_exact(void) { + enum { + n = 257, + guard = 19, + alloc_n = n + guard, + }; + const uint64_t active_bytes = (uint64_t)n * sizeof(float); + const uint64_t alloc_bytes = (uint64_t)alloc_n * sizeof(float); + const uint32_t tmp_poison = 0x7fc1a500u; + const uint32_t out_poison = 0x7fc25a00u; + float a_host[alloc_n]; + float b_host[alloc_n]; + float c_host[alloc_n]; + float tmp_host[alloc_n]; + float out_host[alloc_n]; + float expected_tmp[n]; + float expected_out[n]; + + for (uint32_t i = 0; i < alloc_n; i++) { + a_host[i] = (float)((int)(i % 17u) - 8) * 0.5f; + b_host[i] = (float)((int)((i * 5u) % 23u) - 11) * 0.25f; + c_host[i] = (float)((int)((i * 7u) % 29u) - 14) * 0.125f; + const uint32_t tmp_bits = tmp_poison + (i & 0xffu); + const uint32_t out_bits = out_poison + (i & 0xffu); + memcpy(tmp_host + i, &tmp_bits, sizeof(tmp_bits)); + memcpy(out_host + i, &out_bits, sizeof(out_bits)); + if (i < n) { + expected_tmp[i] = a_host[i] + b_host[i]; + expected_out[i] = expected_tmp[i] + c_host[i]; + } + } + + ds4_gpu_tensor *a = ds4_gpu_tensor_alloc(alloc_bytes); + ds4_gpu_tensor *b = ds4_gpu_tensor_alloc(alloc_bytes); + ds4_gpu_tensor *c = ds4_gpu_tensor_alloc(alloc_bytes); + ds4_gpu_tensor *tmp_base = ds4_gpu_tensor_alloc(alloc_bytes); + ds4_gpu_tensor *out_base = ds4_gpu_tensor_alloc(alloc_bytes); + ds4_gpu_tensor *tmp = tmp_base + ? ds4_gpu_tensor_view(tmp_base, 0u, active_bytes) : NULL; + ds4_gpu_tensor *out = out_base + ? ds4_gpu_tensor_view(out_base, 0u, active_bytes) : NULL; + TEST_ASSERT(a && b && c && tmp_base && out_base && tmp && out); + + volatile uint32_t sequence_count = 0u; + volatile uint32_t sequence[2] = {0u, 0u}; + test_metal_progress_callback_ctx callback0 = { + .sequence_count = &sequence_count, + .sequence = sequence, + .calls = 0u, + .marker = 1u, + }; + test_metal_progress_callback_ctx callback1 = { + .sequence_count = &sequence_count, + .sequence = sequence, + .calls = 0u, + .marker = 2u, + }; + + bool submitted = false; + if (a && b && c && tmp_base && out_base && tmp && out) { + TEST_ASSERT(ds4_gpu_tensor_write(a, 0u, a_host, alloc_bytes) != 0); + TEST_ASSERT(ds4_gpu_tensor_write(b, 0u, b_host, alloc_bytes) != 0); + TEST_ASSERT(ds4_gpu_tensor_write(c, 0u, c_host, alloc_bytes) != 0); + TEST_ASSERT(ds4_gpu_tensor_write( + tmp_base, 0u, tmp_host, alloc_bytes) != 0); + TEST_ASSERT(ds4_gpu_tensor_write( + out_base, 0u, out_host, alloc_bytes) != 0); + TEST_ASSERT(ds4_gpu_commands_active() == 0); + + const int begun = ds4_gpu_begin_commands(); + TEST_ASSERT(begun != 0); + const int first = begun + ? ds4_gpu_add_tensor(tmp, a, b, n) : 0; + TEST_ASSERT(first != 0); + const int flushed0 = first + ? ds4_gpu_flush_commands_progress( + test_metal_progress_callback, &callback0) + : 0; + TEST_ASSERT(flushed0 != 0); + const int second = flushed0 + ? ds4_gpu_add_tensor(out, tmp, c, n) : 0; + TEST_ASSERT(second != 0); + const int flushed1 = second + ? ds4_gpu_flush_commands_progress( + test_metal_progress_callback, &callback1) + : 0; + TEST_ASSERT(flushed1 != 0); + submitted = flushed1 != 0; + + const int drained = ds4_gpu_commands_active() + ? ds4_gpu_end_commands() : 0; + TEST_ASSERT(drained != 0); + TEST_ASSERT(ds4_gpu_commands_active() == 0); + } + + test_float_compare_stats tmp_stats = {0}; + test_float_compare_stats out_stats = {0}; + size_t guard_mismatches = 0u; + if (submitted && tmp_base && out_base) { + TEST_ASSERT(ds4_gpu_tensor_read( + tmp_base, 0u, tmp_host, alloc_bytes) != 0); + TEST_ASSERT(ds4_gpu_tensor_read( + out_base, 0u, out_host, alloc_bytes) != 0); + tmp_stats = test_compare_float_bits(expected_tmp, tmp_host, n); + out_stats = test_compare_float_bits(expected_out, out_host, n); + for (uint32_t i = n; i < alloc_n; i++) { + uint32_t tmp_bits = 0u; + uint32_t out_bits = 0u; + memcpy(&tmp_bits, tmp_host + i, sizeof(tmp_bits)); + memcpy(&out_bits, out_host + i, sizeof(out_bits)); + if (tmp_bits != tmp_poison + (i & 0xffu) || + out_bits != out_poison + (i & 0xffu)) { + guard_mismatches++; + } + } + } + + const uint32_t calls0 = + __atomic_load_n(&callback0.calls, __ATOMIC_ACQUIRE); + const uint32_t calls1 = + __atomic_load_n(&callback1.calls, __ATOMIC_ACQUIRE); + const uint32_t completed = + __atomic_load_n(&sequence_count, __ATOMIC_ACQUIRE); + const uint32_t observed0 = + __atomic_load_n(sequence + 0u, __ATOMIC_ACQUIRE); + const uint32_t observed1 = + __atomic_load_n(sequence + 1u, __ATOMIC_ACQUIRE); + const bool distinct_contexts = + (observed0 == 1u && observed1 == 2u) || + (observed0 == 2u && observed1 == 1u); + fprintf(stderr, + "ds4-test: Metal progress flush exact callbacks=%u/%u " + "completed=%u order=%u,%u tmp=%zu out=%zu guard=%zu\n", + calls0, calls1, completed, observed0, observed1, + tmp_stats.mismatch_count, out_stats.mismatch_count, + guard_mismatches); + TEST_ASSERT(calls0 == 1u); + TEST_ASSERT(calls1 == 1u); + TEST_ASSERT(completed == 2u); + /* Completion-handler ordering across different command buffers is not a + * public contract. The dependent tensor result proves GPU submission + * order; here only prove that both distinct contexts were delivered. */ + TEST_ASSERT(distinct_contexts); + TEST_ASSERT(tmp_stats.mismatch_count == 0u && tmp_stats.max_ulp == 0u); + TEST_ASSERT(out_stats.mismatch_count == 0u && out_stats.max_ulp == 0u); + TEST_ASSERT(guard_mismatches == 0u); + + if (ds4_gpu_commands_active()) { + TEST_ASSERT(ds4_gpu_end_commands() != 0); + } + ds4_gpu_tensor_free(out); + ds4_gpu_tensor_free(tmp); + ds4_gpu_tensor_free(out_base); + ds4_gpu_tensor_free(tmp_base); + ds4_gpu_tensor_free(c); + ds4_gpu_tensor_free(b); + ds4_gpu_tensor_free(a); +} +#endif + static void test_dspark_cache_window_crop(void) { TEST_ASSERT(ds4_test_dspark_cache_window_crop()); } @@ -5630,6 +5809,516 @@ static void test_metal_zero_prefix_prefill_mask_cache_exact(void) { #endif #if defined(__APPLE__) +typedef enum { + TEST_METAL_ATTN_OUT_HC_Q8, + TEST_METAL_ATTN_OUT_HC_Q4, +} test_metal_attn_out_hc_kind; + +static uint32_t test_metal_attn_out_hc_poison_bits( + uint32_t tag, + uint64_t index) { + return 0x7fc00000u | + ((tag & 0x1fu) << 16u) | + (uint32_t)((index + 1u) & 0xffffu); +} + +static void test_metal_attn_out_hc_fill_poison( + float *dst, + uint64_t count, + uint32_t tag) { + for (uint64_t i = 0; i < count; i++) { + const uint32_t bits = test_metal_attn_out_hc_poison_bits(tag, i); + memcpy(dst + i, &bits, sizeof(bits)); + } +} + +static size_t test_metal_attn_out_hc_canary_mismatches( + const float *values, + uint64_t begin, + uint64_t end, + uint32_t tag) { + size_t mismatches = 0u; + for (uint64_t i = begin; i < end; i++) { + uint32_t bits = 0u; + memcpy(&bits, values + i, sizeof(bits)); + if (bits != test_metal_attn_out_hc_poison_bits(tag, i)) { + mismatches++; + } + } + return mismatches; +} + +static size_t test_metal_attn_out_hc_poisoned_values( + const float *values, + uint64_t count, + uint32_t tag) { + size_t poisoned = 0u; + for (uint64_t i = 0; i < count; i++) { + uint32_t bits = 0u; + memcpy(&bits, values + i, sizeof(bits)); + if (bits == test_metal_attn_out_hc_poison_bits(tag, i)) { + poisoned++; + } + } + return poisoned; +} + +static int test_metal_batch_attn_out_hc_fused_call( + test_metal_attn_out_hc_kind kind, + ds4_gpu_tensor *out, + ds4_gpu_tensor *out_hc, + const ds4_gpu_tensor *residual_hc, + const ds4_gpu_tensor *split, + ds4_gpu_tensor *low, + ds4_gpu_tensor *group_tmp, + ds4_gpu_tensor *low_tmp, + const void *model_map, + uint64_t model_size, + uint64_t out_b_offset, + const ds4_gpu_tensor *heads, + uint32_t n_tokens) { + const uint32_t group_dim = 4096u; + const uint32_t rank = 1024u; + const uint32_t n_groups = 8u; + const uint32_t out_dim = 4096u; + const uint32_t n_hc = 4u; + if (kind == TEST_METAL_ATTN_OUT_HC_Q4) { + return ds4_gpu_attention_output_q4_K_batch_hc_tensor( + out, out_hc, residual_hc, split, low, group_tmp, low_tmp, + model_map, model_size, 0u, out_b_offset, 12u, + group_dim, rank, n_groups, out_dim, heads, n_tokens, n_hc); + } + return ds4_gpu_attention_output_q8_batch_hc_tensor( + out, out_hc, residual_hc, split, low, group_tmp, low_tmp, + model_map, model_size, 0u, out_b_offset, + group_dim, rank, n_groups, out_dim, heads, n_tokens, n_hc); +} + +static void test_metal_batch_attn_out_hc_fusion_exact_case( + test_metal_attn_out_hc_kind kind) { + const bool q4 = kind == TEST_METAL_ATTN_OUT_HC_Q4; + const uint32_t n_tokens = 32u; + const uint32_t alloc_tokens = n_tokens + 1u; + const uint32_t group_dim = 4096u; + const uint32_t rank = 1024u; + const uint32_t n_groups = 8u; + const uint32_t low_dim = n_groups * rank; + const uint32_t out_dim = 4096u; + const uint32_t n_hc = 4u; + const uint64_t mix_hc = 2ull * n_hc + (uint64_t)n_hc * n_hc; + const uint64_t page = (uint64_t)getpagesize(); + const uint64_t row_a_bytes = q4 + ? (uint64_t)(group_dim / 256u) * 144u + : (uint64_t)(group_dim / 32u) * 34u; + const uint64_t row_b_bytes = q4 + ? (uint64_t)(low_dim / 256u) * 144u + : (uint64_t)(low_dim / 32u) * 34u; + const uint64_t out_a_bytes = (uint64_t)low_dim * row_a_bytes; + const uint64_t out_b_offset = test_round_up_u64(out_a_bytes, page); + const uint64_t out_b_bytes = (uint64_t)out_dim * row_b_bytes; + const uint64_t model_bytes = + test_round_up_u64(out_b_offset + out_b_bytes, page); + const uint64_t heads_count = + (uint64_t)alloc_tokens * n_groups * group_dim; + const uint64_t low_count = (uint64_t)alloc_tokens * low_dim; + const uint64_t out_count = (uint64_t)alloc_tokens * out_dim; + const uint64_t hc_count = + (uint64_t)alloc_tokens * n_hc * out_dim; + const uint64_t split_count = (uint64_t)alloc_tokens * mix_hc; + const uint64_t active_heads_count = + (uint64_t)n_tokens * n_groups * group_dim; + const uint64_t active_low_count = (uint64_t)n_tokens * low_dim; + const uint64_t active_out_count = (uint64_t)n_tokens * out_dim; + const uint64_t active_hc_count = + (uint64_t)n_tokens * n_hc * out_dim; + const uint64_t active_split_count = (uint64_t)n_tokens * mix_hc; + const uint64_t heads_bytes = heads_count * sizeof(float); + const uint64_t low_bytes = low_count * sizeof(float); + const uint64_t out_bytes = out_count * sizeof(float); + const uint64_t hc_bytes = hc_count * sizeof(float); + const uint64_t split_bytes = split_count * sizeof(float); + const uint64_t active_heads_bytes = active_heads_count * sizeof(float); + const uint64_t active_low_bytes = active_low_count * sizeof(float); + const uint64_t active_out_bytes = active_out_count * sizeof(float); + const uint64_t active_hc_bytes = active_hc_count * sizeof(float); + const uint64_t active_split_bytes = active_split_count * sizeof(float); + const uint64_t scratch_bytes = + (uint64_t)alloc_tokens * low_dim * sizeof(uint16_t); + enum { + ref_low_tag = 1u, + fused_low_tag = 2u, + ref_out_tag = 3u, + fused_out_tag = 4u, + ref_hc_tag = 5u, + fused_hc_tag = 6u, + }; + + const char *disable_fusion_env = + "DS4_METAL_DISABLE_PRE_M5_BATCH_ATTN_OUT_HC_FUSION"; + const char *require_q8_env = + "DS4_METAL_REQUIRE_PRE_M5_BATCH_ATTN_OUT_HC_FUSION"; + const char *require_q4_env = + "DS4_METAL_REQUIRE_Q4_BATCH_ATTN_OUT_HC_FUSION"; + const char *require_q4_rhs_env = + "DS4_METAL_REQUIRE_Q4_ATTN_OUT_B_F16_RHS"; + const char *disable_q4_rhs_env = + "DS4_METAL_DISABLE_Q4_ATTN_OUT_B_F16_RHS"; + const char *stage_profile_env = "DS4_METAL_ATTN_OUT_STAGE_PROFILE"; + const char *q8_profile_env = "DS4_METAL_Q8_PREFILL_PROFILE"; + char *saved_disable_fusion = test_save_env(disable_fusion_env); + char *saved_require_q8 = test_save_env(require_q8_env); + char *saved_require_q4 = test_save_env(require_q4_env); + char *saved_require_q4_rhs = test_save_env(require_q4_rhs_env); + char *saved_disable_q4_rhs = test_save_env(disable_q4_rhs_env); + char *saved_stage_profile = test_save_env(stage_profile_env); + char *saved_q8_profile = test_save_env(q8_profile_env); + + void *model_raw = NULL; + float *heads_host = NULL; + float *residual_host = NULL; + float *split_host = NULL; + float *ref_low_host = NULL; + float *fused_low_host = NULL; + float *ref_out_host = NULL; + float *fused_out_host = NULL; + float *ref_hc_host = NULL; + float *fused_hc_host = NULL; + ds4_gpu_tensor *heads_base = NULL; + ds4_gpu_tensor *residual_base = NULL; + ds4_gpu_tensor *split_base = NULL; + ds4_gpu_tensor *ref_low_base = NULL; + ds4_gpu_tensor *fused_low_base = NULL; + ds4_gpu_tensor *ref_out_base = NULL; + ds4_gpu_tensor *fused_out_base = NULL; + ds4_gpu_tensor *ref_hc_base = NULL; + ds4_gpu_tensor *fused_hc_base = NULL; + ds4_gpu_tensor *group_tmp = NULL; + ds4_gpu_tensor *low_tmp = NULL; + ds4_gpu_tensor *heads = NULL; + ds4_gpu_tensor *residual = NULL; + ds4_gpu_tensor *split = NULL; + ds4_gpu_tensor *ref_low = NULL; + ds4_gpu_tensor *fused_low = NULL; + ds4_gpu_tensor *ref_out = NULL; + ds4_gpu_tensor *fused_out = NULL; + ds4_gpu_tensor *ref_hc = NULL; + ds4_gpu_tensor *fused_hc = NULL; + + TEST_ASSERT(posix_memalign( + &model_raw, (size_t)page, (size_t)model_bytes) == 0); + if (model_raw) { + memset(model_raw, 0, (size_t)model_bytes); + if (q4) { + test_fill_q4_K_weights( + model_raw, group_dim, low_dim, 211u); + test_fill_q4_K_weights( + (uint8_t *)model_raw + out_b_offset, + low_dim, out_dim, 307u); + } else { + test_fill_q8_0_weights( + model_raw, group_dim, low_dim, 211u); + test_fill_q8_0_weights( + (uint8_t *)model_raw + out_b_offset, + low_dim, out_dim, 307u); + } + } + + heads_host = malloc((size_t)heads_bytes); + residual_host = malloc((size_t)hc_bytes); + split_host = malloc((size_t)split_bytes); + ref_low_host = malloc((size_t)low_bytes); + fused_low_host = malloc((size_t)low_bytes); + ref_out_host = malloc((size_t)out_bytes); + fused_out_host = malloc((size_t)out_bytes); + ref_hc_host = malloc((size_t)hc_bytes); + fused_hc_host = malloc((size_t)hc_bytes); + heads_base = ds4_gpu_tensor_alloc(heads_bytes); + residual_base = ds4_gpu_tensor_alloc(hc_bytes); + split_base = ds4_gpu_tensor_alloc(split_bytes); + ref_low_base = ds4_gpu_tensor_alloc(low_bytes); + fused_low_base = ds4_gpu_tensor_alloc(low_bytes); + ref_out_base = ds4_gpu_tensor_alloc(out_bytes); + fused_out_base = ds4_gpu_tensor_alloc(out_bytes); + ref_hc_base = ds4_gpu_tensor_alloc(hc_bytes); + fused_hc_base = ds4_gpu_tensor_alloc(hc_bytes); + group_tmp = ds4_gpu_tensor_alloc(scratch_bytes); + low_tmp = ds4_gpu_tensor_alloc(sizeof(float)); + heads = heads_base + ? ds4_gpu_tensor_view(heads_base, 0u, active_heads_bytes) : NULL; + residual = residual_base + ? ds4_gpu_tensor_view(residual_base, 0u, active_hc_bytes) : NULL; + split = split_base + ? ds4_gpu_tensor_view(split_base, 0u, active_split_bytes) : NULL; + ref_low = ref_low_base + ? ds4_gpu_tensor_view(ref_low_base, 0u, active_low_bytes) : NULL; + fused_low = fused_low_base + ? ds4_gpu_tensor_view(fused_low_base, 0u, active_low_bytes) : NULL; + ref_out = ref_out_base + ? ds4_gpu_tensor_view(ref_out_base, 0u, active_out_bytes) : NULL; + fused_out = fused_out_base + ? ds4_gpu_tensor_view(fused_out_base, 0u, active_out_bytes) : NULL; + ref_hc = ref_hc_base + ? ds4_gpu_tensor_view(ref_hc_base, 0u, active_hc_bytes) : NULL; + fused_hc = fused_hc_base + ? ds4_gpu_tensor_view(fused_hc_base, 0u, active_hc_bytes) : NULL; + + const bool allocated = model_raw && heads_host && residual_host && + split_host && ref_low_host && fused_low_host && ref_out_host && + fused_out_host && ref_hc_host && fused_hc_host && heads_base && + residual_base && split_base && ref_low_base && fused_low_base && + ref_out_base && fused_out_base && ref_hc_base && fused_hc_base && + group_tmp && low_tmp && heads && residual && split && ref_low && + fused_low && ref_out && fused_out && ref_hc && fused_hc; + TEST_ASSERT(allocated); + + test_float_compare_stats low_stats = {0}; + test_float_compare_stats hc_stats = {0}; + size_t reject_writes = 0u; + size_t active_poisoned = 0u; + size_t guard_mismatches = 0u; + size_t fused_out_writes = 0u; + if (allocated) { + for (uint64_t i = 0; i < heads_count; i++) { + const int value = + (int)((i * 17u + (i ^ (i >> 7u)) * 3u + + (q4 ? 19u : 7u)) % 127u) - 63; + heads_host[i] = (float)value / 96.0f; + } + for (uint64_t i = 0; i < hc_count; i++) { + const int value = + (int)((i * 19u + (i ^ (i >> 5u)) * 7u + + (q4 ? 23u : 11u)) % 149u) - 74; + residual_host[i] = (float)value / 80.0f; + } + for (uint32_t t = 0; t < alloc_tokens; t++) { + float *row = split_host + (uint64_t)t * mix_hc; + for (uint32_t h = 0; h < n_hc; h++) { + row[h] = 0.0f; + row[n_hc + h] = + 0.55f + (float)((t + h * 3u) % 11u) / 32.0f; + } + for (uint32_t dst_hc = 0; dst_hc < n_hc; dst_hc++) { + for (uint32_t src_hc = 0; src_hc < n_hc; src_hc++) { + const int value = + (int)((t * 5u + dst_hc * 7u + src_hc * 11u + + (q4 ? 3u : 0u)) % 17u) - 8; + row[2u * n_hc + dst_hc * n_hc + src_hc] = + (float)value / 24.0f; + } + } + } + test_metal_attn_out_hc_fill_poison( + ref_low_host, low_count, ref_low_tag); + test_metal_attn_out_hc_fill_poison( + fused_low_host, low_count, fused_low_tag); + test_metal_attn_out_hc_fill_poison( + ref_out_host, out_count, ref_out_tag); + test_metal_attn_out_hc_fill_poison( + fused_out_host, out_count, fused_out_tag); + test_metal_attn_out_hc_fill_poison( + ref_hc_host, hc_count, ref_hc_tag); + test_metal_attn_out_hc_fill_poison( + fused_hc_host, hc_count, fused_hc_tag); + + TEST_ASSERT(ds4_gpu_tensor_write( + heads_base, 0u, heads_host, heads_bytes) != 0); + TEST_ASSERT(ds4_gpu_tensor_write( + residual_base, 0u, residual_host, hc_bytes) != 0); + TEST_ASSERT(ds4_gpu_tensor_write( + split_base, 0u, split_host, split_bytes) != 0); + TEST_ASSERT(ds4_gpu_tensor_write( + ref_low_base, 0u, ref_low_host, low_bytes) != 0); + TEST_ASSERT(ds4_gpu_tensor_write( + fused_low_base, 0u, fused_low_host, low_bytes) != 0); + TEST_ASSERT(ds4_gpu_tensor_write( + ref_out_base, 0u, ref_out_host, out_bytes) != 0); + TEST_ASSERT(ds4_gpu_tensor_write( + fused_out_base, 0u, fused_out_host, out_bytes) != 0); + TEST_ASSERT(ds4_gpu_tensor_write( + ref_hc_base, 0u, ref_hc_host, hc_bytes) != 0); + TEST_ASSERT(ds4_gpu_tensor_write( + fused_hc_base, 0u, fused_hc_host, hc_bytes) != 0); + TEST_ASSERT(ds4_gpu_set_model_map(model_raw, model_bytes) != 0); + + TEST_ASSERT(unsetenv(disable_fusion_env) == 0); + TEST_ASSERT(unsetenv(require_q8_env) == 0); + TEST_ASSERT(unsetenv(require_q4_env) == 0); + TEST_ASSERT(unsetenv(require_q4_rhs_env) == 0); + TEST_ASSERT(unsetenv(disable_q4_rhs_env) == 0); + TEST_ASSERT(unsetenv(stage_profile_env) == 0); + TEST_ASSERT(unsetenv(q8_profile_env) == 0); + TEST_ASSERT(setenv( + q4 ? require_q4_env : require_q8_env, + "1", 1) == 0); + if (q4) { + TEST_ASSERT(setenv(require_q4_rhs_env, "1", 1) == 0); + } + ds4_gpu_set_quality(false); + ds4_gpu_test_set_flags(q4 + ? DS4_GPU_TEST_BATCH_ATTN_OUT_Q4_HC_FUSION + : DS4_GPU_TEST_BATCH_ATTN_OUT_Q8_HC_FUSION); + + /* The force hook removes only the production minimum. Neighbors of + * the 32-row tile must still fail closed and leave every destination + * byte untouched. */ + TEST_ASSERT(test_metal_batch_attn_out_hc_fused_call( + kind, fused_out_base, fused_hc_base, residual_base, + split_base, fused_low_base, group_tmp, low_tmp, + model_raw, model_bytes, out_b_offset, heads_base, + n_tokens - 1u) == -1); + TEST_ASSERT(test_metal_batch_attn_out_hc_fused_call( + kind, fused_out_base, fused_hc_base, residual_base, + split_base, fused_low_base, group_tmp, low_tmp, + model_raw, model_bytes, out_b_offset, heads_base, + n_tokens + 1u) == -1); + TEST_ASSERT(ds4_gpu_tensor_read( + fused_low_base, 0u, fused_low_host, low_bytes) != 0); + TEST_ASSERT(ds4_gpu_tensor_read( + fused_out_base, 0u, fused_out_host, out_bytes) != 0); + TEST_ASSERT(ds4_gpu_tensor_read( + fused_hc_base, 0u, fused_hc_host, hc_bytes) != 0); + reject_writes += test_metal_attn_out_hc_canary_mismatches( + fused_low_host, 0u, low_count, fused_low_tag); + reject_writes += test_metal_attn_out_hc_canary_mismatches( + fused_out_host, 0u, out_count, fused_out_tag); + reject_writes += test_metal_attn_out_hc_canary_mismatches( + fused_hc_host, 0u, hc_count, fused_hc_tag); + + int reference_ok = 0; + if (q4) { + reference_ok = ds4_gpu_attention_output_q4_K_batch_tensor( + ref_out, ref_low, group_tmp, low_tmp, + model_raw, model_bytes, 0u, out_b_offset, 12u, + group_dim, rank, n_groups, out_dim, heads, n_tokens); + } else { + reference_ok = ds4_gpu_attention_output_q8_batch_tensor( + ref_out, ref_low, group_tmp, low_tmp, + model_raw, model_bytes, 0u, out_b_offset, + group_dim, rank, n_groups, out_dim, heads, n_tokens); + } + TEST_ASSERT(reference_ok == 1); + TEST_ASSERT(ds4_gpu_hc_expand_split_tensor( + ref_hc, ref_out, residual, split, + out_dim, n_hc) != 0); + TEST_ASSERT(test_metal_batch_attn_out_hc_fused_call( + kind, fused_out, fused_hc, residual, split, + fused_low, group_tmp, low_tmp, + model_raw, model_bytes, out_b_offset, heads, + n_tokens) == 1); + + TEST_ASSERT(ds4_gpu_tensor_read( + ref_low_base, 0u, ref_low_host, low_bytes) != 0); + TEST_ASSERT(ds4_gpu_tensor_read( + fused_low_base, 0u, fused_low_host, low_bytes) != 0); + TEST_ASSERT(ds4_gpu_tensor_read( + ref_out_base, 0u, ref_out_host, out_bytes) != 0); + TEST_ASSERT(ds4_gpu_tensor_read( + fused_out_base, 0u, fused_out_host, out_bytes) != 0); + TEST_ASSERT(ds4_gpu_tensor_read( + ref_hc_base, 0u, ref_hc_host, hc_bytes) != 0); + TEST_ASSERT(ds4_gpu_tensor_read( + fused_hc_base, 0u, fused_hc_host, hc_bytes) != 0); + + low_stats = test_compare_float_bits( + ref_low_host, fused_low_host, (size_t)active_low_count); + hc_stats = test_compare_float_bits( + ref_hc_host, fused_hc_host, (size_t)active_hc_count); + active_poisoned += test_metal_attn_out_hc_poisoned_values( + ref_low_host, active_low_count, ref_low_tag); + active_poisoned += test_metal_attn_out_hc_poisoned_values( + fused_low_host, active_low_count, fused_low_tag); + active_poisoned += test_metal_attn_out_hc_poisoned_values( + ref_out_host, active_out_count, ref_out_tag); + active_poisoned += test_metal_attn_out_hc_poisoned_values( + ref_hc_host, active_hc_count, ref_hc_tag); + active_poisoned += test_metal_attn_out_hc_poisoned_values( + fused_hc_host, active_hc_count, fused_hc_tag); + fused_out_writes = test_metal_attn_out_hc_canary_mismatches( + fused_out_host, 0u, out_count, fused_out_tag); + guard_mismatches += test_metal_attn_out_hc_canary_mismatches( + ref_low_host, active_low_count, low_count, ref_low_tag); + guard_mismatches += test_metal_attn_out_hc_canary_mismatches( + fused_low_host, active_low_count, low_count, fused_low_tag); + guard_mismatches += test_metal_attn_out_hc_canary_mismatches( + ref_out_host, active_out_count, out_count, ref_out_tag); + guard_mismatches += test_metal_attn_out_hc_canary_mismatches( + ref_hc_host, active_hc_count, hc_count, ref_hc_tag); + guard_mismatches += test_metal_attn_out_hc_canary_mismatches( + fused_hc_host, active_hc_count, hc_count, fused_hc_tag); + } + + fprintf(stderr, + "ds4-test: Metal %s batch attention-output HC exact N=%u " + "low=%zu/%llu hc=%zu/%llu max_ulp=%u/%u " + "active_poison=%zu guard=%zu fused_out_writes=%zu " + "reject_writes=%zu\n", + q4 ? "Q4_K/F16-RHS" : "Q8_0", + n_tokens, + low_stats.mismatch_count, + (unsigned long long)active_low_count, + hc_stats.mismatch_count, + (unsigned long long)active_hc_count, + low_stats.max_ulp, + hc_stats.max_ulp, + active_poisoned, + guard_mismatches, + fused_out_writes, + reject_writes); + TEST_ASSERT(low_stats.mismatch_count == 0u && low_stats.max_ulp == 0u); + TEST_ASSERT(hc_stats.mismatch_count == 0u && hc_stats.max_ulp == 0u); + TEST_ASSERT(active_poisoned == 0u); + TEST_ASSERT(guard_mismatches == 0u); + TEST_ASSERT(fused_out_writes == 0u); + TEST_ASSERT(reject_writes == 0u); + + ds4_gpu_test_set_flags(0u); + ds4_gpu_set_quality(false); + test_restore_env(q8_profile_env, saved_q8_profile); + test_restore_env(stage_profile_env, saved_stage_profile); + test_restore_env(disable_q4_rhs_env, saved_disable_q4_rhs); + test_restore_env(require_q4_rhs_env, saved_require_q4_rhs); + test_restore_env(require_q4_env, saved_require_q4); + test_restore_env(require_q8_env, saved_require_q8); + test_restore_env(disable_fusion_env, saved_disable_fusion); + ds4_gpu_tensor_free(fused_hc); + ds4_gpu_tensor_free(ref_hc); + ds4_gpu_tensor_free(fused_out); + ds4_gpu_tensor_free(ref_out); + ds4_gpu_tensor_free(fused_low); + ds4_gpu_tensor_free(ref_low); + ds4_gpu_tensor_free(split); + ds4_gpu_tensor_free(residual); + ds4_gpu_tensor_free(heads); + ds4_gpu_tensor_free(low_tmp); + ds4_gpu_tensor_free(group_tmp); + ds4_gpu_tensor_free(fused_hc_base); + ds4_gpu_tensor_free(ref_hc_base); + ds4_gpu_tensor_free(fused_out_base); + ds4_gpu_tensor_free(ref_out_base); + ds4_gpu_tensor_free(fused_low_base); + ds4_gpu_tensor_free(ref_low_base); + ds4_gpu_tensor_free(split_base); + ds4_gpu_tensor_free(residual_base); + ds4_gpu_tensor_free(heads_base); + free(fused_hc_host); + free(ref_hc_host); + free(fused_out_host); + free(ref_out_host); + free(fused_low_host); + free(ref_low_host); + free(split_host); + free(residual_host); + free(heads_host); + free(model_raw); +} + +static void test_metal_batch_attn_out_hc_fusion_exact(void) { + test_metal_batch_attn_out_hc_fusion_exact_case( + TEST_METAL_ATTN_OUT_HC_Q8); + test_metal_batch_attn_out_hc_fusion_exact_case( + TEST_METAL_ATTN_OUT_HC_Q4); +} + static void test_metal_hc_split_weighted_sum_norm_batch_exact(void) { /* Compare the batched HC+RMSNorm fusion against the exact two-dispatch * sequence used by the reference path at DS4's production dimensions. */ @@ -6899,6 +7588,13 @@ static void test_metal_router_weights_batch_exact(void) { #endif static void test_metal_kernel_group(void) { +#if defined(__APPLE__) + if (test_env_bool("DS4_TEST_METAL_RESIDENT_ORACLES_ONLY")) { + test_metal_flush_commands_progress_exact(); + test_metal_batch_attn_out_hc_fusion_exact(); + return; + } +#endif test_metal_f16_matvec_fast_nr0_4(); test_metal_f16_prefill_matmul(); test_metal_q8_0_prefill_matmul(); @@ -6907,6 +7603,7 @@ static void test_metal_kernel_group(void) { test_dspark_cache_window_crop(); test_metal_q8_0_decode_pair_exact(); #if defined(__APPLE__) + test_metal_flush_commands_progress_exact(); test_metal_q8_0_decode_rows_exact(); test_metal_q8_attention_output_static_batch_exact(); test_metal_q4_attention_output_tiny_batch_exact(); @@ -6925,6 +7622,7 @@ static void test_metal_kernel_group(void) { test_metal_contiguous_compressed_f16_attention_exact(); test_metal_persistent_zero_attention_mask_exact(); test_metal_zero_prefix_prefill_mask_cache_exact(); + test_metal_batch_attn_out_hc_fusion_exact(); test_metal_hc_split_weighted_sum_norm_batch_exact(); test_metal_hc_producer_pre_norm_compound_exact(); test_metal_output_hc_weights4_exact(); @@ -9025,6 +9723,7 @@ static void test_print_help(const char *prog) { puts(" DS4_TEST_VECTOR_FILE=FILE Official fixture. Default: flash-0731/official.vec."); puts(" DS4_TEST_LOCAL_GOLDEN_FILE=FILE Local fixture. Default: flash-0731/local-golden.vec."); puts(" DS4_TEST_MPP_EQ_CASE=NAME Run only Tensor equivalence cases whose id contains NAME."); + puts(" DS4_TEST_METAL_RESIDENT_ORACLES_ONLY=1 Restrict --metal-kernels to resident optimization oracles."); puts(" DS4_TEST_MTP=FILE Legacy MTP support GGUF for --mtp-verify-depth."); puts(" DS4_TEST_DSPARK=FILE DSpark support GGUF for --dspark-verify-depth."); puts(" DS4_TEST_CONTINUED_PREFILL_TOKENS=N Large suffix size for --glm53-continued-prefill."); diff --git a/tests/test_rocm_q4_dense_pair.cpp b/tests/test_rocm_q4_dense_pair.cpp index bd4f58128..5d060c101 100644 --- a/tests/test_rocm_q4_dense_pair.cpp +++ b/tests/test_rocm_q4_dense_pair.cpp @@ -39,6 +39,10 @@ constexpr uint32_t kM1 = 33u; constexpr uint32_t kQ4Type = 12u; constexpr uint32_t kQ8Type = 8u; constexpr uint32_t kTailK = 1024u; +constexpr uint32_t kQbOutDim = 32768u; +constexpr uint32_t kQbHeads = 64u; +constexpr uint32_t kQbHeadDim = 512u; +constexpr uint32_t kQbRot = 64u; constexpr uint32_t kAttnGroupDim = 4096u; constexpr uint32_t kAttnRank = 32u; constexpr uint32_t kAttnGroups = 8u; @@ -60,6 +64,18 @@ constexpr const char *kPrefillDisable = "DS4_ROCM_DISABLE_Q4_PREFILL_TILE8"; constexpr const char *kPrefillRequire = "DS4_ROCM_REQUIRE_Q4_PREFILL_TILE8"; +constexpr const char *kPrefillK1024Tile4Disable = + "DS4_ROCM_DISABLE_Q4_PREFILL_K1024_TILE4"; +constexpr const char *kQbF16Enable = + "DS4_ROCM_ENABLE_Q4_ATTN_Q_B_F16_CACHE"; +constexpr const char *kQbF16Disable = + "DS4_ROCM_DISABLE_Q4_ATTN_Q_B_F16_CACHE"; +constexpr const char *kQbF16Require = + "DS4_ROCM_REQUIRE_Q4_ATTN_Q_B_F16_CACHE"; +constexpr const char *kQbF16MinTokens = + "DS4_ROCM_Q4_ATTN_Q_B_F16_CACHE_MIN_TOKENS"; +constexpr const char *kQbF16OutputEnable = + "DS4_ROCM_ENABLE_Q4_ATTN_Q_B_F16_OUTPUT"; constexpr const char *kGroupedDecodeEnable = "DS4_ROCM_ENABLE_Q4_GROUPED_ATTN_A"; constexpr const char *kGroupedDecodeDisable = @@ -116,6 +132,7 @@ struct aligned_model { uint64_t attn_b_q8_offset = 0; uint64_t tail_k1024_offset = 0; uint64_t tail_k1024_pair_offset = 0; + uint64_t q_b_k1024_offset = 0; ~aligned_model() { std::free(data); } @@ -282,6 +299,8 @@ bool make_model(aligned_model *model) { (kTailK / kQkK) * sizeof(block_q4_K_test); const uint64_t tail0_bytes = (uint64_t)kM0 * tail_row_bytes; const uint64_t tail1_bytes = (uint64_t)kM1 * tail_row_bytes; + const uint64_t q_b_k1024_bytes = + (uint64_t)kQbOutDim * tail_row_bytes; const uint64_t attn_b_q8_row_bytes = (kAttnLowDim / 32u) * sizeof(block_q8_0_test); const uint64_t attn_b_q8_bytes = @@ -300,8 +319,10 @@ bool make_model(aligned_model *model) { model->tail_k1024_offset + tail0_bytes, page); model->attn_b_q8_offset = round_up( model->tail_k1024_pair_offset + tail1_bytes, page); - model->size = round_up( + model->q_b_k1024_offset = round_up( model->attn_b_q8_offset + attn_b_q8_bytes, page); + model->size = round_up( + model->q_b_k1024_offset + q_b_k1024_bytes, page); void *storage = nullptr; if (posix_memalign(&storage, (size_t)page, (size_t)model->size) != 0) { return false; @@ -333,6 +354,9 @@ bool make_model(aligned_model *model) { fill_q4_rows(reinterpret_cast( model->data + model->tail_k1024_pair_offset), kM1, kTailK, 0xa4093822u); + fill_q4_rows(reinterpret_cast( + model->data + model->q_b_k1024_offset), + kQbOutDim, kTailK, 0x082efa98u); fill_q8_0_rows(reinterpret_cast( model->data + model->attn_b_q8_offset), kAttnOutDim, kAttnLowDim, 0x03707344u); @@ -702,6 +726,7 @@ bool run_prefill_parity_case(const aligned_model &model, uint32_t n_tokens, env_snapshot enable(kPrefillEnable); env_snapshot disable(kPrefillDisable); env_snapshot require(kPrefillRequire); + env_snapshot k1024_tile4_disable(kPrefillK1024Tile4Disable); // The authoritative rollback remains the reference now that the tiled // path is default-on. @@ -718,6 +743,7 @@ bool run_prefill_parity_case(const aligned_model &model, uint32_t n_tokens, (void)unsetenv(kPrefillEnable); (void)unsetenv(kPrefillDisable); (void)setenv(kPrefillRequire, "1", 1); + (void)unsetenv(kPrefillK1024Tile4Disable); const int candidate_rc = ds4_gpu_matmul_quant_tensor( candidate_gpu.ptr, model.data, model.size, offset, kQ4Type, in_dim, out_dim, x_gpu.ptr, n_tokens); @@ -758,6 +784,227 @@ bool run_prefill_parity_case(const aligned_model &model, uint32_t n_tokens, return ok; } +bool run_q_b_f16_null_qhalf_case(const aligned_model &model) { + constexpr uint32_t n_tokens = 32u; + static_assert(kQbHeads * kQbHeadDim == kQbOutDim, + "q_b test head geometry must cover the projection"); + const size_t logical_count = (size_t)n_tokens * kQbOutDim; + const size_t allocation_count = logical_count + kOutputGuardFloats; + const size_t q_half_guard = kOutputGuardFloats; + const size_t q_half_count = logical_count + q_half_guard; + std::vector x; + fill_activation(&x, n_tokens, kTailK); + const std::vector sentinel = sentinel_values(allocation_count); + const std::vector q_half_sentinel(q_half_count, 0x7e55u); + tensor_owner x_gpu(x.size() * sizeof(float)); + tensor_owner provided_out(allocation_count * sizeof(float)); + tensor_owner scratch_out(allocation_count * sizeof(float)); + tensor_owner reference_out(logical_count * sizeof(float)); + tensor_owner q_half_gpu(q_half_count * sizeof(uint16_t)); + if (!x_gpu.ptr || !provided_out.ptr || !scratch_out.ptr || + !reference_out.ptr || !q_half_gpu.ptr || + !write_tensor(x_gpu.ptr, x) || + !write_tensor(provided_out.ptr, sentinel) || + !write_tensor(scratch_out.ptr, sentinel) || + !ds4_gpu_tensor_write(q_half_gpu.ptr, 0, q_half_sentinel.data(), + q_half_sentinel.size() * sizeof(uint16_t))) { + std::fprintf(stderr, + "q_b F16 null-q_half: tensor allocation/write FAIL\n"); + return false; + } + + env_snapshot enable(kQbF16Enable); + env_snapshot disable(kQbF16Disable); + env_snapshot require(kQbF16Require); + env_snapshot min_tokens(kQbF16MinTokens); + env_snapshot f16_output(kQbF16OutputEnable); + (void)setenv(kQbF16Enable, "1", 1); + (void)unsetenv(kQbF16Disable); + (void)setenv(kQbF16Require, "1", 1); + (void)setenv(kQbF16MinTokens, "32", 1); + (void)setenv(kQbF16OutputEnable, "1", 1); + + const uint64_t weight_bytes = + (uint64_t)kQbOutDim * (kTailK / kQkK) * + sizeof(block_q4_K_test); + const ds4_gpu_q4_attn_q_b_f16_sidecar_desc desc = { + model.q_b_k1024_offset, + weight_bytes, + kTailK, + kQbOutDim, + kQ4Type, + 0u, + }; + uint64_t prepared_bytes = 0; + const int prepare_rc = ds4_gpu_prepare_q4_attn_q_b_f16_sidecars( + model.data, model.size, &desc, 1u, n_tokens, 0u, + &prepared_bytes); + + /* The release default must keep writing C=F32 and must not touch q_half, + * even when a large staging tensor is supplied by a test caller. */ + (void)setenv(kQbF16OutputEnable, "0", 1); + int default_rc = 0; + if (prepare_rc > 0) { + default_rc = ds4_gpu_attn_q_b_f16_head_rms_rope_tail_tensor( + provided_out.ptr, q_half_gpu.ptr, model.data, model.size, + model.q_b_k1024_offset, kQ4Type, kTailK, kQbOutDim, + x_gpu.ptr, n_tokens, kQbHeads, kQbHeadDim, kQbRot, + 17u, 0u, false, 10000.0f, 1.0f, 0.0f, 1.0f, + 32.0f, 1.0f, 1.0e-6f); + } + std::vector default_host(allocation_count); + std::vector default_half_host(q_half_count); + const bool default_read_ok = default_rc > 0 && + read_tensor(provided_out.ptr, &default_host) && + ds4_gpu_tensor_read(q_half_gpu.ptr, 0, default_half_host.data(), + default_half_host.size() * sizeof(uint16_t)); + bool ok = prepare_rc > 0 && default_read_ok; + if (default_read_ok) { + ok = output_guard_unchanged( + default_host, sentinel, logical_count, + "q_b default-F32 output canary") && ok; + uint64_t untouched_or_nonfinite = 0; + for (size_t i = 0; i < logical_count; i++) { + if (!std::isfinite(default_host[i]) || + std::memcmp(&default_host[i], &sentinel[i], + sizeof(float)) == 0) { + untouched_or_nonfinite++; + } + } + uint64_t half_mismatches = 0; + for (size_t i = 0; i < q_half_count; i++) { + if (default_half_host[i] != q_half_sentinel[i]) { + half_mismatches++; + } + } + std::fprintf(stderr, + "q_b default-F32 writes finite output: failures=%llu/%zu; " + "q_half mismatches=%llu/%zu %s\n", + (unsigned long long)untouched_or_nonfinite, + logical_count, + (unsigned long long)half_mismatches, q_half_count, + untouched_or_nonfinite == 0 && half_mismatches == 0 + ? "PASS" : "FAIL"); + ok = untouched_or_nonfinite == 0 && half_mismatches == 0 && ok; + } + + (void)setenv(kQbF16OutputEnable, "1", 1); + if (!write_tensor(provided_out.ptr, sentinel) || + !ds4_gpu_tensor_write(q_half_gpu.ptr, 0, q_half_sentinel.data(), + q_half_sentinel.size() * sizeof(uint16_t))) { + ok = false; + } + int provided_rc = 0; + if (prepare_rc > 0) { + provided_rc = ds4_gpu_attn_q_b_f16_head_rms_rope_tail_tensor( + provided_out.ptr, q_half_gpu.ptr, model.data, model.size, + model.q_b_k1024_offset, kQ4Type, kTailK, kQbOutDim, + x_gpu.ptr, n_tokens, kQbHeads, kQbHeadDim, kQbRot, + 17u, 0u, false, 10000.0f, 1.0f, 0.0f, 1.0f, + 32.0f, 1.0f, 1.0e-6f); + } + + std::vector provided_host(allocation_count); + std::vector q_half_host(q_half_count); + const bool provided_read_ok = provided_rc > 0 && + read_tensor(provided_out.ptr, &provided_host) && + ds4_gpu_tensor_read(q_half_gpu.ptr, 0, q_half_host.data(), + q_half_host.size() * sizeof(uint16_t)); + ok = provided_read_ok && ok; + if (provided_read_ok) { + ok = output_guard_unchanged( + provided_host, sentinel, logical_count, + "q_b F16 provided-q_half output canary") && ok; + uint64_t nonfinite = 0; + for (size_t i = 0; i < logical_count; i++) { + if (!std::isfinite(provided_host[i])) nonfinite++; + } + uint64_t half_guard_mismatches = 0; + for (size_t i = logical_count; i < q_half_count; i++) { + if (q_half_host[i] != q_half_sentinel[i]) { + half_guard_mismatches++; + } + } + std::fprintf(stderr, + "q_b F16 provided-q_half: nonfinite=%llu/%zu; " + "canary mismatches=%llu/%zu %s\n", + (unsigned long long)nonfinite, logical_count, + (unsigned long long)half_guard_mismatches, q_half_guard, + nonfinite == 0 && half_guard_mismatches == 0 + ? "PASS" : "FAIL"); + ok = nonfinite == 0 && half_guard_mismatches == 0 && ok; + } + + /* Re-expand the accepted F16 projection exactly and feed the established + * F32 epilogue. This is a bitwise oracle for the fused half-input tail, + * with a nonzero production-shape projection rather than a zero smoke test. */ + int reference_rc = 0; + std::vector reference_input(logical_count); + if (provided_read_ok) { + for (size_t i = 0; i < logical_count; i++) { + reference_input[i] = fp16_to_float(q_half_host[i]); + } + if (write_tensor(reference_out.ptr, reference_input)) { + reference_rc = ds4_gpu_head_rms_norm_rope_tail_tensor( + reference_out.ptr, n_tokens, kQbHeads, kQbHeadDim, kQbRot, + 17u, 0u, false, 10000.0f, 1.0f, 0.0f, 1.0f, + 32.0f, 1.0f, 1.0e-6f); + } + } + std::vector reference_host(logical_count); + if (reference_rc > 0 && read_tensor(reference_out.ptr, &reference_host)) { + provided_host.resize(logical_count); + ok = bitwise_equal(provided_host, reference_host, + "q_b F16 fused tail vs expanded-F16 F32 tail") && ok; + } else { + std::fprintf(stderr, + "q_b F16 expanded-F16 epilogue reference rc=%d FAIL\n", + reference_rc); + ok = false; + } + + /* Non-Apple graphs pass NULL q_half. The backend-owned Q_F16 region must + * produce the exact same fused result as an explicit staging tensor. */ + int scratch_rc = 0; + if (prepare_rc > 0) { + scratch_rc = ds4_gpu_attn_q_b_f16_head_rms_rope_tail_tensor( + scratch_out.ptr, nullptr, model.data, model.size, + model.q_b_k1024_offset, kQ4Type, kTailK, kQbOutDim, + x_gpu.ptr, n_tokens, kQbHeads, kQbHeadDim, kQbRot, + 17u, 0u, false, 10000.0f, 1.0f, 0.0f, 1.0f, + 32.0f, 1.0f, 1.0e-6f); + } + std::vector scratch_host(allocation_count); + if (scratch_rc > 0 && read_tensor(scratch_out.ptr, &scratch_host)) { + ok = output_guard_unchanged( + scratch_host, sentinel, logical_count, + "q_b F16 null-q_half output canary") && ok; + scratch_host.resize(logical_count); + if (provided_read_ok) { + ok = bitwise_equal(scratch_host, provided_host, + "q_b F16 null vs provided q_half") && ok; + } else { + ok = false; + } + } else { + std::fprintf(stderr, + "q_b F16 null-q_half dispatch/read rc=%d FAIL\n", + scratch_rc); + ok = false; + } + + const int release_rc = ds4_gpu_release_q4_attn_q_b_f16_sidecars(); + ok = release_rc != 0 && ok; + std::fprintf(stderr, + "q_b F16 q_half staging: prepare=%d default=%d provided=%d " + "scratch=%d " + "prepared=%.2f MiB release=%d %s\n", + prepare_rc, default_rc, provided_rc, scratch_rc, + (double)prepared_bytes / 1048576.0, release_rc, + ok ? "PASS" : "FAIL"); + return ok; +} + bool run_prefill_gate_guards(const aligned_model &model) { constexpr uint32_t n_tokens = 9u; std::vector x; @@ -1644,6 +1891,11 @@ int main(int argc, char **argv) { const bool prefill_tail128_ok = run_prefill_parity_case( model, 128u, model.tail_k1024_offset, kM0, true, "prefill K=1024 M=65 n_tok=128 (K-tail nb=4)", kTailK); + const bool prefill_q_b_tile4_ok = run_prefill_parity_case( + model, 9u, model.q_b_k1024_offset, kQbOutDim, false, + "prefill q_b K=1024 M=32768 n_tok=9 (tile4)", kTailK); + const bool q_b_f16_null_qhalf_ok = + run_q_b_f16_null_qhalf_case(model); const bool prefill_single9_ok = run_prefill_parity_case( model, 9u, model.attn_b_offset, kAttnOutDim, true, "prefill K=256 M=65 n_tok=9 (K-tail nb=1)", kAttnLowDim); @@ -1680,6 +1932,7 @@ int main(int argc, char **argv) { ok = prefill9_ok && prefill30_ok && prefill128_ok && prefill_tail9_ok && prefill_tail128_ok && prefill_single9_ok && + prefill_q_b_tile4_ok && q_b_f16_null_qhalf_ok && prefill_single128_ok && prefill_pair9_ok && prefill_pair30_reverse_ok && prefill_pair128_ok && attention9_ok && attention30_ok && attention128_ok && attention_q8_9_ok && From 750b5fcbdcaad14bac55461963dcd190d3bc4b09 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Thu, 27 Aug 2026 13:09:41 +0200 Subject: [PATCH 123/189] Optimize greedy token decode across GPU backends --- ENVIRONMENT_VARIABLES.md | 64 ++-- Makefile | 18 +- ds4.c | 181 +++++++--- ds4_cuda.cu | 3 +- ds4_gpu.h | 3 +- ds4_metal.m | 155 ++++++++- metal/argsort.metal | 126 +++++++ rocm/ds4_rocm_q4.cuh | 31 +- scripts/environment_variables.tsv | 42 ++- tests/ds4_test.c | 3 +- tests/test_metal_argmax_top1.c | 561 ++++++++++++++++++++++++++++++ tests/test_metal_q4_attn_exactn.c | 2 +- tests/test_rocm_q4_dense_pair.cpp | 134 ++++++- 13 files changed, 1206 insertions(+), 117 deletions(-) create mode 100644 tests/test_metal_argmax_top1.c diff --git a/ENVIRONMENT_VARIABLES.md b/ENVIRONMENT_VARIABLES.md index b04889e20..e56577805 100644 --- a/ENVIRONMENT_VARIABLES.md +++ b/ENVIRONMENT_VARIABLES.md @@ -22,6 +22,12 @@ changing them. Unless a row says otherwise: - `STATS`, `PROFILE`, and `ORACLE` are diagnostic and may perturb timing; - benchmark controls and candidates in separate processes. +## Greedy top-1 readback + +| Variable | Default behavior and purpose | +| --- | --- | +| `DS4_DISABLE_GREEDY_TOP1_READBACK=1` | Restore the legacy full-logits readback and CPU argmax. Unset uses device top-1 plus a four-byte readback for eligible single-tier greedy generation, including SSD streaming, on Metal, CUDA, and ROCm. | + ## Metal | Variable | Default behavior and purpose | @@ -29,6 +35,8 @@ changing them. Unless a row says otherwise: | `DS4_METAL_PREFILL_CHUNK=N` | Set the prefill cap when `--prefill-chunk` is absent; the CLI option takes precedence. This historical name is consumed by the shared graph planner rather than by a Metal kernel alone. | | `DS4_METAL_NO_RESIDENCY=1` | Skip creation and residency requests for the model-view residency set. Diagnostic rollback for resident, non-streaming models. | | `DS4_METAL_DISABLE_QUEUE_RESIDENCY_SET=1` | Still create, commit, and request the model residency set, but do not attach it to Metal command queues. This isolates queue-residency behavior without disabling the complete residency policy. | +| `DS4_METAL_DISABLE_DECODE_ARGMAX_TOP1=1` | Restore the generic full argsort used by Metal argmax. Unset uses the dedicated two-stage top-1 reduction for eligible large decode rows. | +| `DS4_METAL_REQUIRE_DECODE_ARGMAX_TOP1=1` | Fail closed instead of using generic argsort when a row of at least 4096 logits cannot execute the dedicated top-1 path. Intended for correctness and performance oracles. | | `DS4_METAL_STREAMING_EXPERT_NOCACHE=1` | Reopen the Metal SSD expert file with `F_NOCACHE` so streamed experts do not displace the dense working set from the page cache. Leave unset for cached `pread`. | | `DS4_METAL_STREAMING_EXPERT_PREAD_SPLIT=N` | Split each expert read into 1–8 aligned requests. The automatic value is 1 below 64 configured cache experts and 4 at 64 or more. | | `DS4_METAL_DISABLE_Q4_DENSE_PAIR=1` | Split the default Metal Q-A/KV Q4 pair back into two standalone projections. | @@ -75,9 +83,9 @@ The detailed Metal A/B contracts and expected oracle counters live in | `DS4_ROCM_Q4_PREFILL_TILE8_STATS=1` | Report dense, pair, attention-batch, and token counters at process exit. | | `DS4_ROCM_ENABLE_Q4_DENSE_PAIR=1` | Share one Q8_K activation quantization between the two Q4 dense projections. This pair remains opt-in. | | `DS4_ROCM_DISABLE_Q4_DENSE_PAIR=1` | Dominant rollback for the ROCm Q4 dense pair. | -| `DS4_ROCM_ENABLE_Q4_GROUPED_ATTN_A=1` | Enable the two-launch grouped attention-A decode path. This path remains opt-in pending model A/B results. | -| `DS4_ROCM_DISABLE_Q4_GROUPED_ATTN_A=1` | Dominant rollback for grouped attention-A decode. | -| `DS4_ROCM_REQUIRE_Q4_GROUPED_ATTN_A=1` | Fail closed if grouped attention-A decode is disabled or ineligible. | +| `DS4_ROCM_ENABLE_Q4_GROUPED_ATTN_A=1` | Extend the two-launch grouped attention-A path outside its default scope. The exact caller-marked resident decode shape `groups=8, N=1, K=4096, M=1024` is automatic; row-at-a-time batch fallbacks are not. | +| `DS4_ROCM_DISABLE_Q4_GROUPED_ATTN_A=1` | Dominant rollback to eight standalone Q4 attention-A projections, including for the resident default. | +| `DS4_ROCM_REQUIRE_Q4_GROUPED_ATTN_A=1` | Request grouped attention-A for eligible non-default shapes and fail closed on fallback; `DISABLE` remains authoritative. | | `DS4_ROCM_Q4_GROUPED_ATTN_A_STATS=1` | Report grouped calls, dispatches, groups, fallbacks, and failures. | Run `make test-strix-rocm-q4-parity` and @@ -118,13 +126,13 @@ above, it is an unstable internal diagnostic or tuning interface. The linked sou remains normative for exact eligibility gates, bounds, and architecture-specific defaults. -Inventory totals: **1069 `DS4_*` runtime variables** and +Inventory totals: **1072 `DS4_*` runtime variables** and **6 external runtime variables**. -The auxiliary inventories contain **112 test/test-fixture entries** +The auxiliary inventories contain **118 test/test-fixture entries** and **19 tool/wrapper entries**.
-Metal (442) +Metal (444) | Variable | Accepted value and default | Effect | Source | | --- | --- | --- | --- | @@ -152,6 +160,7 @@ and **19 tool/wrapper entries**. | `DS4_METAL_DISABLE_COMPRESSOR_STORE_ONE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables compressor store one. | [ds4_metal.m:23249](ds4_metal.m#L23249) | | `DS4_METAL_DISABLE_CONTIG_F16_F16_COPY` | value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables | Disables contig F16 F16 copy. | [ds4_metal.m:29562](ds4_metal.m#L29562) | | `DS4_METAL_DISABLE_CONTIG_F32_F16_COPY` | value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables | Disables contig F32 F16 copy. | [ds4_metal.m:29338](ds4_metal.m#L29338) | +| `DS4_METAL_DISABLE_DECODE_ARGMAX_TOP1` | presence rollback; unset uses the dedicated two-dispatch top-1 reduction for eligible large Metal rows; any defined value including empty or 0 restores the generic full argsort | Restore the generic indexer argsort for decode argmax A/B and emergency rollback. | [ds4_metal.m:21517](ds4_metal.m#L21517) | | `DS4_METAL_DISABLE_DECODE_NORM_EXACT_VIEWS` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables decode norm exact views. | [ds4_metal.m:36207](ds4_metal.m#L36207) | | `DS4_METAL_DISABLE_DECODE_RAW_GATHERED_ATTN` | presence rollback; unset: raw-only decode uses gathered attention; any value including 0 restores the legacy raw path | Restores the separate raw-only attention path instead of gathered staging and attention. | [ds4_metal.m:32968](ds4_metal.m#L32968) | | `DS4_METAL_DISABLE_DECODE_RAW_PACKED32` | presence rollback; unset: raw-only gathered attention may use packed32; any value including 0 disables it for raw-only layers | Disables the packed32 reduce kernel for raw-only gathered attention while leaving compressed layers unchanged. | [ds4_metal.m:31464](ds4_metal.m#L31464) | @@ -500,6 +509,7 @@ and **19 tool/wrapper entries**. | `DS4_METAL_Q_STAGE_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for q stage. | [ds4.c:29270](ds4.c#L29270) | | `DS4_METAL_REPEAT_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the repeat Metal kernel source file loaded at runtime. | [ds4_metal.m:4949](ds4_metal.m#L4949) | | `DS4_METAL_REQUIRE_COMPRESSOR_EXACT_POOL_RATIO4` | presence strict check; unset: fallback allowed; any value including 0 requires the path | Requires compressor exact pool ratio4 and makes eligible fallback fail closed. | [ds4_metal.m:26387](ds4_metal.m#L26387) | +| `DS4_METAL_REQUIRE_DECODE_ARGMAX_TOP1` | presence fail-closed assertion for rows of at least 4096 logits; unset permits generic fallback; any defined value including empty or 0 rejects disabled, ineligible, or failed dedicated top-1 preflight | Require the dedicated Metal top-1 reduction so correctness and performance oracles cannot silently exercise generic argsort. | [ds4_metal.m:21515](ds4_metal.m#L21515) | | `DS4_METAL_REQUIRE_EXACT_ROWS_PERSISTENT_CACHE` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Requires exact rows persistent cache and makes eligible fallback fail closed. | [ds4_metal.m:13002](ds4_metal.m#L13002) | | `DS4_METAL_REQUIRE_GATHERED_KV_STAGE` | presence strict check; unset: fallback allowed; any value including 0 requires the path | Requires gathered KV stage and makes eligible fallback fail closed. | [ds4_metal.m:29680](ds4_metal.m#L29680) | | `DS4_METAL_REQUIRE_IQ2_XXS_SSD_PREFILL_MM` | value-aware boolean; default implicit fail-closed only with complete selected-address domain; explicit 1 is strict, 0 permits fallback | Makes eligible IQ2_XXS/Q2_K grouped SSD-prefill MM fail closed. | [ds4_metal.m:44782](ds4_metal.m#L44782) | @@ -926,7 +936,7 @@ and **19 tool/wrapper entries**. | `DS4_ROCM_DISABLE_IQ2_SELECTED_EXPERT_VIEWS` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable iq2 selected expert views. | [ds4.c:21079](ds4.c#L21079) | | `DS4_ROCM_DISABLE_IQ2_STREAM_ADDR_TABLE` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable iq2 stream addr table. | [ds4.c:6548](ds4.c#L6548) | | `DS4_ROCM_DISABLE_Q4_DENSE_PAIR` | presence rollback; unset leaves opt-in policy unchanged | Disable/roll back rocm disable q4 dense pair. | [rocm/ds4_rocm_q4.cuh:435](rocm/ds4_rocm_q4.cuh#L435) | -| `DS4_ROCM_DISABLE_Q4_GROUPED_ATTN_A` | presence rollback; DISABLE wins over enable/require | Disable/roll back rocm disable q4 grouped attn a. | [rocm/ds4_rocm_q4.cuh:700](rocm/ds4_rocm_q4.cuh#L700) | +| `DS4_ROCM_DISABLE_Q4_GROUPED_ATTN_A` | presence rollback; unset permits the caller-marked resident decode production-shape default and explicit ENABLE/REQUIRE; any defined value including empty or 0 disables all grouped attention-A paths and wins over ENABLE/REQUIRE | Restore eight standalone Q4 attention-A projections instead of the two-dispatch grouped path. | [rocm/ds4_rocm_q4.cuh:868](rocm/ds4_rocm_q4.cuh#L868) | | `DS4_ROCM_DISABLE_Q4_PREFILL_TILE8` | presence rollback; TILE8 is default for 9..4096 tokens | Disable/roll back rocm disable q4 prefill tile8. | [rocm/ds4_rocm_q4.cuh:448](rocm/ds4_rocm_q4.cuh#L448) | | `DS4_ROCM_DISABLE_Q4_SELECTED_EXPERT_VIEWS` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable q4 selected expert views. | [ds4.c:21135](ds4.c#L21135) | | `DS4_ROCM_DISABLE_RESIDENT_IQ2_SORTED` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable resident iq2 sorted. | [rocm/ds4_rocm_moe_launch.cuh:747](rocm/ds4_rocm_moe_launch.cuh#L747) | @@ -963,19 +973,19 @@ and **19 tool/wrapper entries**. | `DS4_ROCM_ENABLE_MXFP4_TILE32` | presence opt-in; unset=off; any defined value including empty or 0 enables the candidate when the MXFP4 sorted-tile path has at least 32 tokens and the expert intermediate dimension is divisible by 32 | Select the ROCm MXFP4 gate/up tile32 kernel, reusing each loaded expert-weight chunk across as many as 32 tokens. | [rocm/ds4_rocm_moe_launch.cuh:786](rocm/ds4_rocm_moe_launch.cuh#L786) | | `DS4_ROCM_ENABLE_MXFP4_TILE4` | presence opt-in; unset=off; any defined value including empty or 0 enables the candidate when the MXFP4 sorted-tile path has at least 5 tokens and neither TILE32 nor LDSB is selected | Select the ROCm MXFP4 gate/up tile4 occupancy variant, reducing staged-activation LDS per block. | [rocm/ds4_rocm_moe_launch.cuh:794](rocm/ds4_rocm_moe_launch.cuh#L794) | | `DS4_ROCM_ENABLE_Q4_DENSE_PAIR` | presence opt-in; unset=off; DISABLE takes precedence | Enable rocm enable q4 dense pair. | [rocm/ds4_rocm_q4.cuh:434](rocm/ds4_rocm_q4.cuh#L434) | -| `DS4_ROCM_ENABLE_Q4_GROUPED_ATTN_A` | presence opt-in; unset=off unless REQUIRE; DISABLE wins | Enable rocm enable q4 grouped attn a. | [rocm/ds4_rocm_q4.cuh:704](rocm/ds4_rocm_q4.cuh#L704) | -| `DS4_ROCM_ENABLE_STREAMING_FULL_EXPERT_ADDR_TABLE` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming full expert addr table. | [ds4.c:18240](ds4.c#L18240) | -| `DS4_ROCM_ENABLE_STREAMING_MADVISE_WILLNEED` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming madvise willneed. | [ds4.c:18209](ds4.c#L18209) | -| `DS4_ROCM_ENABLE_STREAMING_PREFILL_BATCH_SELECTED_ADDR` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming prefill batch selected addr. | [ds4.c:18569](ds4.c#L18569) | -| `DS4_ROCM_ENABLE_STREAMING_PREFILL_CACHE_SEED` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming prefill cache seed. | [ds4.c:21258](ds4.c#L21258) | -| `DS4_ROCM_ENABLE_STREAMING_PREFILL_LAYER_PAGEIN` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming prefill layer pagein. | [ds4.c:18431](ds4.c#L18431) | -| `DS4_ROCM_ENABLE_STREAMING_PREFILL_LAYER_READAHEAD` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming prefill layer readahead. | [ds4.c:18441](ds4.c#L18441) | -| `DS4_ROCM_ENABLE_STREAMING_PREFILL_SELECTED_MADVISE` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming prefill selected madvise. | [ds4.c:18421](ds4.c#L18421) | -| `DS4_ROCM_ENABLE_STREAMING_PREFILL_SELECTED_PAGEIN` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming prefill selected pagein. | [ds4.c:18411](ds4.c#L18411) | -| `DS4_ROCM_ENABLE_STREAMING_PREFILL_SELECTED_READAHEAD` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming prefill selected readahead. | [ds4.c:19955](ds4.c#L19955) | -| `DS4_ROCM_ENABLE_STREAMING_PREFILL_SELECTED_READAHEAD_SHARED` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming prefill selected readahead shared. | [ds4.c:19957](ds4.c#L19957) | -| `DS4_ROCM_ENABLE_STREAMING_READAHEAD` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming readahead. | [ds4.c:18202](ds4.c#L18202) | -| `DS4_ROCM_ENABLE_STREAMING_STATIC_DECODE_MAP` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming static decode map. | [ds4.c:18221](ds4.c#L18221) | +| `DS4_ROCM_ENABLE_Q4_GROUPED_ATTN_A` | presence opt-in outside the default scope; the exact caller-marked resident decode shape groups=8, N=1, K=4096, M=1024 is automatic, while row-at-a-time batch fallbacks are not; DISABLE wins | Enable grouped Q4 attention-A for eligible slices, non-production shapes, or explicit experiments in addition to the resident decode default. | [rocm/ds4_rocm_q4.cuh:872](rocm/ds4_rocm_q4.cuh#L872) | +| `DS4_ROCM_ENABLE_STREAMING_FULL_EXPERT_ADDR_TABLE` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming full expert addr table. | [ds4.c:18255](ds4.c#L18255) | +| `DS4_ROCM_ENABLE_STREAMING_MADVISE_WILLNEED` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming madvise willneed. | [ds4.c:18224](ds4.c#L18224) | +| `DS4_ROCM_ENABLE_STREAMING_PREFILL_BATCH_SELECTED_ADDR` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming prefill batch selected addr. | [ds4.c:18584](ds4.c#L18584) | +| `DS4_ROCM_ENABLE_STREAMING_PREFILL_CACHE_SEED` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming prefill cache seed. | [ds4.c:21273](ds4.c#L21273) | +| `DS4_ROCM_ENABLE_STREAMING_PREFILL_LAYER_PAGEIN` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming prefill layer pagein. | [ds4.c:18446](ds4.c#L18446) | +| `DS4_ROCM_ENABLE_STREAMING_PREFILL_LAYER_READAHEAD` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming prefill layer readahead. | [ds4.c:18456](ds4.c#L18456) | +| `DS4_ROCM_ENABLE_STREAMING_PREFILL_SELECTED_MADVISE` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming prefill selected madvise. | [ds4.c:18436](ds4.c#L18436) | +| `DS4_ROCM_ENABLE_STREAMING_PREFILL_SELECTED_PAGEIN` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming prefill selected pagein. | [ds4.c:18426](ds4.c#L18426) | +| `DS4_ROCM_ENABLE_STREAMING_PREFILL_SELECTED_READAHEAD` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming prefill selected readahead. | [ds4.c:19970](ds4.c#L19970) | +| `DS4_ROCM_ENABLE_STREAMING_PREFILL_SELECTED_READAHEAD_SHARED` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming prefill selected readahead shared. | [ds4.c:19972](ds4.c#L19972) | +| `DS4_ROCM_ENABLE_STREAMING_READAHEAD` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming readahead. | [ds4.c:18217](ds4.c#L18217) | +| `DS4_ROCM_ENABLE_STREAMING_STATIC_DECODE_MAP` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming static decode map. | [ds4.c:18236](ds4.c#L18236) | | `DS4_ROCM_GLM_CAUSAL_ATTN_GEMM` | Enabled by default when unset. Exact "0" or an empty value disables; every other nonempty value enables (including false/off/no), because cuda_env_present only tests nonempty and != "0". Eligibility still requires causal_range && !has_selected; a failed GEMM helper falls through to the scalar attention kernel. | Use FP16 BLAS GEMMs for dense causal GLM indexed prefill; =0 is the correctness/performance rollback to the scalar attention kernel. | [rocm/ds4_rocm_glm.cuh:3283](rocm/ds4_rocm_glm.cuh#L3283) | | `DS4_ROCM_GLM_DISABLE_STREAMING_EXPERT_CACHE` | Pure presence flag: any defined value, including empty or "0", disables. Unset leaves automatic GLM streaming expert-cache eligibility enabled for supported model/quant/quality/SSD configurations. On ROCm builds DS4_METAL_GLM_DISABLE_STREAMING_EXPERT_CACHE is an accepted fallback alias. | Disable selected/resident streamed-expert cache paths and force generic/full-layer expert handling for GLM SSD streaming. | [ds4.c:21197](ds4.c#L21197) | | `DS4_ROCM_GLM_DISABLE_STREAMING_SEED_BEFORE_PREFILL` | Pure presence flag: any defined value, including empty or "0", disables. Unset seeds before prefill whenever SSD streaming is active. On ROCm builds DS4_METAL_GLM_DISABLE_STREAMING_SEED_BEFORE_PREFILL is an accepted fallback alias. | Skip the pre-prefill hotlist seed of the streaming expert cache in both one-shot GLM generation and session setup. | [ds4.c:51073](ds4.c#L51073) | @@ -1015,11 +1025,11 @@ and **19 tool/wrapper entries**. | `DS4_ROCM_MOE_PATH_DEBUG` | presence diagnostic; unset=off; any defined value including empty or 0 enables it | Print ROCm routed-MoE path selection, sorted-tile scratch state, and MXFP4 gate/up launch diagnostics to stderr. | [rocm/ds4_rocm_moe_launch.cuh:831](rocm/ds4_rocm_moe_launch.cuh#L831) | | `DS4_ROCM_MOE_WRITE_CLAMPED_ACT` | Pure presence sentinel: any defined value, including empty or "0", is active; DS4_METAL_MOE_WRITE_CLAMPED_ACT is an accepted fallback alias. On ROCm the variable is only consumed as a path-admission veto: it disables selected-expert cache/address-table, selected-slot and CPU-router/fused optimized paths. No ROCm call site parses a clamp amount or directly enables a write-clamped kernel. | Force shared graph selection away from optimizations incompatible with the clamped-intermediate MoE diagnostic; on ROCm this is a compatibility/rollback gate, not itself a clamped-write implementation. | [ds4.c:18526](ds4.c#L18526) | | `DS4_ROCM_MXFP4_DOWN_RGROUP` | nonempty value is parsed by strtol and a numeric prefix is sufficient; integers 1..8 are accepted; unset, empty, invalid, or out-of-range values use 1 | Set how many 32-row output blocks each ROCm MXFP4 tiled down-projection block computes, reducing the first launch-grid dimension as the value increases. | [rocm/ds4_rocm_moe_launch.cuh:801](rocm/ds4_rocm_moe_launch.cuh#L801) | -| `DS4_ROCM_Q4_GROUPED_ATTN_A_STATS` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Print counters for rocm q4 grouped attn a stats. | [rocm/ds4_rocm_q4.cuh:480](rocm/ds4_rocm_q4.cuh#L480) | +| `DS4_ROCM_Q4_GROUPED_ATTN_A_STATS` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Print counters for rocm q4 grouped attn a stats. | [rocm/ds4_rocm_q4.cuh:614](rocm/ds4_rocm_q4.cuh#L614) | | `DS4_ROCM_Q4_PREFILL_TILE8_STATS` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Print counters for rocm q4 prefill tile8 stats. | [rocm/ds4_rocm_q4.cuh:514](rocm/ds4_rocm_q4.cuh#L514) | | `DS4_ROCM_Q8_DECODE_SHAREDX_64K` | sampled once; unset: enabled; present empty or exact 0: disabled; every other present value: enabled; effective only for one-token non-prequant Q8_0 matmul with 8192 < in_dim <= 16384 | Allows the ROCm shared-input Q8 decode kernel to use up to 64 KiB dynamic LDS for wide inputs; an unsupported/failed LDS launch automatically falls back to the regular kernel. | [rocm/ds4_rocm_runtime.cuh:4805](rocm/ds4_rocm_runtime.cuh#L4805) | -| `DS4_ROCM_Q_STAGE_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm q stage profile. | [ds4.c:29269](ds4.c#L29269) | -| `DS4_ROCM_REQUIRE_Q4_GROUPED_ATTN_A` | presence fail-closed assertion; also requests candidate unless disabled | Require rocm require q4 grouped attn a and fail instead of silently falling back. | [rocm/ds4_rocm_q4.cuh:702](rocm/ds4_rocm_q4.cuh#L702) | +| `DS4_ROCM_Q_STAGE_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm q stage profile. | [ds4.c:29284](ds4.c#L29284) | +| `DS4_ROCM_REQUIRE_Q4_GROUPED_ATTN_A` | presence fail-closed assertion; also requests the candidate outside the caller-marked resident decode default; DISABLE remains authoritative and causes failure | Require grouped Q4 attention-A and fail instead of silently falling back. | [rocm/ds4_rocm_q4.cuh:870](rocm/ds4_rocm_q4.cuh#L870) | | `DS4_ROCM_REQUIRE_Q4_PREFILL_TILE8` | presence fail-closed assertion for eligible TILE8 calls | Require rocm require q4 prefill tile8 and fail instead of silently falling back. | [rocm/ds4_rocm_q4.cuh:452](rocm/ds4_rocm_q4.cuh#L452) | | `DS4_ROCM_STREAMING_DECODE_PREFILL_MAX` | primary nonempty value over the Metal alias; parsed by strtol when it has a numeric prefix (trailing text is accepted); <= 0 disables, values > UINT32_MAX clamp, no numeric prefix uses automatic default: 64 for Flash with uniform Q4_K/MXFP4 experts, 18 for other Pro/Flash, otherwise 0; the disable flag dominates | Sets the largest short, non-quality SSD-streaming prefill batch routed through the decode-style path instead of canonical layer-major prefill. | [ds4.c:31961](ds4.c#L31961) | | `DS4_ROCM_STREAMING_EXPERT_AUTO_PRELOAD_CAP` | primary nonempty value over the Metal alias; strict full-string strtoul; valid values > UINT32_MAX clamp, invalid uses 4096, and 0 means no cap (not disabled); when CLI preload is auto/0, unset defaults to cap 4096 except ROCm GLM52, where absent/empty disables automatic preload entirely | Caps the number of hot experts synchronously seeded into the SSD-streaming expert cache in automatic preload mode; an explicit CLI preload count bypasses this cap, and setting this variable opts ROCm GLM52 back into auto preload. | [ds4.c:21454](ds4.c#L21454) | @@ -1165,7 +1175,7 @@ and **19 tool/wrapper entries**.
-General and shared (76) +General and shared (77) | Variable | Accepted value and default | Effect | Source | | --- | --- | --- | --- | @@ -1180,8 +1190,9 @@ and **19 tool/wrapper entries**. | `DS4_CPU_DUMP_LOGITS` | filesystem path; unset disables read/write | Dump or select diagnostic data for cpu dump logits. | [ds4.c:39086](ds4.c#L39086) | | `DS4_CPU_DUMP_PREFILL_LOGITS` | filesystem path; unset disables read/write | Dump or select diagnostic data for cpu dump prefill logits. | [ds4.c:41606](ds4.c#L41606) | | `DS4_DECODE_PROFILE_DETAIL` | presence flag; unset=off | Print per-stage timing for the single-token CPU FFN path. | [ds4.c:12209](ds4.c#L12209) | -| `DS4_EXPERT_HOTLIST` | nonempty filesystem path; unset=off; currently Metal-only | Load an expert hotlist for Metal expert profiling/streaming. | [ds4.c:60369](ds4.c#L60369) | -| `DS4_EXPERT_PROFILE` | presence diagnostic flag; unset=off | Collect timing/profile diagnostics for expert profile. | [ds4.c:60367](ds4.c#L60367) | +| `DS4_DISABLE_GREEDY_TOP1_READBACK` | presence rollback; unset uses device top-1 plus a 4-byte readback for eligible single-tier greedy generation, including SSD streaming; any defined value including empty or 0 restores full-logits readback and CPU argmax | Restore the legacy per-token full-logits host readback for greedy generation A/B and emergency rollback. | [ds4.c:51964](ds4.c#L51964) | +| `DS4_EXPERT_HOTLIST` | nonempty filesystem path; unset=off; currently Metal-only | Load an expert hotlist for Metal expert profiling/streaming. | [ds4.c:60637](ds4.c#L60637) | +| `DS4_EXPERT_PROFILE` | presence diagnostic flag; unset=off | Collect timing/profile diagnostics for expert profile. | [ds4.c:60635](ds4.c#L60635) | | `DS4_FORCE_CUDA_PEER` | presence flag read once at CUDA init; unset uses automatic transfer selection; any defined value including 0 enables | Force cross-device transfers through cudaMemcpyPeerAsync for diagnostics. | [ds4_cuda.cu:364](ds4_cuda.cu#L364) | | `DS4_FORCE_HOST_BOUNCE` | presence flag read once at CUDA init; unset uses automatic transfer selection; any defined value including 0 enables | Force cross-device transfers through pinned host bounce buffers for diagnostics. | [ds4_cuda.cu:365](ds4_cuda.cu#L365) | | `DS4_LOCK_FILE` | path string; default /tmp/ds4.lock | Override the single-instance lock file. | [ds4.c:51999](ds4.c#L51999) | @@ -1285,6 +1296,7 @@ them as part of its own test contract. | `DS4_TEST_LOGPROB_AUTO_METAL` | presence flag; unset forces DS4_METAL_DISABLE_METAL4=1; any presence including empty or 0 removes that rollback and permits automatic Metal selection | Runs official log-probability vectors with automatic Metal-path selection instead of the fixed pre-Metal4 baseline. | [tests/ds4_test.c:7235](tests/ds4_test.c#L7235) | | `DS4_TEST_LONG_PROMPT` | nonempty readable prompt-file path; unset/empty defaults to tests/long_context_story_prompt.txt | Selects the rendered story prompt for the long-context fact-recall test. | [tests/ds4_test.c:6970](tests/ds4_test.c#L6970) | | `DS4_TEST_LONG_WORDS` | atoi integer; unset/empty/nonnumeric defaults to 0; valid range is 0..DS4_TEST_CONTEXT-128 and numeric prefixes are accepted | Adds repeated words to alternating CUDA session-batch prompts to exercise long-prefill rows. | [tests/test_cuda_session_batch.c:135](tests/test_cuda_session_batch.c#L135) | +| `DS4_TEST_METAL_ARGMAX_TOP1_TIMING` | presence flag; unset runs correctness only; any defined value including empty or 0 also runs the GGUF-free resident production-shape A/B | Measures generic argsort versus dedicated Metal top-1 and full-readback versus 4-byte greedy selection using resident synthetic logits, excluding model and SSD I/O. | [tests/test_metal_argmax_top1.c:545](tests/test_metal_argmax_top1.c#L545) | | `DS4_TEST_METAL_EXACTN_BATCH_HEAD` | nonempty value other than exact 0 enables; unset/empty/0 disables; false/off also enable | Enables the Metal exact-N batch-head path and requires its attempt/use counters for every eligible oracle case. | [tests/test_metal_exactn_oracle.c:401](tests/test_metal_exactn_oracle.c#L401) | | `DS4_TEST_METAL_EXACTN_ORACLE` | presence flag compiled only with DS4_TEST_HOOKS; absent from normal production builds | Force allocation of the exact-N Metal verifier/oracle workspace in tests. | [ds4.c:61991](ds4.c#L61991) | | `DS4_TEST_MIXED_INITIAL` | integer 128..context-1; default 128 | Set the initial prefill length for the CUDA mixed-batch oracle. | [tests/test_cuda_mixed_batch.c:111](tests/test_cuda_mixed_batch.c#L111) | diff --git a/Makefile b/Makefile index cffc6b839..ea967cab9 100644 --- a/Makefile +++ b/Makefile @@ -69,7 +69,7 @@ DS4_LINK_LIBS ?= $(CUDA_LDLIBS) METAL_LDLIBS := $(LDLIBS) endif -.PHONY: all help clean test test-ssd environment-docs test-quantizer-indexer-q4 test-rocm test-glm53-kda-rocm test-metal-session-batch test-metal-session-batch-ssd test-metal-q4-streams test-metal-indexer-q4 test-metal-q4-attn-exactn test-metal-q4-qb-f16-cache test-metal-q4-qb-f16-cache-timing test-metal-exactn-oracle test-metal-dspark-capture test-metal-iq2-midonly test-metal-iq2-ssd-grouped-mm test-metal-iq2-live-index test-mxfp4-metal test-mxfp4-cuda test-mxfp4-rocm test-mmq-parity-cuda test-rocm-q4-parity test-rocm-q4-dense test-rocm-q4-pair test-rocm-q4-prefill test-strix-rocm-q4-parity test-strix-rocm-q4-prefill test-strix-rocm-q4-prefill-long test-cuda-session-batch test-cuda-mixed-batch dspark-acceptance dspark-verify-depth rocm-dspark-acceptance rocm-dspark-verify-depth mtp-verify-depth cpu cuda cuda-spark cuda-generic cuda-regression strix-halo rocm +.PHONY: all help clean test test-ssd environment-docs test-quantizer-indexer-q4 test-rocm test-glm53-kda-rocm test-metal-session-batch test-metal-session-batch-ssd test-metal-q4-streams test-metal-indexer-q4 test-metal-q4-attn-exactn test-metal-q4-qb-f16-cache test-metal-q4-qb-f16-cache-timing test-metal-exactn-oracle test-metal-dspark-capture test-metal-argmax-top1 bench-metal-argmax-top1 test-metal-iq2-midonly test-metal-iq2-ssd-grouped-mm test-metal-iq2-live-index test-mxfp4-metal test-mxfp4-cuda test-mxfp4-rocm test-mmq-parity-cuda test-rocm-q4-parity test-rocm-q4-dense test-rocm-q4-pair test-rocm-q4-prefill test-strix-rocm-q4-parity test-strix-rocm-q4-prefill test-strix-rocm-q4-prefill-long test-cuda-session-batch test-cuda-mixed-batch dspark-acceptance dspark-verify-depth rocm-dspark-acceptance rocm-dspark-verify-depth mtp-verify-depth cpu cuda cuda-spark cuda-generic cuda-regression strix-halo rocm gguf-tools/deepseek4-quantize: gguf-tools/deepseek4-quantize.c gguf-tools/quants.c gguf-tools/quants.h $(MAKE) -C gguf-tools deepseek4-quantize @@ -100,6 +100,8 @@ help: @echo " make test-metal-q4-qb-f16-cache Oracle for M1-M4 Q4 q_b sidecar and transient F16 paths" @echo " make test-metal-q4-qb-f16-cache-timing Compare Q4 direct, sidecar, and transient production at N=4096" @echo " make test-metal-dspark-capture Check fused DSpark HC capture bitwise" + @echo " make test-metal-argmax-top1 Check the resident production-shape Metal decode argmax" + @echo " make bench-metal-argmax-top1 Time full argsort versus resident top-1 without GGUF/SSD" @echo " make test-metal-iq2-midonly Check M1 IQ2 addr mid-only output and sentinels" @echo " make test-metal-iq2-live-index Check IQ2 SSD live-cache index policy and fallback" @echo " make test-rocm-q4-parity Run ROCm Q4_K dense/pair/prefill oracle (or SKIP without HIP)" @@ -239,6 +241,18 @@ tests/test_metal_dspark_capture: tests/test_metal_dspark_capture.o ds4_metal.o test-metal-dspark-capture: tests/test_metal_dspark_capture ./tests/test_metal_dspark_capture +tests/test_metal_argmax_top1.o: tests/test_metal_argmax_top1.c ds4_gpu.h + $(CC) $(CFLAGS) -fno-fast-math -I. -c -o $@ $< + +tests/test_metal_argmax_top1: tests/test_metal_argmax_top1.o ds4_metal.o + $(CC) $(CFLAGS) -o $@ $^ $(METAL_LDLIBS) + +test-metal-argmax-top1: tests/test_metal_argmax_top1 + env -u DS4_TEST_METAL_ARGMAX_TOP1_TIMING ./tests/test_metal_argmax_top1 + +bench-metal-argmax-top1: tests/test_metal_argmax_top1 + DS4_TEST_METAL_ARGMAX_TOP1_TIMING=1 ./tests/test_metal_argmax_top1 + tests/test_metal_iq2_midonly.o: tests/test_metal_iq2_midonly.c ds4_gpu.h $(CC) $(CFLAGS) -I. -c -o $@ $< @@ -879,4 +893,4 @@ mxfp4-dot-test: tests/test_mxfp4_dot.c ./tests/test_mxfp4_dot clean: - rm -f ds4 ds4-server ds4-bench ds4-eval ds4-agent ds4_cpu ds4_native ds4_server_test ds4_test ds4_agent_test gguf-tools/quality-testing/score_official gguf-tools/quality-testing/score_official.o speed-bench/metal_decode_schedule_bench speed-bench/metal_prefill_variant_bench speed-bench/*.o tests/test_q4k_dot tests/test_mxfp4_dot tests/test_quantizer_indexer_q4 tests/test_mxfp4_metal tests/test_mxfp4_rocm tests/bench_mxfp4_rocm tests/test_mxfp4_cuda tests/test_rocm_q4_dense_pair tests/test_metal_session_batch tests/test_metal_q4_streams tests/test_metal_indexer_q4 tests/test_metal_q4_attn_exactn tests/test_metal_q4_qb_f16_cache tests/test_metal_exactn_oracle tests/test_metal_dspark_capture tests/test_metal_iq2_midonly tests/test_metal_iq2_ssd_grouped_mm tests/test_metal_iq2_live_index tests/test_glm53_kda tests/test_glm53_kda_rocm tests/test_glm53_vision_engine tests/test_glm53_vision_prompt tests/test_gpu_xdev tests/test_gpu_model_cache tests/test_gpu_lookup_cache_strict tests/test_engine_mgpu_refusal tests/test_engine_mgpu_runtime tests/test_engine_correctness tests/test_sampling tests/test_cuda_session_batch tests/test_cuda_mixed_batch tests/*.o *.o tests/cuda_long_context_smoke tests/cuda_long_context_smoke.o + rm -f ds4 ds4-server ds4-bench ds4-eval ds4-agent ds4_cpu ds4_native ds4_server_test ds4_test ds4_agent_test gguf-tools/quality-testing/score_official gguf-tools/quality-testing/score_official.o speed-bench/metal_decode_schedule_bench speed-bench/metal_prefill_variant_bench speed-bench/*.o tests/test_q4k_dot tests/test_mxfp4_dot tests/test_quantizer_indexer_q4 tests/test_mxfp4_metal tests/test_mxfp4_rocm tests/bench_mxfp4_rocm tests/test_mxfp4_cuda tests/test_rocm_q4_dense_pair tests/test_metal_session_batch tests/test_metal_q4_streams tests/test_metal_indexer_q4 tests/test_metal_q4_attn_exactn tests/test_metal_q4_qb_f16_cache tests/test_metal_exactn_oracle tests/test_metal_dspark_capture tests/test_metal_argmax_top1 tests/test_metal_iq2_midonly tests/test_metal_iq2_ssd_grouped_mm tests/test_metal_iq2_live_index tests/test_glm53_kda tests/test_glm53_kda_rocm tests/test_glm53_vision_engine tests/test_glm53_vision_prompt tests/test_gpu_xdev tests/test_gpu_model_cache tests/test_gpu_lookup_cache_strict tests/test_engine_mgpu_refusal tests/test_engine_mgpu_runtime tests/test_engine_correctness tests/test_sampling tests/test_cuda_session_batch tests/test_cuda_mixed_batch tests/*.o *.o tests/cuda_long_context_smoke tests/cuda_long_context_smoke.o diff --git a/ds4.c b/ds4.c index ae0757b07..e90624189 100644 --- a/ds4.c +++ b/ds4.c @@ -21502,7 +21502,8 @@ static bool metal_graph_attention_output_dense_quant_low( uint64_t rank, uint32_t group0, uint32_t group_cnt, - const ds4_gpu_tensor *heads); + const ds4_gpu_tensor *heads, + bool resident_decode); static bool metal_graph_attention_output_dense_quant_tp( ds4_gpu_tensor *out, ds4_gpu_tensor *low, @@ -25204,7 +25205,7 @@ static bool metal_graph_encode_decode_layer_phase( metal_graph_attn_low(g), g, model, layer->attn_output_a, group_dim, rank, 0, n_groups, - metal_graph_heads(g)); + metal_graph_heads(g), true); #if !defined(DS4_NO_GPU) && !defined(DS4_ROCM_BUILD) if (ok) { ok = ds4_gpu_matmul_q4_K_hc_expand_tensor( @@ -25250,7 +25251,8 @@ static bool metal_graph_encode_decode_layer_phase( rank, 0, n_groups, - metal_graph_heads(g)); + metal_graph_heads(g), + true); if (ok) ok = metal_graph_matmul_dense_quant_tensor(attn_out_dst, model, layer->attn_output_b, @@ -27653,7 +27655,8 @@ static bool metal_graph_attention_output_dense_quant_low( uint64_t rank, uint32_t group0, uint32_t group_cnt, - const ds4_gpu_tensor *heads) { + const ds4_gpu_tensor *heads, + bool resident_decode) { (void)g; if (!low || !model || !out_a || !heads || group_dim == 0 || rank == 0 || group_cnt == 0) { @@ -27682,7 +27685,8 @@ static bool metal_graph_attention_output_dense_quant_low( rank, group0, group_cnt, - heads); + heads, + resident_decode ? 1 : 0); if (q4_slice_rc > 0) return true; if (q4_slice_rc < 0) return false; } @@ -27756,7 +27760,8 @@ static bool metal_graph_attention_output_dense_quant_tp( rank, group0, group_cnt, - heads)) { + heads, + true)) { return false; } return metal_graph_matmul_dense_quant_kslice(out, @@ -27849,7 +27854,8 @@ static bool metal_graph_attention_output_dense_quant_batch( rank, 0, n_groups, - heads_row); + heads_row, + false); if (ok) ok = metal_graph_matmul_dense_quant_tensor(out_row, model, out_b, @@ -32565,7 +32571,8 @@ static bool metal_graph_eval_token_raw_swa_streaming( const ds4_weights *weights, int token, uint32_t pos, - float *logits) { + float *logits, + int *top_id) { if (g->raw_cap == 0) { fprintf(stderr, "ds4: Metal graph raw KV cache is not allocated\n"); return false; @@ -32575,6 +32582,13 @@ static bool metal_graph_eval_token_raw_swa_streaming( glm_graph_env_present("DS4_ROCM_GRAPH_TOKEN_PROFILE", "DS4_METAL_GRAPH_TOKEN_PROFILE"); const bool throttle = graph_power_throttle_enabled(g); + const bool need_output = logits != NULL || top_id != NULL; + if (need_output && !weights_have_output_head(weights)) { + fprintf(stderr, + "ds4: SSD streaming decode requested logits/top-1 without " + "a complete output head\n"); + return false; + } const double t0 = (profile || throttle) ? now_sec() : 0.0; const uint32_t raw_row = pos % g->raw_cap; const uint32_t n_raw = metal_graph_raw_span_for_batch(g, pos, 1); @@ -32630,25 +32644,38 @@ static bool metal_graph_eval_token_raw_swa_streaming( ok = metal_graph_dspark_capture_decode_layer(g, il); } } - if (ok && logits) { + if (ok && need_output) { ok = metal_graph_encode_output_head(g, model, weights, weights->output->dim[1]); } + if (ok && top_id) { + ok = ds4_gpu_argmax_tensor(metal_graph_comp_selected(g), + metal_graph_logits(g), + DS4_N_VOCAB) != 0; + } const double t_encoded = (profile || throttle) ? now_sec() : 0.0; if (ok) ok = ds4_gpu_end_commands() != 0; const double t_done = (profile || throttle) ? now_sec() : 0.0; + if (ok && top_id) { + int32_t device_top = -1; + ok = ds4_gpu_tensor_read(metal_graph_comp_selected(g), 0, + &device_top, sizeof(device_top)) != 0 && + device_top >= 0 && (uint32_t)device_top < DS4_N_VOCAB; + if (ok) *top_id = (int)device_top; + } if (ok && logits) { ok = ds4_gpu_tensor_read(metal_graph_logits(g), 0, logits, (uint64_t)DS4_N_VOCAB * sizeof(float)) != 0; } const double t_read = (profile || throttle) ? now_sec() : 0.0; if (profile) { fprintf(stderr, - "ds4: metal SSD streaming batched token pos=%u encode=%.3f ms execute=%.3f ms read=%.3f ms total=%.3f ms logits=%d\n", + "ds4: metal SSD streaming batched token pos=%u encode=%.3f ms execute=%.3f ms read=%.3f ms total=%.3f ms logits=%d top=%d\n", pos, (t_encoded - t0) * 1000.0, (t_done - t_encoded) * 1000.0, (t_read - t_done) * 1000.0, (t_read - t0) * 1000.0, - logits != NULL); + logits != NULL, + top_id != NULL); } if (ok && throttle) { graph_power_note_decode_token(g, t_read - t0); @@ -32672,7 +32699,7 @@ static bool metal_graph_eval_token_raw_swa_streaming( } if (!static_decode_map && il + 1 < DS4_N_LAYER) { metal_graph_stream_readahead_layer_decode(model, weights, il + 1); - } else if (!static_decode_map && logits) { + } else if (!static_decode_map && need_output) { metal_graph_stream_readahead_output(model, weights); } if (ok) ok = ds4_gpu_begin_commands() != 0; @@ -32705,31 +32732,44 @@ static bool metal_graph_eval_token_raw_swa_streaming( } } - if (ok && logits && !static_decode_map) ok = metal_graph_stream_map_output(model, weights); + if (ok && need_output && !static_decode_map) ok = metal_graph_stream_map_output(model, weights); const double t_head0 = profile ? now_sec() : 0.0; - if (ok && logits) ok = ds4_gpu_begin_commands() != 0; - if (ok && logits) ok = metal_graph_encode_output_head(g, model, weights, weights->output->dim[1]); + if (ok && need_output) ok = ds4_gpu_begin_commands() != 0; + if (ok && need_output) ok = metal_graph_encode_output_head(g, model, weights, weights->output->dim[1]); + if (ok && top_id) { + ok = ds4_gpu_argmax_tensor(metal_graph_comp_selected(g), + metal_graph_logits(g), + DS4_N_VOCAB) != 0; + } const double t_head_encoded = profile ? now_sec() : 0.0; - if (ok && logits) ok = ds4_gpu_end_commands() != 0; + if (ok && need_output) ok = ds4_gpu_end_commands() != 0; const double t_done = (profile || throttle) ? now_sec() : 0.0; + if (ok && top_id) { + int32_t device_top = -1; + ok = ds4_gpu_tensor_read(metal_graph_comp_selected(g), 0, + &device_top, sizeof(device_top)) != 0 && + device_top >= 0 && (uint32_t)device_top < DS4_N_VOCAB; + if (ok) *top_id = (int)device_top; + } if (ok && logits) { ok = ds4_gpu_tensor_read(metal_graph_logits(g), 0, logits, (uint64_t)DS4_N_VOCAB * sizeof(float)) != 0; } const double t_read = (profile || throttle) ? now_sec() : 0.0; if (profile) { - if (logits) { + if (need_output) { encode_s += t_head_encoded - t_head0; execute_s += t_done - t_head_encoded; } fprintf(stderr, - "ds4: metal SSD streaming token pos=%u encode=%.3f ms execute=%.3f ms read=%.3f ms total=%.3f ms logits=%d\n", + "ds4: metal SSD streaming token pos=%u encode=%.3f ms execute=%.3f ms read=%.3f ms total=%.3f ms logits=%d top=%d\n", pos, encode_s * 1000.0, execute_s * 1000.0, (t_read - t_done) * 1000.0, (t_read - t0) * 1000.0, - logits != NULL); + logits != NULL, + top_id != NULL); } if (ok) graph_power_note_decode_token(g, t_read - t0); if (!ok) { @@ -32749,7 +32789,8 @@ static bool metal_graph_eval_token_raw_swa( uint32_t pos, float *logits) { if (g && g->ssd_streaming) { - return metal_graph_eval_token_raw_swa_streaming(g, model, weights, token, pos, logits); + return metal_graph_eval_token_raw_swa_streaming( + g, model, weights, token, pos, logits, NULL); } const bool profile = @@ -33214,34 +33255,50 @@ typedef struct { * Keeping intermediate rows device-resident avoids turning verification into a * sequence of large CPU readbacks. */ static bool metal_graph_eval_token_raw_swa_top( - ds4_gpu_graph *g, - const ds4_model *model, - const ds4_weights *weights, - int token, - uint32_t pos, - int *top_id, - float *logits, - bool allow_split_top1, + ds4_gpu_graph *g, + const ds4_model *model, + const ds4_weights *weights, + int token, + uint32_t pos, + int *top_id, + float *logits, + bool allow_split_top1, metal_graph_top2_result *top2, - bool force_fast_attention) { + bool force_fast_attention) { if (!top_id) return false; if (top2) memset(top2, 0, sizeof(*top2)); + /* SSD-backed decode must use the mapper on every backend. CUDA's + * optional approximate/split helpers are resident-only; run the exact + * streaming path instead. TP output owns only a vocabulary slice, so a + * local full-row argmax would be stale and is rejected until it has an + * explicit cross-rank merge. */ + if (g && g->ssd_streaming) { + if (g->tp_world >= 2u) { + fprintf(stderr, + "ds4: SSD streaming top-1 is unsupported with tensor " + "parallel output\n"); + return false; + } + return metal_graph_eval_token_raw_swa_streaming( + g, model, weights, token, pos, logits, top_id); + } const bool fast_attention = allow_split_top1 && logits == NULL && (force_fast_attention || metal_graph_cuda_greedy_splitkv_requested()); - if (top2) top2->fast_attention = fast_attention; - const int old_fast_attention = - ds4_gpu_set_decode_fast_attention(fast_attention ? 1 : 0); - const bool profile = getenv("DS4_METAL_GRAPH_TOKEN_PROFILE") != NULL; - const double t0 = profile ? now_sec() : 0.0; const bool split_top1 = allow_split_top1 && logits == NULL && top2 == NULL && + g && g->cuda_tp_output && metal_graph_cuda_greedy_split_top1_requested(); + if (top2) top2->fast_attention = fast_attention; + const int old_fast_attention = + ds4_gpu_set_decode_fast_attention(fast_attention ? 1 : 0); + const bool profile = getenv("DS4_METAL_GRAPH_TOKEN_PROFILE") != NULL; + const double t0 = profile ? now_sec() : 0.0; if (split_top1) { int output_tiers[DS4_MAX_GPUS] = {0}; uint32_t output_ways = 0; @@ -33328,7 +33385,7 @@ static bool metal_graph_eval_token_raw_swa_top( 0, values, sizeof(values)) != 0; - if (ok && ids[0] <= (uint32_t)INT32_MAX && ids[1] <= (uint32_t)INT32_MAX) { + if (ok && ids[0] < DS4_N_VOCAB && ids[1] < DS4_N_VOCAB) { top2->id0 = (int)ids[0]; top2->id1 = (int)ids[1]; top2->value0 = values[0]; @@ -33339,7 +33396,11 @@ static bool metal_graph_eval_token_raw_swa_top( ok = false; } } else if (ok) { - ok = ds4_gpu_tensor_read(metal_graph_comp_selected(g), 0, top_id, sizeof(*top_id)) != 0; + int32_t device_top = -1; + ok = ds4_gpu_tensor_read(metal_graph_comp_selected(g), 0, + &device_top, sizeof(device_top)) != 0 && + device_top >= 0 && (uint32_t)device_top < DS4_N_VOCAB; + if (ok) *top_id = (int)device_top; } if (ok && logits) { ok = ds4_gpu_tensor_read(metal_graph_logits(g), 0, logits, (uint64_t)DS4_N_VOCAB * sizeof(float)) != 0; @@ -55770,7 +55831,25 @@ static int generate_metal_graph_raw_swa( int pos = prompt->len; int n_generated = 0; int n_decode_eval = 0; + /* Greedy decode needs only the winning id after each graph evaluation. + * Resident and SSD-mapped paths can both reduce on the device and transfer + * one int32 instead of DS4_N_VOCAB floats; diagnostics retain full logits. + * CUDA and ROCm already have a dedicated top-1 kernel, while Metal selects + * its two-stage decode reduction. */ + const bool greedy_top1_readback = + !quality && + g.tp_world < 2u && + !trace_top && + !token_timing && + !graph_power_throttle_enabled(&g) && + getenv("DS4_METAL_GRAPH_TOKEN_PROFILE") == NULL && + getenv("DS4_ROCM_GRAPH_TOKEN_PROFILE") == NULL && + getenv("DS4_DISABLE_GREEDY_TOP1_READBACK") == NULL; const double t_decode0 = now_sec(); + /* Both arms need the initial CPU selection from prefill logits. Keep it + * inside the same decode timing window so short rollback A/B runs have an + * identical perimeter. */ + int next_token = sample_argmax(logits, DS4_N_VOCAB); for (int i = 0; i < n_predict && pos < ctx_size; i++) { if (trace_top) { char label[64]; @@ -55778,7 +55857,7 @@ static int generate_metal_graph_raw_swa( print_top_logits(stderr, label, vocab, logits, DS4_N_VOCAB, 10); } - int token = sample_argmax(logits, DS4_N_VOCAB); + const int token = next_token; if (vocab_token_is_generation_stop(vocab, token)) break; if (emit) emit(emit_ud, token); @@ -55790,17 +55869,33 @@ static int generate_metal_graph_raw_swa( } const double t_eval0 = token_timing ? now_sec() : 0.0; - ok = metal_graph_eval_token_raw_swa(&g, - model, - weights, - (uint32_t)token, - (uint32_t)pos, - logits); + if (greedy_top1_readback) { + ok = metal_graph_eval_token_raw_swa_top(&g, + model, + weights, + token, + (uint32_t)pos, + &next_token, + NULL, + false, + NULL, + false); + } else { + ok = metal_graph_eval_token_raw_swa(&g, + model, + weights, + (uint32_t)token, + (uint32_t)pos, + logits); + } if (!ok) break; if (token_timing) { const double t_eval1 = now_sec(); fprintf(stderr, "ds4: gpu decode eval %d took %.3f ms\n", n_decode_eval + 1, (t_eval1 - t_eval0) * 1000.0); } + if (!greedy_top1_readback) { + next_token = sample_argmax(logits, DS4_N_VOCAB); + } n_decode_eval++; pos++; } diff --git a/ds4_cuda.cu b/ds4_cuda.cu index 2c7a70f79..73294953a 100644 --- a/ds4_cuda.cu +++ b/ds4_cuda.cu @@ -43869,7 +43869,8 @@ extern "C" int ds4_gpu_attention_output_low_q4_K_slice_tensor( ds4_gpu_tensor *low, const void *model_map, uint64_t model_size, uint64_t out_a_offset, uint64_t group_dim, uint64_t rank, uint32_t group0, uint32_t group_cnt, - const ds4_gpu_tensor *heads) { + const ds4_gpu_tensor *heads, int resident_decode) { + (void)resident_decode; const int oracle = cuda_env_flag_enabled( "DS4_CUDA_Q4_GROUPED_ATTN_A_ORACLE", 0); if (oracle) cuda_q4_grouped_attn_a_oracle_register_report(); diff --git a/ds4_gpu.h b/ds4_gpu.h index af30002e4..df3afba0b 100644 --- a/ds4_gpu.h +++ b/ds4_gpu.h @@ -2818,7 +2818,8 @@ int ds4_gpu_attention_output_low_q4_K_slice_tensor( uint64_t rank, uint32_t group0, uint32_t group_cnt, - const ds4_gpu_tensor *heads); + const ds4_gpu_tensor *heads, + int resident_decode); int ds4_gpu_attention_output_low_q8_rows_exact_tensor( ds4_gpu_tensor *low, diff --git a/ds4_metal.m b/ds4_metal.m index bcdf95eeb..b2f3e917c 100644 --- a/ds4_metal.m +++ b/ds4_metal.m @@ -198,6 +198,8 @@ static id g_soft_max_f32_4_pipeline; static id g_argsort_f32_i32_desc_pipeline; static id g_argsort_merge_f32_i32_desc_pipeline; +static id g_dsv4_argmax_top1_stage1_pipeline; +static id g_dsv4_argmax_top1_stage2_pipeline; static id g_sum_rows_f32_f32_pipeline; static id g_dsv4_topk_mask_pipeline; static id g_dsv4_topk_mask_scatter_pipeline; @@ -379,6 +381,9 @@ __strong id router_weight_sum_buffer; __strong id indexer_head_scores_buffer; __strong id indexer_topk_buffer; + __strong id argmax_top1_scratch_v; + __strong id argmax_top1_scratch_i; + uint32_t argmax_top1_seq; __strong id indexed_topk_buffer; __strong id f16_round_scratch_buffer; __strong id raw_store_round_buffer; @@ -6793,6 +6798,15 @@ static int ds4_gpu_encode_rope_tail_inplace( int32_t len; } ds4_gpu_kargs_argsort_merge; +/* Matches ds4_metal_args_dsv4_argmax_top1 in metal/argsort.metal. */ +typedef struct { + int32_t n_vocab; + int32_t n_tg; +} ds4_gpu_dsv4_argmax_top1_args; + +_Static_assert(sizeof(ds4_gpu_dsv4_argmax_top1_args) == 8u, + "Metal top-1 argmax argument ABI changed"); + typedef struct { int64_t ne00; int64_t ne01; @@ -11732,6 +11746,8 @@ void ds4_gpu_cleanup(void) { g_soft_max_f32_4_pipeline = nil; g_argsort_f32_i32_desc_pipeline = nil; g_argsort_merge_f32_i32_desc_pipeline = nil; + g_dsv4_argmax_top1_stage1_pipeline = nil; + g_dsv4_argmax_top1_stage2_pipeline = nil; g_sum_rows_f32_f32_pipeline = nil; g_dsv4_topk_mask_pipeline = nil; g_dsv4_topk_mask_scatter_pipeline = nil; @@ -11845,6 +11861,9 @@ void ds4_gpu_cleanup(void) { g_router_weight_sum_buffer = nil; g_indexer_head_scores_buffer = nil; g_indexer_topk_buffer = nil; + g_stream_scratch[si].argmax_top1_scratch_v = nil; + g_stream_scratch[si].argmax_top1_scratch_i = nil; + g_stream_scratch[si].argmax_top1_seq = 0u; g_indexed_topk_buffer = nil; g_f16_round_scratch_buffer = nil; g_raw_store_round_buffer = nil; @@ -21363,17 +21382,145 @@ int ds4_gpu_indexer_topk_tensor( return 1; } -int ds4_gpu_argmax_tensor( +/* Dedicated two-dispatch top-1 reduction for decode logits. The generic + * indexer path sorts every block and merges its winners even though argmax + * consumes one index. Stage 1 scans fixed vocabulary chunks; stage 2 merges + * the stage-1 (value,index) pairs. Separate dispatches provide the device-wide + * ordering that a single-dispatch completion counter cannot guarantee. */ +static int ds4_gpu_argmax_top1_tensor( ds4_gpu_tensor *out_idx, const ds4_gpu_tensor *logits, uint32_t n_vocab) { + enum { + DS4_ARGMAX_TOP1_SLOTS = 8u, + DS4_ARGMAX_TOP1_GROUPS = 128u, + DS4_ARGMAX_TOP1_THREADS = 256u, + }; + if (!g_initialized && !ds4_gpu_init()) return 0; if (!out_idx || !logits || n_vocab == 0) return 0; if (ds4_gpu_tensor_bytes(out_idx) < sizeof(int32_t) || ds4_gpu_tensor_bytes(logits) < (uint64_t)n_vocab * sizeof(float)) { - fprintf(stderr, "ds4: Metal graph argmax received undersized buffers\n"); + fprintf(stderr, "ds4: Metal top-1 argmax received undersized buffers\n"); return 0; } + @autoreleasepool { + ds4_gpu_stream_scratch_state *stream_scratch = + &g_stream_scratch[g_ds4_stream]; + const uint32_t slot = + stream_scratch->argmax_top1_seq++ % DS4_ARGMAX_TOP1_SLOTS; + if (!stream_scratch->argmax_top1_scratch_v || + !stream_scratch->argmax_top1_scratch_i) { + stream_scratch->argmax_top1_scratch_v = nil; + stream_scratch->argmax_top1_scratch_i = nil; + stream_scratch->argmax_top1_scratch_v = + [g_device newBufferWithLength: + DS4_ARGMAX_TOP1_SLOTS * DS4_ARGMAX_TOP1_GROUPS * + sizeof(float) + options:MTLResourceStorageModeShared]; + stream_scratch->argmax_top1_scratch_i = + [g_device newBufferWithLength: + DS4_ARGMAX_TOP1_SLOTS * DS4_ARGMAX_TOP1_GROUPS * + sizeof(int32_t) + options:MTLResourceStorageModeShared]; + if (!stream_scratch->argmax_top1_scratch_v || + !stream_scratch->argmax_top1_scratch_i) { + stream_scratch->argmax_top1_scratch_v = nil; + stream_scratch->argmax_top1_scratch_i = nil; + return -1; + } + } + if (!g_dsv4_argmax_top1_stage1_pipeline || + !g_dsv4_argmax_top1_stage2_pipeline) { + g_dsv4_argmax_top1_stage1_pipeline = + ds4_gpu_get_pipeline( + "kernel_dsv4_argmax_top1_stage1_f32"); + g_dsv4_argmax_top1_stage2_pipeline = + ds4_gpu_get_pipeline( + "kernel_dsv4_argmax_top1_stage2_f32"); + if (!g_dsv4_argmax_top1_stage1_pipeline || + !g_dsv4_argmax_top1_stage2_pipeline) { + g_dsv4_argmax_top1_stage1_pipeline = nil; + g_dsv4_argmax_top1_stage2_pipeline = nil; + return -1; + } + } + if (g_dsv4_argmax_top1_stage1_pipeline.threadExecutionWidth != 32u || + g_dsv4_argmax_top1_stage2_pipeline.threadExecutionWidth != 32u || + g_dsv4_argmax_top1_stage1_pipeline.maxTotalThreadsPerThreadgroup < + DS4_ARGMAX_TOP1_THREADS || + g_dsv4_argmax_top1_stage2_pipeline.maxTotalThreadsPerThreadgroup < + DS4_ARGMAX_TOP1_THREADS) { + return -1; + } + + const NSUInteger scratch_v_offset = + (NSUInteger)slot * DS4_ARGMAX_TOP1_GROUPS * sizeof(float); + const NSUInteger scratch_i_offset = + (NSUInteger)slot * DS4_ARGMAX_TOP1_GROUPS * sizeof(int32_t); + + ds4_gpu_dsv4_argmax_top1_args args = { + .n_vocab = (int32_t)n_vocab, + .n_tg = DS4_ARGMAX_TOP1_GROUPS, + }; + int owned = 0; + id cb = ds4_gpu_command_buffer(&owned); + if (!cb) return 0; + + id enc = ds4_gpu_compute_encoder(cb); + if (!enc) return 0; + [enc setComputePipelineState:g_dsv4_argmax_top1_stage1_pipeline]; + [enc setBytes:&args length:sizeof(args) atIndex:0]; + [enc setBuffer:ds4_gpu_tensor_buffer(logits) + offset:ds4_gpu_tensor_offset(logits) atIndex:1]; + [enc setBuffer:stream_scratch->argmax_top1_scratch_v + offset:scratch_v_offset atIndex:2]; + [enc setBuffer:stream_scratch->argmax_top1_scratch_i + offset:scratch_i_offset atIndex:3]; + [enc setThreadgroupMemoryLength:64u atIndex:0]; + [enc dispatchThreadgroups:MTLSizeMake(DS4_ARGMAX_TOP1_GROUPS, 1, 1) + threadsPerThreadgroup: + MTLSizeMake(DS4_ARGMAX_TOP1_THREADS, 1, 1)]; + + [enc setComputePipelineState:g_dsv4_argmax_top1_stage2_pipeline]; + [enc setBytes:&args length:sizeof(args) atIndex:0]; + [enc setBuffer:stream_scratch->argmax_top1_scratch_v + offset:scratch_v_offset atIndex:1]; + [enc setBuffer:stream_scratch->argmax_top1_scratch_i + offset:scratch_i_offset atIndex:2]; + [enc setBuffer:ds4_gpu_tensor_buffer(out_idx) + offset:ds4_gpu_tensor_offset(out_idx) atIndex:3]; + [enc setThreadgroupMemoryLength:64u atIndex:0]; + [enc dispatchThreadgroups:MTLSizeMake(1, 1, 1) + threadsPerThreadgroup: + MTLSizeMake(DS4_ARGMAX_TOP1_THREADS, 1, 1)]; + ds4_gpu_end_compute_encoder(cb, enc); + + if (!ds4_gpu_finish_command_buffer(cb, owned, + "top-1 argmax")) { + return 0; + } + } + return 1; +} + +int ds4_gpu_argmax_tensor( + ds4_gpu_tensor *out_idx, + const ds4_gpu_tensor *logits, + uint32_t n_vocab) { + /* Production decode has 129,280 logits. Small rows already fit the + * generic one-pass sorter and do not amortize the scan grid. */ + const bool large_row = + n_vocab >= 4096u && n_vocab <= (uint32_t)INT32_MAX; + const bool require_top1 = + getenv("DS4_METAL_REQUIRE_DECODE_ARGMAX_TOP1") != NULL; + if (large_row && !g_quality_mode && !g_batch_encoder_concurrent && + getenv("DS4_METAL_DISABLE_DECODE_ARGMAX_TOP1") == NULL) { + const int top1 = + ds4_gpu_argmax_top1_tensor(out_idx, logits, n_vocab); + if (top1 >= 0) return top1; + } + if (large_row && require_top1) return 0; return ds4_gpu_indexer_topk_tensor(out_idx, logits, n_vocab, 1, 1); } @@ -32084,7 +32231,9 @@ int ds4_gpu_attention_output_low_q4_K_slice_tensor( uint64_t rank, uint32_t group0, uint32_t group_cnt, - const ds4_gpu_tensor *heads) { + const ds4_gpu_tensor *heads, + int resident_decode) { + (void)resident_decode; if (!g_initialized && !ds4_gpu_init()) return 0; if (!low || !heads || !model_map || group_dim == 0 || rank == 0 || group_cnt == 0 || group_dim > UINT32_MAX || rank > UINT32_MAX) { diff --git a/metal/argsort.metal b/metal/argsort.metal index 77c473f96..8f84860ba 100644 --- a/metal/argsort.metal +++ b/metal/argsort.metal @@ -273,3 +273,129 @@ kernel void kernel_argsort_merge_f32_i32( // Host-visible merge variant used by DS4 top-k selection. template [[host_name("kernel_argsort_merge_f32_i32_desc")]] kernel argsort_merge_t kernel_argsort_merge_f32_i32; + +/* Decode only needs the best vocabulary id, not a sorted candidate list. + * Stage 1 scans disjoint chunks and publishes one lexicographic winner + * per threadgroup. Stage 2 reduces those pairs after a dispatch boundary, + * which is the required device-wide ordering without atomics. */ +struct ds4_metal_args_dsv4_argmax_top1 { + int32_t n_vocab; + int32_t n_tg; +}; + +static inline void ds4_argmax_top1_merge( + float ov, int oi, thread float *bv, thread int *bi) { + if (ov > *bv || + (ov == *bv && (*bi < 0 || (oi >= 0 && oi < *bi)))) { + *bv = ov; + *bi = oi; + } +} + +kernel void kernel_dsv4_argmax_top1_stage1_f32( + constant ds4_metal_args_dsv4_argmax_top1 &args, + device const float *src0, + device float *scratch_v, + device int32_t *scratch_i, + threadgroup char *shmem_raw [[threadgroup(0)]], + uint tgpig [[threadgroup_position_in_grid]], + ushort tiitg [[thread_index_in_threadgroup]], + ushort tiisg [[thread_index_in_simdgroup]], + ushort sgitg [[simdgroup_index_in_threadgroup]]) { + threadgroup float *tg_v = (threadgroup float *)shmem_raw; + threadgroup int32_t *tg_i = + (threadgroup int32_t *)(tg_v + 8); + + const uint n = (uint)args.n_vocab; + const uint ntg = (uint)args.n_tg; + const uint chunk = (n + ntg - 1u) / ntg; + const uint begin = tgpig * chunk; + const uint end = min(n, begin + chunk); + + float bv = -INFINITY; + int bi = -1; + for (uint i = begin + (uint)tiitg; i < end; i += 256u) { + const float v = src0[i]; + if (v > bv) { + bv = v; + bi = (int)i; + } + } + for (ushort off = 16u; off > 0u; off >>= 1u) { + ds4_argmax_top1_merge(simd_shuffle_xor(bv, off), + simd_shuffle_xor(bi, off), + &bv, &bi); + } + if (tiisg == 0u) { + tg_v[sgitg] = bv; + tg_i[sgitg] = bi; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + if (sgitg == 0u) { + if (tiisg >= 8u) { + bv = -INFINITY; + bi = -1; + } else { + bv = tg_v[tiisg]; + bi = tg_i[tiisg]; + } + for (ushort off = 4u; off > 0u; off >>= 1u) { + ds4_argmax_top1_merge(simd_shuffle_xor(bv, off), + simd_shuffle_xor(bi, off), + &bv, &bi); + } + if (tiisg == 0u) { + scratch_v[tgpig] = bv; + scratch_i[tgpig] = bi; + } + } +} + +kernel void kernel_dsv4_argmax_top1_stage2_f32( + constant ds4_metal_args_dsv4_argmax_top1 &args, + device const float *scratch_v, + device const int32_t *scratch_i, + device int32_t *dst, + threadgroup char *shmem_raw [[threadgroup(0)]], + ushort tiitg [[thread_index_in_threadgroup]], + ushort tiisg [[thread_index_in_simdgroup]], + ushort sgitg [[simdgroup_index_in_threadgroup]]) { + threadgroup float *tg_v = (threadgroup float *)shmem_raw; + threadgroup int32_t *tg_i = + (threadgroup int32_t *)(tg_v + 8); + + const uint ntg = (uint)args.n_tg; + float bv = -INFINITY; + int bi = -1; + if ((uint)tiitg < ntg) { + bv = scratch_v[tiitg]; + bi = scratch_i[tiitg]; + } + for (ushort off = 16u; off > 0u; off >>= 1u) { + ds4_argmax_top1_merge(simd_shuffle_xor(bv, off), + simd_shuffle_xor(bi, off), + &bv, &bi); + } + if (tiisg == 0u) { + tg_v[sgitg] = bv; + tg_i[sgitg] = bi; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + if (sgitg == 0u) { + if (tiisg >= 8u) { + bv = -INFINITY; + bi = -1; + } else { + bv = tg_v[tiisg]; + bi = tg_i[tiisg]; + } + for (ushort off = 4u; off > 0u; off >>= 1u) { + ds4_argmax_top1_merge(simd_shuffle_xor(bv, off), + simd_shuffle_xor(bi, off), + &bv, &bi); + } + /* sample_argmax starts at index zero with -INF. Preserve that edge + * contract when every logit is NaN or -INF. */ + if (tiisg == 0u) dst[0] = bi >= 0 ? bi : 0; + } +} diff --git a/rocm/ds4_rocm_q4.cuh b/rocm/ds4_rocm_q4.cuh index 156c02f04..3f91dd886 100644 --- a/rocm/ds4_rocm_q4.cuh +++ b/rocm/ds4_rocm_q4.cuh @@ -520,6 +520,30 @@ static int rocm_q4_K_dense_pair_requested(void) { getenv("DS4_ROCM_DISABLE_Q4_DENSE_PAIR") == NULL; } +enum { + ROCM_Q4_GROUPED_ATTN_A_DEFAULT_K = 4096u, + ROCM_Q4_GROUPED_ATTN_A_DEFAULT_M = 1024u, + ROCM_Q4_GROUPED_ATTN_A_DEFAULT_GROUPS = 8u, +}; + +static int rocm_q4_K_grouped_attn_a_resident_default_scope( + uint64_t group_dim, + uint64_t rank, + uint32_t group0, + uint32_t group_cnt, + int resident_decode) { + /* A batch fallback can pass one row at a time through this same API, so + * the caller explicitly identifies true decode. Default only the + * production decode shape after the complete model has been made + * resident; explicit ENABLE keeps the existing experimental surface. */ + return resident_decode && + !g_ssd_streaming_mode && + group_dim == ROCM_Q4_GROUPED_ATTN_A_DEFAULT_K && + rank == ROCM_Q4_GROUPED_ATTN_A_DEFAULT_M && + group0 == 0u && + group_cnt == ROCM_Q4_GROUPED_ATTN_A_DEFAULT_GROUPS; +} + static int rocm_q4_K_prefill_tile8_scope(uint64_t n_tok) { /* Keep decode/speculative micro-batches on the latency-oriented legacy * kernel. 4096 is DS4's largest supported prefill chunk and bounds the @@ -839,13 +863,16 @@ extern "C" int ds4_gpu_attention_output_low_q4_K_slice_tensor( ds4_gpu_tensor *low, const void *model_map, uint64_t model_size, uint64_t out_a_offset, uint64_t group_dim, uint64_t rank, uint32_t group0, uint32_t group_cnt, - const ds4_gpu_tensor *heads) { + const ds4_gpu_tensor *heads, int resident_decode) { const int disabled = getenv("DS4_ROCM_DISABLE_Q4_GROUPED_ATTN_A") != NULL; const int required = getenv("DS4_ROCM_REQUIRE_Q4_GROUPED_ATTN_A") != NULL; const int enabled = getenv("DS4_ROCM_ENABLE_Q4_GROUPED_ATTN_A") != NULL; + const int resident_default = + rocm_q4_K_grouped_attn_a_resident_default_scope( + group_dim, rank, group0, group_cnt, resident_decode); /* DISABLE is authoritative; REQUIRE reports that rollback as a failure * instead of allowing the graph to false-green through its fallback. */ if (disabled) { @@ -856,7 +883,7 @@ extern "C" int ds4_gpu_attention_output_low_q4_K_slice_tensor( } return rocm_q4_K_grouped_attn_a_result(required ? -1 : 0, 0u); } - if (!enabled && !required) { + if (!resident_default && !enabled && !required) { return rocm_q4_K_grouped_attn_a_result(0, 0u); } const int pre_enqueue_failure = required ? -1 : 0; diff --git a/scripts/environment_variables.tsv b/scripts/environment_variables.tsv index 854f7c66c..f4993a2b0 100644 --- a/scripts/environment_variables.tsv +++ b/scripts/environment_variables.tsv @@ -12,9 +12,10 @@ runtime/cli DS4_CLI_FORCE_SESSION Pure presence flag: any defined value, includi runtime/core DS4_BATCHED_FFN Pure presence flag: any defined value, including empty or "0", enables. Unset leaves the default shared-expert-batched FFN path (or its configured fallback). It is read only by CPU layer-major prefill and takes precedence over shared-batch and token-parallel FFN choices. Run the complete CPU prefill FFN in chunks through layer_ffn_batch instead of the default shared-expert-only batched path. ds4.c:14411 runtime/core DS4_BATCHED_ROPE_MAX Nonempty value parsed by strtol without full-string validation; integers 0..65536 are accepted, otherwise default 4096. Zero disables batched RoPE for every nonempty prompt. Effective only when prefix batch attention is selected and DS4_NO_BATCHED_ROPE is absent. Set the largest CPU prefix-prefill token batch that applies RoPE and inverse RoPE with the batched kernels. ds4.c:13911 runtime/core DS4_DECODE_PROFILE_DETAIL presence flag; unset=off Print per-stage timing for the single-token CPU FFN path. ds4.c:12209 -runtime/core DS4_EXPERT_HOTLIST nonempty filesystem path; unset=off; currently Metal-only Load an expert hotlist for Metal expert profiling/streaming. ds4.c:60369 -runtime/core DS4_EXPERT_PROFILE presence diagnostic flag; unset=off Collect timing/profile diagnostics for expert profile. ds4.c:60367 -runtime/core DS4_LOCK_FILE path string; default /tmp/ds4.lock Override the single-instance lock file. ds4.c:51999 +runtime/core DS4_DISABLE_GREEDY_TOP1_READBACK presence rollback; unset uses device top-1 plus a 4-byte readback for eligible single-tier greedy generation, including SSD streaming; any defined value including empty or 0 restores full-logits readback and CPU argmax Restore the legacy per-token full-logits host readback for greedy generation A/B and emergency rollback. ds4.c:51964 +runtime/core DS4_EXPERT_HOTLIST nonempty filesystem path; unset=off; currently Metal-only Load an expert hotlist for Metal expert profiling/streaming. ds4.c:60637 +runtime/core DS4_EXPERT_PROFILE presence diagnostic flag; unset=off Collect timing/profile diagnostics for expert profile. ds4.c:60635 +runtime/core DS4_LOCK_FILE path string; default /tmp/ds4.lock Override the single-instance lock file. ds4.c:52260 runtime/core DS4_NO_BATCHED_ATTN presence rollback flag; unset keeps default/optimized path Disable/roll back no batched attn. ds4.c:14410 runtime/core DS4_NO_BATCHED_ROPE presence rollback flag; unset keeps default/optimized path Disable/roll back no batched rope. ds4.c:13918 runtime/core DS4_NO_PARALLEL_ATTN_ROWS presence rollback flag; unset keeps default/optimized path Disable/roll back no parallel attn rows. ds4.c:13904 @@ -493,6 +494,7 @@ runtime/metal DS4_METAL_DISABLE_COMPRESSOR_RATIO4_PACK_FUSION presence rollback; runtime/metal DS4_METAL_DISABLE_COMPRESSOR_STORE_ONE presence rollback; unset: automatic/default path; any value including 0 disables Disables compressor store one. ds4_metal.m:23249 runtime/metal DS4_METAL_DISABLE_CONTIG_F16_F16_COPY value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Disables contig F16 F16 copy. ds4_metal.m:29562 runtime/metal DS4_METAL_DISABLE_CONTIG_F32_F16_COPY value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Disables contig F32 F16 copy. ds4_metal.m:29338 +runtime/metal DS4_METAL_DISABLE_DECODE_ARGMAX_TOP1 presence rollback; unset uses the dedicated two-dispatch top-1 reduction for eligible large Metal rows; any defined value including empty or 0 restores the generic full argsort Restore the generic indexer argsort for decode argmax A/B and emergency rollback. ds4_metal.m:21517 runtime/metal DS4_METAL_DISABLE_DECODE_NORM_EXACT_VIEWS presence rollback; unset: automatic/default path; any value including 0 disables Disables decode norm exact views. ds4_metal.m:36207 runtime/metal DS4_METAL_DISABLE_DECODE_RAW_GATHERED_ATTN presence rollback; unset: raw-only decode uses gathered attention; any value including 0 restores the legacy raw path Restores the separate raw-only attention path instead of gathered staging and attention. ds4_metal.m:32968 runtime/metal DS4_METAL_DISABLE_DECODE_RAW_PACKED32 presence rollback; unset: raw-only gathered attention may use packed32; any value including 0 disables it for raw-only layers Disables the packed32 reduce kernel for raw-only gathered attention while leaving compressed layers unchanged. ds4_metal.m:31464 @@ -852,6 +854,7 @@ runtime/metal DS4_METAL_Q8_PREFILL_PROFILE_FILTER substring matched against gene runtime/metal DS4_METAL_Q_STAGE_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for q stage. ds4.c:29270 runtime/metal DS4_METAL_REPEAT_SOURCE file path; unset/empty: use the in-tree Metal source file Overrides the repeat Metal kernel source file loaded at runtime. ds4_metal.m:4949 runtime/metal DS4_METAL_REQUIRE_COMPRESSOR_EXACT_POOL_RATIO4 presence strict check; unset: fallback allowed; any value including 0 requires the path Requires compressor exact pool ratio4 and makes eligible fallback fail closed. ds4_metal.m:26387 +runtime/metal DS4_METAL_REQUIRE_DECODE_ARGMAX_TOP1 presence fail-closed assertion for rows of at least 4096 logits; unset permits generic fallback; any defined value including empty or 0 rejects disabled, ineligible, or failed dedicated top-1 preflight Require the dedicated Metal top-1 reduction so correctness and performance oracles cannot silently exercise generic argsort. ds4_metal.m:21515 runtime/metal DS4_METAL_REQUIRE_EXACT_ROWS_PERSISTENT_CACHE nonempty boolean; unset/empty or exact 0: off; every other value: on Requires exact rows persistent cache and makes eligible fallback fail closed. ds4_metal.m:13002 runtime/metal DS4_METAL_REQUIRE_GATHERED_KV_STAGE presence strict check; unset: fallback allowed; any value including 0 requires the path Requires gathered KV stage and makes eligible fallback fail closed. ds4_metal.m:29680 runtime/metal DS4_METAL_REQUIRE_IQ2_XXS_SSD_PREFILL_MM value-aware boolean; default implicit fail-closed only with complete selected-address domain; explicit 1 is strict, 0 permits fallback Makes eligible IQ2_XXS/Q2_K grouped SSD-prefill MM fail closed. ds4_metal.m:44782 @@ -955,7 +958,7 @@ runtime/rocm DS4_ROCM_DISABLE_IQ2_STREAM_ADDR_TABLE presence rollback flag; unse runtime/rocm DS4_ROCM_DISABLE_Q4_ATTN_Q_B_F16_CACHE value-aware rollback; unset/0/false/no/off keeps the experiment available, empty or any other value disables it Disable the resident ROCm Q4_K attn_q_b-to-F16 prefill cache even when ENABLE or REQUIRE is set. rocm/ds4_rocm_q4_qb_sidecar.cuh:140 runtime/rocm DS4_ROCM_DISABLE_Q4_ATTN_Q_B_TRANSIENT_F16 value-aware rollback; unset/0/false/no/off keeps the automatic path eligible, empty or any other non-false value disables it Disable per-layer transient Q4_K attn_q_b-to-F16 scratch for device-image or device-range-resident weights; the existing resident-cache controls remain independent and otherwise eligible calls use native Q4. rocm/ds4_rocm_q4_qb_sidecar.cuh:145 runtime/rocm DS4_ROCM_DISABLE_Q4_DENSE_PAIR presence rollback; unset leaves opt-in policy unchanged Disable/roll back rocm disable q4 dense pair. rocm/ds4_rocm_q4.cuh:435 -runtime/rocm DS4_ROCM_DISABLE_Q4_GROUPED_ATTN_A presence rollback; DISABLE wins over enable/require Disable/roll back rocm disable q4 grouped attn a. rocm/ds4_rocm_q4.cuh:700 +runtime/rocm DS4_ROCM_DISABLE_Q4_GROUPED_ATTN_A presence rollback; unset permits the caller-marked resident decode production-shape default and explicit ENABLE/REQUIRE; any defined value including empty or 0 disables all grouped attention-A paths and wins over ENABLE/REQUIRE Restore eight standalone Q4 attention-A projections instead of the two-dispatch grouped path. rocm/ds4_rocm_q4.cuh:868 runtime/rocm DS4_ROCM_DISABLE_Q4_PREFILL_K1024_TILE4 value-aware rollback; unset/0/false/no/off keeps the exact K=1024 four-block kernel enabled; empty or any other value disables Restore the generic eight-block Q4_K tiled-prefill kernel for K=1024. rocm/ds4_rocm_q4.cuh:550 runtime/rocm DS4_ROCM_DISABLE_Q4_PREFILL_TILE8 presence rollback; TILE8 is default for 9..4096 tokens Disable/roll back rocm disable q4 prefill tile8. rocm/ds4_rocm_q4.cuh:448 runtime/rocm DS4_ROCM_DISABLE_Q4_SELECTED_EXPERT_VIEWS presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable q4 selected expert views. ds4.c:21135 @@ -995,19 +998,19 @@ runtime/rocm DS4_ROCM_ENABLE_MXFP4_TILE4 presence opt-in; unset=off; any defined runtime/rocm DS4_ROCM_ENABLE_Q4_ATTN_Q_B_F16_CACHE value-aware persistent-cache opt-in, default off; unset/0/false/no/off is off; DISABLE cancels an optional persistent request but leaves the automatic transient path independent; REQUIRE plus DISABLE fails closed Prewarm and use persistent resident F16 sidecars for eligible ROCm Q4_K attn_q_b prefills. rocm/ds4_rocm_q4_qb_sidecar.cuh:130 runtime/rocm DS4_ROCM_ENABLE_Q4_ATTN_Q_B_F16_OUTPUT value-aware experimental opt-in; unset/empty/0/false/no/off keeps the release F32 projection boundary; other nonempty values enable Write eligible resident Q4_K attn_q_b GEMM output in F16 and run the half-input norm/RoPE epilogue; SSD remains excluded. rocm/ds4_rocm_q4_qb_sidecar.cuh:152 runtime/rocm DS4_ROCM_ENABLE_Q4_DENSE_PAIR presence opt-in; unset=off; DISABLE takes precedence Enable rocm enable q4 dense pair. rocm/ds4_rocm_q4.cuh:434 -runtime/rocm DS4_ROCM_ENABLE_Q4_GROUPED_ATTN_A presence opt-in; unset=off unless REQUIRE; DISABLE wins Enable rocm enable q4 grouped attn a. rocm/ds4_rocm_q4.cuh:704 -runtime/rocm DS4_ROCM_ENABLE_STREAMING_FULL_EXPERT_ADDR_TABLE presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming full expert addr table. ds4.c:18240 -runtime/rocm DS4_ROCM_ENABLE_STREAMING_MADVISE_WILLNEED presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming madvise willneed. ds4.c:18209 -runtime/rocm DS4_ROCM_ENABLE_STREAMING_PREFILL_BATCH_SELECTED_ADDR presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming prefill batch selected addr. ds4.c:18569 -runtime/rocm DS4_ROCM_ENABLE_STREAMING_PREFILL_CACHE_SEED presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming prefill cache seed. ds4.c:21258 -runtime/rocm DS4_ROCM_ENABLE_STREAMING_PREFILL_LAYER_PAGEIN presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming prefill layer pagein. ds4.c:18431 -runtime/rocm DS4_ROCM_ENABLE_STREAMING_PREFILL_LAYER_READAHEAD presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming prefill layer readahead. ds4.c:18441 -runtime/rocm DS4_ROCM_ENABLE_STREAMING_PREFILL_SELECTED_MADVISE presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming prefill selected madvise. ds4.c:18421 -runtime/rocm DS4_ROCM_ENABLE_STREAMING_PREFILL_SELECTED_PAGEIN presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming prefill selected pagein. ds4.c:18411 -runtime/rocm DS4_ROCM_ENABLE_STREAMING_PREFILL_SELECTED_READAHEAD presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming prefill selected readahead. ds4.c:19955 -runtime/rocm DS4_ROCM_ENABLE_STREAMING_PREFILL_SELECTED_READAHEAD_SHARED presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming prefill selected readahead shared. ds4.c:19957 -runtime/rocm DS4_ROCM_ENABLE_STREAMING_READAHEAD presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming readahead. ds4.c:18202 -runtime/rocm DS4_ROCM_ENABLE_STREAMING_STATIC_DECODE_MAP presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming static decode map. ds4.c:18221 +runtime/rocm DS4_ROCM_ENABLE_Q4_GROUPED_ATTN_A presence opt-in outside the default scope; the exact caller-marked resident decode shape groups=8, N=1, K=4096, M=1024 is automatic, while row-at-a-time batch fallbacks are not; DISABLE wins Enable grouped Q4 attention-A for eligible slices, non-production shapes, or explicit experiments in addition to the resident decode default. rocm/ds4_rocm_q4.cuh:872 +runtime/rocm DS4_ROCM_ENABLE_STREAMING_FULL_EXPERT_ADDR_TABLE presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming full expert addr table. ds4.c:18255 +runtime/rocm DS4_ROCM_ENABLE_STREAMING_MADVISE_WILLNEED presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming madvise willneed. ds4.c:18224 +runtime/rocm DS4_ROCM_ENABLE_STREAMING_PREFILL_BATCH_SELECTED_ADDR presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming prefill batch selected addr. ds4.c:18584 +runtime/rocm DS4_ROCM_ENABLE_STREAMING_PREFILL_CACHE_SEED presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming prefill cache seed. ds4.c:21273 +runtime/rocm DS4_ROCM_ENABLE_STREAMING_PREFILL_LAYER_PAGEIN presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming prefill layer pagein. ds4.c:18446 +runtime/rocm DS4_ROCM_ENABLE_STREAMING_PREFILL_LAYER_READAHEAD presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming prefill layer readahead. ds4.c:18456 +runtime/rocm DS4_ROCM_ENABLE_STREAMING_PREFILL_SELECTED_MADVISE presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming prefill selected madvise. ds4.c:18436 +runtime/rocm DS4_ROCM_ENABLE_STREAMING_PREFILL_SELECTED_PAGEIN presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming prefill selected pagein. ds4.c:18426 +runtime/rocm DS4_ROCM_ENABLE_STREAMING_PREFILL_SELECTED_READAHEAD presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming prefill selected readahead. ds4.c:19970 +runtime/rocm DS4_ROCM_ENABLE_STREAMING_PREFILL_SELECTED_READAHEAD_SHARED presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming prefill selected readahead shared. ds4.c:19972 +runtime/rocm DS4_ROCM_ENABLE_STREAMING_READAHEAD presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming readahead. ds4.c:18217 +runtime/rocm DS4_ROCM_ENABLE_STREAMING_STATIC_DECODE_MAP presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming static decode map. ds4.c:18236 runtime/rocm DS4_ROCM_GLM_CAUSAL_ATTN_GEMM Enabled by default when unset. Exact "0" or an empty value disables; every other nonempty value enables (including false/off/no), because cuda_env_present only tests nonempty and != "0". Eligibility still requires causal_range && !has_selected; a failed GEMM helper falls through to the scalar attention kernel. Use FP16 BLAS GEMMs for dense causal GLM indexed prefill; =0 is the correctness/performance rollback to the scalar attention kernel. rocm/ds4_rocm_glm.cuh:3283 runtime/rocm DS4_ROCM_GLM_DISABLE_STREAMING_EXPERT_CACHE Pure presence flag: any defined value, including empty or "0", disables. Unset leaves automatic GLM streaming expert-cache eligibility enabled for supported model/quant/quality/SSD configurations. On ROCm builds DS4_METAL_GLM_DISABLE_STREAMING_EXPERT_CACHE is an accepted fallback alias. Disable selected/resident streamed-expert cache paths and force generic/full-layer expert handling for GLM SSD streaming. ds4.c:21197 runtime/rocm DS4_ROCM_GLM_DISABLE_STREAMING_SEED_BEFORE_PREFILL Pure presence flag: any defined value, including empty or "0", disables. Unset seeds before prefill whenever SSD streaming is active. On ROCm builds DS4_METAL_GLM_DISABLE_STREAMING_SEED_BEFORE_PREFILL is an accepted fallback alias. Skip the pre-prefill hotlist seed of the streaming expert cache in both one-shot GLM generation and session setup. ds4.c:51073 @@ -1050,12 +1053,12 @@ runtime/rocm DS4_ROCM_MXFP4_DOWN_RGROUP nonempty value is parsed by strtol and a runtime/rocm DS4_ROCM_Q4_ATTN_Q_B_F16_CACHE_MB integer MiB with a full-string parse; default 3072; accepted range 1..65536, values above clamp and invalid or smaller values restore the default Cap device memory used by resident ROCm Q4_K attn_q_b F16 sidecars. rocm/ds4_rocm_q4_qb_sidecar.cuh:162 runtime/rocm DS4_ROCM_Q4_ATTN_Q_B_F16_CACHE_MIN_TOKENS integer token count with a full-string parse; default 512; accepted range 32..UINT32_MAX, values above clamp and invalid or smaller values restore the default Set the minimum prefill batch eligible to prepare or use the ROCm Q4 attn_q_b F16 sidecars. rocm/ds4_rocm_q4_qb_sidecar.cuh:156 runtime/rocm DS4_ROCM_Q4_ATTN_Q_B_TRANSIENT_F16_MIN_TOKENS full-string unsigned token count; default 4096; accepted range 32..UINT32_MAX; values above clamp and invalid or smaller values restore 4096 Set the minimum device-resident, non-SSD prefill batch eligible for per-layer transient ROCm Q4_K attn_q_b-to-F16 expansion. rocm/ds4_rocm_q4_qb_sidecar.cuh:150 -runtime/rocm DS4_ROCM_Q4_GROUPED_ATTN_A_STATS presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Print counters for rocm q4 grouped attn a stats. rocm/ds4_rocm_q4.cuh:480 +runtime/rocm DS4_ROCM_Q4_GROUPED_ATTN_A_STATS presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Print counters for rocm q4 grouped attn a stats. rocm/ds4_rocm_q4.cuh:614 runtime/rocm DS4_ROCM_Q4_PREFILL_TILE8_STATS presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Print counters for rocm q4 prefill tile8 stats. rocm/ds4_rocm_q4.cuh:514 runtime/rocm DS4_ROCM_Q8_DECODE_SHAREDX_64K sampled once; unset: enabled; present empty or exact 0: disabled; every other present value: enabled; effective only for one-token non-prequant Q8_0 matmul with 8192 < in_dim <= 16384 Allows the ROCm shared-input Q8 decode kernel to use up to 64 KiB dynamic LDS for wide inputs; an unsupported/failed LDS launch automatically falls back to the regular kernel. rocm/ds4_rocm_runtime.cuh:4805 runtime/rocm DS4_ROCM_Q_STAGE_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm q stage profile. ds4.c:30038 runtime/rocm DS4_ROCM_REQUIRE_Q4_ATTN_Q_B_F16_CACHE value-aware strict opt-in, default off; unset/0/false/no/off is off, empty or any other value requires eligible batches to use the cache; DISABLE wins Fail an eligible ROCm prefill instead of falling back when the resident Q4_K attn_q_b F16 specialization cannot be prepared or dispatched. rocm/ds4_rocm_q4_qb_sidecar.cuh:135 -runtime/rocm DS4_ROCM_REQUIRE_Q4_GROUPED_ATTN_A presence fail-closed assertion; also requests candidate unless disabled Require rocm require q4 grouped attn a and fail instead of silently falling back. rocm/ds4_rocm_q4.cuh:702 +runtime/rocm DS4_ROCM_REQUIRE_Q4_GROUPED_ATTN_A presence fail-closed assertion; also requests the candidate outside the caller-marked resident decode default; DISABLE remains authoritative and causes failure Require grouped Q4 attention-A and fail instead of silently falling back. rocm/ds4_rocm_q4.cuh:870 runtime/rocm DS4_ROCM_REQUIRE_Q4_PREFILL_TILE8 presence fail-closed assertion for eligible TILE8 calls Require rocm require q4 prefill tile8 and fail instead of silently falling back. rocm/ds4_rocm_q4.cuh:452 runtime/rocm DS4_ROCM_STREAMING_DECODE_PREFILL_MAX primary nonempty value over the Metal alias; parsed by strtol when it has a numeric prefix (trailing text is accepted); <= 0 disables, values > UINT32_MAX clamp, no numeric prefix uses automatic default: 64 for Flash with uniform Q4_K/MXFP4 experts, 18 for other Pro/Flash, otherwise 0; the disable flag dominates Sets the largest short, non-quality SSD-streaming prefill batch routed through the decode-style path instead of canonical layer-major prefill. ds4.c:31961 runtime/rocm DS4_ROCM_STREAMING_EXPERT_AUTO_PRELOAD_CAP primary nonempty value over the Metal alias; strict full-string strtoul; valid values > UINT32_MAX clamp, invalid uses 4096, and 0 means no cap (not disabled); when CLI preload is auto/0, unset defaults to cap 4096 except ROCm GLM52, where absent/empty disables automatic preload entirely Caps the number of hot experts synchronously seeded into the SSD-streaming expert cache in automatic preload mode; an explicit CLI preload count bypasses this cap, and setting this variable opts ROCm GLM52 back into auto preload. ds4.c:21454 @@ -1140,6 +1143,7 @@ test-only DS4_TEST_LOCAL_GOLDEN_FILE nonempty readable fixture path; unset/empty test-only DS4_TEST_LOGPROB_AUTO_METAL presence flag; unset forces DS4_METAL_DISABLE_METAL4=1; any presence including empty or 0 removes that rollback and permits automatic Metal selection Runs official log-probability vectors with automatic Metal-path selection instead of the fixed pre-Metal4 baseline. tests/ds4_test.c:7235 test-only DS4_TEST_LONG_PROMPT nonempty readable prompt-file path; unset/empty defaults to tests/long_context_story_prompt.txt Selects the rendered story prompt for the long-context fact-recall test. tests/ds4_test.c:6970 test-only DS4_TEST_LONG_WORDS atoi integer; unset/empty/nonnumeric defaults to 0; valid range is 0..DS4_TEST_CONTEXT-128 and numeric prefixes are accepted Adds repeated words to alternating CUDA session-batch prompts to exercise long-prefill rows. tests/test_cuda_session_batch.c:135 +test-only DS4_TEST_METAL_ARGMAX_TOP1_TIMING presence flag; unset runs correctness only; any defined value including empty or 0 also runs the GGUF-free resident production-shape A/B Measures generic argsort versus dedicated Metal top-1 and full-readback versus 4-byte greedy selection using resident synthetic logits, excluding model and SSD I/O. tests/test_metal_argmax_top1.c:545 test-only DS4_TEST_METAL_EXACTN_BATCH_HEAD nonempty value other than exact 0 enables; unset/empty/0 disables; false/off also enable Enables the Metal exact-N batch-head path and requires its attempt/use counters for every eligible oracle case. tests/test_metal_exactn_oracle.c:401 test-only DS4_TEST_METAL_EXACTN_ORACLE presence flag compiled only with DS4_TEST_HOOKS; absent from normal production builds Force allocation of the exact-N Metal verifier/oracle workspace in tests. ds4.c:67329 test-only DS4_TEST_METAL_Q4_QB_F16_CACHE_TIMING presence flag; unset runs correctness only; any defined value including empty or 0 also runs resident cold/warm timing Enables the production-shape timing phase for the Metal Q4_K attn_q_b F16 sidecar oracle. tests/test_metal_q4_qb_f16_cache.c:59 diff --git a/tests/ds4_test.c b/tests/ds4_test.c index 63574d9c4..9c9d7fee5 100644 --- a/tests/ds4_test.c +++ b/tests/ds4_test.c @@ -1764,7 +1764,8 @@ static void test_metal_q4_attention_output_tiny_batch_exact_case( rank, 0, n_groups, - heads_row) != 0); + heads_row, + 0) != 0); if (out_b_type == 8u) { TEST_ASSERT(ds4_gpu_matmul_q8_0_tensor( out_row, diff --git a/tests/test_metal_argmax_top1.c b/tests/test_metal_argmax_top1.c new file mode 100644 index 000000000..80a7f2a09 --- /dev/null +++ b/tests/test_metal_argmax_top1.c @@ -0,0 +1,561 @@ +#define _DARWIN_C_SOURCE + +#include "ds4_gpu.h" + +#include +#include +#include +#include +#include +#include + +#ifdef __APPLE__ + +enum { + PROD_VOCAB = 129280u, + GUARD_WORDS = 8u, + OVERLAP_CALLS = 17u, + BENCH_CALLS = 512u, + TAIL_BENCH_CALLS = 128u, + BENCH_SAMPLES = 8u, +}; + +static const uint32_t k_guard = 0x7fc12345u; +static const char *k_disable = "DS4_METAL_DISABLE_DECODE_ARGMAX_TOP1"; +static const char *k_require = "DS4_METAL_REQUIRE_DECODE_ARGMAX_TOP1"; + +bool ds4_log_is_tty(FILE *fp) { + (void)fp; + return false; +} + +static double now_sec(void) { + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (double)ts.tv_sec + (double)ts.tv_nsec / 1.0e9; +} + +static float f32_from_bits(uint32_t bits) { + float value = 0.0f; + memcpy(&value, &bits, sizeof(value)); + return value; +} + +static int cpu_argmax(const float *v, uint32_t n) { + int best = 0; + float best_v = f32_from_bits(0xff800000u); + for (uint32_t i = 0; i < n; i++) { + if (v[i] > best_v) { + best_v = v[i]; + best = (int)i; + } + } + return best; +} + +/* Mirrors the eight-way ILP used by production sample_argmax for the finite + * benchmark row. The correctness oracle above intentionally stays scalar so + * its edge-case behavior remains obvious. */ +static int cpu_argmax_decode(const float *v, uint32_t n) { + int best = 0; + float best_v = f32_from_bits(0xff800000u); + int bi[8] = {0}; + float bv[8] = { + best_v, best_v, best_v, best_v, + best_v, best_v, best_v, best_v, + }; + uint32_t i = 0; + for (; n - i >= 8u; i += 8u) { + for (uint32_t lane = 0; lane < 8u; lane++) { + const float x = v[i + lane]; + if (x > bv[lane]) { + bv[lane] = x; + bi[lane] = (int)(i + lane); + } + } + } + for (uint32_t lane = 0; lane < 8u; lane++) { + if (bv[lane] > best_v || + (bv[lane] == best_v && bi[lane] < best)) { + best_v = bv[lane]; + best = bi[lane]; + } + } + for (; i < n; i++) { + if (v[i] > best_v) { + best_v = v[i]; + best = (int)i; + } + } + return best; +} + +static void fill_case(float *v, uint32_t n, uint32_t kind) { + for (uint32_t i = 0; i < n; i++) { + const int32_t centered = (int32_t)((i * 2654435761u + 17u) % 8191u) - 4095; + v[i] = (float)centered / 257.0f; + } + switch (kind) { + case 0: + v[91357u] = 1000.0f; + break; + case 1: + v[17u] = 2000.0f; + v[80003u] = 2000.0f; + break; + case 2: + v[9u] = f32_from_bits(0x7f800000u); + v[42000u] = f32_from_bits(0x7f800000u); + break; + case 3: + for (uint32_t i = 0; i < n; i++) { + v[i] = f32_from_bits(0xff800000u); + } + break; + case 4: + for (uint32_t i = 0; i < n; i++) { + v[i] = f32_from_bits(0x7fc00001u); + } + break; + case 5: + for (uint32_t i = 0; i < n; i++) { + v[i] = f32_from_bits((i & 1u) ? 0x7fc00001u : 0xff800000u); + } + v[777u] = -100.0f; + break; + default: + abort(); + } +} + +static int set_candidate(bool candidate) { + if (candidate) { + return unsetenv(k_disable) == 0 && + setenv(k_require, "1", 1) == 0; + } + return unsetenv(k_require) == 0 && + setenv(k_disable, "1", 1) == 0; +} + +static int run_one(ds4_gpu_tensor *out, const ds4_gpu_tensor *logits, + uint32_t n_vocab, bool candidate, int32_t *result) { + return set_candidate(candidate) && + ds4_gpu_argmax_tensor(out, logits, n_vocab) && + ds4_gpu_tensor_read(out, 0, result, sizeof(*result)); +} + +static int check_guarded_cases(void) { + const size_t total = (size_t)PROD_VOCAB + 2u * GUARD_WORDS; + uint32_t *host = malloc(total * sizeof(*host)); + ds4_gpu_tensor *input_base = ds4_gpu_tensor_alloc(total * sizeof(float)); + ds4_gpu_tensor *input = input_base ? ds4_gpu_tensor_view( + input_base, (uint64_t)GUARD_WORDS * sizeof(float), + (uint64_t)PROD_VOCAB * sizeof(float)) : NULL; + ds4_gpu_tensor *out_base = ds4_gpu_tensor_alloc(3u * sizeof(uint32_t)); + ds4_gpu_tensor *out = out_base ? ds4_gpu_tensor_view( + out_base, sizeof(uint32_t), sizeof(uint32_t)) : NULL; + int ok = host && input_base && input && out_base && out; + + for (uint32_t kind = 0; ok && kind < 6u; kind++) { + for (size_t i = 0; i < total; i++) host[i] = k_guard; + float *values = (float *)(host + GUARD_WORDS); + fill_case(values, PROD_VOCAB, kind); + const int expected = cpu_argmax(values, PROD_VOCAB); + uint32_t out_words[3] = {k_guard, k_guard, k_guard}; + int32_t control = -1; + int32_t candidate = -1; + ok = ds4_gpu_tensor_write(input_base, 0, host, + total * sizeof(*host)) && + ds4_gpu_tensor_write(out_base, 0, out_words, + sizeof(out_words)) && + run_one(out, input, PROD_VOCAB, false, &control) && + ds4_gpu_tensor_write(out_base, 0, out_words, + sizeof(out_words)) && + run_one(out, input, PROD_VOCAB, true, &candidate) && + ds4_gpu_tensor_read(out_base, 0, out_words, + sizeof(out_words)) && + ds4_gpu_tensor_read(input_base, 0, host, + total * sizeof(*host)); + if (!ok) break; + if (candidate != expected || + (kind < 3u && control != candidate) || + out_words[0] != k_guard || out_words[2] != k_guard) { + fprintf(stderr, + "argmax case %u failed: expected=%d control=%d candidate=%d " + "guards=%08x/%08x\n", + kind, expected, control, candidate, + out_words[0], out_words[2]); + ok = 0; + } + for (uint32_t i = 0; ok && i < GUARD_WORDS; i++) { + if (host[i] != k_guard || + host[GUARD_WORDS + PROD_VOCAB + i] != k_guard) { + fprintf(stderr, "argmax input guard changed in case %u\n", kind); + ok = 0; + } + } + } + + ds4_gpu_tensor_free(out); + ds4_gpu_tensor_free(out_base); + ds4_gpu_tensor_free(input); + ds4_gpu_tensor_free(input_base); + free(host); + if (ok) fprintf(stderr, "Metal top-1 production-shape oracle: PASS\n"); + return ok; +} + +static int check_vocab_shapes(void) { + static const uint32_t sizes[] = {4096u, 4097u, 129279u, 129281u}; + const uint32_t max_n = sizes[sizeof(sizes) / sizeof(sizes[0]) - 1u]; + float *host = malloc((size_t)max_n * sizeof(*host)); + ds4_gpu_tensor *logits = ds4_gpu_tensor_alloc( + (uint64_t)max_n * sizeof(float)); + ds4_gpu_tensor *out = ds4_gpu_tensor_alloc(sizeof(int32_t)); + int ok = host && logits && out; + for (size_t shape = 0; + ok && shape < sizeof(sizes) / sizeof(sizes[0]); + shape++) { + const uint32_t n = sizes[shape]; + for (uint32_t i = 0; i < n; i++) { + host[i] = -(float)((i * 17u + 3u) % 8191u); + } + host[n - 1u] = 2000.0f; + const int expected = cpu_argmax(host, n); + int32_t control = -1; + int32_t candidate = -1; + ok = ds4_gpu_tensor_write(logits, 0, host, + (uint64_t)n * sizeof(*host)) && + run_one(out, logits, n, false, &control) && + run_one(out, logits, n, true, &candidate); + if (ok && (control != expected || candidate != expected)) { + fprintf(stderr, + "Metal top-1 shape %u failed: expected=%d control=%d candidate=%d\n", + n, expected, control, candidate); + ok = 0; + } + } + ds4_gpu_tensor_free(out); + ds4_gpu_tensor_free(logits); + free(host); + if (ok) fprintf(stderr, "Metal top-1 tail/empty-group shapes: PASS\n"); + return ok; +} + +static int check_scratch_wrap(void) { + const size_t values = (size_t)OVERLAP_CALLS * PROD_VOCAB; + float *host = malloc(values * sizeof(*host)); + int32_t expected[OVERLAP_CALLS]; + int32_t results[OVERLAP_CALLS + 2u]; + ds4_gpu_tensor *logits = ds4_gpu_tensor_alloc(values * sizeof(float)); + ds4_gpu_tensor *out_base = ds4_gpu_tensor_alloc(sizeof(results)); + ds4_gpu_tensor *rows[OVERLAP_CALLS] = {0}; + ds4_gpu_tensor *outs[OVERLAP_CALLS] = {0}; + int ok = host && logits && out_base; + + for (uint32_t row = 0; ok && row < OVERLAP_CALLS; row++) { + float *v = host + (size_t)row * PROD_VOCAB; + for (uint32_t i = 0; i < PROD_VOCAB; i++) { + v[i] = -(float)((i + row * 13u) % 4093u); + } + const uint32_t winner = (row * 7919u + 23u) % PROD_VOCAB; + v[winner] = 1000.0f + (float)row; + expected[row] = (int32_t)winner; + rows[row] = ds4_gpu_tensor_view( + logits, (uint64_t)row * PROD_VOCAB * sizeof(float), + (uint64_t)PROD_VOCAB * sizeof(float)); + outs[row] = ds4_gpu_tensor_view( + out_base, (uint64_t)(row + 1u) * sizeof(int32_t), + sizeof(int32_t)); + ok = rows[row] && outs[row]; + } + for (uint32_t i = 0; i < OVERLAP_CALLS + 2u; i++) { + results[i] = (int32_t)k_guard; + } + if (ok) { + ok = ds4_gpu_tensor_write(logits, 0, host, + values * sizeof(float)) && + ds4_gpu_tensor_write(out_base, 0, results, sizeof(results)) && + set_candidate(true) && ds4_gpu_begin_commands(); + } + for (uint32_t row = 0; ok && row < OVERLAP_CALLS; row++) { + ok = ds4_gpu_argmax_tensor(outs[row], rows[row], PROD_VOCAB); + if (ok && row + 1u < OVERLAP_CALLS) { + ok = ds4_gpu_flush_commands(); + } + } + if (ok) ok = ds4_gpu_end_commands(); + else (void)ds4_gpu_synchronize(); + if (ok) ok = ds4_gpu_tensor_read(out_base, 0, results, sizeof(results)); + if (ok && ((uint32_t)results[0] != k_guard || + (uint32_t)results[OVERLAP_CALLS + 1u] != k_guard)) { + fprintf(stderr, "argmax overlap output guard changed\n"); + ok = 0; + } + for (uint32_t row = 0; ok && row < OVERLAP_CALLS; row++) { + if (results[row + 1u] != expected[row]) { + fprintf(stderr, + "argmax scratch wrap failed at call %u: expected=%d got=%d\n", + row, expected[row], results[row + 1u]); + ok = 0; + } + } + + for (uint32_t row = 0; row < OVERLAP_CALLS; row++) { + ds4_gpu_tensor_free(outs[row]); + ds4_gpu_tensor_free(rows[row]); + } + ds4_gpu_tensor_free(out_base); + ds4_gpu_tensor_free(logits); + free(host); + if (ok) { + fprintf(stderr, + "Metal top-1 scratch-ring wrap (%u command-buffer submissions " + "without host waits): PASS\n", + OVERLAP_CALLS); + } + return ok; +} + +static int timed_batch(ds4_gpu_tensor *out, + const ds4_gpu_tensor *logits, + bool candidate, + uint32_t calls, + double *elapsed) { + if (!set_candidate(candidate) || !ds4_gpu_begin_commands()) return 0; + const double t0 = now_sec(); + int ok = 1; + for (uint32_t i = 0; ok && i < calls; i++) { + ok = ds4_gpu_argmax_tensor(out, logits, PROD_VOCAB); + } + if (ok) ok = ds4_gpu_end_commands(); + else (void)ds4_gpu_synchronize(); + const double t1 = now_sec(); + if (ok && elapsed) *elapsed = t1 - t0; + return ok; +} + +static int cmp_double(const void *a, const void *b) { + const double x = *(const double *)a; + const double y = *(const double *)b; + return (x > y) - (x < y); +} + +static int timed_selection_tail( + ds4_gpu_tensor *out, + ds4_gpu_tensor *logits, + const ds4_gpu_tensor *producer, + float *readback, + int expected, + bool candidate, + uint32_t calls, + double *elapsed) { + if (!set_candidate(true)) return 0; + const double t0 = now_sec(); + int ok = 1; + for (uint32_t i = 0; ok && i < calls; i++) { + int32_t got = -1; + ok = ds4_gpu_begin_commands(); + if (ok) { + ok = ds4_gpu_tensor_copy( + logits, 0, producer, 0, + (uint64_t)PROD_VOCAB * sizeof(float)); + } + if (ok && candidate) { + ok = ds4_gpu_argmax_tensor(out, logits, PROD_VOCAB); + } + if (ok) ok = ds4_gpu_end_commands(); + else (void)ds4_gpu_synchronize(); + if (ok && candidate) { + ok = ds4_gpu_tensor_read(out, 0, &got, sizeof(got)); + } else if (ok) { + ok = ds4_gpu_tensor_read(logits, 0, readback, + (uint64_t)PROD_VOCAB * sizeof(*readback)); + if (ok) got = cpu_argmax_decode(readback, PROD_VOCAB); + } + if (ok && got != expected) { + fprintf(stderr, + "Metal selection-tail mismatch: candidate=%d expected=%d got=%d\n", + candidate, expected, got); + ok = 0; + } + } + const double t1 = now_sec(); + if (ok && elapsed) *elapsed = t1 - t0; + return ok; +} + +static int run_selection_tail_benchmark( + ds4_gpu_tensor *out, + ds4_gpu_tensor *logits, + const ds4_gpu_tensor *producer, + const float *values) { + float *readback = malloc((size_t)PROD_VOCAB * sizeof(*readback)); + int ok = readback != NULL; + const int expected = cpu_argmax_decode(values, PROD_VOCAB); + double ignored = 0.0; + if (ok) { + ok = timed_selection_tail(out, logits, producer, readback, expected, + false, 16u, &ignored); + } + if (ok) { + ok = timed_selection_tail(out, logits, producer, readback, expected, + true, 16u, &ignored); + } + + double control[BENCH_SAMPLES] = {0}; + double candidate[BENCH_SAMPLES] = {0}; + uint32_t nc = 0; + uint32_t nn = 0; + for (uint32_t cycle = 0; ok && cycle < BENCH_SAMPLES / 2u; cycle++) { + const bool order[4] = {false, true, true, false}; + for (uint32_t j = 0; ok && j < 4u; j++) { + double elapsed = 0.0; + ok = timed_selection_tail(out, logits, producer, + readback, expected, + order[j], TAIL_BENCH_CALLS, &elapsed); + if (!ok) break; + if (order[j]) candidate[nn++] = elapsed; + else control[nc++] = elapsed; + } + } + if (ok && (nc != BENCH_SAMPLES || nn != BENCH_SAMPLES)) ok = 0; + if (ok) { + qsort(control, BENCH_SAMPLES, sizeof(control[0]), cmp_double); + qsort(candidate, BENCH_SAMPLES, sizeof(candidate[0]), cmp_double); + const double control_median = + 0.5 * (control[BENCH_SAMPLES / 2u - 1u] + + control[BENCH_SAMPLES / 2u]); + const double candidate_median = + 0.5 * (candidate[BENCH_SAMPLES / 2u - 1u] + + candidate[BENCH_SAMPLES / 2u]); + const double control_us = + control_median * 1.0e6 / TAIL_BENCH_CALLS; + const double candidate_us = + candidate_median * 1.0e6 / TAIL_BENCH_CALLS; + const double saved_us = control_us - candidate_us; + fprintf(stderr, + "Metal resident greedy-selection tail A/B (%u logits, " + "%u calls/sample, %u samples): full-read+CPU=%.3f us " + "p95=%.3f us device-top1+4B=%.3f us p95=%.3f us " + "saved=%.3f us speedup=%.2fx reduction=%.1f%%\n", + PROD_VOCAB, TAIL_BENCH_CALLS, BENCH_SAMPLES, + control_us, + control[BENCH_SAMPLES - 1u] * 1.0e6 / TAIL_BENCH_CALLS, + candidate_us, + candidate[BENCH_SAMPLES - 1u] * 1.0e6 / TAIL_BENCH_CALLS, + saved_us, control_us / candidate_us, + saved_us * 100.0 / control_us); + fprintf(stderr, + "Metal resident greedy-selection tail excludes GGUF/model access and SSD I/O.\n"); + } + free(readback); + return ok; +} + +static int run_benchmark(void) { + float *host = malloc((size_t)PROD_VOCAB * sizeof(*host)); + ds4_gpu_tensor *logits = ds4_gpu_tensor_alloc( + (uint64_t)PROD_VOCAB * sizeof(float)); + ds4_gpu_tensor *producer = ds4_gpu_tensor_alloc( + (uint64_t)PROD_VOCAB * sizeof(float)); + ds4_gpu_tensor *out = ds4_gpu_tensor_alloc(sizeof(int32_t)); + int ok = host && logits && producer && out; + if (ok) { + fill_case(host, PROD_VOCAB, 0u); + ok = ds4_gpu_tensor_write(logits, 0, host, + (uint64_t)PROD_VOCAB * sizeof(float)) && + ds4_gpu_tensor_write(producer, 0, host, + (uint64_t)PROD_VOCAB * sizeof(float)); + } + + double ignored = 0.0; + if (ok) ok = timed_batch(out, logits, false, 64u, &ignored); + if (ok) ok = timed_batch(out, logits, true, 64u, &ignored); + + double control[BENCH_SAMPLES] = {0}; + double candidate[BENCH_SAMPLES] = {0}; + uint32_t nc = 0; + uint32_t nn = 0; + for (uint32_t cycle = 0; ok && cycle < BENCH_SAMPLES / 2u; cycle++) { + const bool order[4] = {false, true, true, false}; + for (uint32_t j = 0; ok && j < 4u; j++) { + double elapsed = 0.0; + ok = timed_batch(out, logits, order[j], BENCH_CALLS, &elapsed); + if (!ok) break; + int32_t got = -1; + ok = ds4_gpu_tensor_read(out, 0, &got, sizeof(got)); + if (!ok || got != 91357) { + fprintf(stderr, + "Metal timed top-1 output mismatch: candidate=%d expected=91357 got=%d\n", + order[j], got); + ok = 0; + break; + } + if (order[j]) candidate[nn++] = elapsed; + else control[nc++] = elapsed; + } + } + if (ok && (nc != BENCH_SAMPLES || nn != BENCH_SAMPLES)) ok = 0; + if (ok) { + qsort(control, BENCH_SAMPLES, sizeof(control[0]), cmp_double); + qsort(candidate, BENCH_SAMPLES, sizeof(candidate[0]), cmp_double); + const double control_median = + 0.5 * (control[BENCH_SAMPLES / 2u - 1u] + + control[BENCH_SAMPLES / 2u]); + const double candidate_median = + 0.5 * (candidate[BENCH_SAMPLES / 2u - 1u] + + candidate[BENCH_SAMPLES / 2u]); + const double control_us = control_median * 1.0e6 / BENCH_CALLS; + const double candidate_us = candidate_median * 1.0e6 / BENCH_CALLS; + const double saved_us = control_us - candidate_us; + const double speedup = control_us / candidate_us; + const double gain = saved_us * 100.0 / control_us; + fprintf(stderr, + "Metal resident warm top-1 A/B (%u logits, %u calls/sample, " + "%u samples): control=%.3f us p95=%.3f us " + "candidate=%.3f us p95=%.3f us saved=%.3f us " + "speedup=%.2fx reduction=%.1f%%\n", + PROD_VOCAB, BENCH_CALLS, BENCH_SAMPLES, + control_us, + control[BENCH_SAMPLES - 1u] * 1.0e6 / BENCH_CALLS, + candidate_us, + candidate[BENCH_SAMPLES - 1u] * 1.0e6 / BENCH_CALLS, + saved_us, speedup, gain); + fprintf(stderr, + "Metal resident top-1 A/B excludes GGUF/model access and SSD I/O.\n"); + } + + if (ok) { + ok = run_selection_tail_benchmark(out, logits, producer, host); + } + + ds4_gpu_tensor_free(out); + ds4_gpu_tensor_free(producer); + ds4_gpu_tensor_free(logits); + free(host); + return ok; +} + +int main(void) { + int ok = ds4_gpu_init(); + if (ok) ok = check_guarded_cases(); + if (ok) ok = check_vocab_shapes(); + if (ok) ok = check_scratch_wrap(); + if (ok && getenv("DS4_TEST_METAL_ARGMAX_TOP1_TIMING") != NULL) { + ok = run_benchmark(); + } + (void)unsetenv(k_disable); + (void)unsetenv(k_require); + ds4_gpu_cleanup(); + return ok ? 0 : 1; +} + +#else + +int main(void) { + fprintf(stderr, "test_metal_argmax_top1: skipped (Metal requires macOS)\n"); + return 0; +} + +#endif diff --git a/tests/test_metal_q4_attn_exactn.c b/tests/test_metal_q4_attn_exactn.c index 8cc2bb4f4..c176883bd 100644 --- a/tests/test_metal_q4_attn_exactn.c +++ b/tests/test_metal_q4_attn_exactn.c @@ -252,7 +252,7 @@ int main(void) { CHECK(heads_row && low_row && out_row, "reference views"); CHECK(ds4_gpu_attention_output_low_q4_K_slice_tensor( low_row, model, model_bytes, 0, - GROUP_DIM, RANK, 0, N_GROUPS, heads_row) != 0, + GROUP_DIM, RANK, 0, N_GROUPS, heads_row, 0) != 0, "reference low projection"); CHECK(ds4_gpu_matmul_quant_tensor( out_row, model, model_bytes, out_b_offset, Q4_K_TYPE, diff --git a/tests/test_rocm_q4_dense_pair.cpp b/tests/test_rocm_q4_dense_pair.cpp index 5d060c101..c1ae0e5c2 100644 --- a/tests/test_rocm_q4_dense_pair.cpp +++ b/tests/test_rocm_q4_dense_pair.cpp @@ -1432,32 +1432,102 @@ bool run_grouped_attention_decode_case(const aligned_model &model) { (void)unsetenv(kGroupedDecodeEnable); (void)unsetenv(kGroupedDecodeDisable); (void)unsetenv(kGroupedDecodeRequire); + ds4_gpu_set_ssd_streaming(false); const int default_rc = ds4_gpu_attention_output_low_q4_K_slice_tensor( candidate_gpu.ptr, model.data, model.size, model.decode_attn_a_offset, kDecodeAttnGroupDim, kDecodeAttnRank, - 0u, kDecodeAttnGroups, heads_gpu.ptr); - bool ok = default_rc == 0; - if (default_rc != 0) { + 0u, kDecodeAttnGroups, heads_gpu.ptr, 1); + std::vector resident_default_host(allocation_count); + std::vector resident_legacy_host(allocation_count); + bool ok = default_rc == 1 && + read_tensor(candidate_gpu.ptr, &resident_default_host) && + read_tensor(legacy_gpu.ptr, &resident_legacy_host); + if (!ok) { std::fprintf(stderr, - "grouped attention-A default gate: expected rc=0 got=%d FAIL\n", + "grouped attention-A resident production default: " + "expected rc=1 got=%d/readback FAIL\n", default_rc); } + if (default_rc == 1) { + ok = output_guard_unchanged( + resident_default_host, sentinel, logical_count, + "grouped attention-A resident default output canary") && ok; + ok = bitwise_equal( + resident_default_host, resident_legacy_host, + "grouped attention-A resident default vs 8 legacy calls") && ok; + } + if (!write_tensor(candidate_gpu.ptr, sentinel)) { + std::fprintf(stderr, + "grouped attention-A gate reset: tensor write FAIL\n"); + return false; + } + + /* The batch fallback passes one row at a time through the same low-level + * API. It must not inherit the automatic one-token decode policy. */ + const int batch_row_default_rc = + ds4_gpu_attention_output_low_q4_K_slice_tensor( + candidate_gpu.ptr, model.data, model.size, + model.decode_attn_a_offset, kDecodeAttnGroupDim, kDecodeAttnRank, + 0u, kDecodeAttnGroups, heads_gpu.ptr, 0); + if (batch_row_default_rc != 0) { + std::fprintf(stderr, + "grouped attention-A batch-row default: " + "expected rc=0 got=%d FAIL\n", + batch_row_default_rc); + ok = false; + } ok = unchanged_after_rejected_call( candidate_gpu.ptr, sentinel, - "grouped attention-A disabled-by-default preserves output") && ok; + "grouped attention-A batch-row context preserves output") && ok; + + /* The production shape is implicit only for a fully resident model. This + * toggles policy state without opening or reading an SSD-backed model. */ + ds4_gpu_set_ssd_streaming(true); + const int streaming_default_rc = + ds4_gpu_attention_output_low_q4_K_slice_tensor( + candidate_gpu.ptr, model.data, model.size, + model.decode_attn_a_offset, kDecodeAttnGroupDim, kDecodeAttnRank, + 0u, kDecodeAttnGroups, heads_gpu.ptr, 1); + ds4_gpu_set_ssd_streaming(false); + if (streaming_default_rc != 0) { + std::fprintf(stderr, + "grouped attention-A streaming default: " + "expected rc=0 got=%d FAIL\n", + streaming_default_rc); + ok = false; + } + ok = unchanged_after_rejected_call( + candidate_gpu.ptr, sentinel, + "grouped attention-A streaming mode preserves output") && ok; - (void)setenv(kGroupedDecodeEnable, "1", 1); (void)setenv(kGroupedDecodeDisable, "1", 1); - const int disabled_only_rc = + const int rollback_rc = + ds4_gpu_attention_output_low_q4_K_slice_tensor( + candidate_gpu.ptr, model.data, model.size, + model.decode_attn_a_offset, kDecodeAttnGroupDim, kDecodeAttnRank, + 0u, kDecodeAttnGroups, heads_gpu.ptr, 1); + if (rollback_rc != 0) { + std::fprintf(stderr, + "grouped attention-A resident rollback: " + "expected rc=0 got=%d FAIL\n", + rollback_rc); + ok = false; + } + ok = unchanged_after_rejected_call( + candidate_gpu.ptr, sentinel, + "grouped attention-A DISABLE rolls back resident default") && ok; + + (void)setenv(kGroupedDecodeEnable, "1", 1); + const int disabled_enabled_rc = ds4_gpu_attention_output_low_q4_K_slice_tensor( candidate_gpu.ptr, model.data, model.size, model.decode_attn_a_offset, kDecodeAttnGroupDim, kDecodeAttnRank, - 0u, kDecodeAttnGroups, heads_gpu.ptr); - if (disabled_only_rc != 0) { + 0u, kDecodeAttnGroups, heads_gpu.ptr, 1); + if (disabled_enabled_rc != 0) { std::fprintf(stderr, "grouped attention-A ENABLE+DISABLE: expected rc=0 got=%d FAIL\n", - disabled_only_rc); + disabled_enabled_rc); ok = false; } ok = unchanged_after_rejected_call( @@ -1469,7 +1539,7 @@ bool run_grouped_attention_decode_case(const aligned_model &model) { ds4_gpu_attention_output_low_q4_K_slice_tensor( candidate_gpu.ptr, model.data, model.size, model.decode_attn_a_offset, kDecodeAttnGroupDim, kDecodeAttnRank, - 0u, kDecodeAttnGroups, heads_gpu.ptr); + 0u, kDecodeAttnGroups, heads_gpu.ptr, 1); if (disabled_rc != -1) { std::fprintf(stderr, "grouped attention-A DISABLE+REQUIRE: expected rc=-1 got=%d FAIL\n", @@ -1484,7 +1554,7 @@ bool run_grouped_attention_decode_case(const aligned_model &model) { const int invalid_rc = ds4_gpu_attention_output_low_q4_K_slice_tensor( candidate_gpu.ptr, model.data, model.size, model.size - 16u, kDecodeAttnGroupDim, kDecodeAttnRank, 0u, kDecodeAttnGroups, - heads_gpu.ptr); + heads_gpu.ptr, 1); if (invalid_rc != -1) { std::fprintf(stderr, "grouped attention-A REQUIRE range guard: expected rc=-1 got=%d FAIL\n", @@ -1498,7 +1568,7 @@ bool run_grouped_attention_decode_case(const aligned_model &model) { const int candidate_rc = ds4_gpu_attention_output_low_q4_K_slice_tensor( candidate_gpu.ptr, model.data, model.size, model.decode_attn_a_offset, kDecodeAttnGroupDim, kDecodeAttnRank, - 0u, kDecodeAttnGroups, heads_gpu.ptr); + 0u, kDecodeAttnGroups, heads_gpu.ptr, 1); std::vector legacy_host(allocation_count); std::vector candidate_host(allocation_count); if (candidate_rc != 1 || !read_tensor(legacy_gpu.ptr, &legacy_host) || @@ -1562,10 +1632,32 @@ bool run_grouped_attention_decode_case(const aligned_model &model) { return false; } } + + /* A slice is deliberately outside the implicit production scope. It + * must fall back while the environment is clean, then dispatch when the + * existing explicit ENABLE override is restored. */ + (void)unsetenv(kGroupedDecodeEnable); + (void)unsetenv(kGroupedDecodeRequire); + const int subset_default_rc = + ds4_gpu_attention_output_low_q4_K_slice_tensor( + subset_candidate.ptr, model.data, model.size, + model.decode_attn_a_offset, kDecodeAttnGroupDim, kDecodeAttnRank, + subset_group0, subset_group_cnt, subset_heads.ptr, 1); + if (subset_default_rc != 0) { + std::fprintf(stderr, + "grouped attention-A non-standard default: " + "expected rc=0 got=%d FAIL\n", + subset_default_rc); + ok = false; + } + ok = unchanged_after_rejected_call( + subset_candidate.ptr, subset_sentinel, + "grouped attention-A non-standard default preserves output") && ok; + (void)setenv(kGroupedDecodeEnable, "1", 1); const int subset_rc = ds4_gpu_attention_output_low_q4_K_slice_tensor( subset_candidate.ptr, model.data, model.size, model.decode_attn_a_offset, kDecodeAttnGroupDim, kDecodeAttnRank, - subset_group0, subset_group_cnt, subset_heads.ptr); + subset_group0, subset_group_cnt, subset_heads.ptr, 1); std::vector subset_legacy_host(subset_sentinel.size()); std::vector subset_candidate_host(subset_sentinel.size()); if (subset_rc != 1 || @@ -1589,10 +1681,16 @@ bool run_grouped_attention_decode_case(const aligned_model &model) { "grouped attention-A subset group0=3 count=2 vs legacy") && ok; std::fprintf(stderr, "grouped attention-A decode groups=8 K=4096 rank=1024: " - "default=%d disabled=%d disabled_required=%d invalid=%d " - "candidate=%d subset=%d %s\n", - default_rc, disabled_only_rc, disabled_rc, invalid_rc, - candidate_rc, subset_rc, + "resident_default=%d batch_row_default=%d " + "streaming_default=%d rollback=%d " + "disabled_enabled=%d disabled_required=%d invalid=%d " + "candidate=%d subset_default=%d subset_enabled=%d " + "stats_expected=calls:10,dispatches:3,groups:18," + "fallbacks:5,failures:2 %s\n", + default_rc, batch_row_default_rc, + streaming_default_rc, rollback_rc, + disabled_enabled_rc, disabled_rc, invalid_rc, + candidate_rc, subset_default_rc, subset_rc, ok ? "PASS" : "FAIL"); return ok; } From 1d3fec3ca8f6a7761ded7b4999e5d1a97ab9c764 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Thu, 27 Aug 2026 18:02:05 +0200 Subject: [PATCH 124/189] bench(metal): isolate resident Q4 decode pair kernels --- Makefile | 10 +- speed-bench/.gitignore | 1 + speed-bench/metal_q4_dense_pair_bench.m | 708 ++++++++++++++++++++++++ 3 files changed, 717 insertions(+), 2 deletions(-) create mode 100644 speed-bench/metal_q4_dense_pair_bench.m diff --git a/Makefile b/Makefile index ea967cab9..aff698cd4 100644 --- a/Makefile +++ b/Makefile @@ -81,7 +81,7 @@ test-quantizer-indexer-q4: gguf-tools/deepseek4-quantize tests/test_quantizer_in ./tests/test_quantizer_indexer_q4 ./gguf-tools/deepseek4-quantize ifeq ($(UNAME_S),Darwin) -.PHONY: metal-decode-schedule-bench metal-prefill-variant-bench check-mxfp4-half-lut test-mxfp4-metal +.PHONY: metal-decode-schedule-bench metal-prefill-variant-bench metal-q4-dense-pair-bench check-mxfp4-half-lut test-mxfp4-metal all: ds4 ds4-server ds4-bench ds4-eval ds4-agent @@ -108,6 +108,7 @@ help: @echo " make test-rocm-q4-prefill Run ROCm Q4 tiled-prefill parity/canary oracle" @echo " make metal-decode-schedule-bench Build the balanced Metal decode schedule benchmark" @echo " make metal-prefill-variant-bench Build the balanced Metal prefill variant benchmark" + @echo " make metal-q4-dense-pair-bench Build the resident Q4 decode pair kernel benchmark" @echo " make check-mxfp4-half-lut Verify the checked-in MXFP4 half LUT matches the generator" @echo " make test-mxfp4-metal Check the MXFP4 half LUT, then run Metal MXFP4 exactness tests" @echo " make test-metal-exactn-oracle Compare Metal exact-N state with sequential decode" @@ -310,6 +311,11 @@ test-metal-exactn-oracle: tests/test_metal_exactn_oracle DS4_TEST_MODEL="$(DS4_TEST_MODEL)" \ ./tests/test_metal_exactn_oracle +speed-bench/metal_q4_dense_pair_bench: speed-bench/metal_q4_dense_pair_bench.m $(METAL_SRCS) + $(CC) $(OBJCFLAGS) -o $@ $< $(METAL_LDLIBS) + +metal-q4-dense-pair-bench: speed-bench/metal_q4_dense_pair_bench + tests/test_mxfp4_metal.o: tests/test_mxfp4_metal.c ds4_gpu.h $(CC) $(CFLAGS) -I. -c -o $@ $< @@ -893,4 +899,4 @@ mxfp4-dot-test: tests/test_mxfp4_dot.c ./tests/test_mxfp4_dot clean: - rm -f ds4 ds4-server ds4-bench ds4-eval ds4-agent ds4_cpu ds4_native ds4_server_test ds4_test ds4_agent_test gguf-tools/quality-testing/score_official gguf-tools/quality-testing/score_official.o speed-bench/metal_decode_schedule_bench speed-bench/metal_prefill_variant_bench speed-bench/*.o tests/test_q4k_dot tests/test_mxfp4_dot tests/test_quantizer_indexer_q4 tests/test_mxfp4_metal tests/test_mxfp4_rocm tests/bench_mxfp4_rocm tests/test_mxfp4_cuda tests/test_rocm_q4_dense_pair tests/test_metal_session_batch tests/test_metal_q4_streams tests/test_metal_indexer_q4 tests/test_metal_q4_attn_exactn tests/test_metal_q4_qb_f16_cache tests/test_metal_exactn_oracle tests/test_metal_dspark_capture tests/test_metal_argmax_top1 tests/test_metal_iq2_midonly tests/test_metal_iq2_ssd_grouped_mm tests/test_metal_iq2_live_index tests/test_glm53_kda tests/test_glm53_kda_rocm tests/test_glm53_vision_engine tests/test_glm53_vision_prompt tests/test_gpu_xdev tests/test_gpu_model_cache tests/test_gpu_lookup_cache_strict tests/test_engine_mgpu_refusal tests/test_engine_mgpu_runtime tests/test_engine_correctness tests/test_sampling tests/test_cuda_session_batch tests/test_cuda_mixed_batch tests/*.o *.o tests/cuda_long_context_smoke tests/cuda_long_context_smoke.o + rm -f ds4 ds4-server ds4-bench ds4-eval ds4-agent ds4_cpu ds4_native ds4_server_test ds4_test ds4_agent_test gguf-tools/quality-testing/score_official gguf-tools/quality-testing/score_official.o speed-bench/metal_decode_schedule_bench speed-bench/metal_prefill_variant_bench speed-bench/metal_q4_dense_pair_bench speed-bench/*.o tests/test_q4k_dot tests/test_mxfp4_dot tests/test_quantizer_indexer_q4 tests/test_mxfp4_metal tests/test_mxfp4_rocm tests/bench_mxfp4_rocm tests/test_mxfp4_cuda tests/test_rocm_q4_dense_pair tests/test_metal_session_batch tests/test_metal_q4_streams tests/test_metal_indexer_q4 tests/test_metal_q4_attn_exactn tests/test_metal_q4_qb_f16_cache tests/test_metal_exactn_oracle tests/test_metal_dspark_capture tests/test_metal_argmax_top1 tests/test_metal_iq2_midonly tests/test_metal_iq2_ssd_grouped_mm tests/test_metal_iq2_live_index tests/test_glm53_kda tests/test_glm53_kda_rocm tests/test_glm53_vision_engine tests/test_glm53_vision_prompt tests/test_gpu_xdev tests/test_gpu_model_cache tests/test_gpu_lookup_cache_strict tests/test_engine_mgpu_refusal tests/test_engine_mgpu_runtime tests/test_engine_correctness tests/test_sampling tests/test_cuda_session_batch tests/test_cuda_mixed_batch tests/*.o *.o tests/cuda_long_context_smoke tests/cuda_long_context_smoke.o diff --git a/speed-bench/.gitignore b/speed-bench/.gitignore index 3eac33ffc..a0f60d58e 100644 --- a/speed-bench/.gitignore +++ b/speed-bench/.gitignore @@ -3,3 +3,4 @@ __pycache__/ local-runs/ metal_decode_schedule_bench metal_prefill_variant_bench +metal_q4_dense_pair_bench diff --git a/speed-bench/metal_q4_dense_pair_bench.m b/speed-bench/metal_q4_dense_pair_bench.m new file mode 100644 index 000000000..3eeabfbb5 --- /dev/null +++ b/speed-bench/metal_q4_dense_pair_bench.m @@ -0,0 +1,708 @@ +#import +#import + +#include +#include +#include +#include +#include +#include +#include + +enum { + IN_DIM = 4096, + OUT0_DIM = 1024, + OUT1_DIM = 512, + QK_K = 256, + Q4_BLOCK_BYTES = 144, + NSG = 2, + NR0 = 2, + THREADS_PER_SIMDGROUP = 32, + GUARD_BYTES = 256, + DEFAULT_SETS = 64, + DEFAULT_DISPATCHES = 256, + DEFAULT_WARMUP_DISPATCHES = 64, + DEFAULT_SAMPLES = 16, +}; + +static const uint32_t k_guard = 0x7fc12345u; + +typedef struct { + uint16_t d; + uint16_t dmin; + uint8_t scales[12]; + uint8_t qs[QK_K / 2]; +} block_q4_K_host; + +typedef struct { + int32_t ne00; + int32_t ne01; + int32_t ne02; + uint64_t nb00; + uint64_t nb01; + uint64_t nb02; + uint64_t nb03; + int32_t ne10; + int32_t ne11; + int32_t ne12; + uint64_t nb10; + uint64_t nb11; + uint64_t nb12; + uint64_t nb13; + int32_t ne0; + int32_t ne1; + int32_t nr0; + int16_t r2; + int16_t r3; +} mul_mv_args; + +_Static_assert(sizeof(block_q4_K_host) == Q4_BLOCK_BYTES, + "Q4_K host fixture must match GGUF/Metal layout"); +_Static_assert(sizeof(mul_mv_args) == 112, + "Metal mul_mv argument ABI changed"); + +typedef struct { + uint32_t sets; + uint32_t dispatches; + uint32_t warmup_dispatches; + uint32_t samples; +} bench_config; + +typedef enum { + ARM_SEPARATE, + ARM_PAIR, +} bench_arm; + +typedef struct { + __strong id device; + __strong id queue; + __strong id standalone_pipeline; + __strong id pair_pipeline; + __strong id w0; + __strong id w1; + __strong id x; + __strong id separate0; + __strong id separate1; + __strong id pair0; + __strong id pair1; + mul_mv_args args0; + mul_mv_args args1; + NSUInteger w0_set_bytes; + NSUInteger w1_set_bytes; + NSUInteger x_offset; + NSUInteger out0_stride; + NSUInteger out1_stride; + uint32_t sets; +} fixture; + +static void usage(FILE *fp, const char *argv0) { + fprintf(fp, + "usage: %s [options]\n" + "\n" + "Kernel-only A/B for production decode Q4_K projections:\n" + " 4096 -> 1024 and 4096 -> 512, n_tok=1\n" + "\n" + " --sets N rotating resident weight sets (default: 64)\n" + " --dispatches N logical projection pairs per sample (default: 256)\n" + " --warmup N logical projection pairs per warmup arm (default: 64)\n" + " --samples N samples per arm; must be even (default: 16)\n" + " -h, --help show this help\n" + "\n" + "Set DS4_SOURCE_ROOT when running outside the repository root.\n", + argv0); +} + +static uint32_t parse_u32(const char *text, const char *option, + uint32_t minimum) { + char *end = NULL; + errno = 0; + unsigned long long value = strtoull(text, &end, 10); + if (errno != 0 || text[0] == '\0' || !end || *end != '\0' || + value < minimum || value > UINT32_MAX) { + fprintf(stderr, "metal-q4-dense-pair-bench: invalid %s: %s\n", + option, text); + exit(2); + } + return (uint32_t)value; +} + +static const char *need_arg(int *i, int argc, char **argv) { + if (*i + 1 >= argc) { + fprintf(stderr, "metal-q4-dense-pair-bench: %s needs a value\n", + argv[*i]); + exit(2); + } + return argv[++*i]; +} + +static bench_config parse_options(int argc, char **argv) { + bench_config cfg = { + .sets = DEFAULT_SETS, + .dispatches = DEFAULT_DISPATCHES, + .warmup_dispatches = DEFAULT_WARMUP_DISPATCHES, + .samples = DEFAULT_SAMPLES, + }; + for (int i = 1; i < argc; i++) { + const char *arg = argv[i]; + if (!strcmp(arg, "-h") || !strcmp(arg, "--help")) { + usage(stdout, argv[0]); + exit(0); + } else if (!strcmp(arg, "--sets")) { + cfg.sets = parse_u32(need_arg(&i, argc, argv), arg, 1); + } else if (!strcmp(arg, "--dispatches")) { + cfg.dispatches = + parse_u32(need_arg(&i, argc, argv), arg, 1); + } else if (!strcmp(arg, "--warmup")) { + cfg.warmup_dispatches = + parse_u32(need_arg(&i, argc, argv), arg, 0); + } else if (!strcmp(arg, "--samples")) { + cfg.samples = parse_u32(need_arg(&i, argc, argv), arg, 2); + } else { + fprintf(stderr, "metal-q4-dense-pair-bench: unknown option: %s\n", + arg); + usage(stderr, argv[0]); + exit(2); + } + } + if ((cfg.samples & 1u) != 0u) { + fprintf(stderr, + "metal-q4-dense-pair-bench: --samples must be even for " + "ABBA/BAAB balance\n"); + exit(2); + } + return cfg; +} + +static NSString *metal_prelude(void) { + return @"#include \n" + "using namespace metal;\n" + "#define MAX(x, y) ((x) > (y) ? (x) : (y))\n" + "#define MIN(x, y) ((x) < (y) ? (x) : (y))\n" + "#define SWAP(x, y) { auto tmp = (x); (x) = (y); (y) = tmp; }\n" + "#define QK8_0 32\n" + "#ifndef QK_K\n#define QK_K 256\n#endif\n" + "#define N_SIMDWIDTH 32\n" + "#define N_R0_Q8_0 2\n" + "#define N_SG_Q8_0 4\n" + "#define FC_MUL_MV 600\n" + "#define FC_MUL_MM 700\n" + "#define FC_BIN 1300\n" + "#define FOR_UNROLL(x) _Pragma(\"clang loop unroll(full)\") for (x)\n" + "#define M_PI_F 3.14159265358979323846f\n" + "enum ds4_sort_order { DS4_SORT_ORDER_ASC, DS4_SORT_ORDER_DESC };\n" + "struct block_q8_0 { half d; int8_t qs[QK8_0]; };\n" + "struct block_q8_K { float d; int8_t qs[QK_K]; " + "int16_t bsums[QK_K / 16]; };\n"; +} + +/* Keep the same concatenation order as ds4_metal.m. The benchmark specializes + * and dispatches the checked-in production kernels; it carries no kernel copy. */ +static NSString *load_metal_source(void) { + static const char *paths[] = { + "metal/activations.metal", + "metal/flash_attn.metal", + "metal/dense.metal", + "metal/moe.metal", + "metal/dsv4_hc.metal", + "metal/unary.metal", + "metal/dsv4_kv.metal", + "metal/dsv4_rope.metal", + "metal/dsv4_misc.metal", + "metal/argsort.metal", + "metal/cpy.metal", + "metal/concat.metal", + "metal/get_rows.metal", + "metal/sum_rows.metal", + "metal/softmax.metal", + "metal/repeat.metal", + "metal/glu.metal", + "metal/norm.metal", + "metal/bin.metal", + "metal/set_rows.metal", + }; + const char *root_env = getenv("DS4_SOURCE_ROOT"); + NSString *root = root_env && root_env[0] + ? [NSString stringWithUTF8String:root_env] + : @"."; + NSMutableString *source = [NSMutableString stringWithString:metal_prelude()]; + for (size_t i = 0; i < sizeof(paths) / sizeof(paths[0]); i++) { + NSString *relative = [NSString stringWithUTF8String:paths[i]]; + NSString *path = [root stringByAppendingPathComponent:relative]; + NSError *error = nil; + NSString *part = [NSString stringWithContentsOfFile:path + encoding:NSUTF8StringEncoding + error:&error]; + if (!part) { + fprintf(stderr, "metal-q4-dense-pair-bench: cannot read %s: %s\n", + [path fileSystemRepresentation], + [[error localizedDescription] UTF8String]); + return nil; + } + [source appendFormat:@"\n// appended %@\n%@\n", relative, part]; + } + return source; +} + +static id make_pipeline( + id device, id library, NSString *name) { + int16_t nsg = NSG; + int16_t nxpsg = 8; + MTLFunctionConstantValues *values = [MTLFunctionConstantValues new]; + [values setConstantValue:&nsg type:MTLDataTypeShort atIndex:600]; + [values setConstantValue:&nxpsg type:MTLDataTypeShort atIndex:601]; + NSError *error = nil; + id function = [library newFunctionWithName:name + constantValues:values + error:&error]; + if (!function) { + fprintf(stderr, "metal-q4-dense-pair-bench: function %s: %s\n", + [name UTF8String], [[error localizedDescription] UTF8String]); + return nil; + } + error = nil; + id pipeline = + [device newComputePipelineStateWithFunction:function error:&error]; + if (!pipeline) { + fprintf(stderr, "metal-q4-dense-pair-bench: pipeline %s: %s\n", + [name UTF8String], [[error localizedDescription] UTF8String]); + } + return pipeline; +} + +static uint32_t lcg_next(uint32_t *state) { + *state = *state * 1664525u + 1013904223u; + return *state; +} + +static void fill_q4(void *storage, NSUInteger bytes, uint32_t seed) { + block_q4_K_host *blocks = storage; + const NSUInteger count = bytes / sizeof(*blocks); + uint32_t state = seed; + for (NSUInteger b = 0; b < count; b++) { + blocks[b].d = (uint16_t)(0x2400u | (lcg_next(&state) & 0x03ffu)); + blocks[b].dmin = + (uint16_t)(0x1c00u | (lcg_next(&state) & 0x03ffu)); + for (size_t i = 0; i < sizeof(blocks[b].scales); i++) { + blocks[b].scales[i] = (uint8_t)(lcg_next(&state) >> 24u); + } + for (size_t i = 0; i < sizeof(blocks[b].qs); i++) { + blocks[b].qs[i] = (uint8_t)(lcg_next(&state) >> 24u); + } + } +} + +static void fill_activation(float *x) { + uint32_t state = 0x243f6a88u; + for (uint32_t i = 0; i < IN_DIM; i++) { + int32_t centered = (int32_t)(lcg_next(&state) & 0xffffu) - 32768; + x[i] = (float)centered / 32768.0f; + } +} + +static NSUInteger align_up(NSUInteger value, NSUInteger alignment) { + return (value + alignment - 1u) / alignment * alignment; +} + +static bool checked_product(NSUInteger a, NSUInteger b, NSUInteger *out) { + if (a != 0 && b > NSUIntegerMax / a) return false; + *out = a * b; + return true; +} + +static id alloc_buffer(id device, NSUInteger bytes, + NSString *label) { + if (bytes == 0 || bytes > device.maxBufferLength) { + fprintf(stderr, + "metal-q4-dense-pair-bench: %s size %.1f MiB exceeds device limit\n", + [label UTF8String], (double)bytes / (1024.0 * 1024.0)); + return nil; + } + id buffer = + [device newBufferWithLength:bytes options:MTLResourceStorageModeShared]; + buffer.label = label; + if (!buffer) { + fprintf(stderr, "metal-q4-dense-pair-bench: allocation failed for %s\n", + [label UTF8String]); + } + return buffer; +} + +static mul_mv_args make_args(uint32_t out_dim) { + const uint64_t row_bytes = (IN_DIM / QK_K) * Q4_BLOCK_BYTES; + return (mul_mv_args){ + .ne00 = IN_DIM, + .ne01 = (int32_t)out_dim, + .ne02 = 1, + .nb00 = 1, + .nb01 = row_bytes, + .nb02 = row_bytes * out_dim, + .nb03 = row_bytes * out_dim, + .ne10 = IN_DIM, + .ne11 = 1, + .ne12 = 1, + .nb10 = sizeof(float), + .nb11 = IN_DIM * sizeof(float), + .nb12 = IN_DIM * sizeof(float), + .nb13 = IN_DIM * sizeof(float), + .ne0 = (int32_t)out_dim, + .ne1 = 1, + .nr0 = NR0, + .r2 = 1, + .r3 = 1, + }; +} + +static void fill_guard_buffer(id buffer) { + uint32_t *words = buffer.contents; + for (NSUInteger i = 0; i < buffer.length / sizeof(*words); i++) { + words[i] = k_guard; + } +} + +static bool init_fixture(fixture *f, const bench_config *cfg) { + f->device = MTLCreateSystemDefaultDevice(); + if (!f->device) { + fprintf(stderr, "metal-q4-dense-pair-bench: no Metal device\n"); + return false; + } + f->queue = [f->device newCommandQueue]; + NSString *source = load_metal_source(); + if (!f->queue || !source) return false; + + NSError *error = nil; + MTLCompileOptions *options = [MTLCompileOptions new]; + id library = + [f->device newLibraryWithSource:source options:options error:&error]; + if (!library) { + fprintf(stderr, "metal-q4-dense-pair-bench: Metal compile failed: %s\n", + [[error localizedDescription] UTF8String]); + return false; + } + f->standalone_pipeline = make_pipeline( + f->device, library, @"kernel_mul_mv_q4_K_dense_f32"); + f->pair_pipeline = make_pipeline( + f->device, library, @"kernel_mul_mv_q4_K_dense_pair_f32"); + if (!f->standalone_pipeline || !f->pair_pipeline) return false; + + f->sets = cfg->sets; + f->w0_set_bytes = (IN_DIM / QK_K) * Q4_BLOCK_BYTES * OUT0_DIM; + f->w1_set_bytes = (IN_DIM / QK_K) * Q4_BLOCK_BYTES * OUT1_DIM; + f->x_offset = GUARD_BYTES; + f->out0_stride = align_up( + GUARD_BYTES + OUT0_DIM * sizeof(float) + GUARD_BYTES, 256u); + f->out1_stride = align_up( + GUARD_BYTES + OUT1_DIM * sizeof(float) + GUARD_BYTES, 256u); + f->args0 = make_args(OUT0_DIM); + f->args1 = make_args(OUT1_DIM); + + NSUInteger w0_bytes = 0, w1_bytes = 0, out0_bytes = 0, out1_bytes = 0; + if (!checked_product(f->w0_set_bytes, cfg->sets, &w0_bytes) || + !checked_product(f->w1_set_bytes, cfg->sets, &w1_bytes) || + !checked_product(f->out0_stride, cfg->sets, &out0_bytes) || + !checked_product(f->out1_stride, cfg->sets, &out1_bytes)) { + fprintf(stderr, "metal-q4-dense-pair-bench: requested sizes overflow\n"); + return false; + } + + f->w0 = alloc_buffer(f->device, w0_bytes, @"q4-w0-resident"); + f->w1 = alloc_buffer(f->device, w1_bytes, @"q4-w1-resident"); + f->x = alloc_buffer(f->device, + GUARD_BYTES + IN_DIM * sizeof(float) + GUARD_BYTES, + @"decode-activation"); + f->separate0 = alloc_buffer(f->device, out0_bytes, @"separate-out0"); + f->separate1 = alloc_buffer(f->device, out1_bytes, @"separate-out1"); + f->pair0 = alloc_buffer(f->device, out0_bytes, @"pair-out0"); + f->pair1 = alloc_buffer(f->device, out1_bytes, @"pair-out1"); + if (!f->w0 || !f->w1 || !f->x || !f->separate0 || !f->separate1 || + !f->pair0 || !f->pair1) { + return false; + } + + fill_q4(f->w0.contents, f->w0.length, 0x41c64e6du); + fill_q4(f->w1.contents, f->w1.length, 0x9e3779b9u); + fill_guard_buffer(f->x); + fill_activation((float *)((uint8_t *)f->x.contents + f->x_offset)); + fill_guard_buffer(f->separate0); + fill_guard_buffer(f->separate1); + fill_guard_buffer(f->pair0); + fill_guard_buffer(f->pair1); + return true; +} + +static void encode_standalone(fixture *f, id enc, + uint32_t set) { + const NSUInteger out0 = (NSUInteger)set * f->out0_stride + GUARD_BYTES; + const NSUInteger out1 = (NSUInteger)set * f->out1_stride + GUARD_BYTES; + [enc setComputePipelineState:f->standalone_pipeline]; + [enc setBytes:&f->args0 length:sizeof(f->args0) atIndex:0]; + [enc setBuffer:f->w0 offset:(NSUInteger)set * f->w0_set_bytes atIndex:1]; + [enc setBuffer:f->x offset:f->x_offset atIndex:2]; + [enc setBuffer:f->separate0 offset:out0 atIndex:3]; + [enc setThreadgroupMemoryLength:32 atIndex:0]; + [enc dispatchThreadgroups:MTLSizeMake((OUT0_DIM + NSG * NR0 - 1) / + (NSG * NR0), 1, 1) + threadsPerThreadgroup:MTLSizeMake(THREADS_PER_SIMDGROUP, NSG, 1)]; + + [enc setBytes:&f->args1 length:sizeof(f->args1) atIndex:0]; + [enc setBuffer:f->w1 offset:(NSUInteger)set * f->w1_set_bytes atIndex:1]; + [enc setBuffer:f->separate1 offset:out1 atIndex:3]; + [enc dispatchThreadgroups:MTLSizeMake((OUT1_DIM + NSG * NR0 - 1) / + (NSG * NR0), 1, 1) + threadsPerThreadgroup:MTLSizeMake(THREADS_PER_SIMDGROUP, NSG, 1)]; +} + +static void encode_pair(fixture *f, id enc, + uint32_t set) { + const NSUInteger out0 = (NSUInteger)set * f->out0_stride + GUARD_BYTES; + const NSUInteger out1 = (NSUInteger)set * f->out1_stride + GUARD_BYTES; + [enc setComputePipelineState:f->pair_pipeline]; + [enc setBytes:&f->args0 length:sizeof(f->args0) atIndex:0]; + [enc setBytes:&f->args1 length:sizeof(f->args1) atIndex:1]; + [enc setBuffer:f->w0 offset:(NSUInteger)set * f->w0_set_bytes atIndex:2]; + [enc setBuffer:f->w1 offset:(NSUInteger)set * f->w1_set_bytes atIndex:3]; + [enc setBuffer:f->x offset:f->x_offset atIndex:4]; + [enc setBuffer:f->pair0 offset:out0 atIndex:5]; + [enc setBuffer:f->pair1 offset:out1 atIndex:6]; + [enc setThreadgroupMemoryLength:32 atIndex:0]; + [enc dispatchThreadgroups:MTLSizeMake((OUT0_DIM + NSG * NR0 - 1) / + (NSG * NR0), 1, 1) + threadsPerThreadgroup:MTLSizeMake(THREADS_PER_SIMDGROUP, NSG, 1)]; +} + +static bool finish_command_buffer(id cb, const char *label) { + [cb commit]; + [cb waitUntilCompleted]; + if (cb.status != MTLCommandBufferStatusCompleted) { + fprintf(stderr, "metal-q4-dense-pair-bench: %s failed: %s\n", label, + [[cb.error localizedDescription] UTF8String]); + return false; + } + return true; +} + +static bool check_guard_records(id buffer, NSUInteger stride, + NSUInteger output_bytes, uint32_t sets, + const char *label) { + const uint32_t *words = buffer.contents; + for (uint32_t set = 0; set < sets; set++) { + const NSUInteger record = (NSUInteger)set * stride; + const NSUInteger output_begin = record + GUARD_BYTES; + const NSUInteger output_end = output_begin + output_bytes; + for (NSUInteger off = record; off < output_begin; off += sizeof(*words)) { + if (words[off / sizeof(*words)] != k_guard) goto bad; + } + for (NSUInteger off = output_end; off < record + stride; + off += sizeof(*words)) { + if (words[off / sizeof(*words)] != k_guard) goto bad; + } + } + return true; + +bad: + fprintf(stderr, "metal-q4-dense-pair-bench: %s canary changed\n", label); + return false; +} + +static bool check_all_guards(fixture *f) { + const uint32_t *x = f->x.contents; + for (NSUInteger off = 0; off < f->x_offset; off += sizeof(*x)) { + if (x[off / sizeof(*x)] != k_guard) goto x_bad; + } + for (NSUInteger off = f->x_offset + IN_DIM * sizeof(float); + off < f->x.length; off += sizeof(*x)) { + if (x[off / sizeof(*x)] != k_guard) goto x_bad; + } + return check_guard_records(f->separate0, f->out0_stride, + OUT0_DIM * sizeof(float), f->sets, + "separate out0") && + check_guard_records(f->separate1, f->out1_stride, + OUT1_DIM * sizeof(float), f->sets, + "separate out1") && + check_guard_records(f->pair0, f->out0_stride, + OUT0_DIM * sizeof(float), f->sets, + "pair out0") && + check_guard_records(f->pair1, f->out1_stride, + OUT1_DIM * sizeof(float), f->sets, + "pair out1"); + +x_bad: + fprintf(stderr, "metal-q4-dense-pair-bench: activation canary changed\n"); + return false; +} + +static bool check_outputs(fixture *f) { + for (uint32_t set = 0; set < f->sets; set++) { + const uint8_t *separate0 = (uint8_t *)f->separate0.contents + + (NSUInteger)set * f->out0_stride + GUARD_BYTES; + const uint8_t *pair0 = (uint8_t *)f->pair0.contents + + (NSUInteger)set * f->out0_stride + GUARD_BYTES; + const uint8_t *separate1 = (uint8_t *)f->separate1.contents + + (NSUInteger)set * f->out1_stride + GUARD_BYTES; + const uint8_t *pair1 = (uint8_t *)f->pair1.contents + + (NSUInteger)set * f->out1_stride + GUARD_BYTES; + if (memcmp(separate0, pair0, OUT0_DIM * sizeof(float)) != 0 || + memcmp(separate1, pair1, OUT1_DIM * sizeof(float)) != 0) { + fprintf(stderr, + "metal-q4-dense-pair-bench: bitwise mismatch at set %u\n", + set); + return false; + } + } + return true; +} + +static bool run_oracle(fixture *f) { + id cb = [f->queue commandBuffer]; + id enc = [cb computeCommandEncoder]; + for (uint32_t set = 0; set < f->sets; set++) { + encode_standalone(f, enc, set); + } + for (uint32_t set = 0; set < f->sets; set++) { + encode_pair(f, enc, set); + } + [enc endEncoding]; + if (!finish_command_buffer(cb, "correctness oracle")) return false; + if (!check_outputs(f) || !check_all_guards(f)) return false; + fprintf(stderr, + "Metal Q4_K dense pair oracle: PASS (%u resident sets, bit-exact, canaries intact)\n", + f->sets); + return true; +} + +static bool run_workload(fixture *f, bench_arm arm, uint32_t dispatches, + uint32_t start_set, double *gpu_seconds) { + id cb = [f->queue commandBuffer]; + id enc = [cb computeCommandEncoder]; + for (uint32_t i = 0; i < dispatches; i++) { + const uint32_t set = (start_set + i) % f->sets; + if (arm == ARM_SEPARATE) encode_standalone(f, enc, set); + else encode_pair(f, enc, set); + } + [enc endEncoding]; + if (!finish_command_buffer(cb, arm == ARM_SEPARATE ? + "separate workload" : "pair workload")) { + return false; + } + const double elapsed = cb.GPUEndTime - cb.GPUStartTime; + if (!(elapsed > 0.0)) { + fprintf(stderr, + "metal-q4-dense-pair-bench: GPU timestamps unavailable\n"); + return false; + } + if (gpu_seconds) *gpu_seconds = elapsed; + return true; +} + +static int compare_double(const void *a, const void *b) { + const double x = *(const double *)a; + const double y = *(const double *)b; + return (x > y) - (x < y); +} + +static double percentile(const double *sorted, uint32_t n, double p) { + const double position = p * (double)(n - 1u); + const uint32_t lo = (uint32_t)position; + const uint32_t hi = lo + 1u < n ? lo + 1u : lo; + const double fraction = position - (double)lo; + return sorted[lo] + (sorted[hi] - sorted[lo]) * fraction; +} + +static bool run_benchmark(fixture *f, const bench_config *cfg) { + double ignored = 0.0; + if (cfg->warmup_dispatches != 0u) { + static const bench_arm warm_order[] = { + ARM_SEPARATE, ARM_PAIR, ARM_PAIR, ARM_SEPARATE, + }; + for (size_t i = 0; i < sizeof(warm_order) / sizeof(warm_order[0]); i++) { + if (!run_workload(f, warm_order[i], cfg->warmup_dispatches, + (uint32_t)(i * cfg->warmup_dispatches) % f->sets, + &ignored)) { + return false; + } + } + } + + double *separate = calloc(cfg->samples, sizeof(*separate)); + double *pair = calloc(cfg->samples, sizeof(*pair)); + if (!separate || !pair) { + fprintf(stderr, "metal-q4-dense-pair-bench: sample allocation failed\n"); + free(separate); + free(pair); + return false; + } + + uint32_t separate_count = 0; + uint32_t pair_count = 0; + bool ok = true; + for (uint32_t cycle = 0; ok && cycle < cfg->samples / 2u; cycle++) { + const bench_arm abba[] = { + ARM_SEPARATE, ARM_PAIR, ARM_PAIR, ARM_SEPARATE, + }; + const bench_arm baab[] = { + ARM_PAIR, ARM_SEPARATE, ARM_SEPARATE, ARM_PAIR, + }; + const bench_arm *order = (cycle & 1u) ? baab : abba; + for (uint32_t j = 0; ok && j < 4u; j++) { + const bench_arm arm = order[j]; + uint32_t arm_index = arm == ARM_SEPARATE ? separate_count : pair_count; + uint32_t start = (uint32_t)((uint64_t)arm_index * cfg->dispatches % + f->sets); + double elapsed = 0.0; + ok = run_workload(f, arm, cfg->dispatches, start, &elapsed); + if (ok && arm == ARM_SEPARATE) separate[separate_count++] = elapsed; + if (ok && arm == ARM_PAIR) pair[pair_count++] = elapsed; + } + } + if (ok && (separate_count != cfg->samples || pair_count != cfg->samples)) { + ok = false; + } + if (ok) ok = check_outputs(f) && check_all_guards(f); + + if (ok) { + qsort(separate, cfg->samples, sizeof(*separate), compare_double); + qsort(pair, cfg->samples, sizeof(*pair), compare_double); + const double separate_median = percentile(separate, cfg->samples, 0.50) * + 1.0e6 / cfg->dispatches; + const double separate_p95 = percentile(separate, cfg->samples, 0.95) * + 1.0e6 / cfg->dispatches; + const double pair_median = percentile(pair, cfg->samples, 0.50) * + 1.0e6 / cfg->dispatches; + const double pair_p95 = percentile(pair, cfg->samples, 0.95) * + 1.0e6 / cfg->dispatches; + const double saved = separate_median - pair_median; + const double working_set = + (double)(f->w0.length + f->w1.length) / (1024.0 * 1024.0); + + printf("Metal Q4_K decode kernel-only A/B\n"); + printf(" shape: n_tok=1, 4096->1024 + 4096->512\n"); + printf(" resident anonymous weights: %.1f MiB across %u rotating sets\n", + working_set, f->sets); + printf(" design: alternating ABBA/BAAB, %u logical calls/sample, " + "%u samples/arm, GPU timestamps\n", + cfg->dispatches, cfg->samples); + printf(" separate (2 dispatches): median %.3f us, p95 %.3f us\n", + separate_median, separate_p95); + printf(" pair (1 dispatch): median %.3f us, p95 %.3f us\n", + pair_median, pair_p95); + printf(" saved: %.3f us/logical call, speedup %.3fx, reduction %.2f%%\n", + saved, separate_median / pair_median, + saved * 100.0 / separate_median); + printf(" correctness: bit-exact outputs; activation/output canaries intact\n"); + printf(" scope: no GGUF, mmap, model runtime, SSD I/O, or CPU wall timing\n"); + } + + free(separate); + free(pair); + return ok; +} + +int main(int argc, char **argv) { + @autoreleasepool { + const bench_config cfg = parse_options(argc, argv); + fixture f = {0}; + if (!init_fixture(&f, &cfg)) return 1; + if (!run_oracle(&f)) return 1; + if (!run_benchmark(&f, &cfg)) return 1; + return 0; + } +} From bdf2829306e6bc545cb7fcce6b91e697a36f053b Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Thu, 27 Aug 2026 18:02:05 +0200 Subject: [PATCH 125/189] fix(metal): make IQ2 LUT staging geometry-safe --- metal/moe.metal | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/metal/moe.metal b/metal/moe.metal index 840eebba1..036a489ef 100644 --- a/metal/moe.metal +++ b/metal/moe.metal @@ -3559,12 +3559,14 @@ void kernel_mul_mv_iq2_xxs_f32_impl( threadgroup uint64_t * svalues = (threadgroup uint64_t *)(shmem); threadgroup uint8_t * ssigns = (threadgroup uint8_t *)(svalues + 256); { - int nval = 4; - int pos = (32*sgitg + tiisg)*nval; - for (int i = 0; i < nval; ++i) svalues[pos + i] = ds4_metal_iq2xxs_grid[pos + i]; - nval = 2; - pos = (32*sgitg + tiisg)*nval; - for (int i = 0; i < nval; ++i) ssigns[pos+i] = ds4_metal_ksigns_iq2xs[pos+i]; + const uint tid = 32u*(uint)sgitg + (uint)tiisg; + const uint nth = 32u*(uint)NSG; + for (uint i = tid; i < 256u; i += nth) { + svalues[i] = ds4_metal_iq2xxs_grid[i]; + } + for (uint i = tid; i < 128u; i += nth) { + ssigns[i] = ds4_metal_ksigns_iq2xs[i]; + } threadgroup_barrier(mem_flags::mem_threadgroup); } @@ -3781,15 +3783,14 @@ void kernel_mul_mv_iq2_xxs_pair_swiglu_mid_only_4096x2048_impl( threadgroup uint64_t *svalues = (threadgroup uint64_t *)(shmem); threadgroup uint8_t *ssigns = (threadgroup uint8_t *)(svalues + 256); { - int base = (32 * sgitg + tiisg); - int pos_a = base << 2; - int pos_b = base << 1; - svalues[pos_a] = ds4_metal_iq2xxs_grid[pos_a]; - svalues[pos_a + 1] = ds4_metal_iq2xxs_grid[pos_a + 1]; - svalues[pos_a + 2] = ds4_metal_iq2xxs_grid[pos_a + 2]; - svalues[pos_a + 3] = ds4_metal_iq2xxs_grid[pos_a + 3]; - ssigns[pos_b] = ds4_metal_ksigns_iq2xs[pos_b]; - ssigns[pos_b + 1] = ds4_metal_ksigns_iq2xs[pos_b + 1]; + const uint tid = 32u * (uint)sgitg + (uint)tiisg; + const uint nth = 32u * (uint)NSG; + for (uint i = tid; i < 256u; i += nth) { + svalues[i] = ds4_metal_iq2xxs_grid[i]; + } + for (uint i = tid; i < 128u; i += nth) { + ssigns[i] = ds4_metal_ksigns_iq2xs[i]; + } threadgroup_barrier(mem_flags::mem_threadgroup); } From 531e5bb37adf8904600ec3d0d44b45f9421b12a3 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Thu, 27 Aug 2026 19:25:50 +0200 Subject: [PATCH 126/189] perf(metal): cull inactive IQ2 prefill SIMDgroups --- ds4_metal.m | 53 ++++++++++++++++++++++++++++++++++++++++++++++++- metal/moe.metal | 2 ++ 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/ds4_metal.m b/ds4_metal.m index b2f3e917c..2a18a5c31 100644 --- a/ds4_metal.m +++ b/ds4_metal.m @@ -48492,6 +48492,43 @@ int ds4_gpu_routed_moe_batch_tensor( getenv("DS4_METAL_DISABLE_MOE_MM_ID_PAIR_SWIGLU") == NULL && getenv("DS4_METAL_MOE_WRITE_CLAMPED_ACT") == NULL && getenv("DS4_METAL_GRAPH_DUMP_PREFIX") == NULL; + /* The specialization is arithmetically valid for every grouped-MM + * IQ2_XXS/Q2_K top-6 shape below. Keep the automatic default narrower: + * resident kernel A/B and full-model SSD A/B measured the exact DS4 + * production geometry at 4096 tokens. An explicit ENABLE opts other + * eligible shapes in; ENABLE=0 and DISABLE=1 are rollback controls. */ + const bool iq2_q2_mm_id_pair_tail_cull_eligible = + use_mm_id && + gate_type == DS4_METAL_TENSOR_IQ2_XXS && + down_type == DS4_METAL_TENSOR_Q2_K && + request_mid_f16 && + n_expert == 6 && + g_tp_split_world == 1 && + n_tokens >= 32u; + const bool iq2_q2_mm_id_pair_tail_cull_production_geometry = + n_total_expert == 256u && + expert_in_dim == 4096u && + expert_mid_dim == 2048u && + out_dim == 4096u && + gate_row_bytes == 1056u && + gate_expert_bytes == 2162688u && + down_row_bytes == 672u && + down_expert_bytes == 2752512u; + const int enable_iq2_xxs_mm_id_pair_tail_simdgroup_cull = + ds4_gpu_env_bool( + "DS4_METAL_ENABLE_IQ2_XXS_MOE_MM_ID_PAIR_TAIL_SIMDGROUP_CULL"); + const int disable_iq2_xxs_mm_id_pair_tail_simdgroup_cull = + ds4_gpu_env_bool( + "DS4_METAL_DISABLE_IQ2_XXS_MOE_MM_ID_PAIR_TAIL_SIMDGROUP_CULL"); + const bool use_iq2_xxs_mm_id_pair_tail_simdgroup_cull = + iq2_q2_mm_id_pair_tail_cull_eligible && + use_mm_id_pair_swiglu && + disable_iq2_xxs_mm_id_pair_tail_simdgroup_cull != 1 && + (enable_iq2_xxs_mm_id_pair_tail_simdgroup_cull == 1 || + (enable_iq2_xxs_mm_id_pair_tail_simdgroup_cull == -1 && + ds4_gpu_device_is_pre_m5_apple_silicon() && + iq2_q2_mm_id_pair_tail_cull_production_geometry && + n_tokens >= 4096u)); /* * The MXFP4 32x32 specialization uses two SIMDgroups and 8 KiB of * threadgroup memory, and exactly culls SIMDgroup 1 on at-most-16-row @@ -48670,7 +48707,21 @@ int ds4_gpu_routed_moe_batch_tensor( (use_mxfp4_mm_id_pair_half_scale ? "kernel_mul_mm_id_mxfp4_pair_swiglu_f16_half_scale" : "kernel_mul_mm_id_mxfp4_pair_swiglu_f16")) : - "kernel_mul_mm_id_iq2_xxs_pair_swiglu_f16"); + (use_iq2_xxs_mm_id_pair_tail_simdgroup_cull ? + "kernel_mul_mm_id_iq2_xxs_pair_swiglu_f16_tail_cull" : + "kernel_mul_mm_id_iq2_xxs_pair_swiglu_f16")); + /* Custom DS4_METAL_MOE_SOURCE files may predate the promoted + * symbol. Preserve automatic-default compatibility by + * falling back to the established pair kernel. An explicit + * ENABLE remains strict so benchmark/bring-up cannot silently + * measure the baseline under the candidate label. */ + if (!pair_swiglu_mm_pipeline && + gate_type == DS4_METAL_TENSOR_IQ2_XXS && + use_iq2_xxs_mm_id_pair_tail_simdgroup_cull && + enable_iq2_xxs_mm_id_pair_tail_simdgroup_cull != 1) { + pair_swiglu_mm_pipeline = ds4_gpu_get_pipeline( + "kernel_mul_mm_id_iq2_xxs_pair_swiglu_f16"); + } } if (!map_pipeline || !gate_mm_pipeline || !up_mm_pipeline || !down_mm_pipeline || (use_mm_id_pair_swiglu && !pair_swiglu_mm_pipeline)) { diff --git a/metal/moe.metal b/metal/moe.metal index 036a489ef..6a36f272e 100644 --- a/metal/moe.metal +++ b/metal/moe.metal @@ -9898,6 +9898,7 @@ kernel void kernel_mul_mm_id_pair_swiglu_f16_compact_tail_impl( } typedef decltype(kernel_mul_mm_id_pair_swiglu_f16_impl) mul_mm_id_pair_swiglu_f16_iq2; +typedef decltype(kernel_mul_mm_id_pair_swiglu_f16_impl) mul_mm_id_pair_swiglu_f16_iq2_tail_cull; typedef decltype(kernel_mul_mm_id_pair_swiglu_f16_impl) mul_mm_id_pair_swiglu_f16_q4; typedef decltype(kernel_mul_mm_id_pair_swiglu_f16_impl) mul_mm_id_pair_swiglu_f16_mxfp4; typedef decltype(kernel_mul_mm_id_pair_swiglu_f16_impl) mul_mm_id_pair_swiglu_f16_mxfp4_tail_cull; @@ -9905,6 +9906,7 @@ typedef decltype(kernel_mul_mm_id_pair_swiglu_f16_compact_tail_impl; +template [[host_name("kernel_mul_mm_id_iq2_xxs_pair_swiglu_f16_tail_cull")]] kernel mul_mm_id_pair_swiglu_f16_iq2_tail_cull kernel_mul_mm_id_pair_swiglu_f16_impl; template [[host_name("kernel_mul_mm_id_q4_K_pair_swiglu_f16")]] kernel mul_mm_id_pair_swiglu_f16_q4 kernel_mul_mm_id_pair_swiglu_f16_impl; template [[host_name("kernel_mul_mm_id_mxfp4_pair_swiglu_f16")]] kernel mul_mm_id_pair_swiglu_f16_mxfp4 kernel_mul_mm_id_pair_swiglu_f16_impl; template [[host_name("kernel_mul_mm_id_mxfp4_pair_swiglu_f16_half_scale")]] kernel mul_mm_id_pair_swiglu_f16_mxfp4 kernel_mul_mm_id_pair_swiglu_f16_impl; From 86eb835439553f63f7b9aa7752cec61fb9523e67 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Thu, 27 Aug 2026 19:26:16 +0200 Subject: [PATCH 127/189] bench(metal): isolate resident IQ2 MoE prefill --- Makefile | 13 +- ds4_metal.m | 22 +- speed-bench/metal_iq2_moe_tail_cull_bench.c | 880 ++++++++++++++++++++ 3 files changed, 910 insertions(+), 5 deletions(-) create mode 100644 speed-bench/metal_iq2_moe_tail_cull_bench.c diff --git a/Makefile b/Makefile index aff698cd4..926262c7b 100644 --- a/Makefile +++ b/Makefile @@ -81,7 +81,7 @@ test-quantizer-indexer-q4: gguf-tools/deepseek4-quantize tests/test_quantizer_in ./tests/test_quantizer_indexer_q4 ./gguf-tools/deepseek4-quantize ifeq ($(UNAME_S),Darwin) -.PHONY: metal-decode-schedule-bench metal-prefill-variant-bench metal-q4-dense-pair-bench check-mxfp4-half-lut test-mxfp4-metal +.PHONY: metal-decode-schedule-bench metal-prefill-variant-bench metal-q4-dense-pair-bench metal-iq2-moe-tail-cull-bench check-mxfp4-half-lut test-mxfp4-metal all: ds4 ds4-server ds4-bench ds4-eval ds4-agent @@ -109,6 +109,7 @@ help: @echo " make metal-decode-schedule-bench Build the balanced Metal decode schedule benchmark" @echo " make metal-prefill-variant-bench Build the balanced Metal prefill variant benchmark" @echo " make metal-q4-dense-pair-bench Build the resident Q4 decode pair kernel benchmark" + @echo " make metal-iq2-moe-tail-cull-bench Build the resident IQ2 pair MoE tail-cull benchmark" @echo " make check-mxfp4-half-lut Verify the checked-in MXFP4 half LUT matches the generator" @echo " make test-mxfp4-metal Check the MXFP4 half LUT, then run Metal MXFP4 exactness tests" @echo " make test-metal-exactn-oracle Compare Metal exact-N state with sequential decode" @@ -316,6 +317,14 @@ speed-bench/metal_q4_dense_pair_bench: speed-bench/metal_q4_dense_pair_bench.m $ metal-q4-dense-pair-bench: speed-bench/metal_q4_dense_pair_bench +speed-bench/metal_iq2_moe_tail_cull_bench.o: speed-bench/metal_iq2_moe_tail_cull_bench.c ds4_gpu.h + $(CC) $(CFLAGS) -I. -c -o $@ $< + +speed-bench/metal_iq2_moe_tail_cull_bench: speed-bench/metal_iq2_moe_tail_cull_bench.o ds4_metal.o + $(CC) $(CFLAGS) -o $@ $^ $(METAL_LDLIBS) + +metal-iq2-moe-tail-cull-bench: speed-bench/metal_iq2_moe_tail_cull_bench + tests/test_mxfp4_metal.o: tests/test_mxfp4_metal.c ds4_gpu.h $(CC) $(CFLAGS) -I. -c -o $@ $< @@ -899,4 +908,4 @@ mxfp4-dot-test: tests/test_mxfp4_dot.c ./tests/test_mxfp4_dot clean: - rm -f ds4 ds4-server ds4-bench ds4-eval ds4-agent ds4_cpu ds4_native ds4_server_test ds4_test ds4_agent_test gguf-tools/quality-testing/score_official gguf-tools/quality-testing/score_official.o speed-bench/metal_decode_schedule_bench speed-bench/metal_prefill_variant_bench speed-bench/metal_q4_dense_pair_bench speed-bench/*.o tests/test_q4k_dot tests/test_mxfp4_dot tests/test_quantizer_indexer_q4 tests/test_mxfp4_metal tests/test_mxfp4_rocm tests/bench_mxfp4_rocm tests/test_mxfp4_cuda tests/test_rocm_q4_dense_pair tests/test_metal_session_batch tests/test_metal_q4_streams tests/test_metal_indexer_q4 tests/test_metal_q4_attn_exactn tests/test_metal_q4_qb_f16_cache tests/test_metal_exactn_oracle tests/test_metal_dspark_capture tests/test_metal_argmax_top1 tests/test_metal_iq2_midonly tests/test_metal_iq2_ssd_grouped_mm tests/test_metal_iq2_live_index tests/test_glm53_kda tests/test_glm53_kda_rocm tests/test_glm53_vision_engine tests/test_glm53_vision_prompt tests/test_gpu_xdev tests/test_gpu_model_cache tests/test_gpu_lookup_cache_strict tests/test_engine_mgpu_refusal tests/test_engine_mgpu_runtime tests/test_engine_correctness tests/test_sampling tests/test_cuda_session_batch tests/test_cuda_mixed_batch tests/*.o *.o tests/cuda_long_context_smoke tests/cuda_long_context_smoke.o + rm -f ds4 ds4-server ds4-bench ds4-eval ds4-agent ds4_cpu ds4_native ds4_server_test ds4_test ds4_agent_test gguf-tools/quality-testing/score_official gguf-tools/quality-testing/score_official.o speed-bench/metal_decode_schedule_bench speed-bench/metal_prefill_variant_bench speed-bench/metal_q4_dense_pair_bench speed-bench/metal_iq2_moe_tail_cull_bench speed-bench/*.o tests/test_q4k_dot tests/test_mxfp4_dot tests/test_quantizer_indexer_q4 tests/test_mxfp4_metal tests/test_mxfp4_rocm tests/bench_mxfp4_rocm tests/test_mxfp4_cuda tests/test_rocm_q4_dense_pair tests/test_metal_session_batch tests/test_metal_q4_streams tests/test_metal_indexer_q4 tests/test_metal_q4_attn_exactn tests/test_metal_q4_qb_f16_cache tests/test_metal_exactn_oracle tests/test_metal_dspark_capture tests/test_metal_argmax_top1 tests/test_metal_iq2_midonly tests/test_metal_iq2_ssd_grouped_mm tests/test_metal_iq2_live_index tests/test_glm53_kda tests/test_glm53_kda_rocm tests/test_glm53_vision_engine tests/test_glm53_vision_prompt tests/test_gpu_xdev tests/test_gpu_model_cache tests/test_gpu_lookup_cache_strict tests/test_engine_mgpu_refusal tests/test_engine_mgpu_runtime tests/test_engine_correctness tests/test_sampling tests/test_cuda_session_batch tests/test_cuda_mixed_batch tests/*.o *.o tests/cuda_long_context_smoke tests/cuda_long_context_smoke.o diff --git a/ds4_metal.m b/ds4_metal.m index 2a18a5c31..55ccae95e 100644 --- a/ds4_metal.m +++ b/ds4_metal.m @@ -1462,6 +1462,11 @@ static void ds4_gpu_close_batch_encoder(void) { static double g_gpu_busy_accum; static uint64_t g_gpu_busy_cbs; +/* Stage profilers split a batch into owned command buffers. Retain the GPU + * interval from the last synchronous completion on the calling thread so + * their reports can exclude CPU encoding, submit, and wait overhead. */ +static _Thread_local double g_last_completed_gpu_seconds; +static _Thread_local int g_last_completed_gpu_time_valid; /* A failed command buffer can leave a cross-threadgroup arrival counter at an * arbitrary partial value. Drop cached ownership instead of CPU-resetting @@ -1475,9 +1480,16 @@ static void ds4_gpu_invalidate_completion_counters(void) { } static int ds4_gpu_wait_command_buffer(id cb, const char *label) { + g_last_completed_gpu_time_valid = 0; [cb waitUntilCompleted]; + const double gpu_start = cb.GPUStartTime; + const double gpu_end = cb.GPUEndTime; + const double busy = gpu_end - gpu_start; + if (isfinite(gpu_start) && isfinite(gpu_end) && busy > 0.0) { + g_last_completed_gpu_seconds = busy; + g_last_completed_gpu_time_valid = 1; + } if (getenv("DS4_METAL_GPU_BUSY_PROFILE")) { - const double busy = cb.GPUEndTime - cb.GPUStartTime; if (busy > 0) g_gpu_busy_accum += busy; if ((++g_gpu_busy_cbs % 64u) == 0u) { fprintf(stderr, "ds4: gpu busy accum %.1f ms over %llu cbs\n", @@ -48945,19 +48957,23 @@ int ds4_gpu_routed_moe_batch_tensor( } else { \ const char *stage_name = (name); \ const double now_ms = ds4_gpu_now_ms(); \ + const double gpu_ms = \ + g_last_completed_gpu_time_valid ? \ + g_last_completed_gpu_seconds * 1000.0 : -1.0; \ const int print_stage = \ !moe_stage_filter || !moe_stage_filter[0] || \ strstr(stage_name, moe_stage_filter) != NULL; \ if (print_stage) { \ fprintf(stderr, \ "ds4: Metal routed MoE stage layer=%u tokens=%u pairs=%u experts=%u " \ - "gate=%s down=%s path=%s mid=%s %s=%.3f ms\n", \ + "gate=%s down=%s path=%s mid=%s " \ + "%s=%.3f ms gpu=%.3f ms\n", \ layer_index, n_tokens, pair_rows, n_expert, \ ds4_gpu_metal_tensor_type_name(gate_type), \ ds4_gpu_metal_tensor_type_name(down_type), \ moe_path, \ request_mid_f16 ? "f16" : "f32", \ - stage_name, now_ms - moe_stage_t0); \ + stage_name, now_ms - moe_stage_t0, gpu_ms); \ } \ moe_stage_t0 = now_ms; \ if (ds4_gpu_begin_commands() == 0) { \ diff --git a/speed-bench/metal_iq2_moe_tail_cull_bench.c b/speed-bench/metal_iq2_moe_tail_cull_bench.c new file mode 100644 index 000000000..d15a017ba --- /dev/null +++ b/speed-bench/metal_iq2_moe_tail_cull_bench.c @@ -0,0 +1,880 @@ +#define _DARWIN_C_SOURCE + +#include "ds4_gpu.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#define IQ2_XXS_TYPE 16u +#define Q2_K_TYPE 10u +#define QK_K 256u +#define IN_DIM 4096u +#define MID_DIM 2048u +#define OUT_DIM 4096u +#define N_TOKENS 4096u +#define N_TOTAL_EXPERT 256u +#define N_EXPERT 6u +#define CLAMP 4.0f +#define GUARD_WORDS 64u +#define GUARD_BYTES ((uint64_t)GUARD_WORDS * sizeof(uint32_t)) +#define GUARD_BITS 0x51a7c3e9u +#define DEFAULT_SAMPLES 8u +#define DEFAULT_WARMUP_CYCLES 1u +#define COMPARE_CHUNK_BYTES (8u * 1024u * 1024u) +#define GIB (1024ull * 1024ull * 1024ull) + +#define PAIR_TAIL_ENABLE_ENV \ + "DS4_METAL_ENABLE_IQ2_XXS_MOE_MM_ID_PAIR_TAIL_SIMDGROUP_CULL" +#define PAIR_TAIL_DISABLE_ENV \ + "DS4_METAL_DISABLE_IQ2_XXS_MOE_MM_ID_PAIR_TAIL_SIMDGROUP_CULL" + +typedef struct { + uint16_t d; + uint16_t qs[QK_K / 8u]; +} block_iq2_xxs; + +typedef struct { + uint8_t scales[QK_K / 16u]; + uint8_t qs[QK_K / 4u]; + uint16_t d; + uint16_t dmin; +} block_q2_K; + +_Static_assert(sizeof(block_iq2_xxs) == 66u, + "IQ2_XXS block layout changed"); +_Static_assert(sizeof(block_q2_K) == 84u, + "Q2_K block layout changed"); + +typedef struct { + uint32_t samples; + uint32_t warmup_cycles; +} bench_config; + +typedef enum { + ARM_BASELINE, + ARM_CANDIDATE, +} bench_arm; + +typedef struct { + void *model; + uint64_t model_size; + uint64_t gate_offset; + uint64_t up_offset; + uint64_t down_offset; + uint64_t gate_row_bytes; + uint64_t gate_expert_bytes; + uint64_t down_row_bytes; + uint64_t down_expert_bytes; + + uint64_t x_bytes; + uint64_t route_count; + uint64_t route_i32_bytes; + uint64_t route_f32_bytes; + uint64_t pair_count; + uint64_t pair_f16_bytes; + uint64_t pair_f32_bytes; + uint64_t expert_count; + uint64_t expert_bytes; + uint64_t out_count; + uint64_t out_bytes; + + ds4_gpu_tensor *x; + ds4_gpu_tensor *selected; + ds4_gpu_tensor *weights; + ds4_gpu_tensor *gate; + ds4_gpu_tensor *up; + ds4_gpu_tensor *mid; + ds4_gpu_tensor *experts; + ds4_gpu_tensor *out; +} fixture; + +typedef struct { + uint8_t *storage; + uint8_t *mid; + uint8_t *experts; + uint8_t *out; + uint64_t bytes; +} oracle_snapshot; + +bool ds4_log_is_tty(FILE *fp) { + (void)fp; + return false; +} + +static void usage(FILE *fp, const char *argv0) { + fprintf(fp, + "usage: %s [options]\n" + "\n" + "Resident production-geometry IQ2_XXS pair routed-MoE tail-cull " + "benchmark.\n" + "Metal stage profiling prints the kernel GPU timestamps; marker " + "lines identify each arm.\n" + "\n" + " --samples N samples per arm, even (default: %u)\n" + " --warmup-cycles N four-run balanced warmup cycles " + "(default: %u)\n" + " -h, --help show this help\n", + argv0, DEFAULT_SAMPLES, DEFAULT_WARMUP_CYCLES); +} + +static uint32_t parse_u32(const char *text, const char *option, + uint32_t minimum) { + char *end = NULL; + errno = 0; + const unsigned long long value = strtoull(text, &end, 10); + if (errno != 0 || !text[0] || !end || *end || + value < minimum || value > UINT32_MAX) { + fprintf(stderr, + "metal-iq2-moe-tail-cull-bench: invalid %s: %s\n", + option, text); + exit(2); + } + return (uint32_t)value; +} + +static const char *need_arg(int *index, int argc, char **argv) { + if (*index + 1 >= argc) { + fprintf(stderr, + "metal-iq2-moe-tail-cull-bench: %s needs a value\n", + argv[*index]); + exit(2); + } + return argv[++*index]; +} + +static bench_config parse_options(int argc, char **argv) { + bench_config config = { + .samples = DEFAULT_SAMPLES, + .warmup_cycles = DEFAULT_WARMUP_CYCLES, + }; + for (int i = 1; i < argc; i++) { + if (!strcmp(argv[i], "-h") || !strcmp(argv[i], "--help")) { + usage(stdout, argv[0]); + exit(0); + } else if (!strcmp(argv[i], "--samples")) { + const char *option = argv[i]; + const char *value = need_arg(&i, argc, argv); + config.samples = parse_u32(value, option, 2u); + } else if (!strcmp(argv[i], "--warmup-cycles")) { + const char *option = argv[i]; + const char *value = need_arg(&i, argc, argv); + config.warmup_cycles = parse_u32(value, option, 0u); + } else { + fprintf(stderr, + "metal-iq2-moe-tail-cull-bench: unknown option: %s\n", + argv[i]); + usage(stderr, argv[0]); + exit(2); + } + } + if ((config.samples & 1u) != 0u) { + fprintf(stderr, + "metal-iq2-moe-tail-cull-bench: --samples must be even\n"); + exit(2); + } + return config; +} + +static uint64_t align_up(uint64_t value, uint64_t alignment) { + return (value + alignment - 1u) / alignment * alignment; +} + +static uint32_t mix32(uint32_t value) { + value ^= value >> 16u; + value *= 0x7feb352du; + value ^= value >> 15u; + value *= 0x846ca68bu; + value ^= value >> 16u; + return value; +} + +static void fill_iq2(block_iq2_xxs *matrix, uint32_t salt) { + const uint32_t blocks_per_row = IN_DIM / QK_K; + for (uint32_t expert = 0; expert < N_TOTAL_EXPERT; expert++) { + for (uint32_t row = 0; row < MID_DIM; row++) { + for (uint32_t block = 0; block < blocks_per_row; block++) { + block_iq2_xxs *b = matrix + + ((uint64_t)expert * MID_DIM + row) * blocks_per_row + + block; + const uint32_t key = salt * 977u + expert * 431u + + row * 37u + block * 811u; + b->d = (uint16_t)(0x1800u + + ((key & 1u) ? 0x0200u : 0u)); + for (uint32_t i = 0; i < QK_K / 8u; i++) { + b->qs[i] = (uint16_t)(key + i * 509u + + (i >> 2u) * 131u); + } + } + } + } +} + +static void fill_q2(block_q2_K *matrix) { + const uint32_t blocks_per_row = MID_DIM / QK_K; + for (uint32_t expert = 0; expert < N_TOTAL_EXPERT; expert++) { + for (uint32_t row = 0; row < OUT_DIM; row++) { + for (uint32_t block = 0; block < blocks_per_row; block++) { + block_q2_K *b = matrix + + ((uint64_t)expert * OUT_DIM + row) * blocks_per_row + + block; + const uint32_t key = expert * 617u + row * 73u + + block * 991u; + for (uint32_t group = 0; group < QK_K / 16u; group++) { + const uint8_t scale = + (uint8_t)(1u + (key + 3u * group) % 7u); + const uint8_t min = + (uint8_t)((key / 5u + group) % 4u); + b->scales[group] = + (uint8_t)(scale | (uint8_t)(min << 4u)); + } + for (uint32_t i = 0; i < QK_K / 4u; i++) { + b->qs[i] = + (uint8_t)(key + 29u * i + (i >> 1u) * 7u); + } + b->d = 0x1800u; + b->dmin = 0x1400u; + } + } + } +} + +static uint64_t touch_model_pages(const void *model, uint64_t bytes, + uint64_t page) { + const volatile uint8_t *data = model; + uint64_t checksum = 0xcbf29ce484222325ull; + for (uint64_t offset = 0; offset < bytes; offset += page) { + checksum ^= data[offset]; + checksum *= 0x100000001b3ull; + } + checksum ^= data[bytes - 1u]; + return checksum; +} + +/* The 256 target counts use 643 complete 32-row tiles plus 4000 tail rows. + * Every tail size 1..31 is present, while the 2/3-tile split keeps counts + * non-uniform (65..127). A per-token hash breaks ties in the remaining-count + * scheduler and keeps all six routed experts unique without changing counts. */ +static int build_routes(int32_t *selected, float *weights) { + static const uint8_t final_remainders[8] = {1, 2, 3, 4, 5, 6, 7, 4}; + uint32_t target[N_TOTAL_EXPERT]; + uint32_t remaining[N_TOTAL_EXPERT]; + uint32_t actual[N_TOTAL_EXPERT] = {0}; + uint64_t target_sum = 0; + + for (uint32_t expert = 0; expert < N_TOTAL_EXPERT; expert++) { + const uint32_t tail = expert < 248u ? + 1u + (expert * 17u) % 31u : + final_remainders[expert - 248u]; + const uint32_t full_tiles = + ((expert * 73u) & 255u) < 131u ? 3u : 2u; + target[expert] = full_tiles * 32u + tail; + remaining[expert] = target[expert]; + target_sum += target[expert]; + } + if (target_sum != (uint64_t)N_TOKENS * N_EXPERT) { + fprintf(stderr, + "metal-iq2-moe-tail-cull-bench: route target sum=%llu\n", + (unsigned long long)target_sum); + return 0; + } + + for (uint32_t token = 0; token < N_TOKENS; token++) { + uint8_t used[N_TOTAL_EXPERT] = {0}; + float raw_weight[N_EXPERT]; + float weight_sum = 0.0f; + for (uint32_t slot = 0; slot < N_EXPERT; slot++) { + uint32_t best = UINT32_MAX; + uint32_t best_remaining = 0; + uint32_t best_hash = 0; + for (uint32_t expert = 0; expert < N_TOTAL_EXPERT; expert++) { + if (used[expert] || remaining[expert] == 0u) continue; + const uint32_t hash = mix32( + token * 0x9e3779b9u ^ slot * 0x85ebca6bu ^ + expert * 0xc2b2ae35u); + if (best == UINT32_MAX || + remaining[expert] > best_remaining || + (remaining[expert] == best_remaining && + hash > best_hash)) { + best = expert; + best_remaining = remaining[expert]; + best_hash = hash; + } + } + if (best == UINT32_MAX) { + fprintf(stderr, + "metal-iq2-moe-tail-cull-bench: route scheduler " + "exhausted at token=%u slot=%u\n", + token, slot); + return 0; + } + const uint64_t route = (uint64_t)token * N_EXPERT + slot; + selected[route] = (int32_t)best; + used[best] = 1u; + remaining[best]--; + actual[best]++; + raw_weight[slot] = 1.0f + (float)(mix32( + token * 0x27d4eb2du ^ slot * 0x165667b1u) % 17u); + weight_sum += raw_weight[slot]; + } + for (uint32_t slot = 0; slot < N_EXPERT; slot++) { + weights[(uint64_t)token * N_EXPERT + slot] = + raw_weight[slot] / weight_sum; + } + } + + bool tails_seen[32] = {false}; + uint32_t min_count = UINT32_MAX; + uint32_t max_count = 0; + for (uint32_t expert = 0; expert < N_TOTAL_EXPERT; expert++) { + if (remaining[expert] != 0u || actual[expert] != target[expert]) { + fprintf(stderr, + "metal-iq2-moe-tail-cull-bench: expert=%u target=%u " + "actual=%u remaining=%u\n", + expert, target[expert], actual[expert], + remaining[expert]); + return 0; + } + tails_seen[actual[expert] & 31u] = true; + if (actual[expert] < min_count) min_count = actual[expert]; + if (actual[expert] > max_count) max_count = actual[expert]; + } + for (uint32_t tail = 1; tail < 32u; tail++) { + if (!tails_seen[tail]) { + fprintf(stderr, + "metal-iq2-moe-tail-cull-bench: missing tail=%u\n", + tail); + return 0; + } + } + if (tails_seen[0]) { + fprintf(stderr, + "metal-iq2-moe-tail-cull-bench: unexpected full-only expert\n"); + return 0; + } + fprintf(stderr, + "DS4_IQ2_MOE_TAIL_SETUP routes=%u experts=%u topk=%u " + "count_min=%u count_max=%u tail_coverage=1..31 " + "unique_per_token=yes\n", + N_TOKENS * N_EXPERT, N_TOTAL_EXPERT, N_EXPERT, + min_count, max_count); + return 1; +} + +static void fill_input(float *x) { + for (uint32_t token = 0; token < N_TOKENS; token++) { + for (uint32_t column = 0; column < IN_DIM; column++) { + const uint32_t bits = mix32( + token * 0x9e3779b9u ^ column * 0x85ebca6bu); + const int32_t centered = (int32_t)(bits & 511u) - 256; + x[(uint64_t)token * IN_DIM + column] = + (float)centered / 1024.0f; + } + } +} + +static void make_guard(uint32_t guard[GUARD_WORDS]) { + for (uint32_t i = 0; i < GUARD_WORDS; i++) { + guard[i] = GUARD_BITS ^ (i * 0x9e3779b9u); + } +} + +static int write_guard(ds4_gpu_tensor *tensor, uint64_t offset) { + uint32_t guard[GUARD_WORDS]; + make_guard(guard); + return ds4_gpu_tensor_write(tensor, offset, guard, sizeof(guard)); +} + +static int check_guard(const char *name, const ds4_gpu_tensor *tensor, + uint64_t offset) { + uint32_t expected[GUARD_WORDS]; + uint32_t actual[GUARD_WORDS]; + make_guard(expected); + if (!ds4_gpu_tensor_read(tensor, offset, actual, sizeof(actual))) { + fprintf(stderr, + "DS4_IQ2_MOE_TAIL_CANARY name=%s result=READ_FAIL\n", + name); + return 0; + } + for (uint32_t i = 0; i < GUARD_WORDS; i++) { + if (actual[i] != expected[i]) { + fprintf(stderr, + "DS4_IQ2_MOE_TAIL_CANARY name=%s result=FAIL " + "word=%u expected=0x%08x actual=0x%08x\n", + name, i, expected[i], actual[i]); + return 0; + } + } + return 1; +} + +static int poison_outputs(fixture *f) { + int ok = ds4_gpu_tensor_fill_f32( + f->gate, -101.0f, (f->pair_f32_bytes + GUARD_BYTES) / sizeof(float)); + ok = ds4_gpu_tensor_fill_f32( + f->up, -102.0f, (f->pair_f32_bytes + GUARD_BYTES) / sizeof(float)) && ok; + ok = ds4_gpu_tensor_fill_f32( + f->mid, -103.0f, (f->pair_f32_bytes + GUARD_BYTES) / sizeof(float)) && ok; + ok = ds4_gpu_tensor_fill_f32( + f->experts, -104.0f, + (f->expert_bytes + GUARD_BYTES) / sizeof(float)) && ok; + ok = ds4_gpu_tensor_fill_f32( + f->out, -105.0f, + (f->out_bytes + GUARD_BYTES) / sizeof(float)) && ok; + ok = write_guard(f->gate, f->pair_f32_bytes) && ok; + ok = write_guard(f->up, f->pair_f32_bytes) && ok; + /* The first guard proves the production F16 mid contract. Full F32 + * capacity remains allocated so a wrong fallback is caught safely. */ + ok = write_guard(f->mid, f->pair_f16_bytes) && ok; + ok = write_guard(f->mid, f->pair_f32_bytes) && ok; + ok = write_guard(f->experts, f->expert_bytes) && ok; + ok = write_guard(f->out, f->out_bytes) && ok; + return ok; +} + +static int check_all_canaries(const fixture *f) { + int ok = check_guard("x", f->x, f->x_bytes); + ok = check_guard("selected", f->selected, f->route_i32_bytes) && ok; + ok = check_guard("weights", f->weights, f->route_f32_bytes) && ok; + ok = check_guard("gate", f->gate, f->pair_f32_bytes) && ok; + ok = check_guard("up", f->up, f->pair_f32_bytes) && ok; + ok = check_guard("mid-f16-boundary", f->mid, f->pair_f16_bytes) && ok; + ok = check_guard("mid-allocation-end", f->mid, f->pair_f32_bytes) && ok; + ok = check_guard("experts", f->experts, f->expert_bytes) && ok; + ok = check_guard("out", f->out, f->out_bytes) && ok; + return ok; +} + +static const char *variant_name(bench_arm arm) { + return arm == ARM_BASELINE ? "baseline" : "candidate"; +} + +static int select_variant(bench_arm arm) { + if (unsetenv(PAIR_TAIL_ENABLE_ENV) != 0 || + unsetenv(PAIR_TAIL_DISABLE_ENV) != 0) { + return 0; + } + return setenv(arm == ARM_BASELINE ? PAIR_TAIL_DISABLE_ENV : + PAIR_TAIL_ENABLE_ENV, "1", 1) == 0; +} + +static int run_once(fixture *f, bench_arm arm, + const char *phase, const char *order, + uint32_t sample, uint32_t cycle, uint32_t position, + bool poison, bool check_canaries) { + if (!select_variant(arm)) { + fprintf(stderr, + "metal-iq2-moe-tail-cull-bench: environment setup failed\n"); + return 0; + } + if (poison && !poison_outputs(f)) { + fprintf(stderr, + "metal-iq2-moe-tail-cull-bench: output poison failed\n"); + return 0; + } + + fprintf(stderr, + "DS4_IQ2_MOE_TAIL_BENCH phase=%s experiment=%s variant=%s " + "sample=%u cycle=%u position=%u order=%s force_resident=1\n", + phase, "pair", variant_name(arm), sample, + cycle, position, order); + fflush(stderr); + + if (!ds4_gpu_begin_commands()) { + fprintf(stderr, + "metal-iq2-moe-tail-cull-bench: begin commands failed\n"); + return 0; + } + bool mid_is_f16 = false; + const int call_ok = ds4_gpu_routed_moe_batch_tensor( + f->out, f->gate, f->up, f->mid, f->experts, + f->model, f->model_size, + f->gate_offset, f->up_offset, f->down_offset, + IQ2_XXS_TYPE, Q2_K_TYPE, + f->gate_expert_bytes, f->gate_row_bytes, + f->down_expert_bytes, f->down_row_bytes, + IN_DIM, MID_DIM, OUT_DIM, + f->selected, f->weights, N_TOTAL_EXPERT, N_EXPERT, CLAMP, f->x, + 0u, N_TOKENS, &mid_is_f16, true); + const int end_ok = ds4_gpu_end_commands(); + int ok = call_ok && end_ok && mid_is_f16; + if (!ok) { + fprintf(stderr, + "DS4_IQ2_MOE_TAIL_BENCH result=FAIL experiment=%s " + "variant=%s call=%d end=%d mid_f16=%d\n", + "pair", variant_name(arm), + call_ok, end_ok, mid_is_f16 ? 1 : 0); + } + if (check_canaries) ok = check_all_canaries(f) && ok; + return ok; +} + +static int snapshot_alloc(oracle_snapshot *snapshot, const fixture *f) { + memset(snapshot, 0, sizeof(*snapshot)); + snapshot->bytes = f->pair_f16_bytes + f->expert_bytes + f->out_bytes; + if (snapshot->bytes > SIZE_MAX) return 0; + snapshot->storage = malloc((size_t)snapshot->bytes); + if (!snapshot->storage) return 0; + snapshot->mid = snapshot->storage; + snapshot->experts = snapshot->mid + f->pair_f16_bytes; + snapshot->out = snapshot->experts + f->expert_bytes; + return 1; +} + +static int capture_snapshot(oracle_snapshot *snapshot, const fixture *f) { + return ds4_gpu_tensor_read( + f->mid, 0, snapshot->mid, f->pair_f16_bytes) && + ds4_gpu_tensor_read( + f->experts, 0, snapshot->experts, f->expert_bytes) && + ds4_gpu_tensor_read( + f->out, 0, snapshot->out, f->out_bytes); +} + +static int tensor_matches(const char *candidate, const char *name, + const ds4_gpu_tensor *tensor, + const uint8_t *expected, uint64_t bytes, + uint8_t *scratch, size_t scratch_bytes) { + uint64_t offset = 0; + while (offset < bytes) { + const size_t chunk = bytes - offset > scratch_bytes ? + scratch_bytes : (size_t)(bytes - offset); + if (!ds4_gpu_tensor_read(tensor, offset, scratch, chunk)) { + fprintf(stderr, + "DS4_IQ2_MOE_TAIL_ORACLE candidate=%s tensor=%s " + "result=READ_FAIL offset=%llu\n", + candidate, name, (unsigned long long)offset); + return 0; + } + if (memcmp(scratch, expected + offset, chunk) != 0) { + size_t mismatch = 0; + while (mismatch < chunk && + scratch[mismatch] == expected[offset + mismatch]) { + mismatch++; + } + fprintf(stderr, + "DS4_IQ2_MOE_TAIL_ORACLE candidate=%s tensor=%s " + "result=MISMATCH byte=%llu expected=0x%02x actual=0x%02x\n", + candidate, name, + (unsigned long long)(offset + mismatch), + expected[offset + mismatch], scratch[mismatch]); + return 0; + } + offset += chunk; + } + return 1; +} + +static int compare_candidate(const char *candidate, const fixture *f, + const oracle_snapshot *baseline, + uint8_t *scratch) { + const int mid_ok = tensor_matches( + candidate, "mid_f16", f->mid, baseline->mid, + f->pair_f16_bytes, scratch, COMPARE_CHUNK_BYTES); + const int experts_ok = tensor_matches( + candidate, "experts_f32", f->experts, baseline->experts, + f->expert_bytes, scratch, COMPARE_CHUNK_BYTES); + const int out_ok = tensor_matches( + candidate, "out_f32", f->out, baseline->out, + f->out_bytes, scratch, COMPARE_CHUNK_BYTES); + const int ok = mid_ok && experts_ok && out_ok; + fprintf(stderr, + "DS4_IQ2_MOE_TAIL_ORACLE candidate=%s result=%s " + "mid_f16=%s experts_f32=%s out_f32=%s canaries=PASS\n", + candidate, ok ? "PASS" : "FAIL", + mid_ok ? "exact" : "mismatch", + experts_ok ? "exact" : "mismatch", + out_ok ? "exact" : "mismatch"); + return ok; +} + +static int run_oracle(fixture *f) { + oracle_snapshot baseline; + if (!snapshot_alloc(&baseline, f)) { + fprintf(stderr, + "metal-iq2-moe-tail-cull-bench: oracle snapshot allocation " + "failed\n"); + return 0; + } + uint8_t *scratch = malloc(COMPARE_CHUNK_BYTES); + int ok = scratch != NULL; + + if (ok) { + ok = run_once(f, ARM_BASELINE, + "oracle", "baseline", 0u, 0u, 0u, true, true); + } + if (ok) ok = capture_snapshot(&baseline, f); + + if (ok) { + ok = run_once(f, ARM_CANDIDATE, + "oracle", "pair", 0u, 0u, 0u, true, true); + } + if (ok) ok = compare_candidate("pair", f, &baseline, scratch); + + free(scratch); + free(baseline.storage); + return ok; +} + +static int run_balanced_block(fixture *f, const char *phase, uint32_t cycles, + uint32_t sample_limit) { + uint32_t arm_samples[2] = {0, 0}; + for (uint32_t cycle = 0; cycle < cycles; cycle++) { + static const bench_arm abba[4] = { + ARM_BASELINE, ARM_CANDIDATE, + ARM_CANDIDATE, ARM_BASELINE, + }; + static const bench_arm baab[4] = { + ARM_CANDIDATE, ARM_BASELINE, + ARM_BASELINE, ARM_CANDIDATE, + }; + const bench_arm *order = (cycle & 1u) ? baab : abba; + const char *order_name = (cycle & 1u) ? "BAAB" : "ABBA"; + for (uint32_t position = 0; position < 4u; position++) { + const bench_arm arm = order[position]; + if (!run_once(f, arm, phase, order_name, + arm_samples[arm]++, cycle, position, + false, false)) { + return 0; + } + } + } + if (arm_samples[ARM_BASELINE] != sample_limit || + arm_samples[ARM_CANDIDATE] != sample_limit) { + fprintf(stderr, + "metal-iq2-moe-tail-cull-bench: %s %s arm count " + "baseline=%u candidate=%u expected=%u\n", + "pair", phase, + arm_samples[ARM_BASELINE], arm_samples[ARM_CANDIDATE], + sample_limit); + return 0; + } + return 1; +} + +static int run_experiment(fixture *f, const bench_config *config) { + if (config->warmup_cycles != 0u && + !run_balanced_block(f, "warmup", + config->warmup_cycles, + config->warmup_cycles * 2u)) { + return 0; + } + if (!run_balanced_block(f, "sample", + config->samples / 2u, config->samples)) { + return 0; + } + fprintf(stderr, + "DS4_IQ2_MOE_TAIL_BENCH phase=complete experiment=%s " + "samples_per_variant=%u warmup_per_variant=%u result=PASS\n", + "pair", config->samples, + config->warmup_cycles * 2u); + return 1; +} + +static void free_tensors(fixture *f) { + ds4_gpu_tensor_free(f->out); + ds4_gpu_tensor_free(f->experts); + ds4_gpu_tensor_free(f->mid); + ds4_gpu_tensor_free(f->up); + ds4_gpu_tensor_free(f->gate); + ds4_gpu_tensor_free(f->weights); + ds4_gpu_tensor_free(f->selected); + ds4_gpu_tensor_free(f->x); + f->out = NULL; + f->experts = NULL; + f->mid = NULL; + f->up = NULL; + f->gate = NULL; + f->weights = NULL; + f->selected = NULL; + f->x = NULL; +} + +static int init_fixture(fixture *f) { + memset(f, 0, sizeof(*f)); + const uint64_t page = (uint64_t)getpagesize(); + f->gate_row_bytes = + (uint64_t)(IN_DIM / QK_K) * sizeof(block_iq2_xxs); + f->gate_expert_bytes = (uint64_t)MID_DIM * f->gate_row_bytes; + const uint64_t gate_tensor_bytes = + (uint64_t)N_TOTAL_EXPERT * f->gate_expert_bytes; + f->down_row_bytes = + (uint64_t)(MID_DIM / QK_K) * sizeof(block_q2_K); + f->down_expert_bytes = (uint64_t)OUT_DIM * f->down_row_bytes; + const uint64_t down_tensor_bytes = + (uint64_t)N_TOTAL_EXPERT * f->down_expert_bytes; + f->gate_offset = 0; + f->up_offset = align_up(gate_tensor_bytes, page); + f->down_offset = align_up(f->up_offset + gate_tensor_bytes, page); + f->model_size = align_up(f->down_offset + down_tensor_bytes, page); + + f->x_bytes = (uint64_t)N_TOKENS * IN_DIM * sizeof(float); + f->route_count = (uint64_t)N_TOKENS * N_EXPERT; + f->route_i32_bytes = f->route_count * sizeof(int32_t); + f->route_f32_bytes = f->route_count * sizeof(float); + f->pair_count = f->route_count * MID_DIM; + f->pair_f16_bytes = f->pair_count * sizeof(_Float16); + f->pair_f32_bytes = f->pair_count * sizeof(float); + f->expert_count = (uint64_t)N_TOKENS * N_EXPERT * OUT_DIM; + f->expert_bytes = f->expert_count * sizeof(float); + f->out_count = (uint64_t)N_TOKENS * OUT_DIM; + f->out_bytes = f->out_count * sizeof(float); + + if (f->gate_row_bytes != 1056u || + f->gate_expert_bytes != 2162688u || + f->down_row_bytes != 672u || + f->down_expert_bytes != 2752512u) { + fprintf(stderr, + "metal-iq2-moe-tail-cull-bench: production layout mismatch " + "gate_row=%llu gate_expert=%llu down_row=%llu " + "down_expert=%llu\n", + (unsigned long long)f->gate_row_bytes, + (unsigned long long)f->gate_expert_bytes, + (unsigned long long)f->down_row_bytes, + (unsigned long long)f->down_expert_bytes); + return 0; + } + + const uint64_t tensor_bytes = + f->x_bytes + f->route_i32_bytes + f->route_f32_bytes + + 3u * (f->pair_f32_bytes + GUARD_BYTES) + + f->expert_bytes + GUARD_BYTES + f->out_bytes + GUARD_BYTES + + 3u * GUARD_BYTES; + const uint64_t oracle_bytes = + f->pair_f16_bytes + f->expert_bytes + f->out_bytes + + COMPARE_CHUNK_BYTES; + const uint64_t setup_host_bytes = + f->x_bytes + f->route_i32_bytes + f->route_f32_bytes; + const uint64_t explicit_peak = + f->model_size + tensor_bytes + oracle_bytes + setup_host_bytes; + fprintf(stderr, + "DS4_IQ2_MOE_TAIL_SETUP geometry=N%u,d%u,mid%u,out%u," + "experts%u,top%u model=%.3f_GiB tensors=%.3f_GiB " + "oracle=%.3f_GiB explicit_peak=%.3f_GiB\n", + N_TOKENS, IN_DIM, MID_DIM, OUT_DIM, + N_TOTAL_EXPERT, N_EXPERT, + (double)f->model_size / (double)GIB, + (double)tensor_bytes / (double)GIB, + (double)oracle_bytes / (double)GIB, + (double)explicit_peak / (double)GIB); + if (explicit_peak >= 5u * GIB) { + fprintf(stderr, + "metal-iq2-moe-tail-cull-bench: explicit peak exceeds " + "5 GiB\n"); + return 0; + } + + if (posix_memalign(&f->model, (size_t)page, + (size_t)f->model_size) != 0) { + fprintf(stderr, + "metal-iq2-moe-tail-cull-bench: model allocation failed\n"); + return 0; + } + fprintf(stderr, + "DS4_IQ2_MOE_TAIL_SETUP phase=fill_weights tensor=gate\n"); + fill_iq2((block_iq2_xxs *)((uint8_t *)f->model + f->gate_offset), + 19u); + fprintf(stderr, + "DS4_IQ2_MOE_TAIL_SETUP phase=fill_weights tensor=up\n"); + fill_iq2((block_iq2_xxs *)((uint8_t *)f->model + f->up_offset), + 47u); + fprintf(stderr, + "DS4_IQ2_MOE_TAIL_SETUP phase=fill_weights tensor=down\n"); + fill_q2((block_q2_K *)((uint8_t *)f->model + f->down_offset)); + const uint64_t checksum = + touch_model_pages(f->model, f->model_size, page); + fprintf(stderr, + "DS4_IQ2_MOE_TAIL_SETUP phase=touch_weights pages=%llu " + "checksum=0x%016llx\n", + (unsigned long long)((f->model_size + page - 1u) / page), + (unsigned long long)checksum); + + float *x_host = malloc((size_t)f->x_bytes); + int32_t *selected_host = malloc((size_t)f->route_i32_bytes); + float *weights_host = malloc((size_t)f->route_f32_bytes); + int ok = x_host && selected_host && weights_host; + if (ok) fill_input(x_host); + if (ok) ok = build_routes(selected_host, weights_host); + if (ok) ok = ds4_gpu_set_model_map(f->model, f->model_size); + + if (ok) f->x = ds4_gpu_tensor_alloc(f->x_bytes + GUARD_BYTES); + if (ok) f->selected = + ds4_gpu_tensor_alloc(f->route_i32_bytes + GUARD_BYTES); + if (ok) f->weights = + ds4_gpu_tensor_alloc(f->route_f32_bytes + GUARD_BYTES); + if (ok) f->gate = + ds4_gpu_tensor_alloc(f->pair_f32_bytes + GUARD_BYTES); + if (ok) f->up = + ds4_gpu_tensor_alloc(f->pair_f32_bytes + GUARD_BYTES); + if (ok) f->mid = + ds4_gpu_tensor_alloc(f->pair_f32_bytes + GUARD_BYTES); + if (ok) f->experts = + ds4_gpu_tensor_alloc(f->expert_bytes + GUARD_BYTES); + if (ok) f->out = ds4_gpu_tensor_alloc(f->out_bytes + GUARD_BYTES); + ok = ok && f->x && f->selected && f->weights && f->gate && f->up && + f->mid && f->experts && f->out; + + if (ok) ok = ds4_gpu_tensor_write(f->x, 0, x_host, f->x_bytes); + if (ok) ok = ds4_gpu_tensor_write( + f->selected, 0, selected_host, f->route_i32_bytes); + if (ok) ok = ds4_gpu_tensor_write( + f->weights, 0, weights_host, f->route_f32_bytes); + if (ok) ok = write_guard(f->x, f->x_bytes); + if (ok) ok = write_guard(f->selected, f->route_i32_bytes); + if (ok) ok = write_guard(f->weights, f->route_f32_bytes); + if (ok) ok = poison_outputs(f); + + free(weights_host); + free(selected_host); + free(x_host); + if (!ok) { + fprintf(stderr, + "metal-iq2-moe-tail-cull-bench: fixture initialization " + "failed\n"); + } + return ok; +} + +int main(int argc, char **argv) { + const bench_config config = parse_options(argc, argv); + + /* This benchmark compares the legacy grouped pair kernels directly. + * Prevent Metal 4 TensorOps/MPP from bypassing both A/B variants on M5+ + * hosts; the promoted automatic policy itself remains pre-M5-only. */ + setenv("DS4_METAL_DISABLE_METAL4", "1", 1); + setenv("DS4_METAL_MOE_STAGE_PROFILE", "1", 1); + setenv("DS4_METAL_MOE_STAGE_PROFILE_LAYER", "0", 1); + unsetenv("DS4_METAL_MOE_STAGE_PROFILE_FILTER"); + unsetenv("DS4_METAL_DISABLE_MOE_MM_ID_PAIR_SWIGLU"); + unsetenv("DS4_METAL_MOE_WRITE_CLAMPED_ACT"); + unsetenv("DS4_METAL_GRAPH_DUMP_PREFIX"); + unsetenv(PAIR_TAIL_ENABLE_ENV); + unsetenv(PAIR_TAIL_DISABLE_ENV); + + if (!ds4_gpu_init()) { + fprintf(stderr, + "metal-iq2-moe-tail-cull-bench: Metal initialization failed\n"); + return 1; + } + ds4_gpu_set_quality(false); + ds4_gpu_set_ssd_streaming(false); + + fixture f; + int ok = init_fixture(&f); + if (ok) ok = run_oracle(&f); + if (ok) ok = run_experiment(&f, &config); + if (ok) ok = check_all_canaries(&f); + (void)select_variant(ARM_BASELINE); + + free_tensors(&f); + ds4_gpu_cleanup(); + free(f.model); + fprintf(stderr, + "DS4_IQ2_MOE_TAIL_BENCH result=%s\n", + ok ? "PASS" : "FAIL"); + return ok ? 0 : 1; +} From 8f6718ee8ccd40a7594de26f3b50687dbe5c5d26 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Thu, 27 Aug 2026 20:35:28 +0200 Subject: [PATCH 128/189] perf(rocm): add opt-in IQ2 prefill wave culling --- rocm/ds4_rocm_moe.cuh | 31 +- rocm/ds4_rocm_moe_launch.cuh | 541 +++++++++++++++++++++++++++++++++-- 2 files changed, 537 insertions(+), 35 deletions(-) diff --git a/rocm/ds4_rocm_moe.cuh b/rocm/ds4_rocm_moe.cuh index 6ed2d5cec..d2827fa39 100644 --- a/rocm/ds4_rocm_moe.cuh +++ b/rocm/ds4_rocm_moe.cuh @@ -4840,7 +4840,7 @@ __device__ __forceinline__ static void iq2_xxs_dequant_dual_pair_tile_half_rowwi } } -template +template __global__ static void moe_gate_up_mid_iq2_hotlist_wmma_n2_kernel( float *mid_out, half *mid_out_h, @@ -4859,6 +4859,10 @@ __global__ static void moe_gate_up_mid_iq2_hotlist_wmma_n2_kernel( uint64_t gate_expert_bytes, uint64_t gate_row_bytes, float clamp) { + /* rocWMMA is collective over the hardware wave. The host selector keeps + * this kernel off wave64 devices so the scalar launch remains the complete + * fallback; retain a uniform device-side guard as a final safety net. */ + if (warpSize != 32) return; extern __shared__ unsigned char raw_sh[]; half *shA = reinterpret_cast(raw_sh); half *shBg0 = shA + MTILES * BM * BK; @@ -4892,7 +4896,10 @@ __global__ static void moe_gate_up_mid_iq2_hotlist_wmma_n2_kernel( frag_a a; frag_b bg0, bu0, bg1, bu1; frag_c accg0, accu0, accg1, accu1; - if (wave < MTILES) { + const bool wmma_active = + wave < MTILES && + (!TAIL_WAVE_CULL || wave * (uint32_t)BM < count - m_group0); + if (wmma_active) { rocwmma::fill_fragment(accg0, 0.0f); rocwmma::fill_fragment(accu0, 0.0f); rocwmma::fill_fragment(accg1, 0.0f); @@ -4928,7 +4935,7 @@ __global__ static void moe_gate_up_mid_iq2_hotlist_wmma_n2_kernel( iq2_xxs_dequant_dual_pair_tile_half_rowwise( shBg0, shBu0, shBg1, shBu1, gew, uew, gate_row_bytes, n0, k0, expert_mid_dim, tid); __syncthreads(); - if (wave < MTILES) { + if (wmma_active) { rocwmma::load_matrix_sync(a, shA + wave * BM * BK, BK); rocwmma::load_matrix_sync(bg0, shBg0, BN); rocwmma::load_matrix_sync(bu0, shBu0, BN); @@ -4942,7 +4949,7 @@ __global__ static void moe_gate_up_mid_iq2_hotlist_wmma_n2_kernel( __syncthreads(); } - if (wave < MTILES) { + if (wmma_active) { rocwmma::store_matrix_sync(shCg0 + wave * BM * BN, accg0, BN, rocwmma::mem_row_major); rocwmma::store_matrix_sync(shCu0 + wave * BM * BN, accu0, BN, rocwmma::mem_row_major); rocwmma::store_matrix_sync(shCg1 + wave * BM * BN, accg1, BN, rocwmma::mem_row_major); @@ -5225,7 +5232,7 @@ __global__ static void moe_down_q2K_hotlist_wmma_kernel( } } -template +template __global__ static void moe_down_q2K_hotlist_wmma_n2_kernel( float *down_out, half *down_out_h, @@ -5243,6 +5250,9 @@ __global__ static void moe_down_q2K_hotlist_wmma_n2_kernel( uint64_t down_row_bytes, uint32_t n_expert, uint32_t n_tokens = 0u) { + /* See the IQ2 gate/up hot-list kernel above. Correct wave64 fallback is + * selected on the host before scalar_max excludes any routed rows. */ + if (warpSize != 32) return; extern __shared__ unsigned char raw_sh[]; half *shA = reinterpret_cast(raw_sh); half *shB0 = shA + MTILES * BM * BK; @@ -5276,7 +5286,10 @@ __global__ static void moe_down_q2K_hotlist_wmma_n2_kernel( frag_b b1; frag_c acc0; frag_c acc1; - if (wave < MTILES) { + const bool wmma_active = + wave < MTILES && + (!TAIL_WAVE_CULL || wave * (uint32_t)BM < count - m_group0); + if (wmma_active) { rocwmma::fill_fragment(acc0, 0.0f); rocwmma::fill_fragment(acc1, 0.0f); } @@ -5328,7 +5341,7 @@ __global__ static void moe_down_q2K_hotlist_wmma_n2_kernel( q2_K_dequant_pair_tile_half_rowwise_staged( shB0, shB1, shW, krel, tid); __syncthreads(); - if (wave < MTILES) { + if (wmma_active) { rocwmma::load_matrix_sync(a, shA + wave * BM * BK, BK); rocwmma::load_matrix_sync(b0, shB0, BN); rocwmma::load_matrix_sync(b1, shB1, BN); @@ -5339,7 +5352,7 @@ __global__ static void moe_down_q2K_hotlist_wmma_n2_kernel( } } - if (wave < MTILES) { + if (wmma_active) { rocwmma::store_matrix_sync(shC + wave * BM * BN, acc0, BN, rocwmma::mem_row_major); } __syncthreads(); @@ -5368,7 +5381,7 @@ __global__ static void moe_down_q2K_hotlist_wmma_n2_kernel( } __syncthreads(); - if (wave < MTILES) { + if (wmma_active) { rocwmma::store_matrix_sync(shC + wave * BM * BN, acc1, BN, rocwmma::mem_row_major); } __syncthreads(); diff --git a/rocm/ds4_rocm_moe_launch.cuh b/rocm/ds4_rocm_moe_launch.cuh index 5c9b8d278..70e912873 100644 --- a/rocm/ds4_rocm_moe_launch.cuh +++ b/rocm/ds4_rocm_moe_launch.cuh @@ -10,6 +10,350 @@ static int routed_moe_align256_checked(uint64_t v, uint64_t *out) { return 1; } +enum { + DS4_ROCM_MOE_ENV_INVALID = -2 +}; + +static const char *const DS4_ROCM_IQ2_Q2_TAIL_CULL_ENABLE_ENV = + "DS4_ROCM_ENABLE_IQ2_MOE_WMMA_TAIL_CULL"; +static const char *const DS4_ROCM_IQ2_Q2_TAIL_CULL_DISABLE_ENV = + "DS4_ROCM_DISABLE_IQ2_MOE_WMMA_TAIL_CULL"; +static const char *const DS4_ROCM_IQ2_Q2_WMMA_PROFILE_ENV = + "DS4_ROCM_IQ2_MOE_WMMA_PROFILE"; + +static pthread_mutex_t g_routed_moe_hotlist_policy_mu = + PTHREAD_MUTEX_INITIALIZER; +static uint32_t g_routed_moe_hotlist_notice_mask; + +static int routed_moe_hotlist_notice_once(uint32_t bit) { + pthread_mutex_lock(&g_routed_moe_hotlist_policy_mu); + const int report = (g_routed_moe_hotlist_notice_mask & bit) == 0u; + g_routed_moe_hotlist_notice_mask |= bit; + pthread_mutex_unlock(&g_routed_moe_hotlist_policy_mu); + return report; +} + +static int routed_moe_env_value_eq(const char *value, + size_t value_len, + const char *literal) { + const size_t literal_len = strlen(literal); + if (value_len != literal_len) return 0; + for (size_t i = 0; i < value_len; i++) { + if (tolower((unsigned char)value[i]) != + tolower((unsigned char)literal[i])) { + return 0; + } + } + return 1; +} + +/* Return -1 for unset, 0/1 for a recognized boolean, and -2 for invalid. + * Do not cache values: benchmark A/B arms may change the environment between + * launches in the same process. */ +static int routed_moe_env_bool_value(const char *name, uint32_t notice_bit) { + const char *value = name ? getenv(name) : NULL; + if (!value) return -1; + while (isspace((unsigned char)*value)) value++; + size_t value_len = strlen(value); + while (value_len != 0u && + isspace((unsigned char)value[value_len - 1u])) { + value_len--; + } + if (value_len == 0u) return 1; + if (routed_moe_env_value_eq(value, value_len, "1") || + routed_moe_env_value_eq(value, value_len, "true") || + routed_moe_env_value_eq(value, value_len, "yes") || + routed_moe_env_value_eq(value, value_len, "on")) { + return 1; + } + if (routed_moe_env_value_eq(value, value_len, "0") || + routed_moe_env_value_eq(value, value_len, "false") || + routed_moe_env_value_eq(value, value_len, "no") || + routed_moe_env_value_eq(value, value_len, "off")) { + return 0; + } + if (routed_moe_hotlist_notice_once(notice_bit)) { + const size_t shown_len = value_len < 96u ? value_len : 96u; + fprintf(stderr, + DS4_GPU_LOG_PREFIX "invalid boolean environment value " + "%s=%.*s%s; treating the opt-in as disabled\n", + name, + (int)shown_len, + value, + shown_len == value_len ? "" : "..."); + } + return DS4_ROCM_MOE_ENV_INVALID; +} + +/* HIP exposes the hardware wave width through cudaDeviceProp::warpSize. The + * active device is thread-local in the runtime, so keep the successful property + * query thread-local too. This avoids a process-wide mutex on every prefill + * layer while still noticing cudaSetDevice changes. Failures stay fail-closed + * and may be retried on a later launch. */ +#if defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__) +static int routed_moe_runtime_warp_size_value(void) { + int device = -1; + const cudaError_t device_err = cudaGetDevice(&device); + if (device_err != cudaSuccess || device < 0) { + if (routed_moe_hotlist_notice_once(1u << 4)) { + fprintf(stderr, + DS4_GPU_LOG_PREFIX "cannot query the active device for " + "routed-MoE rocWMMA; using scalar fallback: %s\n", + cudaGetErrorString(device_err)); + } + (void)cudaGetLastError(); + return 0; + } + static thread_local int cached_device = -1; + static thread_local int cached_warp_size = 0; + if (cached_device == device && cached_warp_size != 0) { + return cached_warp_size; + } + + cudaDeviceProp prop = {}; + const cudaError_t prop_err = cudaGetDeviceProperties(&prop, device); + if (prop_err != cudaSuccess || prop.warpSize <= 0) { + if (routed_moe_hotlist_notice_once(1u << 5)) { + fprintf(stderr, + DS4_GPU_LOG_PREFIX "cannot query runtime warpSize for " + "routed-MoE rocWMMA; using scalar fallback: %s\n", + cudaGetErrorString(prop_err)); + } + (void)cudaGetLastError(); + return 0; + } + cached_device = device; + cached_warp_size = prop.warpSize; + return cached_warp_size; +} +#endif + +static int routed_moe_iq2_q2_tail_wave_cull_resolve( + int runtime_warp_size, + int *enable_value, + int *disable_value) { + const int enable = routed_moe_env_bool_value( + DS4_ROCM_IQ2_Q2_TAIL_CULL_ENABLE_ENV, 1u << 0); + const int disable = routed_moe_env_bool_value( + DS4_ROCM_IQ2_Q2_TAIL_CULL_DISABLE_ENV, 1u << 1); + if (enable_value) *enable_value = enable; + if (disable_value) *disable_value = disable; + if (enable == DS4_ROCM_MOE_ENV_INVALID || + disable == DS4_ROCM_MOE_ENV_INVALID) { + return 0; + } + if (disable == 1) { + if (enable == 1 && routed_moe_hotlist_notice_once(1u << 2)) { + fprintf(stderr, + DS4_GPU_LOG_PREFIX "IQ2/Q2 hot-list tail-wave cull " + "disabled by %s (overrides %s)\n", + DS4_ROCM_IQ2_Q2_TAIL_CULL_DISABLE_ENV, + DS4_ROCM_IQ2_Q2_TAIL_CULL_ENABLE_ENV); + } + return 0; + } + if (enable != 1) return 0; /* Explicit opt-in: default is off. */ + if (runtime_warp_size != 32) { + if (routed_moe_hotlist_notice_once(1u << 3)) { + fprintf(stderr, + DS4_GPU_LOG_PREFIX "IQ2/Q2 hot-list tail-wave cull " + "requested with runtime warpSize=%d; using scalar " + "fallback\n", + runtime_warp_size); + } + return 0; + } + if (routed_moe_hotlist_notice_once(1u << 6)) { + fprintf(stderr, + DS4_GPU_LOG_PREFIX "IQ2/Q2 hot-list tail-wave cull policy enabled " + "(opt-in, runtime warpSize=32)\n"); + } + return 1; +} + +#if defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__) +typedef struct { + cudaEvent_t start; + cudaEvent_t end; +} routed_moe_wmma_profile_timer; + +static int routed_moe_wmma_profile_timer_destroy( + routed_moe_wmma_profile_timer *timer, + const char *what) { + if (!timer) return 0; + int ok = 1; + if (timer->start) { + const cudaError_t err = cudaEventDestroy(timer->start); + timer->start = NULL; + if (!cuda_ok(err, what)) ok = 0; + } + if (timer->end) { + const cudaError_t err = cudaEventDestroy(timer->end); + timer->end = NULL; + if (!cuda_ok(err, what)) ok = 0; + } + return ok; +} + +static int routed_moe_wmma_profile_timer_begin( + routed_moe_wmma_profile_timer *timer, + const char *what) { + if (!timer) return 0; + timer->start = NULL; + timer->end = NULL; + cudaError_t err = cudaEventCreate(&timer->start); + if (err != cudaSuccess) return cuda_ok(err, what); + err = cudaEventCreate(&timer->end); + if (err != cudaSuccess) { + (void)cuda_ok(err, what); + (void)routed_moe_wmma_profile_timer_destroy(timer, what); + return 0; + } + err = cudaEventRecord(timer->start, 0); + if (err != cudaSuccess) { + (void)cuda_ok(err, what); + (void)routed_moe_wmma_profile_timer_destroy(timer, what); + return 0; + } + return 1; +} + +static int routed_moe_wmma_profile_timer_finish( + routed_moe_wmma_profile_timer *timer, + float *elapsed_ms, + const char *what) { + if (!timer || !timer->start || !timer->end || !elapsed_ms) { + if (timer) { + (void)routed_moe_wmma_profile_timer_destroy(timer, what); + } + return 0; + } + int ok = cuda_ok(cudaEventRecord(timer->end, 0), what); + if (ok) ok = cuda_ok(cudaEventSynchronize(timer->end), what); + if (ok) { + float ms = 0.0f; + ok = cuda_ok(cudaEventElapsedTime(&ms, timer->start, timer->end), + what); + if (ok) *elapsed_ms = ms; + } + if (!routed_moe_wmma_profile_timer_destroy(timer, what)) ok = 0; + return ok; +} +#endif + +static int routed_moe_wmma_profile_emit( + int enabled, + int tail_wave_cull, + uint32_t n_tokens, + uint64_t assignments, + int gate_up_seen, + float gate_up_ms, + int down_seen, + float down_ms, + int call_ok) { + if (!enabled) return call_ok; + const int complete = call_ok && gate_up_seen && down_seen; + const float shown_gate_up_ms = gate_up_seen ? gate_up_ms : -1.0f; + const float shown_down_ms = down_seen ? down_ms : -1.0f; + const float total_ms = gate_up_seen && down_seen + ? gate_up_ms + down_ms + : -1.0f; + const int printed = fprintf(stderr, + DS4_GPU_LOG_PREFIX "DS4_ROCM_IQ2_MOE_WMMA_PROFILE " + "tail_cull=%d tokens=%u assignments=%llu " + "gate_up_gpu_ms=%.6f down_gpu_ms=%.6f " + "wmma_total_gpu_ms=%.6f timing=cudaEvent result=%s\n", + tail_wave_cull, + n_tokens, + (unsigned long long)assignments, + shown_gate_up_ms, + shown_down_ms, + total_ms, + complete ? "PASS" : "FAIL"); + if (printed < 0 || fflush(stderr) != 0) return 0; + return complete; +} + +#if defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__) +template +static void routed_moe_launch_iq2_hotlist_wmma_n2( + dim3 grid, + dim3 block, + size_t shmem, + int tail_wave_cull, + float *mid_out, + half *mid_out_h, + const char *gate_base, + const char *up_base, + const float *x, + const half *x_h, + const float *weights, + const uint32_t *counts, + const uint32_t *offsets, + const uint32_t *pairs, + const uint32_t *hot_experts, + uint32_t hot_count, + uint32_t expert_in_dim, + uint32_t expert_mid_dim, + uint64_t gate_expert_bytes, + uint64_t gate_row_bytes, + float clamp) { + if (tail_wave_cull) { + moe_gate_up_mid_iq2_hotlist_wmma_n2_kernel< + MTILES, 16, 16, 16, OUT_F16, X_F16, true> + <<>>( + mid_out, mid_out_h, gate_base, up_base, x, x_h, weights, + counts, offsets, pairs, hot_experts, hot_count, expert_in_dim, + expert_mid_dim, gate_expert_bytes, gate_row_bytes, clamp); + } else { + moe_gate_up_mid_iq2_hotlist_wmma_n2_kernel< + MTILES, 16, 16, 16, OUT_F16, X_F16, false> + <<>>( + mid_out, mid_out_h, gate_base, up_base, x, x_h, weights, + counts, offsets, pairs, hot_experts, hot_count, expert_in_dim, + expert_mid_dim, gate_expert_bytes, gate_row_bytes, clamp); + } +} + +template +static void routed_moe_launch_q2_down_hotlist_wmma_n2( + dim3 grid, + dim3 block, + size_t shmem, + int tail_wave_cull, + float *down_out, + half *down_out_h, + const char *down_base, + const float *mid, + const half *mid_h, + const uint32_t *counts, + const uint32_t *offsets, + const uint32_t *pairs, + const uint32_t *hot_experts, + uint32_t hot_count, + uint32_t expert_mid_dim, + uint32_t out_dim, + uint64_t down_expert_bytes, + uint64_t down_row_bytes, + uint32_t n_expert) { + if (tail_wave_cull) { + moe_down_q2K_hotlist_wmma_n2_kernel< + MTILES, 16, 16, 16, MID_F16, OUT_F16, false, true> + <<>>( + down_out, down_out_h, down_base, mid, mid_h, counts, offsets, + pairs, hot_experts, hot_count, expert_mid_dim, out_dim, + down_expert_bytes, down_row_bytes, n_expert); + } else { + moe_down_q2K_hotlist_wmma_n2_kernel< + MTILES, 16, 16, 16, MID_F16, OUT_F16, false, false> + <<>>( + down_out, down_out_h, down_base, mid, mid_h, counts, offsets, + pairs, hot_experts, hot_count, expert_mid_dim, out_dim, + down_expert_bytes, down_row_bytes, n_expert); + } +} +#endif + enum { DS4_ROCM_MOE_DECODE_PROFILE_GATE_RESIDENT_START = 0, DS4_ROCM_MOE_DECODE_PROFILE_GATE_RESIDENT_END, @@ -204,6 +548,11 @@ static int routed_moe_q2_float_down_launch( const uint32_t *offsets, const uint32_t *sorted_pairs, uint32_t *hot_experts_dev, + int hotlist_wmma_wave32, + int tail_wave_cull, + int wmma_profile, + int *wmma_profile_seen, + float *wmma_profile_ms, uint32_t n_tokens, uint32_t n_total_expert, uint32_t n_expert, @@ -215,6 +564,7 @@ static int routed_moe_q2_float_down_launch( n_tokens == 0u || n_total_expert == 0u || n_total_expert > DS4_ROCM_MAX_N_EXPERT || n_expert == 0u || n_expert > DS4_ROCM_N_EXPERT_USED || (expert_mid_dim % CUDA_QK_K) != 0u || expert_mid_dim == 0u || out_dim == 0u || + (wmma_profile && (!wmma_profile_seen || !wmma_profile_ms)) || !cuda_tensor_has_elems3(mid, n_tokens, n_expert, expert_mid_dim, sizeof(float)) || !cuda_tensor_has_elems3(down, n_tokens, n_expert, out_dim, sizeof(float)) || !cuda_tensor_has_elems2(out, n_tokens, out_dim, sizeof(float))) { @@ -240,6 +590,7 @@ static int routed_moe_q2_float_down_launch( const uint32_t hot_threshold = 8u; #if defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__) const int use_wmma_hot = n_tokens >= hot_threshold && hot_experts_dev && + hotlist_wmma_wave32 && !g_quality_mode && (expert_mid_dim % 16u) == 0u && (out_dim % 16u) == 0u; #else @@ -331,6 +682,13 @@ static int routed_moe_q2_float_down_launch( #if defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__) if (use_wmma_hot && hot_count != 0u) { + routed_moe_wmma_profile_timer profile_timer = {}; + if (wmma_profile && + !routed_moe_wmma_profile_timer_begin( + &profile_timer, + "routed_moe q2 wmma down profile begin")) { + return 0; + } constexpr uint32_t bm = 16u, bn = 16u, bk = 16u; const int no_n2 = 0; const uint32_t wmma_mtiles = 4u; @@ -343,22 +701,26 @@ static int routed_moe_q2_float_down_launch( const size_t shmem_n2 = (mt * bm * bk + 2u * bk * bn) * sizeof(half) + (mt * bm * bn) * sizeof(float) + 2u * bn * 84u; if (use_f16_down && hot_mid_f16 && mid_h_hot) { - moe_down_q2K_hotlist_wmma_n2_kernel<4,16,16,16,true,true><<>>( + routed_moe_launch_q2_down_hotlist_wmma_n2<4,true,true>( + grid, block, shmem_n2, tail_wave_cull, NULL, down_h, down_w, NULL, mid_h_hot, counts, offsets, sorted_pairs, hot_experts_dev, hot_count, expert_mid_dim, out_dim, down_expert_bytes, down_row_bytes, n_expert); } else if (use_f16_down) { - moe_down_q2K_hotlist_wmma_n2_kernel<4,16,16,16,false,true><<>>( + routed_moe_launch_q2_down_hotlist_wmma_n2<4,false,true>( + grid, block, shmem_n2, tail_wave_cull, NULL, down_h, down_w, (const float *)mid->ptr, NULL, counts, offsets, sorted_pairs, hot_experts_dev, hot_count, expert_mid_dim, out_dim, down_expert_bytes, down_row_bytes, n_expert); } else if (hot_mid_f16 && mid_h_hot) { - moe_down_q2K_hotlist_wmma_n2_kernel<4,16,16,16,true,false><<>>( + routed_moe_launch_q2_down_hotlist_wmma_n2<4,true,false>( + grid, block, shmem_n2, tail_wave_cull, (float *)down->ptr, NULL, down_w, NULL, mid_h_hot, counts, offsets, sorted_pairs, hot_experts_dev, hot_count, expert_mid_dim, out_dim, down_expert_bytes, down_row_bytes, n_expert); } else { - moe_down_q2K_hotlist_wmma_n2_kernel<4,16,16,16><<>>( + routed_moe_launch_q2_down_hotlist_wmma_n2<4,false,false>( + grid, block, shmem_n2, tail_wave_cull, (float *)down->ptr, NULL, down_w, (const float *)mid->ptr, NULL, counts, offsets, sorted_pairs, hot_experts_dev, hot_count, expert_mid_dim, out_dim, down_expert_bytes, down_row_bytes, n_expert); @@ -371,22 +733,26 @@ static int routed_moe_q2_float_down_launch( const size_t shmem_n2 = (mt * bm * bk + 2u * bk * bn) * sizeof(half) + (mt * bm * bn) * sizeof(float) + 2u * bn * 84u; if (use_f16_down && hot_mid_f16 && mid_h_hot) { - moe_down_q2K_hotlist_wmma_n2_kernel<16,16,16,16,true,true><<>>( + routed_moe_launch_q2_down_hotlist_wmma_n2<16,true,true>( + grid, block, shmem_n2, tail_wave_cull, NULL, down_h, down_w, NULL, mid_h_hot, counts, offsets, sorted_pairs, hot_experts_dev, hot_count, expert_mid_dim, out_dim, down_expert_bytes, down_row_bytes, n_expert); } else if (use_f16_down) { - moe_down_q2K_hotlist_wmma_n2_kernel<16,16,16,16,false,true><<>>( + routed_moe_launch_q2_down_hotlist_wmma_n2<16,false,true>( + grid, block, shmem_n2, tail_wave_cull, NULL, down_h, down_w, (const float *)mid->ptr, NULL, counts, offsets, sorted_pairs, hot_experts_dev, hot_count, expert_mid_dim, out_dim, down_expert_bytes, down_row_bytes, n_expert); } else if (hot_mid_f16 && mid_h_hot) { - moe_down_q2K_hotlist_wmma_n2_kernel<16,16,16,16,true,false><<>>( + routed_moe_launch_q2_down_hotlist_wmma_n2<16,true,false>( + grid, block, shmem_n2, tail_wave_cull, (float *)down->ptr, NULL, down_w, NULL, mid_h_hot, counts, offsets, sorted_pairs, hot_experts_dev, hot_count, expert_mid_dim, out_dim, down_expert_bytes, down_row_bytes, n_expert); } else { - moe_down_q2K_hotlist_wmma_n2_kernel<16,16,16,16><<>>( + routed_moe_launch_q2_down_hotlist_wmma_n2<16,false,false>( + grid, block, shmem_n2, tail_wave_cull, (float *)down->ptr, NULL, down_w, (const float *)mid->ptr, NULL, counts, offsets, sorted_pairs, hot_experts_dev, hot_count, expert_mid_dim, out_dim, down_expert_bytes, down_row_bytes, n_expert); @@ -399,22 +765,26 @@ static int routed_moe_q2_float_down_launch( const size_t shmem_n2 = (mt * bm * bk + 2u * bk * bn) * sizeof(half) + (mt * bm * bn) * sizeof(float) + 2u * bn * 84u; if (use_f16_down && hot_mid_f16 && mid_h_hot) { - moe_down_q2K_hotlist_wmma_n2_kernel<8,16,16,16,true,true><<>>( + routed_moe_launch_q2_down_hotlist_wmma_n2<8,true,true>( + grid, block, shmem_n2, tail_wave_cull, NULL, down_h, down_w, NULL, mid_h_hot, counts, offsets, sorted_pairs, hot_experts_dev, hot_count, expert_mid_dim, out_dim, down_expert_bytes, down_row_bytes, n_expert); } else if (use_f16_down) { - moe_down_q2K_hotlist_wmma_n2_kernel<8,16,16,16,false,true><<>>( + routed_moe_launch_q2_down_hotlist_wmma_n2<8,false,true>( + grid, block, shmem_n2, tail_wave_cull, NULL, down_h, down_w, (const float *)mid->ptr, NULL, counts, offsets, sorted_pairs, hot_experts_dev, hot_count, expert_mid_dim, out_dim, down_expert_bytes, down_row_bytes, n_expert); } else if (hot_mid_f16 && mid_h_hot) { - moe_down_q2K_hotlist_wmma_n2_kernel<8,16,16,16,true,false><<>>( + routed_moe_launch_q2_down_hotlist_wmma_n2<8,true,false>( + grid, block, shmem_n2, tail_wave_cull, (float *)down->ptr, NULL, down_w, NULL, mid_h_hot, counts, offsets, sorted_pairs, hot_experts_dev, hot_count, expert_mid_dim, out_dim, down_expert_bytes, down_row_bytes, n_expert); } else { - moe_down_q2K_hotlist_wmma_n2_kernel<8,16,16,16><<>>( + routed_moe_launch_q2_down_hotlist_wmma_n2<8,false,false>( + grid, block, shmem_n2, tail_wave_cull, (float *)down->ptr, NULL, down_w, (const float *)mid->ptr, NULL, counts, offsets, sorted_pairs, hot_experts_dev, hot_count, expert_mid_dim, out_dim, down_expert_bytes, down_row_bytes, n_expert); @@ -451,7 +821,26 @@ static int routed_moe_q2_float_down_launch( counts, offsets, sorted_pairs, hot_experts_dev, hot_count, expert_mid_dim, out_dim, down_expert_bytes, down_row_bytes); } - if (!cuda_ok(cudaGetLastError(), "routed_moe iq2/q2 float-down wmma launch")) return 0; + const cudaError_t launch_err = cudaGetLastError(); + if (!cuda_ok( + launch_err, + "routed_moe iq2/q2 float-down wmma launch")) { + if (wmma_profile) { + (void)routed_moe_wmma_profile_timer_destroy( + &profile_timer, + "routed_moe q2 wmma down profile cleanup"); + } + return 0; + } + if (wmma_profile) { + if (!routed_moe_wmma_profile_timer_finish( + &profile_timer, + wmma_profile_ms, + "routed_moe q2 wmma down profile finish")) { + return 0; + } + *wmma_profile_seen = 1; + } } #endif @@ -597,12 +986,42 @@ static int routed_moe_launch( const int mxfp4_path = plan.mxfp4_path; const uint64_t gate_bytes = plan.gate_bytes; const uint64_t down_bytes = plan.down_bytes; + int runtime_warp_size = 0; +#if defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__) + if ((iq2_gate_path && n_tokens > 1u) || + (q2k_path && n_tokens >= 32u)) { + runtime_warp_size = routed_moe_runtime_warp_size_value(); + } +#endif + const int hotlist_wmma_wave32 = runtime_warp_size == 32; + int iq2_q2_tail_cull_enable_value = -1; + int iq2_q2_tail_cull_disable_value = -1; + const int iq2_q2_tail_wave_cull = iq2_path && n_tokens > 1u + ? routed_moe_iq2_q2_tail_wave_cull_resolve( + runtime_warp_size, + &iq2_q2_tail_cull_enable_value, + &iq2_q2_tail_cull_disable_value) + : 0; + const int iq2_q2_wmma_profile_value = iq2_path && n_tokens > 1u + ? routed_moe_env_bool_value( + DS4_ROCM_IQ2_Q2_WMMA_PROFILE_ENV, 1u << 7) + : -1; + const int iq2_q2_wmma_profile = iq2_q2_wmma_profile_value == 1; + int iq2_q2_profile_gate_up_seen = 0; + int iq2_q2_profile_down_seen = 0; + float iq2_q2_profile_gate_up_ms = 0.0f; + float iq2_q2_profile_down_ms = 0.0f; uint64_t pair_count64 = 0; if (!cuda_u64_mul_checked(n_tokens, n_expert, &pair_count64) || pair_count64 > UINT32_MAX) { return 0; } const uint32_t pair_count = (uint32_t)pair_count64; + if (iq2_q2_wmma_profile && !hotlist_wmma_wave32) { + return routed_moe_wmma_profile_emit( + 1, iq2_q2_tail_wave_cull, n_tokens, pair_count64, + 0, 0.0f, 0, 0.0f, 0); + } const ds4_gpu_tensor *selected_exec = selected; const char *gate_w = NULL; const char *up_w = NULL; @@ -760,6 +1179,12 @@ static int routed_moe_launch( !cuda_u64_mul_checked(midq_count, sizeof(cuda_block_q8_K), &midq_bytes)) { return 0; } + if (iq2_q2_wmma_profile && + (q2k_path || down->bytes < xq_bytes || gate->bytes < midq_bytes)) { + return routed_moe_wmma_profile_emit( + 1, iq2_q2_tail_wave_cull, n_tokens, pair_count64, + 0, 0.0f, 0, 0.0f, 0); + } if (!q2k_path && down->bytes >= xq_bytes && gate->bytes >= midq_bytes) { cuda_block_q8_K *xq = (cuda_block_q8_K *)down->ptr; cuda_block_q8_K *midq = (cuda_block_q8_K *)gate->ptr; @@ -996,7 +1421,16 @@ static int routed_moe_launch( } } if (ok) ok = cuda_stream_batch_selected_mark_inflight(); - return ok; + return routed_moe_wmma_profile_emit( + iq2_q2_wmma_profile, + iq2_q2_tail_wave_cull, + n_tokens, + pair_count64, + iq2_q2_profile_gate_up_seen, + iq2_q2_profile_gate_up_ms, + iq2_q2_profile_down_seen, + iq2_q2_profile_down_ms, + ok); } if (ok && use_sorted_pairs) { const uint32_t bucket_count = n_total_expert; @@ -1163,7 +1597,18 @@ static int routed_moe_launch( n_tokens >= iq2_gate_hot_threshold && n_expert == 6u && !write_gate_up && sorted_pairs && sorted_offsets && sorted_counts && tile_experts && iq2_gate_hot_dev && use_expert_tiles && (expert_in_dim % 16u) == 0u && (expert_mid_dim % 16u) == 0u && - !g_quality_mode; + hotlist_wmma_wave32 && !g_quality_mode; + if (iq2_gate_path && n_tokens > 1u && + getenv("DS4_ROCM_MOE_PATH_DEBUG") != NULL) { + fprintf(stderr, + DS4_GPU_LOG_PREFIX "IQ2 hot-list selector runtime_warp=%d " + "wmma_eligible=%u tail_policy=%d enable=%d disable=%d\n", + runtime_warp_size, + use_iq2_gate_wmma, + iq2_q2_tail_wave_cull, + iq2_q2_tail_cull_enable_value, + iq2_q2_tail_cull_disable_value); + } if (use_iq2_gate_wmma) { uint32_t h_counts[DS4_ROCM_MAX_N_EXPERT] = {0}; if (!cuda_ok(cudaMemcpy(h_counts, sorted_counts, n_total_expert * sizeof(uint32_t), cudaMemcpyDeviceToHost), @@ -1660,6 +2105,15 @@ static int routed_moe_launch( ok = cuda_ok(cudaGetLastError(), "routed_moe gate/up launch"); #if defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__) if (ok && use_iq2_gate_wmma && iq2_gate_hot_count != 0u) { + routed_moe_wmma_profile_timer profile_timer = {}; + if (iq2_q2_wmma_profile && + !routed_moe_wmma_profile_timer_begin( + &profile_timer, + "routed_moe iq2 wmma gate/up profile begin")) { + return routed_moe_wmma_profile_emit( + 1, iq2_q2_tail_wave_cull, n_tokens, pair_count64, + 0, 0.0f, 0, 0.0f, 0); + } constexpr uint32_t bm = 16u, bn = 16u, bk = 16u; const uint32_t wmma_mtiles = 4u; if (wmma_mtiles == 4u) { @@ -1671,25 +2125,29 @@ static int routed_moe_launch( const size_t shmem_n2 = (mt * bm * bk + 4u * bk * bn) * sizeof(half) + (4u * mt * bm * bn) * sizeof(float); if (use_iq2_hot_f16_mid && use_iq2_x_f16) { - moe_gate_up_mid_iq2_hotlist_wmma_n2_kernel<4,16,16,16,true,true><<>>( + routed_moe_launch_iq2_hotlist_wmma_n2<4,true,true>( + grid, block, shmem_n2, iq2_q2_tail_wave_cull, NULL, iq2_hot_mid_h, gate_w, up_w, (const float *)x->ptr, iq2_x_h, (const float *)weights->ptr, sorted_counts, sorted_offsets, sorted_pairs, iq2_gate_hot_dev, iq2_gate_hot_count, expert_in_dim, expert_mid_dim, gate_expert_bytes, gate_row_bytes, clamp); } else if (use_iq2_hot_f16_mid) { - moe_gate_up_mid_iq2_hotlist_wmma_n2_kernel<4,16,16,16,true><<>>( + routed_moe_launch_iq2_hotlist_wmma_n2<4,true,false>( + grid, block, shmem_n2, iq2_q2_tail_wave_cull, NULL, iq2_hot_mid_h, gate_w, up_w, (const float *)x->ptr, NULL, (const float *)weights->ptr, sorted_counts, sorted_offsets, sorted_pairs, iq2_gate_hot_dev, iq2_gate_hot_count, expert_in_dim, expert_mid_dim, gate_expert_bytes, gate_row_bytes, clamp); } else if (use_iq2_x_f16) { - moe_gate_up_mid_iq2_hotlist_wmma_n2_kernel<4,16,16,16,false,true><<>>( + routed_moe_launch_iq2_hotlist_wmma_n2<4,false,true>( + grid, block, shmem_n2, iq2_q2_tail_wave_cull, (float *)mid->ptr, NULL, gate_w, up_w, (const float *)x->ptr, iq2_x_h, (const float *)weights->ptr, sorted_counts, sorted_offsets, sorted_pairs, iq2_gate_hot_dev, iq2_gate_hot_count, expert_in_dim, expert_mid_dim, gate_expert_bytes, gate_row_bytes, clamp); } else { - moe_gate_up_mid_iq2_hotlist_wmma_n2_kernel<4,16,16,16><<>>( + routed_moe_launch_iq2_hotlist_wmma_n2<4,false,false>( + grid, block, shmem_n2, iq2_q2_tail_wave_cull, (float *)mid->ptr, NULL, gate_w, up_w, (const float *)x->ptr, NULL, (const float *)weights->ptr, sorted_counts, sorted_offsets, sorted_pairs, iq2_gate_hot_dev, iq2_gate_hot_count, expert_in_dim, expert_mid_dim, @@ -1704,32 +2162,50 @@ static int routed_moe_launch( const size_t shmem_n2 = (mt * bm * bk + 4u * bk * bn) * sizeof(half) + (4u * mt * bm * bn) * sizeof(float); if (use_iq2_hot_f16_mid && use_iq2_x_f16) { - moe_gate_up_mid_iq2_hotlist_wmma_n2_kernel<8,16,16,16,true,true><<>>( + routed_moe_launch_iq2_hotlist_wmma_n2<8,true,true>( + grid, block, shmem_n2, iq2_q2_tail_wave_cull, NULL, iq2_hot_mid_h, gate_w, up_w, (const float *)x->ptr, iq2_x_h, (const float *)weights->ptr, sorted_counts, sorted_offsets, sorted_pairs, iq2_gate_hot_dev, iq2_gate_hot_count, expert_in_dim, expert_mid_dim, gate_expert_bytes, gate_row_bytes, clamp); } else if (use_iq2_hot_f16_mid) { - moe_gate_up_mid_iq2_hotlist_wmma_n2_kernel<8,16,16,16,true><<>>( + routed_moe_launch_iq2_hotlist_wmma_n2<8,true,false>( + grid, block, shmem_n2, iq2_q2_tail_wave_cull, NULL, iq2_hot_mid_h, gate_w, up_w, (const float *)x->ptr, NULL, (const float *)weights->ptr, sorted_counts, sorted_offsets, sorted_pairs, iq2_gate_hot_dev, iq2_gate_hot_count, expert_in_dim, expert_mid_dim, gate_expert_bytes, gate_row_bytes, clamp); } else if (use_iq2_x_f16) { - moe_gate_up_mid_iq2_hotlist_wmma_n2_kernel<8,16,16,16,false,true><<>>( + routed_moe_launch_iq2_hotlist_wmma_n2<8,false,true>( + grid, block, shmem_n2, iq2_q2_tail_wave_cull, (float *)mid->ptr, NULL, gate_w, up_w, (const float *)x->ptr, iq2_x_h, (const float *)weights->ptr, sorted_counts, sorted_offsets, sorted_pairs, iq2_gate_hot_dev, iq2_gate_hot_count, expert_in_dim, expert_mid_dim, gate_expert_bytes, gate_row_bytes, clamp); } else { - moe_gate_up_mid_iq2_hotlist_wmma_n2_kernel<8,16,16,16><<>>( + routed_moe_launch_iq2_hotlist_wmma_n2<8,false,false>( + grid, block, shmem_n2, iq2_q2_tail_wave_cull, (float *)mid->ptr, NULL, gate_w, up_w, (const float *)x->ptr, NULL, (const float *)weights->ptr, sorted_counts, sorted_offsets, sorted_pairs, iq2_gate_hot_dev, iq2_gate_hot_count, expert_in_dim, expert_mid_dim, gate_expert_bytes, gate_row_bytes, clamp); } } - ok = cuda_ok(cudaGetLastError(), "routed_moe iq2 wmma hot gate/up launch"); + const cudaError_t launch_err = cudaGetLastError(); + ok = cuda_ok( + launch_err, + "routed_moe iq2 wmma hot gate/up launch"); + if (!ok && iq2_q2_wmma_profile) { + (void)routed_moe_wmma_profile_timer_destroy( + &profile_timer, + "routed_moe iq2 wmma gate/up profile cleanup"); + } else if (iq2_q2_wmma_profile) { + ok = routed_moe_wmma_profile_timer_finish( + &profile_timer, + &iq2_q2_profile_gate_up_ms, + "routed_moe iq2 wmma gate/up profile finish"); + if (ok) iq2_q2_profile_gate_up_seen = 1; + } } #endif } @@ -1852,6 +2328,10 @@ static int routed_moe_launch( ok = routed_moe_q2_float_down_launch( out, down, mid, iq2_hot_mid_h, use_iq2_hot_f16_mid, down_w, sorted_counts, sorted_offsets, sorted_pairs, tile_experts, + hotlist_wmma_wave32, iq2_q2_tail_wave_cull, + iq2_q2_wmma_profile, + &iq2_q2_profile_down_seen, + &iq2_q2_profile_down_ms, n_tokens, n_total_expert, n_expert, expert_mid_dim, out_dim, down_expert_bytes, down_row_bytes); } @@ -2070,7 +2550,16 @@ static int routed_moe_launch( ok = cuda_ok(cudaGetLastError(), "routed_moe sum launch"); } if (ok && compact_selected) ok = cuda_stream_selected_mark_inflight(); - return ok; + return routed_moe_wmma_profile_emit( + iq2_q2_wmma_profile, + iq2_q2_tail_wave_cull, + n_tokens, + pair_count64, + iq2_q2_profile_gate_up_seen, + iq2_q2_profile_gate_up_ms, + iq2_q2_profile_down_seen, + iq2_q2_profile_down_ms, + ok); } const ds4_rocm_runtime_config *cfg = cuda_runtime_config(); @@ -2189,7 +2678,7 @@ static int routed_moe_launch( const uint64_t f16_low_gate_bytes = (uint64_t)bucket_count * sizeof(uint32_t); const uint64_t f16_low_down_bytes = (uint64_t)bucket_count * sizeof(uint32_t); #if defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__) - const int moe_wmma_hot = !g_quality_mode && + const int moe_wmma_hot = hotlist_wmma_wave32 && !g_quality_mode && expert_in_dim % 16u == 0u && expert_mid_dim % 16u == 0u && out_dim % 16u == 0u; From 57898645c29ac8d60fe512d60db8256a4f1affe6 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Thu, 27 Aug 2026 20:35:54 +0200 Subject: [PATCH 129/189] bench(gpu): isolate resident IQ2 MoE prefill --- ENVIRONMENT_VARIABLES.md | 6 +- Makefile | 24 +- ds4_cuda.cu | 228 ++++- scripts/environment_variables.tsv | 19 +- speed-bench/README.md | 46 + speed-bench/gpu_iq2_moe_prefill_bench.c | 1058 +++++++++++++++++++++++ 6 files changed, 1369 insertions(+), 12 deletions(-) create mode 100644 speed-bench/gpu_iq2_moe_prefill_bench.c diff --git a/ENVIRONMENT_VARIABLES.md b/ENVIRONMENT_VARIABLES.md index e56577805..50fb1cbde 100644 --- a/ENVIRONMENT_VARIABLES.md +++ b/ENVIRONMENT_VARIABLES.md @@ -70,8 +70,9 @@ The detailed Metal A/B contracts and expected oracle counters live in | `DS4_CUDA_ENABLE_Q8_FOLD=1` | Enable the experimental one-shot Q8_1 producer-to-consumer fold. | | `DS4_CUDA_NO_Q8_FOLD=1` | Dominant rollback for the Q8_1 fold. | | `DS4_CUDA_Q8_FOLD_ORACLE=1` | Compare fresh canonical Q8_1 bytes and consumer outputs. Use with `DS4_CUDA_DECODE_GRAPHS=0`; require nonzero calls and zero mismatches/skips. | +| `DS4_CUDA_MOE_PROFILE=1` | Print `cudaEvent` stage timings for routed-MoE launches, including resident IQ2 pair/SwiGLU/Q2-down/sum and aligned SoA/direct-D2R fast paths. Profiling is diagnostic and synchronizes the measured stream; the added aligned-IQ2 recorder deliberately excludes one-token graph-captured decode. | -## ROCm Q4 +## ROCm Q4 and IQ2/Q2 diagnostics | Variable | Default behavior and purpose | | --- | --- | @@ -87,6 +88,9 @@ The detailed Metal A/B contracts and expected oracle counters live in | `DS4_ROCM_DISABLE_Q4_GROUPED_ATTN_A=1` | Dominant rollback to eight standalone Q4 attention-A projections, including for the resident default. | | `DS4_ROCM_REQUIRE_Q4_GROUPED_ATTN_A=1` | Request grouped attention-A for eligible non-default shapes and fail closed on fallback; `DISABLE` remains authoritative. | | `DS4_ROCM_Q4_GROUPED_ATTN_A_STATS=1` | Report grouped calls, dispatches, groups, fallbacks, and failures. | +| `DS4_ROCM_ENABLE_IQ2_MOE_WMMA_TAIL_CULL=1` | Opt into inactive-tail wave culling in the resident IQ2 gate/up and Q2 down hot-list rocWMMA prefill kernels. Requires a runtime wave width of 32; default is off pending hardware benchmarks. | +| `DS4_ROCM_DISABLE_IQ2_MOE_WMMA_TAIL_CULL=1` | Dominant value-aware rollback for the IQ2/Q2 rocWMMA tail-wave candidate. Use as the baseline arm even though the candidate is currently off by default. | +| `DS4_ROCM_IQ2_MOE_WMMA_PROFILE=1` | Measure only the affected IQ2 gate/up and Q2 down rocWMMA kernels with GPU events. A profiled eligible call fails rather than reporting a partial or scalar-fallback measurement. | Run `make test-strix-rocm-q4-parity` and `make test-strix-rocm-q4-prefill` on a `gfx1151` Strix Halo host before making diff --git a/Makefile b/Makefile index 926262c7b..99e7834de 100644 --- a/Makefile +++ b/Makefile @@ -69,7 +69,7 @@ DS4_LINK_LIBS ?= $(CUDA_LDLIBS) METAL_LDLIBS := $(LDLIBS) endif -.PHONY: all help clean test test-ssd environment-docs test-quantizer-indexer-q4 test-rocm test-glm53-kda-rocm test-metal-session-batch test-metal-session-batch-ssd test-metal-q4-streams test-metal-indexer-q4 test-metal-q4-attn-exactn test-metal-q4-qb-f16-cache test-metal-q4-qb-f16-cache-timing test-metal-exactn-oracle test-metal-dspark-capture test-metal-argmax-top1 bench-metal-argmax-top1 test-metal-iq2-midonly test-metal-iq2-ssd-grouped-mm test-metal-iq2-live-index test-mxfp4-metal test-mxfp4-cuda test-mxfp4-rocm test-mmq-parity-cuda test-rocm-q4-parity test-rocm-q4-dense test-rocm-q4-pair test-rocm-q4-prefill test-strix-rocm-q4-parity test-strix-rocm-q4-prefill test-strix-rocm-q4-prefill-long test-cuda-session-batch test-cuda-mixed-batch dspark-acceptance dspark-verify-depth rocm-dspark-acceptance rocm-dspark-verify-depth mtp-verify-depth cpu cuda cuda-spark cuda-generic cuda-regression strix-halo rocm +.PHONY: all help clean test test-ssd environment-docs test-quantizer-indexer-q4 test-rocm test-glm53-kda-rocm test-metal-session-batch test-metal-session-batch-ssd test-metal-q4-streams test-metal-indexer-q4 test-metal-q4-attn-exactn test-metal-q4-qb-f16-cache test-metal-q4-qb-f16-cache-timing test-metal-exactn-oracle test-metal-dspark-capture test-metal-argmax-top1 bench-metal-argmax-top1 test-metal-iq2-midonly test-metal-iq2-ssd-grouped-mm test-metal-iq2-live-index test-mxfp4-metal test-mxfp4-cuda test-mxfp4-rocm test-mmq-parity-cuda test-rocm-q4-parity test-rocm-q4-dense test-rocm-q4-pair test-rocm-q4-prefill test-strix-rocm-q4-parity test-strix-rocm-q4-prefill test-strix-rocm-q4-prefill-long test-cuda-session-batch test-cuda-mixed-batch dspark-acceptance dspark-verify-depth rocm-dspark-acceptance rocm-dspark-verify-depth mtp-verify-depth cpu cuda cuda-spark cuda-generic cuda-regression strix-halo rocm cuda-iq2-moe-prefill-bench rocm-iq2-moe-prefill-bench gguf-tools/deepseek4-quantize: gguf-tools/deepseek4-quantize.c gguf-tools/quants.c gguf-tools/quants.h $(MAKE) -C gguf-tools deepseek4-quantize @@ -363,6 +363,8 @@ help: @echo " make rocm-dspark-acceptance Build ROCm and run the DSpark acceptance fixture" @echo " make rocm-dspark-verify-depth Build ROCm and run the DSpark verifier invariant" @echo " make test-mxfp4-rocm Build and run the synthetic ROCm MXFP4 MoE test" + @echo " make rocm-iq2-moe-prefill-bench Build the resident ROCm IQ2/Q2 WMMA A/B harness" + @echo " make cuda-iq2-moe-prefill-bench CUDA_ARCH=sm_N Build the resident CUDA IQ2/Q2 profiling harness" @echo " make test-rocm Core regression suite on ROCm-only hosts" @echo " make cpu Build CPU-only ./ds4, ./ds4-server, ./ds4-bench, ./ds4-eval, and ./ds4-agent" @echo " make test Build and run tests" @@ -492,6 +494,24 @@ cuda/mmq/test/test_mmq_parity: cuda/mmq/test/test_mmq_parity.cu $(MMQ_OBJS) test-mmq-parity-cuda: cuda/mmq/test/test_mmq_parity ./cuda/mmq/test/test_mmq_parity + +speed-bench/gpu_iq2_moe_prefill_bench_rocm.o: speed-bench/gpu_iq2_moe_prefill_bench.c ds4_gpu.h + $(CC) $(filter-out -ffast-math,$(CFLAGS)) $(ROCM_HOST_CFLAGS) -std=c11 -DDS4_ROCM_BUILD -DDS4_BENCH_ROCM -I. -c -o $@ $< + +speed-bench/gpu_iq2_moe_prefill_bench_rocm: speed-bench/gpu_iq2_moe_prefill_bench_rocm.o ds4_rocm.o + $(HIPCC) $(ROCM_CFLAGS) -o $@ $^ $(ROCM_LDLIBS) + +rocm-iq2-moe-prefill-bench: + $(MAKE) --no-print-directory -B speed-bench/gpu_iq2_moe_prefill_bench_rocm ROCM_ARCH="$(ROCM_ARCH)" + +speed-bench/gpu_iq2_moe_prefill_bench_cuda.o: speed-bench/gpu_iq2_moe_prefill_bench.c ds4_gpu.h + $(CC) $(filter-out -ffast-math,$(CFLAGS)) -std=c11 -DDS4_BENCH_CUDA -I. -c -o $@ $< + +speed-bench/gpu_iq2_moe_prefill_bench_cuda: speed-bench/gpu_iq2_moe_prefill_bench_cuda.o ds4_cuda.o $(MMQ_OBJS) + $(NVCC) $(NVCCFLAGS) -std=c++17 $(MMQ_INCLUDES) -o $@ $^ $(CUDA_LDLIBS) + +cuda-iq2-moe-prefill-bench: + $(MAKE) --no-print-directory -B speed-bench/gpu_iq2_moe_prefill_bench_cuda CUDA_ARCH="$(CUDA_ARCH)" endif environment-docs: @@ -908,4 +928,4 @@ mxfp4-dot-test: tests/test_mxfp4_dot.c ./tests/test_mxfp4_dot clean: - rm -f ds4 ds4-server ds4-bench ds4-eval ds4-agent ds4_cpu ds4_native ds4_server_test ds4_test ds4_agent_test gguf-tools/quality-testing/score_official gguf-tools/quality-testing/score_official.o speed-bench/metal_decode_schedule_bench speed-bench/metal_prefill_variant_bench speed-bench/metal_q4_dense_pair_bench speed-bench/metal_iq2_moe_tail_cull_bench speed-bench/*.o tests/test_q4k_dot tests/test_mxfp4_dot tests/test_quantizer_indexer_q4 tests/test_mxfp4_metal tests/test_mxfp4_rocm tests/bench_mxfp4_rocm tests/test_mxfp4_cuda tests/test_rocm_q4_dense_pair tests/test_metal_session_batch tests/test_metal_q4_streams tests/test_metal_indexer_q4 tests/test_metal_q4_attn_exactn tests/test_metal_q4_qb_f16_cache tests/test_metal_exactn_oracle tests/test_metal_dspark_capture tests/test_metal_argmax_top1 tests/test_metal_iq2_midonly tests/test_metal_iq2_ssd_grouped_mm tests/test_metal_iq2_live_index tests/test_glm53_kda tests/test_glm53_kda_rocm tests/test_glm53_vision_engine tests/test_glm53_vision_prompt tests/test_gpu_xdev tests/test_gpu_model_cache tests/test_gpu_lookup_cache_strict tests/test_engine_mgpu_refusal tests/test_engine_mgpu_runtime tests/test_engine_correctness tests/test_sampling tests/test_cuda_session_batch tests/test_cuda_mixed_batch tests/*.o *.o tests/cuda_long_context_smoke tests/cuda_long_context_smoke.o + rm -f ds4 ds4-server ds4-bench ds4-eval ds4-agent ds4_cpu ds4_native ds4_server_test ds4_test ds4_agent_test gguf-tools/quality-testing/score_official gguf-tools/quality-testing/score_official.o speed-bench/metal_decode_schedule_bench speed-bench/metal_prefill_variant_bench speed-bench/metal_q4_dense_pair_bench speed-bench/metal_iq2_moe_tail_cull_bench speed-bench/gpu_iq2_moe_prefill_bench_rocm speed-bench/gpu_iq2_moe_prefill_bench_cuda speed-bench/*.o tests/test_q4k_dot tests/test_mxfp4_dot tests/test_quantizer_indexer_q4 tests/test_mxfp4_metal tests/test_mxfp4_rocm tests/bench_mxfp4_rocm tests/test_mxfp4_cuda tests/test_rocm_q4_dense_pair tests/test_metal_session_batch tests/test_metal_q4_streams tests/test_metal_indexer_q4 tests/test_metal_q4_attn_exactn tests/test_metal_q4_qb_f16_cache tests/test_metal_exactn_oracle tests/test_metal_dspark_capture tests/test_metal_argmax_top1 tests/test_metal_iq2_midonly tests/test_metal_iq2_ssd_grouped_mm tests/test_metal_iq2_live_index tests/test_glm53_kda tests/test_glm53_kda_rocm tests/test_glm53_vision_engine tests/test_glm53_vision_prompt tests/test_gpu_xdev tests/test_gpu_model_cache tests/test_gpu_lookup_cache_strict tests/test_engine_mgpu_refusal tests/test_engine_mgpu_runtime tests/test_engine_correctness tests/test_sampling tests/test_cuda_session_batch tests/test_cuda_mixed_batch tests/*.o *.o tests/cuda_long_context_smoke tests/cuda_long_context_smoke.o diff --git a/ds4_cuda.cu b/ds4_cuda.cu index 73294953a..2522c09a9 100644 --- a/ds4_cuda.cu +++ b/ds4_cuda.cu @@ -30712,6 +30712,137 @@ extern "C" int ds4_cuda_test_iq2_ssd_grouped_raw_layout( expert_in_dim, expert_mid_dim, out_dim); } +enum { CUDA_MOE_FAST_PROFILE_MAX_EVENTS = 5 }; + +static std::atomic g_cuda_moe_fast_profile_reports{0}; + +extern "C" uint64_t ds4_cuda_test_moe_fast_profile_report_count(void) { + return g_cuda_moe_fast_profile_reports.load(std::memory_order_relaxed); +} + +typedef struct { + cudaEvent_t events[CUDA_MOE_FAST_PROFILE_MAX_EVENTS]; + cudaStream_t stream; + uint32_t event_count; + int active; +} cuda_moe_fast_profile; + +/* The legacy routed-MoE profiler is below the IQ2 MMQ early returns. Keep a + * small event-only recorder for those paths so profiling never adds a stream + * synchronization (or even a CUDA call) unless DS4_CUDA_MOE_PROFILE is set. + * Destroying a recorded event is non-blocking; this makes failed MMQ entries + * safe to abandon before the established fallback is entered. */ +static void cuda_moe_fast_profile_destroy(cuda_moe_fast_profile *profile) { + if (!profile || (!profile->active && profile->event_count == 0u)) return; + for (uint32_t i = 0; i < CUDA_MOE_FAST_PROFILE_MAX_EVENTS; i++) { + if (profile->events[i]) { + (void)cudaEventDestroy(profile->events[i]); + } + } + memset(profile, 0, sizeof(*profile)); +} + +static void cuda_moe_fast_profile_begin( + cuda_moe_fast_profile *profile, + cudaStream_t stream, + uint32_t event_count) { + if (!profile) return; + memset(profile, 0, sizeof(*profile)); + if (getenv("DS4_CUDA_MOE_PROFILE") == NULL) return; + if (event_count < 2u || + event_count > CUDA_MOE_FAST_PROFILE_MAX_EVENTS) { + return; + } + cudaStreamCaptureStatus capture = cudaStreamCaptureStatusNone; + const cudaError_t capture_err = + cudaStreamIsCapturing(stream, &capture); + if (capture_err != cudaSuccess || + capture != cudaStreamCaptureStatusNone) { + if (capture_err != cudaSuccess) (void)cudaGetLastError(); + return; + } + profile->stream = stream; + profile->event_count = event_count; + for (uint32_t i = 0; i < event_count; i++) { + cudaEvent_t event = NULL; + if (cudaEventCreate(&event) != cudaSuccess) { + cuda_moe_fast_profile_destroy(profile); + return; + } + profile->events[i] = event; + } + if (cudaEventRecord(profile->events[0], stream) != cudaSuccess) { + cuda_moe_fast_profile_destroy(profile); + return; + } + profile->active = 1; +} + +static void cuda_moe_fast_profile_mark( + cuda_moe_fast_profile *profile, + uint32_t event_index) { + if (!profile || !profile->active || + event_index >= profile->event_count || + cudaEventRecord(profile->events[event_index], profile->stream) != + cudaSuccess) { + if (profile && profile->active) { + cuda_moe_fast_profile_destroy(profile); + } + } +} + +static void cuda_moe_fast_profile_report( + cuda_moe_fast_profile *profile, + const char *path, + uint32_t n_tokens, + uint64_t assignments, + const char *const *stage_names) { + if (!profile || !profile->active) return; + const uint32_t event_count = profile->event_count; + float stage_ms[CUDA_MOE_FAST_PROFILE_MAX_EVENTS - 1] = {}; + float total_ms = 0.0f; + int ok = cudaEventSynchronize(profile->events[event_count - 1u]) == + cudaSuccess; + for (uint32_t i = 1; ok && i < event_count; i++) { + ok = cudaEventElapsedTime( + &stage_ms[i - 1u], profile->events[i - 1u], + profile->events[i]) == cudaSuccess; + } + if (ok) { + ok = cudaEventElapsedTime( + &total_ms, profile->events[0], + profile->events[event_count - 1u]) == cudaSuccess; + } + if (ok) { + char line[512] = {}; + int written = snprintf( + line, sizeof(line), + "ds4: CUDA MoE profile path=%s tokens=%u assignments=%llu", + path, n_tokens, (unsigned long long)assignments); + size_t used = written > 0 ? (size_t)written : 0u; + if (used >= sizeof(line)) used = sizeof(line) - 1u; + for (uint32_t i = 1; i < event_count && used < sizeof(line); i++) { + written = snprintf( + line + used, sizeof(line) - used, " %s=%.3f", + stage_names[i - 1u], stage_ms[i - 1u]); + if (written < 0) break; + const size_t appended = (size_t)written; + used += appended < sizeof(line) - used + ? appended : sizeof(line) - used - 1u; + } + if (used < sizeof(line)) { + (void)snprintf( + line + used, sizeof(line) - used, + " total=%.3f ms (cudaEvent)", total_ms); + } + if (fprintf(stderr, "%s\n", line) >= 0) { + g_cuda_moe_fast_profile_reports.fetch_add( + 1u, std::memory_order_relaxed); + } + } + cuda_moe_fast_profile_destroy(profile); +} + static int routed_moe_launch( ds4_gpu_tensor *out, ds4_gpu_tensor *gate, @@ -30790,12 +30921,22 @@ static int routed_moe_launch( if (gate_aligned && up_aligned && down_aligned) { const cudaStream_t aligned_stream = n_tokens == 1u ? cuda_decode_stream() : (cudaStream_t)0; + cuda_moe_fast_profile aligned_profile; + if (n_tokens > 1u) { + memset(&aligned_profile, 0, sizeof(aligned_profile)); + } + const char *aligned_profile_path = NULL; const int dspark_tiny_aligned_vec = n_tokens >= 2u && n_tokens <= 5u && cuda_env_flag_enabled( "DS4_CUDA_DSPARK_TINY_ALIGNED_VEC", 0); int rc = -1; if (n_tokens == 1u || dspark_tiny_aligned_vec) { + if (dspark_tiny_aligned_vec) { + aligned_profile_path = "iq2_aligned_tiny_vec"; + cuda_moe_fast_profile_begin( + &aligned_profile, aligned_stream, 4u); + } rc = ds4_mmq_iq2_xxs_aligned_moe_gate_up_mid_vec( gate_aligned, up_aligned, (const float *)x->ptr, @@ -30806,6 +30947,9 @@ static int routed_moe_launch( (int)n_tokens, (int)n_total_expert, (int)n_expert, clamp, aligned_stream); if (rc == 0) { + if (dspark_tiny_aligned_vec) { + cuda_moe_fast_profile_mark(&aligned_profile, 1u); + } const uint32_t assignments = n_tokens * n_expert; rc = ds4_mmq_q2_K_aligned_moe_vec( down_aligned, (const float *)mid->ptr, @@ -30815,6 +30959,9 @@ static int routed_moe_launch( (int)assignments, (int)n_total_expert, /*n_expert_used=*/1, aligned_stream); + if (rc == 0 && dspark_tiny_aligned_vec) { + cuda_moe_fast_profile_mark(&aligned_profile, 2u); + } } if (rc == 0 && dspark_tiny_aligned_vec) { static int logged_dspark_tiny_aligned_vec = 0; @@ -30846,6 +30993,17 @@ static int routed_moe_launch( const int direct_applicable = g_cuda_direct_q2_prefill && direct_shape_fits && (assignments >= 1024u || direct_gb10); + /* A failed tiny-vector experiment may enter this path. Its + * partial profile must not leak into the independent fused + * attempt or the legacy fallback. */ + if (aligned_profile.event_count > 0u) { + cuda_moe_fast_profile_destroy(&aligned_profile); + } + aligned_profile_path = direct_applicable + ? "iq2_aligned_direct_d2r" + : "iq2_aligned_soa"; + cuda_moe_fast_profile_begin( + &aligned_profile, aligned_stream, 3u); if (direct_applicable) { size_t input_q8_bytes = 0; size_t down_q8_bytes = 0; @@ -30885,6 +31043,14 @@ static int routed_moe_launch( /* Only a pre-enqueue NOT_APPLICABLE result may retry the * materialized SoA path. Other failures may follow enqueue. */ if (rc == DS4_MMQ_NOT_APPLICABLE) { + aligned_profile_path = "iq2_aligned_soa"; + if (direct_applicable) { + if (aligned_profile.event_count > 0u) { + cuda_moe_fast_profile_destroy(&aligned_profile); + } + cuda_moe_fast_profile_begin( + &aligned_profile, aligned_stream, 3u); + } rc = ds4_mmq_iq2_xxs_q2_K_moe_fused_soa( gate_aligned, up_aligned, down_aligned, (const float *)x->ptr, @@ -30897,6 +31063,9 @@ static int routed_moe_launch( (int)n_total_expert, (int)n_expert, clamp, aligned_stream); } + if (rc == 0) { + cuda_moe_fast_profile_mark(&aligned_profile, 1u); + } } if (rc == 0) { const uint64_t n = (uint64_t)n_tokens * out_dim; @@ -30906,6 +31075,24 @@ static int routed_moe_launch( NULL, out_dim, n_expert, n_tokens, /*guard_nonfinite=*/1); if (cuda_ok(cudaGetLastError(), "aligned moe sum launch")) { + if (n_tokens > 1u && + aligned_profile.event_count > 0u) { + static const char *const vector_stage_names[] = { + "iq2_gateup_swiglu", "q2_down", "sum" + }; + static const char *const fused_stage_names[] = { + "fused_iq2_gateup_swiglu_q2_down", "sum" + }; + const uint32_t aligned_profile_events = + aligned_profile.event_count; + cuda_moe_fast_profile_mark( + &aligned_profile, aligned_profile_events - 1u); + cuda_moe_fast_profile_report( + &aligned_profile, aligned_profile_path, + n_tokens, (uint64_t)n_tokens * n_expert, + aligned_profile_events == 4u + ? vector_stage_names : fused_stage_names); + } static int logged = 0; if (!logged) { logged = 1; @@ -30916,6 +31103,9 @@ static int routed_moe_launch( } rc = -1; } + if (n_tokens > 1u && aligned_profile.event_count > 0u) { + cuda_moe_fast_profile_destroy(&aligned_profile); + } fprintf(stderr, "ds4: aligned routed-MoE returned %d " "(layer=%u n_tokens=%u)\n", @@ -31191,6 +31381,9 @@ static int routed_moe_launch( } g_iq2_ssd_grouped_attempts.fetch_add( 1, std::memory_order_relaxed); + cuda_moe_fast_profile grouped_profile = {}; + cuda_moe_fast_profile_begin( + &grouped_profile, grouped_stream, 3u); int rc = ds4_mmq_iq2_xxs_q2_K_moe_fused_raw( gate_w, up_w, down_w, (const float *)x->ptr, @@ -31202,6 +31395,7 @@ static int routed_moe_launch( (int)n_tokens, (int)stream_binding.weight_domain, (int)n_expert, clamp, grouped_stream); if (rc == DS4_MMQ_NOT_APPLICABLE) { + cuda_moe_fast_profile_destroy(&grouped_profile); g_iq2_ssd_grouped_not_applicable.fetch_add( 1, std::memory_order_relaxed); if (grouped_required) { @@ -31219,6 +31413,7 @@ static int routed_moe_launch( 1, std::memory_order_relaxed); } else { if (rc == 0) { + cuda_moe_fast_profile_mark(&grouped_profile, 1u); const uint64_t n = (uint64_t)n_tokens * out_dim; moe_mmq_sum_kernel<<< (uint32_t)((n + 255u) / 256u), 256, 0, @@ -31228,8 +31423,19 @@ static int routed_moe_launch( /*guard_nonfinite=*/1); rc = cuda_ok(cudaGetLastError(), "IQ2 SSD grouped moe sum launch") ? 0 : -1; + if (rc == 0) { + static const char *const grouped_stage_names[] = { + "fused_iq2_gateup_swiglu_q2_down", "sum" + }; + cuda_moe_fast_profile_mark(&grouped_profile, 2u); + cuda_moe_fast_profile_report( + &grouped_profile, "iq2_ssd_grouped_raw", + n_tokens, (uint64_t)n_tokens * n_expert, + grouped_stage_names); + } } if (rc != 0) { + cuda_moe_fast_profile_destroy(&grouped_profile); g_iq2_ssd_grouped_failures.fetch_add( 1, std::memory_order_relaxed); if (grouped_required) { @@ -31266,6 +31472,9 @@ static int routed_moe_launch( } if (down_w) { const uint64_t n_assignments = (uint64_t)n_tokens * n_expert; + cuda_moe_fast_profile resident_profile = {}; + cuda_moe_fast_profile_begin( + &resident_profile, (cudaStream_t)0, 5u); int rc = ds4_mmq_iq2_xxs_moe_pair( gate_w, up_w, (const float *)x->ptr, (const int32_t *)selected->ptr, @@ -31274,6 +31483,7 @@ static int routed_moe_launch( (int)n_tokens, (int)n_total_expert, (int)n_expert, (cudaStream_t)0); if (rc == 0) { + cuda_moe_fast_profile_mark(&resident_profile, 1u); const uint64_t mid_floats = n_assignments * expert_mid_dim; moe_mmq_swiglu_weighted_clamp_kernel<<<(uint32_t)((mid_floats + 255) / 256), 256>>>( (float *)mid->ptr, @@ -31281,6 +31491,9 @@ static int routed_moe_launch( (const float *)weights->ptr, expert_mid_dim, n_tokens, n_expert, clamp); rc = cuda_ok(cudaGetLastError(), "mmq moe swiglu launch") ? 0 : -1; + if (rc == 0) { + cuda_moe_fast_profile_mark(&resident_profile, 2u); + } } if (rc == 0) { rc = ds4_mmq_q2_K_moe( @@ -31291,6 +31504,9 @@ static int routed_moe_launch( (int)n_assignments, (int)n_total_expert, /*n_expert_used=*/1, (cudaStream_t)0); + if (rc == 0) { + cuda_moe_fast_profile_mark(&resident_profile, 3u); + } } if (rc == 0) { const uint64_t n = (uint64_t)n_tokens * out_dim; @@ -31298,9 +31514,19 @@ static int routed_moe_launch( (float *)out->ptr, (const float *)down->ptr, NULL, out_dim, n_expert, n_tokens, /*guard_nonfinite=*/1); - if (cuda_ok(cudaGetLastError(), "mmq moe sum launch")) return 1; + if (cuda_ok(cudaGetLastError(), "mmq moe sum launch")) { + static const char *const resident_stage_names[] = { + "iq2_pair", "swiglu", "q2_down", "sum" + }; + cuda_moe_fast_profile_mark(&resident_profile, 4u); + cuda_moe_fast_profile_report( + &resident_profile, "iq2_mmq_resident", + n_tokens, n_assignments, resident_stage_names); + return 1; + } rc = -1; } + cuda_moe_fast_profile_destroy(&resident_profile); fprintf(stderr, "ds4: mmq routed-MoE tier rc=%d (layer=%u n_tokens=%u); falling back\n", rc, layer_index, n_tokens); } diff --git a/scripts/environment_variables.tsv b/scripts/environment_variables.tsv index f4993a2b0..8ac779a2b 100644 --- a/scripts/environment_variables.tsv +++ b/scripts/environment_variables.tsv @@ -953,6 +953,7 @@ runtime/rocm DS4_ROCM_DISABLE_GLM_STREAMING_PREFILL_FULL_LAYER integer selector/ runtime/rocm DS4_ROCM_DISABLE_GLM_STREAMING_PREFILL_FULL_LAYER_PREPARE presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable glm streaming prefill full layer prepare. ds4.c:42988 runtime/rocm DS4_ROCM_DISABLE_GLM_STREAMING_PREFILL_SELECTED_ASYNC_LOAD presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable glm streaming prefill selected async load. ds4.c:46681 runtime/rocm DS4_ROCM_DISABLE_GLM_STREAMING_SELECTED_ASYNC_LOAD presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable glm streaming selected async load. ds4.c:44589 +runtime/rocm DS4_ROCM_DISABLE_IQ2_MOE_WMMA_TAIL_CULL value-aware boolean evaluated on every eligible launch; unset is inactive; after trimming whitespace, empty or 1/true/yes/on enables and 0/false/no/off disables (words case-insensitive); invalid values disable the candidate with a one-time diagnostic; true overrides ENABLE Force the opt-in tail-wave cull off for mixed IQ2_XXS gate/up plus Q2_K down hot-list rocWMMA prefill, providing the A/B baseline and rollback arm. rocm/ds4_rocm_moe_launch.cuh:20 runtime/rocm DS4_ROCM_DISABLE_IQ2_SELECTED_EXPERT_VIEWS presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable iq2 selected expert views. ds4.c:21094 runtime/rocm DS4_ROCM_DISABLE_IQ2_STREAM_ADDR_TABLE presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable iq2 stream addr table. ds4.c:6548 runtime/rocm DS4_ROCM_DISABLE_Q4_ATTN_Q_B_F16_CACHE value-aware rollback; unset/0/false/no/off keeps the experiment available, empty or any other value disables it Disable the resident ROCm Q4_K attn_q_b-to-F16 prefill cache even when ENABLE or REQUIRE is set. rocm/ds4_rocm_q4_qb_sidecar.cuh:140 @@ -991,6 +992,7 @@ runtime/rocm DS4_ROCM_DISABLE_STREAMING_SPLIT_SELECTED presence rollback flag; u runtime/rocm DS4_ROCM_DISABLE_STREAMING_STATIC_DECODE_MAP presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming static decode map. ds4.c:18216 runtime/rocm DS4_ROCM_DISABLE_STREAMING_STATIC_MAP_STATE_CACHE presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming static map state cache. ds4.c:18229 runtime/rocm DS4_ROCM_DSV4_PREQUANT_DECODE sampled once; unset: enabled; present empty or exact 0: disabled; every other present value: enabled; quality mode and GLM models force it off regardless ROCm DeepSeek-V4 decode: quantizes one-token F32 activations to Q8 once and selects the prequantized Q8_0/DP4A projection kernels instead of the full-F32 activation paths. rocm/ds4_rocm_runtime.cuh:4775 +runtime/rocm DS4_ROCM_ENABLE_IQ2_MOE_WMMA_TAIL_CULL value-aware boolean evaluated on every eligible launch; unset defaults off; after trimming whitespace, empty or 1/true/yes/on enables and 0/false/no/off disables (words case-insensitive); invalid values fail closed with a one-time diagnostic; true DISABLE overrides Opt into skipping inactive 16-row tail waves in both wave32 mixed IQ2_XXS gate/up and Q2_K down hot-list rocWMMA prefill kernels; other shapes and paths retain their existing fallback. rocm/ds4_rocm_moe_launch.cuh:18 runtime/rocm DS4_ROCM_ENABLE_MXFP4_LDSB presence opt-in; unset=off; any defined value including empty or 0 enables the candidate when the MXFP4 path, sorted expert tiles, token count >= 128, LDS-size limit, and dimension-alignment gates all pass Select the ROCm MXFP4 prefill gate/up kernel that stages eight gate and eight up weight rows in LDS and reuses them across expert tiles of up to 128 tokens. rocm/ds4_rocm_moe_launch.cuh:782 runtime/rocm DS4_ROCM_ENABLE_MXFP4_ROW64 presence opt-in; unset=off; any defined value including empty or 0 enables the candidate when the MXFP4 sorted-tile path has at least 8 tokens and the TILE32, LDSB, and TILE4 candidates are not selected Select the ROCm MXFP4 gate/up tile8 occupancy variant with 64 row slots and 512 threads per block. rocm/ds4_rocm_moe_launch.cuh:798 runtime/rocm DS4_ROCM_ENABLE_MXFP4_TILE32 presence opt-in; unset=off; any defined value including empty or 0 enables the candidate when the MXFP4 sorted-tile path has at least 32 tokens and the expert intermediate dimension is divisible by 32 Select the ROCm MXFP4 gate/up tile32 kernel, reusing each loaded expert-weight chunk across as many as 32 tokens. rocm/ds4_rocm_moe_launch.cuh:786 @@ -1035,14 +1037,15 @@ runtime/rocm DS4_ROCM_GRAPH_DUMP_NAME nonempty substring filter; unset=all tenso runtime/rocm DS4_ROCM_GRAPH_DUMP_NONINVASIVE truthy value under the shared parser; unset lets dumping select conservative kernels Keep production ROCm kernel selection while graph dumping. rocm/ds4_rocm_runtime.cuh:4822 runtime/rocm DS4_ROCM_GRAPH_DUMP_POS unsigned token position; unset=all positions Filter ROCm graph dumps by position. ds4.c:16869 runtime/rocm DS4_ROCM_GRAPH_DUMP_PREFIX nonempty output path prefix; unset=off Enable ROCm intermediate graph/tensor dumps. ds4_cuda.cu:397 -runtime/rocm DS4_ROCM_GRAPH_DUMP_TRACE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Emit trace diagnostics for rocm graph dump trace. ds4.c:16899 -runtime/rocm DS4_ROCM_GRAPH_OUTPUT_ROW Nonempty string with a leading strtoul-parsable unsigned value < n_tokens selects that zero-based row. Default, empty, unparsable, or out-of-range selects n_tokens-1. Trailing characters are accepted because full consumption/errno are not checked. A nonempty ROCm value takes precedence; otherwise DS4_METAL_GRAPH_OUTPUT_ROW is a fallback alias. Choose which prefill hidden-state row is sent through the output head to produce logits, primarily for graph/correctness diagnostics. ds4.c:35845 -runtime/rocm DS4_ROCM_GRAPH_PREFILL_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm graph prefill profile. ds4.c:32038 -runtime/rocm DS4_ROCM_GRAPH_PREFILL_SPLIT_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm graph prefill split profile. ds4.c:35768 -runtime/rocm DS4_ROCM_GRAPH_TOKEN_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm graph token profile. ds4.c:31722 -runtime/rocm DS4_ROCM_INDEXER_STAGE_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm indexer stage profile. ds4.c:29265 -runtime/rocm DS4_ROCM_LAYER_STAGE_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm layer stage profile. ds4.c:29104 -runtime/rocm DS4_ROCM_LAYER_STAGE_PROFILE_LAYER layer filter subordinate to DS4_ROCM_LAYER_STAGE_PROFILE; unset or whitespace-only: all layers allowed by the parent flag; otherwise the whitespace-trimmed value must be a complete base-10 strtoul result <= UINT32_MAX equal to the current layer; invalid values match none Restricts the ROCm layer/prefill stage profiler to one layer; it does not enable profiling by itself. ds4.c:29105 +runtime/rocm DS4_ROCM_GRAPH_DUMP_TRACE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Emit trace diagnostics for rocm graph dump trace. ds4.c:16914 +runtime/rocm DS4_ROCM_GRAPH_OUTPUT_ROW Nonempty string with a leading strtoul-parsable unsigned value < n_tokens selects that zero-based row. Default, empty, unparsable, or out-of-range selects n_tokens-1. Trailing characters are accepted because full consumption/errno are not checked. A nonempty ROCm value takes precedence; otherwise DS4_METAL_GRAPH_OUTPUT_ROW is a fallback alias. Choose which prefill hidden-state row is sent through the output head to produce logits, primarily for graph/correctness diagnostics. ds4.c:35860 +runtime/rocm DS4_ROCM_GRAPH_PREFILL_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm graph prefill profile. ds4.c:32053 +runtime/rocm DS4_ROCM_GRAPH_PREFILL_SPLIT_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm graph prefill split profile. ds4.c:35783 +runtime/rocm DS4_ROCM_GRAPH_TOKEN_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm graph token profile. ds4.c:31737 +runtime/rocm DS4_ROCM_INDEXER_STAGE_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm indexer stage profile. ds4.c:29280 +runtime/rocm DS4_ROCM_IQ2_MOE_WMMA_PROFILE value-aware boolean evaluated on every N>1 mixed IQ2/Q2 launch; unset is off; after trimming whitespace, empty or 1/true/yes/on enables and 0/false/no/off disables (words case-insensitive); invalid values disable profiling with a one-time diagnostic Use HIP cudaEvent timing to report gate/up, down, and summed GPU milliseconds for the affected hot-list rocWMMA stages, excluding SSD/CPU time; profiling is strict and fails the backend call if either stage or event timing is unavailable. rocm/ds4_rocm_moe_launch.cuh:22 +runtime/rocm DS4_ROCM_LAYER_STAGE_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm layer stage profile. ds4.c:29119 +runtime/rocm DS4_ROCM_LAYER_STAGE_PROFILE_LAYER layer filter subordinate to DS4_ROCM_LAYER_STAGE_PROFILE; unset or whitespace-only: all layers allowed by the parent flag; otherwise the whitespace-trimmed value must be a complete base-10 strtoul result <= UINT32_MAX equal to the current layer; invalid values match none Restricts the ROCm layer/prefill stage profiler to one layer; it does not enable profiling by itself. ds4.c:29120 runtime/rocm DS4_ROCM_MOE_DECODE_DOWN_RPB sampled once; nonempty value is parsed by strtoul (a numeric prefix is sufficient), cast to uint32_t, and accepted only if 1/2/4/8/16/32; unset/empty/invalid inherits DS4_ROCM_MOE_DECODE_RPB, with defaults quality=8, non-quality SSD=2, resident=1 Sets output rows (warps) per block for ROCm Q2_K routed-MoE decode down-projection kernels; threads per block are value * 32. rocm/ds4_rocm_runtime.cuh:4842 runtime/rocm DS4_ROCM_MOE_DECODE_GATE_RPB sampled once; nonempty value is parsed by strtoul (a numeric prefix is sufficient), cast to uint32_t, and accepted only if 1/2/4/8/16/32; unset/empty/invalid defaults to 1 in non-quality SSD mode when DS4_ROCM_MOE_DECODE_RPB is unset/empty, otherwise inherits the resolved base RPB Sets output rows (warps) per block for ROCm Q2_K routed-MoE decode gate/up kernels; threads per block are value * 32. rocm/ds4_rocm_runtime.cuh:4836 runtime/rocm DS4_ROCM_MOE_DECODE_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm moe decode profile. rocm/ds4_rocm_moe_launch.cuh:82 diff --git a/speed-bench/README.md b/speed-bench/README.md index ca0c997fd..460235c16 100644 --- a/speed-bench/README.md +++ b/speed-bench/README.md @@ -98,3 +98,49 @@ with one process and engine per variant before promotion. Environment variables consumed while the engine opens, including the Metal streaming `F_NOCACHE` controls, cannot be compared with `--candidate-env` in this harness and likewise require separate processes. + +### Resident IQ2/Q2 MoE prefill on ROCm and CUDA + +The backend-neutral fixture uses the production `N=4096`, 256-expert, top-6 +IQ2_XXS/Q2_K geometry and a deterministic routing distribution containing +every 32-row tail from 1 through 31. Weights and tensors are resident and SSD +streaming is disabled, so the reported GPU-event intervals isolate kernel +work rather than storage throughput. The fixture needs roughly 4 GiB of +explicit host/device storage in addition to backend runtime overhead. + +On a wave32 ROCm host, build and run the real balanced A/B with: + +``` +make rocm-iq2-moe-prefill-bench ROCM_ARCH=gfx1151 +./speed-bench/gpu_iq2_moe_prefill_bench_rocm +``` + +The baseline sets the dominant rollback +`DS4_ROCM_DISABLE_IQ2_MOE_WMMA_TAIL_CULL=1`; the candidate sets +`DS4_ROCM_ENABLE_IQ2_MOE_WMMA_TAIL_CULL=1`. The harness alternates ABBA/BAAB, +requires bit-exact intermediate scratch and final tensors, verifies allocation +canaries, and prints `cudaEvent` time for only the IQ2 gate/up and Q2 down +rocWMMA kernels. The candidate remains opt-in until real-hardware results show +a repeatable win. A wave64 device takes the scalar fallback and therefore +cannot produce a valid candidate timing. + +On CUDA, build and run the measurement-only current path with an explicit +architecture: + +``` +make cuda-iq2-moe-prefill-bench CUDA_ARCH=sm_121 +./speed-bench/gpu_iq2_moe_prefill_bench_cuda +``` + +CUDA prints `DS4_CUDA_MOE_PROFILE` stage times for the resident IQ2 MMQ path +and performs a structural/canary oracle. Every marked call must produce exactly +one completed fast-path profile record or the harness fails. It intentionally +does not claim an A/B result: CUDA's cooperative D2R CTA has different +synchronization and tail semantics, so the ROCm/Metal wave-cull selector cannot +be copied safely. +On a discrete CUDA GPU with enough VRAM, prefix the run with +`DS4_CUDA_COPY_MODEL=1` to keep the raw expert weights in device memory and +remove mapped-host/PCIe stalls from the measured kernel interval; this adds +about 1.7 GiB of device storage. Leave it unset on GB10/UMA when measuring the +normal aligned-artifact production selector, because forcing a raw model copy +changes that residency path. diff --git a/speed-bench/gpu_iq2_moe_prefill_bench.c b/speed-bench/gpu_iq2_moe_prefill_bench.c new file mode 100644 index 000000000..261f87086 --- /dev/null +++ b/speed-bench/gpu_iq2_moe_prefill_bench.c @@ -0,0 +1,1058 @@ +#define _POSIX_C_SOURCE 200809L + +#include "ds4_gpu.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(DS4_BENCH_ROCM) && defined(DS4_BENCH_CUDA) +#error "define exactly one of DS4_BENCH_ROCM or DS4_BENCH_CUDA" +#elif !defined(DS4_BENCH_ROCM) && !defined(DS4_BENCH_CUDA) +#error "define exactly one of DS4_BENCH_ROCM or DS4_BENCH_CUDA" +#endif + +/* + * This source intentionally has no implicit backend selection. Compile it + * once with DS4_BENCH_ROCM and link the ROCm implementation, or once with + * DS4_BENCH_CUDA and link the CUDA implementation. + * + * ROCm exposes a real in-process A/B policy switch. CUDA currently has no + * independent tail-cull candidate selected by this harness; its build is a + * measurement-only run of the current production path. In particular, the + * CUDA markers never label two executions of the same path as an A/B result. + */ +#if defined(DS4_BENCH_ROCM) +#define BENCH_BACKEND "rocm" +#define TAIL_ENABLE_ENV "DS4_ROCM_ENABLE_IQ2_MOE_WMMA_TAIL_CULL" +#define TAIL_DISABLE_ENV "DS4_ROCM_DISABLE_IQ2_MOE_WMMA_TAIL_CULL" +#define ROCM_PROFILE_ENV "DS4_ROCM_IQ2_MOE_WMMA_PROFILE" +#else +#define BENCH_BACKEND "cuda" +#define CUDA_PROFILE_ENV "DS4_CUDA_MOE_PROFILE" +#endif + +#define IQ2_XXS_TYPE 16u +#define Q2_K_TYPE 10u +#define QK_K 256u +#define IN_DIM 4096u +#define MID_DIM 2048u +#define OUT_DIM 4096u +#define N_TOKENS 4096u +#define N_TOTAL_EXPERT 256u +#define N_EXPERT 6u +#define CLAMP 4.0f +#define GUARD_WORDS 64u +#define GUARD_BYTES ((uint64_t)GUARD_WORDS * sizeof(uint32_t)) +#define GUARD_BITS 0x51a7c3e9u +#define DEFAULT_SAMPLES 8u +#define DEFAULT_WARMUPS 2u +#define IO_CHUNK_BYTES (8u * 1024u * 1024u) +#define GIB (1024ull * 1024ull * 1024ull) + +typedef struct { + uint16_t d; + uint16_t qs[QK_K / 8u]; +} block_iq2_xxs; + +typedef struct { + uint8_t scales[QK_K / 16u]; + uint8_t qs[QK_K / 4u]; + uint16_t d; + uint16_t dmin; +} block_q2_K; + +_Static_assert(sizeof(block_iq2_xxs) == 66u, + "IQ2_XXS block layout changed"); +_Static_assert(sizeof(block_q2_K) == 84u, + "Q2_K block layout changed"); + +typedef struct { + uint32_t samples; + uint32_t warmups; +} bench_config; + +typedef enum { + ARM_BASELINE, + ARM_CANDIDATE, + ARM_CURRENT, +} bench_arm; + +typedef struct { + void *model; + uint64_t model_size; + uint64_t gate_offset; + uint64_t up_offset; + uint64_t down_offset; + uint64_t gate_row_bytes; + uint64_t gate_expert_bytes; + uint64_t down_row_bytes; + uint64_t down_expert_bytes; + + uint64_t x_bytes; + uint64_t route_count; + uint64_t route_i32_bytes; + uint64_t route_f32_bytes; + uint64_t pair_count; + uint64_t pair_f32_bytes; + uint64_t expert_count; + uint64_t expert_bytes; + uint64_t out_count; + uint64_t out_bytes; + + ds4_gpu_tensor *x; + ds4_gpu_tensor *selected; + ds4_gpu_tensor *weights; + ds4_gpu_tensor *gate; + ds4_gpu_tensor *up; + ds4_gpu_tensor *mid; + ds4_gpu_tensor *experts; + ds4_gpu_tensor *out; +} fixture; + +#if defined(DS4_BENCH_ROCM) +typedef struct { + uint8_t *storage; + uint8_t *gate; + uint8_t *up; + uint8_t *mid; + uint8_t *experts; + uint8_t *out; + uint64_t bytes; +} oracle_snapshot; +#endif + +bool ds4_log_is_tty(FILE *fp) { + (void)fp; + return false; +} + +#if defined(DS4_BENCH_CUDA) +extern uint64_t ds4_cuda_test_moe_fast_profile_report_count(void); +#endif + +static void usage(FILE *fp, const char *argv0) { + fprintf(fp, + "usage: %s [options]\n" + "\n" + "Resident production-geometry IQ2_XXS/Q2_K routed-MoE prefill " + "benchmark (%s build).\n", + argv0, BENCH_BACKEND); +#if defined(DS4_BENCH_ROCM) + fprintf(fp, + "ROCm performs a real balanced A/B: DISABLE=1 is baseline and " + "ENABLE=1 is candidate. The harness enables the GPU-only " + "IQ2/Q2 WMMA profiler.\n" + " --samples N samples per arm, even (default: %u)\n" + " --warmups N warmups per arm, even (default: %u)\n", + DEFAULT_SAMPLES, DEFAULT_WARMUPS); +#else + fprintf(fp, + "CUDA is measurement-only: every marker is variant=current and " + "DS4_CUDA_MOE_PROFILE is enabled.\n" + "No reliable baseline/candidate selector exists in this process; " + "this harness does not report a false A/B.\n" + " --samples N current-path measured runs (default: %u)\n" + " --warmups N current-path warmup runs (default: %u)\n", + DEFAULT_SAMPLES, DEFAULT_WARMUPS); +#endif + fprintf(fp, " -h, --help show this help\n"); +} + +static uint32_t parse_u32(const char *text, const char *option) { + char *end = NULL; + errno = 0; + const unsigned long long value = strtoull(text, &end, 10); + if (errno != 0 || !text[0] || !end || *end || value > UINT32_MAX) { + fprintf(stderr, "gpu-iq2-moe-prefill-bench: invalid %s: %s\n", + option, text); + exit(2); + } + return (uint32_t)value; +} + +static const char *need_arg(int *index, int argc, char **argv) { + if (*index + 1 >= argc) { + fprintf(stderr, "gpu-iq2-moe-prefill-bench: %s needs a value\n", + argv[*index]); + exit(2); + } + return argv[++*index]; +} + +static bench_config parse_options(int argc, char **argv) { + bench_config config = { + .samples = DEFAULT_SAMPLES, + .warmups = DEFAULT_WARMUPS, + }; + for (int i = 1; i < argc; i++) { + if (!strcmp(argv[i], "-h") || !strcmp(argv[i], "--help")) { + usage(stdout, argv[0]); + exit(0); + } else if (!strcmp(argv[i], "--samples")) { + const char *option = argv[i]; + config.samples = parse_u32(need_arg(&i, argc, argv), option); + } else if (!strcmp(argv[i], "--warmups")) { + const char *option = argv[i]; + config.warmups = parse_u32(need_arg(&i, argc, argv), option); + } else { + fprintf(stderr, "gpu-iq2-moe-prefill-bench: unknown option: %s\n", + argv[i]); + usage(stderr, argv[0]); + exit(2); + } + } + if (config.samples == 0u) { + fprintf(stderr, "gpu-iq2-moe-prefill-bench: --samples must be nonzero\n"); + exit(2); + } +#if defined(DS4_BENCH_ROCM) + if ((config.samples & 1u) != 0u || (config.warmups & 1u) != 0u) { + fprintf(stderr, + "gpu-iq2-moe-prefill-bench: ROCm --samples and --warmups " + "must be even\n"); + exit(2); + } +#endif + return config; +} + +static uint64_t align_up(uint64_t value, uint64_t alignment) { + return (value + alignment - 1u) / alignment * alignment; +} + +static uint32_t mix32(uint32_t value) { + value ^= value >> 16u; + value *= 0x7feb352du; + value ^= value >> 15u; + value *= 0x846ca68bu; + value ^= value >> 16u; + return value; +} + +static void fill_iq2(block_iq2_xxs *matrix, uint32_t salt) { + const uint32_t blocks_per_row = IN_DIM / QK_K; + for (uint32_t expert = 0; expert < N_TOTAL_EXPERT; expert++) { + for (uint32_t row = 0; row < MID_DIM; row++) { + for (uint32_t block = 0; block < blocks_per_row; block++) { + block_iq2_xxs *b = matrix + + ((uint64_t)expert * MID_DIM + row) * blocks_per_row + block; + const uint32_t key = salt * 977u + expert * 431u + + row * 37u + block * 811u; + b->d = (uint16_t)(0x1800u + ((key & 1u) ? 0x0200u : 0u)); + for (uint32_t i = 0; i < QK_K / 8u; i++) { + b->qs[i] = (uint16_t)(key + i * 509u + + (i >> 2u) * 131u); + } + } + } + } +} + +static void fill_q2(block_q2_K *matrix) { + const uint32_t blocks_per_row = MID_DIM / QK_K; + for (uint32_t expert = 0; expert < N_TOTAL_EXPERT; expert++) { + for (uint32_t row = 0; row < OUT_DIM; row++) { + for (uint32_t block = 0; block < blocks_per_row; block++) { + block_q2_K *b = matrix + + ((uint64_t)expert * OUT_DIM + row) * blocks_per_row + block; + const uint32_t key = expert * 617u + row * 73u + block * 991u; + for (uint32_t group = 0; group < QK_K / 16u; group++) { + const uint8_t scale = + (uint8_t)(1u + (key + 3u * group) % 7u); + const uint8_t min = + (uint8_t)((key / 5u + group) % 4u); + b->scales[group] = + (uint8_t)(scale | (uint8_t)(min << 4u)); + } + for (uint32_t i = 0; i < QK_K / 4u; i++) { + b->qs[i] = (uint8_t)(key + 29u * i + (i >> 1u) * 7u); + } + b->d = 0x1800u; + b->dmin = 0x1400u; + } + } + } +} + +static uint64_t touch_model_pages(const void *model, uint64_t bytes, + uint64_t page) { + const volatile uint8_t *data = model; + uint64_t checksum = 0xcbf29ce484222325ull; + for (uint64_t offset = 0; offset < bytes; offset += page) { + checksum ^= data[offset]; + checksum *= 0x100000001b3ull; + } + checksum ^= data[bytes - 1u]; + return checksum; +} + +/* Every 32-row tail size 1..31 is represented at production N=4096/top-6. */ +static int build_routes(int32_t *selected, float *weights) { + static const uint8_t final_remainders[8] = {1, 2, 3, 4, 5, 6, 7, 4}; + uint32_t target[N_TOTAL_EXPERT]; + uint32_t remaining[N_TOTAL_EXPERT]; + uint32_t actual[N_TOTAL_EXPERT] = {0}; + uint64_t target_sum = 0; + + for (uint32_t expert = 0; expert < N_TOTAL_EXPERT; expert++) { + const uint32_t tail = expert < 248u ? + 1u + (expert * 17u) % 31u : final_remainders[expert - 248u]; + const uint32_t full_tiles = + ((expert * 73u) & 255u) < 131u ? 3u : 2u; + target[expert] = full_tiles * 32u + tail; + remaining[expert] = target[expert]; + target_sum += target[expert]; + } + if (target_sum != (uint64_t)N_TOKENS * N_EXPERT) { + fprintf(stderr, "gpu-iq2-moe-prefill-bench: route target sum=%llu\n", + (unsigned long long)target_sum); + return 0; + } + + for (uint32_t token = 0; token < N_TOKENS; token++) { + uint8_t used[N_TOTAL_EXPERT] = {0}; + float raw_weight[N_EXPERT]; + float weight_sum = 0.0f; + for (uint32_t slot = 0; slot < N_EXPERT; slot++) { + uint32_t best = UINT32_MAX; + uint32_t best_remaining = 0; + uint32_t best_hash = 0; + for (uint32_t expert = 0; expert < N_TOTAL_EXPERT; expert++) { + if (used[expert] || remaining[expert] == 0u) continue; + const uint32_t hash = mix32( + token * 0x9e3779b9u ^ slot * 0x85ebca6bu ^ + expert * 0xc2b2ae35u); + if (best == UINT32_MAX || remaining[expert] > best_remaining || + (remaining[expert] == best_remaining && hash > best_hash)) { + best = expert; + best_remaining = remaining[expert]; + best_hash = hash; + } + } + if (best == UINT32_MAX) { + fprintf(stderr, + "gpu-iq2-moe-prefill-bench: route scheduler exhausted " + "at token=%u slot=%u\n", token, slot); + return 0; + } + const uint64_t route = (uint64_t)token * N_EXPERT + slot; + selected[route] = (int32_t)best; + used[best] = 1u; + remaining[best]--; + actual[best]++; + raw_weight[slot] = 1.0f + (float)(mix32( + token * 0x27d4eb2du ^ slot * 0x165667b1u) % 17u); + weight_sum += raw_weight[slot]; + } + for (uint32_t slot = 0; slot < N_EXPERT; slot++) { + weights[(uint64_t)token * N_EXPERT + slot] = + raw_weight[slot] / weight_sum; + } + } + + bool tails_seen[32] = {false}; + uint32_t min_count = UINT32_MAX; + uint32_t max_count = 0; + for (uint32_t expert = 0; expert < N_TOTAL_EXPERT; expert++) { + if (remaining[expert] != 0u || actual[expert] != target[expert]) { + fprintf(stderr, + "gpu-iq2-moe-prefill-bench: expert=%u target=%u " + "actual=%u remaining=%u\n", + expert, target[expert], actual[expert], remaining[expert]); + return 0; + } + tails_seen[actual[expert] & 31u] = true; + if (actual[expert] < min_count) min_count = actual[expert]; + if (actual[expert] > max_count) max_count = actual[expert]; + } + for (uint32_t tail = 1; tail < 32u; tail++) { + if (!tails_seen[tail]) { + fprintf(stderr, + "gpu-iq2-moe-prefill-bench: missing tail=%u\n", tail); + return 0; + } + } + if (tails_seen[0]) { + fprintf(stderr, + "gpu-iq2-moe-prefill-bench: unexpected full-only expert\n"); + return 0; + } + fprintf(stderr, + "DS4_GPU_IQ2_MOE_PREFILL_SETUP backend=%s routes=%u experts=%u " + "topk=%u count_min=%u count_max=%u tail_coverage=1..31 " + "unique_per_token=yes\n", + BENCH_BACKEND, N_TOKENS * N_EXPERT, N_TOTAL_EXPERT, N_EXPERT, + min_count, max_count); + return 1; +} + +static void fill_input(float *x) { + for (uint32_t token = 0; token < N_TOKENS; token++) { + for (uint32_t column = 0; column < IN_DIM; column++) { + const uint32_t bits = mix32( + token * 0x9e3779b9u ^ column * 0x85ebca6bu); + const int32_t centered = (int32_t)(bits & 511u) - 256; + x[(uint64_t)token * IN_DIM + column] = + (float)centered / 1024.0f; + } + } +} + +static void make_guard(uint32_t guard[GUARD_WORDS]) { + for (uint32_t i = 0; i < GUARD_WORDS; i++) { + guard[i] = GUARD_BITS ^ (i * 0x9e3779b9u); + } +} + +static int write_guard(ds4_gpu_tensor *tensor, uint64_t offset) { + uint32_t guard[GUARD_WORDS]; + make_guard(guard); + return ds4_gpu_tensor_write(tensor, offset, guard, sizeof(guard)); +} + +static int check_guard(const char *name, const ds4_gpu_tensor *tensor, + uint64_t offset) { + uint32_t expected[GUARD_WORDS]; + uint32_t actual[GUARD_WORDS]; + make_guard(expected); + if (!ds4_gpu_tensor_read(tensor, offset, actual, sizeof(actual))) { + fprintf(stderr, + "DS4_GPU_IQ2_MOE_PREFILL_CANARY backend=%s name=%s " + "result=READ_FAIL\n", BENCH_BACKEND, name); + return 0; + } + for (uint32_t i = 0; i < GUARD_WORDS; i++) { + if (actual[i] != expected[i]) { + fprintf(stderr, + "DS4_GPU_IQ2_MOE_PREFILL_CANARY backend=%s name=%s " + "result=FAIL word=%u expected=0x%08x actual=0x%08x\n", + BENCH_BACKEND, name, i, expected[i], actual[i]); + return 0; + } + } + return 1; +} + +static int poison_outputs(fixture *f) { + int ok = ds4_gpu_tensor_fill_f32( + f->gate, -101.0f, (f->pair_f32_bytes + GUARD_BYTES) / sizeof(float)); + ok = ds4_gpu_tensor_fill_f32( + f->up, -102.0f, + (f->pair_f32_bytes + GUARD_BYTES) / sizeof(float)) && ok; + ok = ds4_gpu_tensor_fill_f32( + f->mid, -103.0f, + (f->pair_f32_bytes + GUARD_BYTES) / sizeof(float)) && ok; + ok = ds4_gpu_tensor_fill_f32( + f->experts, -104.0f, + (f->expert_bytes + GUARD_BYTES) / sizeof(float)) && ok; + ok = ds4_gpu_tensor_fill_f32( + f->out, -105.0f, + (f->out_bytes + GUARD_BYTES) / sizeof(float)) && ok; + ok = write_guard(f->gate, f->pair_f32_bytes) && ok; + ok = write_guard(f->up, f->pair_f32_bytes) && ok; + ok = write_guard(f->mid, f->pair_f32_bytes) && ok; + ok = write_guard(f->experts, f->expert_bytes) && ok; + ok = write_guard(f->out, f->out_bytes) && ok; + return ok; +} + +static int check_all_canaries(const fixture *f) { + int ok = check_guard("x", f->x, f->x_bytes); + ok = check_guard("selected", f->selected, f->route_i32_bytes) && ok; + ok = check_guard("weights", f->weights, f->route_f32_bytes) && ok; + ok = check_guard("gate", f->gate, f->pair_f32_bytes) && ok; + ok = check_guard("up", f->up, f->pair_f32_bytes) && ok; + ok = check_guard("mid-f32", f->mid, f->pair_f32_bytes) && ok; + ok = check_guard("experts-f32", f->experts, f->expert_bytes) && ok; + ok = check_guard("out-f32", f->out, f->out_bytes) && ok; + return ok; +} + +static const char *variant_name(bench_arm arm) { + switch (arm) { + case ARM_BASELINE: return "baseline"; + case ARM_CANDIDATE: return "candidate"; + case ARM_CURRENT: return "current"; + } + return "invalid"; +} + +static int select_variant(bench_arm arm) { +#if defined(DS4_BENCH_ROCM) + if (arm != ARM_BASELINE && arm != ARM_CANDIDATE) return 0; + if (unsetenv(TAIL_ENABLE_ENV) != 0 || unsetenv(TAIL_DISABLE_ENV) != 0) { + return 0; + } + return setenv(arm == ARM_BASELINE ? TAIL_DISABLE_ENV : TAIL_ENABLE_ENV, + "1", 1) == 0; +#else + return arm == ARM_CURRENT; +#endif +} + +static int run_once(fixture *f, bench_arm arm, + const char *phase, const char *order, + uint32_t sample, uint32_t cycle, uint32_t position, + bool poison, bool check_canaries) { + if (!select_variant(arm)) { + fprintf(stderr, + "gpu-iq2-moe-prefill-bench: backend policy setup failed\n"); + return 0; + } + if (poison && !poison_outputs(f)) { + fprintf(stderr, "gpu-iq2-moe-prefill-bench: output poison failed\n"); + return 0; + } + + fprintf(stderr, + "DS4_GPU_IQ2_MOE_PREFILL_BENCH backend=%s phase=%s " + "variant=%s sample=%u cycle=%u position=%u order=%s " + "force_resident=1 ssd_streaming=0\n", + BENCH_BACKEND, phase, variant_name(arm), sample, + cycle, position, order); + fflush(stderr); + + if (!ds4_gpu_begin_commands()) { + fprintf(stderr, "gpu-iq2-moe-prefill-bench: begin commands failed\n"); + return 0; + } +#if defined(DS4_BENCH_CUDA) + const uint64_t profile_reports_before = + ds4_cuda_test_moe_fast_profile_report_count(); +#endif + bool mid_is_f16 = false; + const int call_ok = ds4_gpu_routed_moe_batch_tensor( + f->out, f->gate, f->up, f->mid, f->experts, + f->model, f->model_size, + f->gate_offset, f->up_offset, f->down_offset, + IQ2_XXS_TYPE, Q2_K_TYPE, + f->gate_expert_bytes, f->gate_row_bytes, + f->down_expert_bytes, f->down_row_bytes, + IN_DIM, MID_DIM, OUT_DIM, + f->selected, f->weights, N_TOTAL_EXPERT, N_EXPERT, CLAMP, f->x, + 0u, N_TOKENS, &mid_is_f16, true); + const int end_ok = ds4_gpu_end_commands(); +#if defined(DS4_BENCH_CUDA) + const uint64_t profile_reports_after = + ds4_cuda_test_moe_fast_profile_report_count(); + const int profile_ok = + profile_reports_after == profile_reports_before + 1u; + if (!profile_ok) { + fprintf(stderr, + "DS4_GPU_IQ2_MOE_PREFILL_BENCH backend=cuda result=FAIL " + "variant=%s fast_profile_reports_before=%llu " + "fast_profile_reports_after=%llu expected_delta=1\n", + variant_name(arm), + (unsigned long long)profile_reports_before, + (unsigned long long)profile_reports_after); + } +#else + const int profile_ok = 1; +#endif + int ok = call_ok && end_ok && profile_ok; + if (!ok) { + fprintf(stderr, + "DS4_GPU_IQ2_MOE_PREFILL_BENCH backend=%s result=FAIL " + "variant=%s call=%d end=%d reported_mid_format=%s\n", + BENCH_BACKEND, variant_name(arm), call_ok, end_ok, + mid_is_f16 ? "f16" : "f32-or-opaque"); + } + if (check_canaries) ok = check_all_canaries(f) && ok; + return ok; +} + +#if defined(DS4_BENCH_ROCM) +static int snapshot_alloc(oracle_snapshot *snapshot, const fixture *f) { + memset(snapshot, 0, sizeof(*snapshot)); + snapshot->bytes = 3u * f->pair_f32_bytes + + f->expert_bytes + f->out_bytes; + if (snapshot->bytes > SIZE_MAX) return 0; + snapshot->storage = malloc((size_t)snapshot->bytes); + if (!snapshot->storage) return 0; + snapshot->gate = snapshot->storage; + snapshot->up = snapshot->gate + f->pair_f32_bytes; + snapshot->mid = snapshot->up + f->pair_f32_bytes; + snapshot->experts = snapshot->mid + f->pair_f32_bytes; + snapshot->out = snapshot->experts + f->expert_bytes; + return 1; +} + +static int capture_snapshot(oracle_snapshot *snapshot, const fixture *f) { + return ds4_gpu_tensor_read( + f->gate, 0, snapshot->gate, f->pair_f32_bytes) && + ds4_gpu_tensor_read( + f->up, 0, snapshot->up, f->pair_f32_bytes) && + ds4_gpu_tensor_read( + f->mid, 0, snapshot->mid, f->pair_f32_bytes) && + ds4_gpu_tensor_read( + f->experts, 0, snapshot->experts, f->expert_bytes) && + ds4_gpu_tensor_read( + f->out, 0, snapshot->out, f->out_bytes); +} + +static int tensor_matches(const char *name, const ds4_gpu_tensor *tensor, + const uint8_t *expected, uint64_t bytes, + uint8_t *scratch) { + uint64_t offset = 0; + while (offset < bytes) { + const size_t chunk = bytes - offset > IO_CHUNK_BYTES ? + IO_CHUNK_BYTES : (size_t)(bytes - offset); + if (!ds4_gpu_tensor_read(tensor, offset, scratch, chunk)) { + fprintf(stderr, + "DS4_GPU_IQ2_MOE_PREFILL_ORACLE backend=rocm " + "candidate=tail-cull tensor=%s result=READ_FAIL " + "offset=%llu\n", name, (unsigned long long)offset); + return 0; + } + if (memcmp(scratch, expected + offset, chunk) != 0) { + size_t mismatch = 0; + while (mismatch < chunk && + scratch[mismatch] == expected[offset + mismatch]) { + mismatch++; + } + fprintf(stderr, + "DS4_GPU_IQ2_MOE_PREFILL_ORACLE backend=rocm " + "candidate=tail-cull tensor=%s result=MISMATCH byte=%llu " + "expected=0x%02x actual=0x%02x\n", + name, (unsigned long long)(offset + mismatch), + expected[offset + mismatch], scratch[mismatch]); + return 0; + } + offset += chunk; + } + return 1; +} + +static int run_correctness(fixture *f) { + oracle_snapshot baseline; + if (!snapshot_alloc(&baseline, f)) { + fprintf(stderr, + "gpu-iq2-moe-prefill-bench: oracle snapshot allocation failed\n"); + return 0; + } + uint8_t *scratch = malloc(IO_CHUNK_BYTES); + int ok = scratch != NULL; + if (ok) { + ok = run_once(f, ARM_BASELINE, "oracle", "BASELINE", + 0u, 0u, 0u, true, true); + } + if (ok) ok = capture_snapshot(&baseline, f); + if (ok) { + ok = run_once(f, ARM_CANDIDATE, "oracle", "CANDIDATE", + 0u, 0u, 0u, true, true); + } + const int gate_ok = ok && tensor_matches( + "gate_scratch", f->gate, baseline.gate, f->pair_f32_bytes, scratch); + const int up_ok = ok && tensor_matches( + "up_scratch", f->up, baseline.up, f->pair_f32_bytes, scratch); + const int mid_ok = ok && tensor_matches( + "mid_scratch", f->mid, baseline.mid, f->pair_f32_bytes, scratch); + const int experts_ok = ok && tensor_matches( + "down_scratch", f->experts, baseline.experts, f->expert_bytes, scratch); + const int out_ok = ok && tensor_matches( + "out_f32", f->out, baseline.out, f->out_bytes, scratch); + ok = ok && gate_ok && up_ok && mid_ok && experts_ok && out_ok; + fprintf(stderr, + "DS4_GPU_IQ2_MOE_PREFILL_ORACLE backend=rocm " + "candidate=tail-cull result=%s gate_scratch=%s up_scratch=%s " + "mid_scratch=%s down_scratch=%s out_f32=%s canaries=%s\n", + ok ? "PASS" : "FAIL", + gate_ok ? "exact" : "mismatch", + up_ok ? "exact" : "mismatch", + mid_ok ? "exact" : "mismatch", + experts_ok ? "exact" : "mismatch", + out_ok ? "exact" : "mismatch", + ok ? "PASS" : "FAIL"); + free(scratch); + free(baseline.storage); + return ok; +} + +static int run_balanced_block(fixture *f, const char *phase, + uint32_t samples_per_arm) { + uint32_t arm_samples[2] = {0, 0}; + const uint32_t cycles = samples_per_arm / 2u; + for (uint32_t cycle = 0; cycle < cycles; cycle++) { + static const bench_arm abba[4] = { + ARM_BASELINE, ARM_CANDIDATE, ARM_CANDIDATE, ARM_BASELINE, + }; + static const bench_arm baab[4] = { + ARM_CANDIDATE, ARM_BASELINE, ARM_BASELINE, ARM_CANDIDATE, + }; + const bench_arm *order = (cycle & 1u) ? baab : abba; + const char *order_name = (cycle & 1u) ? "BAAB" : "ABBA"; + for (uint32_t position = 0; position < 4u; position++) { + const bench_arm arm = order[position]; + if (!run_once(f, arm, phase, order_name, + arm_samples[arm]++, cycle, position, + false, false)) { + return 0; + } + } + } + return arm_samples[ARM_BASELINE] == samples_per_arm && + arm_samples[ARM_CANDIDATE] == samples_per_arm; +} + +static int run_experiment(fixture *f, const bench_config *config) { + if (config->warmups != 0u && + !run_balanced_block(f, "warmup", config->warmups)) { + return 0; + } + if (!run_balanced_block(f, "sample", config->samples)) return 0; + fprintf(stderr, + "DS4_GPU_IQ2_MOE_PREFILL_BENCH backend=rocm phase=complete " + "mode=real-ab samples_per_variant=%u warmups_per_variant=%u " + "result=PASS\n", config->samples, config->warmups); + return 1; +} +#else +static int validate_current_tensor(const char *name, + const ds4_gpu_tensor *tensor, + uint64_t bytes, float poison, + bool dense_f32) { + uint8_t *scratch = malloc(IO_CHUNK_BYTES); + if (!scratch) return 0; + uint64_t hash = 0xcbf29ce484222325ull; + uint64_t changed = 0; + uint64_t unchanged = 0; + uint64_t offset = 0; + int ok = 1; + uint8_t poison_bytes[sizeof(float)]; + memcpy(poison_bytes, &poison, sizeof(poison_bytes)); + while (offset < bytes && ok) { + const size_t chunk = bytes - offset > IO_CHUNK_BYTES ? + IO_CHUNK_BYTES : (size_t)(bytes - offset); + if (!ds4_gpu_tensor_read(tensor, offset, scratch, chunk)) { + ok = 0; + break; + } + if (dense_f32) { + const float *values = (const float *)scratch; + const size_t count = chunk / sizeof(float); + for (size_t i = 0; i < count; i++) { + if (!isfinite(values[i])) { + fprintf(stderr, + "DS4_GPU_IQ2_MOE_PREFILL_ORACLE backend=cuda " + "mode=current-single-run tensor=%s " + "result=NONFINITE element=%llu\n", name, + (unsigned long long)(offset / sizeof(float) + i)); + ok = 0; + break; + } + if (values[i] != poison) { + changed++; + } else { + unchanged++; + } + } + } else { + for (size_t i = 0; i < chunk; i++) { + const uint8_t expected = + poison_bytes[(size_t)((offset + i) % sizeof(float))]; + if (scratch[i] != expected) { + changed++; + } else { + unchanged++; + } + } + } + for (size_t i = 0; i < chunk; i++) { + hash ^= scratch[i]; + hash *= 0x100000001b3ull; + } + offset += chunk; + } + if (ok && (dense_f32 ? unchanged != 0u : changed == 0u)) ok = 0; + fprintf(stderr, + "DS4_GPU_IQ2_MOE_PREFILL_ORACLE backend=cuda " + "mode=current-single-run scope=structural tensor=%s layout=%s " + "result=%s " + "written=%llu unchanged_poison=%llu units=%s hash=0x%016llx\n", + name, dense_f32 ? "dense_f32" : "opaque_scratch", + ok ? "PASS" : "FAIL", + (unsigned long long)changed, (unsigned long long)unchanged, + dense_f32 ? "elements" : "bytes", + (unsigned long long)hash); + free(scratch); + return ok; +} + +static int run_correctness(fixture *f) { + int ok = run_once(f, ARM_CURRENT, "oracle", "CURRENT", + 0u, 0u, 0u, true, true); + const int gate_ok = ok && validate_current_tensor( + "gate_scratch", f->gate, f->pair_f32_bytes, -101.0f, false); + const int up_ok = ok && validate_current_tensor( + "up_scratch", f->up, f->pair_f32_bytes, -102.0f, false); + const int mid_ok = ok && validate_current_tensor( + "mid_scratch", f->mid, f->pair_f32_bytes, -103.0f, false); + const int experts_ok = ok && validate_current_tensor( + "down_f32", f->experts, f->expert_bytes, -104.0f, true); + const int out_ok = ok && validate_current_tensor( + "out_f32", f->out, f->out_bytes, -105.0f, true); + ok = ok && gate_ok && up_ok && mid_ok && experts_ok && out_ok; + fprintf(stderr, + "DS4_GPU_IQ2_MOE_PREFILL_ORACLE backend=cuda " + "mode=current-single-run scope=structural result=%s " + "canaries=%s numerical_ab=NOT_AVAILABLE\n", + ok ? "PASS" : "FAIL", ok ? "PASS" : "FAIL"); + return ok; +} + +static int run_experiment(fixture *f, const bench_config *config) { + for (uint32_t i = 0; i < config->warmups; i++) { + if (!run_once(f, ARM_CURRENT, "warmup", "CURRENT", + i, 0u, 0u, false, false)) { + return 0; + } + } + for (uint32_t i = 0; i < config->samples; i++) { + if (!run_once(f, ARM_CURRENT, "sample", "CURRENT", + i, 0u, 0u, false, false)) { + return 0; + } + } + fprintf(stderr, + "DS4_GPU_IQ2_MOE_PREFILL_BENCH backend=cuda phase=complete " + "mode=measurement-only variant=current samples=%u warmups=%u " + "profiler=%s numerical_ab=NOT_AVAILABLE result=PASS\n", + config->samples, config->warmups, CUDA_PROFILE_ENV); + return 1; +} +#endif + +static void free_tensors(fixture *f) { + ds4_gpu_tensor_free(f->out); + ds4_gpu_tensor_free(f->experts); + ds4_gpu_tensor_free(f->mid); + ds4_gpu_tensor_free(f->up); + ds4_gpu_tensor_free(f->gate); + ds4_gpu_tensor_free(f->weights); + ds4_gpu_tensor_free(f->selected); + ds4_gpu_tensor_free(f->x); + f->out = NULL; + f->experts = NULL; + f->mid = NULL; + f->up = NULL; + f->gate = NULL; + f->weights = NULL; + f->selected = NULL; + f->x = NULL; +} + +static int init_fixture(fixture *f) { + memset(f, 0, sizeof(*f)); + const long page_long = sysconf(_SC_PAGESIZE); + if (page_long <= 0) { + fprintf(stderr, "gpu-iq2-moe-prefill-bench: page size unavailable\n"); + return 0; + } + const uint64_t page = (uint64_t)page_long; + f->gate_row_bytes = + (uint64_t)(IN_DIM / QK_K) * sizeof(block_iq2_xxs); + f->gate_expert_bytes = (uint64_t)MID_DIM * f->gate_row_bytes; + const uint64_t gate_tensor_bytes = + (uint64_t)N_TOTAL_EXPERT * f->gate_expert_bytes; + f->down_row_bytes = + (uint64_t)(MID_DIM / QK_K) * sizeof(block_q2_K); + f->down_expert_bytes = (uint64_t)OUT_DIM * f->down_row_bytes; + const uint64_t down_tensor_bytes = + (uint64_t)N_TOTAL_EXPERT * f->down_expert_bytes; + f->gate_offset = 0; + f->up_offset = align_up(gate_tensor_bytes, page); + f->down_offset = align_up(f->up_offset + gate_tensor_bytes, page); + f->model_size = align_up(f->down_offset + down_tensor_bytes, page); + + f->x_bytes = (uint64_t)N_TOKENS * IN_DIM * sizeof(float); + f->route_count = (uint64_t)N_TOKENS * N_EXPERT; + f->route_i32_bytes = f->route_count * sizeof(int32_t); + f->route_f32_bytes = f->route_count * sizeof(float); + f->pair_count = f->route_count * MID_DIM; + f->pair_f32_bytes = f->pair_count * sizeof(float); + f->expert_count = (uint64_t)N_TOKENS * N_EXPERT * OUT_DIM; + f->expert_bytes = f->expert_count * sizeof(float); + f->out_count = (uint64_t)N_TOKENS * OUT_DIM; + f->out_bytes = f->out_count * sizeof(float); + + if (f->gate_row_bytes != 1056u || + f->gate_expert_bytes != 2162688u || + f->down_row_bytes != 672u || + f->down_expert_bytes != 2752512u) { + fprintf(stderr, + "gpu-iq2-moe-prefill-bench: production layout mismatch " + "gate_row=%llu gate_expert=%llu down_row=%llu " + "down_expert=%llu\n", + (unsigned long long)f->gate_row_bytes, + (unsigned long long)f->gate_expert_bytes, + (unsigned long long)f->down_row_bytes, + (unsigned long long)f->down_expert_bytes); + return 0; + } + + const uint64_t tensor_bytes = + f->x_bytes + f->route_i32_bytes + f->route_f32_bytes + + 3u * (f->pair_f32_bytes + GUARD_BYTES) + + f->expert_bytes + GUARD_BYTES + f->out_bytes + GUARD_BYTES + + 3u * GUARD_BYTES; +#if defined(DS4_BENCH_ROCM) + const uint64_t oracle_bytes = + 3u * f->pair_f32_bytes + f->expert_bytes + + f->out_bytes + IO_CHUNK_BYTES; +#else + const uint64_t oracle_bytes = IO_CHUNK_BYTES; +#endif + const uint64_t setup_host_bytes = + f->x_bytes + f->route_i32_bytes + f->route_f32_bytes; + const uint64_t explicit_peak = + f->model_size + tensor_bytes + oracle_bytes + setup_host_bytes; + fprintf(stderr, + "DS4_GPU_IQ2_MOE_PREFILL_SETUP backend=%s " + "geometry=N%u,d%u,mid%u,out%u,experts%u,top%u " + "model=%.3f_GiB tensors=%.3f_GiB oracle=%.3f_GiB " + "explicit_peak=%.3f_GiB resident=1\n", + BENCH_BACKEND, N_TOKENS, IN_DIM, MID_DIM, OUT_DIM, + N_TOTAL_EXPERT, N_EXPERT, + (double)f->model_size / (double)GIB, + (double)tensor_bytes / (double)GIB, + (double)oracle_bytes / (double)GIB, + (double)explicit_peak / (double)GIB); + if (explicit_peak >= 5u * GIB) { + fprintf(stderr, + "gpu-iq2-moe-prefill-bench: explicit peak exceeds 5 GiB\n"); + return 0; + } + + if (posix_memalign(&f->model, (size_t)page, + (size_t)f->model_size) != 0) { + fprintf(stderr, + "gpu-iq2-moe-prefill-bench: model allocation failed\n"); + return 0; + } + fprintf(stderr, + "DS4_GPU_IQ2_MOE_PREFILL_SETUP backend=%s phase=fill_weights " + "tensor=gate\n", BENCH_BACKEND); + fill_iq2((block_iq2_xxs *)((uint8_t *)f->model + f->gate_offset), 19u); + fprintf(stderr, + "DS4_GPU_IQ2_MOE_PREFILL_SETUP backend=%s phase=fill_weights " + "tensor=up\n", BENCH_BACKEND); + fill_iq2((block_iq2_xxs *)((uint8_t *)f->model + f->up_offset), 47u); + fprintf(stderr, + "DS4_GPU_IQ2_MOE_PREFILL_SETUP backend=%s phase=fill_weights " + "tensor=down\n", BENCH_BACKEND); + fill_q2((block_q2_K *)((uint8_t *)f->model + f->down_offset)); + const uint64_t checksum = touch_model_pages(f->model, f->model_size, page); + fprintf(stderr, + "DS4_GPU_IQ2_MOE_PREFILL_SETUP backend=%s phase=touch_weights " + "pages=%llu checksum=0x%016llx resident=1\n", + BENCH_BACKEND, + (unsigned long long)((f->model_size + page - 1u) / page), + (unsigned long long)checksum); + + float *x_host = malloc((size_t)f->x_bytes); + int32_t *selected_host = malloc((size_t)f->route_i32_bytes); + float *weights_host = malloc((size_t)f->route_f32_bytes); + int ok = x_host && selected_host && weights_host; + if (ok) fill_input(x_host); + if (ok) ok = build_routes(selected_host, weights_host); + if (ok) ok = ds4_gpu_set_model_map(f->model, f->model_size); + + if (ok) f->x = ds4_gpu_tensor_alloc(f->x_bytes + GUARD_BYTES); + if (ok) f->selected = + ds4_gpu_tensor_alloc(f->route_i32_bytes + GUARD_BYTES); + if (ok) f->weights = + ds4_gpu_tensor_alloc(f->route_f32_bytes + GUARD_BYTES); + if (ok) f->gate = + ds4_gpu_tensor_alloc(f->pair_f32_bytes + GUARD_BYTES); + if (ok) f->up = + ds4_gpu_tensor_alloc(f->pair_f32_bytes + GUARD_BYTES); + if (ok) f->mid = + ds4_gpu_tensor_alloc(f->pair_f32_bytes + GUARD_BYTES); + if (ok) f->experts = + ds4_gpu_tensor_alloc(f->expert_bytes + GUARD_BYTES); + if (ok) f->out = ds4_gpu_tensor_alloc(f->out_bytes + GUARD_BYTES); + ok = ok && f->x && f->selected && f->weights && f->gate && f->up && + f->mid && f->experts && f->out; + + if (ok) ok = ds4_gpu_tensor_write(f->x, 0, x_host, f->x_bytes); + if (ok) ok = ds4_gpu_tensor_write( + f->selected, 0, selected_host, f->route_i32_bytes); + if (ok) ok = ds4_gpu_tensor_write( + f->weights, 0, weights_host, f->route_f32_bytes); + if (ok) ok = write_guard(f->x, f->x_bytes); + if (ok) ok = write_guard(f->selected, f->route_i32_bytes); + if (ok) ok = write_guard(f->weights, f->route_f32_bytes); + if (ok) ok = poison_outputs(f); + + free(weights_host); + free(selected_host); + free(x_host); + if (!ok) { + fprintf(stderr, + "gpu-iq2-moe-prefill-bench: fixture initialization failed\n"); + } + return ok; +} + +static int configure_backend(void) { +#if defined(DS4_BENCH_ROCM) + return unsetenv(TAIL_ENABLE_ENV) == 0 && + unsetenv(TAIL_DISABLE_ENV) == 0 && + setenv(ROCM_PROFILE_ENV, "1", 1) == 0; +#else + /* Presence, including value zero, enables the existing CUDA profiler. */ + return setenv(CUDA_PROFILE_ENV, "1", 1) == 0; +#endif +} + +int main(int argc, char **argv) { + const bench_config config = parse_options(argc, argv); + if (!configure_backend()) { + fprintf(stderr, + "gpu-iq2-moe-prefill-bench: backend environment setup failed\n"); + return 1; + } + + if (!ds4_gpu_init()) { + fprintf(stderr, + "gpu-iq2-moe-prefill-bench: %s initialization failed\n", + BENCH_BACKEND); + /* Both GPU backends make cleanup idempotent for partially initialized + * state; do not strand a stream, handle, or allocation on init error. */ + ds4_gpu_cleanup(); + return 1; + } + ds4_gpu_set_quality(false); + ds4_gpu_set_ssd_streaming(false); + + fixture f; + const int fixture_ready = init_fixture(&f); + int ok = fixture_ready; + if (ok) ok = run_correctness(&f); + if (ok) ok = run_experiment(&f, &config); + /* A failing dispatch must not suppress the final overrun check. */ + if (fixture_ready) { + const int canaries_ok = check_all_canaries(&f); + ok = canaries_ok && ok; + } + +#if defined(DS4_BENCH_ROCM) + (void)unsetenv(TAIL_ENABLE_ENV); + (void)unsetenv(TAIL_DISABLE_ENV); + (void)unsetenv(ROCM_PROFILE_ENV); +#endif + free_tensors(&f); + ds4_gpu_cleanup(); + free(f.model); + fprintf(stderr, + "DS4_GPU_IQ2_MOE_PREFILL_BENCH backend=%s result=%s\n", + BENCH_BACKEND, ok ? "PASS" : "FAIL"); + return ok ? 0 : 1; +} From eec016c1513a4a2aee5a1158d00b8a5551642976 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Fri, 28 Aug 2026 10:35:59 +0200 Subject: [PATCH 130/189] perf(metal): cull inactive Q4 prefill SIMDgroups Keep cooperative staging and threadgroup barriers uniform while skipping fragment loads, MMA, and stores for inactive token-row SIMDgroups. Enable the measured Q4_K prefill scopes on Apple M1-M4 with a rollback environment variable, and add a GGUF-free resident kernel benchmark plus boundary coverage. --- ENVIRONMENT_VARIABLES.md | 5 +- Makefile | 10 +- ds4_metal.m | 35 +- metal/dense.metal | 60 +- scripts/environment_variables.tsv | 1 + speed-bench/.gitignore | 1 + speed-bench/README.md | 17 + speed-bench/metal_q4_mm_tail_cull_bench.m | 780 ++++++++++++++++++++++ tests/test_metal_indexer_q4.c | 4 +- 9 files changed, 883 insertions(+), 30 deletions(-) create mode 100644 speed-bench/metal_q4_mm_tail_cull_bench.m diff --git a/ENVIRONMENT_VARIABLES.md b/ENVIRONMENT_VARIABLES.md index 50fb1cbde..05657ecd3 100644 --- a/ENVIRONMENT_VARIABLES.md +++ b/ENVIRONMENT_VARIABLES.md @@ -130,13 +130,13 @@ above, it is an unstable internal diagnostic or tuning interface. The linked sou remains normative for exact eligibility gates, bounds, and architecture-specific defaults. -Inventory totals: **1072 `DS4_*` runtime variables** and +Inventory totals: **1073 `DS4_*` runtime variables** and **6 external runtime variables**. The auxiliary inventories contain **118 test/test-fixture entries** and **19 tool/wrapper entries**.
-Metal (444) +Metal (445) | Variable | Accepted value and default | Effect | Source | | --- | --- | --- | --- | @@ -277,6 +277,7 @@ and **19 tool/wrapper entries**. | `DS4_METAL_DISABLE_Q4_GROUPED_BOUNDARY` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 grouped boundary. | [ds4_metal.m:42149](ds4_metal.m#L42149) | | `DS4_METAL_DISABLE_Q4_GROUPED_EXPERTS` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 grouped experts. | [ds4_metal.m:42130](ds4_metal.m#L42130) | | `DS4_METAL_DISABLE_Q4_MV_CLASSIC` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 MV classic. | [ds4_metal.m:21571](ds4_metal.m#L21571) | +| `DS4_METAL_DISABLE_Q4_PREFILL_TAIL_SIMDGROUP_CULL` | presence rollback; unset: automatic on Apple M1-M4 for Q4_K single-tile prefill N=9..16 and production attn_q_b tails through N=65; any value including 0 disables | Restores the legacy four-SIMDgroup Q4_K prefill kernel on the measured short-prefill scopes. | [ds4_metal.m:22088](ds4_metal.m#L22088) | | `DS4_METAL_DISABLE_Q4_QKV_COMPRESSOR_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 QKV compressor fuse. | [ds4_metal.m:22083](ds4_metal.m#L22083) | | `DS4_METAL_DISABLE_Q4_SELECTED_EXPERT_VIEWS` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 selected expert views. | [ds4.c:21136](ds4.c#L21136) | | `DS4_METAL_DISABLE_Q4_SSD_PREFILL_ATTN_OUT_EXACTN` | value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables | Disables Q4 SSD prefill attn out exactn. | [ds4_metal.m:28095](ds4_metal.m#L28095) | diff --git a/Makefile b/Makefile index 99e7834de..407cd0a6c 100644 --- a/Makefile +++ b/Makefile @@ -81,7 +81,7 @@ test-quantizer-indexer-q4: gguf-tools/deepseek4-quantize tests/test_quantizer_in ./tests/test_quantizer_indexer_q4 ./gguf-tools/deepseek4-quantize ifeq ($(UNAME_S),Darwin) -.PHONY: metal-decode-schedule-bench metal-prefill-variant-bench metal-q4-dense-pair-bench metal-iq2-moe-tail-cull-bench check-mxfp4-half-lut test-mxfp4-metal +.PHONY: metal-decode-schedule-bench metal-prefill-variant-bench metal-q4-dense-pair-bench metal-q4-mm-tail-cull-bench metal-iq2-moe-tail-cull-bench check-mxfp4-half-lut test-mxfp4-metal all: ds4 ds4-server ds4-bench ds4-eval ds4-agent @@ -109,6 +109,7 @@ help: @echo " make metal-decode-schedule-bench Build the balanced Metal decode schedule benchmark" @echo " make metal-prefill-variant-bench Build the balanced Metal prefill variant benchmark" @echo " make metal-q4-dense-pair-bench Build the resident Q4 decode pair kernel benchmark" + @echo " make metal-q4-mm-tail-cull-bench Build the resident Q4 prefill tail-cull kernel benchmark" @echo " make metal-iq2-moe-tail-cull-bench Build the resident IQ2 pair MoE tail-cull benchmark" @echo " make check-mxfp4-half-lut Verify the checked-in MXFP4 half LUT matches the generator" @echo " make test-mxfp4-metal Check the MXFP4 half LUT, then run Metal MXFP4 exactness tests" @@ -317,6 +318,11 @@ speed-bench/metal_q4_dense_pair_bench: speed-bench/metal_q4_dense_pair_bench.m $ metal-q4-dense-pair-bench: speed-bench/metal_q4_dense_pair_bench +speed-bench/metal_q4_mm_tail_cull_bench: speed-bench/metal_q4_mm_tail_cull_bench.m $(METAL_SRCS) + $(CC) $(OBJCFLAGS) -o $@ $< $(METAL_LDLIBS) + +metal-q4-mm-tail-cull-bench: speed-bench/metal_q4_mm_tail_cull_bench + speed-bench/metal_iq2_moe_tail_cull_bench.o: speed-bench/metal_iq2_moe_tail_cull_bench.c ds4_gpu.h $(CC) $(CFLAGS) -I. -c -o $@ $< @@ -928,4 +934,4 @@ mxfp4-dot-test: tests/test_mxfp4_dot.c ./tests/test_mxfp4_dot clean: - rm -f ds4 ds4-server ds4-bench ds4-eval ds4-agent ds4_cpu ds4_native ds4_server_test ds4_test ds4_agent_test gguf-tools/quality-testing/score_official gguf-tools/quality-testing/score_official.o speed-bench/metal_decode_schedule_bench speed-bench/metal_prefill_variant_bench speed-bench/metal_q4_dense_pair_bench speed-bench/metal_iq2_moe_tail_cull_bench speed-bench/gpu_iq2_moe_prefill_bench_rocm speed-bench/gpu_iq2_moe_prefill_bench_cuda speed-bench/*.o tests/test_q4k_dot tests/test_mxfp4_dot tests/test_quantizer_indexer_q4 tests/test_mxfp4_metal tests/test_mxfp4_rocm tests/bench_mxfp4_rocm tests/test_mxfp4_cuda tests/test_rocm_q4_dense_pair tests/test_metal_session_batch tests/test_metal_q4_streams tests/test_metal_indexer_q4 tests/test_metal_q4_attn_exactn tests/test_metal_q4_qb_f16_cache tests/test_metal_exactn_oracle tests/test_metal_dspark_capture tests/test_metal_argmax_top1 tests/test_metal_iq2_midonly tests/test_metal_iq2_ssd_grouped_mm tests/test_metal_iq2_live_index tests/test_glm53_kda tests/test_glm53_kda_rocm tests/test_glm53_vision_engine tests/test_glm53_vision_prompt tests/test_gpu_xdev tests/test_gpu_model_cache tests/test_gpu_lookup_cache_strict tests/test_engine_mgpu_refusal tests/test_engine_mgpu_runtime tests/test_engine_correctness tests/test_sampling tests/test_cuda_session_batch tests/test_cuda_mixed_batch tests/*.o *.o tests/cuda_long_context_smoke tests/cuda_long_context_smoke.o + rm -f ds4 ds4-server ds4-bench ds4-eval ds4-agent ds4_cpu ds4_native ds4_server_test ds4_test ds4_agent_test gguf-tools/quality-testing/score_official gguf-tools/quality-testing/score_official.o speed-bench/metal_decode_schedule_bench speed-bench/metal_prefill_variant_bench speed-bench/metal_q4_dense_pair_bench speed-bench/metal_q4_mm_tail_cull_bench speed-bench/metal_iq2_moe_tail_cull_bench speed-bench/gpu_iq2_moe_prefill_bench_rocm speed-bench/gpu_iq2_moe_prefill_bench_cuda speed-bench/*.o tests/test_q4k_dot tests/test_mxfp4_dot tests/test_quantizer_indexer_q4 tests/test_mxfp4_metal tests/test_mxfp4_rocm tests/bench_mxfp4_rocm tests/test_mxfp4_cuda tests/test_rocm_q4_dense_pair tests/test_metal_session_batch tests/test_metal_q4_streams tests/test_metal_indexer_q4 tests/test_metal_q4_attn_exactn tests/test_metal_q4_qb_f16_cache tests/test_metal_exactn_oracle tests/test_metal_dspark_capture tests/test_metal_argmax_top1 tests/test_metal_iq2_midonly tests/test_metal_iq2_ssd_grouped_mm tests/test_metal_iq2_live_index tests/test_glm53_kda tests/test_glm53_kda_rocm tests/test_glm53_vision_engine tests/test_glm53_vision_prompt tests/test_gpu_xdev tests/test_gpu_model_cache tests/test_gpu_lookup_cache_strict tests/test_engine_mgpu_refusal tests/test_engine_mgpu_runtime tests/test_engine_correctness tests/test_sampling tests/test_cuda_session_batch tests/test_cuda_mixed_batch tests/*.o *.o tests/cuda_long_context_smoke tests/cuda_long_context_smoke.o diff --git a/ds4_metal.m b/ds4_metal.m index 55ccae95e..1c6c2fb4b 100644 --- a/ds4_metal.m +++ b/ds4_metal.m @@ -22050,10 +22050,38 @@ int ds4_gpu_matmul_q8_0_decode_mpp_model_view_tensor( } } -static const char *ds4_gpu_q4_mm_name(uint32_t weight_type) { +static const char *ds4_gpu_q4_mm_name( + uint32_t weight_type, + uint64_t in_dim, + uint64_t out_dim, + uint64_t n_tok) { switch (weight_type) { case DS4_METAL_TENSOR_Q4_0: return "kernel_mul_mm_q4_0_f32"; - case DS4_METAL_TENSOR_Q4_K: return "kernel_mul_mm_q4_K_f32"; + case DS4_METAL_TENSOR_Q4_K: { + /* + * The legacy 64x32 kernel maps token rows 0..15 to SIMDgroups 0/1 + * and rows 16..31 to SIMDgroups 2/3. When the final token tile has + * at most 16 rows, the latter pair can skip its fragment loads, MMA, + * and store while all four SIMDgroups still perform cooperative + * staging and reach every threadgroup barrier. Keep a narrow + * rollback and avoid the candidate when it cannot cull a whole pair. + */ + const uint64_t tail = n_tok % 32u; + const bool measured_platform = + ds4_gpu_device_is_pre_m5_apple_silicon(); + const bool measured_output_geometry = (out_dim % 64u) == 0u; + const bool production_q_b = + in_dim == 1024u && out_dim == 32768u && n_tok <= 65u; + const bool single_tile_prefill = n_tok <= 16u; + const bool tail_cull = + n_tok > 8u && tail > 0u && tail <= 16u && + measured_platform && + measured_output_geometry && + (single_tile_prefill || production_q_b) && + getenv("DS4_METAL_DISABLE_Q4_PREFILL_TAIL_SIMDGROUP_CULL") == NULL; + return tail_cull ? "kernel_mul_mm_q4_K_f32_tail_cull" + : "kernel_mul_mm_q4_K_f32"; + } default: return NULL; } } @@ -22261,7 +22289,8 @@ static int ds4_gpu_matmul_quant_impl_tensor( ds4_gpu_warn_mpp_fallback(); } - const char *mm_fn = ds4_gpu_q4_mm_name(weight_type); + const char *mm_fn = ds4_gpu_q4_mm_name( + weight_type, in_dim, out_dim, n_tok); const bool bc_inp = (in_dim % 32u) != 0; const bool bc_out = (out_dim % 64u) != 0 || (n_tok % 32u) != 0; id pipeline = diff --git a/metal/dense.metal b/metal/dense.metal index 490047181..b9402ec00 100644 --- a/metal/dense.metal +++ b/metal/dense.metal @@ -2076,7 +2076,7 @@ template [[host_name("kernel_mul_mm_q8_0_f32_nax_direct_rhs_n128")]] kernel mul_ // Tiled matrix-matrix kernel used for prompt batches larger than 8. DS4 uses // this to turn prefill into large simdgroup matrix operations; each block_q // contains 16*nl weights. -template +template kernel void kernel_mul_mm( constant ds4_metal_args_mul_mm & args, device const char * src0, @@ -2104,6 +2104,12 @@ kernel void kernel_mul_mm( // if this block is of 64x32 shape or smaller const short nr0 = (args.ne0 - r0 < NR0) ? (args.ne0 - r0) : NR0; const short nr1 = (args.ne1 - r1 < NR1) ? (args.ne1 - r1) : NR1; + // SIMDgroups 0/1 own token rows 0..15 and SIMDgroups 2/3 own rows + // 16..31. Every thread still participates in cooperative A/B staging and + // every threadgroup barrier; on a short final Q4_K tile only the waves with + // valid token rows construct fragments, execute MMA, and store results. + const bool mma_active = + !CULL_TAIL_SIMDGROUPS || 16*(short)(sgitg/2) < nr1; // a thread shouldn't load data outside of the matrix const short lr0 = ((short)tiitg/NL0) < nr0 ? ((short)tiitg/NL0) : nr0 - 1; // 0 .. 63 @@ -2134,8 +2140,10 @@ kernel void kernel_mul_mm( simdgroup_float8x8 mc[8]; - for (short i = 0; i < 8; i++){ - mc[i] = make_filled_simdgroup_matrix(0.f); + if (mma_active) { + for (short i = 0; i < 8; i++){ + mc[i] = make_filled_simdgroup_matrix(0.f); + } } for (int loop_k = 0; loop_k < args.ne00; loop_k += NK) { @@ -2210,27 +2218,29 @@ kernel void kernel_mul_mm( threadgroup const S0 * lsma = (sa + 4*64*(sgitg%2)); threadgroup const S1 * lsmb = (sb + 2*64*(sgitg/2)); - FOR_UNROLL (short ik = 0; ik < NK/8; ik++) { - simdgroup_barrier(mem_flags::mem_none); + if (mma_active) { + FOR_UNROLL (short ik = 0; ik < NK/8; ik++) { + simdgroup_barrier(mem_flags::mem_none); - FOR_UNROLL (short i = 0; i < 4; i++) { - simdgroup_load(ma[i], lsma + 64*i, 8, 0, false); - } + FOR_UNROLL (short i = 0; i < 4; i++) { + simdgroup_load(ma[i], lsma + 64*i, 8, 0, false); + } - simdgroup_barrier(mem_flags::mem_none); + simdgroup_barrier(mem_flags::mem_none); - FOR_UNROLL (short i = 0; i < 2; i++) { - simdgroup_load(mb[i], lsmb + 64*i, 8, 0, false); - } + FOR_UNROLL (short i = 0; i < 2; i++) { + simdgroup_load(mb[i], lsmb + 64*i, 8, 0, false); + } - simdgroup_barrier(mem_flags::mem_none); + simdgroup_barrier(mem_flags::mem_none); - FOR_UNROLL (short i = 0; i < 8; i++){ - simdgroup_multiply_accumulate(mc[i], mb[i/4], ma[i%4], mc[i]); - } + FOR_UNROLL (short i = 0; i < 8; i++){ + simdgroup_multiply_accumulate(mc[i], mb[i/4], ma[i%4], mc[i]); + } - lsma += 8*64; - lsmb += 4*64; + lsma += 8*64; + lsmb += 4*64; + } } } @@ -2240,8 +2250,10 @@ kernel void kernel_mul_mm( (r0 + 32*(sgitg & 1)) + \ (r1 + 16*(sgitg >> 1)) * args.ne0 + im*args.ne1*args.ne0; - for (short i = 0; i < 8; i++) { - simdgroup_store(mc[i], C + 8*(i%4) + 8*args.ne0*(i/4), args.ne0, 0, false); + if (mma_active) { + for (short i = 0; i < 8; i++) { + simdgroup_store(mc[i], C + 8*(i%4) + 8*args.ne0*(i/4), args.ne0, 0, false); + } } } else { // block is smaller than 64x32, we should avoid writing data outside of the matrix @@ -2249,8 +2261,10 @@ kernel void kernel_mul_mm( threadgroup float * temp_str = ((threadgroup float *) shmem) + 32*(sgitg&1) + (16*(sgitg >> 1))*NR0; - for (short i = 0; i < 8; i++) { - simdgroup_store(mc[i], temp_str + 8*(i%4) + 8*NR0*(i/4), NR0, 0, false); + if (mma_active) { + for (short i = 0; i < 8; i++) { + simdgroup_store(mc[i], temp_str + 8*(i%4) + 8*NR0*(i/4), NR0, 0, false); + } } threadgroup_barrier(mem_flags::mem_threadgroup); @@ -2489,6 +2503,7 @@ kernel void kernel_mul_mm_f16_f32_scaled( } typedef decltype(kernel_mul_mm) mul_mm_t; +typedef decltype(kernel_mul_mm) mul_mm_q4_K_tail_cull_t; // Host-visible prefill matmul variants for F16 and Q8_0 weights. template [[host_name("kernel_mul_mm_f16_f32")]] kernel mul_mm_t kernel_mul_mm; @@ -2500,6 +2515,7 @@ template [[host_name("kernel_mul_mm_f16_f16_rhs")]] kernel mul_mm_t kernel_mul_m template [[host_name("kernel_mul_mm_q8_0_f32")]] kernel mul_mm_t kernel_mul_mm; template [[host_name("kernel_mul_mm_q4_0_f32")]] kernel mul_mm_t kernel_mul_mm; template [[host_name("kernel_mul_mm_q4_K_f32")]] kernel mul_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_q4_K_f32_tail_cull")]] kernel mul_mm_q4_K_tail_cull_t kernel_mul_mm; // Q4_K output projection with a pre-materialized F16 RHS. The ordinary F32 // variant performs this same F32-to-F16 conversion every time a 64-row weight // tile revisits the activation matrix; this variant lets the host perform it diff --git a/scripts/environment_variables.tsv b/scripts/environment_variables.tsv index 8ac779a2b..cedb0459d 100644 --- a/scripts/environment_variables.tsv +++ b/scripts/environment_variables.tsv @@ -613,6 +613,7 @@ runtime/metal DS4_METAL_DISABLE_Q4_GROUP8_EXPERT_TABLE presence rollback; unset: runtime/metal DS4_METAL_DISABLE_Q4_GROUPED_BOUNDARY presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 grouped boundary. ds4_metal.m:42149 runtime/metal DS4_METAL_DISABLE_Q4_GROUPED_EXPERTS presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 grouped experts. ds4_metal.m:42130 runtime/metal DS4_METAL_DISABLE_Q4_MV_CLASSIC presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 MV classic. ds4_metal.m:21571 +runtime/metal DS4_METAL_DISABLE_Q4_PREFILL_TAIL_SIMDGROUP_CULL presence rollback; unset: automatic on Apple M1-M4 for Q4_K single-tile prefill N=9..16 and production attn_q_b tails through N=65; any value including 0 disables Restores the legacy four-SIMDgroup Q4_K prefill kernel on the measured short-prefill scopes. ds4_metal.m:22088 runtime/metal DS4_METAL_DISABLE_Q4_QKV_COMPRESSOR_FUSE presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 QKV compressor fuse. ds4_metal.m:22083 runtime/metal DS4_METAL_DISABLE_Q4_SELECTED_EXPERT_VIEWS presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 selected expert views. ds4.c:21136 runtime/metal DS4_METAL_DISABLE_Q4_SSD_PREFILL_ATTN_OUT_EXACTN value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Disables Q4 SSD prefill attn out exactn. ds4_metal.m:28095 diff --git a/speed-bench/.gitignore b/speed-bench/.gitignore index a0f60d58e..f84a32acc 100644 --- a/speed-bench/.gitignore +++ b/speed-bench/.gitignore @@ -4,3 +4,4 @@ local-runs/ metal_decode_schedule_bench metal_prefill_variant_bench metal_q4_dense_pair_bench +metal_q4_mm_tail_cull_bench diff --git a/speed-bench/README.md b/speed-bench/README.md index 460235c16..3f205ccfb 100644 --- a/speed-bench/README.md +++ b/speed-bench/README.md @@ -99,6 +99,23 @@ Environment variables consumed while the engine opens, including the Metal streaming `F_NOCACHE` controls, cannot be compared with `--candidate-env` in this harness and likewise require separate processes. +### Metal Q4_K generic-MM tail cull + +Build and run the GGUF-free kernel-only comparison with: + +``` +make metal-q4-mm-tail-cull-bench +./speed-bench/metal_q4_mm_tail_cull_bench +``` + +The default `4096 -> 1024` shape models a Flash Q-A projection. Use +`--in-dim 1024 --out-dim 32768` to measure `attn_q_b`. Both arms dispatch the +checked-in production Metal kernels with resident rotating Q4_K weights and +GPU timestamps. The harness covers `N=9,16,17,31,33,47,63,65`, alternates +ABBA/BAAB, and requires bit-exact outputs plus intact input/output canaries. +No GGUF access, SSD I/O, upload, readback, or CPU wall time is included in a +measured command buffer. + ### Resident IQ2/Q2 MoE prefill on ROCm and CUDA The backend-neutral fixture uses the production `N=4096`, 256-expert, top-6 diff --git a/speed-bench/metal_q4_mm_tail_cull_bench.m b/speed-bench/metal_q4_mm_tail_cull_bench.m new file mode 100644 index 000000000..361c0afe3 --- /dev/null +++ b/speed-bench/metal_q4_mm_tail_cull_bench.m @@ -0,0 +1,780 @@ +#import +#import + +#include +#include +#include +#include +#include +#include +#include +#include + +enum { + QK_K = 256, + Q4_BLOCK_BYTES = 144, + THREADS_PER_GROUP = 128, + THREADGROUP_MEMORY_BYTES = 8192, + GUARD_BYTES = 256, + MAX_TOKENS = 65, + DEFAULT_IN_DIM = 4096, + DEFAULT_OUT_DIM = 1024, + DEFAULT_SAMPLES = 12, + DEFAULT_WARMUP_DISPATCHES = 16, +}; + +static const uint32_t k_guard = 0x7fc12345u; +static const uint32_t k_baseline_poison = 0x7fc0b001u; +static const uint32_t k_candidate_poison = 0x7fc0c001u; +static const uint32_t k_token_cases[] = { + 9u, 16u, 17u, 31u, 33u, 47u, 63u, 65u, +}; + +typedef struct { + uint16_t d; + uint16_t dmin; + uint8_t scales[12]; + uint8_t qs[QK_K / 2]; +} block_q4_K_host; + +typedef struct { + int32_t ne00; + int32_t ne02; + uint64_t nb01; + uint64_t nb02; + uint64_t nb03; + int32_t ne12; + uint64_t nb10; + uint64_t nb11; + uint64_t nb12; + uint64_t nb13; + int32_t ne0; + int32_t ne1; + int16_t r2; + int16_t r3; +} mul_mm_args; + +_Static_assert(sizeof(block_q4_K_host) == Q4_BLOCK_BYTES, + "Q4_K host fixture must match GGUF/Metal layout"); +_Static_assert(sizeof(mul_mm_args) == 88, + "Metal mul_mm argument ABI changed"); + +typedef struct { + uint32_t in_dim; + uint32_t out_dim; + uint32_t sets; + uint32_t dispatches; + uint32_t warmup_dispatches; + uint32_t samples; + bool sets_explicit; + bool dispatches_explicit; +} bench_config; + +typedef enum { + ARM_BASELINE, + ARM_CANDIDATE, +} bench_arm; + +typedef struct { + __strong id device; + __strong id queue; + __strong id baseline_pipeline; + __strong id candidate_pipeline; + __strong id weights; + __strong id x; + __strong id baseline_out; + __strong id candidate_out; + uint8_t *x_snapshot; + NSUInteger weight_set_bytes; + NSUInteger output_stride; + NSUInteger x_offset; + uint32_t in_dim; + uint32_t out_dim; + uint32_t sets; +} fixture; + +static void usage(FILE *fp, const char *argv0) { + fprintf(fp, + "usage: %s [options]\n" + "\n" + "Kernel-only A/B of the production Metal Q4_K generic-MM tail " + "SIMDgroup cull.\n" + "\n" + " --in-dim N K dimension (default: %u)\n" + " --out-dim N M dimension (default: %u)\n" + " --sets N rotating resident weight sets (default: auto)\n" + " --dispatches N dispatches per sample (default: auto)\n" + " --warmup N dispatches per warmup arm (default: %u)\n" + " --samples N samples per arm, multiple of 4 (default: %u)\n" + " -h, --help show this help\n" + "\n" + "The default shape is the Flash Q-A projection 4096->1024. " + "Use --in-dim 1024 --out-dim 32768 for attn_q_b.\n" + "Set DS4_SOURCE_ROOT when running outside the repository root.\n", + argv0, DEFAULT_IN_DIM, DEFAULT_OUT_DIM, + DEFAULT_WARMUP_DISPATCHES, DEFAULT_SAMPLES); +} + +static uint32_t parse_u32(const char *text, const char *option, + uint32_t minimum) { + char *end = NULL; + errno = 0; + const unsigned long long value = strtoull(text, &end, 10); + if (errno != 0 || !text[0] || !end || *end || + value < minimum || value > UINT32_MAX) { + fprintf(stderr, "metal-q4-mm-tail-cull-bench: invalid %s: %s\n", + option, text); + exit(2); + } + return (uint32_t)value; +} + +static const char *need_arg(int *index, int argc, char **argv) { + if (*index + 1 >= argc) { + fprintf(stderr, "metal-q4-mm-tail-cull-bench: %s needs a value\n", + argv[*index]); + exit(2); + } + return argv[++*index]; +} + +static bench_config parse_options(int argc, char **argv) { + bench_config config = { + .in_dim = DEFAULT_IN_DIM, + .out_dim = DEFAULT_OUT_DIM, + .sets = 0u, + .dispatches = 0u, + .warmup_dispatches = DEFAULT_WARMUP_DISPATCHES, + .samples = DEFAULT_SAMPLES, + }; + for (int i = 1; i < argc; i++) { + if (!strcmp(argv[i], "-h") || !strcmp(argv[i], "--help")) { + usage(stdout, argv[0]); + exit(0); + } else if (!strcmp(argv[i], "--in-dim")) { + const char *option = argv[i]; + config.in_dim = + parse_u32(need_arg(&i, argc, argv), option, QK_K); + } else if (!strcmp(argv[i], "--out-dim")) { + const char *option = argv[i]; + config.out_dim = + parse_u32(need_arg(&i, argc, argv), option, 64u); + } else if (!strcmp(argv[i], "--sets")) { + const char *option = argv[i]; + config.sets = parse_u32(need_arg(&i, argc, argv), option, 1u); + config.sets_explicit = true; + } else if (!strcmp(argv[i], "--dispatches")) { + const char *option = argv[i]; + config.dispatches = + parse_u32(need_arg(&i, argc, argv), option, 1u); + config.dispatches_explicit = true; + } else if (!strcmp(argv[i], "--warmup")) { + const char *option = argv[i]; + config.warmup_dispatches = + parse_u32(need_arg(&i, argc, argv), option, 0u); + } else if (!strcmp(argv[i], "--samples")) { + const char *option = argv[i]; + config.samples = + parse_u32(need_arg(&i, argc, argv), option, 4u); + } else { + fprintf(stderr, + "metal-q4-mm-tail-cull-bench: unknown option: %s\n", + argv[i]); + usage(stderr, argv[0]); + exit(2); + } + } + if ((config.in_dim % QK_K) != 0u || + (config.out_dim % 64u) != 0u) { + fprintf(stderr, + "metal-q4-mm-tail-cull-bench: in-dim must be divisible by " + "%u and out-dim by 64\n", QK_K); + exit(2); + } + if ((config.samples % 4u) != 0u) { + fprintf(stderr, + "metal-q4-mm-tail-cull-bench: --samples must be divisible " + "by 4 for equal ABBA/BAAB cycles\n"); + exit(2); + } + return config; +} + +static bool checked_mul(NSUInteger a, NSUInteger b, NSUInteger *out) { + if (a != 0u && b > NSUIntegerMax / a) return false; + *out = a * b; + return true; +} + +static NSUInteger align_up(NSUInteger value, NSUInteger alignment) { + return (value + alignment - 1u) / alignment * alignment; +} + +static NSString *metal_prelude(void) { + return @"#include \n" + "using namespace metal;\n" + "#define MAX(x, y) ((x) > (y) ? (x) : (y))\n" + "#define MIN(x, y) ((x) < (y) ? (x) : (y))\n" + "#define SWAP(x, y) { auto tmp = (x); (x) = (y); (y) = tmp; }\n" + "#define QK8_0 32\n" + "#ifndef QK_K\n#define QK_K 256\n#endif\n" + "#define N_SIMDWIDTH 32\n" + "#define N_R0_Q8_0 2\n" + "#define N_SG_Q8_0 4\n" + "#define FC_MUL_MV 600\n" + "#define FC_MUL_MM 700\n" + "#define FC_BIN 1300\n" + "#define FOR_UNROLL(x) _Pragma(\"clang loop unroll(full)\") for (x)\n" + "#define M_PI_F 3.14159265358979323846f\n" + "enum ds4_sort_order { DS4_SORT_ORDER_ASC, DS4_SORT_ORDER_DESC };\n" + "struct block_q8_0 { half d; int8_t qs[QK8_0]; };\n" + "struct block_q8_K { float d; int8_t qs[QK_K]; " + "int16_t bsums[QK_K / 16]; };\n"; +} + +static NSString *load_metal_source(void) { + static const char *paths[] = { + "metal/activations.metal", + "metal/flash_attn.metal", + "metal/dense.metal", + "metal/moe.metal", + "metal/dsv4_hc.metal", + "metal/unary.metal", + "metal/dsv4_kv.metal", + "metal/dsv4_rope.metal", + "metal/dsv4_misc.metal", + "metal/argsort.metal", + "metal/cpy.metal", + "metal/concat.metal", + "metal/get_rows.metal", + "metal/sum_rows.metal", + "metal/softmax.metal", + "metal/repeat.metal", + "metal/glu.metal", + "metal/norm.metal", + "metal/bin.metal", + "metal/set_rows.metal", + }; + const char *root_env = getenv("DS4_SOURCE_ROOT"); + NSString *root = root_env && root_env[0] + ? [NSString stringWithUTF8String:root_env] + : @"."; + NSMutableString *source = [NSMutableString stringWithString:metal_prelude()]; + for (size_t i = 0; i < sizeof(paths) / sizeof(paths[0]); i++) { + NSString *relative = [NSString stringWithUTF8String:paths[i]]; + NSString *path = [root stringByAppendingPathComponent:relative]; + NSError *error = nil; + NSString *part = [NSString stringWithContentsOfFile:path + encoding:NSUTF8StringEncoding + error:&error]; + if (!part) { + fprintf(stderr, + "metal-q4-mm-tail-cull-bench: cannot read %s: %s\n", + [path fileSystemRepresentation], + [[error localizedDescription] UTF8String]); + return nil; + } + [source appendFormat:@"\n// appended %@\n%@\n", relative, part]; + } + return source; +} + +static id make_mm_pipeline( + id device, id library, NSString *name) { + bool bc_inp = false; + bool bc_out = true; + MTLFunctionConstantValues *values = [MTLFunctionConstantValues new]; + [values setConstantValue:&bc_inp type:MTLDataTypeBool atIndex:700]; + [values setConstantValue:&bc_out type:MTLDataTypeBool atIndex:701]; + NSError *error = nil; + id function = [library newFunctionWithName:name + constantValues:values + error:&error]; + if (!function) { + fprintf(stderr, "metal-q4-mm-tail-cull-bench: function %s: %s\n", + [name UTF8String], [[error localizedDescription] UTF8String]); + return nil; + } + id pipeline = + [device newComputePipelineStateWithFunction:function error:&error]; + if (!pipeline) { + fprintf(stderr, "metal-q4-mm-tail-cull-bench: pipeline %s: %s\n", + [name UTF8String], [[error localizedDescription] UTF8String]); + } + return pipeline; +} + +static uint32_t lcg_next(uint32_t *state) { + *state = *state * 1664525u + 1013904223u; + return *state; +} + +static void fill_q4(void *storage, NSUInteger bytes) { + block_q4_K_host *blocks = storage; + const NSUInteger count = bytes / sizeof(*blocks); + uint32_t state = 0x41c64e6du; + for (NSUInteger b = 0; b < count; b++) { + blocks[b].d = (uint16_t)(0x2400u | (lcg_next(&state) & 0x03ffu)); + blocks[b].dmin = + (uint16_t)(0x1c00u | (lcg_next(&state) & 0x03ffu)); + for (size_t i = 0; i < sizeof(blocks[b].scales); i++) { + blocks[b].scales[i] = (uint8_t)lcg_next(&state); + } + for (size_t i = 0; i < sizeof(blocks[b].qs); i++) { + blocks[b].qs[i] = (uint8_t)lcg_next(&state); + } + } +} + +static void fill_guard(id buffer) { + uint32_t *words = buffer.contents; + for (NSUInteger i = 0; i < buffer.length / sizeof(*words); i++) { + words[i] = k_guard; + } +} + +static void fill_output_payload(id buffer, NSUInteger stride, + uint32_t sets, NSUInteger payload_bytes, + uint32_t poison) { + for (uint32_t set = 0; set < sets; set++) { + uint32_t *payload = (uint32_t *)((uint8_t *)buffer.contents + + (NSUInteger)set * stride + GUARD_BYTES); + for (NSUInteger offset = 0; offset < payload_bytes; + offset += sizeof(*payload)) { + payload[offset / sizeof(*payload)] = poison; + } + } +} + +static void fill_activation(float *values, NSUInteger count) { + for (NSUInteger i = 0; i < count; i++) { + const int32_t centered = (int32_t)((i * 37u + i / 11u) % 257u) - 128; + values[i] = (float)centered / 96.0f; + } +} + +static id alloc_buffer(id device, NSUInteger bytes, + NSString *label) { + id buffer = + [device newBufferWithLength:bytes options:MTLResourceStorageModeShared]; + if (!buffer) { + fprintf(stderr, + "metal-q4-mm-tail-cull-bench: allocation failed for %s " + "(%llu bytes)\n", + [label UTF8String], (unsigned long long)bytes); + return nil; + } + buffer.label = label; + return buffer; +} + +static bool init_fixture(fixture *f, bench_config *config) { + f->device = MTLCreateSystemDefaultDevice(); + if (!f->device) { + fprintf(stderr, "metal-q4-mm-tail-cull-bench: no Metal device\n"); + return false; + } + f->queue = [f->device newCommandQueue]; + NSString *source = load_metal_source(); + if (!f->queue || !source) return false; + + NSError *error = nil; + id library = + [f->device newLibraryWithSource:source options:nil error:&error]; + if (!library) { + fprintf(stderr, + "metal-q4-mm-tail-cull-bench: Metal compile failed: %s\n", + [[error localizedDescription] UTF8String]); + return false; + } + f->baseline_pipeline = make_mm_pipeline( + f->device, library, @"kernel_mul_mm_q4_K_f32"); + f->candidate_pipeline = make_mm_pipeline( + f->device, library, @"kernel_mul_mm_q4_K_f32_tail_cull"); + if (!f->baseline_pipeline || !f->candidate_pipeline) return false; + if (f->baseline_pipeline.threadExecutionWidth != 32u || + f->candidate_pipeline.threadExecutionWidth != 32u || + f->baseline_pipeline.maxTotalThreadsPerThreadgroup < THREADS_PER_GROUP || + f->candidate_pipeline.maxTotalThreadsPerThreadgroup < THREADS_PER_GROUP) { + fprintf(stderr, + "metal-q4-mm-tail-cull-bench: unexpected pipeline geometry\n"); + return false; + } + + f->in_dim = config->in_dim; + f->out_dim = config->out_dim; + const NSUInteger row_bytes = + (NSUInteger)(f->in_dim / QK_K) * Q4_BLOCK_BYTES; + if (!checked_mul(row_bytes, f->out_dim, &f->weight_set_bytes)) { + fprintf(stderr, "metal-q4-mm-tail-cull-bench: weight size overflow\n"); + return false; + } + if (!config->sets_explicit) { + const NSUInteger target = 144u * 1024u * 1024u; + NSUInteger sets = (target + f->weight_set_bytes - 1u) / + f->weight_set_bytes; + if (sets < 1u) sets = 1u; + if (sets > 64u) sets = 64u; + config->sets = (uint32_t)sets; + } + if (!config->dispatches_explicit) { + const NSUInteger target = 512u * 1024u * 1024u; + NSUInteger dispatches = + (target + f->weight_set_bytes - 1u) / f->weight_set_bytes; + if (dispatches < 8u) dispatches = 8u; + if (dispatches > 256u) dispatches = 256u; + config->dispatches = (uint32_t)dispatches; + } + f->sets = config->sets; + + NSUInteger max_output_bytes = 0u; + if (!checked_mul((NSUInteger)MAX_TOKENS, f->out_dim, + &max_output_bytes) || + !checked_mul(max_output_bytes, sizeof(float), &max_output_bytes)) { + fprintf(stderr, "metal-q4-mm-tail-cull-bench: output size overflow\n"); + return false; + } + f->output_stride = + align_up(GUARD_BYTES + max_output_bytes + GUARD_BYTES, 256u); + f->x_offset = GUARD_BYTES; + + NSUInteger weight_bytes = 0u; + NSUInteger output_bytes = 0u; + NSUInteger x_payload_bytes = 0u; + if (!checked_mul(f->weight_set_bytes, f->sets, &weight_bytes) || + !checked_mul(f->output_stride, f->sets, &output_bytes) || + !checked_mul((NSUInteger)MAX_TOKENS, f->in_dim, &x_payload_bytes) || + !checked_mul(x_payload_bytes, sizeof(float), &x_payload_bytes)) { + fprintf(stderr, "metal-q4-mm-tail-cull-bench: fixture size overflow\n"); + return false; + } + f->weights = alloc_buffer(f->device, weight_bytes, @"q4-resident-weights"); + f->x = alloc_buffer(f->device, + GUARD_BYTES + x_payload_bytes + GUARD_BYTES, + @"prefill-activation"); + f->baseline_out = + alloc_buffer(f->device, output_bytes, @"q4-baseline-output"); + f->candidate_out = + alloc_buffer(f->device, output_bytes, @"q4-candidate-output"); + if (!f->weights || !f->x || !f->baseline_out || !f->candidate_out) { + return false; + } + + fill_q4(f->weights.contents, f->weights.length); + fill_guard(f->x); + fill_activation((float *)((uint8_t *)f->x.contents + f->x_offset), + (NSUInteger)MAX_TOKENS * f->in_dim); + fill_guard(f->baseline_out); + fill_guard(f->candidate_out); + f->x_snapshot = malloc(f->x.length); + if (!f->x_snapshot) { + fprintf(stderr, + "metal-q4-mm-tail-cull-bench: input snapshot allocation failed\n"); + return false; + } + memcpy(f->x_snapshot, f->x.contents, f->x.length); + return true; +} + +static mul_mm_args make_args(const fixture *f, uint32_t n_tokens) { + const uint64_t row_bytes = + (uint64_t)(f->in_dim / QK_K) * Q4_BLOCK_BYTES; + return (mul_mm_args) { + .ne00 = (int32_t)f->in_dim, + .ne02 = 1, + .nb01 = row_bytes, + .nb02 = row_bytes * f->out_dim, + .nb03 = row_bytes * f->out_dim, + .ne12 = 1, + .nb10 = sizeof(float), + .nb11 = (uint64_t)f->in_dim * sizeof(float), + .nb12 = (uint64_t)f->in_dim * n_tokens * sizeof(float), + .nb13 = (uint64_t)f->in_dim * n_tokens * sizeof(float), + .ne0 = (int32_t)f->out_dim, + .ne1 = (int32_t)n_tokens, + .r2 = 1, + .r3 = 1, + }; +} + +static void encode_dispatch(fixture *f, id encoder, + bench_arm arm, uint32_t n_tokens, + uint32_t set) { + const mul_mm_args args = make_args(f, n_tokens); + id output = arm == ARM_BASELINE + ? f->baseline_out : f->candidate_out; + [encoder setComputePipelineState:arm == ARM_BASELINE + ? f->baseline_pipeline : f->candidate_pipeline]; + [encoder setBytes:&args length:sizeof(args) atIndex:0]; + [encoder setBuffer:f->weights + offset:(NSUInteger)set * f->weight_set_bytes + atIndex:1]; + [encoder setBuffer:f->x offset:f->x_offset atIndex:2]; + [encoder setBuffer:output + offset:(NSUInteger)set * f->output_stride + GUARD_BYTES + atIndex:3]; + [encoder setThreadgroupMemoryLength:THREADGROUP_MEMORY_BYTES atIndex:0]; + [encoder dispatchThreadgroups:MTLSizeMake((n_tokens + 31u) / 32u, + f->out_dim / 64u, 1u) + threadsPerThreadgroup:MTLSizeMake(THREADS_PER_GROUP, 1u, 1u)]; +} + +static bool finish_command_buffer(id cb, const char *label) { + [cb commit]; + [cb waitUntilCompleted]; + if (cb.status == MTLCommandBufferStatusCompleted) return true; + fprintf(stderr, "metal-q4-mm-tail-cull-bench: %s failed: %s\n", + label, cb.error ? [[cb.error localizedDescription] UTF8String] + : "unknown Metal error"); + return false; +} + +static bool check_guard_words(const uint8_t *bytes, NSUInteger begin, + NSUInteger end, const char *label, + uint32_t set, uint32_t n_tokens) { + for (NSUInteger offset = begin; offset < end; offset += sizeof(uint32_t)) { + uint32_t actual = 0u; + memcpy(&actual, bytes + offset, sizeof(actual)); + if (actual != k_guard) { + fprintf(stderr, + "metal-q4-mm-tail-cull-bench: %s canary changed " + "N=%u set=%u offset=%llu\n", + label, n_tokens, set, (unsigned long long)offset); + return false; + } + } + return true; +} + +static bool check_outputs(fixture *f, uint32_t n_tokens) { + const NSUInteger payload_bytes = + (NSUInteger)n_tokens * f->out_dim * sizeof(float); + const uint8_t *baseline = f->baseline_out.contents; + const uint8_t *candidate = f->candidate_out.contents; + for (uint32_t set = 0; set < f->sets; set++) { + const NSUInteger record = (NSUInteger)set * f->output_stride; + const NSUInteger payload = record + GUARD_BYTES; + const NSUInteger payload_end = payload + payload_bytes; + if (memcmp(baseline + payload, candidate + payload, payload_bytes) != 0) { + fprintf(stderr, + "metal-q4-mm-tail-cull-bench: bitwise mismatch " + "N=%u set=%u\n", n_tokens, set); + return false; + } + if (!check_guard_words(baseline, record, payload, + "baseline prefix", set, n_tokens) || + !check_guard_words(candidate, record, payload, + "candidate prefix", set, n_tokens) || + !check_guard_words(baseline, payload_end, + record + f->output_stride, + "baseline suffix", set, n_tokens) || + !check_guard_words(candidate, payload_end, + record + f->output_stride, + "candidate suffix", set, n_tokens)) { + return false; + } + } + if (memcmp(f->x.contents, f->x_snapshot, f->x.length) != 0) { + fprintf(stderr, + "metal-q4-mm-tail-cull-bench: input was modified at N=%u\n", + n_tokens); + return false; + } + return true; +} + +static bool run_oracle_case(fixture *f, uint32_t n_tokens) { + fill_guard(f->baseline_out); + fill_guard(f->candidate_out); + const NSUInteger payload_bytes = + (NSUInteger)n_tokens * f->out_dim * sizeof(float); + fill_output_payload(f->baseline_out, f->output_stride, f->sets, + payload_bytes, k_baseline_poison); + fill_output_payload(f->candidate_out, f->output_stride, f->sets, + payload_bytes, k_candidate_poison); + id cb = [f->queue commandBuffer]; + id encoder = [cb computeCommandEncoder]; + for (uint32_t set = 0; set < f->sets; set++) { + encode_dispatch(f, encoder, ARM_BASELINE, n_tokens, set); + } + for (uint32_t set = 0; set < f->sets; set++) { + encode_dispatch(f, encoder, ARM_CANDIDATE, n_tokens, set); + } + [encoder endEncoding]; + if (!finish_command_buffer(cb, "oracle")) return false; + return check_outputs(f, n_tokens); +} + +static bool run_workload(fixture *f, bench_arm arm, uint32_t n_tokens, + uint32_t dispatches, uint32_t start_set, + double *gpu_seconds) { + id cb = [f->queue commandBuffer]; + id encoder = [cb computeCommandEncoder]; + for (uint32_t i = 0; i < dispatches; i++) { + encode_dispatch(f, encoder, arm, n_tokens, + (start_set + i) % f->sets); + } + [encoder endEncoding]; + if (!finish_command_buffer(cb, arm == ARM_BASELINE + ? "baseline workload" : "candidate workload")) { + return false; + } + const double elapsed = cb.GPUEndTime - cb.GPUStartTime; + if (!(elapsed > 0.0)) { + fprintf(stderr, + "metal-q4-mm-tail-cull-bench: GPU timestamps unavailable\n"); + return false; + } + *gpu_seconds = elapsed; + return true; +} + +static int compare_double(const void *lhs, const void *rhs) { + const double a = *(const double *)lhs; + const double b = *(const double *)rhs; + return (a > b) - (a < b); +} + +static double percentile(const double *sorted, uint32_t count, double p) { + const double position = p * (double)(count - 1u); + const uint32_t lo = (uint32_t)position; + const uint32_t hi = lo + 1u < count ? lo + 1u : lo; + const double fraction = position - (double)lo; + return sorted[lo] + (sorted[hi] - sorted[lo]) * fraction; +} + +static bool run_benchmark_case(fixture *f, const bench_config *config, + uint32_t n_tokens) { + double ignored = 0.0; + if (config->warmup_dispatches != 0u) { + static const bench_arm warm_order[] = { + ARM_BASELINE, ARM_CANDIDATE, + ARM_CANDIDATE, ARM_BASELINE, + }; + for (size_t i = 0; i < sizeof(warm_order) / sizeof(warm_order[0]); i++) { + if (!run_workload(f, warm_order[i], n_tokens, + config->warmup_dispatches, + (uint32_t)(i * config->warmup_dispatches) % f->sets, + &ignored)) { + return false; + } + } + } + + double *baseline = calloc(config->samples, sizeof(*baseline)); + double *candidate = calloc(config->samples, sizeof(*candidate)); + double *baseline_sorted = calloc(config->samples, sizeof(*baseline_sorted)); + double *candidate_sorted = calloc(config->samples, sizeof(*candidate_sorted)); + if (!baseline || !candidate || !baseline_sorted || !candidate_sorted) { + fprintf(stderr, + "metal-q4-mm-tail-cull-bench: sample allocation failed\n"); + free(baseline); + free(candidate); + free(baseline_sorted); + free(candidate_sorted); + return false; + } + + uint32_t baseline_count = 0u; + uint32_t candidate_count = 0u; + bool ok = true; + for (uint32_t cycle = 0; ok && cycle < config->samples / 2u; cycle++) { + static const bench_arm abba[] = { + ARM_BASELINE, ARM_CANDIDATE, + ARM_CANDIDATE, ARM_BASELINE, + }; + static const bench_arm baab[] = { + ARM_CANDIDATE, ARM_BASELINE, + ARM_BASELINE, ARM_CANDIDATE, + }; + const bench_arm *order = (cycle & 1u) ? baab : abba; + for (uint32_t position = 0; ok && position < 4u; position++) { + const bench_arm arm = order[position]; + const uint32_t arm_index = arm == ARM_BASELINE + ? baseline_count : candidate_count; + const uint32_t start_set = + (uint32_t)((uint64_t)arm_index * config->dispatches % f->sets); + double elapsed = 0.0; + ok = run_workload(f, arm, n_tokens, config->dispatches, + start_set, &elapsed); + if (ok && arm == ARM_BASELINE) baseline[baseline_count++] = elapsed; + if (ok && arm == ARM_CANDIDATE) candidate[candidate_count++] = elapsed; + } + } + if (ok && (baseline_count != config->samples || + candidate_count != config->samples)) { + ok = false; + } + if (ok) ok = check_outputs(f, n_tokens); + + if (ok) { + memcpy(baseline_sorted, baseline, + config->samples * sizeof(*baseline_sorted)); + memcpy(candidate_sorted, candidate, + config->samples * sizeof(*candidate_sorted)); + qsort(baseline_sorted, config->samples, + sizeof(*baseline_sorted), compare_double); + qsort(candidate_sorted, config->samples, + sizeof(*candidate_sorted), compare_double); + const double scale = 1.0e6 / config->dispatches; + const double baseline_median = + percentile(baseline_sorted, config->samples, 0.50) * scale; + const double baseline_p95 = + percentile(baseline_sorted, config->samples, 0.95) * scale; + const double candidate_median = + percentile(candidate_sorted, config->samples, 0.50) * scale; + const double candidate_p95 = + percentile(candidate_sorted, config->samples, 0.95) * scale; + double log_speedup = 0.0; + for (uint32_t i = 0; i < config->samples; i++) { + log_speedup += log(baseline[i] / candidate[i]); + } + const double paired_speedup = exp(log_speedup / config->samples); + const double reduction = + (baseline_median - candidate_median) * 100.0 / baseline_median; + printf(" N=%-2u baseline %.3f us [p95 %.3f] " + "candidate %.3f us [p95 %.3f] " + "median %.3fx (%+.2f%%), paired-gmean %.3fx\n", + n_tokens, baseline_median, baseline_p95, + candidate_median, candidate_p95, + baseline_median / candidate_median, reduction, + paired_speedup); + } + + free(baseline); + free(candidate); + free(baseline_sorted); + free(candidate_sorted); + return ok; +} + +int main(int argc, char **argv) { + @autoreleasepool { + bench_config config = parse_options(argc, argv); + fixture f = {0}; + if (!init_fixture(&f, &config)) return 1; + + printf("Metal Q4_K generic-MM tail-cull kernel-only A/B\n"); + printf(" shape: %u -> %u, resident anonymous Q4_K weights\n", + config.in_dim, config.out_dim); + printf(" working set: %.1f MiB across %u rotating sets\n", + (double)f.weights.length / (1024.0 * 1024.0), f.sets); + printf(" design: ABBA/BAAB, %u dispatches/sample, " + "%u samples/arm, GPU timestamps\n", + config.dispatches, config.samples); + for (size_t i = 0; + i < sizeof(k_token_cases) / sizeof(k_token_cases[0]); i++) { + if (!run_oracle_case(&f, k_token_cases[i])) return 1; + if (!run_benchmark_case(&f, &config, k_token_cases[i])) return 1; + } + fprintf(stderr, + "Metal Q4_K MM tail-cull oracle: PASS " + "(N=9,16,17,31,33,47,63,65; bit-exact; " + "distinct payload poison; canaries intact)\n"); + printf(" correctness: bit-exact outputs; input/output canaries intact\n"); + printf(" scope: no GGUF, mmap, model runtime, SSD I/O, uploads, " + "readback, or CPU wall timing in measured command buffers\n"); + free(f.x_snapshot); + return 0; + } +} diff --git a/tests/test_metal_indexer_q4.c b/tests/test_metal_indexer_q4.c index 759a039cd..b45dc4de1 100644 --- a/tests/test_metal_indexer_q4.c +++ b/tests/test_metal_indexer_q4.c @@ -312,7 +312,9 @@ static bool compare_case(const float *actual, const float *reference, } int main(void) { - static const uint32_t token_cases[] = {1u, 8u, 9u, 31u, 32u, 33u}; + static const uint32_t token_cases[] = { + 1u, 8u, 9u, 16u, 17u, 31u, 32u, 33u, + }; const uint64_t row_bytes = (INDEXER_IN_DIM / QK_K) * sizeof(block_q4_K); const uint64_t weight_bytes = (uint64_t)INDEXER_OUT_DIM * row_bytes; From 301b9e34f10bf7f3a36bab5904ec5fde7b2de266 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Fri, 28 Aug 2026 15:11:41 +0200 Subject: [PATCH 131/189] perf(rocm): gate SSD Q4 K1024 tile4 on residency --- rocm/ds4_rocm_q4.cuh | 135 ++++++++++++++-- tests/test_rocm_q4_dense_pair.cpp | 260 +++++++++++++++++++++++++++++- 2 files changed, 380 insertions(+), 15 deletions(-) diff --git a/rocm/ds4_rocm_q4.cuh b/rocm/ds4_rocm_q4.cuh index 3f91dd886..19caecae2 100644 --- a/rocm/ds4_rocm_q4.cuh +++ b/rocm/ds4_rocm_q4.cuh @@ -561,23 +561,103 @@ static int rocm_q4_K_prefill_tile8_required(void) { return getenv("DS4_ROCM_REQUIRE_Q4_PREFILL_TILE8") != NULL; } -static int rocm_q4_K_prefill_k1024_tile4_requested( +enum { + ROCM_Q4_PREFILL_K1024_TILE4_REQUIRED_FAILURE = -1, + ROCM_Q4_PREFILL_K1024_TILE4_FALLBACK = 0, + ROCM_Q4_PREFILL_K1024_TILE4_USE = 1, +}; + +/* Keep this decision independent from the device lookup so the complete + * policy matrix has a hardware-free oracle. Resident execution preserves + * the established automatic default. SSD execution stays opt-in and may + * only select TILE4 after the exact weight range has been found in actual + * device storage; mapped/registered host memory is deliberately insufficient. + * REQUIRE requests the candidate as well as asserting it, while DISABLE is + * authoritative in both modes. */ +static int rocm_q4_K_prefill_k1024_tile4_policy( + int ssd_streaming, + int weight_device_resident, + int ssd_enabled, + int disabled, + int required) { + if (disabled) { + return required ? ROCM_Q4_PREFILL_K1024_TILE4_REQUIRED_FAILURE + : ROCM_Q4_PREFILL_K1024_TILE4_FALLBACK; + } + if (!ssd_streaming) return ROCM_Q4_PREFILL_K1024_TILE4_USE; + if (!ssd_enabled && !required) { + return ROCM_Q4_PREFILL_K1024_TILE4_FALLBACK; + } + if (!weight_device_resident) { + return required ? ROCM_Q4_PREFILL_K1024_TILE4_REQUIRED_FAILURE + : ROCM_Q4_PREFILL_K1024_TILE4_FALLBACK; + } + return ROCM_Q4_PREFILL_K1024_TILE4_USE; +} + +/* Test-only pure-policy entry point. It intentionally performs no HIP call, + * so hosts with a ROCm toolchain but no visible device can still validate the + * SSD default, residency gate, and DISABLE/REQUIRE precedence. */ +extern "C" int ds4_rocm_test_q4_prefill_k1024_tile4_policy( + int ssd_streaming, + int weight_device_resident, + int ssd_enabled, + int disabled, + int required) { + return rocm_q4_K_prefill_k1024_tile4_policy( + ssd_streaming != 0, weight_device_resident != 0, + ssd_enabled != 0, disabled != 0, required != 0); +} + +static int rocm_q4_K_prefill_k1024_tile4_resolve( uint64_t blocks, - uint64_t out_dim) { - /* This is the production attn_q_b shape. Keep SSD streaming on the - * established TILE8 path: only a fully resident model can use the new - * specialization, and the dedicated switch provides a narrow rollback. */ - return !g_ssd_streaming_mode && - blocks == ROCM_Q4_PREFILL_K1024_KBLOCK_TILE && - out_dim == DS4_ROCM_Q4_ATTN_Q_B_OUT_DIM && - rocm_q4_attn_q_b_env_bool( - "DS4_ROCM_DISABLE_Q4_PREFILL_K1024_TILE4") != 1; + uint64_t out_dim, + const void *model_map, + uint64_t weight_offset, + uint64_t weight_bytes, + const char *weight_ptr) { + if (blocks != ROCM_Q4_PREFILL_K1024_KBLOCK_TILE || + out_dim != DS4_ROCM_Q4_ATTN_Q_B_OUT_DIM) { + return ROCM_Q4_PREFILL_K1024_TILE4_FALLBACK; + } + + const int enabled = rocm_q4_attn_q_b_env_bool( + "DS4_ROCM_ENABLE_Q4_PREFILL_K1024_TILE4_SSD") == 1; + const int disabled = rocm_q4_attn_q_b_env_bool( + "DS4_ROCM_DISABLE_Q4_PREFILL_K1024_TILE4") == 1; + const int required = rocm_q4_attn_q_b_env_bool( + "DS4_ROCM_REQUIRE_Q4_PREFILL_K1024_TILE4") == 1; + const char *resident_ptr = g_ssd_streaming_mode + ? rocm_q4_attn_q_b_device_resident_source( + model_map, weight_offset, weight_bytes) + : weight_ptr; + const int weight_device_resident = + resident_ptr != NULL && resident_ptr == weight_ptr; + const int decision = rocm_q4_K_prefill_k1024_tile4_policy( + g_ssd_streaming_mode, weight_device_resident, enabled, disabled, + required); + if (decision == ROCM_Q4_PREFILL_K1024_TILE4_REQUIRED_FAILURE) { + if (disabled) { + fprintf(stderr, + "ds4: required ROCm Q4_K prefill K1024 tile4 is " + "disabled\n"); + } else { + fprintf(stderr, + "ds4: required ROCm Q4_K prefill K1024 tile4 has no " + "device-resident SSD weight range " + "(offset=%llu bytes=%llu)\n", + (unsigned long long)weight_offset, + (unsigned long long)weight_bytes); + } + } + return decision; } static uint64_t g_rocm_q4_prefill_tile8_dense_calls; static uint64_t g_rocm_q4_prefill_tile8_pair_calls; static uint64_t g_rocm_q4_prefill_tile8_attention_batch_calls; static uint64_t g_rocm_q4_prefill_k1024_tile4_calls; +static uint64_t g_rocm_q4_prefill_k1024_tile4_ssd_calls; static uint64_t g_rocm_q4_prefill_tile8_tokens; static int g_rocm_q4_prefill_tile8_report_registered; static pthread_mutex_t g_rocm_q4_prefill_tile8_stats_mutex = @@ -639,16 +719,20 @@ static void rocm_q4_K_prefill_tile8_report(void) { g_rocm_q4_prefill_tile8_attention_batch_calls; const uint64_t k1024_tile4_calls = g_rocm_q4_prefill_k1024_tile4_calls; + const uint64_t k1024_tile4_ssd_calls = + g_rocm_q4_prefill_k1024_tile4_ssd_calls; const uint64_t tokens = g_rocm_q4_prefill_tile8_tokens; pthread_mutex_unlock(&g_rocm_q4_prefill_tile8_stats_mutex); fprintf(stderr, - "ds4: ROCm Q4_K prefill tile8 stats: " + "ds4: ROCm Q4_K tiled-prefill stats: " "dense_calls=%llu pair_calls=%llu attention_batch_calls=%llu " - "k1024_tile4_calls=%llu tokens=%llu\n", + "k1024_tile4_calls=%llu k1024_tile4_ssd_calls=%llu " + "tokens=%llu\n", (unsigned long long)dense_calls, (unsigned long long)pair_calls, (unsigned long long)attention_batch_calls, (unsigned long long)k1024_tile4_calls, + (unsigned long long)k1024_tile4_ssd_calls, (unsigned long long)tokens); } @@ -668,6 +752,9 @@ static void rocm_q4_K_prefill_tile8_note( g_rocm_q4_prefill_tile8_pair_calls += pair_calls; g_rocm_q4_prefill_tile8_attention_batch_calls += attention_batch_calls; g_rocm_q4_prefill_k1024_tile4_calls += k1024_tile4_calls; + if (k1024_tile4_calls && g_ssd_streaming_mode) { + g_rocm_q4_prefill_k1024_tile4_ssd_calls += k1024_tile4_calls; + } g_rocm_q4_prefill_tile8_tokens += tokens; pthread_mutex_unlock(&g_rocm_q4_prefill_tile8_stats_mutex); } @@ -693,6 +780,11 @@ extern "C" int ds4_rocm_matmul_q4_K_tensor( const int prefill_scope = rocm_q4_K_prefill_tile8_scope(n_tok); const int prefill_tile8 = rocm_q4_K_prefill_tile8_requested(); const int prefill_tile8_required = rocm_q4_K_prefill_tile8_required(); + const int k1024_tile4_shape = + blocks == ROCM_Q4_PREFILL_K1024_KBLOCK_TILE && + out_dim == DS4_ROCM_Q4_ATTN_Q_B_OUT_DIM; + const int k1024_tile4_required = rocm_q4_attn_q_b_env_bool( + "DS4_ROCM_REQUIRE_Q4_PREFILL_K1024_TILE4") == 1; if (prefill_scope && prefill_tile8_required && !prefill_tile8) { fprintf(stderr, "ds4: required ROCm Q4_K prefill tile8 is disabled " @@ -700,10 +792,27 @@ extern "C" int ds4_rocm_matmul_q4_K_tensor( (unsigned long long)n_tok); return 0; } + if (prefill_scope && k1024_tile4_shape && k1024_tile4_required && + !prefill_tile8) { + fprintf(stderr, + "ds4: required ROCm Q4_K prefill K1024 tile4 cannot run " + "because tiled prefill is disabled (n_tok=%llu)\n", + (unsigned long long)n_tok); + return 0; + } const char *wptr = cuda_model_range_ptr(model_map, weight_offset, weight_bytes, "q4_K dense"); if (!wptr) return 0; + int k1024_tile4 = ROCM_Q4_PREFILL_K1024_TILE4_FALLBACK; + if (prefill_scope && prefill_tile8) { + k1024_tile4 = rocm_q4_K_prefill_k1024_tile4_resolve( + blocks, out_dim, model_map, weight_offset, weight_bytes, wptr); + if (k1024_tile4 == + ROCM_Q4_PREFILL_K1024_TILE4_REQUIRED_FAILURE) { + return 0; + } + } cuda_block_q8_K *xq = rocm_q4_K_prequant_alloc( n_tok, blocks, "q4_K dense prequant"); if (!xq) return 0; @@ -715,7 +824,7 @@ extern "C" int ds4_rocm_matmul_q4_K_tensor( if (!cuda_ok(cudaGetLastError(), "q4_K dense quantize launch")) return 0; if (prefill_scope && prefill_tile8) { - if (rocm_q4_K_prefill_k1024_tile4_requested(blocks, out_dim)) { + if (k1024_tile4 == ROCM_Q4_PREFILL_K1024_TILE4_USE) { const dim3 tiled_grid( (unsigned)((out_dim - 1u) / ROCM_Q4_PREFILL_K1024_ROWS + 1u), diff --git a/tests/test_rocm_q4_dense_pair.cpp b/tests/test_rocm_q4_dense_pair.cpp index c1ae0e5c2..e5cae4143 100644 --- a/tests/test_rocm_q4_dense_pair.cpp +++ b/tests/test_rocm_q4_dense_pair.cpp @@ -28,8 +28,13 @@ #include #include #include +#include #include +extern "C" int ds4_rocm_test_q4_prefill_k1024_tile4_policy( + int ssd_streaming, int weight_device_resident, int ssd_enabled, + int disabled, int required); + namespace { constexpr uint32_t kQkK = 256u; @@ -66,6 +71,10 @@ constexpr const char *kPrefillRequire = "DS4_ROCM_REQUIRE_Q4_PREFILL_TILE8"; constexpr const char *kPrefillK1024Tile4Disable = "DS4_ROCM_DISABLE_Q4_PREFILL_K1024_TILE4"; +constexpr const char *kPrefillK1024Tile4SsdEnable = + "DS4_ROCM_ENABLE_Q4_PREFILL_K1024_TILE4_SSD"; +constexpr const char *kPrefillK1024Tile4Require = + "DS4_ROCM_REQUIRE_Q4_PREFILL_K1024_TILE4"; constexpr const char *kQbF16Enable = "DS4_ROCM_ENABLE_Q4_ATTN_Q_B_F16_CACHE"; constexpr const char *kQbF16Disable = @@ -702,6 +711,31 @@ bool output_guard_unchanged(const std::vector &values, return mismatches == 0; } +bool output_body_overwritten(const std::vector &values, + const std::vector &sentinel, + size_t logical_count, + const char *label) { + if (values.size() != sentinel.size() || logical_count > values.size()) { + std::fprintf(stderr, "%s: invalid body geometry FAIL\n", label); + return false; + } + uint64_t unchanged = 0; + size_t first = logical_count; + for (size_t i = 0; i < logical_count; i++) { + if (std::memcmp(&values[i], &sentinel[i], sizeof(float)) == 0) { + if (unchanged == 0) first = i; + unchanged++; + } + } + std::fprintf(stderr, "%s: unchanged=%llu/%zu %s\n", + label, (unsigned long long)unchanged, logical_count, + unchanged == 0 ? "PASS" : "FAIL"); + if (unchanged != 0) { + std::fprintf(stderr, " first unwritten output at float %zu\n", first); + } + return unchanged == 0; +} + bool run_prefill_parity_case(const aligned_model &model, uint32_t n_tokens, uint64_t offset, uint32_t out_dim, bool compare_cpu, const char *label, @@ -727,12 +761,18 @@ bool run_prefill_parity_case(const aligned_model &model, uint32_t n_tokens, env_snapshot disable(kPrefillDisable); env_snapshot require(kPrefillRequire); env_snapshot k1024_tile4_disable(kPrefillK1024Tile4Disable); + env_snapshot k1024_tile4_ssd_enable(kPrefillK1024Tile4SsdEnable); + env_snapshot k1024_tile4_require(kPrefillK1024Tile4Require); + const bool require_k1024_tile4 = + in_dim == kTailK && out_dim == kQbOutDim; // The authoritative rollback remains the reference now that the tiled // path is default-on. (void)unsetenv(kPrefillEnable); (void)setenv(kPrefillDisable, "1", 1); (void)unsetenv(kPrefillRequire); + (void)unsetenv(kPrefillK1024Tile4SsdEnable); + (void)unsetenv(kPrefillK1024Tile4Require); const int legacy_rc = ds4_gpu_matmul_quant_tensor( legacy_gpu.ptr, model.data, model.size, offset, kQ4Type, in_dim, out_dim, x_gpu.ptr, n_tokens); @@ -744,6 +784,12 @@ bool run_prefill_parity_case(const aligned_model &model, uint32_t n_tokens, (void)unsetenv(kPrefillDisable); (void)setenv(kPrefillRequire, "1", 1); (void)unsetenv(kPrefillK1024Tile4Disable); + (void)unsetenv(kPrefillK1024Tile4SsdEnable); + if (require_k1024_tile4) { + (void)setenv(kPrefillK1024Tile4Require, "1", 1); + } else { + (void)unsetenv(kPrefillK1024Tile4Require); + } const int candidate_rc = ds4_gpu_matmul_quant_tensor( candidate_gpu.ptr, model.data, model.size, offset, kQ4Type, in_dim, out_dim, x_gpu.ptr, n_tokens); @@ -1005,6 +1051,194 @@ bool run_q_b_f16_null_qhalf_case(const aligned_model &model) { return ok; } +bool run_prefill_k1024_tile4_policy_oracle() { + struct policy_case { + const char *label; + int ssd_streaming; + int device_resident; + int ssd_enabled; + int disabled; + int required; + int expected; + }; + const policy_case cases[] = { + {"resident default remains automatic", 0, 0, 0, 0, 0, 1}, + {"resident rollback", 0, 1, 0, 1, 0, 0}, + {"resident DISABLE dominates REQUIRE", 0, 1, 1, 1, 1, -1}, + {"SSD default stays conservative", 1, 1, 0, 0, 0, 0}, + {"SSD opt-in rejects a nonresident range", 1, 0, 1, 0, 0, 0}, + {"SSD opt-in accepts a resident range", 1, 1, 1, 0, 0, 1}, + {"SSD REQUIRE rejects a nonresident range", 1, 0, 0, 0, 1, -1}, + {"SSD REQUIRE requests a resident range", 1, 1, 0, 0, 1, 1}, + {"SSD DISABLE dominates ENABLE", 1, 1, 1, 1, 0, 0}, + {"SSD DISABLE dominates REQUIRE", 1, 1, 1, 1, 1, -1}, + }; + + bool ok = true; + for (const policy_case &test : cases) { + const int got = ds4_rocm_test_q4_prefill_k1024_tile4_policy( + test.ssd_streaming, test.device_resident, test.ssd_enabled, + test.disabled, test.required); + if (got != test.expected) { + std::fprintf(stderr, + "K1024 TILE4 policy %s: expected=%d got=%d FAIL\n", + test.label, test.expected, got); + ok = false; + } + } + std::fprintf(stderr, + "ROCm Q4 K1024 TILE4 policy oracle: cases=%zu %s\n", + sizeof(cases) / sizeof(cases[0]), ok ? "PASS" : "FAIL"); + return ok; +} + +bool run_prefill_k1024_tile4_ssd_case(const aligned_model &model) { + constexpr uint32_t n_tokens = 9u; + const size_t logical_count = (size_t)n_tokens * kQbOutDim; + const size_t allocation_count = logical_count + kOutputGuardFloats; + const std::vector sentinel = sentinel_values(allocation_count); + std::vector x; + fill_activation(&x, n_tokens, kTailK); + tensor_owner x_gpu(x.size() * sizeof(float)); + tensor_owner tile8_gpu(allocation_count * sizeof(float)); + tensor_owner tile4_gpu(allocation_count * sizeof(float)); + if (!x_gpu.ptr || !tile8_gpu.ptr || !tile4_gpu.ptr || + !write_tensor(x_gpu.ptr, x) || + !write_tensor(tile8_gpu.ptr, sentinel) || + !write_tensor(tile4_gpu.ptr, sentinel)) { + std::fprintf(stderr, "prefill K1024 TILE4 SSD: setup FAIL\n"); + return false; + } + + env_snapshot tile8_disable(kPrefillDisable); + env_snapshot tile8_require(kPrefillRequire); + env_snapshot tile4_enable(kPrefillK1024Tile4SsdEnable); + env_snapshot tile4_disable(kPrefillK1024Tile4Disable); + env_snapshot tile4_require(kPrefillK1024Tile4Require); + env_snapshot cache_limit("DS4_ROCM_STREAM_MODEL_CACHE_GB"); + (void)unsetenv(kPrefillDisable); + (void)setenv(kPrefillRequire, "1", 1); + (void)unsetenv(kPrefillK1024Tile4SsdEnable); + (void)unsetenv(kPrefillK1024Tile4Disable); + (void)unsetenv(kPrefillK1024Tile4Require); + (void)setenv("DS4_ROCM_STREAM_MODEL_CACHE_GB", "1", 1); + + FILE *model_file = std::tmpfile(); + void *ssd_model_map = MAP_FAILED; + bool model_map_switched = false; + bool ok = model_file != nullptr && model.size <= (uint64_t)SIZE_MAX; + if (ok) { + ok = std::fwrite(model.data, 1u, (size_t)model.size, model_file) == + (size_t)model.size && + std::fflush(model_file) == 0; + } + const int model_fd = ok ? fileno(model_file) : -1; + if (ok && model_fd >= 0) { + ssd_model_map = mmap(nullptr, (size_t)model.size, PROT_READ, + MAP_PRIVATE, model_fd, 0); + ok = ssd_model_map != MAP_FAILED; + } else { + ok = false; + } + if (ok) { + ok = ds4_gpu_synchronize() != 0 && + ds4_gpu_set_model_map(ssd_model_map, model.size) != 0; + model_map_switched = ok; + } + if (ok) ok = ds4_gpu_set_model_fd(model_fd) != 0; + + int tile8_rc = 0; + int tile4_rc = 0; + int rejected_rc = 1; + if (ok) { + /* Switching modes releases prior synthetic resident ranges. The + * default call reloads q_b from the file into the normal SSD device + * cache but must retain TILE8 because the new SSD candidate is off. */ + ds4_gpu_set_ssd_streaming(true); + tile8_rc = ds4_gpu_matmul_quant_tensor( + tile8_gpu.ptr, ssd_model_map, model.size, + model.q_b_k1024_offset, + kQ4Type, kTailK, kQbOutDim, x_gpu.ptr, n_tokens); + + /* REQUIRE is also an opt-in. The second call reuses the already + * cached device range and cannot false-green through TILE8. */ + (void)setenv(kPrefillK1024Tile4SsdEnable, "1", 1); + (void)setenv(kPrefillK1024Tile4Require, "1", 1); + tile4_rc = ds4_gpu_matmul_quant_tensor( + tile4_gpu.ptr, ssd_model_map, model.size, + model.q_b_k1024_offset, + kQ4Type, kTailK, kQbOutDim, x_gpu.ptr, n_tokens); + } + + std::vector tile8_host(allocation_count); + std::vector tile4_host(allocation_count); + const bool read_ok = tile8_rc != 0 && tile4_rc != 0 && + read_tensor(tile8_gpu.ptr, &tile8_host) && + read_tensor(tile4_gpu.ptr, &tile4_host); + bool parity_ok = read_ok; + if (read_ok) { + parity_ok = output_body_overwritten( + tile8_host, sentinel, logical_count, + "prefill K1024 TILE8 SSD output body"); + parity_ok = output_body_overwritten( + tile4_host, sentinel, logical_count, + "prefill K1024 TILE4 SSD output body") && + parity_ok; + parity_ok = output_guard_unchanged( + tile8_host, sentinel, logical_count, + "prefill K1024 TILE8 SSD output canary"); + parity_ok = output_guard_unchanged( + tile4_host, sentinel, logical_count, + "prefill K1024 TILE4 SSD output canary") && + parity_ok; + tile8_host.resize(logical_count); + tile4_host.resize(logical_count); + parity_ok = bitwise_equal( + tile4_host, tile8_host, + "prefill K1024 TILE4 SSD vs TILE8 SSD") && + parity_ok; + } + + if (ok) { + (void)setenv(kPrefillK1024Tile4Disable, "1", 1); + if (write_tensor(tile4_gpu.ptr, sentinel)) { + rejected_rc = ds4_gpu_matmul_quant_tensor( + tile4_gpu.ptr, ssd_model_map, model.size, + model.q_b_k1024_offset, kQ4Type, kTailK, kQbOutDim, + x_gpu.ptr, n_tokens); + } + } + const bool rejected_ok = rejected_rc == 0 && + unchanged_after_rejected_call( + tile4_gpu.ptr, sentinel, + "prefill K1024 TILE4 SSD DISABLE+REQUIRE preserves output"); + + /* Even a failed candidate may follow an accepted baseline launch. Drain + * it before the mode transition releases the backing range cache. */ + (void)ds4_gpu_synchronize(); + ds4_gpu_set_ssd_streaming(false); + (void)ds4_gpu_set_model_fd(-1); + bool model_map_restored = !model_map_switched; + if (model_map_switched && + !ds4_gpu_set_model_map(model.data, model.size)) { + std::fprintf(stderr, + "prefill K1024 TILE4 SSD: model-map restore FAIL\n"); + ok = false; + } else if (model_map_switched) { + model_map_restored = true; + } + if (ssd_model_map != MAP_FAILED && model_map_restored) { + (void)munmap(ssd_model_map, (size_t)model.size); + } + if (model_file) std::fclose(model_file); + ok = ok && parity_ok && rejected_ok; + std::fprintf(stderr, + "prefill q_b K1024 TILE4 SSD: tile8=%d tile4=%d " + "rejected=%d %s\n", + tile8_rc, tile4_rc, rejected_rc, ok ? "PASS" : "FAIL"); + return ok; +} + bool run_prefill_gate_guards(const aligned_model &model) { constexpr uint32_t n_tokens = 9u; std::vector x; @@ -1862,6 +2096,7 @@ int main(int argc, char **argv) { bool run_prefill = true; bool run_grouped_decode = true; bool run_prefill_long = false; + bool run_policy_only = false; if (argc == 2 && std::strcmp(argv[1], "--dense") == 0) { run_pair = false; run_prefill = false; @@ -1885,11 +2120,17 @@ int main(int argc, char **argv) { run_pair = false; run_grouped_decode = false; run_prefill_long = true; + } else if (argc == 2 && std::strcmp(argv[1], "--policy") == 0) { + run_dense = false; + run_pair = false; + run_prefill = false; + run_grouped_decode = false; + run_policy_only = true; } else if (argc > 1 && !(argc == 2 && std::strcmp(argv[1], "--all") == 0)) { std::fprintf(stderr, "usage: %s [--all|--dense|--pair|--grouped-decode|" - "--prefill|--prefill-long]\n", + "--prefill|--prefill-long|--policy]\n", argv[0]); return 2; } @@ -1897,6 +2138,9 @@ int main(int argc, char **argv) { env_snapshot prefill_enable(kPrefillEnable); env_snapshot prefill_disable(kPrefillDisable); env_snapshot prefill_require(kPrefillRequire); + env_snapshot tile4_ssd_enable(kPrefillK1024Tile4SsdEnable); + env_snapshot tile4_disable(kPrefillK1024Tile4Disable); + env_snapshot tile4_require(kPrefillK1024Tile4Require); env_snapshot grouped_enable(kGroupedDecodeEnable); env_snapshot grouped_disable(kGroupedDecodeDisable); env_snapshot grouped_require(kGroupedDecodeRequire); @@ -1904,11 +2148,17 @@ int main(int argc, char **argv) { (void)unsetenv(kPrefillEnable); (void)unsetenv(kPrefillDisable); (void)unsetenv(kPrefillRequire); + (void)unsetenv(kPrefillK1024Tile4SsdEnable); + (void)unsetenv(kPrefillK1024Tile4Disable); + (void)unsetenv(kPrefillK1024Tile4Require); (void)unsetenv(kGroupedDecodeEnable); (void)unsetenv(kGroupedDecodeDisable); (void)unsetenv(kGroupedDecodeRequire); (void)unsetenv(kGroupedDecodeStats); + const bool policy_ok = run_prefill_k1024_tile4_policy_oracle(); + if (run_policy_only || !policy_ok) return policy_ok ? 0 : 1; + if (detect_rocm_device() <= 0) { const char *require_device = std::getenv("DS4_TEST_REQUIRE_ROCM_DEVICE"); @@ -1924,7 +2174,7 @@ int main(int argc, char **argv) { } aligned_model model; - bool ok = make_model(&model); + bool ok = policy_ok && make_model(&model); if (!ok) { std::fprintf(stderr, "ROCm Q4 dense/pair/prefill: fixture allocation FAIL\n"); @@ -2043,6 +2293,12 @@ int main(int argc, char **argv) { "prefill stress K=4096 M=33 n_tok=4096"); ok = long_ok && ok; } + /* Run the file-backed SSD oracle last. Switching model maps marks the + * process multi-model and intentionally disables optional caches; no + * later parity case should inherit that conservative policy. */ + const bool prefill_q_b_tile4_ssd_ok = + run_prefill_k1024_tile4_ssd_case(model); + ok = prefill_q_b_tile4_ssd_ok && ok; } // Registered host ranges must be released before their aligned backing From cd25137126073c299bcf0d01c63414d98af99688 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Fri, 28 Aug 2026 15:11:54 +0200 Subject: [PATCH 132/189] bench(rocm): isolate resident Q4 prefill kernels --- Makefile | 14 +- speed-bench/README.md | 36 ++ speed-bench/rocm_q4_prefill_bench.cpp | 899 ++++++++++++++++++++++++++ 3 files changed, 947 insertions(+), 2 deletions(-) create mode 100644 speed-bench/rocm_q4_prefill_bench.cpp diff --git a/Makefile b/Makefile index 407cd0a6c..57e68eff7 100644 --- a/Makefile +++ b/Makefile @@ -69,7 +69,7 @@ DS4_LINK_LIBS ?= $(CUDA_LDLIBS) METAL_LDLIBS := $(LDLIBS) endif -.PHONY: all help clean test test-ssd environment-docs test-quantizer-indexer-q4 test-rocm test-glm53-kda-rocm test-metal-session-batch test-metal-session-batch-ssd test-metal-q4-streams test-metal-indexer-q4 test-metal-q4-attn-exactn test-metal-q4-qb-f16-cache test-metal-q4-qb-f16-cache-timing test-metal-exactn-oracle test-metal-dspark-capture test-metal-argmax-top1 bench-metal-argmax-top1 test-metal-iq2-midonly test-metal-iq2-ssd-grouped-mm test-metal-iq2-live-index test-mxfp4-metal test-mxfp4-cuda test-mxfp4-rocm test-mmq-parity-cuda test-rocm-q4-parity test-rocm-q4-dense test-rocm-q4-pair test-rocm-q4-prefill test-strix-rocm-q4-parity test-strix-rocm-q4-prefill test-strix-rocm-q4-prefill-long test-cuda-session-batch test-cuda-mixed-batch dspark-acceptance dspark-verify-depth rocm-dspark-acceptance rocm-dspark-verify-depth mtp-verify-depth cpu cuda cuda-spark cuda-generic cuda-regression strix-halo rocm cuda-iq2-moe-prefill-bench rocm-iq2-moe-prefill-bench +.PHONY: all help clean test test-ssd environment-docs test-quantizer-indexer-q4 test-rocm test-glm53-kda-rocm test-metal-session-batch test-metal-session-batch-ssd test-metal-q4-streams test-metal-indexer-q4 test-metal-q4-attn-exactn test-metal-q4-qb-f16-cache test-metal-q4-qb-f16-cache-timing test-metal-exactn-oracle test-metal-dspark-capture test-metal-argmax-top1 bench-metal-argmax-top1 test-metal-iq2-midonly test-metal-iq2-ssd-grouped-mm test-metal-iq2-live-index test-mxfp4-metal test-mxfp4-cuda test-mxfp4-rocm test-mmq-parity-cuda test-rocm-q4-parity test-rocm-q4-dense test-rocm-q4-pair test-rocm-q4-prefill test-strix-rocm-q4-parity test-strix-rocm-q4-prefill test-strix-rocm-q4-prefill-long test-cuda-session-batch test-cuda-mixed-batch dspark-acceptance dspark-verify-depth rocm-dspark-acceptance rocm-dspark-verify-depth mtp-verify-depth cpu cuda cuda-spark cuda-generic cuda-regression strix-halo rocm cuda-iq2-moe-prefill-bench rocm-iq2-moe-prefill-bench rocm-q4-prefill-bench gguf-tools/deepseek4-quantize: gguf-tools/deepseek4-quantize.c gguf-tools/quants.c gguf-tools/quants.h $(MAKE) -C gguf-tools deepseek4-quantize @@ -370,6 +370,7 @@ help: @echo " make rocm-dspark-verify-depth Build ROCm and run the DSpark verifier invariant" @echo " make test-mxfp4-rocm Build and run the synthetic ROCm MXFP4 MoE test" @echo " make rocm-iq2-moe-prefill-bench Build the resident ROCm IQ2/Q2 WMMA A/B harness" + @echo " make rocm-q4-prefill-bench Build the resident ROCm Q4 dense/pair/q_b A/B harness" @echo " make cuda-iq2-moe-prefill-bench CUDA_ARCH=sm_N Build the resident CUDA IQ2/Q2 profiling harness" @echo " make test-rocm Core regression suite on ROCm-only hosts" @echo " make cpu Build CPU-only ./ds4, ./ds4-server, ./ds4-bench, ./ds4-eval, and ./ds4-agent" @@ -510,6 +511,15 @@ speed-bench/gpu_iq2_moe_prefill_bench_rocm: speed-bench/gpu_iq2_moe_prefill_benc rocm-iq2-moe-prefill-bench: $(MAKE) --no-print-directory -B speed-bench/gpu_iq2_moe_prefill_bench_rocm ROCM_ARCH="$(ROCM_ARCH)" +speed-bench/rocm_q4_prefill_bench.o: speed-bench/rocm_q4_prefill_bench.cpp ds4_gpu.h + $(HIPCC) $(ROCM_CFLAGS) -DDS4_ROCM_BUILD -std=c++17 -fno-fast-math -I. -c -o $@ $< + +speed-bench/rocm_q4_prefill_bench: speed-bench/rocm_q4_prefill_bench.o ds4_rocm.o ds4_rocm_compat.o ds4_rocm_unavailable.o + $(HIPCC) $(ROCM_CFLAGS) -o $@ $^ $(ROCM_LDLIBS) + +rocm-q4-prefill-bench: + $(MAKE) --no-print-directory -B speed-bench/rocm_q4_prefill_bench ROCM_ARCH="$(ROCM_ARCH)" + speed-bench/gpu_iq2_moe_prefill_bench_cuda.o: speed-bench/gpu_iq2_moe_prefill_bench.c ds4_gpu.h $(CC) $(filter-out -ffast-math,$(CFLAGS)) -std=c11 -DDS4_BENCH_CUDA -I. -c -o $@ $< @@ -934,4 +944,4 @@ mxfp4-dot-test: tests/test_mxfp4_dot.c ./tests/test_mxfp4_dot clean: - rm -f ds4 ds4-server ds4-bench ds4-eval ds4-agent ds4_cpu ds4_native ds4_server_test ds4_test ds4_agent_test gguf-tools/quality-testing/score_official gguf-tools/quality-testing/score_official.o speed-bench/metal_decode_schedule_bench speed-bench/metal_prefill_variant_bench speed-bench/metal_q4_dense_pair_bench speed-bench/metal_q4_mm_tail_cull_bench speed-bench/metal_iq2_moe_tail_cull_bench speed-bench/gpu_iq2_moe_prefill_bench_rocm speed-bench/gpu_iq2_moe_prefill_bench_cuda speed-bench/*.o tests/test_q4k_dot tests/test_mxfp4_dot tests/test_quantizer_indexer_q4 tests/test_mxfp4_metal tests/test_mxfp4_rocm tests/bench_mxfp4_rocm tests/test_mxfp4_cuda tests/test_rocm_q4_dense_pair tests/test_metal_session_batch tests/test_metal_q4_streams tests/test_metal_indexer_q4 tests/test_metal_q4_attn_exactn tests/test_metal_q4_qb_f16_cache tests/test_metal_exactn_oracle tests/test_metal_dspark_capture tests/test_metal_argmax_top1 tests/test_metal_iq2_midonly tests/test_metal_iq2_ssd_grouped_mm tests/test_metal_iq2_live_index tests/test_glm53_kda tests/test_glm53_kda_rocm tests/test_glm53_vision_engine tests/test_glm53_vision_prompt tests/test_gpu_xdev tests/test_gpu_model_cache tests/test_gpu_lookup_cache_strict tests/test_engine_mgpu_refusal tests/test_engine_mgpu_runtime tests/test_engine_correctness tests/test_sampling tests/test_cuda_session_batch tests/test_cuda_mixed_batch tests/*.o *.o tests/cuda_long_context_smoke tests/cuda_long_context_smoke.o + rm -f ds4 ds4-server ds4-bench ds4-eval ds4-agent ds4_cpu ds4_native ds4_server_test ds4_test ds4_agent_test gguf-tools/quality-testing/score_official gguf-tools/quality-testing/score_official.o speed-bench/metal_decode_schedule_bench speed-bench/metal_prefill_variant_bench speed-bench/metal_q4_dense_pair_bench speed-bench/metal_q4_mm_tail_cull_bench speed-bench/metal_iq2_moe_tail_cull_bench speed-bench/gpu_iq2_moe_prefill_bench_rocm speed-bench/gpu_iq2_moe_prefill_bench_cuda speed-bench/rocm_q4_prefill_bench speed-bench/*.o tests/test_q4k_dot tests/test_mxfp4_dot tests/test_quantizer_indexer_q4 tests/test_mxfp4_metal tests/test_mxfp4_rocm tests/bench_mxfp4_rocm tests/test_mxfp4_cuda tests/test_rocm_q4_dense_pair tests/test_metal_session_batch tests/test_metal_q4_streams tests/test_metal_indexer_q4 tests/test_metal_q4_attn_exactn tests/test_metal_q4_qb_f16_cache tests/test_metal_exactn_oracle tests/test_metal_dspark_capture tests/test_metal_argmax_top1 tests/test_metal_iq2_midonly tests/test_metal_iq2_ssd_grouped_mm tests/test_metal_iq2_live_index tests/test_glm53_kda tests/test_glm53_kda_rocm tests/test_glm53_vision_engine tests/test_glm53_vision_prompt tests/test_gpu_xdev tests/test_gpu_model_cache tests/test_gpu_lookup_cache_strict tests/test_engine_mgpu_refusal tests/test_engine_mgpu_runtime tests/test_engine_correctness tests/test_sampling tests/test_cuda_session_batch tests/test_cuda_mixed_batch tests/*.o *.o tests/cuda_long_context_smoke tests/cuda_long_context_smoke.o diff --git a/speed-bench/README.md b/speed-bench/README.md index 3f205ccfb..4cf6a72f2 100644 --- a/speed-bench/README.md +++ b/speed-bench/README.md @@ -116,6 +116,42 @@ ABBA/BAAB, and requires bit-exact outputs plus intact input/output canaries. No GGUF access, SSD I/O, upload, readback, or CPU wall time is included in a measured command buffer. +### Resident ROCm Q4_K prefill + +Build the production-dispatch A/B harness on a ROCm host with: + +``` +make rocm-q4-prefill-bench ROCM_ARCH=gfx1151 +./speed-bench/rocm_q4_prefill_bench +``` + +The fixture copies four rotating sets of synthetic GGUF-layout Q4_K weights +to device memory before warmup and forces SSD streaming off. HIP events then +measure only the activation quantizer and Q4 projection kernels. The three +default comparisons are: + +- `dense`: legacy versus TILE8 at the Flash Q-A `K=4096,M=1024` shape; +- `pair`: two TILE8 calls versus the fused Q-A/KV + `K=4096,M=(1024+512)` path; +- `qb`: TILE8 versus TILE4 at the production `attn_q_b` + `K=1024,M=32768` shape. + +The default token set is `9,17,33,128,512`, covering the first row after each +small token-tile boundary plus medium prefill. Use `--full` for +`9,16,17,31,32,33,128,512,4096`, or select a focused run such as: + +``` +./speed-bench/rocm_q4_prefill_bench \ + --case qb --tokens 9,17,33,128,512,4096 --samples 8 +``` + +Every case rotates identical resident weight sets between arms, alternates +ABBA/BAAB, checks the outputs bit-for-bit, and verifies allocation guards. +Fixture creation, the host-to-device residency copy, warmup, oracle readback, +and environment-gate changes are outside the reported HIP-event intervals. +`candidate_delta_pct` is negative when the candidate is faster; the companion +`speedup_pct` reports the positive speedup convention. + ### Resident IQ2/Q2 MoE prefill on ROCm and CUDA The backend-neutral fixture uses the production `N=4096`, 256-expert, top-6 diff --git a/speed-bench/rocm_q4_prefill_bench.cpp b/speed-bench/rocm_q4_prefill_bench.cpp new file mode 100644 index 000000000..61f7602d6 --- /dev/null +++ b/speed-bench/rocm_q4_prefill_bench.cpp @@ -0,0 +1,899 @@ +// SPDX-License-Identifier: MIT +// Resident, GPU-event-only ROCm Q4_K prefill microbenchmark. + +#include "ds4_gpu.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +constexpr uint32_t kQ4Type = 12u; +constexpr uint32_t kQkK = 256u; +constexpr uint32_t kDenseK = 4096u; +constexpr uint32_t kDenseM = 1024u; +constexpr uint32_t kKvM = 512u; +constexpr uint32_t kQbK = 1024u; +constexpr uint32_t kQbM = 32768u; +constexpr uint32_t kDefaultSets = 4u; +constexpr uint32_t kDefaultSamples = 8u; +constexpr uint32_t kDefaultWarmup = 2u; +constexpr uint32_t kGuardWords = 64u; +constexpr uint64_t kCompareChunk = 4u * 1024u * 1024u; + +constexpr const char *kPrefillEnable = + "DS4_ROCM_ENABLE_Q4_PREFILL_TILE8"; +constexpr const char *kPrefillDisable = + "DS4_ROCM_DISABLE_Q4_PREFILL_TILE8"; +constexpr const char *kPrefillRequire = + "DS4_ROCM_REQUIRE_Q4_PREFILL_TILE8"; +constexpr const char *kK1024Tile4Disable = + "DS4_ROCM_DISABLE_Q4_PREFILL_K1024_TILE4"; +constexpr const char *kK1024Tile4SsdEnable = + "DS4_ROCM_ENABLE_Q4_PREFILL_K1024_TILE4_SSD"; +constexpr const char *kK1024Tile4Require = + "DS4_ROCM_REQUIRE_Q4_PREFILL_K1024_TILE4"; + +struct block_q4_K_host { + uint16_t d; + uint16_t dmin; + uint8_t scales[12]; + uint8_t qs[kQkK / 2u]; +}; + +static_assert(sizeof(block_q4_K_host) == 144u, + "Q4_K fixture must match the raw GGUF layout"); + +enum class bench_case { + all, + dense, + pair, + qb, +}; + +struct config { + bench_case selected = bench_case::all; + std::vector tokens = {9u, 17u, 33u, 128u, 512u}; + uint32_t sets = kDefaultSets; + uint32_t samples = kDefaultSamples; + uint32_t warmup = kDefaultWarmup; +}; + +struct weight_set { + uint64_t dense_offset = 0; + uint64_t kv_offset = 0; + uint64_t qb_offset = 0; +}; + +struct model_fixture { + uint8_t *data = nullptr; + uint64_t size = 0; + uint64_t resident_bytes = 0; + FILE *file = nullptr; + std::vector weights; + std::vector span_offsets; + std::vector span_sizes; + + ~model_fixture() { + if (data && size != 0u) (void)munmap(data, static_cast(size)); + if (file) std::fclose(file); + } + model_fixture() = default; + model_fixture(const model_fixture &) = delete; + model_fixture &operator=(const model_fixture &) = delete; +}; + +struct tensor_owner { + ds4_gpu_tensor *ptr = nullptr; + + explicit tensor_owner(uint64_t bytes) : ptr(ds4_gpu_tensor_alloc(bytes)) {} + ~tensor_owner() { ds4_gpu_tensor_free(ptr); } + tensor_owner(const tensor_owner &) = delete; + tensor_owner &operator=(const tensor_owner &) = delete; +}; + +struct env_snapshot { + const char *name; + bool existed; + std::string value; + + explicit env_snapshot(const char *key) + : name(key), existed(std::getenv(key) != nullptr), + value(existed ? std::getenv(key) : "") {} + ~env_snapshot() { + if (existed) { + (void)setenv(name, value.c_str(), 1); + } else { + (void)unsetenv(name); + } + } + env_snapshot(const env_snapshot &) = delete; + env_snapshot &operator=(const env_snapshot &) = delete; +}; + +struct event_timer { + hipEvent_t begin = nullptr; + hipEvent_t end = nullptr; + + event_timer() { + if (hipEventCreate(&begin) != hipSuccess || + hipEventCreate(&end) != hipSuccess) { + std::fprintf(stderr, + "rocm-q4-prefill-bench: HIP event allocation failed\n"); + std::exit(1); + } + } + ~event_timer() { + if (begin) (void)hipEventDestroy(begin); + if (end) (void)hipEventDestroy(end); + } + + bool measure(const std::function &dispatch, float *milliseconds) { + if (hipEventRecord(begin, 0) != hipSuccess) return false; + if (!dispatch()) return false; + if (hipEventRecord(end, 0) != hipSuccess || + hipEventSynchronize(end) != hipSuccess || + hipEventElapsedTime(milliseconds, begin, end) != hipSuccess) { + return false; + } + return true; + } +}; + +struct arm { + const char *name; + std::function prepare; + std::function dispatch; +}; + +struct stats { + double minimum = 0.0; + double median = 0.0; + double p95 = 0.0; + double mean = 0.0; +}; + +uint64_t align_up(uint64_t value, uint64_t alignment) { + return (value + alignment - 1u) / alignment * alignment; +} + +bool checked_mul(uint64_t a, uint64_t b, uint64_t *out) { + if (a != 0u && b > std::numeric_limits::max() / a) return false; + *out = a * b; + return true; +} + +uint32_t lcg_next(uint32_t *state) { + *state = *state * 1664525u + 1013904223u; + return *state; +} + +void fill_q4(void *storage, uint64_t bytes, uint32_t seed) { + auto *blocks = static_cast(storage); + const uint64_t count = bytes / sizeof(*blocks); + uint32_t state = seed; + for (uint64_t i = 0; i < count; i++) { + // Positive, finite FP16 scales. The payload is deterministic but does + // not need a CPU oracle: the benchmark compares production GPU paths. + blocks[i].d = static_cast(0x2400u + (lcg_next(&state) & 0xffu)); + blocks[i].dmin = + static_cast(0x2000u + (lcg_next(&state) & 0xffu)); + for (uint8_t &v : blocks[i].scales) { + v = static_cast(lcg_next(&state) >> 24u); + } + for (uint8_t &v : blocks[i].qs) { + v = static_cast(lcg_next(&state) >> 24u); + } + } +} + +uint64_t q4_weight_bytes(uint32_t in_dim, uint32_t out_dim) { + return static_cast(out_dim) * (in_dim / kQkK) * + sizeof(block_q4_K_host); +} + +bool make_model(model_fixture *model, uint32_t sets) { + constexpr uint64_t page = 4096u; + const uint64_t dense_bytes = q4_weight_bytes(kDenseK, kDenseM); + const uint64_t kv_bytes = q4_weight_bytes(kDenseK, kKvM); + const uint64_t qb_bytes = q4_weight_bytes(kQbK, kQbM); + + model->weights.resize(sets); + model->span_offsets.reserve(static_cast(sets) * 3u); + model->span_sizes.reserve(static_cast(sets) * 3u); + uint64_t cursor = 0; + auto append = [&](uint64_t bytes) { + const uint64_t offset = align_up(cursor, page); + cursor = offset + bytes; + model->span_offsets.push_back(offset); + model->span_sizes.push_back(bytes); + model->resident_bytes += bytes; + return offset; + }; + for (uint32_t i = 0; i < sets; i++) { + model->weights[i].dense_offset = append(dense_bytes); + model->weights[i].kv_offset = append(kv_bytes); + model->weights[i].qb_offset = append(qb_bytes); + } + model->size = align_up(cursor, page); + void *storage = nullptr; + if (posix_memalign(&storage, static_cast(page), + static_cast(model->size)) != 0) { + return false; + } + auto *staging = static_cast(storage); + std::memset(staging, 0, static_cast(model->size)); + for (uint32_t i = 0; i < sets; i++) { + fill_q4(staging + model->weights[i].dense_offset, dense_bytes, + 0x243f6a88u ^ (i * 0x9e3779b9u)); + fill_q4(staging + model->weights[i].kv_offset, kv_bytes, + 0x85a308d3u ^ (i * 0x7f4a7c15u)); + fill_q4(staging + model->weights[i].qb_offset, qb_bytes, + 0x13198a2eu ^ (i * 0x94d049bbu)); + } + + FILE *file = std::tmpfile(); + if (!file || + std::fwrite(staging, 1u, static_cast(model->size), file) != + static_cast(model->size) || + std::fflush(file) != 0) { + std::free(staging); + if (file) std::fclose(file); + return false; + } + void *mapping = mmap(nullptr, static_cast(model->size), PROT_READ, + MAP_PRIVATE, fileno(file), 0); + std::free(staging); + if (mapping == MAP_FAILED) { + std::fclose(file); + return false; + } + model->data = static_cast(mapping); + model->file = file; + return true; +} + +void fill_activation(std::vector *values, uint32_t n_tokens, + uint32_t in_dim) { + values->resize(static_cast(n_tokens) * in_dim); + for (uint32_t token = 0; token < n_tokens; token++) { + for (uint32_t block = 0; block < in_dim / kQkK; block++) { + float *dst = values->data() + + static_cast(token) * in_dim + block * kQkK; + for (uint32_t i = 0; i < kQkK; i++) { + const int q = static_cast((i * 73u + token * 37u + + block * 19u) % 241u) - 120; + dst[i] = static_cast(q) / 32.0f; + } + dst[0] = ((token + block) & 1u) ? 127.0f / 32.0f + : -127.0f / 32.0f; + } + } +} + +std::vector guard_pattern() { + std::vector guard(kGuardWords); + for (uint32_t i = 0; i < kGuardWords; i++) guard[i] = 0x7fc12000u + i; + return guard; +} + +bool prepare_guard(ds4_gpu_tensor *tensor, uint64_t logical_bytes) { + const std::vector guard = guard_pattern(); + return ds4_gpu_tensor_write(tensor, logical_bytes, guard.data(), + guard.size() * sizeof(guard[0])) != 0; +} + +bool poison_output(ds4_gpu_tensor *tensor, uint64_t logical_bytes, + uint32_t pattern) { + if (!tensor || logical_bytes == 0u || + logical_bytes % sizeof(uint32_t) != 0u) { + return false; + } + const uint64_t chunk_bytes = std::min(kCompareChunk, logical_bytes); + std::vector poison( + static_cast(chunk_bytes / sizeof(uint32_t)), pattern); + for (uint64_t offset = 0; offset < logical_bytes; offset += chunk_bytes) { + const uint64_t count = std::min(chunk_bytes, logical_bytes - offset); + if (!ds4_gpu_tensor_write(tensor, offset, poison.data(), count)) { + return false; + } + } + return prepare_guard(tensor, logical_bytes); +} + +bool check_guard(const ds4_gpu_tensor *tensor, uint64_t logical_bytes, + const char *label) { + const std::vector expected = guard_pattern(); + std::vector got(expected.size()); + if (!ds4_gpu_tensor_read(tensor, logical_bytes, got.data(), + got.size() * sizeof(got[0]))) { + std::fprintf(stderr, "%s: guard read failed\n", label); + return false; + } + if (got != expected) { + const auto mismatch = std::mismatch(got.begin(), got.end(), + expected.begin()); + std::fprintf(stderr, "%s: output guard overwritten at word %zu\n", + label, + static_cast(mismatch.first - got.begin())); + return false; + } + return true; +} + +bool bitwise_equal(const ds4_gpu_tensor *a, const ds4_gpu_tensor *b, + uint64_t bytes, const char *label) { + const uint64_t chunk = std::min(kCompareChunk, bytes); + std::vector lhs(static_cast(chunk)); + std::vector rhs(static_cast(chunk)); + for (uint64_t offset = 0; offset < bytes; offset += chunk) { + const uint64_t count = std::min(chunk, bytes - offset); + if (!ds4_gpu_tensor_read(a, offset, lhs.data(), count) || + !ds4_gpu_tensor_read(b, offset, rhs.data(), count)) { + std::fprintf(stderr, "%s: oracle read failed\n", label); + return false; + } + if (std::memcmp(lhs.data(), rhs.data(), static_cast(count)) != 0) { + uint64_t first = 0; + while (first < count && lhs[static_cast(first)] == + rhs[static_cast(first)]) { + first++; + } + std::fprintf(stderr, + "%s: bitwise mismatch at output byte %llu\n", label, + static_cast(offset + first)); + return false; + } + } + return true; +} + +void select_legacy() { + (void)unsetenv(kPrefillEnable); + (void)setenv(kPrefillDisable, "1", 1); + (void)unsetenv(kPrefillRequire); + (void)unsetenv(kK1024Tile4Disable); + (void)unsetenv(kK1024Tile4SsdEnable); + (void)unsetenv(kK1024Tile4Require); +} + +void select_tile8(bool disable_k1024_tile4) { + (void)unsetenv(kPrefillEnable); + (void)unsetenv(kPrefillDisable); + (void)setenv(kPrefillRequire, "1", 1); + (void)unsetenv(kK1024Tile4SsdEnable); + (void)unsetenv(kK1024Tile4Require); + if (disable_k1024_tile4) { + (void)setenv(kK1024Tile4Disable, "1", 1); + } else { + (void)unsetenv(kK1024Tile4Disable); + } +} + +void select_k1024_tile4() { + select_tile8(false); + (void)setenv(kK1024Tile4Require, "1", 1); +} + +double percentile(std::vector sorted, double fraction) { + std::sort(sorted.begin(), sorted.end()); + if (sorted.empty()) return 0.0; + const double position = fraction * static_cast(sorted.size() - 1u); + const size_t lo = static_cast(std::floor(position)); + const size_t hi = static_cast(std::ceil(position)); + const double alpha = position - static_cast(lo); + return sorted[lo] + (sorted[hi] - sorted[lo]) * alpha; +} + +stats summarize(const std::vector &samples) { + stats out; + out.minimum = *std::min_element(samples.begin(), samples.end()); + out.median = percentile(samples, 0.5); + out.p95 = percentile(samples, 0.95); + for (double sample : samples) out.mean += sample; + out.mean /= static_cast(samples.size()); + return out; +} + +bool benchmark_arms(const char *case_name, uint32_t n_tokens, uint32_t in_dim, + uint32_t out_dim, const config &cfg, const arm &baseline, + const arm &candidate, + const std::function &oracle_prepare, + const std::function &oracle) { + // Validate every rotating weight set and prime reusable Q8_K scratch + // before any timed event. This catches data-dependent path errors without + // admitting readback or comparison work into the HIP-event interval. + for (uint32_t set = 0; set < cfg.sets; set++) { + if (!oracle_prepare()) { + std::fprintf(stderr, + "rocm-q4-prefill-bench: %s oracle poison failed " + "for weight set %u\n", + case_name, set); + return false; + } + baseline.prepare(); + if (!baseline.dispatch(set) || !ds4_gpu_synchronize()) return false; + candidate.prepare(); + if (!candidate.dispatch(set) || !ds4_gpu_synchronize()) return false; + if (!oracle()) { + std::fprintf(stderr, + "rocm-q4-prefill-bench: %s oracle failed for " + "weight set %u\n", + case_name, set); + return false; + } + } + + for (uint32_t i = 0; i < cfg.warmup; i++) { + const uint32_t set = i % cfg.sets; + baseline.prepare(); + if (!baseline.dispatch(set) || !ds4_gpu_synchronize()) return false; + candidate.prepare(); + if (!candidate.dispatch(set) || !ds4_gpu_synchronize()) return false; + } + + event_timer timer; + std::vector a_samples; + std::vector b_samples; + a_samples.reserve(cfg.samples); + b_samples.reserve(cfg.samples); + + auto take = [&](const arm &which, uint32_t set, + std::vector *samples) { + which.prepare(); + float elapsed = 0.0f; + const bool ok = timer.measure( + [&]() { return which.dispatch(set); }, &elapsed); + if (!ok) { + std::fprintf(stderr, + "rocm-q4-prefill-bench: %s/%s timed dispatch failed\n", + case_name, which.name); + return false; + } + samples->push_back(static_cast(elapsed)); + return true; + }; + + // Each cycle contributes two samples per arm. Alternating ABBA/BAAB + // balances first/last position, while both arms see identical weight sets. + for (uint32_t cycle = 0; a_samples.size() < cfg.samples; cycle++) { + const uint32_t set0 = (cycle * 2u) % cfg.sets; + const uint32_t set1 = (cycle * 2u + 1u) % cfg.sets; + if ((cycle & 1u) == 0u) { + if (!take(baseline, set0, &a_samples) || + !take(candidate, set0, &b_samples) || + !take(candidate, set1, &b_samples) || + !take(baseline, set1, &a_samples)) return false; + } else { + if (!take(candidate, set0, &b_samples) || + !take(baseline, set0, &a_samples) || + !take(baseline, set1, &a_samples) || + !take(candidate, set1, &b_samples)) return false; + } + } + + const stats a = summarize(a_samples); + const stats b = summarize(b_samples); + std::vector paired_delta; + paired_delta.reserve(a_samples.size()); + for (size_t i = 0; i < a_samples.size(); i++) { + paired_delta.push_back((b_samples[i] / a_samples[i] - 1.0) * 100.0); + } + const double paired_median = percentile(paired_delta, 0.5); + const double median_delta = (b.median / a.median - 1.0) * 100.0; + const double speedup = (a.median / b.median - 1.0) * 100.0; + const double macs = static_cast(n_tokens) * in_dim * out_dim; + const double a_gmac_s = macs / (a.median * 1.0e6); + const double b_gmac_s = macs / (b.median * 1.0e6); + + std::printf( + "DS4_ROCM_Q4_PREFILL_BENCH case=%s N=%u K=%u M=%u " + "baseline=%s candidate=%s samples=%u sets=%u " + "baseline_ms_p50=%.6f candidate_ms_p50=%.6f " + "baseline_ms_min=%.6f candidate_ms_min=%.6f " + "baseline_ms_p95=%.6f candidate_ms_p95=%.6f " + "baseline_gmac_s=%.3f candidate_gmac_s=%.3f " + "candidate_delta_pct=%.3f paired_delta_pct_p50=%.3f " + "speedup_pct=%.3f\n", + case_name, n_tokens, in_dim, out_dim, baseline.name, candidate.name, + cfg.samples, cfg.sets, a.median, b.median, a.minimum, b.minimum, + a.p95, b.p95, a_gmac_s, b_gmac_s, median_delta, paired_median, + speedup); + std::fflush(stdout); + return true; +} + +bool allocate_io(uint32_t n_tokens, uint32_t in_dim, uint64_t out_elements, + tensor_owner *x, tensor_owner *out_a, tensor_owner *out_b) { + std::vector activation; + fill_activation(&activation, n_tokens, in_dim); + if (!x->ptr || !out_a->ptr || !out_b->ptr || + !ds4_gpu_tensor_write(x->ptr, 0, activation.data(), + activation.size() * sizeof(float))) { + return false; + } + const uint64_t logical_bytes = out_elements * sizeof(float); + return poison_output(out_a->ptr, logical_bytes, 0x7fc10001u) && + poison_output(out_b->ptr, logical_bytes, 0x7fc20002u); +} + +bool run_dense(const model_fixture &model, const config &cfg, + uint32_t n_tokens) { + uint64_t out_elements = 0; + if (!checked_mul(n_tokens, kDenseM, &out_elements)) return false; + const uint64_t logical_bytes = out_elements * sizeof(float); + const uint64_t allocation_bytes = logical_bytes + + kGuardWords * sizeof(uint32_t); + tensor_owner x(static_cast(n_tokens) * kDenseK * sizeof(float)); + tensor_owner legacy(allocation_bytes); + tensor_owner tiled(allocation_bytes); + if (!allocate_io(n_tokens, kDenseK, out_elements, &x, &legacy, &tiled)) { + std::fprintf(stderr, "dense N=%u: tensor setup failed\n", n_tokens); + return false; + } + const arm baseline = { + "legacy", select_legacy, + [&](uint32_t set) { + return ds4_gpu_matmul_quant_tensor( + legacy.ptr, model.data, model.size, + model.weights[set].dense_offset, kQ4Type, kDenseK, + kDenseM, x.ptr, n_tokens) != 0; + }}; + const arm candidate = { + "tile8", []() { select_tile8(false); }, + [&](uint32_t set) { + return ds4_gpu_matmul_quant_tensor( + tiled.ptr, model.data, model.size, + model.weights[set].dense_offset, kQ4Type, kDenseK, + kDenseM, x.ptr, n_tokens) != 0; + }}; + if (!benchmark_arms( + "dense", n_tokens, kDenseK, kDenseM, cfg, baseline, candidate, + [&]() { + return poison_output(legacy.ptr, logical_bytes, 0x7fc10001u) && + poison_output(tiled.ptr, logical_bytes, 0x7fc20002u); + }, + [&]() { + return bitwise_equal(legacy.ptr, tiled.ptr, logical_bytes, + "dense legacy vs tile8") && + check_guard(legacy.ptr, logical_bytes, + "dense legacy oracle") && + check_guard(tiled.ptr, logical_bytes, + "dense tile8 oracle"); + })) return false; + return bitwise_equal(legacy.ptr, tiled.ptr, logical_bytes, + "dense legacy vs tile8") && + check_guard(legacy.ptr, logical_bytes, "dense legacy") && + check_guard(tiled.ptr, logical_bytes, "dense tile8"); +} + +bool run_pair(const model_fixture &model, const config &cfg, + uint32_t n_tokens) { + uint64_t out0_elements = 0, out1_elements = 0; + if (!checked_mul(n_tokens, kDenseM, &out0_elements) || + !checked_mul(n_tokens, kKvM, &out1_elements)) return false; + const uint64_t out0_bytes = out0_elements * sizeof(float); + const uint64_t out1_bytes = out1_elements * sizeof(float); + const uint64_t guard_bytes = kGuardWords * sizeof(uint32_t); + tensor_owner x(static_cast(n_tokens) * kDenseK * sizeof(float)); + tensor_owner separate0(out0_bytes + guard_bytes); + tensor_owner separate1(out1_bytes + guard_bytes); + tensor_owner pair0(out0_bytes + guard_bytes); + tensor_owner pair1(out1_bytes + guard_bytes); + std::vector activation; + fill_activation(&activation, n_tokens, kDenseK); + if (!x.ptr || !separate0.ptr || !separate1.ptr || !pair0.ptr || !pair1.ptr || + !ds4_gpu_tensor_write(x.ptr, 0, activation.data(), + activation.size() * sizeof(float)) || + !prepare_guard(separate0.ptr, out0_bytes) || + !prepare_guard(separate1.ptr, out1_bytes) || + !prepare_guard(pair0.ptr, out0_bytes) || + !prepare_guard(pair1.ptr, out1_bytes)) { + std::fprintf(stderr, "pair N=%u: tensor setup failed\n", n_tokens); + return false; + } + const arm baseline = { + "two_dense_tile8", []() { select_tile8(false); }, + [&](uint32_t set) { + return ds4_gpu_matmul_quant_tensor( + separate0.ptr, model.data, model.size, + model.weights[set].dense_offset, kQ4Type, kDenseK, + kDenseM, x.ptr, n_tokens) != 0 && + ds4_gpu_matmul_quant_tensor( + separate1.ptr, model.data, model.size, + model.weights[set].kv_offset, kQ4Type, kDenseK, kKvM, + x.ptr, n_tokens) != 0; + }}; + const arm candidate = { + "pair_tile8", []() { select_tile8(false); }, + [&](uint32_t set) { + return ds4_gpu_matmul_q4_K_pair_tensor( + pair0.ptr, pair1.ptr, model.data, model.size, + model.weights[set].dense_offset, + model.weights[set].kv_offset, kDenseK, kDenseM, kKvM, + x.ptr, n_tokens) != 0; + }}; + if (!benchmark_arms( + "pair", n_tokens, kDenseK, kDenseM + kKvM, cfg, baseline, + candidate, + [&]() { + return poison_output(separate0.ptr, out0_bytes, 0x7fc10001u) && + poison_output(separate1.ptr, out1_bytes, 0x7fc20002u) && + poison_output(pair0.ptr, out0_bytes, 0x7fc30003u) && + poison_output(pair1.ptr, out1_bytes, 0x7fc40004u); + }, + [&]() { + return bitwise_equal(separate0.ptr, pair0.ptr, out0_bytes, + "pair q_a output") && + bitwise_equal(separate1.ptr, pair1.ptr, out1_bytes, + "pair kv output") && + check_guard(separate0.ptr, out0_bytes, + "pair separate q_a oracle") && + check_guard(separate1.ptr, out1_bytes, + "pair separate kv oracle") && + check_guard(pair0.ptr, out0_bytes, + "pair fused q_a oracle") && + check_guard(pair1.ptr, out1_bytes, + "pair fused kv oracle"); + })) return false; + return bitwise_equal(separate0.ptr, pair0.ptr, out0_bytes, + "pair q_a output") && + bitwise_equal(separate1.ptr, pair1.ptr, out1_bytes, + "pair kv output") && + check_guard(separate0.ptr, out0_bytes, "pair separate q_a") && + check_guard(separate1.ptr, out1_bytes, "pair separate kv") && + check_guard(pair0.ptr, out0_bytes, "pair fused q_a") && + check_guard(pair1.ptr, out1_bytes, "pair fused kv"); +} + +bool run_qb(const model_fixture &model, const config &cfg, + uint32_t n_tokens) { + uint64_t out_elements = 0; + if (!checked_mul(n_tokens, kQbM, &out_elements)) return false; + const uint64_t logical_bytes = out_elements * sizeof(float); + const uint64_t allocation_bytes = logical_bytes + + kGuardWords * sizeof(uint32_t); + tensor_owner x(static_cast(n_tokens) * kQbK * sizeof(float)); + tensor_owner tile8(allocation_bytes); + tensor_owner tile4(allocation_bytes); + if (!allocate_io(n_tokens, kQbK, out_elements, &x, &tile8, &tile4)) { + std::fprintf(stderr, "q_b N=%u: tensor setup failed\n", n_tokens); + return false; + } + const arm baseline = { + "tile8", []() { select_tile8(true); }, + [&](uint32_t set) { + return ds4_gpu_matmul_quant_tensor( + tile8.ptr, model.data, model.size, + model.weights[set].qb_offset, kQ4Type, kQbK, kQbM, + x.ptr, n_tokens) != 0; + }}; + const arm candidate = { + "tile4", select_k1024_tile4, + [&](uint32_t set) { + return ds4_gpu_matmul_quant_tensor( + tile4.ptr, model.data, model.size, + model.weights[set].qb_offset, kQ4Type, kQbK, kQbM, + x.ptr, n_tokens) != 0; + }}; + if (!benchmark_arms( + "q_b", n_tokens, kQbK, kQbM, cfg, baseline, candidate, + [&]() { + return poison_output(tile8.ptr, logical_bytes, 0x7fc10001u) && + poison_output(tile4.ptr, logical_bytes, 0x7fc20002u); + }, + [&]() { + return bitwise_equal(tile8.ptr, tile4.ptr, logical_bytes, + "q_b tile8 vs tile4") && + check_guard(tile8.ptr, logical_bytes, + "q_b tile8 oracle") && + check_guard(tile4.ptr, logical_bytes, + "q_b tile4 oracle"); + })) return false; + return bitwise_equal(tile8.ptr, tile4.ptr, logical_bytes, + "q_b tile8 vs tile4") && + check_guard(tile8.ptr, logical_bytes, "q_b tile8") && + check_guard(tile4.ptr, logical_bytes, "q_b tile4"); +} + +void usage(FILE *stream, const char *argv0) { + std::fprintf( + stream, + "usage: %s [options]\n\n" + "Resident ROCm Q4_K prefill kernel A/B (HIP event timing only).\n\n" + " --case all|dense|pair|qb comparison to run (default: all)\n" + " --tokens N[,N...] token counts, each 9..4096\n" + " --full use 9,16,17,31,32,33,128,512,4096\n" + " --sets N rotating resident weight sets (default: %u)\n" + " --samples N samples/arm, multiple of 4 (default: %u)\n" + " --warmup N untimed dispatches/arm (default: %u)\n" + " -h, --help show this help\n\n" + "dense compares legacy with TILE8 at K=4096,M=1024; pair compares\n" + "two TILE8 projections with the fused K=4096,M=(1024+512) path; qb\n" + "compares TILE8 with TILE4 at the production K=1024,M=32768 shape.\n", + argv0, kDefaultSets, kDefaultSamples, kDefaultWarmup); +} + +uint32_t parse_u32(const char *text, const char *option, uint32_t minimum, + uint32_t maximum) { + char *end = nullptr; + errno = 0; + const unsigned long value = std::strtoul(text, &end, 10); + if (errno != 0 || !text[0] || !end || *end || value < minimum || + value > maximum) { + std::fprintf(stderr, "invalid %s: %s\n", option, text); + std::exit(2); + } + return static_cast(value); +} + +const char *need_value(int *index, int argc, char **argv) { + if (*index + 1 >= argc) { + std::fprintf(stderr, "%s requires a value\n", argv[*index]); + std::exit(2); + } + return argv[++*index]; +} + +std::vector parse_tokens(const char *text) { + std::vector result; + const char *cursor = text; + while (*cursor) { + const char *comma = std::strchr(cursor, ','); + const std::string item(cursor, + comma ? static_cast(comma - cursor) + : std::strlen(cursor)); + result.push_back(parse_u32(item.c_str(), "--tokens", 9u, 4096u)); + if (!comma) break; + cursor = comma + 1; + if (!*cursor) { + std::fprintf(stderr, "invalid --tokens: trailing comma\n"); + std::exit(2); + } + } + if (result.empty()) { + std::fprintf(stderr, "--tokens cannot be empty\n"); + std::exit(2); + } + std::sort(result.begin(), result.end()); + result.erase(std::unique(result.begin(), result.end()), result.end()); + return result; +} + +config parse_options(int argc, char **argv) { + config cfg; + for (int i = 1; i < argc; i++) { + if (!std::strcmp(argv[i], "-h") || !std::strcmp(argv[i], "--help")) { + usage(stdout, argv[0]); + std::exit(0); + } else if (!std::strcmp(argv[i], "--case")) { + const char *value = need_value(&i, argc, argv); + if (!std::strcmp(value, "all")) cfg.selected = bench_case::all; + else if (!std::strcmp(value, "dense")) cfg.selected = bench_case::dense; + else if (!std::strcmp(value, "pair")) cfg.selected = bench_case::pair; + else if (!std::strcmp(value, "qb")) cfg.selected = bench_case::qb; + else { + std::fprintf(stderr, "invalid --case: %s\n", value); + std::exit(2); + } + } else if (!std::strcmp(argv[i], "--tokens")) { + cfg.tokens = parse_tokens(need_value(&i, argc, argv)); + } else if (!std::strcmp(argv[i], "--full")) { + cfg.tokens = {9u, 16u, 17u, 31u, 32u, 33u, 128u, 512u, 4096u}; + } else if (!std::strcmp(argv[i], "--sets")) { + cfg.sets = parse_u32(need_value(&i, argc, argv), "--sets", 1u, 32u); + } else if (!std::strcmp(argv[i], "--samples")) { + cfg.samples = + parse_u32(need_value(&i, argc, argv), "--samples", 4u, 1000u); + } else if (!std::strcmp(argv[i], "--warmup")) { + cfg.warmup = + parse_u32(need_value(&i, argc, argv), "--warmup", 0u, 100u); + } else { + std::fprintf(stderr, "unknown option: %s\n", argv[i]); + usage(stderr, argv[0]); + std::exit(2); + } + } + if ((cfg.samples % 4u) != 0u) { + std::fprintf(stderr, + "--samples must be a multiple of 4 for ABBA/BAAB balance\n"); + std::exit(2); + } + return cfg; +} + +bool includes(bench_case selected, bench_case wanted) { + return selected == bench_case::all || selected == wanted; +} + +} // namespace + +int main(int argc, char **argv) { + const config cfg = parse_options(argc, argv); + env_snapshot enable_guard(kPrefillEnable); + env_snapshot disable_guard(kPrefillDisable); + env_snapshot require_guard(kPrefillRequire); + env_snapshot tile4_guard(kK1024Tile4Disable); + env_snapshot tile4_ssd_guard(kK1024Tile4SsdEnable); + env_snapshot tile4_require_guard(kK1024Tile4Require); + + int device_count = 0; + hipError_t hip_rc = hipGetDeviceCount(&device_count); + if (hip_rc != hipSuccess || device_count <= 0) { + std::fprintf(stderr, + "rocm-q4-prefill-bench: no visible HIP device (%s)\n", + hip_rc == hipSuccess ? "device count is zero" + : hipGetErrorString(hip_rc)); + return 77; + } + hipDeviceProp_t properties{}; + if (hipGetDeviceProperties(&properties, 0) != hipSuccess) { + std::fprintf(stderr, + "rocm-q4-prefill-bench: cannot query device properties\n"); + return 1; + } + if (!ds4_gpu_init()) { + std::fprintf(stderr, "rocm-q4-prefill-bench: ds4_gpu_init failed\n"); + return 1; + } + + bool ok = true; + model_fixture model; + if (!make_model(&model, cfg.sets)) { + std::fprintf(stderr, "rocm-q4-prefill-bench: model fixture allocation failed\n"); + ok = false; + } + if (ok) { + ds4_gpu_set_ssd_streaming(false); + const uint64_t max_tensor = q4_weight_bytes(kQbK, kQbM); + if (!ds4_gpu_set_model_fd(fileno(model.file)) || + !ds4_gpu_set_model_map_spans( + model.data, model.size, model.span_offsets.data(), + model.span_sizes.data(), + static_cast(model.span_offsets.size()), max_tensor) || + !ds4_gpu_synchronize()) { + std::fprintf(stderr, + "rocm-q4-prefill-bench: device-resident weight copy failed\n"); + ok = false; + } + } + + if (ok) { + std::printf( + "DS4_ROCM_Q4_PREFILL_SETUP device=%s arch=%s warp=%d sets=%u " + "resident_mib=%.2f timing=hip_events ssd_streaming=off\n", + properties.name, properties.gcnArchName, properties.warpSize, + cfg.sets, static_cast(model.resident_bytes) / 1048576.0); + std::fflush(stdout); + for (uint32_t n_tokens : cfg.tokens) { + if (includes(cfg.selected, bench_case::dense)) { + ok = run_dense(model, cfg, n_tokens) && ok; + } + if (ok && includes(cfg.selected, bench_case::pair)) { + ok = run_pair(model, cfg, n_tokens) && ok; + } + if (ok && includes(cfg.selected, bench_case::qb)) { + ok = run_qb(model, cfg, n_tokens) && ok; + } + if (!ok) break; + } + } + + (void)ds4_gpu_set_model_fd(-1); + ds4_gpu_cleanup(); + std::fprintf(stderr, "rocm-q4-prefill-bench: %s\n", ok ? "PASS" : "FAIL"); + return ok ? 0 : 1; +} From 06ed164899333725bbf24a9918de082288d9d5ac Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Fri, 28 Aug 2026 15:12:09 +0200 Subject: [PATCH 133/189] perf(rocm): add opt-in gfx1151 WMMA prefill attention --- rocm/ds4_rocm_attention.cuh | 23 ++ rocm/ds4_rocm_attention_launch.cuh | 326 +++++++++++++++++++++-------- 2 files changed, 258 insertions(+), 91 deletions(-) diff --git a/rocm/ds4_rocm_attention.cuh b/rocm/ds4_rocm_attention.cuh index a5cb44036..43cd0e243 100644 --- a/rocm/ds4_rocm_attention.cuh +++ b/rocm/ds4_rocm_attention.cuh @@ -1423,6 +1423,11 @@ __global__ static void attention_indexed_mixed_heads8_online_kernel( } } +#if defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__) +/* gfx1151/wave32 mixed-attention core. MODE 0 handles contiguous prefill, + * MODE 1 indexed mixed attention, and MODE 2 the causal raw ring. The tuned + * MODE 1/2 vector path remains controlled by the capability-checked launch + * gates in ds4_rocm_attention_launch.cuh. */ template __global__ static void attention_mixed_heads16_wmma_kernel( float *heads, @@ -1442,6 +1447,10 @@ __global__ static void attention_mixed_heads16_wmma_kernel( uint32_t ratio, uint32_t n_head, uint32_t head_dim) { + static_assert(MODE >= 0 && MODE <= 2, + "gfx1151 WMMA attention mode is invalid"); + static_assert(HEADS == 16u || HEADS == 32u, + "gfx1151 WMMA attention requires 16 or 32 heads/workgroup"); constexpr uint32_t KEYS = 80u; constexpr uint32_t DIMS = 64u; constexpr uint32_t TILE = 16u; @@ -1449,6 +1458,19 @@ __global__ static void attention_mixed_heads16_wmma_kernel( constexpr uint32_t HEAD_TILES = HEADS / TILE; constexpr uint32_t KEY_TILES = KEYS / TILE; constexpr uint32_t WORKGROUP = HEADS * 16u; + constexpr size_t SHARED_BYTES = + (size_t)HEADS * STRIDE * sizeof(half) + + (size_t)KEYS * STRIDE * sizeof(half) + + (size_t)HEADS * STRIDE * sizeof(float) + + (size_t)HEADS * 3u * sizeof(float) + + 256u * sizeof(uint32_t) + + DS4_ROCM_ATTENTION_INDEXED_TOPK_CAP * sizeof(uint32_t) + + 3u * sizeof(uint32_t); + static_assert(KEYS % TILE == 0u && DIMS % TILE == 0u && + HEADS % TILE == 0u && WORKGROUP <= 512u, + "gfx1151 WMMA attention tile geometry changed"); + static_assert(SHARED_BYTES <= 64u * 1024u, + "gfx1151 WMMA attention exceeds the validated LDS budget"); const uint32_t t = (uint32_t)blockIdx.x; const uint32_t head_base = (uint32_t)blockIdx.y * HEADS; if (t >= n_tokens || head_dim != 512u) return; @@ -1813,6 +1835,7 @@ __global__ static void attention_mixed_heads16_wmma_kernel( } } } +#endif __global__ static void attention_static_mixed_heads8_online_kernel( float *heads, diff --git a/rocm/ds4_rocm_attention_launch.cuh b/rocm/ds4_rocm_attention_launch.cuh index 1b7b79537..1cf86971c 100644 --- a/rocm/ds4_rocm_attention_launch.cuh +++ b/rocm/ds4_rocm_attention_launch.cuh @@ -1,4 +1,181 @@ extern "C" int ds4_gpu_store_raw_kv_tensor(ds4_gpu_tensor *raw_cache, const ds4_gpu_tensor *kv, uint32_t raw_cap, uint32_t row, uint32_t head_dim); + +/* The gfx1151 WMMA attention kernels deliberately stay behind per-path + * opt-ins until their approximate-F16 arithmetic has a wider prompt oracle. + * The common rollback wins over both opt-ins. Environment values are read on + * every call so an in-process A/B harness can switch arms without rebuilding + * or reinitializing the backend. */ +static const char *const DS4_ROCM_GFX1151_WMMA_RING_ENABLE_ENV = + "DS4_ROCM_ENABLE_GFX1151_PREFILL_WMMA_RING"; +static const char *const DS4_ROCM_GFX1151_WMMA_INDEXED_ENABLE_ENV = + "DS4_ROCM_ENABLE_GFX1151_PREFILL_WMMA_INDEXED"; +static const char *const DS4_ROCM_GFX1151_WMMA_DISABLE_ENV = + "DS4_ROCM_DISABLE_GFX1151_PREFILL_WMMA_ATTN"; + +enum { + DS4_ROCM_ATTN_ENV_UNSET = -1, + DS4_ROCM_ATTN_ENV_INVALID = -2, +}; + +static pthread_mutex_t g_rocm_attention_wmma_notice_mu = + PTHREAD_MUTEX_INITIALIZER; +static uint32_t g_rocm_attention_wmma_notice_mask; + +static int rocm_attention_wmma_notice_once(uint32_t bit) { + pthread_mutex_lock(&g_rocm_attention_wmma_notice_mu); + const int report = (g_rocm_attention_wmma_notice_mask & bit) == 0u; + g_rocm_attention_wmma_notice_mask |= bit; + pthread_mutex_unlock(&g_rocm_attention_wmma_notice_mu); + return report; +} + +static int rocm_attention_env_value_eq( + const char *value, size_t value_len, const char *literal) { + const size_t literal_len = strlen(literal); + if (value_len != literal_len) return 0; + for (size_t i = 0; i < value_len; i++) { + if (tolower((unsigned char)value[i]) != + tolower((unsigned char)literal[i])) { + return 0; + } + } + return 1; +} + +/* Return -1 for unset, 0/1 for recognized booleans, and -2 for invalid. + * Empty is an explicit presence opt-in; conventional false spellings remain + * safe and invalid values fail closed. */ +static int rocm_attention_env_bool_value( + const char *name, uint32_t invalid_notice_bit) { + const char *value = name ? getenv(name) : NULL; + if (!value) return DS4_ROCM_ATTN_ENV_UNSET; + while (isspace((unsigned char)*value)) value++; + size_t value_len = strlen(value); + while (value_len != 0u && + isspace((unsigned char)value[value_len - 1u])) { + value_len--; + } + if (value_len == 0u) return 1; + if (rocm_attention_env_value_eq(value, value_len, "1") || + rocm_attention_env_value_eq(value, value_len, "true") || + rocm_attention_env_value_eq(value, value_len, "yes") || + rocm_attention_env_value_eq(value, value_len, "on")) { + return 1; + } + if (rocm_attention_env_value_eq(value, value_len, "0") || + rocm_attention_env_value_eq(value, value_len, "false") || + rocm_attention_env_value_eq(value, value_len, "no") || + rocm_attention_env_value_eq(value, value_len, "off")) { + return 0; + } + if (rocm_attention_wmma_notice_once(invalid_notice_bit)) { + const size_t shown_len = value_len < 96u ? value_len : 96u; + fprintf(stderr, + DS4_GPU_LOG_PREFIX "invalid boolean environment value " + "%s=%.*s%s; gfx1151 WMMA attention remains disabled\n", + name, + (int)shown_len, + value, + shown_len == value_len ? "" : "..."); + } + return DS4_ROCM_ATTN_ENV_INVALID; +} + +/* Cache a successful device-property query per host thread and active device. + * This is intentionally not a process-global, unkeyed architecture flag: + * HIP's active device is thread-local and callers may switch devices. */ +static int rocm_attention_runtime_is_gfx1151_wave32(void) { +#if defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__) + int device = -1; + const cudaError_t device_err = cudaGetDevice(&device); + if (device_err != cudaSuccess || device < 0) { + if (rocm_attention_wmma_notice_once(1u << 3)) { + fprintf(stderr, + DS4_GPU_LOG_PREFIX "cannot query active device for " + "gfx1151 WMMA attention; using fallback: %s\n", + cudaGetErrorString(device_err)); + } + (void)cudaGetLastError(); + return 0; + } + + static thread_local int cached_device = -1; + static thread_local int cached_eligible = -1; + if (cached_device == device && cached_eligible >= 0) { + return cached_eligible; + } + + cudaDeviceProp prop = {}; + const cudaError_t prop_err = cudaGetDeviceProperties(&prop, device); + if (prop_err != cudaSuccess) { + if (rocm_attention_wmma_notice_once(1u << 4)) { + fprintf(stderr, + DS4_GPU_LOG_PREFIX "cannot query device properties for " + "gfx1151 WMMA attention; using fallback: %s\n", + cudaGetErrorString(prop_err)); + } + (void)cudaGetLastError(); + return 0; + } + + static const char kGfx1151[] = "gfx1151"; + const int arch_matches = + strncmp(prop.gcnArchName, kGfx1151, sizeof(kGfx1151) - 1u) == 0 && + (prop.gcnArchName[sizeof(kGfx1151) - 1u] == '\0' || + prop.gcnArchName[sizeof(kGfx1151) - 1u] == ':'); + cached_device = device; + cached_eligible = arch_matches && prop.warpSize == 32 ? 1 : 0; + if (!cached_eligible && rocm_attention_wmma_notice_once(1u << 5)) { + fprintf(stderr, + DS4_GPU_LOG_PREFIX "gfx1151 WMMA attention requested on " + "unsupported device arch=%s warpSize=%d; using fallback\n", + prop.gcnArchName, + prop.warpSize); + } + return cached_eligible; +#else + return 0; +#endif +} + +static int rocm_attention_gfx1151_wmma_enabled( + const char *enable_env, + uint32_t enable_invalid_notice_bit, + uint32_t enabled_notice_bit) { + const int enable = rocm_attention_env_bool_value( + enable_env, enable_invalid_notice_bit); + if (enable == DS4_ROCM_ATTN_ENV_INVALID || enable != 1) return 0; + const int disable = rocm_attention_env_bool_value( + DS4_ROCM_GFX1151_WMMA_DISABLE_ENV, 1u << 2); + if (disable == DS4_ROCM_ATTN_ENV_INVALID) return 0; + if (disable == 1) { + if (enable == 1 && rocm_attention_wmma_notice_once(1u << 6)) { + fprintf(stderr, + DS4_GPU_LOG_PREFIX "gfx1151 WMMA attention disabled by " + "%s (overrides %s)\n", + DS4_ROCM_GFX1151_WMMA_DISABLE_ENV, + enable_env); + } + return 0; + } + if (g_quality_mode) { + if (rocm_attention_wmma_notice_once(1u << 7)) { + fprintf(stderr, + DS4_GPU_LOG_PREFIX "gfx1151 WMMA attention is unavailable " + "in quality mode; using fallback\n"); + } + return 0; + } + if (!rocm_attention_runtime_is_gfx1151_wave32()) return 0; + if (rocm_attention_wmma_notice_once(enabled_notice_bit)) { + fprintf(stderr, + DS4_GPU_LOG_PREFIX "gfx1151 wave32 WMMA attention enabled " + "by %s (32 heads/workgroup, 80 keys/tile, float2 KV staging)\n", + enable_env); + } + return 1; +} + extern "C" int ds4_gpu_kv_fp8_store_raw_tensor( ds4_gpu_tensor *kv, ds4_gpu_tensor *raw_cache, @@ -367,30 +544,37 @@ static int attention_decode_batch_launch( model_map, sinks_offset, (uint64_t)n_head * sizeof(float), "attn_sinks"); if (!sinks) return 0; const int fast_window_attention = !g_quality_mode; - const bool use_wmma_ring = - ds4_rocm_gfx1151_flag("DS4_ROCM_ATTN_WMMA32_RING"); - if (use_wmma_ring && !use_comp_mask && n_tokens > 1u && - head_dim == 512u && fast_window_attention) { - dim3 grid(n_tokens, (n_head + 31u) / 32u, 1); - attention_mixed_heads16_wmma_kernel<2, 32><<>>((float *)heads->ptr, - sinks, - (const float *)q->ptr, - (const float *)raw_kv->ptr, - n_comp ? (const float *)comp_kv->ptr : (const float *)raw_kv->ptr, - NULL, - n_tokens, - pos0, - n_raw, - raw_cap, - raw_start, - n_comp, - 0, - window, - ratio, - n_head, - head_dim); - return cuda_ok(cudaGetLastError(), "attention ring wmma32 launch"); +#if defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__) + if (!use_comp_mask && n_tokens > 1u && head_dim == 512u && + n_head == 64u && (uint64_t)n_raw <= (uint64_t)pos0 + n_tokens && + rocm_attention_gfx1151_wmma_enabled( + DS4_ROCM_GFX1151_WMMA_RING_ENABLE_ENV, + 1u << 0, + 1u << 8)) { + dim3 grid(n_tokens, 2u, 1u); + attention_mixed_heads16_wmma_kernel<2, 32, true> + <<>>((float *)heads->ptr, + sinks, + (const float *)q->ptr, + (const float *)raw_kv->ptr, + n_comp ? (const float *)comp_kv->ptr + : (const float *)raw_kv->ptr, + NULL, + n_tokens, + pos0, + n_raw, + raw_cap, + raw_start, + n_comp, + 0u, + window, + ratio, + n_head, + head_dim); + return cuda_ok(cudaGetLastError(), + "attention raw-ring gfx1151 wmma32 tile80 launch"); } +#endif if (!cuda_attention_score_buffer_fits(n_comp)) { if (!use_comp_mask && head_dim == 512u) { dim3 online_grid(n_tokens, (n_head + 7u) / 8u, 1); @@ -621,75 +805,35 @@ extern "C" int ds4_gpu_attention_indexed_mixed_batch_heads_tensor( head_dim == 512 && top_k <= DS4_ROCM_ATTENTION_INDEXED_TOPK_CAP) { #if defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__) + if (n_head == 64u && top_k == 512u && + (uint64_t)n_raw <= (uint64_t)pos0 + n_tokens && + rocm_attention_gfx1151_wmma_enabled( + DS4_ROCM_GFX1151_WMMA_INDEXED_ENABLE_ENV, + 1u << 1, + 1u << 9)) { + dim3 grid(n_tokens, 2u, 1u); + attention_mixed_heads16_wmma_kernel<1, 32, true> + <<>>((float *)heads->ptr, + sinks, + (const float *)q->ptr, + (const float *)raw_kv->ptr, + (const float *)comp_kv->ptr, + topk_ptr, + n_tokens, + pos0, + n_raw, + raw_cap, + raw_start, + n_comp, + top_k, + window, + ratio, + n_head, + head_dim); + return cuda_ok(cudaGetLastError(), + "attention indexed gfx1151 wmma32 tile80 launch"); + } if (!g_quality_mode && n_head <= 64u) { - const bool use_wmma32 = - ds4_rocm_gfx1151_flag("DS4_ROCM_ATTN_WMMA32_INDEXED"); - if (use_wmma32) { - dim3 grid(n_tokens, (n_head + 31u) / 32u, 1); - const bool use_vec2 = - ds4_rocm_gfx1151_flag("DS4_ROCM_ATTN_F32_VEC2"); - if (use_vec2) { - attention_mixed_heads16_wmma_kernel<1, 32, true><<>>((float *)heads->ptr, - sinks, - (const float *)q->ptr, - (const float *)raw_kv->ptr, - (const float *)comp_kv->ptr, - topk_ptr, - n_tokens, - pos0, - n_raw, - raw_cap, - raw_start, - n_comp, - top_k, - window, - ratio, - n_head, - head_dim); - return cuda_ok(cudaGetLastError(), "attention indexed wmma32 f32 vec2 launch"); - } - attention_mixed_heads16_wmma_kernel<1, 32><<>>((float *)heads->ptr, - sinks, - (const float *)q->ptr, - (const float *)raw_kv->ptr, - (const float *)comp_kv->ptr, - topk_ptr, - n_tokens, - pos0, - n_raw, - raw_cap, - raw_start, - n_comp, - top_k, - window, - ratio, - n_head, - head_dim); - return cuda_ok(cudaGetLastError(), "attention indexed wmma32 launch"); - } - const char *wmma_env = getenv("DS4_ROCM_ATTN_WMMA16_INDEXED"); - const bool use_wmma = wmma_env && wmma_env[0] != '\0' && wmma_env[0] != '0'; - if (use_wmma) { - dim3 grid(n_tokens, (n_head + 15u) / 16u, 1); - attention_mixed_heads16_wmma_kernel<1, 16><<>>((float *)heads->ptr, - sinks, - (const float *)q->ptr, - (const float *)raw_kv->ptr, - (const float *)comp_kv->ptr, - topk_ptr, - n_tokens, - pos0, - n_raw, - raw_cap, - raw_start, - n_comp, - top_k, - window, - ratio, - n_head, - head_dim); - return cuda_ok(cudaGetLastError(), "attention indexed wmma16 launch"); - } dim3 grid(n_tokens, (n_head + 31u) / 32u, 1); attention_indexed_mixed_heads8_online_kernel<8, 32><<>>((float *)heads->ptr, sinks, From 0539d7673db1e6f6a9b5b979f098fa35c9e2ec0c Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Fri, 28 Aug 2026 15:12:46 +0200 Subject: [PATCH 134/189] docs(rocm): document prefill experiment controls --- scripts/environment_variables.tsv | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/scripts/environment_variables.tsv b/scripts/environment_variables.tsv index cedb0459d..511b3c058 100644 --- a/scripts/environment_variables.tsv +++ b/scripts/environment_variables.tsv @@ -950,6 +950,7 @@ runtime/mtp DS4_MTP_TIMING Pure presence flag; default off. When set, timestamps runtime/rocm DS4_ROCM_DECODE_STAGE_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm decode stage profile. ds4.c:18259 runtime/rocm DS4_ROCM_DECODE_STAGE_PROFILE_LAYER layer filter subordinate to DS4_ROCM_DECODE_STAGE_PROFILE; unset or whitespace-only: all layers allowed by the parent flag; otherwise the whitespace-trimmed value must be a complete base-10 strtoul result <= UINT32_MAX equal to the current layer; invalid values match none Restricts the ROCm decode stage profiler to one layer; it does not enable profiling by itself. ds4.c:29131 runtime/rocm DS4_ROCM_DISABLE_BATCH_INDEXER_QUERY_PRUNE presence rollback; unset prunes unused zero-prefix indexer query work on eligible resident ROCm prefills; any value including 0 disables Restore transient indexer Q projection, RoPE, QAT, and weight projection before compressed rows exceed top-k. ds4.c:30131 +runtime/rocm DS4_ROCM_DISABLE_GFX1151_PREFILL_WMMA_ATTN value-aware rollback evaluated on every eligible launch; unset/0/false/no/off is inactive, while empty or 1/true/yes/on disables; invalid values fail closed; true overrides both ENABLE variables Force both experimental gfx1151 mixed-attention WMMA prefill paths back to their established scalar/online fallbacks for correctness or A/B measurement. rocm/ds4_rocm_attention_launch.cuh:13 runtime/rocm DS4_ROCM_DISABLE_GLM_STREAMING_PREFILL_FULL_LAYER integer selector/tuning value; unset or invalid uses internal automatic/default value Disable/roll back rocm disable glm streaming prefill full layer. ds4.c:42970 runtime/rocm DS4_ROCM_DISABLE_GLM_STREAMING_PREFILL_FULL_LAYER_PREPARE presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable glm streaming prefill full layer prepare. ds4.c:42988 runtime/rocm DS4_ROCM_DISABLE_GLM_STREAMING_PREFILL_SELECTED_ASYNC_LOAD presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable glm streaming prefill selected async load. ds4.c:46681 @@ -961,7 +962,7 @@ runtime/rocm DS4_ROCM_DISABLE_Q4_ATTN_Q_B_F16_CACHE value-aware rollback; unset/ runtime/rocm DS4_ROCM_DISABLE_Q4_ATTN_Q_B_TRANSIENT_F16 value-aware rollback; unset/0/false/no/off keeps the automatic path eligible, empty or any other non-false value disables it Disable per-layer transient Q4_K attn_q_b-to-F16 scratch for device-image or device-range-resident weights; the existing resident-cache controls remain independent and otherwise eligible calls use native Q4. rocm/ds4_rocm_q4_qb_sidecar.cuh:145 runtime/rocm DS4_ROCM_DISABLE_Q4_DENSE_PAIR presence rollback; unset leaves opt-in policy unchanged Disable/roll back rocm disable q4 dense pair. rocm/ds4_rocm_q4.cuh:435 runtime/rocm DS4_ROCM_DISABLE_Q4_GROUPED_ATTN_A presence rollback; unset permits the caller-marked resident decode production-shape default and explicit ENABLE/REQUIRE; any defined value including empty or 0 disables all grouped attention-A paths and wins over ENABLE/REQUIRE Restore eight standalone Q4 attention-A projections instead of the two-dispatch grouped path. rocm/ds4_rocm_q4.cuh:868 -runtime/rocm DS4_ROCM_DISABLE_Q4_PREFILL_K1024_TILE4 value-aware rollback; unset/0/false/no/off keeps the exact K=1024 four-block kernel enabled; empty or any other value disables Restore the generic eight-block Q4_K tiled-prefill kernel for K=1024. rocm/ds4_rocm_q4.cuh:550 +runtime/rocm DS4_ROCM_DISABLE_Q4_PREFILL_K1024_TILE4 value-aware authoritative rollback; unset/0/false/no/off preserves the resident automatic default and any explicit SSD request; empty or any other value disables; overrides ENABLE and causes REQUIRE to fail closed Restore the generic eight-block Q4_K tiled-prefill kernel for K=1024 in both resident and SSD-streaming execution. rocm/ds4_rocm_q4.cuh:624 runtime/rocm DS4_ROCM_DISABLE_Q4_PREFILL_TILE8 presence rollback; TILE8 is default for 9..4096 tokens Disable/roll back rocm disable q4 prefill tile8. rocm/ds4_rocm_q4.cuh:448 runtime/rocm DS4_ROCM_DISABLE_Q4_SELECTED_EXPERT_VIEWS presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable q4 selected expert views. ds4.c:21135 runtime/rocm DS4_ROCM_DISABLE_RESIDENT_IQ2_SORTED presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable resident iq2 sorted. rocm/ds4_rocm_moe_launch.cuh:747 @@ -993,6 +994,8 @@ runtime/rocm DS4_ROCM_DISABLE_STREAMING_SPLIT_SELECTED presence rollback flag; u runtime/rocm DS4_ROCM_DISABLE_STREAMING_STATIC_DECODE_MAP presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming static decode map. ds4.c:18216 runtime/rocm DS4_ROCM_DISABLE_STREAMING_STATIC_MAP_STATE_CACHE presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming static map state cache. ds4.c:18229 runtime/rocm DS4_ROCM_DSV4_PREQUANT_DECODE sampled once; unset: enabled; present empty or exact 0: disabled; every other present value: enabled; quality mode and GLM models force it off regardless ROCm DeepSeek-V4 decode: quantizes one-token F32 activations to Q8 once and selects the prequantized Q8_0/DP4A projection kernels instead of the full-F32 activation paths. rocm/ds4_rocm_runtime.cuh:4775 +runtime/rocm DS4_ROCM_ENABLE_GFX1151_PREFILL_WMMA_INDEXED value-aware per-launch opt-in, default off; empty or 1/true/yes/on enables and 0/false/no/off disables; invalid values fail closed; DISABLE wins; eligibility additionally requires actual gfx1151, runtime warpSize=32, quality off, N>1, 64 model heads, head_dim=512, and top_k=512 Use 32-head rocWMMA workgroups with 80-key tiles and vectorized float2-to-half2 KV staging for DeepSeek-V4 indexed mixed-attention prefill; all failed gates preserve the existing heads32/heads16 fallback. rocm/ds4_rocm_attention_launch.cuh:11 +runtime/rocm DS4_ROCM_ENABLE_GFX1151_PREFILL_WMMA_RING value-aware per-launch opt-in, default off; empty or 1/true/yes/on enables and 0/false/no/off disables; invalid values fail closed; DISABLE wins; eligibility additionally requires actual gfx1151, runtime warpSize=32, quality off, unmasked N>1, 64 model heads, and head_dim=512 Use 32-head rocWMMA workgroups with 80-key tiles and vectorized float2-to-half2 KV staging for causal raw-ring/mixed prefill; all failed gates preserve the existing online or score-buffer fallback. rocm/ds4_rocm_attention_launch.cuh:9 runtime/rocm DS4_ROCM_ENABLE_IQ2_MOE_WMMA_TAIL_CULL value-aware boolean evaluated on every eligible launch; unset defaults off; after trimming whitespace, empty or 1/true/yes/on enables and 0/false/no/off disables (words case-insensitive); invalid values fail closed with a one-time diagnostic; true DISABLE overrides Opt into skipping inactive 16-row tail waves in both wave32 mixed IQ2_XXS gate/up and Q2_K down hot-list rocWMMA prefill kernels; other shapes and paths retain their existing fallback. rocm/ds4_rocm_moe_launch.cuh:18 runtime/rocm DS4_ROCM_ENABLE_MXFP4_LDSB presence opt-in; unset=off; any defined value including empty or 0 enables the candidate when the MXFP4 path, sorted expert tiles, token count >= 128, LDS-size limit, and dimension-alignment gates all pass Select the ROCm MXFP4 prefill gate/up kernel that stages eight gate and eight up weight rows in LDS and reuses them across expert tiles of up to 128 tokens. rocm/ds4_rocm_moe_launch.cuh:782 runtime/rocm DS4_ROCM_ENABLE_MXFP4_ROW64 presence opt-in; unset=off; any defined value including empty or 0 enables the candidate when the MXFP4 sorted-tile path has at least 8 tokens and the TILE32, LDSB, and TILE4 candidates are not selected Select the ROCm MXFP4 gate/up tile8 occupancy variant with 64 row slots and 512 threads per block. rocm/ds4_rocm_moe_launch.cuh:798 @@ -1002,6 +1005,7 @@ runtime/rocm DS4_ROCM_ENABLE_Q4_ATTN_Q_B_F16_CACHE value-aware persistent-cache runtime/rocm DS4_ROCM_ENABLE_Q4_ATTN_Q_B_F16_OUTPUT value-aware experimental opt-in; unset/empty/0/false/no/off keeps the release F32 projection boundary; other nonempty values enable Write eligible resident Q4_K attn_q_b GEMM output in F16 and run the half-input norm/RoPE epilogue; SSD remains excluded. rocm/ds4_rocm_q4_qb_sidecar.cuh:152 runtime/rocm DS4_ROCM_ENABLE_Q4_DENSE_PAIR presence opt-in; unset=off; DISABLE takes precedence Enable rocm enable q4 dense pair. rocm/ds4_rocm_q4.cuh:434 runtime/rocm DS4_ROCM_ENABLE_Q4_GROUPED_ATTN_A presence opt-in outside the default scope; the exact caller-marked resident decode shape groups=8, N=1, K=4096, M=1024 is automatic, while row-at-a-time batch fallbacks are not; DISABLE wins Enable grouped Q4 attention-A for eligible slices, non-production shapes, or explicit experiments in addition to the resident decode default. rocm/ds4_rocm_q4.cuh:872 +runtime/rocm DS4_ROCM_ENABLE_Q4_PREFILL_K1024_TILE4_SSD value-aware SSD-only opt-in, default off; unset/0/false/no/off retains TILE8, while empty or any other value requests TILE4; eligibility additionally requires N=9..4096, K=1024, M=32768, TILE8 enabled, and the complete weight range in device storage rather than mapped/registered host memory; DISABLE wins Allow the four-lane K=1024 Q4_K prefill specialization to consume an already device-resident/cache-backed attn_q_b weight range during SSD streaming without changing model I/O. rocm/ds4_rocm_q4.cuh:624 runtime/rocm DS4_ROCM_ENABLE_STREAMING_FULL_EXPERT_ADDR_TABLE presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming full expert addr table. ds4.c:18255 runtime/rocm DS4_ROCM_ENABLE_STREAMING_MADVISE_WILLNEED presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming madvise willneed. ds4.c:18224 runtime/rocm DS4_ROCM_ENABLE_STREAMING_PREFILL_BATCH_SELECTED_ADDR presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming prefill batch selected addr. ds4.c:18584 @@ -1058,11 +1062,12 @@ runtime/rocm DS4_ROCM_Q4_ATTN_Q_B_F16_CACHE_MB integer MiB with a full-string pa runtime/rocm DS4_ROCM_Q4_ATTN_Q_B_F16_CACHE_MIN_TOKENS integer token count with a full-string parse; default 512; accepted range 32..UINT32_MAX, values above clamp and invalid or smaller values restore the default Set the minimum prefill batch eligible to prepare or use the ROCm Q4 attn_q_b F16 sidecars. rocm/ds4_rocm_q4_qb_sidecar.cuh:156 runtime/rocm DS4_ROCM_Q4_ATTN_Q_B_TRANSIENT_F16_MIN_TOKENS full-string unsigned token count; default 4096; accepted range 32..UINT32_MAX; values above clamp and invalid or smaller values restore 4096 Set the minimum device-resident, non-SSD prefill batch eligible for per-layer transient ROCm Q4_K attn_q_b-to-F16 expansion. rocm/ds4_rocm_q4_qb_sidecar.cuh:150 runtime/rocm DS4_ROCM_Q4_GROUPED_ATTN_A_STATS presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Print counters for rocm q4 grouped attn a stats. rocm/ds4_rocm_q4.cuh:614 -runtime/rocm DS4_ROCM_Q4_PREFILL_TILE8_STATS presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Print counters for rocm q4 prefill tile8 stats. rocm/ds4_rocm_q4.cuh:514 +runtime/rocm DS4_ROCM_Q4_PREFILL_TILE8_STATS presence diagnostic; unset=off; any defined value including empty or 0 prints at exit Print tiled-prefill dense/pair/attention counters plus total and SSD-specific K=1024 TILE4 dispatch counts. rocm/ds4_rocm_q4.cuh:741 runtime/rocm DS4_ROCM_Q8_DECODE_SHAREDX_64K sampled once; unset: enabled; present empty or exact 0: disabled; every other present value: enabled; effective only for one-token non-prequant Q8_0 matmul with 8192 < in_dim <= 16384 Allows the ROCm shared-input Q8 decode kernel to use up to 64 KiB dynamic LDS for wide inputs; an unsupported/failed LDS launch automatically falls back to the regular kernel. rocm/ds4_rocm_runtime.cuh:4805 runtime/rocm DS4_ROCM_Q_STAGE_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm q stage profile. ds4.c:30038 runtime/rocm DS4_ROCM_REQUIRE_Q4_ATTN_Q_B_F16_CACHE value-aware strict opt-in, default off; unset/0/false/no/off is off, empty or any other value requires eligible batches to use the cache; DISABLE wins Fail an eligible ROCm prefill instead of falling back when the resident Q4_K attn_q_b F16 specialization cannot be prepared or dispatched. rocm/ds4_rocm_q4_qb_sidecar.cuh:135 runtime/rocm DS4_ROCM_REQUIRE_Q4_GROUPED_ATTN_A presence fail-closed assertion; also requests the candidate outside the caller-marked resident decode default; DISABLE remains authoritative and causes failure Require grouped Q4 attention-A and fail instead of silently falling back. rocm/ds4_rocm_q4.cuh:870 +runtime/rocm DS4_ROCM_REQUIRE_Q4_PREFILL_K1024_TILE4 value-aware fail-closed assertion and SSD opt-in; unset/0/false/no/off is off; empty or any other value requires eligible N=9..4096, K=1024, M=32768 dense calls to select TILE4; SSD also requires an actual device-resident weight range; DISABLE wins Prevent a K=1024 TILE4 correctness/performance oracle from silently falling back to TILE8, including during SSD-streaming A/B runs. rocm/ds4_rocm_q4.cuh:628 runtime/rocm DS4_ROCM_REQUIRE_Q4_PREFILL_TILE8 presence fail-closed assertion for eligible TILE8 calls Require rocm require q4 prefill tile8 and fail instead of silently falling back. rocm/ds4_rocm_q4.cuh:452 runtime/rocm DS4_ROCM_STREAMING_DECODE_PREFILL_MAX primary nonempty value over the Metal alias; parsed by strtol when it has a numeric prefix (trailing text is accepted); <= 0 disables, values > UINT32_MAX clamp, no numeric prefix uses automatic default: 64 for Flash with uniform Q4_K/MXFP4 experts, 18 for other Pro/Flash, otherwise 0; the disable flag dominates Sets the largest short, non-quality SSD-streaming prefill batch routed through the decode-style path instead of canonical layer-major prefill. ds4.c:31961 runtime/rocm DS4_ROCM_STREAMING_EXPERT_AUTO_PRELOAD_CAP primary nonempty value over the Metal alias; strict full-string strtoul; valid values > UINT32_MAX clamp, invalid uses 4096, and 0 means no cap (not disabled); when CLI preload is auto/0, unset defaults to cap 4096 except ROCm GLM52, where absent/empty disables automatic preload entirely Caps the number of hot experts synchronously seeded into the SSD-streaming expert cache in automatic preload mode; an explicit CLI preload count bypasses this cap, and setting this variable opts ROCm GLM52 back into auto preload. ds4.c:21454 From 841c72fbb788905478459558e0b28c9844431915 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Fri, 28 Aug 2026 15:31:38 +0200 Subject: [PATCH 135/189] fix(cuda): gate Q4 persistent kernel on residency --- cuda/mmq/ds4_mmq.cu | 43 +++++++++++++++++++- cuda/mmq/ds4_mmq.h | 22 +++++++++- cuda/mmq/test/test_mmq_parity.cu | 70 ++++++++++++++++++++++++++++---- ds4_cuda.cu | 57 +++++++++++++++++++++++--- 4 files changed, 175 insertions(+), 17 deletions(-) diff --git a/cuda/mmq/ds4_mmq.cu b/cuda/mmq/ds4_mmq.cu index d2891c6de..ed531b15a 100644 --- a/cuda/mmq/ds4_mmq.cu +++ b/cuda/mmq/ds4_mmq.cu @@ -4028,6 +4028,7 @@ int ds4_mmq_dense_vec_impl( int M, int N, int K, + int q4_weight_device_resident, cudaStream_t stream) { if (!W || !X_f32 || !out_f32) { @@ -4078,6 +4079,7 @@ int ds4_mmq_dense_vec_impl( } if (q4_k1024_exact && gb10_optimizations_enabled() && (enable || q4_k1024_oracle) && !disable && + q4_weight_device_resident > 0 && (((uintptr_t)W & 15u) == 0u)) { const uint64_t row_tiles = ((uint64_t)(uint32_t)M + 7u) / 8u; const int nsm = ggml_cuda_info().devices[dev].nsm; @@ -6809,14 +6811,51 @@ extern "C" int ds4_mmq_q8_0_dense_vec( const void * W, const float * X, float * out, int M, int N, int K, cudaStream_t stream) { return ds4_mmq_dense_vec_impl( - "ds4_mmq_q8_0_dense_vec", W, X, out, M, N, K, stream); + "ds4_mmq_q8_0_dense_vec", W, X, out, M, N, K, + /*q4_weight_device_resident=*/0, stream); +} + +static int ds4_mmq_pointer_is_device_resident( + const void *ptr, int expected_device) { + if (!ptr || expected_device < 0) return 0; + cudaPointerAttributes attr = {}; + const cudaError_t err = cudaPointerGetAttributes(&attr, ptr); + if (err != cudaSuccess) { + (void)cudaGetLastError(); + return 0; + } +#if CUDART_VERSION >= 10000 + return attr.type == cudaMemoryTypeDevice && + attr.device == expected_device; +#else + return attr.memoryType == cudaMemoryTypeDevice && + attr.device == expected_device; +#endif } extern "C" int ds4_mmq_q4_K_dense_vec( const void * W, const float * X, float * out, int M, int N, int K, cudaStream_t stream) { + int weight_device_resident = 0; + /* Keep pointer introspection out of generic Q4 MMVQ traffic. It is only + * needed when the exact-shape persistent candidate could be considered; + * the full runtime uses the explicit provenance API below instead. */ + if (M == 32768 && N == 1 && K == 1024) { + weight_device_resident = ds4_mmq_pointer_is_device_resident( + W, ggml_cuda_get_device()); + } + return ds4_mmq_dense_vec_impl( + "ds4_mmq_q4_K_dense_vec", W, X, out, M, N, K, + weight_device_resident, stream); +} + +extern "C" int ds4_mmq_q4_K_dense_vec_with_weight_residency( + const void * W, const float * X, float * out, + int M, int N, int K, int weight_device_resident, + cudaStream_t stream) { return ds4_mmq_dense_vec_impl( - "ds4_mmq_q4_K_dense_vec", W, X, out, M, N, K, stream); + "ds4_mmq_q4_K_dense_vec_with_weight_residency", + W, X, out, M, N, K, weight_device_resident > 0, stream); } extern "C" int ds4_mmq_q4_K_grouped_vec( diff --git a/cuda/mmq/ds4_mmq.h b/cuda/mmq/ds4_mmq.h index 759ff8e52..526a08dfc 100644 --- a/cuda/mmq/ds4_mmq.h +++ b/cuda/mmq/ds4_mmq.h @@ -968,6 +968,24 @@ int ds4_mmq_q4_K_dense_vec( int K, cudaStream_t stream); +// Full-runtime form of ds4_mmq_q4_K_dense_vec. The CUDA model resolver +// supplies weight_device_resident from allocation/cache provenance so the +// exact K1024 persistent candidate can reject mapped-host/HMM weights without +// issuing cudaPointerGetAttributes on every decode dispatch. Only a positive +// hint admits that candidate; zero/negative provenance remains fail-closed and +// canonical MMVQ stays available. The legacy entry +// above performs its own pointer-attribute query for standalone callers and +// benchmarks that allocate W with cudaMalloc. +int ds4_mmq_q4_K_dense_vec_with_weight_residency( + const void * W_q4_K, + const float * X_f32, + float * out_f32, + int M, + int N, + int K, + int weight_device_resident, + cudaStream_t stream); + // Exact grouped one-row Q4_K MMVQ for AProjQ4 attention-A on a single GB10. // W is [n_groups][M][K], X is [n_groups][K], and out is // [n_groups][M]. Each group retains the canonical dense_vec reduction tree; @@ -1018,7 +1036,9 @@ int ds4_mmq_q4_K_grouped_batch_vec( // run it with DS4_CUDA_DECODE_GRAPHS=0. Set // DS4_CUDA_Q4_K1024_PERSISTENT_STATS=1 for the atexit counter summary. // DS4_CUDA_NO_Q4_GB10_FAST=1 is the umbrella rollback for this and the -// GB10 Q4 activation scratch. Other shapes and devices retain canonical MMVQ. +// GB10 Q4 activation scratch. The candidate additionally requires W to be in +// CUDA device allocation/cache storage; mapped host and managed/HMM pointers +// retain canonical MMVQ. Other shapes and devices retain canonical MMVQ. void ds4_mmq_q4_K_k1024_persistent_counters( uint64_t *candidates, uint64_t *uses, diff --git a/cuda/mmq/test/test_mmq_parity.cu b/cuda/mmq/test/test_mmq_parity.cu index 8b7468548..520354309 100644 --- a/cuda/mmq/test/test_mmq_parity.cu +++ b/cuda/mmq/test/test_mmq_parity.cu @@ -1984,7 +1984,13 @@ bool run_q4_K_dense_vec_gb10_parity( cudaMemcpyDeviceToHost, stream); int rc_required_disabled = 0; int rc_oracle = 0; + int rc_nonresident_fallback = 0; + int rc_nonresident_required = 0; + cudaError_t nonresident_guard_setup_err = cudaSuccess; std::vector oracle((size_t)M * N); + std::vector nonresident_fallback((size_t)M * N); + std::vector nonresident_guard( + (size_t)M * N * sizeof(float)); if (persistent_k1024) { setenv("DS4_CUDA_NO_Q4_K1024_PERSISTENT", "1", 1); rc_required_disabled = ds4_mmq_q4_K_dense_vec( @@ -1996,16 +2002,51 @@ bool run_q4_K_dense_vec_gb10_parity( unsetenv("DS4_CUDA_Q4_K1024_PERSISTENT_ORACLE"); cudaMemcpyAsync(oracle.data(), dGot, oracle.size() * sizeof(float), cudaMemcpyDeviceToHost, stream); + + /* Simulate the full runtime resolving W from mapped host/HMM rather + * than a cudaMalloc cache. ENABLE must fall back to canonical MMVQ; + * REQUIRE must reject before enqueue and leave the sentinel intact. */ + unsetenv("DS4_CUDA_REQUIRE_Q4_K1024_PERSISTENT"); + rc_nonresident_fallback = + ds4_mmq_q4_K_dense_vec_with_weight_residency( + dW, dX, dGot, M, N, K, + /*weight_device_resident=*/0, stream); + cudaMemcpyAsync(nonresident_fallback.data(), dGot, + nonresident_fallback.size() * sizeof(float), + cudaMemcpyDeviceToHost, stream); + cudaMemsetAsync(dGot, 0xa5, (size_t)M * N * sizeof(float), stream); + nonresident_guard_setup_err = cudaStreamSynchronize(stream); + + setenv("DS4_CUDA_REQUIRE_Q4_K1024_PERSISTENT", "1", 1); + rc_nonresident_required = + ds4_mmq_q4_K_dense_vec_with_weight_residency( + dW, dX, dGot, M, N, K, + /*weight_device_resident=*/0, stream); + cudaMemcpyAsync(nonresident_guard.data(), dGot, + nonresident_guard.size(), cudaMemcpyDeviceToHost, + stream); } const cudaError_t sync_err = cudaStreamSynchronize(stream); size_t mismatches = 0; size_t oracle_output_mismatches = 0; + size_t nonresident_fallback_mismatches = 0; + size_t nonresident_guard_mismatches = 0; for (size_t i = 0; i < ref.size(); i++) { if (std::memcmp(&ref[i], &got[i], sizeof(float)) != 0) mismatches++; if (persistent_k1024 && std::memcmp(&ref[i], &oracle[i], sizeof(float)) != 0) { oracle_output_mismatches++; } + if (persistent_k1024 && + std::memcmp(&ref[i], &nonresident_fallback[i], + sizeof(float)) != 0) { + nonresident_fallback_mismatches++; + } + } + if (persistent_k1024) { + for (unsigned char value : nonresident_guard) { + if (value != 0xa5u) nonresident_guard_mismatches++; + } } uint64_t candidates1 = 0, uses1 = 0, fallbacks1 = 0; @@ -2015,24 +2056,36 @@ bool run_q4_K_dense_vec_gb10_parity( &candidates1, &uses1, &fallbacks1, &require_failures1, &oracle_calls1, &oracle_mismatches1, &oracle_skips1); const bool counter_ok = !persistent_k1024 || - (candidates1 - candidates0 >= 4u && + (candidates1 - candidates0 >= 6u && uses1 - uses0 >= 2u && - fallbacks1 - fallbacks0 >= 2u && - require_failures1 - require_failures0 >= 1u && + fallbacks1 - fallbacks0 >= 4u && + require_failures1 - require_failures0 >= 2u && oracle_calls1 - oracle_calls0 >= 1u && oracle_mismatches1 == oracle_mismatches0 && oracle_skips1 == oracle_skips0); ok = rc_ref == 0 && rc_got == 0 && (!persistent_k1024 || rc_required_disabled != 0) && (!persistent_k1024 || rc_oracle == 0) && + (!persistent_k1024 || rc_nonresident_fallback == 0) && + (!persistent_k1024 || rc_nonresident_required != 0) && + nonresident_guard_setup_err == cudaSuccess && sync_err == cudaSuccess && - mismatches == 0 && oracle_output_mismatches == 0 && counter_ok; + mismatches == 0 && oracle_output_mismatches == 0 && + nonresident_fallback_mismatches == 0 && + nonresident_guard_mismatches == 0 && counter_ok; fprintf(stderr, "rc_ref=%d rc_candidate=%d rc_required_disabled=%d " - "rc_oracle=%d mismatches=%zu oracle_output_mismatches=%zu " - "counter_delta=%llu/%llu/%llu/%llu/%llu/%llu/%llu sync=%s\n%s\n\n", - rc_ref, rc_got, rc_required_disabled, rc_oracle, mismatches, - oracle_output_mismatches, + "rc_oracle=%d rc_nonresident_fallback=%d " + "rc_nonresident_required=%d mismatches=%zu " + "oracle_output_mismatches=%zu " + "nonresident_fallback_mismatches=%zu " + "nonresident_guard_mismatches=%zu " + "counter_delta=%llu/%llu/%llu/%llu/%llu/%llu/%llu " + "guard_setup=%s sync=%s\n%s\n\n", + rc_ref, rc_got, rc_required_disabled, rc_oracle, + rc_nonresident_fallback, rc_nonresident_required, mismatches, + oracle_output_mismatches, nonresident_fallback_mismatches, + nonresident_guard_mismatches, (unsigned long long)(candidates1 - candidates0), (unsigned long long)(uses1 - uses0), (unsigned long long)(fallbacks1 - fallbacks0), @@ -2040,6 +2093,7 @@ bool run_q4_K_dense_vec_gb10_parity( (unsigned long long)(oracle_calls1 - oracle_calls0), (unsigned long long)(oracle_mismatches1 - oracle_mismatches0), (unsigned long long)(oracle_skips1 - oracle_skips0), + cudaGetErrorString(nonresident_guard_setup_err), cudaGetErrorString(sync_err), ok ? "PASS" : "FAIL"); diff --git a/ds4_cuda.cu b/ds4_cuda.cu index 2522c09a9..02c84c3ce 100644 --- a/ds4_cuda.cu +++ b/ds4_cuda.cu @@ -1932,13 +1932,54 @@ extern "C" int ds4_gpu_register_support_map(const void *map, uint64_t size, uint return 1; } +static int cuda_resolved_model_range_is_device_resident( + const void *model_map, + uint64_t offset, + uint64_t bytes, + const char *resolved_ptr) { + if (!resolved_ptr) return 0; + if (model_map == g_model_host_base && + g_model_device_owned && g_model_device_base && + resolved_ptr == g_model_device_base + offset) { + return 1; + } + const uint64_t end = offset + bytes; + if (end < offset) return 0; + for (const cuda_model_range &r : g_model_ranges) { + if (r.host_base != model_map || !r.device_ptr || + offset < r.offset) { + continue; + } + const uint64_t delta = offset - r.offset; + if (delta > r.bytes || bytes > r.bytes - delta) continue; + const char *range_ptr = r.device_ptr + delta; + if (range_ptr == resolved_ptr) { + /* host_registered ranges are CUDA-addressable but page-backed by + * the mmap. All other cuda_model_range entries originate from + * cudaMalloc (standalone or arena-backed) and are resident even + * on GB10's physically unified memory. */ + return r.host_registered == 0; + } + } + return 0; +} + static const char *cuda_resolve_weight_ptr(const void *model_map, uint64_t offset, uint64_t bytes, int logical_tier, - const char *label) { + const char *label, + int *device_resident = NULL) { + if (device_resident) *device_resident = 0; if (g_n_gpus <= 1) { - return cuda_model_range_ptr(model_map, offset, bytes, label); + const char *ptr = cuda_model_range_ptr( + model_map, offset, bytes, label); + if (device_resident) { + *device_resident = + cuda_resolved_model_range_is_device_resident( + model_map, offset, bytes, ptr); + } + return ptr; } if (g_support_host_base && model_map == g_support_host_base) { offset += g_support_offset_bias; @@ -1953,6 +1994,7 @@ static const char *cuda_resolve_weight_ptr(const void *model_map, void *dev_ptr = NULL; if (ds4_gpu_lookup_cache_strict(offset, bytes, physical_device, &dev_ptr) && dev_ptr) { + if (device_resident) *device_resident = 1; return (const char *)dev_ptr; } /* GLM multi-tier: generic launchers resolve by the OUT tensor's tier, @@ -1964,6 +2006,7 @@ static const char *cuda_resolve_weight_ptr(const void *model_map, cur_dev != physical_device && ds4_gpu_lookup_cache_strict(offset, bytes, cur_dev, &dev_ptr) && dev_ptr) { + if (device_resident) *device_resident = 1; return (const char *)dev_ptr; } fprintf(stderr, @@ -40360,9 +40403,10 @@ static int cuda_matmul_q4_K_tensor( return 0; } const int logical_tier = ds4_tensor_device_idx(out); - const char *wptr = cuda_resolve_weight_ptr(model_map, weight_offset, - weight_bytes, logical_tier, - "q4_K dense"); + int weight_device_resident = 0; + const char *wptr = cuda_resolve_weight_ptr( + model_map, weight_offset, weight_bytes, logical_tier, + "q4_K dense", &weight_device_resident); if (!wptr) return 0; /* The scalar-token kernel below rereads every weight row for every token, * making prefill scale almost linearly with batch length. MMQ tiles both @@ -40374,9 +40418,10 @@ static int cuda_matmul_q4_K_tensor( * regular MMQ wins once enough token columns can share each weight * tile. Both consume the GGUF Q4_K layout directly. */ const int rc = n_tok <= 8u - ? ds4_mmq_q4_K_dense_vec( + ? ds4_mmq_q4_K_dense_vec_with_weight_residency( wptr, (const float *)x->ptr, (float *)out->ptr, (int)out_dim, (int)n_tok, (int)in_dim, + weight_device_resident, cuda_decode_stream()) : ds4_mmq_q4_K_dense( wptr, (const float *)x->ptr, (float *)out->ptr, From 149d1aaeba9be36ed599e90f29e0a09a80f51d19 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Fri, 28 Aug 2026 18:48:49 +0200 Subject: [PATCH 136/189] bench(cuda): isolate resident Q4 prefill kernels --- Makefile | 18 +- ds4_cuda.cu | 69 +- ds4_gpu.h | 14 + speed-bench/README.md | 50 ++ speed-bench/cuda_q4_prefill_bench.cu | 1170 ++++++++++++++++++++++++++ 5 files changed, 1315 insertions(+), 6 deletions(-) create mode 100644 speed-bench/cuda_q4_prefill_bench.cu diff --git a/Makefile b/Makefile index 57e68eff7..d2b90c955 100644 --- a/Makefile +++ b/Makefile @@ -69,7 +69,7 @@ DS4_LINK_LIBS ?= $(CUDA_LDLIBS) METAL_LDLIBS := $(LDLIBS) endif -.PHONY: all help clean test test-ssd environment-docs test-quantizer-indexer-q4 test-rocm test-glm53-kda-rocm test-metal-session-batch test-metal-session-batch-ssd test-metal-q4-streams test-metal-indexer-q4 test-metal-q4-attn-exactn test-metal-q4-qb-f16-cache test-metal-q4-qb-f16-cache-timing test-metal-exactn-oracle test-metal-dspark-capture test-metal-argmax-top1 bench-metal-argmax-top1 test-metal-iq2-midonly test-metal-iq2-ssd-grouped-mm test-metal-iq2-live-index test-mxfp4-metal test-mxfp4-cuda test-mxfp4-rocm test-mmq-parity-cuda test-rocm-q4-parity test-rocm-q4-dense test-rocm-q4-pair test-rocm-q4-prefill test-strix-rocm-q4-parity test-strix-rocm-q4-prefill test-strix-rocm-q4-prefill-long test-cuda-session-batch test-cuda-mixed-batch dspark-acceptance dspark-verify-depth rocm-dspark-acceptance rocm-dspark-verify-depth mtp-verify-depth cpu cuda cuda-spark cuda-generic cuda-regression strix-halo rocm cuda-iq2-moe-prefill-bench rocm-iq2-moe-prefill-bench rocm-q4-prefill-bench +.PHONY: all help clean test test-ssd environment-docs test-quantizer-indexer-q4 test-rocm test-glm53-kda-rocm test-metal-session-batch test-metal-session-batch-ssd test-metal-q4-streams test-metal-indexer-q4 test-metal-q4-attn-exactn test-metal-q4-qb-f16-cache test-metal-q4-qb-f16-cache-timing test-metal-exactn-oracle test-metal-dspark-capture test-metal-argmax-top1 bench-metal-argmax-top1 test-metal-iq2-midonly test-metal-iq2-ssd-grouped-mm test-metal-iq2-live-index test-mxfp4-metal test-mxfp4-cuda test-mxfp4-rocm test-mmq-parity-cuda test-rocm-q4-parity test-rocm-q4-dense test-rocm-q4-pair test-rocm-q4-prefill test-strix-rocm-q4-parity test-strix-rocm-q4-prefill test-strix-rocm-q4-prefill-long test-cuda-session-batch test-cuda-mixed-batch dspark-acceptance dspark-verify-depth rocm-dspark-acceptance rocm-dspark-verify-depth mtp-verify-depth cpu cuda cuda-spark cuda-generic cuda-regression strix-halo rocm cuda-iq2-moe-prefill-bench cuda-q4-prefill-bench rocm-iq2-moe-prefill-bench rocm-q4-prefill-bench gguf-tools/deepseek4-quantize: gguf-tools/deepseek4-quantize.c gguf-tools/quants.c gguf-tools/quants.h $(MAKE) -C gguf-tools deepseek4-quantize @@ -372,6 +372,7 @@ help: @echo " make rocm-iq2-moe-prefill-bench Build the resident ROCm IQ2/Q2 WMMA A/B harness" @echo " make rocm-q4-prefill-bench Build the resident ROCm Q4 dense/pair/q_b A/B harness" @echo " make cuda-iq2-moe-prefill-bench CUDA_ARCH=sm_N Build the resident CUDA IQ2/Q2 profiling harness" + @echo " make cuda-q4-prefill-bench CUDA_ARCH=sm_N Build the resident CUDA Q4 dense/pair/q_b harness" @echo " make test-rocm Core regression suite on ROCm-only hosts" @echo " make cpu Build CPU-only ./ds4, ./ds4-server, ./ds4-bench, ./ds4-eval, and ./ds4-agent" @echo " make test Build and run tests" @@ -528,6 +529,19 @@ speed-bench/gpu_iq2_moe_prefill_bench_cuda: speed-bench/gpu_iq2_moe_prefill_benc cuda-iq2-moe-prefill-bench: $(MAKE) --no-print-directory -B speed-bench/gpu_iq2_moe_prefill_bench_cuda CUDA_ARCH="$(CUDA_ARCH)" + +speed-bench/cuda_q4_prefill_bench.o: speed-bench/cuda_q4_prefill_bench.cu ds4_gpu.h + $(NVCC) $(NVCCFLAGS) -std=c++17 -DDS4_BENCH_CUDA -I. -c -o $@ $< + +speed-bench/cuda_q4_prefill_bench: speed-bench/cuda_q4_prefill_bench.o ds4_cuda.o $(MMQ_OBJS) + $(NVCC) $(NVCCFLAGS) -std=c++17 $(MMQ_INCLUDES) -o $@ $^ $(CUDA_LDLIBS) + +cuda-q4-prefill-bench: + @if [ -z "$(strip $(CUDA_ARCH))" ]; then \ + echo "error: specify CUDA_ARCH, for example: make cuda-q4-prefill-bench CUDA_ARCH=sm_121"; \ + exit 2; \ + fi + $(MAKE) --no-print-directory -B speed-bench/cuda_q4_prefill_bench CUDA_ARCH="$(CUDA_ARCH)" endif environment-docs: @@ -944,4 +958,4 @@ mxfp4-dot-test: tests/test_mxfp4_dot.c ./tests/test_mxfp4_dot clean: - rm -f ds4 ds4-server ds4-bench ds4-eval ds4-agent ds4_cpu ds4_native ds4_server_test ds4_test ds4_agent_test gguf-tools/quality-testing/score_official gguf-tools/quality-testing/score_official.o speed-bench/metal_decode_schedule_bench speed-bench/metal_prefill_variant_bench speed-bench/metal_q4_dense_pair_bench speed-bench/metal_q4_mm_tail_cull_bench speed-bench/metal_iq2_moe_tail_cull_bench speed-bench/gpu_iq2_moe_prefill_bench_rocm speed-bench/gpu_iq2_moe_prefill_bench_cuda speed-bench/rocm_q4_prefill_bench speed-bench/*.o tests/test_q4k_dot tests/test_mxfp4_dot tests/test_quantizer_indexer_q4 tests/test_mxfp4_metal tests/test_mxfp4_rocm tests/bench_mxfp4_rocm tests/test_mxfp4_cuda tests/test_rocm_q4_dense_pair tests/test_metal_session_batch tests/test_metal_q4_streams tests/test_metal_indexer_q4 tests/test_metal_q4_attn_exactn tests/test_metal_q4_qb_f16_cache tests/test_metal_exactn_oracle tests/test_metal_dspark_capture tests/test_metal_argmax_top1 tests/test_metal_iq2_midonly tests/test_metal_iq2_ssd_grouped_mm tests/test_metal_iq2_live_index tests/test_glm53_kda tests/test_glm53_kda_rocm tests/test_glm53_vision_engine tests/test_glm53_vision_prompt tests/test_gpu_xdev tests/test_gpu_model_cache tests/test_gpu_lookup_cache_strict tests/test_engine_mgpu_refusal tests/test_engine_mgpu_runtime tests/test_engine_correctness tests/test_sampling tests/test_cuda_session_batch tests/test_cuda_mixed_batch tests/*.o *.o tests/cuda_long_context_smoke tests/cuda_long_context_smoke.o + rm -f ds4 ds4-server ds4-bench ds4-eval ds4-agent ds4_cpu ds4_native ds4_server_test ds4_test ds4_agent_test gguf-tools/quality-testing/score_official gguf-tools/quality-testing/score_official.o speed-bench/metal_decode_schedule_bench speed-bench/metal_prefill_variant_bench speed-bench/metal_q4_dense_pair_bench speed-bench/metal_q4_mm_tail_cull_bench speed-bench/metal_iq2_moe_tail_cull_bench speed-bench/gpu_iq2_moe_prefill_bench_rocm speed-bench/gpu_iq2_moe_prefill_bench_cuda speed-bench/rocm_q4_prefill_bench speed-bench/cuda_q4_prefill_bench speed-bench/*.o tests/test_q4k_dot tests/test_mxfp4_dot tests/test_quantizer_indexer_q4 tests/test_mxfp4_metal tests/test_mxfp4_rocm tests/bench_mxfp4_rocm tests/test_mxfp4_cuda tests/test_rocm_q4_dense_pair tests/test_metal_session_batch tests/test_metal_q4_streams tests/test_metal_indexer_q4 tests/test_metal_q4_attn_exactn tests/test_metal_q4_qb_f16_cache tests/test_metal_exactn_oracle tests/test_metal_dspark_capture tests/test_metal_argmax_top1 tests/test_metal_iq2_midonly tests/test_metal_iq2_ssd_grouped_mm tests/test_metal_iq2_live_index tests/test_glm53_kda tests/test_glm53_kda_rocm tests/test_glm53_vision_engine tests/test_glm53_vision_prompt tests/test_gpu_xdev tests/test_gpu_model_cache tests/test_gpu_lookup_cache_strict tests/test_engine_mgpu_refusal tests/test_engine_mgpu_runtime tests/test_engine_correctness tests/test_sampling tests/test_cuda_session_batch tests/test_cuda_mixed_batch tests/*.o *.o tests/cuda_long_context_smoke tests/cuda_long_context_smoke.o diff --git a/ds4_cuda.cu b/ds4_cuda.cu index 02c84c3ce..34670bf93 100644 --- a/ds4_cuda.cu +++ b/ds4_cuda.cu @@ -1766,6 +1766,15 @@ extern "C" int ds4_cuda_q8_fold_take_q81(const void *src, uint64_t in_dim, return cuda_q8_fold_take(src, in_dim, stream, q81); } +/* Test-only fail-closed control used by the resident Q4 benchmark. Keep this + * out of the environment surface: production dispatch retains its established + * MMQ-to-Q8_K rollback unless an explicit test caller enables strict mode. */ +static int g_cuda_test_q4_mmq_strict; + +extern "C" void ds4_cuda_test_set_q4_mmq_strict(int required) { + g_cuda_test_q4_mmq_strict = required != 0; +} + static int cuda_use_mmq(void) { static int init = 0; static int use = 0; @@ -2627,6 +2636,30 @@ static int cuda_q4_attn_q_b_source_is_device_resident( #endif } +extern "C" int ds4_cuda_test_model_range_is_device_resident( + const void *model_map, + uint64_t model_size, + uint64_t offset, + uint64_t bytes, + int logical_tier) { + if (!model_map || model_size == 0u || bytes == 0u || + offset > model_size || bytes > model_size - offset || + g_n_gpus != 1 || logical_tier != 0 || + model_map != g_model_host_base || + model_size != g_model_registered_size || + !g_model_device_owned || !g_model_device_base) { + return 0; + } + + const char *resolved = g_model_device_base + offset; + if (!cuda_resolved_model_range_is_device_resident( + model_map, offset, bytes, resolved)) { + return 0; + } + return cuda_q4_attn_q_b_source_is_device_resident( + resolved, g_gpu[logical_tier].device_id); +} + /* Match the existing CUDA weight-cache safety floor without coupling this * cache to the Q8-specific reserve environment variable. The explicit * future-session reserve supplied by ds4.c is added independently. */ @@ -7756,6 +7789,7 @@ extern "C" int ds4_gpu_init(void) { } extern "C" void ds4_gpu_cleanup(void) { + g_cuda_test_q4_mmq_strict = 0; g_stream_expert_persistent_runtime_ready = 0; (void)cudaDeviceSynchronize(); /* The resident q_b GEMMs may still reference cache-owned allocations. @@ -40430,15 +40464,28 @@ static int cuda_matmul_q4_K_tensor( if (rc == 0) return 1; fprintf(stderr, "ds4: Q4_K MMQ returned %d " - "(in=%llu out=%llu n_tok=%llu); falling back\n", + "(in=%llu out=%llu n_tok=%llu)%s\n", rc, (unsigned long long)in_dim, (unsigned long long)out_dim, - (unsigned long long)n_tok); + (unsigned long long)n_tok, + g_cuda_test_q4_mmq_strict + ? "; strict benchmark mode rejects fallback" + : "; falling back"); + if (g_cuda_test_q4_mmq_strict) return 0; if (in_dim == 1024u && out_dim == 32768u && n_tok == 1u && getenv("DS4_CUDA_REQUIRE_Q4_K1024_PERSISTENT") != NULL) { return 0; } } + if (g_cuda_test_q4_mmq_strict) { + fprintf(stderr, + "ds4: Q4_K strict benchmark mode found no MMQ dispatch " + "(in=%llu out=%llu n_tok=%llu)\n", + (unsigned long long)in_dim, + (unsigned long long)out_dim, + (unsigned long long)n_tok); + return 0; + } void *tmp = cuda_tmp_alloc_on(logical_tier, n_tok * blocks * sizeof(cuda_block_q8_K), "q4_K dense prequant"); @@ -40553,14 +40600,28 @@ static int cuda_matmul_q4_K_pair_tensor_impl( if (rc == 0) return 1; fprintf(stderr, "ds4: Q4_K %s pair returned %d " - "(in=%llu out0=%llu out1=%llu n_tok=%llu); falling back\n", + "(in=%llu out0=%llu out1=%llu n_tok=%llu)%s\n", n_tok <= 8u ? "MMVQ" : "MMQ", rc, (unsigned long long)in_dim, (unsigned long long)out0_dim, (unsigned long long)out1_dim, - (unsigned long long)n_tok); + (unsigned long long)n_tok, + g_cuda_test_q4_mmq_strict + ? "; strict benchmark mode rejects fallback" + : "; falling back"); + if (g_cuda_test_q4_mmq_strict) return 0; if (gb10_canonical) return 0; } + if (g_cuda_test_q4_mmq_strict) { + fprintf(stderr, + "ds4: Q4_K pair strict benchmark mode found no MMQ " + "dispatch (in=%llu out0=%llu out1=%llu n_tok=%llu)\n", + (unsigned long long)in_dim, + (unsigned long long)out0_dim, + (unsigned long long)out1_dim, + (unsigned long long)n_tok); + return 0; + } /* The Q8_K pair is the established decode/microbatch rollback only. * For prefill, preserve DS4_CUDA_MMQ=0 and MMQ rejection semantics by diff --git a/ds4_gpu.h b/ds4_gpu.h index df3afba0b..33b029ec9 100644 --- a/ds4_gpu.h +++ b/ds4_gpu.h @@ -134,6 +134,20 @@ int ds4_gpu_synchronize(void); int ds4_gpu_set_model_map(const void *model_map, uint64_t model_size); int ds4_gpu_set_model_fd(int fd); int ds4_gpu_set_model_fd_for_map(int fd, const void *model_map); +#if defined(DS4_BENCH_CUDA) || \ + (!defined(__APPLE__) && !defined(DS4_ROCM_BUILD) && \ + !defined(DS4_NO_GPU)) +/* CUDA benchmark/test controls. These are deliberately explicit API hooks + * rather than environment knobs: production callers must not depend on + * strict dispatch or backend-internal model provenance. */ +int ds4_cuda_test_model_range_is_device_resident( + const void *model_map, + uint64_t model_size, + uint64_t offset, + uint64_t bytes, + int logical_tier); +void ds4_cuda_test_set_q4_mmq_strict(int required); +#endif /* Prepare a second, fully resident support GGUF without replacing the active * target-model mapping used by SSD streaming. */ int ds4_gpu_prepare_support_model(const void *model_map, uint64_t model_size, diff --git a/speed-bench/README.md b/speed-bench/README.md index 4cf6a72f2..db1f3cf7a 100644 --- a/speed-bench/README.md +++ b/speed-bench/README.md @@ -152,6 +152,56 @@ and environment-gate changes are outside the reported HIP-event intervals. `candidate_delta_pct` is negative when the candidate is faster; the companion `speedup_pct` reports the positive speedup convention. +### Resident CUDA Q4_K prefill + +Build the production-API kernel harness on a CUDA host with an explicit +architecture: + +``` +make cuda-q4-prefill-bench CUDA_ARCH=sm_121 +./speed-bench/cuda_q4_prefill_bench --path mmq +``` + +The default `dense`, `pair`, and `qb` cases use the same production shapes and +token set as the ROCm harness. The synthetic GGUF-layout weights are copied by +the backend into a `cudaMalloc` allocation before warmup. A CUDA test hook +checks the backend-owned pointer provenance and device attributes of every +dense, KV, and q_b range in every rotating weight set; a global free-memory +delta is printed only as a diagnostic and is not accepted as residency proof. +CUDA events measure the production backend GPU interval, including +stream-ordered scratch allocation/free, tail clears, activation quantization, +Q4 projection, and output sanitization. Model uploads, output poisoning, full +finite-output scans, sampled CPU Q4_K oracles, canary checks, and warmups are +outside the event interval. + +`cuda_use_mmq()` caches its first decision for the life of the process, so the +dense and `attn_q_b` MMQ-versus-Q8_K comparison deliberately uses separate +processes. The MMQ process also enables a test-only strict control, so an MMQ +rejection fails the run instead of silently measuring the Q8_K fallback. Run +both orders to balance thermal/order drift: + +``` +# ABBA +./speed-bench/cuda_q4_prefill_bench --path legacy --case dense +./speed-bench/cuda_q4_prefill_bench --path mmq --case dense +./speed-bench/cuda_q4_prefill_bench --path mmq --case dense +./speed-bench/cuda_q4_prefill_bench --path legacy --case dense + +# BAAB +./speed-bench/cuda_q4_prefill_bench --path mmq --case dense +./speed-bench/cuda_q4_prefill_bench --path legacy --case dense +./speed-bench/cuda_q4_prefill_bench --path legacy --case dense +./speed-bench/cuda_q4_prefill_bench --path mmq --case dense +``` + +Repeat with `--case qb` for `K=1024,M=32768`. Each result records the immutable +path as `path=legacy` or `path=mmq`; do not toggle `DS4_CUDA_MMQ` around calls +inside another harness. The default MMQ `pair` case is a true in-process +ABBA/BAAB comparison between two public dense calls and the public fused pair +API, with bit-exact pair outputs. Prefill pair is skipped under `--path legacy` +because the CUDA pair API intentionally returns control to two independent +dense projections for `N>8` when MMQ is disabled. + ### Resident IQ2/Q2 MoE prefill on ROCm and CUDA The backend-neutral fixture uses the production `N=4096`, 256-expert, top-6 diff --git a/speed-bench/cuda_q4_prefill_bench.cu b/speed-bench/cuda_q4_prefill_bench.cu new file mode 100644 index 000000000..4cea324ec --- /dev/null +++ b/speed-bench/cuda_q4_prefill_bench.cu @@ -0,0 +1,1170 @@ +// SPDX-License-Identifier: MIT +// Resident, CUDA-event-only Q4_K prefill microbenchmark. + +#include "ds4_gpu.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +constexpr uint32_t kQ4Type = 12u; +constexpr uint32_t kQkK = 256u; +constexpr uint32_t kDenseK = 4096u; +constexpr uint32_t kDenseM = 1024u; +constexpr uint32_t kKvM = 512u; +constexpr uint32_t kQbK = 1024u; +constexpr uint32_t kQbM = 32768u; +constexpr uint32_t kDefaultSets = 4u; +constexpr uint32_t kDefaultSamples = 8u; +constexpr uint32_t kDefaultWarmup = 2u; +constexpr uint32_t kGuardWords = 64u; +constexpr uint64_t kCompareChunk = 4u * 1024u * 1024u; + +struct block_q4_K_host { + uint16_t d; + uint16_t dmin; + uint8_t scales[12]; + uint8_t qs[kQkK / 2u]; +}; + +static_assert(sizeof(block_q4_K_host) == 144u, + "Q4_K fixture must match the raw GGUF layout"); + +enum class bench_case { + all, + dense, + pair, + qb, +}; + +enum class cuda_path { + mmq, + legacy, +}; + +struct config { + bench_case selected = bench_case::all; + cuda_path path = cuda_path::mmq; + std::vector tokens = {9u, 17u, 33u, 128u, 512u}; + uint32_t sets = kDefaultSets; + uint32_t samples = kDefaultSamples; + uint32_t warmup = kDefaultWarmup; +}; + +struct weight_set { + uint64_t dense_offset = 0; + uint64_t kv_offset = 0; + uint64_t qb_offset = 0; +}; + +struct model_fixture { + uint8_t *data = nullptr; + uint64_t size = 0; + uint64_t payload_bytes = 0; + std::vector weights; + + ~model_fixture() { std::free(data); } + model_fixture() = default; + model_fixture(const model_fixture &) = delete; + model_fixture &operator=(const model_fixture &) = delete; +}; + +struct tensor_owner { + ds4_gpu_tensor *ptr = nullptr; + + explicit tensor_owner(uint64_t bytes) : ptr(ds4_gpu_tensor_alloc(bytes)) {} + ~tensor_owner() { ds4_gpu_tensor_free(ptr); } + tensor_owner(const tensor_owner &) = delete; + tensor_owner &operator=(const tensor_owner &) = delete; +}; + +struct env_snapshot { + const char *name; + bool existed; + std::string value; + + explicit env_snapshot(const char *key) + : name(key), existed(std::getenv(key) != nullptr), + value(existed ? std::getenv(key) : "") {} + ~env_snapshot() { + if (existed) { + (void)setenv(name, value.c_str(), 1); + } else { + (void)unsetenv(name); + } + } + env_snapshot(const env_snapshot &) = delete; + env_snapshot &operator=(const env_snapshot &) = delete; +}; + +struct event_timer { + cudaEvent_t begin = nullptr; + cudaEvent_t end = nullptr; + + event_timer() { + if (cudaEventCreate(&begin) != cudaSuccess || + cudaEventCreate(&end) != cudaSuccess) { + std::fprintf(stderr, + "cuda-q4-prefill-bench: CUDA event allocation failed\n"); + std::exit(1); + } + } + ~event_timer() { + if (begin) (void)cudaEventDestroy(begin); + if (end) (void)cudaEventDestroy(end); + } + + bool measure(const std::function &dispatch, float *milliseconds) { + // The harness disables decode graphs. Production Q4 eager dispatch + // passes cuda_decode_stream()==0 to both MMQ (including its async + // scratch pool) and the Q8_K fallback kernels, so bracketing stream 0 + // covers every enqueue in the public call. + if (cudaEventRecord(begin, nullptr) != cudaSuccess) return false; + if (!dispatch()) return false; + if (cudaEventRecord(end, nullptr) != cudaSuccess || + cudaEventSynchronize(end) != cudaSuccess || + cudaEventElapsedTime(milliseconds, begin, end) != cudaSuccess) { + return false; + } + return true; + } +}; + +struct arm { + const char *name; + std::function dispatch; +}; + +struct stats { + double minimum = 0.0; + double median = 0.0; + double p95 = 0.0; + double mean = 0.0; +}; + +uint64_t align_up(uint64_t value, uint64_t alignment) { + return (value + alignment - 1u) / alignment * alignment; +} + +bool checked_mul(uint64_t a, uint64_t b, uint64_t *out) { + if (a != 0u && b > std::numeric_limits::max() / a) return false; + *out = a * b; + return true; +} + +uint32_t lcg_next(uint32_t *state) { + *state = *state * 1664525u + 1013904223u; + return *state; +} + +void fill_q4(void *storage, uint64_t bytes, uint32_t seed) { + auto *blocks = static_cast(storage); + const uint64_t count = bytes / sizeof(*blocks); + uint32_t state = seed; + for (uint64_t i = 0; i < count; i++) { + // Positive, finite FP16 scales. Payload values are deterministic. + blocks[i].d = static_cast(0x2400u + + (lcg_next(&state) & 0xffu)); + blocks[i].dmin = static_cast(0x2000u + + (lcg_next(&state) & 0xffu)); + for (uint8_t &v : blocks[i].scales) { + v = static_cast(lcg_next(&state) >> 24u); + } + for (uint8_t &v : blocks[i].qs) { + v = static_cast(lcg_next(&state) >> 24u); + } + } +} + +uint64_t q4_weight_bytes(uint32_t in_dim, uint32_t out_dim) { + return static_cast(out_dim) * (in_dim / kQkK) * + sizeof(block_q4_K_host); +} + +bool make_model(model_fixture *model, uint32_t sets) { + constexpr uint64_t page = 4096u; + const uint64_t dense_bytes = q4_weight_bytes(kDenseK, kDenseM); + const uint64_t kv_bytes = q4_weight_bytes(kDenseK, kKvM); + const uint64_t qb_bytes = q4_weight_bytes(kQbK, kQbM); + + model->weights.resize(sets); + uint64_t cursor = 0; + auto append = [&](uint64_t bytes) { + const uint64_t offset = align_up(cursor, page); + cursor = offset + bytes; + model->payload_bytes += bytes; + return offset; + }; + for (uint32_t i = 0; i < sets; i++) { + model->weights[i].dense_offset = append(dense_bytes); + model->weights[i].kv_offset = append(kv_bytes); + model->weights[i].qb_offset = append(qb_bytes); + } + model->size = align_up(cursor, page); + void *storage = nullptr; + if (posix_memalign(&storage, static_cast(page), + static_cast(model->size)) != 0) { + return false; + } + model->data = static_cast(storage); + std::memset(model->data, 0, static_cast(model->size)); + for (uint32_t i = 0; i < sets; i++) { + fill_q4(model->data + model->weights[i].dense_offset, dense_bytes, + 0x243f6a88u ^ (i * 0x9e3779b9u)); + fill_q4(model->data + model->weights[i].kv_offset, kv_bytes, + 0x85a308d3u ^ (i * 0x7f4a7c15u)); + fill_q4(model->data + model->weights[i].qb_offset, qb_bytes, + 0x13198a2eu ^ (i * 0x94d049bbu)); + } + return true; +} + +void fill_activation(std::vector *values, uint32_t n_tokens, + uint32_t in_dim) { + values->resize(static_cast(n_tokens) * in_dim); + for (uint32_t token = 0; token < n_tokens; token++) { + for (uint32_t block = 0; block < in_dim / kQkK; block++) { + float *dst = values->data() + + static_cast(token) * in_dim + block * kQkK; + for (uint32_t i = 0; i < kQkK; i++) { + const int q = static_cast((i * 73u + token * 37u + + block * 19u) % 241u) - 120; + dst[i] = static_cast(q) / 32.0f; + } + dst[0] = ((token + block) & 1u) ? 127.0f / 32.0f + : -127.0f / 32.0f; + } + } +} + +std::vector guard_pattern(uint32_t salt) { + std::vector guard(kGuardWords); + for (uint32_t i = 0; i < kGuardWords; i++) { + guard[i] = 0x7fc12000u ^ salt ^ (i * 0x00010101u); + } + return guard; +} + +bool prepare_guard(ds4_gpu_tensor *tensor, uint64_t logical_bytes, + uint32_t salt) { + const std::vector guard = guard_pattern(salt); + return ds4_gpu_tensor_write(tensor, logical_bytes, guard.data(), + guard.size() * sizeof(guard[0])) != 0; +} + +bool poison_output(ds4_gpu_tensor *tensor, uint64_t logical_bytes, + uint32_t pattern, uint32_t guard_salt) { + if (!tensor || logical_bytes == 0u || + logical_bytes % sizeof(uint32_t) != 0u) { + return false; + } + const uint64_t chunk_bytes = std::min(kCompareChunk, logical_bytes); + std::vector poison( + static_cast(chunk_bytes / sizeof(uint32_t)), pattern); + for (uint64_t offset = 0; offset < logical_bytes; offset += chunk_bytes) { + const uint64_t count = std::min(chunk_bytes, logical_bytes - offset); + if (!ds4_gpu_tensor_write(tensor, offset, poison.data(), count)) { + return false; + } + } + return prepare_guard(tensor, logical_bytes, guard_salt); +} + +bool check_guard(const ds4_gpu_tensor *tensor, uint64_t logical_bytes, + uint32_t salt, const char *label) { + const std::vector expected = guard_pattern(salt); + std::vector got(expected.size()); + if (!ds4_gpu_tensor_read(tensor, logical_bytes, got.data(), + got.size() * sizeof(got[0]))) { + std::fprintf(stderr, "%s: guard read failed\n", label); + return false; + } + if (got != expected) { + const auto mismatch = std::mismatch(got.begin(), got.end(), + expected.begin()); + std::fprintf(stderr, "%s: guard overwritten at word %zu\n", label, + static_cast(mismatch.first - got.begin())); + return false; + } + return true; +} + +bool output_is_finite(const ds4_gpu_tensor *tensor, uint64_t logical_bytes, + const char *label) { + const uint64_t chunk_bytes = std::min(kCompareChunk, logical_bytes); + std::vector values( + static_cast(chunk_bytes / sizeof(float))); + for (uint64_t offset = 0; offset < logical_bytes; offset += chunk_bytes) { + const uint64_t count = std::min(chunk_bytes, logical_bytes - offset); + if (!ds4_gpu_tensor_read(tensor, offset, values.data(), count)) { + std::fprintf(stderr, "%s: output read failed\n", label); + return false; + } + const size_t n = static_cast(count / sizeof(float)); + for (size_t i = 0; i < n; i++) { + if (!std::isfinite(values[i])) { + std::fprintf(stderr, + "%s: non-finite/unwritten output at element %llu\n", + label, + static_cast( + offset / sizeof(float) + i)); + return false; + } + } + } + return true; +} + +bool bitwise_equal(const ds4_gpu_tensor *a, const ds4_gpu_tensor *b, + uint64_t bytes, const char *label) { + const uint64_t chunk = std::min(kCompareChunk, bytes); + std::vector lhs(static_cast(chunk)); + std::vector rhs(static_cast(chunk)); + for (uint64_t offset = 0; offset < bytes; offset += chunk) { + const uint64_t count = std::min(chunk, bytes - offset); + if (!ds4_gpu_tensor_read(a, offset, lhs.data(), count) || + !ds4_gpu_tensor_read(b, offset, rhs.data(), count)) { + std::fprintf(stderr, "%s: oracle read failed\n", label); + return false; + } + if (std::memcmp(lhs.data(), rhs.data(), static_cast(count)) != + 0) { + uint64_t first = 0; + while (first < count && lhs[static_cast(first)] == + rhs[static_cast(first)]) { + first++; + } + std::fprintf(stderr, "%s: bitwise mismatch at output byte %llu\n", + label, + static_cast(offset + first)); + return false; + } + } + return true; +} + +float fp16_to_float(uint16_t h) { + const float sign = (h & 0x8000u) ? -1.0f : 1.0f; + const uint32_t exponent = (h >> 10u) & 0x1fu; + const uint32_t mantissa = h & 0x3ffu; + if (exponent == 0u) { + return mantissa == 0u + ? std::copysign(0.0f, sign) + : sign * std::ldexp(static_cast(mantissa), -24); + } + if (exponent == 31u) { + return mantissa == 0u + ? std::copysign(std::numeric_limits::infinity(), sign) + : std::numeric_limits::quiet_NaN(); + } + return sign * std::ldexp(1.0f + static_cast(mantissa) / 1024.0f, + static_cast(exponent) - 15); +} + +void get_scale_min_k4(uint32_t index, const uint8_t *packed, + uint8_t *scale, uint8_t *minimum) { + if (index < 4u) { + *scale = packed[index] & 63u; + *minimum = packed[index + 4u] & 63u; + } else { + *scale = static_cast((packed[index + 4u] & 0x0fu) | + ((packed[index - 4u] >> 6u) << 4u)); + *minimum = static_cast((packed[index + 4u] >> 4u) | + ((packed[index] >> 6u) << 4u)); + } +} + +void dequantize_q4_row(const block_q4_K_host *blocks, uint32_t in_dim, + std::vector *row) { + row->resize(in_dim); + float *dst = row->data(); + const uint32_t n_blocks = in_dim / kQkK; + for (uint32_t block = 0; block < n_blocks; block++) { + const float d = fp16_to_float(blocks[block].d); + const float dmin = fp16_to_float(blocks[block].dmin); + const uint8_t *q = blocks[block].qs; + uint32_t scale_index = 0; + for (uint32_t group = 0; group < kQkK; group += 64u) { + uint8_t sc0 = 0, min0 = 0, sc1 = 0, min1 = 0; + get_scale_min_k4(scale_index, blocks[block].scales, + &sc0, &min0); + get_scale_min_k4(scale_index + 1u, blocks[block].scales, + &sc1, &min1); + const float ds0 = d * sc0; + const float dm0 = dmin * min0; + const float ds1 = d * sc1; + const float dm1 = dmin * min1; + for (uint32_t i = 0; i < 32u; i++) { + *dst++ = ds0 * static_cast(q[i] & 0x0fu) - dm0; + } + for (uint32_t i = 0; i < 32u; i++) { + *dst++ = ds1 * static_cast(q[i] >> 4u) - dm1; + } + q += 32u; + scale_index += 2u; + (void)group; + } + } +} + +std::vector sample_indices(uint32_t extent) { + std::vector values = { + 0u, extent / 3u, extent / 2u, extent - 1u, + }; + std::sort(values.begin(), values.end()); + values.erase(std::unique(values.begin(), values.end()), values.end()); + return values; +} + +bool sampled_cpu_oracle(const ds4_gpu_tensor *output, + const model_fixture &model, uint64_t weight_offset, + const std::vector &activation, + uint32_t n_tokens, uint32_t in_dim, + uint32_t out_dim, const char *label) { + const std::vector tokens = sample_indices(n_tokens); + const std::vector rows = sample_indices(out_dim); + const uint64_t blocks_per_row = in_dim / kQkK; + const uint64_t row_bytes = blocks_per_row * sizeof(block_q4_K_host); + std::vector weight_row; + // cuda/mmq/test/test_mmq_parity.cu validates Q4_K MMQ against this same + // dequantized-weight x original-F32 reference at 0.20*sqrt(K) absolute + // and 5% relative error. Use a small shared envelope for both that Q8_1 + // path and the legacy 256-value Q8_K activation quantizer. Every fixture + // block pins |x|max to 127/32, keeping the Q8_K step bounded and stable. + const double abs_tol = 0.25 * std::sqrt(static_cast(in_dim)); + constexpr double rel_tol = 0.06; + + for (uint32_t row : rows) { + const auto *blocks = reinterpret_cast( + model.data + weight_offset + static_cast(row) * row_bytes); + dequantize_q4_row(blocks, in_dim, &weight_row); + for (uint32_t token : tokens) { + const float *x = activation.data() + + static_cast(token) * in_dim; + float reference = 0.0f; + for (uint32_t k = 0; k < in_dim; k++) { + reference += weight_row[k] * x[k]; + } + float got = 0.0f; + const uint64_t element = static_cast(token) * out_dim + row; + if (!ds4_gpu_tensor_read(output, element * sizeof(float), &got, + sizeof(got))) { + std::fprintf(stderr, "%s: sampled output read failed\n", label); + return false; + } + const double abs_error = std::fabs(static_cast(got) - + reference); + const double rel_error = reference != 0.0f + ? abs_error / std::fabs(static_cast(reference)) + : (abs_error == 0.0 ? 0.0 + : std::numeric_limits::infinity()); + if (!std::isfinite(got) || + (abs_error > abs_tol && rel_error > rel_tol)) { + std::fprintf( + stderr, + "%s: CPU oracle mismatch token=%u row=%u got=%.7g " + "reference=%.7g abs=%.5g rel=%.5g limits=%.5g/%.3g\n", + label, token, row, got, reference, abs_error, rel_error, + abs_tol, rel_tol); + return false; + } + } + } + return true; +} + +double percentile(std::vector sorted, double fraction) { + std::sort(sorted.begin(), sorted.end()); + if (sorted.empty()) return 0.0; + const double position = fraction * static_cast(sorted.size() - 1u); + const size_t lo = static_cast(std::floor(position)); + const size_t hi = static_cast(std::ceil(position)); + const double alpha = position - static_cast(lo); + return sorted[lo] + (sorted[hi] - sorted[lo]) * alpha; +} + +stats summarize(const std::vector &samples) { + stats out; + out.minimum = *std::min_element(samples.begin(), samples.end()); + out.median = percentile(samples, 0.5); + out.p95 = percentile(samples, 0.95); + for (double sample : samples) out.mean += sample; + out.mean /= static_cast(samples.size()); + return out; +} + +const char *path_name(cuda_path path) { + return path == cuda_path::mmq ? "mmq" : "legacy"; +} + +bool benchmark_single_path( + const char *case_name, uint32_t n_tokens, uint32_t in_dim, + uint32_t out_dim, const config &cfg, const arm &which, + const std::function &oracle_prepare, + const std::function &oracle) { + // Validate every rotating weight set and prime lazy Q8 scratch before + // recording a CUDA event. Upload, poison, readback, and CPU work remain + // outside the measured interval. + for (uint32_t set = 0; set < cfg.sets; set++) { + if (!oracle_prepare() || !which.dispatch(set) || + !ds4_gpu_synchronize() || !oracle(set)) { + std::fprintf(stderr, + "cuda-q4-prefill-bench: %s oracle failed for " + "weight set %u\n", + case_name, set); + return false; + } + } + + for (uint32_t i = 0; i < cfg.warmup; i++) { + if (!which.dispatch(i % cfg.sets) || !ds4_gpu_synchronize()) { + return false; + } + } + + event_timer timer; + std::vector samples; + samples.reserve(cfg.samples); + for (uint32_t i = 0; i < cfg.samples; i++) { + float elapsed = 0.0f; + if (!timer.measure([&]() { return which.dispatch(i % cfg.sets); }, + &elapsed)) { + std::fprintf(stderr, + "cuda-q4-prefill-bench: %s/%s timed dispatch failed\n", + case_name, which.name); + return false; + } + samples.push_back(static_cast(elapsed)); + } + + const stats result = summarize(samples); + const double macs = static_cast(n_tokens) * in_dim * out_dim; + const double gmac_s = macs / (result.median * 1.0e6); + std::printf( + "DS4_CUDA_Q4_PREFILL_BENCH case=%s path=%s N=%u K=%u M=%u " + "variant=%s samples=%u sets=%u ms_p50=%.6f ms_min=%.6f " + "ms_p95=%.6f ms_mean=%.6f gmac_s=%.3f\n", + case_name, path_name(cfg.path), n_tokens, in_dim, out_dim, which.name, + cfg.samples, cfg.sets, result.median, result.minimum, result.p95, + result.mean, gmac_s); + std::fflush(stdout); + return true; +} + +bool benchmark_pair_arms( + uint32_t n_tokens, const config &cfg, const arm &baseline, + const arm &candidate, const std::function &oracle_prepare, + const std::function &oracle) { + for (uint32_t set = 0; set < cfg.sets; set++) { + if (!oracle_prepare() || !baseline.dispatch(set) || + !ds4_gpu_synchronize() || !candidate.dispatch(set) || + !ds4_gpu_synchronize() || !oracle(set)) { + std::fprintf(stderr, + "cuda-q4-prefill-bench: pair oracle failed for " + "weight set %u\n", + set); + return false; + } + } + + for (uint32_t i = 0; i < cfg.warmup; i++) { + const uint32_t set = i % cfg.sets; + if (!baseline.dispatch(set) || !ds4_gpu_synchronize() || + !candidate.dispatch(set) || !ds4_gpu_synchronize()) { + return false; + } + } + + event_timer timer; + std::vector a_samples; + std::vector b_samples; + a_samples.reserve(cfg.samples); + b_samples.reserve(cfg.samples); + auto take = [&](const arm &which, uint32_t set, + std::vector *samples) { + float elapsed = 0.0f; + if (!timer.measure([&]() { return which.dispatch(set); }, &elapsed)) { + std::fprintf(stderr, + "cuda-q4-prefill-bench: pair/%s timed dispatch failed\n", + which.name); + return false; + } + samples->push_back(static_cast(elapsed)); + return true; + }; + + // Two samples per arm per cycle. Alternating ABBA/BAAB balances order + // while both arms see identical rotating resident weights. + for (uint32_t cycle = 0; a_samples.size() < cfg.samples; cycle++) { + const uint32_t set0 = (cycle * 2u) % cfg.sets; + const uint32_t set1 = (cycle * 2u + 1u) % cfg.sets; + if ((cycle & 1u) == 0u) { + if (!take(baseline, set0, &a_samples) || + !take(candidate, set0, &b_samples) || + !take(candidate, set1, &b_samples) || + !take(baseline, set1, &a_samples)) return false; + } else { + if (!take(candidate, set0, &b_samples) || + !take(baseline, set0, &a_samples) || + !take(baseline, set1, &a_samples) || + !take(candidate, set1, &b_samples)) return false; + } + } + + const stats a = summarize(a_samples); + const stats b = summarize(b_samples); + std::vector paired_delta; + paired_delta.reserve(a_samples.size()); + for (size_t i = 0; i < a_samples.size(); i++) { + paired_delta.push_back((b_samples[i] / a_samples[i] - 1.0) * 100.0); + } + const double paired_median = percentile(paired_delta, 0.5); + const double median_delta = (b.median / a.median - 1.0) * 100.0; + const double speedup = (a.median / b.median - 1.0) * 100.0; + const double macs = static_cast(n_tokens) * kDenseK * + (kDenseM + kKvM); + std::printf( + "DS4_CUDA_Q4_PREFILL_BENCH case=pair path=%s N=%u K=%u M=%u " + "baseline=%s candidate=%s samples=%u sets=%u " + "baseline_ms_p50=%.6f candidate_ms_p50=%.6f " + "baseline_ms_min=%.6f candidate_ms_min=%.6f " + "baseline_ms_p95=%.6f candidate_ms_p95=%.6f " + "baseline_gmac_s=%.3f candidate_gmac_s=%.3f " + "candidate_delta_pct=%.3f paired_delta_pct_p50=%.3f " + "speedup_pct=%.3f\n", + path_name(cfg.path), n_tokens, kDenseK, kDenseM + kKvM, + baseline.name, candidate.name, cfg.samples, cfg.sets, a.median, + b.median, a.minimum, b.minimum, a.p95, b.p95, + macs / (a.median * 1.0e6), macs / (b.median * 1.0e6), + median_delta, paired_median, speedup); + std::fflush(stdout); + return true; +} + +bool run_dense(const model_fixture &model, const config &cfg, + uint32_t n_tokens) { + uint64_t out_elements = 0; + if (!checked_mul(n_tokens, kDenseM, &out_elements)) return false; + const uint64_t x_bytes = static_cast(n_tokens) * kDenseK * + sizeof(float); + const uint64_t out_bytes = out_elements * sizeof(float); + const uint64_t guard_bytes = kGuardWords * sizeof(uint32_t); + tensor_owner x(x_bytes + guard_bytes); + tensor_owner output(out_bytes + guard_bytes); + std::vector activation; + fill_activation(&activation, n_tokens, kDenseK); + if (!x.ptr || !output.ptr || + !ds4_gpu_tensor_write(x.ptr, 0, activation.data(), x_bytes) || + !prepare_guard(x.ptr, x_bytes, 0x1000u)) { + std::fprintf(stderr, "dense N=%u: tensor setup failed\n", n_tokens); + return false; + } + const arm path = { + cfg.path == cuda_path::mmq ? "mmq" : "legacy_q8k", + [&](uint32_t set) { + return ds4_gpu_matmul_quant_tensor( + output.ptr, model.data, model.size, + model.weights[set].dense_offset, kQ4Type, kDenseK, + kDenseM, x.ptr, n_tokens) != 0; + }}; + return benchmark_single_path( + "dense", n_tokens, kDenseK, kDenseM, cfg, path, + [&]() { + return poison_output(output.ptr, out_bytes, 0x7fc10001u, + 0x2000u); + }, + [&](uint32_t set) { + return output_is_finite(output.ptr, out_bytes, "dense output") && + sampled_cpu_oracle(output.ptr, model, + model.weights[set].dense_offset, + activation, n_tokens, kDenseK, kDenseM, + "dense") && + check_guard(x.ptr, x_bytes, 0x1000u, "dense input") && + check_guard(output.ptr, out_bytes, 0x2000u, + "dense output"); + }) && + check_guard(x.ptr, x_bytes, 0x1000u, "dense input final") && + check_guard(output.ptr, out_bytes, 0x2000u, "dense output final"); +} + +bool run_pair(const model_fixture &model, const config &cfg, + uint32_t n_tokens) { + if (cfg.path == cuda_path::legacy) { + std::printf( + "DS4_CUDA_Q4_PREFILL_SKIP case=pair path=legacy N=%u " + "reason=fused_prefill_api_requires_mmq\n", + n_tokens); + std::fflush(stdout); + return true; + } + + const uint64_t x_bytes = static_cast(n_tokens) * kDenseK * + sizeof(float); + const uint64_t out0_bytes = static_cast(n_tokens) * kDenseM * + sizeof(float); + const uint64_t out1_bytes = static_cast(n_tokens) * kKvM * + sizeof(float); + const uint64_t guard_bytes = kGuardWords * sizeof(uint32_t); + tensor_owner x(x_bytes + guard_bytes); + tensor_owner separate0(out0_bytes + guard_bytes); + tensor_owner separate1(out1_bytes + guard_bytes); + tensor_owner pair0(out0_bytes + guard_bytes); + tensor_owner pair1(out1_bytes + guard_bytes); + std::vector activation; + fill_activation(&activation, n_tokens, kDenseK); + if (!x.ptr || !separate0.ptr || !separate1.ptr || !pair0.ptr || + !pair1.ptr || + !ds4_gpu_tensor_write(x.ptr, 0, activation.data(), x_bytes) || + !prepare_guard(x.ptr, x_bytes, 0x3000u)) { + std::fprintf(stderr, "pair N=%u: tensor setup failed\n", n_tokens); + return false; + } + const arm baseline = { + "two_dense_mmq", + [&](uint32_t set) { + return ds4_gpu_matmul_quant_tensor( + separate0.ptr, model.data, model.size, + model.weights[set].dense_offset, kQ4Type, kDenseK, + kDenseM, x.ptr, n_tokens) != 0 && + ds4_gpu_matmul_quant_tensor( + separate1.ptr, model.data, model.size, + model.weights[set].kv_offset, kQ4Type, kDenseK, + kKvM, x.ptr, n_tokens) != 0; + }}; + const arm candidate = { + "pair_mmq", + [&](uint32_t set) { + return ds4_gpu_matmul_q4_K_pair_tensor( + pair0.ptr, pair1.ptr, model.data, model.size, + model.weights[set].dense_offset, + model.weights[set].kv_offset, kDenseK, kDenseM, kKvM, + x.ptr, n_tokens) != 0; + }}; + return benchmark_pair_arms( + n_tokens, cfg, baseline, candidate, + [&]() { + return poison_output(separate0.ptr, out0_bytes, 0x7fc10001u, + 0x4000u) && + poison_output(separate1.ptr, out1_bytes, 0x7fc20002u, + 0x5000u) && + poison_output(pair0.ptr, out0_bytes, 0x7fc30003u, + 0x6000u) && + poison_output(pair1.ptr, out1_bytes, 0x7fc40004u, + 0x7000u); + }, + [&](uint32_t set) { + return bitwise_equal(separate0.ptr, pair0.ptr, out0_bytes, + "pair q_a output") && + bitwise_equal(separate1.ptr, pair1.ptr, out1_bytes, + "pair kv output") && + output_is_finite(separate0.ptr, out0_bytes, + "pair q_a output") && + output_is_finite(separate1.ptr, out1_bytes, + "pair kv output") && + sampled_cpu_oracle(separate0.ptr, model, + model.weights[set].dense_offset, + activation, n_tokens, kDenseK, kDenseM, + "pair q_a") && + sampled_cpu_oracle(separate1.ptr, model, + model.weights[set].kv_offset, + activation, n_tokens, kDenseK, kKvM, + "pair kv") && + check_guard(x.ptr, x_bytes, 0x3000u, "pair input") && + check_guard(separate0.ptr, out0_bytes, 0x4000u, + "pair separate q_a") && + check_guard(separate1.ptr, out1_bytes, 0x5000u, + "pair separate kv") && + check_guard(pair0.ptr, out0_bytes, 0x6000u, + "pair fused q_a") && + check_guard(pair1.ptr, out1_bytes, 0x7000u, + "pair fused kv"); + }) && + check_guard(x.ptr, x_bytes, 0x3000u, "pair input final") && + check_guard(separate0.ptr, out0_bytes, 0x4000u, + "pair separate q_a final") && + check_guard(separate1.ptr, out1_bytes, 0x5000u, + "pair separate kv final") && + check_guard(pair0.ptr, out0_bytes, 0x6000u, + "pair fused q_a final") && + check_guard(pair1.ptr, out1_bytes, 0x7000u, + "pair fused kv final"); +} + +bool run_qb(const model_fixture &model, const config &cfg, + uint32_t n_tokens) { + uint64_t out_elements = 0; + if (!checked_mul(n_tokens, kQbM, &out_elements)) return false; + const uint64_t x_bytes = static_cast(n_tokens) * kQbK * + sizeof(float); + const uint64_t out_bytes = out_elements * sizeof(float); + const uint64_t guard_bytes = kGuardWords * sizeof(uint32_t); + tensor_owner x(x_bytes + guard_bytes); + tensor_owner output(out_bytes + guard_bytes); + std::vector activation; + fill_activation(&activation, n_tokens, kQbK); + if (!x.ptr || !output.ptr || + !ds4_gpu_tensor_write(x.ptr, 0, activation.data(), x_bytes) || + !prepare_guard(x.ptr, x_bytes, 0x8000u)) { + std::fprintf(stderr, "q_b N=%u: tensor setup failed\n", n_tokens); + return false; + } + const arm path = { + cfg.path == cuda_path::mmq ? "mmq" : "legacy_q8k", + [&](uint32_t set) { + return ds4_gpu_matmul_quant_tensor( + output.ptr, model.data, model.size, + model.weights[set].qb_offset, kQ4Type, kQbK, kQbM, + x.ptr, n_tokens) != 0; + }}; + return benchmark_single_path( + "q_b", n_tokens, kQbK, kQbM, cfg, path, + [&]() { + return poison_output(output.ptr, out_bytes, 0x7fc10001u, + 0x9000u); + }, + [&](uint32_t set) { + return output_is_finite(output.ptr, out_bytes, "q_b output") && + sampled_cpu_oracle(output.ptr, model, + model.weights[set].qb_offset, + activation, n_tokens, kQbK, kQbM, + "q_b") && + check_guard(x.ptr, x_bytes, 0x8000u, "q_b input") && + check_guard(output.ptr, out_bytes, 0x9000u, + "q_b output"); + }) && + check_guard(x.ptr, x_bytes, 0x8000u, "q_b input final") && + check_guard(output.ptr, out_bytes, 0x9000u, "q_b output final"); +} + +void usage(FILE *stream, const char *argv0) { + std::fprintf( + stream, + "usage: %s [options]\n\n" + "Resident CUDA Q4_K prefill kernel benchmark (CUDA event timing).\n\n" + " --path mmq|legacy process-wide path (default: mmq)\n" + " --case all|dense|pair|qb case to run (default: all)\n" + " --tokens N[,N...] token counts, each 9..4096\n" + " --full use 9,16,17,31,32,33,128,512,4096\n" + " --sets N rotating resident weight sets (default: %u)\n" + " --samples N samples/arm, multiple of 4 (default: %u)\n" + " --warmup N untimed dispatches/arm (default: %u)\n" + " -h, --help show this help\n\n" + "Dense and q_b measure one immutable process path. Run separate " + "legacy/MMQ\nprocesses (preferably ABBA/BAAB) to compare them because " + "the CUDA backend\ncaches DS4_CUDA_MMQ on its first dispatch. Pair is " + "an in-process ABBA/BAAB\ncomparison of two MMQ projections against " + "the fused public pair API.\n", + argv0, kDefaultSets, kDefaultSamples, kDefaultWarmup); +} + +uint32_t parse_u32(const char *text, const char *option, uint32_t minimum, + uint32_t maximum) { + char *end = nullptr; + errno = 0; + const unsigned long value = std::strtoul(text, &end, 10); + if (errno != 0 || !text[0] || !end || *end || value < minimum || + value > maximum) { + std::fprintf(stderr, "invalid %s: %s\n", option, text); + std::exit(2); + } + return static_cast(value); +} + +const char *need_value(int *index, int argc, char **argv) { + if (*index + 1 >= argc) { + std::fprintf(stderr, "%s requires a value\n", argv[*index]); + std::exit(2); + } + return argv[++*index]; +} + +std::vector parse_tokens(const char *text) { + std::vector result; + const char *cursor = text; + while (*cursor) { + const char *comma = std::strchr(cursor, ','); + const std::string item(cursor, + comma ? static_cast(comma - cursor) + : std::strlen(cursor)); + result.push_back(parse_u32(item.c_str(), "--tokens", 9u, 4096u)); + if (!comma) break; + cursor = comma + 1; + if (!*cursor) { + std::fprintf(stderr, "invalid --tokens: trailing comma\n"); + std::exit(2); + } + } + if (result.empty()) { + std::fprintf(stderr, "--tokens cannot be empty\n"); + std::exit(2); + } + std::sort(result.begin(), result.end()); + result.erase(std::unique(result.begin(), result.end()), result.end()); + return result; +} + +config parse_options(int argc, char **argv) { + config cfg; + for (int i = 1; i < argc; i++) { + if (!std::strcmp(argv[i], "-h") || !std::strcmp(argv[i], "--help")) { + usage(stdout, argv[0]); + std::exit(0); + } else if (!std::strcmp(argv[i], "--path")) { + const char *value = need_value(&i, argc, argv); + if (!std::strcmp(value, "mmq")) cfg.path = cuda_path::mmq; + else if (!std::strcmp(value, "legacy")) { + cfg.path = cuda_path::legacy; + } else { + std::fprintf(stderr, "invalid --path: %s\n", value); + std::exit(2); + } + } else if (!std::strcmp(argv[i], "--case")) { + const char *value = need_value(&i, argc, argv); + if (!std::strcmp(value, "all")) cfg.selected = bench_case::all; + else if (!std::strcmp(value, "dense")) { + cfg.selected = bench_case::dense; + } else if (!std::strcmp(value, "pair")) { + cfg.selected = bench_case::pair; + } else if (!std::strcmp(value, "qb")) { + cfg.selected = bench_case::qb; + } else { + std::fprintf(stderr, "invalid --case: %s\n", value); + std::exit(2); + } + } else if (!std::strcmp(argv[i], "--tokens")) { + cfg.tokens = parse_tokens(need_value(&i, argc, argv)); + } else if (!std::strcmp(argv[i], "--full")) { + cfg.tokens = {9u, 16u, 17u, 31u, 32u, 33u, 128u, 512u, 4096u}; + } else if (!std::strcmp(argv[i], "--sets")) { + cfg.sets = parse_u32(need_value(&i, argc, argv), "--sets", 1u, + 32u); + } else if (!std::strcmp(argv[i], "--samples")) { + cfg.samples = parse_u32(need_value(&i, argc, argv), "--samples", + 4u, 1000u); + } else if (!std::strcmp(argv[i], "--warmup")) { + cfg.warmup = parse_u32(need_value(&i, argc, argv), "--warmup", + 0u, 100u); + } else { + std::fprintf(stderr, "unknown option: %s\n", argv[i]); + usage(stderr, argv[0]); + std::exit(2); + } + } + if ((cfg.samples % 4u) != 0u) { + std::fprintf(stderr, + "--samples must be a multiple of 4 for balanced runs\n"); + std::exit(2); + } + if (cfg.path == cuda_path::legacy && cfg.selected == bench_case::pair) { + std::fprintf(stderr, + "--case pair requires --path mmq for prefill N > 8\n"); + std::exit(2); + } + return cfg; +} + +bool includes(bench_case selected, bench_case wanted) { + return selected == bench_case::all || selected == wanted; +} + +bool install_resident_model(const model_fixture &model, + size_t *resident_delta, + bool *resident_delta_valid) { + size_t free_before = 0, total_before = 0; + size_t free_after = 0, total_after = 0; + const bool have_before = + cudaMemGetInfo(&free_before, &total_before) == cudaSuccess; + if (!have_before) (void)cudaGetLastError(); + if (!ds4_gpu_set_model_map(model.data, model.size) || + !ds4_gpu_synchronize()) { + return false; + } + const bool have_after = + cudaMemGetInfo(&free_after, &total_after) == cudaSuccess; + if (!have_after) (void)cudaGetLastError(); + (void)total_before; + (void)total_after; + *resident_delta_valid = have_before && have_after; + *resident_delta = *resident_delta_valid && free_before >= free_after + ? free_before - free_after : 0u; + return true; +} + +bool verify_resident_weight_ranges(const model_fixture &model) { + const uint64_t dense_bytes = q4_weight_bytes(kDenseK, kDenseM); + const uint64_t kv_bytes = q4_weight_bytes(kDenseK, kKvM); + const uint64_t qb_bytes = q4_weight_bytes(kQbK, kQbM); + for (uint32_t set = 0; set < model.weights.size(); set++) { + struct range_desc { + const char *name; + uint64_t offset; + uint64_t bytes; + }; + const range_desc ranges[] = { + {"dense", model.weights[set].dense_offset, dense_bytes}, + {"kv", model.weights[set].kv_offset, kv_bytes}, + {"q_b", model.weights[set].qb_offset, qb_bytes}, + }; + for (const range_desc &range : ranges) { + if (!ds4_cuda_test_model_range_is_device_resident( + model.data, model.size, range.offset, range.bytes, 0)) { + std::fprintf( + stderr, + "cuda-q4-prefill-bench: nonresident %s weight range " + "set=%u offset=%llu bytes=%llu\n", + range.name, set, + static_cast(range.offset), + static_cast(range.bytes)); + return false; + } + } + } + return true; +} + +bool verify_mmq_prefill_dispatch(const model_fixture &model) { + // For N > 8 the public pair API succeeds only through MMQ. Probe it + // before printing any timing to initialize and attest the process-wide + // decision. Strict mode separately rejects fallback on every measured + // dense, pair, and q_b dispatch. + constexpr uint32_t n_tokens = 9u; + const uint64_t x_bytes = static_cast(n_tokens) * kDenseK * + sizeof(float); + const uint64_t out0_bytes = static_cast(n_tokens) * kDenseM * + sizeof(float); + const uint64_t out1_bytes = static_cast(n_tokens) * kKvM * + sizeof(float); + tensor_owner x(x_bytes); + tensor_owner out0(out0_bytes); + tensor_owner out1(out1_bytes); + std::vector activation; + fill_activation(&activation, n_tokens, kDenseK); + return x.ptr && out0.ptr && out1.ptr && + ds4_gpu_tensor_write(x.ptr, 0, activation.data(), x_bytes) && + ds4_gpu_matmul_q4_K_pair_tensor( + out0.ptr, out1.ptr, model.data, model.size, + model.weights[0].dense_offset, model.weights[0].kv_offset, + kDenseK, kDenseM, kKvM, x.ptr, n_tokens) && + ds4_gpu_synchronize(); +} + +} // namespace + +int main(int argc, char **argv) { + const config cfg = parse_options(argc, argv); + env_snapshot mmq_guard("DS4_CUDA_MMQ"); + env_snapshot copy_guard("DS4_CUDA_COPY_MODEL"); + env_snapshot pair_guard("DS4_CUDA_DISABLE_Q4_DENSE_PAIR"); + env_snapshot graph_guard("DS4_CUDA_DECODE_GRAPHS"); + if (setenv("DS4_CUDA_MMQ", + cfg.path == cuda_path::mmq ? "1" : "0", 1) != 0 || + setenv("DS4_CUDA_COPY_MODEL", "1", 1) != 0 || + setenv("DS4_CUDA_DECODE_GRAPHS", "0", 1) != 0 || + unsetenv("DS4_CUDA_DISABLE_Q4_DENSE_PAIR") != 0) { + std::fprintf(stderr, + "cuda-q4-prefill-bench: environment setup failed\n"); + return 1; + } + + int device_count = 0; + const cudaError_t count_rc = cudaGetDeviceCount(&device_count); + if (count_rc != cudaSuccess || device_count <= 0) { + std::fprintf(stderr, + "cuda-q4-prefill-bench: no visible CUDA device (%s)\n", + count_rc == cudaSuccess ? "device count is zero" + : cudaGetErrorString(count_rc)); + return 77; + } + cudaDeviceProp properties{}; + if (cudaGetDeviceProperties(&properties, 0) != cudaSuccess) { + std::fprintf(stderr, + "cuda-q4-prefill-bench: cannot query device properties\n"); + return 1; + } + if (!ds4_gpu_init()) { + std::fprintf(stderr, "cuda-q4-prefill-bench: ds4_gpu_init failed\n"); + return 1; + } + ds4_cuda_test_set_q4_mmq_strict( + cfg.path == cuda_path::mmq ? 1 : 0); + + bool ok = true; + model_fixture model; + if (!make_model(&model, cfg.sets)) { + std::fprintf(stderr, + "cuda-q4-prefill-bench: model fixture allocation failed\n"); + ok = false; + } + size_t resident_delta = 0; + bool resident_delta_valid = false; + if (ok) { + ds4_gpu_set_quality(false); + ds4_gpu_set_ssd_streaming(false); + if (!install_resident_model( + model, &resident_delta, &resident_delta_valid)) { + std::fprintf( + stderr, + "cuda-q4-prefill-bench: model-map installation failed\n"); + ok = false; + } else if (!verify_resident_weight_ranges(model)) { + std::fprintf( + stderr, + "cuda-q4-prefill-bench: explicit CUDA model provenance " + "check failed; refusing PCIe/HMM-contaminated timings\n"); + ok = false; + } + } + if (ok && cfg.path == cuda_path::mmq && + !verify_mmq_prefill_dispatch(model)) { + std::fprintf(stderr, + "cuda-q4-prefill-bench: MMQ prefill proof probe failed; " + "refusing to label fallback timings as MMQ\n"); + ok = false; + } + + if (ok) { + std::printf( + "DS4_CUDA_Q4_PREFILL_SETUP device=%s cc=%d.%d warp=%d path=%s " + "sets=%u resident_payload_mib=%.2f device_free_delta_mib=%.2f " + "device_free_delta_valid=%d timing=cuda_events " + "ssd_streaming=off model_storage=cudaMalloc " + "residency=backend_provenance strict_mmq=%d " + "dispatch_stream=legacy_default\n", + properties.name, properties.major, properties.minor, + properties.warpSize, path_name(cfg.path), cfg.sets, + static_cast(model.payload_bytes) / 1048576.0, + static_cast(resident_delta) / 1048576.0, + resident_delta_valid ? 1 : 0, + cfg.path == cuda_path::mmq ? 1 : 0); + std::fflush(stdout); + for (uint32_t n_tokens : cfg.tokens) { + if (includes(cfg.selected, bench_case::dense)) { + ok = run_dense(model, cfg, n_tokens) && ok; + } + if (ok && includes(cfg.selected, bench_case::pair)) { + ok = run_pair(model, cfg, n_tokens) && ok; + } + if (ok && includes(cfg.selected, bench_case::qb)) { + ok = run_qb(model, cfg, n_tokens) && ok; + } + if (!ok) break; + } + } + + ds4_gpu_cleanup(); + std::fprintf(stderr, "cuda-q4-prefill-bench: %s\n", ok ? "PASS" : "FAIL"); + return ok ? 0 : 1; +} From ab6486fe522a7c86f808807d74c903fedea08540 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Fri, 28 Aug 2026 19:49:53 +0200 Subject: [PATCH 137/189] perf(rocm): add compressed Q4 WMMA prefill kernel --- ENVIRONMENT_VARIABLES.md | 72 ++--- rocm/ds4_rocm_q4.cuh | 458 +++++++++++++++++++++++++++++- scripts/environment_variables.tsv | 66 +++-- tests/test_rocm_q4_dense_pair.cpp | 323 ++++++++++++++++++++- 4 files changed, 845 insertions(+), 74 deletions(-) diff --git a/ENVIRONMENT_VARIABLES.md b/ENVIRONMENT_VARIABLES.md index 05657ecd3..e15ecd976 100644 --- a/ENVIRONMENT_VARIABLES.md +++ b/ENVIRONMENT_VARIABLES.md @@ -81,6 +81,10 @@ The detailed Metal A/B contracts and expected oracle counters live in | `DS4_ROCM_DISABLE_Q4_PREFILL_TILE8=1` | Restore the legacy Q4 prefill kernel. TILE8 is automatic for validated chunks of 9 through 4096 tokens. | | `DS4_ROCM_REQUIRE_Q4_PREFILL_TILE8=1` | Fail closed when an eligible Q4 prefill call cannot use TILE8. | | `DS4_ROCM_ENABLE_Q4_PREFILL_TILE8=1` | Legacy spelling retained for migration notes only. The runtime does not read it; TILE8 is automatic and this setting is ignored. | +| `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA=1` | Opt into the experimental resident gfx1151 wave32 Q4_K WMMA64 prefill kernel for 256–4096 tokens. It replaces Q8_K activation scratch with transient F16 register dequantization and F32 accumulation. | +| `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_SSD=1` | Allow that WMMA64 path during SSD streaming only when the complete projection weight range is already backed by physical device storage. It never treats mapped/registered host memory as resident. | +| `DS4_ROCM_DISABLE_Q4_PREFILL_WMMA=1` | Dominant value-aware rollback for the WMMA64 experiment. | +| `DS4_ROCM_REQUIRE_Q4_PREFILL_WMMA=1` | Request WMMA64 and fail closed on an unsupported device/shape, quality mode, rollback, or SSD weight range that is not physically device-resident. Intended for strict A/B oracles. | | `DS4_ROCM_Q4_PREFILL_TILE8_STATS=1` | Report dense, pair, attention-batch, and token counters at process exit. | | `DS4_ROCM_ENABLE_Q4_DENSE_PAIR=1` | Share one Q8_K activation quantization between the two Q4 dense projections. This pair remains opt-in. | | `DS4_ROCM_DISABLE_Q4_DENSE_PAIR=1` | Dominant rollback for the ROCm Q4 dense pair. | @@ -928,7 +932,7 @@ and **19 tool/wrapper entries**.
-ROCm (140) +ROCm (144) | Variable | Accepted value and default | Effect | Source | | --- | --- | --- | --- | @@ -943,35 +947,36 @@ and **19 tool/wrapper entries**. | `DS4_ROCM_DISABLE_Q4_DENSE_PAIR` | presence rollback; unset leaves opt-in policy unchanged | Disable/roll back rocm disable q4 dense pair. | [rocm/ds4_rocm_q4.cuh:435](rocm/ds4_rocm_q4.cuh#L435) | | `DS4_ROCM_DISABLE_Q4_GROUPED_ATTN_A` | presence rollback; unset permits the caller-marked resident decode production-shape default and explicit ENABLE/REQUIRE; any defined value including empty or 0 disables all grouped attention-A paths and wins over ENABLE/REQUIRE | Restore eight standalone Q4 attention-A projections instead of the two-dispatch grouped path. | [rocm/ds4_rocm_q4.cuh:868](rocm/ds4_rocm_q4.cuh#L868) | | `DS4_ROCM_DISABLE_Q4_PREFILL_TILE8` | presence rollback; TILE8 is default for 9..4096 tokens | Disable/roll back rocm disable q4 prefill tile8. | [rocm/ds4_rocm_q4.cuh:448](rocm/ds4_rocm_q4.cuh#L448) | -| `DS4_ROCM_DISABLE_Q4_SELECTED_EXPERT_VIEWS` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable q4 selected expert views. | [ds4.c:21135](ds4.c#L21135) | -| `DS4_ROCM_DISABLE_RESIDENT_IQ2_SORTED` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable resident iq2 sorted. | [rocm/ds4_rocm_moe_launch.cuh:747](rocm/ds4_rocm_moe_launch.cuh#L747) | -| `DS4_ROCM_DISABLE_ROUTED_PAIR_SWIGLU_FUSION` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable routed pair swiglu fusion. | [ds4.c:18528](ds4.c#L18528) | -| `DS4_ROCM_DISABLE_STREAMING_COLD_DECODE_PREFILL` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming cold decode prefill. | [ds4.c:32006](ds4.c#L32006) | -| `DS4_ROCM_DISABLE_STREAMING_DECODE_PREFILL` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming decode prefill. | [ds4.c:31955](ds4.c#L31955) | -| `DS4_ROCM_DISABLE_STREAMING_EXPERT_ADDR_TABLE` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming expert addr table. | [ds4.c:18524](ds4.c#L18524) | -| `DS4_ROCM_DISABLE_STREAMING_EXPERT_HOTLIST` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming expert hotlist. | [ds4.c:21289](ds4.c#L21289) | -| `DS4_ROCM_DISABLE_STREAMING_FULL_EXPERT_ADDR_TABLE` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming full expert addr table. | [ds4.c:18242](ds4.c#L18242) | -| `DS4_ROCM_DISABLE_STREAMING_LAYER_BATCH` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming layer batch. | [ds4.c:18238](ds4.c#L18238) | -| `DS4_ROCM_DISABLE_STREAMING_MADVISE_WILLNEED` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming madvise willneed. | [ds4.c:18211](ds4.c#L18211) | -| `DS4_ROCM_DISABLE_STREAMING_PREFILL_BATCH_SELECTED_ADDR` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill batch selected addr. | [ds4.c:18522](ds4.c#L18522) | -| `DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_MADVISE` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill layer madvise. | [ds4.c:18465](ds4.c#L18465) | -| `DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_PAGEIN` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill layer pagein. | [ds4.c:18433](ds4.c#L18433) | -| `DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_PAGEIN_OVERLAP` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill layer pagein overlap. | [ds4.c:19320](ds4.c#L19320) | -| `DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_PREAD` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill layer pread. | [ds4.c:18453](ds4.c#L18453) | -| `DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_PREPARE` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill layer prepare. | [ds4.c:18445](ds4.c#L18445) | -| `DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_PREPARE_OVERLAP` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill layer prepare overlap. | [ds4.c:19318](ds4.c#L19318) | -| `DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_READAHEAD` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill layer readahead. | [ds4.c:18443](ds4.c#L18443) | -| `DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_ASYNC_LOAD` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill selected async load. | [ds4.c:46423](ds4.c#L46423) | -| `DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_MADVISE` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill selected madvise. | [ds4.c:18423](ds4.c#L18423) | -| `DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_PAGEIN` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill selected pagein. | [ds4.c:18413](ds4.c#L18413) | -| `DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_PROFILE` | presence rollback flag; unset keeps automatic/default path | Collect timing/profile diagnostics for rocm disable streaming prefill selected profile. | [ds4.c:18907](ds4.c#L18907) | -| `DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_READAHEAD` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill selected readahead. | [ds4.c:19959](ds4.c#L19959) | -| `DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_READAHEAD_SHARED` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill selected readahead shared. | [ds4.c:19969](ds4.c#L19969) | -| `DS4_ROCM_DISABLE_STREAMING_READAHEAD` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming readahead. | [ds4.c:18204](ds4.c#L18204) | -| `DS4_ROCM_DISABLE_STREAMING_SELECTED_ASYNC_LOAD` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming selected async load. | [ds4.c:44326](ds4.c#L44326) | -| `DS4_ROCM_DISABLE_STREAMING_SPLIT_SELECTED` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming split selected. | [rocm/ds4_rocm_moe_launch.cuh:661](rocm/ds4_rocm_moe_launch.cuh#L661) | -| `DS4_ROCM_DISABLE_STREAMING_STATIC_DECODE_MAP` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming static decode map. | [ds4.c:18216](ds4.c#L18216) | -| `DS4_ROCM_DISABLE_STREAMING_STATIC_MAP_STATE_CACHE` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming static map state cache. | [ds4.c:18229](ds4.c#L18229) | +| `DS4_ROCM_DISABLE_Q4_PREFILL_WMMA` | value-aware authoritative rollback for the experimental path; unset/0/false/no/off permits ENABLE or REQUIRE, while empty or any other value disables; REQUIRE then fails closed | Prevent the gfx1151 Q4_K prefill WMMA64 experiment from dispatching and retain the Q8_K-plus-TILE8/TILE4 path. | [rocm/ds4_rocm_q4.cuh:775](rocm/ds4_rocm_q4.cuh#L775) | +| `DS4_ROCM_DISABLE_Q4_SELECTED_EXPERT_VIEWS` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable q4 selected expert views. | [ds4.c:21150](ds4.c#L21150) | +| `DS4_ROCM_DISABLE_RESIDENT_IQ2_SORTED` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable resident iq2 sorted. | [rocm/ds4_rocm_moe_launch.cuh:751](rocm/ds4_rocm_moe_launch.cuh#L751) | +| `DS4_ROCM_DISABLE_ROUTED_PAIR_SWIGLU_FUSION` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable routed pair swiglu fusion. | [ds4.c:18543](ds4.c#L18543) | +| `DS4_ROCM_DISABLE_STREAMING_COLD_DECODE_PREFILL` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming cold decode prefill. | [ds4.c:32021](ds4.c#L32021) | +| `DS4_ROCM_DISABLE_STREAMING_DECODE_PREFILL` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming decode prefill. | [ds4.c:31970](ds4.c#L31970) | +| `DS4_ROCM_DISABLE_STREAMING_EXPERT_ADDR_TABLE` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming expert addr table. | [ds4.c:18539](ds4.c#L18539) | +| `DS4_ROCM_DISABLE_STREAMING_EXPERT_HOTLIST` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming expert hotlist. | [ds4.c:21304](ds4.c#L21304) | +| `DS4_ROCM_DISABLE_STREAMING_FULL_EXPERT_ADDR_TABLE` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming full expert addr table. | [ds4.c:18257](ds4.c#L18257) | +| `DS4_ROCM_DISABLE_STREAMING_LAYER_BATCH` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming layer batch. | [ds4.c:18253](ds4.c#L18253) | +| `DS4_ROCM_DISABLE_STREAMING_MADVISE_WILLNEED` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming madvise willneed. | [ds4.c:18226](ds4.c#L18226) | +| `DS4_ROCM_DISABLE_STREAMING_PREFILL_BATCH_SELECTED_ADDR` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill batch selected addr. | [ds4.c:18537](ds4.c#L18537) | +| `DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_MADVISE` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill layer madvise. | [ds4.c:18480](ds4.c#L18480) | +| `DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_PAGEIN` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill layer pagein. | [ds4.c:18448](ds4.c#L18448) | +| `DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_PAGEIN_OVERLAP` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill layer pagein overlap. | [ds4.c:19335](ds4.c#L19335) | +| `DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_PREAD` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill layer pread. | [ds4.c:18468](ds4.c#L18468) | +| `DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_PREPARE` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill layer prepare. | [ds4.c:18460](ds4.c#L18460) | +| `DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_PREPARE_OVERLAP` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill layer prepare overlap. | [ds4.c:19333](ds4.c#L19333) | +| `DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_READAHEAD` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill layer readahead. | [ds4.c:18458](ds4.c#L18458) | +| `DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_ASYNC_LOAD` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill selected async load. | [ds4.c:46684](ds4.c#L46684) | +| `DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_MADVISE` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill selected madvise. | [ds4.c:18438](ds4.c#L18438) | +| `DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_PAGEIN` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill selected pagein. | [ds4.c:18428](ds4.c#L18428) | +| `DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_PROFILE` | presence rollback flag; unset keeps automatic/default path | Collect timing/profile diagnostics for rocm disable streaming prefill selected profile. | [ds4.c:18922](ds4.c#L18922) | +| `DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_READAHEAD` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill selected readahead. | [ds4.c:19974](ds4.c#L19974) | +| `DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_READAHEAD_SHARED` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill selected readahead shared. | [ds4.c:19984](ds4.c#L19984) | +| `DS4_ROCM_DISABLE_STREAMING_READAHEAD` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming readahead. | [ds4.c:18219](ds4.c#L18219) | +| `DS4_ROCM_DISABLE_STREAMING_SELECTED_ASYNC_LOAD` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming selected async load. | [ds4.c:44587](ds4.c#L44587) | +| `DS4_ROCM_DISABLE_STREAMING_SPLIT_SELECTED` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming split selected. | [rocm/ds4_rocm_moe_launch.cuh:665](rocm/ds4_rocm_moe_launch.cuh#L665) | +| `DS4_ROCM_DISABLE_STREAMING_STATIC_DECODE_MAP` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming static decode map. | [ds4.c:18231](ds4.c#L18231) | +| `DS4_ROCM_DISABLE_STREAMING_STATIC_MAP_STATE_CACHE` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming static map state cache. | [ds4.c:18244](ds4.c#L18244) | | `DS4_ROCM_DSV4_PREQUANT_DECODE` | sampled once; unset: enabled; present empty or exact 0: disabled; every other present value: enabled; quality mode and GLM models force it off regardless | ROCm DeepSeek-V4 decode: quantizes one-token F32 activations to Q8 once and selects the prequantized Q8_0/DP4A projection kernels instead of the full-F32 activation paths. | [rocm/ds4_rocm_runtime.cuh:4775](rocm/ds4_rocm_runtime.cuh#L4775) | | `DS4_ROCM_ENABLE_MXFP4_LDSB` | presence opt-in; unset=off; any defined value including empty or 0 enables the candidate when the MXFP4 path, sorted expert tiles, token count >= 128, LDS-size limit, and dimension-alignment gates all pass | Select the ROCm MXFP4 prefill gate/up kernel that stages eight gate and eight up weight rows in LDS and reuses them across expert tiles of up to 128 tokens. | [rocm/ds4_rocm_moe_launch.cuh:782](rocm/ds4_rocm_moe_launch.cuh#L782) | | `DS4_ROCM_ENABLE_MXFP4_ROW64` | presence opt-in; unset=off; any defined value including empty or 0 enables the candidate when the MXFP4 sorted-tile path has at least 8 tokens and the TILE32, LDSB, and TILE4 candidates are not selected | Select the ROCm MXFP4 gate/up tile8 occupancy variant with 64 row slots and 512 threads per block. | [rocm/ds4_rocm_moe_launch.cuh:798](rocm/ds4_rocm_moe_launch.cuh#L798) | @@ -979,6 +984,8 @@ and **19 tool/wrapper entries**. | `DS4_ROCM_ENABLE_MXFP4_TILE4` | presence opt-in; unset=off; any defined value including empty or 0 enables the candidate when the MXFP4 sorted-tile path has at least 5 tokens and neither TILE32 nor LDSB is selected | Select the ROCm MXFP4 gate/up tile4 occupancy variant, reducing staged-activation LDS per block. | [rocm/ds4_rocm_moe_launch.cuh:794](rocm/ds4_rocm_moe_launch.cuh#L794) | | `DS4_ROCM_ENABLE_Q4_DENSE_PAIR` | presence opt-in; unset=off; DISABLE takes precedence | Enable rocm enable q4 dense pair. | [rocm/ds4_rocm_q4.cuh:434](rocm/ds4_rocm_q4.cuh#L434) | | `DS4_ROCM_ENABLE_Q4_GROUPED_ATTN_A` | presence opt-in outside the default scope; the exact caller-marked resident decode shape groups=8, N=1, K=4096, M=1024 is automatic, while row-at-a-time batch fallbacks are not; DISABLE wins | Enable grouped Q4 attention-A for eligible slices, non-production shapes, or explicit experiments in addition to the resident decode default. | [rocm/ds4_rocm_q4.cuh:872](rocm/ds4_rocm_q4.cuh#L872) | +| `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA` | value-aware opt-in, default off; unset/0/false/no/off retains the canonical path; empty or any other value requests WMMA64 only for N=256..4096, K a positive multiple of 256, resident non-quality execution on gfx1151 wave32; DISABLE wins | Benchmark the compressed Q4_K-to-F16 register dequantization plus 64x64 WMMA prefill kernel without Q8_K activation scratch or an F16 weight sidecar. | [rocm/ds4_rocm_q4.cuh:771](rocm/ds4_rocm_q4.cuh#L771) | +| `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_SSD` | value-aware SSD-only opt-in, default off; unset/0/false/no/off retains TILE8/TILE4, while empty or any other value requests WMMA64; eligibility additionally requires the complete projection weight range in physical device storage rather than mapped/registered host memory; DISABLE wins | Allow the compressed WMMA64 kernel to consume an already device-resident/cache-backed Q4_K projection during SSD streaming without changing model I/O. | [rocm/ds4_rocm_q4.cuh:773](rocm/ds4_rocm_q4.cuh#L773) | | `DS4_ROCM_ENABLE_STREAMING_FULL_EXPERT_ADDR_TABLE` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming full expert addr table. | [ds4.c:18255](ds4.c#L18255) | | `DS4_ROCM_ENABLE_STREAMING_MADVISE_WILLNEED` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming madvise willneed. | [ds4.c:18224](ds4.c#L18224) | | `DS4_ROCM_ENABLE_STREAMING_PREFILL_BATCH_SELECTED_ADDR` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming prefill batch selected addr. | [ds4.c:18584](ds4.c#L18584) | @@ -1036,8 +1043,9 @@ and **19 tool/wrapper entries**. | `DS4_ROCM_Q_STAGE_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm q stage profile. | [ds4.c:29284](ds4.c#L29284) | | `DS4_ROCM_REQUIRE_Q4_GROUPED_ATTN_A` | presence fail-closed assertion; also requests the candidate outside the caller-marked resident decode default; DISABLE remains authoritative and causes failure | Require grouped Q4 attention-A and fail instead of silently falling back. | [rocm/ds4_rocm_q4.cuh:870](rocm/ds4_rocm_q4.cuh#L870) | | `DS4_ROCM_REQUIRE_Q4_PREFILL_TILE8` | presence fail-closed assertion for eligible TILE8 calls | Require rocm require q4 prefill tile8 and fail instead of silently falling back. | [rocm/ds4_rocm_q4.cuh:452](rocm/ds4_rocm_q4.cuh#L452) | -| `DS4_ROCM_STREAMING_DECODE_PREFILL_MAX` | primary nonempty value over the Metal alias; parsed by strtol when it has a numeric prefix (trailing text is accepted); <= 0 disables, values > UINT32_MAX clamp, no numeric prefix uses automatic default: 64 for Flash with uniform Q4_K/MXFP4 experts, 18 for other Pro/Flash, otherwise 0; the disable flag dominates | Sets the largest short, non-quality SSD-streaming prefill batch routed through the decode-style path instead of canonical layer-major prefill. | [ds4.c:31961](ds4.c#L31961) | -| `DS4_ROCM_STREAMING_EXPERT_AUTO_PRELOAD_CAP` | primary nonempty value over the Metal alias; strict full-string strtoul; valid values > UINT32_MAX clamp, invalid uses 4096, and 0 means no cap (not disabled); when CLI preload is auto/0, unset defaults to cap 4096 except ROCm GLM52, where absent/empty disables automatic preload entirely | Caps the number of hot experts synchronously seeded into the SSD-streaming expert cache in automatic preload mode; an explicit CLI preload count bypasses this cap, and setting this variable opts ROCm GLM52 back into auto preload. | [ds4.c:21454](ds4.c#L21454) | +| `DS4_ROCM_REQUIRE_Q4_PREFILL_WMMA` | value-aware strict opt-in; unset/0/false/no/off is off; empty or any other value requires every selected Q4 dense or attention-output projection to use WMMA64; unsupported shape/device, quality mode, DISABLE, or an SSD weight range without physical device residency fails before the relevant dispatch | Prevent the ROCm Q4 prefill WMMA64 correctness/performance oracle from silently timing TILE8/TILE4; the fused q_a/KV pair yields to separately checked dense calls. | [rocm/ds4_rocm_q4.cuh:777](rocm/ds4_rocm_q4.cuh#L777) | +| `DS4_ROCM_STREAMING_DECODE_PREFILL_MAX` | primary nonempty value over the Metal alias; parsed by strtol when it has a numeric prefix (trailing text is accepted); <= 0 disables, values > UINT32_MAX clamp, no numeric prefix uses automatic default: 64 for Flash with uniform Q4_K/MXFP4 experts, 18 for other Pro/Flash, otherwise 0; the disable flag dominates | Sets the largest short, non-quality SSD-streaming prefill batch routed through the decode-style path instead of canonical layer-major prefill. | [ds4.c:31976](ds4.c#L31976) | +| `DS4_ROCM_STREAMING_EXPERT_AUTO_PRELOAD_CAP` | primary nonempty value over the Metal alias; strict full-string strtoul; valid values > UINT32_MAX clamp, invalid uses 4096, and 0 means no cap (not disabled); when CLI preload is auto/0, unset defaults to cap 4096 except ROCm GLM52, where absent/empty disables automatic preload entirely | Caps the number of hot experts synchronously seeded into the SSD-streaming expert cache in automatic preload mode; an explicit CLI preload count bypasses this cap, and setting this variable opts ROCm GLM52 back into auto preload. | [ds4.c:21469](ds4.c#L21469) | | `DS4_ROCM_STREAMING_EXPERT_CACHE_VERBOSE` | presence flag; unset=off | Print verbose ROCm streaming expert-cache seed/load diagnostics. | [rocm/ds4_rocm_runtime.cuh:2904](rocm/ds4_rocm_runtime.cuh#L2904) | | `DS4_ROCM_STREAMING_EXPERT_HOTLIST` | Nonempty filesystem path; a nonempty ROCm value takes precedence, otherwise DS4_METAL_STREAMING_EXPERT_HOTLIST is a fallback. The file contains whitespace-separated layer expert hits rows; blank/comment lines are ignored, zero-hit rows skipped, malformed/open/read errors fail seeding. Unset/empty uses the built-in Pro/Flash/GLM52 hotlist. Effective only when non-cold SSD hotlist seeding is enabled and cache/preload budget is nonzero. | Select a custom ranked expert hotlist used to preseed the streaming resident expert cache before decode. | [ds4.c:21494](ds4.c#L21494) | | `DS4_ROCM_STREAMING_EXPERT_HOTLIST_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm streaming expert hotlist profile. | [ds4.c:32244](ds4.c#L32244) | diff --git a/rocm/ds4_rocm_q4.cuh b/rocm/ds4_rocm_q4.cuh index 19caecae2..c71def35a 100644 --- a/rocm/ds4_rocm_q4.cuh +++ b/rocm/ds4_rocm_q4.cuh @@ -123,6 +123,188 @@ enum { static_assert((sizeof(cuda_block_q8_K) % sizeof(uint32_t)) == 0u, "ROCm Q8_K LDS copies require a whole number of words"); +#if defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__) +/* Experimental compressed Q4_K MMQ for resident gfx1151 prefill. + * + * This deliberately mirrors the live Q8 WMMA kernel: four wave32s compute a + * 64-token x 64-row output tile, K advances by 32, only the 64x32 activation + * tile is staged in 4 KiB of LDS, and accumulator fragments are written + * straight to the output. Each lane dequantizes one Q4_K row/group directly + * into two F16 register vectors, so there is no Q8_K activation scratch and no + * persistent F16 weight sidecar. + * + * Arithmetic is not bit-identical to Q4_K x Q8_K: activations and transient + * weights round to F16 before F32 WMMA accumulation. Host policy therefore + * keeps this path opt-in, physically device-resident, gfx1151-only, and out of + * quality mode until device-side A/B plus a prompt-level oracle promote it. + */ +enum { + ROCM_Q4_WMMA_M_TILE = 64u, + ROCM_Q4_WMMA_N_TILE = 64u, + ROCM_Q4_WMMA_K_TILE = 32u, + ROCM_Q4_WMMA_WAVES = 4u, + ROCM_Q4_WMMA_FRAGMENT = 16u, +}; + +typedef _Float16 __attribute__((ext_vector_type(16))) ds4_q4_half16_t; +typedef float __attribute__((ext_vector_type(8))) ds4_q4_float8_t; +typedef uint8_t __attribute__((ext_vector_type(16))) ds4_q4_uchar16_t; + +__launch_bounds__(128, 2) +__global__ static void rocm_matmul_q4_K_prefill_wmma64_strided_kernel( + float *out, + const char *w_base, + const float *x, + uint32_t n_tok, + uint32_t n_groups, + uint32_t in_dim, + uint32_t out_dim, + uint64_t row_bytes, + uint64_t x_token_stride, + uint64_t x_group_stride, + uint64_t out_token_stride) { + if (warpSize != 32) return; + + const uint32_t tid = threadIdx.x; + const uint32_t wave = tid >> 5u; + const uint32_t lane = tid & 31u; + const uint32_t lane16 = lane & 15u; + const uint32_t group = blockIdx.z; + const uint32_t row0 = blockIdx.x * ROCM_Q4_WMMA_N_TILE; + const uint32_t tok0 = blockIdx.y * ROCM_Q4_WMMA_M_TILE; + if (group >= n_groups) return; + + const uint32_t wave_row0 = row0 + wave * ROCM_Q4_WMMA_FRAGMENT; + const uint32_t my_row = wave_row0 + lane16; + const uint32_t safe_row = my_row < out_dim ? my_row : out_dim - 1u; + const cuda_block_q4_K *row_blocks = + reinterpret_cast( + w_base + ((uint64_t)group * out_dim + safe_row) * row_bytes); + const uint32_t q4_blocks = in_dim / CUDA_QK_K; + + ds4_q4_float8_t acc0 = {0.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 0.0f}; + ds4_q4_float8_t acc1 = acc0; + ds4_q4_float8_t acc2 = acc0; + ds4_q4_float8_t acc3 = acc0; + __shared__ _Float16 lds_x[ROCM_Q4_WMMA_M_TILE * ROCM_Q4_WMMA_K_TILE]; + + for (uint32_t block_index = 0u; block_index < q4_blocks; + block_index++) { + const cuda_block_q4_K *block = row_blocks + block_index; + const float block_d = dev_f16_to_f32(block->d); + const float block_dm = dev_f16_to_f32(block->dmin); + for (uint32_t qpair = 0u; qpair < 4u; qpair++) { + /* Adjacent 32-value groups are the low/high nibbles of the same + * 32 payload bytes. Keep them in registers across both K tiles + * instead of issuing the same global loads twice. */ + ds4_q4_uchar16_t packed0; + ds4_q4_uchar16_t packed1; + __builtin_memcpy( + &packed0, block->qs + qpair * 32u, sizeof(packed0)); + __builtin_memcpy( + &packed1, + block->qs + qpair * 32u + ROCM_Q4_WMMA_FRAGMENT, + sizeof(packed1)); + + #pragma unroll + for (uint32_t nibble = 0u; nibble < 2u; nibble++) { + const uint32_t qgroup = qpair * 2u + nibble; + const uint32_t group32 = block_index * 8u + qgroup; + for (uint32_t j = tid; + j < ROCM_Q4_WMMA_M_TILE * ROCM_Q4_WMMA_K_TILE; + j += blockDim.x) { + const uint32_t tok_local = j >> 5u; + const uint32_t kk = j & 31u; + const uint32_t tok = tok0 + tok_local; + float value = 0.0f; + if (tok < n_tok) { + value = x[(uint64_t)tok * x_token_stride + + (uint64_t)group * x_group_stride + + (uint64_t)group32 * ROCM_Q4_WMMA_K_TILE + + kk]; + } + lds_x[j] = (_Float16)value; + } + __syncthreads(); + + uint8_t scale = 0u; + uint8_t minimum = 0u; + dev_q4_K_get_scale_min( + qgroup, block->scales, &scale, &minimum); + const float d = block_d * (float)scale; + const float dm = block_dm * (float)minimum; + const uint32_t shift = nibble * 4u; + ds4_q4_half16_t weights0; + ds4_q4_half16_t weights1; + #pragma unroll + for (uint32_t i = 0u; i < ROCM_Q4_WMMA_FRAGMENT; i++) { + const uint8_t q0 = (packed0[i] >> shift) & 0x0fu; + const uint8_t q1 = (packed1[i] >> shift) & 0x0fu; + weights0[i] = (_Float16)(d * (float)q0 - dm); + weights1[i] = (_Float16)(d * (float)q1 - dm); + } + + #pragma unroll + for (uint32_t token_tile = 0u; token_tile < 4u; + token_tile++) { + const uint32_t token_local = + token_tile * ROCM_Q4_WMMA_FRAGMENT + lane16; + const _Float16 *activation = + lds_x + token_local * ROCM_Q4_WMMA_K_TILE; + const ds4_q4_half16_t activation0 = + *reinterpret_cast(activation); + const ds4_q4_half16_t activation1 = + *reinterpret_cast( + activation + ROCM_Q4_WMMA_FRAGMENT); + if (token_tile == 0u) { + acc0 = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32( + weights0, activation0, acc0); + acc0 = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32( + weights1, activation1, acc0); + } else if (token_tile == 1u) { + acc1 = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32( + weights0, activation0, acc1); + acc1 = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32( + weights1, activation1, acc1); + } else if (token_tile == 2u) { + acc2 = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32( + weights0, activation0, acc2); + acc2 = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32( + weights1, activation1, acc2); + } else { + acc3 = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32( + weights0, activation0, acc3); + acc3 = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32( + weights1, activation1, acc3); + } + } + __syncthreads(); + } + } + } + + #pragma unroll + for (uint32_t token_tile = 0u; token_tile < 4u; token_tile++) { + const uint32_t tok = + tok0 + token_tile * ROCM_Q4_WMMA_FRAGMENT + lane16; + if (tok >= n_tok) continue; + const ds4_q4_float8_t acc = token_tile == 0u + ? acc0 + : (token_tile == 1u ? acc1 + : (token_tile == 2u ? acc2 : acc3)); + #pragma unroll + for (uint32_t j = 0u; j < 8u; j++) { + const uint32_t row = wave_row0 + 2u * j + (lane >> 4u); + if (row < out_dim) { + out[(uint64_t)tok * out_token_stride + + (uint64_t)group * out_dim + row] = acc[j]; + } + } + } +} +#endif + __device__ __forceinline__ static void rocm_dot_q4_K_q8_K_block8_reuse_weights( const cuda_block_q4_K *x, @@ -561,6 +743,120 @@ static int rocm_q4_K_prefill_tile8_required(void) { return getenv("DS4_ROCM_REQUIRE_Q4_PREFILL_TILE8") != NULL; } +enum { + ROCM_Q4_PREFILL_WMMA_REQUIRED_FAILURE = -1, + ROCM_Q4_PREFILL_WMMA_FALLBACK = 0, + ROCM_Q4_PREFILL_WMMA_USE = 1, +}; + +/* Test oracle for strict dispatch: REQUIRE must attest this launch wrapper, + * not merely produce output through a canonical fallback. */ +static uint64_t g_rocm_q4_prefill_wmma_launches; + +extern "C" void ds4_rocm_test_q4_prefill_wmma_reset(void) { + __atomic_store_n(&g_rocm_q4_prefill_wmma_launches, 0u, + __ATOMIC_RELAXED); +} + +extern "C" uint64_t ds4_rocm_test_q4_prefill_wmma_get_calls(void) { + return __atomic_load_n(&g_rocm_q4_prefill_wmma_launches, + __ATOMIC_RELAXED); +} + +static int rocm_q4_K_prefill_wmma_select( + uint64_t n_tok, + uint64_t in_dim, + uint64_t out_dim, + int weight_device_resident) { + const int enabled = rocm_q4_attn_q_b_env_bool( + "DS4_ROCM_ENABLE_Q4_PREFILL_WMMA") == 1; + const int ssd_enabled = rocm_q4_attn_q_b_env_bool( + "DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_SSD") == 1; + const int disabled = rocm_q4_attn_q_b_env_bool( + "DS4_ROCM_DISABLE_Q4_PREFILL_WMMA") == 1; + const int required = rocm_q4_attn_q_b_env_bool( + "DS4_ROCM_REQUIRE_Q4_PREFILL_WMMA") == 1; + const int requested = enabled || required || + (g_ssd_streaming_mode && ssd_enabled); + if (!requested) return ROCM_Q4_PREFILL_WMMA_FALLBACK; + + const int shape_ok = + n_tok >= 256u && n_tok <= 4096u && + in_dim != 0u && (in_dim % CUDA_QK_K) == 0u && + out_dim != 0u && in_dim <= UINT32_MAX && + out_dim <= UINT32_MAX && n_tok <= UINT32_MAX; + const int storage_ok = !g_ssd_streaming_mode || + ((ssd_enabled || required) && + weight_device_resident); + const int eligible = !disabled && shape_ok && storage_ok && + !g_quality_mode && rocm_attention_runtime_is_gfx1151_wave32(); + if (eligible) return ROCM_Q4_PREFILL_WMMA_USE; + if (!required) return ROCM_Q4_PREFILL_WMMA_FALLBACK; + + fprintf(stderr, + DS4_GPU_LOG_PREFIX + "required Q4_K prefill WMMA is unavailable " + "(N=%llu K=%llu M=%llu disabled=%d ssd=%d ssd_opt=%d " + "resident=%d quality=%d)\n", + (unsigned long long)n_tok, + (unsigned long long)in_dim, + (unsigned long long)out_dim, + disabled, + g_ssd_streaming_mode ? 1 : 0, + ssd_enabled, + weight_device_resident, + g_quality_mode ? 1 : 0); + return ROCM_Q4_PREFILL_WMMA_REQUIRED_FAILURE; +} + +static int rocm_q4_K_prefill_wmma_launch( + float *out, + const char *w, + const float *x, + uint32_t n_tok, + uint32_t n_groups, + uint32_t in_dim, + uint32_t out_dim, + uint64_t row_bytes, + uint64_t x_token_stride, + uint64_t x_group_stride, + uint64_t out_token_stride, + const char *label) { +#if defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__) + if (!out || !w || !x || n_groups == 0u) return 0; + const dim3 grid( + (unsigned)(((uint64_t)out_dim + ROCM_Q4_WMMA_N_TILE - 1u) / + ROCM_Q4_WMMA_N_TILE), + (unsigned)(((uint64_t)n_tok + ROCM_Q4_WMMA_M_TILE - 1u) / + ROCM_Q4_WMMA_M_TILE), + n_groups); + rocm_matmul_q4_K_prefill_wmma64_strided_kernel<<>>( + out, w, x, n_tok, n_groups, in_dim, out_dim, row_bytes, + x_token_stride, x_group_stride, out_token_stride); + const int ok = cuda_ok( + cudaGetLastError(), label ? label : "q4_K prefill WMMA64 launch"); + if (ok) { + __atomic_fetch_add(&g_rocm_q4_prefill_wmma_launches, 1u, + __ATOMIC_RELAXED); + } + return ok; +#else + (void)out; + (void)w; + (void)x; + (void)n_tok; + (void)n_groups; + (void)in_dim; + (void)out_dim; + (void)row_bytes; + (void)x_token_stride; + (void)x_group_stride; + (void)out_token_stride; + (void)label; + return 0; +#endif +} + enum { ROCM_Q4_PREFILL_K1024_TILE4_REQUIRED_FAILURE = -1, ROCM_Q4_PREFILL_K1024_TILE4_FALLBACK = 0, @@ -804,6 +1100,47 @@ extern "C" int ds4_rocm_matmul_q4_K_tensor( const char *wptr = cuda_model_range_ptr(model_map, weight_offset, weight_bytes, "q4_K dense"); if (!wptr) return 0; + const char *resident_wptr = g_ssd_streaming_mode + ? rocm_q4_attn_q_b_device_resident_source( + model_map, weight_offset, weight_bytes) + : wptr; + const int weight_device_resident = + resident_wptr != NULL && resident_wptr == wptr; + int prefill_wmma = rocm_q4_K_prefill_wmma_select( + n_tok, in_dim, out_dim, weight_device_resident); + if (prefill_wmma == ROCM_Q4_PREFILL_WMMA_REQUIRED_FAILURE) return 0; + if (prefill_wmma == ROCM_Q4_PREFILL_WMMA_USE && + prefill_scope && prefill_tile8_required) { + if (rocm_q4_attn_q_b_env_bool( + "DS4_ROCM_REQUIRE_Q4_PREFILL_WMMA") == 1) { + fprintf(stderr, + DS4_GPU_LOG_PREFIX + "Q4_K prefill cannot require both WMMA and TILE8\n"); + return 0; + } + /* REQUIRE_TILE8 owns the dispatch when WMMA is only an optional + * experiment. */ + prefill_wmma = ROCM_Q4_PREFILL_WMMA_FALLBACK; + } + if (prefill_wmma == ROCM_Q4_PREFILL_WMMA_USE && + k1024_tile4_shape && k1024_tile4_required) { + if (rocm_q4_attn_q_b_env_bool( + "DS4_ROCM_REQUIRE_Q4_PREFILL_WMMA") == 1) { + fprintf(stderr, + DS4_GPU_LOG_PREFIX + "Q4_K prefill cannot require both WMMA and K1024 TILE4\n"); + return 0; + } + prefill_wmma = ROCM_Q4_PREFILL_WMMA_FALLBACK; + } + if (prefill_wmma == ROCM_Q4_PREFILL_WMMA_USE) { + return rocm_q4_K_prefill_wmma_launch( + reinterpret_cast(out->ptr), wptr, + reinterpret_cast(x->ptr), + (uint32_t)n_tok, 1u, (uint32_t)in_dim, (uint32_t)out_dim, + row_bytes, in_dim, 0u, out_dim, + "q4_K dense prefill WMMA64 launch"); + } int k1024_tile4 = ROCM_Q4_PREFILL_K1024_TILE4_FALLBACK; if (prefill_scope && prefill_tile8) { k1024_tile4 = rocm_q4_K_prefill_k1024_tile4_resolve( @@ -877,10 +1214,38 @@ extern "C" int ds4_gpu_matmul_q4_K_pair_tensor( const ds4_gpu_tensor *x, uint64_t n_tok) { const int prefill_scope = rocm_q4_K_prefill_tile8_scope(n_tok); - const int prefill_pair = prefill_scope && - rocm_q4_K_prefill_tile8_requested(); const int prefill_required = prefill_scope && rocm_q4_K_prefill_tile8_required(); + const int wmma_required = rocm_q4_attn_q_b_env_bool( + "DS4_ROCM_REQUIRE_Q4_PREFILL_WMMA") == 1; + + /* The fused pair consumes a shared Q8_K activation tile. When the direct + * F16 WMMA experiment is selected, return before validation/enqueue so the + * graph's established fallback issues two dense calls and both can take + * the strict WMMA path. This also prevents REQUIRE from falsely passing + * after silently measuring the TILE8 pair. SSD preflight is deliberately + * optimistic about residency: it only decides whether to yield; each dense + * fallback then proves its exact physical device range before enqueue. */ + if (!prefill_required) { + const int wmma0 = rocm_q4_K_prefill_wmma_select( + n_tok, in_dim, out0_dim, 1); + const int wmma1 = rocm_q4_K_prefill_wmma_select( + n_tok, in_dim, out1_dim, 1); + if (wmma0 == ROCM_Q4_PREFILL_WMMA_REQUIRED_FAILURE || + wmma1 == ROCM_Q4_PREFILL_WMMA_REQUIRED_FAILURE || + wmma0 == ROCM_Q4_PREFILL_WMMA_USE || + wmma1 == ROCM_Q4_PREFILL_WMMA_USE) { + return 0; + } + } else if (wmma_required) { + fprintf(stderr, + DS4_GPU_LOG_PREFIX + "Q4_K prefill pair cannot require both WMMA and TILE8\n"); + return 0; + } + + const int prefill_pair = prefill_scope && + rocm_q4_K_prefill_tile8_requested(); if (prefill_required && !prefill_pair) { fprintf(stderr, "ds4: required ROCm Q4_K prefill tile8 pair is disabled " @@ -1061,9 +1426,11 @@ static int rocm_q4_K_prefill_tile8_quant_launch( uint32_t in_dim, uint32_t out_dim, uint64_t row_bytes, + int prefill_wmma, const char *label) { uint64_t n_rows = 0; uint64_t xq_token_stride = 0; + uint64_t x_token_stride = 0; uint64_t out_token_stride = 0; const uint64_t blocks = in_dim / CUDA_QK_K; if (!out || !w || !x || n_tok == 0u || n_groups == 0u || @@ -1074,10 +1441,19 @@ static int rocm_q4_K_prefill_tile8_quant_launch( * eight groups, so even the 4096-token ceiling remains in range. */ n_rows > UINT16_MAX || !cuda_u64_mul_checked(n_groups, blocks, &xq_token_stride) || + !cuda_u64_mul_checked(n_groups, in_dim, &x_token_stride) || !cuda_u64_mul_checked(n_groups, out_dim, &out_token_stride)) { return 0; } + if (prefill_wmma == ROCM_Q4_PREFILL_WMMA_USE) { + return rocm_q4_K_prefill_wmma_launch( + out, w, x, n_tok, n_groups, in_dim, out_dim, row_bytes, + x_token_stride, in_dim, out_token_stride, + label ? label : "q4_K attention-output WMMA64 launch") + ? 1 : -1; + } + cuda_block_q8_K *xq = rocm_q4_K_prequant_alloc( n_rows, blocks, label ? label : "q4_K prefill tile8 prequant"); if (!xq) return 0; @@ -1127,8 +1503,19 @@ extern "C" int ds4_gpu_attention_output_q4_K_batch_tensor( const int tile8_requested = rocm_q4_K_prefill_tile8_requested(); const int tile8_required = tile8_scope && rocm_q4_K_prefill_tile8_required(); + const int wmma_enabled = rocm_q4_attn_q_b_env_bool( + "DS4_ROCM_ENABLE_Q4_PREFILL_WMMA") == 1; + const int wmma_ssd_enabled = rocm_q4_attn_q_b_env_bool( + "DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_SSD") == 1; + const int wmma_disabled = rocm_q4_attn_q_b_env_bool( + "DS4_ROCM_DISABLE_Q4_PREFILL_WMMA") == 1; + const int wmma_required = rocm_q4_attn_q_b_env_bool( + "DS4_ROCM_REQUIRE_Q4_PREFILL_WMMA") == 1; + const int wmma_requested = wmma_required || + (!wmma_disabled && ((!g_ssd_streaming_mode && wmma_enabled) || + (g_ssd_streaming_mode && wmma_ssd_enabled))); if (!tile8_scope) return 0; - if (!tile8_requested) { + if (!tile8_requested && !wmma_requested) { if (tile8_required) { fprintf(stderr, "ds4: required ROCm Q4_K attention-output prefill " @@ -1138,7 +1525,8 @@ extern "C" int ds4_gpu_attention_output_q4_K_batch_tensor( } return 0; } - const int pre_enqueue_failure = tile8_required ? -1 : 0; + const int pre_enqueue_failure = + (tile8_required || wmma_required) ? -1 : 0; if (!out || !low || !heads || !model_map || group_dim == 0u || rank == 0u || n_groups == 0u || out_dim == 0u || @@ -1203,13 +1591,61 @@ extern "C" int ds4_gpu_attention_output_q4_K_batch_tensor( model_map, out_b_offset, out_b_bytes, "q4_K attention output B"); if (!out_a || !out_b) return pre_enqueue_failure; - /* A: one quantization over [token,group] plus one z-grouped tile8 launch; - * no group pack/unpack buffers or n_tokens*n_groups dispatch loop. */ + const char *resident_out_a = g_ssd_streaming_mode + ? rocm_q4_attn_q_b_device_resident_source( + model_map, out_a_offset, out_a_bytes) + : out_a; + const char *resident_out_b = g_ssd_streaming_mode && out_b_type == 12u + ? rocm_q4_attn_q_b_device_resident_source( + model_map, out_b_offset, out_b_bytes) + : out_b; + const int out_a_device_resident = + resident_out_a != NULL && resident_out_a == out_a; + const int out_b_device_resident = + resident_out_b != NULL && resident_out_b == out_b; + + /* Resolve every strict WMMA decision before A can enqueue work. That + * keeps REQUIRE fail-closed: an ineligible B projection can never make + * the graph replay a fallback over an already submitted A projection. */ + int a_wmma = rocm_q4_K_prefill_wmma_select( + n_tokens, group_dim, rank, out_a_device_resident); + int b_wmma = out_b_type == 12u + ? rocm_q4_K_prefill_wmma_select( + n_tokens, low_dim, out_dim, out_b_device_resident) + : ROCM_Q4_PREFILL_WMMA_FALLBACK; + if (a_wmma == ROCM_Q4_PREFILL_WMMA_REQUIRED_FAILURE || + b_wmma == ROCM_Q4_PREFILL_WMMA_REQUIRED_FAILURE) { + return -1; + } + if (tile8_required && + (a_wmma == ROCM_Q4_PREFILL_WMMA_USE || + b_wmma == ROCM_Q4_PREFILL_WMMA_USE)) { + if (wmma_required) { + fprintf(stderr, + DS4_GPU_LOG_PREFIX + "Q4_K attention-output prefill cannot require both " + "WMMA and TILE8\n"); + return -1; + } + a_wmma = ROCM_Q4_PREFILL_WMMA_FALLBACK; + b_wmma = ROCM_Q4_PREFILL_WMMA_FALLBACK; + } + if (!tile8_requested && + (a_wmma != ROCM_Q4_PREFILL_WMMA_USE || + (out_b_type == 12u && + b_wmma != ROCM_Q4_PREFILL_WMMA_USE))) { + return pre_enqueue_failure; + } + + /* A: the WMMA candidate consumes F32 heads directly; TILE8 retains one + * quantization over [token,group]. Neither path needs group pack/unpack + * buffers or an n_tokens*n_groups dispatch loop. */ const int a_rc = rocm_q4_K_prefill_tile8_quant_launch( reinterpret_cast(low->ptr), out_a, reinterpret_cast(heads->ptr), n_tokens, n_groups, (uint32_t)group_dim, (uint32_t)rank, row_a_bytes, - "q4_K attention output A prequant"); + a_wmma, + "q4_K attention output A WMMA64/tile8"); if (a_rc <= 0) { return a_rc < 0 ? -1 : pre_enqueue_failure; } @@ -1220,7 +1656,8 @@ extern "C" int ds4_gpu_attention_output_q4_K_batch_tensor( reinterpret_cast(out->ptr), out_b, reinterpret_cast(low->ptr), n_tokens, 1u, (uint32_t)low_dim, (uint32_t)out_dim, row_b_bytes, - "q4_K attention output B prequant"); + b_wmma, + "q4_K attention output B WMMA64/tile8"); } else { b_rc = ds4_gpu_matmul_q8_0_tensor( out, model_map, model_size, out_b_offset, low_dim, out_dim, @@ -1228,6 +1665,9 @@ extern "C" int ds4_gpu_attention_output_q4_K_batch_tensor( } if (b_rc <= 0) return -1; - rocm_q4_K_prefill_tile8_note(0u, 0u, 1u, 0u, n_tokens); + if (a_wmma != ROCM_Q4_PREFILL_WMMA_USE && + b_wmma != ROCM_Q4_PREFILL_WMMA_USE) { + rocm_q4_K_prefill_tile8_note(0u, 0u, 1u, 0u, n_tokens); + } return 1; } diff --git a/scripts/environment_variables.tsv b/scripts/environment_variables.tsv index 511b3c058..69d349e4c 100644 --- a/scripts/environment_variables.tsv +++ b/scripts/environment_variables.tsv @@ -964,35 +964,36 @@ runtime/rocm DS4_ROCM_DISABLE_Q4_DENSE_PAIR presence rollback; unset leaves opt- runtime/rocm DS4_ROCM_DISABLE_Q4_GROUPED_ATTN_A presence rollback; unset permits the caller-marked resident decode production-shape default and explicit ENABLE/REQUIRE; any defined value including empty or 0 disables all grouped attention-A paths and wins over ENABLE/REQUIRE Restore eight standalone Q4 attention-A projections instead of the two-dispatch grouped path. rocm/ds4_rocm_q4.cuh:868 runtime/rocm DS4_ROCM_DISABLE_Q4_PREFILL_K1024_TILE4 value-aware authoritative rollback; unset/0/false/no/off preserves the resident automatic default and any explicit SSD request; empty or any other value disables; overrides ENABLE and causes REQUIRE to fail closed Restore the generic eight-block Q4_K tiled-prefill kernel for K=1024 in both resident and SSD-streaming execution. rocm/ds4_rocm_q4.cuh:624 runtime/rocm DS4_ROCM_DISABLE_Q4_PREFILL_TILE8 presence rollback; TILE8 is default for 9..4096 tokens Disable/roll back rocm disable q4 prefill tile8. rocm/ds4_rocm_q4.cuh:448 -runtime/rocm DS4_ROCM_DISABLE_Q4_SELECTED_EXPERT_VIEWS presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable q4 selected expert views. ds4.c:21135 -runtime/rocm DS4_ROCM_DISABLE_RESIDENT_IQ2_SORTED presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable resident iq2 sorted. rocm/ds4_rocm_moe_launch.cuh:747 -runtime/rocm DS4_ROCM_DISABLE_ROUTED_PAIR_SWIGLU_FUSION presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable routed pair swiglu fusion. ds4.c:18528 -runtime/rocm DS4_ROCM_DISABLE_STREAMING_COLD_DECODE_PREFILL presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming cold decode prefill. ds4.c:32006 -runtime/rocm DS4_ROCM_DISABLE_STREAMING_DECODE_PREFILL presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming decode prefill. ds4.c:31955 -runtime/rocm DS4_ROCM_DISABLE_STREAMING_EXPERT_ADDR_TABLE presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming expert addr table. ds4.c:18524 -runtime/rocm DS4_ROCM_DISABLE_STREAMING_EXPERT_HOTLIST presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming expert hotlist. ds4.c:21289 -runtime/rocm DS4_ROCM_DISABLE_STREAMING_FULL_EXPERT_ADDR_TABLE presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming full expert addr table. ds4.c:18242 -runtime/rocm DS4_ROCM_DISABLE_STREAMING_LAYER_BATCH presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming layer batch. ds4.c:18238 -runtime/rocm DS4_ROCM_DISABLE_STREAMING_MADVISE_WILLNEED presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming madvise willneed. ds4.c:18211 -runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_BATCH_SELECTED_ADDR presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming prefill batch selected addr. ds4.c:18522 -runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_MADVISE presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming prefill layer madvise. ds4.c:18465 -runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_PAGEIN presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming prefill layer pagein. ds4.c:18433 -runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_PAGEIN_OVERLAP presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming prefill layer pagein overlap. ds4.c:19320 -runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_PREAD presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming prefill layer pread. ds4.c:18453 -runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_PREPARE presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming prefill layer prepare. ds4.c:18445 -runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_PREPARE_OVERLAP presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming prefill layer prepare overlap. ds4.c:19318 -runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_READAHEAD presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming prefill layer readahead. ds4.c:18443 -runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_ASYNC_LOAD presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming prefill selected async load. ds4.c:46423 -runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_MADVISE presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming prefill selected madvise. ds4.c:18423 -runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_PAGEIN presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming prefill selected pagein. ds4.c:18413 -runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_PROFILE presence rollback flag; unset keeps automatic/default path Collect timing/profile diagnostics for rocm disable streaming prefill selected profile. ds4.c:18907 -runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_READAHEAD presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming prefill selected readahead. ds4.c:19959 -runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_READAHEAD_SHARED presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming prefill selected readahead shared. ds4.c:19969 -runtime/rocm DS4_ROCM_DISABLE_STREAMING_READAHEAD presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming readahead. ds4.c:18204 -runtime/rocm DS4_ROCM_DISABLE_STREAMING_SELECTED_ASYNC_LOAD presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming selected async load. ds4.c:44326 -runtime/rocm DS4_ROCM_DISABLE_STREAMING_SPLIT_SELECTED presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming split selected. rocm/ds4_rocm_moe_launch.cuh:661 -runtime/rocm DS4_ROCM_DISABLE_STREAMING_STATIC_DECODE_MAP presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming static decode map. ds4.c:18216 -runtime/rocm DS4_ROCM_DISABLE_STREAMING_STATIC_MAP_STATE_CACHE presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming static map state cache. ds4.c:18229 +runtime/rocm DS4_ROCM_DISABLE_Q4_PREFILL_WMMA value-aware authoritative rollback for the experimental path; unset/0/false/no/off permits ENABLE or REQUIRE, while empty or any other value disables; REQUIRE then fails closed Prevent the gfx1151 Q4_K prefill WMMA64 experiment from dispatching and retain the Q8_K-plus-TILE8/TILE4 path. rocm/ds4_rocm_q4.cuh:775 +runtime/rocm DS4_ROCM_DISABLE_Q4_SELECTED_EXPERT_VIEWS presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable q4 selected expert views. ds4.c:21150 +runtime/rocm DS4_ROCM_DISABLE_RESIDENT_IQ2_SORTED presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable resident iq2 sorted. rocm/ds4_rocm_moe_launch.cuh:751 +runtime/rocm DS4_ROCM_DISABLE_ROUTED_PAIR_SWIGLU_FUSION presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable routed pair swiglu fusion. ds4.c:18543 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_COLD_DECODE_PREFILL presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming cold decode prefill. ds4.c:32021 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_DECODE_PREFILL presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming decode prefill. ds4.c:31970 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_EXPERT_ADDR_TABLE presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming expert addr table. ds4.c:18539 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_EXPERT_HOTLIST presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming expert hotlist. ds4.c:21304 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_FULL_EXPERT_ADDR_TABLE presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming full expert addr table. ds4.c:18257 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_LAYER_BATCH presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming layer batch. ds4.c:18253 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_MADVISE_WILLNEED presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming madvise willneed. ds4.c:18226 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_BATCH_SELECTED_ADDR presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming prefill batch selected addr. ds4.c:18537 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_MADVISE presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming prefill layer madvise. ds4.c:18480 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_PAGEIN presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming prefill layer pagein. ds4.c:18448 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_PAGEIN_OVERLAP presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming prefill layer pagein overlap. ds4.c:19335 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_PREAD presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming prefill layer pread. ds4.c:18468 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_PREPARE presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming prefill layer prepare. ds4.c:18460 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_PREPARE_OVERLAP presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming prefill layer prepare overlap. ds4.c:19333 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_READAHEAD presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming prefill layer readahead. ds4.c:18458 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_ASYNC_LOAD presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming prefill selected async load. ds4.c:46684 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_MADVISE presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming prefill selected madvise. ds4.c:18438 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_PAGEIN presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming prefill selected pagein. ds4.c:18428 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_PROFILE presence rollback flag; unset keeps automatic/default path Collect timing/profile diagnostics for rocm disable streaming prefill selected profile. ds4.c:18922 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_READAHEAD presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming prefill selected readahead. ds4.c:19974 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_READAHEAD_SHARED presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming prefill selected readahead shared. ds4.c:19984 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_READAHEAD presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming readahead. ds4.c:18219 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_SELECTED_ASYNC_LOAD presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming selected async load. ds4.c:44587 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_SPLIT_SELECTED presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming split selected. rocm/ds4_rocm_moe_launch.cuh:665 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_STATIC_DECODE_MAP presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming static decode map. ds4.c:18231 +runtime/rocm DS4_ROCM_DISABLE_STREAMING_STATIC_MAP_STATE_CACHE presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable streaming static map state cache. ds4.c:18244 runtime/rocm DS4_ROCM_DSV4_PREQUANT_DECODE sampled once; unset: enabled; present empty or exact 0: disabled; every other present value: enabled; quality mode and GLM models force it off regardless ROCm DeepSeek-V4 decode: quantizes one-token F32 activations to Q8 once and selects the prequantized Q8_0/DP4A projection kernels instead of the full-F32 activation paths. rocm/ds4_rocm_runtime.cuh:4775 runtime/rocm DS4_ROCM_ENABLE_GFX1151_PREFILL_WMMA_INDEXED value-aware per-launch opt-in, default off; empty or 1/true/yes/on enables and 0/false/no/off disables; invalid values fail closed; DISABLE wins; eligibility additionally requires actual gfx1151, runtime warpSize=32, quality off, N>1, 64 model heads, head_dim=512, and top_k=512 Use 32-head rocWMMA workgroups with 80-key tiles and vectorized float2-to-half2 KV staging for DeepSeek-V4 indexed mixed-attention prefill; all failed gates preserve the existing heads32/heads16 fallback. rocm/ds4_rocm_attention_launch.cuh:11 runtime/rocm DS4_ROCM_ENABLE_GFX1151_PREFILL_WMMA_RING value-aware per-launch opt-in, default off; empty or 1/true/yes/on enables and 0/false/no/off disables; invalid values fail closed; DISABLE wins; eligibility additionally requires actual gfx1151, runtime warpSize=32, quality off, unmasked N>1, 64 model heads, and head_dim=512 Use 32-head rocWMMA workgroups with 80-key tiles and vectorized float2-to-half2 KV staging for causal raw-ring/mixed prefill; all failed gates preserve the existing online or score-buffer fallback. rocm/ds4_rocm_attention_launch.cuh:9 @@ -1006,6 +1007,8 @@ runtime/rocm DS4_ROCM_ENABLE_Q4_ATTN_Q_B_F16_OUTPUT value-aware experimental opt runtime/rocm DS4_ROCM_ENABLE_Q4_DENSE_PAIR presence opt-in; unset=off; DISABLE takes precedence Enable rocm enable q4 dense pair. rocm/ds4_rocm_q4.cuh:434 runtime/rocm DS4_ROCM_ENABLE_Q4_GROUPED_ATTN_A presence opt-in outside the default scope; the exact caller-marked resident decode shape groups=8, N=1, K=4096, M=1024 is automatic, while row-at-a-time batch fallbacks are not; DISABLE wins Enable grouped Q4 attention-A for eligible slices, non-production shapes, or explicit experiments in addition to the resident decode default. rocm/ds4_rocm_q4.cuh:872 runtime/rocm DS4_ROCM_ENABLE_Q4_PREFILL_K1024_TILE4_SSD value-aware SSD-only opt-in, default off; unset/0/false/no/off retains TILE8, while empty or any other value requests TILE4; eligibility additionally requires N=9..4096, K=1024, M=32768, TILE8 enabled, and the complete weight range in device storage rather than mapped/registered host memory; DISABLE wins Allow the four-lane K=1024 Q4_K prefill specialization to consume an already device-resident/cache-backed attn_q_b weight range during SSD streaming without changing model I/O. rocm/ds4_rocm_q4.cuh:624 +runtime/rocm DS4_ROCM_ENABLE_Q4_PREFILL_WMMA value-aware opt-in, default off; unset/0/false/no/off retains the canonical path; empty or any other value requests WMMA64 only for N=256..4096, K a positive multiple of 256, resident non-quality execution on gfx1151 wave32; DISABLE wins Benchmark the compressed Q4_K-to-F16 register dequantization plus 64x64 WMMA prefill kernel without Q8_K activation scratch or an F16 weight sidecar. rocm/ds4_rocm_q4.cuh:771 +runtime/rocm DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_SSD value-aware SSD-only opt-in, default off; unset/0/false/no/off retains TILE8/TILE4, while empty or any other value requests WMMA64; eligibility additionally requires the complete projection weight range in physical device storage rather than mapped/registered host memory; DISABLE wins Allow the compressed WMMA64 kernel to consume an already device-resident/cache-backed Q4_K projection during SSD streaming without changing model I/O. rocm/ds4_rocm_q4.cuh:773 runtime/rocm DS4_ROCM_ENABLE_STREAMING_FULL_EXPERT_ADDR_TABLE presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming full expert addr table. ds4.c:18255 runtime/rocm DS4_ROCM_ENABLE_STREAMING_MADVISE_WILLNEED presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming madvise willneed. ds4.c:18224 runtime/rocm DS4_ROCM_ENABLE_STREAMING_PREFILL_BATCH_SELECTED_ADDR presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming prefill batch selected addr. ds4.c:18584 @@ -1069,8 +1072,9 @@ runtime/rocm DS4_ROCM_REQUIRE_Q4_ATTN_Q_B_F16_CACHE value-aware strict opt-in, d runtime/rocm DS4_ROCM_REQUIRE_Q4_GROUPED_ATTN_A presence fail-closed assertion; also requests the candidate outside the caller-marked resident decode default; DISABLE remains authoritative and causes failure Require grouped Q4 attention-A and fail instead of silently falling back. rocm/ds4_rocm_q4.cuh:870 runtime/rocm DS4_ROCM_REQUIRE_Q4_PREFILL_K1024_TILE4 value-aware fail-closed assertion and SSD opt-in; unset/0/false/no/off is off; empty or any other value requires eligible N=9..4096, K=1024, M=32768 dense calls to select TILE4; SSD also requires an actual device-resident weight range; DISABLE wins Prevent a K=1024 TILE4 correctness/performance oracle from silently falling back to TILE8, including during SSD-streaming A/B runs. rocm/ds4_rocm_q4.cuh:628 runtime/rocm DS4_ROCM_REQUIRE_Q4_PREFILL_TILE8 presence fail-closed assertion for eligible TILE8 calls Require rocm require q4 prefill tile8 and fail instead of silently falling back. rocm/ds4_rocm_q4.cuh:452 -runtime/rocm DS4_ROCM_STREAMING_DECODE_PREFILL_MAX primary nonempty value over the Metal alias; parsed by strtol when it has a numeric prefix (trailing text is accepted); <= 0 disables, values > UINT32_MAX clamp, no numeric prefix uses automatic default: 64 for Flash with uniform Q4_K/MXFP4 experts, 18 for other Pro/Flash, otherwise 0; the disable flag dominates Sets the largest short, non-quality SSD-streaming prefill batch routed through the decode-style path instead of canonical layer-major prefill. ds4.c:31961 -runtime/rocm DS4_ROCM_STREAMING_EXPERT_AUTO_PRELOAD_CAP primary nonempty value over the Metal alias; strict full-string strtoul; valid values > UINT32_MAX clamp, invalid uses 4096, and 0 means no cap (not disabled); when CLI preload is auto/0, unset defaults to cap 4096 except ROCm GLM52, where absent/empty disables automatic preload entirely Caps the number of hot experts synchronously seeded into the SSD-streaming expert cache in automatic preload mode; an explicit CLI preload count bypasses this cap, and setting this variable opts ROCm GLM52 back into auto preload. ds4.c:21454 +runtime/rocm DS4_ROCM_REQUIRE_Q4_PREFILL_WMMA value-aware strict opt-in; unset/0/false/no/off is off; empty or any other value requires every selected Q4 dense or attention-output projection to use WMMA64; unsupported shape/device, quality mode, DISABLE, or an SSD weight range without physical device residency fails before the relevant dispatch Prevent the ROCm Q4 prefill WMMA64 correctness/performance oracle from silently timing TILE8/TILE4; the fused q_a/KV pair yields to separately checked dense calls. rocm/ds4_rocm_q4.cuh:777 +runtime/rocm DS4_ROCM_STREAMING_DECODE_PREFILL_MAX primary nonempty value over the Metal alias; parsed by strtol when it has a numeric prefix (trailing text is accepted); <= 0 disables, values > UINT32_MAX clamp, no numeric prefix uses automatic default: 64 for Flash with uniform Q4_K/MXFP4 experts, 18 for other Pro/Flash, otherwise 0; the disable flag dominates Sets the largest short, non-quality SSD-streaming prefill batch routed through the decode-style path instead of canonical layer-major prefill. ds4.c:31976 +runtime/rocm DS4_ROCM_STREAMING_EXPERT_AUTO_PRELOAD_CAP primary nonempty value over the Metal alias; strict full-string strtoul; valid values > UINT32_MAX clamp, invalid uses 4096, and 0 means no cap (not disabled); when CLI preload is auto/0, unset defaults to cap 4096 except ROCm GLM52, where absent/empty disables automatic preload entirely Caps the number of hot experts synchronously seeded into the SSD-streaming expert cache in automatic preload mode; an explicit CLI preload count bypasses this cap, and setting this variable opts ROCm GLM52 back into auto preload. ds4.c:21469 runtime/rocm DS4_ROCM_STREAMING_EXPERT_CACHE_VERBOSE presence flag; unset=off Print verbose ROCm streaming expert-cache seed/load diagnostics. rocm/ds4_rocm_runtime.cuh:2904 runtime/rocm DS4_ROCM_STREAMING_EXPERT_HOTLIST Nonempty filesystem path; a nonempty ROCm value takes precedence, otherwise DS4_METAL_STREAMING_EXPERT_HOTLIST is a fallback. The file contains whitespace-separated layer expert hits rows; blank/comment lines are ignored, zero-hit rows skipped, malformed/open/read errors fail seeding. Unset/empty uses the built-in Pro/Flash/GLM52 hotlist. Effective only when non-cold SSD hotlist seeding is enabled and cache/preload budget is nonzero. Select a custom ranked expert hotlist used to preseed the streaming resident expert cache before decode. ds4.c:21494 runtime/rocm DS4_ROCM_STREAMING_EXPERT_HOTLIST_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm streaming expert hotlist profile. ds4.c:32244 diff --git a/tests/test_rocm_q4_dense_pair.cpp b/tests/test_rocm_q4_dense_pair.cpp index e5cae4143..11af5fde3 100644 --- a/tests/test_rocm_q4_dense_pair.cpp +++ b/tests/test_rocm_q4_dense_pair.cpp @@ -34,6 +34,8 @@ extern "C" int ds4_rocm_test_q4_prefill_k1024_tile4_policy( int ssd_streaming, int weight_device_resident, int ssd_enabled, int disabled, int required); +extern "C" void ds4_rocm_test_q4_prefill_wmma_reset(void); +extern "C" uint64_t ds4_rocm_test_q4_prefill_wmma_get_calls(void); namespace { @@ -58,6 +60,7 @@ constexpr uint32_t kDecodeAttnRank = 1024u; constexpr uint32_t kDecodeAttnGroups = 8u; constexpr uint32_t kDecodeAttnLowDim = kDecodeAttnGroups * kDecodeAttnRank; +constexpr uint32_t kDecodeAttnOutDim = 4096u; constexpr size_t kOutputGuardFloats = 257u; constexpr float kCpuAbsTolerance = 2.0e-3f; constexpr float kCpuRelTolerance = 3.0e-5f; @@ -75,6 +78,14 @@ constexpr const char *kPrefillK1024Tile4SsdEnable = "DS4_ROCM_ENABLE_Q4_PREFILL_K1024_TILE4_SSD"; constexpr const char *kPrefillK1024Tile4Require = "DS4_ROCM_REQUIRE_Q4_PREFILL_K1024_TILE4"; +constexpr const char *kPrefillWmmaEnable = + "DS4_ROCM_ENABLE_Q4_PREFILL_WMMA"; +constexpr const char *kPrefillWmmaSsdEnable = + "DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_SSD"; +constexpr const char *kPrefillWmmaDisable = + "DS4_ROCM_DISABLE_Q4_PREFILL_WMMA"; +constexpr const char *kPrefillWmmaRequire = + "DS4_ROCM_REQUIRE_Q4_PREFILL_WMMA"; constexpr const char *kQbF16Enable = "DS4_ROCM_ENABLE_Q4_ATTN_Q_B_F16_CACHE"; constexpr const char *kQbF16Disable = @@ -142,6 +153,7 @@ struct aligned_model { uint64_t tail_k1024_offset = 0; uint64_t tail_k1024_pair_offset = 0; uint64_t q_b_k1024_offset = 0; + uint64_t decode_attn_b_offset = 0; ~aligned_model() { std::free(data); } @@ -310,6 +322,10 @@ bool make_model(aligned_model *model) { const uint64_t tail1_bytes = (uint64_t)kM1 * tail_row_bytes; const uint64_t q_b_k1024_bytes = (uint64_t)kQbOutDim * tail_row_bytes; + const uint64_t decode_attn_b_row_bytes = + (kDecodeAttnLowDim / kQkK) * sizeof(block_q4_K_test); + const uint64_t decode_attn_b_bytes = + (uint64_t)kDecodeAttnOutDim * decode_attn_b_row_bytes; const uint64_t attn_b_q8_row_bytes = (kAttnLowDim / 32u) * sizeof(block_q8_0_test); const uint64_t attn_b_q8_bytes = @@ -330,8 +346,10 @@ bool make_model(aligned_model *model) { model->tail_k1024_pair_offset + tail1_bytes, page); model->q_b_k1024_offset = round_up( model->attn_b_q8_offset + attn_b_q8_bytes, page); - model->size = round_up( + model->decode_attn_b_offset = round_up( model->q_b_k1024_offset + q_b_k1024_bytes, page); + model->size = round_up( + model->decode_attn_b_offset + decode_attn_b_bytes, page); void *storage = nullptr; if (posix_memalign(&storage, (size_t)page, (size_t)model->size) != 0) { return false; @@ -366,6 +384,9 @@ bool make_model(aligned_model *model) { fill_q4_rows(reinterpret_cast( model->data + model->q_b_k1024_offset), kQbOutDim, kTailK, 0x082efa98u); + fill_q4_rows(reinterpret_cast( + model->data + model->decode_attn_b_offset), + kDecodeAttnOutDim, kDecodeAttnLowDim, 0x452821e6u); fill_q8_0_rows(reinterpret_cast( model->data + model->attn_b_q8_offset), kAttnOutDim, kAttnLowDim, 0x03707344u); @@ -523,6 +544,38 @@ bool close_to_cpu(const std::vector &got, return tolerance_failures == 0; } +bool close_with_tolerance(const std::vector &got, + const std::vector &expected, + float abs_tolerance, + float rel_tolerance, + const char *label) { + if (got.size() != expected.size()) return false; + uint64_t failures = 0; + float max_abs = 0.0f; + float max_rel = 0.0f; + size_t worst = 0; + for (size_t i = 0; i < got.size(); i++) { + const float diff = std::fabs(got[i] - expected[i]); + const float rel = diff / std::max(1.0f, std::fabs(expected[i])); + if (diff > max_abs) { + max_abs = diff; + worst = i; + } + max_rel = std::max(max_rel, rel); + if (!std::isfinite(got[i]) || !std::isfinite(expected[i]) || + diff > abs_tolerance + rel_tolerance * std::fabs(expected[i])) { + failures++; + } + } + std::fprintf(stderr, + "%s: failures=%llu/%zu max_abs=%g max_rel=%g worst=%zu " + "tolerance(abs=%g rel=%g) %s\n", + label, (unsigned long long)failures, got.size(), max_abs, + max_rel, worst, abs_tolerance, rel_tolerance, + failures == 0u ? "PASS" : "FAIL"); + return failures == 0u; +} + bool bitwise_equal(const std::vector &got, const std::vector &expected, const char *label) { @@ -2068,6 +2121,260 @@ bool run_pair_opt_in_guards(const aligned_model &model) { return ok; } +bool run_prefill_wmma_smoke(const aligned_model &model) { +#if DS4_TEST_HAS_HIP_RUNTIME + hipDeviceProp_t properties{}; + if (hipGetDeviceProperties(&properties, 0) != hipSuccess || + properties.warpSize != 32 || + std::strncmp(properties.gcnArchName, "gfx1151", 7u) != 0) { + std::fprintf(stderr, + "ROCm Q4 WMMA64 prefill: SKIP (requires gfx1151 wave32)\n"); + return true; + } +#else + (void)model; + return true; +#endif + + constexpr uint32_t n_tokens = 257u; + const size_t logical_count = (size_t)n_tokens * kM0; + /* A broken N-tail store could write the remaining 63 rows of the final + * 64-token tile. Cover that full footprint, not just the normal API + * canary. */ + constexpr size_t wmma_guard_floats = (64u - 1u) * kM0; + const size_t allocation_count = logical_count + wmma_guard_floats; + const std::vector sentinel = sentinel_values(allocation_count); + std::vector x; + fill_activation(&x, n_tokens, kK); + tensor_owner x_gpu(x.size() * sizeof(float)); + tensor_owner tile8_gpu(allocation_count * sizeof(float)); + tensor_owner wmma_gpu(allocation_count * sizeof(float)); + if (!x_gpu.ptr || !tile8_gpu.ptr || !wmma_gpu.ptr || + !write_tensor(x_gpu.ptr, x) || !write_tensor(tile8_gpu.ptr, sentinel) || + !write_tensor(wmma_gpu.ptr, sentinel)) { + std::fprintf(stderr, "ROCm Q4 WMMA64 prefill: setup FAIL\n"); + return false; + } + + env_snapshot tile8_enable(kPrefillEnable); + env_snapshot tile8_disable(kPrefillDisable); + env_snapshot tile8_require(kPrefillRequire); + env_snapshot tile4_require(kPrefillK1024Tile4Require); + env_snapshot wmma_enable(kPrefillWmmaEnable); + env_snapshot wmma_ssd_enable(kPrefillWmmaSsdEnable); + env_snapshot wmma_disable(kPrefillWmmaDisable); + env_snapshot wmma_require(kPrefillWmmaRequire); + + (void)unsetenv(kPrefillEnable); + (void)unsetenv(kPrefillDisable); + (void)setenv(kPrefillRequire, "1", 1); + (void)unsetenv(kPrefillK1024Tile4Require); + (void)unsetenv(kPrefillWmmaEnable); + (void)unsetenv(kPrefillWmmaSsdEnable); + (void)unsetenv(kPrefillWmmaDisable); + (void)unsetenv(kPrefillWmmaRequire); + ds4_rocm_test_q4_prefill_wmma_reset(); + const int tile8_rc = ds4_gpu_matmul_quant_tensor( + tile8_gpu.ptr, model.data, model.size, model.weight0_offset, kQ4Type, + kK, kM0, x_gpu.ptr, n_tokens); + const uint64_t tile8_wmma_calls = + ds4_rocm_test_q4_prefill_wmma_get_calls(); + + (void)unsetenv(kPrefillRequire); + (void)setenv(kPrefillWmmaEnable, "1", 1); + (void)unsetenv(kPrefillWmmaDisable); + (void)setenv(kPrefillWmmaRequire, "1", 1); + ds4_rocm_test_q4_prefill_wmma_reset(); + const int wmma_rc = ds4_gpu_matmul_quant_tensor( + wmma_gpu.ptr, model.data, model.size, model.weight0_offset, kQ4Type, + kK, kM0, x_gpu.ptr, n_tokens); + const uint64_t wmma_calls = + ds4_rocm_test_q4_prefill_wmma_get_calls(); + + std::vector tile8(allocation_count); + std::vector wmma(allocation_count); + bool ok = tile8_rc != 0 && wmma_rc != 0 && tile8_wmma_calls == 0u && + wmma_calls == 1u && + read_tensor(tile8_gpu.ptr, &tile8) && + read_tensor(wmma_gpu.ptr, &wmma); + if (ok) { + ok = output_body_overwritten(tile8, sentinel, logical_count, + "WMMA64 TILE8 output body") && ok; + ok = output_body_overwritten(wmma, sentinel, logical_count, + "WMMA64 candidate output body") && ok; + ok = output_guard_unchanged(tile8, sentinel, logical_count, + "WMMA64 TILE8 output canary") && ok; + ok = output_guard_unchanged(wmma, sentinel, logical_count, + "WMMA64 candidate output canary") && ok; + tile8.resize(logical_count); + wmma.resize(logical_count); + ok = close_with_tolerance(wmma, tile8, 2.0f, 3.0e-2f, + "WMMA64 vs TILE8 N/M tail") && ok; + } + + (void)setenv(kPrefillWmmaDisable, "1", 1); + if (!write_tensor(wmma_gpu.ptr, sentinel)) return false; + const int rejected_rc = ds4_gpu_matmul_quant_tensor( + wmma_gpu.ptr, model.data, model.size, model.weight0_offset, kQ4Type, + kK, kM0, x_gpu.ptr, n_tokens); + ok = rejected_rc == 0 && + unchanged_after_rejected_call( + wmma_gpu.ptr, sentinel, + "WMMA64 DISABLE+REQUIRE preserves output") && ok; + std::fprintf(stderr, + "ROCm Q4 WMMA64 prefill: tile8=%d/%llu wmma=%d/%llu " + "rejected=%d %s\n", + tile8_rc, (unsigned long long)tile8_wmma_calls, + wmma_rc, (unsigned long long)wmma_calls, + rejected_rc, ok ? "PASS" : "FAIL"); + return ok; +} + +bool run_attention_output_wmma_smoke(const aligned_model &model) { +#if DS4_TEST_HAS_HIP_RUNTIME + hipDeviceProp_t properties{}; + if (hipGetDeviceProperties(&properties, 0) != hipSuccess || + properties.warpSize != 32 || + std::strncmp(properties.gcnArchName, "gfx1151", 7u) != 0) { + std::fprintf(stderr, + "ROCm Q4 production output WMMA64: SKIP " + "(requires gfx1151 wave32)\n"); + return true; + } +#else + (void)model; + return true; +#endif + + constexpr uint32_t n_tokens = 257u; + const size_t heads_count = + (size_t)n_tokens * kDecodeAttnGroups * kDecodeAttnGroupDim; + const size_t low_count = (size_t)n_tokens * kDecodeAttnLowDim; + const size_t out_count = (size_t)n_tokens * kDecodeAttnOutDim; + constexpr size_t low_guard = (64u - 1u) * kDecodeAttnLowDim; + constexpr size_t out_guard = (64u - 1u) * kDecodeAttnOutDim; + std::vector heads_host; + fill_activation( + &heads_host, n_tokens * kDecodeAttnGroups, kDecodeAttnGroupDim); + const std::vector low_sentinel = + sentinel_values(low_count + low_guard); + const std::vector out_sentinel = + sentinel_values(out_count + out_guard); + + tensor_owner heads_gpu(heads_count * sizeof(float)); + tensor_owner tile8_low(low_sentinel.size() * sizeof(float)); + tensor_owner tile8_out(out_sentinel.size() * sizeof(float)); + tensor_owner wmma_low(low_sentinel.size() * sizeof(float)); + tensor_owner wmma_out(out_sentinel.size() * sizeof(float)); + if (!heads_gpu.ptr || !tile8_low.ptr || !tile8_out.ptr || + !wmma_low.ptr || !wmma_out.ptr || + !write_tensor(heads_gpu.ptr, heads_host) || + !write_tensor(tile8_low.ptr, low_sentinel) || + !write_tensor(tile8_out.ptr, out_sentinel) || + !write_tensor(wmma_low.ptr, low_sentinel) || + !write_tensor(wmma_out.ptr, out_sentinel)) { + std::fprintf(stderr, + "ROCm Q4 production output WMMA64: setup FAIL\n"); + return false; + } + + env_snapshot tile8_enable(kPrefillEnable); + env_snapshot tile8_disable(kPrefillDisable); + env_snapshot tile8_require(kPrefillRequire); + env_snapshot tile4_require(kPrefillK1024Tile4Require); + env_snapshot wmma_enable(kPrefillWmmaEnable); + env_snapshot wmma_ssd_enable(kPrefillWmmaSsdEnable); + env_snapshot wmma_disable(kPrefillWmmaDisable); + env_snapshot wmma_require(kPrefillWmmaRequire); + + ds4_gpu_set_ssd_streaming(false); + (void)unsetenv(kPrefillEnable); + (void)unsetenv(kPrefillDisable); + (void)setenv(kPrefillRequire, "1", 1); + (void)unsetenv(kPrefillK1024Tile4Require); + (void)unsetenv(kPrefillWmmaEnable); + (void)unsetenv(kPrefillWmmaSsdEnable); + (void)unsetenv(kPrefillWmmaDisable); + (void)unsetenv(kPrefillWmmaRequire); + ds4_rocm_test_q4_prefill_wmma_reset(); + const int tile8_rc = ds4_gpu_attention_output_q4_K_batch_tensor( + tile8_out.ptr, tile8_low.ptr, nullptr, nullptr, + model.data, model.size, model.decode_attn_a_offset, + model.decode_attn_b_offset, kQ4Type, kDecodeAttnGroupDim, + kDecodeAttnRank, kDecodeAttnGroups, kDecodeAttnOutDim, + heads_gpu.ptr, n_tokens); + const uint64_t tile8_wmma_calls = + ds4_rocm_test_q4_prefill_wmma_get_calls(); + + (void)unsetenv(kPrefillRequire); + (void)setenv(kPrefillWmmaEnable, "1", 1); + (void)unsetenv(kPrefillWmmaDisable); + (void)setenv(kPrefillWmmaRequire, "1", 1); + ds4_rocm_test_q4_prefill_wmma_reset(); + const int wmma_rc = ds4_gpu_attention_output_q4_K_batch_tensor( + wmma_out.ptr, wmma_low.ptr, nullptr, nullptr, + model.data, model.size, model.decode_attn_a_offset, + model.decode_attn_b_offset, kQ4Type, kDecodeAttnGroupDim, + kDecodeAttnRank, kDecodeAttnGroups, kDecodeAttnOutDim, + heads_gpu.ptr, n_tokens); + const uint64_t wmma_calls = + ds4_rocm_test_q4_prefill_wmma_get_calls(); + + std::vector tile8_low_host(low_sentinel.size()); + std::vector tile8_out_host(out_sentinel.size()); + std::vector wmma_low_host(low_sentinel.size()); + std::vector wmma_out_host(out_sentinel.size()); + bool ok = tile8_rc == 1 && wmma_rc == 1 && + tile8_wmma_calls == 0u && wmma_calls == 2u && + read_tensor(tile8_low.ptr, &tile8_low_host) && + read_tensor(tile8_out.ptr, &tile8_out_host) && + read_tensor(wmma_low.ptr, &wmma_low_host) && + read_tensor(wmma_out.ptr, &wmma_out_host); + if (ok) { + ok = output_body_overwritten( + tile8_low_host, low_sentinel, low_count, + "production output-A TILE8 body") && ok; + ok = output_body_overwritten( + tile8_out_host, out_sentinel, out_count, + "production output-B TILE8 body") && ok; + ok = output_body_overwritten( + wmma_low_host, low_sentinel, low_count, + "production output-A WMMA64 body") && ok; + ok = output_body_overwritten( + wmma_out_host, out_sentinel, out_count, + "production output-B WMMA64 body") && ok; + ok = output_guard_unchanged( + tile8_low_host, low_sentinel, low_count, + "production output-A TILE8 N-tail") && ok; + ok = output_guard_unchanged( + tile8_out_host, out_sentinel, out_count, + "production output-B TILE8 N-tail") && ok; + ok = output_guard_unchanged( + wmma_low_host, low_sentinel, low_count, + "production output-A WMMA64 N-tail") && ok; + ok = output_guard_unchanged( + wmma_out_host, out_sentinel, out_count, + "production output-B WMMA64 N-tail") && ok; + tile8_low_host.resize(low_count); + tile8_out_host.resize(out_count); + wmma_low_host.resize(low_count); + wmma_out_host.resize(out_count); + ok = close_with_tolerance( + wmma_low_host, tile8_low_host, 2.0f, 3.0e-2f, + "production output-A WMMA64 vs TILE8") && ok; + ok = close_with_tolerance( + wmma_out_host, tile8_out_host, 16.0f, 8.0e-2f, + "production output-A+B WMMA64 vs TILE8") && ok; + } + std::fprintf(stderr, + "ROCm Q4 production output WMMA64: tile8=%d/%llu " + "wmma=%d/%llu %s\n", + tile8_rc, (unsigned long long)tile8_wmma_calls, + wmma_rc, (unsigned long long)wmma_calls, + ok ? "PASS" : "FAIL"); + return ok; +} + int detect_rocm_device() { #if DS4_TEST_HAS_HIP_RUNTIME int count = 0; @@ -2141,6 +2448,10 @@ int main(int argc, char **argv) { env_snapshot tile4_ssd_enable(kPrefillK1024Tile4SsdEnable); env_snapshot tile4_disable(kPrefillK1024Tile4Disable); env_snapshot tile4_require(kPrefillK1024Tile4Require); + env_snapshot wmma_enable(kPrefillWmmaEnable); + env_snapshot wmma_ssd_enable(kPrefillWmmaSsdEnable); + env_snapshot wmma_disable(kPrefillWmmaDisable); + env_snapshot wmma_require(kPrefillWmmaRequire); env_snapshot grouped_enable(kGroupedDecodeEnable); env_snapshot grouped_disable(kGroupedDecodeDisable); env_snapshot grouped_require(kGroupedDecodeRequire); @@ -2151,6 +2462,10 @@ int main(int argc, char **argv) { (void)unsetenv(kPrefillK1024Tile4SsdEnable); (void)unsetenv(kPrefillK1024Tile4Disable); (void)unsetenv(kPrefillK1024Tile4Require); + (void)unsetenv(kPrefillWmmaEnable); + (void)unsetenv(kPrefillWmmaSsdEnable); + (void)unsetenv(kPrefillWmmaDisable); + (void)unsetenv(kPrefillWmmaRequire); (void)unsetenv(kGroupedDecodeEnable); (void)unsetenv(kGroupedDecodeDisable); (void)unsetenv(kGroupedDecodeRequire); @@ -2276,6 +2591,9 @@ int main(int argc, char **argv) { "attention prefill Q4-A/Q8-B groups=8 K=4096 rank=32 M=65 " "n_tok=30 (token-tail nt=6)", kQ8Type); + const bool prefill_wmma_ok = run_prefill_wmma_smoke(model); + const bool output_wmma_ok = + run_attention_output_wmma_smoke(model); const bool gate_ok = run_prefill_gate_guards(model); ok = prefill9_ok && prefill30_ok && prefill128_ok && prefill_tail9_ok && @@ -2284,7 +2602,8 @@ int main(int argc, char **argv) { prefill_single128_ok && prefill_pair9_ok && prefill_pair30_reverse_ok && prefill_pair128_ok && attention9_ok && attention30_ok && attention128_ok && attention_q8_9_ok && - attention_q8_30_ok && gate_ok && ok; + attention_q8_30_ok && prefill_wmma_ok && output_wmma_ok && + gate_ok && ok; if (run_prefill_long) { // A 64 MiB activation and a roughly 0.5 Gi-op projection stress // arbitrary token-grid tails without the much slower CPU oracle. From 53d25760d46fe7606ad4c3fdae06e7eebf2335a7 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Fri, 28 Aug 2026 19:50:13 +0200 Subject: [PATCH 138/189] bench(rocm): cover Q4 attention output WMMA --- Makefile | 2 +- speed-bench/README.md | 36 ++- speed-bench/rocm_q4_prefill_bench.cpp | 384 ++++++++++++++++++++++++-- 3 files changed, 391 insertions(+), 31 deletions(-) diff --git a/Makefile b/Makefile index d2b90c955..8616f656c 100644 --- a/Makefile +++ b/Makefile @@ -370,7 +370,7 @@ help: @echo " make rocm-dspark-verify-depth Build ROCm and run the DSpark verifier invariant" @echo " make test-mxfp4-rocm Build and run the synthetic ROCm MXFP4 MoE test" @echo " make rocm-iq2-moe-prefill-bench Build the resident ROCm IQ2/Q2 WMMA A/B harness" - @echo " make rocm-q4-prefill-bench Build the resident ROCm Q4 dense/pair/q_b A/B harness" + @echo " make rocm-q4-prefill-bench Build the resident ROCm Q4 projection/WMMA A/B harness" @echo " make cuda-iq2-moe-prefill-bench CUDA_ARCH=sm_N Build the resident CUDA IQ2/Q2 profiling harness" @echo " make cuda-q4-prefill-bench CUDA_ARCH=sm_N Build the resident CUDA Q4 dense/pair/q_b harness" @echo " make test-rocm Core regression suite on ROCm-only hosts" diff --git a/speed-bench/README.md b/speed-bench/README.md index db1f3cf7a..890a65c4c 100644 --- a/speed-bench/README.md +++ b/speed-bench/README.md @@ -127,26 +127,46 @@ make rocm-q4-prefill-bench ROCM_ARCH=gfx1151 The fixture copies four rotating sets of synthetic GGUF-layout Q4_K weights to device memory before warmup and forces SSD streaming off. HIP events then -measure only the activation quantizer and Q4 projection kernels. The three -default comparisons are: +measure only activation conversion/quantization and projection kernels. The +comparisons are: - `dense`: legacy versus TILE8 at the Flash Q-A `K=4096,M=1024` shape; - `pair`: two TILE8 calls versus the fused Q-A/KV `K=4096,M=(1024+512)` path; - `qb`: TILE8 versus TILE4 at the production `attn_q_b` `K=1024,M=32768` shape. - -The default token set is `9,17,33,128,512`, covering the first row after each -small token-tile boundary plus medium prefill. Use `--full` for -`9,16,17,31,32,33,128,512,4096`, or select a focused run such as: +- `outb`: TILE8 versus the experimental compressed WMMA64 kernel at the + production `output_b` `K=8192,M=4096` shape; +- `output`: the complete grouped `output_a` plus `output_b` production API, + comparing two TILE8 projections with two direct WMMA64 projections. + +On gfx1151 wave32, `dense` and `qb` also emit a second WMMA64 comparison for +`N>=256`. The candidate keeps Q4_K weights compressed, rounds each transient +32-value weight group and the activation tile to F16 in the kernel, accumulates +through WMMA in F32, and avoids both Q8_K activation scratch and persistent F16 +weight sidecars. It remains opt-in in the runtime; the benchmark uses the +strict REQUIRE gate so a rejected dispatch fails instead of timing a fallback. +The benchmark always keeps SSD streaming disabled. Runtime SSD experiments +need the separate `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_SSD=1` gate and only accept +projection ranges already backed by physical device memory, so model I/O is +never folded into the kernel result. + +The default token set is `9,17,33,128,257,512`, covering the first row after +the small TILE8 boundaries and the first token after a 64-token WMMA boundary. +Use `--full` for `9,16,17,31,32,33,128,257,512,4096`, or select a focused run +such as: ``` ./speed-bench/rocm_q4_prefill_bench \ - --case qb --tokens 9,17,33,128,512,4096 --samples 8 + --case output --tokens 256,257,512,1024,2048,4096 --samples 8 ``` Every case rotates identical resident weight sets between arms, alternates -ABBA/BAAB, checks the outputs bit-for-bit, and verifies allocation guards. +ABBA/BAAB, and verifies allocation guards. Comparisons among the integer Q4 +paths remain bit-exact. WMMA64 has a deliberate F16 arithmetic boundary, so +those comparisons require finite results within an explicit absolute/relative +smoke tolerance; a model-logit or prompt oracle is still required before +enabling the kernel by default. Fixture creation, the host-to-device residency copy, warmup, oracle readback, and environment-gate changes are outside the reported HIP-event intervals. `candidate_delta_pct` is negative when the candidate is faster; the companion diff --git a/speed-bench/rocm_q4_prefill_bench.cpp b/speed-bench/rocm_q4_prefill_bench.cpp index 61f7602d6..935bfb2ec 100644 --- a/speed-bench/rocm_q4_prefill_bench.cpp +++ b/speed-bench/rocm_q4_prefill_bench.cpp @@ -28,10 +28,16 @@ constexpr uint32_t kDenseM = 1024u; constexpr uint32_t kKvM = 512u; constexpr uint32_t kQbK = 1024u; constexpr uint32_t kQbM = 32768u; +constexpr uint32_t kOutputGroups = 8u; +constexpr uint32_t kOutputRank = 1024u; +constexpr uint32_t kOutputLowDim = kOutputGroups * kOutputRank; +constexpr uint32_t kOutputM = 4096u; constexpr uint32_t kDefaultSets = 4u; constexpr uint32_t kDefaultSamples = 8u; constexpr uint32_t kDefaultWarmup = 2u; -constexpr uint32_t kGuardWords = 64u; +/* Catch a bad N-tail predicate anywhere in the final 64-token WMMA tile, + * including the widest q_b output used by this harness. */ +constexpr uint32_t kGuardWords = (64u - 1u) * kQbM; constexpr uint64_t kCompareChunk = 4u * 1024u * 1024u; constexpr const char *kPrefillEnable = @@ -46,6 +52,14 @@ constexpr const char *kK1024Tile4SsdEnable = "DS4_ROCM_ENABLE_Q4_PREFILL_K1024_TILE4_SSD"; constexpr const char *kK1024Tile4Require = "DS4_ROCM_REQUIRE_Q4_PREFILL_K1024_TILE4"; +constexpr const char *kWmmaEnable = + "DS4_ROCM_ENABLE_Q4_PREFILL_WMMA"; +constexpr const char *kWmmaSsdEnable = + "DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_SSD"; +constexpr const char *kWmmaDisable = + "DS4_ROCM_DISABLE_Q4_PREFILL_WMMA"; +constexpr const char *kWmmaRequire = + "DS4_ROCM_REQUIRE_Q4_PREFILL_WMMA"; struct block_q4_K_host { uint16_t d; @@ -62,20 +76,25 @@ enum class bench_case { dense, pair, qb, + outb, + output, }; struct config { bench_case selected = bench_case::all; - std::vector tokens = {9u, 17u, 33u, 128u, 512u}; + std::vector tokens = {9u, 17u, 33u, 128u, 257u, 512u}; uint32_t sets = kDefaultSets; uint32_t samples = kDefaultSamples; uint32_t warmup = kDefaultWarmup; + bool wmma_supported = false; }; struct weight_set { uint64_t dense_offset = 0; uint64_t kv_offset = 0; uint64_t qb_offset = 0; + uint64_t output_a_offset = 0; + uint64_t output_b_offset = 0; }; struct model_fixture { @@ -210,10 +229,14 @@ bool make_model(model_fixture *model, uint32_t sets) { const uint64_t dense_bytes = q4_weight_bytes(kDenseK, kDenseM); const uint64_t kv_bytes = q4_weight_bytes(kDenseK, kKvM); const uint64_t qb_bytes = q4_weight_bytes(kQbK, kQbM); + const uint64_t output_a_bytes = + q4_weight_bytes(kDenseK, kOutputLowDim); + const uint64_t output_b_bytes = + q4_weight_bytes(kOutputLowDim, kOutputM); model->weights.resize(sets); - model->span_offsets.reserve(static_cast(sets) * 3u); - model->span_sizes.reserve(static_cast(sets) * 3u); + model->span_offsets.reserve(static_cast(sets) * 5u); + model->span_sizes.reserve(static_cast(sets) * 5u); uint64_t cursor = 0; auto append = [&](uint64_t bytes) { const uint64_t offset = align_up(cursor, page); @@ -227,6 +250,8 @@ bool make_model(model_fixture *model, uint32_t sets) { model->weights[i].dense_offset = append(dense_bytes); model->weights[i].kv_offset = append(kv_bytes); model->weights[i].qb_offset = append(qb_bytes); + model->weights[i].output_a_offset = append(output_a_bytes); + model->weights[i].output_b_offset = append(output_b_bytes); } model->size = align_up(cursor, page); void *storage = nullptr; @@ -243,6 +268,10 @@ bool make_model(model_fixture *model, uint32_t sets) { 0x85a308d3u ^ (i * 0x7f4a7c15u)); fill_q4(staging + model->weights[i].qb_offset, qb_bytes, 0x13198a2eu ^ (i * 0x94d049bbu)); + fill_q4(staging + model->weights[i].output_a_offset, output_a_bytes, + 0x03707344u ^ (i * 0x369dea0fu)); + fill_q4(staging + model->weights[i].output_b_offset, output_b_bytes, + 0xa4093822u ^ (i * 0xdb4f0b91u)); } FILE *file = std::tmpfile(); @@ -361,6 +390,58 @@ bool bitwise_equal(const ds4_gpu_tensor *a, const ds4_gpu_tensor *b, return true; } +bool numerically_close(const ds4_gpu_tensor *got, + const ds4_gpu_tensor *reference, + uint64_t bytes, + const char *label, + float abs_tolerance = 2.0f, + float rel_tolerance = 3.0e-2f) { + if ((bytes % sizeof(float)) != 0u) return false; + const uint64_t chunk_bytes = + std::min(kCompareChunk, bytes) & ~(uint64_t)(sizeof(float) - 1u); + std::vector lhs(static_cast(chunk_bytes / sizeof(float))); + std::vector rhs(static_cast(chunk_bytes / sizeof(float))); + uint64_t failures = 0u; + uint64_t compared = 0u; + float max_abs = 0.0f; + float max_rel = 0.0f; + uint64_t worst = 0u; + for (uint64_t offset = 0u; offset < bytes; offset += chunk_bytes) { + const uint64_t count = std::min(chunk_bytes, bytes - offset); + if (!ds4_gpu_tensor_read(got, offset, lhs.data(), count) || + !ds4_gpu_tensor_read(reference, offset, rhs.data(), count)) { + std::fprintf(stderr, "%s: oracle read failed\n", label); + return false; + } + const uint64_t values = count / sizeof(float); + for (uint64_t i = 0u; i < values; i++) { + const float diff = std::fabs(lhs[(size_t)i] - rhs[(size_t)i]); + const float rel = diff / + std::max(1.0f, std::fabs(rhs[(size_t)i])); + if (diff > max_abs) { + max_abs = diff; + worst = compared + i; + } + max_rel = std::max(max_rel, rel); + if (!std::isfinite(lhs[(size_t)i]) || + !std::isfinite(rhs[(size_t)i]) || + diff > abs_tolerance + + rel_tolerance * std::fabs(rhs[(size_t)i])) { + failures++; + } + } + compared += values; + } + std::fprintf(stderr, + "%s: failures=%llu/%llu max_abs=%g max_rel=%g worst=%llu " + "tolerance(abs=%g rel=%g) %s\n", + label, (unsigned long long)failures, + (unsigned long long)compared, max_abs, max_rel, + (unsigned long long)worst, abs_tolerance, rel_tolerance, + failures == 0u ? "PASS" : "FAIL"); + return failures == 0u; +} + void select_legacy() { (void)unsetenv(kPrefillEnable); (void)setenv(kPrefillDisable, "1", 1); @@ -368,6 +449,10 @@ void select_legacy() { (void)unsetenv(kK1024Tile4Disable); (void)unsetenv(kK1024Tile4SsdEnable); (void)unsetenv(kK1024Tile4Require); + (void)unsetenv(kWmmaEnable); + (void)unsetenv(kWmmaSsdEnable); + (void)unsetenv(kWmmaDisable); + (void)unsetenv(kWmmaRequire); } void select_tile8(bool disable_k1024_tile4) { @@ -376,6 +461,10 @@ void select_tile8(bool disable_k1024_tile4) { (void)setenv(kPrefillRequire, "1", 1); (void)unsetenv(kK1024Tile4SsdEnable); (void)unsetenv(kK1024Tile4Require); + (void)unsetenv(kWmmaEnable); + (void)unsetenv(kWmmaSsdEnable); + (void)unsetenv(kWmmaDisable); + (void)unsetenv(kWmmaRequire); if (disable_k1024_tile4) { (void)setenv(kK1024Tile4Disable, "1", 1); } else { @@ -388,6 +477,18 @@ void select_k1024_tile4() { (void)setenv(kK1024Tile4Require, "1", 1); } +void select_wmma() { + select_tile8(false); + /* WMMA and TILE8 are separate strict contracts. The candidate must not + * inherit REQUIRE_TILE8 from the baseline selector. */ + (void)unsetenv(kPrefillRequire); + (void)setenv(kWmmaEnable, "1", 1); + (void)unsetenv(kWmmaSsdEnable); + (void)unsetenv(kWmmaDisable); + (void)setenv(kWmmaRequire, "1", 1); + (void)unsetenv(kK1024Tile4Require); +} + double percentile(std::vector sorted, double fraction) { std::sort(sorted.begin(), sorted.end()); if (sorted.empty()) return 0.0; @@ -574,10 +675,45 @@ bool run_dense(const model_fixture &model, const config &cfg, check_guard(tiled.ptr, logical_bytes, "dense tile8 oracle"); })) return false; - return bitwise_equal(legacy.ptr, tiled.ptr, logical_bytes, - "dense legacy vs tile8") && - check_guard(legacy.ptr, logical_bytes, "dense legacy") && - check_guard(tiled.ptr, logical_bytes, "dense tile8"); + if (!bitwise_equal(legacy.ptr, tiled.ptr, logical_bytes, + "dense legacy vs tile8") || + !check_guard(legacy.ptr, logical_bytes, "dense legacy") || + !check_guard(tiled.ptr, logical_bytes, "dense tile8")) { + return false; + } + if (!cfg.wmma_supported || n_tokens < 256u) return true; + + const arm wmma_baseline = { + "tile8", []() { select_tile8(false); }, + [&](uint32_t set) { + return ds4_gpu_matmul_quant_tensor( + legacy.ptr, model.data, model.size, + model.weights[set].dense_offset, kQ4Type, kDenseK, + kDenseM, x.ptr, n_tokens) != 0; + }}; + const arm wmma_candidate = { + "wmma64", select_wmma, + [&](uint32_t set) { + return ds4_gpu_matmul_quant_tensor( + tiled.ptr, model.data, model.size, + model.weights[set].dense_offset, kQ4Type, kDenseK, + kDenseM, x.ptr, n_tokens) != 0; + }}; + return benchmark_arms( + "dense_wmma", n_tokens, kDenseK, kDenseM, cfg, wmma_baseline, + wmma_candidate, + [&]() { + return poison_output(legacy.ptr, logical_bytes, 0x7fc10001u) && + poison_output(tiled.ptr, logical_bytes, 0x7fc20002u); + }, + [&]() { + return numerically_close(tiled.ptr, legacy.ptr, logical_bytes, + "dense WMMA64 vs TILE8") && + check_guard(legacy.ptr, logical_bytes, + "dense WMMA64 baseline oracle") && + check_guard(tiled.ptr, logical_bytes, + "dense WMMA64 candidate oracle"); + }); } bool run_pair(const model_fixture &model, const config &cfg, @@ -703,10 +839,178 @@ bool run_qb(const model_fixture &model, const config &cfg, check_guard(tile4.ptr, logical_bytes, "q_b tile4 oracle"); })) return false; - return bitwise_equal(tile8.ptr, tile4.ptr, logical_bytes, - "q_b tile8 vs tile4") && - check_guard(tile8.ptr, logical_bytes, "q_b tile8") && - check_guard(tile4.ptr, logical_bytes, "q_b tile4"); + if (!bitwise_equal(tile8.ptr, tile4.ptr, logical_bytes, + "q_b tile8 vs tile4") || + !check_guard(tile8.ptr, logical_bytes, "q_b tile8") || + !check_guard(tile4.ptr, logical_bytes, "q_b tile4")) { + return false; + } + if (!cfg.wmma_supported || n_tokens < 256u) return true; + + const arm wmma_baseline = { + "tile4", select_k1024_tile4, + [&](uint32_t set) { + return ds4_gpu_matmul_quant_tensor( + tile8.ptr, model.data, model.size, + model.weights[set].qb_offset, kQ4Type, kQbK, kQbM, + x.ptr, n_tokens) != 0; + }}; + const arm wmma_candidate = { + "wmma64", select_wmma, + [&](uint32_t set) { + return ds4_gpu_matmul_quant_tensor( + tile4.ptr, model.data, model.size, + model.weights[set].qb_offset, kQ4Type, kQbK, kQbM, + x.ptr, n_tokens) != 0; + }}; + return benchmark_arms( + "q_b_wmma", n_tokens, kQbK, kQbM, cfg, wmma_baseline, + wmma_candidate, + [&]() { + return poison_output(tile8.ptr, logical_bytes, 0x7fc10001u) && + poison_output(tile4.ptr, logical_bytes, 0x7fc20002u); + }, + [&]() { + return numerically_close(tile4.ptr, tile8.ptr, logical_bytes, + "q_b WMMA64 vs TILE4") && + check_guard(tile8.ptr, logical_bytes, + "q_b WMMA64 baseline oracle") && + check_guard(tile4.ptr, logical_bytes, + "q_b WMMA64 candidate oracle"); + }); +} + +bool run_output_b(const model_fixture &model, const config &cfg, + uint32_t n_tokens) { + if (!cfg.wmma_supported || n_tokens < 256u) return true; + uint64_t out_elements = 0; + if (!checked_mul(n_tokens, kOutputM, &out_elements)) return false; + const uint64_t logical_bytes = out_elements * sizeof(float); + const uint64_t allocation_bytes = logical_bytes + + kGuardWords * sizeof(uint32_t); + tensor_owner x( + static_cast(n_tokens) * kOutputLowDim * sizeof(float)); + tensor_owner tile8(allocation_bytes); + tensor_owner wmma(allocation_bytes); + if (!allocate_io(n_tokens, kOutputLowDim, out_elements, + &x, &tile8, &wmma)) { + std::fprintf(stderr, "output_b N=%u: tensor setup failed\n", n_tokens); + return false; + } + + const arm baseline = { + "tile8", []() { select_tile8(false); }, + [&](uint32_t set) { + return ds4_gpu_matmul_quant_tensor( + tile8.ptr, model.data, model.size, + model.weights[set].output_b_offset, kQ4Type, + kOutputLowDim, kOutputM, x.ptr, n_tokens) != 0; + }}; + const arm candidate = { + "wmma64", select_wmma, + [&](uint32_t set) { + return ds4_gpu_matmul_quant_tensor( + wmma.ptr, model.data, model.size, + model.weights[set].output_b_offset, kQ4Type, + kOutputLowDim, kOutputM, x.ptr, n_tokens) != 0; + }}; + return benchmark_arms( + "output_b_wmma", n_tokens, kOutputLowDim, kOutputM, cfg, + baseline, candidate, + [&]() { + return poison_output(tile8.ptr, logical_bytes, 0x7fc10001u) && + poison_output(wmma.ptr, logical_bytes, 0x7fc20002u); + }, + [&]() { + return numerically_close(wmma.ptr, tile8.ptr, logical_bytes, + "output_b WMMA64 vs TILE8") && + check_guard(tile8.ptr, logical_bytes, + "output_b TILE8 oracle") && + check_guard(wmma.ptr, logical_bytes, + "output_b WMMA64 oracle"); + }); +} + +bool run_attention_output(const model_fixture &model, const config &cfg, + uint32_t n_tokens) { + if (!cfg.wmma_supported || n_tokens < 256u) return true; + uint64_t heads_elements = 0; + uint64_t low_elements = 0; + uint64_t out_elements = 0; + if (!checked_mul(n_tokens, + static_cast(kOutputGroups) * kDenseK, + &heads_elements) || + !checked_mul(n_tokens, kOutputLowDim, &low_elements) || + !checked_mul(n_tokens, kOutputM, &out_elements)) { + return false; + } + const uint64_t low_bytes = low_elements * sizeof(float); + const uint64_t out_bytes = out_elements * sizeof(float); + const uint64_t guard_bytes = kGuardWords * sizeof(uint32_t); + tensor_owner heads(heads_elements * sizeof(float)); + tensor_owner tile8_low(low_bytes + guard_bytes); + tensor_owner tile8_out(out_bytes + guard_bytes); + tensor_owner wmma_low(low_bytes + guard_bytes); + tensor_owner wmma_out(out_bytes + guard_bytes); + std::vector activation; + fill_activation(&activation, n_tokens, kOutputGroups * kDenseK); + if (!heads.ptr || !tile8_low.ptr || !tile8_out.ptr || !wmma_low.ptr || + !wmma_out.ptr || + !ds4_gpu_tensor_write(heads.ptr, 0, activation.data(), + activation.size() * sizeof(float))) { + std::fprintf(stderr, + "attention_output N=%u: tensor setup failed\n", n_tokens); + return false; + } + + auto poison_all = [&]() { + return poison_output(tile8_low.ptr, low_bytes, 0x7fc10001u) && + poison_output(tile8_out.ptr, out_bytes, 0x7fc20002u) && + poison_output(wmma_low.ptr, low_bytes, 0x7fc30003u) && + poison_output(wmma_out.ptr, out_bytes, 0x7fc40004u); + }; + if (!poison_all()) return false; + + const arm baseline = { + "tile8_ab", []() { select_tile8(false); }, + [&](uint32_t set) { + return ds4_gpu_attention_output_q4_K_batch_tensor( + tile8_out.ptr, tile8_low.ptr, nullptr, nullptr, + model.data, model.size, + model.weights[set].output_a_offset, + model.weights[set].output_b_offset, kQ4Type, + kDenseK, kOutputRank, kOutputGroups, kOutputM, + heads.ptr, n_tokens) > 0; + }}; + const arm candidate = { + "wmma64_ab", select_wmma, + [&](uint32_t set) { + return ds4_gpu_attention_output_q4_K_batch_tensor( + wmma_out.ptr, wmma_low.ptr, nullptr, nullptr, + model.data, model.size, + model.weights[set].output_a_offset, + model.weights[set].output_b_offset, kQ4Type, + kDenseK, kOutputRank, kOutputGroups, kOutputM, + heads.ptr, n_tokens) > 0; + }}; + return benchmark_arms( + "attention_output_ab_wmma", n_tokens, kOutputLowDim, + kDenseK + kOutputM, cfg, baseline, candidate, poison_all, + [&]() { + return numerically_close(wmma_low.ptr, tile8_low.ptr, low_bytes, + "output_a grouped WMMA64 vs TILE8") && + numerically_close(wmma_out.ptr, tile8_out.ptr, out_bytes, + "output_a+b WMMA64 vs TILE8", + 16.0f, 8.0e-2f) && + check_guard(tile8_low.ptr, low_bytes, + "output_a TILE8 oracle") && + check_guard(wmma_low.ptr, low_bytes, + "output_a WMMA64 oracle") && + check_guard(tile8_out.ptr, out_bytes, + "output_b TILE8 oracle") && + check_guard(wmma_out.ptr, out_bytes, + "output_b WMMA64 oracle"); + }); } void usage(FILE *stream, const char *argv0) { @@ -714,16 +1018,21 @@ void usage(FILE *stream, const char *argv0) { stream, "usage: %s [options]\n\n" "Resident ROCm Q4_K prefill kernel A/B (HIP event timing only).\n\n" - " --case all|dense|pair|qb comparison to run (default: all)\n" + " --case all|dense|pair|qb|outb|output\n" + " comparison to run (default: all)\n" " --tokens N[,N...] token counts, each 9..4096\n" - " --full use 9,16,17,31,32,33,128,512,4096\n" + " --full use 9,16,17,31,32,33,128,257,512,4096\n" " --sets N rotating resident weight sets (default: %u)\n" " --samples N samples/arm, multiple of 4 (default: %u)\n" " --warmup N untimed dispatches/arm (default: %u)\n" " -h, --help show this help\n\n" - "dense compares legacy with TILE8 at K=4096,M=1024; pair compares\n" - "two TILE8 projections with the fused K=4096,M=(1024+512) path; qb\n" - "compares TILE8 with TILE4 at the production K=1024,M=32768 shape.\n", + "dense compares legacy/TILE8 and, on gfx1151 for N>=256, TILE8/\n" + "WMMA64 at K=4096,M=1024. pair compares two TILE8 projections with\n" + "the fused K=4096,M=(1024+512) path. qb adds TILE4/WMMA64 at the\n" + "K=1024,M=32768 shape. outb isolates K=8192,M=4096; output measures\n" + "the production grouped output_a plus output_b API. WMMA comparisons\n" + "use a finite/toleranced oracle because their F16 boundary is not\n" + "bit-identical to the Q8_K activation path.\n", argv0, kDefaultSets, kDefaultSamples, kDefaultWarmup); } @@ -785,6 +1094,8 @@ config parse_options(int argc, char **argv) { else if (!std::strcmp(value, "dense")) cfg.selected = bench_case::dense; else if (!std::strcmp(value, "pair")) cfg.selected = bench_case::pair; else if (!std::strcmp(value, "qb")) cfg.selected = bench_case::qb; + else if (!std::strcmp(value, "outb")) cfg.selected = bench_case::outb; + else if (!std::strcmp(value, "output")) cfg.selected = bench_case::output; else { std::fprintf(stderr, "invalid --case: %s\n", value); std::exit(2); @@ -792,7 +1103,8 @@ config parse_options(int argc, char **argv) { } else if (!std::strcmp(argv[i], "--tokens")) { cfg.tokens = parse_tokens(need_value(&i, argc, argv)); } else if (!std::strcmp(argv[i], "--full")) { - cfg.tokens = {9u, 16u, 17u, 31u, 32u, 33u, 128u, 512u, 4096u}; + cfg.tokens = { + 9u, 16u, 17u, 31u, 32u, 33u, 128u, 257u, 512u, 4096u}; } else if (!std::strcmp(argv[i], "--sets")) { cfg.sets = parse_u32(need_value(&i, argc, argv), "--sets", 1u, 32u); } else if (!std::strcmp(argv[i], "--samples")) { @@ -822,13 +1134,17 @@ bool includes(bench_case selected, bench_case wanted) { } // namespace int main(int argc, char **argv) { - const config cfg = parse_options(argc, argv); + config cfg = parse_options(argc, argv); env_snapshot enable_guard(kPrefillEnable); env_snapshot disable_guard(kPrefillDisable); env_snapshot require_guard(kPrefillRequire); env_snapshot tile4_guard(kK1024Tile4Disable); env_snapshot tile4_ssd_guard(kK1024Tile4SsdEnable); env_snapshot tile4_require_guard(kK1024Tile4Require); + env_snapshot wmma_enable_guard(kWmmaEnable); + env_snapshot wmma_ssd_enable_guard(kWmmaSsdEnable); + env_snapshot wmma_disable_guard(kWmmaDisable); + env_snapshot wmma_require_guard(kWmmaRequire); int device_count = 0; hipError_t hip_rc = hipGetDeviceCount(&device_count); @@ -845,6 +1161,20 @@ int main(int argc, char **argv) { "rocm-q4-prefill-bench: cannot query device properties\n"); return 1; } + cfg.wmma_supported = properties.warpSize == 32 && + std::strncmp(properties.gcnArchName, "gfx1151", 7u) == 0; + const bool wmma_only_case = + cfg.selected == bench_case::outb || cfg.selected == bench_case::output; + const bool has_wmma_tokens = std::any_of( + cfg.tokens.begin(), cfg.tokens.end(), + [](uint32_t n_tokens) { return n_tokens >= 256u; }); + if (wmma_only_case && (!cfg.wmma_supported || !has_wmma_tokens)) { + std::fprintf(stderr, + "rocm-q4-prefill-bench: SKIP (%s requires gfx1151 " + "wave32 and at least one N>=256 sample)\n", + cfg.selected == bench_case::outb ? "outb" : "output"); + return 77; + } if (!ds4_gpu_init()) { std::fprintf(stderr, "rocm-q4-prefill-bench: ds4_gpu_init failed\n"); return 1; @@ -858,7 +1188,9 @@ int main(int argc, char **argv) { } if (ok) { ds4_gpu_set_ssd_streaming(false); - const uint64_t max_tensor = q4_weight_bytes(kQbK, kQbM); + const uint64_t max_tensor = std::max( + q4_weight_bytes(kOutputLowDim, kOutputM), + q4_weight_bytes(kDenseK, kOutputLowDim)); if (!ds4_gpu_set_model_fd(fileno(model.file)) || !ds4_gpu_set_model_map_spans( model.data, model.size, model.span_offsets.data(), @@ -874,9 +1206,11 @@ int main(int argc, char **argv) { if (ok) { std::printf( "DS4_ROCM_Q4_PREFILL_SETUP device=%s arch=%s warp=%d sets=%u " - "resident_mib=%.2f timing=hip_events ssd_streaming=off\n", + "resident_mib=%.2f timing=hip_events ssd_streaming=off " + "wmma64=%s\n", properties.name, properties.gcnArchName, properties.warpSize, - cfg.sets, static_cast(model.resident_bytes) / 1048576.0); + cfg.sets, static_cast(model.resident_bytes) / 1048576.0, + cfg.wmma_supported ? "available" : "skipped"); std::fflush(stdout); for (uint32_t n_tokens : cfg.tokens) { if (includes(cfg.selected, bench_case::dense)) { @@ -888,6 +1222,12 @@ int main(int argc, char **argv) { if (ok && includes(cfg.selected, bench_case::qb)) { ok = run_qb(model, cfg, n_tokens) && ok; } + if (ok && includes(cfg.selected, bench_case::outb)) { + ok = run_output_b(model, cfg, n_tokens) && ok; + } + if (ok && includes(cfg.selected, bench_case::output)) { + ok = run_attention_output(model, cfg, n_tokens) && ok; + } if (!ok) break; } } From 01ff4ee8db0451c4d6d731285ae9ba9dd37fbce4 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Fri, 28 Aug 2026 20:09:59 +0200 Subject: [PATCH 139/189] perf(cuda): eliminate Q4 prefill output passes --- cuda/mmq/ds4_mmq.cu | 143 +++++++++++++++++++-- cuda/mmq/ds4_mmq.h | 14 +++ cuda/mmq/mmq.cuh | 40 +++++- cuda/mmq/test/test_mmq_parity.cu | 206 +++++++++++++++++++++++++++++++ ds4_cuda.cu | 46 ++++++- 5 files changed, 431 insertions(+), 18 deletions(-) diff --git a/cuda/mmq/ds4_mmq.cu b/cuda/mmq/ds4_mmq.cu index ed531b15a..194f49efc 100644 --- a/cuda/mmq/ds4_mmq.cu +++ b/cuda/mmq/ds4_mmq.cu @@ -1125,7 +1125,9 @@ int ds4_mmq_dense_impl( fprintf(stderr, "%s: mul_mat_q_case launch failed: %s\n", tag, cudaGetErrorString(err)); return -3; } - ds4_mmq_sanitize_f32(out_f32, (uint64_t)M * (uint64_t)N, stream); + if constexpr (type != GGML_TYPE_Q4_K) { + ds4_mmq_sanitize_f32(out_f32, (uint64_t)M * (uint64_t)N, stream); + } return 0; } @@ -1261,8 +1263,6 @@ int ds4_mmq_q4_K_dense_pair_impl( tag, cudaGetErrorString(err)); return -3; } - ds4_mmq_sanitize_f32( - out0_f32, (uint64_t)M0 * (uint64_t)N, stream); if (out_memset_enabled()) { cudaMemsetAsync(out1_f32, 0, out1_bytes, stream); @@ -1300,8 +1300,130 @@ int ds4_mmq_q4_K_dense_pair_impl( tag, cudaGetErrorString(err)); return -4; } - ds4_mmq_sanitize_f32( - out1_f32, (uint64_t)M1 * (uint64_t)N, stream); + return 0; +} + +/* Token-batched grouped Q4_K projection for attention output-A. The source + * is token-major [N][G][K], while MMQ stores directly into token-major + * [N][G][M]. Quantizing the strided source as G channels removes the old + * pack/unpack copies and shares one scratch allocation/quantizer launch. + * Keep one MMQ launch per group: its stream-K partition and reduction tree + * are therefore identical to the established per-group dense call. */ +int ds4_mmq_q4_K_grouped_dense_impl( + const void *W, + const float *X, + float *out, + int M, + int N, + int K, + int n_groups, + cudaStream_t stream) { + const char *tag = "ds4_mmq_q4_K_grouped_dense"; + if (!W || !X || !out) { + fprintf(stderr, "%s: null pointer\n", tag); + return -1; + } + if (M <= 0 || N <= 0 || K <= 0 || n_groups <= 0 || + K % QK_K != 0) { + fprintf(stderr, "%s: bad shape M=%d N=%d K=%d groups=%d\n", + tag, M, N, K, n_groups); + return -1; + } + + const int dev = ggml_cuda_get_device(); + const int cc = ggml_cuda_info().devices[dev].cc; + ggml_backend_cuda_context *ctx = get_ctx_for_device(dev); + if (!ctx) { + fprintf(stderr, "%s: failed to get cuda context for device %d\n", + tag, dev); + return -1; + } + ds4_pool_set_stream(stream); + + const int64_t ne10_padded = GGML_PAD((int64_t)K, MATRIX_ROW_PADDING); + const size_t blocks_per_col = + (size_t)ne10_padded / (4u * (size_t)QK8_1); + const size_t bytes_per_col = + blocks_per_col * sizeof(block_q8_1_mmq); + if ((size_t)N > SIZE_MAX / bytes_per_col) return -1; + const size_t channel_bytes = (size_t)N * bytes_per_col; + if ((size_t)n_groups > SIZE_MAX / channel_bytes) return -1; + const size_t payload_bytes = (size_t)n_groups * channel_bytes; + const size_t slack_blocks = (size_t)get_mmq_x_max_host(cc); + if (slack_blocks > SIZE_MAX / sizeof(block_q8_1_mmq)) return -1; + const size_t slack_bytes = slack_blocks * sizeof(block_q8_1_mmq); + if (payload_bytes > SIZE_MAX - slack_bytes) return -1; + + const int64_t row_blocks = (int64_t)K / QK_K; + if ((size_t)M > SIZE_MAX / (size_t)row_blocks / + sizeof(block_q4_K)) return -1; + const size_t group_weight_bytes = + (size_t)M * (size_t)row_blocks * sizeof(block_q4_K); + const int64_t low_dim = (int64_t)M * n_groups; + if (low_dim > INT_MAX || + (uint64_t)low_dim > UINT64_MAX / (uint64_t)N) return -1; + + ggml_cuda_pool_alloc y_q8_1( + ctx->pool(), payload_bytes + slack_bytes); + ybuf_memset(y_q8_1.get(), payload_bytes + slack_bytes, stream); + quantize_mmq_q8_1_cuda( + X, /*ids=*/nullptr, (void *)y_q8_1.get(), GGML_TYPE_Q4_K, + /*ne00=*/K, + /*s01=*/(int64_t)n_groups * K, + /*s02=*/(int64_t)K, + /*s03=*/(int64_t)n_groups * N * K, + /*ne0=*/ne10_padded, /*ne1=*/N, + /*ne2=*/n_groups, /*ne3=*/1, stream); + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) { + fprintf(stderr, "%s: quantize failed: %s\n", + tag, cudaGetErrorString(err)); + return -2; + } + + if (out_memset_enabled()) { + cudaMemsetAsync(out, 0, + (size_t)N * (size_t)low_dim * sizeof(float), stream); + } + const bool use_stream_k = + (GGML_CUDA_CC_IS_NVIDIA(cc) && + ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_VOLTA) || + GGML_CUDA_CC_IS_CDNA(cc); + for (int g = 0; g < n_groups; ++g) { + const mmq_args args = { + /*x=*/(const char *)W + (size_t)g * group_weight_bytes, + /*type_x=*/GGML_TYPE_Q4_K, + /*y=*/(const int *)(y_q8_1.get() + (size_t)g * channel_bytes), + /*ids_dst=*/nullptr, + /*expert_bounds=*/nullptr, + /*dst=*/out + (int64_t)g * M, + /*ncols_x=*/(int64_t)K, + /*nrows_x=*/(int64_t)M, + /*ncols_dst=*/(int64_t)N, + /*stride_row_x=*/row_blocks, + /*ncols_y=*/(int64_t)N, + /*nrows_dst=*/low_dim, + /*nchannels_x=*/1, + /*nchannels_y=*/1, + /*stride_channel_x=*/0, + /*stride_channel_y=*/(int64_t)(channel_bytes / sizeof(int)), + /*stride_channel_dst=*/0, + /*nsamples_x=*/1, + /*nsamples_y=*/1, + /*stride_sample_x=*/0, + /*stride_sample_y=*/(int64_t)(channel_bytes / sizeof(int)), + /*stride_sample_dst=*/0, + /*use_stream_k=*/use_stream_k, + /*ncols_max=*/(int64_t)N, + }; + mul_mat_q_case(*ctx, args, stream); + err = cudaGetLastError(); + if (err != cudaSuccess) { + fprintf(stderr, "%s: group %d launch failed: %s\n", + tag, g, cudaGetErrorString(err)); + return -3; + } + } return 0; } @@ -1541,6 +1663,13 @@ extern "C" int ds4_mmq_q4_K_dense_pair( W0, W1, X, out0, out1, M0, M1, N, K, stream); } +extern "C" int ds4_mmq_q4_K_grouped_dense( + const void *W, const float *X, float *out, + int M, int N, int K, int n_groups, cudaStream_t stream) { + return ds4_mmq_q4_K_grouped_dense_impl( + W, X, out, M, N, K, n_groups, stream); +} + extern "C" int ds4_mmq_mxfp4_dense( const void * W, const float * X, float * out, int M, int N, int K, cudaStream_t stream) { @@ -1809,7 +1938,7 @@ int ds4_mmq_moe_impl( fprintf(stderr, "%s: mul_mat_q_case (moe) launch failed: %s\n", tag, cudaGetErrorString(err)); return -4; } - if (sanitize_out) { + if (sanitize_out && type != GGML_TYPE_Q4_K) { ds4_mmq_sanitize_f32(out_f32, (uint64_t)M * (uint64_t)ne_get_rows, stream); } return 0; @@ -2568,7 +2697,7 @@ int ds4_mmq_moe_pair_impl( } } } - if (sanitize_out) { + if (sanitize_out && type != GGML_TYPE_Q4_K) { ds4_mmq_sanitize_f32(out_a, (uint64_t)M * (uint64_t)ne_get_rows, stream); ds4_mmq_sanitize_f32(out_b, (uint64_t)M * (uint64_t)ne_get_rows, stream); } diff --git a/cuda/mmq/ds4_mmq.h b/cuda/mmq/ds4_mmq.h index 526a08dfc..3fd6018a9 100644 --- a/cuda/mmq/ds4_mmq.h +++ b/cuda/mmq/ds4_mmq.h @@ -205,6 +205,20 @@ int ds4_mmq_q4_K_dense_pair( int K, cudaStream_t stream); +// Prefill attention output-A with W=[groups][M][K], +// X=[N][groups][K], and out=[N][groups][M]. It quantizes the strided source +// in one launch and writes each group directly to the final token-major +// layout, while preserving the established per-group MMQ reduction tree. +int ds4_mmq_q4_K_grouped_dense( + const void * W_q4_K, + const float * X_f32, + float * out_f32, + int M, + int N, + int K, + int n_groups, + cudaStream_t stream); + int ds4_mmq_mxfp4_dense( const void * W_mxfp4, const float * X_f32, diff --git a/cuda/mmq/mmq.cuh b/cuda/mmq/mmq.cuh index db2f61e22..177bc1b03 100644 --- a/cuda/mmq/mmq.cuh +++ b/cuda/mmq/mmq.cuh @@ -3663,6 +3663,7 @@ static __device__ __forceinline__ void mul_mat_q_process_tile( const int * __restrict__ ids_dst, float * __restrict__ dst, float * __restrict__ tmp_fixup, const int stride_row_x, const int ncols_y, const int stride_col_dst, const int tile_x_max_i, const int tile_y_max_j, const int kb0_start, const int kb0_stop, + const int blocks_per_ne00_total, const char * __restrict__ x_soa, const int64_t soa_blocks) { constexpr int warp_size = ggml_cuda_get_physical_warp_size(); @@ -3752,6 +3753,22 @@ static __device__ __forceinline__ void mul_mat_q_process_tile( __syncthreads(); } + /* AProjQ4 dense prefill used to run a separate full-output sanitize + * kernel after every MMQ. Preserve that contract in the producer + * epilogue instead. A stream-K block may publish only the leading + * partial of a split tile; sanitizing that partial would change the + * eventual sum, so only a block that owns the complete K range may fold + * non-finite values here. Split tiles are handled after their final + * accumulation in mul_mat_q_stream_k_fixup below. */ + if constexpr (type == GGML_TYPE_Q4_K && !fixup) { + if (kb0_start == 0 && kb0_stop == blocks_per_ne00_total) { +#pragma unroll + for (int l = 0; l < mmq_x*mmq_y / (nwarps*warp_size); ++l) { + if (!isfinite(sum[l])) sum[l] = 0.0f; + } + } + } + if (fixup) { write_back(sum, nullptr, tmp_fixup + blockIdx.x*(mmq_x*mmq_y), mmq_y, mmq_y, mmq_x); } else { @@ -3871,7 +3888,8 @@ static __global__ void mul_mat_q( constexpr bool fixup = false; mul_mat_q_process_tile (x, offset_x, y + offset_y, ids_dst_shared, dst + offset_dst, tmp_fixup, stride_row_x, ncols_y, stride_col_dst, - tile_x_max_i, tile_y_max_j, 0, blocks_per_ne00.z, x_soa, soa_blocks); + tile_x_max_i, tile_y_max_j, 0, blocks_per_ne00.z, + blocks_per_ne00.z, x_soa, soa_blocks); return; } #endif // (defined(GGML_USE_HIP) && !defined(CDNA4) && !defined(CDNA3)) || __CUDA_ARCH__ < GGML_CUDA_CC_VOLTA @@ -3957,7 +3975,8 @@ static __global__ void mul_mat_q( constexpr bool fixup = false; // All but (potentially) the last iterations write their data to dst rather than the fixup buffer. mul_mat_q_process_tile (x, offset_x, y + offset_y, ids_dst_shared, dst + offset_dst, tmp_fixup, stride_row_x, ncols_y, stride_col_dst, - tile_x_max_i, tile_y_max_j, kb0_start, kb0_stop, x_soa, soa_blocks); + tile_x_max_i, tile_y_max_j, kb0_start, kb0_stop, + blocks_per_ne00.z, x_soa, soa_blocks); kbc += blocks_per_ne00.z; kbc -= fastmodulo(kbc, blocks_per_ne00); @@ -4026,7 +4045,8 @@ static __global__ void mul_mat_q( constexpr bool fixup = true; // Last index writes its data to fixup buffer to avoid data races with other blocks. mul_mat_q_process_tile (x, offset_x, y + offset_y, ids_dst_shared, dst + offset_dst, tmp_fixup, stride_row_x, ncols_y, stride_col_dst, - tile_x_max_i, tile_y_max_j, kb0_start, kb0_stop, x_soa, soa_blocks); + tile_x_max_i, tile_y_max_j, kb0_start, kb0_stop, + blocks_per_ne00.z, x_soa, soa_blocks); } template @@ -4132,7 +4152,12 @@ static __global__ void mul_mat_q_stream_k_fixup( return; } - dst[j*stride_col_dst + i] += sum[j0/nwarps]; + const int dst_idx = j*stride_col_dst + i; + float value = dst[dst_idx] + sum[j0/nwarps]; + if constexpr (type == GGML_TYPE_Q4_K) { + if (!isfinite(value)) value = 0.0f; + } + dst[dst_idx] = value; } return; } @@ -4168,7 +4193,12 @@ static __global__ void mul_mat_q_stream_k_fixup( return; } - dst[ids_dst_shared[j]*stride_col_dst + i] += sum[j0/nwarps]; + const int dst_idx = ids_dst_shared[j]*stride_col_dst + i; + float value = dst[dst_idx] + sum[j0/nwarps]; + if constexpr (type == GGML_TYPE_Q4_K) { + if (!isfinite(value)) value = 0.0f; + } + dst[dst_idx] = value; } } diff --git a/cuda/mmq/test/test_mmq_parity.cu b/cuda/mmq/test/test_mmq_parity.cu index 520354309..e7a813654 100644 --- a/cuda/mmq/test/test_mmq_parity.cu +++ b/cuda/mmq/test/test_mmq_parity.cu @@ -642,6 +642,196 @@ bool run_q4_K_dense_pair_parity( return ok; } +// Prefill attention output-A verifier. The grouped entry consumes +// X=[N][G][K] and writes out=[N][G][M]. Build the reference with exactly G +// ordinary dense calls, using D2D 2D copies to pack/unpack each group. This +// keeps the quantizer and MMQ reduction tree identical while independently +// checking the grouped entry's strided quantization and output pitch. +bool run_q4_K_grouped_dense_parity( + int M, int N, int K, int n_groups, uint32_t seed, + bool inject_nonfinite = false) { + fprintf(stderr, + "=== Q4_K/GROUPED_DENSE M=%d N=%d K=%d groups=%d seed=%u%s ===\n", + M, N, K, n_groups, seed, + inject_nonfinite ? " nonfinite" : ""); + + std::mt19937 rng(seed); + std::normal_distribution nd(0.0f, 1.0f); + const int blocks_per_row = K / QK_K_LOCAL; + const size_t blocks_per_group = (size_t)M * blocks_per_row; + std::vector W((size_t)n_groups * blocks_per_group); + for (auto &blk : W) generate_random_block_q4_K(&blk, rng); + std::vector X((size_t)N * n_groups * K); + for (float &v : X) v = nd(rng); + + // One NaN scale makes this row's accumulator non-finite. Both the + // ordinary dense reference and the grouped entry must apply Q4_K's + // fused sanitize contract and publish +0.0f for every token. + const int nonfinite_group = n_groups / 2; + const int nonfinite_row = M / 2; + if (inject_nonfinite) { + block_q4_K &blk = + W[((size_t)nonfinite_group * M + nonfinite_row) * + blocks_per_row]; + set_half_from_u16(blk.data.d, (uint16_t)0x7e00u); + } + + constexpr size_t guard_floats = 64; + constexpr uint8_t guard_byte = 0xa5; + const size_t output_count = (size_t)N * n_groups * M; + const size_t output_bytes = output_count * sizeof(float); + const size_t guarded_count = output_count + 2u * guard_floats; + const size_t guarded_bytes = guarded_count * sizeof(float); + + cudaStream_t stream = nullptr; + void *dW = nullptr; + float *dX = nullptr; + float *dGroupX = nullptr; + float *dGroupOut = nullptr; + float *dRef = nullptr; + float *dGotStorage = nullptr; + bool allocated = cudaStreamCreate(&stream) == cudaSuccess && + cudaMalloc(&dW, W.size() * sizeof(block_q4_K)) == cudaSuccess && + cudaMalloc(&dX, X.size() * sizeof(float)) == cudaSuccess && + cudaMalloc(&dGroupX, (size_t)N * K * sizeof(float)) == cudaSuccess && + cudaMalloc(&dGroupOut, (size_t)N * M * sizeof(float)) == cudaSuccess && + cudaMalloc(&dRef, output_bytes) == cudaSuccess && + cudaMalloc(&dGotStorage, guarded_bytes) == cudaSuccess; + const auto cleanup = [&]() { + if (dGotStorage) cudaFree(dGotStorage); + if (dRef) cudaFree(dRef); + if (dGroupOut) cudaFree(dGroupOut); + if (dGroupX) cudaFree(dGroupX); + if (dX) cudaFree(dX); + if (dW) cudaFree(dW); + if (stream) cudaStreamDestroy(stream); + }; + if (!allocated) { + fprintf(stderr, "Q4_K grouped dense parity allocation failed: %s\n\n", + cudaGetErrorString(cudaGetLastError())); + cleanup(); + return false; + } + + float *const dGot = dGotStorage + guard_floats; + cudaError_t enqueue_err = cudaMemcpyAsync( + dW, W.data(), W.size() * sizeof(block_q4_K), + cudaMemcpyHostToDevice, stream); + if (enqueue_err == cudaSuccess) { + enqueue_err = cudaMemcpyAsync( + dX, X.data(), X.size() * sizeof(float), + cudaMemcpyHostToDevice, stream); + } + if (enqueue_err == cudaSuccess) { + enqueue_err = cudaMemsetAsync(dRef, 0x5a, output_bytes, stream); + } + if (enqueue_err == cudaSuccess) { + enqueue_err = cudaMemsetAsync( + dGotStorage, guard_byte, guarded_bytes, stream); + } + + int rc_ref = enqueue_err == cudaSuccess ? 0 : -100; + for (int g = 0; g < n_groups && rc_ref == 0; ++g) { + cudaError_t err = cudaMemcpy2DAsync( + dGroupX, (size_t)K * sizeof(float), + dX + (size_t)g * K, + (size_t)n_groups * K * sizeof(float), + (size_t)K * sizeof(float), (size_t)N, + cudaMemcpyDeviceToDevice, stream); + if (err != cudaSuccess) { + enqueue_err = err; + rc_ref = -101; + break; + } + rc_ref = ds4_mmq_q4_K_dense( + (const char *)dW + (size_t)g * blocks_per_group * + sizeof(block_q4_K), + dGroupX, dGroupOut, M, N, K, stream); + if (rc_ref != 0) break; + err = cudaMemcpy2DAsync( + dRef + (size_t)g * M, + (size_t)n_groups * M * sizeof(float), + dGroupOut, (size_t)M * sizeof(float), + (size_t)M * sizeof(float), (size_t)N, + cudaMemcpyDeviceToDevice, stream); + if (err != cudaSuccess) { + enqueue_err = err; + rc_ref = -102; + } + } + + const int rc_got = enqueue_err == cudaSuccess + ? ds4_mmq_q4_K_grouped_dense( + dW, dX, dGot, M, N, K, n_groups, stream) + : -100; + + std::vector ref(output_count); + std::vector got(output_count); + std::vector guarded(guarded_bytes); + if (enqueue_err == cudaSuccess) { + enqueue_err = cudaMemcpyAsync( + ref.data(), dRef, output_bytes, cudaMemcpyDeviceToHost, stream); + } + if (enqueue_err == cudaSuccess) { + enqueue_err = cudaMemcpyAsync( + got.data(), dGot, output_bytes, cudaMemcpyDeviceToHost, stream); + } + if (enqueue_err == cudaSuccess) { + enqueue_err = cudaMemcpyAsync( + guarded.data(), dGotStorage, guarded_bytes, + cudaMemcpyDeviceToHost, stream); + } + const cudaError_t sync_err = cudaStreamSynchronize(stream); + + size_t mismatches = 0; + size_t nonfinite_ref = 0; + size_t nonfinite_got = 0; + for (size_t i = 0; i < output_count; ++i) { + if (std::memcmp(&ref[i], &got[i], sizeof(float)) != 0) mismatches++; + if (!std::isfinite(ref[i])) nonfinite_ref++; + if (!std::isfinite(got[i])) nonfinite_got++; + } + const size_t guard_bytes = guard_floats * sizeof(float); + size_t canary_mismatches = 0; + for (size_t i = 0; i < guard_bytes; ++i) { + if (guarded[i] != guard_byte) canary_mismatches++; + } + for (size_t i = guard_bytes + output_bytes; i < guarded.size(); ++i) { + if (guarded[i] != guard_byte) canary_mismatches++; + } + + size_t sanitize_mismatches = 0; + if (inject_nonfinite) { + const uint32_t positive_zero = 0; + for (int t = 0; t < N; ++t) { + const size_t i = + ((size_t)t * n_groups + nonfinite_group) * M + + nonfinite_row; + uint32_t ref_bits = 0; + uint32_t got_bits = 0; + std::memcpy(&ref_bits, &ref[i], sizeof(ref_bits)); + std::memcpy(&got_bits, &got[i], sizeof(got_bits)); + if (ref_bits != positive_zero || got_bits != positive_zero) { + sanitize_mismatches++; + } + } + } + + const bool ok = rc_ref == 0 && rc_got == 0 && + enqueue_err == cudaSuccess && sync_err == cudaSuccess && + mismatches == 0 && nonfinite_ref == 0 && nonfinite_got == 0 && + canary_mismatches == 0 && sanitize_mismatches == 0; + fprintf(stderr, + "rc_ref=%d rc_grouped=%d enqueue=%s sync=%s " + "mismatches=%zu nonfinite=%zu/%zu canary=%zu sanitize=%zu: %s\n\n", + rc_ref, rc_got, cudaGetErrorString(enqueue_err), + cudaGetErrorString(sync_err), mismatches, nonfinite_ref, + nonfinite_got, canary_mismatches, sanitize_mismatches, + ok ? "PASS" : "FAIL"); + cleanup(); + return ok; +} + // IQ2_XXS internally accumulates in int8 via SIMD intrinsics // (__vsub4 / __vcmpne4 in vec_dot_iq2_xxs_q8_1) and applies the scale // post-accumulation, while the CPU reference does per-element float @@ -2291,6 +2481,22 @@ int main(int argc, char ** argv) { /*M0=*/65, /*M1=*/129, /*N=*/129, /*K=*/1024, 0xC4FE52); all_ok &= run_q4_K_dense_pair_parity( /*M0=*/96, /*M1=*/33, /*N=*/128, /*K=*/4096, 0xC4FE53); + // Grouped attention output-A prefill: exercise each token-tile tail + // around 8/16/32/128, with at least three groups in every case. The + // final case also injects a NaN Q4 scale to verify fused sanitization. + all_ok &= run_q4_K_grouped_dense_parity( + /*M=*/33, /*N=*/9, /*K=*/256, /*groups=*/3, 0xC4D009); + all_ok &= run_q4_K_grouped_dense_parity( + /*M=*/65, /*N=*/17, /*K=*/512, /*groups=*/4, 0xC4D011); + all_ok &= run_q4_K_grouped_dense_parity( + /*M=*/47, /*N=*/33, /*K=*/768, /*groups=*/5, 0xC4D021); + all_ok &= run_q4_K_grouped_dense_parity( + /*M=*/31, /*N=*/127, /*K=*/256, /*groups=*/3, 0xC4D07F); + all_ok &= run_q4_K_grouped_dense_parity( + /*M=*/31, /*N=*/128, /*K=*/256, /*groups=*/3, 0xC4D080); + all_ok &= run_q4_K_grouped_dense_parity( + /*M=*/31, /*N=*/129, /*K=*/512, /*groups=*/3, 0xC4D081, + /*inject_nonfinite=*/true); // MoE (_id) path. Small expert counts + small shapes for fast verification. // Per-token-distinct routing with top_k=2 or 6. diff --git a/ds4_cuda.cu b/ds4_cuda.cu index 34670bf93..06414af27 100644 --- a/ds4_cuda.cu +++ b/ds4_cuda.cu @@ -43910,22 +43910,24 @@ extern "C" int ds4_gpu_attention_output_q4_K_batch_tensor( uint64_t out_dim, const ds4_gpu_tensor *heads, uint32_t n_tokens) { const int grouped_batch_require = cuda_env_flag_enabled( "DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_BATCH", 0); + const int grouped_prefill_require = cuda_env_flag_enabled( + "DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_PREFILL", 0); if (!out || !low || !group_tmp || !low_tmp || !heads || !model_map || group_dim == 0 || rank == 0 || n_groups == 0 || out_dim == 0 || n_tokens < 2u || group_dim > INT_MAX || rank > INT_MAX || out_dim > INT_MAX || n_tokens > INT_MAX || !cuda_use_mmq()) { - return grouped_batch_require ? -1 : 0; + return (grouped_batch_require || grouped_prefill_require) ? -1 : 0; } const uint64_t low_dim = (uint64_t)n_groups * rank; if (low_dim > INT_MAX || (group_dim % CUDA_QK_K) != 0u || (low_dim % CUDA_QK_K) != 0u) { - return grouped_batch_require ? -1 : 0; + return (grouped_batch_require || grouped_prefill_require) ? -1 : 0; } const uint64_t row_a_bytes = (group_dim / CUDA_QK_K) * sizeof(cuda_block_q4_K); if (rank > UINT64_MAX / row_a_bytes || n_groups > UINT64_MAX / (rank * row_a_bytes)) { - return grouped_batch_require ? -1 : 0; + return (grouped_batch_require || grouped_prefill_require) ? -1 : 0; } const uint64_t out_a_bytes = (uint64_t)n_groups * rank * row_a_bytes; if (out_a_offset > model_size || @@ -43935,16 +43937,22 @@ extern "C" int ds4_gpu_attention_output_q4_K_batch_tensor( out->bytes < (uint64_t)n_tokens * out_dim * sizeof(float) || group_tmp->bytes < (uint64_t)n_tokens * group_dim * sizeof(float) || low_tmp->bytes < (uint64_t)n_tokens * rank * sizeof(float)) { - return grouped_batch_require ? -1 : 0; + return (grouped_batch_require || grouped_prefill_require) ? -1 : 0; } const int logical_tier = ds4_tensor_device_idx(out); const char *out_a = cuda_resolve_weight_ptr( model_map, out_a_offset, out_a_bytes, logical_tier, "q4 attn_out_a"); - if (!out_a) return grouped_batch_require ? -1 : 0; + if (!out_a) { + return (grouped_batch_require || grouped_prefill_require) ? -1 : 0; + } const int grouped_oracle = cuda_env_flag_enabled( "DS4_CUDA_Q4_GROUPED_ATTN_A_ORACLE", 0); const int grouped_batch_enable = cuda_env_flag_enabled( "DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_BATCH", 0); + const int grouped_prefill_enable = cuda_env_flag_enabled( + "DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_PREFILL", 0); + const int grouped_prefill_disable = + getenv("DS4_CUDA_NO_Q4_GROUPED_ATTN_A_PREFILL") != NULL; if (grouped_oracle) cuda_q4_grouped_attn_a_oracle_register_report(); const int grouped_gb10 = @@ -43953,6 +43961,13 @@ extern "C" int ds4_gpu_attention_output_q4_K_batch_tensor( ds4_tensor_device_idx(heads) == logical_tier && cuda_q4_gb10_fast_path_enabled( logical_tier, "DS4_CUDA_NO_Q4_GROUPED_ATTN_A"); + const int grouped_prefill_gb10 = + n_tokens > 8u && n_groups <= INT_MAX && + ds4_tensor_device_idx(low) == logical_tier && + ds4_tensor_device_idx(heads) == logical_tier && + !grouped_prefill_disable && + cuda_q4_gb10_fast_path_enabled( + logical_tier, "DS4_CUDA_NO_Q4_GROUPED_ATTN_A"); if (grouped_batch_require && (!grouped_batch_enable || !grouped_gb10)) { fprintf(stderr, @@ -43960,7 +43975,26 @@ extern "C" int ds4_gpu_attention_output_q4_K_batch_tensor( "not eligible\n"); return -1; } - if (grouped_gb10) { + if (grouped_prefill_require && !grouped_prefill_gb10) { + fprintf(stderr, + "ds4: required CUDA Q4 grouped attention-A prefill path " + "is not eligible\n"); + return -1; + } + if ((grouped_prefill_enable || grouped_prefill_require) && + grouped_prefill_gb10) { + const int rc = ds4_mmq_q4_K_grouped_dense( + out_a, (const float *)heads->ptr, (float *)low->ptr, + (int)rank, (int)n_tokens, (int)group_dim, (int)n_groups, + cuda_decode_stream()); + if (rc != 0) { + fprintf(stderr, + "ds4: CUDA Q4 grouped attention-A prefill returned %d; " + "failing closed after candidate dispatch\n", + rc); + return -1; + } + } else if (grouped_gb10) { /* DSpark verification stores [token][group][K]. The opt-in batch * entry flattens (token, group) into channels while keeping * ncols_dst=1, preserving the canonical per-pair Q8_1 quantization From bdd28e343331f6f9d8db7169dea1cd00802b22ff Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Fri, 28 Aug 2026 20:10:18 +0200 Subject: [PATCH 140/189] bench(cuda): isolate grouped Q4 attention prefill --- ENVIRONMENT_VARIABLES.md | 6 + Makefile | 2 +- QA_BEFORE_RELEASES.md | 11 +- README.md | 13 + scripts/environment_variables.tsv | 3 + speed-bench/README.md | 25 +- speed-bench/cuda_q4_prefill_bench.cu | 380 ++++++++++++++++++++++++--- 7 files changed, 401 insertions(+), 39 deletions(-) diff --git a/ENVIRONMENT_VARIABLES.md b/ENVIRONMENT_VARIABLES.md index e15ecd976..96089687e 100644 --- a/ENVIRONMENT_VARIABLES.md +++ b/ENVIRONMENT_VARIABLES.md @@ -63,6 +63,9 @@ The detailed Metal A/B contracts and expected oracle counters live in | `DS4_CUDA_NO_Q4_GB10_FAST=1` | Umbrella rollback for the GB10-specific Q4 choices; it does not disable the older cross-CUDA dense pair. | | `DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_BATCH=1` | Enable grouped attention-A for two-to-eight-token GB10 verifier batches. | | `DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_BATCH=1` | Fail closed if that grouped batch path is unavailable. | +| `DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_PREFILL=1` | Enable the GB10 Q4 attention-A prefill candidate for more than eight tokens. It quantizes the token-major grouped input directly and removes the eight pack/unpack copies while preserving each group’s MMQ reduction tree. | +| `DS4_CUDA_NO_Q4_GROUPED_ATTN_A_PREFILL=1` | Dominant rollback from the grouped Q4 attention-A prefill candidate to eight pack/MMQ/unpack projections. Any defined value disables the candidate. | +| `DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_PREFILL=1` | Request the grouped Q4 attention-A prefill candidate and fail before enqueue when its GB10, shape, residency, or buffer contract is unavailable. | | `DS4_CUDA_Q4_GROUPED_ATTN_A_ORACLE=1` | Compare grouped attention-A with the canonical result and retain the canonical output. Disable graph capture for this diagnostic. | | `DS4_CUDA_ENABLE_Q4_K1024_PERSISTENT=1` | Enable the experimental persistent-CTA kernel for the exact `32768x1024` Q4 shape. | | `DS4_CUDA_NO_Q4_K1024_PERSISTENT=1` | Roll back the persistent K1024 experiment. | @@ -643,6 +646,7 @@ and **19 tool/wrapper entries**. | `DS4_CUDA_ENABLE_IQ2_XXS_SSD_PREFILL_MMQ` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Enable the CUDA IQ2 XXS SSD prefill MMQ experimental path. | [ds4_cuda.cu:4625](ds4_cuda.cu#L4625) | | `DS4_CUDA_ENABLE_Q4_ATTN_OUT_HC_FUSE` | value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on | Opt in to the fused Q4 attention-output/HC expansion path. | [ds4_cuda.cu:37375](ds4_cuda.cu#L37375) | | `DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_BATCH` | value-aware opt-in, default off; nonempty value other than exact 0 enables; rollback wins | Enable flattened grouped attention-A MMQ for two-to-eight-token GB10 batches. | [cuda/mmq/ds4_mmq.cu:4303](cuda/mmq/ds4_mmq.cu#L4303) | +| `DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_PREFILL` | value-aware opt-in, default off; unset/empty/exact 0 is off, any other nonempty value requests the path; REQUIRE also requests it; local/global rollback wins | Enable direct-strided grouped Q4_K attention-A MMQ for GB10 prefill widths above eight tokens, removing per-group pack/unpack copies while preserving the per-group reduction tree. | [ds4_cuda.cu:41928](ds4_cuda.cu#L41928) | | `DS4_CUDA_ENABLE_Q4_K1024_PERSISTENT` | presence flag, default off; any defined value including 0 requests the path; rollback wins | Enable the GB10 persistent-CTA kernel for M=32768, N=1, K=1024 Q4. | [cuda/mmq/ds4_mmq.cu:3905](cuda/mmq/ds4_mmq.cu#L3905) | | `DS4_CUDA_ENABLE_Q8_FOLD` | strict flag, default off; only exact value 1 enables; overridden by DS4_CUDA_NO_Q8_FOLD | Enable one-shot producer-to-consumer reuse of freshly quantized Q8_1 data. | [ds4_cuda.cu:785](ds4_cuda.cu#L785) | | `DS4_CUDA_ENABLE_STREAMING_EXPERT_PERSISTENT_CACHE` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Enable streaming expert persistent cache in CUDA SSD streaming. | [ds4_cuda.cu:4111](ds4_cuda.cu#L4111) | @@ -790,6 +794,7 @@ and **19 tool/wrapper entries**. | `DS4_CUDA_NO_Q4_GB10_FAST` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the GB10-specific Q4 fast-path family. | [cuda/mmq/ds4_mmq.cu:3908](cuda/mmq/ds4_mmq.cu#L3908) | | `DS4_CUDA_NO_Q4_GROUPED_ATTN_A` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q4 grouped attn a CUDA Q4 optimization. | [cuda/mmq/ds4_mmq.cu:4295](cuda/mmq/ds4_mmq.cu#L4295) | | `DS4_CUDA_NO_Q4_GROUPED_ATTN_A_BATCH` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q4 grouped attn a batch CUDA Q4 optimization. | [cuda/mmq/ds4_mmq.cu:4305](cuda/mmq/ds4_mmq.cu#L4305) | +| `DS4_CUDA_NO_Q4_GROUPED_ATTN_A_PREFILL` | presence kill switch; default unset; any defined value including empty or 0 disables and dominates ENABLE/REQUIRE | Restore the eight pack/MMQ/unpack Q4 attention-A prefill projections. | [ds4_cuda.cu:41930](ds4_cuda.cu#L41930) | | `DS4_CUDA_NO_Q4_K1024_PERSISTENT` | presence kill switch, default off; any defined value including 0 disables | Disable the Q4 K1024 persistent CUDA Q4 optimization. | [cuda/mmq/ds4_mmq.cu:3907](cuda/mmq/ds4_mmq.cu#L3907) | | `DS4_CUDA_NO_Q8_ALIGNED_DENSE_SCRATCH` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q8 aligned dense scratch CUDA Q8 optimization. | [cuda/mmq/ds4_mmq.cu:5578](cuda/mmq/ds4_mmq.cu#L5578) | | `DS4_CUDA_NO_Q8_ALIGNED_PERSISTENT` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q8 aligned persistent CUDA Q8 optimization. | [cuda/mmq/ds4_mmq.cu:5410](cuda/mmq/ds4_mmq.cu#L5410) | @@ -856,6 +861,7 @@ and **19 tool/wrapper entries**. | `DS4_CUDA_Q_NORM_ROPE_FUSE` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control or tune the CUDA q norm rope fuse path. | [ds4.c:17418](ds4.c#L17418) | | `DS4_CUDA_REQUIRE_IQ2_XXS_SSD_PREFILL_MMQ` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Require the CUDA IQ2 XXS SSD prefill MMQ path; fail closed when unavailable. | [ds4_cuda.cu:4631](ds4_cuda.cu#L4631) | | `DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_BATCH` | value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on | Fail if grouped batched attention-A cannot be used. | [ds4_cuda.cu:40198](ds4_cuda.cu#L40198) | +| `DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_PREFILL` | value-aware fail-closed assertion, default off; unset/empty/exact 0 is off, any other nonempty value requests the candidate and rejects ineligibility before enqueue | Require the GB10 grouped Q4_K attention-A prefill path instead of silently using pack/MMQ/unpack. | [ds4_cuda.cu:41889](ds4_cuda.cu#L41889) | | `DS4_CUDA_REQUIRE_Q4_K1024_PERSISTENT` | presence flag, default off; any defined value including 0 makes ineligible candidate fail closed | Fail when the exact Q4 K1024 persistent candidate is unavailable instead of using MMVQ. | [cuda/mmq/ds4_mmq.cu:3929](cuda/mmq/ds4_mmq.cu#L3929) | | `DS4_CUDA_REQUIRE_STREAMING_EXPERT_PERSISTENT_CACHE` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Require streaming expert persistent cache in CUDA SSD streaming; fail closed when unavailable. | [ds4_cuda.cu:4117](ds4_cuda.cu#L4117) | | `DS4_CUDA_REQUIRE_STREAMING_SELECTED_BATCH_IO` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Require streaming selected batch I/O in CUDA SSD streaming; fail closed when unavailable. | [ds4_cuda.cu:4738](ds4_cuda.cu#L4738) | diff --git a/Makefile b/Makefile index 8616f656c..30f1bee48 100644 --- a/Makefile +++ b/Makefile @@ -372,7 +372,7 @@ help: @echo " make rocm-iq2-moe-prefill-bench Build the resident ROCm IQ2/Q2 WMMA A/B harness" @echo " make rocm-q4-prefill-bench Build the resident ROCm Q4 projection/WMMA A/B harness" @echo " make cuda-iq2-moe-prefill-bench CUDA_ARCH=sm_N Build the resident CUDA IQ2/Q2 profiling harness" - @echo " make cuda-q4-prefill-bench CUDA_ARCH=sm_N Build the resident CUDA Q4 dense/pair/q_b harness" + @echo " make cuda-q4-prefill-bench CUDA_ARCH=sm_N Build the resident CUDA Q4 dense/pair/q_b/output-A harness" @echo " make test-rocm Core regression suite on ROCm-only hosts" @echo " make cpu Build CPU-only ./ds4, ./ds4-server, ./ds4-bench, ./ds4-eval, and ./ds4-agent" @echo " make test Build and run tests" diff --git a/QA_BEFORE_RELEASES.md b/QA_BEFORE_RELEASES.md index aed06495e..631a3ed82 100644 --- a/QA_BEFORE_RELEASES.md +++ b/QA_BEFORE_RELEASES.md @@ -1003,7 +1003,16 @@ Do not use high-performance Hugging Face Xet mode while vLLM is resident. `DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_BATCH=1`, and `DS4_CUDA_Q4_GROUPED_ATTN_A_ORACLE=1`; require `batch_candidates>0`, `batch_calls>0`, `batch_mismatches=0`, and `batch_skips=0`, plus - byte-identical stdout. Benchmark the K1024 persistent kernel as a + byte-identical stdout. Build the resident prefill harness and run + `./speed-bench/cuda_q4_prefill_bench --path mmq --case outa --tokens + 127,128,129,257,512,2048,4096 --samples 16 --warmup 4`; require bitwise + equality between `pack8_mmq_unpack` and `grouped_dense`, finite/canary/CPU + oracle success, and record the paired median. Then compare full-model + prefills with + `DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_PREFILL=1` plus + `DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_PREFILL=1` against the dominant + `DS4_CUDA_NO_Q4_GROUPED_ATTN_A_PREFILL=1` rollback. Benchmark the K1024 + persistent kernel as a separate fail-closed arm with both `DS4_CUDA_ENABLE_Q4_K1024_PERSISTENT=1` and `DS4_CUDA_REQUIRE_Q4_K1024_PERSISTENT=1`; its rollback is diff --git a/README.md b/README.md index f65c7ed77..cd0360461 100644 --- a/README.md +++ b/README.md @@ -756,6 +756,13 @@ arithmetic: dispatch while keeping `ncols_dst=1`; `DS4_CUDA_NO_Q4_GROUPED_ATTN_A_BATCH=1` restores the per-token grouped loop and `DS4_CUDA_NO_Q4_GROUPED_ATTN_A=1` restores the per-group loop; +- for prefill widths above eight, the experimental + `DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_PREFILL=1` path quantizes the strided + `[token][group][K]` input in one launch and writes each group directly into + `[token][group][rank]`. It removes the eight F32 pack/unpack copies while + keeping one established stream-K MMQ reduction per group. Add + `DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_PREFILL=1` for fail-closed tests; + `DS4_CUDA_NO_Q4_GROUPED_ATTN_A_PREFILL=1` is the dominant local rollback; - attention-output B keeps its canonical MMVQ result and the ordinary HC epilogue inside the graph-compatible fused call. The row-packed epilogue remains oracle-only until a GB10 device run proves it bit-exact; @@ -788,6 +795,12 @@ directly enqueues the canonical reference instead of consuming an unchecked candidate. The scratch, grouped, and persistent paths are also covered by `make test-mmq-parity-cuda CUDA_ARCH=sm_121`. +CUDA Q4_K MMQ performs its non-finite output guard in the final write-back +(or after the final stream-K fixup) instead of launching a separate +full-output sanitizer. Finite results and the per-group reduction order are +unchanged; the resident CUDA prefill harness checks them bit-for-bit against +the former pack/MMQ/unpack path. + Two additional CUDA fusions remain experimental until a device oracle passes on the target GPU. `DS4_CUDA_ENABLE_HC_NORM_MIX_FUSE=1` combines HC RMSNorm with the narrow F16 mixer when the selected standalone kernels have the same diff --git a/scripts/environment_variables.tsv b/scripts/environment_variables.tsv index 69d349e4c..2fc3146e4 100644 --- a/scripts/environment_variables.tsv +++ b/scripts/environment_variables.tsv @@ -85,6 +85,7 @@ runtime/cuda DS4_CUDA_ENABLE_Q4_ATTN_OUT_HC_FUSE value-aware flag, default off; runtime/cuda DS4_CUDA_ENABLE_Q4_ATTN_Q_B_F16_CACHE value-aware persistent-cache opt-in, default off; unset/empty/0/false/no/off is off; DISABLE cancels an optional persistent request but leaves the automatic transient path independent; REQUIRE plus DISABLE fails closed Prewarm and use persistent resident F16 sidecars for eligible single-GPU Q4_K attn_q_b prefills. ds4_cuda.cu:2490 runtime/cuda DS4_CUDA_ENABLE_Q4_ATTN_Q_B_F16_OUTPUT value-aware experimental opt-in; unset/empty/0/false/no/off keeps the release F32 projection boundary; other nonempty values enable Write eligible resident Q4_K attn_q_b GEMM output in F16 and run the half-input norm/RoPE epilogue; SSD remains excluded. ds4_cuda.cu:2523 runtime/cuda DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_BATCH value-aware opt-in, default off; nonempty value other than exact 0 enables; rollback wins Enable flattened grouped attention-A MMQ for two-to-eight-token GB10 batches. cuda/mmq/ds4_mmq.cu:4303 +runtime/cuda DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_PREFILL value-aware opt-in, default off; unset/empty/exact 0 is off, any other nonempty value requests the path; REQUIRE also requests it; local/global rollback wins Enable direct-strided grouped Q4_K attention-A MMQ for GB10 prefill widths above eight tokens, removing per-group pack/unpack copies while preserving the per-group reduction tree. ds4_cuda.cu:41928 runtime/cuda DS4_CUDA_ENABLE_Q4_K1024_PERSISTENT presence flag, default off; any defined value including 0 requests the path; rollback wins Enable the GB10 persistent-CTA kernel for M=32768, N=1, K=1024 Q4. cuda/mmq/ds4_mmq.cu:3905 runtime/cuda DS4_CUDA_ENABLE_Q8_FOLD strict flag, default off; only exact value 1 enables; overridden by DS4_CUDA_NO_Q8_FOLD Enable one-shot producer-to-consumer reuse of freshly quantized Q8_1 data. ds4_cuda.cu:785 runtime/cuda DS4_CUDA_ENABLE_STREAMING_EXPERT_PERSISTENT_CACHE false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Enable streaming expert persistent cache in CUDA SSD streaming. ds4_cuda.cu:4111 @@ -232,6 +233,7 @@ runtime/cuda DS4_CUDA_NO_Q4_DENSE_SCRATCH presence kill switch; default unset (e runtime/cuda DS4_CUDA_NO_Q4_GB10_FAST presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the GB10-specific Q4 fast-path family. cuda/mmq/ds4_mmq.cu:3908 runtime/cuda DS4_CUDA_NO_Q4_GROUPED_ATTN_A presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q4 grouped attn a CUDA Q4 optimization. cuda/mmq/ds4_mmq.cu:4295 runtime/cuda DS4_CUDA_NO_Q4_GROUPED_ATTN_A_BATCH presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q4 grouped attn a batch CUDA Q4 optimization. cuda/mmq/ds4_mmq.cu:4305 +runtime/cuda DS4_CUDA_NO_Q4_GROUPED_ATTN_A_PREFILL presence kill switch; default unset; any defined value including empty or 0 disables and dominates ENABLE/REQUIRE Restore the eight pack/MMQ/unpack Q4 attention-A prefill projections. ds4_cuda.cu:41930 runtime/cuda DS4_CUDA_NO_Q4_K1024_PERSISTENT presence kill switch, default off; any defined value including 0 disables Disable the Q4 K1024 persistent CUDA Q4 optimization. cuda/mmq/ds4_mmq.cu:3907 runtime/cuda DS4_CUDA_NO_Q8_ALIGNED_DENSE_SCRATCH presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q8 aligned dense scratch CUDA Q8 optimization. cuda/mmq/ds4_mmq.cu:5578 runtime/cuda DS4_CUDA_NO_Q8_ALIGNED_PERSISTENT presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q8 aligned persistent CUDA Q8 optimization. cuda/mmq/ds4_mmq.cu:5410 @@ -302,6 +304,7 @@ runtime/cuda DS4_CUDA_Q_NORM_ROPE_FUSE value-aware boolean; default on; unset/em runtime/cuda DS4_CUDA_REQUIRE_IQ2_XXS_SSD_PREFILL_MMQ false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Require the CUDA IQ2 XXS SSD prefill MMQ path; fail closed when unavailable. ds4_cuda.cu:4631 runtime/cuda DS4_CUDA_REQUIRE_Q4_ATTN_Q_B_F16_CACHE value-aware strict opt-in, default off; unset/empty/0/false/no/off is off, any other nonempty value requires eligible batches to use the cache; DISABLE wins Fail an eligible CUDA prefill instead of falling back when the resident Q4_K attn_q_b F16 specialization cannot be prepared or dispatched. ds4_cuda.cu:2492; ds4_cuda.cu:2497 runtime/cuda DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_BATCH value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on Fail if grouped batched attention-A cannot be used. ds4_cuda.cu:40198 +runtime/cuda DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_PREFILL value-aware fail-closed assertion, default off; unset/empty/exact 0 is off, any other nonempty value requests the candidate and rejects ineligibility before enqueue Require the GB10 grouped Q4_K attention-A prefill path instead of silently using pack/MMQ/unpack. ds4_cuda.cu:41889 runtime/cuda DS4_CUDA_REQUIRE_Q4_K1024_PERSISTENT presence flag, default off; any defined value including 0 makes ineligible candidate fail closed Fail when the exact Q4 K1024 persistent candidate is unavailable instead of using MMVQ. cuda/mmq/ds4_mmq.cu:3929 runtime/cuda DS4_CUDA_REQUIRE_STREAMING_EXPERT_PERSISTENT_CACHE false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Require streaming expert persistent cache in CUDA SSD streaming; fail closed when unavailable. ds4_cuda.cu:4117 runtime/cuda DS4_CUDA_REQUIRE_STREAMING_SELECTED_BATCH_IO false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Require streaming selected batch I/O in CUDA SSD streaming; fail closed when unavailable. ds4_cuda.cu:4738 diff --git a/speed-bench/README.md b/speed-bench/README.md index 890a65c4c..8987a0b0e 100644 --- a/speed-bench/README.md +++ b/speed-bench/README.md @@ -182,11 +182,13 @@ make cuda-q4-prefill-bench CUDA_ARCH=sm_121 ./speed-bench/cuda_q4_prefill_bench --path mmq ``` -The default `dense`, `pair`, and `qb` cases use the same production shapes and -token set as the ROCm harness. The synthetic GGUF-layout weights are copied by +The default `dense`, `pair`, `qb`, and GB10-only `outa` cases use production +shapes and include both sides of the 128-column MMQ tail. The synthetic +GGUF-layout weights are copied by the backend into a `cudaMalloc` allocation before warmup. A CUDA test hook checks the backend-owned pointer provenance and device attributes of every -dense, KV, and q_b range in every rotating weight set; a global free-memory +dense, KV, q_b, output-A, and minimal output-B range in every rotating weight +set; a global free-memory delta is printed only as a diagnostic and is not accepted as residency proof. CUDA events measure the production backend GPU interval, including stream-ordered scratch allocation/free, tail clears, activation quantization, @@ -222,6 +224,23 @@ API, with bit-exact pair outputs. Prefill pair is skipped under `--path legacy` because the CUDA pair API intentionally returns control to two independent dense projections for `N>8` when MMQ is disabled. +On GB10, isolate the production Flash attention-output A geometry +(`groups=8`, `K=4096`, `rank=1024`) and its 127/128/129 token tails with: + +``` +./speed-bench/cuda_q4_prefill_bench \ + --path mmq --case outa --tokens 127,128,129,257,512,2048 \ + --samples 16 --warmup 4 +``` + +This is an in-process ABBA/BAAB comparison between the current eight-group +pack/MMQ/unpack sequence and the strictly required grouped-prefill dispatch. +The public API must also execute output-B, so the fixture uses a valid Q4_K +`K=8192,M=256` output-B common to both arms. It is 6.25% of output-A's MACs; +the result prints `focus_macs_per_token` and `common_macs_per_token` separately +and checks the two arms bit-for-bit. SSD streaming is forced off and every +measured weight range must resolve to backend-owned device storage. + ### Resident IQ2/Q2 MoE prefill on ROCm and CUDA The backend-neutral fixture uses the production `N=4096`, 256-expert, top-6 diff --git a/speed-bench/cuda_q4_prefill_bench.cu b/speed-bench/cuda_q4_prefill_bench.cu index 4cea324ec..09dbea6b8 100644 --- a/speed-bench/cuda_q4_prefill_bench.cu +++ b/speed-bench/cuda_q4_prefill_bench.cu @@ -27,12 +27,26 @@ constexpr uint32_t kDenseM = 1024u; constexpr uint32_t kKvM = 512u; constexpr uint32_t kQbK = 1024u; constexpr uint32_t kQbM = 32768u; +constexpr uint32_t kOutputGroups = 8u; +constexpr uint32_t kOutputRank = 1024u; +constexpr uint32_t kOutputLowDim = kOutputGroups * kOutputRank; +constexpr uint32_t kOutputMinB = 256u; constexpr uint32_t kDefaultSets = 4u; constexpr uint32_t kDefaultSamples = 8u; constexpr uint32_t kDefaultWarmup = 2u; constexpr uint32_t kGuardWords = 64u; constexpr uint64_t kCompareChunk = 4u * 1024u * 1024u; +constexpr const char *kGroupedPrefillEnable = + "DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_PREFILL"; +constexpr const char *kGroupedPrefillDisable = + "DS4_CUDA_NO_Q4_GROUPED_ATTN_A_PREFILL"; +constexpr const char *kGroupedPrefillRequire = + "DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_PREFILL"; +constexpr const char *kGroupedGlobalDisable = + "DS4_CUDA_NO_Q4_GROUPED_ATTN_A"; +constexpr const char *kGb10GlobalDisable = "DS4_CUDA_NO_Q4_GB10_FAST"; + struct block_q4_K_host { uint16_t d; uint16_t dmin; @@ -48,6 +62,7 @@ enum class bench_case { dense, pair, qb, + outa, }; enum class cuda_path { @@ -58,7 +73,8 @@ enum class cuda_path { struct config { bench_case selected = bench_case::all; cuda_path path = cuda_path::mmq; - std::vector tokens = {9u, 17u, 33u, 128u, 512u}; + std::vector tokens = {9u, 17u, 33u, 127u, 128u, 129u, 257u, + 512u}; uint32_t sets = kDefaultSets; uint32_t samples = kDefaultSamples; uint32_t warmup = kDefaultWarmup; @@ -68,6 +84,8 @@ struct weight_set { uint64_t dense_offset = 0; uint64_t kv_offset = 0; uint64_t qb_offset = 0; + uint64_t output_a_offset = 0; + uint64_t output_b_offset = 0; }; struct model_fixture { @@ -146,6 +164,10 @@ struct event_timer { struct arm { const char *name; std::function dispatch; + // Host-only path selection must happen before the start event. Otherwise + // an idle device can execute that event while setenv()/unsetenv() is still + // running, folding host-side gate switching into the reported GPU time. + std::function select; }; struct stats { @@ -199,6 +221,10 @@ bool make_model(model_fixture *model, uint32_t sets) { const uint64_t dense_bytes = q4_weight_bytes(kDenseK, kDenseM); const uint64_t kv_bytes = q4_weight_bytes(kDenseK, kKvM); const uint64_t qb_bytes = q4_weight_bytes(kQbK, kQbM); + const uint64_t output_a_bytes = + q4_weight_bytes(kDenseK, kOutputLowDim); + const uint64_t output_b_bytes = + q4_weight_bytes(kOutputLowDim, kOutputMinB); model->weights.resize(sets); uint64_t cursor = 0; @@ -212,6 +238,8 @@ bool make_model(model_fixture *model, uint32_t sets) { model->weights[i].dense_offset = append(dense_bytes); model->weights[i].kv_offset = append(kv_bytes); model->weights[i].qb_offset = append(qb_bytes); + model->weights[i].output_a_offset = append(output_a_bytes); + model->weights[i].output_b_offset = append(output_b_bytes); } model->size = align_up(cursor, page); void *storage = nullptr; @@ -228,6 +256,10 @@ bool make_model(model_fixture *model, uint32_t sets) { 0x85a308d3u ^ (i * 0x7f4a7c15u)); fill_q4(model->data + model->weights[i].qb_offset, qb_bytes, 0x13198a2eu ^ (i * 0x94d049bbu)); + fill_q4(model->data + model->weights[i].output_a_offset, + output_a_bytes, 0xa4093822u ^ (i * 0x2545f491u)); + fill_q4(model->data + model->weights[i].output_b_offset, + output_b_bytes, 0x299f31d0u ^ (i * 0x369dea0fu)); } return true; } @@ -486,6 +518,80 @@ bool sampled_cpu_oracle(const ds4_gpu_tensor *output, return true; } +bool sampled_grouped_cpu_oracle( + const ds4_gpu_tensor *output, const model_fixture &model, + uint64_t weight_offset, const std::vector &activation, + uint32_t n_tokens, const char *label) { + const std::vector tokens = sample_indices(n_tokens); + const std::vector rows = sample_indices(kOutputRank); + const uint64_t blocks_per_row = kDenseK / kQkK; + const uint64_t row_bytes = blocks_per_row * sizeof(block_q4_K_host); + std::vector weight_row; + const double abs_tol = 0.25 * std::sqrt(static_cast(kDenseK)); + constexpr double rel_tol = 0.06; + + for (uint32_t group = 0; group < kOutputGroups; group++) { + for (uint32_t row : rows) { + const uint64_t weight_row_index = + static_cast(group) * kOutputRank + row; + const auto *blocks = + reinterpret_cast( + model.data + weight_offset + weight_row_index * row_bytes); + dequantize_q4_row(blocks, kDenseK, &weight_row); + for (uint32_t token : tokens) { + const float *x = activation.data() + + (static_cast(token) * kOutputGroups + group) * + kDenseK; + float reference = 0.0f; + for (uint32_t k = 0; k < kDenseK; k++) { + reference += weight_row[k] * x[k]; + } + const uint64_t element = + static_cast(token) * kOutputLowDim + + static_cast(group) * kOutputRank + row; + float got = 0.0f; + if (!ds4_gpu_tensor_read(output, element * sizeof(float), + &got, sizeof(got))) { + std::fprintf(stderr, + "%s: grouped output read failed\n", label); + return false; + } + const double abs_error = std::fabs( + static_cast(got) - reference); + const double rel_error = reference != 0.0f + ? abs_error / std::fabs(static_cast(reference)) + : (abs_error == 0.0 + ? 0.0 + : std::numeric_limits::infinity()); + if (!std::isfinite(got) || + (abs_error > abs_tol && rel_error > rel_tol)) { + std::fprintf( + stderr, + "%s: grouped CPU oracle mismatch token=%u group=%u " + "row=%u got=%.7g reference=%.7g abs=%.5g rel=%.5g " + "limits=%.5g/%.3g\n", + label, token, group, row, got, reference, abs_error, + rel_error, abs_tol, rel_tol); + return false; + } + } + } + } + return true; +} + +bool select_grouped_prefill_baseline() { + return unsetenv(kGroupedPrefillEnable) == 0 && + setenv(kGroupedPrefillDisable, "1", 1) == 0 && + unsetenv(kGroupedPrefillRequire) == 0; +} + +bool select_grouped_prefill_candidate() { + return setenv(kGroupedPrefillEnable, "1", 1) == 0 && + unsetenv(kGroupedPrefillDisable) == 0 && + setenv(kGroupedPrefillRequire, "1", 1) == 0; +} + double percentile(std::vector sorted, double fraction) { std::sort(sorted.begin(), sorted.end()); if (sorted.empty()) return 0.0; @@ -510,6 +616,10 @@ const char *path_name(cuda_path path) { return path == cuda_path::mmq ? "mmq" : "legacy"; } +bool select_arm(const arm &which) { + return !which.select || which.select(); +} + bool benchmark_single_path( const char *case_name, uint32_t n_tokens, uint32_t in_dim, uint32_t out_dim, const config &cfg, const arm &which, @@ -519,7 +629,8 @@ bool benchmark_single_path( // recording a CUDA event. Upload, poison, readback, and CPU work remain // outside the measured interval. for (uint32_t set = 0; set < cfg.sets; set++) { - if (!oracle_prepare() || !which.dispatch(set) || + if (!oracle_prepare() || !select_arm(which) || + !which.dispatch(set) || !ds4_gpu_synchronize() || !oracle(set)) { std::fprintf(stderr, "cuda-q4-prefill-bench: %s oracle failed for " @@ -530,7 +641,8 @@ bool benchmark_single_path( } for (uint32_t i = 0; i < cfg.warmup; i++) { - if (!which.dispatch(i % cfg.sets) || !ds4_gpu_synchronize()) { + if (!select_arm(which) || !which.dispatch(i % cfg.sets) || + !ds4_gpu_synchronize()) { return false; } } @@ -540,7 +652,8 @@ bool benchmark_single_path( samples.reserve(cfg.samples); for (uint32_t i = 0; i < cfg.samples; i++) { float elapsed = 0.0f; - if (!timer.measure([&]() { return which.dispatch(i % cfg.sets); }, + if (!select_arm(which) || + !timer.measure([&]() { return which.dispatch(i % cfg.sets); }, &elapsed)) { std::fprintf(stderr, "cuda-q4-prefill-bench: %s/%s timed dispatch failed\n", @@ -565,24 +678,29 @@ bool benchmark_single_path( } bool benchmark_pair_arms( - uint32_t n_tokens, const config &cfg, const arm &baseline, - const arm &candidate, const std::function &oracle_prepare, + const char *case_name, uint32_t n_tokens, uint32_t in_dim, + uint32_t out_dim, uint64_t focus_macs_per_token, + uint64_t common_macs_per_token, const config &cfg, + const arm &baseline, const arm &candidate, + const std::function &oracle_prepare, const std::function &oracle) { for (uint32_t set = 0; set < cfg.sets; set++) { - if (!oracle_prepare() || !baseline.dispatch(set) || - !ds4_gpu_synchronize() || !candidate.dispatch(set) || + if (!oracle_prepare() || !select_arm(baseline) || + !baseline.dispatch(set) || !ds4_gpu_synchronize() || + !select_arm(candidate) || !candidate.dispatch(set) || !ds4_gpu_synchronize() || !oracle(set)) { std::fprintf(stderr, - "cuda-q4-prefill-bench: pair oracle failed for " + "cuda-q4-prefill-bench: %s oracle failed for " "weight set %u\n", - set); + case_name, set); return false; } } for (uint32_t i = 0; i < cfg.warmup; i++) { const uint32_t set = i % cfg.sets; - if (!baseline.dispatch(set) || !ds4_gpu_synchronize() || + if (!select_arm(baseline) || !baseline.dispatch(set) || + !ds4_gpu_synchronize() || !select_arm(candidate) || !candidate.dispatch(set) || !ds4_gpu_synchronize()) { return false; } @@ -596,10 +714,12 @@ bool benchmark_pair_arms( auto take = [&](const arm &which, uint32_t set, std::vector *samples) { float elapsed = 0.0f; - if (!timer.measure([&]() { return which.dispatch(set); }, &elapsed)) { + if (!select_arm(which) || + !timer.measure([&]() { return which.dispatch(set); }, &elapsed)) { std::fprintf(stderr, - "cuda-q4-prefill-bench: pair/%s timed dispatch failed\n", - which.name); + "cuda-q4-prefill-bench: %s/%s timed dispatch " + "failed\n", + case_name, which.name); return false; } samples->push_back(static_cast(elapsed)); @@ -634,20 +754,23 @@ bool benchmark_pair_arms( const double paired_median = percentile(paired_delta, 0.5); const double median_delta = (b.median / a.median - 1.0) * 100.0; const double speedup = (a.median / b.median - 1.0) * 100.0; - const double macs = static_cast(n_tokens) * kDenseK * - (kDenseM + kKvM); + const double macs = static_cast(n_tokens) * + (focus_macs_per_token + common_macs_per_token); std::printf( - "DS4_CUDA_Q4_PREFILL_BENCH case=pair path=%s N=%u K=%u M=%u " + "DS4_CUDA_Q4_PREFILL_BENCH case=%s path=%s N=%u K=%u M=%u " "baseline=%s candidate=%s samples=%u sets=%u " + "focus_macs_per_token=%llu common_macs_per_token=%llu " "baseline_ms_p50=%.6f candidate_ms_p50=%.6f " "baseline_ms_min=%.6f candidate_ms_min=%.6f " "baseline_ms_p95=%.6f candidate_ms_p95=%.6f " "baseline_gmac_s=%.3f candidate_gmac_s=%.3f " "candidate_delta_pct=%.3f paired_delta_pct_p50=%.3f " "speedup_pct=%.3f\n", - path_name(cfg.path), n_tokens, kDenseK, kDenseM + kKvM, - baseline.name, candidate.name, cfg.samples, cfg.sets, a.median, - b.median, a.minimum, b.minimum, a.p95, b.p95, + case_name, path_name(cfg.path), n_tokens, in_dim, out_dim, + baseline.name, candidate.name, cfg.samples, cfg.sets, + static_cast(focus_macs_per_token), + static_cast(common_macs_per_token), + a.median, b.median, a.minimum, b.minimum, a.p95, b.p95, macs / (a.median * 1.0e6), macs / (b.median * 1.0e6), median_delta, paired_median, speedup); std::fflush(stdout); @@ -679,7 +802,8 @@ bool run_dense(const model_fixture &model, const config &cfg, output.ptr, model.data, model.size, model.weights[set].dense_offset, kQ4Type, kDenseK, kDenseM, x.ptr, n_tokens) != 0; - }}; + }, + {}}; return benchmark_single_path( "dense", n_tokens, kDenseK, kDenseM, cfg, path, [&]() { @@ -743,7 +867,8 @@ bool run_pair(const model_fixture &model, const config &cfg, separate1.ptr, model.data, model.size, model.weights[set].kv_offset, kQ4Type, kDenseK, kKvM, x.ptr, n_tokens) != 0; - }}; + }, + {}}; const arm candidate = { "pair_mmq", [&](uint32_t set) { @@ -752,9 +877,12 @@ bool run_pair(const model_fixture &model, const config &cfg, model.weights[set].dense_offset, model.weights[set].kv_offset, kDenseK, kDenseM, kKvM, x.ptr, n_tokens) != 0; - }}; + }, + {}}; return benchmark_pair_arms( - n_tokens, cfg, baseline, candidate, + "pair", n_tokens, kDenseK, kDenseM + kKvM, + static_cast(kDenseK) * (kDenseM + kKvM), 0u, + cfg, baseline, candidate, [&]() { return poison_output(separate0.ptr, out0_bytes, 0x7fc10001u, 0x4000u) && @@ -828,7 +956,8 @@ bool run_qb(const model_fixture &model, const config &cfg, output.ptr, model.data, model.size, model.weights[set].qb_offset, kQ4Type, kQbK, kQbM, x.ptr, n_tokens) != 0; - }}; + }, + {}}; return benchmark_single_path( "q_b", n_tokens, kQbK, kQbM, cfg, path, [&]() { @@ -849,15 +978,157 @@ bool run_qb(const model_fixture &model, const config &cfg, check_guard(output.ptr, out_bytes, 0x9000u, "q_b output final"); } +bool run_output_a(const model_fixture &model, const config &cfg, + uint32_t n_tokens) { + if (cfg.path == cuda_path::legacy) { + std::printf( + "DS4_CUDA_Q4_PREFILL_SKIP case=outa_plus_min_b path=legacy " + "N=%u reason=grouped_prefill_requires_mmq\n", + n_tokens); + std::fflush(stdout); + return true; + } + + uint64_t heads_elements = 0; + uint64_t low_elements = 0; + uint64_t out_elements = 0; + if (!checked_mul(n_tokens, + static_cast(kOutputGroups) * kDenseK, + &heads_elements) || + !checked_mul(n_tokens, kOutputLowDim, &low_elements) || + !checked_mul(n_tokens, kOutputMinB, &out_elements)) { + return false; + } + const uint64_t heads_bytes = heads_elements * sizeof(float); + const uint64_t low_bytes = low_elements * sizeof(float); + const uint64_t out_bytes = out_elements * sizeof(float); + const uint64_t group_tmp_bytes = + static_cast(n_tokens) * kDenseK * sizeof(float); + const uint64_t low_tmp_bytes = + static_cast(n_tokens) * kOutputRank * sizeof(float); + const uint64_t guard_bytes = kGuardWords * sizeof(uint32_t); + tensor_owner heads(heads_bytes + guard_bytes); + tensor_owner baseline_low(low_bytes + guard_bytes); + tensor_owner baseline_out(out_bytes + guard_bytes); + tensor_owner candidate_low(low_bytes + guard_bytes); + tensor_owner candidate_out(out_bytes + guard_bytes); + tensor_owner group_tmp(group_tmp_bytes + guard_bytes); + tensor_owner low_tmp(low_tmp_bytes + guard_bytes); + std::vector activation; + fill_activation(&activation, n_tokens, kOutputGroups * kDenseK); + if (!heads.ptr || !baseline_low.ptr || !baseline_out.ptr || + !candidate_low.ptr || !candidate_out.ptr || !group_tmp.ptr || + !low_tmp.ptr || + !ds4_gpu_tensor_write(heads.ptr, 0, activation.data(), heads_bytes) || + !prepare_guard(heads.ptr, heads_bytes, 0xa000u)) { + std::fprintf(stderr, + "outa_plus_min_b N=%u: tensor setup failed\n", + n_tokens); + return false; + } + + const arm baseline = { + "pack8_mmq_unpack", + [&](uint32_t set) { + return ds4_gpu_attention_output_q4_K_batch_tensor( + baseline_out.ptr, baseline_low.ptr, group_tmp.ptr, + low_tmp.ptr, model.data, model.size, + model.weights[set].output_a_offset, + model.weights[set].output_b_offset, kQ4Type, + kDenseK, kOutputRank, kOutputGroups, kOutputMinB, + heads.ptr, n_tokens) > 0; + }, + select_grouped_prefill_baseline}; + const arm candidate = { + "grouped_dense", + [&](uint32_t set) { + return ds4_gpu_attention_output_q4_K_batch_tensor( + candidate_out.ptr, candidate_low.ptr, group_tmp.ptr, + low_tmp.ptr, model.data, model.size, + model.weights[set].output_a_offset, + model.weights[set].output_b_offset, kQ4Type, + kDenseK, kOutputRank, kOutputGroups, kOutputMinB, + heads.ptr, n_tokens) > 0; + }, + select_grouped_prefill_candidate}; + const uint64_t output_a_macs_per_token = + static_cast(kOutputGroups) * kDenseK * kOutputRank; + const uint64_t output_b_macs_per_token = + static_cast(kOutputLowDim) * kOutputMinB; + const bool ok = benchmark_pair_arms( + "outa_plus_min_b", n_tokens, kDenseK, kOutputLowDim, + output_a_macs_per_token, output_b_macs_per_token, cfg, + baseline, candidate, + [&]() { + return poison_output(baseline_low.ptr, low_bytes, 0x7fc10001u, + 0xb000u) && + poison_output(baseline_out.ptr, out_bytes, 0x7fc20002u, + 0xc000u) && + poison_output(candidate_low.ptr, low_bytes, 0x7fc30003u, + 0xd000u) && + poison_output(candidate_out.ptr, out_bytes, 0x7fc40004u, + 0xe000u) && + prepare_guard(group_tmp.ptr, group_tmp_bytes, 0xf000u) && + prepare_guard(low_tmp.ptr, low_tmp_bytes, 0x11000u); + }, + [&](uint32_t set) { + return bitwise_equal(baseline_low.ptr, candidate_low.ptr, + low_bytes, + "output_a grouped dense vs pack/MMQ") && + bitwise_equal(baseline_out.ptr, candidate_out.ptr, + out_bytes, + "output_a minimal-B final output") && + output_is_finite(baseline_low.ptr, low_bytes, + "output_a low") && + output_is_finite(baseline_out.ptr, out_bytes, + "output_a minimal-B output") && + sampled_grouped_cpu_oracle( + baseline_low.ptr, model, + model.weights[set].output_a_offset, activation, + n_tokens, "output_a") && + check_guard(heads.ptr, heads_bytes, 0xa000u, + "output_a heads") && + check_guard(baseline_low.ptr, low_bytes, 0xb000u, + "output_a baseline low") && + check_guard(baseline_out.ptr, out_bytes, 0xc000u, + "output_a baseline out") && + check_guard(candidate_low.ptr, low_bytes, 0xd000u, + "output_a candidate low") && + check_guard(candidate_out.ptr, out_bytes, 0xe000u, + "output_a candidate out") && + check_guard(group_tmp.ptr, group_tmp_bytes, 0xf000u, + "output_a group scratch") && + check_guard(low_tmp.ptr, low_tmp_bytes, 0x11000u, + "output_a low scratch"); + }); + return ok && + check_guard(heads.ptr, heads_bytes, 0xa000u, + "output_a heads final") && + check_guard(baseline_low.ptr, low_bytes, 0xb000u, + "output_a baseline low final") && + check_guard(baseline_out.ptr, out_bytes, 0xc000u, + "output_a baseline out final") && + check_guard(candidate_low.ptr, low_bytes, 0xd000u, + "output_a candidate low final") && + check_guard(candidate_out.ptr, out_bytes, 0xe000u, + "output_a candidate out final") && + check_guard(group_tmp.ptr, group_tmp_bytes, 0xf000u, + "output_a group scratch final") && + check_guard(low_tmp.ptr, low_tmp_bytes, 0x11000u, + "output_a low scratch final"); +} + void usage(FILE *stream, const char *argv0) { std::fprintf( stream, "usage: %s [options]\n\n" "Resident CUDA Q4_K prefill kernel benchmark (CUDA event timing).\n\n" " --path mmq|legacy process-wide path (default: mmq)\n" - " --case all|dense|pair|qb case to run (default: all)\n" + " --case all|dense|pair|qb|outa\n" + " case to run (default: all)\n" " --tokens N[,N...] token counts, each 9..4096\n" - " --full use 9,16,17,31,32,33,128,512,4096\n" + " --full use 9,16,17,31,32,33,127,128,129,257," + "512,2048,4096\n" " --sets N rotating resident weight sets (default: %u)\n" " --samples N samples/arm, multiple of 4 (default: %u)\n" " --warmup N untimed dispatches/arm (default: %u)\n" @@ -866,7 +1137,10 @@ void usage(FILE *stream, const char *argv0) { "legacy/MMQ\nprocesses (preferably ABBA/BAAB) to compare them because " "the CUDA backend\ncaches DS4_CUDA_MMQ on its first dispatch. Pair is " "an in-process ABBA/BAAB\ncomparison of two MMQ projections against " - "the fused public pair API.\n", + "the fused public pair API. outa\ncompares the current eight-group " + "pack/MMQ/unpack path against the strict\ngrouped-prefill candidate " + "at the production attention-A shape. It includes\na common minimal " + "Q4 output-B (M=256) whose MACs are reported separately.\n", argv0, kDefaultSets, kDefaultSamples, kDefaultWarmup); } @@ -940,6 +1214,8 @@ config parse_options(int argc, char **argv) { cfg.selected = bench_case::pair; } else if (!std::strcmp(value, "qb")) { cfg.selected = bench_case::qb; + } else if (!std::strcmp(value, "outa")) { + cfg.selected = bench_case::outa; } else { std::fprintf(stderr, "invalid --case: %s\n", value); std::exit(2); @@ -947,7 +1223,8 @@ config parse_options(int argc, char **argv) { } else if (!std::strcmp(argv[i], "--tokens")) { cfg.tokens = parse_tokens(need_value(&i, argc, argv)); } else if (!std::strcmp(argv[i], "--full")) { - cfg.tokens = {9u, 16u, 17u, 31u, 32u, 33u, 128u, 512u, 4096u}; + cfg.tokens = {9u, 16u, 17u, 31u, 32u, 33u, 127u, + 128u, 129u, 257u, 512u, 2048u, 4096u}; } else if (!std::strcmp(argv[i], "--sets")) { cfg.sets = parse_u32(need_value(&i, argc, argv), "--sets", 1u, 32u); @@ -968,9 +1245,12 @@ config parse_options(int argc, char **argv) { "--samples must be a multiple of 4 for balanced runs\n"); std::exit(2); } - if (cfg.path == cuda_path::legacy && cfg.selected == bench_case::pair) { + if (cfg.path == cuda_path::legacy && + (cfg.selected == bench_case::pair || + cfg.selected == bench_case::outa)) { std::fprintf(stderr, - "--case pair requires --path mmq for prefill N > 8\n"); + "--case pair/outa requires --path mmq for prefill " + "N > 8\n"); std::exit(2); } return cfg; @@ -1007,6 +1287,10 @@ bool verify_resident_weight_ranges(const model_fixture &model) { const uint64_t dense_bytes = q4_weight_bytes(kDenseK, kDenseM); const uint64_t kv_bytes = q4_weight_bytes(kDenseK, kKvM); const uint64_t qb_bytes = q4_weight_bytes(kQbK, kQbM); + const uint64_t output_a_bytes = + q4_weight_bytes(kDenseK, kOutputLowDim); + const uint64_t output_b_bytes = + q4_weight_bytes(kOutputLowDim, kOutputMinB); for (uint32_t set = 0; set < model.weights.size(); set++) { struct range_desc { const char *name; @@ -1017,6 +1301,9 @@ bool verify_resident_weight_ranges(const model_fixture &model) { {"dense", model.weights[set].dense_offset, dense_bytes}, {"kv", model.weights[set].kv_offset, kv_bytes}, {"q_b", model.weights[set].qb_offset, qb_bytes}, + {"output_a", model.weights[set].output_a_offset, output_a_bytes}, + {"output_b_min", model.weights[set].output_b_offset, + output_b_bytes}, }; for (const range_desc &range : ranges) { if (!ds4_cuda_test_model_range_is_device_resident( @@ -1069,11 +1356,21 @@ int main(int argc, char **argv) { env_snapshot copy_guard("DS4_CUDA_COPY_MODEL"); env_snapshot pair_guard("DS4_CUDA_DISABLE_Q4_DENSE_PAIR"); env_snapshot graph_guard("DS4_CUDA_DECODE_GRAPHS"); + env_snapshot grouped_enable_guard(kGroupedPrefillEnable); + env_snapshot grouped_disable_guard(kGroupedPrefillDisable); + env_snapshot grouped_require_guard(kGroupedPrefillRequire); + env_snapshot grouped_global_disable_guard(kGroupedGlobalDisable); + env_snapshot gb10_global_disable_guard(kGb10GlobalDisable); if (setenv("DS4_CUDA_MMQ", cfg.path == cuda_path::mmq ? "1" : "0", 1) != 0 || setenv("DS4_CUDA_COPY_MODEL", "1", 1) != 0 || setenv("DS4_CUDA_DECODE_GRAPHS", "0", 1) != 0 || - unsetenv("DS4_CUDA_DISABLE_Q4_DENSE_PAIR") != 0) { + unsetenv("DS4_CUDA_DISABLE_Q4_DENSE_PAIR") != 0 || + unsetenv(kGroupedPrefillEnable) != 0 || + unsetenv(kGroupedPrefillDisable) != 0 || + unsetenv(kGroupedPrefillRequire) != 0 || + unsetenv(kGroupedGlobalDisable) != 0 || + unsetenv(kGb10GlobalDisable) != 0) { std::fprintf(stderr, "cuda-q4-prefill-bench: environment setup failed\n"); return 1; @@ -1094,6 +1391,16 @@ int main(int argc, char **argv) { "cuda-q4-prefill-bench: cannot query device properties\n"); return 1; } + const bool grouped_prefill_supported = + device_count == 1 && properties.major == 12 && + properties.minor == 1 && properties.warpSize == 32; + if (cfg.selected == bench_case::outa && + !grouped_prefill_supported) { + std::fprintf(stderr, + "cuda-q4-prefill-bench: SKIP (outa requires one " + "GB10/sm_121 device)\n"); + return 77; + } if (!ds4_gpu_init()) { std::fprintf(stderr, "cuda-q4-prefill-bench: ds4_gpu_init failed\n"); return 1; @@ -1142,13 +1449,14 @@ int main(int argc, char **argv) { "device_free_delta_valid=%d timing=cuda_events " "ssd_streaming=off model_storage=cudaMalloc " "residency=backend_provenance strict_mmq=%d " - "dispatch_stream=legacy_default\n", + "grouped_attn_a_prefill=%s dispatch_stream=legacy_default\n", properties.name, properties.major, properties.minor, properties.warpSize, path_name(cfg.path), cfg.sets, static_cast(model.payload_bytes) / 1048576.0, static_cast(resident_delta) / 1048576.0, resident_delta_valid ? 1 : 0, - cfg.path == cuda_path::mmq ? 1 : 0); + cfg.path == cuda_path::mmq ? 1 : 0, + grouped_prefill_supported ? "available" : "skipped"); std::fflush(stdout); for (uint32_t n_tokens : cfg.tokens) { if (includes(cfg.selected, bench_case::dense)) { @@ -1160,6 +1468,10 @@ int main(int argc, char **argv) { if (ok && includes(cfg.selected, bench_case::qb)) { ok = run_qb(model, cfg, n_tokens) && ok; } + if (ok && grouped_prefill_supported && + includes(cfg.selected, bench_case::outa)) { + ok = run_output_a(model, cfg, n_tokens) && ok; + } if (!ok) break; } } From 8c1d88fd693101713cd41cf5853e4a3228cb259a Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Fri, 28 Aug 2026 21:38:15 +0200 Subject: [PATCH 141/189] perf(metal): stream transient q_b F16 from SSD --- Makefile | 12 ++ ds4_gpu.h | 4 + ds4_metal.m | 284 ++++++++++++++++++++++++++--- scripts/environment_variables.tsv | 4 +- tests/test_metal_q4_qb_f16_cache.c | 223 +++++++++++++++++++--- 5 files changed, 476 insertions(+), 51 deletions(-) diff --git a/Makefile b/Makefile index 30f1bee48..5e8e60f2d 100644 --- a/Makefile +++ b/Makefile @@ -217,11 +217,23 @@ test-metal-q4-qb-f16-cache: tests/test_metal_q4_qb_f16_cache -u DS4_METAL_DISABLE_Q4_ATTN_Q_B_F16_RHS \ -u DS4_METAL_DISABLE_Q4_ATTN_Q_B_TRANSIENT_F16 \ -u DS4_METAL_Q4_ATTN_Q_B_TRANSIENT_F16_MIN_TOKENS \ + -u DS4_METAL_UNRETAINED_COMMAND_BUFFERS \ -u DS4_TEST_METAL_Q4_QB_F16_CACHE_TIMING \ -u DS4_TEST_METAL_Q4_QB_F16_CACHE_TIMING_TOKENS \ DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MIN_TOKENS=32 \ DS4_METAL_REQUIRE_Q4_ATTN_Q_B_F16_CACHE=1 \ ./tests/test_metal_q4_qb_f16_cache + env -u DS4_METAL_DISABLE_Q4_ATTN_Q_B_F16_CACHE \ + -u DS4_METAL_ENABLE_Q4_ATTN_Q_B_F16_CACHE_WITH_SSD_STREAMING \ + -u DS4_METAL_DISABLE_Q4_ATTN_Q_B_F16_RHS \ + -u DS4_METAL_DISABLE_Q4_ATTN_Q_B_TRANSIENT_F16 \ + -u DS4_METAL_Q4_ATTN_Q_B_TRANSIENT_F16_MIN_TOKENS \ + -u DS4_TEST_METAL_Q4_QB_F16_CACHE_TIMING \ + -u DS4_TEST_METAL_Q4_QB_F16_CACHE_TIMING_TOKENS \ + DS4_METAL_UNRETAINED_COMMAND_BUFFERS=1 \ + DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MIN_TOKENS=32 \ + DS4_METAL_REQUIRE_Q4_ATTN_Q_B_F16_CACHE=1 \ + ./tests/test_metal_q4_qb_f16_cache test-metal-q4-qb-f16-cache-timing: tests/test_metal_q4_qb_f16_cache env -u DS4_METAL_DISABLE_Q4_ATTN_Q_B_F16_CACHE \ diff --git a/ds4_gpu.h b/ds4_gpu.h index 33b029ec9..5f0de67f9 100644 --- a/ds4_gpu.h +++ b/ds4_gpu.h @@ -319,6 +319,10 @@ typedef struct ds4_gpu_q4_attn_q_b_f16_cache_report { uint64_t fallbacks; uint64_t rejects; uint64_t build_circuit_open; + uint64_t transient_exact_views_created; + uint64_t transient_exact_views_live; + uint64_t model_exact_cache_entries; + uint64_t model_exact_cache_bytes; } ds4_gpu_q4_attn_q_b_f16_cache_report; /* Test observability for the resident Metal Q4_K attn_q_b F16 sidecar. */ void ds4_gpu_test_q4_attn_q_b_f16_cache_report( diff --git a/ds4_metal.m b/ds4_metal.m index 1c6c2fb4b..884a3bf7a 100644 --- a/ds4_metal.m +++ b/ds4_metal.m @@ -567,13 +567,97 @@ g_q4_qb_transient_f16_scratch[DS4_GPU_MAX_STREAMS]; static NSUInteger g_q4_qb_transient_f16_scratch_capacity[DS4_GPU_MAX_STREAMS]; -/* Protect only allocation/publication and cleanup. Encoding never retains - * this mutex; command ordering and the one-in-flight-batch-per-stream rule - * protect reuse after a buffer has been captured strongly by the caller. */ +typedef struct { + const void *model_map; + uint64_t model_size; + uint64_t generation; + uint64_t reserve_bytes; + bool admitted; +} ds4_gpu_q4_qb_transient_f16_admission; +static ds4_gpu_q4_qb_transient_f16_admission + g_q4_qb_transient_f16_admission[DS4_GPU_MAX_STREAMS]; +static uint64_t g_q4_qb_transient_f16_exact_views_created; +static uint64_t g_q4_qb_transient_f16_exact_views_live; +/* Protect allocation/publication, SSD admission metadata, and cleanup. + * Encoding never retains this mutex; command ordering and the + * one-in-flight-batch-per-stream rule protect reuse after a buffer has been + * captured strongly by the caller. */ static pthread_mutex_t g_q4_qb_transient_f16_scratch_mu = PTHREAD_MUTEX_INITIALIZER; static int g_initialized; +static void ds4_gpu_q4_qb_transient_f16_admission_clear(void) { + pthread_mutex_lock(&g_q4_qb_transient_f16_scratch_mu); + memset(g_q4_qb_transient_f16_admission, 0, + sizeof(g_q4_qb_transient_f16_admission)); + pthread_mutex_unlock(&g_q4_qb_transient_f16_scratch_mu); +} + +static bool ds4_gpu_q4_qb_transient_f16_admission_begin( + int stream, + const void *model_map, + uint64_t model_size, + uint64_t reserve_bytes, + bool *same_contract) { + if (stream < 0 || stream >= DS4_GPU_MAX_STREAMS) return false; + pthread_mutex_lock(&g_q4_qb_transient_f16_scratch_mu); + const bool invalidated = + g_q4_qb_transient_f16_admission[stream].admitted; + if (same_contract) { + *same_contract = invalidated && + g_q4_qb_transient_f16_admission[stream].model_map == model_map && + g_q4_qb_transient_f16_admission[stream].model_size == model_size && + g_q4_qb_transient_f16_admission[stream].reserve_bytes == + reserve_bytes; + } + g_q4_qb_transient_f16_admission[stream] = + (ds4_gpu_q4_qb_transient_f16_admission) { + .model_map = model_map, + .model_size = model_size, + .generation = 0u, + .reserve_bytes = reserve_bytes, + .admitted = false, + }; + pthread_mutex_unlock(&g_q4_qb_transient_f16_scratch_mu); + return invalidated; +} + +static void ds4_gpu_q4_qb_transient_f16_admission_commit( + int stream, + const void *model_map, + uint64_t model_size, + uint64_t generation, + uint64_t reserve_bytes) { + if (stream < 0 || stream >= DS4_GPU_MAX_STREAMS) return; + pthread_mutex_lock(&g_q4_qb_transient_f16_scratch_mu); + ds4_gpu_q4_qb_transient_f16_admission *state = + &g_q4_qb_transient_f16_admission[stream]; + if (state->model_map == model_map && + state->model_size == model_size) { + state->generation = generation; + state->reserve_bytes = reserve_bytes; + state->admitted = true; + } + pthread_mutex_unlock(&g_q4_qb_transient_f16_scratch_mu); +} + +static bool ds4_gpu_q4_qb_transient_f16_admission_allows( + int stream, + const void *model_map, + uint64_t model_size, + uint64_t generation) { + if (stream < 0 || stream >= DS4_GPU_MAX_STREAMS) return false; + pthread_mutex_lock(&g_q4_qb_transient_f16_scratch_mu); + const ds4_gpu_q4_qb_transient_f16_admission *state = + &g_q4_qb_transient_f16_admission[stream]; + const bool allowed = state->admitted && + state->model_map == model_map && + state->model_size == model_size && + state->generation == generation; + pthread_mutex_unlock(&g_q4_qb_transient_f16_scratch_mu); + return allowed; +} + static void ds4_gpu_q4_attn_q_b_f16_cache_clear(int reset_stats) { pthread_mutex_lock(&g_q4_attn_q_b_f16_build_mu); pthread_mutex_lock(&g_q4_attn_q_b_f16_cache_mu); @@ -614,6 +698,15 @@ static void ds4_gpu_q4_attn_q_b_f16_cache_clear(int reset_stats) { pthread_mutex_unlock(&g_q4_attn_q_b_f16_build_mu); } +static void ds4_gpu_q4_attn_q_b_f16_advance_generation(void) { + pthread_mutex_lock(&g_q4_attn_q_b_f16_cache_mu); + g_q4_attn_q_b_f16_cache_generation++; + if (g_q4_attn_q_b_f16_cache_generation == 0u) { + g_q4_attn_q_b_f16_cache_generation = 1u; + } + pthread_mutex_unlock(&g_q4_attn_q_b_f16_cache_mu); +} + void ds4_gpu_test_q4_attn_q_b_f16_cache_report( ds4_gpu_q4_attn_q_b_f16_cache_report *report) { if (!report) return; @@ -632,6 +725,15 @@ void ds4_gpu_test_q4_attn_q_b_f16_cache_report( .build_circuit_open = (uint64_t)( g_q4_attn_q_b_f16_build_circuit_state != DS4_Q4_ATTN_Q_B_F16_CIRCUIT_CLOSED), + .transient_exact_views_created = __atomic_load_n( + &g_q4_qb_transient_f16_exact_views_created, + __ATOMIC_ACQUIRE), + .transient_exact_views_live = __atomic_load_n( + &g_q4_qb_transient_f16_exact_views_live, + __ATOMIC_ACQUIRE), + .model_exact_cache_entries = g_model_buffer_cache + ? (uint64_t)[g_model_buffer_cache count] : 0u, + .model_exact_cache_bytes = g_model_buffer_cache_bytes, }; pthread_mutex_unlock(&g_q4_attn_q_b_f16_cache_mu); } @@ -644,6 +746,13 @@ void ds4_gpu_test_q4_attn_q_b_f16_cache_reset(void) { return; } ds4_gpu_q4_attn_q_b_f16_cache_clear(1); + ds4_gpu_q4_qb_transient_f16_admission_clear(); + __atomic_store_n(&g_q4_qb_transient_f16_exact_views_created, + 0u, + __ATOMIC_RELEASE); + __atomic_store_n(&g_q4_qb_transient_f16_exact_views_live, + 0u, + __ATOMIC_RELEASE); __atomic_store_n(&g_q4_attn_q_b_f16_ssd_admission_blocked, 0, __ATOMIC_RELEASE); @@ -669,6 +778,7 @@ int ds4_gpu_release_q4_attn_q_b_f16_sidecars(void) { * breaker open and must not poison the next resident session. */ if (has_entries && g_initialized && !ds4_gpu_synchronize()) return 0; ds4_gpu_q4_attn_q_b_f16_cache_clear(0); + ds4_gpu_q4_qb_transient_f16_admission_clear(); __atomic_store_n(&g_q4_attn_q_b_f16_ssd_admission_blocked, 0, __ATOMIC_RELEASE); @@ -5020,6 +5130,9 @@ void ds4_gpu_set_glm_model(bool enabled) { void ds4_gpu_set_ssd_streaming(bool enabled) { const int was_ssd_streaming = g_ssd_streaming_mode; g_ssd_streaming_mode = enabled ? 1 : 0; + if (g_ssd_streaming_mode != was_ssd_streaming) { + ds4_gpu_q4_qb_transient_f16_admission_clear(); + } /* A real resident->streaming transition must re-evaluate sidecar * admission against the streaming reserve. Reasserting an already active * SSD mode is configuration plumbing, not a lifecycle boundary, and must @@ -11618,8 +11731,16 @@ void ds4_gpu_cleanup(void) { for (int si = 0; si < DS4_GPU_MAX_STREAMS; si++) { g_q4_qb_transient_f16_scratch[si] = nil; g_q4_qb_transient_f16_scratch_capacity[si] = 0u; + g_q4_qb_transient_f16_admission[si] = + (ds4_gpu_q4_qb_transient_f16_admission) {0}; } pthread_mutex_unlock(&g_q4_qb_transient_f16_scratch_mu); + __atomic_store_n(&g_q4_qb_transient_f16_exact_views_created, + 0u, + __ATOMIC_RELEASE); + __atomic_store_n(&g_q4_qb_transient_f16_exact_views_live, + 0u, + __ATOMIC_RELEASE); ds4_gpu_stream_expert_pread_pool_shutdown(); ds4_gpu_stream_expert_cache_clear_all(1); ds4_gpu_stream_expert_cache_live_release(); @@ -25786,6 +25907,27 @@ static uint64_t ds4_gpu_q4_attn_q_b_f16_effective_working_set_reserve( inner_offset); } +static uint64_t ds4_gpu_q4_attn_q_b_f16_exact_source_bytes( + uint64_t model_size, + uint64_t weight_offset, + uint64_t weight_bytes) { + if (model_size == 0u || weight_offset > model_size || + weight_bytes > model_size - weight_offset) { + return UINT64_MAX; + } + const uint64_t page = (uint64_t)getpagesize(); + const uint64_t page_offset = weight_offset & ~(page - 1u); + const uint64_t leading = weight_offset - page_offset; + if (weight_bytes > UINT64_MAX - leading || + leading + weight_bytes > UINT64_MAX - (page - 1u)) { + return UINT64_MAX; + } + uint64_t view_bytes = round_up_u64(leading + weight_bytes, page); + const uint64_t remaining = model_size - page_offset; + if (view_bytes > remaining) view_bytes = remaining; + return view_bytes; +} + static void ds4_gpu_q4_attn_q_b_f16_suppress_builds( ds4_gpu_q4_attn_q_b_f16_circuit_state state) { if (state == DS4_Q4_ATTN_Q_B_F16_CIRCUIT_CLOSED) return; @@ -25911,7 +26053,9 @@ static int ds4_gpu_q4_qb_transient_f16_scratch_ensure_locked( /* Resolve the complete transient path and reserve its one stream-local * expansion buffer before prefill timing starts. Per-layer Q4->F16 work is * intentionally not performed here: unlike PSO creation and allocation, that - * dequantization is a real recurring cost of the optimized projection. */ + * dequantization is a real recurring cost of the optimized projection. SSD + * streaming additionally reserves room for one short-lived exact Q4 source + * view; the view itself is created only when its layer is encoded. */ static int ds4_gpu_prepare_q4_attn_q_b_transient_f16( const void *model_map, uint64_t model_size, @@ -25925,7 +26069,7 @@ static int ds4_gpu_prepare_q4_attn_q_b_transient_f16( "DS4_METAL_Q4_ATTN_Q_B_TRANSIENT_F16_MIN_TOKENS", 4096u, 32u, UINT32_MAX); if (max_prefill_rows < min_tokens || - g_ssd_streaming_mode || g_quality_mode || + g_quality_mode || g_batch_encoder_concurrent || !ds4_gpu_device_is_pre_m5_apple_silicon() || ds4_gpu_mpp_available() || @@ -25934,6 +26078,7 @@ static int ds4_gpu_prepare_q4_attn_q_b_transient_f16( } uint64_t f16_bytes = 0u; + uint64_t max_source_view_bytes = 0u; for (uint32_t i = 0; i < count; i++) { const ds4_gpu_q4_attn_q_b_f16_sidecar_desc *desc = &descs[i]; if (desc->weight_type != DS4_METAL_TENSOR_Q4_K || @@ -25953,6 +26098,14 @@ static int ds4_gpu_prepare_q4_attn_q_b_transient_f16( return 0; } f16_bytes = desc_f16_bytes; + if (g_ssd_streaming_mode) { + const uint64_t source_view_bytes = + ds4_gpu_q4_attn_q_b_f16_exact_source_bytes( + model_size, desc->weight_offset, desc->weight_bytes); + if (source_view_bytes > max_source_view_bytes) { + max_source_view_bytes = source_view_bytes; + } + } } if (!model_map || f16_bytes == 0u || f16_bytes > NSUIntegerMax) { return 0; @@ -25983,19 +26136,30 @@ static int ds4_gpu_prepare_q4_attn_q_b_transient_f16( } const int stream = g_ds4_stream; + const uint64_t effective_working_set_reserve_bytes = + ds4_gpu_q4_attn_q_b_f16_effective_working_set_reserve( + working_set_reserve_bytes, NULL); bool allocated = false; pthread_mutex_lock(&g_q4_qb_transient_f16_scratch_mu); const bool needs_allocation = !g_q4_qb_transient_f16_scratch[stream] || g_q4_qb_transient_f16_scratch_capacity[stream] < (NSUInteger)f16_bytes; - uint64_t admission_bytes = f16_bytes; - if (UINT64_MAX - admission_bytes < working_set_reserve_bytes) { + uint64_t admission_bytes = needs_allocation ? f16_bytes : 0u; + const uint64_t source_view_bytes = + g_ssd_streaming_mode ? max_source_view_bytes : 0u; + if (UINT64_MAX - admission_bytes < source_view_bytes) { + admission_bytes = UINT64_MAX; + } else { + admission_bytes += source_view_bytes; + } + if (UINT64_MAX - admission_bytes < + effective_working_set_reserve_bytes) { admission_bytes = UINT64_MAX; } else { - admission_bytes += working_set_reserve_bytes; + admission_bytes += effective_working_set_reserve_bytes; } - if ((needs_allocation && + if (((needs_allocation || g_ssd_streaming_mode) && !ds4_gpu_q4_attn_q_b_f16_working_set_has_room( admission_bytes)) || !ds4_gpu_q4_qb_transient_f16_scratch_ensure_locked( @@ -26034,6 +26198,16 @@ int ds4_gpu_prepare_q4_attn_q_b_f16_sidecars( } if (!g_initialized && !ds4_gpu_init()) return 0; + const uint64_t transient_effective_reserve_bytes = + ds4_gpu_q4_attn_q_b_f16_effective_working_set_reserve( + working_set_reserve_bytes, NULL); + bool ssd_transient_same_contract = false; + const bool ssd_transient_invalidated = g_ssd_streaming_mode && + ds4_gpu_q4_qb_transient_f16_admission_begin( + g_ds4_stream, model_map, model_size, + transient_effective_reserve_bytes, + &ssd_transient_same_contract); + const int required = ds4_gpu_env_bool("DS4_METAL_REQUIRE_Q4_ATTN_Q_B_F16_CACHE") == 1; const int transient_disabled = @@ -26048,9 +26222,26 @@ int ds4_gpu_prepare_q4_attn_q_b_f16_sidecars( * operator explicitly selects the legacy path, makes it mandatory, or * opts into the established SSD-streaming hybrid. */ if (!required && !transient_disabled && !ssd_sidecar_opt_in) { - return ds4_gpu_prepare_q4_attn_q_b_transient_f16( + const int rc = ds4_gpu_prepare_q4_attn_q_b_transient_f16( model_map, model_size, descs, count, max_prefill_rows, working_set_reserve_bytes, prepared_bytes); + if (ssd_transient_invalidated && + (rc != 1 || !ssd_transient_same_contract)) { + /* The stream slot is shared by sessions. If an admitted slot is + * rejected or repurposed, invalidate every session that cached + * the old generation so it cannot skip its next preflight. */ + ds4_gpu_q4_attn_q_b_f16_advance_generation(); + } + if (rc == 1 && g_ssd_streaming_mode) { + ds4_gpu_q4_qb_transient_f16_admission_commit( + g_ds4_stream, model_map, model_size, + ds4_gpu_q4_attn_q_b_f16_cache_generation(), + transient_effective_reserve_bytes); + } + return rc; + } + if (ssd_transient_invalidated) { + ds4_gpu_q4_attn_q_b_f16_advance_generation(); } const int disabled = ds4_gpu_env_bool("DS4_METAL_DISABLE_Q4_ATTN_Q_B_F16_CACHE") == 1; @@ -26730,11 +26921,12 @@ static int ds4_gpu_attn_q_b_f16_head_rms_rope_tail_tensor_impl( return 1; } -/* Resident pre-M5 production path: rebuild one Q4_K q_b matrix into a +/* Pre-M5 production path: rebuild one Q4_K q_b matrix into a * stream-local 64 MiB F16 buffer, consume it immediately with the compact - * F16 RHS, and reuse that allocation for the next layer. This removes the - * native kernel's per-output-tile dequantization without retaining a 64 MiB - * sidecar for every layer. */ + * F16 RHS, and reuse that allocation for the next layer. Under SSD streaming + * the current Q4 source is a short-lived exact no-copy view retained by its + * command buffer. This removes the native kernel's per-output-tile + * dequantization without retaining a 64 MiB sidecar for every layer. */ static int ds4_gpu_attn_q_b_transient_f16_head_rms_rope_tail_tensor( ds4_gpu_tensor *out, ds4_gpu_tensor *q_half, @@ -26766,7 +26958,7 @@ static int ds4_gpu_attn_q_b_transient_f16_head_rms_rope_tail_tensor( "DS4_METAL_Q4_ATTN_Q_B_TRANSIENT_F16_MIN_TOKENS", 4096u, 32u, UINT32_MAX); if ((uint64_t)n_tok < min_tokens || - g_ssd_streaming_mode || g_quality_mode || + g_quality_mode || g_batch_encoder_concurrent || !ds4_gpu_device_is_pre_m5_apple_silicon() || ds4_gpu_mpp_available() || @@ -26811,6 +27003,12 @@ static int ds4_gpu_attn_q_b_transient_f16_head_rms_rope_tail_tensor( g_ds4_stream < 0 || g_ds4_stream >= DS4_GPU_MAX_STREAMS) { return 0; } + if (g_ssd_streaming_mode && + !ds4_gpu_q4_qb_transient_f16_admission_allows( + g_ds4_stream, model_map, model_size, + ds4_gpu_q4_attn_q_b_f16_cache_generation())) { + return 0; + } const bool mm_bc_out = (n_tok & 31u) != 0u; id dequant_pipeline = @@ -26831,12 +27029,6 @@ static int ds4_gpu_attn_q_b_transient_f16_head_rms_rope_tail_tensor( return 0; } - uint64_t weight_inner = 0u; - id weight_buffer = ds4_gpu_wrap_model_range( - model_map, model_size, weight_offset, weight_bytes, - &weight_inner); - if (!weight_buffer || weight_inner > NSUIntegerMax) return 0; - const int stream = g_ds4_stream; id scratch = nil; pthread_mutex_lock(&g_q4_qb_transient_f16_scratch_mu); @@ -26844,8 +27036,17 @@ static int ds4_gpu_attn_q_b_transient_f16_head_rms_rope_tail_tensor( !g_q4_qb_transient_f16_scratch[stream] || g_q4_qb_transient_f16_scratch_capacity[stream] < (NSUInteger)f16_bytes; - if ((needs_allocation && - !ds4_gpu_q4_attn_q_b_f16_working_set_has_room(f16_bytes)) || + uint64_t admission_bytes = needs_allocation ? f16_bytes : 0u; + if (g_ssd_streaming_mode) { + const uint64_t source_view_bytes = + ds4_gpu_q4_attn_q_b_f16_exact_source_bytes( + model_size, weight_offset, weight_bytes); + admission_bytes = UINT64_MAX - admission_bytes < source_view_bytes + ? UINT64_MAX + : admission_bytes + source_view_bytes; + } + if (((needs_allocation || g_ssd_streaming_mode) && + !ds4_gpu_q4_attn_q_b_f16_working_set_has_room(admission_bytes)) || !ds4_gpu_q4_qb_transient_f16_scratch_ensure_locked( stream, f16_bytes)) { pthread_mutex_unlock(&g_q4_qb_transient_f16_scratch_mu); @@ -26854,9 +27055,42 @@ static int ds4_gpu_attn_q_b_transient_f16_head_rms_rope_tail_tensor( scratch = g_q4_qb_transient_f16_scratch[stream]; pthread_mutex_unlock(&g_q4_qb_transient_f16_scratch_mu); + uint64_t weight_inner = 0u; + id weight_buffer = + ds4_gpu_q4_attn_q_b_f16_wrap_source( + model_map, model_size, weight_offset, weight_bytes, + &weight_inner); + if (!weight_buffer || weight_inner > NSUIntegerMax) return 0; + int owned = 0; id cb = ds4_gpu_command_buffer(&owned); if (!cb) return 0; + if (g_ssd_streaming_mode) { + /* Retained command buffers normally own every bound resource. Keep + * the exact mmap view alive explicitly as well so the diagnostic + * unretained-command-buffer mode and asynchronous progress flushes + * obey the same lifetime contract. The completion block releases the + * ~18 MiB view immediately instead of accumulating one per layer. */ + id retained_weight_buffer = weight_buffer; + dispatch_group_enter(g_progress_completion_group); + __atomic_add_fetch( + &g_q4_qb_transient_f16_exact_views_created, + 1u, + __ATOMIC_RELEASE); + __atomic_add_fetch( + &g_q4_qb_transient_f16_exact_views_live, + 1u, + __ATOMIC_RELEASE); + [cb addCompletedHandler:^(id completed) { + (void)completed; + (void)[retained_weight_buffer length]; + __atomic_sub_fetch( + &g_q4_qb_transient_f16_exact_views_live, + 1u, + __ATOMIC_RELEASE); + dispatch_group_leave(g_progress_completion_group); + }]; + } /* Copy may share the caller's current serial batch encoder. Close it * before dequantization, then close the dequant encoder as well: each @@ -26950,8 +27184,8 @@ int ds4_gpu_attn_q_b_f16_head_rms_rope_tail_tensor( /* Default mode never falls through to the multi-GiB sidecar cache. * A non-candidate or any failure before the output writer returns the * native-Q4 fallback sentinel directly. The explicit SSD hybrid - * remains on its persistent sidecar path because transient scratch is - * deliberately resident-only. */ + * remains on its persistent sidecar path; ordinary SSD streaming uses + * one exact Q4 view and the same stream-local transient scratch. */ return ds4_gpu_attn_q_b_transient_f16_head_rms_rope_tail_tensor( out, q_half, model_map, model_size, weight_offset, weight_type, in_dim, out_dim, x, n_tok, n_head, head_dim, n_rot, pos0, diff --git a/scripts/environment_variables.tsv b/scripts/environment_variables.tsv index 2fc3146e4..264d06cb5 100644 --- a/scripts/environment_variables.tsv +++ b/scripts/environment_variables.tsv @@ -602,7 +602,7 @@ runtime/metal DS4_METAL_DISABLE_Q4_ATTN_OUT_HC_FUSE presence rollback; unset: au runtime/metal DS4_METAL_DISABLE_Q4_ATTN_OUT_TINY_BATCH value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Disables Q4 attn out tiny batch. ds4_metal.m:28396 runtime/metal DS4_METAL_DISABLE_Q4_ATTN_Q_B_F16_CACHE value-aware boolean; unset/0: persistent sidecar remains eligible; empty/1/true/yes/on disables; REQUIRE plus DISABLE fails closed Disables only the persistent pre-M5 Q4_K attn_q_b F16 weight sidecar; unless REQUIRE is set, the default transient F16 path remains independently eligible, so also set DS4_METAL_DISABLE_Q4_ATTN_Q_B_TRANSIENT_F16=1 to force native per-tile Q4 dequantization. ds4_metal.m:25819; ds4_metal.m:26131 runtime/metal DS4_METAL_DISABLE_Q4_ATTN_Q_B_F16_RHS value-aware boolean; unset/0/false/no/off keeps the resident path enabled; empty/1/true/yes/on disables Disables one-time F32-to-F16 RHS materialization for the resident pre-M5 Q4_K attn_q_b F16 sidecar and restores repeated per-tile F32 staging. ds4_metal.m:25821; ds4_metal.m:26136 -runtime/metal DS4_METAL_DISABLE_Q4_ATTN_Q_B_TRANSIENT_F16 value-aware rollback; unset/0/false/no/off keeps the automatic path enabled; empty/1/true/yes/on disables Disables per-layer transient Q4_K attn_q_b-to-F16 scratch for long resident, non-SSD prefill; an enabled/required and eligible persistent sidecar may still run, otherwise dispatch returns to native Q4. ds4_metal.m:25804; ds4_metal.m:26707 +runtime/metal DS4_METAL_DISABLE_Q4_ATTN_Q_B_TRANSIENT_F16 value-aware rollback; unset/0/false/no/off keeps the automatic path enabled; empty/1/true/yes/on disables Disables per-layer transient Q4_K attn_q_b-to-F16 scratch for long prefill, including the SSD-safe exact-view path; an enabled/required and eligible persistent sidecar may still run, otherwise dispatch returns to native Q4. ds4_metal.m:26212; ds4_metal.m:27162 runtime/metal DS4_METAL_DISABLE_Q4_BATCH_EXPERT_TABLE presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 batch expert table. ds4_metal.m:44883 runtime/metal DS4_METAL_DISABLE_Q4_DENSE_PAIR presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 dense pair. ds4_metal.m:21990 runtime/metal DS4_METAL_DISABLE_Q4_EXACT_BOUNDARY presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 exact boundary. ds4_metal.m:42239 @@ -829,7 +829,7 @@ runtime/metal DS4_METAL_Q4_ADDR_USE_RESOURCES presence control; unset: off/defau runtime/metal DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MB integer 1..65536 MiB; default 3072; above max clamps, below min/invalid restores default Caps copied F16 sidecar storage for resident Q4_K attn_q_b weights. ds4_metal.m:25861; ds4_metal.m:26210 runtime/metal DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MIN_TOKENS integer 32..UINT32_MAX; default 512; below min/invalid restores default Sets the minimum resident, or explicitly enabled SSD-hybrid, prefill batch that may build or use the Q4_K attn_q_b F16 sidecar; smaller tails remain non-candidates. ds4_metal.m:25823; ds4_metal.m:26138 runtime/metal DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints resident Q4_K attn_q_b F16 cache counters and circuit state at Metal cleanup. ds4_metal.m:11494 -runtime/metal DS4_METAL_Q4_ATTN_Q_B_TRANSIENT_F16_MIN_TOKENS integer token count; default 4096; invalid or values below 32 restore 4096 Sets the minimum long-N resident, non-SSD prefill batch eligible for per-layer transient Q4_K attn_q_b-to-F16 scratch; smaller chunks and tails stay on native Q4. ds4_metal.m:25688; ds4_metal.m:26529 +runtime/metal DS4_METAL_Q4_ATTN_Q_B_TRANSIENT_F16_MIN_TOKENS integer token count; default 4096; invalid or values below 32 restore 4096 Sets the minimum long-N resident or SSD-streamed prefill batch eligible for per-layer transient Q4_K attn_q_b-to-F16 scratch; smaller chunks and tails stay on native Q4. ds4_metal.m:26065; ds4_metal.m:26944 runtime/metal DS4_METAL_Q4_EXPERT_GROUP_SIZE positive uint32; default 32; clamped to total expert count Sets experts processed per grouped Q4 dispatch. ds4_metal.m:34088 runtime/metal DS4_METAL_Q4_EXPERT_TABLE_GROUP_SIZE integer 2..total experts; default/invalid 1 (ungrouped) Sets grouped exact-view width while building Q4 expert tables. ds4_metal.m:19604 runtime/metal DS4_METAL_Q4_EXPERT_TABLE_PROFILE presence diagnostic; unset: off; any value including 0 enables Prints timing/profile diagnostics for Q4 expert table. ds4_metal.m:20103 diff --git a/tests/test_metal_q4_qb_f16_cache.c b/tests/test_metal_q4_qb_f16_cache.c index a036f17e2..5f7e4f582 100644 --- a/tests/test_metal_q4_qb_f16_cache.c +++ b/tests/test_metal_q4_qb_f16_cache.c @@ -1265,9 +1265,10 @@ int main(void) { "DISABLE accounting"); CHECK(unsetenv(k_disable) == 0, "clear cache disable env"); - /* The SSD-streaming extension is opt-in: its default must continue to - * drop resident-only sidecars and reject the optimization without - * rebuilding them. */ + /* The persistent-sidecar SSD extension is opt-in: its default must + * continue to drop resident-only sidecars and, under REQUIRE, reject the + * persistent cache path without rebuilding it. The independent transient + * path is exercised below with REQUIRE cleared. */ gate_before = gate_after; CHECK(gate_before.entries != 0u, "default SSD transition lacks a resident sidecar to release"); @@ -1533,10 +1534,9 @@ int main(void) { /* Exercise the public production selector, not only its benchmark hook. * Lower the threshold for this bounded oracle so aligned and boundary - * geometries fit in the guarded allocations. Default mode must use one - * transient scratch without publishing persistent sidecars, while SSD - * streaming must return the clean native-Q4 fallback sentinel before - * touching output. */ + * geometries fit in the guarded allocations. Resident and SSD-streamed + * modes must use one transient scratch without publishing persistent + * sidecars. */ ds4_gpu_test_q4_attn_q_b_f16_cache_reset(); CHECK(unsetenv(k_require) == 0, "clear REQUIRE for transient production oracle"); @@ -1773,54 +1773,228 @@ int main(void) { transient_report.builds == 0u, "transient production published a persistent sidecar"); + /* Install a second model identity whose registered view covers only a + * disjoint prefix. Its byte-identical Q4 matrix starts at a deliberately + * non-page-aligned, uncovered offset, so this succeeds only when the SSD + * path creates an exact owned source view. Running inside a real command + * batch also exercises the completion-handler lifetime under + * DS4_METAL_UNRETAINED_COMMAND_BUFFERS. */ + const uint64_t ssd_transient_weight_offset = page + SSD_SOURCE_LEADING; + const uint64_t ssd_transient_model_bytes = align_up( + ssd_transient_weight_offset + weight_bytes, page); + void *ssd_transient_model = NULL; + CHECK(posix_memalign(&ssd_transient_model, (size_t)page, + (size_t)ssd_transient_model_bytes) == 0, + "SSD transient exact-view model allocation"); + memset(ssd_transient_model, 0, (size_t)ssd_transient_model_bytes); + fill_q4_matrix((block_q4_K *)((uint8_t *)ssd_transient_model + + ssd_transient_weight_offset)); + + const uint32_t ssd_transient_tokens = 64u; + const uint64_t ssd_transient_output_count = + (uint64_t)ssd_transient_tokens * OUT_DIM; + const uint64_t ssd_transient_output_bytes = + ssd_transient_output_count * sizeof(float); + const uint64_t ssd_transient_half_count = + (uint64_t)ssd_transient_tokens * IN_DIM; + const uint64_t ssd_transient_half_bytes = + ssd_transient_half_count * sizeof(uint16_t); + poison_f32(reference_host, output_storage_count, k_reference_poison); poison_f32(candidate_host, output_storage_count, k_candidate_poison); poison_f16(q_half_host, q_half_storage_count); + CHECK(ds4_gpu_tensor_write( + reference_base, 0, reference_host, + output_storage_count * sizeof(float)) != 0, + "transient SSD reference poison upload"); CHECK(ds4_gpu_tensor_write( candidate_base, 0, candidate_host, output_storage_count * sizeof(float)) != 0, - "transient SSD fallback poison upload"); + "transient SSD candidate poison upload"); CHECK(ds4_gpu_tensor_write( q_half_base, 0, q_half_host, q_half_storage_count * sizeof(uint16_t)) != 0, - "transient SSD fallback q_half poison upload"); + "transient SSD q_half poison upload"); + ds4_gpu_tensor *transient_ssd_reference = ds4_gpu_tensor_view( + reference_base, GUARD_FLOATS * sizeof(float), + ssd_transient_output_bytes); ds4_gpu_tensor *transient_ssd_out = ds4_gpu_tensor_view( candidate_base, GUARD_FLOATS * sizeof(float), - (uint64_t)64u * OUT_DIM * sizeof(float)); + ssd_transient_output_bytes); ds4_gpu_tensor *transient_ssd_half = ds4_gpu_tensor_view( q_half_base, GUARD_HALFS * sizeof(uint16_t), - (uint64_t)64u * IN_DIM * sizeof(uint16_t)); - CHECK(transient_ssd_out && transient_ssd_half, - "transient SSD fallback tensor views"); + ssd_transient_half_bytes); + CHECK(transient_ssd_reference && transient_ssd_out && transient_ssd_half, + "transient SSD exact-view tensor views"); + CHECK(run_reference( + transient_ssd_reference, model, model_bytes, x, + ssd_transient_tokens) == 1, + "transient SSD native Q4 reference"); + ds4_gpu_tensor_free(transient_ssd_reference); + CHECK(ds4_gpu_tensor_read( + reference_base, 0, reference_host, + output_storage_count * sizeof(float)) != 0, + "transient SSD reference readback"); + + /* Stream 0 already owns the resident oracle's 64 MiB scratch. Move the + * SSD transient case to a fresh stream so its successful preflight also + * covers cold scratch + exact-source admission and allocation. */ + ds4_gpu_set_stream(1); ds4_gpu_set_ssd_streaming(true); - CHECK(run_candidate( + CHECK(ds4_gpu_set_model_map_range( + ssd_transient_model, ssd_transient_model_bytes, + 0u, page, page) != 0, + "install disjoint SSD transient model prefix"); + const ds4_gpu_q4_attn_q_b_f16_sidecar_desc ssd_transient_desc = { + .weight_offset = ssd_transient_weight_offset, + .weight_bytes = weight_bytes, + .in_dim = IN_DIM, + .out_dim = OUT_DIM, + .weight_type = Q4_K_TYPE, + .layer = 0u, + }; + ds4_gpu_q4_attn_q_b_f16_cache_report ssd_lifetime_before; + ds4_gpu_q4_attn_q_b_f16_cache_report ssd_lifetime_mid; + ds4_gpu_q4_attn_q_b_f16_cache_report ssd_lifetime_after; + ds4_gpu_stream_test_stats ssd_transients_before; + ds4_gpu_stream_test_stats ssd_transients_mid; + ds4_gpu_stream_test_stats ssd_transients_after; + ds4_gpu_test_q4_attn_q_b_f16_cache_report(&ssd_lifetime_before); + + /* A rejected session reserve must latch the transient selector off. The + * public call then returns the native-Q4 fallback sentinel without + * touching either output, even though its smaller runtime-only gate would + * otherwise fit. A later successful preflight re-arms the same slot. */ + uint64_t ssd_transient_prepared_bytes = UINT64_MAX; + CHECK(ds4_gpu_prepare_q4_attn_q_b_f16_sidecars( + ssd_transient_model, ssd_transient_model_bytes, + &ssd_transient_desc, 1u, 4096u, UINT64_MAX, + &ssd_transient_prepared_bytes) == 0, + "transient SSD oversized-reserve preflight rejection"); + CHECK(ssd_transient_prepared_bytes == 0u, + "transient SSD rejected preflight allocated scratch"); + CHECK(ds4_gpu_begin_commands() != 0, + "begin transient SSD rejected command batch"); + CHECK(run_candidate_at( transient_ssd_out, transient_ssd_half, - model, model_bytes, x, 64u) == 0, - "transient path must fall back under SSD streaming"); + ssd_transient_model, ssd_transient_model_bytes, + ssd_transient_weight_offset, x, + ssd_transient_tokens) == 0, + "transient SSD rejected preflight escaped admission latch"); + CHECK(ds4_gpu_end_commands() != 0, + "finish transient SSD rejected command batch"); + CHECK(ds4_gpu_tensor_read( + candidate_base, 0, candidate_host, + output_storage_count * sizeof(float)) != 0, + "transient SSD rejected output readback"); + CHECK(ds4_gpu_tensor_read( + q_half_base, 0, q_half_host, + q_half_storage_count * sizeof(uint16_t)) != 0, + "transient SSD rejected q_half readback"); + CHECK(count_poison_f32_mismatches( + candidate_host, 0u, output_storage_count, + k_candidate_poison) == 0u, + "transient SSD rejected preflight touched output"); + CHECK(count_poison_f16_mismatches( + q_half_host, q_half_storage_count) == 0u, + "transient SSD rejected preflight touched q_half"); + + ssd_transient_prepared_bytes = UINT64_MAX; + CHECK(ds4_gpu_prepare_q4_attn_q_b_f16_sidecars( + ssd_transient_model, ssd_transient_model_bytes, + &ssd_transient_desc, 1u, 4096u, 0u, + &ssd_transient_prepared_bytes) == 1, + "transient SSD successful preflight re-arm"); + CHECK(ssd_transient_prepared_bytes == f16_cache_bytes, + "transient SSD cold preflight did not allocate one scratch"); + ds4_gpu_test_stream_stats(&ssd_transients_before); + CHECK(ds4_gpu_begin_commands() != 0, + "begin transient SSD exact-view command batch"); + CHECK(run_candidate_at( + transient_ssd_out, transient_ssd_half, + ssd_transient_model, ssd_transient_model_bytes, + ssd_transient_weight_offset, x, + ssd_transient_tokens) == 1, + "encode transient SSD exact-view candidate"); + ds4_gpu_test_q4_attn_q_b_f16_cache_report(&ssd_lifetime_mid); + ds4_gpu_test_stream_stats(&ssd_transients_mid); + CHECK(ssd_lifetime_mid.transient_exact_views_created == + ssd_lifetime_before.transient_exact_views_created + 1u && + ssd_lifetime_mid.transient_exact_views_live == + ssd_lifetime_before.transient_exact_views_live + 1u && + ssd_lifetime_mid.model_exact_cache_entries == + ssd_lifetime_before.model_exact_cache_entries && + ssd_lifetime_mid.model_exact_cache_bytes == + ssd_lifetime_before.model_exact_cache_bytes && + ssd_transients_mid.transient_references == + ssd_transients_before.transient_references, + "transient SSD exact source was not command-buffer-owned"); + CHECK(ds4_gpu_end_commands() != 0, + "finish transient SSD exact-view command batch"); + ds4_gpu_test_q4_attn_q_b_f16_cache_report(&ssd_lifetime_after); + ds4_gpu_test_stream_stats(&ssd_transients_after); + CHECK(ssd_lifetime_after.transient_exact_views_created == + ssd_lifetime_before.transient_exact_views_created + 1u && + ssd_lifetime_after.transient_exact_views_live == + ssd_lifetime_before.transient_exact_views_live && + ssd_lifetime_after.model_exact_cache_entries == + ssd_lifetime_before.model_exact_cache_entries && + ssd_lifetime_after.model_exact_cache_bytes == + ssd_lifetime_before.model_exact_cache_bytes && + ssd_transients_after.transient_references == + ssd_transients_before.transient_references, + "transient SSD exact source leaked past command completion"); ds4_gpu_tensor_free(transient_ssd_half); ds4_gpu_tensor_free(transient_ssd_out); CHECK(ds4_gpu_tensor_read( candidate_base, 0, candidate_host, output_storage_count * sizeof(float)) != 0, - "transient SSD fallback output readback"); + "transient SSD candidate readback"); + uint64_t transient_ssd_first = UINT64_MAX; + CHECK(count_bit_mismatches( + reference_host + GUARD_FLOATS, + candidate_host + GUARD_FLOATS, + ssd_transient_output_count, + &transient_ssd_first) == 0u, + "transient SSD exact-view bitwise mismatch"); CHECK(count_poison_f32_mismatches( - candidate_host, 0u, output_storage_count, + candidate_host, 0u, GUARD_FLOATS, + k_candidate_poison) == 0u && + count_poison_f32_mismatches( + candidate_host, + GUARD_FLOATS + ssd_transient_output_count, + output_storage_count, k_candidate_poison) == 0u, - "transient SSD fallback touched output"); + "transient SSD exact-view touched output guards"); CHECK(ds4_gpu_tensor_read( q_half_base, 0, q_half_host, q_half_storage_count * sizeof(uint16_t)) != 0, - "transient SSD fallback q_half readback"); - CHECK(count_poison_f16_mismatches( - q_half_host, q_half_storage_count) == 0u, - "transient SSD fallback touched q_half"); + "transient SSD q_half readback"); + CHECK(count_poison_f16_mismatches_range( + q_half_host, 0u, GUARD_HALFS) == 0u && + count_poison_f16_mismatches_range( + q_half_host, GUARD_HALFS, + GUARD_HALFS + ssd_transient_half_count) == + ssd_transient_half_count && + count_poison_f16_mismatches_range( + q_half_host, + GUARD_HALFS + ssd_transient_half_count, + q_half_storage_count) == 0u, + "transient SSD q_half payload/canary mismatch"); + ds4_gpu_test_q4_attn_q_b_f16_cache_report(&transient_report); + CHECK(transient_report.entries == 0u && + transient_report.bytes == 0u && + transient_report.lookups == 0u && + transient_report.builds == 0u, + "transient SSD exact-view published a persistent sidecar"); ds4_gpu_set_ssd_streaming(false); + ds4_gpu_set_stream(0); CHECK(setenv(k_require, "1", 1) == 0, "restore REQUIRE after transient production oracle"); CHECK(unsetenv(k_transient_min_tokens) == 0, "restore transient production threshold"); fprintf(stderr, "Metal Q4 attn_q_b transient F16 production selector " - "N=32/33/64 and SSD fallback: PASS\n"); + "N=32/33/64 and SSD exact-view: PASS\n"); /* The input, including both guards, is immutable across every path. */ CHECK(ds4_gpu_tensor_read( @@ -2363,6 +2537,7 @@ int main(void) { free(input_readback); free(input_host); free(support_model); + free(ssd_transient_model); free(model); CHECK(unsetenv(k_require) == 0, "clear cache require env at exit"); From 6962d61fd8e2386b40b0c8d01afb3022a03ce7d5 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Fri, 28 Aug 2026 23:27:59 +0200 Subject: [PATCH 142/189] perf(metal): write small prefill attention directly Use a single FlashAttention workgroup for prefill with at most 32 keys, writing normalized output in place of the 32-way temporary and reducer. Preserve overlap and pipeline-failure fallbacks, and add exact topology, lifetime, rollback, alias, and GPU-only timing oracles. --- ds4_gpu.h | 6 + ds4_metal.m | 216 ++++++++---- scripts/environment_variables.tsv | 1 + tests/ds4_test.c | 542 +++++++++++++++++++++++++++++- 4 files changed, 701 insertions(+), 64 deletions(-) diff --git a/ds4_gpu.h b/ds4_gpu.h index 5f0de67f9..20db38b70 100644 --- a/ds4_gpu.h +++ b/ds4_gpu.h @@ -394,8 +394,14 @@ enum { DS4_GPU_TEST_ATTN_OUT_LOW_Q8_STATIC = 1u << 9, DS4_GPU_TEST_BATCH_ATTN_OUT_Q8_HC_FUSION = 1u << 10, DS4_GPU_TEST_BATCH_ATTN_OUT_Q4_HC_FUSION = 1u << 11, + DS4_GPU_TEST_FLASH_ATTN_SMALL_PREFILL_NWG32 = 1u << 12, + DS4_GPU_TEST_FLASH_ATTN_SMALL_PREFILL_NWG1_FAILURE = 1u << 13, }; void ds4_gpu_test_set_flags(uint32_t flags); +double ds4_gpu_test_last_completed_gpu_ms(void); +uint32_t ds4_gpu_test_last_flash_attn_prefill_nwg(void); +int ds4_gpu_test_reset_flash_attn_tmp(void); +uint64_t ds4_gpu_test_flash_attn_tmp_bytes(void); void ds4_gpu_release_zero_prefix_prefill_mask_cache(void); #else static inline int ds4_gpu_device_is_pre_m5_apple_silicon(void) { return 0; } diff --git a/ds4_metal.m b/ds4_metal.m index 884a3bf7a..a695e3698 100644 --- a/ds4_metal.m +++ b/ds4_metal.m @@ -115,6 +115,7 @@ static id g_hc_weighted_sum_capture_last_pipeline; static id g_output_hc_weights4_pipeline; static uint32_t g_test_flags; +static _Thread_local uint32_t g_test_last_flash_attn_prefill_nwg; static id g_hc_expand_pipeline; static id g_unary_sigmoid_pipeline; static id g_unary_silu_pipeline; @@ -4350,6 +4351,12 @@ static uint32_t ds4_gpu_flash_attn_vec_nsg(uint32_t n_keys, uint32_t nwg, uint32 return nsg; } +static bool ds4_gpu_flash_attn_small_prefill_direct(uint32_t n_keys) { + return n_keys <= 32u && + getenv("DS4_METAL_DISABLE_SMALL_PREFILL_DIRECT") == NULL && + (g_test_flags & DS4_GPU_TEST_FLASH_ATTN_SMALL_PREFILL_NWG32) == 0u; +} + static int ds4_gpu_trace_allocs(void) { static int initialized; static int enabled; @@ -10150,6 +10157,27 @@ int ds4_gpu_test_iq2_addr_mid_only_oracle( void ds4_gpu_test_set_flags(uint32_t flags) { g_test_flags = flags; + g_test_last_flash_attn_prefill_nwg = 0u; +} + +double ds4_gpu_test_last_completed_gpu_ms(void) { + return g_last_completed_gpu_time_valid + ? g_last_completed_gpu_seconds * 1000.0 : -1.0; +} + +uint32_t ds4_gpu_test_last_flash_attn_prefill_nwg(void) { + return g_test_last_flash_attn_prefill_nwg; +} + +int ds4_gpu_test_reset_flash_attn_tmp(void) { + if (!ds4_gpu_synchronize()) return 0; + g_flash_attn_tmp_buffer = nil; + g_flash_attn_tmp_bytes = 0u; + return 1; +} + +uint64_t ds4_gpu_test_flash_attn_tmp_bytes(void) { + return (uint64_t)g_flash_attn_tmp_bytes; } ds4_gpu_tensor *ds4_gpu_tensor_alloc(uint64_t bytes) { @@ -34020,8 +34048,11 @@ static int ds4_gpu_encode_flash_attention_prefill_static_mixed_heads_vec( } const uint32_t ncpsg = 32; - const uint32_t nwg = 32; - const uint32_t nsg = ds4_gpu_flash_attn_vec_nsg(n_keys, nwg, ncpsg); + bool direct_output = + ds4_gpu_flash_attn_small_prefill_direct(n_keys) && + !ds4_gpu_tensor_prefixes_overlap(heads, q_bytes, q, q_bytes); + uint32_t nwg = direct_output ? 1u : 32u; + uint32_t nsg = ds4_gpu_flash_attn_vec_nsg(n_keys, nwg, ncpsg); const NSUInteger row_bytes = (NSUInteger)head_dim * sizeof(float); const NSUInteger row_bytes_f16 = (NSUInteger)head_dim * sizeof(uint16_t); const NSUInteger mask_bytes = (NSUInteger)n_keys * (NSUInteger)n_tokens * sizeof(uint16_t); @@ -34031,8 +34062,9 @@ static int ds4_gpu_encode_flash_attention_prefill_static_mixed_heads_vec( ? (NSUInteger)ncpsg * (2u * row_bytes_f16 + (NSUInteger)n_tokens * sizeof(uint16_t)) : 1u; const NSUInteger nrows = (NSUInteger)n_tokens * n_head; - const NSUInteger tmp_bytes = nrows * (NSUInteger)head_dim * (NSUInteger)nwg * sizeof(float) + - nrows * (2u * (NSUInteger)nwg) * sizeof(float); + NSUInteger tmp_bytes = direct_output ? 0u : + nrows * (NSUInteger)head_dim * (NSUInteger)nwg * sizeof(float) + + nrows * (2u * (NSUInteger)nwg) * sizeof(float); id mask_buffer = ds4_gpu_new_transient_buffer(mask_bytes, "ds4_flash_attn_mask"); @@ -34045,10 +34077,11 @@ static int ds4_gpu_encode_flash_attention_prefill_static_mixed_heads_vec( &g_flash_attn_pad_bytes, pad_bytes, "ds4_flash_attn_pad") || - !ds4_gpu_ensure_scratch_buffer(&g_flash_attn_tmp_buffer, - &g_flash_attn_tmp_bytes, - tmp_bytes, - "ds4_flash_attn_tmp")) { + (!direct_output && + !ds4_gpu_ensure_scratch_buffer(&g_flash_attn_tmp_buffer, + &g_flash_attn_tmp_bytes, + tmp_bytes, + "ds4_flash_attn_tmp"))) { return 0; } @@ -34128,16 +34161,39 @@ static int ds4_gpu_encode_flash_attention_prefill_static_mixed_heads_vec( if (!pad_pipeline) return 0; } id vec_pipeline = - ds4_gpu_get_flash_attn_vec_pipeline("kernel_flash_attn_ext_vec_f16_dk512_dv512", - true, true, false, false, has_kvpad, - false, - (int32_t)head_dim, - (int32_t)head_dim, - (int32_t)nsg, - (int32_t)nwg); - id reduce_pipeline = - ds4_gpu_get_flash_attn_reduce_pipeline((int32_t)head_dim, (int32_t)nwg); - if (!vec_pipeline || !reduce_pipeline) return 0; + direct_output && + (g_test_flags & + DS4_GPU_TEST_FLASH_ATTN_SMALL_PREFILL_NWG1_FAILURE) != 0u + ? nil + : ds4_gpu_get_flash_attn_vec_pipeline( + "kernel_flash_attn_ext_vec_f16_dk512_dv512", + true, true, false, false, has_kvpad, false, + (int32_t)head_dim, (int32_t)head_dim, + (int32_t)nsg, (int32_t)nwg); + if (direct_output && !vec_pipeline) { + direct_output = false; + nwg = 32u; + nsg = ds4_gpu_flash_attn_vec_nsg(n_keys, nwg, ncpsg); + tmp_bytes = + nrows * (NSUInteger)head_dim * (NSUInteger)nwg * sizeof(float) + + nrows * (2u * (NSUInteger)nwg) * sizeof(float); + if (!ds4_gpu_ensure_scratch_buffer(&g_flash_attn_tmp_buffer, + &g_flash_attn_tmp_bytes, + tmp_bytes, + "ds4_flash_attn_tmp")) { + return 0; + } + vec_pipeline = ds4_gpu_get_flash_attn_vec_pipeline( + "kernel_flash_attn_ext_vec_f16_dk512_dv512", + true, true, false, false, has_kvpad, false, + (int32_t)head_dim, (int32_t)head_dim, + (int32_t)nsg, (int32_t)nwg); + } + id reduce_pipeline = direct_output ? nil : + ds4_gpu_get_flash_attn_reduce_pipeline( + (int32_t)head_dim, (int32_t)nwg); + if (!vec_pipeline || (!direct_output && !reduce_pipeline)) return 0; + g_test_last_flash_attn_prefill_nwg = nwg; if (has_kvpad) { ds4_gpu_flash_attn_pad_args pad_args = { @@ -34220,25 +34276,29 @@ static int ds4_gpu_encode_flash_attention_prefill_static_mixed_heads_vec( [enc setBuffer:mask_buffer offset:0 atIndex:4]; [enc setBuffer:sinks_buf offset:sinks_offset atIndex:5]; [enc setBuffer:g_flash_attn_pad_buffer offset:0 atIndex:6]; - [enc setBuffer:g_flash_attn_tmp_buffer offset:0 atIndex:7]; + [enc setBuffer:direct_output ? headsbuf : g_flash_attn_tmp_buffer + offset:direct_output ? ds4_gpu_tensor_offset(heads) : 0u + atIndex:7]; [enc setThreadgroupMemoryLength:shared_bytes atIndex:0]; [enc dispatchThreadgroups:MTLSizeMake(n_tokens, n_head, nwg) threadsPerThreadgroup:MTLSizeMake(32, nsg, 1)]; ds4_gpu_end_compute_encoder(cb, enc); DS4_METAL_PROFILE_FLASH_ATTN_STAGE("attention_vec"); - ds4_gpu_flash_attn_reduce_args reduce_args = { - .nrows = (int32_t)nrows, - }; - enc = ds4_gpu_compute_encoder(cb); - [enc setComputePipelineState:reduce_pipeline]; - [enc setBytes:&reduce_args length:sizeof(reduce_args) atIndex:0]; - [enc setBuffer:g_flash_attn_tmp_buffer offset:0 atIndex:1]; - [enc setBuffer:headsbuf offset:ds4_gpu_tensor_offset(heads) atIndex:2]; - [enc dispatchThreadgroups:MTLSizeMake(nrows, 1, 1) - threadsPerThreadgroup:MTLSizeMake(32u * nwg, 1, 1)]; - ds4_gpu_end_compute_encoder(cb, enc); - DS4_METAL_PROFILE_FLASH_ATTN_STAGE("attention_reduce"); + if (!direct_output) { + ds4_gpu_flash_attn_reduce_args reduce_args = { + .nrows = (int32_t)nrows, + }; + enc = ds4_gpu_compute_encoder(cb); + [enc setComputePipelineState:reduce_pipeline]; + [enc setBytes:&reduce_args length:sizeof(reduce_args) atIndex:0]; + [enc setBuffer:g_flash_attn_tmp_buffer offset:0 atIndex:1]; + [enc setBuffer:headsbuf offset:ds4_gpu_tensor_offset(heads) atIndex:2]; + [enc dispatchThreadgroups:MTLSizeMake(nrows, 1, 1) + threadsPerThreadgroup:MTLSizeMake(32u * nwg, 1, 1)]; + ds4_gpu_end_compute_encoder(cb, enc); + DS4_METAL_PROFILE_FLASH_ATTN_STAGE("attention_reduce"); + } #undef DS4_METAL_PROFILE_FLASH_ATTN_STAGE return 1; @@ -34634,8 +34694,11 @@ static int ds4_gpu_encode_flash_attention_prefill_raw_heads( } const uint32_t ncpsg = 32; - const uint32_t nwg = 32; - const uint32_t nsg = ds4_gpu_flash_attn_vec_nsg(n_tokens, nwg, ncpsg); + bool direct_output = + ds4_gpu_flash_attn_small_prefill_direct(n_tokens) && + !ds4_gpu_tensor_prefixes_overlap(heads, q_bytes, q, q_bytes); + uint32_t nwg = direct_output ? 1u : 32u; + uint32_t nsg = ds4_gpu_flash_attn_vec_nsg(n_tokens, nwg, ncpsg); const NSUInteger row_bytes = (NSUInteger)head_dim * sizeof(float); const NSUInteger row_bytes_f16 = (NSUInteger)head_dim * sizeof(uint16_t); const NSUInteger mask_bytes = (NSUInteger)n_tokens * (NSUInteger)n_tokens * sizeof(uint16_t); @@ -34644,8 +34707,9 @@ static int ds4_gpu_encode_flash_attention_prefill_raw_heads( const NSUInteger pad_bytes = 2u * (NSUInteger)ncpsg * row_bytes_f16 + (NSUInteger)ncpsg * (NSUInteger)n_tokens * sizeof(uint16_t); const NSUInteger nrows = (NSUInteger)n_tokens * n_head; - const NSUInteger tmp_bytes = nrows * (NSUInteger)head_dim * (NSUInteger)nwg * sizeof(float) + - nrows * (2u * (NSUInteger)nwg) * sizeof(float); + NSUInteger tmp_bytes = direct_output ? 0u : + nrows * (NSUInteger)head_dim * (NSUInteger)nwg * sizeof(float) + + nrows * (2u * (NSUInteger)nwg) * sizeof(float); id mask_buffer = ds4_gpu_new_transient_buffer(mask_bytes, "ds4_flash_attn_mask"); @@ -34658,10 +34722,11 @@ static int ds4_gpu_encode_flash_attention_prefill_raw_heads( &g_flash_attn_kv_bytes, kv_f16_bytes, "ds4_flash_attn_kv_f16") || - !ds4_gpu_ensure_scratch_buffer(&g_flash_attn_tmp_buffer, - &g_flash_attn_tmp_bytes, - tmp_bytes, - "ds4_flash_attn_tmp")) { + (!direct_output && + !ds4_gpu_ensure_scratch_buffer(&g_flash_attn_tmp_buffer, + &g_flash_attn_tmp_bytes, + tmp_bytes, + "ds4_flash_attn_tmp"))) { return 0; } @@ -34698,16 +34763,39 @@ static int ds4_gpu_encode_flash_attention_prefill_raw_heads( if (!pad_pipeline) return 0; } id vec_pipeline = - ds4_gpu_get_flash_attn_vec_pipeline("kernel_flash_attn_ext_vec_f16_dk512_dv512", - true, true, false, false, true, - false, - (int32_t)head_dim, - (int32_t)head_dim, - (int32_t)nsg, - (int32_t)nwg); - id reduce_pipeline = - ds4_gpu_get_flash_attn_reduce_pipeline((int32_t)head_dim, (int32_t)nwg); - if (!vec_pipeline || !reduce_pipeline) return 0; + direct_output && + (g_test_flags & + DS4_GPU_TEST_FLASH_ATTN_SMALL_PREFILL_NWG1_FAILURE) != 0u + ? nil + : ds4_gpu_get_flash_attn_vec_pipeline( + "kernel_flash_attn_ext_vec_f16_dk512_dv512", + true, true, false, false, true, false, + (int32_t)head_dim, (int32_t)head_dim, + (int32_t)nsg, (int32_t)nwg); + if (direct_output && !vec_pipeline) { + direct_output = false; + nwg = 32u; + nsg = ds4_gpu_flash_attn_vec_nsg(n_tokens, nwg, ncpsg); + tmp_bytes = + nrows * (NSUInteger)head_dim * (NSUInteger)nwg * sizeof(float) + + nrows * (2u * (NSUInteger)nwg) * sizeof(float); + if (!ds4_gpu_ensure_scratch_buffer(&g_flash_attn_tmp_buffer, + &g_flash_attn_tmp_bytes, + tmp_bytes, + "ds4_flash_attn_tmp")) { + return 0; + } + vec_pipeline = ds4_gpu_get_flash_attn_vec_pipeline( + "kernel_flash_attn_ext_vec_f16_dk512_dv512", + true, true, false, false, true, false, + (int32_t)head_dim, (int32_t)head_dim, + (int32_t)nsg, (int32_t)nwg); + } + id reduce_pipeline = direct_output ? nil : + ds4_gpu_get_flash_attn_reduce_pipeline( + (int32_t)head_dim, (int32_t)nwg); + if (!vec_pipeline || (!direct_output && !reduce_pipeline)) return 0; + g_test_last_flash_attn_prefill_nwg = nwg; if (!ds4_gpu_encode_cpy_f32_f16_1d(cb, rawbuf, @@ -34800,25 +34888,29 @@ static int ds4_gpu_encode_flash_attention_prefill_raw_heads( [enc setBuffer:mask_buffer offset:0 atIndex:4]; [enc setBuffer:sinks_buf offset:sinks_offset atIndex:5]; [enc setBuffer:g_flash_attn_pad_buffer offset:0 atIndex:6]; - [enc setBuffer:g_flash_attn_tmp_buffer offset:0 atIndex:7]; + [enc setBuffer:direct_output ? headsbuf : g_flash_attn_tmp_buffer + offset:direct_output ? ds4_gpu_tensor_offset(heads) : 0u + atIndex:7]; [enc setThreadgroupMemoryLength:shared_bytes atIndex:0]; [enc dispatchThreadgroups:MTLSizeMake(n_tokens, n_head, nwg) threadsPerThreadgroup:MTLSizeMake(32, nsg, 1)]; ds4_gpu_end_compute_encoder(cb, enc); DS4_METAL_PROFILE_FLASH_ATTN_STAGE("attention_vec"); - ds4_gpu_flash_attn_reduce_args reduce_args = { - .nrows = (int32_t)nrows, - }; - enc = ds4_gpu_compute_encoder(cb); - [enc setComputePipelineState:reduce_pipeline]; - [enc setBytes:&reduce_args length:sizeof(reduce_args) atIndex:0]; - [enc setBuffer:g_flash_attn_tmp_buffer offset:0 atIndex:1]; - [enc setBuffer:headsbuf offset:ds4_gpu_tensor_offset(heads) atIndex:2]; - [enc dispatchThreadgroups:MTLSizeMake(nrows, 1, 1) - threadsPerThreadgroup:MTLSizeMake(32u * nwg, 1, 1)]; - ds4_gpu_end_compute_encoder(cb, enc); - DS4_METAL_PROFILE_FLASH_ATTN_STAGE("attention_reduce"); + if (!direct_output) { + ds4_gpu_flash_attn_reduce_args reduce_args = { + .nrows = (int32_t)nrows, + }; + enc = ds4_gpu_compute_encoder(cb); + [enc setComputePipelineState:reduce_pipeline]; + [enc setBytes:&reduce_args length:sizeof(reduce_args) atIndex:0]; + [enc setBuffer:g_flash_attn_tmp_buffer offset:0 atIndex:1]; + [enc setBuffer:headsbuf offset:ds4_gpu_tensor_offset(heads) atIndex:2]; + [enc dispatchThreadgroups:MTLSizeMake(nrows, 1, 1) + threadsPerThreadgroup:MTLSizeMake(32u * nwg, 1, 1)]; + ds4_gpu_end_compute_encoder(cb, enc); + DS4_METAL_PROFILE_FLASH_ATTN_STAGE("attention_reduce"); + } #undef DS4_METAL_PROFILE_FLASH_ATTN_STAGE return 1; diff --git a/scripts/environment_variables.tsv b/scripts/environment_variables.tsv index 264d06cb5..20cbbfda5 100644 --- a/scripts/environment_variables.tsv +++ b/scripts/environment_variables.tsv @@ -635,6 +635,7 @@ runtime/metal DS4_METAL_DISABLE_SHARED_DOWN_HC_FUSION nonempty boolean; unset/em runtime/metal DS4_METAL_DISABLE_SHARED_GATE_UP_SWIGLU_FUSION presence rollback; unset: automatic/default path; any value including 0 disables Disables shared gate up SwiGLU fusion. ds4.c:17567 runtime/metal DS4_METAL_DISABLE_SHARED_KV_PAD presence rollback; unset: automatic/default path; any value including 0 disables Disables shared KV pad. ds4_metal.m:31510 runtime/metal DS4_METAL_DISABLE_SHARED_ROPE_COEFF presence rollback; unset: automatic/default path; any value including 0 disables Disables shared RoPE coeff. ds4_metal.m:6341 +runtime/metal DS4_METAL_DISABLE_SMALL_PREFILL_DIRECT presence rollback; unset: direct one-workgroup FlashAttention output for prefill with at most 32 keys; any defined value including 0 restores split reduction Disables direct normalized output for small Metal prefill and restores the 32-way temporary plus reduction path. ds4_metal.m:4356 runtime/metal DS4_METAL_DISABLE_STREAMING_COLD_DECODE_PREFILL presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming cold decode prefill. ds4.c:32007 runtime/metal DS4_METAL_DISABLE_STREAMING_COMPACT_ADDR presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming compact address. ds4_metal.m:14598 runtime/metal DS4_METAL_DISABLE_STREAMING_DECODE_PREFILL presence rollback; unset: automatic/default path; any value including 0 disables Disables streaming decode prefill. ds4.c:31956 diff --git a/tests/ds4_test.c b/tests/ds4_test.c index 9c9d7fee5..e7271ae6c 100644 --- a/tests/ds4_test.c +++ b/tests/ds4_test.c @@ -4358,7 +4358,8 @@ static void test_metal_compressor_ratio4_exact_pool_decode_case( kv_cur, sc_cur, ref_state_kv, ref_state_score, ref_comp, model_raw, model_bytes, 0, ape_type, norm_offset, 0, head_dim, ratio, pos, 0, 0, 0, - 10000.0f, 1.0f, 0.0f, 1.0f, 32.0f, 1.0f, 1.0e-6f, false); + 10000.0f, 1.0f, 0.0f, 1.0f, 32.0f, 1.0f, 1.0e-6f, + false, false, false); TEST_ASSERT(ref_ok != 0); TEST_ASSERT(unsetenv(disable_env) == 0); @@ -4372,7 +4373,8 @@ static void test_metal_compressor_ratio4_exact_pool_decode_case( kv_cur, sc_cur, exact_state_kv, exact_state_score, exact_comp, model_raw, model_bytes, 0, ape_type, norm_offset, 0, head_dim, ratio, pos, 0, 0, 0, - 10000.0f, 1.0f, 0.0f, 1.0f, 32.0f, 1.0f, 1.0e-6f, false); + 10000.0f, 1.0f, 0.0f, 1.0f, 32.0f, 1.0f, 1.0e-6f, + false, false, false); TEST_ASSERT(exact_ok != 0); TEST_ASSERT(ds4_gpu_tensor_read( @@ -5807,6 +5809,539 @@ static void test_metal_zero_prefix_prefill_mask_cache_exact(void) { test_metal_zero_prefix_prefill_mask_cache_exact_kind( TEST_METAL_PREFILL_MASK_CACHE_RATIO128, 47); } + +static double test_metal_small_prefill_gpu_batch_ms( + test_metal_prefill_mask_cache_kind kind, + bool masked, + const test_metal_prefill_mask_cache_shape *shape, + ds4_gpu_tensor *heads, + const void *model_map, + uint64_t model_size, + const ds4_gpu_tensor *q, + const ds4_gpu_tensor *raw, + const ds4_gpu_tensor *comp, + const ds4_gpu_tensor *comp_mask, + uint32_t n_head, + uint32_t head_dim, + uint32_t flags, + uint32_t repeats, + uint32_t expected_nwg) { + ds4_gpu_test_set_flags(flags); + const int begin_ok = ds4_gpu_begin_commands(); + TEST_ASSERT(begin_ok != 0); + if (!begin_ok) return -1.0; + for (uint32_t rep = 0u; rep < repeats; rep++) { + const int call_ok = test_metal_zero_prefix_prefill_mask_cache_call( + kind, heads, model_map, model_size, q, raw, comp, comp_mask, + shape, masked, n_head, head_dim); + TEST_ASSERT(call_ok != 0); + if (!call_ok) { + (void)ds4_gpu_end_commands(); + return -1.0; + } + } + const int end_ok = ds4_gpu_end_commands(); + TEST_ASSERT(end_ok != 0); + if (!end_ok) return -1.0; + const uint32_t selected_nwg = + ds4_gpu_test_last_flash_attn_prefill_nwg(); + TEST_ASSERT(selected_nwg == expected_nwg); + if (selected_nwg != expected_nwg) return -1.0; + const double gpu_ms = ds4_gpu_test_last_completed_gpu_ms(); + TEST_ASSERT(gpu_ms > 0.0); + return gpu_ms > 0.0 ? gpu_ms / (double)repeats : -1.0; +} + +static void test_metal_small_prefill_direct_exact(void) { + const uint32_t head_dim = 512u; + const uint32_t n_head = 2u; + const uint32_t max_tokens = 19u; + const uint32_t max_comp = 14u; + const uint64_t guard_bytes = 256u; + const uint64_t raw_count = (uint64_t)max_tokens * head_dim; + const uint64_t comp_count = (uint64_t)max_comp * head_dim; + const uint64_t q_count = + (uint64_t)max_tokens * n_head * head_dim; + const uint64_t mask_count = (uint64_t)max_tokens * max_comp; + const uint64_t raw_bytes = raw_count * sizeof(float); + const uint64_t comp_bytes = comp_count * sizeof(uint16_t); + const uint64_t q_bytes = q_count * sizeof(float); + const uint64_t mask_bytes = mask_count * sizeof(float); + const uint64_t heads_base_bytes = guard_bytes + q_bytes + guard_bytes; + const uint64_t page = (uint64_t)getpagesize(); + const char *disable_env = "DS4_METAL_DISABLE_SMALL_PREFILL_DIRECT"; + char *saved_disable = test_save_env(disable_env); + const char *stage_profile_env = "DS4_METAL_FLASH_ATTN_STAGE_PROFILE"; + char *saved_stage_profile = test_save_env(stage_profile_env); + + typedef struct { + const char *name; + test_metal_prefill_mask_cache_kind kind; + bool masked; + test_metal_prefill_mask_cache_shape shape; + } test_metal_small_prefill_case; + static const test_metal_small_prefill_case cases[] = { + {"raw-1", TEST_METAL_PREFILL_MASK_CACHE_RAW, false, + {1u, 0u, 0u, 0u}}, + {"raw-7", TEST_METAL_PREFILL_MASK_CACHE_RAW, false, + {7u, 0u, 5u, 0u}}, + {"raw-19", TEST_METAL_PREFILL_MASK_CACHE_RAW, false, + {19u, 0u, 11u, 0u}}, + {"static-12", TEST_METAL_PREFILL_MASK_CACHE_RATIO4, false, + {7u, 5u, 5u, 4u}}, + {"static-32", TEST_METAL_PREFILL_MASK_CACHE_RATIO4, false, + {19u, 13u, 11u, 4u}}, + {"masked-12", TEST_METAL_PREFILL_MASK_CACHE_RATIO4, true, + {7u, 5u, 5u, 4u}}, + {"masked-32", TEST_METAL_PREFILL_MASK_CACHE_RATIO4, true, + {19u, 13u, 11u, 4u}}, + {"static-33-fallback", TEST_METAL_PREFILL_MASK_CACHE_RATIO4, false, + {19u, 14u, 11u, 4u}}, + }; + enum { + TEST_SMALL_PREFILL_DIRECT_COLD = 0, + TEST_SMALL_PREFILL_PSO_FALLBACK, + TEST_SMALL_PREFILL_ENV_ROLLBACK, + TEST_SMALL_PREFILL_DIRECT_REPLAY, + TEST_SMALL_PREFILL_ARM_COUNT, + }; + + ds4_gpu_tensor *raw = ds4_gpu_tensor_alloc(raw_bytes); + ds4_gpu_tensor *comp = ds4_gpu_tensor_alloc(comp_bytes); + ds4_gpu_tensor *q = ds4_gpu_tensor_alloc(q_bytes); + ds4_gpu_tensor *comp_mask = ds4_gpu_tensor_alloc(mask_bytes); + ds4_gpu_tensor *heads_base = ds4_gpu_tensor_alloc(heads_base_bytes); + float *raw_host = malloc((size_t)raw_bytes); + uint16_t *comp_host = malloc((size_t)comp_bytes); + float *q_host = malloc((size_t)q_bytes); + float *mask_host = malloc((size_t)mask_bytes); + uint8_t *initial = malloc((size_t)heads_base_bytes); + uint8_t *snapshots = malloc( + (size_t)(TEST_SMALL_PREFILL_ARM_COUNT * heads_base_bytes)); + /* Metal retains a no-copy model view after this oracle returns. Static + * page-aligned storage keeps that backing valid until backend cleanup. */ + static uint8_t model_storage[16384] + __attribute__((aligned(16384))); + void *model_raw = model_storage; + const bool model_storage_ok = page <= sizeof(model_storage); + TEST_ASSERT(model_storage_ok); + TEST_ASSERT(raw != NULL); + TEST_ASSERT(comp != NULL); + TEST_ASSERT(q != NULL); + TEST_ASSERT(comp_mask != NULL); + TEST_ASSERT(heads_base != NULL); + TEST_ASSERT(raw_host != NULL); + TEST_ASSERT(comp_host != NULL); + TEST_ASSERT(q_host != NULL); + TEST_ASSERT(mask_host != NULL); + TEST_ASSERT(initial != NULL); + TEST_ASSERT(snapshots != NULL); + + size_t direct_mismatches = 0u; + size_t replay_mismatches = 0u; + size_t fallback_mismatches = 0u; + size_t alias_mismatches = 0u; + size_t alias_guard_mismatches = 0u; + size_t guard_mismatches = 0u; + size_t nonfinite = 0u; + size_t total_words = 0u; + size_t completed_cases = 0u; + const bool allocated = raw && comp && q && comp_mask && heads_base && + raw_host && comp_host && q_host && mask_host && initial && + snapshots && model_storage_ok; + if (allocated) { + memset(model_raw, 0, (size_t)page); + ((float *)model_raw)[0] = -0.375f; + ((float *)model_raw)[1] = 0.21875f; + for (uint64_t i = 0; i < raw_count; i++) { + const int value = (int)((i * 17u + + (i ^ (i >> 5u)) * 11u + 23u) % 211u) - 105; + raw_host[i] = (float)value / 192.0f; + } + for (uint64_t i = 0; i < comp_count; i++) { + const int value = (int)((i * 23u + + (i ^ (i >> 4u)) * 7u + 29u) % 193u) - 96; + comp_host[i] = test_float_to_f16( + 0.125f + (float)value / 224.0f); + } + for (uint64_t i = 0; i < q_count; i++) { + const int value = (int)((i * 31u + + (i ^ (i >> 3u)) * 5u + 37u) % 227u) - 113; + q_host[i] = (float)value / 208.0f; + } + for (uint64_t i = 0; i < mask_count; i++) { + mask_host[i] = (i % 5u) == 0u + ? -65504.0f : -(float)((i % 13u) + 1u) / 16.0f; + } + for (uint64_t i = 0; i < heads_base_bytes; i++) { + initial[i] = (uint8_t)(0xa5u ^ (uint8_t)(i * 37u)); + } + const uint32_t poison = 0x7fc12345u; + for (uint64_t i = 0; i < q_count; i++) { + memcpy(initial + guard_bytes + i * sizeof(poison), + &poison, sizeof(poison)); + } + + TEST_ASSERT(ds4_gpu_tensor_write( + raw, 0u, raw_host, raw_bytes) != 0); + TEST_ASSERT(ds4_gpu_tensor_write( + comp, 0u, comp_host, comp_bytes) != 0); + TEST_ASSERT(ds4_gpu_tensor_write( + q, 0u, q_host, q_bytes) != 0); + TEST_ASSERT(ds4_gpu_tensor_write( + comp_mask, 0u, mask_host, mask_bytes) != 0); + TEST_ASSERT(ds4_gpu_set_model_map(model_raw, page) != 0); + ds4_gpu_set_quality(false); + TEST_ASSERT(unsetenv(disable_env) == 0); + TEST_ASSERT(unsetenv(stage_profile_env) == 0); + + for (size_t case_i = 0; + case_i < sizeof(cases) / sizeof(cases[0]); case_i++) { + const test_metal_small_prefill_case *c = &cases[case_i]; + const uint64_t output_count = + (uint64_t)c->shape.n_tokens * n_head * head_dim; + const uint64_t output_bytes = output_count * sizeof(float); + ds4_gpu_tensor *heads = ds4_gpu_tensor_view( + heads_base, guard_bytes, output_bytes); + TEST_ASSERT(heads != NULL); + bool case_ok = heads != NULL; + for (uint32_t arm = 0u; + arm < TEST_SMALL_PREFILL_ARM_COUNT && case_ok; arm++) { + const bool cold_scratch_case = + case_i == 0u || case_i == 3u; + if (cold_scratch_case && + (arm == TEST_SMALL_PREFILL_DIRECT_COLD || + arm == TEST_SMALL_PREFILL_PSO_FALLBACK)) { + TEST_ASSERT(ds4_gpu_test_reset_flash_attn_tmp() != 0); + TEST_ASSERT(ds4_gpu_test_flash_attn_tmp_bytes() == 0u); + } + TEST_ASSERT(ds4_gpu_tensor_write( + heads_base, 0u, initial, heads_base_bytes) != 0); + const bool env_rollback = + arm == TEST_SMALL_PREFILL_ENV_ROLLBACK; + const int env_ok = env_rollback + ? setenv(disable_env, "0", 1) + : unsetenv(disable_env); + TEST_ASSERT(env_ok == 0); + ds4_gpu_test_set_flags( + arm == TEST_SMALL_PREFILL_PSO_FALLBACK + ? DS4_GPU_TEST_FLASH_ATTN_SMALL_PREFILL_NWG1_FAILURE + : 0u); + const int call_ok = + test_metal_zero_prefix_prefill_mask_cache_call( + c->kind, heads, model_raw, page, q, raw, comp, + comp_mask, &c->shape, c->masked, + n_head, head_dim); + TEST_ASSERT(call_ok != 0); + const uint32_t n_keys = + c->shape.n_tokens + c->shape.n_comp; + const bool expect_direct = + (arm == TEST_SMALL_PREFILL_DIRECT_COLD || + arm == TEST_SMALL_PREFILL_DIRECT_REPLAY) && + n_keys <= 32u; + const uint32_t expected_nwg = expect_direct ? 1u : 32u; + const uint32_t selected_nwg = + ds4_gpu_test_last_flash_attn_prefill_nwg(); + TEST_ASSERT(selected_nwg == expected_nwg); + if (cold_scratch_case && + arm == TEST_SMALL_PREFILL_DIRECT_COLD) { + TEST_ASSERT( + ds4_gpu_test_flash_attn_tmp_bytes() == 0u); + } + if (cold_scratch_case && + arm == TEST_SMALL_PREFILL_PSO_FALLBACK) { + TEST_ASSERT( + ds4_gpu_test_flash_attn_tmp_bytes() != 0u); + } + const int read_ok = call_ok && ds4_gpu_tensor_read( + heads_base, 0u, + snapshots + (uint64_t)arm * heads_base_bytes, + heads_base_bytes) != 0; + TEST_ASSERT(read_ok != 0); + case_ok = read_ok != 0 && selected_nwg == expected_nwg; + } + ds4_gpu_test_set_flags(0u); + ds4_gpu_tensor_free(heads); + if (!case_ok) continue; + + const uint8_t *direct = snapshots + + TEST_SMALL_PREFILL_DIRECT_COLD * heads_base_bytes; + const uint8_t *baseline = snapshots + + TEST_SMALL_PREFILL_ENV_ROLLBACK * heads_base_bytes; + const uint8_t *replay = snapshots + + TEST_SMALL_PREFILL_DIRECT_REPLAY * heads_base_bytes; + const uint8_t *fallback = snapshots + + TEST_SMALL_PREFILL_PSO_FALLBACK * heads_base_bytes; + const test_float_compare_stats direct_stats = + test_compare_float_bits( + (const float *)(baseline + guard_bytes), + (const float *)(direct + guard_bytes), + (size_t)output_count); + const test_float_compare_stats replay_stats = + test_compare_float_bits( + (const float *)(baseline + guard_bytes), + (const float *)(replay + guard_bytes), + (size_t)output_count); + const test_float_compare_stats fallback_stats = + test_compare_float_bits( + (const float *)(baseline + guard_bytes), + (const float *)(fallback + guard_bytes), + (size_t)output_count); + direct_mismatches += direct_stats.mismatch_count; + replay_mismatches += replay_stats.mismatch_count; + fallback_mismatches += fallback_stats.mismatch_count; + total_words += (size_t)output_count; + completed_cases++; + + for (uint32_t arm = 0u; + arm < TEST_SMALL_PREFILL_ARM_COUNT; arm++) { + const uint8_t *snapshot = + snapshots + (uint64_t)arm * heads_base_bytes; + for (uint64_t i = 0; i < heads_base_bytes; i++) { + const bool in_output = i >= guard_bytes && + i < guard_bytes + output_bytes; + if (!in_output && snapshot[i] != initial[i]) { + guard_mismatches++; + } + } + const float *values = + (const float *)(snapshot + guard_bytes); + for (uint64_t i = 0; i < output_count; i++) { + if (!isfinite(values[i])) nonfinite++; + } + } + fprintf(stderr, + "ds4-test: small-prefill direct %s keys=%u " + "direct=%zu/%llu replay=%zu/%llu fallback=%zu/%llu\n", + c->name, + c->shape.n_tokens + c->shape.n_comp, + direct_stats.mismatch_count, + (unsigned long long)output_count, + replay_stats.mismatch_count, + (unsigned long long)output_count, + fallback_stats.mismatch_count, + (unsigned long long)output_count); + } + + /* The legacy split path permits heads to alias q because its first + * dispatch finishes reading q before the reducer writes heads. The + * direct kernel must preserve that API behavior by falling back. */ + { + static const size_t alias_case_indices[] = {1u, 3u, 5u}; + for (size_t alias_i = 0u; + alias_i < sizeof(alias_case_indices) / + sizeof(alias_case_indices[0]); + alias_i++) { + const test_metal_small_prefill_case *c = + &cases[alias_case_indices[alias_i]]; + const uint64_t output_count = + (uint64_t)c->shape.n_tokens * n_head * head_dim; + const uint64_t output_bytes = output_count * sizeof(float); + ds4_gpu_tensor *heads_ref = ds4_gpu_tensor_view( + heads_base, guard_bytes, output_bytes); + ds4_gpu_tensor *q_alias = ds4_gpu_tensor_view( + heads_base, guard_bytes, q_bytes); + ds4_gpu_tensor *heads_alias = ds4_gpu_tensor_view( + heads_base, guard_bytes, output_bytes); + TEST_ASSERT(heads_ref != NULL); + TEST_ASSERT(q_alias != NULL); + TEST_ASSERT(heads_alias != NULL); + if (heads_ref && q_alias && heads_alias) { + TEST_ASSERT(ds4_gpu_tensor_write( + q, 0u, q_host, q_bytes) != 0); + TEST_ASSERT(ds4_gpu_tensor_write( + heads_base, 0u, initial, heads_base_bytes) != 0); + TEST_ASSERT(setenv(disable_env, "0", 1) == 0); + ds4_gpu_test_set_flags(0u); + const int ref_ok = + test_metal_zero_prefix_prefill_mask_cache_call( + c->kind, heads_ref, model_raw, page, q, raw, + comp, comp_mask, &c->shape, c->masked, + n_head, head_dim); + TEST_ASSERT(ref_ok != 0); + TEST_ASSERT( + ds4_gpu_test_last_flash_attn_prefill_nwg() == 32u); + const int ref_read_ok = ref_ok && ds4_gpu_tensor_read( + heads_base, 0u, snapshots, heads_base_bytes) != 0; + TEST_ASSERT(ref_read_ok != 0); + + TEST_ASSERT(ds4_gpu_tensor_write( + heads_base, 0u, initial, heads_base_bytes) != 0); + TEST_ASSERT(ds4_gpu_tensor_write( + q_alias, 0u, q_host, q_bytes) != 0); + const int before_read_ok = ds4_gpu_tensor_read( + heads_base, 0u, snapshots + heads_base_bytes, + heads_base_bytes) != 0; + TEST_ASSERT(before_read_ok != 0); + TEST_ASSERT(unsetenv(disable_env) == 0); + ds4_gpu_test_set_flags(0u); + const int alias_ok = + test_metal_zero_prefix_prefill_mask_cache_call( + c->kind, heads_alias, model_raw, page, q_alias, + raw, comp, comp_mask, &c->shape, c->masked, + n_head, head_dim); + TEST_ASSERT(alias_ok != 0); + const uint32_t alias_nwg = + ds4_gpu_test_last_flash_attn_prefill_nwg(); + TEST_ASSERT(alias_nwg == 32u); + const int alias_read_ok = alias_ok && + ds4_gpu_tensor_read( + heads_base, 0u, + snapshots + 2u * heads_base_bytes, + heads_base_bytes) != 0; + TEST_ASSERT(alias_read_ok != 0); + if (ref_read_ok && before_read_ok && alias_read_ok) { + const test_float_compare_stats alias_stats = + test_compare_float_bits( + (const float *)(snapshots + guard_bytes), + (const float *)(snapshots + + 2u * heads_base_bytes + guard_bytes), + (size_t)output_count); + alias_mismatches += alias_stats.mismatch_count; + size_t case_guard_mismatches = 0u; + const uint8_t *before = + snapshots + heads_base_bytes; + const uint8_t *after = + snapshots + 2u * heads_base_bytes; + for (uint64_t i = 0u; + i < heads_base_bytes; i++) { + const bool in_output = i >= guard_bytes && + i < guard_bytes + output_bytes; + if (!in_output && before[i] != after[i]) { + case_guard_mismatches++; + } + } + alias_guard_mismatches += case_guard_mismatches; + fprintf(stderr, + "ds4-test: small-prefill q/heads alias " + "%s nwg=%u mismatches=%zu/%llu guards=%zu\n", + c->name, alias_nwg, + alias_stats.mismatch_count, + (unsigned long long)output_count, + case_guard_mismatches); + } + } + ds4_gpu_test_set_flags(0u); + TEST_ASSERT(unsetenv(disable_env) == 0); + ds4_gpu_tensor_free(heads_alias); + ds4_gpu_tensor_free(q_alias); + ds4_gpu_tensor_free(heads_ref); + } + } + + if (test_env_bool("DS4_TEST_METAL_SMALL_PREFILL_TIMING")) { + /* GPUStartTime/GPUEndTime excludes CPU mask construction, + * encoding, submit, and wait time. Both arms run the same + * copy/pad kernels; their only topology difference is NWG=32 + + * reduce versus direct NWG=1 output. Alternate pair order to + * limit thermal/order bias. Keep this opt-in so the exactness + * oracle remains suitable for routine test runs. */ + static const size_t timing_case_indices[] = {2u, 4u}; + const uint32_t timing_repeats = 16u; + const uint32_t timing_samples = 4u; + for (size_t timing_i = 0u; + timing_i < sizeof(timing_case_indices) / + sizeof(timing_case_indices[0]); + timing_i++) { + const test_metal_small_prefill_case *c = + &cases[timing_case_indices[timing_i]]; + const uint64_t output_bytes = + (uint64_t)c->shape.n_tokens * n_head * head_dim * + sizeof(float); + ds4_gpu_tensor *heads = ds4_gpu_tensor_view( + heads_base, guard_bytes, output_bytes); + TEST_ASSERT(heads != NULL); + if (!heads) continue; + + (void)test_metal_small_prefill_gpu_batch_ms( + c->kind, c->masked, &c->shape, heads, model_raw, page, + q, raw, comp, comp_mask, n_head, head_dim, + DS4_GPU_TEST_FLASH_ATTN_SMALL_PREFILL_NWG32, 4u, 32u); + (void)test_metal_small_prefill_gpu_batch_ms( + c->kind, c->masked, &c->shape, heads, model_raw, page, + q, raw, comp, comp_mask, n_head, head_dim, + 0u, 4u, 1u); + + double baseline_sum = 0.0; + double direct_sum = 0.0; + double log_speedup_sum = 0.0; + uint32_t valid_samples = 0u; + for (uint32_t sample = 0u; + sample < timing_samples; sample++) { + double pair_ms[2] = {-1.0, -1.0}; + for (uint32_t order_i = 0u; order_i < 2u; order_i++) { + const uint32_t arm = (sample & 1u) != 0u + ? 1u - order_i : order_i; + pair_ms[arm] = + test_metal_small_prefill_gpu_batch_ms( + c->kind, c->masked, &c->shape, + heads, model_raw, page, q, raw, comp, + comp_mask, n_head, head_dim, + arm == 0u + ? DS4_GPU_TEST_FLASH_ATTN_SMALL_PREFILL_NWG32 + : 0u, + timing_repeats, + arm == 0u ? 32u : 1u); + } + if (pair_ms[0] > 0.0 && pair_ms[1] > 0.0) { + baseline_sum += pair_ms[0]; + direct_sum += pair_ms[1]; + log_speedup_sum += log(pair_ms[0] / pair_ms[1]); + valid_samples++; + } + } + TEST_ASSERT(valid_samples == timing_samples); + if (valid_samples != 0u) { + const double baseline_ms = + baseline_sum / (double)valid_samples; + const double direct_ms = + direct_sum / (double)valid_samples; + const double speedup = + exp(log_speedup_sum / (double)valid_samples); + fprintf(stderr, + "ds4-test: small-prefill GPU-only kernel-chain %s " + "legacy=%.6f ms direct=%.6f ms speedup=%.3fx " + "throughput=%.1f%%\n", + c->name, baseline_ms, direct_ms, speedup, + (speedup - 1.0) * 100.0); + } + ds4_gpu_tensor_free(heads); + } + } + } + + ds4_gpu_test_set_flags(0u); + test_restore_env(stage_profile_env, saved_stage_profile); + test_restore_env(disable_env, saved_disable); + fprintf(stderr, + "ds4-test: small-prefill direct exact cases=%zu/%zu " + "words=%zu direct=%zu replay=%zu fallback=%zu alias=%zu " + "guards=%zu alias_guards=%zu nonfinite=%zu\n", + completed_cases, sizeof(cases) / sizeof(cases[0]), total_words, + direct_mismatches, replay_mismatches, fallback_mismatches, + alias_mismatches, guard_mismatches, alias_guard_mismatches, + nonfinite); + TEST_ASSERT(completed_cases == sizeof(cases) / sizeof(cases[0])); + TEST_ASSERT(direct_mismatches == 0u); + TEST_ASSERT(replay_mismatches == 0u); + TEST_ASSERT(fallback_mismatches == 0u); + TEST_ASSERT(alias_mismatches == 0u); + TEST_ASSERT(guard_mismatches == 0u); + TEST_ASSERT(alias_guard_mismatches == 0u); + TEST_ASSERT(nonfinite == 0u); + + free(snapshots); + free(initial); + free(mask_host); + free(q_host); + free(comp_host); + free(raw_host); + ds4_gpu_tensor_free(heads_base); + ds4_gpu_tensor_free(comp_mask); + ds4_gpu_tensor_free(q); + ds4_gpu_tensor_free(comp); + ds4_gpu_tensor_free(raw); +} #endif #if defined(__APPLE__) @@ -7592,6 +8127,7 @@ static void test_metal_kernel_group(void) { #if defined(__APPLE__) if (test_env_bool("DS4_TEST_METAL_RESIDENT_ORACLES_ONLY")) { test_metal_flush_commands_progress_exact(); + test_metal_small_prefill_direct_exact(); test_metal_batch_attn_out_hc_fusion_exact(); return; } @@ -7623,6 +8159,7 @@ static void test_metal_kernel_group(void) { test_metal_contiguous_compressed_f16_attention_exact(); test_metal_persistent_zero_attention_mask_exact(); test_metal_zero_prefix_prefill_mask_cache_exact(); + test_metal_small_prefill_direct_exact(); test_metal_batch_attn_out_hc_fusion_exact(); test_metal_hc_split_weighted_sum_norm_batch_exact(); test_metal_hc_producer_pre_norm_compound_exact(); @@ -9725,6 +10262,7 @@ static void test_print_help(const char *prog) { puts(" DS4_TEST_LOCAL_GOLDEN_FILE=FILE Local fixture. Default: flash-0731/local-golden.vec."); puts(" DS4_TEST_MPP_EQ_CASE=NAME Run only Tensor equivalence cases whose id contains NAME."); puts(" DS4_TEST_METAL_RESIDENT_ORACLES_ONLY=1 Restrict --metal-kernels to resident optimization oracles."); + puts(" DS4_TEST_METAL_SMALL_PREFILL_TIMING=1 Include the small-prefill GPU-only microbenchmark."); puts(" DS4_TEST_MTP=FILE Legacy MTP support GGUF for --mtp-verify-depth."); puts(" DS4_TEST_DSPARK=FILE DSpark support GGUF for --dspark-verify-depth."); puts(" DS4_TEST_CONTINUED_PREFILL_TOKENS=N Large suffix size for --glm53-continued-prefill."); From 21d5b2570d7f00675de96e1fc89975f29bf8ad97 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Sat, 29 Aug 2026 09:13:09 +0200 Subject: [PATCH 143/189] perf(metal): share q4 prefill rhs across qkv projections --- .gitignore | 1 + Makefile | 37 +- ds4.c | 18 +- ds4_gpu.h | 4 +- ds4_metal.m | 215 +++++++++- speed-bench/.gitignore | 1 + speed-bench/metal_q4_prefill_pair_bench.m | 405 ++++++++++++++++++ tests/test_metal_q4_prefill_pair.c | 491 ++++++++++++++++++++++ tests/test_metal_q4_streams.c | 2 +- 9 files changed, 1159 insertions(+), 15 deletions(-) create mode 100644 speed-bench/metal_q4_prefill_pair_bench.m create mode 100644 tests/test_metal_q4_prefill_pair.c diff --git a/.gitignore b/.gitignore index 4db2bbf6b..c77b669c0 100644 --- a/.gitignore +++ b/.gitignore @@ -27,6 +27,7 @@ /tests/test_metal_session_batch /tests/test_metal_indexer_q4 /tests/test_metal_q4_qb_f16_cache +/tests/test_metal_q4_prefill_pair /tests/test_metal_q4_streams /tests/test_mxfp4_cuda /tests/test_mxfp4_dot diff --git a/Makefile b/Makefile index 5e8e60f2d..71c71ed63 100644 --- a/Makefile +++ b/Makefile @@ -69,7 +69,7 @@ DS4_LINK_LIBS ?= $(CUDA_LDLIBS) METAL_LDLIBS := $(LDLIBS) endif -.PHONY: all help clean test test-ssd environment-docs test-quantizer-indexer-q4 test-rocm test-glm53-kda-rocm test-metal-session-batch test-metal-session-batch-ssd test-metal-q4-streams test-metal-indexer-q4 test-metal-q4-attn-exactn test-metal-q4-qb-f16-cache test-metal-q4-qb-f16-cache-timing test-metal-exactn-oracle test-metal-dspark-capture test-metal-argmax-top1 bench-metal-argmax-top1 test-metal-iq2-midonly test-metal-iq2-ssd-grouped-mm test-metal-iq2-live-index test-mxfp4-metal test-mxfp4-cuda test-mxfp4-rocm test-mmq-parity-cuda test-rocm-q4-parity test-rocm-q4-dense test-rocm-q4-pair test-rocm-q4-prefill test-strix-rocm-q4-parity test-strix-rocm-q4-prefill test-strix-rocm-q4-prefill-long test-cuda-session-batch test-cuda-mixed-batch dspark-acceptance dspark-verify-depth rocm-dspark-acceptance rocm-dspark-verify-depth mtp-verify-depth cpu cuda cuda-spark cuda-generic cuda-regression strix-halo rocm cuda-iq2-moe-prefill-bench cuda-q4-prefill-bench rocm-iq2-moe-prefill-bench rocm-q4-prefill-bench +.PHONY: all help clean test test-ssd environment-docs test-quantizer-indexer-q4 test-rocm test-glm53-kda-rocm test-metal-session-batch test-metal-session-batch-ssd test-metal-q4-streams test-metal-q4-prefill-pair test-metal-indexer-q4 test-metal-q4-attn-exactn test-metal-q4-qb-f16-cache test-metal-q4-qb-f16-cache-timing test-metal-exactn-oracle test-metal-dspark-capture test-metal-argmax-top1 bench-metal-argmax-top1 test-metal-iq2-midonly test-metal-iq2-ssd-grouped-mm test-metal-iq2-live-index test-mxfp4-metal test-mxfp4-cuda test-mxfp4-rocm test-mmq-parity-cuda test-rocm-q4-parity test-rocm-q4-dense test-rocm-q4-pair test-rocm-q4-prefill test-strix-rocm-q4-parity test-strix-rocm-q4-prefill test-strix-rocm-q4-prefill-long test-cuda-session-batch test-cuda-mixed-batch dspark-acceptance dspark-verify-depth rocm-dspark-acceptance rocm-dspark-verify-depth mtp-verify-depth cpu cuda cuda-spark cuda-generic cuda-regression strix-halo rocm cuda-iq2-moe-prefill-bench cuda-q4-prefill-bench rocm-iq2-moe-prefill-bench rocm-q4-prefill-bench gguf-tools/deepseek4-quantize: gguf-tools/deepseek4-quantize.c gguf-tools/quants.c gguf-tools/quants.h $(MAKE) -C gguf-tools deepseek4-quantize @@ -81,7 +81,7 @@ test-quantizer-indexer-q4: gguf-tools/deepseek4-quantize tests/test_quantizer_in ./tests/test_quantizer_indexer_q4 ./gguf-tools/deepseek4-quantize ifeq ($(UNAME_S),Darwin) -.PHONY: metal-decode-schedule-bench metal-prefill-variant-bench metal-q4-dense-pair-bench metal-q4-mm-tail-cull-bench metal-iq2-moe-tail-cull-bench check-mxfp4-half-lut test-mxfp4-metal +.PHONY: metal-decode-schedule-bench metal-prefill-variant-bench metal-q4-dense-pair-bench metal-q4-prefill-pair-bench metal-q4-mm-tail-cull-bench metal-iq2-moe-tail-cull-bench check-mxfp4-half-lut test-mxfp4-metal all: ds4 ds4-server ds4-bench ds4-eval ds4-agent @@ -95,6 +95,7 @@ help: @echo " make environment-docs Generate and verify the environment variable inventory" @echo " make test-metal-session-batch-ssd Exact-logit Metal SSD union control/candidate oracle" @echo " make test-metal-q4-streams Check resident Q4 Metal stream overlap" + @echo " make test-metal-q4-prefill-pair Runtime oracle for the M1-M4 Q4 prefill pair" @echo " make test-metal-indexer-q4 Check the production-shape Q4_K indexer projection" @echo " make test-metal-q4-attn-exactn Bitwise/canary oracle for M1-M4 SSD-prefill Q4 attention output" @echo " make test-metal-q4-qb-f16-cache Oracle for M1-M4 Q4 q_b sidecar and transient F16 paths" @@ -109,6 +110,7 @@ help: @echo " make metal-decode-schedule-bench Build the balanced Metal decode schedule benchmark" @echo " make metal-prefill-variant-bench Build the balanced Metal prefill variant benchmark" @echo " make metal-q4-dense-pair-bench Build the resident Q4 decode pair kernel benchmark" + @echo " make metal-q4-prefill-pair-bench Build the resident Q4 prefill pair F16-RHS benchmark" @echo " make metal-q4-mm-tail-cull-bench Build the resident Q4 prefill tail-cull kernel benchmark" @echo " make metal-iq2-moe-tail-cull-bench Build the resident IQ2 pair MoE tail-cull benchmark" @echo " make check-mxfp4-half-lut Verify the checked-in MXFP4 half LUT matches the generator" @@ -181,6 +183,30 @@ test-metal-q4-streams: tests/test_metal_q4_streams env -u DS4_METAL_MODEL_UNTRACKED ./tests/test_metal_q4_streams DS4_METAL_MODEL_UNTRACKED=1 ./tests/test_metal_q4_streams +tests/test_metal_q4_prefill_pair.o: tests/test_metal_q4_prefill_pair.c ds4_gpu.h + $(CC) $(CFLAGS) -I. -c -o $@ $< + +tests/test_metal_q4_prefill_pair: tests/test_metal_q4_prefill_pair.o ds4_metal.o + $(CC) $(CFLAGS) -o $@ $^ $(METAL_LDLIBS) + +test-metal-q4-prefill-pair: tests/test_metal_q4_prefill_pair + env -u DS4_METAL_ENABLE_Q4_PREFILL_PAIR_F16_RHS \ + -u DS4_METAL_DISABLE_Q4_PREFILL_PAIR_F16_RHS \ + -u DS4_METAL_REQUIRE_Q4_PREFILL_PAIR_F16_RHS \ + -u DS4_METAL_DISABLE_Q4_DENSE_PAIR \ + -u DS4_METAL_DISABLE_CONTIG_F32_F16_COPY \ + -u DS4_METAL_MODEL_UNTRACKED \ + -u DS4_METAL_UNRETAINED_COMMAND_BUFFERS \ + ./tests/test_metal_q4_prefill_pair + env -u DS4_METAL_ENABLE_Q4_PREFILL_PAIR_F16_RHS \ + -u DS4_METAL_DISABLE_Q4_PREFILL_PAIR_F16_RHS \ + -u DS4_METAL_REQUIRE_Q4_PREFILL_PAIR_F16_RHS \ + -u DS4_METAL_DISABLE_Q4_DENSE_PAIR \ + -u DS4_METAL_DISABLE_CONTIG_F32_F16_COPY \ + -u DS4_METAL_MODEL_UNTRACKED \ + DS4_METAL_UNRETAINED_COMMAND_BUFFERS=1 \ + ./tests/test_metal_q4_prefill_pair + tests/test_metal_indexer_q4.o: tests/test_metal_indexer_q4.c ds4_gpu.h $(CC) $(CFLAGS) -I. -c -o $@ $< @@ -330,6 +356,11 @@ speed-bench/metal_q4_dense_pair_bench: speed-bench/metal_q4_dense_pair_bench.m $ metal-q4-dense-pair-bench: speed-bench/metal_q4_dense_pair_bench +speed-bench/metal_q4_prefill_pair_bench: speed-bench/metal_q4_prefill_pair_bench.m $(METAL_SRCS) + $(CC) $(OBJCFLAGS) -o $@ $< $(METAL_LDLIBS) + +metal-q4-prefill-pair-bench: speed-bench/metal_q4_prefill_pair_bench + speed-bench/metal_q4_mm_tail_cull_bench: speed-bench/metal_q4_mm_tail_cull_bench.m $(METAL_SRCS) $(CC) $(OBJCFLAGS) -o $@ $< $(METAL_LDLIBS) @@ -970,4 +1001,4 @@ mxfp4-dot-test: tests/test_mxfp4_dot.c ./tests/test_mxfp4_dot clean: - rm -f ds4 ds4-server ds4-bench ds4-eval ds4-agent ds4_cpu ds4_native ds4_server_test ds4_test ds4_agent_test gguf-tools/quality-testing/score_official gguf-tools/quality-testing/score_official.o speed-bench/metal_decode_schedule_bench speed-bench/metal_prefill_variant_bench speed-bench/metal_q4_dense_pair_bench speed-bench/metal_q4_mm_tail_cull_bench speed-bench/metal_iq2_moe_tail_cull_bench speed-bench/gpu_iq2_moe_prefill_bench_rocm speed-bench/gpu_iq2_moe_prefill_bench_cuda speed-bench/rocm_q4_prefill_bench speed-bench/cuda_q4_prefill_bench speed-bench/*.o tests/test_q4k_dot tests/test_mxfp4_dot tests/test_quantizer_indexer_q4 tests/test_mxfp4_metal tests/test_mxfp4_rocm tests/bench_mxfp4_rocm tests/test_mxfp4_cuda tests/test_rocm_q4_dense_pair tests/test_metal_session_batch tests/test_metal_q4_streams tests/test_metal_indexer_q4 tests/test_metal_q4_attn_exactn tests/test_metal_q4_qb_f16_cache tests/test_metal_exactn_oracle tests/test_metal_dspark_capture tests/test_metal_argmax_top1 tests/test_metal_iq2_midonly tests/test_metal_iq2_ssd_grouped_mm tests/test_metal_iq2_live_index tests/test_glm53_kda tests/test_glm53_kda_rocm tests/test_glm53_vision_engine tests/test_glm53_vision_prompt tests/test_gpu_xdev tests/test_gpu_model_cache tests/test_gpu_lookup_cache_strict tests/test_engine_mgpu_refusal tests/test_engine_mgpu_runtime tests/test_engine_correctness tests/test_sampling tests/test_cuda_session_batch tests/test_cuda_mixed_batch tests/*.o *.o tests/cuda_long_context_smoke tests/cuda_long_context_smoke.o + rm -f ds4 ds4-server ds4-bench ds4-eval ds4-agent ds4_cpu ds4_native ds4_server_test ds4_test ds4_agent_test gguf-tools/quality-testing/score_official gguf-tools/quality-testing/score_official.o speed-bench/metal_decode_schedule_bench speed-bench/metal_prefill_variant_bench speed-bench/metal_q4_dense_pair_bench speed-bench/metal_q4_prefill_pair_bench speed-bench/metal_q4_mm_tail_cull_bench speed-bench/metal_iq2_moe_tail_cull_bench speed-bench/gpu_iq2_moe_prefill_bench_rocm speed-bench/gpu_iq2_moe_prefill_bench_cuda speed-bench/rocm_q4_prefill_bench speed-bench/cuda_q4_prefill_bench speed-bench/*.o tests/test_q4k_dot tests/test_mxfp4_dot tests/test_quantizer_indexer_q4 tests/test_mxfp4_metal tests/test_mxfp4_rocm tests/bench_mxfp4_rocm tests/test_mxfp4_cuda tests/test_rocm_q4_dense_pair tests/test_metal_session_batch tests/test_metal_q4_streams tests/test_metal_q4_prefill_pair tests/test_metal_indexer_q4 tests/test_metal_q4_attn_exactn tests/test_metal_q4_qb_f16_cache tests/test_metal_exactn_oracle tests/test_metal_dspark_capture tests/test_metal_argmax_top1 tests/test_metal_iq2_midonly tests/test_metal_iq2_ssd_grouped_mm tests/test_metal_iq2_live_index tests/test_glm53_kda tests/test_glm53_kda_rocm tests/test_glm53_vision_engine tests/test_glm53_vision_prompt tests/test_gpu_xdev tests/test_gpu_model_cache tests/test_gpu_lookup_cache_strict tests/test_engine_mgpu_refusal tests/test_engine_mgpu_runtime tests/test_engine_correctness tests/test_sampling tests/test_cuda_session_batch tests/test_cuda_mixed_batch tests/*.o *.o cuda/mmq/*.o cuda/mmq/test/*.o tests/cuda_long_context_smoke tests/cuda_long_context_smoke.o diff --git a/ds4.c b/ds4.c index e90624189..9661c2cd9 100644 --- a/ds4.c +++ b/ds4.c @@ -23959,7 +23959,7 @@ static bool metal_graph_encode_decode_layer_phase( if (!resume_after_qa_kv_raw && ok && qkv_rms_fused && qkv_proj_q4 && g->cuda_qkv_pair && !qkv_pair_projected && !metal_graph_use_reference_qkv_pair_proj()) { - qkv_pair_projected = ds4_gpu_matmul_q4_K_pair_tensor( + const int pair_rc = ds4_gpu_matmul_q4_K_pair_tensor( metal_graph_qr(g), metal_graph_kv_raw(g), model->map, @@ -23970,7 +23970,12 @@ static bool metal_graph_encode_decode_layer_phase( q_rank, DS4_N_HEAD_DIM, metal_graph_attn_norm(g), - 1) != 0; + 1); + if (pair_rc < 0) { + ok = false; + } else { + qkv_pair_projected = pair_rc > 0; + } } if (!resume_after_qa_kv_raw && ok && !qkv_pair_projected) { ok = metal_graph_matmul_dense_quant_tensor(metal_graph_qr(g), @@ -30177,12 +30182,17 @@ static bool metal_graph_encode_layer_attention_batch( !metal_graph_use_reference_qkv_pair_proj() && n_tokens >= 2u && layer->attn_q_a->type == DS4_TENSOR_Q4_K && layer->attn_kv->type == DS4_TENSOR_Q4_K) { - qkv_q4_pair_projected = ds4_gpu_matmul_q4_K_pair_tensor( + const int pair_rc = ds4_gpu_matmul_q4_K_pair_tensor( metal_graph_batch_qr(g), metal_graph_batch_kv_raw(g), model->map, model->size, layer->attn_q_a->abs_offset, layer->attn_kv->abs_offset, DS4_N_EMBD, q_rank, DS4_N_HEAD_DIM, - metal_graph_batch_attn_norm(g), n_tokens) != 0; + metal_graph_batch_attn_norm(g), n_tokens); + if (pair_rc < 0) { + ok = false; + } else { + qkv_q4_pair_projected = pair_rc > 0; + } } if (ok && !qkv_q4_pair_projected) { ok = metal_graph_matmul_q8_0_named_tensor("attn_q_a", diff --git a/ds4_gpu.h b/ds4_gpu.h index 20db38b70..173e3d98f 100644 --- a/ds4_gpu.h +++ b/ds4_gpu.h @@ -1125,8 +1125,8 @@ int ds4_gpu_matmul_q4_K_pair_decode_tensor( const ds4_gpu_tensor *x); /* Optional dense Q4_K pair for decode, microbatch, and prefill. Backends - * without a beneficial paired kernel return zero and request separate - * fallback matmuls from the graph. */ + * return 1 when the pair was encoded, 0 to request separate fallback + * matmuls from the graph, and -1 after a required/attempted-path error. */ int ds4_gpu_matmul_q4_K_pair_tensor( ds4_gpu_tensor *out0, ds4_gpu_tensor *out1, diff --git a/ds4_metal.m b/ds4_metal.m index a695e3698..92632e862 100644 --- a/ds4_metal.m +++ b/ds4_metal.m @@ -50,6 +50,15 @@ /* kernel_mul_mm_mpp_direct_rhs double-buffers two 64x32 half tiles. */ enum { DS4_METAL_MPP_DIRECT_RHS_SMEM = 2u * 64u * 32u * sizeof(uint16_t), + /* One fixed-capacity, stream-local sidecar avoids replacing a buffer that + * an unretained/in-flight command buffer may still reference. 8192 is + * above every currently supported q_a/KV input width; 128 is the largest + * exact token tile admitted by the experimental pair path. */ + DS4_METAL_Q4_PAIR_RHS_MAX_IN = 8192u, + DS4_METAL_Q4_PAIR_RHS_MAX_TOKENS = 128u, + DS4_METAL_Q4_PAIR_RHS_BYTES = + DS4_METAL_Q4_PAIR_RHS_MAX_IN * + DS4_METAL_Q4_PAIR_RHS_MAX_TOKENS * sizeof(uint16_t), }; @class DS4MetalQ4ExpertTable; @@ -386,6 +395,7 @@ __strong id argmax_top1_scratch_i; uint32_t argmax_top1_seq; __strong id indexed_topk_buffer; + __strong id q4_pair_rhs_f16_buffer; __strong id f16_round_scratch_buffer; __strong id raw_store_round_buffer; __strong id moe_gate_scratch_buffer; @@ -416,6 +426,7 @@ NSUInteger indexer_head_scores_bytes; NSUInteger indexer_topk_bytes; NSUInteger indexed_topk_bytes; + NSUInteger q4_pair_rhs_f16_bytes; NSUInteger f16_round_scratch_bytes; NSUInteger raw_store_round_bytes; NSUInteger moe_gate_scratch_bytes; @@ -452,6 +463,7 @@ #define g_indexer_head_scores_buffer DS4_STREAM_SCRATCH(indexer_head_scores_buffer) #define g_indexer_topk_buffer DS4_STREAM_SCRATCH(indexer_topk_buffer) #define g_indexed_topk_buffer DS4_STREAM_SCRATCH(indexed_topk_buffer) +#define g_q4_pair_rhs_f16_buffer DS4_STREAM_SCRATCH(q4_pair_rhs_f16_buffer) #define g_f16_round_scratch_buffer DS4_STREAM_SCRATCH(f16_round_scratch_buffer) #define g_raw_store_round_buffer DS4_STREAM_SCRATCH(raw_store_round_buffer) #define g_moe_gate_scratch_buffer DS4_STREAM_SCRATCH(moe_gate_scratch_buffer) @@ -482,6 +494,7 @@ #define g_indexer_head_scores_bytes DS4_STREAM_SCRATCH(indexer_head_scores_bytes) #define g_indexer_topk_bytes DS4_STREAM_SCRATCH(indexer_topk_bytes) #define g_indexed_topk_bytes DS4_STREAM_SCRATCH(indexed_topk_bytes) +#define g_q4_pair_rhs_f16_bytes DS4_STREAM_SCRATCH(q4_pair_rhs_f16_bytes) #define g_f16_round_scratch_bytes DS4_STREAM_SCRATCH(f16_round_scratch_bytes) #define g_raw_store_round_bytes DS4_STREAM_SCRATCH(raw_store_round_bytes) #define g_moe_gate_scratch_bytes DS4_STREAM_SCRATCH(moe_gate_scratch_bytes) @@ -4775,6 +4788,7 @@ void ds4_gpu_print_memory_report(const char *label) { uint64_t router = 0; uint64_t indexer = 0; uint64_t moe = 0; + uint64_t q4_pair_rhs = 0; uint64_t f16_round = 0; uint64_t raw_store = 0; for (int si = 0; si < DS4_GPU_MAX_STREAMS; si++) { @@ -4808,12 +4822,15 @@ void ds4_gpu_print_memory_report(const char *label) { (uint64_t)scratch_state->moe_q4_gate_slots_bytes + (uint64_t)scratch_state->moe_q4_up_slots_bytes + (uint64_t)scratch_state->moe_q4_down_slots_bytes; + q4_pair_rhs += + (uint64_t)scratch_state->q4_pair_rhs_f16_bytes; f16_round += (uint64_t)scratch_state->f16_round_scratch_bytes; raw_store += (uint64_t)scratch_state->raw_store_round_bytes; } const uint64_t scratch = flash_mask + flash_pad + flash_tmp + flash_blk + flash_ring + flash_kv + compressor + router + - indexer + moe + f16_round + raw_store; + indexer + moe + q4_pair_rhs + f16_round + + raw_store; pthread_mutex_lock(&g_tensor_mu); const uint64_t tensor_live_snap = g_tensor_alloc_live_bytes; @@ -5109,7 +5126,7 @@ void ds4_gpu_print_memory_report(const char *label) { (g_metal4_tensor_api_compile_supported ? "available" : "disabled"), g_metal4_m5_neural_accelerators_hint ? "likely" : "not detected"); fprintf(stderr, - "ds4: scratch %.2f MiB (flash mask %.2f, pad %.2f, tmp %.2f, blk %.2f, ring %.2f, kv %.2f, compressor %.2f, router %.2f, indexer %.2f, moe %.2f, f16 %.2f, raw-store %.2f)\n", + "ds4: scratch %.2f MiB (flash mask %.2f, pad %.2f, tmp %.2f, blk %.2f, ring %.2f, kv %.2f, compressor %.2f, router %.2f, indexer %.2f, moe %.2f, q4-pair-rhs %.2f, f16 %.2f, raw-store %.2f)\n", ds4_gpu_mib(scratch), ds4_gpu_mib(flash_mask), ds4_gpu_mib(flash_pad), @@ -5121,6 +5138,7 @@ void ds4_gpu_print_memory_report(const char *label) { ds4_gpu_mib(router), ds4_gpu_mib(indexer), ds4_gpu_mib(moe), + ds4_gpu_mib(q4_pair_rhs), ds4_gpu_mib(f16_round), ds4_gpu_mib(raw_store)); if (color) fputs(reset, stderr); @@ -5563,6 +5581,12 @@ static int ds4_gpu_encode_cpy_f32_f16_1d( NSUInteger dst_off, uint32_t n); +static bool ds4_gpu_tensor_prefixes_overlap( + const ds4_gpu_tensor *a, + uint64_t a_bytes, + const ds4_gpu_tensor *b, + uint64_t b_bytes); + static int ds4_gpu_encode_cpy_f32_f16_2d( id cb, id src, @@ -12026,6 +12050,7 @@ void ds4_gpu_cleanup(void) { g_stream_scratch[si].argmax_top1_scratch_i = nil; g_stream_scratch[si].argmax_top1_seq = 0u; g_indexed_topk_buffer = nil; + g_q4_pair_rhs_f16_buffer = nil; g_f16_round_scratch_buffer = nil; g_raw_store_round_buffer = nil; g_moe_gate_scratch_buffer = nil; @@ -12076,6 +12101,7 @@ void ds4_gpu_cleanup(void) { g_indexer_head_scores_bytes = 0; g_indexer_topk_bytes = 0; g_indexed_topk_bytes = 0; + g_q4_pair_rhs_f16_bytes = 0; g_f16_round_scratch_bytes = 0; g_raw_store_round_bytes = 0; g_moe_gate_scratch_bytes = 0; @@ -22733,18 +22759,195 @@ int ds4_gpu_matmul_q8_0_pair_tensor( return 1; } +/* Experimental q_a/KV prefill pair. The legacy M64xN32 Q4 kernel narrows + * every 32xK RHS tile once per 64-row output band. q_a and KV therefore + * replay the same F32->F16 conversion 24 times at the 1024+512 production + * shape. Materialize one stream-local half RHS and feed it to both unchanged + * 128-thread kernels instead. A fixed-capacity sidecar makes reuse safe for + * retained and unretained command-buffer modes; command-queue ordering plus + * tracked resource hazards serialize consecutive uses on one stream. */ +static int ds4_gpu_try_q4_K_prefill_pair_f16_rhs( + ds4_gpu_tensor *out0, ds4_gpu_tensor *out1, + const void *model_map, uint64_t model_size, + uint64_t weight0_offset, uint64_t weight1_offset, + uint64_t in_dim, uint64_t out0_dim, uint64_t out1_dim, + const ds4_gpu_tensor *x, uint64_t n_tok) { + const bool exact_tokens = + n_tok >= 32u && n_tok <= DS4_METAL_Q4_PAIR_RHS_MAX_TOKENS && + (n_tok % 32u) == 0u; + if (!out0 || !out1 || !model_map || !x || !exact_tokens || + !ds4_gpu_device_is_pre_m5_apple_silicon() || + g_batch_encoder_concurrent || + g_ds4_stream < 0 || g_ds4_stream >= DS4_GPU_MAX_STREAMS || + in_dim == 0u || in_dim > DS4_METAL_Q4_PAIR_RHS_MAX_IN || + (in_dim % 256u) != 0u || + out0_dim == 0u || out1_dim == 0u || + out0_dim > INT32_MAX || out1_dim > INT32_MAX || + (out0_dim % 64u) != 0u || (out1_dim % 64u) != 0u || + ds4_gpu_env_bool("DS4_METAL_DISABLE_CONTIG_F32_F16_COPY") == 1) { + return 0; + } + + const uint64_t row_bytes = (in_dim / 256u) * 144u; + if (out0_dim > UINT64_MAX / row_bytes || + out1_dim > UINT64_MAX / row_bytes || + in_dim > UINT64_MAX / n_tok / sizeof(float) || + out0_dim > UINT64_MAX / n_tok / sizeof(float) || + out1_dim > UINT64_MAX / n_tok / sizeof(float)) { + return 0; + } + const uint64_t w0_bytes = out0_dim * row_bytes; + const uint64_t w1_bytes = out1_dim * row_bytes; + const uint64_t x_bytes = in_dim * n_tok * sizeof(float); + const uint64_t o0_bytes = out0_dim * n_tok * sizeof(float); + const uint64_t o1_bytes = out1_dim * n_tok * sizeof(float); + if (weight0_offset > model_size || w0_bytes > model_size - weight0_offset || + weight1_offset > model_size || w1_bytes > model_size - weight1_offset || + ds4_gpu_tensor_bytes(x) < x_bytes || + ds4_gpu_tensor_bytes(out0) < o0_bytes || + ds4_gpu_tensor_bytes(out1) < o1_bytes || + ds4_gpu_tensor_prefixes_overlap(x, x_bytes, out0, o0_bytes) || + ds4_gpu_tensor_prefixes_overlap(x, x_bytes, out1, o1_bytes) || + ds4_gpu_tensor_prefixes_overlap(out0, o0_bytes, out1, o1_bytes)) { + return 0; + } + + @autoreleasepool { + uint64_t inner0 = 0u; + uint64_t inner1 = 0u; + id w0 = ds4_gpu_wrap_model_range( + model_map, model_size, weight0_offset, w0_bytes, &inner0); + id w1 = ds4_gpu_wrap_model_range( + model_map, model_size, weight1_offset, w1_bytes, &inner1); + id xb = ds4_gpu_tensor_buffer(x); + id o0 = ds4_gpu_tensor_buffer(out0); + id o1 = ds4_gpu_tensor_buffer(out1); + id mm_pipeline = + ds4_gpu_get_mul_mm_pipeline( + "kernel_mul_mm_q4_K_f16_rhs", false, false); + if (!w0 || !w1 || !xb || !o0 || !o1 || + inner0 > NSUIntegerMax || inner1 > NSUIntegerMax || + !g_cpy_contig_f32_f16_pipeline || !mm_pipeline || + mm_pipeline.threadExecutionWidth != 32u || + mm_pipeline.maxTotalThreadsPerThreadgroup < 128u || + g_cpy_contig_f32_f16_pipeline.maxTotalThreadsPerThreadgroup == 0u || + !ds4_gpu_ensure_scratch_buffer( + &g_q4_pair_rhs_f16_buffer, + &g_q4_pair_rhs_f16_bytes, + DS4_METAL_Q4_PAIR_RHS_BYTES, + "ds4_q4_pair_rhs_f16")) { + return 0; + } + id rhs_f16 = g_q4_pair_rhs_f16_buffer; + if (!rhs_f16) return 0; + + ds4_gpu_mul_mm_args args0 = + ds4_gpu_make_mm_args(in_dim, out0_dim, n_tok, row_bytes); + ds4_gpu_mul_mm_args args1 = + ds4_gpu_make_mm_args(in_dim, out1_dim, n_tok, row_bytes); + const uint64_t rhs_row_bytes = in_dim * sizeof(uint16_t); + const uint64_t rhs_bytes = rhs_row_bytes * n_tok; + args0.nb10 = args1.nb10 = sizeof(uint16_t); + args0.nb11 = args1.nb11 = rhs_row_bytes; + args0.nb12 = args0.nb13 = rhs_bytes; + args1.nb12 = args1.nb13 = rhs_bytes; + + int owned = 0; + id cb = ds4_gpu_command_buffer(&owned); + if (!cb) return 0; + + /* All fallible resource/PSO work above happens before this first + * dispatch. The copy touches only scratch, so the graph can still use + * its normal output fallback if encoder creation fails. */ + if (!ds4_gpu_encode_cpy_f32_f16_1d( + cb, xb, ds4_gpu_tensor_offset(x), rhs_f16, 0u, + (uint32_t)(in_dim * n_tok))) { + return 0; + } + if (!owned) ds4_gpu_close_batch_encoder(); + + /* Ending the copy encoder establishes an explicit producer/consumer + * boundary. Both output dispatches are then encoded without another + * fallible host operation between them. */ + id enc = ds4_gpu_compute_encoder(cb); + if (!enc) return 0; + [enc setComputePipelineState:mm_pipeline]; + [enc setThreadgroupMemoryLength:6144u atIndex:0]; + + [enc setBytes:&args0 length:sizeof(args0) atIndex:0]; + [enc setBuffer:w0 offset:(NSUInteger)inner0 atIndex:1]; + [enc setBuffer:rhs_f16 offset:0 atIndex:2]; + [enc setBuffer:o0 offset:ds4_gpu_tensor_offset(out0) atIndex:3]; + [enc dispatchThreadgroups:MTLSizeMake( + (NSUInteger)n_tok / 32u, + (NSUInteger)out0_dim / 64u, + 1u) + threadsPerThreadgroup:MTLSizeMake(128u, 1u, 1u)]; + + [enc setBytes:&args1 length:sizeof(args1) atIndex:0]; + [enc setBuffer:w1 offset:(NSUInteger)inner1 atIndex:1]; + [enc setBuffer:o1 offset:ds4_gpu_tensor_offset(out1) atIndex:3]; + [enc dispatchThreadgroups:MTLSizeMake( + (NSUInteger)n_tok / 32u, + (NSUInteger)out1_dim / 64u, + 1u) + threadsPerThreadgroup:MTLSizeMake(128u, 1u, 1u)]; + ds4_gpu_end_compute_encoder(cb, enc); + + return ds4_gpu_finish_command_buffer( + cb, owned, "paired Q4_K prefill F16 RHS") ? 1 : -1; + } +} + int ds4_gpu_matmul_q4_K_pair_tensor( ds4_gpu_tensor *out0, ds4_gpu_tensor *out1, const void *model_map, uint64_t model_size, uint64_t weight0_offset, uint64_t weight1_offset, uint64_t in_dim, uint64_t out0_dim, uint64_t out1_dim, const ds4_gpu_tensor *x, uint64_t n_tok) { - if (!g_initialized && !ds4_gpu_init()) return 0; + const int pair_enable = + ds4_gpu_env_bool("DS4_METAL_ENABLE_Q4_PREFILL_PAIR_F16_RHS"); + const int pair_disable = + ds4_gpu_env_bool("DS4_METAL_DISABLE_Q4_PREFILL_PAIR_F16_RHS"); + const int pair_require = + ds4_gpu_env_bool("DS4_METAL_REQUIRE_Q4_PREFILL_PAIR_F16_RHS"); + const bool prefill_pair_call = n_tok >= 32u; + const bool global_pair_disabled = + getenv("DS4_METAL_DISABLE_Q4_DENSE_PAIR") != NULL; + const bool pair_requested = + !global_pair_disabled && pair_disable != 1 && + (pair_enable == 1 || pair_require == 1); + + if (!g_initialized && !ds4_gpu_init()) { + return pair_require == 1 && prefill_pair_call ? -1 : 0; + } + if (pair_requested) { + const int candidate = ds4_gpu_try_q4_K_prefill_pair_f16_rhs( + out0, out1, model_map, model_size, + weight0_offset, weight1_offset, + in_dim, out0_dim, out1_dim, x, n_tok); + if (candidate > 0) return 1; + if (candidate < 0) return -1; + } + if (pair_require == 1 && prefill_pair_call) { + fprintf(stderr, + "ds4: Metal Q4 prefill pair F16-RHS required but unavailable " + "(disabled=%d global_pair_disabled=%d pre_m5=%d " + "in=%llu out0=%llu out1=%llu tokens=%llu)\n", + pair_disable == 1, + global_pair_disabled, + ds4_gpu_device_is_pre_m5_apple_silicon(), + (unsigned long long)in_dim, + (unsigned long long)out0_dim, + (unsigned long long)out1_dim, + (unsigned long long)n_tok); + return -1; + } if (!out0 || !out1 || !model_map || !x || n_tok == 0 || n_tok > 8u || in_dim == 0 || (in_dim % 256u) != 0 || in_dim > UINT32_MAX || out0_dim == 0 || out1_dim == 0 || out0_dim > UINT32_MAX || out1_dim > UINT32_MAX || - getenv("DS4_METAL_DISABLE_Q4_DENSE_PAIR") != NULL) { + global_pair_disabled) { return 0; } @autoreleasepool { @@ -22788,6 +22991,7 @@ int ds4_gpu_matmul_q4_K_pair_tensor( id cb = ds4_gpu_command_buffer(&owned); if (!cb) return 0; id enc = ds4_gpu_compute_encoder(cb); + if (!enc) return 0; [enc setComputePipelineState:pipeline]; [enc setBytes:&args0 length:sizeof(args0) atIndex:0]; [enc setBytes:&args1 length:sizeof(args1) atIndex:1]; @@ -22801,7 +23005,8 @@ int ds4_gpu_matmul_q4_K_pair_tensor( [enc dispatchThreadgroups:MTLSizeMake((max_out + 3u) / 4u, n_tok, 1) threadsPerThreadgroup:MTLSizeMake(32, nsg, 1)]; ds4_gpu_end_compute_encoder(cb, enc); - return ds4_gpu_finish_command_buffer(cb, owned, "paired Q4_K matvec"); + return ds4_gpu_finish_command_buffer( + cb, owned, "paired Q4_K matvec") ? 1 : -1; } } diff --git a/speed-bench/.gitignore b/speed-bench/.gitignore index f84a32acc..334d0d53e 100644 --- a/speed-bench/.gitignore +++ b/speed-bench/.gitignore @@ -4,4 +4,5 @@ local-runs/ metal_decode_schedule_bench metal_prefill_variant_bench metal_q4_dense_pair_bench +metal_q4_prefill_pair_bench metal_q4_mm_tail_cull_bench diff --git a/speed-bench/metal_q4_prefill_pair_bench.m b/speed-bench/metal_q4_prefill_pair_bench.m new file mode 100644 index 000000000..50e8be9b3 --- /dev/null +++ b/speed-bench/metal_q4_prefill_pair_bench.m @@ -0,0 +1,405 @@ +#import +#import + +#include +#include +#include +#include +#include +#include + +enum { + IN_DIM = 4096, + OUT0_DIM = 1024, + OUT1_DIM = 512, + MAX_TOKENS = 128, + QK_K = 256, + Q4_BLOCK_BYTES = 144, + THREADS = 128, + SMEM = 6144, + GUARD_BYTES = 256, +}; + +static const uint32_t k_guard = 0x7fc12345u; +static const uint32_t k_baseline_poison = 0x7fc0b001u; +static const uint32_t k_pair_poison = 0x7fc0c001u; +static const uint32_t k_tokens[] = {32u, 64u, 96u, 128u}; + +typedef struct { + uint16_t d, dmin; + uint8_t scales[12]; + uint8_t qs[QK_K / 2]; +} block_q4_K_host; + +typedef struct { + int32_t ne00, ne02; + uint64_t nb01, nb02, nb03; + int32_t ne12; + uint64_t nb10, nb11, nb12, nb13; + int32_t ne0, ne1; + int16_t r2, r3; +} mul_mm_args; + +_Static_assert(sizeof(block_q4_K_host) == Q4_BLOCK_BYTES, "Q4_K ABI"); +_Static_assert(sizeof(mul_mm_args) == 88, "mul_mm ABI"); + +typedef struct { + uint32_t sets, dispatches, warmup, samples; +} config; + +typedef enum { ARM_BASELINE, ARM_PAIR_F16 } arm; + +typedef struct { + __strong id device; + __strong id queue; + __strong id mm_f32; + __strong id mm_f16; + __strong id copy; + __strong id w0, w1, x, rhs; + __strong id base0, base1, pair0, pair1; + NSUInteger w0_set_bytes, w1_set_bytes; + NSUInteger x_off, rhs_off, out0_stride, out1_stride; + uint32_t sets; + uint8_t *x_snapshot; +} fixture; + +static void usage(const char *argv0) { + printf("usage: %s [--sets N] [--dispatches N] [--warmup N] [--samples N]\n", + argv0); +} + +static uint32_t parse_u32(const char *s, const char *name, uint32_t min) { + char *end = NULL; + errno = 0; + unsigned long long v = strtoull(s, &end, 10); + if (errno || !s[0] || !end || *end || v < min || v > UINT32_MAX) { + fprintf(stderr, "metal-q4-prefill-pair-bench: invalid %s: %s\n", name, s); + exit(2); + } + return (uint32_t)v; +} + +static const char *arg_value(int *i, int argc, char **argv) { + if (*i + 1 >= argc) { + fprintf(stderr, "metal-q4-prefill-pair-bench: %s needs a value\n", argv[*i]); + exit(2); + } + return argv[++*i]; +} + +static config parse_options(int argc, char **argv) { + config c = {.sets=64u, .dispatches=32u, .warmup=8u, .samples=12u}; + for (int i = 1; i < argc; i++) { + if (!strcmp(argv[i], "-h") || !strcmp(argv[i], "--help")) { + usage(argv[0]); exit(0); + } else if (!strcmp(argv[i], "--sets")) { + c.sets = parse_u32(arg_value(&i, argc, argv), "--sets", 1u); + } else if (!strcmp(argv[i], "--dispatches")) { + c.dispatches = parse_u32(arg_value(&i, argc, argv), "--dispatches", 1u); + } else if (!strcmp(argv[i], "--warmup")) { + c.warmup = parse_u32(arg_value(&i, argc, argv), "--warmup", 0u); + } else if (!strcmp(argv[i], "--samples")) { + c.samples = parse_u32(arg_value(&i, argc, argv), "--samples", 2u); + } else { + usage(argv[0]); exit(2); + } + } + if (c.samples & 1u) { + fprintf(stderr, "metal-q4-prefill-pair-bench: --samples must be even\n"); + exit(2); + } + return c; +} + +static NSString *prelude(void) { + return @"#include \nusing namespace metal;\n" + "#define MAX(x,y) ((x)>(y)?(x):(y))\n" + "#define MIN(x,y) ((x)<(y)?(x):(y))\n" + "#define SWAP(x,y) { auto t=(x); (x)=(y); (y)=t; }\n" + "#define QK8_0 32\n#ifndef QK_K\n#define QK_K 256\n#endif\n" + "#define N_SIMDWIDTH 32\n#define N_R0_Q8_0 2\n#define N_SG_Q8_0 4\n" + "#define FC_MUL_MV 600\n#define FC_MUL_MM 700\n#define FC_BIN 1300\n" + "#define FOR_UNROLL(x) _Pragma(\"clang loop unroll(full)\") for (x)\n" + "#define M_PI_F 3.14159265358979323846f\n" + "enum ds4_sort_order { DS4_SORT_ORDER_ASC, DS4_SORT_ORDER_DESC };\n" + "struct block_q8_0 { half d; int8_t qs[QK8_0]; };\n" + "struct block_q8_K { float d; int8_t qs[QK_K]; int16_t bsums[QK_K/16]; };\n"; +} + +static NSString *load_source(void) { + static const char *files[] = { + "metal/activations.metal", "metal/flash_attn.metal", "metal/dense.metal", + "metal/moe.metal", "metal/dsv4_hc.metal", "metal/unary.metal", + "metal/dsv4_kv.metal", "metal/dsv4_rope.metal", "metal/dsv4_misc.metal", + "metal/argsort.metal", "metal/cpy.metal", "metal/concat.metal", + "metal/get_rows.metal", "metal/sum_rows.metal", "metal/softmax.metal", + "metal/repeat.metal", "metal/glu.metal", "metal/norm.metal", + "metal/bin.metal", "metal/set_rows.metal", + }; + NSString *root = getenv("DS4_SOURCE_ROOT") + ? [NSString stringWithUTF8String:getenv("DS4_SOURCE_ROOT")] : @"."; + NSMutableString *s = [NSMutableString stringWithString:prelude()]; + for (size_t i = 0; i < sizeof(files)/sizeof(files[0]); i++) { + NSString *p = [root stringByAppendingPathComponent: + [NSString stringWithUTF8String:files[i]]]; + NSError *e = nil; + NSString *part = [NSString stringWithContentsOfFile:p + encoding:NSUTF8StringEncoding error:&e]; + if (!part) { + fprintf(stderr, "metal-q4-prefill-pair-bench: read %s: %s\n", + files[i], [[e localizedDescription] UTF8String]); + return nil; + } + [s appendFormat:@"\n%@\n", part]; + } + return s; +} + +static id pipeline(id d, + id l, NSString *name, bool mm) { + NSError *e = nil; + id fn = nil; + if (mm) { + bool no = false; + MTLFunctionConstantValues *v = [MTLFunctionConstantValues new]; + [v setConstantValue:&no type:MTLDataTypeBool atIndex:700]; + [v setConstantValue:&no type:MTLDataTypeBool atIndex:701]; + fn = [l newFunctionWithName:name constantValues:v error:&e]; + } else { + fn = [l newFunctionWithName:name]; + } + if (!fn) { + fprintf(stderr, "metal-q4-prefill-pair-bench: function %s: %s\n", + [name UTF8String], e ? [[e localizedDescription] UTF8String] : "missing"); + return nil; + } + id p = [d newComputePipelineStateWithFunction:fn error:&e]; + if (!p) fprintf(stderr, "metal-q4-prefill-pair-bench: pipeline %s: %s\n", + [name UTF8String], [[e localizedDescription] UTF8String]); + return p; +} + +static uint32_t rng(uint32_t *s) { *s = *s * 1664525u + 1013904223u; return *s; } +static void fill_q4(void *ptr, NSUInteger bytes, uint32_t seed) { + block_q4_K_host *b = ptr; + for (NSUInteger i = 0; i < bytes / sizeof(*b); i++) { + b[i].d = (uint16_t)(0x2400u | (rng(&seed) & 0x3ffu)); + b[i].dmin = (uint16_t)(0x1c00u | (rng(&seed) & 0x3ffu)); + for (size_t j=0;j>24); + for (size_t j=0;j>24); + } +} +static void fill_guard(id b) { + uint32_t *p=b.contents; for (NSUInteger i=0;i b, NSUInteger stride, uint32_t sets, + NSUInteger bytes, uint32_t poison) { + for (uint32_t s = 0; s < sets; s++) { + uint32_t *p = (uint32_t *)((uint8_t *)b.contents + + (NSUInteger)s * stride + GUARD_BYTES); + for (NSUInteger i = 0; i < bytes / sizeof(*p); i++) p[i] = poison; + } +} +static void fill_x(float *x) { + uint32_t s=0x243f6a88u; + for (NSUInteger i=0;i<(NSUInteger)IN_DIM*MAX_TOKENS;i++) + x[i]=((int32_t)(rng(&s)&0xffffu)-32768)/32768.0f; +} +static NSUInteger align_up(NSUInteger x, NSUInteger a) { return (x+a-1)/a*a; } +static id alloc(id d, NSUInteger n, NSString *label) { + if (!n || n>d.maxBufferLength) return nil; + id b=[d newBufferWithLength:n options:MTLResourceStorageModeShared]; + b.label=label; return b; +} + +static bool is_pre_m5_apple_silicon_name(const char *name) { + return name && !strncmp(name, "Apple M", 7) && + name[7] >= '1' && name[7] <= '4' && + (name[8] == '\0' || name[8] == ' '); +} + +static bool init_fixture(fixture *f, const config *c, id device) { + f->device=device; + f->queue=[f->device newCommandQueue]; + NSString *src=load_source(); if (!f->device || !f->queue || !src) return false; + NSError *e=nil; + id l=[f->device newLibraryWithSource:src options:nil error:&e]; + if (!l) { fprintf(stderr,"metal-q4-prefill-pair-bench: compile: %s\n", + [[e localizedDescription] UTF8String]); return false; } + f->mm_f32=pipeline(f->device,l,@"kernel_mul_mm_q4_K_f32",true); + f->mm_f16=pipeline(f->device,l,@"kernel_mul_mm_q4_K_f16_rhs",true); + f->copy=pipeline(f->device,l,@"kernel_cpy_contig_f32_f16_4",false); + if (!f->mm_f32 || !f->mm_f16 || !f->copy || + f->mm_f32.maxTotalThreadsPerThreadgroupmm_f16.maxTotalThreadsPerThreadgroupsets=c->sets; + f->w0_set_bytes=(IN_DIM/QK_K)*Q4_BLOCK_BYTES*OUT0_DIM; + f->w1_set_bytes=(IN_DIM/QK_K)*Q4_BLOCK_BYTES*OUT1_DIM; + f->x_off=f->rhs_off=GUARD_BYTES; + f->out0_stride=align_up(GUARD_BYTES+(NSUInteger)MAX_TOKENS*OUT0_DIM*4+GUARD_BYTES,256); + f->out1_stride=align_up(GUARD_BYTES+(NSUInteger)MAX_TOKENS*OUT1_DIM*4+GUARD_BYTES,256); + f->w0=alloc(f->device,f->w0_set_bytes*c->sets,@"pair-w0"); + f->w1=alloc(f->device,f->w1_set_bytes*c->sets,@"pair-w1"); + f->x=alloc(f->device,GUARD_BYTES+(NSUInteger)MAX_TOKENS*IN_DIM*4+GUARD_BYTES,@"pair-x"); + f->rhs=alloc(f->device,GUARD_BYTES+(NSUInteger)MAX_TOKENS*IN_DIM*2+GUARD_BYTES,@"pair-rhs"); + f->base0=alloc(f->device,f->out0_stride*c->sets,@"base0"); + f->base1=alloc(f->device,f->out1_stride*c->sets,@"base1"); + f->pair0=alloc(f->device,f->out0_stride*c->sets,@"pair0"); + f->pair1=alloc(f->device,f->out1_stride*c->sets,@"pair1"); + if (!f->w0||!f->w1||!f->x||!f->rhs||!f->base0||!f->base1||!f->pair0||!f->pair1) return false; + fill_q4(f->w0.contents,f->w0.length,0x41c64e6du); + fill_q4(f->w1.contents,f->w1.length,0x9e3779b9u); + fill_guard(f->x); fill_x((float *)((uint8_t *)f->x.contents+f->x_off)); + f->x_snapshot=malloc(f->x.length); if (!f->x_snapshot) return false; + memcpy(f->x_snapshot,f->x.contents,f->x.length); + return true; +} + +static mul_mm_args args(uint32_t out, uint32_t n, bool half_rhs) { + uint64_t row=(IN_DIM/QK_K)*Q4_BLOCK_BYTES, elem=half_rhs?2u:4u; + return (mul_mm_args){.ne00=IN_DIM,.ne02=1,.nb01=row,.nb02=row*out,.nb03=row*out, + .ne12=1,.nb10=elem,.nb11=(uint64_t)IN_DIM*elem, + .nb12=(uint64_t)IN_DIM*n*elem,.nb13=(uint64_t)IN_DIM*n*elem, + .ne0=(int32_t)out,.ne1=(int32_t)n,.r2=1,.r3=1}; +} + +static void encode_mm(fixture *f,id e,bool half, + uint32_t n,uint32_t set,id o0,id o1) { + mul_mm_args a0=args(OUT0_DIM,n,half), a1=args(OUT1_DIM,n,half); + [e setComputePipelineState:half?f->mm_f16:f->mm_f32]; + [e setThreadgroupMemoryLength:SMEM atIndex:0]; + [e setBytes:&a0 length:sizeof(a0) atIndex:0]; + [e setBuffer:f->w0 offset:set*f->w0_set_bytes atIndex:1]; + [e setBuffer:half?f->rhs:f->x offset:half?f->rhs_off:f->x_off atIndex:2]; + [e setBuffer:o0 offset:set*f->out0_stride+GUARD_BYTES atIndex:3]; + [e dispatchThreadgroups:MTLSizeMake(n/32,OUT0_DIM/64,1) + threadsPerThreadgroup:MTLSizeMake(THREADS,1,1)]; + [e setBytes:&a1 length:sizeof(a1) atIndex:0]; + [e setBuffer:f->w1 offset:set*f->w1_set_bytes atIndex:1]; + [e setBuffer:o1 offset:set*f->out1_stride+GUARD_BYTES atIndex:3]; + [e dispatchThreadgroups:MTLSizeMake(n/32,OUT1_DIM/64,1) + threadsPerThreadgroup:MTLSizeMake(THREADS,1,1)]; +} + +static void encode_copy(fixture *f,id e,uint32_t n) { + uint32_t elems=n*IN_DIM; + NSUInteger work=(elems+3u)/4u, nth=256u; + if (nth>f->copy.maxTotalThreadsPerThreadgroup) nth=f->copy.maxTotalThreadsPerThreadgroup; + [e setComputePipelineState:f->copy]; [e setBytes:&elems length:4 atIndex:0]; + [e setBuffer:f->x offset:f->x_off atIndex:1]; [e setBuffer:f->rhs offset:f->rhs_off atIndex:2]; + [e dispatchThreadgroups:MTLSizeMake((work+nth-1)/nth,1,1) + threadsPerThreadgroup:MTLSizeMake(nth,1,1)]; +} + +static bool finish(id cb,const char *label,double *secs) { + [cb commit]; [cb waitUntilCompleted]; + if (cb.status!=MTLCommandBufferStatusCompleted) { + fprintf(stderr,"metal-q4-prefill-pair-bench: %s: %s\n",label, + cb.error?[[cb.error localizedDescription] UTF8String]:"failed"); return false; + } + double t=cb.GPUEndTime-cb.GPUStartTime; if (!(t>0)) return false; + if (secs) *secs=t; return true; +} + +static bool workload(fixture *f,arm a,uint32_t n,uint32_t calls,uint32_t start,double *secs) { + id cb=[f->queue commandBuffer]; + if (a==ARM_BASELINE) { + id e=[cb computeCommandEncoder]; + for(uint32_t i=0;isets; + encode_mm(f,e,false,n,s,f->base0,f->base1); } + [e endEncoding]; + } else { + for(uint32_t i=0;isets; + id c=[cb computeCommandEncoder]; encode_copy(f,c,n); [c endEncoding]; + id m=[cb computeCommandEncoder]; encode_mm(f,m,true,n,s,f->pair0,f->pair1); [m endEncoding]; + } + } + return finish(cb,a==ARM_BASELINE?"baseline":"pair-f16",secs); +} + +static bool guards(fixture *f,id b,NSUInteger stride,NSUInteger payload,const char *name) { + uint32_t *p=b.contents; + for(uint32_t s=0;ssets;s++) { + NSUInteger base=s*stride, begin=base+GUARD_BYTES, end=begin+payload; + for(NSUInteger x=base;xrhs); fill_guard(f->base0); fill_guard(f->base1); fill_guard(f->pair0); fill_guard(f->pair1); + NSUInteger b0=(NSUInteger)n*OUT0_DIM*4, b1=(NSUInteger)n*OUT1_DIM*4; + fill_payload(f->base0, f->out0_stride, f->sets, b0, k_baseline_poison); + fill_payload(f->base1, f->out1_stride, f->sets, b1, k_baseline_poison); + fill_payload(f->pair0, f->out0_stride, f->sets, b0, k_pair_poison); + fill_payload(f->pair1, f->out1_stride, f->sets, b1, k_pair_poison); + double ignored; + if (!workload(f,ARM_BASELINE,n,f->sets,0,&ignored) || + !workload(f,ARM_PAIR_F16,n,f->sets,0,&ignored)) return false; + for(uint32_t s=0;ssets;s++) { + uint8_t *a0=(uint8_t *)f->base0.contents+s*f->out0_stride+GUARD_BYTES; + uint8_t *p0=(uint8_t *)f->pair0.contents+s*f->out0_stride+GUARD_BYTES; + uint8_t *a1=(uint8_t *)f->base1.contents+s*f->out1_stride+GUARD_BYTES; + uint8_t *p1=(uint8_t *)f->pair1.contents+s*f->out1_stride+GUARD_BYTES; + if(memcmp(a0,p0,b0)||memcmp(a1,p1,b1)) { fprintf(stderr,"mismatch N=%u set=%u\n",n,s); return false; } + } + if(memcmp(f->x.contents,f->x_snapshot,f->x.length)) return false; + uint32_t *r=f->rhs.contents; + for(NSUInteger x=0;xrhs_off;x+=4) if(r[x/4]!=k_guard) return false; + for(NSUInteger x=f->rhs_off+(NSUInteger)n*IN_DIM*2;xrhs.length;x+=4) if(r[x/4]!=k_guard) return false; + if(!guards(f,f->base0,f->out0_stride,b0,"base0")||!guards(f,f->base1,f->out1_stride,b1,"base1")|| + !guards(f,f->pair0,f->out0_stride,b0,"pair0")||!guards(f,f->pair1,f->out1_stride,b1,"pair1")) return false; + fprintf(stderr,"Metal Q4 prefill pair oracle N=%u: PASS (bit-exact, canaries intact)\n",n); + return true; +} + +static int cmp(const void *a,const void *b){double x=*(double*)a,y=*(double*)b;return(x>y)-(xwarmup;i++) if(!workload(f,(i==0||i==3)?ARM_BASELINE:ARM_PAIR_F16,n,c->warmup,i*c->warmup,&tmp)) return false; + double *b=calloc(c->samples,sizeof(*b)),*p=calloc(c->samples,sizeof(*p)); if(!b||!p) return false; + uint32_t nb=0,np=0; + for(uint32_t cyc=0;cycsamples/2;cyc++) { + arm abba[]={ARM_BASELINE,ARM_PAIR_F16,ARM_PAIR_F16,ARM_BASELINE}; + arm baab[]={ARM_PAIR_F16,ARM_BASELINE,ARM_BASELINE,ARM_PAIR_F16}; arm *order=cyc&1?baab:abba; + for(int j=0;j<4;j++) { arm a=order[j]; double t; + uint32_t idx=a==ARM_BASELINE?nb:np; + if(!workload(f,a,n,c->dispatches,(uint64_t)idx*c->dispatches%f->sets,&t)){free(b);free(p);return false;} + if(a==ARM_BASELINE)b[nb++]=t;else p[np++]=t; + } + } + double bm=median(b,c->samples)*1e6/c->dispatches, pm=median(p,c->samples)*1e6/c->dispatches; + printf(" N=%-3u baseline 2xQ4/F32 %.3f us | pair copy+2xQ4/F16 %.3f us | %+.2f%%\n", + n,bm,pm,(bm-pm)*100.0/bm); free(b);free(p); return true; +} + +int main(int argc,char **argv) { @autoreleasepool { + config c=parse_options(argc,argv); + id device=MTLCreateSystemDefaultDevice(); + if(!device) { + fprintf(stderr,"metal-q4-prefill-pair-bench: SKIP: no Metal device\n"); + return 0; + } + const char *device_name=[[device name] UTF8String]; + fprintf(stderr,"metal-q4-prefill-pair-bench: device=%s\n", + device_name?device_name:"unknown"); + if(!is_pre_m5_apple_silicon_name(device_name)) { + fprintf(stderr, + "metal-q4-prefill-pair-bench: SKIP: runtime candidate requires Apple M1-M4\n"); + return 0; + } + fixture f={0}; if(!init_fixture(&f,&c,device)) return 1; + printf("Metal Q4 q_a/KV prefill pair, resident GPU-event-only\n"); + printf(" 4096->1024 + 4096->512, %.1f MiB weights, %u sets, %u calls/sample\n", + (double)(f.w0.length+f.w1.length)/1048576.0,c.sets,c.dispatches); + for(size_t i=0;i +#include +#include +#include +#include +#include + +#ifdef __APPLE__ + +/* Standalone Metal tests link the backend without ds4.c. */ +bool ds4_log_is_tty(FILE *fp) { + (void)fp; + return false; +} + +enum { + Q4_K_TYPE = 12u, + QK_K = 256u, + IN_DIM = 4096u, + OUT0_DIM = 1024u, + OUT1_DIM = 512u, + MAX_TOKENS = 128u, + TEST_STREAMS = 2u, +}; + +static const uint32_t k_tokens[] = {32u, 64u, 96u, 128u}; +static const float k_poison = -12345.25f; + +typedef struct { + uint16_t d; + uint16_t dmin; + uint8_t scales[12]; + uint8_t qs[QK_K / 2u]; +} block_q4_K; + +_Static_assert(sizeof(block_q4_K) == 144u, "Q4_K ABI"); + +typedef struct { + void *model; + uint64_t model_size; + uint64_t row_bytes; + uint64_t weight_offset[2]; + uint64_t weight_bytes[2]; + ds4_gpu_tensor *x[TEST_STREAMS]; + ds4_gpu_tensor *baseline[TEST_STREAMS][2]; + ds4_gpu_tensor *candidate[TEST_STREAMS][2]; + float *x_host[TEST_STREAMS]; +} fixture; + +static void fail(const char *what) { + fprintf(stderr, "Metal Q4 prefill pair runtime FAIL: %s\n", what); + exit(1); +} + +static void set_flag(const char *name, bool enabled) { + const int rc = enabled ? setenv(name, "1", 1) : unsetenv(name); + if (rc != 0) fail("environment update"); +} + +static void set_pair_controls(bool enabled, bool disabled, bool required) { + set_flag("DS4_METAL_ENABLE_Q4_PREFILL_PAIR_F16_RHS", enabled); + set_flag("DS4_METAL_DISABLE_Q4_PREFILL_PAIR_F16_RHS", disabled); + set_flag("DS4_METAL_REQUIRE_Q4_PREFILL_PAIR_F16_RHS", required); +} + +static uint64_t align_up(uint64_t value, uint64_t alignment) { + return (value + alignment - 1u) / alignment * alignment; +} + +static uint32_t lcg(uint32_t *state) { + *state = *state * 1664525u + 1013904223u; + return *state; +} + +static void fill_q4(block_q4_K *blocks, uint64_t count, uint32_t seed) { + for (uint64_t i = 0; i < count; i++) { + blocks[i].d = (uint16_t)(0x2400u | (lcg(&seed) & 0x03ffu)); + blocks[i].dmin = + (uint16_t)(0x1c00u | (lcg(&seed) & 0x03ffu)); + for (uint32_t j = 0; j < sizeof(blocks[i].scales); j++) { + blocks[i].scales[j] = (uint8_t)(lcg(&seed) >> 24u); + } + for (uint32_t j = 0; j < sizeof(blocks[i].qs); j++) { + blocks[i].qs[j] = (uint8_t)(lcg(&seed) >> 24u); + } + } +} + +static void fill_x(float *x, uint32_t stream) { + for (uint64_t i = 0; i < (uint64_t)MAX_TOKENS * IN_DIM; i++) { + const int32_t v = (int32_t)( + (i * 29u + stream * 47u + ((uint64_t)stream ^ i) * 3u) % + 255u) - 127; + x[i] = (float)v / 128.0f; + } +} + +static uint64_t checksum(const void *ptr, uint64_t bytes) { + const uint8_t *p = ptr; + uint64_t hash = UINT64_C(1469598103934665603); + for (uint64_t i = 0; i < bytes; i++) { + hash ^= p[i]; + hash *= UINT64_C(1099511628211); + } + return hash; +} + +static uint64_t output_count(uint32_t output) { + return (uint64_t)MAX_TOKENS * + (output == 0u ? OUT0_DIM : OUT1_DIM); +} + +static uint64_t active_output_count(uint32_t output, uint32_t n_tokens) { + return (uint64_t)n_tokens * + (output == 0u ? OUT0_DIM : OUT1_DIM); +} + +static void fixture_init(fixture *f) { + memset(f, 0, sizeof(*f)); + f->row_bytes = (IN_DIM / QK_K) * sizeof(block_q4_K); + const uint64_t page = (uint64_t)getpagesize(); + f->weight_offset[0] = 0u; + f->weight_bytes[0] = (uint64_t)OUT0_DIM * f->row_bytes; + f->weight_bytes[1] = (uint64_t)OUT1_DIM * f->row_bytes; + f->weight_offset[1] = align_up(f->weight_bytes[0] + page, page); + + f->model_size = align_up(f->weight_offset[1] + f->weight_bytes[1], + page); + if (posix_memalign(&f->model, (size_t)page, (size_t)f->model_size) != 0) { + fail("synthetic model allocation"); + } + memset(f->model, 0, (size_t)f->model_size); + fill_q4((block_q4_K *)((uint8_t *)f->model + f->weight_offset[0]), + f->weight_bytes[0] / sizeof(block_q4_K), 0x41c64e6du); + fill_q4((block_q4_K *)((uint8_t *)f->model + f->weight_offset[1]), + f->weight_bytes[1] / sizeof(block_q4_K), 0x9e3779b9u); + + ds4_gpu_set_quality(false); + ds4_gpu_set_ssd_streaming(true); + const uint64_t offsets[2] = { + f->weight_offset[0], f->weight_offset[1], + }; + const uint64_t sizes[2] = { + f->weight_bytes[0], f->weight_bytes[1], + }; + if (!ds4_gpu_set_model_map_spans(f->model, f->model_size, + offsets, sizes, 2u, + f->weight_bytes[0])) { + fail("SSD-style model spans"); + } + + const uint64_t x_count = (uint64_t)MAX_TOKENS * IN_DIM; + for (uint32_t stream = 0; stream < TEST_STREAMS; stream++) { + f->x_host[stream] = malloc((size_t)x_count * sizeof(float)); + if (!f->x_host[stream]) fail("host activation allocation"); + fill_x(f->x_host[stream], stream); + f->x[stream] = ds4_gpu_tensor_alloc(x_count * sizeof(float)); + if (!f->x[stream] || + !ds4_gpu_tensor_write(f->x[stream], 0u, f->x_host[stream], + x_count * sizeof(float))) { + fail("activation tensor"); + } + for (uint32_t output = 0; output < 2u; output++) { + const uint64_t bytes = output_count(output) * sizeof(float); + f->baseline[stream][output] = ds4_gpu_tensor_alloc(bytes); + f->candidate[stream][output] = ds4_gpu_tensor_alloc(bytes); + if (!f->baseline[stream][output] || + !f->candidate[stream][output]) { + fail("output tensor allocation"); + } + } + } + fprintf(stderr, + "Metal Q4 prefill pair runtime: synthetic=%.2f MiB spans=2 " + "ssd=1 max_tokens=%u\n", + (double)f->model_size / 1048576.0, MAX_TOKENS); +} + +static void fixture_destroy(fixture *f) { + ds4_gpu_set_stream(0); + (void)ds4_gpu_synchronize(); + for (uint32_t stream = 0; stream < TEST_STREAMS; stream++) { + for (uint32_t output = 0; output < 2u; output++) { + ds4_gpu_tensor_free(f->candidate[stream][output]); + ds4_gpu_tensor_free(f->baseline[stream][output]); + } + ds4_gpu_tensor_free(f->x[stream]); + free(f->x_host[stream]); + } + ds4_gpu_cleanup(); + free(f->model); + memset(f, 0, sizeof(*f)); +} + +static void poison_outputs(fixture *f, uint32_t stream) { + for (uint32_t output = 0; output < 2u; output++) { + const uint64_t count = output_count(output); + if (!ds4_gpu_tensor_fill_f32(f->baseline[stream][output], + k_poison, count) || + !ds4_gpu_tensor_fill_f32(f->candidate[stream][output], + k_poison, count)) { + fail("output poison"); + } + } +} + +static int run_baseline(fixture *f, uint32_t stream, uint32_t n_tokens) { + ds4_gpu_set_stream((int)stream); + if (!ds4_gpu_begin_commands()) return 0; + int ok = ds4_gpu_matmul_quant_tensor( + f->baseline[stream][0], f->model, f->model_size, + f->weight_offset[0], Q4_K_TYPE, IN_DIM, OUT0_DIM, + f->x[stream], n_tokens); + if (ok) { + ok = ds4_gpu_matmul_quant_tensor( + f->baseline[stream][1], f->model, f->model_size, + f->weight_offset[1], Q4_K_TYPE, IN_DIM, OUT1_DIM, + f->x[stream], n_tokens); + } + const int ended = ds4_gpu_end_commands(); + return ok && ended; +} + +static int run_candidate(fixture *f, uint32_t stream, uint32_t n_tokens) { + ds4_gpu_set_stream((int)stream); + if (!ds4_gpu_begin_commands()) return 0; + const int rc = ds4_gpu_matmul_q4_K_pair_tensor( + f->candidate[stream][0], f->candidate[stream][1], + f->model, f->model_size, + f->weight_offset[0], f->weight_offset[1], + IN_DIM, OUT0_DIM, OUT1_DIM, f->x[stream], n_tokens); + const int ended = ds4_gpu_end_commands(); + return rc == 1 && ended; +} + +static void check_poison_suffix(const float *values, uint64_t begin, + uint64_t count, const char *label) { + for (uint64_t i = begin; i < count; i++) { + if (memcmp(&values[i], &k_poison, sizeof(k_poison)) != 0) { + fprintf(stderr, + "Metal Q4 prefill pair suffix write label=%s index=%llu\n", + label, (unsigned long long)i); + fail("output suffix canary"); + } + } +} + +static void check_tensor_all_poison(const ds4_gpu_tensor *tensor, + uint64_t count, const char *label) { + float *values = malloc((size_t)count * sizeof(float)); + if (!values || + !ds4_gpu_tensor_read(tensor, 0u, values, count * sizeof(float))) { + fail("poison readback"); + } + check_poison_suffix(values, 0u, count, label); + free(values); +} + +static void check_outputs(fixture *f, uint32_t stream, uint32_t n_tokens) { + for (uint32_t output = 0; output < 2u; output++) { + const uint64_t count = output_count(output); + const uint64_t active = active_output_count(output, n_tokens); + float *base = malloc((size_t)count * sizeof(float)); + float *candidate = malloc((size_t)count * sizeof(float)); + if (!base || !candidate) fail("host output allocation"); + if (!ds4_gpu_tensor_read(f->baseline[stream][output], 0u, base, + count * sizeof(float)) || + !ds4_gpu_tensor_read(f->candidate[stream][output], 0u, candidate, + count * sizeof(float))) { + fail("output readback"); + } + if (memcmp(base, candidate, active * sizeof(float)) != 0) { + uint64_t first = 0u; + while (first < active && + memcmp(&base[first], &candidate[first], + sizeof(float)) == 0) { + first++; + } + fprintf(stderr, + "Metal Q4 prefill pair mismatch N=%u stream=%u output=%u " + "first=%llu base=%g candidate=%g\n", + n_tokens, stream, output, (unsigned long long)first, + first < active ? base[first] : 0.0f, + first < active ? candidate[first] : 0.0f); + fail("bitwise parity"); + } + check_poison_suffix(base, active, count, "baseline"); + check_poison_suffix(candidate, active, count, "candidate"); + fprintf(stderr, + "Metal Q4 prefill pair N=%u stream=%u output=%u " + "checksum=%016llx bitwise=1 canary=1\n", + n_tokens, stream, output, + (unsigned long long)checksum(base, active * sizeof(float))); + free(candidate); + free(base); + } +} + +static void check_x_unchanged(fixture *f, uint32_t stream) { + const uint64_t count = (uint64_t)MAX_TOKENS * IN_DIM; + float *actual = malloc((size_t)count * sizeof(float)); + if (!actual) fail("activation readback allocation"); + if (!ds4_gpu_tensor_read(f->x[stream], 0u, actual, + count * sizeof(float)) || + memcmp(actual, f->x_host[stream], count * sizeof(float)) != 0) { + fail("activation modified"); + } + free(actual); +} + +static void test_shapes(fixture *f) { + set_pair_controls(true, false, true); + for (uint32_t i = 0; i < sizeof(k_tokens) / sizeof(k_tokens[0]); i++) { + const uint32_t n_tokens = k_tokens[i]; + poison_outputs(f, 0u); + if (!run_baseline(f, 0u, n_tokens)) fail("baseline projection"); + if (!run_candidate(f, 0u, n_tokens)) fail("required pair projection"); + check_outputs(f, 0u, n_tokens); + check_x_unchanged(f, 0u); + } +} + +static void test_alias_rejection(fixture *f) { + const uint32_t n_tokens = 32u; + const uint64_t alias_bytes = + active_output_count(0u, n_tokens) * sizeof(float); + ds4_gpu_tensor *alias = + ds4_gpu_tensor_view(f->x[0], 0u, alias_bytes); + if (!alias) fail("alias tensor view"); + poison_outputs(f, 0u); + set_pair_controls(true, false, false); + const int rc = ds4_gpu_matmul_q4_K_pair_tensor( + alias, f->candidate[0][1], f->model, f->model_size, + f->weight_offset[0], f->weight_offset[1], + IN_DIM, OUT0_DIM, OUT1_DIM, f->x[0], n_tokens); + ds4_gpu_tensor_free(alias); + if (rc != 0) fail("alias was not rejected with fallback status"); + check_x_unchanged(f, 0u); + + check_tensor_all_poison(f->candidate[0][1], output_count(1u), + "alias-reject"); + fprintf(stderr, "Metal Q4 prefill pair alias rejection: PASS\n"); +} + +static void test_required_failure_is_local(fixture *f) { + const uint32_t n_tokens = 32u; + const uint64_t active = active_output_count(0u, n_tokens); + const uint64_t count = output_count(0u); + float *reference = malloc((size_t)active * sizeof(float)); + float *actual = malloc((size_t)count * sizeof(float)); + if (!reference || !actual) fail("REQUIRE oracle allocation"); + + set_pair_controls(false, false, false); + if (!ds4_gpu_matmul_quant_tensor( + f->baseline[0][0], f->model, f->model_size, + f->weight_offset[0], Q4_K_TYPE, IN_DIM, OUT0_DIM, + f->x[0], n_tokens) || + !ds4_gpu_tensor_read(f->baseline[0][0], 0u, reference, + active * sizeof(float))) { + fail("REQUIRE reference matmul"); + } + + poison_outputs(f, 0u); + set_pair_controls(true, true, true); + const int rc = ds4_gpu_matmul_q4_K_pair_tensor( + f->candidate[0][0], f->candidate[0][1], + f->model, f->model_size, + f->weight_offset[0], f->weight_offset[1], + IN_DIM, OUT0_DIM, OUT1_DIM, f->x[0], n_tokens); + if (rc != -1) { + fprintf(stderr, + "Metal Q4 prefill pair REQUIRE negative returned %d, expected -1\n", + rc); + fail("REQUIRE negative status"); + } + check_tensor_all_poison(f->candidate[0][0], output_count(0u), + "require-negative-out0"); + check_tensor_all_poison(f->candidate[0][1], output_count(1u), + "require-negative-out1"); + + set_pair_controls(false, false, false); + if (!ds4_gpu_matmul_quant_tensor( + f->baseline[0][0], f->model, f->model_size, + f->weight_offset[0], Q4_K_TYPE, IN_DIM, OUT0_DIM, + f->x[0], n_tokens)) { + fail("REQUIRE failure contaminated next quant matmul"); + } + if (!ds4_gpu_tensor_read(f->baseline[0][0], 0u, actual, + count * sizeof(float)) || + memcmp(actual, reference, active * sizeof(float)) != 0) { + fail("REQUIRE failure changed next quant matmul"); + } + check_poison_suffix(actual, active, count, "require-next-matmul"); + free(actual); + free(reference); + fprintf(stderr, + "Metal Q4 prefill pair REQUIRE negative: PASS " + "rc=-1 next_matmul_bitwise=1\n"); +} + +static void test_two_stream_async(fixture *f) { + static const uint32_t stream_tokens[TEST_STREAMS] = {96u, 128u}; + set_pair_controls(true, false, true); + for (uint32_t stream = 0; stream < TEST_STREAMS; stream++) { + poison_outputs(f, stream); + if (!run_baseline(f, stream, stream_tokens[stream])) { + fail("async baseline"); + } + } + + uint32_t submitted = 0u; + for (uint32_t stream = 0; stream < TEST_STREAMS; stream++) { + ds4_gpu_set_stream((int)stream); + if (!ds4_gpu_begin_commands()) fail("async begin"); + const int rc = ds4_gpu_matmul_q4_K_pair_tensor( + f->candidate[stream][0], f->candidate[stream][1], + f->model, f->model_size, + f->weight_offset[0], f->weight_offset[1], + IN_DIM, OUT0_DIM, OUT1_DIM, f->x[stream], + stream_tokens[stream]); + if (rc != 1 || !ds4_gpu_end_commands_async()) { + fail("async pair submission"); + } + submitted++; + } + for (uint32_t stream = 0; stream < submitted; stream++) { + if (!ds4_gpu_wait_stream((int)stream)) fail("async stream wait"); + } + ds4_gpu_set_stream(0); + for (uint32_t stream = 0; stream < TEST_STREAMS; stream++) { + check_outputs(f, stream, stream_tokens[stream]); + check_x_unchanged(f, stream); + } + fprintf(stderr, + "Metal Q4 prefill pair async streams: PASS streams=2 " + "tokens=96,128 scratch_isolation=bitwise\n"); +} + +int main(void) { + set_pair_controls(false, false, false); + set_flag("DS4_METAL_DISABLE_Q4_DENSE_PAIR", false); + set_flag("DS4_METAL_DISABLE_CONTIG_F32_F16_COPY", false); + + if (!ds4_gpu_init()) { + fprintf(stderr, + "test_metal_q4_prefill_pair: SKIP: no Metal device\n"); + return 0; + } + if (!ds4_gpu_device_is_pre_m5_apple_silicon()) { + fprintf(stderr, + "test_metal_q4_prefill_pair: SKIP: requires Apple M1-M4\n"); + ds4_gpu_cleanup(); + return 0; + } + + fixture f; + fixture_init(&f); + test_shapes(&f); + test_alias_rejection(&f); + test_required_failure_is_local(&f); + test_two_stream_async(&f); + fixture_destroy(&f); + fprintf(stderr, + "test_metal_q4_prefill_pair PASS tokens=32,64,96,128 " + "bitwise=1 require=1 alias=1 ssd_spans=1 streams=2 " + "unretained=%s\n", + getenv("DS4_METAL_UNRETAINED_COMMAND_BUFFERS") ? "on" : "off"); + return 0; +} + +#else + +int main(void) { + fprintf(stderr, + "test_metal_q4_prefill_pair: SKIP: Metal requires macOS\n"); + return 0; +} + +#endif diff --git a/tests/test_metal_q4_streams.c b/tests/test_metal_q4_streams.c index 97c92dfa2..447c6db4c 100644 --- a/tests/test_metal_q4_streams.c +++ b/tests/test_metal_q4_streams.c @@ -343,7 +343,7 @@ static int encode_pair(fixture *f, ds4_gpu_tensor *out0, return ds4_gpu_matmul_q4_K_pair_tensor( out0, out1, f->model, f->model_size, f->weight0_offset, f->weight1_offset, - IN_DIM, OUT0_DIM, OUT1_DIM, x, rows); + IN_DIM, OUT0_DIM, OUT1_DIM, x, rows) > 0; } static int run_arm_once_with_options(fixture *f, test_arm arm, From 5d2e080ea4ddfded2effba0a5ab93ed5e900e1c3 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Sat, 29 Aug 2026 09:13:40 +0200 Subject: [PATCH 144/189] perf(rocm): quantize q4 prefill rhs per wave32 --- rocm/ds4_rocm_moe.cuh | 135 ++++++++ rocm/ds4_rocm_norm_rope.cuh | 52 ++- rocm/ds4_rocm_q4.cuh | 350 +++++++++++++++++-- speed-bench/rocm_q4_prefill_bench.cpp | 226 +++++++++++-- tests/test_rocm_q4_dense_pair.cpp | 464 +++++++++++++++++++++++++- 5 files changed, 1153 insertions(+), 74 deletions(-) diff --git a/rocm/ds4_rocm_moe.cuh b/rocm/ds4_rocm_moe.cuh index d2827fa39..025d3c676 100644 --- a/rocm/ds4_rocm_moe.cuh +++ b/rocm/ds4_rocm_moe.cuh @@ -818,6 +818,141 @@ __global__ static void q8_K_quantize_kernel(cuda_block_q8_K *out, const float *x if (tid == 0) yb->d = 1.0f / iscale_s; } +/* Wave32 Q8_K activation quantizer for the Q4 prefill path. + * + * A 256-thread workgroup quantizes up to eight independent 256-value blocks, + * one per wave. Each lane owns the eight values lane + 32*k. The max + * reduction carries the original element index so equal absolute maxima keep + * the lowest index, matching q8_K_quantize_kernel's left-biased reduction. + * The scale, lrintf conversion, clamp and 16-value bsums are otherwise the + * canonical operations above. No cross-wave communication or LDS is used. + * + * Dispatch is deliberately restricted to a runtime-proven wave32 target in + * ds4_rocm_q4.cuh. Keep the device guard as a second fail-closed boundary in + * case a future caller bypasses that host policy. */ +enum { + ROCM_Q8_K_WAVE32_WIDTH = 32u, + ROCM_Q8_K_WAVE32_BLOCK_THREADS = 256u, + ROCM_Q8_K_WAVE32_WAVES_PER_BLOCK = + ROCM_Q8_K_WAVE32_BLOCK_THREADS / ROCM_Q8_K_WAVE32_WIDTH, +}; + +__global__ static void q8_K_quantize_wave32_kernel( + cuda_block_q8_K *out, + const float *x, + uint32_t in_dim, + uint32_t n_rows) { + if (warpSize != ROCM_Q8_K_WAVE32_WIDTH) return; + + const uint32_t lane = threadIdx.x & (ROCM_Q8_K_WAVE32_WIDTH - 1u); + const uint32_t wave = threadIdx.x / ROCM_Q8_K_WAVE32_WIDTH; + const uint32_t blocks_per_row = in_dim / CUDA_QK_K; + const uint32_t b = blockIdx.x * ROCM_Q8_K_WAVE32_WAVES_PER_BLOCK + wave; + const uint32_t row = blockIdx.y; + if (row >= n_rows || b >= blocks_per_row) return; + + const float *xr = x + (uint64_t)row * in_dim + + (uint64_t)b * CUDA_QK_K; + cuda_block_q8_K *yb = out + (uint64_t)row * blocks_per_row + b; + + uint32_t max_index = lane; + float max_value = xr[max_index]; + float max_abs = fabsf(max_value); + #pragma unroll + for (uint32_t k = 1u; k < CUDA_QK_K / ROCM_Q8_K_WAVE32_WIDTH; + k++) { + const uint32_t index = lane + k * ROCM_Q8_K_WAVE32_WIDTH; + const float value = xr[index]; + const float value_abs = fabsf(value); + if (value_abs > max_abs || + (value_abs == max_abs && index < max_index)) { + max_abs = value_abs; + max_value = value; + max_index = index; + } + } + + for (uint32_t offset = ROCM_Q8_K_WAVE32_WIDTH / 2u; + offset != 0u; offset >>= 1u) { +#if defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__) + const float other_abs = __shfl_down(max_abs, offset, 32); + const float other_value = __shfl_down(max_value, offset, 32); + const uint32_t other_index = __shfl_down(max_index, offset, 32); +#else + const float other_abs = __shfl_down_sync( + FULL_WARP_MASK, max_abs, offset, 32); + const float other_value = __shfl_down_sync( + FULL_WARP_MASK, max_value, offset, 32); + const uint32_t other_index = __shfl_down_sync( + FULL_WARP_MASK, max_index, offset, 32); +#endif + if (lane + offset < ROCM_Q8_K_WAVE32_WIDTH && + (other_abs > max_abs || + (other_abs == max_abs && other_index < max_index))) { + max_abs = other_abs; + max_value = other_value; + max_index = other_index; + } + } + +#if defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__) + const float amax = __shfl(max_abs, 0, 32); + const float signed_max = __shfl(max_value, 0, 32); +#else + const float amax = __shfl_sync(FULL_WARP_MASK, max_abs, 0, 32); + const float signed_max = __shfl_sync( + FULL_WARP_MASK, max_value, 0, 32); +#endif + if (amax == 0.0f) { + #pragma unroll + for (uint32_t k = 0u; + k < CUDA_QK_K / ROCM_Q8_K_WAVE32_WIDTH; k++) { + yb->qs[lane + k * ROCM_Q8_K_WAVE32_WIDTH] = 0; + } + if (lane < CUDA_QK_K / 16u) yb->bsums[lane] = 0; + if (lane == 0u) yb->d = 0.0f; + return; + } + + const float iscale = -127.0f / signed_max; + int qv[CUDA_QK_K / ROCM_Q8_K_WAVE32_WIDTH]; + #pragma unroll + for (uint32_t k = 0u; + k < CUDA_QK_K / ROCM_Q8_K_WAVE32_WIDTH; k++) { + const uint32_t index = lane + k * ROCM_Q8_K_WAVE32_WIDTH; + int q = (int)lrintf(iscale * xr[index]); + if (q > 127) q = 127; + if (q < -128) q = -128; + qv[k] = q; + yb->qs[index] = (int8_t)q; + } + + /* A width-16 reduction simultaneously produces the low- and high-half + * bsum for each contiguous 32-value slice. Only lanes 0 and 16 store. */ + #pragma unroll + for (uint32_t offset = 8u; offset != 0u; offset >>= 1u) { + #pragma unroll + for (uint32_t k = 0u; + k < CUDA_QK_K / ROCM_Q8_K_WAVE32_WIDTH; k++) { +#if defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__) + qv[k] += __shfl_down(qv[k], offset, 16); +#else + qv[k] += __shfl_down_sync( + FULL_WARP_MASK, qv[k], offset, 16); +#endif + } + } + if (lane == 0u || lane == 16u) { + const uint32_t half = lane >> 4u; + #pragma unroll + for (uint32_t k = 0u; + k < CUDA_QK_K / ROCM_Q8_K_WAVE32_WIDTH; k++) { + yb->bsums[2u * k + half] = (int16_t)qv[k]; + } + } + if (lane == 0u) yb->d = 1.0f / iscale; +} + __global__ static DS4_ROCM_UNUSED void moe_gate_up_mid_kernel( float *gate_out, float *up_out, diff --git a/rocm/ds4_rocm_norm_rope.cuh b/rocm/ds4_rocm_norm_rope.cuh index 0b3009207..308049d0a 100644 --- a/rocm/ds4_rocm_norm_rope.cuh +++ b/rocm/ds4_rocm_norm_rope.cuh @@ -921,6 +921,40 @@ static int rocm_q4_attn_q_b_transient_f16_head_rms_rope_tail_tensor( return 1; } +/* A strict Q8_K-wave32 request belongs to the exact Q4 path. The F16 + * sidecars must yield before validation, allocation, dequantization or GEMM; + * the caller can then enter the canonical Q4 fallback, whose selector either + * launches the required quantizer or fails closed on an unsupported device. */ +enum { + ROCM_Q4_ATTN_Q_B_Q8_WAVE32_CONFLICT = -1, + ROCM_Q4_ATTN_Q_B_Q8_WAVE32_KEEP_F16 = 0, + ROCM_Q4_ATTN_Q_B_Q8_WAVE32_YIELD = 1, +}; + +static int rocm_q4_attn_q_b_required_q8_wave32_policy( + uint32_t weight_type, + uint32_t n_tok, + int q8_wave32_required, + int f16_cache_required) { + if (weight_type != DS4_ROCM_Q4_K_TYPE || n_tok <= 8u || + !q8_wave32_required) { + return ROCM_Q4_ATTN_Q_B_Q8_WAVE32_KEEP_F16; + } + return f16_cache_required + ? ROCM_Q4_ATTN_Q_B_Q8_WAVE32_CONFLICT + : ROCM_Q4_ATTN_Q_B_Q8_WAVE32_YIELD; +} + +extern "C" int ds4_rocm_test_q4_attn_q_b_yield_to_q8_wave32_policy( + uint32_t weight_type, + uint32_t n_tok, + int q8_wave32_required, + int f16_cache_required) { + return rocm_q4_attn_q_b_required_q8_wave32_policy( + weight_type, n_tok, q8_wave32_required != 0, + f16_cache_required != 0); +} + extern "C" int ds4_gpu_attn_q_b_f16_head_rms_rope_tail_tensor( ds4_gpu_tensor *out, ds4_gpu_tensor *q_half, @@ -945,8 +979,24 @@ extern "C" int ds4_gpu_attn_q_b_f16_head_rms_rope_tail_tensor( float beta_fast, float beta_slow, float eps) { + const int q8_wave32_required = rocm_q4_attn_q_b_env_bool( + "DS4_ROCM_REQUIRE_Q4_PREFILL_Q8_K_WAVE32") == 1; + const int f16_cache_required = rocm_q4_attn_q_b_f16_required(); + const int q8_wave32_policy = + rocm_q4_attn_q_b_required_q8_wave32_policy( + weight_type, n_tok, q8_wave32_required, f16_cache_required); + if (q8_wave32_policy == ROCM_Q4_ATTN_Q_B_Q8_WAVE32_CONFLICT) { + fprintf(stderr, + DS4_GPU_LOG_PREFIX + "Q4 attn_q_b prefill cannot require both the F16 cache " + "and the Q8_K wave32 quantizer\n"); + return -1; + } + if (q8_wave32_policy == ROCM_Q4_ATTN_Q_B_Q8_WAVE32_YIELD) { + return 0; + } if (weight_type == DS4_ROCM_Q4_K_TYPE) { - const int required = rocm_q4_attn_q_b_f16_required(); + const int required = f16_cache_required; const int persistent_requested = required || (rocm_q4_attn_q_b_f16_enabled() && diff --git a/rocm/ds4_rocm_q4.cuh b/rocm/ds4_rocm_q4.cuh index c71def35a..c7fb7909a 100644 --- a/rocm/ds4_rocm_q4.cuh +++ b/rocm/ds4_rocm_q4.cuh @@ -743,6 +743,196 @@ static int rocm_q4_K_prefill_tile8_required(void) { return getenv("DS4_ROCM_REQUIRE_Q4_PREFILL_TILE8") != NULL; } +enum { + ROCM_Q4_PREFILL_Q8_WAVE32_REQUIRED_FAILURE = -1, + ROCM_Q4_PREFILL_Q8_WAVE32_FALLBACK = 0, + ROCM_Q4_PREFILL_Q8_WAVE32_USE = 1, +}; + +static uint64_t g_rocm_q4_prefill_q8_wave32_launches; + +extern "C" void ds4_rocm_test_q4_prefill_q8_wave32_reset(void) { + __atomic_store_n(&g_rocm_q4_prefill_q8_wave32_launches, 0u, + __ATOMIC_RELAXED); +} + +extern "C" uint64_t ds4_rocm_test_q4_prefill_q8_wave32_get_calls(void) { + return __atomic_load_n(&g_rocm_q4_prefill_q8_wave32_launches, + __ATOMIC_RELAXED); +} + +/* Pure policy kept separate from device discovery so the precedence and + * fail-closed contract remain testable on hosts without a visible GPU. + * REQUIRE is also an opt-in; DISABLE is authoritative. */ +static int rocm_q4_K_prefill_q8_wave32_policy( + int prefill_scope, + int runtime_compatible, + int enabled, + int disabled, + int required) { + if (!enabled && !required) { + return ROCM_Q4_PREFILL_Q8_WAVE32_FALLBACK; + } + if (disabled || !prefill_scope || !runtime_compatible) { + return required ? ROCM_Q4_PREFILL_Q8_WAVE32_REQUIRED_FAILURE + : ROCM_Q4_PREFILL_Q8_WAVE32_FALLBACK; + } + return ROCM_Q4_PREFILL_Q8_WAVE32_USE; +} + +extern "C" int ds4_rocm_test_q4_prefill_q8_wave32_policy( + int prefill_scope, + int runtime_compatible, + int enabled, + int disabled, + int required) { + return rocm_q4_K_prefill_q8_wave32_policy( + prefill_scope != 0, runtime_compatible != 0, enabled != 0, + disabled != 0, required != 0); +} + +static int rocm_q4_K_prefill_q8_wave32_required(void) { + return rocm_q4_attn_q_b_env_bool( + "DS4_ROCM_REQUIRE_Q4_PREFILL_Q8_K_WAVE32") == 1; +} + +static int rocm_q4_K_prefill_q8_wave32_select(uint64_t n_tok) { + const int enabled = rocm_q4_attn_q_b_env_bool( + "DS4_ROCM_ENABLE_Q4_PREFILL_Q8_K_WAVE32") == 1; + const int disabled = rocm_q4_attn_q_b_env_bool( + "DS4_ROCM_DISABLE_Q4_PREFILL_Q8_K_WAVE32") == 1; + const int required = rocm_q4_K_prefill_q8_wave32_required(); + const int prefill_scope = rocm_q4_K_prefill_tile8_scope(n_tok); + const int requested = enabled || required; + const int runtime_compatible = requested && !disabled && prefill_scope + ? rocm_attention_runtime_is_gfx1151_wave32(0) + : 0; + const int decision = rocm_q4_K_prefill_q8_wave32_policy( + prefill_scope, runtime_compatible, enabled, disabled, required); + if (decision == ROCM_Q4_PREFILL_Q8_WAVE32_REQUIRED_FAILURE) { + fprintf(stderr, + DS4_GPU_LOG_PREFIX + "required Q4 prefill Q8_K wave32 quantizer is unavailable " + "(N=%llu scope=%d compatible=%d disabled=%d)\n", + (unsigned long long)n_tok, prefill_scope, + runtime_compatible, disabled); + } + return decision; +} + +static int rocm_q4_K_q8_quantize_launch( + cuda_block_q8_K *out, + const float *x, + uint32_t in_dim, + uint32_t n_rows, + int q8_wave32, + int q8_wave32_required, + const char *label) { + const uint32_t blocks = in_dim / CUDA_QK_K; + if (q8_wave32 == ROCM_Q4_PREFILL_Q8_WAVE32_USE) { + /* Re-read the active device immediately before enqueue. Selection + * may have happened before range resolution/allocation, and HIP's + * active device is thread-local. Optional use falls back safely; + * REQUIRE cannot silently enqueue either the wrong kernel or the + * canonical one. */ + if (!rocm_attention_runtime_is_gfx1151_wave32(0)) { + if (q8_wave32_required) { + fprintf(stderr, + DS4_GPU_LOG_PREFIX + "required Q8_K wave32 quantizer lost its gfx1151 " + "wave32 device before enqueue\n"); + return 0; + } + q8_wave32 = ROCM_Q4_PREFILL_Q8_WAVE32_FALLBACK; + } + } + if (q8_wave32 == ROCM_Q4_PREFILL_Q8_WAVE32_USE) { + const dim3 grid( + (unsigned)((blocks + ROCM_Q8_K_WAVE32_WAVES_PER_BLOCK - 1u) / + ROCM_Q8_K_WAVE32_WAVES_PER_BLOCK), + n_rows, 1u); + q8_K_quantize_wave32_kernel<<< + grid, ROCM_Q8_K_WAVE32_BLOCK_THREADS>>>( + out, x, in_dim, n_rows); + const int ok = cuda_ok(cudaGetLastError(), label); + if (ok) { + __atomic_fetch_add(&g_rocm_q4_prefill_q8_wave32_launches, 1u, + __ATOMIC_RELAXED); + } + return ok; + } + + const dim3 grid((unsigned)blocks, n_rows, 1u); + q8_K_quantize_kernel<<>>(out, x, in_dim, n_rows); + return cuda_ok(cudaGetLastError(), label); +} + +/* Raw-layout oracle used only by the ROCm Q4 parity test. It lets the test + * compare every Q8_K byte instead of relying solely on downstream dot + * products to expose a quantizer mismatch. */ +extern "C" int ds4_rocm_test_q8_K_quantize_tensor( + ds4_gpu_tensor *out, + const ds4_gpu_tensor *x, + uint32_t in_dim, + uint32_t n_rows, + int use_wave32) { + if (!out || !x || !out->ptr || !x->ptr || in_dim == 0u || n_rows == 0u || + (in_dim % CUDA_QK_K) != 0u) { + return 0; + } + uint64_t x_bytes = 0; + uint64_t out_bytes = 0; + if (!cuda_u64_mul3_checked(n_rows, in_dim, sizeof(float), &x_bytes) || + !cuda_u64_mul3_checked(n_rows, in_dim / CUDA_QK_K, + sizeof(cuda_block_q8_K), &out_bytes) || + x->bytes < x_bytes || out->bytes < out_bytes) { + return 0; + } + const int decision = use_wave32 + ? (rocm_attention_runtime_is_gfx1151_wave32(0) + ? ROCM_Q4_PREFILL_Q8_WAVE32_USE + : ROCM_Q4_PREFILL_Q8_WAVE32_REQUIRED_FAILURE) + : ROCM_Q4_PREFILL_Q8_WAVE32_FALLBACK; + if (decision == ROCM_Q4_PREFILL_Q8_WAVE32_REQUIRED_FAILURE) return 0; + return rocm_q4_K_q8_quantize_launch( + reinterpret_cast(out->ptr), + reinterpret_cast(x->ptr), in_dim, n_rows, decision, + use_wave32 != 0, + use_wave32 ? "Q8_K wave32 raw oracle launch" + : "Q8_K canonical raw oracle launch"); +} + +/* Benchmark-only enqueue hook. The resident ROCm harness validates the + * active gfx1151/wave32 device, dimensions, allocations and output guards + * before entering its HIP-event interval. Keep this entry deliberately free + * of device queries, allocation, environment parsing and cudaGetLastError so + * the measured interval contains only the selected quantizer dispatch. The + * following end event/synchronization reports any asynchronous failure. */ +extern "C" void ds4_rocm_bench_q8_K_quantize_enqueue( + void *out, + const void *x, + uint32_t in_dim, + uint32_t n_rows, + int use_wave32) { + const uint32_t blocks = in_dim / CUDA_QK_K; + if (use_wave32) { + const dim3 grid( + (unsigned)((blocks + ROCM_Q8_K_WAVE32_WAVES_PER_BLOCK - 1u) / + ROCM_Q8_K_WAVE32_WAVES_PER_BLOCK), + n_rows, 1u); + q8_K_quantize_wave32_kernel<<< + grid, ROCM_Q8_K_WAVE32_BLOCK_THREADS>>>( + reinterpret_cast(out), + reinterpret_cast(x), in_dim, n_rows); + return; + } + + const dim3 grid((unsigned)blocks, n_rows, 1u); + q8_K_quantize_kernel<<>>( + reinterpret_cast(out), + reinterpret_cast(x), in_dim, n_rows); +} + enum { ROCM_Q4_PREFILL_WMMA_REQUIRED_FAILURE = -1, ROCM_Q4_PREFILL_WMMA_FALLBACK = 0, @@ -789,7 +979,8 @@ static int rocm_q4_K_prefill_wmma_select( ((ssd_enabled || required) && weight_device_resident); const int eligible = !disabled && shape_ok && storage_ok && - !g_quality_mode && rocm_attention_runtime_is_gfx1151_wave32(); + !g_quality_mode && + rocm_attention_runtime_is_gfx1151_wave32(requested); if (eligible) return ROCM_Q4_PREFILL_WMMA_USE; if (!required) return ROCM_Q4_PREFILL_WMMA_FALLBACK; @@ -1076,6 +1267,12 @@ extern "C" int ds4_rocm_matmul_q4_K_tensor( const int prefill_scope = rocm_q4_K_prefill_tile8_scope(n_tok); const int prefill_tile8 = rocm_q4_K_prefill_tile8_requested(); const int prefill_tile8_required = rocm_q4_K_prefill_tile8_required(); + const int q8_wave32 = rocm_q4_K_prefill_q8_wave32_select(n_tok); + const int q8_wave32_required = + rocm_q4_K_prefill_q8_wave32_required(); + if (q8_wave32 == ROCM_Q4_PREFILL_Q8_WAVE32_REQUIRED_FAILURE) { + return 0; + } const int k1024_tile4_shape = blocks == ROCM_Q4_PREFILL_K1024_KBLOCK_TILE && out_dim == DS4_ROCM_Q4_ATTN_Q_B_OUT_DIM; @@ -1109,6 +1306,21 @@ extern "C" int ds4_rocm_matmul_q4_K_tensor( int prefill_wmma = rocm_q4_K_prefill_wmma_select( n_tok, in_dim, out_dim, weight_device_resident); if (prefill_wmma == ROCM_Q4_PREFILL_WMMA_REQUIRED_FAILURE) return 0; + if (prefill_wmma == ROCM_Q4_PREFILL_WMMA_USE && + q8_wave32 == ROCM_Q4_PREFILL_Q8_WAVE32_USE && + q8_wave32_required) { + if (rocm_q4_attn_q_b_env_bool( + "DS4_ROCM_REQUIRE_Q4_PREFILL_WMMA") == 1) { + fprintf(stderr, + DS4_GPU_LOG_PREFIX + "Q4_K prefill cannot require both direct WMMA and " + "the Q8_K wave32 quantizer\n"); + return 0; + } + /* The direct F16 WMMA path has no Q8_K RHS. A strict quantizer + * request therefore owns dispatch and retains the exact matmul. */ + prefill_wmma = ROCM_Q4_PREFILL_WMMA_FALLBACK; + } if (prefill_wmma == ROCM_Q4_PREFILL_WMMA_USE && prefill_scope && prefill_tile8_required) { if (rocm_q4_attn_q_b_env_bool( @@ -1154,11 +1366,13 @@ extern "C" int ds4_rocm_matmul_q4_K_tensor( n_tok, blocks, "q4_K dense prequant"); if (!xq) return 0; - const dim3 qgrid((unsigned)blocks, (unsigned)n_tok, 1u); - q8_K_quantize_kernel<<>>( + if (!rocm_q4_K_q8_quantize_launch( xq, reinterpret_cast(x->ptr), - (uint32_t)in_dim, (uint32_t)n_tok); - if (!cuda_ok(cudaGetLastError(), "q4_K dense quantize launch")) return 0; + (uint32_t)in_dim, (uint32_t)n_tok, q8_wave32, + q8_wave32_required, + "q4_K dense quantize launch")) { + return 0; + } if (prefill_scope && prefill_tile8) { if (k1024_tile4 == ROCM_Q4_PREFILL_K1024_TILE4_USE) { @@ -1201,6 +1415,29 @@ extern "C" int ds4_rocm_matmul_q4_K_tensor( return cuda_ok(cudaGetLastError(), "q4_K dense matmul launch"); } +/* Only REQUIRE_TILE8 makes the fused pair itself a strict contract. A + * Q8-wave32-only REQUIRE belongs to the quantizer and can still be satisfied + * by the graph's two dense fallbacks. Keep this pre-enqueue policy in one + * place so validation, alias/range rejection and scratch allocation cannot + * silently weaken the pair contract. Decode is outside prefill scope and + * therefore keeps its legacy optional status. */ +static int rocm_q4_K_pair_pre_enqueue_failure_policy( + int prefill_scope, + int tile8_required, + int q8_wave32_required) { + (void)q8_wave32_required; + return prefill_scope && tile8_required ? -1 : 0; +} + +extern "C" int ds4_rocm_test_q4_pair_pre_enqueue_failure_policy( + int prefill_scope, + int tile8_required, + int q8_wave32_required) { + return rocm_q4_K_pair_pre_enqueue_failure_policy( + prefill_scope != 0, tile8_required != 0, + q8_wave32_required != 0); +} + extern "C" int ds4_gpu_matmul_q4_K_pair_tensor( ds4_gpu_tensor *out0, ds4_gpu_tensor *out1, @@ -1214,10 +1451,26 @@ extern "C" int ds4_gpu_matmul_q4_K_pair_tensor( const ds4_gpu_tensor *x, uint64_t n_tok) { const int prefill_scope = rocm_q4_K_prefill_tile8_scope(n_tok); - const int prefill_required = prefill_scope && - rocm_q4_K_prefill_tile8_required(); + const int tile8_required = rocm_q4_K_prefill_tile8_required(); + const int prefill_required = prefill_scope && tile8_required; const int wmma_required = rocm_q4_attn_q_b_env_bool( "DS4_ROCM_REQUIRE_Q4_PREFILL_WMMA") == 1; + const int q8_wave32 = rocm_q4_K_prefill_q8_wave32_select(n_tok); + const int q8_wave32_required = + rocm_q4_K_prefill_q8_wave32_required(); + if (q8_wave32 == ROCM_Q4_PREFILL_Q8_WAVE32_REQUIRED_FAILURE) { + return -1; + } + const int pre_enqueue_failure = + rocm_q4_K_pair_pre_enqueue_failure_policy( + prefill_scope, tile8_required, q8_wave32_required); + if (q8_wave32_required && wmma_required) { + fprintf(stderr, + DS4_GPU_LOG_PREFIX + "Q4_K prefill pair cannot require both direct WMMA and " + "the Q8_K wave32 quantizer\n"); + return -1; + } /* The fused pair consumes a shared Q8_K activation tile. When the direct * F16 WMMA experiment is selected, return before validation/enqueue so the @@ -1226,14 +1479,16 @@ extern "C" int ds4_gpu_matmul_q4_K_pair_tensor( * after silently measuring the TILE8 pair. SSD preflight is deliberately * optimistic about residency: it only decides whether to yield; each dense * fallback then proves its exact physical device range before enqueue. */ - if (!prefill_required) { + if (!prefill_required && !q8_wave32_required) { const int wmma0 = rocm_q4_K_prefill_wmma_select( n_tok, in_dim, out0_dim, 1); const int wmma1 = rocm_q4_K_prefill_wmma_select( n_tok, in_dim, out1_dim, 1); if (wmma0 == ROCM_Q4_PREFILL_WMMA_REQUIRED_FAILURE || - wmma1 == ROCM_Q4_PREFILL_WMMA_REQUIRED_FAILURE || - wmma0 == ROCM_Q4_PREFILL_WMMA_USE || + wmma1 == ROCM_Q4_PREFILL_WMMA_REQUIRED_FAILURE) { + return -1; + } + if (wmma0 == ROCM_Q4_PREFILL_WMMA_USE || wmma1 == ROCM_Q4_PREFILL_WMMA_USE) { return 0; } @@ -1241,7 +1496,7 @@ extern "C" int ds4_gpu_matmul_q4_K_pair_tensor( fprintf(stderr, DS4_GPU_LOG_PREFIX "Q4_K prefill pair cannot require both WMMA and TILE8\n"); - return 0; + return -1; } const int prefill_pair = prefill_scope && @@ -1251,7 +1506,7 @@ extern "C" int ds4_gpu_matmul_q4_K_pair_tensor( "ds4: required ROCm Q4_K prefill tile8 pair is disabled " "(n_tok=%llu)\n", (unsigned long long)n_tok); - return 0; + return -1; } /* Decode keeps its original, separately gated pair path. Prefill uses @@ -1259,10 +1514,12 @@ extern "C" int ds4_gpu_matmul_q4_K_pair_tensor( * one tiled launch across the two projections. */ const int decode_pair = n_tok <= 8u && rocm_q4_K_dense_pair_requested(); - if ((!prefill_pair && !decode_pair) || - !out0 || !out1 || out0 == out1 || + if (!prefill_pair && !decode_pair) { + return pre_enqueue_failure; + } + if (!out0 || !out1 || out0 == out1 || (out0 && out1 && out0->ptr == out1->ptr)) { - return 0; + return pre_enqueue_failure; } uint64_t blocks0 = 0, blocks1 = 0; @@ -1275,7 +1532,7 @@ extern "C" int ds4_gpu_matmul_q4_K_pair_tensor( in_dim, out1_dim, x, n_tok, &blocks1, &row_bytes1, &weight1_bytes) || blocks0 != blocks1 || row_bytes0 != row_bytes1) { - return 0; + return pre_enqueue_failure; } uint64_t out0_bytes = 0; uint64_t out1_bytes = 0; @@ -1283,24 +1540,24 @@ extern "C" int ds4_gpu_matmul_q4_K_pair_tensor( !cuda_u64_mul3_checked(n_tok, out1_dim, sizeof(float), &out1_bytes) || rocm_q4_K_byte_ranges_overlap(out0->ptr, out0_bytes, out1->ptr, out1_bytes)) { - return 0; + return pre_enqueue_failure; } const char *w0 = cuda_model_range_ptr(model_map, weight0_offset, weight0_bytes, "q4_K dense pair0"); const char *w1 = cuda_model_range_ptr(model_map, weight1_offset, weight1_bytes, "q4_K dense pair1"); - if (!w0 || !w1) return 0; + if (!w0 || !w1) return pre_enqueue_failure; cuda_block_q8_K *xq = rocm_q4_K_prequant_alloc( n_tok, blocks0, "q4_K dense pair prequant"); - if (!xq) return 0; + if (!xq) return pre_enqueue_failure; - const dim3 qgrid((unsigned)blocks0, (unsigned)n_tok, 1u); - q8_K_quantize_kernel<<>>( + if (!rocm_q4_K_q8_quantize_launch( xq, reinterpret_cast(x->ptr), - (uint32_t)in_dim, (uint32_t)n_tok); - if (!cuda_ok(cudaGetLastError(), "q4_K dense pair quantize launch")) { - return 0; + (uint32_t)in_dim, (uint32_t)n_tok, q8_wave32, + q8_wave32_required, + "q4_K dense pair quantize launch")) { + return -1; } if (prefill_pair) { @@ -1318,7 +1575,7 @@ extern "C" int ds4_gpu_matmul_q4_K_pair_tensor( const int ok = cuda_ok(cudaGetLastError(), "q4_K dense prefill pair tile8 launch"); if (ok) rocm_q4_K_prefill_tile8_note(0u, 1u, 0u, 0u, n_tok); - return ok; + return ok ? 1 : -1; } const uint64_t out0_tiles = (out0_dim - 1u) / 32u + 1u; @@ -1330,7 +1587,8 @@ extern "C" int ds4_gpu_matmul_q4_K_pair_tensor( reinterpret_cast(out1->ptr), w0, w1, xq, row_bytes0, (uint32_t)blocks0, (uint32_t)out0_dim, (uint32_t)out1_dim, (uint32_t)n_tok); - return cuda_ok(cudaGetLastError(), "q4_K dense pair matmul launch"); + return cuda_ok(cudaGetLastError(), "q4_K dense pair matmul launch") + ? 1 : -1; } extern "C" int ds4_gpu_attention_output_low_q4_K_slice_tensor( @@ -1427,6 +1685,8 @@ static int rocm_q4_K_prefill_tile8_quant_launch( uint32_t out_dim, uint64_t row_bytes, int prefill_wmma, + int q8_wave32, + int q8_wave32_required, const char *label) { uint64_t n_rows = 0; uint64_t xq_token_stride = 0; @@ -1458,11 +1718,10 @@ static int rocm_q4_K_prefill_tile8_quant_launch( n_rows, blocks, label ? label : "q4_K prefill tile8 prequant"); if (!xq) return 0; - const dim3 qgrid((unsigned)blocks, (unsigned)n_rows, 1u); - q8_K_quantize_kernel<<>>( - xq, x, in_dim, (uint32_t)n_rows); - if (!cuda_ok(cudaGetLastError(), - "q4_K prefill tile8 quantize launch")) { + if (!rocm_q4_K_q8_quantize_launch( + xq, x, in_dim, (uint32_t)n_rows, q8_wave32, + q8_wave32_required, + "q4_K prefill tile8 quantize launch")) { return 0; } @@ -1503,6 +1762,12 @@ extern "C" int ds4_gpu_attention_output_q4_K_batch_tensor( const int tile8_requested = rocm_q4_K_prefill_tile8_requested(); const int tile8_required = tile8_scope && rocm_q4_K_prefill_tile8_required(); + const int q8_wave32 = rocm_q4_K_prefill_q8_wave32_select(n_tokens); + const int q8_wave32_required = + rocm_q4_K_prefill_q8_wave32_required(); + if (q8_wave32 == ROCM_Q4_PREFILL_Q8_WAVE32_REQUIRED_FAILURE) { + return -1; + } const int wmma_enabled = rocm_q4_attn_q_b_env_bool( "DS4_ROCM_ENABLE_Q4_PREFILL_WMMA") == 1; const int wmma_ssd_enabled = rocm_q4_attn_q_b_env_bool( @@ -1516,17 +1781,17 @@ extern "C" int ds4_gpu_attention_output_q4_K_batch_tensor( (g_ssd_streaming_mode && wmma_ssd_enabled))); if (!tile8_scope) return 0; if (!tile8_requested && !wmma_requested) { - if (tile8_required) { + if (tile8_required || q8_wave32_required) { fprintf(stderr, "ds4: required ROCm Q4_K attention-output prefill " - "tile8 is disabled (n_tok=%u)\n", + "exact path is disabled (n_tok=%u)\n", n_tokens); return -1; } return 0; } const int pre_enqueue_failure = - (tile8_required || wmma_required) ? -1 : 0; + (tile8_required || wmma_required || q8_wave32_required) ? -1 : 0; if (!out || !low || !heads || !model_map || group_dim == 0u || rank == 0u || n_groups == 0u || out_dim == 0u || @@ -1617,6 +1882,19 @@ extern "C" int ds4_gpu_attention_output_q4_K_batch_tensor( b_wmma == ROCM_Q4_PREFILL_WMMA_REQUIRED_FAILURE) { return -1; } + if (q8_wave32_required && + (a_wmma == ROCM_Q4_PREFILL_WMMA_USE || + b_wmma == ROCM_Q4_PREFILL_WMMA_USE)) { + if (wmma_required) { + fprintf(stderr, + DS4_GPU_LOG_PREFIX + "Q4_K attention-output prefill cannot require both " + "direct WMMA and the Q8_K wave32 quantizer\n"); + return -1; + } + a_wmma = ROCM_Q4_PREFILL_WMMA_FALLBACK; + b_wmma = ROCM_Q4_PREFILL_WMMA_FALLBACK; + } if (tile8_required && (a_wmma == ROCM_Q4_PREFILL_WMMA_USE || b_wmma == ROCM_Q4_PREFILL_WMMA_USE)) { @@ -1644,7 +1922,7 @@ extern "C" int ds4_gpu_attention_output_q4_K_batch_tensor( reinterpret_cast(low->ptr), out_a, reinterpret_cast(heads->ptr), n_tokens, n_groups, (uint32_t)group_dim, (uint32_t)rank, row_a_bytes, - a_wmma, + a_wmma, q8_wave32, q8_wave32_required, "q4_K attention output A WMMA64/tile8"); if (a_rc <= 0) { return a_rc < 0 ? -1 : pre_enqueue_failure; @@ -1656,7 +1934,7 @@ extern "C" int ds4_gpu_attention_output_q4_K_batch_tensor( reinterpret_cast(out->ptr), out_b, reinterpret_cast(low->ptr), n_tokens, 1u, (uint32_t)low_dim, (uint32_t)out_dim, row_b_bytes, - b_wmma, + b_wmma, q8_wave32, q8_wave32_required, "q4_K attention output B WMMA64/tile8"); } else { b_rc = ds4_gpu_matmul_q8_0_tensor( diff --git a/speed-bench/rocm_q4_prefill_bench.cpp b/speed-bench/rocm_q4_prefill_bench.cpp index 935bfb2ec..3c4a110ce 100644 --- a/speed-bench/rocm_q4_prefill_bench.cpp +++ b/speed-bench/rocm_q4_prefill_bench.cpp @@ -19,6 +19,12 @@ #include #include +/* Deliberately local to this harness: the backend entry performs only the + * selected kernel enqueue after this process has completed all validation. */ +extern "C" void ds4_rocm_bench_q8_K_quantize_enqueue( + void *out, const void *x, uint32_t in_dim, uint32_t n_rows, + int use_wave32); + namespace { constexpr uint32_t kQ4Type = 12u; @@ -35,6 +41,7 @@ constexpr uint32_t kOutputM = 4096u; constexpr uint32_t kDefaultSets = 4u; constexpr uint32_t kDefaultSamples = 8u; constexpr uint32_t kDefaultWarmup = 2u; +constexpr uint32_t kRawQ8GuardWords = 64u; /* Catch a bad N-tail predicate anywhere in the final 64-token WMMA tile, * including the widest q_b output used by this harness. */ constexpr uint32_t kGuardWords = (64u - 1u) * kQbM; @@ -60,6 +67,12 @@ constexpr const char *kWmmaDisable = "DS4_ROCM_DISABLE_Q4_PREFILL_WMMA"; constexpr const char *kWmmaRequire = "DS4_ROCM_REQUIRE_Q4_PREFILL_WMMA"; +constexpr const char *kQ8Wave32Enable = + "DS4_ROCM_ENABLE_Q4_PREFILL_Q8_K_WAVE32"; +constexpr const char *kQ8Wave32Disable = + "DS4_ROCM_DISABLE_Q4_PREFILL_Q8_K_WAVE32"; +constexpr const char *kQ8Wave32Require = + "DS4_ROCM_REQUIRE_Q4_PREFILL_Q8_K_WAVE32"; struct block_q4_K_host { uint16_t d; @@ -71,6 +84,15 @@ struct block_q4_K_host { static_assert(sizeof(block_q4_K_host) == 144u, "Q4_K fixture must match the raw GGUF layout"); +struct block_q8_K_host { + float d; + int8_t qs[kQkK]; + int16_t bsums[kQkK / 16u]; +}; + +static_assert(sizeof(block_q8_K_host) == 292u, + "Q8_K fixture must match the ROCm activation layout"); + enum class bench_case { all, dense, @@ -168,7 +190,9 @@ struct event_timer { hipEventElapsedTime(milliseconds, begin, end) != hipSuccess) { return false; } - return true; + /* Outside the measured interval: surface an immediate launch/config + * error from enqueue-only benchmark hooks instead of false-greening. */ + return hipGetLastError() == hipSuccess; } }; @@ -185,6 +209,11 @@ struct stats { double mean = 0.0; }; +enum class benchmark_rate { + macs, + quantized_values, +}; + uint64_t align_up(uint64_t value, uint64_t alignment) { return (value + alignment - 1u) / alignment * alignment; } @@ -313,20 +342,22 @@ void fill_activation(std::vector *values, uint32_t n_tokens, } } -std::vector guard_pattern() { - std::vector guard(kGuardWords); - for (uint32_t i = 0; i < kGuardWords; i++) guard[i] = 0x7fc12000u + i; +std::vector guard_pattern(uint32_t words = kGuardWords) { + std::vector guard(words); + for (uint32_t i = 0; i < words; i++) guard[i] = 0x7fc12000u + i; return guard; } -bool prepare_guard(ds4_gpu_tensor *tensor, uint64_t logical_bytes) { - const std::vector guard = guard_pattern(); +bool prepare_guard(ds4_gpu_tensor *tensor, uint64_t logical_bytes, + uint32_t guard_words = kGuardWords) { + const std::vector guard = guard_pattern(guard_words); return ds4_gpu_tensor_write(tensor, logical_bytes, guard.data(), guard.size() * sizeof(guard[0])) != 0; } bool poison_output(ds4_gpu_tensor *tensor, uint64_t logical_bytes, - uint32_t pattern) { + uint32_t pattern, + uint32_t guard_words = kGuardWords) { if (!tensor || logical_bytes == 0u || logical_bytes % sizeof(uint32_t) != 0u) { return false; @@ -340,12 +371,13 @@ bool poison_output(ds4_gpu_tensor *tensor, uint64_t logical_bytes, return false; } } - return prepare_guard(tensor, logical_bytes); + return prepare_guard(tensor, logical_bytes, guard_words); } bool check_guard(const ds4_gpu_tensor *tensor, uint64_t logical_bytes, - const char *label) { - const std::vector expected = guard_pattern(); + const char *label, + uint32_t guard_words = kGuardWords) { + const std::vector expected = guard_pattern(guard_words); std::vector got(expected.size()); if (!ds4_gpu_tensor_read(tensor, logical_bytes, got.data(), got.size() * sizeof(got[0]))) { @@ -453,6 +485,9 @@ void select_legacy() { (void)unsetenv(kWmmaSsdEnable); (void)unsetenv(kWmmaDisable); (void)unsetenv(kWmmaRequire); + (void)unsetenv(kQ8Wave32Enable); + (void)unsetenv(kQ8Wave32Disable); + (void)unsetenv(kQ8Wave32Require); } void select_tile8(bool disable_k1024_tile4) { @@ -465,6 +500,9 @@ void select_tile8(bool disable_k1024_tile4) { (void)unsetenv(kWmmaSsdEnable); (void)unsetenv(kWmmaDisable); (void)unsetenv(kWmmaRequire); + (void)unsetenv(kQ8Wave32Enable); + (void)unsetenv(kQ8Wave32Disable); + (void)unsetenv(kQ8Wave32Require); if (disable_k1024_tile4) { (void)setenv(kK1024Tile4Disable, "1", 1); } else { @@ -513,7 +551,8 @@ bool benchmark_arms(const char *case_name, uint32_t n_tokens, uint32_t in_dim, uint32_t out_dim, const config &cfg, const arm &baseline, const arm &candidate, const std::function &oracle_prepare, - const std::function &oracle) { + const std::function &oracle, + benchmark_rate rate = benchmark_rate::macs) { // Validate every rotating weight set and prime reusable Q8_K scratch // before any timed event. This catches data-dependent path errors without // admitting readback or comparison work into the HIP-event interval. @@ -596,23 +635,40 @@ bool benchmark_arms(const char *case_name, uint32_t n_tokens, uint32_t in_dim, const double paired_median = percentile(paired_delta, 0.5); const double median_delta = (b.median / a.median - 1.0) * 100.0; const double speedup = (a.median / b.median - 1.0) * 100.0; - const double macs = static_cast(n_tokens) * in_dim * out_dim; - const double a_gmac_s = macs / (a.median * 1.0e6); - const double b_gmac_s = macs / (b.median * 1.0e6); - - std::printf( - "DS4_ROCM_Q4_PREFILL_BENCH case=%s N=%u K=%u M=%u " - "baseline=%s candidate=%s samples=%u sets=%u " - "baseline_ms_p50=%.6f candidate_ms_p50=%.6f " - "baseline_ms_min=%.6f candidate_ms_min=%.6f " - "baseline_ms_p95=%.6f candidate_ms_p95=%.6f " - "baseline_gmac_s=%.3f candidate_gmac_s=%.3f " - "candidate_delta_pct=%.3f paired_delta_pct_p50=%.3f " - "speedup_pct=%.3f\n", - case_name, n_tokens, in_dim, out_dim, baseline.name, candidate.name, - cfg.samples, cfg.sets, a.median, b.median, a.minimum, b.minimum, - a.p95, b.p95, a_gmac_s, b_gmac_s, median_delta, paired_median, - speedup); + const double work = static_cast(n_tokens) * in_dim * + (rate == benchmark_rate::macs ? out_dim : 1u); + const double a_rate = work / (a.median * 1.0e6); + const double b_rate = work / (b.median * 1.0e6); + + if (rate == benchmark_rate::quantized_values) { + std::printf( + "DS4_ROCM_Q4_PREFILL_BENCH case=%s N=%u K=%u " + "baseline=%s candidate=%s samples=%u sets=%u " + "baseline_ms_p50=%.6f candidate_ms_p50=%.6f " + "baseline_ms_min=%.6f candidate_ms_min=%.6f " + "baseline_ms_p95=%.6f candidate_ms_p95=%.6f " + "baseline_gvalue_s=%.3f candidate_gvalue_s=%.3f " + "candidate_delta_pct=%.3f paired_delta_pct_p50=%.3f " + "speedup_pct=%.3f\n", + case_name, n_tokens, in_dim, baseline.name, candidate.name, + cfg.samples, cfg.sets, a.median, b.median, a.minimum, b.minimum, + a.p95, b.p95, a_rate, b_rate, median_delta, paired_median, + speedup); + } else { + std::printf( + "DS4_ROCM_Q4_PREFILL_BENCH case=%s N=%u K=%u M=%u " + "baseline=%s candidate=%s samples=%u sets=%u " + "baseline_ms_p50=%.6f candidate_ms_p50=%.6f " + "baseline_ms_min=%.6f candidate_ms_min=%.6f " + "baseline_ms_p95=%.6f candidate_ms_p95=%.6f " + "baseline_gmac_s=%.3f candidate_gmac_s=%.3f " + "candidate_delta_pct=%.3f paired_delta_pct_p50=%.3f " + "speedup_pct=%.3f\n", + case_name, n_tokens, in_dim, out_dim, baseline.name, + candidate.name, cfg.samples, cfg.sets, a.median, b.median, + a.minimum, b.minimum, a.p95, b.p95, a_rate, b_rate, + median_delta, paired_median, speedup); + } std::fflush(stdout); return true; } @@ -631,6 +687,98 @@ bool allocate_io(uint32_t n_tokens, uint32_t in_dim, uint64_t out_elements, poison_output(out_b->ptr, logical_bytes, 0x7fc20002u); } +bool run_q8_quantizer(const config &cfg, ds4_gpu_tensor *x, + uint32_t n_tokens) { + int active_device = -1; + hipDeviceProp_t properties{}; + if (hipGetDevice(&active_device) != hipSuccess || active_device < 0 || + hipGetDeviceProperties(&properties, active_device) != hipSuccess || + properties.warpSize != 32 || + std::strncmp(properties.gcnArchName, "gfx1151", 7u) != 0) { + std::fprintf(stderr, + "dense_q8_wave32 N=%u: active device is not " + "gfx1151 wave32\n", + n_tokens); + return false; + } + + uint64_t block_count = 0; + uint64_t logical_bytes = 0; + uint64_t x_bytes = 0; + if (!x || + !checked_mul(n_tokens, kDenseK / kQkK, &block_count) || + !checked_mul(block_count, sizeof(block_q8_K_host), &logical_bytes) || + !checked_mul(static_cast(n_tokens) * kDenseK, + sizeof(float), &x_bytes) || + logical_bytes > std::numeric_limits::max() - + kRawQ8GuardWords * sizeof(uint32_t) || + ds4_gpu_tensor_bytes(x) < x_bytes) { + std::fprintf(stderr, + "dense_q8_wave32 N=%u: raw tensor size overflow\n", + n_tokens); + return false; + } + + const uint64_t allocation_bytes = logical_bytes + + kRawQ8GuardWords * sizeof(uint32_t); + tensor_owner canonical(allocation_bytes); + tensor_owner wave32(allocation_bytes); + if (!canonical.ptr || !wave32.ptr) { + std::fprintf(stderr, + "dense_q8_wave32 N=%u: raw tensor allocation failed\n", + n_tokens); + return false; + } + + /* contents() synchronizes; resolve all raw device pointers before any + * event is recorded so the timed callbacks contain one launch only. */ + const void *x_device = ds4_gpu_tensor_contents(x); + void *canonical_device = ds4_gpu_tensor_contents(canonical.ptr); + void *wave32_device = ds4_gpu_tensor_contents(wave32.ptr); + if (!x_device || !canonical_device || !wave32_device) { + std::fprintf(stderr, + "dense_q8_wave32 N=%u: device pointer resolution failed\n", + n_tokens); + return false; + } + + const arm q8_canonical = { + "q8_canonical_raw", []() {}, + [&](uint32_t) { + ds4_rocm_bench_q8_K_quantize_enqueue( + canonical_device, x_device, kDenseK, n_tokens, 0); + return true; + }}; + const arm q8_wave32 = { + "q8_wave32_raw", []() {}, + [&](uint32_t) { + ds4_rocm_bench_q8_K_quantize_enqueue( + wave32_device, x_device, kDenseK, n_tokens, 1); + return true; + }}; + + config q8_cfg = cfg; + q8_cfg.sets = 1u; // Raw quantization has no rotating weight set. + return benchmark_arms( + "dense_q8_wave32", n_tokens, kDenseK, 1u, q8_cfg, + q8_canonical, q8_wave32, + [&]() { + return poison_output(canonical.ptr, logical_bytes, 0x5a5a5a5au, + kRawQ8GuardWords) && + poison_output(wave32.ptr, logical_bytes, 0xa5a5a5a5u, + kRawQ8GuardWords); + }, + [&]() { + return bitwise_equal(canonical.ptr, wave32.ptr, logical_bytes, + "raw canonical vs wave32 Q8_K") && + check_guard(canonical.ptr, logical_bytes, + "raw canonical Q8_K", kRawQ8GuardWords) && + check_guard(wave32.ptr, logical_bytes, + "raw wave32 Q8_K", kRawQ8GuardWords); + }, + benchmark_rate::quantized_values); +} + bool run_dense(const model_fixture &model, const config &cfg, uint32_t n_tokens) { uint64_t out_elements = 0; @@ -681,7 +829,10 @@ bool run_dense(const model_fixture &model, const config &cfg, !check_guard(tiled.ptr, logical_bytes, "dense tile8")) { return false; } - if (!cfg.wmma_supported || n_tokens < 256u) return true; + if (!cfg.wmma_supported) return true; + if (!run_q8_quantizer(cfg, x.ptr, n_tokens)) return false; + + if (n_tokens < 256u) return true; const arm wmma_baseline = { "tile8", []() { select_tile8(false); }, @@ -760,7 +911,7 @@ bool run_pair(const model_fixture &model, const config &cfg, pair0.ptr, pair1.ptr, model.data, model.size, model.weights[set].dense_offset, model.weights[set].kv_offset, kDenseK, kDenseM, kKvM, - x.ptr, n_tokens) != 0; + x.ptr, n_tokens) == 1; }}; if (!benchmark_arms( "pair", n_tokens, kDenseK, kDenseM + kKvM, cfg, baseline, @@ -1026,8 +1177,10 @@ void usage(FILE *stream, const char *argv0) { " --samples N samples/arm, multiple of 4 (default: %u)\n" " --warmup N untimed dispatches/arm (default: %u)\n" " -h, --help show this help\n\n" - "dense compares legacy/TILE8 and, on gfx1151 for N>=256, TILE8/\n" - "WMMA64 at K=4096,M=1024. pair compares two TILE8 projections with\n" + "dense compares legacy/TILE8, times the raw canonical/wave32 Q8_K\n" + "quantizer kernels on gfx1151, and TILE8/WMMA64 there for N>=256\n" + "at K=4096,M=1024. pair compares\n" + "two TILE8 projections with\n" "the fused K=4096,M=(1024+512) path. qb adds TILE4/WMMA64 at the\n" "K=1024,M=32768 shape. outb isolates K=8192,M=4096; output measures\n" "the production grouped output_a plus output_b API. WMMA comparisons\n" @@ -1145,6 +1298,12 @@ int main(int argc, char **argv) { env_snapshot wmma_ssd_enable_guard(kWmmaSsdEnable); env_snapshot wmma_disable_guard(kWmmaDisable); env_snapshot wmma_require_guard(kWmmaRequire); + env_snapshot q8_wave32_enable_guard(kQ8Wave32Enable); + env_snapshot q8_wave32_disable_guard(kQ8Wave32Disable); + env_snapshot q8_wave32_require_guard(kQ8Wave32Require); + (void)unsetenv(kQ8Wave32Enable); + (void)unsetenv(kQ8Wave32Disable); + (void)unsetenv(kQ8Wave32Require); int device_count = 0; hipError_t hip_rc = hipGetDeviceCount(&device_count); @@ -1207,9 +1366,10 @@ int main(int argc, char **argv) { std::printf( "DS4_ROCM_Q4_PREFILL_SETUP device=%s arch=%s warp=%d sets=%u " "resident_mib=%.2f timing=hip_events ssd_streaming=off " - "wmma64=%s\n", + "wmma64=%s q8_wave32=%s\n", properties.name, properties.gcnArchName, properties.warpSize, cfg.sets, static_cast(model.resident_bytes) / 1048576.0, + cfg.wmma_supported ? "available" : "skipped", cfg.wmma_supported ? "available" : "skipped"); std::fflush(stdout); for (uint32_t n_tokens : cfg.tokens) { diff --git a/tests/test_rocm_q4_dense_pair.cpp b/tests/test_rocm_q4_dense_pair.cpp index 11af5fde3..f219c3135 100644 --- a/tests/test_rocm_q4_dense_pair.cpp +++ b/tests/test_rocm_q4_dense_pair.cpp @@ -36,6 +36,19 @@ extern "C" int ds4_rocm_test_q4_prefill_k1024_tile4_policy( int disabled, int required); extern "C" void ds4_rocm_test_q4_prefill_wmma_reset(void); extern "C" uint64_t ds4_rocm_test_q4_prefill_wmma_get_calls(void); +extern "C" int ds4_rocm_test_q4_prefill_q8_wave32_policy( + int prefill_scope, int runtime_compatible, int enabled, int disabled, + int required); +extern "C" void ds4_rocm_test_q4_prefill_q8_wave32_reset(void); +extern "C" uint64_t ds4_rocm_test_q4_prefill_q8_wave32_get_calls(void); +extern "C" int ds4_rocm_test_q8_K_quantize_tensor( + ds4_gpu_tensor *out, const ds4_gpu_tensor *x, uint32_t in_dim, + uint32_t n_rows, int use_wave32); +extern "C" int ds4_rocm_test_q4_attn_q_b_yield_to_q8_wave32_policy( + uint32_t weight_type, uint32_t n_tok, int q8_wave32_required, + int f16_cache_required); +extern "C" int ds4_rocm_test_q4_pair_pre_enqueue_failure_policy( + int prefill_scope, int tile8_required, int q8_wave32_required); namespace { @@ -86,6 +99,12 @@ constexpr const char *kPrefillWmmaDisable = "DS4_ROCM_DISABLE_Q4_PREFILL_WMMA"; constexpr const char *kPrefillWmmaRequire = "DS4_ROCM_REQUIRE_Q4_PREFILL_WMMA"; +constexpr const char *kPrefillQ8Wave32Enable = + "DS4_ROCM_ENABLE_Q4_PREFILL_Q8_K_WAVE32"; +constexpr const char *kPrefillQ8Wave32Disable = + "DS4_ROCM_DISABLE_Q4_PREFILL_Q8_K_WAVE32"; +constexpr const char *kPrefillQ8Wave32Require = + "DS4_ROCM_REQUIRE_Q4_PREFILL_Q8_K_WAVE32"; constexpr const char *kQbF16Enable = "DS4_ROCM_ENABLE_Q4_ATTN_Q_B_F16_CACHE"; constexpr const char *kQbF16Disable = @@ -412,6 +431,30 @@ void fill_activation(std::vector *x, uint32_t n_tokens, } } +/* Exercise the two canonical edge cases for max selection: all-zero blocks + * and equal absolute maxima with opposite signs at different indices. */ +void fill_q8_wave32_activation(std::vector *x, uint32_t n_tokens, + uint32_t in_dim) { + x->resize((uint64_t)n_tokens * in_dim); + for (uint32_t token = 0; token < n_tokens; token++) { + for (uint32_t b = 0; b < in_dim / kQkK; b++) { + float *block = x->data() + (uint64_t)token * in_dim + b * kQkK; + if (((token + b) % 7u) == 0u) { + std::fill(block, block + kQkK, 0.0f); + continue; + } + for (uint32_t i = 0; i < kQkK; i++) { + const int value = + (int)((i * 29u + token * 17u + b * 11u) % 97u) - 48; + block[i] = (float)value / 16.0f; + } + const float first = ((token + b) & 1u) ? 4.0f : -4.0f; + block[5] = first; + block[224] = -first; + } + } +} + void quantize_q8_K_cpu(const float *x, block_q8_K_test *out) { float amax = 0.0f; float maxv = 0.0f; @@ -698,7 +741,7 @@ bool run_pair_case(const aligned_model &model, uint32_t n_tokens, std::vector dense1_host(count1); std::vector pair0_host(sentinel0.size()); std::vector pair1_host(sentinel1.size()); - if (dense_rc0 == 0 || dense_rc1 == 0 || pair_rc == 0 || + if (dense_rc0 == 0 || dense_rc1 == 0 || pair_rc <= 0 || !read_tensor(dense0.ptr, &dense0_host) || !read_tensor(dense1.ptr, &dense1_host) || !read_tensor(pair0.ptr, &pair0_host) || @@ -1145,6 +1188,318 @@ bool run_prefill_k1024_tile4_policy_oracle() { return ok; } +bool run_prefill_q8_wave32_policy_oracle() { + struct policy_case { + const char *label; + int scope; + int compatible; + int enabled; + int disabled; + int required; + int expected; + }; + const policy_case cases[] = { + {"default remains canonical", 1, 1, 0, 0, 0, 0}, + {"ENABLE selects compatible prefill", 1, 1, 1, 0, 0, 1}, + {"ENABLE falls back outside prefill", 0, 1, 1, 0, 0, 0}, + {"ENABLE falls back on incompatible runtime", 1, 0, 1, 0, 0, 0}, + {"DISABLE dominates ENABLE", 1, 1, 1, 1, 0, 0}, + {"REQUIRE is an opt-in", 1, 1, 0, 0, 1, 1}, + {"REQUIRE rejects non-prefill", 0, 1, 0, 0, 1, -1}, + {"REQUIRE rejects incompatible runtime", 1, 0, 0, 0, 1, -1}, + {"DISABLE dominates REQUIRE", 1, 1, 1, 1, 1, -1}, + }; + bool ok = true; + for (const policy_case &test : cases) { + const int got = ds4_rocm_test_q4_prefill_q8_wave32_policy( + test.scope, test.compatible, test.enabled, test.disabled, + test.required); + if (got != test.expected) { + std::fprintf(stderr, + "Q8_K wave32 policy %s: expected=%d got=%d FAIL\n", + test.label, test.expected, got); + ok = false; + } + } + struct q_b_policy_case { + const char *label; + uint32_t weight_type; + uint32_t n_tok; + int required; + int f16_required; + int expected; + }; + const q_b_policy_case q_b_cases[] = { + {"Q4 prefill REQUIRE yields", kQ4Type, 32u, 1, 0, 1}, + {"Q4 long prefill REQUIRE yields", kQ4Type, 4097u, 1, 0, 1}, + {"dual REQUIRE conflicts", kQ4Type, 32u, 1, 1, -1}, + {"Q4 decode does not claim prefill", kQ4Type, 8u, 1, 1, 0}, + {"Q8 prefill is unrelated", kQ8Type, 32u, 1, 1, 0}, + {"Q4 prefill without Q8 REQUIRE keeps F16 policy", + kQ4Type, 32u, 0, 1, 0}, + }; + for (const q_b_policy_case &test : q_b_cases) { + const int got = + ds4_rocm_test_q4_attn_q_b_yield_to_q8_wave32_policy( + test.weight_type, test.n_tok, test.required, + test.f16_required); + if (got != test.expected) { + std::fprintf(stderr, + "Q8_K wave32 q_b yield policy %s: " + "expected=%d got=%d FAIL\n", + test.label, test.expected, got); + ok = false; + } + } + std::fprintf(stderr, + "ROCm Q4 Q8_K wave32 policy oracle: selector=%zu q_b=%zu " + "%s\n", + sizeof(cases) / sizeof(cases[0]), + sizeof(q_b_cases) / sizeof(q_b_cases[0]), + ok ? "PASS" : "FAIL"); + return ok; +} + +bool run_pair_pre_enqueue_policy_oracle() { + struct policy_case { + const char *label; + int prefill_scope; + int tile8_required; + int q8_wave32_required; + int expected; + }; + const policy_case cases[] = { + {"optional prefill rejection falls back", 1, 0, 0, 0}, + {"TILE8 REQUIRE rejection fails closed", 1, 1, 0, -1}, + {"Q8 wave32 REQUIRE can use dense fallback", 1, 0, 1, 0}, + {"TILE8 remains strict with dual REQUIRE", 1, 1, 1, -1}, + {"decode ignores prefill TILE8 REQUIRE", 0, 1, 0, 0}, + {"optional decode rejection falls back", 0, 0, 0, 0}, + }; + + bool ok = true; + for (const policy_case &test : cases) { + const int got = ds4_rocm_test_q4_pair_pre_enqueue_failure_policy( + test.prefill_scope, test.tile8_required, + test.q8_wave32_required); + if (got != test.expected) { + std::fprintf(stderr, + "Q4 pair pre-enqueue policy %s: " + "expected=%d got=%d FAIL\n", + test.label, test.expected, got); + ok = false; + } + } + + /* Exercise the public selector without a GPU: null tensors reach the + * validation rejection only when the pair is selected. */ + env_snapshot tile8_disable(kPrefillDisable); + env_snapshot tile8_require(kPrefillRequire); + env_snapshot dense_pair_enable("DS4_ROCM_ENABLE_Q4_DENSE_PAIR"); + env_snapshot dense_pair_disable("DS4_ROCM_DISABLE_Q4_DENSE_PAIR"); + env_snapshot wmma_enable(kPrefillWmmaEnable); + env_snapshot wmma_disable(kPrefillWmmaDisable); + env_snapshot wmma_require(kPrefillWmmaRequire); + env_snapshot q8_enable(kPrefillQ8Wave32Enable); + env_snapshot q8_disable(kPrefillQ8Wave32Disable); + env_snapshot q8_require(kPrefillQ8Wave32Require); + (void)unsetenv(kPrefillDisable); + (void)setenv(kPrefillRequire, "1", 1); + (void)unsetenv("DS4_ROCM_ENABLE_Q4_DENSE_PAIR"); + (void)unsetenv("DS4_ROCM_DISABLE_Q4_DENSE_PAIR"); + (void)unsetenv(kPrefillWmmaEnable); + (void)unsetenv(kPrefillWmmaDisable); + (void)unsetenv(kPrefillWmmaRequire); + (void)unsetenv(kPrefillQ8Wave32Enable); + (void)unsetenv(kPrefillQ8Wave32Disable); + (void)unsetenv(kPrefillQ8Wave32Require); + const int required_validation = ds4_gpu_matmul_q4_K_pair_tensor( + nullptr, nullptr, nullptr, 0u, 0u, 0u, + kK, kM0, kM1, nullptr, 9u); + + (void)unsetenv(kPrefillRequire); + (void)setenv(kPrefillDisable, "1", 1); + const int opt_out = ds4_gpu_matmul_q4_K_pair_tensor( + nullptr, nullptr, nullptr, 0u, 0u, 0u, + kK, kM0, kM1, nullptr, 9u); + + (void)unsetenv(kPrefillDisable); + (void)setenv(kPrefillRequire, "1", 1); + (void)setenv("DS4_ROCM_ENABLE_Q4_DENSE_PAIR", "1", 1); + const int decode = ds4_gpu_matmul_q4_K_pair_tensor( + nullptr, nullptr, nullptr, 0u, 0u, 0u, + kK, kM0, kM1, nullptr, 1u); + if (required_validation != -1 || opt_out != 0 || decode != 0) { + std::fprintf(stderr, + "Q4 pair public pre-enqueue policy: " + "required=%d opt_out=%d decode=%d FAIL\n", + required_validation, opt_out, decode); + ok = false; + } + std::fprintf(stderr, + "ROCm Q4 pair pre-enqueue policy oracle: cases=%zu " + "required=%d opt_out=%d decode=%d %s\n", + sizeof(cases) / sizeof(cases[0]), required_validation, + opt_out, decode, ok ? "PASS" : "FAIL"); + return ok; +} + +bool run_prefill_q8_wave32_oracle(const aligned_model &model) { +#if DS4_TEST_HAS_HIP_RUNTIME + int device = 0; + hipDeviceProp_t properties = {}; + if (hipGetDevice(&device) != hipSuccess || + hipGetDeviceProperties(&properties, device) != hipSuccess || + properties.warpSize != 32 || + std::strncmp(properties.gcnArchName, "gfx1151", 7u) != 0) { + std::fprintf(stderr, + "ROCm Q4 Q8_K wave32 oracle: SKIP " + "(requires gfx1151 wave32)\n"); + return true; + } + + bool ok = true; + constexpr uint32_t n_tokens = 9u; + const uint32_t dimensions[] = {256u, 1024u, 4096u}; + for (uint32_t in_dim : dimensions) { + std::vector x; + fill_q8_wave32_activation(&x, n_tokens, in_dim); + const size_t block_count = + (size_t)n_tokens * (in_dim / kQkK); + const size_t bytes = block_count * sizeof(block_q8_K_test); + tensor_owner x_gpu(x.size() * sizeof(float)); + tensor_owner legacy_gpu(bytes); + tensor_owner wave_gpu(bytes); + if (!x_gpu.ptr || !legacy_gpu.ptr || !wave_gpu.ptr || + !write_tensor(x_gpu.ptr, x)) { + std::fprintf(stderr, + "Q8_K wave32 raw K=%u allocation/write FAIL\n", + in_dim); + ok = false; + continue; + } + const int legacy_rc = ds4_rocm_test_q8_K_quantize_tensor( + legacy_gpu.ptr, x_gpu.ptr, in_dim, n_tokens, 0); + const int wave_rc = ds4_rocm_test_q8_K_quantize_tensor( + wave_gpu.ptr, x_gpu.ptr, in_dim, n_tokens, 1); + std::vector legacy(block_count); + std::vector wave(block_count); + std::vector cpu(block_count); + const bool read_ok = legacy_rc != 0 && wave_rc != 0 && + ds4_gpu_tensor_read(legacy_gpu.ptr, 0, legacy.data(), bytes) != 0 && + ds4_gpu_tensor_read(wave_gpu.ptr, 0, wave.data(), bytes) != 0; + for (size_t i = 0; i < block_count; i++) { + quantize_q8_K_cpu(x.data() + i * kQkK, &cpu[i]); + } + const bool pair_equal = read_ok && + std::memcmp(legacy.data(), wave.data(), bytes) == 0; + const bool cpu_equal = read_ok && + std::memcmp(legacy.data(), cpu.data(), bytes) == 0; + std::fprintf(stderr, + "Q8_K wave32 raw K=%u blocks=%zu legacy=%d wave=%d " + "pair=%s cpu=%s %s\n", + in_dim, block_count, legacy_rc, wave_rc, + pair_equal ? "bitwise" : "MISMATCH", + cpu_equal ? "bitwise" : "MISMATCH", + pair_equal && cpu_equal ? "PASS" : "FAIL"); + ok = pair_equal && cpu_equal && ok; + } + + env_snapshot tile8_disable(kPrefillDisable); + env_snapshot tile8_require(kPrefillRequire); + env_snapshot wmma_disable(kPrefillWmmaDisable); + env_snapshot q8_enable(kPrefillQ8Wave32Enable); + env_snapshot q8_disable(kPrefillQ8Wave32Disable); + env_snapshot q8_require(kPrefillQ8Wave32Require); + (void)unsetenv(kPrefillDisable); + (void)setenv(kPrefillRequire, "1", 1); + (void)setenv(kPrefillWmmaDisable, "1", 1); + (void)unsetenv(kPrefillQ8Wave32Enable); + (void)unsetenv(kPrefillQ8Wave32Disable); + (void)setenv(kPrefillQ8Wave32Require, "1", 1); + + std::vector x; + fill_q8_wave32_activation(&x, n_tokens, kK); + tensor_owner x_gpu(x.size() * sizeof(float)); + tensor_owner out_gpu((uint64_t)n_tokens * kM0 * sizeof(float)); + bool dispatch_ok = x_gpu.ptr && out_gpu.ptr && write_tensor(x_gpu.ptr, x); + ds4_rocm_test_q4_prefill_q8_wave32_reset(); + const int rc = dispatch_ok ? ds4_gpu_matmul_quant_tensor( + out_gpu.ptr, model.data, model.size, model.weight0_offset, kQ4Type, + kK, kM0, x_gpu.ptr, n_tokens) : 0; + const uint64_t calls = + ds4_rocm_test_q4_prefill_q8_wave32_get_calls(); + std::vector got((uint64_t)n_tokens * kM0); + dispatch_ok = dispatch_ok && rc != 0 && calls == 1u && + read_tensor(out_gpu.ptr, &got); + if (dispatch_ok) { + const std::vector cpu = dense_reference( + model.data + model.weight0_offset, x, kM0, n_tokens); + dispatch_ok = close_to_cpu( + got, cpu, "Q8_K wave32 REQUIRE public dispatch vs CPU"); + } + + /* Q8 REQUIRE owns only activation quantization. With TILE8 opted out, + * the pair must yield so the graph can issue two dense calls, each of + * which still attests the wave32 quantizer. */ + tensor_owner fallback0((uint64_t)n_tokens * kM0 * sizeof(float)); + tensor_owner fallback1((uint64_t)n_tokens * kM1 * sizeof(float)); + (void)setenv(kPrefillDisable, "1", 1); + (void)unsetenv(kPrefillRequire); + (void)unsetenv(kPrefillQ8Wave32Disable); + ds4_rocm_test_q4_prefill_q8_wave32_reset(); + const int pair_fallback_rc = fallback0.ptr && fallback1.ptr + ? ds4_gpu_matmul_q4_K_pair_tensor( + fallback0.ptr, fallback1.ptr, model.data, model.size, + model.weight0_offset, model.weight1_offset, + kK, kM0, kM1, x_gpu.ptr, n_tokens) + : -1; + const uint64_t pair_fallback_calls = + ds4_rocm_test_q4_prefill_q8_wave32_get_calls(); + const int fallback_rc0 = pair_fallback_rc == 0 + ? ds4_gpu_matmul_quant_tensor( + fallback0.ptr, model.data, model.size, model.weight0_offset, + kQ4Type, kK, kM0, x_gpu.ptr, n_tokens) + : 0; + const int fallback_rc1 = fallback_rc0 != 0 + ? ds4_gpu_matmul_quant_tensor( + fallback1.ptr, model.data, model.size, model.weight1_offset, + kQ4Type, kK, kM1, x_gpu.ptr, n_tokens) + : 0; + const uint64_t dense_fallback_calls = + ds4_rocm_test_q4_prefill_q8_wave32_get_calls(); + const bool pair_fallback = pair_fallback_rc == 0 && + pair_fallback_calls == 0u && fallback_rc0 != 0 && + fallback_rc1 != 0 && dense_fallback_calls == 2u; + + (void)unsetenv(kPrefillDisable); + (void)setenv(kPrefillRequire, "1", 1); + (void)setenv(kPrefillQ8Wave32Disable, "1", 1); + ds4_rocm_test_q4_prefill_q8_wave32_reset(); + const int rejected_rc = ds4_gpu_matmul_quant_tensor( + out_gpu.ptr, model.data, model.size, model.weight0_offset, kQ4Type, + kK, kM0, x_gpu.ptr, n_tokens); + const uint64_t rejected_calls = + ds4_rocm_test_q4_prefill_q8_wave32_get_calls(); + const bool rejected = rejected_rc == 0 && rejected_calls == 0u; + std::fprintf(stderr, + "ROCm Q4 Q8_K wave32 dispatch: rc=%d calls=%llu " + "pair_fallback=%d/%llu dense_fallback=%d,%d/%llu " + "disable+require=%d/%llu %s\n", + rc, (unsigned long long)calls, pair_fallback_rc, + (unsigned long long)pair_fallback_calls, + fallback_rc0, fallback_rc1, + (unsigned long long)dense_fallback_calls, + rejected_rc, + (unsigned long long)rejected_calls, + dispatch_ok && pair_fallback && rejected ? "PASS" : "FAIL"); + return dispatch_ok && pair_fallback && rejected && ok; +#else + (void)model; + return true; +#endif +} + bool run_prefill_k1024_tile4_ssd_case(const aligned_model &model) { constexpr uint32_t n_tokens = 9u; const size_t logical_count = (size_t)n_tokens * kQbOutDim; @@ -1420,7 +1775,7 @@ bool run_prefill_pair_case(const aligned_model &model, uint32_t n_tokens, std::vector legacy1_host(sentinel1.size()); std::vector pair0_host(sentinel0.size()); std::vector pair1_host(sentinel1.size()); - if (legacy_rc0 == 0 || legacy_rc1 == 0 || pair_rc == 0 || + if (legacy_rc0 == 0 || legacy_rc1 == 0 || pair_rc <= 0 || !read_tensor(legacy0.ptr, &legacy0_host) || !read_tensor(legacy1.ptr, &legacy1_host) || !read_tensor(pair0.ptr, &pair0_host) || @@ -2032,9 +2387,21 @@ bool run_pair_guards(const aligned_model &model) { env_snapshot prefill_enable(kPrefillEnable); env_snapshot prefill_disable(kPrefillDisable); env_snapshot prefill_require(kPrefillRequire); + env_snapshot wmma_enable(kPrefillWmmaEnable); + env_snapshot wmma_disable(kPrefillWmmaDisable); + env_snapshot wmma_require(kPrefillWmmaRequire); + env_snapshot q8_enable(kPrefillQ8Wave32Enable); + env_snapshot q8_disable(kPrefillQ8Wave32Disable); + env_snapshot q8_require(kPrefillQ8Wave32Require); (void)unsetenv(kPrefillEnable); (void)setenv(kPrefillDisable, "1", 1); (void)unsetenv(kPrefillRequire); + (void)unsetenv(kPrefillWmmaEnable); + (void)unsetenv(kPrefillWmmaDisable); + (void)unsetenv(kPrefillWmmaRequire); + (void)unsetenv(kPrefillQ8Wave32Enable); + (void)unsetenv(kPrefillQ8Wave32Disable); + (void)unsetenv(kPrefillQ8Wave32Require); const int rc = ds4_gpu_matmul_q4_K_pair_tensor( out0.ptr, out1.ptr, model.data, model.size, model.weight0_offset, model.weight1_offset, @@ -2048,6 +2415,80 @@ bool run_pair_guards(const aligned_model &model) { ok = unchanged_after_rejected_call( out1.ptr, sentinel1, "pair n_tok=9 preserves out1") && ok; + (void)unsetenv(kPrefillDisable); + (void)setenv(kPrefillRequire, "1", 1); + if (!write_tensor(out0.ptr, sentinel0) || + !write_tensor(out1.ptr, sentinel1)) { + return false; + } + const int validation_rc = ds4_gpu_matmul_q4_K_pair_tensor( + out0.ptr, out1.ptr, model.data, model.size, + model.weight0_offset, model.weight1_offset, + kK - 1u, kM0, kM1, x_gpu.ptr, n_tokens); + ok = validation_rc == -1 && unchanged_after_rejected_call( + out0.ptr, sentinel0, + "required pair validation preserves out0") && ok; + ok = unchanged_after_rejected_call( + out1.ptr, sentinel1, + "required pair validation preserves out1") && ok; + if (validation_rc != -1) { + std::fprintf(stderr, + "required pair validation: expected rc=-1 got=%d FAIL\n", + validation_rc); + } + + if (!write_tensor(out0.ptr, sentinel0) || + !write_tensor(out1.ptr, sentinel1)) { + return false; + } + const int range_rc = ds4_gpu_matmul_q4_K_pair_tensor( + out0.ptr, out1.ptr, model.data, model.size, + model.size - 16u, model.weight1_offset, + kK, kM0, kM1, x_gpu.ptr, n_tokens); + ok = range_rc == -1 && unchanged_after_rejected_call( + out0.ptr, sentinel0, "required pair range preserves out0") && ok; + ok = unchanged_after_rejected_call( + out1.ptr, sentinel1, + "required pair range preserves out1") && ok; + if (range_rc != -1) { + std::fprintf(stderr, + "required pair range: expected rc=-1 got=%d FAIL\n", + range_rc); + } + + const size_t strict_shared_count = + (size_t)n_tokens * ((size_t)kM0 + kM1); + const std::vector strict_shared_sentinel = + sentinel_values(strict_shared_count); + tensor_owner strict_shared(strict_shared_count * sizeof(float)); + tensor_owner strict_overlap0(ds4_gpu_tensor_view( + strict_shared.ptr, 0u, + (uint64_t)n_tokens * kM0 * sizeof(float))); + tensor_owner strict_overlap1(ds4_gpu_tensor_view( + strict_shared.ptr, + ((uint64_t)n_tokens * kM0 - 1u) * sizeof(float), + (uint64_t)n_tokens * kM1 * sizeof(float))); + if (!strict_shared.ptr || !strict_overlap0.ptr || !strict_overlap1.ptr || + !write_tensor(strict_shared.ptr, strict_shared_sentinel)) { + std::fprintf(stderr, "required pair overlap guard: setup FAIL\n"); + return false; + } + const int strict_overlap_rc = ds4_gpu_matmul_q4_K_pair_tensor( + strict_overlap0.ptr, strict_overlap1.ptr, model.data, model.size, + model.weight0_offset, model.weight1_offset, + kK, kM0, kM1, x_gpu.ptr, n_tokens); + ok = strict_overlap_rc == -1 && unchanged_after_rejected_call( + strict_shared.ptr, strict_shared_sentinel, + "required pair partial-overlap preserves storage") && ok; + if (strict_overlap_rc != -1) { + std::fprintf(stderr, + "required pair partial-overlap: " + "expected rc=-1 got=%d FAIL\n", + strict_overlap_rc); + } + + (void)setenv(kPrefillDisable, "1", 1); + (void)unsetenv(kPrefillRequire); const size_t shared_count = (size_t)kM0 + kM1; const std::vector shared_sentinel = sentinel_values(shared_count); tensor_owner shared(shared_count * sizeof(float)); @@ -2164,6 +2605,9 @@ bool run_prefill_wmma_smoke(const aligned_model &model) { env_snapshot wmma_ssd_enable(kPrefillWmmaSsdEnable); env_snapshot wmma_disable(kPrefillWmmaDisable); env_snapshot wmma_require(kPrefillWmmaRequire); + env_snapshot q8_wave32_enable(kPrefillQ8Wave32Enable); + env_snapshot q8_wave32_disable(kPrefillQ8Wave32Disable); + env_snapshot q8_wave32_require(kPrefillQ8Wave32Require); (void)unsetenv(kPrefillEnable); (void)unsetenv(kPrefillDisable); @@ -2173,6 +2617,9 @@ bool run_prefill_wmma_smoke(const aligned_model &model) { (void)unsetenv(kPrefillWmmaSsdEnable); (void)unsetenv(kPrefillWmmaDisable); (void)unsetenv(kPrefillWmmaRequire); + (void)unsetenv(kPrefillQ8Wave32Enable); + (void)unsetenv(kPrefillQ8Wave32Disable); + (void)unsetenv(kPrefillQ8Wave32Require); ds4_rocm_test_q4_prefill_wmma_reset(); const int tile8_rc = ds4_gpu_matmul_quant_tensor( tile8_gpu.ptr, model.data, model.size, model.weight0_offset, kQ4Type, @@ -2452,6 +2899,9 @@ int main(int argc, char **argv) { env_snapshot wmma_ssd_enable(kPrefillWmmaSsdEnable); env_snapshot wmma_disable(kPrefillWmmaDisable); env_snapshot wmma_require(kPrefillWmmaRequire); + env_snapshot q8_wave32_enable(kPrefillQ8Wave32Enable); + env_snapshot q8_wave32_disable(kPrefillQ8Wave32Disable); + env_snapshot q8_wave32_require(kPrefillQ8Wave32Require); env_snapshot grouped_enable(kGroupedDecodeEnable); env_snapshot grouped_disable(kGroupedDecodeDisable); env_snapshot grouped_require(kGroupedDecodeRequire); @@ -2466,12 +2916,17 @@ int main(int argc, char **argv) { (void)unsetenv(kPrefillWmmaSsdEnable); (void)unsetenv(kPrefillWmmaDisable); (void)unsetenv(kPrefillWmmaRequire); + (void)unsetenv(kPrefillQ8Wave32Enable); + (void)unsetenv(kPrefillQ8Wave32Disable); + (void)unsetenv(kPrefillQ8Wave32Require); (void)unsetenv(kGroupedDecodeEnable); (void)unsetenv(kGroupedDecodeDisable); (void)unsetenv(kGroupedDecodeRequire); (void)unsetenv(kGroupedDecodeStats); - const bool policy_ok = run_prefill_k1024_tile4_policy_oracle(); + const bool policy_ok = run_prefill_k1024_tile4_policy_oracle() && + run_prefill_q8_wave32_policy_oracle() && + run_pair_pre_enqueue_policy_oracle(); if (run_policy_only || !policy_ok) return policy_ok ? 0 : 1; if (detect_rocm_device() <= 0) { @@ -2539,6 +2994,7 @@ int main(int argc, char **argv) { std::fprintf(stderr, "ROCm Q4 tiled prefill parity " "(forced DISABLE vs default+REQUIRE):\n"); + const bool q8_wave32_ok = run_prefill_q8_wave32_oracle(model); const bool prefill9_ok = run_prefill_parity_case( model, 9u, model.weight0_offset, kM0, true, "prefill K=4096 M=65 n_tok=9"); @@ -2595,7 +3051,7 @@ int main(int argc, char **argv) { const bool output_wmma_ok = run_attention_output_wmma_smoke(model); const bool gate_ok = run_prefill_gate_guards(model); - ok = prefill9_ok && prefill30_ok && prefill128_ok && + ok = q8_wave32_ok && prefill9_ok && prefill30_ok && prefill128_ok && prefill_tail9_ok && prefill_tail128_ok && prefill_single9_ok && prefill_q_b_tile4_ok && q_b_f16_null_qhalf_ok && From 5a7505ac98d0d1932934fadb6b61280eeddf250b Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Sat, 29 Aug 2026 09:14:13 +0200 Subject: [PATCH 145/189] perf(cuda): submit grouped q4 prefill in one grid --- cuda/mmq/ds4_mmq.cu | 122 ++++++++++++++++++----- cuda/mmq/ds4_mmq.h | 18 ++++ cuda/mmq/mmq.cuh | 139 ++++++++++++++++++--------- ds4_cuda.cu | 111 +++++++++++++++------ speed-bench/cuda_q4_prefill_bench.cu | 43 ++++++--- 5 files changed, 319 insertions(+), 114 deletions(-) diff --git a/cuda/mmq/ds4_mmq.cu b/cuda/mmq/ds4_mmq.cu index 194f49efc..406aa5a48 100644 --- a/cuda/mmq/ds4_mmq.cu +++ b/cuda/mmq/ds4_mmq.cu @@ -1149,17 +1149,17 @@ int ds4_mmq_q4_K_dense_pair_impl( const char *tag = "ds4_mmq_q4_K_dense_pair"; if (!W0 || !W1 || !X_f32 || !out0_f32 || !out1_f32) { fprintf(stderr, "%s: null pointer\n", tag); - return -1; + return DS4_MMQ_NOT_APPLICABLE; } if (M0 <= 0 || M1 <= 0 || N <= 0 || K <= 0 || K % 256 != 0) { fprintf(stderr, "%s: bad shape M0=%d M1=%d N=%d K=%d\n", tag, M0, M1, N, K); - return -1; + return DS4_MMQ_NOT_APPLICABLE; } if ((size_t)M0 > SIZE_MAX / (size_t)N / sizeof(float) || (size_t)M1 > SIZE_MAX / (size_t)N / sizeof(float)) { fprintf(stderr, "%s: output size overflow\n", tag); - return -1; + return DS4_MMQ_NOT_APPLICABLE; } const size_t out0_bytes = (size_t)M0 * (size_t)N * sizeof(float); const size_t out1_bytes = (size_t)M1 * (size_t)N * sizeof(float); @@ -1170,18 +1170,20 @@ int ds4_mmq_q4_K_dense_pair_impl( : (size_t)(out0_addr - out1_addr) < out1_bytes; if (outputs_overlap) { fprintf(stderr, "%s: output ranges overlap\n", tag); - return -1; + return DS4_MMQ_NOT_APPLICABLE; } const int dev = ggml_cuda_get_device(); const int cc = ggml_cuda_info().devices[dev].cc; - if (!ds4_mmq_k_tile_supported(tag, K, cc)) return -1; + if (!ds4_mmq_k_tile_supported(tag, K, cc)) { + return DS4_MMQ_NOT_APPLICABLE; + } ggml_backend_cuda_context *ctx = get_ctx_for_device(dev); if (!ctx) { fprintf(stderr, "%s: failed to get cuda context for device %d\n", tag, dev); - return -1; + return DS4_MMQ_NOT_APPLICABLE; } ds4_pool_set_stream(stream); @@ -1194,13 +1196,13 @@ int ds4_mmq_q4_K_dense_pair_impl( if ((size_t)N > SIZE_MAX / bytes_per_col || slack_blocks > SIZE_MAX / sizeof(block_q8_1_mmq)) { fprintf(stderr, "%s: activation scratch size overflow\n", tag); - return -1; + return DS4_MMQ_NOT_APPLICABLE; } const size_t payload_bytes = (size_t)N * bytes_per_col; const size_t slack_bytes = slack_blocks * sizeof(block_q8_1_mmq); if (payload_bytes > SIZE_MAX - slack_bytes) { fprintf(stderr, "%s: activation scratch size overflow\n", tag); - return -1; + return DS4_MMQ_NOT_APPLICABLE; } const size_t nbytes_q8_1 = payload_bytes + slack_bytes; @@ -1307,8 +1309,11 @@ int ds4_mmq_q4_K_dense_pair_impl( * is token-major [N][G][K], while MMQ stores directly into token-major * [N][G][M]. Quantizing the strided source as G channels removes the old * pack/unpack copies and shares one scratch allocation/quantizer launch. - * Keep one MMQ launch per group: its stream-K partition and reduction tree - * are therefore identical to the established per-group dense call. */ + * + * single_grid=false retains the established one-MMQ-launch-per-group path. + * single_grid=true maps groups to grid.z in one launch, but isolates each + * z-slice's stream-k coordinate space. Its grid.x, partial-K ownership and + * fixup order are therefore identical to the former per-group invocation. */ int ds4_mmq_q4_K_grouped_dense_impl( const void *W, const float *X, @@ -1317,17 +1322,23 @@ int ds4_mmq_q4_K_grouped_dense_impl( int N, int K, int n_groups, + bool single_grid, cudaStream_t stream) { - const char *tag = "ds4_mmq_q4_K_grouped_dense"; + const char *tag = single_grid + ? "ds4_mmq_q4_K_grouped_dense_single_grid" + : "ds4_mmq_q4_K_grouped_dense"; + const int pre_enqueue_failure = single_grid + ? DS4_MMQ_NOT_APPLICABLE + : -1; if (!W || !X || !out) { fprintf(stderr, "%s: null pointer\n", tag); - return -1; + return pre_enqueue_failure; } if (M <= 0 || N <= 0 || K <= 0 || n_groups <= 0 || K % QK_K != 0) { fprintf(stderr, "%s: bad shape M=%d N=%d K=%d groups=%d\n", tag, M, N, K, n_groups); - return -1; + return pre_enqueue_failure; } const int dev = ggml_cuda_get_device(); @@ -1336,7 +1347,7 @@ int ds4_mmq_q4_K_grouped_dense_impl( if (!ctx) { fprintf(stderr, "%s: failed to get cuda context for device %d\n", tag, dev); - return -1; + return pre_enqueue_failure; } ds4_pool_set_stream(stream); @@ -1345,23 +1356,39 @@ int ds4_mmq_q4_K_grouped_dense_impl( (size_t)ne10_padded / (4u * (size_t)QK8_1); const size_t bytes_per_col = blocks_per_col * sizeof(block_q8_1_mmq); - if ((size_t)N > SIZE_MAX / bytes_per_col) return -1; + if ((size_t)N > SIZE_MAX / bytes_per_col) return pre_enqueue_failure; const size_t channel_bytes = (size_t)N * bytes_per_col; - if ((size_t)n_groups > SIZE_MAX / channel_bytes) return -1; + if ((size_t)n_groups > SIZE_MAX / channel_bytes) { + return pre_enqueue_failure; + } const size_t payload_bytes = (size_t)n_groups * channel_bytes; const size_t slack_blocks = (size_t)get_mmq_x_max_host(cc); - if (slack_blocks > SIZE_MAX / sizeof(block_q8_1_mmq)) return -1; + if (slack_blocks > SIZE_MAX / sizeof(block_q8_1_mmq)) { + return pre_enqueue_failure; + } const size_t slack_bytes = slack_blocks * sizeof(block_q8_1_mmq); - if (payload_bytes > SIZE_MAX - slack_bytes) return -1; + if (payload_bytes > SIZE_MAX - slack_bytes) return pre_enqueue_failure; const int64_t row_blocks = (int64_t)K / QK_K; if ((size_t)M > SIZE_MAX / (size_t)row_blocks / - sizeof(block_q4_K)) return -1; + sizeof(block_q4_K)) return pre_enqueue_failure; const size_t group_weight_bytes = (size_t)M * (size_t)row_blocks * sizeof(block_q4_K); + const int64_t group_weight_blocks = (int64_t)M * row_blocks; const int64_t low_dim = (int64_t)M * n_groups; if (low_dim > INT_MAX || - (uint64_t)low_dim > UINT64_MAX / (uint64_t)N) return -1; + (uint64_t)low_dim > UINT64_MAX / (uint64_t)N) { + return pre_enqueue_failure; + } + /* The grouped kernel ABI narrows strides and weight-block offsets to int. + * Reject before allocating or enqueueing so an optional caller can safely + * fall back to the established per-group launch loop. */ + if (single_grid && + (n_groups > 65535 || group_weight_blocks > INT_MAX || + group_weight_blocks * (int64_t)n_groups > INT_MAX || + channel_bytes / sizeof(int) > (size_t)INT_MAX)) { + return DS4_MMQ_NOT_APPLICABLE; + } ggml_cuda_pool_alloc y_q8_1( ctx->pool(), payload_bytes + slack_bytes); @@ -1389,6 +1416,42 @@ int ds4_mmq_q4_K_grouped_dense_impl( (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_VOLTA) || GGML_CUDA_CC_IS_CDNA(cc); + if (single_grid) { + const mmq_args args = { + /*x=*/(const char *)W, + /*type_x=*/GGML_TYPE_Q4_K, + /*y=*/(const int *)y_q8_1.get(), + /*ids_dst=*/nullptr, + /*expert_bounds=*/nullptr, + /*dst=*/out, + /*ncols_x=*/(int64_t)K, + /*nrows_x=*/(int64_t)M, + /*ncols_dst=*/(int64_t)N, + /*stride_row_x=*/row_blocks, + /*ncols_y=*/(int64_t)N, + /*nrows_dst=*/low_dim, + /*nchannels_x=*/(int64_t)n_groups, + /*nchannels_y=*/(int64_t)n_groups, + /*stride_channel_x=*/group_weight_blocks, + /*stride_channel_y=*/(int64_t)(channel_bytes / sizeof(int)), + /*stride_channel_dst=*/(int64_t)M, + /*nsamples_x=*/1, + /*nsamples_y=*/1, + /*stride_sample_x=*/0, + /*stride_sample_y=*/0, + /*stride_sample_dst=*/0, + /*use_stream_k=*/use_stream_k, + /*ncols_max=*/(int64_t)N, + }; + mul_mat_q_case_grouped_channels(*ctx, args, stream); + err = cudaGetLastError(); + if (err != cudaSuccess) { + fprintf(stderr, "%s: grouped launch failed: %s\n", + tag, cudaGetErrorString(err)); + return -3; + } + return 0; + } for (int g = 0; g < n_groups; ++g) { const mmq_args args = { /*x=*/(const char *)W + (size_t)g * group_weight_bytes, @@ -1667,7 +1730,14 @@ extern "C" int ds4_mmq_q4_K_grouped_dense( const void *W, const float *X, float *out, int M, int N, int K, int n_groups, cudaStream_t stream) { return ds4_mmq_q4_K_grouped_dense_impl( - W, X, out, M, N, K, n_groups, stream); + W, X, out, M, N, K, n_groups, false, stream); +} + +extern "C" int ds4_mmq_q4_K_grouped_dense_single_grid( + const void *W, const float *X, float *out, + int M, int N, int K, int n_groups, cudaStream_t stream) { + return ds4_mmq_q4_K_grouped_dense_impl( + W, X, out, M, N, K, n_groups, true, stream); } extern "C" int ds4_mmq_mxfp4_dense( @@ -4430,21 +4500,21 @@ int ds4_mmq_dense_pair_vec_impl( if (!W0 || !W1 || !X_f32 || !out0_f32 || !out1_f32) { fprintf(stderr, "%s: null pointer\n", tag); - return -1; + return DS4_MMQ_NOT_APPLICABLE; } if (M0 <= 0 || M1 <= 0 || N <= 0 || K <= 0) { fprintf(stderr, "%s: bad shape M0=%d M1=%d N=%d K=%d\n", tag, M0, M1, N, K); - return -1; + return DS4_MMQ_NOT_APPLICABLE; } if (K % 256 != 0) { fprintf(stderr, "%s: K=%d must be a multiple of 256\n", tag, K); - return -1; + return DS4_MMQ_NOT_APPLICABLE; } if (N > MMVQ_MAX_BATCH_SIZE) { fprintf(stderr, "%s: N=%d exceeds MMVQ_MAX_BATCH_SIZE=%d\n", tag, N, MMVQ_MAX_BATCH_SIZE); - return -1; + return DS4_MMQ_NOT_APPLICABLE; } const int dev = ggml_cuda_get_device(); @@ -4452,7 +4522,7 @@ int ds4_mmq_dense_pair_vec_impl( if (!ctx) { fprintf(stderr, "%s: failed to get cuda context for device %d\n", tag, dev); - return -1; + return DS4_MMQ_NOT_APPLICABLE; } ds4_pool_set_stream(stream); diff --git a/cuda/mmq/ds4_mmq.h b/cuda/mmq/ds4_mmq.h index 3fd6018a9..85a19cf3b 100644 --- a/cuda/mmq/ds4_mmq.h +++ b/cuda/mmq/ds4_mmq.h @@ -193,6 +193,8 @@ int ds4_mmq_q4_K_dense( // ds4_mmq_q4_K_dense_pair_vec: N is not limited to the MMVQ batch ceiling, // M0 and M1 may differ, and each leg preserves ds4_mmq_q4_K_dense's // reduction and output layout. The two output ranges must be disjoint. +// Shape/capability rejection returns DS4_MMQ_NOT_APPLICABLE before enqueue; +// negative values report an attempted-path launch failure. int ds4_mmq_q4_K_dense_pair( const void * W0_q4_K, const void * W1_q4_K, @@ -219,6 +221,20 @@ int ds4_mmq_q4_K_grouped_dense( int n_groups, cudaStream_t stream); +// Opt-in sibling of ds4_mmq_q4_K_grouped_dense that submits all groups in one +// grid, with group selected by grid.z. Stream-k coordinates and fixup storage +// are isolated per grid.z slice, preserving the reduction tree and output bits +// of the established one-launch-per-group implementation. +int ds4_mmq_q4_K_grouped_dense_single_grid( + const void * W_q4_K, + const float * X_f32, + float * out_f32, + int M, + int N, + int K, + int n_groups, + cudaStream_t stream); + int ds4_mmq_mxfp4_dense( const void * W_mxfp4, const float * X_f32, @@ -1066,6 +1082,8 @@ void ds4_mmq_q4_K_k1024_persistent_counters( // activation quantization. Each output is dispatched through the same MMVQ // entry as ds4_mmq_q4_K_dense_vec, so its reduction and output bits are // unchanged; M0 and M1 may differ (the DS4 Q-A/KV decode shape does). +// Shape/capability rejection returns DS4_MMQ_NOT_APPLICABLE before enqueue; +// negative values report an attempted-path launch failure. int ds4_mmq_q4_K_dense_pair_vec( const void * W0_q4_K, const void * W1_q4_K, diff --git a/cuda/mmq/mmq.cuh b/cuda/mmq/mmq.cuh index 177bc1b03..0b03cc2d6 100644 --- a/cuda/mmq/mmq.cuh +++ b/cuda/mmq/mmq.cuh @@ -3779,7 +3779,7 @@ static __device__ __forceinline__ void mul_mat_q_process_tile( // The mul_mat_q kernel implements "stream-k" work partitioning as described in https://arxiv.org/abs/2301.03598 -template +template #if defined(GGML_USE_HIP) #if defined(RDNA4) || defined(RDNA3) || defined(RDNA2) || defined(CDNA) || defined(GCN) __launch_bounds__(ggml_cuda_get_physical_warp_size()*mmq_get_nwarps_device(), 2) @@ -3813,6 +3813,22 @@ static __global__ void mul_mat_q( const uint32_t nty = (nrows_x + mmq_y - 1) / mmq_y; // Number of tiles y + /* Dense grouped dispatch: grid.z is an outer channel selector while each + * z-slice keeps the exact grid.x / stream-k partition of the former + * one-launch-per-channel path. Offset the three channel bases here and + * let the ordinary tile code see a single logical channel. This preserves + * each output's K reduction tree while removing the host launch loop. */ + const int grid_z_channel = grid_z_channels ? (int)blockIdx.z : 0; + const int grid_z_offset_x = grid_z_channels + ? grid_z_channel * stride_channel_x : 0; + if constexpr (grid_z_channels) { + y += (int64_t)grid_z_channel * stride_channel_y; + dst += (int64_t)grid_z_channel * stride_channel_dst; + if (tmp_fixup != nullptr) { + tmp_fixup += (int64_t)grid_z_channel * gridDim.x * mmq_x * mmq_y; + } + } + // Initialize the ids for writing back data with just the index. // For regular matrix multiplications this is never changed. // For MoE the correct indices are loaded from ids_dst. @@ -3832,9 +3848,13 @@ static __global__ void mul_mat_q( // On non-CDNA AMD or old CUDA the performance with stream-k was worse, use conventional tiling instead: #if (defined(GGML_USE_HIP) && !defined(CDNA)) || __CUDA_ARCH__ < GGML_CUDA_CC_VOLTA { - const uint2 tmp2 = fast_div_modulo(blockIdx.z, nchannels_y); - const int wt = tmp2.x; - const int zt = tmp2.y; + int wt = 0; + int zt = 0; + if constexpr (!grid_z_channels) { + const uint2 tmp2 = fast_div_modulo(blockIdx.z, nchannels_y); + wt = tmp2.x; + zt = tmp2.y; + } const int jt = blockIdx.y; const int it = blockIdx.x; @@ -3883,7 +3903,7 @@ static __global__ void mul_mat_q( const int tile_x_max_i = nrows_x - it*mmq_y - 1; const int tile_y_max_j = col_diff - jt*mmq_x - 1; - const int offset_x = fastdiv(wt, sample_ratio)*stride_sample_x + fastdiv(zt, channel_ratio)*stride_channel_x + it*mmq_y*stride_row_x; + const int offset_x = grid_z_offset_x + fastdiv(wt, sample_ratio)*stride_sample_x + fastdiv(zt, channel_ratio)*stride_channel_x + it*mmq_y*stride_row_x; constexpr bool fixup = false; mul_mat_q_process_tile @@ -3970,7 +3990,7 @@ static __global__ void mul_mat_q( const int tile_x_max_i = nrows_x - it*mmq_y - 1; const int tile_y_max_j = col_diff - jt*mmq_x - 1; - const int offset_x = fastdiv(wt, sample_ratio)*stride_sample_x + fastdiv(zt, channel_ratio)*stride_channel_x + it*mmq_y*stride_row_x; + const int offset_x = grid_z_offset_x + fastdiv(wt, sample_ratio)*stride_sample_x + fastdiv(zt, channel_ratio)*stride_channel_x + it*mmq_y*stride_row_x; constexpr bool fixup = false; // All but (potentially) the last iterations write their data to dst rather than the fixup buffer. mul_mat_q_process_tile @@ -4040,7 +4060,7 @@ static __global__ void mul_mat_q( const int tile_x_max_i = nrows_x - it*mmq_y - 1; const int tile_y_max_j = col_diff - jt*mmq_x - 1; - const int offset_x = fastdiv(wt, sample_ratio)*stride_sample_x + fastdiv(zt, channel_ratio)*stride_channel_x + it*mmq_y*stride_row_x; + const int offset_x = grid_z_offset_x + fastdiv(wt, sample_ratio)*stride_sample_x + fastdiv(zt, channel_ratio)*stride_channel_x + it*mmq_y*stride_row_x; constexpr bool fixup = true; // Last index writes its data to fixup buffer to avoid data races with other blocks. mul_mat_q_process_tile @@ -4049,7 +4069,7 @@ static __global__ void mul_mat_q( blocks_per_ne00.z, x_soa, soa_blocks); } -template +template __launch_bounds__(ggml_cuda_get_physical_warp_size()*mmq_get_nwarps_device()/2, 1) static __global__ void mul_mat_q_stream_k_fixup( const int32_t * __restrict__ ids_dst, const int32_t * __restrict__ expert_bounds, float * __restrict__ dst, @@ -4064,6 +4084,11 @@ static __global__ void mul_mat_q_stream_k_fixup( constexpr int nwarps = mmq_get_nwarps_device()/2; constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + if constexpr (grid_z_channels) { + dst += (int64_t)blockIdx.z * stride_channel_dst; + tmp_last_tile += (int64_t)blockIdx.z * gridDim.x * mmq_x * mmq_y; + } + float sum[mmq_x / nwarps] = {0.0f}; const int i = blockIdx.y*warp_size + threadIdx.x; @@ -4225,7 +4250,7 @@ static size_t mmq_get_nbytes_shared(const int mmq_x, const int mmq_y, const int return nbs_ids + nbs_x + GGML_PAD(nbs_y, nwarps*warp_size*sizeof(int)); } -template +template static void launch_mul_mat_q(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) { const int id = ggml_cuda_get_device(); const int cc = ggml_cuda_info().devices[id].cc; @@ -4238,30 +4263,38 @@ static void launch_mul_mat_q(ggml_backend_cuda_context & ctx, const mmq_args & a const int nbytes_shared = mmq_get_nbytes_shared(mmq_x, mmq_y, cc, warp_size, nwarps); - CUDA_SET_SHARED_MEMORY_LIMIT((mul_mat_q), nbytes_shared); - CUDA_SET_SHARED_MEMORY_LIMIT((mul_mat_q), nbytes_shared); + CUDA_SET_SHARED_MEMORY_LIMIT((mul_mat_q), nbytes_shared); + CUDA_SET_SHARED_MEMORY_LIMIT((mul_mat_q), nbytes_shared); const int nty = (args.nrows_x + mmq_y - 1) / mmq_y; const int ntx = (args.ncols_max + mmq_x - 1) / mmq_x; - const int ntzw = args.nchannels_y * args.nsamples_y; - const dim3 block_nums_xy_tiling(nty, ntx, ntzw); + const int ntzw = grid_z_channels ? 1 : args.nchannels_y * args.nsamples_y; + const int grid_z = grid_z_channels ? args.nchannels_y : ntzw; + const dim3 block_nums_xy_tiling(nty, ntx, grid_z); GGML_ASSERT(args.nchannels_y % args.nchannels_x == 0); GGML_ASSERT(args.nsamples_y % args.nsamples_x == 0); - const int channel_ratio = args.nchannels_y / args.nchannels_x; - const int sample_ratio = args.nsamples_y / args.nsamples_x; + if constexpr (grid_z_channels) { + GGML_ASSERT(args.ids_dst == nullptr && args.expert_bounds == nullptr); + GGML_ASSERT(args.nchannels_x == args.nchannels_y); + GGML_ASSERT(args.nsamples_x == 1 && args.nsamples_y == 1); + } + const int channel_ratio = grid_z_channels ? 1 : args.nchannels_y / args.nchannels_x; + const int sample_ratio = grid_z_channels ? 1 : args.nsamples_y / args.nsamples_x; + const int logical_nchannels_y = grid_z_channels ? 1 : args.nchannels_y; + const int logical_nsamples_y = grid_z_channels ? 1 : args.nsamples_y; const uint3 blocks_per_ne00_fd = init_fastdiv_values(args.ncols_x / ggml_cuda_type_traits::qk); const uint3 ntx_fd = init_fastdiv_values(ntx); - const uint3 nchannels_y_fd = init_fastdiv_values(args.nchannels_y); - const uint3 nsamples_y_fd = init_fastdiv_values(args.nsamples_y); + const uint3 nchannels_y_fd = init_fastdiv_values(logical_nchannels_y); + const uint3 nsamples_y_fd = init_fastdiv_values(logical_nsamples_y); const uint3 channel_ratio_fd = init_fastdiv_values(channel_ratio); const uint3 sample_ratio_fd = init_fastdiv_values(sample_ratio); if (!args.use_stream_k) { if (args.nrows_x % mmq_y == 0) { constexpr bool need_check = false; - mul_mat_q<<>> + mul_mat_q<<>> (args.x, args.y, args.ids_dst, args.expert_bounds, args.dst, nullptr, blocks_per_ne00_fd, args.nrows_x, args.ncols_dst, args.stride_row_x, args.ncols_y, args.nrows_dst, channel_ratio_fd, nchannels_y_fd, args.stride_channel_x, args.stride_channel_y, args.stride_channel_dst, @@ -4269,7 +4302,7 @@ static void launch_mul_mat_q(ggml_backend_cuda_context & ctx, const mmq_args & a ntx_fd, args.x_soa, args.soa_blocks); } else { constexpr bool need_check = true; - mul_mat_q<<>> + mul_mat_q<<>> (args.x, args.y, args.ids_dst, args.expert_bounds, args.dst, nullptr, blocks_per_ne00_fd, args.nrows_x, args.ncols_dst, args.stride_row_x, args.ncols_y, args.nrows_dst, channel_ratio_fd, nchannels_y_fd, args.stride_channel_x, args.stride_channel_y, args.stride_channel_dst, @@ -4284,7 +4317,8 @@ static void launch_mul_mat_q(ggml_backend_cuda_context & ctx, const mmq_args & a const int ntiles_dst = ntx * nty * ntzw; const int tiles_nwaves = (ntiles_dst + nsm - 1) / nsm; const int tiles_efficiency_percent = 100 * ntiles_dst / (nsm*tiles_nwaves); - const dim3 block_nums_stream_k(GGML_CUDA_CC_IS_NVIDIA(cc) && tiles_efficiency_percent >= 90 ? ntiles_dst : nsm, 1, 1); + const unsigned stream_k_grid_x = GGML_CUDA_CC_IS_NVIDIA(cc) && tiles_efficiency_percent >= 90 ? ntiles_dst : nsm; + const dim3 block_nums_stream_k(stream_k_grid_x, 1, grid_z_channels ? args.nchannels_y : 1); GGML_ASSERT(ntiles_dst * blocks_per_ne00_fd.z < (1 << 30)); // Assert that variable kbc will not overflow. @@ -4293,18 +4327,19 @@ static void launch_mul_mat_q(ggml_backend_cuda_context & ctx, const mmq_args & a ggml_cuda_pool & pool = ctx.pool(id); ggml_cuda_pool_alloc tmp_fixup(pool); if (fixup_needed) { - tmp_fixup.alloc(block_nums_stream_k.x * mmq_x*mmq_y); + tmp_fixup.alloc((size_t)block_nums_stream_k.x * (size_t)block_nums_stream_k.z * mmq_x*mmq_y); CUDA_CHECK(cudaMemsetAsync(tmp_fixup.ptr, 0, - (size_t)block_nums_stream_k.x * (size_t)mmq_x * (size_t)mmq_y * sizeof(float), + (size_t)block_nums_stream_k.x * (size_t)block_nums_stream_k.z * + (size_t)mmq_x * (size_t)mmq_y * sizeof(float), stream)); } - const dim3 block_nums_fixup(block_nums_stream_k.x, mmq_y/warp_size, 1); + const dim3 block_nums_fixup(block_nums_stream_k.x, mmq_y/warp_size, block_nums_stream_k.z); const dim3 block_dims_fixup(block_dims.x, block_dims.y/2, block_dims.z); if (args.nrows_x % mmq_y == 0) { constexpr bool need_check = false; - mul_mat_q<<>> + mul_mat_q<<>> (args.x, args.y, args.ids_dst, args.expert_bounds, args.dst, tmp_fixup.ptr, blocks_per_ne00_fd, args.nrows_x, args.ncols_dst, args.stride_row_x, args.ncols_y, args.nrows_dst, channel_ratio_fd, nchannels_y_fd, args.stride_channel_x, args.stride_channel_y, args.stride_channel_dst, @@ -4316,13 +4351,13 @@ static void launch_mul_mat_q(ggml_backend_cuda_context & ctx, const mmq_args & a } CUDA_CHECK(cudaGetLastError()); - mul_mat_q_stream_k_fixup<<>> + mul_mat_q_stream_k_fixup<<>> (args.ids_dst, args.expert_bounds, args.dst, tmp_fixup.ptr, blocks_per_ne00_fd, args.nrows_x, args.ncols_dst, args.nrows_dst, nchannels_y_fd, args.stride_channel_dst, nsamples_y_fd, args.stride_sample_dst, ntx_fd); } else { constexpr bool need_check = true; - mul_mat_q<<>> + mul_mat_q<<>> (args.x, args.y, args.ids_dst, args.expert_bounds, args.dst, tmp_fixup.ptr, blocks_per_ne00_fd, args.nrows_x, args.ncols_dst, args.stride_row_x, args.ncols_y, args.nrows_dst, channel_ratio_fd, nchannels_y_fd, args.stride_channel_x, args.stride_channel_y, args.stride_channel_dst, @@ -4334,15 +4369,15 @@ static void launch_mul_mat_q(ggml_backend_cuda_context & ctx, const mmq_args & a } CUDA_CHECK(cudaGetLastError()); - mul_mat_q_stream_k_fixup<<>> + mul_mat_q_stream_k_fixup<<>> (args.ids_dst, args.expert_bounds, args.dst, tmp_fixup.ptr, blocks_per_ne00_fd, args.nrows_x, args.ncols_dst, args.nrows_dst, nchannels_y_fd, args.stride_channel_dst, nsamples_y_fd, args.stride_sample_dst, ntx_fd); } } -template -void mul_mat_q_case(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) { +template +static void mul_mat_q_case_impl(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) { const int id = ggml_cuda_get_device(); const int cc = ggml_cuda_info().devices[id].cc; const size_t smpbo = ggml_cuda_info().devices[id].smpbo; @@ -4372,52 +4407,52 @@ void mul_mat_q_case(ggml_backend_cuda_context & ctx, const mmq_args & args, cuda switch (mmq_x_best) { case 8: - launch_mul_mat_q(ctx, args, stream); + launch_mul_mat_q(ctx, args, stream); break; case 16: - launch_mul_mat_q(ctx, args, stream); + launch_mul_mat_q(ctx, args, stream); break; case 24: - launch_mul_mat_q(ctx, args, stream); + launch_mul_mat_q(ctx, args, stream); break; case 32: - launch_mul_mat_q(ctx, args, stream); + launch_mul_mat_q(ctx, args, stream); break; case 40: - launch_mul_mat_q(ctx, args, stream); + launch_mul_mat_q(ctx, args, stream); break; case 48: - launch_mul_mat_q(ctx, args, stream); + launch_mul_mat_q(ctx, args, stream); break; case 56: - launch_mul_mat_q(ctx, args, stream); + launch_mul_mat_q(ctx, args, stream); break; case 64: - launch_mul_mat_q(ctx, args, stream); + launch_mul_mat_q(ctx, args, stream); break; case 72: - launch_mul_mat_q(ctx, args, stream); + launch_mul_mat_q(ctx, args, stream); break; case 80: - launch_mul_mat_q(ctx, args, stream); + launch_mul_mat_q(ctx, args, stream); break; case 88: - launch_mul_mat_q(ctx, args, stream); + launch_mul_mat_q(ctx, args, stream); break; case 96: - launch_mul_mat_q(ctx, args, stream); + launch_mul_mat_q(ctx, args, stream); break; case 104: - launch_mul_mat_q(ctx, args, stream); + launch_mul_mat_q(ctx, args, stream); break; case 112: - launch_mul_mat_q(ctx, args, stream); + launch_mul_mat_q(ctx, args, stream); break; case 120: - launch_mul_mat_q(ctx, args, stream); + launch_mul_mat_q(ctx, args, stream); break; case 128: - launch_mul_mat_q(ctx, args, stream); + launch_mul_mat_q(ctx, args, stream); break; default: fprintf(stderr, "mmq_x_best=%d\n", mmq_x_best); @@ -4426,6 +4461,20 @@ void mul_mat_q_case(ggml_backend_cuda_context & ctx, const mmq_args & args, cuda } } +template +void mul_mat_q_case(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) { + mul_mat_q_case_impl(ctx, args, stream); +} + +/* Dense-only grouped entry used by ds4's Q4 attention output-A path. Each + * grid.z slice is reduction-isolated, so the result remains bit-identical to + * invoking mul_mat_q_case once per channel on the same stream. */ +template +void mul_mat_q_case_grouped_channels( + ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) { + mul_mat_q_case_impl(ctx, args, stream); +} + #define DECL_MMQ_CASE(type) \ template void mul_mat_q_case(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) \ diff --git a/ds4_cuda.cu b/ds4_cuda.cu index 06414af27..3d7522aaf 100644 --- a/ds4_cuda.cu +++ b/ds4_cuda.cu @@ -40606,10 +40606,13 @@ static int cuda_matmul_q4_K_pair_tensor_impl( (unsigned long long)out0_dim, (unsigned long long)out1_dim, (unsigned long long)n_tok, - g_cuda_test_q4_mmq_strict - ? "; strict benchmark mode rejects fallback" - : "; falling back"); - if (g_cuda_test_q4_mmq_strict) return 0; + rc == DS4_MMQ_NOT_APPLICABLE + ? (g_cuda_test_q4_mmq_strict + ? "; strict benchmark mode rejects fallback" + : "; falling back") + : "; attempted path failed closed"); + if (rc != DS4_MMQ_NOT_APPLICABLE) return -1; + if (g_cuda_test_q4_mmq_strict) return -1; if (gb10_canonical) return 0; } if (g_cuda_test_q4_mmq_strict) { @@ -40620,7 +40623,7 @@ static int cuda_matmul_q4_K_pair_tensor_impl( (unsigned long long)out0_dim, (unsigned long long)out1_dim, (unsigned long long)n_tok); - return 0; + return -1; } /* The Q8_K pair is the established decode/microbatch rollback only. @@ -40641,7 +40644,7 @@ static int cuda_matmul_q4_K_pair_tensor_impl( q8_K_quantize_kernel<<>>( xq, (const float *)x->ptr, (uint32_t)in_dim, (uint32_t)n_tok); if (!cuda_ok(cudaGetLastError(), "q4_K dense pair quantize launch")) { - return 0; + return -1; } const uint64_t max_out = out0_dim > out1_dim ? out0_dim : out1_dim; @@ -40658,7 +40661,9 @@ static int cuda_matmul_q4_K_pair_tensor_impl( (uint32_t)out0_dim, (uint32_t)out1_dim, (uint32_t)n_tok); - return cuda_ok(cudaGetLastError(), "q4_K dense pair matmul launch"); + return cuda_ok(cudaGetLastError(), "q4_K dense pair matmul launch") + ? 1 + : -1; } static uint64_t g_q4_attn_hc_oracle_calls; @@ -43912,22 +43917,26 @@ extern "C" int ds4_gpu_attention_output_q4_K_batch_tensor( "DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_BATCH", 0); const int grouped_prefill_require = cuda_env_flag_enabled( "DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_PREFILL", 0); + const int grouped_single_grid_require = cuda_env_flag_enabled( + "DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_SINGLE_GRID", 0); + const int any_grouped_require = grouped_batch_require || + grouped_prefill_require || grouped_single_grid_require; if (!out || !low || !group_tmp || !low_tmp || !heads || !model_map || group_dim == 0 || rank == 0 || n_groups == 0 || out_dim == 0 || n_tokens < 2u || group_dim > INT_MAX || rank > INT_MAX || out_dim > INT_MAX || n_tokens > INT_MAX || !cuda_use_mmq()) { - return (grouped_batch_require || grouped_prefill_require) ? -1 : 0; + return any_grouped_require ? -1 : 0; } const uint64_t low_dim = (uint64_t)n_groups * rank; if (low_dim > INT_MAX || (group_dim % CUDA_QK_K) != 0u || (low_dim % CUDA_QK_K) != 0u) { - return (grouped_batch_require || grouped_prefill_require) ? -1 : 0; + return any_grouped_require ? -1 : 0; } const uint64_t row_a_bytes = (group_dim / CUDA_QK_K) * sizeof(cuda_block_q4_K); if (rank > UINT64_MAX / row_a_bytes || n_groups > UINT64_MAX / (rank * row_a_bytes)) { - return (grouped_batch_require || grouped_prefill_require) ? -1 : 0; + return any_grouped_require ? -1 : 0; } const uint64_t out_a_bytes = (uint64_t)n_groups * rank * row_a_bytes; if (out_a_offset > model_size || @@ -43937,13 +43946,13 @@ extern "C" int ds4_gpu_attention_output_q4_K_batch_tensor( out->bytes < (uint64_t)n_tokens * out_dim * sizeof(float) || group_tmp->bytes < (uint64_t)n_tokens * group_dim * sizeof(float) || low_tmp->bytes < (uint64_t)n_tokens * rank * sizeof(float)) { - return (grouped_batch_require || grouped_prefill_require) ? -1 : 0; + return any_grouped_require ? -1 : 0; } const int logical_tier = ds4_tensor_device_idx(out); const char *out_a = cuda_resolve_weight_ptr( model_map, out_a_offset, out_a_bytes, logical_tier, "q4 attn_out_a"); if (!out_a) { - return (grouped_batch_require || grouped_prefill_require) ? -1 : 0; + return any_grouped_require ? -1 : 0; } const int grouped_oracle = cuda_env_flag_enabled( "DS4_CUDA_Q4_GROUPED_ATTN_A_ORACLE", 0); @@ -43953,6 +43962,15 @@ extern "C" int ds4_gpu_attention_output_q4_K_batch_tensor( "DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_PREFILL", 0); const int grouped_prefill_disable = getenv("DS4_CUDA_NO_Q4_GROUPED_ATTN_A_PREFILL") != NULL; + const int grouped_single_grid_enable = cuda_env_flag_enabled( + "DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_SINGLE_GRID", 0); + const int grouped_single_grid_disable = cuda_env_flag_enabled( + "DS4_CUDA_DISABLE_Q4_GROUPED_ATTN_A_SINGLE_GRID", 0); + const int grouped_single_grid_selected = + (grouped_single_grid_enable || grouped_single_grid_require) && + !grouped_single_grid_disable; + const int grouped_prefill_selected = grouped_prefill_enable || + grouped_prefill_require || grouped_single_grid_selected; if (grouped_oracle) cuda_q4_grouped_attn_a_oracle_register_report(); const int grouped_gb10 = @@ -43981,16 +43999,44 @@ extern "C" int ds4_gpu_attention_output_q4_K_batch_tensor( "is not eligible\n"); return -1; } - if ((grouped_prefill_enable || grouped_prefill_require) && - grouped_prefill_gb10) { - const int rc = ds4_mmq_q4_K_grouped_dense( - out_a, (const float *)heads->ptr, (float *)low->ptr, - (int)rank, (int)n_tokens, (int)group_dim, (int)n_groups, - cuda_decode_stream()); + if (grouped_single_grid_require && + (grouped_single_grid_disable || !grouped_prefill_gb10)) { + fprintf(stderr, + "ds4: required CUDA Q4 grouped attention-A single-grid " + "path is not eligible\n"); + return -1; + } + if (grouped_prefill_selected && grouped_prefill_gb10) { + int rc = grouped_single_grid_selected + ? ds4_mmq_q4_K_grouped_dense_single_grid( + out_a, (const float *)heads->ptr, (float *)low->ptr, + (int)rank, (int)n_tokens, (int)group_dim, (int)n_groups, + cuda_decode_stream()) + : ds4_mmq_q4_K_grouped_dense( + out_a, (const float *)heads->ptr, (float *)low->ptr, + (int)rank, (int)n_tokens, (int)group_dim, (int)n_groups, + cuda_decode_stream()); + /* NOT_APPLICABLE is returned before allocation or enqueue, so the + * opt-in candidate can safely fall back to the established grouped + * implementation. Every other failure may follow an enqueue and + * remains fail-closed. */ + if (rc == DS4_MMQ_NOT_APPLICABLE && grouped_single_grid_selected) { + if (grouped_single_grid_require) { + fprintf(stderr, + "ds4: required CUDA Q4 grouped attention-A " + "single-grid dispatch was not applicable\n"); + return -1; + } + rc = ds4_mmq_q4_K_grouped_dense( + out_a, (const float *)heads->ptr, (float *)low->ptr, + (int)rank, (int)n_tokens, (int)group_dim, (int)n_groups, + cuda_decode_stream()); + } if (rc != 0) { fprintf(stderr, - "ds4: CUDA Q4 grouped attention-A prefill returned %d; " + "ds4: CUDA Q4 grouped attention-A %s returned %d; " "failing closed after candidate dispatch\n", + grouped_single_grid_selected ? "single-grid" : "prefill", rc); return -1; } @@ -44020,7 +44066,7 @@ extern "C" int ds4_gpu_attention_output_q4_K_batch_tensor( (float *)low->ptr + (uint64_t)t * low_dim, (int)rank, (int)group_dim, (int)n_groups, cuda_decode_stream()); - if (rc == DS4_MMQ_NOT_APPLICABLE) return 0; + if (rc == DS4_MMQ_NOT_APPLICABLE) return -1; if (rc != 0) return -1; } } else if (batch_rc != 0) { @@ -44047,32 +44093,35 @@ extern "C" int ds4_gpu_attention_output_q4_K_batch_tensor( (uint64_t)n_groups * group_dim * sizeof(float), group_dim * sizeof(float), n_tokens, cudaMemcpyDeviceToDevice, cuda_decode_stream()); - if (!cuda_ok(ce, "q4 attention output heads pack")) return 0; + if (!cuda_ok(ce, "q4 attention output heads pack")) return -1; const int rc = ds4_mmq_q4_K_dense( out_a + (uint64_t)g * rank * row_a_bytes, (const float *)group_tmp->ptr, (float *)low_tmp->ptr, (int)rank, (int)n_tokens, (int)group_dim, cuda_decode_stream()); - if (rc != 0) return 0; + if (rc != 0) return -1; ce = cudaMemcpy2DAsync( (float *)low->ptr + (uint64_t)g * rank, low_dim * sizeof(float), low_tmp->ptr, rank * sizeof(float), rank * sizeof(float), n_tokens, cudaMemcpyDeviceToDevice, cuda_decode_stream()); - if (!cuda_ok(ce, "q4 attention output low unpack")) return 0; + if (!cuda_ok(ce, "q4 attention output low unpack")) return -1; } } + int b_rc = 0; if (out_b_type == 12u) { - return cuda_matmul_q4_K_tensor(out, model_map, model_size, + b_rc = cuda_matmul_q4_K_tensor(out, model_map, model_size, out_b_offset, low_dim, out_dim, low, n_tokens); - } - if (out_b_type == 8u) { - return cuda_matmul_q8_0_tensor_labeled(out, model_map, model_size, - out_b_offset, low_dim, out_dim, - low, n_tokens, "q4 attn_output_b"); - } - return 0; + } else if (out_b_type == 8u) { + b_rc = cuda_matmul_q8_0_tensor_labeled( + out, model_map, model_size, out_b_offset, low_dim, out_dim, + low, n_tokens, "q4 attn_output_b"); + } + /* Attention-A has already been enqueued. A B rejection or launch error + * can no longer request whole-operation fallback without replaying A, so + * preserve the tri-state compound-operation contract and fail closed. */ + return b_rc > 0 ? 1 : -1; } static void cuda_q4_grouped_attn_a_oracle_report(void) { diff --git a/speed-bench/cuda_q4_prefill_bench.cu b/speed-bench/cuda_q4_prefill_bench.cu index 09dbea6b8..ed1e09d3d 100644 --- a/speed-bench/cuda_q4_prefill_bench.cu +++ b/speed-bench/cuda_q4_prefill_bench.cu @@ -43,6 +43,12 @@ constexpr const char *kGroupedPrefillDisable = "DS4_CUDA_NO_Q4_GROUPED_ATTN_A_PREFILL"; constexpr const char *kGroupedPrefillRequire = "DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_PREFILL"; +constexpr const char *kGroupedSingleGridEnable = + "DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_SINGLE_GRID"; +constexpr const char *kGroupedSingleGridDisable = + "DS4_CUDA_DISABLE_Q4_GROUPED_ATTN_A_SINGLE_GRID"; +constexpr const char *kGroupedSingleGridRequire = + "DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_SINGLE_GRID"; constexpr const char *kGroupedGlobalDisable = "DS4_CUDA_NO_Q4_GROUPED_ATTN_A"; constexpr const char *kGb10GlobalDisable = "DS4_CUDA_NO_Q4_GB10_FAST"; @@ -581,15 +587,21 @@ bool sampled_grouped_cpu_oracle( } bool select_grouped_prefill_baseline() { - return unsetenv(kGroupedPrefillEnable) == 0 && - setenv(kGroupedPrefillDisable, "1", 1) == 0 && - unsetenv(kGroupedPrefillRequire) == 0; + return setenv(kGroupedPrefillEnable, "1", 1) == 0 && + unsetenv(kGroupedPrefillDisable) == 0 && + setenv(kGroupedPrefillRequire, "1", 1) == 0 && + unsetenv(kGroupedSingleGridEnable) == 0 && + setenv(kGroupedSingleGridDisable, "1", 1) == 0 && + unsetenv(kGroupedSingleGridRequire) == 0; } bool select_grouped_prefill_candidate() { return setenv(kGroupedPrefillEnable, "1", 1) == 0 && unsetenv(kGroupedPrefillDisable) == 0 && - setenv(kGroupedPrefillRequire, "1", 1) == 0; + setenv(kGroupedPrefillRequire, "1", 1) == 0 && + setenv(kGroupedSingleGridEnable, "1", 1) == 0 && + unsetenv(kGroupedSingleGridDisable) == 0 && + setenv(kGroupedSingleGridRequire, "1", 1) == 0; } double percentile(std::vector sorted, double fraction) { @@ -876,7 +888,7 @@ bool run_pair(const model_fixture &model, const config &cfg, pair0.ptr, pair1.ptr, model.data, model.size, model.weights[set].dense_offset, model.weights[set].kv_offset, kDenseK, kDenseM, kKvM, - x.ptr, n_tokens) != 0; + x.ptr, n_tokens) == 1; }, {}}; return benchmark_pair_arms( @@ -1028,7 +1040,7 @@ bool run_output_a(const model_fixture &model, const config &cfg, } const arm baseline = { - "pack8_mmq_unpack", + "grouped_8_grids", [&](uint32_t set) { return ds4_gpu_attention_output_q4_K_batch_tensor( baseline_out.ptr, baseline_low.ptr, group_tmp.ptr, @@ -1040,7 +1052,7 @@ bool run_output_a(const model_fixture &model, const config &cfg, }, select_grouped_prefill_baseline}; const arm candidate = { - "grouped_dense", + "grouped_single_grid", [&](uint32_t set) { return ds4_gpu_attention_output_q4_K_batch_tensor( candidate_out.ptr, candidate_low.ptr, group_tmp.ptr, @@ -1074,7 +1086,7 @@ bool run_output_a(const model_fixture &model, const config &cfg, [&](uint32_t set) { return bitwise_equal(baseline_low.ptr, candidate_low.ptr, low_bytes, - "output_a grouped dense vs pack/MMQ") && + "output_a single-grid vs eight-grid") && bitwise_equal(baseline_out.ptr, candidate_out.ptr, out_bytes, "output_a minimal-B final output") && @@ -1137,9 +1149,10 @@ void usage(FILE *stream, const char *argv0) { "legacy/MMQ\nprocesses (preferably ABBA/BAAB) to compare them because " "the CUDA backend\ncaches DS4_CUDA_MMQ on its first dispatch. Pair is " "an in-process ABBA/BAAB\ncomparison of two MMQ projections against " - "the fused public pair API. outa\ncompares the current eight-group " - "pack/MMQ/unpack path against the strict\ngrouped-prefill candidate " - "at the production attention-A shape. It includes\na common minimal " + "the fused public pair API. outa\ncompares the existing grouped path " + "with one MMQ grid per group against the strict\nsingle-grid candidate " + "at the production attention-A shape, isolating grid.z submission. It " + "includes\na common minimal " "Q4 output-B (M=256) whose MACs are reported separately.\n", argv0, kDefaultSets, kDefaultSamples, kDefaultWarmup); } @@ -1344,7 +1357,7 @@ bool verify_mmq_prefill_dispatch(const model_fixture &model) { ds4_gpu_matmul_q4_K_pair_tensor( out0.ptr, out1.ptr, model.data, model.size, model.weights[0].dense_offset, model.weights[0].kv_offset, - kDenseK, kDenseM, kKvM, x.ptr, n_tokens) && + kDenseK, kDenseM, kKvM, x.ptr, n_tokens) == 1 && ds4_gpu_synchronize(); } @@ -1359,6 +1372,9 @@ int main(int argc, char **argv) { env_snapshot grouped_enable_guard(kGroupedPrefillEnable); env_snapshot grouped_disable_guard(kGroupedPrefillDisable); env_snapshot grouped_require_guard(kGroupedPrefillRequire); + env_snapshot grouped_single_grid_enable_guard(kGroupedSingleGridEnable); + env_snapshot grouped_single_grid_disable_guard(kGroupedSingleGridDisable); + env_snapshot grouped_single_grid_require_guard(kGroupedSingleGridRequire); env_snapshot grouped_global_disable_guard(kGroupedGlobalDisable); env_snapshot gb10_global_disable_guard(kGb10GlobalDisable); if (setenv("DS4_CUDA_MMQ", @@ -1369,6 +1385,9 @@ int main(int argc, char **argv) { unsetenv(kGroupedPrefillEnable) != 0 || unsetenv(kGroupedPrefillDisable) != 0 || unsetenv(kGroupedPrefillRequire) != 0 || + unsetenv(kGroupedSingleGridEnable) != 0 || + unsetenv(kGroupedSingleGridDisable) != 0 || + unsetenv(kGroupedSingleGridRequire) != 0 || unsetenv(kGroupedGlobalDisable) != 0 || unsetenv(kGb10GlobalDisable) != 0) { std::fprintf(stderr, From 278ee3801e499064b6fa1e20e8cd4b3319bdfb73 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Sat, 29 Aug 2026 09:14:30 +0200 Subject: [PATCH 146/189] docs: register q4 prefill kernel controls --- ENVIRONMENT_VARIABLES.md | 9 +++++++++ scripts/environment_variables.tsv | 9 +++++++++ 2 files changed, 18 insertions(+) diff --git a/ENVIRONMENT_VARIABLES.md b/ENVIRONMENT_VARIABLES.md index 96089687e..dc877917d 100644 --- a/ENVIRONMENT_VARIABLES.md +++ b/ENVIRONMENT_VARIABLES.md @@ -40,6 +40,9 @@ changing them. Unless a row says otherwise: | `DS4_METAL_STREAMING_EXPERT_NOCACHE=1` | Reopen the Metal SSD expert file with `F_NOCACHE` so streamed experts do not displace the dense working set from the page cache. Leave unset for cached `pread`. | | `DS4_METAL_STREAMING_EXPERT_PREAD_SPLIT=N` | Split each expert read into 1–8 aligned requests. The automatic value is 1 below 64 configured cache experts and 4 at 64 or more. | | `DS4_METAL_DISABLE_Q4_DENSE_PAIR=1` | Split the default Metal Q-A/KV Q4 pair back into two standalone projections. | +| `DS4_METAL_ENABLE_Q4_PREFILL_PAIR_F16_RHS=1` | On Apple M1–M4, opt into the Q-A/KV prefill pair that materializes their shared F32 activation as F16 once for exact 32-token tiles through N=128. | +| `DS4_METAL_DISABLE_Q4_PREFILL_PAIR_F16_RHS=1` | Dominant rollback to two standalone Q4_K/F32-RHS prefill projections. | +| `DS4_METAL_REQUIRE_Q4_PREFILL_PAIR_F16_RHS=1` | Request the shared-F16-RHS pair and fail closed if its device, shape, storage, or buffer contract is unavailable. Intended for strict A/B oracles. | | `DS4_METAL_DISABLE_M1_IQ2_MID_ONLY=1` | Restore the canonical IQ2 address-table gate/up producer on the exact M1 SSD-streaming decode shape. The specialization is automatic by default. | | `DS4_METAL_REQUIRE_M1_IQ2_MID_ONLY=1` | Fail closed when an otherwise eligible M1 IQ2 mid-only dispatch cannot use the specialization. | | `DS4_METAL_ENABLE_M1_IQ2_MID_ONLY=1` | Legacy spelling retained for migration notes only. The runtime does not read it; the path is automatic and this setting is ignored. | @@ -66,6 +69,9 @@ The detailed Metal A/B contracts and expected oracle counters live in | `DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_PREFILL=1` | Enable the GB10 Q4 attention-A prefill candidate for more than eight tokens. It quantizes the token-major grouped input directly and removes the eight pack/unpack copies while preserving each group’s MMQ reduction tree. | | `DS4_CUDA_NO_Q4_GROUPED_ATTN_A_PREFILL=1` | Dominant rollback from the grouped Q4 attention-A prefill candidate to eight pack/MMQ/unpack projections. Any defined value disables the candidate. | | `DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_PREFILL=1` | Request the grouped Q4 attention-A prefill candidate and fail before enqueue when its GB10, shape, residency, or buffer contract is unavailable. | +| `DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_SINGLE_GRID=1` | On eligible GB10 prefills, submit the eight grouped attention-A MMQs as one grid.z launch while retaining a separate stream-K coordinate and fixup slice per group. | +| `DS4_CUDA_DISABLE_Q4_GROUPED_ATTN_A_SINGLE_GRID=1` | Dominant rollback from the single-grid experiment to the established one-MMQ-grid-per-group path. | +| `DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_SINGLE_GRID=1` | Require the single-grid grouped attention-A submission and fail closed on rollback, ineligibility, or dispatch failure. | | `DS4_CUDA_Q4_GROUPED_ATTN_A_ORACLE=1` | Compare grouped attention-A with the canonical result and retain the canonical output. Disable graph capture for this diagnostic. | | `DS4_CUDA_ENABLE_Q4_K1024_PERSISTENT=1` | Enable the experimental persistent-CTA kernel for the exact `32768x1024` Q4 shape. | | `DS4_CUDA_NO_Q4_K1024_PERSISTENT=1` | Roll back the persistent K1024 experiment. | @@ -84,6 +90,9 @@ The detailed Metal A/B contracts and expected oracle counters live in | `DS4_ROCM_DISABLE_Q4_PREFILL_TILE8=1` | Restore the legacy Q4 prefill kernel. TILE8 is automatic for validated chunks of 9 through 4096 tokens. | | `DS4_ROCM_REQUIRE_Q4_PREFILL_TILE8=1` | Fail closed when an eligible Q4 prefill call cannot use TILE8. | | `DS4_ROCM_ENABLE_Q4_PREFILL_TILE8=1` | Legacy spelling retained for migration notes only. The runtime does not read it; TILE8 is automatic and this setting is ignored. | +| `DS4_ROCM_ENABLE_Q4_PREFILL_Q8_K_WAVE32=1` | On gfx1151 wave32, opt into the no-LDS Q8_K activation quantizer that assigns one 256-value block to each wave before the exact Q4 prefill matmul. | +| `DS4_ROCM_DISABLE_Q4_PREFILL_Q8_K_WAVE32=1` | Dominant rollback to the canonical one-workgroup-per-Q8_K-block quantizer. | +| `DS4_ROCM_REQUIRE_Q4_PREFILL_Q8_K_WAVE32=1` | Require the wave32 quantizer for strict prefill A/B runs; unsupported scope/device, rollback, or an incompatible required F16/WMMA path fails closed. | | `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA=1` | Opt into the experimental resident gfx1151 wave32 Q4_K WMMA64 prefill kernel for 256–4096 tokens. It replaces Q8_K activation scratch with transient F16 register dequantization and F32 accumulation. | | `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_SSD=1` | Allow that WMMA64 path during SSD streaming only when the complete projection weight range is already backed by physical device storage. It never treats mapped/registered host memory as resident. | | `DS4_ROCM_DISABLE_Q4_PREFILL_WMMA=1` | Dominant value-aware rollback for the WMMA64 experiment. | diff --git a/scripts/environment_variables.tsv b/scripts/environment_variables.tsv index 20cbbfda5..a4c84ba99 100644 --- a/scripts/environment_variables.tsv +++ b/scripts/environment_variables.tsv @@ -59,6 +59,7 @@ runtime/cuda DS4_CUDA_DISABLE_Q4_ATTN_OUT_HC_FUSE presence kill switch; default runtime/cuda DS4_CUDA_DISABLE_Q4_ATTN_Q_B_F16_CACHE value-aware rollback; unset/empty/0/false/no/off keeps the experiment available, any other nonempty value disables it Disable the resident Q4_K attn_q_b-to-F16 prefill cache even when ENABLE or REQUIRE is set. ds4_cuda.cu:2502 runtime/cuda DS4_CUDA_DISABLE_Q4_ATTN_Q_B_TRANSIENT_F16 value-aware rollback; unset/empty/0/false/no/off keeps the automatic transient path eligible, any other nonempty value disables it Disable per-layer transient Q4_K attn_q_b-to-F16 scratch for physically device-resident single-GPU model images; the existing resident-cache controls remain independent and otherwise eligible calls use native Q4. ds4_cuda.cu:2514 runtime/cuda DS4_CUDA_DISABLE_Q4_DENSE_PAIR presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q4 dense pair CUDA Q4 optimization. ds4_cuda.cu:20402 +runtime/cuda DS4_CUDA_DISABLE_Q4_GROUPED_ATTN_A_SINGLE_GRID value-aware authoritative rollback; unset/empty/exact 0 permits the opt-in, every other nonempty value disables it and makes REQUIRE fail closed Restore the established one-MMQ-launch-per-group grouped attention-A prefill path instead of mapping the groups to grid.z. ds4_cuda.cu:41943 runtime/cuda DS4_CUDA_DISABLE_Q8_HC_EXPAND_FUSED false-like-aware flag, default off; 0/false/no/off is off, other nonempty values request split; force-fused wins Request the split Q8 shared-down/HC path when safe. ds4_cuda.cu:2086 runtime/cuda DS4_CUDA_DISABLE_QKV_RMS_FUSED presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the CUDA QKV RMS fused optimization/path. ds4_cuda.cu:369; ds4.c:17428 runtime/cuda DS4_CUDA_DISABLE_SHARED_GATE_UP_PAIR presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the CUDA shared gate up pair optimization/path. ds4_cuda.cu:24293 @@ -86,6 +87,7 @@ runtime/cuda DS4_CUDA_ENABLE_Q4_ATTN_Q_B_F16_CACHE value-aware persistent-cache runtime/cuda DS4_CUDA_ENABLE_Q4_ATTN_Q_B_F16_OUTPUT value-aware experimental opt-in; unset/empty/0/false/no/off keeps the release F32 projection boundary; other nonempty values enable Write eligible resident Q4_K attn_q_b GEMM output in F16 and run the half-input norm/RoPE epilogue; SSD remains excluded. ds4_cuda.cu:2523 runtime/cuda DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_BATCH value-aware opt-in, default off; nonempty value other than exact 0 enables; rollback wins Enable flattened grouped attention-A MMQ for two-to-eight-token GB10 batches. cuda/mmq/ds4_mmq.cu:4303 runtime/cuda DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_PREFILL value-aware opt-in, default off; unset/empty/exact 0 is off, any other nonempty value requests the path; REQUIRE also requests it; local/global rollback wins Enable direct-strided grouped Q4_K attention-A MMQ for GB10 prefill widths above eight tokens, removing per-group pack/unpack copies while preserving the per-group reduction tree. ds4_cuda.cu:41928 +runtime/cuda DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_SINGLE_GRID value-aware opt-in, default off; unset/empty/exact 0 is off, every other nonempty value requests the candidate; REQUIRE also requests it; DISABLE wins Submit all eligible GB10 grouped Q4_K attention-A prefill projections in one grid.z launch while isolating each group's stream-K coordinates and fixup storage. ds4_cuda.cu:41941 runtime/cuda DS4_CUDA_ENABLE_Q4_K1024_PERSISTENT presence flag, default off; any defined value including 0 requests the path; rollback wins Enable the GB10 persistent-CTA kernel for M=32768, N=1, K=1024 Q4. cuda/mmq/ds4_mmq.cu:3905 runtime/cuda DS4_CUDA_ENABLE_Q8_FOLD strict flag, default off; only exact value 1 enables; overridden by DS4_CUDA_NO_Q8_FOLD Enable one-shot producer-to-consumer reuse of freshly quantized Q8_1 data. ds4_cuda.cu:785 runtime/cuda DS4_CUDA_ENABLE_STREAMING_EXPERT_PERSISTENT_CACHE false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Enable streaming expert persistent cache in CUDA SSD streaming. ds4_cuda.cu:4111 @@ -305,6 +307,7 @@ runtime/cuda DS4_CUDA_REQUIRE_IQ2_XXS_SSD_PREFILL_MMQ false-like-aware flag, def runtime/cuda DS4_CUDA_REQUIRE_Q4_ATTN_Q_B_F16_CACHE value-aware strict opt-in, default off; unset/empty/0/false/no/off is off, any other nonempty value requires eligible batches to use the cache; DISABLE wins Fail an eligible CUDA prefill instead of falling back when the resident Q4_K attn_q_b F16 specialization cannot be prepared or dispatched. ds4_cuda.cu:2492; ds4_cuda.cu:2497 runtime/cuda DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_BATCH value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on Fail if grouped batched attention-A cannot be used. ds4_cuda.cu:40198 runtime/cuda DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_PREFILL value-aware fail-closed assertion, default off; unset/empty/exact 0 is off, any other nonempty value requests the candidate and rejects ineligibility before enqueue Require the GB10 grouped Q4_K attention-A prefill path instead of silently using pack/MMQ/unpack. ds4_cuda.cu:41889 +runtime/cuda DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_SINGLE_GRID value-aware fail-closed assertion, default off; unset/empty/exact 0 is off, every other nonempty value requests the candidate; DISABLE or ineligibility fails before enqueue Require one grid.z MMQ submission for eligible GB10 grouped Q4_K attention-A prefill instead of falling back to one launch per group. ds4_cuda.cu:41896 runtime/cuda DS4_CUDA_REQUIRE_Q4_K1024_PERSISTENT presence flag, default off; any defined value including 0 makes ineligible candidate fail closed Fail when the exact Q4 K1024 persistent candidate is unavailable instead of using MMVQ. cuda/mmq/ds4_mmq.cu:3929 runtime/cuda DS4_CUDA_REQUIRE_STREAMING_EXPERT_PERSISTENT_CACHE false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Require streaming expert persistent cache in CUDA SSD streaming; fail closed when unavailable. ds4_cuda.cu:4117 runtime/cuda DS4_CUDA_REQUIRE_STREAMING_SELECTED_BATCH_IO false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Require streaming selected batch I/O in CUDA SSD streaming; fail closed when unavailable. ds4_cuda.cu:4738 @@ -616,6 +619,7 @@ runtime/metal DS4_METAL_DISABLE_Q4_GROUP8_EXPERT_TABLE presence rollback; unset: runtime/metal DS4_METAL_DISABLE_Q4_GROUPED_BOUNDARY presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 grouped boundary. ds4_metal.m:42149 runtime/metal DS4_METAL_DISABLE_Q4_GROUPED_EXPERTS presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 grouped experts. ds4_metal.m:42130 runtime/metal DS4_METAL_DISABLE_Q4_MV_CLASSIC presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 MV classic. ds4_metal.m:21571 +runtime/metal DS4_METAL_DISABLE_Q4_PREFILL_PAIR_F16_RHS value-aware authoritative rollback; unset/0/false/no/off permits the opt-in, while empty/1/true/yes/on or another nonempty value disables it; REQUIRE then fails closed Restore two independent Q4_K/F32-RHS q_a and KV prefill matmuls instead of materializing their shared F16 RHS once. ds4_metal.m:22911 runtime/metal DS4_METAL_DISABLE_Q4_PREFILL_TAIL_SIMDGROUP_CULL presence rollback; unset: automatic on Apple M1-M4 for Q4_K single-tile prefill N=9..16 and production attn_q_b tails through N=65; any value including 0 disables Restores the legacy four-SIMDgroup Q4_K prefill kernel on the measured short-prefill scopes. ds4_metal.m:22088 runtime/metal DS4_METAL_DISABLE_Q4_QKV_COMPRESSOR_FUSE presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 QKV compressor fuse. ds4_metal.m:22083 runtime/metal DS4_METAL_DISABLE_Q4_SELECTED_EXPERT_VIEWS presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 selected expert views. ds4.c:21136 @@ -722,6 +726,7 @@ runtime/metal DS4_METAL_ENABLE_Q4_GROUP24_EXPERT_TABLE presence opt-in; unset: o runtime/metal DS4_METAL_ENABLE_Q4_GROUP6_EXPERT_TABLE presence opt-in; unset: off/automatic; any value including 0 enables Enables Q4 group6 expert table. ds4_metal.m:42166 runtime/metal DS4_METAL_ENABLE_Q4_GROUP8_EXPERT_TABLE presence opt-in; unset: off/automatic; any value including 0 enables Enables Q4 group8 expert table. ds4_metal.m:42183 runtime/metal DS4_METAL_ENABLE_Q4_GROUPED_EXPERTS presence opt-in; unset: off/automatic; any value including 0 enables Enables Q4 grouped experts. ds4_metal.m:42129 +runtime/metal DS4_METAL_ENABLE_Q4_PREFILL_PAIR_F16_RHS value-aware opt-in, default off; empty/1/true/yes/on or another nonempty value enables, while 0/false/no/off disables; REQUIRE also requests it; DISABLE wins On Apple M1-M4, convert the shared q_a/KV prefill activation to F16 once and feed two bit-exact Q4_K kernels for N=32,64,96,128. ds4_metal.m:22909 runtime/metal DS4_METAL_ENABLE_Q4_QKV_COMPRESSOR_FUSE presence opt-in; unset: off/automatic; any value including 0 enables Enables Q4 QKV compressor fuse. ds4.c:23140 runtime/metal DS4_METAL_ENABLE_Q4_SELECTED_EXPERT_VIEWS presence opt-in; unset: off/automatic; any value including 0 enables Enables Q4 selected expert views. ds4.c:20977 runtime/metal DS4_METAL_ENABLE_Q4_SSD_PREFILL_ATTN_OUT_EXACTN value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Enables Q4 SSD prefill attn out exactn. ds4_metal.m:28097 @@ -870,6 +875,7 @@ runtime/metal DS4_METAL_REQUIRE_Q4_ATTN_OUT_B_F16_RHS value-aware boolean; unset runtime/metal DS4_METAL_REQUIRE_Q4_ATTN_OUT_TINY_BATCH value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Requires Q4 attn out tiny batch and makes eligible fallback fail closed. ds4_metal.m:28373 runtime/metal DS4_METAL_REQUIRE_Q4_ATTN_Q_B_F16_CACHE value-aware boolean; unset/0: fallback allowed; empty/1/true/yes/on requires; DISABLE fails closed Requires the sidecar for resident, or explicitly enabled SSD-hybrid, pre-M5 Q4_K Flash attn_q_b batches at or above the configured minimum; decode and below-min batches remain non-candidates. ds4_metal.m:25801; ds4_metal.m:26129; ds4_metal.m:26704 runtime/metal DS4_METAL_REQUIRE_Q4_BATCH_ATTN_OUT_HC_FUSION presence strict check; unset permits fallback; any value including 0 requires the exact resident Q4_K/F16-RHS output-B-to-HC4 tail Fail closed instead of replaying the separate Q4 output-B and HC path when the fused candidate is ineligible or fails. ds4_metal.m:31455 +runtime/metal DS4_METAL_REQUIRE_Q4_PREFILL_PAIR_F16_RHS value-aware strict opt-in; unset/0/false/no/off is off; empty/1/true/yes/on or another nonempty value requires the candidate for prefill pair calls; unsupported device/shape, DISABLE, or an attempted-path error fails closed Require the Apple M1-M4 shared-F16-RHS q_a/KV Q4_K prefill pair instead of silently running two standalone matmuls. ds4_metal.m:22913 runtime/metal DS4_METAL_REQUIRE_Q4_SSD_PREFILL_ATTN_OUT_EXACTN value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Requires Q4 SSD prefill attn out exactn and makes eligible fallback fail closed. ds4_metal.m:28089 runtime/metal DS4_METAL_REQUIRE_Q4_SSD_PREFILL_ATTN_OUT_SCALE_META value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Requires shared scale/min metadata in the Q4 SSD prefill attention-output exact-N kernel and makes fallback fail closed. ds4_metal.m:28209 runtime/metal DS4_METAL_REQUIRE_Q4_SSD_SESSION_UNION nonempty boolean; unset/empty or exact 0: off; every other value: on Requires Q4 SSD session union and makes eligible fallback fail closed. ds4.c:65164 @@ -967,6 +973,7 @@ runtime/rocm DS4_ROCM_DISABLE_Q4_ATTN_Q_B_TRANSIENT_F16 value-aware rollback; un runtime/rocm DS4_ROCM_DISABLE_Q4_DENSE_PAIR presence rollback; unset leaves opt-in policy unchanged Disable/roll back rocm disable q4 dense pair. rocm/ds4_rocm_q4.cuh:435 runtime/rocm DS4_ROCM_DISABLE_Q4_GROUPED_ATTN_A presence rollback; unset permits the caller-marked resident decode production-shape default and explicit ENABLE/REQUIRE; any defined value including empty or 0 disables all grouped attention-A paths and wins over ENABLE/REQUIRE Restore eight standalone Q4 attention-A projections instead of the two-dispatch grouped path. rocm/ds4_rocm_q4.cuh:868 runtime/rocm DS4_ROCM_DISABLE_Q4_PREFILL_K1024_TILE4 value-aware authoritative rollback; unset/0/false/no/off preserves the resident automatic default and any explicit SSD request; empty or any other value disables; overrides ENABLE and causes REQUIRE to fail closed Restore the generic eight-block Q4_K tiled-prefill kernel for K=1024 in both resident and SSD-streaming execution. rocm/ds4_rocm_q4.cuh:624 +runtime/rocm DS4_ROCM_DISABLE_Q4_PREFILL_Q8_K_WAVE32 value-aware authoritative rollback; unset/0/false/no/off permits ENABLE or REQUIRE, while empty or any other value disables; REQUIRE then fails closed Restore the canonical one-workgroup-per-Q8_K-block activation quantizer for Q4 prefill. rocm/ds4_rocm_q4.cuh:803 runtime/rocm DS4_ROCM_DISABLE_Q4_PREFILL_TILE8 presence rollback; TILE8 is default for 9..4096 tokens Disable/roll back rocm disable q4 prefill tile8. rocm/ds4_rocm_q4.cuh:448 runtime/rocm DS4_ROCM_DISABLE_Q4_PREFILL_WMMA value-aware authoritative rollback for the experimental path; unset/0/false/no/off permits ENABLE or REQUIRE, while empty or any other value disables; REQUIRE then fails closed Prevent the gfx1151 Q4_K prefill WMMA64 experiment from dispatching and retain the Q8_K-plus-TILE8/TILE4 path. rocm/ds4_rocm_q4.cuh:775 runtime/rocm DS4_ROCM_DISABLE_Q4_SELECTED_EXPERT_VIEWS presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable q4 selected expert views. ds4.c:21150 @@ -1011,6 +1018,7 @@ runtime/rocm DS4_ROCM_ENABLE_Q4_ATTN_Q_B_F16_OUTPUT value-aware experimental opt runtime/rocm DS4_ROCM_ENABLE_Q4_DENSE_PAIR presence opt-in; unset=off; DISABLE takes precedence Enable rocm enable q4 dense pair. rocm/ds4_rocm_q4.cuh:434 runtime/rocm DS4_ROCM_ENABLE_Q4_GROUPED_ATTN_A presence opt-in outside the default scope; the exact caller-marked resident decode shape groups=8, N=1, K=4096, M=1024 is automatic, while row-at-a-time batch fallbacks are not; DISABLE wins Enable grouped Q4 attention-A for eligible slices, non-production shapes, or explicit experiments in addition to the resident decode default. rocm/ds4_rocm_q4.cuh:872 runtime/rocm DS4_ROCM_ENABLE_Q4_PREFILL_K1024_TILE4_SSD value-aware SSD-only opt-in, default off; unset/0/false/no/off retains TILE8, while empty or any other value requests TILE4; eligibility additionally requires N=9..4096, K=1024, M=32768, TILE8 enabled, and the complete weight range in device storage rather than mapped/registered host memory; DISABLE wins Allow the four-lane K=1024 Q4_K prefill specialization to consume an already device-resident/cache-backed attn_q_b weight range during SSD streaming without changing model I/O. rocm/ds4_rocm_q4.cuh:624 +runtime/rocm DS4_ROCM_ENABLE_Q4_PREFILL_Q8_K_WAVE32 value-aware opt-in, default off; unset/0/false/no/off retains the canonical quantizer, while empty or any other value requests the candidate for N=9..4096 on gfx1151 wave32; DISABLE wins Quantize eight independent Q8_K activation blocks per 256-thread workgroup using one wave32 per block, without LDS or workgroup barriers, before the exact Q4 prefill matmul (TILE8 or its legacy rollback). rocm/ds4_rocm_q4.cuh:801 runtime/rocm DS4_ROCM_ENABLE_Q4_PREFILL_WMMA value-aware opt-in, default off; unset/0/false/no/off retains the canonical path; empty or any other value requests WMMA64 only for N=256..4096, K a positive multiple of 256, resident non-quality execution on gfx1151 wave32; DISABLE wins Benchmark the compressed Q4_K-to-F16 register dequantization plus 64x64 WMMA prefill kernel without Q8_K activation scratch or an F16 weight sidecar. rocm/ds4_rocm_q4.cuh:771 runtime/rocm DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_SSD value-aware SSD-only opt-in, default off; unset/0/false/no/off retains TILE8/TILE4, while empty or any other value requests WMMA64; eligibility additionally requires the complete projection weight range in physical device storage rather than mapped/registered host memory; DISABLE wins Allow the compressed WMMA64 kernel to consume an already device-resident/cache-backed Q4_K projection during SSD streaming without changing model I/O. rocm/ds4_rocm_q4.cuh:773 runtime/rocm DS4_ROCM_ENABLE_STREAMING_FULL_EXPERT_ADDR_TABLE presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming full expert addr table. ds4.c:18255 @@ -1075,6 +1083,7 @@ runtime/rocm DS4_ROCM_Q_STAGE_PROFILE presence/nonempty diagnostic; unset=off (p runtime/rocm DS4_ROCM_REQUIRE_Q4_ATTN_Q_B_F16_CACHE value-aware strict opt-in, default off; unset/0/false/no/off is off, empty or any other value requires eligible batches to use the cache; DISABLE wins Fail an eligible ROCm prefill instead of falling back when the resident Q4_K attn_q_b F16 specialization cannot be prepared or dispatched. rocm/ds4_rocm_q4_qb_sidecar.cuh:135 runtime/rocm DS4_ROCM_REQUIRE_Q4_GROUPED_ATTN_A presence fail-closed assertion; also requests the candidate outside the caller-marked resident decode default; DISABLE remains authoritative and causes failure Require grouped Q4 attention-A and fail instead of silently falling back. rocm/ds4_rocm_q4.cuh:870 runtime/rocm DS4_ROCM_REQUIRE_Q4_PREFILL_K1024_TILE4 value-aware fail-closed assertion and SSD opt-in; unset/0/false/no/off is off; empty or any other value requires eligible N=9..4096, K=1024, M=32768 dense calls to select TILE4; SSD also requires an actual device-resident weight range; DISABLE wins Prevent a K=1024 TILE4 correctness/performance oracle from silently falling back to TILE8, including during SSD-streaming A/B runs. rocm/ds4_rocm_q4.cuh:628 +runtime/rocm DS4_ROCM_REQUIRE_Q4_PREFILL_Q8_K_WAVE32 value-aware strict opt-in; unset/0/false/no/off is off; empty or any other value requires gfx1151 wave32 and N=9..4096; DISABLE and conflicts with required WMMA or the required q_b F16 cache fail closed Require the no-LDS wave32 Q8_K activation quantizer for exact Q4 prefill instead of silently using the canonical quantizer or an F16 side path. rocm/ds4_rocm_q4.cuh:796 runtime/rocm DS4_ROCM_REQUIRE_Q4_PREFILL_TILE8 presence fail-closed assertion for eligible TILE8 calls Require rocm require q4 prefill tile8 and fail instead of silently falling back. rocm/ds4_rocm_q4.cuh:452 runtime/rocm DS4_ROCM_REQUIRE_Q4_PREFILL_WMMA value-aware strict opt-in; unset/0/false/no/off is off; empty or any other value requires every selected Q4 dense or attention-output projection to use WMMA64; unsupported shape/device, quality mode, DISABLE, or an SSD weight range without physical device residency fails before the relevant dispatch Prevent the ROCm Q4 prefill WMMA64 correctness/performance oracle from silently timing TILE8/TILE4; the fused q_a/KV pair yields to separately checked dense calls. rocm/ds4_rocm_q4.cuh:777 runtime/rocm DS4_ROCM_STREAMING_DECODE_PREFILL_MAX primary nonempty value over the Metal alias; parsed by strtol when it has a numeric prefix (trailing text is accepted); <= 0 disables, values > UINT32_MAX clamp, no numeric prefix uses automatic default: 64 for Flash with uniform Q4_K/MXFP4 experts, 18 for other Pro/Flash, otherwise 0; the disable flag dominates Sets the largest short, non-quality SSD-streaming prefill batch routed through the decode-style path instead of canonical layer-major prefill. ds4.c:31976 From 6ab475bf2c523d48a51a8a67cea243cfa3a4535d Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Sat, 29 Aug 2026 10:47:00 +0200 Subject: [PATCH 147/189] Update rocm/ds4_rocm_attention.cuh Co-authored-by: Frank --- rocm/ds4_rocm_attention.cuh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rocm/ds4_rocm_attention.cuh b/rocm/ds4_rocm_attention.cuh index 43cd0e243..216935d93 100644 --- a/rocm/ds4_rocm_attention.cuh +++ b/rocm/ds4_rocm_attention.cuh @@ -113,7 +113,8 @@ __global__ static void attention_output_b_f16_wmma_64x64_kernel( frag_c acc; rocwmma::fill_fragment(acc, 0.0f); - for (uint32_t k0 = 0; k0 < K; k0 += 16u) { +#pragma unroll + for (uint32_t k0 = 0; k0 < K; k0 + 16u) { const uint32_t ja = tid * 2u; const uint32_t a_row = ja & 63u; const uint32_t a_k = ja >> 6u; From fc761527d6ee93e25089d5bb7cef005c10eddd5f Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Sat, 29 Aug 2026 10:47:25 +0200 Subject: [PATCH 148/189] Update rocm/ds4_rocm_attention.cuh Co-authored-by: Frank --- rocm/ds4_rocm_attention.cuh | 1 + 1 file changed, 1 insertion(+) diff --git a/rocm/ds4_rocm_attention.cuh b/rocm/ds4_rocm_attention.cuh index 216935d93..75bc08e24 100644 --- a/rocm/ds4_rocm_attention.cuh +++ b/rocm/ds4_rocm_attention.cuh @@ -1606,6 +1606,7 @@ __global__ static void attention_mixed_heads16_wmma_kernel( frag_c score_acc; if (wave < HEAD_TILES * KEY_TILES) rocwmma::fill_fragment(score_acc, 0.0f); +#pragma unroll for (uint32_t d0 = 0; d0 < 512u; d0 += TILE) { if (tid < HEADS * TILE) { const uint32_t hl = tid / TILE; From 7a697804a7dee67fbd382a36da15035da2a1126a Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Sat, 29 Aug 2026 10:47:46 +0200 Subject: [PATCH 149/189] Update rocm/ds4_rocm_attention.cuh Co-authored-by: Frank --- rocm/ds4_rocm_attention.cuh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rocm/ds4_rocm_attention.cuh b/rocm/ds4_rocm_attention.cuh index 75bc08e24..a5a399de2 100644 --- a/rocm/ds4_rocm_attention.cuh +++ b/rocm/ds4_rocm_attention.cuh @@ -1717,6 +1717,8 @@ __global__ static void attention_mixed_heads16_wmma_kernel( } sh_qp[hl * STRIDE + kl] = __float2half(p); } + } + __syncthreads(); #pragma unroll for (uint32_t delta = 1u; delta < 16u; delta <<= 1u) { block_sum += __shfl_xor_sync(FULL_WARP_MASK, block_sum, delta, 32); From 38d8ec5eb344f0a95ca5bda4ac62a3adfc1deb73 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Sat, 29 Aug 2026 10:48:03 +0200 Subject: [PATCH 150/189] Update rocm/ds4_rocm_attention.cuh Co-authored-by: Frank --- rocm/ds4_rocm_attention.cuh | 1 + 1 file changed, 1 insertion(+) diff --git a/rocm/ds4_rocm_attention.cuh b/rocm/ds4_rocm_attention.cuh index a5a399de2..75b23b59a 100644 --- a/rocm/ds4_rocm_attention.cuh +++ b/rocm/ds4_rocm_attention.cuh @@ -1736,6 +1736,7 @@ __global__ static void attention_mixed_heads16_wmma_kernel( accum[i] *= old_scale[out_idx / 512u]; } +#pragma unroll for (uint32_t dim0 = 0; dim0 < 512u; dim0 += DIMS) { if constexpr (F32_VEC2) { for (uint32_t idx = tid; idx < KEYS * (DIMS / 2u); idx += blockDim.x) { From 9ee2beb64a91ac3747df4540ab91daccc9d107f9 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Sat, 29 Aug 2026 10:48:21 +0200 Subject: [PATCH 151/189] Update rocm/ds4_rocm_indexer.cuh Co-authored-by: Frank --- rocm/ds4_rocm_indexer.cuh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rocm/ds4_rocm_indexer.cuh b/rocm/ds4_rocm_indexer.cuh index 173ee9995..7e1e0d940 100644 --- a/rocm/ds4_rocm_indexer.cuh +++ b/rocm/ds4_rocm_indexer.cuh @@ -336,7 +336,8 @@ __global__ static void indexer_top1_rows_kernel( indices[tid] = best_index; __syncthreads(); - for (uint32_t stride = THREADS / 2u; stride > 0u; stride >>= 1u) { +#pragma unroll + for (uint32_t stride = THREADS / 2; stride > 0u; stride >>= 1u) { if (tid < stride && topk_score_better(values[tid + stride], indices[tid + stride], values[tid], indices[tid])) { From 1ca06844543cf1558827f04b32f2c792e484f95e Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Sat, 29 Aug 2026 10:48:49 +0200 Subject: [PATCH 152/189] Update rocm/ds4_rocm_matmul.cuh Co-authored-by: Frank --- rocm/ds4_rocm_matmul.cuh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rocm/ds4_rocm_matmul.cuh b/rocm/ds4_rocm_matmul.cuh index 19c0a2659..e5af30f2f 100644 --- a/rocm/ds4_rocm_matmul.cuh +++ b/rocm/ds4_rocm_matmul.cuh @@ -51,6 +51,7 @@ __global__ static void matmul_f16_smallm_wmma_kernel( rocwmma::fill_fragment(c, 0.0f); for (uint32_t k0 = 0; k0 < k; k0 += 16u) { +#pragma unroll for (uint32_t p = tid; p < BM * 8u; p += WAVES * 32u) { const uint32_t j = p * 2u; const uint32_t kk = j & 15u; @@ -59,6 +60,7 @@ __global__ static void matmul_f16_smallm_wmma_kernel( *reinterpret_cast( w + (k0 + kk) + (uint64_t)(m0 + row) * k); } +#pragma unroll for (uint32_t p = tid; p < 8u * BN; p += WAVES * 32u) { const uint32_t j = p * 2u; const uint32_t kk = j & 15u; From 4c37ac87d8ada062e2747e006d9de157065af03f Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Sat, 29 Aug 2026 10:49:11 +0200 Subject: [PATCH 153/189] Update rocm/ds4_rocm_matmul.cuh Co-authored-by: Frank --- rocm/ds4_rocm_matmul.cuh | 54 ++++++++++++++++++++++++++++++---------- 1 file changed, 41 insertions(+), 13 deletions(-) diff --git a/rocm/ds4_rocm_matmul.cuh b/rocm/ds4_rocm_matmul.cuh index e5af30f2f..c90b1b597 100644 --- a/rocm/ds4_rocm_matmul.cuh +++ b/rocm/ds4_rocm_matmul.cuh @@ -82,20 +82,48 @@ __global__ static void matmul_f16_smallm_wmma_kernel( __global__ static void matmul_f16_tinym24_wmma_kernel( float *out, const half *w, const half *x) { - constexpr uint32_t M=24u, N=2048u, K=16384u, BM=32u, BN=64u, WAVES=8u; - __shared__ half sa[BM*16u]; __shared__ half sb[16u*BN]; __shared__ float sc[BM*BN]; - const uint32_t tid=threadIdx.x, wave=tid>>5u, wm=wave&1u, wn=wave>>1u, n0=blockIdx.x*BN; - using fa_t=rocwmma::fragment; - using fb_t=rocwmma::fragment; - using fc_t=rocwmma::fragment; - fa_t a; fb_t b; fc_t c; rocwmma::fill_fragment(c,0.0f); - for (uint32_t k0=0;k0>5u; sa[j]=r>4u; *reinterpret_cast(sb+j)=*reinterpret_cast(x+k0+k+(uint64_t)(n0+n)*K); } - __syncthreads(); rocwmma::load_matrix_sync(a,sa+wm*16u,BM); rocwmma::load_matrix_sync(b,sb+wn*256u,16u); rocwmma::mma_sync(c,a,b,c); __syncthreads(); + constexpr uint32_t M = 24u, N = 2048u, K = 16384u; + constexpr uint32_t BM = 32u, BN = 64u, WAVES = 8u; + __shared__ half sa[BM * 16u]; + __shared__ half sb[16u * BN]; + __shared__ float sc[BM * BN]; + const uint32_t tid = threadIdx.x, wave = tid >> 5u; + const uint32_t wm = wave & 1u, wn = wave >> 1u; + const uint32_t n0 = blockIdx.x * BN; + using fa_t = rocwmma::fragment; + using fb_t = rocwmma::fragment; + using fc_t = rocwmma::fragment; + fa_t a; + fb_t b; + fc_t c; + rocwmma::fill_fragment(c, 0.0f); +#pragma unroll + for (uint32_t k0 = 0; k0 < K; k0 += 16u) { +#pragma unroll + for (uint32_t j = tid; j < BM * 16u; j += WAVES * 32u) { + const uint32_t r = j & 31u; + const uint32_t k = j >> 5u; + sa[j] = r < M ? w[(uint64_t)r * K + k0 + k] : __float2half(0.0f); + } +#pragma unroll + for (uint32_t p = tid; p < 8u * BN; p += WAVES * 32u) { + const uint32_t j = p * 2u, k = j & 15u, n = j >> 4u; + *reinterpret_cast(sb + j) = *reinterpret_cast(x + k0 + k + (uint64_t)(n0 + n) * K); + } + __syncthreads(); + rocwmma::load_matrix_sync(a, sa + wm * 16u, BM); + rocwmma::load_matrix_sync(b, sb + wn * 256u, 16u); + rocwmma::mma_sync(c, a, b, c); + __syncthreads(); + } + rocwmma::store_matrix_sync(sc + wm * 16u + wn * 16u * BM, c, + BM, rocwmma::mem_col_major); + __syncthreads(); +#pragma unroll + for (uint32_t i = tid; i < M * BN; i += WAVES * 32u) { + const uint32_t n = i / M, r = i - n * M; + out[r + (uint64_t)(n0 + n) * M] = sc[r + n * BM]; } - rocwmma::store_matrix_sync(sc+wm*16u+wn*16u*BM,c,BM,rocwmma::mem_col_major); __syncthreads(); - for (uint32_t i=tid;i From 7ab7fb923c316aa19a20e109e0857e4565be5029 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Sat, 29 Aug 2026 10:49:38 +0200 Subject: [PATCH 154/189] Update rocm/ds4_rocm_norm_rope.cuh Co-authored-by: Frank --- rocm/ds4_rocm_norm_rope.cuh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/rocm/ds4_rocm_norm_rope.cuh b/rocm/ds4_rocm_norm_rope.cuh index 308049d0a..949fcb181 100644 --- a/rocm/ds4_rocm_norm_rope.cuh +++ b/rocm/ds4_rocm_norm_rope.cuh @@ -996,9 +996,8 @@ extern "C" int ds4_gpu_attn_q_b_f16_head_rms_rope_tail_tensor( return 0; } if (weight_type == DS4_ROCM_Q4_K_TYPE) { - const int required = f16_cache_required; const int persistent_requested = - required || + f16_cache_required || (rocm_q4_attn_q_b_f16_enabled() && !rocm_q4_attn_q_b_f16_disabled()); if (persistent_requested) { From 40f7681bbf14ca9dd2b6af0ae8a14ba94e9a25ad Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Sat, 29 Aug 2026 10:49:58 +0200 Subject: [PATCH 155/189] Update rocm/ds4_rocm_q4.cuh Co-authored-by: Frank --- rocm/ds4_rocm_q4.cuh | 1 + 1 file changed, 1 insertion(+) diff --git a/rocm/ds4_rocm_q4.cuh b/rocm/ds4_rocm_q4.cuh index c7fb7909a..18dc10e51 100644 --- a/rocm/ds4_rocm_q4.cuh +++ b/rocm/ds4_rocm_q4.cuh @@ -194,6 +194,7 @@ __global__ static void rocm_matmul_q4_K_prefill_wmma64_strided_kernel( const cuda_block_q4_K *block = row_blocks + block_index; const float block_d = dev_f16_to_f32(block->d); const float block_dm = dev_f16_to_f32(block->dmin); +#pragma unroll for (uint32_t qpair = 0u; qpair < 4u; qpair++) { /* Adjacent 32-value groups are the low/high nibbles of the same * 32 payload bytes. Keep them in registers across both K tiles From e9bf171b851ee85ab43a0af85941324b848de775 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Sat, 29 Aug 2026 10:50:23 +0200 Subject: [PATCH 156/189] Update rocm/ds4_rocm_q4.cuh Co-authored-by: Frank --- rocm/ds4_rocm_q4.cuh | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/rocm/ds4_rocm_q4.cuh b/rocm/ds4_rocm_q4.cuh index 18dc10e51..6f33df66a 100644 --- a/rocm/ds4_rocm_q4.cuh +++ b/rocm/ds4_rocm_q4.cuh @@ -1032,21 +1032,6 @@ static int rocm_q4_K_prefill_wmma_launch( __ATOMIC_RELAXED); } return ok; -#else - (void)out; - (void)w; - (void)x; - (void)n_tok; - (void)n_groups; - (void)in_dim; - (void)out_dim; - (void)row_bytes; - (void)x_token_stride; - (void)x_group_stride; - (void)out_token_stride; - (void)label; - return 0; -#endif } enum { From 1d42ce41fbaac4ed16e8c8a3560e369aef01b04f Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Sat, 29 Aug 2026 10:50:47 +0200 Subject: [PATCH 157/189] Update rocm/ds4_rocm_q4.cuh Co-authored-by: Frank --- rocm/ds4_rocm_q4.cuh | 1 - 1 file changed, 1 deletion(-) diff --git a/rocm/ds4_rocm_q4.cuh b/rocm/ds4_rocm_q4.cuh index 6f33df66a..11c855c99 100644 --- a/rocm/ds4_rocm_q4.cuh +++ b/rocm/ds4_rocm_q4.cuh @@ -1014,7 +1014,6 @@ static int rocm_q4_K_prefill_wmma_launch( uint64_t x_group_stride, uint64_t out_token_stride, const char *label) { -#if defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__) if (!out || !w || !x || n_groups == 0u) return 0; const dim3 grid( (unsigned)(((uint64_t)out_dim + ROCM_Q4_WMMA_N_TILE - 1u) / From ccb2cf5d7337fe0d2847511ab59c253506d26102 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Sat, 29 Aug 2026 10:51:07 +0200 Subject: [PATCH 158/189] Update speed-bench/.gitignore Co-authored-by: Frank --- speed-bench/.gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/speed-bench/.gitignore b/speed-bench/.gitignore index 334d0d53e..68e3a2dfd 100644 --- a/speed-bench/.gitignore +++ b/speed-bench/.gitignore @@ -6,3 +6,7 @@ metal_prefill_variant_bench metal_q4_dense_pair_bench metal_q4_prefill_pair_bench metal_q4_mm_tail_cull_bench +metal_iq2_moe_tail_cull_bench +gpu_iq2_moe_prefill_bench +cuda_q4_prefill_bench +rocm_q4_prefill_bench From 1ff1e150a1a812d5ffe0ddf3bf1379f39b144110 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Sat, 29 Aug 2026 10:51:25 +0200 Subject: [PATCH 159/189] Update .gitignore Co-authored-by: Frank --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index c77b669c0..f300c00a3 100644 --- a/.gitignore +++ b/.gitignore @@ -36,6 +36,7 @@ /tests/test_sampling /tests/test_glm53_kda /tests/test_quantizer_indexer_q4 +/tests/test_rocm_q4_dense_pair *.o *.dSYM/ __pycache__/ From 3c8fbf34d66a46e5c06fd6efd3501592a2e92960 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Sat, 29 Aug 2026 11:37:08 +0200 Subject: [PATCH 160/189] Remove dead code across inference backends --- Makefile | 2 +- STRIXHALO.md | 2 +- cuda/mmq/ds4_ggml_stubs.cu | 4 - cuda/mmq/ds4_mmq.cu | 14 - ds4.c | 867 +------------------------------- ds4_cuda.cu | 468 ----------------- ds4_distributed.c | 25 - ds4_distributed.h | 1 - ds4_kvstore.c | 19 - ds4_kvstore.h | 10 +- ds4_metal.m | 304 +---------- ds4_rocm.cu | 9 - ds4_server.c | 7 - ds4_tp.c | 36 -- ds4_tp.h | 10 - linenoise.c | 22 - metal/dsv4_misc.metal | 293 ----------- metal/glu.metal | 28 -- metal/moe.metal | 251 --------- metal/sum_rows.metal | 1 - rocm/ds4_rocm_attention.cuh | 86 ---- rocm/ds4_rocm_glm.cuh | 65 --- rocm/ds4_rocm_hipblaslt.cuh | 137 ----- rocm/ds4_rocm_moe.cuh | 535 -------------------- rocm/ds4_rocm_norm_rope.cuh | 38 -- rocm/ds4_rocm_q8.cuh | 542 -------------------- rocm/ds4_rocm_runtime.cuh | 210 -------- rocm/ds4_rocm_shared_expert.cuh | 192 ------- 28 files changed, 25 insertions(+), 4153 deletions(-) delete mode 100644 rocm/ds4_rocm_hipblaslt.cuh diff --git a/Makefile b/Makefile index 71c71ed63..e9e8eed8b 100644 --- a/Makefile +++ b/Makefile @@ -59,7 +59,7 @@ HIPCC ?= $(shell command -v hipcc 2>/dev/null || echo /opt/rocm/bin/hipcc) ROCM_ARCH ?= gfx1151 ROCM_HOST_CFLAGS ?= -fPIC ROCM_CFLAGS ?= -O3 -ffast-math -g -fno-finite-math-only -pthread -D__HIP_PLATFORM_AMD__ -Wno-unused-command-line-argument --offload-arch=$(ROCM_ARCH) -ROCM_LDLIBS ?= -lm -pthread -lhipblas -lhipblaslt -lrocblas +ROCM_LDLIBS ?= -lm -pthread -lhipblas -lrocblas ROCM_MMQ_Y ?= 64 ROCM_MMQ_FLAGS := $(ROCM_CFLAGS) -std=c++17 -DGGML_USE_HIP -DDS4_HIP_MMQ_Y=$(ROCM_MMQ_Y) $(MMQ_INCLUDES) ROCM_MMQ_OBJS := cuda/mmq/ds4_ggml_stubs.rocm.o cuda/mmq/ds4_mmq.rocm.o cuda/mmq/quantize.rocm.o cuda/mmq/mmid.rocm.o cuda/mmq/mmvq.rocm.o cuda/mmq/d2r_stubs.rocm.o diff --git a/STRIXHALO.md b/STRIXHALO.md index 611239b7f..f5f7b5a8f 100644 --- a/STRIXHALO.md +++ b/STRIXHALO.md @@ -18,7 +18,7 @@ sudo apt-get update sudo apt-get install -y \ hipcc rocminfo rocm-smi \ libamdhip64-dev \ - libhipblas-dev libhipblaslt-dev \ + libhipblas-dev \ librocblas-dev \ librocwmma-dev \ libhipcub-dev diff --git a/cuda/mmq/ds4_ggml_stubs.cu b/cuda/mmq/ds4_ggml_stubs.cu index 1a5df364e..18154ddd4 100644 --- a/cuda/mmq/ds4_ggml_stubs.cu +++ b/cuda/mmq/ds4_ggml_stubs.cu @@ -147,10 +147,6 @@ extern "C" void ds4_pool_set_stream(cudaStream_t stream) { t_ds4_pool_stream = stream; } -extern "C" cudaStream_t ds4_pool_get_stream(void) { - return t_ds4_pool_stream; -} - namespace { struct ds4_naive_pool : public ggml_cuda_pool { diff --git a/cuda/mmq/ds4_mmq.cu b/cuda/mmq/ds4_mmq.cu index 406aa5a48..30b18b9a3 100644 --- a/cuda/mmq/ds4_mmq.cu +++ b/cuda/mmq/ds4_mmq.cu @@ -333,16 +333,6 @@ static void *ds4_mmq_aligned_q81_scratch(int device, size_t bytes) { ? g_aligned_q81_scratch_ptr : nullptr; } -// Legacy generic-scratch ABI. It must stay false: the grouped arena has a -// stricter lease contract and callers should inspect its counters/report API. -extern "C" int ds4_mmq_q81_persistent_enabled(void) { - return 0; -} - -extern "C" void *ds4_mmq_q81_scratch_ptr(void) { - return g_q81_scratch_ptr; -} - /* Test-only preflight hook. It deliberately traverses the production * acquire path (including owner/default-stream/capture checks and resize * retirement) without enqueueing a synthetic large MMQ fixture. */ @@ -671,10 +661,6 @@ static int64_t d2r_min_cols() { return cached; } -extern "C" size_t ds4_mmq_q81_scratch_bytes(void) { - return g_q81_scratch_bytes; -} - extern "C" int ds4_mmq_init(int device) { if (device < 0) { fprintf(stderr, "ds4_mmq_init: invalid device %d\n", device); diff --git a/ds4.c b/ds4.c index 9661c2cd9..88637bd5f 100644 --- a/ds4.c +++ b/ds4.c @@ -8234,26 +8234,6 @@ static void matvec_q8_0_3d_slice_prequant( ds4_parallel_for(out_dim, matvec_q8_0_worker, &ctx); } -static DS4_MAYBE_UNUSED void matvec_q8_0_3d_slice( - float * out, - const ds4_model * m, - const ds4_tensor * w, - const float * x, - uint64_t slice) { - if (w->type != DS4_TENSOR_Q8_0 || w->ndim != 3) ds4_die("expected a 3D Q8_0 tensor"); - - const uint64_t in_dim = w->dim[0]; - const uint64_t blocks = (in_dim + 31) / 32; - int8_t *xq = xmalloc((size_t)blocks * 32); - float *xscale = xmalloc((size_t)blocks * sizeof(xscale[0])); - - quantize_q8_0_activation(x, xq, xscale, in_dim); - matvec_q8_0_3d_slice_prequant(out, m, w, xq, xscale, slice); - - free(xscale); - free(xq); -} - /* Compute two Q8_0 projections from the same input, used by gate/up and * compressor kv/score pairs. */ static void matvec_q8_0_pair_prequant( @@ -10061,40 +10041,6 @@ static void quantize_mid_pairs_worker(void *vctx, uint64_t p0, uint64_t p1) { } } -typedef struct { - float *down_pair; - const uint8_t *base[DS4_MAX_EXPERT]; - const block_q8_K *midq; - const uint32_t *pair_ids; - const uint32_t *expert_offset; - const uint32_t *active_expert; - uint64_t in_dim; - uint64_t out_dim; - uint64_t row_bytes[DS4_MAX_EXPERT]; - uint64_t midq_blocks; -} matvec_q2_k_batch_down_ctx; - -static DS4_MAYBE_UNUSED void matvec_q2_k_batch_down_worker(void *vctx, uint64_t task0, uint64_t task1) { - matvec_q2_k_batch_down_ctx *ctx = vctx; - - for (uint64_t task = task0; task < task1; task++) { - const uint32_t active_idx = (uint32_t)(task / ctx->out_dim); - const uint64_t row = task - (uint64_t)active_idx * ctx->out_dim; - const uint32_t expert = ctx->active_expert[active_idx]; - const uint32_t begin = ctx->expert_offset[expert]; - const uint32_t end = ctx->expert_offset[expert + 1]; - const block_q2_K *br = (const block_q2_K *)(ctx->base[expert] + row * ctx->row_bytes[expert]); - - for (uint32_t i = begin; i < end; i++) { - const uint32_t pair_id = ctx->pair_ids[i]; - const block_q8_K *xq = ctx->midq + (uint64_t)pair_id * ctx->midq_blocks; - ds4_vec_dot_q2_K_q8_K((int)ctx->in_dim, - ctx->down_pair + (uint64_t)pair_id * ctx->out_dim + row, - br, xq); - } - } -} - typedef struct { float *moe; const uint8_t *base[DS4_MAX_EXPERT]; @@ -10895,27 +10841,6 @@ static void matvec_q8_0_batch_accum_rows_worker(void *vctx, uint64_t row0, uint6 } } -typedef struct { - float *moe; - const float *down_pair; - uint32_t n_tok; - uint64_t out_dim; -} sum_down_pairs_ctx; - -static DS4_MAYBE_UNUSED void sum_down_pairs_worker(void *vctx, uint64_t row0, uint64_t row1) { - sum_down_pairs_ctx *ctx = vctx; - for (uint64_t idx = row0; idx < row1; idx++) { - const uint32_t token = (uint32_t)(idx / ctx->out_dim); - const uint64_t row = idx - (uint64_t)token * ctx->out_dim; - float acc = 0.0f; - for (uint32_t slot = 0; slot < DS4_N_EXPERT_USED; slot++) { - const uint64_t pair_id = (uint64_t)token * DS4_N_EXPERT_USED + slot; - acc += ctx->down_pair[pair_id * ctx->out_dim + row]; - } - ctx->moe[idx] = acc; - } -} - /* ========================================================================= * Hyper-Connection Transforms. * ========================================================================= @@ -39755,8 +39680,6 @@ static uint32_t glm53_graph_resume_prefill_min_tokens(void) { #define DS4_GLM_METAL_FULL_ATTN_DEFAULT_CONTEXT 4096u #define DS4_GLM_METAL_STREAMING_FULL_ATTN_CONTEXT 8192u #define DS4_GLM_METAL_FULL_ATTN_LAYER_FLUSH_CONTEXT 2048u -#define DS4_GLM_METAL_DISPLAY_PROGRESS_LAYER_TOKENS 32u -#define DS4_GLM_METAL_SMALL_PREFILL_STAGE_SYNC_TOKENS 0u #define DS4_GLM_METAL_LONG_CONTEXT_THRESHOLD 65536u #define DS4_GLM_METAL_LONG_CONTEXT_FULL_ATTN_CONTEXT 4096u #define DS4_GLM_METAL_INDEXED_PREFILL_CHUNK_TOKENS 4096u @@ -44322,14 +44245,6 @@ static void glm_graph_report_prefill_display_progress( (int)((uint64_t)absolute_base + (uint64_t)work_total)); } -static bool glm_graph_small_prefill_stage_sync( - uint32_t n_tokens, - bool logits_requested) { - return logits_requested && - n_tokens > 0 && - n_tokens <= DS4_GLM_METAL_SMALL_PREFILL_STAGE_SYNC_TOKENS; -} - static uint32_t glm_graph_indexed_decode_split_min_block_rows(void) { return 32u; } @@ -44366,11 +44281,6 @@ static bool glm_graph_indexed_decode_split_group8_available(uint32_t n_selected) #endif } -static bool glm_graph_prefill_stage_sync_boundary(void) { - if (ds4_gpu_end_commands() == 0) return false; - return ds4_gpu_begin_commands() != 0; -} - static bool glm_graph_indexed_prefill_attention_boundary(void) { #ifdef DS4_ROCM_BUILD /* @@ -45547,10 +45457,7 @@ static bool glm_graph_warm_compact_indexer_store( return false; } - const bool profile = false; - const double t0 = 0.0; bool ok = ds4_gpu_begin_commands() != 0; - uint32_t warmed = 0; for (uint32_t il = g->layer_start; ok && il <= g->layer_end; il++) { if (!glm_graph_layer_uses_full_indexer(il)) continue; const ds4_layer_weights *l = &weights->layer[il]; @@ -45583,18 +45490,9 @@ static bool glm_graph_warm_compact_indexer_store( DS4_ROPE_YARN_BETA_FAST, DS4_ROPE_YARN_BETA_SLOW, glm_graph_compact_cache_is_f16()) != 0; - if (ok) warmed++; } if (ok) ok = ds4_gpu_end_commands() != 0; else (void)ds4_gpu_synchronize(); - - if (profile) { - fprintf(stderr, - "ds4: GLM compact indexer warmup pos=%u layers=%u %.3f ms\n", - warm_pos, - warmed, - (now_sec() - t0) * 1000.0); - } return ok; } @@ -46879,7 +46777,6 @@ static bool glm_graph_profile_router_selection_batch( static bool glm_graph_prefill_stage_boundary( bool stage_profile, - bool stage_sync, const char *part, const char *stage, uint32_t il, @@ -46889,7 +46786,6 @@ static bool glm_graph_prefill_stage_boundary( if (stage_profile) { return glm_graph_profile_stage(true, part, stage, il, pos0, n_tokens, stage_t0); } - if (stage_sync) return glm_graph_prefill_stage_sync_boundary(); return true; } @@ -48675,7 +48571,6 @@ static bool glm_graph_encode_sparse_ffn_indexed_batch_routed_moe( ds4_gpu_tensor *next, uint32_t n_tokens, bool stage_profile, - bool stage_sync, double *stage_t0) { const char *failed_stage = "setup"; if (!g || !model || !l || !after_attn || !next || @@ -48766,7 +48661,6 @@ static bool glm_graph_encode_sparse_ffn_indexed_batch_routed_moe( ds4_gpu_tensor_free(logits_view); } if (ok) ok = glm_graph_prefill_stage_boundary(stage_profile, - stage_sync, "glm_indexed_ffn", "router", il, @@ -48828,7 +48722,6 @@ static bool glm_graph_encode_sparse_ffn_indexed_batch_routed_moe( !use_grouped_moe) != 0; } if (ok) ok = glm_graph_prefill_stage_boundary(stage_profile, - stage_sync, "glm_indexed_ffn", "routed_moe", il, @@ -48978,7 +48871,6 @@ static bool glm_graph_encode_sparse_ffn_indexed_batch_routed_moe( ds4_gpu_tensor_free(after_attn_view); } if (ok) ok = glm_graph_prefill_stage_boundary(stage_profile, - stage_sync, "glm_indexed_ffn", "shared_expert", il, @@ -49005,7 +48897,6 @@ static bool glm_graph_encode_sparse_ffn_indexed_batch_routed_moe( } } if (ok) ok = glm_graph_prefill_stage_boundary(stage_profile, - stage_sync, "glm_indexed_ffn", "residual", il, @@ -49063,7 +48954,6 @@ static bool glm_graph_encode_ffn_batch( uint32_t n_tokens, bool full_layer_prefill, bool stage_profile, - bool stage_sync, double *stage_t0) { if (!g || !model || !weights || !l || !after_attn || !next || n_tokens == 0) return false; @@ -49077,7 +48967,6 @@ static bool glm_graph_encode_ffn_batch( n_tokens, DS4_RMS_EPS) != 0; if (ok) ok = glm_graph_prefill_stage_boundary(stage_profile, - stage_sync, "glm_ffn", "ffn_norm", il, @@ -49114,7 +49003,6 @@ static bool glm_graph_encode_ffn_batch( g->glm53 ? DS4_SWIGLU_CLAMP_EXP : 0.0f); if (fused_gate_up) { ok = glm_graph_prefill_stage_boundary(stage_profile, - stage_sync, "glm_ffn", "dense_gate_up_swiglu", il, @@ -49137,7 +49025,6 @@ static bool glm_graph_encode_ffn_batch( g->batch_ffn_norm, n_tokens); if (ok) ok = glm_graph_prefill_stage_boundary(stage_profile, - stage_sync, "glm_ffn", "dense_gate_up", il, @@ -49151,7 +49038,6 @@ static bool glm_graph_encode_ffn_batch( g->glm53 ? DS4_SWIGLU_CLAMP_EXP : 0.0f, 1.0f) != 0; if (ok) ok = glm_graph_prefill_stage_boundary(stage_profile, - stage_sync, "glm_ffn", "dense_swiglu", il, @@ -49177,7 +49063,6 @@ static bool glm_graph_encode_ffn_batch( (uint64_t)n_tokens * DS4_N_EMBD, il, pos0); } if (ok) ok = glm_graph_prefill_stage_boundary(stage_profile, - stage_sync, "glm_ffn", "dense_down", il, @@ -49197,7 +49082,6 @@ static bool glm_graph_encode_ffn_batch( (uint32_t)residual_elems) != 0; } if (ok) ok = glm_graph_prefill_stage_boundary(stage_profile, - stage_sync, "glm_ffn", "residual", il, @@ -49260,7 +49144,6 @@ static bool glm_graph_encode_ffn_batch( } } if (ok) ok = glm_graph_prefill_stage_boundary(stage_profile, - stage_sync, "glm_ffn", "router", il, @@ -49328,7 +49211,6 @@ static bool glm_graph_encode_ffn_batch( g->glm53 ? DS4_SWIGLU_CLAMP_EXP : 0.0f); \ if (fused_shared) { \ ok = glm_graph_prefill_stage_boundary(stage_profile, \ - stage_sync, \ "glm_ffn", \ "shared_gate_up_swiglu", \ il, \ @@ -49351,7 +49233,6 @@ static bool glm_graph_encode_ffn_batch( g->batch_ffn_norm, \ n_tokens); \ if (ok) ok = glm_graph_prefill_stage_boundary(stage_profile, \ - stage_sync, \ "glm_ffn", \ "shared_gate_up", \ il, \ @@ -49365,7 +49246,6 @@ static bool glm_graph_encode_ffn_batch( g->glm53 ? DS4_SWIGLU_CLAMP_EXP : 0.0f, \ 1.0f) != 0; \ if (ok) ok = glm_graph_prefill_stage_boundary(stage_profile, \ - stage_sync, \ "glm_ffn", \ "shared_swiglu", \ il, \ @@ -49382,7 +49262,6 @@ static bool glm_graph_encode_ffn_batch( g->batch_shared_mid, \ n_tokens); \ if (ok) ok = glm_graph_prefill_stage_boundary(stage_profile, \ - stage_sync, \ "glm_ffn", \ "shared_down", \ il, \ @@ -49456,7 +49335,6 @@ static bool glm_graph_encode_ffn_batch( if (!ok) fprintf(stderr, "ds4: GLM TP batch gate failed (layer %u)\n", il); } if (ok) ok = glm_graph_prefill_stage_boundary(stage_profile, - stage_sync, "glm_ffn", "routed_moe", il, @@ -49501,7 +49379,6 @@ static bool glm_graph_encode_ffn_batch( (uint32_t)residual_elems) != 0; } if (ok) ok = glm_graph_prefill_stage_boundary(stage_profile, - stage_sync, "glm_ffn", "residual", il, @@ -50368,7 +50245,6 @@ static bool glm_graph_verify_rows( n, false, false, - false, NULL); if (ok) { ds4_gpu_tensor *tmp = cur; @@ -50625,9 +50501,7 @@ static bool glm_graph_forward_tokens( work_total, true); - const bool stage_sync = - glm_graph_small_prefill_stage_sync(n_tokens, logits_out != NULL); - const uint32_t layer_flush_interval = stage_sync ? 0u : + const uint32_t layer_flush_interval = glm_graph_full_prefill_layer_flush_interval(n_tokens, n_rows, logits_out != NULL); @@ -50698,10 +50572,9 @@ static bool glm_graph_forward_tokens( metal_graph_stream_prefill_layer_prepare_ahead() : 1u; if (trace) { glm_graph_full_prefill_tracef( - "mode pos=%u tokens=%u stage_sync=%u layer_flush_interval=%u progress_flush_interval=%u drain_interval=%u", + "mode pos=%u tokens=%u layer_flush_interval=%u progress_flush_interval=%u drain_interval=%u", pos0, n_tokens, - stage_sync ? 1u : 0u, layer_flush_interval, progress_flush_interval, drain_interval); @@ -50825,8 +50698,6 @@ static bool glm_graph_forward_tokens( #define DS4_GLM_PROFILE_PREFILL_STAGE(part_, name_) do { \ if (ok && layer_stage_profile) { \ ok = metal_graph_layer_stage_profile_boundary((part_), (name_), il, pos0, n_tokens, &layer_stage_t0); \ - } else if (ok && stage_sync) { \ - ok = glm_graph_prefill_stage_sync_boundary(); \ } \ } while (0) for (uint32_t il = g->layer_start; ok && il <= g->layer_end; il++) { @@ -51444,7 +51315,6 @@ static bool glm_graph_forward_tokens( n_tokens, full_layer_prefill, layer_stage_profile, - stage_sync, layer_stage_profile ? &layer_stage_t0 : NULL); } if (ok && g->glm53) { @@ -51972,9 +51842,7 @@ static bool glm_graph_forward_indexed_tokens( const bool use_batch_indexer_weights_proj = true; const bool use_batch_attn_out_proj = true; const bool use_batch_ffn = glm_graph_indexed_prefill_batch_ffn(); - const bool stage_sync = - glm_graph_small_prefill_stage_sync(n_tokens, logits_out != NULL); - const uint32_t layer_flush_interval = stage_sync ? 0u : + const uint32_t layer_flush_interval = glm_graph_full_prefill_layer_flush_interval(n_tokens, n_tokens, logits_out != NULL); @@ -52107,8 +51975,6 @@ static bool glm_graph_forward_indexed_tokens( } \ if (ok && layer_stage_profile) { \ ok = metal_graph_layer_stage_profile_boundary((part_), (name_), il, pos0, n_tokens, &layer_stage_t0); \ - } else if (ok && stage_sync) { \ - ok = glm_graph_prefill_stage_sync_boundary(); \ } \ } while (0) ds4_gpu_tensor *last_indexer_selected = NULL; @@ -53077,7 +52943,6 @@ static bool glm_graph_forward_indexed_tokens( n_tokens, false, layer_stage_profile, - stage_sync, layer_stage_profile ? &layer_stage_t0 : NULL); } else if (ok) { const bool use_batch_ffn_norm = @@ -53109,7 +52974,6 @@ static bool glm_graph_forward_indexed_tokens( next, n_tokens, layer_stage_profile, - stage_sync, layer_stage_profile ? &layer_stage_t0 : NULL); } else for (uint32_t t = 0; ok && t < n_tokens; t++) { ds4_gpu_tensor *after_attn_view = @@ -53931,13 +53795,8 @@ static bool glm_graph_forward_token( } } - const bool decode_output_profile = false; const bool merge_indexed_output = - logits_out != NULL && use_indexed_attention && !decode_output_profile; - double decode_output_stage_t0 = decode_output_profile ? now_sec() : 0.0; - const bool decode_flush_profile = false; - uint32_t decode_flush_layer0 = 0; - double decode_flush_stage_t0 = decode_flush_profile ? now_sec() : 0.0; + logits_out != NULL && use_indexed_attention; const bool static_decode_map = !input_hc && @@ -54852,24 +54711,7 @@ static bool glm_graph_forward_token( decode_layer_flush_interval != 0 && il < g->layer_end && (slice_layer_done % decode_layer_flush_interval) == 0) { - if (decode_flush_profile) { - ok = ds4_gpu_flush_commands() != 0; - if (ok) ok = ds4_gpu_synchronize() != 0; - if (ok) { - const double now = now_sec(); - fprintf(stderr, - "ds4: GLM decode layer flush pos=%u layers=%u..%u %.3f ms\n", - pos, - decode_flush_layer0, - il, - (now - decode_flush_stage_t0) * 1000.0); - decode_flush_layer0 = il + 1u; - decode_flush_stage_t0 = now; - ok = ds4_gpu_begin_commands() != 0; - } - } else { - ok = ds4_gpu_flush_commands() != 0; - } + ok = ds4_gpu_flush_commands() != 0; } if (streaming_decode_sync_each_layer) { if (ok) ok = ds4_gpu_end_commands() != 0; @@ -54898,14 +54740,6 @@ static bool glm_graph_forward_token( } else if (!ok) { (void)ds4_gpu_synchronize(); } - if (decode_output_profile) { - const double now = now_sec(); - fprintf(stderr, - "ds4: GLM decode output profile pos=%u layers=%.3f ms\n", - pos, - (now - decode_output_stage_t0) * 1000.0); - decode_output_stage_t0 = now; - } if (ok && output_hc && !defer_completion) { ok = ds4_gpu_tensor_read(g->glm53 ? g->hc_cur : g->cur, 0, @@ -54916,23 +54750,6 @@ static bool glm_graph_forward_token( } if (ok && logits_out && !defer_completion) { if (use_indexed_attention) { - if (!merge_indexed_output) { - if (g->ssd_streaming && !static_decode_map) { - ok = glm_graph_stream_map_output(g, model, weights); - } - if (ok) ok = glm_graph_begin_commands_if_needed(); - if (ok) ok = glm_graph_encode_output_head(g, model, weights); - if (ok) ok = glm_graph_end_commands_if_active(); - else (void)ds4_gpu_synchronize(); - if (decode_output_profile) { - const double now = now_sec(); - fprintf(stderr, - "ds4: GLM decode output profile pos=%u output_head=%.3f ms\n", - pos, - (now - decode_output_stage_t0) * 1000.0); - decode_output_stage_t0 = now; - } - } if (ok) { if (glm_debug_hidden_dump_layer() < 0) glm_debug_dump_hidden_row(g->cur, 0); @@ -54940,13 +54757,6 @@ static bool glm_graph_forward_token( 0, logits_out, (uint64_t)DS4_N_VOCAB * sizeof(float)) != 0; - if (decode_output_profile) { - const double now = now_sec(); - fprintf(stderr, - "ds4: GLM decode output profile pos=%u logits_read=%.3f ms\n", - pos, - (now - decode_output_stage_t0) * 1000.0); - } } } else { if (g->ssd_streaming && !static_decode_map) { @@ -54961,13 +54771,6 @@ static bool glm_graph_forward_token( weights, g->glm53 ? g->hc_cur : g->cur, logits_out); - if (decode_output_profile) { - const double now = now_sec(); - fprintf(stderr, - "ds4: GLM decode output profile pos=%u fallback_output=%.3f ms\n", - pos, - (now - decode_output_stage_t0) * 1000.0); - } } } if (ok && @@ -54994,482 +54797,6 @@ static bool glm_graph_forward_token( #undef DS4_GLM_FT_FAIL } -static int glm_metal_first_token_logits( - const ds4_model *model, - const ds4_weights *weights, - int token, - float *logits_out) { - if (!model || !weights || !logits_out) return 1; - if (token < 0 || token >= (int)DS4_N_VOCAB) { - fprintf(stderr, "ds4: GLM token %d is outside vocab\n", token); - return 1; - } - if (!weights->token_embd || weights->token_embd->type != DS4_TENSOR_Q8_0 || - !weights->output_norm || weights->output_norm->type != DS4_TENSOR_F32 || - !weights->output || weights->output->type != DS4_TENSOR_Q8_0 || - weights->output_norm->dim[0] != DS4_N_EMBD || - weights->output->dim[0] != DS4_N_EMBD || - weights->output->dim[1] != DS4_N_VOCAB) { - fprintf(stderr, "ds4: GLM Metal first-token path found unexpected embedding/output layout\n"); - return 1; - } - if (DS4_N_LAYER <= DS4_N_NEXTN_PREDICT) { - fprintf(stderr, "ds4: GLM Metal first-token path has no normal transformer layers\n"); - return 1; - } - - const uint32_t normal_layers = DS4_N_LAYER - DS4_N_NEXTN_PREDICT; - const uint64_t heads_dim = (uint64_t)DS4_N_HEAD * DS4_N_VALUE_MLA; - uint64_t kv_raw_dim = 0; - uint64_t dense_hidden_max = DS4_N_FF_EXP; - bool generic_routed_moe = false; - for (uint32_t il = 0; il < normal_layers; il++) { - const ds4_layer_weights *l = &weights->layer[il]; - if (l->attn_kv_a_mqa && l->attn_kv_a_mqa->dim[1] > kv_raw_dim) { - kv_raw_dim = l->attn_kv_a_mqa->dim[1]; - } - if (il < DS4_N_LEADING_DENSE && l->ffn_gate && - l->ffn_gate->dim[1] > dense_hidden_max) { - dense_hidden_max = l->ffn_gate->dim[1]; - } - if (glm_graph_layer_uses_generic_routed_moe(l)) generic_routed_moe = true; - } - if (kv_raw_dim < DS4_N_KV_LORA) { - fprintf(stderr, "ds4: GLM Metal first-token path found no valid KV projection\n"); - return 1; - } - - const uint64_t emb_bytes = (uint64_t)DS4_N_EMBD * sizeof(float); - const uint64_t sparse_mid_elems = (uint64_t)DS4_N_EXPERT_USED * DS4_N_FF_EXP; - const uint64_t ffn_mid_elems = - dense_hidden_max > sparse_mid_elems ? dense_hidden_max : sparse_mid_elems; - const uint64_t routed_mid_bytes = - (uint64_t)DS4_N_EXPERT_USED * DS4_N_FF_EXP * sizeof(float); - const uint64_t routed_down_bytes = - (uint64_t)DS4_N_EXPERT_USED * DS4_N_EMBD * sizeof(float); - const uint64_t logits_bytes = (uint64_t)DS4_N_VOCAB * sizeof(float); - - ds4_gpu_tensor *cur = NULL; - ds4_gpu_tensor *attn_norm = NULL; - ds4_gpu_tensor *kv_raw = NULL; - ds4_gpu_tensor *kv_norm = NULL; - ds4_gpu_tensor *heads = NULL; - ds4_gpu_tensor *attn_out = NULL; - ds4_gpu_tensor *after_attn = NULL; - ds4_gpu_tensor *ffn_norm = NULL; - ds4_gpu_tensor *ffn_gate = NULL; - ds4_gpu_tensor *ffn_up = NULL; - ds4_gpu_tensor *ffn_mid = NULL; - ds4_gpu_tensor *routed_gate = NULL; - ds4_gpu_tensor *routed_up = NULL; - ds4_gpu_tensor *routed_down = NULL; - ds4_gpu_tensor *ffn_out = NULL; - ds4_gpu_tensor *ffn_sum = NULL; - ds4_gpu_tensor *next = NULL; - ds4_gpu_tensor *router_logits = NULL; - ds4_gpu_tensor *router_probs = NULL; - ds4_gpu_tensor *router_selected = NULL; - ds4_gpu_tensor *router_weights = NULL; - ds4_gpu_tensor *logits = NULL; - - int ok = 1; -#define DS4_GLM_FIRST_ALLOC_TENSOR(var, bytes_) \ - do { \ - (var) = ds4_gpu_tensor_alloc((bytes_)); \ - if (!(var)) { \ - fprintf(stderr, "ds4: GLM Metal first-token path could not allocate %s\n", #var); \ - ok = 0; \ - } \ - } while (0) - - DS4_GLM_FIRST_ALLOC_TENSOR(cur, emb_bytes); - DS4_GLM_FIRST_ALLOC_TENSOR(attn_norm, emb_bytes); - DS4_GLM_FIRST_ALLOC_TENSOR(kv_raw, kv_raw_dim * sizeof(float)); - DS4_GLM_FIRST_ALLOC_TENSOR(kv_norm, (uint64_t)DS4_N_KV_LORA * sizeof(float)); - DS4_GLM_FIRST_ALLOC_TENSOR(heads, heads_dim * sizeof(float)); - DS4_GLM_FIRST_ALLOC_TENSOR(attn_out, emb_bytes); - DS4_GLM_FIRST_ALLOC_TENSOR(after_attn, emb_bytes); - DS4_GLM_FIRST_ALLOC_TENSOR(ffn_norm, emb_bytes); - DS4_GLM_FIRST_ALLOC_TENSOR(ffn_gate, dense_hidden_max * sizeof(float)); - DS4_GLM_FIRST_ALLOC_TENSOR(ffn_up, dense_hidden_max * sizeof(float)); - DS4_GLM_FIRST_ALLOC_TENSOR(ffn_mid, ffn_mid_elems * sizeof(float)); - if (generic_routed_moe) { - DS4_GLM_FIRST_ALLOC_TENSOR(routed_gate, routed_mid_bytes); - DS4_GLM_FIRST_ALLOC_TENSOR(routed_up, routed_mid_bytes); - DS4_GLM_FIRST_ALLOC_TENSOR(routed_down, routed_down_bytes); - } - DS4_GLM_FIRST_ALLOC_TENSOR(ffn_out, emb_bytes); - DS4_GLM_FIRST_ALLOC_TENSOR(ffn_sum, emb_bytes); - DS4_GLM_FIRST_ALLOC_TENSOR(next, emb_bytes); - DS4_GLM_FIRST_ALLOC_TENSOR(router_logits, (uint64_t)DS4_N_EXPERT * sizeof(float)); - DS4_GLM_FIRST_ALLOC_TENSOR(router_probs, (uint64_t)DS4_N_EXPERT * sizeof(float)); - DS4_GLM_FIRST_ALLOC_TENSOR(router_selected, (uint64_t)DS4_N_EXPERT_USED * sizeof(int32_t)); - DS4_GLM_FIRST_ALLOC_TENSOR(router_weights, (uint64_t)DS4_N_EXPERT_USED * sizeof(float)); - DS4_GLM_FIRST_ALLOC_TENSOR(logits, logits_bytes); -#undef DS4_GLM_FIRST_ALLOC_TENSOR - - if (ok) { - ok = ds4_gpu_embed_token_q8_0_tensor(cur, - model->map, - model->size, - weights->token_embd->abs_offset, - DS4_N_VOCAB, - (uint32_t)token, - DS4_N_EMBD); - } - for (uint32_t il = 0; ok && il < normal_layers; il++) { - const ds4_layer_weights *gl = &weights->layer[il]; - const uint64_t gl_kv_raw_dim = gl->attn_kv_a_mqa ? gl->attn_kv_a_mqa->dim[1] : 0; - if (!gl->attn_norm || - !gl->attn_kv_a_mqa || - !gl->attn_kv_a_norm || - !gl->attn_v_b || - !gl->attn_output || - !gl->ffn_norm || - gl_kv_raw_dim < DS4_N_KV_LORA || - gl_kv_raw_dim > kv_raw_dim || - gl->attn_kv_a_mqa->type != DS4_TENSOR_Q8_0 || - gl->attn_kv_a_mqa->dim[0] != DS4_N_EMBD || - gl->attn_v_b->type != DS4_TENSOR_Q8_0 || - gl->attn_v_b->dim[0] != DS4_N_KV_LORA || - gl->attn_v_b->dim[1] != DS4_N_VALUE_MLA || - gl->attn_v_b->dim[2] != DS4_N_HEAD || - gl->attn_output->type != DS4_TENSOR_Q8_0 || - gl->attn_output->dim[0] != heads_dim || - gl->attn_output->dim[1] != DS4_N_EMBD) { - fprintf(stderr, - "ds4: GLM Metal first-token path found unexpected attention layout in layer %u\n", - il); - ok = 0; - break; - } - - if (ok) ok = ds4_gpu_rms_norm_weight_tensor(attn_norm, cur, - model->map, model->size, - gl->attn_norm->abs_offset, - DS4_N_EMBD, DS4_RMS_EPS); - if (ok) ok = ds4_gpu_matmul_q8_0_tensor(kv_raw, - model->map, - model->size, - gl->attn_kv_a_mqa->abs_offset, - DS4_N_EMBD, - gl_kv_raw_dim, - attn_norm, - 1); - if (ok) ok = ds4_gpu_rms_norm_weight_tensor(kv_norm, kv_raw, - model->map, model->size, - gl->attn_kv_a_norm->abs_offset, - DS4_N_KV_LORA, DS4_RMS_EPS); - if (ok) ok = ds4_gpu_matmul_q8_0_tensor(heads, - model->map, - model->size, - gl->attn_v_b->abs_offset, - DS4_N_KV_LORA, - heads_dim, - kv_norm, - 1); - if (ok) ok = ds4_gpu_matmul_q8_0_tensor(attn_out, - model->map, - model->size, - gl->attn_output->abs_offset, - heads_dim, - DS4_N_EMBD, - heads, - 1); - if (ok) ok = ds4_gpu_add_tensor(after_attn, cur, attn_out, DS4_N_EMBD); - if (ok) ok = ds4_gpu_rms_norm_weight_tensor(ffn_norm, after_attn, - model->map, model->size, - gl->ffn_norm->abs_offset, - DS4_N_EMBD, DS4_RMS_EPS); - if (il < DS4_N_LEADING_DENSE) { - const uint64_t gl_ffn_hidden = gl->ffn_gate ? gl->ffn_gate->dim[1] : 0; - if (!gl->ffn_gate || - !gl->ffn_up || - !gl->ffn_down || - gl->ffn_gate->type != DS4_TENSOR_Q8_0 || - gl->ffn_up->type != DS4_TENSOR_Q8_0 || - gl->ffn_down->type != DS4_TENSOR_Q8_0 || - gl->ffn_gate->dim[0] != DS4_N_EMBD || - gl->ffn_up->dim[0] != DS4_N_EMBD || - gl->ffn_up->dim[1] != gl_ffn_hidden || - gl->ffn_down->dim[0] != gl_ffn_hidden || - gl->ffn_down->dim[1] != DS4_N_EMBD || - gl_ffn_hidden > dense_hidden_max) { - fprintf(stderr, - "ds4: GLM Metal first-token path found unexpected dense FFN layout in layer %u\n", - il); - ok = 0; - break; - } - if (ok) ok = ds4_gpu_matmul_q8_0_tensor(ffn_gate, - model->map, - model->size, - gl->ffn_gate->abs_offset, - DS4_N_EMBD, - gl_ffn_hidden, - ffn_norm, - 1); - if (ok) ok = ds4_gpu_matmul_q8_0_tensor(ffn_up, - model->map, - model->size, - gl->ffn_up->abs_offset, - DS4_N_EMBD, - gl_ffn_hidden, - ffn_norm, - 1); - if (ok) ok = ds4_gpu_swiglu_tensor(ffn_mid, ffn_gate, ffn_up, - (uint32_t)gl_ffn_hidden, 0.0f, 1.0f); - if (ok) ok = ds4_gpu_matmul_q8_0_tensor(ffn_out, - model->map, - model->size, - gl->ffn_down->abs_offset, - gl_ffn_hidden, - DS4_N_EMBD, - ffn_mid, - 1); - if (ok) ok = ds4_gpu_add_tensor(next, after_attn, ffn_out, DS4_N_EMBD); - } else { - const uint32_t gl_gate_type = gl->ffn_gate_exps ? gl->ffn_gate_exps->type : 0; - const uint32_t gl_up_type = gl->ffn_up_exps ? gl->ffn_up_exps->type : 0; - const bool gl_gate_pair_supported = - glm_graph_gate_pair_type_supported(gl_gate_type, gl_up_type); - uint64_t gate_in = 0, gate_out = 0, gate_row_bytes = 0; - uint64_t up_in = 0, up_out = 0, up_row_bytes = 0; - uint64_t down_in = 0, down_out = 0, down_row_bytes = 0; - - if (!gl->ffn_gate_inp || - !gl->ffn_exp_probs_b || - !gl->ffn_gate_exps || - !gl->ffn_up_exps || - !gl->ffn_down_exps || - !gl->ffn_gate_shexp || - !gl->ffn_up_shexp || - !gl->ffn_down_shexp || - gl->ffn_gate_inp->type != DS4_TENSOR_F32 || - gl->ffn_gate_inp->dim[0] != DS4_N_EMBD || - gl->ffn_gate_inp->dim[1] != DS4_N_EXPERT || - gl->ffn_exp_probs_b->type != DS4_TENSOR_F32 || - gl->ffn_exp_probs_b->dim[0] != DS4_N_EXPERT || - !gl_gate_pair_supported || - !glm_graph_down_type_supported(gl->ffn_down_exps->type) || - gl->ffn_gate_shexp->type != DS4_TENSOR_Q8_0 || - gl->ffn_up_shexp->type != DS4_TENSOR_Q8_0 || - gl->ffn_down_shexp->type != DS4_TENSOR_Q8_0 || - gl->ffn_gate_shexp->dim[0] != DS4_N_EMBD || - gl->ffn_gate_shexp->dim[1] != DS4_N_FF_EXP || - gl->ffn_up_shexp->dim[0] != DS4_N_EMBD || - gl->ffn_up_shexp->dim[1] != DS4_N_FF_EXP || - gl->ffn_down_shexp->dim[0] != DS4_N_FF_EXP || - gl->ffn_down_shexp->dim[1] != DS4_N_EMBD || - sparse_mid_elems > ffn_mid_elems) { - fprintf(stderr, - "ds4: GLM Metal first-token path found unexpected sparse FFN layout in layer %u\n", - il); - ok = 0; - break; - } - - (void)tensor_expert_bytes(model, gl->ffn_gate_exps, 0, - &gate_in, &gate_out, &gate_row_bytes); - (void)tensor_expert_bytes(model, gl->ffn_up_exps, 0, - &up_in, &up_out, &up_row_bytes); - (void)tensor_expert_bytes(model, gl->ffn_down_exps, 0, - &down_in, &down_out, &down_row_bytes); - if (gate_in != DS4_N_EMBD || - up_in != DS4_N_EMBD || - down_in != DS4_N_FF_EXP || - gate_out != DS4_N_FF_EXP || - up_out != DS4_N_FF_EXP || - down_out != DS4_N_EMBD) { - fprintf(stderr, - "ds4: GLM Metal first-token path found unexpected expert strides in layer %u\n", - il); - ok = 0; - break; - } - - if (ok) ok = ds4_gpu_matmul_f32_tensor(router_logits, - model->map, - model->size, - gl->ffn_gate_inp->abs_offset, - DS4_N_EMBD, - DS4_N_EXPERT, - ffn_norm, - 1); - if (ok) ok = ds4_gpu_glm_router_select_tensor(router_selected, - router_weights, - router_probs, - model->map, - model->size, - gl->ffn_exp_probs_b->abs_offset, - router_logits, - DS4_N_EXPERT, - DS4_N_EXPERT_USED, - DS4_EXPERT_WEIGHT_SCALE); - if (ok) { - const ds4_gpu_stream_expert_table table = { - .model_map = model->map, - .model_size = model->size, - .layer = il, - .n_total_expert = DS4_N_EXPERT, - .gate_offset = gl->ffn_gate_exps->abs_offset, - .up_offset = gl->ffn_up_exps->abs_offset, - .down_offset = gl->ffn_down_exps->abs_offset, - .gate_expert_bytes = gate_out * gate_row_bytes, - .down_expert_bytes = down_out * down_row_bytes, - }; - ok = ds4_gpu_glm_stream_expert_cache_begin_selected_load_tensor( - &table, - router_selected, - DS4_N_EXPERT_USED) != 0; - } - ds4_glm_gpu_graph route_g = { - .routed_gate = routed_gate, - .routed_up = routed_up, - .routed_down = routed_down, - .ssd_streaming = false, - .glm53 = ds4_model_is_glm53(), - }; - if (ok) ok = glm_graph_routed_moe_one_dispatch( - &route_g, - model, - gl, - il, - ffn_out, - ffn_mid, - gate_out * gate_row_bytes, - gate_row_bytes, - up_out * up_row_bytes, - up_row_bytes, - down_out * down_row_bytes, - down_row_bytes, - router_selected, - router_weights, - ffn_norm, - false); - if (ok) ok = ds4_gpu_shared_gate_up_swiglu_q8_0_tensor( - ffn_gate, - ffn_up, - ffn_mid, - model->map, - model->size, - gl->ffn_gate_shexp->abs_offset, - gl->ffn_up_shexp->abs_offset, - DS4_N_EMBD, - DS4_N_FF_EXP, - ffn_norm, - DS4_SWIGLU_CLAMP_EXP); - if (ok) ok = ds4_gpu_matmul_q8_0_tensor(ffn_sum, - model->map, - model->size, - gl->ffn_down_shexp->abs_offset, - DS4_N_FF_EXP, - DS4_N_EMBD, - ffn_mid, - 1); - if (ok) ok = ds4_gpu_add_tensor(attn_out, ffn_out, ffn_sum, DS4_N_EMBD); - if (ok) ok = ds4_gpu_add_tensor(next, after_attn, attn_out, DS4_N_EMBD); - } - - if (ok) { - ds4_gpu_tensor *tmp = cur; - cur = next; - next = tmp; - } - } - if (ok) ok = ds4_gpu_rms_norm_weight_tensor(ffn_norm, cur, - model->map, model->size, - weights->output_norm->abs_offset, - DS4_N_EMBD, DS4_RMS_EPS); - if (ok) ok = ds4_gpu_matmul_q8_0_tensor(logits, - model->map, - model->size, - weights->output->abs_offset, - DS4_N_EMBD, - DS4_N_VOCAB, - ffn_norm, - 1); - if (ok) ok = ds4_gpu_tensor_read(logits, 0, logits_out, logits_bytes) != 0; - - ds4_gpu_tensor_free(router_weights); - ds4_gpu_tensor_free(router_selected); - ds4_gpu_tensor_free(router_probs); - ds4_gpu_tensor_free(router_logits); - ds4_gpu_tensor_free(logits); - ds4_gpu_tensor_free(next); - ds4_gpu_tensor_free(ffn_sum); - ds4_gpu_tensor_free(ffn_out); - ds4_gpu_tensor_free(routed_down); - ds4_gpu_tensor_free(routed_up); - ds4_gpu_tensor_free(routed_gate); - ds4_gpu_tensor_free(ffn_mid); - ds4_gpu_tensor_free(ffn_up); - ds4_gpu_tensor_free(ffn_gate); - ds4_gpu_tensor_free(ffn_norm); - ds4_gpu_tensor_free(after_attn); - ds4_gpu_tensor_free(attn_out); - ds4_gpu_tensor_free(heads); - ds4_gpu_tensor_free(kv_norm); - ds4_gpu_tensor_free(kv_raw); - ds4_gpu_tensor_free(attn_norm); - ds4_gpu_tensor_free(cur); - return ok ? 0 : 1; -} - -static DS4_MAYBE_UNUSED int generate_glm_metal_first_token( - const ds4_model * model, - const ds4_vocab * vocab, - const ds4_weights * weights, - const token_vec * prompt, - int n_predict, - int ctx_size, - ds4_token_emit_fn emit, - ds4_generation_done_fn done, - void * emit_ud) { - fprintf(stderr, "ds4: using GLM Metal first-token generation path\n"); - - if (prompt->len != 1 || prompt->len > ctx_size) { - fprintf(stderr, - "ds4: GLM Metal generation currently supports exactly one prompt token; " - "multi-token prefill needs the GLM KV/DSA graph\n"); - return 1; - } - if (n_predict <= 0) { - if (done) done(emit_ud); - return 0; - } - if (n_predict > 1) { - fprintf(stderr, - "ds4: GLM Metal generation currently emits only the first generated token; " - "stopping after one token\n"); - } - - float *logits = xmalloc((size_t)DS4_N_VOCAB * sizeof(logits[0])); - const double t0 = now_sec(); - const int rc = glm_metal_first_token_logits(model, weights, prompt->v[0], logits); - const double t1 = now_sec(); - if (rc != 0) { - free(logits); - return 1; - } - - if (getenv("DS4_TRACE_TOP") != NULL) { - print_top_logits(stderr, "GLM first-token", vocab, logits, DS4_N_VOCAB, 10); - } - const int token = sample_argmax(logits, DS4_N_VOCAB); - if (!vocab_token_is_generation_stop(vocab, token) && emit) emit(emit_ud, token); - if (done) done(emit_ud); - - const double eval_s = t1 - t0; - ds4_log(stderr, - DS4_LOG_TIMING, - "ds4: GLM first-token eval: %.2f t/s\n", - eval_s > 0.0 ? 1.0 / eval_s : 0.0); - - free(logits); - return 0; -} - static int generate_glm_metal_argmax( const ds4_model * model, const ds4_vocab * vocab, @@ -61170,187 +60497,6 @@ static int glm_metal_compare_i32_list( } #ifndef DS4_NO_GPU -static void glm_metal_q8_diag_fill_input(float *x, uint64_t n_tok, uint64_t in_dim) { - for (uint64_t t = 0; t < n_tok; t++) { - for (uint64_t i = 0; i < in_dim; i++) { - uint32_t s = (uint32_t)(0x9e3779b9u ^ (uint32_t)(t * 0x85ebca6bu) ^ - (uint32_t)(i * 0xc2b2ae35u)); - s ^= s >> 16; - s *= 0x7feb352du; - s ^= s >> 15; - s *= 0x846ca68bu; - s ^= s >> 16; - const float centered = ((float)(int32_t)(s & 0xffffu) - 32768.0f) / 32768.0f; - const float scale = 0.25f + 0.015625f * (float)((i + 3u * t) & 31u); - x[t * in_dim + i] = centered * scale; - } - } -} - -static void glm_metal_q8_diag_reference( - float *out, - const ds4_model *model, - const ds4_tensor *w, - const float *x, - uint64_t n_tok) { - const uint64_t out_dim = w->elements / w->dim[0]; - for (uint64_t t = 0; t < n_tok; t++) { - matvec_q8_0_f32_ref(out + t * out_dim, - model, - w, - x + t * w->dim[0]); - } -} - -static int glm_metal_graph_test_q8_prefill_one( - ds4_engine *e, - const char *name, - const ds4_tensor *w, - int strict) { - if (!w) return 1; - if (w->type != DS4_TENSOR_Q8_0 || w->ndim < 2 || w->dim[0] == 0) { - fprintf(stderr, "ds4: GLM Q8 prefill diagnostic found unexpected %s layout\n", - name); - return 0; - } - - const ds4_model *model = &e->model; - const uint64_t in_dim = w->dim[0]; - const uint64_t out_dim = w->elements / in_dim; - const uint32_t cases[] = { 16u, 17u, 31u, 32u }; - int ok = 1; - - printf(" q8_prefill_diag: %s ndim=%u in=%llu out=%llu strict=%d\n", - name, - w->ndim, - (unsigned long long)in_dim, - (unsigned long long)out_dim, - strict); - - for (uint32_t ci = 0; ci < sizeof(cases) / sizeof(cases[0]); ci++) { - const uint32_t n_tok = cases[ci]; - if (in_dim > UINT64_MAX / n_tok / sizeof(float) || - out_dim > UINT64_MAX / n_tok / sizeof(float) || - out_dim > UINT32_MAX / n_tok) { - fprintf(stderr, "ds4: GLM Q8 prefill diagnostic size overflow in %s token case %u\n", - name, - n_tok); - ok = 0; - if (strict) break; - continue; - } - - const uint64_t x_bytes = (uint64_t)n_tok * in_dim * sizeof(float); - const uint64_t out_bytes = (uint64_t)n_tok * out_dim * sizeof(float); - if (x_bytes > SIZE_MAX || out_bytes > SIZE_MAX) { - fprintf(stderr, "ds4: GLM Q8 prefill diagnostic host allocation is too large in %s token case %u\n", - name, - n_tok); - ok = 0; - if (strict) break; - continue; - } - - char label[128]; - snprintf(label, sizeof(label), "q8_prefill_diag_%s_tok%u", name, n_tok); - - float *x_host = xmalloc((size_t)x_bytes); - float *cpu_out = xmalloc((size_t)out_bytes); - float *gpu_out = xmalloc((size_t)out_bytes); - ds4_gpu_tensor *x_gpu = ds4_gpu_tensor_alloc(x_bytes); - ds4_gpu_tensor *out_gpu = ds4_gpu_tensor_alloc(out_bytes); - int case_ok = x_gpu && out_gpu; - - if (!case_ok) { - fprintf(stderr, "ds4: GLM Q8 prefill diagnostic could not allocate %s token case %u\n", - name, - n_tok); - } - if (case_ok) { - glm_metal_q8_diag_fill_input(x_host, n_tok, in_dim); - glm_metal_q8_diag_reference(cpu_out, model, w, x_host, n_tok); - case_ok = ds4_gpu_tensor_write(x_gpu, 0, x_host, x_bytes) != 0; - } - if (case_ok) { - case_ok = ds4_gpu_matmul_q8_0_tensor(out_gpu, - model->map, - model->size, - w->abs_offset, - in_dim, - out_dim, - x_gpu, - n_tok); - } - if (case_ok) { - case_ok = ds4_gpu_tensor_read(out_gpu, 0, gpu_out, out_bytes) != 0; - } - if (case_ok) { - case_ok = glm_metal_compare_f32(label, - cpu_out, - gpu_out, - (uint32_t)((uint64_t)n_tok * out_dim), - strict ? 5.0e-2f : 3.0e38f); - } - - ds4_gpu_tensor_free(out_gpu); - ds4_gpu_tensor_free(x_gpu); - free(gpu_out); - free(cpu_out); - free(x_host); - - if (!case_ok) { - ok = 0; - if (strict) break; - } - } - - return ok || !strict; -} - -static int glm_metal_graph_test_q8_prefill( - ds4_engine *e, - const ds4_layer_weights *layer) { - return 1; - - if (!layer) { - fprintf(stderr, "ds4: GLM Q8 prefill diagnostic requires layer weights\n"); - return 0; - } - - const int strict = 0; - int ok = 1; - const ds4_layer_weights *sparse = - DS4_N_LEADING_DENSE < DS4_N_LAYER ? &e->weights.layer[DS4_N_LEADING_DENSE] : NULL; - -#define DS4_GLM_Q8_DIAG_ONE(label, tensor) \ - do { \ - if (!glm_metal_graph_test_q8_prefill_one(e, label, tensor, strict)) { \ - ok = 0; \ - if (strict) goto done; \ - } \ - } while (0) - - DS4_GLM_Q8_DIAG_ONE("layer0.attn_q_a", layer->attn_q_a); - DS4_GLM_Q8_DIAG_ONE("layer0.attn_q_b", layer->attn_q_b); - DS4_GLM_Q8_DIAG_ONE("layer0.attn_kv_a_mqa", layer->attn_kv_a_mqa); - DS4_GLM_Q8_DIAG_ONE("layer0.attn_k_b", layer->attn_k_b); - DS4_GLM_Q8_DIAG_ONE("layer0.attn_v_b", layer->attn_v_b); - DS4_GLM_Q8_DIAG_ONE("layer0.attn_output", layer->attn_output); - DS4_GLM_Q8_DIAG_ONE("layer0.ffn_gate", layer->ffn_gate); - DS4_GLM_Q8_DIAG_ONE("layer0.ffn_up", layer->ffn_up); - DS4_GLM_Q8_DIAG_ONE("layer0.ffn_down", layer->ffn_down); - - if (sparse) { - DS4_GLM_Q8_DIAG_ONE("layer3.ffn_gate_shexp", sparse->ffn_gate_shexp); - DS4_GLM_Q8_DIAG_ONE("layer3.ffn_up_shexp", sparse->ffn_up_shexp); - DS4_GLM_Q8_DIAG_ONE("layer3.ffn_down_shexp", sparse->ffn_down_shexp); - } - -done: -#undef DS4_GLM_Q8_DIAG_ONE - return strict ? ok : 1; -} - static int glm_metal_graph_test_multitok_attention( ds4_engine *e, const ds4_tokens *prompt, @@ -62258,8 +61404,6 @@ static int glm_metal_graph_test(ds4_engine *e, const ds4_tokens *prompt) { } if (ok) ok = glm_metal_graph_test_multitok_attention(e, prompt, layer); if (ok) ok = glm_metal_graph_test_decode_attention(e, prompt, layer); - if (ok) ok = glm_metal_graph_test_q8_prefill(e, layer); - if (ok) { memcpy(cpu_cur, cpu_emb, emb_bytes); for (uint32_t il = 0; il < sparse_il; il++) { @@ -72273,7 +71417,6 @@ static bool glm53_graph_encode_native_session_batch( rows, false, false, - false, NULL); stage = "FFN steering"; if (ok) ok = glm_graph_apply_directional_steering_ffn( diff --git a/ds4_cuda.cu b/ds4_cuda.cu index 3d7522aaf..c363875a7 100644 --- a/ds4_cuda.cu +++ b/ds4_cuda.cu @@ -37,7 +37,6 @@ #endif #define CUDA_QK_K 256 -#define DS4_CUDA_UNUSED __attribute__((unused)) /* Environment switches used by opt-in CUDA experiments are value-aware: * spelling a switch as 0/off/no/false must behave like leaving it unset. @@ -10477,49 +10476,6 @@ __device__ __forceinline__ static int32_t dot_i8x32_aligned_int4( return dot; } -__global__ static DS4_CUDA_UNUSED void matmul_q8_0_kernel( - float *out, - const unsigned char *w, - const float *x, - uint64_t in_dim, - uint64_t out_dim, - uint64_t n_tok) { - uint64_t row = (uint64_t)blockIdx.x; - uint64_t tok = (uint64_t)blockIdx.y; - if (row >= out_dim || tok >= n_tok) return; - const uint64_t blocks = (in_dim + 31) / 32; - const unsigned char *wr = w + row * blocks * 34; - const float *xr = x + tok * in_dim; - float acc = 0.0f; - - for (uint64_t b = threadIdx.x; b < blocks; b += blockDim.x) { - uint64_t i0 = b * 32; - uint64_t bn = in_dim - i0 < 32 ? in_dim - i0 : 32; - float amax = 0.0f; - for (uint64_t i = 0; i < bn; i++) amax = fmaxf(amax, fabsf(xr[i0 + i])); - float d = amax / 127.0f; - float id = d != 0.0f ? 1.0f / d : 0.0f; - const __half *scale_h = (const __half *)(wr + b * 34); - const int8_t *qs = (const int8_t *)(wr + b * 34 + 2); - int dot = 0; - for (uint64_t i = 0; i < bn; i++) { - int q = (int)lrintf(xr[i0 + i] * id); - q = q > 127 ? 127 : (q < -128 ? -128 : q); - dot += (int)qs[i] * q; - } - acc += __half2float(*scale_h) * d * (float)dot; - } - - __shared__ float partial[256]; - partial[threadIdx.x] = acc; - __syncthreads(); - for (uint32_t stride = blockDim.x >> 1; stride > 0; stride >>= 1) { - if (threadIdx.x < stride) partial[threadIdx.x] += partial[threadIdx.x + stride]; - __syncthreads(); - } - if (threadIdx.x == 0) out[tok * out_dim + row] = partial[0]; -} - __global__ static void quantize_q8_0_f32_kernel( int8_t *xq, float *xscale, @@ -12751,38 +12707,6 @@ __device__ static float model_scalar_dev(const void *base, uint64_t offset, uint return ((const float *)p)[idx]; } -__device__ static float rope_yarn_ramp_cpu_equiv_dev(float low, float high, int i0) { - float y = ((float)(i0 / 2) - low) / fmaxf(0.001f, high - low); - return 1.0f - fminf(1.0f, fmaxf(0.0f, y)); -} - -__device__ static DS4_CUDA_UNUSED void rope_tail_one_dev(float *x, uint32_t head_dim, uint32_t n_rot, uint32_t pos, uint32_t n_ctx_orig, float freq_base, float freq_scale, float ext_factor, float attn_factor, float beta_fast, float beta_slow) { - uint32_t n_nope = head_dim - n_rot; - float corr0 = 0.0f, corr1 = 0.0f; - if (ext_factor != 0.0f) { - float denom = 2.0f * logf(freq_base); - corr0 = fmaxf(0.0f, floorf((float)n_rot * logf((float)n_ctx_orig / (beta_fast * 2.0f * (float)M_PI)) / denom)); - corr1 = fminf((float)(n_rot - 1), ceilf((float)n_rot * logf((float)n_ctx_orig / (beta_slow * 2.0f * (float)M_PI)) / denom)); - } - for (uint32_t i = 0; i < n_rot; i += 2) { - float theta_extrap = (float)pos * powf(freq_base, -((float)i) / (float)n_rot); - float theta_interp = freq_scale * theta_extrap; - float theta = theta_interp; - float mscale = attn_factor; - if (ext_factor != 0.0f) { - float mix = rope_yarn_ramp_cpu_equiv_dev(corr0, corr1, (int)i) * ext_factor; - theta = theta_interp * (1.0f - mix) + theta_extrap * mix; - mscale *= 1.0f + 0.1f * logf(1.0f / freq_scale); - } - float c = cosf(theta) * mscale; - float s = sinf(theta) * mscale; - float x0 = x[n_nope + i]; - float x1 = x[n_nope + i + 1]; - x[n_nope + i] = x0 * c - x1 * s; - x[n_nope + i + 1] = x0 * s + x1 * c; - } -} - __device__ static void fp8_kv_quantize_row( float *xr, uint32_t head_dim, @@ -16195,18 +16119,6 @@ __device__ __forceinline__ void tt_ldmatrix_x2_trans_addr(uint32_t (&r)[2], unsi #endif } -__device__ __forceinline__ void tt_ldmatrix_x4(uint32_t (&r)[4], const void *p) { - tt_ldmatrix_x4_addr(r, tt_smem_addr(p)); -} - -__device__ __forceinline__ void tt_ldmatrix_x2(uint32_t (&r)[2], const void *p) { - tt_ldmatrix_x2_addr(r, tt_smem_addr(p)); -} - -__device__ __forceinline__ void tt_ldmatrix_x2_trans(uint32_t (&r)[2], const void *p) { - tt_ldmatrix_x2_trans_addr(r, tt_smem_addr(p)); -} - __device__ __forceinline__ void tt_mma_m16n8k16_f16_f32( float *d, const uint32_t (&a)[4], @@ -26078,45 +25990,6 @@ __device__ static void dev_dot_iq2_xxs_q8_K_block4( for (uint32_t p = 0; p < n; p++) acc[p] += 0.125f * xd * ys[p]->d * (float)bsum[p]; } -__device__ static DS4_CUDA_UNUSED void dev_dot_iq2_xxs_q8_K_block8( - const cuda_block_iq2_xxs *x, - const cuda_block_q8_K *y0, - const cuda_block_q8_K *y1, - const cuda_block_q8_K *y2, - const cuda_block_q8_K *y3, - const cuda_block_q8_K *y4, - const cuda_block_q8_K *y5, - const cuda_block_q8_K *y6, - const cuda_block_q8_K *y7, - uint32_t n, - float acc[8]) { - const float xd = dev_f16_to_f32(x->d); - const uint16_t *q2 = x->qs; - int32_t bsum[8] = {0, 0, 0, 0, 0, 0, 0, 0}; - const int8_t *q8[8] = { - y0 ? y0->qs : NULL, y1 ? y1->qs : NULL, y2 ? y2->qs : NULL, y3 ? y3->qs : NULL, - y4 ? y4->qs : NULL, y5 ? y5->qs : NULL, y6 ? y6->qs : NULL, y7 ? y7->qs : NULL, - }; - for (int ib32 = 0; ib32 < CUDA_QK_K / 32; ib32++) { - const uint32_t aux0 = (uint32_t)q2[0] | ((uint32_t)q2[1] << 16); - const uint32_t aux1 = (uint32_t)q2[2] | ((uint32_t)q2[3] << 16); - q2 += 4; - const uint32_t ls = 2u * (aux1 >> 28) + 1u; - const uint8_t a0 = (uint8_t)(aux0 & 0xffu); - const uint8_t a1 = (uint8_t)((aux0 >> 8) & 0xffu); - const uint8_t a2 = (uint8_t)((aux0 >> 16) & 0xffu); - const uint8_t a3 = (uint8_t)((aux0 >> 24) & 0xffu); - for (uint32_t p = 0; p < n; p++) { - int32_t sumi = 0; - sumi += dev_dot_iq2_pair_16(a0, (aux1 >> 0) & 127u, a1, (aux1 >> 7) & 127u, q8[p] + ib32 * 32); - sumi += dev_dot_iq2_pair_16(a2, (aux1 >> 14) & 127u, a3, (aux1 >> 21) & 127u, q8[p] + ib32 * 32 + 16); - bsum[p] += sumi * (int32_t)ls; - } - } - const cuda_block_q8_K *ys[8] = { y0, y1, y2, y3, y4, y5, y6, y7 }; - for (uint32_t p = 0; p < n; p++) acc[p] += 0.125f * xd * ys[p]->d * (float)bsum[p]; -} - __device__ static void dev_q4_K_get_scale_min( uint32_t j, const uint8_t *scales, @@ -26710,162 +26583,6 @@ __global__ static void q8_K_quantize_sidecar_kernel( if (tid == 0u) yb->d = 1.0f / iscale_s; } -__global__ static DS4_CUDA_UNUSED void moe_gate_up_mid_kernel( - float *gate_out, - float *up_out, - float *mid_out, - const char *gate_base, - const char *up_base, - const cuda_block_q8_K *xq, - const int32_t *selected, - const float *weights, - uint64_t gate_expert_bytes, - uint64_t gate_row_bytes, - uint32_t xq_blocks, - uint32_t expert_mid_dim, - uint32_t n_expert, - float clamp) { - uint32_t row = blockIdx.x; - uint32_t pair = blockIdx.y; - if (row >= expert_mid_dim) return; - uint32_t tok = pair / n_expert; - uint32_t slot = pair - tok * n_expert; - int32_t expert_i = selected[(uint64_t)tok * n_expert + slot]; - if (expert_i < 0) expert_i = 0; - uint32_t expert = (uint32_t)expert_i; - const cuda_block_iq2_xxs *gr = (const cuda_block_iq2_xxs *)(gate_base + (uint64_t)expert * gate_expert_bytes + (uint64_t)row * gate_row_bytes); - const cuda_block_iq2_xxs *ur = (const cuda_block_iq2_xxs *)(up_base + (uint64_t)expert * gate_expert_bytes + (uint64_t)row * gate_row_bytes); - const cuda_block_q8_K *xqb = xq + (uint64_t)tok * xq_blocks; - float gate = 0.0f; - float up = 0.0f; - for (uint32_t b = threadIdx.x; b < xq_blocks; b += blockDim.x) { - gate += dev_dot_iq2_xxs_q8_K_block(gr + b, xqb + b); - up += dev_dot_iq2_xxs_q8_K_block(ur + b, xqb + b); - } - __shared__ float partial_gate[256]; - __shared__ float partial_up[256]; - partial_gate[threadIdx.x] = gate; - partial_up[threadIdx.x] = up; - __syncthreads(); - for (uint32_t stride = blockDim.x >> 1; stride > 0; stride >>= 1) { - if (threadIdx.x < stride) { - partial_gate[threadIdx.x] += partial_gate[threadIdx.x + stride]; - partial_up[threadIdx.x] += partial_up[threadIdx.x + stride]; - } - __syncthreads(); - } - if (threadIdx.x == 0) { - gate = partial_gate[0]; - up = partial_up[0]; - if (clamp > 1.0e-6f) { - if (gate > clamp) gate = clamp; - if (up > clamp) up = clamp; - if (up < -clamp) up = -clamp; - } - const uint64_t off = (uint64_t)pair * expert_mid_dim + row; - gate_out[off] = gate; - up_out[off] = up; - mid_out[off] = (gate / (1.0f + expf(-gate))) * up * weights[(uint64_t)tok * n_expert + slot]; - } -} - -__global__ static DS4_CUDA_UNUSED void moe_gate_up_mid_warp8_kernel( - float *gate_out, - float *up_out, - float *mid_out, - const char *gate_base, - const char *up_base, - const cuda_block_q8_K *xq, - const int32_t *selected, - const float *weights, - uint64_t gate_expert_bytes, - uint64_t gate_row_bytes, - uint32_t xq_blocks, - uint32_t expert_mid_dim, - uint32_t n_expert, - float clamp) { - uint32_t lane = threadIdx.x & 31u; - uint32_t warp = threadIdx.x >> 5u; - uint32_t row = blockIdx.x * 8u + warp; - uint32_t pair = blockIdx.y; - if (row >= expert_mid_dim) return; - uint32_t tok = pair / n_expert; - uint32_t slot = pair - tok * n_expert; - int32_t expert_i = selected[(uint64_t)tok * n_expert + slot]; - if (expert_i < 0) expert_i = 0; - uint32_t expert = (uint32_t)expert_i; - const cuda_block_iq2_xxs *gr = (const cuda_block_iq2_xxs *)(gate_base + (uint64_t)expert * gate_expert_bytes + (uint64_t)row * gate_row_bytes); - const cuda_block_iq2_xxs *ur = (const cuda_block_iq2_xxs *)(up_base + (uint64_t)expert * gate_expert_bytes + (uint64_t)row * gate_row_bytes); - const cuda_block_q8_K *xqb = xq + (uint64_t)tok * xq_blocks; - float gate = 0.0f; - float up = 0.0f; - for (uint32_t b = lane; b < xq_blocks; b += 32u) { - gate += dev_dot_iq2_xxs_q8_K_block(gr + b, xqb + b); - up += dev_dot_iq2_xxs_q8_K_block(ur + b, xqb + b); - } - gate = warp_sum_f32(gate); - up = warp_sum_f32(up); - if (lane == 0) { - if (clamp > 1.0e-6f) { - if (gate > clamp) gate = clamp; - if (up > clamp) up = clamp; - if (up < -clamp) up = -clamp; - } - const uint64_t off = (uint64_t)pair * expert_mid_dim + row; - gate_out[off] = gate; - up_out[off] = up; - mid_out[off] = (gate / (1.0f + expf(-gate))) * up * weights[(uint64_t)tok * n_expert + slot]; - } -} - -__global__ static DS4_CUDA_UNUSED void moe_gate_up_mid_hwarp16_kernel( - float *gate_out, - float *up_out, - float *mid_out, - const char *gate_base, - const char *up_base, - const cuda_block_q8_K *xq, - const int32_t *selected, - const float *weights, - uint64_t gate_expert_bytes, - uint64_t gate_row_bytes, - uint32_t xq_blocks, - uint32_t expert_mid_dim, - uint32_t n_expert, - float clamp) { - uint32_t lane = threadIdx.x & 15u; - uint32_t row = blockIdx.x * 16u + (threadIdx.x >> 4u); - uint32_t pair = blockIdx.y; - if (row >= expert_mid_dim) return; - uint32_t tok = pair / n_expert; - uint32_t slot = pair - tok * n_expert; - int32_t expert_i = selected[(uint64_t)tok * n_expert + slot]; - if (expert_i < 0) expert_i = 0; - uint32_t expert = (uint32_t)expert_i; - const cuda_block_iq2_xxs *gr = (const cuda_block_iq2_xxs *)(gate_base + (uint64_t)expert * gate_expert_bytes + (uint64_t)row * gate_row_bytes); - const cuda_block_iq2_xxs *ur = (const cuda_block_iq2_xxs *)(up_base + (uint64_t)expert * gate_expert_bytes + (uint64_t)row * gate_row_bytes); - const cuda_block_q8_K *xqb = xq + (uint64_t)tok * xq_blocks; - float gate = 0.0f; - float up = 0.0f; - for (uint32_t b = lane; b < xq_blocks; b += 16u) { - gate += dev_dot_iq2_xxs_q8_K_block(gr + b, xqb + b); - up += dev_dot_iq2_xxs_q8_K_block(ur + b, xqb + b); - } - gate = half_warp_sum_f32(gate, lane); - up = half_warp_sum_f32(up, lane); - if (lane == 0) { - if (clamp > 1.0e-6f) { - if (gate > clamp) gate = clamp; - if (up > clamp) up = clamp; - if (up < -clamp) up = -clamp; - } - const uint64_t off = (uint64_t)pair * expert_mid_dim + row; - gate_out[off] = gate; - up_out[off] = up; - mid_out[off] = (gate / (1.0f + expf(-gate))) * up * weights[(uint64_t)tok * n_expert + slot]; - } -} - // perf-04: launch-geometry tuning for the routed-MoE gate/up decode kernels // (moe_gate_up_mid_qwarp32 / _decode_lut_qwarp32 / _decode_q4K_qwarp32). Each // block processes MOE_DECODE_ROW_TILES tiles of 32 rows (row_lane in [0,32)). @@ -27283,68 +27000,6 @@ __global__ static void moe_gate_up_mid_sorted_qwarp32_kernel( } } -__global__ static DS4_CUDA_UNUSED void moe_gate_up_mid_expert_tile8_kernel( - float *gate_out, - float *up_out, - float *mid_out, - const char *gate_base, - const char *up_base, - const cuda_block_q8_K *xq, - const uint32_t *sorted_pairs, - const uint32_t *offsets, - const uint32_t *counts, - const uint32_t *tile_total, - const uint32_t *tile_experts, - const uint32_t *tile_starts, - const float *weights, - uint64_t gate_expert_bytes, - uint64_t gate_row_bytes, - uint32_t xq_blocks, - uint32_t expert_mid_dim, - uint32_t n_expert, - float clamp) { - uint32_t tile = blockIdx.y; - if (tile >= *tile_total) return; - uint32_t group = threadIdx.x >> 3u; - uint32_t lane = threadIdx.x & 7u; - uint32_t pair_slot = group & 7u; - uint32_t row_lane = group >> 3u; - uint32_t expert = tile_experts[tile]; - uint32_t local_pair = tile_starts[tile] + pair_slot; - if (local_pair >= counts[expert]) return; - uint32_t sorted_idx = offsets[expert] + local_pair; - uint32_t pair = sorted_pairs[sorted_idx]; - uint32_t tok = pair / n_expert; - uint32_t slot = pair - tok * n_expert; - const cuda_block_q8_K *xqb = xq + (uint64_t)tok * xq_blocks; - - for (uint32_t rr = 0; rr < 2u; rr++) { - uint32_t row = blockIdx.x * 8u + row_lane + rr * 4u; - if (row >= expert_mid_dim) continue; - const cuda_block_iq2_xxs *gr = (const cuda_block_iq2_xxs *)(gate_base + (uint64_t)expert * gate_expert_bytes + (uint64_t)row * gate_row_bytes); - const cuda_block_iq2_xxs *ur = (const cuda_block_iq2_xxs *)(up_base + (uint64_t)expert * gate_expert_bytes + (uint64_t)row * gate_row_bytes); - float gate = 0.0f; - float up = 0.0f; - for (uint32_t b = lane; b < xq_blocks; b += 8u) { - gate += dev_dot_iq2_xxs_q8_K_block(gr + b, xqb + b); - up += dev_dot_iq2_xxs_q8_K_block(ur + b, xqb + b); - } - gate = quarter_warp_sum_f32(gate, lane); - up = quarter_warp_sum_f32(up, lane); - if (lane == 0) { - if (clamp > 1.0e-6f) { - if (gate > clamp) gate = clamp; - if (up > clamp) up = clamp; - if (up < -clamp) up = -clamp; - } - const uint64_t off = (uint64_t)pair * expert_mid_dim + row; - gate_out[off] = gate; - up_out[off] = up; - mid_out[off] = (gate / (1.0f + expf(-gate))) * up * weights[(uint64_t)tok * n_expert + slot]; - } - } -} - __global__ static void moe_gate_up_mid_expert_tile4_row32_kernel( float *gate_out, float *up_out, @@ -27754,90 +27409,6 @@ __global__ static void moe_gate_up_mid_sorted_p2_qwarp32_kernel( } } -__global__ static DS4_CUDA_UNUSED void moe_down_kernel( - float *down_out, - const char *down_base, - const cuda_block_q8_K *midq, - const int32_t *selected, - uint64_t down_expert_bytes, - uint64_t down_row_bytes, - uint32_t midq_blocks, - uint32_t out_dim, - uint32_t n_expert) { - uint32_t row = blockIdx.x; - uint32_t pair = blockIdx.y; - if (row >= out_dim) return; - uint32_t tok = pair / n_expert; - uint32_t slot = pair - tok * n_expert; - int32_t expert_i = selected[(uint64_t)tok * n_expert + slot]; - if (expert_i < 0) expert_i = 0; - const cuda_block_q2_K *wr = (const cuda_block_q2_K *)(down_base + (uint64_t)(uint32_t)expert_i * down_expert_bytes + (uint64_t)row * down_row_bytes); - const cuda_block_q8_K *xq = midq + (uint64_t)pair * midq_blocks; - float acc = 0.0f; - for (uint32_t b = threadIdx.x; b < midq_blocks; b += blockDim.x) acc += dev_dot_q2_K_q8_K_block(wr + b, xq + b); - __shared__ float partial[256]; - partial[threadIdx.x] = acc; - __syncthreads(); - for (uint32_t stride = blockDim.x >> 1; stride > 0; stride >>= 1) { - if (threadIdx.x < stride) partial[threadIdx.x] += partial[threadIdx.x + stride]; - __syncthreads(); - } - if (threadIdx.x == 0) down_out[(uint64_t)pair * out_dim + row] = partial[0]; -} - -__global__ static DS4_CUDA_UNUSED void moe_down_warp8_kernel( - float *down_out, - const char *down_base, - const cuda_block_q8_K *midq, - const int32_t *selected, - uint64_t down_expert_bytes, - uint64_t down_row_bytes, - uint32_t midq_blocks, - uint32_t out_dim, - uint32_t n_expert) { - uint32_t lane = threadIdx.x & 31u; - uint32_t warp = threadIdx.x >> 5u; - uint32_t row = blockIdx.x * 8u + warp; - uint32_t pair = blockIdx.y; - if (row >= out_dim) return; - uint32_t tok = pair / n_expert; - uint32_t slot = pair - tok * n_expert; - int32_t expert_i = selected[(uint64_t)tok * n_expert + slot]; - if (expert_i < 0) expert_i = 0; - const cuda_block_q2_K *wr = (const cuda_block_q2_K *)(down_base + (uint64_t)(uint32_t)expert_i * down_expert_bytes + (uint64_t)row * down_row_bytes); - const cuda_block_q8_K *xq = midq + (uint64_t)pair * midq_blocks; - float acc = 0.0f; - for (uint32_t b = lane; b < midq_blocks; b += 32u) acc += dev_dot_q2_K_q8_K_block(wr + b, xq + b); - acc = warp_sum_f32(acc); - if (lane == 0) down_out[(uint64_t)pair * out_dim + row] = acc; -} - -__global__ static DS4_CUDA_UNUSED void moe_down_hwarp16_kernel( - float *down_out, - const char *down_base, - const cuda_block_q8_K *midq, - const int32_t *selected, - uint64_t down_expert_bytes, - uint64_t down_row_bytes, - uint32_t midq_blocks, - uint32_t out_dim, - uint32_t n_expert) { - uint32_t lane = threadIdx.x & 15u; - uint32_t row = blockIdx.x * 16u + (threadIdx.x >> 4u); - uint32_t pair = blockIdx.y; - if (row >= out_dim) return; - uint32_t tok = pair / n_expert; - uint32_t slot = pair - tok * n_expert; - int32_t expert_i = selected[(uint64_t)tok * n_expert + slot]; - if (expert_i < 0) expert_i = 0; - const cuda_block_q2_K *wr = (const cuda_block_q2_K *)(down_base + (uint64_t)(uint32_t)expert_i * down_expert_bytes + (uint64_t)row * down_row_bytes); - const cuda_block_q8_K *xq = midq + (uint64_t)pair * midq_blocks; - float acc = 0.0f; - for (uint32_t b = lane; b < midq_blocks; b += 16u) acc += dev_dot_q2_K_q8_K_block(wr + b, xq + b); - acc = half_warp_sum_f32(acc, lane); - if (lane == 0) down_out[(uint64_t)pair * out_dim + row] = acc; -} - __global__ static void moe_down_qwarp32_kernel( float *down_out, const char *down_base, @@ -30050,45 +29621,6 @@ __global__ static void moe_down_sorted_qwarp32_kernel( if (lane == 0) down_out[(uint64_t)pair * out_dim + row] = acc; } -__global__ static DS4_CUDA_UNUSED void moe_down_expert_tile8_kernel( - float *down_out, - const char *down_base, - const cuda_block_q8_K *midq, - const uint32_t *sorted_pairs, - const uint32_t *offsets, - const uint32_t *counts, - const uint32_t *tile_total, - const uint32_t *tile_experts, - const uint32_t *tile_starts, - uint64_t down_expert_bytes, - uint64_t down_row_bytes, - uint32_t midq_blocks, - uint32_t out_dim, - uint32_t n_expert) { - uint32_t tile = blockIdx.y; - if (tile >= *tile_total) return; - uint32_t group = threadIdx.x >> 3u; - uint32_t lane = threadIdx.x & 7u; - uint32_t pair_slot = group & 7u; - uint32_t row_lane = group >> 3u; - uint32_t expert = tile_experts[tile]; - uint32_t local_pair = tile_starts[tile] + pair_slot; - if (local_pair >= counts[expert]) return; - uint32_t sorted_idx = offsets[expert] + local_pair; - uint32_t pair = sorted_pairs[sorted_idx]; - const cuda_block_q8_K *xq = midq + (uint64_t)pair * midq_blocks; - - for (uint32_t rr = 0; rr < 2u; rr++) { - uint32_t row = blockIdx.x * 8u + row_lane + rr * 4u; - if (row >= out_dim) continue; - const cuda_block_q2_K *wr = (const cuda_block_q2_K *)(down_base + (uint64_t)expert * down_expert_bytes + (uint64_t)row * down_row_bytes); - float acc = 0.0f; - for (uint32_t b = lane; b < midq_blocks; b += 8u) acc += dev_dot_q2_K_q8_K_block(wr + b, xq + b); - acc = quarter_warp_sum_f32(acc, lane); - if (lane == 0) down_out[(uint64_t)pair * out_dim + row] = acc; - } -} - __global__ static void moe_down_expert_tile4_row32_kernel( float *down_out, const char *down_base, diff --git a/ds4_distributed.c b/ds4_distributed.c index 3d797b5dd..388748611 100644 --- a/ds4_distributed.c +++ b/ds4_distributed.c @@ -65,7 +65,6 @@ #define DS4_DIST_ACTIVATION_BITS_DEFAULT 32u #define DS4_DIST_ROUTE_F_OUTPUT_LOGITS 0x00000001u #define DS4_DIST_ROUTE_RETURN_UPSTREAM 1u -#define DS4_DIST_RECV_TRANSPORT_ERROR 1 #define DS4_DIST_RECV_REMOTE_ERROR 2 #define DS4_DIST_SNAPSHOT_CHUNK_BYTES (8u * 1024u * 1024u) @@ -8140,30 +8139,6 @@ void ds4_dist_options_free(ds4_dist_options *opt) { free(opt); } -void ds4_dist_usage(FILE *fp) { - fprintf(fp, - " --role ROLE\n" - " Distributed role: coordinator or worker.\n" - " --layers A:B\n" - " Inclusive distributed layer slice, e.g. 10:20 or 21:output.\n" - " --listen HOST PORT\n" - " Coordinator TCP listen address. Workers may later use it to force their data listener.\n" - " --coordinator HOST PORT\n" - " Coordinator TCP address for --role worker.\n" - " --dist-prefill-chunk N\n" - " Coordinator prefill pipeline chunk size. Default: session cap, normally 4096.\n" - " Non-default values are experimental and can change logits unless validated.\n" - " --dist-prefill-window N\n" - " Coordinator max end-to-end prefill chunks in flight. Default: workers+2, capped at 8.\n" - " --dist-activation-bits N\n" - " Coordinator hidden-state transport width: 32, 16, or 8. Default: 32.\n" - " --dist-replay-check\n" - " Coordinator diagnostic: reset and replay the prompt, then compare logits.\n" - " --debug\n" - " Print coordinator route/debug logs. Workers keep their normal logs without this.\n" - ); -} - ds4_dist_cli_parse_result ds4_dist_parse_cli_arg( const char *arg, int *index, diff --git a/ds4_distributed.h b/ds4_distributed.h index 35c5343bd..6493799f0 100644 --- a/ds4_distributed.h +++ b/ds4_distributed.h @@ -43,7 +43,6 @@ typedef enum { bool ds4_dist_enabled(const ds4_dist_options *opt); ds4_dist_options *ds4_dist_options_create(void); void ds4_dist_options_free(ds4_dist_options *opt); -void ds4_dist_usage(FILE *fp); ds4_dist_cli_parse_result ds4_dist_parse_cli_arg( const char *arg, int *index, diff --git a/ds4_kvstore.c b/ds4_kvstore.c index 5b73de7b6..40b0e2b5a 100644 --- a/ds4_kvstore.c +++ b/ds4_kvstore.c @@ -1168,25 +1168,6 @@ bool ds4_kvstore_store_live_prefix(ds4_kvstore *kc, hooks, err, err_len); } -bool ds4_kvstore_maybe_store_continued(ds4_kvstore *kc, - ds4_engine *engine, - ds4_session *session, - const ds4_kvstore_trailer_hooks *hooks, - char *err, - size_t err_len) { - const ds4_tokens *tokens = ds4_session_tokens(session); - if (!tokens) return false; - const int target = ds4_kvstore_continued_store_target(kc, tokens->len); - if (target == 0) return false; - if (ds4_kvstore_store_live_prefix(kc, engine, session, tokens, target, - "continued", hooks, err, err_len)) - { - ds4_kvstore_note_store(kc, target); - return true; - } - return false; -} - int ds4_kvstore_find_text_prefix(ds4_kvstore *kc, const char *prompt_text, int model_id, int quant_bits, int ctx_size) { if (!prompt_text) return -1; diff --git a/ds4_kvstore.h b/ds4_kvstore.h index 28ccdb7ea..47e826cc5 100644 --- a/ds4_kvstore.h +++ b/ds4_kvstore.h @@ -17,7 +17,7 @@ #define DS4_KVSTORE_EXT_THINKING_VISIBLE (1u << 2) #define DS4_KVSTORE_EXT_SESSION_TITLE (1u << 3) -typedef enum { +enum { DS4_KVSTORE_REASON_UNKNOWN = 0, DS4_KVSTORE_REASON_COLD = 1, DS4_KVSTORE_REASON_CONTINUED = 2, @@ -25,7 +25,7 @@ typedef enum { DS4_KVSTORE_REASON_SHUTDOWN = 4, DS4_KVSTORE_REASON_AGENT_SYSTEM = 5, DS4_KVSTORE_REASON_AGENT_SESSION = 6, -} ds4_kvstore_reason; +}; typedef enum { DS4_KVSTORE_LOG_DEFAULT, @@ -180,12 +180,6 @@ bool ds4_kvstore_store_live_prefix(ds4_kvstore *kc, const ds4_kvstore_trailer_hooks *hooks, char *err, size_t err_len); -bool ds4_kvstore_maybe_store_continued(ds4_kvstore *kc, - ds4_engine *engine, - ds4_session *session, - const ds4_kvstore_trailer_hooks *hooks, - char *err, - size_t err_len); int ds4_kvstore_try_load_text(ds4_kvstore *kc, ds4_engine *engine, ds4_session *session, diff --git a/ds4_metal.m b/ds4_metal.m index 92632e862..005b1c743 100644 --- a/ds4_metal.m +++ b/ds4_metal.m @@ -101,7 +101,6 @@ static id g_cpy_contig_f16_f32_pipeline; static id g_cpy_contig_f16_f16_pipeline; static id g_flash_kv_stage_f16_pipeline; -static id g_swiglu_pipeline; static id g_swiglu_flat_pipeline; static id g_add_pipeline; static id g_add2_pipeline; @@ -252,13 +251,11 @@ static id g_glm_indexer_score_one_direct_pipeline; static id g_glm_indexer_scores_batch_pipeline; static id g_glm_indexer_scores_tiled_pipeline; -static id g_glm_indexer_scores_tiled_f32_pipeline; static id g_glm_qk_lowrank_pipeline; static id g_glm_qk_lowrank_glm52_pipeline; static id g_glm_qk_lowrank_glm52_sg_pipeline; static id g_glm_qk_lowrank_batch_pipeline; static id g_glm_qk_lowrank_batch_glm52_t4_pipeline; -static id g_glm_value_project_q8_0_pipeline; static id g_glm_value_project_q8_0_batch_heads_pipeline; static id g_glm_value_project_q8_0_batch_heads_mma_pipeline; static id g_glm_attention_indexed_decode_pipeline; @@ -275,11 +272,8 @@ static id g_glm_attention_indexed_batch_lora_group8_vec_valid_fullheads_pipeline; static id g_glm_attention_indexed_batch_lora_group8_vec_causal_pipeline; static id g_glm_attention_indexed_batch_lora_group8_vec_causal_fullheads_pipeline; -static id g_glm_q4_k_pair_swiglu_f32_pipeline; static id g_glm_q4_k_pair_swiglu2_f32_pipeline; static id g_glm_q4_k_pair_swiglu4_f32_pipeline; -static id g_glm_q4_k_pair_swiglu2_mapped_f32_pipeline; -static id g_glm_q4_k_pair_swiglu2_mapped_row_f32_pipeline; static id g_glm_q2_k_pair_swiglu_f32_pipeline; static id g_glm_q2_k_addr_pair_swiglu2_f32_pipeline; static id g_glm_q2_k_addr_pair_swiglu2_masked_f32_pipeline; @@ -290,8 +284,6 @@ static id g_glm_q2_k_addr_down_f32_pipeline; static id g_glm_q4_k_addr_down_f32_pipeline; static id g_glm_q5_k_pair_swiglu_f32_pipeline; -static id g_glm_q5_k_pair_swiglu_mapped_f32_pipeline; -static id g_glm_q5_k_pair_swiglu_mapped_row_f32_pipeline; static id g_glm_q5_k_down_f32_pipeline; static id g_glm_q6_k_down_f32_pipeline; static id g_dsv4_router_weights_batch_pipeline; @@ -376,7 +368,6 @@ __strong id flash_attn_pad_buffer; __strong id flash_attn_tmp_buffer; __strong id flash_attn_blk_buffer; - __strong id flash_attn_ring_buffer; __strong id flash_attn_kv_buffer; __strong id glm_flash_attn_mask_buffer; __strong id compressor_pool_kv_buffer; @@ -410,7 +401,6 @@ NSUInteger flash_attn_pad_bytes; NSUInteger flash_attn_tmp_bytes; NSUInteger flash_attn_blk_bytes; - NSUInteger flash_attn_ring_bytes; NSUInteger flash_attn_kv_bytes; NSUInteger glm_flash_attn_mask_bytes; NSUInteger compressor_pool_kv_bytes; @@ -447,7 +437,6 @@ #define g_flash_attn_pad_buffer DS4_STREAM_SCRATCH(flash_attn_pad_buffer) #define g_flash_attn_tmp_buffer DS4_STREAM_SCRATCH(flash_attn_tmp_buffer) #define g_flash_attn_blk_buffer DS4_STREAM_SCRATCH(flash_attn_blk_buffer) -#define g_flash_attn_ring_buffer DS4_STREAM_SCRATCH(flash_attn_ring_buffer) #define g_flash_attn_kv_buffer DS4_STREAM_SCRATCH(flash_attn_kv_buffer) #define g_glm_flash_attn_mask_buffer DS4_STREAM_SCRATCH(glm_flash_attn_mask_buffer) #define g_compressor_pool_kv_buffer DS4_STREAM_SCRATCH(compressor_pool_kv_buffer) @@ -478,7 +467,6 @@ #define g_flash_attn_pad_bytes DS4_STREAM_SCRATCH(flash_attn_pad_bytes) #define g_flash_attn_tmp_bytes DS4_STREAM_SCRATCH(flash_attn_tmp_bytes) #define g_flash_attn_blk_bytes DS4_STREAM_SCRATCH(flash_attn_blk_bytes) -#define g_flash_attn_ring_bytes DS4_STREAM_SCRATCH(flash_attn_ring_bytes) #define g_flash_attn_kv_bytes DS4_STREAM_SCRATCH(flash_attn_kv_bytes) #define g_glm_flash_attn_mask_bytes DS4_STREAM_SCRATCH(glm_flash_attn_mask_bytes) #define g_compressor_pool_kv_bytes DS4_STREAM_SCRATCH(compressor_pool_kv_bytes) @@ -4782,7 +4770,6 @@ void ds4_gpu_print_memory_report(const char *label) { uint64_t flash_pad = 0; uint64_t flash_tmp = 0; uint64_t flash_blk = cached_prefill_blk_bytes; - uint64_t flash_ring = 0; uint64_t flash_kv = 0; uint64_t compressor = 0; uint64_t router = 0; @@ -4800,7 +4787,6 @@ void ds4_gpu_print_memory_report(const char *label) { flash_pad += (uint64_t)scratch_state->flash_attn_pad_bytes; flash_tmp += (uint64_t)scratch_state->flash_attn_tmp_bytes; flash_blk += (uint64_t)scratch_state->flash_attn_blk_bytes; - flash_ring += (uint64_t)scratch_state->flash_attn_ring_bytes; flash_kv += (uint64_t)scratch_state->flash_attn_kv_bytes; compressor += (uint64_t)scratch_state->compressor_pool_kv_bytes + (uint64_t)scratch_state->compressor_pool_score_bytes + @@ -4828,7 +4814,7 @@ void ds4_gpu_print_memory_report(const char *label) { raw_store += (uint64_t)scratch_state->raw_store_round_bytes; } const uint64_t scratch = flash_mask + flash_pad + flash_tmp + flash_blk + - flash_ring + flash_kv + compressor + router + + flash_kv + compressor + router + indexer + moe + q4_pair_rhs + f16_round + raw_store; @@ -5126,13 +5112,12 @@ void ds4_gpu_print_memory_report(const char *label) { (g_metal4_tensor_api_compile_supported ? "available" : "disabled"), g_metal4_m5_neural_accelerators_hint ? "likely" : "not detected"); fprintf(stderr, - "ds4: scratch %.2f MiB (flash mask %.2f, pad %.2f, tmp %.2f, blk %.2f, ring %.2f, kv %.2f, compressor %.2f, router %.2f, indexer %.2f, moe %.2f, q4-pair-rhs %.2f, f16 %.2f, raw-store %.2f)\n", + "ds4: scratch %.2f MiB (flash mask %.2f, pad %.2f, tmp %.2f, blk %.2f, kv %.2f, compressor %.2f, router %.2f, indexer %.2f, moe %.2f, q4-pair-rhs %.2f, f16 %.2f, raw-store %.2f)\n", ds4_gpu_mib(scratch), ds4_gpu_mib(flash_mask), ds4_gpu_mib(flash_pad), ds4_gpu_mib(flash_tmp), ds4_gpu_mib(flash_blk), - ds4_gpu_mib(flash_ring), ds4_gpu_mib(flash_kv), ds4_gpu_mib(compressor), ds4_gpu_mib(router), @@ -5264,7 +5249,6 @@ static int ds4_gpu_model_map_log_enabled(void) { "#endif\n" "#define N_SIMDWIDTH 32\n" "#define N_R0_Q8_0 2\n" -"#define N_SG_Q8_0 4\n" "#define FC_MUL_MV 600\n" "#define FC_MUL_MM 700\n" "#define FC_BIN 1300\n" @@ -7897,23 +7881,6 @@ int ds4_gpu_init(void) { return 0; } - fn = [library newFunctionWithName:@"kernel_swiglu_f32"]; - if (!fn) { - fprintf(stderr, "ds4: Metal kernel_swiglu_f32 function not found\n"); - g_queue = nil; - g_device = nil; - return 0; - } - - g_swiglu_pipeline = [g_device newComputePipelineStateWithFunction:fn error:&error]; - if (!g_swiglu_pipeline) { - fprintf(stderr, "ds4: Metal kernel_swiglu_f32 pipeline failed: %s\n", - [[error localizedDescription] UTF8String]); - g_queue = nil; - g_device = nil; - return 0; - } - fn = [library newFunctionWithName:@"kernel_swiglu_flat_f32"]; if (!fn) { fprintf(stderr, "ds4: Metal kernel_swiglu_flat_f32 function not found\n"); @@ -9603,8 +9570,6 @@ int ds4_gpu_init(void) { ds4_gpu_get_pipeline("kernel_glm_indexer_scores_batch"); g_glm_indexer_scores_tiled_pipeline = ds4_gpu_get_pipeline("kernel_glm_indexer_scores_tiled"); - g_glm_indexer_scores_tiled_f32_pipeline = - ds4_gpu_get_pipeline("kernel_glm_indexer_scores_tiled_f32"); g_glm_qk_lowrank_pipeline = ds4_gpu_get_pipeline("kernel_glm_qk_lowrank_q8_0"); g_glm_qk_lowrank_glm52_pipeline = @@ -9615,8 +9580,6 @@ int ds4_gpu_init(void) { ds4_gpu_get_pipeline("kernel_glm_qk_lowrank_q8_0_batch"); g_glm_qk_lowrank_batch_glm52_t4_pipeline = ds4_gpu_get_pipeline("kernel_glm_qk_lowrank_q8_0_batch_glm52_t4"); - g_glm_value_project_q8_0_pipeline = - ds4_gpu_get_pipeline("kernel_glm_value_project_q8_0"); g_glm_value_project_q8_0_batch_heads_pipeline = ds4_gpu_get_pipeline("kernel_glm_value_project_q8_0_batch_heads"); g_glm_value_project_q8_0_batch_heads_mma_pipeline = @@ -9649,16 +9612,10 @@ int ds4_gpu_init(void) { ds4_gpu_get_pipeline("kernel_glm_attention_indexed_batch_lora_group8_vec_causal"); g_glm_attention_indexed_batch_lora_group8_vec_causal_fullheads_pipeline = ds4_gpu_get_pipeline("kernel_glm_attention_indexed_batch_lora_group8_vec_causal_fullheads"); - g_glm_q4_k_pair_swiglu_f32_pipeline = - ds4_gpu_get_pipeline("kernel_glm_q4_K_pair_swiglu_f32"); g_glm_q4_k_pair_swiglu2_f32_pipeline = ds4_gpu_get_pipeline("kernel_glm_q4_K_pair_swiglu2_f32"); g_glm_q4_k_pair_swiglu4_f32_pipeline = ds4_gpu_get_pipeline("kernel_glm_q4_K_pair_swiglu4_f32"); - g_glm_q4_k_pair_swiglu2_mapped_f32_pipeline = - ds4_gpu_get_pipeline("kernel_glm_q4_K_pair_swiglu2_mapped_f32"); - g_glm_q4_k_pair_swiglu2_mapped_row_f32_pipeline = - ds4_gpu_get_pipeline("kernel_glm_q4_K_pair_swiglu2_mapped_row_f32"); g_glm_q2_k_pair_swiglu_f32_pipeline = ds4_gpu_get_pipeline("kernel_glm_q2_K_pair_swiglu_f32"); g_glm_q2_k_addr_pair_swiglu2_f32_pipeline = @@ -9679,10 +9636,6 @@ int ds4_gpu_init(void) { ds4_gpu_get_pipeline("kernel_glm_q4_K_addr_down_simd_f32"); g_glm_q5_k_pair_swiglu_f32_pipeline = ds4_gpu_get_pipeline("kernel_glm_q5_K_pair_swiglu_f32"); - g_glm_q5_k_pair_swiglu_mapped_f32_pipeline = - ds4_gpu_get_pipeline("kernel_glm_q5_K_pair_swiglu_mapped_f32"); - g_glm_q5_k_pair_swiglu_mapped_row_f32_pipeline = - ds4_gpu_get_pipeline("kernel_glm_q5_K_pair_swiglu_mapped_row_f32"); g_glm_q5_k_down_f32_pipeline = ds4_gpu_get_pipeline("kernel_glm_q5_K_down_f32"); g_glm_q6_k_down_f32_pipeline = @@ -9727,13 +9680,11 @@ int ds4_gpu_init(void) { !g_glm_indexer_score_one_direct_pipeline || !g_glm_indexer_scores_batch_pipeline || !g_glm_indexer_scores_tiled_pipeline || - !g_glm_indexer_scores_tiled_f32_pipeline || !g_glm_qk_lowrank_pipeline || !g_glm_qk_lowrank_glm52_pipeline || !g_glm_qk_lowrank_glm52_sg_pipeline || !g_glm_qk_lowrank_batch_pipeline || !g_glm_qk_lowrank_batch_glm52_t4_pipeline || - !g_glm_value_project_q8_0_pipeline || !g_glm_value_project_q8_0_batch_heads_pipeline || !g_glm_value_project_q8_0_batch_heads_mma_pipeline || !g_glm_attention_indexed_decode_pipeline || @@ -9750,11 +9701,8 @@ int ds4_gpu_init(void) { !g_glm_attention_indexed_batch_lora_group8_vec_valid_fullheads_pipeline || !g_glm_attention_indexed_batch_lora_group8_vec_causal_pipeline || !g_glm_attention_indexed_batch_lora_group8_vec_causal_fullheads_pipeline || - !g_glm_q4_k_pair_swiglu_f32_pipeline || !g_glm_q4_k_pair_swiglu2_f32_pipeline || !g_glm_q4_k_pair_swiglu4_f32_pipeline || - !g_glm_q4_k_pair_swiglu2_mapped_f32_pipeline || - !g_glm_q4_k_pair_swiglu2_mapped_row_f32_pipeline || !g_glm_q2_k_pair_swiglu_f32_pipeline || !g_glm_q2_k_addr_pair_swiglu2_f32_pipeline || !g_glm_q2_k_addr_pair_swiglu2_masked_f32_pipeline || @@ -9765,8 +9713,6 @@ int ds4_gpu_init(void) { !g_glm_q2_k_addr_down_f32_pipeline || !g_glm_q4_k_addr_down_f32_pipeline || !g_glm_q5_k_pair_swiglu_f32_pipeline || - !g_glm_q5_k_pair_swiglu_mapped_f32_pipeline || - !g_glm_q5_k_pair_swiglu_mapped_row_f32_pipeline || !g_glm_q5_k_down_f32_pipeline || !g_glm_q6_k_down_f32_pipeline || !g_dsv4_hc_expand4_pipeline) { @@ -11826,7 +11772,6 @@ void ds4_gpu_cleanup(void) { g_cpy_contig_f16_f32_pipeline = nil; g_cpy_contig_f16_f16_pipeline = nil; g_flash_kv_stage_f16_pipeline = nil; - g_swiglu_pipeline = nil; g_swiglu_flat_pipeline = nil; g_add_pipeline = nil; g_add2_pipeline = nil; @@ -11976,13 +11921,11 @@ void ds4_gpu_cleanup(void) { g_glm_indexer_score_one_direct_pipeline = nil; g_glm_indexer_scores_batch_pipeline = nil; g_glm_indexer_scores_tiled_pipeline = nil; - g_glm_indexer_scores_tiled_f32_pipeline = nil; g_glm_qk_lowrank_pipeline = nil; g_glm_qk_lowrank_glm52_pipeline = nil; g_glm_qk_lowrank_glm52_sg_pipeline = nil; g_glm_qk_lowrank_batch_pipeline = nil; g_glm_qk_lowrank_batch_glm52_t4_pipeline = nil; - g_glm_value_project_q8_0_pipeline = nil; g_glm_value_project_q8_0_batch_heads_pipeline = nil; g_glm_value_project_q8_0_batch_heads_mma_pipeline = nil; g_glm_attention_indexed_decode_pipeline = nil; @@ -11999,11 +11942,8 @@ void ds4_gpu_cleanup(void) { g_glm_attention_indexed_batch_lora_group8_vec_valid_fullheads_pipeline = nil; g_glm_attention_indexed_batch_lora_group8_vec_causal_pipeline = nil; g_glm_attention_indexed_batch_lora_group8_vec_causal_fullheads_pipeline = nil; - g_glm_q4_k_pair_swiglu_f32_pipeline = nil; g_glm_q4_k_pair_swiglu2_f32_pipeline = nil; g_glm_q4_k_pair_swiglu4_f32_pipeline = nil; - g_glm_q4_k_pair_swiglu2_mapped_f32_pipeline = nil; - g_glm_q4_k_pair_swiglu2_mapped_row_f32_pipeline = nil; g_glm_q2_k_pair_swiglu_f32_pipeline = nil; g_glm_q2_k_addr_pair_swiglu2_f32_pipeline = nil; g_glm_q2_k_addr_pair_swiglu2_masked_f32_pipeline = nil; @@ -12014,8 +11954,6 @@ void ds4_gpu_cleanup(void) { g_glm_q2_k_addr_down_f32_pipeline = nil; g_glm_q4_k_addr_down_f32_pipeline = nil; g_glm_q5_k_pair_swiglu_f32_pipeline = nil; - g_glm_q5_k_pair_swiglu_mapped_f32_pipeline = nil; - g_glm_q5_k_pair_swiglu_mapped_row_f32_pipeline = nil; g_glm_q5_k_down_f32_pipeline = nil; g_glm_q6_k_down_f32_pipeline = nil; g_dsv4_router_weights_batch_pipeline = nil; @@ -12031,7 +11969,6 @@ void ds4_gpu_cleanup(void) { g_flash_attn_pad_buffer = nil; g_flash_attn_tmp_buffer = nil; g_flash_attn_blk_buffer = nil; - g_flash_attn_ring_buffer = nil; g_flash_attn_kv_buffer = nil; g_glm_flash_attn_mask_buffer = nil; g_compressor_pool_kv_buffer = nil; @@ -12085,7 +12022,6 @@ void ds4_gpu_cleanup(void) { g_flash_attn_pad_bytes = 0; g_flash_attn_tmp_bytes = 0; g_flash_attn_blk_bytes = 0; - g_flash_attn_ring_bytes = 0; g_flash_attn_kv_bytes = 0; g_glm_flash_attn_mask_bytes = 0; g_compressor_pool_kv_bytes = 0; @@ -41720,15 +41656,12 @@ static int ds4_gpu_glm_indexer_scores_batch_grouped_tensor( } const bool force_scalar = g_quality_mode; - const bool use_tiled_f32 = false; const bool use_tiled = !force_scalar && n_tokens >= 8u && n_head == 32u && head_dim == 128u; id pipeline = use_tiled - ? ds4_gpu_hot_pipeline(use_tiled_f32 ? g_glm_indexer_scores_tiled_f32_pipeline - : g_glm_indexer_scores_tiled_pipeline, - use_tiled_f32 ? "kernel_glm_indexer_scores_tiled_f32" - : "kernel_glm_indexer_scores_tiled") + ? ds4_gpu_hot_pipeline(g_glm_indexer_scores_tiled_pipeline, + "kernel_glm_indexer_scores_tiled") : ds4_gpu_hot_pipeline(g_glm_indexer_scores_batch_pipeline, "kernel_glm_indexer_scores_batch"); if (!pipeline) return 0; @@ -41765,13 +41698,8 @@ static int ds4_gpu_glm_indexer_scores_batch_grouped_tensor( const NSUInteger q_shared = 8u * 128u; const NSUInteger k_shared = 32u * 128u; const NSUInteger dot_shared = 8u * 32u; - if (use_tiled_f32) { - [enc setThreadgroupMemoryLength:(q_shared + k_shared + dot_shared) * - sizeof(float) atIndex:0]; - } else { - [enc setThreadgroupMemoryLength:(q_shared + k_shared) * sizeof(uint16_t) + - dot_shared * sizeof(float) atIndex:0]; - } + [enc setThreadgroupMemoryLength:(q_shared + k_shared) * sizeof(uint16_t) + + dot_shared * sizeof(float) atIndex:0]; [enc dispatchThreadgroups:MTLSizeMake(((NSUInteger)n_rows + 31u) / 32u, ((NSUInteger)n_tokens + 7u) / 8u, 1) @@ -44334,11 +44262,6 @@ static bool ds4_gpu_glm_routed_moe_batch_grouped_available( ds4_gpu_routed_mm_f16_rhs_pipeline(down_type) != nil; } -static bool ds4_gpu_glm_grouped_moe_layer_enabled(uint32_t layer_index) { - (void)layer_index; - return true; -} - static int ds4_gpu_glm_routed_moe_batch_grouped_tensor( ds4_gpu_tensor *out, ds4_gpu_tensor *mid, @@ -44364,7 +44287,6 @@ static int ds4_gpu_glm_routed_moe_batch_grouped_tensor( uint32_t n_total_expert, uint32_t n_expert, float swiglu_clamp, - uint32_t layer_index, const ds4_gpu_tensor *x, uint32_t n_tokens) { if (!ds4_gpu_glm_routed_moe_batch_grouped_available(gate_type, @@ -44389,14 +44311,13 @@ static int ds4_gpu_glm_routed_moe_batch_grouped_tensor( return 0; } - const bool mid_f16 = true; const NSUInteger mm_id_threadgroup_bytes = 8192u; const uint64_t compact_mid_values = (uint64_t)pair_rows * expert_mid_dim; const uint64_t down_values = (uint64_t)pair_rows * out_dim; const uint64_t x_values = (uint64_t)n_tokens * expert_in_dim; const uint64_t out_values = (uint64_t)n_tokens * out_dim; if (compact_mid_values > UINT64_MAX / sizeof(float) || - compact_mid_values > UINT64_MAX / (mid_f16 ? sizeof(uint16_t) : sizeof(float)) || + compact_mid_values > UINT64_MAX / sizeof(uint16_t) || down_values > UINT64_MAX / sizeof(float) || x_values > UINT64_MAX / sizeof(float) || out_values > UINT64_MAX / sizeof(float)) { @@ -44404,7 +44325,7 @@ static int ds4_gpu_glm_routed_moe_batch_grouped_tensor( } const uint64_t gate_scratch_bytes = compact_mid_values * sizeof(float); - const uint64_t mid_bytes = compact_mid_values * (mid_f16 ? sizeof(uint16_t) : sizeof(float)); + const uint64_t mid_bytes = compact_mid_values * sizeof(uint16_t); const uint64_t down_scratch_bytes = down_values * sizeof(float); const uint64_t x_bytes = x_values * sizeof(float); const uint64_t out_bytes = out_values * sizeof(float); @@ -44495,7 +44416,7 @@ static int ds4_gpu_glm_routed_moe_batch_grouped_tensor( ds4_gpu_make_mul_mm_id_args_src1_size(expert_mid_dim, out_dim, n_total_expert, down_row_bytes, down_expert_bytes, n_expert, n_expert, n_tokens, - mid_f16 ? sizeof(uint16_t) : sizeof(float)); + sizeof(uint16_t)); gate_args.tp_rank = g_tp_split_rank; gate_args.tp_world = g_tp_split_world; gate_args.tp_expert_base = (int32_t)first_expert; @@ -44509,57 +44430,13 @@ static int ds4_gpu_glm_routed_moe_batch_grouped_tensor( id cb = ds4_gpu_command_buffer(&owned); if (!cb) return 0; - const bool glm_moe_stage_profile = false; - const char *glm_moe_stage_filter = NULL; - double glm_moe_stage_t0 = 0.0; - if (glm_moe_stage_profile) { - if (ds4_gpu_end_commands() == 0 || ds4_gpu_begin_commands() == 0) { - return 0; - } - cb = ds4_gpu_command_buffer(&owned); - if (!cb) return 0; - glm_moe_stage_t0 = ds4_gpu_now_ms(); - } - int ok = 1; -#define DS4_METAL_PROFILE_GLM_GROUPED_MOE_STAGE(name) do { \ - if (ok && glm_moe_stage_profile) { \ - if (ds4_gpu_end_commands() == 0) { \ - ok = 0; \ - } else { \ - const char *stage_name = (name); \ - const double now_ms = ds4_gpu_now_ms(); \ - const int print_stage = \ - !glm_moe_stage_filter || !glm_moe_stage_filter[0] || \ - strstr(stage_name, glm_moe_stage_filter) != NULL; \ - if (print_stage) { \ - fprintf(stderr, \ - "ds4: Metal GLM grouped routed MoE stage layer=%u tokens=%u pairs=%u experts=%u " \ - "gate=%s down=%s mid=%s %s=%.3f ms\n", \ - layer_index, n_tokens, pair_rows, n_expert, \ - ds4_gpu_metal_tensor_type_name(gate_type), \ - ds4_gpu_metal_tensor_type_name(down_type), \ - mid_f16 ? "f16" : "f32", \ - stage_name, now_ms - glm_moe_stage_t0); \ - } \ - glm_moe_stage_t0 = now_ms; \ - if (ds4_gpu_begin_commands() == 0) { \ - ok = 0; \ - } else { \ - cb = ds4_gpu_command_buffer(&owned); \ - if (!cb) ok = 0; \ - } \ - } \ - } \ - } while (0) - ok = ds4_gpu_encode_mul_mm_id_map(cb, map_pipeline, &map_args, &gate_args, selectedbuf, ds4_gpu_tensor_offset(selected)); - DS4_METAL_PROFILE_GLM_GROUPED_MOE_STAGE("map"); if (ok) { ok = ds4_gpu_encode_mul_mm_id_mapped_tile(cb, gate_pipeline, @@ -44572,7 +44449,6 @@ static int ds4_gpu_glm_routed_moe_batch_grouped_tensor( 0, mm_id_threadgroup_bytes); } - DS4_METAL_PROFILE_GLM_GROUPED_MOE_STAGE("gate"); if (ok) { ok = ds4_gpu_encode_mul_mm_id_mapped_tile(cb, up_pipeline, @@ -44585,7 +44461,6 @@ static int ds4_gpu_glm_routed_moe_batch_grouped_tensor( (NSUInteger)gate_scratch_bytes, mm_id_threadgroup_bytes); } - DS4_METAL_PROFILE_GLM_GROUPED_MOE_STAGE("up"); if (ok) { ok = ds4_gpu_encode_moe_swiglu_weight(cb, g_moe_gate_scratch_buffer, @@ -44599,10 +44474,8 @@ static int ds4_gpu_glm_routed_moe_batch_grouped_tensor( expert_mid_dim, pair_rows, swiglu_clamp, - mid_f16); + true); } - DS4_METAL_PROFILE_GLM_GROUPED_MOE_STAGE("activation_weight"); - id down_dst = n_expert == 1 ? outbuf : g_moe_down_scratch_buffer; NSUInteger down_dst_off = n_expert == 1 ? ds4_gpu_tensor_offset(out) : 0; if (ok) { @@ -44617,7 +44490,6 @@ static int ds4_gpu_glm_routed_moe_batch_grouped_tensor( down_dst_off, mm_id_threadgroup_bytes); } - DS4_METAL_PROFILE_GLM_GROUPED_MOE_STAGE("down"); if (ok && n_expert > 1) { ok = ds4_gpu_encode_moe_sum_experts(cb, down_dst, @@ -44628,13 +44500,11 @@ static int ds4_gpu_glm_routed_moe_batch_grouped_tensor( n_expert, n_tokens); } - DS4_METAL_PROFILE_GLM_GROUPED_MOE_STAGE("sum"); if (!ok) return 0; if (!ds4_gpu_finish_command_buffer(cb, owned, "GLM grouped routed batch MoE")) { return 0; } -#undef DS4_METAL_PROFILE_GLM_GROUPED_MOE_STAGE } return 1; @@ -44666,7 +44536,6 @@ static int ds4_gpu_glm_routed_moe_batch_grouped_addr_tensor( uint32_t n_total_expert, uint32_t n_expert, float swiglu_clamp, - uint32_t layer_index, const ds4_gpu_tensor *x, uint32_t n_tokens, ds4_gpu_stream_expert_cache_entry * const *resources, @@ -44689,7 +44558,6 @@ static int ds4_gpu_glm_routed_moe_batch_grouped_addr_tensor( return 0; } - const bool mid_f16 = true; const NSUInteger mm_id_threadgroup_bytes = 8192u; const uint64_t compact_mid_values = (uint64_t)pair_rows * expert_mid_dim; const uint64_t down_values = (uint64_t)pair_rows * out_dim; @@ -44784,56 +44652,13 @@ static int ds4_gpu_glm_routed_moe_batch_grouped_addr_tensor( id cb = ds4_gpu_command_buffer(&owned); if (!cb) return 0; - const bool glm_moe_stage_profile = false; - const char *glm_moe_stage_filter = NULL; - double glm_moe_stage_t0 = 0.0; - if (glm_moe_stage_profile) { - if (ds4_gpu_end_commands() == 0 || ds4_gpu_begin_commands() == 0) { - return 0; - } - cb = ds4_gpu_command_buffer(&owned); - if (!cb) return 0; - glm_moe_stage_t0 = ds4_gpu_now_ms(); - } - int ok = 1; -#define DS4_METAL_PROFILE_GLM_GROUPED_ADDR_MOE_STAGE(name) do { \ - if (ok && glm_moe_stage_profile) { \ - if (ds4_gpu_end_commands() == 0) { \ - ok = 0; \ - } else { \ - const char *stage_name = (name); \ - const double now_ms = ds4_gpu_now_ms(); \ - const int print_stage = \ - !glm_moe_stage_filter || !glm_moe_stage_filter[0] || \ - strstr(stage_name, glm_moe_stage_filter) != NULL; \ - if (print_stage) { \ - fprintf(stderr, \ - "ds4: Metal GLM grouped-address routed MoE stage layer=%u tokens=%u pairs=%u experts=%u " \ - "gate=%s down=%s mid=f16 %s=%.3f ms\n", \ - layer_index, n_tokens, pair_rows, n_expert, \ - ds4_gpu_metal_tensor_type_name(gate_type), \ - ds4_gpu_metal_tensor_type_name(down_type), \ - stage_name, now_ms - glm_moe_stage_t0); \ - } \ - glm_moe_stage_t0 = now_ms; \ - if (ds4_gpu_begin_commands() == 0) { \ - ok = 0; \ - } else { \ - cb = ds4_gpu_command_buffer(&owned); \ - if (!cb) ok = 0; \ - } \ - } \ - } \ - } while (0) - ok = ds4_gpu_encode_mul_mm_id_map(cb, map_pipeline, &map_args, &gate_args, selectedbuf, ds4_gpu_tensor_offset(selected)); - DS4_METAL_PROFILE_GLM_GROUPED_ADDR_MOE_STAGE("map"); if (ok) { ok = ds4_gpu_encode_mul_mm_id_addr_mapped_tile(cb, gate_pipeline, @@ -44849,7 +44674,6 @@ static int ds4_gpu_glm_routed_moe_batch_grouped_addr_tensor( 0, overflow_gate); } - DS4_METAL_PROFILE_GLM_GROUPED_ADDR_MOE_STAGE("gate"); if (ok) { ok = ds4_gpu_encode_mul_mm_id_addr_mapped_tile(cb, up_pipeline, @@ -44865,7 +44689,6 @@ static int ds4_gpu_glm_routed_moe_batch_grouped_addr_tensor( 1, overflow_up); } - DS4_METAL_PROFILE_GLM_GROUPED_ADDR_MOE_STAGE("up"); if (ok) { ok = ds4_gpu_encode_moe_swiglu_weight(cb, g_moe_gate_scratch_buffer, @@ -44879,10 +44702,8 @@ static int ds4_gpu_glm_routed_moe_batch_grouped_addr_tensor( expert_mid_dim, pair_rows, swiglu_clamp, - mid_f16); + true); } - DS4_METAL_PROFILE_GLM_GROUPED_ADDR_MOE_STAGE("activation_weight"); - id down_dst = n_expert == 1 ? outbuf : g_moe_down_scratch_buffer; NSUInteger down_dst_off = n_expert == 1 ? ds4_gpu_tensor_offset(out) : 0; if (ok) { @@ -44900,7 +44721,6 @@ static int ds4_gpu_glm_routed_moe_batch_grouped_addr_tensor( 2, overflow_down); } - DS4_METAL_PROFILE_GLM_GROUPED_ADDR_MOE_STAGE("down"); if (ok && n_expert > 1) { ok = ds4_gpu_encode_moe_sum_experts(cb, down_dst, @@ -44911,14 +44731,12 @@ static int ds4_gpu_glm_routed_moe_batch_grouped_addr_tensor( n_expert, n_tokens); } - DS4_METAL_PROFILE_GLM_GROUPED_ADDR_MOE_STAGE("sum"); if (!ok) return 0; if (!ds4_gpu_finish_command_buffer(cb, owned, "GLM grouped-address routed batch MoE")) { return 0; } -#undef DS4_METAL_PROFILE_GLM_GROUPED_ADDR_MOE_STAGE } return 1; @@ -44953,8 +44771,7 @@ static int ds4_gpu_glm_routed_moe_batch_tensor_impl( const ds4_gpu_tensor *x, uint32_t n_tokens, uint32_t mid_token_stride, - bool allow_grouped, - bool force_scalar_q4_pair) { + bool allow_grouped) { if (!g_initialized && !ds4_gpu_init()) return 0; if (!out || !mid || !model_map || !selected || !weights || !x || n_tokens == 0 || @@ -45007,7 +44824,6 @@ static int ds4_gpu_glm_routed_moe_batch_tensor_impl( if (allow_grouped && (!g_ssd_streaming_mode || ds4_gpu_glm_streaming_prefill_full_layer_active()) && - ds4_gpu_glm_grouped_moe_layer_enabled(layer_index) && ds4_gpu_glm_routed_moe_batch_grouped_available(gate_type, up_type, down_type, @@ -45037,7 +44853,6 @@ static int ds4_gpu_glm_routed_moe_batch_tensor_impl( n_total_expert, n_expert, swiglu_clamp, - layer_index, x, n_tokens); } @@ -45149,12 +44964,6 @@ static int ds4_gpu_glm_routed_moe_batch_tensor_impl( down_type, n_expert, n_tokens); - const BOOL enable_q4_pair4 = true; - const BOOL q4_scalar_pair = false; - const BOOL q4_pair2 = - !use_stream_expert_addr_table && - !gate_pair_q5 && !q4_scalar_pair && - (force_scalar_q4_pair || !enable_q4_pair4); id pair_pipeline = use_stream_expert_addr_table ? (gate_pair_q2 ? @@ -45168,14 +44977,8 @@ static int ds4_gpu_glm_routed_moe_batch_tensor_impl( gate_pair_q5 ? ds4_gpu_hot_pipeline(g_glm_q5_k_pair_swiglu_f32_pipeline, "kernel_glm_q5_K_pair_swiglu_f32") : - (q4_scalar_pair ? - ds4_gpu_hot_pipeline(g_glm_q4_k_pair_swiglu_f32_pipeline, - "kernel_glm_q4_K_pair_swiglu_f32") : - q4_pair2 ? - ds4_gpu_hot_pipeline(g_glm_q4_k_pair_swiglu2_f32_pipeline, - "kernel_glm_q4_K_pair_swiglu2_f32") : - ds4_gpu_hot_pipeline(g_glm_q4_k_pair_swiglu4_f32_pipeline, - "kernel_glm_q4_K_pair_swiglu4_f32")); + ds4_gpu_hot_pipeline(g_glm_q4_k_pair_swiglu4_f32_pipeline, + "kernel_glm_q4_K_pair_swiglu4_f32"); id down_pipeline = use_stream_expert_addr_table ? (down_scalar_q2 ? @@ -45261,7 +45064,6 @@ static int ds4_gpu_glm_routed_moe_batch_tensor_impl( n_total_expert, n_expert, swiglu_clamp, - layer_index, x, n_tokens, stream_resources, @@ -45284,63 +45086,6 @@ static int ds4_gpu_glm_routed_moe_batch_tensor_impl( id cb = ds4_gpu_command_buffer(&owned); if (!cb) return 0; - const bool glm_moe_stage_profile = false; - const char *glm_moe_stage_filter = NULL; - const char *glm_pair_path = use_stream_expert_addr_table ? - (gate_pair_q2 ? "q2_stream_addr_swiglu" : - "q4_stream_addr_swiglu") : - gate_pair_q2 ? "q2_scalar_swiglu" : - gate_pair_q5 ? "q5_pair_simd_swiglu" : - (q4_scalar_pair ? "q4_scalar_swiglu" : - (q4_pair2 ? "q4_pair2_simd_swiglu" : - "q4_pair4_simd_swiglu")); - const char *glm_down_path = - use_stream_expert_addr_table ? - (down_scalar_q2 ? "q2_stream_addr_down" : "q4_stream_addr_down_simd") : - down_scalar_q2 ? "q2_down_scalar" : - down_scalar_q4 ? "q4_down_simd" : - down_simd_q5 ? "q5_down_simd" : "q6_down_simd"; - double glm_moe_stage_t0 = 0.0; - if (glm_moe_stage_profile) { - if (ds4_gpu_end_commands() == 0 || ds4_gpu_begin_commands() == 0) { - return 0; - } - cb = ds4_gpu_command_buffer(&owned); - if (!cb) return 0; - glm_moe_stage_t0 = ds4_gpu_now_ms(); - } - int ok = 1; -#define DS4_METAL_PROFILE_GLM_MOE_BATCH_STAGE(name) do { \ - if (ok && glm_moe_stage_profile) { \ - if (ds4_gpu_end_commands() == 0) { \ - ok = 0; \ - } else { \ - const char *stage_name = (name); \ - const double now_ms = ds4_gpu_now_ms(); \ - const int print_stage = \ - !glm_moe_stage_filter || !glm_moe_stage_filter[0] || \ - strstr(stage_name, glm_moe_stage_filter) != NULL; \ - if (print_stage) { \ - fprintf(stderr, \ - "ds4: Metal GLM routed MoE batch stage layer=%u tokens=%u experts=%u " \ - "gate=%s down=%s pair=%s down_path=%s %s=%.3f ms\n", \ - layer_index, n_tokens, n_expert, \ - ds4_gpu_metal_tensor_type_name(gate_type), \ - ds4_gpu_metal_tensor_type_name(down_type), \ - glm_pair_path, glm_down_path, \ - stage_name, now_ms - glm_moe_stage_t0); \ - } \ - glm_moe_stage_t0 = now_ms; \ - if (ds4_gpu_begin_commands() == 0) { \ - ok = 0; \ - } else { \ - cb = ds4_gpu_command_buffer(&owned); \ - if (!cb) ok = 0; \ - } \ - } \ - } \ - } while (0) - ds4_gpu_glm_routed_moe_args args = { .tp_rank = g_tp_split_rank, .tp_world = g_tp_split_world, @@ -45367,13 +45112,7 @@ static int ds4_gpu_glm_routed_moe_batch_tensor_impl( (NSUInteger)((expert_mid_dim + 7u) / 8u)) : gate_pair_q5 ? (NSUInteger)((expert_mid_dim + 7u) / 8u) : use_stream_expert_addr_table ? (NSUInteger)((expert_mid_dim + 3u) / 4u) : - q4_scalar_pair ? (NSUInteger)expert_mid_dim : - q4_pair2 ? (NSUInteger)((expert_mid_dim + 1u) / 2u) : (NSUInteger)((expert_mid_dim + 7u) / 8u); - const NSUInteger pair_threadgroup_bytes = - q4_scalar_pair ? 512u * sizeof(float) : 0u; - const NSUInteger pair_threads = - q4_scalar_pair ? 256u : 64u; const NSUInteger down_x_groups = down_scalar_q2 ? (NSUInteger)((out_dim + 7u) / 8u) : down_simd_q4 ? (NSUInteger)((out_dim + 3u) / 4u) : @@ -45412,16 +45151,11 @@ static int ds4_gpu_glm_routed_moe_batch_tensor_impl( if (stream_overflow_gate) [enc useResource:stream_overflow_gate usage:MTLResourceUsageRead]; if (stream_overflow_up) [enc useResource:stream_overflow_up usage:MTLResourceUsageRead]; } - if (pair_threadgroup_bytes != 0u) { - [enc setThreadgroupMemoryLength:pair_threadgroup_bytes atIndex:0]; - } [enc dispatchThreadgroups:MTLSizeMake(pair_x_groups, (NSUInteger)n_expert, (NSUInteger)n_tokens) - threadsPerThreadgroup:MTLSizeMake(pair_threads, 1, 1)]; + threadsPerThreadgroup:MTLSizeMake(64u, 1, 1)]; ds4_gpu_end_compute_encoder(cb, enc); - DS4_METAL_PROFILE_GLM_MOE_BATCH_STAGE("pair"); - enc = ds4_gpu_compute_encoder(cb); [enc setComputePipelineState:down_pipeline]; [enc setBytes:&args length:sizeof(args) atIndex:0]; @@ -45445,11 +45179,7 @@ static int ds4_gpu_glm_routed_moe_batch_tensor_impl( 1) threadsPerThreadgroup:MTLSizeMake(down_threads, 1, 1)]; ds4_gpu_end_compute_encoder(cb, enc); - DS4_METAL_PROFILE_GLM_MOE_BATCH_STAGE("down"); - - if (!ok) return 0; if (!ds4_gpu_finish_command_buffer(cb, owned, "GLM routed batch MoE")) return 0; -#undef DS4_METAL_PROFILE_GLM_MOE_BATCH_STAGE } return 1; @@ -45514,8 +45244,7 @@ int ds4_gpu_glm_routed_moe_batch_tensor( x, n_tokens, mid_token_stride, - true, - false); + true); } int ds4_gpu_glm_routed_moe_batch_direct_scalar_q4_tensor( @@ -45575,7 +45304,6 @@ int ds4_gpu_glm_routed_moe_batch_direct_scalar_q4_tensor( x, n_tokens, mid_token_stride, - false, false); } diff --git a/ds4_rocm.cu b/ds4_rocm.cu index 652b43c30..7ec56d841 100644 --- a/ds4_rocm.cu +++ b/ds4_rocm.cu @@ -1,6 +1,5 @@ #ifdef __HIP_PLATFORM_AMD__ #include "ds4_rocm.h" -#include #include #define FULL_WARP_MASK 0xFFFFFFFFFFFFFFFFULL @@ -65,7 +64,6 @@ extern "C" int ds4_mmq_q2_K_moe_down_sum6_vec( #endif #define CUDA_QK_K 256 -#define DS4_ROCM_UNUSED __attribute__((unused)) enum { /* attention_decode_mixed_kernel stores raw-window scores plus visible @@ -114,13 +112,6 @@ typedef struct { static_assert(sizeof(cuda_block_mxfp4) == 17, "cuda_block_mxfp4 must match the GGUF MXFP4 block layout"); -/* Twice the MXFP4 values so each 32-value sub-block can use signed-int8 - * dp4a; the factor of 1/2 is folded into the sub-block scale. */ -__device__ __constant__ static const int8_t cuda_mxfp4_values_x2[16] = { - 0, 1, 2, 3, 4, 6, 8, 12, - 0, -1, -2, -3, -4, -6, -8, -12, -}; - #include "ds4_iq2_tables_cuda.inc" #include "rocm/ds4_rocm_runtime.cuh" diff --git a/ds4_server.c b/ds4_server.c index dfe87cb61..bc770ca97 100644 --- a/ds4_server.c +++ b/ds4_server.c @@ -3285,13 +3285,6 @@ static char *render_live_tool_tail_for_syntax(server_model_syntax syntax, return render_deepseek_live_tool_tail(msgs, start, think_mode); } -static DS4_SERVER_MAYBE_UNUSED char *render_live_tool_tail( - const chat_msgs *msgs, int start, - ds4_think_mode think_mode) { - return render_live_tool_tail_for_syntax(SERVER_MODEL_SYNTAX_DEEPSEEK, - msgs, start, NULL, think_mode); -} - static void chat_msg_collect_tool_call_ids(const chat_msg *m, stop_list *ids) { if (!m || !ids) return; id_list_push_unique(ids, m->tool_call_id); diff --git a/ds4_tp.c b/ds4_tp.c index 6c50f0127..b5326ceb2 100644 --- a/ds4_tp.c +++ b/ds4_tp.c @@ -370,18 +370,6 @@ bool ds4_tp_enabled(const ds4_tp_options *opt) { return opt && opt->role != DS4_TP_NONE; } -void ds4_tp_usage(FILE *fp) { - fprintf(fp, - "Tensor parallelism (two identical machines):\n" - " --tensor-parallel Use --role/--listen/--coordinator for a 50/50 TP pair.\n" - " --transport Gate transport (default auto).\n" - " --rdma-device Select a verbs device such as rdma_en1.\n" - " --rdma-gid-index Select the local verbs GID index.\n" - " --tensor-parallel-token-prefill\n" - " GLM diagnostic: prefill one token at a time.\n" - " --debug-hash Cross-check hidden state every n tokens.\n"); -} - int ds4_tp_parse_cli_arg( const char *arg, int *index, @@ -1505,7 +1493,6 @@ void ds4_tp_free(ds4_tp *tp) { int ds4_tp_rank(const ds4_tp *tp) { return tp->rank; } bool ds4_tp_is_rdma(const ds4_tp *tp) { return tp->rdma_active; } -uint32_t ds4_tp_peer_ctx(const ds4_tp *tp) { return tp->peer_ctx; } bool ds4_tp_failed(const ds4_tp *tp) { return tp && atomic_load_explicit(&tp->failed, memory_order_acquire); } @@ -2181,29 +2168,6 @@ int ds4_tp_recv_verify_commit(ds4_tp *tp, int32_t *full_accept, int32_t *replay_ return 1; } -int ds4_tp_hash_check(ds4_tp *tp, uint64_t seq, uint64_t hash, char *err, size_t errlen) { - struct { uint64_t seq; uint64_t hash; } mine = { seq, hash }, theirs; - if (!tp_send_frame(tp->control_fd, DS4_TP_FRAME_HASH, &mine, sizeof(mine))) { - tp_set_err(err, errlen, "tp: hash send failed"); - return 0; - } - uint32_t type = 0, bytes = 0; - if (!tp_read_frame_header(tp->control_fd, &type, &bytes) || - type != DS4_TP_FRAME_HASH || bytes != sizeof(theirs) || - !tp_read_full(tp->control_fd, &theirs, sizeof(theirs))) { - tp_set_err(err, errlen, "tp: hash recv failed"); - return 0; - } - if (theirs.seq != seq || theirs.hash != hash) { - tp_set_err(err, errlen, - "tp: LOCKSTEP DIVERGENCE at seq %llu: local %016llx peer %016llx", - (unsigned long long)seq, - (unsigned long long)hash, (unsigned long long)theirs.hash); - return -1; - } - return 1; -} - /* ------------------------------------------------------------------------ * Worker main loop. * --------------------------------------------------------------------- */ diff --git a/ds4_tp.h b/ds4_tp.h index ceb4dd2de..a6ee0af72 100644 --- a/ds4_tp.h +++ b/ds4_tp.h @@ -4,7 +4,6 @@ #include #include #include -#include #include "ds4.h" @@ -81,8 +80,6 @@ int ds4_tp_adopt_distributed_options( ds4_distributed_options *dist, char *err, size_t errlen); -void ds4_tp_usage(FILE *fp); - /* Validates option combinations that TP cannot run with (SSD streaming, * distributed mode, MTP drafting, CPU backend). */ int ds4_tp_validate_engine_options( @@ -103,7 +100,6 @@ void ds4_tp_free(ds4_tp *tp); int ds4_tp_rank(const ds4_tp *tp); bool ds4_tp_is_rdma(const ds4_tp *tp); -uint32_t ds4_tp_peer_ctx(const ds4_tp *tp); bool ds4_tp_failed(const ds4_tp *tp); void ds4_tp_mark_failed(ds4_tp *tp); @@ -183,9 +179,7 @@ typedef enum { DS4_TP_FRAME_REWIND = 3, DS4_TP_FRAME_INVALIDATE = 4, DS4_TP_FRAME_STOP = 5, - DS4_TP_FRAME_HASH = 6, DS4_TP_FRAME_RDMA_INFO = 7, - DS4_TP_FRAME_SYNC_ACK = 8, DS4_TP_FRAME_RDMA_READY = 9, DS4_TP_FRAME_LOGITS = 10, DS4_TP_FRAME_VERIFY = 11, @@ -218,10 +212,6 @@ int ds4_tp_recv_command( size_t errlen); void ds4_tp_command_free(ds4_tp_command *command); -/* Debug lockstep check: both sides send their hidden-state hash for a token - * and compare. Returns 0 on transport failure, -1 on hash mismatch. */ -int ds4_tp_hash_check(ds4_tp *tp, uint64_t seq, uint64_t hash, char *err, size_t errlen); - /* Vocab-split output head: the worker ships its logits half to the leader * after every eval (and after a sync) on the control socket. */ int ds4_tp_send_logits_half(ds4_tp *tp, const float *half, uint32_t count); diff --git a/linenoise.c b/linenoise.c index 9639422ef..68146c505 100644 --- a/linenoise.c +++ b/linenoise.c @@ -109,12 +109,10 @@ #include #include #include -#include #include #include #include #include -#include #include #include "linenoise.h" @@ -122,7 +120,6 @@ #define LINENOISE_MAX_LINE (1024*1024) // That will get dynamically allocated #define LINENOISE_INITIAL_BUFLEN 4096 #define PASTE_FOLD_THRESHOLD 200 // Min bytes to fold a single-line paste. -#define PASTE_FOLD_CONTEXT 8 // Context chars kept around generic folds. #define HISTORY_FOLD_THRESHOLD 4096 // Min bytes to fold single-line history. #define HISTORY_FOLD_MULTILINE_LINES 16 // Min lines to fold shorter history. #define HISTORY_FOLD_CONTEXT 96 // Context chars kept around history folds. @@ -486,14 +483,12 @@ static int utf8SingleCharWidth(const char *s, size_t len) { } enum KEY_ACTION{ - KEY_NULL = 0, /* NULL */ CTRL_A = 1, /* Ctrl+a */ CTRL_B = 2, /* Ctrl-b */ CTRL_C = 3, /* Ctrl-c */ CTRL_D = 4, /* Ctrl-d */ CTRL_E = 5, /* Ctrl-e */ CTRL_F = 6, /* Ctrl-f */ - CTRL_H = 8, /* Ctrl-h */ TAB = 9, /* Tab */ CTRL_K = 11, /* Ctrl+k */ CTRL_L = 12, /* Ctrl+l */ @@ -514,24 +509,7 @@ int linenoiseHistoryAdd(const char *line); #define REFRESH_ALL (REFRESH_CLEAN|REFRESH_WRITE) // Do both. static void refreshLine(struct linenoiseState *l); -/* Debugging macro. */ -#if 0 -FILE *lndebug_fp = NULL; -#define lndebug(...) \ - do { \ - if (lndebug_fp == NULL) { \ - lndebug_fp = fopen("/tmp/lndebug.txt","a"); \ - fprintf(lndebug_fp, \ - "[%d %d %d] p: %d, rows: %d, rpos: %d, max: %d, oldmax: %d\n", \ - (int)l->len,(int)l->pos,(int)l->oldpos,plen,rows,rpos, \ - (int)l->oldrows,old_rows); \ - } \ - fprintf(lndebug_fp, ", " __VA_ARGS__); \ - fflush(lndebug_fp); \ - } while (0) -#else #define lndebug(fmt, ...) -#endif /* ======================= Low level terminal handling ====================== */ diff --git a/metal/dsv4_misc.metal b/metal/dsv4_misc.metal index 4d1536b7a..210a6bc96 100644 --- a/metal/dsv4_misc.metal +++ b/metal/dsv4_misc.metal @@ -2047,139 +2047,6 @@ kernel void kernel_glm_indexer_scores_batch( if (tid == 0) *dst = score; } -kernel void kernel_glm_indexer_scores_tiled_f32( - constant ds4_metal_args_glm_indexer_scores_batch & args, - device const char *q, - device const char *weights, - device const char *indexer_key_cache, - device char *scores, - threadgroup float *shared [[threadgroup(0)]], - uint2 tgpig [[threadgroup_position_in_grid]], - ushort tid [[thread_index_in_threadgroup]], - ushort lane [[thread_index_in_simdgroup]], - ushort sg [[simdgroup_index_in_threadgroup]]) { - constexpr uint TM = 8; - constexpr uint TN = 32; - constexpr uint TS = 8; - constexpr uint D = 128; - - const uint row_base = tgpig.x * TN; - const uint token_base = tgpig.y * TM; - - threadgroup float *qtg = shared; - threadgroup float *ktg = qtg + TM*D; - threadgroup float *dot = ktg + TN*D; - - const uint last_token = min(token_base + TM, args.n_tokens); - const uint max_visible = last_token > token_base ? - glm_indexer_batch_visible_rows(args, last_token - 1u) : 0u; - - if (row_base >= max_visible) { - for (uint i = tid; i < TM*TN; i += 128) { - const uint tr = i / TN; - const uint rc = i - tr*TN; - const uint token = token_base + tr; - const uint row = row_base + rc; - if (token < args.n_tokens && row < args.n_rows) { - device float *dst = (device float *)(scores + - (uint64_t)token * args.score_token_stride) + row; - *dst = -INFINITY; - } - } - return; - } - - for (uint i = tid; i < TN*D; i += 128) { - const uint rc = i / D; - const uint d = i - rc*D; - const uint row = row_base + rc; - float v = 0.0f; - if (row < args.n_rows) { - v = glm_cache_load_f32_or_f16(indexer_key_cache, - (uint64_t)row * args.head_dim + d, - args.cache_f16); - } - ktg[i] = v; - } - - const uint cell0 = lane; - const uint cell1 = lane + 32u; - const uint token_row0 = cell0 >> 3; - const uint token_row1 = cell1 >> 3; - const uint sub0 = cell0 & 7u; - const uint sub1 = cell1 & 7u; - const uint col0 = (uint)sg * TS + sub0; - const uint col1 = (uint)sg * TS + sub1; - const uint token0 = token_base + token_row0; - const uint token1 = token_base + token_row1; - const uint row0 = row_base + col0; - const uint row1 = row_base + col1; - - float acc0 = 0.0f; - float acc1 = 0.0f; - - threadgroup_barrier(mem_flags::mem_threadgroup); - - for (uint head = 0; head < args.n_head; head++) { - for (uint i = tid; i < TM*D; i += 128) { - const uint tr = i / D; - const uint d = i - tr*D; - const uint token = token_base + tr; - float v = 0.0f; - if (token < args.n_tokens) { - device const float *qrow = (device const float *)(q + - (uint64_t)token * args.q_token_stride + - (uint64_t)head * args.q_head_stride); - v = qrow[d]; - } - qtg[i] = v; - } - - threadgroup_barrier(mem_flags::mem_threadgroup); - - simdgroup_float8x8 mdot = make_filled_simdgroup_matrix(0.0f); - for (uint db = 0; db < D/TS; db++) { - simdgroup_float8x8 mq; - simdgroup_float8x8 mk; - simdgroup_load(mq, qtg + db*TS, D, 0, false); - simdgroup_load(mk, ktg + ((uint)sg * TS) * D + db*TS, D, 0, true); - simdgroup_multiply_accumulate(mdot, mq, mk, mdot); - } - - simdgroup_store(mdot, dot + (uint)sg * TS, TN, 0, false); - - threadgroup_barrier(mem_flags::mem_threadgroup); - - if (token0 < args.n_tokens && row0 < args.n_rows) { - device const float *w = (device const float *)(weights + - (uint64_t)token0 * args.weights_token_stride); - const float s = dot[token_row0*TN + col0]; - acc0 += max(s * args.scale, 0.0f) * w[head]; - } - if (token1 < args.n_tokens && row1 < args.n_rows) { - device const float *w = (device const float *)(weights + - (uint64_t)token1 * args.weights_token_stride); - const float s = dot[token_row1*TN + col1]; - acc1 += max(s * args.scale, 0.0f) * w[head]; - } - - threadgroup_barrier(mem_flags::mem_threadgroup); - } - - if (token0 < args.n_tokens && row0 < args.n_rows) { - const uint visible = glm_indexer_batch_visible_rows(args, token0); - device float *dst = (device float *)(scores + - (uint64_t)token0 * args.score_token_stride) + row0; - *dst = row0 < visible ? acc0 : -INFINITY; - } - if (token1 < args.n_tokens && row1 < args.n_rows) { - const uint visible = glm_indexer_batch_visible_rows(args, token1); - device float *dst = (device float *)(scores + - (uint64_t)token1 * args.score_token_stride) + row1; - *dst = row1 < visible ? acc1 : -INFINITY; - } -} - kernel void kernel_glm_indexer_scores_tiled( constant ds4_metal_args_glm_indexer_scores_batch & args, device const char *q, @@ -2589,34 +2456,6 @@ kernel void kernel_glm_qk_lowrank_q8_0_batch_glm52_t4( } } -kernel void kernel_glm_value_project_q8_0( - constant ds4_metal_args_glm_qk_lowrank & args, - device const char *weight, - device const char *lora, - device char *heads, - threadgroup float *x [[threadgroup(0)]], - uint tid [[thread_index_in_threadgroup]], - ushort3 ntg_u [[threads_per_threadgroup]], - uint3 tgpig [[threadgroup_position_in_grid]]) { - const uint head = tgpig.x; - if (head >= args.n_head) return; - const uint nth = ntg_u.x; - device const float *src = - (device const float *)(lora + (uint64_t)head * args.kv_lora_dim * sizeof(float)); - for (uint j = tid; j < args.kv_lora_dim; j += nth) { - x[j] = src[j]; - } - threadgroup_barrier(mem_flags::mem_threadgroup); - - device float *out = - (device float *)(heads + (uint64_t)head * args.qk_dim * sizeof(float)); - for (uint d = tid; d < args.qk_dim; d += nth) { - device const char *row = - weight + ((uint64_t)head * args.qk_dim + d) * args.row_bytes; - out[d] = glm_quant_dot_row_tg_f32(args.weight_type, row, x, args.kv_lora_dim); - } -} - kernel void kernel_glm_value_project_q8_0_batch_heads( constant ds4_metal_args_glm_qk_lowrank_batch & args, device const char *weight, @@ -5434,118 +5273,6 @@ kernel void kernel_dsv4_sort_i32_rows_asc( } } -static inline void dsv4_attend_f32_row_as_f16( - device const char *kv, - uint64_t row_stride, - uint row, - half4 q0, - half4 q1, - half4 q2, - half4 q3, - float scale, - ushort lane, - thread float &M, - thread float &S, - thread float4 &o0, - thread float4 &o1, - thread float4 &o2, - thread float4 &o3) { - device const float4 *kv4 = (device const float4 *)(kv + (uint64_t)row * row_stride); - const half4 k0 = (half4)kv4[lane + 0]; - const half4 k1 = (half4)kv4[lane + 32]; - const half4 k2 = (half4)kv4[lane + 64]; - const half4 k3 = (half4)kv4[lane + 96]; - - float score = dot((float4)q0, (float4)k0) + - dot((float4)q1, (float4)k1) + - dot((float4)q2, (float4)k2) + - dot((float4)q3, (float4)k3); - score = simd_sum(score) * scale; - - const float old_m = M; - const float new_m = max(M, score); - const float old_scale = exp(old_m - new_m); - const float row_scale = exp(score - new_m); - - S = S * old_scale + row_scale; - o0 *= old_scale; - o1 *= old_scale; - o2 *= old_scale; - o3 *= old_scale; - - o0 += (float4)k0 * row_scale; - o1 += (float4)k1 * row_scale; - o2 += (float4)k2 * row_scale; - o3 += (float4)k3 * row_scale; - M = new_m; -} - -static inline void dsv4_attend_shared_f32_row_as_f16( - threadgroup const float4 *kv4, - half4 q0, - half4 q1, - half4 q2, - half4 q3, - float scale, - ushort lane, - thread float &M, - thread float &S, - thread float4 &o0, - thread float4 &o1, - thread float4 &o2, - thread float4 &o3) { - const half4 k0 = (half4)kv4[lane + 0]; - const half4 k1 = (half4)kv4[lane + 32]; - const half4 k2 = (half4)kv4[lane + 64]; - const half4 k3 = (half4)kv4[lane + 96]; - - float score = dot((float4)q0, (float4)k0) + - dot((float4)q1, (float4)k1) + - dot((float4)q2, (float4)k2) + - dot((float4)q3, (float4)k3); - score = simd_sum(score) * scale; - - const float old_m = M; - const float new_m = max(M, score); - const float old_scale = exp(old_m - new_m); - const float row_scale = exp(score - new_m); - - S = S * old_scale + row_scale; - o0 *= old_scale; - o1 *= old_scale; - o2 *= old_scale; - o3 *= old_scale; - - o0 += (float4)k0 * row_scale; - o1 += (float4)k1 * row_scale; - o2 += (float4)k2 * row_scale; - o3 += (float4)k3 * row_scale; - M = new_m; -} - -static inline void dsv4_attend_shared_f32_row_as_f16_at( - threadgroup const float4 *kv4, - uint row_in_tg, - half4 q0, - half4 q1, - half4 q2, - half4 q3, - float scale, - ushort lane, - thread float &M, - thread float &S, - thread float4 &o0, - thread float4 &o1, - thread float4 &o2, - thread float4 &o3) { - dsv4_attend_shared_f32_row_as_f16(kv4 + row_in_tg * 128u, - q0, q1, q2, q3, - scale, - lane, - M, S, - o0, o1, o2, o3); -} - static inline void dsv4_attend_shared_h4_row( threadgroup const half4 *kv4, half4 q0, @@ -6179,26 +5906,6 @@ kernel void kernel_dsv4_indexed_mixed_attention_heads8_split_reduce( } } -static inline float dsv4_indexer_dot128_shared_q( - float4 c0, - float4 c1, - float4 c2, - float4 c3, - threadgroup const float4 *q4, - ushort lane) { - float sum = 0.0f; - if (lane < 8) { - const ushort ib = lane >> 1; - const ushort il = lane & 1; - const ushort base = ib*8 + il*4; - sum += dot(c0, q4[base + 0]); - sum += dot(c1, q4[base + 1]); - sum += dot(c2, q4[base + 2]); - sum += dot(c3, q4[base + 3]); - } - return simd_sum(sum); -} - // Tiled prefill score builder for the sparse-compressed attention indexer. // // The kernel covers an 8-token by 32-compressed-row rectangle: K is copied into diff --git a/metal/glu.metal b/metal/glu.metal index dde0c22fd..80af3117a 100644 --- a/metal/glu.metal +++ b/metal/glu.metal @@ -11,34 +11,6 @@ struct ds4_metal_args_glu { float limit; }; -// SwiGLU activation for the FFN inner state. DS4 clamps the shared expert with -// the same swiglu_limit used by routed experts. -kernel void kernel_swiglu_f32( - constant ds4_metal_args_glu & args, - device const char * src0, - device const char * src1, - device char * dst, - uint tgpig[[threadgroup_position_in_grid]], - uint tpitg[[thread_position_in_threadgroup]], - uint ntg[[threads_per_threadgroup]]) { - device const float * src0_row = (device const float *) ((device const char *) src0 + tgpig*args.nb01) + args.i00; - device const float * src1_row = (device const float *) ((device const char *) src1 + tgpig*args.nb11) + args.i10; - device float * dst_row = (device float *) ((device char *) dst + tgpig*args.nb1); - - for (int i0 = tpitg; i0 < args.ne0; i0 += ntg) { - float x0 = src0_row[i0]; - float x1 = src1_row[i0]; - if (args.limit > 1.0e-6f) { - x0 = min(x0, args.limit); - x1 = clamp(x1, -args.limit, args.limit); - } - - const float silu = ds4_silu(x0); - - dst_row[i0] = silu*x1*args.alpha; - } -} - kernel void kernel_swiglu_flat_f32( constant ds4_metal_args_glu & args, device const char * src0, diff --git a/metal/moe.metal b/metal/moe.metal index 6a36f272e..60db8bc5c 100644 --- a/metal/moe.metal +++ b/metal/moe.metal @@ -880,20 +880,6 @@ void dequantize_q2_K(device const block_q2_K *xb, short il, thread type4x4 & reg } } -static inline float ds4_glm_q2_K_value(device const block_q2_K *blocks, uint k) { - const uint block = k / QK_K; - const uint idx = k - block * QK_K; - device const block_q2_K *xb = blocks + block; - const uint group = idx / 16u; - const uint l = idx - group * 16u; - const uint q_base = 32u * (group / 8u) + 16u * (group & 1u); - const uint shift = ((group / 2u) & 3u) * 2u; - const uint q = ((uint)xb->qs[q_base + l] >> shift) & 0x03u; - const uint sc = (uint)xb->scales[group]; - return (float)xb->d * (float)(sc & 0x0fu) * (float)q - - (float)xb->dmin * (float)(sc >> 4u); -} - static inline uchar2 get_scale_min_k4_just2(int j, int k, device const uchar * q) { return j < 4 ? uchar2{uchar(q[j+0+k] & 63), uchar(q[j+4+k] & 63)} : uchar2{uchar((q[j+4+k] & 0xF) | ((q[j-4+k] & 0xc0) >> 2)), @@ -914,115 +900,6 @@ static inline float ds4_glm_q4_K_value(device const block_q4_K *blocks, uint k) (float)xb->dmin * (float)sm.y; } -static inline float ds4_glm_q5_K_value(device const block_q5_K *blocks, uint k) { - const uint block = k / QK_K; - const uint idx = k - block * QK_K; - device const block_q5_K *xb = blocks + block; - const uint group = idx / 32u; - const uint l = idx - group * 32u; - const uchar2 sm = get_scale_min_k4_just2((int)group, 0, xb->scales); - const uint ql_base = (group >> 1u) * 32u + l; - const uint shift = (group & 1u) * 4u; - uint q = (xb->qs[ql_base] >> shift) & 0x0Fu; - q += (xb->qh[l] & (uchar)(1u << group)) ? 16u : 0u; - return (float)xb->d * (float)sm.x * (float)q - - (float)xb->dmin * (float)sm.y; -} - -static inline float ds4_glm_q6_K_value(device const block_q6_K *blocks, uint k) { - const uint block = k / QK_K; - const uint idx = k - block * QK_K; - device const block_q6_K *xb = blocks + block; - const uint n128 = idx >> 7u; - const uint r = idx & 127u; - const uint l = r & 31u; - const uint quarter = r >> 5u; - const uint ql_base = n128 * 64u; - const uint qh_base = n128 * 32u; - const uint sc_base = n128 * 8u; - uint q; - int sc; - - if (quarter == 0u) { - q = (xb->ql[ql_base + l] & 0x0Fu) | (((xb->qh[qh_base + l] >> 0u) & 3u) << 4u); - sc = (int)xb->scales[sc_base + l / 16u + 0u]; - } else if (quarter == 1u) { - q = (xb->ql[ql_base + 32u + l] & 0x0Fu) | (((xb->qh[qh_base + l] >> 2u) & 3u) << 4u); - sc = (int)xb->scales[sc_base + l / 16u + 2u]; - } else if (quarter == 2u) { - q = (xb->ql[ql_base + l] >> 4u) | (((xb->qh[qh_base + l] >> 4u) & 3u) << 4u); - sc = (int)xb->scales[sc_base + l / 16u + 4u]; - } else { - q = (xb->ql[ql_base + 32u + l] >> 4u) | (((xb->qh[qh_base + l] >> 6u) & 3u) << 4u); - sc = (int)xb->scales[sc_base + l / 16u + 6u]; - } - - return (float)xb->d * (float)sc * (float)((int)q - 32); -} - -kernel void kernel_glm_q4_K_pair_swiglu_f32( - constant ds4_metal_glm_routed_moe_args &args, - device const char *gate, - device const char *up, - device const float *x, - device const int32_t *selected, - device const float *weights, - device float *mid, - threadgroup float *scratch [[threadgroup(0)]], - uint3 tgpig [[threadgroup_position_in_grid]], - uint tid [[thread_index_in_threadgroup]]) { - const uint ntg = 256u; - const uint row = tgpig.x; - const uint slot = tgpig.y; - const uint token = tgpig.z; - if (row >= args.mid_dim || slot >= args.n_expert_used || token >= args.n_tokens) return; - - const uint64_t selected_off = (uint64_t)token * args.n_expert_used + slot; - const uint64_t mid_off = (uint64_t)token * args.mid_token_stride + - (uint64_t)slot * args.mid_dim + row; - const int expert = selected[selected_off]; - if (!ds4_tp_owns_expert(expert, args.n_total_expert, - args.tp_rank, args.tp_world)) { - if (tid == 0u) mid[mid_off] = 0.0f; - return; - } - const int local_expert = expert - args.tp_expert_base; - - device const block_q4_K *gate_row = - (device const block_q4_K *)(gate + - (uint64_t)(uint)local_expert * args.gate_expert_bytes + - (uint64_t)row * args.gate_row_bytes); - device const block_q4_K *up_row = - (device const block_q4_K *)(up + - (uint64_t)(uint)local_expert * args.up_expert_bytes + - (uint64_t)row * args.up_row_bytes); - - float acc_gate = 0.0f; - float acc_up = 0.0f; - device const float *token_x = x + (uint64_t)token * args.in_dim; - for (uint k = tid; k < args.in_dim; k += ntg) { - const float xv = token_x[k]; - acc_gate += ds4_glm_q4_K_value(gate_row, k) * xv; - acc_up += ds4_glm_q4_K_value(up_row, k) * xv; - } - - scratch[tid] = acc_gate; - scratch[ntg + tid] = acc_up; - threadgroup_barrier(mem_flags::mem_threadgroup); - for (uint stride = ntg >> 1u; stride > 0u; stride >>= 1u) { - if (tid < stride) { - scratch[tid] += scratch[tid + stride]; - scratch[ntg + tid] += scratch[ntg + tid + stride]; - } - threadgroup_barrier(mem_flags::mem_threadgroup); - } - - if (tid == 0u) { - mid[mid_off] = ds4_glm_swiglu(scratch[0], scratch[ntg], - args.swiglu_clamp) * weights[selected_off]; - } -} - template static inline void glm_q2_K_pair_swiglu_simd_f32_impl( ds4_metal_glm_routed_moe_args args, @@ -1646,71 +1523,6 @@ kernel void kernel_glm_q4_K_pair_swiglu4_f32( expert - args.tp_expert_base, tiisg, sgitg); } -kernel void kernel_glm_q4_K_pair_swiglu2_mapped_f32( - constant ds4_metal_glm_routed_moe_args &args, - device const char *gate, - device const char *up, - device const float *x, - device const uint32_t *htpe, - device const int32_t *hids, - device const float *weights, - device float *mid, - threadgroup float *scratch [[threadgroup(0)]], - uint3 tgpig [[threadgroup_position_in_grid]], - ushort tiisg [[thread_index_in_simdgroup]], - ushort sgitg [[simdgroup_index_in_threadgroup]]) { - const uint expert = tgpig.z; - if (expert >= args.n_total_expert) return; - if (!ds4_tp_owns_expert((int)expert, args.n_total_expert, - args.tp_rank, args.tp_world)) return; - const uint count = htpe[expert]; - const uint map_base = tgpig.y * 32u; - for (uint i = 0; i < 32u; i++) { - const uint map_row = map_base + i; - if (map_row >= count) break; - const int id = hids[(uint64_t)expert * args.n_tokens + map_row]; - if (id < 0) continue; - const uint token = (uint)id / args.n_expert_used; - const uint slot = (uint)id - token * args.n_expert_used; - if (slot >= args.n_expert_used || token >= args.n_tokens) continue; - const uint64_t selected_off = (uint64_t)token * args.n_expert_used + slot; - glm_q4_K_pair_swiglu_simd_f32_impl( - args, gate, up, x, weights, mid, scratch, - tgpig, slot, token, selected_off, - (int)expert - args.tp_expert_base, tiisg, sgitg); - } -} - -kernel void kernel_glm_q4_K_pair_swiglu2_mapped_row_f32( - constant ds4_metal_glm_routed_moe_args &args, - device const char *gate, - device const char *up, - device const float *x, - device const uint32_t *htpe, - device const int32_t *hids, - device const float *weights, - device float *mid, - threadgroup float *scratch [[threadgroup(0)]], - uint3 tgpig [[threadgroup_position_in_grid]], - ushort tiisg [[thread_index_in_simdgroup]], - ushort sgitg [[simdgroup_index_in_threadgroup]]) { - const uint expert = tgpig.z; - const uint map_row = tgpig.y; - if (expert >= args.n_total_expert || map_row >= htpe[expert]) return; - if (!ds4_tp_owns_expert((int)expert, args.n_total_expert, - args.tp_rank, args.tp_world)) return; - const int id = hids[(uint64_t)expert * args.n_tokens + map_row]; - if (id < 0) return; - const uint token = (uint)id / args.n_expert_used; - const uint slot = (uint)id - token * args.n_expert_used; - if (slot >= args.n_expert_used || token >= args.n_tokens) return; - const uint64_t selected_off = (uint64_t)token * args.n_expert_used + slot; - glm_q4_K_pair_swiglu_simd_f32_impl( - args, gate, up, x, weights, mid, scratch, - tgpig, slot, token, selected_off, - (int)expert - args.tp_expert_base, tiisg, sgitg); -} - static inline void glm_q5_K_pair_swiglu_f32_impl( constant ds4_metal_glm_routed_moe_args &args, device const char *gate, @@ -1910,65 +1722,6 @@ kernel void kernel_glm_q5_K_pair_swiglu_f32( tgpig, slot, token, selected_off, expert, tiisg, sgitg); } -kernel void kernel_glm_q5_K_pair_swiglu_mapped_f32( - constant ds4_metal_glm_routed_moe_args &args, - device const char *gate, - device const char *up, - device const float *x, - device const uint32_t *htpe, - device const int32_t *hids, - device const float *weights, - device float *mid, - threadgroup float *scratch [[threadgroup(0)]], - uint3 tgpig [[threadgroup_position_in_grid]], - ushort tiisg [[thread_index_in_simdgroup]], - ushort sgitg [[simdgroup_index_in_threadgroup]]) { - const uint expert = tgpig.z; - if (expert >= args.n_total_expert) return; - const uint count = htpe[expert]; - const uint map_base = tgpig.y * 32u; - for (uint i = 0; i < 32u; i++) { - const uint map_row = map_base + i; - if (map_row >= count) break; - const int id = hids[(uint64_t)expert * args.n_tokens + map_row]; - if (id < 0) continue; - const uint token = (uint)id / args.n_expert_used; - const uint slot = (uint)id - token * args.n_expert_used; - if (slot >= args.n_expert_used || token >= args.n_tokens) continue; - const uint64_t selected_off = (uint64_t)token * args.n_expert_used + slot; - glm_q5_K_pair_swiglu_f32_impl( - args, gate, up, x, weights, mid, scratch, - tgpig, slot, token, selected_off, (int)expert, tiisg, sgitg); - } -} - -kernel void kernel_glm_q5_K_pair_swiglu_mapped_row_f32( - constant ds4_metal_glm_routed_moe_args &args, - device const char *gate, - device const char *up, - device const float *x, - device const uint32_t *htpe, - device const int32_t *hids, - device const float *weights, - device float *mid, - threadgroup float *scratch [[threadgroup(0)]], - uint3 tgpig [[threadgroup_position_in_grid]], - ushort tiisg [[thread_index_in_simdgroup]], - ushort sgitg [[simdgroup_index_in_threadgroup]]) { - const uint expert = tgpig.z; - const uint map_row = tgpig.y; - if (expert >= args.n_total_expert || map_row >= htpe[expert]) return; - const int id = hids[(uint64_t)expert * args.n_tokens + map_row]; - if (id < 0) return; - const uint token = (uint)id / args.n_expert_used; - const uint slot = (uint)id - token * args.n_expert_used; - if (slot >= args.n_expert_used || token >= args.n_tokens) return; - const uint64_t selected_off = (uint64_t)token * args.n_expert_used + slot; - glm_q5_K_pair_swiglu_f32_impl( - args, gate, up, x, weights, mid, scratch, - tgpig, slot, token, selected_off, (int)expert, tiisg, sgitg); -} - kernel void kernel_glm_q5_K_down_f32( constant ds4_metal_glm_routed_moe_args &args, device const char *down, @@ -2914,10 +2667,6 @@ struct ds4_metal_q4_expert_table { array experts [[id(0)]]; }; -struct ds4_metal_expert_address_table { - device const uint64_t *addrs; -}; - struct ds4_metal_stream_expert_validate_args { uint32_t n_total_expert; uint32_t n_expert; diff --git a/metal/sum_rows.metal b/metal/sum_rows.metal index 8efea9b39..c77a0b509 100644 --- a/metal/sum_rows.metal +++ b/metal/sum_rows.metal @@ -2,7 +2,6 @@ #define FC_SUM_ROWS 1400 -#define OP_SUM_ROWS_NUM_SUM_ROWS 10 #define OP_SUM_ROWS_NUM_MEAN 11 struct ds4_metal_args_sum_rows { diff --git a/rocm/ds4_rocm_attention.cuh b/rocm/ds4_rocm_attention.cuh index 75b23b59a..53025aef8 100644 --- a/rocm/ds4_rocm_attention.cuh +++ b/rocm/ds4_rocm_attention.cuh @@ -1018,92 +1018,6 @@ __global__ static void attention_decode_mixed_kernel( } } -__global__ static void attention_indexed_mixed_scalar_kernel( - float *heads, - const float *sinks, - const float *q, - const float *raw_kv, - const float *comp_kv, - const int32_t *topk, - uint32_t n_tokens, - uint32_t pos0, - uint32_t n_raw, - uint32_t raw_cap, - uint32_t raw_start, - uint32_t n_comp, - uint32_t top_k, - uint32_t window, - uint32_t ratio, - uint32_t n_head, - uint32_t head_dim) { - const uint64_t idx = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; - const uint64_t total = (uint64_t)n_tokens * n_head * head_dim; - if (idx >= total) return; - const uint32_t d = (uint32_t)(idx % head_dim); - const uint64_t th = idx / head_dim; - const uint32_t h = (uint32_t)(th % n_head); - const uint32_t t = (uint32_t)(th / n_head); - const uint32_t qpos = pos0 + t; - const uint32_t last_pos = pos0 + n_tokens - 1u; - const uint32_t first_raw_pos = last_pos + 1u - n_raw; - const float *qh = q + ((uint64_t)t * n_head + h) * head_dim; - const float scale = rsqrtf((float)head_dim); - const uint32_t visible = ratio ? (qpos + 1u) / ratio : n_comp; - float max_score = sinks[h]; - - for (uint32_t r = 0; r < n_raw; r++) { - const uint32_t kpos = first_raw_pos + r; - if (kpos > qpos) continue; - if (window != 0 && qpos - kpos >= window) continue; - const uint32_t row = raw_cap ? ((raw_start + r) % raw_cap) : r; - const float *kv = raw_kv + (uint64_t)row * head_dim; - float s = 0.0f; - for (uint32_t i = 0; i < head_dim; i++) s += qh[i] * kv[i]; - s *= scale; - if (s > max_score) max_score = s; - } - for (uint32_t u = 0; u < top_k; u++) { - const int32_t ci = topk[(uint64_t)t * top_k + u]; - if (ci < 0) continue; - const uint32_t c = (uint32_t)ci; - if (c >= n_comp || c >= visible) continue; - const float *kv = comp_kv + (uint64_t)c * head_dim; - float s = 0.0f; - for (uint32_t i = 0; i < head_dim; i++) s += qh[i] * kv[i]; - s *= scale; - if (s > max_score) max_score = s; - } - - float denom = expf(sinks[h] - max_score); - float acc = 0.0f; - for (uint32_t r = 0; r < n_raw; r++) { - const uint32_t kpos = first_raw_pos + r; - if (kpos > qpos) continue; - if (window != 0 && qpos - kpos >= window) continue; - const uint32_t row = raw_cap ? ((raw_start + r) % raw_cap) : r; - const float *kv = raw_kv + (uint64_t)row * head_dim; - float s = 0.0f; - for (uint32_t i = 0; i < head_dim; i++) s += qh[i] * kv[i]; - const float w = expf(s * scale - max_score); - denom += w; - acc += w * kv[d]; - } - for (uint32_t u = 0; u < top_k; u++) { - const int32_t ci = topk[(uint64_t)t * top_k + u]; - if (ci < 0) continue; - const uint32_t c = (uint32_t)ci; - if (c >= n_comp || c >= visible) continue; - const float *kv = comp_kv + (uint64_t)c * head_dim; - float s = 0.0f; - for (uint32_t i = 0; i < head_dim; i++) s += qh[i] * kv[i]; - s *= scale; - const float w = expf(s - max_score); - denom += w; - acc += w * kv[d]; - } - heads[idx] = acc / denom; -} - __global__ static void attention_indexed_mixed_kernel( float *heads, const float *sinks, diff --git a/rocm/ds4_rocm_glm.cuh b/rocm/ds4_rocm_glm.cuh index 948af9d7a..4741ea977 100644 --- a/rocm/ds4_rocm_glm.cuh +++ b/rocm/ds4_rocm_glm.cuh @@ -1103,31 +1103,6 @@ __global__ static void glm_store_indexer_k_kernel( } } -__global__ static void glm_k_b_project_q8_0_kernel( - float *out, - const unsigned char *weight, - const float *kv_norm, - uint32_t n_tokens, - uint32_t kv_lora_dim, - uint32_t qk_nope, - uint32_t n_head, - uint32_t row_bytes) { - const uint32_t token = blockIdx.x; - const uint32_t head = blockIdx.y; - const uint32_t q = threadIdx.x + blockIdx.z * blockDim.x; - if (token >= n_tokens || head >= n_head || q >= qk_nope) return; - const float *kv = kv_norm + (uint64_t)token * kv_lora_dim; - float acc = 0.0f; - const uint32_t b = q >> 5u; - const uint32_t j_in_block = q & 31u; - for (uint32_t j = 0; j < kv_lora_dim; j++) { - const unsigned char *row = weight + ((uint64_t)head * kv_lora_dim + j) * row_bytes; - const unsigned char *blk = row + (uint64_t)b * 34u; - acc += q8_0_scale_scalar(blk) * (float)((const int8_t *)(blk + 2u))[j_in_block] * kv[j]; - } - out[((uint64_t)token * n_head + head) * qk_nope + q] = acc; -} - __global__ static void glm_k_b_project_q8_0_head_kernel( float *out, const unsigned char *weight, @@ -1162,46 +1137,6 @@ __global__ static void glm_k_b_project_q8_0_head_kernel( } } -__global__ static void glm_q8_project_rows_kernel( - float *out, - const unsigned char *weight, - const float *x, - uint32_t n_tokens, - uint32_t n_head, - uint32_t in_dim, - uint32_t out_dim, - uint32_t x_stride, - uint32_t x_head_stride, - uint32_t row_bytes) { - const uint32_t out_row = blockIdx.x; - const uint32_t token = blockIdx.y; - if (token >= n_tokens || out_row >= n_head * out_dim) return; - const uint32_t head = out_row / out_dim; - const uint32_t d = out_row - head * out_dim; - const float *xr = x + (uint64_t)token * x_stride + (uint64_t)head * x_head_stride; - const unsigned char *row = weight + ((uint64_t)head * out_dim + d) * row_bytes; - float acc = 0.0f; - const uint32_t blocks = (in_dim + 31u) >> 5u; - for (uint32_t b = threadIdx.x; b < blocks; b += blockDim.x) { - const uint32_t base = b << 5u; - const uint32_t count = min(32u, in_dim - base); - const unsigned char *blk = row + (uint64_t)b * 34u; - const float scale = q8_0_scale_scalar(blk); - const int8_t *qs = (const int8_t *)(blk + 2u); - for (uint32_t i = 0; i < count; i++) acc += scale * (float)qs[i] * xr[base + i]; - } - __shared__ float partial[256]; - partial[threadIdx.x] = acc; - __syncthreads(); - for (uint32_t stride = blockDim.x >> 1u; stride > 0u; stride >>= 1u) { - if (threadIdx.x < stride) partial[threadIdx.x] += partial[threadIdx.x + stride]; - __syncthreads(); - } - if (threadIdx.x == 0u) { - out[((uint64_t)token * n_head + head) * out_dim + d] = partial[0]; - } -} - __global__ static void glm_q8_project_head_kernel( float *out, const unsigned char *weight, diff --git a/rocm/ds4_rocm_hipblaslt.cuh b/rocm/ds4_rocm_hipblaslt.cuh deleted file mode 100644 index ce5c15a86..000000000 --- a/rocm/ds4_rocm_hipblaslt.cuh +++ /dev/null @@ -1,137 +0,0 @@ -/* HIP-only hipBLASLt state and helpers. - * Included from ds4_cuda.cu under __HIP_PLATFORM_AMD__ to keep ROCm - * planning/cache code out of the CUDA host runtime body. */ - -static hipblasLtHandle_t g_hipblaslt; -static int g_hipblaslt_ready; -struct cuda_hipblaslt_gemm_plan { - uint32_t out_dim; - uint32_t n_tok; - uint32_t in_dim; - hipblasLtMatmulDesc_t desc; - hipblasLtMatrixLayout_t a_desc; - hipblasLtMatrixLayout_t b_desc; - hipblasLtMatrixLayout_t c_desc; - hipblasLtMatrixLayout_t d_desc; - hipblasLtMatmulAlgo_t algo; -}; -static std::vector g_hipblaslt_gemm_plans; - -static void hipblaslt_gemm_plan_clear(void) { - for (size_t i = 0; i < g_hipblaslt_gemm_plans.size(); i++) { - cuda_hipblaslt_gemm_plan &p = g_hipblaslt_gemm_plans[i]; - if (p.d_desc) (void)hipblasLtMatrixLayoutDestroy(p.d_desc); - if (p.c_desc) (void)hipblasLtMatrixLayoutDestroy(p.c_desc); - if (p.b_desc) (void)hipblasLtMatrixLayoutDestroy(p.b_desc); - if (p.a_desc) (void)hipblasLtMatrixLayoutDestroy(p.a_desc); - if (p.desc) (void)hipblasLtMatmulDescDestroy(p.desc); - } - g_hipblaslt_gemm_plans.clear(); -} - -static int hipblaslt_ok(hipblasStatus_t st, const char *what) { - if (st == HIPBLAS_STATUS_SUCCESS) return 1; - fprintf(stderr, "ds4: hipBLASLt %s failed: status %d\n", what, (int)st); - return 0; -} - -static cuda_hipblaslt_gemm_plan *hipblaslt_gemm_plan_get( - uint32_t out_dim, - uint32_t n_tok, - uint32_t in_dim, - const char *label) { - for (size_t i = 0; i < g_hipblaslt_gemm_plans.size(); i++) { - cuda_hipblaslt_gemm_plan &p = g_hipblaslt_gemm_plans[i]; - if (p.out_dim == out_dim && p.n_tok == n_tok && p.in_dim == in_dim) return &p; - } - - hipblasLtMatmulDesc_t desc = NULL; - hipblasLtMatrixLayout_t a_desc = NULL, b_desc = NULL, c_desc = NULL, d_desc = NULL; - hipblasLtMatmulPreference_t pref = NULL; - hipblasLtMatmulHeuristicResult_t heur[8]; - int returned = 0; - int ok = 0; - do { - if (!hipblaslt_ok(hipblasLtMatmulDescCreate(&desc, HIPBLAS_COMPUTE_32F, HIP_R_32F), - "matmul desc create")) break; - hipblasOperation_t op_a = HIPBLAS_OP_T; - hipblasOperation_t op_b = HIPBLAS_OP_N; - if (!hipblaslt_ok(hipblasLtMatmulDescSetAttribute(desc, HIPBLASLT_MATMUL_DESC_TRANSA, - &op_a, sizeof(op_a)), - "set transA")) break; - if (!hipblaslt_ok(hipblasLtMatmulDescSetAttribute(desc, HIPBLASLT_MATMUL_DESC_TRANSB, - &op_b, sizeof(op_b)), - "set transB")) break; - if (!hipblaslt_ok(hipblasLtMatrixLayoutCreate(&a_desc, HIP_R_16F, in_dim, out_dim, in_dim), - "A layout create")) break; - if (!hipblaslt_ok(hipblasLtMatrixLayoutCreate(&b_desc, HIP_R_16F, in_dim, n_tok, in_dim), - "B layout create")) break; - if (!hipblaslt_ok(hipblasLtMatrixLayoutCreate(&c_desc, HIP_R_16F, out_dim, n_tok, out_dim), - "C layout create")) break; - if (!hipblaslt_ok(hipblasLtMatrixLayoutCreate(&d_desc, HIP_R_16F, out_dim, n_tok, out_dim), - "D layout create")) break; - if (!hipblaslt_ok(hipblasLtMatmulPreferenceCreate(&pref), "preference create")) break; - const size_t max_workspace = 0; - if (!hipblaslt_ok(hipblasLtMatmulPreferenceSetAttribute( - pref, HIPBLASLT_MATMUL_PREF_MAX_WORKSPACE_BYTES, - &max_workspace, sizeof(max_workspace)), - "set max workspace")) break; - if (!hipblaslt_ok(hipblasLtMatmulAlgoGetHeuristic(g_hipblaslt, desc, - a_desc, b_desc, c_desc, d_desc, - pref, 8, heur, &returned), - "algo heuristic")) break; - if (returned <= 0 || heur[0].state != HIPBLAS_STATUS_SUCCESS) { - fprintf(stderr, "ds4: hipBLASLt no algo for %s m=%u n=%u k=%u\n", - label ? label : "gemm", out_dim, n_tok, in_dim); - break; - } - ok = 1; - } while (0); - if (pref) (void)hipblasLtMatmulPreferenceDestroy(pref); - if (!ok) { - if (d_desc) (void)hipblasLtMatrixLayoutDestroy(d_desc); - if (c_desc) (void)hipblasLtMatrixLayoutDestroy(c_desc); - if (b_desc) (void)hipblasLtMatrixLayoutDestroy(b_desc); - if (a_desc) (void)hipblasLtMatrixLayoutDestroy(a_desc); - if (desc) (void)hipblasLtMatmulDescDestroy(desc); - return NULL; - } - - cuda_hipblaslt_gemm_plan p; - p.out_dim = out_dim; - p.n_tok = n_tok; - p.in_dim = in_dim; - p.desc = desc; - p.a_desc = a_desc; - p.b_desc = b_desc; - p.c_desc = c_desc; - p.d_desc = d_desc; - p.algo = heur[0].algo; - g_hipblaslt_gemm_plans.push_back(p); - return &g_hipblaslt_gemm_plans.back(); -} - -static int hipblaslt_gemm_tn_f16_out_f16( - __half *out, - const __half *w_rowmajor_out_in, - const __half *x_rowmajor_tok_in, - uint32_t out_dim, - uint32_t n_tok, - uint32_t in_dim, - const char *label) { - if (!g_hipblaslt_ready || !out || !w_rowmajor_out_in || !x_rowmajor_tok_in || - out_dim == 0 || n_tok == 0 || in_dim == 0) return 0; - cuda_hipblaslt_gemm_plan *p = hipblaslt_gemm_plan_get(out_dim, n_tok, in_dim, label); - if (!p) return 0; - const float alpha = 1.0f; - const float beta = 0.0f; - return hipblaslt_ok(hipblasLtMatmul(g_hipblaslt, p->desc, &alpha, - w_rowmajor_out_in, p->a_desc, - x_rowmajor_tok_in, p->b_desc, - &beta, - out, p->c_desc, - out, p->d_desc, - &p->algo, - NULL, 0, 0), - label ? label : "gemm"); -} diff --git a/rocm/ds4_rocm_moe.cuh b/rocm/ds4_rocm_moe.cuh index 025d3c676..bc46a70f3 100644 --- a/rocm/ds4_rocm_moe.cuh +++ b/rocm/ds4_rocm_moe.cuh @@ -228,45 +228,6 @@ __device__ static void dev_dot_iq2_xxs_q8_K_block4( for (uint32_t p = 0; p < n; p++) acc[p] += 0.125f * xd * ys[p]->d * (float)bsum[p]; } -__device__ static DS4_ROCM_UNUSED void dev_dot_iq2_xxs_q8_K_block8( - const cuda_block_iq2_xxs *x, - const cuda_block_q8_K *y0, - const cuda_block_q8_K *y1, - const cuda_block_q8_K *y2, - const cuda_block_q8_K *y3, - const cuda_block_q8_K *y4, - const cuda_block_q8_K *y5, - const cuda_block_q8_K *y6, - const cuda_block_q8_K *y7, - uint32_t n, - float acc[8]) { - const float xd = dev_f16_to_f32(x->d); - const uint16_t *q2 = x->qs; - int32_t bsum[8] = {0, 0, 0, 0, 0, 0, 0, 0}; - const int8_t *q8[8] = { - y0 ? y0->qs : NULL, y1 ? y1->qs : NULL, y2 ? y2->qs : NULL, y3 ? y3->qs : NULL, - y4 ? y4->qs : NULL, y5 ? y5->qs : NULL, y6 ? y6->qs : NULL, y7 ? y7->qs : NULL, - }; - for (int ib32 = 0; ib32 < CUDA_QK_K / 32; ib32++) { - const uint32_t aux0 = (uint32_t)q2[0] | ((uint32_t)q2[1] << 16); - const uint32_t aux1 = (uint32_t)q2[2] | ((uint32_t)q2[3] << 16); - q2 += 4; - const uint32_t ls = 2u * (aux1 >> 28) + 1u; - const uint8_t a0 = (uint8_t)(aux0 & 0xffu); - const uint8_t a1 = (uint8_t)((aux0 >> 8) & 0xffu); - const uint8_t a2 = (uint8_t)((aux0 >> 16) & 0xffu); - const uint8_t a3 = (uint8_t)((aux0 >> 24) & 0xffu); - for (uint32_t p = 0; p < n; p++) { - int32_t sumi = 0; - sumi += dev_dot_iq2_pair_16(a0, (aux1 >> 0) & 127u, a1, (aux1 >> 7) & 127u, q8[p] + ib32 * 32); - sumi += dev_dot_iq2_pair_16(a2, (aux1 >> 14) & 127u, a3, (aux1 >> 21) & 127u, q8[p] + ib32 * 32 + 16); - bsum[p] += sumi * (int32_t)ls; - } - } - const cuda_block_q8_K *ys[8] = { y0, y1, y2, y3, y4, y5, y6, y7 }; - for (uint32_t p = 0; p < n; p++) acc[p] += 0.125f * xd * ys[p]->d * (float)bsum[p]; -} - __device__ static void dev_q4_K_get_scale_min( uint32_t j, const uint8_t *scales, @@ -354,30 +315,6 @@ __device__ __forceinline__ static void dev_mxfp4_unpack2x4( *high = (int32_t)dev_mxfp4_unpack4(packed >> 4u); } -/* One q8_K chunk covers eight consecutive 32-value MXFP4 blocks. MXFP4 - * stores the first 16 values in the low nibbles and the second 16 in the - * high nibbles, rather than interleaving them. */ -__device__ static float dev_dot_mxfp4_q8_K_block( - const cuda_block_mxfp4 *x8, - const cuda_block_q8_K *y) { - float chunk = 0.0f; - #pragma unroll - for (uint32_t sb = 0; sb < 8u; sb++) { - const cuda_block_mxfp4 *x = x8 + sb; - const int8_t *q8 = y->qs + sb * 32u; - int32_t bsum = 0; - #pragma unroll - for (uint32_t j = 0; j < 16u; j += 4u) { - int32_t wlo, whi; - dev_mxfp4_unpack2x4(x->qs + j, &wlo, &whi); - bsum = __dp4a(wlo, *(const int32_t *)(q8 + j), bsum); - bsum = __dp4a(whi, *(const int32_t *)(q8 + 16u + j), bsum); - } - chunk += dev_e8m0_to_f32(x->e) * (float)bsum; - } - return 0.5f * y->d * chunk; -} - /* Split one 32-value MXFP4 block across a pair of lanes. A wave therefore * reads 16 consecutive 17-byte blocks instead of having each quarter-wave * lane jump by a full 136-byte Q8_K chunk. This mirrors the coalesced Metal @@ -953,162 +890,6 @@ __global__ static void q8_K_quantize_wave32_kernel( if (lane == 0u) yb->d = 1.0f / iscale; } -__global__ static DS4_ROCM_UNUSED void moe_gate_up_mid_kernel( - float *gate_out, - float *up_out, - float *mid_out, - const char *gate_base, - const char *up_base, - const cuda_block_q8_K *xq, - const int32_t *selected, - const float *weights, - uint64_t gate_expert_bytes, - uint64_t gate_row_bytes, - uint32_t xq_blocks, - uint32_t expert_mid_dim, - uint32_t n_expert, - float clamp) { - uint32_t row = blockIdx.x; - uint32_t pair = blockIdx.y; - if (row >= expert_mid_dim) return; - uint32_t tok = pair / n_expert; - uint32_t slot = pair - tok * n_expert; - int32_t expert_i = selected[(uint64_t)tok * n_expert + slot]; - if (expert_i < 0) expert_i = 0; - uint32_t expert = (uint32_t)expert_i; - const cuda_block_iq2_xxs *gr = (const cuda_block_iq2_xxs *)(gate_base + (uint64_t)expert * gate_expert_bytes + (uint64_t)row * gate_row_bytes); - const cuda_block_iq2_xxs *ur = (const cuda_block_iq2_xxs *)(up_base + (uint64_t)expert * gate_expert_bytes + (uint64_t)row * gate_row_bytes); - const cuda_block_q8_K *xqb = xq + (uint64_t)tok * xq_blocks; - float gate = 0.0f; - float up = 0.0f; - for (uint32_t b = threadIdx.x; b < xq_blocks; b += blockDim.x) { - gate += dev_dot_iq2_xxs_q8_K_block(gr + b, xqb + b); - up += dev_dot_iq2_xxs_q8_K_block(ur + b, xqb + b); - } - __shared__ float partial_gate[256]; - __shared__ float partial_up[256]; - partial_gate[threadIdx.x] = gate; - partial_up[threadIdx.x] = up; - __syncthreads(); - for (uint32_t stride = blockDim.x >> 1; stride > 0; stride >>= 1) { - if (threadIdx.x < stride) { - partial_gate[threadIdx.x] += partial_gate[threadIdx.x + stride]; - partial_up[threadIdx.x] += partial_up[threadIdx.x + stride]; - } - __syncthreads(); - } - if (threadIdx.x == 0) { - gate = partial_gate[0]; - up = partial_up[0]; - if (clamp > 1.0e-6f) { - if (gate > clamp) gate = clamp; - if (up > clamp) up = clamp; - if (up < -clamp) up = -clamp; - } - const uint64_t off = (uint64_t)pair * expert_mid_dim + row; - gate_out[off] = gate; - up_out[off] = up; - mid_out[off] = (gate / (1.0f + expf(-gate))) * up * weights[(uint64_t)tok * n_expert + slot]; - } -} - -__global__ static DS4_ROCM_UNUSED void moe_gate_up_mid_warp8_kernel( - float *gate_out, - float *up_out, - float *mid_out, - const char *gate_base, - const char *up_base, - const cuda_block_q8_K *xq, - const int32_t *selected, - const float *weights, - uint64_t gate_expert_bytes, - uint64_t gate_row_bytes, - uint32_t xq_blocks, - uint32_t expert_mid_dim, - uint32_t n_expert, - float clamp) { - uint32_t lane = threadIdx.x & 31u; - uint32_t warp = threadIdx.x >> 5u; - uint32_t row = blockIdx.x * 8u + warp; - uint32_t pair = blockIdx.y; - if (row >= expert_mid_dim) return; - uint32_t tok = pair / n_expert; - uint32_t slot = pair - tok * n_expert; - int32_t expert_i = selected[(uint64_t)tok * n_expert + slot]; - if (expert_i < 0) expert_i = 0; - uint32_t expert = (uint32_t)expert_i; - const cuda_block_iq2_xxs *gr = (const cuda_block_iq2_xxs *)(gate_base + (uint64_t)expert * gate_expert_bytes + (uint64_t)row * gate_row_bytes); - const cuda_block_iq2_xxs *ur = (const cuda_block_iq2_xxs *)(up_base + (uint64_t)expert * gate_expert_bytes + (uint64_t)row * gate_row_bytes); - const cuda_block_q8_K *xqb = xq + (uint64_t)tok * xq_blocks; - float gate = 0.0f; - float up = 0.0f; - for (uint32_t b = lane; b < xq_blocks; b += 32u) { - gate += dev_dot_iq2_xxs_q8_K_block(gr + b, xqb + b); - up += dev_dot_iq2_xxs_q8_K_block(ur + b, xqb + b); - } - gate = warp_sum_f32(gate); - up = warp_sum_f32(up); - if (lane == 0) { - if (clamp > 1.0e-6f) { - if (gate > clamp) gate = clamp; - if (up > clamp) up = clamp; - if (up < -clamp) up = -clamp; - } - const uint64_t off = (uint64_t)pair * expert_mid_dim + row; - gate_out[off] = gate; - up_out[off] = up; - mid_out[off] = (gate / (1.0f + expf(-gate))) * up * weights[(uint64_t)tok * n_expert + slot]; - } -} - -__global__ static DS4_ROCM_UNUSED void moe_gate_up_mid_hwarp16_kernel( - float *gate_out, - float *up_out, - float *mid_out, - const char *gate_base, - const char *up_base, - const cuda_block_q8_K *xq, - const int32_t *selected, - const float *weights, - uint64_t gate_expert_bytes, - uint64_t gate_row_bytes, - uint32_t xq_blocks, - uint32_t expert_mid_dim, - uint32_t n_expert, - float clamp) { - uint32_t lane = threadIdx.x & 15u; - uint32_t row = blockIdx.x * 16u + (threadIdx.x >> 4u); - uint32_t pair = blockIdx.y; - if (row >= expert_mid_dim) return; - uint32_t tok = pair / n_expert; - uint32_t slot = pair - tok * n_expert; - int32_t expert_i = selected[(uint64_t)tok * n_expert + slot]; - if (expert_i < 0) expert_i = 0; - uint32_t expert = (uint32_t)expert_i; - const cuda_block_iq2_xxs *gr = (const cuda_block_iq2_xxs *)(gate_base + (uint64_t)expert * gate_expert_bytes + (uint64_t)row * gate_row_bytes); - const cuda_block_iq2_xxs *ur = (const cuda_block_iq2_xxs *)(up_base + (uint64_t)expert * gate_expert_bytes + (uint64_t)row * gate_row_bytes); - const cuda_block_q8_K *xqb = xq + (uint64_t)tok * xq_blocks; - float gate = 0.0f; - float up = 0.0f; - for (uint32_t b = lane; b < xq_blocks; b += 16u) { - gate += dev_dot_iq2_xxs_q8_K_block(gr + b, xqb + b); - up += dev_dot_iq2_xxs_q8_K_block(ur + b, xqb + b); - } - gate = half_warp_sum_f32(gate, lane); - up = half_warp_sum_f32(up, lane); - if (lane == 0) { - if (clamp > 1.0e-6f) { - if (gate > clamp) gate = clamp; - if (up > clamp) up = clamp; - if (up < -clamp) up = -clamp; - } - const uint64_t off = (uint64_t)pair * expert_mid_dim + row; - gate_out[off] = gate; - up_out[off] = up; - mid_out[off] = (gate / (1.0f + expf(-gate))) * up * weights[(uint64_t)tok * n_expert + slot]; - } -} - __global__ static void moe_gate_up_mid_qwarp32_kernel( float *gate_out, float *up_out, @@ -1438,21 +1219,6 @@ __global__ static void moe_prefix_sorted_pairs_kernel( } } -__global__ static void moe_scatter_sorted_pairs_kernel( - uint32_t *sorted_pairs, - uint32_t *cursors, - const int32_t *selected, - uint32_t pair_count, - uint32_t n_total_expert) { - uint32_t pair = (uint32_t)((uint64_t)blockIdx.x * blockDim.x + threadIdx.x); - if (pair >= pair_count) return; - int32_t expert_i = selected[pair]; - if (expert_i < 0) expert_i = 0; - if ((uint32_t)expert_i >= n_total_expert) return; - uint32_t pos = atomicAdd(cursors + (uint32_t)expert_i, 1u); - sorted_pairs[pos] = pair; -} - /* Keep pair order stable inside each expert bucket. The MoE WMMA kernels are * row-position sensitive enough that atomic append order changes logits. */ __global__ static void moe_scatter_sorted_pairs_deterministic_kernel( @@ -1567,68 +1333,6 @@ __global__ static void moe_gate_up_mid_sorted_qwarp32_kernel( } } -__global__ static DS4_ROCM_UNUSED void moe_gate_up_mid_expert_tile8_kernel( - float *gate_out, - float *up_out, - float *mid_out, - const char *gate_base, - const char *up_base, - const cuda_block_q8_K *xq, - const uint32_t *sorted_pairs, - const uint32_t *offsets, - const uint32_t *counts, - const uint32_t *tile_total, - const uint32_t *tile_experts, - const uint32_t *tile_starts, - const float *weights, - uint64_t gate_expert_bytes, - uint64_t gate_row_bytes, - uint32_t xq_blocks, - uint32_t expert_mid_dim, - uint32_t n_expert, - float clamp) { - uint32_t tile = blockIdx.y; - if (tile >= *tile_total) return; - uint32_t group = threadIdx.x >> 3u; - uint32_t lane = threadIdx.x & 7u; - uint32_t pair_slot = group & 7u; - uint32_t row_lane = group >> 3u; - uint32_t expert = tile_experts[tile]; - uint32_t local_pair = tile_starts[tile] + pair_slot; - if (local_pair >= counts[expert]) return; - uint32_t sorted_idx = offsets[expert] + local_pair; - uint32_t pair = sorted_pairs[sorted_idx]; - uint32_t tok = pair / n_expert; - uint32_t slot = pair - tok * n_expert; - const cuda_block_q8_K *xqb = xq + (uint64_t)tok * xq_blocks; - - for (uint32_t rr = 0; rr < 2u; rr++) { - uint32_t row = blockIdx.x * 8u + row_lane + rr * 4u; - if (row >= expert_mid_dim) continue; - const cuda_block_iq2_xxs *gr = (const cuda_block_iq2_xxs *)(gate_base + (uint64_t)expert * gate_expert_bytes + (uint64_t)row * gate_row_bytes); - const cuda_block_iq2_xxs *ur = (const cuda_block_iq2_xxs *)(up_base + (uint64_t)expert * gate_expert_bytes + (uint64_t)row * gate_row_bytes); - float gate = 0.0f; - float up = 0.0f; - for (uint32_t b = lane; b < xq_blocks; b += 8u) { - gate += dev_dot_iq2_xxs_q8_K_block(gr + b, xqb + b); - up += dev_dot_iq2_xxs_q8_K_block(ur + b, xqb + b); - } - gate = quarter_warp_sum_f32(gate, lane); - up = quarter_warp_sum_f32(up, lane); - if (lane == 0) { - if (clamp > 1.0e-6f) { - if (gate > clamp) gate = clamp; - if (up > clamp) up = clamp; - if (up < -clamp) up = -clamp; - } - const uint64_t off = (uint64_t)pair * expert_mid_dim + row; - gate_out[off] = gate; - up_out[off] = up; - mid_out[off] = (gate / (1.0f + expf(-gate))) * up * weights[(uint64_t)tok * n_expert + slot]; - } - } -} - __global__ static void moe_gate_up_mid_expert_tile4_row32_kernel( float *gate_out, float *up_out, @@ -2770,90 +2474,6 @@ __global__ static void moe_gate_up_mid_mxfp4_expert_tile32_row32_kernel( } } -__global__ static DS4_ROCM_UNUSED void moe_down_kernel( - float *down_out, - const char *down_base, - const cuda_block_q8_K *midq, - const int32_t *selected, - uint64_t down_expert_bytes, - uint64_t down_row_bytes, - uint32_t midq_blocks, - uint32_t out_dim, - uint32_t n_expert) { - uint32_t row = blockIdx.x; - uint32_t pair = blockIdx.y; - if (row >= out_dim) return; - uint32_t tok = pair / n_expert; - uint32_t slot = pair - tok * n_expert; - int32_t expert_i = selected[(uint64_t)tok * n_expert + slot]; - if (expert_i < 0) expert_i = 0; - const cuda_block_q2_K *wr = (const cuda_block_q2_K *)(down_base + (uint64_t)(uint32_t)expert_i * down_expert_bytes + (uint64_t)row * down_row_bytes); - const cuda_block_q8_K *xq = midq + (uint64_t)pair * midq_blocks; - float acc = 0.0f; - for (uint32_t b = threadIdx.x; b < midq_blocks; b += blockDim.x) acc += dev_dot_q2_K_q8_K_block(wr + b, xq + b); - __shared__ float partial[256]; - partial[threadIdx.x] = acc; - __syncthreads(); - for (uint32_t stride = blockDim.x >> 1; stride > 0; stride >>= 1) { - if (threadIdx.x < stride) partial[threadIdx.x] += partial[threadIdx.x + stride]; - __syncthreads(); - } - if (threadIdx.x == 0) down_out[(uint64_t)pair * out_dim + row] = partial[0]; -} - -__global__ static DS4_ROCM_UNUSED void moe_down_warp8_kernel( - float *down_out, - const char *down_base, - const cuda_block_q8_K *midq, - const int32_t *selected, - uint64_t down_expert_bytes, - uint64_t down_row_bytes, - uint32_t midq_blocks, - uint32_t out_dim, - uint32_t n_expert) { - uint32_t lane = threadIdx.x & 31u; - uint32_t warp = threadIdx.x >> 5u; - uint32_t row = blockIdx.x * 8u + warp; - uint32_t pair = blockIdx.y; - if (row >= out_dim) return; - uint32_t tok = pair / n_expert; - uint32_t slot = pair - tok * n_expert; - int32_t expert_i = selected[(uint64_t)tok * n_expert + slot]; - if (expert_i < 0) expert_i = 0; - const cuda_block_q2_K *wr = (const cuda_block_q2_K *)(down_base + (uint64_t)(uint32_t)expert_i * down_expert_bytes + (uint64_t)row * down_row_bytes); - const cuda_block_q8_K *xq = midq + (uint64_t)pair * midq_blocks; - float acc = 0.0f; - for (uint32_t b = lane; b < midq_blocks; b += 32u) acc += dev_dot_q2_K_q8_K_block(wr + b, xq + b); - acc = warp_sum_f32(acc); - if (lane == 0) down_out[(uint64_t)pair * out_dim + row] = acc; -} - -__global__ static DS4_ROCM_UNUSED void moe_down_hwarp16_kernel( - float *down_out, - const char *down_base, - const cuda_block_q8_K *midq, - const int32_t *selected, - uint64_t down_expert_bytes, - uint64_t down_row_bytes, - uint32_t midq_blocks, - uint32_t out_dim, - uint32_t n_expert) { - uint32_t lane = threadIdx.x & 15u; - uint32_t row = blockIdx.x * 16u + (threadIdx.x >> 4u); - uint32_t pair = blockIdx.y; - if (row >= out_dim) return; - uint32_t tok = pair / n_expert; - uint32_t slot = pair - tok * n_expert; - int32_t expert_i = selected[(uint64_t)tok * n_expert + slot]; - if (expert_i < 0) expert_i = 0; - const cuda_block_q2_K *wr = (const cuda_block_q2_K *)(down_base + (uint64_t)(uint32_t)expert_i * down_expert_bytes + (uint64_t)row * down_row_bytes); - const cuda_block_q8_K *xq = midq + (uint64_t)pair * midq_blocks; - float acc = 0.0f; - for (uint32_t b = lane; b < midq_blocks; b += 16u) acc += dev_dot_q2_K_q8_K_block(wr + b, xq + b); - acc = half_warp_sum_f32(acc, lane); - if (lane == 0) down_out[(uint64_t)pair * out_dim + row] = acc; -} - __global__ static void moe_down_qwarp32_kernel( float *down_out, const char *down_base, @@ -3634,45 +3254,6 @@ __global__ static void moe_down_sorted_qwarp32_kernel( if (lane == 0) down_out[(uint64_t)pair * out_dim + row] = acc; } -__global__ static DS4_ROCM_UNUSED void moe_down_expert_tile8_kernel( - float *down_out, - const char *down_base, - const cuda_block_q8_K *midq, - const uint32_t *sorted_pairs, - const uint32_t *offsets, - const uint32_t *counts, - const uint32_t *tile_total, - const uint32_t *tile_experts, - const uint32_t *tile_starts, - uint64_t down_expert_bytes, - uint64_t down_row_bytes, - uint32_t midq_blocks, - uint32_t out_dim, - uint32_t n_expert) { - uint32_t tile = blockIdx.y; - if (tile >= *tile_total) return; - uint32_t group = threadIdx.x >> 3u; - uint32_t lane = threadIdx.x & 7u; - uint32_t pair_slot = group & 7u; - uint32_t row_lane = group >> 3u; - uint32_t expert = tile_experts[tile]; - uint32_t local_pair = tile_starts[tile] + pair_slot; - if (local_pair >= counts[expert]) return; - uint32_t sorted_idx = offsets[expert] + local_pair; - uint32_t pair = sorted_pairs[sorted_idx]; - const cuda_block_q8_K *xq = midq + (uint64_t)pair * midq_blocks; - - for (uint32_t rr = 0; rr < 2u; rr++) { - uint32_t row = blockIdx.x * 8u + row_lane + rr * 4u; - if (row >= out_dim) continue; - const cuda_block_q2_K *wr = (const cuda_block_q2_K *)(down_base + (uint64_t)expert * down_expert_bytes + (uint64_t)row * down_row_bytes); - float acc = 0.0f; - for (uint32_t b = lane; b < midq_blocks; b += 8u) acc += dev_dot_q2_K_q8_K_block(wr + b, xq + b); - acc = quarter_warp_sum_f32(acc, lane); - if (lane == 0) down_out[(uint64_t)pair * out_dim + row] = acc; - } -} - __global__ static void moe_down_expert_tile4_row32_kernel( float *down_out, const char *down_base, @@ -4123,21 +3704,6 @@ __device__ __forceinline__ static float q2_K_dequant_256_scaled_w32( } -__device__ __forceinline__ static float q2_K_dequant_256_direct(const unsigned char *blk, uint32_t i) { - const uint16_t d_bits = (uint16_t)blk[80] | ((uint16_t)blk[81] << 8); - const uint16_t dmin_bits = (uint16_t)blk[82] | ((uint16_t)blk[83] << 8); - const unsigned char *sc = blk; - const unsigned char *qs = blk + 16u; - const uint32_t g = i >> 4u; - const uint32_t within = g & 7u; - const uint32_t qi = (g >> 3u) * 32u + (within & 1u) * 16u + (i & 15u); - const uint32_t shift = (within >> 1u) * 2u; - const float q = (float)((qs[qi] >> shift) & 3u); - const float scale = (float)(sc[g] & 0x0fu); - const float mn = (float)(sc[g] >> 4u); - return dev_f16_to_f32(d_bits) * scale * q - dev_f16_to_f32(dmin_bits) * mn; -} - template __device__ __forceinline__ static void q2_K_dequant_tile_half_rowwise( half *shB, @@ -4178,107 +3744,6 @@ __device__ __forceinline__ static void q2_K_dequant_tile_half_rowwise( } } -template -__device__ __forceinline__ static void q2_K_dequant_dual_tile_half_rowwise( - half *shB0, - half *shB1, - const unsigned char *base0, - const unsigned char *base1, - uint64_t row_bytes, - uint32_t n0, - uint32_t k0, - uint32_t out_dim, - uint32_t tid) { - const uint32_t g = (k0 & 255u) >> 4u; - const uint32_t within = g & 7u; - const uint32_t qbase = (g >> 3u) * 32u + (within & 1u) * 16u; - const uint32_t shift = (within >> 1u) * 2u; - constexpr uint32_t KG = 2u; - for (uint32_t j = tid; j < (uint32_t)(BN * (BK / KG)); j += blockDim.x) { - const uint32_t nn = j / (uint32_t)(BK / KG); - const uint32_t kk0 = (j - nn * (uint32_t)(BK / KG)) * KG; - const uint32_t row = n0 + nn; - if (row < out_dim) { - const unsigned char *blk0 = base0 + (uint64_t)row * row_bytes + (uint64_t)(k0 >> 8u) * 84u; - const unsigned char *blk1 = base1 + (uint64_t)row * row_bytes + (uint64_t)(k0 >> 8u) * 84u; - const float d0 = dev_f16_to_f32((uint16_t)blk0[80] | ((uint16_t)blk0[81] << 8)); - const float dm0 = dev_f16_to_f32((uint16_t)blk0[82] | ((uint16_t)blk0[83] << 8)); - const float d1 = dev_f16_to_f32((uint16_t)blk1[80] | ((uint16_t)blk1[81] << 8)); - const float dm1 = dev_f16_to_f32((uint16_t)blk1[82] | ((uint16_t)blk1[83] << 8)); - const float s0 = (float)(blk0[g] & 0x0fu); - const float m0 = (float)(blk0[g] >> 4u); - const float s1 = (float)(blk1[g] & 0x0fu); - const float m1 = (float)(blk1[g] >> 4u); -#pragma unroll - for (uint32_t u = 0; u < KG; u++) { - const uint32_t kk = kk0 + u; - const float q0 = (float)((blk0[16u + qbase + kk] >> shift) & 3u); - const float q1 = (float)((blk1[16u + qbase + kk] >> shift) & 3u); - const uint32_t sj = kk * (uint32_t)BN + nn; - shB0[sj] = __float2half(d0 * s0 * q0 - dm0 * m0); - shB1[sj] = __float2half(d1 * s1 * q1 - dm1 * m1); - } - } else { -#pragma unroll - for (uint32_t u = 0; u < KG; u++) { - const uint32_t kk = kk0 + u; - const uint32_t sj = kk * (uint32_t)BN + nn; - shB0[sj] = __float2half(0.0f); - shB1[sj] = __float2half(0.0f); - } - } - } -} - -template -__device__ __forceinline__ static void q2_K_dequant_pair_tile_half_rowwise( - half *shB0, - half *shB1, - const unsigned char *base, - uint64_t row_bytes, - uint32_t n0, - uint32_t k0, - uint32_t out_dim, - uint32_t tid) { - const uint32_t g = (k0 & 255u) >> 4u; - const uint32_t within = g & 7u; - const uint32_t qbase = (g >> 3u) * 32u + (within & 1u) * 16u; - const uint32_t shift = (within >> 1u) * 2u; - constexpr uint32_t KG = 4u; - constexpr uint32_t UNITS_PER_TILE = (uint32_t)(BN * (BK / KG)); - for (uint32_t j = tid; j < 2u * UNITS_PER_TILE; j += blockDim.x) { - const uint32_t tile = j / UNITS_PER_TILE; - const uint32_t rem = j - tile * UNITS_PER_TILE; - const uint32_t nn = rem / (uint32_t)(BK / KG); - const uint32_t kk0 = (rem - nn * (uint32_t)(BK / KG)) * KG; - const uint32_t row = n0 + tile * (uint32_t)BN + nn; - half *shB = tile == 0u ? shB0 : shB1; - uint32_t v0 = 0u; - uint32_t v1 = 0u; - if (row < out_dim) { - const unsigned char *blk = base + (uint64_t)row * row_bytes + (uint64_t)(k0 >> 8u) * 84u; - const float d = dev_f16_to_f32((uint16_t)blk[80] | ((uint16_t)blk[81] << 8)); - const float dm = dev_f16_to_f32((uint16_t)blk[82] | ((uint16_t)blk[83] << 8)); - const float s = (float)(blk[g] & 0x0fu); - const float m = (float)(blk[g] >> 4u); - const uint32_t qbits = *reinterpret_cast(blk + 16u + qbase + kk0); - const uint32_t q0 = (qbits >> shift) & 3u; - const uint32_t q1 = (qbits >> (8u + shift)) & 3u; - const uint32_t q2 = (qbits >> (16u + shift)) & 3u; - const uint32_t q3 = (qbits >> (24u + shift)) & 3u; - const float ds = d * s; - const float dmm = dm * m; - v0 = dev_pack_half2_bits(ds * (float)q0 - dmm, - ds * (float)q1 - dmm); - v1 = dev_pack_half2_bits(ds * (float)q2 - dmm, - ds * (float)q3 - dmm); - } - half *dst = shB + nn * (uint32_t)BK + kk0; - *reinterpret_cast(dst) = v0; - *reinterpret_cast(dst + 2u) = v1; - } -} - template __device__ __forceinline__ static void q2_K_dequant_pair_tile_half_rowwise_staged( half *shB0, diff --git a/rocm/ds4_rocm_norm_rope.cuh b/rocm/ds4_rocm_norm_rope.cuh index 949fcb181..a7cb07e07 100644 --- a/rocm/ds4_rocm_norm_rope.cuh +++ b/rocm/ds4_rocm_norm_rope.cuh @@ -357,12 +357,6 @@ __device__ static float dsv4_e2m1fn_dequant_dev(float x) { return sign * dsv4_e2m1fn_value_dev(best); } -__device__ static float model_scalar_dev(const void *base, uint64_t offset, uint32_t type, uint64_t idx) { - const char *p = (const char *)base + offset; - if (type == 1u) return __half2float(((const __half *)p)[idx]); - return ((const float *)p)[idx]; -} - __device__ static float model_ape_value_dev(const void *base, uint64_t offset, uint32_t type, uint32_t width, uint32_t row, uint32_t col) { const char *p = (const char *)base + offset; @@ -377,38 +371,6 @@ __device__ static float model_ape_value_dev(const void *base, uint64_t offset, u return ((const float *)p)[(uint64_t)row * width + col]; } -__device__ static float rope_yarn_ramp_cpu_equiv_dev(float low, float high, int i0) { - float y = ((float)(i0 / 2) - low) / fmaxf(0.001f, high - low); - return 1.0f - fminf(1.0f, fmaxf(0.0f, y)); -} - -__device__ static DS4_ROCM_UNUSED void rope_tail_one_dev(float *x, uint32_t head_dim, uint32_t n_rot, uint32_t pos, uint32_t n_ctx_orig, float freq_base, float freq_scale, float ext_factor, float attn_factor, float beta_fast, float beta_slow) { - uint32_t n_nope = head_dim - n_rot; - float corr0 = 0.0f, corr1 = 0.0f; - if (ext_factor != 0.0f) { - float denom = 2.0f * logf(freq_base); - corr0 = fmaxf(0.0f, floorf((float)n_rot * logf((float)n_ctx_orig / (beta_fast * 2.0f * (float)M_PI)) / denom)); - corr1 = fminf((float)(n_rot - 1), ceilf((float)n_rot * logf((float)n_ctx_orig / (beta_slow * 2.0f * (float)M_PI)) / denom)); - } - for (uint32_t i = 0; i < n_rot; i += 2) { - float theta_extrap = (float)pos * powf(freq_base, -((float)i) / (float)n_rot); - float theta_interp = freq_scale * theta_extrap; - float theta = theta_interp; - float mscale = attn_factor; - if (ext_factor != 0.0f) { - float mix = rope_yarn_ramp_cpu_equiv_dev(corr0, corr1, (int)i) * ext_factor; - theta = theta_interp * (1.0f - mix) + theta_extrap * mix; - mscale *= 1.0f + 0.1f * logf(1.0f / freq_scale); - } - float c = cosf(theta) * mscale; - float s = sinf(theta) * mscale; - float x0 = x[n_nope + i]; - float x1 = x[n_nope + i + 1]; - x[n_nope + i] = x0 * c - x1 * s; - x[n_nope + i + 1] = x0 * s + x1 * c; - } -} - extern "C" int ds4_gpu_rms_norm_plain_tensor(ds4_gpu_tensor *out, const ds4_gpu_tensor *x, uint32_t n, float eps) { if (!cuda_tensor_has_f32(out, n) || !cuda_tensor_has_f32(x, n)) return 0; if (n == 0u) return 1; diff --git a/rocm/ds4_rocm_q8.cuh b/rocm/ds4_rocm_q8.cuh index bc0ffb2a9..f287d9318 100644 --- a/rocm/ds4_rocm_q8.cuh +++ b/rocm/ds4_rocm_q8.cuh @@ -35,49 +35,6 @@ __device__ __forceinline__ static int32_t dot_i8_block(const int8_t *a, const in return dot; } -__global__ static DS4_ROCM_UNUSED void matmul_q8_0_kernel( - float *out, - const unsigned char *w, - const float *x, - uint64_t in_dim, - uint64_t out_dim, - uint64_t n_tok) { - uint64_t row = (uint64_t)blockIdx.x; - uint64_t tok = (uint64_t)blockIdx.y; - if (row >= out_dim || tok >= n_tok) return; - const uint64_t blocks = (in_dim + 31) / 32; - const unsigned char *wr = w + row * blocks * 34; - const float *xr = x + tok * in_dim; - float acc = 0.0f; - - for (uint64_t b = threadIdx.x; b < blocks; b += blockDim.x) { - uint64_t i0 = b * 32; - uint64_t bn = in_dim - i0 < 32 ? in_dim - i0 : 32; - float amax = 0.0f; - for (uint64_t i = 0; i < bn; i++) amax = fmaxf(amax, fabsf(xr[i0 + i])); - float d = amax / 127.0f; - float id = d != 0.0f ? 1.0f / d : 0.0f; - const __half *scale_h = (const __half *)(wr + b * 34); - const int8_t *qs = (const int8_t *)(wr + b * 34 + 2); - int dot = 0; - for (uint64_t i = 0; i < bn; i++) { - int q = (int)lrintf(xr[i0 + i] * id); - q = q > 127 ? 127 : (q < -128 ? -128 : q); - dot += (int)qs[i] * q; - } - acc += __half2float(*scale_h) * d * (float)dot; - } - - __shared__ float partial[256]; - partial[threadIdx.x] = acc; - __syncthreads(); - for (uint32_t stride = blockDim.x >> 1; stride > 0; stride >>= 1) { - if (threadIdx.x < stride) partial[threadIdx.x] += partial[threadIdx.x + stride]; - __syncthreads(); - } - if (threadIdx.x == 0) out[tok * out_dim + row] = partial[0]; -} - __global__ static void quantize_q8_0_f32_kernel( int8_t *xq, float *xscale, @@ -147,33 +104,6 @@ __global__ static void matmul_q8_0_preq_kernel( if (threadIdx.x == 0) out[tok * out_dim + row] = partial[0]; } -__global__ static void matmul_q8_0_preq_warp8_kernel( - float *out, - const unsigned char *w, - const int8_t *xq, - const float *xscale, - uint64_t in_dim, - uint64_t out_dim, - uint64_t blocks, - int use_dp4a) { - uint64_t row = (uint64_t)blockIdx.x * 8u + (threadIdx.x >> 5u); - uint32_t lane = threadIdx.x & 31u; - if (row >= out_dim) return; - const unsigned char *wr = w + row * blocks * 34; - float acc = 0.0f; - for (uint64_t b = lane; b < blocks; b += 32u) { - uint64_t i0 = b * 32; - uint64_t bn = in_dim - i0 < 32 ? in_dim - i0 : 32; - const __half *scale_h = (const __half *)(wr + b * 34); - const int8_t *qs = (const int8_t *)(wr + b * 34 + 2); - const int8_t *xqb = xq + b * 32; - int dot = dot_i8_block(qs, xqb, bn, use_dp4a); - acc += __half2float(*scale_h) * xscale[b] * (float)dot; - } - acc = warp_sum_f32(acc); - if (lane == 0) out[row] = acc; -} - __global__ static void matmul_q8_0_preq_rows_w32_kernel( float *out, const unsigned char *w, @@ -247,108 +177,6 @@ __global__ static void matmul_q8_0_pair_preq_warp8_kernel( } } -__global__ static void shared_gate_up_swiglu_q8_0_pair_preq_warp8_kernel( - float *gate, - float *up, - float *mid, - const unsigned char *wg, - const unsigned char *wu, - const int8_t *xq, - const float *xscale, - uint64_t in_dim, - uint64_t out_dim, - uint64_t blocks, - int use_dp4a, - int store_gate_up, - float clamp) { - const uint64_t row = (uint64_t)blockIdx.x * 8u + (threadIdx.x >> 5u); - const uint32_t lane = threadIdx.x & 31u; - if (row >= out_dim) return; - const unsigned char *gr = wg + row * blocks * 34u; - const unsigned char *ur = wu + row * blocks * 34u; - float g = 0.0f; - float u = 0.0f; - for (uint64_t b = lane; b < blocks; b += 32u) { - const uint64_t i0 = b * 32u; - const uint64_t bn = in_dim - i0 < 32u ? in_dim - i0 : 32u; - const int8_t *xqb = xq + b * 32u; - const float xs = xscale[b]; - const __half *gscale_h = (const __half *)(gr + b * 34u); - const int8_t *gqs = (const int8_t *)(gr + b * 34u + 2u); - const __half *uscale_h = (const __half *)(ur + b * 34u); - const int8_t *uqs = (const int8_t *)(ur + b * 34u + 2u); - const int gdot = dot_i8_block(gqs, xqb, bn, use_dp4a); - const int udot = dot_i8_block(uqs, xqb, bn, use_dp4a); - g += __half2float(*gscale_h) * xs * (float)gdot; - u += __half2float(*uscale_h) * xs * (float)udot; - } - g = warp_sum_f32(g); - u = warp_sum_f32(u); - if (lane == 0u) { - if (store_gate_up) { - gate[row] = g; - up[row] = u; - } - float sg = g; - float su = u; - if (clamp > 1.0e-6f) { - sg = fminf(sg, clamp); - su = fminf(fmaxf(su, -clamp), clamp); - } - mid[row] = (sg / (1.0f + expf(-sg))) * su; - } -} - -__global__ static void matmul_q8_0_hc_expand_preq_warp8_kernel( - float *out_hc, - float *block_out, - const float *block_add, - const float *residual_hc, - const float *split, - const unsigned char *w, - const int8_t *xq, - const float *xscale, - uint64_t in_dim, - uint64_t out_dim, - uint32_t n_embd, - uint32_t n_hc, - uint64_t blocks, - int has_add, - int use_dp4a) { - const uint64_t row = (uint64_t)blockIdx.x * 8u + (threadIdx.x >> 5u); - const uint32_t lane = threadIdx.x & 31u; - if (row >= out_dim) return; - const unsigned char *wr = w + row * blocks * 34; - float acc = 0.0f; - for (uint64_t b = lane; b < blocks; b += 32u) { - const uint64_t i0 = b * 32; - const uint64_t bn = in_dim - i0 < 32 ? in_dim - i0 : 32; - const __half *scale_h = (const __half *)(wr + b * 34); - const int8_t *qs = (const int8_t *)(wr + b * 34 + 2); - const int8_t *xqb = xq + b * 32; - int dot = dot_i8_block(qs, xqb, bn, use_dp4a); - acc += __half2float(*scale_h) * xscale[b] * (float)dot; - } - acc = warp_sum_f32(acc); - if (lane == 0) { - const uint32_t d = (uint32_t)row; - block_out[d] = acc; - float block_v = acc; - if (has_add) block_v += block_add[d]; - const float *post = split + n_hc; - const float *comb = split + 2u * n_hc; - for (uint32_t dst_hc = 0; dst_hc < n_hc; dst_hc++) { - float hc_acc = block_v * post[dst_hc]; - for (uint32_t src_hc = 0; src_hc < n_hc; src_hc++) { - const float comb_v = comb[dst_hc + (uint64_t)src_hc * n_hc]; - const float res_v = residual_hc[(uint64_t)src_hc * n_embd + d]; - hc_acc += comb_v * res_v; - } - out_hc[(uint64_t)dst_hc * n_embd + d] = hc_acc; - } - } -} - __global__ static void matmul_q8_0_preq_batch_warp8_kernel( float *out, const unsigned char *w, @@ -396,47 +224,6 @@ __device__ static float q8_0_scale_broadcast_w32(const unsigned char *blk) { #endif } -__device__ static float q8_block_sum_w32(float v) { - __shared__ float sh[32]; - const uint32_t tid = threadIdx.x; - const uint32_t lane = tid & 31u; - const uint32_t wid = tid >> 5u; - const uint32_t nwarp = (blockDim.x + 31u) >> 5u; - v = warp_sum_f32(v); - if (lane == 0u) sh[wid] = v; - __syncthreads(); - v = (tid < nwarp) ? sh[lane] : 0.0f; - if (wid == 0u) v = warp_sum_f32(v); - if (tid == 0u) sh[0] = v; - __syncthreads(); - return sh[0]; -} - -__global__ static void matmul_q8_0_f32_small_block_w32_kernel( - float *out, - const unsigned char *w, - const float *x, - uint32_t n_blocks, - uint64_t out_dim, - uint64_t row_bytes) { - const uint64_t row = (uint64_t)blockIdx.x; - if (row >= out_dim) return; - const uint32_t tid = threadIdx.x; - const uint32_t lane = tid & 31u; - const uint32_t wave = tid >> 5u; - const uint32_t waves_per_block = blockDim.x >> 5u; - const unsigned char *wr = w + row * row_bytes; - float acc = 0.0f; - for (uint32_t b = wave; b < n_blocks; b += waves_per_block) { - const unsigned char *blk = wr + (uint64_t)b * 34u; - const float d = q8_0_scale_broadcast_w32(blk); - const int8_t q = ((const int8_t *)(blk + 2u))[lane]; - acc += d * (float)q * x[((uint64_t)b << 5u) + lane]; - } - acc = q8_block_sum_w32(acc); - if (tid == 0u) out[row] = acc; -} - __global__ static void matmul_q8_0_f32_warp8_kernel( float *out, const unsigned char *w, @@ -847,78 +634,6 @@ __global__ static void matmul_q8_0_f32_batch_wmma_rowtile_kernel( } } -template -__global__ static void matmul_q8_0_f32_batch_wmma_onthefly_kernel( - float *out, - const unsigned char *w, - const float *x, - uint32_t n_tokens, - uint32_t in_dim, - uint32_t out_dim, - uint64_t row_bytes) { - extern __shared__ unsigned char raw_sh[]; - half *shA = reinterpret_cast(raw_sh); - half *shB = shA + BM * BK; - float *shC = reinterpret_cast(shB + TILES_N * BK * BN); - const uint32_t tid = threadIdx.x; - const uint32_t wave = tid >> 5u; - const uint32_t t0 = (uint32_t)blockIdx.y * BM; - const uint32_t row0 = (uint32_t)blockIdx.x * TILES_N * BN; - - using frag_a = rocwmma::fragment; - using frag_b = rocwmma::fragment; - using frag_c = rocwmma::fragment; - frag_a a; - frag_b b; - frag_c acc; - if (wave < TILES_N) rocwmma::fill_fragment(acc, 0.0f); - - for (uint32_t k0 = 0; k0 < in_dim; k0 += BK) { - for (uint32_t j = tid; j < BM * BK; j += blockDim.x) { - const uint32_t m = j / BK; - const uint32_t kk = j - m * BK; - const uint32_t t = t0 + m; - shA[j] = (t < n_tokens && k0 + kk < in_dim) - ? __float2half(x[(uint64_t)t * in_dim + k0 + kk]) - : __float2half(0.0f); - } - for (uint32_t j = tid; j < TILES_N * BK * BN; j += blockDim.x) { - const uint32_t tn = j / (BK * BN); - const uint32_t rem = j - tn * BK * BN; - const uint32_t kk = rem / BN; - const uint32_t nn = rem - kk * BN; - const uint32_t row = row0 + tn * BN + nn; - const uint32_t k = k0 + kk; - if (row < out_dim && k < in_dim) { - const unsigned char *blk = w + (uint64_t)row * row_bytes + (uint64_t)(k >> 5u) * 34u; - const float d = __half2float(*(const half *)blk); - const int8_t q = ((const int8_t *)(blk + 2u))[k & 31u]; - shB[j] = __float2half(d * (float)q); - } else { - shB[j] = __float2half(0.0f); - } - } - __syncthreads(); - if (wave < TILES_N) { - rocwmma::load_matrix_sync(a, shA, BK); - rocwmma::load_matrix_sync(b, shB + wave * BK * BN, BN); - rocwmma::mma_sync(acc, a, b, acc); - } - __syncthreads(); - } - - if (wave < TILES_N) rocwmma::store_matrix_sync(shC + wave * BM * BN, acc, BN, rocwmma::mem_row_major); - __syncthreads(); - for (uint32_t j = tid; j < TILES_N * BM * BN; j += blockDim.x) { - const uint32_t tn = j / (BM * BN); - const uint32_t rem = j - tn * BM * BN; - const uint32_t m = rem / BN; - const uint32_t nn = rem - m * BN; - const uint32_t t = t0 + m; - const uint32_t row = row0 + tn * BN + nn; - if (t < n_tokens && row < out_dim) out[(uint64_t)t * out_dim + row] = shC[j]; - } -} #endif __global__ static void matmul_q8_0_pair_f32_warp8_kernel( @@ -1232,182 +947,6 @@ __device__ static float q8_0_scale_broadcast_oldhip_w32(const unsigned char *blk #endif } -__global__ static void matmul_q8_0_hc_partial16_w32_kernel( - float *partial, - const unsigned char *w, - const float *x, - uint32_t out_dim, - uint64_t row_bytes) { - extern __shared__ float shx[]; - const uint32_t tid = threadIdx.x; - const uint32_t lane = tid & 31u; - const uint32_t wave = tid >> 5; - const uint32_t rows_per_block = blockDim.x >> 5; - const uint32_t split = blockIdx.y; - const uint32_t b0 = split << 4; - for (uint32_t i = tid; i < 512u; i += blockDim.x) shx[i] = x[((uint64_t)b0 << 5) + i]; - __syncthreads(); - - const uint32_t row = blockIdx.x * rows_per_block + wave; - if (row >= out_dim) return; - const unsigned char *wr = w + (uint64_t)row * row_bytes; - float acc = 0.0f; -#pragma unroll - for (uint32_t bb = 0; bb < 16u; bb++) { - const uint32_t b = b0 + bb; - const unsigned char *blk = wr + (uint64_t)b * 34u; - const float d = q8_0_scale_broadcast_oldhip_w32(blk); - const int8_t q = ((const int8_t *)(blk + 2u))[lane]; - acc += d * (float)q * shx[(bb << 5) + lane]; - } - acc = warp_sum_f32_oldhip_w32(acc); - if (lane == 0u) partial[(uint64_t)split * out_dim + row] = acc; -} - -__global__ static void matmul_q8_0_hc_partial_w32_kernel( - float *partial, - const unsigned char *w, - const float *x, - uint32_t n_blocks, - uint32_t out_dim, - uint64_t row_bytes, - uint32_t n_splits) { - extern __shared__ float shx[]; - const uint32_t tid = threadIdx.x; - const uint32_t lane = tid & 31u; - const uint32_t wave = tid >> 5; - const uint32_t rows_per_block = blockDim.x >> 5; - const uint32_t split = blockIdx.y; - const uint32_t chunk = (n_blocks + n_splits - 1u) / n_splits; - const uint32_t b0 = split * chunk; - const uint32_t b1 = min(n_blocks, b0 + chunk); - const uint32_t chunk_blocks = b1 > b0 ? b1 - b0 : 0u; - for (uint32_t i = tid; i < (chunk_blocks << 5); i += blockDim.x) shx[i] = x[((uint64_t)b0 << 5) + i]; - __syncthreads(); - - const uint32_t row = blockIdx.x * rows_per_block + wave; - if (row >= out_dim) return; - const unsigned char *wr = w + (uint64_t)row * row_bytes; - float acc = 0.0f; - for (uint32_t bb = 0; bb < chunk_blocks; bb++) { - const uint32_t b = b0 + bb; - const unsigned char *blk = wr + (uint64_t)b * 34u; - const float d = q8_0_scale_broadcast_oldhip_w32(blk); - const int8_t q = ((const int8_t *)(blk + 2u))[lane]; - acc += d * (float)q * shx[(bb << 5) + lane]; - } - acc = warp_sum_f32_oldhip_w32(acc); - if (lane == 0u) partial[(uint64_t)split * out_dim + row] = acc; -} - -__global__ static void hc_expand_partial_kernel( - float *out_hc, - float *block_out, - const float *partial, - const float *residual_hc, - const float *split, - uint32_t out_dim, - uint32_t n_hc, - uint32_t n_splits, - int store_block_out) { - const uint32_t row = blockIdx.x * blockDim.x + threadIdx.x; - if (row >= out_dim) return; - float acc = 0.0f; - for (uint32_t s = 0; s < n_splits; s++) acc += partial[(uint64_t)s * out_dim + row]; - if (store_block_out) block_out[row] = acc; - const float *post = split + n_hc; - const float *comb = split + 2u * n_hc; - for (uint32_t dst = 0; dst < n_hc; dst++) { - float v = acc * post[dst]; - for (uint32_t src = 0; src < n_hc; src++) { - v += comb[dst + (uint64_t)src * n_hc] * residual_hc[(uint64_t)src * out_dim + row]; - } - out_hc[(uint64_t)dst * out_dim + row] = v; - } -} - -__global__ static void hc_expand_add_partial_kernel( - float *out_hc, - float *block_out, - const float *partial, - const float *block_add, - const float *residual_hc, - const float *split, - uint32_t out_dim, - uint32_t n_hc, - uint32_t n_splits, - int store_block_out) { - const uint32_t row = blockIdx.x * blockDim.x + threadIdx.x; - if (row >= out_dim) return; - float acc = 0.0f; - for (uint32_t s = 0; s < n_splits; s++) acc += partial[(uint64_t)s * out_dim + row]; - if (store_block_out) block_out[row] = acc; - const float block = acc + block_add[row]; - const float *post = split + n_hc; - const float *comb = split + 2u * n_hc; - for (uint32_t dst = 0; dst < n_hc; dst++) { - float v = block * post[dst]; - for (uint32_t src = 0; src < n_hc; src++) { - v += comb[dst + (uint64_t)src * n_hc] * residual_hc[(uint64_t)src * out_dim + row]; - } - out_hc[(uint64_t)dst * out_dim + row] = v; - } -} - -__global__ static void hc_expand_add_partial4_kernel( - float *out_hc, - float *block_out, - const float *partial, - const float *block_add, - const float *residual_hc, - const float *split, - uint32_t out_dim, - uint32_t n_hc, - int store_block_out) { - const uint32_t row = blockIdx.x * blockDim.x + threadIdx.x; - if (row >= out_dim) return; - float acc = 0.0f; -#pragma unroll - for (uint32_t s = 0; s < 4u; s++) acc += partial[(uint64_t)s * out_dim + row]; - if (store_block_out) block_out[row] = acc; - const float block = acc + block_add[row]; - const float *post = split + n_hc; - const float *comb = split + 2u * n_hc; - for (uint32_t dst = 0; dst < n_hc; dst++) { - float v = block * post[dst]; - for (uint32_t src = 0; src < n_hc; src++) { - v += comb[dst + (uint64_t)src * n_hc] * residual_hc[(uint64_t)src * out_dim + row]; - } - out_hc[(uint64_t)dst * out_dim + row] = v; - } -} - -__global__ static void hc_expand_partial16_kernel( - float *out_hc, - float *block_out, - const float *partial, - const float *residual_hc, - const float *split, - uint32_t out_dim, - uint32_t n_hc, - int store_block_out) { - const uint32_t row = blockIdx.x * blockDim.x + threadIdx.x; - if (row >= out_dim) return; - float acc = 0.0f; -#pragma unroll - for (uint32_t s = 0; s < 16u; s++) acc += partial[(uint64_t)s * out_dim + row]; - if (store_block_out) block_out[row] = acc; - const float *post = split + n_hc; - const float *comb = split + 2u * n_hc; - for (uint32_t dst = 0; dst < n_hc; dst++) { - float v = acc * post[dst]; - for (uint32_t src = 0; src < n_hc; src++) { - v += comb[dst + (uint64_t)src * n_hc] * residual_hc[(uint64_t)src * out_dim + row]; - } - out_hc[(uint64_t)dst * out_dim + row] = v; - } -} - __global__ static void grouped_q8_0_a_f32_warp8_kernel( float *low, const unsigned char *w, @@ -1719,87 +1258,6 @@ __global__ static void grouped_q8_0_a_f32_batch_sharedx_chunked_strided_w32_kern } } -#if defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__) -template -__global__ static void grouped_q8_0_a_f32_batch_wmma_onthefly_kernel( - float *low, - const unsigned char *w, - const float *heads, - uint32_t n_tokens, - uint32_t n_groups, - uint32_t group_dim, - uint32_t rank, - uint64_t row_bytes) { - extern __shared__ unsigned char raw_sh[]; - half *shA = reinterpret_cast(raw_sh); - half *shB = shA + BM * BK; - float *shC = reinterpret_cast(shB + TILES_N * BK * BN); - const uint32_t tid = threadIdx.x; - const uint32_t wave = tid >> 5u; - const uint32_t row_tiles_per_group = (rank + TILES_N * BN - 1u) / (TILES_N * BN); - const uint32_t g = (uint32_t)blockIdx.x / row_tiles_per_group; - const uint32_t row_tile = (uint32_t)blockIdx.x - g * row_tiles_per_group; - const uint32_t row0 = row_tile * TILES_N * BN; - const uint32_t t0 = (uint32_t)blockIdx.y * BM; - if (g >= n_groups) return; - - using frag_a = rocwmma::fragment; - using frag_b = rocwmma::fragment; - using frag_c = rocwmma::fragment; - frag_a a; - frag_b b; - frag_c acc; - if (wave < TILES_N) rocwmma::fill_fragment(acc, 0.0f); - - for (uint32_t k0 = 0; k0 < group_dim; k0 += BK) { - for (uint32_t j = tid; j < BM * BK; j += blockDim.x) { - const uint32_t m = j / BK; - const uint32_t kk = j - m * BK; - const uint32_t t = t0 + m; - const uint32_t k = k0 + kk; - shA[j] = (t < n_tokens && k < group_dim) - ? __float2half(heads[((uint64_t)t * n_groups + g) * group_dim + k]) - : __float2half(0.0f); - } - for (uint32_t j = tid; j < TILES_N * BK * BN; j += blockDim.x) { - const uint32_t tn = j / (BK * BN); - const uint32_t rem = j - tn * BK * BN; - const uint32_t kk = rem / BN; - const uint32_t nn = rem - kk * BN; - const uint32_t row = row0 + tn * BN + nn; - const uint32_t k = k0 + kk; - if (row < rank && k < group_dim) { - const unsigned char *blk = w + ((uint64_t)g * rank + row) * row_bytes + (uint64_t)(k >> 5u) * 34u; - const float d = __half2float(*(const half *)blk); - const int8_t q = ((const int8_t *)(blk + 2u))[k & 31u]; - shB[j] = __float2half(d * (float)q); - } else { - shB[j] = __float2half(0.0f); - } - } - __syncthreads(); - if (wave < TILES_N) { - rocwmma::load_matrix_sync(a, shA, BK); - rocwmma::load_matrix_sync(b, shB + wave * BK * BN, BN); - rocwmma::mma_sync(acc, a, b, acc); - } - __syncthreads(); - } - - if (wave < TILES_N) rocwmma::store_matrix_sync(shC + wave * BM * BN, acc, BN, rocwmma::mem_row_major); - __syncthreads(); - for (uint32_t j = tid; j < TILES_N * BM * BN; j += blockDim.x) { - const uint32_t tn = j / (BM * BN); - const uint32_t rem = j - tn * BM * BN; - const uint32_t m = rem / BN; - const uint32_t nn = rem - m * BN; - const uint32_t t = t0 + m; - const uint32_t row = row0 + tn * BN + nn; - if (t < n_tokens && row < rank) low[((uint64_t)t * n_groups + g) * rank + row] = shC[j]; - } -} -#endif - __global__ static void dequant_q8_0_to_f16_kernel( __half *out, const unsigned char *w, diff --git a/rocm/ds4_rocm_runtime.cuh b/rocm/ds4_rocm_runtime.cuh index dfb8bb009..39cddd0cf 100644 --- a/rocm/ds4_rocm_runtime.cuh +++ b/rocm/ds4_rocm_runtime.cuh @@ -30,7 +30,6 @@ enum { static int g_rocblas_f16_solution_set; static int g_rocblas_f16_solutions_disabled; static int g_rocblas_attention_b_solution_disabled; -#include "ds4_rocm_hipblaslt.cuh" #endif static int g_quality_mode; static int g_glm_model; @@ -535,14 +534,6 @@ static int cuda_tensor_has_i32(const ds4_gpu_tensor *t, uint64_t elems) { return cuda_tensor_has_elems(t, elems, sizeof(int32_t)); } -static int cuda_tensor_has_f16(const ds4_gpu_tensor *t, uint64_t elems) { - return cuda_tensor_has_elems(t, elems, sizeof(__half)); -} - -static int cuda_tensor_has_u16(const ds4_gpu_tensor *t, uint64_t elems) { - return cuda_tensor_has_elems(t, elems, sizeof(uint16_t)); -} - static const char *cuda_model_range_ptr_from_fd( const void *model_map, uint64_t offset, @@ -576,8 +567,6 @@ __global__ static void dequant_q8_0_to_f16_transpose_kernel( uint64_t out_dim, uint64_t blocks); -static void cuda_shared_gate_up_async_cleanup(void); - static void *cuda_tmp_alloc(uint64_t bytes, const char *what) { if (bytes == 0) return NULL; if (g_cuda_tmp_bytes >= bytes) return g_cuda_tmp; @@ -3647,191 +3636,6 @@ static int cuda_stream_batch_selected_prepare( return ok; } -static int cuda_stream_layer_expert_cache_prepare_batch( - const void *model_map, - uint32_t layer, - const ds4_gpu_tensor *selected, - uint32_t n_tokens, - uint32_t n_total_expert, - uint32_t n_selected, - uint64_t gate_offset, - uint64_t up_offset, - uint64_t down_offset, - uint64_t gate_expert_bytes, - uint64_t down_expert_bytes, - const ds4_gpu_tensor **selected_exec, - const char ***gate_ptrs, - const char ***up_ptrs, - const char ***down_ptrs, - uint32_t *unique_out) { - if (!selected || - !selected_exec || - !gate_ptrs || - !up_ptrs || - !down_ptrs || - !unique_out || - !cuda_tensor_has_elems2(selected, n_tokens, n_selected, sizeof(int32_t)) || - n_tokens <= 1 || - n_total_expert == 0 || - n_total_expert > DS4_ROCM_MAX_N_EXPERT || - n_selected == 0 || - n_selected > DS4_ROCM_N_EXPERT_USED || - gate_expert_bytes == 0 || - down_expert_bytes == 0) { - return 0; - } - const char *layer_gate = NULL; - const char *layer_up = NULL; - const char *layer_down = NULL; - if (!cuda_stream_layer_expert_cache_apply(model_map, - layer, - n_total_expert, - gate_offset, - up_offset, - down_offset, - gate_expert_bytes, - down_expert_bytes, - &layer_gate, - &layer_up, - &layer_down)) { - return 0; - } - - uint64_t n_ids64 = 0; - if (!cuda_u64_mul_checked(n_tokens, n_selected, &n_ids64) || - n_ids64 > SIZE_MAX / sizeof(int32_t)) { - return 0; - } - int32_t *ids = (int32_t *)malloc((size_t)n_ids64 * sizeof(ids[0])); - int32_t *compact_ids = - (int32_t *)malloc((size_t)n_ids64 * sizeof(compact_ids[0])); - if (!ids || !compact_ids) { - free(ids); - free(compact_ids); - return 0; - } - - int ok = cuda_ok(cudaMemcpy(ids, - selected->ptr, - (size_t)n_ids64 * sizeof(ids[0]), - cudaMemcpyDeviceToHost), - "streaming full-layer selected ids copy"); - - int32_t map[DS4_ROCM_MAX_N_EXPERT]; - int32_t unique_ids[DS4_ROCM_MAX_N_EXPERT]; - for (uint32_t i = 0; i < DS4_ROCM_MAX_N_EXPERT; i++) map[i] = -1; - uint32_t unique_count = 0; - for (uint64_t i = 0; ok && i < n_ids64; i++) { - const int32_t expert = ids[i]; - if (expert < 0 || (uint32_t)expert >= n_total_expert) { - fprintf(stderr, - DS4_GPU_LOG_PREFIX "streaming full-layer selected expert id %d " - "outside 0..%u (layer=%u)\n", - expert, - n_total_expert, - layer); - ok = 0; - break; - } - int32_t slot = map[(uint32_t)expert]; - if (slot < 0) { - if (unique_count >= DS4_ROCM_MAX_N_EXPERT) { - ok = 0; - break; - } - slot = (int32_t)unique_count; - map[(uint32_t)expert] = slot; - unique_ids[unique_count++] = expert; - } - compact_ids[i] = slot; - } - if (ok && unique_count == 0) ok = 0; - if (ok && !cuda_stream_batch_selected_ensure_buffers(n_ids64, unique_count)) { - ok = 0; - } - if (ok && !cuda_stream_selected_ensure_stream()) ok = 0; - - const char *gate_host[DS4_ROCM_MAX_N_EXPERT] = {0}; - const char *up_host[DS4_ROCM_MAX_N_EXPERT] = {0}; - const char *down_host[DS4_ROCM_MAX_N_EXPERT] = {0}; - for (uint32_t u = 0; ok && u < unique_count; u++) { - const uint64_t expert = (uint64_t)(uint32_t)unique_ids[u]; - uint64_t gate_rel = 0; - uint64_t down_rel = 0; - if (!cuda_u64_mul_checked(expert, gate_expert_bytes, &gate_rel) || - !cuda_u64_mul_checked(expert, down_expert_bytes, &down_rel)) { - ok = 0; - break; - } - gate_host[u] = layer_gate + gate_rel; - up_host[u] = layer_up + gate_rel; - down_host[u] = layer_down + down_rel; - } - - if (ok) { - cudaError_t err = cudaMemcpyAsync(g_stream_batch_selected_cache.selected_ids, - compact_ids, - (size_t)n_ids64 * sizeof(compact_ids[0]), - cudaMemcpyHostToDevice, - g_stream_selected_upload_stream); - if (err == cudaSuccess) { - err = cudaMemcpyAsync(g_stream_batch_selected_cache.gate_ptrs, - gate_host, - unique_count * sizeof(gate_host[0]), - cudaMemcpyHostToDevice, - g_stream_selected_upload_stream); - } - if (err == cudaSuccess) { - err = cudaMemcpyAsync(g_stream_batch_selected_cache.up_ptrs, - up_host, - unique_count * sizeof(up_host[0]), - cudaMemcpyHostToDevice, - g_stream_selected_upload_stream); - } - if (err == cudaSuccess) { - err = cudaMemcpyAsync(g_stream_batch_selected_cache.down_ptrs, - down_host, - unique_count * sizeof(down_host[0]), - cudaMemcpyHostToDevice, - g_stream_selected_upload_stream); - } - if (err == cudaSuccess) err = cudaStreamSynchronize(g_stream_selected_upload_stream); - if (err != cudaSuccess) { - fprintf(stderr, - DS4_GPU_LOG_PREFIX "streaming full-layer selected table upload failed: %s\n", - cudaGetErrorString(err)); - (void)cudaGetLastError(); - ok = 0; - } - } - - if (ok) { - g_stream_batch_selected_cache.loaded = 0; - g_stream_batch_selected_cache.model_map = model_map; - g_stream_batch_selected_cache.layer = layer; - g_stream_batch_selected_cache.n_total_expert = n_total_expert; - g_stream_batch_selected_cache.n_selected = n_selected; - g_stream_batch_selected_cache.n_tokens = n_tokens; - g_stream_batch_selected_cache.n_unique = unique_count; - g_stream_batch_selected_cache.gate_offset = gate_offset; - g_stream_batch_selected_cache.up_offset = up_offset; - g_stream_batch_selected_cache.down_offset = down_offset; - g_stream_batch_selected_cache.gate_expert_bytes = gate_expert_bytes; - g_stream_batch_selected_cache.down_expert_bytes = down_expert_bytes; - *selected_exec = &g_stream_batch_selected_cache.selected_tensor; - *gate_ptrs = g_stream_batch_selected_cache.gate_ptrs; - *up_ptrs = g_stream_batch_selected_cache.up_ptrs; - *down_ptrs = g_stream_batch_selected_cache.down_ptrs; - *unique_out = unique_count; - } else { - g_stream_batch_selected_cache.loaded = 0; - } - - free(ids); - free(compact_ids); - return ok; -} - static int cuda_stream_layer_expert_cache_seed_selected( const void *model_map, uint64_t model_size, @@ -5871,11 +5675,6 @@ extern "C" int ds4_gpu_init(void) { __atomic_store_n(&g_rocblas_attention_b_solution_disabled, 0, __ATOMIC_RELAXED); g_rocblas_ready = 1; } - if (!g_hipblaslt_ready) { - if (hipblaslt_ok(hipblasLtCreate(&g_hipblaslt), "create handle")) { - g_hipblaslt_ready = 1; - } - } #endif return 1; } @@ -5884,10 +5683,6 @@ extern "C" void ds4_gpu_cleanup(void) { (void)cudaDeviceSynchronize(); (void)ds4_gpu_release_q4_attn_q_b_f16_sidecars(); cuda_stream_cache_stats_print("cleanup"); - cuda_shared_gate_up_async_cleanup(); -#ifdef __HIP_PLATFORM_AMD__ - hipblaslt_gemm_plan_clear(); -#endif if (g_cublas_ready) { (void)cublasDestroy(g_cublas); g_cublas_ready = 0; @@ -5902,11 +5697,6 @@ extern "C" void ds4_gpu_cleanup(void) { __atomic_store_n(&g_rocblas_f16_solutions_disabled, 0, __ATOMIC_RELAXED); __atomic_store_n(&g_rocblas_attention_b_solution_disabled, 0, __ATOMIC_RELAXED); } - if (g_hipblaslt_ready) { - (void)hipblasLtDestroy(g_hipblaslt); - g_hipblaslt_ready = 0; - g_hipblaslt = NULL; - } #endif cuda_model_range_release_all(); cuda_q8_f16_cache_release_all(); diff --git a/rocm/ds4_rocm_shared_expert.cuh b/rocm/ds4_rocm_shared_expert.cuh index 198c652ce..945e3fcac 100644 --- a/rocm/ds4_rocm_shared_expert.cuh +++ b/rocm/ds4_rocm_shared_expert.cuh @@ -195,198 +195,6 @@ extern "C" int ds4_gpu_shared_gate_up_swiglu_q8_0_rows_tensor( n_tok); } -static cudaStream_t g_shared_gate_up_stream = NULL; -static cudaEvent_t g_shared_gate_up_ready_event = NULL; -static void *g_shared_gate_up_tmp = NULL; -static uint64_t g_shared_gate_up_tmp_bytes = 0; -static int g_shared_gate_up_pending = 0; - -static int cuda_shared_gate_up_async_wait_internal(void) { - if (!g_shared_gate_up_pending) return 1; - cudaError_t err = cudaStreamSynchronize(g_shared_gate_up_stream); - g_shared_gate_up_pending = 0; - if (err != cudaSuccess) { - fprintf(stderr, DS4_GPU_LOG_PREFIX "shared gate/up async wait failed: %s\n", cudaGetErrorString(err)); - (void)cudaGetLastError(); - return 0; - } - return 1; -} - -static void *cuda_shared_gate_up_async_tmp_alloc(uint64_t bytes) { - if (bytes == 0) return NULL; - if (g_shared_gate_up_tmp_bytes >= bytes) return g_shared_gate_up_tmp; - if (g_shared_gate_up_tmp) { - (void)cuda_shared_gate_up_async_wait_internal(); - (void)cudaFree(g_shared_gate_up_tmp); - g_shared_gate_up_tmp = NULL; - g_shared_gate_up_tmp_bytes = 0; - } - void *ptr = NULL; - cudaError_t err = cudaMalloc(&ptr, (size_t)bytes); - if (err != cudaSuccess) { - fprintf(stderr, DS4_GPU_LOG_PREFIX "shared gate/up async temp alloc failed (%.2f MiB): %s\n", - (double)bytes / 1048576.0, cudaGetErrorString(err)); - (void)cudaGetLastError(); - return NULL; - } - g_shared_gate_up_tmp = ptr; - g_shared_gate_up_tmp_bytes = bytes; - return g_shared_gate_up_tmp; -} - -static void cuda_shared_gate_up_async_cleanup(void) { - if (g_shared_gate_up_stream) { - (void)cuda_shared_gate_up_async_wait_internal(); - } - if (g_shared_gate_up_tmp) { - (void)cudaFree(g_shared_gate_up_tmp); - g_shared_gate_up_tmp = NULL; - g_shared_gate_up_tmp_bytes = 0; - } - if (g_shared_gate_up_ready_event) { - (void)cudaEventDestroy(g_shared_gate_up_ready_event); - g_shared_gate_up_ready_event = NULL; - } - if (g_shared_gate_up_stream) { - (void)cudaStreamDestroy(g_shared_gate_up_stream); - g_shared_gate_up_stream = NULL; - } -} - -extern "C" int ds4_gpu_shared_gate_up_swiglu_q8_0_async_tensor( - ds4_gpu_tensor *gate, - ds4_gpu_tensor *up, - ds4_gpu_tensor *mid, - const void *model_map, - uint64_t model_size, - uint64_t gate_offset, - uint64_t up_offset, - uint64_t in_dim, - uint64_t out_dim, - const ds4_gpu_tensor *x, - float clamp) { - if (g_quality_mode || cuda_runtime_config()->graph_dump) return 0; - if (g_shared_gate_up_pending && !cuda_shared_gate_up_async_wait_internal()) return 0; - if (!gate || !up || !mid || !model_map || !x || - in_dim == 0u || out_dim == 0u || in_dim > UINT32_MAX || out_dim > UINT32_MAX) { - return 0; - } - const uint64_t blocks = (in_dim + 31u) / 32u; - uint64_t row_bytes = 0; - uint64_t weight_bytes = 0; - if (!cuda_u64_mul_checked(blocks, 34u, &row_bytes) || - !cuda_u64_mul_checked(out_dim, row_bytes, &weight_bytes)) { - return 0; - } - if (g_quality_mode || - !gate || !up || !mid || !model_map || !x || - in_dim == 0u || out_dim == 0u || in_dim > UINT32_MAX || out_dim > UINT32_MAX || - gate_offset > model_size || up_offset > model_size || - weight_bytes > model_size - gate_offset || - weight_bytes > model_size - up_offset || - x->bytes < in_dim * sizeof(float) || - gate->bytes < out_dim * sizeof(float) || - up->bytes < out_dim * sizeof(float) || - mid->bytes < out_dim * sizeof(float)) { - return 0; - } - const char *wg = cuda_model_range_ptr(model_map, gate_offset, weight_bytes, "shared_gate_q8_pair_async"); - const char *wu = cuda_model_range_ptr(model_map, up_offset, weight_bytes, "shared_up_q8_pair_async"); - if (!wg || !wu) return 0; - if (!g_shared_gate_up_stream) { - int least_priority = 0; - int greatest_priority = 0; -#ifdef __HIP_PLATFORM_AMD__ - hipError_t err = hipDeviceGetStreamPriorityRange(&least_priority, &greatest_priority); - if (err == hipSuccess) { - err = hipStreamCreateWithPriority(&g_shared_gate_up_stream, cudaStreamNonBlocking, least_priority); - } else { - (void)cudaGetLastError(); - err = hipStreamCreateWithFlags(&g_shared_gate_up_stream, cudaStreamNonBlocking); - } - if (err != hipSuccess) return 0; -#else - cudaError_t err = cudaDeviceGetStreamPriorityRange(&least_priority, &greatest_priority); - if (err == cudaSuccess) { - err = cudaStreamCreateWithPriority(&g_shared_gate_up_stream, cudaStreamNonBlocking, least_priority); - } else { - (void)cudaGetLastError(); - err = cudaStreamCreateWithFlags(&g_shared_gate_up_stream, cudaStreamNonBlocking); - } - if (err != cudaSuccess) return 0; -#endif - } - if (!g_shared_gate_up_ready_event) { - cudaError_t err = cudaEventCreateWithFlags(&g_shared_gate_up_ready_event, cudaEventDisableTiming); - if (err != cudaSuccess) { - fprintf(stderr, DS4_GPU_LOG_PREFIX "shared gate/up async event create failed: %s\n", cudaGetErrorString(err)); - (void)cudaGetLastError(); - return 0; - } - } - /* - * This stream is intentionally non-blocking so it can overlap routed MoE. - * Non-blocking streams do not inherit default-stream ordering, so explicitly - * wait until the default-stream producer of x (ffn_norm) has completed before - * quantizing it here. - */ - cudaError_t dep_err = cudaEventRecord(g_shared_gate_up_ready_event, 0); - if (dep_err != cudaSuccess) { - fprintf(stderr, DS4_GPU_LOG_PREFIX "shared gate/up async dependency record failed: %s\n", cudaGetErrorString(dep_err)); - (void)cudaGetLastError(); - return 0; - } -#ifdef __HIP_PLATFORM_AMD__ - dep_err = hipStreamWaitEvent(g_shared_gate_up_stream, g_shared_gate_up_ready_event, 0); -#else - dep_err = cudaStreamWaitEvent(g_shared_gate_up_stream, g_shared_gate_up_ready_event, 0); -#endif - if (dep_err != cudaSuccess) { - fprintf(stderr, DS4_GPU_LOG_PREFIX "shared gate/up async dependency wait failed: %s\n", cudaGetErrorString(dep_err)); - (void)cudaGetLastError(); - return 0; - } - const uint64_t xq_bytes = blocks * 32u; - const uint64_t scale_offset = (xq_bytes + 15u) & ~15ull; - const uint64_t tmp_bytes = scale_offset + blocks * sizeof(float); - void *tmp = cuda_shared_gate_up_async_tmp_alloc(tmp_bytes); - if (!tmp) return 0; - int8_t *xq = (int8_t *)tmp; - float *xscale = (float *)((char *)tmp + scale_offset); - const int use_dp4a = 1; - dim3 qgrid((unsigned)blocks, 1, 1); - quantize_q8_0_f32_kernel<<>>(xq, xscale, (const float *)x->ptr, in_dim, blocks); - if (!cuda_ok(cudaGetLastError(), "shared gate/up async quantize launch")) return 0; - matmul_q8_0_pair_preq_warp8_kernel<<<((unsigned)out_dim + 7u) / 8u, 256, 0, g_shared_gate_up_stream>>>( - (float *)gate->ptr, - (float *)up->ptr, - reinterpret_cast(wg), - reinterpret_cast(wu), - xq, - xscale, - in_dim, - out_dim, - out_dim, - blocks, - use_dp4a); - if (!cuda_ok(cudaGetLastError(), "shared gate/up async pair launch")) return 0; - swiglu_kernel<<<((unsigned)out_dim + 255u) / 256u, 256, 0, g_shared_gate_up_stream>>>( - (float *)mid->ptr, - (const float *)gate->ptr, - (const float *)up->ptr, - (uint32_t)out_dim, - clamp, - 1.0f); - if (!cuda_ok(cudaGetLastError(), "shared gate/up async swiglu launch")) return 0; - g_shared_gate_up_pending = 1; - return 1; -} - -extern "C" int ds4_gpu_shared_gate_up_async_wait(void) { - return cuda_shared_gate_up_async_wait_internal(); -} - extern "C" int ds4_gpu_shared_gate_up_swiglu_q8_0_batch_tensor( ds4_gpu_tensor *gate, ds4_gpu_tensor *up, From 701d4d1a2616990c1694b5feb246a7d50c086151 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Sat, 29 Aug 2026 12:36:27 +0200 Subject: [PATCH 161/189] Update metal/moe.metal Co-authored-by: Frank --- metal/moe.metal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metal/moe.metal b/metal/moe.metal index 60db8bc5c..3b4e86ec5 100644 --- a/metal/moe.metal +++ b/metal/moe.metal @@ -2603,7 +2603,7 @@ void dequantize_iq2_xxs(device const block_iq2_xxs * xb, short il, thread type4x const float dl = d * (0.5f + (aux32_s >> 28)); constant const ushort * values = ds4_metal_iq2xxs_half_values[aux8[2*il+0]]; uint8_t signs = ksigns_iq2xs[(aux32_s >> 14*il) & 127]; - for (int i = 0; i < 8; ++i) { + FOR_UNROLL (int i = 0; i < 8; ++i) { const ushort bits = values[i] ^ (signs & kmask_iq2xs[i] ? 0x8000u : 0x0000u); reg[i/4][i%4] = dl * (float)as_type(bits); } From e6ee504942786abaa8abf29252d59fa5e5726434 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Sat, 29 Aug 2026 12:36:38 +0200 Subject: [PATCH 162/189] Update metal/moe.metal Co-authored-by: Frank --- metal/moe.metal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metal/moe.metal b/metal/moe.metal index 3b4e86ec5..f7076c4d3 100644 --- a/metal/moe.metal +++ b/metal/moe.metal @@ -2609,7 +2609,7 @@ void dequantize_iq2_xxs(device const block_iq2_xxs * xb, short il, thread type4x } values = ds4_metal_iq2xxs_half_values[aux8[2*il+1]]; signs = ksigns_iq2xs[(aux32_s >> (14*il+7)) & 127]; - for (int i = 0; i < 8; ++i) { + FOR_UNROLL (int i = 0; i < 8; ++i) { const ushort bits = values[i] ^ (signs & kmask_iq2xs[i] ? 0x8000u : 0x0000u); reg[2+i/4][i%4] = dl * (float)as_type(bits); } From 441b55ff1ef55c209bc8e682274d4ddd8a7543ec Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Sat, 29 Aug 2026 12:36:50 +0200 Subject: [PATCH 163/189] Update metal/moe.metal Co-authored-by: Frank --- metal/moe.metal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metal/moe.metal b/metal/moe.metal index f7076c4d3..0f5b03368 100644 --- a/metal/moe.metal +++ b/metal/moe.metal @@ -9954,7 +9954,7 @@ kernel void kernel_mul_mm_id_mpp( device const block_q * xb = x_base + (chunk / nl); if (is_same::value && FC_mul_mm_bc_inp) { - for (short i = 0; i < 16; i++) { + FOR_UNROLL (short i = 0; i < 16; i++) { const short sx = 2*il0 + i/8; const short sy = (tiitg/NL0)/8; const short lx = i%8; From a01b14533a20903feea6f0f95f80a4cb607a2a8a Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Sat, 29 Aug 2026 12:37:04 +0200 Subject: [PATCH 164/189] Update rocm/ds4_rocm_attention.cuh Co-authored-by: Frank --- rocm/ds4_rocm_attention.cuh | 1 - 1 file changed, 1 deletion(-) diff --git a/rocm/ds4_rocm_attention.cuh b/rocm/ds4_rocm_attention.cuh index 53025aef8..acf302241 100644 --- a/rocm/ds4_rocm_attention.cuh +++ b/rocm/ds4_rocm_attention.cuh @@ -1631,7 +1631,6 @@ __global__ static void attention_mixed_heads16_wmma_kernel( } sh_qp[hl * STRIDE + kl] = __float2half(p); } - } __syncthreads(); #pragma unroll for (uint32_t delta = 1u; delta < 16u; delta <<= 1u) { From ac319c1e3384ea33390becaa75e991647f71d5ae Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Sat, 29 Aug 2026 13:03:02 +0200 Subject: [PATCH 165/189] Optimize ROCm DSpark verification prefill --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index e9e8eed8b..91aee2c96 100644 --- a/Makefile +++ b/Makefile @@ -549,7 +549,7 @@ test-mmq-parity-cuda: cuda/mmq/test/test_mmq_parity speed-bench/gpu_iq2_moe_prefill_bench_rocm.o: speed-bench/gpu_iq2_moe_prefill_bench.c ds4_gpu.h $(CC) $(filter-out -ffast-math,$(CFLAGS)) $(ROCM_HOST_CFLAGS) -std=c11 -DDS4_ROCM_BUILD -DDS4_BENCH_ROCM -I. -c -o $@ $< -speed-bench/gpu_iq2_moe_prefill_bench_rocm: speed-bench/gpu_iq2_moe_prefill_bench_rocm.o ds4_rocm.o +speed-bench/gpu_iq2_moe_prefill_bench_rocm: speed-bench/gpu_iq2_moe_prefill_bench_rocm.o ds4_rocm.o $(ROCM_MMQ_OBJS) $(HIPCC) $(ROCM_CFLAGS) -o $@ $^ $(ROCM_LDLIBS) rocm-iq2-moe-prefill-bench: From 1f88ece4f6b3f9ae660d09835405e3a2f83cc408 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Sat, 29 Aug 2026 13:03:18 +0200 Subject: [PATCH 166/189] Optimize CUDA persistent expert planning --- ds4_cuda.cu | 275 ++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 248 insertions(+), 27 deletions(-) diff --git a/ds4_cuda.cu b/ds4_cuda.cu index c363875a7..bb77a288a 100644 --- a/ds4_cuda.cu +++ b/ds4_cuda.cu @@ -4727,6 +4727,7 @@ static int cuda_stream_expert_persistent_plan_build( const int32_t *selected_ids, uint32_t n_selected, cuda_stream_expert_persistent_plan *plan) { + enum { dense_expert_stack_capacity = 384 }; g_stream_expert_persistent_plan_attempts.fetch_add( 1, std::memory_order_relaxed); if (!state || !table || !selected_ids || n_selected == 0 || !plan || @@ -4750,28 +4751,44 @@ static int cuda_stream_expert_persistent_plan_build( return 0; } + cuda_stream_expert_persistent_key key_template = {}; + if (!cuda_stream_expert_persistent_key_make(&key_template, table, 0)) { + cuda_stream_expert_persistent_note_reject( + &g_stream_expert_persistent_overflow_rejects); + return 0; + } + std::vector unique_keys; std::vector input_unique; + uint32_t expert_to_unique_stack[dense_expert_stack_capacity]; + std::vector expert_to_unique_heap; + uint32_t *expert_to_unique = NULL; uint32_t duplicate_count = 0; try { - unique_keys.reserve(n_selected); + /* Expert ids form a small dense domain. Keep only one full key per + * expert instead of reserving n_selected keys and scanning them for + * every token/top-k pair. */ + unique_keys.reserve(std::min(n_selected, table->n_total_expert)); input_unique.reserve(n_selected); + if (table->n_total_expert <= dense_expert_stack_capacity) { + std::fill_n(expert_to_unique_stack, + table->n_total_expert, + UINT32_MAX); + expert_to_unique = expert_to_unique_stack; + } else { + expert_to_unique_heap.assign(table->n_total_expert, UINT32_MAX); + expert_to_unique = expert_to_unique_heap.data(); + } for (uint32_t i = 0; i < n_selected; i++) { - cuda_stream_expert_persistent_key key = {}; - if (!cuda_stream_expert_persistent_key_make( - &key, table, selected_ids[i])) { + const int32_t selected_id = selected_ids[i]; + if (selected_id < 0 || + (uint32_t)selected_id >= key_template.n_total_expert) { cuda_stream_expert_persistent_note_reject( &g_stream_expert_persistent_overflow_rejects); return 0; } - uint32_t unique_index = UINT32_MAX; - for (uint32_t u = 0; u < unique_keys.size(); u++) { - if (cuda_stream_expert_persistent_key_equal( - &unique_keys[u], &key)) { - unique_index = u; - break; - } - } + const uint32_t expert_id = (uint32_t)selected_id; + uint32_t unique_index = expert_to_unique[expert_id]; if (unique_index == UINT32_MAX) { if (unique_keys.size() >= UINT32_MAX) { cuda_stream_expert_persistent_note_reject( @@ -4779,8 +4796,19 @@ static int cuda_stream_expert_persistent_plan_build( return 0; } unique_index = (uint32_t)unique_keys.size(); + cuda_stream_expert_persistent_key key = key_template; + key.expert_id = expert_id; unique_keys.push_back(key); + expert_to_unique[expert_id] = unique_index; } else { + /* The immutable key template was validated before the loop; + * duplicates therefore need only verify the dense map's own + * expert-id invariant. */ + if (unique_index >= unique_keys.size() || + unique_keys[unique_index].expert_id != expert_id) { + cuda_stream_expert_persistent_note_reject(NULL); + return 0; + } duplicate_count++; } input_unique.push_back(unique_index); @@ -4802,9 +4830,21 @@ static int cuda_stream_expert_persistent_plan_build( return 0; } std::vector protected_slots; + uint8_t mismatched_expert_keys_stack[dense_expert_stack_capacity]; + std::vector mismatched_expert_keys_heap; + uint8_t *mismatched_expert_keys = NULL; std::vector unique_slots; try { protected_slots.assign(candidate.next.slots.size(), 0u); + if (unique_keys.size() <= dense_expert_stack_capacity) { + std::fill_n(mismatched_expert_keys_stack, + unique_keys.size(), + (uint8_t)0u); + mismatched_expert_keys = mismatched_expert_keys_stack; + } else { + mismatched_expert_keys_heap.assign(unique_keys.size(), 0u); + mismatched_expert_keys = mismatched_expert_keys_heap.data(); + } unique_slots.assign(unique_keys.size(), UINT32_MAX); candidate.remap.resize(n_selected); candidate.loads.reserve(unique_keys.size()); @@ -4812,10 +4852,35 @@ static int cuda_stream_expert_persistent_plan_build( cuda_stream_expert_persistent_note_reject(NULL); return 0; } + /* Derive a request-local exact-key index with one increasing slot scan. + * It deliberately does not enter persistent state: commit/rollback stay + * unchanged, and keys from other models or layers cannot alias. */ for (uint32_t slot = 0; slot < candidate.next.slots.size(); slot++) { - if (candidate.next.slots[slot].pin_count != 0) { + const cuda_stream_expert_persistent_entry *entry = + &candidate.next.slots[slot]; + if (entry->pin_count != 0) { protected_slots[slot] = 1u; } + if (!entry->valid || + entry->key.expert_id >= key_template.n_total_expert) { + continue; + } + const uint32_t unique_index = + expert_to_unique[entry->key.expert_id]; + if (unique_index == UINT32_MAX || + unique_index >= unique_keys.size()) { + continue; + } + if (cuda_stream_expert_persistent_key_equal( + &entry->key, &unique_keys[unique_index])) { + /* Preserve the old linear lookup's deterministic first-slot + * behaviour if a damaged state ever contains duplicate keys. */ + if (unique_slots[unique_index] == UINT32_MAX) { + unique_slots[unique_index] = slot; + } + } else { + mismatched_expert_keys[unique_index] = 1u; + } } uint32_t hit_count = 0; @@ -4827,9 +4892,15 @@ static int cuda_stream_expert_persistent_plan_build( * cold miss before a resident key; a one-pass planner would otherwise be * able to evict that later requested resident slot. */ for (uint32_t u = 0; u < unique_keys.size(); u++) { - uint32_t slot = 0; - if (cuda_stream_expert_persistent_find_key( - &candidate.next, &unique_keys[u], &slot)) { + const uint32_t slot = unique_slots[u]; + if (slot != UINT32_MAX) { + if (slot >= candidate.next.slots.size() || + !candidate.next.slots[slot].valid || + !cuda_stream_expert_persistent_key_equal( + &candidate.next.slots[slot].key, &unique_keys[u])) { + cuda_stream_expert_persistent_note_reject(NULL); + return 0; + } protected_slots[slot] = 1u; if (!cuda_stream_expert_persistent_touch( &candidate.next, slot)) { @@ -4842,16 +4913,7 @@ static int cuda_stream_expert_persistent_plan_build( continue; } miss_count++; - for (const cuda_stream_expert_persistent_entry &entry : - candidate.next.slots) { - if (entry.valid && - entry.key.expert_id == unique_keys[u].expert_id && - !cuda_stream_expert_persistent_key_equal( - &entry.key, &unique_keys[u])) { - key_miss_count++; - break; - } - } + if (mismatched_expert_keys[u]) key_miss_count++; } /* Only slots outside the complete request hit-set and outside active @@ -5472,12 +5534,171 @@ static int cuda_stream_expert_persistent_test_rejections(void) { return 1; } +static int cuda_stream_expert_persistent_test_dense_index(void) { + unsigned char model_a = 0; + unsigned char model_b = 0; + ds4_gpu_stream_expert_table table_a = {}; + ds4_gpu_stream_expert_table table_b = {}; + ds4_gpu_stream_expert_table table_c = {}; + cuda_stream_expert_persistent_state state = {}; + if (!cuda_stream_expert_persistent_test_table_make( + &table_a, &model_a, 3u, 8u, 16u, 8u) || + !cuda_stream_expert_persistent_test_table_make( + &table_b, &model_a, 4u, 8u, 16u, 8u) || + !cuda_stream_expert_persistent_test_table_make( + &table_c, &model_b, 3u, 8u, 16u, 8u) || + !cuda_stream_expert_persistent_state_init(&state, 4u, 16u, 8u)) { + return 0; + } + + const int32_t ids_a[2] = {0, 1}; + cuda_stream_expert_persistent_plan fill_a = {}; + if (!cuda_stream_expert_persistent_plan_build( + &state, &table_a, ids_a, 2u, &fill_a) || + !cuda_stream_expert_persistent_plan_commit(&state, &fill_a)) { + return 0; + } + const int32_t ids_b[2] = {0, 2}; + cuda_stream_expert_persistent_plan fill_b = {}; + if (!cuda_stream_expert_persistent_plan_build( + &state, &table_b, ids_b, 2u, &fill_b) || + !cuda_stream_expert_persistent_plan_commit(&state, &fill_b)) { + return 0; + } + + cuda_stream_expert_persistent_state before = {}; + if (!cuda_stream_expert_persistent_state_copy(&before, &state)) return 0; + const uint64_t key_misses_before = + g_stream_expert_persistent_key_misses.load(); + + /* Slot zero and slot two deliberately contain expert zero from different + * layers of the same model. The dense expert index must still choose the + * exact full key, preserve first-seen dedup order, and leave the live + * state untouched. */ + const int32_t repeated_a[6] = {0, 0, 1, 0, 1, 1}; + cuda_stream_expert_persistent_plan hit_a = {}; + if (!cuda_stream_expert_persistent_plan_build( + &state, &table_a, repeated_a, 6u, &hit_a) || + hit_a.unique_count != 2u || hit_a.hit_count != 2u || + hit_a.miss_count != 0u || hit_a.duplicate_count != 4u || + !hit_a.loads.empty() || hit_a.slot_base != 0u || + hit_a.weight_domain != 2u || hit_a.remap.size() != 6u || + hit_a.remap[0] != 0u || hit_a.remap[1] != 0u || + hit_a.remap[2] != 1u || hit_a.remap[3] != 0u || + hit_a.remap[4] != 1u || hit_a.remap[5] != 1u || + g_stream_expert_persistent_key_misses.load() != key_misses_before || + !cuda_stream_expert_persistent_state_equal(&state, &before)) { + return 0; + } + cuda_stream_expert_persistent_plan_rollback(&hit_a); + + const int32_t repeated_b[3] = {0, 2, 0}; + cuda_stream_expert_persistent_plan hit_b = {}; + if (!cuda_stream_expert_persistent_plan_build( + &state, &table_b, repeated_b, 3u, &hit_b) || + hit_b.unique_count != 2u || hit_b.hit_count != 2u || + hit_b.miss_count != 0u || hit_b.duplicate_count != 1u || + !hit_b.loads.empty() || hit_b.slot_base != 2u || + hit_b.weight_domain != 2u || hit_b.remap.size() != 3u || + hit_b.remap[0] != 0u || hit_b.remap[1] != 1u || + hit_b.remap[2] != 0u || + g_stream_expert_persistent_key_misses.load() != key_misses_before || + !cuda_stream_expert_persistent_state_equal(&state, &before)) { + return 0; + } + cuda_stream_expert_persistent_plan_rollback(&hit_b); + + /* A third full key with the same dense expert id is a real miss. Count + * it once even though two resident entries share that expert number. */ + const int32_t id_c[1] = {0}; + cuda_stream_expert_persistent_plan miss_c = {}; + if (!cuda_stream_expert_persistent_plan_build( + &state, &table_c, id_c, 1u, &miss_c) || + miss_c.hit_count != 0u || miss_c.miss_count != 1u || + miss_c.loads.size() != 1u || !miss_c.loads[0].had_victim || + g_stream_expert_persistent_key_misses.load() != + key_misses_before + 1u || + !cuda_stream_expert_persistent_state_equal(&state, &before)) { + return 0; + } + cuda_stream_expert_persistent_plan_rollback(&miss_c); + return cuda_stream_expert_persistent_state_equal(&state, &before); +} + +static int cuda_stream_expert_persistent_test_prefill_index(void) { + enum { + test_n_expert = 384, + test_n_selected = 8192 * 6, + }; + unsigned char model_marker = 0; + ds4_gpu_stream_expert_table table = {}; + cuda_stream_expert_persistent_state state = {}; + cuda_stream_expert_persistent_state before = {}; + std::vector selected; + try { + selected.resize(test_n_selected); + } catch (...) { + return 0; + } + /* Five is coprime to 384, so the first 384 entries visit every expert + * exactly once and subsequent entries exercise dense duplicate lookup. */ + for (uint32_t i = 0; i < test_n_selected; i++) { + selected[i] = (int32_t)((i * 5u) % test_n_expert); + } + if (!cuda_stream_expert_persistent_test_table_make( + &table, &model_marker, 9u, test_n_expert, 16u, 8u) || + !cuda_stream_expert_persistent_state_init( + &state, test_n_expert, 16u, 8u) || + !cuda_stream_expert_persistent_state_copy(&before, &state)) { + return 0; + } + + cuda_stream_expert_persistent_plan prefill = {}; + if (!cuda_stream_expert_persistent_plan_build( + &state, &table, selected.data(), test_n_selected, &prefill) || + prefill.unique_count != test_n_expert || + prefill.hit_count != 0u || prefill.miss_count != test_n_expert || + prefill.duplicate_count != test_n_selected - test_n_expert || + prefill.loads.size() != test_n_expert || + prefill.remap.size() != test_n_selected || + prefill.slot_base != 0u || + prefill.weight_domain != test_n_expert || + !cuda_stream_expert_persistent_state_equal(&state, &before)) { + return 0; + } + for (uint32_t i = 0; i < test_n_selected; i++) { + if (prefill.remap[i] != i % test_n_expert) return 0; + } + for (uint32_t u = 0; u < test_n_expert; u++) { + if (prefill.loads[u].slot != u || + prefill.loads[u].key.expert_id != (u * 5u) % test_n_expert) { + return 0; + } + } + cuda_stream_expert_persistent_plan_rollback(&prefill); + if (!cuda_stream_expert_persistent_state_equal(&state, &before)) return 0; + + const int32_t invalid_low[1] = {-1}; + const int32_t invalid_high[1] = {test_n_expert}; + cuda_stream_expert_persistent_plan rejected = {}; + if (cuda_stream_expert_persistent_plan_build( + &state, &table, invalid_low, 1u, &rejected) || + cuda_stream_expert_persistent_plan_build( + &state, &table, invalid_high, 1u, &rejected) || + !cuda_stream_expert_persistent_state_equal(&state, &before)) { + return 0; + } + return 1; +} + extern "C" int ds4_cuda_test_stream_expert_persistent_planner(void) { g_stream_expert_persistent_oracle_runs.fetch_add( 1, std::memory_order_relaxed); const int ok = cuda_stream_expert_persistent_test_basic() && cuda_stream_expert_persistent_test_protection() && - cuda_stream_expert_persistent_test_rejections(); + cuda_stream_expert_persistent_test_rejections() && + cuda_stream_expert_persistent_test_dense_index() && + cuda_stream_expert_persistent_test_prefill_index(); if (!ok) { g_stream_expert_persistent_oracle_failures.fetch_add( 1, std::memory_order_relaxed); From dbc4eb8feebe36d7361d8b787c67efd40696128d Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Sat, 29 Aug 2026 15:38:15 +0200 Subject: [PATCH 167/189] Optimize Metal IQ2 top-8 MoE prefill --- Makefile | 12 ++- ds4_gpu.h | 1 + ds4_metal.m | 51 ++++++++++++- speed-bench/.gitignore | 1 + speed-bench/README.md | 28 +++++++ speed-bench/metal_iq2_moe_tail_cull_bench.c | 85 +++++++++++++++++---- speed-bench/metal_iq2_moe_top8_pair_bench.c | 2 + 7 files changed, 161 insertions(+), 19 deletions(-) create mode 100644 speed-bench/metal_iq2_moe_top8_pair_bench.c diff --git a/Makefile b/Makefile index 91aee2c96..0d1376677 100644 --- a/Makefile +++ b/Makefile @@ -81,7 +81,7 @@ test-quantizer-indexer-q4: gguf-tools/deepseek4-quantize tests/test_quantizer_in ./tests/test_quantizer_indexer_q4 ./gguf-tools/deepseek4-quantize ifeq ($(UNAME_S),Darwin) -.PHONY: metal-decode-schedule-bench metal-prefill-variant-bench metal-q4-dense-pair-bench metal-q4-prefill-pair-bench metal-q4-mm-tail-cull-bench metal-iq2-moe-tail-cull-bench check-mxfp4-half-lut test-mxfp4-metal +.PHONY: metal-decode-schedule-bench metal-prefill-variant-bench metal-q4-dense-pair-bench metal-q4-prefill-pair-bench metal-q4-mm-tail-cull-bench metal-iq2-moe-tail-cull-bench metal-iq2-moe-top8-pair-bench check-mxfp4-half-lut test-mxfp4-metal all: ds4 ds4-server ds4-bench ds4-eval ds4-agent @@ -113,6 +113,7 @@ help: @echo " make metal-q4-prefill-pair-bench Build the resident Q4 prefill pair F16-RHS benchmark" @echo " make metal-q4-mm-tail-cull-bench Build the resident Q4 prefill tail-cull kernel benchmark" @echo " make metal-iq2-moe-tail-cull-bench Build the resident IQ2 pair MoE tail-cull benchmark" + @echo " make metal-iq2-moe-top8-pair-bench Build the resident GLM-shape IQ2 top-8 pair-fusion benchmark" @echo " make check-mxfp4-half-lut Verify the checked-in MXFP4 half LUT matches the generator" @echo " make test-mxfp4-metal Check the MXFP4 half LUT, then run Metal MXFP4 exactness tests" @echo " make test-metal-exactn-oracle Compare Metal exact-N state with sequential decode" @@ -374,6 +375,14 @@ speed-bench/metal_iq2_moe_tail_cull_bench: speed-bench/metal_iq2_moe_tail_cull_b metal-iq2-moe-tail-cull-bench: speed-bench/metal_iq2_moe_tail_cull_bench +speed-bench/metal_iq2_moe_top8_pair_bench.o: speed-bench/metal_iq2_moe_top8_pair_bench.c speed-bench/metal_iq2_moe_tail_cull_bench.c ds4_gpu.h + $(CC) $(CFLAGS) -I. -c -o $@ $< + +speed-bench/metal_iq2_moe_top8_pair_bench: speed-bench/metal_iq2_moe_top8_pair_bench.o ds4_metal.o + $(CC) $(CFLAGS) -o $@ $^ $(METAL_LDLIBS) + +metal-iq2-moe-top8-pair-bench: speed-bench/metal_iq2_moe_top8_pair_bench + tests/test_mxfp4_metal.o: tests/test_mxfp4_metal.c ds4_gpu.h $(CC) $(CFLAGS) -I. -c -o $@ $< @@ -1001,4 +1010,5 @@ mxfp4-dot-test: tests/test_mxfp4_dot.c ./tests/test_mxfp4_dot clean: + rm -f speed-bench/metal_iq2_moe_top8_pair_bench rm -f ds4 ds4-server ds4-bench ds4-eval ds4-agent ds4_cpu ds4_native ds4_server_test ds4_test ds4_agent_test gguf-tools/quality-testing/score_official gguf-tools/quality-testing/score_official.o speed-bench/metal_decode_schedule_bench speed-bench/metal_prefill_variant_bench speed-bench/metal_q4_dense_pair_bench speed-bench/metal_q4_prefill_pair_bench speed-bench/metal_q4_mm_tail_cull_bench speed-bench/metal_iq2_moe_tail_cull_bench speed-bench/gpu_iq2_moe_prefill_bench_rocm speed-bench/gpu_iq2_moe_prefill_bench_cuda speed-bench/rocm_q4_prefill_bench speed-bench/cuda_q4_prefill_bench speed-bench/*.o tests/test_q4k_dot tests/test_mxfp4_dot tests/test_quantizer_indexer_q4 tests/test_mxfp4_metal tests/test_mxfp4_rocm tests/bench_mxfp4_rocm tests/test_mxfp4_cuda tests/test_rocm_q4_dense_pair tests/test_metal_session_batch tests/test_metal_q4_streams tests/test_metal_q4_prefill_pair tests/test_metal_indexer_q4 tests/test_metal_q4_attn_exactn tests/test_metal_q4_qb_f16_cache tests/test_metal_exactn_oracle tests/test_metal_dspark_capture tests/test_metal_argmax_top1 tests/test_metal_iq2_midonly tests/test_metal_iq2_ssd_grouped_mm tests/test_metal_iq2_live_index tests/test_glm53_kda tests/test_glm53_kda_rocm tests/test_glm53_vision_engine tests/test_glm53_vision_prompt tests/test_gpu_xdev tests/test_gpu_model_cache tests/test_gpu_lookup_cache_strict tests/test_engine_mgpu_refusal tests/test_engine_mgpu_runtime tests/test_engine_correctness tests/test_sampling tests/test_cuda_session_batch tests/test_cuda_mixed_batch tests/*.o *.o cuda/mmq/*.o cuda/mmq/test/*.o tests/cuda_long_context_smoke tests/cuda_long_context_smoke.o diff --git a/ds4_gpu.h b/ds4_gpu.h index 173e3d98f..cc449770e 100644 --- a/ds4_gpu.h +++ b/ds4_gpu.h @@ -396,6 +396,7 @@ enum { DS4_GPU_TEST_BATCH_ATTN_OUT_Q4_HC_FUSION = 1u << 11, DS4_GPU_TEST_FLASH_ATTN_SMALL_PREFILL_NWG32 = 1u << 12, DS4_GPU_TEST_FLASH_ATTN_SMALL_PREFILL_NWG1_FAILURE = 1u << 13, + DS4_GPU_TEST_REQUIRE_IQ2_TOP8_PAIR_SWIGLU = 1u << 14, }; void ds4_gpu_test_set_flags(uint32_t flags); double ds4_gpu_test_last_completed_gpu_ms(void); diff --git a/ds4_metal.m b/ds4_metal.m index 005b1c743..54ff5c203 100644 --- a/ds4_metal.m +++ b/ds4_metal.m @@ -48776,13 +48776,39 @@ int ds4_gpu_routed_moe_batch_tensor( * order, and the same epilogue math as separate GEMMs + SwiGLU, so the * mid tensor is bit-identical. */ - const bool use_mm_id_pair_swiglu = + /* Repeated resident A/B on M1 Max covers this exact GLM prefill + * shape at 4096 tokens. Keep the production default inside that + * measured domain: the SSD path has different page-fault locality, + * and other batch sizes/devices need their own promotion data. The + * REQUIRE test flag may opt the same exact resident shape in on other + * Metal devices so the portable harness can collect that data. */ + const bool measured_top8_iq2_pair_swiglu_shape = + n_expert == 8 && + n_total_expert == 288u && + gate_type == DS4_METAL_TENSOR_IQ2_XXS && + down_type == DS4_METAL_TENSOR_Q2_K && + expert_in_dim == 4096u && + expert_mid_dim == 2048u && + out_dim == 4096u && + gate_row_bytes == 1056u && + gate_expert_bytes == 2162688u && + down_row_bytes == 672u && + down_expert_bytes == 2752512u && + clamp == 0.0f && + n_tokens == 4096u && + !g_ssd_streaming_mode; + const bool use_measured_top8_iq2_pair_swiglu = + measured_top8_iq2_pair_swiglu_shape && + (ds4_gpu_device_name_contains("M1 Max") || + (g_test_flags & + DS4_GPU_TEST_REQUIRE_IQ2_TOP8_PAIR_SWIGLU) != 0u); + bool use_mm_id_pair_swiglu = use_mm_id && !(gate_type == DS4_METAL_TENSOR_IQ2_XXS && (ds4_gpu_routed_mm_mpp_mask() & 3) == 3) && - g_tp_split_world != 2 && /* pair-swiglu mm kernel lacks expert ownership */ + g_tp_split_world == 1 && /* pair-swiglu mm kernel lacks expert ownership */ request_mid_f16 && - n_expert == 6 && + (n_expert == 6 || use_measured_top8_iq2_pair_swiglu) && ((gate_type == DS4_METAL_TENSOR_IQ2_XXS && down_type == DS4_METAL_TENSOR_Q2_K) || (gate_type == DS4_METAL_TENSOR_Q4_K && @@ -48792,6 +48818,15 @@ int ds4_gpu_routed_moe_batch_tensor( getenv("DS4_METAL_DISABLE_MOE_MM_ID_PAIR_SWIGLU") == NULL && getenv("DS4_METAL_MOE_WRITE_CLAMPED_ACT") == NULL && getenv("DS4_METAL_GRAPH_DUMP_PREFIX") == NULL; + if ((g_test_flags & DS4_GPU_TEST_REQUIRE_IQ2_TOP8_PAIR_SWIGLU) != 0u && + !(measured_top8_iq2_pair_swiglu_shape && + use_mm_id_pair_swiglu)) { + fprintf(stderr, + "ds4: required Metal IQ2 top-8 MM ID pair-SwiGLU path " + "was not selected tokens=%u experts=%u\n", + n_tokens, n_expert); + return 0; + } /* The specialization is arithmetically valid for every grouped-MM * IQ2_XXS/Q2_K top-6 shape below. Keep the automatic default narrower: * resident kernel A/B and full-model SSD A/B measured the exact DS4 @@ -49022,6 +49057,16 @@ int ds4_gpu_routed_moe_batch_tensor( pair_swiglu_mm_pipeline = ds4_gpu_get_pipeline( "kernel_mul_mm_id_iq2_xxs_pair_swiglu_f16"); } + /* A custom Metal source may also predate the base pair + * symbol. The measured top-8 production default must remain + * fail-soft; only REQUIRE is allowed to turn that into a hard + * error so benchmark labels cannot hide a fallback. */ + if (!pair_swiglu_mm_pipeline && + use_measured_top8_iq2_pair_swiglu && + (g_test_flags & + DS4_GPU_TEST_REQUIRE_IQ2_TOP8_PAIR_SWIGLU) == 0u) { + use_mm_id_pair_swiglu = false; + } } if (!map_pipeline || !gate_mm_pipeline || !up_mm_pipeline || !down_mm_pipeline || (use_mm_id_pair_swiglu && !pair_swiglu_mm_pipeline)) { diff --git a/speed-bench/.gitignore b/speed-bench/.gitignore index 68e3a2dfd..0c92e7210 100644 --- a/speed-bench/.gitignore +++ b/speed-bench/.gitignore @@ -7,6 +7,7 @@ metal_q4_dense_pair_bench metal_q4_prefill_pair_bench metal_q4_mm_tail_cull_bench metal_iq2_moe_tail_cull_bench +metal_iq2_moe_top8_pair_bench gpu_iq2_moe_prefill_bench cuda_q4_prefill_bench rocm_q4_prefill_bench diff --git a/speed-bench/README.md b/speed-bench/README.md index 8987a0b0e..a543df68c 100644 --- a/speed-bench/README.md +++ b/speed-bench/README.md @@ -116,6 +116,34 @@ ABBA/BAAB, and requires bit-exact outputs plus intact input/output canaries. No GGUF access, SSD I/O, upload, readback, or CPU wall time is included in a measured command buffer. +### Metal resident IQ2/Q2 routed MoE + +Build the production top-6 tail-cull fixture or the GLM-shape top-8 pair-fusion +fixture with: + +``` +make metal-iq2-moe-tail-cull-bench +./speed-bench/metal_iq2_moe_tail_cull_bench --samples 12 --warmup-cycles 2 + +make metal-iq2-moe-top8-pair-bench +./speed-bench/metal_iq2_moe_top8_pair_bench --samples 12 --warmup-cycles 2 +``` + +Both fixtures keep synthetic IQ2_XXS gate/up and Q2_K down weights resident, +alternate variants in ABBA/BAAB order, and report per-stage GPU timestamps. +The top-8 fixture matches the `4096 -> 2048 -> 4096`, 288-expert GLM routed +MoE geometry. Its baseline launches gate and up separately; its candidate +uses the grouped pair-SwiGLU kernel. It requires bit-exact F16 mid rows, F32 +expert rows and final output, plus intact allocation canaries. The measured +stages contain no GGUF loading, file-backed model mapping, application SSD +reads, uploads, readback, or CPU wall timing. The anonymous fixture is fully +touched before oracle and warm-up, so a change in these GPU timestamps is +attributable to the kernel path rather than SSD streaming. + +The automatic top-8 dispatch is intentionally limited to the measured +M1 Max, resident, 4096-token shape. SSD streaming and other batch/device +shapes keep the separate gate/up path until they have their own A/B data. + ### Resident ROCm Q4_K prefill Build the production-dispatch A/B harness on a ROCm host with: diff --git a/speed-bench/metal_iq2_moe_tail_cull_bench.c b/speed-bench/metal_iq2_moe_tail_cull_bench.c index d15a017ba..f6e0188a2 100644 --- a/speed-bench/metal_iq2_moe_tail_cull_bench.c +++ b/speed-bench/metal_iq2_moe_tail_cull_bench.c @@ -18,9 +18,21 @@ #define MID_DIM 2048u #define OUT_DIM 4096u #define N_TOKENS 4096u +#ifdef DS4_METAL_IQ2_MOE_TOP8_PAIR_BENCH +#define N_TOTAL_EXPERT 288u +#define N_EXPERT 8u +#define CLAMP 0.0f +#define BENCH_DESCRIPTION \ + "Resident GLM-geometry IQ2_XXS routed-MoE top-8 pair-fusion benchmark." +#define EXPERIMENT_NAME "top8-pair-fusion" +#else #define N_TOTAL_EXPERT 256u #define N_EXPERT 6u #define CLAMP 4.0f +#define BENCH_DESCRIPTION \ + "Resident production-geometry IQ2_XXS pair routed-MoE tail-cull benchmark." +#define EXPERIMENT_NAME "pair-tail-cull" +#endif #define GUARD_WORDS 64u #define GUARD_BYTES ((uint64_t)GUARD_WORDS * sizeof(uint32_t)) #define GUARD_BITS 0x51a7c3e9u @@ -33,6 +45,8 @@ "DS4_METAL_ENABLE_IQ2_XXS_MOE_MM_ID_PAIR_TAIL_SIMDGROUP_CULL" #define PAIR_TAIL_DISABLE_ENV \ "DS4_METAL_DISABLE_IQ2_XXS_MOE_MM_ID_PAIR_TAIL_SIMDGROUP_CULL" +#define PAIR_FUSION_DISABLE_ENV \ + "DS4_METAL_DISABLE_MOE_MM_ID_PAIR_SWIGLU" typedef struct { uint16_t d; @@ -111,8 +125,7 @@ static void usage(FILE *fp, const char *argv0) { fprintf(fp, "usage: %s [options]\n" "\n" - "Resident production-geometry IQ2_XXS pair routed-MoE tail-cull " - "benchmark.\n" + BENCH_DESCRIPTION "\n" "Metal stage profiling prints the kernel GPU timestamps; marker " "lines identify each arm.\n" "\n" @@ -256,27 +269,57 @@ static uint64_t touch_model_pages(const void *model, uint64_t bytes, return checksum; } -/* The 256 target counts use 643 complete 32-row tiles plus 4000 tail rows. - * Every tail size 1..31 is present, while the 2/3-tile split keeps counts - * non-uniform (65..127). A per-token hash breaks ties in the remaining-count - * scheduler and keeps all six routed experts unique without changing counts. */ +/* Construct non-uniform expert counts with every final tile size 1..31. + * A per-token hash breaks ties in the remaining-count scheduler and keeps all + * routed experts unique without changing the target counts. */ static int build_routes(int32_t *selected, float *weights) { +#ifndef DS4_METAL_IQ2_MOE_TOP8_PAIR_BENCH static const uint8_t final_remainders[8] = {1, 2, 3, 4, 5, 6, 7, 4}; +#endif uint32_t target[N_TOTAL_EXPERT]; uint32_t remaining[N_TOTAL_EXPERT]; uint32_t actual[N_TOTAL_EXPERT] = {0}; uint64_t target_sum = 0; for (uint32_t expert = 0; expert < N_TOTAL_EXPERT; expert++) { +#ifdef DS4_METAL_IQ2_MOE_TOP8_PAIR_BENCH + const uint32_t tail = 1u + (expert * 17u) % 31u; + target[expert] = 3u * 32u + tail; +#else const uint32_t tail = expert < 248u ? 1u + (expert * 17u) % 31u : final_remainders[expert - 248u]; const uint32_t full_tiles = ((expert * 73u) & 255u) < 131u ? 3u : 2u; target[expert] = full_tiles * 32u + tail; - remaining[expert] = target[expert]; +#endif target_sum += target[expert]; } +#ifdef DS4_METAL_IQ2_MOE_TOP8_PAIR_BENCH + uint64_t deficit = (uint64_t)N_TOKENS * N_EXPERT - target_sum; + while (deficit != 0u) { + bool progressed = false; + /* Preserve experts 0..30 as one complete permutation of tails. */ + for (uint32_t expert = 31u; + expert < N_TOTAL_EXPERT && deficit != 0u; + expert++) { + if ((target[expert] & 31u) == 31u) continue; + target[expert]++; + target_sum++; + deficit--; + progressed = true; + } + if (!progressed) { + fprintf(stderr, + "metal-iq2-moe-tail-cull-bench: route target " + "distribution exhausted\n"); + return 0; + } + } +#endif + for (uint32_t expert = 0; expert < N_TOTAL_EXPERT; expert++) { + remaining[expert] = target[expert]; + } if (target_sum != (uint64_t)N_TOKENS * N_EXPERT) { fprintf(stderr, "metal-iq2-moe-tail-cull-bench: route target sum=%llu\n", @@ -451,16 +494,28 @@ static int check_all_canaries(const fixture *f) { } static const char *variant_name(bench_arm arm) { +#ifdef DS4_METAL_IQ2_MOE_TOP8_PAIR_BENCH + return arm == ARM_BASELINE ? "separate" : "fused"; +#else return arm == ARM_BASELINE ? "baseline" : "candidate"; +#endif } static int select_variant(bench_arm arm) { +#ifdef DS4_METAL_IQ2_MOE_TOP8_PAIR_BENCH + ds4_gpu_test_set_flags(arm == ARM_CANDIDATE + ? DS4_GPU_TEST_REQUIRE_IQ2_TOP8_PAIR_SWIGLU : 0u); + return arm == ARM_BASELINE + ? setenv(PAIR_FUSION_DISABLE_ENV, "1", 1) == 0 + : unsetenv(PAIR_FUSION_DISABLE_ENV) == 0; +#else if (unsetenv(PAIR_TAIL_ENABLE_ENV) != 0 || unsetenv(PAIR_TAIL_DISABLE_ENV) != 0) { return 0; } return setenv(arm == ARM_BASELINE ? PAIR_TAIL_DISABLE_ENV : PAIR_TAIL_ENABLE_ENV, "1", 1) == 0; +#endif } static int run_once(fixture *f, bench_arm arm, @@ -481,7 +536,7 @@ static int run_once(fixture *f, bench_arm arm, fprintf(stderr, "DS4_IQ2_MOE_TAIL_BENCH phase=%s experiment=%s variant=%s " "sample=%u cycle=%u position=%u order=%s force_resident=1\n", - phase, "pair", variant_name(arm), sample, + phase, EXPERIMENT_NAME, variant_name(arm), sample, cycle, position, order); fflush(stderr); @@ -507,7 +562,7 @@ static int run_once(fixture *f, bench_arm arm, fprintf(stderr, "DS4_IQ2_MOE_TAIL_BENCH result=FAIL experiment=%s " "variant=%s call=%d end=%d mid_f16=%d\n", - "pair", variant_name(arm), + EXPERIMENT_NAME, variant_name(arm), call_ok, end_ok, mid_is_f16 ? 1 : 0); } if (check_canaries) ok = check_all_canaries(f) && ok; @@ -613,7 +668,7 @@ static int run_oracle(fixture *f) { ok = run_once(f, ARM_CANDIDATE, "oracle", "pair", 0u, 0u, 0u, true, true); } - if (ok) ok = compare_candidate("pair", f, &baseline, scratch); + if (ok) ok = compare_candidate(EXPERIMENT_NAME, f, &baseline, scratch); free(scratch); free(baseline.storage); @@ -648,7 +703,7 @@ static int run_balanced_block(fixture *f, const char *phase, uint32_t cycles, fprintf(stderr, "metal-iq2-moe-tail-cull-bench: %s %s arm count " "baseline=%u candidate=%u expected=%u\n", - "pair", phase, + EXPERIMENT_NAME, phase, arm_samples[ARM_BASELINE], arm_samples[ARM_CANDIDATE], sample_limit); return 0; @@ -670,7 +725,7 @@ static int run_experiment(fixture *f, const bench_config *config) { fprintf(stderr, "DS4_IQ2_MOE_TAIL_BENCH phase=complete experiment=%s " "samples_per_variant=%u warmup_per_variant=%u result=PASS\n", - "pair", config->samples, + EXPERIMENT_NAME, config->samples, config->warmup_cycles * 2u); return 1; } @@ -843,13 +898,13 @@ int main(int argc, char **argv) { const bench_config config = parse_options(argc, argv); /* This benchmark compares the legacy grouped pair kernels directly. - * Prevent Metal 4 TensorOps/MPP from bypassing both A/B variants on M5+ - * hosts; the promoted automatic policy itself remains pre-M5-only. */ + * Prevent Metal 4 TensorOps/MPP from bypassing either A/B variant on + * M5+ hosts; production defaults remain independently hardware-gated. */ setenv("DS4_METAL_DISABLE_METAL4", "1", 1); setenv("DS4_METAL_MOE_STAGE_PROFILE", "1", 1); setenv("DS4_METAL_MOE_STAGE_PROFILE_LAYER", "0", 1); unsetenv("DS4_METAL_MOE_STAGE_PROFILE_FILTER"); - unsetenv("DS4_METAL_DISABLE_MOE_MM_ID_PAIR_SWIGLU"); + unsetenv(PAIR_FUSION_DISABLE_ENV); unsetenv("DS4_METAL_MOE_WRITE_CLAMPED_ACT"); unsetenv("DS4_METAL_GRAPH_DUMP_PREFIX"); unsetenv(PAIR_TAIL_ENABLE_ENV); diff --git a/speed-bench/metal_iq2_moe_top8_pair_bench.c b/speed-bench/metal_iq2_moe_top8_pair_bench.c new file mode 100644 index 000000000..e53ec3916 --- /dev/null +++ b/speed-bench/metal_iq2_moe_top8_pair_bench.c @@ -0,0 +1,2 @@ +#define DS4_METAL_IQ2_MOE_TOP8_PAIR_BENCH 1 +#include "metal_iq2_moe_tail_cull_bench.c" From 59a174f2b0b01a2ffe6583688be3c825b504bfaf Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Sat, 29 Aug 2026 20:50:13 +0200 Subject: [PATCH 168/189] Fix ROCm Q4 attention and MMQ build regressions --- Makefile | 4 ++-- cuda/mmq/ds4_mmq.cu | 8 +++++++- rocm/ds4_rocm_attention.cuh | 3 +-- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 0d1376677..e7ab65fa3 100644 --- a/Makefile +++ b/Makefile @@ -567,7 +567,7 @@ rocm-iq2-moe-prefill-bench: speed-bench/rocm_q4_prefill_bench.o: speed-bench/rocm_q4_prefill_bench.cpp ds4_gpu.h $(HIPCC) $(ROCM_CFLAGS) -DDS4_ROCM_BUILD -std=c++17 -fno-fast-math -I. -c -o $@ $< -speed-bench/rocm_q4_prefill_bench: speed-bench/rocm_q4_prefill_bench.o ds4_rocm.o ds4_rocm_compat.o ds4_rocm_unavailable.o +speed-bench/rocm_q4_prefill_bench: speed-bench/rocm_q4_prefill_bench.o ds4_rocm.o $(ROCM_MMQ_OBJS) ds4_rocm_compat.o ds4_rocm_unavailable.o $(HIPCC) $(ROCM_CFLAGS) -o $@ $^ $(ROCM_LDLIBS) rocm-q4-prefill-bench: @@ -797,7 +797,7 @@ ds4_rocm_unavailable.o: ds4_rocm_unavailable.cu tests/test_rocm_q4_dense_pair.o: tests/test_rocm_q4_dense_pair.cpp ds4_gpu.h $(HIPCC) $(ROCM_CFLAGS) -DDS4_ROCM_BUILD -std=c++17 -fno-fast-math -I. -c -o $@ $< -tests/test_rocm_q4_dense_pair: tests/test_rocm_q4_dense_pair.o ds4_rocm.o ds4_rocm_compat.o ds4_rocm_unavailable.o +tests/test_rocm_q4_dense_pair: tests/test_rocm_q4_dense_pair.o ds4_rocm.o $(ROCM_MMQ_OBJS) ds4_rocm_compat.o ds4_rocm_unavailable.o $(HIPCC) $(ROCM_CFLAGS) -o $@ $^ $(ROCM_LDLIBS) # Keep the public test target usable on development hosts without ROCm. The diff --git a/cuda/mmq/ds4_mmq.cu b/cuda/mmq/ds4_mmq.cu index 30b18b9a3..10008c03e 100644 --- a/cuda/mmq/ds4_mmq.cu +++ b/cuda/mmq/ds4_mmq.cu @@ -7009,7 +7009,13 @@ static int ds4_mmq_pointer_is_device_resident( (void)cudaGetLastError(); return 0; } -#if CUDART_VERSION >= 10000 +#if defined(GGML_USE_HIP) && HIP_VERSION >= 60000000 + return attr.type == cudaMemoryTypeDevice && + attr.device == expected_device; +#elif defined(GGML_USE_HIP) + return attr.memoryType == cudaMemoryTypeDevice && + attr.device == expected_device; +#elif CUDART_VERSION >= 10000 return attr.type == cudaMemoryTypeDevice && attr.device == expected_device; #else diff --git a/rocm/ds4_rocm_attention.cuh b/rocm/ds4_rocm_attention.cuh index acf302241..71aa9389b 100644 --- a/rocm/ds4_rocm_attention.cuh +++ b/rocm/ds4_rocm_attention.cuh @@ -113,8 +113,7 @@ __global__ static void attention_output_b_f16_wmma_64x64_kernel( frag_c acc; rocwmma::fill_fragment(acc, 0.0f); -#pragma unroll - for (uint32_t k0 = 0; k0 < K; k0 + 16u) { + for (uint32_t k0 = 0; k0 < K; k0 += 16u) { const uint32_t ja = tid * 2u; const uint32_t a_row = ja & 63u; const uint32_t a_k = ja >> 6u; From 6a51d69844cd96dadca9972be7211dbd0abec605 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Sat, 29 Aug 2026 21:58:00 +0200 Subject: [PATCH 169/189] Optimize ROCm Q4 prefill WMMA kernels --- ENVIRONMENT_VARIABLES.md | 22 +- rocm/ds4_rocm_q4.cuh | 284 ++++++++++++++--- scripts/environment_variables.tsv | 9 +- speed-bench/README.md | 48 ++- speed-bench/rocm_q4_prefill_bench.cpp | 425 ++++++++++++++++++++++++-- tests/test_rocm_q4_dense_pair.cpp | 97 ++++-- 6 files changed, 767 insertions(+), 118 deletions(-) diff --git a/ENVIRONMENT_VARIABLES.md b/ENVIRONMENT_VARIABLES.md index dc877917d..9287e9252 100644 --- a/ENVIRONMENT_VARIABLES.md +++ b/ENVIRONMENT_VARIABLES.md @@ -93,10 +93,11 @@ The detailed Metal A/B contracts and expected oracle counters live in | `DS4_ROCM_ENABLE_Q4_PREFILL_Q8_K_WAVE32=1` | On gfx1151 wave32, opt into the no-LDS Q8_K activation quantizer that assigns one 256-value block to each wave before the exact Q4 prefill matmul. | | `DS4_ROCM_DISABLE_Q4_PREFILL_Q8_K_WAVE32=1` | Dominant rollback to the canonical one-workgroup-per-Q8_K-block quantizer. | | `DS4_ROCM_REQUIRE_Q4_PREFILL_Q8_K_WAVE32=1` | Require the wave32 quantizer for strict prefill A/B runs; unsupported scope/device, rollback, or an incompatible required F16/WMMA path fails closed. | -| `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA=1` | Opt into the experimental resident gfx1151 wave32 Q4_K WMMA64 prefill kernel for 256–4096 tokens. It replaces Q8_K activation scratch with transient F16 register dequantization and F32 accumulation. | -| `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_SSD=1` | Allow that WMMA64 path during SSD streaming only when the complete projection weight range is already backed by physical device storage. It never treats mapped/registered host memory as resident. | -| `DS4_ROCM_DISABLE_Q4_PREFILL_WMMA=1` | Dominant value-aware rollback for the WMMA64 experiment. | -| `DS4_ROCM_REQUIRE_Q4_PREFILL_WMMA=1` | Request WMMA64 and fail closed on an unsupported device/shape, quality mode, rollback, or SSD weight range that is not physically device-resident. Intended for strict A/B oracles. | +| `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA=1` | Opt into the experimental resident gfx1151 wave32 direct-Q4 WMMA prefill kernel for 256–4096 tokens. It replaces Q8_K activation scratch with transient F16 register dequantization and F32 accumulation, using 64 rows below output dimension 1024, 128 rows below 8192, and 256 rows otherwise; the wider variants use two-wide F32-to-F16 activation staging. | +| `DS4_ROCM_Q4_PREFILL_WMMA_ROW_TILE=64|128|256` | Override the direct-Q4 WMMA output-row tile. The default uses 64 rows below output dimension 1024, 128 below 8192, and 256 otherwise; `64` also retains the prior kernel geometry as an A/B control. | +| `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_SSD=1` | Allow that direct-Q4 WMMA path during SSD streaming only when the complete projection weight range is already backed by physical device storage. It never treats mapped/registered host memory as resident. | +| `DS4_ROCM_DISABLE_Q4_PREFILL_WMMA=1` | Dominant value-aware rollback for the direct-Q4 WMMA experiment. | +| `DS4_ROCM_REQUIRE_Q4_PREFILL_WMMA=1` | Request direct-Q4 WMMA and fail closed on an unsupported device/shape, quality mode, rollback, or SSD weight range that is not physically device-resident. Intended for strict A/B oracles. | | `DS4_ROCM_Q4_PREFILL_TILE8_STATS=1` | Report dense, pair, attention-batch, and token counters at process exit. | | `DS4_ROCM_ENABLE_Q4_DENSE_PAIR=1` | Share one Q8_K activation quantization between the two Q4 dense projections. This pair remains opt-in. | | `DS4_ROCM_DISABLE_Q4_DENSE_PAIR=1` | Dominant rollback for the ROCm Q4 dense pair. | @@ -146,7 +147,7 @@ above, it is an unstable internal diagnostic or tuning interface. The linked sou remains normative for exact eligibility gates, bounds, and architecture-specific defaults. -Inventory totals: **1073 `DS4_*` runtime variables** and +Inventory totals: **1074 `DS4_*` runtime variables** and **6 external runtime variables**. The auxiliary inventories contain **118 test/test-fixture entries** and **19 tool/wrapper entries**. @@ -947,7 +948,7 @@ and **19 tool/wrapper entries**.
-ROCm (144) +ROCm (145) | Variable | Accepted value and default | Effect | Source | | --- | --- | --- | --- | @@ -962,7 +963,7 @@ and **19 tool/wrapper entries**. | `DS4_ROCM_DISABLE_Q4_DENSE_PAIR` | presence rollback; unset leaves opt-in policy unchanged | Disable/roll back rocm disable q4 dense pair. | [rocm/ds4_rocm_q4.cuh:435](rocm/ds4_rocm_q4.cuh#L435) | | `DS4_ROCM_DISABLE_Q4_GROUPED_ATTN_A` | presence rollback; unset permits the caller-marked resident decode production-shape default and explicit ENABLE/REQUIRE; any defined value including empty or 0 disables all grouped attention-A paths and wins over ENABLE/REQUIRE | Restore eight standalone Q4 attention-A projections instead of the two-dispatch grouped path. | [rocm/ds4_rocm_q4.cuh:868](rocm/ds4_rocm_q4.cuh#L868) | | `DS4_ROCM_DISABLE_Q4_PREFILL_TILE8` | presence rollback; TILE8 is default for 9..4096 tokens | Disable/roll back rocm disable q4 prefill tile8. | [rocm/ds4_rocm_q4.cuh:448](rocm/ds4_rocm_q4.cuh#L448) | -| `DS4_ROCM_DISABLE_Q4_PREFILL_WMMA` | value-aware authoritative rollback for the experimental path; unset/0/false/no/off permits ENABLE or REQUIRE, while empty or any other value disables; REQUIRE then fails closed | Prevent the gfx1151 Q4_K prefill WMMA64 experiment from dispatching and retain the Q8_K-plus-TILE8/TILE4 path. | [rocm/ds4_rocm_q4.cuh:775](rocm/ds4_rocm_q4.cuh#L775) | +| `DS4_ROCM_DISABLE_Q4_PREFILL_WMMA` | value-aware authoritative rollback for the experimental path; unset/0/false/no/off permits ENABLE or REQUIRE, while empty or any other value disables; REQUIRE then fails closed | Prevent the gfx1151 direct-Q4 WMMA prefill experiment from dispatching and retain the Q8_K-plus-TILE8/TILE4 path. | [rocm/ds4_rocm_q4.cuh:1022](rocm/ds4_rocm_q4.cuh#L1022) | | `DS4_ROCM_DISABLE_Q4_SELECTED_EXPERT_VIEWS` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable q4 selected expert views. | [ds4.c:21150](ds4.c#L21150) | | `DS4_ROCM_DISABLE_RESIDENT_IQ2_SORTED` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable resident iq2 sorted. | [rocm/ds4_rocm_moe_launch.cuh:751](rocm/ds4_rocm_moe_launch.cuh#L751) | | `DS4_ROCM_DISABLE_ROUTED_PAIR_SWIGLU_FUSION` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable routed pair swiglu fusion. | [ds4.c:18543](ds4.c#L18543) | @@ -999,8 +1000,8 @@ and **19 tool/wrapper entries**. | `DS4_ROCM_ENABLE_MXFP4_TILE4` | presence opt-in; unset=off; any defined value including empty or 0 enables the candidate when the MXFP4 sorted-tile path has at least 5 tokens and neither TILE32 nor LDSB is selected | Select the ROCm MXFP4 gate/up tile4 occupancy variant, reducing staged-activation LDS per block. | [rocm/ds4_rocm_moe_launch.cuh:794](rocm/ds4_rocm_moe_launch.cuh#L794) | | `DS4_ROCM_ENABLE_Q4_DENSE_PAIR` | presence opt-in; unset=off; DISABLE takes precedence | Enable rocm enable q4 dense pair. | [rocm/ds4_rocm_q4.cuh:434](rocm/ds4_rocm_q4.cuh#L434) | | `DS4_ROCM_ENABLE_Q4_GROUPED_ATTN_A` | presence opt-in outside the default scope; the exact caller-marked resident decode shape groups=8, N=1, K=4096, M=1024 is automatic, while row-at-a-time batch fallbacks are not; DISABLE wins | Enable grouped Q4 attention-A for eligible slices, non-production shapes, or explicit experiments in addition to the resident decode default. | [rocm/ds4_rocm_q4.cuh:872](rocm/ds4_rocm_q4.cuh#L872) | -| `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA` | value-aware opt-in, default off; unset/0/false/no/off retains the canonical path; empty or any other value requests WMMA64 only for N=256..4096, K a positive multiple of 256, resident non-quality execution on gfx1151 wave32; DISABLE wins | Benchmark the compressed Q4_K-to-F16 register dequantization plus 64x64 WMMA prefill kernel without Q8_K activation scratch or an F16 weight sidecar. | [rocm/ds4_rocm_q4.cuh:771](rocm/ds4_rocm_q4.cuh#L771) | -| `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_SSD` | value-aware SSD-only opt-in, default off; unset/0/false/no/off retains TILE8/TILE4, while empty or any other value requests WMMA64; eligibility additionally requires the complete projection weight range in physical device storage rather than mapped/registered host memory; DISABLE wins | Allow the compressed WMMA64 kernel to consume an already device-resident/cache-backed Q4_K projection during SSD streaming without changing model I/O. | [rocm/ds4_rocm_q4.cuh:773](rocm/ds4_rocm_q4.cuh#L773) | +| `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA` | value-aware opt-in, default off; unset/0/false/no/off retains the canonical path; empty or any other value requests direct-Q4 WMMA only for N=256..4096, K a positive multiple of 256, resident non-quality execution on gfx1151 wave32; DISABLE wins | Benchmark compressed Q4_K-to-F16 register dequantization plus shape-selected 64-token by 64/128/256-row WMMA tiles and two-wide activation staging on the wider tiles, without Q8_K activation scratch or an F16 weight sidecar. | [rocm/ds4_rocm_q4.cuh:1018](rocm/ds4_rocm_q4.cuh#L1018) | +| `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_SSD` | value-aware SSD-only opt-in, default off; unset/0/false/no/off retains TILE8/TILE4, while empty or any other value requests direct-Q4 WMMA; eligibility additionally requires the complete projection weight range in physical device storage rather than mapped/registered host memory; DISABLE wins | Allow the compressed direct-Q4 WMMA kernel to consume an already device-resident/cache-backed Q4_K projection during SSD streaming without changing model I/O. | [rocm/ds4_rocm_q4.cuh:1020](rocm/ds4_rocm_q4.cuh#L1020) | | `DS4_ROCM_ENABLE_STREAMING_FULL_EXPERT_ADDR_TABLE` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming full expert addr table. | [ds4.c:18255](ds4.c#L18255) | | `DS4_ROCM_ENABLE_STREAMING_MADVISE_WILLNEED` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming madvise willneed. | [ds4.c:18224](ds4.c#L18224) | | `DS4_ROCM_ENABLE_STREAMING_PREFILL_BATCH_SELECTED_ADDR` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming prefill batch selected addr. | [ds4.c:18584](ds4.c#L18584) | @@ -1054,11 +1055,12 @@ and **19 tool/wrapper entries**. | `DS4_ROCM_MXFP4_DOWN_RGROUP` | nonempty value is parsed by strtol and a numeric prefix is sufficient; integers 1..8 are accepted; unset, empty, invalid, or out-of-range values use 1 | Set how many 32-row output blocks each ROCm MXFP4 tiled down-projection block computes, reducing the first launch-grid dimension as the value increases. | [rocm/ds4_rocm_moe_launch.cuh:801](rocm/ds4_rocm_moe_launch.cuh#L801) | | `DS4_ROCM_Q4_GROUPED_ATTN_A_STATS` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Print counters for rocm q4 grouped attn a stats. | [rocm/ds4_rocm_q4.cuh:614](rocm/ds4_rocm_q4.cuh#L614) | | `DS4_ROCM_Q4_PREFILL_TILE8_STATS` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Print counters for rocm q4 prefill tile8 stats. | [rocm/ds4_rocm_q4.cuh:514](rocm/ds4_rocm_q4.cuh#L514) | +| `DS4_ROCM_Q4_PREFILL_WMMA_ROW_TILE` | unsigned integer; unset, empty, malformed, negative, or values other than 64/128/256 use shape selection (64 rows when M<1024, 128 when M<8192, otherwise 256); 64 retains the previous geometry | Override the number of output rows sharing each direct-Q4 64x32 activation tile for controlled 64/128/256-row ROCm WMMA A/B measurements. | [rocm/ds4_rocm_q4.cuh:1126](rocm/ds4_rocm_q4.cuh#L1126) | | `DS4_ROCM_Q8_DECODE_SHAREDX_64K` | sampled once; unset: enabled; present empty or exact 0: disabled; every other present value: enabled; effective only for one-token non-prequant Q8_0 matmul with 8192 < in_dim <= 16384 | Allows the ROCm shared-input Q8 decode kernel to use up to 64 KiB dynamic LDS for wide inputs; an unsupported/failed LDS launch automatically falls back to the regular kernel. | [rocm/ds4_rocm_runtime.cuh:4805](rocm/ds4_rocm_runtime.cuh#L4805) | | `DS4_ROCM_Q_STAGE_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm q stage profile. | [ds4.c:29284](ds4.c#L29284) | | `DS4_ROCM_REQUIRE_Q4_GROUPED_ATTN_A` | presence fail-closed assertion; also requests the candidate outside the caller-marked resident decode default; DISABLE remains authoritative and causes failure | Require grouped Q4 attention-A and fail instead of silently falling back. | [rocm/ds4_rocm_q4.cuh:870](rocm/ds4_rocm_q4.cuh#L870) | | `DS4_ROCM_REQUIRE_Q4_PREFILL_TILE8` | presence fail-closed assertion for eligible TILE8 calls | Require rocm require q4 prefill tile8 and fail instead of silently falling back. | [rocm/ds4_rocm_q4.cuh:452](rocm/ds4_rocm_q4.cuh#L452) | -| `DS4_ROCM_REQUIRE_Q4_PREFILL_WMMA` | value-aware strict opt-in; unset/0/false/no/off is off; empty or any other value requires every selected Q4 dense or attention-output projection to use WMMA64; unsupported shape/device, quality mode, DISABLE, or an SSD weight range without physical device residency fails before the relevant dispatch | Prevent the ROCm Q4 prefill WMMA64 correctness/performance oracle from silently timing TILE8/TILE4; the fused q_a/KV pair yields to separately checked dense calls. | [rocm/ds4_rocm_q4.cuh:777](rocm/ds4_rocm_q4.cuh#L777) | +| `DS4_ROCM_REQUIRE_Q4_PREFILL_WMMA` | value-aware strict opt-in; unset/0/false/no/off is off; empty or any other value requires every selected Q4 dense or attention-output projection to use direct-Q4 WMMA; unsupported shape/device, quality mode, DISABLE, or an SSD weight range without physical device residency fails before the relevant dispatch | Prevent the ROCm Q4 prefill WMMA correctness/performance oracle from silently timing TILE8/TILE4; the fused q_a/KV pair yields to separately checked dense calls. | [rocm/ds4_rocm_q4.cuh:1024](rocm/ds4_rocm_q4.cuh#L1024) | | `DS4_ROCM_STREAMING_DECODE_PREFILL_MAX` | primary nonempty value over the Metal alias; parsed by strtol when it has a numeric prefix (trailing text is accepted); <= 0 disables, values > UINT32_MAX clamp, no numeric prefix uses automatic default: 64 for Flash with uniform Q4_K/MXFP4 experts, 18 for other Pro/Flash, otherwise 0; the disable flag dominates | Sets the largest short, non-quality SSD-streaming prefill batch routed through the decode-style path instead of canonical layer-major prefill. | [ds4.c:31976](ds4.c#L31976) | | `DS4_ROCM_STREAMING_EXPERT_AUTO_PRELOAD_CAP` | primary nonempty value over the Metal alias; strict full-string strtoul; valid values > UINT32_MAX clamp, invalid uses 4096, and 0 means no cap (not disabled); when CLI preload is auto/0, unset defaults to cap 4096 except ROCm GLM52, where absent/empty disables automatic preload entirely | Caps the number of hot experts synchronously seeded into the SSD-streaming expert cache in automatic preload mode; an explicit CLI preload count bypasses this cap, and setting this variable opts ROCm GLM52 back into auto preload. | [ds4.c:21469](ds4.c#L21469) | | `DS4_ROCM_STREAMING_EXPERT_CACHE_VERBOSE` | presence flag; unset=off | Print verbose ROCm streaming expert-cache seed/load diagnostics. | [rocm/ds4_rocm_runtime.cuh:2904](rocm/ds4_rocm_runtime.cuh#L2904) | diff --git a/rocm/ds4_rocm_q4.cuh b/rocm/ds4_rocm_q4.cuh index 11c855c99..38b718363 100644 --- a/rocm/ds4_rocm_q4.cuh +++ b/rocm/ds4_rocm_q4.cuh @@ -124,14 +124,18 @@ static_assert((sizeof(cuda_block_q8_K) % sizeof(uint32_t)) == 0u, "ROCm Q8_K LDS copies require a whole number of words"); #if defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__) -/* Experimental compressed Q4_K MMQ for resident gfx1151 prefill. +/* Experimental direct-Q4 WMMA for resident gfx1151 prefill. * - * This deliberately mirrors the live Q8 WMMA kernel: four wave32s compute a - * 64-token x 64-row output tile, K advances by 32, only the 64x32 activation - * tile is staged in 4 KiB of LDS, and accumulator fragments are written - * straight to the output. Each lane dequantizes one Q4_K row/group directly - * into two F16 register vectors, so there is no Q8_K activation scratch and no - * persistent F16 weight sidecar. + * This deliberately mirrors the live Q8 WMMA kernel: each wave32 owns 16 + * output rows, the workgroup computes 64 tokens, K advances by 32, only the + * 64x32 activation tile is staged in 4 KiB of LDS, and accumulator fragments + * are written straight to the output. The row tile is shape-selected to 64, + * 128, or 256 so every activation tile is reused as broadly as the output + * shape permits; the retained 64-row instantiation is also an A/B control. + * The wider candidates can stage two adjacent F32 activations per iteration, + * matching the established Q8 WMMA load/conversion pattern. Each lane + * dequantizes one Q4_K row/group directly into two F16 register vectors, so + * there is no Q8_K activation scratch and no persistent F16 weight sidecar. * * Arithmetic is not bit-identical to Q4_K x Q8_K: activations and transient * weights round to F16 before F32 WMMA accumulation. Host policy therefore @@ -139,19 +143,27 @@ static_assert((sizeof(cuda_block_q8_K) % sizeof(uint32_t)) == 0u, * quality mode until device-side A/B plus a prompt-level oracle promote it. */ enum { - ROCM_Q4_WMMA_M_TILE = 64u, - ROCM_Q4_WMMA_N_TILE = 64u, + ROCM_Q4_WMMA_TOKEN_TILE = 64u, ROCM_Q4_WMMA_K_TILE = 32u, - ROCM_Q4_WMMA_WAVES = 4u, ROCM_Q4_WMMA_FRAGMENT = 16u, }; +#if defined(__HIP_DEVICE_COMPILE__) && __HIP_DEVICE_COMPILE__ && \ + defined(__gfx1151__) && \ + (!defined(__AMDGCN_WAVEFRONT_SIZE__) || \ + __AMDGCN_WAVEFRONT_SIZE__ == 32) +#define DS4_ROCM_Q4_GFX1151_WMMA_ROWTILE_DEVICE 1 typedef _Float16 __attribute__((ext_vector_type(16))) ds4_q4_half16_t; typedef float __attribute__((ext_vector_type(8))) ds4_q4_float8_t; typedef uint8_t __attribute__((ext_vector_type(16))) ds4_q4_uchar16_t; +#else +#define DS4_ROCM_Q4_GFX1151_WMMA_ROWTILE_DEVICE 0 +#endif -__launch_bounds__(128, 2) -__global__ static void rocm_matmul_q4_K_prefill_wmma64_strided_kernel( +template +__launch_bounds__(WAVES * 32u, MIN_BLOCKS) +__global__ static void rocm_matmul_q4_K_prefill_wmma_rowtile_strided_kernel( float *out, const char *w_base, const float *x, @@ -163,6 +175,7 @@ __global__ static void rocm_matmul_q4_K_prefill_wmma64_strided_kernel( uint64_t x_token_stride, uint64_t x_group_stride, uint64_t out_token_stride) { +#if DS4_ROCM_Q4_GFX1151_WMMA_ROWTILE_DEVICE if (warpSize != 32) return; const uint32_t tid = threadIdx.x; @@ -170,8 +183,10 @@ __global__ static void rocm_matmul_q4_K_prefill_wmma64_strided_kernel( const uint32_t lane = tid & 31u; const uint32_t lane16 = lane & 15u; const uint32_t group = blockIdx.z; - const uint32_t row0 = blockIdx.x * ROCM_Q4_WMMA_N_TILE; - const uint32_t tok0 = blockIdx.y * ROCM_Q4_WMMA_M_TILE; + static_assert(ROW_TILE == WAVES * ROCM_Q4_WMMA_FRAGMENT, + "one Q4 WMMA wave must own exactly 16 output rows"); + const uint32_t row0 = blockIdx.x * ROW_TILE; + const uint32_t tok0 = blockIdx.y * ROCM_Q4_WMMA_TOKEN_TILE; if (group >= n_groups) return; const uint32_t wave_row0 = row0 + wave * ROCM_Q4_WMMA_FRAGMENT; @@ -187,7 +202,10 @@ __global__ static void rocm_matmul_q4_K_prefill_wmma64_strided_kernel( ds4_q4_float8_t acc1 = acc0; ds4_q4_float8_t acc2 = acc0; ds4_q4_float8_t acc3 = acc0; - __shared__ _Float16 lds_x[ROCM_Q4_WMMA_M_TILE * ROCM_Q4_WMMA_K_TILE]; + /* The 16-lane _Float16 vector loads below have 32-byte alignment on the + * AMDGPU target; half2 itself would require only four bytes. */ + __shared__ __align__(32) _Float16 + lds_x[ROCM_Q4_WMMA_TOKEN_TILE * ROCM_Q4_WMMA_K_TILE]; for (uint32_t block_index = 0u; block_index < q4_blocks; block_index++) { @@ -212,20 +230,43 @@ __global__ static void rocm_matmul_q4_K_prefill_wmma64_strided_kernel( for (uint32_t nibble = 0u; nibble < 2u; nibble++) { const uint32_t qgroup = qpair * 2u + nibble; const uint32_t group32 = block_index * 8u + qgroup; - for (uint32_t j = tid; - j < ROCM_Q4_WMMA_M_TILE * ROCM_Q4_WMMA_K_TILE; - j += blockDim.x) { - const uint32_t tok_local = j >> 5u; - const uint32_t kk = j & 31u; - const uint32_t tok = tok0 + tok_local; - float value = 0.0f; - if (tok < n_tok) { - value = x[(uint64_t)tok * x_token_stride + - (uint64_t)group * x_group_stride + - (uint64_t)group32 * ROCM_Q4_WMMA_K_TILE + - kk]; + if constexpr (LOAD2) { + for (uint32_t j = tid * 2u; + j < ROCM_Q4_WMMA_TOKEN_TILE * ROCM_Q4_WMMA_K_TILE; + j += blockDim.x * 2u) { + const uint32_t tok_local = j >> 5u; + const uint32_t kk = j & 31u; + const uint32_t tok = tok0 + tok_local; + half2 value = __floats2half2_rn(0.0f, 0.0f); + if (tok < n_tok) { + const float2 pair = + *reinterpret_cast( + x + (uint64_t)tok * x_token_stride + + (uint64_t)group * x_group_stride + + (uint64_t)group32 * + ROCM_Q4_WMMA_K_TILE + + kk); + value = __floats2half2_rn(pair.x, pair.y); + } + *reinterpret_cast(lds_x + j) = value; + } + } else { + for (uint32_t j = tid; + j < ROCM_Q4_WMMA_TOKEN_TILE * ROCM_Q4_WMMA_K_TILE; + j += blockDim.x) { + const uint32_t tok_local = j >> 5u; + const uint32_t kk = j & 31u; + const uint32_t tok = tok0 + tok_local; + float value = 0.0f; + if (tok < n_tok) { + value = x[(uint64_t)tok * x_token_stride + + (uint64_t)group * x_group_stride + + (uint64_t)group32 * + ROCM_Q4_WMMA_K_TILE + + kk]; + } + lds_x[j] = (_Float16)value; } - lds_x[j] = (_Float16)value; } __syncthreads(); @@ -303,7 +344,21 @@ __global__ static void rocm_matmul_q4_K_prefill_wmma64_strided_kernel( } } } +#else + (void)out; + (void)w_base; + (void)x; + (void)n_tok; + (void)n_groups; + (void)in_dim; + (void)out_dim; + (void)row_bytes; + (void)x_token_stride; + (void)x_group_stride; + (void)out_token_stride; +#endif } +#undef DS4_ROCM_Q4_GFX1151_WMMA_ROWTILE_DEVICE #endif __device__ __forceinline__ static void @@ -1001,6 +1056,94 @@ static int rocm_q4_K_prefill_wmma_select( return ROCM_Q4_PREFILL_WMMA_REQUIRED_FAILURE; } +static int rocm_q4_K_prefill_wmma_load2_compatible( + const float *x, + uint64_t x_token_stride, + uint64_t x_group_stride) { + return x && (((uintptr_t)x & 7u) == 0u) && + ((x_token_stride & 1u) == 0u) && + ((x_group_stride & 1u) == 0u); +} + +static void rocm_q4_K_prefill_wmma_enqueue( + float *out, + const char *w, + const float *x, + uint32_t n_tok, + uint32_t n_groups, + uint32_t in_dim, + uint32_t out_dim, + uint64_t row_bytes, + uint64_t x_token_stride, + uint64_t x_group_stride, + uint64_t out_token_stride, + uint32_t row_tile, + int load2) { + const dim3 grid( + (unsigned)(((uint64_t)out_dim + row_tile - 1u) / row_tile), + (unsigned)(((uint64_t)n_tok + ROCM_Q4_WMMA_TOKEN_TILE - 1u) / + ROCM_Q4_WMMA_TOKEN_TILE), + n_groups); + if (row_tile == 256u) { + if (load2) { + rocm_matmul_q4_K_prefill_wmma_rowtile_strided_kernel< + 256u, 16u, 1u, true><<>>( + out, w, x, n_tok, n_groups, in_dim, out_dim, row_bytes, + x_token_stride, x_group_stride, out_token_stride); + } else { + rocm_matmul_q4_K_prefill_wmma_rowtile_strided_kernel< + 256u, 16u, 1u, false><<>>( + out, w, x, n_tok, n_groups, in_dim, out_dim, row_bytes, + x_token_stride, x_group_stride, out_token_stride); + } + } else if (row_tile == 128u) { + if (load2) { + rocm_matmul_q4_K_prefill_wmma_rowtile_strided_kernel< + 128u, 8u, 1u, true><<>>( + out, w, x, n_tok, n_groups, in_dim, out_dim, row_bytes, + x_token_stride, x_group_stride, out_token_stride); + } else { + rocm_matmul_q4_K_prefill_wmma_rowtile_strided_kernel< + 128u, 8u, 1u, false><<>>( + out, w, x, n_tok, n_groups, in_dim, out_dim, row_bytes, + x_token_stride, x_group_stride, out_token_stride); + } + } else { + /* Preserve the original kernel's occupancy contract so the retained + * 64-row benchmark arm differs only in row geometry. */ + rocm_matmul_q4_K_prefill_wmma_rowtile_strided_kernel< + 64u, 4u, 2u, false> + <<>>( + out, w, x, n_tok, n_groups, in_dim, out_dim, row_bytes, + x_token_stride, x_group_stride, out_token_stride); + } +} + +static uint32_t rocm_q4_K_prefill_wmma_row_tile(uint32_t out_dim) { + const uint32_t shape_tile = out_dim >= 8192u + ? 256u + : (out_dim >= 1024u ? 128u : 64u); + const char *raw = getenv("DS4_ROCM_Q4_PREFILL_WMMA_ROW_TILE"); + if (raw) { + while (isspace((unsigned char)*raw)) raw++; + /* strtoull accepts a leading minus and wraps some values into the + * whitelist. Treat every negative spelling as an invalid unsigned + * override so the benchmark cannot silently select the wrong arm. */ + if (*raw == '-') return shape_tile; + } + const uint64_t requested = rocm_q4_attn_q_b_env_u64( + "DS4_ROCM_Q4_PREFILL_WMMA_ROW_TILE", 0u, 0u, UINT64_MAX); + if (requested == 64u || requested == 128u || requested == 256u) { + return (uint32_t)requested; + } + return shape_tile; +} + +extern "C" uint32_t ds4_rocm_test_q4_prefill_wmma_row_tile( + uint32_t out_dim) { + return rocm_q4_K_prefill_wmma_row_tile(out_dim); +} + static int rocm_q4_K_prefill_wmma_launch( float *out, const char *w, @@ -1015,17 +1158,21 @@ static int rocm_q4_K_prefill_wmma_launch( uint64_t out_token_stride, const char *label) { if (!out || !w || !x || n_groups == 0u) return 0; - const dim3 grid( - (unsigned)(((uint64_t)out_dim + ROCM_Q4_WMMA_N_TILE - 1u) / - ROCM_Q4_WMMA_N_TILE), - (unsigned)(((uint64_t)n_tok + ROCM_Q4_WMMA_M_TILE - 1u) / - ROCM_Q4_WMMA_M_TILE), - n_groups); - rocm_matmul_q4_K_prefill_wmma64_strided_kernel<<>>( + + /* Match the proven Q8 row geometry so a 64x32 activation tile is loaded + * once for 128 rows, or once for 256 rows on the large q_b/output shapes. + * Small projections retain 64 rows instead of launching mostly idle waves; + * that instantiation is also the deterministic previous-kernel A/B control. + * Invalid overrides deliberately fall back to shape selection. */ + const uint32_t row_tile = rocm_q4_K_prefill_wmma_row_tile(out_dim); + const int load2 = row_tile != 64u && + rocm_q4_K_prefill_wmma_load2_compatible( + x, x_token_stride, x_group_stride); + rocm_q4_K_prefill_wmma_enqueue( out, w, x, n_tok, n_groups, in_dim, out_dim, row_bytes, - x_token_stride, x_group_stride, out_token_stride); + x_token_stride, x_group_stride, out_token_stride, row_tile, load2); const int ok = cuda_ok( - cudaGetLastError(), label ? label : "q4_K prefill WMMA64 launch"); + cudaGetLastError(), label ? label : "q4_K prefill WMMA rowtile launch"); if (ok) { __atomic_fetch_add(&g_rocm_q4_prefill_wmma_launches, 1u, __ATOMIC_RELAXED); @@ -1033,6 +1180,61 @@ static int rocm_q4_K_prefill_wmma_launch( return ok; } +/* Resident, enqueue-only hooks for the ROCm microbenchmark. Pointer lookup, + * policy, environment parsing, and launch-error reporting stay outside HIP + * event intervals; the explicit tile and loader arguments make each arm + * self-attesting instead of relying on a mutable process environment. */ +extern "C" const void *ds4_rocm_bench_q4_K_resident_weight_ptr( + const void *model_map, + uint64_t model_size, + uint64_t weight_offset, + uint64_t weight_bytes) { + if (!cuda_model_range_fits(model_size, weight_offset, weight_bytes)) { + return NULL; + } + return rocm_q4_attn_q_b_device_resident_source( + model_map, weight_offset, weight_bytes); +} + +extern "C" int ds4_rocm_bench_q4_K_wmma_enqueue( + void *out, + const void *w, + const void *x, + uint32_t n_tok, + uint32_t n_groups, + uint32_t in_dim, + uint32_t out_dim, + uint64_t row_bytes, + uint64_t x_token_stride, + uint64_t x_group_stride, + uint64_t out_token_stride, + uint32_t row_tile, + int load2) { + if (!out || !w || !x || n_tok == 0u || n_groups == 0u || + in_dim == 0u || out_dim == 0u || + (in_dim % CUDA_QK_K) != 0u || + (load2 != 0 && load2 != 1) || + (load2 && row_tile == 64u) || + (row_tile != 64u && row_tile != 128u && row_tile != 256u)) { + return 0; + } + const uint64_t minimum_row_bytes = + ((uint64_t)in_dim / CUDA_QK_K) * sizeof(cuda_block_q4_K); + if (row_bytes < minimum_row_bytes || + (load2 && !rocm_q4_K_prefill_wmma_load2_compatible( + reinterpret_cast(x), + x_token_stride, x_group_stride))) { + return 0; + } + rocm_q4_K_prefill_wmma_enqueue( + reinterpret_cast(out), + reinterpret_cast(w), + reinterpret_cast(x), + n_tok, n_groups, in_dim, out_dim, row_bytes, + x_token_stride, x_group_stride, out_token_stride, row_tile, load2); + return 1; +} + enum { ROCM_Q4_PREFILL_K1024_TILE4_REQUIRED_FAILURE = -1, ROCM_Q4_PREFILL_K1024_TILE4_FALLBACK = 0, @@ -1336,7 +1538,7 @@ extern "C" int ds4_rocm_matmul_q4_K_tensor( reinterpret_cast(x->ptr), (uint32_t)n_tok, 1u, (uint32_t)in_dim, (uint32_t)out_dim, row_bytes, in_dim, 0u, out_dim, - "q4_K dense prefill WMMA64 launch"); + "q4_K dense prefill WMMA rowtile launch"); } int k1024_tile4 = ROCM_Q4_PREFILL_K1024_TILE4_FALLBACK; if (prefill_scope && prefill_tile8) { @@ -1695,7 +1897,7 @@ static int rocm_q4_K_prefill_tile8_quant_launch( return rocm_q4_K_prefill_wmma_launch( out, w, x, n_tok, n_groups, in_dim, out_dim, row_bytes, x_token_stride, in_dim, out_token_stride, - label ? label : "q4_K attention-output WMMA64 launch") + label ? label : "q4_K attention-output WMMA rowtile launch") ? 1 : -1; } @@ -1908,7 +2110,7 @@ extern "C" int ds4_gpu_attention_output_q4_K_batch_tensor( reinterpret_cast(heads->ptr), n_tokens, n_groups, (uint32_t)group_dim, (uint32_t)rank, row_a_bytes, a_wmma, q8_wave32, q8_wave32_required, - "q4_K attention output A WMMA64/tile8"); + "q4_K attention output A WMMA rowtile/tile8"); if (a_rc <= 0) { return a_rc < 0 ? -1 : pre_enqueue_failure; } @@ -1920,7 +2122,7 @@ extern "C" int ds4_gpu_attention_output_q4_K_batch_tensor( reinterpret_cast(low->ptr), n_tokens, 1u, (uint32_t)low_dim, (uint32_t)out_dim, row_b_bytes, b_wmma, q8_wave32, q8_wave32_required, - "q4_K attention output B WMMA64/tile8"); + "q4_K attention output B WMMA rowtile/tile8"); } else { b_rc = ds4_gpu_matmul_q8_0_tensor( out, model_map, model_size, out_b_offset, low_dim, out_dim, diff --git a/scripts/environment_variables.tsv b/scripts/environment_variables.tsv index a4c84ba99..d8d13395f 100644 --- a/scripts/environment_variables.tsv +++ b/scripts/environment_variables.tsv @@ -975,7 +975,7 @@ runtime/rocm DS4_ROCM_DISABLE_Q4_GROUPED_ATTN_A presence rollback; unset permits runtime/rocm DS4_ROCM_DISABLE_Q4_PREFILL_K1024_TILE4 value-aware authoritative rollback; unset/0/false/no/off preserves the resident automatic default and any explicit SSD request; empty or any other value disables; overrides ENABLE and causes REQUIRE to fail closed Restore the generic eight-block Q4_K tiled-prefill kernel for K=1024 in both resident and SSD-streaming execution. rocm/ds4_rocm_q4.cuh:624 runtime/rocm DS4_ROCM_DISABLE_Q4_PREFILL_Q8_K_WAVE32 value-aware authoritative rollback; unset/0/false/no/off permits ENABLE or REQUIRE, while empty or any other value disables; REQUIRE then fails closed Restore the canonical one-workgroup-per-Q8_K-block activation quantizer for Q4 prefill. rocm/ds4_rocm_q4.cuh:803 runtime/rocm DS4_ROCM_DISABLE_Q4_PREFILL_TILE8 presence rollback; TILE8 is default for 9..4096 tokens Disable/roll back rocm disable q4 prefill tile8. rocm/ds4_rocm_q4.cuh:448 -runtime/rocm DS4_ROCM_DISABLE_Q4_PREFILL_WMMA value-aware authoritative rollback for the experimental path; unset/0/false/no/off permits ENABLE or REQUIRE, while empty or any other value disables; REQUIRE then fails closed Prevent the gfx1151 Q4_K prefill WMMA64 experiment from dispatching and retain the Q8_K-plus-TILE8/TILE4 path. rocm/ds4_rocm_q4.cuh:775 +runtime/rocm DS4_ROCM_DISABLE_Q4_PREFILL_WMMA value-aware authoritative rollback for the experimental path; unset/0/false/no/off permits ENABLE or REQUIRE, while empty or any other value disables; REQUIRE then fails closed Prevent the gfx1151 direct-Q4 WMMA prefill experiment from dispatching and retain the Q8_K-plus-TILE8/TILE4 path. rocm/ds4_rocm_q4.cuh:1022 runtime/rocm DS4_ROCM_DISABLE_Q4_SELECTED_EXPERT_VIEWS presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable q4 selected expert views. ds4.c:21150 runtime/rocm DS4_ROCM_DISABLE_RESIDENT_IQ2_SORTED presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable resident iq2 sorted. rocm/ds4_rocm_moe_launch.cuh:751 runtime/rocm DS4_ROCM_DISABLE_ROUTED_PAIR_SWIGLU_FUSION presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable routed pair swiglu fusion. ds4.c:18543 @@ -1019,8 +1019,8 @@ runtime/rocm DS4_ROCM_ENABLE_Q4_DENSE_PAIR presence opt-in; unset=off; DISABLE t runtime/rocm DS4_ROCM_ENABLE_Q4_GROUPED_ATTN_A presence opt-in outside the default scope; the exact caller-marked resident decode shape groups=8, N=1, K=4096, M=1024 is automatic, while row-at-a-time batch fallbacks are not; DISABLE wins Enable grouped Q4 attention-A for eligible slices, non-production shapes, or explicit experiments in addition to the resident decode default. rocm/ds4_rocm_q4.cuh:872 runtime/rocm DS4_ROCM_ENABLE_Q4_PREFILL_K1024_TILE4_SSD value-aware SSD-only opt-in, default off; unset/0/false/no/off retains TILE8, while empty or any other value requests TILE4; eligibility additionally requires N=9..4096, K=1024, M=32768, TILE8 enabled, and the complete weight range in device storage rather than mapped/registered host memory; DISABLE wins Allow the four-lane K=1024 Q4_K prefill specialization to consume an already device-resident/cache-backed attn_q_b weight range during SSD streaming without changing model I/O. rocm/ds4_rocm_q4.cuh:624 runtime/rocm DS4_ROCM_ENABLE_Q4_PREFILL_Q8_K_WAVE32 value-aware opt-in, default off; unset/0/false/no/off retains the canonical quantizer, while empty or any other value requests the candidate for N=9..4096 on gfx1151 wave32; DISABLE wins Quantize eight independent Q8_K activation blocks per 256-thread workgroup using one wave32 per block, without LDS or workgroup barriers, before the exact Q4 prefill matmul (TILE8 or its legacy rollback). rocm/ds4_rocm_q4.cuh:801 -runtime/rocm DS4_ROCM_ENABLE_Q4_PREFILL_WMMA value-aware opt-in, default off; unset/0/false/no/off retains the canonical path; empty or any other value requests WMMA64 only for N=256..4096, K a positive multiple of 256, resident non-quality execution on gfx1151 wave32; DISABLE wins Benchmark the compressed Q4_K-to-F16 register dequantization plus 64x64 WMMA prefill kernel without Q8_K activation scratch or an F16 weight sidecar. rocm/ds4_rocm_q4.cuh:771 -runtime/rocm DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_SSD value-aware SSD-only opt-in, default off; unset/0/false/no/off retains TILE8/TILE4, while empty or any other value requests WMMA64; eligibility additionally requires the complete projection weight range in physical device storage rather than mapped/registered host memory; DISABLE wins Allow the compressed WMMA64 kernel to consume an already device-resident/cache-backed Q4_K projection during SSD streaming without changing model I/O. rocm/ds4_rocm_q4.cuh:773 +runtime/rocm DS4_ROCM_ENABLE_Q4_PREFILL_WMMA value-aware opt-in, default off; unset/0/false/no/off retains the canonical path; empty or any other value requests direct-Q4 WMMA only for N=256..4096, K a positive multiple of 256, resident non-quality execution on gfx1151 wave32; DISABLE wins Benchmark compressed Q4_K-to-F16 register dequantization plus shape-selected 64-token by 64/128/256-row WMMA tiles and two-wide activation staging on the wider tiles, without Q8_K activation scratch or an F16 weight sidecar. rocm/ds4_rocm_q4.cuh:1018 +runtime/rocm DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_SSD value-aware SSD-only opt-in, default off; unset/0/false/no/off retains TILE8/TILE4, while empty or any other value requests direct-Q4 WMMA; eligibility additionally requires the complete projection weight range in physical device storage rather than mapped/registered host memory; DISABLE wins Allow the compressed direct-Q4 WMMA kernel to consume an already device-resident/cache-backed Q4_K projection during SSD streaming without changing model I/O. rocm/ds4_rocm_q4.cuh:1020 runtime/rocm DS4_ROCM_ENABLE_STREAMING_FULL_EXPERT_ADDR_TABLE presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming full expert addr table. ds4.c:18255 runtime/rocm DS4_ROCM_ENABLE_STREAMING_MADVISE_WILLNEED presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming madvise willneed. ds4.c:18224 runtime/rocm DS4_ROCM_ENABLE_STREAMING_PREFILL_BATCH_SELECTED_ADDR presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming prefill batch selected addr. ds4.c:18584 @@ -1078,6 +1078,7 @@ runtime/rocm DS4_ROCM_Q4_ATTN_Q_B_F16_CACHE_MIN_TOKENS integer token count with runtime/rocm DS4_ROCM_Q4_ATTN_Q_B_TRANSIENT_F16_MIN_TOKENS full-string unsigned token count; default 4096; accepted range 32..UINT32_MAX; values above clamp and invalid or smaller values restore 4096 Set the minimum device-resident, non-SSD prefill batch eligible for per-layer transient ROCm Q4_K attn_q_b-to-F16 expansion. rocm/ds4_rocm_q4_qb_sidecar.cuh:150 runtime/rocm DS4_ROCM_Q4_GROUPED_ATTN_A_STATS presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Print counters for rocm q4 grouped attn a stats. rocm/ds4_rocm_q4.cuh:614 runtime/rocm DS4_ROCM_Q4_PREFILL_TILE8_STATS presence diagnostic; unset=off; any defined value including empty or 0 prints at exit Print tiled-prefill dense/pair/attention counters plus total and SSD-specific K=1024 TILE4 dispatch counts. rocm/ds4_rocm_q4.cuh:741 +runtime/rocm DS4_ROCM_Q4_PREFILL_WMMA_ROW_TILE unsigned integer; unset, empty, malformed, negative, or values other than 64/128/256 use shape selection (64 rows when M<1024, 128 when M<8192, otherwise 256); 64 retains the previous geometry Override the number of output rows sharing each direct-Q4 64x32 activation tile for controlled 64/128/256-row ROCm WMMA A/B measurements. rocm/ds4_rocm_q4.cuh:1126 runtime/rocm DS4_ROCM_Q8_DECODE_SHAREDX_64K sampled once; unset: enabled; present empty or exact 0: disabled; every other present value: enabled; effective only for one-token non-prequant Q8_0 matmul with 8192 < in_dim <= 16384 Allows the ROCm shared-input Q8 decode kernel to use up to 64 KiB dynamic LDS for wide inputs; an unsupported/failed LDS launch automatically falls back to the regular kernel. rocm/ds4_rocm_runtime.cuh:4805 runtime/rocm DS4_ROCM_Q_STAGE_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm q stage profile. ds4.c:30038 runtime/rocm DS4_ROCM_REQUIRE_Q4_ATTN_Q_B_F16_CACHE value-aware strict opt-in, default off; unset/0/false/no/off is off, empty or any other value requires eligible batches to use the cache; DISABLE wins Fail an eligible ROCm prefill instead of falling back when the resident Q4_K attn_q_b F16 specialization cannot be prepared or dispatched. rocm/ds4_rocm_q4_qb_sidecar.cuh:135 @@ -1085,7 +1086,7 @@ runtime/rocm DS4_ROCM_REQUIRE_Q4_GROUPED_ATTN_A presence fail-closed assertion; runtime/rocm DS4_ROCM_REQUIRE_Q4_PREFILL_K1024_TILE4 value-aware fail-closed assertion and SSD opt-in; unset/0/false/no/off is off; empty or any other value requires eligible N=9..4096, K=1024, M=32768 dense calls to select TILE4; SSD also requires an actual device-resident weight range; DISABLE wins Prevent a K=1024 TILE4 correctness/performance oracle from silently falling back to TILE8, including during SSD-streaming A/B runs. rocm/ds4_rocm_q4.cuh:628 runtime/rocm DS4_ROCM_REQUIRE_Q4_PREFILL_Q8_K_WAVE32 value-aware strict opt-in; unset/0/false/no/off is off; empty or any other value requires gfx1151 wave32 and N=9..4096; DISABLE and conflicts with required WMMA or the required q_b F16 cache fail closed Require the no-LDS wave32 Q8_K activation quantizer for exact Q4 prefill instead of silently using the canonical quantizer or an F16 side path. rocm/ds4_rocm_q4.cuh:796 runtime/rocm DS4_ROCM_REQUIRE_Q4_PREFILL_TILE8 presence fail-closed assertion for eligible TILE8 calls Require rocm require q4 prefill tile8 and fail instead of silently falling back. rocm/ds4_rocm_q4.cuh:452 -runtime/rocm DS4_ROCM_REQUIRE_Q4_PREFILL_WMMA value-aware strict opt-in; unset/0/false/no/off is off; empty or any other value requires every selected Q4 dense or attention-output projection to use WMMA64; unsupported shape/device, quality mode, DISABLE, or an SSD weight range without physical device residency fails before the relevant dispatch Prevent the ROCm Q4 prefill WMMA64 correctness/performance oracle from silently timing TILE8/TILE4; the fused q_a/KV pair yields to separately checked dense calls. rocm/ds4_rocm_q4.cuh:777 +runtime/rocm DS4_ROCM_REQUIRE_Q4_PREFILL_WMMA value-aware strict opt-in; unset/0/false/no/off is off; empty or any other value requires every selected Q4 dense or attention-output projection to use direct-Q4 WMMA; unsupported shape/device, quality mode, DISABLE, or an SSD weight range without physical device residency fails before the relevant dispatch Prevent the ROCm Q4 prefill WMMA correctness/performance oracle from silently timing TILE8/TILE4; the fused q_a/KV pair yields to separately checked dense calls. rocm/ds4_rocm_q4.cuh:1024 runtime/rocm DS4_ROCM_STREAMING_DECODE_PREFILL_MAX primary nonempty value over the Metal alias; parsed by strtol when it has a numeric prefix (trailing text is accepted); <= 0 disables, values > UINT32_MAX clamp, no numeric prefix uses automatic default: 64 for Flash with uniform Q4_K/MXFP4 experts, 18 for other Pro/Flash, otherwise 0; the disable flag dominates Sets the largest short, non-quality SSD-streaming prefill batch routed through the decode-style path instead of canonical layer-major prefill. ds4.c:31976 runtime/rocm DS4_ROCM_STREAMING_EXPERT_AUTO_PRELOAD_CAP primary nonempty value over the Metal alias; strict full-string strtoul; valid values > UINT32_MAX clamp, invalid uses 4096, and 0 means no cap (not disabled); when CLI preload is auto/0, unset defaults to cap 4096 except ROCm GLM52, where absent/empty disables automatic preload entirely Caps the number of hot experts synchronously seeded into the SSD-streaming expert cache in automatic preload mode; an explicit CLI preload count bypasses this cap, and setting this variable opts ROCm GLM52 back into auto preload. ds4.c:21469 runtime/rocm DS4_ROCM_STREAMING_EXPERT_CACHE_VERBOSE presence flag; unset=off Print verbose ROCm streaming expert-cache seed/load diagnostics. rocm/ds4_rocm_runtime.cuh:2904 diff --git a/speed-bench/README.md b/speed-bench/README.md index a543df68c..8b63662eb 100644 --- a/speed-bench/README.md +++ b/speed-bench/README.md @@ -156,6 +156,9 @@ make rocm-q4-prefill-bench ROCM_ARCH=gfx1151 The fixture copies four rotating sets of synthetic GGUF-layout Q4_K weights to device memory before warmup and forces SSD streaming off. HIP events then measure only activation conversion/quantization and projection kernels. The +row-geometry and activation-loader comparisons use resident pointers plus an +explicit enqueue-only hook, excluding environment parsing, model lookup, and +policy selection. The comparisons are: - `dense`: legacy versus TILE8 at the Flash Q-A `K=4096,M=1024` shape; @@ -163,38 +166,53 @@ comparisons are: `K=4096,M=(1024+512)` path; - `qb`: TILE8 versus TILE4 at the production `attn_q_b` `K=1024,M=32768` shape. -- `outb`: TILE8 versus the experimental compressed WMMA64 kernel at the +- `outb`: TILE8 versus the experimental compressed direct-Q4 WMMA kernel at the production `output_b` `K=8192,M=4096` shape; - `output`: the complete grouped `output_a` plus `output_b` production API, - comparing two TILE8 projections with two direct WMMA64 projections. + comparing two TILE8 projections with two direct WMMA projections. -On gfx1151 wave32, `dense` and `qb` also emit a second WMMA64 comparison for +On gfx1151 wave32, `dense` and `qb` also emit a direct-Q4 WMMA comparison for `N>=256`. The candidate keeps Q4_K weights compressed, rounds each transient 32-value weight group and the activation tile to F16 in the kernel, accumulates through WMMA in F32, and avoids both Q8_K activation scratch and persistent F16 -weight sidecars. It remains opt-in in the runtime; the benchmark uses the -strict REQUIRE gate so a rejected dispatch fails instead of timing a fallback. +weight sidecars. Its shape-selected row tile uses 64 rows below output dimension +1024, 128 below 8192, and 256 otherwise; the wider variants stage two adjacent +F32 activations into one F16 pair, matching the established Q8 WMMA loader. +`dense`, `outb`, and `output` also emit a bit-exact 64-row versus shape-selected +scalar-loader A/B. The large `q_b` shape instead reports adjacent scalar-loader +64→128 and 128→256 comparisons, so +register pressure in the 512-thread candidate cannot hide a better midpoint. +These direct comparisons measure the net effect of the broader row geometry, +including its changed workgroup and occupancy contract, separately from the +candidate's arithmetic change versus TILE8. `q_b` additionally compares scalar +versus two-wide activation staging at fixed 128- and 256-row geometry. +The direct hook receives both tile and loader explicitly, so every arm attests +its own configuration. It remains opt-in in the runtime; the production-API +comparison uses the strict REQUIRE gate so a rejected dispatch fails instead +of timing a fallback. The benchmark always keeps SSD streaming disabled. Runtime SSD experiments need the separate `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_SSD=1` gate and only accept projection ranges already backed by physical device memory, so model I/O is never folded into the kernel result. -The default token set is `9,17,33,128,257,512`, covering the first row after -the small TILE8 boundaries and the first token after a 64-token WMMA boundary. -Use `--full` for `9,16,17,31,32,33,128,257,512,4096`, or select a focused run -such as: +The default token set is `9,17,33,128,256,257,512`, covering the first row +after the small TILE8 boundaries plus both the exact 256-token occupancy case +and the first token after a 64-token WMMA boundary. Use `--full` for +`9,16,17,31,32,33,128,256,257,512,4096`, or select a focused run such as: ``` ./speed-bench/rocm_q4_prefill_bench \ - --case output --tokens 256,257,512,1024,2048,4096 --samples 8 + --case qb --tokens 256,257,512,1024,2048,2049,4096 \ + --sets 4 --warmup 4 --samples 12 ``` Every case rotates identical resident weight sets between arms, alternates -ABBA/BAAB, and verifies allocation guards. Comparisons among the integer Q4 -paths remain bit-exact. WMMA64 has a deliberate F16 arithmetic boundary, so -those comparisons require finite results within an explicit absolute/relative -smoke tolerance; a model-logit or prompt oracle is still required before -enabling the kernel by default. +ABBA/BAAB, and verifies allocation guards both before and after timing. +Comparisons among the integer Q4 +paths remain bit-exact. Direct-Q4 WMMA has a deliberate F16 arithmetic +boundary, so those comparisons require finite results within an explicit +absolute/relative smoke tolerance; a model-logit or prompt oracle is still +required before enabling the kernel by default. Fixture creation, the host-to-device residency copy, warmup, oracle readback, and environment-gate changes are outside the reported HIP-event intervals. `candidate_delta_pct` is negative when the candidate is faster; the companion diff --git a/speed-bench/rocm_q4_prefill_bench.cpp b/speed-bench/rocm_q4_prefill_bench.cpp index 3c4a110ce..368491d62 100644 --- a/speed-bench/rocm_q4_prefill_bench.cpp +++ b/speed-bench/rocm_q4_prefill_bench.cpp @@ -24,6 +24,15 @@ extern "C" void ds4_rocm_bench_q8_K_quantize_enqueue( void *out, const void *x, uint32_t in_dim, uint32_t n_rows, int use_wave32); +extern "C" const void *ds4_rocm_bench_q4_K_resident_weight_ptr( + const void *model_map, uint64_t model_size, uint64_t weight_offset, + uint64_t weight_bytes); +extern "C" int ds4_rocm_bench_q4_K_wmma_enqueue( + void *out, const void *w, const void *x, uint32_t n_tok, + uint32_t n_groups, uint32_t in_dim, uint32_t out_dim, + uint64_t row_bytes, uint64_t x_token_stride, + uint64_t x_group_stride, uint64_t out_token_stride, + uint32_t row_tile, int load2); namespace { @@ -67,6 +76,8 @@ constexpr const char *kWmmaDisable = "DS4_ROCM_DISABLE_Q4_PREFILL_WMMA"; constexpr const char *kWmmaRequire = "DS4_ROCM_REQUIRE_Q4_PREFILL_WMMA"; +constexpr const char *kWmmaRowTile = + "DS4_ROCM_Q4_PREFILL_WMMA_ROW_TILE"; constexpr const char *kQ8Wave32Enable = "DS4_ROCM_ENABLE_Q4_PREFILL_Q8_K_WAVE32"; constexpr const char *kQ8Wave32Disable = @@ -104,7 +115,8 @@ enum class bench_case { struct config { bench_case selected = bench_case::all; - std::vector tokens = {9u, 17u, 33u, 128u, 257u, 512u}; + std::vector tokens = { + 9u, 17u, 33u, 128u, 256u, 257u, 512u}; uint32_t sets = kDefaultSets; uint32_t samples = kDefaultSamples; uint32_t warmup = kDefaultWarmup; @@ -253,6 +265,31 @@ uint64_t q4_weight_bytes(uint32_t in_dim, uint32_t out_dim) { sizeof(block_q4_K_host); } +uint32_t shape_wmma_row_tile(uint32_t out_dim) { + return out_dim >= 8192u ? 256u : (out_dim >= 1024u ? 128u : 64u); +} + +bool resolve_resident_weights( + const model_fixture &model, + uint64_t weight_set::*offset_member, + uint64_t weight_bytes, + std::vector *resolved) { + resolved->resize(model.weights.size()); + for (size_t i = 0; i < model.weights.size(); i++) { + const uint64_t offset = model.weights[i].*offset_member; + (*resolved)[i] = ds4_rocm_bench_q4_K_resident_weight_ptr( + model.data, model.size, offset, weight_bytes); + if (!(*resolved)[i]) { + std::fprintf(stderr, + "rocm-q4-prefill-bench: weight set %zu is not " + "physically device-resident\n", + i); + return false; + } + } + return true; +} + bool make_model(model_fixture *model, uint32_t sets) { constexpr uint64_t page = 4096u; const uint64_t dense_bytes = q4_weight_bytes(kDenseK, kDenseM); @@ -334,7 +371,10 @@ void fill_activation(std::vector *values, uint32_t n_tokens, for (uint32_t i = 0; i < kQkK; i++) { const int q = static_cast((i * 73u + token * 37u + block * 19u) % 241u) - 120; - dst[i] = static_cast(q) / 32.0f; + /* Exercise real F32->F16 rounding in the direct-WMMA + * scalar/load2 oracle; division by 32 made every fixture + * value exactly representable in F16. */ + dst[i] = static_cast(q) / 37.0f; } dst[0] = ((token + block) & 1u) ? 127.0f / 32.0f : -127.0f / 32.0f; @@ -485,6 +525,7 @@ void select_legacy() { (void)unsetenv(kWmmaSsdEnable); (void)unsetenv(kWmmaDisable); (void)unsetenv(kWmmaRequire); + (void)unsetenv(kWmmaRowTile); (void)unsetenv(kQ8Wave32Enable); (void)unsetenv(kQ8Wave32Disable); (void)unsetenv(kQ8Wave32Require); @@ -500,6 +541,7 @@ void select_tile8(bool disable_k1024_tile4) { (void)unsetenv(kWmmaSsdEnable); (void)unsetenv(kWmmaDisable); (void)unsetenv(kWmmaRequire); + (void)unsetenv(kWmmaRowTile); (void)unsetenv(kQ8Wave32Enable); (void)unsetenv(kQ8Wave32Disable); (void)unsetenv(kQ8Wave32Require); @@ -515,7 +557,7 @@ void select_k1024_tile4() { (void)setenv(kK1024Tile4Require, "1", 1); } -void select_wmma() { +void select_wmma_shape() { select_tile8(false); /* WMMA and TILE8 are separate strict contracts. The candidate must not * inherit REQUIRE_TILE8 from the baseline selector. */ @@ -525,6 +567,7 @@ void select_wmma() { (void)unsetenv(kWmmaDisable); (void)setenv(kWmmaRequire, "1", 1); (void)unsetenv(kK1024Tile4Require); + (void)unsetenv(kWmmaRowTile); } double percentile(std::vector sorted, double fraction) { @@ -625,6 +668,23 @@ bool benchmark_arms(const char *case_name, uint32_t n_tokens, uint32_t in_dim, } } + /* Re-run one set after the timed samples. Event synchronization catches + * asynchronous launch failures, while this final readback also catches a + * geometry-dependent overwrite or wrong result that appears only after + * repeated launches. */ + const uint32_t post_set = cfg.sets - 1u; + if (!oracle_prepare()) return false; + baseline.prepare(); + if (!baseline.dispatch(post_set) || !ds4_gpu_synchronize()) return false; + candidate.prepare(); + if (!candidate.dispatch(post_set) || !ds4_gpu_synchronize()) return false; + if (!oracle()) { + std::fprintf(stderr, + "rocm-q4-prefill-bench: %s post-timing oracle failed\n", + case_name); + return false; + } + const stats a = summarize(a_samples); const stats b = summarize(b_samples); std::vector paired_delta; @@ -843,14 +903,14 @@ bool run_dense(const model_fixture &model, const config &cfg, kDenseM, x.ptr, n_tokens) != 0; }}; const arm wmma_candidate = { - "wmma64", select_wmma, + "wmma_shape", select_wmma_shape, [&](uint32_t set) { return ds4_gpu_matmul_quant_tensor( tiled.ptr, model.data, model.size, model.weights[set].dense_offset, kQ4Type, kDenseK, kDenseM, x.ptr, n_tokens) != 0; }}; - return benchmark_arms( + if (!benchmark_arms( "dense_wmma", n_tokens, kDenseK, kDenseM, cfg, wmma_baseline, wmma_candidate, [&]() { @@ -859,11 +919,57 @@ bool run_dense(const model_fixture &model, const config &cfg, }, [&]() { return numerically_close(tiled.ptr, legacy.ptr, logical_bytes, - "dense WMMA64 vs TILE8") && + "dense WMMA shape vs TILE8") && + check_guard(legacy.ptr, logical_bytes, + "dense WMMA baseline oracle") && + check_guard(tiled.ptr, logical_bytes, + "dense WMMA shape oracle"); + })) return false; + + void *const x_device = ds4_gpu_tensor_contents(x.ptr); + void *const rows64_device = ds4_gpu_tensor_contents(legacy.ptr); + void *const shape_device = ds4_gpu_tensor_contents(tiled.ptr); + std::vector dense_weights; + if (!x_device || !rows64_device || !shape_device || + !resolve_resident_weights( + model, &weight_set::dense_offset, + q4_weight_bytes(kDenseK, kDenseM), &dense_weights)) { + std::fprintf(stderr, + "dense_wmma_rows N=%u: direct setup failed\n", n_tokens); + return false; + } + const uint64_t row_bytes = q4_weight_bytes(kDenseK, 1u); + const arm geometry_baseline = { + "wmma_rows64_scalar", []() {}, + [&](uint32_t set) { + return ds4_rocm_bench_q4_K_wmma_enqueue( + rows64_device, dense_weights[set], x_device, + n_tokens, 1u, kDenseK, kDenseM, row_bytes, + kDenseK, 0u, kDenseM, 64u, 0) != 0; + }}; + const arm geometry_candidate = { + "wmma_shape_scalar", []() {}, + [&](uint32_t set) { + return ds4_rocm_bench_q4_K_wmma_enqueue( + shape_device, dense_weights[set], x_device, + n_tokens, 1u, kDenseK, kDenseM, row_bytes, + kDenseK, 0u, kDenseM, + shape_wmma_row_tile(kDenseM), 0) != 0; + }}; + return benchmark_arms( + "dense_wmma_rows", n_tokens, kDenseK, kDenseM, cfg, + geometry_baseline, geometry_candidate, + [&]() { + return poison_output(legacy.ptr, logical_bytes, 0x7fc10001u) && + poison_output(tiled.ptr, logical_bytes, 0x7fc20002u); + }, + [&]() { + return bitwise_equal(legacy.ptr, tiled.ptr, logical_bytes, + "dense WMMA rows64 vs shape") && check_guard(legacy.ptr, logical_bytes, - "dense WMMA64 baseline oracle") && + "dense WMMA rows64 oracle") && check_guard(tiled.ptr, logical_bytes, - "dense WMMA64 candidate oracle"); + "dense WMMA shape oracle"); }); } @@ -1007,14 +1113,14 @@ bool run_qb(const model_fixture &model, const config &cfg, x.ptr, n_tokens) != 0; }}; const arm wmma_candidate = { - "wmma64", select_wmma, + "wmma_shape", select_wmma_shape, [&](uint32_t set) { return ds4_gpu_matmul_quant_tensor( tile4.ptr, model.data, model.size, model.weights[set].qb_offset, kQ4Type, kQbK, kQbM, x.ptr, n_tokens) != 0; }}; - return benchmark_arms( + if (!benchmark_arms( "q_b_wmma", n_tokens, kQbK, kQbM, cfg, wmma_baseline, wmma_candidate, [&]() { @@ -1023,11 +1129,152 @@ bool run_qb(const model_fixture &model, const config &cfg, }, [&]() { return numerically_close(tile4.ptr, tile8.ptr, logical_bytes, - "q_b WMMA64 vs TILE4") && + "q_b WMMA shape vs TILE4") && + check_guard(tile8.ptr, logical_bytes, + "q_b WMMA baseline oracle") && + check_guard(tile4.ptr, logical_bytes, + "q_b WMMA shape oracle"); + })) return false; + + void *const x_device = ds4_gpu_tensor_contents(x.ptr); + void *const rows64_device = ds4_gpu_tensor_contents(tile8.ptr); + void *const shape_device = ds4_gpu_tensor_contents(tile4.ptr); + std::vector qb_weights; + if (!x_device || !rows64_device || !shape_device || + !resolve_resident_weights( + model, &weight_set::qb_offset, + q4_weight_bytes(kQbK, kQbM), &qb_weights)) { + std::fprintf(stderr, + "q_b_wmma_rows N=%u: direct setup failed\n", n_tokens); + return false; + } + const uint64_t row_bytes = q4_weight_bytes(kQbK, 1u); + const arm geometry_baseline = { + "wmma_rows64_scalar", []() {}, + [&](uint32_t set) { + return ds4_rocm_bench_q4_K_wmma_enqueue( + rows64_device, qb_weights[set], x_device, + n_tokens, 1u, kQbK, kQbM, row_bytes, + kQbK, 0u, kQbM, 64u, 0) != 0; + }}; + const arm rows128_candidate = { + "wmma_rows128_scalar", []() {}, + [&](uint32_t set) { + return ds4_rocm_bench_q4_K_wmma_enqueue( + shape_device, qb_weights[set], x_device, + n_tokens, 1u, kQbK, kQbM, row_bytes, + kQbK, 0u, kQbM, 128u, 0) != 0; + }}; + if (!benchmark_arms( + "q_b_wmma_rows64_128", n_tokens, kQbK, kQbM, cfg, + geometry_baseline, rows128_candidate, + [&]() { + return poison_output(tile8.ptr, logical_bytes, 0x7fc10001u) && + poison_output(tile4.ptr, logical_bytes, 0x7fc20002u); + }, + [&]() { + return bitwise_equal(tile8.ptr, tile4.ptr, logical_bytes, + "q_b WMMA rows64 vs rows128") && + check_guard(tile8.ptr, logical_bytes, + "q_b WMMA rows64 oracle") && + check_guard(tile4.ptr, logical_bytes, + "q_b WMMA rows128 oracle"); + })) return false; + + const arm rows128_baseline = { + "wmma_rows128_scalar", []() {}, + [&](uint32_t set) { + return ds4_rocm_bench_q4_K_wmma_enqueue( + rows64_device, qb_weights[set], x_device, + n_tokens, 1u, kQbK, kQbM, row_bytes, + kQbK, 0u, kQbM, 128u, 0) != 0; + }}; + const arm rows256_candidate = { + "wmma_rows256_scalar", []() {}, + [&](uint32_t set) { + return ds4_rocm_bench_q4_K_wmma_enqueue( + shape_device, qb_weights[set], x_device, + n_tokens, 1u, kQbK, kQbM, row_bytes, + kQbK, 0u, kQbM, 256u, 0) != 0; + }}; + if (!benchmark_arms( + "q_b_wmma_rows128_256", n_tokens, kQbK, kQbM, cfg, + rows128_baseline, rows256_candidate, + [&]() { + return poison_output(tile8.ptr, logical_bytes, 0x7fc10001u) && + poison_output(tile4.ptr, logical_bytes, 0x7fc20002u); + }, + [&]() { + return bitwise_equal(tile8.ptr, tile4.ptr, logical_bytes, + "q_b WMMA rows128 vs rows256") && check_guard(tile8.ptr, logical_bytes, - "q_b WMMA64 baseline oracle") && + "q_b WMMA rows128 oracle") && check_guard(tile4.ptr, logical_bytes, - "q_b WMMA64 candidate oracle"); + "q_b WMMA rows256 oracle"); + })) return false; + + const arm rows128_scalar = { + "wmma_rows128_scalar", []() {}, + [&](uint32_t set) { + return ds4_rocm_bench_q4_K_wmma_enqueue( + rows64_device, qb_weights[set], x_device, + n_tokens, 1u, kQbK, kQbM, row_bytes, + kQbK, 0u, kQbM, 128u, 0) != 0; + }}; + const arm rows128_load2 = { + "wmma_rows128_load2", []() {}, + [&](uint32_t set) { + return ds4_rocm_bench_q4_K_wmma_enqueue( + shape_device, qb_weights[set], x_device, + n_tokens, 1u, kQbK, kQbM, row_bytes, + kQbK, 0u, kQbM, 128u, 1) != 0; + }}; + if (!benchmark_arms( + "q_b_wmma_load2_128", n_tokens, kQbK, kQbM, cfg, + rows128_scalar, rows128_load2, + [&]() { + return poison_output(tile8.ptr, logical_bytes, 0x7fc10001u) && + poison_output(tile4.ptr, logical_bytes, 0x7fc20002u); + }, + [&]() { + return bitwise_equal(tile8.ptr, tile4.ptr, logical_bytes, + "q_b WMMA rows128 scalar vs load2") && + check_guard(tile8.ptr, logical_bytes, + "q_b WMMA rows128 scalar oracle") && + check_guard(tile4.ptr, logical_bytes, + "q_b WMMA rows128 load2 oracle"); + })) return false; + + const arm rows256_scalar = { + "wmma_rows256_scalar", []() {}, + [&](uint32_t set) { + return ds4_rocm_bench_q4_K_wmma_enqueue( + rows64_device, qb_weights[set], x_device, + n_tokens, 1u, kQbK, kQbM, row_bytes, + kQbK, 0u, kQbM, 256u, 0) != 0; + }}; + const arm rows256_load2 = { + "wmma_rows256_load2", []() {}, + [&](uint32_t set) { + return ds4_rocm_bench_q4_K_wmma_enqueue( + shape_device, qb_weights[set], x_device, + n_tokens, 1u, kQbK, kQbM, row_bytes, + kQbK, 0u, kQbM, 256u, 1) != 0; + }}; + return benchmark_arms( + "q_b_wmma_load2_256", n_tokens, kQbK, kQbM, cfg, + rows256_scalar, rows256_load2, + [&]() { + return poison_output(tile8.ptr, logical_bytes, 0x7fc10001u) && + poison_output(tile4.ptr, logical_bytes, 0x7fc20002u); + }, + [&]() { + return bitwise_equal(tile8.ptr, tile4.ptr, logical_bytes, + "q_b WMMA rows256 scalar vs load2") && + check_guard(tile8.ptr, logical_bytes, + "q_b WMMA rows256 scalar oracle") && + check_guard(tile4.ptr, logical_bytes, + "q_b WMMA rows256 load2 oracle"); }); } @@ -1058,14 +1305,14 @@ bool run_output_b(const model_fixture &model, const config &cfg, kOutputLowDim, kOutputM, x.ptr, n_tokens) != 0; }}; const arm candidate = { - "wmma64", select_wmma, + "wmma_shape", select_wmma_shape, [&](uint32_t set) { return ds4_gpu_matmul_quant_tensor( wmma.ptr, model.data, model.size, model.weights[set].output_b_offset, kQ4Type, kOutputLowDim, kOutputM, x.ptr, n_tokens) != 0; }}; - return benchmark_arms( + if (!benchmark_arms( "output_b_wmma", n_tokens, kOutputLowDim, kOutputM, cfg, baseline, candidate, [&]() { @@ -1074,11 +1321,58 @@ bool run_output_b(const model_fixture &model, const config &cfg, }, [&]() { return numerically_close(wmma.ptr, tile8.ptr, logical_bytes, - "output_b WMMA64 vs TILE8") && + "output_b WMMA shape vs TILE8") && check_guard(tile8.ptr, logical_bytes, "output_b TILE8 oracle") && check_guard(wmma.ptr, logical_bytes, - "output_b WMMA64 oracle"); + "output_b WMMA shape oracle"); + })) return false; + + void *const x_device = ds4_gpu_tensor_contents(x.ptr); + void *const rows64_device = ds4_gpu_tensor_contents(tile8.ptr); + void *const shape_device = ds4_gpu_tensor_contents(wmma.ptr); + std::vector output_b_weights; + if (!x_device || !rows64_device || !shape_device || + !resolve_resident_weights( + model, &weight_set::output_b_offset, + q4_weight_bytes(kOutputLowDim, kOutputM), &output_b_weights)) { + std::fprintf(stderr, + "output_b_wmma_rows N=%u: direct setup failed\n", + n_tokens); + return false; + } + const uint64_t row_bytes = q4_weight_bytes(kOutputLowDim, 1u); + const arm geometry_baseline = { + "wmma_rows64_scalar", []() {}, + [&](uint32_t set) { + return ds4_rocm_bench_q4_K_wmma_enqueue( + rows64_device, output_b_weights[set], x_device, + n_tokens, 1u, kOutputLowDim, kOutputM, row_bytes, + kOutputLowDim, 0u, kOutputM, 64u, 0) != 0; + }}; + const arm geometry_candidate = { + "wmma_shape_scalar", []() {}, + [&](uint32_t set) { + return ds4_rocm_bench_q4_K_wmma_enqueue( + shape_device, output_b_weights[set], x_device, + n_tokens, 1u, kOutputLowDim, kOutputM, row_bytes, + kOutputLowDim, 0u, kOutputM, + shape_wmma_row_tile(kOutputM), 0) != 0; + }}; + return benchmark_arms( + "output_b_wmma_rows", n_tokens, kOutputLowDim, kOutputM, cfg, + geometry_baseline, geometry_candidate, + [&]() { + return poison_output(tile8.ptr, logical_bytes, 0x7fc10001u) && + poison_output(wmma.ptr, logical_bytes, 0x7fc20002u); + }, + [&]() { + return bitwise_equal(tile8.ptr, wmma.ptr, logical_bytes, + "output_b WMMA rows64 vs shape") && + check_guard(tile8.ptr, logical_bytes, + "output_b WMMA rows64 oracle") && + check_guard(wmma.ptr, logical_bytes, + "output_b WMMA shape oracle"); }); } @@ -1134,7 +1428,7 @@ bool run_attention_output(const model_fixture &model, const config &cfg, heads.ptr, n_tokens) > 0; }}; const arm candidate = { - "wmma64_ab", select_wmma, + "wmma_shape_ab", select_wmma_shape, [&](uint32_t set) { return ds4_gpu_attention_output_q4_K_batch_tensor( wmma_out.ptr, wmma_low.ptr, nullptr, nullptr, @@ -1144,23 +1438,93 @@ bool run_attention_output(const model_fixture &model, const config &cfg, kDenseK, kOutputRank, kOutputGroups, kOutputM, heads.ptr, n_tokens) > 0; }}; - return benchmark_arms( + if (!benchmark_arms( "attention_output_ab_wmma", n_tokens, kOutputLowDim, kDenseK + kOutputM, cfg, baseline, candidate, poison_all, [&]() { return numerically_close(wmma_low.ptr, tile8_low.ptr, low_bytes, - "output_a grouped WMMA64 vs TILE8") && + "output_a grouped WMMA shape vs TILE8") && numerically_close(wmma_out.ptr, tile8_out.ptr, out_bytes, - "output_a+b WMMA64 vs TILE8", + "output_a+b WMMA shape vs TILE8", 16.0f, 8.0e-2f) && check_guard(tile8_low.ptr, low_bytes, "output_a TILE8 oracle") && check_guard(wmma_low.ptr, low_bytes, - "output_a WMMA64 oracle") && + "output_a WMMA shape oracle") && check_guard(tile8_out.ptr, out_bytes, "output_b TILE8 oracle") && check_guard(wmma_out.ptr, out_bytes, - "output_b WMMA64 oracle"); + "output_b WMMA shape oracle"); + })) return false; + + void *const heads_device = ds4_gpu_tensor_contents(heads.ptr); + void *const rows64_low_device = ds4_gpu_tensor_contents(tile8_low.ptr); + void *const rows64_out_device = ds4_gpu_tensor_contents(tile8_out.ptr); + void *const shape_low_device = ds4_gpu_tensor_contents(wmma_low.ptr); + void *const shape_out_device = ds4_gpu_tensor_contents(wmma_out.ptr); + std::vector output_a_weights; + std::vector output_b_weights; + if (!heads_device || !rows64_low_device || !rows64_out_device || + !shape_low_device || !shape_out_device || + !resolve_resident_weights( + model, &weight_set::output_a_offset, + q4_weight_bytes(kDenseK, kOutputLowDim), &output_a_weights) || + !resolve_resident_weights( + model, &weight_set::output_b_offset, + q4_weight_bytes(kOutputLowDim, kOutputM), &output_b_weights)) { + std::fprintf(stderr, + "attention_output_ab_wmma_rows N=%u: direct setup failed\n", + n_tokens); + return false; + } + const uint64_t row_a_bytes = q4_weight_bytes(kDenseK, 1u); + const uint64_t row_b_bytes = q4_weight_bytes(kOutputLowDim, 1u); + const arm geometry_baseline = { + "wmma_rows64_ab_scalar", []() {}, + [&](uint32_t set) { + return ds4_rocm_bench_q4_K_wmma_enqueue( + rows64_low_device, output_a_weights[set], heads_device, + n_tokens, kOutputGroups, kDenseK, kOutputRank, + row_a_bytes, kOutputGroups * kDenseK, kDenseK, + kOutputLowDim, 64u, 0) != 0 && + ds4_rocm_bench_q4_K_wmma_enqueue( + rows64_out_device, output_b_weights[set], + rows64_low_device, n_tokens, 1u, kOutputLowDim, + kOutputM, row_b_bytes, kOutputLowDim, 0u, kOutputM, + 64u, 0) != 0; + }}; + const arm geometry_candidate = { + "wmma_shape_ab_scalar", []() {}, + [&](uint32_t set) { + return ds4_rocm_bench_q4_K_wmma_enqueue( + shape_low_device, output_a_weights[set], heads_device, + n_tokens, kOutputGroups, kDenseK, kOutputRank, + row_a_bytes, kOutputGroups * kDenseK, kDenseK, + kOutputLowDim, + shape_wmma_row_tile(kOutputRank), 0) != 0 && + ds4_rocm_bench_q4_K_wmma_enqueue( + shape_out_device, output_b_weights[set], + shape_low_device, n_tokens, 1u, kOutputLowDim, + kOutputM, row_b_bytes, kOutputLowDim, 0u, kOutputM, + shape_wmma_row_tile(kOutputM), 0) != 0; + }}; + return benchmark_arms( + "attention_output_ab_wmma_rows", n_tokens, kOutputLowDim, + kDenseK + kOutputM, cfg, geometry_baseline, geometry_candidate, + poison_all, + [&]() { + return bitwise_equal(tile8_low.ptr, wmma_low.ptr, low_bytes, + "output_a WMMA rows64 vs shape") && + bitwise_equal(tile8_out.ptr, wmma_out.ptr, out_bytes, + "output_a+b WMMA rows64 vs shape") && + check_guard(tile8_low.ptr, low_bytes, + "output_a WMMA rows64 oracle") && + check_guard(wmma_low.ptr, low_bytes, + "output_a WMMA shape oracle") && + check_guard(tile8_out.ptr, out_bytes, + "output_b WMMA rows64 oracle") && + check_guard(wmma_out.ptr, out_bytes, + "output_b WMMA shape oracle"); }); } @@ -1172,16 +1536,16 @@ void usage(FILE *stream, const char *argv0) { " --case all|dense|pair|qb|outb|output\n" " comparison to run (default: all)\n" " --tokens N[,N...] token counts, each 9..4096\n" - " --full use 9,16,17,31,32,33,128,257,512,4096\n" + " --full use 9,16,17,31,32,33,128,256,257,512,4096\n" " --sets N rotating resident weight sets (default: %u)\n" " --samples N samples/arm, multiple of 4 (default: %u)\n" " --warmup N untimed dispatches/arm (default: %u)\n" " -h, --help show this help\n\n" "dense compares legacy/TILE8, times the raw canonical/wave32 Q8_K\n" - "quantizer kernels on gfx1151, and TILE8/WMMA64 there for N>=256\n" + "quantizer kernels on gfx1151, and TILE8/direct WMMA there for N>=256\n" "at K=4096,M=1024. pair compares\n" "two TILE8 projections with\n" - "the fused K=4096,M=(1024+512) path. qb adds TILE4/WMMA64 at the\n" + "the fused K=4096,M=(1024+512) path. qb adds TILE4/direct WMMA at the\n" "K=1024,M=32768 shape. outb isolates K=8192,M=4096; output measures\n" "the production grouped output_a plus output_b API. WMMA comparisons\n" "use a finite/toleranced oracle because their F16 boundary is not\n" @@ -1257,7 +1621,8 @@ config parse_options(int argc, char **argv) { cfg.tokens = parse_tokens(need_value(&i, argc, argv)); } else if (!std::strcmp(argv[i], "--full")) { cfg.tokens = { - 9u, 16u, 17u, 31u, 32u, 33u, 128u, 257u, 512u, 4096u}; + 9u, 16u, 17u, 31u, 32u, 33u, 128u, 256u, 257u, 512u, + 4096u}; } else if (!std::strcmp(argv[i], "--sets")) { cfg.sets = parse_u32(need_value(&i, argc, argv), "--sets", 1u, 32u); } else if (!std::strcmp(argv[i], "--samples")) { @@ -1298,6 +1663,7 @@ int main(int argc, char **argv) { env_snapshot wmma_ssd_enable_guard(kWmmaSsdEnable); env_snapshot wmma_disable_guard(kWmmaDisable); env_snapshot wmma_require_guard(kWmmaRequire); + env_snapshot wmma_row_tile_guard(kWmmaRowTile); env_snapshot q8_wave32_enable_guard(kQ8Wave32Enable); env_snapshot q8_wave32_disable_guard(kQ8Wave32Disable); env_snapshot q8_wave32_require_guard(kQ8Wave32Require); @@ -1366,10 +1732,11 @@ int main(int argc, char **argv) { std::printf( "DS4_ROCM_Q4_PREFILL_SETUP device=%s arch=%s warp=%d sets=%u " "resident_mib=%.2f timing=hip_events ssd_streaming=off " - "wmma64=%s q8_wave32=%s\n", + "wmma_rowtiles=%s wmma_loaders=%s q8_wave32=%s\n", properties.name, properties.gcnArchName, properties.warpSize, cfg.sets, static_cast(model.resident_bytes) / 1048576.0, - cfg.wmma_supported ? "available" : "skipped", + cfg.wmma_supported ? "64,128,256" : "skipped", + cfg.wmma_supported ? "scalar,load2" : "skipped", cfg.wmma_supported ? "available" : "skipped"); std::fflush(stdout); for (uint32_t n_tokens : cfg.tokens) { diff --git a/tests/test_rocm_q4_dense_pair.cpp b/tests/test_rocm_q4_dense_pair.cpp index f219c3135..71146ec4b 100644 --- a/tests/test_rocm_q4_dense_pair.cpp +++ b/tests/test_rocm_q4_dense_pair.cpp @@ -36,6 +36,8 @@ extern "C" int ds4_rocm_test_q4_prefill_k1024_tile4_policy( int disabled, int required); extern "C" void ds4_rocm_test_q4_prefill_wmma_reset(void); extern "C" uint64_t ds4_rocm_test_q4_prefill_wmma_get_calls(void); +extern "C" uint32_t ds4_rocm_test_q4_prefill_wmma_row_tile( + uint32_t out_dim); extern "C" int ds4_rocm_test_q4_prefill_q8_wave32_policy( int prefill_scope, int runtime_compatible, int enabled, int disabled, int required); @@ -99,6 +101,8 @@ constexpr const char *kPrefillWmmaDisable = "DS4_ROCM_DISABLE_Q4_PREFILL_WMMA"; constexpr const char *kPrefillWmmaRequire = "DS4_ROCM_REQUIRE_Q4_PREFILL_WMMA"; +constexpr const char *kPrefillWmmaRowTile = + "DS4_ROCM_Q4_PREFILL_WMMA_ROW_TILE"; constexpr const char *kPrefillQ8Wave32Enable = "DS4_ROCM_ENABLE_Q4_PREFILL_Q8_K_WAVE32"; constexpr const char *kPrefillQ8Wave32Disable = @@ -1147,6 +1151,51 @@ bool run_q_b_f16_null_qhalf_case(const aligned_model &model) { return ok; } +bool run_prefill_wmma_row_tile_policy_oracle() { + struct row_tile_case { + const char *label; + const char *value; + uint32_t out_dim; + uint32_t expected; + }; + const row_tile_case cases[] = { + {"automatic small", nullptr, 1023u, 64u}, + {"automatic normal lower boundary", nullptr, 1024u, 128u}, + {"automatic normal upper boundary", nullptr, 8191u, 128u}, + {"automatic large", nullptr, 8192u, 256u}, + {"explicit 64", "64", 32768u, 64u}, + {"explicit 128", "128", 32768u, 128u}, + {"explicit 256", "256", 1024u, 256u}, + {"non-whitelisted integer", "257", 1024u, 128u}, + {"negative integer", "-1", 1024u, 128u}, + {"negative wrapped whitelist", "-18446744073709551552", 1024u, + 128u}, + {"malformed", "128x", 1024u, 128u}, + {"empty", "", 8192u, 256u}, + }; + + env_snapshot row_tile(kPrefillWmmaRowTile); + bool ok = true; + for (const row_tile_case &test : cases) { + const int env_rc = test.value + ? setenv(kPrefillWmmaRowTile, test.value, 1) + : unsetenv(kPrefillWmmaRowTile); + const uint32_t got = env_rc == 0 + ? ds4_rocm_test_q4_prefill_wmma_row_tile(test.out_dim) + : 0u; + if (env_rc != 0 || got != test.expected) { + std::fprintf(stderr, + "Q4 WMMA row-tile policy %s: expected=%u got=%u " + "env_rc=%d FAIL\n", + test.label, test.expected, got, env_rc); + ok = false; + } + } + std::fprintf(stderr, "Q4 WMMA row-tile policy: %s\n", + ok ? "PASS" : "FAIL"); + return ok; +} + bool run_prefill_k1024_tile4_policy_oracle() { struct policy_case { const char *label; @@ -1300,6 +1349,7 @@ bool run_pair_pre_enqueue_policy_oracle() { env_snapshot wmma_enable(kPrefillWmmaEnable); env_snapshot wmma_disable(kPrefillWmmaDisable); env_snapshot wmma_require(kPrefillWmmaRequire); + env_snapshot wmma_row_tile(kPrefillWmmaRowTile); env_snapshot q8_enable(kPrefillQ8Wave32Enable); env_snapshot q8_disable(kPrefillQ8Wave32Disable); env_snapshot q8_require(kPrefillQ8Wave32Require); @@ -1310,6 +1360,7 @@ bool run_pair_pre_enqueue_policy_oracle() { (void)unsetenv(kPrefillWmmaEnable); (void)unsetenv(kPrefillWmmaDisable); (void)unsetenv(kPrefillWmmaRequire); + (void)unsetenv(kPrefillWmmaRowTile); (void)unsetenv(kPrefillQ8Wave32Enable); (void)unsetenv(kPrefillQ8Wave32Disable); (void)unsetenv(kPrefillQ8Wave32Require); @@ -2569,7 +2620,8 @@ bool run_prefill_wmma_smoke(const aligned_model &model) { properties.warpSize != 32 || std::strncmp(properties.gcnArchName, "gfx1151", 7u) != 0) { std::fprintf(stderr, - "ROCm Q4 WMMA64 prefill: SKIP (requires gfx1151 wave32)\n"); + "ROCm Q4 direct-WMMA prefill: SKIP " + "(requires gfx1151 wave32)\n"); return true; } #else @@ -2593,7 +2645,7 @@ bool run_prefill_wmma_smoke(const aligned_model &model) { if (!x_gpu.ptr || !tile8_gpu.ptr || !wmma_gpu.ptr || !write_tensor(x_gpu.ptr, x) || !write_tensor(tile8_gpu.ptr, sentinel) || !write_tensor(wmma_gpu.ptr, sentinel)) { - std::fprintf(stderr, "ROCm Q4 WMMA64 prefill: setup FAIL\n"); + std::fprintf(stderr, "ROCm Q4 direct-WMMA prefill: setup FAIL\n"); return false; } @@ -2605,6 +2657,7 @@ bool run_prefill_wmma_smoke(const aligned_model &model) { env_snapshot wmma_ssd_enable(kPrefillWmmaSsdEnable); env_snapshot wmma_disable(kPrefillWmmaDisable); env_snapshot wmma_require(kPrefillWmmaRequire); + env_snapshot wmma_row_tile(kPrefillWmmaRowTile); env_snapshot q8_wave32_enable(kPrefillQ8Wave32Enable); env_snapshot q8_wave32_disable(kPrefillQ8Wave32Disable); env_snapshot q8_wave32_require(kPrefillQ8Wave32Require); @@ -2617,6 +2670,7 @@ bool run_prefill_wmma_smoke(const aligned_model &model) { (void)unsetenv(kPrefillWmmaSsdEnable); (void)unsetenv(kPrefillWmmaDisable); (void)unsetenv(kPrefillWmmaRequire); + (void)unsetenv(kPrefillWmmaRowTile); (void)unsetenv(kPrefillQ8Wave32Enable); (void)unsetenv(kPrefillQ8Wave32Disable); (void)unsetenv(kPrefillQ8Wave32Require); @@ -2646,17 +2700,17 @@ bool run_prefill_wmma_smoke(const aligned_model &model) { read_tensor(wmma_gpu.ptr, &wmma); if (ok) { ok = output_body_overwritten(tile8, sentinel, logical_count, - "WMMA64 TILE8 output body") && ok; + "direct-WMMA TILE8 output body") && ok; ok = output_body_overwritten(wmma, sentinel, logical_count, - "WMMA64 candidate output body") && ok; + "direct-WMMA candidate output body") && ok; ok = output_guard_unchanged(tile8, sentinel, logical_count, - "WMMA64 TILE8 output canary") && ok; + "direct-WMMA TILE8 output canary") && ok; ok = output_guard_unchanged(wmma, sentinel, logical_count, - "WMMA64 candidate output canary") && ok; + "direct-WMMA candidate output canary") && ok; tile8.resize(logical_count); wmma.resize(logical_count); ok = close_with_tolerance(wmma, tile8, 2.0f, 3.0e-2f, - "WMMA64 vs TILE8 N/M tail") && ok; + "direct-WMMA vs TILE8 N/M tail") && ok; } (void)setenv(kPrefillWmmaDisable, "1", 1); @@ -2667,9 +2721,9 @@ bool run_prefill_wmma_smoke(const aligned_model &model) { ok = rejected_rc == 0 && unchanged_after_rejected_call( wmma_gpu.ptr, sentinel, - "WMMA64 DISABLE+REQUIRE preserves output") && ok; + "direct-WMMA DISABLE+REQUIRE preserves output") && ok; std::fprintf(stderr, - "ROCm Q4 WMMA64 prefill: tile8=%d/%llu wmma=%d/%llu " + "ROCm Q4 direct-WMMA prefill: tile8=%d/%llu wmma=%d/%llu " "rejected=%d %s\n", tile8_rc, (unsigned long long)tile8_wmma_calls, wmma_rc, (unsigned long long)wmma_calls, @@ -2684,7 +2738,7 @@ bool run_attention_output_wmma_smoke(const aligned_model &model) { properties.warpSize != 32 || std::strncmp(properties.gcnArchName, "gfx1151", 7u) != 0) { std::fprintf(stderr, - "ROCm Q4 production output WMMA64: SKIP " + "ROCm Q4 production output direct-WMMA: SKIP " "(requires gfx1151 wave32)\n"); return true; } @@ -2721,7 +2775,7 @@ bool run_attention_output_wmma_smoke(const aligned_model &model) { !write_tensor(wmma_low.ptr, low_sentinel) || !write_tensor(wmma_out.ptr, out_sentinel)) { std::fprintf(stderr, - "ROCm Q4 production output WMMA64: setup FAIL\n"); + "ROCm Q4 production output direct-WMMA: setup FAIL\n"); return false; } @@ -2733,6 +2787,7 @@ bool run_attention_output_wmma_smoke(const aligned_model &model) { env_snapshot wmma_ssd_enable(kPrefillWmmaSsdEnable); env_snapshot wmma_disable(kPrefillWmmaDisable); env_snapshot wmma_require(kPrefillWmmaRequire); + env_snapshot wmma_row_tile(kPrefillWmmaRowTile); ds4_gpu_set_ssd_streaming(false); (void)unsetenv(kPrefillEnable); @@ -2743,6 +2798,7 @@ bool run_attention_output_wmma_smoke(const aligned_model &model) { (void)unsetenv(kPrefillWmmaSsdEnable); (void)unsetenv(kPrefillWmmaDisable); (void)unsetenv(kPrefillWmmaRequire); + (void)unsetenv(kPrefillWmmaRowTile); ds4_rocm_test_q4_prefill_wmma_reset(); const int tile8_rc = ds4_gpu_attention_output_q4_K_batch_tensor( tile8_out.ptr, tile8_low.ptr, nullptr, nullptr, @@ -2786,10 +2842,10 @@ bool run_attention_output_wmma_smoke(const aligned_model &model) { "production output-B TILE8 body") && ok; ok = output_body_overwritten( wmma_low_host, low_sentinel, low_count, - "production output-A WMMA64 body") && ok; + "production output-A direct-WMMA body") && ok; ok = output_body_overwritten( wmma_out_host, out_sentinel, out_count, - "production output-B WMMA64 body") && ok; + "production output-B direct-WMMA body") && ok; ok = output_guard_unchanged( tile8_low_host, low_sentinel, low_count, "production output-A TILE8 N-tail") && ok; @@ -2798,23 +2854,23 @@ bool run_attention_output_wmma_smoke(const aligned_model &model) { "production output-B TILE8 N-tail") && ok; ok = output_guard_unchanged( wmma_low_host, low_sentinel, low_count, - "production output-A WMMA64 N-tail") && ok; + "production output-A direct-WMMA N-tail") && ok; ok = output_guard_unchanged( wmma_out_host, out_sentinel, out_count, - "production output-B WMMA64 N-tail") && ok; + "production output-B direct-WMMA N-tail") && ok; tile8_low_host.resize(low_count); tile8_out_host.resize(out_count); wmma_low_host.resize(low_count); wmma_out_host.resize(out_count); ok = close_with_tolerance( wmma_low_host, tile8_low_host, 2.0f, 3.0e-2f, - "production output-A WMMA64 vs TILE8") && ok; + "production output-A direct-WMMA vs TILE8") && ok; ok = close_with_tolerance( wmma_out_host, tile8_out_host, 16.0f, 8.0e-2f, - "production output-A+B WMMA64 vs TILE8") && ok; + "production output-A+B direct-WMMA vs TILE8") && ok; } std::fprintf(stderr, - "ROCm Q4 production output WMMA64: tile8=%d/%llu " + "ROCm Q4 production output direct-WMMA: tile8=%d/%llu " "wmma=%d/%llu %s\n", tile8_rc, (unsigned long long)tile8_wmma_calls, wmma_rc, (unsigned long long)wmma_calls, @@ -2899,6 +2955,7 @@ int main(int argc, char **argv) { env_snapshot wmma_ssd_enable(kPrefillWmmaSsdEnable); env_snapshot wmma_disable(kPrefillWmmaDisable); env_snapshot wmma_require(kPrefillWmmaRequire); + env_snapshot wmma_row_tile(kPrefillWmmaRowTile); env_snapshot q8_wave32_enable(kPrefillQ8Wave32Enable); env_snapshot q8_wave32_disable(kPrefillQ8Wave32Disable); env_snapshot q8_wave32_require(kPrefillQ8Wave32Require); @@ -2916,6 +2973,7 @@ int main(int argc, char **argv) { (void)unsetenv(kPrefillWmmaSsdEnable); (void)unsetenv(kPrefillWmmaDisable); (void)unsetenv(kPrefillWmmaRequire); + (void)unsetenv(kPrefillWmmaRowTile); (void)unsetenv(kPrefillQ8Wave32Enable); (void)unsetenv(kPrefillQ8Wave32Disable); (void)unsetenv(kPrefillQ8Wave32Require); @@ -2924,7 +2982,8 @@ int main(int argc, char **argv) { (void)unsetenv(kGroupedDecodeRequire); (void)unsetenv(kGroupedDecodeStats); - const bool policy_ok = run_prefill_k1024_tile4_policy_oracle() && + const bool policy_ok = run_prefill_wmma_row_tile_policy_oracle() && + run_prefill_k1024_tile4_policy_oracle() && run_prefill_q8_wave32_policy_oracle() && run_pair_pre_enqueue_policy_oracle(); if (run_policy_only || !policy_ok) return policy_ok ? 0 : 1; From 5dd2e3cb0e793d6f6e335dc1acc027ed9e280e81 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Sat, 29 Aug 2026 22:57:01 +0200 Subject: [PATCH 170/189] Optimize CUDA Q4 prefill with 16-warp MMQ --- ENVIRONMENT_VARIABLES.md | 10 +- Makefile | 19 +- cuda/mmq/ds4_mmq.cu | 234 +++++++++++++++ cuda/mmq/ds4_mmq.h | 38 +++ cuda/mmq/ds4_mmq_q4_16warp.cu | 360 +++++++++++++++++++++++ cuda/mmq/ds4_mmq_q4_16warp.cuh | 55 ++++ cuda/mmq/test/test_mmq_parity.cu | 353 ++++++++++++++++++++++- ds4_cuda.cu | 33 +++ ds4_gpu.h | 7 + scripts/environment_variables.tsv | 3 + speed-bench/README.md | 32 +++ speed-bench/cuda_q4_prefill_bench.cu | 407 ++++++++++++++++++++++++--- 12 files changed, 1504 insertions(+), 47 deletions(-) create mode 100644 cuda/mmq/ds4_mmq_q4_16warp.cu create mode 100644 cuda/mmq/ds4_mmq_q4_16warp.cuh diff --git a/ENVIRONMENT_VARIABLES.md b/ENVIRONMENT_VARIABLES.md index 9287e9252..02717e8f8 100644 --- a/ENVIRONMENT_VARIABLES.md +++ b/ENVIRONMENT_VARIABLES.md @@ -76,6 +76,9 @@ The detailed Metal A/B contracts and expected oracle counters live in | `DS4_CUDA_ENABLE_Q4_K1024_PERSISTENT=1` | Enable the experimental persistent-CTA kernel for the exact `32768x1024` Q4 shape. | | `DS4_CUDA_NO_Q4_K1024_PERSISTENT=1` | Roll back the persistent K1024 experiment. | | `DS4_CUDA_REQUIRE_Q4_K1024_PERSISTENT=1` | Require the K1024 candidate before enqueue instead of silently using canonical MMVQ. | +| `DS4_CUDA_Q4_MMQ_16WARP=1` | Request the experimental exact-integer m128n128 16-warp Q4_K kernel for eligible complete-K CUDA prefills; ineligible shapes fall back. | +| `DS4_CUDA_NO_Q4_MMQ_16WARP=1` | Roll back the 16-warp Q4_K prefill experiment. This value-aware switch overrides request and require. | +| `DS4_CUDA_REQUIRE_Q4_MMQ_16WARP=1` | Request the 16-warp Q4_K prefill kernel and fail closed instead of silently measuring another MMQ/Q8_K path. Decode/speculative batches of at most eight tokens remain on MMVQ. | | `DS4_CUDA_ENABLE_Q8_FOLD=1` | Enable the experimental one-shot Q8_1 producer-to-consumer fold. | | `DS4_CUDA_NO_Q8_FOLD=1` | Dominant rollback for the Q8_1 fold. | | `DS4_CUDA_Q8_FOLD_ORACLE=1` | Compare fresh canonical Q8_1 bytes and consumer outputs. Use with `DS4_CUDA_DECODE_GRAPHS=0`; require nonzero calls and zero mismatches/skips. | @@ -147,7 +150,7 @@ above, it is an unstable internal diagnostic or tuning interface. The linked sou remains normative for exact eligibility gates, bounds, and architecture-specific defaults. -Inventory totals: **1074 `DS4_*` runtime variables** and +Inventory totals: **1077 `DS4_*` runtime variables** and **6 external runtime variables**. The auxiliary inventories contain **118 test/test-fixture entries** and **19 tool/wrapper entries**. @@ -606,7 +609,7 @@ and **19 tool/wrapper entries**.
-CUDA (331) +CUDA (334) | Variable | Accepted value and default | Effect | Source | | --- | --- | --- | --- | @@ -806,6 +809,7 @@ and **19 tool/wrapper entries**. | `DS4_CUDA_NO_Q4_GROUPED_ATTN_A_BATCH` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q4 grouped attn a batch CUDA Q4 optimization. | [cuda/mmq/ds4_mmq.cu:4305](cuda/mmq/ds4_mmq.cu#L4305) | | `DS4_CUDA_NO_Q4_GROUPED_ATTN_A_PREFILL` | presence kill switch; default unset; any defined value including empty or 0 disables and dominates ENABLE/REQUIRE | Restore the eight pack/MMQ/unpack Q4 attention-A prefill projections. | [ds4_cuda.cu:41930](ds4_cuda.cu#L41930) | | `DS4_CUDA_NO_Q4_K1024_PERSISTENT` | presence kill switch, default off; any defined value including 0 disables | Disable the Q4 K1024 persistent CUDA Q4 optimization. | [cuda/mmq/ds4_mmq.cu:3907](cuda/mmq/ds4_mmq.cu#L3907) | +| `DS4_CUDA_NO_Q4_MMQ_16WARP` | value-aware rollback, default off; unset/empty/exact 0 permits the experiment, every other nonempty value disables it and overrides REQUEST/REQUIRE | Disable the experimental complete-K CUDA Q4_K m128n128 16-warp prefill kernel. | [cuda/mmq/ds4_mmq.cu:1059](cuda/mmq/ds4_mmq.cu#L1059) | | `DS4_CUDA_NO_Q8_ALIGNED_DENSE_SCRATCH` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q8 aligned dense scratch CUDA Q8 optimization. | [cuda/mmq/ds4_mmq.cu:5578](cuda/mmq/ds4_mmq.cu#L5578) | | `DS4_CUDA_NO_Q8_ALIGNED_PERSISTENT` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q8 aligned persistent CUDA Q8 optimization. | [cuda/mmq/ds4_mmq.cu:5410](cuda/mmq/ds4_mmq.cu#L5410) | | `DS4_CUDA_NO_Q8_BATCH_EXACT_TOK2` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q8 batch exact tok2 CUDA Q8 optimization. | [ds4_cuda.cu:19852](ds4_cuda.cu#L19852) | @@ -856,6 +860,7 @@ and **19 tool/wrapper entries**. | `DS4_CUDA_Q4_GROUPED_ATTN_A_ORACLE` | value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on | Compare grouped attention-A against the canonical per-group result. | [ds4_cuda.cu:1477](ds4_cuda.cu#L1477) | | `DS4_CUDA_Q4_K1024_PERSISTENT_ORACLE` | value-aware flag, default off; nonempty value other than exact 0 enables and implies candidate admission | Bitwise-compare the exact-shape persistent Q4 K1024 kernel with canonical MMVQ and retain canonical output. | [cuda/mmq/ds4_mmq.cu:3738](cuda/mmq/ds4_mmq.cu#L3738) | | `DS4_CUDA_Q4_K1024_PERSISTENT_STATS` | value-aware flag, default off; nonempty value other than exact 0 enables | Print exact-shape persistent Q4 K1024 dispatch counters at exit. | [cuda/mmq/ds4_mmq.cu:3737](cuda/mmq/ds4_mmq.cu#L3737) | +| `DS4_CUDA_Q4_MMQ_16WARP` | value-aware opt-in cached on first Q4_K dense MMQ call; unset/empty/exact 0 is off, every other nonempty value requests the candidate; rollback wins and ineligible shapes fall back | Enable the experimental exact-integer CUDA Q4_K m128n128 16-warp kernel for eligible complete-K dense prefills. | [cuda/mmq/ds4_mmq.cu:1052](cuda/mmq/ds4_mmq.cu#L1052) | | `DS4_CUDA_Q8_F16_ALL` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Control the Q8 F16 all CUDA quantized-matmul/cache optimization. | [ds4_cuda.cu:2358](ds4_cuda.cu#L2358) | | `DS4_CUDA_Q8_F16_CACHE_MB` | unsigned integer MiB, full-string parse; default unlimited; 0 disables this cache | Limit the selective Q8-to-F16 derived-weight cache. | [ds4_cuda.cu:2218](ds4_cuda.cu#L2218) | | `DS4_CUDA_Q8_F16_CACHE_RESERVE_MB` | unsigned integer MiB, full-string parse; default is VRAM-dependent (>=112 GiB: 512; >=40 GiB: max(768,1%); smaller: max(4096,5%)) | Reserve free VRAM when growing the selective Q8-to-F16 cache. | [ds4_cuda.cu:2224](ds4_cuda.cu#L2224) | @@ -873,6 +878,7 @@ and **19 tool/wrapper entries**. | `DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_BATCH` | value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on | Fail if grouped batched attention-A cannot be used. | [ds4_cuda.cu:40198](ds4_cuda.cu#L40198) | | `DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_PREFILL` | value-aware fail-closed assertion, default off; unset/empty/exact 0 is off, any other nonempty value requests the candidate and rejects ineligibility before enqueue | Require the GB10 grouped Q4_K attention-A prefill path instead of silently using pack/MMQ/unpack. | [ds4_cuda.cu:41889](ds4_cuda.cu#L41889) | | `DS4_CUDA_REQUIRE_Q4_K1024_PERSISTENT` | presence flag, default off; any defined value including 0 makes ineligible candidate fail closed | Fail when the exact Q4 K1024 persistent candidate is unavailable instead of using MMVQ. | [cuda/mmq/ds4_mmq.cu:3929](cuda/mmq/ds4_mmq.cu#L3929) | +| `DS4_CUDA_REQUIRE_Q4_MMQ_16WARP` | value-aware fail-closed prefill opt-in cached on first Q4_K dense MMQ call; unset/empty/exact 0 is off, every other nonempty value requests and requires the candidate for N>8; rollback, disabled MMQ, ineligibility, or preflight failure prevents fallback; decode/speculative N<=8 remains on MMVQ | Require the experimental CUDA Q4_K 16-warp prefill kernel so benchmark runs cannot silently measure another path. | [cuda/mmq/ds4_mmq.cu:1056](cuda/mmq/ds4_mmq.cu#L1056); [ds4_cuda.cu:38208](ds4_cuda.cu#L38208) | | `DS4_CUDA_REQUIRE_STREAMING_EXPERT_PERSISTENT_CACHE` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Require streaming expert persistent cache in CUDA SSD streaming; fail closed when unavailable. | [ds4_cuda.cu:4117](ds4_cuda.cu#L4117) | | `DS4_CUDA_REQUIRE_STREAMING_SELECTED_BATCH_IO` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Require streaming selected batch I/O in CUDA SSD streaming; fail closed when unavailable. | [ds4_cuda.cu:4738](ds4_cuda.cu#L4738) | | `DS4_CUDA_REQUIRE_STREAMING_SELECTED_EVENT_PIPELINE` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Require streaming selected event pipeline in CUDA SSD streaming; fail closed when unavailable. | [ds4_cuda.cu:4870](ds4_cuda.cu#L4870) | diff --git a/Makefile b/Makefile index e7ab65fa3..e92db436e 100644 --- a/Makefile +++ b/Makefile @@ -51,7 +51,7 @@ endif NVCCFLAGS ?= -O3 -g -lineinfo --use_fast_math $(NVCC_ARCH_FLAGS) -Xcompiler $(NATIVE_CPU_FLAG) -Xcompiler -pthread # Vendored llama.cpp mmq prefill tier (cuda/mmq/, see cuda/mmq/VENDOR.md). MMQ_INCLUDES := -Icuda/mmq -MMQ_OBJS := cuda/mmq/ds4_ggml_stubs.o cuda/mmq/ds4_mmq.o cuda/mmq/ds4_mmq_d2r.o cuda/mmq/quantize.o cuda/mmq/mmid.o cuda/mmq/mmvq.o cuda/mmq/ds4_repack.o +MMQ_OBJS := cuda/mmq/ds4_ggml_stubs.o cuda/mmq/ds4_mmq.o cuda/mmq/ds4_mmq_d2r.o cuda/mmq/ds4_mmq_q4_16warp.o cuda/mmq/quantize.o cuda/mmq/mmid.o cuda/mmq/mmvq.o cuda/mmq/ds4_repack.o CORE_OBJS = ds4.o ds4_image.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_cuda.o ds4_layer_pack.o $(MMQ_OBJS) CPU_CORE_OBJS = ds4_cpu.o ds4_image.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o CUDA_LDLIBS ?= -lm -Xcompiler -pthread -L$(CUDA_HOME)/targets/sbsa-linux/lib -L$(CUDA_HOME)/lib64 -lcudart -lcublas @@ -69,7 +69,7 @@ DS4_LINK_LIBS ?= $(CUDA_LDLIBS) METAL_LDLIBS := $(LDLIBS) endif -.PHONY: all help clean test test-ssd environment-docs test-quantizer-indexer-q4 test-rocm test-glm53-kda-rocm test-metal-session-batch test-metal-session-batch-ssd test-metal-q4-streams test-metal-q4-prefill-pair test-metal-indexer-q4 test-metal-q4-attn-exactn test-metal-q4-qb-f16-cache test-metal-q4-qb-f16-cache-timing test-metal-exactn-oracle test-metal-dspark-capture test-metal-argmax-top1 bench-metal-argmax-top1 test-metal-iq2-midonly test-metal-iq2-ssd-grouped-mm test-metal-iq2-live-index test-mxfp4-metal test-mxfp4-cuda test-mxfp4-rocm test-mmq-parity-cuda test-rocm-q4-parity test-rocm-q4-dense test-rocm-q4-pair test-rocm-q4-prefill test-strix-rocm-q4-parity test-strix-rocm-q4-prefill test-strix-rocm-q4-prefill-long test-cuda-session-batch test-cuda-mixed-batch dspark-acceptance dspark-verify-depth rocm-dspark-acceptance rocm-dspark-verify-depth mtp-verify-depth cpu cuda cuda-spark cuda-generic cuda-regression strix-halo rocm cuda-iq2-moe-prefill-bench cuda-q4-prefill-bench rocm-iq2-moe-prefill-bench rocm-q4-prefill-bench +.PHONY: all help clean test test-ssd environment-docs test-quantizer-indexer-q4 test-rocm test-glm53-kda-rocm test-metal-session-batch test-metal-session-batch-ssd test-metal-q4-streams test-metal-q4-prefill-pair test-metal-indexer-q4 test-metal-q4-attn-exactn test-metal-q4-qb-f16-cache test-metal-q4-qb-f16-cache-timing test-metal-exactn-oracle test-metal-dspark-capture test-metal-argmax-top1 bench-metal-argmax-top1 test-metal-iq2-midonly test-metal-iq2-ssd-grouped-mm test-metal-iq2-live-index test-mxfp4-metal test-mxfp4-cuda test-mxfp4-rocm test-mmq-parity-cuda test-mmq-q4-16warp-cuda test-rocm-q4-parity test-rocm-q4-dense test-rocm-q4-pair test-rocm-q4-prefill test-strix-rocm-q4-parity test-strix-rocm-q4-prefill test-strix-rocm-q4-prefill-long test-cuda-session-batch test-cuda-mixed-batch dspark-acceptance dspark-verify-depth rocm-dspark-acceptance rocm-dspark-verify-depth mtp-verify-depth cpu cuda cuda-spark cuda-generic cuda-regression strix-halo rocm cuda-iq2-moe-prefill-bench cuda-q4-prefill-bench rocm-iq2-moe-prefill-bench rocm-q4-prefill-bench gguf-tools/deepseek4-quantize: gguf-tools/deepseek4-quantize.c gguf-tools/quants.c gguf-tools/quants.h $(MAKE) -C gguf-tools deepseek4-quantize @@ -413,6 +413,7 @@ help: @echo " make cuda-generic Build CUDA for a generic local CUDA GPU" @echo " make cuda CUDA_ARCH=sm_N Build CUDA with an explicit nvcc -arch value" @echo " make test-mmq-parity-cuda CUDA_ARCH=sm_N Run quantized CUDA kernel parity tests" + @echo " make test-mmq-q4-16warp-cuda CUDA_ARCH=sm_N Run focused Q4 16-warp bitwise/canary oracle" @echo " make test-rocm-q4-parity Run ROCm Q4_K dense/pair/prefill oracle" @echo " make test-strix-rocm-q4-prefill Require gfx1151 and run tiled-prefill oracle" @echo " make test-strix-rocm-q4-parity Require a visible gfx1151 device and run the Q4 tests" @@ -549,12 +550,15 @@ tests/test_mxfp4_cuda: tests/test_mxfp4_cuda.cu $(MMQ_OBJS) test-mxfp4-cuda: tests/test_mxfp4_cuda ./tests/test_mxfp4_cuda -cuda/mmq/test/test_mmq_parity: cuda/mmq/test/test_mmq_parity.cu $(MMQ_OBJS) - $(NVCC) $(NVCCFLAGS) -std=c++17 $(MMQ_INCLUDES) -o $@ $^ $(CUDA_LDLIBS) +cuda/mmq/test/test_mmq_parity: cuda/mmq/test/test_mmq_parity.cu cuda/mmq/ds4_mmq.h $(MMQ_OBJS) + $(NVCC) $(NVCCFLAGS) -std=c++17 $(MMQ_INCLUDES) -o $@ $< $(MMQ_OBJS) $(CUDA_LDLIBS) test-mmq-parity-cuda: cuda/mmq/test/test_mmq_parity ./cuda/mmq/test/test_mmq_parity +test-mmq-q4-16warp-cuda: cuda/mmq/test/test_mmq_parity + ./cuda/mmq/test/test_mmq_parity --q4-16warp + speed-bench/gpu_iq2_moe_prefill_bench_rocm.o: speed-bench/gpu_iq2_moe_prefill_bench.c ds4_gpu.h $(CC) $(filter-out -ffast-math,$(CFLAGS)) $(ROCM_HOST_CFLAGS) -std=c11 -DDS4_ROCM_BUILD -DDS4_BENCH_ROCM -I. -c -o $@ $< @@ -582,7 +586,7 @@ speed-bench/gpu_iq2_moe_prefill_bench_cuda: speed-bench/gpu_iq2_moe_prefill_benc cuda-iq2-moe-prefill-bench: $(MAKE) --no-print-directory -B speed-bench/gpu_iq2_moe_prefill_bench_cuda CUDA_ARCH="$(CUDA_ARCH)" -speed-bench/cuda_q4_prefill_bench.o: speed-bench/cuda_q4_prefill_bench.cu ds4_gpu.h +speed-bench/cuda_q4_prefill_bench.o: speed-bench/cuda_q4_prefill_bench.cu ds4_gpu.h cuda/mmq/ds4_mmq.h cuda/mmq/ds4_mmq_q4_16warp.cuh $(NVCC) $(NVCCFLAGS) -std=c++17 -DDS4_BENCH_CUDA -I. -c -o $@ $< speed-bench/cuda_q4_prefill_bench: speed-bench/cuda_q4_prefill_bench.o ds4_cuda.o $(MMQ_OBJS) @@ -734,12 +738,15 @@ ds4_cuda.o: ds4_cuda.cu ds4_gpu.h ds4_gpu_mgpu.h ds4_glm53_vision_gpu.cuh ds4_iq cuda/mmq/ds4_ggml_stubs.o: cuda/mmq/ds4_ggml_stubs.cu cuda/mmq/ds4_mmq.h cuda/mmq/ds4_ggml_stubs.h cuda/mmq/common.cuh $(NVCC) $(NVCCFLAGS) -std=c++17 $(MMQ_INCLUDES) -c -o $@ $< -cuda/mmq/ds4_mmq.o: cuda/mmq/ds4_mmq.cu cuda/mmq/ds4_mmq.h cuda/mmq/ds4_mmq_d2r.cuh cuda/mmq/mmq.cuh cuda/mmq/common.cuh cuda/mmq/ds4_ggml_stubs.h cuda/mmq/quantize.cuh cuda/mmq/mmid.cuh cuda/mmq/vecdotq.cuh cuda/mmq/mma.cuh +cuda/mmq/ds4_mmq.o: cuda/mmq/ds4_mmq.cu cuda/mmq/ds4_mmq.h cuda/mmq/ds4_mmq_d2r.cuh cuda/mmq/ds4_mmq_q4_16warp.cuh cuda/mmq/mmq.cuh cuda/mmq/common.cuh cuda/mmq/ds4_ggml_stubs.h cuda/mmq/quantize.cuh cuda/mmq/mmid.cuh cuda/mmq/vecdotq.cuh cuda/mmq/mma.cuh $(NVCC) $(NVCCFLAGS) -std=c++17 $(MMQ_INCLUDES) -c -o $@ $< cuda/mmq/ds4_mmq_d2r.o: cuda/mmq/ds4_mmq_d2r.cu cuda/mmq/ds4_mmq_d2r.cuh cuda/mmq/mmq.cuh cuda/mmq/common.cuh cuda/mmq/ds4_ggml_stubs.h cuda/mmq/vecdotq.cuh cuda/mmq/mma.cuh $(NVCC) $(NVCCFLAGS) -std=c++17 $(MMQ_INCLUDES) -c -o $@ $< +cuda/mmq/ds4_mmq_q4_16warp.o: cuda/mmq/ds4_mmq_q4_16warp.cu cuda/mmq/ds4_mmq_q4_16warp.cuh cuda/mmq/mmq.cuh cuda/mmq/common.cuh cuda/mmq/ds4_ggml_stubs.h cuda/mmq/vecdotq.cuh cuda/mmq/mma.cuh + $(NVCC) $(NVCCFLAGS) -std=c++17 $(MMQ_INCLUDES) -c -o $@ $< + cuda/mmq/quantize.o: cuda/mmq/quantize.cu cuda/mmq/quantize.cuh cuda/mmq/common.cuh cuda/mmq/ds4_ggml_stubs.h cuda/mmq/mmq.cuh $(NVCC) $(NVCCFLAGS) -std=c++17 $(MMQ_INCLUDES) -c -o $@ $< diff --git a/cuda/mmq/ds4_mmq.cu b/cuda/mmq/ds4_mmq.cu index 10008c03e..4448270d1 100644 --- a/cuda/mmq/ds4_mmq.cu +++ b/cuda/mmq/ds4_mmq.cu @@ -25,6 +25,9 @@ #include "quantize.cuh" #include "mmid.cuh" #include "ds4_mmq_d2r.cuh" +#if !defined(GGML_USE_HIP) +#include "ds4_mmq_q4_16warp.cuh" +#endif #include #include @@ -961,6 +964,181 @@ bool ds4_mmq_k_tile_supported(const char *tag, int K, int cc) { return true; } +#if !defined(GGML_USE_HIP) +static bool ds4_q4_test_q8_1_layout( + int N, int K, size_t *payload_bytes, size_t *total_bytes) { + if (N <= 0 || K <= 0 || (K % QK_K) != 0) return false; + const int64_t padded_k = GGML_PAD((int64_t)K, MATRIX_ROW_PADDING); + const size_t blocks_per_column = + (size_t)padded_k / (4u * (size_t)QK8_1); + if ((size_t)N > SIZE_MAX / blocks_per_column) return false; + const size_t blocks = (size_t)N * blocks_per_column; + if (blocks > SIZE_MAX / sizeof(block_q8_1_mmq)) return false; + const size_t payload = blocks * sizeof(block_q8_1_mmq); + const size_t slack = 128u * sizeof(block_q8_1_mmq); + if (payload > SIZE_MAX - slack) return false; + if (payload_bytes) *payload_bytes = payload; + if (total_bytes) *total_bytes = payload + slack; + return true; +} + +extern "C" size_t ds4_mmq_q4_K_q8_1_scratch_bytes(int N, int K) { + size_t total = 0; + return ds4_q4_test_q8_1_layout(N, K, nullptr, &total) ? total : 0; +} + +extern "C" int ds4_mmq_q4_K_quantize_q8_1_for_test( + const float *X_f32, void *q8_ds4, size_t q8_bytes, + int N, int K, cudaStream_t stream) { + size_t total = 0; + if (!X_f32 || !q8_ds4 || + !ds4_q4_test_q8_1_layout(N, K, nullptr, &total) || + q8_bytes < total) { + return -1; + } + cudaError_t err = cudaMemsetAsync(q8_ds4, 0, total, stream); + if (err != cudaSuccess) return -2; + quantize_mmq_q8_1_cuda( + X_f32, /*ids=*/nullptr, q8_ds4, GGML_TYPE_Q4_K, + /*ne00=*/K, /*s11=*/(int64_t)K, /*s12=*/0, /*s13=*/0, + /*ne0=*/GGML_PAD((int64_t)K, MATRIX_ROW_PADDING), + /*ne1=*/N, /*ne2=*/1, /*ne3=*/1, stream); + err = cudaGetLastError(); + return err == cudaSuccess ? 0 : -3; +} + +extern "C" int ds4_mmq_q4_K_dense_preq_reference_for_test( + const void *W_q4_K, const void *q8_ds4, size_t q8_bytes, + float *out_f32, int M, int N, int K, int use_stream_k, + cudaStream_t stream) { + size_t payload = 0, total = 0; + if (!W_q4_K || !q8_ds4 || !out_f32 || M <= 0 || + !ds4_q4_test_q8_1_layout(N, K, &payload, &total) || + q8_bytes < total) { + return -1; + } + const int dev = ggml_cuda_get_device(); + const int cc = ggml_cuda_info().devices[dev].cc; + if (!ds4_mmq_k_tile_supported( + "ds4_mmq_q4_K_dense_preq_reference_for_test", K, cc)) { + return -1; + } + ggml_backend_cuda_context *ctx = get_ctx_for_device(dev); + if (!ctx) return -1; + ds4_pool_set_stream(stream); + const int64_t stride_row_x = (int64_t)K / QK_K; + const int64_t stride_y = (int64_t)(payload / sizeof(int)); + const mmq_args args = { + /*x=*/(const char *)W_q4_K, + /*type_x=*/GGML_TYPE_Q4_K, + /*y=*/(const int *)q8_ds4, + /*ids_dst=*/nullptr, + /*expert_bounds=*/nullptr, + /*dst=*/out_f32, + /*ncols_x=*/(int64_t)K, /*nrows_x=*/(int64_t)M, + /*ncols_dst=*/(int64_t)N, + /*stride_row_x=*/stride_row_x, /*ncols_y=*/(int64_t)N, + /*nrows_dst=*/(int64_t)M, + /*nchannels_x=*/1, /*nchannels_y=*/1, + /*stride_channel_x=*/0, /*stride_channel_y=*/stride_y, + /*stride_channel_dst=*/0, + /*nsamples_x=*/1, /*nsamples_y=*/1, + /*stride_sample_x=*/0, /*stride_sample_y=*/stride_y, + /*stride_sample_dst=*/0, + /*use_stream_k=*/use_stream_k != 0, + /*ncols_max=*/(int64_t)N, + }; + mul_mat_q_case(*ctx, args, stream); + const cudaError_t err = cudaGetLastError(); + return err == cudaSuccess ? 0 : -2; +} + +static int ds4_q4_16warp_prepare_once(int device); + +extern "C" int ds4_mmq_q4_K_dense_preq_16warp_for_test( + const void *W_q4_K, const void *q8_ds4, size_t q8_bytes, + float *out_f32, int M, int N, int K, cudaStream_t stream) { + size_t total = 0; + if (!W_q4_K || !q8_ds4 || !out_f32 || + !ds4_q4_test_q8_1_layout(N, K, nullptr, &total) || + q8_bytes < total) { + return -1; + } + // This direct oracle hook deliberately accepts an N tail (the production + // selector is stricter until NVIDIA measurements justify broadening it), + // but it must reject shapes that cannot be represented by this kernel + // before enqueueing any work. + if (M < 128 || (M % 128) != 0 || N < 512 || + K < 1024 || K > 4096 || (K % QK_K) != 0) { + return DS4_MMQ_NOT_APPLICABLE; + } + const int dev = ggml_cuda_get_device(); + const int cc = ggml_cuda_info().devices[dev].cc; + if (!ds4_mmq_q4_K_dense_16warp_available(cc) || + ds4_q4_16warp_prepare_once(dev) != 0) { + return DS4_MMQ_NOT_APPLICABLE; + } + return ds4_mmq_q4_K_dense_16warp_enqueue( + W_q4_K, q8_ds4, out_f32, M, N, K, stream); +} + +enum { + DS4_Q4_16WARP_REQUEST = 1, + DS4_Q4_16WARP_REQUIRE = 2, + DS4_Q4_16WARP_DISABLE = 4, +}; + +static bool ds4_q4_16warp_env_enabled(const char *name) { + const char *value = getenv(name); + return value && value[0] && !(value[0] == '0' && value[1] == '\0'); +} + +static int ds4_q4_16warp_mode(void) { + static const int cached = [] { + int mode = 0; + if (ds4_q4_16warp_env_enabled("DS4_CUDA_Q4_MMQ_16WARP")) { + mode |= DS4_Q4_16WARP_REQUEST; + } + if (ds4_q4_16warp_env_enabled( + "DS4_CUDA_REQUIRE_Q4_MMQ_16WARP")) { + mode |= DS4_Q4_16WARP_REQUEST | DS4_Q4_16WARP_REQUIRE; + } + if (ds4_q4_16warp_env_enabled("DS4_CUDA_NO_Q4_MMQ_16WARP")) { + mode |= DS4_Q4_16WARP_DISABLE; + } + return mode; + }(); + return cached; +} + +static int ds4_q4_16warp_prepare_once(int device) { + static std::mutex mutex; + static std::atomic state[GGML_CUDA_MAX_DEVICES]; + if (device < 0 || device >= GGML_CUDA_MAX_DEVICES) return -1; + int cached = state[device].load(std::memory_order_acquire); + if (cached == 1) return 0; + if (cached < 0) return cached; + std::lock_guard lock(mutex); + cached = state[device].load(std::memory_order_relaxed); + if (cached == 1) return 0; + if (cached < 0) return cached; + const int rc = ds4_mmq_q4_K_dense_16warp_prepare(); + state[device].store(rc == 0 ? 1 : rc, std::memory_order_release); + return rc; +} + +static bool ds4_q4_canonical_owns_complete_k(int M, int N, int nsm) { + if (M <= 0 || N <= 0 || nsm <= 0) return false; + const int64_t tiles_m = ((int64_t)M + 127) / 128; + const int64_t tiles_n = ((int64_t)N + 127) / 128; + if (tiles_m > INT64_MAX / tiles_n) return false; + const int64_t tiles = tiles_m * tiles_n; + const int64_t waves = (tiles + nsm - 1) / nsm; + if (waves <= 0 || (int64_t)nsm > INT64_MAX / waves) return false; + return (100 * tiles) / ((int64_t)nsm * waves) >= 90; +} +#endif + template int ds4_mmq_dense_impl( const char * tag, @@ -991,6 +1169,48 @@ int ds4_mmq_dense_impl( const int cc = ggml_cuda_info().devices[dev].cc; if (!ds4_mmq_k_tile_supported(tag, K, cc)) return -1; + bool use_q4_16warp = false; +#if !defined(GGML_USE_HIP) + if constexpr (type == GGML_TYPE_Q4_K) { + const int mode = ds4_q4_16warp_mode(); + const bool disabled = (mode & DS4_Q4_16WARP_DISABLE) != 0; + const bool requested = (mode & DS4_Q4_16WARP_REQUEST) != 0; + const bool required = (mode & DS4_Q4_16WARP_REQUIRE) != 0; + const bool shape_supported = + ds4_mmq_q4_K_dense_16warp_supported(cc, M, N, K) != 0; + // The complete-K parity gate below models canonical m128n128 MMQ. + // A DS4_CUDA_MMQ_X_MAX sweep can make the reference choose a smaller + // X tile and therefore a different stream-K/fixup decision. + const bool canonical_x128 = get_mmq_x_max_host(cc) == 128; + const bool complete_k = ds4_q4_canonical_owns_complete_k( + M, N, ggml_cuda_info().devices[dev].nsm); + if (required && (disabled || !shape_supported || !canonical_x128 || + !complete_k)) { + fprintf(stderr, + "%s: required Q4 16-warp path is ineligible " + "(disabled=%d shape=%d x128=%d complete_k=%d " + "M=%d N=%d K=%d)\n", + tag, disabled ? 1 : 0, shape_supported ? 1 : 0, + canonical_x128 ? 1 : 0, complete_k ? 1 : 0, M, N, K); + return DS4_MMQ_NOT_APPLICABLE; + } + if (requested && !disabled && shape_supported && canonical_x128 && + complete_k) { + const int prep = ds4_q4_16warp_prepare_once(dev); + if (prep == 0) { + use_q4_16warp = true; + } else if (required) { + fprintf(stderr, + "%s: required Q4 16-warp preflight failed: %d\n", + tag, prep); + return DS4_MMQ_NOT_APPLICABLE; + } else { + (void)cudaGetLastError(); + } + } + } +#endif + ggml_backend_cuda_context * ctx = get_ctx_for_device(dev); if (!ctx) { fprintf(stderr, "%s: failed to get cuda context for device %d\n", tag, dev); @@ -1087,6 +1307,20 @@ int ds4_mmq_dense_impl( (void)cudaMemsetAsync(out_f32, 0, (size_t)M * (size_t)N * sizeof(float), stream); } +#if !defined(GGML_USE_HIP) + if constexpr (type == GGML_TYPE_Q4_K) { + if (use_q4_16warp) { + const int rc = ds4_mmq_q4_K_dense_16warp_enqueue( + W, src1_q8_1.get(), out_f32, M, N, K, stream); + if (rc != 0) { + fprintf(stderr, "%s: Q4 16-warp launch failed: %d\n", tag, rc); + return -3; + } + return 0; + } + } +#endif + const mmq_args args = { /*x=*/(const char *)W, /*type_x=*/type, diff --git a/cuda/mmq/ds4_mmq.h b/cuda/mmq/ds4_mmq.h index 85a19cf3b..67daae548 100644 --- a/cuda/mmq/ds4_mmq.h +++ b/cuda/mmq/ds4_mmq.h @@ -188,6 +188,44 @@ int ds4_mmq_q4_K_dense( int K, cudaStream_t stream); +// CUDA benchmark/test boundary for separating activation quantization from +// the Q4_K MMQ kernel. The scratch layout is canonical block_q8_1_mmq DS4 +// ([K/128][N]) plus a zeroed 128-column tail. These helpers never transfer +// model weights and never synchronize the stream. +size_t ds4_mmq_q4_K_q8_1_scratch_bytes(int N, int K); + +int ds4_mmq_q4_K_quantize_q8_1_for_test( + const float * X_f32, + void * q8_ds4, + size_t q8_bytes, + int N, + int K, + cudaStream_t stream); + +// Enqueue-only A/B arms over a caller-owned, already-quantized activation. +// The reference can disable stream-K to give the 16-warp candidate the same +// complete-K reduction tree. Zero is success; nonzero is rejection/failure. +int ds4_mmq_q4_K_dense_preq_reference_for_test( + const void * W_q4_K, + const void * q8_ds4, + size_t q8_bytes, + float * out_f32, + int M, + int N, + int K, + int use_stream_k, + cudaStream_t stream); + +int ds4_mmq_q4_K_dense_preq_16warp_for_test( + const void * W_q4_K, + const void * q8_ds4, + size_t q8_bytes, + float * out_f32, + int M, + int N, + int K, + cudaStream_t stream); + // Two dense Q4_K MMQ projections that share one token-tiled Q8_1 // activation buffer. This is the prefill sibling of // ds4_mmq_q4_K_dense_pair_vec: N is not limited to the MMVQ batch ceiling, diff --git a/cuda/mmq/ds4_mmq_q4_16warp.cu b/cuda/mmq/ds4_mmq_q4_16warp.cu new file mode 100644 index 000000000..38f05d052 --- /dev/null +++ b/cuda/mmq/ds4_mmq_q4_16warp.cu @@ -0,0 +1,360 @@ +// SPDX-License-Identifier: MIT +// Dense Q4_K x canonical-MMQ-Q8_1, m128n128, 16-warp experiment. +// +// The canonical Turing/Ampere MMQ kernel assigns two 16-row MMA minitiles to +// each of eight warps at N=128. That leaves 64 F32 accumulators per thread +// and can spill on shallow-K, wide-M prefill projections. This kernel keeps +// the canonical shared representation and arithmetic but splits each 128-row +// tile over four N-warps for each 32-row band. Each warp therefore owns +// 32 rows x 32 columns and carries 32 accumulators. Keeping both 16-row A +// fragments in one warp also preserves the canonical reuse of each B load. +// +// Numerical contract: +// * canonical Q4_K nibble/scales/min unpack; +// * canonical Q8_1 DS4 (half scale + half sum) activation blocks; +// * identical ascending sequence of eight K32 folds per Q4_K block; +// * the two canonical F32 accumulation statements are kept verbatim; +// * one CTA owns the complete K reduction (no stream-K/fixup tree). + +#include "ds4_mmq_q4_16warp.cuh" + +#include "common.cuh" +#include "mmq.cuh" + +#include +#include + +namespace { +namespace q4w16 { + +constexpr int kMTile = 128; +constexpr int kNTile = 128; +constexpr int kRowGroups = 4; +constexpr int kColWarps = 4; +constexpr int kWarps = kRowGroups * kColWarps; +constexpr int kThreads = 32 * kWarps; +constexpr int kRowFrag = 2; +constexpr int kNFrag = kNTile / 8; +constexpr int kNFragPerWarp = kNFrag / kColWarps; +constexpr int kMetadataWarps = kMTile / 16; +constexpr int kWeightStride = MMQ_MMA_TILE_X_K_Q8_1; +constexpr int kYStrideInts = sizeof(block_q8_1_mmq) / sizeof(int); +constexpr int kYChunks16 = sizeof(block_q8_1_mmq) / 16; + +constexpr size_t kWeightTileBytes = + (size_t)kMTile * (size_t)kWeightStride * sizeof(int); +constexpr size_t kYTileBytes = + (size_t)kNTile * sizeof(block_q8_1_mmq); +constexpr size_t kSharedBytes = kWeightTileBytes + kYTileBytes; + +static_assert(kWarps == 16, "Q4 16-warp decomposition changed"); +static_assert(kThreads == 512, "Q4 16-warp CTA must have 512 threads"); +static_assert(kRowGroups * kRowFrag * 16 == kMTile, + "Q4 split-N row coverage changed"); +static_assert(kNFragPerWarp == 4, "Q4 split-N fragment count changed"); +static_assert(kWeightStride == 76, "canonical Q4_K MMA row stride changed"); +static_assert(kYStrideInts == 36, "canonical Q8_1 DS4 stride changed"); +static_assert(kYChunks16 == 9, "canonical Q8_1 DS4 block size changed"); +static_assert(kSharedBytes == 57344, "Q4 16-warp shared-memory model changed"); +static_assert(kSharedBytes <= 99ull * 1024ull, + "Q4 16-warp kernel exceeds the intended opt-in shared limit"); + +__device__ __forceinline__ int lane_id() { + return (int)threadIdx.x; +} + +__device__ __forceinline__ int warp_id() { + return (int)threadIdx.y; +} + +__device__ __forceinline__ int linear_tid() { + return (warp_id() << 5) | lane_id(); +} + +__device__ __forceinline__ void load_weight_tile( + const block_q4_K * __restrict__ W, + int * __restrict__ tile, + int cta_row0, + int blocks_per_row, + int kb) { + int *x_qs = tile; + half2 *x_dm = reinterpret_cast(x_qs + 2 * MMQ_TILE_NE_K); + const int lane = lane_id(); + const int warp = warp_id(); + + // Canonical load_tiles_q4_K nibble expansion. With 16 warps each warp + // visits eight rows; all 128 rows are covered exactly once. +#pragma unroll + for (int row = warp; row < kMTile; row += kWarps) { + const block_q4_K *b = + W + (uint64_t)(cta_row0 + row) * (uint64_t)blocks_per_row + kb; + const int qs0 = get_int_b4(b->qs, lane); + x_qs[row * kWeightStride + 16 * (lane / 8) + lane % 8 + 0] = + (qs0 >> 0) & 0x0F0F0F0F; + x_qs[row * kWeightStride + 16 * (lane / 8) + lane % 8 + 8] = + (qs0 >> 4) & 0x0F0F0F0F; + } + + // The canonical loader uses 16 rows/warp and two lanes/row for metadata. + // Only eight warps participate, so the extra split-N warps do not + // duplicate any metadata row. + if (warp < kMetadataWarps) { + const int row = warp * 16 + lane / 2; + const int ksc = lane & 1; + const block_q4_K *b = + W + (uint64_t)(cta_row0 + row) * (uint64_t)blocks_per_row + kb; + const int *scales = reinterpret_cast(b->scales); + const int sc32 = unpack_scales_q45_K(scales, ksc + 0); + const int m32 = unpack_scales_q45_K(scales, ksc + 2); + const uint8_t *sc8 = reinterpret_cast(&sc32); + const uint8_t *m8 = reinterpret_cast(&m32); + const half2 dm = b->dm * make_half2(1.0f, -1.0f); +#pragma unroll + for (int l = 0; l < (int)sizeof(int); ++l) { + x_dm[row * kWeightStride + (int)sizeof(int) * ksc + l] = + dm * make_half2(sc8[l], m8[l]); + } + } +} + +__device__ __forceinline__ void load_y_tile( + const block_q8_1_mmq * __restrict__ q8, + block_q8_1_mmq * __restrict__ tile, + int N, + int col0, + int k128) { + const int tid = linear_tid(); + constexpr int threads_per_col = kThreads / kNTile; + static_assert(threads_per_col == 4, + "Q8_1 DS4 copy mapping changed"); + const int col = tid & (kNTile - 1); +#pragma unroll + for (int chunk = tid >> 7; chunk < kYChunks16; + chunk += threads_per_col) { + int4 value = make_int4(0, 0, 0, 0); + if (col0 + col < N) { + const char *src = reinterpret_cast( + q8 + (uint64_t)k128 * (uint64_t)N + (uint64_t)(col0 + col)); + value = *reinterpret_cast(src + chunk * 16); + } + char *dst = reinterpret_cast(tile + col); + *reinterpret_cast(dst + chunk * 16) = value; + } +} + +template +__device__ __forceinline__ void fold_y_half( + float (&acc)[kNFragPerWarp][kRowFrag][TileC::ne], + const int * __restrict__ x_tile, + const block_q8_1_mmq * __restrict__ y_tile, + int x_group0) { + static_assert(TileC::ne == 4, + "expected m16n8 s32 accumulator fragment"); + const half2 *x_dm = reinterpret_cast( + x_tile + 2 * MMQ_TILE_NE_K); + const int warp = warp_id(); + const int row0 = (warp / kColWarps) * (kRowFrag * 16); + const int nf0 = (warp % kColWarps) * kNFragPerWarp; + const int c0 = TileC::get_j(0); + const int c1 = TileC::get_j(1); + const int r0 = TileC::get_i(0); + const int r1 = TileC::get_i(2); + + // K32-phased A loads keep only the two fragments needed for this 32-row + // band live, instead of canonical MMQ's eight K32 phases at once. Each + // B fragment is reused by both A fragments exactly as in canonical MMQ. + // For every output element folds remain in canonical group order 0..7. +#pragma unroll + for (int local_group = 0; local_group < 4; ++local_group) { + const int x_group = x_group0 + local_group; + TileA A[kRowFrag]; + float2 dmA[kRowFrag][2]; +#pragma unroll + for (int nr = 0; nr < kRowFrag; ++nr) { + const int frag_row0 = row0 + nr * 16; + ggml_cuda_mma::load_ldmatrix( + A[nr], + x_tile + frag_row0 * kWeightStride + x_group * QI8_1, + kWeightStride); + dmA[nr][0] = __half22float2( + x_dm[(frag_row0 + r0) * kWeightStride + x_group]); + dmA[nr][1] = __half22float2( + x_dm[(frag_row0 + r1) * kWeightStride + x_group]); + } + +#pragma unroll + for (int nf = 0; nf < kNFragPerWarp; ++nf) { + const int col_base = (nf0 + nf) * 8; + TileB B; + const int *b_qs = reinterpret_cast( + &y_tile[col_base].qs[local_group * QK8_1]); + // Canonical NVIDIA MMQ deliberately uses load_generic for B. + ggml_cuda_mma::load_generic(B, b_qs, kYStrideInts); + + const float2 dsB[2] = { + __half22float2(y_tile[col_base + c0].ds4[local_group]), + __half22float2(y_tile[col_base + c1].ds4[local_group]), + }; + + // These are the canonical vec_dot_q8_1_q8_1_mma accumulation + // statements. Do not fuse the min correction into the dot fold + // or change their order: parity depends on this reduction tree. +#pragma unroll + for (int nr = 0; nr < kRowFrag; ++nr) { + TileC C; + ggml_cuda_mma::mma(C, A[nr], B); +#pragma unroll + for (int l = 0; l < TileC::ne; ++l) { + acc[nf][nr][l] += + dmA[nr][l / 2].x * dsB[l % 2].x * C.x[l]; + acc[nf][nr][l] += + dmA[nr][l / 2].y * dsB[l % 2].y; + } + } + } + } +} + +__global__ __launch_bounds__(kThreads, 1) +void dense_q4_16warp_kernel( + const block_q4_K * __restrict__ W, + const block_q8_1_mmq * __restrict__ q8, + float * __restrict__ out, + int M, + int N, + int K) { +#if defined(TURING_MMA_AVAILABLE) + using tile_A = ggml_cuda_mma::tile<16, 8, int>; + using tile_B = ggml_cuda_mma::tile<8, 8, int>; + using tile_C = ggml_cuda_mma::tile<16, 8, int>; + + extern __shared__ __align__(16) unsigned char dynamic_smem[]; + int *x_tile = reinterpret_cast(dynamic_smem); + block_q8_1_mmq *y_tile = reinterpret_cast( + dynamic_smem + kWeightTileBytes); + + const int cta_row0 = (int)blockIdx.x * kMTile; + const int col0 = (int)blockIdx.y * kNTile; + const int blocks_per_row = K / QK_K; + float acc[kNFragPerWarp][kRowFrag][tile_C::ne] = {}; + + for (int kb = 0; kb < blocks_per_row; ++kb) { + load_weight_tile(W, x_tile, cta_row0, blocks_per_row, kb); + load_y_tile(q8, y_tile, N, col0, 2 * kb + 0); + __syncthreads(); + + fold_y_half(acc, x_tile, y_tile, 0); + __syncthreads(); + + load_y_tile(q8, y_tile, N, col0, 2 * kb + 1); + __syncthreads(); + + fold_y_half(acc, x_tile, y_tile, 4); + // Protect both shared tiles before the following K256 iteration. + __syncthreads(); + } + + const int warp = warp_id(); + const int out_row0 = + cta_row0 + (warp / kColWarps) * (kRowFrag * 16); + const int out_col0 = + col0 + (warp % kColWarps) * (kNFragPerWarp * 8); +#pragma unroll + for (int nf = 0; nf < kNFragPerWarp; ++nf) { +#pragma unroll + for (int nr = 0; nr < kRowFrag; ++nr) { +#pragma unroll + for (int l = 0; l < tile_C::ne; ++l) { + const int row = out_row0 + nr * 16 + tile_C::get_i(l); + const int col = out_col0 + nf * 8 + tile_C::get_j(l); + if (col < N) { + const float value = isfinite(acc[nf][nr][l]) + ? acc[nf][nr][l] : 0.0f; + out[(uint64_t)col * (uint64_t)M + (uint64_t)row] = + value; + } + } + } + } +#else + GGML_UNUSED_VARS(W, q8, out, M, N, K); + NO_DEVICE_CODE; +#endif +} + +} // namespace q4w16 +} // anonymous namespace + +extern "C" int ds4_mmq_q4_K_dense_16warp_available(int cc) { + return GGML_CUDA_CC_IS_NVIDIA(cc) && + ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_AMPERE; +} + +extern "C" int ds4_mmq_q4_K_dense_16warp_supported( + int cc, int M, int N, int K) { + if (!ds4_mmq_q4_K_dense_16warp_available(cc)) { + return 0; + } + return M >= 2048 && (M % q4w16::kMTile) == 0 && + N >= 512 && (N % q4w16::kNTile) == 0 && + K >= 1024 && K <= 4096 && (K % QK_K) == 0; +} + +extern "C" int ds4_mmq_q4_K_dense_16warp_prepare(void) { + using namespace q4w16; + int device = -1; + cudaError_t err = cudaGetDevice(&device); + if (err != cudaSuccess) { + return -1; + } + cudaDeviceProp prop; + err = cudaGetDeviceProperties(&prop, device); + if (err != cudaSuccess || prop.major < 8 || + prop.maxThreadsPerBlock < kThreads) { + return -2; + } +#if CUDART_VERSION >= 9000 + if ((size_t)prop.sharedMemPerBlockOptin < kSharedBytes) { + return -2; + } +#else + if ((size_t)prop.sharedMemPerBlock < kSharedBytes) { + return -2; + } +#endif + err = cudaFuncSetAttribute( + dense_q4_16warp_kernel, + cudaFuncAttributeMaxDynamicSharedMemorySize, + (int)kSharedBytes); + return err == cudaSuccess ? 0 : -3; +} + +extern "C" int ds4_mmq_q4_K_dense_16warp_enqueue( + const void *W, + const void *q8_ds4, + float *out, + int M, + int N, + int K, + cudaStream_t stream) { + using namespace q4w16; + if (!W || !q8_ds4 || !out || M <= 0 || N <= 0 || K <= 0 || + (M % kMTile) != 0 || (K % QK_K) != 0) { + return -1; + } + + // Convert before adding the tile bias: N is a positive signed int, but + // N + 127 would otherwise overflow for a (syntactically valid) INT_MAX + // direct-enqueue request. + const dim3 grid((unsigned)M / (unsigned)kMTile, + ((unsigned)N + (unsigned)kNTile - 1u) / + (unsigned)kNTile, + 1); + const dim3 block(32, kWarps, 1); + dense_q4_16warp_kernel<<>>( + static_cast(W), + static_cast(q8_ds4), + out, M, N, K); + const cudaError_t err = cudaGetLastError(); + return err == cudaSuccess ? 0 : -4; +} diff --git a/cuda/mmq/ds4_mmq_q4_16warp.cuh b/cuda/mmq/ds4_mmq_q4_16warp.cuh new file mode 100644 index 000000000..cb76aa607 --- /dev/null +++ b/cuda/mmq/ds4_mmq_q4_16warp.cuh @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: MIT +// Internal CUDA Q4_K dense-prefill experiment. This header intentionally +// exposes only the pre-quantized enqueue boundary; allocation and Q8_1 +// quantization stay owned by ds4_mmq.cu. + +#pragma once + +#if defined(GGML_USE_HIP) +#include "vendors/hip.h" +#else +#include +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +// Returns non-zero when the requested CUDA compute capability can execute +// the m128n128, 16-warp integer-MMA kernel. +int ds4_mmq_q4_K_dense_16warp_available(int cc); + +// Conservative production admission gate. Availability and shape are both +// checked; the initial gate admits only complete 128x128 output tiles so the +// canonical A/B arm also selects mmq_x=128. A false result must fall back. +int ds4_mmq_q4_K_dense_16warp_supported(int cc, int M, int N, int K); + +// Opt in the 56 KiB dynamic-shared-memory launch on the current device. +// Call once during device initialization (and once again after switching to a +// different device) before enqueue. The operation is idempotent. +int ds4_mmq_q4_K_dense_16warp_prepare(void); + +// Enqueue-only dense Q4_K GEMM over an already resident canonical MMQ Q8_1 +// activation buffer. +// +// W raw row-major block_q4_K, [M][K/256] +// q8_ds4 block_q8_1_mmq DS4 (half scale + half sum), [K/128][N] +// out column-major float, [N][M] +// +// ds4_mmq_q4_K_dense_16warp_prepare must have succeeded on the current device. +// The kernel owns the complete K reduction for every output tile: it never +// uses stream-K and writes every valid output exactly once. No allocation, +// memset, quantization, synchronization, or host/device copy is performed. +// Returns 0 after a successful enqueue and a negative value otherwise. +int ds4_mmq_q4_K_dense_16warp_enqueue( + const void * W, + const void * q8_ds4, + float * out, + int M, + int N, + int K, + cudaStream_t stream); + +#ifdef __cplusplus +} +#endif diff --git a/cuda/mmq/test/test_mmq_parity.cu b/cuda/mmq/test/test_mmq_parity.cu index e7a813654..9ba4217b9 100644 --- a/cuda/mmq/test/test_mmq_parity.cu +++ b/cuda/mmq/test/test_mmq_parity.cu @@ -548,6 +548,289 @@ bool run_q4_K(int M, int N, int K, uint32_t seed, float abs_scale = 0.20f) { return ok; } +// Resident-kernel oracle for the opt-in m128n128/16-warp Q4_K prefill +// candidate. All arms consume the same caller-owned DS4 Q8_1 activation. +// The complete-K reference is always checked; for a no-fixup shape the real +// stream-K production arm must also be bit-identical. Guard regions catch +// output overruns independently of numerical parity. +bool run_q4_K_dense_16warp_parity( + int M, int N, int K, uint32_t seed, + bool check_production, bool check_rejection) { + fprintf(stderr, + "=== Q4_K/DENSE_16WARP M=%d N=%d K=%d seed=%u%s ===\n", + M, N, K, seed, check_production ? " stream-k" : ""); + + if (M <= 0 || N <= 0 || K <= 0 || K % QK_K_LOCAL != 0 || + (size_t)M > SIZE_MAX / (size_t)N / sizeof(float)) { + fprintf(stderr, "invalid 16-warp parity shape\n\n"); + return false; + } + + std::mt19937 rng(seed); + std::normal_distribution nd(0.0f, 1.0f); + const int blocks_per_row = K / QK_K_LOCAL; + std::vector W((size_t)M * blocks_per_row); + for (auto &blk : W) generate_random_block_q4_K(&blk, rng); + std::vector X((size_t)N * K); + for (float &v : X) v = nd(rng); + + const size_t q8_bytes = ds4_mmq_q4_K_q8_1_scratch_bytes(N, K); + if (q8_bytes == 0) { + fprintf(stderr, "Q4_K 16-warp scratch-size query rejected shape\n\n"); + return false; + } + + constexpr size_t guard_floats = 64; + constexpr uint8_t ref_guard_byte = 0xa5; + constexpr uint8_t prod_guard_byte = 0xc3; + constexpr uint8_t got_guard_byte = 0x5a; + constexpr uint8_t reject_byte = 0x3c; + const size_t output_count = (size_t)M * N; + const size_t output_bytes = output_count * sizeof(float); + const size_t guard_bytes = guard_floats * sizeof(float); + const size_t guarded_bytes = output_bytes + 2u * guard_bytes; + + cudaStream_t stream = nullptr; + void *dW = nullptr; + float *dX = nullptr; + void *dQ8 = nullptr; + float *dRefStorage = nullptr; + float *dProdStorage = nullptr; + float *dGotStorage = nullptr; + const bool allocated = cudaStreamCreate(&stream) == cudaSuccess && + cudaMalloc(&dW, W.size() * sizeof(block_q4_K)) == cudaSuccess && + cudaMalloc(&dX, X.size() * sizeof(float)) == cudaSuccess && + cudaMalloc(&dQ8, q8_bytes) == cudaSuccess && + cudaMalloc(&dRefStorage, guarded_bytes) == cudaSuccess && + (!check_production || + cudaMalloc(&dProdStorage, guarded_bytes) == cudaSuccess) && + cudaMalloc(&dGotStorage, guarded_bytes) == cudaSuccess; + const auto cleanup = [&]() { + if (dGotStorage) cudaFree(dGotStorage); + if (dProdStorage) cudaFree(dProdStorage); + if (dRefStorage) cudaFree(dRefStorage); + if (dQ8) cudaFree(dQ8); + if (dX) cudaFree(dX); + if (dW) cudaFree(dW); + if (stream) cudaStreamDestroy(stream); + }; + if (!allocated) { + fprintf(stderr, "Q4_K 16-warp parity allocation failed: %s\n\n", + cudaGetErrorString(cudaGetLastError())); + cleanup(); + return false; + } + + float *const dRef = dRefStorage + guard_floats; + float *const dProd = check_production + ? dProdStorage + guard_floats : nullptr; + float *const dGot = dGotStorage + guard_floats; + cudaError_t enqueue_err = cudaMemcpyAsync( + dW, W.data(), W.size() * sizeof(block_q4_K), + cudaMemcpyHostToDevice, stream); + if (enqueue_err == cudaSuccess) { + enqueue_err = cudaMemcpyAsync( + dX, X.data(), X.size() * sizeof(float), + cudaMemcpyHostToDevice, stream); + } + if (enqueue_err == cudaSuccess) { + enqueue_err = cudaMemsetAsync( + dRefStorage, ref_guard_byte, guarded_bytes, stream); + } + if (enqueue_err == cudaSuccess && check_production) { + enqueue_err = cudaMemsetAsync( + dProdStorage, prod_guard_byte, guarded_bytes, stream); + } + if (enqueue_err == cudaSuccess) { + enqueue_err = cudaMemsetAsync( + dGotStorage, got_guard_byte, guarded_bytes, stream); + } + + const int rc_quant = enqueue_err == cudaSuccess + ? ds4_mmq_q4_K_quantize_q8_1_for_test( + dX, dQ8, q8_bytes, N, K, stream) + : -100; + const int rc_ref = rc_quant == 0 + ? ds4_mmq_q4_K_dense_preq_reference_for_test( + dW, dQ8, q8_bytes, dRef, M, N, K, + /*use_stream_k=*/0, stream) + : -100; + const int rc_prod = rc_ref != 0 ? -100 : check_production + ? ds4_mmq_q4_K_dense_preq_reference_for_test( + dW, dQ8, q8_bytes, dProd, M, N, K, + /*use_stream_k=*/1, stream) + : 0; + const int rc_got = rc_prod == 0 + ? ds4_mmq_q4_K_dense_preq_16warp_for_test( + dW, dQ8, q8_bytes, dGot, M, N, K, stream) + : -100; + + std::vector ref_guarded(guarded_bytes); + std::vector prod_guarded( + check_production ? guarded_bytes : 0u); + std::vector got_guarded(guarded_bytes); + if (rc_got == 0) { + enqueue_err = cudaMemcpyAsync( + ref_guarded.data(), dRefStorage, guarded_bytes, + cudaMemcpyDeviceToHost, stream); + } + if (enqueue_err == cudaSuccess && rc_got == 0 && check_production) { + enqueue_err = cudaMemcpyAsync( + prod_guarded.data(), dProdStorage, guarded_bytes, + cudaMemcpyDeviceToHost, stream); + } + if (enqueue_err == cudaSuccess && rc_got == 0) { + enqueue_err = cudaMemcpyAsync( + got_guarded.data(), dGotStorage, guarded_bytes, + cudaMemcpyDeviceToHost, stream); + } + const cudaError_t sync_err = cudaStreamSynchronize(stream); + + size_t ref_got_mismatches = 0; + size_t ref_prod_mismatches = 0; + size_t prod_got_mismatches = 0; + size_t nonfinite_ref = 0; + size_t nonfinite_prod = 0; + size_t nonfinite_got = 0; + uint32_t first_ref_bits = 0; + uint32_t first_got_bits = 0; + uint32_t first_prod_bits = 0; + size_t first_ref_got_bad = SIZE_MAX; + size_t first_ref_prod_bad = SIZE_MAX; + if (enqueue_err == cudaSuccess && sync_err == cudaSuccess && rc_got == 0) { + for (size_t i = 0; i < output_count; ++i) { + const uint8_t *const ref_bits_ptr = + ref_guarded.data() + guard_bytes + i * sizeof(float); + const uint8_t *const got_bits_ptr = + got_guarded.data() + guard_bytes + i * sizeof(float); + const uint8_t *const prod_bits_ptr = check_production + ? prod_guarded.data() + guard_bytes + i * sizeof(float) + : nullptr; + float ref_value = 0.0f; + float prod_value = 0.0f; + float got_value = 0.0f; + std::memcpy(&ref_value, ref_bits_ptr, sizeof(ref_value)); + std::memcpy(&got_value, got_bits_ptr, sizeof(got_value)); + if (check_production) { + std::memcpy(&prod_value, prod_bits_ptr, sizeof(prod_value)); + } + if (!std::isfinite(ref_value)) nonfinite_ref++; + if (check_production && !std::isfinite(prod_value)) { + nonfinite_prod++; + } + if (!std::isfinite(got_value)) nonfinite_got++; + if (std::memcmp(ref_bits_ptr, got_bits_ptr, sizeof(float)) != 0) { + if (first_ref_got_bad == SIZE_MAX) { + first_ref_got_bad = i; + std::memcpy(&first_ref_bits, ref_bits_ptr, + sizeof(first_ref_bits)); + std::memcpy(&first_got_bits, got_bits_ptr, + sizeof(first_got_bits)); + } + ref_got_mismatches++; + } + if (check_production && + std::memcmp(ref_bits_ptr, prod_bits_ptr, sizeof(float)) != 0) { + if (first_ref_prod_bad == SIZE_MAX) { + first_ref_prod_bad = i; + std::memcpy(&first_prod_bits, prod_bits_ptr, + sizeof(first_prod_bits)); + } + ref_prod_mismatches++; + } + if (check_production && + std::memcmp(prod_bits_ptr, got_bits_ptr, sizeof(float)) != 0) { + prod_got_mismatches++; + } + } + } + + const auto guard_mismatches = [=]( + const std::vector &bytes, uint8_t expected) { + size_t bad = 0; + for (size_t i = 0; i < guard_bytes; ++i) { + if (bytes[i] != expected) bad++; + } + for (size_t i = guard_bytes + output_bytes; i < bytes.size(); ++i) { + if (bytes[i] != expected) bad++; + } + return bad; + }; + const size_t ref_canary = guard_mismatches(ref_guarded, ref_guard_byte); + const size_t prod_canary = check_production + ? guard_mismatches(prod_guarded, prod_guard_byte) : 0; + const size_t got_canary = guard_mismatches(got_guarded, got_guard_byte); + + int rc_reject = DS4_MMQ_NOT_APPLICABLE; + size_t reject_writes = 0; + cudaError_t reject_sync = cudaSuccess; + if (check_rejection && enqueue_err == cudaSuccess && + sync_err == cudaSuccess) { + cudaError_t reject_err = cudaMemsetAsync( + dGotStorage, reject_byte, guarded_bytes, stream); + rc_reject = reject_err == cudaSuccess + ? ds4_mmq_q4_K_dense_preq_16warp_for_test( + dW, dQ8, q8_bytes, dGot, M, /*N=*/511, K, stream) + : -100; + std::vector rejected(guarded_bytes); + if (reject_err == cudaSuccess) { + reject_err = cudaMemcpyAsync( + rejected.data(), dGotStorage, guarded_bytes, + cudaMemcpyDeviceToHost, stream); + } + reject_sync = cudaStreamSynchronize(stream); + if (reject_err == cudaSuccess && reject_sync == cudaSuccess) { + for (uint8_t byte : rejected) { + if (byte != reject_byte) reject_writes++; + } + } else { + reject_writes = SIZE_MAX; + } + } + + const bool rejection_ok = !check_rejection || + (rc_reject == DS4_MMQ_NOT_APPLICABLE && + reject_sync == cudaSuccess && reject_writes == 0); + const bool production_ok = !check_production || + (ref_prod_mismatches == 0 && prod_got_mismatches == 0 && + nonfinite_prod == 0 && prod_canary == 0); + const bool ok = rc_quant == 0 && rc_ref == 0 && rc_prod == 0 && + rc_got == 0 && enqueue_err == cudaSuccess && + sync_err == cudaSuccess && ref_got_mismatches == 0 && + nonfinite_ref == 0 && nonfinite_got == 0 && ref_canary == 0 && + got_canary == 0 && production_ok && rejection_ok; + fprintf(stderr, + "quant/ref/stream/16w=%d/%d/%d/%d enqueue=%s sync=%s " + "bits(ref-16w/ref-stream/stream-16w)=%zu/%zu/%zu " + "nonfinite=%zu/%zu/%zu canary=%zu/%zu/%zu " + "reject=%d/%zu: %s\n", + rc_quant, rc_ref, rc_prod, rc_got, + cudaGetErrorString(enqueue_err), cudaGetErrorString(sync_err), + ref_got_mismatches, ref_prod_mismatches, prod_got_mismatches, + nonfinite_ref, nonfinite_prod, nonfinite_got, + ref_canary, prod_canary, got_canary, rc_reject, reject_writes, + ok ? "PASS" : "FAIL"); + if (first_ref_got_bad != SIZE_MAX) { + fprintf(stderr, + "first ref/16w mismatch at output[%zu]: " + "ref=0x%08x got=0x%08x\n", + first_ref_got_bad, first_ref_bits, first_got_bits); + } + if (first_ref_prod_bad != SIZE_MAX) { + uint32_t ref_bits = 0; + const uint8_t *const ptr = ref_guarded.data() + guard_bytes + + first_ref_prod_bad * sizeof(float); + std::memcpy(&ref_bits, ptr, sizeof(ref_bits)); + fprintf(stderr, + "first ref/stream mismatch at output[%zu]: " + "ref=0x%08x stream=0x%08x\n", + first_ref_prod_bad, ref_bits, first_prod_bits); + } + fputc('\n', stderr); + cleanup(); + return ok; +} + // Prefill dense-pair verifier. The candidate shares only the canonical // token-tiled Q8_1 activation; both weight legs still run the ordinary Q4_K // MMQ kernel, so their outputs must match two independent dense calls bitwise. @@ -2442,12 +2725,80 @@ bool run_q4_K_grouped_vec_parity( } // namespace int main(int argc, char ** argv) { - (void)argc; (void)argv; int rc = ds4_mmq_init(0); if (rc != 0) { fprintf(stderr, "ds4_mmq_init failed: %d\n", rc); return 1; } bool all_ok = true; + if (argc == 2 && std::strcmp(argv[1], "--q4-16warp") == 0) { + // The production arm must select mmq_x=128 for the no-fixup proof. + // Mirror get_mmq_x_max_host's numeric-prefix parsing and reject a + // narrower experiment override before MMQ caches it. + const char *const mmq_x_env = std::getenv("DS4_CUDA_MMQ_X_MAX"); + if (mmq_x_env && mmq_x_env[0]) { + char *end = nullptr; + const long value = std::strtol(mmq_x_env, &end, 10); + if (end != mmq_x_env && value >= 8 && + (value > INT32_MAX || (value / 8) * 8 < 128)) { + fprintf(stderr, + "Q4 16-warp oracle requires DS4_CUDA_MMQ_X_MAX>=128 " + "(got %s)\n", + mmq_x_env); + return 1; + } + } + + int device = -1; + cudaDeviceProp prop = {}; + const cudaError_t device_err = cudaGetDevice(&device); + const cudaError_t prop_err = device_err == cudaSuccess + ? cudaGetDeviceProperties(&prop, device) : device_err; + if (prop_err != cudaSuccess || prop.multiProcessorCount <= 0) { + fprintf(stderr, "Q4 16-warp geometry query failed: %s\n", + cudaGetErrorString(prop_err)); + return 1; + } + + // N=512 and mmq_x=128 give four N tiles. Choose the smallest M-tile + // count >=16 for which 4*tiles_m is divisible by nSM. Stream-K then + // launches one block per tile (100% efficiency) and needs no fixup. + int gcd = prop.multiProcessorCount; + int remainder = 4; + while (remainder != 0) { + const int next = gcd % remainder; + gcd = remainder; + remainder = next; + } + const int tiles_m_step = prop.multiProcessorCount / gcd; + int64_t tiles_m = tiles_m_step; + while (tiles_m < 16) tiles_m += tiles_m_step; + if (tiles_m > INT32_MAX / 128) { + fprintf(stderr, "Q4 16-warp geometry overflow\n"); + return 1; + } + const int no_fixup_M = (int)(128 * tiles_m); + fprintf(stderr, + "Q4 16-warp no-fixup geometry: nsm=%d tiles_m=%lld " + "tiles=%lld M=%d\n", + prop.multiProcessorCount, (long long)tiles_m, + (long long)(4 * tiles_m), no_fixup_M); + + // Exact N tile with a device-dependent, provably no-fixup M, plus an + // N-tail case at the top of the K envelope. Keep the larger K=4096 + // case at two arms so the focused oracle remains reasonably small. + all_ok &= run_q4_K_dense_16warp_parity( + /*M=*/no_fixup_M, /*N=*/512, /*K=*/1024, 0xC4160001u, + /*check_production=*/true, + /*check_rejection=*/true); + all_ok &= run_q4_K_dense_16warp_parity( + /*M=*/2176, /*N=*/513, /*K=*/4096, 0xC4160002u, + /*check_production=*/false, + /*check_rejection=*/false); + fprintf(stderr, "===================\n"); + fprintf(stderr, "Q4 16-WARP %s\n", all_ok ? "PASS" : "FAILED"); + return all_ok ? 0 : 1; + } + // Q8_0 all_ok &= run_q8_0(/*M=*/64, /*N=*/4, /*K=*/256, 0xC0FFEE); all_ok &= run_q8_0(/*M=*/128, /*N=*/8, /*K=*/512, 0xDEADBEE); diff --git a/ds4_cuda.cu b/ds4_cuda.cu index bb77a288a..499acf234 100644 --- a/ds4_cuda.cu +++ b/ds4_cuda.cu @@ -2659,6 +2659,23 @@ extern "C" int ds4_cuda_test_model_range_is_device_resident( resolved, g_gpu[logical_tier].device_id); } +extern "C" int ds4_cuda_test_model_range_device_ptr( + const void *model_map, + uint64_t model_size, + uint64_t offset, + uint64_t bytes, + int logical_tier, + const void **device_ptr) { + if (device_ptr) *device_ptr = NULL; + if (!device_ptr || + !ds4_cuda_test_model_range_is_device_resident( + model_map, model_size, offset, bytes, logical_tier)) { + return 0; + } + *device_ptr = g_model_device_base + offset; + return 1; +} + /* Match the existing CUDA weight-cache safety floor without coupling this * cache to the Q8-specific reserve environment variable. The explicit * future-session reserve supplied by ds4.c is added independently. */ @@ -40195,6 +40212,12 @@ static int cuda_matmul_q4_K_tensor( model_map, weight_offset, weight_bytes, logical_tier, "q4_K dense", &weight_device_resident); if (!wptr) return 0; + // The 16-warp experiment is a prefill specialization. Keep decode and + // speculative micro-batches on MMVQ, but make a strict prefill request + // fail closed even when the global MMQ selector was disabled before this + // call (otherwise the legacy Q8_K fallback could be measured silently). + const int require_q4_16warp = n_tok > 8u && cuda_env_flag_enabled( + "DS4_CUDA_REQUIRE_Q4_MMQ_16WARP", 0); /* The scalar-token kernel below rereads every weight row for every token, * making prefill scale almost linearly with batch length. MMQ tiles both * axes and shares the Q4_K weights across activation columns, matching the @@ -40224,6 +40247,7 @@ static int cuda_matmul_q4_K_tensor( g_cuda_test_q4_mmq_strict ? "; strict benchmark mode rejects fallback" : "; falling back"); + if (require_q4_16warp) return 0; if (g_cuda_test_q4_mmq_strict) return 0; if (in_dim == 1024u && out_dim == 32768u && n_tok == 1u && getenv("DS4_CUDA_REQUIRE_Q4_K1024_PERSISTENT") != NULL) { @@ -40239,6 +40263,15 @@ static int cuda_matmul_q4_K_tensor( (unsigned long long)n_tok); return 0; } + if (require_q4_16warp) { + fprintf(stderr, + "ds4: required Q4 16-warp prefill found no MMQ dispatch " + "(in=%llu out=%llu n_tok=%llu)\n", + (unsigned long long)in_dim, + (unsigned long long)out_dim, + (unsigned long long)n_tok); + return 0; + } void *tmp = cuda_tmp_alloc_on(logical_tier, n_tok * blocks * sizeof(cuda_block_q8_K), "q4_K dense prequant"); diff --git a/ds4_gpu.h b/ds4_gpu.h index cc449770e..9b986720d 100644 --- a/ds4_gpu.h +++ b/ds4_gpu.h @@ -146,6 +146,13 @@ int ds4_cuda_test_model_range_is_device_resident( uint64_t offset, uint64_t bytes, int logical_tier); +int ds4_cuda_test_model_range_device_ptr( + const void *model_map, + uint64_t model_size, + uint64_t offset, + uint64_t bytes, + int logical_tier, + const void **device_ptr); void ds4_cuda_test_set_q4_mmq_strict(int required); #endif /* Prepare a second, fully resident support GGUF without replacing the active diff --git a/scripts/environment_variables.tsv b/scripts/environment_variables.tsv index d8d13395f..74c5560df 100644 --- a/scripts/environment_variables.tsv +++ b/scripts/environment_variables.tsv @@ -237,6 +237,7 @@ runtime/cuda DS4_CUDA_NO_Q4_GROUPED_ATTN_A presence kill switch; default unset ( runtime/cuda DS4_CUDA_NO_Q4_GROUPED_ATTN_A_BATCH presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q4 grouped attn a batch CUDA Q4 optimization. cuda/mmq/ds4_mmq.cu:4305 runtime/cuda DS4_CUDA_NO_Q4_GROUPED_ATTN_A_PREFILL presence kill switch; default unset; any defined value including empty or 0 disables and dominates ENABLE/REQUIRE Restore the eight pack/MMQ/unpack Q4 attention-A prefill projections. ds4_cuda.cu:41930 runtime/cuda DS4_CUDA_NO_Q4_K1024_PERSISTENT presence kill switch, default off; any defined value including 0 disables Disable the Q4 K1024 persistent CUDA Q4 optimization. cuda/mmq/ds4_mmq.cu:3907 +runtime/cuda DS4_CUDA_NO_Q4_MMQ_16WARP value-aware rollback, default off; unset/empty/exact 0 permits the experiment, every other nonempty value disables it and overrides REQUEST/REQUIRE Disable the experimental complete-K CUDA Q4_K m128n128 16-warp prefill kernel. cuda/mmq/ds4_mmq.cu:1059 runtime/cuda DS4_CUDA_NO_Q8_ALIGNED_DENSE_SCRATCH presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q8 aligned dense scratch CUDA Q8 optimization. cuda/mmq/ds4_mmq.cu:5578 runtime/cuda DS4_CUDA_NO_Q8_ALIGNED_PERSISTENT presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q8 aligned persistent CUDA Q8 optimization. cuda/mmq/ds4_mmq.cu:5410 runtime/cuda DS4_CUDA_NO_Q8_BATCH_EXACT_TOK2 presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q8 batch exact tok2 CUDA Q8 optimization. ds4_cuda.cu:19852 @@ -290,6 +291,7 @@ runtime/cuda DS4_CUDA_Q4_ATTN_Q_B_TRANSIENT_F16_MIN_TOKENS full-string unsigned runtime/cuda DS4_CUDA_Q4_GROUPED_ATTN_A_ORACLE value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on Compare grouped attention-A against the canonical per-group result. ds4_cuda.cu:1477 runtime/cuda DS4_CUDA_Q4_K1024_PERSISTENT_ORACLE value-aware flag, default off; nonempty value other than exact 0 enables and implies candidate admission Bitwise-compare the exact-shape persistent Q4 K1024 kernel with canonical MMVQ and retain canonical output. cuda/mmq/ds4_mmq.cu:3738 runtime/cuda DS4_CUDA_Q4_K1024_PERSISTENT_STATS value-aware flag, default off; nonempty value other than exact 0 enables Print exact-shape persistent Q4 K1024 dispatch counters at exit. cuda/mmq/ds4_mmq.cu:3737 +runtime/cuda DS4_CUDA_Q4_MMQ_16WARP value-aware opt-in cached on first Q4_K dense MMQ call; unset/empty/exact 0 is off, every other nonempty value requests the candidate; rollback wins and ineligible shapes fall back Enable the experimental exact-integer CUDA Q4_K m128n128 16-warp kernel for eligible complete-K dense prefills. cuda/mmq/ds4_mmq.cu:1052 runtime/cuda DS4_CUDA_Q8_F16_ALL presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Control the Q8 F16 all CUDA quantized-matmul/cache optimization. ds4_cuda.cu:2358 runtime/cuda DS4_CUDA_Q8_F16_CACHE_MB unsigned integer MiB, full-string parse; default unlimited; 0 disables this cache Limit the selective Q8-to-F16 derived-weight cache. ds4_cuda.cu:2218 runtime/cuda DS4_CUDA_Q8_F16_CACHE_RESERVE_MB unsigned integer MiB, full-string parse; default is VRAM-dependent (>=112 GiB: 512; >=40 GiB: max(768,1%); smaller: max(4096,5%)) Reserve free VRAM when growing the selective Q8-to-F16 cache. ds4_cuda.cu:2224 @@ -309,6 +311,7 @@ runtime/cuda DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_BATCH value-aware flag, default runtime/cuda DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_PREFILL value-aware fail-closed assertion, default off; unset/empty/exact 0 is off, any other nonempty value requests the candidate and rejects ineligibility before enqueue Require the GB10 grouped Q4_K attention-A prefill path instead of silently using pack/MMQ/unpack. ds4_cuda.cu:41889 runtime/cuda DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_SINGLE_GRID value-aware fail-closed assertion, default off; unset/empty/exact 0 is off, every other nonempty value requests the candidate; DISABLE or ineligibility fails before enqueue Require one grid.z MMQ submission for eligible GB10 grouped Q4_K attention-A prefill instead of falling back to one launch per group. ds4_cuda.cu:41896 runtime/cuda DS4_CUDA_REQUIRE_Q4_K1024_PERSISTENT presence flag, default off; any defined value including 0 makes ineligible candidate fail closed Fail when the exact Q4 K1024 persistent candidate is unavailable instead of using MMVQ. cuda/mmq/ds4_mmq.cu:3929 +runtime/cuda DS4_CUDA_REQUIRE_Q4_MMQ_16WARP value-aware fail-closed prefill opt-in cached on first Q4_K dense MMQ call; unset/empty/exact 0 is off, every other nonempty value requests and requires the candidate for N>8; rollback, disabled MMQ, ineligibility, or preflight failure prevents fallback; decode/speculative N<=8 remains on MMVQ Require the experimental CUDA Q4_K 16-warp prefill kernel so benchmark runs cannot silently measure another path. cuda/mmq/ds4_mmq.cu:1056; ds4_cuda.cu:38208 runtime/cuda DS4_CUDA_REQUIRE_STREAMING_EXPERT_PERSISTENT_CACHE false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Require streaming expert persistent cache in CUDA SSD streaming; fail closed when unavailable. ds4_cuda.cu:4117 runtime/cuda DS4_CUDA_REQUIRE_STREAMING_SELECTED_BATCH_IO false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Require streaming selected batch I/O in CUDA SSD streaming; fail closed when unavailable. ds4_cuda.cu:4738 runtime/cuda DS4_CUDA_REQUIRE_STREAMING_SELECTED_EVENT_PIPELINE false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Require streaming selected event pipeline in CUDA SSD streaming; fail closed when unavailable. ds4_cuda.cu:4870 diff --git a/speed-bench/README.md b/speed-bench/README.md index 8b63662eb..eb076aa43 100644 --- a/speed-bench/README.md +++ b/speed-bench/README.md @@ -270,6 +270,38 @@ API, with bit-exact pair outputs. Prefill pair is skipped under `--path legacy` because the CUDA pair API intentionally returns control to two independent dense projections for `N>8` when MMQ is disabled. +To isolate the experimental Q4_K m128n128 16-warp GEMM from activation +quantization and every storage effect, run the focused bitwise/canary oracle +and then the resident prequantized A/B benchmark: + +``` +make test-mmq-q4-16warp-cuda CUDA_ARCH=sm_121 +make cuda-q4-prefill-bench CUDA_ARCH=sm_121 +./speed-bench/cuda_q4_prefill_bench \ + --path mmq --kernel-16warp --case qb \ + --tokens 512,1024,2048,2049,4096 --sets 4 --samples 16 --warmup 4 +``` + +The benchmark quantizes X to canonical Q8_1 DS4 once, before timing, and uses +CUDA events around only the complete-K reference GEMM or the raw 16-warp GEMM. +It alternates the two arms ABBA/BAAB over resident Q4_K weight sets, compares +their complete outputs bit-for-bit, checks finite values and canaries, and +samples a CPU Q4_K oracle before and after timing. Results attest +`timing=kernel_only_prequant`; SSD streaming, model upload, Q8_1 quantization, +allocation, host copies, and oracle work are outside the samples. Use +`--case dense` as an exploratory `K=4096,M=1024` datapoint; the initial +production admission gate is deliberately limited to larger, complete +128x128 projections such as q_b. + +The production path remains opt-in until the NVIDIA oracle passes and the +paired median is a repeatable win. `DS4_CUDA_Q4_MMQ_16WARP=1` requests it and +falls back on ineligible shapes. For an attested production benchmark, +`DS4_CUDA_REQUIRE_Q4_MMQ_16WARP=1` also prevents a Q8_K/MMQ fallback; +`DS4_CUDA_NO_Q4_MMQ_16WARP=1` is the value-aware rollback. The strict path +requires Ampere or newer, N and M divisible by 128, `M>=2048`, `N>=512`, +`1024<=K<=4096`, the default 128-column MMQ selector, and a canonical tiling +that owns complete K without stream-K fixup. + On GB10, isolate the production Flash attention-output A geometry (`groups=8`, `K=4096`, `rank=1024`) and its 127/128/129 token tails with: diff --git a/speed-bench/cuda_q4_prefill_bench.cu b/speed-bench/cuda_q4_prefill_bench.cu index ed1e09d3d..109173c08 100644 --- a/speed-bench/cuda_q4_prefill_bench.cu +++ b/speed-bench/cuda_q4_prefill_bench.cu @@ -2,6 +2,8 @@ // Resident, CUDA-event-only Q4_K prefill microbenchmark. #include "ds4_gpu.h" +#include "cuda/mmq/ds4_mmq.h" +#include "cuda/mmq/ds4_mmq_q4_16warp.cuh" #include @@ -84,6 +86,7 @@ struct config { uint32_t sets = kDefaultSets; uint32_t samples = kDefaultSamples; uint32_t warmup = kDefaultWarmup; + bool kernel_16warp = false; }; struct weight_set { @@ -115,6 +118,21 @@ struct tensor_owner { tensor_owner &operator=(const tensor_owner &) = delete; }; +struct cuda_buffer { + void *ptr = nullptr; + cudaError_t status = cudaSuccess; + + explicit cuda_buffer(size_t bytes) { + if (bytes == 0u) return; + status = cudaMalloc(&ptr, bytes); + } + ~cuda_buffer() { + if (ptr) (void)cudaFree(ptr); + } + cuda_buffer(const cuda_buffer &) = delete; + cuda_buffer &operator=(const cuda_buffer &) = delete; +}; + struct env_snapshot { const char *name; bool existed; @@ -152,10 +170,9 @@ struct event_timer { } bool measure(const std::function &dispatch, float *milliseconds) { - // The harness disables decode graphs. Production Q4 eager dispatch - // passes cuda_decode_stream()==0 to both MMQ (including its async - // scratch pool) and the Q8_K fallback kernels, so bracketing stream 0 - // covers every enqueue in the public call. + // Every harness arm uses stream 0. Production paths enqueue their + // complete backend work there; kernel-only arms enqueue exactly one + // already-prepared GEMM there. if (cudaEventRecord(begin, nullptr) != cudaSuccess) return false; if (!dispatch()) return false; if (cudaEventRecord(end, nullptr) != cudaSuccess || @@ -174,6 +191,10 @@ struct arm { // an idle device can execute that event while setenv()/unsetenv() is still // running, folding host-side gate switching into the reported GPU time. std::function select; + // Optional checked boundary for oracle passes. Timed samples continue to + // use dispatch so an enqueue-only experimental arm can keep its safety + // preflight outside the CUDA-event interval. + std::function oracle_dispatch = {}; }; struct stats { @@ -628,29 +649,51 @@ const char *path_name(cuda_path path) { return path == cuda_path::mmq ? "mmq" : "legacy"; } +const char *case_scope(const config &cfg) { + switch (cfg.selected) { + case bench_case::all: + return cfg.kernel_16warp ? "dense,q_b" : "dense,pair,q_b,outa"; + case bench_case::dense: return "dense"; + case bench_case::pair: return "pair"; + case bench_case::qb: return "q_b"; + case bench_case::outa: return "outa"; + } + return "unknown"; +} + bool select_arm(const arm &which) { return !which.select || which.select(); } +bool dispatch_oracle_arm(const arm &which, uint32_t set) { + return which.oracle_dispatch ? which.oracle_dispatch(set) + : which.dispatch(set); +} + bool benchmark_single_path( const char *case_name, uint32_t n_tokens, uint32_t in_dim, uint32_t out_dim, const config &cfg, const arm &which, const std::function &oracle_prepare, const std::function &oracle) { + auto validate = [&](const char *phase) { + for (uint32_t set = 0; set < cfg.sets; set++) { + if (!oracle_prepare() || !select_arm(which) || + !dispatch_oracle_arm(which, set) || !ds4_gpu_synchronize() || + !oracle(set)) { + std::fprintf(stderr, + "cuda-q4-prefill-bench: %s %s oracle failed " + "for weight set %u\n", + case_name, phase, set); + return false; + } + } + return true; + }; + // Validate every rotating weight set and prime lazy Q8 scratch before // recording a CUDA event. Upload, poison, readback, and CPU work remain // outside the measured interval. - for (uint32_t set = 0; set < cfg.sets; set++) { - if (!oracle_prepare() || !select_arm(which) || - !which.dispatch(set) || - !ds4_gpu_synchronize() || !oracle(set)) { - std::fprintf(stderr, - "cuda-q4-prefill-bench: %s oracle failed for " - "weight set %u\n", - case_name, set); - return false; - } - } + if (!validate("pre-timing")) return false; for (uint32_t i = 0; i < cfg.warmup; i++) { if (!select_arm(which) || !which.dispatch(i % cfg.sets) || @@ -675,6 +718,20 @@ bool benchmark_single_path( samples.push_back(static_cast(elapsed)); } + const uint32_t timed_set = (cfg.samples - 1u) % cfg.sets; + if (!oracle(timed_set)) { + std::fprintf(stderr, + "cuda-q4-prefill-bench: %s timed-output oracle failed " + "for weight set %u\n", + case_name, timed_set); + return false; + } + + // Timed launches overwrite the validated buffers. Re-run the complete + // finite/CPU/guard oracle for every resident weight set after timing so a + // late or state-dependent corruption cannot survive as a benchmark result. + if (!validate("post-timing")) return false; + const stats result = summarize(samples); const double macs = static_cast(n_tokens) * in_dim * out_dim; const double gmac_s = macs / (result.median * 1.0e6); @@ -695,19 +752,26 @@ bool benchmark_pair_arms( uint64_t common_macs_per_token, const config &cfg, const arm &baseline, const arm &candidate, const std::function &oracle_prepare, - const std::function &oracle) { - for (uint32_t set = 0; set < cfg.sets; set++) { - if (!oracle_prepare() || !select_arm(baseline) || - !baseline.dispatch(set) || !ds4_gpu_synchronize() || - !select_arm(candidate) || !candidate.dispatch(set) || - !ds4_gpu_synchronize() || !oracle(set)) { - std::fprintf(stderr, - "cuda-q4-prefill-bench: %s oracle failed for " - "weight set %u\n", - case_name, set); - return false; + const std::function &oracle, + const char *timing_scope = "production_api_cuda_events") { + auto validate = [&](const char *phase) { + for (uint32_t set = 0; set < cfg.sets; set++) { + if (!oracle_prepare() || !select_arm(baseline) || + !dispatch_oracle_arm(baseline, set) || + !ds4_gpu_synchronize() || !select_arm(candidate) || + !dispatch_oracle_arm(candidate, set) || + !ds4_gpu_synchronize() || !oracle(set)) { + std::fprintf(stderr, + "cuda-q4-prefill-bench: %s %s oracle failed " + "for weight set %u\n", + case_name, phase, set); + return false; + } } - } + return true; + }; + + if (!validate("pre-timing")) return false; for (uint32_t i = 0; i < cfg.warmup; i++) { const uint32_t set = i % cfg.sets; @@ -756,6 +820,22 @@ bool benchmark_pair_arms( } } + // samples is a multiple of four, so the final ABBA/BAAB cycle leaves both + // output buffers holding weight set samples-1. Check those exact timed + // results before oracle_prepare is allowed to poison or reset any guard. + const uint32_t timed_set = (cfg.samples - 1u) % cfg.sets; + if (!oracle(timed_set)) { + std::fprintf(stderr, + "cuda-q4-prefill-bench: %s timed-output oracle failed " + "for weight set %u\n", + case_name, timed_set); + return false; + } + + // Both arms must still agree bit-for-bit (and with their CPU/guard + // oracle) after the measured ABBA/BAAB sequence, for every weight set. + if (!validate("post-timing")) return false; + const stats a = summarize(a_samples); const stats b = summarize(b_samples); std::vector paired_delta; @@ -771,6 +851,7 @@ bool benchmark_pair_arms( std::printf( "DS4_CUDA_Q4_PREFILL_BENCH case=%s path=%s N=%u K=%u M=%u " "baseline=%s candidate=%s samples=%u sets=%u " + "timing=%s " "focus_macs_per_token=%llu common_macs_per_token=%llu " "baseline_ms_p50=%.6f candidate_ms_p50=%.6f " "baseline_ms_min=%.6f candidate_ms_min=%.6f " @@ -780,6 +861,7 @@ bool benchmark_pair_arms( "speedup_pct=%.3f\n", case_name, path_name(cfg.path), n_tokens, in_dim, out_dim, baseline.name, candidate.name, cfg.samples, cfg.sets, + timing_scope, static_cast(focus_macs_per_token), static_cast(common_macs_per_token), a.median, b.median, a.minimum, b.minimum, a.p95, b.p95, @@ -789,6 +871,161 @@ bool benchmark_pair_arms( return true; } +bool run_q4_16warp_kernel( + const model_fixture &model, const config &cfg, uint32_t n_tokens, + uint32_t in_dim, uint32_t out_dim, + uint64_t weight_set::*weight_offset_member, const char *case_name) { + uint64_t x_elements = 0; + uint64_t out_elements = 0; + if (!checked_mul(n_tokens, in_dim, &x_elements) || + !checked_mul(n_tokens, out_dim, &out_elements)) { + return false; + } + const uint64_t x_bytes = x_elements * sizeof(float); + const uint64_t out_bytes = out_elements * sizeof(float); + const uint64_t guard_bytes = kGuardWords * sizeof(uint32_t); + const uint64_t weight_bytes = q4_weight_bytes(in_dim, out_dim); + const size_t q8_bytes = ds4_mmq_q4_K_q8_1_scratch_bytes( + static_cast(n_tokens), static_cast(in_dim)); + if (q8_bytes == 0u) { + std::fprintf(stderr, + "cuda-q4-prefill-bench: %s kernel-only Q8_1 scratch " + "shape rejected for N=%u K=%u\n", + case_name, n_tokens, in_dim); + return false; + } + + tensor_owner x(x_bytes + guard_bytes); + tensor_owner reference(out_bytes + guard_bytes); + tensor_owner candidate(out_bytes + guard_bytes); + cuda_buffer prequant(q8_bytes); + std::vector activation; + fill_activation(&activation, n_tokens, in_dim); + if (!x.ptr || !reference.ptr || !candidate.ptr || !prequant.ptr || + prequant.status != cudaSuccess || + !ds4_gpu_tensor_write(x.ptr, 0, activation.data(), x_bytes) || + !prepare_guard(x.ptr, x_bytes, 0x12000u)) { + std::fprintf(stderr, + "cuda-q4-prefill-bench: %s N=%u kernel-only tensor/" + "scratch setup failed (%s)\n", + case_name, n_tokens, + prequant.status == cudaSuccess + ? "tensor setup" + : cudaGetErrorString(prequant.status)); + return false; + } + + const auto *x_device = static_cast( + ds4_gpu_tensor_contents(x.ptr)); + auto *reference_device = static_cast( + ds4_gpu_tensor_contents(reference.ptr)); + auto *candidate_device = static_cast( + ds4_gpu_tensor_contents(candidate.ptr)); + if (!x_device || !reference_device || !candidate_device) { + std::fprintf(stderr, + "cuda-q4-prefill-bench: %s N=%u device tensor pointer " + "lookup failed\n", + case_name, n_tokens); + return false; + } + + std::vector weight_device(cfg.sets, nullptr); + for (uint32_t set = 0; set < cfg.sets; set++) { + const uint64_t offset = model.weights[set].*weight_offset_member; + if (!ds4_cuda_test_model_range_device_ptr( + model.data, model.size, offset, weight_bytes, + /*logical_tier=*/0, &weight_device[set]) || + !weight_device[set]) { + std::fprintf(stderr, + "cuda-q4-prefill-bench: %s N=%u resident weight " + "pointer lookup failed for set %u\n", + case_name, n_tokens, set); + return false; + } + } + + // Quantize exactly once. Both A/B arms consume this immutable canonical + // DS4 Q8_1 buffer, and no quantizer, memset, allocation, or copy is issued + // by either timed dispatch. + const int quant_rc = ds4_mmq_q4_K_quantize_q8_1_for_test( + x_device, prequant.ptr, q8_bytes, static_cast(n_tokens), + static_cast(in_dim), /*stream=*/nullptr); + if (quant_rc != 0 || !ds4_gpu_synchronize() || + !check_guard(x.ptr, x_bytes, 0x12000u, + "kernel-only prequant input")) { + std::fprintf(stderr, + "cuda-q4-prefill-bench: %s N=%u prequantization " + "failed rc=%d\n", + case_name, n_tokens, quant_rc); + return false; + } + + const arm baseline = { + "canonical_preq_complete_k", + [&](uint32_t set) { + return ds4_mmq_q4_K_dense_preq_reference_for_test( + weight_device[set], prequant.ptr, q8_bytes, + reference_device, static_cast(out_dim), + static_cast(n_tokens), static_cast(in_dim), + /*use_stream_k=*/0, /*stream=*/nullptr) == 0; + }, + {}}; + const arm candidate_arm = { + "q4_16warp_m128n128", + [&](uint32_t set) { + return ds4_mmq_q4_K_dense_16warp_enqueue( + weight_device[set], prequant.ptr, candidate_device, + static_cast(out_dim), + static_cast(n_tokens), static_cast(in_dim), + /*stream=*/nullptr) == 0; + }, + {}, + [&](uint32_t set) { + return ds4_mmq_q4_K_dense_preq_16warp_for_test( + weight_device[set], prequant.ptr, q8_bytes, + candidate_device, static_cast(out_dim), + static_cast(n_tokens), static_cast(in_dim), + /*stream=*/nullptr) == 0; + }}; + const bool ok = benchmark_pair_arms( + case_name, n_tokens, in_dim, out_dim, + static_cast(in_dim) * out_dim, 0u, cfg, + baseline, candidate_arm, + [&]() { + return poison_output(reference.ptr, out_bytes, 0x7fc50005u, + 0x13000u) && + poison_output(candidate.ptr, out_bytes, 0x7fc60006u, + 0x14000u); + }, + [&](uint32_t set) { + const uint64_t offset = + model.weights[set].*weight_offset_member; + return bitwise_equal(reference.ptr, candidate.ptr, out_bytes, + "16-warp vs canonical complete-K") && + output_is_finite(reference.ptr, out_bytes, + "16-warp canonical output") && + output_is_finite(candidate.ptr, out_bytes, + "16-warp candidate output") && + sampled_cpu_oracle( + reference.ptr, model, offset, activation, n_tokens, + in_dim, out_dim, "16-warp canonical") && + check_guard(x.ptr, x_bytes, 0x12000u, + "16-warp input") && + check_guard(reference.ptr, out_bytes, 0x13000u, + "16-warp canonical output") && + check_guard(candidate.ptr, out_bytes, 0x14000u, + "16-warp candidate output"); + }, + "kernel_only_prequant"); + return ok && + check_guard(x.ptr, x_bytes, 0x12000u, + "16-warp input final") && + check_guard(reference.ptr, out_bytes, 0x13000u, + "16-warp canonical output final") && + check_guard(candidate.ptr, out_bytes, 0x14000u, + "16-warp candidate output final"); +} + bool run_dense(const model_fixture &model, const config &cfg, uint32_t n_tokens) { uint64_t out_elements = 0; @@ -1139,11 +1376,12 @@ void usage(FILE *stream, const char *argv0) { " --case all|dense|pair|qb|outa\n" " case to run (default: all)\n" " --tokens N[,N...] token counts, each 9..4096\n" - " --full use 9,16,17,31,32,33,127,128,129,257," - "512,2048,4096\n" + " --full use 9,16,17,31,32,33,127,128,129,256," + "257,512,1024,2048,2049,4096\n" " --sets N rotating resident weight sets (default: %u)\n" " --samples N samples/arm, multiple of 4 (default: %u)\n" " --warmup N untimed dispatches/arm (default: %u)\n" + " --kernel-16warp prequantized canonical-vs-16-warp A/B\n" " -h, --help show this help\n\n" "Dense and q_b measure one immutable process path. Run separate " "legacy/MMQ\nprocesses (preferably ABBA/BAAB) to compare them because " @@ -1153,7 +1391,13 @@ void usage(FILE *stream, const char *argv0) { "with one MMQ grid per group against the strict\nsingle-grid candidate " "at the production attention-A shape, isolating grid.z submission. It " "includes\na common minimal " - "Q4 output-B (M=256) whose MACs are reported separately.\n", + "Q4 output-B (M=256) whose MACs are reported separately. " + "DS4_CUDA_MMQ_X_MAX\nmay explicitly select an 8..128 multiple-of-8 " + "sweep point; the setup line\nattests it, or prints auto when the " + "variable is unset. --kernel-16warp requires\n--path mmq and " + "--case dense/qb; with --case all it runs only dense and q_b. Its " + "token\ncounts must be >=512; without --tokens/--full it uses " + "512,1024,2048,2049,4096.\n", argv0, kDefaultSets, kDefaultSamples, kDefaultWarmup); } @@ -1205,6 +1449,7 @@ std::vector parse_tokens(const char *text) { config parse_options(int argc, char **argv) { config cfg; + bool tokens_explicit = false; for (int i = 1; i < argc; i++) { if (!std::strcmp(argv[i], "-h") || !std::strcmp(argv[i], "--help")) { usage(stdout, argv[0]); @@ -1235,9 +1480,12 @@ config parse_options(int argc, char **argv) { } } else if (!std::strcmp(argv[i], "--tokens")) { cfg.tokens = parse_tokens(need_value(&i, argc, argv)); + tokens_explicit = true; } else if (!std::strcmp(argv[i], "--full")) { cfg.tokens = {9u, 16u, 17u, 31u, 32u, 33u, 127u, - 128u, 129u, 257u, 512u, 2048u, 4096u}; + 128u, 129u, 256u, 257u, 512u, 1024u, 2048u, + 2049u, 4096u}; + tokens_explicit = true; } else if (!std::strcmp(argv[i], "--sets")) { cfg.sets = parse_u32(need_value(&i, argc, argv), "--sets", 1u, 32u); @@ -1247,6 +1495,8 @@ config parse_options(int argc, char **argv) { } else if (!std::strcmp(argv[i], "--warmup")) { cfg.warmup = parse_u32(need_value(&i, argc, argv), "--warmup", 0u, 100u); + } else if (!std::strcmp(argv[i], "--kernel-16warp")) { + cfg.kernel_16warp = true; } else { std::fprintf(stderr, "unknown option: %s\n", argv[i]); usage(stderr, argv[0]); @@ -1258,6 +1508,34 @@ config parse_options(int argc, char **argv) { "--samples must be a multiple of 4 for balanced runs\n"); std::exit(2); } + if (cfg.kernel_16warp && !tokens_explicit) { + cfg.tokens = {512u, 1024u, 2048u, 2049u, 4096u}; + } + if (cfg.kernel_16warp) { + const auto below_minimum = std::find_if( + cfg.tokens.begin(), cfg.tokens.end(), + [](uint32_t value) { return value < 512u; }); + if (below_minimum != cfg.tokens.end()) { + std::fprintf(stderr, + "--kernel-16warp requires every token count to be " + ">=512 (got %u)\n", + *below_minimum); + std::exit(2); + } + } + if (cfg.kernel_16warp && cfg.path != cuda_path::mmq) { + std::fprintf(stderr, + "--kernel-16warp requires --path mmq\n"); + std::exit(2); + } + if (cfg.kernel_16warp && + (cfg.selected == bench_case::pair || + cfg.selected == bench_case::outa)) { + std::fprintf(stderr, + "--kernel-16warp supports only --case dense, qb, or " + "all (all runs dense+qb only)\n"); + std::exit(2); + } if (cfg.path == cuda_path::legacy && (cfg.selected == bench_case::pair || cfg.selected == bench_case::outa)) { @@ -1273,6 +1551,21 @@ bool includes(bench_case selected, bench_case wanted) { return selected == bench_case::all || selected == wanted; } +std::string mmq_x_max_attestation() { + const char *value = std::getenv("DS4_CUDA_MMQ_X_MAX"); + if (!value || !value[0]) return "auto"; + const uint32_t parsed = + parse_u32(value, "DS4_CUDA_MMQ_X_MAX", 8u, 128u); + if ((parsed % 8u) != 0u) { + std::fprintf(stderr, + "invalid DS4_CUDA_MMQ_X_MAX: %s (must be a multiple " + "of 8)\n", + value); + std::exit(2); + } + return std::to_string(parsed); +} + bool install_resident_model(const model_fixture &model, size_t *resident_delta, bool *resident_delta_valid) { @@ -1365,6 +1658,11 @@ bool verify_mmq_prefill_dispatch(const model_fixture &model) { int main(int argc, char **argv) { const config cfg = parse_options(argc, argv); + // get_mmq_x_max_host() caches this process-wide on its first call. Read + // and validate the inherited sweep request before backend initialization, + // then attest it in the setup record instead of silently contaminating a + // supposedly default run. + const std::string mmq_x_max = mmq_x_max_attestation(); env_snapshot mmq_guard("DS4_CUDA_MMQ"); env_snapshot copy_guard("DS4_CUDA_COPY_MODEL"); env_snapshot pair_guard("DS4_CUDA_DISABLE_Q4_DENSE_PAIR"); @@ -1410,6 +1708,12 @@ int main(int argc, char **argv) { "cuda-q4-prefill-bench: cannot query device properties\n"); return 1; } + if (cfg.kernel_16warp && device_count != 1) { + std::fprintf(stderr, + "cuda-q4-prefill-bench: SKIP (--kernel-16warp " + "requires exactly one visible CUDA device)\n"); + return 77; + } const bool grouped_prefill_supported = device_count == 1 && properties.major == 12 && properties.minor == 1 && properties.warpSize == 32; @@ -1424,6 +1728,18 @@ int main(int argc, char **argv) { std::fprintf(stderr, "cuda-q4-prefill-bench: ds4_gpu_init failed\n"); return 1; } + if (cfg.kernel_16warp) { + const int prepare_rc = ds4_mmq_q4_K_dense_16warp_prepare(); + if (prepare_rc != 0) { + std::fprintf( + stderr, + "cuda-q4-prefill-bench: --kernel-16warp prepare failed " + "rc=%d\n", + prepare_rc); + ds4_gpu_cleanup(); + return 1; + } + } ds4_cuda_test_set_q4_mmq_strict( cfg.path == cuda_path::mmq ? 1 : 0); @@ -1464,30 +1780,45 @@ int main(int argc, char **argv) { if (ok) { std::printf( "DS4_CUDA_Q4_PREFILL_SETUP device=%s cc=%d.%d warp=%d path=%s " - "sets=%u resident_payload_mib=%.2f device_free_delta_mib=%.2f " - "device_free_delta_valid=%d timing=cuda_events " + "mmq_x_max=%s sets=%u resident_payload_mib=%.2f " + "device_free_delta_mib=%.2f " + "device_free_delta_valid=%d timing=%s kernel_16warp=%d " + "cases=%s " "ssd_streaming=off model_storage=cudaMalloc " "residency=backend_provenance strict_mmq=%d " "grouped_attn_a_prefill=%s dispatch_stream=legacy_default\n", properties.name, properties.major, properties.minor, - properties.warpSize, path_name(cfg.path), cfg.sets, + properties.warpSize, path_name(cfg.path), mmq_x_max.c_str(), + cfg.sets, static_cast(model.payload_bytes) / 1048576.0, static_cast(resident_delta) / 1048576.0, resident_delta_valid ? 1 : 0, + cfg.kernel_16warp ? "kernel_only_prequant" : "cuda_events", + cfg.kernel_16warp ? 1 : 0, + case_scope(cfg), cfg.path == cuda_path::mmq ? 1 : 0, grouped_prefill_supported ? "available" : "skipped"); std::fflush(stdout); for (uint32_t n_tokens : cfg.tokens) { if (includes(cfg.selected, bench_case::dense)) { - ok = run_dense(model, cfg, n_tokens) && ok; + ok = (cfg.kernel_16warp + ? run_q4_16warp_kernel( + model, cfg, n_tokens, kDenseK, kDenseM, + &weight_set::dense_offset, "dense") + : run_dense(model, cfg, n_tokens)) && ok; } - if (ok && includes(cfg.selected, bench_case::pair)) { + if (ok && !cfg.kernel_16warp && + includes(cfg.selected, bench_case::pair)) { ok = run_pair(model, cfg, n_tokens) && ok; } if (ok && includes(cfg.selected, bench_case::qb)) { - ok = run_qb(model, cfg, n_tokens) && ok; + ok = (cfg.kernel_16warp + ? run_q4_16warp_kernel( + model, cfg, n_tokens, kQbK, kQbM, + &weight_set::qb_offset, "q_b") + : run_qb(model, cfg, n_tokens)) && ok; } - if (ok && grouped_prefill_supported && + if (ok && !cfg.kernel_16warp && grouped_prefill_supported && includes(cfg.selected, bench_case::outa)) { ok = run_output_a(model, cfg, n_tokens) && ok; } From 3504a3f1b22e283e646f91bfde9851252318ab47 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Sun, 30 Aug 2026 12:13:05 +0200 Subject: [PATCH 171/189] Extend CUDA Q4 16-warp MMQ with Stream-K --- ENVIRONMENT_VARIABLES.md | 10 +- cuda/mmq/ds4_mmq.cu | 299 +++++++++--- cuda/mmq/ds4_mmq.h | 21 +- cuda/mmq/ds4_mmq_q4_16warp.cu | 348 +++++++++++++- cuda/mmq/ds4_mmq_q4_16warp.cuh | 44 +- cuda/mmq/mmq.cuh | 29 +- cuda/mmq/test/test_mmq_parity.cu | 661 +++++++++++++++++++++++++-- ds4_cuda.cu | 17 +- gguf-tools/Makefile | 4 +- scripts/environment_variables.tsv | 6 +- speed-bench/README.md | 46 +- speed-bench/cuda_q4_prefill_bench.cu | 444 ++++++++++++++++-- 12 files changed, 1724 insertions(+), 205 deletions(-) diff --git a/ENVIRONMENT_VARIABLES.md b/ENVIRONMENT_VARIABLES.md index 02717e8f8..f75891ce1 100644 --- a/ENVIRONMENT_VARIABLES.md +++ b/ENVIRONMENT_VARIABLES.md @@ -76,9 +76,9 @@ The detailed Metal A/B contracts and expected oracle counters live in | `DS4_CUDA_ENABLE_Q4_K1024_PERSISTENT=1` | Enable the experimental persistent-CTA kernel for the exact `32768x1024` Q4 shape. | | `DS4_CUDA_NO_Q4_K1024_PERSISTENT=1` | Roll back the persistent K1024 experiment. | | `DS4_CUDA_REQUIRE_Q4_K1024_PERSISTENT=1` | Require the K1024 candidate before enqueue instead of silently using canonical MMVQ. | -| `DS4_CUDA_Q4_MMQ_16WARP=1` | Request the experimental exact-integer m128n128 16-warp Q4_K kernel for eligible complete-K CUDA prefills; ineligible shapes fall back. | +| `DS4_CUDA_Q4_MMQ_16WARP=1` | Request the experimental exact-integer m128n128 16-warp Q4_K kernel for eligible CUDA prefills. Standalone dense admits `M>=1024`; dense-pair shares one Q8_1 activation and admits legs down to `M=512`. Candidate grids require at least 80% whole-tile SM-wave efficiency; below the canonical 90% cutoff the kernel mirrors canonical Stream-K partitioning and fixup so the FP32 reduction tree is unchanged. Ineligible optional shapes fall back. | | `DS4_CUDA_NO_Q4_MMQ_16WARP=1` | Roll back the 16-warp Q4_K prefill experiment. This value-aware switch overrides request and require. | -| `DS4_CUDA_REQUIRE_Q4_MMQ_16WARP=1` | Request the 16-warp Q4_K prefill kernel and fail closed instead of silently measuring another MMQ/Q8_K path. Decode/speculative batches of at most eight tokens remain on MMVQ. | +| `DS4_CUDA_REQUIRE_Q4_MMQ_16WARP=1` | Request the 16-warp Q4_K prefill kernel and fail closed instead of silently measuring another MMQ/Q8_K path; a dense-pair requires both legs to be eligible. Decode/speculative batches of at most eight tokens remain on MMVQ. | | `DS4_CUDA_ENABLE_Q8_FOLD=1` | Enable the experimental one-shot Q8_1 producer-to-consumer fold. | | `DS4_CUDA_NO_Q8_FOLD=1` | Dominant rollback for the Q8_1 fold. | | `DS4_CUDA_Q8_FOLD_ORACLE=1` | Compare fresh canonical Q8_1 bytes and consumer outputs. Use with `DS4_CUDA_DECODE_GRAPHS=0`; require nonzero calls and zero mismatches/skips. | @@ -809,7 +809,7 @@ and **19 tool/wrapper entries**. | `DS4_CUDA_NO_Q4_GROUPED_ATTN_A_BATCH` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q4 grouped attn a batch CUDA Q4 optimization. | [cuda/mmq/ds4_mmq.cu:4305](cuda/mmq/ds4_mmq.cu#L4305) | | `DS4_CUDA_NO_Q4_GROUPED_ATTN_A_PREFILL` | presence kill switch; default unset; any defined value including empty or 0 disables and dominates ENABLE/REQUIRE | Restore the eight pack/MMQ/unpack Q4 attention-A prefill projections. | [ds4_cuda.cu:41930](ds4_cuda.cu#L41930) | | `DS4_CUDA_NO_Q4_K1024_PERSISTENT` | presence kill switch, default off; any defined value including 0 disables | Disable the Q4 K1024 persistent CUDA Q4 optimization. | [cuda/mmq/ds4_mmq.cu:3907](cuda/mmq/ds4_mmq.cu#L3907) | -| `DS4_CUDA_NO_Q4_MMQ_16WARP` | value-aware rollback, default off; unset/empty/exact 0 permits the experiment, every other nonempty value disables it and overrides REQUEST/REQUIRE | Disable the experimental complete-K CUDA Q4_K m128n128 16-warp prefill kernel. | [cuda/mmq/ds4_mmq.cu:1059](cuda/mmq/ds4_mmq.cu#L1059) | +| `DS4_CUDA_NO_Q4_MMQ_16WARP` | value-aware rollback, default off; unset/empty/exact 0 permits the experiment, every other nonempty value disables it and overrides REQUEST/REQUIRE | Disable the experimental Stream-K-compatible CUDA Q4_K m128n128 16-warp prefill kernel. | [cuda/mmq/ds4_mmq.cu:1133](cuda/mmq/ds4_mmq.cu#L1133) | | `DS4_CUDA_NO_Q8_ALIGNED_DENSE_SCRATCH` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q8 aligned dense scratch CUDA Q8 optimization. | [cuda/mmq/ds4_mmq.cu:5578](cuda/mmq/ds4_mmq.cu#L5578) | | `DS4_CUDA_NO_Q8_ALIGNED_PERSISTENT` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q8 aligned persistent CUDA Q8 optimization. | [cuda/mmq/ds4_mmq.cu:5410](cuda/mmq/ds4_mmq.cu#L5410) | | `DS4_CUDA_NO_Q8_BATCH_EXACT_TOK2` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q8 batch exact tok2 CUDA Q8 optimization. | [ds4_cuda.cu:19852](ds4_cuda.cu#L19852) | @@ -860,7 +860,7 @@ and **19 tool/wrapper entries**. | `DS4_CUDA_Q4_GROUPED_ATTN_A_ORACLE` | value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on | Compare grouped attention-A against the canonical per-group result. | [ds4_cuda.cu:1477](ds4_cuda.cu#L1477) | | `DS4_CUDA_Q4_K1024_PERSISTENT_ORACLE` | value-aware flag, default off; nonempty value other than exact 0 enables and implies candidate admission | Bitwise-compare the exact-shape persistent Q4 K1024 kernel with canonical MMVQ and retain canonical output. | [cuda/mmq/ds4_mmq.cu:3738](cuda/mmq/ds4_mmq.cu#L3738) | | `DS4_CUDA_Q4_K1024_PERSISTENT_STATS` | value-aware flag, default off; nonempty value other than exact 0 enables | Print exact-shape persistent Q4 K1024 dispatch counters at exit. | [cuda/mmq/ds4_mmq.cu:3737](cuda/mmq/ds4_mmq.cu#L3737) | -| `DS4_CUDA_Q4_MMQ_16WARP` | value-aware opt-in cached on first Q4_K dense MMQ call; unset/empty/exact 0 is off, every other nonempty value requests the candidate; rollback wins and ineligible shapes fall back | Enable the experimental exact-integer CUDA Q4_K m128n128 16-warp kernel for eligible complete-K dense prefills. | [cuda/mmq/ds4_mmq.cu:1052](cuda/mmq/ds4_mmq.cu#L1052) | +| `DS4_CUDA_Q4_MMQ_16WARP` | value-aware opt-in cached on the first Q4_K dense or dense-pair MMQ call; unset/empty/exact 0 is off, every other nonempty value requests the candidate; rollback wins; standalone dense requires M>=1024 while dense-pair admits legs down to M=512 and shares one Q8_1 activation; grids require at least 80% whole-tile SM-wave efficiency and use the canonical Stream-K partition/fixup below its 90% cutoff; ineligible optional shapes fall back | Enable the experimental exact-integer CUDA Q4_K m128n128 16-warp kernel for eligible dense and dense-pair prefills without changing the canonical FP32 reduction tree. | [cuda/mmq/ds4_mmq.cu:1126](cuda/mmq/ds4_mmq.cu#L1126) | | `DS4_CUDA_Q8_F16_ALL` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Control the Q8 F16 all CUDA quantized-matmul/cache optimization. | [ds4_cuda.cu:2358](ds4_cuda.cu#L2358) | | `DS4_CUDA_Q8_F16_CACHE_MB` | unsigned integer MiB, full-string parse; default unlimited; 0 disables this cache | Limit the selective Q8-to-F16 derived-weight cache. | [ds4_cuda.cu:2218](ds4_cuda.cu#L2218) | | `DS4_CUDA_Q8_F16_CACHE_RESERVE_MB` | unsigned integer MiB, full-string parse; default is VRAM-dependent (>=112 GiB: 512; >=40 GiB: max(768,1%); smaller: max(4096,5%)) | Reserve free VRAM when growing the selective Q8-to-F16 cache. | [ds4_cuda.cu:2224](ds4_cuda.cu#L2224) | @@ -878,7 +878,7 @@ and **19 tool/wrapper entries**. | `DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_BATCH` | value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on | Fail if grouped batched attention-A cannot be used. | [ds4_cuda.cu:40198](ds4_cuda.cu#L40198) | | `DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_PREFILL` | value-aware fail-closed assertion, default off; unset/empty/exact 0 is off, any other nonempty value requests the candidate and rejects ineligibility before enqueue | Require the GB10 grouped Q4_K attention-A prefill path instead of silently using pack/MMQ/unpack. | [ds4_cuda.cu:41889](ds4_cuda.cu#L41889) | | `DS4_CUDA_REQUIRE_Q4_K1024_PERSISTENT` | presence flag, default off; any defined value including 0 makes ineligible candidate fail closed | Fail when the exact Q4 K1024 persistent candidate is unavailable instead of using MMVQ. | [cuda/mmq/ds4_mmq.cu:3929](cuda/mmq/ds4_mmq.cu#L3929) | -| `DS4_CUDA_REQUIRE_Q4_MMQ_16WARP` | value-aware fail-closed prefill opt-in cached on first Q4_K dense MMQ call; unset/empty/exact 0 is off, every other nonempty value requests and requires the candidate for N>8; rollback, disabled MMQ, ineligibility, or preflight failure prevents fallback; decode/speculative N<=8 remains on MMVQ | Require the experimental CUDA Q4_K 16-warp prefill kernel so benchmark runs cannot silently measure another path. | [cuda/mmq/ds4_mmq.cu:1056](cuda/mmq/ds4_mmq.cu#L1056); [ds4_cuda.cu:38208](ds4_cuda.cu#L38208) | +| `DS4_CUDA_REQUIRE_Q4_MMQ_16WARP` | value-aware fail-closed prefill opt-in cached on the first Q4_K dense or dense-pair MMQ call; unset/empty/exact 0 is off, every other nonempty value requests and requires the candidate for N>8; a dense-pair is rejected before allocation unless both legs are eligible; rollback, disabled MMQ, ineligibility, or preflight failure prevents fallback; decode/speculative N<=8 remains on MMVQ | Require the experimental CUDA Q4_K 16-warp prefill kernel so benchmark runs cannot silently measure another path. | [cuda/mmq/ds4_mmq.cu:1130](cuda/mmq/ds4_mmq.cu#L1130); [ds4_cuda.cu:38208](ds4_cuda.cu#L38208); [ds4_cuda.cu:38358](ds4_cuda.cu#L38358) | | `DS4_CUDA_REQUIRE_STREAMING_EXPERT_PERSISTENT_CACHE` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Require streaming expert persistent cache in CUDA SSD streaming; fail closed when unavailable. | [ds4_cuda.cu:4117](ds4_cuda.cu#L4117) | | `DS4_CUDA_REQUIRE_STREAMING_SELECTED_BATCH_IO` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Require streaming selected batch I/O in CUDA SSD streaming; fail closed when unavailable. | [ds4_cuda.cu:4738](ds4_cuda.cu#L4738) | | `DS4_CUDA_REQUIRE_STREAMING_SELECTED_EVENT_PIPELINE` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Require streaming selected event pipeline in CUDA SSD streaming; fail closed when unavailable. | [ds4_cuda.cu:4870](ds4_cuda.cu#L4870) | diff --git a/cuda/mmq/ds4_mmq.cu b/cuda/mmq/ds4_mmq.cu index 4448270d1..28efff074 100644 --- a/cuda/mmq/ds4_mmq.cu +++ b/cuda/mmq/ds4_mmq.cu @@ -982,6 +982,49 @@ static bool ds4_q4_test_q8_1_layout( return true; } +/* The candidate and its caller-owned fixup buffer model canonical m128n128. + * Confirm the real canonical picker chooses that tile: a width limit alone is + * insufficient because resource constraints or a ceil-division plateau can + * retain a narrower width and change both partitioning and scratch size. */ +static bool ds4_q4_test_reference_uses_m128n128( + int device, int cc, int N) { + if (device < 0 || device >= GGML_CUDA_MAX_DEVICES || N <= 0 || + get_mmq_y_host(cc) != 128) { + return false; + } + const size_t smpbo = ggml_cuda_info().devices[device].smpbo; + const int warp_size = ggml_cuda_info().devices[device].warp_size; + const int nwarps = mmq_get_nwarps_host(cc, warp_size); + const int mmq_x_max = get_mmq_x_max_host(cc); + int mmq_x_best = 0; + int64_t ntiles_x_best = INT64_MAX; + for (int mmq_x = 8; + mmq_x <= mmq_x_max && ntiles_x_best > 1; + mmq_x += 8) { + const int granularity = mmq_get_granularity_host(mmq_x, cc); + if (mmq_x % granularity != 0 || + mmq_get_nbytes_shared( + mmq_x, 128, cc, warp_size, nwarps) > smpbo) { + continue; + } + const int64_t ntiles_x = + ((int64_t)N + mmq_x - 1) / mmq_x; + if (ntiles_x < ntiles_x_best) { + mmq_x_best = mmq_x; + ntiles_x_best = ntiles_x; + } + } + return mmq_x_best == 128; +} + +extern "C" int +ds4_mmq_q4_K_dense_preq_reference_m128n128_for_test(int N) { + const int dev = ggml_cuda_get_device(); + if (dev < 0 || dev >= GGML_CUDA_MAX_DEVICES) return 0; + return ds4_q4_test_reference_uses_m128n128( + dev, ggml_cuda_info().devices[dev].cc, N) ? 1 : 0; +} + extern "C" size_t ds4_mmq_q4_K_q8_1_scratch_bytes(int N, int K) { size_t total = 0; return ds4_q4_test_q8_1_layout(N, K, nullptr, &total) ? total : 0; @@ -1010,6 +1053,7 @@ extern "C" int ds4_mmq_q4_K_quantize_q8_1_for_test( extern "C" int ds4_mmq_q4_K_dense_preq_reference_for_test( const void *W_q4_K, const void *q8_ds4, size_t q8_bytes, float *out_f32, int M, int N, int K, int use_stream_k, + void *stream_k_fixup, size_t stream_k_fixup_bytes, cudaStream_t stream) { size_t payload = 0, total = 0; if (!W_q4_K || !q8_ds4 || !out_f32 || M <= 0 || @@ -1018,6 +1062,7 @@ extern "C" int ds4_mmq_q4_K_dense_preq_reference_for_test( return -1; } const int dev = ggml_cuda_get_device(); + if (dev < 0 || dev >= GGML_CUDA_MAX_DEVICES) return -1; const int cc = ggml_cuda_info().devices[dev].cc; if (!ds4_mmq_k_tile_supported( "ds4_mmq_q4_K_dense_preq_reference_for_test", K, cc)) { @@ -1025,6 +1070,22 @@ extern "C" int ds4_mmq_q4_K_dense_preq_reference_for_test( } ggml_backend_cuda_context *ctx = get_ctx_for_device(dev); if (!ctx) return -1; + if ((stream_k_fixup == nullptr && stream_k_fixup_bytes != 0u) || + (stream_k_fixup != nullptr && + ((uintptr_t)stream_k_fixup % alignof(float)) != 0) || + (stream_k_fixup_bytes % sizeof(float)) != 0u) { + return -1; + } + if (stream_k_fixup != nullptr) { + if (!use_stream_k) return -1; + if (!ds4_q4_test_reference_uses_m128n128(dev, cc, N)) { + return DS4_MMQ_NOT_APPLICABLE; + } + const size_t required = + ds4_mmq_q4_K_dense_16warp_streamk_scratch_bytes( + M, N, ggml_cuda_info().devices[dev].nsm); + if (required > stream_k_fixup_bytes) return -1; + } ds4_pool_set_stream(stream); const int64_t stride_row_x = (int64_t)K / QK_K; const int64_t stride_y = (int64_t)(payload / sizeof(int)); @@ -1047,6 +1108,10 @@ extern "C" int ds4_mmq_q4_K_dense_preq_reference_for_test( /*stride_sample_dst=*/0, /*use_stream_k=*/use_stream_k != 0, /*ncols_max=*/(int64_t)N, + /*x_soa=*/nullptr, + /*soa_blocks=*/0, + /*stream_k_fixup=*/static_cast(stream_k_fixup), + /*stream_k_fixup_elements=*/stream_k_fixup_bytes / sizeof(float), }; mul_mat_q_case(*ctx, args, stream); const cudaError_t err = cudaGetLastError(); @@ -1064,7 +1129,7 @@ extern "C" int ds4_mmq_q4_K_dense_preq_16warp_for_test( q8_bytes < total) { return -1; } - // This direct oracle hook deliberately accepts an N tail (the production + // This Stream-K oracle hook deliberately accepts an N tail (the production // selector is stricter until NVIDIA measurements justify broadening it), // but it must reject shapes that cannot be represented by this kernel // before enqueueing any work. @@ -1078,8 +1143,17 @@ extern "C" int ds4_mmq_q4_K_dense_preq_16warp_for_test( ds4_q4_16warp_prepare_once(dev) != 0) { return DS4_MMQ_NOT_APPLICABLE; } - return ds4_mmq_q4_K_dense_16warp_enqueue( - W_q4_K, q8_ds4, out_f32, M, N, K, stream); + ggml_backend_cuda_context *ctx = get_ctx_for_device(dev); + if (!ctx) return DS4_MMQ_NOT_APPLICABLE; + ds4_pool_set_stream(stream); + const int nsm = ggml_cuda_info().devices[dev].nsm; + const size_t fixup_bytes = + ds4_mmq_q4_K_dense_16warp_streamk_scratch_bytes(M, N, nsm); + ggml_cuda_pool_alloc fixup(ctx->pool()); + if (fixup_bytes != 0u) fixup.alloc(fixup_bytes); + return ds4_mmq_q4_K_dense_16warp_streamk_enqueue( + W_q4_K, q8_ds4, out_f32, fixup.get(), fixup_bytes, + M, N, K, nsm, stream); } enum { @@ -1127,7 +1201,12 @@ static int ds4_q4_16warp_prepare_once(int device) { return rc; } -static bool ds4_q4_canonical_owns_complete_k(int M, int N, int nsm) { +/* Keep the 16-warp experiment on geometries with enough independent output + * tiles to occupy the device well. Below canonical's 90% whole-tile cutoff, + * the candidate mirrors canonical stream-K partitioning and fixup; this 80% + * gate is therefore only an admission/performance heuristic, not a numerical + * shortcut. On GB10 the Q-A/KV N=4096 shapes score 88%. */ +static bool ds4_q4_16warp_grid_efficient(int M, int N, int nsm) { if (M <= 0 || N <= 0 || nsm <= 0) return false; const int64_t tiles_m = ((int64_t)M + 127) / 128; const int64_t tiles_n = ((int64_t)N + 127) / 128; @@ -1135,7 +1214,70 @@ static bool ds4_q4_canonical_owns_complete_k(int M, int N, int nsm) { const int64_t tiles = tiles_m * tiles_n; const int64_t waves = (tiles + nsm - 1) / nsm; if (waves <= 0 || (int64_t)nsm > INT64_MAX / waves) return false; - return (100 * tiles) / ((int64_t)nsm * waves) >= 90; + return (100 * tiles) / ((int64_t)nsm * waves) >= 80; +} + +static bool ds4_q4_16warp_pair_leg_shape_supported( + int cc, int M, int N, int K) { + return ds4_mmq_q4_K_dense_16warp_available(cc) && + M >= 512 && (M % 128) == 0 && + N >= 512 && (N % 128) == 0 && + K >= 1024 && K <= 4096 && (K % QK_K) == 0; +} + +/* Resolve the experiment before allocation or enqueue. The standalone path + * uses the public M>=1024 gate; a dense-pair leg may go down to M=512 because + * Q-A/KV pairs contain a 512-row leg. Each candidate grid must retain at least + * 80% whole-tile SM-wave efficiency; scheduling itself follows canonical + * stream-K whenever canonical would split K. */ +static int ds4_q4_16warp_select( + const char *tag, int device, int cc, int M, int N, int K, + bool pair_leg, bool *selected) { + if (!selected) return DS4_MMQ_NOT_APPLICABLE; + *selected = false; + const int mode = ds4_q4_16warp_mode(); + const bool disabled = (mode & DS4_Q4_16WARP_DISABLE) != 0; + const bool requested = (mode & DS4_Q4_16WARP_REQUEST) != 0; + const bool required = (mode & DS4_Q4_16WARP_REQUIRE) != 0; + if (!requested) return 0; + + const bool shape_supported = pair_leg + ? ds4_q4_16warp_pair_leg_shape_supported(cc, M, N, K) + : ds4_mmq_q4_K_dense_16warp_supported(cc, M, N, K) != 0; + // The exact oracle models canonical m128n128 MMQ. Check the selector's + // actual result, not only its upper bound: resource limits and equal + // ceil-division plateaus can make it retain a narrower tile. + const bool canonical_x128 = + ds4_q4_test_reference_uses_m128n128(device, cc, N); + const bool grid_efficient = ds4_q4_16warp_grid_efficient( + M, N, ggml_cuda_info().devices[device].nsm); + if (required && (disabled || !shape_supported || !canonical_x128 || + !grid_efficient)) { + fprintf(stderr, + "%s: required Q4 16-warp path is ineligible " + "(scope=%s disabled=%d shape=%d x128=%d grid_eff=%d " + "M=%d N=%d K=%d)\n", + tag, pair_leg ? "pair-leg" : "dense", + disabled ? 1 : 0, shape_supported ? 1 : 0, + canonical_x128 ? 1 : 0, grid_efficient ? 1 : 0, M, N, K); + return DS4_MMQ_NOT_APPLICABLE; + } + if (disabled || !shape_supported || !canonical_x128 || !grid_efficient) { + return 0; + } + + const int prep = ds4_q4_16warp_prepare_once(device); + if (prep == 0) { + *selected = true; + return 0; + } + if (required) { + fprintf(stderr, "%s: required Q4 16-warp preflight failed: %d\n", + tag, prep); + return DS4_MMQ_NOT_APPLICABLE; + } + (void)cudaGetLastError(); + return 0; } #endif @@ -1169,44 +1311,14 @@ int ds4_mmq_dense_impl( const int cc = ggml_cuda_info().devices[dev].cc; if (!ds4_mmq_k_tile_supported(tag, K, cc)) return -1; - bool use_q4_16warp = false; #if !defined(GGML_USE_HIP) + bool use_q4_16warp = false; if constexpr (type == GGML_TYPE_Q4_K) { - const int mode = ds4_q4_16warp_mode(); - const bool disabled = (mode & DS4_Q4_16WARP_DISABLE) != 0; - const bool requested = (mode & DS4_Q4_16WARP_REQUEST) != 0; - const bool required = (mode & DS4_Q4_16WARP_REQUIRE) != 0; - const bool shape_supported = - ds4_mmq_q4_K_dense_16warp_supported(cc, M, N, K) != 0; - // The complete-K parity gate below models canonical m128n128 MMQ. - // A DS4_CUDA_MMQ_X_MAX sweep can make the reference choose a smaller - // X tile and therefore a different stream-K/fixup decision. - const bool canonical_x128 = get_mmq_x_max_host(cc) == 128; - const bool complete_k = ds4_q4_canonical_owns_complete_k( - M, N, ggml_cuda_info().devices[dev].nsm); - if (required && (disabled || !shape_supported || !canonical_x128 || - !complete_k)) { - fprintf(stderr, - "%s: required Q4 16-warp path is ineligible " - "(disabled=%d shape=%d x128=%d complete_k=%d " - "M=%d N=%d K=%d)\n", - tag, disabled ? 1 : 0, shape_supported ? 1 : 0, - canonical_x128 ? 1 : 0, complete_k ? 1 : 0, M, N, K); - return DS4_MMQ_NOT_APPLICABLE; - } - if (requested && !disabled && shape_supported && canonical_x128 && - complete_k) { - const int prep = ds4_q4_16warp_prepare_once(dev); - if (prep == 0) { - use_q4_16warp = true; - } else if (required) { - fprintf(stderr, - "%s: required Q4 16-warp preflight failed: %d\n", - tag, prep); - return DS4_MMQ_NOT_APPLICABLE; - } else { - (void)cudaGetLastError(); - } + const int select_rc = ds4_q4_16warp_select( + tag, dev, cc, M, N, K, /*pair_leg=*/false, + &use_q4_16warp); + if (select_rc != 0) { + return select_rc; } } #endif @@ -1310,10 +1422,19 @@ int ds4_mmq_dense_impl( #if !defined(GGML_USE_HIP) if constexpr (type == GGML_TYPE_Q4_K) { if (use_q4_16warp) { - const int rc = ds4_mmq_q4_K_dense_16warp_enqueue( - W, src1_q8_1.get(), out_f32, M, N, K, stream); + const int nsm = ggml_cuda_info().devices[dev].nsm; + const size_t fixup_bytes = + ds4_mmq_q4_K_dense_16warp_streamk_scratch_bytes( + M, N, nsm); + ggml_cuda_pool_alloc fixup(ctx->pool()); + if (fixup_bytes != 0u) fixup.alloc(fixup_bytes); + const int rc = ds4_mmq_q4_K_dense_16warp_streamk_enqueue( + W, src1_q8_1.get(), out_f32, fixup.get(), fixup_bytes, + M, N, K, nsm, stream); if (rc != 0) { - fprintf(stderr, "%s: Q4 16-warp launch failed: %d\n", tag, rc); + fprintf(stderr, + "%s: Q4 16-warp stream-K launch failed: %d\n", + tag, rc); return -3; } return 0; @@ -1399,6 +1520,19 @@ int ds4_mmq_q4_K_dense_pair_impl( return DS4_MMQ_NOT_APPLICABLE; } +#if !defined(GGML_USE_HIP) + bool use_q4_16warp0 = false; + bool use_q4_16warp1 = false; + int select_rc = ds4_q4_16warp_select( + tag, dev, cc, M0, N, K, /*pair_leg=*/true, + &use_q4_16warp0); + if (select_rc != 0) return select_rc; + select_rc = ds4_q4_16warp_select( + tag, dev, cc, M1, N, K, /*pair_leg=*/true, + &use_q4_16warp1); + if (select_rc != 0) return select_rc; +#endif + ggml_backend_cuda_context *ctx = get_ctx_for_device(dev); if (!ctx) { fprintf(stderr, "%s: failed to get cuda context for device %d\n", @@ -1449,6 +1583,27 @@ int ds4_mmq_q4_K_dense_pair_impl( ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_VOLTA) || GGML_CUDA_CC_IS_CDNA(cc); +#if !defined(GGML_USE_HIP) + const int q4_16warp_nsm = ggml_cuda_info().devices[dev].nsm; + const size_t q4_16warp_fixup0 = use_q4_16warp0 + ? ds4_mmq_q4_K_dense_16warp_streamk_scratch_bytes( + M0, N, q4_16warp_nsm) + : 0u; + const size_t q4_16warp_fixup1 = use_q4_16warp1 + ? ds4_mmq_q4_K_dense_16warp_streamk_scratch_bytes( + M1, N, q4_16warp_nsm) + : 0u; + const size_t q4_16warp_fixup_bytes = + q4_16warp_fixup0 > q4_16warp_fixup1 + ? q4_16warp_fixup0 : q4_16warp_fixup1; + // Both legs are ordered on the same stream, so one allocation can be + // cleared and reused after the first leg's fixup has consumed it. + ggml_cuda_pool_alloc q4_16warp_fixup(ctx->pool()); + if (q4_16warp_fixup_bytes != 0u) { + q4_16warp_fixup.alloc(q4_16warp_fixup_bytes); + } +#endif + if (out_memset_enabled()) { cudaMemsetAsync(out0_f32, 0, out0_bytes, stream); } @@ -1478,12 +1633,28 @@ int ds4_mmq_q4_K_dense_pair_impl( /*use_stream_k=*/use_stream_k, /*ncols_max=*/(int64_t)N, }; - mul_mat_q_case(*ctx, args0, stream); - err = cudaGetLastError(); - if (err != cudaSuccess) { - fprintf(stderr, "%s: first mul_mat_q_case launch failed: %s\n", - tag, cudaGetErrorString(err)); - return -3; +#if !defined(GGML_USE_HIP) + if (use_q4_16warp0) { + const int rc = ds4_mmq_q4_K_dense_16warp_streamk_enqueue( + W0, src1_q8_1.get(), out0_f32, + q4_16warp_fixup.get(), q4_16warp_fixup_bytes, + M0, N, K, q4_16warp_nsm, stream); + if (rc != 0) { + fprintf(stderr, + "%s: first Q4 16-warp stream-K launch failed: %d\n", + tag, rc); + return -3; + } + } else +#endif + { + mul_mat_q_case(*ctx, args0, stream); + err = cudaGetLastError(); + if (err != cudaSuccess) { + fprintf(stderr, "%s: first mul_mat_q_case launch failed: %s\n", + tag, cudaGetErrorString(err)); + return -3; + } } if (out_memset_enabled()) { @@ -1515,12 +1686,28 @@ int ds4_mmq_q4_K_dense_pair_impl( /*use_stream_k=*/use_stream_k, /*ncols_max=*/(int64_t)N, }; - mul_mat_q_case(*ctx, args1, stream); - err = cudaGetLastError(); - if (err != cudaSuccess) { - fprintf(stderr, "%s: second mul_mat_q_case launch failed: %s\n", - tag, cudaGetErrorString(err)); - return -4; +#if !defined(GGML_USE_HIP) + if (use_q4_16warp1) { + const int rc = ds4_mmq_q4_K_dense_16warp_streamk_enqueue( + W1, src1_q8_1.get(), out1_f32, + q4_16warp_fixup.get(), q4_16warp_fixup_bytes, + M1, N, K, q4_16warp_nsm, stream); + if (rc != 0) { + fprintf(stderr, + "%s: second Q4 16-warp stream-K launch failed: %d\n", + tag, rc); + return -4; + } + } else +#endif + { + mul_mat_q_case(*ctx, args1, stream); + err = cudaGetLastError(); + if (err != cudaSuccess) { + fprintf(stderr, "%s: second mul_mat_q_case launch failed: %s\n", + tag, cudaGetErrorString(err)); + return -4; + } } return 0; } diff --git a/cuda/mmq/ds4_mmq.h b/cuda/mmq/ds4_mmq.h index 67daae548..3e34e97ed 100644 --- a/cuda/mmq/ds4_mmq.h +++ b/cuda/mmq/ds4_mmq.h @@ -188,6 +188,7 @@ int ds4_mmq_q4_K_dense( int K, cudaStream_t stream); +#if !defined(GGML_USE_HIP) // CUDA benchmark/test boundary for separating activation quantization from // the Q4_K MMQ kernel. The scratch layout is canonical block_q8_1_mmq DS4 // ([K/128][N]) plus a zeroed 128-column tail. These helpers never transfer @@ -202,9 +203,17 @@ int ds4_mmq_q4_K_quantize_q8_1_for_test( int K, cudaStream_t stream); +// Return non-zero only when the current device's canonical Q4_K MMQ picker +// selects m128n128 for this activation-column count. Kernel A/B harnesses use +// this to reject tail geometries whose Stream-K tile partition is different. +int ds4_mmq_q4_K_dense_preq_reference_m128n128_for_test(int N); + // Enqueue-only A/B arms over a caller-owned, already-quantized activation. -// The reference can disable stream-K to give the 16-warp candidate the same -// complete-K reduction tree. Zero is success; nonzero is rejection/failure. +// The reference may disable stream-K for a complete-K control; the 16-warp +// candidate helper follows the production canonical stream-K/fixup policy. +// Passing caller-owned reference fixup storage keeps pool alloc/free nodes out +// of CUDA-event kernel benchmarks; null/zero retains the production pool. +// Zero is success; nonzero is rejection/failure. int ds4_mmq_q4_K_dense_preq_reference_for_test( const void * W_q4_K, const void * q8_ds4, @@ -214,6 +223,8 @@ int ds4_mmq_q4_K_dense_preq_reference_for_test( int N, int K, int use_stream_k, + void * stream_k_fixup, + size_t stream_k_fixup_bytes, cudaStream_t stream); int ds4_mmq_q4_K_dense_preq_16warp_for_test( @@ -225,12 +236,18 @@ int ds4_mmq_q4_K_dense_preq_16warp_for_test( int N, int K, cudaStream_t stream); +#endif // Two dense Q4_K MMQ projections that share one token-tiled Q8_1 // activation buffer. This is the prefill sibling of // ds4_mmq_q4_K_dense_pair_vec: N is not limited to the MMVQ batch ceiling, // M0 and M1 may differ, and each leg preserves ds4_mmq_q4_K_dense's // reduction and output layout. The two output ranges must be disjoint. +// On CUDA, the opt-in 16-warp experiment selects each leg independently +// (down to M=512) when its output-tile grid retains at least 80% SM-wave +// efficiency; below canonical's whole-tile cutoff it mirrors canonical +// stream-K partitioning and fixup. REQUIRE rejects the pair before allocation +// unless both legs can use the candidate. // Shape/capability rejection returns DS4_MMQ_NOT_APPLICABLE before enqueue; // negative values report an attempted-path launch failure. int ds4_mmq_q4_K_dense_pair( diff --git a/cuda/mmq/ds4_mmq_q4_16warp.cu b/cuda/mmq/ds4_mmq_q4_16warp.cu index 38f05d052..6601f9881 100644 --- a/cuda/mmq/ds4_mmq_q4_16warp.cu +++ b/cuda/mmq/ds4_mmq_q4_16warp.cu @@ -14,7 +14,8 @@ // * canonical Q8_1 DS4 (half scale + half sum) activation blocks; // * identical ascending sequence of eight K32 folds per Q4_K block; // * the two canonical F32 accumulation statements are kept verbatim; -// * one CTA owns the complete K reduction (no stream-K/fixup tree). +// * direct tiling and canonical Stream-K/fixup reduction trees are both +// available; the caller chooses explicitly at the enqueue boundary. #include "ds4_mmq_q4_16warp.cuh" @@ -40,6 +41,7 @@ constexpr int kMetadataWarps = kMTile / 16; constexpr int kWeightStride = MMQ_MMA_TILE_X_K_Q8_1; constexpr int kYStrideInts = sizeof(block_q8_1_mmq) / sizeof(int); constexpr int kYChunks16 = sizeof(block_q8_1_mmq) / 16; +constexpr size_t kTileElements = (size_t)kMTile * (size_t)kNTile; constexpr size_t kWeightTileBytes = (size_t)kMTile * (size_t)kWeightStride * sizeof(int); @@ -55,6 +57,8 @@ static_assert(kNFragPerWarp == 4, "Q4 split-N fragment count changed"); static_assert(kWeightStride == 76, "canonical Q4_K MMA row stride changed"); static_assert(kYStrideInts == 36, "canonical Q8_1 DS4 stride changed"); static_assert(kYChunks16 == 9, "canonical Q8_1 DS4 block size changed"); +static_assert(MMQ_ITER_K % QK_K == 0 && MMQ_ITER_K / QK_K == 1, + "Stream-K scheduler must restore canonical K alignment"); static_assert(kSharedBytes == 57344, "Q4 16-warp shared-memory model changed"); static_assert(kSharedBytes <= 99ull * 1024ull, "Q4 16-warp kernel exceeds the intended opt-in shared limit"); @@ -71,10 +75,12 @@ __device__ __forceinline__ int linear_tid() { return (warp_id() << 5) | lane_id(); } +template __device__ __forceinline__ void load_weight_tile( const block_q4_K * __restrict__ W, int * __restrict__ tile, int cta_row0, + int M, int blocks_per_row, int kb) { int *x_qs = tile; @@ -86,8 +92,10 @@ __device__ __forceinline__ void load_weight_tile( // visits eight rows; all 128 rows are covered exactly once. #pragma unroll for (int row = warp; row < kMTile; row += kWarps) { + const int global_row = need_check && cta_row0 + row >= M + ? M - 1 : cta_row0 + row; const block_q4_K *b = - W + (uint64_t)(cta_row0 + row) * (uint64_t)blocks_per_row + kb; + W + (uint64_t)global_row * (uint64_t)blocks_per_row + kb; const int qs0 = get_int_b4(b->qs, lane); x_qs[row * kWeightStride + 16 * (lane / 8) + lane % 8 + 0] = (qs0 >> 0) & 0x0F0F0F0F; @@ -101,8 +109,10 @@ __device__ __forceinline__ void load_weight_tile( if (warp < kMetadataWarps) { const int row = warp * 16 + lane / 2; const int ksc = lane & 1; + const int global_row = need_check && cta_row0 + row >= M + ? M - 1 : cta_row0 + row; const block_q4_K *b = - W + (uint64_t)(cta_row0 + row) * (uint64_t)blocks_per_row + kb; + W + (uint64_t)global_row * (uint64_t)blocks_per_row + kb; const int *scales = reinterpret_cast(b->scales); const int sc32 = unpack_scales_q45_K(scales, ksc + 0); const int m32 = unpack_scales_q45_K(scales, ksc + 2); @@ -215,31 +225,34 @@ __device__ __forceinline__ void fold_y_half( } } -__global__ __launch_bounds__(kThreads, 1) -void dense_q4_16warp_kernel( +template +__device__ __forceinline__ void process_tile_range( const block_q4_K * __restrict__ W, const block_q8_1_mmq * __restrict__ q8, float * __restrict__ out, + float * __restrict__ tmp_fixup, int M, int N, - int K) { + int K, + int it, + int jt, + int kb_start, + int kb_stop, + int * __restrict__ x_tile, + block_q8_1_mmq * __restrict__ y_tile) { #if defined(TURING_MMA_AVAILABLE) using tile_A = ggml_cuda_mma::tile<16, 8, int>; using tile_B = ggml_cuda_mma::tile<8, 8, int>; using tile_C = ggml_cuda_mma::tile<16, 8, int>; - extern __shared__ __align__(16) unsigned char dynamic_smem[]; - int *x_tile = reinterpret_cast(dynamic_smem); - block_q8_1_mmq *y_tile = reinterpret_cast( - dynamic_smem + kWeightTileBytes); - - const int cta_row0 = (int)blockIdx.x * kMTile; - const int col0 = (int)blockIdx.y * kNTile; + const int cta_row0 = it * kMTile; + const int col0 = jt * kNTile; const int blocks_per_row = K / QK_K; float acc[kNFragPerWarp][kRowFrag][tile_C::ne] = {}; - for (int kb = 0; kb < blocks_per_row; ++kb) { - load_weight_tile(W, x_tile, cta_row0, blocks_per_row, kb); + for (int kb = kb_start; kb < kb_stop; ++kb) { + load_weight_tile( + W, x_tile, cta_row0, M, blocks_per_row, kb); load_y_tile(q8, y_tile, N, col0, 2 * kb + 0); __syncthreads(); @@ -267,21 +280,180 @@ void dense_q4_16warp_kernel( for (int l = 0; l < tile_C::ne; ++l) { const int row = out_row0 + nr * 16 + tile_C::get_i(l); const int col = out_col0 + nf * 8 + tile_C::get_j(l); - if (col < N) { - const float value = isfinite(acc[nf][nr][l]) - ? acc[nf][nr][l] : 0.0f; + if constexpr (to_fixup) { + // Canonical MMQ always materializes a complete 128x128 + // final-partial tile in block-private storage. Tail rows + // and columns are masked only when the fixup publishes it. + tmp_fixup[(size_t)blockIdx.x * kTileElements + + (size_t)(col - col0) * kMTile + + (size_t)(row - cta_row0)] = acc[nf][nr][l]; + } else if (row < M && col < N) { + float value = acc[nf][nr][l]; + // A leading Stream-K partial is deliberately left + // unsanitized. Canonical fixup sanitizes only after the + // complete reduction tree has been reconstructed. + if (kb_start == 0 && kb_stop == blocks_per_row && + !isfinite(value)) { + value = 0.0f; + } out[(uint64_t)col * (uint64_t)M + (uint64_t)row] = value; } } } } +#else + GGML_UNUSED_VARS(W, q8, out, tmp_fixup, M, N, K, it, jt, + kb_start, kb_stop); + GGML_UNUSED_VARS(x_tile, y_tile); + NO_DEVICE_CODE; +#endif +} + +__global__ __launch_bounds__(kThreads, 1) +void dense_q4_16warp_kernel( + const block_q4_K * __restrict__ W, + const block_q8_1_mmq * __restrict__ q8, + float * __restrict__ out, + int M, + int N, + int K) { +#if defined(TURING_MMA_AVAILABLE) + extern __shared__ __align__(16) unsigned char dynamic_smem[]; + int *x_tile = reinterpret_cast(dynamic_smem); + block_q8_1_mmq *y_tile = reinterpret_cast( + dynamic_smem + kWeightTileBytes); + const int it = (int)blockIdx.x; + const int jt = (int)blockIdx.y; + process_tile_range( + W, q8, out, nullptr, M, N, K, it, jt, + 0, K / QK_K, x_tile, y_tile); #else GGML_UNUSED_VARS(W, q8, out, M, N, K); NO_DEVICE_CODE; #endif } +// The integer partition and flattened tile order intentionally mirror +// mul_mat_q. One CTA may finish a leading split +// tile, own zero or more complete tiles, and publish one trailing prefix for +// canonical mul_mat_q_stream_k_fixup. +template +__global__ __launch_bounds__(kThreads, 1) +void dense_q4_16warp_streamk_kernel( + const block_q4_K * __restrict__ W, + const block_q8_1_mmq * __restrict__ q8, + float * __restrict__ out, + float * __restrict__ tmp_fixup, + int M, + int N, + int K) { +#if defined(TURING_MMA_AVAILABLE) + extern __shared__ __align__(16) unsigned char dynamic_smem[]; + int *x_tile = reinterpret_cast(dynamic_smem); + block_q8_1_mmq *y_tile = reinterpret_cast( + dynamic_smem + kWeightTileBytes); + + // Convert before adding the tile bias so syntactically valid INT_MAX + // dimensions cannot overflow signed arithmetic in device code. + const int nty = (int)(((unsigned)M + (unsigned)kMTile - 1u) / + (unsigned)kMTile); + const int ntx = (int)(((unsigned)N + (unsigned)kNTile - 1u) / + (unsigned)kNTile); + const int blocks_per_row = K / QK_K; + const int64_t total = (int64_t)nty * ntx * blocks_per_row; + + int kbc = (int)((int64_t)blockIdx.x * total / gridDim.x); + const int kbc_stop = + (int)((int64_t)(blockIdx.x + 1) * total / gridDim.x); + + int kb_start = kbc % blocks_per_row; + int kb_stop = min(blocks_per_row, kb_start + kbc_stop - kbc); + while (kbc < kbc_stop && kb_stop == blocks_per_row) { + const int tile = kbc / blocks_per_row; + const int jt = tile % ntx; + const int it = tile / ntx; + process_tile_range( + W, q8, out, tmp_fixup, M, N, K, it, jt, + kb_start, kb_stop, x_tile, y_tile); + + kbc += blocks_per_row; + kbc -= kbc % blocks_per_row; + kb_start = 0; + kb_stop = min(blocks_per_row, kbc_stop - kbc); + } + + if (kbc >= kbc_stop) { + return; + } + + const int tile = kbc / blocks_per_row; + const int jt = tile % ntx; + const int it = tile / ntx; + process_tile_range( + W, q8, out, tmp_fixup, M, N, K, it, jt, + kb_start, kb_stop, x_tile, y_tile); +#else + GGML_UNUSED_VARS(W, q8, out, tmp_fixup, M, N, K); + NO_DEVICE_CODE; +#endif +} + +struct streamk_schedule { + unsigned nty; + unsigned ntx; + unsigned ntiles; + unsigned grid_x; + bool fixup_needed; + size_t scratch_bytes; +}; + +static bool make_streamk_schedule( + int M, int N, int nsm, streamk_schedule *schedule) { + if (M <= 0 || N <= 0 || nsm <= 0 || schedule == nullptr) { + return false; + } + + const uint64_t nty64 = + ((uint64_t)(unsigned)M + (uint64_t)kMTile - 1u) / kMTile; + const uint64_t ntx64 = + ((uint64_t)(unsigned)N + (uint64_t)kNTile - 1u) / kNTile; + if (nty64 == 0 || ntx64 == 0 || nty64 > UINT32_MAX || + ntx64 > UINT32_MAX || nty64 > UINT32_MAX / ntx64) { + return false; + } + + const uint64_t ntiles64 = nty64 * ntx64; + const uint64_t nsm64 = (uint64_t)(unsigned)nsm; + const uint64_t nwaves = (ntiles64 + nsm64 - 1u) / nsm64; + if (nwaves == 0 || nsm64 > UINT64_MAX / nwaves) { + return false; + } + const uint64_t wave_slots = nsm64 * nwaves; + const uint64_t efficiency = 100u * ntiles64 / wave_slots; + const uint64_t grid64 = efficiency >= 90u ? ntiles64 : nsm64; + if (grid64 == 0 || grid64 > UINT32_MAX) { + return false; + } + + const bool fixup_needed = ntiles64 % grid64 != 0; + size_t bytes = 0; + if (fixup_needed) { + if (grid64 > SIZE_MAX / kTileElements / sizeof(float)) { + return false; + } + bytes = (size_t)grid64 * kTileElements * sizeof(float); + } + + schedule->nty = (unsigned)nty64; + schedule->ntx = (unsigned)ntx64; + schedule->ntiles = (unsigned)ntiles64; + schedule->grid_x = (unsigned)grid64; + schedule->fixup_needed = fixup_needed; + schedule->scratch_bytes = bytes; + return true; +} + } // namespace q4w16 } // anonymous namespace @@ -295,7 +467,7 @@ extern "C" int ds4_mmq_q4_K_dense_16warp_supported( if (!ds4_mmq_q4_K_dense_16warp_available(cc)) { return 0; } - return M >= 2048 && (M % q4w16::kMTile) == 0 && + return M >= 1024 && (M % q4w16::kMTile) == 0 && N >= 512 && (N % q4w16::kNTile) == 0 && K >= 1024 && K <= 4096 && (K % QK_K) == 0; } @@ -326,6 +498,20 @@ extern "C" int ds4_mmq_q4_K_dense_16warp_prepare(void) { dense_q4_16warp_kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, (int)kSharedBytes); + if (err != cudaSuccess) { + return -3; + } + err = cudaFuncSetAttribute( + dense_q4_16warp_streamk_kernel, + cudaFuncAttributeMaxDynamicSharedMemorySize, + (int)kSharedBytes); + if (err != cudaSuccess) { + return -3; + } + err = cudaFuncSetAttribute( + dense_q4_16warp_streamk_kernel, + cudaFuncAttributeMaxDynamicSharedMemorySize, + (int)kSharedBytes); return err == cudaSuccess ? 0 : -3; } @@ -346,10 +532,14 @@ extern "C" int ds4_mmq_q4_K_dense_16warp_enqueue( // Convert before adding the tile bias: N is a positive signed int, but // N + 127 would otherwise overflow for a (syntactically valid) INT_MAX // direct-enqueue request. - const dim3 grid((unsigned)M / (unsigned)kMTile, - ((unsigned)N + (unsigned)kNTile - 1u) / - (unsigned)kNTile, - 1); + const unsigned grid_y = + ((unsigned)N + (unsigned)kNTile - 1u) / (unsigned)kNTile; + // CUDA guarantees only 65535 blocks on y/z. Production shapes are far + // below this, but reject oversized raw-enqueue requests before launch. + if (grid_y > 65535u) { + return -1; + } + const dim3 grid((unsigned)M / (unsigned)kMTile, grid_y, 1); const dim3 block(32, kWarps, 1); dense_q4_16warp_kernel<<>>( static_cast(W), @@ -358,3 +548,115 @@ extern "C" int ds4_mmq_q4_K_dense_16warp_enqueue( const cudaError_t err = cudaGetLastError(); return err == cudaSuccess ? 0 : -4; } + +extern "C" size_t ds4_mmq_q4_K_dense_16warp_streamk_scratch_bytes( + int M, int N, int nsm) { + q4w16::streamk_schedule schedule; + return q4w16::make_streamk_schedule(M, N, nsm, &schedule) + ? schedule.scratch_bytes : 0; +} + +extern "C" int ds4_mmq_q4_K_dense_16warp_streamk_enqueue( + const void *W, + const void *q8_ds4, + float *out, + void *scratch, + size_t scratch_bytes, + int M, + int N, + int K, + int nsm, + cudaStream_t stream) { + using namespace q4w16; + if (!W || !q8_ds4 || !out || M <= 0 || N <= 0 || K <= 0 || + nsm <= 0 || (K % QK_K) != 0) { + return -1; + } + + streamk_schedule schedule; + if (!make_streamk_schedule(M, N, nsm, &schedule)) { + return -1; + } + + const uint64_t blocks_per_row = (uint64_t)(unsigned)K / QK_K; + const uint64_t total = (uint64_t)schedule.ntiles * blocks_per_row; + // Match the canonical launcher's invariant: the device scheduler stores + // flattened K-block coordinates in signed int variables. + if (blocks_per_row == 0 || total >= (1ull << 30)) { + return -1; + } + + // Canonical Stream-K degenerates to one complete-K CTA per output tile + // at high whole-tile efficiency. Preserve that policy while using the + // cheaper 2-D direct launch for the aligned production shapes: it avoids + // per-CTA flattened-index divisions and needs neither scratch nor fixup. + if (!schedule.fixup_needed && (M % kMTile) == 0) { + return ds4_mmq_q4_K_dense_16warp_enqueue( + W, q8_ds4, out, M, N, K, stream); + } + + if (schedule.fixup_needed) { + if (!scratch || scratch_bytes < schedule.scratch_bytes || + ((uintptr_t)scratch % alignof(float)) != 0) { + return -1; + } + const cudaError_t memset_err = cudaMemsetAsync( + scratch, 0, schedule.scratch_bytes, stream); + if (memset_err != cudaSuccess) { + return -2; + } + } + + const dim3 grid(schedule.grid_x, 1, 1); + const dim3 block(32, kWarps, 1); + float *tmp_fixup = schedule.fixup_needed + ? static_cast(scratch) : nullptr; + if ((M % kMTile) == 0) { + dense_q4_16warp_streamk_kernel + <<>>( + static_cast(W), + static_cast(q8_ds4), + out, tmp_fixup, M, N, K); + } else { + dense_q4_16warp_streamk_kernel + <<>>( + static_cast(W), + static_cast(q8_ds4), + out, tmp_fixup, M, N, K); + } + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) { + return -4; + } + + if (!schedule.fixup_needed) { + return 0; + } + + const uint3 blocks_per_ne00_fd = init_fastdiv_values(blocks_per_row); + const uint3 one_fd = init_fastdiv_values(1); + const uint3 ntx_fd = init_fastdiv_values(schedule.ntx); + const dim3 fixup_grid(schedule.grid_x, kMTile / 32, 1); + const dim3 fixup_block(32, 4, 1); + if ((M % kMTile) == 0) { + constexpr bool need_check = false; + mul_mat_q_stream_k_fixup< + GGML_TYPE_Q4_K, kNTile, need_check, false> + <<>>( + /*ids_dst=*/nullptr, /*expert_bounds=*/nullptr, out, + tmp_fixup, blocks_per_ne00_fd, M, N, M, + one_fd, /*stride_channel_dst=*/0, + one_fd, /*stride_sample_dst=*/0, ntx_fd); + } else { + constexpr bool need_check = true; + mul_mat_q_stream_k_fixup< + GGML_TYPE_Q4_K, kNTile, need_check, false> + <<>>( + /*ids_dst=*/nullptr, /*expert_bounds=*/nullptr, out, + tmp_fixup, blocks_per_ne00_fd, M, N, M, + one_fd, /*stride_channel_dst=*/0, + one_fd, /*stride_sample_dst=*/0, ntx_fd); + } + err = cudaGetLastError(); + return err == cudaSuccess ? 0 : -5; +} diff --git a/cuda/mmq/ds4_mmq_q4_16warp.cuh b/cuda/mmq/ds4_mmq_q4_16warp.cuh index cb76aa607..e9d39b4d1 100644 --- a/cuda/mmq/ds4_mmq_q4_16warp.cuh +++ b/cuda/mmq/ds4_mmq_q4_16warp.cuh @@ -11,6 +11,10 @@ #include #endif +#include + +#if !defined(GGML_USE_HIP) + #ifdef __cplusplus extern "C" { #endif @@ -19,9 +23,11 @@ extern "C" { // the m128n128, 16-warp integer-MMA kernel. int ds4_mmq_q4_K_dense_16warp_available(int cc); -// Conservative production admission gate. Availability and shape are both -// checked; the initial gate admits only complete 128x128 output tiles so the -// canonical A/B arm also selects mmq_x=128. A false result must fall back. +// Conservative standalone production admission gate. Availability and shape +// are both checked; it admits M>=1024 and only complete 128x128 output tiles. +// The dispatcher separately enforces the m128n128 reference selector and its +// candidate-grid efficiency gate. A false result must fall back. The pair +// dispatcher has a separate per-leg M>=512 gate. int ds4_mmq_q4_K_dense_16warp_supported(int cc, int M, int N, int K); // Opt in the 56 KiB dynamic-shared-memory launch on the current device. @@ -50,6 +56,38 @@ int ds4_mmq_q4_K_dense_16warp_enqueue( int K, cudaStream_t stream); +// Return the caller-owned fixup storage required by the canonical Stream-K +// partition for this dense MxN output shape and SM count. Zero means either +// that no fixup is necessary (the selected grid owns complete tiles) or that +// the arguments/size cannot be represented; enqueue repeats all validation. +// The storage, when non-zero, is a byte buffer and need only remain valid until +// the work already enqueued on `stream` has completed. +size_t ds4_mmq_q4_K_dense_16warp_streamk_scratch_bytes( + int M, + int N, + int nsm); + +// Enqueue the same 16-warp arithmetic using canonical CUDA MMQ Stream-K +// scheduling and its exact Q4_K fixup reduction tree. W, q8_ds4 and out use +// the layouts documented above. `scratch` must provide at least the size +// returned by ds4_mmq_q4_K_dense_16warp_streamk_scratch_bytes; it may be null +// when that function returns zero. The routine performs only asynchronous +// memset/kernel operations and does not allocate or synchronize. +// ds4_mmq_q4_K_dense_16warp_prepare must have succeeded on the current device. +int ds4_mmq_q4_K_dense_16warp_streamk_enqueue( + const void * W, + const void * q8_ds4, + float * out, + void * scratch, + size_t scratch_bytes, + int M, + int N, + int K, + int nsm, + cudaStream_t stream); + #ifdef __cplusplus } #endif + +#endif // !defined(GGML_USE_HIP) diff --git a/cuda/mmq/mmq.cuh b/cuda/mmq/mmq.cuh index 0b03cc2d6..89582a754 100644 --- a/cuda/mmq/mmq.cuh +++ b/cuda/mmq/mmq.cuh @@ -4238,6 +4238,10 @@ struct mmq_args { // ignored; soa_blocks = pair count (Q2_K) or block count (IQ2_XXS). // Trailing fields so existing aggregate initializers value-init them. const char * x_soa; int64_t soa_blocks; + // Optional caller-owned Stream-K fixup tile storage. Production callers + // normally leave this null and use the CUDA pool; kernel-only A/B harnesses + // provide it so cudaMallocAsync/cudaFreeAsync are outside their events. + float * stream_k_fixup; size_t stream_k_fixup_elements; }; template @@ -4326,12 +4330,19 @@ static void launch_mul_mat_q(ggml_backend_cuda_context & ctx, const mmq_args & a ggml_cuda_pool & pool = ctx.pool(id); ggml_cuda_pool_alloc tmp_fixup(pool); + float * tmp_fixup_ptr = nullptr; if (fixup_needed) { - tmp_fixup.alloc((size_t)block_nums_stream_k.x * (size_t)block_nums_stream_k.z * mmq_x*mmq_y); - CUDA_CHECK(cudaMemsetAsync(tmp_fixup.ptr, 0, - (size_t)block_nums_stream_k.x * (size_t)block_nums_stream_k.z * - (size_t)mmq_x * (size_t)mmq_y * sizeof(float), - stream)); + const size_t fixup_elements = + (size_t)block_nums_stream_k.x * + (size_t)block_nums_stream_k.z * mmq_x * mmq_y; + if (args.stream_k_fixup != nullptr) { + GGML_ASSERT(args.stream_k_fixup_elements >= fixup_elements); + tmp_fixup_ptr = args.stream_k_fixup; + } else { + tmp_fixup_ptr = tmp_fixup.alloc(fixup_elements); + } + CUDA_CHECK(cudaMemsetAsync( + tmp_fixup_ptr, 0, fixup_elements * sizeof(float), stream)); } const dim3 block_nums_fixup(block_nums_stream_k.x, mmq_y/warp_size, block_nums_stream_k.z); @@ -4340,7 +4351,7 @@ static void launch_mul_mat_q(ggml_backend_cuda_context & ctx, const mmq_args & a if (args.nrows_x % mmq_y == 0) { constexpr bool need_check = false; mul_mat_q<<>> - (args.x, args.y, args.ids_dst, args.expert_bounds, args.dst, tmp_fixup.ptr, + (args.x, args.y, args.ids_dst, args.expert_bounds, args.dst, tmp_fixup_ptr, blocks_per_ne00_fd, args.nrows_x, args.ncols_dst, args.stride_row_x, args.ncols_y, args.nrows_dst, channel_ratio_fd, nchannels_y_fd, args.stride_channel_x, args.stride_channel_y, args.stride_channel_dst, sample_ratio_fd, nsamples_y_fd, args.stride_sample_x, args.stride_sample_y, args.stride_sample_dst, @@ -4352,13 +4363,13 @@ static void launch_mul_mat_q(ggml_backend_cuda_context & ctx, const mmq_args & a CUDA_CHECK(cudaGetLastError()); mul_mat_q_stream_k_fixup<<>> - (args.ids_dst, args.expert_bounds, args.dst, tmp_fixup.ptr, blocks_per_ne00_fd, args.nrows_x, args.ncols_dst, + (args.ids_dst, args.expert_bounds, args.dst, tmp_fixup_ptr, blocks_per_ne00_fd, args.nrows_x, args.ncols_dst, args.nrows_dst, nchannels_y_fd, args.stride_channel_dst, nsamples_y_fd, args.stride_sample_dst, ntx_fd); } else { constexpr bool need_check = true; mul_mat_q<<>> - (args.x, args.y, args.ids_dst, args.expert_bounds, args.dst, tmp_fixup.ptr, + (args.x, args.y, args.ids_dst, args.expert_bounds, args.dst, tmp_fixup_ptr, blocks_per_ne00_fd, args.nrows_x, args.ncols_dst, args.stride_row_x, args.ncols_y, args.nrows_dst, channel_ratio_fd, nchannels_y_fd, args.stride_channel_x, args.stride_channel_y, args.stride_channel_dst, sample_ratio_fd, nsamples_y_fd, args.stride_sample_x, args.stride_sample_y, args.stride_sample_dst, @@ -4370,7 +4381,7 @@ static void launch_mul_mat_q(ggml_backend_cuda_context & ctx, const mmq_args & a CUDA_CHECK(cudaGetLastError()); mul_mat_q_stream_k_fixup<<>> - (args.ids_dst, args.expert_bounds, args.dst, tmp_fixup.ptr, blocks_per_ne00_fd, args.nrows_x, args.ncols_dst, + (args.ids_dst, args.expert_bounds, args.dst, tmp_fixup_ptr, blocks_per_ne00_fd, args.nrows_x, args.ncols_dst, args.nrows_dst, nchannels_y_fd, args.stride_channel_dst, nsamples_y_fd, args.stride_sample_dst, ntx_fd); } diff --git a/cuda/mmq/test/test_mmq_parity.cu b/cuda/mmq/test/test_mmq_parity.cu index 9ba4217b9..60391b1a2 100644 --- a/cuda/mmq/test/test_mmq_parity.cu +++ b/cuda/mmq/test/test_mmq_parity.cu @@ -20,6 +20,7 @@ // -o test_mmq_parity #include "ds4_mmq.h" +#include "ds4_mmq_q4_16warp.cuh" #include "iq2_host_tables.h" // Pull in the block_* struct definitions. We use the CUDA decl/impl mode @@ -104,6 +105,34 @@ private: bool active_ = false; }; +cudaError_t enqueue_scratch_guard_copy( + const void *storage, size_t payload_bytes, size_t guard_bytes, + uint8_t *host_guards, cudaStream_t stream) { + if (!storage || !host_guards || guard_bytes == 0) { + // cudaError_t is an integer-compatible type in both the CUDA runtime and + // the host-only syntax-check stub; avoid depending on a stubbed enum. + return static_cast(1); + } + const auto *bytes = static_cast(storage); + cudaError_t err = cudaMemcpyAsync( + host_guards, bytes, guard_bytes, cudaMemcpyDeviceToHost, stream); + if (err == cudaSuccess) { + err = cudaMemcpyAsync( + host_guards + guard_bytes, bytes + guard_bytes + payload_bytes, + guard_bytes, cudaMemcpyDeviceToHost, stream); + } + return err; +} + +size_t scratch_guard_mismatches( + const std::vector &guards, uint8_t expected) { + size_t bad = 0; + for (uint8_t value : guards) { + if (value != expected) bad++; + } + return bad; +} + // -------------------------------------------------------------------------- // Half-precision conversion (standalone, no CUDA host fp16 needed). // -------------------------------------------------------------------------- @@ -549,18 +578,20 @@ bool run_q4_K(int M, int N, int K, uint32_t seed, float abs_scale = 0.20f) { } // Resident-kernel oracle for the opt-in m128n128/16-warp Q4_K prefill -// candidate. All arms consume the same caller-owned DS4 Q8_1 activation. -// The complete-K reference is always checked; for a no-fixup shape the real -// stream-K production arm must also be bit-identical. Guard regions catch -// output overruns independently of numerical parity. +// candidate. The raw canonical/candidate arms consume the same caller-owned +// DS4 Q8_1 activation. The canonical arm uses the production stream-K policy, +// including fixup when selected. The production API can be checked as a third, +// independently guarded output without folding its quantizer into the raw +// kernel comparison. bool run_q4_K_dense_16warp_parity( - int M, int N, int K, uint32_t seed, - bool check_production, bool check_rejection) { + int M, int N, int K, int nsm, uint32_t seed, + bool check_public_dense, bool check_rejection) { fprintf(stderr, "=== Q4_K/DENSE_16WARP M=%d N=%d K=%d seed=%u%s ===\n", - M, N, K, seed, check_production ? " stream-k" : ""); + M, N, K, seed, check_public_dense ? " production" : ""); - if (M <= 0 || N <= 0 || K <= 0 || K % QK_K_LOCAL != 0 || + if (M <= 0 || N <= 0 || K <= 0 || nsm <= 0 || + K % QK_K_LOCAL != 0 || (size_t)M > SIZE_MAX / (size_t)N / sizeof(float)) { fprintf(stderr, "invalid 16-warp parity shape\n\n"); return false; @@ -585,10 +616,18 @@ bool run_q4_K_dense_16warp_parity( constexpr uint8_t prod_guard_byte = 0xc3; constexpr uint8_t got_guard_byte = 0x5a; constexpr uint8_t reject_byte = 0x3c; + constexpr uint8_t scratch_guard_byte = 0x7d; const size_t output_count = (size_t)M * N; const size_t output_bytes = output_count * sizeof(float); const size_t guard_bytes = guard_floats * sizeof(float); const size_t guarded_bytes = output_bytes + 2u * guard_bytes; + const size_t scratch_bytes = + ds4_mmq_q4_K_dense_16warp_streamk_scratch_bytes(M, N, nsm); + if (scratch_bytes > SIZE_MAX - 2u * guard_bytes) { + fprintf(stderr, "Q4_K 16-warp scratch guard size overflow\n\n"); + return false; + } + const size_t scratch_guarded_bytes = scratch_bytes + 2u * guard_bytes; cudaStream_t stream = nullptr; void *dW = nullptr; @@ -597,15 +636,18 @@ bool run_q4_K_dense_16warp_parity( float *dRefStorage = nullptr; float *dProdStorage = nullptr; float *dGotStorage = nullptr; + void *dScratchStorage = nullptr; const bool allocated = cudaStreamCreate(&stream) == cudaSuccess && cudaMalloc(&dW, W.size() * sizeof(block_q4_K)) == cudaSuccess && cudaMalloc(&dX, X.size() * sizeof(float)) == cudaSuccess && cudaMalloc(&dQ8, q8_bytes) == cudaSuccess && cudaMalloc(&dRefStorage, guarded_bytes) == cudaSuccess && - (!check_production || + (!check_public_dense || cudaMalloc(&dProdStorage, guarded_bytes) == cudaSuccess) && - cudaMalloc(&dGotStorage, guarded_bytes) == cudaSuccess; + cudaMalloc(&dGotStorage, guarded_bytes) == cudaSuccess && + cudaMalloc(&dScratchStorage, scratch_guarded_bytes) == cudaSuccess; const auto cleanup = [&]() { + if (dScratchStorage) cudaFree(dScratchStorage); if (dGotStorage) cudaFree(dGotStorage); if (dProdStorage) cudaFree(dProdStorage); if (dRefStorage) cudaFree(dRefStorage); @@ -622,9 +664,11 @@ bool run_q4_K_dense_16warp_parity( } float *const dRef = dRefStorage + guard_floats; - float *const dProd = check_production + float *const dProd = check_public_dense ? dProdStorage + guard_floats : nullptr; float *const dGot = dGotStorage + guard_floats; + void *const dScratch = + static_cast(dScratchStorage) + guard_bytes; cudaError_t enqueue_err = cudaMemcpyAsync( dW, W.data(), W.size() * sizeof(block_q4_K), cudaMemcpyHostToDevice, stream); @@ -637,7 +681,7 @@ bool run_q4_K_dense_16warp_parity( enqueue_err = cudaMemsetAsync( dRefStorage, ref_guard_byte, guarded_bytes, stream); } - if (enqueue_err == cudaSuccess && check_production) { + if (enqueue_err == cudaSuccess && check_public_dense) { enqueue_err = cudaMemsetAsync( dProdStorage, prod_guard_byte, guarded_bytes, stream); } @@ -645,6 +689,11 @@ bool run_q4_K_dense_16warp_parity( enqueue_err = cudaMemsetAsync( dGotStorage, got_guard_byte, guarded_bytes, stream); } + if (enqueue_err == cudaSuccess) { + enqueue_err = cudaMemsetAsync( + dScratchStorage, scratch_guard_byte, scratch_guarded_bytes, + stream); + } const int rc_quant = enqueue_err == cudaSuccess ? ds4_mmq_q4_K_quantize_q8_1_for_test( @@ -653,28 +702,43 @@ bool run_q4_K_dense_16warp_parity( const int rc_ref = rc_quant == 0 ? ds4_mmq_q4_K_dense_preq_reference_for_test( dW, dQ8, q8_bytes, dRef, M, N, K, - /*use_stream_k=*/0, stream) + /*use_stream_k=*/1, + dScratch, scratch_bytes, + stream) : -100; - const int rc_prod = rc_ref != 0 ? -100 : check_production - ? ds4_mmq_q4_K_dense_preq_reference_for_test( - dW, dQ8, q8_bytes, dProd, M, N, K, - /*use_stream_k=*/1, stream) + std::vector ref_scratch_guards(2u * guard_bytes); + if (enqueue_err == cudaSuccess && rc_ref == 0) { + enqueue_err = enqueue_scratch_guard_copy( + dScratchStorage, scratch_bytes, guard_bytes, + ref_scratch_guards.data(), stream); + } + const int rc_prod = rc_ref != 0 || enqueue_err != cudaSuccess + ? -100 : check_public_dense + ? ds4_mmq_q4_K_dense( + dW, dX, dProd, M, N, K, stream) : 0; const int rc_got = rc_prod == 0 - ? ds4_mmq_q4_K_dense_preq_16warp_for_test( - dW, dQ8, q8_bytes, dGot, M, N, K, stream) + ? ds4_mmq_q4_K_dense_16warp_streamk_enqueue( + dW, dQ8, dGot, dScratch, scratch_bytes, + M, N, K, nsm, stream) : -100; + std::vector got_scratch_guards(2u * guard_bytes); + if (enqueue_err == cudaSuccess && rc_got == 0) { + enqueue_err = enqueue_scratch_guard_copy( + dScratchStorage, scratch_bytes, guard_bytes, + got_scratch_guards.data(), stream); + } std::vector ref_guarded(guarded_bytes); std::vector prod_guarded( - check_production ? guarded_bytes : 0u); + check_public_dense ? guarded_bytes : 0u); std::vector got_guarded(guarded_bytes); if (rc_got == 0) { enqueue_err = cudaMemcpyAsync( ref_guarded.data(), dRefStorage, guarded_bytes, cudaMemcpyDeviceToHost, stream); } - if (enqueue_err == cudaSuccess && rc_got == 0 && check_production) { + if (enqueue_err == cudaSuccess && rc_got == 0 && check_public_dense) { enqueue_err = cudaMemcpyAsync( prod_guarded.data(), dProdStorage, guarded_bytes, cudaMemcpyDeviceToHost, stream); @@ -703,7 +767,7 @@ bool run_q4_K_dense_16warp_parity( ref_guarded.data() + guard_bytes + i * sizeof(float); const uint8_t *const got_bits_ptr = got_guarded.data() + guard_bytes + i * sizeof(float); - const uint8_t *const prod_bits_ptr = check_production + const uint8_t *const prod_bits_ptr = check_public_dense ? prod_guarded.data() + guard_bytes + i * sizeof(float) : nullptr; float ref_value = 0.0f; @@ -711,11 +775,11 @@ bool run_q4_K_dense_16warp_parity( float got_value = 0.0f; std::memcpy(&ref_value, ref_bits_ptr, sizeof(ref_value)); std::memcpy(&got_value, got_bits_ptr, sizeof(got_value)); - if (check_production) { + if (check_public_dense) { std::memcpy(&prod_value, prod_bits_ptr, sizeof(prod_value)); } if (!std::isfinite(ref_value)) nonfinite_ref++; - if (check_production && !std::isfinite(prod_value)) { + if (check_public_dense && !std::isfinite(prod_value)) { nonfinite_prod++; } if (!std::isfinite(got_value)) nonfinite_got++; @@ -729,7 +793,7 @@ bool run_q4_K_dense_16warp_parity( } ref_got_mismatches++; } - if (check_production && + if (check_public_dense && std::memcmp(ref_bits_ptr, prod_bits_ptr, sizeof(float)) != 0) { if (first_ref_prod_bad == SIZE_MAX) { first_ref_prod_bad = i; @@ -738,7 +802,7 @@ bool run_q4_K_dense_16warp_parity( } ref_prod_mismatches++; } - if (check_production && + if (check_public_dense && std::memcmp(prod_bits_ptr, got_bits_ptr, sizeof(float)) != 0) { prod_got_mismatches++; } @@ -757,9 +821,13 @@ bool run_q4_K_dense_16warp_parity( return bad; }; const size_t ref_canary = guard_mismatches(ref_guarded, ref_guard_byte); - const size_t prod_canary = check_production + const size_t prod_canary = check_public_dense ? guard_mismatches(prod_guarded, prod_guard_byte) : 0; const size_t got_canary = guard_mismatches(got_guarded, got_guard_byte); + const size_t ref_scratch_canary = + scratch_guard_mismatches(ref_scratch_guards, scratch_guard_byte); + const size_t got_scratch_canary = + scratch_guard_mismatches(got_scratch_guards, scratch_guard_byte); int rc_reject = DS4_MMQ_NOT_APPLICABLE; size_t reject_writes = 0; @@ -791,24 +859,27 @@ bool run_q4_K_dense_16warp_parity( const bool rejection_ok = !check_rejection || (rc_reject == DS4_MMQ_NOT_APPLICABLE && reject_sync == cudaSuccess && reject_writes == 0); - const bool production_ok = !check_production || + const bool production_ok = !check_public_dense || (ref_prod_mismatches == 0 && prod_got_mismatches == 0 && nonfinite_prod == 0 && prod_canary == 0); const bool ok = rc_quant == 0 && rc_ref == 0 && rc_prod == 0 && rc_got == 0 && enqueue_err == cudaSuccess && sync_err == cudaSuccess && ref_got_mismatches == 0 && nonfinite_ref == 0 && nonfinite_got == 0 && ref_canary == 0 && - got_canary == 0 && production_ok && rejection_ok; + got_canary == 0 && ref_scratch_canary == 0 && + got_scratch_canary == 0 && production_ok && rejection_ok; fprintf(stderr, - "quant/ref/stream/16w=%d/%d/%d/%d enqueue=%s sync=%s " - "bits(ref-16w/ref-stream/stream-16w)=%zu/%zu/%zu " + "quant/ref/production/16w=%d/%d/%d/%d enqueue=%s sync=%s " + "bits(ref-16w/ref-production/production-16w)=%zu/%zu/%zu " "nonfinite=%zu/%zu/%zu canary=%zu/%zu/%zu " - "reject=%d/%zu: %s\n", + "scratch_canary(ref/16w)=%zu/%zu reject=%d/%zu: %s\n", rc_quant, rc_ref, rc_prod, rc_got, cudaGetErrorString(enqueue_err), cudaGetErrorString(sync_err), ref_got_mismatches, ref_prod_mismatches, prod_got_mismatches, nonfinite_ref, nonfinite_prod, nonfinite_got, - ref_canary, prod_canary, got_canary, rc_reject, reject_writes, + ref_canary, prod_canary, got_canary, + ref_scratch_canary, got_scratch_canary, + rc_reject, reject_writes, ok ? "PASS" : "FAIL"); if (first_ref_got_bad != SIZE_MAX) { fprintf(stderr, @@ -822,8 +893,8 @@ bool run_q4_K_dense_16warp_parity( first_ref_prod_bad * sizeof(float); std::memcpy(&ref_bits, ptr, sizeof(ref_bits)); fprintf(stderr, - "first ref/stream mismatch at output[%zu]: " - "ref=0x%08x stream=0x%08x\n", + "first ref/production mismatch at output[%zu]: " + "ref=0x%08x production=0x%08x\n", first_ref_prod_bad, ref_bits, first_prod_bits); } fputc('\n', stderr); @@ -831,6 +902,424 @@ bool run_q4_K_dense_16warp_parity( return ok; } +// Kernel-only Q-A/KV pair oracle. Quantize X exactly once, then compare two +// canonical stream-K launches against two 16-warp stream-K launches over the +// same immutable Q8_1 DS4 buffer and the real required production pair API. +// Keeping every output independently guarded catches a bad M0/M1 stride or a +// cross-leg overwrite as well as a numerical mismatch. +bool run_q4_K_dense_pair_16warp_parity( + int M0, int M1, int N, int K, int nsm, uint32_t seed) { + fprintf(stderr, + "=== Q4_K/DENSE_PAIR_16WARP M0=%d M1=%d N=%d K=%d seed=%u ===\n", + M0, M1, N, K, seed); + + if (M0 <= 0 || M1 <= 0 || N <= 0 || K <= 0 || nsm <= 0 || + K % QK_K_LOCAL != 0 || + (size_t)M0 > SIZE_MAX / (size_t)N / sizeof(float) || + (size_t)M1 > SIZE_MAX / (size_t)N / sizeof(float)) { + fprintf(stderr, "invalid 16-warp pair parity shape\n\n"); + return false; + } + + std::mt19937 rng(seed); + std::normal_distribution nd(0.0f, 1.0f); + const int blocks_per_row = K / QK_K_LOCAL; + std::vector W0((size_t)M0 * blocks_per_row); + std::vector W1((size_t)M1 * blocks_per_row); + for (auto &blk : W0) generate_random_block_q4_K(&blk, rng); + for (auto &blk : W1) generate_random_block_q4_K(&blk, rng); + std::vector X((size_t)N * K); + for (float &v : X) v = nd(rng); + + const size_t q8_bytes = ds4_mmq_q4_K_q8_1_scratch_bytes(N, K); + if (q8_bytes == 0) { + fprintf(stderr, "Q4_K 16-warp pair scratch-size query rejected shape\n\n"); + return false; + } + + constexpr size_t guard_floats = 64; + constexpr size_t guard_bytes = guard_floats * sizeof(float); + constexpr uint8_t ref0_guard_byte = 0xa5; + constexpr uint8_t ref1_guard_byte = 0xb6; + constexpr uint8_t got0_guard_byte = 0x5a; + constexpr uint8_t got1_guard_byte = 0x69; + constexpr uint8_t prod0_guard_byte = 0xc3; + constexpr uint8_t prod1_guard_byte = 0xd4; + constexpr uint8_t reject_guard_byte = 0x3c; + constexpr uint8_t scratch_guard_byte = 0x7d; + const size_t count0 = (size_t)M0 * N; + const size_t count1 = (size_t)M1 * N; + const size_t bytes0 = count0 * sizeof(float); + const size_t bytes1 = count1 * sizeof(float); + const size_t guarded0 = bytes0 + 2u * guard_bytes; + const size_t guarded1 = bytes1 + 2u * guard_bytes; + const size_t scratch0_bytes = + ds4_mmq_q4_K_dense_16warp_streamk_scratch_bytes(M0, N, nsm); + const size_t scratch1_bytes = + ds4_mmq_q4_K_dense_16warp_streamk_scratch_bytes(M1, N, nsm); + const size_t scratch_bytes = + scratch0_bytes > scratch1_bytes ? scratch0_bytes : scratch1_bytes; + if (scratch_bytes > SIZE_MAX - 2u * guard_bytes) { + fprintf(stderr, "Q4_K 16-warp pair scratch guard size overflow\n\n"); + return false; + } + const size_t scratch_guarded_bytes = scratch_bytes + 2u * guard_bytes; + + cudaStream_t stream = nullptr; + void *dW0 = nullptr; + void *dW1 = nullptr; + float *dX = nullptr; + void *dQ8 = nullptr; + float *dRef0Storage = nullptr; + float *dRef1Storage = nullptr; + float *dGot0Storage = nullptr; + float *dGot1Storage = nullptr; + float *dProd0Storage = nullptr; + float *dProd1Storage = nullptr; + void *dScratchStorage = nullptr; + const bool allocated = cudaStreamCreate(&stream) == cudaSuccess && + cudaMalloc(&dW0, W0.size() * sizeof(block_q4_K)) == cudaSuccess && + cudaMalloc(&dW1, W1.size() * sizeof(block_q4_K)) == cudaSuccess && + cudaMalloc(&dX, X.size() * sizeof(float)) == cudaSuccess && + cudaMalloc(&dQ8, q8_bytes) == cudaSuccess && + cudaMalloc(&dRef0Storage, guarded0) == cudaSuccess && + cudaMalloc(&dRef1Storage, guarded1) == cudaSuccess && + cudaMalloc(&dGot0Storage, guarded0) == cudaSuccess && + cudaMalloc(&dGot1Storage, guarded1) == cudaSuccess && + cudaMalloc(&dProd0Storage, guarded0) == cudaSuccess && + cudaMalloc(&dProd1Storage, guarded1) == cudaSuccess && + cudaMalloc(&dScratchStorage, scratch_guarded_bytes) == cudaSuccess; + const auto cleanup = [&]() { + if (dScratchStorage) cudaFree(dScratchStorage); + if (dProd1Storage) cudaFree(dProd1Storage); + if (dProd0Storage) cudaFree(dProd0Storage); + if (dGot1Storage) cudaFree(dGot1Storage); + if (dGot0Storage) cudaFree(dGot0Storage); + if (dRef1Storage) cudaFree(dRef1Storage); + if (dRef0Storage) cudaFree(dRef0Storage); + if (dQ8) cudaFree(dQ8); + if (dX) cudaFree(dX); + if (dW1) cudaFree(dW1); + if (dW0) cudaFree(dW0); + if (stream) cudaStreamDestroy(stream); + }; + if (!allocated) { + fprintf(stderr, "Q4_K 16-warp pair allocation failed: %s\n\n", + cudaGetErrorString(cudaGetLastError())); + cleanup(); + return false; + } + + float *const dRef0 = dRef0Storage + guard_floats; + float *const dRef1 = dRef1Storage + guard_floats; + float *const dGot0 = dGot0Storage + guard_floats; + float *const dGot1 = dGot1Storage + guard_floats; + float *const dProd0 = dProd0Storage + guard_floats; + float *const dProd1 = dProd1Storage + guard_floats; + void *const dScratch = + static_cast(dScratchStorage) + guard_bytes; + cudaError_t enqueue_err = cudaMemcpyAsync( + dW0, W0.data(), W0.size() * sizeof(block_q4_K), + cudaMemcpyHostToDevice, stream); + if (enqueue_err == cudaSuccess) { + enqueue_err = cudaMemcpyAsync( + dW1, W1.data(), W1.size() * sizeof(block_q4_K), + cudaMemcpyHostToDevice, stream); + } + if (enqueue_err == cudaSuccess) { + enqueue_err = cudaMemcpyAsync( + dX, X.data(), X.size() * sizeof(float), + cudaMemcpyHostToDevice, stream); + } + if (enqueue_err == cudaSuccess) { + enqueue_err = cudaMemsetAsync( + dRef0Storage, ref0_guard_byte, guarded0, stream); + } + if (enqueue_err == cudaSuccess) { + enqueue_err = cudaMemsetAsync( + dRef1Storage, ref1_guard_byte, guarded1, stream); + } + if (enqueue_err == cudaSuccess) { + enqueue_err = cudaMemsetAsync( + dGot0Storage, got0_guard_byte, guarded0, stream); + } + if (enqueue_err == cudaSuccess) { + enqueue_err = cudaMemsetAsync( + dGot1Storage, got1_guard_byte, guarded1, stream); + } + if (enqueue_err == cudaSuccess) { + enqueue_err = cudaMemsetAsync( + dProd0Storage, prod0_guard_byte, guarded0, stream); + } + if (enqueue_err == cudaSuccess) { + enqueue_err = cudaMemsetAsync( + dProd1Storage, prod1_guard_byte, guarded1, stream); + } + if (enqueue_err == cudaSuccess) { + enqueue_err = cudaMemsetAsync( + dScratchStorage, scratch_guard_byte, scratch_guarded_bytes, + stream); + } + + const int rc_quant = enqueue_err == cudaSuccess + ? ds4_mmq_q4_K_quantize_q8_1_for_test( + dX, dQ8, q8_bytes, N, K, stream) + : -100; + const int rc_ref0 = rc_quant == 0 + ? ds4_mmq_q4_K_dense_preq_reference_for_test( + dW0, dQ8, q8_bytes, dRef0, M0, N, K, + /*use_stream_k=*/1, + dScratch, scratch_bytes, + stream) + : -100; + const int rc_ref1 = rc_ref0 == 0 + ? ds4_mmq_q4_K_dense_preq_reference_for_test( + dW1, dQ8, q8_bytes, dRef1, M1, N, K, + /*use_stream_k=*/1, + dScratch, scratch_bytes, + stream) + : -100; + std::vector ref_scratch_guards(2u * guard_bytes); + if (enqueue_err == cudaSuccess && rc_ref1 == 0) { + enqueue_err = enqueue_scratch_guard_copy( + dScratchStorage, scratch_bytes, guard_bytes, + ref_scratch_guards.data(), stream); + } + const int rc_got0 = rc_ref1 == 0 && enqueue_err == cudaSuccess + ? ds4_mmq_q4_K_dense_16warp_streamk_enqueue( + dW0, dQ8, dGot0, dScratch, scratch_bytes, + M0, N, K, nsm, stream) + : -100; + const int rc_got1 = rc_got0 == 0 + ? ds4_mmq_q4_K_dense_16warp_streamk_enqueue( + dW1, dQ8, dGot1, dScratch, scratch_bytes, + M1, N, K, nsm, stream) + : -100; + std::vector got_scratch_guards(2u * guard_bytes); + if (enqueue_err == cudaSuccess && rc_got1 == 0) { + enqueue_err = enqueue_scratch_guard_copy( + dScratchStorage, scratch_bytes, guard_bytes, + got_scratch_guards.data(), stream); + } + const int rc_pair = rc_got1 == 0 && enqueue_err == cudaSuccess + ? ds4_mmq_q4_K_dense_pair( + dW0, dW1, dX, dProd0, dProd1, M0, M1, N, K, stream) + : -100; + + std::vector ref0_guarded(guarded0); + std::vector ref1_guarded(guarded1); + std::vector got0_guarded(guarded0); + std::vector got1_guarded(guarded1); + std::vector prod0_guarded(guarded0); + std::vector prod1_guarded(guarded1); + if (rc_pair == 0) { + enqueue_err = cudaMemcpyAsync( + ref0_guarded.data(), dRef0Storage, guarded0, + cudaMemcpyDeviceToHost, stream); + } + if (enqueue_err == cudaSuccess && rc_pair == 0) { + enqueue_err = cudaMemcpyAsync( + ref1_guarded.data(), dRef1Storage, guarded1, + cudaMemcpyDeviceToHost, stream); + } + if (enqueue_err == cudaSuccess && rc_pair == 0) { + enqueue_err = cudaMemcpyAsync( + got0_guarded.data(), dGot0Storage, guarded0, + cudaMemcpyDeviceToHost, stream); + } + if (enqueue_err == cudaSuccess && rc_pair == 0) { + enqueue_err = cudaMemcpyAsync( + got1_guarded.data(), dGot1Storage, guarded1, + cudaMemcpyDeviceToHost, stream); + } + if (enqueue_err == cudaSuccess && rc_pair == 0) { + enqueue_err = cudaMemcpyAsync( + prod0_guarded.data(), dProd0Storage, guarded0, + cudaMemcpyDeviceToHost, stream); + } + if (enqueue_err == cudaSuccess && rc_pair == 0) { + enqueue_err = cudaMemcpyAsync( + prod1_guarded.data(), dProd1Storage, guarded1, + cudaMemcpyDeviceToHost, stream); + } + const cudaError_t sync_err = cudaStreamSynchronize(stream); + + struct leg_result { + size_t mismatches = 0; + size_t nonfinite_ref = 0; + size_t nonfinite_got = 0; + size_t ref_canary = 0; + size_t got_canary = 0; + size_t first_bad = SIZE_MAX; + uint32_t first_ref_bits = 0; + uint32_t first_got_bits = 0; + }; + const auto inspect_leg = [&](const std::vector &ref, + const std::vector &got, + size_t count, size_t output_bytes, + uint8_t expected_ref, + uint8_t expected_got) { + leg_result result; + if (enqueue_err != cudaSuccess || sync_err != cudaSuccess || + rc_pair != 0) { + return result; + } + for (size_t i = 0; i < count; ++i) { + const uint8_t *const ref_ptr = + ref.data() + guard_bytes + i * sizeof(float); + const uint8_t *const got_ptr = + got.data() + guard_bytes + i * sizeof(float); + float ref_value = 0.0f; + float got_value = 0.0f; + std::memcpy(&ref_value, ref_ptr, sizeof(ref_value)); + std::memcpy(&got_value, got_ptr, sizeof(got_value)); + if (!std::isfinite(ref_value)) result.nonfinite_ref++; + if (!std::isfinite(got_value)) result.nonfinite_got++; + if (std::memcmp(ref_ptr, got_ptr, sizeof(float)) != 0) { + if (result.first_bad == SIZE_MAX) { + result.first_bad = i; + std::memcpy(&result.first_ref_bits, ref_ptr, + sizeof(result.first_ref_bits)); + std::memcpy(&result.first_got_bits, got_ptr, + sizeof(result.first_got_bits)); + } + result.mismatches++; + } + } + for (size_t i = 0; i < guard_bytes; ++i) { + if (ref[i] != expected_ref) result.ref_canary++; + if (got[i] != expected_got) result.got_canary++; + } + for (size_t i = guard_bytes + output_bytes; i < ref.size(); ++i) { + if (ref[i] != expected_ref) result.ref_canary++; + } + for (size_t i = guard_bytes + output_bytes; i < got.size(); ++i) { + if (got[i] != expected_got) result.got_canary++; + } + return result; + }; + + const leg_result leg0 = inspect_leg( + ref0_guarded, got0_guarded, count0, bytes0, + ref0_guard_byte, got0_guard_byte); + const leg_result leg1 = inspect_leg( + ref1_guarded, got1_guarded, count1, bytes1, + ref1_guard_byte, got1_guard_byte); + const leg_result prod0 = inspect_leg( + ref0_guarded, prod0_guarded, count0, bytes0, + ref0_guard_byte, prod0_guard_byte); + const leg_result prod1 = inspect_leg( + ref1_guarded, prod1_guarded, count1, bytes1, + ref1_guard_byte, prod1_guard_byte); + const size_t ref_scratch_canary = + scratch_guard_mismatches(ref_scratch_guards, scratch_guard_byte); + const size_t got_scratch_canary = + scratch_guard_mismatches(got_scratch_guards, scratch_guard_byte); + + // REQUIRE must reject the whole pair before either leg can enqueue. A + // 384-row second leg is aligned but below the pair admission floor; poison + // both complete guarded ranges so even a partial first-leg launch is seen. + int rc_reject = -100; + size_t reject_writes = SIZE_MAX; + cudaError_t reject_sync = cudaSuccess; + if (enqueue_err == cudaSuccess && sync_err == cudaSuccess && rc_pair == 0) { + cudaError_t reject_err = cudaMemsetAsync( + dProd0Storage, reject_guard_byte, guarded0, stream); + if (reject_err == cudaSuccess) { + reject_err = cudaMemsetAsync( + dProd1Storage, reject_guard_byte, guarded1, stream); + } + rc_reject = reject_err == cudaSuccess + ? ds4_mmq_q4_K_dense_pair( + dW0, dW1, dX, dProd0, dProd1, + M0, /*M1=*/384, N, K, stream) + : -100; + std::vector rejected0(guarded0); + std::vector rejected1(guarded1); + if (reject_err == cudaSuccess) { + reject_err = cudaMemcpyAsync( + rejected0.data(), dProd0Storage, guarded0, + cudaMemcpyDeviceToHost, stream); + } + if (reject_err == cudaSuccess) { + reject_err = cudaMemcpyAsync( + rejected1.data(), dProd1Storage, guarded1, + cudaMemcpyDeviceToHost, stream); + } + reject_sync = cudaStreamSynchronize(stream); + if (reject_err == cudaSuccess && reject_sync == cudaSuccess) { + reject_writes = 0; + for (uint8_t byte : rejected0) { + if (byte != reject_guard_byte) reject_writes++; + } + for (uint8_t byte : rejected1) { + if (byte != reject_guard_byte) reject_writes++; + } + } + } + const bool rejection_ok = + rc_reject == DS4_MMQ_NOT_APPLICABLE && + reject_sync == cudaSuccess && reject_writes == 0; + const bool ok = rc_quant == 0 && rc_ref0 == 0 && rc_ref1 == 0 && + rc_got0 == 0 && rc_got1 == 0 && rc_pair == 0 && + enqueue_err == cudaSuccess && + sync_err == cudaSuccess && leg0.mismatches == 0 && + leg1.mismatches == 0 && prod0.mismatches == 0 && + prod1.mismatches == 0 && leg0.nonfinite_ref == 0 && + leg0.nonfinite_got == 0 && leg1.nonfinite_ref == 0 && + leg1.nonfinite_got == 0 && prod0.nonfinite_got == 0 && + prod1.nonfinite_got == 0 && leg0.ref_canary == 0 && + leg0.got_canary == 0 && leg1.ref_canary == 0 && + leg1.got_canary == 0 && prod0.got_canary == 0 && + prod1.got_canary == 0 && ref_scratch_canary == 0 && + got_scratch_canary == 0 && rejection_ok; + fprintf(stderr, + "quant/ref0/ref1/16w0/16w1/pair=%d/%d/%d/%d/%d/%d " + "enqueue=%s sync=%s bits(raw/prod)=%zu/%zu,%zu/%zu " + "nonfinite(ref/raw/prod)=%zu/%zu/%zu,%zu/%zu/%zu " + "canary(ref/raw/prod)=%zu/%zu/%zu,%zu/%zu/%zu " + "scratch_canary(ref/16w)=%zu/%zu " + "reject=%d/%zu reject_sync=%s: %s\n", + rc_quant, rc_ref0, rc_ref1, rc_got0, rc_got1, rc_pair, + cudaGetErrorString(enqueue_err), cudaGetErrorString(sync_err), + leg0.mismatches, prod0.mismatches, + leg1.mismatches, prod1.mismatches, + leg0.nonfinite_ref, leg0.nonfinite_got, prod0.nonfinite_got, + leg1.nonfinite_ref, leg1.nonfinite_got, prod1.nonfinite_got, + leg0.ref_canary, leg0.got_canary, prod0.got_canary, + leg1.ref_canary, leg1.got_canary, prod1.got_canary, + ref_scratch_canary, got_scratch_canary, + rc_reject, reject_writes, cudaGetErrorString(reject_sync), + ok ? "PASS" : "FAIL"); + if (leg0.first_bad != SIZE_MAX) { + fprintf(stderr, + "first pair leg0 mismatch at output[%zu]: " + "ref=0x%08x got=0x%08x\n", + leg0.first_bad, leg0.first_ref_bits, leg0.first_got_bits); + } + if (leg1.first_bad != SIZE_MAX) { + fprintf(stderr, + "first pair leg1 mismatch at output[%zu]: " + "ref=0x%08x got=0x%08x\n", + leg1.first_bad, leg1.first_ref_bits, leg1.first_got_bits); + } + if (prod0.first_bad != SIZE_MAX) { + fprintf(stderr, + "first pair production leg0 mismatch at output[%zu]: " + "ref=0x%08x got=0x%08x\n", + prod0.first_bad, prod0.first_ref_bits, prod0.first_got_bits); + } + if (prod1.first_bad != SIZE_MAX) { + fprintf(stderr, + "first pair production leg1 mismatch at output[%zu]: " + "ref=0x%08x got=0x%08x\n", + prod1.first_bad, prod1.first_ref_bits, prod1.first_got_bits); + } + fputc('\n', stderr); + cleanup(); + return ok; +} + // Prefill dense-pair verifier. The candidate shares only the canonical // token-tiled Q8_1 activation; both weight legs still run the ordinary Q4_K // MMQ kernel, so their outputs must match two independent dense calls bitwise. @@ -2725,15 +3214,30 @@ bool run_q4_K_grouped_vec_parity( } // namespace int main(int argc, char ** argv) { + const bool q4_16warp_oracle = + argc == 2 && std::strcmp(argv[1], "--q4-16warp") == 0; + scoped_env_override require_16warp( + "DS4_CUDA_REQUIRE_Q4_MMQ_16WARP"); + scoped_env_override disable_16warp( + "DS4_CUDA_NO_Q4_MMQ_16WARP"); + // Production dense/pair oracles must fail closed if any leg falls back. + // Set the required mode before initialization can observe its process-wide + // cache, and neutralize an inherited rollback request. + if (q4_16warp_oracle && + (!require_16warp.set("1") || !disable_16warp.set("0"))) { + fprintf(stderr, "Q4 16-warp oracle environment setup failed\n"); + return 1; + } + int rc = ds4_mmq_init(0); if (rc != 0) { fprintf(stderr, "ds4_mmq_init failed: %d\n", rc); return 1; } bool all_ok = true; - if (argc == 2 && std::strcmp(argv[1], "--q4-16warp") == 0) { - // The production arm must select mmq_x=128 for the no-fixup proof. - // Mirror get_mmq_x_max_host's numeric-prefix parsing and reject a - // narrower experiment override before MMQ caches it. + if (q4_16warp_oracle) { + // The canonical production baseline must select mmq_x=128. Mirror + // get_mmq_x_max_host's numeric-prefix parsing and reject a narrower + // experiment override before MMQ caches it. const char *const mmq_x_env = std::getenv("DS4_CUDA_MMQ_X_MAX"); if (mmq_x_env && mmq_x_env[0]) { char *end = nullptr; @@ -2758,6 +3262,35 @@ int main(int argc, char ** argv) { cudaGetErrorString(prop_err)); return 1; } + const int cc = prop.major * 100 + prop.minor * 10; + if (!ds4_mmq_q4_K_dense_16warp_available(cc)) { + fprintf(stderr, + "Q4 16-warp oracle unsupported on cc=%d.%d\n", + prop.major, prop.minor); + return 77; + } + const int prepare_rc = ds4_mmq_q4_K_dense_16warp_prepare(); + if (prepare_rc != 0) { + fprintf(stderr, + "Q4 16-warp oracle prepare failed: %d\n", prepare_rc); + return 1; + } + + const auto grid_efficiency = [&](int M, int N) { + const int64_t tiles_m = ((int64_t)M + 127) / 128; + const int64_t tiles_n = ((int64_t)N + 127) / 128; + const int64_t tiles = tiles_m * tiles_n; + const int64_t waves = + (tiles + prop.multiProcessorCount - 1) / + prop.multiProcessorCount; + return (int)(100 * tiles / + ((int64_t)prop.multiProcessorCount * waves)); + }; + const int dense_4096_eff = grid_efficiency(1024, 4096); + const int kv_4096_eff = grid_efficiency(512, 4096); + const bool public_dense_4096 = dense_4096_eff >= 80; + const bool public_pair_4096 = + public_dense_4096 && kv_4096_eff >= 80; // N=512 and mmq_x=128 give four N tiles. Choose the smallest M-tile // count >=16 for which 4*tiles_m is divisible by nSM. Stream-K then @@ -2777,23 +3310,59 @@ int main(int argc, char ** argv) { return 1; } const int no_fixup_M = (int)(128 * tiles_m); + fprintf(stderr, "Q4 16-warp no-fixup geometry: nsm=%d tiles_m=%lld " "tiles=%lld M=%d\n", prop.multiProcessorCount, (long long)tiles_m, (long long)(4 * tiles_m), no_fixup_M); - // Exact N tile with a device-dependent, provably no-fixup M, plus an - // N-tail case at the top of the K envelope. Keep the larger K=4096 - // case at two arms so the focused oracle remains reasonably small. + // Cover a device-dependent no-fixup baseline, an N tail, and the real + // 4096-token production dense/pair envelope. all_ok &= run_q4_K_dense_16warp_parity( - /*M=*/no_fixup_M, /*N=*/512, /*K=*/1024, 0xC4160001u, - /*check_production=*/true, + /*M=*/no_fixup_M, /*N=*/512, /*K=*/1024, + prop.multiProcessorCount, 0xC4160001u, + /*check_public_dense=*/false, /*check_rejection=*/true); all_ok &= run_q4_K_dense_16warp_parity( - /*M=*/2176, /*N=*/513, /*K=*/4096, 0xC4160002u, - /*check_production=*/false, + // N=601 keeps the canonical selector on m128n128 while retaining + // an N-tail, so external-scratch Stream-K remains bit-comparable. + /*M=*/2176, /*N=*/601, /*K=*/4096, + prop.multiProcessorCount, 0xC4160002u, + /*check_public_dense=*/false, + /*check_rejection=*/false); + // Production Q-A and Q-A/KV-pair envelope: exercise the newly admitted + // M=1024 dense leg at a full 4096-token context, then validate the + // asymmetric 1024+512 pair with one shared Q8_1 activation. + all_ok &= run_q4_K_dense_16warp_parity( + /*M=*/1024, /*N=*/4096, /*K=*/4096, + prop.multiProcessorCount, 0xC4160003u, + /*check_public_dense=*/public_dense_4096, /*check_rejection=*/false); + if (!public_dense_4096) { + fprintf(stderr, + "Q4 16-warp public dense N=4096 SKIP: grid efficiency " + "%d%% < 80%% (nsm=%d)\n", + dense_4096_eff, prop.multiProcessorCount); + } + if (public_pair_4096) { + all_ok &= run_q4_K_dense_pair_16warp_parity( + /*M0=*/1024, /*M1=*/512, /*N=*/4096, /*K=*/4096, + prop.multiProcessorCount, 0xC4160004u); + } else { + // Keep raw coverage of the pair-only 512-row leg even when this + // device's SM geometry makes the required public pair ineligible. + all_ok &= run_q4_K_dense_16warp_parity( + /*M=*/512, /*N=*/4096, /*K=*/4096, + prop.multiProcessorCount, 0xC4160005u, + /*check_public_dense=*/false, + /*check_rejection=*/false); + fprintf(stderr, + "Q4 16-warp public pair N=4096 SKIP: grid efficiency " + "dense=%d%% kv=%d%% (need both >=80%%, nsm=%d)\n", + dense_4096_eff, kv_4096_eff, + prop.multiProcessorCount); + } fprintf(stderr, "===================\n"); fprintf(stderr, "Q4 16-WARP %s\n", all_ok ? "PASS" : "FAILED"); return all_ok ? 0 : 1; diff --git a/ds4_cuda.cu b/ds4_cuda.cu index 499acf234..d60558e83 100644 --- a/ds4_cuda.cu +++ b/ds4_cuda.cu @@ -40366,6 +40366,8 @@ static int cuda_matmul_q4_K_pair_tensor_impl( const int gb10_canonical = cuda_q4_gb10_fast_path_enabled( logical_tier, "DS4_CUDA_DISABLE_Q4_DENSE_PAIR"); + const int require_q4_16warp = n_tok > 8u && cuda_env_flag_enabled( + "DS4_CUDA_REQUIRE_Q4_MMQ_16WARP", 0); if (cuda_use_mmq()) { /* Share one canonical Q8_1 activation across both projections: * MMVQ covers decode/speculative widths and token-tiled MMQ covers @@ -40393,14 +40395,27 @@ static int cuda_matmul_q4_K_pair_tensor_impl( (unsigned long long)out1_dim, (unsigned long long)n_tok, rc == DS4_MMQ_NOT_APPLICABLE - ? (g_cuda_test_q4_mmq_strict + ? (require_q4_16warp + ? "; required 16-warp path rejects fallback" + : g_cuda_test_q4_mmq_strict ? "; strict benchmark mode rejects fallback" : "; falling back") : "; attempted path failed closed"); if (rc != DS4_MMQ_NOT_APPLICABLE) return -1; + if (require_q4_16warp) return -1; if (g_cuda_test_q4_mmq_strict) return -1; if (gb10_canonical) return 0; } + if (require_q4_16warp) { + fprintf(stderr, + "ds4: required Q4 16-warp prefill found no pair MMQ " + "dispatch (in=%llu out0=%llu out1=%llu n_tok=%llu)\n", + (unsigned long long)in_dim, + (unsigned long long)out0_dim, + (unsigned long long)out1_dim, + (unsigned long long)n_tok); + return -1; + } if (g_cuda_test_q4_mmq_strict) { fprintf(stderr, "ds4: Q4_K pair strict benchmark mode found no MMQ " diff --git a/gguf-tools/Makefile b/gguf-tools/Makefile index 37d5d2cda..53357faf9 100644 --- a/gguf-tools/Makefile +++ b/gguf-tools/Makefile @@ -32,10 +32,10 @@ endif NVCCFLAGS ?= -O3 --use_fast_math $(NVCC_ARCH_FLAGS) -Xcompiler $(NATIVE_CPU_FLAG) -Xcompiler -pthread QUALITY_LDLIBS ?= -lm -Xcompiler -pthread -L$(CUDA_HOME)/targets/sbsa-linux/lib -L$(CUDA_HOME)/lib64 -lcudart -lcublas QUALITY_TARGETS := ds4.o ds4_cuda.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o ds4_image.o rax.o ds4_gpu_args.o \ - cuda/mmq/ds4_ggml_stubs.o cuda/mmq/ds4_mmq.o cuda/mmq/ds4_mmq_d2r.o \ + cuda/mmq/ds4_ggml_stubs.o cuda/mmq/ds4_mmq.o cuda/mmq/ds4_mmq_d2r.o cuda/mmq/ds4_mmq_q4_16warp.o \ cuda/mmq/quantize.o cuda/mmq/mmid.o cuda/mmq/mmvq.o cuda/mmq/ds4_repack.o QUALITY_OBJS := ../ds4.o ../ds4_cuda.o ../ds4_distributed.o ../ds4_tp.o ../ds4_ssd.o ../ds4_layer_pack.o ../ds4_image.o ../rax.o ../ds4_gpu_args.o \ - ../cuda/mmq/ds4_ggml_stubs.o ../cuda/mmq/ds4_mmq.o ../cuda/mmq/ds4_mmq_d2r.o \ + ../cuda/mmq/ds4_ggml_stubs.o ../cuda/mmq/ds4_mmq.o ../cuda/mmq/ds4_mmq_d2r.o ../cuda/mmq/ds4_mmq_q4_16warp.o \ ../cuda/mmq/quantize.o ../cuda/mmq/mmid.o ../cuda/mmq/mmvq.o ../cuda/mmq/ds4_repack.o QUALITY_LINK := $(NVCC) $(NVCCFLAGS) -I.. -o LLAMA_CPP_LDLIBS ?= -L$(LLAMA_CPP_BIN) -Wl,-rpath,$(abspath $(LLAMA_CPP_BIN)) \ diff --git a/scripts/environment_variables.tsv b/scripts/environment_variables.tsv index 74c5560df..8a5ec0134 100644 --- a/scripts/environment_variables.tsv +++ b/scripts/environment_variables.tsv @@ -237,7 +237,7 @@ runtime/cuda DS4_CUDA_NO_Q4_GROUPED_ATTN_A presence kill switch; default unset ( runtime/cuda DS4_CUDA_NO_Q4_GROUPED_ATTN_A_BATCH presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q4 grouped attn a batch CUDA Q4 optimization. cuda/mmq/ds4_mmq.cu:4305 runtime/cuda DS4_CUDA_NO_Q4_GROUPED_ATTN_A_PREFILL presence kill switch; default unset; any defined value including empty or 0 disables and dominates ENABLE/REQUIRE Restore the eight pack/MMQ/unpack Q4 attention-A prefill projections. ds4_cuda.cu:41930 runtime/cuda DS4_CUDA_NO_Q4_K1024_PERSISTENT presence kill switch, default off; any defined value including 0 disables Disable the Q4 K1024 persistent CUDA Q4 optimization. cuda/mmq/ds4_mmq.cu:3907 -runtime/cuda DS4_CUDA_NO_Q4_MMQ_16WARP value-aware rollback, default off; unset/empty/exact 0 permits the experiment, every other nonempty value disables it and overrides REQUEST/REQUIRE Disable the experimental complete-K CUDA Q4_K m128n128 16-warp prefill kernel. cuda/mmq/ds4_mmq.cu:1059 +runtime/cuda DS4_CUDA_NO_Q4_MMQ_16WARP value-aware rollback, default off; unset/empty/exact 0 permits the experiment, every other nonempty value disables it and overrides REQUEST/REQUIRE Disable the experimental Stream-K-compatible CUDA Q4_K m128n128 16-warp prefill kernel. cuda/mmq/ds4_mmq.cu:1133 runtime/cuda DS4_CUDA_NO_Q8_ALIGNED_DENSE_SCRATCH presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q8 aligned dense scratch CUDA Q8 optimization. cuda/mmq/ds4_mmq.cu:5578 runtime/cuda DS4_CUDA_NO_Q8_ALIGNED_PERSISTENT presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q8 aligned persistent CUDA Q8 optimization. cuda/mmq/ds4_mmq.cu:5410 runtime/cuda DS4_CUDA_NO_Q8_BATCH_EXACT_TOK2 presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q8 batch exact tok2 CUDA Q8 optimization. ds4_cuda.cu:19852 @@ -291,7 +291,7 @@ runtime/cuda DS4_CUDA_Q4_ATTN_Q_B_TRANSIENT_F16_MIN_TOKENS full-string unsigned runtime/cuda DS4_CUDA_Q4_GROUPED_ATTN_A_ORACLE value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on Compare grouped attention-A against the canonical per-group result. ds4_cuda.cu:1477 runtime/cuda DS4_CUDA_Q4_K1024_PERSISTENT_ORACLE value-aware flag, default off; nonempty value other than exact 0 enables and implies candidate admission Bitwise-compare the exact-shape persistent Q4 K1024 kernel with canonical MMVQ and retain canonical output. cuda/mmq/ds4_mmq.cu:3738 runtime/cuda DS4_CUDA_Q4_K1024_PERSISTENT_STATS value-aware flag, default off; nonempty value other than exact 0 enables Print exact-shape persistent Q4 K1024 dispatch counters at exit. cuda/mmq/ds4_mmq.cu:3737 -runtime/cuda DS4_CUDA_Q4_MMQ_16WARP value-aware opt-in cached on first Q4_K dense MMQ call; unset/empty/exact 0 is off, every other nonempty value requests the candidate; rollback wins and ineligible shapes fall back Enable the experimental exact-integer CUDA Q4_K m128n128 16-warp kernel for eligible complete-K dense prefills. cuda/mmq/ds4_mmq.cu:1052 +runtime/cuda DS4_CUDA_Q4_MMQ_16WARP value-aware opt-in cached on the first Q4_K dense or dense-pair MMQ call; unset/empty/exact 0 is off, every other nonempty value requests the candidate; rollback wins; standalone dense requires M>=1024 while dense-pair admits legs down to M=512 and shares one Q8_1 activation; grids require at least 80% whole-tile SM-wave efficiency and use the canonical Stream-K partition/fixup below its 90% cutoff; ineligible optional shapes fall back Enable the experimental exact-integer CUDA Q4_K m128n128 16-warp kernel for eligible dense and dense-pair prefills without changing the canonical FP32 reduction tree. cuda/mmq/ds4_mmq.cu:1126 runtime/cuda DS4_CUDA_Q8_F16_ALL presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Control the Q8 F16 all CUDA quantized-matmul/cache optimization. ds4_cuda.cu:2358 runtime/cuda DS4_CUDA_Q8_F16_CACHE_MB unsigned integer MiB, full-string parse; default unlimited; 0 disables this cache Limit the selective Q8-to-F16 derived-weight cache. ds4_cuda.cu:2218 runtime/cuda DS4_CUDA_Q8_F16_CACHE_RESERVE_MB unsigned integer MiB, full-string parse; default is VRAM-dependent (>=112 GiB: 512; >=40 GiB: max(768,1%); smaller: max(4096,5%)) Reserve free VRAM when growing the selective Q8-to-F16 cache. ds4_cuda.cu:2224 @@ -311,7 +311,7 @@ runtime/cuda DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_BATCH value-aware flag, default runtime/cuda DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_PREFILL value-aware fail-closed assertion, default off; unset/empty/exact 0 is off, any other nonempty value requests the candidate and rejects ineligibility before enqueue Require the GB10 grouped Q4_K attention-A prefill path instead of silently using pack/MMQ/unpack. ds4_cuda.cu:41889 runtime/cuda DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_SINGLE_GRID value-aware fail-closed assertion, default off; unset/empty/exact 0 is off, every other nonempty value requests the candidate; DISABLE or ineligibility fails before enqueue Require one grid.z MMQ submission for eligible GB10 grouped Q4_K attention-A prefill instead of falling back to one launch per group. ds4_cuda.cu:41896 runtime/cuda DS4_CUDA_REQUIRE_Q4_K1024_PERSISTENT presence flag, default off; any defined value including 0 makes ineligible candidate fail closed Fail when the exact Q4 K1024 persistent candidate is unavailable instead of using MMVQ. cuda/mmq/ds4_mmq.cu:3929 -runtime/cuda DS4_CUDA_REQUIRE_Q4_MMQ_16WARP value-aware fail-closed prefill opt-in cached on first Q4_K dense MMQ call; unset/empty/exact 0 is off, every other nonempty value requests and requires the candidate for N>8; rollback, disabled MMQ, ineligibility, or preflight failure prevents fallback; decode/speculative N<=8 remains on MMVQ Require the experimental CUDA Q4_K 16-warp prefill kernel so benchmark runs cannot silently measure another path. cuda/mmq/ds4_mmq.cu:1056; ds4_cuda.cu:38208 +runtime/cuda DS4_CUDA_REQUIRE_Q4_MMQ_16WARP value-aware fail-closed prefill opt-in cached on the first Q4_K dense or dense-pair MMQ call; unset/empty/exact 0 is off, every other nonempty value requests and requires the candidate for N>8; a dense-pair is rejected before allocation unless both legs are eligible; rollback, disabled MMQ, ineligibility, or preflight failure prevents fallback; decode/speculative N<=8 remains on MMVQ Require the experimental CUDA Q4_K 16-warp prefill kernel so benchmark runs cannot silently measure another path. cuda/mmq/ds4_mmq.cu:1130; ds4_cuda.cu:38208; ds4_cuda.cu:38358 runtime/cuda DS4_CUDA_REQUIRE_STREAMING_EXPERT_PERSISTENT_CACHE false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Require streaming expert persistent cache in CUDA SSD streaming; fail closed when unavailable. ds4_cuda.cu:4117 runtime/cuda DS4_CUDA_REQUIRE_STREAMING_SELECTED_BATCH_IO false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Require streaming selected batch I/O in CUDA SSD streaming; fail closed when unavailable. ds4_cuda.cu:4738 runtime/cuda DS4_CUDA_REQUIRE_STREAMING_SELECTED_EVENT_PIPELINE false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate Require streaming selected event pipeline in CUDA SSD streaming; fail closed when unavailable. ds4_cuda.cu:4870 diff --git a/speed-bench/README.md b/speed-bench/README.md index eb076aa43..33951fed8 100644 --- a/speed-bench/README.md +++ b/speed-bench/README.md @@ -278,29 +278,51 @@ and then the resident prequantized A/B benchmark: make test-mmq-q4-16warp-cuda CUDA_ARCH=sm_121 make cuda-q4-prefill-bench CUDA_ARCH=sm_121 ./speed-bench/cuda_q4_prefill_bench \ - --path mmq --kernel-16warp --case qb \ - --tokens 512,1024,2048,2049,4096 --sets 4 --samples 16 --warmup 4 + --path mmq --kernel-16warp --case dense \ + --tokens 512,1024,2048,2049,4096,6144,8192 \ + --sets 4 --samples 16 --warmup 4 +./speed-bench/cuda_q4_prefill_bench \ + --path mmq --kernel-16warp --case pair \ + --tokens 512,1024,2048,2049,4096,6144,8192 \ + --sets 4 --samples 16 --warmup 4 ``` The benchmark quantizes X to canonical Q8_1 DS4 once, before timing, and uses -CUDA events around only the complete-K reference GEMM or the raw 16-warp GEMM. -It alternates the two arms ABBA/BAAB over resident Q4_K weight sets, compares -their complete outputs bit-for-bit, checks finite values and canaries, and -samples a CPU Q4_K oracle before and after timing. Results attest +CUDA events around only the production-policy Stream-K reference or the +16-warp kernel with the same partition/fixup policy. Both arms therefore +include fixup whenever the canonical dispatcher would use it, including the +GB10 `M=1024,N=4096` case. One guarded fixup allocation is created before the +samples and reused by both A/B arms and by the two pair legs, so CUDA pool +allocation/free nodes cannot bias the kernel delta. +The dense case is the production Q-A shape `K=4096,M=1024`; the pair case +reuses that same prequantized activation across the asymmetric Q-A/KV shapes +`M0=1024,M1=512` and times two GEMMs in each arm. It alternates the +arms ABBA/BAAB over resident Q4_K weight sets, compares every complete output +bit-for-bit, checks finite values and independent output canaries, and samples +a CPU Q4_K oracle before and after timing. Results attest `timing=kernel_only_prequant`; SSD streaming, model upload, Q8_1 quantization, allocation, host copies, and oracle work are outside the samples. Use -`--case dense` as an exploratory `K=4096,M=1024` datapoint; the initial -production admission gate is deliberately limited to larger, complete -128x128 projections such as q_b. +`--case qb` for the additional `K=1024,M=32768` large-projection datapoint. +The focused test also invokes the real standalone dense and pair dispatchers in +required mode at `N=4096`, so fallback fails instead of producing a misleading +canonical-path pass. A negative pair case verifies that an ineligible 384-row +leg returns `DS4_MMQ_NOT_APPLICABLE` without touching either guarded output. +The CLI accepts contexts through 8192 tokens; the default kernel-only sweep +adds 6144 and 8192 to expose long-context scaling. A custom token tail must +also make the canonical picker select `m128n128`; the harness checks the real +device selector up front and reports `SKIP` instead of running a mismatched +Stream-K partition. The production path remains opt-in until the NVIDIA oracle passes and the paired median is a repeatable win. `DS4_CUDA_Q4_MMQ_16WARP=1` requests it and falls back on ineligible shapes. For an attested production benchmark, `DS4_CUDA_REQUIRE_Q4_MMQ_16WARP=1` also prevents a Q8_K/MMQ fallback; `DS4_CUDA_NO_Q4_MMQ_16WARP=1` is the value-aware rollback. The strict path -requires Ampere or newer, N and M divisible by 128, `M>=2048`, `N>=512`, -`1024<=K<=4096`, the default 128-column MMQ selector, and a canonical tiling -that owns complete K without stream-K fixup. +requires Ampere or newer, complete 128x128 output tiles, `N>=512`, +`1024<=K<=4096`, the default 128-column MMQ selector, and at least 80% final-wave +grid efficiency. Single dense admission starts at `M=1024`; the Q-A/KV pair +admission also accepts its `M1=512` leg when both projections select the +16-warp path. On GB10, isolate the production Flash attention-output A geometry (`groups=8`, `K=4096`, `rank=1024`) and its 127/128/129 token tails with: diff --git a/speed-bench/cuda_q4_prefill_bench.cu b/speed-bench/cuda_q4_prefill_bench.cu index 109173c08..2e82962e3 100644 --- a/speed-bench/cuda_q4_prefill_bench.cu +++ b/speed-bench/cuda_q4_prefill_bench.cu @@ -133,6 +133,75 @@ struct cuda_buffer { cuda_buffer &operator=(const cuda_buffer &) = delete; }; +struct guarded_cuda_buffer { + void *storage = nullptr; + void *ptr = nullptr; + size_t logical_bytes = 0; + uint32_t salt = 0; + cudaError_t status = cudaSuccess; + + guarded_cuda_buffer(size_t bytes, uint32_t guard_salt) + : logical_bytes(bytes), salt(guard_salt) { + if (bytes == 0u) return; + constexpr size_t guard_bytes = kGuardWords * sizeof(uint32_t); + if (bytes > std::numeric_limits::max() - 2u * guard_bytes) { + status = static_cast(1); + return; + } + status = cudaMalloc(&storage, bytes + 2u * guard_bytes); + if (status != cudaSuccess) return; + ptr = static_cast(storage) + guard_bytes; + std::vector prefix(kGuardWords); + std::vector suffix(kGuardWords); + for (uint32_t i = 0; i < kGuardWords; ++i) { + prefix[i] = 0x6b8b4567u ^ salt ^ (i * 0x00010101u); + suffix[i] = 0x327b23c6u ^ salt ^ (i * 0x01000101u); + } + status = cudaMemcpy(storage, prefix.data(), guard_bytes, + cudaMemcpyHostToDevice); + if (status == cudaSuccess) { + status = cudaMemcpy( + static_cast(ptr) + bytes, suffix.data(), + guard_bytes, cudaMemcpyHostToDevice); + } + } + + ~guarded_cuda_buffer() { + if (storage) (void)cudaFree(storage); + } + + bool intact(const char *label) const { + if (logical_bytes == 0u) return true; + constexpr size_t guard_bytes = kGuardWords * sizeof(uint32_t); + std::vector prefix(kGuardWords); + std::vector suffix(kGuardWords); + if (cudaMemcpy(prefix.data(), storage, guard_bytes, + cudaMemcpyDeviceToHost) != cudaSuccess || + cudaMemcpy(suffix.data(), + static_cast(ptr) + logical_bytes, + guard_bytes, cudaMemcpyDeviceToHost) != cudaSuccess) { + std::fprintf(stderr, "%s: scratch guard read failed\n", label); + return false; + } + for (uint32_t i = 0; i < kGuardWords; ++i) { + const uint32_t expected_prefix = + 0x6b8b4567u ^ salt ^ (i * 0x00010101u); + const uint32_t expected_suffix = + 0x327b23c6u ^ salt ^ (i * 0x01000101u); + if (prefix[i] != expected_prefix || suffix[i] != expected_suffix) { + std::fprintf(stderr, + "%s: scratch guard overwritten at word %u\n", + label, i); + return false; + } + } + return true; + } + + guarded_cuda_buffer(const guarded_cuda_buffer &) = delete; + guarded_cuda_buffer &operator=(const guarded_cuda_buffer &) = delete; +}; + struct env_snapshot { const char *name; bool existed; @@ -214,6 +283,23 @@ bool checked_mul(uint64_t a, uint64_t b, uint64_t *out) { return true; } +bool current_device_nsm(int *nsm) { + if (!nsm) return false; + int device = -1; + cudaDeviceProp prop = {}; + const cudaError_t device_err = cudaGetDevice(&device); + const cudaError_t prop_err = device_err == cudaSuccess + ? cudaGetDeviceProperties(&prop, device) : device_err; + if (prop_err != cudaSuccess || prop.multiProcessorCount <= 0) { + std::fprintf(stderr, + "cuda-q4-prefill-bench: CUDA geometry query failed: %s\n", + cudaGetErrorString(prop_err)); + return false; + } + *nsm = prop.multiProcessorCount; + return true; +} + uint32_t lcg_next(uint32_t *state) { *state = *state * 1664525u + 1013904223u; return *state; @@ -652,7 +738,8 @@ const char *path_name(cuda_path path) { const char *case_scope(const config &cfg) { switch (cfg.selected) { case bench_case::all: - return cfg.kernel_16warp ? "dense,q_b" : "dense,pair,q_b,outa"; + return cfg.kernel_16warp ? "dense,pair,q_b" + : "dense,pair,q_b,outa"; case bench_case::dense: return "dense"; case bench_case::pair: return "pair"; case bench_case::qb: return "q_b"; @@ -894,24 +981,33 @@ bool run_q4_16warp_kernel( case_name, n_tokens, in_dim); return false; } + int nsm = 0; + if (!current_device_nsm(&nsm)) return false; + const size_t fixup_bytes = + ds4_mmq_q4_K_dense_16warp_streamk_scratch_bytes( + static_cast(out_dim), static_cast(n_tokens), nsm); tensor_owner x(x_bytes + guard_bytes); tensor_owner reference(out_bytes + guard_bytes); tensor_owner candidate(out_bytes + guard_bytes); cuda_buffer prequant(q8_bytes); + guarded_cuda_buffer fixup(fixup_bytes, 0x1a000u); std::vector activation; fill_activation(&activation, n_tokens, in_dim); if (!x.ptr || !reference.ptr || !candidate.ptr || !prequant.ptr || - prequant.status != cudaSuccess || + prequant.status != cudaSuccess || fixup.status != cudaSuccess || + (fixup_bytes != 0u && !fixup.ptr) || !ds4_gpu_tensor_write(x.ptr, 0, activation.data(), x_bytes) || !prepare_guard(x.ptr, x_bytes, 0x12000u)) { std::fprintf(stderr, "cuda-q4-prefill-bench: %s N=%u kernel-only tensor/" "scratch setup failed (%s)\n", case_name, n_tokens, - prequant.status == cudaSuccess - ? "tensor setup" - : cudaGetErrorString(prequant.status)); + prequant.status != cudaSuccess + ? cudaGetErrorString(prequant.status) + : fixup.status != cudaSuccess + ? cudaGetErrorString(fixup.status) + : "tensor setup"); return false; } @@ -944,9 +1040,9 @@ bool run_q4_16warp_kernel( } } - // Quantize exactly once. Both A/B arms consume this immutable canonical - // DS4 Q8_1 buffer, and no quantizer, memset, allocation, or copy is issued - // by either timed dispatch. + // Quantize exactly once. Both A/B arms consume this immutable canonical + // DS4 Q8_1 buffer. Timed work is restricted to each GEMM's scheduling, + // required stream-K scratch clear, producer, and fixup kernels. const int quant_rc = ds4_mmq_q4_K_quantize_q8_1_for_test( x_device, prequant.ptr, q8_bytes, static_cast(n_tokens), static_cast(in_dim), /*stream=*/nullptr); @@ -961,31 +1057,33 @@ bool run_q4_16warp_kernel( } const arm baseline = { - "canonical_preq_complete_k", + "canonical_preq_stream_k", [&](uint32_t set) { return ds4_mmq_q4_K_dense_preq_reference_for_test( weight_device[set], prequant.ptr, q8_bytes, reference_device, static_cast(out_dim), static_cast(n_tokens), static_cast(in_dim), - /*use_stream_k=*/0, /*stream=*/nullptr) == 0; + /*use_stream_k=*/1, fixup.ptr, fixup_bytes, + /*stream=*/nullptr) == 0; }, {}}; const arm candidate_arm = { - "q4_16warp_m128n128", + "q4_16warp_m128n128_stream_k", [&](uint32_t set) { - return ds4_mmq_q4_K_dense_16warp_enqueue( + return ds4_mmq_q4_K_dense_16warp_streamk_enqueue( weight_device[set], prequant.ptr, candidate_device, + fixup.ptr, fixup_bytes, static_cast(out_dim), static_cast(n_tokens), static_cast(in_dim), - /*stream=*/nullptr) == 0; + nsm, /*stream=*/nullptr) == 0; }, {}, [&](uint32_t set) { - return ds4_mmq_q4_K_dense_preq_16warp_for_test( - weight_device[set], prequant.ptr, q8_bytes, - candidate_device, static_cast(out_dim), + return ds4_mmq_q4_K_dense_16warp_streamk_enqueue( + weight_device[set], prequant.ptr, candidate_device, + fixup.ptr, fixup_bytes, static_cast(out_dim), static_cast(n_tokens), static_cast(in_dim), - /*stream=*/nullptr) == 0; + nsm, /*stream=*/nullptr) == 0; }}; const bool ok = benchmark_pair_arms( case_name, n_tokens, in_dim, out_dim, @@ -1001,7 +1099,7 @@ bool run_q4_16warp_kernel( const uint64_t offset = model.weights[set].*weight_offset_member; return bitwise_equal(reference.ptr, candidate.ptr, out_bytes, - "16-warp vs canonical complete-K") && + "16-warp vs canonical stream-K") && output_is_finite(reference.ptr, out_bytes, "16-warp canonical output") && output_is_finite(candidate.ptr, out_bytes, @@ -1014,7 +1112,8 @@ bool run_q4_16warp_kernel( check_guard(reference.ptr, out_bytes, 0x13000u, "16-warp canonical output") && check_guard(candidate.ptr, out_bytes, 0x14000u, - "16-warp candidate output"); + "16-warp candidate output") && + fixup.intact("16-warp Stream-K scratch"); }, "kernel_only_prequant"); return ok && @@ -1023,7 +1122,232 @@ bool run_q4_16warp_kernel( check_guard(reference.ptr, out_bytes, 0x13000u, "16-warp canonical output final") && check_guard(candidate.ptr, out_bytes, 0x14000u, - "16-warp candidate output final"); + "16-warp candidate output final") && + fixup.intact("16-warp Stream-K scratch final"); +} + +bool run_q4_16warp_pair_kernel( + const model_fixture &model, const config &cfg, uint32_t n_tokens) { + uint64_t x_elements = 0; + uint64_t out0_elements = 0; + uint64_t out1_elements = 0; + if (!checked_mul(n_tokens, kDenseK, &x_elements) || + !checked_mul(n_tokens, kDenseM, &out0_elements) || + !checked_mul(n_tokens, kKvM, &out1_elements)) { + return false; + } + const uint64_t x_bytes = x_elements * sizeof(float); + const uint64_t out0_bytes = out0_elements * sizeof(float); + const uint64_t out1_bytes = out1_elements * sizeof(float); + const uint64_t guard_bytes = kGuardWords * sizeof(uint32_t); + const uint64_t weight0_bytes = q4_weight_bytes(kDenseK, kDenseM); + const uint64_t weight1_bytes = q4_weight_bytes(kDenseK, kKvM); + const size_t q8_bytes = ds4_mmq_q4_K_q8_1_scratch_bytes( + static_cast(n_tokens), static_cast(kDenseK)); + if (q8_bytes == 0u) { + std::fprintf(stderr, + "cuda-q4-prefill-bench: pair kernel-only Q8_1 scratch " + "shape rejected for N=%u K=%u\n", + n_tokens, kDenseK); + return false; + } + int nsm = 0; + if (!current_device_nsm(&nsm)) return false; + const size_t fixup0_bytes = + ds4_mmq_q4_K_dense_16warp_streamk_scratch_bytes( + static_cast(kDenseM), static_cast(n_tokens), nsm); + const size_t fixup1_bytes = + ds4_mmq_q4_K_dense_16warp_streamk_scratch_bytes( + static_cast(kKvM), static_cast(n_tokens), nsm); + const size_t fixup_bytes = fixup0_bytes > fixup1_bytes + ? fixup0_bytes : fixup1_bytes; + + tensor_owner x(x_bytes + guard_bytes); + tensor_owner reference0(out0_bytes + guard_bytes); + tensor_owner reference1(out1_bytes + guard_bytes); + tensor_owner candidate0(out0_bytes + guard_bytes); + tensor_owner candidate1(out1_bytes + guard_bytes); + cuda_buffer prequant(q8_bytes); + guarded_cuda_buffer fixup(fixup_bytes, 0x1b000u); + std::vector activation; + fill_activation(&activation, n_tokens, kDenseK); + if (!x.ptr || !reference0.ptr || !reference1.ptr || !candidate0.ptr || + !candidate1.ptr || !prequant.ptr || prequant.status != cudaSuccess || + fixup.status != cudaSuccess || + (fixup_bytes != 0u && !fixup.ptr) || + !ds4_gpu_tensor_write(x.ptr, 0, activation.data(), x_bytes) || + !prepare_guard(x.ptr, x_bytes, 0x15000u)) { + std::fprintf(stderr, + "cuda-q4-prefill-bench: pair N=%u kernel-only tensor/" + "scratch setup failed (%s)\n", + n_tokens, + prequant.status != cudaSuccess + ? cudaGetErrorString(prequant.status) + : fixup.status != cudaSuccess + ? cudaGetErrorString(fixup.status) + : "tensor setup"); + return false; + } + + const auto *x_device = static_cast( + ds4_gpu_tensor_contents(x.ptr)); + auto *reference0_device = static_cast( + ds4_gpu_tensor_contents(reference0.ptr)); + auto *reference1_device = static_cast( + ds4_gpu_tensor_contents(reference1.ptr)); + auto *candidate0_device = static_cast( + ds4_gpu_tensor_contents(candidate0.ptr)); + auto *candidate1_device = static_cast( + ds4_gpu_tensor_contents(candidate1.ptr)); + if (!x_device || !reference0_device || !reference1_device || + !candidate0_device || !candidate1_device) { + std::fprintf(stderr, + "cuda-q4-prefill-bench: pair N=%u device tensor pointer " + "lookup failed\n", + n_tokens); + return false; + } + + std::vector weight0_device(cfg.sets, nullptr); + std::vector weight1_device(cfg.sets, nullptr); + for (uint32_t set = 0; set < cfg.sets; set++) { + if (!ds4_cuda_test_model_range_device_ptr( + model.data, model.size, model.weights[set].dense_offset, + weight0_bytes, /*logical_tier=*/0, &weight0_device[set]) || + !weight0_device[set] || + !ds4_cuda_test_model_range_device_ptr( + model.data, model.size, model.weights[set].kv_offset, + weight1_bytes, /*logical_tier=*/0, &weight1_device[set]) || + !weight1_device[set]) { + std::fprintf(stderr, + "cuda-q4-prefill-bench: pair N=%u resident weight " + "pointer lookup failed for set %u\n", + n_tokens, set); + return false; + } + } + + // One immutable activation quantization is shared by both Q-A/KV legs and + // is deliberately outside every event interval. + const int quant_rc = ds4_mmq_q4_K_quantize_q8_1_for_test( + x_device, prequant.ptr, q8_bytes, static_cast(n_tokens), + static_cast(kDenseK), /*stream=*/nullptr); + if (quant_rc != 0 || !ds4_gpu_synchronize() || + !check_guard(x.ptr, x_bytes, 0x15000u, + "kernel-only pair prequant input")) { + std::fprintf(stderr, + "cuda-q4-prefill-bench: pair N=%u prequantization " + "failed rc=%d\n", + n_tokens, quant_rc); + return false; + } + + const arm baseline = { + "two_canonical_preq_stream_k", + [&](uint32_t set) { + return ds4_mmq_q4_K_dense_preq_reference_for_test( + weight0_device[set], prequant.ptr, q8_bytes, + reference0_device, static_cast(kDenseM), + static_cast(n_tokens), static_cast(kDenseK), + /*use_stream_k=*/1, fixup.ptr, fixup_bytes, + /*stream=*/nullptr) == 0 && + ds4_mmq_q4_K_dense_preq_reference_for_test( + weight1_device[set], prequant.ptr, q8_bytes, + reference1_device, static_cast(kKvM), + static_cast(n_tokens), static_cast(kDenseK), + /*use_stream_k=*/1, fixup.ptr, fixup_bytes, + /*stream=*/nullptr) == 0; + }, + {}}; + const arm candidate = { + "two_q4_16warp_m128n128_stream_k", + [&](uint32_t set) { + return ds4_mmq_q4_K_dense_16warp_streamk_enqueue( + weight0_device[set], prequant.ptr, candidate0_device, + fixup.ptr, fixup_bytes, + static_cast(kDenseM), + static_cast(n_tokens), static_cast(kDenseK), + nsm, /*stream=*/nullptr) == 0 && + ds4_mmq_q4_K_dense_16warp_streamk_enqueue( + weight1_device[set], prequant.ptr, candidate1_device, + fixup.ptr, fixup_bytes, + static_cast(kKvM), + static_cast(n_tokens), static_cast(kDenseK), + nsm, /*stream=*/nullptr) == 0; + }, + {}, + [&](uint32_t set) { + return ds4_mmq_q4_K_dense_16warp_streamk_enqueue( + weight0_device[set], prequant.ptr, candidate0_device, + fixup.ptr, fixup_bytes, static_cast(kDenseM), + static_cast(n_tokens), static_cast(kDenseK), + nsm, /*stream=*/nullptr) == 0 && + ds4_mmq_q4_K_dense_16warp_streamk_enqueue( + weight1_device[set], prequant.ptr, candidate1_device, + fixup.ptr, fixup_bytes, static_cast(kKvM), + static_cast(n_tokens), static_cast(kDenseK), + nsm, /*stream=*/nullptr) == 0; + }}; + const bool ok = benchmark_pair_arms( + "pair", n_tokens, kDenseK, kDenseM + kKvM, + static_cast(kDenseK) * (kDenseM + kKvM), 0u, cfg, + baseline, candidate, + [&]() { + return poison_output(reference0.ptr, out0_bytes, 0x7fc70007u, + 0x16000u) && + poison_output(reference1.ptr, out1_bytes, 0x7fc80008u, + 0x17000u) && + poison_output(candidate0.ptr, out0_bytes, 0x7fc90009u, + 0x18000u) && + poison_output(candidate1.ptr, out1_bytes, 0x7fca000au, + 0x19000u); + }, + [&](uint32_t set) { + return bitwise_equal(reference0.ptr, candidate0.ptr, out0_bytes, + "16-warp pair q_a vs canonical") && + bitwise_equal(reference1.ptr, candidate1.ptr, out1_bytes, + "16-warp pair kv vs canonical") && + output_is_finite(reference0.ptr, out0_bytes, + "16-warp pair q_a canonical") && + output_is_finite(reference1.ptr, out1_bytes, + "16-warp pair kv canonical") && + output_is_finite(candidate0.ptr, out0_bytes, + "16-warp pair q_a candidate") && + output_is_finite(candidate1.ptr, out1_bytes, + "16-warp pair kv candidate") && + sampled_cpu_oracle( + reference0.ptr, model, + model.weights[set].dense_offset, activation, n_tokens, + kDenseK, kDenseM, "16-warp pair q_a canonical") && + sampled_cpu_oracle( + reference1.ptr, model, model.weights[set].kv_offset, + activation, n_tokens, kDenseK, kKvM, + "16-warp pair kv canonical") && + check_guard(x.ptr, x_bytes, 0x15000u, + "16-warp pair input") && + check_guard(reference0.ptr, out0_bytes, 0x16000u, + "16-warp pair q_a canonical") && + check_guard(reference1.ptr, out1_bytes, 0x17000u, + "16-warp pair kv canonical") && + check_guard(candidate0.ptr, out0_bytes, 0x18000u, + "16-warp pair q_a candidate") && + check_guard(candidate1.ptr, out1_bytes, 0x19000u, + "16-warp pair kv candidate") && + fixup.intact("16-warp pair Stream-K scratch"); + }, + "kernel_only_prequant"); + return ok && + check_guard(x.ptr, x_bytes, 0x15000u, + "16-warp pair input final") && + check_guard(reference0.ptr, out0_bytes, 0x16000u, + "16-warp pair q_a canonical final") && + check_guard(reference1.ptr, out1_bytes, 0x17000u, + "16-warp pair kv canonical final") && + check_guard(candidate0.ptr, out0_bytes, 0x18000u, + "16-warp pair q_a candidate final") && + check_guard(candidate1.ptr, out1_bytes, 0x19000u, + "16-warp pair kv candidate final") && + fixup.intact("16-warp pair Stream-K scratch final"); } bool run_dense(const model_fixture &model, const config &cfg, @@ -1375,9 +1699,9 @@ void usage(FILE *stream, const char *argv0) { " --path mmq|legacy process-wide path (default: mmq)\n" " --case all|dense|pair|qb|outa\n" " case to run (default: all)\n" - " --tokens N[,N...] token counts, each 9..4096\n" + " --tokens N[,N...] token counts, each 9..8192\n" " --full use 9,16,17,31,32,33,127,128,129,256," - "257,512,1024,2048,2049,4096\n" + "257,512,1024,2048,2049,4096,6144,8192\n" " --sets N rotating resident weight sets (default: %u)\n" " --samples N samples/arm, multiple of 4 (default: %u)\n" " --warmup N untimed dispatches/arm (default: %u)\n" @@ -1395,9 +1719,11 @@ void usage(FILE *stream, const char *argv0) { "DS4_CUDA_MMQ_X_MAX\nmay explicitly select an 8..128 multiple-of-8 " "sweep point; the setup line\nattests it, or prints auto when the " "variable is unset. --kernel-16warp requires\n--path mmq and " - "--case dense/qb; with --case all it runs only dense and q_b. Its " - "token\ncounts must be >=512; without --tokens/--full it uses " - "512,1024,2048,2049,4096.\n", + "--case dense/pair/qb; with --case all it runs dense, pair, and q_b. Its " + "pair arm\nshares one prequantized activation across M=1024+512. Token " + "counts must be\n>=512 and select canonical m128n128 on the active " + "device; without --tokens/--full it uses " + "512,1024,2048,2049,4096,6144,8192.\n", argv0, kDefaultSets, kDefaultSamples, kDefaultWarmup); } @@ -1414,6 +1740,11 @@ uint32_t parse_u32(const char *text, const char *option, uint32_t minimum, return static_cast(value); } +bool env_value_enabled(const char *name) { + const char *value = std::getenv(name); + return value && value[0] && !(value[0] == '0' && value[1] == '\0'); +} + const char *need_value(int *index, int argc, char **argv) { if (*index + 1 >= argc) { std::fprintf(stderr, "%s requires a value\n", argv[*index]); @@ -1430,7 +1761,7 @@ std::vector parse_tokens(const char *text) { const std::string item(cursor, comma ? static_cast(comma - cursor) : std::strlen(cursor)); - result.push_back(parse_u32(item.c_str(), "--tokens", 9u, 4096u)); + result.push_back(parse_u32(item.c_str(), "--tokens", 9u, 8192u)); if (!comma) break; cursor = comma + 1; if (!*cursor) { @@ -1484,7 +1815,7 @@ config parse_options(int argc, char **argv) { } else if (!std::strcmp(argv[i], "--full")) { cfg.tokens = {9u, 16u, 17u, 31u, 32u, 33u, 127u, 128u, 129u, 256u, 257u, 512u, 1024u, 2048u, - 2049u, 4096u}; + 2049u, 4096u, 6144u, 8192u}; tokens_explicit = true; } else if (!std::strcmp(argv[i], "--sets")) { cfg.sets = parse_u32(need_value(&i, argc, argv), "--sets", 1u, @@ -1509,7 +1840,7 @@ config parse_options(int argc, char **argv) { std::exit(2); } if (cfg.kernel_16warp && !tokens_explicit) { - cfg.tokens = {512u, 1024u, 2048u, 2049u, 4096u}; + cfg.tokens = {512u, 1024u, 2048u, 2049u, 4096u, 6144u, 8192u}; } if (cfg.kernel_16warp) { const auto below_minimum = std::find_if( @@ -1528,12 +1859,10 @@ config parse_options(int argc, char **argv) { "--kernel-16warp requires --path mmq\n"); std::exit(2); } - if (cfg.kernel_16warp && - (cfg.selected == bench_case::pair || - cfg.selected == bench_case::outa)) { + if (cfg.kernel_16warp && cfg.selected == bench_case::outa) { std::fprintf(stderr, - "--kernel-16warp supports only --case dense, qb, or " - "all (all runs dense+qb only)\n"); + "--kernel-16warp supports only --case dense, pair, qb, " + "or all (all runs dense+pair+qb)\n"); std::exit(2); } if (cfg.path == cuda_path::legacy && @@ -1551,7 +1880,7 @@ bool includes(bench_case selected, bench_case wanted) { return selected == bench_case::all || selected == wanted; } -std::string mmq_x_max_attestation() { +std::string mmq_x_max_attestation(bool require_m128n128) { const char *value = std::getenv("DS4_CUDA_MMQ_X_MAX"); if (!value || !value[0]) return "auto"; const uint32_t parsed = @@ -1563,6 +1892,13 @@ std::string mmq_x_max_attestation() { value); std::exit(2); } + if (require_m128n128 && parsed != 128u) { + std::fprintf(stderr, + "--kernel-16warp requires DS4_CUDA_MMQ_X_MAX=128 " + "when the variable is set (got %s)\n", + value); + std::exit(2); + } return std::to_string(parsed); } @@ -1629,10 +1965,11 @@ bool verify_resident_weight_ranges(const model_fixture &model) { } bool verify_mmq_prefill_dispatch(const model_fixture &model) { - // For N > 8 the public pair API succeeds only through MMQ. Probe it - // before printing any timing to initialize and attest the process-wide - // decision. Strict mode separately rejects fallback on every measured - // dense, pair, and q_b dispatch. + // For N > 8 the public pair API succeeds only through MMQ. Non-required + // production runs use this small probe to initialize and attest the + // process-wide decision. The caller skips it for raw-kernel and required + // 16-warp runs, whose measured dispatches provide their own fail-closed + // proof at an eligible shape. constexpr uint32_t n_tokens = 9u; const uint64_t x_bytes = static_cast(n_tokens) * kDenseK * sizeof(float); @@ -1662,7 +1999,8 @@ int main(int argc, char **argv) { // and validate the inherited sweep request before backend initialization, // then attest it in the setup record instead of silently contaminating a // supposedly default run. - const std::string mmq_x_max = mmq_x_max_attestation(); + const std::string mmq_x_max = + mmq_x_max_attestation(cfg.kernel_16warp); env_snapshot mmq_guard("DS4_CUDA_MMQ"); env_snapshot copy_guard("DS4_CUDA_COPY_MODEL"); env_snapshot pair_guard("DS4_CUDA_DISABLE_Q4_DENSE_PAIR"); @@ -1739,6 +2077,18 @@ int main(int argc, char **argv) { ds4_gpu_cleanup(); return 1; } + for (uint32_t tokens : cfg.tokens) { + if (!ds4_mmq_q4_K_dense_preq_reference_m128n128_for_test( + static_cast(tokens))) { + std::fprintf( + stderr, + "cuda-q4-prefill-bench: SKIP (--kernel-16warp N=%u " + "does not select canonical m128n128 on this device)\n", + tokens); + ds4_gpu_cleanup(); + return 77; + } + } } ds4_cuda_test_set_q4_mmq_strict( cfg.path == cuda_path::mmq ? 1 : 0); @@ -1769,7 +2119,13 @@ int main(int argc, char **argv) { ok = false; } } - if (ok && cfg.path == cuda_path::mmq && + // The N=9 probe is intentionally outside the 16-warp admission envelope. + // Kernel-only runs prove their raw path directly; required production runs + // fail closed at each measured dispatch, so probing here would be a false + // failure before the requested shape is reached. + const bool skip_mmq_probe = cfg.kernel_16warp || env_value_enabled( + "DS4_CUDA_REQUIRE_Q4_MMQ_16WARP"); + if (ok && cfg.path == cuda_path::mmq && !skip_mmq_probe && !verify_mmq_prefill_dispatch(model)) { std::fprintf(stderr, "cuda-q4-prefill-bench: MMQ prefill proof probe failed; " @@ -1807,9 +2163,11 @@ int main(int argc, char **argv) { &weight_set::dense_offset, "dense") : run_dense(model, cfg, n_tokens)) && ok; } - if (ok && !cfg.kernel_16warp && - includes(cfg.selected, bench_case::pair)) { - ok = run_pair(model, cfg, n_tokens) && ok; + if (ok && includes(cfg.selected, bench_case::pair)) { + ok = (cfg.kernel_16warp + ? run_q4_16warp_pair_kernel( + model, cfg, n_tokens) + : run_pair(model, cfg, n_tokens)) && ok; } if (ok && includes(cfg.selected, bench_case::qb)) { ok = (cfg.kernel_16warp From 58cc2f4b7deb7f054ab50c07ac7fb3b1652c8b55 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Sun, 30 Aug 2026 12:39:32 +0200 Subject: [PATCH 172/189] Enable ROCm Q4 prefill WMMA by default --- ENVIRONMENT_VARIABLES.md | 18 ++-- rocm/ds4_rocm_q4.cuh | 103 ++++++++++++++++------ scripts/environment_variables.tsv | 12 +-- speed-bench/README.md | 13 +-- speed-bench/rocm_q4_prefill_bench.cpp | 4 +- tests/test_rocm_q4_dense_pair.cpp | 119 +++++++++++++++++++++++--- 6 files changed, 211 insertions(+), 58 deletions(-) diff --git a/ENVIRONMENT_VARIABLES.md b/ENVIRONMENT_VARIABLES.md index f75891ce1..4623a8c45 100644 --- a/ENVIRONMENT_VARIABLES.md +++ b/ENVIRONMENT_VARIABLES.md @@ -93,14 +93,14 @@ The detailed Metal A/B contracts and expected oracle counters live in | `DS4_ROCM_DISABLE_Q4_PREFILL_TILE8=1` | Restore the legacy Q4 prefill kernel. TILE8 is automatic for validated chunks of 9 through 4096 tokens. | | `DS4_ROCM_REQUIRE_Q4_PREFILL_TILE8=1` | Fail closed when an eligible Q4 prefill call cannot use TILE8. | | `DS4_ROCM_ENABLE_Q4_PREFILL_TILE8=1` | Legacy spelling retained for migration notes only. The runtime does not read it; TILE8 is automatic and this setting is ignored. | -| `DS4_ROCM_ENABLE_Q4_PREFILL_Q8_K_WAVE32=1` | On gfx1151 wave32, opt into the no-LDS Q8_K activation quantizer that assigns one 256-value block to each wave before the exact Q4 prefill matmul. | +| `DS4_ROCM_ENABLE_Q4_PREFILL_Q8_K_WAVE32=1` | On gfx1151 wave32, opt into the no-LDS Q8_K activation quantizer that assigns one 256-value block to each wave before the exact Q4 prefill matmul. This exact path takes precedence over automatic direct-Q4 WMMA; `REQUIRE_Q4_PREFILL_WMMA` overrides an optional request, while dual REQUIRE fails closed. | | `DS4_ROCM_DISABLE_Q4_PREFILL_Q8_K_WAVE32=1` | Dominant rollback to the canonical one-workgroup-per-Q8_K-block quantizer. | | `DS4_ROCM_REQUIRE_Q4_PREFILL_Q8_K_WAVE32=1` | Require the wave32 quantizer for strict prefill A/B runs; unsupported scope/device, rollback, or an incompatible required F16/WMMA path fails closed. | -| `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA=1` | Opt into the experimental resident gfx1151 wave32 direct-Q4 WMMA prefill kernel for 256–4096 tokens. It replaces Q8_K activation scratch with transient F16 register dequantization and F32 accumulation, using 64 rows below output dimension 1024, 128 rows below 8192, and 256 rows otherwise; the wider variants use two-wide F32-to-F16 activation staging. | +| `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA=0/1` | Compatibility control for the automatic resident gfx1151 wave32 direct-Q4 WMMA prefill kernel at 256–4096 tokens: unset or `1` permits the default, while explicit `0` opts out for resident execution. It replaces Q8_K activation scratch with transient F16 register dequantization and F32 accumulation, using 64 rows below output dimension 1024, 128 rows below 8192, and 256 rows otherwise. | | `DS4_ROCM_Q4_PREFILL_WMMA_ROW_TILE=64|128|256` | Override the direct-Q4 WMMA output-row tile. The default uses 64 rows below output dimension 1024, 128 below 8192, and 256 otherwise; `64` also retains the prior kernel geometry as an A/B control. | | `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_SSD=1` | Allow that direct-Q4 WMMA path during SSD streaming only when the complete projection weight range is already backed by physical device storage. It never treats mapped/registered host memory as resident. | -| `DS4_ROCM_DISABLE_Q4_PREFILL_WMMA=1` | Dominant value-aware rollback for the direct-Q4 WMMA experiment. | -| `DS4_ROCM_REQUIRE_Q4_PREFILL_WMMA=1` | Request direct-Q4 WMMA and fail closed on an unsupported device/shape, quality mode, rollback, or SSD weight range that is not physically device-resident. Intended for strict A/B oracles. | +| `DS4_ROCM_DISABLE_Q4_PREFILL_WMMA=1` | Dominant value-aware opt-out for the automatic direct-Q4 WMMA path, including explicit resident or SSD requests. | +| `DS4_ROCM_REQUIRE_Q4_PREFILL_WMMA=1` | Assert the automatic direct-Q4 WMMA dispatch and fail closed on an unsupported device/shape, quality mode, rollback, or SSD weight range that is not physically device-resident. It is no longer needed to enable eligible resident execution and remains intended for strict A/B oracles. | | `DS4_ROCM_Q4_PREFILL_TILE8_STATS=1` | Report dense, pair, attention-batch, and token counters at process exit. | | `DS4_ROCM_ENABLE_Q4_DENSE_PAIR=1` | Share one Q8_K activation quantization between the two Q4 dense projections. This pair remains opt-in. | | `DS4_ROCM_DISABLE_Q4_DENSE_PAIR=1` | Dominant rollback for the ROCm Q4 dense pair. | @@ -969,7 +969,7 @@ and **19 tool/wrapper entries**. | `DS4_ROCM_DISABLE_Q4_DENSE_PAIR` | presence rollback; unset leaves opt-in policy unchanged | Disable/roll back rocm disable q4 dense pair. | [rocm/ds4_rocm_q4.cuh:435](rocm/ds4_rocm_q4.cuh#L435) | | `DS4_ROCM_DISABLE_Q4_GROUPED_ATTN_A` | presence rollback; unset permits the caller-marked resident decode production-shape default and explicit ENABLE/REQUIRE; any defined value including empty or 0 disables all grouped attention-A paths and wins over ENABLE/REQUIRE | Restore eight standalone Q4 attention-A projections instead of the two-dispatch grouped path. | [rocm/ds4_rocm_q4.cuh:868](rocm/ds4_rocm_q4.cuh#L868) | | `DS4_ROCM_DISABLE_Q4_PREFILL_TILE8` | presence rollback; TILE8 is default for 9..4096 tokens | Disable/roll back rocm disable q4 prefill tile8. | [rocm/ds4_rocm_q4.cuh:448](rocm/ds4_rocm_q4.cuh#L448) | -| `DS4_ROCM_DISABLE_Q4_PREFILL_WMMA` | value-aware authoritative rollback for the experimental path; unset/0/false/no/off permits ENABLE or REQUIRE, while empty or any other value disables; REQUIRE then fails closed | Prevent the gfx1151 direct-Q4 WMMA prefill experiment from dispatching and retain the Q8_K-plus-TILE8/TILE4 path. | [rocm/ds4_rocm_q4.cuh:1022](rocm/ds4_rocm_q4.cuh#L1022) | +| `DS4_ROCM_DISABLE_Q4_PREFILL_WMMA` | value-aware authoritative opt-out for the automatic resident path and explicit SSD/REQUIRE requests; unset/0/false/no/off leaves policy unchanged, while empty or any other value disables; REQUIRE then fails closed | Prevent the gfx1151 direct-Q4 WMMA prefill path from dispatching and retain the Q8_K-plus-TILE8/TILE4 path. | [rocm/ds4_rocm_q4.cuh:1068](rocm/ds4_rocm_q4.cuh#L1068) | | `DS4_ROCM_DISABLE_Q4_SELECTED_EXPERT_VIEWS` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable q4 selected expert views. | [ds4.c:21150](ds4.c#L21150) | | `DS4_ROCM_DISABLE_RESIDENT_IQ2_SORTED` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable resident iq2 sorted. | [rocm/ds4_rocm_moe_launch.cuh:751](rocm/ds4_rocm_moe_launch.cuh#L751) | | `DS4_ROCM_DISABLE_ROUTED_PAIR_SWIGLU_FUSION` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable routed pair swiglu fusion. | [ds4.c:18543](ds4.c#L18543) | @@ -1006,8 +1006,8 @@ and **19 tool/wrapper entries**. | `DS4_ROCM_ENABLE_MXFP4_TILE4` | presence opt-in; unset=off; any defined value including empty or 0 enables the candidate when the MXFP4 sorted-tile path has at least 5 tokens and neither TILE32 nor LDSB is selected | Select the ROCm MXFP4 gate/up tile4 occupancy variant, reducing staged-activation LDS per block. | [rocm/ds4_rocm_moe_launch.cuh:794](rocm/ds4_rocm_moe_launch.cuh#L794) | | `DS4_ROCM_ENABLE_Q4_DENSE_PAIR` | presence opt-in; unset=off; DISABLE takes precedence | Enable rocm enable q4 dense pair. | [rocm/ds4_rocm_q4.cuh:434](rocm/ds4_rocm_q4.cuh#L434) | | `DS4_ROCM_ENABLE_Q4_GROUPED_ATTN_A` | presence opt-in outside the default scope; the exact caller-marked resident decode shape groups=8, N=1, K=4096, M=1024 is automatic, while row-at-a-time batch fallbacks are not; DISABLE wins | Enable grouped Q4 attention-A for eligible slices, non-production shapes, or explicit experiments in addition to the resident decode default. | [rocm/ds4_rocm_q4.cuh:872](rocm/ds4_rocm_q4.cuh#L872) | -| `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA` | value-aware opt-in, default off; unset/0/false/no/off retains the canonical path; empty or any other value requests direct-Q4 WMMA only for N=256..4096, K a positive multiple of 256, resident non-quality execution on gfx1151 wave32; DISABLE wins | Benchmark compressed Q4_K-to-F16 register dequantization plus shape-selected 64-token by 64/128/256-row WMMA tiles and two-wide activation staging on the wider tiles, without Q8_K activation scratch or an F16 weight sidecar. | [rocm/ds4_rocm_q4.cuh:1018](rocm/ds4_rocm_q4.cuh#L1018) | -| `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_SSD` | value-aware SSD-only opt-in, default off; unset/0/false/no/off retains TILE8/TILE4, while empty or any other value requests direct-Q4 WMMA; eligibility additionally requires the complete projection weight range in physical device storage rather than mapped/registered host memory; DISABLE wins | Allow the compressed direct-Q4 WMMA kernel to consume an already device-resident/cache-backed Q4_K projection during SSD streaming without changing model I/O. | [rocm/ds4_rocm_q4.cuh:1020](rocm/ds4_rocm_q4.cuh#L1020) | +| `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA` | value-aware compatibility control for the automatic resident path; unset or any value other than 0/false/no/off permits direct-Q4 WMMA for N=256..4096, K a positive multiple of 256, resident non-quality execution on gfx1151 wave32; explicit 0/false/no/off opts out for resident execution; it never bypasses the SSD gate and DISABLE wins | Use compressed Q4_K-to-F16 register dequantization plus shape-selected 64-token by 64/128/256-row WMMA tiles and two-wide activation staging on the wider tiles, without Q8_K activation scratch or an F16 weight sidecar. | [rocm/ds4_rocm_q4.cuh:1064](rocm/ds4_rocm_q4.cuh#L1064) | +| `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_SSD` | value-aware SSD-only opt-in, default off; unset/0/false/no/off retains TILE8/TILE4, while empty or any other value requests direct-Q4 WMMA; eligibility additionally requires the complete projection weight range in physical device storage rather than mapped/registered host memory; DISABLE wins | Allow the compressed direct-Q4 WMMA kernel to consume an already device-resident/cache-backed Q4_K projection during SSD streaming without changing model I/O. | [rocm/ds4_rocm_q4.cuh:1066](rocm/ds4_rocm_q4.cuh#L1066) | | `DS4_ROCM_ENABLE_STREAMING_FULL_EXPERT_ADDR_TABLE` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming full expert addr table. | [ds4.c:18255](ds4.c#L18255) | | `DS4_ROCM_ENABLE_STREAMING_MADVISE_WILLNEED` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming madvise willneed. | [ds4.c:18224](ds4.c#L18224) | | `DS4_ROCM_ENABLE_STREAMING_PREFILL_BATCH_SELECTED_ADDR` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming prefill batch selected addr. | [ds4.c:18584](ds4.c#L18584) | @@ -1061,12 +1061,12 @@ and **19 tool/wrapper entries**. | `DS4_ROCM_MXFP4_DOWN_RGROUP` | nonempty value is parsed by strtol and a numeric prefix is sufficient; integers 1..8 are accepted; unset, empty, invalid, or out-of-range values use 1 | Set how many 32-row output blocks each ROCm MXFP4 tiled down-projection block computes, reducing the first launch-grid dimension as the value increases. | [rocm/ds4_rocm_moe_launch.cuh:801](rocm/ds4_rocm_moe_launch.cuh#L801) | | `DS4_ROCM_Q4_GROUPED_ATTN_A_STATS` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Print counters for rocm q4 grouped attn a stats. | [rocm/ds4_rocm_q4.cuh:614](rocm/ds4_rocm_q4.cuh#L614) | | `DS4_ROCM_Q4_PREFILL_TILE8_STATS` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Print counters for rocm q4 prefill tile8 stats. | [rocm/ds4_rocm_q4.cuh:514](rocm/ds4_rocm_q4.cuh#L514) | -| `DS4_ROCM_Q4_PREFILL_WMMA_ROW_TILE` | unsigned integer; unset, empty, malformed, negative, or values other than 64/128/256 use shape selection (64 rows when M<1024, 128 when M<8192, otherwise 256); 64 retains the previous geometry | Override the number of output rows sharing each direct-Q4 64x32 activation tile for controlled 64/128/256-row ROCm WMMA A/B measurements. | [rocm/ds4_rocm_q4.cuh:1126](rocm/ds4_rocm_q4.cuh#L1126) | +| `DS4_ROCM_Q4_PREFILL_WMMA_ROW_TILE` | unsigned integer; unset, empty, malformed, negative, or values other than 64/128/256 use shape selection (64 rows when M<1024, 128 when M<8192, otherwise 256); 64 retains the previous geometry | Override the number of output rows sharing each direct-Q4 64x32 activation tile for controlled 64/128/256-row ROCm WMMA A/B measurements. | [rocm/ds4_rocm_q4.cuh:1173](rocm/ds4_rocm_q4.cuh#L1173) | | `DS4_ROCM_Q8_DECODE_SHAREDX_64K` | sampled once; unset: enabled; present empty or exact 0: disabled; every other present value: enabled; effective only for one-token non-prequant Q8_0 matmul with 8192 < in_dim <= 16384 | Allows the ROCm shared-input Q8 decode kernel to use up to 64 KiB dynamic LDS for wide inputs; an unsupported/failed LDS launch automatically falls back to the regular kernel. | [rocm/ds4_rocm_runtime.cuh:4805](rocm/ds4_rocm_runtime.cuh#L4805) | | `DS4_ROCM_Q_STAGE_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm q stage profile. | [ds4.c:29284](ds4.c#L29284) | | `DS4_ROCM_REQUIRE_Q4_GROUPED_ATTN_A` | presence fail-closed assertion; also requests the candidate outside the caller-marked resident decode default; DISABLE remains authoritative and causes failure | Require grouped Q4 attention-A and fail instead of silently falling back. | [rocm/ds4_rocm_q4.cuh:870](rocm/ds4_rocm_q4.cuh#L870) | | `DS4_ROCM_REQUIRE_Q4_PREFILL_TILE8` | presence fail-closed assertion for eligible TILE8 calls | Require rocm require q4 prefill tile8 and fail instead of silently falling back. | [rocm/ds4_rocm_q4.cuh:452](rocm/ds4_rocm_q4.cuh#L452) | -| `DS4_ROCM_REQUIRE_Q4_PREFILL_WMMA` | value-aware strict opt-in; unset/0/false/no/off is off; empty or any other value requires every selected Q4 dense or attention-output projection to use direct-Q4 WMMA; unsupported shape/device, quality mode, DISABLE, or an SSD weight range without physical device residency fails before the relevant dispatch | Prevent the ROCm Q4 prefill WMMA correctness/performance oracle from silently timing TILE8/TILE4; the fused q_a/KV pair yields to separately checked dense calls. | [rocm/ds4_rocm_q4.cuh:1024](rocm/ds4_rocm_q4.cuh#L1024) | +| `DS4_ROCM_REQUIRE_Q4_PREFILL_WMMA` | value-aware strict assertion, not required for the automatic resident default; unset/0/false/no/off is off; empty or any other value requires every selected Q4 dense or attention-output projection to use direct-Q4 WMMA; unsupported shape/device, quality mode, DISABLE, or an SSD weight range without physical device residency fails before the relevant dispatch | Prevent the ROCm Q4 prefill WMMA correctness/performance oracle from silently timing TILE8/TILE4; the fused q_a/KV pair yields to separately checked dense calls. | [rocm/ds4_rocm_q4.cuh:1070](rocm/ds4_rocm_q4.cuh#L1070) | | `DS4_ROCM_STREAMING_DECODE_PREFILL_MAX` | primary nonempty value over the Metal alias; parsed by strtol when it has a numeric prefix (trailing text is accepted); <= 0 disables, values > UINT32_MAX clamp, no numeric prefix uses automatic default: 64 for Flash with uniform Q4_K/MXFP4 experts, 18 for other Pro/Flash, otherwise 0; the disable flag dominates | Sets the largest short, non-quality SSD-streaming prefill batch routed through the decode-style path instead of canonical layer-major prefill. | [ds4.c:31976](ds4.c#L31976) | | `DS4_ROCM_STREAMING_EXPERT_AUTO_PRELOAD_CAP` | primary nonempty value over the Metal alias; strict full-string strtoul; valid values > UINT32_MAX clamp, invalid uses 4096, and 0 means no cap (not disabled); when CLI preload is auto/0, unset defaults to cap 4096 except ROCm GLM52, where absent/empty disables automatic preload entirely | Caps the number of hot experts synchronously seeded into the SSD-streaming expert cache in automatic preload mode; an explicit CLI preload count bypasses this cap, and setting this variable opts ROCm GLM52 back into auto preload. | [ds4.c:21469](ds4.c#L21469) | | `DS4_ROCM_STREAMING_EXPERT_CACHE_VERBOSE` | presence flag; unset=off | Print verbose ROCm streaming expert-cache seed/load diagnostics. | [rocm/ds4_rocm_runtime.cuh:2904](rocm/ds4_rocm_runtime.cuh#L2904) | diff --git a/rocm/ds4_rocm_q4.cuh b/rocm/ds4_rocm_q4.cuh index 38b718363..382101605 100644 --- a/rocm/ds4_rocm_q4.cuh +++ b/rocm/ds4_rocm_q4.cuh @@ -124,7 +124,7 @@ static_assert((sizeof(cuda_block_q8_K) % sizeof(uint32_t)) == 0u, "ROCm Q8_K LDS copies require a whole number of words"); #if defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__) -/* Experimental direct-Q4 WMMA for resident gfx1151 prefill. +/* Direct-Q4 WMMA for resident gfx1151 prefill. * * This deliberately mirrors the live Q8 WMMA kernel: each wave32 owns 16 * output rows, the workgroup computes 64 tokens, K advances by 32, only the @@ -139,8 +139,9 @@ static_assert((sizeof(cuda_block_q8_K) % sizeof(uint32_t)) == 0u, * * Arithmetic is not bit-identical to Q4_K x Q8_K: activations and transient * weights round to F16 before F32 WMMA accumulation. Host policy therefore - * keeps this path opt-in, physically device-resident, gfx1151-only, and out of - * quality mode until device-side A/B plus a prompt-level oracle promote it. + * enables this path by default only for physically device-resident, gfx1151 + * wave32 prefill outside quality mode. SSD streaming remains separately + * gated, and DISABLE provides an authoritative rollback. */ enum { ROCM_Q4_WMMA_TOKEN_TILE = 64u, @@ -1009,21 +1010,66 @@ extern "C" uint64_t ds4_rocm_test_q4_prefill_wmma_get_calls(void) { __ATOMIC_RELAXED); } +/* Resident execution is automatic after validation. Preserve an explicit + * ENABLE=0 as a compatibility opt-out, while REQUIRE remains a strict + * assertion rather than the switch that turns the candidate on. SSD + * streaming deliberately keeps its separate opt-in and residency contract. */ +static int rocm_q4_K_prefill_wmma_requested_policy( + int ssd_streaming, + int enabled, + int ssd_enabled, + int disabled, + int required) { + if (required) return 1; + if (disabled) return 0; + return ssd_streaming ? ssd_enabled == 1 : enabled != 0; +} + +/* An explicitly selected exact Q8_K quantizer owns an optional WMMA default. + * REQUIRE_WMMA is the only control that may override an optional Q8 request; + * dual REQUIRE is rejected by each public dispatch before enqueue. */ +static int rocm_q4_K_prefill_wmma_yields_to_q8_wave32( + int q8_wave32, + int wmma_required) { + return q8_wave32 == ROCM_Q4_PREFILL_Q8_WAVE32_USE && !wmma_required; +} + +extern "C" int ds4_rocm_test_q4_prefill_wmma_yields_to_q8_wave32( + int q8_selected, + int wmma_required) { + const int q8_wave32 = q8_selected + ? ROCM_Q4_PREFILL_Q8_WAVE32_USE + : ROCM_Q4_PREFILL_Q8_WAVE32_FALLBACK; + return rocm_q4_K_prefill_wmma_yields_to_q8_wave32( + q8_wave32, wmma_required != 0); +} + +extern "C" int ds4_rocm_test_q4_prefill_wmma_requested_policy( + int ssd_streaming, + int enabled, + int ssd_enabled, + int disabled, + int required) { + return rocm_q4_K_prefill_wmma_requested_policy( + ssd_streaming != 0, enabled, ssd_enabled, disabled != 0, + required != 0); +} + static int rocm_q4_K_prefill_wmma_select( uint64_t n_tok, uint64_t in_dim, uint64_t out_dim, int weight_device_resident) { const int enabled = rocm_q4_attn_q_b_env_bool( - "DS4_ROCM_ENABLE_Q4_PREFILL_WMMA") == 1; + "DS4_ROCM_ENABLE_Q4_PREFILL_WMMA"); const int ssd_enabled = rocm_q4_attn_q_b_env_bool( "DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_SSD") == 1; const int disabled = rocm_q4_attn_q_b_env_bool( "DS4_ROCM_DISABLE_Q4_PREFILL_WMMA") == 1; const int required = rocm_q4_attn_q_b_env_bool( "DS4_ROCM_REQUIRE_Q4_PREFILL_WMMA") == 1; - const int requested = enabled || required || - (g_ssd_streaming_mode && ssd_enabled); + const int requested = rocm_q4_K_prefill_wmma_requested_policy( + g_ssd_streaming_mode, enabled, ssd_enabled, disabled, required); if (!requested) return ROCM_Q4_PREFILL_WMMA_FALLBACK; const int shape_ok = @@ -1034,9 +1080,10 @@ static int rocm_q4_K_prefill_wmma_select( const int storage_ok = !g_ssd_streaming_mode || ((ssd_enabled || required) && weight_device_resident); + const int explicit_request = enabled == 1 || ssd_enabled || required; const int eligible = !disabled && shape_ok && storage_ok && !g_quality_mode && - rocm_attention_runtime_is_gfx1151_wave32(requested); + rocm_attention_runtime_is_gfx1151_wave32(explicit_request); if (eligible) return ROCM_Q4_PREFILL_WMMA_USE; if (!required) return ROCM_Q4_PREFILL_WMMA_FALLBACK; @@ -1494,19 +1541,22 @@ extern "C" int ds4_rocm_matmul_q4_K_tensor( n_tok, in_dim, out_dim, weight_device_resident); if (prefill_wmma == ROCM_Q4_PREFILL_WMMA_REQUIRED_FAILURE) return 0; if (prefill_wmma == ROCM_Q4_PREFILL_WMMA_USE && - q8_wave32 == ROCM_Q4_PREFILL_Q8_WAVE32_USE && - q8_wave32_required) { - if (rocm_q4_attn_q_b_env_bool( - "DS4_ROCM_REQUIRE_Q4_PREFILL_WMMA") == 1) { + q8_wave32 == ROCM_Q4_PREFILL_Q8_WAVE32_USE) { + const int wmma_required = rocm_q4_attn_q_b_env_bool( + "DS4_ROCM_REQUIRE_Q4_PREFILL_WMMA") == 1; + if (wmma_required && q8_wave32_required) { fprintf(stderr, DS4_GPU_LOG_PREFIX "Q4_K prefill cannot require both direct WMMA and " "the Q8_K wave32 quantizer\n"); return 0; } - /* The direct F16 WMMA path has no Q8_K RHS. A strict quantizer - * request therefore owns dispatch and retains the exact matmul. */ - prefill_wmma = ROCM_Q4_PREFILL_WMMA_FALLBACK; + /* The direct F16 WMMA path has no Q8_K RHS. An explicitly selected + * exact quantizer therefore owns an optional automatic WMMA path. */ + if (rocm_q4_K_prefill_wmma_yields_to_q8_wave32( + q8_wave32, wmma_required)) { + prefill_wmma = ROCM_Q4_PREFILL_WMMA_FALLBACK; + } } if (prefill_wmma == ROCM_Q4_PREFILL_WMMA_USE && prefill_scope && prefill_tile8_required) { @@ -1660,13 +1710,15 @@ extern "C" int ds4_gpu_matmul_q4_K_pair_tensor( } /* The fused pair consumes a shared Q8_K activation tile. When the direct - * F16 WMMA experiment is selected, return before validation/enqueue so the + * F16 WMMA path is selected, return before validation/enqueue so the * graph's established fallback issues two dense calls and both can take * the strict WMMA path. This also prevents REQUIRE from falsely passing * after silently measuring the TILE8 pair. SSD preflight is deliberately * optimistic about residency: it only decides whether to yield; each dense * fallback then proves its exact physical device range before enqueue. */ - if (!prefill_required && !q8_wave32_required) { + if (!prefill_required && + !rocm_q4_K_prefill_wmma_yields_to_q8_wave32( + q8_wave32, wmma_required)) { const int wmma0 = rocm_q4_K_prefill_wmma_select( n_tok, in_dim, out0_dim, 1); const int wmma1 = rocm_q4_K_prefill_wmma_select( @@ -1956,16 +2008,16 @@ extern "C" int ds4_gpu_attention_output_q4_K_batch_tensor( return -1; } const int wmma_enabled = rocm_q4_attn_q_b_env_bool( - "DS4_ROCM_ENABLE_Q4_PREFILL_WMMA") == 1; + "DS4_ROCM_ENABLE_Q4_PREFILL_WMMA"); const int wmma_ssd_enabled = rocm_q4_attn_q_b_env_bool( "DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_SSD") == 1; const int wmma_disabled = rocm_q4_attn_q_b_env_bool( "DS4_ROCM_DISABLE_Q4_PREFILL_WMMA") == 1; const int wmma_required = rocm_q4_attn_q_b_env_bool( "DS4_ROCM_REQUIRE_Q4_PREFILL_WMMA") == 1; - const int wmma_requested = wmma_required || - (!wmma_disabled && ((!g_ssd_streaming_mode && wmma_enabled) || - (g_ssd_streaming_mode && wmma_ssd_enabled))); + const int wmma_requested = rocm_q4_K_prefill_wmma_requested_policy( + g_ssd_streaming_mode, wmma_enabled, wmma_ssd_enabled, + wmma_disabled, wmma_required); if (!tile8_scope) return 0; if (!tile8_requested && !wmma_requested) { if (tile8_required || q8_wave32_required) { @@ -2069,18 +2121,21 @@ extern "C" int ds4_gpu_attention_output_q4_K_batch_tensor( b_wmma == ROCM_Q4_PREFILL_WMMA_REQUIRED_FAILURE) { return -1; } - if (q8_wave32_required && + if (q8_wave32 == ROCM_Q4_PREFILL_Q8_WAVE32_USE && (a_wmma == ROCM_Q4_PREFILL_WMMA_USE || b_wmma == ROCM_Q4_PREFILL_WMMA_USE)) { - if (wmma_required) { + if (wmma_required && q8_wave32_required) { fprintf(stderr, DS4_GPU_LOG_PREFIX "Q4_K attention-output prefill cannot require both " "direct WMMA and the Q8_K wave32 quantizer\n"); return -1; } - a_wmma = ROCM_Q4_PREFILL_WMMA_FALLBACK; - b_wmma = ROCM_Q4_PREFILL_WMMA_FALLBACK; + if (rocm_q4_K_prefill_wmma_yields_to_q8_wave32( + q8_wave32, wmma_required)) { + a_wmma = ROCM_Q4_PREFILL_WMMA_FALLBACK; + b_wmma = ROCM_Q4_PREFILL_WMMA_FALLBACK; + } } if (tile8_required && (a_wmma == ROCM_Q4_PREFILL_WMMA_USE || diff --git a/scripts/environment_variables.tsv b/scripts/environment_variables.tsv index 8a5ec0134..27cec9a16 100644 --- a/scripts/environment_variables.tsv +++ b/scripts/environment_variables.tsv @@ -978,7 +978,7 @@ runtime/rocm DS4_ROCM_DISABLE_Q4_GROUPED_ATTN_A presence rollback; unset permits runtime/rocm DS4_ROCM_DISABLE_Q4_PREFILL_K1024_TILE4 value-aware authoritative rollback; unset/0/false/no/off preserves the resident automatic default and any explicit SSD request; empty or any other value disables; overrides ENABLE and causes REQUIRE to fail closed Restore the generic eight-block Q4_K tiled-prefill kernel for K=1024 in both resident and SSD-streaming execution. rocm/ds4_rocm_q4.cuh:624 runtime/rocm DS4_ROCM_DISABLE_Q4_PREFILL_Q8_K_WAVE32 value-aware authoritative rollback; unset/0/false/no/off permits ENABLE or REQUIRE, while empty or any other value disables; REQUIRE then fails closed Restore the canonical one-workgroup-per-Q8_K-block activation quantizer for Q4 prefill. rocm/ds4_rocm_q4.cuh:803 runtime/rocm DS4_ROCM_DISABLE_Q4_PREFILL_TILE8 presence rollback; TILE8 is default for 9..4096 tokens Disable/roll back rocm disable q4 prefill tile8. rocm/ds4_rocm_q4.cuh:448 -runtime/rocm DS4_ROCM_DISABLE_Q4_PREFILL_WMMA value-aware authoritative rollback for the experimental path; unset/0/false/no/off permits ENABLE or REQUIRE, while empty or any other value disables; REQUIRE then fails closed Prevent the gfx1151 direct-Q4 WMMA prefill experiment from dispatching and retain the Q8_K-plus-TILE8/TILE4 path. rocm/ds4_rocm_q4.cuh:1022 +runtime/rocm DS4_ROCM_DISABLE_Q4_PREFILL_WMMA value-aware authoritative opt-out for the automatic resident path and explicit SSD/REQUIRE requests; unset/0/false/no/off leaves policy unchanged, while empty or any other value disables; REQUIRE then fails closed Prevent the gfx1151 direct-Q4 WMMA prefill path from dispatching and retain the Q8_K-plus-TILE8/TILE4 path. rocm/ds4_rocm_q4.cuh:1068 runtime/rocm DS4_ROCM_DISABLE_Q4_SELECTED_EXPERT_VIEWS presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable q4 selected expert views. ds4.c:21150 runtime/rocm DS4_ROCM_DISABLE_RESIDENT_IQ2_SORTED presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable resident iq2 sorted. rocm/ds4_rocm_moe_launch.cuh:751 runtime/rocm DS4_ROCM_DISABLE_ROUTED_PAIR_SWIGLU_FUSION presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable routed pair swiglu fusion. ds4.c:18543 @@ -1021,9 +1021,9 @@ runtime/rocm DS4_ROCM_ENABLE_Q4_ATTN_Q_B_F16_OUTPUT value-aware experimental opt runtime/rocm DS4_ROCM_ENABLE_Q4_DENSE_PAIR presence opt-in; unset=off; DISABLE takes precedence Enable rocm enable q4 dense pair. rocm/ds4_rocm_q4.cuh:434 runtime/rocm DS4_ROCM_ENABLE_Q4_GROUPED_ATTN_A presence opt-in outside the default scope; the exact caller-marked resident decode shape groups=8, N=1, K=4096, M=1024 is automatic, while row-at-a-time batch fallbacks are not; DISABLE wins Enable grouped Q4 attention-A for eligible slices, non-production shapes, or explicit experiments in addition to the resident decode default. rocm/ds4_rocm_q4.cuh:872 runtime/rocm DS4_ROCM_ENABLE_Q4_PREFILL_K1024_TILE4_SSD value-aware SSD-only opt-in, default off; unset/0/false/no/off retains TILE8, while empty or any other value requests TILE4; eligibility additionally requires N=9..4096, K=1024, M=32768, TILE8 enabled, and the complete weight range in device storage rather than mapped/registered host memory; DISABLE wins Allow the four-lane K=1024 Q4_K prefill specialization to consume an already device-resident/cache-backed attn_q_b weight range during SSD streaming without changing model I/O. rocm/ds4_rocm_q4.cuh:624 -runtime/rocm DS4_ROCM_ENABLE_Q4_PREFILL_Q8_K_WAVE32 value-aware opt-in, default off; unset/0/false/no/off retains the canonical quantizer, while empty or any other value requests the candidate for N=9..4096 on gfx1151 wave32; DISABLE wins Quantize eight independent Q8_K activation blocks per 256-thread workgroup using one wave32 per block, without LDS or workgroup barriers, before the exact Q4 prefill matmul (TILE8 or its legacy rollback). rocm/ds4_rocm_q4.cuh:801 -runtime/rocm DS4_ROCM_ENABLE_Q4_PREFILL_WMMA value-aware opt-in, default off; unset/0/false/no/off retains the canonical path; empty or any other value requests direct-Q4 WMMA only for N=256..4096, K a positive multiple of 256, resident non-quality execution on gfx1151 wave32; DISABLE wins Benchmark compressed Q4_K-to-F16 register dequantization plus shape-selected 64-token by 64/128/256-row WMMA tiles and two-wide activation staging on the wider tiles, without Q8_K activation scratch or an F16 weight sidecar. rocm/ds4_rocm_q4.cuh:1018 -runtime/rocm DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_SSD value-aware SSD-only opt-in, default off; unset/0/false/no/off retains TILE8/TILE4, while empty or any other value requests direct-Q4 WMMA; eligibility additionally requires the complete projection weight range in physical device storage rather than mapped/registered host memory; DISABLE wins Allow the compressed direct-Q4 WMMA kernel to consume an already device-resident/cache-backed Q4_K projection during SSD streaming without changing model I/O. rocm/ds4_rocm_q4.cuh:1020 +runtime/rocm DS4_ROCM_ENABLE_Q4_PREFILL_Q8_K_WAVE32 value-aware opt-in, default off; unset/0/false/no/off retains the canonical quantizer, while empty or any other value requests the candidate for N=9..4096 on gfx1151 wave32; DISABLE wins; a selected exact Q8 path takes precedence over automatic direct-Q4 WMMA, REQUIRE_WMMA overrides an optional request, and dual REQUIRE fails closed Quantize eight independent Q8_K activation blocks per 256-thread workgroup using one wave32 per block, without LDS or workgroup barriers, before the exact Q4 prefill matmul (TILE8 or its legacy rollback). rocm/ds4_rocm_q4.cuh:858 +runtime/rocm DS4_ROCM_ENABLE_Q4_PREFILL_WMMA value-aware compatibility control for the automatic resident path; unset or any value other than 0/false/no/off permits direct-Q4 WMMA for N=256..4096, K a positive multiple of 256, resident non-quality execution on gfx1151 wave32; explicit 0/false/no/off opts out for resident execution; it never bypasses the SSD gate and DISABLE wins Use compressed Q4_K-to-F16 register dequantization plus shape-selected 64-token by 64/128/256-row WMMA tiles and two-wide activation staging on the wider tiles, without Q8_K activation scratch or an F16 weight sidecar. rocm/ds4_rocm_q4.cuh:1064 +runtime/rocm DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_SSD value-aware SSD-only opt-in, default off; unset/0/false/no/off retains TILE8/TILE4, while empty or any other value requests direct-Q4 WMMA; eligibility additionally requires the complete projection weight range in physical device storage rather than mapped/registered host memory; DISABLE wins Allow the compressed direct-Q4 WMMA kernel to consume an already device-resident/cache-backed Q4_K projection during SSD streaming without changing model I/O. rocm/ds4_rocm_q4.cuh:1066 runtime/rocm DS4_ROCM_ENABLE_STREAMING_FULL_EXPERT_ADDR_TABLE presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming full expert addr table. ds4.c:18255 runtime/rocm DS4_ROCM_ENABLE_STREAMING_MADVISE_WILLNEED presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming madvise willneed. ds4.c:18224 runtime/rocm DS4_ROCM_ENABLE_STREAMING_PREFILL_BATCH_SELECTED_ADDR presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming prefill batch selected addr. ds4.c:18584 @@ -1081,7 +1081,7 @@ runtime/rocm DS4_ROCM_Q4_ATTN_Q_B_F16_CACHE_MIN_TOKENS integer token count with runtime/rocm DS4_ROCM_Q4_ATTN_Q_B_TRANSIENT_F16_MIN_TOKENS full-string unsigned token count; default 4096; accepted range 32..UINT32_MAX; values above clamp and invalid or smaller values restore 4096 Set the minimum device-resident, non-SSD prefill batch eligible for per-layer transient ROCm Q4_K attn_q_b-to-F16 expansion. rocm/ds4_rocm_q4_qb_sidecar.cuh:150 runtime/rocm DS4_ROCM_Q4_GROUPED_ATTN_A_STATS presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Print counters for rocm q4 grouped attn a stats. rocm/ds4_rocm_q4.cuh:614 runtime/rocm DS4_ROCM_Q4_PREFILL_TILE8_STATS presence diagnostic; unset=off; any defined value including empty or 0 prints at exit Print tiled-prefill dense/pair/attention counters plus total and SSD-specific K=1024 TILE4 dispatch counts. rocm/ds4_rocm_q4.cuh:741 -runtime/rocm DS4_ROCM_Q4_PREFILL_WMMA_ROW_TILE unsigned integer; unset, empty, malformed, negative, or values other than 64/128/256 use shape selection (64 rows when M<1024, 128 when M<8192, otherwise 256); 64 retains the previous geometry Override the number of output rows sharing each direct-Q4 64x32 activation tile for controlled 64/128/256-row ROCm WMMA A/B measurements. rocm/ds4_rocm_q4.cuh:1126 +runtime/rocm DS4_ROCM_Q4_PREFILL_WMMA_ROW_TILE unsigned integer; unset, empty, malformed, negative, or values other than 64/128/256 use shape selection (64 rows when M<1024, 128 when M<8192, otherwise 256); 64 retains the previous geometry Override the number of output rows sharing each direct-Q4 64x32 activation tile for controlled 64/128/256-row ROCm WMMA A/B measurements. rocm/ds4_rocm_q4.cuh:1173 runtime/rocm DS4_ROCM_Q8_DECODE_SHAREDX_64K sampled once; unset: enabled; present empty or exact 0: disabled; every other present value: enabled; effective only for one-token non-prequant Q8_0 matmul with 8192 < in_dim <= 16384 Allows the ROCm shared-input Q8 decode kernel to use up to 64 KiB dynamic LDS for wide inputs; an unsupported/failed LDS launch automatically falls back to the regular kernel. rocm/ds4_rocm_runtime.cuh:4805 runtime/rocm DS4_ROCM_Q_STAGE_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm q stage profile. ds4.c:30038 runtime/rocm DS4_ROCM_REQUIRE_Q4_ATTN_Q_B_F16_CACHE value-aware strict opt-in, default off; unset/0/false/no/off is off, empty or any other value requires eligible batches to use the cache; DISABLE wins Fail an eligible ROCm prefill instead of falling back when the resident Q4_K attn_q_b F16 specialization cannot be prepared or dispatched. rocm/ds4_rocm_q4_qb_sidecar.cuh:135 @@ -1089,7 +1089,7 @@ runtime/rocm DS4_ROCM_REQUIRE_Q4_GROUPED_ATTN_A presence fail-closed assertion; runtime/rocm DS4_ROCM_REQUIRE_Q4_PREFILL_K1024_TILE4 value-aware fail-closed assertion and SSD opt-in; unset/0/false/no/off is off; empty or any other value requires eligible N=9..4096, K=1024, M=32768 dense calls to select TILE4; SSD also requires an actual device-resident weight range; DISABLE wins Prevent a K=1024 TILE4 correctness/performance oracle from silently falling back to TILE8, including during SSD-streaming A/B runs. rocm/ds4_rocm_q4.cuh:628 runtime/rocm DS4_ROCM_REQUIRE_Q4_PREFILL_Q8_K_WAVE32 value-aware strict opt-in; unset/0/false/no/off is off; empty or any other value requires gfx1151 wave32 and N=9..4096; DISABLE and conflicts with required WMMA or the required q_b F16 cache fail closed Require the no-LDS wave32 Q8_K activation quantizer for exact Q4 prefill instead of silently using the canonical quantizer or an F16 side path. rocm/ds4_rocm_q4.cuh:796 runtime/rocm DS4_ROCM_REQUIRE_Q4_PREFILL_TILE8 presence fail-closed assertion for eligible TILE8 calls Require rocm require q4 prefill tile8 and fail instead of silently falling back. rocm/ds4_rocm_q4.cuh:452 -runtime/rocm DS4_ROCM_REQUIRE_Q4_PREFILL_WMMA value-aware strict opt-in; unset/0/false/no/off is off; empty or any other value requires every selected Q4 dense or attention-output projection to use direct-Q4 WMMA; unsupported shape/device, quality mode, DISABLE, or an SSD weight range without physical device residency fails before the relevant dispatch Prevent the ROCm Q4 prefill WMMA correctness/performance oracle from silently timing TILE8/TILE4; the fused q_a/KV pair yields to separately checked dense calls. rocm/ds4_rocm_q4.cuh:1024 +runtime/rocm DS4_ROCM_REQUIRE_Q4_PREFILL_WMMA value-aware strict assertion, not required for the automatic resident default; unset/0/false/no/off is off; empty or any other value requires every selected Q4 dense or attention-output projection to use direct-Q4 WMMA; unsupported shape/device, quality mode, DISABLE, or an SSD weight range without physical device residency fails before the relevant dispatch Prevent the ROCm Q4 prefill WMMA correctness/performance oracle from silently timing TILE8/TILE4; the fused q_a/KV pair yields to separately checked dense calls. rocm/ds4_rocm_q4.cuh:1070 runtime/rocm DS4_ROCM_STREAMING_DECODE_PREFILL_MAX primary nonempty value over the Metal alias; parsed by strtol when it has a numeric prefix (trailing text is accepted); <= 0 disables, values > UINT32_MAX clamp, no numeric prefix uses automatic default: 64 for Flash with uniform Q4_K/MXFP4 experts, 18 for other Pro/Flash, otherwise 0; the disable flag dominates Sets the largest short, non-quality SSD-streaming prefill batch routed through the decode-style path instead of canonical layer-major prefill. ds4.c:31976 runtime/rocm DS4_ROCM_STREAMING_EXPERT_AUTO_PRELOAD_CAP primary nonempty value over the Metal alias; strict full-string strtoul; valid values > UINT32_MAX clamp, invalid uses 4096, and 0 means no cap (not disabled); when CLI preload is auto/0, unset defaults to cap 4096 except ROCm GLM52, where absent/empty disables automatic preload entirely Caps the number of hot experts synchronously seeded into the SSD-streaming expert cache in automatic preload mode; an explicit CLI preload count bypasses this cap, and setting this variable opts ROCm GLM52 back into auto preload. ds4.c:21469 runtime/rocm DS4_ROCM_STREAMING_EXPERT_CACHE_VERBOSE presence flag; unset=off Print verbose ROCm streaming expert-cache seed/load diagnostics. rocm/ds4_rocm_runtime.cuh:2904 diff --git a/speed-bench/README.md b/speed-bench/README.md index 33951fed8..6c9090c64 100644 --- a/speed-bench/README.md +++ b/speed-bench/README.md @@ -166,7 +166,7 @@ comparisons are: `K=4096,M=(1024+512)` path; - `qb`: TILE8 versus TILE4 at the production `attn_q_b` `K=1024,M=32768` shape. -- `outb`: TILE8 versus the experimental compressed direct-Q4 WMMA kernel at the +- `outb`: TILE8 versus the compressed direct-Q4 WMMA kernel at the production `output_b` `K=8192,M=4096` shape; - `output`: the complete grouped `output_a` plus `output_b` production API, comparing two TILE8 projections with two direct WMMA projections. @@ -187,9 +187,10 @@ including its changed workgroup and occupancy contract, separately from the candidate's arithmetic change versus TILE8. `q_b` additionally compares scalar versus two-wide activation staging at fixed 128- and 256-row geometry. The direct hook receives both tile and loader explicitly, so every arm attests -its own configuration. It remains opt-in in the runtime; the production-API -comparison uses the strict REQUIRE gate so a rejected dispatch fails instead -of timing a fallback. +its own configuration. Eligible resident runtime calls use direct-Q4 WMMA by +default; set `DS4_ROCM_DISABLE_Q4_PREFILL_WMMA=1` to opt out. The +production-API comparison still uses the strict REQUIRE gate so a rejected +dispatch fails instead of timing a fallback. The benchmark always keeps SSD streaming disabled. Runtime SSD experiments need the separate `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_SSD=1` gate and only accept projection ranges already backed by physical device memory, so model I/O is @@ -211,8 +212,8 @@ ABBA/BAAB, and verifies allocation guards both before and after timing. Comparisons among the integer Q4 paths remain bit-exact. Direct-Q4 WMMA has a deliberate F16 arithmetic boundary, so those comparisons require finite results within an explicit -absolute/relative smoke tolerance; a model-logit or prompt oracle is still -required before enabling the kernel by default. +absolute/relative smoke tolerance; release validation must also include the +model-logit or prompt oracle that guards the automatic policy. Fixture creation, the host-to-device residency copy, warmup, oracle readback, and environment-gate changes are outside the reported HIP-event intervals. `candidate_delta_pct` is negative when the candidate is faster; the companion diff --git a/speed-bench/rocm_q4_prefill_bench.cpp b/speed-bench/rocm_q4_prefill_bench.cpp index 368491d62..59c3bfcc0 100644 --- a/speed-bench/rocm_q4_prefill_bench.cpp +++ b/speed-bench/rocm_q4_prefill_bench.cpp @@ -523,7 +523,7 @@ void select_legacy() { (void)unsetenv(kK1024Tile4Require); (void)unsetenv(kWmmaEnable); (void)unsetenv(kWmmaSsdEnable); - (void)unsetenv(kWmmaDisable); + (void)setenv(kWmmaDisable, "1", 1); (void)unsetenv(kWmmaRequire); (void)unsetenv(kWmmaRowTile); (void)unsetenv(kQ8Wave32Enable); @@ -539,7 +539,7 @@ void select_tile8(bool disable_k1024_tile4) { (void)unsetenv(kK1024Tile4Require); (void)unsetenv(kWmmaEnable); (void)unsetenv(kWmmaSsdEnable); - (void)unsetenv(kWmmaDisable); + (void)setenv(kWmmaDisable, "1", 1); (void)unsetenv(kWmmaRequire); (void)unsetenv(kWmmaRowTile); (void)unsetenv(kQ8Wave32Enable); diff --git a/tests/test_rocm_q4_dense_pair.cpp b/tests/test_rocm_q4_dense_pair.cpp index 71146ec4b..d45982599 100644 --- a/tests/test_rocm_q4_dense_pair.cpp +++ b/tests/test_rocm_q4_dense_pair.cpp @@ -36,6 +36,11 @@ extern "C" int ds4_rocm_test_q4_prefill_k1024_tile4_policy( int disabled, int required); extern "C" void ds4_rocm_test_q4_prefill_wmma_reset(void); extern "C" uint64_t ds4_rocm_test_q4_prefill_wmma_get_calls(void); +extern "C" int ds4_rocm_test_q4_prefill_wmma_requested_policy( + int ssd_streaming, int enabled, int ssd_enabled, int disabled, + int required); +extern "C" int ds4_rocm_test_q4_prefill_wmma_yields_to_q8_wave32( + int q8_selected, int wmma_required); extern "C" uint32_t ds4_rocm_test_q4_prefill_wmma_row_tile( uint32_t out_dim); extern "C" int ds4_rocm_test_q4_prefill_q8_wave32_policy( @@ -863,6 +868,7 @@ bool run_prefill_parity_case(const aligned_model &model, uint32_t n_tokens, env_snapshot k1024_tile4_disable(kPrefillK1024Tile4Disable); env_snapshot k1024_tile4_ssd_enable(kPrefillK1024Tile4SsdEnable); env_snapshot k1024_tile4_require(kPrefillK1024Tile4Require); + env_snapshot wmma_disable(kPrefillWmmaDisable); const bool require_k1024_tile4 = in_dim == kTailK && out_dim == kQbOutDim; @@ -873,6 +879,7 @@ bool run_prefill_parity_case(const aligned_model &model, uint32_t n_tokens, (void)unsetenv(kPrefillRequire); (void)unsetenv(kPrefillK1024Tile4SsdEnable); (void)unsetenv(kPrefillK1024Tile4Require); + (void)setenv(kPrefillWmmaDisable, "1", 1); const int legacy_rc = ds4_gpu_matmul_quant_tensor( legacy_gpu.ptr, model.data, model.size, offset, kQ4Type, in_dim, out_dim, x_gpu.ptr, n_tokens); @@ -885,6 +892,7 @@ bool run_prefill_parity_case(const aligned_model &model, uint32_t n_tokens, (void)setenv(kPrefillRequire, "1", 1); (void)unsetenv(kPrefillK1024Tile4Disable); (void)unsetenv(kPrefillK1024Tile4SsdEnable); + (void)setenv(kPrefillWmmaDisable, "1", 1); if (require_k1024_tile4) { (void)setenv(kPrefillK1024Tile4Require, "1", 1); } else { @@ -1196,6 +1204,67 @@ bool run_prefill_wmma_row_tile_policy_oracle() { return ok; } +bool run_prefill_wmma_requested_policy_oracle() { + struct policy_case { + const char *label; + int ssd_streaming; + int enabled; + int ssd_enabled; + int disabled; + int required; + int expected; + }; + const policy_case cases[] = { + {"resident unset is automatic", 0, -1, 0, 0, 0, 1}, + {"resident ENABLE=0 compatibility opt-out", 0, 0, 0, 0, 0, 0}, + {"resident ENABLE=1 remains accepted", 0, 1, 0, 0, 0, 1}, + {"resident DISABLE opts out", 0, -1, 0, 1, 0, 0}, + {"resident REQUIRE requests after ENABLE=0", 0, 0, 0, 0, 1, 1}, + {"DISABLE+REQUIRE reaches strict rejection", 0, -1, 0, 1, 1, 1}, + {"SSD default stays conservative", 1, -1, 0, 0, 0, 0}, + {"generic ENABLE does not bypass SSD gate", 1, 1, 0, 0, 0, 0}, + {"SSD gate requests the candidate", 1, -1, 1, 0, 0, 1}, + {"SSD DISABLE opts out", 1, -1, 1, 1, 0, 0}, + {"SSD REQUIRE requests strict validation", 1, -1, 0, 0, 1, 1}, + }; + + bool ok = true; + for (const policy_case &test : cases) { + const int got = ds4_rocm_test_q4_prefill_wmma_requested_policy( + test.ssd_streaming, test.enabled, test.ssd_enabled, + test.disabled, test.required); + if (got != test.expected) { + std::fprintf(stderr, + "Q4 WMMA request policy %s: expected=%d got=%d " + "FAIL\n", + test.label, test.expected, got); + ok = false; + } + } + const int optional_q8_yield = + ds4_rocm_test_q4_prefill_wmma_yields_to_q8_wave32(1, 0); + const int absent_q8_yield = + ds4_rocm_test_q4_prefill_wmma_yields_to_q8_wave32(0, 0); + const int strict_wmma_yield = + ds4_rocm_test_q4_prefill_wmma_yields_to_q8_wave32(1, 1); + if (optional_q8_yield != 1 || absent_q8_yield != 0 || + strict_wmma_yield != 0) { + std::fprintf(stderr, + "Q4 WMMA/Q8 precedence: optional=%d absent=%d " + "strict=%d FAIL\n", + optional_q8_yield, absent_q8_yield, + strict_wmma_yield); + ok = false; + } + std::fprintf(stderr, + "ROCm Q4 WMMA request policy oracle: cases=%zu " + "q8_yield=%d/%d/%d %s\n", + sizeof(cases) / sizeof(cases[0]), optional_q8_yield, + absent_q8_yield, strict_wmma_yield, + ok ? "PASS" : "FAIL"); + return ok; +} + bool run_prefill_k1024_tile4_policy_oracle() { struct policy_case { const char *label; @@ -2668,7 +2737,7 @@ bool run_prefill_wmma_smoke(const aligned_model &model) { (void)unsetenv(kPrefillK1024Tile4Require); (void)unsetenv(kPrefillWmmaEnable); (void)unsetenv(kPrefillWmmaSsdEnable); - (void)unsetenv(kPrefillWmmaDisable); + (void)setenv(kPrefillWmmaDisable, "1", 1); (void)unsetenv(kPrefillWmmaRequire); (void)unsetenv(kPrefillWmmaRowTile); (void)unsetenv(kPrefillQ8Wave32Enable); @@ -2682,9 +2751,9 @@ bool run_prefill_wmma_smoke(const aligned_model &model) { ds4_rocm_test_q4_prefill_wmma_get_calls(); (void)unsetenv(kPrefillRequire); - (void)setenv(kPrefillWmmaEnable, "1", 1); + (void)unsetenv(kPrefillWmmaEnable); (void)unsetenv(kPrefillWmmaDisable); - (void)setenv(kPrefillWmmaRequire, "1", 1); + (void)unsetenv(kPrefillWmmaRequire); ds4_rocm_test_q4_prefill_wmma_reset(); const int wmma_rc = ds4_gpu_matmul_quant_tensor( wmma_gpu.ptr, model.data, model.size, model.weight0_offset, kQ4Type, @@ -2714,20 +2783,47 @@ bool run_prefill_wmma_smoke(const aligned_model &model) { } (void)setenv(kPrefillWmmaDisable, "1", 1); + (void)unsetenv(kPrefillWmmaRequire); + if (!write_tensor(wmma_gpu.ptr, sentinel)) return false; + ds4_rocm_test_q4_prefill_wmma_reset(); + const int opt_out_rc = ds4_gpu_matmul_quant_tensor( + wmma_gpu.ptr, model.data, model.size, model.weight0_offset, kQ4Type, + kK, kM0, x_gpu.ptr, n_tokens); + const uint64_t opt_out_wmma_calls = + ds4_rocm_test_q4_prefill_wmma_get_calls(); + std::vector opt_out(allocation_count); + const bool opt_out_read = opt_out_rc != 0 && opt_out_wmma_calls == 0u && + read_tensor(wmma_gpu.ptr, &opt_out); + ok = opt_out_read && ok; + if (opt_out_read) { + ok = output_guard_unchanged( + opt_out, sentinel, logical_count, + "direct-WMMA opt-out output canary") && ok; + opt_out.resize(logical_count); + ok = bitwise_equal(opt_out, tile8, + "direct-WMMA opt-out vs TILE8") && ok; + } + + (void)setenv(kPrefillWmmaRequire, "1", 1); if (!write_tensor(wmma_gpu.ptr, sentinel)) return false; + ds4_rocm_test_q4_prefill_wmma_reset(); const int rejected_rc = ds4_gpu_matmul_quant_tensor( wmma_gpu.ptr, model.data, model.size, model.weight0_offset, kQ4Type, kK, kM0, x_gpu.ptr, n_tokens); - ok = rejected_rc == 0 && + const uint64_t rejected_wmma_calls = + ds4_rocm_test_q4_prefill_wmma_get_calls(); + ok = rejected_rc == 0 && rejected_wmma_calls == 0u && unchanged_after_rejected_call( wmma_gpu.ptr, sentinel, "direct-WMMA DISABLE+REQUIRE preserves output") && ok; std::fprintf(stderr, - "ROCm Q4 direct-WMMA prefill: tile8=%d/%llu wmma=%d/%llu " - "rejected=%d %s\n", + "ROCm Q4 direct-WMMA prefill: tile8=%d/%llu " + "default=%d/%llu opt_out=%d/%llu rejected=%d/%llu %s\n", tile8_rc, (unsigned long long)tile8_wmma_calls, wmma_rc, (unsigned long long)wmma_calls, - rejected_rc, ok ? "PASS" : "FAIL"); + opt_out_rc, (unsigned long long)opt_out_wmma_calls, + rejected_rc, (unsigned long long)rejected_wmma_calls, + ok ? "PASS" : "FAIL"); return ok; } @@ -2796,7 +2892,7 @@ bool run_attention_output_wmma_smoke(const aligned_model &model) { (void)unsetenv(kPrefillK1024Tile4Require); (void)unsetenv(kPrefillWmmaEnable); (void)unsetenv(kPrefillWmmaSsdEnable); - (void)unsetenv(kPrefillWmmaDisable); + (void)setenv(kPrefillWmmaDisable, "1", 1); (void)unsetenv(kPrefillWmmaRequire); (void)unsetenv(kPrefillWmmaRowTile); ds4_rocm_test_q4_prefill_wmma_reset(); @@ -2810,9 +2906,9 @@ bool run_attention_output_wmma_smoke(const aligned_model &model) { ds4_rocm_test_q4_prefill_wmma_get_calls(); (void)unsetenv(kPrefillRequire); - (void)setenv(kPrefillWmmaEnable, "1", 1); + (void)unsetenv(kPrefillWmmaEnable); (void)unsetenv(kPrefillWmmaDisable); - (void)setenv(kPrefillWmmaRequire, "1", 1); + (void)unsetenv(kPrefillWmmaRequire); ds4_rocm_test_q4_prefill_wmma_reset(); const int wmma_rc = ds4_gpu_attention_output_q4_K_batch_tensor( wmma_out.ptr, wmma_low.ptr, nullptr, nullptr, @@ -2982,7 +3078,8 @@ int main(int argc, char **argv) { (void)unsetenv(kGroupedDecodeRequire); (void)unsetenv(kGroupedDecodeStats); - const bool policy_ok = run_prefill_wmma_row_tile_policy_oracle() && + const bool policy_ok = run_prefill_wmma_requested_policy_oracle() && + run_prefill_wmma_row_tile_policy_oracle() && run_prefill_k1024_tile4_policy_oracle() && run_prefill_q8_wave32_policy_oracle() && run_pair_pre_enqueue_policy_oracle(); From 95f4c3cbad43bc165126b21e8361e6ac05096688 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Sun, 30 Aug 2026 15:43:10 +0200 Subject: [PATCH 173/189] Coalesce CUDA Q4 16-warp activation loads --- cuda/mmq/ds4_mmq_q4_16warp.cu | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/cuda/mmq/ds4_mmq_q4_16warp.cu b/cuda/mmq/ds4_mmq_q4_16warp.cu index 6601f9881..92a179bdb 100644 --- a/cuda/mmq/ds4_mmq_q4_16warp.cu +++ b/cuda/mmq/ds4_mmq_q4_16warp.cu @@ -48,6 +48,7 @@ constexpr size_t kWeightTileBytes = constexpr size_t kYTileBytes = (size_t)kNTile * sizeof(block_q8_1_mmq); constexpr size_t kSharedBytes = kWeightTileBytes + kYTileBytes; +constexpr int kYTileVectors = (int)(kYTileBytes / sizeof(int4)); static_assert(kWarps == 16, "Q4 16-warp decomposition changed"); static_assert(kThreads == 512, "Q4 16-warp CTA must have 512 threads"); @@ -57,6 +58,8 @@ static_assert(kNFragPerWarp == 4, "Q4 split-N fragment count changed"); static_assert(kWeightStride == 76, "canonical Q4_K MMA row stride changed"); static_assert(kYStrideInts == 36, "canonical Q8_1 DS4 stride changed"); static_assert(kYChunks16 == 9, "canonical Q8_1 DS4 block size changed"); +static_assert(kYTileBytes % sizeof(int4) == 0, + "Q8_1 tile must support vectorized copies"); static_assert(MMQ_ITER_K % QK_K == 0 && MMQ_ITER_K / QK_K == 1, "Stream-K scheduler must restore canonical K alignment"); static_assert(kSharedBytes == 57344, "Q4 16-warp shared-memory model changed"); @@ -134,6 +137,26 @@ __device__ __forceinline__ void load_y_tile( int col0, int k128) { const int tid = linear_tid(); + + // The production selector admits complete N128 tiles. Copy those as one + // contiguous vector range, matching canonical MMQ's flat cooperative + // load. The former column-major mapping made every warp issue 144-byte- + // strided global loads; flattening turns each warp's accesses into adjacent + // 16-byte vectors while preserving the shared representation byte-for-byte. + if (col0 <= N - kNTile) { + const int4 * __restrict__ src = reinterpret_cast( + q8 + (uint64_t)k128 * (uint64_t)N + (uint64_t)col0); + int4 * __restrict__ dst = reinterpret_cast(tile); +#pragma unroll + for (int vector = tid; vector < kYTileVectors; + vector += kThreads) { + dst[vector] = src[vector]; + } + return; + } + + // Keep the guarded per-column copy for the N tail accepted by the direct + // oracle hook. Production never takes this path. constexpr int threads_per_col = kThreads / kNTile; static_assert(threads_per_col == 4, "Q8_1 DS4 copy mapping changed"); From 5968837a7371b48336ad2a9b2ccc8f0216a08375 Mon Sep 17 00:00:00 2001 From: kyuz0 Date: Sat, 29 Aug 2026 21:35:28 +0100 Subject: [PATCH 174/189] fix(rocm): stabilize gfx1151 MMQ routing maps (cherry picked from commit c012d5cedfcf9fd23e5ed6718247ec9a16f6bbe5) --- cuda/mmq/ds4_mmq.cu | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/cuda/mmq/ds4_mmq.cu b/cuda/mmq/ds4_mmq.cu index 28efff074..c87064f8d 100644 --- a/cuda/mmq/ds4_mmq.cu +++ b/cuda/mmq/ds4_mmq.cu @@ -1429,7 +1429,7 @@ int ds4_mmq_dense_impl( ggml_cuda_pool_alloc fixup(ctx->pool()); if (fixup_bytes != 0u) fixup.alloc(fixup_bytes); const int rc = ds4_mmq_q4_K_dense_16warp_streamk_enqueue( - W, src1_q8_1.get(), out_f32, fixup.get(), fixup_bytes, + W, src1_q8_1, out_f32, fixup.get(), fixup_bytes, M, N, K, nsm, stream); if (rc != 0) { fprintf(stderr, @@ -2710,15 +2710,18 @@ int ds4_mmq_moe_pair_impl( } } - if (persistent_pair_maps) { - const auto & maps = g_mmq_pair_maps[dev]; - ids_src1 = maps.ids_src1; - ids_dst = maps.ids_dst; - expert_bounds = maps.expert_bounds; - } else if (!direct_gateup_q8) { - ids_src1 = ids_src1_alloc.alloc(ctx->pool(), ne_get_rows); - ids_dst = ids_dst_alloc.alloc(ctx->pool(), ne_get_rows); - expert_bounds = expert_bounds_alloc.alloc(ctx->pool(), n_experts + 1); + if (!direct_gateup_q8) { + if (persistent_pair_maps) { + const auto & maps = g_mmq_pair_maps[dev]; + ids_src1 = maps.ids_src1; + ids_dst = maps.ids_dst; + expert_bounds = maps.expert_bounds; + } else { + ids_src1 = ids_src1_alloc.alloc(ctx->pool(), ne_get_rows); + ids_dst = ids_dst_alloc.alloc(ctx->pool(), ne_get_rows); + expert_bounds = expert_bounds_alloc.alloc( + ctx->pool(), n_experts + 1); + } } const int si1 = n_expert_used; From 58c2112f258bf915e02e94b8bf47402c01fdc5bb Mon Sep 17 00:00:00 2001 From: Donato Capitella Date: Sun, 30 Aug 2026 12:40:52 +0100 Subject: [PATCH 175/189] server: support fixed-length greedy decode (cherry picked from commit e1147cdcbd1cf2fa6191cd7d3b684ab3e59fdf66) --- ds4.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ds4.c b/ds4.c index 88637bd5f..393400b87 100644 --- a/ds4.c +++ b/ds4.c @@ -77422,7 +77422,7 @@ static int ds4_session_eval_speculative_argmax_impl( accepted[0] = first_token; return 1; } - if (s->engine->glm_mtp && DS4_N_NEXTN_PREDICT != 0 && + if (!ignore_eos && s->engine->glm_mtp && DS4_N_NEXTN_PREDICT != 0 && s->glm_graph_ready) { if (ds4_session_tp_leader(s)) { ds4_engine *ge = s->engine; @@ -77532,7 +77532,7 @@ static int ds4_session_eval_speculative_argmax_impl( errlen); } - if (metal_graph_cuda_splitkv_spec_requested() && + if (!ignore_eos && metal_graph_cuda_splitkv_spec_requested() && (!e->mtp_ready || e->mtp_draft_tokens <= 1)) { int extra = ds4_session_eval_splitkv_spec_after_first( s, From d29f53b23c3912a2984795139210482cc3bbcb33 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Sun, 30 Aug 2026 20:10:58 +0200 Subject: [PATCH 176/189] metal: optimize Q4 attention output prefill routing Use a fixed-route direct kernel for the production Q4 attention output-A shape on Apple M1-M4, avoiding the generic route-map and work-list path. Add fail-closed controls, a production-shape oracle, and a resident GPU-only benchmark. --- ENVIRONMENT_VARIABLES.md | 12 +- Makefile | 26 +- ds4_metal.m | 126 ++- metal/moe.metal | 134 +++ scripts/environment_variables.tsv | 8 +- speed-bench/README.md | 22 + .../metal_q4_attn_out_a_direct_bench.m | 995 ++++++++++++++++++ tests/test_metal_q4_attn_out_a_direct.c | 539 ++++++++++ 8 files changed, 1847 insertions(+), 15 deletions(-) create mode 100644 speed-bench/metal_q4_attn_out_a_direct_bench.m create mode 100644 tests/test_metal_q4_attn_out_a_direct.c diff --git a/ENVIRONMENT_VARIABLES.md b/ENVIRONMENT_VARIABLES.md index 4623a8c45..5415363b6 100644 --- a/ENVIRONMENT_VARIABLES.md +++ b/ENVIRONMENT_VARIABLES.md @@ -43,6 +43,8 @@ changing them. Unless a row says otherwise: | `DS4_METAL_ENABLE_Q4_PREFILL_PAIR_F16_RHS=1` | On Apple M1–M4, opt into the Q-A/KV prefill pair that materializes their shared F32 activation as F16 once for exact 32-token tiles through N=128. | | `DS4_METAL_DISABLE_Q4_PREFILL_PAIR_F16_RHS=1` | Dominant rollback to two standalone Q4_K/F32-RHS prefill projections. | | `DS4_METAL_REQUIRE_Q4_PREFILL_PAIR_F16_RHS=1` | Request the shared-F16-RHS pair and fail closed if its device, shape, storage, or buffer contract is unavailable. Intended for strict A/B oracles. | +| `DS4_METAL_DISABLE_Q4_ATTN_OUT_A_DIRECT=1` | Restore the generic route-map/work-list Q4 attention output-A path. Unset uses the bit-identical fixed-route direct kernel by default for eligible Apple M1–M4 `4096 -> 1024`, eight-group prefills at N=512–4096. Any defined value, including `0`, disables the specialization. | +| `DS4_METAL_REQUIRE_Q4_ATTN_OUT_A_DIRECT=1` | Require the fixed-route Q4 attention output-A kernel and fail closed before dispatch if its device, shape, concurrency, pipeline, or memory contract is unavailable. Intended for production-shape correctness and performance oracles; `DISABLE` wins. | | `DS4_METAL_DISABLE_M1_IQ2_MID_ONLY=1` | Restore the canonical IQ2 address-table gate/up producer on the exact M1 SSD-streaming decode shape. The specialization is automatic by default. | | `DS4_METAL_REQUIRE_M1_IQ2_MID_ONLY=1` | Fail closed when an otherwise eligible M1 IQ2 mid-only dispatch cannot use the specialization. | | `DS4_METAL_ENABLE_M1_IQ2_MID_ONLY=1` | Legacy spelling retained for migration notes only. The runtime does not read it; the path is automatic and this setting is ignored. | @@ -150,13 +152,13 @@ above, it is an unstable internal diagnostic or tuning interface. The linked sou remains normative for exact eligibility gates, bounds, and architecture-specific defaults. -Inventory totals: **1077 `DS4_*` runtime variables** and +Inventory totals: **1079 `DS4_*` runtime variables** and **6 external runtime variables**. The auxiliary inventories contain **118 test/test-fixture entries** and **19 tool/wrapper entries**.
-Metal (445) +Metal (447) | Variable | Accepted value and default | Effect | Source | | --- | --- | --- | --- | @@ -280,8 +282,9 @@ and **19 tool/wrapper entries**. | `DS4_METAL_DISABLE_PRE_M5_ROUTER_SIMD_WEIGHTS_FUSION` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 router simd weights fusion. | [ds4_metal.m:35836](ds4_metal.m#L35836) | | `DS4_METAL_DISABLE_PRE_M5_ROUTER_TRANSFORM_FINALIZE_FUSION` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 router transform finalize fusion. | [ds4_metal.m:35840](ds4_metal.m#L35840) | | `DS4_METAL_DISABLE_PRO_Q4_EXPERT_ADDRESS_AUTO` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pro Q4 expert address auto. | [ds4_metal.m:19710](ds4_metal.m#L19710) | -| `DS4_METAL_DISABLE_PRO_Q4_EXPERT_TABLE_AUTO` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pro Q4 expert table auto. | [ds4.c:21040](ds4.c#L21040) | -| `DS4_METAL_DISABLE_PRO_Q4_EXPERT_TABLE_PRELOAD` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pro Q4 expert table preload. | [ds4.c:58780](ds4.c#L58780) | +| `DS4_METAL_DISABLE_PRO_Q4_EXPERT_TABLE_AUTO` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pro Q4 expert table auto. | [ds4.c:21568](ds4.c#L21568) | +| `DS4_METAL_DISABLE_PRO_Q4_EXPERT_TABLE_PRELOAD` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pro Q4 expert table preload. | [ds4.c:63088](ds4.c#L63088) | +| `DS4_METAL_DISABLE_Q4_ATTN_OUT_A_DIRECT` | presence rollback; unset enables the automatic fixed-route path for eligible Apple M1-M4 long prefills; any defined value including 0 disables | Restore the generic route-map/work-list Q4 attention output-A path instead of the bit-identical fixed-route direct kernel. | [ds4_metal.m:31562](ds4_metal.m#L31562) | | `DS4_METAL_DISABLE_Q4_ATTN_OUT_HC_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 attn out HC fuse. | [ds4_metal.m:47624](ds4_metal.m#L47624) | | `DS4_METAL_DISABLE_Q4_ATTN_OUT_TINY_BATCH` | value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables | Disables Q4 attn out tiny batch. | [ds4_metal.m:28396](ds4_metal.m#L28396) | | `DS4_METAL_DISABLE_Q4_BATCH_EXPERT_TABLE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 batch expert table. | [ds4_metal.m:44883](ds4_metal.m#L44883) | @@ -540,6 +543,7 @@ and **19 tool/wrapper entries**. | `DS4_METAL_REQUIRE_IQ2_XXS_SSD_PREFILL_MM` | value-aware boolean; default implicit fail-closed only with complete selected-address domain; explicit 1 is strict, 0 permits fallback | Makes eligible IQ2_XXS/Q2_K grouped SSD-prefill MM fail closed. | [ds4_metal.m:44782](ds4_metal.m#L44782) | | `DS4_METAL_REQUIRE_M1_IQ2_MID_ONLY` | presence strict check; unset: fallback allowed; any value including 0 requires the path | Requires M1 IQ2 mid only and makes eligible fallback fail closed. | [ds4_metal.m:42074](ds4_metal.m#L42074) | | `DS4_METAL_REQUIRE_OUTPUT_HC_WEIGHTS4` | presence strict check; unset: fallback allowed; any value including 0 requires the path | Requires output HC weights4 and makes eligible fallback fail closed. | [ds4_metal.m:46789](ds4_metal.m#L46789) | +| `DS4_METAL_REQUIRE_Q4_ATTN_OUT_A_DIRECT` | presence strict check; unset permits the automatic path or fallback; any defined value including 0 requires the direct kernel and DISABLE wins | Require the bit-identical fixed-route Q4 attention output-A kernel and fail closed before dispatch when its production contract is unavailable. | [ds4_metal.m:31560](ds4_metal.m#L31560) | | `DS4_METAL_REQUIRE_Q4_ATTN_OUT_TINY_BATCH` | value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables | Requires Q4 attn out tiny batch and makes eligible fallback fail closed. | [ds4_metal.m:28373](ds4_metal.m#L28373) | | `DS4_METAL_REQUIRE_Q4_SSD_PREFILL_ATTN_OUT_EXACTN` | value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables | Requires Q4 SSD prefill attn out exactn and makes eligible fallback fail closed. | [ds4_metal.m:28089](ds4_metal.m#L28089) | | `DS4_METAL_REQUIRE_Q4_SSD_PREFILL_ATTN_OUT_SCALE_META` | value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables | Requires shared scale/min metadata in the Q4 SSD prefill attention-output exact-N kernel and makes fallback fail closed. | [ds4_metal.m:28209](ds4_metal.m#L28209) | diff --git a/Makefile b/Makefile index e92db436e..4ca3a045c 100644 --- a/Makefile +++ b/Makefile @@ -69,7 +69,7 @@ DS4_LINK_LIBS ?= $(CUDA_LDLIBS) METAL_LDLIBS := $(LDLIBS) endif -.PHONY: all help clean test test-ssd environment-docs test-quantizer-indexer-q4 test-rocm test-glm53-kda-rocm test-metal-session-batch test-metal-session-batch-ssd test-metal-q4-streams test-metal-q4-prefill-pair test-metal-indexer-q4 test-metal-q4-attn-exactn test-metal-q4-qb-f16-cache test-metal-q4-qb-f16-cache-timing test-metal-exactn-oracle test-metal-dspark-capture test-metal-argmax-top1 bench-metal-argmax-top1 test-metal-iq2-midonly test-metal-iq2-ssd-grouped-mm test-metal-iq2-live-index test-mxfp4-metal test-mxfp4-cuda test-mxfp4-rocm test-mmq-parity-cuda test-mmq-q4-16warp-cuda test-rocm-q4-parity test-rocm-q4-dense test-rocm-q4-pair test-rocm-q4-prefill test-strix-rocm-q4-parity test-strix-rocm-q4-prefill test-strix-rocm-q4-prefill-long test-cuda-session-batch test-cuda-mixed-batch dspark-acceptance dspark-verify-depth rocm-dspark-acceptance rocm-dspark-verify-depth mtp-verify-depth cpu cuda cuda-spark cuda-generic cuda-regression strix-halo rocm cuda-iq2-moe-prefill-bench cuda-q4-prefill-bench rocm-iq2-moe-prefill-bench rocm-q4-prefill-bench +.PHONY: all help clean test test-ssd environment-docs test-quantizer-indexer-q4 test-rocm test-glm53-kda-rocm test-metal-session-batch test-metal-session-batch-ssd test-metal-q4-streams test-metal-q4-prefill-pair test-metal-indexer-q4 test-metal-q4-attn-exactn test-metal-q4-attn-out-a-direct test-metal-q4-qb-f16-cache test-metal-q4-qb-f16-cache-timing test-metal-exactn-oracle test-metal-dspark-capture test-metal-argmax-top1 bench-metal-argmax-top1 test-metal-iq2-midonly test-metal-iq2-ssd-grouped-mm test-metal-iq2-live-index test-mxfp4-metal test-mxfp4-cuda test-mxfp4-rocm test-mmq-parity-cuda test-mmq-q4-16warp-cuda test-rocm-q4-parity test-rocm-q4-dense test-rocm-q4-pair test-rocm-q4-prefill test-strix-rocm-q4-parity test-strix-rocm-q4-prefill test-strix-rocm-q4-prefill-long test-cuda-session-batch test-cuda-mixed-batch dspark-acceptance dspark-verify-depth rocm-dspark-acceptance rocm-dspark-verify-depth mtp-verify-depth cpu cuda cuda-spark cuda-generic cuda-regression strix-halo rocm cuda-iq2-moe-prefill-bench cuda-q4-prefill-bench rocm-iq2-moe-prefill-bench rocm-q4-prefill-bench gguf-tools/deepseek4-quantize: gguf-tools/deepseek4-quantize.c gguf-tools/quants.c gguf-tools/quants.h $(MAKE) -C gguf-tools deepseek4-quantize @@ -81,7 +81,7 @@ test-quantizer-indexer-q4: gguf-tools/deepseek4-quantize tests/test_quantizer_in ./tests/test_quantizer_indexer_q4 ./gguf-tools/deepseek4-quantize ifeq ($(UNAME_S),Darwin) -.PHONY: metal-decode-schedule-bench metal-prefill-variant-bench metal-q4-dense-pair-bench metal-q4-prefill-pair-bench metal-q4-mm-tail-cull-bench metal-iq2-moe-tail-cull-bench metal-iq2-moe-top8-pair-bench check-mxfp4-half-lut test-mxfp4-metal +.PHONY: metal-decode-schedule-bench metal-prefill-variant-bench metal-q4-dense-pair-bench metal-q4-prefill-pair-bench metal-q4-mm-tail-cull-bench metal-q4-attn-out-a-direct-bench metal-iq2-moe-tail-cull-bench metal-iq2-moe-top8-pair-bench check-mxfp4-half-lut test-mxfp4-metal all: ds4 ds4-server ds4-bench ds4-eval ds4-agent @@ -98,6 +98,7 @@ help: @echo " make test-metal-q4-prefill-pair Runtime oracle for the M1-M4 Q4 prefill pair" @echo " make test-metal-indexer-q4 Check the production-shape Q4_K indexer projection" @echo " make test-metal-q4-attn-exactn Bitwise/canary oracle for M1-M4 SSD-prefill Q4 attention output" + @echo " make test-metal-q4-attn-out-a-direct Production-shape oracle for M1-M4 Q4 output-A direct routing" @echo " make test-metal-q4-qb-f16-cache Oracle for M1-M4 Q4 q_b sidecar and transient F16 paths" @echo " make test-metal-q4-qb-f16-cache-timing Compare Q4 direct, sidecar, and transient production at N=4096" @echo " make test-metal-dspark-capture Check fused DSpark HC capture bitwise" @@ -112,6 +113,7 @@ help: @echo " make metal-q4-dense-pair-bench Build the resident Q4 decode pair kernel benchmark" @echo " make metal-q4-prefill-pair-bench Build the resident Q4 prefill pair F16-RHS benchmark" @echo " make metal-q4-mm-tail-cull-bench Build the resident Q4 prefill tail-cull kernel benchmark" + @echo " make metal-q4-attn-out-a-direct-bench Build the resident Q4 attention output-A direct benchmark" @echo " make metal-iq2-moe-tail-cull-bench Build the resident IQ2 pair MoE tail-cull benchmark" @echo " make metal-iq2-moe-top8-pair-bench Build the resident GLM-shape IQ2 top-8 pair-fusion benchmark" @echo " make check-mxfp4-half-lut Verify the checked-in MXFP4 half LUT matches the generator" @@ -232,6 +234,19 @@ test-metal-q4-attn-exactn: tests/test_metal_q4_attn_exactn -u DS4_METAL_DISABLE_Q4_MV_CLASSIC \ ./tests/test_metal_q4_attn_exactn +tests/test_metal_q4_attn_out_a_direct.o: tests/test_metal_q4_attn_out_a_direct.c ds4_gpu.h + $(CC) $(CFLAGS) -I. -c -o $@ $< + +tests/test_metal_q4_attn_out_a_direct: tests/test_metal_q4_attn_out_a_direct.o ds4_metal.o + $(CC) $(CFLAGS) -o $@ $^ $(METAL_LDLIBS) + +test-metal-q4-attn-out-a-direct: tests/test_metal_q4_attn_out_a_direct + env -u DS4_METAL_DISABLE_Q4_ATTN_OUT_A_DIRECT \ + -u DS4_METAL_REQUIRE_Q4_ATTN_OUT_A_DIRECT \ + -u DS4_METAL_DISABLE_Q4_ATTN_OUT_B_F16_RHS \ + -u DS4_METAL_REQUIRE_Q4_ATTN_OUT_B_F16_RHS \ + ./tests/test_metal_q4_attn_out_a_direct + tests/test_metal_q4_qb_f16_cache.o: tests/test_metal_q4_qb_f16_cache.c ds4_gpu.h $(CC) $(CFLAGS) -I. -c -o $@ $< @@ -367,6 +382,11 @@ speed-bench/metal_q4_mm_tail_cull_bench: speed-bench/metal_q4_mm_tail_cull_bench metal-q4-mm-tail-cull-bench: speed-bench/metal_q4_mm_tail_cull_bench +speed-bench/metal_q4_attn_out_a_direct_bench: speed-bench/metal_q4_attn_out_a_direct_bench.m $(METAL_SRCS) + $(CC) $(OBJCFLAGS) -o $@ $< $(METAL_LDLIBS) + +metal-q4-attn-out-a-direct-bench: speed-bench/metal_q4_attn_out_a_direct_bench + speed-bench/metal_iq2_moe_tail_cull_bench.o: speed-bench/metal_iq2_moe_tail_cull_bench.c ds4_gpu.h $(CC) $(CFLAGS) -I. -c -o $@ $< @@ -1018,4 +1038,4 @@ mxfp4-dot-test: tests/test_mxfp4_dot.c clean: rm -f speed-bench/metal_iq2_moe_top8_pair_bench - rm -f ds4 ds4-server ds4-bench ds4-eval ds4-agent ds4_cpu ds4_native ds4_server_test ds4_test ds4_agent_test gguf-tools/quality-testing/score_official gguf-tools/quality-testing/score_official.o speed-bench/metal_decode_schedule_bench speed-bench/metal_prefill_variant_bench speed-bench/metal_q4_dense_pair_bench speed-bench/metal_q4_prefill_pair_bench speed-bench/metal_q4_mm_tail_cull_bench speed-bench/metal_iq2_moe_tail_cull_bench speed-bench/gpu_iq2_moe_prefill_bench_rocm speed-bench/gpu_iq2_moe_prefill_bench_cuda speed-bench/rocm_q4_prefill_bench speed-bench/cuda_q4_prefill_bench speed-bench/*.o tests/test_q4k_dot tests/test_mxfp4_dot tests/test_quantizer_indexer_q4 tests/test_mxfp4_metal tests/test_mxfp4_rocm tests/bench_mxfp4_rocm tests/test_mxfp4_cuda tests/test_rocm_q4_dense_pair tests/test_metal_session_batch tests/test_metal_q4_streams tests/test_metal_q4_prefill_pair tests/test_metal_indexer_q4 tests/test_metal_q4_attn_exactn tests/test_metal_q4_qb_f16_cache tests/test_metal_exactn_oracle tests/test_metal_dspark_capture tests/test_metal_argmax_top1 tests/test_metal_iq2_midonly tests/test_metal_iq2_ssd_grouped_mm tests/test_metal_iq2_live_index tests/test_glm53_kda tests/test_glm53_kda_rocm tests/test_glm53_vision_engine tests/test_glm53_vision_prompt tests/test_gpu_xdev tests/test_gpu_model_cache tests/test_gpu_lookup_cache_strict tests/test_engine_mgpu_refusal tests/test_engine_mgpu_runtime tests/test_engine_correctness tests/test_sampling tests/test_cuda_session_batch tests/test_cuda_mixed_batch tests/*.o *.o cuda/mmq/*.o cuda/mmq/test/*.o tests/cuda_long_context_smoke tests/cuda_long_context_smoke.o + rm -f ds4 ds4-server ds4-bench ds4-eval ds4-agent ds4_cpu ds4_native ds4_server_test ds4_test ds4_agent_test gguf-tools/quality-testing/score_official gguf-tools/quality-testing/score_official.o speed-bench/metal_decode_schedule_bench speed-bench/metal_prefill_variant_bench speed-bench/metal_q4_dense_pair_bench speed-bench/metal_q4_prefill_pair_bench speed-bench/metal_q4_mm_tail_cull_bench speed-bench/metal_q4_attn_out_a_direct_bench speed-bench/metal_iq2_moe_tail_cull_bench speed-bench/gpu_iq2_moe_prefill_bench_rocm speed-bench/gpu_iq2_moe_prefill_bench_cuda speed-bench/rocm_q4_prefill_bench speed-bench/cuda_q4_prefill_bench speed-bench/*.o tests/test_q4k_dot tests/test_mxfp4_dot tests/test_quantizer_indexer_q4 tests/test_mxfp4_metal tests/test_mxfp4_rocm tests/bench_mxfp4_rocm tests/test_mxfp4_cuda tests/test_rocm_q4_dense_pair tests/test_metal_session_batch tests/test_metal_q4_streams tests/test_metal_q4_prefill_pair tests/test_metal_indexer_q4 tests/test_metal_q4_attn_exactn tests/test_metal_q4_attn_out_a_direct tests/test_metal_q4_qb_f16_cache tests/test_metal_exactn_oracle tests/test_metal_dspark_capture tests/test_metal_argmax_top1 tests/test_metal_iq2_midonly tests/test_metal_iq2_ssd_grouped_mm tests/test_metal_iq2_live_index tests/test_glm53_kda tests/test_glm53_kda_rocm tests/test_glm53_vision_engine tests/test_glm53_vision_prompt tests/test_gpu_xdev tests/test_gpu_model_cache tests/test_gpu_lookup_cache_strict tests/test_engine_mgpu_refusal tests/test_engine_mgpu_runtime tests/test_engine_correctness tests/test_sampling tests/test_cuda_session_batch tests/test_cuda_mixed_batch tests/*.o *.o cuda/mmq/*.o cuda/mmq/test/*.o tests/cuda_long_context_smoke tests/cuda_long_context_smoke.o diff --git a/ds4_metal.m b/ds4_metal.m index 54ff5c203..717d50d1c 100644 --- a/ds4_metal.m +++ b/ds4_metal.m @@ -6401,6 +6401,17 @@ static int ds4_gpu_encode_attn_out_low_mpp( id dst, NSUInteger dst_off); +static int ds4_gpu_encode_attn_out_low_q4_direct( + id cb, + id pipeline, + const ds4_gpu_mul_mm_id_args *mm_args, + id src0, + NSUInteger src0_off, + id src1, + NSUInteger src1_off, + id dst, + NSUInteger dst_off); + static ds4_gpu_mul_mm_id_map_args ds4_gpu_make_mul_mm_id_map_args( uint32_t src0_cols, uint32_t src0_experts, @@ -31545,8 +31556,13 @@ static int ds4_gpu_attention_output_q4_K_batch_impl( ds4_gpu_env_bool("DS4_METAL_REQUIRE_Q4_ATTN_OUT_B_F16_RHS") == 1; const bool disable_f16_rhs = ds4_gpu_env_bool("DS4_METAL_DISABLE_Q4_ATTN_OUT_B_F16_RHS") == 1; + const bool require_direct_low = + getenv("DS4_METAL_REQUIRE_Q4_ATTN_OUT_A_DIRECT") != NULL; + const bool disable_direct_low = + getenv("DS4_METAL_DISABLE_Q4_ATTN_OUT_A_DIRECT") != NULL; - if (!require_f16_rhs && n_tokens >= 6u && n_tokens <= 31u) { + if (!require_f16_rhs && !require_direct_low && + n_tokens >= 6u && n_tokens <= 31u) { const int exactn_rc = ds4_gpu_attention_output_q4_K_ssd_prefill_exactn_tensor( out, @@ -31569,7 +31585,8 @@ static int ds4_gpu_attention_output_q4_K_batch_impl( const bool require_tiny = tiny_scope && ds4_gpu_env_bool("DS4_METAL_REQUIRE_Q4_ATTN_OUT_TINY_BATCH") == 1; - const int failure_rc = (require_tiny || require_f16_rhs) ? -1 : 0; + const int failure_rc = + (require_tiny || require_f16_rhs || require_direct_low) ? -1 : 0; if (!g_initialized && !ds4_gpu_init()) return failure_rc; if (!out || !low || !heads || !model_map || group_dim == 0 || rank == 0 || n_groups == 0 || out_dim == 0 || n_tokens == 0 || @@ -31768,9 +31785,62 @@ static int ds4_gpu_attention_output_q4_K_batch_impl( use_mpp_low = false; } } + + /* On pre-M5 GPUs the production attention output-A route is fixed: + * slot g always consumes Q4 weight group g. The generic MoE kernel + * nevertheless builds and reads an expert-major route map and work + * list before scattering back to the same token/group layout. The + * direct kernel removes only those indirections; its dequantization, + * F16 staging, MMA order, and final stores are bit-identical. Keep + * the default to the measured long-prefill range and leave Metal4 on + * its cooperative-tensor path. */ + const bool direct_low_eligible = + !use_mpp_low && !use_tiny_exact && + n_tokens >= 512u && n_tokens <= 4096u && + group_dim == 4096u && rank == 1024u && n_groups == 8u && + ds4_gpu_device_is_pre_m5_apple_silicon() && + g_tp_split_world == 1 && + !g_batch_encoder_concurrent; + bool use_direct_low = + direct_low_eligible && !disable_direct_low; + id direct_low_pipeline = nil; + if (use_direct_low) { + direct_low_pipeline = ds4_gpu_get_pipeline( + "kernel_attn_out_low_q4_K_legacy_direct"); + const NSUInteger max_tgmem = + (NSUInteger)[g_device maxThreadgroupMemoryLength]; + const NSUInteger static_tgmem = direct_low_pipeline + ? direct_low_pipeline.staticThreadgroupMemoryLength + : 0u; + if (!direct_low_pipeline || + direct_low_pipeline.threadExecutionWidth != 32u || + direct_low_pipeline.maxTotalThreadsPerThreadgroup < 128u || + static_tgmem > max_tgmem || + 8192u > max_tgmem - static_tgmem) { + use_direct_low = false; + } + } + if (require_direct_low && !use_direct_low) { + fprintf(stderr, + "ds4: required Metal Q4 attention output-A direct path " + "is ineligible (rows=%u shape=%llux%llux%u disabled=%u " + "pre_m5=%u tp_world=%d concurrent=%u mpp=%u tiny=%u)\n", + n_tokens, + (unsigned long long)group_dim, + (unsigned long long)rank, + n_groups, + disable_direct_low ? 1u : 0u, + ds4_gpu_device_is_pre_m5_apple_silicon() ? 1u : 0u, + g_tp_split_world, + g_batch_encoder_concurrent ? 1u : 0u, + use_mpp_low ? 1u : 0u, + use_tiny_exact ? 1u : 0u); + return -1; + } + const int selected_failure_rc = use_direct_low ? -1 : failure_rc; const NSUInteger ids_bytes = (NSUInteger)n_tokens * (NSUInteger)n_groups * sizeof(int32_t); id group_ids_buffer = nil; - if (!use_mpp_low && !use_tiny_exact) { + if (!use_mpp_low && !use_direct_low && !use_tiny_exact) { if (getenv("DS4_METAL_DISABLE_ATTN_OUT_IDS_CACHE") != NULL) { group_ids_buffer = ds4_gpu_new_transient_buffer(ids_bytes, "attention output Q4 group ids"); @@ -31798,7 +31868,9 @@ static int ds4_gpu_attention_output_q4_K_batch_impl( if (!out_a_buf) return failure_rc; const bool had_batch = g_batch_cb != nil; - if (!had_batch && ds4_gpu_begin_commands() == 0) return failure_rc; + if (!had_batch && ds4_gpu_begin_commands() == 0) { + return selected_failure_rc; + } bool ok = true; int owned = 0; @@ -31868,6 +31940,17 @@ static int ds4_gpu_attention_output_q4_K_batch_impl( ds4_gpu_tensor_offset(heads), ds4_gpu_tensor_buffer(low), ds4_gpu_tensor_offset(low)) != 0; + } else if (use_direct_low) { + ok = ds4_gpu_encode_attn_out_low_q4_direct( + cb, + direct_low_pipeline, + &mm_args, + out_a_buf, + (NSUInteger)out_a_inner, + ds4_gpu_tensor_buffer(heads), + ds4_gpu_tensor_offset(heads), + ds4_gpu_tensor_buffer(low), + ds4_gpu_tensor_offset(low)) != 0; } else { ds4_gpu_mul_mm_id_map_args map_args = ds4_gpu_make_mul_mm_id_map_args((uint32_t)group_dim, @@ -31970,7 +32053,7 @@ static int ds4_gpu_attention_output_q4_K_batch_impl( if (!had_batch) { ok = ds4_gpu_end_commands() != 0 && ok; } - return ok ? 1 : failure_rc; + return ok ? 1 : selected_failure_rc; } } @@ -39052,6 +39135,39 @@ static int ds4_gpu_encode_mul_mm_id_mapped( 8192u); } +static int ds4_gpu_encode_attn_out_low_q4_direct( + id cb, + id pipeline, + const ds4_gpu_mul_mm_id_args *mm_args, + id src0, + NSUInteger src0_off, + id src1, + NSUInteger src1_off, + id dst, + NSUInteger dst_off) { + if (!cb || !pipeline || !mm_args || !src0 || !src1 || !dst || + mm_args->ne00 <= 0 || mm_args->ne0 <= 0 || + mm_args->ne02 <= 0 || mm_args->ne1 <= 0 || mm_args->ne21 <= 0) { + return 0; + } + + id enc = ds4_gpu_compute_encoder(cb); + if (!enc) return 0; + [enc setComputePipelineState:pipeline]; + [enc setBytes:mm_args length:sizeof(*mm_args) atIndex:0]; + [enc setBuffer:src0 offset:src0_off atIndex:1]; + [enc setBuffer:src1 offset:src1_off atIndex:2]; + [enc setBuffer:dst offset:dst_off atIndex:3]; + [enc setThreadgroupMemoryLength:8192u atIndex:0]; + [enc dispatchThreadgroups:MTLSizeMake( + ((NSUInteger)mm_args->ne21 + 31u) / 32u, + ((NSUInteger)mm_args->ne0 + 63u) / 64u, + (NSUInteger)mm_args->ne02) + threadsPerThreadgroup:MTLSizeMake(128u, 1u, 1u)]; + ds4_gpu_end_compute_encoder(cb, enc); + return 1; +} + static int ds4_gpu_encode_attn_out_low_mpp( id cb, id pipeline, diff --git a/metal/moe.metal b/metal/moe.metal index 0f5b03368..c61cbfe5f 100644 --- a/metal/moe.metal +++ b/metal/moe.metal @@ -8975,6 +8975,140 @@ kernel void kernel_mul_mm_id( } } +// Fixed-routing Q4_K attention output-A projection for the production +// [token][group][K] -> [token][group][rank] layout. The generic routed +// matmul builds an expert-major map, work list, and scatter description even +// though attention output-A always routes slot g to weight group g. Keep the +// same Q4 dequantization, F32->F16 staging, K-loop order, simdgroup MMA, and +// final scatter arithmetic while deriving that fixed route directly from the +// dispatch coordinates. +kernel void kernel_attn_out_low_q4_K_legacy_direct( + constant ds4_metal_args_mul_mm_id & args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem [[threadgroup(0)]], + uint3 tgpig [[threadgroup_position_in_grid]], + ushort tiitg [[thread_index_in_threadgroup]], + ushort tiisg [[thread_index_in_simdgroup]], + ushort sgitg [[simdgroup_index_in_threadgroup]]) { + constexpr int NR0 = 64; + constexpr int NR1 = 32; + constexpr int NK = 32; + constexpr int NL0 = NK/16; + constexpr int NL1 = NK/8; + constexpr short Q4_NL = 16; + constexpr int SA_BYTES = NR0*NR1*(int)sizeof(half); + + const int group = (int)tgpig.z; + const int r0 = (int)tgpig.y*NR0; + const int r1 = (int)tgpig.x*NR1; + if (group >= args.ne02 || r0 >= args.ne0 || r1 >= args.ne21) return; + + const short nr0 = (args.ne0 - r0 < NR0) ? (args.ne0 - r0) : NR0; + const short nr1 = (args.ne21 - r1 < NR1) ? (args.ne21 - r1) : NR1; + const short lr0 = ((short)tiitg/NL0) < nr0 ? + ((short)tiitg/NL0) : nr0 - 1; + const short lr1 = ((short)tiitg/NL1) < nr1 ? + ((short)tiitg/NL1) : nr1 - 1; + const short il0 = tiitg % NL0; + short il = il0; + + threadgroup half *sa = (threadgroup half *)shmem; + threadgroup half *sb = + (threadgroup half *)(shmem + SA_BYTES); + + const uint64_t offset0 = (uint64_t)group*args.nb02; + const short offset1 = il0/Q4_NL; + device const block_q4_K *x = + (device const block_q4_K *)(src0 + args.nb01*(r0 + lr0) + offset0) + + offset1; + + const short iy = 8*(tiitg % NL1); + device const float *y = (device const float *)(src1 + + args.nb12*(r1 + lr1) + + args.nb11*group + + args.nb10*iy); + + simdgroup_half8x8 ma[4]; + simdgroup_half8x8 mb[2]; + simdgroup_float8x8 mc[8]; + FOR_UNROLL (short i = 0; i < 8; i++) { + mc[i] = make_filled_simdgroup_matrix(0.0f); + } + + for (int loop_k = 0; loop_k < args.ne00; loop_k += NK) { + half4x4 temp_a; + dequantize_q4_K(x, il, temp_a); + + threadgroup_barrier(mem_flags::mem_threadgroup); + + FOR_UNROLL (short i = 0; i < 16; i++) { + const short sx = 2*il0 + i/8; + const short sy = (tiitg/NL0)/8; + const short lx = (tiitg/NL0)%8; + const short ly = i%8; + const short ib = 8*sx + sy; + *(sa + 64*ib + 8*ly + lx) = temp_a[i/4][i%4]; + } + + const short sx = tiitg%NL1; + const short sy = (tiitg/NL1)/8; + const short ly = (tiitg/NL1)%8; + const short ib = 4*sx + sy; + *(threadgroup half2x4 *)(sb + 64*ib + 8*ly) = + half2x4(*((device float2x4 *)y)); + + il = (il + 2 < Q4_NL) ? il + 2 : il % 2; + x = (il < 2) ? x + (2 + Q4_NL - 1)/Q4_NL : x; + y += NK; + + threadgroup_barrier(mem_flags::mem_threadgroup); + + threadgroup const half *lsma = sa + 4*64*(sgitg%2); + threadgroup const half *lsmb = sb + 2*64*(sgitg/2); + FOR_UNROLL (short ik = 0; ik < NK/8; ik++) { + simdgroup_barrier(mem_flags::mem_none); + FOR_UNROLL (short i = 0; i < 4; i++) { + simdgroup_load(ma[i], lsma + 64*i, 8, 0, false); + } + simdgroup_barrier(mem_flags::mem_none); + FOR_UNROLL (short i = 0; i < 2; i++) { + simdgroup_load(mb[i], lsmb + 64*i, 8, 0, false); + } + simdgroup_barrier(mem_flags::mem_none); + FOR_UNROLL (short i = 0; i < 8; i++) { + simdgroup_multiply_accumulate(mc[i], mb[i/4], ma[i%4], mc[i]); + } + lsma += 8*64; + lsmb += 4*64; + } + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + threadgroup float *temp_str = + ((threadgroup float *)shmem) + 32*(sgitg&1) + + (16*(sgitg >> 1))*NR0; + FOR_UNROLL (short i = 0; i < 8; i++) { + simdgroup_store(mc[i], temp_str + 8*(i%4) + + 8*NR0*(i/4), NR0, 0, false); + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + for (short j = sgitg; j < nr1; j += 4) { + device float *D = (device float *)dst + r0 + group*args.ne0 + + (uint64_t)(r1 + j)*args.ne1*args.ne0; + device float4 *D4 = (device float4 *)D; + threadgroup float *C = (threadgroup float *)shmem + j*NR0; + threadgroup float4 *C4 = (threadgroup float4 *)C; + + int i = tiisg; + for (; i < nr0/4; i += 32) D4[i] = C4[i]; + i = 4*(nr0/4) + tiisg; + for (; i < nr0; i += 32) D[i] = C[i]; + } +} + // Address-table variant used by SSD streaming. The routing ids remain the // model's original expert ids, but each expert's resident buffer is found via a // GPU-address table instead of a contiguous full-layer tensor. diff --git a/scripts/environment_variables.tsv b/scripts/environment_variables.tsv index 27cec9a16..fb97eb637 100644 --- a/scripts/environment_variables.tsv +++ b/scripts/environment_variables.tsv @@ -601,9 +601,10 @@ runtime/metal DS4_METAL_DISABLE_PRE_M5_ROUTER_SIMD_FINALIZE presence rollback; u runtime/metal DS4_METAL_DISABLE_PRE_M5_ROUTER_SIMD_WEIGHTS_FUSION presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 router simd weights fusion. ds4_metal.m:35836 runtime/metal DS4_METAL_DISABLE_PRE_M5_ROUTER_TRANSFORM_FINALIZE_FUSION presence rollback; unset: automatic/default path; any value including 0 disables Disables pre M5 router transform finalize fusion. ds4_metal.m:35840 runtime/metal DS4_METAL_DISABLE_PRO_Q4_EXPERT_ADDRESS_AUTO presence rollback; unset: automatic/default path; any value including 0 disables Disables pro Q4 expert address auto. ds4_metal.m:19710 -runtime/metal DS4_METAL_DISABLE_PRO_Q4_EXPERT_TABLE_AUTO presence rollback; unset: automatic/default path; any value including 0 disables Disables pro Q4 expert table auto. ds4.c:21040 -runtime/metal DS4_METAL_DISABLE_PRO_Q4_EXPERT_TABLE_PRELOAD presence rollback; unset: automatic/default path; any value including 0 disables Disables pro Q4 expert table preload. ds4.c:58780 -runtime/metal DS4_METAL_DISABLE_Q4_ATTN_OUT_B_F16_RHS value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Disables resident pre-M5 Q4 attention output-B F16 RHS materialization and restores per-tile F32 staging. ds4_metal.m:28533 +runtime/metal DS4_METAL_DISABLE_PRO_Q4_EXPERT_TABLE_AUTO presence rollback; unset: automatic/default path; any value including 0 disables Disables pro Q4 expert table auto. ds4.c:21568 +runtime/metal DS4_METAL_DISABLE_PRO_Q4_EXPERT_TABLE_PRELOAD presence rollback; unset: automatic/default path; any value including 0 disables Disables pro Q4 expert table preload. ds4.c:63088 +runtime/metal DS4_METAL_DISABLE_Q4_ATTN_OUT_A_DIRECT presence rollback; unset enables the automatic fixed-route path for eligible Apple M1-M4 long prefills; any defined value including 0 disables Restore the generic route-map/work-list Q4 attention output-A path instead of the bit-identical fixed-route direct kernel. ds4_metal.m:31562 +runtime/metal DS4_METAL_DISABLE_Q4_ATTN_OUT_B_F16_RHS value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Disables resident pre-M5 Q4 attention output-B F16 RHS materialization and restores per-tile F32 staging. ds4_metal.m:31558 runtime/metal DS4_METAL_DISABLE_Q4_ATTN_OUT_HC_FUSE presence rollback; unset: automatic/default path; any value including 0 disables Disables Q4 attn out HC fuse. ds4_metal.m:47624 runtime/metal DS4_METAL_DISABLE_Q4_ATTN_OUT_TINY_BATCH value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Disables Q4 attn out tiny batch. ds4_metal.m:28396 runtime/metal DS4_METAL_DISABLE_Q4_ATTN_Q_B_F16_CACHE value-aware boolean; unset/0: persistent sidecar remains eligible; empty/1/true/yes/on disables; REQUIRE plus DISABLE fails closed Disables only the persistent pre-M5 Q4_K attn_q_b F16 weight sidecar; unless REQUIRE is set, the default transient F16 path remains independently eligible, so also set DS4_METAL_DISABLE_Q4_ATTN_Q_B_TRANSIENT_F16=1 to force native per-tile Q4 dequantization. ds4_metal.m:25819; ds4_metal.m:26131 @@ -874,6 +875,7 @@ runtime/metal DS4_METAL_REQUIRE_IQ2_XXS_SSD_PREFILL_MM value-aware boolean; defa runtime/metal DS4_METAL_REQUIRE_M1_IQ2_MID_ONLY presence strict check; unset: fallback allowed; any value including 0 requires the path Requires M1 IQ2 mid only and makes eligible fallback fail closed. ds4_metal.m:42074 runtime/metal DS4_METAL_REQUIRE_OUTPUT_HC_WEIGHTS4 presence strict check; unset: fallback allowed; any value including 0 requires the path Requires output HC weights4 and makes eligible fallback fail closed. ds4_metal.m:46789 runtime/metal DS4_METAL_REQUIRE_PRE_M5_BATCH_ATTN_OUT_HC_FUSION presence strict check; unset permits fallback; any value including 0 requires the exact resident Q8_0 output-B-to-HC4 tail Fail closed instead of replaying the separate Q8 output-B and HC path when the fused candidate is ineligible or fails. ds4_metal.m:30536 +runtime/metal DS4_METAL_REQUIRE_Q4_ATTN_OUT_A_DIRECT presence strict check; unset permits the automatic path or fallback; any defined value including 0 requires the direct kernel and DISABLE wins Require the bit-identical fixed-route Q4 attention output-A kernel and fail closed before dispatch when its production contract is unavailable. ds4_metal.m:31560 runtime/metal DS4_METAL_REQUIRE_Q4_ATTN_OUT_B_F16_RHS value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Requires the resident pre-M5 Q4 attention output-B F16 RHS path and makes an ineligible or disabled candidate fail closed. ds4_metal.m:28551 runtime/metal DS4_METAL_REQUIRE_Q4_ATTN_OUT_TINY_BATCH value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables Requires Q4 attn out tiny batch and makes eligible fallback fail closed. ds4_metal.m:28373 runtime/metal DS4_METAL_REQUIRE_Q4_ATTN_Q_B_F16_CACHE value-aware boolean; unset/0: fallback allowed; empty/1/true/yes/on requires; DISABLE fails closed Requires the sidecar for resident, or explicitly enabled SSD-hybrid, pre-M5 Q4_K Flash attn_q_b batches at or above the configured minimum; decode and below-min batches remain non-candidates. ds4_metal.m:25801; ds4_metal.m:26129; ds4_metal.m:26704 diff --git a/speed-bench/README.md b/speed-bench/README.md index 6c9090c64..f931f12d6 100644 --- a/speed-bench/README.md +++ b/speed-bench/README.md @@ -116,6 +116,28 @@ ABBA/BAAB, and requires bit-exact outputs plus intact input/output canaries. No GGUF access, SSD I/O, upload, readback, or CPU wall time is included in a measured command buffer. +### Metal Q4_K attention output-A direct routing + +Build and run the production-shape, resident GPU comparison with: + +``` +make metal-q4-attn-out-a-direct-bench +./speed-bench/metal_q4_attn_out_a_direct_bench \ + --n 512,1024,2048,4096 --samples 8 --warmup 2 +``` + +The fixed `4096 -> 1024 x 8` geometry compares the current map-plus-routed +path, the routed kernel with a prebuilt map, and the fixed-route direct kernel +on the production Apple M1–M4 and N=512–4096 scope. All three pairwise +comparisons are scheduled in balanced ABBA/BAAB blocks and timed only with Metal +GPU start/end timestamps. The harness requires all three full outputs to be +bit-identical, hashes immutable weights, heads, ids, and the prebuilt map, and +checks prefix/suffix canaries around every allocation. Fixture construction, +map prebuilding, and validation are outside the samples. There is no GGUF, +SSD I/O, model upload, GPU readback, or CPU wall timing in measured command +buffers, so these numbers isolate the kernel and routing overhead rather than +SSD-streaming noise. + ### Metal resident IQ2/Q2 routed MoE Build the production top-6 tail-cull fixture or the GLM-shape top-8 pair-fusion diff --git a/speed-bench/metal_q4_attn_out_a_direct_bench.m b/speed-bench/metal_q4_attn_out_a_direct_bench.m new file mode 100644 index 000000000..3f13528bb --- /dev/null +++ b/speed-bench/metal_q4_attn_out_a_direct_bench.m @@ -0,0 +1,995 @@ +#import +#import + +#include +#include +#include +#include +#include +#include +#include + +/* Resident, production-shape comparison for Q4 attention output-A. The + * measured command buffers contain only Metal dispatches; fixture creation, + * map construction for the routed-only arm, and all validation stay outside + * the timed region. */ +enum { + K_DIM = 4096, + M_DIM = 1024, + GROUPS = 8, + QK_K = 256, + Q4_BLOCK_BYTES = 144, + THREADS_PER_GROUP = 128, + THREADGROUP_MEMORY_BYTES = 8192, + GUARD_BYTES = 4096, + MIN_TOKENS = 512, + MAX_TOKENS = 4096, + MAX_TOKEN_CASES = 16, + MAX_SAMPLES = 64, + MAX_WARMUP = 32, + DEFAULT_SAMPLES = 8, + DEFAULT_WARMUP = 2, +}; + +static const uint32_t k_guard = 0x7fc12345u; +static const uint32_t k_poison_current = 0x7fc0a001u; +static const uint32_t k_poison_routed = 0x7fc0b001u; +static const uint32_t k_poison_direct = 0x7fc0c001u; +static const uint32_t k_default_tokens[] = {512u, 1024u, 2048u, 4096u}; + +typedef struct { + uint16_t d; + uint16_t dmin; + uint8_t scales[12]; + uint8_t qs[QK_K / 2]; +} block_q4_K_host; + +typedef struct { + int32_t ne02; + int32_t ne10; + int32_t ne11; + uint64_t nb11; + uint64_t nb12; + int32_t ne21; + int32_t ne20; + uint64_t nb21; +} map_args; + +typedef struct { + int32_t ne00; + int32_t ne02; + uint64_t nb01; + uint64_t nb02; + uint64_t nb03; + int32_t ne11; + uint64_t nb10; + uint64_t nb11; + uint64_t nb12; + uint64_t nb13; + int32_t ne20; + int32_t ne21; + int32_t ne0; + int32_t ne1; + int16_t r2; + int16_t r3; + int32_t tp_rank; + int32_t tp_world; + int32_t tp_expert_base; +} mm_args; + +_Static_assert(sizeof(block_q4_K_host) == Q4_BLOCK_BYTES, + "Q4_K host fixture must match the Metal ABI"); +_Static_assert(sizeof(map_args) == 48, "map argument ABI changed"); +_Static_assert(sizeof(mm_args) == 104, "routed-MM argument ABI changed"); + +typedef struct { + NSUInteger tpe_bytes; + NSUInteger hids_bytes; + NSUInteger work_offset; + NSUInteger total_bytes; + NSUInteger work_cap; +} map_layout; + +typedef struct { + uint32_t tokens[MAX_TOKEN_CASES]; + uint32_t token_count; + uint32_t samples; + uint32_t warmup; +} bench_config; + +typedef enum { + ARM_CURRENT, + ARM_ROUTED_ONLY, + ARM_DIRECT, +} bench_arm; + +typedef struct { + __strong id device; + __strong id queue; + __strong id map_pipeline; + __strong id routed_pipeline; + __strong id direct_pipeline; + __strong id weights; + NSUInteger weights_bytes; + uint64_t weights_hash; +} fixture; + +typedef struct { + __strong id heads; + __strong id current_out; + __strong id routed_out; + __strong id direct_out; + __strong id ids; + __strong id current_map; + __strong id prebuilt_map; + NSUInteger heads_bytes; + NSUInteger output_bytes; + NSUInteger ids_bytes; + map_layout layout; + map_args map; + mm_args mm; + uint64_t heads_hash; + uint64_t ids_hash; + uint64_t prebuilt_map_hash; + uint32_t tokens; +} case_buffers; + +typedef struct { + double first[MAX_SAMPLES]; + double second[MAX_SAMPLES]; +} pair_samples; + +static void usage(FILE *fp, const char *argv0) { + fprintf(fp, + "usage: %s [options]\n" + "\n" + "Resident Metal Q4_K attention output-A kernel comparison.\n" + "\n" + " --n LIST comma-separated token counts, %u..%u\n" + " (default: 512,1024,2048,4096)\n" + " --samples N samples per arm, multiple of 4, max %u\n" + " (default: %u)\n" + " --warmup N warmup dispatches per arm, max %u\n" + " (default: %u)\n" + " -h, --help show this help\n" + "\n" + "The fixed production geometry is 4096 -> 1024 across 8 groups.\n" + "Set DS4_SOURCE_ROOT when running outside the repository root.\n", + argv0, MIN_TOKENS, MAX_TOKENS, MAX_SAMPLES, DEFAULT_SAMPLES, + MAX_WARMUP, DEFAULT_WARMUP); +} + +static const char *need_arg(int *index, int argc, char **argv) { + if (*index + 1 >= argc) { + fprintf(stderr, + "metal-q4-attn-out-a-direct-bench: %s needs a value\n", + argv[*index]); + exit(2); + } + return argv[++*index]; +} + +static uint32_t parse_u32(const char *text, const char *option, + uint32_t minimum, uint32_t maximum) { + char *end = NULL; + errno = 0; + const unsigned long long value = strtoull(text, &end, 10); + if (errno != 0 || !text[0] || !end || *end || + value < minimum || value > maximum) { + fprintf(stderr, + "metal-q4-attn-out-a-direct-bench: invalid %s: %s\n", + option, text); + exit(2); + } + return (uint32_t)value; +} + +static void parse_token_list(bench_config *config, const char *text) { + const char *cursor = text; + config->token_count = 0u; + while (*cursor) { + char *end = NULL; + errno = 0; + const unsigned long long value = strtoull(cursor, &end, 10); + if (errno != 0 || end == cursor || value < MIN_TOKENS || + value > MAX_TOKENS || (*end != ',' && *end != '\0') || + config->token_count == MAX_TOKEN_CASES) { + fprintf(stderr, + "metal-q4-attn-out-a-direct-bench: invalid --n list: %s\n", + text); + exit(2); + } + for (uint32_t i = 0; i < config->token_count; i++) { + if (config->tokens[i] == (uint32_t)value) { + fprintf(stderr, + "metal-q4-attn-out-a-direct-bench: duplicate N=%llu\n", + value); + exit(2); + } + } + config->tokens[config->token_count++] = (uint32_t)value; + if (*end == '\0') break; + cursor = end + 1; + if (!*cursor) { + fprintf(stderr, + "metal-q4-attn-out-a-direct-bench: invalid --n list: %s\n", + text); + exit(2); + } + } + if (config->token_count == 0u) { + fprintf(stderr, + "metal-q4-attn-out-a-direct-bench: --n list is empty\n"); + exit(2); + } +} + +static bench_config parse_options(int argc, char **argv) { + bench_config config = { + .samples = DEFAULT_SAMPLES, + .warmup = DEFAULT_WARMUP, + }; + config.token_count = (uint32_t)(sizeof(k_default_tokens) / + sizeof(k_default_tokens[0])); + memcpy(config.tokens, k_default_tokens, sizeof(k_default_tokens)); + + for (int i = 1; i < argc; i++) { + if (!strcmp(argv[i], "-h") || !strcmp(argv[i], "--help")) { + usage(stdout, argv[0]); + exit(0); + } else if (!strcmp(argv[i], "--n")) { + parse_token_list(&config, need_arg(&i, argc, argv)); + } else if (!strcmp(argv[i], "--samples")) { + const char *option = argv[i]; + config.samples = parse_u32(need_arg(&i, argc, argv), option, + 4u, MAX_SAMPLES); + } else if (!strcmp(argv[i], "--warmup")) { + const char *option = argv[i]; + config.warmup = parse_u32(need_arg(&i, argc, argv), option, + 0u, MAX_WARMUP); + } else { + fprintf(stderr, + "metal-q4-attn-out-a-direct-bench: unknown option: %s\n", + argv[i]); + usage(stderr, argv[0]); + exit(2); + } + } + if ((config.samples % 4u) != 0u) { + fprintf(stderr, + "metal-q4-attn-out-a-direct-bench: --samples must be " + "divisible by 4 for balanced ABBA/BAAB blocks\n"); + exit(2); + } + return config; +} + +static bool checked_add(NSUInteger a, NSUInteger b, NSUInteger *out) { + if (b > NSUIntegerMax - a) return false; + *out = a + b; + return true; +} + +static bool checked_mul(NSUInteger a, NSUInteger b, NSUInteger *out) { + if (a != 0u && b > NSUIntegerMax / a) return false; + *out = a * b; + return true; +} + +static NSUInteger align_up(NSUInteger value, NSUInteger alignment) { + return (value + alignment - 1u) / alignment * alignment; +} + +static uint32_t lcg_next(uint32_t *state) { + *state = *state * 1664525u + 1013904223u; + return *state; +} + +static uint64_t hash_bytes(const void *raw, NSUInteger bytes) { + const uint8_t *data = raw; + uint64_t hash = UINT64_C(1469598103934665603); + for (NSUInteger i = 0; i < bytes; i++) { + hash ^= data[i]; + hash *= UINT64_C(1099511628211); + } + return hash; +} + +static bool is_pre_m5_apple_silicon_name(const char *name) { + return name && !strncmp(name, "Apple M", 7) && + name[7] >= '1' && name[7] <= '4' && + (name[8] == '\0' || name[8] == ' '); +} + +static NSString *metal_prelude(void) { + return @"#include \n" + "using namespace metal;\n" + "#define MAX(x, y) ((x) > (y) ? (x) : (y))\n" + "#define MIN(x, y) ((x) < (y) ? (x) : (y))\n" + "#define SWAP(x, y) { auto tmp = (x); (x) = (y); (y) = tmp; }\n" + "#define QK8_0 32\n" + "#ifndef QK_K\n#define QK_K 256\n#endif\n" + "#define N_SIMDWIDTH 32\n" + "#define N_R0_Q8_0 2\n" + "#define N_SG_Q8_0 4\n" + "#define FC_MUL_MV 600\n" + "#define FC_MUL_MM 700\n" + "#define FC_BIN 1300\n" + "#define FOR_UNROLL(x) _Pragma(\"clang loop unroll(full)\") for (x)\n" + "#define M_PI_F 3.14159265358979323846f\n" + "enum ds4_sort_order { DS4_SORT_ORDER_ASC, DS4_SORT_ORDER_DESC };\n" + "struct block_q8_0 { half d; int8_t qs[QK8_0]; };\n" + "struct block_q8_K { float d; int8_t qs[QK_K]; " + "int16_t bsums[QK_K / 16]; };\n"; +} + +static NSString *load_metal_source(void) { + static const char *paths[] = { + "metal/activations.metal", + "metal/flash_attn.metal", + "metal/dense.metal", + "metal/moe.metal", + "metal/dsv4_hc.metal", + "metal/unary.metal", + "metal/dsv4_kv.metal", + "metal/dsv4_rope.metal", + "metal/dsv4_misc.metal", + "metal/argsort.metal", + "metal/cpy.metal", + "metal/concat.metal", + "metal/get_rows.metal", + "metal/sum_rows.metal", + "metal/softmax.metal", + "metal/repeat.metal", + "metal/glu.metal", + "metal/norm.metal", + "metal/bin.metal", + "metal/set_rows.metal", + }; + const char *root_env = getenv("DS4_SOURCE_ROOT"); + NSString *root = root_env && root_env[0] + ? [NSString stringWithUTF8String:root_env] + : [[NSFileManager defaultManager] currentDirectoryPath]; + NSMutableString *source = [NSMutableString stringWithString:metal_prelude()]; + for (size_t i = 0; i < sizeof(paths) / sizeof(paths[0]); i++) { + NSString *relative = [NSString stringWithUTF8String:paths[i]]; + NSString *path = [root stringByAppendingPathComponent:relative]; + NSError *error = nil; + NSString *part = [NSString stringWithContentsOfFile:path + encoding:NSUTF8StringEncoding + error:&error]; + if (!part) { + fprintf(stderr, + "metal-q4-attn-out-a-direct-bench: cannot read %s: %s\n", + [path fileSystemRepresentation], + [[error localizedDescription] UTF8String]); + return nil; + } + [source appendFormat:@"\n// appended %@\n%@\n", relative, part]; + } + return source; +} + +static id make_pipeline( + id device, id library, NSString *name, + bool routed_constants) { + NSError *error = nil; + id function = nil; + if (routed_constants) { + bool bc_inp = false; + MTLFunctionConstantValues *values = [MTLFunctionConstantValues new]; + [values setConstantValue:&bc_inp + type:MTLDataTypeBool + atIndex:700]; + function = [library newFunctionWithName:name + constantValues:values + error:&error]; + } else { + function = [library newFunctionWithName:name]; + } + if (!function) { + fprintf(stderr, + "metal-q4-attn-out-a-direct-bench: function %s: %s\n", + [name UTF8String], + error ? [[error localizedDescription] UTF8String] : "missing"); + return nil; + } + id pipeline = + [device newComputePipelineStateWithFunction:function error:&error]; + if (!pipeline) { + fprintf(stderr, + "metal-q4-attn-out-a-direct-bench: pipeline %s: %s\n", + [name UTF8String], [[error localizedDescription] UTF8String]); + } + return pipeline; +} + +static id guarded_buffer(id device, + NSUInteger payload_bytes, + NSString *label) { + NSUInteger total = 0u; + if (!checked_add(payload_bytes, 2u * GUARD_BYTES, &total)) return nil; + id buffer = + [device newBufferWithLength:total options:MTLResourceStorageModeShared]; + if (!buffer) { + fprintf(stderr, + "metal-q4-attn-out-a-direct-bench: allocation failed for " + "%s (%llu bytes)\n", + [label UTF8String], (unsigned long long)total); + return nil; + } + buffer.label = label; + uint32_t *words = buffer.contents; + for (NSUInteger i = 0; i < total / sizeof(*words); i++) { + words[i] = k_guard; + } + return buffer; +} + +static void *payload(id buffer) { + return (uint8_t *)buffer.contents + GUARD_BYTES; +} + +static bool check_canary(id buffer, NSUInteger payload_bytes, + const char *label, uint32_t tokens) { + const uint32_t *words = buffer.contents; + const NSUInteger suffix = (GUARD_BYTES + payload_bytes) / sizeof(*words); + for (NSUInteger i = 0; i < GUARD_BYTES / sizeof(*words); i++) { + if (words[i] != k_guard || words[suffix + i] != k_guard) { + fprintf(stderr, + "metal-q4-attn-out-a-direct-bench: %s canary changed " + "N=%u word=%llu\n", + label, tokens, (unsigned long long)i); + return false; + } + } + return true; +} + +static void fill_weights(fixture *f) { + block_q4_K_host *blocks = payload(f->weights); + const NSUInteger count = f->weights_bytes / sizeof(*blocks); + uint32_t state = 0x31415926u; + for (NSUInteger block = 0; block < count; block++) { + blocks[block].d = + (uint16_t)(0x2800u | (lcg_next(&state) & 0x01ffu)); + blocks[block].dmin = + (uint16_t)(0x2000u | (lcg_next(&state) & 0x01ffu)); + for (size_t i = 0; i < sizeof(blocks[block].scales); i++) { + blocks[block].scales[i] = (uint8_t)(lcg_next(&state) >> 24); + } + for (size_t i = 0; i < sizeof(blocks[block].qs); i++) { + blocks[block].qs[i] = (uint8_t)(lcg_next(&state) >> 24); + } + } +} + +static bool init_fixture(fixture *f) { + f->device = MTLCreateSystemDefaultDevice(); + if (!f->device) { + fprintf(stderr, + "metal-q4-attn-out-a-direct-bench: no Metal device\n"); + return false; + } + if (!is_pre_m5_apple_silicon_name([f->device.name UTF8String])) { + fprintf(stderr, + "metal-q4-attn-out-a-direct-bench: production route requires " + "Apple M1-M4 (device: %s)\n", + [f->device.name UTF8String]); + return false; + } + f->queue = [f->device newCommandQueue]; + NSString *source = load_metal_source(); + if (!f->queue || !source) return false; + + NSError *error = nil; + MTLCompileOptions *options = [MTLCompileOptions new]; + id library = + [f->device newLibraryWithSource:source options:options error:&error]; + if (!library) { + fprintf(stderr, + "metal-q4-attn-out-a-direct-bench: Metal compile failed: %s\n", + [[error localizedDescription] UTF8String]); + return false; + } + f->map_pipeline = make_pipeline( + f->device, library, @"kernel_mul_mm_id_map0_ne20_8", false); + f->routed_pipeline = make_pipeline( + f->device, library, @"kernel_mul_mm_id_q4_K_f32", true); + f->direct_pipeline = make_pipeline( + f->device, library, + @"kernel_attn_out_low_q4_K_legacy_direct", false); + if (!f->map_pipeline || !f->routed_pipeline || !f->direct_pipeline || + f->map_pipeline.maxTotalThreadsPerThreadgroup < GROUPS || + f->routed_pipeline.threadExecutionWidth != 32u || + f->direct_pipeline.threadExecutionWidth != 32u || + f->routed_pipeline.maxTotalThreadsPerThreadgroup < THREADS_PER_GROUP || + f->direct_pipeline.maxTotalThreadsPerThreadgroup < THREADS_PER_GROUP || + f->routed_pipeline.staticThreadgroupMemoryLength + + THREADGROUP_MEMORY_BYTES > f->device.maxThreadgroupMemoryLength || + f->direct_pipeline.staticThreadgroupMemoryLength + + THREADGROUP_MEMORY_BYTES > f->device.maxThreadgroupMemoryLength) { + fprintf(stderr, + "metal-q4-attn-out-a-direct-bench: unexpected pipeline " + "geometry or threadgroup memory limit\n"); + return false; + } + + const NSUInteger row_bytes = + (NSUInteger)(K_DIM / QK_K) * Q4_BLOCK_BYTES; + NSUInteger rows = 0u; + if (!checked_mul((NSUInteger)GROUPS, M_DIM, &rows) || + !checked_mul(rows, row_bytes, &f->weights_bytes)) { + fprintf(stderr, + "metal-q4-attn-out-a-direct-bench: weight size overflow\n"); + return false; + } + f->weights = guarded_buffer(f->device, f->weights_bytes, + @"Q4 output-A weights"); + if (!f->weights) return false; + fill_weights(f); + f->weights_hash = hash_bytes(payload(f->weights), f->weights_bytes); + return true; +} + +static map_layout make_map_layout(uint32_t tokens) { + const NSUInteger pair_rows = (NSUInteger)tokens * GROUPS; + const NSUInteger tpe_bytes = GROUPS * 2u * sizeof(uint32_t); + const NSUInteger hids_bytes = pair_rows * sizeof(int32_t); + const NSUInteger work_offset = align_up(tpe_bytes + hids_bytes, 8u); + const NSUInteger work_cap = + (pair_rows + 31u * GROUPS + 31u) / 32u; + return (map_layout) { + .tpe_bytes = tpe_bytes, + .hids_bytes = hids_bytes, + .work_offset = work_offset, + .total_bytes = work_offset + 8u + work_cap * 2u * sizeof(uint32_t), + .work_cap = work_cap, + }; +} + +static void fill_heads(float *values, NSUInteger count) { + for (NSUInteger i = 0; i < count; i++) { + const uint32_t bits = + (uint32_t)(i * 1103515245u + 12345u + (i >> 7u)); + values[i] = ((float)((int32_t)((bits >> 16u) & 0x7ffu) - 1024)) / + 1024.0f; + } +} + +static void fill_ids(int32_t *ids, uint32_t tokens) { + for (uint32_t token = 0; token < tokens; token++) { + for (uint32_t group = 0; group < GROUPS; group++) { + ids[(NSUInteger)token * GROUPS + group] = (int32_t)group; + } + } +} + +static bool init_case(case_buffers *c, fixture *f, uint32_t tokens) { + c->tokens = tokens; + c->layout = make_map_layout(tokens); + NSUInteger head_values = 0u; + NSUInteger output_values = 0u; + if (!checked_mul((NSUInteger)tokens, GROUPS * K_DIM, &head_values) || + !checked_mul(head_values, sizeof(float), &c->heads_bytes) || + !checked_mul((NSUInteger)tokens, GROUPS * M_DIM, &output_values) || + !checked_mul(output_values, sizeof(float), &c->output_bytes) || + !checked_mul((NSUInteger)tokens, GROUPS * sizeof(int32_t), + &c->ids_bytes)) { + fprintf(stderr, + "metal-q4-attn-out-a-direct-bench: N=%u size overflow\n", + tokens); + return false; + } + + c->heads = guarded_buffer(f->device, c->heads_bytes, @"attention heads"); + c->current_out = guarded_buffer(f->device, c->output_bytes, + @"map+routed output"); + c->routed_out = guarded_buffer(f->device, c->output_bytes, + @"routed-only output"); + c->direct_out = guarded_buffer(f->device, c->output_bytes, + @"direct output"); + c->ids = guarded_buffer(f->device, c->ids_bytes, @"fixed route ids"); + c->current_map = guarded_buffer(f->device, c->layout.total_bytes, + @"current route map"); + c->prebuilt_map = guarded_buffer(f->device, c->layout.total_bytes, + @"prebuilt route map"); + if (!c->heads || !c->current_out || !c->routed_out || !c->direct_out || + !c->ids || !c->current_map || !c->prebuilt_map) { + return false; + } + + fill_heads(payload(c->heads), head_values); + fill_ids(payload(c->ids), tokens); + c->heads_hash = hash_bytes(payload(c->heads), c->heads_bytes); + c->ids_hash = hash_bytes(payload(c->ids), c->ids_bytes); + + const uint64_t row_bytes = + (uint64_t)(K_DIM / QK_K) * Q4_BLOCK_BYTES; + c->map = (map_args) { + .ne02 = GROUPS, + .ne10 = K_DIM, + .ne11 = GROUPS, + .nb11 = (uint64_t)K_DIM * sizeof(float), + .nb12 = (uint64_t)GROUPS * K_DIM * sizeof(float), + .ne21 = (int32_t)tokens, + .ne20 = GROUPS, + .nb21 = (uint64_t)GROUPS * sizeof(int32_t), + }; + c->mm = (mm_args) { + .ne00 = K_DIM, + .ne02 = GROUPS, + .nb01 = row_bytes, + .nb02 = (uint64_t)M_DIM * row_bytes, + .nb03 = (uint64_t)GROUPS * M_DIM * row_bytes, + .ne11 = GROUPS, + .nb10 = sizeof(float), + .nb11 = (uint64_t)K_DIM * sizeof(float), + .nb12 = (uint64_t)GROUPS * K_DIM * sizeof(float), + .nb13 = (uint64_t)tokens * GROUPS * K_DIM * sizeof(float), + .ne20 = GROUPS, + .ne21 = (int32_t)tokens, + .ne0 = M_DIM, + .ne1 = GROUPS, + .r2 = 1, + .r3 = 1, + .tp_rank = 0, + .tp_world = 1, + .tp_expert_base = 0, + }; + return true; +} + +static void encode_map(id encoder, + fixture *f, case_buffers *c, + id map_buffer) { + [encoder setComputePipelineState:f->map_pipeline]; + [encoder setBytes:&c->map length:sizeof(c->map) atIndex:0]; + [encoder setBuffer:c->ids offset:GUARD_BYTES atIndex:1]; + [encoder setBuffer:map_buffer offset:GUARD_BYTES atIndex:2]; + [encoder setBuffer:map_buffer + offset:GUARD_BYTES + c->layout.tpe_bytes + atIndex:3]; + [encoder setBuffer:map_buffer + offset:GUARD_BYTES + c->layout.work_offset + atIndex:4]; + const NSUInteger staging = GROUPS * GROUPS * sizeof(uint16_t); + const NSUInteger scatter = GROUPS * 2u * sizeof(uint32_t); + [encoder setThreadgroupMemoryLength: + staging > scatter ? staging : scatter atIndex:0]; + [encoder dispatchThreadgroups:MTLSizeMake(1u, 1u, 1u) + threadsPerThreadgroup:MTLSizeMake(GROUPS, 1u, 1u)]; +} + +static void encode_routed(id encoder, + fixture *f, case_buffers *c, + id map_buffer, + id output) { + [encoder setComputePipelineState:f->routed_pipeline]; + [encoder setBytes:&c->mm length:sizeof(c->mm) atIndex:0]; + [encoder setBuffer:f->weights offset:GUARD_BYTES atIndex:1]; + [encoder setBuffer:c->heads offset:GUARD_BYTES atIndex:2]; + [encoder setBuffer:map_buffer offset:GUARD_BYTES atIndex:3]; + [encoder setBuffer:map_buffer + offset:GUARD_BYTES + c->layout.tpe_bytes + atIndex:4]; + [encoder setBuffer:output offset:GUARD_BYTES atIndex:5]; + [encoder setBuffer:map_buffer + offset:GUARD_BYTES + c->layout.work_offset + atIndex:6]; + [encoder setThreadgroupMemoryLength:THREADGROUP_MEMORY_BYTES atIndex:0]; + [encoder dispatchThreadgroups: + MTLSizeMake(c->layout.work_cap, (M_DIM + 63u) / 64u, 1u) + threadsPerThreadgroup:MTLSizeMake(THREADS_PER_GROUP, 1u, 1u)]; +} + +static void encode_direct(id encoder, + fixture *f, case_buffers *c) { + [encoder setComputePipelineState:f->direct_pipeline]; + [encoder setBytes:&c->mm length:sizeof(c->mm) atIndex:0]; + [encoder setBuffer:f->weights offset:GUARD_BYTES atIndex:1]; + [encoder setBuffer:c->heads offset:GUARD_BYTES atIndex:2]; + [encoder setBuffer:c->direct_out offset:GUARD_BYTES atIndex:3]; + [encoder setThreadgroupMemoryLength:THREADGROUP_MEMORY_BYTES atIndex:0]; + [encoder dispatchThreadgroups: + MTLSizeMake((c->tokens + 31u) / 32u, + (M_DIM + 63u) / 64u, GROUPS) + threadsPerThreadgroup:MTLSizeMake(THREADS_PER_GROUP, 1u, 1u)]; +} + +static bool finish_command_buffer(id cb, + const char *label) { + [cb commit]; + [cb waitUntilCompleted]; + if (cb.status == MTLCommandBufferStatusCompleted) return true; + fprintf(stderr, + "metal-q4-attn-out-a-direct-bench: %s failed: %s\n", + label, cb.error ? [[cb.error localizedDescription] UTF8String] + : "unknown Metal error"); + return false; +} + +static bool build_prebuilt_map(fixture *f, case_buffers *c) { + id cb = [f->queue commandBuffer]; + id encoder = [cb computeCommandEncoder]; + encode_map(encoder, f, c, c->prebuilt_map); + [encoder endEncoding]; + if (!finish_command_buffer(cb, "prebuilt map")) return false; + c->prebuilt_map_hash = + hash_bytes(payload(c->prebuilt_map), c->layout.total_bytes); + return true; +} + +static bool run_arm(fixture *f, case_buffers *c, bench_arm arm, + double *gpu_seconds) { + id cb = [f->queue commandBuffer]; + id encoder = [cb computeCommandEncoder]; + switch (arm) { + case ARM_CURRENT: + encode_map(encoder, f, c, c->current_map); + encode_routed(encoder, f, c, c->current_map, c->current_out); + break; + case ARM_ROUTED_ONLY: + encode_routed(encoder, f, c, c->prebuilt_map, c->routed_out); + break; + case ARM_DIRECT: + encode_direct(encoder, f, c); + break; + } + [encoder endEncoding]; + const char *label = arm == ARM_CURRENT ? "map+routed" : + arm == ARM_ROUTED_ONLY ? "routed-only" : "direct"; + if (!finish_command_buffer(cb, label)) return false; + const double elapsed = cb.GPUEndTime - cb.GPUStartTime; + if (!(elapsed > 0.0) || !isfinite(elapsed)) { + fprintf(stderr, + "metal-q4-attn-out-a-direct-bench: GPU timestamps " + "unavailable for %s\n", label); + return false; + } + *gpu_seconds = elapsed; + return true; +} + +static void poison_output(id output, NSUInteger bytes, + uint32_t poison) { + uint32_t *words = payload(output); + for (NSUInteger i = 0; i < bytes / sizeof(*words); i++) { + words[i] = poison; + } +} + +static bool check_bitwise_outputs(case_buffers *c) { + const uint32_t *current = payload(c->current_out); + const uint32_t *routed = payload(c->routed_out); + const uint32_t *direct = payload(c->direct_out); + const NSUInteger count = c->output_bytes / sizeof(uint32_t); + for (NSUInteger i = 0; i < count; i++) { + if (current[i] != routed[i] || current[i] != direct[i]) { + fprintf(stderr, + "metal-q4-attn-out-a-direct-bench: bitwise mismatch " + "N=%u element=%llu current=%08x routed=%08x direct=%08x\n", + c->tokens, (unsigned long long)i, + current[i], routed[i], direct[i]); + return false; + } + } + return true; +} + +static bool check_integrity(fixture *f, case_buffers *c) { + bool ok = true; + if (hash_bytes(payload(f->weights), f->weights_bytes) != f->weights_hash) { + fprintf(stderr, + "metal-q4-attn-out-a-direct-bench: weights changed N=%u\n", + c->tokens); + ok = false; + } + if (hash_bytes(payload(c->heads), c->heads_bytes) != c->heads_hash) { + fprintf(stderr, + "metal-q4-attn-out-a-direct-bench: heads changed N=%u\n", + c->tokens); + ok = false; + } + if (hash_bytes(payload(c->ids), c->ids_bytes) != c->ids_hash) { + fprintf(stderr, + "metal-q4-attn-out-a-direct-bench: ids changed N=%u\n", + c->tokens); + ok = false; + } + if (hash_bytes(payload(c->prebuilt_map), c->layout.total_bytes) != + c->prebuilt_map_hash) { + fprintf(stderr, + "metal-q4-attn-out-a-direct-bench: routed-only map changed " + "N=%u\n", c->tokens); + ok = false; + } + if (hash_bytes(payload(c->current_map), c->layout.total_bytes) != + c->prebuilt_map_hash) { + fprintf(stderr, + "metal-q4-attn-out-a-direct-bench: rebuilt map differs " + "N=%u\n", c->tokens); + ok = false; + } + ok = check_canary(f->weights, f->weights_bytes, + "weights", c->tokens) && ok; + ok = check_canary(c->heads, c->heads_bytes, + "heads", c->tokens) && ok; + ok = check_canary(c->current_out, c->output_bytes, + "map+routed output", c->tokens) && ok; + ok = check_canary(c->routed_out, c->output_bytes, + "routed-only output", c->tokens) && ok; + ok = check_canary(c->direct_out, c->output_bytes, + "direct output", c->tokens) && ok; + ok = check_canary(c->ids, c->ids_bytes, + "ids", c->tokens) && ok; + ok = check_canary(c->current_map, c->layout.total_bytes, + "current map", c->tokens) && ok; + ok = check_canary(c->prebuilt_map, c->layout.total_bytes, + "prebuilt map", c->tokens) && ok; + return ok; +} + +static bool run_oracle(fixture *f, case_buffers *c) { + poison_output(c->current_out, c->output_bytes, k_poison_current); + poison_output(c->routed_out, c->output_bytes, k_poison_routed); + poison_output(c->direct_out, c->output_bytes, k_poison_direct); + double ignored = 0.0; + if (!run_arm(f, c, ARM_CURRENT, &ignored) || + !run_arm(f, c, ARM_ROUTED_ONLY, &ignored) || + !run_arm(f, c, ARM_DIRECT, &ignored)) { + return false; + } + return check_bitwise_outputs(c) && check_integrity(f, c); +} + +static bool warm_up(fixture *f, case_buffers *c, uint32_t warmup) { + static const bench_arm orders[][3] = { + {ARM_CURRENT, ARM_ROUTED_ONLY, ARM_DIRECT}, + {ARM_DIRECT, ARM_ROUTED_ONLY, ARM_CURRENT}, + {ARM_ROUTED_ONLY, ARM_CURRENT, ARM_DIRECT}, + {ARM_DIRECT, ARM_CURRENT, ARM_ROUTED_ONLY}, + {ARM_CURRENT, ARM_DIRECT, ARM_ROUTED_ONLY}, + {ARM_ROUTED_ONLY, ARM_DIRECT, ARM_CURRENT}, + }; + double ignored = 0.0; + for (uint32_t cycle = 0; cycle < warmup; cycle++) { + const bench_arm *order = orders[cycle % + (sizeof(orders) / sizeof(orders[0]))]; + for (uint32_t position = 0; position < 3u; position++) { + if (!run_arm(f, c, order[position], &ignored)) return false; + } + } + return true; +} + +static bool run_balanced_pair(fixture *f, case_buffers *c, + bench_arm first, bench_arm second, + uint32_t samples, pair_samples *result) { + uint32_t first_count = 0u; + uint32_t second_count = 0u; + for (uint32_t cycle = 0; cycle < samples / 2u; cycle++) { + const bench_arm abba[] = {first, second, second, first}; + const bench_arm baab[] = {second, first, first, second}; + const bench_arm *order = (cycle & 1u) ? baab : abba; + for (uint32_t position = 0; position < 4u; position++) { + double elapsed = 0.0; + if (!run_arm(f, c, order[position], &elapsed)) return false; + if (order[position] == first) { + result->first[first_count++] = elapsed; + } else { + result->second[second_count++] = elapsed; + } + } + } + return first_count == samples && second_count == samples; +} + +static int compare_double(const void *lhs, const void *rhs) { + const double a = *(const double *)lhs; + const double b = *(const double *)rhs; + return (a > b) - (a < b); +} + +static double median(const double *values, uint32_t count) { + double sorted[MAX_SAMPLES]; + memcpy(sorted, values, count * sizeof(*sorted)); + qsort(sorted, count, sizeof(*sorted), compare_double); + if (count & 1u) return sorted[count / 2u]; + return 0.5 * (sorted[count / 2u - 1u] + sorted[count / 2u]); +} + +static double paired_geomean_ratio(const double *baseline, + const double *candidate, + uint32_t count) { + double log_sum = 0.0; + for (uint32_t i = 0; i < count; i++) { + log_sum += log(baseline[i] / candidate[i]); + } + return exp(log_sum / count); +} + +static bool run_benchmark_case(fixture *f, const bench_config *config, + uint32_t tokens) { + @autoreleasepool { + case_buffers c = {0}; + if (!init_case(&c, f, tokens) || !build_prebuilt_map(f, &c) || + !run_oracle(f, &c) || !warm_up(f, &c, config->warmup)) { + return false; + } + + pair_samples current_direct = {0}; + pair_samples current_routed = {0}; + pair_samples routed_direct = {0}; + if (!run_balanced_pair(f, &c, ARM_CURRENT, ARM_DIRECT, + config->samples, ¤t_direct) || + !run_balanced_pair(f, &c, ARM_CURRENT, ARM_ROUTED_ONLY, + config->samples, ¤t_routed) || + !run_balanced_pair(f, &c, ARM_ROUTED_ONLY, ARM_DIRECT, + config->samples, &routed_direct) || + !check_bitwise_outputs(&c) || !check_integrity(f, &c)) { + return false; + } + + const double current_ms = + median(current_direct.first, config->samples) * 1.0e3; + const double direct_ms = + median(current_direct.second, config->samples) * 1.0e3; + const double routed_ms = + median(current_routed.second, config->samples) * 1.0e3; + const double direct_ratio = paired_geomean_ratio( + current_direct.first, current_direct.second, config->samples); + const double map_ratio = paired_geomean_ratio( + current_routed.first, current_routed.second, config->samples); + double map_delta[MAX_SAMPLES]; + for (uint32_t i = 0; i < config->samples; i++) { + map_delta[i] = (current_routed.first[i] - + current_routed.second[i]) * 1.0e6; + } + const double map_delta_us = median(map_delta, config->samples); + const double routed_to_direct = paired_geomean_ratio( + routed_direct.first, routed_direct.second, config->samples); + + printf(" N=%-4u current(map+routed) %8.3f ms " + "routed-only %8.3f ms direct %8.3f ms\n", + tokens, current_ms, routed_ms, direct_ms); + printf(" paired direct gain %6.3fx (%+6.2f%%); " + "map+dispatch %+8.3f us (%+6.2f%%); " + "paired direct vs routed %6.3fx (%+6.2f%%); PASS\n", + direct_ratio, (direct_ratio - 1.0) * 100.0, + map_delta_us, (map_ratio - 1.0) * 100.0, + routed_to_direct, (routed_to_direct - 1.0) * 100.0); + fflush(stdout); + return true; + } +} + +int main(int argc, char **argv) { + @autoreleasepool { + const bench_config config = parse_options(argc, argv); + fixture f = {0}; + if (!init_fixture(&f)) return 1; + + printf("Metal Q4_K attention output-A direct kernel benchmark\n"); + printf(" device: %s\n", [f.device.name UTF8String]); + printf(" shape: 4096 -> 1024 x 8 groups; resident anonymous Q4_K " + "weights (%.1f MiB)\n", + (double)f.weights_bytes / (1024.0 * 1024.0)); + printf(" design: current map+routed, routed-only prebuilt map, " + "and fixed-route direct\n"); + printf(" schedule: balanced ABBA/BAAB pair blocks, " + "%u samples/arm/pair, " + "%u warmup dispatches/arm, GPU timestamps only\n", + config.samples, config.warmup); + + for (uint32_t i = 0; i < config.token_count; i++) { + if (!run_benchmark_case(&f, &config, config.tokens[i])) return 1; + } + printf(" correctness: all three outputs bit-identical; weights, " + "heads, ids, and prebuilt map hashes unchanged; all " + "prefix/suffix canaries intact\n"); + printf(" scope: no GGUF, SSD I/O, model upload, GPU readback, or " + "CPU wall timing in measured command buffers\n"); + return 0; + } +} diff --git a/tests/test_metal_q4_attn_out_a_direct.c b/tests/test_metal_q4_attn_out_a_direct.c new file mode 100644 index 000000000..50d3aabc1 --- /dev/null +++ b/tests/test_metal_q4_attn_out_a_direct.c @@ -0,0 +1,539 @@ +#define _DARWIN_C_SOURCE + +/* Production-shape, GGUF-free oracle for the pre-M5 Metal Q4_K attention + * output-A fixed-route specialization. The established routed kernel is the + * bitwise baseline; output-B deliberately stays small to keep this focused + * test's resident memory footprint prudent. */ + +#include "ds4_gpu.h" + +#include +#include +#include +#include +#include +#include + +bool ds4_log_is_tty(FILE *fp) { + (void)fp; + return false; +} + +#ifdef __APPLE__ + +enum { + Q4_K_TYPE = 12u, + Q8_0_TYPE = 8u, + QK_K = 256u, + QK8_0 = 32u, + GROUP_DIM = 4096u, + RANK = 1024u, + N_GROUPS = 8u, + LOW_DIM = N_GROUPS * RANK, + OUT_DIM = 64u, + ACTIVE_ROWS = 513u, + REJECT_ROWS = 511u, + ALLOC_ROWS = ACTIVE_ROWS + 1u, + GUARD_ELEMENTS = 64u, +}; + +typedef struct { + uint16_t d; + uint16_t dmin; + uint8_t scales[12]; + uint8_t qs[QK_K / 2u]; +} block_q4_K; + +typedef struct { + uint16_t d; + int8_t qs[QK8_0]; +} block_q8_0; + +static const char *k_disable = + "DS4_METAL_DISABLE_Q4_ATTN_OUT_A_DIRECT"; +static const char *k_require = + "DS4_METAL_REQUIRE_Q4_ATTN_OUT_A_DIRECT"; +static const char *k_disable_f16_rhs = + "DS4_METAL_DISABLE_Q4_ATTN_OUT_B_F16_RHS"; +static const char *k_require_f16_rhs = + "DS4_METAL_REQUIRE_Q4_ATTN_OUT_B_F16_RHS"; + +static void fail(const char *what) { + fprintf(stderr, "Metal Q4 output-A direct oracle FAIL: %s\n", what); + exit(1); +} + +#define CHECK(expr, what) do { if (!(expr)) fail(what); } while (0) + +static uint64_t align_up(uint64_t value, uint64_t alignment) { + return (value + alignment - 1u) & ~(alignment - 1u); +} + +static uint64_t hash_bytes(const void *raw, uint64_t bytes) { + const uint8_t *p = raw; + uint64_t hash = UINT64_C(1469598103934665603); + for (uint64_t i = 0; i < bytes; i++) { + hash ^= p[i]; + hash *= UINT64_C(1099511628211); + } + return hash; +} + +static void pack_scales(uint8_t packed[12], + const uint8_t scale[8], + const uint8_t minimum[8]) { + memset(packed, 0, 12u); + for (uint32_t group = 0; group < 4u; group++) { + packed[group] = scale[group] & 63u; + packed[group + 4u] = minimum[group] & 63u; + } + for (uint32_t group = 4u; group < 8u; group++) { + packed[group + 4u] = (scale[group] & 15u) | + ((minimum[group] & 15u) << 4u); + packed[group - 4u] |= (scale[group] >> 4u) << 6u; + packed[group] |= (minimum[group] >> 4u) << 6u; + } +} + +static void fill_q4_matrix(void *raw, uint32_t in_dim, + uint32_t rows, uint32_t salt) { + CHECK(sizeof(block_q4_K) == 144u, "unexpected Q4_K block size"); + CHECK((in_dim % QK_K) == 0u, "unaligned Q4_K fixture"); + const uint32_t blocks_per_row = in_dim / QK_K; + block_q4_K *matrix = raw; + for (uint32_t row = 0; row < rows; row++) { + for (uint32_t block = 0; block < blocks_per_row; block++) { + block_q4_K *b = matrix + + (uint64_t)row * blocks_per_row + block; + const uint32_t key = salt + row * 1009u + block * 313u + + (row ^ (block * 17u)); + uint8_t scale[8]; + uint8_t minimum[8]; + for (uint32_t group = 0; group < 8u; group++) { + scale[group] = (uint8_t)((key + group * 7u) % 64u); + minimum[group] = + (uint8_t)((key / 3u + group * 5u) % 64u); + } + pack_scales(b->scales, scale, minimum); + for (uint32_t i = 0; i < QK_K / 2u; i++) { + b->qs[i] = + (uint8_t)(key + i * 37u + (i >> 2u) * 11u); + } + /* Exact binary half scales keep the fixture deterministic. */ + b->d = 0x2400u; + b->dmin = 0x1c00u; + } + } +} + +static void fill_q8_matrix(void *raw, uint32_t in_dim, + uint32_t rows, uint32_t salt) { + CHECK(sizeof(block_q8_0) == 34u, "unexpected Q8_0 block size"); + CHECK((in_dim % QK8_0) == 0u, "unaligned Q8_0 fixture"); + const uint32_t blocks_per_row = in_dim / QK8_0; + block_q8_0 *matrix = raw; + for (uint32_t row = 0; row < rows; row++) { + for (uint32_t block = 0; block < blocks_per_row; block++) { + block_q8_0 *b = matrix + + (uint64_t)row * blocks_per_row + block; + const uint32_t key = salt + row * 977u + block * 101u; + b->d = 0x2000u; + for (uint32_t i = 0; i < QK8_0; i++) { + b->qs[i] = (int8_t)((int)((key + i * 29u) % 127u) - 63); + } + } + } +} + +static void fill_heads(float *heads) { + for (uint32_t row = 0; row < ALLOC_ROWS; row++) { + for (uint32_t i = 0; i < N_GROUPS * GROUP_DIM; i++) { + const uint32_t key = + i * 41u + row * 271u + ((i >> 2u) ^ (row * 19u)); + heads[(uint64_t)row * N_GROUPS * GROUP_DIM + i] = + (float)((int)(key % 255u) - 127) / 512.0f; + } + } +} + +static void poison(float *values, uint64_t count, uint32_t base) { + for (uint64_t i = 0; i < count; i++) { + const uint32_t bits = base + (uint32_t)(i & 0xffffu); + memcpy(&values[i], &bits, sizeof(bits)); + } +} + +static uint64_t count_poison_mismatches(const float *actual, + uint64_t begin, + uint64_t end, + uint32_t base) { + uint64_t mismatches = 0; + for (uint64_t i = begin; i < end; i++) { + uint32_t bits = 0; + memcpy(&bits, &actual[i], sizeof(bits)); + if (bits != base + (uint32_t)(i & 0xffffu)) mismatches++; + } + return mismatches; +} + +static uint64_t count_bit_mismatches(const float *reference, + const float *actual, + uint64_t count, + uint64_t *first) { + uint64_t mismatches = 0; + *first = UINT64_MAX; + for (uint64_t i = 0; i < count; i++) { + if (memcmp(&reference[i], &actual[i], sizeof(float)) != 0) { + if (*first == UINT64_MAX) *first = i; + mismatches++; + } + } + return mismatches; +} + +static void upload_poison(ds4_gpu_tensor *low, + ds4_gpu_tensor *out, + float *low_host, + float *out_host, + uint64_t low_count, + uint64_t out_count, + uint32_t low_poison, + uint32_t out_poison) { + poison(low_host, low_count, low_poison); + poison(out_host, out_count, out_poison); + CHECK(ds4_gpu_tensor_write(low, 0, low_host, + low_count * sizeof(float)) != 0, + "low poison upload"); + CHECK(ds4_gpu_tensor_write(out, 0, out_host, + out_count * sizeof(float)) != 0, + "out poison upload"); +} + +static void read_outputs(ds4_gpu_tensor *low, + ds4_gpu_tensor *out, + float *low_host, + float *out_host, + uint64_t low_count, + uint64_t out_count) { + CHECK(ds4_gpu_tensor_read(low, 0, low_host, + low_count * sizeof(float)) != 0, + "low read"); + CHECK(ds4_gpu_tensor_read(out, 0, out_host, + out_count * sizeof(float)) != 0, + "out read"); +} + +static void check_inputs_immutable(ds4_gpu_tensor *heads, + float *heads_host, + uint64_t heads_bytes, + uint64_t heads_hash, + const void *model, + uint64_t model_bytes, + uint64_t model_hash) { + CHECK(ds4_gpu_tensor_read(heads, 0, heads_host, heads_bytes) != 0, + "heads immutability read"); + CHECK(hash_bytes(heads_host, heads_bytes) == heads_hash, + "heads were modified"); + CHECK(hash_bytes(model, model_bytes) == model_hash, + "model weights were modified"); +} + +int main(void) { + const uint64_t page = (uint64_t)getpagesize(); + const uint64_t row_a_bytes = + (GROUP_DIM / QK_K) * sizeof(block_q4_K); + const uint64_t out_a_bytes = (uint64_t)LOW_DIM * row_a_bytes; + const uint64_t out_b_offset = align_up(out_a_bytes, page); + const uint64_t row_b_bytes = + (LOW_DIM / QK8_0) * sizeof(block_q8_0); + const uint64_t out_b_bytes = (uint64_t)OUT_DIM * row_b_bytes; + const uint64_t model_bytes = + align_up(out_b_offset + out_b_bytes, page); + const uint64_t heads_payload_count = + (uint64_t)ALLOC_ROWS * N_GROUPS * GROUP_DIM; + const uint64_t low_payload_count = (uint64_t)ALLOC_ROWS * LOW_DIM; + const uint64_t out_payload_count = (uint64_t)ALLOC_ROWS * OUT_DIM; + const uint64_t heads_storage_count = + GUARD_ELEMENTS + heads_payload_count + GUARD_ELEMENTS; + const uint64_t low_storage_count = + GUARD_ELEMENTS + low_payload_count + GUARD_ELEMENTS; + const uint64_t out_storage_count = + GUARD_ELEMENTS + out_payload_count + GUARD_ELEMENTS; + const uint64_t active_low_count = (uint64_t)ACTIVE_ROWS * LOW_DIM; + const uint64_t active_out_count = (uint64_t)ACTIVE_ROWS * OUT_DIM; + const uint64_t heads_payload_bytes = + heads_payload_count * sizeof(float); + const uint64_t heads_storage_bytes = + heads_storage_count * sizeof(float); + + CHECK(unsetenv(k_disable) == 0, "clear direct disable env"); + CHECK(unsetenv(k_require) == 0, "clear direct require env"); + CHECK(unsetenv(k_disable_f16_rhs) == 0, + "clear output-B F16 disable env"); + CHECK(unsetenv(k_require_f16_rhs) == 0, + "clear output-B F16 require env"); + + CHECK(ds4_gpu_init() != 0, "Metal init"); + if (!ds4_gpu_device_is_pre_m5_apple_silicon()) { + fprintf(stderr, + "Metal Q4 output-A direct oracle SKIP: requires Apple M1--M4\n"); + ds4_gpu_cleanup(); + return 0; + } + + void *model = NULL; + CHECK(posix_memalign(&model, (size_t)page, (size_t)model_bytes) == 0, + "model allocation"); + memset(model, 0, (size_t)model_bytes); + fill_q4_matrix(model, GROUP_DIM, LOW_DIM, 211u); + fill_q8_matrix((uint8_t *)model + out_b_offset, + LOW_DIM, OUT_DIM, 307u); + const uint64_t model_hash = hash_bytes(model, model_bytes); + + float *heads_host = malloc((size_t)heads_storage_bytes); + float *low_host = malloc((size_t)low_storage_count * sizeof(float)); + float *out_host = malloc((size_t)out_storage_count * sizeof(float)); + float *baseline_low = malloc((size_t)active_low_count * sizeof(float)); + float *baseline_out = malloc((size_t)active_out_count * sizeof(float)); + CHECK(heads_host && low_host && out_host && baseline_low && baseline_out, + "host tensor allocation"); + poison(heads_host, heads_storage_count, 0x7fc00000u); + fill_heads(heads_host + GUARD_ELEMENTS); + const uint64_t heads_hash = + hash_bytes(heads_host, heads_storage_bytes); + + ds4_gpu_tensor *heads_base = + ds4_gpu_tensor_alloc(heads_storage_bytes); + ds4_gpu_tensor *low_base = + ds4_gpu_tensor_alloc(low_storage_count * sizeof(float)); + ds4_gpu_tensor *out_base = + ds4_gpu_tensor_alloc(out_storage_count * sizeof(float)); + CHECK(heads_base && low_base && out_base, + "guarded Metal base allocation"); + ds4_gpu_tensor *heads = ds4_gpu_tensor_view( + heads_base, GUARD_ELEMENTS * sizeof(float), heads_payload_bytes); + ds4_gpu_tensor *low = ds4_gpu_tensor_view( + low_base, GUARD_ELEMENTS * sizeof(float), + low_payload_count * sizeof(float)); + ds4_gpu_tensor *out = ds4_gpu_tensor_view( + out_base, GUARD_ELEMENTS * sizeof(float), + out_payload_count * sizeof(float)); + CHECK(heads && low && out, "guarded Metal tensor views"); + CHECK(ds4_gpu_tensor_write(heads_base, 0, heads_host, + heads_storage_bytes) != 0, + "heads upload"); + + /* Quality mode also disables the Metal4 cooperative path. SSD mode is + * intentional: this is the exact production dispatch policy under test. */ + ds4_gpu_set_quality(true); + ds4_gpu_set_ssd_streaming(true); + CHECK(ds4_gpu_set_model_map(model, model_bytes) != 0, "model map"); + + const uint32_t low_poison = 0x7fc10000u; + const uint32_t out_poison = 0x7fc20000u; + upload_poison(low_base, out_base, low_host, out_host, + low_storage_count, out_storage_count, + low_poison, out_poison); + CHECK(setenv(k_disable, "1", 1) == 0, "select routed baseline"); + CHECK(setenv(k_require, "1", 1) == 0, + "require direct for kill-switch preflight"); + CHECK(ds4_gpu_attention_output_q4_K_batch_tensor( + out, low, NULL, NULL, model, model_bytes, + 0, out_b_offset, Q8_0_TYPE, GROUP_DIM, RANK, + N_GROUPS, OUT_DIM, heads, ACTIVE_ROWS) == -1, + "direct disable must win over REQUIRE"); + read_outputs(low_base, out_base, low_host, out_host, + low_storage_count, out_storage_count); + CHECK(count_poison_mismatches(low_host, 0, low_storage_count, + low_poison) == 0, + "direct kill-switch preflight modified low"); + CHECK(count_poison_mismatches(out_host, 0, out_storage_count, + out_poison) == 0, + "direct kill-switch preflight modified out"); + CHECK(unsetenv(k_require) == 0, "clear REQUIRE for baseline"); + CHECK(ds4_gpu_attention_output_q4_K_batch_tensor( + out, low, NULL, NULL, model, model_bytes, + 0, out_b_offset, Q8_0_TYPE, GROUP_DIM, RANK, + N_GROUPS, OUT_DIM, heads, ACTIVE_ROWS) == 1, + "routed baseline dispatch"); + read_outputs(low_base, out_base, low_host, out_host, + low_storage_count, out_storage_count); + CHECK(count_poison_mismatches( + low_host, GUARD_ELEMENTS, + GUARD_ELEMENTS + active_low_count, low_poison) == + active_low_count, + "baseline did not overwrite every active low value"); + CHECK(count_poison_mismatches( + out_host, GUARD_ELEMENTS, + GUARD_ELEMENTS + active_out_count, out_poison) == + active_out_count, + "baseline did not overwrite every active out value"); + CHECK(count_poison_mismatches(low_host, 0, GUARD_ELEMENTS, + low_poison) == 0, + "baseline low prefix canary"); + CHECK(count_poison_mismatches( + low_host, GUARD_ELEMENTS + active_low_count, + GUARD_ELEMENTS + low_payload_count, + low_poison) == 0, + "baseline low tail canary"); + CHECK(count_poison_mismatches( + low_host, GUARD_ELEMENTS + low_payload_count, + low_storage_count, low_poison) == 0, + "baseline low suffix canary"); + CHECK(count_poison_mismatches(out_host, 0, GUARD_ELEMENTS, + out_poison) == 0, + "baseline out prefix canary"); + CHECK(count_poison_mismatches( + out_host, GUARD_ELEMENTS + active_out_count, + GUARD_ELEMENTS + out_payload_count, + out_poison) == 0, + "baseline out tail canary"); + CHECK(count_poison_mismatches( + out_host, GUARD_ELEMENTS + out_payload_count, + out_storage_count, out_poison) == 0, + "baseline out suffix canary"); + memcpy(baseline_low, low_host + GUARD_ELEMENTS, + (size_t)active_low_count * sizeof(float)); + memcpy(baseline_out, out_host + GUARD_ELEMENTS, + (size_t)active_out_count * sizeof(float)); + check_inputs_immutable(heads_base, heads_host, heads_storage_bytes, + heads_hash, + model, model_bytes, model_hash); + + upload_poison(low_base, out_base, low_host, out_host, + low_storage_count, out_storage_count, + low_poison, out_poison); + CHECK(unsetenv(k_disable) == 0, "enable direct candidate"); + CHECK(setenv(k_require, "1", 1) == 0, "require direct candidate"); + CHECK(ds4_gpu_attention_output_q4_K_batch_tensor( + out, low, NULL, NULL, model, model_bytes, + 0, out_b_offset, Q8_0_TYPE, GROUP_DIM, RANK, + N_GROUPS, OUT_DIM, heads, ACTIVE_ROWS) == 1, + "direct candidate dispatch"); + read_outputs(low_base, out_base, low_host, out_host, + low_storage_count, out_storage_count); + CHECK(count_poison_mismatches( + low_host, GUARD_ELEMENTS, + GUARD_ELEMENTS + active_low_count, low_poison) == + active_low_count, + "direct candidate did not overwrite every active low value"); + CHECK(count_poison_mismatches( + out_host, GUARD_ELEMENTS, + GUARD_ELEMENTS + active_out_count, out_poison) == + active_out_count, + "direct candidate did not overwrite every active out value"); + + uint64_t first_low = UINT64_MAX; + uint64_t first_out = UINT64_MAX; + const uint64_t low_mismatch = count_bit_mismatches( + baseline_low, low_host + GUARD_ELEMENTS, + active_low_count, &first_low); + const uint64_t out_mismatch = count_bit_mismatches( + baseline_out, out_host + GUARD_ELEMENTS, + active_out_count, &first_out); + const uint64_t low_prefix_mismatch = count_poison_mismatches( + low_host, 0, GUARD_ELEMENTS, low_poison); + const uint64_t low_tail_mismatch = count_poison_mismatches( + low_host, GUARD_ELEMENTS + active_low_count, + GUARD_ELEMENTS + low_payload_count, low_poison); + const uint64_t low_suffix_mismatch = count_poison_mismatches( + low_host, GUARD_ELEMENTS + low_payload_count, + low_storage_count, low_poison); + const uint64_t out_prefix_mismatch = count_poison_mismatches( + out_host, 0, GUARD_ELEMENTS, out_poison); + const uint64_t out_tail_mismatch = count_poison_mismatches( + out_host, GUARD_ELEMENTS + active_out_count, + GUARD_ELEMENTS + out_payload_count, out_poison); + const uint64_t out_suffix_mismatch = count_poison_mismatches( + out_host, GUARD_ELEMENTS + out_payload_count, + out_storage_count, out_poison); + fprintf(stderr, + "Metal Q4 output-A direct N=%u low=%llu/%llu out=%llu/%llu " + "low_guard=%llu/%llu/%llu out_guard=%llu/%llu/%llu\n", + ACTIVE_ROWS, + (unsigned long long)low_mismatch, + (unsigned long long)active_low_count, + (unsigned long long)out_mismatch, + (unsigned long long)active_out_count, + (unsigned long long)low_prefix_mismatch, + (unsigned long long)low_tail_mismatch, + (unsigned long long)low_suffix_mismatch, + (unsigned long long)out_prefix_mismatch, + (unsigned long long)out_tail_mismatch, + (unsigned long long)out_suffix_mismatch); + if (low_mismatch != 0) { + fprintf(stderr, " first low mismatch index=%llu\n", + (unsigned long long)first_low); + } + if (out_mismatch != 0) { + fprintf(stderr, " first out mismatch index=%llu\n", + (unsigned long long)first_out); + } + CHECK(low_mismatch == 0, "direct low bitwise mismatch"); + CHECK(out_mismatch == 0, "direct final output bitwise mismatch"); + CHECK(low_prefix_mismatch == 0, "direct low prefix canary"); + CHECK(low_tail_mismatch == 0, "direct low tail canary"); + CHECK(low_suffix_mismatch == 0, "direct low suffix canary"); + CHECK(out_prefix_mismatch == 0, "direct out prefix canary"); + CHECK(out_tail_mismatch == 0, "direct out tail canary"); + CHECK(out_suffix_mismatch == 0, "direct out suffix canary"); + check_inputs_immutable(heads_base, heads_host, heads_storage_bytes, + heads_hash, + model, model_bytes, model_hash); + + /* REQUIRE must fail closed before encoding anything just below the + * production threshold. The entire low/out allocations are poison here, + * so this also detects partial work from an ineligible dispatch. */ + const uint32_t reject_low_poison = 0x7fc30000u; + const uint32_t reject_out_poison = 0x7fc40000u; + upload_poison(low_base, out_base, low_host, out_host, + low_storage_count, out_storage_count, + reject_low_poison, reject_out_poison); + CHECK(ds4_gpu_attention_output_q4_K_batch_tensor( + out, low, NULL, NULL, model, model_bytes, + 0, out_b_offset, Q8_0_TYPE, GROUP_DIM, RANK, + N_GROUPS, OUT_DIM, heads, REJECT_ROWS) == -1, + "N=511 REQUIRE must fail closed"); + read_outputs(low_base, out_base, low_host, out_host, + low_storage_count, out_storage_count); + CHECK(count_poison_mismatches(low_host, 0, low_storage_count, + reject_low_poison) == 0, + "N=511 REQUIRE modified low"); + CHECK(count_poison_mismatches(out_host, 0, out_storage_count, + reject_out_poison) == 0, + "N=511 REQUIRE modified out"); + check_inputs_immutable(heads_base, heads_host, heads_storage_bytes, + heads_hash, + model, model_bytes, model_hash); + + CHECK(unsetenv(k_require) == 0, "clear direct REQUIRE"); + CHECK(unsetenv(k_disable) == 0, "clear direct disable"); + ds4_gpu_set_ssd_streaming(false); + ds4_gpu_set_quality(false); + ds4_gpu_tensor_free(out); + ds4_gpu_tensor_free(low); + ds4_gpu_tensor_free(heads); + ds4_gpu_tensor_free(out_base); + ds4_gpu_tensor_free(low_base); + ds4_gpu_tensor_free(heads_base); + ds4_gpu_cleanup(); + free(baseline_out); + free(baseline_low); + free(out_host); + free(low_host); + free(heads_host); + free(model); + fprintf(stderr, + "Metal Q4 output-A direct oracle PASS N=513 bitwise=1 " + "tail=1 immutable=1 reject_N511=1\n"); + return 0; +} + +#else + +int main(void) { + fprintf(stderr, "Metal Q4 output-A direct oracle SKIP: non-Apple host\n"); + return 0; +} + +#endif From 1e0c220bcbeb7c5e0d96a489a652f131bb144b64 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Mon, 31 Aug 2026 09:58:18 +0200 Subject: [PATCH 177/189] rocm: restore PR887 WMMA attention scheduling --- rocm/ds4_rocm_attention.cuh | 3 --- 1 file changed, 3 deletions(-) diff --git a/rocm/ds4_rocm_attention.cuh b/rocm/ds4_rocm_attention.cuh index 71aa9389b..7eaf90cd3 100644 --- a/rocm/ds4_rocm_attention.cuh +++ b/rocm/ds4_rocm_attention.cuh @@ -1519,7 +1519,6 @@ __global__ static void attention_mixed_heads16_wmma_kernel( frag_c score_acc; if (wave < HEAD_TILES * KEY_TILES) rocwmma::fill_fragment(score_acc, 0.0f); -#pragma unroll for (uint32_t d0 = 0; d0 < 512u; d0 += TILE) { if (tid < HEADS * TILE) { const uint32_t hl = tid / TILE; @@ -1630,7 +1629,6 @@ __global__ static void attention_mixed_heads16_wmma_kernel( } sh_qp[hl * STRIDE + kl] = __float2half(p); } - __syncthreads(); #pragma unroll for (uint32_t delta = 1u; delta < 16u; delta <<= 1u) { block_sum += __shfl_xor_sync(FULL_WARP_MASK, block_sum, delta, 32); @@ -1648,7 +1646,6 @@ __global__ static void attention_mixed_heads16_wmma_kernel( accum[i] *= old_scale[out_idx / 512u]; } -#pragma unroll for (uint32_t dim0 = 0; dim0 < 512u; dim0 += DIMS) { if constexpr (F32_VEC2) { for (uint32_t idx = tid; idx < KEYS * (DIMS / 2u); idx += blockDim.x) { From bc5cfc67ca7e5b256096b657e58a88ba6883a4f6 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Mon, 31 Aug 2026 13:11:56 +0200 Subject: [PATCH 178/189] rocm: fix Q4 prefill parity blockers --- ENVIRONMENT_VARIABLES.md | 12 ++-- rocm/ds4_rocm_moe.cuh | 35 +++++----- rocm/ds4_rocm_q4.cuh | 39 ++++++++++- scripts/environment_variables.tsv | 6 +- tests/test_rocm_q4_dense_pair.cpp | 108 ++++++++++++++++++++++-------- 5 files changed, 143 insertions(+), 57 deletions(-) diff --git a/ENVIRONMENT_VARIABLES.md b/ENVIRONMENT_VARIABLES.md index 5415363b6..da0caa630 100644 --- a/ENVIRONMENT_VARIABLES.md +++ b/ENVIRONMENT_VARIABLES.md @@ -98,11 +98,11 @@ The detailed Metal A/B contracts and expected oracle counters live in | `DS4_ROCM_ENABLE_Q4_PREFILL_Q8_K_WAVE32=1` | On gfx1151 wave32, opt into the no-LDS Q8_K activation quantizer that assigns one 256-value block to each wave before the exact Q4 prefill matmul. This exact path takes precedence over automatic direct-Q4 WMMA; `REQUIRE_Q4_PREFILL_WMMA` overrides an optional request, while dual REQUIRE fails closed. | | `DS4_ROCM_DISABLE_Q4_PREFILL_Q8_K_WAVE32=1` | Dominant rollback to the canonical one-workgroup-per-Q8_K-block quantizer. | | `DS4_ROCM_REQUIRE_Q4_PREFILL_Q8_K_WAVE32=1` | Require the wave32 quantizer for strict prefill A/B runs; unsupported scope/device, rollback, or an incompatible required F16/WMMA path fails closed. | -| `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA=0/1` | Compatibility control for the automatic resident gfx1151 wave32 direct-Q4 WMMA prefill kernel at 256–4096 tokens: unset or `1` permits the default, while explicit `0` opts out for resident execution. It replaces Q8_K activation scratch with transient F16 register dequantization and F32 accumulation, using 64 rows below output dimension 1024, 128 rows below 8192, and 256 rows otherwise. | +| `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA=0/1` | Compatibility control for the resident gfx1151 wave32 direct-Q4 WMMA prefill kernel at 256–4096 tokens. Unset keeps the automatic standalone/attention-output-A path, while the numerically compounded all-Q4 attention-output B stays on Q8_K+TILE8; an enabled value also opts B into direct WMMA for experiments, and an explicit false value opts out of resident WMMA entirely. The kernel uses transient F16 register dequantization and F32 accumulation, with 64 rows below output dimension 1024, 128 rows below 8192, and 256 rows otherwise. | | `DS4_ROCM_Q4_PREFILL_WMMA_ROW_TILE=64|128|256` | Override the direct-Q4 WMMA output-row tile. The default uses 64 rows below output dimension 1024, 128 below 8192, and 256 otherwise; `64` also retains the prior kernel geometry as an A/B control. | -| `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_SSD=1` | Allow that direct-Q4 WMMA path during SSD streaming only when the complete projection weight range is already backed by physical device storage. It never treats mapped/registered host memory as resident. | +| `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_SSD=1` | Explicitly allow direct-Q4 WMMA during SSD streaming, including attention-output B, only when each complete projection weight range is already backed by physical device storage. It never treats mapped/registered host memory as resident. | | `DS4_ROCM_DISABLE_Q4_PREFILL_WMMA=1` | Dominant value-aware opt-out for the automatic direct-Q4 WMMA path, including explicit resident or SSD requests. | -| `DS4_ROCM_REQUIRE_Q4_PREFILL_WMMA=1` | Assert the automatic direct-Q4 WMMA dispatch and fail closed on an unsupported device/shape, quality mode, rollback, or SSD weight range that is not physically device-resident. It is no longer needed to enable eligible resident execution and remains intended for strict A/B oracles. | +| `DS4_ROCM_REQUIRE_Q4_PREFILL_WMMA=1` | Require direct-Q4 WMMA for every selected projection and fail closed on an unsupported device/shape, quality mode, rollback, or SSD weight range that is not physically device-resident. In an all-Q4 attention-output batch this explicitly selects both A and the experimental B path; it is intended for strict A/B oracles, not quality-sensitive defaults. | | `DS4_ROCM_Q4_PREFILL_TILE8_STATS=1` | Report dense, pair, attention-batch, and token counters at process exit. | | `DS4_ROCM_ENABLE_Q4_DENSE_PAIR=1` | Share one Q8_K activation quantization between the two Q4 dense projections. This pair remains opt-in. | | `DS4_ROCM_DISABLE_Q4_DENSE_PAIR=1` | Dominant rollback for the ROCm Q4 dense pair. | @@ -1010,8 +1010,8 @@ and **19 tool/wrapper entries**. | `DS4_ROCM_ENABLE_MXFP4_TILE4` | presence opt-in; unset=off; any defined value including empty or 0 enables the candidate when the MXFP4 sorted-tile path has at least 5 tokens and neither TILE32 nor LDSB is selected | Select the ROCm MXFP4 gate/up tile4 occupancy variant, reducing staged-activation LDS per block. | [rocm/ds4_rocm_moe_launch.cuh:794](rocm/ds4_rocm_moe_launch.cuh#L794) | | `DS4_ROCM_ENABLE_Q4_DENSE_PAIR` | presence opt-in; unset=off; DISABLE takes precedence | Enable rocm enable q4 dense pair. | [rocm/ds4_rocm_q4.cuh:434](rocm/ds4_rocm_q4.cuh#L434) | | `DS4_ROCM_ENABLE_Q4_GROUPED_ATTN_A` | presence opt-in outside the default scope; the exact caller-marked resident decode shape groups=8, N=1, K=4096, M=1024 is automatic, while row-at-a-time batch fallbacks are not; DISABLE wins | Enable grouped Q4 attention-A for eligible slices, non-production shapes, or explicit experiments in addition to the resident decode default. | [rocm/ds4_rocm_q4.cuh:872](rocm/ds4_rocm_q4.cuh#L872) | -| `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA` | value-aware compatibility control for the automatic resident path; unset or any value other than 0/false/no/off permits direct-Q4 WMMA for N=256..4096, K a positive multiple of 256, resident non-quality execution on gfx1151 wave32; explicit 0/false/no/off opts out for resident execution; it never bypasses the SSD gate and DISABLE wins | Use compressed Q4_K-to-F16 register dequantization plus shape-selected 64-token by 64/128/256-row WMMA tiles and two-wide activation staging on the wider tiles, without Q8_K activation scratch or an F16 weight sidecar. | [rocm/ds4_rocm_q4.cuh:1064](rocm/ds4_rocm_q4.cuh#L1064) | -| `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_SSD` | value-aware SSD-only opt-in, default off; unset/0/false/no/off retains TILE8/TILE4, while empty or any other value requests direct-Q4 WMMA; eligibility additionally requires the complete projection weight range in physical device storage rather than mapped/registered host memory; DISABLE wins | Allow the compressed direct-Q4 WMMA kernel to consume an already device-resident/cache-backed Q4_K projection during SSD streaming without changing model I/O. | [rocm/ds4_rocm_q4.cuh:1066](rocm/ds4_rocm_q4.cuh#L1066) | +| `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA` | value-aware compatibility control; unset keeps automatic resident direct-Q4 WMMA for standalone dense and attention-output A while all-Q4 attention-output B remains on Q8_K+TILE8; empty or any value other than 0/false/no/off also opts B into experimental direct WMMA; explicit 0/false/no/off opts out of resident WMMA; N=256..4096, K a positive multiple of 256, resident non-quality gfx1151 wave32 only; SSD has a separate gate and DISABLE wins | Use compressed Q4_K-to-F16 register dequantization plus shape-selected 64-token by 64/128/256-row WMMA tiles and two-wide activation staging on the wider tiles, without Q8_K activation scratch or an F16 weight sidecar. | [rocm/ds4_rocm_q4.cuh:1089](rocm/ds4_rocm_q4.cuh#L1089) | +| `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_SSD` | value-aware SSD-only opt-in, default off; unset/0/false/no/off retains TILE8/TILE4, while empty or any other value requests direct-Q4 WMMA including attention-output B; eligibility additionally requires each complete projection weight range in physical device storage rather than mapped/registered host memory; DISABLE wins | Allow the compressed direct-Q4 WMMA kernel to consume an already device-resident/cache-backed Q4_K projection during SSD streaming without changing model I/O. | [rocm/ds4_rocm_q4.cuh:1091](rocm/ds4_rocm_q4.cuh#L1091) | | `DS4_ROCM_ENABLE_STREAMING_FULL_EXPERT_ADDR_TABLE` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming full expert addr table. | [ds4.c:18255](ds4.c#L18255) | | `DS4_ROCM_ENABLE_STREAMING_MADVISE_WILLNEED` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming madvise willneed. | [ds4.c:18224](ds4.c#L18224) | | `DS4_ROCM_ENABLE_STREAMING_PREFILL_BATCH_SELECTED_ADDR` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming prefill batch selected addr. | [ds4.c:18584](ds4.c#L18584) | @@ -1070,7 +1070,7 @@ and **19 tool/wrapper entries**. | `DS4_ROCM_Q_STAGE_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm q stage profile. | [ds4.c:29284](ds4.c#L29284) | | `DS4_ROCM_REQUIRE_Q4_GROUPED_ATTN_A` | presence fail-closed assertion; also requests the candidate outside the caller-marked resident decode default; DISABLE remains authoritative and causes failure | Require grouped Q4 attention-A and fail instead of silently falling back. | [rocm/ds4_rocm_q4.cuh:870](rocm/ds4_rocm_q4.cuh#L870) | | `DS4_ROCM_REQUIRE_Q4_PREFILL_TILE8` | presence fail-closed assertion for eligible TILE8 calls | Require rocm require q4 prefill tile8 and fail instead of silently falling back. | [rocm/ds4_rocm_q4.cuh:452](rocm/ds4_rocm_q4.cuh#L452) | -| `DS4_ROCM_REQUIRE_Q4_PREFILL_WMMA` | value-aware strict assertion, not required for the automatic resident default; unset/0/false/no/off is off; empty or any other value requires every selected Q4 dense or attention-output projection to use direct-Q4 WMMA; unsupported shape/device, quality mode, DISABLE, or an SSD weight range without physical device residency fails before the relevant dispatch | Prevent the ROCm Q4 prefill WMMA correctness/performance oracle from silently timing TILE8/TILE4; the fused q_a/KV pair yields to separately checked dense calls. | [rocm/ds4_rocm_q4.cuh:1070](rocm/ds4_rocm_q4.cuh#L1070) | +| `DS4_ROCM_REQUIRE_Q4_PREFILL_WMMA` | value-aware strict assertion, not required for the automatic resident default; unset/0/false/no/off is off; empty or any other value requires every selected Q4 dense or attention-output projection to use direct-Q4 WMMA, explicitly including experimental all-Q4 attention-output B; unsupported shape/device, quality mode, DISABLE, or an SSD weight range without physical device residency fails before dispatch | Prevent the ROCm Q4 prefill WMMA correctness/performance oracle from silently timing TILE8/TILE4; quality-sensitive defaults retain Q8_K+TILE8 for attention-output B. | [rocm/ds4_rocm_q4.cuh:1095](rocm/ds4_rocm_q4.cuh#L1095) | | `DS4_ROCM_STREAMING_DECODE_PREFILL_MAX` | primary nonempty value over the Metal alias; parsed by strtol when it has a numeric prefix (trailing text is accepted); <= 0 disables, values > UINT32_MAX clamp, no numeric prefix uses automatic default: 64 for Flash with uniform Q4_K/MXFP4 experts, 18 for other Pro/Flash, otherwise 0; the disable flag dominates | Sets the largest short, non-quality SSD-streaming prefill batch routed through the decode-style path instead of canonical layer-major prefill. | [ds4.c:31976](ds4.c#L31976) | | `DS4_ROCM_STREAMING_EXPERT_AUTO_PRELOAD_CAP` | primary nonempty value over the Metal alias; strict full-string strtoul; valid values > UINT32_MAX clamp, invalid uses 4096, and 0 means no cap (not disabled); when CLI preload is auto/0, unset defaults to cap 4096 except ROCm GLM52, where absent/empty disables automatic preload entirely | Caps the number of hot experts synchronously seeded into the SSD-streaming expert cache in automatic preload mode; an explicit CLI preload count bypasses this cap, and setting this variable opts ROCm GLM52 back into auto preload. | [ds4.c:21469](ds4.c#L21469) | | `DS4_ROCM_STREAMING_EXPERT_CACHE_VERBOSE` | presence flag; unset=off | Print verbose ROCm streaming expert-cache seed/load diagnostics. | [rocm/ds4_rocm_runtime.cuh:2904](rocm/ds4_rocm_runtime.cuh#L2904) | diff --git a/rocm/ds4_rocm_moe.cuh b/rocm/ds4_rocm_moe.cuh index bc46a70f3..ce86b0e15 100644 --- a/rocm/ds4_rocm_moe.cuh +++ b/rocm/ds4_rocm_moe.cuh @@ -758,11 +758,12 @@ __global__ static void q8_K_quantize_kernel(cuda_block_q8_K *out, const float *x /* Wave32 Q8_K activation quantizer for the Q4 prefill path. * * A 256-thread workgroup quantizes up to eight independent 256-value blocks, - * one per wave. Each lane owns the eight values lane + 32*k. The max - * reduction carries the original element index so equal absolute maxima keep - * the lowest index, matching q8_K_quantize_kernel's left-biased reduction. - * The scale, lrintf conversion, clamp and 16-value bsums are otherwise the - * canonical operations above. No cross-wave communication or LDS is used. + * one per wave. Each lane owns the eight values lane + 32*k. Visiting those + * values in three-bit reversed order, then reducing lanes with strict-greater + * shuffle comparisons, reproduces q8_K_quantize_kernel's complete 256-thread + * reduction tree including its tie behavior. The scale, lrintf conversion, + * clamp and 16-value bsums are otherwise the canonical operations above. No + * cross-wave communication or LDS is used. * * Dispatch is deliberately restricted to a runtime-proven wave32 target in * ds4_rocm_q4.cuh. Keep the device guard as a second fail-closed boundary in @@ -792,20 +793,23 @@ __global__ static void q8_K_quantize_wave32_kernel( (uint64_t)b * CUDA_QK_K; cuda_block_q8_K *yb = out + (uint64_t)row * blocks_per_row + b; - uint32_t max_index = lane; - float max_value = xr[max_index]; + float max_value = xr[lane]; float max_abs = fabsf(max_value); #pragma unroll - for (uint32_t k = 1u; k < CUDA_QK_K / ROCM_Q8_K_WAVE32_WIDTH; - k++) { + for (uint32_t order = 1u; + order < CUDA_QK_K / ROCM_Q8_K_WAVE32_WIDTH; order++) { + /* q8_K_quantize_kernel compares strides 128, 64 and 32 before + * crossing lanes. Bit-reversing the local k bits gives the same + * priority order: 0,4,2,6,1,5,3,7. */ + const uint32_t k = ((order & 1u) << 2u) | + (order & 2u) | + ((order & 4u) >> 2u); const uint32_t index = lane + k * ROCM_Q8_K_WAVE32_WIDTH; const float value = xr[index]; const float value_abs = fabsf(value); - if (value_abs > max_abs || - (value_abs == max_abs && index < max_index)) { + if (value_abs > max_abs) { max_abs = value_abs; max_value = value; - max_index = index; } } @@ -814,21 +818,16 @@ __global__ static void q8_K_quantize_wave32_kernel( #if defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__) const float other_abs = __shfl_down(max_abs, offset, 32); const float other_value = __shfl_down(max_value, offset, 32); - const uint32_t other_index = __shfl_down(max_index, offset, 32); #else const float other_abs = __shfl_down_sync( FULL_WARP_MASK, max_abs, offset, 32); const float other_value = __shfl_down_sync( FULL_WARP_MASK, max_value, offset, 32); - const uint32_t other_index = __shfl_down_sync( - FULL_WARP_MASK, max_index, offset, 32); #endif if (lane + offset < ROCM_Q8_K_WAVE32_WIDTH && - (other_abs > max_abs || - (other_abs == max_abs && other_index < max_index))) { + other_abs > max_abs) { max_abs = other_abs; max_value = other_value; - max_index = other_index; } } diff --git a/rocm/ds4_rocm_q4.cuh b/rocm/ds4_rocm_q4.cuh index 382101605..d32fb8e13 100644 --- a/rocm/ds4_rocm_q4.cuh +++ b/rocm/ds4_rocm_q4.cuh @@ -1025,6 +1025,20 @@ static int rocm_q4_K_prefill_wmma_requested_policy( return ssd_streaming ? ssd_enabled == 1 : enabled != 0; } +/* A direct attention-output B consumes the already approximate output of A. + * Unlike standalone dense and attention-output A, do not select it from the + * resident automatic default: require an explicit resident or SSD request. */ +static int rocm_q4_K_prefill_wmma_b_requested_policy( + int ssd_streaming, + int enabled, + int ssd_enabled, + int disabled, + int required) { + if (required) return 1; + if (disabled) return 0; + return ssd_streaming ? ssd_enabled == 1 : enabled == 1; +} + /* An explicitly selected exact Q8_K quantizer owns an optional WMMA default. * REQUIRE_WMMA is the only control that may override an optional Q8 request; * dual REQUIRE is rejected by each public dispatch before enqueue. */ @@ -1055,6 +1069,17 @@ extern "C" int ds4_rocm_test_q4_prefill_wmma_requested_policy( required != 0); } +extern "C" int ds4_rocm_test_q4_prefill_wmma_b_requested_policy( + int ssd_streaming, + int enabled, + int ssd_enabled, + int disabled, + int required) { + return rocm_q4_K_prefill_wmma_b_requested_policy( + ssd_streaming != 0, enabled, ssd_enabled, disabled != 0, + required != 0); +} + static int rocm_q4_K_prefill_wmma_select( uint64_t n_tok, uint64_t in_dim, @@ -2113,7 +2138,13 @@ extern "C" int ds4_gpu_attention_output_q4_K_batch_tensor( * the graph replay a fallback over an already submitted A projection. */ int a_wmma = rocm_q4_K_prefill_wmma_select( n_tokens, group_dim, rank, out_a_device_resident); - int b_wmma = out_b_type == 12u + /* The all-Q4 A+B direct path compounds two F16 input/dequantization + * approximations. Keep the validated A candidate automatic, but leave B + * on exact Q8_K+TILE8 unless the user explicitly requests WMMA. */ + const int b_wmma_requested = rocm_q4_K_prefill_wmma_b_requested_policy( + g_ssd_streaming_mode, wmma_enabled, wmma_ssd_enabled, wmma_disabled, + wmma_required); + int b_wmma = out_b_type == 12u && b_wmma_requested ? rocm_q4_K_prefill_wmma_select( n_tokens, low_dim, out_dim, out_b_device_resident) : ROCM_Q4_PREFILL_WMMA_FALLBACK; @@ -2185,8 +2216,10 @@ extern "C" int ds4_gpu_attention_output_q4_K_batch_tensor( } if (b_rc <= 0) return -1; - if (a_wmma != ROCM_Q4_PREFILL_WMMA_USE && - b_wmma != ROCM_Q4_PREFILL_WMMA_USE) { + const int used_q4_tile8 = + a_wmma != ROCM_Q4_PREFILL_WMMA_USE || + (out_b_type == 12u && b_wmma != ROCM_Q4_PREFILL_WMMA_USE); + if (used_q4_tile8) { rocm_q4_K_prefill_tile8_note(0u, 0u, 1u, 0u, n_tokens); } return 1; diff --git a/scripts/environment_variables.tsv b/scripts/environment_variables.tsv index fb97eb637..9f072558c 100644 --- a/scripts/environment_variables.tsv +++ b/scripts/environment_variables.tsv @@ -1024,8 +1024,8 @@ runtime/rocm DS4_ROCM_ENABLE_Q4_DENSE_PAIR presence opt-in; unset=off; DISABLE t runtime/rocm DS4_ROCM_ENABLE_Q4_GROUPED_ATTN_A presence opt-in outside the default scope; the exact caller-marked resident decode shape groups=8, N=1, K=4096, M=1024 is automatic, while row-at-a-time batch fallbacks are not; DISABLE wins Enable grouped Q4 attention-A for eligible slices, non-production shapes, or explicit experiments in addition to the resident decode default. rocm/ds4_rocm_q4.cuh:872 runtime/rocm DS4_ROCM_ENABLE_Q4_PREFILL_K1024_TILE4_SSD value-aware SSD-only opt-in, default off; unset/0/false/no/off retains TILE8, while empty or any other value requests TILE4; eligibility additionally requires N=9..4096, K=1024, M=32768, TILE8 enabled, and the complete weight range in device storage rather than mapped/registered host memory; DISABLE wins Allow the four-lane K=1024 Q4_K prefill specialization to consume an already device-resident/cache-backed attn_q_b weight range during SSD streaming without changing model I/O. rocm/ds4_rocm_q4.cuh:624 runtime/rocm DS4_ROCM_ENABLE_Q4_PREFILL_Q8_K_WAVE32 value-aware opt-in, default off; unset/0/false/no/off retains the canonical quantizer, while empty or any other value requests the candidate for N=9..4096 on gfx1151 wave32; DISABLE wins; a selected exact Q8 path takes precedence over automatic direct-Q4 WMMA, REQUIRE_WMMA overrides an optional request, and dual REQUIRE fails closed Quantize eight independent Q8_K activation blocks per 256-thread workgroup using one wave32 per block, without LDS or workgroup barriers, before the exact Q4 prefill matmul (TILE8 or its legacy rollback). rocm/ds4_rocm_q4.cuh:858 -runtime/rocm DS4_ROCM_ENABLE_Q4_PREFILL_WMMA value-aware compatibility control for the automatic resident path; unset or any value other than 0/false/no/off permits direct-Q4 WMMA for N=256..4096, K a positive multiple of 256, resident non-quality execution on gfx1151 wave32; explicit 0/false/no/off opts out for resident execution; it never bypasses the SSD gate and DISABLE wins Use compressed Q4_K-to-F16 register dequantization plus shape-selected 64-token by 64/128/256-row WMMA tiles and two-wide activation staging on the wider tiles, without Q8_K activation scratch or an F16 weight sidecar. rocm/ds4_rocm_q4.cuh:1064 -runtime/rocm DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_SSD value-aware SSD-only opt-in, default off; unset/0/false/no/off retains TILE8/TILE4, while empty or any other value requests direct-Q4 WMMA; eligibility additionally requires the complete projection weight range in physical device storage rather than mapped/registered host memory; DISABLE wins Allow the compressed direct-Q4 WMMA kernel to consume an already device-resident/cache-backed Q4_K projection during SSD streaming without changing model I/O. rocm/ds4_rocm_q4.cuh:1066 +runtime/rocm DS4_ROCM_ENABLE_Q4_PREFILL_WMMA value-aware compatibility control; unset keeps automatic resident direct-Q4 WMMA for standalone dense and attention-output A while all-Q4 attention-output B remains on Q8_K+TILE8; empty or any value other than 0/false/no/off also opts B into experimental direct WMMA; explicit 0/false/no/off opts out of resident WMMA; N=256..4096, K a positive multiple of 256, resident non-quality gfx1151 wave32 only; SSD has a separate gate and DISABLE wins Use compressed Q4_K-to-F16 register dequantization plus shape-selected 64-token by 64/128/256-row WMMA tiles and two-wide activation staging on the wider tiles, without Q8_K activation scratch or an F16 weight sidecar. rocm/ds4_rocm_q4.cuh:1089 +runtime/rocm DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_SSD value-aware SSD-only opt-in, default off; unset/0/false/no/off retains TILE8/TILE4, while empty or any other value requests direct-Q4 WMMA including attention-output B; eligibility additionally requires each complete projection weight range in physical device storage rather than mapped/registered host memory; DISABLE wins Allow the compressed direct-Q4 WMMA kernel to consume an already device-resident/cache-backed Q4_K projection during SSD streaming without changing model I/O. rocm/ds4_rocm_q4.cuh:1091 runtime/rocm DS4_ROCM_ENABLE_STREAMING_FULL_EXPERT_ADDR_TABLE presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming full expert addr table. ds4.c:18255 runtime/rocm DS4_ROCM_ENABLE_STREAMING_MADVISE_WILLNEED presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming madvise willneed. ds4.c:18224 runtime/rocm DS4_ROCM_ENABLE_STREAMING_PREFILL_BATCH_SELECTED_ADDR presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming prefill batch selected addr. ds4.c:18584 @@ -1091,7 +1091,7 @@ runtime/rocm DS4_ROCM_REQUIRE_Q4_GROUPED_ATTN_A presence fail-closed assertion; runtime/rocm DS4_ROCM_REQUIRE_Q4_PREFILL_K1024_TILE4 value-aware fail-closed assertion and SSD opt-in; unset/0/false/no/off is off; empty or any other value requires eligible N=9..4096, K=1024, M=32768 dense calls to select TILE4; SSD also requires an actual device-resident weight range; DISABLE wins Prevent a K=1024 TILE4 correctness/performance oracle from silently falling back to TILE8, including during SSD-streaming A/B runs. rocm/ds4_rocm_q4.cuh:628 runtime/rocm DS4_ROCM_REQUIRE_Q4_PREFILL_Q8_K_WAVE32 value-aware strict opt-in; unset/0/false/no/off is off; empty or any other value requires gfx1151 wave32 and N=9..4096; DISABLE and conflicts with required WMMA or the required q_b F16 cache fail closed Require the no-LDS wave32 Q8_K activation quantizer for exact Q4 prefill instead of silently using the canonical quantizer or an F16 side path. rocm/ds4_rocm_q4.cuh:796 runtime/rocm DS4_ROCM_REQUIRE_Q4_PREFILL_TILE8 presence fail-closed assertion for eligible TILE8 calls Require rocm require q4 prefill tile8 and fail instead of silently falling back. rocm/ds4_rocm_q4.cuh:452 -runtime/rocm DS4_ROCM_REQUIRE_Q4_PREFILL_WMMA value-aware strict assertion, not required for the automatic resident default; unset/0/false/no/off is off; empty or any other value requires every selected Q4 dense or attention-output projection to use direct-Q4 WMMA; unsupported shape/device, quality mode, DISABLE, or an SSD weight range without physical device residency fails before the relevant dispatch Prevent the ROCm Q4 prefill WMMA correctness/performance oracle from silently timing TILE8/TILE4; the fused q_a/KV pair yields to separately checked dense calls. rocm/ds4_rocm_q4.cuh:1070 +runtime/rocm DS4_ROCM_REQUIRE_Q4_PREFILL_WMMA value-aware strict assertion, not required for the automatic resident default; unset/0/false/no/off is off; empty or any other value requires every selected Q4 dense or attention-output projection to use direct-Q4 WMMA, explicitly including experimental all-Q4 attention-output B; unsupported shape/device, quality mode, DISABLE, or an SSD weight range without physical device residency fails before dispatch Prevent the ROCm Q4 prefill WMMA correctness/performance oracle from silently timing TILE8/TILE4; quality-sensitive defaults retain Q8_K+TILE8 for attention-output B. rocm/ds4_rocm_q4.cuh:1095 runtime/rocm DS4_ROCM_STREAMING_DECODE_PREFILL_MAX primary nonempty value over the Metal alias; parsed by strtol when it has a numeric prefix (trailing text is accepted); <= 0 disables, values > UINT32_MAX clamp, no numeric prefix uses automatic default: 64 for Flash with uniform Q4_K/MXFP4 experts, 18 for other Pro/Flash, otherwise 0; the disable flag dominates Sets the largest short, non-quality SSD-streaming prefill batch routed through the decode-style path instead of canonical layer-major prefill. ds4.c:31976 runtime/rocm DS4_ROCM_STREAMING_EXPERT_AUTO_PRELOAD_CAP primary nonempty value over the Metal alias; strict full-string strtoul; valid values > UINT32_MAX clamp, invalid uses 4096, and 0 means no cap (not disabled); when CLI preload is auto/0, unset defaults to cap 4096 except ROCm GLM52, where absent/empty disables automatic preload entirely Caps the number of hot experts synchronously seeded into the SSD-streaming expert cache in automatic preload mode; an explicit CLI preload count bypasses this cap, and setting this variable opts ROCm GLM52 back into auto preload. ds4.c:21469 runtime/rocm DS4_ROCM_STREAMING_EXPERT_CACHE_VERBOSE presence flag; unset=off Print verbose ROCm streaming expert-cache seed/load diagnostics. rocm/ds4_rocm_runtime.cuh:2904 diff --git a/tests/test_rocm_q4_dense_pair.cpp b/tests/test_rocm_q4_dense_pair.cpp index d45982599..11301d811 100644 --- a/tests/test_rocm_q4_dense_pair.cpp +++ b/tests/test_rocm_q4_dense_pair.cpp @@ -39,6 +39,9 @@ extern "C" uint64_t ds4_rocm_test_q4_prefill_wmma_get_calls(void); extern "C" int ds4_rocm_test_q4_prefill_wmma_requested_policy( int ssd_streaming, int enabled, int ssd_enabled, int disabled, int required); +extern "C" int ds4_rocm_test_q4_prefill_wmma_b_requested_policy( + int ssd_streaming, int enabled, int ssd_enabled, int disabled, + int required); extern "C" int ds4_rocm_test_q4_prefill_wmma_yields_to_q8_wave32( int q8_selected, int wmma_required); extern "C" uint32_t ds4_rocm_test_q4_prefill_wmma_row_tile( @@ -440,8 +443,8 @@ void fill_activation(std::vector *x, uint32_t n_tokens, } } -/* Exercise the two canonical edge cases for max selection: all-zero blocks - * and equal absolute maxima with opposite signs at different indices. */ +/* Exercise the canonical max-selection edge cases: all-zero blocks and equal + * opposite-sign maxima spanning both the per-lane and cross-lane tree. */ void fill_q8_wave32_activation(std::vector *x, uint32_t n_tokens, uint32_t in_dim) { x->resize((uint64_t)n_tokens * in_dim); @@ -458,22 +461,46 @@ void fill_q8_wave32_activation(std::vector *x, uint32_t n_tokens, block[i] = (float)value / 16.0f; } const float first = ((token + b) & 1u) ? 4.0f : -4.0f; - block[5] = first; - block[224] = -first; + uint32_t first_index = 5u; + uint32_t second_index = 224u; + switch ((token + b) % 3u) { + case 1u: + first_index = 0u; + second_index = 128u; + break; + case 2u: + first_index = 1u; + second_index = 16u; + break; + default: + break; + } + block[first_index] = first; + block[second_index] = -first; } } } void quantize_q8_K_cpu(const float *x, block_q8_K_test *out) { - float amax = 0.0f; - float maxv = 0.0f; + float abs_part[kQkK]; + float val_part[kQkK]; for (uint32_t i = 0; i < kQkK; i++) { - const float av = std::fabs(x[i]); - if (av > amax) { - amax = av; - maxv = x[i]; + abs_part[i] = std::fabs(x[i]); + val_part[i] = x[i]; + } + /* Mirror the canonical GPU reduction literally. A linear scan chooses a + * different signed maximum when equal magnitudes occur, which changes the + * raw Q8_K bytes even though the resulting dot product is equivalent. */ + for (uint32_t stride = kQkK >> 1u; stride != 0u; stride >>= 1u) { + for (uint32_t i = 0; i < stride; i++) { + if (abs_part[i + stride] > abs_part[i]) { + abs_part[i] = abs_part[i + stride]; + val_part[i] = val_part[i + stride]; + } } } + const float amax = abs_part[0]; + const float maxv = val_part[0]; if (amax == 0.0f) { std::memset(out, 0, sizeof(*out)); return; @@ -1213,19 +1240,20 @@ bool run_prefill_wmma_requested_policy_oracle() { int disabled; int required; int expected; + int b_expected; }; const policy_case cases[] = { - {"resident unset is automatic", 0, -1, 0, 0, 0, 1}, - {"resident ENABLE=0 compatibility opt-out", 0, 0, 0, 0, 0, 0}, - {"resident ENABLE=1 remains accepted", 0, 1, 0, 0, 0, 1}, - {"resident DISABLE opts out", 0, -1, 0, 1, 0, 0}, - {"resident REQUIRE requests after ENABLE=0", 0, 0, 0, 0, 1, 1}, - {"DISABLE+REQUIRE reaches strict rejection", 0, -1, 0, 1, 1, 1}, - {"SSD default stays conservative", 1, -1, 0, 0, 0, 0}, - {"generic ENABLE does not bypass SSD gate", 1, 1, 0, 0, 0, 0}, - {"SSD gate requests the candidate", 1, -1, 1, 0, 0, 1}, - {"SSD DISABLE opts out", 1, -1, 1, 1, 0, 0}, - {"SSD REQUIRE requests strict validation", 1, -1, 0, 0, 1, 1}, + {"resident unset is automatic", 0, -1, 0, 0, 0, 1, 0}, + {"resident ENABLE=0 compatibility opt-out", 0, 0, 0, 0, 0, 0, 0}, + {"resident ENABLE=1 remains accepted", 0, 1, 0, 0, 0, 1, 1}, + {"resident DISABLE opts out", 0, -1, 0, 1, 0, 0, 0}, + {"resident REQUIRE requests after ENABLE=0", 0, 0, 0, 0, 1, 1, 1}, + {"DISABLE+REQUIRE reaches strict rejection", 0, -1, 0, 1, 1, 1, 1}, + {"SSD default stays conservative", 1, -1, 0, 0, 0, 0, 0}, + {"generic ENABLE does not bypass SSD gate", 1, 1, 0, 0, 0, 0, 0}, + {"SSD gate requests the candidate", 1, -1, 1, 0, 0, 1, 1}, + {"SSD DISABLE opts out", 1, -1, 1, 1, 0, 0, 0}, + {"SSD REQUIRE requests strict validation", 1, -1, 0, 0, 1, 1, 1}, }; bool ok = true; @@ -1240,6 +1268,17 @@ bool run_prefill_wmma_requested_policy_oracle() { test.label, test.expected, got); ok = false; } + const int b_got = + ds4_rocm_test_q4_prefill_wmma_b_requested_policy( + test.ssd_streaming, test.enabled, test.ssd_enabled, + test.disabled, test.required); + if (b_got != test.b_expected) { + std::fprintf(stderr, + "Q4 attention-output B WMMA policy %s: " + "expected=%d got=%d FAIL\n", + test.label, test.b_expected, b_got); + ok = false; + } } const int optional_q8_yield = ds4_rocm_test_q4_prefill_wmma_yields_to_q8_wave32(1, 0); @@ -2039,6 +2078,21 @@ bool run_attention_prefill_case(const aligned_model &model, return false; } + /* Register/copy A as one contiguous tensor before the row-wise oracle + * requests eight contained group views. Reversing that order creates + * overlapping cache registrations (eight subranges followed by their + * superset) and can fail before either candidate kernel is launched. */ + const uint64_t out_a_row_bytes = + (kAttnGroupDim / kQkK) * sizeof(block_q4_K_test); + const uint64_t out_a_bytes = + (uint64_t)kAttnGroups * kAttnRank * out_a_row_bytes; + if (!ds4_gpu_cache_model_range( + model.data, model.size, model.attn_a_offset, out_a_bytes, + "ROCm Q4 attention-prefill fixture A")) { + std::fprintf(stderr, "%s: attention-output-A preload FAIL\n", label); + return false; + } + env_snapshot enable(kPrefillEnable); env_snapshot disable(kPrefillDisable); env_snapshot require(kPrefillRequire); @@ -2924,7 +2978,7 @@ bool run_attention_output_wmma_smoke(const aligned_model &model) { std::vector wmma_low_host(low_sentinel.size()); std::vector wmma_out_host(out_sentinel.size()); bool ok = tile8_rc == 1 && wmma_rc == 1 && - tile8_wmma_calls == 0u && wmma_calls == 2u && + tile8_wmma_calls == 0u && wmma_calls == 1u && read_tensor(tile8_low.ptr, &tile8_low_host) && read_tensor(tile8_out.ptr, &tile8_out_host) && read_tensor(wmma_low.ptr, &wmma_low_host) && @@ -2941,7 +2995,7 @@ bool run_attention_output_wmma_smoke(const aligned_model &model) { "production output-A direct-WMMA body") && ok; ok = output_body_overwritten( wmma_out_host, out_sentinel, out_count, - "production output-B direct-WMMA body") && ok; + "production output-B A-WMMA/B-TILE8 body") && ok; ok = output_guard_unchanged( tile8_low_host, low_sentinel, low_count, "production output-A TILE8 N-tail") && ok; @@ -2953,7 +3007,7 @@ bool run_attention_output_wmma_smoke(const aligned_model &model) { "production output-A direct-WMMA N-tail") && ok; ok = output_guard_unchanged( wmma_out_host, out_sentinel, out_count, - "production output-B direct-WMMA N-tail") && ok; + "production output-B A-WMMA/B-TILE8 N-tail") && ok; tile8_low_host.resize(low_count); tile8_out_host.resize(out_count); wmma_low_host.resize(low_count); @@ -2963,11 +3017,11 @@ bool run_attention_output_wmma_smoke(const aligned_model &model) { "production output-A direct-WMMA vs TILE8") && ok; ok = close_with_tolerance( wmma_out_host, tile8_out_host, 16.0f, 8.0e-2f, - "production output-A+B direct-WMMA vs TILE8") && ok; + "production output A-WMMA/B-TILE8 vs all-TILE8") && ok; } std::fprintf(stderr, - "ROCm Q4 production output direct-WMMA: tile8=%d/%llu " - "wmma=%d/%llu %s\n", + "ROCm Q4 production output WMMA safety default: " + "tile8=%d/%llu A-WMMA/B-TILE8=%d/%llu %s\n", tile8_rc, (unsigned long long)tile8_wmma_calls, wmma_rc, (unsigned long long)wmma_calls, ok ? "PASS" : "FAIL"); From 377fbe5c3187d6405926afb61f4c1d0a8d4cff2d Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Mon, 31 Aug 2026 13:28:42 +0200 Subject: [PATCH 179/189] cuda: extend Q4 16-warp to attention output B --- ENVIRONMENT_VARIABLES.md | 4 +-- Makefile | 2 +- cuda/mmq/ds4_mmq.cu | 2 +- cuda/mmq/ds4_mmq_q4_16warp.cu | 2 +- cuda/mmq/ds4_mmq_q4_16warp.cuh | 1 + cuda/mmq/test/test_mmq_parity.cu | 25 ++++++++++++++++ scripts/environment_variables.tsv | 2 +- speed-bench/README.md | 19 ++++++++---- speed-bench/cuda_q4_prefill_bench.cu | 44 ++++++++++++++++++++++------ 9 files changed, 81 insertions(+), 20 deletions(-) diff --git a/ENVIRONMENT_VARIABLES.md b/ENVIRONMENT_VARIABLES.md index da0caa630..8d08976f1 100644 --- a/ENVIRONMENT_VARIABLES.md +++ b/ENVIRONMENT_VARIABLES.md @@ -78,7 +78,7 @@ The detailed Metal A/B contracts and expected oracle counters live in | `DS4_CUDA_ENABLE_Q4_K1024_PERSISTENT=1` | Enable the experimental persistent-CTA kernel for the exact `32768x1024` Q4 shape. | | `DS4_CUDA_NO_Q4_K1024_PERSISTENT=1` | Roll back the persistent K1024 experiment. | | `DS4_CUDA_REQUIRE_Q4_K1024_PERSISTENT=1` | Require the K1024 candidate before enqueue instead of silently using canonical MMVQ. | -| `DS4_CUDA_Q4_MMQ_16WARP=1` | Request the experimental exact-integer m128n128 16-warp Q4_K kernel for eligible CUDA prefills. Standalone dense admits `M>=1024`; dense-pair shares one Q8_1 activation and admits legs down to `M=512`. Candidate grids require at least 80% whole-tile SM-wave efficiency; below the canonical 90% cutoff the kernel mirrors canonical Stream-K partitioning and fixup so the FP32 reduction tree is unchanged. Ineligible optional shapes fall back. | +| `DS4_CUDA_Q4_MMQ_16WARP=1` | Request the experimental exact-integer m128n128 16-warp Q4_K kernel for eligible CUDA prefills. Standalone dense admits `M>=1024` and `K<=8192`, including attention output-B; dense-pair shares one Q8_1 activation, admits legs down to `M=512`, and remains bounded to `K<=4096`. Candidate grids require at least 80% whole-tile SM-wave efficiency; below the canonical 90% cutoff the kernel mirrors canonical Stream-K partitioning and fixup so the FP32 reduction tree is unchanged. Ineligible optional shapes fall back. | | `DS4_CUDA_NO_Q4_MMQ_16WARP=1` | Roll back the 16-warp Q4_K prefill experiment. This value-aware switch overrides request and require. | | `DS4_CUDA_REQUIRE_Q4_MMQ_16WARP=1` | Request the 16-warp Q4_K prefill kernel and fail closed instead of silently measuring another MMQ/Q8_K path; a dense-pair requires both legs to be eligible. Decode/speculative batches of at most eight tokens remain on MMVQ. | | `DS4_CUDA_ENABLE_Q8_FOLD=1` | Enable the experimental one-shot Q8_1 producer-to-consumer fold. | @@ -864,7 +864,7 @@ and **19 tool/wrapper entries**. | `DS4_CUDA_Q4_GROUPED_ATTN_A_ORACLE` | value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on | Compare grouped attention-A against the canonical per-group result. | [ds4_cuda.cu:1477](ds4_cuda.cu#L1477) | | `DS4_CUDA_Q4_K1024_PERSISTENT_ORACLE` | value-aware flag, default off; nonempty value other than exact 0 enables and implies candidate admission | Bitwise-compare the exact-shape persistent Q4 K1024 kernel with canonical MMVQ and retain canonical output. | [cuda/mmq/ds4_mmq.cu:3738](cuda/mmq/ds4_mmq.cu#L3738) | | `DS4_CUDA_Q4_K1024_PERSISTENT_STATS` | value-aware flag, default off; nonempty value other than exact 0 enables | Print exact-shape persistent Q4 K1024 dispatch counters at exit. | [cuda/mmq/ds4_mmq.cu:3737](cuda/mmq/ds4_mmq.cu#L3737) | -| `DS4_CUDA_Q4_MMQ_16WARP` | value-aware opt-in cached on the first Q4_K dense or dense-pair MMQ call; unset/empty/exact 0 is off, every other nonempty value requests the candidate; rollback wins; standalone dense requires M>=1024 while dense-pair admits legs down to M=512 and shares one Q8_1 activation; grids require at least 80% whole-tile SM-wave efficiency and use the canonical Stream-K partition/fixup below its 90% cutoff; ineligible optional shapes fall back | Enable the experimental exact-integer CUDA Q4_K m128n128 16-warp kernel for eligible dense and dense-pair prefills without changing the canonical FP32 reduction tree. | [cuda/mmq/ds4_mmq.cu:1126](cuda/mmq/ds4_mmq.cu#L1126) | +| `DS4_CUDA_Q4_MMQ_16WARP` | value-aware opt-in cached on the first Q4_K dense or dense-pair MMQ call; unset/empty/exact 0 is off, every other nonempty value requests the candidate; rollback wins; standalone dense requires M>=1024 and admits K<=8192 including attention output-B, while dense-pair admits legs down to M=512, remains bounded to K<=4096, and shares one Q8_1 activation; grids require at least 80% whole-tile SM-wave efficiency and use the canonical Stream-K partition/fixup below its 90% cutoff; ineligible optional shapes fall back | Enable the experimental exact-integer CUDA Q4_K m128n128 16-warp kernel for eligible dense and dense-pair prefills without changing the canonical FP32 reduction tree. | [cuda/mmq/ds4_mmq.cu:1126](cuda/mmq/ds4_mmq.cu#L1126) | | `DS4_CUDA_Q8_F16_ALL` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Control the Q8 F16 all CUDA quantized-matmul/cache optimization. | [ds4_cuda.cu:2358](ds4_cuda.cu#L2358) | | `DS4_CUDA_Q8_F16_CACHE_MB` | unsigned integer MiB, full-string parse; default unlimited; 0 disables this cache | Limit the selective Q8-to-F16 derived-weight cache. | [ds4_cuda.cu:2218](ds4_cuda.cu#L2218) | | `DS4_CUDA_Q8_F16_CACHE_RESERVE_MB` | unsigned integer MiB, full-string parse; default is VRAM-dependent (>=112 GiB: 512; >=40 GiB: max(768,1%); smaller: max(4096,5%)) | Reserve free VRAM when growing the selective Q8-to-F16 cache. | [ds4_cuda.cu:2224](ds4_cuda.cu#L2224) | diff --git a/Makefile b/Makefile index 4ca3a045c..2044945ac 100644 --- a/Makefile +++ b/Makefile @@ -445,7 +445,7 @@ help: @echo " make rocm-iq2-moe-prefill-bench Build the resident ROCm IQ2/Q2 WMMA A/B harness" @echo " make rocm-q4-prefill-bench Build the resident ROCm Q4 projection/WMMA A/B harness" @echo " make cuda-iq2-moe-prefill-bench CUDA_ARCH=sm_N Build the resident CUDA IQ2/Q2 profiling harness" - @echo " make cuda-q4-prefill-bench CUDA_ARCH=sm_N Build the resident CUDA Q4 dense/pair/q_b/output-A harness" + @echo " make cuda-q4-prefill-bench CUDA_ARCH=sm_N Build the resident CUDA Q4 dense/pair/q_b/output-A/output-B harness" @echo " make test-rocm Core regression suite on ROCm-only hosts" @echo " make cpu Build CPU-only ./ds4, ./ds4-server, ./ds4-bench, ./ds4-eval, and ./ds4-agent" @echo " make test Build and run tests" diff --git a/cuda/mmq/ds4_mmq.cu b/cuda/mmq/ds4_mmq.cu index c87064f8d..6e90605a2 100644 --- a/cuda/mmq/ds4_mmq.cu +++ b/cuda/mmq/ds4_mmq.cu @@ -1134,7 +1134,7 @@ extern "C" int ds4_mmq_q4_K_dense_preq_16warp_for_test( // but it must reject shapes that cannot be represented by this kernel // before enqueueing any work. if (M < 128 || (M % 128) != 0 || N < 512 || - K < 1024 || K > 4096 || (K % QK_K) != 0) { + K < 1024 || K > 8192 || (K % QK_K) != 0) { return DS4_MMQ_NOT_APPLICABLE; } const int dev = ggml_cuda_get_device(); diff --git a/cuda/mmq/ds4_mmq_q4_16warp.cu b/cuda/mmq/ds4_mmq_q4_16warp.cu index 92a179bdb..f1b22b423 100644 --- a/cuda/mmq/ds4_mmq_q4_16warp.cu +++ b/cuda/mmq/ds4_mmq_q4_16warp.cu @@ -492,7 +492,7 @@ extern "C" int ds4_mmq_q4_K_dense_16warp_supported( } return M >= 1024 && (M % q4w16::kMTile) == 0 && N >= 512 && (N % q4w16::kNTile) == 0 && - K >= 1024 && K <= 4096 && (K % QK_K) == 0; + K >= 1024 && K <= 8192 && (K % QK_K) == 0; } extern "C" int ds4_mmq_q4_K_dense_16warp_prepare(void) { diff --git a/cuda/mmq/ds4_mmq_q4_16warp.cuh b/cuda/mmq/ds4_mmq_q4_16warp.cuh index e9d39b4d1..2434330a2 100644 --- a/cuda/mmq/ds4_mmq_q4_16warp.cuh +++ b/cuda/mmq/ds4_mmq_q4_16warp.cuh @@ -25,6 +25,7 @@ int ds4_mmq_q4_K_dense_16warp_available(int cc); // Conservative standalone production admission gate. Availability and shape // are both checked; it admits M>=1024 and only complete 128x128 output tiles. +// The K envelope covers the production 8192-wide attention output projection. // The dispatcher separately enforces the m128n128 reference selector and its // candidate-grid efficiency gate. A false result must fall back. The pair // dispatcher has a separate per-leg M>=512 gate. diff --git a/cuda/mmq/test/test_mmq_parity.cu b/cuda/mmq/test/test_mmq_parity.cu index 60391b1a2..7dd3cf10c 100644 --- a/cuda/mmq/test/test_mmq_parity.cu +++ b/cuda/mmq/test/test_mmq_parity.cu @@ -3269,6 +3269,14 @@ int main(int argc, char ** argv) { prop.major, prop.minor); return 77; } + if (!ds4_mmq_q4_K_dense_16warp_supported( + cc, /*M=*/4096, /*N=*/768, /*K=*/8192) || + ds4_mmq_q4_K_dense_16warp_supported( + cc, /*M=*/4096, /*N=*/768, /*K=*/8448)) { + fprintf(stderr, + "Q4 16-warp K=8192 admission envelope is invalid\n"); + return 1; + } const int prepare_rc = ds4_mmq_q4_K_dense_16warp_prepare(); if (prepare_rc != 0) { fprintf(stderr, @@ -3288,9 +3296,11 @@ int main(int argc, char ** argv) { }; const int dense_4096_eff = grid_efficiency(1024, 4096); const int kv_4096_eff = grid_efficiency(512, 4096); + const int output_b_8192_eff = grid_efficiency(4096, 768); const bool public_dense_4096 = dense_4096_eff >= 80; const bool public_pair_4096 = public_dense_4096 && kv_4096_eff >= 80; + const bool public_output_b_8192 = output_b_8192_eff >= 80; // N=512 and mmq_x=128 give four N tiles. Choose the smallest M-tile // count >=16 for which 4*tiles_m is divisible by nSM. Stream-K then @@ -3345,6 +3355,21 @@ int main(int argc, char ** argv) { "%d%% < 80%% (nsm=%d)\n", dense_4096_eff, prop.multiProcessorCount); } + // Production output-B has M=4096,K=8192. N=768 is a smaller, + // independent set of complete N128 tiles that selects the same + // scratch-free direct kernel as N=2048 on GB10, while the dedicated + // speed benchmark covers the real N=2048 timing geometry. + all_ok &= run_q4_K_dense_16warp_parity( + /*M=*/4096, /*N=*/768, /*K=*/8192, + prop.multiProcessorCount, 0xC4160006u, + /*check_public_dense=*/public_output_b_8192, + /*check_rejection=*/false); + if (!public_output_b_8192) { + fprintf(stderr, + "Q4 16-warp public output-B K=8192 SKIP: grid " + "efficiency %d%% < 80%% (nsm=%d)\n", + output_b_8192_eff, prop.multiProcessorCount); + } if (public_pair_4096) { all_ok &= run_q4_K_dense_pair_16warp_parity( /*M0=*/1024, /*M1=*/512, /*N=*/4096, /*K=*/4096, diff --git a/scripts/environment_variables.tsv b/scripts/environment_variables.tsv index 9f072558c..6c6af683b 100644 --- a/scripts/environment_variables.tsv +++ b/scripts/environment_variables.tsv @@ -291,7 +291,7 @@ runtime/cuda DS4_CUDA_Q4_ATTN_Q_B_TRANSIENT_F16_MIN_TOKENS full-string unsigned runtime/cuda DS4_CUDA_Q4_GROUPED_ATTN_A_ORACLE value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on Compare grouped attention-A against the canonical per-group result. ds4_cuda.cu:1477 runtime/cuda DS4_CUDA_Q4_K1024_PERSISTENT_ORACLE value-aware flag, default off; nonempty value other than exact 0 enables and implies candidate admission Bitwise-compare the exact-shape persistent Q4 K1024 kernel with canonical MMVQ and retain canonical output. cuda/mmq/ds4_mmq.cu:3738 runtime/cuda DS4_CUDA_Q4_K1024_PERSISTENT_STATS value-aware flag, default off; nonempty value other than exact 0 enables Print exact-shape persistent Q4 K1024 dispatch counters at exit. cuda/mmq/ds4_mmq.cu:3737 -runtime/cuda DS4_CUDA_Q4_MMQ_16WARP value-aware opt-in cached on the first Q4_K dense or dense-pair MMQ call; unset/empty/exact 0 is off, every other nonempty value requests the candidate; rollback wins; standalone dense requires M>=1024 while dense-pair admits legs down to M=512 and shares one Q8_1 activation; grids require at least 80% whole-tile SM-wave efficiency and use the canonical Stream-K partition/fixup below its 90% cutoff; ineligible optional shapes fall back Enable the experimental exact-integer CUDA Q4_K m128n128 16-warp kernel for eligible dense and dense-pair prefills without changing the canonical FP32 reduction tree. cuda/mmq/ds4_mmq.cu:1126 +runtime/cuda DS4_CUDA_Q4_MMQ_16WARP value-aware opt-in cached on the first Q4_K dense or dense-pair MMQ call; unset/empty/exact 0 is off, every other nonempty value requests the candidate; rollback wins; standalone dense requires M>=1024 and admits K<=8192 including attention output-B, while dense-pair admits legs down to M=512, remains bounded to K<=4096, and shares one Q8_1 activation; grids require at least 80% whole-tile SM-wave efficiency and use the canonical Stream-K partition/fixup below its 90% cutoff; ineligible optional shapes fall back Enable the experimental exact-integer CUDA Q4_K m128n128 16-warp kernel for eligible dense and dense-pair prefills without changing the canonical FP32 reduction tree. cuda/mmq/ds4_mmq.cu:1126 runtime/cuda DS4_CUDA_Q8_F16_ALL presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies Control the Q8 F16 all CUDA quantized-matmul/cache optimization. ds4_cuda.cu:2358 runtime/cuda DS4_CUDA_Q8_F16_CACHE_MB unsigned integer MiB, full-string parse; default unlimited; 0 disables this cache Limit the selective Q8-to-F16 derived-weight cache. ds4_cuda.cu:2218 runtime/cuda DS4_CUDA_Q8_F16_CACHE_RESERVE_MB unsigned integer MiB, full-string parse; default is VRAM-dependent (>=112 GiB: 512; >=40 GiB: max(768,1%); smaller: max(4096,5%)) Reserve free VRAM when growing the selective Q8-to-F16 cache. ds4_cuda.cu:2224 diff --git a/speed-bench/README.md b/speed-bench/README.md index f931f12d6..7108dca9d 100644 --- a/speed-bench/README.md +++ b/speed-bench/README.md @@ -308,6 +308,11 @@ make cuda-q4-prefill-bench CUDA_ARCH=sm_121 --path mmq --kernel-16warp --case pair \ --tokens 512,1024,2048,2049,4096,6144,8192 \ --sets 4 --samples 16 --warmup 4 +DS4_CUDA_MMQ_X_MAX=128 \ +./speed-bench/cuda_q4_prefill_bench \ + --path mmq --kernel-16warp --case outb \ + --tokens 2048 \ + --sets 4 --samples 16 --warmup 4 ``` The benchmark quantizes X to canonical Q8_1 DS4 once, before timing, and uses @@ -325,7 +330,10 @@ bit-for-bit, checks finite values and independent output canaries, and samples a CPU Q4_K oracle before and after timing. Results attest `timing=kernel_only_prequant`; SSD streaming, model upload, Q8_1 quantization, allocation, host copies, and oracle work are outside the samples. Use -`--case qb` for the additional `K=1024,M=32768` large-projection datapoint. +`--case qb` for the additional `K=1024,M=32768` large-projection datapoint and +`--case outb` for the real attention output-B `K=8192,M=4096` geometry. The +focused oracle covers that wider K with complete N128 tiles; the resident +benchmark retains the real `N=2048` prefill geometry for performance claims. The focused test also invokes the real standalone dense and pair dispatchers in required mode at `N=4096`, so fallback fails instead of producing a misleading canonical-path pass. A negative pair case verifies that an ineligible 384-row @@ -342,10 +350,11 @@ falls back on ineligible shapes. For an attested production benchmark, `DS4_CUDA_REQUIRE_Q4_MMQ_16WARP=1` also prevents a Q8_K/MMQ fallback; `DS4_CUDA_NO_Q4_MMQ_16WARP=1` is the value-aware rollback. The strict path requires Ampere or newer, complete 128x128 output tiles, `N>=512`, -`1024<=K<=4096`, the default 128-column MMQ selector, and at least 80% final-wave -grid efficiency. Single dense admission starts at `M=1024`; the Q-A/KV pair -admission also accepts its `M1=512` leg when both projections select the -16-warp path. +`1024<=K<=8192`, the default 128-column MMQ selector, and at least 80% final-wave +grid efficiency. Single dense admission starts at `M=1024`, including the +`K=8192` output-B projection; the Q-A/KV pair admission also accepts its +`M1=512` leg when both projections select the 16-warp path, and remains bounded +to `K<=4096`. On GB10, isolate the production Flash attention-output A geometry (`groups=8`, `K=4096`, `rank=1024`) and its 127/128/129 token tails with: diff --git a/speed-bench/cuda_q4_prefill_bench.cu b/speed-bench/cuda_q4_prefill_bench.cu index 2e82962e3..254f8c554 100644 --- a/speed-bench/cuda_q4_prefill_bench.cu +++ b/speed-bench/cuda_q4_prefill_bench.cu @@ -33,6 +33,7 @@ constexpr uint32_t kOutputGroups = 8u; constexpr uint32_t kOutputRank = 1024u; constexpr uint32_t kOutputLowDim = kOutputGroups * kOutputRank; constexpr uint32_t kOutputMinB = 256u; +constexpr uint32_t kOutputM = 4096u; constexpr uint32_t kDefaultSets = 4u; constexpr uint32_t kDefaultSamples = 8u; constexpr uint32_t kDefaultWarmup = 2u; @@ -71,6 +72,7 @@ enum class bench_case { pair, qb, outa, + outb, }; enum class cuda_path { @@ -101,6 +103,7 @@ struct model_fixture { uint8_t *data = nullptr; uint64_t size = 0; uint64_t payload_bytes = 0; + uint32_t output_b_rows = kOutputMinB; std::vector weights; ~model_fixture() { std::free(data); } @@ -329,7 +332,9 @@ uint64_t q4_weight_bytes(uint32_t in_dim, uint32_t out_dim) { sizeof(block_q4_K_host); } -bool make_model(model_fixture *model, uint32_t sets) { +bool make_model(model_fixture *model, uint32_t sets, + uint32_t output_b_rows) { + if (output_b_rows < kOutputMinB) return false; constexpr uint64_t page = 4096u; const uint64_t dense_bytes = q4_weight_bytes(kDenseK, kDenseM); const uint64_t kv_bytes = q4_weight_bytes(kDenseK, kKvM); @@ -337,8 +342,9 @@ bool make_model(model_fixture *model, uint32_t sets) { const uint64_t output_a_bytes = q4_weight_bytes(kDenseK, kOutputLowDim); const uint64_t output_b_bytes = - q4_weight_bytes(kOutputLowDim, kOutputMinB); + q4_weight_bytes(kOutputLowDim, output_b_rows); + model->output_b_rows = output_b_rows; model->weights.resize(sets); uint64_t cursor = 0; auto append = [&](uint64_t bytes) { @@ -738,12 +744,13 @@ const char *path_name(cuda_path path) { const char *case_scope(const config &cfg) { switch (cfg.selected) { case bench_case::all: - return cfg.kernel_16warp ? "dense,pair,q_b" + return cfg.kernel_16warp ? "dense,pair,q_b,output_b" : "dense,pair,q_b,outa"; case bench_case::dense: return "dense"; case bench_case::pair: return "pair"; case bench_case::qb: return "q_b"; case bench_case::outa: return "outa"; + case bench_case::outb: return "output_b"; } return "unknown"; } @@ -1697,7 +1704,7 @@ void usage(FILE *stream, const char *argv0) { "usage: %s [options]\n\n" "Resident CUDA Q4_K prefill kernel benchmark (CUDA event timing).\n\n" " --path mmq|legacy process-wide path (default: mmq)\n" - " --case all|dense|pair|qb|outa\n" + " --case all|dense|pair|qb|outa|outb\n" " case to run (default: all)\n" " --tokens N[,N...] token counts, each 9..8192\n" " --full use 9,16,17,31,32,33,127,128,129,256," @@ -1719,7 +1726,8 @@ void usage(FILE *stream, const char *argv0) { "DS4_CUDA_MMQ_X_MAX\nmay explicitly select an 8..128 multiple-of-8 " "sweep point; the setup line\nattests it, or prints auto when the " "variable is unset. --kernel-16warp requires\n--path mmq and " - "--case dense/pair/qb; with --case all it runs dense, pair, and q_b. Its " + "--case dense/pair/qb/outb; with --case all it also runs the real " + "K=8192,M=4096 output-B. Its " "pair arm\nshares one prequantized activation across M=1024+512. Token " "counts must be\n>=512 and select canonical m128n128 on the active " "device; without --tokens/--full it uses " @@ -1805,6 +1813,8 @@ config parse_options(int argc, char **argv) { cfg.selected = bench_case::qb; } else if (!std::strcmp(value, "outa")) { cfg.selected = bench_case::outa; + } else if (!std::strcmp(value, "outb")) { + cfg.selected = bench_case::outb; } else { std::fprintf(stderr, "invalid --case: %s\n", value); std::exit(2); @@ -1862,7 +1872,12 @@ config parse_options(int argc, char **argv) { if (cfg.kernel_16warp && cfg.selected == bench_case::outa) { std::fprintf(stderr, "--kernel-16warp supports only --case dense, pair, qb, " - "or all (all runs dense+pair+qb)\n"); + "outb, or all\n"); + std::exit(2); + } + if (!cfg.kernel_16warp && cfg.selected == bench_case::outb) { + std::fprintf(stderr, + "--case outb requires --kernel-16warp\n"); std::exit(2); } if (cfg.path == cuda_path::legacy && @@ -1932,7 +1947,7 @@ bool verify_resident_weight_ranges(const model_fixture &model) { const uint64_t output_a_bytes = q4_weight_bytes(kDenseK, kOutputLowDim); const uint64_t output_b_bytes = - q4_weight_bytes(kOutputLowDim, kOutputMinB); + q4_weight_bytes(kOutputLowDim, model.output_b_rows); for (uint32_t set = 0; set < model.weights.size(); set++) { struct range_desc { const char *name; @@ -1944,7 +1959,9 @@ bool verify_resident_weight_ranges(const model_fixture &model) { {"kv", model.weights[set].kv_offset, kv_bytes}, {"q_b", model.weights[set].qb_offset, qb_bytes}, {"output_a", model.weights[set].output_a_offset, output_a_bytes}, - {"output_b_min", model.weights[set].output_b_offset, + {model.output_b_rows == kOutputMinB + ? "output_b_min" : "output_b", + model.weights[set].output_b_offset, output_b_bytes}, }; for (const range_desc &range : ranges) { @@ -2095,7 +2112,10 @@ int main(int argc, char **argv) { bool ok = true; model_fixture model; - if (!make_model(&model, cfg.sets)) { + const uint32_t output_b_rows = + cfg.kernel_16warp && includes(cfg.selected, bench_case::outb) + ? kOutputM : kOutputMinB; + if (!make_model(&model, cfg.sets, output_b_rows)) { std::fprintf(stderr, "cuda-q4-prefill-bench: model fixture allocation failed\n"); ok = false; @@ -2176,6 +2196,12 @@ int main(int argc, char **argv) { &weight_set::qb_offset, "q_b") : run_qb(model, cfg, n_tokens)) && ok; } + if (ok && cfg.kernel_16warp && + includes(cfg.selected, bench_case::outb)) { + ok = run_q4_16warp_kernel( + model, cfg, n_tokens, kOutputLowDim, kOutputM, + &weight_set::output_b_offset, "output_b") && ok; + } if (ok && !cfg.kernel_16warp && grouped_prefill_supported && includes(cfg.selected, bench_case::outa)) { ok = run_output_a(model, cfg, n_tokens) && ok; From 5c5b1d516e185982a218d22e248715f27c38bbfa Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Tue, 1 Sep 2026 20:39:32 +0200 Subject: [PATCH 180/189] Fix ROCm build after upstream merge --- cuda/mmq/vendors/hip.h | 4 ++ rocm/ds4_rocm_attention.cuh | 79 ------------------------ rocm/ds4_rocm_attention_launch.cuh | 99 ++++++++++-------------------- 3 files changed, 37 insertions(+), 145 deletions(-) diff --git a/cuda/mmq/vendors/hip.h b/cuda/mmq/vendors/hip.h index 27885edc0..5099a4437 100644 --- a/cuda/mmq/vendors/hip.h +++ b/cuda/mmq/vendors/hip.h @@ -58,6 +58,7 @@ #define cudaDeviceProp hipDeviceProp_t #define cudaDeviceSynchronize hipDeviceSynchronize #define cudaError_t hipError_t +#define cudaErrorInvalidValue hipErrorInvalidValue #define cudaErrorMemoryAllocation hipErrorOutOfMemory #define cudaErrorPeerAccessAlreadyEnabled hipErrorPeerAccessAlreadyEnabled #define cudaErrorPeerAccessNotEnabled hipErrorPeerAccessNotEnabled @@ -97,7 +98,10 @@ #define cudaMemset hipMemset #define cudaMemsetAsync hipMemsetAsync #define cudaMemGetInfo hipMemGetInfo +#define cudaMemoryTypeDevice hipMemoryTypeDevice #define cudaOccupancyMaxPotentialBlockSize hipOccupancyMaxPotentialBlockSize +#define cudaPointerAttributes hipPointerAttribute_t +#define cudaPointerGetAttributes hipPointerGetAttributes #define cudaSetDevice hipSetDevice #define cuDeviceGet hipDeviceGet #define CUdevice hipDevice_t diff --git a/rocm/ds4_rocm_attention.cuh b/rocm/ds4_rocm_attention.cuh index 7eaf90cd3..a162fd180 100644 --- a/rocm/ds4_rocm_attention.cuh +++ b/rocm/ds4_rocm_attention.cuh @@ -204,85 +204,6 @@ __global__ static void attention_prefill_raw_kernel( } } -/* Non-causal attention used by the DSpark draft block. Every draft query sees - * the complete visible raw-KV ring plus the learned per-head sink. Keep the - * scalar accumulation order aligned with the CUDA reference path. */ -__global__ static void attention_noncausal_raw_batch_heads_kernel( - float *heads, - const float *sinks, - const float *q, - const float *raw_kv, - uint32_t n_tokens, - uint32_t n_raw, - uint32_t raw_cap, - uint32_t raw_start, - uint32_t n_head, - uint32_t head_dim) { - const uint32_t tok = blockIdx.x; - const uint32_t h = blockIdx.y; - if (tok >= n_tokens || h >= n_head) return; - - extern __shared__ float sh_scores[]; - __shared__ float partial[256]; - __shared__ float max_s; - __shared__ float denom; - const float *qh = q + ((uint64_t)tok * n_head + h) * head_dim; - const float scale = rsqrtf((float)head_dim); - - for (uint32_t r = threadIdx.x; r < n_raw; r += blockDim.x) { - const uint32_t row = (raw_start + r) % raw_cap; - const float *kv = raw_kv + (uint64_t)row * head_dim; - float dot = 0.0f; - for (uint32_t d = 0; d < head_dim; d++) dot += qh[d] * kv[d]; - sh_scores[r] = dot * scale; - } - __syncthreads(); - - float local_max = sinks[h]; - for (uint32_t r = threadIdx.x; r < n_raw; r += blockDim.x) { - local_max = fmaxf(local_max, sh_scores[r]); - } - partial[threadIdx.x] = local_max; - __syncthreads(); - for (uint32_t stride = blockDim.x >> 1u; stride > 0u; stride >>= 1u) { - if (threadIdx.x < stride) { - partial[threadIdx.x] = - fmaxf(partial[threadIdx.x], partial[threadIdx.x + stride]); - } - __syncthreads(); - } - if (threadIdx.x == 0) max_s = partial[0]; - __syncthreads(); - - float den_local = 0.0f; - for (uint32_t r = threadIdx.x; r < n_raw; r += blockDim.x) { - sh_scores[r] = expf(sh_scores[r] - max_s); - den_local += sh_scores[r]; - } - partial[threadIdx.x] = den_local; - __syncthreads(); - for (uint32_t stride = blockDim.x >> 1u; stride > 0u; stride >>= 1u) { - if (threadIdx.x < stride) { - partial[threadIdx.x] += partial[threadIdx.x + stride]; - } - __syncthreads(); - } - if (threadIdx.x == 0) { - denom = partial[0] + expf(sinks[h] - max_s); - } - __syncthreads(); - - float *oh = heads + ((uint64_t)tok * n_head + h) * head_dim; - for (uint32_t d = threadIdx.x; d < head_dim; d += blockDim.x) { - float acc = 0.0f; - for (uint32_t r = 0; r < n_raw; r++) { - const uint32_t row = (raw_start + r) % raw_cap; - acc += raw_kv[(uint64_t)row * head_dim + d] * sh_scores[r]; - } - oh[d] = acc / denom; - } -} - __global__ static void attention_prefill_mixed_kernel( float *heads, const float *sinks, diff --git a/rocm/ds4_rocm_attention_launch.cuh b/rocm/ds4_rocm_attention_launch.cuh index 1cf86971c..54bd54b7c 100644 --- a/rocm/ds4_rocm_attention_launch.cuh +++ b/rocm/ds4_rocm_attention_launch.cuh @@ -84,12 +84,14 @@ static int rocm_attention_env_bool_value( /* Cache a successful device-property query per host thread and active device. * This is intentionally not a process-global, unkeyed architecture flag: * HIP's active device is thread-local and callers may switch devices. */ -static int rocm_attention_runtime_is_gfx1151_wave32(void) { +static int rocm_attention_runtime_is_gfx1151_wave32( + int report_unavailable) { #if defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__) int device = -1; const cudaError_t device_err = cudaGetDevice(&device); if (device_err != cudaSuccess || device < 0) { - if (rocm_attention_wmma_notice_once(1u << 3)) { + if (report_unavailable && + rocm_attention_wmma_notice_once(1u << 3)) { fprintf(stderr, DS4_GPU_LOG_PREFIX "cannot query active device for " "gfx1151 WMMA attention; using fallback: %s\n", @@ -102,13 +104,20 @@ static int rocm_attention_runtime_is_gfx1151_wave32(void) { static thread_local int cached_device = -1; static thread_local int cached_eligible = -1; if (cached_device == device && cached_eligible >= 0) { + if (!cached_eligible && report_unavailable && + rocm_attention_wmma_notice_once(1u << 5)) { + fprintf(stderr, + DS4_GPU_LOG_PREFIX "gfx1151 WMMA attention requested on " + "a cached unsupported device; using fallback\n"); + } return cached_eligible; } cudaDeviceProp prop = {}; const cudaError_t prop_err = cudaGetDeviceProperties(&prop, device); if (prop_err != cudaSuccess) { - if (rocm_attention_wmma_notice_once(1u << 4)) { + if (report_unavailable && + rocm_attention_wmma_notice_once(1u << 4)) { fprintf(stderr, DS4_GPU_LOG_PREFIX "cannot query device properties for " "gfx1151 WMMA attention; using fallback: %s\n", @@ -125,7 +134,8 @@ static int rocm_attention_runtime_is_gfx1151_wave32(void) { prop.gcnArchName[sizeof(kGfx1151) - 1u] == ':'); cached_device = device; cached_eligible = arch_matches && prop.warpSize == 32 ? 1 : 0; - if (!cached_eligible && rocm_attention_wmma_notice_once(1u << 5)) { + if (!cached_eligible && report_unavailable && + rocm_attention_wmma_notice_once(1u << 5)) { fprintf(stderr, DS4_GPU_LOG_PREFIX "gfx1151 WMMA attention requested on " "unsupported device arch=%s warpSize=%d; using fallback\n", @@ -134,6 +144,7 @@ static int rocm_attention_runtime_is_gfx1151_wave32(void) { } return cached_eligible; #else + (void)report_unavailable; return 0; #endif } @@ -166,7 +177,7 @@ static int rocm_attention_gfx1151_wmma_enabled( } return 0; } - if (!rocm_attention_runtime_is_gfx1151_wave32()) return 0; + if (!rocm_attention_runtime_is_gfx1151_wave32(1)) return 0; if (rocm_attention_wmma_notice_once(enabled_notice_bit)) { fprintf(stderr, DS4_GPU_LOG_PREFIX "gfx1151 wave32 WMMA attention enabled " @@ -214,20 +225,28 @@ extern "C" int ds4_gpu_attention_noncausal_raw_batch_heads_tensor( uint32_t raw_start, uint32_t n_head, uint32_t head_dim) { + uint64_t sink_bytes = 0; + uint64_t head_elems = 0; + uint64_t head_bytes = 0; + uint64_t raw_bytes = 0; + const bool sizes_ok = + cuda_u64_mul_checked(n_head, sizeof(float), &sink_bytes) && + cuda_u64_mul3_checked(n_tokens, n_head, head_dim, &head_elems) && + cuda_u64_mul_checked(head_elems, sizeof(float), &head_bytes) && + cuda_u64_mul3_checked(raw_cap, head_dim, sizeof(float), &raw_bytes); if (!heads || !q || !raw_kv || !model_map || + !sizes_ok || n_tokens == 0 || n_raw == 0 || raw_cap < n_raw || raw_start >= raw_cap || n_head == 0 || head_dim == 0 || - sinks_offset > model_size || - (uint64_t)n_head * sizeof(float) > model_size - sinks_offset || - heads->bytes < (uint64_t)n_tokens * n_head * head_dim * sizeof(float) || - q->bytes < (uint64_t)n_tokens * n_head * head_dim * sizeof(float) || - raw_kv->bytes < (uint64_t)raw_cap * head_dim * sizeof(float)) { + sinks_offset > model_size || sink_bytes > model_size - sinks_offset || + heads->bytes < head_bytes || q->bytes < head_bytes || + raw_kv->bytes < raw_bytes) { return 0; } const float *sinks = (const float *)cuda_model_range_ptr( model_map, sinks_offset, - (uint64_t)n_head * sizeof(float), + sink_bytes, "dspark_attn_sinks"); if (!sinks) return 0; @@ -257,8 +276,8 @@ extern "C" int ds4_gpu_attention_noncausal_raw_batch_heads_tensor( if (verify_left > 0) { verify_left--; (void)cudaDeviceSynchronize(); - const uint64_t qn = (uint64_t)n_tokens * n_head * head_dim; - const uint64_t kn = (uint64_t)raw_cap * head_dim; + const uint64_t qn = head_elems; + const uint64_t kn = raw_bytes / sizeof(float); std::vector hq(qn), hkv(kn), hout(qn), hsink(n_head); (void)cudaMemcpy(hq.data(), q->ptr, qn * sizeof(float), cudaMemcpyDeviceToHost); @@ -266,7 +285,7 @@ extern "C" int ds4_gpu_attention_noncausal_raw_batch_heads_tensor( cudaMemcpyDeviceToHost); (void)cudaMemcpy(hout.data(), heads->ptr, qn * sizeof(float), cudaMemcpyDeviceToHost); - (void)cudaMemcpy(hsink.data(), sinks, (uint64_t)n_head * sizeof(float), + (void)cudaMemcpy(hsink.data(), sinks, sink_bytes, cudaMemcpyDeviceToHost); double max_abs = 0.0; double max_rel = 0.0; @@ -651,58 +670,6 @@ extern "C" int ds4_gpu_attention_decode_raw_batch_heads_tensor( n_head, head_dim); } -extern "C" int ds4_gpu_attention_noncausal_raw_batch_heads_tensor( - ds4_gpu_tensor *heads, - const void *model_map, - uint64_t model_size, - uint64_t sinks_offset, - const ds4_gpu_tensor *q, - const ds4_gpu_tensor *raw_kv, - uint32_t n_tokens, - uint32_t n_raw, - uint32_t raw_cap, - uint32_t raw_start, - uint32_t n_head, - uint32_t head_dim) { - uint64_t sink_bytes = 0; - uint64_t head_elems = 0; - uint64_t head_bytes = 0; - uint64_t raw_bytes = 0; - const bool sizes_ok = - cuda_u64_mul_checked(n_head, sizeof(float), &sink_bytes) && - cuda_u64_mul3_checked(n_tokens, n_head, head_dim, &head_elems) && - cuda_u64_mul_checked(head_elems, sizeof(float), &head_bytes) && - cuda_u64_mul3_checked(raw_cap, head_dim, sizeof(float), &raw_bytes); - if (!heads || !q || !raw_kv || !model_map || - !sizes_ok || - n_tokens == 0 || n_raw == 0 || raw_cap < n_raw || - raw_start >= raw_cap || n_head == 0 || head_dim == 0 || - sinks_offset > model_size || sink_bytes > model_size - sinks_offset || - heads->bytes < head_bytes || q->bytes < head_bytes || - raw_kv->bytes < raw_bytes) { - return 0; - } - const float *sinks = (const float *)cuda_model_range_ptr( - model_map, sinks_offset, sink_bytes, "dspark_attn_sinks"); - if (!sinks) return 0; - const size_t shmem = (size_t)n_raw * sizeof(float); - if (shmem > 32768u) return 0; - dim3 grid(n_tokens, n_head, 1); - attention_noncausal_raw_batch_heads_kernel<<>>( - (float *)heads->ptr, - sinks, - (const float *)q->ptr, - (const float *)raw_kv->ptr, - n_tokens, - n_raw, - raw_cap, - raw_start, - n_head, - head_dim); - return cuda_ok(cudaGetLastError(), - "attention noncausal raw batch heads launch"); -} - extern "C" int ds4_gpu_attention_decode_mixed_batch_heads_tensor( ds4_gpu_tensor *heads, const void *model_map, From 555f68a235ec18abe460f2834b1df9ebc142173d Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Tue, 1 Sep 2026 22:00:14 +0200 Subject: [PATCH 181/189] Default ROCm Q4 prefill WMMA to K64 staging --- ENVIRONMENT_VARIABLES.md | 6 +- rocm/ds4_rocm_q4.cuh | 375 +++++++++++++++++++++++++- scripts/environment_variables.tsv | 1 + speed-bench/README.md | 18 ++ speed-bench/rocm_q4_prefill_bench.cpp | 177 +++++++++++- tests/test_rocm_q4_dense_pair.cpp | 169 +++++++++++- 6 files changed, 719 insertions(+), 27 deletions(-) diff --git a/ENVIRONMENT_VARIABLES.md b/ENVIRONMENT_VARIABLES.md index 8d08976f1..4de3c2931 100644 --- a/ENVIRONMENT_VARIABLES.md +++ b/ENVIRONMENT_VARIABLES.md @@ -99,6 +99,7 @@ The detailed Metal A/B contracts and expected oracle counters live in | `DS4_ROCM_DISABLE_Q4_PREFILL_Q8_K_WAVE32=1` | Dominant rollback to the canonical one-workgroup-per-Q8_K-block quantizer. | | `DS4_ROCM_REQUIRE_Q4_PREFILL_Q8_K_WAVE32=1` | Require the wave32 quantizer for strict prefill A/B runs; unsupported scope/device, rollback, or an incompatible required F16/WMMA path fails closed. | | `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA=0/1` | Compatibility control for the resident gfx1151 wave32 direct-Q4 WMMA prefill kernel at 256–4096 tokens. Unset keeps the automatic standalone/attention-output-A path, while the numerically compounded all-Q4 attention-output B stays on Q8_K+TILE8; an enabled value also opts B into direct WMMA for experiments, and an explicit false value opts out of resident WMMA entirely. The kernel uses transient F16 register dequantization and F32 accumulation, with 64 rows below output dimension 1024, 128 rows below 8192, and 256 rows otherwise. | +| `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_K64=0/1` | K64/P80 is the default staging mode for an otherwise eligible direct-Q4 WMMA launch. It stages two adjacent 32-value Q4_K groups in one padded LDS tile, halving workgroup barriers while preserving activation traffic and K32 accumulation order. Unset or true keeps K64/P80; `0`/`false`/`no`/`off` rolls back selectively to K32, while `DS4_ROCM_DISABLE_Q4_PREFILL_WMMA=1` rolls back direct WMMA entirely. | | `DS4_ROCM_Q4_PREFILL_WMMA_ROW_TILE=64|128|256` | Override the direct-Q4 WMMA output-row tile. The default uses 64 rows below output dimension 1024, 128 below 8192, and 256 otherwise; `64` also retains the prior kernel geometry as an A/B control. | | `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_SSD=1` | Explicitly allow direct-Q4 WMMA during SSD streaming, including attention-output B, only when each complete projection weight range is already backed by physical device storage. It never treats mapped/registered host memory as resident. | | `DS4_ROCM_DISABLE_Q4_PREFILL_WMMA=1` | Dominant value-aware opt-out for the automatic direct-Q4 WMMA path, including explicit resident or SSD requests. | @@ -152,7 +153,7 @@ above, it is an unstable internal diagnostic or tuning interface. The linked sou remains normative for exact eligibility gates, bounds, and architecture-specific defaults. -Inventory totals: **1079 `DS4_*` runtime variables** and +Inventory totals: **1080 `DS4_*` runtime variables** and **6 external runtime variables**. The auxiliary inventories contain **118 test/test-fixture entries** and **19 tool/wrapper entries**. @@ -958,7 +959,7 @@ and **19 tool/wrapper entries**.
-ROCm (145) +ROCm (146) | Variable | Accepted value and default | Effect | Source | | --- | --- | --- | --- | @@ -1011,6 +1012,7 @@ and **19 tool/wrapper entries**. | `DS4_ROCM_ENABLE_Q4_DENSE_PAIR` | presence opt-in; unset=off; DISABLE takes precedence | Enable rocm enable q4 dense pair. | [rocm/ds4_rocm_q4.cuh:434](rocm/ds4_rocm_q4.cuh#L434) | | `DS4_ROCM_ENABLE_Q4_GROUPED_ATTN_A` | presence opt-in outside the default scope; the exact caller-marked resident decode shape groups=8, N=1, K=4096, M=1024 is automatic, while row-at-a-time batch fallbacks are not; DISABLE wins | Enable grouped Q4 attention-A for eligible slices, non-production shapes, or explicit experiments in addition to the resident decode default. | [rocm/ds4_rocm_q4.cuh:872](rocm/ds4_rocm_q4.cuh#L872) | | `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA` | value-aware compatibility control; unset keeps automatic resident direct-Q4 WMMA for standalone dense and attention-output A while all-Q4 attention-output B remains on Q8_K+TILE8; empty or any value other than 0/false/no/off also opts B into experimental direct WMMA; explicit 0/false/no/off opts out of resident WMMA; N=256..4096, K a positive multiple of 256, resident non-quality gfx1151 wave32 only; SSD has a separate gate and DISABLE wins | Use compressed Q4_K-to-F16 register dequantization plus shape-selected 64-token by 64/128/256-row WMMA tiles and two-wide activation staging on the wider tiles, without Q8_K activation scratch or an F16 weight sidecar. | [rocm/ds4_rocm_q4.cuh:1089](rocm/ds4_rocm_q4.cuh#L1089) | +| `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_K64` | value-aware compatibility control for the default K64/P80 stage; unset, empty, or any value other than 0/false/no/off selects K64/P80 after the normal direct-Q4 WMMA device, shape, residency, and quality gates pass; 0/false/no/off rolls back only to the established K32 stage; DS4_ROCM_DISABLE_Q4_PREFILL_WMMA wins | Stage two adjacent 32-value Q4_K groups and a 64-value activation slice in one padded P80 LDS tile, halving workgroup barriers while preserving activation traffic and the K32 accumulation order. | [rocm/ds4_rocm_q4.cuh:1537](rocm/ds4_rocm_q4.cuh#L1537) | | `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_SSD` | value-aware SSD-only opt-in, default off; unset/0/false/no/off retains TILE8/TILE4, while empty or any other value requests direct-Q4 WMMA including attention-output B; eligibility additionally requires each complete projection weight range in physical device storage rather than mapped/registered host memory; DISABLE wins | Allow the compressed direct-Q4 WMMA kernel to consume an already device-resident/cache-backed Q4_K projection during SSD streaming without changing model I/O. | [rocm/ds4_rocm_q4.cuh:1091](rocm/ds4_rocm_q4.cuh#L1091) | | `DS4_ROCM_ENABLE_STREAMING_FULL_EXPERT_ADDR_TABLE` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming full expert addr table. | [ds4.c:18255](ds4.c#L18255) | | `DS4_ROCM_ENABLE_STREAMING_MADVISE_WILLNEED` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming madvise willneed. | [ds4.c:18224](ds4.c#L18224) | diff --git a/rocm/ds4_rocm_q4.cuh b/rocm/ds4_rocm_q4.cuh index d32fb8e13..d887540fd 100644 --- a/rocm/ds4_rocm_q4.cuh +++ b/rocm/ds4_rocm_q4.cuh @@ -146,8 +146,15 @@ static_assert((sizeof(cuda_block_q8_K) % sizeof(uint32_t)) == 0u, enum { ROCM_Q4_WMMA_TOKEN_TILE = 64u, ROCM_Q4_WMMA_K_TILE = 32u, + ROCM_Q4_WMMA_K64_TILE = 64u, + ROCM_Q4_WMMA_K64_LDS_PITCH = 80u, ROCM_Q4_WMMA_FRAGMENT = 16u, }; +static_assert(ROCM_Q4_WMMA_K64_TILE == 2u * ROCM_Q4_WMMA_K_TILE, + "K64 must combine exactly two adjacent Q4_K qgroups"); +static_assert(ROCM_Q4_WMMA_K64_LDS_PITCH >= ROCM_Q4_WMMA_K64_TILE && + (ROCM_Q4_WMMA_K64_LDS_PITCH % ROCM_Q4_WMMA_FRAGMENT) == 0u, + "K64 LDS rows must preserve aligned half16 WMMA loads"); #if defined(__HIP_DEVICE_COMPILE__) && __HIP_DEVICE_COMPILE__ && \ defined(__gfx1151__) && \ @@ -359,6 +366,218 @@ __global__ static void rocm_matmul_q4_K_prefill_wmma_rowtile_strided_kernel( (void)out_token_stride; #endif } + +/* Default K64 variant. The K32 kernel above intentionally remains + * byte-for-byte unchanged so its code generation stays a stable rollback and + * A/B baseline. K64 stages two adjacent 32-value activation groups + * at once and consumes the low/high Q4 nibbles in their original order, + * reducing the LDS barrier pairs from sixteen to eight per Q4_K block. + * + * A padded 80-half LDS pitch keeps every 16-half WMMA load 32-byte aligned + * while rotating successive token rows across LDS banks. Natural pitch 64 + * would map every 128-byte row to the same bank pattern on RDNA wave32. */ +template +__launch_bounds__(WAVES * 32u, MIN_BLOCKS) +__global__ static void +rocm_matmul_q4_K_prefill_wmma_k64_p80_rowtile_strided_kernel( + float *out, + const char *w_base, + const float *x, + uint32_t n_tok, + uint32_t n_groups, + uint32_t in_dim, + uint32_t out_dim, + uint64_t row_bytes, + uint64_t x_token_stride, + uint64_t x_group_stride, + uint64_t out_token_stride) { +#if DS4_ROCM_Q4_GFX1151_WMMA_ROWTILE_DEVICE + if (warpSize != 32) return; + + const uint32_t tid = threadIdx.x; + const uint32_t wave = tid >> 5u; + const uint32_t lane = tid & 31u; + const uint32_t lane16 = lane & 15u; + const uint32_t group = blockIdx.z; + static_assert(ROW_TILE == WAVES * ROCM_Q4_WMMA_FRAGMENT, + "one Q4 WMMA wave must own exactly 16 output rows"); + const uint32_t row0 = blockIdx.x * ROW_TILE; + const uint32_t tok0 = blockIdx.y * ROCM_Q4_WMMA_TOKEN_TILE; + if (group >= n_groups) return; + + const uint32_t wave_row0 = row0 + wave * ROCM_Q4_WMMA_FRAGMENT; + const uint32_t my_row = wave_row0 + lane16; + const uint32_t safe_row = my_row < out_dim ? my_row : out_dim - 1u; + const cuda_block_q4_K *row_blocks = + reinterpret_cast( + w_base + ((uint64_t)group * out_dim + safe_row) * row_bytes); + const uint32_t q4_blocks = in_dim / CUDA_QK_K; + + ds4_q4_float8_t acc0 = {0.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 0.0f}; + ds4_q4_float8_t acc1 = acc0; + ds4_q4_float8_t acc2 = acc0; + ds4_q4_float8_t acc3 = acc0; + __shared__ __align__(32) _Float16 + lds_x[ROCM_Q4_WMMA_TOKEN_TILE * ROCM_Q4_WMMA_K64_LDS_PITCH]; + + for (uint32_t block_index = 0u; block_index < q4_blocks; + block_index++) { + const cuda_block_q4_K *block = row_blocks + block_index; + const float block_d = dev_f16_to_f32(block->d); + const float block_dm = dev_f16_to_f32(block->dmin); +#pragma unroll + for (uint32_t qpair = 0u; qpair < 4u; qpair++) { + ds4_q4_uchar16_t packed0; + ds4_q4_uchar16_t packed1; + __builtin_memcpy( + &packed0, block->qs + qpair * 32u, sizeof(packed0)); + __builtin_memcpy( + &packed1, + block->qs + qpair * 32u + ROCM_Q4_WMMA_FRAGMENT, + sizeof(packed1)); + + const uint32_t group32_base = block_index * 8u + qpair * 2u; + if constexpr (LOAD2) { + for (uint32_t j = tid * 2u; + j < ROCM_Q4_WMMA_TOKEN_TILE * ROCM_Q4_WMMA_K64_TILE; + j += blockDim.x * 2u) { + const uint32_t tok_local = j >> 6u; + const uint32_t kk = j & 63u; + const uint32_t tok = tok0 + tok_local; + half2 value = __floats2half2_rn(0.0f, 0.0f); + if (tok < n_tok) { + const float2 pair = + *reinterpret_cast( + x + (uint64_t)tok * x_token_stride + + (uint64_t)group * x_group_stride + + (uint64_t)group32_base * + ROCM_Q4_WMMA_K_TILE + + kk); + value = __floats2half2_rn(pair.x, pair.y); + } + *reinterpret_cast( + lds_x + tok_local * ROCM_Q4_WMMA_K64_LDS_PITCH + + kk) = value; + } + } else { + for (uint32_t j = tid; + j < ROCM_Q4_WMMA_TOKEN_TILE * ROCM_Q4_WMMA_K64_TILE; + j += blockDim.x) { + const uint32_t tok_local = j >> 6u; + const uint32_t kk = j & 63u; + const uint32_t tok = tok0 + tok_local; + float value = 0.0f; + if (tok < n_tok) { + value = x[(uint64_t)tok * x_token_stride + + (uint64_t)group * x_group_stride + + (uint64_t)group32_base * + ROCM_Q4_WMMA_K_TILE + + kk]; + } + lds_x[tok_local * ROCM_Q4_WMMA_K64_LDS_PITCH + kk] = + (_Float16)value; + } + } + __syncthreads(); + + /* Do not unroll the two nibbles: one pair of half16 weight + * vectors must die before the next one is materialized. This + * caps VGPR pressure while preserving qgroup accumulation order. */ +#pragma unroll 1 + for (uint32_t nibble = 0u; nibble < 2u; nibble++) { + const uint32_t qgroup = qpair * 2u + nibble; + uint8_t scale = 0u; + uint8_t minimum = 0u; + dev_q4_K_get_scale_min( + qgroup, block->scales, &scale, &minimum); + const float d = block_d * (float)scale; + const float dm = block_dm * (float)minimum; + const uint32_t shift = nibble * 4u; + ds4_q4_half16_t weights0; + ds4_q4_half16_t weights1; +#pragma unroll + for (uint32_t i = 0u; i < ROCM_Q4_WMMA_FRAGMENT; i++) { + const uint8_t q0 = (packed0[i] >> shift) & 0x0fu; + const uint8_t q1 = (packed1[i] >> shift) & 0x0fu; + weights0[i] = (_Float16)(d * (float)q0 - dm); + weights1[i] = (_Float16)(d * (float)q1 - dm); + } + +#pragma unroll + for (uint32_t token_tile = 0u; token_tile < 4u; + token_tile++) { + const uint32_t token_local = + token_tile * ROCM_Q4_WMMA_FRAGMENT + lane16; + const _Float16 *activation = + lds_x + + token_local * ROCM_Q4_WMMA_K64_LDS_PITCH + + nibble * ROCM_Q4_WMMA_K_TILE; + const ds4_q4_half16_t activation0 = + *reinterpret_cast(activation); + const ds4_q4_half16_t activation1 = + *reinterpret_cast( + activation + ROCM_Q4_WMMA_FRAGMENT); + if (token_tile == 0u) { + acc0 = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32( + weights0, activation0, acc0); + acc0 = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32( + weights1, activation1, acc0); + } else if (token_tile == 1u) { + acc1 = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32( + weights0, activation0, acc1); + acc1 = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32( + weights1, activation1, acc1); + } else if (token_tile == 2u) { + acc2 = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32( + weights0, activation0, acc2); + acc2 = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32( + weights1, activation1, acc2); + } else { + acc3 = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32( + weights0, activation0, acc3); + acc3 = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32( + weights1, activation1, acc3); + } + } + } + __syncthreads(); + } + } + +#pragma unroll + for (uint32_t token_tile = 0u; token_tile < 4u; token_tile++) { + const uint32_t tok = + tok0 + token_tile * ROCM_Q4_WMMA_FRAGMENT + lane16; + if (tok >= n_tok) continue; + const ds4_q4_float8_t acc = token_tile == 0u + ? acc0 + : (token_tile == 1u ? acc1 + : (token_tile == 2u ? acc2 : acc3)); +#pragma unroll + for (uint32_t j = 0u; j < 8u; j++) { + const uint32_t row = wave_row0 + 2u * j + (lane >> 4u); + if (row < out_dim) { + out[(uint64_t)tok * out_token_stride + + (uint64_t)group * out_dim + row] = acc[j]; + } + } + } +#else + (void)out; + (void)w_base; + (void)x; + (void)n_tok; + (void)n_groups; + (void)in_dim; + (void)out_dim; + (void)row_bytes; + (void)x_token_stride; + (void)x_group_stride; + (void)out_token_stride; +#endif +} #undef DS4_ROCM_Q4_GFX1151_WMMA_ROWTILE_DEVICE #endif @@ -999,10 +1218,13 @@ enum { /* Test oracle for strict dispatch: REQUIRE must attest this launch wrapper, * not merely produce output through a canonical fallback. */ static uint64_t g_rocm_q4_prefill_wmma_launches; +static uint64_t g_rocm_q4_prefill_wmma_k64_launches; extern "C" void ds4_rocm_test_q4_prefill_wmma_reset(void) { __atomic_store_n(&g_rocm_q4_prefill_wmma_launches, 0u, __ATOMIC_RELAXED); + __atomic_store_n(&g_rocm_q4_prefill_wmma_k64_launches, 0u, + __ATOMIC_RELAXED); } extern "C" uint64_t ds4_rocm_test_q4_prefill_wmma_get_calls(void) { @@ -1010,6 +1232,20 @@ extern "C" uint64_t ds4_rocm_test_q4_prefill_wmma_get_calls(void) { __ATOMIC_RELAXED); } +extern "C" uint64_t ds4_rocm_test_q4_prefill_wmma_k64_get_calls(void) { + return __atomic_load_n(&g_rocm_q4_prefill_wmma_k64_launches, + __ATOMIC_RELAXED); +} + +static int rocm_q4_K_prefill_wmma_k64_control_policy(int control) { + return control != 0; +} + +extern "C" int ds4_rocm_test_q4_prefill_wmma_k64_control_policy( + int control) { + return rocm_q4_K_prefill_wmma_k64_control_policy(control); +} + /* Resident execution is automatic after validation. Preserve an explicit * ENABLE=0 as a compatibility opt-out, while REQUIRE remains a strict * assertion rather than the switch that turns the candidate on. SSD @@ -1191,6 +1427,60 @@ static void rocm_q4_K_prefill_wmma_enqueue( } } +/* Keep the K64 launch family separate from the established K32 entry points. + * Besides making rollback a single environment change, this prevents compiler + * changes caused by a template K_TILE branch from moving the A/B baseline. */ +static void rocm_q4_K_prefill_wmma_k64_enqueue( + float *out, + const char *w, + const float *x, + uint32_t n_tok, + uint32_t n_groups, + uint32_t in_dim, + uint32_t out_dim, + uint64_t row_bytes, + uint64_t x_token_stride, + uint64_t x_group_stride, + uint64_t out_token_stride, + uint32_t row_tile, + int load2) { + const dim3 grid( + (unsigned)(((uint64_t)out_dim + row_tile - 1u) / row_tile), + (unsigned)(((uint64_t)n_tok + ROCM_Q4_WMMA_TOKEN_TILE - 1u) / + ROCM_Q4_WMMA_TOKEN_TILE), + n_groups); + if (row_tile == 256u) { + if (load2) { + rocm_matmul_q4_K_prefill_wmma_k64_p80_rowtile_strided_kernel< + 256u, 16u, 1u, true><<>>( + out, w, x, n_tok, n_groups, in_dim, out_dim, row_bytes, + x_token_stride, x_group_stride, out_token_stride); + } else { + rocm_matmul_q4_K_prefill_wmma_k64_p80_rowtile_strided_kernel< + 256u, 16u, 1u, false><<>>( + out, w, x, n_tok, n_groups, in_dim, out_dim, row_bytes, + x_token_stride, x_group_stride, out_token_stride); + } + } else if (row_tile == 128u) { + if (load2) { + rocm_matmul_q4_K_prefill_wmma_k64_p80_rowtile_strided_kernel< + 128u, 8u, 1u, true><<>>( + out, w, x, n_tok, n_groups, in_dim, out_dim, row_bytes, + x_token_stride, x_group_stride, out_token_stride); + } else { + rocm_matmul_q4_K_prefill_wmma_k64_p80_rowtile_strided_kernel< + 128u, 8u, 1u, false><<>>( + out, w, x, n_tok, n_groups, in_dim, out_dim, row_bytes, + x_token_stride, x_group_stride, out_token_stride); + } + } else { + rocm_matmul_q4_K_prefill_wmma_k64_p80_rowtile_strided_kernel< + 64u, 4u, 2u, false><<>>( + out, w, x, n_tok, n_groups, in_dim, out_dim, row_bytes, + x_token_stride, x_group_stride, out_token_stride); + } +} + static uint32_t rocm_q4_K_prefill_wmma_row_tile(uint32_t out_dim) { const uint32_t shape_tile = out_dim >= 8192u ? 256u @@ -1240,14 +1530,33 @@ static int rocm_q4_K_prefill_wmma_launch( const int load2 = row_tile != 64u && rocm_q4_K_prefill_wmma_load2_compatible( x, x_token_stride, x_group_stride); - rocm_q4_K_prefill_wmma_enqueue( - out, w, x, n_tok, n_groups, in_dim, out_dim, row_bytes, - x_token_stride, x_group_stride, out_token_stride, row_tile, load2); - const int ok = cuda_ok( - cudaGetLastError(), label ? label : "q4_K prefill WMMA rowtile launch"); + /* K64 is the automatic geometry once the normal direct-Q4 WMMA selector + * has accepted this launch. Preserve a value-aware, selective K32 + * rollback: unset/true selects K64 and 0/false/no/off selects K32. */ + const int k64_control = rocm_q4_attn_q_b_env_bool( + "DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_K64"); + const int use_k64 = + rocm_q4_K_prefill_wmma_k64_control_policy(k64_control); + if (use_k64) { + rocm_q4_K_prefill_wmma_k64_enqueue( + out, w, x, n_tok, n_groups, in_dim, out_dim, row_bytes, + x_token_stride, x_group_stride, out_token_stride, row_tile, load2); + } else { + rocm_q4_K_prefill_wmma_enqueue( + out, w, x, n_tok, n_groups, in_dim, out_dim, row_bytes, + x_token_stride, x_group_stride, out_token_stride, row_tile, load2); + } + const char *launch_label = use_k64 + ? "q4_K prefill WMMA K64/P80 rowtile launch" + : (label ? label : "q4_K prefill WMMA rowtile launch"); + const int ok = cuda_ok(cudaGetLastError(), launch_label); if (ok) { __atomic_fetch_add(&g_rocm_q4_prefill_wmma_launches, 1u, __ATOMIC_RELAXED); + if (use_k64) { + __atomic_fetch_add(&g_rocm_q4_prefill_wmma_k64_launches, 1u, + __ATOMIC_RELAXED); + } } return ok; } @@ -1307,6 +1616,62 @@ extern "C" int ds4_rocm_bench_q4_K_wmma_enqueue( return 1; } +/* Variant hook for a resident, same-process K32/K64 comparison. The legacy + * benchmark hook above remains a strict K32 wrapper so existing binaries and + * source call sites retain their meaning. */ +extern "C" int ds4_rocm_bench_q4_K_wmma_variant_enqueue( + void *out, + const void *w, + const void *x, + uint32_t n_tok, + uint32_t n_groups, + uint32_t in_dim, + uint32_t out_dim, + uint64_t row_bytes, + uint64_t x_token_stride, + uint64_t x_group_stride, + uint64_t out_token_stride, + uint32_t row_tile, + uint32_t k_tile, + int load2) { + if (k_tile != ROCM_Q4_WMMA_K_TILE && + k_tile != ROCM_Q4_WMMA_K64_TILE) { + return 0; + } + if (!out || !w || !x || n_tok == 0u || n_groups == 0u || + in_dim == 0u || out_dim == 0u || + (in_dim % CUDA_QK_K) != 0u || + (load2 != 0 && load2 != 1) || + (load2 && row_tile == 64u) || + (row_tile != 64u && row_tile != 128u && row_tile != 256u)) { + return 0; + } + const uint64_t minimum_row_bytes = + ((uint64_t)in_dim / CUDA_QK_K) * sizeof(cuda_block_q4_K); + if (row_bytes < minimum_row_bytes || + (load2 && !rocm_q4_K_prefill_wmma_load2_compatible( + reinterpret_cast(x), + x_token_stride, x_group_stride))) { + return 0; + } + if (k_tile == ROCM_Q4_WMMA_K64_TILE) { + rocm_q4_K_prefill_wmma_k64_enqueue( + reinterpret_cast(out), + reinterpret_cast(w), + reinterpret_cast(x), + n_tok, n_groups, in_dim, out_dim, row_bytes, + x_token_stride, x_group_stride, out_token_stride, row_tile, load2); + } else { + rocm_q4_K_prefill_wmma_enqueue( + reinterpret_cast(out), + reinterpret_cast(w), + reinterpret_cast(x), + n_tok, n_groups, in_dim, out_dim, row_bytes, + x_token_stride, x_group_stride, out_token_stride, row_tile, load2); + } + return 1; +} + enum { ROCM_Q4_PREFILL_K1024_TILE4_REQUIRED_FAILURE = -1, ROCM_Q4_PREFILL_K1024_TILE4_FALLBACK = 0, diff --git a/scripts/environment_variables.tsv b/scripts/environment_variables.tsv index 6c6af683b..2702a8bfa 100644 --- a/scripts/environment_variables.tsv +++ b/scripts/environment_variables.tsv @@ -1025,6 +1025,7 @@ runtime/rocm DS4_ROCM_ENABLE_Q4_GROUPED_ATTN_A presence opt-in outside the defau runtime/rocm DS4_ROCM_ENABLE_Q4_PREFILL_K1024_TILE4_SSD value-aware SSD-only opt-in, default off; unset/0/false/no/off retains TILE8, while empty or any other value requests TILE4; eligibility additionally requires N=9..4096, K=1024, M=32768, TILE8 enabled, and the complete weight range in device storage rather than mapped/registered host memory; DISABLE wins Allow the four-lane K=1024 Q4_K prefill specialization to consume an already device-resident/cache-backed attn_q_b weight range during SSD streaming without changing model I/O. rocm/ds4_rocm_q4.cuh:624 runtime/rocm DS4_ROCM_ENABLE_Q4_PREFILL_Q8_K_WAVE32 value-aware opt-in, default off; unset/0/false/no/off retains the canonical quantizer, while empty or any other value requests the candidate for N=9..4096 on gfx1151 wave32; DISABLE wins; a selected exact Q8 path takes precedence over automatic direct-Q4 WMMA, REQUIRE_WMMA overrides an optional request, and dual REQUIRE fails closed Quantize eight independent Q8_K activation blocks per 256-thread workgroup using one wave32 per block, without LDS or workgroup barriers, before the exact Q4 prefill matmul (TILE8 or its legacy rollback). rocm/ds4_rocm_q4.cuh:858 runtime/rocm DS4_ROCM_ENABLE_Q4_PREFILL_WMMA value-aware compatibility control; unset keeps automatic resident direct-Q4 WMMA for standalone dense and attention-output A while all-Q4 attention-output B remains on Q8_K+TILE8; empty or any value other than 0/false/no/off also opts B into experimental direct WMMA; explicit 0/false/no/off opts out of resident WMMA; N=256..4096, K a positive multiple of 256, resident non-quality gfx1151 wave32 only; SSD has a separate gate and DISABLE wins Use compressed Q4_K-to-F16 register dequantization plus shape-selected 64-token by 64/128/256-row WMMA tiles and two-wide activation staging on the wider tiles, without Q8_K activation scratch or an F16 weight sidecar. rocm/ds4_rocm_q4.cuh:1089 +runtime/rocm DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_K64 value-aware compatibility control for the default K64/P80 stage; unset, empty, or any value other than 0/false/no/off selects K64/P80 after the normal direct-Q4 WMMA device, shape, residency, and quality gates pass; 0/false/no/off rolls back only to the established K32 stage; DS4_ROCM_DISABLE_Q4_PREFILL_WMMA wins Stage two adjacent 32-value Q4_K groups and a 64-value activation slice in one padded P80 LDS tile, halving workgroup barriers while preserving activation traffic and the K32 accumulation order. rocm/ds4_rocm_q4.cuh:1537 runtime/rocm DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_SSD value-aware SSD-only opt-in, default off; unset/0/false/no/off retains TILE8/TILE4, while empty or any other value requests direct-Q4 WMMA including attention-output B; eligibility additionally requires each complete projection weight range in physical device storage rather than mapped/registered host memory; DISABLE wins Allow the compressed direct-Q4 WMMA kernel to consume an already device-resident/cache-backed Q4_K projection during SSD streaming without changing model I/O. rocm/ds4_rocm_q4.cuh:1091 runtime/rocm DS4_ROCM_ENABLE_STREAMING_FULL_EXPERT_ADDR_TABLE presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming full expert addr table. ds4.c:18255 runtime/rocm DS4_ROCM_ENABLE_STREAMING_MADVISE_WILLNEED presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming madvise willneed. ds4.c:18224 diff --git a/speed-bench/README.md b/speed-bench/README.md index 7108dca9d..3992a9011 100644 --- a/speed-bench/README.md +++ b/speed-bench/README.md @@ -213,6 +213,24 @@ its own configuration. Eligible resident runtime calls use direct-Q4 WMMA by default; set `DS4_ROCM_DISABLE_Q4_PREFILL_WMMA=1` to opt out. The production-API comparison still uses the strict REQUIRE gate so a rejected dispatch fails instead of timing a fallback. + +Eligible direct-Q4 WMMA launches use K64/P80 staging by default. It stages two +adjacent 32-value Q4_K groups and a 64-value activation slice in one padded +P80 LDS tile, halving workgroup barriers while retaining the K32 activation +traffic and accumulation order. Leave +`DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_K64` unset or set it to a true value for this +default; set it to `0`, `false`, `no`, or `off` to restore K32 staging. +`DS4_ROCM_DISABLE_Q4_PREFILL_WMMA=1` rolls the whole direct-WMMA path back to +Q8_K plus TILE8/TILE4. + +Use the production 2048-token sweep to isolate default K64/P80 from the K32 +rollback without SSD-streaming or policy-selection noise: + +``` +./speed-bench/rocm_q4_prefill_bench \ + --case all --tokens 2048 --sets 4 --warmup 4 --samples 12 +``` + The benchmark always keeps SSD streaming disabled. Runtime SSD experiments need the separate `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_SSD=1` gate and only accept projection ranges already backed by physical device memory, so model I/O is diff --git a/speed-bench/rocm_q4_prefill_bench.cpp b/speed-bench/rocm_q4_prefill_bench.cpp index 59c3bfcc0..48ecdc0e9 100644 --- a/speed-bench/rocm_q4_prefill_bench.cpp +++ b/speed-bench/rocm_q4_prefill_bench.cpp @@ -33,6 +33,12 @@ extern "C" int ds4_rocm_bench_q4_K_wmma_enqueue( uint64_t row_bytes, uint64_t x_token_stride, uint64_t x_group_stride, uint64_t out_token_stride, uint32_t row_tile, int load2); +extern "C" int ds4_rocm_bench_q4_K_wmma_variant_enqueue( + void *out, const void *w, const void *x, uint32_t n_tok, + uint32_t n_groups, uint32_t in_dim, uint32_t out_dim, + uint64_t row_bytes, uint64_t x_token_stride, + uint64_t x_group_stride, uint64_t out_token_stride, + uint32_t row_tile, uint32_t k_tile, int load2); namespace { @@ -78,6 +84,8 @@ constexpr const char *kWmmaRequire = "DS4_ROCM_REQUIRE_Q4_PREFILL_WMMA"; constexpr const char *kWmmaRowTile = "DS4_ROCM_Q4_PREFILL_WMMA_ROW_TILE"; +constexpr const char *kWmmaK64 = + "DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_K64"; constexpr const char *kQ8Wave32Enable = "DS4_ROCM_ENABLE_Q4_PREFILL_Q8_K_WAVE32"; constexpr const char *kQ8Wave32Disable = @@ -526,6 +534,7 @@ void select_legacy() { (void)setenv(kWmmaDisable, "1", 1); (void)unsetenv(kWmmaRequire); (void)unsetenv(kWmmaRowTile); + (void)unsetenv(kWmmaK64); (void)unsetenv(kQ8Wave32Enable); (void)unsetenv(kQ8Wave32Disable); (void)unsetenv(kQ8Wave32Require); @@ -542,6 +551,7 @@ void select_tile8(bool disable_k1024_tile4) { (void)setenv(kWmmaDisable, "1", 1); (void)unsetenv(kWmmaRequire); (void)unsetenv(kWmmaRowTile); + (void)unsetenv(kWmmaK64); (void)unsetenv(kQ8Wave32Enable); (void)unsetenv(kQ8Wave32Disable); (void)unsetenv(kQ8Wave32Require); @@ -568,6 +578,7 @@ void select_wmma_shape() { (void)setenv(kWmmaRequire, "1", 1); (void)unsetenv(kK1024Tile4Require); (void)unsetenv(kWmmaRowTile); + (void)unsetenv(kWmmaK64); } double percentile(std::vector sorted, double fraction) { @@ -956,7 +967,7 @@ bool run_dense(const model_fixture &model, const config &cfg, kDenseK, 0u, kDenseM, shape_wmma_row_tile(kDenseM), 0) != 0; }}; - return benchmark_arms( + if (!benchmark_arms( "dense_wmma_rows", n_tokens, kDenseK, kDenseM, cfg, geometry_baseline, geometry_candidate, [&]() { @@ -970,6 +981,39 @@ bool run_dense(const model_fixture &model, const config &cfg, "dense WMMA rows64 oracle") && check_guard(tiled.ptr, logical_bytes, "dense WMMA shape oracle"); + })) return false; + + const uint32_t fixed_row_tile = shape_wmma_row_tile(kDenseM); + const arm k32_baseline = { + "wmma_k32_rows128_load2", []() {}, + [&](uint32_t set) { + return ds4_rocm_bench_q4_K_wmma_variant_enqueue( + rows64_device, dense_weights[set], x_device, + n_tokens, 1u, kDenseK, kDenseM, row_bytes, + kDenseK, 0u, kDenseM, fixed_row_tile, 32u, 1) != 0; + }}; + const arm k64_candidate = { + "wmma_k64p80_rows128_load2", []() {}, + [&](uint32_t set) { + return ds4_rocm_bench_q4_K_wmma_variant_enqueue( + shape_device, dense_weights[set], x_device, + n_tokens, 1u, kDenseK, kDenseM, row_bytes, + kDenseK, 0u, kDenseM, fixed_row_tile, 64u, 1) != 0; + }}; + return benchmark_arms( + "dense_wmma_k32_k64", n_tokens, kDenseK, kDenseM, cfg, + k32_baseline, k64_candidate, + [&]() { + return poison_output(legacy.ptr, logical_bytes, 0x7fc10001u) && + poison_output(tiled.ptr, logical_bytes, 0x7fc20002u); + }, + [&]() { + return bitwise_equal(legacy.ptr, tiled.ptr, logical_bytes, + "dense WMMA K32 vs K64/P80") && + check_guard(legacy.ptr, logical_bytes, + "dense WMMA K32 oracle") && + check_guard(tiled.ptr, logical_bytes, + "dense WMMA K64/P80 oracle"); }); } @@ -1261,7 +1305,7 @@ bool run_qb(const model_fixture &model, const config &cfg, n_tokens, 1u, kQbK, kQbM, row_bytes, kQbK, 0u, kQbM, 256u, 1) != 0; }}; - return benchmark_arms( + if (!benchmark_arms( "q_b_wmma_load2_256", n_tokens, kQbK, kQbM, cfg, rows256_scalar, rows256_load2, [&]() { @@ -1275,6 +1319,38 @@ bool run_qb(const model_fixture &model, const config &cfg, "q_b WMMA rows256 scalar oracle") && check_guard(tile4.ptr, logical_bytes, "q_b WMMA rows256 load2 oracle"); + })) return false; + + const arm k32_baseline = { + "wmma_k32_rows256_load2", []() {}, + [&](uint32_t set) { + return ds4_rocm_bench_q4_K_wmma_variant_enqueue( + rows64_device, qb_weights[set], x_device, + n_tokens, 1u, kQbK, kQbM, row_bytes, + kQbK, 0u, kQbM, 256u, 32u, 1) != 0; + }}; + const arm k64_candidate = { + "wmma_k64p80_rows256_load2", []() {}, + [&](uint32_t set) { + return ds4_rocm_bench_q4_K_wmma_variant_enqueue( + shape_device, qb_weights[set], x_device, + n_tokens, 1u, kQbK, kQbM, row_bytes, + kQbK, 0u, kQbM, 256u, 64u, 1) != 0; + }}; + return benchmark_arms( + "q_b_wmma_k32_k64", n_tokens, kQbK, kQbM, cfg, + k32_baseline, k64_candidate, + [&]() { + return poison_output(tile8.ptr, logical_bytes, 0x7fc10001u) && + poison_output(tile4.ptr, logical_bytes, 0x7fc20002u); + }, + [&]() { + return bitwise_equal(tile8.ptr, tile4.ptr, logical_bytes, + "q_b WMMA K32 vs K64/P80") && + check_guard(tile8.ptr, logical_bytes, + "q_b WMMA K32 oracle") && + check_guard(tile4.ptr, logical_bytes, + "q_b WMMA K64/P80 oracle"); }); } @@ -1359,7 +1435,7 @@ bool run_output_b(const model_fixture &model, const config &cfg, kOutputLowDim, 0u, kOutputM, shape_wmma_row_tile(kOutputM), 0) != 0; }}; - return benchmark_arms( + if (!benchmark_arms( "output_b_wmma_rows", n_tokens, kOutputLowDim, kOutputM, cfg, geometry_baseline, geometry_candidate, [&]() { @@ -1373,6 +1449,41 @@ bool run_output_b(const model_fixture &model, const config &cfg, "output_b WMMA rows64 oracle") && check_guard(wmma.ptr, logical_bytes, "output_b WMMA shape oracle"); + })) return false; + + const uint32_t fixed_row_tile = shape_wmma_row_tile(kOutputM); + const arm k32_baseline = { + "wmma_k32_rows128_load2", []() {}, + [&](uint32_t set) { + return ds4_rocm_bench_q4_K_wmma_variant_enqueue( + rows64_device, output_b_weights[set], x_device, + n_tokens, 1u, kOutputLowDim, kOutputM, row_bytes, + kOutputLowDim, 0u, kOutputM, fixed_row_tile, + 32u, 1) != 0; + }}; + const arm k64_candidate = { + "wmma_k64p80_rows128_load2", []() {}, + [&](uint32_t set) { + return ds4_rocm_bench_q4_K_wmma_variant_enqueue( + shape_device, output_b_weights[set], x_device, + n_tokens, 1u, kOutputLowDim, kOutputM, row_bytes, + kOutputLowDim, 0u, kOutputM, fixed_row_tile, + 64u, 1) != 0; + }}; + return benchmark_arms( + "output_b_wmma_k32_k64", n_tokens, kOutputLowDim, kOutputM, cfg, + k32_baseline, k64_candidate, + [&]() { + return poison_output(tile8.ptr, logical_bytes, 0x7fc10001u) && + poison_output(wmma.ptr, logical_bytes, 0x7fc20002u); + }, + [&]() { + return bitwise_equal(tile8.ptr, wmma.ptr, logical_bytes, + "output_b WMMA K32 vs K64/P80") && + check_guard(tile8.ptr, logical_bytes, + "output_b WMMA K32 oracle") && + check_guard(wmma.ptr, logical_bytes, + "output_b WMMA K64/P80 oracle"); }); } @@ -1508,7 +1619,7 @@ bool run_attention_output(const model_fixture &model, const config &cfg, kOutputM, row_b_bytes, kOutputLowDim, 0u, kOutputM, shape_wmma_row_tile(kOutputM), 0) != 0; }}; - return benchmark_arms( + if (!benchmark_arms( "attention_output_ab_wmma_rows", n_tokens, kOutputLowDim, kDenseK + kOutputM, cfg, geometry_baseline, geometry_candidate, poison_all, @@ -1525,6 +1636,54 @@ bool run_attention_output(const model_fixture &model, const config &cfg, "output_b WMMA rows64 oracle") && check_guard(wmma_out.ptr, out_bytes, "output_b WMMA shape oracle"); + })) return false; + + const uint32_t fixed_row_a_tile = shape_wmma_row_tile(kOutputRank); + const uint32_t fixed_row_b_tile = shape_wmma_row_tile(kOutputM); + const arm k32_baseline = { + "wmma_k32_ab_rows128_load2", []() {}, + [&](uint32_t set) { + return ds4_rocm_bench_q4_K_wmma_variant_enqueue( + rows64_low_device, output_a_weights[set], heads_device, + n_tokens, kOutputGroups, kDenseK, kOutputRank, + row_a_bytes, kOutputGroups * kDenseK, kDenseK, + kOutputLowDim, fixed_row_a_tile, 32u, 1) != 0 && + ds4_rocm_bench_q4_K_wmma_variant_enqueue( + rows64_out_device, output_b_weights[set], + rows64_low_device, n_tokens, 1u, kOutputLowDim, + kOutputM, row_b_bytes, kOutputLowDim, 0u, kOutputM, + fixed_row_b_tile, 32u, 1) != 0; + }}; + const arm k64_candidate = { + "wmma_k64p80_ab_rows128_load2", []() {}, + [&](uint32_t set) { + return ds4_rocm_bench_q4_K_wmma_variant_enqueue( + shape_low_device, output_a_weights[set], heads_device, + n_tokens, kOutputGroups, kDenseK, kOutputRank, + row_a_bytes, kOutputGroups * kDenseK, kDenseK, + kOutputLowDim, fixed_row_a_tile, 64u, 1) != 0 && + ds4_rocm_bench_q4_K_wmma_variant_enqueue( + shape_out_device, output_b_weights[set], + shape_low_device, n_tokens, 1u, kOutputLowDim, + kOutputM, row_b_bytes, kOutputLowDim, 0u, kOutputM, + fixed_row_b_tile, 64u, 1) != 0; + }}; + return benchmark_arms( + "attention_output_ab_wmma_k32_k64", n_tokens, kOutputLowDim, + kDenseK + kOutputM, cfg, k32_baseline, k64_candidate, poison_all, + [&]() { + return bitwise_equal(tile8_low.ptr, wmma_low.ptr, low_bytes, + "output_a WMMA K32 vs K64/P80") && + bitwise_equal(tile8_out.ptr, wmma_out.ptr, out_bytes, + "output_a+b WMMA K32 vs K64/P80") && + check_guard(tile8_low.ptr, low_bytes, + "output_a WMMA K32 oracle") && + check_guard(wmma_low.ptr, low_bytes, + "output_a WMMA K64/P80 oracle") && + check_guard(tile8_out.ptr, out_bytes, + "output_b WMMA K32 oracle") && + check_guard(wmma_out.ptr, out_bytes, + "output_b WMMA K64/P80 oracle"); }); } @@ -1549,7 +1708,9 @@ void usage(FILE *stream, const char *argv0) { "K=1024,M=32768 shape. outb isolates K=8192,M=4096; output measures\n" "the production grouped output_a plus output_b API. WMMA comparisons\n" "use a finite/toleranced oracle because their F16 boundary is not\n" - "bit-identical to the Q8_K activation path.\n", + "bit-identical to the Q8_K activation path. At N>=256 the direct\n" + "arms also compare rollback K32 with default K64/P80 at fixed\n" + "production geometry and load2, requiring bitwise-identical output.\n", argv0, kDefaultSets, kDefaultSamples, kDefaultWarmup); } @@ -1664,6 +1825,7 @@ int main(int argc, char **argv) { env_snapshot wmma_disable_guard(kWmmaDisable); env_snapshot wmma_require_guard(kWmmaRequire); env_snapshot wmma_row_tile_guard(kWmmaRowTile); + env_snapshot wmma_k64_guard(kWmmaK64); env_snapshot q8_wave32_enable_guard(kQ8Wave32Enable); env_snapshot q8_wave32_disable_guard(kQ8Wave32Disable); env_snapshot q8_wave32_require_guard(kQ8Wave32Require); @@ -1732,11 +1894,14 @@ int main(int argc, char **argv) { std::printf( "DS4_ROCM_Q4_PREFILL_SETUP device=%s arch=%s warp=%d sets=%u " "resident_mib=%.2f timing=hip_events ssd_streaming=off " - "wmma_rowtiles=%s wmma_loaders=%s q8_wave32=%s\n", + "wmma_rowtiles=%s wmma_loaders=%s wmma_k_stages=%s " + "wmma_k_default=%s q8_wave32=%s\n", properties.name, properties.gcnArchName, properties.warpSize, cfg.sets, static_cast(model.resident_bytes) / 1048576.0, cfg.wmma_supported ? "64,128,256" : "skipped", cfg.wmma_supported ? "scalar,load2" : "skipped", + cfg.wmma_supported ? "32,64p80" : "skipped", + cfg.wmma_supported ? "64p80" : "skipped", cfg.wmma_supported ? "available" : "skipped"); std::fflush(stdout); for (uint32_t n_tokens : cfg.tokens) { diff --git a/tests/test_rocm_q4_dense_pair.cpp b/tests/test_rocm_q4_dense_pair.cpp index 11301d811..78b4e50ce 100644 --- a/tests/test_rocm_q4_dense_pair.cpp +++ b/tests/test_rocm_q4_dense_pair.cpp @@ -36,6 +36,9 @@ extern "C" int ds4_rocm_test_q4_prefill_k1024_tile4_policy( int disabled, int required); extern "C" void ds4_rocm_test_q4_prefill_wmma_reset(void); extern "C" uint64_t ds4_rocm_test_q4_prefill_wmma_get_calls(void); +extern "C" uint64_t ds4_rocm_test_q4_prefill_wmma_k64_get_calls(void); +extern "C" int ds4_rocm_test_q4_prefill_wmma_k64_control_policy( + int control); extern "C" int ds4_rocm_test_q4_prefill_wmma_requested_policy( int ssd_streaming, int enabled, int ssd_enabled, int disabled, int required); @@ -111,6 +114,8 @@ constexpr const char *kPrefillWmmaRequire = "DS4_ROCM_REQUIRE_Q4_PREFILL_WMMA"; constexpr const char *kPrefillWmmaRowTile = "DS4_ROCM_Q4_PREFILL_WMMA_ROW_TILE"; +constexpr const char *kPrefillWmmaK64 = + "DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_K64"; constexpr const char *kPrefillQ8Wave32Enable = "DS4_ROCM_ENABLE_Q4_PREFILL_Q8_K_WAVE32"; constexpr const char *kPrefillQ8Wave32Disable = @@ -1295,11 +1300,25 @@ bool run_prefill_wmma_requested_policy_oracle() { strict_wmma_yield); ok = false; } + const int k64_unset = + ds4_rocm_test_q4_prefill_wmma_k64_control_policy(-1); + const int k64_false = + ds4_rocm_test_q4_prefill_wmma_k64_control_policy(0); + const int k64_true = + ds4_rocm_test_q4_prefill_wmma_k64_control_policy(1); + if (k64_unset != 1 || k64_false != 0 || k64_true != 1) { + std::fprintf(stderr, + "Q4 WMMA K64 default policy: unset=%d false=%d " + "true=%d FAIL\n", + k64_unset, k64_false, k64_true); + ok = false; + } std::fprintf(stderr, "ROCm Q4 WMMA request policy oracle: cases=%zu " - "q8_yield=%d/%d/%d %s\n", + "q8_yield=%d/%d/%d k64=%d/%d/%d %s\n", sizeof(cases) / sizeof(cases[0]), optional_q8_yield, absent_q8_yield, strict_wmma_yield, + k64_unset, k64_false, k64_true, ok ? "PASS" : "FAIL"); return ok; } @@ -1458,6 +1477,7 @@ bool run_pair_pre_enqueue_policy_oracle() { env_snapshot wmma_disable(kPrefillWmmaDisable); env_snapshot wmma_require(kPrefillWmmaRequire); env_snapshot wmma_row_tile(kPrefillWmmaRowTile); + env_snapshot wmma_k64(kPrefillWmmaK64); env_snapshot q8_enable(kPrefillQ8Wave32Enable); env_snapshot q8_disable(kPrefillQ8Wave32Disable); env_snapshot q8_require(kPrefillQ8Wave32Require); @@ -1469,6 +1489,7 @@ bool run_pair_pre_enqueue_policy_oracle() { (void)unsetenv(kPrefillWmmaDisable); (void)unsetenv(kPrefillWmmaRequire); (void)unsetenv(kPrefillWmmaRowTile); + (void)setenv(kPrefillWmmaK64, "0", 1); (void)unsetenv(kPrefillQ8Wave32Enable); (void)unsetenv(kPrefillQ8Wave32Disable); (void)unsetenv(kPrefillQ8Wave32Require); @@ -2754,10 +2775,12 @@ bool run_prefill_wmma_smoke(const aligned_model &model) { constexpr uint32_t n_tokens = 257u; const size_t logical_count = (size_t)n_tokens * kM0; - /* A broken N-tail store could write the remaining 63 rows of the final - * 64-token tile. Cover that full footprint, not just the normal API - * canary. */ - constexpr size_t wmma_guard_floats = (64u - 1u) * kM0; + /* A broken N-tail store could write the remaining 63 tokens of the final + * tile, while a simultaneous M-tail failure could address the remaining + * 63 rows of this 64-row launch. Cover both predicates failing together, + * not just the normal API canary. */ + constexpr size_t wmma_guard_floats = + (64u - 1u) * kM0 + (64u - 1u); const size_t allocation_count = logical_count + wmma_guard_floats; const std::vector sentinel = sentinel_values(allocation_count); std::vector x; @@ -2765,9 +2788,11 @@ bool run_prefill_wmma_smoke(const aligned_model &model) { tensor_owner x_gpu(x.size() * sizeof(float)); tensor_owner tile8_gpu(allocation_count * sizeof(float)); tensor_owner wmma_gpu(allocation_count * sizeof(float)); - if (!x_gpu.ptr || !tile8_gpu.ptr || !wmma_gpu.ptr || + tensor_owner k64_gpu(allocation_count * sizeof(float)); + if (!x_gpu.ptr || !tile8_gpu.ptr || !wmma_gpu.ptr || !k64_gpu.ptr || !write_tensor(x_gpu.ptr, x) || !write_tensor(tile8_gpu.ptr, sentinel) || - !write_tensor(wmma_gpu.ptr, sentinel)) { + !write_tensor(wmma_gpu.ptr, sentinel) || + !write_tensor(k64_gpu.ptr, sentinel)) { std::fprintf(stderr, "ROCm Q4 direct-WMMA prefill: setup FAIL\n"); return false; } @@ -2781,6 +2806,7 @@ bool run_prefill_wmma_smoke(const aligned_model &model) { env_snapshot wmma_disable(kPrefillWmmaDisable); env_snapshot wmma_require(kPrefillWmmaRequire); env_snapshot wmma_row_tile(kPrefillWmmaRowTile); + env_snapshot wmma_k64(kPrefillWmmaK64); env_snapshot q8_wave32_enable(kPrefillQ8Wave32Enable); env_snapshot q8_wave32_disable(kPrefillQ8Wave32Disable); env_snapshot q8_wave32_require(kPrefillQ8Wave32Require); @@ -2794,6 +2820,7 @@ bool run_prefill_wmma_smoke(const aligned_model &model) { (void)setenv(kPrefillWmmaDisable, "1", 1); (void)unsetenv(kPrefillWmmaRequire); (void)unsetenv(kPrefillWmmaRowTile); + (void)setenv(kPrefillWmmaK64, "0", 1); (void)unsetenv(kPrefillQ8Wave32Enable); (void)unsetenv(kPrefillQ8Wave32Disable); (void)unsetenv(kPrefillQ8Wave32Require); @@ -2803,39 +2830,70 @@ bool run_prefill_wmma_smoke(const aligned_model &model) { kK, kM0, x_gpu.ptr, n_tokens); const uint64_t tile8_wmma_calls = ds4_rocm_test_q4_prefill_wmma_get_calls(); + const uint64_t tile8_k64_calls = + ds4_rocm_test_q4_prefill_wmma_k64_get_calls(); (void)unsetenv(kPrefillRequire); (void)unsetenv(kPrefillWmmaEnable); (void)unsetenv(kPrefillWmmaDisable); (void)unsetenv(kPrefillWmmaRequire); + (void)setenv(kPrefillWmmaK64, "0", 1); ds4_rocm_test_q4_prefill_wmma_reset(); const int wmma_rc = ds4_gpu_matmul_quant_tensor( wmma_gpu.ptr, model.data, model.size, model.weight0_offset, kQ4Type, kK, kM0, x_gpu.ptr, n_tokens); const uint64_t wmma_calls = ds4_rocm_test_q4_prefill_wmma_get_calls(); + const uint64_t wmma_k64_calls = + ds4_rocm_test_q4_prefill_wmma_k64_get_calls(); + + /* The production default must select K64 when no override is present. */ + (void)unsetenv(kPrefillWmmaK64); + ds4_rocm_test_q4_prefill_wmma_reset(); + const int k64_default_rc = ds4_gpu_matmul_quant_tensor( + k64_gpu.ptr, model.data, model.size, model.weight0_offset, kQ4Type, + kK, kM0, x_gpu.ptr, n_tokens); + const uint64_t k64_default_wmma_calls = + ds4_rocm_test_q4_prefill_wmma_get_calls(); + const uint64_t k64_default_calls = + ds4_rocm_test_q4_prefill_wmma_k64_get_calls(); std::vector tile8(allocation_count); std::vector wmma(allocation_count); - bool ok = tile8_rc != 0 && wmma_rc != 0 && tile8_wmma_calls == 0u && - wmma_calls == 1u && + std::vector k64_default(allocation_count); + bool ok = tile8_rc != 0 && wmma_rc != 0 && k64_default_rc != 0 && + tile8_wmma_calls == 0u && tile8_k64_calls == 0u && + wmma_calls == 1u && wmma_k64_calls == 0u && + k64_default_wmma_calls == 1u && k64_default_calls == 1u && read_tensor(tile8_gpu.ptr, &tile8) && - read_tensor(wmma_gpu.ptr, &wmma); + read_tensor(wmma_gpu.ptr, &wmma) && + read_tensor(k64_gpu.ptr, &k64_default); if (ok) { ok = output_body_overwritten(tile8, sentinel, logical_count, "direct-WMMA TILE8 output body") && ok; ok = output_body_overwritten(wmma, sentinel, logical_count, "direct-WMMA candidate output body") && ok; + ok = output_body_overwritten( + k64_default, sentinel, logical_count, + "direct-WMMA default K64 output body") && ok; ok = output_guard_unchanged(tile8, sentinel, logical_count, "direct-WMMA TILE8 output canary") && ok; ok = output_guard_unchanged(wmma, sentinel, logical_count, "direct-WMMA candidate output canary") && ok; + ok = output_guard_unchanged( + k64_default, sentinel, logical_count, + "direct-WMMA default K64 output canary") && ok; tile8.resize(logical_count); wmma.resize(logical_count); + k64_default.resize(logical_count); ok = close_with_tolerance(wmma, tile8, 2.0f, 3.0e-2f, "direct-WMMA vs TILE8 N/M tail") && ok; + ok = bitwise_equal(k64_default, wmma, + "direct-WMMA default K64 vs K32 N/M tail") && ok; } + /* Return to the neutral K32 setting for the remaining policy cases. */ + (void)setenv(kPrefillWmmaK64, "0", 1); (void)setenv(kPrefillWmmaDisable, "1", 1); (void)unsetenv(kPrefillWmmaRequire); if (!write_tensor(wmma_gpu.ptr, sentinel)) return false; @@ -2845,8 +2903,11 @@ bool run_prefill_wmma_smoke(const aligned_model &model) { kK, kM0, x_gpu.ptr, n_tokens); const uint64_t opt_out_wmma_calls = ds4_rocm_test_q4_prefill_wmma_get_calls(); + const uint64_t opt_out_k64_calls = + ds4_rocm_test_q4_prefill_wmma_k64_get_calls(); std::vector opt_out(allocation_count); const bool opt_out_read = opt_out_rc != 0 && opt_out_wmma_calls == 0u && + opt_out_k64_calls == 0u && read_tensor(wmma_gpu.ptr, &opt_out); ok = opt_out_read && ok; if (opt_out_read) { @@ -2866,17 +2927,28 @@ bool run_prefill_wmma_smoke(const aligned_model &model) { kK, kM0, x_gpu.ptr, n_tokens); const uint64_t rejected_wmma_calls = ds4_rocm_test_q4_prefill_wmma_get_calls(); + const uint64_t rejected_k64_calls = + ds4_rocm_test_q4_prefill_wmma_k64_get_calls(); ok = rejected_rc == 0 && rejected_wmma_calls == 0u && + rejected_k64_calls == 0u && unchanged_after_rejected_call( wmma_gpu.ptr, sentinel, "direct-WMMA DISABLE+REQUIRE preserves output") && ok; std::fprintf(stderr, - "ROCm Q4 direct-WMMA prefill: tile8=%d/%llu " - "default=%d/%llu opt_out=%d/%llu rejected=%d/%llu %s\n", + "ROCm Q4 direct-WMMA prefill: tile8=%d/%llu/%llu " + "K32=%d/%llu/%llu K64-default=%d/%llu/%llu " + "opt_out=%d/%llu/%llu rejected=%d/%llu/%llu %s\n", tile8_rc, (unsigned long long)tile8_wmma_calls, + (unsigned long long)tile8_k64_calls, wmma_rc, (unsigned long long)wmma_calls, + (unsigned long long)wmma_k64_calls, + k64_default_rc, + (unsigned long long)k64_default_wmma_calls, + (unsigned long long)k64_default_calls, opt_out_rc, (unsigned long long)opt_out_wmma_calls, + (unsigned long long)opt_out_k64_calls, rejected_rc, (unsigned long long)rejected_wmma_calls, + (unsigned long long)rejected_k64_calls, ok ? "PASS" : "FAIL"); return ok; } @@ -2938,6 +3010,7 @@ bool run_attention_output_wmma_smoke(const aligned_model &model) { env_snapshot wmma_disable(kPrefillWmmaDisable); env_snapshot wmma_require(kPrefillWmmaRequire); env_snapshot wmma_row_tile(kPrefillWmmaRowTile); + env_snapshot wmma_k64(kPrefillWmmaK64); ds4_gpu_set_ssd_streaming(false); (void)unsetenv(kPrefillEnable); @@ -2949,6 +3022,7 @@ bool run_attention_output_wmma_smoke(const aligned_model &model) { (void)setenv(kPrefillWmmaDisable, "1", 1); (void)unsetenv(kPrefillWmmaRequire); (void)unsetenv(kPrefillWmmaRowTile); + (void)setenv(kPrefillWmmaK64, "0", 1); ds4_rocm_test_q4_prefill_wmma_reset(); const int tile8_rc = ds4_gpu_attention_output_q4_K_batch_tensor( tile8_out.ptr, tile8_low.ptr, nullptr, nullptr, @@ -2958,11 +3032,14 @@ bool run_attention_output_wmma_smoke(const aligned_model &model) { heads_gpu.ptr, n_tokens); const uint64_t tile8_wmma_calls = ds4_rocm_test_q4_prefill_wmma_get_calls(); + const uint64_t tile8_k64_calls = + ds4_rocm_test_q4_prefill_wmma_k64_get_calls(); (void)unsetenv(kPrefillRequire); (void)unsetenv(kPrefillWmmaEnable); (void)unsetenv(kPrefillWmmaDisable); (void)unsetenv(kPrefillWmmaRequire); + (void)setenv(kPrefillWmmaK64, "0", 1); ds4_rocm_test_q4_prefill_wmma_reset(); const int wmma_rc = ds4_gpu_attention_output_q4_K_batch_tensor( wmma_out.ptr, wmma_low.ptr, nullptr, nullptr, @@ -2972,13 +3049,17 @@ bool run_attention_output_wmma_smoke(const aligned_model &model) { heads_gpu.ptr, n_tokens); const uint64_t wmma_calls = ds4_rocm_test_q4_prefill_wmma_get_calls(); + const uint64_t wmma_k64_calls = + ds4_rocm_test_q4_prefill_wmma_k64_get_calls(); std::vector tile8_low_host(low_sentinel.size()); std::vector tile8_out_host(out_sentinel.size()); std::vector wmma_low_host(low_sentinel.size()); std::vector wmma_out_host(out_sentinel.size()); bool ok = tile8_rc == 1 && wmma_rc == 1 && - tile8_wmma_calls == 0u && wmma_calls == 1u && + tile8_wmma_calls == 0u && tile8_k64_calls == 0u && + wmma_calls == 1u && + wmma_k64_calls == 0u && read_tensor(tile8_low.ptr, &tile8_low_host) && read_tensor(tile8_out.ptr, &tile8_out_host) && read_tensor(wmma_low.ptr, &wmma_low_host) && @@ -3019,11 +3100,68 @@ bool run_attention_output_wmma_smoke(const aligned_model &model) { wmma_out_host, tile8_out_host, 16.0f, 8.0e-2f, "production output A-WMMA/B-TILE8 vs all-TILE8") && ok; } + + int k64_default_rc = 0; + uint64_t k64_default_wmma_calls = 0u; + uint64_t k64_default_calls = 0u; + std::vector k64_default_low_host(low_sentinel.size()); + std::vector k64_default_out_host(out_sentinel.size()); + if (ok) { + ok = write_tensor(wmma_low.ptr, low_sentinel) && + write_tensor(wmma_out.ptr, out_sentinel); + } + if (ok) { + /* Attest the production K64 default, not its explicit opt-in alias. */ + (void)unsetenv(kPrefillWmmaK64); + ds4_rocm_test_q4_prefill_wmma_reset(); + k64_default_rc = ds4_gpu_attention_output_q4_K_batch_tensor( + wmma_out.ptr, wmma_low.ptr, nullptr, nullptr, + model.data, model.size, model.decode_attn_a_offset, + model.decode_attn_b_offset, kQ4Type, kDecodeAttnGroupDim, + kDecodeAttnRank, kDecodeAttnGroups, kDecodeAttnOutDim, + heads_gpu.ptr, n_tokens); + k64_default_wmma_calls = + ds4_rocm_test_q4_prefill_wmma_get_calls(); + k64_default_calls = ds4_rocm_test_q4_prefill_wmma_k64_get_calls(); + ok = k64_default_rc == 1 && k64_default_wmma_calls == 1u && + k64_default_calls == 1u && + read_tensor(wmma_low.ptr, &k64_default_low_host) && + read_tensor(wmma_out.ptr, &k64_default_out_host); + } + if (ok) { + ok = output_body_overwritten( + k64_default_low_host, low_sentinel, low_count, + "production output-A default K64 direct-WMMA body") && ok; + ok = output_body_overwritten( + k64_default_out_host, out_sentinel, out_count, + "production output-B A-default-K64/B-TILE8 body") && ok; + ok = output_guard_unchanged( + k64_default_low_host, low_sentinel, low_count, + "production output-A default K64 direct-WMMA N-tail") && ok; + ok = output_guard_unchanged( + k64_default_out_host, out_sentinel, out_count, + "production output-B A-default-K64/B-TILE8 N-tail") && ok; + k64_default_low_host.resize(low_count); + k64_default_out_host.resize(out_count); + ok = bitwise_equal( + k64_default_low_host, wmma_low_host, + "production output-A default K64 vs K32") && ok; + ok = bitwise_equal( + k64_default_out_host, wmma_out_host, + "production output A-default-K64/B-TILE8 vs " + "A-K32/B-TILE8") && ok; + } std::fprintf(stderr, "ROCm Q4 production output WMMA safety default: " - "tile8=%d/%llu A-WMMA/B-TILE8=%d/%llu %s\n", + "tile8=%d/%llu/%llu A-K32/B-TILE8=%d/%llu/%llu " + "A-K64-default/B-TILE8=%d/%llu/%llu %s\n", tile8_rc, (unsigned long long)tile8_wmma_calls, + (unsigned long long)tile8_k64_calls, wmma_rc, (unsigned long long)wmma_calls, + (unsigned long long)wmma_k64_calls, + k64_default_rc, + (unsigned long long)k64_default_wmma_calls, + (unsigned long long)k64_default_calls, ok ? "PASS" : "FAIL"); return ok; } @@ -3106,6 +3244,7 @@ int main(int argc, char **argv) { env_snapshot wmma_disable(kPrefillWmmaDisable); env_snapshot wmma_require(kPrefillWmmaRequire); env_snapshot wmma_row_tile(kPrefillWmmaRowTile); + env_snapshot wmma_k64_global(kPrefillWmmaK64); env_snapshot q8_wave32_enable(kPrefillQ8Wave32Enable); env_snapshot q8_wave32_disable(kPrefillQ8Wave32Disable); env_snapshot q8_wave32_require(kPrefillQ8Wave32Require); @@ -3124,6 +3263,8 @@ int main(int argc, char **argv) { (void)unsetenv(kPrefillWmmaDisable); (void)unsetenv(kPrefillWmmaRequire); (void)unsetenv(kPrefillWmmaRowTile); + /* Neutralize the new K64 default for every non-K64-specific oracle. */ + (void)setenv(kPrefillWmmaK64, "0", 1); (void)unsetenv(kPrefillQ8Wave32Enable); (void)unsetenv(kPrefillQ8Wave32Disable); (void)unsetenv(kPrefillQ8Wave32Require); From 75503b9166bdea802a183b9f9371ae272411efe1 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Tue, 1 Sep 2026 23:55:45 +0200 Subject: [PATCH 182/189] Fix ROCm Q4 attention output WMMA quality drift --- ENVIRONMENT_VARIABLES.md | 18 +- rocm/ds4_rocm_q4.cuh | 90 +++- scripts/environment_variables.tsv | 12 +- speed-bench/README.md | 35 +- speed-bench/rocm_q4_prefill_bench.cpp | 163 +++++-- tests/test_rocm_q4_dense_pair.cpp | 633 +++++++++++++++++++------- 6 files changed, 691 insertions(+), 260 deletions(-) diff --git a/ENVIRONMENT_VARIABLES.md b/ENVIRONMENT_VARIABLES.md index 4de3c2931..2c5d3c980 100644 --- a/ENVIRONMENT_VARIABLES.md +++ b/ENVIRONMENT_VARIABLES.md @@ -98,12 +98,12 @@ The detailed Metal A/B contracts and expected oracle counters live in | `DS4_ROCM_ENABLE_Q4_PREFILL_Q8_K_WAVE32=1` | On gfx1151 wave32, opt into the no-LDS Q8_K activation quantizer that assigns one 256-value block to each wave before the exact Q4 prefill matmul. This exact path takes precedence over automatic direct-Q4 WMMA; `REQUIRE_Q4_PREFILL_WMMA` overrides an optional request, while dual REQUIRE fails closed. | | `DS4_ROCM_DISABLE_Q4_PREFILL_Q8_K_WAVE32=1` | Dominant rollback to the canonical one-workgroup-per-Q8_K-block quantizer. | | `DS4_ROCM_REQUIRE_Q4_PREFILL_Q8_K_WAVE32=1` | Require the wave32 quantizer for strict prefill A/B runs; unsupported scope/device, rollback, or an incompatible required F16/WMMA path fails closed. | -| `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA=0/1` | Compatibility control for the resident gfx1151 wave32 direct-Q4 WMMA prefill kernel at 256–4096 tokens. Unset keeps the automatic standalone/attention-output-A path, while the numerically compounded all-Q4 attention-output B stays on Q8_K+TILE8; an enabled value also opts B into direct WMMA for experiments, and an explicit false value opts out of resident WMMA entirely. The kernel uses transient F16 register dequantization and F32 accumulation, with 64 rows below output dimension 1024, 128 rows below 8192, and 256 rows otherwise. | +| `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA=0/1` | Compatibility control for the resident gfx1151 wave32 direct-Q4 WMMA prefill kernel at 256–4096 tokens. Unset keeps the automatic standalone and attention-output-A paths, while attention-output B stays on Q8_K+TILE8. A true value explicitly retains those eligible A paths but no longer opts B into direct WMMA; an explicit false value opts out unless `REQUIRE` is also set. Use `DISABLE=1` for an authoritative rollback. The kernel uses transient F16 register dequantization and F32 accumulation, with 64 rows below output dimension 1024, 128 rows below 8192, and 256 rows otherwise. | | `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_K64=0/1` | K64/P80 is the default staging mode for an otherwise eligible direct-Q4 WMMA launch. It stages two adjacent 32-value Q4_K groups in one padded LDS tile, halving workgroup barriers while preserving activation traffic and K32 accumulation order. Unset or true keeps K64/P80; `0`/`false`/`no`/`off` rolls back selectively to K32, while `DS4_ROCM_DISABLE_Q4_PREFILL_WMMA=1` rolls back direct WMMA entirely. | | `DS4_ROCM_Q4_PREFILL_WMMA_ROW_TILE=64|128|256` | Override the direct-Q4 WMMA output-row tile. The default uses 64 rows below output dimension 1024, 128 below 8192, and 256 otherwise; `64` also retains the prior kernel geometry as an A/B control. | -| `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_SSD=1` | Explicitly allow direct-Q4 WMMA during SSD streaming, including attention-output B, only when each complete projection weight range is already backed by physical device storage. It never treats mapped/registered host memory as resident. | +| `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_SSD=1` | Explicitly allow direct-Q4 WMMA during SSD streaming only when each complete projection weight range is already backed by physical device storage. It opts attention-output A into WMMA while B remains on Q8_K+TILE8, and never treats mapped/registered host memory as resident. | | `DS4_ROCM_DISABLE_Q4_PREFILL_WMMA=1` | Dominant value-aware opt-out for the automatic direct-Q4 WMMA path, including explicit resident or SSD requests. | -| `DS4_ROCM_REQUIRE_Q4_PREFILL_WMMA=1` | Require direct-Q4 WMMA for every selected projection and fail closed on an unsupported device/shape, quality mode, rollback, or SSD weight range that is not physically device-resident. In an all-Q4 attention-output batch this explicitly selects both A and the experimental B path; it is intended for strict A/B oracles, not quality-sensitive defaults. | +| `DS4_ROCM_REQUIRE_Q4_PREFILL_WMMA=1` | Require direct-Q4 WMMA for every selected projection and fail closed on an unsupported device/shape, quality mode, rollback, or SSD weight range that is not physically device-resident. In an all-Q4 attention-output batch this is the only control that selects the numerically compounded direct-WMMA B stage; it is a diagnostic assertion for strict kernel A/B oracles, not a quality-sensitive runtime setting. | | `DS4_ROCM_Q4_PREFILL_TILE8_STATS=1` | Report dense, pair, attention-batch, and token counters at process exit. | | `DS4_ROCM_ENABLE_Q4_DENSE_PAIR=1` | Share one Q8_K activation quantization between the two Q4 dense projections. This pair remains opt-in. | | `DS4_ROCM_DISABLE_Q4_DENSE_PAIR=1` | Dominant rollback for the ROCm Q4 dense pair. | @@ -974,7 +974,7 @@ and **19 tool/wrapper entries**. | `DS4_ROCM_DISABLE_Q4_DENSE_PAIR` | presence rollback; unset leaves opt-in policy unchanged | Disable/roll back rocm disable q4 dense pair. | [rocm/ds4_rocm_q4.cuh:435](rocm/ds4_rocm_q4.cuh#L435) | | `DS4_ROCM_DISABLE_Q4_GROUPED_ATTN_A` | presence rollback; unset permits the caller-marked resident decode production-shape default and explicit ENABLE/REQUIRE; any defined value including empty or 0 disables all grouped attention-A paths and wins over ENABLE/REQUIRE | Restore eight standalone Q4 attention-A projections instead of the two-dispatch grouped path. | [rocm/ds4_rocm_q4.cuh:868](rocm/ds4_rocm_q4.cuh#L868) | | `DS4_ROCM_DISABLE_Q4_PREFILL_TILE8` | presence rollback; TILE8 is default for 9..4096 tokens | Disable/roll back rocm disable q4 prefill tile8. | [rocm/ds4_rocm_q4.cuh:448](rocm/ds4_rocm_q4.cuh#L448) | -| `DS4_ROCM_DISABLE_Q4_PREFILL_WMMA` | value-aware authoritative opt-out for the automatic resident path and explicit SSD/REQUIRE requests; unset/0/false/no/off leaves policy unchanged, while empty or any other value disables; REQUIRE then fails closed | Prevent the gfx1151 direct-Q4 WMMA prefill path from dispatching and retain the Q8_K-plus-TILE8/TILE4 path. | [rocm/ds4_rocm_q4.cuh:1068](rocm/ds4_rocm_q4.cuh#L1068) | +| `DS4_ROCM_DISABLE_Q4_PREFILL_WMMA` | value-aware authoritative opt-out for the automatic resident path and explicit SSD/REQUIRE requests; unset/0/false/no/off leaves policy unchanged, while empty or any other value disables; REQUIRE then fails closed | Prevent the gfx1151 direct-Q4 WMMA prefill path from dispatching and retain the Q8_K-plus-TILE8/TILE4 path. | [rocm/ds4_rocm_q4.cuh:1358](rocm/ds4_rocm_q4.cuh#L1358) | | `DS4_ROCM_DISABLE_Q4_SELECTED_EXPERT_VIEWS` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable q4 selected expert views. | [ds4.c:21150](ds4.c#L21150) | | `DS4_ROCM_DISABLE_RESIDENT_IQ2_SORTED` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable resident iq2 sorted. | [rocm/ds4_rocm_moe_launch.cuh:751](rocm/ds4_rocm_moe_launch.cuh#L751) | | `DS4_ROCM_DISABLE_ROUTED_PAIR_SWIGLU_FUSION` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable routed pair swiglu fusion. | [ds4.c:18543](ds4.c#L18543) | @@ -1011,9 +1011,9 @@ and **19 tool/wrapper entries**. | `DS4_ROCM_ENABLE_MXFP4_TILE4` | presence opt-in; unset=off; any defined value including empty or 0 enables the candidate when the MXFP4 sorted-tile path has at least 5 tokens and neither TILE32 nor LDSB is selected | Select the ROCm MXFP4 gate/up tile4 occupancy variant, reducing staged-activation LDS per block. | [rocm/ds4_rocm_moe_launch.cuh:794](rocm/ds4_rocm_moe_launch.cuh#L794) | | `DS4_ROCM_ENABLE_Q4_DENSE_PAIR` | presence opt-in; unset=off; DISABLE takes precedence | Enable rocm enable q4 dense pair. | [rocm/ds4_rocm_q4.cuh:434](rocm/ds4_rocm_q4.cuh#L434) | | `DS4_ROCM_ENABLE_Q4_GROUPED_ATTN_A` | presence opt-in outside the default scope; the exact caller-marked resident decode shape groups=8, N=1, K=4096, M=1024 is automatic, while row-at-a-time batch fallbacks are not; DISABLE wins | Enable grouped Q4 attention-A for eligible slices, non-production shapes, or explicit experiments in addition to the resident decode default. | [rocm/ds4_rocm_q4.cuh:872](rocm/ds4_rocm_q4.cuh#L872) | -| `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA` | value-aware compatibility control; unset keeps automatic resident direct-Q4 WMMA for standalone dense and attention-output A while all-Q4 attention-output B remains on Q8_K+TILE8; empty or any value other than 0/false/no/off also opts B into experimental direct WMMA; explicit 0/false/no/off opts out of resident WMMA; N=256..4096, K a positive multiple of 256, resident non-quality gfx1151 wave32 only; SSD has a separate gate and DISABLE wins | Use compressed Q4_K-to-F16 register dequantization plus shape-selected 64-token by 64/128/256-row WMMA tiles and two-wide activation staging on the wider tiles, without Q8_K activation scratch or an F16 weight sidecar. | [rocm/ds4_rocm_q4.cuh:1089](rocm/ds4_rocm_q4.cuh#L1089) | -| `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_K64` | value-aware compatibility control for the default K64/P80 stage; unset, empty, or any value other than 0/false/no/off selects K64/P80 after the normal direct-Q4 WMMA device, shape, residency, and quality gates pass; 0/false/no/off rolls back only to the established K32 stage; DS4_ROCM_DISABLE_Q4_PREFILL_WMMA wins | Stage two adjacent 32-value Q4_K groups and a 64-value activation slice in one padded P80 LDS tile, halving workgroup barriers while preserving activation traffic and the K32 accumulation order. | [rocm/ds4_rocm_q4.cuh:1537](rocm/ds4_rocm_q4.cuh#L1537) | -| `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_SSD` | value-aware SSD-only opt-in, default off; unset/0/false/no/off retains TILE8/TILE4, while empty or any other value requests direct-Q4 WMMA including attention-output B; eligibility additionally requires each complete projection weight range in physical device storage rather than mapped/registered host memory; DISABLE wins | Allow the compressed direct-Q4 WMMA kernel to consume an already device-resident/cache-backed Q4_K projection during SSD streaming without changing model I/O. | [rocm/ds4_rocm_q4.cuh:1091](rocm/ds4_rocm_q4.cuh#L1091) | +| `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA` | value-aware compatibility control; unset keeps automatic resident direct-Q4 WMMA for standalone dense and attention-output A while attention-output B remains on Q8_K+TILE8; empty or any value other than 0/false/no/off explicitly retains the same eligible A paths but no longer opts B into direct WMMA; explicit 0/false/no/off opts out unless REQUIRE is set, while DISABLE is the authoritative rollback; N=256..4096, K a positive multiple of 256, resident non-quality gfx1151 wave32 only; SSD has a separate gate | Use compressed Q4_K-to-F16 register dequantization plus shape-selected 64-token by 64/128/256-row WMMA tiles and two-wide activation staging on the wider tiles, without Q8_K activation scratch or an F16 weight sidecar. | [rocm/ds4_rocm_q4.cuh:1354](rocm/ds4_rocm_q4.cuh#L1354) | +| `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_K64` | value-aware compatibility control for the default K64/P80 stage; unset, empty, or any value other than 0/false/no/off selects K64/P80 after the normal direct-Q4 WMMA device, shape, residency, and quality gates pass; 0/false/no/off rolls back only to the established K32 stage; DS4_ROCM_DISABLE_Q4_PREFILL_WMMA wins | Stage two adjacent 32-value Q4_K groups and a 64-value activation slice in one padded P80 LDS tile, halving workgroup barriers while preserving activation traffic and the K32 accumulation order. | [rocm/ds4_rocm_q4.cuh:1566](rocm/ds4_rocm_q4.cuh#L1566) | +| `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_SSD` | value-aware SSD-only opt-in, default off; unset/0/false/no/off retains TILE8/TILE4, while empty or any other value requests direct-Q4 WMMA for eligible standalone projections and attention-output A but leaves attention-output B on Q8_K+TILE8; eligibility additionally requires each complete projection weight range in physical device storage rather than mapped/registered host memory; DISABLE wins | Allow the compressed direct-Q4 WMMA kernel to consume an already device-resident/cache-backed Q4_K projection during SSD streaming without changing model I/O. | [rocm/ds4_rocm_q4.cuh:1356](rocm/ds4_rocm_q4.cuh#L1356) | | `DS4_ROCM_ENABLE_STREAMING_FULL_EXPERT_ADDR_TABLE` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming full expert addr table. | [ds4.c:18255](ds4.c#L18255) | | `DS4_ROCM_ENABLE_STREAMING_MADVISE_WILLNEED` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming madvise willneed. | [ds4.c:18224](ds4.c#L18224) | | `DS4_ROCM_ENABLE_STREAMING_PREFILL_BATCH_SELECTED_ADDR` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming prefill batch selected addr. | [ds4.c:18584](ds4.c#L18584) | @@ -1067,12 +1067,12 @@ and **19 tool/wrapper entries**. | `DS4_ROCM_MXFP4_DOWN_RGROUP` | nonempty value is parsed by strtol and a numeric prefix is sufficient; integers 1..8 are accepted; unset, empty, invalid, or out-of-range values use 1 | Set how many 32-row output blocks each ROCm MXFP4 tiled down-projection block computes, reducing the first launch-grid dimension as the value increases. | [rocm/ds4_rocm_moe_launch.cuh:801](rocm/ds4_rocm_moe_launch.cuh#L801) | | `DS4_ROCM_Q4_GROUPED_ATTN_A_STATS` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Print counters for rocm q4 grouped attn a stats. | [rocm/ds4_rocm_q4.cuh:614](rocm/ds4_rocm_q4.cuh#L614) | | `DS4_ROCM_Q4_PREFILL_TILE8_STATS` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Print counters for rocm q4 prefill tile8 stats. | [rocm/ds4_rocm_q4.cuh:514](rocm/ds4_rocm_q4.cuh#L514) | -| `DS4_ROCM_Q4_PREFILL_WMMA_ROW_TILE` | unsigned integer; unset, empty, malformed, negative, or values other than 64/128/256 use shape selection (64 rows when M<1024, 128 when M<8192, otherwise 256); 64 retains the previous geometry | Override the number of output rows sharing each direct-Q4 64x32 activation tile for controlled 64/128/256-row ROCm WMMA A/B measurements. | [rocm/ds4_rocm_q4.cuh:1173](rocm/ds4_rocm_q4.cuh#L1173) | +| `DS4_ROCM_Q4_PREFILL_WMMA_ROW_TILE` | unsigned integer; unset, empty, malformed, negative, or values other than 64/128/256 use shape selection (64 rows when M<1024, 128 when M<8192, otherwise 256); 64 retains the previous geometry | Override the number of output rows sharing each direct-Q4 64x32 activation tile for controlled 64/128/256-row ROCm WMMA A/B measurements. | [rocm/ds4_rocm_q4.cuh:1517](rocm/ds4_rocm_q4.cuh#L1517) | | `DS4_ROCM_Q8_DECODE_SHAREDX_64K` | sampled once; unset: enabled; present empty or exact 0: disabled; every other present value: enabled; effective only for one-token non-prequant Q8_0 matmul with 8192 < in_dim <= 16384 | Allows the ROCm shared-input Q8 decode kernel to use up to 64 KiB dynamic LDS for wide inputs; an unsupported/failed LDS launch automatically falls back to the regular kernel. | [rocm/ds4_rocm_runtime.cuh:4805](rocm/ds4_rocm_runtime.cuh#L4805) | | `DS4_ROCM_Q_STAGE_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm q stage profile. | [ds4.c:29284](ds4.c#L29284) | | `DS4_ROCM_REQUIRE_Q4_GROUPED_ATTN_A` | presence fail-closed assertion; also requests the candidate outside the caller-marked resident decode default; DISABLE remains authoritative and causes failure | Require grouped Q4 attention-A and fail instead of silently falling back. | [rocm/ds4_rocm_q4.cuh:870](rocm/ds4_rocm_q4.cuh#L870) | | `DS4_ROCM_REQUIRE_Q4_PREFILL_TILE8` | presence fail-closed assertion for eligible TILE8 calls | Require rocm require q4 prefill tile8 and fail instead of silently falling back. | [rocm/ds4_rocm_q4.cuh:452](rocm/ds4_rocm_q4.cuh#L452) | -| `DS4_ROCM_REQUIRE_Q4_PREFILL_WMMA` | value-aware strict assertion, not required for the automatic resident default; unset/0/false/no/off is off; empty or any other value requires every selected Q4 dense or attention-output projection to use direct-Q4 WMMA, explicitly including experimental all-Q4 attention-output B; unsupported shape/device, quality mode, DISABLE, or an SSD weight range without physical device residency fails before dispatch | Prevent the ROCm Q4 prefill WMMA correctness/performance oracle from silently timing TILE8/TILE4; quality-sensitive defaults retain Q8_K+TILE8 for attention-output B. | [rocm/ds4_rocm_q4.cuh:1095](rocm/ds4_rocm_q4.cuh#L1095) | +| `DS4_ROCM_REQUIRE_Q4_PREFILL_WMMA` | value-aware strict diagnostic assertion, not required for the automatic standalone/attention-output-A resident default; unset/0/false/no/off is off; empty or any other value requires every selected Q4 dense or attention-output projection to use direct-Q4 WMMA and is the only control that selects the numerically compounded attention-output B stage; unsupported shape/device, quality mode, DISABLE, or an SSD weight range without physical device residency fails before dispatch | Prevent a strict ROCm Q4 prefill WMMA kernel oracle from silently timing TILE8/TILE4; normal runtime controls keep attention-output B on Q8_K+TILE8. | [rocm/ds4_rocm_q4.cuh:1360](rocm/ds4_rocm_q4.cuh#L1360) | | `DS4_ROCM_STREAMING_DECODE_PREFILL_MAX` | primary nonempty value over the Metal alias; parsed by strtol when it has a numeric prefix (trailing text is accepted); <= 0 disables, values > UINT32_MAX clamp, no numeric prefix uses automatic default: 64 for Flash with uniform Q4_K/MXFP4 experts, 18 for other Pro/Flash, otherwise 0; the disable flag dominates | Sets the largest short, non-quality SSD-streaming prefill batch routed through the decode-style path instead of canonical layer-major prefill. | [ds4.c:31976](ds4.c#L31976) | | `DS4_ROCM_STREAMING_EXPERT_AUTO_PRELOAD_CAP` | primary nonempty value over the Metal alias; strict full-string strtoul; valid values > UINT32_MAX clamp, invalid uses 4096, and 0 means no cap (not disabled); when CLI preload is auto/0, unset defaults to cap 4096 except ROCm GLM52, where absent/empty disables automatic preload entirely | Caps the number of hot experts synchronously seeded into the SSD-streaming expert cache in automatic preload mode; an explicit CLI preload count bypasses this cap, and setting this variable opts ROCm GLM52 back into auto preload. | [ds4.c:21469](ds4.c#L21469) | | `DS4_ROCM_STREAMING_EXPERT_CACHE_VERBOSE` | presence flag; unset=off | Print verbose ROCm streaming expert-cache seed/load diagnostics. | [rocm/ds4_rocm_runtime.cuh:2904](rocm/ds4_rocm_runtime.cuh#L2904) | diff --git a/rocm/ds4_rocm_q4.cuh b/rocm/ds4_rocm_q4.cuh index d887540fd..8760961f6 100644 --- a/rocm/ds4_rocm_q4.cuh +++ b/rocm/ds4_rocm_q4.cuh @@ -1261,18 +1261,34 @@ static int rocm_q4_K_prefill_wmma_requested_policy( return ssd_streaming ? ssd_enabled == 1 : enabled != 0; } -/* A direct attention-output B consumes the already approximate output of A. - * Unlike standalone dense and attention-output A, do not select it from the - * resident automatic default: require an explicit resident or SSD request. */ -static int rocm_q4_K_prefill_wmma_b_requested_policy( +/* Attention output is a two-stage A -> B projection. A retains the validated + * standalone policy: resident execution is automatic, while SSD keeps its + * explicit gate and physical-device-residency contract. */ +static int rocm_q4_K_prefill_wmma_attention_a_requested_policy( int ssd_streaming, int enabled, int ssd_enabled, int disabled, int required) { - if (required) return 1; - if (disabled) return 0; - return ssd_streaming ? ssd_enabled == 1 : enabled == 1; + return rocm_q4_K_prefill_wmma_requested_policy( + ssd_streaming, enabled, ssd_enabled, disabled, required); +} + +/* Applying direct WMMA to both stages compounds the two F16 approximations. + * Keep attention-output B behind REQUIRE so ordinary ENABLE A/B measurements + * retain the exact Q8_K+TILE8 second stage. Test REQUIRE before DISABLE so + * DISABLE+REQUIRE reaches the normal selector and fails closed. */ +static int rocm_q4_K_prefill_wmma_attention_b_requested_policy( + int ssd_streaming, + int enabled, + int ssd_enabled, + int disabled, + int required) { + (void)ssd_streaming; + (void)enabled; + (void)ssd_enabled; + (void)disabled; + return required != 0; } /* An explicitly selected exact Q8_K quantizer owns an optional WMMA default. @@ -1305,13 +1321,26 @@ extern "C" int ds4_rocm_test_q4_prefill_wmma_requested_policy( required != 0); } -extern "C" int ds4_rocm_test_q4_prefill_wmma_b_requested_policy( +extern "C" int +ds4_rocm_test_q4_prefill_wmma_attention_a_requested_policy( int ssd_streaming, int enabled, int ssd_enabled, int disabled, int required) { - return rocm_q4_K_prefill_wmma_b_requested_policy( + return rocm_q4_K_prefill_wmma_attention_a_requested_policy( + ssd_streaming != 0, enabled, ssd_enabled, disabled != 0, + required != 0); +} + +extern "C" int +ds4_rocm_test_q4_prefill_wmma_attention_b_requested_policy( + int ssd_streaming, + int enabled, + int ssd_enabled, + int disabled, + int required) { + return rocm_q4_K_prefill_wmma_attention_b_requested_policy( ssd_streaming != 0, enabled, ssd_enabled, disabled != 0, required != 0); } @@ -2405,11 +2434,23 @@ extern "C" int ds4_gpu_attention_output_q4_K_batch_tensor( "DS4_ROCM_DISABLE_Q4_PREFILL_WMMA") == 1; const int wmma_required = rocm_q4_attn_q_b_env_bool( "DS4_ROCM_REQUIRE_Q4_PREFILL_WMMA") == 1; - const int wmma_requested = rocm_q4_K_prefill_wmma_requested_policy( - g_ssd_streaming_mode, wmma_enabled, wmma_ssd_enabled, - wmma_disabled, wmma_required); - if (!tile8_scope) return 0; - if (!tile8_requested && !wmma_requested) { + const int a_wmma_requested = + rocm_q4_K_prefill_wmma_attention_a_requested_policy( + g_ssd_streaming_mode, wmma_enabled, wmma_ssd_enabled, + wmma_disabled, wmma_required); + const int b_wmma_requested = + rocm_q4_K_prefill_wmma_attention_b_requested_policy( + g_ssd_streaming_mode, wmma_enabled, wmma_ssd_enabled, + wmma_disabled, wmma_required); + const int attention_wmma_requested = + a_wmma_requested || b_wmma_requested; + if (!tile8_scope) { + /* REQUIRE is a strict diagnostic assertion. Do not let an + * unsupported attention-output shape escape into a grouped/per-token + * fallback that never attests the requested WMMA kernel. */ + return wmma_required ? -1 : 0; + } + if (!tile8_requested && !attention_wmma_requested) { if (tile8_required || q8_wave32_required) { fprintf(stderr, "ds4: required ROCm Q4_K attention-output prefill " @@ -2498,17 +2539,16 @@ extern "C" int ds4_gpu_attention_output_q4_K_batch_tensor( const int out_b_device_resident = resident_out_b != NULL && resident_out_b == out_b; - /* Resolve every strict WMMA decision before A can enqueue work. That - * keeps REQUIRE fail-closed: an ineligible B projection can never make - * the graph replay a fallback over an already submitted A projection. */ - int a_wmma = rocm_q4_K_prefill_wmma_select( - n_tokens, group_dim, rank, out_a_device_resident); - /* The all-Q4 A+B direct path compounds two F16 input/dequantization - * approximations. Keep the validated A candidate automatic, but leave B - * on exact Q8_K+TILE8 unless the user explicitly requests WMMA. */ - const int b_wmma_requested = rocm_q4_K_prefill_wmma_b_requested_policy( - g_ssd_streaming_mode, wmma_enabled, wmma_ssd_enabled, wmma_disabled, - wmma_required); + /* Resolve both independently requested stages before A can enqueue work. + * This keeps REQUIRE fail-closed: an ineligible B projection can never + * make the graph replay a fallback over an already submitted A projection. + * A retains the validated resident automatic default; B does not inherit + * it because applying direct WMMA to both stages compounds their F16 + * approximations. */ + int a_wmma = a_wmma_requested + ? rocm_q4_K_prefill_wmma_select( + n_tokens, group_dim, rank, out_a_device_resident) + : ROCM_Q4_PREFILL_WMMA_FALLBACK; int b_wmma = out_b_type == 12u && b_wmma_requested ? rocm_q4_K_prefill_wmma_select( n_tokens, low_dim, out_dim, out_b_device_resident) diff --git a/scripts/environment_variables.tsv b/scripts/environment_variables.tsv index 2702a8bfa..1c2e11cc9 100644 --- a/scripts/environment_variables.tsv +++ b/scripts/environment_variables.tsv @@ -980,7 +980,7 @@ runtime/rocm DS4_ROCM_DISABLE_Q4_GROUPED_ATTN_A presence rollback; unset permits runtime/rocm DS4_ROCM_DISABLE_Q4_PREFILL_K1024_TILE4 value-aware authoritative rollback; unset/0/false/no/off preserves the resident automatic default and any explicit SSD request; empty or any other value disables; overrides ENABLE and causes REQUIRE to fail closed Restore the generic eight-block Q4_K tiled-prefill kernel for K=1024 in both resident and SSD-streaming execution. rocm/ds4_rocm_q4.cuh:624 runtime/rocm DS4_ROCM_DISABLE_Q4_PREFILL_Q8_K_WAVE32 value-aware authoritative rollback; unset/0/false/no/off permits ENABLE or REQUIRE, while empty or any other value disables; REQUIRE then fails closed Restore the canonical one-workgroup-per-Q8_K-block activation quantizer for Q4 prefill. rocm/ds4_rocm_q4.cuh:803 runtime/rocm DS4_ROCM_DISABLE_Q4_PREFILL_TILE8 presence rollback; TILE8 is default for 9..4096 tokens Disable/roll back rocm disable q4 prefill tile8. rocm/ds4_rocm_q4.cuh:448 -runtime/rocm DS4_ROCM_DISABLE_Q4_PREFILL_WMMA value-aware authoritative opt-out for the automatic resident path and explicit SSD/REQUIRE requests; unset/0/false/no/off leaves policy unchanged, while empty or any other value disables; REQUIRE then fails closed Prevent the gfx1151 direct-Q4 WMMA prefill path from dispatching and retain the Q8_K-plus-TILE8/TILE4 path. rocm/ds4_rocm_q4.cuh:1068 +runtime/rocm DS4_ROCM_DISABLE_Q4_PREFILL_WMMA value-aware authoritative opt-out for the automatic resident path and explicit SSD/REQUIRE requests; unset/0/false/no/off leaves policy unchanged, while empty or any other value disables; REQUIRE then fails closed Prevent the gfx1151 direct-Q4 WMMA prefill path from dispatching and retain the Q8_K-plus-TILE8/TILE4 path. rocm/ds4_rocm_q4.cuh:1358 runtime/rocm DS4_ROCM_DISABLE_Q4_SELECTED_EXPERT_VIEWS presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable q4 selected expert views. ds4.c:21150 runtime/rocm DS4_ROCM_DISABLE_RESIDENT_IQ2_SORTED presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable resident iq2 sorted. rocm/ds4_rocm_moe_launch.cuh:751 runtime/rocm DS4_ROCM_DISABLE_ROUTED_PAIR_SWIGLU_FUSION presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable routed pair swiglu fusion. ds4.c:18543 @@ -1024,9 +1024,9 @@ runtime/rocm DS4_ROCM_ENABLE_Q4_DENSE_PAIR presence opt-in; unset=off; DISABLE t runtime/rocm DS4_ROCM_ENABLE_Q4_GROUPED_ATTN_A presence opt-in outside the default scope; the exact caller-marked resident decode shape groups=8, N=1, K=4096, M=1024 is automatic, while row-at-a-time batch fallbacks are not; DISABLE wins Enable grouped Q4 attention-A for eligible slices, non-production shapes, or explicit experiments in addition to the resident decode default. rocm/ds4_rocm_q4.cuh:872 runtime/rocm DS4_ROCM_ENABLE_Q4_PREFILL_K1024_TILE4_SSD value-aware SSD-only opt-in, default off; unset/0/false/no/off retains TILE8, while empty or any other value requests TILE4; eligibility additionally requires N=9..4096, K=1024, M=32768, TILE8 enabled, and the complete weight range in device storage rather than mapped/registered host memory; DISABLE wins Allow the four-lane K=1024 Q4_K prefill specialization to consume an already device-resident/cache-backed attn_q_b weight range during SSD streaming without changing model I/O. rocm/ds4_rocm_q4.cuh:624 runtime/rocm DS4_ROCM_ENABLE_Q4_PREFILL_Q8_K_WAVE32 value-aware opt-in, default off; unset/0/false/no/off retains the canonical quantizer, while empty or any other value requests the candidate for N=9..4096 on gfx1151 wave32; DISABLE wins; a selected exact Q8 path takes precedence over automatic direct-Q4 WMMA, REQUIRE_WMMA overrides an optional request, and dual REQUIRE fails closed Quantize eight independent Q8_K activation blocks per 256-thread workgroup using one wave32 per block, without LDS or workgroup barriers, before the exact Q4 prefill matmul (TILE8 or its legacy rollback). rocm/ds4_rocm_q4.cuh:858 -runtime/rocm DS4_ROCM_ENABLE_Q4_PREFILL_WMMA value-aware compatibility control; unset keeps automatic resident direct-Q4 WMMA for standalone dense and attention-output A while all-Q4 attention-output B remains on Q8_K+TILE8; empty or any value other than 0/false/no/off also opts B into experimental direct WMMA; explicit 0/false/no/off opts out of resident WMMA; N=256..4096, K a positive multiple of 256, resident non-quality gfx1151 wave32 only; SSD has a separate gate and DISABLE wins Use compressed Q4_K-to-F16 register dequantization plus shape-selected 64-token by 64/128/256-row WMMA tiles and two-wide activation staging on the wider tiles, without Q8_K activation scratch or an F16 weight sidecar. rocm/ds4_rocm_q4.cuh:1089 -runtime/rocm DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_K64 value-aware compatibility control for the default K64/P80 stage; unset, empty, or any value other than 0/false/no/off selects K64/P80 after the normal direct-Q4 WMMA device, shape, residency, and quality gates pass; 0/false/no/off rolls back only to the established K32 stage; DS4_ROCM_DISABLE_Q4_PREFILL_WMMA wins Stage two adjacent 32-value Q4_K groups and a 64-value activation slice in one padded P80 LDS tile, halving workgroup barriers while preserving activation traffic and the K32 accumulation order. rocm/ds4_rocm_q4.cuh:1537 -runtime/rocm DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_SSD value-aware SSD-only opt-in, default off; unset/0/false/no/off retains TILE8/TILE4, while empty or any other value requests direct-Q4 WMMA including attention-output B; eligibility additionally requires each complete projection weight range in physical device storage rather than mapped/registered host memory; DISABLE wins Allow the compressed direct-Q4 WMMA kernel to consume an already device-resident/cache-backed Q4_K projection during SSD streaming without changing model I/O. rocm/ds4_rocm_q4.cuh:1091 +runtime/rocm DS4_ROCM_ENABLE_Q4_PREFILL_WMMA value-aware compatibility control; unset keeps automatic resident direct-Q4 WMMA for standalone dense and attention-output A while attention-output B remains on Q8_K+TILE8; empty or any value other than 0/false/no/off explicitly retains the same eligible A paths but no longer opts B into direct WMMA; explicit 0/false/no/off opts out unless REQUIRE is set, while DISABLE is the authoritative rollback; N=256..4096, K a positive multiple of 256, resident non-quality gfx1151 wave32 only; SSD has a separate gate Use compressed Q4_K-to-F16 register dequantization plus shape-selected 64-token by 64/128/256-row WMMA tiles and two-wide activation staging on the wider tiles, without Q8_K activation scratch or an F16 weight sidecar. rocm/ds4_rocm_q4.cuh:1354 +runtime/rocm DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_K64 value-aware compatibility control for the default K64/P80 stage; unset, empty, or any value other than 0/false/no/off selects K64/P80 after the normal direct-Q4 WMMA device, shape, residency, and quality gates pass; 0/false/no/off rolls back only to the established K32 stage; DS4_ROCM_DISABLE_Q4_PREFILL_WMMA wins Stage two adjacent 32-value Q4_K groups and a 64-value activation slice in one padded P80 LDS tile, halving workgroup barriers while preserving activation traffic and the K32 accumulation order. rocm/ds4_rocm_q4.cuh:1566 +runtime/rocm DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_SSD value-aware SSD-only opt-in, default off; unset/0/false/no/off retains TILE8/TILE4, while empty or any other value requests direct-Q4 WMMA for eligible standalone projections and attention-output A but leaves attention-output B on Q8_K+TILE8; eligibility additionally requires each complete projection weight range in physical device storage rather than mapped/registered host memory; DISABLE wins Allow the compressed direct-Q4 WMMA kernel to consume an already device-resident/cache-backed Q4_K projection during SSD streaming without changing model I/O. rocm/ds4_rocm_q4.cuh:1356 runtime/rocm DS4_ROCM_ENABLE_STREAMING_FULL_EXPERT_ADDR_TABLE presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming full expert addr table. ds4.c:18255 runtime/rocm DS4_ROCM_ENABLE_STREAMING_MADVISE_WILLNEED presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming madvise willneed. ds4.c:18224 runtime/rocm DS4_ROCM_ENABLE_STREAMING_PREFILL_BATCH_SELECTED_ADDR presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming prefill batch selected addr. ds4.c:18584 @@ -1084,7 +1084,7 @@ runtime/rocm DS4_ROCM_Q4_ATTN_Q_B_F16_CACHE_MIN_TOKENS integer token count with runtime/rocm DS4_ROCM_Q4_ATTN_Q_B_TRANSIENT_F16_MIN_TOKENS full-string unsigned token count; default 4096; accepted range 32..UINT32_MAX; values above clamp and invalid or smaller values restore 4096 Set the minimum device-resident, non-SSD prefill batch eligible for per-layer transient ROCm Q4_K attn_q_b-to-F16 expansion. rocm/ds4_rocm_q4_qb_sidecar.cuh:150 runtime/rocm DS4_ROCM_Q4_GROUPED_ATTN_A_STATS presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Print counters for rocm q4 grouped attn a stats. rocm/ds4_rocm_q4.cuh:614 runtime/rocm DS4_ROCM_Q4_PREFILL_TILE8_STATS presence diagnostic; unset=off; any defined value including empty or 0 prints at exit Print tiled-prefill dense/pair/attention counters plus total and SSD-specific K=1024 TILE4 dispatch counts. rocm/ds4_rocm_q4.cuh:741 -runtime/rocm DS4_ROCM_Q4_PREFILL_WMMA_ROW_TILE unsigned integer; unset, empty, malformed, negative, or values other than 64/128/256 use shape selection (64 rows when M<1024, 128 when M<8192, otherwise 256); 64 retains the previous geometry Override the number of output rows sharing each direct-Q4 64x32 activation tile for controlled 64/128/256-row ROCm WMMA A/B measurements. rocm/ds4_rocm_q4.cuh:1173 +runtime/rocm DS4_ROCM_Q4_PREFILL_WMMA_ROW_TILE unsigned integer; unset, empty, malformed, negative, or values other than 64/128/256 use shape selection (64 rows when M<1024, 128 when M<8192, otherwise 256); 64 retains the previous geometry Override the number of output rows sharing each direct-Q4 64x32 activation tile for controlled 64/128/256-row ROCm WMMA A/B measurements. rocm/ds4_rocm_q4.cuh:1517 runtime/rocm DS4_ROCM_Q8_DECODE_SHAREDX_64K sampled once; unset: enabled; present empty or exact 0: disabled; every other present value: enabled; effective only for one-token non-prequant Q8_0 matmul with 8192 < in_dim <= 16384 Allows the ROCm shared-input Q8 decode kernel to use up to 64 KiB dynamic LDS for wide inputs; an unsupported/failed LDS launch automatically falls back to the regular kernel. rocm/ds4_rocm_runtime.cuh:4805 runtime/rocm DS4_ROCM_Q_STAGE_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm q stage profile. ds4.c:30038 runtime/rocm DS4_ROCM_REQUIRE_Q4_ATTN_Q_B_F16_CACHE value-aware strict opt-in, default off; unset/0/false/no/off is off, empty or any other value requires eligible batches to use the cache; DISABLE wins Fail an eligible ROCm prefill instead of falling back when the resident Q4_K attn_q_b F16 specialization cannot be prepared or dispatched. rocm/ds4_rocm_q4_qb_sidecar.cuh:135 @@ -1092,7 +1092,7 @@ runtime/rocm DS4_ROCM_REQUIRE_Q4_GROUPED_ATTN_A presence fail-closed assertion; runtime/rocm DS4_ROCM_REQUIRE_Q4_PREFILL_K1024_TILE4 value-aware fail-closed assertion and SSD opt-in; unset/0/false/no/off is off; empty or any other value requires eligible N=9..4096, K=1024, M=32768 dense calls to select TILE4; SSD also requires an actual device-resident weight range; DISABLE wins Prevent a K=1024 TILE4 correctness/performance oracle from silently falling back to TILE8, including during SSD-streaming A/B runs. rocm/ds4_rocm_q4.cuh:628 runtime/rocm DS4_ROCM_REQUIRE_Q4_PREFILL_Q8_K_WAVE32 value-aware strict opt-in; unset/0/false/no/off is off; empty or any other value requires gfx1151 wave32 and N=9..4096; DISABLE and conflicts with required WMMA or the required q_b F16 cache fail closed Require the no-LDS wave32 Q8_K activation quantizer for exact Q4 prefill instead of silently using the canonical quantizer or an F16 side path. rocm/ds4_rocm_q4.cuh:796 runtime/rocm DS4_ROCM_REQUIRE_Q4_PREFILL_TILE8 presence fail-closed assertion for eligible TILE8 calls Require rocm require q4 prefill tile8 and fail instead of silently falling back. rocm/ds4_rocm_q4.cuh:452 -runtime/rocm DS4_ROCM_REQUIRE_Q4_PREFILL_WMMA value-aware strict assertion, not required for the automatic resident default; unset/0/false/no/off is off; empty or any other value requires every selected Q4 dense or attention-output projection to use direct-Q4 WMMA, explicitly including experimental all-Q4 attention-output B; unsupported shape/device, quality mode, DISABLE, or an SSD weight range without physical device residency fails before dispatch Prevent the ROCm Q4 prefill WMMA correctness/performance oracle from silently timing TILE8/TILE4; quality-sensitive defaults retain Q8_K+TILE8 for attention-output B. rocm/ds4_rocm_q4.cuh:1095 +runtime/rocm DS4_ROCM_REQUIRE_Q4_PREFILL_WMMA value-aware strict diagnostic assertion, not required for the automatic standalone/attention-output-A resident default; unset/0/false/no/off is off; empty or any other value requires every selected Q4 dense or attention-output projection to use direct-Q4 WMMA and is the only control that selects the numerically compounded attention-output B stage; unsupported shape/device, quality mode, DISABLE, or an SSD weight range without physical device residency fails before dispatch Prevent a strict ROCm Q4 prefill WMMA kernel oracle from silently timing TILE8/TILE4; normal runtime controls keep attention-output B on Q8_K+TILE8. rocm/ds4_rocm_q4.cuh:1360 runtime/rocm DS4_ROCM_STREAMING_DECODE_PREFILL_MAX primary nonempty value over the Metal alias; parsed by strtol when it has a numeric prefix (trailing text is accepted); <= 0 disables, values > UINT32_MAX clamp, no numeric prefix uses automatic default: 64 for Flash with uniform Q4_K/MXFP4 experts, 18 for other Pro/Flash, otherwise 0; the disable flag dominates Sets the largest short, non-quality SSD-streaming prefill batch routed through the decode-style path instead of canonical layer-major prefill. ds4.c:31976 runtime/rocm DS4_ROCM_STREAMING_EXPERT_AUTO_PRELOAD_CAP primary nonempty value over the Metal alias; strict full-string strtoul; valid values > UINT32_MAX clamp, invalid uses 4096, and 0 means no cap (not disabled); when CLI preload is auto/0, unset defaults to cap 4096 except ROCm GLM52, where absent/empty disables automatic preload entirely Caps the number of hot experts synchronously seeded into the SSD-streaming expert cache in automatic preload mode; an explicit CLI preload count bypasses this cap, and setting this variable opts ROCm GLM52 back into auto preload. ds4.c:21469 runtime/rocm DS4_ROCM_STREAMING_EXPERT_CACHE_VERBOSE presence flag; unset=off Print verbose ROCm streaming expert-cache seed/load diagnostics. rocm/ds4_rocm_runtime.cuh:2904 diff --git a/speed-bench/README.md b/speed-bench/README.md index 3992a9011..98e38b1f1 100644 --- a/speed-bench/README.md +++ b/speed-bench/README.md @@ -191,7 +191,8 @@ comparisons are: - `outb`: TILE8 versus the compressed direct-Q4 WMMA kernel at the production `output_b` `K=8192,M=4096` shape; - `output`: the complete grouped `output_a` plus `output_b` production API, - comparing two TILE8 projections with two direct WMMA projections. + comparing an all-TILE8 rollback with the production A-WMMA/B-TILE8 + pipeline. On gfx1151 wave32, `dense` and `qb` also emit a direct-Q4 WMMA comparison for `N>=256`. The candidate keeps Q4_K weights compressed, rounds each transient @@ -209,10 +210,16 @@ including its changed workgroup and occupancy contract, separately from the candidate's arithmetic change versus TILE8. `q_b` additionally compares scalar versus two-wide activation staging at fixed 128- and 256-row geometry. The direct hook receives both tile and loader explicitly, so every arm attests -its own configuration. Eligible resident runtime calls use direct-Q4 WMMA by -default; set `DS4_ROCM_DISABLE_Q4_PREFILL_WMMA=1` to opt out. The -production-API comparison still uses the strict REQUIRE gate so a rejected -dispatch fails instead of timing a fallback. +its own configuration. Eligible standalone resident calls and attention-output +A use direct-Q4 WMMA by default; set `DS4_ROCM_DISABLE_Q4_PREFILL_WMMA=1` to +opt out. Attention-output B remains on Q8_K+TILE8. The production comparison +sets `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA=1` without REQUIRE, which explicitly +retains that same A-WMMA/B-TILE8 policy. The hard B oracle replays TILE8 over +the same WMMA-low intermediate and requires bitwise-identical output. The composed +A-WMMA/B-TILE8 versus all-TILE8 delta remains visible as a non-gating +diagnostic because it includes A's deliberate F16 boundary rather than +isolating B correctness. Raw two-stage direct-WMMA row-geometry and K32/K64 +comparisons remain diagnostic-only kernel measurements. Eligible direct-Q4 WMMA launches use K64/P80 staging by default. It stages two adjacent 32-value Q4_K groups and a 64-value activation slice in one padded @@ -233,8 +240,10 @@ rollback without SSD-streaming or policy-selection noise: The benchmark always keeps SSD streaming disabled. Runtime SSD experiments need the separate `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_SSD=1` gate and only accept -projection ranges already backed by physical device memory, so model I/O is -never folded into the kernel result. +projection ranges already backed by physical device memory. That gate opts +attention-output A into WMMA but retains TILE8 for B; only the strict, +diagnostic `DS4_ROCM_REQUIRE_Q4_PREFILL_WMMA=1` selects direct WMMA for B. +Model I/O is therefore never folded into the kernel result. The default token set is `9,17,33,128,256,257,512`, covering the first row after the small TILE8 boundaries plus both the exact 256-token occupancy case @@ -249,11 +258,13 @@ and the first token after a 64-token WMMA boundary. Use `--full` for Every case rotates identical resident weight sets between arms, alternates ABBA/BAAB, and verifies allocation guards both before and after timing. -Comparisons among the integer Q4 -paths remain bit-exact. Direct-Q4 WMMA has a deliberate F16 arithmetic -boundary, so those comparisons require finite results within an explicit -absolute/relative smoke tolerance; release validation must also include the -model-logit or prompt oracle that guards the automatic policy. +Comparisons among the integer Q4 paths remain bit-exact. Direct-Q4 WMMA has a +deliberate F16 arithmetic boundary, so single-stage comparisons require finite +results within an explicit absolute/relative smoke tolerance. For the chained +production output, the B-stage same-input replay is bit-exact while the +end-to-end delta is informational; release validation must also include the +model-logit or prompt oracle that guards the automatic standalone and +attention-output-A policy. Fixture creation, the host-to-device residency copy, warmup, oracle readback, and environment-gate changes are outside the reported HIP-event intervals. `candidate_delta_pct` is negative when the candidate is faster; the companion diff --git a/speed-bench/rocm_q4_prefill_bench.cpp b/speed-bench/rocm_q4_prefill_bench.cpp index 48ecdc0e9..2e6f13956 100644 --- a/speed-bench/rocm_q4_prefill_bench.cpp +++ b/speed-bench/rocm_q4_prefill_bench.cpp @@ -39,6 +39,9 @@ extern "C" int ds4_rocm_bench_q4_K_wmma_variant_enqueue( uint64_t row_bytes, uint64_t x_token_stride, uint64_t x_group_stride, uint64_t out_token_stride, uint32_t row_tile, uint32_t k_tile, int load2); +extern "C" void ds4_rocm_test_q4_prefill_wmma_reset(void); +extern "C" uint64_t ds4_rocm_test_q4_prefill_wmma_get_calls(void); +extern "C" uint64_t ds4_rocm_test_q4_prefill_wmma_k64_get_calls(void); namespace { @@ -475,13 +478,15 @@ bool numerically_close(const ds4_gpu_tensor *got, uint64_t bytes, const char *label, float abs_tolerance = 2.0f, - float rel_tolerance = 3.0e-2f) { + float rel_tolerance = 3.0e-2f, + bool gate = true) { if ((bytes % sizeof(float)) != 0u) return false; const uint64_t chunk_bytes = std::min(kCompareChunk, bytes) & ~(uint64_t)(sizeof(float) - 1u); std::vector lhs(static_cast(chunk_bytes / sizeof(float))); std::vector rhs(static_cast(chunk_bytes / sizeof(float))); uint64_t failures = 0u; + uint64_t nonfinite = 0u; uint64_t compared = 0u; float max_abs = 0.0f; float max_rel = 0.0f; @@ -495,6 +500,12 @@ bool numerically_close(const ds4_gpu_tensor *got, } const uint64_t values = count / sizeof(float); for (uint64_t i = 0u; i < values; i++) { + if (!std::isfinite(lhs[(size_t)i]) || + !std::isfinite(rhs[(size_t)i])) { + failures++; + nonfinite++; + continue; + } const float diff = std::fabs(lhs[(size_t)i] - rhs[(size_t)i]); const float rel = diff / std::max(1.0f, std::fabs(rhs[(size_t)i])); @@ -503,9 +514,7 @@ bool numerically_close(const ds4_gpu_tensor *got, worst = compared + i; } max_rel = std::max(max_rel, rel); - if (!std::isfinite(lhs[(size_t)i]) || - !std::isfinite(rhs[(size_t)i]) || - diff > abs_tolerance + + if (diff > abs_tolerance + rel_tolerance * std::fabs(rhs[(size_t)i])) { failures++; } @@ -513,13 +522,15 @@ bool numerically_close(const ds4_gpu_tensor *got, compared += values; } std::fprintf(stderr, - "%s: failures=%llu/%llu max_abs=%g max_rel=%g worst=%llu " - "tolerance(abs=%g rel=%g) %s\n", + "%s: failures=%llu/%llu nonfinite=%llu max_abs=%g " + "max_rel=%g worst=%llu tolerance(abs=%g rel=%g) %s\n", label, (unsigned long long)failures, - (unsigned long long)compared, max_abs, max_rel, + (unsigned long long)compared, + (unsigned long long)nonfinite, max_abs, max_rel, (unsigned long long)worst, abs_tolerance, rel_tolerance, - failures == 0u ? "PASS" : "FAIL"); - return failures == 0u; + failures == 0u ? "PASS" : + (gate || nonfinite != 0u ? "FAIL" : "DIAGNOSTIC")); + return nonfinite == 0u && (failures == 0u || !gate); } void select_legacy() { @@ -581,6 +592,22 @@ void select_wmma_shape() { (void)unsetenv(kWmmaK64); } +void select_wmma_attention_a_tile8_b() { + select_tile8(false); + /* ENABLE is the production A-only request. Do not use REQUIRE here: + * REQUIRE deliberately promotes the numerically compounded output-B + * direct-WMMA path for diagnostics. */ + (void)unsetenv(kPrefillRequire); + (void)setenv(kWmmaEnable, "1", 1); + (void)unsetenv(kWmmaSsdEnable); + (void)unsetenv(kWmmaDisable); + (void)unsetenv(kWmmaRequire); + (void)unsetenv(kK1024Tile4Require); + (void)unsetenv(kWmmaRowTile); + (void)unsetenv(kWmmaK64); + ds4_rocm_test_q4_prefill_wmma_reset(); +} + double percentile(std::vector sorted, double fraction) { std::sort(sorted.begin(), sorted.end()); if (sorted.empty()) return 0.0; @@ -1508,10 +1535,11 @@ bool run_attention_output(const model_fixture &model, const config &cfg, tensor_owner tile8_out(out_bytes + guard_bytes); tensor_owner wmma_low(low_bytes + guard_bytes); tensor_owner wmma_out(out_bytes + guard_bytes); + tensor_owner replay_out(out_bytes + guard_bytes); std::vector activation; fill_activation(&activation, n_tokens, kOutputGroups * kDenseK); if (!heads.ptr || !tile8_low.ptr || !tile8_out.ptr || !wmma_low.ptr || - !wmma_out.ptr || + !wmma_out.ptr || !replay_out.ptr || !ds4_gpu_tensor_write(heads.ptr, 0, activation.data(), activation.size() * sizeof(float))) { std::fprintf(stderr, @@ -1519,13 +1547,19 @@ bool run_attention_output(const model_fixture &model, const config &cfg, return false; } - auto poison_all = [&]() { + auto poison_outputs = [&]() { return poison_output(tile8_low.ptr, low_bytes, 0x7fc10001u) && poison_output(tile8_out.ptr, out_bytes, 0x7fc20002u) && poison_output(wmma_low.ptr, low_bytes, 0x7fc30003u) && poison_output(wmma_out.ptr, out_bytes, 0x7fc40004u); }; - if (!poison_all()) return false; + auto poison_production = [&]() { + return poison_outputs() && + poison_output(replay_out.ptr, out_bytes, 0x7fc50005u); + }; + if (!poison_production()) return false; + + uint32_t production_candidate_set = 0u; const arm baseline = { "tile8_ab", []() { select_tile8(false); }, @@ -1539,8 +1573,9 @@ bool run_attention_output(const model_fixture &model, const config &cfg, heads.ptr, n_tokens) > 0; }}; const arm candidate = { - "wmma_shape_ab", select_wmma_shape, + "wmma_a_k64p80_b_tile8", select_wmma_attention_a_tile8_b, [&](uint32_t set) { + production_candidate_set = set; return ds4_gpu_attention_output_q4_K_batch_tensor( wmma_out.ptr, wmma_low.ptr, nullptr, nullptr, model.data, model.size, @@ -1550,22 +1585,84 @@ bool run_attention_output(const model_fixture &model, const config &cfg, heads.ptr, n_tokens) > 0; }}; if (!benchmark_arms( - "attention_output_ab_wmma", n_tokens, kOutputLowDim, - kDenseK + kOutputM, cfg, baseline, candidate, poison_all, + "attention_output_a_wmma_b_tile8", n_tokens, kOutputLowDim, + kDenseK + kOutputM, cfg, baseline, candidate, poison_production, [&]() { - return numerically_close(wmma_low.ptr, tile8_low.ptr, low_bytes, - "output_a grouped WMMA shape vs TILE8") && - numerically_close(wmma_out.ptr, tile8_out.ptr, out_bytes, - "output_a+b WMMA shape vs TILE8", - 16.0f, 8.0e-2f) && - check_guard(tile8_low.ptr, low_bytes, - "output_a TILE8 oracle") && - check_guard(wmma_low.ptr, low_bytes, - "output_a WMMA shape oracle") && - check_guard(tile8_out.ptr, out_bytes, - "output_b TILE8 oracle") && - check_guard(wmma_out.ptr, out_bytes, - "output_b WMMA shape oracle"); + const uint64_t wmma_calls = + ds4_rocm_test_q4_prefill_wmma_get_calls(); + const uint64_t k64_calls = + ds4_rocm_test_q4_prefill_wmma_k64_get_calls(); + bool oracle_ok = wmma_calls == 1u && k64_calls == 1u; + std::fprintf(stderr, + "output production dispatch: WMMA=%llu/1 " + "K64=%llu/1 %s\n", + (unsigned long long)wmma_calls, + (unsigned long long)k64_calls, + oracle_ok ? "PASS" : "FAIL"); + oracle_ok = numerically_close( + wmma_low.ptr, tile8_low.ptr, low_bytes, + "output_a grouped WMMA shape vs TILE8") && oracle_ok; + + /* A changes the values presented to B, so comparing the composed + * candidate against an all-TILE8 run conflates A's deliberate F16 + * boundary with B correctness. Keep the delta visible, but do + * not make it the production-path gate. */ + oracle_ok = numerically_close( + wmma_out.ptr, tile8_out.ptr, out_bytes, + "output A-WMMA/B-TILE8 vs all-TILE8 (diagnostic only)", + 16.0f, 8.0e-2f, false) && oracle_ok; + + /* Hard B oracle: replay exact TILE8 with the very same WMMA-low + * intermediate consumed by the production candidate. */ + select_tile8(false); + const bool replay_ok = + ds4_gpu_matmul_quant_tensor( + replay_out.ptr, model.data, model.size, + model.weights[production_candidate_set].output_b_offset, + kQ4Type, kOutputLowDim, kOutputM, wmma_low.ptr, + n_tokens) != 0 && + ds4_gpu_synchronize(); + if (!replay_ok) { + std::fprintf(stderr, + "output_b same-low TILE8 replay dispatch FAIL\n"); + oracle_ok = false; + } else { + const uint64_t replay_wmma_calls = + ds4_rocm_test_q4_prefill_wmma_get_calls(); + const uint64_t replay_k64_calls = + ds4_rocm_test_q4_prefill_wmma_k64_get_calls(); + if (replay_wmma_calls != wmma_calls || + replay_k64_calls != k64_calls) { + std::fprintf(stderr, + "output_b same-low TILE8 replay used WMMA: " + "WMMA=%llu/%llu K64=%llu/%llu FAIL\n", + (unsigned long long)replay_wmma_calls, + (unsigned long long)wmma_calls, + (unsigned long long)replay_k64_calls, + (unsigned long long)k64_calls); + oracle_ok = false; + } + oracle_ok = bitwise_equal( + wmma_out.ptr, replay_out.ptr, out_bytes, + "output_b production TILE8 vs same-low TILE8 replay") && + oracle_ok; + } + oracle_ok = check_guard( + tile8_low.ptr, low_bytes, "output_a TILE8 oracle") && + oracle_ok; + oracle_ok = check_guard( + wmma_low.ptr, low_bytes, "output_a WMMA shape oracle") && + oracle_ok; + oracle_ok = check_guard( + tile8_out.ptr, out_bytes, "output_b all-TILE8 oracle") && + oracle_ok; + oracle_ok = check_guard( + wmma_out.ptr, out_bytes, + "output_b production A-WMMA/B-TILE8 oracle") && oracle_ok; + oracle_ok = check_guard( + replay_out.ptr, out_bytes, + "output_b same-low TILE8 replay oracle") && oracle_ok; + return oracle_ok; })) return false; void *const heads_device = ds4_gpu_tensor_contents(heads.ptr); @@ -1622,7 +1719,7 @@ bool run_attention_output(const model_fixture &model, const config &cfg, if (!benchmark_arms( "attention_output_ab_wmma_rows", n_tokens, kOutputLowDim, kDenseK + kOutputM, cfg, geometry_baseline, geometry_candidate, - poison_all, + poison_outputs, [&]() { return bitwise_equal(tile8_low.ptr, wmma_low.ptr, low_bytes, "output_a WMMA rows64 vs shape") && @@ -1670,7 +1767,7 @@ bool run_attention_output(const model_fixture &model, const config &cfg, }}; return benchmark_arms( "attention_output_ab_wmma_k32_k64", n_tokens, kOutputLowDim, - kDenseK + kOutputM, cfg, k32_baseline, k64_candidate, poison_all, + kDenseK + kOutputM, cfg, k32_baseline, k64_candidate, poison_outputs, [&]() { return bitwise_equal(tile8_low.ptr, wmma_low.ptr, low_bytes, "output_a WMMA K32 vs K64/P80") && @@ -1706,9 +1803,11 @@ void usage(FILE *stream, const char *argv0) { "two TILE8 projections with\n" "the fused K=4096,M=(1024+512) path. qb adds TILE4/direct WMMA at the\n" "K=1024,M=32768 shape. outb isolates K=8192,M=4096; output measures\n" - "the production grouped output_a plus output_b API. WMMA comparisons\n" + "the production grouped output_a plus output_b API as all-TILE8 vs\n" + "A-WMMA/B-TILE8. Its hard B oracle replays TILE8 on the same low; the\n" + "composed all-TILE8 delta is diagnostic only. Other WMMA comparisons\n" "use a finite/toleranced oracle because their F16 boundary is not\n" - "bit-identical to the Q8_K activation path. At N>=256 the direct\n" + "bit-identical to the Q8_K activation path. At N>=256 the raw direct\n" "arms also compare rollback K32 with default K64/P80 at fixed\n" "production geometry and load2, requiring bitwise-identical output.\n", argv0, kDefaultSets, kDefaultSamples, kDefaultWarmup); diff --git a/tests/test_rocm_q4_dense_pair.cpp b/tests/test_rocm_q4_dense_pair.cpp index 78b4e50ce..e082059cc 100644 --- a/tests/test_rocm_q4_dense_pair.cpp +++ b/tests/test_rocm_q4_dense_pair.cpp @@ -42,7 +42,12 @@ extern "C" int ds4_rocm_test_q4_prefill_wmma_k64_control_policy( extern "C" int ds4_rocm_test_q4_prefill_wmma_requested_policy( int ssd_streaming, int enabled, int ssd_enabled, int disabled, int required); -extern "C" int ds4_rocm_test_q4_prefill_wmma_b_requested_policy( +extern "C" int +ds4_rocm_test_q4_prefill_wmma_attention_a_requested_policy( + int ssd_streaming, int enabled, int ssd_enabled, int disabled, + int required); +extern "C" int +ds4_rocm_test_q4_prefill_wmma_attention_b_requested_policy( int ssd_streaming, int enabled, int ssd_enabled, int disabled, int required); extern "C" int ds4_rocm_test_q4_prefill_wmma_yields_to_q8_wave32( @@ -632,13 +637,20 @@ bool close_with_tolerance(const std::vector &got, const std::vector &expected, float abs_tolerance, float rel_tolerance, - const char *label) { + const char *label, + bool gate = true) { if (got.size() != expected.size()) return false; uint64_t failures = 0; + uint64_t nonfinite = 0; float max_abs = 0.0f; float max_rel = 0.0f; size_t worst = 0; for (size_t i = 0; i < got.size(); i++) { + if (!std::isfinite(got[i]) || !std::isfinite(expected[i])) { + failures++; + nonfinite++; + continue; + } const float diff = std::fabs(got[i] - expected[i]); const float rel = diff / std::max(1.0f, std::fabs(expected[i])); if (diff > max_abs) { @@ -646,18 +658,19 @@ bool close_with_tolerance(const std::vector &got, worst = i; } max_rel = std::max(max_rel, rel); - if (!std::isfinite(got[i]) || !std::isfinite(expected[i]) || - diff > abs_tolerance + rel_tolerance * std::fabs(expected[i])) { + if (diff > abs_tolerance + rel_tolerance * std::fabs(expected[i])) { failures++; } } std::fprintf(stderr, - "%s: failures=%llu/%zu max_abs=%g max_rel=%g worst=%zu " - "tolerance(abs=%g rel=%g) %s\n", - label, (unsigned long long)failures, got.size(), max_abs, - max_rel, worst, abs_tolerance, rel_tolerance, - failures == 0u ? "PASS" : "FAIL"); - return failures == 0u; + "%s: failures=%llu/%zu nonfinite=%llu max_abs=%g " + "max_rel=%g worst=%zu tolerance(abs=%g rel=%g) %s\n", + label, (unsigned long long)failures, got.size(), + (unsigned long long)nonfinite, max_abs, max_rel, worst, + abs_tolerance, rel_tolerance, + failures == 0u ? "PASS" : + (gate || nonfinite != 0u ? "FAIL" : "DIAGNOSTIC")); + return nonfinite == 0u && (failures == 0u || !gate); } bool bitwise_equal(const std::vector &got, @@ -873,6 +886,30 @@ bool output_body_overwritten(const std::vector &values, return unchanged == 0; } +bool output_body_finite(const std::vector &values, + size_t logical_count, + const char *label) { + if (logical_count > values.size()) { + std::fprintf(stderr, "%s: invalid body geometry FAIL\n", label); + return false; + } + uint64_t nonfinite = 0u; + size_t first = logical_count; + for (size_t i = 0; i < logical_count; i++) { + if (!std::isfinite(values[i])) { + if (nonfinite == 0u) first = i; + nonfinite++; + } + } + std::fprintf(stderr, "%s: nonfinite=%llu/%zu %s\n", + label, (unsigned long long)nonfinite, logical_count, + nonfinite == 0u ? "PASS" : "FAIL"); + if (nonfinite != 0u) { + std::fprintf(stderr, " first non-finite output at float %zu\n", first); + } + return nonfinite == 0u; +} + bool run_prefill_parity_case(const aligned_model &model, uint32_t n_tokens, uint64_t offset, uint32_t out_dim, bool compare_cpu, const char *label, @@ -1244,21 +1281,28 @@ bool run_prefill_wmma_requested_policy_oracle() { int ssd_enabled; int disabled; int required; - int expected; - int b_expected; + int standalone_expected; + int attention_a_expected; + int attention_b_expected; }; const policy_case cases[] = { - {"resident unset is automatic", 0, -1, 0, 0, 0, 1, 0}, - {"resident ENABLE=0 compatibility opt-out", 0, 0, 0, 0, 0, 0, 0}, - {"resident ENABLE=1 remains accepted", 0, 1, 0, 0, 0, 1, 1}, - {"resident DISABLE opts out", 0, -1, 0, 1, 0, 0, 0}, - {"resident REQUIRE requests after ENABLE=0", 0, 0, 0, 0, 1, 1, 1}, - {"DISABLE+REQUIRE reaches strict rejection", 0, -1, 0, 1, 1, 1, 1}, - {"SSD default stays conservative", 1, -1, 0, 0, 0, 0, 0}, - {"generic ENABLE does not bypass SSD gate", 1, 1, 0, 0, 0, 0, 0}, - {"SSD gate requests the candidate", 1, -1, 1, 0, 0, 1, 1}, - {"SSD DISABLE opts out", 1, -1, 1, 1, 0, 0, 0}, - {"SSD REQUIRE requests strict validation", 1, -1, 0, 0, 1, 1, 1}, + {"resident unset keeps A automatic", 0, -1, 0, 0, 0, 1, 1, 0}, + {"resident ENABLE=0 compatibility opt-out", 0, 0, 0, 0, 0, + 0, 0, 0}, + {"resident ENABLE=1 opts attention A only", 0, 1, 0, 0, 0, + 1, 1, 0}, + {"resident DISABLE opts out", 0, -1, 0, 1, 0, 0, 0, 0}, + {"resident REQUIRE requests both attention stages", 0, 0, 0, 0, 1, + 1, 1, 1}, + {"DISABLE+REQUIRE reaches strict rejection", 0, -1, 0, 1, 1, + 1, 1, 1}, + {"SSD default stays conservative", 1, -1, 0, 0, 0, 0, 0, 0}, + {"generic ENABLE does not bypass SSD gate", 1, 1, 0, 0, 0, + 0, 0, 0}, + {"SSD gate opts attention A only", 1, -1, 1, 0, 0, 1, 1, 0}, + {"SSD DISABLE opts out", 1, -1, 1, 1, 0, 0, 0, 0}, + {"SSD REQUIRE requests both attention stages", 1, -1, 0, 0, 1, + 1, 1, 1}, }; bool ok = true; @@ -1266,22 +1310,33 @@ bool run_prefill_wmma_requested_policy_oracle() { const int got = ds4_rocm_test_q4_prefill_wmma_requested_policy( test.ssd_streaming, test.enabled, test.ssd_enabled, test.disabled, test.required); - if (got != test.expected) { + if (got != test.standalone_expected) { std::fprintf(stderr, - "Q4 WMMA request policy %s: expected=%d got=%d " - "FAIL\n", - test.label, test.expected, got); + "Q4 standalone WMMA request policy %s: " + "expected=%d got=%d FAIL\n", + test.label, test.standalone_expected, got); + ok = false; + } + const int a_got = + ds4_rocm_test_q4_prefill_wmma_attention_a_requested_policy( + test.ssd_streaming, test.enabled, test.ssd_enabled, + test.disabled, test.required); + if (a_got != test.attention_a_expected) { + std::fprintf(stderr, + "Q4 attention-output A WMMA policy %s: " + "expected=%d got=%d FAIL\n", + test.label, test.attention_a_expected, a_got); ok = false; } const int b_got = - ds4_rocm_test_q4_prefill_wmma_b_requested_policy( + ds4_rocm_test_q4_prefill_wmma_attention_b_requested_policy( test.ssd_streaming, test.enabled, test.ssd_enabled, test.disabled, test.required); - if (b_got != test.b_expected) { + if (b_got != test.attention_b_expected) { std::fprintf(stderr, "Q4 attention-output B WMMA policy %s: " "expected=%d got=%d FAIL\n", - test.label, test.b_expected, b_got); + test.label, test.attention_b_expected, b_got); ok = false; } } @@ -2987,15 +3042,14 @@ bool run_attention_output_wmma_smoke(const aligned_model &model) { tensor_owner heads_gpu(heads_count * sizeof(float)); tensor_owner tile8_low(low_sentinel.size() * sizeof(float)); tensor_owner tile8_out(out_sentinel.size() * sizeof(float)); - tensor_owner wmma_low(low_sentinel.size() * sizeof(float)); - tensor_owner wmma_out(out_sentinel.size() * sizeof(float)); + tensor_owner candidate_low(low_sentinel.size() * sizeof(float)); + tensor_owner candidate_out(out_sentinel.size() * sizeof(float)); + tensor_owner replay_out(out_sentinel.size() * sizeof(float)); if (!heads_gpu.ptr || !tile8_low.ptr || !tile8_out.ptr || - !wmma_low.ptr || !wmma_out.ptr || + !candidate_low.ptr || !candidate_out.ptr || !replay_out.ptr || !write_tensor(heads_gpu.ptr, heads_host) || !write_tensor(tile8_low.ptr, low_sentinel) || - !write_tensor(tile8_out.ptr, out_sentinel) || - !write_tensor(wmma_low.ptr, low_sentinel) || - !write_tensor(wmma_out.ptr, out_sentinel)) { + !write_tensor(tile8_out.ptr, out_sentinel)) { std::fprintf(stderr, "ROCm Q4 production output direct-WMMA: setup FAIL\n"); return false; @@ -3011,157 +3065,384 @@ bool run_attention_output_wmma_smoke(const aligned_model &model) { env_snapshot wmma_require(kPrefillWmmaRequire); env_snapshot wmma_row_tile(kPrefillWmmaRowTile); env_snapshot wmma_k64(kPrefillWmmaK64); + env_snapshot q8_wave32_enable(kPrefillQ8Wave32Enable); + env_snapshot q8_wave32_disable(kPrefillQ8Wave32Disable); + env_snapshot q8_wave32_require(kPrefillQ8Wave32Require); + + struct batch_result { + int rc = 0; + uint64_t wmma_calls = 0u; + uint64_t k64_calls = 0u; + bool write_ok = false; + bool low_read = false; + bool out_read = false; + }; + + struct replay_result { + int rc = 0; + uint64_t wmma_calls = 0u; + uint64_t k64_calls = 0u; + bool write_ok = false; + bool out_read = false; + }; + + auto clear_controls = [&]() { + (void)unsetenv(kPrefillEnable); + (void)unsetenv(kPrefillDisable); + (void)unsetenv(kPrefillRequire); + (void)unsetenv(kPrefillK1024Tile4Require); + (void)unsetenv(kPrefillWmmaEnable); + (void)unsetenv(kPrefillWmmaSsdEnable); + (void)unsetenv(kPrefillWmmaDisable); + (void)unsetenv(kPrefillWmmaRequire); + (void)unsetenv(kPrefillWmmaRowTile); + (void)unsetenv(kPrefillWmmaK64); + (void)unsetenv(kPrefillQ8Wave32Enable); + (void)unsetenv(kPrefillQ8Wave32Disable); + (void)unsetenv(kPrefillQ8Wave32Require); + }; + + auto run_batch = [&](ds4_gpu_tensor *low, ds4_gpu_tensor *out, + std::vector *low_host, + std::vector *out_host) { + batch_result result; + ds4_rocm_test_q4_prefill_wmma_reset(); + result.write_ok = write_tensor(low, low_sentinel) && + write_tensor(out, out_sentinel); + if (result.write_ok) { + result.rc = ds4_gpu_attention_output_q4_K_batch_tensor( + out, low, nullptr, nullptr, + model.data, model.size, model.decode_attn_a_offset, + model.decode_attn_b_offset, kQ4Type, kDecodeAttnGroupDim, + kDecodeAttnRank, kDecodeAttnGroups, kDecodeAttnOutDim, + heads_gpu.ptr, n_tokens); + } + result.wmma_calls = ds4_rocm_test_q4_prefill_wmma_get_calls(); + result.k64_calls = ds4_rocm_test_q4_prefill_wmma_k64_get_calls(); + if (result.write_ok) { + result.low_read = read_tensor(low, low_host); + result.out_read = read_tensor(out, out_host); + } + return result; + }; + + auto batch_ready = [&](const batch_result &result, + uint64_t expected_wmma, + uint64_t expected_k64, + const char *label) { + const bool ready = result.write_ok && result.rc == 1 && + result.wmma_calls == expected_wmma && + result.k64_calls == expected_k64 && + result.low_read && result.out_read; + std::fprintf(stderr, + "%s: rc=%d WMMA=%llu/%llu K64=%llu/%llu " + "write=%d read=%d/%d %s\n", + label, result.rc, + (unsigned long long)result.wmma_calls, + (unsigned long long)expected_wmma, + (unsigned long long)result.k64_calls, + (unsigned long long)expected_k64, + result.write_ok ? 1 : 0, + result.low_read ? 1 : 0, + result.out_read ? 1 : 0, + ready ? "PASS" : "FAIL"); + return ready; + }; + + auto batch_memory_ok = [&](const batch_result &result, + const std::vector &low_host, + const std::vector &out_host, + const char *label) { + if (!result.low_read || !result.out_read) return false; + const std::string low_body = std::string(label) + " low body"; + const std::string out_body = std::string(label) + " out body"; + const std::string low_finite = std::string(label) + " low finite"; + const std::string out_finite = std::string(label) + " out finite"; + const std::string low_guard = std::string(label) + " low N-tail"; + const std::string out_guard = std::string(label) + " out N-tail"; + bool memory_ok = output_body_overwritten( + low_host, low_sentinel, low_count, low_body.c_str()); + memory_ok = output_body_overwritten( + out_host, out_sentinel, out_count, out_body.c_str()) && memory_ok; + memory_ok = output_body_finite( + low_host, low_count, low_finite.c_str()) && memory_ok; + memory_ok = output_body_finite( + out_host, out_count, out_finite.c_str()) && memory_ok; + memory_ok = output_guard_unchanged( + low_host, low_sentinel, low_count, low_guard.c_str()) && memory_ok; + memory_ok = output_guard_unchanged( + out_host, out_sentinel, out_count, out_guard.c_str()) && memory_ok; + return memory_ok; + }; + + auto run_tile8_replay = [&](const ds4_gpu_tensor *low, + std::vector *out_host) { + replay_result result; + clear_controls(); + (void)setenv(kPrefillRequire, "1", 1); + (void)setenv(kPrefillWmmaDisable, "1", 1); + ds4_rocm_test_q4_prefill_wmma_reset(); + result.write_ok = write_tensor(replay_out.ptr, out_sentinel); + if (result.write_ok) { + result.rc = ds4_gpu_matmul_quant_tensor( + replay_out.ptr, model.data, model.size, + model.decode_attn_b_offset, kQ4Type, + kDecodeAttnLowDim, kDecodeAttnOutDim, low, n_tokens); + } + result.wmma_calls = ds4_rocm_test_q4_prefill_wmma_get_calls(); + result.k64_calls = ds4_rocm_test_q4_prefill_wmma_k64_get_calls(); + if (result.write_ok) { + result.out_read = read_tensor(replay_out.ptr, out_host); + } + return result; + }; + + auto replay_ready = [&](const replay_result &result, + const std::vector &out_host, + const char *label) { + bool ready = result.write_ok && result.rc != 0 && + result.wmma_calls == 0u && result.k64_calls == 0u && + result.out_read; + std::fprintf(stderr, + "%s: rc=%d WMMA=%llu K64=%llu write=%d read=%d %s\n", + label, result.rc, + (unsigned long long)result.wmma_calls, + (unsigned long long)result.k64_calls, + result.write_ok ? 1 : 0, + result.out_read ? 1 : 0, + ready ? "PASS" : "FAIL"); + if (result.out_read) { + const std::string body = std::string(label) + " body"; + const std::string finite = std::string(label) + " finite"; + const std::string guard = std::string(label) + " N-tail"; + ready = output_body_overwritten( + out_host, out_sentinel, out_count, body.c_str()) && ready; + ready = output_body_finite( + out_host, out_count, finite.c_str()) && ready; + ready = output_guard_unchanged( + out_host, out_sentinel, out_count, guard.c_str()) && ready; + } + return ready; + }; ds4_gpu_set_ssd_streaming(false); - (void)unsetenv(kPrefillEnable); - (void)unsetenv(kPrefillDisable); + + /* Forced TILE8 is the exact baseline. */ + clear_controls(); (void)setenv(kPrefillRequire, "1", 1); - (void)unsetenv(kPrefillK1024Tile4Require); - (void)unsetenv(kPrefillWmmaEnable); - (void)unsetenv(kPrefillWmmaSsdEnable); (void)setenv(kPrefillWmmaDisable, "1", 1); - (void)unsetenv(kPrefillWmmaRequire); - (void)unsetenv(kPrefillWmmaRowTile); + std::vector tile8_low_host(low_sentinel.size()); + std::vector tile8_out_host(out_sentinel.size()); + const batch_result tile8_result = run_batch( + tile8_low.ptr, tile8_out.ptr, &tile8_low_host, &tile8_out_host); + + /* The production default keeps validated WMMA on A and exact TILE8 on B. */ + clear_controls(); + std::vector default_low_host(low_sentinel.size()); + std::vector default_out_host(out_sentinel.size()); + const batch_result default_result = run_batch( + candidate_low.ptr, candidate_out.ptr, + &default_low_host, &default_out_host); + + /* ENABLE opts only A into direct WMMA. Pin K32 as the rollback arm. */ + clear_controls(); + (void)setenv(kPrefillWmmaEnable, "1", 1); (void)setenv(kPrefillWmmaK64, "0", 1); - ds4_rocm_test_q4_prefill_wmma_reset(); - const int tile8_rc = ds4_gpu_attention_output_q4_K_batch_tensor( - tile8_out.ptr, tile8_low.ptr, nullptr, nullptr, - model.data, model.size, model.decode_attn_a_offset, - model.decode_attn_b_offset, kQ4Type, kDecodeAttnGroupDim, - kDecodeAttnRank, kDecodeAttnGroups, kDecodeAttnOutDim, - heads_gpu.ptr, n_tokens); - const uint64_t tile8_wmma_calls = - ds4_rocm_test_q4_prefill_wmma_get_calls(); - const uint64_t tile8_k64_calls = - ds4_rocm_test_q4_prefill_wmma_k64_get_calls(); - - (void)unsetenv(kPrefillRequire); - (void)unsetenv(kPrefillWmmaEnable); - (void)unsetenv(kPrefillWmmaDisable); - (void)unsetenv(kPrefillWmmaRequire); + std::vector k32_low_host(low_sentinel.size()); + std::vector k32_out_host(out_sentinel.size()); + const batch_result k32_result = run_batch( + candidate_low.ptr, candidate_out.ptr, &k32_low_host, &k32_out_host); + + std::vector k32_replay_host(out_sentinel.size()); + replay_result k32_replay_result; + if (k32_result.rc == 1 && k32_result.low_read) { + k32_replay_result = run_tile8_replay( + candidate_low.ptr, &k32_replay_host); + } + + /* Run K64 regardless of every earlier comparison result. */ + clear_controls(); + (void)setenv(kPrefillWmmaEnable, "1", 1); + std::vector k64_low_host(low_sentinel.size()); + std::vector k64_out_host(out_sentinel.size()); + const batch_result k64_result = run_batch( + candidate_low.ptr, candidate_out.ptr, &k64_low_host, &k64_out_host); + + /* REQUIRE is the only policy that opts both A and B into direct WMMA. */ + clear_controls(); + (void)setenv(kPrefillWmmaRequire, "1", 1); (void)setenv(kPrefillWmmaK64, "0", 1); - ds4_rocm_test_q4_prefill_wmma_reset(); - const int wmma_rc = ds4_gpu_attention_output_q4_K_batch_tensor( - wmma_out.ptr, wmma_low.ptr, nullptr, nullptr, - model.data, model.size, model.decode_attn_a_offset, - model.decode_attn_b_offset, kQ4Type, kDecodeAttnGroupDim, - kDecodeAttnRank, kDecodeAttnGroups, kDecodeAttnOutDim, - heads_gpu.ptr, n_tokens); - const uint64_t wmma_calls = - ds4_rocm_test_q4_prefill_wmma_get_calls(); - const uint64_t wmma_k64_calls = - ds4_rocm_test_q4_prefill_wmma_k64_get_calls(); + std::vector strict_k32_low_host(low_sentinel.size()); + std::vector strict_k32_out_host(out_sentinel.size()); + const batch_result strict_k32_result = run_batch( + candidate_low.ptr, candidate_out.ptr, + &strict_k32_low_host, &strict_k32_out_host); - std::vector tile8_low_host(low_sentinel.size()); - std::vector tile8_out_host(out_sentinel.size()); - std::vector wmma_low_host(low_sentinel.size()); - std::vector wmma_out_host(out_sentinel.size()); - bool ok = tile8_rc == 1 && wmma_rc == 1 && - tile8_wmma_calls == 0u && tile8_k64_calls == 0u && - wmma_calls == 1u && - wmma_k64_calls == 0u && - read_tensor(tile8_low.ptr, &tile8_low_host) && - read_tensor(tile8_out.ptr, &tile8_out_host) && - read_tensor(wmma_low.ptr, &wmma_low_host) && - read_tensor(wmma_out.ptr, &wmma_out_host); - if (ok) { - ok = output_body_overwritten( - tile8_low_host, low_sentinel, low_count, - "production output-A TILE8 body") && ok; - ok = output_body_overwritten( - tile8_out_host, out_sentinel, out_count, - "production output-B TILE8 body") && ok; - ok = output_body_overwritten( - wmma_low_host, low_sentinel, low_count, - "production output-A direct-WMMA body") && ok; - ok = output_body_overwritten( - wmma_out_host, out_sentinel, out_count, - "production output-B A-WMMA/B-TILE8 body") && ok; - ok = output_guard_unchanged( - tile8_low_host, low_sentinel, low_count, - "production output-A TILE8 N-tail") && ok; - ok = output_guard_unchanged( - tile8_out_host, out_sentinel, out_count, - "production output-B TILE8 N-tail") && ok; - ok = output_guard_unchanged( - wmma_low_host, low_sentinel, low_count, - "production output-A direct-WMMA N-tail") && ok; - ok = output_guard_unchanged( - wmma_out_host, out_sentinel, out_count, - "production output-B A-WMMA/B-TILE8 N-tail") && ok; - tile8_low_host.resize(low_count); - tile8_out_host.resize(out_count); - wmma_low_host.resize(low_count); - wmma_out_host.resize(out_count); - ok = close_with_tolerance( - wmma_low_host, tile8_low_host, 2.0f, 3.0e-2f, - "production output-A direct-WMMA vs TILE8") && ok; - ok = close_with_tolerance( - wmma_out_host, tile8_out_host, 16.0f, 8.0e-2f, - "production output A-WMMA/B-TILE8 vs all-TILE8") && ok; - } - - int k64_default_rc = 0; - uint64_t k64_default_wmma_calls = 0u; - uint64_t k64_default_calls = 0u; - std::vector k64_default_low_host(low_sentinel.size()); - std::vector k64_default_out_host(out_sentinel.size()); - if (ok) { - ok = write_tensor(wmma_low.ptr, low_sentinel) && - write_tensor(wmma_out.ptr, out_sentinel); + clear_controls(); + (void)setenv(kPrefillWmmaRequire, "1", 1); + std::vector strict_k64_low_host(low_sentinel.size()); + std::vector strict_k64_out_host(out_sentinel.size()); + const batch_result strict_k64_result = run_batch( + candidate_low.ptr, candidate_out.ptr, + &strict_k64_low_host, &strict_k64_out_host); + + const bool tile8_ready = batch_ready( + tile8_result, 0u, 0u, "production forced TILE8"); + const bool default_ready = batch_ready( + default_result, 1u, 1u, + "production clean-default A-K64/B-TILE8"); + const bool k32_ready = batch_ready( + k32_result, 1u, 0u, "production ENABLE A-K32/B-TILE8"); + const bool k64_ready = batch_ready( + k64_result, 1u, 1u, "production ENABLE A-K64/B-TILE8"); + const bool strict_k32_ready = batch_ready( + strict_k32_result, 2u, 0u, "production REQUIRE A+B K32"); + const bool strict_k64_ready = batch_ready( + strict_k64_result, 2u, 2u, "production REQUIRE A+B K64"); + + const bool tile8_memory = batch_memory_ok( + tile8_result, tile8_low_host, tile8_out_host, + "production forced TILE8"); + const bool default_memory = batch_memory_ok( + default_result, default_low_host, default_out_host, + "production clean-default A-K64/B-TILE8"); + const bool k32_memory = batch_memory_ok( + k32_result, k32_low_host, k32_out_host, + "production ENABLE A-K32/B-TILE8"); + const bool k64_memory = batch_memory_ok( + k64_result, k64_low_host, k64_out_host, + "production ENABLE A-K64/B-TILE8"); + const bool strict_k32_memory = batch_memory_ok( + strict_k32_result, strict_k32_low_host, strict_k32_out_host, + "production REQUIRE A+B K32"); + const bool strict_k64_memory = batch_memory_ok( + strict_k64_result, strict_k64_low_host, strict_k64_out_host, + "production REQUIRE A+B K64"); + const bool k32_replay_ready = replay_ready( + k32_replay_result, k32_replay_host, + "production K32-low standalone B-TILE8 replay"); + + tile8_low_host.resize(low_count); + tile8_out_host.resize(out_count); + default_low_host.resize(low_count); + default_out_host.resize(out_count); + k32_low_host.resize(low_count); + k32_out_host.resize(out_count); + k32_replay_host.resize(out_count); + k64_low_host.resize(low_count); + k64_out_host.resize(out_count); + strict_k32_low_host.resize(low_count); + strict_k32_out_host.resize(out_count); + strict_k64_low_host.resize(low_count); + strict_k64_out_host.resize(out_count); + + bool comparisons_ok = true; + if (tile8_result.low_read && default_result.low_read) { + comparisons_ok = close_with_tolerance( + default_low_host, tile8_low_host, + 2.0f, 3.0e-2f, + "production clean-default A-WMMA vs forced TILE8") && + comparisons_ok; + } else { + comparisons_ok = false; + } + if (tile8_result.out_read && default_result.out_read) { + comparisons_ok = close_with_tolerance( + default_out_host, tile8_out_host, + 16.0f, 8.0e-2f, + "production clean-default A-WMMA/B-TILE8 vs all-TILE8", + false) && comparisons_ok; + } else { + comparisons_ok = false; } - if (ok) { - /* Attest the production K64 default, not its explicit opt-in alias. */ - (void)unsetenv(kPrefillWmmaK64); - ds4_rocm_test_q4_prefill_wmma_reset(); - k64_default_rc = ds4_gpu_attention_output_q4_K_batch_tensor( - wmma_out.ptr, wmma_low.ptr, nullptr, nullptr, - model.data, model.size, model.decode_attn_a_offset, - model.decode_attn_b_offset, kQ4Type, kDecodeAttnGroupDim, - kDecodeAttnRank, kDecodeAttnGroups, kDecodeAttnOutDim, - heads_gpu.ptr, n_tokens); - k64_default_wmma_calls = - ds4_rocm_test_q4_prefill_wmma_get_calls(); - k64_default_calls = ds4_rocm_test_q4_prefill_wmma_k64_get_calls(); - ok = k64_default_rc == 1 && k64_default_wmma_calls == 1u && - k64_default_calls == 1u && - read_tensor(wmma_low.ptr, &k64_default_low_host) && - read_tensor(wmma_out.ptr, &k64_default_out_host); + if (tile8_result.low_read && k32_result.low_read) { + comparisons_ok = close_with_tolerance( + k32_low_host, tile8_low_host, 2.0f, 3.0e-2f, + "production ENABLE output-A K32 vs TILE8") && comparisons_ok; + } else { + comparisons_ok = false; } - if (ok) { - ok = output_body_overwritten( - k64_default_low_host, low_sentinel, low_count, - "production output-A default K64 direct-WMMA body") && ok; - ok = output_body_overwritten( - k64_default_out_host, out_sentinel, out_count, - "production output-B A-default-K64/B-TILE8 body") && ok; - ok = output_guard_unchanged( - k64_default_low_host, low_sentinel, low_count, - "production output-A default K64 direct-WMMA N-tail") && ok; - ok = output_guard_unchanged( - k64_default_out_host, out_sentinel, out_count, - "production output-B A-default-K64/B-TILE8 N-tail") && ok; - k64_default_low_host.resize(low_count); - k64_default_out_host.resize(out_count); - ok = bitwise_equal( - k64_default_low_host, wmma_low_host, - "production output-A default K64 vs K32") && ok; - ok = bitwise_equal( - k64_default_out_host, wmma_out_host, - "production output A-default-K64/B-TILE8 vs " - "A-K32/B-TILE8") && ok; + if (k32_result.out_read && k32_replay_result.out_read) { + comparisons_ok = bitwise_equal( + k32_out_host, k32_replay_host, + "production ENABLE K32 B vs standalone TILE8 replay") && + comparisons_ok; + } else { + comparisons_ok = false; + } + if (k32_result.low_read && k64_result.low_read) { + comparisons_ok = bitwise_equal( + k64_low_host, k32_low_host, + "production ENABLE output-A K64 vs K32") && comparisons_ok; + } else { + comparisons_ok = false; + } + if (k32_result.out_read && k64_result.out_read) { + comparisons_ok = bitwise_equal( + k64_out_host, k32_out_host, + "production ENABLE A-K64/B-TILE8 vs A-K32/B-TILE8") && + comparisons_ok; + } else { + comparisons_ok = false; } + if (default_result.low_read && k64_result.low_read) { + comparisons_ok = bitwise_equal( + default_low_host, k64_low_host, + "production clean-default A vs explicit ENABLE A-K64") && + comparisons_ok; + } else { + comparisons_ok = false; + } + if (default_result.out_read && k64_result.out_read) { + comparisons_ok = bitwise_equal( + default_out_host, k64_out_host, + "production clean-default B vs explicit ENABLE B-TILE8") && + comparisons_ok; + } else { + comparisons_ok = false; + } + if (strict_k32_result.low_read && strict_k64_result.low_read) { + comparisons_ok = bitwise_equal( + strict_k64_low_host, strict_k32_low_host, + "production REQUIRE output-A K64 vs K32") && comparisons_ok; + } else { + comparisons_ok = false; + } + if (strict_k32_result.out_read && strict_k64_result.out_read) { + comparisons_ok = bitwise_equal( + strict_k64_out_host, strict_k32_out_host, + "production REQUIRE output A+B K64 vs K32") && comparisons_ok; + } else { + comparisons_ok = false; + } + + const bool ok = tile8_ready && default_ready && k32_ready && k64_ready && + strict_k32_ready && strict_k64_ready && tile8_memory && + default_memory && k32_memory && k64_memory && strict_k32_memory && + strict_k64_memory && k32_replay_ready && comparisons_ok; std::fprintf(stderr, - "ROCm Q4 production output WMMA safety default: " - "tile8=%d/%llu/%llu A-K32/B-TILE8=%d/%llu/%llu " - "A-K64-default/B-TILE8=%d/%llu/%llu %s\n", - tile8_rc, (unsigned long long)tile8_wmma_calls, - (unsigned long long)tile8_k64_calls, - wmma_rc, (unsigned long long)wmma_calls, - (unsigned long long)wmma_k64_calls, - k64_default_rc, - (unsigned long long)k64_default_wmma_calls, - (unsigned long long)k64_default_calls, + "ROCm Q4 production output WMMA policy: " + "tile8=%d/%llu/%llu default=%d/%llu/%llu " + "enable-k32=%d/%llu/%llu enable-k64=%d/%llu/%llu " + "require-k32=%d/%llu/%llu require-k64=%d/%llu/%llu %s\n", + tile8_result.rc, + (unsigned long long)tile8_result.wmma_calls, + (unsigned long long)tile8_result.k64_calls, + default_result.rc, + (unsigned long long)default_result.wmma_calls, + (unsigned long long)default_result.k64_calls, + k32_result.rc, + (unsigned long long)k32_result.wmma_calls, + (unsigned long long)k32_result.k64_calls, + k64_result.rc, + (unsigned long long)k64_result.wmma_calls, + (unsigned long long)k64_result.k64_calls, + strict_k32_result.rc, + (unsigned long long)strict_k32_result.wmma_calls, + (unsigned long long)strict_k32_result.k64_calls, + strict_k64_result.rc, + (unsigned long long)strict_k64_result.wmma_calls, + (unsigned long long)strict_k64_result.k64_calls, ok ? "PASS" : "FAIL"); return ok; } From 9f70fd5e8f438c26b3a30bcdad41827437612226 Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Wed, 2 Sep 2026 07:13:30 +0200 Subject: [PATCH 183/189] Fix ROCm 10 DeepSeek Vision build --- rocm/ds4_rocm_deepseek4_vision.cuh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rocm/ds4_rocm_deepseek4_vision.cuh b/rocm/ds4_rocm_deepseek4_vision.cuh index aeb521774..641d48d9c 100644 --- a/rocm/ds4_rocm_deepseek4_vision.cuh +++ b/rocm/ds4_rocm_deepseek4_vision.cuh @@ -236,7 +236,7 @@ extern "C" int ds4_gpu_attention_visual_mixed_batch_heads_tensor( if (!cuda_ok(cudaGetLastError(), "visual attention KV pack launch")) return 0; - const float alpha = rsqrtf((float)head_dim); + const float alpha = 1.0f / sqrtf((float)head_dim); const float beta = 0.0f; cublasStatus_t status = cublasSgemmStridedBatched( g_cublas, CUBLAS_OP_T, CUBLAS_OP_N, From 9a448bb1d52efedac1cba670b8bc36876ace8a6b Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Wed, 2 Sep 2026 07:16:07 +0200 Subject: [PATCH 184/189] Add Metal benchmark and test binaries --- speed-bench/metal_q4_attn_out_a_direct_bench | Bin 0 -> 74048 bytes tests/test_metal_argmax_top1 | Bin 0 -> 901592 bytes tests/test_metal_dspark_capture | Bin 0 -> 901160 bytes tests/test_metal_iq2_live_index | Bin 0 -> 901624 bytes tests/test_metal_iq2_midonly | Bin 0 -> 901048 bytes tests/test_metal_iq2_ssd_grouped_mm | Bin 0 -> 919136 bytes tests/test_metal_q4_attn_exactn | Bin 0 -> 927576 bytes tests/test_metal_q4_attn_out_a_direct | Bin 0 -> 924032 bytes 8 files changed, 0 insertions(+), 0 deletions(-) create mode 100755 speed-bench/metal_q4_attn_out_a_direct_bench create mode 100755 tests/test_metal_argmax_top1 create mode 100755 tests/test_metal_dspark_capture create mode 100755 tests/test_metal_iq2_live_index create mode 100755 tests/test_metal_iq2_midonly create mode 100755 tests/test_metal_iq2_ssd_grouped_mm create mode 100755 tests/test_metal_q4_attn_exactn create mode 100755 tests/test_metal_q4_attn_out_a_direct diff --git a/speed-bench/metal_q4_attn_out_a_direct_bench b/speed-bench/metal_q4_attn_out_a_direct_bench new file mode 100755 index 0000000000000000000000000000000000000000..21d3e78e1e8987a7b0d43998f69030469f055baf GIT binary patch literal 74048 zcmeHw3w%`7wfCNxgiOLKK!6aCoDk52S0Drk0c7&xArL|me7D17<|G-IN0%cWNoFQMu=jrVcfa3v z&d<%->$&${d+p~r8Q=Q)vu}Gbrek>YNQ00}M#k=95^{_UL7IzXHkTL9DZR0@Vi8fJ zAFdot>VlD-eJH?et}4C0Dw@sp1JN`l@OY3Zp-TKQn|<UiQ*%L>J8_WJzQ-VS!Gz1>q4dpCh*c-u3Ew9Vy{rKw=-E0C0ef%*ef*%E5IY4 zuJ-({Rd$!nYYLlVPdLUgv8S$VWV&Wvaex|ib{k95u!Xxde zc2o#^HIW?~+uml?o~R$;m$t{ph23eXm8mOY?FH5=^;5?=;obGa1E{QGN3AtrQ%pYq z=Z{GNR7Xmp+R#_VtL*T|HJAo2W^aP3&=lQHj9n4gihYWYxkzU7f_Y{0OH1Y{(E*B1 zDbs{3e-wkJj804~f+%^61r)Se9?H+%U-Q$lipG!RWC07u>p*Y)#WI{IScI79*d_TXSN5k%qq8Mxi@FF z+wZbje0H~Me2q%YxmNagLGk7kNAO{n>|Y;YpXvQ*D(ZD{`wL~C#c?6ig*luBYl_|I z_8tQ@y1f*!;|n8OtLW81GP=D#yHtIO-5&YUZ7$4-mVx~%k*MFkKpdjacuWyaaYzr% zNrcxA*RcaihY|U*pUtx8v*D_#NlWJ$9VEE@8)+MaCU% zYZ7;~73+hE6V_FfBV>|$n8Hjy>)rMQ)7$Na9&ev9=t5r_SZFSV(-2eGs56DPuOE2s zXiv7e`7IXK9b@j6gyhD#EKt!pM!#nR3mi1+nohM2(1j6xUl?1a2br#EW^+Q{##fTq ziW37^)dM3~a1!LVAPr*c?)z$eLJ)Is%K6NqJ;Rz<-lPQ9?+*RI;R zB4oE1g9kv9dPUG)GY0pAW_p=TZs@`K%`yyJzU*aIHFS;lT>H@Y@U7sbWB9l1%?dUf zhK7oCfr>wXb{hWi0~(oWk8wAfT!;GZdy(a}4lNCDc#-Xb&!)|Nkxjni2kfN&MOJkh zetNyVeXv?a=r^n?jmpT>@1eG~LYK;08dY8zFK-8E=TY7bqPz`L*yL8U{~MHbhrxJ) zO<`3V41+eN$LBx0Z8EE(x}-nLR>Tc4-I4+PIDDL_XL+*<*@~@stm+f^{)8bhw4l_q z`w`u+RRr)>4QEwyqt93sMQ`}+VOKH8TXb<7(@yug+hE*}d zaHXM>5eD?h2-CCy4Qc(G4Fh>w4(ZusvQP2YyqASr z_nN}3e`R3_G0on~44y|_nox%$CZ>^PoTjJd1`b6TPE^XYFmnhfT5xgRX1 zF!cqs?Jbl+{V_`5`+<{Rs82BeaGx;Q6fQy?ida(cgphj`yyn+g_{=LToPzk~cHv*% zc3pf>Z}**R6|%pE>^I!z;Ls7o@f(7^6ZA0XsxFP;W0)(B7>0z_!$O zIHm8)A;bIn?@aWsu|qBsa=#aHvmr1?2IQzen~$>aR~X;#AnieF zHVh8^8N76foy?3=V%LuPrU%%`M=`#sZ)AaIaC~5h;wzo$Pj1sI{Feou$?<`qY{)Bp znG$*$yt*V)FkQz^Wa$)nnh#Yz#gEFLm%ysBXr0A4r}2r+!58e<(4eKKGCn^^WSg;% zPNDv7N4^<-(FTZdoTBOt;Bl1}+5o#WhQ0-#{1Ya5#MTJVX^zmrcE+tk_#E-J;F0~o z)aU4nk2!5n=pzo(LPt4F4Yd$NxgTR(IwjmE;jZ@aa z&lzHHXy_SkXHck-!?e(^I7|)g0Oa%HSbgvXqM=<2g)d2l>h%Pd9T0kpL#!#(Pp0^w zN`Cm!f2x#1HoPp2kS!u(tfNFs_f1lK}^(}U-Drc+?yXX z_KOQ*4-q!De}0hZY9C$-o^jaLG8&tt*H>p9-fA$8K5iH``uJ~QYajE=*{>R8yD*2u(=CpBI5VS=Kt+!v$ehS)r zh1RQKPTMfp`OTm2u6RFxtEUxvjzJBpcUPcZH0Sbl_pj?`HXlNI3u!;n>qvW%UP1aT z(u+vXBRzxkE2Jlpwjn)&^bpd|k?u#jXZ_5xH}&1K@Algc{pt2Khelta3tqE!&3(r2 z#RZLRrHAJCHf~&=!GgVvfr?Ly6T-*mB!oYipAbH=C?VWhnGk;G_7#Wze!Kh7H&YYB zUrq;qRzjHilja&8dx4~2o`|_T)W3D;oN)6{$QX>FtAMvm2M&29PJ0Hkg={v-rh}KR z$PSB@4fG5S7qY`iHWRX$ifq4FS=K8!K*$at*{dLXl_J}NWM=_a>k#M_#Jw6H*G7^Z z1Kt=#_8TK)7e&aj-oZA5B8!gQ+?oSkjw1U-tZbln@RX1}1s)%Jljej^Pf7@XM6y5M zJM+l>d(j_zXIk~O4;!vXEh~DUxhxK8UFy0rD_{nt)b(YBsfA@3Ss8s=()*TWv6_dC zYv+a%+sY3ab%~>m8FNSP8_jT2So82ZuuDF+y~f;Q5`Iw8tNh7|A7dU_iG4kf!-l;) zMs>FV(s-Vc(kJ`2w4q7t%Hh`l7GW%mKs~YreYBt#gPsX`W=exwk5om65Cbmz9R zqPPie{nP|^(VdT%6}^{Hww~x8WV-b6YiB&x+gQf#JW`f{^g@5k73bTln&IDHp`PTw zcMUA>!yatKTAWpk7<;%+IM5gK?!H^@xG%BCI)s1X#!nD8L|?jg@65G7+&fdc>*_}7 zu33%QI>c=vJ6VW*c)D)TiKSQv{-ihVH>MajCh6Bz7qxnEVmS z{{X3V>YQ-woY#uIMk4$(CStBLs&m~o^f%gY3v}NkeZZOsYu8e1^7Y2H1)CL#RU9iU;%&tt$hCWODa z2evZ%?m2r^-#wrHJfW?1J7dTBHNSAnhqxFFgIguxf^w*wQCU|?Hhh7 z@cDqWmK{ZGwqQ*-g1VV{(0VX1vQq&pBRdl3^1eNcV&_WMx&febc1Xe3#Tdy-8<-f=P6rh;Hw z0%S4nNwzQKEEkgv6a-I^EZW|JwjafQRF5%pH`m%W>V))<~*3Kfd9r4rA z_K%=X{-pgS@e7C#AE*IJc8#hBQ$n|pRtBr(9eG(8R6%5N#+86KA_6L&wq}T ziSYC5sto-68p-@O`Z@8EeopP;=f_cgVwU}aey)a}6EU7}PoF~L@A#PV9W36|sPA3a z7&oxIvBwpQ8`=UeqKQ3b**jyP%MMzd8DSlSt zh~Dd-GSXU~fwN;9?%cMO?>Nq5!gyjS&W5ds3G9RRk3MLX%oHu4u5%Y3aCF*cKAE)Zi= z$qvaN#_}R%B4TX3DuWmsLo)wkF=p%%V?9xRY>XM%zJsm(T2Ik=t!1jIEyK8Uc*`2r z*0QnGS%$oX)a+$#&FQ8#X~@#yCotzH;!M}O2K%C% zte&m>44BpP+(8{|A01HQTo-3bw<1aLru5?my|Z}?bo=U@$9gg+*KL{lxN!@b=O=g!4O{m3+uwMRCH+`o@9=&_hxaQiNHUC>!ViOvH7~q{V3H~PAi+MS@B;+bH44gDqFF9WB^@)xc)knvxx`6Qn!S_G10p=iLU}tH$AeH6Hh@Jy2Ex>WcS4 z20iw72E6mphv?l8)tlN#cXJ*N2ZrhaS0>=D{}nckVuAThVHq^K(>(GD3vR>R@SWEM zDsDy$wGUw7W9u2euQ`H!@lo7E(O9H=zpn<+9pd27&D?Hk=sL0+&vt#)mxYg7NeB1n zxC`tLd9oeuOY%cPd=CUY>eKi1>;)c=xc?fAIHkQe(fvqt4@&YByL2{~3rJ^!VnCM{4Ab=BV>s{pg|{^&^iM#HtrR*d5LEkd5clVR(BHu4ZMaUf-8K>B4hlUd z^!IZ(AoM4Jx2-8g8;a1*!X0Q+@MW~E%NR^Ch4(^!AnLR%4SU!^{mJ4)WuHuE;u9;j z9PIh}v!|91d;Qe@73WV~bKCxhu4$Wd=*04eEB62RNzYTSZmJk-*y`zxGk9;-d*gwT z8!IMcJ>luoKTy#Z?MNQqYiK`qS4F=OY!VyQ=b?U#H7K+@Ijvuuz;92j?3c`@aUSJU zBJ!yb`C$=xDIz~IBA*qJ9~+UM5RuP|$md7oXLRPDwddqxo({I_^!y$Hdxv0Sp&s|p zNhdO}k5%U+K8F}+4$&i)42Y>X_&grI?}7NiJhE38JUW=|dBn(89P7*W=&`pQhIxz5 z8?Aj!dmb?umoL@Ny`>rN9d1IupE1OT&LYOXLcGy9A1Y)?XTzPoOD6??T|O&{KxB+)LAe1y`NgQ=rV?c9)b?e3cNnl&JBk9)9%*)^uaBIx1@P{ z+@D5zYtZJ_0j53ax}nQ0XWOf`;oT>#&uwewppRywuS(HpCFr|i%mYOjD|%&JSWokS z0Ww%?*2H7o(kbg!9Dh%<8D~pA&S~AEyT20Pbhkq5*S~<9fZIplonAk7(tvq_&Yq)j z-rv~66nqM)55||hp>Fp*NPk9}rR&T6IZM}jc_I8t@n**Rz!vORC0^fB)TIRV#ay*s zS-U>RJ869Ha^M=w_a@8}%SIRO`TRxQj@BpHj!z8y9mRnR+$9KlkF*&Y5TrQ*xq$@aEfXh`=To3pAfcN#XaU|`1=!m(i_)B_`41^ z+w~Wz@2*)vyqDv?c(LxzimB(G@dS<(?eL)=W`O<^B#PZcypPWQ^7i}ao;GQ9-EOmP zFptyGh+WFJ(Afwvo32Yfsl$0Z3%+elXM2)p%%lI#Y-2mHet%*7CGJO*{{13jm@cS? z{dUMvAFoE=Qn|xHtN*W*JM;qO{vo>DW0*VIA-5F!&xR37xeR4ih;p+p(B2HhlG@%^ zW7<0n?Tu;c0b}neQEh!4We%a3!<=!8DDx<2lSP^HzGImM7biPQKtPnmO1@GWj-2R=8s{Y)-4)aw_;3DpY8zd8zhRQyT46PFz)-{U?25(gwXTX_;YI(p;S}X|y3N33nkpF9GjvjXQhY%UDxz zCR>BO?t1KXij8m6eBXj~jPF13U5XLw%J%hFoI7I}FuEG=5x>mDSp>;o3RLJ&7n~J$ zTED>e3ZS)MZ=uIItN`@7+jW)rbaZNBJX^f=M*I0kZ>&2nWi1`9OLh#WI~h7(ZUt@o z`tO~~hK8?W~1Gu@Gs88*r)QnEcQKL*Xpn*(t{qo|DiJw`Q&*~{#OEi zDQxZqJOh4uGCPSsKEN8lcG9;P%2{+kC=+}-|KYI5&yG#}j5Hv0g`jT(o%X!h0)GTJ zogW_rd<_ZvD>^6c%xIp9whRt=L8CM6CJs|WKLSMCQ;>EVf0-XdtVYS#a=AgF0N^8d z?{tQL>oXv9E2pR6Y}mO!I%h7zo)BjzvDcz|g5#NS&I{cUd`a;Uv1e!4BW{OHYO4#7 ze6~yA%K*toAL9E@y3?e0q9&{>*Cw#ZM-iiRW^BP3_z2o}LwozriFORJO?NU!5xd6_ z-;pxk$34p!@bvutASFa&@5p0%_y=*;TZzeWpy>yj!ug0HP8TtmPO?~cRT{-)8qqLz zR2s$OK+p)+vpadbO#|MFxz2bS>u0NN46pMiYfL| zLieNX1b+d@`)Khrx;xsLoiQy)e#p{oc#z{?<_9shR!jj-G24Q%b|ZX&c*!GtBI(n8 zAM#TuKW*u7W5&|qpJCo$S8OkP4?YRt-WF$yHu|;(Ht9UgW07G#!@ULVFL!1*rUj2c zmhRiAUSEqlt8*L<>~dFdJhQ|}_XK>LVBR=qHeifI&mCtCakwim;I2S9!%^So_acAB zg+3B?kBV|y1f=i0-b6dr@qWWxBKnQS=n?e)R9-*OMZY~uvb^6E8ui;#ppif5UWw$W z-<||*L)82-FmxyAIJXJeL0tB6(6(^epwJ%FX+zZ9GcfcT__(xy&cM*KqRwh8{F=%r zFonOueH-uhUs4RL>x==+K}W2e^AG+UfS3IU>_pB#=gc_Qh&hNc-1#Cg$uhk4uaBf1 zfASH0KRdJDd~~>S_Zdd^uqIUiNn68b|9}9{PSW_L>{G z4@E4r8iS`1V*$`CNdNhD`_DfE-oBpUzEQciYysXHLyM|Uf|zga9TaSa+&?ZQtJZzS zfZ#ijJBZY_<`vvIvY={He8aYrH6;c!dMV$qO@rU?y>wK)RQV{M3`+^V0=btiW%tvR z;9BwRY9{PH3)uF&>_mw=0cnWF$!5ujF@sAX~gnM{3aX!|OOY*Tdxg;O+&L#QD-SBa?zodR@ zH~eAU@TG3}BfH^eb;BRq4Szy6{Jd`X`Q7kmbi+4w!!PNEKerqHf^PWb-SDfr;a}ek z|K@J^OS|FQy5ZM#!*_JUukVKM>xRF&8~&}`@B`iO@9c)Zp&S0DZus|h!{2;yKF)9a z!M-%=(M4dCCpnVAoimOE%HNkJ02+~4a_8S)GZPW$5tgxOQ=X9=kUoD*3gG(4s^!5H ze^h!v;IVua*EKcW5tViVk4#&EN2JosSMP{SYTecHsDV6B)u+l5A}Xc&fmj{oBTo;} zM2kCks+_7va_#Lpysmo*uW4WEA+_eR4|<@0V?E-KZS7IQ*i#85$bON4Z;Lk>labwm zFX-6gMiSU=guti9tn<0-)5KvQY)Hbp=q*X9$R0>qa2UcpLD<V@o<-Xo#k)EkXn1^V^446oYitt&=gfpHIxz0dMfW|BOZKaxUI{*W2I4k$!l@eJ7jN;#pBE&9w)8>=C7A*c5l7K2Wy-?R+>}3 zMDp34veySCujF<)8j{$!adonz9>9eptRbhEVE7Y#i(}l1ym1zv&o$2N_l>iRv)Mhe z)iI>B4rGodbii^bHlL5?e>jNx{RQ-yH;8pb{p?hhuw?*pn=+bSlayc4!ac{E|tw3 z<#6Ed&AQj2z@ajz*|M|VhcVT?5(3vH%QmlMkx)~=d^tL5qegizw>d7iw2%g{n656T zkcj@oAQDecsHl-2DU^FA} z%}%G;ZZkVA^%Kl4Id_6NpLJobz^rMmnUt$?XjU?#nr4T4m6a+%cz;( zC9kf>bVi|Uua_Nmmn?MzGUgy(okzCV_(U!_WhX_U!|t^ElGx(B&M8Gqf0yKD%NQKlsP z$1?3MtHY1^X$J3B{1A{0ze#a*lHW^{7yYCS7GA52fWhFKo|K7If#Pam;q_Uo$4U*^ zQdZXLZ0Tw#3)xu`Y2$GHzuwtuJD>>ZmdTdENZigeQ#_w=skI3d*35a$Swa{EX z&pcb2G=5^<#L3fgr%awSIe*&JX_Im%<>yVQNs?WDr)2ZynQ0K1-5whj=QU!vs;DwA zswgR~Fc(%9kBz03lvWl?w@pvNFX~`LR5oem737dLQk}(7BiW`)c9$>T?2}e_Z{oiH z9(tpz!h9&yINTOrtcH0$6kuZF6xgW7*djCNq#SAD6vfnK1bL0$rL5O#gku7rxV;S3 zjv2B;wcG8X#euylp996T&$|Q5Uly&e*z5UZFW>cI!5x?9m#lt|2OG95ECXZs0%XgU zM&*y6R3kaP5;g!>l5FE>=Nds(Ci@No?V-g(J7Nr5Ay8|vyQEQ5U~DytA2VfqZq2As z7#mcUx>uG)jp3zXiykQwZD^%e)Z{|e$_p#;H{qT8&N|E{Ue?(jv1+i$wxC(qn=(;P zfhij-Wfiu=gm8^gG*z0_d*o`r-GLd4{1QzfKA(RsxqC-PM6Sl_W_}^xHOL;1+Y?dq z$SRAvUPciyrw*)FE3y7tXs1t`{L9;zRoi{5>|Qj{?sd|UhW2NENp{Fi1PkA)sgW~t z)P(%iibHHlUhyi=U(Q}dJ(OANFrgrLsHDr$;-}xvA)*94j%H#2h!Yw=VLC0I zLccdv7)n;S)pS{KwvD++cxE#GENyGihY=~x-rdIeJ}VDO3q3B%Zl2;YedDdlH?3evA!;q}> zSvay{y@>_644cNxglJBZ5xA^LyS?_jXm=&-kRkQOel^+6BwnVeBC9{Ab z3M}H~am?-`fJsdjC6~IaXpjd~7oEsErn3@$n}B^amBp}cp!4%W1W_%7)mgc!a~JL{ zP`2dWa@i9>E?}rJFN2mcW#+2EF+=&Kt_m65;6<@~C1SKBIqbK{wi4NE_1O8%1?IWc z5%w1Hz5W8(Rf}2?%Z;s^!jCDL@t-h+i$YTG!JMV&^(}dK=Xj+0nG!N2Q&|89?(3Xc|h}k<^jzEng=uw{IB*va)7>H#Sb9_ zl#f7@1N6-);q<{OLH^|_Ab!|@ptpc7Gu>V{|KTE(F0xl5HETB)oy9N9Y0e1@6B;eZuwg{L-=9v`Om%uaZq1hbg z-zrlsmcLu@6@PZ+9~6AW4_*07c)KK~^j}y0jCo4=7vq-;{)6IcdODA;45*~mA_r^ds8uZV$1)v;9so#KMMZE%72gZ8NTN3d`z8LEvI=v^MK|7 z%>$YTG!JMV&^(}dK=Xj+0nG!N2Q&|89?(3Xc|h}k<^jzEng=uwXdci!pm{*^faU?s z1DXdk4`?3HJfL|%^MK|7%>$YTG!JMV&^(}dK=Xj+0nG!N2Q&|89?(3Xc|h}k<^jzE zng=uwXdci!pm{*^!2ib{xT1m@pUjd4oGW0ZfEEGW0$YTG!JMV z&^(}dK=Xj+0nG!N2Q&|89?(3Xc|h}k<^jzEng=uwXdci!pm{*^faU?s1DXdk4`?3H zJfL|%^MK|7%>$YTG!JMV&^(}dK=Xj+0nG!N2Q&|89?(3Xc|h}k=7Ik^9*|(R4yl)p zv1~wqjwik&$5<-z69i6qX6nIMKo9^(uY=wcP1oW7=dK%pRDyIZQa#cpq@76bB7KUa z*D+l((lDeek)|NcMY{;x!1ScY_oX4 zAcE}ixIL7ydu!}2yHB-B+2dwrd9~d~|AJCegL&ly zrVuTDpW7olWQ$iWce@?sZnT_B$1=EsV>uPF&+l>Za)g@GTU#l+(05$aUGFoy^3q=bT^vHx1c2?$MbX!!BW^?guwY7>*iyaoPw?eLwJ+ccv6pzl#`coF;iy7Fx z{g5*|>kvG$)$fzdb+QFv#uA-Wq23Gvfk|`A?~VIF_n!W+p;dRj=Pq6Oz4{0AAnAum zMx!ZZEFShJ*c+}3uw)vuF#S33KFC}z@UsHnF7SAR!rwhoksm1V&7%}NQ{bOoso+xt z-jt=_3kAL%6A+d6eStroqu@2bsXo_Z0wVsss=UCr2z>WMh5x+359BKNO9F4sQ}F!) zpD{(j|1R+B1^$J=R|`BoPHA62;AsNiAn>aMzE9wJF?eANjwy)y^8+D&Qw&}waN|^^ zek)_}^#acm{CfqyRN((1@J#~$74YF`lQd1S_nN>>0{>9pO9g&T-~oZB#4Gw+1wKLG zO#&|!c(cIm0%rwEc>#e-0^cTZlfatjF0k{C$Bh6?lAtQeHsdnF8M`@S6qRB=9=~-YoDJ18tZ8&99`d;v`S{$)M9iIO#|7 zNnf3RX+9=>6(@Pp{{V>|!bv}pPx|V7P4hSDt2oJ%KDJ`~Awc?(e9~9vcbe}>U&TqD z^bb=84*}ATUu-#59zBo$&>yrBzg!Z{YXCPtLqc3SER4vBv1NX`d_SH#3%jj0;lzj*FSiS3PA?d3)@ku|GGI$7(ek7mt)%B9rPtsR$lIP{e#9t(z^wss1)?3n7 zaaI3j%HSbDKUtN!BeI|VsCwVG=7!o~%lYS(h^wss7)^pNVagrx}35g!UNk5WL z`s#X5>p$tMILVVfrWgJYPWq92(pUEfv|k{76({)%=|}QOU)@j8{(|&XT-7)6(mU%P z$tQhv|4}IPRb16iiP4YblYZoWgvwWORex-Zek7mtBljz$ui_WekK~iSx}Tx_4e6`+ z#q=Zjq_6IOXg@^yDtXJD|WLm%aBBQQn6!$<8mCA!{P85D)48Bgm#b{;h;TZg-82pfechu;p zf{WRbv3PKl|G>aAC;zBLA48H2Bl!8gU=_s8JBh`}F@!EuEU z)t+Z!@E2q7S7PwjV(@nryrX@e$KZ+R36MJYSH|FZF?dl7ULJ#gF9!F=;J;9C6sbIq z#^6uJ;7`ZkzmCCQiNW{4>?=sWN7{?@8qyz;-ay)q^d{1uk^X}87SaKvw~-DaQLG(C zdI#yRNbe#24e5QPza!BeJpkbaBw2c*b4zYF|A^#++qxRdC5{5JFBxA}kOP9*B?yu>|M_PitDu4+oAslFO^mL>)AjToi@9L-@|raIaDwox2P_w z+a{xexc#FGTW_7&>UKIUxaC!ANAqc$t6__=9QI} zm}f63E3TTisLTupbR|x_5HXj0WOt3Xx*fLhkmyA6A`#r&c5Wm5k#owIl)7wHcDnYCWmNhs9^VC6OR8q)tM|Q8I+pr++?39C zdz=QF_FqtM6DMqCqZJdaacACqgDh1~KBI;EE>uJ-96OmDoKpg>gf< z#fQ7}C=r`-G2L<59nG|Rai?yrtE$6QwyhR{QSUAAal~6Ldm_lOtwi&AP5Bjhluhvh zA&Q3aA@0d5A~ylzB;9V0$hpH?R~5(AS(K9v0wJbhL^&n4lWnd z2cpdFGs{0^#ZJyyA+br-A*-o7aA zC~()*04cEe=DBS0>ew2QT!T;c7C3mHTtW;k>*ACPSW{mS@D~*YzJ_}F5;hlDs%6Io zj9esQE^3K_(|Qr5n9#x78lOLQSYlbhxF~8U#xxD%B^cn5L=7dP8Ejm66N_)st9?@|!j*V%1d#?dN^-M7T$vpcB%4!o4Z z!h!dN%B*7cxtGiIK5G@$7qh>9yc^4+$IIpv7vl|$RmM9c{w@tOxWjF+VR^%dG<$i# zjmOMW=VdudywJ#T*Vnsib4uK+TqNktv3c`yy!End8&?fC)a5Wmbw!>T?*Lu2f|@O6 z8=v>gTw=VF<*c-La%!*ukwKp~C)YD!Wlg@zJ8`8WciQBY(`v0&&kEkx^k*= zy0bYhcS?SZoIf6~EH7)ffc^2;|LoO-S5$N~P|qeQpy70(-^27c9^n-(cnwI`Bhk~J z>4w)!de%!pyZbjMOGxOtt3I1xnfclBUrc^(*i$*qX|n#+1xsHV^H}ba(hb|vo;J)k z8KxZi!#~H~a<1|G+?O0Lk1GDF^U$7ee>uN&?XYh@yQk@&^U5*FE>qsoDD z|6k>gPyOtf1jnh}pWNhs_3=?(3@fxfIjjHYjw^qjz3JL}ub%m%|Em4&^n31Jksf~N z(5Zjzdg3q3R@U9J-Fob3&-{OV+4uR;XY6lOoc_!Czu#in^uYMvjcmE0{Id^!^Vq|+ W&AZ0FIqQ*QZ@qIW_wKt8X#WRillb!h literal 0 HcmV?d00001 diff --git a/tests/test_metal_argmax_top1 b/tests/test_metal_argmax_top1 new file mode 100755 index 0000000000000000000000000000000000000000..d99aabd0d8e682e54baa639133b1f866c3f7d5d3 GIT binary patch literal 901592 zcmd443tZIKwdlWpGaxez2q-EJhKEFD@D(FQNMad4qk>{Z+BC^I1vDn$Clt9aD8!SNqAN@5K^i?ZW6Az`1{!tCpV3F&5@GW?9etE^?fz%uE{p^w--;@%610~-5 zulra^dU5gNvt|{~UN~#^6HgXDSvvDZd?y~Z==1TJ$&W?iH{$4yzqnW*zgcrj7c8hK zoj3bNeA{RF@#O{zNxb{tjkp8&WIhDlP4jUffbXXPG!pOrcN=_9KJ&!Q_|{bV+xKz+ ziNw4A-Hh+4il@sf9xX1fc&rk@8~dB|gdg9J0^=v~8*y~UUtB!9toX6{(pCNMMtlnw z`tc5tZyb;l=sU@sLG7mv>_$}Y~GRxrcQ4Os6w3VQ%L$PWT>!6{Cf9gTxd zDz8!TcjK!GHU@mbAjMS=+>a#?_`7*sT;IN$I)D2%2l^@T?)ZXuf%K5<>~HF` zeta!){t^EtJ+^8;~N7uWEW z?DgZz2qa1TpP&<&(p%#Dc>v$p06vL#Z{M|={h{LGtEg^?Z{QmCll|OSspd!g{P^l@`X!wI z-0jC8|GVQ#N0jjsPK0V^iys|`H=85W+JSxtO%V}J?x*0b0KNubzc~%^iS|iqP^Y!q}~jie6}#|`d_#YXqn z4~DqUPYZQlS=_^YrA~AC>Utj98>-ftY<90nvwJt!#a)~=TCHndtbgYosmC(Oj(|>TGh2 za;0wjMPb?4|2E%1`ex_P3R7n^&NmWHHPsZFw1)X1g!lStdxheg)Y?MR6PxGvAWU6n zO}mv##;ENT<;|cSQh%6t&$ut{s~ML{+;s8n`C-6*F|lTxo_B@3u{GmNhMK~#NK?TV zxi#Zj_Cz#(Q?0zmp0_lfcqzQ`AFrAlO^jU|Z8m6{tHs#cvy`!p>!GGy_{-$RuYa$m zo&Pki@yp-mH{M0s7j;t_+l=uZGySwtM*HQP1p`;SQ;@Ra-GVz;yjPICVq3w4=Is>^ zG;bgAyB|GU@W6^UDt;H6Rry9pdgaD-?^nzw@3BDMW8LyLk@qZl&js?H>z4OAd4tFs z9LO8oEpG#PIV*M)%)+g}CF64bGr3?EZUrtGmveb+#Vp(kTr$q^!uJ|Op3Q1hD|Qx` z=(85NF5FDH$uEJ2vatNvt1wEr zf^?h2n6u1c&1c^(EPp^XnIc{;oD`_959!^U+xN9J7%qLGtp0S^&)!4^lfc6G5HfIm0b@noF=Z1m1pC32y4dAYM{N_4m zlHUASpib)e7x3C23)D#)^mz-^IfA^B**DjDggP4@1@|7yb#>N(yJgnRbuJ}6iS+Lp zpQj{$rb=%=m(!G`#XD+?Rgs6WPGhWv9!=0z`;qLXwpsrC-9J`EiCVa$L8+p12Izux ziEE^{IsD<)e^o_g2D_e~{4c6#lF=XD{Rrtj{NYlZrNxIb^-tZvl@&0f&;j#pOxb=sm zC;G#+gfqibOjf+A`lj0M)#nA?x%;oaBAMs2%NT2D>9p?m@toBCSWk!U_x22)Z&y+drL3n8be&)TL^!()W5gX0QV(f*-N(lKy6=# zdkPl^T`yvd!(ZP~@ctq2#3w@3w5|}twE3YS)9!(f_l1ui=^0XRKD5WQui&43fnh)C zb*?U=k#e7zx|FN`GYfYLk4b&L-fEt=d)zfJ-ds-aL<8J1u+u^C(;Hleg!Bcm@ zQ@6oW{dw2()E)5DZSYio-t|0n2RwBfJk_6fJx|?H0S~Q!hgQHtE8w9OITsr$X5m)g zl5shaS&idEmv2=o-Y;;#KT9vcKQ&V)3WTD>&}Cn)U`$HjpGoXT;ZH4lxs`sY0UHV? zYvu!%ua+Hg%v5qIztmY8T6@1* z@p^$RMs;Pw%ZjP5ly!hW7~8?vnY8c&<>BGZyqIuHUZjf2U;6%f$CZ*JQ#4DyBXo+z zu{2_aV~iy~U$a~YeXjDj$HCiTzsAdKlr6E}nwNc3Wms2}bv@n<;BEOHcy-=+4exaD zCXp_DMfgr4JgGFB{S7}EKQ2tU5__wz8|nVsWIeVVQQjJJO`%1?2b9;hK=Y~-s ziVJD<4JGdmWFBKoV~a5w7uDGEbj`SPdzAOweQJB#5#&wr57@nB_*UkuZI9jCuU5HM zvi>z>CKG#=!fy$)G_DMWAJpZmcQjktujP4j<7Qti@F4{)`2iSYFpi@ zjoOOV0>g@91)oRdH5ykOFKCA+hO;N8QU1iyNsV7mYaVwow|U%iD_RQP#H|JXUEIr* z*@jz3nH{)~mzYO?N_N0IN%xJbTXjih}z}3w!x; zXlmCJhIM})s$k)MdD!SGc1fL@tYzk+5E^by3+(;WCFymPn?kxGJt(_8AnVO`>$Mn1 zUPG!HbF`K=rc$Pp{cUu%w`HXAo;PY&Wt*mbE7f*MQ%00l2JeuNSyW-Qd;(*N~#pe+pa) ze#$I-F2Dd=km!Q(;N&E0yEWphG&Iw%IHMnwV z-yJG_mb4GQB?`C)fm;9^=bvEvilu!6RQf~GzIfnz1NR87F2Wsz`5`bUNYcL#OtTf3 ze&iQ}>v_vnn5n?D^jGNZUbN%sBiANEy^T(4V z-doE18;?Jux{&?#G;zOkTgZ}aoA1dYj@luJ8 zl=A*~vcyMAd4IfhKJl?q{zv}u#C@ZcHx)kAR;il)&8T97`<>G4vfljiZ0s->I!s-D zM-Lg>&Y2oDjP3`|S{`KBJY-lq`?;d;>E7nH?(KYZxN9u+ z|JCQ~r?(qwJt?OYy(D&yUXbel->z_V|z87w^mVUV=}3lTz$%gSUL0 zo9%6zh=a$p8S&vGZDIKEiMD+FP59IB;TdffeE33J6h7;|eFpx6xHqHR?Y*_x!WowJhkM8&O?AHA_&o15X@A<9nTRp$g{diB4?)UY4ApSPHcZc}D zv3s|Pzs2r-SNu(O@7vZV>-* zySHBaV!QX>#GhsNt{49iyZ0C3KWO*9D*iOP_ow2|uzOz-f1=&%5%`w<_wK4NtZS>ZH>GOo9@fQYaz>MNk&2Ae zX4KclJIO!4%e7VDM%%qBJHp07qwlC6rMZ4b8cOl)iv76-S?Bk}WuFVipIhWVy9MLR ziR-#RXug`T$O~T`QQjl@YWsHFPjNrNwS}m)a+WLOTxUa<*N!eP85wRM=QcNbJU4p0 z6yi2?cjf5r?j-C+cV|O)H%QVs^O-sG4JNz~-Q5uUvz+_P=Id#)7Aw+f?wA=Uy*Z@Z1{(uYUOJil2Y@>k<7Pd9~o@&wWtQuU}T>g&ygZ=YRH2 z#Ruf=4CL+XmiHBTed`FcxDNCiFINn3o5y{!dz?>Nw*J%d!AkHJgx4P%cIbQN7l7ChH z*TNf>W^Qalo>R3(&H#qSl5r~C_lnl!tHUkOn&kYBEl+IwE9g`>Gp2>An0DYhfN3zy z{8g!O=C3?k|Ld?Pv+c_q?eSkF$IVPgN-arfB~QWw$xlutEu$nM>7mri!Shg(nSB5{ zHI33-*7AiJ#hmSv(cS2@i>_(6&@=R>$KkiofBN4w6&s8TUy zP%%c%t4Wmq<0`dIKkU)ra$s>=hcm0e76eZX9FL0PgN-88U|Y-0!|$tNr+X^~cf*?YyzF z_3@f<>hGl$rnR3P-aDx6OF7q>PkwQDAikulzt61DLgIdG$S6(F;z|?JENzE#nfqMR ztb|!{B?%wEr!I^@k7TfWPrRCaC6v4srr8ODb^o!1H|rj}{5kPAOphg)?DXd^%6%DL z#$M#YU~Fp3E84uzEyj8JT*i)ewMK=^dy=s}4nBSU|2W2#3@r;yuOdCjzg{9N^1kTg zyEL`#gF$NhMdDwih`!T5$H!P(G9D*5(}YsK2YmEjWz8Vte;3@K0Nf1VeABw0pPl!s z+ST`8($CVX?r*8%yu6t6GxEFdxbl8syiL8)HHBH=s}ULhxcy4YXzj}F%IIyN%-xh} z5jkGUbgF-<%pmEX$n;X?%gQEm%h8wprwODRbfWWrA%xK^a}HmoizmZ(Fd;Uno=4J5W#V?aKt)wx2RB zF@ZAQ+`etWGQXvaJvvavcl$EIw!K3cZ2$aYF!A-tpZWv%GgZsKESZ_USc!n5K=B3l{Z57V-*gutt2nI26Tr2DfI{t=da zDquk zMEKZ)r1!^{bBVynUD$W+)3OK1UM)P$yU;H$UJxHS;49tl?HL7*!sU$74a<<7v}z&k2#q$?=L z*K(IE?H>ZXoJWR(Gn{x5ej0uPejfe+{5kkoM8vMfXMSTn_##{E$B)JDz?U;#IC~a% z?XhY2*kQ!x;d9F#I|m?hf`B zeLvYc($z}erLDm}v@q}ggE2zpIdti9jdPlT^O})!TL|~=+`Z>&u5-o^&v~O!*CmNu zyb*aspF?DipV9XkgRZ}NNABjj%OGXsai2a$`lrK_znU~fTCjX~_nv+Qn}Gyh(9?cVRpkNomXmNC{L z_e8opLmdsz`{n;2j;FyPG?Y%;TABBgL)5xh@CEZL1NlYY_|g?}-Dc#z&B%Qk>);sj zo?=}nWWBZ_c6YKi^H*;2-Q*7>Ut~c8X+c@A#GtP1r4FU+SCVlmsWhSOFZI^;2kSAI zG3QGkqo}(lE)o}kv*5yUW*o8>G*WXohI?bhovczr5*kR;kl~V%qqPAI1qPfE7lI4L z^}vPU?s{O^a5Zp9Lez&ZS;GzvNk|$vIN{)*)>;Q29GqYFelkfzd-1`bTfYhG;~cyM6CT@PfF zpPZl_Os0I|-IJSFSqn#1oR}@V@q76e5T9WQc zID!846P(aNFs+t&rMh^}*#U1BpquwJ12>nnX3f564*q8YeocB$PeUNRA&~ygfHz2w z^wf~P2DlBldQCZ`%-R93lNaGB>6TX=$a`YICh{yEd$+veK;HKUY$Pw-lhiFQE0A{= z_f{>O6;vQk+MP&V9=@DuWNv+RhC`PtL!jwUXuAhA9tN$OSZ`+5n}PKfhW@k%Yc7;E z7vj0V+Pi2pc(|ZM|Eix0)+*NC2hgMn*Voe8z}j1TRsJWBbHo{goMVCjbrtzZx>B0I&hogxu8(GnaAHo^!1hf_L4C{vT zK{Kv}d&pMcg1XOstThbr)Av26)~#cWmEew1zrYEuI_m!j9<4&q6ONYq+dd9;*;mwy zc9a;q*V)e#JoD`Y9{!S!jErpC&vUQO_o{w2G7j)aAKGZM;A!TJ2|s@)$j`rm?_u{N zd~a>?u=`w_=)dsCuLrXa1jb|$X?|Wanem^*JWOOh9$;Q3pr;xSzcJ|iM&QMUsG0W9 zqm1y0^K#`vG$C3ih3I*zqtp&2?|G}F7`oZ5) zm&RNhuxa8B8N9UN@U9-J%EBF}==HNgRMi&jL(VbhlNKiQN(|+ENL(3f3V%xd&Dbcd zT()C(f@uTobMc3Yd5y}!Qq(%$P^jL0-fT~Q&iKgRg?w=#Be-jV)>0NX%6?Xn976t4=Hznazcc5z-uuk=))l@3T}AgqJ~Cwe*%xL% z^++@JF~R#PGf{BGHdqCtSiQIy$CH>~@hhdSizsk%0?CflL4PTKl4FnRP7L&(?yY;8K= zbw{;+oy)^i(J(FCxf1+|;1~OwmEbQhD_7(dK5n=r;2a{FIQ7zKprmmJD z=(wTfos5&{5GA}r!ss6*{GNnoA^Q-1o3PB$X5+)6r+%lqo_hCGbnjty7juc8`es?< za^=VR8c7OKQ7wVsFc+1o`S4lz->AU6G;Uyyv?ynE57kve zTL;#zcG{IaGdIMZS-K=-M#Hn;vz0`sUO6UvX2Ys1+bp}r6}mQR7QD+CtEN{+t5G?r zxr^E^`ihQfn&U(I-+Dkbofe*8P!0ogg)MEYgf}u5EuX5UF6i(WG`z3sju~n5@0ii< z=-<>)*}uu5nJwk!eb)8`{hORxxMhxczqRA({!L;>QU+e+iNhji4h>U>PSUqE*bAOw zzkBhwOKk_2n&a30F2@$V%p8AaX?XmLjWX6&<(f34y4*o!%{KX16$eAc%ed96#M+`_ZI^cgr=2OX>jQ)*q|Mar9uCTCli@>-1+ z*UE70Y~c3;j7HZa@W|eV{>YWc8W}hQ+dR$W9L5@{XN^c(hOtKCStBn5*2qT2MC|6O zjr}@hy$nO&R?nKT4?%c)6Iq>jZee9Vd`r9+l(D$zX_NFXDw4)_j6*bb&GmLY+7OC5pjD;Rn z8hSg!>F?x7)vI`^sv7u<>_v_TlxwEzp-XL_bDs_0NgB|6zxKMRa^UOHBh~Askw*OF z*P|;HE<|mpn5{NeyuRW@fl?brl&Xy*lCTd@65rYN;|TTGuU^qC&T2#d&a&gaqP(TX z8F^Y_FKl+w!nbh-D)+26Oi(V-zmy#Fr3>6Fa)6mKOBT81Md_Cel=l{j#m~nIqEX8i)ISv(fBSF8gJPYcbiE zlktn$_b?92GUHVD+RCA5TH$AJldf-QABv-^_B#$V>pMk#)A6;?UPYM5-@L%vV7~>?`REj?ne-r-e z_@7I9LPJ4x!p?$~i6*DYR*fyB&rH}@ke#rh;Pr&p3+fV(LxC}bYKXF$DzfNL!qzE>i$i(JM5W5Dc@N~7#!`-_iw^x%Gu8O-NssK z-xT3(597}7igF|Sdb3zJdssuGnO}1xYm;%yV(op%p0dhda;BN>nZ<$q$rfqPZ2#7m zZeu-Vu?N+#o@6cFNm-rFse457^>p40?i=ZR>9;rc(ex>Oq+fdv(#Kr-NPn^X@vk5D zq!G63@gkSqg!ggqeuJ!8X6%1}`#x`53i1p0{<2SRFd7btY!NOx!UMjs!LjeUMO!xS z_qU}Czp;+C^i)+1*pldd98OzCUEh{l^>I9SKVu#~VGhoY^3P-2?|ntW7dGQJ?upyE zmbsQTrv9F@DfUeb#`_z0RgZO_7yiLmIO$_=_=oV^KJe)Ga}9?+pdXt*EZ%8m-`KZk z^iKD4IXjUh<4v=zGt^T1DC|sWE)==+l-PFylZ*_q3x8;oD(VETny{m%(rxLu7r*E+ zzKS}^&%15VTjK8FopU1L5dsGf<}7@89`fCY^0^rf+G$y8S^rk~&OxGSF>or z#d_eR-)-?VP1Ocdr_|p9thB!qetulqkF4$>%$fYKd+0*<7UM$~H*Bl0ogP+G7-gv` zJlnUX@K6TluEd(cjo0zML+86Ph(#eW>}N9&)Lnd1N0^N-d+u|47^?3oL^9^N~{xR@= zl(F6~bSU<^jE~S<_c5Om7;_oR`oLHUTt4!E!0EJ!Jkao5&c*Coz>1vzB4hu}W1MTG z4z0Ii!`r^1wgnG0^?`=l-kf0jcgCab>4%#7Nqm_#Ug$YU%a676Z*l4mn59T49+`XQSs@__Z!Z>AwD4v61ExWus} zBX4Qn_`JwZtbf3tlpGgdy1y)AGjY@K{_z_L@14&)g_HKCIvCFjkWaeYeUt{J=6nwrAL z!P9JL9+!MVO*6o24eM$v>^t8rFsQ)^#VVxA20c^emio)!nW82~=%Tl0Gq{*5U8lF7 zGoN#&*)@zfe4tlP^tmIGusWD@{THlBb@d8>-~eBUoZ|oDVv?~Gu1ck&*Rn3FG8?;WjCiZQaw6z z!Tn#1`*7NpMZMkICAvKZF;?*3vt-$k!6l1cu89aK7&3S1%kLZdI)!$E_#V{pAv19P z$)qh=MuWrim9OY&73#c1KQpSAz4A9hoO35G(P-^FcK0Kf#2zt)wVc7)xL~knw)WI| ziu@-st;n((`Z&XYjRlEustUbz(Ffsi&Wq^Rj`gVB8kfR3hW@nBb}warq57xwpQv^| z%(+AQDfUNCn+3iH@b@+z>v13Me%v_R_i$M_h0_`@7(!feMom8#eL}e;=B*mvS7$`m zXF|VVMz<7>4#I*iA_DzLB)SzNx<2JGMcA9nch;fPMUI80)THOeB`)mWWG;KCY2Or; zZc{Z);heMXnVPnK|694Xx%b#@72kW%wmLIBJ`bA6i#%xEGR_k3AUrSfQ)~VmYSffW z72|x@Y28ZrEyAC?yT-ap_`_VzAF__|(JvpKv((tziTm{Knx?r1i%ZUPwyDF{i!MXp zCjSn-^sW$hHQ{mzCo1o%-NRgGhxBmEyRyriBmTmBvkNmrUFa7N?^_zR-d$(CczUGr zCQ3cT#O@s@9#;!T4m6|-L-dt4QT53VbmGiT04 z`f+GVd3lDM)sjh{qM4m*IIl^(6<#}xGIKXZc!mE}Yv#@k(BDXG=T?^*rxz!yQFXJC zX^|((fsyv@O~BTv?%|8&oD-~fn4M+wA8K-=6R-`hY3dkj_gXlwq}ztBpTxPe>?aR3 z^-tMreQM4_=&T-U+MA-rv>E&7w@p+{xWy^d$OvT=>KO zO5i_S!aO*ZGX9D%@(xUXUCFmtuNtZsbH|0y)oP!R$hTPU`P4a%LLmL(@G&^U|erqvf*wJr| zHBYlx9E^Kj@&Rk-{_>2x#X}d4fyc<0zQFqbh_+^N4%E_A(T*GyBV#CejmXl!B~N&0 z=l-YkayzhDDeI%g>@t`gpKzw^J|8bLA1$#L$C{WAo>b4bGnQGDo6@vjScMJ*|5V2AIqFZOZLN1ezmyUC{h|0lm`{*% zhp`7YF0;QPeG-0mM(dwHZSjbOyV!?Cj^Zx6==7X<8PgV@UU&u?oJm`Xkt?w&D0+gn zOb=1ht7u2`;6txH@om^WcNwF?@A}p^#>ePsA;MoHOl<}^#i zbf!QzLRTlCtFORcOq$Toqh&J%k4`^10o+*zgHAuK#u(2zU~+C-f)tA#Jaq?JqE1^ML;p`QLZN-}aNhv?rT^i}224 zJ;vbQ!Jihu}y=7gPgJ zV=iN*%w}xcbp8S@9w2>=PHW+gQ?ym)wG93#aH-!@WBl*!R8>27hrcwM^SR4$weK45 zu0|d@y=Q`3`=hVuGsevg?Xl)!I^koY7Zbi&teG7(jP=PmZ3t zsH;N89J;GyEDsL)^tB4;ZiF#1d<1m23pyLI$7%h87HOHfc-TUry-Mb*iupRrex{eD zo>JxuowKWiF&bD_l~Ej^HPKlXB=rO?!m9?Vm=dA21T{*^&JkKOL|vV)V}bcfR8jhT z4J<1WJo$f!J}lp1i#GJa^@=rn5w{h5k4egJG7SAjn>TY`*PWr?5ZjS zo-#+)AN+Yg=gztGYaab_Fdo8_E9ui)jGyqlv(QdrfG3|qA8#3G@i^#H9yBO?L&t%R zU)H(ENV0a$(w|$w2~VB@{-eMOU7jtk5Zrn{4+iS1*Lv#xJjI=*@MNh|c=B5M`3GQx zc4h4lJ|y9u@Z_W2c(N&FpH=8b`nl;;u%Bkfe(n(dzx5LwoD2Tf{WLqIpDF)``zdpk zr3o?*nCOV0D<#lXSt}Vz@ z7G%6dykoo%n3KpQokl}iXK!?8;ISma1Ntfb?93u98rDr_5_w7FrW<9Z9CXRIl9_7R zi@M898uZj%W};8ML~g2OE{-u5LAhy9{582r+IR8;&2|=hbvb*sMqv|=?3c07y_55| z*NmLlH^sL8@4vR|XVsNkvKMv0uVkErzqD@3MlZ;lOX?7vuk4L`dDq)ELp8}c%~{R4 zn=@tVQvA0~Hj^dJY3f>fq-`wqj8sjsPj)vmCj?#GJ1D>zIYuv$}kS0iAa6{IM5y zuMKIpqd9^A{mJC}7|&s;b~PJ@P4!Eei_*Cjzlxmzd$jCt4)}`fZNgXP!B^%o zZnB@eRqn_*%HAfj9+!uneSy8r1Yao=UMV=xpCj|>d}T3wf5LuO-;J-72l~=MIT<(E zt8djGkuwDM-eePS5#A~EXRh!`_G&4+8r~~(DSLIWPabCe{+>pXsR4s-S^#- zA7tCsXy*J5Xj*7hctHp2Sm^aEZxWoX@L70;@Db#T7~~FzlRFIAhb7J6_j_GgJe70d z6wZhFoD=gnFHXh=VG=e7M%@M>C{MKXuE%Wp_Bsro(R65~ylLlN+Jp_*W_XXBF@tBz z@JIc;%Q;CEi9MIO#LvGh#N{5^SFD<>zsiPZ09t5;5+m2DbqO--YDgw3~qF& z_{61LF20n@!k2Qqm2pTpc!NXA!IOkP9ZT`^PDeJp6P}PL{1{!vCq^~qDD!y~8r!v0 zxeq{}`&esf(6XF0H|D6K{j4_!^c*~AYNU1NnZa}BC(wKMb7l{|qY&hmv+RC;xepj= z&%S|v9hn1}pEIZ49t&aLG}Uy0)%o`J`9l>-lGkGjb0|KDxX>p6HeJJnqhofxM%EJYO&` z!k&3t$BRyrdgq{Pl5=gaj&S&VN=tD@N}zl-<+dPS*5m7W+|Abo@^%ICjt29%qn{PX z^9Ayxjy!ZzDbeIn?=IHiUilUVca^M*qJ4S(`QO61bMFKG`OhbO)BFSXUGu-c_jk-c z_->s4Wae$A%>S@!^PevBzkHId&2s(xCo}&x<}-%*Ps0s`28KZc!=Zr@(7@Z^8fl2~ z+yxDcf(DL5133{Y-cA|CoFy+Y#B2RE%Q0xc2n~F(KG*#%a)8i+Vhtxl3pvojENGzy zTKEW__dEKzVo7NH3UI6dN1_VpRPW|3>SSM&aLK!QmkvM&60RYDL< z#cqZ476~J(N?5%|7@fC-lL&V(_xZGCJhb-F-sFt$s=ovrtD%AM(8Y<{z|{cf0q1yV z^P>abjjsW?{q%P{wDdV|;a)(U0}LM!U~K z%lnp3{Xb8;@j32qqy87POF#dg((WzlFM%d*qTO4-)c|)B?cM^eL1-*UySd%y<`%Hc z0vn`V-f~{0$6LU*2yBpcvC+C6>~*w@9DO_3>u49*DoDF~_3sY(^&q+|7C1NT6Il$o zK=#$4yL?6a19DhrvAv1CKc*8o2N~MY`CDI6Kn^Puc^zAw&4-+=A>2m2e>OXk!DI-X~|pa z3GYO%FGgO^Lzj>4In%b(9iBzpM!cMRS<}+Z;fcge#LKvs6qigGwG|`Rlwtol6!}2zNiE<{jjCyCXD#0Q%=6$r#r432;Y>I)F1&Fc@8oUsLb0#x zIP6ey#_-nSj1gVM8Bx!iTp0b#sfD8^*fZ2Krx$9^bS=!;P`ogAck#mU zt;Gu`JZoP#jr;W{mPE(bEK%{5i-+xeV)5{uC5uPw+=?7Eck!s54!%iLifq!t*tIgQ z6XAnxUoJgjV@%q=T)Hg}zXRWa--*8l|0I4R{we%}_^0vD;Ge-iJH+no!cJSUFH8SN z8LSquuV-KGYG7YT)U2JWIh$*Ti!+ul`pL!Yy`wU+NISrJtQbEZ`mUBZe!axmlXpoR z|ER>-mwotow0RKtwPSufInAS-=x;NN$qQ$kwJ*D+LyOg7Zt1Zq^Qgc}de3g@aVpbCx(7a^_3D;BfHlB=cB{20Z)~Ce+pgo| zKQ^qp(MEf>d^@3NoGXfT%$Z!bXVIX$9LB#tY1m8i9j4x%M8c+;hQdT;>TF}YI&lV=|AGWJ3G3cP_oEblZTl+tZigpP{zdasR<><#o4U4!3tU*bM&oi0}z&^H*- zJA|Nr2t^OU_f{8&VZX2O?m^yt89e8xTXY5N8=nRB3T^BosWGb$?`_}w<>u_&Urx$x z{j!a7Za?^t*uDtu`rlyp_WYQ(>>X^knatQHj#fKUm&VP&PBAleY4nV?6!A?n+J>mi z_Ms~C!bp`V=c;{_7hdSZC7QXr*6OYD7Tl&et#am?q^+`oE7NyGE%arn%-!<-eeO6H z`x|4`+p~#wSmCWZu&Y^z6Z-!X?T~lTGCqe1i@lB2sGN50PQFLn&X{{ilW#}L+m|fT zl8o3J#HtLxEe~%{NTaULlDCGtaw#Wu$@kZMeYju5k0Jj*H`;A+*d*-Z%$tdhb1d&{ z`c#kQ4MQK#o#e~hC1Q)+$y(-HVujyu&i*?#GWSsCRnb#^nV3<#=U%70c{#>g$2%x< z!4U6!*x28~cLa31=)J!B!p+<@x7TUBHCX$lY3aW8rYqBqn9L@pq;qC>nJH(-eoy8O zYgu_a_{JS>1x09*hm3|0K?VXdkQ}W*Mh@n z(AUjc^1)l!hAe{o_nFOh?VC2TZ(LbCk#+F^>th1zWIXF77yVog?@9b`gnj_t8+mFH zG^gRtv4+mGhJ?r7xRw`Du7$NM^CPs;%3KHeeSh9M`FVO@PpQD^{O)?5?qAQ>r`=Zm z+td4y$V}bWvFHbrGSznWu9w?4J@#IzGC32`xz=TH=cpJ5*7!cwxSKUTnKdqXNsNi$ z3GB1ijaSWBYzm~kUxZciCa;nAcen_g1s9G()-ob<8N(X0?5fK+TDMQ-J*>Qutts^J z=D{rUij#Ma{c*l(iyQlrUIQLoTR7l@-_8$bZTI1>eUy1sp=*+7zCo?*m9lDOVaiI+ z{FbmtHy1atc4I7d7;%heKhIR!BlL2Xw`f9-En!pLm$3K$hA}x1P}S#v13Tvu+xSbHM)1u;)yJ8SUH?hr(6lv6 zwHYsH1M;dhHGhk?#yYoK+^N0re>^?~n{I~|k@tVQ-W%(if=&7VFWP}!`Ty&7yuiF) z;QeDKba@iGtbh+TYCZGbf;K*dR(3%vf6!jC{@;#s7u2-@eV4`Y^iBtKy+yO+Rfnok zTAV7%55>+|i*X>61j0r~3pzPTk668`aE)j4{Aw-2(SSZq($*BLF8rk2qo;{oqNELT zuPppzl^@QIy-gl#B<=H&V>h!-+hf(3y*S@Jyhk9<#5ah0(}psG!IBtbA9b8Fq>1kj z_Z2=LNXsQ{@3P+UW|6l!-|y`O%usvg%3f-WX{0@KlK7+TnI?Q|3U7Ue@D7b{B+o;C zSk3piTez36=9|Gaa*xLMxOel8W)gfs?%>P$9)-y9CawPgc|UX<9sbvQRMXcwuMnPL zg`fBLWE%T9&vM7X_d@k|u)qEn!#a7N`{3Vw>G%HKw_`TA=KSENg?T^NF~5WNQ=RA3 z-P}Jo%Fyvo=35l(rDNK72bXxoQnigQL+lA6Nv;QfzB4C{h?O?X_8N1bJ_ z6CNf!vYa~Fv0=6?jhWG2th~bC4(o3iRJ!!JRqiuMlkX6$;@cEuybG2wC`HHWGlW#N zF>fv0GhMAa$zS)s{S$RB2Cvk2;SuHCL7giJi!LPY(E$E~TelG14xUIlQ`XuTW z7*q0Jt@{Qd-)ZJK4V*ItE{HRYa0g}Ur7zq?`gqH+0eM|u*b8)gKPON6xEtE^S)m_% z$$un}zm|OV$iuQ%$-SibE$C^HljTnO@MN)-#r5OP=q$3Ij7uvx+KRb1GbsJr{xZM6 zWGrsA*X`cRIi7Xw+td5D=gJG*t;#*5)GzH7+cBvNIbNqf|XAIyClsNy3#7TcuAi`+!zR_66RZYP3;GG!?Nt(*PHI( zZO)b4BjwB7>!toa+N#&MZ0GG<-?TRFx#woG!HmDMSCaNMsRN~;cEZ&7`n zV)s6mv^ee_{$Q{kkUcq1i*y3xI`G({m&LB)K={&^J9!g$U_L%)72d0Mbt2nlp*u2+ zo^q+3u;=qdFL$8R%G0dQ4$dwf^U{~auKmp8_Y4Nk+GM6N<6D~!Pz4N;cI_;mIj73FxmsP5rC zD%vp%|6@#2KQ4Oi;cC^xao6&-Rv-ScwYiH9@g6a1`AgO_*sg!tM`egkk2`VweP{!G zzZ!kchpg`?8NHeLoGIow!hE!h`QHzbfBs-vbIyTzRu1R zySEx!m>kBs4f`0ei#Z0blX_*IKQ~Tte}m27IqbP){#&tQJBE&3=t1Zsg0>8&OwAn)2zoUoLo# z8nmhp*`t6v>?GbcB+uQ6j!}hHHyR@Cjd%WZS>xb;t8T1WkyNl@MRLK1D+U%EUy;JQ z#4PfcZ?*Be4i3gz?2BaF+bJt&hJU87C5(%}cW$%0(MN9Y6n`_0-BP!9u7?+2f3h2{k^i; z%)G&~YL8);BJb#U=PGk1Jm%EsDehrfgmXAN=QMW_ay~uduzS~NQShcvXCr3@?8drU zu>Ytu^LFoV_NFM-OEl*e{QM{2!Qbb-L)73;Sr4({XW4HOv3q1gFFG3<u>YR|)eqllfZ1T!{_6=xXCZSj6{T_A?7SCCu-Fvpfw_VQ8 z$}#tHa`{|+u9Rcg32Kpm&Am3B@kB8*^+t2gw`1&3`(e90B zZ_kQTU3+9sxqn5T>Xn!b?;>vEj@6uOPU~3wQenq$KbUXh&SiD<^1?jRhWYS+M|nW+ zCpxgbDs`;vnP%oJOPM-z@s> zZ@KvD`X1VL+;FHR%N}B0@r@NnYe2e_BtQ-;WyJ zJgya8kK~y~HILhaKZ`d_AK|Z1;jTa8uRCUO{Rw}^-6mJ(o0Huq*+W+IUCFI{8@V1H z>tMWoWBlRHxzKsp`*!y(c-ZFJm9OP+)|d1-e4nr$J1*z$7qRh=$ambqCcvBU)5hT* zp5!>IwVs_7qh5TCZ<%+A&Q)x)r-17KYpF8E{-q}GUT`IeAqCI!4b)>PHRCp}*jNx{ zcJXTe(gfM|cjh?s`x@3>S^-g^R)U z#;seim2cQ@E%*uULEO&Is~fGx$bxud&w>HQr~>Yj3dRxs9^ov~Gl|Dd+*t9;iEmUa zp7?sj!xJ}EjGg#F#n%(xulUo%9TnRr@|#78zb+_E+)z-H$aj?z-z<1J;k|T-uUP-eoHO$}}Bh$CsH_T=VRlVAfQ$;7+JeWOhmufuy zf!5f$#n9O9G&Xi2PszC`QTA%?7FwX~*5d}(iWtRxhKk(_e=Hlc*ZPMl?3H;-VWORL z-m<&pj&Dxw>emJ$i&W6gPXhg4ll%wkO4_-Vwcf#9Wuvs0KFR*omo|ovQ@t$cmA7WA z>GJNK56~1LSHw234qK`F7CyfA~i%Vf@ctULI-{>`nohR(G;!B~wXUPax- z*u2kUoLlL;$mR7~bm#KgwXYpVrjoP|kwNa9;~&QrNqeldN=th*iEk_MhC=4&INxz9 zR{nA9q5;O9`aLk^vXK3jTUF8h$d9J7%nUEzO3L#3(sxiUQ46V3 zj7Q>AAysAlsUtLG`aagZt+a>ho`=eK53UYnO;_(q{TmjQ?UXS)cEsS?7R`4YnB%q& zIEQO}E&5n8j>oncT=l*D-&kvVyQV4eDc+>e-<6D;eD_JlR%Bki4lUeu|1)JfMTegY z{@Nh^JqFi$@YnF2r#9rD0onjdkSEonBewwCW*m?BM6*+z%dEM(0 zzPY!{m%flXZ%hy3ya;cXJD{P^BC=qi$U7;7>y28~acHlTvru{IqLvR1<1%TppR-*udVo(Z6@P4zhoWERqEc?xuZ0M_%aSZ`31Uc^fJ5{RVTsj7h%H zbU?;M+A8^T%b&@JV4nEiiJWn&O2PFhWn>KpX?fni+uicE>kqXrytWbDMI|&TZv@0P zJ$m3LR*S)swpRE7Yr@YDqCLCdxxx>oGZqy(KZy3sV_mFaT{Mm!W-HT7d8sNjoHJzA zF=Kez{YHaxxpDHP&B!1Zko7O3x0ZD)bDhmO#LHVAS{;j_k)}E4!Opt?YkUj6ref#H%Y!O_iXPHk~ zm%F(WS*cl^rrpIm<=$cyGOq0RLvh2nBRR<($#CvSPBTWIq8oH*QI^W3Ti2iAj-<0O zGJcL0ZkZN{ceo?tk?-|8lJe~EG#L*YcsO4;YO(A26LsErC&AswTDi;q7y45|-EzMm zXLPyCevGu5z+LuS!lDEFuDk3FZ}8rukLP=O`mOJ>$4VZ3^yO-YE*5Lt%^0|&;k#pt`F5pIbCpE${de|%Rij`@ zHREde76pBJc{1{soWabT$BL6wSDmhly7fIrE9VgzBl!m3*U-5C4%Xm#lQt$<)b>~t zHn9lsrX;r8&T zaYB}B409=Y+%ImwGnjs8oNFZM#U}1d{B731`L%&IrXugkx4p6n%iW@^-3zo=-m*)Z z#zXq~U3eb!`L^GK^MALxuSXxB-QRP`JB5YN)q^?Y7YV2>QXB?8QC#E}zl$HQ!#mMEkmbdr`)0<#oS5 zYQ4CNbJ&vR<<>`<>#W~k-@UT>H|ybVUX!=_$hwr@pi7JOY-YUpT^Ze;<7xB+{4Q^L zn=vk}9i3Q*v0qv-XH4Fa>u1YhOE<5NGB3d?#drF#vND$FSa2Uh>j*>)N8Lb>u<>Y_v#7IPaFCm zlNRR`T%yOyW?%PzU!<>x@As`UF{ao+c;zi-J35oGTCDQ|^|f<0FXs-roi=6ACaGKM z-NC))Izzt$PhPDzwsY*Lb+0MDsphwRfHz&Ox8E)69eG2&7pNDW>T0JQ=c#WceK7NF zX!8ba2WWrd=c)^ypnnf4oOWburUOZ|>E$25SAdthZ)-Q4Uttp`-;N3oy9WC_u$y}} z|5biJC0Hg>WceXEx^0O~?u`Z;Z2V47x44n`_-^qK;uE^XLv6y#HuHXK73=%vyZ-`s z!&G?06nI5GJR^_0|H*E6h|cSNNSj0tF%y|V@Cg5#gzq-JRA~Fd*7>3vD(2g7Nyv4` zT+VRx48@w(xtTePU@uyY{z+^+lZmh9I|nAdU67w`oL+6?ohE6{;r4|NzUws{{ekFe zXC_a*v=`k^@w4!I?QR`k@y&vw!k}J>vzX?)i*>rv!qSsQGgu{Yvr;Ws1%kKhm)TW}nuyqp`Qyp-KQ+08NbXQ1QhS;*3AOubd+ zQ_@V-e`EUXznks1`)FbP?Z&0TKUZ?sB=;#g&-lSOmweMVXxDY22)nKj|2`3v$uww- zZ)l}I$v!#*w-p%0_f#85$4<$u@25NEx0q$W?(K1ruG={QXJHJAC^I^^2bbTt2m8yc zpp0S;Z}lC_;5WBp*@Mrq*RBcFUvoqKvL8#m|ATyyvCIJ(%fxz@^mj==kN*A$PVYOi zjC^ZP?usN|`fH#KzXPt}qEG+*-oHv1{v`HV$>fP0hkQrrebN(EzfSDqblk5I_kZ`> ze|LJEu{gn4EaDqvemN)3BjYBr%kKm2e1Z6r7k%4j;P#4q@+ZyJr02zXHjwu^;ReR7 zyGuytLNAC3HihL0UuQ0(g%e2FH{Q7(cHdOLkrgO{VXS`yzir26XZ8dR2;6E-#RgsCJa^O366O;OR4JpIPpmsY~iKL{}u_$nU8z&jxV4f{p1G z`jbz8))795j3?jE7dm~Ed6^KT>8A`g$dw_igHX?>GC#mxSv-Yt&u9Gen1{*C$0T_7 zMC4RI@AmKx;Jkb{02!oMi*Rga9A41E^QUs}vq_t!k3prXN_dNYA7l&Q=+6I1+?&QnRiq8yr@OOs z(gc!pLRPb|ND@Fq&?K^T0!j=R6m%3^2%821K|n_WN)m`{!5I!RZa9-5F63xN6c}N| z0YnkSVbgJ$ja7+6MT`qAFnRu0opX{l;@r>kez@Nc=T~*AuC1=Bx@xa_1AF}Wld@@YbHx-6J6-Wmqr-(Jjgov#N>%7dE`e1 za{1iFDKrTwGPmY_h_y}JOkBAit#jW;SnG@_y|VA>#H9;1u8}@zWtWX>s?f`pG1Lci zDC2wX@XFfmjAMVtt@AWhr=*{0_C1a>wmw0f`;@l%GwM};?mfH|*^jQfrat4lmQ4;m zHMi5(r8hA5_6U0??&WKd{^^x!F62-qeZNy7aWx~@k2gIusb&Rl`pR9iGM{o9K0A(% zAEb4zcEWbm0eTTz0NQDP$ukYjRfwTwc+kAVMeumr+$Xqjf)#96~(CNkw zTL0C2ZfptjI)|psd|*{+=z>+#?_aR$`b7@C4Bq?TecIJw?*{s8+O<0diR&mdU1yQ$ zy7Emo=ql3t#Lb&YBa4Q<4Z?V z8s&Qk>nRh_DYosT=ag zS*bPL6|`tzex&yDT5yijPp<{9l6HO~aQVRHGjBA><=T6rTiZI6G)>~%fxy0oyd?`9 zW2Ua+J;>``neBeCeofqkiSI8ZO#Jcq{R`rg+#?r`PODhI=1$6FS~~Wgb@(HtLG?e} z=B($O$RYP5%%`RDW(6>f^l85x6`ENwn>j@8IjF^T;o5mF@TZ1wrk1*B+mAx(q}U>Z zI~h(5Fyka_1Yw_ghtg>^in4ObES99`C#uFv1R@!?;6?r5yP9a z%juK9uXV1OH?QNG_bErdUE{XpExK)+@1cRceM4{Gmgl+y|DfK!U3USOy?vd0ujeI+U+eAbfxn;l9ld=+@Q)C`vv=W9O&{lqtl{3uGY;xRe`ie1yYMV5 zw|MS^XVq=nDi`PW_T|CTaR>gZd;506v+7R#0|YJcLP=+C@5=n!w^cbMomcm+eB%znC7lBZ zA5XZXVPNmdj=tBc62(8Lcjcw{uMq#>-j###7m9y~O6O|UN9}YL{{O&pn$7dB|Gji> zf~U&QDIM3uPm0!6Ir`toQ_AvHe6M~Un$6voydTXzt;W#oli07_oZWBHJWXp)s=EPo z8NV2erDT47=Vy#lS&X=59EvM+$O``GSU=BXaTGsC+(GEaF3^ZNYZ z-JO}IQhi&mqYj66cd4=2=j@?ieb(QBHD9s2Ina~#y|Ewb|LjRR${v8Vxb@ic%*`%6 z99wt)^9iACg3W#K|%WG%+)*%jyWggHooD`OWT%6-EJy-VAU7wADV$pXuu|X zMcXl|tESU#9MG;%?ZN@arKao5GF{hYI|f}&o2OI9RU1Lu=SCjp_L(nW@K@w8n&?gwtM?6}zVp306VV8=D=vd6e_rLW;3O)FZgIf|BO&Z26~Rdhqpu7OuL zwW5AbN73$&)61UayDKKRp}W&rbg9!-6h&MNak2dKMq}A%zG`6PjqSXTn$0M zN!SwNF6V2b<7~Lw=V*AAu*ortU#zCRILi33g1wMO7#AK#CXwqSSNv6PI9xT8f?PF| zzg_TtciM{(EvV)hx9`HU+@1LxGCz;Z8<6=~WY#U&g{R0XZ+(kA!uM_3rnh`rLwCa7 zjBz$pS32}b#jKg8X)ntd_k?P9;Lm+<$DBcX7cn<8X<*cvYTBUcprD!{XHZS`k?{ke zy>bB~yE&mSuns1N_e^c95&HWOzxoW?|Q?yTM+N6{!K6$?<5BF(4d9Np!KG)Oq z#k903Dfur>Nm)mH8vaT6KV_}(Qu@b1k;N&yU!0s0%A1b@cLskDankocJ;22}KoIKz z!R!$TVUNJY#{za)3yz$g@_f!+DJSE!3vXvkN!gGyE2VGNl$290M^=+7BrC*sS4s!% zpNGDPX#8dsZwl?6J3Hka-^7$JUBOvjxq`Cx&v;h9H0!PV?_03yir}E4 zDd_Q@U{6g5XhE-AgBw?SAvYq8i?8+5Kiy(e9E79Y1qu7p_g1p0XQV?=GoG zS+4XvjsJ-$pM`0!oA(EGY^tNr^cj71k?mdj-yvDklsuQ>zKvXc7rY!kL<=n)xM4}+ zN#<;aIFm~Y83#AyOiLN=imEvo6j~Fih1Ph`NjTqk?6vHGE8}(S>6xSSo$NRXKS(*{ zER23q^VV#4c5bSp4n7C}OEZy^G1~qe#VJS9=cLHG-&Z%^l)9U@CHIXi;&_o3(-Hh5 z>625I6ZhGKU#0FY8I`&(auipKh0^bjG8jq=@mf?n=oO!J(QYhtTh z9_nw1C5N2*wB^{194Dkt%N?}Rr)9t2p>D`i%l%~1rg9W z|Hm*b>K$YG=z7DLy^k@$-|3V8hA{E}Q~aEp8}HI)pLYG`$x%c8xaJ1_BN;#JdoVI- zpCfh5SB$y#(cjn)?95O)!A zz4FcSZtr|k`29)smhf8UKo;`J*nbKzcg0UmG2D%ZhLac1xbIGRi*Fj|`9dP6rG#R) zBWa69aPEE<`7{#!)y8s{A$vYout#SVd`2hUm^%862`Qtq?oRP_nUr!09gHL&R@|4$ zc`5CtG2pIw(3>Q07LExoPATqkPf9rYQZUS0$o`>qtWiC}-2`jG%{L22I%lQK;A4ME zbS--bo=Nb{t3>yTU-;ydjqsflJ|$)Jyg4cT+_O`r1N$)fv5|Zk&A;Hw|7-le&%Ees zeAC&lw1Tp+g0g|^Q7b41Pm;IO$@BK%PZ7StDj#3ERy=uq_KRzZ_}?e#daz^C3d#q2 zW^14$`zmgH82H{lv}~^ZE_hZWbIS9nyOLK=f77yQ5ueJ>hlBNHgdgeTs6R$~`e;au zXUBpC@4q@q&BJaw!and1c>C-P+S9di22=~yw-I-QzO2-x&sl#h#x~2`j`S%e;&$cB zmwB|g6II*wZk+PuUMG^P}$W8`mtO-F7*`Sf`Am49om! zg7K->k@*?%TbVzV{k5w@wf?fNe*$GjX!Mpk@qOs1TGr#lPD_~V%h)sVfmO?@G(9VF z<>>Ir=V#}!kM9(9qDw?YL$_F8gS1rTzRAL8Qe?5hBS1ITKHli z`liB>4q5Px%u5NaJ<}Gf;w_)O%frHJdcwy8p9%2EckCD>^oBxbXmHg) z_F29>0sAa8Dk;bBINf`faxT`E%}4v@h#btjjK0_D52iY=(YDbCQ8v9~h+|AWbC2?V zKs;w^N(Re(MOacS{)CFyB=@CNYQEwMDHBoVo&7TYUwU2T=>E)6R19E0J$l>4y}lJ! z^G1vHzJqdY3Q|!%d@~v*ixNU1I8Ov7>>|N)(ecM`%v3&KQ-gP_fAe^y$^Ghrt{aHIK>W_$)kmZs)}dxM{ScWySO>ootWmrLze@O3 z!LPWQaXkF0;8$GDI39jg@GGun91p)L_!T2lCH$)3SBy-R@T-DfF)~%cuL^#}$TSIl z)$rp!dByK|Sm&A#uL#pm&_DU#;z?Vs=hr#)biR{(L!7ZqYq^V%eT~d(csOt2sfpw+ zJeg;lz#7sMMV!}n#;Em&Mj5mA%wzn@8+zI{#SQP-b=dUJQ6)t1e1DV}J?J?^Cd(6t7WbT>& z?{7*TVO=Tp0j)mOt@$0;*<B_e4br0jjE&8xNK6XHZZ1Vm)|*GH z@3M7`y!&(818di$UI;!jwWav|v?qK&7R!18Fx_Y?gU$BxfP1RDwY{|G=+`5&`)H_Y zFBxZBGMjVspIPbrrTtX)`-?|9?KYG)=V`hHB4b86YXt@m-J zWGvt3>^H;SWDVi0T0ZTg>|v1e5VF=;#78Gfz0E)!jg~$F=UAdjIhV5nU2W$*%Csn1 zoAOY16WE6rRw{N9eSy#VXJJ~-nQ>`bH1wWMxtD&V><`;ln4UvF!*gt0y2R;2T$;M8 zbOK?iT6mMZUArF`Cp-gT9E8h0C|Lvh=6YG@iZi}eZjAAja=peExRPIT{$%O5lqt$% z9xKpS$O+y4>^Ns=jj`Ty?}1wKos^iF|1%v`$Ak+uemXFuL;3 z+;j9>w5=D@!x84dRr;~t=A5Y(v$L>A&Qp#M&-rmZwqR#SXGoI|dw(}!hplwJ0gSxw z83-fkl>3c^PLGO+q!z7X&ttT=Zv|0k9y8FW$9#tFPtuS7Xa1!%3q4xHLXV^2 zeBq@vRiTcCs!)y3#pmP;<_qErZP=|jqN+ke8cu5F{)!eyLQ^GUEur%nALlMMg)nZr z+Y##7QP|`B`NAI1(;7~)&v!TT6w=;&#+Ygd@09Ie{aey=sVijfF4A&>v@*7f61yt- z{U_3q%Dx|IQ{&O0jMcc;f%&2sYppZhSVx{onM4nx!|2!47Utw^XD`cf()Mc z&Nnt<)97DgTPjB{1wYH8yR8(uqyv9oMFG+G^SBTfsb` z?4=d^8aqqNOam|GuUh?gYW^?A$Wgdn<0@RcT<`gpS+1*aRk{8y>}R%;^Bg~4*Ntf# ze{~UhjG>AzvaY*>{Qsr(;27|V**_PJTOs!~vu8AL?BXt^{7&^t;T2El6LJxn|NXnjj?zAnbtUrogWiC==qI{_R6k>nalTs?>!+op3 zIZI`Io%>s(8YwHZg;A-*I|=J>Ysr>c;$MI^X9-HH59ZJFS$^4z;i0d+44IgZ*h*R0 z%KCPcl!p)DL0(16ehPSxmNo7=_C986uF|R4iDCSA=YNPJs5GN*bz+#q<&pWX5y(Cg zIlg3!wH+Bu9YthC%q`gx;W(SU9a-KGU67{n4(6B9QD1b_NckSZdif5IHQ*kx;52M>R(M1}5X%FOI_LGorrBB1J0X(sV_3)~9xJF1iKS#zA;zS?9FARHQ z=Oyn58g7THbiL$nsJRa%m4C4_Qs!+v(q{Chjp$DvY`TFloyfnMk z5m9;!`KOad)xA`dV?@&FRw$7U(QLnLf{(VFL7Py4@(>sH%8oBEf-m1rZbPJ z2Wg{1s$VXO4%SA6R=-m82ySHc#v+}tRq~HN=DS^k_Gu1X?!u1^(MEOVY|UefpDTKN z@$*GbEZ$J`B>!s`zg+ax;#Z1(zj$NOo!QTpRq!q4tLGcHgtI+-Pw~CZH}3AWW%K!- z;(ML%-PFGn6{b!o+grA-?8nsYMf32V!{7Om^+k{3-|ui7Jhe<488>rX+5Rf+;E(wC zJDmE5%W6kn#r)=zn*dSv2yzH0!}t``w%u zgwO8k=ZiQCqwkZk59Lny6mwSKuiT~l8`_1LS_hBV`L7u}PKWjx@;He7rM1|J>dIw_ zEskL2zIp3qTZTBpOZ#52Jn`kRmu+EwMYRvI7fa|~YvoflvISK?Qxr`81Xn*>v=}!8 zIVBB}H?%9GB!8li>l@&jf%%qr(cJ-bb`|oE=9|m6knc*)K7B_zvT^(JMJ+dV`Vl(a zN%+a?4Mn0$(c=WhCMtc@Nzr2+c1iU3rlrSVWT@utcUzASsWSCUk*%+{r>`r!6ds>J zw;H)@y&bjmR!8{_)SKw*9rV)T2rvD*@AAYA?evmPounT$dL;FLa}ZH~WZ$UBvLD@l z%{-0B6LmTBH63~Tif@C~BSp&jE4Wg=>C29ia$KvK_U(i#D9b%}@|H&00I|i=2B2@f z!V#qG+uz88AzDyrG3R{Tw*r@NHoFJ^*CDr!YukFM^MXtNL)rSb!4*C}?)DH|p?@@v zIWuIB!9EWMz8O2YlrEw4B~r!Kab~D9HuSL5Sz3eturs3c%|qBa{BnoXcZ}Co(~p2h zR0*!cA0>~LK% zUjf{*#O<0kE0=eA#=F8x$MsvD_{ptTZTWHRRa-EV^w;UD54J{#5qV%;(CU zn!K&-fj--m4H}rdwd}{me=Zw0^LcF5YejcpOA?oCQt)>I|8L;`F!?WK3;O(}?BCd{ z!R_G3EBKccd;#$88PAn1oxHxRq|f>?_ms6|8SUV&Q}A^~Nx;sB_L|ABmEG3oHDwdC zQnwamCC@MOFWyp?i#)kY{#?`(_`etA))dxgBd_MJ&A%61T~i?bp{D;%@lP`Sx8sjr zvbiXC5qclW89@5Rwa&QGO2*i=At|$uoNl@4@n5Gze@wl2REzh-FL}0TGymT?V@EuO z{(eCBqPK)ry(zxTobvC;TINMn#&j+UcTSP)*yu*CcS>7Uw1Xk$C zdlA3oJ{0zo=^cQPIY0ad#c#hwB>kCS!li#v3w_!D>*=O#mG_U@%fVZ`*kJ6`t)TZ+{nne1+%|K`T}9XT3W7rFNR!CTEfb=R)lpZ#xluF0E{@~(o+LAaS`&(PMd z3A^K(sfpbyx5jH8J%Kkv(kJAu(J13SEwsU>h42OQ1@XD~oO}*Gt)VmT%W3TA8PDBt zY22ON=!(tyT}on&#vJW3-WM6d+aRaVcMWeIJj|O1y~vN{)*BMw~HBzsKTOPBvoxH2_vvFP+hLxt^n|VJJC2+yZ_kuCwXHeeaK~KqpXu9_v9bu(~v_&hd{1X{e zeg7|cC+|tsSnp1WtcnI@SnASH^5*e?cc91sYOwz@ei551_2**y&h58STXqGr-WkRESv2ctUe?oMSXYZ>y|Xjtw1V|% z!P{vW{ z;S-CZhx(RBSAV=DddhFBqOWy?)r306ZSgv8-}0^I&05VlNU!70E$p4g^=)PSJ^L$(9h&w^Zx(1 zcuHB%8du>=nqT2cda6%Q(%wG7Nt>s2N{XKunUp&-D(Pg2mh_=BGU}w}@%$b9aTOl# z-?jKHZ#zRroP5CJ{hqkDi?yviT*;n~i2q@M$NQ0%*c$(~Pusc#7>~>2`4$?hiT^-L z-qNJ8Py9iTcQ5v`l=$bJ$(}jPf6XEOeJy26sUy^k*J#f;*W%_IQstag#XcQruR=-x z$$L}Aj6uJ5Pt>;FozK^Wk9v2vGiA%&&SXC1K-Qn&)BkdFzhiImZXbC!le}9=-pwNK zD#^Qi@@^4%H<`Tqlr~M;0I?C7+#{XN-dQ*6P^o!Avx=FAWsetoh21sT33-`W6Dl&B zoZ+5cB9g_A^K5m-gNxJ%aU*>@B3o;8`NuHap^Numdm!8buCA&gAy&PW863(Rm zhc~%@{h*e#q&z68=h&#E>#BTF)6hjvY|QirqrJ5bXJ1(}^&kg(xN@;QBeDn|Pq^&= zps>f(UU1BsfnV-*Zx2&L_;}(U#{Hc54#1Z_7(J$RLc-5r-=*G4dSxys6PQe3!YS|m zyTX&Exe`Z=BMpVz73*<%J@X%QCOtNvgi%&9xgYyhr=Lw@M*z!%IehNL&qsTCW ze~}@OJ~|mO!j&-My<0mb-Bs2p$?HlUaqog?@0F zuzNP2+4$vtx2f2ZpP&Ew6*lK)Y|l;DpfT8@(X8d)$oa2eb^hza;m*~+bO!l5=B&P_ z{r`dXU)CMghjafi=R}Wi4lo^kFXWTEy*n3}^Y(#ZQG^|4ec~C`C(bheBXgshqUBE8 z7{g0kqtnN|dfL{Hu~DnJt8O59Cu;yg;~0DPg{}sijS~`p!@YU^WsO8*d}`h~8*2m+ zFY~f=9OX=Uw41wD-Q2ZG{V{rS7M-=Q!V#>kPv_o%@!V56jXO>MxBnT#se3BFb!BC} zir$xsK0WBO>reIh264Z1rr4d#I|qzu3+ZQut@qW_UN&7Bi!B0J*_f-_mlZ#X=Om~XhJI-E_#^r1Td zzbsvwbxLf(Cb^R*tmy^PQQ=_!9PytcV>oH~BWdZ#*r1U2Dhs(grh`gLhpb+bmdjn4 zU$VYDl`+7@_#CA#l16#@$z1j`zWOYOlfA3%CYj5WbO*)@Pv)?VJZ!%7d1c%reco`Y4-3{D1`ATEg==#rh1;5hR!&_f>Lh!VPql+}%*{yQ)bDVjvV{Y^)ZG)^I zI=P=>En({y-T5;cAY-Wu^GzFI*ZEJbb*}zzeg2k3oem=V%2wIwlWcFvsoy>)VM{pq zHlF=Ch2)X@MY-q3-87ZEl<2cIRrVkZ{mrn(JZ|p1adR(^lt1=HW{o7z+~k>zt;R=~ z@8*pl|D`;JJs3P@WGrJc;>UN_W`$9f)qNRK9^1vy@4d;R%8Z9HQ}blumD-I`ZYsO7 z=O<5#Duj=W(OyE1PL!R$Qg#}Uqn^8un(D?u^iDxnEOc=%8fi(DYu_ ziYKKHTbC^FbFvrEN>c-T8`A%oxYd!6^>9}$Dnaq?WYorKeb1v`;V*sEFIfxb{Cae5 zg=@^V2$S~=>_V@py>nz8Me4x>>WQ4ClKU3qu7z`xC|7)Ik!vFT82SaPcW|Gg%nz~0 zn>&SKJjIm76UW0?8&5Q(?Io?W`O+__99`=&?;8@{a^FzI>oRtZX!?r#Uk*DuHTjr# zi=a+?EjkR-{0&oQ{xmIDx-w6T9#h-WavAriwv>|{+^hHiWuug` z(SdYb5AO)lwOrZXy@_MVBa!(O`BK5%RFXIBTk~gzN}8Nwp7NM!qOIE6B|?iT&|ID~ zyu+C4h${VpyLBsfbmd$-=SM@et=Cg`Mej0Svz_z5za$g)d;H(a6dNFuRaTzr)Fx9p zGGzqFG*ZTe$RzR{y2g}8%0^=kUuK5G?Qto1XB+*&un5R+$ zObxohQ-1Gx_J<2Ej5V^;mc~EsvOk;jdjUzaS zaCuk3Yp{kumqm^L)SFB6yMA`=9vk{!@6?@(Trzhjbw}!r2fLbuTfrJ$DP^-Gc2D}; z5o+EkqUlTOj=XEXgS3flWBo&q!v5@|?(AdEpcC-iBbRj#Ww4`9>Q9tWq23XXG)7R~ zny62Ib!gGDzmU3dW5-^uO!gOMT{d^x+}@6k)S-yduGm~R{(7qpDf=Mn71W{D{sD6e zZt79Su#6gap08Z`2htWwe?aJ?y2J;MIZeMv;u-j7uPwo>ct)?7}K;G~UzH;u> zEe(?SI{rP8zVa0g)_1AL5A)w4M(!2pP#VPlANh}n=L|k+W<5B*#~NlDlQA$!qUGUwK7HXz4QIM9)?` z{*yA5haIx}g!Vl(tVox0rm$S<9Qz7Ajj3As=Uuh(5=WS4nf&+Qzo%Bdp8t2`zYqWY zz~Mj4$$vKg1HgBNd8TUo58;2PR=$+?5bNcCIR7KG@;&@Fj?&5{-}YixPLXc~VZK$| zKcx1pP=?i7X;>wD&Sr4$uq%A;m!zYqtES6*>cgz-6*(em4qu|3*+Cx4JaBK$68%vh zpR-TXHq9(Ao4fK7?I2}3YF7DWb1yINGq+*w_?*8IS5`h_?r#&cgS59%_muaZ+lTnx z?792n%K^Bt?QnMn;Bsf>&v1DYtCep=J6zt%YULaDbKC~r(Q1un-ufrBOQz>Ea3)`5 zrk#$OiM+j#cL(FMWz^S3*0;mak?2M0Oc1)sBwWgQ@z>lTQNkU#+*L56eEMAay-{W5 zz2;sAjUT`l_<6X~hRJv>m^-7Q7}G^Fw(~Nd6vO`cVEw1{E%|IlH}lOJxmTpIha)qi zTVCtEA&otp{I}gZ(%8erf7`tyjXi?+Z@YJ-u}3ifZTF5e_6Xts+5mc?{0|TK=Ur^> zNufOIoQ>PslR1p(dADzQ_>vXghU?xdY={_HSXMQD%l>PA<2`--zQV6EiwbAv-MOWZ zHr!xNq7XYY&B54%IT>@j5_Cq^^28U;B-NV0k zcFxM4Hhl|oT{5@j`CZBMEv)5ue#f`T$-Eb1;^VFm_=FlK-AlYD`|mD1dF$@NUu)jN z54FzW_iNtp51ny^Cs_+Tsl|F$AivC?o8#zU)-y;4_Vkn%?%B&+Rd_EgvxK@fl{)C| z=Ez*?#QtjBiP+7VS)fIFGPO?lU770_-7bCmpV)+>*p3VHZ=wv0p)8DMe9yRks5!3Z zeYVxit8eH#_{!+JM#Qj z1?TzG!}9#W>AuWVhc@dw#`PH!+`4P$_OlsPZaoe+EjG`;gL->>sH6VVP_2F`?cArK zzWU=s^Xe7tvDf;iOw{Vjr#b59&vw?|oA0Xsbx2VCeI0}A?{|jO+p?5gEqjoD`Q4gX z!^}6c0>_|=zGGUh>_?w~Zn9{nee5-py7L)+)~Z*|Io4B+neHU51-%*LthLUaIJjrc z;ox)fx%h(kg84%DLiyZ$VT{?l#%OS*?aoA3nG>$ihg_)L!`g%CO4ylRdd7v?rGev0 zS)Wtz7l~_s&z+r)5>3k<=i+Wj(xAeoT+3Ke+f+B+yqoR5n>ER=uicJo{3Q-;v+ReH z^V~B7!s+L378vcVVgByq#S!*wX~6U>9t@rEQkA1(`DzJ+!|wPu9b2I*r_2CUiyjQ31ScTCH;Zb(p^eeM*_}0UHtk+bUPY zHU3ukgTwsI7JM=AKcWlnVf|@3q5|OmHq3w0g0BSrtBb+M2EczZ%>NGyz7F_f7lTg- zfdAJp|9%U82k;+W4E~Y;_@l%8yMXtRk4J!iHvs-(I{o0=mMb@Y+tPnPKW+5|*Cob- z)N7fS@G;LNb7_^(GV9D`?1}UFHOG(-^KJc*H^n~eJW$P%HNr>gy5NZIyAgUv zWE}t*$dd!^hp*D9)48eXj@;BdXRa!%M?32o-F@|`j zKQgR#X6e8C6Jr&^`VqF2d(}z%tRT)T6_YPt%i0`ks3KPe@d9@#ZZrCtgT7+WS1~eF zpsy48?D4epRgAuh(N{70sz6_J(3hH{WW1pG9_pnpogF0Q!e6^sJ9{7L+Dy9ELn{oO z4MAsOZ%Q6r`lO7F+|^-cJ@BoY7@~(MH&_qCMUP+qzuMWliNX3=WUc*A?NU=`KYb_c zmu%35`P=CWY5Eu7s{D(}qAhdlX`~?&S(#VLy)b|0dKs^1+<)1A9#iJ}%y~bj@f0*F z!+rio0`|kM&UeOW>?aUBC;K539nL_if9^$6|7qs>x-(77RsTLschGOq!EJFl4t{@& zgEreqzm5AocgZ;b*4Ej_yPoqnQYK@j4E2|Ak7b5K%XYim*^NPw*?U6bv)8-RvX_Qu zWS4ZfJiDM{PPWWhNE%}&T+3O3SmPe#jTO6sU*6=nOXTPNvWcDLzS>yh60zsFlL#03 z1s6J91^$89Wy4f_aXb89wZo6F@W)vACFkNy{D)2a8`|Mt+YbK%wo3R9weXkvQ%!u` z#P_wszp5Snmll3M3xE9&Jxu)HnfRBt!@sm0{=Y2zt``2D@A{bdPnh^=?eIId!~d&= z@3rt7o3l;)-<$Z6?eIg|;lFO-yKH<%j)}k4#Mj#4f1mf${A;lAn+Y!_PoJP}sXl+~ zaDQ`GtzOEGBQKx4WL*T93-gCk&hqISa<{3=y4&=BDD#JfXS*{0QC?e_543ZEb>*Po zb)>FzY}-5jj^J5!+FX3$#Jaq6#=Qg|s!Q(DlRTsI> zN7cO!)_(`uK%qi`(JHS@?HY`0eZUxOVtA zw8Iaz@UOS<+t=&DcKE(__&-pNB^`q-{Py+w>UQ{-x5Gba;rF)i+t=%?cKB)S@ISKf zFR}33*XzV~_>t}K-?s20Z2WfhI=mge)(-zw3;%+TvPGV*6Z^ne#hvi;dff!f|Cf6G zIe6{s^-&-7rF~n~zP?y(*D+vKd&(H3fWEraExWBvP1a^LE~-o{#{OtWF4M#4ho;dF zbu%|;x3xB`4P*CJrJriU2z@Qjqv~y7y;OV|*X?0EByA;kVytddeOa%up8bAR%mbvq z7q0h7H~Yw&NGoH4GtG=o*W)+)4{d#}w!TAK*neFQb6s+mMPaD}^?{}FwR%zreJjS7Q+wGq{@`!#v4U zSKWEGVpYo7dHh#Ir{YhYTX86L%ba6o-^{^$sN&N(OQ%1VTk%SVEeD1;SI;R6*GuS^ z71PI)HOpz>OP{Zj@!m|zlZ+k82=mEZp2+Feul1XEe!Gpkt@yi$m%h&g+*5q^c;OcO zQa>Z;Zzw(V_Hp$mv?X1Q{dEWHk6q9~3~=(cT}O1m+SZ0SPX_A+gl%_4pLMf;u`$WT zJ51#92O;{}MXrO#cE#(gCmjmnYY-mn1!znf>Q7DQoS8$rX}xRo*#i1A+Ona4u=#56 zW$okwviTj*O8+8Qmwg|B@vU(Wgy zX+uVOR<4XsXV6dA2$T6xkw-)S`Lh;|{+hb4@^fKt#yi^9;$uEFE^%Cw@BbdzsYB81 zN#9D|Zr#(Fy{-APavRyVWy_(;{*DO!M)=oJFI)Nc&C2z{%Z5|BnjESR7dXcBKMB7l zRL>*)2y+EL39k&*2M}J&S<`G}Zd`S(zY+hBj2}hc+KqXBCwY*+f-{(;L(16SkXdy0 zEwBn^Y@UB7VN$lG96Uq5%NSJFQI11L`b?tl&E&O|vF*Uh`pchj3EwmkJD8v5X3mCv z`q)jOb3~1OxtCR7;nPC7I!;-Vx28`3BYQ)>qZ|pn7l3&cc-bd8DxjQ5`BnM#xLdE{ zj^;b8d`aCL>?fmI%Ts6IINPvEXx#^GN#j${6dsS^N;#pwWtPR2amd-($QHSnV_c@( zSYrvUoRhpVfNm@Qr{eXa?5mafGt7eP5daqt|0~gh@W0vO{{V2@pKIKYs|E7EJpul_ zb4R$`w>}j&3R+7kr|$21s&mY$99VKj5HE6dGGPt`z^qKSXa!j?lAfCt-Ok1s<;EH} zDmTU$rChIZJuYRNIknz;`~)rYd%5e3I!!q^;I?cAI$x=+4As-Id6MQ(aHxwbH#__3 zbyxWOlS$)x>{gv+$8w05|D{)In{I*5k=c3vM@X~e!_~wwPF=Zrc9#CgZ0%r4=RALL zXNR(XS;SQ&I{X!74u36nQt%$K@Olwf@^@hWZo;PWFYymr@!cf;3gWLY5f@vE%(of5x*_yuPC zCM$lq6~9&DClWu=jDOLJUuea@Eb$A8UuecZW5v(2;_D^;F5>Sp37ew-D*OybLkFEisGu;NEs@l_K4 z8{&Us#@{RdQXf*7E7FvAUFpDbN@Dt;+3ndHxNS9ODTW+pEH9GsS-QwQz1FPWF$;`tFg6szFB< z@h#!2sTx!)^_@2I*seHz9kxG+{UkNeJThFPuFAWsL)HGM#?r9*X71i+9|7-5nql%5 zsLVYw*15^1(;Z%oE%|E^Vo#uQy?G zCpVLm*wh%~QvP2OnuC~MBwXrFGyekH1y^(t%bfTn*huEL4+*W~7Oh0$h1LfbS~kZL zC%96-1lMcfen+1H`;{yGRHZv~F8do#S++b3II|z-rA>@AqsQEnz-s zZ`0CfrA?0jYZha6;|4I!RH+C=RcD9jAUc0{70MZ z;GghI*vrcG8ZY5~H&d(sag>aQ6ODXym0y>p&$9IKyn;(Ip0(iDDVMv_l^bK!;{G7- z2oLA}HK)7D!PzRiyxQeX^z<+2*=?1yMa{emp~~^8md(OUecVc)-R99xKA=s% zQqQy6`9k2Nos>RcsvhFM{$lX)SLx{%{9xdVsk;@}@O#BieXhXX&k{ejxdPiGm97;3VEltM{}uRUtkLY0eZyYk zGXAB0iOypwv(gXhL)aER=_}pDzEr92S^RG_!|Le^txT`(smr}#Hg1B#jWyDMspm86 zSF-Uvc`x<)62ipR#p6o(A>O2OB#Zi#U~CyEWi!zTCr-+!Te)5%1XtGQkVX26E7Nc7 z1$?aWBJhD_$}GcP<3~k@cg&O(N$S6Uj{X4u(kE8`>vQzE{EKZ=|L1e`d-)f=ssG|!{T}`$UsBmqB4-Cg z7h=1EE+)fFa za@ojTW#iEAxIR9=&*k$k11}Ri%Jr;iu4PZQ;FjX1vR|^-&NWYRXU?qNxMshuw_e7W z?=b7^Q-S%ihqL~Ru8#UseYASUd^losnuadeDXH3Sg?9jv4`r{rB@!!F}nPy3AoFRRi9l%OHUX8sPx2V4^ z={UdN=|4V6c*Gk^fsuS{-Y@jI%Sg$TY!u@9*7*Em*J%FX_zUnCKugh2leUMuqAXmg ztAm00B!@L%=r`|${!pQxVB7_TqN9f~s=UVjZO!`wT zT&b6r0+Tbuq<{Wx=yx^cm%fXV|7}zLR7F48NEG_(kbfP05}}X3agZ&)jI-j58w2F; z2+Y>Omi)V*?=t20+4Ao)8A+&=aK(;&EGNr`uLaH^y^IexdHNj56rC={qx(Q ze>_&{Kf{)PyD9%^;Kc5n!u7p?{4Z$!lR_VVhD|>$QOTbkApgIC@e{A~{~Gjn3H<~k z(w6@eXnsy=)Z*gFL72_=;L2M(AK|6U-~wZccTB@z|6JipMMql>rDC2Vi%SE zUp3|5uIMKle-`>1k$HhE&oR7FEZsn0*op@8%_D2 zSMKx%H@^V=p{D#BZ24a><-cFi_ZqW>{&wWw zuK8yQef$|V{W_Drv|qOV?*is5@k;*Zq2JY%|8ZOX=S}&?EBeXCIHA7_`FCmlTZBIT z#;cWnHsq&WqU6^DrDBFEBam|U+BM${BLXiAwnPjQk#CA zNxwWm{;Pnw)t0{&`p08b`Ir8UDnGTR{CyPtWFt%H??wK-n!kt8$Dd)-PfJnqPY94d z1sFf^s{F5k{w|@PU`YQ)m7g`H{BerD*N7JS`;mXY=I)=v^zn=R6x;C+{;|`f!{ieO4H*YXJ@|mQM}QMO zwfq)7lZ21izfpXCYx4Q4(EkVY|DpNc7W()bbF|I$FE&UYEnfArl8rj-p!BN*M)L6u z+!0F0nqReZb48c&#zew}|0dkcywx%W{vzv23;$KZh5swK1KRL6q@NfFK6|zle!TGv zuFMw?1)sji2JN4N^gOG7GnqIqdllv|7iIT9A16%O^|`d`6}0PxW?xaIAwi|VYs|D{ zcobOa=@GRp|&Ws$PKh@&8W>8Awb66wM=39B1LACbeo3AS%^2v2K0Dz``DvxOI!J#p0I!wiKZ6xb?nU^SoF7{> zTXIb~f5esY(@Jw&h(0v{ua#zyTbDb_>~z~QzG=~Gf{w`e8SczLdMDhv+$&<^wbF}Z zor84qo@r~Gt;?4znjcy+9>g6NNb`em-4}q@O7px&-wDm4fH+&uCoP&gEjhR0n&rnT zn;&=3#|GfF()>C?uVUQXAt26{^C64oiG}m|1(~+}VK%6b- zJr>Q!EIC)Th z^$h`dtu!lB^$F0F@$B2=J$GOpQtd7Cdor$&_DkqVyLFcMTSr>`wi?>op=Q59`Zs=H znrqB{!x`YDJ^TXKcbI;|Va@+J{sR0NBdmTKb7DfjtA#5z`Uo(7;#I$A3G};~{RZjZ zhz#eKnC}2T#))imD?t8AV9KE{@;@lJBLDsHS!c?>!IuAiQ~vt} zrWyI0HUBK3kH5gCpVn2$|3ZNLCBVen^3Q?pB2)e+ExP_WLN~#f0G!BwyU_mu`afv? zu|gkzV}UKdN&gWGSM+}aFmtT@_0NRVW1tQF6dwd}@89m-o?3M+q_wS#r+GpK&La)+ zd(ALx^_zs{N!V>|VJ8S1NSLIv3w&DccKXeF*G)gsRp8`}&Un=ilCpXJZd3L|U?qQI zas8txm*`uShe@Q@MS4g1{MV9RRh}maj@O7#IL!ay`dTOtEt)?Ve*u0s={5V;-4svh z{|N38=$y}Xu72Wrvpo3mtMV`&KF6a~dG2ND{``1Twq^yB=O5#8_`k$o;9?ySd^26$ z75yF-j^y(RV46w8m>bN#{t=tUZSdFy4`lG#JZ>|2{9Rx|;SuWae;_>YmyBx5vrR%1 z`3oe!j(f_@eDjpIj1MbU^XOK-{R_IXE+G1`^X+2^+a7I(S^4(9guT`lCi%9HFv&OJ z)iNd^-xdj6oY6c&le{p##r~?i8)M4;0`Q{O=Wuh;i^{iAQcg{Mehj>tN-f~NHZsVBf++*uH>vnmqmGs;u zawfH=Tk7AjTdXiE-M^EtF>PUz?v*OtUgOvBYZ)Gp?yhFK&yknMq5CJ&q3qmnGu<N`!DoUNkgi`e^BV-mwI8Qp@&L?Cm;=bfRQwe8EvJZ2Ryo(Y4BqYC9VD*!Xv@h zp?D-4uL+N?*qg2n{}$nazvMQLJ<^#}@Hu8R4Es*9)_7Vo*wbG2gRP{IC4&Z0Y>9c6wZOJ(iw~5ah z6KqI#_s|ap;I-1M&d?pmc{m`>mb0ftb2M~B&g*eM45YaK^8t9RG>7%p*J5Wj2gKQOIxLzQ&=EPi;!62xO?&cX`i=m+ zR+_i>QRDSF0dXeH1mg?xL-O4V9g(vm?qh*82VPDc@fv@;Qurhq8gb0290;6qI!By6 zr*am*ITw^*yb!>zRhH_$N){PQYy!4Dtwsyqp1(Rz9eE;vPP|pN{$k1fDezKPj^IuW zlzZ|O$Q^G~5-#=Z5N@2Z(iO!|%s>i(| zkk_XE@JcYQBV5|(HMq>r9SEGCdz3JHe(n+cW}bMBu>t&A^W?;p`c7-@e$+*1Ewb=4 z(3Qw7bAB>McWnSITTi!IvR8m3CDV8P&Ab~sqp7~yp_?k>JoUdg$N*WWE(y@+eis|R61zucOqWS?DY8A-J4^?%T* z)Z6aBN*R%~OF3!flW{eC5{(^-Pm&Q$Tq0#;9OYlyoAH$Y2*P~i>3H%p9KT&g-V5Mo zr|X!-FAO+IR}k*Qd~N&&sB}GJrR#g#_GR)6VZx&yJf#f16u>7*rHk>m#pg7zvj653 z?wmkAH^}^WlChj{=|B7vw|%-kBFs+L2l&l&B^Zwc@Uzn;_EYruK5&w*eYm#-@|!+L zrK^l^sdJ*w_WAW1VZuYzYx)i{?|xqZuLLX4q-_;mTY#55+lV_bkk?Xk-hB+=lE!Co z+v~87FuPv=9>1MuV*~hEb>o1haqh>Ow;6j8S`Sqvta>owm!su3e9^hnlo|F^0m@D>(sXe)aO>31wm>L z^t%CZww%(=3eE8r&2hNL18EKnR{8m3KU2peiEE#q!wIwN`Y`;WW9geq-_6H=^8lIq z_Zoxn`x&cAxwsmC;|S9}%J}Un{+s-5eWF&`H*l7~%9C%AVaCtWkYI#bX}Am;qU%iD zSNK}>vOZL$;TbQi(SHkcM0U~hBJf*fFP1kP6O1P<*}uSTU&c=oW|#33_(gWv80X z22N<4Pcr-5rf$8)n?iet=J!WhFXl(V?%uY??|iQG{*^!*VqQE@Ol;3*OUHw zPltaK{sR2#!55ef{I_^5IqHyOktxS<%GMd^HH!>hQ-<|I4?gQ~=h*x^!G99`q4hW5 zMaTY50-sMIB{27XWg0YhS7LTomO#Q^rHvR{#^qlvY{Ff?v zUSkoi=-`u1mJUK?Ol0an`fy4Ip{5SzE1YEG*SNlZ=%Am&KL>vS{-xlH4rcPtJs&cc z#W)oiTx~LJuyo*enQ6LP=%Is&xL)?aO55dcaWKEsN$FsXIeWHUo7R!W|l-(PEm$qjF?gxBU+1-#n zQ1)#l8BJH1dLBgF?btP&mTiOLEINaU7v1FIz7a_0KnFeB-G-N7wGYy^3jfQ2m3-}q zE9Iw^|A7cSKCBJT=683WKI))OIhc3_wB6C zf3U9Gs`rPL-V=<&xGm=#{w}g_HZ6;~+)34CsYB8a5}D4QGt0wA3WxnaxKf4_F0sn+ z50s&Glp)HN%fe~?L2$gry9y`ScnjAzfHFM5;eP{v0e;z!E--)Lzoprdp$-|iKTXZY zB0~%5RQ0&ol;KZ84?ZvBJ_5d!;q#~AKgle^(mttrd|KcW3`5{4ug~K8hERrwIQ&oJ zFTh`*_$L^D;J>BG)UlVc|5NG-s>rVRH9J#=sn?i`!{VX4O* zRXu*%*1=&@2U7(e9Zbgc4MzvV9sUXU3-D(s{t3n%{I~qW)%dA7W7oASC9e3Ie9 z^^GC@V;p`B|2f;19v$v4iEp!|QqPm(L@!CkhtzAaNf-K>w&g6Yl%G~x_NL5>v0g>E zr12E)hKtxzX@7-I6Y)}}Kf|3FNasXn{RHPXZ5u2yNWT2Z!uuESQnxrGlI7hc(JA1aIXo(8YO0`WF<(Qm#;`f@D1@xV%X8Ham35U-@0ero_fGkwgzTX;7DE9o1F`?o+m=F^|1 z?fqHXWR>I3E|)YV89Csz?6U0v_0iQbRC_*?7ar_@L8yt3*q! zi^2UpUu!;`$fQ1cjZG>Y$wmlqZtCOD(rdvPR(j9lw!CKBpUbep5v?{@`bSFs*G&1p zRXDtdi7RKPyt6GE{HoYs(u{3Pvvkz_s^D<`N8uzJ$8deOVS{dS_z&YRz`tjfX@md4 zf6GQ&=g6?AO@?G!hK;5S?+ZQn?8AKoeAO1eBsN%d&K>T+OL_CZB=8Bw+X9b$dIQ%t z9veK~;opwG0DpnvpJ2Skf6EKD4*Fn&+xVYH?~>=|UoiP^RP>ln#Z_(bJ(dj5iyby~ zApIM`X@1_+!E*{H*;tF~n}iM~IsAXXUx0r-_@bvL`EOZg>i`*CZ8H1=dMY2*nKG;r zdg$O`+&MP?S}AuOln%B7uXIpr>R_?Jqk}43-`(ioZijyX{sR0NihqJJkAJg2cDc%b z>GMb$M7BA^Nm`$lMI<%HG=^v>yK4#Jx3#_E^CfqsTt9*V0zKhH>F1FM7h?&Og1s)y+xW1XBai+t6 z4gLcB8H#^`k;i|_Z|!{UtDl(E#($xe-t)gT`R6KnUZX#*>Ti@;GSnc$P&4gQES%;V z!SNcs6i%|y1J^f)e4gX*cf((Re<}C^lg59`5?cqz(6vnlY2TC%mY6cc3q5q;#T{ew zUj%=bsRL=BR6Z{1=%`4$5P+-3OsnbaebuYHqyEa ze*yl+>B65krulE7e3<3EALXEpzx0n(eW%sl*4247gAJ<3Tj3v*W#$SNHK*``W{=k0=_0ZG-GIVW|;V|@+4yd0Z!=pkE9q71T zn?E+>xXaYRF5p!;m}Kf;xxj<>5U!81bQ|SuA^rmVdrD0mEa1NdTWIQ_KjlE?b=Z%> z_-DJ7p7Y~P{=ZiAyv8hC(Lp&hln$`v$9Hko7TKP*b#R-hgQ*IK{y(md`Z1PzGXZ}A z{tP99*SLfKmRpnz2lDHXVO^UHf3S6Miz&m+LJvMS;5JXQ_>Ylt0DpA8(AL2iQwP@x zJUSSL>!VKJM!nC&Ux0r-_)-oA@!vAa;-7XU<)DrK0$T^8O#W9YdS2slT+zW;TZR!* z4on?PuyruP)Io;Aq5qHTdjuU|ztZp*;CCw-*eA_@%Wzu<$k4S-hC6H>3^!$n5qjvL z6Yl#{EgcM%a$xEp&(^_EQwMH=M+Yuk-(%HoNrglEkLz254%X2A<1fJ9SRyib zjeqdpl5Oh%8P>JQApIj%7qU$m-WPi4U?1*A@KrhNC*=VC(9&!j^fPtvw!ou2W|YNf28VGACuljMUVC$_Z&J%vSjf0lyYF|K>9aI z=ANd^&ncV)V=b<49c|+}+JF27_$42TkvtXA9tOr<+ z2|JH}p_dNKjTX!Tf$0QHClh9tf=N&?1s2SG0^RRbY4$3(=kKhT~7eW?YLBQSk{>0`q5lz;TJ{v+-@ zKo=$AlI9F?Nn7KGxX8a=Tx2W}7dbM-g;(R>aG|#zSEXI+rpm(UC{u^YLZ3CaV%FTG z40N((deMaY)Pjo^xNP9EO}JqBr|dKyfCMohgsH}fb$)uo*#DjKgC~w-wmG7Kgxeg zSG%6&(8q17XI*SMT}?Wl04wG316*Zmpy5XjDaZMg(SKvdSm)RuVxi zO%DG{_zUots69Yl;|2a(l5G1$x^~I^L8cA;0(#08B$?@2C-g|iA8`9w__CKm`V6e2 zY{;l!3{_#u`#WIdKf{(Y4msB$Cvr*uM%kq}p_^blEOgQFQe0m%cCOjse+Yj8eo3#u zRPo>9we_E?pIFo;=NneK&v}Jjl2NYcd5wE<$fc<*x<4&cBj zqYUD};Fu#Sat@#(<}uMUwG_n?MKrX*F;GxZG}(o>iYBOe>)A$GVo6z=R<~)TV%eO4 zW+0B>w3Q?8ckKZ#$$NO7_x?SfxA)w67rUGufq-ur%Qlp#s66|HY*FIwAL zvDGX&j;}Z!)Og1vJ$;9lrh$(GpFEb^zXfB_mR;S1`zU__{F|`PJw<2uZt|IrpBRDk zXpfsnc_Q73pcindd)(Zh>~LW`6VF?YhyAx2F9BGx$q3N1Y`omgLVx|A^J}Zdiv^bQ zivm^JkL)v*8|2&-uN*eU^Of_%`I{QA5AdXa56};BHIy^3n=ponr|HM5@j`&5{DMK> zV&h%zE(9_0H2rGPkI61Rz>NQ=sH0W_6O)4888 z!}ibdiq&{OtMSf)PG;j}M+o1-k3(xqnhrGQQhr|pPjz$*bQBwJaetwjiKmrwqZ;p! zn%{?@F>JiD0m1<$o>oVz)p+j$Pj&PbXm2)N2zk#cI4AYJOWmJF)Q=4-&R8 z`DyyiR^zP)p7hHF^=9K0MhW|vcv^lm|B_u+s`)Jgb!v>aH>ejcWH9kG{Swr83xFs6 zUI4Y?QrjS&@#uspkI*kxjh6x})%$ax*VuT!=!I!aewu!L)OgPTOZq(pTF%C6VJj?R z;%WMYsPRSuOZmlvo?+u1uoL2#c$$7bYCIFLq~Ac$&)Ili*$bhM(65CWuP3mSUsuo% z*m#Q_gw9NUntl&pbE=09z>u4UsbZYkVg;%WKO{7dD$AFkwg8#JAb*S)oHQ#tpS$8t@-FVuKffG7Pb zLDSfH)0_pTN91=vjdxDX?drk|+BI}JSP_Z8@qY`pGnLdhfY+oHxh zs^<3z=ukG^XYRs5CO=KTSJim?fhYa;f=00Me((^^F!3~dW~lLYsrhXO4QJy$?<2cco-nDlV9c7uX_%wxge)t{eagdSEi0rR| zwFmiQk-y3*C%K$^*)>CFHNVk_L;b-F`aQ0OxiQvXIHSysa=eDQYD)+CUC>A-OtH0} z=&7bT2TwP)^@pS$Bb zs=lITZ>7HBf2-zy1o6n0AA=52^QU?)N1l@)qcZH3Dfa)M^-l6ydT(?g*3vPJYiXL_ zYLU);CuJ@DcZ!2G|29y1Zn95%R%B26C}k~Oia1msb@mj;PuwKu<16NY8t-G@ai5n2 z@NwYjzbM$(=SIB#7}nB$BK_A%zTeYRsAKd~`lPQ&<0rM1)re2}EeCDYpr0G`!}n?+ zBmL%s;!F_V+#vr`PqA3lh033XFe?8%M|Iz%NBQ+i`M0a(uhlDiff;goIR67R)}#FO zIR6752R_b0Irn)6_i|o^N?x1PG$^mJ2;1bO?lZx5wc*a7`*Ah&s{>R9U-7F@lKF}L2z!F* zpAG#gq_^zzIZ0V-$ayI3)J?pBI-@+i5SPm62D*jKqXqe9dyD(zJbXo4gf;D#9AsL* zv;s%{gnYplI5v*04Q0vgD{RAh{-!doJT$g_qn70s(x9?j2VKCH1>fYNaXwcqO9g1t zasF4xG#|&Gz*AXhuQ~QnWy$C-Y;#cFVg4Ji-gt+ZbR~N#ZAhtO1L}eL5#{|k!Xoe; zHrnIYqY*VtDa4LQ~EPEf~3)p6ONhOsjV726joode`551y8 zlBUCXMTams4>z#`+|n@{&M}0Vr>PPPz>~4tzG^D{V4L&bPZ7m-4s*%=q4_Jp4SG#ySNk0eD0 zJ57g^iVk1PdH9LPC=a~fHy-c(eNK6R&+aMfa4I^GzAxh`*3-C1{R%o91ZKRNe(edQ zAEW4S6&P8E6N(P|<@DkIOX=es#ff-#v53+KAEKp?e|b>)e;}^bZ{Gw))&Us#tnnYn zBUsVlJGC6O|4?+;BIg1BU&;gTS&qm1mwA*2_}%EAs0Zl4%Xxgm=#T@<0yX{Gqex$` z=s@cjS%;&F4q0;gZelv6k9S4kYb!0L^ufnz=@%;Lzt89}4;a#+RvRM@A&>L6a$9;+ z)8UY!LyDY-pZFZ*f%jeUPOp?id4O-!K_~0*2BX6SU?Q}(^byiOiuBP2)@V9>r06hO zPTx&5Q~G!ZcOu@^9ZuP6%7jUrm1>($7?M8L#S6yHBxI zAf+Dz9b$Bn52X(t-}EB?pNHn1TJ&X__X$JujtA&gT=`?8wCFwrv2bU^L?pW2EtI^xe2<2EkkEsSir<< z$TLwVe>;laEz#2SmDl}l;>T*57mya|bq+N7p)~aZeZyXh*O2BIyg#D8lcI&W$!YqD zyVW#Ls^vTmsoWOaI)-u>_u;~}SZv<0;3US>=~2>%DxEHhAsOK^t{GR^o5+p=Ty+9c1ng! zThl!Yp2}YFh`eb2r8Z3%$}17H3RlB=*{D3<%M zGwAnn9Q6#pa|`W!BbRxXI+vS3Yb9)D;;C&Y zm&$-9&F?rU zwT*^$wU@pX>MhQMociHGQ1q9F-o5<*GHs7k44&FB^}WtYJASww8T}f@jprYc)|+bl zrv1;GNlUX8evB>GX(8f}tu}+UXUndhoehAT%Dxt~sa>)m)B4|X@Y>j6ReWY2F8g7B zpVx_h@TvJ*?Yce6`k*`T z)Rw|Q&umcU`?_7g2Ge{G|0dw6EYdE*dyDO5yq_2VY8;Arf2dCK10M(8uu0w*$U4+@ zP~ye4gbwR;3o3jY%OCQH@&`jk={q2;`U0l>=WUeo!Y|pvcC)fxxd|N2S=T^~!%_a>I_Wa_IPk}q@_z*x>7CpP<)4i5 ze`M2O|GP-Dsr_pqqqgxQ;?=*-ls{7~e2f@dI&(3eO{}#yPHbncfZYcl6#`4qtqG|b$ z%H{VKEr?gYk|}?%TK-_A{4~GG_Ft)#zYuuRdo!pv@|Vk>Nm#}HU#sP>%~b54C*%FZ zb)d%4DF0}kv<7?}__)oD<(~x^l|P%xU(aJ-sc*3V#z&O@b-DaF146v|MQZs2ZG;O) zm9_2t0HypTD35IaMN0Xf2cGPo0(wR(|I5Jc(98CJL$m+Oiv6d_ct3G6sBs+1KTan- z4L%OMVM}BA2SP^WkE8ORMEN&2mY>##P3=$n3o6Gj#H*jf*gr`vzpqmMRhs?hDCIW- zPxc=Gx(W7^?Vkc{43%G5-%vSfQxyC6lJS0GH&Ek5lz*a5>I^;(d?i!<7LZZAWGa6F z%0Hm7{IsWNT7G4H=q7q2UVRc%e!W`$o=W-0YxYl4%I^$3>Foqs+ZX4uD1)rSWZ=*1 zgo?gKX`rfe?PNslhYVUdd2OBnoD zF%kdqC4a3Q8s^Qu!dw-zi!k`lL?Zq(L703ElL$Wu6||Kp3oI z780f%Fzpmfs)orPAPiJ7X@uzrOh*MXO~dpWDD+e@$%F|5CQQLh)G#*(3SlZ{GGV#` z(^bJFXqW?$LXe6XOBj4(BN5LBsm#MP416ZMRLls%;Qt?q`2PoCj2h9m2R{bwq*a-hnF?En7PXK0ug1M$)7TXA)sF+&9Oaf+-g1M++ zF53wERm>&AOa*4Dg85a$JZ&qyrDDnn^BgeGDVT3H%yq1RRLl>AnF-8H1#?2fJZ&fB ztC*97nGMWr1#?8hl-UV_iur;tbAg$wU`jMhoV}2(Vm>C!OTfIOVD@U5N_!z)#S{}} zAutOS%q|U+Y!F^jF?$H}3NWuIn5`P-ltD;UG202V447pKX1#`a+(DS8Vm1+G1u!cV z%o+_-<{(T|F}Z|U4a{l1?E)+vrxlacNB)Hm{$n%8ZfUZm^2MD z%}Fq-m=_82H(>szV3IY=O(&tRikU^2d|>hw%wz?V%e~M-=&E9p2(uNKtqNwWhPl~7 z=%8XI5N10t+ZD_R4U^bX@KZ6P2=jMf{;ptRHOz&Uf}4toBa8@)s9*+Ym={_Jjw&XK zFmD6%wt{&~!`y8p@G7PcVcrAgJp~i0VG3IdcU7OnD-S-poWQa7Bv6f_JsKm zm=6_-Ot%iBQRrprLI1uIwV7^c= zb?wwSZljxULd9@|IS$Nm1yieG@V(|ED&`Jk^xwC{ZxqZW4U_00l&F}igee84RKb*M zn9Cl*UKR5@Va@_`R>AzBVe;AtyHw0Cg!u`WpA^hV4HM@nY*jI52=f~-zbTk6G|Y8R zVZDkuPM8W{DiqAe8s?yvutvpvN|=klTvRZ{8V3JJ$y6~12y+RTOA2O>hIz|JSg2y& zA8w-rpTh6!maOja>@gt-UI zJq5E;!@Sj27^`A(2=f4#2MXpD4b$0Q7@=a85k`l9M(TKZj(btVeC992s+a|Yu>;0V z!OYSy^8MIL{5RA=!+h3W_(8?ABupQq(?>~%(=ab|5KgKX8^ZJlroV!@ z6NGymo`J`!e)2@yDc`N8wcW8{xDP`((XPt(plKaup!+@uj|1(GaH4|{PSlKWq6r8e z3OW|yL?@wpwA(k=uCtYosDp!a}hom^d*E7U5Id^uOOW0GK9y1u0S}^ z)d(m0D#D4rhVU5BzagAxKEjD^ML5yz2#*H+JHm;I2q*eB!im0za1-eJ2q*d>!ij!@ zaH2;LZUp@T;Y5!koai?QCt8Yd{1=MGylkRBA)M%M2q#*BaQt_P#-MDXmk>_$D#D4@ zARPZ4qA`K~-$eaBj_3n~6Gi_$HV_njG@B^;VH{EPH3L!fr(**^(P!xYK-4edh@$Nq zh@yQT!?P=D!}Na~YNv5T(asE@18^o+*_qqx%ZEjLhi6O%^*cP;IUvogbS^yyZHUe~ zW`fq@qI1A0_%CqKSHajz!v9Z}d5cdI4(YMqJ{$)bo&9Ownfr>#$Co#v4||JU)qI`= zmY%ab0s14ZhJ3zlk9>T^>w(ICGa6xd&a#*F8HfUN+zrz1+l?nfx@pXl|wa^uUo`50LIk@D2GLr0;y- zeN|O2vS-u!s)mfMuL(>(T74Z+^Qi)s^tu3=%I0${MAj<{aylP43))n#(~xO;eFI*r zuR%)T0hq6Ypj~j5@1QM`Jg-?#*RzmmdQAnd)fauA zQp-=%i{@6+YYK3r*VCZC<7#M!mnlCtu`WQ-i|pC7zJ@_YdC1QyaArn+!*4TrX>~?( zE9Er=c+$-TdK_0{UR~unE0^=~6MG`8scu~%({u|5Px(s7lb#a^OrDy4IyKKwU`fCB zphdVE@+|8ne}7T4i?=+^;rsHiIhEBH*!zgr2DE^U_X~Zm0pF{Y%jzo{5Z1K5^^gVP z?2f(<$$?ku!A*Sj5#{_r&95HmQ918{X0rJO(sz0By$U%$e190UX+8V_SwlVG%#P|I zp2fAUNX>0$#CizaZdHM%#66Aq| zFY788IHSbzQD+TYMoCp?E;p196GB=Va=2%QA}zeXE!z;^IYyioirr7qb9)Z;w;T0m z7}iC=yMs3jd|rO#IL@+-))aA_h1>p=mzx-ectov>VxU{J<@Pv}1-c5|19a~7F*c{~ zPHqWVD}=R%-p&>)p0z-iQ-)BipD2$^zq^HgTruey;Eeu)e~jZhtVO+m$*Z6_!(B{4 z9=A~T{n0&z-5rgVWF03#AJ-_}P#w}EJ3rZ#DVjof#W=E+^Rz-@(Lx zkS9S#w%VOPM9Su{zVb1apd2pAK3oaX-n&1#gHWi$_fML}ZD*85Lgvo7=j@(@HQ=M- zPF3O#ened3zl*y}iQDH9aeMr`xZ9PuogNXl-M@=_M2SoD$wRh+zHa|6ZiN!p;Sq6d z{#{&MN7>Fa&Na>V7G!X@$+5uBt6=9DsH0HS(G1j)1L~Xp6>sRVDnGdmSAzbMYjEiz zIJoFfhXt9W@L@xxbe_909DTZ4pW`zb{7l;%pYRciJV%vx9yL@dKv)YrE4r*-P!or_aa85cUe&_Kj4*xdMaGjq~Wq5|K$>g|G)VR({T$0W7=StANOOo4hC4oIm zQid&8EdR4z&lh11D(VkiX)IofenjoELeB~3gL#4eL;exM4x``G|D7wHj8Yr)69f9D zhM~4jxOagg44$79dt%IR1g6p+eRsjjcWbC_I--s9Tv&2Pu7v)7en{t8U1qHxLupdq z3*g&SXW(D%l-^L3;|9JLOZ;f?yEC7@>wrG^&_DX{Z)NMEC(xHiqEDMGcPEdqppOfO z^^?yYY>Q`DN4e8CNAl?B7~?YWzjdl3j5iLvF~ZQnBt_YBX`F-c-ZrEok9VKPO7o!u z-oGlgE{aCF7&|a7QhIXum$B0G2oJ2U-!I>P9V^Wu`~&s=bgYz2;oALov64In;2-od z{>NB+!vNvh{jago(-gn1e!ns|bdvuAISw)ovMIWlq9n)YB5q@uIS5yvSBBXHIu)16 zYboxH<~Y1l&3E}UGSClC0uH2^4O~ZmLtD<(aIe0%oaVCLa+>%Dw8X=Xi5Q2{F$O(= zebS@35^4iD?O%ZNr)c^B>DSzJ;07dQRu0)Aa-RrUow`1eIxm421aCGhYi zN~_wj%7-xV9tA$3xF&o#F-A&Q3mP_cFr}tIE31#v{bua zFhmN${W9%-$q*?H_nF%L(jihZ?(rU+5`VcoCQHk;`(eZ7yjE!UBZik8L*9YNTWcEz zQx9raC5}Ce2f_`nCOExXG&hQON?BF3Cc)uN%UmNrY$56B7-Wp7_RUB@pL7i69m5Q5 z(H5z`KIcY7;osXjhuDQ>oeF%Kq$`CPUM+QewWvmH@mY;%xmXj$J62PA2Bdch=^1!O z#}M8S7JxL$U{i`u^J|F5%m}h8*)+@}EJCwY{ZX6Kb#q!;?oDrne$yKL#~JP21%1U8 zeWl5qsQ0-m+g0s5j>Aln(+HdgVE(5*6pKDI&D(g-2G}zqrh;+5@rF7(#bGOlmPjhym9ao&a@9_heXL}!*mR0&x z*4~z;t8`82lS3^v9;IK_ytv~=&7wZ=Z3;YGLv2X0RagX@c55~5fk?Xy zJ`lxv-RVMHCg3rS$>U{j3WI-vsDG;8aT~6e6UMz&I^%nn@$ZWCCH##-2X0KVEhk*k zJBF?19gjc0#CLPpjFf)WXtR#>^}Sc|>7_HjD=59RrLe{^@nlVyM|n-?635M<{DRV6 zz`9`!Erz}FU@s5YD+Bf_fW2BF-vZdH0QM??z4BnM{i?ml{+g}iy%z3oVjXu|ZwR9_ zJX_-o2jl7ir0aD)SfKaELhLyg#7|4qb7QP58&Oa z@o&X3alys~0S;W+6yiG_UJwvu?7sy zw?^wMWv%hgJsx8_*3cv~fj@+`qm7sGUWV)Fd$;utV((GIO9~xh7ScFW>B*IlE$TtF zFtVeTAJsd?9l72=Kz_9Dr?yt<#+7Jwf&Ry(z+?Q`*}lwXLC?a$Bp? zJA@769gc^g{Z!3V+L{B}+OU7o*4hJW2i*-Q&js|8So9Mc^pk1mCsYo7B`M+H&T-`Z2u_uGU`yYqLnzW;3A!?P0%97+&%r+TMhy_JSAe zWqf_o-N~=}-yM(e$!-DQ1LjXY6L4U{d0z0cSL_v~cdM=njt;GK%l3)lJyM9KLuTOJ z9f{Z4d%ccNyN7$)Hx5R6wD*c3+S(|Yh`M1POw`jTMG{5Z7(f*5CW0t*>kCT%1a7+C zp!P%maS> zEY=vEa0h!W8n57Anc#tbP@c{$2*eso9%nIT-^G46+kxw2!5B6SW7sN;VZ$(nC1cDo zV9ZLY4{u+19e&~&+%Im5QhxDD}%2$l9*9LPI9erQC&|cV# z^%Sjbk0UOvZL#Oi4a}jvt-II@Ya$9eim=-VE5zQ`4|df@ajw)B%Hj7!ZTcD4@O7ZmPzLnJQ{;R4Dr9$%M#X(j@wdbO3l1WEDiiH(X@5?6?FJ5OXOZff zTRTLm$NX{OC?`~$<1Ocr-#Lu!l+HbQ@9!i24fqexHmL38f|uiAZ;o*aepZNwI7bkN z`qn+PYr?GpZh#F}tmx^B|4h1zKcXz8TY3871)6@1y7`KOWZitlg@_xh>P35R53J28 z%~8mM%0D0Bl-E4aI$UVS%07G}ozZ!SZjh1RFwI?L8#WE2n#OdBi!`Q!RyU?`DyTc1 zy?KZ_$mq;xjEwWeJCMHOC|TwyjwFiyJVL(r5r-2+zaA#vw-JZR)Lk4bQ$O_AVd#5t z=z~Mi7l)uv4n}{CwV;m*VZJyQILf21@{sEgt_RKrcw6*UJFJCxyfbbq-qPEM=Ua0n zclA1v`Y!fNu!*hM4S5{HyXXeYBLP+}EyTc$@j~0VH;8ksY;7#T*cj%Aexvo54WZoi zJLo^ucJUo*mo4^OT814rrVaYd&yyPaO%UecKlK~lN9{K>&w8cMoJ;+t3i)6!nz9u0 zF7=&q^c8B`Kls2db@jce9owRuiIC4jx$7bKME)%H6lBSI_nd%dxQG|+IcZ@qw_tuJ zZUO0#9L_DEG;RW;rEwKJ&0Wb8xri^Je+cwPUAu&gaw;LLb8c3!BEkw$ZRT z?WcI*oR7}^-t>o$* zMEPiZ<-ED#JlMSH`sovdH=Ucc_0vm5Yf8s{Q&jW*ny+ha@3~TQXV2PF&y#P}c)t2Y z&7BjMYf4^STPp1-s`>HiiJFMDmuqf*w7xViw6JE(#V=}#XZ}(11MK02^=g1RzZ;Y_ zL_-;uq1-Wg%(JK?noASw>o-(?v_2sxv~aEgZN&j&>&zi567o^*3F`VR1-6BK)6nk5 zq@b=Mbix z;|fZ{LJDi*FCMLNobg9Z3d&7wdks_GH7M^|l=p3v_ncbZw^816*u#8^a=nf6o>R;F zc89|Y&Z+Ckw^816YI)yAdEY^K_oBS-qP*u&-ZGSz(~Y|qs2eVqcfVX-4>4XXFU_m7 zo>-aCBAhAj6Dvy{!>PPK*2K^HZ{nhfeL3ur~o(w>H*@U`} z!1~As?I95BqbRJ8T!4$h`X~zPqbRJ8BCtN1pstS!bZ*sUj;4i$j>db3aoxdsmeQd% zNb{8racNxTWa=(H@K)-S`VjTkd$=bbU+bb+^v4+V%V_jZ6Z-AvyJ;_JU33WjlT-WW z%!o=j(8_&vJ@(E#x1U2g)Xfc2U!UAL>lfG<}i6(xUq$>8~M?GosB1l zg|rm5A*><3j-#`aT!Ses;uz+xGMY1uJp>c>fI-DAg>Iw|$L)Veez>I&3b_&K2QuZl zHsJl#D?Rzt>Tq3ZRj@wwU#-vTW86-ct;6@(Q710As4Zb%Rf_VK^0;tTU)l;+Yh2E_ zA{^H7 z!g|#Ej~F{6VtNRDWANR>d3bJ-$x9R_$nCR@xCXSJycSgWkeBJ~&~uNWN5M}9yxQm>u$8%Ch56#d>(h@C6ffhtl4FwIhiz^_ zr+X`TX@cG(r&5=+wQP0F)~}({wl{`YgMOIuO3NdEly2vzZw)d`_1ERZYz@X$9gKBc z0B5bjI_0m4hBDZnBR`P#zKJmi;T%p%~=vCP%hR&mUZ9_TbKJ!&ep#x7F!xaCDeJ1PqJtk{3)dlHF*Y1Wn(^hmgu3ax zbWG=$Q<=g`aXX_ekJGq*@^B>16!#Cqbs68z{07)>(z*y}OC=|Qj3v~LF5$VriB-fq zrHts2;HMsk@FBo$=hDJ>qr9j@fz!dCc41 z2WJ&-;tIRyy5D{6Q|s~E^g>V0dNs<{a=EXK1?Ohg%3V53#X3H<+=;Wwb=lcNxYUia zIKZA%mSmhs&v3+dPm%A8Rz_(xWG3+C=m%A5T{N_VVsGKTr@H1g^%Wd~jFvFoAckR{ zK%aL1m-*tGWt4CABjP*aJetZvWuWzJ=+eGIkb^Om+9I{hVAvz%{p#q$R9;P+wz`nvIU<=VEkJ<8%e<%v}9@kWc!5~I~;0sc4gqS5M` zW|aI=jaJ(kM#=xyuvCLJF4b}8&{U@gqt$D{5KGIiV^i^LA$MLyKcV&MnA91t+swD4 zEwlXbjNNWX>eT*5Yw94QG&{j4C5<&&=S(n4(Uy!CMK6#qctV|Z&}@Kk>z@lm|BfNP03YxAo{<9pv+tLiJ*t5x-UHNR>kFJ%NL z`6m)?0B=p#x30d7y=7I;b@^38d3-;5ntv9;ChMK6tFh<3-ZMAf82qgNPROG4F4dQC zo^+$9kbh3cRbRups;p;5eqiu({$C)R3!d)(2WWwgufB@4eRb`WKsD$NdY-4i9|s z;tR9KUQC;P+P{an*uR%~%GsM{Iv#*i#Wgvz|Ls9 z$6UR~vFHWRF_5pnICu7Y{*RfLo~7UVLm( zFJNB-R%bgrb&TJ3bA`QKQ7;=xW8Un2{yohZXKT%6IT?|GNULmRMx^`rF&0U$7n5Io z!g3#by6P*2H__1);fGUe+$Xcb`l3n^Di6CTdVi@Sg>w6aUE^J zIN({i9X-LjoVbqqzpigBUi-+`a*gLsT#Lc?`UgBGwex=TEqH2eqGry8IoTwZP-8dWEXXRINxF60* z1v*>YFVD|LoXVk`bWUf7`xW_hSce}YSq%PLxd&^hcX;d7eZ2L`2fX$2N4)jkAzr$5 zl$Rd-gO{pK@YYKwdF$;nytVRs-irUJSs$F^ttSIMFpEW_Hw{{!>EPn<2gN?k1P!xTOj@R6>W!alk>33z?&e%3k-wH7WH1bSQ%kWwl0Y z8FZ+;1`Kq#koiPWp{q?%CEC+&l)r2u+5>c`MEltd9m+=I9(q)wE$oIKWoSo*(4`7( zpfbiJS=zOFO6oX#fo zkIdmk({;9~Whac*#6fncGx~GZMT1bjo`)mxu9&r~FX#}^0if~J&Oyy6%Z~Au8PF5o z&bB5)zuC~OYR?4A4Ct0hdc8BzGP7e#%dD=gQY-&3N-5A|CUjW*o3mv;>gFY_%+Ci3 zkGH`*gEdKpD_4wlKrXE#cVOI>>#jwRP_F(zJPJ5^pZ+YqiEwsq7vY%BSn_2%oMU0U z&)aSc^#(<6}-AE!s}rzwg}pxYk|)}R8`^Ov-h%IZ(LNwID7J&}G4dOCR4+ZHx5fV) z?WBpB_R{WX^b^!8?B;UTl@rK@_i%2i)r~vW$dy>HdU_0&yq+8)d9NP|-#8y}0?zs- zqEEyfjge@~8m1pC#m5YhhG!0y9P|z+I`f7ctXpmb>f`Sv>WAOM2sdVyR9{SM%%$oY z^B8RAG0@9giG5O4ib?8;IN0l)Dw@(o*gXRM3eShdCr#3x@h0i5XG~HN*1qo0IS*~` z80y1<3(wQ`r{H=VS4-^gGf?N3`|2#0y789EPO9!LIti~p_gkB3?TvjE*4cjp4(p>+ zp;%Lg+;9}!V8alkk92!spO%`E>;8*Kj&rb)<0sX4a)1l*lyECD)@D!+5W@q6L!dmJ# z($e|> z2MpOD0BOiJgdGg90qIQX{wrM%A)It`Lw@s6{{^_$bfd6;rVEGs6Hy11XkQ%mEY#L$ zKU+B+-_Ik8eWjl0W|L$?6#H;nqAN|39npCv$)4yOljKY^(HMPH=-|_ zBzMr-Gq$NWPGHPFs!#oWvo7`g>)5yC$b0A9p!W1^g0ueYj{R&WWe+|DWs}>Mp_T9h zWKBKeP&f4q>I{CA1wS>UnbBQ1@^8|7`k$rw)-0qs>!CD5pj-Cb@Q6ZljMUU0 zlXS-(>6Ag;Gc8s2Os6>Pnb`A2=z5*@OAr949fU) zAZKGR2R=xg9rR1$Y$Gm%I;KyC%@X17l7Xub7tM8n?%bFJ#HBFuBZ$YCt8;y+3v?&0j($T@zX1IMSI>Uf8#zAjpM$!bhje<4i7lFEXE678 zfHn7TSkvR`_5}8N$jcp9YwUGhQ678nwz%}T=q$hnmmMx=TqodTM*Ac;Tx{HRuvxD$ zM(b_lljrl2fA^=15)XG@ z;m8R;Pnn$n{U+4UoxK}pjyN0KgSs5!iZbc52`VPm`iv6b=`$a!FD*0(hranjYPwjx(1GNikAJk5$y}07~Q=7O}zyB4aA@>{1r?MX# z&Zsaxz#k9(b~vMo;teTbJ+gfQAZw2^syLTbJ`UWv1bXhs^;jkQn^Al&%}MURGXJ8& z-yZido+&i?VPZTxRXk}pd~Dps!bwggRiFhrE~OBDMm=eMs8GH^?t(sDiS?#on@*th z6vnV3j^}adaOrW`;IhSKhsz$90hdFO{Jax!EyzRG=|*Q^2FAyiur_e`^w5DXvCfdk z1#3a}!Bh1c3VDo)$XEM5oDuDayNQ3Xz<-wB6Ih5kp=Vl@-Y|@x0f@I7Jp7?j647qm z@!b4<+(U=7&m4^l>HJ6Yd%QjFsQgUiPwCe^CQNZu(r2-g-E{7Cb3Vj#7aq@Fba)P< zhwboeRBiKFW4XL!JR36bkC$Xa9}a8vo{-V=xr0w|^4c9bUpIW(MsR3rk^**emJjT4 z-l}_y)&NpK5HESTW+Y%>+#7Mex9}6N6ZE0NeXLIx?Fhneur?<+AuU?}ya>JN+;g5x z-NiIeTA$y+StqU0GvM3ij{I+7zk0~d;V#*|9CfLK&4O{+Ands_d>(v5;InUcqa4rr zUU<%Ha|Y&2*h+c^>1E*TuK@n26NtwdV1XkijRRi*pVI;#egV~c#EAwPS&9_h6ay8|~$#aUz= zp1G;G9LV2N`qy4Vk8r`E2Y)*18Juyg;z|M}T$rB%-o@p@MFgH6BB4zcm$!C%Yy;B4 z8SCC;-Qy+6Xuo9Z{iyr>cn{$=-Ec!QC~W9Orag$vK@(N&UiJ z2gvpmpY{d?IHNdxCuTHU+iw{s&hx5-ODXd#!CZ@tDGUl&^_A+dz`%? zOPosXKdAR?I*!sh4{OA|*&hy(fX8c?+!&0lcy}aBD7-gV>W{n&j}MXhfgZYND;+v+ zCmrg;mmI3$A`WfmBM!ZXOUFl09T#5Ah&-g@B3}Q{4r?{V_gx#isB8v)CaoR&YR`dB zora$Up7$1F43f(ZeEjvAYd- z`Zm^!XEDBzFCz!PYM#qZ*yo~?#~{ofy|V2xBI4@nKRD)S$T373OE})umHLPg{&6%0 zY(!ca_}jR=H|=>3RK(O;Xk z8`wgACjoYL{bykm|DT0XISyd`;pJ%~m*WMd9BFDfwqd`}v>d--48zzRk*wka@%_YS zReTxdymrtn`WSs*9%tVedyS{~JhJ(}D$911BRa&UAx=-kL4WKm%eyzqyFgC!w46t0 zddF4CqZ8!FTaka;n0JtxH*EAz?Qj-j-LZ;c5mdg)oA!MxQGV@Rvs-7Bb1HeA7H1<& zf}VssT;YJd#KZkqtB?8^^7+)GU)!L6+oGS_p}*T>?uOr|UMEnSqPAZ-8GcpBi+l!` zB3)|36(2SF48DW;LJr6I3B7-}2zxH_Y03wd{1@f^Mt*~CIDgC}|37#5_qmCuG4@k? zx&vQ0^5w_|9{0s^S~!1s4)=E{FW^5x9%VR-C;#kBm2Zc#c_A-~`x3BPzpii1Nxvsi zK3*cd3rO?&R^&zg&v)R5NBC6WOR#?-f7_Yh$$yaYC3|{fZB7^y;*cKHeyAKtpqFvM zF62LWD(DBQZ*Nf#nK$ylK+=$gkD3O}vy{d-V8{nC0rY1!jY9f%oR3)JOnUf=u?V9y zqh;zP8fDr>93)eBaiB~+#r`sN6Z^^38-6fxsJo%4!y%~4!Kl+%jMXuCzep#vb%D>I zp6Xf$xgOy*z}W&1f5Anl@1s2YdpYqI^e;>>;q0B0;ES3YjqjG+g{`UlosbXJJJw%& zjj}H)-WyhR3q%;zb0Nl4YUgcme;3pQdr*4h?|EBI#{*&1uCz8m?SipwFYch)?yMtN42Qh$|tdy82P3UgGyM#rOYZ+D1GhQxEa9 zOx?wkGIbNbmZ=x&YA9?!1a&YNbrFj?i9uaOItdxTb8uPw-r>MW7=2ak6Y(&4v9`V~V?^@X#*y@y{LBI&rS1nMWG zKh+H@I}91sO_J&pzY}q(E{ZXKt;h8euBUKO-zjoeq;FNIkKH?w9 zmnhAvRE|vCPoetCpl?BViIY(FL_K`5(AMUnY&3V3qkPoQNcTv{Xib~~x~3t_Qx59` zSznZi-tE=el=^OVE{AVhELG3e+X6@RLi;YFRGv|Cx?bW)nYIx}$kam|E>m}Lm`vTo zZZaibwtw#TOX~|t_crFVn#v`amzLo8FMoGmk>D?*%)drnPbobrU`savNG2F1X)nJcqKVX?0T5`WC+JzG_}hp~p%RaBM@Xs-e8+>LUqp|m#QA&>uVVuos~?~y+Bqqo%iGq|TQ_Z!f0 z&;@q1$ZfWBFSJ>M2OXAa8}SoRT07_IA_S~yq?u^z8QcA_@5w0r0$~L|>7|U) z7^$FCFYUuU_Ff4zA5~#Y55=4oi1LsQfzZi;J62ldQdtwE`bAaZ9JWmJf4X2VjFj_q z6Q`(pB}&S>GNkJkIgIN0VSj17>w>dlQ!4pOQ(HAo&=>zN`7dTVoTf21pgSkYXD1xa zOgNmK;9YihZ2Rq-`VHJzeJ}Xk;XGiF5Yh?$G?xA1d{{P`WSI}k%p}A49~*ZV$re2di%!E?C? z(3J=$nu%{q5luHr2BM1)-VYQ$m)S&LM>x@S2nW5_7d(76cZ25PO2kz*$0${`!hf*P zCuwZDgK|(mya1UkjX9_<2LmU}bmW9RgVrUa>@1r5R`Jq=&I8|@oBCQ-LMv{0!aJKb zB{&G&I-VaM-7mm3$R^q;Xt8c|Pz!!I&+|^7w&lZg9)_GL3){`- zm$U}fb##z{cS6{3-XC;+yZLK+w}@Xe`0@BP9xdWidOO9Z3?3ez;^7qk_>3O&!*TUM zS)z3O4cdFr_!R2E3-P${p5u-3`}BUghggjAe}^`1$Ur#uFo82-F%B$CSenJP)-C5+ zXW|lH%uI-FscW?}i*Mc0j+^87{H}zdEp5!sE4bD!SyNiMW=(D7mX*}X{qVG|d3Igr zomIut^{23aJdhTDsmmM>@6ORlLHRd z7!2}S0Y1tdakg?8ukz73wV(I`aCJEE&Lb@JH!N)oH#nd@AYLun8I5r>fL#q7&n-=; zXo+@-wZSyVmaDQ~Wtp2e8L|w>sGSgIBJOWbj^ zUTVcj8Lc_#r6Hh0LH`blG2kVfNo7purI$MDB>dMzdg&_0^=o>m3~fCI=}$x2+uZTq zBE|$)+?S#4M}kgb!v7wsVrVdQ5?KHVts zkhf#xry!T>v>bJ68(mbk(z%GUjRAE5HG;MRHG#G+st@OBEs3>qsOw~>_WSteGc8z{*LZ- z&dNOSGVa^ymA3T@+CT;VB}d_U7v;?PA0)#w6j?_9ct9FQAfx-+F?fc9c1inQnk!-v zwnq)Cn@Dm$Sw0PNTANcGT4Ucrn<9J`ZAg#zGv0(OAF}U3Wjxvt$v?w=o`yyH zA=w4U)@d^IH%ik6&uUj|vdN09J!G^#BinMcP7g=FOUj76b3*BlTm)pDAe)#Gc^A*C zD)e>&{b!2uC;PUuSL{RiH?>bX$(q=Q)_;nwZN$5bZKgHI8|^Y#mE9Ri_VSeNMRpml zVbLcvyNuCf=ntArl=VC8ioT%PBwoWYHW{qSZo?*8eUV+FNWL<*s2uhuNpaX)RPMod?L8IS(Eg#RZTgX{iEVn4EJ?Ag9{I;2pRQB}xos$C zBGVxAZjdRqBAd83$P_!h30Z4ZR+lr%atHgGd+4*8Z43=~q@lJKeS-{XKt}E50s5<) zziKbCi=@f4`h)By$^JuI(Ky}CNwJL`@@#6G8%V3EZ7!3niEaL9C`Y4hN;Mh!nAVTJ z)?{c$BqQ4#(`2xVW|N~0GWQ0X(E6S9zdc=T8>(FnXjsNBw0@UurS`2@*yU}K{fBl5 zhF$2Nag@dr#K$N$`WWMv3HyPdLi)y^o4B#AenVibu9t+doc6+dTIlk=0!Eq7{KQ?j zKZg6Va7xQl+=2T$eL3kU$-0ruN8Ae8A-eBL_wHgo?r#o&PbOsY9EiGI583;;r*-|d z3$`Pv-RoK!%zE4>2f&X9)WI<;!47nW&LPJZ)RD`Qp9|ZAcf&=`iZy=%dX`Mjg|%n? z^n8CnReI5)bwRTiPYj&Bd6%KvmR*K9Z|riO^U1EjIll$8GtXP`!2AmCx9oE6_VKR3 zZXZMTN0!z62EvJVUST!=s`6hzz6iK2z->X;&a67~VwGP6%p%BumD8AjJSQN}EW~*R z`A$H-Nm=*J6Ok9E#G6BDM69ecAB1cI(nT8GD2=l#2&bfX2mEH0_gZ=1JOP*qz$C1= zXZ{dz5GKdTW8&-wui<(C7mD|-;+Jwaw%w^8!>_z(Znv`5yko_A^Tid{%~>lhnE$cj z4|Cv(GV|aSzvEr2-_2QBe}G;vCuLnU2jShTDR_5^-ua?;upICXmVtN3NeW%7!>Wut8sNSfKsP4!&gX)j!j@Hz-(Iz8MXVe}d zV8aO5Z~*ML7`mcPy1jz-@o~T)vvp)fBhK? zzZ~~LeiRlsCk66rpx5MZ=Q+|AG- zMq(|RC7Hce-ZSqqdi_}sKX%CVL9r}d1Z}UcVDcy zXih?1?nK>{;a$0QEAXxy>TD|Nk?1nH4zHW({kKQgVcvhW4)2{AV!5(!u;prab3Oi_ zt;ei{FVQdmTYa&)5B{I*gKL`W?k_xkHut-~>fD!=U}+vR|Dx-bHO+PR7alX4``ur6 z%v{wxX8vWz%;viLtMYtJYms!YY09-u$n5KDMU0?*78-%;tXg*Bvt*xaKkQ zuR3Nn*WF*W-{mDV*WF*U?lNvFeo9Ad6kmMuQFsD2vclQLeDF}@dL8sZxzN%rGG z+VJ%tAB+(crutyi{!G- z9l{3fA#2efYqX0!$^HYoEP-7}mpwA|61_p;kFqzY7rj64CEl#Xdn;vpubZRcO9Edt zd$idQV91Ze(A_A|cMR{M{(LxIC|hZi@V*TG(*~bbgq7{!E$$?T-BAq#i^5P3rx8|zurhnd(RRPa{Z-tU!}X2n z#;@J}r?gTaYnnz9(l`s>hy?gbIIp;3&SA>-pQe$w;u8EUF2nbt8a@|S%uY->!g)nM z(&L}$_|Np)k94!LE}OR?-H4S}%qjmHWm~lJ68t|dn+?mW&Gks5`hO#h2}oln%JCb@ z@iEeX574{iH|ocH3jY)G6$zwoE5oOUUvb+U4IdKn^$Eg##flqd{mPs0Q#o&r#(nV0 z3+Akqx8Sn^pCH^{fd7!)^83*7qB#rqJ6Ha0{sR6hJ8{1g_ksA%Vq#{-z^VAgXp%2y zeG~W7+%h7oHskv=CQg1&-VuJv|8M<;ntc@hXMGeB@$I%M%vb--I+K08#>H94Pw3j{ zW*?#dX&<3Pe6O&|{=Zv?&He6wxZj0y)=<2|Df@CYkCXqO_oqzk##vY4dxBBxb@Klo zSKAvSCW8;PU6sGuiQ!u_yc@%NGQ2Ot2QYjv!*^o%aE9;3@I4v6FT)RD_$Y>tVfdj8 zKb+x5GW=+UAII?cUsi+elNmmV;iofv3d7G~_<0QfGQ%%o_;iNPWcZZ~pTqF$82)vJ z-@x#j8NPtwcQE`ehTp^R?=bv6hW~)!KVtYp41bj2|H1Gl82%)~pJDj#8UAO6KgaOr z8U7E3uV(mb3}4IecNqRY!`CB{0xRqW%x9Pf05xAF#HmRU&`>y8GaSRuVr|F;ny?#CWe25;kPmT zPKLKId=bOH%kcXdzJ%dFX82DT{&R-^lHtE*_|pvk9mD^~@V_#AIm7?X@Kp?dh2d{7 z{4Iu;7~a~*H}nOZkFCb{N@+|~`ht<^3k=_q;awQsgW}6is7#?{0)Y`#qbitTO0X?zTnn8FEq6O@r`3e&NB=@ zh2fuN_!$hJ%J69n|02ULVE82rzm(yZGyE!sU(4_U!>?!fO$`4A!*65woeXbb_#%dX zm*Mv_dQ|2ORy@NJRi=L^~zsqt)~`5Z~>3ye+v|9+0t zt$AK(o)>71#LS8Rx1J;68Fcf!&^#|RpCiEs1MhY;pCf5~ff*a0VfZNw|187LVE9yq zPh|T-izV=7(S5U+cSI!!*^!*?hN0H;rlWC zK!!Imd@RGqG5q5U{{+L2Vfd#Q{%M9!WcXR&6d?CZX$?&@w{w;>z%kaeve}Lf+GW=nN|BT^}G5l8ye~RJ1W%wT$ z{uhQXWB3aUU&-*78U8xM-(>i^3}4sCYtNh=`9}X#C)>s`!pVW*TQa;0!+S8iH^aAO z_#lSw!0@3A--Y3OFnn)@k6`#nhBq<%;QxoXH;<31y8r+0otY#v35$?DEJ=()Cb+R1 z1v5$5QB*{%Xl1c#fTBfQp^!kT4a8b5A}AD?wTd+A53UE(y2<;>Mm6 z!2F)?b7#UBz_2Jv{fJiNNX< z_MXhoO-J?*V;{D@!<)U@y*W!->uzt$UGKel7itUdqlMp(&X6}7KFhW?aSmDDRNQ~q zly{j&i(5Iz)#FU=H~6j07{~n=A@@gTj0}n&-t9^V`%jI)O>(xI@M=FD8kG0oYZ<$E zkMQOuXN!~g#E-nQCjWwua*oy;@-OFxO`#mj=`a`ajX$sm)}2yO?(c)O%vy8c^#sJ>G6E_3>W%ncsT*`TOZ_ z>-W>cZ`hV`7TlecZ<-L0@&?Iz+qHQqjR{P7N&k9W{A%)DKO z5jgEFFy7FbZk{uzYet(1UC$V1M7;QKiMRDzzmKhLUK-%tIh~H$i(1}ZleY!DMoD!- zUdiYqsY=A8%_^O=xTM8A|#j z5A{+9W|lRYe|gJF*NruVcRvzsIV*qnQTP#h`0ReaWAFfcdZp%p1p6}hUfz_smVa?K zL`!}J*;O!Zy+NY4KYSxOb(wNsz;7?G{u+wwIOb8}lz5M5TD<+KHj{Xx z?#%Dr&fyJ3gE*zVG~Mw!Z*ZEt#p&YRLO1U!@^-zOH|%&{^r+iq9}B%vSQiTWeafKO zWi$-qZ8>>EV;lcoBl(@(yj5OF84K+?@{XJ5SK~&?KxpscU*5cT{v~aLh_8@%3y43T zH$eZT+poAS#J{|;`2v1~x5B*kD*wW(;>IRk;h#;=^t}P_iQI$n6n^DSfprp}JGorm z{z}&TBlb!5v)W$22|qd&`cdk37w@wRZ#sEQw7bAh^5&h;b`#F4yqPWGRDh3Swk03o zdXjhbGNk|b&4pH-$LSBo;ZEEOA3s2RPM#C(KWTb`Jp=P~>RCYh;_a{SBW>aY|H41VFjHE}EZ&~hQkL=de(YX$CDA?->~BK;TlX{QAiC|g zrf2ZpdUK(??|!=6IE1`#>Z@enR?<@(8)ztlr=%@y)A1zPuZPmPgMVpf|Hb?snxt9W z6_QS;o|Ehv?I+oO1M@M;;vV`Y1KA}yS??%#mN(nyA$Po_wa#5Luq*W90^TA8PYTi7 zK`Z+H_lq(-o>4(pLb<#Xv)}Et(E-+EnDbs3!FzhoG&heaMuwG;*5c?n2TR})aZ_}3 zj`la)^uA-q#+EPOeu_ZU+yVX$#!H(*erkzlvdjBUC2!Q1xxD4(VB_!b@&U2WAg!%s z9c8EA2(67asv5`t9T{jM3th-w-vr9v^kPwfvi{%O&3a z9p$zXc|6(4>uBk3q#aIy?lls$T(1A)%CsfO^CDdj@%Bxj^1qufr2MC0J_=1_uJ{%G z55MJx+gmR0!i(JOqvK1qZ^q0fJ%=e9ar-Mxm-kQ$$wL8o$w!|5(~;%>hx`rJ;qt!s zS@PHG7xI@C%HL&#A^GczIW{7H&xi77l0T8tlD`uNq&+6v=ixRoe;GO*XU^Y=yI!N; zGRd!te7ng%ednww-X%6I-q;>p>oKYhQ)jC%kGMV5o%7!Baq3XZpU`{NlV56HTH=vD zCf<$-#TTW`c)JmEH}zW|>TeJ8?=_O%*$A&XeQ<*P1GHkd(Xa&`koGKc%WHJfZhX|a zP7Cz!bROgF25rha+&|Y3n(oNw9qzYu-HD%L$e#~rV>+$34N+T8HdkH3ThjXf!5qtv z8)SE`H|XW9$$wg?_w|Sr%E{# zV4vmgR32rBj7p}CQtU6e`{--x>k#hc?XBuiK3>&kvVA>ud?{&?H>kzjO!}R?C)+RL zW+VTRZ%03d4mM#(p`!g(4Z*MA;vsJIW#27bsWc*6nME|-BQ^K91O1uJ9n(!;gSY4WWpa0w#5;g= z36ETdALo6v1pD8Vz0~_L-jx3|HkJC0w|j-$4G+f$?H{)if#lDy@WZZforgEj0=eZLb3Mi|lrslp)=yrAKf$DS7 zu?iY5LJ#a@e-ZNA=nn9{v5_Eo7kU>R4sYaP{;d@+N!`ZNM&eDQD*SGDf_5)Dk;6~N ze>4+Bws;w@zE|tqVb=jS(uddl)GY06oA*eeVjS|4>2~@wJx&u()`$>`#ILlw{G+s>D2s;Q>WdZdGKAfG-+zlqB|PwDyQfw}TV zF7+cc9d=!km5uvs!j^Imr*#`?Z7%g}KXF$ZWL#ThTsvf3dt}@>$hZ#3IFF^=QPi1- zxd=CJP`bF+&XI4sew6Y~u;15aqP+uC%CNdHZ9{x}+DAL1>I1&&c19oM-a=zoSL>cH znwLs_={7R=GRv1~Sm_>PdiG0Y%aV{Sq8HQ<&RBHjW852d!kwB^*50tZCku9tk1?hv zzx32Haq~jBE%Yi`=0hLfPx<&&@y_w4w_!D9G2UeC(z9e)x|s}LE?q`hAL#VZO8my# z*T*{V#=k&3BAY~h)w?>CQ&!u(zE79cqwmVtLi!6EUL5Z>re8%pNu3(pQ+DEzD~OwI z=0M*1EG->SSNewmWyj|1tX~}!tUna(*yW4eAF(SNPX#PKs|68$hXo=90?Lp9Pk+uUr z*C!h)(B8`%@}&cgOPeg6J3!*z%RMaO=Y9B5*FmPEpBKA)4Rb}es2im6kR>I$k4d$g za2JmI23^ld_VwCKw6D`&Fis*|5^D%YQrZyTyaPZd5%Y-?!#K9kPVN7u)h^@RlN zbQ}LuKZ|*$-1JN{H$C5#+d3^)BB(~dO7#>g=k+wJ3eR5&VBBb9AwENX&X|8n<>NJ7yFhsx)XEe z7k{+8+3jlx(Dz8(@8U;f!hf_GZ?D4?*(!D-TTjj!LEeUw$6@4kD0v~I_?6|Nqsv^5_&bypQ2QyTTHa_pjNGl~Oi?9fS#iqp`cBeT)9gypT+g|?bi=FD zH)Bqx<-+Hypr;%04fE(jDHA=W4UOd&8e=ZDo(atb-$6Qc8s94nz9(_@Hm!L-IkI~d zb|Skg=(8oRMEJw$b5iY&;4iT&A}zI;rYRdq566N zdzk~g=?7%`W|8S_Dc^RKb9>7B9LgPeKC2@<<$jrig8SXZ+JMMU-cR0_6&LhkSAVOap1QzTSaS7H)rYn(x^7YL z|0Ul$A>Ws4`JRM)FLEboIo`>)OUwFCypF8zN%}>fnMc?%PT0fTm)u_&zNb|5{{fmu zQ>ZVIv5XP;kA*JoWPWlla#&=n=2+Yt8y z!ms4bdj@$cy4wUTZ&U0QAwMU+cl3>iTNx zd=zc!D%zAIZzFYKorchOWblj4OKZ@JZi5#!9lfS9G!6~x2ci=qgSQ#2=AV&e3;oE> z?6%%=@1~(=^ryr8^~h>;+`izo-u#|2XIq`)eROo}qigQ=o3SHedj>X?uX$v2+_S;E zN0@OVpykAN3FMCWz_`|%8OZfMnAQ<}XnMdm=rY3}m{}67^`+71^b2R)mt!&3&Aktu zLS)zm%Hy6Fy;hkkpcvmH7reF&tvH&w9B5u-A?aT0NB7D@_o_o?p}X!_0Br>N(fHX$ zaC{e2O^M+jy(sxoM-4?xTWxk`nvmWl^I4?5JTIhoNti;@ z;iA(GC(Pldq38<^)Y7ga%hnLRs3guBK|1xEaq`Nvt+emfb3i)3bI5OWAQv63e~~MY zTaVl?8W+f|6mx7q_X9;cMs4sSR~D0}62>_d)Ta-g61&Jol#>X34;nL!#8< z#I@dafmJcjy{2NGhc9YPMZc1PB}WITl}9~lxcF}ZH+j?$!9Rh2@~A?=F2Z$1ses^U zaCDR!C^!}z8>KE2+z#9>O7#azyd?|f4J>(-Z_&Jg6)s~ zo&i5ge<7IiXMHKhs_2%lHg+@BchEM@G;+mW zyVUszk8?6)oa2l|*7|7^*|Z7xScx8W0s4XHOw!++N!KcI1*%Q@?#$m@VqJ>vUWTrw z(~hn-3EmdJod|&La-SP;37$V<;`RNUx+bf$$PYvy|&%@_M%PHgA?W zQ#bP1gIs2wXjU|RuBqqyM1PIsfiGX+ZF2a|6Tt&7!)r^WPBH7d4OK#3t!!aw+}p6}o0jbhxu*oSYeF>TOE?rlEl zHR!XZce8@@0e-WcRo~qV9=XmH+~byUM7(`amqWZg;TQNU%%73`_AUO-#&2OhGeUe; zfZIqu>l;eH(4>v(Itq=aMGrk2pPf(nMe@>Ld_U)<_bAW*KX}R1WhlHq1b!I|-wdK3 zFQor==8waAp4TV{8mwmv{}|^F)$6?ve`^X)Ec7Bf!u^Z%*}wZq#uc9J4L#zf?aOL2 z?U!^xZ|90pC(s9vpbs8JA3TIU*vMEz=C>4MjnH^~p{yIji~aix&$32*WT>GG`1195 z*8Im8`swc$j+mWe4f~7N@|pp)&y|?N{CP+8xC5R}fu=vC1)9#o9K@HA8kqTIY+&ZI zX@Qx)f$m8SG?l~}V}_Z5++nds|6x(|{g_6V%2D0mPk54XU&-hqpV7ygnbEa}&lq#! z)2N_~DW0MHbpMY#XRayDekw$J8Je?Bk!mju(O!ghxHKBS3HG88EeNf)kbBpANc&E) z9|_S{Lr1QRKEb`_pE2)s0^R);ng3!>#se6jjru3ce{J- zdv#i|7n8J3*Jfw?E-~{0YMTC+Ii54?h5NFuoYv4|_hL6y#5VczM(hy zGTS#uoVT869RGpIyU3P0((uFc z?x4JfaMaxfoxF{`4_#I2dZ6SH{(Cmdal7Y_w3~QC?Ika@Avb++BX$2}p-amu32)Dz z3xgkp>@UDx_TSV?_!k>$SgKLgkI&$%WiQbdWS-X=UY+hUrkCMvIA5k&wrql|1OL-! z$vy=)Zv{MXi=oGd6R$Ah4VkYvr>FJxdrlu2A4-F!ztGc~i=W*gyEfQKc^^$Ms%rRT ze0x;axhuTZHCo1dE0J4QHZ?Ex<5stI+L$lO$UPimsD#-DWtC(fYCumy-&%sc^&MEo zBuB}Y*nKN@=wD0FzrGeb>Qd-}MHgFwE>itr7ehm!J@M*L1%j#T>0Gn!TBAdR9TX&`F64g(^r(F2~M*M1a}pj zYUc?)H%c9+Er`3z<-c2$dJ=so*L{K2cf8@3zR5Rz&`Cdi(=g`!vdQPZ)%ig`WAAL{ zy!!Sws`K)UYTT{ME_63GG9UaDod}s0&H+6Ouo4=4I&2i@=BAE8$ncqCY=| z{`?47%HmH#Lyum99(|N{^{}|_?zk5$di7)I)g|EikpWe9g;Pc+;DeF^FmgbFB^@Q` z+Wl{6=lACsCx?@F?o?L$ne*6}6;L_Pd4v9|4b^`&`Tv_yy^s9+3u7Ax+$3|TDNg&2 z;oj`$;)4D=D5sscp$z=H*_Tq-_QYiN8~7#f*P0LHZEv>$-7Ht~D)Qz=(k6MiK}_;` zy_oRObz;I}*NRD=>M4VdbiZS4KYX=`-osjFsbOUsu74Yrs+jqiY|J9VRj%XQ zG_ra)bALHe#x@yq$(qiHXAFz+(}5D)jf@gmV>C=;PIshFu-RaOuVf$9NTW@|YUa7G z>JU&bm~G0_%{C3${FfVsx{m%w>f}O68)kSM(7f5O;v39_O4@LEUm)#yp>gvC*2K_U z(3kreXP`?)`iaI5I?!7Gg|eqCpaNxuE6n+YE6|aqu{UC2<^>m8ub;tR<9;(Ca?DXA z`#Plk($B@-7|3P38ymZQ)Q0iMm_v;H(7|&1nBLfu6RZ`w;$wd{zVI4Vt;dBjFU7b~ z>2O2+^VK%vifE(HU4)_O=mwR}`rbm<`~o$}G}hlkIu_6CW1$bJeE6%ZsZf9i29vQtOC1`7j~}J|Oz3Sj?lTuxFm99i&v&AY#g&ZNbb1-H z9U)Gs3rTAM;mACv^MCqYHd(8t&h^~ziI1H=^$PqPWjwQm`lt!b^IQT=%3f&0PUtnZ zh2gO$e8%x|Dcdd1xU z{Ky)I;A-tB*?tu>+nreMWj{wzd)^sh{Ho=~;y%_)>?p4T^~lgT_(_j@S0%8&k$uhZ zNG0~v^XT2XjNtBehF${8fsHyCO`TmV>0_ps_vS z-Y6CSV3exlKRnmA5&quTk*}a#V{MjUEhS7bGtmWHvYiMV-r>mgIee*k6JD6^?y3?8SYN z%o{*QYrKCUFm(q;E!u4Up+{Eouy1mV{12WM3?K*&nE8+&_IEe7}G>Ve-{?1oOjn1s>ca+i7QT z6YvFQ8qON|^z#^pPdwKcGnI5toqLfrnfdux^q~JB`yW9c%jsz^AJ;SB|CX`Ijo0V< ze>FWnc;$@+!Eb4Q=x81M=NlcFZ@G4Irs0<~wI#ln;qf{*GMn-kO`dAwYX%x0O&nDh zhxysWJG~$MZq6?Fz3gsy8{U!eYZ+-zH&Pnjlyy4N-xt3d3j))JGZywp9zt$!k8nE) zyY3B#_028EPKpD z@r{UxZx3VNEhl1rmbTA{E9^Ee!tFf9#^JPm3@`Wy|7Y>WIkMGjPR%U??{em`*o|7C z%3}38#lnp00#%0&E_`iaM}ETe znLT7pf1y7=(aPo>Fe%S$<_E;D)Y)qCvU5*!RYq=6S3L*1C*69+6*y^RPwa{wd@|fd z<`ZUZ%%Ewskz2gRmL2sS>JPfiJ62@!U-{D0J1Uc^4%UD1(LwL7Zw|VlKO0RL&rV9u zFbdO+?B8^)h&E1E#>##v!>WvBJbd$t9rI7FIk@$!j}LC$ed3^Rwr{!LxAn2uEoD2V zJW+9Q!6R=ST;Qucm?!fzpEr-uay&GyG0j_7NS@9;S=e%BCW|-=ytj5ONSWBxz%(%T zlp51$J5IS=8E5^TJbcR7uLk*K)aF=m)Dz{-7-}T4ZUi!KII@p-vhEnl+T@VnzO2F1 zkd0ce6B+qS^3nU_{L#Z)6?a!|PTMd+`iP1))*RxSK>yGavm#8V?|6X!8t6SRrQeV~ zBe{>WmM-#H`#;gfx^I`GOFoY~k+~DRM%4o%9}Z2)_43ckUQ=zT8PWA&O>HR>m9(^CQV_E1|F@ZI*?@0QDKY`k?+#w2R?Kq2ETlLcdj;3HESINq=@zOh0Lhw1I`;v2j4= zvnI5)=AvWBxU7rz6K{{f{H6L0*DY;jo40O?vmc-FNbB+IPU4zDe>Q>sY8GLl18rNs zCr7WhRH5t07(vo0_V-~{pgT$bJPj=RP-QRr4bJmyqTiT--x(cySod}`w^uB3+=<^} z+*K?XtTrx)Qr#qsN5PLqsVuNg-^7a?9qZnC1Jiv`YAb2#gPtpECE+=Mqc?1>4C&`B z`FRF;C_1_~u`Eseh;A%ynoC`bQKR7ZX!t$`{`b(X7wYG|%pK%1N05iUZpP6EZtk=}>h<8))QPme zU*%eFeBki+A!xN7j4IKwmqo|NiMz)`^Pk^hC;GO$i~A$ILfk)kpGmM>?3MB|*Ww?3 zS$UQzd#6)u;m3dDM)rcOiWi6AL>@?B!8r3tv1>Jt4;O?VrE3} zvdH8^*Xc3s+RU5#Ti0MevbDWik6+hj-gBun4EsXtrA)%}AN`5K*i*Ix>B``t_1N(j4Mbdj}Xk!QQigSaLSrf*8L6S1p-Uqnv6fq%xV@Y*@k zWnCale;s;cJJj{MA=jGgcG}_P*vpzHaw-K`CJ+=vTb zSbp8LhQEi~7(M<%(@!7K;riKTaQtjnFgwbq4!1vPXO9r3w6mu$GpJ{oixYmik$;2# zGo9sx*R>3foZ}LCdNqD@9mC_aqXQE<4r?kQe-h8HX?G$Iuf&bWNTCJ5&U&h|hc@SH zQ_6mu_hDTKxP~>+N9jCQ zk_HJs9InU7-;c%z&bVJ>#tHULolR~?J@=t3`>;+xAEej9>f(()y8e-`^TCbeN%o)X zauFHQrH7Uw*C9iC|AGw3WG$`8W%jCP-dxI}_N!BJCkPSHYofjB=i3I z{NoY!tIK&v<00lS(N`-7OWa6blRnt3PJ9{aYa;uPX-@w!8@FM;xrjKz{l~4W6Nl+N zpr4IzLUTMVI-Vgq!v&XF`JwfXXNgv{%jxnAW{{ia#=knLECGHG(D?`q@iJYO#WE?0meH3NWx@`st zvn1R$UnefnRs6ZeAMh2gZR^PqK&A%^Jc2 z(KYBFop=k-MVU`WFO&GZ)Vo(*W}zP+@S2Y;V;s0JyuL-)GMB$LlezpT@&Z3lRzLUO zT9=n}${x1uN1B(;@ETR<<3EP?&eg+@;k|Q%2~XD%>nD%1e)6*HA)|d3Ko9R>8vtG@ z`aNqXl`EpuM)sYFn_TE(4kEl7twWuDHXw;~@4EdGs}6bo0=ll#E@dxvlATC@7M2HD zgasCP^rOuEXGfe75SpA3aLWGlGXnplE)8in>`99=<(xpg-Cn0nbO)!L!ea*6QzZSt zndnky-d5)XTBoOyxuh>>53S2vkCoW>SIL}8kTR5U(0RT<(g!Vz+OW6yS{iHkgnv1J+{Uz9f!}a$cu(Jm-!Ty+Xk-IMR z{x9N6*c7AwiGAz+U%Jd$ACodk|JZfLJ26_9L3(HAKFM#qF5^?{yx}}+9=`H&#$AT( zhZiJ2N8AZHT92pfwv6gio0gZ7Eh|{xTQwN{*4Zx^!81Rmf9_J+J?mY$VO}B364D^y{sI5OGp%_-_H6yqUad-a z-WyuWsVq%vl>GbAt0n)HrQ;fP8;+Qh51`vIZG(o=@VwfV3Qd?f!o8L_4oSWPaAYu6$*oQhs+Hg92^)mMe zNEyv%PQxEjM)~xcPF{P=@-8bNudIC;xXFYU zw0(hNpVP{|mbMbvukXt$m=<1N)&3?q{yO8Y=BMWM5_TD;wwveJC1OXNX}d~HZMVv? zYlj_q)plD>gzVmQ>|(IXfF@zqGuN*D?Q!f3>;lll&WkDKYH${xZZk z4`UE$0ZeJrnA?oQZLOJzF z*Ge+R`p5q^FE!ZDQSUaV$s9qxtb3QV)BHb#aOua^&REsUn&1kkA@Ci0?)6-2 z(Kz;qjrPhJFIi(`ulcES){*5Z@#;Hw+8X_SX>O>G-&ni&iu0^Q_CVZ0yi(UCi*9LL zxu|R71N^V-m<~>VVC9bV4U6}f7IWkWWG@WmxOQ<;j_e`GUAeB!h69J#!`{xP*H)@1 zkHvc~vNV6u)(y%{^DH_O{m0t%jA80&L!$pQQ?3<^XErjPxtaJX$nO-PVg3#oCgw)g zQm@CZ^1eYTIA>6#?9**yH~n-O?nK7PI)(HxTV$U(a_|^+Y|xhf%(mxaFC|~cE`pEHduoYK(=#u$jL9_veg2z* zadiRJCNto#XJ6s|m(Acl#*s2-tLrx59BW%0^(bMz6uJ+HcBB9IIrjl&A-BR|rso>f z&KhjnHR*K*c{cP~eL#3`K!58~!m25!iz?g{$F^hwUd8opRzWM3|Z@c ziTi*~p1KdnOCAi-Wd;qfw&1peu=M}41FYOF%}aGZ&z_rb`5(c}#%RV4;%D0c>vjH- z=~b-Zt({tTfi)#_OmOnk0nP>os>=v#GT#WkcaVR-#mytzFR-M{{`9Xg8%p-EmOy)G zM|;>rdoXAZj}E*gxG*YTJ>tt(-(Ktq?k{u)f9TGc*UL@W6S6S#cLOci8~&Z!P;-b| z&fsV8{dq<6()HBK`aL~#+lhfs*z2-Q_PWSCIO`+oE!^j28S1jtDSFNo{eDeecel!* zk6=Diy&?AOQIPV^SUPuL#`3uX^Df7p{vnS#d`0YgIQD^3*P6Vmv145^6tlueSvxG8>|q!{vp2`@w*wn<=C^oE|0m0 zC&d2paJ(^V@>XKU`cNM0qL$d@h5YWs?}zw(S>~7$?XB2R25WOSQ|GIrt@M*Cw`6Y) znW1}X=CaRCm)oT7oS})fs`0}-#oBQ@dsx468@a<~FdJ<;<3<@XO1LX9y#9Eg zIuRaX{bE~6VQLr)*$(3-Q z#_lcTp^m>W#|j)DtlB{fyfZ|#!5n{_b3*S#tMUEK>hXt+-0`c;-0?4)xsS76vVK*j zHQsIJ>hk@jWT3vYXLT1VHMV5*c;cui;CxDfsRk2H#k>Nwah^*R2!69rZG6+M@&vQ* zbR+vt2M8`0g8OKd3pU`Nboi&A;6ilWLR0k?Tw0*gOI_*`@cMHND<#XYa$~sXg|%AB zV{OX)IacER#`cP6;)`~vZn&%1PdfISDogP9;O|Y9A^0HppsBhDJ_bh{^^M>W;1MqMrQmVkaV}LS_!jUjF7<`rN#IE?^{L<`)XfsN`bh9H@G`ghK=2Ci z3iN-$tH7(=>OH}K1^?Bp{v+51+ita0@QdIV-RdpD>%i;WYO~xz($J?}mQ2OH~QJ4}70Xy#$6gax3QGZjM{65xaTdd2VG3ei;0) zTRkWEukhDjqtvs47eHU&R?i526#S@Ltq{BjyvVJV3w{Osid#J)cqZ}9bg9P$&jHVI zsYQb4f#FU23f0gW!WMb)Dd&;G-^e zjo>D5lS^GC_$TmBE;T~1i!yY%)lk9F;ApoRBsdlv>sI-K+kxA;)fIv}f;+m^<${yI zNp5wi;52ZWTlE#(72MUWdI>%ke6CwvEVvuEn_FEVI0u~LR(`=NNY4t|zu;BiRW6k! z_^;rQyM zw~7;704{JVuizozA#N2Vcm#NaTe$@H0{3#O=H1|a;C{^i3BCe+gZst+ z;L&b%Nbp$jShxC7V`S4<&i@I%h_qcq`xo2`+>7=vxF5J5?O*U^;LB+Lg0BEyLHif{ zCSktmR-X!f3;Y)CU+~-Dw`u=^-vz%*`xm?eyo2^HcpP{f?O*UM;9F?_f+v9|(f$SB z4!)iCFL(-gid)qPwrQJol=_F@7r`$^sn-PGiJLoV|AOxZ-|bdag6{+0=TuEDlUvOf>_V@0dDO##qruT0 zRVsKL`CP~Tf3R`!g?jIq_}%zgl&aX1uX02W$K1HbRQHSfSp3F%)IEaRf!lf1bip0L z9X;w!!AamGkD4kt4V>mtMS{D6yL!}Zg3kq?>rs;gvj=)(H;)=GI0u}A{xA3<@I@YV zqu^fPULG}8a6fQAkGf7Ud+9c^_wE|O?5EqvKD(;~7k~>qYJ}h+;2|D0RPYG!2#*>h zcocY)N97A14Ib@LR|p;p9_vw;3myj^=TVmmz6E@XNA(pv2|UT8dI`QAe7i?oEO-id zibq`__)hSh9_1H&H~4Ok>L!@6%f|aWDogN8@Jx@&5X^XH;~bCbB6uEno=2q#ei;0) zMt;AI}=6}$qx!lR-DuL7^~D3{>B zg8%AK&0m0RuMX#byV;x;8#5Akl@$AuX)sug5LnY!TBG-Z-U?S zsQrT90>9-^-w1vi{I*AZDfnIRyB<|1cn5fgM|~l97kHOPeJc19@FyPik>D@DUwG69 zg7<*;c+?KTUxUB)sP_c#2k-Z&{|Npb{JlqQ6?_nU(4*c0YrU+ruNBtmHbFDc|Ji@6 z-TNnBY}$$9b4^^#3YxR~~Z5Jv&C}!+Z}7Fh*iO(oipm+pED> zb4P`^<-Cg?!<9y@F?y5?tayWV_ePXjgRwOMITB@P$i==gd*2PfMBWan?G7a_HNZR?(&NtK@ku*Ovzo8~Y(k_H{p`m^q zN$Ux%r=floN$U%(FK28bX}QpH4K*r~HW1oCLk*9lCnoA=so7ChRKF<&*jS6yN>_Me4&u}gf?SIv9@ zW3h`YnX9Q><=MV5HpKgN^9CAU@$H*8a3g2QDsV6Ny>=n~%4Gq)1|ahSvi8F~$=b|8 z9j%86SBJ;kuW7p=$9wPQ;M)-EX-*gVc`39pTWC+W{m5*BH4a1AUpE?HOhYcXX!mrSWy*>&(}Muj`r&g$(FBcc9Ue??OI5 z-}!u*d|mib`4ag$@U`Xh@kQ~?O<$Ao560u0`2NMWmG8fN+xb4=`yFgRu}U5`OfFd@z;+ z`A<3wEquRCddkJ6+gMTe2fLig`HKB^EM$q3pL|zQH=S%7FL04V*I@v+<|A z1MG(!BYf2iKMsL+gePj}%^grXuVjFC^T_H$%<=CoFsp0Og?6J04a7~&Dn}O@4&IF} zR5K!9>#t(&9$~6J;-*)=+C%>-=I&mmx>#tRR;jybN7@O)lp~*zzbqj zd%V3iEz~f@ncENLo zs$%hnxqD8G`j607fLFw*t%5%Re-fkK5_}8zmKe2J@Vuc~e-(50ycqR{&{lz0#i-W> ze*yj?Mr{y02|OuAtrt9sx*TPyb%M_YpKGeW3FfTM?$M@t0WAEy+hta7S;btqmO~{2 zx9pL5@sfeNhnUp6GjH##{ha!?gg@+Ld)X_xHevoCZ9k)x|H-ZVi+(Hq zZx8t|I-czK|C8hY)>i&+Y~}x)kpHnE|79xO@ozc)uWRLhcq{)#$p4U#|H@|7Fm&l3A1DUfHS-*oz_m>F&g-lUS1-%l}_U zo2-M$S*GVGyOZqodIOs1=@o{wmt;E#J$+cehBsLsHDpc1Fsw4sKhS684BOn$`z?3k zPk2belr~?EDe;}0HHiMIypYkEx$oa>3>aU)=^d!7}*!wwWSIR#6mBcp;(`BZWM}{Nk zgrdmP%wa>Fjb9NMs*`=AyS^7!<*kg&&E&v zr-q-sbk$##N?OxGY31HLcue+jeoc9@w&mPq*ga6qet;hf7P__D=mux+#F5ZlhUg2d zN;!M^`_S?_pj+(|hjNq|2bn8xm>R^1Jy1u*5f1)_n?}JBj;Sqm)YPlaC z@y_P%Ozwm&lXC3O7>a!@Mt|<@5q|Zs=jsnD_1(4T`v;P_J6U-0#lueBL=m^tjSKfa zWW*l5A0y3nb4F9j{V;VTW$D~i*h$}2xI0AqfOZjiU*X5lMGow72bAo!9mMx#i~l5h zZ(KnA2HJbj)C_k}*L#+sj={Hb-)*gX-T=Lq|5k?w29th?d)FtdQ^Ci>WDnCB?lm0W za<5^UT@>>37JkI_lrz5G`5!CT;>UtRM zb13aIbbhGxm-kK!4>X4EHaZLsoCxh94d0`x({DO=8yU-3bLQ>@;koK)?qB1Z!8aD# zrF_YJhbfn0(k6Fd3m?y=u0$?90ZrtRxp%j6&M8^vg^%|k|6*rv<(}8rhCRq2#&f5qXPd{G zOPl?SHo3sogb@UF4HDR6bIMk@!@(b+VHsP1KL%Ap!gskw5=~_m04O zD>INAl-ZZut@RarD0jgAQXO>X99@_vIp@${cvkvUDJSVq%zUn8^S zoT%J=w1=|!it{jEQvb3)RMJZS)5i;cRJ1p$=JMSZ`rmCtO?9<5#vI{(Zn+Cq_F7Ay zeM0xy&b_e0&;R_`siR|rDSY)W@-F=T8T810_}|dehxKjP!ycp4`1OKl>w3yq_(0nH z?q0_949XM*y3U(m;RYJtD1Wl)lHp*FOuzc z@THcYADVvJH+*TRuUAXBsnn6&BkR*FMx$NutC0>(#z^Bz!XRp48Meba7WJap}E5bX~7E*Nh z65r@he3xQUhxV8F$wUX0dS!q1nDk4G>F7FRIIKE-A|`vRPSrEIl+^Pu!+HNq(vsKN zxLe$dUMBxPb06Z1NcY*l7rw~Xd=W|iUwXk&+NHFg*1BqoUcg%cU3K0P?9SRuw9~Z7 z-OZSy7jzvY~SZ_;&hTHfl& zTgK^m+eCWCsb*caav#2&O$_TSuePKGorQT|(OFhOYmCrY_MzkKJEPv>$ZyF@C2`F9 z1-+%XMQ`b3Uq-o#pC|Am<`PUf4=iQ=n5HG!mNpaZKWj6={*yKtUuaYAXa2wGFT$&G z52MJTSJ5{&(w5}BtH>%DJ2-tx68DPB-5Y1`+g~|(s$Z#Oe&P`PFZX{-|03sV!juZD``QQ; zZX`CacN=;q`#5@tqbH@>er*@8?@2sOmi^Xe)rDl__a8d%Q%F6Yr|SltUz?oo!4&x> zd6c^SrE=7AsW)e}da^fHWE*#P)g!xW>35!?o$o_`lD6%qUF&fG;WpEch26-#O#A51 z{Fyhz`RUI_xH~uOjS|`D&B>bW)jCrV|6!fUxdWoJZGY>u*`LUVlv#z0=g^rx`r4s4 zLKhuj0=krQS6!+-1^$(OO3LsKF#8Am^?x4huMhQ2jy@*sJ1um-M5-Mf>X-h8-ee%l zTlA(s#L+KtR{a}CZ)!qslJnxHmwBx947woia3nkB8EO)X%r zT|fR4`5q+=&V8ZD_Co4kc=S1FwT#v2gQ^ar_v&%}QO0qH%-{g(M*4VZ-|0r@h7;7c z%sHHv|7~$C?&b{dHTSwZHSFasrulPTTP{4Lbs7H4c=xCN zIU~5yoz|$w%FHKRPM8xoZ*n zZ;Em_v*k_Ey}WU9>pkr)W0Pr_by1e+{?DVUOz3Fzso)*H%R+oNj|!j zGd5qtst*sCvS#&x#G!-(;w9 z>|+dEJXH18?K+{sd&y$mr=`-TmGQ0*=f}1u_&%l|gKtCgCAp;G1iUGEEWqyoczsW) zsrl6%=Zu-VLtE`0N4dq@Cy+6FsE_kVM+fMuRy$=E?rSxVxe6iFC%w4-HW0l=3c&^rd^XP8n}MpyiCr^~BrXl4c28!b{QN zCEF$BRo;=1Jc*vjeNH=$F_x2e2+k*-uhE&mK^}6pXK@Cy_?gW&ZeXruTfy4)u>uQZIl+U`GXEqfPy`F%6>hpzlw zAQ;w__afg!SEj#UuBlUbB#+5FQ|;qUyZIRZ0oodGLXDO>xPy3T7e7zKZ`-xp?^lj{ z&aCW^vr&>yxw9*bMei3Z?MuqK8n=hw56R~%V4+oMGud9J&Cd1{+Dx!>h*#RiK#?nj zXMVd-&UJ?0FZ3F;P3B9AmP%%G7I5#J&9APdq)27uW>DFaHL)4AVhxN zfW7qlHH0Vh1@N`xZ5VXPOK84N^M!#vdxY~AI_;g~t%eU+10hZWclVICv*o+lE&W9- z^FVRT2en~ds4eqD?U?gx&wW~M{kBO{sK1bRbL2iedD|rXE?+ij5ncQh=JcN69wpMN zjv%*&p?iyN*K}8mU)|;PYrT85CwPK35j(r{^7`j_qvzJP*tPTTTOAWrZr;V2ow|HD zUp@D)iro5Dp>vk3J#ol5x2-n0gGu)piO90YwJb{vy+yU|2dD4$;ZEdm0sh-kW-`8& zKHJ^KkzJzm%fGxy6@Hs!ucjs02PikkPR^V=y7*_>F2UZ9otoha${jIB7%S*|Y}~vB zRVrt^oH%qJC1K-M?&i>ZeunwH$h$f2LEcK7J3?`9!*ulGc>BMaCVKDH)XOO9=PK&y zO6qGQ^L8VG)Sq+jPw6ku+l9wy((j+39uI|d)Z-ytSf?{IZ`bj0=6KLUMQ(hAZu&ak z<9rW7yMeDC-)rd3P3Q&-C4K0tlKw@MyYOugnxmU`vi}sKJuB|e1!TVOFy$iicgVr2 zEpuL7j{c;^Fefiz%*TDYH$lHUBzJs^oce>%PR-r@Tn;w2Z_!DI6NjX^@x%O}%-zkA zI0^Ij+Dx=3VT#@)Ze+auzx2`~SLcRO=BQ=P&Of_)c^0~Z+__uN97|ZAl=dk7ko13v zp|ya?#3Q;;iI03To^u_Uv$I0lcZ8lG<@5idC#*tml<|V-raixO`1@MImwtgd3eD$r zvNyv+(&xz@9(m9Fb>f>P{p>XV@+kU6XI{^lFVtfa%I9mwJ2K9>n0~MX9WLxw{3PhU zG{s&M3Rn6GIddp|X(RoI+zaNs!6bcYn*BZLL^K(*$jqjU_$Fwo3{tiEv6<EkKFp!ot*P6a%?~RcVb42 zzX5p`H#=^5J?D2CdN7w?9Oo~Chw6Jo2lsc631-i3v%Egb8yt(w`tTx0X6+$d;f-2t zrr2L-Guhs*dAT#P>nSa}(nJsAT}IKvI{QA+^9d!P`P>ijBRct;g!w7;;K;FLJFJ7f z1x@Zm6+POy*Up(!SPNbF{*6$a;W4PpD+IM&qWx-%9cSCAQ$7CIb_w=c?COzyvQ8%R zgEj5sJ*IgBn)Ligq`XVE6UmRn|5yCb=bcsFHMiU$l5AfZ^0OqAhQ*kDbUxy3OVgxn zj-no~qCT&rUPn^DBanB)Id30Yrzrg8wNrUxQDmIFyKscMmh-leGVbHY;MZhlY_lYp z`$GAoudH+>G`N#F8=7QP6_e)5Bx6jrQL{sEQLnOP?7KOTVlG`ansnZTo-)gwqVH)< zHJ7w(O z*+Ltd%>53rQM{Wp_tYKU@W+8#!WlXDmd0Y@n#h{KNOL?mQQzS`a_;!Xm998_hxe7} z0_F0Ck{PG(@cxE&B6Iz9^iws&SF@8kcxDQgs>mnam1J`5ZGQxXOMu6m-BANsDBJc<4%w40%oa8F(j%KX24VyDx`dB9@oR_>jb zd6-pFj>z%7Rx-b=(*#GBklKr$8L(@O)Yogegl3F*RAY5(ERXhf>lSo$(!AbU-g?HqDy%x z#}n?fcl_?o3$oq5cW7VF&!c^%nkl?p+quE-toya-4mzxBlB_!JX^}fzPK)O^iB^7S zu1oT6P^SOuF5Z1vyn7*I18ayHO8>;$kqtg+`|xpiO`dqS8J8sMK2P2V*^Q2|Ph>)C zH^Wk_TI}^3d}ikbtp7gmK2unIQuST?yKtLgbY5^N^T3e5imKm(i zLWkt-c*)~2V;ZyEQ*z=Rzki&bm)X7@!rjga@QGw~mo-zB;-2j$}w_e}jj5WO(tn0hWmPwp0Gl_fqocmv8oynkXC!pKOT%hD##%)Dz z_AVfI81pVHsK~TlrH>O`_-n!oud~X$V5;u>((DhRiyx5@X|scR-TDJ)B~xcMJ~{QW z#`hxZeba->`))Ua`^oPJ(vnkW>U)2RSgS4S{wu+jA3iVYo)Rp2hw`R9a87xQuLxM$URAKgik) z+TFR(Tkp|Acgq<={>s>2C4DnIJZ2T2kGj`)%~P(4=h@4TBJV_|?+WeP+KpS;OS`+( zzO5O=8@X>w!WEhR*ctY1X?`;4yXJDgbC*E1*9=s%zifJd`=7ZtV&(+OUeYad+j2+6 z+1^;FkU6{>ztxqv@lz#!>Pf5SsoU~ZZ|I_Kc13%l0;e{$8lunUriPWpa<04++c|ZO{-#QO-4PPzo zLi5Y-GW6WcKer1l$xg!VFN7nyY8LY5M)H;$f1#D#o4W@H^IgKrgU5B-3}jmGi@%P> z`ZC&F;=}@xwQG~~p3fvZny{9jBZT#h6S$@S=qF*hbozLI8aLUrIj_lkHH0f}kKj)7 zl%)55CfSGij|^x38Nzv@C7kq@aQ247(R)DIJJ2eekIoRz!!6--Z3$<4I2^qfl=pvH zh4a=K!nvmI~sbY6+)vOE_!7;pn}g+yT`poKSS*uZU2*P=HmU`hfiZH zu(oWxGp8Ju5sAdln{HKgjI$>sGR_Y9`7eIHy?FSv@c&l+YrCV{cPbg!It&kC)Nn^` zU*Zb8{|D~B!M%2iZv9Pi$w1xS(7PM4%jg)=5qQ@^^zLtxYX-`=MRaf8w?Ged?${1| zJns?O5*Kmb)=B@gsM0mtvlWc^l>s%nYL-eWLd_ zT~~Q;@4B~B&A?l^@BT}^a33SGA?nhit{In3=xSt7>N@xRy`w78_g#!fGC!MiXC~v3 zdFu*S+{(A-&*rq+KN{1%avLYL3@XX8o+V#5A$RUWj_l&zlN$P)y*D$@LjDD3%YN67 zGv%!SM{bp!Z!N`-*gb+-LLG>FmH!X$m&kqa+|k}=6aTvGK04oeG~~Vn^KtqSO<#Mi zwVHn^f8%iTdM$tRspmt@eIFn#DHpRJjB$5OC@q3@T2ixqmX>=6L+mDFGEcg87~>D` z`Fgy>{XOSfi#9s*ME}OV;#@~hNp@9?e$}1asMB#xH)}3-wX89S{)as6BX7kCPl((W z{jU?eB5!DkKN;JWpi6o^#`g4+?GGS3$0Wb-+no|#lE`nyBQ1HaI^VjQu%&GBG3i&B zpOAKQJ^Vk0c5of-VKnXHTE_F&aCiCDw3kuSzUzGzcPB;ZF@(ga)A@ck>juIop|6s3 z4ii1KRXPt~FX@ytQhz#)-jic?(vPlBeE+_kBe3@}rjY*Y)p=#hB+bJquVAXtr#{-) zp4Y*cF5`q)`ac(6Ci#iOY{S=k||r*HgWQ%~zh-Y`C3-qct-_t%Ye@Zp}s zfa+j&C^t-_+%(N{m+30+8T;Wf!~5Z~OuinjxbikeoAL{dw&mCIf03(Q`44kl);S%G z1Dm+VO2+B`G252U!yL~Z=L+%=hwjpb_~gxqkv*SZK6S3>cd_qGd8nW<){HG5;rnn| z0rB{m6Oeg;f_SGdmvM*8^PHGIg zZ)ikE*K=BXT(>l)`edKXseR5~!j^xTf0y*ix}h7|szTRx?GFBvxvF+TD?LS%IVzvf zZaqc&12hk`>}EH6%7^W&hd*j#3b@aG{(#!_l7UAG$q#(JceAWtrr49=>q^d@dd<}G zO7b_0d($M3a?Yw4y?HNhD2ls-@1a96775MauMO|X-57B%uISkl;2m*aO1NUy;ZOMe z2g*~-Bcx4C8Bc}Zkt(9Ci{6w9|A^fm@GE-Lr}&kz!rIXPhy2T0@D9wIba)|obd8_9 zq}q4uxLE(!W}5w`HdE{xZ6?}pXfw(FJEmeBbq>7kMsDb~^{ehJca+Y;Ue>)@?259j z%+T2Po=_OiYctgjVy=GFY1enLw^H)_thP(ESB2bGh)H}aFeByKf}7Y+@So;BW;hMV z$w9agS=JA8oPO`lWd6WJKDdw*r@#AKC+jZkA7rj|?G|)&FS1wW(%F;0L)QAn8{Vlm zUQ(8p2Ct4|4EX=?_U`dfR@eUjGcyUyB-}#oA(DisWP*y30HRPPiD0-?D`36gDM73T zBDK+K1(XD$r@^RY6s09SCtTXo%pi&yY*K3t&{oub6{NMdb4mhMbMc1U@^CTV_h&yd z7!mF1_x1Yy{&-&d^6dLwYwfkyUVH6f;1Y6yaF!?C+-W$1oWAkVK#$~dvH@3%jIPtj z?0ntZ!M&{B$p?2ar`5En0s6mK?bCe@-o{P?R#fLtHd#T<{Q~fpeQ{EUCpBMTG44k-|eau|^(YnjAH?@9c>%_8t;Ljj*DTiA2ne|@Xr`}_{-*IN$ zHNfeg%--bl@7x~0tr)s0yQD5dOIc%9>wG#7+8KTOmib3#fp^ELqZOW+buMf{U&|z0 zYsC(}=x8i&0l*tMZ>(=O_WH9KU+rn+o?7O_NoN;?}jkg=*i1C}l9`C#e7o~PXdx_{~q9N(C5)p4zJTX+lc zLr;A8BV=f@;r<5iDSiaq4KUZck;P`Y9a-#dWU*OpM;5zVvRKy;Ba7Vwj$iB^Qui|9 zJao6A*~lg0x>sh4*Ywdr^@Z=zU7jdb)T0J4>S4{o)hRcAsenj zcX;WTb?am^s)+KAAv@8V!#W?3-qoe2de>@=aZx^cjfK%U)Y+LE`@s3UE6BTng@!M3 zr3F*2wLX)j5|Y1xGQC zdCcn^#&{uV``Y3)Y046VBS?eZ#H_EA=33yR5!`O@cq@)0tM6cb9eG0atS;b0?`b|h zN-|>I33Bd%ot6;;muJdw=TWxkMpQ#@R%-mP2Njg=0S&n`(!(l_DGIWE|~F4wgjJ2li5?NF!%p0jT3kO*K_y( zzW-zJZzyLKyYsmJFSvO5xp>}o=oz0ax_Nzm@Hx@|gN7$XmSFZpdWTTso)?Oz9PoO^Y)gWDvZ)ZOVHmKe+P6-nCl8OnCLE;Bvys&%LI~R&cZl z{!8?)2sq1&%x~V7_!ik(H?Y6qB4alb-N_a%>?JAhtJLu-=ku>-1X@tp4c%Gc8Ze!>{4S3*HvgQ!gz0~@u9=r$ z6)b=j`g3?=bxFohBA1rDo#9V z8?vtP@G~8xN58teV|+t~2g^0^0}(9nM>V_ z8*2gYQXt3AA}rW=@Au%I6=(@C7A7rye_V6c+spkA&cPNqZ&()+*T&ei=UViId%=Xu z2zS&6S~}w;8}L=MBP;Eg4UYhyK)8E8Va)++CiNzTUDWF$p9MbmjV~h23EEK$93=rq z<$|@o^oqXu=$qFUXkn~ZPro)b*o|B9QmW>%vfTC1Iun0ks_?kdOIURp+VyZUHY=>8 zEtJ0t-B7P(wS@5}(r2~lb^IEWGhrjzhD6|hjrtxf?_{c(%fFMavF4^8LodM-Kk1|^ zDSjZE{(wipQ_D)gtzvMj2tMOChA&u0KK&Q|nK{08MvMGj;?|?1fQ~LaMou^@rnLK2 z%8K$K(e^}S##^n!g~1IGyZLWZrg#i1Qf^+1RVSFpZ<%VttTbh@hBDWXccN8xva2@@l z{!n!6)hG3H1O1y!Kh@u8TcdrAhQBCJclff?bm;eEgZI+L)(D-ejnKJva9gq;jmJ@J zux5)U@`mGL(n3eKX-ov0(o2Z`yOn?7o}rh!BIV2n4x;6Vc503|<2jw@2J%#6EBSra z1@$Y`&$?!^E2-j^%K}UNg#F32H|}#=cV^?Bd7oRf*^JeE=5)SiPxgazxA^|8NOZr& zim|r%hFG@uzF5CtGzyE^{gB05*ts8^c*!TPyVj6@tG7K(Fk$;*r9bEJyJ|}nbXjji zR#~n)y}&~Q;SS#Zn^!&kU`JwY#^gY$aiM9IZN#gd>>=E3D*0-i1;*~J?|ql|T<&`M zlcb~DKWTb+|0ki9KYVLi=QE%9$G`bW=kE@EvS=K3z_VLFiMQ%!UCqFka_{cLL{Bi8GOZT{vUMpB0SD)p!D%ZG!>)wFJ+X7wAK~^As?_p#OeiwFAi(ChD z;M0cCCxbgh`N4_*GsPa?KE)pQ56%cWr`TgZm|{;jIK{r~$Q1j6u|+MH{}y^odg$_m zu2^flzp@A#G56pK&eKoh5oh|Hsa8weo+7&oo-+QAMRqs)ws}RR2TNnu&nk139>=!+ zQq6luf2$=A81mjQAt?PY!RPxUt(KQsde>^rHZU~cvS170kNXE&-sfI4bTBOaF2O-n zKda!D*NT>QQ5I`SWuEJyj4t+b^Hxkhn3qtS(FMPvdX1m^$>0ZRYRy{hFEDTVOz^tm z@<$~EaO3kw?YSt3tzWx}cZa$uv)q*zH_8sQ_@M<=tOuIk%$uxGN|`H^e2Y7j@eNOC z!0lfDkZhkn?!GwxKRa9R9@UI&*u{Ry%^ugo9ybPB8VgPB2fxDIfA(#?HNHLNS}#uX zDycnYY}vw7IR#}y!>^>isnk1# z`mdlJCD8I>WN}5v;&$CGczvsoEkX|P5N?MjKfJGBY4<|#-t>*NurJ+L`Y7$c zj=sDJ-Y;YxD>fB{x<~p#9anOW&760XSoU?XDRmkTA8WxX)&ia77WMSb(V28A>p<6R z>}2uZ-)Ailt<*X`F2@Q<)-exx;R@(uCvjTW&$=cYbk+p#!n3afjfZ;+fmLV!5#e=q zKEytDyj6H8kGJ^X<;!)Zz@D?bpXy2PUIG75&ij&`S3C8(6oyCZ9BIN8E8w%rhl}^f zsK}9jg#1Yr+o8MVUsD?PoAMV^Mx^c97k9AF1Go2dQm;Sfl3*9|hfz}k%`^FkKllMW zR+Vc&MG<}edT;;n@~{7}cg_@#r8oUrI%v<5jaI0Gc9bvj_`kP2#$R=()lxMMeHZGv zXJh|R)zTDu<_qYCUTL))Lr%z;72Yj<6V5O+_Imf#^!qX7F4)vscB|EL?_=n4^s`!) zoljZC-kz%B0Xaw%O^Ihk%mn|Yc zwzrn$*El%ug3N;#LPJ(RP9ayC)U zPSWqh7LM{Bz<*?je5)<&0B%;%LvHo~)l!&MV}7 z#j^j8^1g@vJ=%`{ z8-KTDH_IPSIq@!go5~qQ88i4QC}%nOhf~hi)Sh_SGq@+~8OnK+yl-OvPkBGW|B+?? zm;8e%XRyoOq;f`6#?^dxP|iy7kD#1lwTE(cP|ov|vzNSkEqkr<9>Rafve(F;NI8iv zJE(HTP{wSRUG_;_&{g!{tU<&f+k4C4QkB)xxv4ZHob|3)1y7t8?zkKnErB-3KZdyi z7xMZe_hQfPW1Z{*rb}}|=x@e`y1@^<=O;ZD!$;)&DWj`O@BR7m4ekUl4t7yT{uFm} z{*vJ|-5t($u-eY!GWPh{P`>2AyJq_1g|fBh5mdldZ1P~KD>5vteyMZLrI z4*PrETh~38+34d1j`xloZ*+aC-!gsNP-tw~h!^nZus^0B+0c~mQu^yktSEv{$ah`e zGtV{5$c6G*)BD`%>#tr?IX>)4hyu zC*#Xq@%&E4b>VsFtUyoUsS*}0#P9RgrTU>Kea3VJZzXpyPu_(yTWVkJEzFZmjB73o zruhObss2DqVldD$3|@{qlEIs5VvKyEx~6(!FSynVJe6|3+A#`PTTwl+&C`!JTKe;O zx3tZ`23tL8z%QG;+1R?w9!cEu#DV+SH4jb99zz&>>4hF;*E}*Ydvf)pHt@I?Tv7R* zufiMVI(1suzUl=J+JNH&#J54aB=gCpo_z{KgLL=nwBAcX-xjpi5fe{J|k9c(UhFed|(kiUbcGBhaqzA9&e;Vbqb1$~z%0SC5)`kyx>&VC2pt-bYlhz2m z-Q5m8{sUZimpO2Blaqki6s}D``jh12MIPzQFtulB|8Q%He zW0fz|C|*c!yL0wEh`N*)x%nF5iyv5z!c-S)-3;KqYX>|iZ!|}(9#Voix0F%He}4@4t?2m;52QAPhC^?eaEl%maWB4A2ZZ; z>hA={MYC>(Pu6*58nWXw_KxyRvQ9yMdJp$m~WN5spSFmH~Cwlg2A^`6Kpq+KsLfWo{dvQyzfR+9H*$Jgfy<=-LI)uA@=3irJZ421DY3o7e%#6ty z|5KBpBYzCo>Td^WBxgUfjLfm5ITEmcO&ZasA2Sa3^6AVm8~4ogiPsj{QCvKMEhXV{ zbguI}sdYY2X5Dm8V%^8IWeB`({C5H^w?o$lZj28maDMomvCr1u0e-xBgB9B4gMar~ z38HnaD^^TwU9ocF!~FjaIP)m~SMh)0Kr5k|_-g+9;M2VPUkD!g7(em4zK|8N_;2yw zz-D`I;U8&NKh_>Ev`2SH4FCCDf!-HtDTD94q93>jfBSb_$u?dkJ^lRHx;y4nv~L!= zBqnZ2%&Gax_lXa>loakEPaE^Ey@|#q%{4%}Xj?6Bda!^oYz0rFG}B_P&G>C72*$BC zd4I-xIeT59F76k10ORagZ9^-YhXrHED;Vv)jyK<<^?_ptHo1dqpjXIT3fry&Uf_AY z;dS0wTOMnz;oaH7XdROEB7?!!Z;IJ#>7LBpq!A6)+A^6sw6;u=%bIezT-K=xa*qY< z@p4(8#>)L0Z;;7_?^VCF-i?;KGhkmR_mzM>O782>QMsD~_DH$U1nl$VJ|D1i1@1Ra_NG4gigPBdHAni{ zy2q+>7oEYWuYyAtpJW7WqXPEj+)rqOC!2uF9mQbv4o`5L?lrkRynl{c><)65cTM$5 zSMXxFJFMVExV%joyui4L;crsswR~obT{BzCS%WkO7*nR(@eiprb-Jg=q zzV!7)gS_Zz$QF&4^Y)G`tEH9p@3T1DjdCqxXp_1H89h zmVvA#!?GV&y5abTTlQ-CN8le}*^ih!F`=r9$b&7I_1wKHyM%w2SoXgw-8lT?EW1Yj z%kW>uzSQJlzdDsXgRM#n+eBs4`8VCNmnhv;_^-nLxBRp4&$8^hOrE$<)wjr#h@Bny zq_Ug&cQf|CmF`yjw?d=jUxa^=W#4M@3=CD>L!MO5YS_z{E#==*WL-+P9RG65zDfR- z_*Yu?4JOZ!P!)E8c?+pB8<;73jDL?=_AI4)691EyeYO19k*-?DnWxE<7^?a&@?=?+ z)$I4n!u$(c_7tUi8voPqc=A7o|2fi`JSm~7Uvr-x{Z{eMJ|FUJ_?gMbz$D*}gO6t4 zQ8B9Ep^SONNxr=hc_#DT0>6a3)myRhd~43(VXQ}I1nH`Cd#jAgB@6^|)?ExvlviII_Z zM%Hr|_@j0wvYv~_)%x6Voz;?vETvt#4dB9~zk_CEMfef!Gl?JZ!jH&qj>TE3S^r6+ z_1}cG{wqA(YBFK1|Ae*vAEJCSPP`|Uc*<(g-jH=?Lf^V)pX?ng8OM5b9(u;GS4-r6 zOjh-*msF<5vnLvEAuPHQ4R14Hx7jng;X9RQD{)t?f~P)Nwl@o%73rjlcKi4)Vr<^v z?Zwxo1=!2F@_#=+VDmm${u|(<&S;hH5c+F6Grk;uC2?vmGTz|J@Ud;&o7fEfSiNL% z^K|0E`MixnnvXpL!-KhR`_ZZLy&rGS9Cq-U7wTP8Tj~p-X$7D578ZcN2a)X+#02bT z<$42lrQGa*J=s01+l36U;9PXx#w+YLIt&E?`3H4-k^3EFPf>~NUGDyP_C(lk>UDW4 zw!@p{P7O3ySwkw?j-#)JO@I~bF%w$R?eAx8Ek^Es(D!V8!HwT-&bffRz*3gSZN^0U zUsqp$Y4bCT@5eV>+WZmYdeFG;VEYZ1_Q2mK>|ou`f;MadW}GwRwES}!53LvbJu&$Y z#Bh#o#dNc-&e5LZxA)%C{JVQ^ZhqP7SK*}{F1~j?ZpHTy-$Q(NDf`<`DAy^^6I?+0 z$!)Ph^gwF#VU`N(cB#@wU59zo+N$w!6K0Pmgm}?bV|2Jy@{U2fpJ4YyCfi z@BW1Q@VC3qbZOR*!zo@Tu16K@)`t~L$$Gw%{o_n}+@%jl=taH4;})*iLN zZP|aIjB3_y&iL&;^0R(7#98(m_#0R+8(1&j9%=>A2{{;Gjad&ap&L+82Hb9ETqfhU z@F(5zi~13ioy(_vRO^G3QDXgI?h-Z<;6Cvo3?kV?qt=K?(Js_-!zUVkMtiO9fJ#K=b13Oq7Tf9*NwvK0@C({ z*Lj5L(`kA=Mxz(>1Ydf4&c7~8{Lii_2TFqNXfUq3Li83^l#J|{ToS`J;dq$ z<@V{{+2EADq9gBeony`(CAZN22mO>&R$lpV&U;UZx9iN^Va{pP<_~GJX+n6S=k2ToI(O7NxnE^#I;vljt@Zr^chF`G6636v3>6vPSFN zLG`9Wm!4!T=k6f)5-hVe|D617k-Ng%r#kn9wKgYtdR{oc&v@-4U0-z;oQ|=p3ZIy+q`F6;@HEM&gyH--#@cs@Gkt(JDj^5fIr_q-fEFv z{p6Fem3bFgb94@vHPKbM>q2DFca?;mpIaPinO78&4&XZ(f#bLE{e}-Y;qeEC1&&u= z08fVjU@4W!pz9zJJ_ z7rsuiD(Qnu-d>&?s+UaqCfwRF#S`+-^{Xv`)-tyK_G0!l+)ag!8JJC2G(qRK;t~Ep z9g1`P+f@6v_yf7<)DZSY){LsRq5C7aYatlz!-MKB26*s=HRRXeS}SYK`smux<_?}{ zLw_ur_Ry|!WJ=!9MiA*(2&~0N%E#6CQCIuI!8VrM2i7evZ~oqYQsCcWfi{R&*hM&s4h< znwz($pRtGKCylvlaYnt;m77Z4@0vCxh1+G97yj!2GBe%X{|D{<#t)pfF>Zw&o2PJg zVjb*cPdN`)XYkyYZR+ITEz=7-neGp~1P#E}EBQ0SNh+5*dXZBn$WOVwoZ}3_ zpGp4=3^+Enq%*hLx;{3zXD4)nyGiretLCgfkNHRoU;VJ7NADw_aKik{KHvN+jr?=- zPx3A&AO9GiqR77!lqnfeJ@k4hAIxNv=J_MyH6E8U9-a7gm%d9beLhYf-!XB>Epemc zun)ibc{Ba2_gGEQKIB0oRp$Tj|1W&B9jhzRs!WScs2_>+a-UwXd_Ez0PxHeHr2HPTeM#Xb`G2c%;CS%h7qURkQu%~bpz|y$py}@HEq%TsIRtn zBKX$1iSFjeX4O0NO>**gvx-AC4*_@o=pC!Jm7eGw3v92>$v!WrJ_J18x>a=!UX)Om z;9LBN=4Aq5!F3_yIL5RiDO@Nwg5wuO;zr|^Mey5?U+VyCSm@FSE@$AXyr)>l{*`h0 zZ|IT(_etUZAg__-LC5IBTIny|xg%odh`5qS+~PiQUyH<*_lcVjiCfSot}qh!E#l6? z*M0VGxx=$=jqbA|e4Tlhd>eV!JNvBNI!AbxaD=bhZ6Qb4LO9B|Lch13xzCF5b-TSq z{~BM{SJt;T&)R+0#<7xZn>G1&)LAwH{|L+8DSt8kV(j(cm)uJ7rmBtHmx1?}Ebf;| zb3Xp_dH+xTsraYz{-4s{O4@(t`!CY7N07YkX{E`>pU?U)|8)G*ExS?a7m@ZMzA)*z z!-7tQy-{f{!haF`f&39U+fS7KJEVP-?`hHxCCyOFeo|>J!G8(zKlx|jpN0HS>F*@% zV|>q%K81MpKo2QRMBeuU`M*j0H!b@CrN4)?Px8G$`iM+%xza>rg3IK;k@PoO_Pt7v zj83w(s$Y^mB6k$Lm3@sgU$g8xYj;FDD*6p;-3%$1}!@R}7IaRkOG5`JEAIutmWye)d z^0H^k!uAqp;d2TYp9Jc2#l5<2M60iU9($Rbm=)`CxDz_!H+MePj$ALx@>F`NAIY#( z_K*R{s?bqJF8q#5vbC7aBk;$%6JCGWa(Dmi;+5;(^#$zr(v3VM_<)sA7ZVuX^BdgQ z*5N%dfvleXty#@kC#?|hV`rstx1r4nKFj~f?jhZ|w5biAKibCa=*@X)V=Mi(7_au| zSQS4wD~GWX9Tr`dUidWTxv$K`#r{wKdrOkfsu!LlI6&th7dYwNY3(_DH8zEoaWA9B z3ThoPa8sHb{1)lfyEyOQ4jOz;p=?>G9jaG%k&p1dqrdp?M6+M>qI=q?eX?MSy#w~y zkz*uPcnj8KOmh=AjlFNds*I>!Fm)T54f8%!at_Wb)=3sOkG8>|oz@}e&fHhtJv_?Y zHQC3J?YL;$NB@(y|L!bp@8H~b3~h%dSC;=f_eLUZ9!8sy4{@KnVnSn#HLgr})plX9 zmNIC&?T>t<(364$eT1-#yS8I~|>- z>A=!M1FdcQ+`fukm#orRH)V|uj!*7qUwXgGpO<6#^Ae^y=hX@QY*TmH$l!Ix}bZ+Gp# z5Z*j7{0-`uKzkVQJ;84m@s2)8V)zo$L5s$kdsHKIw>Qz~GN~`6J4a0jG5@8sLHaI9jE&mR z1|Q@iF6aGZQ{FoI`8~pGZXOw|UfC~rK53L69)3dJU4@^M+Z{8zdx+`E9!B-#V* z+`Y~14>k@;szef51$ z@*5aW1jczQGPN_e1-Z|c-=~$Xu^9(sq2Lwged#^WD7%VpJZ&jU)ID-nI4996bk6>C zPrwR`-sVD=EXKzIhZi$epTl8fE|QrDUxdRkw7-MBbrv?6%76>ua4mT7c^pmwho3x+ z54_`)5tX|nQhroyZdfKS>Q}%Rf{|w;I z?3>_~_5J~G>8xd)Ei~)ScE=v^e9jG`W4(oX#sG`uM;%@3@6*3yBl3dnJM_*?YWTf6;ulp5d&Vb!i=4=D`PQz|% zx@l+k%SLB%){Vg7A?#N+(*}#XoT{&q^sEh)$J`AWk|_%Yd!Vn;v2G?WaKs(dqQ$yz znuFfTnX*>hIhE}HbN5q8EBU__Q=Fx>HOv{iXiyb&;TSq=vR7E{@^1DPH2igXJGzJH zCF|^^){QI)`y7p)8@{l{NXDpfcl6fJ4*Q+CuS8{`r|EkIu>U>!A$#~AjEO^bAJ6&s z0B~U-GW|iw_6H;5AA+nufi-C;Yf?x^-6&bMBqg}*#7tb{J zsfFK?%e+S1^!XD{_qEfr%Jj9um9+VLk-oCtn!cj1KGq`7P?xbo`<5v$eWqkrTm7za zy6u|bt(&ed)CJ}r2W{x;XHFzqWdYU+`i3z(i=H(QIR7j zE^?g`-`4n?1LKkN)UA@${s@~5smOZT*;lpFmWhm08+KhRY^=DhbY!*C>r&r4IB&WT zc(b^Vp!3wbNZXfhy@RmjoTr+7Pz7OND=M#LZ#62|S6-{SuR~rdJwWsh-})|OD~<{uatYTsVXZrX62hAZBlGx>Fn5h*f3xLAa7Fi+J;D|6 zJKDF{v-C~&D&v4@^-nrYMvhtI+0s_y-U6R%`WvUUNAk@`Jo8t`xy@UDp`86)%(oLB_{hW;iRc03LSP5+`k58-1}dzqfupV79Q0*%=M%sGyqPVsY4SRL={vtqd(j`_Ef_25=>lwP!J|H1 zfPI6d`MM7dzZ%!Qo;F{2-fJ0;^!6XOc5q;_AM`37If^hvM6HW3E`-fv#2jk}FO{VRwVGFJB$qmydSfajW3*P4YzGajW*_uM=+dne&#d z+LymVSnwFp?^}K5zPuJ$&X=7ht>+v``hF|>oF|!eV}53^h;t?39yH9H&$u56EH2Kg z7`}ir8uugCV)r9Ogcl8Wbp93?oj>;q*J6W5=#OonU6z}BYJKj7nmL?*aj+U2dz9T0 ze+@bnq3QlM&ThJ}ci1`F3c;W3&S}mF-bMawa928@@a4Pv^5xrE&%W$UjLzJd(83wm z+nLT;W{hKRhjNmc3uwN0%rA_Q_gL@PxA>n;oq7xEgE6s`(GPmsfAN>S6>0hv;VC*e zyU@60wa2&M8haF8)+OCzirSHHmF{loIl;Nq5nxq9 zuPKY8ZyBnLL+CHC&opZAynO>MRi1=~hFVxZ@1mi0&CU8%w+T z!dU@f3md`aKX7&-VZoX0oKvk9-AnItPSv;U3G_W`@8N%5g7!*x1=?HGhxV#XshoQp zV=U~yaPD!exA4cJYcP3KT~oIyp?{-#gbb`Nz$^mg0c>(BX( zYs(Y&44*9<_Oizy{y=)Te}BDqE%Mdyl0Ij8 cqnV#BuW}W$}vm|oj?fq=7b`Xxz1m!&&%r|}oU8s)Gg`4}(1~WW^+<{v&k^5q{7aX&o@tSk(fu2yG>02IS>GfDGzxYRQ)1EgRTkLI~ zCo}xuv-JM2jo|&Sar4O2J?bTMKQX#bFU{O|l|OUCRdLQ4IB@OYP2cgCPRfWr^p#y=vcZ|1QuJ2p+ip>4nNnwX!r`wSo_jm*9vQKndmNK zs6D@Pk8x*|C!8hw*U@{v!mH@J$$!4@^j^Twq-l)1*(%%c;PtI|bVys#7tLl&ODV@X7}pX__YM9DrlVoKrJ_5aN3V0ke`2Fz zSq$&%x`N1ex0btE!#S5U?TqYC+L1en_td)Rz59w&W#Yw<3#?=u;qg{CM)oS~Rd=sp zud>4IRd~OU|N0l%tC&3Z5{Jym@O{7jm0-l7J*S{YfILH;A@WO6!J`+ z$J$xxUwoHbdzY`;ujnn7L#&}(*fD$ZEwmvD-@ACfOEfeK81KLpEhx`%-krN4$7&kO zXVP-6x{LGhM#i!{+iGeY3f!~L@Mfb+kRqK<=U#737GcJ$G|OsYZ#xT}h-Oo#o4uhNav+e70LE{@PAPdc_H?HCciBigJlI;3~zL9R?y9Q zS(eEeougN_`m|ozx39GQ><5aer#K+lQ+i~*O9_v74ZQ&Jyxcn>^m6Z|eB(p1HMqWC zAha>1*#C22YJFVmq{j10LizN+Ym;k27djkWoKuH$-Fr5B+~`=xhQc+jJ)7@x;g1PT zr=IECFAi=#ps**zx$U0Ko8)(MC%_eIW6eH58+4YZx4*}f(7!b1qU6TrU(8(+YOcAy zCu;*TLf#+r&xt4@EpgIt)@Tog_$d(ySE`HSTQiJ?)IW0dmr|z+PGtQ`-)<8 zI0kfA%fG#Zdt(E-v*mY9F=<-4V^~o%l{E3)$av<=fR0pbI48Jh$)cVt-X?Axik$91 z|4=7)E6dqa$R_Dec3vFp;5=V$>Repm2v(%crQL-W|7 z^uUmJ�i!AMks(6!|AdcmcOHW2tp<>C&ibXx1_wxv-zjf@-KlQLIZFYTD*dU*X+IeGI3r}ZOb75F7{%06e% zRX*bUr^@!CLmcNVDve`LmDtI@$*#ObIqYjxhSzmL(3cbN$5G#O=;>tk;8*wp$BXzc zo!N8DIDJvBOxw-ctu=(P+iEQ?vfbdK>{J$zM{QR5ZhMN&847rw7=Dd-6X%+GI_|g8 zxY*C)TBC8k&*EN;#sRC*@?MC>aV`;!do~)E{8`*n#EB=Uy`-qc%h+8@ySD8Z8%)iJ zyzM}I=hr-YbjNrbYcpfey@UR5WX+3~TTdAqS>vPOwFuEms%4vj;wX&oni2IwS|nM zE#5jso4KEn9b~V&)xTm;u;b@i8pE2DzdcWtx*dI&fa}-#P|4 zy3^wSZzD~|x+!+&FNOQQ?uEdsc&yMBa&#EsChOz^peC$vyLT{w` z7RL_NWONGWlUBAss&VtE&qsRMp_1!Ai2Zxqpg9oj z*#C2@rTuEl-o^XFhdo2!0TayI!yA|Mu8p=q?UYS`gUs>Az*PIy=n(xJJ4k`Z9^;%6 zyEevZ5^Ut>l-OPD)8er;TvcQp+=)CezMr*QHZo-o`2*}=eyBSn+~;YHt=h6-N74t3NYmJ>u{K}_3B7Zxw*oTpiZNY|ML&Vn5^dc)bfbttASqIBGi`IQUjrso| z5AWifD?i7>T}gL-M+r7uVytVt)GyuA$>&&=?oqt&Ks$Idr_k^9)y1`Pb`QPQeZUHM z1IZjOz}`v2sn|;PJaZP02-iQo~k_(!T0?V@NJkK+vyR>GtV?P2u zuftQoJ!3~sNGqxV)7cjT@^9rnO-l}2se!i|}WVa%|djvKf$}hI&B+>7x-q^}r zOSs3A7YN<%iVMFVfuR8r7>mZ=j$N&)2u#h3z?AsY zc;Aym(iQ`og;rr_Xgm~mm_~#cai+($)Dccj@(lFdTbIj`Jj^v<@%7Z;+lS}7#m!; zc#4s^E?gXI$05g-yxA9u4V9A*{xDeXl5QHhynifl&a(gLv7ctloHM|*@Hp=O9==0- z9k>9$a0%g~6%L;#`xzGz?(H4Q@o)m)kkE<#t`<}GH5UfUkqIpH3om7ZVO7RL?$4bK zM;9{Iv;})ezKZv-6VQp5Bs9 zH1Fy<_B_(V_t3uMx6KNjgl8n5_-D!1QyhYYM*RoH}y(-lGx5QBQ z0mqI-8F#SDxQosGX1nVS=6`N#sBvJJ9{h9%dCDs|wg1%L1ni?c@9l+K25`=Zr9qx$JxXB^wG~_#Zj%%I3UF_lOT) z$9l=!9$?+NmHw&!hnZ*ReY}+Ld=r;JIGHq=;eR6x&&r+dbA!KXW(^1TqU%BmZ7HLz zSF@hA>D?P-Ht;>H&B#f~lk?QM!611K|B3zF^=hB7X8-x zczoVeXwg-ngC$c!2Sx^L(fKINpZ_=M_#~Qpy6v2uI(?OmjpXoHY-~8Nm1N={#XgMa z@BQG2Xls7J3dXnUom4An`U+zo=Gp2$^v;>-nrG;0CpbE@$nmcNcW16}{C@C$=61*b zs{G)9=;+!56M5!@$X_k$Gz>O#UnO zsy*=~`6KP@;9lZc%a61jIn~+!NBYxJ_r|8wm%#Q$9A?4LM;``Ef7 z>$DxrBw!)g(65Z}^az}k8aFen`(x3(7O;9^o)v<&ew;I8c(D5yE=&oC?q!50M)F-I z7aX}1S8Gq?ynUH6yP_McFP?wvrl=4|KA&PL?*X1#=` zpk1eVif|vEqUQexp2AJPKjS4V-tu(kukr;Rn*l!&2R|YEE>-Xo#lUhrFy9HzkMi

r2Kk?=f0O&P*#0i~2wvjNB^MfALi;?G@o?q+k6j4v45AF( zJFVcJ?-uZk&w0i*ueq19gg?&|Iey{KGr+Ing+I>#tMUtfo&hf97ydj049YM3 zc?NisU-0>SUbGHBg1^x&I8yP3%Z>GX9FYJ2S<5C-?Z#Hzkf;#JWj;}4+WNL5B_C1 z6y0CzJ=#uBV*W8~mSLa0>8Icjw(|4SpyzS$VBOdy)%pE|3&Zsp+^G`J3r}v&^e-na zQ#Oi#Ki`33?uJ)C=0a`_JP!2&gAYSfy5ZNmunXD_Z!P{v`d8V&-GFar)IS0*x45^r z@R6c`{W5z*FFf-HtPvlA|H1?K=7X*5QJw)0p22oRzt{;!7cyZqOk)NFK1wbn=uq&um)t2FHUt~IrzrocC)R0t-TNqZzN@L1QUC3~+s zc>mrmcU;}|?MwEa*>~`EtMFE9(`Ek6_0vnR+XTNfae*s1Ok==!dE1Nah7nffe%j1f zyyc6G2YZ*H$tl*hUGz!u!q@1(5Bdje{pb89$?u7)8+~4}J^vgl;eF2_=4EhLcs$)T z)Znt@qSeSsFCyKWjNR4jL*8WUdZC*g%(cOB*MJJuud*&?Ox~xAPWEQ)?46`1qI0v0 z`8NbT`7G>KqK`J`jmj$~e221B?i;JHp%E$X>7sef`}&kO`tE=|ANxAdwnfVuO}>x# z*UKELJk!2sr#kK1)2DsH^*-fY{1WA1lPB87XnCf6zwXn%gQcIfFC7_kw0+Qq&$TbM z$exc4pR=})f1hh#!3?K;^A#T*Kj>NC_I;i5627E;R~7XczpsDRzEGd`L3=*ez9%Rz z`Agczzt6Sr*{hxQ9qiLS(W<`f`ya|n|C09oy-)iNUNhm14|<*XoltDXuk?f7IZs2| z)V`AUxyQ|C<`MW)I}UXB&Y2Hxbiuz}tMwDypHaEFUij$)j_zQ+j*fFXZxU%AR8izM zc79ud#qaD1O!yAC{hee7o`l<1EnC#A`f}Nqui#S|s%t0s`*8$ktN;2%oW1BC_7oAE z-StlIoY(&s@3erA|FX>y$vHX)qpzMYZ7IC}Kc$_zvphfEYAWV_spN6TSm*D>=E(Kf z97%^>XFxwsKtGqEqm{|{iEmDZj%?#R^NxFw!A5Lv?0{wsm%--7=J|#$ zJO*tYdXTb3SGO=;yazQ@HaKQpfc_csmUeKv0~}Al#>8OC%U&LMezJRT#SYf+hmZ?a z-cZ&IKM{Nt8yusTH#@dBew#jQ|KeZPyW@UUUxMuo=A*D+dHn3jm+5YUHRqZDyqJ4n zx8x%S&AtKsN7;bo-?`lVZ@`X5HEmwd$BxFqYaRO;*y5GSh<8`<=3FE-pe2Rr-QM&1O~{j5aB=q`6^@OzB$9L6|> zF-~QSv6J<28e=TGE*a40Wad7Tw8I!{jeSy|vA+3W@0^9mU2|CH7TwG{)uT#8f6X{+ zjFHV4T08*y_<&p;G@Q@4EM}2yOl9({N`ulRm-|}4SBU53w}O~z>n5t!A_T1mj?km1L@lU z)3*V@j`}A2TuJ@EzrU&eVfyq)OUkex(x+!8wbXyinzNNYy>Wm1?0pj)SoyF>`0eZN zp#6{1r#Eg`432-cE`LN_y{yaoDYFYW-A}t%hiAEHM{h}g_@z|zVbjouP3PP_gY)-c z=p$sJkI-NGu&cf74ZOg8qbH-s;I!TnLN3c4@+R4MUvPm{ScE?rc`Wn(nLHL*T@v!x z+NeC1u;j7I^XL4CJQg=9k445a$J9rAb&k}geGu|krH{&E_cSrio*Shjyt?n6#qwKo z=F%S@a@x4{MWK#S{X??D)tQU@4p}X-h245LK=>#Gyiq z?jOZn?Ro6$ktwusK2Q3Oz1ly;BKyKGnQWN7R8%Hg16?uv--`e5WU_)$jr-Zgn)CNL z!C6RXHT~(L&&$E zs!8kV93Si7;_}T{@40--de0LR+iI}cTGNj&mM><@WZ8D-o=?Xx%Wli9o+LSFrYkdF zy7C>vBp=D>maLOs9FK^D}+hp~Odx|nrXp6$R z$;fcow(aMP+*`HN$Z%J12A+q%ls9L6Ri64hzOK<%U@J7fTXuSD6J&c~AnT4RT!k%$ za@H*F##X+T(vTrObpZ5FZ}?SQvbp~DB@3FTy9SIxulvSDOLCjT2V8qfmR#1{g?#8d zS6p3qr+d$oC71Qc7V!CmJ8L|_^9i?)oEqxy^3`?hbO-wr?zv6!r(`2fl8u+Gf5N_~ z#~rlzmq%Z`Ie$eIXdU#r8ZuJQ8EiY++Z1;(xPU(XJm%?UXu0oV%YGI*@Za#9$}5@V z$&nuaCff90wBtSM?m>p7Iy*d!H}$o1S6MPFW1n*rx~UJKtBX8KW2~{(xNH1dS&KE_ zAC6`Y`e9c9*;OawJ{>uEn6o0@nz5_E!D4VQAN}KfKI}HJck8^;*am0{!{1K7e$khv z4FLb0Ga_l>UB5K#K;B+}rpSJ(kNtfZzVu7e@&;rzb7yQ!DlI(cm!?g&Dp$~U!S$BT z-m#YeL!H;5gO40O3~#A?`S4xQe(r$JQa=wf-hJsCGRM`{cRzcBZ{72*HG;*n;T&iB zb8`FSm#pUiZ(SjST%B|K`N8jgMcxhjwO*xghJO>ZeJ(WqMri#F(ERJMYjjH;PpUaUQIx@v7&iu~SUhyW!;1SOyFX!rg&ki}u zSu~*>`e)yXPOEe*#V0zp9mW#gZDnQ$Z=^r1$ox(CH`&2&66UO|$)dcd4xRb;-JIan zXZfcxt@%^UxokOO-%C2nvW7Kt^|TH1XN-}QWje2E6f67zjkyPzq&9|Y{ITE44(8vpXRThvGHx7F&& z32u*^J*JrUs=sHK!A0cgvdQ-o@KAHVZKWsJ2p^lrU4uq+bUV;5ZbV0{ z5%+bD2F^QU2+4efmBul1}wT z`+ga7oyK>FIX<)Ru1McsCOvfTi+xWx@3X!~baU<`t>nU5TMlD0=I17y82*`Y)54pK zn-Sh%++pD#7&j^W4aTeQ*cy0AJ2$w2d)HI>Cs-4%mVMZ(bC-{WtB;WP4*s2~_jw2$ zjJ|!Mv#d7ph!SkIlF z=s3(E&)Mo`O*S}<4cW?eaCvf?Rf*hxPU%Sp$IEf^nEO6D8(X7vx4!bras1@7pU3eY z!hLc4uoFH_ht)QXMYQc@tbx(CXVUg#%wcUzziZ%QxB6EUn!9bhqXVzjui%!v!0~g_ zut~+oIrMSlsmH6Cr`{UrybfS}8pt{|i18lGxDR1Z!aco`8197225n4!`F($`FJ&J4 z!j>~96ATU2*$8y+t?p-CNqO9xuYv9^yf3XDKD`^-JLe1eesx)}J@V%5FX*T8o?3Pd zb$^34Tuoc9qD^0C&72vcjln-s?#7x+Id4hp*;JE`E+4S?)A7O0a(M_GufgbzI z?3*^!xb{dsp?k2!jEmMx?yGmV(I&5pd(&20xAmjq@t_og5>avgb(3z?;^>h4^FjH58c1l+>fQ-yP=^5zXl7Zs3$w$+zsiXJn<4wlmAV@ zGyg@uG?vm2tHK7S#<|=*EFM|&HEF)TW$4`XKGLcko5)vQKH9$6Vz{_axy2{Pi2IVKY8}e@*iYtjqEwR&2<MwZ7{5kj{SyftT1+CvTC-SNhImOnPoimSVbH^P5N`#*LMtV{63 z$LH`S@Q64o;p=k?dajzA({uIQ5j__TvJ$>A_oAMP1nJzQ)HTq)BKXfNbTL}t`vq63 zBaZU&{oeX`k2ik$+)+I`7vTMqI>F=u6+PF&w3+nmw4`HlL|++^TtK0?69WZtjf4PGZb^RW`Xehy>&O~&|$$CtnG zzU2BY#&|OG;}soeTt`I4bqaMQf7gMjL6m73KE?{a1`OPDvbRw9<>fv0(q&_N_MNRS z_f9(fzV)vneK|6FO6cfF)>q(3_qY$ty@GvKitM|+HOIgfFur?i@Ojc7VZU>XbZvd; zPde*}*5(}2HZsQ6v*-(OU*|-P6=F^1jRVeQ%f4oG&L5zRQsK;o+~Aw|QDA~ENeOQR zHlzn2JUn*VpBT# z7d=hMZ%e-VExMzuwMXE0b!H}>;^{~kPesZMb;Fkl`k#$$Z?j&m85g{PJm!uNyoT46kzeYy#u+*T z9X$LeXG|R$I5K!sgwEBGM`LNmr*bNEAw4XYtCww#EzbZV=39lsl zw!)V=;f;jfQuu35cn9G(6fSnc`w72BSaoR+ptfI2+r|Gq%J_*kuEw?5vy0z91^g?` z%t#v1KaGLr`)K6f6#j|!EG2x>{NsK4$iJ`gPqh9n!g;*skO|%hc19B~oW0PvY2i`0 zX8gZhWDmrzI*QHOkQcm#cCH*O9o6*k4fvyLN^h!VAK0Y(XzAg*mWjiLSZwPncG)CWn7v z+@$a}<0gimH*Okog0tH!g-1?R zeVaia(d)Wx8hl?Ve4i(H9b*vX^)4pw=Xkvuc)bO#J;EpPdav-mu|~XJ%2|26)CjL9 z+aAEc?i6U7+R>r8-pAd*KVi28Iot@~qj-H@kn!O^Zw&!I!uhKy|0uW>p?RydCJ5J> zBk*%2{{=q|-_+&lF65rb>MJtQx6s=Iwa7OlXD)|tt682||1x9kcBR+- zl<^kM$QJ9ioesa{GyP~BZV zm!FbXF#lcB$z6;q`!Mp0$CEACM(mfVKO>;OefuNXhIpO7E_TLmFJ%RgX=pw+(WmG< zyh9tGr!P*sx#wrv&H7Jz-Ff(lCkehc)!6G-o^LSM2Q=64J8s&M>Cl$l;%}Cb=VVsE zZe|Q3{Eg|GLz~jVos?buoWt8_Ogi9i*jtD$?asM=oS{oIDFYh4Tl~#hwSl_!Q`en> zDcUMHdD-~W!Y@4F^zA34Q@@=5Dd8*wXDQ)FhkET9;Jx(X1hb!;v+S|K`Opx}*?8*I{G5ERB&0FDU2UNKi%eTG!{x?J z58sOW3B1%lz2i{QZ;$N9oJ5{@3_7SiEN|Bt`gj8VO6#Kf65XRew;0(d?*m94{3B#n z%lK~KyNnN;+=a(@JO01<3e*mrhXDU$m>1#LbmFuR)LMK|ByI|E+W$!oN;zU6W~nfXVSFW;*(gC-+$92U+bf0o7G0a~ki z(@Z$I`v5Zag(fX$0E2mN?0(58Gr~j6|CH{dz}Ze{h1w8Lob24nJ{z=VPT%=iue&mZ z=LhFU_UCVIab)Eu_7;cQXjhEs3$QGgGA+4`$)}_fAM_5ccu$R|=x1+9zkjOR*x^ac zKf-$_y>3^fi#=Y0$6dJ{Tk2Y$EIVMY5WHLCg8zm-t;YR*>JXgE7H?|;a<^r?Z7A8# z*LmM$=MN^trl;fde-4wU#PRc&L`tSUFcSKylB1D`!e^dxi8)PD~E>tnexPU zTO+L|;q08te^%6TRS0VhncrD9`W`<(7GmRPbqFnz{Wg{i^MCC3x@eKidZ{(cl?UnZJT} zme@ynL#`ll{g3_AT*0c?z$e+R+I6$Z>&*tnDEk;=eQ2($Gy!<)RJ?q_@<>Cc-vngrZ>>LzvoryU~)gxXdk1Bx!0LsRTg_U_+HUWO$UO3h6FwXP@1=jP{e$LOwA?eufz=cPGa2DE%)42S z_@{M3FxqQiVOaQ)KL50L5DW`Wj}hPD(f;v^bL*bs@VtUU!R`B`)!1(`w1zsEU;X<} z>KC3LXFjK4CmB7RJziJqU%d&ff6Y5@*Sf~p<3nS~zYRFu^|ip`KIW`5H#T%+47P58 z*#qFK`ZEW5bCPiqT939L2W6%HdeN7Cf8``RVCWarOf9>Co;OL))qnY8GfC*){u$-XpuFB1Yftnokq&r(5O1fyH`4O=qn!6RU*fLC z8r4raa~9iM61%4}cM!Jf2DA4V!X6}ny~t4ZB#G>^lF-eG4T_&Q#u?>N$~z2CflS=+ z6erkoorI@|?g^qi#eY;_-+;4u*)m@O4|x;c<$RGdvcf;WSNwwSdD?atZewboB@egb zZytXteRAeBGn@`yYrVUeIESYg7EX%9jZ)tYPmvKG9Em$$ak&=rmmZFb#O0X0p3vR! z7~(JTNK# zORieJ_XXZm7LRev#5?J8fd##fu6tH57@j3P{869(XYw0{CO5l-E4&Ur`21$w!GPun z4ryNpVeuXB%BB79$OX6e8#g7q3s?Mx@(6BEEt|vK&1Mc~F_+gcr{7>dc{Q^9tFVz0 zYv#Pl?XE<3)`8g#j$V<`hm`yaz0-!TS(ROD8#20~`xOPT!S`_={%tJ!+}uS=T$QoP zRkx1!A{KF{ti`Mm+hT%N!m4#YT<5_2Z)lVFiR^gx(zA#9J3OiRz9IB!WTL-wbBVue zWRl(HmPu*O)tjcD6)^M6=)K%f6w zA4J=}Z~kS3WBdM#(6;ZIf9Zz*iz_yGA9@~?b3>+Pv2d*KnP#vKk)+BZl45AICKB-vGse8ktu%I}gqNT-~d);hRL0^~U-B zovxs>{-lMQfMwOA^X)vwzx{{k-oT$0m-b`4sh>I1+^en$%u5yYSn)@ItwXGZg~W-D zU4mOm``e6sxIZ)~1se3_`hj`ag_8ZmlhC1K5jtc;hjgbwc4=fM`2_X808IhMKX=F5 zj73Bj>POhC!j5+Y7HG3~1d**-z#%3jCWb$vQ>zX`o@^nH$B?Y!IAITty!?B}m1_Kr z=oiPGD%x9`Tw0gOd7a+7@e=pJq5iz{Y_%Mn!CRfoTN|`M<8bba4vlCa&XGr_g|i}Y z>xhf;4;hiTCx}y@Pa?BAI@_}ED?pErFPl&2h_co5GW^TWJlaPjhmX=9=~IgC=>Dbd zc^;nuon~x)5ZSL)Q>J);pXEAvob-|lWui}#5xzf??jF*KkNp|^o`Yj4thwBsWZVvv zIAz`$Nm~(VXVWOFX)RwF-(hH3+j#6pL9eYLdBIUXcKTixDc8hr%?q9viNBF}(Nz=w z4Sz5z5@lSlKRe66|Q&CYK&b8oq&Jl87xEldXJZ*uafBCm9R6DS2dxLksiPHMb-m~Di7$APdZZ`%f+wYCV~J!&4Ba_%!fK-{y$ zX^h%NSF^`<#;0w3^`xVBv1eh74wPsQmJuF#dYm%MIElw`#wjDbmcGdUtw^7?(GO?+ z$Owl_pV(u7SE~R2^7iiWQB_y}|CyOUCWL$L7fmoK31}&TA_dEYpaf{80*ZnyXlRY1 z^+DUJASM_d4dR6Xub|};S~CYx(4a(Z0ijQ^Jgpa6Yg=2BVB48Q#gK3r2~kg= z(CYL0{qg%_Ui-}2`|PtXYp>g0du^vr=p$~Q)|+;cKV9n%+Y)VhFa~1|V*R;41{*=S z45vT$0(1H!`*NDC{w!9x5j(s4(r@P_U+fR^2e{?TyW`(g(bb<6yUgv+5Vt=|=!4d@ zk^fD9klk3z`o7a2bX2!Lw{*3MGAp0a7y3kufEdHzDk*w-b(3fFQu#{>>vC`PgKf!;OwOBb73BnV&l_ySXU>N zf{Q1VLYtyWk$FBO9fq8vZ|c0_jr>10=XGRs53)LUR4gJc<8IwS;pkS{&(eOl;?+91 zgPm!6V$f6R=m{*Lt~cg6E^`TfN86kpYoIE}gc0 z662Tjr%R*$zg$WjwxpJw-qPdijg5l3XZV9p(k8~ZPJMOwljhs;cU*`qOnTTKD06aA z32h!v+l$dJ_S8OKD;@TW@AP0UetUzXzwP9`05O=61x>uG&ML`!8OVE?=qFj!lTCd& z)XQ0rMQ5FAgx&~eDYIM1ow=jlAmphKUBfAn~WA@mrUj(AlS!1%#$)14C zYPD<#5$tnw7?TaOd+6ed{X>wkD&(Jr56*FXV9LGOW)E$WZ|AI{0Xt_UpYvw+4sSMo zGimm=3sz^B>%Q&ZXXg-WWq#4t$FlwBq~%oLPv6X5*Nm;z*~{*^XI1q*i&wQEBMBZU zu%d-qt5?f^-n%$Sw$f8#NHriU_`7YToVRssrB$rmvX9OWTAVNRT-P(W_#JEsM>CZF~~;d zT6G=B9OwsV(*+Zp6({$f8%r(3oO8k2)r#!*7*h_YJJ_FHTt#Mz1KOoYe;&T(x}ct5qq;DUXhA;w(i? zf%t!}T`HVqPSjV~x?iKe5$M zPNvnugYbOtA3TWE^a+LU;CX7G{%FCOMPY2{^6}GI5x?gNY}uar4&v*GPM5fIzMA$s zIzhJmFwbYvF7;~-X9B-|%1- zdjS3$N3YSGs_X&ANpm`nvT9c&_z{Jsh^`!D41`a@5uJ-xJ+XI1gPn4@_B`H~Y+lZo zXkGbPSGCM}3!2BO5bNcXGLtk?3tLfvlhdRX<-u4d~5%j||brwc` zs>Pg@?1DE=Fnh%fpLNby3crG;y*H10$^4qQ>-KVU~yZI}Uy; zR#6J=@UsR?{;u@kxFm~q^Bd%ezR8(T#f~!P_`#{(oE0UeShQjpzjydi*L+V>@WBDj z{UiDgcUUgZBTkskCW}`1`KIV(FUm@`i15u`)PGp^37`B>dP08_p~HUYF?$zNd%>%c z;oIJz?rJ&=ua?{zknXi}Y;eyn9l7tro`D7PWfRnxFBuX{dWSP(*bx}}(Y5%rneyk2 z4Q{9WI{2^3FVDAT+>vj&zRy$m)?~gpiEo$EhKcZi%R;ot(0@~}Yugz2BWNXOHZ9se zomhewy72_xV%+okTdg&QCZ*Z?^uP3?mhrT!5WlJzuWHa9Vro)wiR-Vjf;P#g@kLV2 z88m!F{C*enAsfYtB4~Fn@ICvSA#1MQ#CpTWp>&f~ADF6`M(BXk&3^8-IpAg;bp`l8 z2ie$Xz#f#~e$qQwlX;ck7H9ZQ@a?$=$475-Ve6r>!Y?Z_K6(pn&$AL*eZW*k za-jZKaUZ-N`{as?%brB} z@retTDup}Ztd|)-~ZHj!z>|>rqL*FG8E%@DsI#Z{8YMsMO zH~O>se{#`(rVE9{!c-l*sN*2?3%_YIzkf1yr`!A8Z`7MK#s0Tw+-}jhBo=u9&7 zrZ;p4Kf6VJxv$8_9p}!uv}4dZo!1ji#?zksA9B_M**k}CrT4Vzj`kpKG-s0Q`K{)+ zo^LsNeTH4bcncq21tz+pvHH0S+XhT?_D5*pHh$Mu)FXRmnEX6{Zi`Pet77|_Ffvw- zk-gLHCwMo{%4xaYk-fXujE-{w8Eq*_RmNqdpr5;^(^}8 zJ95L^a~1Jr4Uvz2cyN!HI=}c`VlaV!$;tp*6e8055??DHEZ^Alb>#X z;O6UH>Fin;6Uj6C3@4pE*nDt1THr(fUplsHT~=9}n+~uhtF$J$`-?GEI)%20MvpES z5gfyOh+gW>p(XH=!UGec6+A=ZXQ=Go-FL?Dj`|n*wbQ=<|E0sL%t)S1KeuVUXWN0U zzGAC#`#RL^Ysl@(DAQNjot(a=xP4vU)mO(RrM~tiuiD*fKB_*-K1DwpxJyKGY1%I|7foVe*6Es{BsrG?er;jLprn}1KN-YZODQ) zWJ4Q>->|zUb_-{3_%QYh;z=33U37xr3g`gl%+4vX9pdT6zU=G#R9Mp*zOJF}H0l-)Il?_Eu{v97WD_iQ?z{P8{ZCIjvYfh;LF|(@NR>_!JgpYK;BE? z{Z#5nqrP--FasRyfxmJhxaORp?f?gMri2(h$mEWHLOjm>v<#awXE(KXx^@*fN6dzD z=EC4yqP6V^{S{y0a9o|{>L_z4I^CC4yy+=$MmC>T7w$Lk9-%bOx1I@(s9x>IlD_*F z>|Tc}$n#ls;bGV)sk8bhcBR3@z+IfP=C91LzVR>WkZ*zPGdj=2dU#j=wNK|)`Tf37 zD&_w|J^J=vP5IQ8_JTlk!)bi4L2~@agvwz&i?h7b%WM_`{x|quBEd50u|siFLA?e8py#uSgi*N!CsD7`hC;buap@b%(;c zdmZb11^)!r>u}dDT21|uZDb325I)gVCEm?Gg=;&w16YlGoPlW!B)?q`Z1uUC%WPZn zpLqIk;!?-fu8*~A=J7kd&D{YHvj#MWbj;uv@!(P$*b}OhjFXG4jC*~@lkbD}y^&@; zx;Amqq4kVih0kg!Ctd5i>(KSdCCA$aOGihpNXLHc$4*M@3ig+0V2`U$!uP+ZxL+{; zFXhMl=nzTNDZPFWvA~BXJ7f7>`ns;4Ro7Nv)hkcsmhs(=vfkY3TfJIwh8fSdiTHey zy~vZL>}`ax^*Q667&^+>vW|-yrdlU8#_~%&%(;$Zw`LX@|HT`~XDk}nr}>NP^Y_iJ z@y)eI#*eS-+n4txm~*+a!2vY@SnGzE*!F{nWpEaC z-a}hd_9qvdH(K;c>rnXewRp11s_$HDe2iWUmn0q2+Kgyj4(=6-3{DP3@OK@;w(RWDKx_HEmDy?dtC}`&&s9T7 z?+(_|3hOz}>(JJD_c^pENZZXGj8|iV{OXSHa^TLI&#)iq8e7)9i<1kx##!PPPBC{YBIhD!4cl;;v{jSJ)KR`5qq|I<4&w)ApmxoA6BS_r$mVdul(i zuD5o#f1um$6O89!@VORzF70Psgs%MVeeM{CQ?S{?Lkm6ree!iy9qOO@rumD_O~n`b zMmcwGByZ|HuiLk$>6_|}`Tj`!vi-G#FE+klx+8lMcr0Hq z_50L0OS8YHIb&T4XZPU4vPgH2BQtc(-Hg7$Hu5zmE9k$`N0<@|{UZZ?67Azej&rzf~xvqQeZLqBl%DEC(#~_<7l2G;~fPzt9%*QT$oE2*GKW*Q!X9a!+To8Dl7k4pT{lh z&ii?k-66Tas+yJr&Yy)$pt?jG{!U(bh1T_@`;%DTMXc{H%=ee#Ircl&>tQqB&iYP@ zpYPuNxcl8%`+wfjy~eW`C$+`Q5qu|&|H6wDlcw7PNXK9+lMnP)`9B|beGaAH#^N(| zpj*i2(6xhg#XR}JTE}^5N6D z57n=6|4;tFF<)TLj)Y!a$G8~!mSMk6J(A7$1CtJb!Mv$yLUV5doF1? zI)FcI_VOfjfWpOpGV$zW6H?jb;EK{}&dSQ}U9dRK=m@pgyxZ8T(K(K8Jwa!(wWlf@ zwa>oRwS#ZryX*Q`b*wY_Z-3^8-)o=sN$M@-=kYwbI>zgB<9NN5Vm-QJe4w?usxte; z%)UjrF@7)Ebet=jPW6YPoG)&ei9CutAr-_X3xl2ipK`&$Q2R{h99uE*VI%sVZ4d z@~<^Axhp%&2d1?j zLjy>sO82koKGRT09bs%J9VZy$_;UaL`oW?%3y??G$En#1eA^ybh=%1G&$#%-03xxZGcb7_V5el zQ?%uEU}g^r{<6-6y#h?M6zBEmHD2Pq@++0y1G_C(p`i(K(B9&fOn@ z{$*(&fVv(dUwKDZ$6n~I|8%|N=KjH5u1(|`wTm%dp%mWxuu^!$Pe|cy?5}*aJ*2#1 zzoB!D_%By`(^_h{BO%J#SX7a+Ms&N7|83~PU9=5esdy7HKe}_U8EY)=;(LMym{f z`!?g-nB?n~T!gO;=Q9QtCH*ctJlG#Vp;+NoCv{E`x5ju|_8&WUqNmqH6trp(R#SSl;vLw^Nr#EHy7?d6S9qHt| zvVtFaJ*|@{YyST}Gl&e3YVdRE(4dF0Sl82;``3T&;6y`nXPuE_HTQE)oqI#)Pn-X0 z##TI7IKY{;kk0xZ1qVvwc{I4tZg3&hp5)@fI=&}7=l~BDld*>RIUN@s2Ik1i-FR>p zFyTQWsl#u(=l?e4G5=P49{8>KpRYXT#|tl&+?xlUh5L%mj^7_H^M|Cp3cE8D4KIE;~`%o zt;>AYpXNa0-B^%P6up%-H`~QQ$w_lun`oIk?lrwbnuDgYTxadw(LLU>w}1SKL%&K+ z8}rK;ldds$?V`hhoj&Hd8au|^2G36)b0fbq<~8wo?wacx8*|ICA-uu=yht$*;| z)sfFH!{&b4*rRJ`jh#OBi{i%qApPlDVxiaUtWY^P@4FSp73v1Mmap;iu@Ahru1KYt5^CE#ChLdsDrwI-^6U1y&wq z4R=Dno6t8a1_ll_l5WM8rTGj~m$8|G_d08-^9*g!(*NsRw$X;R`1zz?3*C8qhq@ga znrr`waX)W05#+mnBOhilPL95rVdp?Qgx41Ub8JGt#lIWd3F_kJFjYY0T>t?45lt#5|k5vr?lU&sv^_ zd|i9bduBYAAUBpPeiS-!%{}QxC)Pepm%M0rymY@TbIFVLMMYw(~Htv)b_KaqZ@BpKdolasazI@fMKvh`XE= zOpDW{&+lhF`XRb>Eq3#4M~~iO8M}Gkj=7Qzjom!ke!S9Y=Wg059d{@6sG-dhd}fma z{}phJ`&_}%{pY&ATCV^X4Lgx6c?bU!)~)0ot;3kS_6uO5HL*A&QRM5G&zh0nDeKL5 zcbj9J5nSVS;)deT=`y*&)sIJ*v*TAHlM5dY zgQGE>TknZh34RrIDNnlmbW?AZeT7nRd#d@LW`EbDsrCrc$PvsdTkB#l zFo%9**~u>KER`Fka+$W)ugaaFa#abTKSDdSA78B$euS=G_b4<=b^0jVo+CXowWSIg z(r(WGXW1utx6AIvxH)G(I8WrlH+&f_nR{W^KK$!zoVyUk{|@mi9{HJ4_{FD6!IwXh z%D$>`(A=L~^aIBH`;7gy%)xBtVivUJnvm{HR6JzPZL~Dg7GD;7z38F4k&f3d2{dTw9Zz-oXUzma4GQY#b>bekFsCK1et9zWhS^SE9kFVYjeYyRA zlE8WAzBcX4rS-MU{Kd+vTzq~X@;jkFy(n9{Uy16Ch=Wh7bofjFu zifrhd@b++3ASC}T^UT=GUa4o*n@C-v$32uXRzCAT&GwM?;r)(VUu9i-Sf5_jDKX8a zCO{*5Aj^4zoj(6NlBp!q{WB#wbihZvacHE@3)F*4rY#p=9DKUk@gX$-Ke#yfB>%;$ z%>SjMgE%!lA%1H9pZBfcbE_Tv`H)zK^Kyz88oQ3pyZEr<;JenIQk}gQf6`b?mWyYx zFOh1Ee0fZ9Xa;c$!Z*O15?QA`t-8PRd!Aom73T?;cv?%5%L=g@_=#O$Xn65(?osC+ zmf?Zzz0L@XP8t*#ZR%cf(`fcO%(<>U+{v`gGw!`U*3y-q5I1^9Qedl(I0onYAD-6d z;mx-_=;<^0q0J9Y>$5R*+kBpvdybJNL zjC)hV(BIB81DIQ^8LK?(pYYr3O|kt?UANMYzg5GSUZI_`ft8*aoWdA=Iurj?;y7G~ zAK^G`9X`$g;zzV#7j41zv_Rvu)`$HIyy!*SC!f!GY53H9`WWyKcaMJFOh04op5y5= z{{h-{t%Y2QwdotgQ!_ z50&fu`P@TGc;=jA-BtX|IKUUfJ*;gX@N4JpvWZUJ)55d4TdoYg#@U_%`rb0fi6Ool z9u@ig()~q)W-cH5$=&f{aGE_5*j^Vlez4PSoryh@b{T*73afSQatB{CJywc% zGi!txd6%+oCa``=SV!YoPsQ-L|EU#^Q(q$Vg?;#zGR`SAKRbnck$ScGc#id@h`vOo zTG65C0m|om&%PFP!wPhiy$2~bH4uICo}@LAiJTdtoa-l zjkZiZR!icj2IkDM`=_8q$Uxov8{5?WDcI<_ zk9-QTtXw;Cqsq}AhkuK9y743?Lf73g@2d=LZlLd1yL@QBVDwFGnC`;W4L!1cuqg6l`=M^O5q`XrSOZr->~Zb z2+gX#YGSmU=Kw$1zt_T7@+#4H&j;_|9c#d`Jj>ha<9{V(pWy$x^QZ$n^SkFNN3ymb z^>U{4ilQ~({M%&>X9lfC+PA)E(3`Ms+P~kxS9d6MFzy|dP5pkr$mkEz?)K?cbmaN0 zL#@#g;_FTH)JKL+3N=5Q8j8GN5yy?QjIxovWSwkV?x~MHLOyasWFL0Pg6txl2W@uG zB{nRwP9DjjOde%cp|c*Z*k8n5I`Ua+uIU$Qu1OC?p5k$+z=D#=*n2{g!4s&@nZN5P7EuVxf zr1ISf{QrjjgDaA8jqISZ!yLInviZo)OZS&!gR83BgnF(i17wdHKJMRJSk2ba{Ae)=}+=b89#gHQ9w^OLvN{jTP*R&he|6yw45 zrHT9AV!qd&ILXkwR(ze_0B^>BGY{DUJM8j6{Z7VMb|OEt+ux&lRv4WA2pWD2nsUlI zJfL+b-|ifH6SgJE(Vb7y#yRopko$q4YX|7Z2Di)+%1HLp_kI>~-fPu+WzITW;KDu@ z44hD(Z+2n-1|}ZCdDrN1?6>8d51Yj=4BhmTDkjY=_UiNbi5?g^!n(xRo zF}EN2&_v;T8~ASKNI3qibyDLOzC8{5LVD=v?cnkqoTuGk8K2n*I#PIWW=J?6F31YC z)sX)z?}HzOvHkNuThXTZI!iTWbHn}IxtI8z%?%SPFKJ9p;(0Q#o|})1AeJO@HRpJ# zr=M914}-@V!{=$Aborgwtfkvu;ra{L0+%e!{wHws_%X=$W}Se?{EyiW9s_nUeK0uX z!i7JN0C(hL-evrR&kF@J@-Z;xNw_50TyrD&&)~Se#rQ5IPqMl2zXKfV%0qVCNZt+R zJ%>MZ<*|3Vf@}?|po=zmKzLQ2Xyuq3yBt;`&2vm;YM~cWH;A z&5VisC&n`_>!AbJOE#{)gfYml2gKoSCb;W&We>$g6mBRL{z&dss&Sjdxa!@BN*U8j zmD27K1JAaLmD2xlN*SX|lrnx}NPRj}#`+Mw9wj*I@j|7;0?`plq5s2`vWAB#rQY+E z76hUxq&mmV{>1`fSkKU$9l>rY-dn!4b0qB>cu00s>X%(MJ{4WaQ9bl2ODXg*gY+M= zXH@APiUeef9>^Fyku?&LIsC}#NyzG6>>17(Ufmgq?EfyfJB@SXcQr1u?rQKMZ%Lj` zyJ_?>#)f!>7v1O$jJ}b3<&r5|j!ns)xAfh~OSc?);OYH`Cf^f1bn`{rEt9|VP{m?T zD?T7I?$r5a{M{ybl72U2aql%rH_pGf1v-(`*?D29r{dkBA3pu=Bz(-1cK+|78}T>M z`EJdVzIUNJFX7+b_io|OUp^GB{N16##KuDrbR|Fao&Ihr@20Vj_jsZ8*?Ua|!7So^ z5|_~Uo0f7ucN=;GG+;({nl*6t?D{q3v$-3HdoOp%rkscjtvE59y(ljT+$CR|>-0RB z=Ru?9>pVyKP(5Efe?4}sxpS$0m3RBZM~OGU|GT-Vp~UXPvt?yx0xKvv0D5wV`#tg*s$|_zUBozg%~1q}kc5Gv$TIli$EP z^aAda80i~l4V11}O#SkWGj~X$ch9~XUP$@`-RPs;+(VIKA?tBJ3OrYO z+wmNYN49+j;W&KWM9bd+Jz7$368r=GP`p^n zZ*6-O`|V%RrC&l*y0PYU-YY7ClI+_cm7r>dz-eU}+vP{fGzn zd(Or_e}ACI3979jvA%0JN`E&1@f!w9Dol9dhM1) zf%>iU*|+x&7^1aXjh?rupg8n9VmwmUK!m!wKl9tf4}Gpq>!h{h%%X$Ip88K;=|{Xg z_Rw#|pLOtnP(A(n_>n;LyA>7t{l*9J_p9?fy-hs8$a2wxq!#==W<2NgpEX9@qpIFt zskf;=N1(f#pz&M4V`TGbk>uj~akSs>S(Y6}9tscrL6O>CNZF(w)w&yN0kYNVmYKv~ z@FurZan@+-V7~7i&{DwjdiX=0*VD3_n137KEqPvVOBv5Ktgk$;uf@mn6lBf$gJvzi z6`io93ft=9O>2_9zI~q}&vRdOOE}s&qopDS_wM@?nON{b;Lk&slf9mO`d`c4+R5Gl z=DC#T7TUg?x^5hV@3PS&d$m*_?VJ&UH)=mW1aFJnKnkymAhR_yZXsfKN07(V?@5b8 zYYGSLT^&aLYQ|4Xu}=Kh!OFNBJ%a3zM_eYKp39u)ns(@4_|v8`bZP1F7k)`z6*{zJ(giOKKO zlV%V3lCsjNMJo!vB(LOS^6m&kbIEHOHZc@y=VWa5&9sgCJ6a0RwZn`#>uRJ@6IV9F zipG4G`|xg;?=twPIG7rXvPkEQC)k&itdsB`%9MoNe(;{@hj90HbXmPedW3b+RX6>O z)m`;3-n-Q;NBZ%9D5pLZ{Ihe$jtcIouDF;VaV9G+3dQX1@^fqSCEL>X9(Q%`=EtCy z^l#&CZ1doM<2ZDpp^i@RkXxovWrkAbKwKH{^uQ?OR=3RiDnoxZ`Zxy?tEb;(p}XDu zx0R1hu(2#IeIAP<<_B(>7gdJ!w6V%9^T@@2zDs*H{?@*NYYOAg~rZ`gVf9?}H3jWG}PjoZP9G8zocVa9Hl~OlxaO)Je zsa7fdDOXB+3zhPX@CBswv6fVL+Vhu=&ay;pG`bpRQe4>Gf+SeE^X3o93Fm|)yV?!w9h181Zo9&lkl5Dd9QYwRB`>>9z4 zM})(FabYtAO9lqM{?Ua^6)chYW&HPIi;$jItrWUbsubMwD;+`pU{Y|bT4@gXN>j*J z+K+tf5Ar*%R+>mYX>(Oluy(D^gY*m@FZ0$Pggza-!COBR9cHfd&rzO}L*WrEV?5E; z7fM1c<2=z8)&d7)qb(CX(c`PUoE7udx3HHprPz1uLCStSow76e&E_}96Fr~0*0V2D z%$`g$cAA5`z4gs+dFwxVY3C(P{bp=#dZhp6rhZqFR$g)-WgySfdG5FQvkLaL@Ncb?-{qU1?6abu45rLmq@zejWwRIWVK3fO zKa_TTJTW)aI`NFqrw2Us%?Ca8gJ{dA=+e#bhi3HY@FMI(ZHz+&>2mQA@@@b=pWo3y zefU=D^7Nk{Ua}(Fe~HDN4GGi3-1%VSlRLj{^!aUfIylwzD06nd6%F*Y>Ke*=1YfUl z;2#KHWks!Q;EZ#HYX^J}xbgM5KM?(^=qz(0c`T+s|3)zO9b>Sw>F-X#n6nsc0x-$^ zO}nLg^Dbvd=K*`!)YlDL3T&%jw>z*Z{%_>>956@k#-7PK+yhK9dkry&RiEhFN5EE^ za#uSr)@K5;#Y3i?3uCRG4XoOfo9UEe-A({jWy-m*8_?~6-2+VPwl}Gxqw_6reJOd8 z*~L$L#OF;6L^qNr`CaSybLe{4d#s5)p4D;b@CB;*PFad`?L5w)=mQBHJiMs z;YV3^*v*&jBzfwGq_Vl0v0#mzP5G&mf5qirAGqbki@M6O1}9MNQp(jp^US+5={K-| zU_KWH{aOla3^37+H{;vJ`rSyLU*)t6^G*CI^PN3J=@;QS#n@u62oZfy(z_eN>pUO_c-Fv7u<~SZ0j{qMxr~Txf5HQu=sYDdTcXDPwm;Df7^- zbQt^pN}2D^l!6nVDg|#oRtj!4D+T|;N|~2`DZPQQQ(6X}SIQhVC>;}s{zED2-~**Y zS-(nIKkqA@ABg^%GzFW9eA~6B^0oI!?e1yrk&3_fK|UVRm9&4UyQHM=t=paXdc3}L z=5+QMds`#93vJ*B*b-mkx0>Ib{I2A;i7`kf4GkS1D)l5eejR#+`G0)PKAL1JhJ`lgth3UI7f0UbDSe0OT@KEsP1{ZdZV1_4W+Sj|V zyHp=CTpPN*{Dsi<%)J!lq-~7Vbm{#3us=oBj%#@@uYaJ`*>5AJAOFQ`b?&Rl?7`vd z#@g5TD;5Rmv|Lirq3^0JaEqGUxd*7=w>Jw1 zW3kkY+zGvL_PWyT1I*>$U7PQ4>XY8;$dj=$8Mb^@6;pA)TW&DrWWzD?WUQ`4r_Ar% zGG|gob|m%Zy)^t8`8~!jjNKr~x_4R&;}{y2Uo?dEA3)|zAREBfw^*-_3vOTH@a|7m zvPVH5Dp^B^=Met}nYZC~Z}2zko&0~2uRb<#Uas~4_kt%ou`2Gvw@mo^ck(r!Iujw? zV1qB=9m&lx9{<0}>EyvV9HrEDI)^+5}*d&1tI-g@Q1JBP*Rfm3_PTW#`k?85lGJcswMGI`ndp!hs+ApySnpvlX$ zvrox$WP$$%wqd{H$0T`U0_9hj^4z^#>F}13@FA@c`RYfKoioc}&Txo!_>Gm8_uk72t|680U-Qgozt}~XR<+{U1_CM(O zmYCh^xhvQog^qJK;WT_Q>ptN3D!-@r{gmH!=KNyPp?tfV{fa8+W}c_CwS+W8td?$d`ZN@xcy#?dQVJ1upsZXo27vc9ILv2d;h`8)ZeeV~3MH{b}s4ck!FWFJHD; z(Pid1fpeaiahW~QXlmCT~hNd{rI}rN$O9S`S0F?jL}KfHGAC; zxO(S5s8een8Q@*?wfa(Q$Bum=Rvw=**1r(i!l{&?5D|t_L_gu;fz6rSKy~Vj{;jM-J{9f8AnfEomtGtF(=9Te# zhO)ml^JdjM^(dYtaf_)-=lLzp37#I;^iAq4@(c+UaxT*Pp}w7ApHCfTz2CCy5~Uva zg7z%^jIrWt7#Ms(dz+>lyulA&@Q~MVzkjno-`NYDL>r6P6D`u2;=w7Q&Io%Hi`*MIsJV50xp1CjjU>}9ei>7Hqme9+V;`kiW5A!nb> zn9Ky`>~HG)p>y}*cwoZ!w;3ykE|Uiy&ZGPo<$?Qe#^L2k>8Q-&kCAqLzY}sG&-^`27iVP;EHFYY!li|e|0>cj>3|-gVQ3rO* z=P=d2lQ};DtHM$mhE=RSokxz3)wixBOY(dd z9~kmIC(eEv-+mc8!jt@>BiO%UOvdmWBCdyMz4Bj0=8*lToK*kizwP0kv3g{I!cJlw z^u~Xic=3|s^XQ|>h{qUN9{D^94SqsnVC3c;`zdf;YkdnaBh!F`kGil;z_82J>mC?C zw)9uo-w=I$(9N$QUpzuIHJ`luT=+ApAAX`3Yx;hGRONz#!*d?dd*sVk4cYl!^FA@$ zI)NRAucRT1Q{LDHYn6hl<)n|fzT-dPJBOi-I)fsgM%lm8WIv~^wX6-DiBUUmqFl^g z9E+O9seAtz~# zUgP#@rc&B@rBZm*bOTSZN1If!->v~qt_ELbf;U%zKUX5B&p=L}&OHM?ji0%Ac}be1 zQ~5~aeas{D@h0}0yL6~jSBFZ$?=YFY4rr|8M6I8{|H$)b665OUCp$O1ccigRyZKtb zhgl!m-_#uv7fp3=V3^tp9dvXBr!Mh>Sh@4ua_3Mk3~ol4(^AGV< z$xq*pynyb5>~H9g?3w5QrR*y!hQIU**+-l4D-YkmS?2xtmVX#KyVJ)xyEC{iF|YcW zvpe{$ej(JYrl>+`CoL! zS;1!TQ@*Q;RkmcDm9p`6XFT4aU+QB(db)6X6FKH%a*J9S2#dHu2Lhx8lZ;=P)i zH=&)5E&)w;`BNRR8t!5dUyjAXTPv8+FMu(Y6KMNtV4|&sqz-=~58Rzc-YQ^PPlc@e z)so#FFT!oM~aJ14`Q%slRh#m*uBGX5L=pH%v41F>6* z4J_RrN6KCHoEPfmSHzaEBk-4h!e2f@$2fqF0sXRxzgPdSJPVdl`7XYq@BWK=bT8@m zJ_oPj`B_|Dr}Mr`PcQp_;C*eb559DtBeYBR2G$=a4!y?O6VFrZgJS5l#<8}*s(%fC ze$keXpiBRx|2iwFGm&3ME6-#c;GZj`M^*c*QHm3Ec$nhz=}rvYU8y_WpgYXH?#j@f zlWb|yTm0N_c=_e%UGT?716RJvsl49P`>F?DvSHW&CHM25@W0f(EATA%6Y^|bBkQ7k zU|?k}^zSjA@8Wkgzij#!Zd<$3kDpP6*WbLJKKuEW&R3NpOUtIGy<)ZHqKBQkYAWGd z;?Yykd64Rdu^L67S`_BpZn^0MLGYR5g~E3ZI$A8lCZ{x>!=zW+1#{|3Da4UX|>GhR=6 zgT*e79z**yMs1uUUSQfEh~BD{cHW|txtMR#6#EL&&Csm9*fDF;bx&k!OBv~npDX6- z^yNc1-xZ;qv*5jNdQNL z{C3fTOzNDw*AVr6Q%-J z|1?&a(C2u2SOPT52TVLneu>8Zm56f@p<6(9`g19bNEMk7yoz=$+z*p(zx0sj(f$sfK6Rb!@tL!d7lHE*9Z9#PHyXTSj5XGY z?tj(0o_fT))+(JI2tDKeUqf0*9i3I4cmC74EYW|(L{hwkYxExSO?ra$4t9vQGkwlI zJmQxg^!K9=dLDfb|IPq(aBt6@qcb?`L;a)tXRtq!Onk#j0`-<>;6RnJ@a230THMGx z*i7m>!T-hr+56J%@r-=~--^g2!KCu~_WR_GCYGnZ zUFh36I{iOFIF;whm+uHVs^V@O=KT)CKh3?-F3(k5AP@8J@Z1I8iSb;) zV?0;o#dn=MtQ4Q)2y&m+F7zGVlxlb`e6Lk!mlNDrToXujZcyi%q5U&TJMY=g+SyjU zYA$hjtZhyA!&{-N;b}>IQfE~xz zwfY_h25+9B_tc)R8z;@>@xk8EulZJt=k5d_Mc@An%;C8?_G>QeU6n)cm0tR?3;Vrb zgMmSpwz{yrdbi3KnkD`UP4X*+hV4b~)}B_iQs_-K=Uv2WH}Rf$>29^XA{AJU{YwK& zb@(ga+u;5;{FQJ0-2MN?m+{xJJ%WRXDI=YyjJ9YzWILCx`=n_vK2u6*+oMXMlPi^i zUw4vjg;pF#7N|CH8m?X53%+_h1;3QVf9;CZ5Jvw!fL^QhRYN+1I{_o!w0&#w*(>C~ z&Quet>@=P$Jym*+VhC`)-`%6Rn{P@#67T8OFVTA`qjG1u5lL}^v15357QZ=>%7&gSqCw_(Zqa+>5W^FvBd+A z=`2P+>fkKKu5Vi@ZTOBJcX=xGi+hH->+HWAetH@0HT<+UZH3R~uW{f3;F1NL^UU4; zNZ$e;(-prB4>?^|ECMDzJJR4c;|Se6o97h4WBMYrdIGS%h6iQYPs6*r^eN_f9(jo- zFUNi?J`Y@2N}i=W@a^IFJaBg-c~M}(^ZVoTz^Ogt9RhZ`j&@SzneUj6ww}6;-;;aB zIXvGVYKNEZ{dO$QV$YE7lZ@BTt_9C!dl25djqg<#FFhu|n>2eZW2!SqI@hdkH7s{* ztI(uVc`0|AFCqT+7;szbLH;8Sok+9y^Nw);9i>wPp|{-szbCywd~>(r(+t24G!T1G z3U;AX>_fz+S;Tz{RXw;1C=s3IKi{~=Z!&%p)Jp>4oLmt%Jz*!)Dd-8nI$_ zCm$=~zxKP6v1L664nFu7EBH{SXTavgiPP8m`nUE)<~hhZ4DaY2`u&oJvY)5SAFlHT z5ANs_dV%MI9&d1MV&G8cZ>{=IJrA#*N_}gqtk%lm53fEp_>7|7`IXZ~41XxQ`KVR@ zli`nK9~trB>JVowiWW?%KMKvh(PLeh=kvEFRf6Lwla5DpzPoc=Yx5Ipv+?J+DRsc3 z+5P}`QVn=~b$gpte;qXJlOMbCzdJq+*y!ZD@GIA@mkM0GQ1-!0)^t;W_9(jj;C#TG z{S4Gb)+N5A(V&$~kS!XePsUm*j+8oZf zvqw6oz-lcg)qM@)(XGPhR`3;UG{n|9+4)3FzX}r<1s~0N=44AzasN~9Y-mn#YA3rl)7K9l;Oq)AF7&8g=c=R!ETi3K&UJ4CI@edty_xg*>`#?hC#6G2hy&NYzfb7c z{4<<5aEp+)z#-9p$*woKW4Vj(3fJ0Li{g2E7#n9iyL}E`0OpKmx{=j1?i$mlfS)%vWGs_J9wk)psmTl$-YI06o-8hu@9TsXEE?e$*0`mFdch`Xwn1p zjX5y7aG}PsU_em=u?fRdlR|CNcBsZ)e&%eIM1?O3Xfz* zodCR;_v#I8%(DOBzSC{1q#Sf?9_2<+?qyRB7~fGn&A>QMHYADo?wnV7r!a|g4P)pt zXG!O0(!hf1O|A^&LB0FTZy+JgHb_rMHdzVK4*AvU~dgft#E%!A6 z|LCdMzFM|@uEtt;K`I*i68(9ZIo(U#J=xFMTNxF<*49B=6x01e+7pYp_rWs!!zlYF z#oL3vb`oPwarCfzo49(&>np6vK0)|}ncsnY%j$~XKA`Bx|HrwNO-=pzg==%^pdHQ} zbN@Qy=&XnCIc{Mtob}K>$I=mJBll~5CF5#-r3=T_g2qzcneWcC;X{Teq}rP6|KXk3 zTu*o3@1ASEe7-tc1&%v|qK30)${3l%K zo=eKQb2*8!VW0Fd=MDyqXP3Q4F=+8`n)HPjw5QFbY(8I@%XIri`l`J%jic;7=fwB- zGp*kYyZy=7`gLsH7pi~IzY*#)ecvrvvqWcdeaJxx$U{Ali;#(@CL(KcE*BeI0wodxJzeNPRplmzDjY~IruP-L0|WqObM zA$dn~NYvm5{!o<$r^VM{&Uoh7Hv$(xNuDI%Qf~ly6{SsNB()=(C^Y$ulXa+h>A#vy3JM8*$(bVzj*`wMm9w05;2(eG!)0~ zFD}2IO!=5Dl;oDx|J}Ts$+u+BFLukY&Su1L!UtX%KAGRQOZVwXKh$>FOyuh}#`Kr? zKuXaaMkxgs&M>L$R9_u0)6s!aTguRX6yL6ZaaU}+1IWb>pa;!xW7^5~QYu*)+l!yn z?00-wJUiC;o371YZIIp>J6Gq#yvs&!a%?X<+;SaD)EIgF#Yjo7W0Jikn?!~=_gFoE&NSK8 znOcVR4kaU>-?Zd`?BsmnKE1$u_yb5UY8$b9^`vvwW*1?bm0v*;^3~Cq*+nCVS7#fX zF232E<8Q>aX8xyrr(XWxUzop<{fr*60vfFTEphN}+TEnkaFH#q@?I_}tli z^FhH(+z$9)q9fGr#3Eh!Fsx_uerrFb9XvB!;YS!Y+itCX>xLfY5Zfpk|!8nibZ8k_+w&V(l8OSXu+U-1iyVh1{g z9VmWJT`|CATR*`VgMV+w{E45u3_mhxtbw7sJNUFc*&L6TpZsf$_+H<)N=y$damFMdMPn3Ugv(Q+|N5lhK2^}-#pnlvX*sH zeX_G|5x-gdM)6DIcci1U&StJ{(!3slR`w%ijPoQcpiQLx&6$1H#DXQ9O<(JC&hW2I zu@-pv-#pBU>g-Bnlof5c!K(i=ZEqQc%*OgxnLeJUvpHv45C1V|goRpuQIbH2q5zA*T)F$DS-XG8VYLoE(5z}U3rkVZ(qRW-i#$`%r^ZiPh z-&;s;W?jrD)w?(HY|d*Fo2Q$f$d@M5mXD|8v!SjHyn=Tm|CzJetXb!*i{cFv1Jy0_ zGkpiUUc8^$f#l*)wwwQBQ->#{Z*PYm$u3u}6k1WJ^ni1YrP+)_Mo|*$BVYRj*lf(R z{FSR792=o+5$@11F!?K2Jv`Ptnq^}+L# zd^0H^A15pIYCY$}Bf-&Q1?*F(U$=8F+DqUrYa;xLRd3dO;Xrfl_2`Wb{+`YEC5Pqr zwdxvHd4r$3cW<8wT>Qk$dvEfXhn@I53FkA&!|rSJtQ=^$u-tA1OLQ3 z-Zy#K_K)K8ST}pfdk2`~OGjPcz(>IT*Obq&XT_IieaT1qHzqI5o*thE{>xYT*Cvm8 zzzy*y>8Dyx=#I&L?kMpSZ#T)eb5_xSowK;VB)gJ31h|91S%ZtuB@^DpVP&42M+GgruF zzvREpQ*Knsm~K$Yc&{Vv$2v54hCIKQJ%^KvcAz_2!4ngh+eN!-ce4LL{%3w-d?pd! zvlsC_hZ2`GnYg6mJjwHy5#KY3_?`jJq=hCf=%B#p7UFwO_9XWrj^q&b`A7TNclQ(D za~%B2n{?^IhQVvIwV(Qxv1Up*j{}d?z0_^){s`!08mEF-pk0X|62gR5(O zeOq?3NBRPIxYpONrHtn-&{OsYT6{cjWj@#X2DH?$f4Uz0TkG?*l<;i$na^W**k1gJ z9`Fq?&*kR1Kj$?|TYcuapLwn!{_}kJ#7*$IYF}Sq=mm^a(Zx6H8$Z{=4)^bRbQ5HC_A*|4kRxU$CiyW!mj7ELkx2^2B2fy0D&tO%yEM zZgycF!Ab;6vH$79I+g+(2W)mA+Tg;D3Woi(;f6r;Z!YYRV8{gxGdZv5!ae~ed{ErV z@E<}2evZh0%mHZ5^rSHF`ZgZxT_(AIT zZvKadfiuKMDKId_#_-#}QaSF|DuXAh-}UBy|JDPB7pL3X-2Y|fe~P_T^?)Z?kwwIf zMOJ}FtIwO!+_Z?*DuFw)!d_{Yx_ztba33tbe8SZJkp3`g5i9 z{b{9)#cHLD&67$QtH+hH4(=v(eftfMxs(6q8_=lElEkl!XQcC0J35KY(mBeFnU38B zSzB~J%KFgV8pE*h36HAuJ!CsaE^y1SepKdMx6DG7L5Fi>0=Eon!N+-`Gu$!@RHjQN zaLcfER3_CeQ?4>O)Z-&g3Nn#tQy$-9EhW2U=BNyEnRAZTDRUrsLdfUl&s0A0oxv^U zz8ZQWxuR4lZSpJSyL+K0vO`oW9ZbIHO9gjXvG$vp>(J0aU2?}d-++#o+_A1I=*Sqz z8-fi&)>v1yYV7)|N5@L$$VcWtc8i9dy&$x%>anqQRr+@MEQCrjwwpTeU0=65y+eAl zWRi$v5@dqI$Rs9zF>=oR>`kS3>f19sj$E)b#mEJ%H#l>0CVi9*Y_9w%QWAp4Ryy!Z z;L<^Lm!)LH*YH=<91a9-{9zf>ba>f@13t#LJD#Km*ht#bR3<++;I2$?;y&5*yI~V3 za|D>9tAa0#?>u0Qg2iG?GS*9hePGHd)&=Dl_l>}$6F7Vx7;~@(*t@`V-hVEsBTtYA zo_|E%KIO5FuZho_>c{|lOkSovEk19GBLn=(qW(zZ|5Gd2HzvyGR!Ds zm>M7ZR6aAO$FwfM39YG{!HdSkVn>DBd0?Mj))1xlIYTa_~ZwhQh4VRyvyZ-XN_?I1Vb#>iAsXJ`j@e`0NqO-qR6Uawf zV!Ha~@%U_T_YZcsX$zCAx@-B3;r9}4ZSz}o+i3gww6WHk*-}7U%N_U{9$-H@gb(sK zPkw6^&v_nyYYFK^@TS^i_B@f{_x9wOe#-Bfc$sz5r^DK(oi7!!?)@~O(2_7Qe#P_szvQ!)dFdN9Nc)M zOmgYUhDR~?pVQx1eMj&4+5U*PzsYZchyD9r_6ZrQ|K!^dDfk}qH( zr=t_SZM~lw+_b=j7|Knk) z=Qj^dJ0Jb(SDolq4cuk>LeQz>E%gzddvzAkSxIERWSSkGiVex*n&TXkxzm-kPrWa_oNQy{xYHmh#?0_q}Shq3TNL zF>QdZY$We?lb3GyaqCic7s<_N%<)F!rFs%(=Ncc(?Y=GVNzY*L=jM<2eW1X++mR z4}5$4+%)nz_XdtRFDtGT!c3R=L>;X#8Hu-7xSxT9YeDfclNu|)I1oJ<| zJ_DWbD|NHo$>v@T-OsDL){a3F6%Qd62UP1No+g$TvqwZ+L7iJx+|T3KGXIod)xFMd z4Zpki&EPkJ-?2N0GXTwc-)Gge{jE=EGyL@yp7rjW6zet*H0ovIfWC)b`Vz4}SCQJw z1NDl(S zrSK*0jAD8_eobQoE73VycM*frjQgoJrY(MKOgam&PJ0O0m~LL;*qGeC34EZOy@?@w zV~F-Aj&@$SlrrDzif8)v>Q7(!PAJdQr{&=8Kz$9mZJwuZOBt!+lS&7b&9#&9%012Cqn!RXmpJiA z%jJv6H?VKF_<45pN6U!Ik3bW|w`-6~Bv1B_%bUc0 zX$1ad@{vu#l+9o-QGS{|EA{Qr1+ z_xLEQYybb5nLsAtPVN^;Laby!iU^pZct~cVfG8+nt(P`Yuz=K3q-q5vfuM$>l@TqK z*aBiJ^H^Iv1|>xg(DXvx9@=$}LZTlkfYppJ$SQm-hEM-{1LTUhA2C zzpTCX+H0@1HtV8JcRau&;GNc@?lDz|@dZl11Mn=}eTKGlPxnk&ERn8>^RwHwlzRD= zH01*S+%0{9mDQ@ToXOgkEWo4z{^1kSsQt^J7q8S9e;fnb;4MGd&nR!1;bYYJ83m8- zMLwr}dpG$c=XcMkdP@>rnl#RrrSP&|nqMmoJf17N_R?gPIVV^Pvag=&rBfS>FHv(= zCEbuRM`s$Psq)e^C=F`>8LzVoI_sWUX&Y%(_kms$W7m3VV@f-LdL!bIyn4aW1K{VC zUYhkv!&)%@OA~20cN`djFMBV|8l{PlrnXx-td9fW_QhVB?p~!T>IA8GqTmGL1?lsykwE%gCrJGhfrpxcu3ybakLYCV8`?+t8;_Uz8=*rh zdqan7xF48@y+d<>@AQ@2C4X3qKg}UN=l1(D>;EkGcde{6UCY@+Dh zcnI2mXacf>$;c44l{L0Z1%hWSqlhad{1SfKxmy_TYkbe2HgDIm)w#hQ^0Voi9v@-k z3cIZx8Ah(~&P;cG1esHvp=u;o*q9XE{~b5nPk8f8{L3hOW|{LtPu6m(I>9&R+7`Ug ze&$HHo3N2@70n4q-`@>8m;U}$Va#D7tb(w8CY=|?d3P3J?<%ZGvIp`Kww$oH6!wA} zR?GjV`Mpk)X1;>6t*eeP%el`hXeIF!jt4Sw&eZs&!@s%%_bY2*ED1PxLCY^kL zkq(@!Ak5|8y)f`|7GY15?#PW=GimmvtK4zg%6Ld0Qeoo%x_IC$dpUX55T-TuL(=KY zJ;KXp^I!Kl3KRd=l`qqNntUrvK79V0eED{fm+wjbi@($Uqy3>PUxxh)@+~&`GVB#5 zU!I-j<@-)&zC7E1R=$JeyWPvT*yPKxdwBV3I`ie&N1@N%>yr$l(#v-{>5km(j?M8$ z+_72U@5eq{Cf97JlrE5TnP-Y-Gm(Zy2`w|w9hu2)&HkS zul-$fI9%<7{s|9W^zx1*?5KwuKRhFEy8R3CYW`I2Af>xbdDHD@yu5=|9(fxq#*Vnh zKe3N#_Cd<|Z{k#MZ8Oz?RnU7;|q3p@j)Xv75O}HKX#b^N-ysuawxqS+s2to?_10NkmOLZ!`1yO zy#K0%9IDrUAcx}IkpZ5cMDAqlCVj52f;qguKz|+$ro6LOzWQYI?q5BzeJ*^C(q94| zFIU^(J>T_N4O;8>GmqkDo+Mm2b-fACv8%oC@0xJlZ$y_QkNoI=y`Iz2u|<3w@-p*g zEP5&p6X7Ez&q}{9ul`-(7G>W>83}#5-sDvt{V#?;3s&cyVOKQIq{YUEee36(1$VLE zC2T~H-E6j!vO0C^M$jhp$JrZ>Np?6(eFDdCF)+uTXrO%f{5v{zZk-9cBI%B_Fqg&n zIjH8Yr-c8XWK(p4IdVL?!Bx~Jn!7&LwJ93%uuG>V5MD)CHTFbH;opUO?KAs@3?54d zAz~RDh`&=`d|NPf;=Pnu`?H=tTvO^9c0>~sZO^9dx{ocye&Pi4s`Yb%U!x~u>rcG;Bir9}*0OP;O!w$%s%lR20eDR-d!H3b&y}e^?^YdKWmW7rh-bk`q?eVj`G2KRe z?wID<+Zn5~$JG9WF@2n{ckvyl^){GqjVb!7>HiYXXd5&X`&afR=H}nUFZCmVU+UW# z_(lJ^)Hf<4apD5&TP7eUp4gA>c!Y{rZsyRV=>qphkLwnI11iL#(D22jK%Cv z7>j8d3uvOo;yvhz#v;O47`(|s=eFDTLcZNS9sJC5byULSPv|(l=H*Z9{SNts-@410 z#dwvHR(&=1vkGV4RbC6>!9WIkn6)Q{{RFxpe?m88ZG2RBJD>-~t?0A^LXUA6{j5#s zF{at;SzF@mq=%c%m~6lX2-)D~aCQ!BjI-WD2^|sF9{*g%OLE-7z@zju<}w-Fz@H=I`_0^qDgD^KIn0z(t(r)yBklBxL2kp)aSp*@|5IX?=(H z(mcOHd_2H;(z6#?hrbZX>_QnU)@oikZ#B;Sn7v3>|G9_VTqt{yG;Dv)>ic=!=6P-R zepk8kJhxlFpEZ5UwPOZm*}wc3vfKm@fm1yAiZgkL_Dgv83pq_5{dl{F)zHXq4gc@r zcQ56O21$mr1bh;2@Hk;+jlx%HZ7OUPVLC4~z%M+`uYHMh5pztu{1_a7=ZFU~hu&cP z8)VxH$)|Ae_~U!Kl^ZR>jzl)YU!YvIOT6#b`PTe@&A>E!8c^%ejn89$|Md59#WUsJ zdbzU*8ME$NzS3E4j{P>{F1|{9;~RV*G~c@am!6{Z4|!LMHn{$+GVLsUzZT+?&#b{+ zx!|PI=Gi{xM|S5Y(dipVyOO`=?i-V+%ba}~7uvBocq}4YiRWv?_h9#qp6}++zRYMz zjt_q`mU9Vp{fYT>`;}{7K>i*4hnIWZq}`br9iL($PvlqRb5`^uzwd%*s#QO&6<=Y! zFTg&loOgX!@Y}z#(B8q_6z@^TKyy34sy+HBS zTF;3y&~GnBZys5FE@h5Sw(clLZjG;>(OLYiq<^Ac*_=tVzOVA|Y8K(bweK=++9M`- z;a?$K_$D6;IkZn@o(I%9=S}R4*h63cCS_gA?_?lr=*87b_TI$Vzq?-QnZ(*Dpr zu7dISix>B`v*NTbi#LCfdqvIVEH8dA^Y)|LTpAo_EY~0FXb^vGaL1#;yoZ~h!K=E{ zVBza;pn+<;+9uxNT*6*4ZRiS9yS5Uh_(WQ@=LNOrWXG1S@C@TK^EofSpSwiG-OL!3 zkX|+|Cv~o>T{cekLgP8lm5i!eE8C?*(DV@UXX$Eggnn20a#|;jd}1yB4Z3W}153y4 zueG8Firw)}1IN^!r|I_%;JD@_STstyy-sjn1$Nyj_|}@(UIkt%U8$G;aneIKSA~P; zFmLW#xs$)``ngfMhlzg=If?71EX!WO|1WjzO{+p5-rp0 z{w=270bd_mmvX^HHJm}Mt3My@xQBC43igzuZyDvzCeAN;q<8m4>;qY2+ABvBCORb^ zE87bjL735nW&MagDBVz{<2?Zo(cFj35gf&xreGj{Xq@$L-T*AdZ?n1QL;r`o$A)kR zh>w*6x;wkPbfx?*rK6*&`jUv3+(te&#n&A*ydL&l)RhSLLxXqG-)=ISRaV70ojx9a zPCs=XIXMDepZ1*Kzhr2~1ljYB7?^JV(ZCE_vL@)*G|^D$*e2Z}-wB4UKV4w_4L^+hLG$tp^$T57=_6^bE#|sN__6-t==r~M(`-{3 zo#m0wZI}Jb0ry=F*B+DiZ8_WBy$~%S}Wc?A}o)vX&IWgkO%zFY!(G}2t z^w}C)lF)Y!5wJzs#d47SN^T_}^f@B_k3oz0JhqdlGy~ z#P0_eL{n#)INlub=wudjGeo#_V4DLqm!9B0)3d8sW!=en>JntYi#a#TwxU)tVC?_2 z9+)3v|3{leXXSrHc7t)ouBAA|=wSr4?|Qz%>PQp6t{1kbz41ZEcV(CV5tVh?lkv@< z@9w!M)1IiY2hWZp3y^GU25oo!RUrTJe3FbO%+*g#vpH{)F5iqn$n-PUeyMW|Bp=JO zH3oxCT#miDdmL*-`%9jQ%eG_P;~2Z;#HE|KH2fX7z8iM&KfXloq+hl-zEI=F`|{W` znSRR;1vV)fn|OrrW9(ep6yBHU96$O-zc)m_B)P+?aCTu-WARJyOXr9)0$@JxF;yn1{sr9sD;})*(HY1pUys>%7(d zzs6YhR^7wS{;xED{Fde}-+oIFJbuIc&$Raf|AfroJjTWFskP!$`@yI7hsPQKuQd>! zD-GT&9X=JmU)(#2Pj%R9+SzO5%kd<8jeHx+mPPBt(0gy6Ip+&5&nkVbN_f_#(91>e ztJm{eN8O_L37&O5<0_i*65*ox37&PW7yi7$YpqxYJgexW$}DmFZg|#X9ad}${A#=x z_7K47@T~Db&d_+ka_(Fb?Brd)LwTb4PeIeWc-M?Gc-Q+AX+DW}tt;%5$zMU6)DG#H z*6`h&T;?nhWFMJjplnV*Ssvd~JQbh%-Fa913ChkrkY#jf@yXl@?}HpZr_I&3kd1F1 z>jqgEc$nqM+vib__`NFhYj1?^wa-rt9eJxq>{u!EZlU#xbGOIy);>5+KEaQ?hEJlm zQe#J6!$;t2yf-qRwKB>VEq&0Xqt{WMc!om5zvkHOp5Es*gm?12Kb+vqU``+ zH0BdL9EbqbhD03+9#;GN+4Uq7*2Sx)8CjQ~FxRJ1ru~Rlzs|&+>6ly5n>Nu2>PgV* zACph?L_DozWpVbx=Gt1r*H**V&iPcnHV?kGeywz$6ZxA1%f{_*g0G$F^0i+E|6Cf8 zYwuzF_f0i=lB>c`4raV_?Dfo*OP3kLyWKlI`NvV*U-&Pc_Fdo&j8#WK{#ggGrU$a7 z(^%8#tmzEaG(P+mXL0s)*R*)Z?(cyl)^$Rb)_q+MDP>)w7cRQ96@Bn0_}$CzMt&C) zJ~7!E7!O)wex$a*zl-*40)Mo=w-e^_bb0nhFYISZr+uCF|G*2|qAz-bG9dqzS z=HgoBWCn9Hoipt;&a~HXrgiPn#==k!LNk1^nUoE<@_Gx z_rSPg@EOgkebJE*xMg2M*`}|?UY7d{Pp4rr;T~Ps?SF29Glg}spSru#g|TMs>$SIu z%eJ+DBb(4T$8`M|D=HLAK~;4PW?aXVWy+@ana3c-F2pTsv-m zfcuP%?i&6YxO5b{puIqR{CkYoK5Uf-b59xGeJJ!Cki6Ex;%GU31HZ!B?v8`fkJkUcP>%3YWxd1q^`ZeyNzj1a(11SRSzqw2 zA9&Xv{2KrbKn7Lo%AXQ`98R*g>rJ2|?CsrcL!be>6Y{9@F5)cG%cTKhxzoz#*Ur7~ zyZpALTBBdgD{H(9xTydgD%LT+21f7l=M6;`JaAKrHD;aq%lK)XZUBdbzgr1&X+y64 zq!;#r(s6FndR^m%{X}7$D?}R}^}=Ebg9aaij!=gChNd8V(;Pu~;S%7T9&LDtd~tB) z81!Td{DJ6%YXg#N-%nhEZoJ{i__e-Gdgxgq{YH;&+~cJm?9z=h=PT)R(Vd>G>py6t z=IHoT?(k{PLemfIe*}xM{}E)~Z#OW_o(?>dW=vzwu3_%3W)7z^ms6k_S3xtb#Fiw< z$ie1dgLCYWvRD&3z`Dz?lP)lLUyUrJnD060>>7FkU6Br~>=<4C!o7Qw9LPcrtNEBg z{f3_Ow;J4YSt5<}G&Ahu$V`;(3ew>(o^|cgo)eVuO=!=_>HR~ymiVV1=R6|5EIx#@ z$VO~Rp24Q%h%cIZznlMJ@=LzEqmR`fI{jx)-)1!7S|7qY|ML@^Qr5D3l{8Fx{v19awa`HdbXPp;V?5_$GB{sSTaPaS^{)#&REa3Wn}f3}Ni@ z>{X1r=2heP`klVV)_F8%z$ek0!t|A$bZqJmw;i>qewFLY-!j*rDG43fjwI$vyd25U|{D*0OnKMq1HeM`9o4+8) z8oNjkyvqlQwkfY@-s#0RG3Pfj_cwq8*MkdRg0@`;ZTliPaxJ(r1DiSCA&Vp%I``rz z-cbX`WH+d>%2u19G4!pmp!}lN>ZJ*grbc?d;Evh9n?uoGd-}qszKI-EanyAJoQoAt2xUPxw^p7PvZ0$>h7AcF zqO2E(jgI{!kn!^NNXFJf_+dCUJsW$yoRD~rmg3yd-$vyPZylCDyruY@;m7u-hoCQw zad^7sT7PtBt=rap)TKLdc%Spe{q|diUc25tk zJWZVTf320F-Q&QgUl8{=VXiG6Wih7*30q~-=h%a~ryu6(DX%bbxpr3fIB@k`+P%!g zrQ3e)lXd312zqVkayfLl7(2GfEug?deMxj5w0RITHL(Wuo=W0d z`iF^c`G`$?tDYH@eS&$A?BW`}#g9xCgl1m_>?1n;Z{Az#EqL*X$GT)doqXbxqdL}9 zWw|!Q7iC!u8GN6}v_`+pFUUEuF4G#5PkjkF;~>UIG;|DMhWFrJ2iwr0x`*Tqtf3RD zx7Fw0mxHZNrkzLHf?$5DY;Qj{Ho;+?=e6dA>Q}m*W%`FqhX0bYKhBtxpu?HNnroY# z97;itGd?6KHWXj+hla5~cikEGgLa=jHqVo#G(9j*`eUut=*Z#w#_%=8sj)-FyxTX^ z+#~MJ;2tppnT*bQr>IXl+WP*HC$~`>v>u;?w^{b=1jhmo-Frmz^*z~gD>P00oC|c< z6Z+Ql<6O%6gBN$pS#cG_{mzTi{?R?(EaHCU#fi7;9=Dvh?IsRgWsg@`NSjX1^SgRY z(61WWS;hHU`@3Y3bsio2Eq$rt>@WV$$O*iAM+>_w>3*!>UatGSV<~mL0)AOFx&yRY zOGedv${nD^x~jofSx#$^yTDG~2OhlZ+|~U~Fx&3yt*>>o=}YdN;O_2sg4^MPt(G67;c{rFyN_qvb;RpVP-{`~%lR*yT4rFHJ;y-pWB>ZDTI=vI z{SQKK%-EcDpNPMfkNzk3iH7IlK2hiU6!Z}^-YGubY4+Ci-$wOfPwfrg_1QWJpT=_? zh3C?{4|+THl&8C`w__7JGSt60aZY^lLS#kg!boO!4|~rI{jAYn;Frm7T_AJbE^G;3 z?#ugl*mt4(*qFmOy1Z$QoyU3=Ex3>{;g$vN@opY8U~7GQmgT-b0=+uHe3-jCY>ULh zWRR~Q=-%BOBObpcjf1hJ+r;^~+gW7_L)*YJ947;zHN8Z-szuK%fB1ZwQ z1iEkaq018p4s8kK4n6XWFLs;Wk7Ms{^;^5EE@jUM)Q+13&v6JkQgUCgekFP3v+>Xn z-e9~0-=Nj~Lhx3NCyV>DZx3MK9>~6(HoRlHp&Of}zjXuroOqhQdu>pk=76XB83W;c zJYY2@WQFn29Ujk1liXG9)Y1`u*N5E^qKB z>X8gb`ZWf3y!Bg!{M%c#!@~=_YsT zZ+L3~9py9izE}&p|6i;HwK~C z+N-uPhX$WQ@ZUcC83uCOmU;cJ0iL?W+_~=#M)Aj9%swIA8J+jO*E>3%cytoB#J`G< z!Xj6Og1?<<4tR8vxwpt2TA7@*ARzlsBLmI1zd#W)x%m^0sPzLZyo_3E|Kw1M&}D1TOurQ^cXf5ian`J2%hQCSwez|dYP zE9cEj$qSNa@@3Z6dA>aPt~(j(<@b(~k z%mDt2_Q-FY?lS+pK=%h$y5i88dR@AUEkk&JwvX%r{nNLzKV%MAVeSck?8#69l%ai| z_iZi`UCev@a&rbeL0RI%bPwv&+UESO|KhKcDEmWb*twK1`)1v9xiobs@AZ~RCvri` z-kNEtf55hDz>2l1H}wtoJ8$k>Z13{y_qvteV)!1_g$}dFTl}FjEZtTe(lJnd|K)}M z34E3Ao_OL4!nYDGe_5`sXuhqmCc@{!0pHF|Erc(?2VH37w{z7u^-;P8x@*xL%YNh>2e>a$ zy#o!s&bBww-*36}Yja}1m&{B45u^*m+c2%J9#FPD#>=ysJaN{@c;e(g;0gN7`WP*G zTW-cL&;GVIj&%m++7AGA9x~@16P9K#1#Sx5&=zBV2|*7+&;#M(K4fg#Q#zUkex?tG zL>sgYRQF)Y(;EH&Iwcq%93H#NZ2+ycb7jnSu&X50BhjHzjTZdPKtJC zqL1?;YrckW>ErB!=13oB8)X(UK4U3QX9Sh|8s!@~7I(nV-)TX@&&Y?5-RH>1{kQS` z%pL9mC#^15c^j$2>=lpeOl>t7I@J96Xpr>y?OR|4%XqVf_YhMei^^73E;@!k=uLiW zT1g<73Lm#%A9nKwr&$kmx@#0|#-9wdDM z?nT0cO|?!NJcS-VNn4~dZ2m(RlNdAA`#58!Fwt105wF!EI!hkKKm8f;-zV-i`q@#N zgd8l$K9~YOmI^=C3prSCWFNezQ`r|;NIzsE{gHzqr;j8>Up-3>*4?kuiEeT*?N^EW zJK58D_wt=QS?L9w$>7N}k9Tn5S#0N%YAaC9--j;oV^VyU97W13KIjw|q z+JDk!_dYMvZeWbXo4rby__Zo7JjR@)Q^G((ivtm=?j8s=B(Cz z|0st7ewq*kJG|k>fI&5RNe1mir zI!4yh!RYOkE?s|$IPKYm$d^ye4~7;?Ha#yTiH-i4-b_loqwp2V;9Zi|Isfv^l*gH= zzU#~c?;?D4X((qV?x_w$TpGHQxiM?;EZs?+lXRDqpqn3%&hV7h2>PeHfE@dDb(en3 zAjUpW<&ODfjLmv*Lh>=n-rZ>rz`AEI)|v1Oo>FT*rnSY`Xuef%U&_;e_M$DO-m1@c z*83jyCempf3#m_S8f0Lmoo`^eoois4Z5gP0*K1f)SF^UJvc{&c)~-U%cqMYi$*jRE zSc{X8GfqS|!@W-lGJmG6|3C4i>UR!&X%2j;{746Tm%SSF`C7w;_|kuHU9MNoPdS3)7ZP-Z`i3a`s;KdPj1Z@wL{1$RumV=t<~>@(WJzU4tFR z$c+=dJ;T#4>M;U6)`?~x+8q4y8rzCLGU z-P(>oYAgBIvJbbOK-WWc1R3KUx}()TXq7pZZ~U8Syysgz$Xc)hdeyPll3)30S|@J0 zVbPy3?m_Od437Ci(YfbssxR4)6CG7_!`N!{bC<%;rWk%Ay|qZ+J;Yn&w(b}Mt36xN%#^bZ@dY) zmg1I=jGnrwV~uy8ze;1e3x3GNjlM8?jJQOeb?ndBiO$eIT?Q=S48N~7JGz7Z{qpOq z#g&|eN7Y!u6Mysio^9U4v9Q`Q)| z&>9o;m5q&$v^b{VyN!8F_-r1Ezvfi@HDlXzh5R?K#AkCad^Y2w`3m`Ho^*jVCIvsu zp*O5+#t*PYHjRMZ6ydk|p4@?RMp$DgYgKUWSSM^SN7vUKjZKANJ`R`X#@G z8;0>+TiI6N%Wo~`Tm88wP}wH`E1+2Ee9k&NlkaVeUFyAMtK)&xSMfdg z?O}nGS9_nk{?Te*@76lze4a0VrEm~=&J1YNqXoX~*6gL7W!KYpX2 z*YN6x#p9GbUbmL@;wavqz^VLE(XV@WV{ql=QPJD@Z}wc_)r`?mUm1SMy!p>({tIxe zJ(vDyJf_OF89YJ}ev zba*K~){d<7)}b{f>Z@65>|=_p_0x7YW6%BLukCF{m-j{9?vxIui@)bbe*nIS#}!=> zzl%`yb*B>D-g(?xKv!;>AFN+JyDav-%JP_-rz9FB&%_e;Q%lD)=0`j9$6-`TC| zPils~vs#PzPCLili^fuSq8)eet^H{v(5akEo68*CK_}vGBV1we+QHG>>ql*^V@;i? z)O&rwX9r(y%~&z4JQlhywcfdrH-?f^Rxa#m4ao9ktz;cUyDSFnNw(f=>^PQM$^_TuMR%(4ShwgZ=zV*#`RBC-Q zhpEhSCG%X)x9~ZYxxNiQYzMHBxPrN^WNp{-t-013y_xY44Jp*xICXKf349hW-F7>2 zBk0h@gx@gl687|5+vl})>p?foKS(1vP$EwHXwQ3bapFX~t3-1r4vMb0v15%{8%r;a zrW9FG(e9Gx>(=HmuaXhGNqT6=v{U`8hClNA8NYgdl8O9czRrEV!}0^CGwBZT?=`gD z`~R|u_a8pMSg0*q6Y>0aZnP*r*M7lE(=;AA7W0Lr5)~uh#zI{08TesHicf32xp1CRnT^>rhlu~O< z2VwsOEzulIXHSpsWly$(sd4-!z5!j^0$meLS`Y2oO?{g_fwpl*brk<9@xo1ub5epP zwSfckS^R?D6m%yWgbz9pr5!?_SDId~av}r`#EdzCKU9)~4Yt={FE^f z_VV;rpXfUKzq`+~j-Zig?Bzz5ztkF=C}SgK6w=R4_-Tq!cL-i8#Jo!Gu%QZGqiF~- zhs)vTE`y(|fY*({>xSWV%i(o@{WbA$n)_qe$O#v&HvP@Drx=)HUuj^zeT9J;_6x^4 z#^}z*t)Dv*zBN8ZHa4})i87ysCDv%soy2$*z-OHZtXbRfh~?~?>Do;k@LT50gx|Vn z!-MH-AZsP^muL4Ev37hpE5A=UHAU9ww#uF{;ZdQtM%uw;t&tIGPsDP*KzMeo-c!u( zTqFND;HEiGX~qi&-8ik0xEEKTIOOng{A@}NA(>;Mz2M0hQ?6NiUFH7K%bRb;qN~ik zUU(+qD`>acm&~`$gr|V|A5fkNZd*=#jWb;YS@96d_+^_TcnR`7!3og0^Q?xaSi9lE zo>BSS8KyG}JeBWi>`2IaswItGaf<4Bc+J_<8;N|mMna|rSW5I{rBA6 zR=fCaWT?5k4Oa8|BFgo9|0N66S^ZAJ_D>f+=i0UWui#gsut-s z*4tggb7tFI^x-(tr`gxAcQhlTIlx>Wq~Ds;_mi)RNv`}?)};+zmX#%F*7?!L8M`|< zYsJ|!Me9~Gw+rtW9{uLPj$b|^UrrH^?GNu-NE5yP6qS^ey}kldULxk%H6^4 zE8g#e|7&Z(bl-}#Dl=t8In(;WoIwX3DLXy$?VFbbfBwPiNxs=X zm>HZkNWKe`Nb^2v9^m(WebVg*RtIk1x6*&R{20o&QcJ*hyXo)OE{zJej}%+rZHB#j z9`YW>cnSOUWAvfs(aW9rv|aRF-!Id4;h@?bq}^4NU$ipv13$kez7Nu#<+LY-_I!@^ zEDzo~NbN}B{C%F{6JGAaOZ=NJf`-^z9+4M={C0<6fSbhCDebroD&3=7$+%_*x_Bm(A;d{g6)4JHG zG3*y{HVF2LIO_zHBhL2)0}-cQup{g|A$T(EtPwmBb{-cz5_VP#{v+%}1zW?;BZ41< zoreVvgq@Xw?}eQeg71W#2L;~>JNFCj4Lh}huZ5kZg0F_1YQbNJo%;lv!p>sB?P2F0 z!EIsZZo!wr&YglUgq?+gKMgwz1fLB%^97#?I~vdCjbUe=;JUCgSFk?p%obb|c5W41 z9d>34J`#3r7F-#2Dg_@5J6{&84LdgqR)?MI1s8{%>jdu(JJ$*>3_H^W=ZBqZ1m}gF zse-e^&Q*dl!_H*E%CIv@@W!w+LGZe;bD7}uuoDrS8g|MBCx@N!f)m2drGk;LbBW;i zurpThlCV=MSQ>W52#yLnqXb8UosojW!p;c6Az^2@U~$+PCRh-5LV|f=XNX`{*!i4b zTG%NT>=$;51bc;@0>R|4bBg4hZ(EaP|rIs&L*DOs;VLEEuS8-VyAGIDZs88FAhcJP~o;5Ihob_6q(Z;_MM@ zjX19feh_hfCwL&@yejx!#QBZjI}zvCf^S8fUkdJxI8B1DMVy_2uST5hg1?S9uLw3p zoNa>JBTl2>wutkR;7bu_i{J|p=LNx^Mx5sbpN%*_6?`V*JSVs@;yf$3F5<)l>m$xH zf@>nq4+U38oQ;Byu>J*Cvi=1hWc>@)vi=3DBhC|oizCh&!Mh{Qf`@*PKdWKcD0 zRd7 zIQ3S+Cw-GW)Z$zGh@U+RK71PcplRD!Zw^i`uApBL`WL33<($QSxk>$Qt9+TTGiCZm zJzYxekDMzvzlnUm33-_O!z9Wrq+HFJd^qTSCS~m8M)ZEhWR2}5*`}#Gmoz*4NzrW6 zhDV;j}(@B((V)CXK zH))}h?bg{2o(X?SXiq^3@-f;ro;K;Mbp%;+BHa<;Be;2baXI7ib5q{&>t5lT@)mQy zQEM^xe(nl<=yr9DgX9PM$ZPPGdyW+3w5iCj=JDnXGS129_st6~z@70U`d$hA&7wO!)i~4*w zhdEcI+gC9*+}B-X${092y6M8G(^2C4Mu%=&jr!>n)L1yr|NPJ}T`F^l#1mKmE+2 zpGvFtWKf4>18G3X3g6>x@n+Vw%GTHy`0`q{rc~a^Iq=WPyf1XmBF==_c8bZHZYKk` z^8SUwo?x8Ico$gTzv5f{=s~<_R6F_a;av>HJxE-Em4DklWb|r>eAxek_^Ol17b*TF zFYj@}lvin_Z}WtyGspgbaHSLf=(qBR>RadjSCPeNJ*3}!xg(u#wfkMtCh$b%ylwKP z+iwCjX9?U<+#d+@^j9`_mm`%!(-nVu@h97z$>wbH9AA1V`4ew#nsv!O5{IT-#vJV$ z;*Q5olV<0pr%6BOK37ijvKO|FFxdosmJJZ|h^*_(_3>11tKkUui`&b53nn6al^u}m z7LP$E<^Sdcv_rC(1kKq?eJPybbuKvseVNRjF>hafRPt}p!twZu+rhkl?f#0b4|Q~m znRI>0R_>|?IE=6I}OnK}(_w}j& zJ@S_5qPcCnfBEc3(4r{5;=a+-Dt(u`oWEaM7W;^|76y27`2K^jx$S8k-og5`2A4I~ zAcwd7Uui3_vdz9_Ez@_F*|#h|_exe@vu|0;^xe%(cst^ za12|YcINRU=j%tpqkCX{VW#V*2~-@NFk;dcGEzGB5J-;PVGde&SjxzF6e z=(Tgd8)RbI%~l>BYC`%$%^*xEQ(pKKW;0odvyphxY~= zxpVB5^2_wCDJ_z}=C4d?v78i#cL<;*@;{j3AcyM*?S)TLS-BZ?xeegcZL^V=HT}PP z*G=!sN+(lRk|C# z_geS<*Zk)`_*?v!JSefQcd@Qb9QVNsh)diD$C}E<+9S#u^@h2*%VRBwH(3v?!%ptK z3(@()9-tjNsut`hOnKfte%cW3@$nl_CE3AQcf|w#)UihT@_IKk2p*Yy-aYY(WcQvJ ze*>jTQ-|E--@PY}^*HmMSntaPy?f$aN!A#Z7awv>h&$7H6S)%`1&^$^G~%+ggV%-@ zHN+p`jVAcz_=D_+z-+?S6_*{}$=a0yoMK-HegK)%1mu9(=w+tc2iX(eC;wlO3x2@;&W>|_GG)U#&rMl&&X1?e zJm=Xd73bJfK6g%RN>BVueFGWG|JOI{#LK;Uw@cSA2e(gZ|6;s;<%N|KCOpzPWv3T5 zjxhD>#3;+jV|E1i{s?2jtSXS#Y!pyjg;C&y) z<>WB@Hf@`~S9PSDH2JLo#`q+8bjM-dtD#KU@F8QCKIBEpe==}ctUqm$onl`>()I>E zZSFX{e3W^Bal4g#%Co|s(Hemkf)A{_%VNR*PA;YGtGercC|!wnx8DTr>TWNKyP^*6 zgPLjc9G9N#*13zhAHA=*V~uN*Bi$<3)@SU0(4O{=(IxoVijTJ%Wq)Gnevf;@Ev!HB z8i&5>j;#rwa|yZ@*z3{9XaDTR?m7&>RzyRM#mcPa(D2odqE4{NAtFu^lJNw zV%}cD<{~}>yocv~hhKAXne&PnTi@Yg@dw1|e6PE~4a|rBZ!z)y!?$?&t?}Gy+JG&i zAZ0WFpVIg;4(MfPJsGIcxrOl?7HuKUtQ+peGW+DO+%=ecH|pLMhCkyjFa^0K^z?8k zcY*8=yLAVMKE#-y-`bKzoct!4{t>SK;zh*A$X3FC?)qulFZIdB@7XNmr@C{n@XLg5 zAZhZA40RB5ydFE@9n>qk&4*}DqHVwAdwU?Wt?PXt@n{)#K;=@7=Hmg}!Wi_l{ z?cY;%ryE{FxOiJ@pw)2Eb(zx>Z44UuZMuC2@psdo6SrI#ZT=H>1Gl>SPm6y}TgzZ8 ziY$NC72G?Q_~x`7nrzAU3HIX^r{@5J*I?&?-fE0Ap{0GJ@gb?Pqc=hW==V*OyFVgZ zO>DOLZoaY1nITBrGy~J@-OQQJ5L#cq(B>oj zT$@hPF{iRkJD7(rtD2f=J*`#K(ilg0qS4Szv;RrQI)!&Fogr2%9DEn zN#0MJ5bH_aWb(ovJyr;fz6G2O-!wW}#+)YRU>0o_e%=P&2qMEsUx-gBlNMc+G5r6= zH#^pRR@*-8VLi43x|(R4*40e%9H(7Dwb8q0__X#-q*UqbN1 z;I$xQ_#W_cj7dk6?CUeoWz0mUF$=xoZ1jqAkU8Wcd&uM5o6q_89M0K;IAg#0mS(gW#QbYU!Yau))6qTYSAS`w+JI&tao)Z1qcB z{^A9Af}h=ZS;+7m&w6|ZcFBhCcm@cs5T9x}W*vCEK>yF7n+#sH!>1g7tzU?7#OKKc z(JxN#;x)vxw4ifb=~`ynI~Z%#lTz-O+fAE1U8YU=5-@Fnhbm0;3;c_xSfwT%x=eNWNa6dVf0v(0!QWDMy*ItR zo8V_k;FID*t~I>OF5a_?{}kIK{8q=GDQg6eM~i3i5iWnMJK=*&8{k>yu>O>`g>>Rs z3eb1i2`|);1i$huyibx0iWqM{ztfAq0*-zeT)hdLy%F5K0o{e`(OviwIDH+s z{YCV2uf^|4a&*HuU;19=CZPv=y*D?(q6?!bp|V(rvmH8CeCzHI-w4FJ=aGNE4$10X zWKGvCfo=z^G1A9=;`KZCK8_qyYq5EX-xwi*&%yRFpo4tI7 zJ}^GwNv3cIUpPSfP_F%jUJN74d zkD5M)$5{Q2yRZFLY2evTO+~+zG17YhooPLOq|QsbowQo3RUR)=h3%EWx3hQ=)~s^` zey$pskz`h~S=9J;5O#|3$?XY_u_x{70fdLB10H9uz6U;g2e@E0pC5gTy4DB2*rxt} zkMAFK9|Oizdv;Qd?k!hMt7?Y_U=K;;eU%kFy{{1nb_ACNe>Tx z% z7Si=-XU;UgbwHOlSG;Tgc0_UXZE5#7a8kM-9}<=rqo7qgEyY?sZ7}yArcTO!lKSdA zf2ErO>20Rp5AVEmeeGl8%(r}&j;Oix$V}t`kslcipy*0%oxmI|6yP4j+c|W;hoa| zntG1V-{X1k#msRN|HZ%W6r|tV1sR)HfLeFk%>PV#1Mu-bbd1*dHwfOfN*}O@ydPe8~d(ZpqEq`4%HMHfQ@CD$_#q=f0-Blqlv3`oYcCI8` z{=RabLsu3W@pHo0BQMt;Ecr|D>Gjieb|~cS;Q;Hsj5y0XOWoqxfUZ%SvEiy2U^R5W zKOZ#XuJagl)1w)`B(M1Vhm%c@`#`BF{$*Q8SU1abxcRF(()PC4C{s)-XgS7Eh=C%s{eMgVB(t;qeCHR** z=2H4T7JEkvx<$eT(;0g+IQB4kq4T3Rfje{H6Xd@k z_z7)#%c}#uH#2wdQC4$5-Uk+>o$mLXxjePwY zcU^6y@8UO1xW|i#znt|)H@=>Doz+$LUcc4Qz)v(td$;tTUH&cI*4bS$4$16v=DJt$ zr+kNH*Q&Wue$nEy@BG~Gbo*P>qjK*u@Z;$^YI}13fp~s)2K;U&d~X)~Z#HMN9PU?p zM&;*BKK|u*`8e|>dn(vvgrA)m1IW!LbzHVLRCI&! zCx8BZ>GkW-6PSp+zjah{tfiEgkOi&%f?27e9|s-+JzXs{+WD z8OIXYh@;zVXkGb$sCE6lZR;}2{wdkYcw~fWJ=W!FUzBgyXWw?q*iRYSTa^FEbgQwP zc}%R6r>}O?{E;+TC%P9>TZ`Za(^$8X-CcW!n`Sp@+&wA7zS;|Wl`y4$vB+xp9=|Gn zsz>AU0&SMuQDd{2vDbPsGDF7b$Lx=tGDH4vGV%VyVa|anuZ?ze1d?LwU-0c&^$oZF z&C~^5Y}gdJeyi%(2X5|?&I9RB!VCV@YrE{97~4jjfi}%78*60lz7%73?xS5t*x%K! zG}f^8?-R_o_Lw}cPv4~;jm1knX+Pm9q!ZtMA?21**Tul@^hdO3z4!?1{!*4rtsmqZzQ17l*2wWaevqg0YdvUuhN5?qbWCrExL!4~u@EZ{e%PE$ofkEreAa@7VGpv|szZ-VZhMW%dZ2 zAKqtu%qOjp&-4E((#wCNWP$qLSbKe&##{FpcWYh!BN8+3GZ;M21J850SHQm(Yo!Xm zJ{sHc20qsmDHiRUc))<6!cPjaC>o1NW4J_ zGzb4jG4Fh_+q{jy`J#?Alk~3Zkbc|`=G5zaag?)#)BS8gn;I6mIN6K-)g)UD!WXUm zzTk>`w&;2n!2!30H(CegJ$U9*{M7-%HC79N?%6^G_p<%i&qa6{ipYsUT|olV@azlX6G?rZM1kx#hRC`dnF0_uL4 zdkjbS1TUEIZ2Nfw)9jxbm}x&JNd6<7EpRMA-ZXm^@D}*{1fCz@zv}sr_QV-`8tu2F=LWuR{;M&T{Ya7tR8GRm-2s2*mC6KTjx%l)y~#`qpyZf;LS0!E-o-- zN!R2|J-I}>j);Fx`|KomjUMnEJ>fl);f;dm2>PRK-S{Ghd6oaygrBB!ymIpl%&>C| zGuotUvdW{I>F2%kM6JFZJTRTcFZa zZoK)QJMo#n>%S}a*7d{J77$zBEiukVzp<=_lKbAQ@8Awz>saaaHf{^N0DgcCZ*VVj zE8JgAy}Gy6xZLf9eaCCdDfX6|&_OEXr?(1jVvl>GItza%dG<dpI8 z$tpYBzMZ&uvh-ZKG3SF6BiHWd;pJCppX{TW!ONFx%f?C;qIEl4xK%6PB=~jmbbh_~GtNlmXqZAa0C_%eI@k$1$$L)z1?r`R|!BFV~Nh&ZmhtdVZ7Gm!3(A zIq=i;akHg;sUJ2`#o%&r!p9SGcE(HVPJZK`;Px1ee@b}Tt6oAg8g3rCc>)zmZALvY9?#Tkt#QU<^KeSJ1_Lc;Wn`H{J9D|x)OYv41K&J z27Vd(IChFVhKosy-fe?;3`4Jvx$owdG2cP%tk7@R+}-!0X!SaH9?2CB0E6r$Ce2y? zX(s&--^(>^!H1~o%fdemyuGuO`aWwP7JrrS0W6$Itd06}kVl|jA$?ghgbd@7msyD+rfJy$7lPX+a4Z^_JPOyXLjx3 zH_~U-^C5jY8;8+z|3u$*OT0YqlE>ii7qBI!Z^Ge6Z}0MF`v>CR1BZ17uK3;Fxc%P1 zG`qn-`Lq4>J-+TL7l(gKTH&xq6Fc92{wy4R23*p8{r@%&uMiH``k?VX)})`c8DNcK zYqGcpayDO7`ojqxQgq&1mz}sA&*$C%T}9zkCV0~ZzxW0+YW;sa*j73|kTEZeuGeV7 zzRI{5Il__#c!ErOKDbm6%xVpP5-rCzdLRD6bRRgHe4htjg`awlHX(<&jqqIJoqotR zbl&pl==I13EOaI~8}06{e+F%6Oc`G@4Tf{~6~2+tU?^1YzZKckDB-X2yS%9T?stYc zQ~!{%m`k0hcZb&}d+FG}!JTHKN0nt?Mmw}04+9p`o}Sv%C~v$M{yCt=NUojm*Q?z{qIm z)}i%&WI++mX)_xxi5B_-(W0F_qouqdR3kkmbow>d!COQ1CQa{gQL88r_3i5!l}uqc z>B_(nwe4DH#D41No+of~8N8``_a>SqIm;p5-(Ih~-fm_4`dn#vO=Rfkkh|&7tuW_i z$uc?5z22QqDbu-f*TvCt54Z2T$-Ohte)$65s`GgRwO^ij_Dk4EbwqTQ;Em?ly;-w; zShIauv;A1J_(lvQ8QHk>ln!M1V(uFtU32n%Q+B#t56px{ssHzTbMqu&qFMFG5J%B= z_SDxeK_4a^zWEsE>UP?Al=dBAESx~j&{Oy|lpoXJeaMI4<+Jb0uP>>7xjq9P>bvY! zD*Gs7Cu$=;f8?`jlA%?>mGB~af+_E= zO`*@@v3qLa-XjH{QoOHgtH0tal|EiWKjB~W@Zu51Py4s@?u_0v<6S|&Qb?28k$C4?<+|^W z9U*<(pKrz{g|X53&G=UgEFb4A^z$x2ASQnJK&|L)o;{oX3AWwnd#s*1)rSO6sxxW# zGkUdmHdXo+UcL{=r~mI82o3>VUL(V9HevAp2Bz8X8mRsIYSzkB*31;v&Q+|TE4inc z%sJilZzx>&2>RB|e#h)}n=I=h)4qEqpl@nz>8Cl)>+;)H2QBH0N2X@J@#RXs25eV$ z%g^=ww4sLo?q0BSlkWOV9DZJx5Eo~x6<5N&6!|v?@m;%%y{C$A`L5kbIr3fGn>lCi z*>XlWeqMVLUPk$alyQ2oTYg8R$DY59>Wz+AAM_skqW9R3KKG~Z0~mvWj71uBgZE4W zJ))oH-`+tsk&r1RWE0|TMdwAEO<%pcwu8CQNc4upkFSBo-^Fjb_kUd=ZQg!p{5#MT z`BkmsTj{3vY%9GO`=4(z|8HtU`|;GQElTftx_ z?~y_IYU5FGU+3y8R3GbTq9AMO^FXajr4iqFdhra#bUI@@jWNE4vA!Donu?y@6z1S6 z^z?X}EYid5`I{nTW4E;q54{}8+IsA*p0Shj`h^_);A`Eq@kZF+MhzU^IxG!;`02y- z_TWJ8UHc2EQ#^PxV=I364fJ$1KIaoI9gAg*;RUS4r>6V&=uTbl(P(eVBWx69yYCUA z*WkTflR>!nmDQ{(t#{>7e?>2}-cOtLmT9l-7Qdgdm_^)i6PMF@CbcQ6%6mId`fAYh zRYoS)3*8-LbjVu0^`B$UW)9ST>2AG4ANFxi1D~}y987^eWZO3Vb(dhQAY~SL)&K zXz+Ij_^EZchJEiYem8pm$Ah!`kv0Db{zCEf(zOWW&O_!r|r#~WJftA-f_>=H-PD*sA$4AVg zT+uG!;0C^hqw5Szv+p-hwC@VWV-n*sk@1Rl~J}I2!#S31Y2zAw^` z#JNxF*d4=c`)b0U1E)UoU4%&Co}=KD?91Ab*UH9H@572F8k!Pv?PoXD=C*amvHBd& z4rwk9y_nWkx+cwPxQE{t2^+(Yi{{aJ)GvH`+nkl~MK3%EW+%={XVR-3snHFrWu28; zuJ=85KmFL=Gq@lEpRpbu`)`D44lVSa#9wbmXJkM05S|*J{Z@KgF}|g|@R{&AdT$_= zuvYjoy=ibeHlJTlu?Eb?um3kv-puX)lgzy5fx}6@fhi|5Cjg#>;&AO93 z^Du4CqhC)k4wd&-{d1|gx5+g$<303dEc9zRFKqslbKd6e=RB3CbHDBzGm*eRkT*)^t%9FV<+PyzDM^liLffdgjY*w(+p^Up;eoDgHtLu&tAfssWE2?Y#K-- zKJ7<9?Z?HGRWr$5Q$HuH#_#fZbHwu`&UnxfPsX*XyNv4+^6QNEv3=_8Ugn*PWyGD` z#%rm=E4Z)>fVUCJC~& zRsWMWNz51^TT?&tIAi8<##~OH-v5$pLOz8HZ>1W2fD`ERs2uIz!5(#MjZGY~kIBVv<=%GRw)z{2cCD=pq_lE>HlyvQ0nUv% z_Ia#x?G=|&j@i>WKNNXkUr^bjurC>6Ia|4tlz#Ud!4aH$xNk1GZ&SVA+0Hh51@8lq zPv__eXDR7OWO@IaSbKA6gY<*VS&jR$Ed1U{XSJ_MlhwLWxXs;ks`vjXaPedKHo9-; ze)Af0gFei(W9SYX9|CVg+X8AYXFi{S$WKV4HQ=4W&ZJrJyyYIvdRgfx=N+KIPwvFe z#?Pgnw`P2n-Mx5ey+_m`or(k6hdjMm_Cx7NK7zbP<9vWR3ZX3;YrTORJYBW7q$l@b z+?B-H&(EeM+N;5PBTM)hYvu-I55c*i`T*ZEOZrExrM{?dLl1m&WU;Sh*;%|Pv>qC; zDIp_Bq@3t_*dY!I!r6kM85# zD!D@UvDf>fA4_{dzvu>!H<(44-P4PPeUGr!tTBA6L|?$hN%$+8_aW_BH^5rp2WPb2 zucqw7gpFs<80>|eLs(Vcq$x4%?ADcA3vPjSD6M1$C3h`(bOp9~hv;J~`UF=(+pl8p zn8MyMHRYY$f5h_gRsnneXI|cBV4oNB@65g*F!2V(0!zu6$ zXth``D^i0iZ}SIxmH30NPA2U<${EJqfvlx-?~shJjyHOWcq8ZA%d9Bvni}9;9*o-8>evU%d3R?p zx<^i zcJcEP=0*GieHIOJ=A^!p8~j~f-iPRa+#Lu`)}GwXe02D8=N$sKk6pMnSG3@VOH%Sg z4~`DAVv~HXtSmHqXnpZWy*u067F}A(nqJle-z1z{Bv;UxU>)+!0t_boqKG6y3D%k=bjzvC|hm) zzpTA`d{ouB|Gy^#WD@S=e$gaGB?DSSZX#F)f)YSQxrmA_D4wrD^jOr^3qk^Dji#-P z(4xe1K(RHm(Mn4YM(qK`(^74#g0;2nIfk1|GALpKWWr^>@6XzM!hopl@Av&L<>Z&%Lqm9|vG!9^$BEa1$V#PaqYNj2UekzRKMVZBEr}i}+`=&%+ zMXwC)*)cZM?nTc@`Tv)FKkyZOPj0h~aq6z1YR(6ux8=}I&Xab8vy+cCaAnOW=co4- z(0t^~(8|5g9%B717p>4fFqm(hvzO%1@}bSf207oB&5wLeTA@#}DLcYi@1u@*JN`M6 zb1n&l&eT@Q5Y`xa?(jOYz05E+Y5G?1b%5_Rmx@88wV0X2^XQD2$aB^w-yY5SAR`+p zI^tulci!jn)PL;_twq)i{F?fV{v^k)ru-+^C#f#YQKCM@;P?!>^VxOm?cZP%5o1r6 z{8I5t6uTi|!*U08>wI&&Nwe%qQt|EK^88Ix;Bg1@Wbwq%s~tfu5CBHy=bd)B6UzY} z;62|P#DE@sG6r-ue9#eZUcX2W*@=zWk@u?=_@-pr*P1r6?Xm1>qyELa84Hm4m3a${ zJghM7@6?xJH@h~7nosd3TIb7I`Y=Y5-Mh*iPdm|1N7}KG zMK>M!xQp{n_vYd%R4XmDranUfZ?&^onYLu>o8|8?($P?k;Wy-_od|7--l-DyDp6B0nW5$e$QZM zGpa*|muL1LeRCK%P}*wJZ2Jq+p|q=W>;aRPX@6=`_7k_vH`*+G16)ZCAe(W)mHc=U zaP^_v-UlYlvbU4UuA&8<|DSm_!Sl&>00cDmc4?i=8ku!?n00t$q)+$N$xdZoYDo7d z?OgKM$jF-Xkm3*D2;2oGjf?$?`TG9@OL;CVO?P4Gm-JKo%oytaHS5)3b70oLdb{|4 z>2I{}8yPV8s`Y-9d?!yl5-?8QV)A59AiDzD6dVPA8oj69DH=04k!wFryVApc)q3k3 zl{hmM8W_p7A2f5DYu``m*x6*+-#2-g_B|%eu;-cdpRi3?B6&|I< z*uc$4=2M;$Jj6OneQAhz>^T~Ee#+V&BA&zy*64v{x*7vp>H>2z@TK*giYGkBk$_L0H+{dYP0dpUK8 zr@VzW#%ZmLy<1iXZeC2D&Ow5I@skPMEHrDCWA`(C&^t$0sWrtt)*n>zZX}Qug~_v#y&v_HFL(b-(H4ww*%itZRlHpXT89 z@%iIfmkQQr9P4xi>vcJDCv4-EIC7^0BVIu+>Onr5f{faWjQXqBG`_Xl`^Kjp-}=Y! zvA-Q(`-$*o7vOu}9Upwjs*kz$k2((u#}z*?5&H^0d8w~ME|aD6;Cl(#-;gZi3HTcq zFmLw|lVTdrV4hr_BiPvejVIJM5Ppm@Yx@Vn6G>kj6bN@CwXw72&V#x2+?(-uFKd|U zc!hJL_%7Y0QBRB^?4E|SL!b9=w8G6VSmCz2tk`St5KRMmgjO+E()&dFUmcnW|J{~! zmQ3R;d32n7915c4oKaWF_nGvLlE3CZzQC@VQS$qhk9qUnsp4!|fuH>x^l>`Vb4H!o zv>qF#Ny6bQyOy=l8R$8;E$v4?bo2g0Yc-d0cLidL+`OlhcbmS=u^)2tmML$x@-prF z+`OMC5Bk(-18y=xJD z)*7`>3d9=e*HkmEKy0#7`hAU3#&NaMiGkRaq&mmy8}~eWYTaY|1b4dUSs!IpPI961 z`CY$cn$ELJljSX=XX=bzU<1#{T+WOiO{L}F#29d6G`{_Bn>z|L7fsCHrD@ypn>iC) z&)wXL$yBfNJ8PHkj=!`mKDBL-6^mEtJd7{Ebn*NNKjs4a9ap}qdC=ZLdry;X^ob`;-_vQhe{84o5QO-!W zdh=_$SoyWh#F-L2AMWe%1MkKc*om!#z7^Qn3mok$<-Oa#LKluShez-$a$qITj`a=1 z_Vd4)@vD#AAv1S1wvuhEYh50^evb6#Df61Z5kto=?9f>)wXVtR&{Qzc+8IH= zRiElodri!lvhk1XaDz1e>nfY z37ygTU*&J-o$`I0!&Ro3vvX*7%KQFBR&Wt#<`nMAUBc6ebGgpgojI3xdHA`jQ#m`I zmpAGqi@S6CdKSMp!9AC&&LU5K6dsnbJuO6u10W0eM(2p(C+~g(~y|``Ozhu+XWBrgF^{;&^qK|TOgQt zhx`Og_<@OL;Gw;LCqKpu?>7Svf{|Ojxxhs3KMfP}T$o6#jn+!*ExC)C$LcG9!(vyyQIX@!Ve?omzcLE?lEt^4>$#%n$~*!hw+*HI-Dv=nm^5( z;9GMzio5OV!JRt%uY}hrtl?^CaI&62bPu>S=k%k{k}&UOTO^wy|98ozQ?i(11_i+x9WGTj{ra zcb|m@Rihtoya!ksAR3)--%9+7Em`!Na=rzx)@6VC&<@e|{4t_SW^KIgH6~D{yxvB4C4OLxPZoTpgO~?{7 zo-D>AdfuJ)j=n6@PUm+VU$nM0SA`;TJ^scHddJyAMn?`JH+r38HE z0N2&f9<_Ufy+kq(&C{MWSA-sUD5Ea9{Vr<1|K%am*4h+*ONJHRlVjm)=d_gy&A9k#;w*2b$GS{Cl*NQv!=-=Wi^qr}Leh6+VSd%Da z6b~qZY-a}R69G;`Be*wF`#}F*q2~3Mhnj1x7{)&D1vvjT-)02^2cH-j9mPE%FZT(F z7VKY}5sJQ?8QT9c{=3xGJlhkF-{uK7%_Y6d6aMrb&u{kM=lM-^zUMd1Ri588E%E#& z@~G!G$b-$En|+J3=MJ6Xt~+qlkKfuam`2VDM;yVEpQ*(kP9pHkI#rTae_e=3;6`WF1ina88X){(iSu3gN{ZpIU9 zaq3b#ZNT1`IcFnkm5CHE39aRa~i@jG5h%r)Rk zeU9|E{5yeP^-281C(9;=_5%Aq1b+K4F17g_ZECIr%We4f2(LeaR}xOn!ydj08O~8} zHTJ{ApqUKc$h!O&Y2|SzKF2Kjem$|0PVb}VEa1HK@=)fyv!;k=N&&8S)7~EXIuAMI ztYgFqIcf;XD+VVww@IGit zGCfK3G5Pz%-0fk_5_N3V_n$LH;_oGYpYn7$YffxS>}lFwxGK~HP2x_T@SaHyP12cL za#qo$LB#cW6ZvPuwW61u-q&5$UuCbU*l~#%2FX5O7Xv%8zb{I)?q0;X?jCsbsXUjU zgUsg1;ojyWGgAGMITgz;vedDQY~o&d$qA3l=SO+M(mpaUz7cy^ohN(n zy>Ksu`uYH0j{FJvL{CTFc$|7KVH}dDI`XG1BY)aQesZ2<8M{ngru{kd78z9?jsUm1 z_o%TnaNK&nxTj=Nl3iGmW1J$FBE}Q6QT6TN+aEaci**Tm!em>*k2~ZS*r&SRe84yQ zb~~vf8_lraHF@&u`?AfT?nh4849W-O*j--a8fW&$l#W`lh$W9YhD<`Sh$VN_-M^w8 zn@l^I_A4gMvIR5h%dbgunUf6c>f5oS{tHhS-Sg9&osQ-pJAj^#;P;UhPv~{z5VE~Z zL5El^8{?!qca!;?6}&r=0*~_@f1RPRNjy&ELC1DFR65rG3v_}%?}|Pc{-!H-)Dynb zzpfk{kF;3+Ch;`Dx#EZJ$`Ngl4Mm=fUS~ON58gFKdXb`l&lL55enJ7FbiK6j%*Ic=8KazphqHF{i`a$9W01Uqw|;Y>`Ul zBq6^Dh-vFQ|C73pkewLzma2MFIvf79RRmwFZwL|t?WgsX@50#p{9;m z-wy8P+W>w?DbS76VMV+_f+zkGtR(5XzXPny!tW!3{aPsfgM9y=GYg`uaSrhJcuLwt zoj0cqd^-Q<3-b#k=a~Z>$$meflhjJNdr&Wb|+}(gE8O`zN-y9FT;Oi z4ze6{g>#>|xF-LRs(hn^+yh;xq)y~2pZoFUP~E=s`qg#lAb|Ob`Dpyo~Sx*i%Pj}v4O+MMDc$^sZtL!UTJ#+K zv9B1~ZFM@BiHurfo#@(+KF2qbHR?{hUqJsld|+Mwtsr?r3|@i1;O=hdz`6XNZ)l}( z8Xk$c6}@)oCwVTe-bx;q-z~2hbBU3E`_evGNz5r?f{9)~GO`tW73s5*{aN=KJju6z zK))XR8oDhVPw&h9U(pxcZBT(PlBv^k=DvIr-8|y%eTg{yyPFr%hou!_y_tUJ}AK1tR6q}H2B1(0gU&aZvIFWycl?+ zJ-z6d_h2i(YdBwNzj%o}`P3gyMW%{xf!Y=BD^)q=U&a5zV(hK?T@OA+=!bCnI#-VS z5^X#B+oRD7<|dIch`;%372c!|I+n}Zxt>t>Bs}WX+yZYupdsdee(MY`yi+}r$;wycdCJ|zyGpO)4>KHKP((J337 z@HLLkdKdGrI__aSt}gkEJY2FaxwD}kdG>6+(YH5~4yPT>?RQOHu6>h9bL<;UivNH~ zv+V1XLKCK%{~7jVlS=pa^)w;w6~E90T9FFPz(;&~8Z-nu^ZD=$TT_D4LmhGTPzioP zHs=Xl^9a*Uraj!GS@tlKX4^wdnqvwBY{*T* zi47y)ZbJXGsb|};P);B`2RWFJ^I4huu57_sQ?~UW7Iamp&UbLZ4%xv~^sqK?he8PW zUT1;t!1sG4xx<2wJmQgjxFPZcZ+ zhX0GQDrep?e~EW*@oppUxMW86#;&u1@>j-2!wYW02cCrU#UoGpJQdpg415>buQ6uD zf@ZEee4dieD62a5xdV^tcL_~E zmfJixH57$cZzOJ0Q)#;6|2M;mp}$``3p(d4=aybn((55{scha_YZRk2yc#C$S4!r0;f6hxYG8 z{%qD>y7*B4(VK>f6;JFj>s4sofEd~E(z8Ni9X|8>Uc!?JFZ(sXO4d0CBo12|)B78FoT#VBH z;lSr5RmgC^qD}9G(D~iKFTB)I?(3>yu8)~JB3!%czD_LNy^^Iui~I7v=Xag>wO?81 zc}|_p1^Dn$KC+g3F>1>t&o;b{S zk-5YLx+g7U>b)yHggh?PJhxjYM$9{di}yHfq`3Hg18b`@lJ>%srvtH@peMV$itC2& zzZ=VCoT1CEjDzu>OP??9$OGSziH{}kLh?4cF#0O^DSj!PJ_$a>>kgqU;l?17X4nHs z&*QzrPdVqUTPa&WJ-)aX`^1VJe*PC@f7Vpm1)I`GN18ooh(jag-?zJ5{9nHP6W~&5 z5p}v}t25G&Gwe<#6)icl|6{`E@n5p_ z@uq&Qt&ix=Vd%~g=uW~fHRhfzWMhT=KvSJ}nkGr9^Fl|EhRHrif;bn zl-%fU?0as1uxQFeYu<O=Ub36B~lLkT(G`Q@xT;sKx3NWmXnXMS$k-d zm$gT(^PY84PKWHQm@#ObTITbPeoixE4aBA>WgOQkWn7b#&gSezDjy4dqx4I4XJ&WA zol2u@N8Riv=ks6oKXZV^Wc#0;S84p2cHEo#FWLVb8h}mDqZYbk>GzzNCB`?Ox;~uc z@YDThQ)^#M%!b$?b6#DVu=dv;_E&ITL^T~ZSZ96!{Av&cbWG`pcnF4(fTgtxAZH>Ka6g=LMe0D#Jmb- z-(U@S7fmBg=%@Ys&UNQkYf_p*Y$D=CHVtsTDHCka&fG*hlYlj~Lw|gV$3c7Bps!8n zq!p_{`fs(N^N4t(CQ@VqlVx|4TyLdp7N}Reo$3ED?7#T!qihp(Hz7-HqVACDrvJ^r z3i{z_iCh01w|?>9$@NBv1(EBqCO^|e#Eo7G(o8$q) zgGScM@OA8fH1;hAzmSd(RBkj`4m%eX@19ay$3wZ4QKI z7a@=2iIij0fDTDE4K4k9hoY2=E;BX_ON~v#3yw|0e%ju@J|pz_g06Ksmo>|#VSQGp zd7~BEeGj$__hHj8pR@{_25cDiW5W=w#irp2Y#Nqf)3Cy|X@K7ByuqP2o{0{xukjq3 z;Q0Mzi&j8;z-4S2vh2IuH4)qi-geVp$D>>9c>_hTiJz zQ}k9HaN#L|9=SUC1Uw{vyO3`Uof#Dfw}Q{`tzYV!?$=B73z+YyTRM8_&RbcF)-@J< zE4(uBZs6fA^zP6phc{-=VZPd?%TFlR?uC!^YT{%YUq?@e?%`y;^f^*{dC1hM`oJIA zDkayYdQP~oOdR6yV8uh|qBSZs{2y!6fj4Qy`z|Klx8^s^joAc!TPOPFpN2mgUsOTqj_6A7lAn+Mm_u2>_k9fN3(*0?Ckny2&HRCqb97yC%KeplR#>pk|vGtq<7VJ9&*^~bCcaG8mYd=Gt! z(5J|pl>PUvGXX2YsM_@;^+_h zh=SwMwH{+XCA->NQ?jt9%&Ki-e`%WO+SiH(Orjsd=!fR>K70)O3*NKrvs@f&ql|b7 zowGC-iT^6s+0E1Y*k8tmG^bBv$5Z`p@fE9x0oZD6cyjG`$HNm*=O0ObfDMnz{g}BO z#ed>fA`?kNR&wUps0o>f&L)!o>dwGpoJ|xTLg&g#aN^|IB)_dJDF#VK8hbMPl6>uo zkWtjY^FP?lYPf-CBu@bkc99JYoHH7Lou%q0e1^)F@r``Xn{*#5_Y2o@hPa(G#1wqb zC10$^A3X|<(Yibe-PKv)Kl$F!CDvuJoA65$9BOo$32xbZwyu z;A)tKEu_xwz&L$$;$~;qAH%n6tp?Hl7VJR8-wS3GlUC#W0q}b%y!;k)@`rL*BjN>h zx!(Eae7;e6wT-Q*&Sc>V?m*N$|BQWDbk3mz&!Qap*QYek-V{uZcksOjGSZ&NO1TGN zQg8GP+SL^oVZ77x>E3_ws8OgzIiE zd3&W~^JU~!V3(qE+&1pFIZ)okf3!3u^jTLc_Gyt7-d`m8lWSiOObZ|GqCMF@2=}JA zdADm!hIVDzIPIHq|7OZ{3e9lol+UG8+eDv$mjY;{>YYG&#ar^a_tYig#)e?j+s`2w8&1bmUr z<CcN7Tto>PMaK`wwJH{|$OxP>G#P5y3sgd_EM#FzGzpad~nD08qwpwG$v2Sw6 zrZe~v#;X2hFqha&&pV1ONh@XC*hbq`M*meNoif5%&CgQWyk2y${Oj%Fr3Lc3EP`IL2>dlOlolI25# zO)H&szMnp*e)g1xP3$Q;4;L|Cg`TJKSGlrN@j8BQPP81HORw5cm*8{2>#2>LCtfiz zX7lZ-;8T$Wyo>LE9$n-9FX#V<3k&NSS?`H%p26qTKx~}*Uo=nsX!H{Qk3MN{{wDi? z&BXkAezyGX3xF%xMCeSOVShTq8QW;;kw2p^i=)gQq|tZ|Ab7@y{}S;Th+ zm(m1Znf9Bm3^9*<#bQ<4yBK#N{Fh@>Li{Y(PAru?_5TrQiDWm28Q&l?PvTp$jh_B9 z-f6xca?1!lC9jlTWFL8J;H8|rLR&hHT`JF5zjsP`M{oaky<^OCPI*^gY)w8Oul|Ii z8<~ZzSxc$nD?L0?e268zh~f{@r<1z)99!$_(2gAYHO9SjdLVobvbZtZBxlIPzdaEr zPV(PM)?*HJw*7v_nnrCk{X^#DdNi*!tCguK5(s{zAC13BauKPcP8?q7B zeLT|rC1NSROdU_LXNd+s&DscF){&>R%Q5-HZ*=o*@`cAAkiVKoG$z8{eiYjF82i^q zo+ht1RD9@)_e0h$&(Ea8d@XgQ; z=+YwE*ZftPG{atCQtf4D_H#}6fvcbLhdRVM9cEr%#sBnU;^O-P)sx@qQ}JMoG5Fzm z{i6p4%RI=f1_n0u_x)sY*Ppy}caf*-+?tnuG}-rj=w?<0FU|5<_I z{{ObF9ffaW1^=*}75VGSb_H1%^CSN# z`9=*WQ5*vHO!#fxZ=rD~>RRX4bzK#-iTYmTzkF77zB`I+kQfc&_8F1~o?5?i&mJ)q zX-9i=Z_dLGf0Sjf1O`7^IA1ouOOs`sqT30X(=)t#Mz%2Xj4XQTss1{7_Dtd!$Ty-Q zA+Jc3J58=UjdIlo@S*T#_iJK&BN(6V);I(&m)<+A_OC9?9b%o2As4xq@j3m>wde6$ zdfaBy(kOXU{VHiR8DTBhLC8Ib%yd&Jty*#pcR=Gi|5E=3PsBv16A z@OpyveX)DuT()-on9xh`lyTtk)BaXCy519N2FJ0T4xwLKu2}GiyT6)B!<GwWf-OuX9&dT_Mxvfd?PX543mU z_rtk?_OaYe|2+Tq@Vv!yEBWxKQ}^`8=3<5w+dJ0bXKEMX>zNUFa!zp8lh9hbg0}{;0e4*B`7fOzsV|>7~>^GIqofW&;M=H1nR{jo+)8W_L^5;wP)vj_h(~IdCurd#%yfwEWJ* zh!|C6&HFw58&cPH%+~{-GWzA?`qbvggvw5#cn|q%Fx$5 z`^32p{aC_#@vd8lZ}@v?p1xPyzUtHVB+(f9m++NXny_#6;_orLEE73%Ct@k4V(XM= zze;RJ&WVH7);3o!iteQ2o4iGm`Q+K(HFL-wMq15!RrRpu#TZj1a#GsbK8LYIfd}Sc zd;0zT6k9M@m69HP7CkZc9YM=VtA)>qdibxn0GqveZO?Co$De}E1h_P3$8ns?T)A5X z?Pwh6N*YE&XKHS{A{Hs7J!Ddo;J*XN)ea-4SiQsZegSuTi-wO-A0^9WUFx@9ze#%d z!_2)oo7bNmY%LdDS_f5ECvfc;yvoO%g>>KdKj2N?g-*{R9Y+0<-K*|SZr$fnx8^9u zzG}`M1CR%hhwbhuZR!5tW6YPvSi;%}USr54K0z;ENqonuZ_^(0FNnb zdwRmU)3hbW6OJsgE{>xEjTE5IEb@dMxb7bO{P)mc)@KLvuRXTjdqZ1foM5}azLs;d z-u;1h&Y8Wy4zOo9=esO>y!-ww-U}YIZ(L&b6!s?UYKadZK zjx1n2n#bzy$iGIhmdxAw+5P?Nt1jTajf}eb7lww`P8#6XTH4;+wzn5%)@fc3ukD8Y zZAM+QJFkz^kLEQ!n0w~*pP0`pFLvf?5q%XMF!Q^ud+;fDe&>^~Ia<&BYHr(I*!@2F ziMee8cN0AA7w-EzdEc6VYv#9+GRW)j8})?NtVwyF``m*^@3BIAnG;8jr89XpbB+F( zc`YnS$fkuGnf5i*r}@?#UuAydt8UUfdpzmu#A%uH?+-oc!uq;V!2eGC`b0;%f&+s6 z)8m~8pY~%b`6;%@oq^+D5L@E~VxO!d_Q@l}K6!NX;un8T+>V#r*e9YnbHSA&?kgRQ zen5IX!8-dvJ%Ya;e{5EVJrsVrustWuSy0c_PkI**ODl(t^Nk#rL)(89xM^Zgu zg9DEO+pKq_7Cp1@;|VL)S_{@c7ryB{@X;#lw+-CXJ73n^;YFP}!}YbMKGWAClsU39oxd{&1Q+r9&*wX9pThhJ zHw3#=%B-Oy%Q#Q*o-^6HvaJ5YKRmx;C^qfeM=-~&;MidDYc9S+zKz2wl6)e8ou$_5 zenIhKCFzd*Ei}fn{AU+mFkCSH);4Ro)=&7c`fSVGz0Ei4EcU7AC|6A$`_ww^Q`#S< zl{)?V8}9^zt>B|%v42#Z#e5IEyvOf@^k1<%4zqXo=;w2TvB_iZ4uVg+9z{0V_nY>R z-ewO;;M=pcLqop;*D9%_9$agLUt2uT>B~mibMa)I1)ltgF^Sgv%I#wUPcCxtU@iFx zJSj@TlQQ@H8r~=9QqI4KBU7M9Mm`0e)liS{sf#-En>_$tZ|dT->dW?Hmwm?r7i{{G*^9fGz1Z*K zh3@g)iN4@$+UiPMcXV0!WC<~W)K&@j)g3(S;YoXE(wz@J+0*J-yO@|j#2uLQFi&UV z4+$S72W2k`z672f;ITM6R#9*98}X^1d%N2Q!PPF*^*iwNJ@E8>@UzR=qh9KD#o`x# z`|!`N9yrE{AN)Dx?tt(4+}o3Hdf4X^kElzRMI(R9cLV!4@rOQVEZQfl(QEX&|DsL3 zx`@B%WybU|d!6uEFaU4$wKzQ-co$B`!0D&>EqrcU)63cK{snv%4jX*f*3;nUr{J?- zE1xw?rom57cJec|w%~>6at(Y#f(8fO_uY7(fURor8f&cKzGC`^ow)Wn(c?&5XJIS0 zyMqoZ&Mmq;U`c1{eZW>j$m)Qz&qkl5!%_1)+ioJA;qoP-A;QxDFfa?+kSy2G7{E`% zL*C1~<+8E67CQ}i)6;oKKd}s|&$0$5_(zq?Hhd#EPkRYk@HW3c{5ZZrv`zf5+ApDP zGmmej27ko+e?aq|$r%u=#z#Trr2E`LnPgh{q&wG}DX%$8;PJQJxeSx9{%fu;WG3 zPh^)_hWxILzKgdR2^=Ud(Vr>2JBpm`S>9o%3C(uyr`k>XKP+-$_^D2Pdm-O8u9^Vv zmBziZ=#EA(Kb-ZX_mS+Z<{3w~%pPaPu5WgtXE=NlutL9AFz&A$^N=r(d0)V@{928f zK5EQoyJP+~<8U~yb1JhF8q2o=-{2umCw9g-+2NZOXC@MljOZix^(d#Kk{^k9(ITS=p9}Cc6@5p z7{?E~7qqrFWeyNmTy*brUS==6%s!Ww>E+T#@kg_u)hd60=W&nK(3u$WKL>8!_4KOk zp6xpW5^O0h_fSZ;)q?z9I)2>;qdPO(k>4NX>?1rj z{E>UdP=Q6P0rPaDoLXPN zZ%?m-CHjqwhP~3nQ9!qE%G?Zm)}zN@Kb_~{_h0&@PuYATbzbCWtMa+`HJJb{5j|=xw_@s-6}%$0h5U!0?K=0U9PqnAw$y^t zM(?T{WpgQhE%}>jWOkA6}&j`OqKu&a(RW&>?7E zi-!ju6o2wc&ufO7b)AR2B+HSPG-a>K9|aHm2InjvGL#y8Y%9gH2FHgL_sAmOtaIa8 z+!@i$wpi;~-K`BKkMsSpQQ&bmYXh-kV;j3-w+p^n)dPZu$=A7G@EJ)#kBRSx?pRpg z6WfZEjA%Lk>1%o#_FzpZ8SnUyPunr-a_is{#b)F;F~ts+@>{V?f9}$qV%n}o#;bJ| zom}PS=aXOTvl?U!?38enw&!tY_1fy~{_bL=wm?MCtCYTll+xc9NH=q*%_^-wH1hc+$fCdx z$8L=HOVG50Y~tsXEpCsSwSFx+EEHaHNvPg?V_W@_ytew%l=pRy3_c-YpS5^$gm^-V zU62m#Y?2I`J0*55sR-@rIW8o-Ys0G*$=;IK8rb^-m)vh(-6x}S&5fF~Md}~0xtZ}H zv+j^}N1&0!rL$ka9w4-KbZ9NMOD_&95B;*-k$Zne`};R~LQ&dmqRlJyovUC4A413JX~u7_^E0!@`YRODqVRu2wEHj;KF zU2o{#QXl8s2r?o3;bU_wbRw*m_C@xXrPlSmgNH9*+}J&|gO8%gku{cZ^*yUwN$>#g z8j&BFvTePCsmzD;FXsR9K0&YMhjTbG!KLl#*09RO(3LsXz}5W%?Kgv?hpViZDf>)O z@E_Ery}I=hxu-B zN$_pA9`(7EZ;0nN%)DzW32x<`@;h}M{h;2N{7vTse?z{J)oNUw`Un5~ZO?Mf=u7?3 zyE15-ADUN&j4=&c@P6zS6QHjX$-ff%dLeq*?&xK!Jl*HkLSMT=U#mS6AB;j@;}=`= zZUz?vovaPW4kvd*-w-7(-bCo^Txe_+dfH0%t_jfL3FvUsc_$g?v)HbVV%#^Af`jiR zY&K1Q-#Rboab*DY^z#$esxqB0(SFfb z$-_0j6KP}ag~)99Kg*+im%1p^pRvwmtamcje`l-%7;8_)y4lln?vEL3cgDKKbJc^N zQuf~&!vXe^KY4nlzvSuBZ?4A3UZQbsroLO!#y%MByDGnBrWM-1R(GUL^)`B7;Q zJczBQfAU?D9$I!fM`lb4{Tq3Qs{*lEq|F1a3srj3?I!*cFWKm4;N3Q;?D)>?g)fs2 zj~<2gzReR^iyx2D7to_9ozHuvYuGcPzYCZ<;$gUR-TnkDuW6 z$KwsYRkNP-VS6y1*oS2cI#2PHjPKg@&cZ1b3*Iq(dY1ODMCR!9^A9G^LtYs^*W2Jv z{ODzzcfU99I15bG|1#>2+=mQ);Wm?hGy6mZ-$e4P2K57()0xL}M_F(*@bw31n(Q1d zVodlx4@s;IwpIAeVV^)eEY_i4GES~Kequ#6whT@EufxQbHS%Peb&8}}8+IVSeuX_) zeM1j1cmluu`p-H=*n_oB^`r>e%o0c>W+G_(6Y%9;`aYnx`?A(Knyhm+tqL z^%Kstd28~U7!yGqhm=TA8hFml3cf&Ff&rZ!sunu77}x;} zRv!*Ce&U+Pwk>qlYbWbcf-bHZc&j)o@TC74M^A_hu;G2mizkS0O-C1ojC;~pbYbP_ z!p5Kr8;vgPxAltCw0TKRn|%WPJor|5B0fa4>^b&2?NK|Sv9b$B<}|o|$@I3zsss;d zQE+Oi_=tV1a|}3X8ZaevJLiM_^y3!hHyPH=8m!CiuoZt?{kks@)7hkH1$Ih&w*4r4bnb%gbyesi>p2hQ>HcWN?dSKlg$^7`ru8$%Ub*l?2S&}i3=>Zv z4V?INK(~N`P%vbX&>2`h5a^|cVc!&d@4llfJ>s@U zhBc9`OJs!mp0)lMLoXv_>(L<$qd+F58d(oxSRy z&$DW!;+{m7(NFl3M$!V>kdKY-C3%y6D0XPETSp_~DKKLX#PXCiG|MUUJQQe7nFo-nw@AV0uw+PE*!v?Hqr;h=tUeQqq(jK;XZBX;M-yv4Y9M<(`+p7gvF7_lw+}Ca z2SDb2w|suBw19Jd_vED4KqD@G9yRJx5Ps>W!@DEa&JSCRaGA;XksV zb6NwzjOOE2=1lxqmg-w(g)2{RKi$9O_o+Hz4fG9MmLCC@t5Q16)45DzK^{EUuX7pj zwU%#{w~g3_MLu8j7Wf?1F`l~6gKkgEpLn$6XUA70<~G5X5OXCYUwiH$2x%T$9C_Q) zl0CUW@%P4FB>`KVz`5JdQvg$uNr6~brRy2T@$wyM@$wzboc$(ve6_8}U@Gy^lWl%8 zF=Z8eM2(kM`*XPoIb#l=#26` zoym{~EFuoD{@)?|IF@=ajsNmnOI{0PK{=vR!d(+Xo;mVy|A*!%GK@81j;gY)ySG5Q zSPS%A){yul)^WNOTd2B7_20~wYuk{fcvxxN$-?>E^mTuJ@V+-4-O7#XE9EWxthAr% zPOUps5B=9ZwNW1GV<~rokJ>jt~*~h)3*K>YrZVs!QG5G(SAnK8jQc%nmVK8 zB%M(|V|=H*tvBTyd1Icfv%B`igN&~OpVxrT(7}KIdwd17XL5JupLIRB3w$tqLvc#b z!@T_E*UsAZU<{(OU6g`9!hMV1&tvb}2p_X^6*OXnXhfdOtRP&c@c>B;RU z-b!+^(ZDNri!j%{4$59a@LnXj8ow)3eAgCvE}6%9W@;vP(Vqu?sSe4vE7i8-H=-@T zr~DQpoLvk&L7QsuoiH%;OhIsYKW9#UPk*$=iM4rAbbKiqq+Xr^0R_{H|b>Hl}I+ zqK{g$R@N*rccN7X&T{Dn{hlLUjd3+IPtCJCVL#*z85+WTMc`GN6K9c23WF`^5=1|S zGA{9Oacl|@bPUCQaI*UGGHpBj_IhJeY~}+!YX#+t**^>q`1;vF)p4}InWJjn=^UKP zx3lO=vhGH6#JCc=oA;7*H{83LZ2yyGzX~5WYX&;#giR{C8*35sOWilqhUTL3M(7Ud zEdDP7_6$tuY{;C_&h{q8tTRKD^c4Fa__R~)e=>~j_HO#DZ_2nMbkt?|T>>}Q!Yt)H zyHxq|UwRh1BH6=;mzhA_ccK?MsRzw4deCHh82o_0(H`ctF4j=dA$V`axWo?D@m&;t z`>>4`E$%Hk2`!FzJ)t&iPd(sEN#;e&eYXAEOP%!-4as3H+L3LG$2bPQsh{Jyx8)A{ zDLd$A&vWJ`oiS)$<}!xrGw2TXV?X$>*fbzlMW>KqY#PL$Xgun}-|2&N#%G={M{oQo z{n*cQ0XBaBitc!`=XBk1M)VE%Nx|W@OolRsh(K>1wHDbT|KeS z3R193N(tA}?`Vc6)|}&sP2`weB>xoo)X?aE+u?mVoLb4N&J>g(*RFw{}jHR zo)XUGEE-)uEfghoLkqDR;+w75{w1^pZT$36?4fGOd%|PlLVQMln@8k@ns!iTE$?3D zoAsV>^F~kjG5XN7rU1P&=kR^lt?uCcUp(P@#=-q3jvdKv>_|3x9Xpa0P8vVr3A2x^ zvpyIS98DdQpta%=9%9}^3niD7Jg${}NP9_ag7f}f-aER%Ogo~zMsyUuKydxJTV^h0 zBx_UN$CQbq6Rn^PokM-t$23YOy0vq#vCff`YERn3xSg0{x<4SxzB|=2b&FPAO?^6R zUTM-CTQU&sD?Vga8|i1FkMFtdPwZ$v)82eqAAjeT8ABOo9NG3Olxg%@v0b}Uojy%+ z_DLsRy8Hyxrvm$~9Ox)*A6*g%KLbzk({oR4f1O)*|D^U;o!0&fZkcnE+W*<Ub~NVJWzL#D?3RfScX*Mh z=Ru!&cQ@tsP_8dyR17KY+di*_Py4F;8b4=_N2i8%!B6a7p*Sm(xT8b1a>VU>D!$@M zz7g$~4T$K(+Rh7~6upuyNH=KMOTeY0ct~_923->Gwrj=|^wrt+ZfM4h zS37v|OXg6x@H{j~W%Sz7Nb8oaVn5nS}fhW|6lB{4`~l5BVKTBv0q8kJ$I* zmtfw3%VT*rR_}mcjLtgV9fc>AAIhz~AF20hZ-C>&@j3s+OG;1fJ3e~HoNrsf!N{AvFK-?L%Wm#y~x3^TXt$@SixHp!sYh&QIbUw|V9_mlcQkh<3_ zk^Lj>u})7fuRU6)Cl}27Etqm+)&Fh z{;%+aeXZ;X?{a?nFKhWbeVw(oSWD4+OZ@!cqF~Nu=ls=viD$X?fX32n;?g+u;(m8; zZQ)&F54er-Chq}vxn-Iulh^~kOBwNhvIE@>-QNS9ZJiD;Tj$XK!v)u2Ba?0XaYScV zW+&)emi;dEh~E9tq}g^A@{vx!mh-KAx)f_N*W4SkdmONgEHko%^Sk6Fs-NGxmWiHb z*i&;o%Ow+#96W65%d$6;wxfd=@1y(6*Lbf(A2+?Nsg_tL<<@Y+bE}?O>H!w_P}g48 z`4jL?c2d#5CVabyx9!&X9BCyoXvrwB)q@8L@iC)~ z?{7hRl^33vJ*EJ9t1&LmOt`N36IR==Tf{GPO8ZM4N&`*;|xzzN<$sS%eH~ zr)(?=pwS82iz3l$*Y*OPWryuWU#HB?vc16nJz;yX<6B*g?M3BR*T-E@w_k@iDvwkm@bm-pgVF}&)*T*1D z=-xGFz9id?ISJbh_G2HqJ>={_~W#u=kf7 zZZ&II>gN9^`Hjf*u-9vlEyh`H{!_|FhRr>&4dTOd-25k%Ka70zRHBc)-TWUZe-L$} zqiVp8Lwyx~w+^t7NfYx2`AdVb=g7Cer~Lj-zH<+2o;{a5#SoJGdI{%g*>z-M*P*`N zs`7oE@@v4+2HAGd*Y7I7)U99k9ng{+RL=mXo^tXUjEzNsJ%xPrV-0v}e0C`F9DUr4 zUBLPekzM1@r2O5fm$N72+rK0)a7tdD9U>2$H05^cd-^7O?=|{fGI73F{x15qyA)YE z`4%+3aS%SRX5NV3Xy!E#8>VzrAT~s4c_21O>9{~_pi=gU0i@;dfE{(3{U+0{p-#~Q z)p@?k4ueNfI)b}&lu~E0QtCWgDRmYqrOtd(opl@brVcab-AUR96v z1)DqhIf|ZxtM4W5_Wt(a6GqqPV{a_#9rz+3dPqCR=TD)|$<%!2o9PEjRrO#GX_N zuHSFcEPE5{QJIn#-HmK&eEjkq37^Uj7N*s4KT@=e{3iH}4*3b^LMM(2ca2cbKo_1K zp+52L=)oISFY~-V^Ku8ys>s)VT}*zZd`0GsJq`^yULju=XiUV*xnW9bs1=$8CWRwj z>(`O3zP1|rK3+bXFQkQ+MXrenF(T_>+x4UL5mKV6y!#S5+N}hPzH0E*wV=OI6$G=~3 zXNXrrf6n3k-(A>w8G4n;+J9*sx0D7Kv5r4sZ4zrZjB<(J>eIl}{`coUu)967BM|cX zoV%}f0e3n-bf!(g&NA9_&PDVGn7Ea9C&*L(Qi^rQMD_MxQx^SBuq$UCof{H&xr7!5 zmZdYE<`=MGdD#p7BnJ5!%Z&X_U{W@2f{UHZ)3aM|*z_i6qcIKV^1jes2P`=JdWQYQ z5S7ihKR0Qi{h3Mg?ER$IG2UMG( z>r*mqY&f4x2hYc2qcJ|;+At`e`}V+(SROIBc+#LJe)LEE;0-2lAGaUPvk?nR>NaC*ev8)8}u&3TqbODYI3X%b8_I3dIz2?<9AaL`=!Rn9)=Lq z?AOl%L!ud*$QRCNZJu%S)|00?b>I5aneb!qqybXnyM_L*Kz_>btWQy1v8kSeHmE#2 zc&5^4NRL88ROb_vQ64lb>cvJXv@owuGVmfRE!vn9h%F&D=3^)1TRqkIQ4d~_WpJK2 z=TnbjyK@rf6@x;)`S}Lt2fMPJ8u}_c*#gh8jrI0TJ3q+1YD@huofjPHo&QA{I5{YF z{ud_Cb6!yQnn|ye!V`GW!L|FSLwdNDC6+m-9xLtb@1PUYt>g07l#i$A!J*so{YT-c zTj8lAGk9LWCwfiSkYu%rtu$t=Q|B$zDVihOS0D0%0#Eik21i{SzLEFZUw5(g;xBfU zD(-)#?Zs|E=fCmfosPpJX=y|?_WiTBN^k{MeEHFsrwx2+7FxB?p3C3Fn_&CJU* z=MI2l@L?zK0m!s}#9C{N@-wvdvOMPawhwHR_J(n z=io-}hIxhdRuyAc&Az>nv;0Zh&{X7qipMmKZ>I9?6zZ5vUDvXYU&9?Hor9loo{T{Y zBH&trC#Zn06AUQkj?R+DpcA@#RrdPgCoXP1v0^X$(oXvC#9)-1QE^1_ZT28T^DDW> z`eV*vdx(4RJkPm2o%AqIdx(8d&E6QXy1*-RHv446>TIWB`z0HWP`b5Y)F5p1*{Ac1 zk?-_jKTEaV_j$V5*s?SS&a{6IF6)RPcNcRjS@bCGJ(FHDAKH5paha0q@W7w6GL}ys zwL;>H+8EZvM~6*EdI@Pp`q3{M7I!Id&E6 zEZ(IF8bk~S!;7qe-iV*LpLgOz@VWakA5!QK-t6)rmr=KTyjymRG5W>3)i37xdnV<6 zGg93zs`vznebDkili@kufR>cp)Hm3hGMe*7*7)n-Ks;q7cPlL6{Gh#PJa{z%yn5{X zzposG9oD@IvkiT#q0b$(4Y*6h*FWK&b0*VXfU~gpJn=FYlsk0gO4Db{oX5=C0;`%I z{0E+_M+UNm{>qNlOSx*!%V!+V*b9RHGsu|}@$21^#_eU?rjBm?g3mI4>(~zyb0>V& z?^b9|3%FRUzR>rxNo~rR?;h^Yz1z^RXPo|;GDqVLqS5MSWJXW_9QdPR@nGfF@h0YI zkk;k}U#N9QT8R5t!tsspSnyy?ThrSf1^s+N_aMy0}Nyu+qKIwDlm1yjea}@swIVfW}m-hZ<=vahosSQS|XQ zCe1bRm*QJR>v}_<`aq{LpjVmDE$H0*Y+|=`4(h%{`4P2)w~CRs8y;Bvi{k$1oS&dy z$n1utwIJ6b4#MlKx4mPGUvc(oQqsH5=@WQzm6!V;iM?{J;;&#=@m9Kt)nekZJWR}% zl-|=GtmKT{&Y1I=n~>H&B?H@3$A-8p)!GonkGv8-1%7Ws)61Pg&G0DS*H{Lyw+<6e z!<~rW${fQZOsVaM4bk`oIdzx9C+I$zdd}IpTS+|Ly{xzKZH4Ceh|>^(4(Xgf8ywMG z-NAcv_IJ-Sn#!1VBpq-4_k@@D^N0Il6N1H@FqM60; zD53@aF;4z?x2YGO~WC{3$FS7X*>jZ3%A=Dlcu*6B&$Tk=P(NuB#XoA=r)mG?OBHI4{+RntFc zps$!YHF{M~u$6nTMZfn<8DnBvC|=f4aKQ1YbaZw3#4~c*%dih$EF8g?la%|32Jcyz zRTn858`_HvT&zkqC)$J2cXhoY^htl@i^Ms1zC|ypZ@I&HFR5h9W^Ji!C;uPSSz)W< z?R92NyRf#YtTA+KQX06%J&j9J46f;{pu0)MA4rCrd`IJU)z4VoGijFnCzEE|e=uo| z{WhuEOx$I69c!(aiSz4#P}V>F68#qKzJs_)p&fRfOHFpm2E_Kz)x#;zM*h`>idf%7!&PwUAE4WYjQ)7?A{&=Oc#xKzq z&5P*ZuMCY+tmFdQ>)ucI40$8ytLTx#&p7AJA9j>wj+txG&_x}2idpIKr4J?Ypsk{v z7ePzWbA_S@y!b2t1JKmSJt-ma2q$AK=G&hEJ8H9v{^|ZH=-0Y4#oJ20e@gqgzPZcH zbH05i>814R7*QN|!H8mM$*=Pm^jP?2#^dT4etz)8iY@rVh)<5B?2GYcNdxW`PDa*!rb-C7{BQ7!UkVRgRFC+RKb(JOgt{Gan zWY7Plch3wkah*k9H73#5FPFI}_^eA`-yZ0kCv^sh@cWfqVPg&uB`dZA{&d^<-LwDOs)BSPwt=PK&tI%?GVNWN<@D)y^ht7uP10w=hg{+A4{wm~ob9>)4S09tADQ;I-S=VMJ9IeH zzL56`nk!n1Z4Go0JS!JZlFz)k_q~9>ihJzDWHvDe+%p|<6sb@2d!&pe3N0<{uA;S=Pc#OUygt;y7wT#zi1Ch#+Q#iKd}Nn zc(7<_eM*kOmm2WpUhria_%VzpkEaFS(oN9CO{CN4OZ@D>7Z&HEWLT_culI#6#-~>M zl+TuLg7*CL+`g-ga~YS?Qqssm`4mm>nvNgAT5u}jE$o+ORX^#6#>$sq3FjE%0QDP1 zjJaF*t?wj@6TVi0Yek-dwQrI?hjT-Lr=WIQNweP>263VAN?TN+Ej&{RC1=AW|H98>SAO1=%-{aulk?Gywd%F7#?txoPYf|BBPU4*5DJCU&9`3c9lI=eV zzTHTj?PtO_*M{uBG97%2UgXGbHW+-%wQnEo6KQN=7#3{S-@$u;&_d%1_n;Y$kaH>gYUoy{iAw0}dYd8=fv z&bRW@g@>ESJGq}_+YS7m|4;BNiTidFaR#&>CGHYX|C8~{1AfiIrwl**${Ju|Pf&Z8M=KNx7m6(8~d@nN>e9{?L`m zLsusqwf05dXmo*p*1H1xX7`=W1Yd&hn}&AC7S5+#jjQe5a_D8gJ%j(6`x{J}Yfm?6 zhJ6{S))P53!z z-k2+gu?J+?AJZTCVawiIvVg?6wXcYt>wgQlt+ILa`E|jXw-fwvIy_NVcq7j5)4Rhf z^?*O_iGIx++}Bn1;7&{nokbPXLU&edpq`Dp$AqN27CrP)AGRt6HahcLWAU0a+xD0= z$HxAY{9OAu=`?KD@s}JfIT1RoM13f$%A0~c{XFhg2}d)2n*USa^a%WVJSA(bd{7i0 zyp46^{5$2b_V^IyI$Y77I_xO6MuL6akvkWgQSCEXqx;p?pSWR1gfSX?oi{ODgMX80 zd-cV^`LxS#=iQuf<_;aoz2_S}Z+HcBV)5Gw3=H1P`j<3x4{H9=J#RQAe@pjZ3i(@@ zPwHA~Rh%6hkL|x;CShXi#t% z^;`n}N|#yBy>r`uN9l5j=`z)n*)}LRfp5Z-F7|V#55HbI$S)jG3>(d*kEb6nsXj!! zb@}?XiEUHeZ-kQlp2Wr z?X&pcE;r;%iSWL&C#zQQAbPzEnb>Y%O89hsFNYS*Wjvzql5btgBb-U}P5oA1U++7} zAKFLka8G`Eec!=;)?lf{dW?qt1ftwa0W8$-dSD0Rn3vXR@nnDRRV$-?ezx)$?YCq+ zv$7!LXDf|d6gfxQMKvqai>oJ(@^m_Q>A=TV`aJpWneJ~Zqr}8ML|=uQyIdLLaOO{R zRBfiv=1Azrb6XwzPOsO|L-Zw2^lUeGBuh4y;r=(V#u>BDay>fobiQ+9?58F2fO*-b zJxsYyi9G15@}4Iz&NvN!j0{RN%jo*JYq%YmSmTmMMxN9!>er-dC%#qpwtltmano#Q!(5Ad3cb*0oo$?Z;qP>1DsNCvHGb~eH*#6y zLgKl;&U~qC1MjfO$FA1OyvMbT-|&BVd-wP#t80(@nHeCHa3}YRCJt+IfMP42+LlYb)Y@Z6P$mf~ zG(a*WnD_hJ&+~*y1nqm?^LhW6&z@)R{p@G&wbxpEt+m%)n>?*0+=1ul&_{~#OS69G zgHE8Q-Rw**2k|uzPDY;^PL?@vDOeTF4^vJcIotzahyL~$!{0GY+rnl zFFA5QvG>N|Z!BL{#b(VP=R35Vy#uwYJk$2xj7SmnnM;EUG5XiK06A?Dv^ZP5Kh6Fr zeN_Kfkk-v({*fyxxFba}g%3Hy1{Xf|=PXkEhAWT@=ghDc`}tr0m~6w@_B?Q2a>Y{K z7hOw+`o5d@WAYpwDc8Qr&HJIrONpM(T&MGsU;Yl}JjS15jtfb%m`A~7w8=}eN0F-j0P>AKmuuZwztIXY=BkCiLJs@Q z##=4lz|*S>1Na?i{|T9MwtQ%^U71sN?qt}%3`ibu$0?${Gs;+z?l4Lj&_&awqVA?Mqrv~eA6ET;|LOQsFh$YBP~?~I5nWqiQuP|%I@ zY|cdv;hYljdXCKqS2gpFg*|5mKF?wNuwS_l?-3H?X@PRFMq zR?E3p{1$Xagy#4cDs%DRA^z)*)^%mst+M+VydKNFz0k`IocYtU{x9HvEA7eGY!&Z3 z1Kx0kzY2bF`Nzbf@N?f7aNk{DQrhqHLe3oy<%}`U2jD~bz8;Hy(EpNSpFRAuoSt_K zzC-ah@7H_2?hXCnLm{3m;&qPYj+BqH7a9mTh|8| zY3?}R=;3vdzi5P+n{^{2{rQHUpSkPlmT;91nL>Q)QoV0^K$7FCVA=dfoc^&X1%P#pIb3=?TuP;IR&wfBvVxPK@_mMlM zfo`9!cIQ%a*Tgv)&CNmPCFRz;AL@SVY5f`k-KTKwZ&NApx=B|r=iH2*?|srdZz}t> zlfNQv@>l=l`1T8ax-n^55B3w2IB(O3n5vPU!KnwB??cSB)wye? z&c3km=4&oIeDkv0C#Jso(=9i@_S34HkB@on=GHN<-n=Y-%+#y%FPOSC_v)!1<}IE2 za@XZ!-uvmHsgLkm!mo*6)_t3%j^g*{{;YGH`TGo6T;HEVpQ^e4EZ^62(Hr377Z2Z` z)BBF^-0^4d@3-J%zQ32j$3MdZh4;S(ZeA+;@RSQB-gsV+4fUwIDu1?O194DSjLi3H(Y7We1IXa}Cjx|nI7#adJY z%ub0X>EZbAug;tyyxz;&0}brnI-2tcdFUJ8L_3p4osxEr_e9poIYB%Bq4DsIm87Db zqqW#SzaaKH`@!|>K`GW^U4VV8fxX$U>JzOS*f)BV*wW~JRm7E!q$rm3(-*Oim>P|3 zu;K$<8}B`8kMY$E=KO%h3J(*l26{Sj?oH&i!b8hB-zz>o3mR!sEZ1uXt;yoq&-hJ! z#(f5GzMT518@zVpn#8gRlu0j9UEPe2X!D;~d<0>N$9VIVj++?k2;(zy)^> zib$KpfAtOi*Te6Q4$hNr)F$IYc{@Io$UVu{a%|YWqmpItm5LuF-hD6(E@t|w4#<~s zgca$*ylCzk&>3RSaS!-oRvbToO$9%}4`2;GR?nqI8#eOp4CZoBdq~_ShiT|dg5Z`z?R zn(seT*Wf~@GR^y2<{Px@;)3S;4KL4?xt=2LwZt|IemL#FN!g2(8G0thlH?xAf9h`y zvS4?Y=RQoT{+x<_{XK}RiT78` zJI^`4)d2@H?LO@|m}##7=HG~ecbyssTY4b#vF3Pd=>3Bo{JxJ=d_d!02rnE?|G&pr zTZ@Ug{;^dhdzNfIXSs8}g8GhaFw?$6Z6MFqz!QJMm}{1Z=Cj-H5t3~Gb@0-+R9`SU zga0!P+>n(Sw}UUV%^Xai>@vz;C9e89d_3x)*@ET&F~;!M>vLx)_DjL?`^TK1-)#GJ za8v!3E<4Sy$|5-LRu2 zVG|5;Uy(nedxhjPC%xRcJIL{y(>-;vsmBJf5998KURRG^-V<4cZ^is+i$chRSv{@F zqdlz3clm9`f3B_@_p)(s?Ra8Lsa}acxQIROnbO%`2Ipi8l>Ru5yjRiJ^T_+_AdMfq z<6F7xNxWk8=3M(z^0n3$0l#0nzS{fA+lIZn-DdDry?OOX^ybz7M|$(>P-nfF{DIRXr~CSR=k$WCg42m5J}N#GJIMws($-*w;Eh$X86aow7LK2=1?xO`s?B-9 zN5Dy2ewVVpq3dlLpEt_vf9-`HXPiB}^0zm4Zn z=<%ZKx!|y5>f6;nbd%;;_#UF$i^pE&miso6cilP* zRA-JA*WR`KTf|cN8Dd8`gFKzt);r^!ef;)wa7GuH)UADwF&1+D z2kF=821cC!F>vIbL8ZVt%cQAxhDkH*wz44htgpXUN@MZ*5a2AFXcM`vYA@qrqh=7k z?YAmk*03P)_U?XljbckJ+Vio4j4i9;z=)if(Pp8i^lw)zMUI_*g4%(UYmx%(Z@N@qTO<#0-KbSc^a9f5ltq zgx%o<@ICy~q6N&Eqnr8ceUEWJFV8>eg?t3xEbR*{hY*tu8Cx{)A#kc_ymBOVw8gJy z$nKM7e?Z>hV(<;Pc>3(xd~0!ef6hl*)}3pKdsT40;<~Jzp4^3J{BO50hsi|X)GZhu@imPo%jTqHmwoX z%(1`Myj9Y>HLm)ypXJ;@yVuihiQ85I z_m%XZ?Hd!?yD_1?8xz{Qk@o%w4DSy-y{18Ja-OE4Y*h~N4etCIZLU&&gZ4it5^RfqWKBtWXw9!Bt^|Vn(8*T1;+s?FR_Mx4Fy!)Ve_qjFs={n>5 zKu(PJKIgpyyw|{c^}JWddu`Oc5?#-%@v=LiEB-mzv8l`eKibf@gU~{?cmViR4Zhrq zy$Bs{Q(_Dw*>@ks0Q7vLUVHTzvuexF2S1D0l5nGkM30dt~0hUS(mg z@?o#48A@Dp-YL}@icEMt@+UGLwkIosf1TlVR|fFcVeVuf5N@k{54x=G`8&+I;eDsw zCUs*{#9B`KPtrd0G+*trZ|v1e97B+LmQV5q@(VDjW=s}#t9ObVyOogxSeq5oOM8-A zb*~LJ_Kln~D8*LRoAoJ7TD?6G`Dla#+g9E+Ya0CN2(rl+JP+rLRjd&{(}g$?i7!G5-S9QKvT>I?Q2Jr`qNQ91$pit;bQzM}ME>?qP7& zKWi!3vl?4VUu-P>cy0i%WJlYR_nWDe*ih2>&d;%<+>Q<9N^B^ju%UdD{p2+4CwZOh zCpo7tox0T6PxfO!*^m8XKlYRT*iU|k{p5F3?|1Dd+7r+{Ir)rjIdR7y0N<*S`BwrD z8(i@C=1Tg#w$NJqMvl3IaQ7dP_1+;S^PhTIm;bq+b@^X1tjqgkS&>~i*5%dEGd?JT z_6}qmMs%)mkg1VJ>!~0p@%DMS-)iU9Zo-LwvXz+u%H3O80zUV9|u4@Y34U z(wx8>l);lN%F5ibaM8G>wPlpmPzEn8zy`FTy==Tw=685&n00M?ds(ql*3IFya~W%G zd)Wl1EXCoyrPz@Q@x4}G!bKM?t>JtAUe=wm9VsTwDqwN{lGFMf2=tgj9I(|-f(Mr| z&+EYhJ#PRHls*L>DF4^sf$VGgzY$#M2`+46?u7&5ebf2>Ie4JEG#c|Z8a%k-{|FCc zNBhv=!9nogAb4;PJUG~n2Vb~&@HsJY^!dwyoz-o~A68uQElsfoRO0sX1aOi1bY)HdY3FDbHjbGXX!am5}S=6oZ+Zon@UIY9MmsBg|nVC${y zZB-tDml89*QfF`_e_hVnFMC@HJaQp+q-1OiiM+2%I&m*%V{d~;>_rBY-z;**?gxX& z&=+EJ8wjsJ?>QwOlFjWTeCRNINN0@lli@3)(HRqX(R1`!yokU^kBEPvESXqGtKc)j zt9SekUafQS>PZ)`*1LGM!Nsemz$=4auKiYc<=Jn42TlpM_CkZw51e^RcIS;dvEFT) z`Dbu^H+c31@HvPb)wAKg%e$RzxG!VFeZ{rmzJd++Ro8}l4L00szs81p>}zbeeFiu- zT<7i?-4B!3u+7+TdoS`2n0o9GIN0}&v+vN|THql#*vH`DpP<7yJQ}$rO{wO*od1HQ z;Lg1c2M-`4iqG$0O>J$&w%LZvFB@5L@8;xE&fev6)@g7b;v)8q)(`F%tsROzY%DR8 zxF4?w8(KLut~()PL(_=mE?i2tKX-joFJj*0OQ$n{mk7UD_cWiLjdK`z!nHJTjdgo< zmh4&A4Rh^T)C0d{=aBradY#K;=SZbYZPq|Xsz0B+UJ3OzM!&INIqj{Wyc^|{$hVfi zPVAqD-55n_&DPU;{`nXa3#d5Y#w*IOm%`%>-t!%eO}LKk(*f6w+~lmeJIBkm39h#Q zXPpJ;!Z*Ej*TeUDq8s9wX-~^rdmF#4?_}AZld9iZ6SSVlR-(4_uEzQ)<(fCOT{jUO z0Jv^g^6k)uveYeGi_^Af{wBTNqq&ZGiEbVieWonES+c?o;(i->spmM|o1hpxyU(U= zLvzre+AeYD?mfOC`+@A!(izqR5AlgWH^)}_d-8@cF6DW!KZEDz$TxZ5o91Hzd0HRq zHsJSI*gJZ-jQuAU&Xw5jgcD9&s9d`MI5*P=^+S8W4-3}xo4%TUr}gxWGPn!OH2x#N zN^=oB^;{ere^=1Vg?t7R=RrKj&m3&a=r{)txpVM9JB%~z2IQsMA*Y87&gh(2iFJHo zyPdgDRQjRrmTb*+?aTwQ{WAx3w>vZ^isd*XKRTu%-n`$XWqbhvzOAMpFL;A zYv(~%w?J3Gqi7tqX67|EG&6c&yA@Y#!rBMjm}A72iaq9gt*tFPivP8tw|?%NT~D7K zAJ1I-T6Y{e(|!Z+cW5uG-PU{sd0J;Y{e2g(mkm*S)S5Tpp=_U8Q#7wQ1U+(8b{FLI zT<8@!eYGQ}k9O&mw%z&u4s}&8czMUN`Lr{TGPUjD%nI@bkXOh0g%8K#+xk8FxO^G{ z_!X&dD-~Y^+ep;ypLrMjc#8M9RM+_GwC^($1J$8*r!Q|(r@Qw(@KLdn)W?0~cP0NY zH1)EvU*y=4PUAnqoSSdWVQ-1`0w)mDZ@={)<{aH`_3c7pp3OQhdLHe)2re7^xQa7$ z#6CN^#3D|Le^+g>cwUx$z1A}Luk2g@z;E#zc*w!=PdRw2coJP`-|+*(2Lc{2GYQ)? z_AifbeDW=)?P^l>MYdF}^}0Jvd|x(IJ&XUVFYi%LyhU`jhyN;5J)I%^i`s(b|D+VY z{YRz1|81r4*tbYGwtVwBqej=4N#~5@6Yt#(CSNAkX1dd}nWlY$%@jWX`QQJT^cr-` zk^BfeR>}HN**Mm!s+=}HRE{jI*x}>x{~<;d@=@v{bex$j=uP*le}$Y;2xL2cPWU2v zeol3cpHq8W5d550C;B;E#k=~pd@66}zi7Zb1HW(bi><>Kq7AuiU!l$=$mdDtIkN4u z(e)&Et)M+*?@A^?cH*5m=!f{cRLZ`kZxP4J@P-QbSQHzk;_9^FhpE^n*y%=nR_G74 z4D^K#PqRXL$MLBmekbdT6I%wH7z^!S8!dnjViR3G{u!-5s+(tL40mjzEy#}j7+0Qq zM&tW@n|_!$_eO7ZZNwI^lCD#SojK8ttF!*^Ax{V{j!$ilJ%Kia8{3rth zPJUT$ccneoFH5}ef8(2F`iX5>eaXWwE6-=epXZxj>sMEtv`hB;Pj5%A1{clwQurP* zpYhk>Od>GY0Jp#UxtCd9Y92{Nrbkk1e5fgcZMr7-!$o`~bhST?JASz*=@yN??b3Qx3bqaNp{uI7B=FS*Wk6cr@S`7rV5=SZ|yYxQPAHR zT^(AB@{NM-R;Bc)@%SoZn_8V{o06Xk`0433s}pQf+>M&x--JD?hBHEQ*>}{R{fhJL0!kG;%+b5=wAA;(SvpFZq@tVvAw7vL{1 zLVM!tvw_uJ%=J<5asB9UmHdlz9_A?Uci!iWI&gzjk zcx!eccrpNgS)^ynDJ z$0Ng$WAOc07x=}PD0}WrjCuWbd_Ax+>Ko^QHwM@EZ!=cqz0Ta@GYQTRv%-Z*{QdD! zz5CLlL7|t*($P&ajBZi~9E#9S*h7z3yia_}vP{kia(>l?8?jCty)iW$e}z8k+mcm0 z-QzdpjiLX_^TsO}t=4!?L9hP4q7O$06uf>y-Fj$Ne)IK(-J`AZkR7BGy5oM8acjPu zF=yCu=F%B+hJ6$L)VauZePE7tJh;d6JLwS>|BNlN;>U;$5e#GV+3)%bZSQG(PGb9e zMLFjkzl84@f0Hu&P4@SR#vkOorRO`uw^O7C0HYYOVUBEIe`boWs%ctBpOaWgI({Y@ zob^ey`?&C1VCF(TvKjVl{u{jL&bq;v+i}qKIhjG8w}$LwUK4eI8u+;06>LXQ&%@OW zfE)wtRQ@O@x?Tq*e5q7?d%lkULI_phAK*ZCvK zDn}FKW+z6Eks&vEa`VMC+;INk9r z$XoOSo;M>y4`q#$+*t?zt1C;d8a2c3wHlYnR#arnOXhKMP??(Vtg(AZrdKf=Or1RmlS+ny5sZG1D}_k_`Gly zt}K90g>x3Av;KB6JbY>|XD->-5`K;DW>seKYhv!-ofY1Dwv+E8-{4pmtMch$^t}<`DjVHH zG24;%x#Pm(4jTCVb6rER=PnHqe|EgcDOJQs-NZ$#pXfNOMD@-#sn;bcPG;c zH%}cRTa!jjCF7GoQp$Mvo8u&%$N|0?XjvP zmsXnZ<=W4aHqK_PyN`P4vvq-SRXWeoiqA#UMs&DYw9i_tSj}StX^XalH;V0E4&IDK zFRPx1ulFtC_&Dy@Rvny}D;rq{OZ?f3)^iWt%i!43G~x$=SBhPl%bCMg_D@y!m%ySG z-CJkOrF-k!pA=5Rx7Qy!ItLwmf%p?Xa;Z~^^~jmA1;izX=kMP`ToZiCG-la8SJ968 zf$b_S4qlq`4sr0SQ`#$$4;tOZK zIo5sud+s-%ARcscS(vgcdkT5Y?zgt_j_%@#ouqx%oJz%nuTWZy+@X{;w}>>GUp26h zu6ZWkiZOoHmP*Z`?s1b(OggqzolDxFxdksX?&iFb;6(j=E2H5ku-y({)_sM+yKmo< z(fey`a4kc*|~L+nM&U zmCsa}`sp|ArZ)_rKZ(2qy@T^|z1w4Pzg6JG8P^(cod42`Ta*F=E(c{^!A+&i!4ak4 zWuwyj!tq1qISc(b`D@SE9h(OoY?vI)@uhV)I`RVe!#yvr$wEhtaW1pn?`VY4kvAC~ znYo+ob9hiS-`R#gV(Tnusn9ZU@s7bur7yGA9Zb}h#j9AGjJ`aEGEX-_SLHsc1bulR zcYG25avd_nB=%{vcPJgj)JxQnktcglZ!S7=3;k=(huPVAr(2OM^yP&m z*euYMFQF~@zK0eMQd>Lcb<&UHoCC7Zj}`mH;0Lm35VHoGR z;>~l2tFlZqm2Q^;^CzJl&acOx^n>f-;pp2nk4e|onRntu)841>k^LLPRsX=g8zxqx z?gtiMQXAsa`oDm-7J{q#FFJ_x-?{rd&p9tg8=ug|r+niU^j>V~icdF8vCAqft5UX^ z_rPbtNp0dcwtEHo+NY0Aj&9U_6`Bv>Qh>gLmwUls>C@wSU|V4=!4FVpTa6xnq2J+U zQ|MDQ`VKPp3+mVT>@n&bsO6#x_Y7|o9kuK4=m5YtQGfqI zH>dtN)NiHVX6&pH-z&5#$M93z<=|dDeXn=-C+Yw5-X5j)Um6C&UxuV{w@Z4oVMvC# zGg9}z=+2J1_kCtBa}wj0EjhNGzUQ+JZ^()s0|r_bt42BW(}zCDW>~X`7=OYMVscAn z@N{~~|3|@}9q2#W*BlAmB<^dD0zRfba6g0pwbWPL6MLJqMcD+(KB7!@wFl$r^zSFu zH+qp%e+Bh-QfBlOv#3E-s0rnrz>7@xlNmwYD{j_@EC zcpPO8oB03YoI#-%T^*kB6OXz}Bo!UL892RCdtP)4I=ly~n_XD_p0*APR^ayA^v%)Z ziId>&Ay$&-z(X-)PuMzsMLsf5yWMRNF!bzhjvh}NE{qO{k;X|&X-84 zdxj+TTt>8EtI_9o8hw5{`uua;A@mp4{te*RbJQ2S(C1s`=RC

R+w03 zbb05y`zXt!j5B8r-%@(A@ho$6`M&NqJzf6yJlBB}k}GtEWF+lvarF63MxUQpdO`EU zo<8r;u=up$dC|~|jagtJKzbmQXzkL4}((At=uE@WjPp$U!`tSY^ zar^r?x=!ViiJ`T~xdZU0%jJw_Gd?@-^Lvrs3uW2O#J`MR!T-(0=vTy(XvCi~icS?J zwb3so=}aH`)d0?7J~Jg;CB5TmSC8V3zUGC<0+-5f2YFa=`0#6cLHq5}v*xote1>cg z+uS46@N~}*``J~`pU(J^n`9pd5GN-(?S!sXPHfFn=|(pWb@=Bt-p2>hvlFZ~b^^(S z;QVIF96Q0((7~~ng$}{j_XaMCR-i{jC!#A2b#$dhbS0g^*}I`@sBU|gQ0vw}=<_|; z5T5ph4t1B`r`FxC?v?$9@op@>YRfyw>d4}zdpW;Ye)D>Oa$x*?J*HPAJGxE|4W;~rp`8n#~8!c?!#^Gwo7qdb&f{oBbu>o z$tOecrw_9yY;=`DoTqos(LDH%FO6&`xZs#o`B#35g|>#@QhrZ^tAk0S;B6f;TznR1 z8Wv0ll_v?`vlorWUZr<`3r%e0`3+?EXE$FIDmV1dYteX~Rlb(HDEFdEJU2RA^&GOB zjgEQ)sdQHD^Z6+IVnnadfmf2E4cxgdpPECbF+bP>EBqN|ueoj-dpg9_K>uuT?F}9A z=E@7(;|2bgc{d#Y2dTa<-r2Uj+tEMAmA3(XO}b)TVK|=J|84ZS9Q!G^Y&(0Fv9d|gI_N{XTq?TUFV!C7*1OxKtI^+I zkQd8z_AdQ=16{4-Jp6EMJN;h6m<0dBGs1ECJ4DR5!|?}|GPVblGS*c}ksVhmWevWC z^kH!QL=49?dgfniZ}iOe7F(ZUh>9Sk-0x8j$a2a>ECB` zOb(!_xTn-368TTzB=gM z?;U~;g3EIw#=kVvsxsd>JSI{EAJ7S*hwxQrMhqN@No6I6mnx1yEXk@W08Z${ z9qwrM^kHPuot{iuhK+gNUA~C*I(LLG?-r>6ho^G}I?Orp3T#`V-8%H11I%Fq-+N}r zdC{LSu7|;6d)kQTGebs3xi}?S&%8DaK0VaBAV1V_yE$J})c{R!w`A1;&M*`Isp?27 z=@RtEO!OA~&5q_Hw?bpw2OrnD=K_3EiK`L`A9`?%8IWulv^bq z!A~wAmFzi!HdI~%pE!g68_EAyVCuh$xpsAe3B328YS+-2BOhLdtfslsK3g?sT&v3- z7^C?&ae2yCjj6%!s+K)8y^{i8&=^Fg_b?95Y7J|`@4j@vzUpG7^b z^E4lSbL(gxh*6cMzIJi@I-CAE=M&TIX{N8752vr!(1z1j&VIXnooK$9W&ghYo78pt zdbTOcwf`%z%8;9YC4-8q|4m`uc1_UwgTD_&90j`H~Lk%~v8lI=I6b zaNr>M-MJH$ezG<^MP4`cWfu28CH93i;7!Wf$ZG_zO1iP$mX=OWw#ugu;NB707xlg8 z)YnYMmJy?V9lTHLvDQ3z+^Xxz3w2%jc<@_qKi*i_)0|T$29Wl8Hx+0faY*R}?0Lm! zxZ|2js&>m=9{}N-(|;#M+284p$`tb zakh%@cg7LTzxsTlzYgrq;F}vMOPr4h`WydjO+P&MykZG6E_q|_)tIXcV55)_z?dkA6&NMETP3j0^5Px{cj3?V(OG zboT85}-F;8Sa>|)|dy8B_sqKY%UZ=MerpV$xkVu*t~#kAw?hwZ?x(D*>I-iUXdN11fK z&yc%6XT1tBhQ<`z|%{0+Zyp8f5v2|ShmF7nWB&C3zyMRh*O(cBOR(WL42qu-=|6({df zTzB0asCzc{Eu-Ia;|}`Ti$mIDqowU_sQZ~BmGZzugxARJeW-Ertwqd33VHr-l`t$F6*Wa1Ibf_~|H74>Fy7k*sF8LKB>8Fczgjc0}tk_4fngjWNWw=d*l1c=X>nU zywJusW&so0+kJKXL_0g;?D3g)vh3;9Z%!CPkhrd1*%Xy|{pYS%wb!}Nvjo$I>IraeRDYpN- zeDod7`DyG!C}r<2i@m=wd9=wqWKr*9#;1Kr!6aW}C(Tjn%}G<+u(>{~va1XGJj%2O zk!I><*u9l%e(=Ywgr@L~r>zXThg+WHmajWP983NS-?}UBB(PWm{s>2 zc-j)?aYsYjaNV8Q=FQ)Mwij0O%$l=1-^wB!D{X?a+S^f^G3Fx+{X}rnT)nPu2dsve zbZy|HyjRIHy66d=t9@OmmXH0uev!Ry4EWM)hvpxlUur+gm)SkP%vxMQd+53cE#fRL zrOvuDh_krNs>&tKPkw%Ghd7BXoSDqe&0UnM_;JKZRBQ$7?yNPzRD6Y(SXD9NAqI(Q zXyPt_N3YYa+5a43#dD8+Y2=u7d?O4FwoJrMSj;}%U@Pu-_C7nDW7l53CtHr@O!_kU zy4_^j6J!K zH8Tl0sC1Y$>=xFPFljY+E{!R2*4S%!pS?1(zi^nnuW39FL%)vA8&?%O*BaiO3%`PA zU#fbu6sL`SFP{~^i04Ppd+=2_J`CRE;Zd=RN0Y##9sca@_{$obO(HIR$KY|6<9F`j zQb@Q2e1!A&g5T)x9dYT+K9O+bfwT8X9P@K z(j1eYW@nj{_#bYXX42em+;bPcCWEho(yd!}q*(o8*ur<*6E%D8I}-QY!N($ZFM9`j z*(TOUZI|f6k`r-0VjGj9b#HNAgE$UtC0$1a`*24?-$};ipmlRE^j-%Yh62fhR(7?9 zFT|!(x8bso&PFK?i1ttw17sm^*q3=_XshL498??)p11bloVTw$CsxumV&a25I5FBx zd=Rrg%os|KTZ&mwnWb1D(1h%A?*YqL$wi}NGm;(TI_~UI{vPrV10NsxHOR?kpPDv4 z$-Odk0sFVAFWX--XPAEhuY4A|Zl^ci9~*b*?M>PdEhST5vYOUIY^YT_kD!=9vibS% zHogZP=uhjuM}O#cn><^Kp{YdQ1Mstn`wX_Bf49hn=F;C+-btiG^v-(TDPWHUoU1Ad zSm!m5r7w&p*x!n@An(g&rgirNrSKv{U&MNir4j2DoZUN(vvI6rx<`BOOKz-J_!hKy z>oV*|yVw&_ds~{@URXdIYV%}$tA$U>uOTr8#di7@#xGNSI@vqzaVWsIe+RxEFBruC zlk$Rr@Pen|b+edv=rb@3zlp)tutg!@Hpt-zBL_PC;7Zy^$qg4t%PrApwnQJ z92P4Ecaw;7QFvvDepKxRKK;+`7kL9-I^LWi&jJ^@i$b^v?tIF+xjJq=QhEk{7qpv3 zztnyK{ZfB&NkzX4vp83lYMJ%%=WZV#bLZIb5peceo*jOdV?W^5X%5QX}RDID{ zDv&E%S%0NGdzE*T2Ky z(EG?q=BzP%`7rWA#X!af3>T70K3a#Y>vQKJm9;Pt{wJe@lgC0SnG_N1$yZsohegIFcZ=zelvxxtmy!l(!o=?cD zxT|W7pS!#FA@9jQsdmG}XuE$BbF&vYO7|mV)7Jv*Bp&QK!!4n&2gg@<_}h_>+(Tb2 z@gTv?#ob<}zd3d?sp%)Scgfv4$Mhxjjr>p9VaZUegTfn&7(`0NSDJyb_LJfnzDM?h z=ZX1;u|3IuOS-+c#y7cZd^73wtxo%myepf#;q&c&Qi6462S2GS)_3)1KlSyUBa1Bj z8?E@eS6h|q_-WsO*v`Z1p|yJVyg5Af`MSq^#{bFqN#$ZM9FmEj6#gqivZJ2=N*yv| z(YP+nm*5jrH=4B)TCFS0jbaOc#&hh&XFKc14!*7RUh&)CVGe6}Px3^-jcfZ7c}@B7 zG3+BOPfnb1CyuGkU6}g7ME1fp)K}f$6<(av6X(nyp-gqXSf;WUKA2cPS^UST zzk;%I%8I0G;2%{zh4UP1kk|B_c;=_f^A*anTiv;rfAtR;-@kxcPV5Kpr!!6NZvwYw zJGhl?a4Xxs9asr=w~`)%{?yl7`2SD%k9dgiUpTLr@XV9hr_|ZHTIO#bIQ1#>Kilu( zJ$z9-qw~6B@ZMW@+VS3{aqv&PaGUzp3GctFzA?vBNX7GnJ4N#QLlz3Vcz>DdfMccD zAcb!q69YKXi9ODsYYTO2*fi!FW~I>IF0UCB=|=;w5G;)~Dm6)X-)Ka`BAwcFQ0pW4|W zwm7)HkhAyfurI`=KeSQST1ebkr~V|=^Y*Iw!)ebZXujTOJd-9^+*ArBd`>ND0D z(PMzRqKTGtjVsGON{VdD-D;fCN5(_1KE$4Y6CZmCzH;B?j+yUT!`SawZG}(7#}0lR4Uxxas^3@gGF9&r(O_oy#UrwvjSJv*57yD@3#F6YF=P*)^1TG%A`Ed}Gk9 z&L&pKw=aQS`-on_S<&kX#v@tb8)zbdUOUo6%TeT*2@bvH7<$dIS84ue|EHwpJ{Z1J z#{VanZ_OqC=4sMr%=!d;6$f0;G2($79gH6!K8A+hzU0IOsT)lh?bU5ajdJGMj%|d_ zz_T65B}Nx<1Mtw8)pskbSp3t9AxVXxW1Lwk|&=x<$3&3b2E{A zlL!A0{52=z$*ap$KgH8CqRp>#xvdf&Br-QU1E$ob7dDvSdd;d*6x=VjVCsS^ln=EVE@s}` z_0jd=`~n=-nTVKd-lU6o)}Gh|=HILX;8oMVFn-zPyzw77)%e9HUOM&on-az^Ty^-x z>Cu=AgQiX}s0R<~0x5&)usbRa`qlXAaPFrATo$-6`7hugUv9zS4aVs3nY{M+G{1G> zApb_gVJWCVa8W_%S_Q)Ti?CWF-)%9@wKDB!ic|v@slrq!qScf)NkhjdsV;`(` zj1!W(H>mzC$g{d6auyqw8cytTsYU*F{2O~5?6yr+;l&RWs2 zO!JXvU_pECTA}&K^zstsqo3QxVfM{}`i7Yk=Np=n6w1w+3*-^aLwUfNhezCb$aeeQ zL0(WSp`Q4nQ6|}@8QRr+>AzsyRWJZ|14`k+J_D09yDgRd1Xu2t?&xE$SYwqBy!3V9 z!>5LSGA_zh_PF}T{5BQ#YmarWH7d#2`@^+R!9cAWJu zu6K%gZ|1qsuX13xKq+|lZ%P+&pBHHhva;+a+Dm@M#AZ?qd_U)St%&Y7U03}2 zm?%5|Khmn|?WJS5ha|+lbk$#)n6G8yIoE1Aaqz#vr$IjKr}Ae&&d0~^hEedi5jwxV zcK4T$eX4#j&PIF|Uf}ziXB3BC?H-mqzj%I5cGCX)pSd z;L~8vyMJhGa?m_w!okmSi}C-l(zf~?x=?;Kdw}w1FnW>a&pK4HhR@{3@IHPF_|r?)mM=qND)-)*eFW$jKIr)|?8k?p792VP z-U*-Ym2L1D`7jV86kdG$0LO*yS(PvITf;Bv&+I-n8Qn39J*O;d#AfKFgqTn3Gp(80kC{h0Gn0F{Gx4{< zk2glVK%G;MlG?q)@yY1EFX){5P|m4mc08y43}bFV7J4R-+5GhO%q{iEZgs`j56iNe zTi{u(z_O|^Ty-tuU&oprN7o!1uolND6YNv)16UZyYnX@sM6~S6X3xhQT%XwreCs!M zjW*8Gd50YPa&Sw$ccEZ81>FvwajBa(&$OS_{3trU?$T(P##+Qa&;+-9Hs$hF)LNHF zN2ScCb!C_K82i1cCm)m-0#`NPN?pBkF?ki+XKh&*nANOyWrwEDZZ9u}Ua$_n9df@F zroPTSi+2;(jWga?n%FPH)Cck#;c?=7=4>)!%yr*8SMO0@{JR1gs^Gnaq%*XqJ`tTu zsn!_nrNayBkiS|7a&8_R?ESnd&ef$XVO9x6b>_MZ2+lXMb`7^YMcCH}!&dzp$om2&sJ5f$chYfX)yep}#S{DVP|42IfpKkqxDS zd&>nc>?Zina~4c>X#@L1!w&7Q^WG=QgJ0?0@$^aOPQNlYcXe&QtFxl-iMe@EFa;JL zkyj0DK2*y1J}_y9{hm_b_*e6sZoh0&`FYKN)}}*qmqUBsg$AeL=QR~SudZf&=;-q< z`33(Y;QZrnJN)f!+L8UHfO7#aAWJCr!TJKH>KunQPYM*eqfm09UQCte4WC!FT!c5_c3nzK{7dHhpYix*u}B-`Rg~V;-y? zU!G+2;+g$V_*3Z~UiHJnWx-2+XZn5xv~2MHN8KXd6I|kLl^Vwn-Epi=7>8)6eH{3D ziAGHy2P#g$sr%R$zb^|sXzycz)5pQ|Q8Xp|Pwbo8x?X)@ybDNIicgj$!6!MNx*(gm z%3;oOnY%pruCtN%qToiA)oVv0@9|dQxGG*EJP`7L{W%8r>!1$n`v%C2d$d_;Nk)c+6 zHumuX#-?>ZZDbSMPX0BVpX1(;(%q_)i)?;`wzKMZ7TV)u_$jnbel*++hl{>lolD60Oya8X+$F*KnvQNmoIhD_f`0`#}L#W1AJAn`xJLpu@c@sh<;eW zUfp(Irvfb4Gmyb+x`1iGQ$n6dvu^ zr6(=3;^CpC7ffP&C+=2Dw{LKHhR&b-4`bGMkM#CE;=yq^?Zq}uh#o2I6{<&eFzw2w zIo3L^Gk%p^0I=eTR`?QazM8{t1%+;OlTO`ouTFajPRv8+ zeml+46Vz8-?IUz9*L*!9=D` zI)l1XO~12j$$!oqC{|mhJ%O@`l<7Q$#xZ&^_`tn@;M*qa$FE;dz`dTsuTR9?9Q&2_ zG33}o4DROG7n{BcPR^Zu)%1Cs^1z9)N}=NmNPo|M_t%~YE)MQ$arHUb6?7^ftj3GOaId}l~>XigC7u9ly-rw+bs6Z!^+;SbyiV<~}ksb2v z?(|_y!<*NPL;q&NlvGCgrTgnl+uW-Z&uMF`)I3+a z?>2E~hxQgbzk~hD_%6Ta*85EL77%BWn5A>j8;gKV4mx8x-_*G&`H7#TtWv%f=o#_{ zh?kSzA-{=Rz+Yqqq=>Nt5W#e zYbGUrm`REMYtkHiEp_MGMaWDOkeP~+nZ_eCjYDSoL&dhhoc9DD?Ze7%+TtE1?fc%< z(pEXCC253on6-@k=&`m+ql?4Oj4U|Kkp&C66XVDXEB<41wm8Fngu03&b!0|3UL0?m z$$Spe9EBP41C(i>rM$VVGRQr2!TujUE*bG&^slX~$11C6VqQq^Ro!Wh3>j$RZf^P* zL$A##!9RMo)83ubVGbICN88%>rN_E5-`FH%zHhN7J=%&Jn0}NQ!M|zuZ4H*rx=xR6 zHE|k-p*IafR$Cq99BLik;5^x&a%4Ju5RJdE&iu|rw$MA=g_vgAdL}D!Ca^W_Orej| z*^4Yk*$c`W8|brmA&g(oVslCxL~j zd+lU1C-vBI>dUOE-yP&^KJA$APson+;QR7BJf?HAWs)tO`@hwesbl3t0tt1b-w2Ne zA^!@N^3E1N^mm0dALnY(kX>y3~0>UrFE<}1*h^eOgy;#pS5+J@=A zGwIHHZ$N?!8g$;-)WHXQD6xomuV8l~blU1G< zDWpu_p5T7_LB6B4M15XDpA~QX2>njinp4a=!MdAYY8_e*{_FWx>f~Ep4h49QeX;7H zxB1l%M9%UBSLOSX8;%WJo8!@hflK*mkr8}j8*7Tkk42L)je+(Sl7EC$e0H|Jj}E2( z(&^=oatz--@w{gRd&;C+1P`v7u6PXEYpE{uMPp^f(PP+jjeZtBJ#w7*UQ54BI!|>q zhkD<@t)z?DlUjH*={#>vv!H#^?GEu%_+Kgf^mY0n-9qEQUuO5ddrI~gI4{5#3RufV zGb<0AM~Bd!gx1-ujoBlmV?WQkvK@#wtF32w-tV?0-Gjauok(;S z;Q?!w%lx( zIm`$BjR)`-;EaAuK6ROnkB9Ui*&<`ypH#>GUfsad=(cB($!Bm57+w`COpnGkYyML0 zb*>$)iMbFw6cek(z*YOY8SVPer{uLT_JzpCl8f2sa_@%sd~v7?+$0PBk-DlEytZSR zWWhHnQ@b_5M)l{Dw>7aour%MLYz1XoC@YdofX-vNI!_Qe%|ho1E=rUeGwieABL=>- z|0Ls5JA&g&j7_-u0_pj*$=$eBEdwu!wh-gMv@N~oKFig6fZ4;u(ZhEp7>2KO@}CiG z>jm4VQ=`N@kpAP`-;OVO?CJDq{Y$L3g7q8+)-KhBOB^P!?yt zikJt*&OR|;UyYqTM=&)(86WcIz=;*C)Qh%cA zBh$Q_*v}>S6H*qY>|Y-~F8DaPQ#M+$GF?nX5kx2PeRT zoDO)9Za)ouICzk5=L1XDp4Gw22SwoTqjLIMqt?5!fKut8P4M|6oITS!AAQTQ4-EC_ zVPH5uf^R#p2ghC5OSYG6xUPLn;Is?-?v%9x`*L6^e^ZZ#1+Vw~NgZ05!1yids$S=^ z`IN;CKA)hK733ZE@)BVG1$pgl7}z`Qze)XAVp-1RbD z?timKiGMun?(`AZM6>y^K2`pmxTe-=6;nS@3{POL@*?tr{R>WSAO=vuY0;^>C@bZx z#bxwKy0YvD)1Vc}NB>Tq_F}YmHQCL(jyz}nGVP1pylcqQTEn7Uc?7xQJ$`pduc568 z)Y1Jgg_2`vC+t3NFnd&)_89m1UdOK9eqL{`>|)hPk=MIAc6A@$z`l&E)q`>YDiNm|P%#(RPX`BqLt@a)GQM~^s!&+y~P1)PEN zbvcxG&dSFp1@cxMO9H35TDyYDA3e?-fV0FeIm6*ez4)&7o7JA5_WFX0H38XcxFb(| z6Wne!xRz}%B^6$eGB9xVt%UdcOnH`lt6Q#jwdM=&C9?}R7j-COe~YsB$cqJ7_cB?_ z!2Rk#*_hVu^sj9Bn3@e%q_)^H{kW9*QJ=L}wvB#2N$k5HFgN3$(A{D=_JhFC&_CtE zr8ex&I)5M;&Y7iIhr2&_^L|TSXWU)t;_jo= z(foS28*!gK+!cMR+}#uEh`U>Yk#YFZ4xqo&`G)!}+%4z7aQCMsO|w^+G}XR=R4{Yu z<$=SKfz2hrXcDlR2+S_VzHt$LHhzP@1~#14MV73rmftUP_aoY9b8#1+fhvQik%{qn zhQ_`gPZM$U7`8XfZBG|Z=hJRGp3>d|aHnRdgQp&^cGizDgH+WhGp7s_U-8-fRpQGA&4xX-OE<8NVpx*{ZS0o!8-3twV zl6?YC2MSK$X)AcDcM5ry-^i!#dD9}EOaIwcxkGY>>tCaN74~1uJr4N5tWJ!-C|xH8 z+}G1a5SlgRuV+NYCS(8h@$2u4`ycWg8q2ZY zaP$5~o;?yp7*1eh(Yk%e7ymo{8ya){Pun zw5w-_7PIYVy>hdTWZAa+?3eEaGQ4MhH0#Kw5{C zXVNtLS0?2Q0IBFw??{gMCc3PKE^fBT^wZwG&`wREXfm(e)_)Uu zny&`FC7K*<#vq@IG;<%%6{eg!w%l^P>#ZGAO~%9?|2(8Il|&f3H?#Dcvv8ge)Z=(?mMB zcL(bX_|AEW=~>`@6Kl;!tUE8ea?@Pm)%y9S`sUzPO5~F!XHPnZe2X>7TX&B0EILUe z_12wcH?KFTXyOWR9-X!_7RVX&pWuG%?^g7@68RW$cPe$vT?LFY039q$vidq@nfB)i zWhce?`Q0PSU7v*KgT6;J9^rY6|5|@OH7UL?CQY^fn^b;R!tuBG@6_iGf@PNi!zsXW zGBCXa*iK>{nur{9@l9>TPV6SZ&tfgAPPQsFrro@cZzy-A<_(g+$#!TnSb$uF{VI4i zan4w;g2%@Z>omKeqOEP_SY)SB$WE;d$)T40DWTXL>$Mj8s{Y=H|DRyEoi-(-33uND z@0|6AeJ#HuCp}M|V0N~PljoW7YR}5Kd+Kpho@p1k<>LEIz}4H+lKeiPNiwUm2X?XW z7@UiN$J^NJI*J@6T#kPQm#NSAL}$9sNdAfjkFV?jEh;Y@zZhJ0WTYIsT(AL;<7Vy2 zwO8=`IQ)BJvx7q`d0xlbBRK9zSbx5D?E%)m{{MUJ;hS1}?$g>+CfZAa_JZI?GI){# zzH|p~pu1(<2Lzso_L|wJi@UL-U}=S!eN;@K8Xyb1Ul znXo*N+dwRcnd$UtyYeV6PQ?_t1u9NUH}iSHWxR z@^SHT+L!6kB{F)pQ|G^_qcttu-Kwm9*cW*!!^wY>e6tp?mWoeGk2(gQ;C_rM_E=vF zmiTAr+;P6I&mpayDcCw1*k72;IpY#+;~U^HCB#);cuu)lYqQ`r8t)bO%BgS9(7y1Y zhVgHLmTP7R9@rLK-m=EfH~2%Ub#$0JCm#Qh9(J$FMo4a!JtNoXUw51L@c(k(GiA)R z^sn2<^BaES*f0jcf6TgxtzBmdssk&=G^ZkOl_BTWBIlM#&gG6cgCCl!bo(0mPYeua z-JI0h%;Dfn?jxX&s(TaT`WRlLyDx>W@|P7psoyCH|8?J@moK=?;XTO$*P1lfo=rMS zbNmhQ>2*i0Vsp!{`V{oN)6?fF{y=PO|Ezm<+uP}RqMaky z!L;9BeoxmK!TvM52XD%oA>GXIm0X9fh^H9da*Gvx#pNr<=(F)9cKJ%hJ!vLpZ2H~F z@SP0%pQfDqgB&|ar#=za$X)AS>gM_%@~zMKuXvLUO2NrpCKWF^xqZa5jTqmr0P)j+ zofEgVYE3;jQ+RjEniBT6CjpnW-MTEUJsEuN=x2Pr{{g;y1$IB`8}Y#}bO-baKVX;d zKo>vCb?9*i{nR`YZ?4kF7FQTvv8n%ET_R^+?bO*y9r1*1$Q<&m{m*nKzlwarL&!h2 zzh~$e{6h1T&3qq8<$TNgy+Sd@607YUYHRC*k8Pek!}ZIxc~@&i4K(m9yxj4f;hqrq zNf-3G|0GYoy3Uzc-8YhJ2Gia$u^sR8~Iz@@uPS%#gGSjL=)2mF{k(Y#+V z(s)OeU}d~GRLOzPoz?g zJPqG;ctDzcBV+P-fcEu--$6H(yr*Zqqx*40`*Tg2Yk!}#1zfyYbYG^kKt9%SD-wgg zKZU-d(C2=9l)GZv-;X>O{}T8w8;8%TcMhlvww_blHCy2ojkI-`wvHeR9CKxI`4-DJ zjCpV86~FvFvH$ey&&pYHkVn0qSMu%0aCHxvJzq5NK?UvG0i;8}u=4C1m+gcMo-IDiYBgn+BhPyZ7BiXB#k1Oj~rF_0Bl!6lr zl?GA=>D)k8|Myv=vhBmPqu6?7(4z5^ynFMSOqGGdqVIewvta{t_!xI=Hs6y0ugJCc zyYJMZ8^@5rn+%V^hUeCI^@WZ+B$nE)FZ_nsT!QscXj1&+?>tx2&%Y_9-|s7hC%vl_ zntFv)vWjExJGK4Q<1dD$uX1Vn0PFu%4(@7iyTl*H{l~_BgVQN5d6n{meD%hiG-U z!GG>YVti-tf3+#ku}>#;Xqhr#AX=7et~rrR>6Hma+WQqfFIPG9(mSDCeuXP2TT0%M z8C|16oh!SO{lvge$C&@@c~z&DUQja+`VKgK{epQCKhd2^^BC8Y#aBD@{ZSh9ooSy7 zuXAMDRQpGaS9xo}G1*;3D>Fr&S+4~@~_eGZMhfJA@Y?+3P znU2h!fjzdXk=aG3I*X?@Jr149x32Moyl!X|UyKn=$nl%s{PHpBQ@VF1KE#T@H1y)o z%L^_F{dUNNknTy0`LhOL)94opWKNkJ=(%YAVCSsf3is^ZeA0r{Sxff~3NJk)H@uW{ zeM={Bo^O6ODgW{1T6(YVlBGY0OkP@!?_SL;U-UER_`pD`s-aN#(52d&Yr~^*G}ghQ z$8=k?DViQiUW|DXUv$pn1nqRU>`c`ki{H1Q;WS%3LGOH9`J?ffg^tA|limCR&x+F-vh_40M}-Mb2Grb>F6<+qsM&rCUDZ|F-JsGLx7Rr->`xEHcOC2 z7LvY7{#JipLxBHv@Vg3s-XHNjYba0U8J%D&=ii>%?%Q<~ToZqij9d#$zmM%#^Z7h| zG4f+St8y#n=bozd?egkL_OE3vp5fNrNL|Sh)qPzVEn zq8q*ws@R_S>efKk7Oj8ZIo)aNC$!bfIxl+H9g8~eeIIMg`ptg&j?Cc3>bR9W!S^8e z=*b{W1I3#%jf`?N z#>huwtG~~oigU{!PevxH#}*nwx0}P_ztCxyPcqtb9z( zJU_fuvD?$^Ux4r8;gX$t0OKc%uXK28WhUj>_TQO9qia%rn&y!_L(lNmBjBWazOf^n zT$vGhJ%zhQ;H$C|i$;3Gd%geCwY>k*wakC!rYr61ytVk0-+U_%Uo?F)*Zwl#d*s_j z_UgLpi_tx9`fO;=n;Hi9x{3Wqcy=G?rZ4o<4?0SPp3>mi@a^KRksW!znHtUe_0*r` zJvDVv-iE1{=B=N47JJCO%pS56yIV3>BfMPu0pi<|xm^28yUZ2d{^et(*g+*9-WIef zujJRb$BH%rkIMbyLoXv|6=8n~1+0FP19?+)_NyQiUb-&@el5Pu&);|E(jN?+ymTRU zm=_q2&YFncUSf=*$3K(j=x$ktHkJ3b$@53GCm@+<5^1z7JgOXC$9{+L+tS`gD1GJQ z>E@1~4Et%?laH?CorNWpYtjvG%d&s%=ATsF8Iu*c&efT2;F}s}Bm6~px>kKfuKA@> zXsV2~qb=qNU_KMr&tMLwGZ&Y`!@dg-o5mbXWv(u}>5I{qVwZ9FS-xb^yP{+Qw_MKY4=n*=$r)Ma>0evb79@f@7^!ZDD&N}d1-?QpB z_B*Q_{wAADY_0WLqFttK18eoy*k;nO&B)ICxobDNi0?b{T&8^q{G$fDkmkhOA0I}Z z?7q)K*W!DQZsy2rXE1iH{~H3|Gv|Zjz|q^^$Ph1@CEBAdu$xcazNW1Vd#&4+DFY8y zP?kd3y_~r<--b{97th8%%-U|5{qvuUG4ca8V(r7!ZkMi>YTsq(S^MYkX2!I4qp{bl z9)ETY<-|G#&JGP|?y_x*@tS@N{(gU)*FIJ_A5x2+mA9!c0J5Gw$GbeUC*A38s>M>>t4$JNUyKjN4QjVH1n}EdpT_<3|UW{ zqi@f*pLFBaC?B$%uyv6e_eaGc$B8csgM)5-gW{3pB+oqH#?>nh{(A^Hb5c973EqY* zYh<`B$%OF6I)(6vO2T#9*-zP3dZzO2zS%D<*+75BaVyKGVZ)^Ux8CmYbd+&j^ z^6k6axMJc|)d1Xge$DL;fM?Qqzvt#F zA)m(N(_Tl$+Ryj&?OhAXLT|cx3dm#HkGVlQ*m|G~S24E7_<7LWL;4HrM+RpX7G2*y zc;A%{&UWWLL!0OvHfDg!X-V6fx;bebRX%hX;*)=Lbz$uk=at--Wv75klHm?{godiBC8s%O;XF;;N5l!2eoW}diTX~4{W@veOqzU3WZ6GV zOotA3lm3QD&$A~bra>#R?|h9o(P+|K;hu9=&mEK(-v!~S4%W}6AOqAT`Nry8#k7G> z;V`S^PFJgWT&-x@1F&ogT^ zL%>b->qX>Ie+LMS{Kh{Ad(*1fOFgkFDKE8QyHlU}--t}5(3rzXl`rT^sW}zR`naXaOTelyPR@cSq?wg%7Au<7fhO8Ty@mzcV2-UOE zJ0Jf;$Lsnrw6veG=!}(o`x0nuIsOXJ9WE@Nm`{78+j-sN@-O;-kII>DwGSyjYJ0BEILd5B$qj3wdBq^Z*D3$eTDeNjbNnm=}h$QDlcPD z@@N|EBEHd=I7hFNWp{Do&Qv~)@g!?4>j7)`Gd2g$Sz~naozi@sjZ4{mbh&%au({7Jj2En9h3yB{+lKI2Sy`Zd7#`k%p^xi023 zrUZX9(ZQAddLN8Ko?p9cf@j;?U7h%U5LSxuJXmXC`)KF54qW%DUwjQL{``CMPPY9d;S$MC745g}8PpB=zB_ZU9?Zegko(h- z|1+2eWHKMfVh)y#ok{y(1M{!>%)h2F{~F8u>uly>{`iH(xn}Q=4FEtKesk6 zQ%RS$kFn6W;hRO0=Nr@6BS>4V*;nD8?@cqlhwed#0=gQ&QpJuIXP0FsWhetk{ zW(|dJ4&9H9Wq5nJzs{S^@pU=XTi}`N@KL*n%j}(jN0o4&!?mAt54LkPNEd;8Q^mdd zU>EWZI=iu~C0B1@Uc%WhgO{DL>ay2cEwhJHXCE{mkD-H`SaZg+m;IL;KLvTPj`;q> zPdQ`dW$TGwhEMXq8TMuCxR-x&*&hFi`$qKolJ}nDp1H^1hBbjw^mAdq+8>&gEYgoL zD0(a>PICk4pU!gQE>J#KA7A9g4O5(G2As@y;|3{?ajfy4<;I<(I4}TRyxE&rLOZ9R zuRF|`Hs|Re@A%py)4|uaNY;a<>l8vWl?tC56`IC*J5BKCO7L6$dm^-(%6u3dMLYfz zVV*C!ZL`+x;3G}&ZspA_`ed={ag;L1hEy_O!@c1pxkj$bvk$uQUg`Nbzgj+mlvX(> zZHfM&u^a!ka7W+1*4PbY+F!WuK6NS?a8%DW8Sq)!T(q^HF(*vlq5hzbpDU!!pDCmb zK2^w={a7JmWdosPK)owIc5K$w(ElXH!BvchD;XCPnGa83K0KcBas~5YbbA%;jc!lx zuX|rX>R?7w@ZF|T)XZw%DGpAT<_tU&Kqpx&`Egxq154~uG zC9fX3Im^F)T(&<-+aB$c;}@MY73ca}KefVp*5vhXUYXxNT7O1=VP?6jH++I`3U_qQ zf-v(*aBAlo$r`NlbB<9mbH`sB>|*_hP?$N$v?Fs<+FY1x zjPH$o$QC}<7cH$-6x3Y6+3B&J704Bv9XpB5(3^wB8TL@xVR^-*_2tZ2%ik+7Slk|d z!~W0dhrwR8!^P?kFj^R_K2;gjUc%tS=L<}}YfH*2N=Wm{?|8)pbeWdX9nYEX#Be7 z3jY{yhQVQB?@_RKJ6DKn5!W1VdUYi@%s$zn%vpvW8qPY%bLIX@Zz?#PRb2%RU){qR zG927Zk7F@&&;~Gf5mz%ryqVC_pT{@#Rt9hJp$@%Jqke}Cx4 z2VDH^?Beg$#Mij^dnNaUF8+?^exr-O&n9e zu|Kh)0K_razQ?#)If%(3g- z_g(DW8n+!me*M}iENt_Ga4B_HA08T49umghXWEDPqeAL?uR`j7k3!_DnI>d?o>17U zcg4$&&AJA_|S?R|AFRQe@lIyU-S}b&G+wFa|RgP6AUisFS%d`wojtdVSHP1d<*?A3?2^N z?cC?+|Kk`e`9U=MG}v_-UgF{~#K9rfJVp~X zFu!My*Yq9Oku*>b-OuH|(wi}MDf7${?xXO`gWy4!I^L-^n^fBDQ^(pO+}* z9Lzd*-8WZnj~s1XchH-UZD?-aF!Jwe|@3BOj>GqL03h^R7Q-3niVt|Db_dCPnY z9}-W%gFH#i=vbr!II*2p7YCQS_P}p&PLJB@e>nR|;hTgfv{*RGR~2|0skWxgHhzGkzZ+W$j4cNDaS6_uCoL)k5z1sGN&s+AMKlM1rJWr{;GC2~ z{Ht;Oinx7l{@2_#I?~UIy*$8LTf=4Em;GO=gZ*LhNX{rc$68zW$uG7`cVK*WU`wjA z0#j%&m1PSulIZ+!aZi7t`~}Rhj>$hjlr}r>W8S;2Tj&t-_@UzNp@Xcm9vy^VhLMh6 zhNcg#VCCMfEq$S@rV73fo;MYzhaxMfH#}mbt0Vp`Wie}4eFCw?;H-1TIcrz$IYz%E zF21Kfoj&kgIoRO*5oj~6M~UxmoY#zACcdU&E8&)UywJ(v|H$&GiA=Lp`1?tq}w;De<+*y_y?3PMxBq|+|zIR)TQr|5AWP= z)@1v13rPlyEMV;`&l3iq!Y1E*#=#@bIEZ}OBXpQHJiOHX?}Mb(TgM`wGUs}Z^=H!H0=j4V|zPsGEdk3F68XKzj zX71I;n&V0*_Ky0RZz?YCpF#b95I!YdIh1>6UYus{hmXYfvn?k+?x#WWlkW#{*@*8z zS8ChG$x|n}i8SR&f`0TL`Kr8$^o69GvbcRVkGKUUj`QIN?380SX?L1$X4}b$-@MLQOS#>orQrvs&e+5hLr0>8hMR|l{Ja}tY?}VMg8BH+ znhT;c9KR{MDx5a^Ty)7;RoiO3j;GCKBm5NOyZoawY1eeS2)uN3-Pjn?{!7t~3Y&W~ zM$Gtp51*+4XMD>3=5%B9D137x{7!AQ4H`Td15O^JCw%J^<21tekueYdUmL9U)##Tdwr)<4H<7{Y73Kbw$Zu!!%`cHT zIrq5~e=q)KuN6)vPBzUyItC9(ILko(668;=b+c@lJ2SWH!W^qBbFFU7xw@lkz^8m| za!_X)$e+B{%klMN>}o{ES})hUq%L6}`?Qz7S;TzEtXKFp){bvQw_4T3ns^O!q2XM) zTt~6DF#259##@ups;dY?$ehf1d-ytAL-w-{{vp>IE~CqZ$5nFg*E8~{_VjN-PL=Lf z_RdwuW1=?XdemgYhKAdJjAG!8>w4LP9X5uB&7#{%4Z{K#~|E~PV3`dy1X+3(2 z8~?fD;lt>@h|jTKbK^fY-$@SXJeSX*DPfmn)5nmh;%noY*ZcHLA8wIsx}rQZ1bH(o zJLR-M>_xs+;L9;{^Ty&bbe|b!P33vTvz}6{xjERBfi1M=_zk?5BRRDL_a;wHvsZE~ zc%rlnBf|z_%iZ^75444`kT5sjYFWM7WU(Jp2dxvyc29J+#MCtqdqN?&yhtJSdrTqY zau%WFR=p#PJ2q=P^l=4rayj%e4!XIFIr*jR-A|71-CtiFVBbPd|%l&*u zy2NG7J7oWSQ;HKmk9f&B+81>c`{(uB&+)(b&H&C>C=WGJ|8*AafS(shebD1~y74=`XYg2tY-!!rbeK&(~`xJ9?gSRg4`l0wq zD*oTW5@DXR?g~az|K9xTBBj&b@44v)GwIh+q>WUXHxPR(F^zV-iL_x#qmTcbkfwH) z+&7Q(b4icf*>ggYr(1VZ9&mdXaYsMw5ZaHt3hpg5ddR$*3-%+o@7#s0O|4@Sr2bVM zg?Sm&Q+CtnA-9%))CTrN2j&@#PR-4YVdun8T;{l7TV&*6Pp_I|#r z@s~=dvTbC}!1-i-B~u+BuE3Yuyc1gf7QOZ^4i>rVhcD?IS}=XD3DfKk3H84pLzB{H zT6ep&w$&G$;?mkzx@3)M!!j-MwVD7O>=8T#4eAm9U zk#x#OH-ZW+P6%7iecvz>vZe%XHH{Xs?oPX$Rp2EHUGtPZy74Mk- z@6?;}$sRbzo(c^(wn5kjlHLjpXl+~R%X#iWo81(3_ToOn^D_SF+D0Dt1p)r}+F8L6 zZJX!^qLg~BfktE}ARKs*_th^8Oqga*H=*`NR6;iwLq8WmM;C$*qe86}j$aQS?PtDw zlK+VL|BwD7Lai(Sg|^zm-+WTS>pWcF#Jq@BTlC87uKUwVNNP~BW*1@ zh{0j5QtBF~@!>9wk0)-#oR^o>wcfj(K3^w#Uk1I4R?auHp5OeMGfzGdo3HL|_T2U2 zAFZ67sWv^&ZPR>0{rkK8GMI5N+Z_j&qy&F|iG!{Ed0*o|ei}|52Zf}`pURPHpFcJ= z#b3u*$Z^NQpI!UNGsqJk3q=|WUSSdEPPuzWJGo=werQrSdh%GkoveEo@{#>?W6A=q5l83-MFZk{)uVjKicGY-V_BedI}62`(``G)#!iwW6ZXF~jDk>~$!n(s?_PWG9iJuqJ& zvl+S1O0K>U><}Fpd9U{Pez!{LY+gN+eQ$ByWWpXdrtnd5&xY{ zKnvy285^<*i0^}o+ggey3@LS-tv30J|Bj-*+Zn>-TTF@!CJH_4`Y) zpV9jL7UDPJS6J)!|IPgr?zMjZ8uu3WTEE}G{W5T1Ei%LT^sUB}`1eQ9nfTiT^*3^< z@HFbp@7shOv2=A!eLNxmY}=fXFhpw+yfc(BRp7(tGIGLGj(*YLfd%Hr`vcy+*N5u_ z^~-XtC(k=8_!(u;`t<-qTkN$UR6Rv^r<2K|#h1{3V_7E|!~T;5`Rtc2R@~Y#_{dnN z4!!hU=$bw->&QpjJMsC%Pa*E;?bt+LhfVa@iFVM@`tJSz zogH)rcF>XqpT!hB zACVSCrjFJphxQ@YA7Jh<1^hYDz9!SYj6RUuqqCx4qYPRXNNjtSi_QMThdadW#WL+} zYJc9D15ed?UFtur@v0A0hPa(g;`8g~oorjQBtD>Xq{Ta*Av8Y262@YDyqa?CaK@zk z6!dJRjZWHo`dxnYG-vZ+lOul=hfXo}gz1T6@8}EIIWwn{jQ==t_&r?Lah=bVN&F%7 ziyw1oEF9(hS>+4x?9=#SJ>U-2%gnj5?O)KB>eGjabL7!{`)6+4eBzw3lV{)N#?8~a z(y20Et8(M!C?9*=)z-pWA9F0}()ixm_cf;XhOY1>1rdJo?8|U|9oN?8k|462pl8GF$*%R#Gj_E$c zOMcm?crfX&u8nRH@fwF3PiHcgoc*PYL6_fmGh>kTVaAWf;C}KsJRi9ly6#OI95Q1o z&;F&GM?6_}7^6u0+N9;$_a>&%UN?~zHEGaKLRx%{a~|nCOnRoBRPOAhGV@c$b_9NZ zsJI>S#ji$0JhX%KM(jB~1TG%mcbaKmL0_t_8uR~6eRs?VgpnIomv`!e+>mBJO4~TJ z$@yQ<_-NVZCQqJS4gInotW=*IEM{!i&SRh2@jbx*gw7oN zO|x%I?2DhOFLLaGZGDku&!7!9fCGthS&cL89Wi4L9f5e@EoyiAEI!9IW0AT}C0^Kj z17VKl1PPcc{ZeQ`pfp-+m3~od`TIa;&Khlk98{~l#Q%O@ny@y$=U;fc-_?`F=M^sA zZjH-2%huu3)V29V2iEU3bYSbzi!I@r$#o6aVC+@0xU}d01L?lZx@?j;zoUvX8S%N9 zTi{8rz6IG%?eGKIL-_nC=hW&M`ZM^vN9(xw0*G8}g;UiQ25YnJ&h&wBx;JrgO!m2P z1cpxPqB|*Zbz;ag$VlnP zM_GOGDIM}-dm>v6M?T87zv7$1=z8$`{d-?r(x7!NFkM(3Ax$(UyBDqHt~2z9A1wDB zqZjK_9`fzDd%LrLjy;cVn!A?UhT%1`Svk9p*=wY^*D|-=4vwn`NiX(3&(gr5zp*y;Dn#rMd8AzOW$IA6>Y2)FjMrZ+8Gnv3tm=_%|3*qAhWW|Xr7 z`hi8&2U2x*)anQB=oI8UvuV+OR(Njr1&dR5^bGno^$f<(EI4blgEcCH`p))_UvY;2=$uUd7TPo=fSkP|HS`8;DBGE6eWwuq z*XRmT^O{|qbWXMH%*$LFj!jl~`ZUQr0INN3#Wef4`E_g>>i|I@g|KE$M(@4NeW z5Ay%W3gk@urHkex@$a#I)rjs+=a}sDVuxa6*fjf3(4eqXy1TPzSNWN}5Be)Vw}iID z7aslQ$V$u)Xm8|ygLg_Br1pn@3d7Fj{u*IeH_oGD++#CV$~g)C2>8{6zmoUao6t7T ztI#>IUD3aELl46`@1!2+WjG^t7W-+v?aVo`vK2fGUd6zxxUFD-(5}u*Ky0mOmrS56s&B zN&bxV{Uk7JB)p9I-23c#P#xZ&oSL^)fIC`Wtis;J2X;jaCT824NEglto64Pe-HBys zC<$(I?IC_i+om{~z~zmSazJ_3aIOTjRM98$|izo#XXP*BL_0 zp{GaE>T;JRc}>`~;=<5KY=0Ctd7yW%3F(s<&w9rPcHNX#Idd+1g!FD1q2Bcn>b*_K z%%%@TrVROn2)unH6CWzJ_ zQ*P*28vm!}~h%jLqPj49*%X;n{Zpye9PD&^c&q zmN;9+Jpa9O@P5iQW>cr&*di-f-TnT>2Uvev>cwvzc`C7M@qzF9&+~411C&d?8EV)E z6niH*B>zu2UZ*alq-ox4k@oVZ1N=v%XP}?ft~l4<(9HTsMGpJna_kRWJH-Cvcd$F( z9tf5`m6l{^!t3ro%O_4gpA2S0yS=G@2jYY~kAYcjxD1U7vn`XBWj~ad2Hgs?W5lgP zrrC0D@`NMB9UWY}&c)elz(wrygp5(o@k92X`H^zbb5{zd;f+4R zE783lMNWJYJ7&pgKcL(NzWnB+$csiUWNu*AOTld6`ISmX&g=#DG{B1!_{Ku3@SL`N zB)QaEZL77=%emM5{xXHo!eA2$r!S}f#?g%2MRMa_`0!GDU%m*)W z{+-Xig&_fK(uG9_Jx+Wk@z@vG8{{xSBlbE0o<1hImu$aop=4`sb16uKxYIy7{H!XmT8wIra?o( z%x8&1hZU~!SmzmRtC?%$*B^qV5wm9pyV1^)UCWVO+lQE&rfV-xg!O0qK|2_WtO@RI zEp&ZtNgk2zbeGpDcN%mqnL@btYwGCY-oo<2bHTlI`>Lelxc4}9kK>+W*s&L$Y-s>mhU#;-36XT>a; znVT3_abKIhlW~vnw3By2uAcH3|4~0JI|=s;*6}@Ioo^Jl!#9O>g{hS@CCkp41Kur4 zz`NyL9lRUe?Ks{ct0m&y6&JeWpMNIe-Ak4e{|WKzg>>)^{|s$-2hVB4JJuT#@$RUL zcg-JGgyMLoxRdej`jhak&U+fXtNUJXIe7QDi+8N+8Jzo}8}7#+#5LUOANuFZ%>(>v zThH~YeO9g*=nvO_54`IL-FLJvb#02DcI&Hl*WBP_8SxR~j^kan!Ml46rexWj2;1-u zIuYLe!lY%|?Gn?NUkmSkN?b&?UP&(Axi)~kgo9v8e7>$VwEM{)>DM77JJtyJW?hL~ zm*t!z-I+QHlf)xs``Cl`^^U%QuWknyd9Mu@c@LYon9gF}RpH>`TZWb$Tr8t4+Hi5% zAcKqD;B#?#a6E0)hKsb5aPv~)REDu8%(h1pYRnBj9WH`Hi9SU)oQ#PT9awKoXT2_i z_0~+*TeBEX*^H|k##b)mEDwClM?Pd9=xO#~%Llpki%4#i-+}$?7de1!!@=XW4Xfii z$T_RA)yr_soOuyn9lzzegKHw!aIPG#HMH+`_5tFj#`vz8$v(jrWcc;$*~{kM%;mh< z)eY2?+15 zcjG=(oSFBh+mqb5cNK>&RC>zsZrodnD^^^ZeW@E~{)-NFgEy%O>ib)0RkE_=Kyu+JJDE%`F-2goN`LpqIGdR)ou%8e}eLjKf^ zdkP)GxA>)RNN_neVba%$F4ShnW?j#?xQ_90E#u@G#>>_4pGoNDu3{Wr$-4JM_GWi* z>=I9w9j(D;&LQ1z5&dMoNxMxleHMsaY5EhNi3*wfU7?V6AE%Ih5G`pgFo@9jgXWAz z$@ik~rWJuuoX2g!7utr}%$BF8YVOTh>>Wmj_PTo73-t}gRt-KDC4*b*jQ(O_k<~r8 zs>0zrBb0|Rw+#EbWiyhtWwttYE9WU5nkXdxm5M-Ec84FiI-UWfNl&nIEaijWAKsPc zhyR8nQ#sqBKHnehcd5T+#Totxw9!=G)31KFu&Ii6YEzcy)o7mQ)kpysV?I*s3GbXC+icD|clUs|=bxgVd0?#g!+EB^s5|Y~oBMwn z?9H(6annt^(8l5)U%0q@M`9ZEE&d_dT;uB}iD}UMJkmZUuD({j(D=_dXV2k#CO>OT ziTUZ{*U9hTf0lh+VjBIro3uX@SLw}OApIV;noix|Gj~z1jaqYLyhNrlMrt3>n&Vbu zv)Qlaf`jNgc6z<&eWfGMu?IuX^0}e$FZ;X|^w08&tE6{WSY9%eJZbiw^r!Y9I_EX9 zfA}lwQSrBL(Z6+`@mD$#{vukH?SVqg2@LJ>p9bEQ&G<74)dqxP#g8gH__$7ncX)yg z^l_+Pg?}Y}2cM~UZN#%LUgZDs#diKq_&NH==D8!9r?OwMUlM-rI3E@n{#wJQWW(n% zWI zsm^+`7_OZk|0>iq9TZd?z=4J8g-{*4>gS#gG+X4?&D?2&Si z+5vj-ROSZVpb52`aO2plA3&eiL#Nk4uh&Ai*T65XhF?sAp09#mT*-cNhhN0M*9^wW z7Di*9K5~5GVNWSuT&ECBt5gVn`4oaxTUdJ&FRml3oE=({C!G^*s(Bc8ylY>e4fK8O zQJqSEMHs7jBPTt|i8`fKaSp}`&CZ7FH2`MfJ0 z?WNAM_RZd|ehKiOCwX80i14pscxrTo_^?wRbmsgg-+qKVvPE&e;pE*6ug{x($M(lf zo^1R6lk$k?AG!C=?Y}kYnf84rr^EA)+ZwleEN0^;l5SJ zzYPoCKpn-Shtc<@A8FU07}^NLZdFLTPg6)gR4JrieyngPJVPP$Y)vuZ|CaYuhIMJL-tQMY!8&C7`;~aU`N6+ zcl6Miz0=FO&tJR*-PNqkor6`y((f-Z`u%X3!_O=EhW5>DNJ^Tnez#ICpXsyajxY2& zbJr*|to0M=>!wX*&7i^xf(g?q-E{(=f5zeNbR&Z9~y*q zDhZcONZRH*=Ew~t?*FlaT)T1-c=~yneAQ z|AA2RL47msLvK6ZS?|*E6!)8=_qKe{%_zR{25B$2^sKpr{(V9|=$^O!P0yHAt$N8;9)Z*YG0slEZtzD~ZUNz;Bxc;Q-&>2pt&7dqcf z-X}=YKa#BbjSf-s)28Clq0!nGUEJB36VEVn;?;#(j~T$67#U_K`|5UWmb}7VTE?RK zOXs29L!ars!tJ}o>@juLsPpWHw0D&`uzbrd;lEBlvd7cy$6pZ_ub=l@W#&EV*PAlm zN?Kz5TKNBo?M`0mH;TMBkhh*WjQMX@r!O=Yj7)OdhcI%g_Zj!|c;CqHlsSnrwAJ_R zXrB_|wSFYpk5-|U6O??`4w(u0tWR0U)_B9V;n_8T(pAVi ze*|lusogwcMeXY&{<+2~?IKxisb|$5>_fhM#42qr#?J(}^iS>m>tlr$nZ16Tr&{~} zoy;cxZn7thpMSIu-NJt4vjfNX0WOT|7OI{_C!6W`6k3dp;T>EPxdw6NakZdZkiYv^ z2qk+|p@)begN4y6pm!L2Xae#W^?%XzOC))$5`U`FEu;`9-NIkdF|0yndyr6jDWyxe z)zdk3si$-GF6LlwqGR|H+38tixG%?9u}jcJMC;KpTx`XfkzMxS%V4FeYq*$wnfgW{ zGG+rhg*~pWq0Eh+;_4du6EE3nIPuF|UBfW$rFW3c_z>=;caR;{_qdndLH1j#6mN75 zx%OBWhyScNBO7Jd7rAkp6gNOJRJvW}#=W9ABO9gJBiy(b6(`w@TDBOQAovg&WyNATV(+I}AEL5&kQ^UJjfezV!t$H1fBp@I{K1YmtVeQ z_)3N?8CU$z=rHJW@yL3UmTpV7YfFPJ#1o%1X=!%f#I!yRZ+y(8<=MrFX~hmNe2BO< zT?y+sPxCF!1toW7CFX}NUnlK8la_0DOH6|ncat{Dq@iufCDV&L_c-a|>??!HSWf9gbJJN5L;6GgbG<_Pf$bgZ-;otoNcxg9zojqvE?>c3boS0U6|yIK{9QMDoo7O#h6CYpq^to{b#AzSEI_9Fk3a?Hwc&H}SwqaW+mlaoHpgF5 zmYw82%f^=RvpQj(y47kK8T7bzZ+~gBj9+o-29I$so%#_IrrR+SrrDpHFw2%+OMUk# z&#M3F&IpbB2i945;+vnFvbg8LWA}BPw{Ef8)Cyh{+e%u1`e38goC0m0RMxMpS1hAr zejge*JR><&Cpx{kUFh(P_Mynl9YT%!tl$goIqDx!F3sc6pR5&cyW7?2ZY7>M4!%CL zk0RaP4sUVxO)}2`_mdeT|3#eIQSzV0^f3No<_EM#Z|h8#{?9HI8tT%+s7+I_Juv~j0 zI4$~9-rrJ(R%}qUcUW^EXU}i0y^#2z86W@e{(@yMe-L?6_VV&=CL1S>1NjUV9Vd>1 z_d=ZC&c0ji+phCwR@XykFC$ki_2yNZvluvgLFX`V_QK((x!=wC3-dUK;U+HZize#5 zR`v)+=<`q>HvhhF&b`Hb!vxlh`dHx}^pSKF8au09-rJ8jtp{j)taRhfQa&R?=GxD= zaYc$VGGw;B+>OguoY^ywVL$1{W$AnD&q5Y7ey@b*W{(zf-pBG?2@k4M2*0UR2p@q5 zIKGCp4`8U|uNRj@pwpafcVLf|X7ANE88-!1PP6(~`ZdXbCG4BbF*aM+dy~hc)2E`@ z-8_rljqZ!MsyV-4FF_!dV`M_kcXIO=(}&X2$=~Jibk0O@(|7Xjx3J)-r%g_Kf;KiZ zfZX*C_tGW3rI7l)sgU}tgso6*mPCN##0!NJ(u*aJJh!POZTYy58MMUkN|QnY3*Cv&1y|^Cr?NOAq6>t^A74|L>dtofv~=97W$JQE#y7P|Co>V;q8 zS-JQNI=ydV*(S)3(18TogoEIHd@pz+-e29oUU2jPxzuY0I)dxD&f~(^W$h8{5#HyL zZs2jQ1LJ(5Fm)>BUisIJ8|AM~%D~njqq-KmgaY=7SIRyC-1o^Y0o?y9x&Zl$Zx%j} zq)&cfg$JQ8K=*EZz*j^Z-{Cc`KHyT;aJANQfg8Wk)d$EXVIOmN=>*Or4cs+YJ&5}n zS0`{T_l4XGtNU}m5uLyq+P^{hv%u+#(MP*7#U|n;Q%DXkbK_o7KBE)Jv`4scFDlOH z1k%}0?#!?Lq&S1;dG`0*xK)Z1o>v57{oOeImwou5eejAo@OWXj@e8lHJmb;K>6u?> zPA?oUQOMYdxH#UR@E3t_uJBv3{p{Q&QTR=oFq*n(?Ct}%3aqr|ec)iR=JVh{C3XXj z|Mz_RPV&evd<8hKvH3He<9Y$@`*OaEzxblyti*J-z8{cJo$q)7^_)JguOhNx9Yz(5<7%-=QRdf1(%WL>g z8?WKMjn{DhEndU8HoS&!9)M3qx(32^jB!I>%+q~$-2EHV*6<+z9e4d^n8&w3Yi;UF z?fJPCN13_l$ZuRickICq~+OXC#Eq*#ite#XXzwA8$9~bEeW&FXlAjf9Jp>g%J_2 zudfdt=+m73Lw8TeWHawyXx1S5um*unXlFeDYzI9`{k!m}hZa%T>yi&*kgjWAB}BU_E+1>6^Ehy?EKxy?E~8 z)>%BI8*QsOe;?wUIe(s=>c;g_K37NG-i|TXD=c4}bx8xIP%zgK5qW ziO4oN4IbjI;TG`Qbrgl<|s?d%7eUv0&oi^f%@s}o*I9_fi>cT*1^-ML0O z0@n9jKM)TQCwl5cD6AMl8I^vBF&Ov7atCSkZD;a2nD$fi9_PP8OAf}czDNHE({3?o zoRgTCMjr~(ZZv5*_ScDN^t15mI?@Wf$(s6>M52A28me zRR@sOukGrNiULazX_c=Sd(s#r2fk{UoDw?Yx$`525Y*&_nfhkZeLEj#O;9h zgB!BJ5)Fy&gs}?a{`L$l(xw%ry}>i+Ai%x;^?kytgf|t!ijK(Br{4?qpJK)QKOj4^ zuA_1#VnGu2MQu32TEVGsq#+B;1|J)+C6Wx+`gh7}#(%!;p`Ap-*`_{@p6Dp|ic3?y zBxfT39CG8jD9-2&a_oP(aVd%eWA=d`+l3kMtU878nMxN+e1s{&4B9se4G&>GNwgN_ zUEzaQ@*!#ReQsn<@I>QV@@BD%CI9O_JNn5S`xE!wQ{u=2?SmfI*Wgm>t9Co6b1R_B z_f&t%_O3$8x>X_k@NE;O*=q?;#F10)1B({pf63howrfEk6yc0qFZN{x9&htzY)|6Y zA$rux;|qTFvjWA zajq@ZGo%Ykq@%PMS8e;8+%f(baccK8cg*3Nv(2~2LB^i+{k>_cS)@DT9D97qK8m3DJDwCFD_(NNoE2rE*TH-h#!Z?4*jC05D zaCdM(bRj!``5$Sl{ndyAW7;=GXY*-0TUqZtlD-jtM$%P5M*W`pdW4KI8}1AZ){5 z{E}JW#hf7u?HE}if=%UqZzr=KV9wLuERvq#OUki~IqBN#F7YqO^l6^DT)!s}E=&o`?2ax+zAUcyJen&##FqBS$Rf!DlNgKG zx}|p~PPCJlRsj|b2BSK`pEze??RKtJ0|K!J3I8~V{gs5zA#-;n)O!ucH1Y>G7kTCn z6{GR{-A;044_}HkWEpg|k^MnWF;4FztcPAI2%n~(k_n%IR{xBSeg(AgYr>Ti0^zZ& zo1Dwlj_dQJ^lIcKqvI_YYE7@=-sm%nKFUq0`PF!M{j`m&3#%M9_&B-LnqJAXljpDG zdC25hhK%B~YJQWG@|VRUz4((-``z>2l2-OVEvGG%mXh+?;*p+ogHv1X*=o|{$Dzu- zzkz$-J#Q|lLPz>78KKb^oa@RvckFibkqw-G8lunq^fmn<+tKDi{+0i4(u+7#Uh)-m z6g)QTVrc6kXzW60Z4@+D!CbYRxoR16)h)NmU*MK|x-Dr%cAJY1rHcEd+!t~0yC-$Y zWbQX%YZ09a&zKPif8*`m_sFe1muS9M(Ak>abVz>UGD6LpyN3RmmKOTzMJtA!7vAOV z9QtBGdg$-W%l>gQYbEtotc-HY2PeCkW1qIO9%bb#_QisUp)W^X725Nh#w$KJ6bJs|gA-*d@iZ^!0h?iUe1+S~C>oxR%t&W@g_w3Oxw z?yn@R(wp+;2F@ej0`?yL+!s2qG9?t9q&1sP#-8Pnw==SNS9>Jw*vNcEIQW^1t5*}J zy|@RiQ+$T~p&LJ*__$2{t{Zo$-lsm2J>PQUDi!w#aZA{b%6Ii|F^ji%42oxS+Dc=ukNG0iDPTOAb`9&rf=o>jJSe$RmDv zpaLC?c;*~z29&l2n~8)pBY)Gr-HH2Y&4-YlkP`<&6oq@zV^q;V)L=H zGxy!gV>&licy01wUp$w0^e*d%6NM8Np~>qhkA>pzWt1}pt-azCt#hP%;JivN*?rSX&W3(j!`i#4bJ zdp+}{f-#|U+EJFNR(J~Y%R=fK84JCE%jSD`Pb=L*yswY^bY2p!;(i|d@e10ewJYUF z*p>19lfDni4X)fvGV^u>%D zy9TG-=n0PTJhE8*sdbpDa)`_1~xJLfra)eqfrU!@Ri5Dj0>eFVBA&+wg>U zL!|Y+yK4QI^p>HM@#mF$vZ#&7!ow=CHoG^YO!3V#|i|D52=a zd-pHaZ%Fbs7f-|{x_)A4^lUHZ>3Eln^yD-r1+PEcaHZk{u`@j#7c_ZOgBRXCw55sf zoyBw1+c9_+&r!bD0exA-n-c86^Qq`idZU}$g8{YPZ5DY-=vQCShq?0A65ZtsHtj<3Lk231wT*9YTk6Xb>e}M<>5oa%fs4Rrm-cA-9ekY#W&+=jJMU+ z&$|Yn<&2%c#n!qZ7t;n@%aZ!^w5~4fVyz$8#b=$>Q@pC;B5UF?-YvYy8f@s|R`{L9 z?y<7imTKnF)wv;bTCvC)Pk8@}o^X^lXc{mk6kVCeo~ntV<`wxN*2=IUZ68$ssIQti zOGfK8+LLVbQP2ziz0x)A^{s+3eD4%Lmiu`xYxD?6PGDwb_xapsR`2=@e*Wu;q0O9wQS+Jm9`afW?bfkK?_oFM zy+7AC|8(D=eBf1tGJS?EYMt!M`%70#{U%tUBHFO1rgQKD+AvP<^1TXeg{#I|LkwPw zW$v-h(aEWna-xjt^a;wPQ#irWZDzXZe5g!Cye8TzRraj z#|!=c-7~2EaL4gN@Lcxx3&-^DY3Rd@>DL(3y|9(P(PCULSX|9`u1;F8_{$~O2vC0y zHu-V;`45LUkEZE!~5LesAa&{0P1n zPcv~lx|ua+Q?DVMhe{nxThA?a+V+g%;9vC)|1kf5G%fh)4%gSw{B+0H(ZKXzU)o~K z9T`F6yY9>kgSXSb(RhEEw%S2{;+o)7lj#c|eG#cm4$0o4g!g?pna=m#z`oFJXZiYw zj$xmEo9yA zz`x*~tnI1&UjvJ!!`Mmv)Nit}{DAvGtZ($@EZr1zwOZqoycMa(-f+dZP;>?1N^syg za}Ln>gJ41gOnBqol)UK6o^UhuXx>2FX3wj_WIt=)w)TiHw~GG76K;AF{P=?Pl6SaY z0UocpB-GUBj8K#|(s;cETIt)4an!CZchAy5Y|o&cA(xlss%{FJ{^>nn`A0_qW25D0JTQI{J5r<1xb&0Pl8 z_w0bZ3cSe3tY>b|)wyEAaFd_+BHp@OpZaX9$M}BHoaij_Pv9bCQaVkfIeqTfz?1ZN%y_WmlTtCn|d1Z`G-hnTUB2KdT&aC5c zHxXAs+yTOmxnALtuPU|a@6@KezmGa=9aTIvNwnRyEVhgG(RcNqLAarai2* z9J{;Hm=`QfOsh~@hTV}g^<5R~kcoZwzT0|RTV&cdDKxX>_IgPob`EGn( zAbITX@Zwpku~ES0RQv+@bbMv1H8hR?NX}iycVFZi(q+vsvRGPkVG--=GiS6==CwO! z&RCz+JrMFfc2TH(3NqjYlpTJxIT^VMowD#ivMJaR{#rCZog0P`k1Xc{3#yKF7%Dk1 z!oM%zAMw8N(l>?=)pZF<_9%1pwS#DbIA2SUeXnq3-*Cyxt%JY!fAO87@Ayt+J$82q z-!Yhuj3vHbf}GVCIm_S|a+V)C3)#B5EyTvH0V~=!Pc3V&qm)#tj@7;cENjLj~ zJL`Sydl$xXb};=t1v;EmVhwicnG+OtfAx-|TcwU$&Dk<%211&L+yXu*u4G42O!7&= zXGzRs%fgj>L$sDo`SrX7-ZGMM`Y2~3JY@%M>hV|ilzrq*=CSA)ZcpO!a>*tq8Jimp zKzdZ?+wLT<>Mxz=Y|VL@%NqXwPnZAG4`$pmjz8djXfN6IvX1ZC&3rJ4p77psCK#hv;`7Gvd*%PTaCVD3Zao*+m*)}9%)_o7v?G_^N_}hji=xp>Grz@J2gK{w`FT3 zTKs|fTk;X|pq^9Qex5?SV}q4$Pc;1u?;RV6iT|7|V_ZTUa^Z0qV5-aCI~?cly=jZ@ zkx%`$5$tZ`=k(hs()uZlz6mF$(Kk1d)>~<70<j7RJvpcBrHxwCKo_Mk>fezaEb&+|e>?t#9=f9^<;}@u*4}z- z*2W|=x0!oxFcl2>65nEb(4%)b7rbgWXVp`k1FNj$k+&Cdm2*wyTGnnP@%KIZHt+BF zrgdU9|9HTY9h_6<*;~Z3AG@|xzCY&zYi|`c@>*k!dVSgxusWs48`HXk?tMGPhU<`7 z_mBH8*2g^TpXJ;#&VQLk+L+}7f@22;LYJ17`^V)3!k1rI9-44PMQBVBHi{mrxyDm= zc$ues=0;D&%tp_snO4$;hYOP~Iy@!m;=?sbqYnr0C#SiRMf>agmHp#hYtVo7tZ-Qx z>rFrJIRX8{>KXS92;NZc3HGFIk@drcC%x0^?SQ|-Ox9PjSZB#*y(Ne5VJ|xipI)2W zbH=9Qr%QVuWcO_7-EZVqblJHrQ;=1y5q*P)Kf{)RH7uVMD6L}cs^Ki_3VdA*ywNS^ zP})iTnX>%W(kL>d*5svkN?Cq(srkn1=LDnr2D(Y}eEopn-+7*bJ?%2tC2!0QKEm_o z-}g+fqVG+6Upk6DXJ6K($+7ooYZG_HDE8!K_T7Uo%9SaB-)umC^&0s*(f=zMw-0pe zw4gJ-EFUVd7QXH28}w2x`grd~=BN)}7+C*YYwOTfPg?bjU{UL_cGF7;FE3rSr2EX< zTLvOmcAq(*#r*S%`Sx7qD4tHu0d$JnJd@W?_p}Th$+@Jx zDC@{?+D(tpwt=$emU!-(-cpo|p7*YSZDre5_8;b-`yb}t^dII&7X6RP-|j!mpYk8( z?;Hs4-QQ~3^QrxC8v`k-2kQxBVOY1xw>)pbw%2 z$a?5IqtJ?cz#Qg%`4Nd#mxt~m{JvW@*%(jxuytZw_Rw6<*um4cre9y_8@!9W;s+nY zt3`vNMMH=9n3bGz6Y0z7qtkxl=-I)W$)mgz+;9DvZ|Qj)&yjteKUH1%*uN_0Y+2&N z2XM4sMQCI@>v`#G4NtzIR}g!Q=f`;OUtHq1LW%#0H8baD3`OGQ{h$89OZeum_V^cMj(V?quD3&e3+$wU$141iXQe|1b2ju9<@E$gx=) z@GWKql|Piq{&Vb=Z2Gk$VY9cdJ+j@{=F#A3{J)yd2gK9%4{ke`IKV%zgLcQZOIdI$ z-+nNBQ4D>?+Bu%i&C8Kf75Bj0VZjIc2SR`Jq}u0rtabCLv({-BgxklymEPLAb^~)u z=@0Qc*<3@|kG@jA1!a9ps|l-FU%3^`tiDC-1gX^z4qT9nY%qNubM}?pAINP+b{*S} zb6Gs+qI+I4e(c%LF6q>xX8aVw)N{*An~@T*x``*?rmU=F>>fCpHwq1zzGAABc&CrO)A1uMIaLW4?4j+Ht8<)?kfAd*KVNBEwvE-k>`N4jD2932~Oz(4mnyl?t%#L(dTpBi~$oq66pG`NlD zLiUK5=Y_+9Z}AMi?bW)FDf{#Nf+feSuzU{n^;o5h^}#hKt;bDCcurw`ZYlcHj(?at zKK1FjaI3>VZr(LPr9JJwNHi>&)y z`&gnJ$2c7?S~vANh5T#fj5-{bt6X_=q3;TJUCj>$2eqEzs~+pF$&G?$s=D8w+p(Q> zg^xXNsWqH)ajamv51P{2UE|QDx%$rGn_oswL5Dag3Ax3K4CwgfJBU4a+@4G4N5y^q zw#sJ>z6+a;QqYYa^g8Ffcs^{zd-yX?uTryAU?y^eGT!C5q|~uC46U#bj#kfzP_%Esk!Ab zaBZz`Qw3w!+S|yS@O14UK5@FVqqo0AdwFQLB-)R&M#e!Cu0J5z^vUn}kxwW)vXJHn zPteb$-at&Y#uj#P;`dLG&KiHP5IW#IM0-n8O(}BqizTVPHd)1_Bd@6cW>X)dlcxMz zk}?+fb}SvX#cDYr|3-W&N*0R$qlSM>L8e%T9^nS&X=>B=iDNEpWHgm0@qaCDc@9BS zKCday3-NE=#(O5+lqbu+hVsZaNUqP?CR^%wzNzHXd|UXTa8zE|OV@U=wl%^}eAO=v zW4?SsnNO8>6J?cLq%kiWh*s&}S!dcnJ#GVUH*~c2O4qfa3w?`taaoz>tBQ2ee3U$lT~ozAjHsZj1kfc*gn= z@~*V!wwd1fy*@yJR}{FV`I@oy8~2p!%i9n8ez@mq)wK#Mwq()mUj7$6p3on&8!MaE&FM`S9R=$<`H08%@4y;!PgL z-6f>Mv$kEr7!b{uGiOgBUkUlOhG5YTdT&hcH*%Q|cc0eU^}IwKL?LNroikJ0_KF*{gSG>q`T&@@``@NIo3gjl~D%fA27fh5dBu^yD7ae)F zepz;#RdP$uRP;sjkrgCIq@HPoQdq;{9KHo9DXiC&RFThf+p$^Yj02thw)s}Iy)dPb zzTY$G6>Mfb*l#D12fn`93fAR6ec3Jf^_Pvwf9kS+v_mI!pZ~7@MG5tvb5i|h@hl!5 zum3N;tN!o-Q~ynQL91PW`rX>IsHD0lICq=XPW7F}^DR7kZnL@vj#;W-ck9@(Sryc+ zoVt{;SM~L_I(|VNOQ>sPm#5T2Iitm4?9@HrKoWT11s9TO7kt>w!dAUjcIxLFJ9Y7x zhT`&4&hHJcoOQW>{YY%mu@#O~c)~BuxYWP)iwi?*R*v?+JQhD`Ybrv|y<8r8esftU zbRB2P!RLH!da#@p({`NuQPPg}QS^*ojPrz#a;9BuYLC#K2|1wy9dbioew7*e`sIw! zR~O@#cmr#o@WO-thqrT&uc|up{yygbIZ5Q2a0|DDs3f7*3qfuw2~i24q5+H5c7W0u zqtP;mtyNUO*dZ9JIfV`_bP5z(at?K13aDq$0ZKb!u@%LZwzee@F8d@XP;S|Pp7;A( zd+%_9U_0}^pZAaTS$nVhde*a^=ULBs)>^#paw60}MW*V2&Aj`z(wR_t?Dg+`vLt~F zq&@Tl?4cY?k%c~iJ~f*%x?}&(ow1vzM323C4|Uv4U0;Zvc&Eg1W>iGa@lM;JiKX!- z=#gwG;WHIAX|64Ni1o`E^xwbbj;)orXK;_=zK9!+%Op z=;yFu?}+;dz27^ybvP6h8y-aeHwiZiw>*?nWy4cU8O-?_zBN|iyf<5I*c;#>ir+Jm zc)s2E-?i%e8=!AX=B{6s4EArXgf0|3{YPV;9^l)4%Kcsm-Kh7zOZg5zz00K83(wj9 zl>0rO@0zE6Wx`7+(~Kdyd#(8l%{Y8R#NWT&_Ivj}pnA@Uq1#)Rs@uz@{Qm59s?8UR z-#bv}1bbq~J`y|jTfx0sz`>ir#WHYm47^Em^i3CkpUzq~8hh{Q*n4k(Dr;Gi^;&@M znx`_BZRUF=dWlM_LvtBnbFKE~K3LYh&D#jieL82^16I2o4a+RfFs3&b5Z1t%_Xn)@ zJM>?VPA94}@Gn~E&BM(F{2vT;j_&W-DY~z7$LJ^3=x{do1ZR2~ycB*mFX!F??85i% zW*=ozsj;gyxb3Y&kSUA9{a9nb4}sY_>r0`?S=I+n0Ix&f=h;o}nTH0#_Tpqi*9h+J zXK&_=;t6~9Z7;24``xU!!4vJ>)X%ZvuZ^;t*JV4rng1#)zTr^&sO-(xuS#bg68Hv+&=;T6ESs8nxKeEK{~Sa*}WV;MWF^1$;s3Hn&b9W)8XKSBS^_gub#jqawg z8F#IxljIq~_!yZQdG}r0-B-Geu!cqG0~v$2y|A%_nKj)m?u+rl#t>%qw4f(H_rh*e z-UZSn_Kv;bg$-9&E%?J&-p745l}X;uW$c*$8z<-v>5re@#r)q`&N?(v&3%Gju3u(< z!d;B*t&PnykW)4@zKcqnQwz{Bz)xw%hWBx=;IPk-?rnqgPaVZy!hM<>ng6Ht&Mxf_ z8=lkCoP)Jza8_LB3Ft%63$JF*3dd@A8`H$mm$S6@{F#E@vD4mv5E^vBIIcr;*?Q;B z>$_drp!Mme()F?;TRj_)pLyxjp9b20h<~l=4j4Z>)@Xdq0mA9`9`oNl_DlSW(Uob~ z)80nhDPVA-IX!xE0=A$Nv2&k*8;dK$jmC|_4Z{t=T^(>5sxu-1^p)etn;aZ)m|O7y z?6HP9f6$Ke2WMEn*zgBqJ1#gW9=ko!PQ#};e}L`A2J_AN1KIBxy9cGio^h-4c=;v2 znQv_PBqN$U0rtiH{G3D3dh`VA(J9DArysd0Vh34gBlp_y+!OstMsmpt{i58Mbro;_Q)(sODC|Q0C)1$VuFxkhuDuXw&Qx=dr@@ zvgS2+L<_Hi9zEvLpizYB9ghNJ4%)Ia8@u60!Grg(5z6I^L2^}KggqlYqH~dUb$TSR z3YQPPFTP)-^2NlfNS?YlXE0#w@3GZ8;#6s@tOnR z#6kGZA^339myyr0skJAKi5K9j4Sk?lx+~}FS7CpH?xb;ask47nsdHdhsdI2hsdMP+ zQpcV$CT>?llW2oInl@6dJ==<(8pb!}+p{k#1P+ew;S-NKzA9YUwE7Er7b-6PWDgES zo7Qu`RsKEE=E1>e(^`cEqD_p2<_Pwsg?5OuugGrVJ+;4f?R*6DWHKW<5g9`;m+jpC z85xn2lWvcSXUSGX=UE$EllQcT}rLk|BVmZIRg0a32x`{sr*DxeK@`D3zov*9BRhCn7Icebg zc5@(p{0O$iz=Hh&!^7><$iZj%7VMJv(0z2p*dSwvoED7RnZY-N8+uPM${J2@G1{Gj z@kADJ!m*zC;E?9;Vc^I;EQZJ1oHMGVjp^7TkH;V1{kfdOoIg4Yybp60=`iqJZt(rN zoaM~FV(gO5m~d7}G6Z`uj^^cb(JR{P+a~LtRrcHB_kPvf>2JnCusEDD|5)?8G?Mb? z`QN^svdZ%9l$TiSk1^-wu{M!i@(!!jE_s;jlIfRd&fY0w;-b+(>iPIA?=1BOFRy$* zM>Z1~o(=ePU@acJkGc4d@U(ra(u^L$x9hr-FleK5oHC_@);Skr&pXk#_sKRkzt@va z@izFLU|da_3Fp(u?&m7fh)->TA2w2_bQt>X}c4*k2#zg)`hT>v-?F29v-(GbkawSP47(i-8Q>XcQ2%2(<@u; z?ASxlz~jJQ_Nz%V>$M_b!_RT19KB0f^*z{<%xl0gcLtFf@ z3G(iOJ=TZ0dsclXX$04QXoU~u0auO3TZ#jAQ&QuA-B*eGx#D(lXLf4bM0Y$iep&Fq z6Tr0?xSj;A^{uv7Jq%ob1?(3CN7-KOgXXy~bK|xVC!Q0&7M&00kleGEv#|q3+sNOtk^x! z7wJlM$EV=>ec&q{lus8$gXZjaY0xsk9vWo*DY;SeRCDIr3S%s4}x+}FBBTHS91~=`D>ip z-*V+_=V^iL_HaIbbq!&tlF6)nNBg6LjxZ{5`g-k_YZrUC0A9`>e=9 z_`i6C@_A>BcIteO;oIK%9@)^azKeH}O;$B;w#^Kq=SauiD9C;NV+vPtm)?)K8+a9W z^v4Ealf@bHm#TlsUBTFCq3ihf(b#R3-C3AKcXFhxQ{hMOwhw1;cmLJ!iRx!>Xoiif^KaP8U3{F|EL;tp`gqPb*0SZ?FcmnrD+;Cj1CuhTuRiaBuMy$z#76$rMl|iEmu@$c$03Tt` za69_*`h4!&;*5SXYfR*o_$hQYP45Hyv8+Sj^}@Sm#zJEf#Fs8ydr=3yu#ZW*f_GSq z4YC*3qWlw;m-{`vu;U6tPCr#H!u#n$@2|!_fQXW zMql;4M_3K_8xP<-`wV!M>XW?mHsSUVWMjhJdD$!WCSkg#exSR4sq@#UmTdg8`?T(7 zKTn$k&z6bhQQ7NlHSNa!AAbeydqe;9VUzK*Vk_}A|5)p^Ez`;|@5BzItj9>BG6lDf z=S*VED;WE`z=1o#g^A$F9q2r6e;6DwxY9_vFf=5(sARr#WlRQ9b*0WQsepjKdPS6 zGSbWc0{K^fL%N$`WA#03nV9p`?`oqCcv^V^_Rp)vdjK7By(Q>7#6p21pd#HnG(xkCaYjvo&f7U<5oZK0O4 zqb%|V-_;pw+1xZ#TQBP#I^9RLiMAqxfk)bZz;=@{Z)7}L%8+gG9mZXHz(a#woy9=Q zr8%-2)4D+Elt$$o9}C^0FS0A`ZgtO4nYYjf_7l5xCcS8?Y(so}@ae4ouX0pw>c7er zeH@~3pdHuZ>u!<1t#{2Zde=k1tHqoD$C>}9nE!gOL-&ipSYpg`=hZCQpNX9nWgjbsBb#sGNjLm+lN8oB)|u~n85_+Zhi}c}F8JDqec$L2vSXT? z3eUjTzS$Q{c#qhJUU)lv$-;ls{G^=6B=3&094^-3KRY&?f7vB{!Tk4#&E#L}Vx6sQ zfrbnTS{skHv$AEszL@>xUV`(-$&G^fQv4)ky^r5l;h!WoTKH}9|G?VN3wI9O{$Lq2 z?LN-^SaQQve1%k7EoJ)5;;A7cj*qwE#X1^RY0u$OgTN?wzX_PTc&^o*U@9f~TiYp>8J zw#;jn`lPn0O^diA|1TkfClv->rPrD>(naAHDl%Sp`4RSc#%Ekx@>=c7&tqp2ePnzf zb39>18F?imn)|T!31nvIAHD1d)-m0}zqRff&3eg7zj@?|LCbSg7w3E9;R0*e7ocxj zMwU9Nd&MAY!${<@0rRY5dHt-(3cud)kaovb^|dCS811gx!=>| z=e#W)BmW9ab8C4!2l#P;fYAy0{kqP@d(K+8@GtP-zvxk`xbbT4oqO`>UFSF#`91VL z!sF$=tXVCq%A)h1W#2E*ZdNJ&BTC~xzCszjn45ajR=7~@-6Iv!eCB^2<0|@kIqmp}{^y{Z zFyYYAMrg8RaGUVe&_UH#$=Iu&Q}2Oe&{xT<$N3fwI)>i@PZf>&h;Q|6_RZD?wY%_U zbP&upI~b0Ihk*NxeGUBn1ZQ_oaBer9H)ZVAInl%(bg00nX&di|a3*&lWh8d@jh0A z*hzbY+d=ZZPk&m^24%YEQa2N3`c0o2p}m4f%nO?`!rEYT4$un0<0Zo1V$9Xoeb6L$ zSG<6Gj%{F8`H#E<@pN$6JoeX9&lad|cs{aLFqS@gUd1BfX9us1gl{igR>t>?-(ME7 z_F_vL=n#=D?NQDjSTnfO8y)W{VAKpunt>B~z|IZ3t&a^{^7~***f%Phty8-=v%Civ zvVQhlS*RtE-=DYv(Np=zUmUT}NCGMyJphFBwlvw~?OII|y|JV1XBf+N?8CVbi0Qy%5c^2k2{ zYsri?WMH<1s^GqUYn zWC`e0F8dS)mhiF@DP@$m`*ax-DPtsc%|oV8T~=wSvjRCxdR+gX`LW@Tz|7EJ_A*!( z?wmKK)cGxa8*nakY$5zt|AsHV6o^c`!sUy9p*^~H+sF;{cR2N2MY*a^Wh{OsJ<`uky-z-D}5xa*k7NLB6R_8?-NC`Z&&OdtY$C%d?g|l9`nD40zuJJrs?^ zzUWvw>;2^+>sUGdXx3@dC!yO1j|wMY$2JMQ5Nm^r%K(h^#81lrTfV|OgOopwJ?yFM zImk|{89Ie-EWT#Y=L_qrd%pG{dZY3{NnC3PXcKREv!}bQ=W90fI+d@8uAzfGCOVhM zLYD}QH2<1g{5J(zKi-aQ&F!o+ai@@pL??1Qz3MCv?)&J^*P+=iFXG(*j~7iNOlzm_ zp|A1#aye}o?}Zy41--eMFv;&~hj3SU_R%)+sjm`u6LCZ6i~8{`d_SMc8s_CIXTLQy zpUS+3IK9`Zd^fVsQ6k!1nGtD6yJgFw^N>RhCX0>ya8s$X@EQ zYIm%0oHhCV@#p8*_6eV!9}B#4bx*S7hXJe~T)GBK_*Y;0(54g6uam5+TpH!ZJxiQq ze(}=!>$qRf&@VUcCE_{~x0v~!TAuotM!4vq%ELA@V&<};i@?z1d8+qA`f;8vo^Q`- z!c}L@HEruudyW#v9%J@C;?;iD*G!n|6MYhm7u{6(n%}z5-n>6IOj{{1@Mk za_(LHuW;*CoU8Np&KUou&RWwJWk@!enr_W%#2#cjvQXi}H#-ZF<$aopEq+^?x}jqv zlksjqhBtWixsH*qQO~rF)<*Nsdf-8|T_nm4yKxiZ`;!nJR$vV&KQaee~q z+_}@;;7L2^j`)Y$H{C;pEXO>#oW9N>-0bmtu-BM;!w&-o8i%;!G0O`JX7lbH zuO#dcmp4lCUHyp7(&f_ey)-?08 z=G}O5%77?Jy!e?5WX>jJ&L&Uh)O`TY)V6>5nc8+Q*Kh0bdUI8;LcM*XH*xL!PSN~! z*2KmIX)o^`%Uf>w)H8^BmMA`p`0S_+jR-Dm_i}Jy`j@L>Zl!OODIEMa!d;r|-Z3B?toP`V!Bx_Im9RyG*;mhxCze=|++!2ZSuf>wo;PaV zjk*_h1$8b0AH%uu26#&Nnwq6ej7b8VoPB2QuDQXC=7HCipQ&cwLib~;9n;KR+O;=` zJ~VnV@)_{g#hq-^H?{R!rf*rXg&x0K;Q68z)qES?DA~8OyLa_~X*)cEyuR%FRl@dD zr|z3cP5&%mUv5qB&dZkwyN9rm$cew;+fCm+*21_5_wORk$YIFQ>DDt91KAr?8sTS< zx$Dc+$7$b5#(c#{WN79vGPV02@TX+z92JaNSF9&Af z*0echt$s6MA;O?*>qM6YlS|RH`K@iuihW4CPEo&sFKtm>r@$-e43j#a0nPu|oYUwL zJ8S$tF-Jc1^t61*5Ar9W_c8IT`Q^h$kH|+B`%wNw)~Y7HcPwH2Z0rFu{-!;k@Obw* zb7rq1i|@xk)d$TvAx0JOvqqmf`HzVC_>e~;{**gC%e}un+;iZp?VZuWEP*|X6eO*yuT^*xa{IEVSH;5Qnc#r*ph#!Ncfep$Gn+;3aSn5XRJ)LDo7M5cQ-X1!^*;D2(`*r;&t4%4?1 z=QjDYeY|}2MLzW7R^w;KK8G(||EJ1M+0zMv%d89b9%&9WGX*GXu&t4|Lx_^A;0!>EbNtE2ghFV_H`7>b^ajF_$N6p= zG@-Bnd5*Jt8Ogc9sP=Ud-o8%D?r>ppPg_fX53p*5u)>nU_MSD80v1h^_ z131SU*WkxKWQ|5}@+9<4_nb*bd>WiM2;W&9n$jYBpr1NP=&FT$;_7~sR%uk$3Fwjf zGZ)&MWo2ck+;^}adeYn1VSW7``{D2TxS_FDnE$VGRBr0O%GLVd4a$)&a}&PoR{vhw zaU9yAeVudAj#J*g?xko)hi^;1bnWW27cqeShOM04tskrP273^EN6UCO2|BW-Gqe;s zu^YDuw-om{ZbP3Erx^eE5-YwQIdYfwf6);?2OasWeT#>gW7;RXm;N^oPKzdHTJb9N z+0caS*u&t0Xw0*Oxi+(XVwGOlT*6j>W709r@WP%@82BU_^ABFw*Hzvmm&Sb23;Sn< zv6pocIdG-i@;2A}y_@6nhF^4g4b_ou!A9qDdtUPWItzZD*C&lx)2 zh!!mg8QSC0n41;P7%RNYqcNiiQyJSl8Z*i(V?a3EJP`ZJPtk%u^=;iVPGYkym^qW! zGp2pTrhg^Qb*ArMXt_o{{rSAYfy3vFpB?LFe7$4yx9LY1{Ml+Krn^*CRz3v?Dr8anb!rBWKO*u)Pa0vbRl9G5n?{%Et6IT9S zv5j8dbn@mxKSTpw^XP|YhvrNJ^KND4nCMIF*Q&mHFV8t(qItSx^{l(lPVV(b$$(M7FED^%YS&>cI;{X%;J(FEgT(`LQ+j6V~)GCEiHiMr>=bY2Ntx{cG($1vXJ>==5_HuvWY2{)sa6Ziau3TH_ITE7RO}qq2{TdWADX0nQEuIR_O&4r&JvYY#8$08dMU zx1G0}NtWGc_Ng7{*$L>`flUF<>a?S-5OmEw^D5cH%$GvuV1W5r-Cec=$VNqxoTu1_ z<9^3V-n4kT7Jh^N@$}|^bLv&fSsQSUuMaq{U_Y=izr@@LVt>bdue6kTx}A3|(7n{@ zUe;gnriJvNy|DY&y{c>wZ`AoqaFYGX9w!Oa#hN#5>aXT|IeQ@Jq3unj#t!8N`rcgXl*@m$)CtR9@18BL z7f!9g?h+k+bnLjof?#^lfPU5^^VsjTbpCEbzevjt#+Z2f$1hfxGr(T>UkT48yoT@x zkY@_7;?6hV2EEgJ;B|Q|{USR^V_^8ygCq2Ixa7c6@Z;b9%zUEH_0@N`n7qKeki0t6 zQku)%!euuiSC3;1%NfhDjOne6?JdYB+>4oC#+Z*`>`Na$GU`|6%uQlufHNXvMy#s_ zK2zyGXSQ~-Uv294&$*!Q_wSsd9sjWf`pBMI#OTF7nKNz&Un* zz&STJ;2fP2aE??3oY#P(k&V}sI%kdq!V7;@;R^!JiTj9KTk4b$_kAyJ zo#N&Yx4hIj0e*@Ge#eW8DsFYaSt+0JzVN1oSA$N&s#53N?0}O?`UPIvA1X~XQS5x7T*=P}-RE+pF)CHu6UG#-!J-NA*oRz_<6>)Q|l}(ytSE z+KV_7A^%0@iu_vUiu?t%2{_rSiI-nZJbnXXQNmusKhOtzW-03htqb}#&tUFoEEG3I zarc)x*~HmZrOwi)!>lQIixk|`xG2pXq*2%^=Cj5`VdK28kir-bh22aT?{3FA+sOMH zls7S8WC61d0scxeO7&5X)+pR*?8Zwr0OpDxqWB4=CR{dU(7V=iPt5V7*1G)b^c?+epwFK()VWeVFuGiRZK*R*J}}Ic4@~>ZXI%TrpIzz% z@y{F?6Q8sv?fTsFG@#9SJgXr&`Cq7CeKBW6kb`XMLSElsaFD)OUSAWF=r3oHB=ZEJ zx1Z|MF9t-ma&Pnw_!IKTcM>y7os9IimQLl(gH^yLv5fru>kWj9=5NT21UNIliTtur zm#*)`5cZhY2c6_v+?t@-YhF=3;^h_9!(VRLRDEN^tGM;JwYW97)wor-<+x?ICAdYn z7jddMX z&BRsV?#Io*O~u`Zn}VB!n~0l$8;dK$jmC|_4Z{t=U5y)z8-&Zp4Z!um^~7c2I^!~N zX}AzBIiLRHPUDW_j^LVchj5L!J-82WyK%d4@8EXe-o$OgZN_cFy^33pTZ>zRTa8wF;cXxhl^X5kl_Gt?9=R^=X zfS!L{s?7#-z}PQ5NBH;vwi<<=H~pYc5D{Q-(RWA)15?vYY(_$r}FL*8X+P&t9yMtbt5VGZ2K14KACoyHs42{QNg%mV4+pWHmW^qa$-QkcV zufnqlm+kqn9GCANpnVOa;i1%h*b8e=*eLkk%rWuAePh^*OE)soeCU|YbCgT&8bw?2 z6Wh8))RrcA%~aOY1L=#_6FJm5o3Whzd7XJ?4MF)jH-b%f)P+r6<8D)iwU9@T3Pbf3yzfI15Pve)x5qP=XTr04^N;kQ-jWjr2B9i z|M&C%X1n;Q9qr=DciP39s)EkhnL(!sn%gv+bj{>DO*r;EMO#BHCigU-2X+=8HU zIIDfUxo7)$(-6|XNWLj!;!XMO^)8RUF>Q2Y3q!R_NML)ynrEF;Zw+|}S) z0NVog!q2SYe>HWlZ5MA@!#87t{BGWMGT-23iN;&urSAP0;V$zl>_e6lW^A=zqHpPT z!d={N>C%#f8NBw(V%&w}rYwD@mi0kOS$wya#kVO7-)~nr{a=V1%$pWXlrxp`&HEN> zdq)ipofsH7z8xIO4>(V*ap{B!e>6YxA>n4e~P>(9l2zSU>i zyb>A!o!U4Ro6}Oh4ScaNO)&ntGlB8MMuR!Y8FhOqW6%}55sd+NQPr?ls5+%jPm(T4 zI`vOy{m!tSv}C9WaFWd z2{_*;eS-AUbhbBhOuUBs)0HQCdWrL|yhXKKPLDkN*>a@TLThWmi3{%dd>|rVKOIqr@9r+XEXNgvs|s&t3@{BVKFfF}}$g9u&zSZ!T;7 zk@U~+qwa@Z3SU?L9NJz8Jj|I`*2W6UrcBuk`E%zw(reC``WQpev~$#F%3BtUZw_`n zw(>W?2N;9{eKQJvB!ncLcyS!~M8j-7pz zGW(P|r|?ag;MWOotw3}|_U#{W*D7-vxDPWlO>_fV7!a-Y^gAoXizuVDuRi>93nMRV zL?6z(JZ3!V3nS0zAN$|iz;(uS)?n(Xda5W=`K|0o7-Ay}&vM@&yZMr|kQ{*=FRf2Z37{S01#k7alherad8>GtMeQJk0t-$(a?j%|6Th zDcJt_a%hNgGvoW@U~?yk?ytKbZl`rkQKU3uUWLw74FuQB+hg#~hcCy-uL8M0d8{8N7g#;MKa`@!^ygFmOtTZwXF`G;pX-fb$^Ucs_J(WHRkp z;p1bM>mqmZFZ@({kQL(E)3C{B^R4r6W_(|}E^=#Xzit>38I#&C(wctZ8(u(J!X+zm ztI?a?4d1cBUua3Q4ekabC1=`wujI^eaM_gq`jE)yy>^)Y+lnJaDdqd~B#F&eg8ZYw z{RDXpJqM?nm`jC>Q8Vp6d|zpNSs=4$d|*^YGdMR_@mbI1G=p=ZRl+&*PkS_nG$%yI z3ZcKvW)5X$2saH4t99o|Df7gi8^{mt+-N9{1X9|_{RwgIY%DhW*;S>^KFV*Ttp~G` zOVG{6FAQI*@Hv5a2pFSBjiYbnE{M{2VvgocdJ#C??VX;C%>d$bheWN1D^vc5xRNrr zMwz+Q$HirMf#KD8?cBMAKH1DI>7h0E|9NQStzUYy3RwH?s=;o`#Ajb0*-AVz(nj;Y z`1;5jsp$u}{V0I{muS!92VEi?i3j)G^u2~f{)2z|>DISmSmb3tzt$G?+29nuDZhMp zP$WH_^YQ4X)4OsvwtRS0x_su-Tj=h?9hq0h8K=>KcscrQ)3&#VN1mo_ zsjzs$?~7n}5m-Dzd@3xy#{V>SjJ6oq=S3dmpSz1pU(H&Jes5td=;I;u%dH!@Pxb2p zCVpL&0qc6@*BrKKug$kQt0STEO@Gco><9wtdK$Y-UXKule`@yhvkW{=w5V!OV=q z%t6UBpM!op&)(CrTPj}D+^oH-3>n7dVYMHQL4M7Q`TsvSHpbC83C-op41Z&MkXe|= zMWXl3&{@hT5`B=b`IR{0;%7KOdzo7bJKR}w>oVe9yB4)oxU?r2oRyC(B%UnVu${Hm zS?0Rr8RVYglRX$K`fu^gIhYN9@^Sj`$VkgC&hvYlG?H;7Cne~w%Bl~BGs3*Hr~JwF z?9YdK70pHF)4EiB=!N{+x;D=Gp36s({S42d4B@u+#>G3u|5YaP|IWgdH}RGyxcaKa z8VXrXa`&s)@X|JeL&R&ox6-rkjf(s_wXL*;yRzK%J?me)9dZie$NK9DTkY^*A^F5! z1b?S~$s%e?I?kmnp-3OvAURMpbQE>Bsk8is$hy=zsV~Vo=Rdi#(?+i9`&9R3M&ED+ z_2*NkG$57BEzXueV$L9-}L)SxaMwaf2|uMk<_}WwY;_RhH3lg>R<~%Pgl{>zb3}pI|vPp z6{i1T>F7XC^N*mh$g{0?1ekfW;>O6<_VX%k!;UdIw_{Xi%~NGsd)x4cJ%hU8A>6Ix z*pHRQ53lF`Ht-TzIxe{P0$1Usv&=Ydy(#iQ*5aiZ^V>x-w+18QyVfpUPFyo%J&m(T zSzVu7+Azm*UZk(n&{;M-W;xIEZQ4>dI&!DNp<_d#1vi143F_5-V5K(&CYwI*9W4G3 zf9u9TaXS5+8cMJF%8h}^d^=5>7!U5--FpN7(95R#-2dtNFDwylR(#doQT$(}_fmUQ z&DVEt(yrEibw+bQ|NZ!1<^Inf$$uXI%e;8~U%@~4p!D-c@L#BS_n#Y1Cx@Q`&sUpv zH_sm);9fiS?k|Tv;{OI7Ejac1^H?X3^C+g@=SD}md1J5l-|b+pBD>1axue#j+$T$% zN1KmYEmkCre`x!LFmL(T%==_>z}Z{YIWmTKIetK#-CSzsm2_lb)_Dc!o2)>~t2X^- zoT|=2`y@AO{VtxSbf?jCobJ$$Hz~!}JzZ8!AaZz{OD}W|h_)WJvV)OBgwdW2=D$A3 zojkPXR`36sP-GAPvVov&N5g*(MZ_O_c3ra6=(gUmq_fj`$Lqt?t#mm!-GQj^O7tOu z&j4&g>_+elnx?sAC&4{vWF@+d2F7_WW4n(rYh)Z`EB`@P_q;}ux>-9ot!WrXKh1ky zno~-{-d}4P#0Mh+HKN)Jtb{Q`+In4|J!T7NeiyEmYdR# zOfT)Glzvp|UO3uZAfJBN#TWDgIpjz;w>>X={XiuX)mm1vc=AnX@NrzqI#>J2dfzya z@A9m~XxA6NHgq1H%1zjGuHbyhPu|DY20ytv6mMA6De}mBuHDslsLMhBn#)-+jj`kz z@jpW|$uE1gclvS$f-`H9*TG@wOU=8&;P$W@Xj&lsX6&eX$7WO3Ja9|j`$+o;Yh&F( znn;$2|7IkPmpH;Fl|B7hPmVq?)-Cf*#aR{IITQ3P=B8xadhVdxU)?>@3S;3Mu;WZh z#RJc{dqH2JPTfPXe@03=jhmNls+aB_!4&<#{<4&G8Z+9d^6vK1-Jx{A_-qOAKp#*q ze**Z2p9Hs(;AIm0O@ika8=qwaGB?+0ukGB=nVW;Kf8%_RY?QUvR=)qmifibfY$yt* z7`GIw_E9VJ|W$#l|KFn!A|^5@XG5JnB1#dK>T`L_`|VIeb>N-a&S9+ zC~{WUfOK|8Gp^+xCFu3`E`iAtn8yzVZgn2&64;>chuh!kbo9Qf(rCOKzqKTMtA zMz2t~xmw?(*-UtWzQL6r5WejpzM)e}U#V~AzP`iyW*#U#clb^g|4=)Efu6Pd;Um&L zowyJBhK|;u4%tCZ+U?F&!HRodV|M7ei1omb7V(_CB|rTR2D-Pl!{nzwL%jTp$ZuC)b3Sj`=S|)B{uA#q<1C!uayl5U6&yC= z&(=BWpL2)CDc-MS+-e?#j%Ylv&F$^X-8Jnq&dy;Yd32-hK)BoNtM8n#p?$D4=HdPTmGtGI>*eww^F%8;FTQd3XOk0{ez{@+J*UPMJ+QIib9USqIUNF#Yy^Llkx z^d86{$U!?#|0)z2PMGA8`b}xq{{`7tHn#S*Qp3}vlg0+qe6L^~_FdkvNNt~?kL+ba zS6kOji6NGgn92UcpmtI1Z)@$6;QmpqA&w8??ooJ;cvC|6k3w_BuM&gJp8w9gpZ16( z?k7EWuO5YNoARX>Ei^J0deJVrhjwS)>%AlUpm}o+qRZJFh+|itZPv_vf^qH9#9Po2 zv-T{{0uPT{KdX;5{8cndcAA`vFt$_ruHpMrv>0-0(6F+rQs_w0 z-=tyPy!?-l-=$%nOUch$Sw0QC1Pu#%`M*MbyKHbu-U5^N%y?_VA8;#h7o}lA-Z+JZ zU7mvH?I`QQ`BBi^c7FVMi#tD3<%UPV4dH!rm-BPx80n@BZade?Z?3DjleE%vNq=MJ z7do4Rq%C zyKEphGOT55YdO8omt$}&%jJ`Pd+49qv&}2#b@E-Ub%^2ZZk-NkB^&$vy@fvLjO%^B zb?3%f)m3(dsaJjZQET0o=mTZBJib?KyjR8#$&=dtNNX9FXun?u?H}Ql@onYydKefiec0OYP23l8r{V7!bMbnO@ofBLFm!DCk>unr3U{G};O$OcLW&HhoQ6e~ z&aLeXf47f&Pxy^}B2CD2Q=!*Qgpc2w6VbkP;&drG0`!A~72ef1(u`mAVV_7jaZTX1 z(m#Ms13km}yeIocbYF|f^JDTH;r?;b6{7cOn&PH2@yH>~$r9-ti%gy4`teqf?s|D{ zslC>6#)Yk6cT1%In{~>&)>FUz>_&`VYC&ERhe=p}o4wLWsyK;Hk)|yp|{$@bHdZYkf zI>Hap4@7A*x;;1j19@)xu6eELY2PDZ(yM*!{b)VEE}cDi(--bVHT^>;K|9wK-rh5k zAf2yETRGpFjNWWr;TvO}ar4o0a|U0$78+ALk$bY%a8_p?HV;|va`z*?){zaradFqu z*v@2KFZ|vN9?!&HsgJdxlzi#jYbsrbL%d|b`m2F0ypKDvoiO&q_6MPOq6!|nCxi|Y z`?DRSTSnOOP<$9`7w2opDVZ1k9i^KKP3agb1&Ryew=cQ{zSYTxkYzd%7#Ps8q%rkiXneK zVHJOZTq#%>xgg8zi>(XuCS(S0Sv9RrkEX92!`b1#HmHmo*pw&KZMIL`iQ{H4!vWf63^i-Pe{ z^v%`hdvo!M%!obHaxxh2Ok78edzNbp@~+qBhYKI4 z59?c#PWm3wEcDX+S^f5PJ$@SSK)RlJUYhrnhPix}^TnJUGcq4@)#!S*|8r9Nk_Lvg zq_ck3w$0mbsjFB34E;3EkY>wYE=Y6y#R~39G5FF3&aCnFm34|VvX0)*`Z*QO1AsI7 z+F7ExVRT`#vnjwv=Jr2Acl%n!A93$Y2J}Pkz%bW1XCs|i$%yT){UiF6yvSa{PSc(h z=%`swtvmfnUc`rYZ{R%-J@r@iC%FOof&L8xtdH%rjKLnxMppA)1oMf#4%s@d%li|& zf_Z9(EJM==Q|5Qyf$sNm>Hb%K?aA|mOIA=nzQOmnez7}5=dlmmn8bet+*r|rxeLA) zkXQ8AzMpb;g^c~Ey&F0|gn4}=z#5D*dc-GA2cR?jr_AT%S<-7x@3Od`vq#KMhTec4 zu+HmQ)$7_NIllePNWJZ|3qFkuYUdzR!e@Fv)oU4dMLNlClv|Asj=3heY)mLzyFVWu z2Hr`hEnSIZxQeG-nQ{XC(cCV>m(AQw_|g-Ze4$V0yV=WkBk44MFIC4C)G@+Kd!1KD zkypnRpHYYM4fgU~PCChmeNDccSWkTD+p(MbU|+?XL5!wm9>+DsHc=wQiWA@nbb;i-e;m=oG(zEfw*SSX%hyR4)j(BnSItyvy z@MkNo(Tl@3`!K2xy73Tk!Wq>k9e}-xHYg7NABj`A^rQA6^y~Q@c;}?laaezAU&n3) zUs=bqPm3Ny->lr0eJ-Dth%=#96)Od;h;^zP>;paHurSd1M zeDwE~>>>TH%7+K2{OzON@`GOa>~CiaHcxx7c~Y>UKKOvq$A+){U)I;%t*??bxB7Ds zzN`M+sWtWt=HxQglDzSuFry3St(-}np>6oUqb{s{TxeQBhS|q<|GQ!n27IsstQev; z0vp*%r{eV6)T6WM!ZY0ik&4s5Bu?um#j!v3UMf!i7jeQ9#YwlEiqo$WcPX4+M_e16 zUZuBt!0A=I+XGIYT*aF`wczKfKr8;*;Ph!|vCdK&8f;{Ttl0jZ;BFSM0-q0or@i1A z?HLCj|5{~*zUh%XWQ99Y_|)%ThyO;~@~QGrUBxTljiJly(R&%06Pt*~u%}^t!XADR zdpAXbr`Z$2zU)!(SLv}WtQ(JAMmc-JUrK)}ggpT|Y5XR}ReRb^j4eC&XKH3)dkc@cf_F@twiQQi zp?vL&kKjMCZAfGU{|RIo>?>;RZP!IkaJED3>eG@ODt+8N?XBz0e!?b~x9#KJ5$(m6 zuy)g)=VD-2Pya0P=sffS{@wkAVW#~mqiuff^(Fr`UjAp59#|;9=EP&I`LDl3e)b~h zYp$37QPTed^GNR)A>$Oct;=nz_}$DoE-$Vmz1pcZ_odGJTI>7l_Nt73@XDA%9<^2N z?oJuwTg&+DHmi)=y)tek&o#6=vCYGkZI<&bWZa9jzdiXzd-;ZwFSY%HTkHPp_N$D~ zdu0qFPip&fTFdzC_N$D3UKu^eBRXY+vl_ef)-qhY<*teicjJ0DotJ@swF&ulL0h^} zP*hh@f0>nS^aaph=JYY?5psDm58XnYbPGD`^a|?{>Al&<|KIVXelCv)vqvZ%@nz&3 z@q>-hEwP4{j^gv!U(e}@9oC-Y67h-c#d=?ww+T8zEAF+5-@n%ydX#Uii*wO^7~X*m z$aeZ^zS&Pj&tlG)v8FZaX7jcWGB|I}XSc3(we}rJKagRRUiZG1OSTGJ6G^}JmUv>e z&iu^spz|Xy zu1`nB^@*)i+#q!GUU=WG3hx_Rrtthy=U=_>-rW`6JN8|LcP@46@K3^LbjOvx<=5h8 zU1klFJQfC~nu{0m$spmE;*)2Q0WQTSPY`!0KKUVWm*SHLiEHJP*cjL|0watJ`2Ub! zK7*_`BF)O)B6}Lv`dNa`fU$veYZMx5_pX%~zE{pu-KbbdPJr3V19(H+Ed7rw9KTy78+_QtYpMoB$8G5k7ltz#(peHHKjRZG@E<}tDkGLMmUka>)Zydw)@-gqlYN-_LU{rICN|-<-E#6 zlzqFgkLcLE6aPF7*>2|f6Fh8y-1g`tx>2_2Z$i7@bFC1KOL@eRKG zP~ScH25*+}osIp+;ve0_n_1xLtCq9uJLs%_jtnw`H(Bm2agLoddqu4{4n3+1e~b-d zn#CRXE_}Cwt9f6#DRK?v6&{CfqpL+`Ll@!N18uzux#fP+?h5_QwY$G=(VmWd8*qQK z#Hry8fa6aa`?fl3T7IOMdPJwEp(}Xq$LJOst>TRbT^)yHck_pSo{t#M!C?F}0m zioCo*XG6pCp^@9sr%0yt=UEbbF*;IguZ({jTM+Qqzzw_+TpwgEQqE_>6kQ7MwxBD> zI6rUwa)%7IhG`Dp3k|t&-map(mzuXfAnsE0_PfMgYTmv;+@1^wa&ougiu% z(4Vu0(#`W8?v&`i{~3K4daLRE*4*(-zn|Wbb?%cTIzL;_`B@{&Z0s7j3>>-tl~((~gf=KkZ?y6V-l?(mv2N z@)3Pi*hl>TW?a`uYn;<9@-lXsN9a?M_y*?7%%D|tbaP3(bFgy}wg+`xg54s=@UOkK zTjVXul&-K-x5!f6TWjKf(>2nm+Y3usKdl^BaMyOW?UA zUuVth7}=`zKbDHR6QgnB<%KRqH_>iJ{)>Nz4K+4b%`Wq;J`tVr-PsK{-|+T^ zD&CbO+{sipZ^7yt{@wuoujad0`)5_?_qyRXqZ8+Qbe=WqnaubC|XLhuf}? ze8l?p2)Y<^KL;||QSuJJwglV#$w#Tb%az-6?o6g#f6e2SEp@yv&^1^Rz3cIUmahMD zYjhU=JlsnCBagj{-{mG~DgK8rFQY3HopJe(p(`t)D{50EH031iX`xRJ|9Vq4e1o+i zFt+5iVAqmp=)1Q@J0Lsi%_QxEPDYCOVvKGbVe3uTTHr+>9x&-Vb`#u{99Iw_FT7;F1aXZ_Jhrw zI}ZGzLE*bIBh|7=gFkE+AEchWWeZ`l^Sc`1cdnMm9;o z(P_k897hL&qrDA|I;}X`%fr#`u|upc>R9i+!MaHE)a*0U$2rW~X@Qnu?BfVMkMn(c7f6mps%2*{ok*@!|%-6ZD;-9c*RvGDUO)z`&j7e1hJj)_% z_CWSh!G-D16u0it51<^?3*UC>U}~Mw%t$}#6wTs->Ea}^c2Y8MZ79AM+O`kc)(CCm z4TMNSdmQxpAmtx|mT5lw^CWZ`Z5_%ysV$33?l%4TwY$&%A++1?(jod#6EKlqSU|Sn-_A&<=^VvHJ zb&ecp>;z3hZl`>e^Fb$$}knH{YW5=y|SF_#QXhe2dmY zL)9?z5+B$ccuZdms1$oBE8SqfaHCw?q$vQRQ zw6j}%K{i^|DYAc+3nTTxf!1gqoZGFs`^Gw|KI#bLv)NH>>PT7dT*uzQIpVLu7yNAa z_Jnlw7(Y)AeHWdZ**n>Mp3i)PG1Q)<=BjKmQukG664z#aulMrdR}y!rHOqA3+VHMb z9`C}xmpGp;B#yvS&^OqF-2Fm3gx*2pJ_Ef&E&der4$uSp{_}f;W^bhmp1*|fsovh; z0O1jDWin0&Cu)x+E7sE2TKrSy40GJviC|;bBs&?|$Y>1_=Kb4D_T~TB&njM_{fd^6 zk=dGm{j8zj@=ocQ9)AV6{kbhJZ|`1LF$G-|xZGxc>hcTw{OWl+*^jtO!LbK%ZQz&! ztIjE~O4X^I)E*S@akK{oeEdBq!REN9k9i5+dH~*f5Z-zS-r7bVGY1`2A8YYffsdVr zOt&L|JtuUd?fQJD*5|q}=CtOYM>n?jcIUp0EF-!h-Y0rud*S;C*Vw;JxDUrcz%kdg z^J=9D_HcK<75&9dofol4BfPG_uRCSvw;6+8{Lg#$f-x9YS8)nml3#x3i_ z=42W)BNLZ}>xr9yn|S{IkrS+Q{rOQtdv%ZLzkB^jr$0&ht363SPV%3|RdHtDkuf)M z_$!FhxGK)bc7#=`O#JVAWrm6K_awLLjNI~4=R3sx)Gse(zCG`ir@5m%$6tkxr{K!7 z#JA3?e8yuCZZ-2NMW1&Zecs9AZFPB0%KoL1!Tle|v z-PoQn-EsB4OZUw<{X6h2`9<4ts_#};@jT-s8#RC474gP_>|yqa-qYUzeQBJ)+x6Rc zQ>g@=$Na1x!`pzTtJ?C z{|oD?q^^=1+`7tdvWnZ2Hgotx{r|Wf+bnpz?C|{hw(CshfplYQR|H-r(MJgf%4Pe7 zeks6PA*)rZ|i56bj$ z&W{TbC!9--q|>c(p7+iu^!WgE+Tap+_5ifH6kIaxqF&Q3{CB8B z{ZX5ZZAlsH0oGRLK801*2zaWab&ug8tEo%8quPqnPZw`TYyHQ&rmBy64#M9L(VlbE zXR&T;@#L8TWE;uL>%ICu5iBN!;Y!nIn2DPC#~jo>Rhbky$0FCtR!x_ z-{zG0cbfUf9ZahI%K_vWV;4hzT5VxIH6Qh^SWfKFRL&gsaK}}=EsH)? zaF@UE{!aY)BOdI(cWU5c&9{)+Lz)Kuzlbfy|BH3qj%Qu`z70LxBIe!@uWrrPZ*oVx zSzFqOkjA5#HZ$>>{lL&}TwVd|Gm$t?u?8eKNEc`Z1Lw+711v z=*$@H0+tO8pDYPLllIOCe5~&e@Z zygDmSAFhW!RVeOd!Y>0SHW1Ey;anIpUu`I!d@;mchW-Oa--|9K+up<+ZidF*hkpcr z3VvfKe(F#tZV#Fq_3aF8^f`8t_!-29g6@5$X~^HKsnmbT&i};w4c7NP85wzei0VGx zRtI3sOHTd(x!BM*=xe>`8~!?E3eiiAi%&0o+7~6x=e73#h9*J#_Ct$Yn&hr|cO#F9 z)`{*&7K6u}w)h=-1Tc?_J2Q%e&)mu4qbs+RZ*hRBs*FzU;t$ zuX(?-R_jULwd(G|ZMN{8{ilYhr612{-)R7IXBvA@?8kik^M3z+M`ORWcf8c^5X&hy8dy}w80%<@8icK#I>P&;DnDGRmAzY zkt{?05`DNIHz>gSr=HBUNM}U4#~NSsXdLh}y60+S72W_F2Axy+qjg783XQABW_}*$ zl5)}am?)WOX;H*8el)1ltJmLQR_144C^i=`p zUuaA6{(#euGIdsGZUCb3ryBXut=lNK9rZr45j^?y%`kA}t z4RgCH6gT`_X9YI%o{-K8V8cBikX4m^?IO+z^epo4=1hR&Ixl43uatT5Q~KU!&Hy(vXYhYaTq;eKt(loK_&+4hpEEXehjDW04swcF zFTr2i(w*L{S8`lkR;_4Huh_qVD@GSicn0mxMrS4XVBbDrD6quVJu@jgcg{L9w#Hsv zbi$lP;l3C3DSII_kN#}eIuiezw2Qa2Vp>a^aWZ$vT##qJo2Oj&O3|20euCi`fs1 z4#-XzJd|GI2(Vi2_X!(D9}c$P#yq!;w%5k%y*7U7B5l0-Gul{?+D2pdW{q&i_wR4( z^Q=Q36)d6Wemh%XS?IR206TfX{xYwfcU+{M?LVWPpFktqz>+n&;d|a$Cc*Ul`kcK& z;HfvO4*}0B`&o-`qrUIwgKrnu73Jl+y6bQrabua2>?iHL((IeB>z){Ebo2%nxI0Ph z{}b&v1it+kx^N*b{Fb;&;lgh&)qd^+*k4v)=Fp|uzn1p*y$HPGpHbh>E?M6I>g(~D z_1)Xwou|t$kl)!y9ro7S$nWeKJjmN$pTXPKS?hWm`F-|(wDPufvwy}pa8HiN4!H9B zUw)TdVt5d+|A6lcdE3dh^7{q6Ej#woI%{z!WcC%@FD3c?gUv2)J4?9F+m7%p-gd&{ zZLK^i-J11mQOOh1!I{O8eQw~z;a3j)9vA$0n>XjP;l~+IS2oD^;P+c(!;SmS%LZ8<{9XWNhNc3) z+6&>gI~YCjnegL1TVV5F$OAgpBKXyM@LNN;_FUD^<$TZe;CCF^;89-|c(+J*@{LF! z4}|}~xeIvyeBIkco<FHKn>)-1c2i8;Tw7*eG`;fQV$f)%@>ab5e zpJq5|EOdxI{#n4UW_ulPn6p2-2DzPkBf)R%!QgqG-f%BGO1!0%G#A^eZuu>CKeW^2 z>EhCd_sOI201mC~>#g=(AVXKuK2K&gciH@Pt$R*OdbcvtJHSx;2qkj=u!80oxcn)-g>nov5}`{|3;guqGoX6j*GvmaVz_8DtSou1J9 zXQ{`G8Eq~1#yOAjHO78;y(ee)Aw0_;#|!I39=!NElrfWhLw%BOCUvtG-%p(e7bv5t zGx9y-kQj{K9X!A;-IpibbCcw4WbbA_dpCZ&Rn9MI``wh|(}J6Tf61wj+s^ZA>MF!P z-{1}j=?ygw(tFGUKgehHYM3j<8;SSzE2SeI>R(hat^amtjK6PEAbB5qi9a4O_0@^? z45Yp-e>L^(^z8&}bUA`G>tS;S^X$J{i@!}CzijDe{IXKZf0H!hXj?gMEbdm^Ex4O; zWw=4a$!ga)H50&Pk;#u{Z-p$mw1Wzo~ zT(sp=V?!|epf)o0+(DlzKJ7HfPv4j^jCGT-^ugXSe+nO&?h7BOwnErp-Yh?vRdhgh7`8{oLwll{q4)R_T@rxg2 z{A125e$5<=enb3i$NaaR|18;}w{#25f}T`XzTdFU?i#A%`#1MHGV4|T^UCh!xsrcP znOU7G;T-w27Wm<{grm9%~%n> zh5f{%qz9#+92`38E$hNSPu|V_SwRox!c&83HN&j(;IxUfHN0!y13ohSh42x-6*Bc$ z(KYmA@`XNjH=$P@nZB`6@BFGw+6LwDSFRAopwK1NLFGr_IPqnsYSoJ+7_%hl`h6bkvEbhW&9&a&^ zKQetdZ`xLc$v0%`#=l<|=C#c-?qV@zvQ7$Rt6vxw)R)#(=f&#r*FpontA@v_R9?&K z;nhuX^gdP{^jzY$x53@!;Evk|XLb)AUS&O~boGyymdU@VduZQr!c6;exBVwy7sfsO z9~~!bnfj%uB-C`AuqJdq>-6u1{y~_bhonC9>V^CpamTeUFlF=r(VaE5hBM1` z-RBpiV~-Ea@Qur;uOB<%Nb%IbfCUBI-8UuWaNO|Ym(qi7j%nZ3)@Or5w|VU#xpQhx zXzDkY+0b7tcP_;@rEEQXQwQE~?#Nrtop{qZ03W2mcj@q53Vhee-p5@RF9HL##<$D#Q6DR*^VqVrV^_|yD_ug=6)SBC!;@Bx3ptGX|-@nC#> zOMmt*`Lp`&!v3}8yX>*pJ~))tqkin7Ug0ItGlmasK(B~$hN{??-`0~cCdwI-VqadH z*)zu(s$ySJ+f}lY7RX=N=9B*v`3u?}V{D4{l0UPpN`Ce_>wVkp&`y-|6^-ynlyh9j zhBZ<43YX>brUm$TiWNJ9Hfx)nPWe|$dcxdpzTnRWEpv(9dsJsUR% zHy3v|?w+&b-)H~G%+zqa*oy5e^>H_>?t9C#o7ua!FIRWoWZUP`U*EmKu}c;!k3!=5 zlj9b02dopX$c>jxJk85@?y!>5EUq6Wa9=!0 zUewcsQ`ZCfCZBde>P-5vebjq5-}}009L~1oR~zi34chR>ntkEJpQMBj{*8SMrNhHz z+;h4AaqjjgtqDhe$9|(?(x(lKo_0pcQ0<*uzLYeO!^&Iq+CC&0TZX(Yvho&4Z&93e zH>iH(NxoZ^CwWgJjJwv~j0sQqmeF7B!-BC8{}%GE^cuY#rSb*Yd(qB*@A-rs#c0>&fR=vxno)x7Xo+@TF?}S@s$?|FyukX-_i; zBF-Q9W}{DRD>(6};ywvZayLLZb}=3nb9Zt}>7$EVOP4HeD_y)eZF*UIUtGo=W$mm9 zhNk;77d(o)%%8IWzS}(uzEfG)!~Leee%9E7q_qb+kq+eHH0~%b?|ZlV{c}^!O#5yF zv+X&8wBIcA&Ap6*^!p6p+HJ$b&%IY=&Q^GBkx5%jq%Ecs&R8FtNL$>2|3zqHB5iRS z{$=3x%F92;vR@zq^EkE zeS=fZi_LeweUTvL90z=jG~(61t?d(h$TI9OWBka;RP-bta?*#4baj-@d7rJm!@3CV7Ojr1`ebTOPbFM|uBhk0pQ)P=Y) zL*__M^(23@c1U?p>D7UYYS<@Yg-sj$>y*$<#5rDnFaIn) zzMOU{hb~KKH_b84KQ9)CuI3*Lef30rXxjT?H?N-*g;pfBLxR?Sh)vDChwxg`eKaY~ zm-?&dlu!-fs@Lg2&K#_scv(%X7XQr4Qe*$(zj(DDouL!scmR4!gYMFyzYKVxGrZsr z9YAMjM>jZ(j=&{?3H?DlqPz8*fNDeWTfKqV_FcfsWs6`>Tphe48_!H|qB-eNdhbi} z`9P)>E1*ml$A`ym=er4ef!bE@PW-5&wYD7^lK&u3HV*ms;7{RRNI&1RkWaF&OoTUk z(eA^S4Pwm7wV6jzo?LqZY1Naz#!<_%R#}#BmGDNskw%0teUBwxkTP5xPl=t6U$`H* zP;I4r-)4>U@8k__tWJ_UtF5A*A)m`@sqQ~#x&P#_SNVX>^^xA$;C}oKWqFve*sUGP zqvh6T1)K*w$XGf|HcQsx`tq;lMQYC#lF1nscheQn_Pot$^31V&8klJp8<=IEY+$xs zXkf0L58TEWwFdVP?jZTp9vTrV!(Z0P(akpT?ruXdXQ}h80h90_CO_r5j{P_4!>**E z@n|^z^`MQLmpeLwpLv>W?nfxwFxpDvT(j&!jDx-T*1W{QW!mbu8gR`CT$aI22KVm- zF5BSdf_pcC%Qd(a;I@Gab_-n4lv{t!!GW|(H(;q1YerV`=84f;Mw!0Nx8Jj^2Y&`_ zsa*p-2TFhE4XCuXS?Fki?KQDV^v;R+RlXbDe_rJueZOL0uKlus+4lb!$o*diX4)GK z%&}iEu*m)+@F3&jV($F9_Rp;6cd!P_hNAv|*2KxS*BF>(KW$*9z1qMW`}aVvy;tHF z&B;D=<6@`np8}_KWed4y<|6wT4?9?Y2=a!m#t-Dt!t(9h!JBW^(H*@Anz^lyNDcP;38W!UrssliCC9~j_FiQq}h zdjzw9oq&133}6wk8?XeJ3G5BbwIb8eqwd?l+C?lrNOU1RN_v*caSQn=-E@Mpu1-ev^OZdHz@taSo8ifoXk1`=?o3w|!SSXr`SH{zd%nf7chw#J}Z&b7R@m z^{op!#9kV8u8E_$z=Nb^bm|mqsQ%02d&}Z;W3Q|~&*(JQ!7FO#vY0?!r)BtKjS6 zEbfA3EcWDV4*c4neOIwzA=W#0XZapntho16F3F^purmhy$^HI-Z=**8uOn^rxbQ6g z?|G`mh8(*dyw(BUG%(kG11S5Zm!HPZWAh`8m`P*ez|hUaUzQS#=x*(5c)XE!9ZY?$ z84;8GU|j+Emuasjo@}x+e&z7*3w-al1EDSS>x)Jw3D|!C$6OG*m+V`|@U`?ykFTNW zvXdRY9z^<4XS_G#(6QwW&_1r-vASnNuTTwnIAzL$ht7pQYoOC==yf>~};aSQoqzTwH1${AL;l5+f?@Dz1YU9w0o3toCgyyW+7-G8x;rRd^%uh{U@#v6v#u`aw1x?KCxsUgvEBkhaMyRhfP za|}NoX{7)D(lMs>NKY5_rz{)0lyrkyuUy?Z^fGaz&u9*i4XrP~T5lL-vSwI6_9a|8 zak-Cmx9kPW2x}&OkoT{WX-G7K?zg(<+20Mx{_cgjI^1gO`O?rvS8v$Q|LBJ3{?*id zIQ>Jv_?>?7{!>3E(!Aq4A=%%L8}?jpQ~s|@Px377ni}4VV9dBC)tz&EHDA-7vQIG= zNV5Mew9XzJdX4gyAgh==ER?cjm>T&-^bdURX*45yzuL6 zZl(svH?4=}k1wmG=k7z#jiKl6MbABmooo+wvi;DXGcQlEw@?q!#h(q#wl^D? zXWzX9n&kY!IHSX6+H;UQg;v1==z5Lf&*Zzn%3tsnZKU-d*%HRb<7*k?-&_kFYJT!l z;#`h@7_P|5U2vFr3Vrp4=Fov3oVg&kiZ_L34UgsX&eGd=&Df@~^9hrGzWumcj%5bs z+P`t(M(DJ#ZU}F?=Nf%XdfH)dl7Ej7CxXAggv)M}V=H{G3D36|5-#~vWWpVNPT{-U z@YyDhJiFe_>n`9B@kSC|CTM+UpHNTq?OueRMLhQ9AS1Kwspj7-`z8Z3?HdiuIj)>K zeAA!JIz@a=D)W`j^kWxfSXX$Z8$5Fo@}fIqZfXeGXN$I&hpjGWZj?Q>+2bzvns?i}vy&{)}W|9P=;bal;Xc4S)N4+^mf@ZNGB?Vq=t_8(KWAe%R> zROS(s@gm+=ZQ`F4=3vriC(w7Ck6EMf&CEke28X6G9x49!i9dlhOUyw6_2&q0<{_A)=1?bV&RzS^GNez1h`Akp^A zhJ>6sPqzKO(j@)j!&VsiW7^NOJO2)I^WCO6)m7Bn7kdrA@_kkLC~fAAO>Onj2rMh`b>6P31^U|O}nkV0DcepgRv&v7mSYHAV^xZz{@LwvHvEmLaT7W zp=snnc>Uq{nk&x2whX?=Dq7&>9XuRgt1i8KsP@h-6GZK=373Wu?-!9w`O~y zTL%^lgvJL|ofiZ5Svl{)vBw<#3dVAfaT;f*ntsT!Z*cqLdIPiUIs>uMxNx56v{TJ0 z+Iv+~zZcK33~y!GR}pUH)6a)HVZ~lpwtX34)6IWXPFSWFmSc}2>?edZ(0+~9i=PJq zmiAhF7T8!Gvas3F2Jo(HFNJr*N(cGw@$zrY_fg({%zWoK;n3W#&3C3flzgJ}btZa` z{vWJw`c!(5=x>0&>HBQ+onxQI_jiS3Zkgl8>BYDHEio|DJ_T5Ew2u~gvQK?v?TL@S z_jl-;{uJgae)=etzUn}qbwqA-X6 zp#KDaJ^I65#;1l+)n*TUEn&`knwj=e!YA@}_iVyMlfR+Oq)*GggfP*Dw=Y3@)>q1U z{`W#X-Ln3O_|6<5(-xf^`)|Bj>xzEro=O{Zpe;JmCeY3F05-xDY{`y|aGz@=^TaN^>OrI}$(phj;rktno$I zi`7<^y9e!J$|8GaGwpV{=n}hEvuG>Dv2C3PZaDds;uO}KGzSPr-laIqC96B3(l!{)~ z6K4U z8=jqPX39}J+02BUtk7QW{^K+KSZK$8Ydv@t<#N}_pursbAUN?}yMbBuegiY@y#^LS zo3+r;2x#d%XzE;OtA;TH-l}$ZE51p3UKj9P3GYU{lZcOwH@ydI*{Q}Z{ow=9A~d(7 zA9bp*IM3n3PR+VTMKBiWAB=726O658op259`5bNku3Lt;f%`{Tv~iBT4gY~^`LPAy zZ?BX;&wdU6-m~P-vi~AKby0m@Rvg;R^bh_6*a>AT+=$<^$^HqybliQ|xz0u}%)}{v zjI;*fKS=uR+=ulJZW-zY>~j!-l0b z_$jy6N2c)ZBzG1rz}~QR+TackN8QCsHQ;Xba3ff!rk^H*yVAo|YwbG6@W`bej{5C@$5s$;yoaN% zvM+B2H^#%!4?F1Ro!~|#aL(Gbc;ws!F4x`xP4osAba8v>E3KXHpsi}a4+Gx}{mjOO zDS8l}eGIOQv|IaBhwaj#Va+Q>!doRn;jKRP=S0U@vBx+0w(fw2^-tLFN1i>5cGLUv z$-MPK5^ohJcq`BTtNTx1{!tobK+UT}hvF~MRjCQfwR;*^U>6&hZ=WnkyA%r2X8D4& zXRaV^oh?Y;Wg3`mcQ;V;t+DXdh49!9;k7aF+y(IB57^V;=x?H3(X8lqJ9ZlRwFY-0 z9^J9DLs)A^8W*G|h(|RhXwBwXXykd?dIL1M9(s(>m!GXr`LgU!fO`z@W!i1{_ZeQz zwxjsl4X<*h8Gp?1YCh#EpnQd_M-{>UIre`D6HorjzykZ91{T`y8kl3h4b++Oi_u>O z;r7!u`)Idp`O!y2^RsCi*%!AGSM&c8gU_+=bn&l(-vi%1%G%R(+-10-I15+T!{uM} zhq@fcKU#ilYOJkH-OgT-I(S$6hJGP`0rv6^9WR7_B2St!=GhDg4@So+PR9Yw*aB7F z0R6wh4SyUcoc3LM<(FOkQ4?NhFE+5ie#F3h`(Z(7;z2=Z;(mkAu@@ScZU551Ec;#q zbM5&C=ADo>8sT5=>x?zRD^X-mV=<6DDUI+KcOk{X@ZLXF^?z_HM@VZvs zVqqV9>;Q6Ga<}+Tj*axEF+MyZ)BlsOQK!(u%6OL8C&>Qa`Cwn z%EVlIGVQMN@1snMj**G2lx+h0CcL$|ICnBmG;4FY_C4-@JNfTS^3q;qrL*IIna)q# zbl&7Y@sHAY9sdFH79aKhU&=Mv3MJPx_mK{NJh`@>^p($ZKu@Pwi+`VyU)lE4_}h() zaiSpocBR4R*jE_HJm0`v`w{~)?H?KV z|1ZxPk!MkKt9i(@2FbKOR(KvVP47tUG4hNt>v>0>4JS{vv-UYi7M<&k?UHYvOgr7f zCCRjY2FIM4|J7fo1CJ-uq_c~zicQ^e?2p~}y^j-rGVR*IjW3zj!~BCxqwPGIcEs?8 zE9YF9=IHN|rAKAj9xoo_t|!wzN#Kskw3YjWLKd*$KZ18SqA3XGY!nMXBe1oPdBi@{uxm7@MpMB z7lY-R+i{oRhT;lv4etDB0e9stVg56-lQp#;{s!hh(wR15yPwDWM{}Vm_!|n1ZZy@0 zzZKoB*2jKZZ1|1X4(fol9oWO5Z}xLW*@IJ?MmVsMJ)Cv=?n1b}+4~DWuc_@$IIxlZ zzjgY~CLGvuaca1ceZX6ow>6TF(QS~C(rt|HgDfz*4?J#kAM_fd`!LUv?$f~hN-{#@ z>j$j+i=WS?EoAeO90(eIz;1=C^JGFl50@kpz6A6- z&%@I;bC3_h?*#uT_``kNve7nP*`(Je6ZRHi zQSwPH*Ie+gd-$YstpNXu@ZB8QaUJhDN_MnD)1x}1FSs0eayIfqb!diXHWUAk#5dpQ zR+?Mx1ovD5hps_;rJ{2^lfdPfebK$a{XT(1PSQ3t;KB(UaByf_=9Ht|IVE;Bl`YH2?QHwEk691i=IBoe`IBs~;T;TDcVhnT z{__z3D2)ezU$)m2*uOMk`S!gA=GpTN%(WW~%(3qgq@CuP?<{+^fthx_fwJd(^BTa3 zt)*|Samz6Y=;=`t@$WbD#vJ>{COp@^+`v5hQUmktiw!KW zF9Nd0Gj`i4?1{rI!rg(p6gLd#$)lAgTjL)>9=*VQCZq9t@@RXJHGaNpk6D1gaWLKSJ60i(dr*Fw5 zeFG)aTKibwNgsb1uohScZ3Tc)^45L%jgo1STcyY)a7KP1SE7<%@KQ7v$UjlZFX&eD zvV`2??jXsnb?g_D-0Di3Xq`^__zgt~awCV6E9w?;V1H_0m_IO*(? zS(Cgng0qFAjFXJ~q>SKxD;#BkstLr>a4x#ohaPvDSclxqdJ=?R>x z^MU&*IN4TAnx5VV{$}ve(x5r((#)EVbavUwdO;tOX&aDfYrj#Z71{3_o-DF&US>Ub z=Qqf-r3pJ(k)2}niXwX=|0vHZfrH6Ua|iLY_Qqan!r*xW^X!WZ^@T4+2_jE)0c zZ$Z{+ow-4f^3)1K8v#M+VGD1K)k2#Mg3x2FApIQ>g#NamyVdq1T(CdkK&>0r4m2`m zO>G6Vfy}F=z3T*N?=rz^!jW~gBM289?a08|F^&wZ9Y;8_uJ&TW1uu7GVC@7)270op zk+McJkXfojlCCvB;>fJ+IunF!lg#=+G(-GkUF!{S!)Xt#>l`LbwA35=SuZ^8Q;HL= z2HaZV$fp!1++=X8g`?d{al*|7w^BILDs|Ia0q%Fg(VnF^#oG+-G2v+2Qk-x*!96M* z?OcizPBLqeaI|?TPB_V|`-P+ZOL4;0fcu4T{lMXbn+#4m`*8ZD6erwVaMIa_4+Mu3 zZUwlzgrl!Yal&l|C!Kvb{Z@(-ZYQ{(2uB~5;)F{@2HYfEH8`Aby}?~C+z4IPn<(5Ea5&-Sg1cO}ao}*mtpN8U;VuS;6K*rO3x&HJ98S2M;LaCr0yvy- zsjRz?02lQ`v&hMW9+>OM&P%}!GqO|mf9{iGp4eOaL-$IYeQe)T;2H-_Ky?EcWJyH>E! zn^z|5b;)Z58{L13MdOtFWZ;+ex?DTYgyq;d2BJS2$oOqwrhSru(%(iSyG9|yMk32< zk!d4X12~VhZfAdWyjtsV8Q?n;-i3HwiQf%b$KES{YUm(q!26j`evp|O-hutMxu4D` zI5x30(Ahd@axL^5Da7ud$=s5*+iUEOh4$yb{j`t9mi_1(pYXj0`7IfK5W9nU6OS=V zZ@Yg)SbK%yvKET}@Id)x_s_DM_|`xFF%bK|f!X#y4a~CN1!~Tm!xcBg82aJ@`s4@n&H41%Xx31jb(>wBl??b;3+7&<#!i8Xy1+>ljcIumb?GG)Wf^4^ zE#*3!#@_bt1j7?JTUYLzzZiKwNNeX?xLHD;Ch$$2ccD+Nias zbF_wO)`{77<>h?}a#(8+jvX}94$+6ozY&||ZqBMohP3=U&bySRez6Aph9~|VH#{MJ zX`sxFoRt;+dd|>_R~n%6I^>v%%U*8bOd8O1qtC%#&pFHl9(s;a7saW%nw`F(kkN~uD z?0?tlKjD-9Q~P`Qm3>w`Bmg}f`=4wA2A}lb_@w_DKItfjM?B19R;X z106dzd&|e}q2K?GTZ?-bcRTJ9Tv^h1c3RSSR+coLl`)vQvs zW3SiPeZ;_Q`;dWIcFe#``+$Ku!}ZNNMjiR@LN9A%?2%l5g*~q2(C<88Bjb*d=Ps`0 zkZ>mrksP0gU-3p7T;32PuNk+53j-Uik|C1QBVAmYa2<|6{xzNi&9ip4#U1~?I-WV> z-?8J_7RsXW&l}I$_J4W&yMDhrUMBF)_-8YAX?&fs&xv1`;*5Dw#x28xdmTR3Jxpdi z(HKWQEsR;m8%L<48ArB%9v}ZPTF6BH2>)tTRqLLjN3e?K+Pd zeXszznx7aaexAS~>yfj)>HpaYoMXeR0e52pr}Y-;WUSYLzcztKXNJEI(#LbbUuE#p zv9Ucs-z&gf0Zw$VC16!Fqy;K!(@&}hWFWsYN-EYPdmB2JRdDy*m)3)1`&e(*b(B_t z8IMvJ3(7jxM-HKT4HEAGwqaV2a`#};O8ZC42 ztvi$O$lk_J<7<0SmRiYM(pC91UX@W#(_Z_#4bgl)IjpA}7TBLVq*n4V>AwOutae}V zkXp&cq_9jktYL4TA+?foNnxGcF!jZjJ^hE6`cS^)IDX@5Zs*?%^WH4(uP!PrZJ zltsGM3;4B1OY8i5*}Jdxjkkb%iMOlMx#4^9E4-5Ta_iHB2cz6)!P@Scf8n0PEy1-= z&yj-EGYuGJ+-~UuZFF~JavSrF{@KAu9QnKe|7qwujo;ziw)ScB{k;xH=OShKN9Q71 zb64KlGo*5tTP2E>kNa> zwx8nlaVr(UOP!EBW_zdUV;; z`iSfV(SFbgb=XTD2i7=p@&fAP*uS%F?Ts2o|B^yul+XL>l=M|!g4WT}Uv z?SzwzJjKI7f6|>LAM+D9bmUHse9TJVSZ9aUHxsX$i)-)0yj!vqe%=W_EfJUXcX&hk zbZP>JEJkigpN=E@qW$4#+O(Z>PYtZ0N>@_-BxAZElN)%e)X{ewTspWs_g&Ix1!um7&STlXAa0bjTbPFo;oru~ojsVy^@T>aW)J56abMnpSu5Rxe04X{ z^Yo+A-L@DT0{s@0EINT*;{*0EYMor`QahlDWc%&0Yvk+cj~7T=Yvg|f9yBtDGxi3T zV?SpgcL5lfXRk3Z-+mgXF=iWf&NaA2xI1u{;u<_%jklr~qpM9|U3?1u22WSRKhLxC z<~nOnbz1YG56ggc&`nfp*Q|p#0_!MaoxWKMF4MR40^WqvI(Q?n4mzpRH|yYK`i4fL zS$s2BY7|7yv7YVlQz?*kG;2!5K&>e$zW9xFTbBF6vdOa7%d8{qW3JfS<)!J=`H-t~ z%rs-K_P8Hi1HT@ecOODCbm8&SWbltuw?okDB=|(*U=3||f$-FE67>~sGPse#QO`-# zS-832YJ{V%lPI5XE5KC>M|~$zci}dJ8zvleos550_UXFZ#C ztYDr^T{aUxKz!}rHFiwuvJ;#?fkOrqI&0cr7#&^lkRSAAZ)~ZK{w^Fc37J*{ZZGkg z>2rs_ow@%Lq$fM4*00w)Yds5%os+q4l=0S^>#7`+&6rSV@B5W=-e3#oTaIVvOg?Ym zjXxjokJ8uylunnzI3~W)nEf9UmS_LVz+C&E1~TRg(jIS{?=1W824>oC8mKwRH`~xo zv~Df=a2WZZ{mqgO+6%Z3yQjv9bqrC%fkw`b2imXHDMR1p@T;Iq#v5R~vkh z{d=Hn1d>%N@$WVIOP1Ye!rA|4Ap8FeWdEOmx%M)k=I0N?w>qzo4b;Bo!Pq-9aWU?V z_$O`yZV|4Xb_uxhMP~{E$QRA|I8(r#dd^;(e)!XoFB8DO+SMAra^Q$?8+^Z#f7{^q zmGGfCV*pK5Rcj_^i!u#WIisyO0|TW@Ic$K5Ni|t=OD5KxSyXDSE~k zYj344hh6z_yP-{Ip5TqOS9`c5+3;fzCp$jns-Z26{HJX-=Io$9CWEW?;#E6#s=464 z@8KAyy|H$vhZFtNZ<~oXD1l>4rfxgI^-tiSP5L+$9_yRHIXu=I+;_k=LkkY?IP0YE zf)nqw&|YEqr-k;@xq>v#tF?@P3EEZlgkYX*0jUbC;Ka*>?O9 z@g?WY49u|)87TYoH}44%Z8kFoi&o`#&J@U2l7=li9h-Irw(ZW?xVx~<(3NvP&V5un z3pu}ttweWO9!j#4tf%cFw4=`Y{M#+ldj@9P|8UFpj)A%MTLz*F7?^KwHL$>b9auzr zJiuABpWrUS4Z-E(4ss@q@zLR~< zCxK&bPMK=J8U5YxhBGgp4DQZEJZDZm7u@X$oMV?=0q&Lr4nCmGH-ozoTyx=<>AS?E zZ}_66w3DIl7G&E7;4@Z>uFd7fOyKi`w<-1W6`M{wwq z>xPGSL6c9?SG%CeC!x(<$nJZg&6%84S=nLf;wdw@U)4P;X5==sT+5lUXhn+A@t*vH zL!*Am=jnHy46R6yII7=$Ze*BnT5~NhYwrI};AB6_H+tQ>2^=))cjUM9cSomzMmsrk zrN4n|ro4`9b@UR^w`6N8_M%qoKrQ{Pn31dSqvUN1?^_>TgDterF*H@LR0KZt`z!DGl8CO>#28 zJ^J3=eWEwbUnDP+?aB|j`QL9~w!P569Q&6B=GylH#XIAWeS>h>xc$)EJ2>n=k;RLmPLJZaZUp3G0lF z^qJPL4qls9UOV%yZPC&WVZ~R!1+p6!*OqQsJP*I#Zm-ul*1f$#r}4k;lacKtdaD(Q z-fsO-y3$4TMX^=T7C3$JV)U5ZfxDM3PAk#f->JI`2Fx?}My0y+c{yn~GBnSgAlfN) zWRK<}=niDsSizNaT}&@8k#nX>jv;2fHr3+}t%V$hxHWp&~1 zx;b;s)*XLM{@G_YZ{j>IXL2R`9yR=3WUG#@?0e&Q`f;-Cd%^wZWd2bag}^V%zC1g} zgyq^<2Ikm349vDqGBC^TYG9__*}$)tbuFc}Hy)LDDcqm>)w={ddtE^1EEaOeDQ1H7#mj8&p**65!3{NS{?K;4;9-=p@R z&H`y~*Cb@R^o`JJhn}LOsk@qj(39|&(;mgC*6xkSwRgztz)O8Y&HvFIPFeO&Xyk3? z9B-$bN_%I~znr1&>h$?*gf%zU#C9C4iD@sH!g~YP3I8GZFPt=Lz&$NoE4UQh3zKE$ zo2!KT1YAc4r+$~t-uyYZbO*Nr+;ZXegX`+xHiP?(a98_Hndb1l6Bh;-O$iRwIl(}X zv8?-2XxLeU6@6~G8e0|jI0cSWZQI9Ods$b})$FsQJB#w&1+*HQ6}vx(fy;ncWv*@~y{*h0Gc|seYs};f!Yuh+ zTT4Iu*WqvB4ze&hPYb*p)|i4k3^R_mAPdVGH_Mrq2Y4@#e%cyHdu6e+M$fwSQx5G8 zr99r+i!=UwdF`bP8j$W{^gHI?y(xQX0w+1-*gv`^aL6IbF`0Od{;q!={cbL})I>Z- zzgq!L`g=2FcE(O;{Y3O##`;N%^fty$$*m~kXSsLY`73*v3hWz=>?p9mpkJps^OEDr zt_g{KoCWTfQ(%9>KdR?0;4tzMovEKB*F`6qpL}2-^)oQr-fm!){l0;j_P-50@j4@S z)icNV`g`&Zp{E^iX>R{<#!b!P|HsXHlY!axMgzGE)WBT(j|S%18-S|Et<1N_;|Alz z6W7!4+EX1N{1V=Ds=(#wZ70T9j}8|w-}dORpZw6FISa#lqYT=UT)0tw#^M6@8~vKJ zCipuJ*a7{T^*(4)G+36)nxA_HrN*VZCB*gUZmIa((Ad#6zxf82Z3o?Q-begqWUTI< zRa`$~q-gPG#iKp$BHT$sx|_51S3`JRB5jR@&V3k@5;$lOx)WVq>Ed)Q>~X#i{L;Bo zO=rWSeS+b$3?CI5K9c<&UBH}e?MuEdnzcV??an*Xy8J$55@{Evz4w*1LCsG^+lSEq z3|;q0qHFGRk{z>o2XfQM!XkSeZJ=?rmb||0nEX%opELPK^jro#5617&^cna?)2ADl zV}H-UZ2MFLvuxE%vbD@W(eyXJyP)!w5ck`b?`gMuC-IL{rXsrw{(qSDx&MFuIP##% zpJg91kh6>i=GX@eGA9w!Vk;~hN>+#1su3SrA6MxCz(3#zGzD)e% z@kgWL!5_~OemwqImPq?3e=JGhj`GLDF3#hR{_w{#&Vnyr9vmtjX*uSMq;nqNC{Mgk z{A1V1hyE}5U>|iAo!vuSzAc^QxpekZ{_*(WR{S0x+-%w<*Zywe01NA_V=G<%AI8o zHITDd2Ikm<4a~I%0+sKr*k{M%WUJNw)B}tY0ngUN-7f{uxz>I%@CV#+;4#^3H4dOJ zkCq=gXV2->?Ti%-%#|DAl?LV{b%M}fnIL6~qVr%Mh&2kLE7SoSfznMHC2yt65I?}! z5p~Z1J-f=`i!Lg+^h9{OyJ(XC%$_%QPvK$udp3ESwGqNJMtme3<(Pz1e2oju!jXqr zn*lc&+`omRd}eJ1oW_NBg`-SnZ3f&5aNC5#KM8jTr+N2g@UMZ7l1K8|h_kobvorAK zG4VDL&#@mcR#67oYyXtM!H1MV_Sy)zX7Wz9DcnGv1B@No(|ux{sL++)g?8kDgiYbN zI?>F8PE=@*HZ;lpU;a@!p8yV{+(&hyT>Eho#+iQuv+ds)m}Ng|V5Ys;z;B@o{U6Zy zY&Y+E1G)d-K<-L0FxQ?26pat$u4^8eTeFY5D&NMf!+A8`ug6guf0a9sJQ}}BewW5| zHG%Mf1vE^xj=K>NE)_oM^kV zaMW)S&f^WycSrCi;tkRKVVCyIdLd=40e2vQb9iGixX-~I%NxHXkK^!$bC&fh^MykD zDkB5g|9^jiH&!3d){)E`bKQTM`A59b1U!~E-ZNo2_CE~Fw%;)@%YMs1-uyH0|CTpo z&(+-YSoy!f%{yXXw!O~49DA*Sx%M+a+1PIFiTuY6#^vA+lpN&^o$D$>{x6mM$M5mR zW1Qvkcw@Bup8P+>*$2?b8Lm2LuuhP^EEAOc=j>J;^wlT`-P8#}LuG={O_crib)|$0 z_90xbAK`-i3FrJ)-9W+xD+m`ni*Uh8!gZFlnsC7pgbR))TyPBG>}#kSN4Vg{gbQ9y zxZniBIlp=$-t6J>=5=aEmp5+|ol>7<-n<0d)yRozoWgq3*44t%F4Z{UYQTM8INGQh zC){LkLxrQgs&T^21vf}I+O8TW+zN30g~MNs6K*p&>F-z5rqwv%c7pqkaI|kVPB`(U zboQ%h>uQ{E;=w}UX!mNIaN@xn;pl^EoN(g7?!wU@)i~kig6k|CeN&ASZUwkb!c~C7 z3AY(sig5HJi{@!f9W?C&G;ehZAlx zxEA5YfWrwl7u-(a#(~2Lw*uVz!d(mwC){Rm{}k?Wa5&+1f|LG!^#pLZglyyd)$8Cr z**4GJ6LIXguQOf$L%ig1UuU{r1Q(MW!5KSAH_8rvz0%BtM$LXDooi)o%$)ZI+U?uR zxO}_JjDh)fB;>q9g6;FTGOjW){^#4H-G9RTqkNwLe%YSO_;11({|#jPH<0n)K*oOq zWyk&IJ=fpX{w5t)Zw1DVS#$OFrq6QopJ`yWJ;T5pd%A(S_RkE=vu_7V7x)qOw!yee z++OUq4V~D(c(OIM0lV7md|!gQ4*gfQ;8(HZUWfdXe+vHV(SJv?-*G1U9e1Qz z%p`09{u_{i_hJiK%DriRzEAZBV}Iqn@O#sOv3r4%p;es8QW^e=ZTMc!KmB4rF#H1N zpYHYjV)1Fjzp<0G=6u}m+2i{OZ;tGT|Db+YnFDirq=Uz z;i;s31K)~&s_>cC)E9Zf@KosVN%rjve*)_fkb9sR0(;nBJFVMF_ly;JvT`rUcOhY7s1-@G?v-qIj61J%-sjYtoz7WG z*4kno&#fRm*;pPoFxy^cV2=G819R<14a~C_8<=lD0#u)GgSOV-wAWbQW_>Qh*^Ar` zeZ7Nw7N;|ioE;j}3Z2!udyikmreB-w%sHpvAIaWh*~Vq-AIZLB@u_kRpLG0k0}vrWk7`7>v1n=<9P_63|1 znL^)=)_D>5QuoTKUyIDX;X<>|Uwel?T^nCiawtA1N*x2#D-4wF-q`S=(H6eTr#g20 zGWNIXo-f(uJ`X#zJCgEwcA_76bjCj(-Ja>;uxWWV)6xVEx`nqzx2Ggwt~IyLOT&cfomfk{evQ*hIP5+)Z0&*)JPepf@zQ zPui7dPlHS7rSym7*{_9T%rJT>IMKx;!ci8Zmx7xN?g8P*-{_^_=7RgBaL_34@^ZrybWA4`8xIxM<0@Y6YVH_$d_#C z&YbTnwsh<-hM)88+h`xX%l2*bA!q!_x34jMo^MYA4y7)#hp6whetD${%eJpDFw4Ho zz)brR1M^R4KmRuGAb!ZYK=NLSgGshNZ{Jy^oBufmX4_{Om}3t$FxRdyFwY)rV7@&N zsQF(e_K3aE+grG2af@(w;A&kPU>i2;T67@M>jL~E(Sgd*fy~|f`bGybvKRd+ys|oM z)+6D2vmOae7?}ngwII_*GCq%-k;Xorq;<)D&i9=+_r~GQM)UTa{M{fvQ`_8$zC zj{ePW+qINN*z4nCzgY_V&iw2@OJyG#Z_G{S{f0TdP~0Eb+V0w-52B}gYnI9K^H*-( ziww-R9|9KAHv^fIXW{lS_x@+THK-CDbq??194bu>U7`Ir{(`7Eg>dSv3k3}2+Zn=cZB-pV@VF9__Ou`POQ_wY@$U9pwVzMNlu zWna!gX$t)y+_CutYL=gP0n%WwXOn3Z-+er?lDUgQ54dHLMDF7fgr z-77?6)LpXA&pa>{oni#I3ff9IooN&w?0~LB@23;y$Vu^mbGKDLaC&nbJ4vj`?8%sO z_H+Kc_KE&qn!P^D{V7}Z&tt9(ID`CMS$JGJOr9HVcK_+cKT4y-K;HiVo<(_dkNFjD zd*y@k`XuVl@7qefq8V0f$;`l#vQ(>Ebb)oGfq$Z(b`Q7s^ZM>^`{It7X-nF7^avk3 z%NN;O>5D{{TSuawW`)~#WQPw<0e2I)4dC|Vh8Gd{!25aO_@`D(Z*v4^byyXgmAXni z<@?15^LDV>b0_U}k3V&R`f&&2<_F9pKVrQ3LoxM0cRM()V>x?-obue78a}`~69>n2 zE{|4_5AV~qPYUqeqdYo>?_1Le&j`ooIC(btJMqp($5oZgqngn@O*=4770?c%@xAzS z@#`J&T;8INj;jjirCD45K^c4A=azAkWCLZROgrhv4`_=I8Lw5w2Fl19^PmGMMGJK1 z=_=mRRvsFEl)lz0|MUy>N0$A(fxQ1`Aa5i11EC|+&x2pjg>P%%-)i`n`Q%N^BT8Ss zL2q-!H|ZVfuky8#OW$grR8?Z1ly#D`PfG83=ndxDCmeacl=A3&VmEc`I==Q;UI1~&nm@=Pgl-i678e;SkaHr2yJ zdrPHv&psRdbG3D$^Oi;PB<~%IO#5#xA6!AX-@T>FbDVo0s{T-XG^Eu}u>EN{+#%^NvsXU-n`8FRjqI6q{iRK6YW6xz&Xmnso#GF};GoR=`rhM&L-0?E-2jv@HKQ|oNF|vHc=wRgco2tW4 zy*xa;eq2qtqf`1~jf{KI3|}ltUf%w>M&yT~2i_8GDJCq{syN`!uw$Kku|1`}*r6RK zhYz>0k8;~7VVzmr*U!f}6lcHZ_CAC&F0}i*FgA2WhIM2A@k#dX{``wgXX^Q1EZRvM z?e1R{`wZFkDKg{}c;I8?@&}A{+xt{uQ`Nad-8Wjrp5QF|;I=EF;U9?PO<$bG zyw1nlTcW`;`-C>KSF{&t-E*(?;Mb+Oaiqq|TpLc{4ntytSOv;SqGwc^y!_2m5`It*o6Xbn0o z^k00d{eryD2)(Qhbb*$b_Zfe8y-RQHZeic7cti18s#79k{u5sm#Li{X*flJ45ox3o zHgTAn2K8?FD99OiH=O}aI<2JBy2lscUZ%dx^QSg(zBsaEPp5s`^78RVAANf zd4ka1NMN7?whEw$H^V8T*$d78>Ex?)bk1A(eD>6((9h-J4_;*s_`+q-RTXDp-S)hB zaOj`pTTfc19?L31Z}Y8xE#B%f>EF$}9Hg+(S_nJ1`@#WWqp;Gw|n6i_&*5_)?}u$N3_D+GQ#Bq>Yjr*f?e)E3U#H z9XLF~cZKLFz`LZigc<+T(UCIRQMMfDAWP80OK(j6ZlNE_k4)GWroJstIexV#{|`3x zXK(q#)PEW8beiv*%R=|-yOcI3U8VJP^^&}m92PCLW>&>oQ_>f-GA6WURK@no-;H*s z-tUzAmP}Y`O>O0TSAEK;X?xtdZ*uG1T3i*Ih5uISq`LRSPhSt-hkVw1q+U61P`{zG zD?igW)t82o`$p&8_8_m7UaghZ7v3Ebjbq1yMv(!GO|)lb%20cb6?vhOK0&TGo@FsU zbt#V`&o?aZz!;TQzHGF`80BY-N@0vjEq```FC0x8m%_-=Vr*+WGA;>goT%z0dB=I# z+$zSys@O-gg>`b{1>aWw?fg?bcmISQVU26!siWoqO_c5PC5|3*BRJii-9%m=ySO@V zI&azZGJNUcCV?||=5K=cT-=r5n!5SWZ;I^yxVTG&W2|kWy#H`<%D3qWba%$Szq`0G z!mR^G+q~}LMuBVTe_6P72fXzR=RPN7hW9VGV%eOl-p|>~cW_bqy+IHI9R5-KVk@W3&t3|(h43dO;j7s{(L{R)pPq!Dqdi5M>&;P`X-m$=7NIus5$7g& z$cz)n^ocruQFeOhwfWANW4^bX9(sju#R)oN4)6KW59DF^u%{y%zBJ}AMwowZ7!X>| zzZ2m>^L^48p=bG)3=ej)B!3nLzT=ENjjzR>^va#|%-F;D(#T$ZjV~py#n)cXeN+kg zpA;7SYkch_!b~3U^sl}0ei(Q1*yszTBU`_YkN(1)ERwk%KQ-_z{%QE2Q-$)EJmTIK zM;`S=Zubm^D;Z~XZb9`f!Tx#dm{k92eEeX_WbjF4ZTfS3?U~>uZ(6E{M_SveBD_c6 zw{=HVr0u<`$S7Y%U-P|9{+zyTtp53+_9Y-{M4|vhp)r__9w|QIK)pZu~(sbx5_x;|{w4HRFre}EH6n_)+JX7!q z?*9;k&Zi4P_dgR{8jRg8xI7rUO>jvtcB>%zCA`jS^84Edd~4%%Wm=F{mPA}J5+QILiWe# zA!zc~NqO*nygV)^uI{Z!AQtmU~i<^76PgVQPL>BS?Ko{3X zxan4`F(oIxm~rDVy@@{4(b=NYvB#jh8JgvN^d{s)^SCR*dc%Il16PKB+KsmQ$cijY z={S`!c6^j@>F(9|OBiPjjWf<}v8=&{hZraSO8W*+wbtmrdg(A{_6@DP{fk99hhx6q z?HhW6Z^>%&eP_SWa=tY;(p~BY826Zen)!k12tj6fbZ5bf(I>79N7=JJi?n<8E)U&L zexkei=;tq_SW}}P1+hys6fBkB-%ubvQ@TU4nN#4@H|yce%Wgd_6h)S5-~3f-6V+{& zHPtUa?N?k*8AS86FKgOo%4c(3418V7mv1Al<{W$0j{ zy9zh%Jn;V{UGdCMkHiPZi61XyetMJIkTZJzhYMN?t2hrNe?vj^9q5enebLNC1p!~S zp;hUl(RXM^+S$@D)A-ObzF9eAt$?v9!d8@ZIA5B8?2j`UN^v};ZOP=NBp z-=Pf5pW?4u*beL8jL+;Ij<2+0dHCViH(B$1Q?&FxI=$ibcXVDUb?Vi$TRHd+)Taep z`HaB0R`lhOoZD$b|809OHPU((#Pouw#p8X zmY;)L4({Uap~~Ie+QYPi=FYLvgpKJCN#~v1QOvJf$90IrC-4vRk+#b_M3zzq$#X|X z2#jlc9sXy{J-)$;?5eyXylY%)q>c7!yErwnw?Ac~+@DNRx!K2XZEEDxDU|=F)JXKK zlvtF1qm{t-fz`xaKt1nEji|la9w5#l(xRMooCl6xoDypQA03ktYa5pm`}i5+t|QJI z{+&P?72QI`)Z@@5+U8~2@Ky48hj7Xr-3A<2CD}4)yZ?%?nJYcT9V>6Men`2SHUuM! z{uy7?^nfd8{(gFBRRes&cqUn@c^35lnZ+FJ3Gn7zDF5ViSXvO@W<@;S7oAvXRnwGL z^nr!~wdr7Fn#NzvJ#}V)b-qE-4ajch2o2b?qm)DbGW^+{kMj60$fpFJHu=Ki^Z163 z-8sX6(A|79pI&3~Ctt~%tJ$}54P_OdP3!9L*?QuM&z=@^`Rqyj;h*;S+?mDEdfsaQx{D_>4BC{VrNB9b_ z-RAPzZ5_?|$M=a7RNFE?{UK>@kv=vH{ieLi=r@{|_D09C@`4A`(0O{N1be z8izE0G3EQ~nW0M3DDm3u!6Biu`1a26wmiT+Wj^L9Db|qqa?8+Je0hHPxwKhwJI*0J z$$oU*2=9jXWo}f_gnX~2oxHT7uUnDs{P&{lG`ywUGbx{_Xu|E2mw1QS__M94%N7Tx z;paT%(t-xwRYrD1yfHy`AeE^bX&T%B=M6{UPWEt&9jdEo(?5>TU0Q3DhWG{hxzSSt z8%M-4z4+)0J$X+=X_$5$%AOoAjPy71j)=iC7Ha=xJf2_frDgEwqtQvo=1IX=FX|Ux zM0uCfCdl?EIAo` zHK%x0)0|UqC$EZcGjk|x?uEF5RX<`3d|%@*?UzgYMf+S4uAG%R)k>?K*1G)4a4_wM z)2y^n(*oA)v68DLzS?PN)~vCu$e*BZ81G2=pvDrA3f&QuCk64nfA2h61gRGUiO(hhKVukG?VORyB2K0G;MV*`!}qfI%|klE$P>=z4y(@OHIXbb0FisblSoJ#mc`b_IO)_Bq9HAV+~g?($`jTYLo z5?WJxN|puCb?PXS;Wf%oFJ0{KXM`rxw~|SQ_vq`1d|Svdm9a6UC>{A$(I~kzC3+?0bNQ_WUas_&IOTw!6L|Wwa@I+^P5I#G%Bot&PybfV zN_YJ5bLFg2#*Ymmk}5d}6yK?=XOq9?5zy-!xy-e(2~SlyOnr|OPi?0h#q95q-fQZ) zWk_g++s2X+N+$r_x2%MB=(m>4M++txIIjR*YT-EAa3(S$mN~DW(oF~6eWTpSfZ`s3 z4kPy#Uq`1i@7 zp(~E_FZ!|MY@~I8kvr*awAW0xom9T$_W5f0PVSM{Wn|y|UisDx3soN{56$&F8ZP%+ zW6K#YsB^{f>s-_$r#o#=nHV>Wey}CY8uICgE5f@=QzM`BLl^Ip3LYJ0YR!?R+>|42 z$ceT>^w45-)@pRx5$L*o(50chvKif%v_9jH>|W=ONUv{w-xq7y!Cd7d#y6^^pHRDN!%a#3>yE;Pnw<3W9L|BNIp;SS+|U$PJ!<87xx8@>M(zC zdau7MZi1#W?(Dwg{2ASr{9;DeC7)H3?+DU>?mv5i{8p03Q=~n~ABkV|`jcz7xLfIsr-BENKqHv-H9S~Ad?yQN0{@V{!T%{p@4r$lXZ-I}`4qXQ< zAAEMGn0a9d_r!?*-|gy_|ITwl8J(RpKT#T_`Jo_rDy=5`rd=L8EyVhW8EXS+!HC7( zAzBk@p-=X~Z=#<;=!;SKQ1aEsoO3VZs%fK>z90JBE%#e)9Kw}r;Z&dzxS`^lAd2_tRaqMeFI~5E%RMtgP=V1nztbbWRIAI zu43AzOQq8`4}CvmWCUR)zJ|pwyJfYNAMNp?vF&Br%Tx}=(IvvW?ehr!N2L?0-th2M z(x0#WV$m|zQ62iI3|&eb$w%b=+L5{Nn%^2M{V5DYzh3CsJ0wf4jF+k{JVc@Nf zp?Sdaw2qca0TlCEp3?vnqElwFPx=M32$jW`bLfJIG&eX&4!ee0&8N zV*c-v5}HriqmEc1>GLfsgOTf?8xywO?}Y7632Dts=>=RmnM`^=B|Q`G#tu%r-=>C& zee93#K5y}NTw1@3cpg18Q70?FyoEYwtkO7Gg*^7g#3uTu-sn~xL)q^CqxfI{MK7jp zeb_f>AM`Xcu5Kj0+R4yn_J0p8%I`Q%CYDKaW(JT-_$*P2tBJEok(X^%GP$O&{!mS8kmt*7_GLjIbaiHZB4prU7E3>ypeUy zd9j~BBQGucVp09m@j+{mBaLs4*4;7P8{f=`#FzVqMwe%YbPro}Q+B9j2adgT(K)~e zND2Z>zY_++|Ig>>{~lN<=hVC`!M&4u-1Xz zroY{MFI2uIzh~d|q;n&I9?<8tKJ{caRcEu_&C#`Z{|I_ zI3KOO-7A5z4Qc&p7JG73$2|{yvFOD!NRPQlBjbwEuhXolo35oVp9sbpu7R&v>xoXr z9u5>wf5#7962C6P8uSc0@vXD{p=-Efc0YB|`t3gIqjUGo)S;bmM0;cppbzb2O*i)~ zM{d46*O|*-!yT~t$HafYiK}>B+<1Q^UK4$I4SPru@fHz}@j~(ZZoFp{?>>utOAf8H zBX7rZ<|Mk3z81WX^&_*FC(qsvu9@#0*gZbj=9FhS{~tm&L?56Xurb2xZ&=K^uT;9U z(|gPf{vU7e9$!^)_5bga1LT}=C-;jaA+{tz>xH|5IU$GvycO_TB<6mP9hzlMu)l4zBNkQ^@N_x{Y@Cmb$npWpZO zeE-<5*?VU8%8!YvcXc-D2Z1BjAopA@&lDOD%S4 zjmr&;i}uhMx~CsvbT1xKoap}u^Na{OdJD8BY`w4_{LD86&JIOoBLo{=yWm}kb^`Uk zL*6Fg3F#ry8`+DZcRTaFAF$hNXeVVAQWpQ8kZonB&K&E*`_kiM{h&F0KIszGVk2)nD>1cvc@Qsh%VqbAsubDU2F%^iA4bi`SYrw|zPv}HAdBmfg?=e>K&w6i@_ul*7dwJMdTEQW+zVIG$0DHCMfUsQaO5$nx z0NfB}&j{z;2{USuRSmSg=RZ1@RTEc*@3MGK^YTLQNAlAbtw^$>XPMGhl5&WnQ+owOl)!_+Eo+(-O57p)mt>~wNrxw-udK4R3m)X~q zbsPKG&EP-hG#0N!=8Z?@-9niYT665p-g6T=^#or|G5-%(GPAy~{UG6kv-;W-(2-=H zU)*=`a%9?1YBjdw``N>JHg%--vzu45-;Q$3yKnZjCGQ)Txw=;+_tSVU(yw-Tao?R>MaKa9Mz^Ew zdD1D+Riukqv3s>0eN=6i?yGY2o+;y5-en%fza3`DSEQUaFX9@4J~fTFCgPd~`LKo3 z-;a9rpl>aJW({qXTDEkySe;q}Gx589c0aZ4Ys!~jRdZbV)6hY8vxl*RILcWcn0}G1 z{%F{N{zPAC9k1>`z_Z&rhV)^KB@ghe8d6A`Bk-^JpC7oSaev=-`L_LDmv8g-b!_Zl zoHZ^lC(<^`G<9I7hZbiwKXYA?=8H|}6w|QTHVs^vbu)9^@xG#@*nHZ|t-(-e@Ia#< zy??IH%pDuy(+K(q|H9kQKsX8c@)5K)g!XEkZ9HpxS~rQzNT`pLk?*DicO7Op^S}#z zW__cv1R5-3U1>3S9!_}Kv<(}3WU>{FU^l{M5shF+icBNKmm`9HBA<@o(i8CydQf$N z8I8UEf7P`!Fel{JL7C;>aLdl4Y^@0lM5cD7ulLmSvG;E4Ws8S)Ux7~X0{dS382LR* zd>FvSBy3_HJ^UiZsiDRi@*!s)CBb72!k<;Fp>@iyP`!z>&SefMn`_CtmK8b{w6?$1 z*f;X8eBVv~VFjl5*Vy>_%PB<~&639iqu~lp~r17gFFa?61^;t|lL!Uy-J9RvYkXek<~)kef!hk~H}` zDGlCvE3(Z?BYeVZo6@koyrud~nS?fJ`c|bSlO{iq2Z@Vd8*Ri+8hNH$SnFX8_<}VS zTpezChO<7|=Ov$}Hu}18?d0%5WcHpJ$()rIwD%z+4c)J$uUF9L>TA*AA<}eyBjTP(vW(QM9t@Wz&zjH&omV=A4Ra02D3 zzb~L~BFMJw{1@zhiVxC-3>Kgd8El~3@_e`a^S$yD8EfHP3>SwSK2fJM{C66i5yyx8h#(@JDHxkpWLVDM4|`nB`Xq+4?IoZGG8@) zn>{(Ky%T-t*A4h;n6l%>>mV{hwp`I1cF_~)_*2rvN1e19N5|6bPN4BPI{uKflj->H z#7Pf}%wTS7XqfY#4GsIlqFvGOACOEM2u#k)qV1RXKwW|^s=9Cx*Hv@ zif@vP5+1xQUvYGy?S0KWdwpQ{XnemCT$@KJXSVtlEl-86rK698eh-Nr;%K4H!|-Qm zt7xtGD=<8pvC=m(|2P51>%j59`~rKIu!(V;a)hG55FB%i}yw5%Q{DBt# zqgkztn`o=}jPT`*U%qDIo>CmX3r2ur8ghl~;5O1_)A z-m}>2;>nn6j18pzBh?AU@2B1mzJg~u@ZoL9wT7(X93jmw{#SLMJm!w*K2Jc4ojfxj z?G&Ez$MKByH9V6y@)n~5{ZDyjOBHq%c&0%N*%e2Z zwdi=#OR5Tu&O=%L2q%nXkfswC&L*vH42!C?2t;P3v`^#A^j zWffoAw|Mker#hg=wbXI4 zu5aq6jcelSKZSmzgPA&^A5&+Fvk(kCS!dV|?QC97|D~`71nqb_!_L#p{dQXa_jA9U z)(>6Uxz?kdpJD^u_#;C*J7u4V(ayFt9maoHw6leKXlJK42i#foC3C>YdvQ7gIN;6u zcIJP{ysvsbdn?crD)99a{=M*!i+}GCuRa&;?BIT-bOvZgG;+|>8$>@ExA=KA#0PptL~e4af&`Q6ybG|vvmE=2ejWS_?QY})lG_Q|@o`zCE) z|Er%^zJdKMe9b<^PRIJ$`sY8Uf5>0XT1*?E_ISR`yM~6if02954eeCD@IFUV4W&F>SJyJKot5&f)mq`8_<3PD}UYR3l&tU9msOru}^?*Vq>6XZbDqjPIgkbX-5H&0aqTskZ^SVaAWLDk-y-P%`^6 zuUy?v@aj)JP5+gV_AzZ!-*%&Z^_Ti>;JiS##`xs^YNyW2y>irEt2DpA`m2*3R-T?7 z{trH_?C~2SKgp&U3H6^)$0y89kiWefhI{@h*e09rXU&AU$<6p;Yktyr3BDwcl7Cr3 zeZv*LW9+NM=bQNJxu%h}jkG_Jb`xncNxO~s*<5#G+g!mKo%*mQ17FYdb>hk7C$Vud zCPqFH`n`VYq%Hc-q!AwVWU$g=x_+(X8t=7|rg-T-qOCpX^RYhI%NQB@qAv`6v3I?g zeiNM{$7gA6UAW-#cUo~I$r@(j{l{&Z%%g?}p9G6~TSFYqVbZ>QaOwX_t`&c|O>@2V zgp0*b&x^z2<;Sr&Cf9ndJ}%dixc@4-X34g>w#%)Nld<^UkZrHsV=WzwF0?Ao#bWD0 zSGMKRK4F^rI)!^=*nw96V_8A?*Q~4TWj%s*u2q~PBzt%>^wtc$eOjh{0jJX2C(zp` z(A%PUJv-@5zE7gLPd%FR@sFe3YpX{)?2~AM?wX*xX6WwIIJ*1f>*(&6F5N|tbBCZG zt(V{E(U&l5I`=N#a{d$d16-POeQ@Q+*v(s87N$HCioQ?(n6-alJ!{P0<{jDP_05J( zPH$)18MCwH+^RP4|8L5XeRiPUqa7t)J4EkaPTQPW24;Q?hVF;9XT#f_v>ivofhK%G zz46HBj7ROKbJXr;*89argaux^kCG-|Beg~MlBMRJZ%qX^%@o2 zyf17u>q3%`w}siM3p~s|$IW*PIj^}q zXW1BBmQP3H3$ER8JoIsb-O%6D<^A`BeVn;s{Q`buKK@ZG%QlIWvra7?m)_&aoTm1dTNxIgD zU#GjKId9J8j34x?YSyZxy9O4!{9Q=<)bHZ&9PU4irZdyKY1pLR?c ziH}rdZfSk_8PEvp^`epOvPlv?;Em-V*Jfkh9ZWe!7DIyu?+K+(iwA}4M$cf5WbmGF z4rwvG7aeFnO=KW*>axD9QDK{ZfoDH+cW9S2w9dJ;d;U(k^D#8BA2~jx8SiUoR65Qoon@?v z<2!JSvwHrQX?nfY;L5C4`av{p<_~+0%dFV^q2j6I^M~(p|8LRs$ujHTm_Ovuj{A^P zA@)*;rYAq(()4WN4Yt##H|ZWaUW(6>Z12oxQ{I6Gc9)dZ-@yE5%+gGQB^mGo`>uvW zh}Yaf;}`SgdkQ^2fqt&0o?ED|fq$nMA6M2$v>$di9IhSvf z&)l~Z@~vkPKHmags_kEN4ADAG*C5|7US%E>NwkL4YTw3}`MZKa+8=Ue=7hT&UvTL* zXf2$S#G2y-)+6YbpA5rSImYjWNiM(t9KAp~t@PB`HyU1|{)pewTI%}z_cQG`X@{|w z#PLBD@19_Pj>CYvNQ+^ByB@=Q4IF5|UMw7NF~D+dCbFwNjLihTH|ulo(0=Ap|NT+N zvU~8ah_%_w)xfCflr7xTTaf z7HyKB5o2iDD*rxH_pdLs3)Duw;}hi_$$lZ_<#{?jS0>NS3+ycA;oB;dM!5?p$E2l< zvXi}UZ1BGEso3}L8)f$(ty(m)y{Fl$VBWcBl=*fOKP30~w~x56^V_i@nY{*)Id0$Y zn1lXXrhk|DIm4|BXYF-`9=bsniO_6-@k+3T7ag0ek?+wj4U<>{uCl_fpgWA=?4$pL z54CsUZOW6ry8^zbSUe72y=3Pi^eUyjPMYM>D(T{2`8J*p5$eoE;ifgcG`fm7>92b> zTlQ+s2wQkf^`yV@zUtGts4KjBUL>DzQ}k~=G%nhaY(@9MSLQziV9fz!f&P=L^E7o{ zMjiP#u`faQPjheHvw5$#72P{S?`1e|^Uh(t)7yP#1^F&O?{~k0b1V6lX0?U&k)@<* zTw{2jEq#`dY`l9m~+Xq4T@& z8KD2n{lSvs{C;N1ahfl`sFUW)JKQrs2Jy|pn_hg^zjb!uZ|q4sq0i2<3kx{AaNE!~ zWB-WO`G7ii`(W(Mz?!V^@7@>}4i64RH>NW$;N8x%2y4>Y^PrW+KUwVg!x#51d}!n^ zzuAg1_lO?hxq|1GZC133=Qf^&(Ho!TY-4Eo4#LXBZr|pNjr07^f)xeUtnI(yY}pdu ztXjhGK+ZY)4gY85l0Fq#XKgStg1vvW>#QNkRWGinnqTp+3J?2l>WbghqgK@9SvRqM zT9q}c%OZFG5A%!qyEy+w|2gU!$yKZmk$jBA=G@e6! z_gm4rL4+%klWpu+$)uV+6KKNnJTFK=M zr;`6&3oGLDkMi>Gy=lwn=^^V!`R`A^Jb?U6sNXtzz<=NHBmdZpA60m9zKgexPWa38 z%l8yij~0!bT>h0_`Nm#H9)+vEci-q$Q@O6@n#6TE*95L{Tw}N{;<|uq7}xn+-{LCf z`X<)^uKrwETxneWxO#CVadqQLxNmerLCCg3>9%hRwtK!Mg1rtoT_3?Fr0|e@+O8C zOH1re=ugRBeA5qvkSPyc9NM(b*FJo+FMB5X|CC}Bb7-PxPIHU8ESfBhnH zc=7nNN8EbdkWCMDv-%$7EXf1Nbwk7G?iRFrWWN>0mLEmF@650B*@y3R&*4_PE~T!( zxz&@Z{M}~#aByu_oz^AJD2tXeUkYKGsZfwo4qUR*vIl$UxvN8 z)OR4C=Lz`j_N4sHqpbs#oKqkjx(QvYIN#41-kHU#Y;HZaaHi1Nzg{t(GrpVX(+Tt~XGblTpG8q$OaCc3V9|FI>C43(9YfAS_nYbK zF=*y_E2lE&ua~aGTz``GD``9<%!$Nb!Rsb+PEc^>&G4SmQ?Bb?e{hEteSowH=#b@< zGd`bl%F*2-_}`9Sr2F8^B=k10VpbRQxoi5?W(CmaIuZgiThHfA7|O4MPO7Cx{$;4W zk+G=2W``f@$U@?Cx1tkqu6dw~`!8(mjn{V|dnU~y3=---f*koh^XNsasp@>ch`-2b zdI?|PuI0@|{0k;F{+;Jq9UNjm<(=XA7Df{*p0G2% zl=uZ+JYi@20^+xN@r0f6TZyM%-SR{Ao$-v*vt7M-!p``v#Ipy| z$B>`JO3v_i*il_0CMP z+Vy>%&&cZiYp)I;q^!EK(lBR!M1S)_S$G=%6RMRj++!=Vp3RouHvRz#*)t=|hZbS~ zYKA|n;9Y#LvntQGh8>;Zt8Y1C)ql=@#Lr&p7XGxvV&Ae=e|QGxw5f}G4XBPLw4Jkbi0Aj$QaIXyM=iA_$~hTWzQ`; z$3B`IvVT!lVz&^^9y8EBm>9B`67q91`*2yJ9YJOUN;z+6&WQF%w(fJyebw9%?JK#e z?jEse)!lcsZ@zm(``O!*s{WhMH`s&Q8$V}5Q=^94KOr6-bkFhFH{AXa&s#Wm&7?0GVXJ?zhnc;nU)2V9 zZfDVRgY3d%R=s2^zNRB9k8}(Tpfkj`N&lgW9KomcyVmewbMPOgP15l@W}@dk&o$fV zSvv#YcIWGj&;jSv)+ZIY`+pk!4`c=G)6e0tM@5^KVV?q5iWWSuyb*kEoQuD=e}vO` zrxiseeLM1@RUd$_B8+bU*>BAm)~#wB*Qxq!GvXr%~R zxq!GFU&i7H^cFx5$>F)}8Y{fJW-7LwZ01kcbRL%fxXy>qVl5}fah_!F*Wr$h>bJ%Z z-EVInxH79@2IpAaj-3@OknXk{tUItCyCU!D{-5Taf8{XOYvw+XU(p}?MfzYysKjnN#}{t74cj~PyPI#@gTA^C`uzwxcGqDuoFUJBu}t&O zt7w-s;^zKI&wsJ({Vvwf7Z|5T+E_^2nrPbu+BTVb-1|iPdfn4D=^oeUp0-V(ZI|kv zwjnzR6YX-sMt`ul`I)3}qF7B zIYpZ^KJe{%8l$|voHsNS-2?iqV6IZlTGt#uXT}h=@QlyhhBvSceGY%=+ar_ZYvRt2 zUn9PWc$88{d$`?sbF#CI_5S5q5KHoo3 zR&YdR{{ZXq7m0S5J75!LO^Chb&s@WPQ1*rXi8W{Tn16QRaT?fH5V9AB(rxJ%P5cLd z{TS@Sit%Ko_pgR*=b0aS>)dNuZ(%G3Kg|Ae<|WKQ>hl$3zvc+BcM^Fgf=%UD|lR<{?E-eZwxg?83#$$Jsz@1LP}IzS7cTdF0D0^&L~+S1^u0pzqc1N9n_Z z&=$7a`q`|Zw7|EskNgfS|_LJz72((;@|6H~7|G=5{ zM*8b0^;I*MiS>Un-*}Mz5{{0878~}5j#=+dpC5Q-|64w99~x^o#~Kn7ZslKj;UlxH zu+futcE!$=?A%M4ojL>Gyj0&zxDh?%5a0Rkm~^|1K4_y4+WB^U^I>GbRCEshZB%^E zxX`AU-0{!3$Gh)scbRifr=HQ3`*CycX|JU}!d!pmYUb)ef1J(r$qQibv(T^hnJ^C6 zqoPKxz(4yLlb1`6%W?}{whX8MIbRx}}Tu6rF1aioz?j zUn!=)wb4&)^wSagX)g5e#5b)a8GDQD7`_IO!_sv%&wB;=)rf3Sy9?)3ZpuB@bylI` zYM|wQwEG**iXdk?EJ7!eUb^b;JKC8SHtU?JU{}W2Jf}1-KzB#4h|Za-^vq_To`=H! zo(!2VNAXF`rRZS6k;F4*u{g$kN?Ae+UdqC<0^$I15$zp5~pvf=lWp`zybw*j35D!Bl4+bzVnZ)VovXP)DdM7N0|!?n}TZ zvzE&JC)^vE#C>1V*}t=4GMB5LBs>3{?b>P7_QJ+-;7W3HWN_{B!p3ee1N8a$V2}FO;IEYo$+%@+3jcftfj_w zQ?+1HD?Ggs>|1rq?d=co9!^wmt{h-Jw*b0KQM!kbO8?oHrVlso&pRnyxLa3qP5W!$ zdQlN`Quv*wyl9J%)~<`3D$O%KES z{-omO`AI99(M8rOWX@7PZ@ZZnnEzrhr#J+bzB>ke7i?-HJVH3Pthe0?#%~;Rj(wQ+ zOakbSZNB%OzC#FOPo=ULIk_)p#P+_=uTTb}C{ec-(^`Y35_ z5psOdd;aJi#{Oo1DC(e(N-s*F|DFz78`)c3Uz=kMZ5r4s9A*C0vVwV8RZh6$L)MQ9 zkXvs|4Xf{WL!*0`^GKFT9&f+j^{vQZoTAvLrPuEBGY^mDMXyr6|MK$nBcIl8_U-L< zJP)+I7kb?2)&LIitaY9?LY?s;n+E@4 zjec(8vz!Bp-{!_=5#Pjf6X%FZHfb+;WNmN0iM=ro#xSHV8GR;A1LeYx}#X|!L-MLVTleCO;KETtF ziB{kEvEKSQya7FmHpFkY(+1I}cuMy00JIYHWfnK??HD5e;UUC}Z%h1~;ehQZW+#x{ zQ@UauvhYbRXm5Qm=-GdQiJTF~nU<8jVRz#2Sh>|08j@Y=gp$-HXSC_fkeb`en;M(5=014fry<>&)sxc%p^=`j79rJ~Q!kfNksl zUHfo3cBUgYv7ZZFA6dR6e{2uCA8mlKiuJ*`Lyv8js)5U-93nu|5*()Ai{3Dj+haHv7(aM%Zc*3zV{=PcJAK9YPIT004ZRENr(Ib` z?Z$d)B5ACj&ka&fg5AzqVJmBehrG4I*uSYQ)7}*43-cYR&QU_`ZQsQ?E*rQW<9clZ z_7QAn`Y-RFFUQA(d;OPpyZ0~NiQJE$7haLYzaxuT$7r^yADQV(+A+=Fy&0Wn$^>6l z_X&NizSqy1_Xz(N-%*$vSR9yV*@0tYb_G6~_eiBbuy~7X!uawAy)!F>VXP6}Uwi73q(db;b3FgboqUJZ{`4=rw~daI z&V8oiXYD|JySXz?`vNnZGx=tRxLSX8eFk;@jXFpAIsfVVBUU8L(j10!zIbme`Br)P z;?uOZ@J7;}JZ)OGvyik$NZZbTcVC;p8B_C$Rur*ES^wLVd>yc$mVc(Yb%fA^G+(gr z$L6e$b&+ffJ)v5DWYIk}{67yFro3O6u}gOz=3DPX9~2gy?w_cv*)&(cPkJXG>{nQ2 z#yj1aM<_oz%~3S&+DCDhNlS6=Fk!NDyO;Oa-0{#;92`g#Vqk@sISQasNAw#{G|1|49FiSNBvCraIS}FwMEzYr_;1W;iS&ng62e?5inU zL%Gtp+Bn-PMoR%`DgI2aKQpE=zwjVqdNa6Gk#B8)$UbSoTmBXD`$D8BV76o#`e_b5c#1>Li;m#p!FpWT&?Y zzy2TVlJiZ;tc_>mhlt%!Jn~QI=vmKavzy9Ehpyn0BFM%`^nW+*_`lE_ zI^+0XFqkKv_<*=_ct`yD8vR_Pv(vs&Y|zcL`rW-oxjh zYgza^T(y<6->XL-E?_JReE4b$BeI=Vk6w0je=}pQxN2-}-{(6zw)v46LHIZce(nxm z_kh29A~Sm7E9l4XD$)6PAT|Z~^$4=#FtTIMWUD?VJL+>o(M^o)3%x_pA5o^}qdzoZ zs&k78)0|2ZraM0{VTN;~2{WA=2<2DsC_H;R*Hv7@xH7mnjO%~I!{g(4czhfWPl)5; ziE%tU{(pgo2S7La9?csIy>I=d3Dcc3O_=5kFkz~bXTlVxzX_9_924rF(wEc!moW~P zG9D8dmkHqO_%P#S@U^Z0y%=7Fm!tZReI0ZlnNSBU%P!UAXJ0M+-3-6~6W!-`i`=d=~r27`dZo@qZ|K zBXJ*e#-(@i@4I@prYv0fCAc8kUcWtfIYN6>_g=p7Rq}4_tcU-PLwEa#`+a8| z^Bm~03;(@;fjDe`==I3jlW-!25$M~|Gs*jF<%Mo?SuYJ(wVd^|aMA$ItcqY`ZsdQ` z#oX6XPau`JwOztG<1NTpb2`_G{~w$7>02fVm*CUJwPV9t9|?TpwdJf)MH>qe!$-?3 z&PQUL;MeNj$aVjH{8t)2o#8Wl+C7d>s~9(Z^W(Q!OMfUnEfb$6LJtAxA_#pXK_}he z(;oOo`fd4xMUkue&%OM?^bgtgIJtU1<%lYaYlQ0?t}1Q3Gr)@w*mN2 zc_wnNGioJ=s*xSmwHbE4+6FF~=Y|aX44$=SsC>$M}0+RNyjNyy+@WN-i(yr|i=Z#)g2yZoE(j4=I@ z<_t4osx!odDbD#OOm@yS;n)8Iz6si>gGQyl86IU_%<$(xtNv5?_OG=re}04V#Gm3# zcm5!mm(j_aDTclk=bmA4Cg0v+=tXhj!Av8A4-*%K{tMx^ENK1%{G{eehrRUzPlr7& zgFhp$^2Mjc>r2J82@Conmed5axNRJFO_NyVnz?jVa)U=&v_&_|e zi044OEQ2Q+IjcS(aQq`sg6=3jWyZ7ms8*V?4ann&o0;FF&4NiX=MH+<3uKIzLGsEb{g^VH=!Hzo?d9EM-oJUz4>erb*4 zmzOA0{8DeiRObZ~ra2o4IahL03unl^&9#y1ajv^LgZMfw@Yw7_nU-k%WEDDv^5W>8 za)h~oX!iQ5gvsUnv+>AH-HP(GADDlqFCvdQtC_Q#kKELwsDXc{3ojxCfpAAvkn^un z&Q@C%^X-*Yp+|GAKy&ci+U4-jfuf(ST%MFyJ&FHd9k}4U2bKqXS&IW*tv9oSe`hX{ z>mk~$bx+Zk%{c8bx&b;~Z&$~=hdA|rY}{h=p&t#hHMqied@R#QD{&y3iAzB8S-4Zq+EOgrYd^OPm0o2QiU|7LfdlIgU0 z?_6d2KGT^@c%f-ynscRzOLZlJUO+I>5Gc(`CrfOXHQ3@t@6Johe*Hxzf1WICCm{hKq+Y z@ox*<T~q8D^e(zqRi8+%(&{#*AOK6S>t|I`=EaysC4|vz@!Wci!h6$-j3AHRj?S z^`+*mZ<;jDOg3Sv^STLBoE;`ic3v}~Xy9w+sx^|S0cax#jU+)U-Ju!e>0I>RGQTa1 zXayqxmHed z;M`v;p*%e&K+L;u8i^c-_S;yb6aN`|D$BSb|d9XHEAhMC22>I<*Ry0 z)81YNYDX2GaEtHFda%}X+!Hw>R-Iu+#ocQN5#-aA)%bze>>`VbFHG;tdD%NWN? z8PAD~>jcJkJiK3yjl$KTv61jUG|th-7Q0C6@%d-lh3}Yk4(WdftUlxKz5(o3THg5X z^$LqG<1AY%Y36w1^DgVIGx%-u?$*I}liw;H^js)>i!W($A-*4l=dM~&7__3=uk-K7 zB(3ishSrWiYf)&e1zJ0bJUR%iwVAzIX0OSTfbSNa%Q~#`67~s|TIUTwzmbn;E4=Ze zv%XlS`&gR4Keis$jSu|L?Ei4mp-=J7zvv_F5eX7%9pN!(R`#Axym-I*5WYUh{2|?> z@&9w;v<}(I7#_~G>K`+6RnF!j9=}GXopBz4&KS|01bs2W`c5!t*(0#gX0^@VwK5^A zxq$s^n+f-_Z>`p!*}M(El9m4C<{H99{^T7c=UE2|xSs|VP{x!)@Wtki5#cr4d~dP_ z1aIhUe(5^NP7V5c%>q|%dXMj8-s7y<>w7bn=O}CIvR{an=Xm*EC!hLf6#s5!r`Lws zq`xl@bII;D&!9gj}-JQIyi&*!puOJ^QB+bf3QUNS^Oj5Mb=|wufCV} zwDz%*b+Fp|j^jzH@C2VF4g;W1vz%|RUqLi;5BWtil(RnJW&R%wChb`*y$n6ov#H)f z+zV!}Q@XPsUfoOnezdFTJ8l^>DWd`zD4UeZ3AC0)!L#Qq{NO#FV&7YuU6|?9x5WU1-T~V(oIQNIoFt@dBX+AeX>##rJmP+ZNmURr5~`&9a73}Zd! zJN-#h9Ov!K<^Qf0XCSd=r2OCDo0V2z!b~UMggH*GLh3$4A>Wj(5FGWHbnT1!dL2gN z-c0`l_!jVJebLr5oApc9Yq|0{Tc!(f_=Y6mC(?}W&_nwpO4;ADi0e%*`gx`=Xg}E3 zI#7x(p)p`Bb8#;F85XbxYQ~81&^RQMH<@y8r@oKDsE@#?yw|Js9{^U3t=u^?~4&uxov5H**zU7prZSDdz-C$jAYV89!c!+Pm~$iT9&rT>Y- z17W_ok+$EgkZ&m>lW;lcA#R`74M1C?<&m;w$q?DE+;ce~i7y-=d3JoNX&M-Pnj?wvqmrC2i7z^XN znY3(YDVWolMqdi+Iy@YDC_at;73Q@Qm+AG-&*IZPtoxj}R&>%%tiu*{umB#Y0qbVO z=G)2659nKcpK$H(eB#w?av3~!){`Mi(VJ+G$ed%OeiTJx1U4s84t>^+Rf-&p&Uz=+%hI1CMUw38B z--kJWUueD`w4V$Qq`(WQ;7l4g)76%Kb=1R}7V1Ap{jy^p@vufRty2D9=vTs;>|)Lt z#II4`a4lm!Qu8&g!{E)^TrV)kUk2vXrHAS{M{Y=NU-lHB*BnJ>Itnf<<$GN`!F~rl z2zS;Ir#;Nldu=bSPWhN42uB|D;#Mn8<3Ru3@5TLx;tXbGI1dn~dB7oXW;Jx)1n&d` z|Gm6$LC%UoZ07NF4_>af9?Y|+d;i3kKsjbypdVpKm8mNfou_bMC^}am^f5;v<2{W~ zn088;>vQd`@Vb0&hmuESO7}W8mpv+1Tt)v+VH_qi9+SYJZ!_n<@>a&l%(;)wzh_q= zSR_5Ww(7MN7XKc`#y91ZI)0IFzXi=SVP8lCFR&{`x1(1?3h*URNc+nP3%dpvr!~$6 zi?f|Z^wTk>f6|&~fDh{Hi+GNBn6Zs|(&puWCt1!2FQ0Jxl)8UCzjPT}!RHLu%?ZY0}dAfh$OxNtW8Bwg9nQ)7A3 z96$dx+3Dpu)9cGt=;)`cxwj4~4d)RypbKN~uW!4=iY}tB8w>gswO(QDWJ`Q6Zyn8i za*4Hd>*(G7uIrj+_Xsx*>d85f=zROF`p3bcEw9Lz$nzykPOu|)xorwipRlSLJ%43F zdUNgh%TAeFrV9h1g*7$qJ4bGI%Xx(|JHUoL1qonTiusN%$o(dR7uilFq3~*$p$-0X z1YHTw-Xh-B+4PSj7ssTVyE+?j-aK-|yK|QpQ%3mg!s(p9X!bA?-zz zmf@7gr$H;PlJ>kw!@nXy=T}z0b(1h~IP)|!N4{txF#Jc9@jo9 zM27;`n2Suh5Io~>jtx=pY-0g7%WhV`+JH55HqYPT8pc(JEmzpKmHF5~@aKDKr@^LV zrvlm$uHC6P`cV9Ly%+Zr#SKg?C4N!}ih7!lqu7rEe9sjWu}Fim(?i>~B= zZRPV`TLC|AIHg?`-?aZOJVVcnP+u*62NCKoS4evT3I~C49@fUaeK*HB%6G-yQ$MQD+lW&?o)!Za4m9+t zIjMs#+71r|{7KCP`0}h3p5oWDjdKTc{prm$gzR^5=kVsf`;L+Yz60CX`@0gqlUhRI z<7R(GyXN!mJUzn^wm0)_TH6;^{(PQ`F?%XqJ?r?Kp7|Z$CanCKm*;QfksiLrpU_#} z9@?q>4fl9?c9KWq7@_<-Q#p%}GOi|UV-8=zzZoS{Yr#kB56rpX>HLEk_W8ee+pv{) zSFxVeMjPt}h3Z?ciD7SAk^0*F9|YVz5r4a_WB*z7U_q~N<7CzyYIt{k!sKAJ>~wt& z1i{jzF5q`}-^-ESzL%TXA2b15Zk3NQWA0Uj?V%D|gzS6i*!QGE{G74vNU)k0G1f7f zO7?}KnhV@d8v1CsRBivf$l%f3(_zp0LVGtnaUo-={>Pg$)+xklac+T36fB zFLapyi%)5IG{K&Br`r!z^n-Lf%lNIQI0No=_IV@aQ=I5T%Aj&V|QY2-!6k=*zj z&xQt&#}!6)EE?5zAv{`h|0v1Bz1t)^Qk`YO6zEa=1iRC3yPxr7$Ii1zPjUWEAInbA zZtMi9PBwkoo3Rm{ibiEOiH&)HG1oaRW~@gI;{Aa0(oHUXYfrt4SNQKx2Kzpu4IhjN zCvq=6ud70}-Gu3mkMJdQ=Xf7P(LzlBy?!9)t+6M57j#we!1(%Vt9^L9pP|t!&pq4T zdbLXnhiHfBM{~W(2ZHsDJnOyqceH-pSkMib_T_8X($iZ!d;1||Srl0ov$t#B72n45 zX`_Yit$D+H*c>-;E$3RS4$PVe) zzo9QA$NoS*cP^9e@DoE5w}m*-_!Q<*L(N=@IS*;Mq0j*rL(`pK>G>Ytf$gW0MSuC0 zy};AGXHr(ox9%zKHU2A2nC$$_gz1iaUt{C#XS_{6ucx2AGNyX@=Md_m2`gBRWDn{Fh`A(TuR<orAOq|Ag8}$o|Mwqs0y(`76 ze@Qpr3#RBD_0L$+3(Y&}&hNZ;+_Y3R<_fR z??oQWYJ=C*Z!S$`IX#G1A19eG%?S{?{#}W0HV;a_wXq-r-phnWvY?f0XeI~RIRkyN zKXYXE^5#5!c@b;l`lmoEI;DIxweHmpA8IW^e0T)AoM;6;+!@1)E5C`)IR8-+J>J28 z^RDN*fGdZqoxQkkbJZa~bNhy7O=q4|cMkW{_1`(}2W#CkH0vpRitC2Ex@IZ&&m&*A z{ebT&VUGm*VI(2h=pE04YaQH6@2E!y=}5?Ip25BLl@t&*YF_3mGkQm3LZ;D2W}tgC zCa}JMKH}F|B*!k_wH^Dj=;2xVNWA|?>WVP#`nE^`I1aYfd1;%~4zT^GnSbXvPkZt8 zibwA~f(|0u_=Ok$yyDS)4l6YFqejntJZwTPSMeM;dikzMS8F4!ae|?_Y@YR=-qHO_{oVI_f)A<_o&UhQa?o$>-*@|HDgU|lQTAqdX90Ax5`IFbm?GJlb3k=7 zS0L}Xv{&c4ng2M`2Osh*8nV7!8jW6Ig`Y+qjLA#1{|G*63|e_d-@b*m2v;j;gW^9U z-uNA|KT^0Mee+(vMY!1keO6~7A9!yM`TU|^%c^(Fzz3LezD4;zrEMGUWv?51wj{$E z$oqVf_$$YGllxWdBlR{ z&fwM@B*&RSzrChpnRn(2dSYZ>V`_)b5&} zk23M!$7iYL;ZgD*ls|;xe6MXCJv|{|UE?5h{&RdUui`90TYTBYdNX%)j|1xCSCEGq z-&Sxh*5Bw~DGkYfd!F~t`d{)*>R+W-_Yz&CI3-ms9X(E(=>G`gB)`ok>HDX*iH2Pp zlw{yS;s#U49{NS`($ml6c|P&NnxnKyvQcT8k4XmJMA~Zh0h&14b|Y~&C_gmz`3dhc?%c|7ugl)ovMl>#M|lTV;S>AI6sf#_S_)g3`dU_v6#RM(N>YrhKm~ zWWkxlU8MYA-CyGKgI{Aw8?H2c53)k^b|Z1;dvOUjxHR}N&t~j4GIoQiN4H$hJ`3p~ zzvP*-*;*z-8|-ahoy+vw)6j^EL#a+0{lEK9o~-6?us7J4L!hTcldk0DLp*q8z)=2 z=8czdUIOOd0_}WM|Z6iL{%J%F;A1og_FB>~i zYI6v`>{II=)_H!!La$EI(Tk=YeA*S#p8qgmD*uD&$)4A7^wq`m*;x9nj6NKL%qm4@ z{pp8VyNKDfD?Pimb-HiP7;feSdo&i{j%+fO=-#p$6nb+E+k8i=`3`;8L*BDzr#Vr^ zo;AYGrw{*1d7G>7m)IM#ugF&-#i^k$M_{8=A3n&p2t$tBOq56dUD?h#UY;8AsQ&?r z4HaAm(XFL>8rhXH(jMiN0ZmVumdt(_=(HkOdiW6Yfl~VUX0Wwrt9vH#pV_ZCG_x%F z82XIlOQ3Yt$OvN2}I@uz?0Xe`Zponez`n zW1g|iXBW(N`>2e((i5tSg|Yl=*0Uvz_P$%+k?RmQde(NrL;ont|T`YN+eTIb4Ie__ua`7wd`yWZY8^1lB!$IeSi4^UgB~^$pkX zZyNp^a)kee9GSs&lP~JytdOVf>%Q)(`+BVFSnP`)<{Yx-n|$>RFEG!$Ej@hXwv4d$ zNgjUGSATFN|NZt`zYUM^w?}TvA}u?7cpLfuME?ECe+F^=!yVba=u`J~rOYntrs7k3 z_?$C}8s?JzS?}wrwe#9l?OPd<*cn014oL)upy^G-ccSL(%&Pge1 zxJJ+2dF~N*@t=9eAxjF*-)4{DX8ME(^=F&8A%!ftO*O~lT_=QOykxyZ^ zq3g-SN3OWHe$|-EUH?($Nr#wM74nYpA5G*e*2IK$$9-+E-@$k2xa;4&VU+y}dY;-f zopSaV8b98~OT9W{^{H*ciF0|2ws|_u`6uPOk$h(p7rAW#?Fx0a%gpx@*@Jvid!D8} zuT0Q?!?K(b-g7a4c6ff&#V3`oc4+*RKh`$&Q6JU$S#^D@pZVT={HO;pHgi2cYSy3* zYQCxZ{R`uKkMP-2{oixz&sF_kml-Rs{-l%YQvK4+Pp$jYX zqyKI0v56A)*SbDyU|+TVr8TeEo;2wvE$AnQkLxGvWA=%{0j%Yti)hUa|LY+sT)(Cc z$+W|)m#bga!3UC$uM#g`fY~NK*?Go`|1Y%-T0P1djpiIrdGVVR&zyd)iO+Ucc=5kg zJmU#}ZfGCq^4wA{ZLQLS-$IOMJg@ z#|kSd*_GNVyiIl5r^CzrSO=D_r8#`j7_FBr$*(-q{=r699<`Fk<^5Erk$d@Es4rTP zi|@b(4YL#MKmXXx+eEt2DT&uQiq^a0{n^uu&q2C#zh~e1KVI3g`|JUOG#@j1SF)>j zy-wT(Mt9C~CU7qqrSpC@{<1MEypm9JK%+}R55mP~$r~Fd>9u-K{jYp#SF$f{vB?ju z2(wp{=l9-NoqJM#_MpFM@%#$cg-|D4yW*S9FD(J;jSxjMxI#{PEz~e z<$(FFP_&yuXrzlm=srOqbZ039pFZY&jr*~=*D`k3Fost%mRB*RQ<&RM<{u2Mk3*p2 zt1(^YUHut9jq-W#clwG(=D%-zMNetJhtoTGN4Q(AJp;5YpiuZL97g6DeCAsWK7(-v zpZ)wpQn;+WXW(ZVIJ(jBPO7twZ)yS$k7Fo)VBm6$4_`W6I&;#MoSPHPr(73Mi7V{-@REs9ru&KPS>yG67- zn05c#-1u7JrSD)L*$_b%oQ%V+?-%~-p6&Y{@@vkaHL_~t&>q??Jo0-uJd-$goix?x ze=!^uPh8{i#$*p)^`7wWOwumnz1SE?|BJ_AlMk90OTIGl#qfIPN%_?NWM4XXO}Ws{ zjpQFr{@9pEw~H^=#cOv@>O%6KMcx>l)SsL;={@k8ywK-a7>~*Ln4zbgxbGYL4)h{DOSWwveZBi(50fp{vM-{+NUjF!t z(2raE$;C72pPT2s)2SaXn(y8p*N@*h1$RvyXZnqg{tML6#J|a!yIWD=ZMpcLehDZ9 zZ?|Kw(;l9>`N;-jMbkfIEDS~j0?pgNySn*lD`Za){%X8$Bwc>_dl@I`PHFQ}jo<$F zy!g}NU%TJlrnU?$4Yv#o7#+2>KxbWpf9R@BlY4}lDJybL&+uUB%UAT`jOpIQ_X&3_ zHahD1v{sFAnsWtXAbaFiS4Ukh9hJGm=h9JM{;i95my+ioG^YF^?)5$T2H9@jWq({t zZ)mc`XXpLM&96Mle+c~}5C13aKbVHDS{}FeGFdhjU$WT?5D4yCuJgu3Q}25|w&zfW zY;3~J!C-}SG3ih-%2gl>veWO~?cN+Zo(6W}Ww?H~)s&f@Ipgxfw)NAz1?kW%8{-W~28jq$nu#RmVLnD_0jo%7wrUS8F=mHR`J z$FT1t&G|3x+rUh8Fth*Q>vY;9p4}nlYb(D9orNR0i2Pj=vpXd9XI%vR)ZAkZJ}XnW zF5pV$Itq@y!};1TaV?^L7f0FO53LACpH>|Dqjb#sytrQ~&h%}vv%rgcLUA76-s8nR zsyKtUna-VF+|LyU-riNUafNIRqPM$9+a8ClF?&NH`td3K*n=~#Jzs&5l%;Qyj(MA@ zD-^v|Av8NvAu{r2g~;}s6oSE*5lSaB@6qR9r)$R94^AAi>WlvLCn@nSBaI z%7(DkVB}ZZ5CX0Z;lw)1gSHPF#UBhVfsyE#(ot*a7wM?w3i;N6LiljIhncksWitRz zWHZp*SG3ljel~O5PJi7jCo?WiSQ>9PkPPWdy8LntmcrMX{|jTgd+{EYp4jfQlI(he z>!ENfI%F+l)E2{0=KM|9_Y5~vR^*&s;UcwfVDGTjpPCDp`(JPL$n|l0_k?A>C=>J_1ha``h66JGJi^opZ+z-Mr1DOV?;%&p{K|elnhF z?`gR&)x*9Vrteh99(Iw-!4o(}qF4{OQ^g=bn<5TD0%&EJ~z zWGCdMn>5B#c)Hf45dLra3K)v-TYePoMVkoJ&~QiKmoLy6|+lNzZZ$30*uT z&C?qnHEHbsj8B7)Ugf<9i4&bf&=;%aQ;ILC_Gg`_E2cRK(15Ee;`>3Lbn1%b-Honz z@vSi|O>^$2uVV9e`Ra-PVxh{YB3=5?Ji;E(P7DLTc8$0!@lE-NeOw|Nr+YSn&QFP9 zq1GzmvG7khU}0}-hls6o$1A2sCa|T*-ow(H%xJ0b3nfE>|rnJDBV?*dfj3Ue%PgU zj>ivQ`^xL^M{NR!<@;5rIqq=ATeil*tN|!IClswz2;LMbgl;0>Oe1>sD1+OnP7-w< zMNiQlFrBFfjZKMAM?B{E$P>eyz49I7TaVJlSX>!#=ABRP+2!gh$<9HgGZwOCYCbDH z`7`3=U$zaMPWroeHi)$}`Bj_+ojKUx0`h}N|X4*$L;rM=#t zx@q&a)J@?2ZzHetXn%t5k}kHXU|6`WV0f7G_|FfcmlW}D-V)B09#49{?}_Cv`ZG4w zZOhn%UcTOXHE63pw2th1uOeQ$O@;9}%5(;J{=zu@W2-|0nLGug);!})ZlCu1R;KZ?#Q zUh3}gpYEj>ix+i&m_C}UU(u;0ho!aApW^CK1idW!u=jf|u4w3PdLwHbVre)JOk zAYOh;aUMqh&Wn3pai0GCoENuUaYkmOIKTGdwkgi&&sokhUfh2v4&2_JfKJ9f-|!d0 z_p4ldr!6sj$38hl^Xh+BU#vHFjvVJP>S55pUd}}kHdAJ%nOA2!KUW%X>2p?4{q>Y!GFw=ov!03yip|Q{sYjV@ZKwG8QM*dhQ z->J?1toAiE@?FwibLmqVP8&Fa~5toY`V9>M^iDSex4;^e>CCmL3Ow@3em3*Jc6IjGxBkA?{6l zT_5|cso)Op#PU^QF|X>Vfe#IMd# zzqtJB$8KDX;tan^v)8$CnTiv?0*{_{i zYl*vEK6IFJ(&4U$2g(1|Q4^nGe@OgM!=I9k?)T_N;y-Ej0XM(lKh$ZTzPtEOs{Nju zF2BgCPcwbJ_gd8#9-@%?{Fg%J<+}=*yMI;49BLwr)ERYpCF6ey|ENR0UoZY4`Cc~c ziS`j_qRtT9>*{lV15C*!fds~Tws+TyGtbU{3%-#pzTr(cnNJ-7X*k5eI{$f+V(9wI`^awuC{GC|I?S>CDPk;}&wiuD~YVoTM&OAk1q%)OX zTDBE;RypyH5-*y&)}^@@yLPP)6E8Y0+7*PRJVX0M1Hv@-KH?(nD$b&v_Yk+fPX4uI zo0@K9^_vVm%CtX*7dv#-v17QI{MVZ_{BT63xiojWNn;N!GR>vASCG~L{s^c3O1dMD zC)!8B6X~Kwf5*DzCEGdjxa!3HD5PCx(o*dmk+8uv(cz0t8a^M8vt`%u0pG1>IQLFy zyt9$TTg#YJ*h`qcFJjG*Clyz1do zom+0GXgv9K|dpzsWCAqeeW#ZbM8GeID@>5^=>m?k0b^cOmJZQJMog=GQflHbN28a`M=v2 zpWktd)u=H`#V6mNiNk*=ZArDCCcpT#-p9N4zCR#Nv`c!6b>R8lJ0%~acb##4h`9B2 z*R?zPPT86|`$u;ZuW=mU%47||F}B3SNw#z5u<~b^`EBHP-tQ%!`Z&rhx0r87$D3du z>QT-)6D!$V|CviZ$=xS&IadNt^JU{-&>MZIp1bJex8CSBIR}f+IBb$6Po>*`)Hu>s z*_D>iUTjKtmfoC1nzJW9fVt(=Io+OW>Xm9=MR)|7s`bJZCN9~YY{Eo)xCt|Coqvrl zcg8?@n$Y)4W~kCWjq7(%eI8)^C6~=1KGOGK5o53h7#g?t34M&M$-u;Sub%%(4r{L=2*!3wNo`;xdYVBJL%X@3cpX$HsNX_ zA-K228K*?MlW(m7G%i{GgtjB2oHN6MW059Wb!t&5|KOa^jMvRNth3JA6~31!&x&fJ zZ?=fPYo4^SuKba!$M2ZIKau0eAADy<&$&^hxiM~V@D=1fpXY&9k#pp`l!FfJ4wwB6 zWxtA_K>4=Ud1URoNd^?%lfAF-B9iW%n}S~e=-#`WN!r5mB+ml+EVbXvPWf)WNp|`z6wA;*;CB$Ncm(t|=*4;_gS@;j|5z_{)9qoq)zA z*K9HMacEqI{g7Lq^}rB~bLzwTU-dC#m|#EZ)@2{`ZISXpv$bw+<^1y{$aP+9osN1l z3l3sayN9y~i|+TYmT%>9-g2lgc=GNVmckL$({qLfW4=4AakNwFf+-roE z0%NoFi{#*i;~f~^AWbw<`$B6U_3SGCrV~Gtc=@L~-k-BN(Ct?4i1oNS+pm!(nwXVq z%?}}S=K=dA&;ZMDuw6~CFTZiIT9lDW+Z(8|RKQenzzT~!6 z*6gB1qOTL7Z;lTE?E2#!eP$tX&iYe%lFZ(G7IDI*##@}a7E&K)tkUd_(6rvPHNoW1 zuye>~^eE6*+bw%GandD-=1R8fF6;O(jduSvhYWUBu zP5xB-&*alR2I}8gZrOJu%3js0tmMz1yJg=ZUwu@Dd+z0(bot@dn74Ayv8p;_+D7~k z$#2Wrx+n41;t8&&4^-!2%(cVtE{%bB(GSr(t$Y32ZSbSbr5kjPI>DX?UW(=_|Nr7Y zihJIKDfT84CfOl{$VA^W-<++&zQE8Z&cCr|U3hBIE7-@<=Q``dIk+6)=Nw51enx)L zBA(4w<>25fzD2(+XE0Lca8HzJGS#kNi-J z*6Lma-S?xj6}m@6JV*3_cI=d106e-l+$QQa@@wGBZ<8D!)J>eR2&Yx?-n*S|P)Etc zwfG+k;Y)EM=kerA(MM=>!07i{pJkm4e;N9Cb_U`*<(}u$Ot%>xNx>;oecgZ7&r-S?Tn}B ztNyF`t2&K#;fQuBK9_h$9!<9MTsYjJxkb2_#eKYjA^B5s%>Wn1P!|UD5E!Yp=6ASF zDWqlLBl{-GnDsO5{Ac$AeR{Q1ht_9HLwSogu1-eAeWm7-&?~HutJt4>5X{lk_zFD&GljOn9b9WWK zy*~%r=yk)e<(E6ze$<71fO4XHJz!7i33~~7kcWYt{(0DD%^!xHZeZW%!rnq|qe@AS(T z-)X%E`ncW`4BqUNe}VF5-DLL7GVSTE&Fv=QkFl4a{*=tEwZUG_^+}GcV?IYym#2u6 zEU&omIlEGUw19k?7hbpA{axjf>`%bY$Z{KK``6ua)uvnmz95`F_!i&cHN*ZM<&w;r z{z_9W$^JU|92-{b(cQAMh;!&{g8j!HWfRSsKVZrx+7;x}JoR}}U3&lLbe*L$eC|Kt zbDX8~LGyJ-lg7!MGwgAuF2$wa!XfTPWSm;6jgED5>DV;J1ziVY>hv4yU1-a2{@JaY z%CyhpTWiN*CQP^UO_*j6HDQW+0&M3=I)&@f91-?mGg^gF%;>u4v+|_JoHEYX_`PNqH49vHy z)4eGX{lEoQ;e9np8NSPxPJOIqBQ{2f_QREqJ{g+5!-D z^q#eR*T-fySiik%&;5#{@2&HlJPY*=?`?9w@76amN9_ep`W^aaT(U28zHievbEApx zM}KvP`R^RXA&+Mdck+CL?GzZ_^!N&Z}bv}M>xFO!yIyGSfYK|O`Izkj=eH? z)qJ5koBiKSaH*hv}SxVlNh;dO}@V)Xr@Ku+4sEc)$^4-`j*{VmMMC6;N@f$*&qi$?}KWb)s zb3;j}1-Ril3fALle~w_04*V`yz%jVc-}&ctYJWDc`b==z|5mT|Q+6+H3&RfMLML)_ zE&C;FZnO@e2n?2S?{xNHt8gqdz>IO*_J$bfNCr4e z{Y;tgxT)UZJ|N$@)O&3@I7~m8ahq~x_qYj%f!70vDSLo&yQp__D)8yA&-IJoQ%?C3 z1wV!Mq?Cnv*ZqCcw^Prx$-qIs+ha^kJs;?T6Q%x5A|E>99%I@oU;7E}^X4x3R&Pnj z=l8{XrzB0YR-QH3GvNHedt*yO<>jO&0B2b5dOu+LPB=m0RRK~~}I(D!ES$|RF{ ze+iADe;s;}YR{eJ$oBQfIrYd%5xUIH)`qlTiF>y6-f505^E~o(j=~0>IWO5qXL2Q{ zNS;7vv>Q8%-JE;gt#y^o?taz0qGSC<_u#?(Oo;8L2{UZJ3DfOZ6QEf3}X#z<<`3qrV>)(w!V%)?Vc~#(2mt$rtsB z^0dD^aiIV-cBl@*kz4ACaap zl8#ql?{xX&d73=U>?0{%<8$o#aiI>*WS^)m3w?Sgw&cu7a|fMQcS`lawjmZ92OoMh zKl-&eba>p=_%gLv{X+e(A zp4&|15JxwJ{{a3e->x^8I{p4*zO^2xr42?#3*U)@-cEYyiSns~emHz5&KTPK4EPxC z|A#5BacnSQqP^UNx(91AxHkzLoCq$KgOd}mw;9h_Bxj!cpiKsLFMW1?H2?c==(9~5 zsITgofj%3bG97nQ->>2WR{mdo_@B}}rkWp;cQrTWSkY~J>7Un`55=dl4lVC7FQS+i z+)K>d@LEfzo^;nOSJ4muyYu4xDbBn=-dkz)%?SG3xp5h=G&i7AQ;#q&{>bC2W!=a8 zXq{obES%8%_^CTThLK-+g#&+dZ7I(sZY}dec%XT40XXQ)i*8?~=`LRSTs+RAjI;li zW%$}NUa;ngY%jG=)ZIxftb;Vh()mcX`M1;oU;mj%xE~!%FXZL8s z0bom}snp!RAv@>~IDPSNlod`Q7frnr+mWkzM)Rce$aiEYw>1CA(D?l2to07E)@vQw zH($7)2|aik`&RM2p}!D8c=_9I*$*Si&gxb64a)w; zE&ErMMIKVwU%6%fE28YwUS;=F_GP#14)V3K_tgpyH?(QE^gqiSIu+tQzVot4N zb>ipA_WAT_B%NA=&zUekmu}RdQ_1#p@}Y-aS-LZua_EZS>vNH{o&HF+pX1-6Rkyo1 z^8(+Z+r4>u?HAChQM6+L^FE9tPf}ic4Bt0ls{NP=lk7)Km|(9oq5L+SUY-xj^8aeO zrFy48w^E>66Aj(^@;h~9w{i?x)d8&v`;#{FEi)&0Ua>RZZlY~w|5H4+>Dj5D=Ej=& zl*oLNTq-@)H1aT4!Fkp+u8qQU;;{ci7v|VG%>XasIfn`^Yo%+-^`iI2vSua)zUl3Tmyz|!tH;H|wZ zkA9PW%G}lKb^7O4{?lU~M98%>yZCjw{RsKYJa{aJe$|=YRC^<9OVM!k^9NT-escVM zDct0a>4(Hi7VAA9n%G}Hos2u0IxYvF9KT_l|De3qvAa!}1m1u7-M;^7yjT5F!TVJ3 zex~sLpRD`DuXUHO?h)3#23r5genGUqvnGb~nlWvQ;L(fFv4q!vzEfB1*@(>Rto@v` zAX@u*tz}co+yyI@~7qzCL&@@Rc+=JLHmf|t0q;kOenywzG|1iUVZhdWia z7{AJUeEFSMnziN1H0JhSjLl-2eFJ5jwGH}JcW!^3xF+U)cx@>@Hxr!So1l7kuPY~+ zdBFW%%meYPD=Fv9({x+%pEJL;cFZt*QuJSZ^iuM5uPYrsIn<=#|HOr*JdSLdYH#iV z%h`L$aQ~ezSm-`fFUk4hrHLjj*&gJ=Ql9WW-w%4gN;PNieeS=L$tRxXM<;f0eN;#? z_(62$%5Pm5fmt|4D3I9J*}#2I^d+SVJp*Qv)U9G>$CQPvZ(}Z7nz52O3PFb)1 zx9&JKbg$QXb(WdaQM-=c;$`g|%^Es}wKP7G7saB-@dULWqdgeS-vE7RU6mVL%$W+| z+)rqa=42)N+ZqS)t?9grUjB%*FyFe;wb}hKaRJti>RZWZA@D){^c}`t@P0_TgLf(R zpe|g&mKJ&&&750o@~7JkKW6_0_p$CaFK3QB| z@h`)xNM5n@EAa})x24Y0h4q9QNAU^iK^9_vbscBMZaty1&T{l2 z%cgEa_ar>(OJ4{tw?qG*ba_PpUQrJI`o4XZq30#MOFy{b3WrxH@1IZ0E5iS9c*Pp` zABQOy<`rGAU+oF|Ve*Jqn0b?ZcJO~)>eO#P@tRLsuj@SEA9-Hl3G<5ZUK+S3yV;ep zoBbO46s`AuPg!T3pK32~^`bk83&2wxeM_>v2V8XcMYq1?KZw(Oc#ghNAH7bQu)bv! zX%4^W*0(%k@@Lwc$=9uKarnh*B zF<(h%`7fKRs&7~iGtbOb-7njZz4|kt8U3Lh1E3)Tp(Wg5y(j@*5FJE*+Ud)e4MLz} zBQ|#4OP&4t*Tz_dqT97&oc;O|LhaW_6P`pCeDf07BYW+t`rX z?=tq*i@8@YHqGqUKccu|;sQ>d@9KLD-zCoX3clBRQ`(zKo$m*gj!vo4{a&i?5$w&E zX^u8b&89w!6^FjINpZCgn7Dh1s~iiAVephVYv7!#AD=q$>aBP97Kc{*PR-gA7jF%m z$a_BTBNqEIs?JC-^_xpM!I2Gt?qRs|P4??(%gw}Fg2y`luym2c%?$qo+34_r5RA{{n7jkF^TxpnJ^|!h63N_OgkQb^vMiJ4e_@F0~3DKqm5uK7M_$ zp^ui~_3WWiwtEjX!?goQvuiupXDnggmUK@o_iaF1`mqOJ8L$e6)ZD$tl;1wsl$Q-N zh9r;G7F&gq%YBtgrsmU!c@v5ne1E)qkL=}rO-rZd5qEGs=ZYs&m&JD(pAC=A_60W{ zdvNOZ&5musGWO`3W6ML%K6{3-0gz2WAL{#N``}>C@y;g#;AL}JcfAg<$K1?5^&Wih zJMGE}wvaB_;5O>j1g_O-4}bLp*9IZOwGogFLYn;uebl4;GRhZIp8ew+RerQp_#pfG z(2K^sb>l5h#;2KE+d_M7DE0fW1RTgEtbxT1I3AF_giq56&Qoz_k{^jAtN zR$3u#?LR&wSkF*Kwi(k;w5tkJ+;?-lZq)sdXw-q>*=FZiu-hyf>mR6it`=k3+r z{X>E+@Xx*d**8{sY)If48FdvbGnsdUA!zFUkjRhn!kfR5NidU1Na+h(b&WUuq~-$?!I4F)?J=;r)*0Okmvgy_A*G?VTjZZEL5V{6ic4GCp;T6djo%F4#&D%xlfAA=0n zj15T#Wh9?Dv>f|R{%u`L+M`zE(HZ6N;|c7+j}INble3}p%Vx$7e%y5q&A@-|%-}57 zcEdl}vEA6{y|#T0`SRi0`S5L>&wUkJ;ZuvI!P}?8Y4h<4JQDe)%jQG096JGH3(`qFpPoT|X~QZL;_uFc3HJ9*h;Kg=X4sFIFx_5B zxE7lAUGtr4KV-rbdxZ&;?Rpa?*$*nbk~8fJALS0`)3gbGG=0$|=Uf`PWn-iA&+7Xr z_`L_&=d0$|IQ)wItZR4wGuX;|3?A@v@_&`Q8sE+2MTcPQX?pi<7zd6+`+rP6*?@de z9^uIk$n$0CC!t@DktcHe!v6^4*+ZnurbT1hJ54yajP%o`bxa!@T1;96dtYCauCvj1 zseGAq57FRY47k__oJ8-i$j7;XD5HBgita&c!hq&a>N&w1i<~voql~rDJzztBbVh7` z9%l@se<do&m@_CAV6Z1#wqy>}- z>m)veE^3{my#PnI;Mi=;aAo>$kXQT12c?(h{QEIvHgpRsjs6PV!m)TOLKmH8@8Tb- zci6__i*yVBAYUo%FWqoXmu?{jIWMeRn81G{bql^roIaXFe1vYH>nvb|ZlT9HK&{_p zUli6w52w83%L_~>-9kCzGl6j$&v=c4W{kyeb`Rac%KXpUSp2un44QQTI;%+CLIZU? ze#_6P1Nxg8QPjx`&fv4hi@%<_0lRcZhVVxHV`wC_&#YPMGTIFu4Hz1HVK? z(vR(9ZT$pkf-Ae1TX@IE#QY|%0czzQs>9fU2-oi;{?DA_co&>ddb9V@^o@1be~#{F zOQ!_i5br;2I(qT~U|C;~p2OYU^e_Gi#?t3y^nEE~Py!zO(wx^(8^d*)0h}lC2QU0N z-M^Ynbx!}A=Kt&3M~eYQA7I4-le219`Wa7b$?8M{bQat>2N}MXO}^$sR-ZQQ@$aL~ z`Rw!KbFAA|CRe(0A8Y&=^b>u|e*bWEahlH$P%qI9$v$EJX~7rEwYGhAXjnJ2Q@Wuu z$txOaaOSMMVDV@N#$Ch<2c+*!;Jm~Ubp0DS-}7m%&(NV{@aL%6ze~1flh@G;C)@pI zJLPXCt{Ht#w~n7X3ZO08uaz!6OpB%vr@f|P@JDsOhH_yY|F5A5j?SiA$3N2K&#=qL zw-5U4=;a-q&G{xR%|6$Kr92K@OS6C41D2!XAL9P|Le&-7Yp`2xB;Sr+hx@;Jlymet ziSB=fm~tuh0Jq#wzMXx56nk}#a?UwvJC<{y!C^t}ijZR9?{56EZc=EgkwIoa%OzsP#p>C;U6({e`#w9Oso<9uuG)jB$$ zF8XP8?*sIppJS-ogWz;{yjv)*`T3p+(f^xJ`u|Db_(X8sk?W4%G9KA$T!^_4si*yy z(NfhjY_E3RSDJ${&{JnWJFKU@Z24$FBEx@!=!xKEQg?9i3RBeTAz# z-JpITzxw@CV-J#G{}`MI-)9}h{~&SVvo+v^@c;Xi3+pW7TpSunTcke>?_=L(%5Y9W z{{U~l<(B;p-(kEx0UmVgJDoE|-*o?V7x`MfDe!RaUl{5QeTwd-i-&`MpZ2o``l$y z6K`;Et%|ls$QNCI?`5x^51om$S1+Os>!EAmzHgzt;r}Lt z|CpDf0~dhG1r+>s7` zuAy%1l;Oc z*Q3`IEx(^Q-6<|vp&INIKIFp(yzX|Y<(}JS?-K5k0H<>uy8dha z6N%H2az!LvZzW%paC$&kHxTIf?CQ^+aOPQ>9pL|>>l#Ca9v8REh!4~CNZby~7QJy> zyjy261V=bsKzY&iJQJq=U&rZ4+2UR!TWCMCKl_>7k$cNP)|P{iw-b=N6Ioj(;oCPR zn3uV(!f);Xk43)FSta4Aik5pKCY(D)c< zEVs@KG+s@ZmyE(afB_b=GvckixQl8sMy?uj{xPNBYY!5&|>wY9!~W`oXef1f=m z;>|s3*JL^L`*r$M`1#jy=-mf9aaS%o_1CEQTla^h;yiLbkV3+JxW?Faa_q$5XeVPSIM?ZeJ%?fQKU3-Nask5;&TmCn(jS2{=WTpVJLX7i5oIwS;sptSkAq#^GMd zsozcB`^dX5A}{||-do+g^YjnuVDi2}UT@ScwcGfcp#I7`%guX}@`3{O+K&r z(Wszrx&tc@Sn9u8==cfxR=DwNS09r@nvqGU|Jl&dz3b%<7n#I86EJ`{;r=RUn#Ll@ zg{8RV&^~7!=g1sqQQsrP34Rp$G#4In%f)t;OS2dCDku5-eYf0^u_~8p-|v>|;MN<04`u)~(f;p~lt<{v1 z{_WJFtD#lXp;^q`dHLp2|Rz^nTRE?fJwB zKVDLQlD7Y}a1!h{fOCa`lVD#jIM6@A+3&)c83~89-CqD_KXAsla3)el{Sj5Z#Q6OD zkoZsc_?+{d*vTwMX3mxmtSruVmLVJE;&=ab=@`>(+sp??9+FPtOy*Ho78GB&vi8Mhpho^EHEFu^`kAvD@+ zzBB9z;?vyWcQAJ#eEN&;@eA|mW5}58@M+l#9fwa#H*oX|`1H+Q_7>5%ADhkDWcHLA z;nPJPtMC`_Y51+-(>D9yc{SqG@0vU2YSE8>Zf;iPclo{BQWiejr1L1JzyI!F?iBV< zojdTh;QuaVapvk){PC3=AD@Z#Z)m&d*gjyb#s8fnrzP6IBR))L_K^116P;V+m$(Lh zp0n|Tr!jlkE&B)3n~sOerrWPXlzoG=6CIsfRBkdKGlHM9&p}-C(aypp_%3ZO3v>keuH?J9*eUa{(m(Ai%PoGt zgta_}>$qEkv1+_?Sbiw?-1FAHh`q^Fr#?Br7oV&&`?mN%c6BBh#7kc$`kw_Yd}h65 z817wh<$^fkv<`U;8l|!9Pkv|qa6bgJ&5;Y7F+Yp?e`Mr=1p8#E%5mSETh7x}F5Uii zuX1B4*X)-2$dtohtXu9d-_HLs>|5M&?bJ_o6rSBo*hqonnB{Q18`n!`tlW1QXq#tVuwe3UrG zY(H_&nSA1z#4&!+@RMgvJ~xiBJd3zzh|}D?)UEGN-TH>_ATA(Xeca72fk!>ZQua|( z7XQ7J(SM?rIXy=$pn4^kGcZOr=Y5f>SGpbJmaFY5mu^e` zi!3J`m}&llPefBL!#?KX?j2p_GVJ$zmAjd8Q`~YhmCqgjR=4bI;*QXNMh+wGZMS^S z{!BDyYKq-Pa<*RKIW^z zi3IM4p?q(gh$b#vr-$7-`Fho9(HGR|Eb6qMG|`n$N*Et_j@}o!aE=lu{Mw>9OWK^% z!f|w;yA2%HI+R&oCwUY+gzn$WKX#Zj<^^~xnY)HMpF9xf$Zaz%{^_*8b(-{?-Sgox z({}bPf%U@m`0OmZdK>cE%F^wQKC^a8_?&L4{Tlz3eA%Qr*WvdVn$XVwf1!Bh)6Wz_ zTeq2fsrE|>nfqH5LKl9b5Sp>sq$k_|Zo&lnIfcl6j}ZRHDSS5rw+%T?emZ;mah4s1 z?hJMI7^3V`lvxvHwRw=UrlU7sn;aNDkT^sCc|U{q_oA$7>;k9bPwJ2ddjr`j^!DBC ztdDy7bM{4B!7OY5ZbGJ0y6W>qHX=WLD!BWn_RF6fJO0N{Mt#uv$=xe{u=}N})_?ML z@t;0<`>FO%(0y&rpEV*l(!)A;@wkpS?rEG1UAx>DAjO*|pUc4vYle%N3 z$KUoE|D?>$5r@ij@5!zsjL!woap@%eCu;V{e@2$qI^_2(Uv&h#1RuUCrO!iGJYD$Z z0rooqWIFtG3cvDzWdW-l`vxEQv7Gx$#TS4b%<`nRWz&Cg_g1w}=DUVAmjP$;sj+*i zAD{H=YVP2ueqb_TvI%eX_*cwZcx!tDd7tm>+&XvRx$Pe@UjN{K`j4+9E139OD})c# z4*vCs*3ABsWB24g9{6=Wu<{?6NSJKG2I`UTiC=N+!a410Jd7jb~>JV8z`?JpJYe=)G|GCsn`D)$1%sc4qILLj6l?(IRH)3z9G0B_Pr^CUA z)Bi8>UYBXuq~3`ih)V2g|M8nsPh=ze&*B~VW~a5~>|luYD=mCS%Toa>G5jvvkzE>R zPgs|DS$m(}J`Gq~MpCaA9~s{vI$p&4kD{!Rc{9d$RN;Rco05;izO~GMZ#X}g!=0-` zz|VF3pT66DI-9DKx0kWbFP$gbs{f!~uS8q(XE|$=k9{rY2jjrYgW!p5Tmqirw~O(6 zvHoP|^d+n(U&7i#WxT8_@O{2RKF>EE z*IfdRpK<9yrQg^3O1As=QCHu6fkzK|eLIdv`Bxl^O5f2LmAGjsWA#B)VuR*@A6)oc zd&gd2?iGK7fBHOe&RwVA<8p8Mjx5R~qW_-hNj!bo>aP9Bd-0X7eYLIRHDl476)aP`vYDIs zZ{P2&UsdYOFn31l97T-g9Wa}*s~XKc@L93e{1)!Kyh-#n`>ddNcL;i1$rxMB=LfMP z18(u#%%L9rZqe@@z|)>v4D{_1=7ajlno`nnd>Z?^cUmC_W^V92_=05F8>pA)KsI9VmlK#Pkra03u_q} z^bf+8Dm%1VbFzlAgD6+ohxa}fE;e|N<2}ym=*xS}3ahau-Lt^jZ{1$GbjhWaOPBK0 z@hszcfak$WYyQpZn%kPs`nL}qcPDz@lec@A(^2Tvyy(}szX5;QrxtPMwa$azGLKz+ zGG_kvRQXeu&5LEr&a9Stfqxz1Y4Xee0exFb{nk*w1A3=^YpCCT-etF^aa;45C$uIR zx{9AEANzKGYxUIDA)(c3>$`;YBT9?v@e5dAhAstKoZnJP|N7fn6Rqj{@B^>2xA@X5 zfKG4rl@tX1oARdq4)l;h7= zwnE}%qfMSvdj)wmb>)%lkAE5RyiXkZAK;|fOUUy~S03&4IeGq*IHli7y6Q6Dgemsz zgwwHSI!Zr0&V13jXdZFGiO0c-TX=t*xnXQH6%SrK4qnVse7qB{cj3t6;K--^?>c?g zy?JRZ_{;s69o$FIQ48!fUaMo8(tx*yxp$S`fw2a=k1KdLkPy z^?YTWhsS!pl66!;{Wu7F<2KJMQXuC+-L1J0^LO+CCVuVpYKp z`gDu{H%#8Qh=*4fyw#_}$2j?~9^WC}?Z3JYb80;EWgPQnEc2&~c~pvCn)@-(D{7wA zk18!Xh~NHvbZu?9}|_idbX zj_U0lYnYv*ojqu%FI z&xe0Gx$$A%Z>1mCc=}ZjVZJ@j`x)2H+xC0v-^p2;rOcxEH-i@|Q(A@!O+4@1MW#~HShhDL0*E-en;53B&V&i78svVsfwhN*4Y%2VsJwr=O%w=($P2)?y% z$o}>8yU625KKQ`v-%%eGx1u4+x1+|BuqlePYEME#4gMXG;ie<|HGoIRkZ~)1O&5l%Ke=dvbN4ZEu{~h18z3=If-YP`ki%I@EX+*o*eFzWcoxhz#Q>D z=4A`%&CC_{1tkY;&iOkw*1#(uKJs2QBVR%9g|q4x|I6b$up8Nw%Xs>cTm3H!*Sqgw zi_D+zNpASE^la)E?(6N;zxTRGvO^{Hxj}VBE~sH%J8PswBd>_}D}D;;--c!(%Wg4k zgBG7pT$6>2sX8BIUgB^6;E3g`{0CLvo@*kDJ~Z`kdB^hPZr+gtjqmh&b{(DJVLo&3 z&LV8+U$=rLKl~&aev$+~Np$$hW($7u4sw9%X2t@3@+ZDq_04}S#xLzFrR9fO23gbB zES+>H+F!D;mUpGW!{^_eUUErazLA$#ZoCNmA87EOxpG^PFKV>r!b<<4>Y_Q!i+sxX z4-G3KZgT=WD1rIuw4=HxE5@3?4*L?NB@P_hF$CGeqzxYy^bbEjSO7j4+NJphoCJ3s z;RopR^T;zM2t7%(tnW=-Lf(a!Ida#y?H(ll;OyYUsy8GyTZTzjgLC;ZZ_>w8)p%A_c!;~ zc=~h{@9NW0@ZmV8-#?7$(eG8j@8vkA~L^@0ACMgL|4 z@p^nHCZtou@@4Sb_t#_)d&JTm;>{(bepv71*9zo@im*QriJKYag$5B0#0 z=+EKDPG5^BC_RUR4`0sHB4O2k9@lpIuJ!~o{|{h+Q(W1rbBN3-`mx~bG6xUO_I6l{ zEBEApH)oL63-@;V9{2RXy=N}99upk3MYyMNJ{`VwmPPMs8|I&PFgFg}Ks@qKU*sU} zxtwtZxY-{%I)HOX(Lwol{Tur_x?4uR55#L*F0dN4K5DW89q(CI$6xqorMt#?Tkq}~ zXAkdf%(gLLpW1)NO;e;#Pm+opf+=9~e&=sOzO&F1d`dq! zbsiQ>I%PE)UImZYdc1R_kM-!gpLN%%4*iOFf&M2uasB^s`DD+vI-I*pG@ zKeT+a2ImlGzwm?IcJh`_KKQRce-iihflo?F%kkYbSo+GUv(axX37oP1p{VHtC4AN%fQ(SeTj3h@g&TVXc( z11qD=!@0Avv)Btm$6bSuA#`Tb%K~~oqs_SlCh%c8AAOVC?-z1GLGtqX(EO|?^V&)Bef_Se3-t;QNq{Te*r%5wu9 zA2at`paloK1MRoq$sX1~rNqy)`nNs9+UFn8g=55j;LR>TN2a@7M91f!V4VK=QRm3c zX!-|O#e_?s{Z&yJ1>0LKSTElC6g(se`kMHOJkJdK3G#Fv&EaP5oHeo%cq~0=E{|-B_VfJ&ek;P| z(#hXK{>Zs7D`GB)S2SY}(i_Huz>rMXl2UtFcy3Hoe?-jfaeOx+zlG-p{%XwJ*4&W1 zHit*k$ zO7sKG=#HwO@f!)}K;P#OW?BBW5BQh(`NSkb-8sp3KJOa)iS$W+((+YVp18SrF>6-& zu`@5&HJ&qyzJ@$xEy+UPWh|s8E2r*Z9-?-Ic}OYmqW7(=ld~41--nMZp^U~yXV^RI zqS1r(h5q-051au%=nr3jcP$!-ZESSV$3K04o6K2zd=r>6n9f~khh4p}bW>V8w!qI+ zzx;S&79S&-|qkY1O1j-{H?$_()E)b)}d0IOz<&5f_Z&y$!v0G+_)+ z0?z=RWS)UMeRxuM5_w{Il6VI3r1D(%jhg4L``#rL*FWpK;JRmhx8LyWfZLzW%Bfhk z>=zaHJv-n6y)Pne5#e=wUq@Kcr9GOz~RF4&KecOhkq$gh0I$aAgx&i_rG)!)0c;!Vns z@AlzNx`CPc9RrVcIRBdJ{MVG(%6k~*fn|*qY_uK9fd~lQ9h<0oTXFrsC#_SEg3vS*p;DTkD zLn>x(7;yV+LJQbA%QjR@%CTN}m9bFf1uBx`sJ3DF%6nwUTJ^ zOuoy{wgy^hlNOd;U`=nCX*C}CJ?HV;ydnH1ObtEt)YRoqHM)M8X7A}(kKe;$i}k;C zXhXUy>u}H6r$Z(0aF!oip1Y81rX$xRA=e}$*Q9V4?lI(=bmSUz=_~t#o4Svvc=CwR zap3%J3e^qM~kfBDP!l?Ut0Zf$w8rXRy%yQ1)a9X`CVbC$@oxYhebv8i zb#yOG9c1?=`*QII_SnrCkWPHi)X)6c>Z;tAR{MsR7UlJ^MpkJZg&tD;V zKYnN(_XvH`IelaKoufB?(phL|q380(D&9q>v@X?pOZ2 zrb~;%aba3q5J8LMp~clME&e-uxBn-!c;UZ*7BBjLfEGXWKSYb;p~Z}&p~b~6E!J2R zM9|_{(BcM9+>Yn0zJ?apeFa)v-HR57|07z=+F*xm#Wh?GEnah`bw_nfLpt-r(Bc~S z{T%rHWY(mu@PrUDMFBeWcpmh^)i;4>8~+yH$lkKu>dh=L@)vkt&b`HxSc^?U?;U`~ zS><8>_OX`O-WLs&(6EAzRWa3f(JVV1`_beOxPwPz|h zM{`)=v3=#(!zJ5)0G9r{+k~n1P7@|zZ#)J0>?-86E0NbKklU_c-F|rpInJ!trI&4{ z-6y%rSNpE#P!7MZ$aqoYJ-u&V$#v3G=zO01)gNW8qWylIch#8;(Y;Le3x2`aG0y0K zV}f@B7g^fqJ}v6azT4JZ+F;V#V}rMmu61%NZB+V`=x!JCT+1_pCz0n$rm>6gU4gNdx^672XWW4BW%d~1O&tIz!Kii!A*ITyGq7GwLZy1vRC zJ9Fl4V_p4mHT-3+C$2$!0vW&h8TLJvbN}un@=YQib7SD-Ir-a|3m;D+??jKkAs_xi z+vC4Vo!vMS zY3%&eL$b_7_U5|#C2!law6pk>cYgg5;OHNR9NiRqm*}OIr^n5aUMjA7*zgtUlEtK# zx(~gS^i|7attnaqtcSmTgTC|m-dP1t9VA|=^N=-OtIhZLnskR}UEa9d>z^$C>4z@~ zex_%6`nw-@7OGyFL#iixqw}Ax@l4Bw$BKr`so8Gq`E%gKH`Vx72ZG~SZ==Ub4IHn9 zAMb^ZAQS9P6Tj_`4^MEo9Q+RNv9&@MtQWMOAZ<>!v`j=Tk9 z+TV8JKL@WjeF`jp66a@yJL|H8?5)|uJl3XaY%=0c+1o|qWgj(*HmW}5tg#MqZ{Qg4 zD4*|q@Fw4r(U7+!w(-&>@r@IKlMikc&l$7L=gEvOkLlak$Upy1Tg$ufbJ4aNX_ssy zWczeF`v~2SEqlFY!sjTDK45opp0m!g=;yUVtu1Cho%mO2i}p<9n?Y-S*+jfcpPTkc zpLvKKQcw4Ojpn1TAM0%LJ_+nbXn^xC{Nn&idt@zTmeC0gQhz}&n`2Anp%=W-hhC}` zy;PLZOWib>HTsPk(MxSaFV%`(%74?0VuccgE9vql9y{=lgybXpc`e(AKd?zjBbDSv(-B-zl8irX}au>vnb zcSFa13A|*By>QuiXdi&K4tpWcxQVlllD%rc@j2i)>)Fkd{^k$XNCtzZA@A?*3y#A- zq{Chr#?dCT-)Zo3WzH`K2HzpQMlu>}uW*?<;==8DGon3JoC}I*4|X23r{8JX(~tHr zJ|Xr+^VJrQ=wYILK7I8mA#~WR$A?A8fPJa|L#_FOUIO|34rDKL0-? z1NKD*1fP1yfEtTo5i;NsWWeV<@jI^Y^fNNx`mZ1ZKGaJF4FAV6GGP1;);tZ9wH`+X zd?@CjE*WqQ^nN)q;N`9iXy(AHBZCFQIkS4|#yvBa%=+X5<{~_CM?Uxl&o~6nIGzpM zNe|q#XR`mTRm_1wHJ-k0w+?@3RZU4=+mPXtj@Jk4(_4Wlcm>Fn=!vJyfy50ELH zJ#+4NhaXbMmDc`2!ST!k>}O>^VCGT^|J%r1`aW<(Pj->EVZ>}YlPz{;Er z3D^>(*eT=@-P2sC(!7GMeba>UZ_?8~#@G#@-@p7~&Un|!rqs*6Q8Y4H402o_&|NA ziE2;Z_$({FYu$N}K1{{NE_GjiFyZ<+?RO$CwR(rx#nH%Hq@_X6@-oJDWG;?dJ+i%K z&&H^zhAdC!rrD&4w|>$6GY50aIEU)^tCD}V{ZCk-_rK!_9pFrt<5Ph%7xYP3_Snvt zv?5Cx{s(=_pSWoBL3E4iyL9pekd3E#t&T*uKa&Vs(FXSnhG6)*nm#{2j_ zxV|KGDS4J7C+d!L(W!cGQUkJ`xeHx3*ZR+?ks7N+dj~x3AbtE6^j_;(^{e(6kXhRB z!$-fynSM?2^y*iS)31_k4x;OlF4ekzZo9?)!f5*N&)DA{1^z+)69@kOQ}g& zeLlXgbl+ntW0|@hdOLJnM{QI}!%xYpJm-+EJra+Xy%P9|11l*U;(9YMf zU%%1=K4KG6X9dkXC>#*HSA7TW`0$sw5nXsE^P(l!VxN;c?Xa&Kv(u+zuCf5AftcNDnq^>Svo+ZsG>3C)7tWS=?bFYC9 zzKwqMb@U%*u0QPy=J=d53ilR5!-&sf-8D8NaQrNub<8Qwm3E+i!GCF-v1Q7}_FOud zuJ{f9Ul~*0DgX2#jr{C{;?Z9r2yLa3i=Bppa1xv~QNAlLgiyt+-IKlpi zxJX`{;7!@lgCE};4?j+_N06_p{x|eRhwRk9oOnY+f)^NhI}duAKfDj?qCm%6(6WNX zS|`S}<)dp-J+sj<`Ab`O=TN`w%*w%P6TIJP>y!P0Y3_gaP!_u9#DD7y>fp5pk=C2{ z%Qp64Zppl`ZLI0v(fxz3x@SPXEIx4`+>XoPjbFRlz*N1kc+LgWX_&Q@R-k4yG zZ2d&Ki^S@U;KK*-jj9o>ae=A)@zkNZ<$~axoIrjYI)u3~7J2>EL$H@(JU-6vH?|{= zKFB!RUVRUBJa!YliQ;CdFJ^ir7~|6Jwwkv}z|K3!#! zk9Yo5wk|iP2ad00-{HxQ&QWh25A4=CtvtSu(Y6-&VH0`p9AS9u?&jFic|O*e`8>yw zF`mL67QC81D=NLA%$v+v64{^Q?~D9RR{Kk1O*w482I8X^J%q-@XQek}f7ZD*4EqN1 z2(H$&TEpgyo-_D#>sZMO;We+<%%|Cdvsv4sBe}!(O=v+Cdj5g+tNgdrPK}#WJ9Xl~ z<*UrNu;)M5!#*XBM=-i2&n4{(vGu7;u}-atN^k2x#;s*v1AORSZ(?Js^ChJ( zmi@4$!J5}KcUPtar#byGwT3=7^yt&cjYrUHoDLUT+t^RRhErpc_i23NtTv-V!uRe4 z!7Oxg>{Ej)R%0dqwzBhrF9UljaLsq~dBGigBbOpeoc>YK3ftAb9P81mBU`>F$>dn51^yj(f*sfmqA zy>PXNeinW`4X)~(WE$`E>&HnhuD%Pd$`3OAu{r-VxS9s8D!*`b>mUbLFFn!u)0tjl zmvy{}{V{BhMlC-P*gccGkIMNDiIcv}4Bc#1vb>F~6${Vo?zXL=LcG*(UM5%bhP>=Rhf^;&pQ2{^hG zTn*!C3;K>6@HFJ~hq-U_E921XU+$k;I&|G}v_B;}r2AgO_g!iYv<3gzFG}yn+M`AN zg?#>=36t#qB-Gssou9?b$F6RARuRf) zwl?d`^pV2gI@fp5LoUoV@E5+dGkoUAm+PE4I?9=&?d;`lL^r*?lr;f!aAHeo2ei2@ z1Z?K+=CDms4&R4?r#RUZ?dQGe3~Qvyt6V+nQ^5$ApU8UDly~ioo*2s=PU!LiI_nl` zCx{;i=7aQLnW_R7`EOsGfi4^9RoQeAC3?qg`>>`TUV{KC^h2P0KqU zaL$i$JeWH+^wu*`p;pdJZGYbzdiSH~km#lV%(2cHkTat@L@(ZYKPGgNb*$F4@BX+? zX!~ceA^AP5;q1k$>=o2-SKSgq)oU{U5De{wlymMt{aV2O!w0Obr9WspoBam#&9^Sl zncajoaAvynlQS7}>5h}Y72)BHge&5=uPQ&}TkYfQYMyWb9kP$TZ9jSiXBO?wQHj?WAkVl#c#hJWc#q^;P|{sow|e!~go?sG`4eZtpJ_MHQVveShc4 z4{DsZ`+W_5&MQ%;fqu>^35Wc$S(RO&Q`doh&MPV2oPoQ#g!4M?I`(Q0YyCjc@`y8W zIbl3RzpOKHnVezB;0%MAE1Y4lvs{0lnme3f_%Yw%`si%j8ux5mO|P?YjnVC+igh+l z>wfh&=Mq?Fntd?!xy}HXI_-oAkT$;6&)-mxfx@jM z`%l9ieEOpa6YM=EO#Y`n73;aDCS3RIlh`IuukhKX?dZ64wu${nbG9jNrgOHb+GCAG z4%mDv=LH^SzvmV0|1hp`3(nnEMI5|&U>@g~syIWScKV0A=aYteJGMLgeRJ4u+@wdG zUo!3I97g2%rK)4tDn`)fR%o%WANCc<&&=Tl+!{Cbf#!hx4w$C}>p&4^*dnW>aE%txr8{6J-`bIZn zzOjuh;oIz4g2x)m{9ZP(+rbN9qld{1S`P%~S$WliBgZ%=Z|-300c)Q10DBR6bFgz| zU*G}N+mZj$X@|_SJy-9}wy{};9~x!mIkUh!&FuwqGc@XC@t!K*yJ;FMSW!Ldu?gX2p21;@t( zPCfWQ;ED;C1x`Inc-6N9r*;xfc|LIJ5aHCn1x^imFmT1RGfPg5Ae?@6$*ISd1+KXI z;gVBZ2@e$Rz3ojsso$v>dhy=d2JpnDf2ZPGgj0AfOkY{?0^uY2P7hWL$~>py&97zO zHvYjM-!>?7XvG2ECp`Ep?-y1KxGwv)s~+6U`|yh4yia-Xh1+Cjw-39wXzcLLe(*;X z)g{)hNuD#%NyfM3$PSA1C9=ii$-(CC64~4(S%>=Z?Vs48k*g#+2L|;yBv36bFVg<`Q*Pg^4?~1W86(^SwE~A~Z^`tJV zh|bKdm`&a5maV@{b;uuRy-*^XJnE269sK+&4!;=wd_Vv5?TD+I(?03cRKphvm`~Sv z(&O{_2Xy6-_G4#yJ=G!h`Gv2tHIU7L*0AL@3-?%8T(aY4@KkbmpyvKPvv|kG)$H58 z3XLk|TRbITLisS)Icwc}C|db>pAc`{{+aAuce6g8deECzP{7&4cZHjK93$5 zzxun_nE#%(UhrAx7S0#!zRGh(!+^NARt<>i+c4nGJ01yIfsQ5EpD#<5U0#y?rl8n;GU1K+zpDukTa==<8)+kQ_P zwt8(U@1@`0gMR2ty81!wZ@AsLcc%U|E35-*%vxL=^3mt1v5pOn_AHW+2dc=U@p9%* zQi1f|%E!5}Ll$Fl`MAKgho*#9_hWuv!5PCf!>z`Co|*06gf0&HkZ6=l!lj6e3!)3doB)X*wySGMaOQD8lX z9DXZzgg&1T=s1tC)q9@3CZ@b?B6t<%Pp6Zqe@c7az!%pMWPIIeue1M+F`V0j?)n?Y zqf@uG=-%;Jql|82bN-Ui!GfGXXtKwr{rauvj8EngvbF z^7Lz~B@``o6> zX-z-w%`EsiGRVfW;a|vwEi;fgCRiQYk#kKPwlLyl=t$buvG?JR(YvS3oJ|`jJz{Eg z3$ms^N$)Auk~{rrde5xR(R)##*`li5`47YE3$OdXPG6`W2nAqx8N6JUq;rN_ICzci_YETzD^YOZrXO+Zddc z{K^{EI-mQBa<-?x7Zo?R653L|sH*+fQ3Y`|$fR#atw=vAUQ1cQI|o=Qf8ZA8Gh^p- z<%W(6td1q(NyNn=J0vhxMlL{Z7{l1FjgT8g@!eF=EjNtdTXKVJ`5l`+nI+)Rb&nS-tH`&1PCE}2T4Le zvZ#QpkwiKPfdJabq9FJRh>l>!AmS(xF$qCH<0vgC3XTb&BYl+-6@<79o4Yd(h~muX zjA4@wQ6VBE9l(6QQ};GRM4frw@BO2mb8l7My0x6Gs!pB44(0l+Epmpnfwt`?-Y0SR zlRni$^S3Y;ZZ6RLCx>aZtuurDt)mD%gZ-+n?DWs^irka~ggu0vgjWcm#5VyC6Fwun zOV~_^ApR7vf$%xuL&ED7xpgxO{V96JOY%I=;Npgh+x*7~~gG1JHgUMzKtzSNTt#yV1lt2WMqjSKwMw-wv+%24zB z-sQPk#UnYb#hebR)cq-yi*iybAINE4#rrGr{!mWJ;PTv*k>$Cqdx&cw*lqrl@tma@ zo9Azx#QRv?|2F@t8_)ZA-M>SgCz)wmRei%ft<%UinQ*Jl{F=7u_U0{;ZzlO>>i$jg zJe%j)y8mUKTT97%51~x=`@m&QAIdi&&Cg4|#pGM8``5{H70*?=|5^EF8TlU}tO$HF zoNvOKpO$>pGtK@kt&uex66Y|aTB)qe}Cui3<6DRne%vve_ z7{)nLYUJ!pgAZGG-)FAc_t?9BlKbAoe|=x!8_=%Mzk}!7O4FN1_U0@xbUd;*{LbQ+ zHSpsP-~$iB2X2E8jDQbh&YiwxvhFOt2|0LM>21xKtVK&1$i!WNHMNGxns=Yf)pF83 zf4|hcTV!43_~s1qWFVh2l>Bfsx3z5xeJ9Wd8f)Y$rvI=) z)-lwYhdn-1`VM>1%y8@tkF0>!vQ$6p_Vo<$3NAaQY98MtY;*(XQSp&1vbP;=uOG{| zeM}jqWt;0IzDejI+s$wGb_G@|TeT=(wN_g{7TF8mjpkf-?Y!ArWIXFvh>Ua;H}&A$ z>?-QYyt{nf>}JkH_@AJzK6tq{*L+_t@9<9b&iF5tcj{0-z^udZX_`MD9xioAs<^TF z6ze>yKImpk0?!qbo6~s~x%m!bNt302}yMG4N~t^N_svv3cvW z@F|<7dF#NDec;@Fa6D$RP?s8>Uuyfa4zoGyT0lmQ-m+Xi$2bw0Q+@7ofUE}+f__u@UUhQM0oPtFkf*n6D9S=qpz zO<)f+kGKo%@5uRCr7z}Pe^=(Qg71N|v?;;b87{Exn`C$SKaT@1gqI*+ z1^-f_ism``VI!qvO@=3t@%%?DdFbB+o+t1u_>xQA1^y4bOZ64~?j&7kxxY$FG_E5} z*0g2Zqg5I{4pmx$F^V+Nv70(+8$Tzo4{VeJMJCEW1^s5uq_g)TG*V9)ZS$lNFZ!mG@m_nAH@3yUC6@>-{4)L> zQ}w!|xt@Kyuc7M)u_G(CM^|-Vu5=VxJ|g!|tG=NgMd#`QywB-pcdXFT&T=Gy?Y{0$gq^FRULY$(RFIvyy<|OL; z-{7W){Fa`oaP#-PgBP;CrEv2B-v3Ww z#|Vrs^j{+|y3p?yII7TpmB5j#B?}x@=r0mjQ0N~jaB!i2h`>RG{(ON03jHpDeG2_o z3Piun5m;R4ze3=xh5ms8rxp793!Gf&?<;Upq5pD$;|u-01dc8AUj}6TA^u*@f``f8 zGkS;2SA>6`hUfcwjLqm1rbV@uMS0WlaiDlAdqL^nwQdyo#hJyAg;o?T>cQ(ZJ&0f@ z*w(kelQmsmf3niyIW4rRYwJyUTDqmRtfRNJO!F^5b{!> zuWPJtBoBrZMmhg^lK<56-fH^CP_{AHxsUT@aYqNX6Peib*K<#Y*!KI%L`DXw=h9Gg zhb)onNe%lrdwaLh+1zo1-ckD^)Z6@-6dm$;F0Lh4@gQ}AHvr<0WF zhnKg&XMOac)ZNFw?Fwdkf)woG2?mN>|B62QUiyZ9HuDBsc{5brTT0L6J3_t%yzivn zy8#OWY4pbdDWfNRSWkF90lVRG7aMd4*qn+r*t0@3(HOUpvH`GB>AL*1u#ZqJYtG^Nn7$s=dXDIh^6fU(xgOx# z2bll8&$q*w|7DO*=3p{*zPDQYi7vA4t=7Ia#@WsoOFyPu?z_pF(_(%fA)~9rIcm!}F@sf|bGM3x4xJ-Hely9V* zu~u5Z9{S|?o$K70Vh<=jJ1*?6BhXewBh1NJV`9xbskuOv6=URCQ2sA%m3XPwK%n>- zl>Y`>X*pJ!)Jf|6uKG6CxKu&*O@LCyQcdxF^in?PDQ{V(IPMFq+| z_N;kq$e@1%YaZCajywNs?&+CVpFNj*9pAO?4BFjtYX{DIL~`DvBj-Kvu~OOz`+D&c zhQD{=k8Q?mul3I;+O!Qno9On(wB`LhD~NaTuK4S~fu3c&ZxcU+e6QV-=Qntly|k|C zTw;btSNIfdw9H&w&EnBfhCSfQNz)C(%uWuWq8OYtcE>Sm9f? z4&SoUk1rCxzku znz$ovn`GhJ0@6hXlrlrqx5_4AL#l6}k`WiFaubZ13dS0@D~QZdFvggnV6<^7@ItsD zHdIoNGo@n~^U;j`HQ>M~&i=UZmmB06mVuAQbaW}|put~|og4EAN*7?CJrnzttWOL5 z8UHAhGf}fR72J}d4R4Wh&y;S(&z$A~7k05a=;=ruL|?39FXBw;_I0w?mfXwqMVRc+ zX64EH5qTX~$9d|38GnuSNP0T^#p?a)c+U~yNBlL8^TrRXRL@r@c(5hgRs7d@mG?G$ z_z#ZJW{uFRSIS;~hP=1Kj)44T88KEF(JKGBa$>D=Vz52s8|tIVz#c)B!MnEoft6Bc z`r%_$cjC6OA0>6%2UPQkyxykmL2uWVEXTuRA|76UC;Q{Y53GM^OxQEtJ6-zld1B{V z7Nj?ZYfHztmX2r%FUMv=8__Z);hGlcXoL2a?peY*=!h8Cqa%W}O=aE8J@nf;x*q)i z9G;w~H7;S_V+Q`!!Y*=UXS~gM<@VSO!|NTXu58C#XQw6DLbUZ^uKHEKNYS$R9$wFw zs{VNUB9DwoioR@R+q(tSm_5^3oq-9o_k8)~&~&-yVWXY@itoV5_=sA^xCibeUBN#B zKKTuN@;`42$LFr;lfPJx!RKyR!~4j4%73)29Fe|t=n-dq@z3Bp*{jpA-8w(t`E?5T z3^r{|m~RQ+smt;lkh;h|2R2<{!R2G#6CdZP6SJ}5npHua9jg`O!Z?{`Pm>`U^5hohfK8&~)CM0Bgcwh{j`sk&zb zIwUf5+$DLg+K2EBdkK8ET90cGJBv%$$F0_58@40cE}@UBxo1Ii_)FT$b3}tK&r$N+ zv2iu~beF{Poa$=KyTV zH<^9$lW}WHGh^wu+56L#3>j;+-H|P43ba|@L&Gxe*Qj!0C?^m8xkK@^=E-lsM}O?I zxb2v7o@K*c%H7EtwAexg#^(y_AFY;u)-e7x92iI(zZWD4;V+?m#aQO9$Wu*?7l?tyt2oC^PS*9*QgQm9JBDx zM!(mu_|3~RY++Tl-osmtFP1)wG3@65ezniUkDK&QpzlO?sv~Z54|EI*r@!I(J#bL$ z-op5IxP^NTaE~!>tb;d2q`GRWz`Y3l#^y2$_afM%D6?=cf;|WLlES?R_7}>q6A;{s zp#EhR?nQvBWftzmV80+T+xNr7JwCAo7QD|Gs@hslTZLC4&&Pd6y_)vpQ-gZd=VSN4 zytApRuI!8|z<o`Vm@9wl6B#%w_~LcTomO*Qk~M!pp2mNDPuz;*{Y+n2hG=>tB9 zoRs#-JW^m?66btnJQ$Zy1>+6+i+#s)KHHlob5H55dHBhBN70)aCv=E(#)-4%XLX2V zbciMB5Duk76wOPw5L5>uBDuCi7FVA4xn6s8^Zo;xXKkff_X8GQ$-d%v+A8&z`o*jN#2F3r^+Cli#RgBEIrv!a z^bDpCJLxoKZv^EH zr@YBldGh~bj5TMe8jAXquVmk}(RqLMO6jXLgH4+AcQ@yVGDkkcxUW$-g6%Q56~uVb zuI1v}S!l3{I*C20_!jC$+-}k&J`gvAxWAkicQbMCT5(OZEs%F1@!PHV03QDiyndsy zWAl_q?$4xeHGM%&OBFa!QJy1oCu`~Qzdj0o6O6a0J7+*ehVF8t_E9=`8Rr?{E8`sd zhG3h*dZHTl1mimEn~i)UWxT9ljIlw%c;in3Y2OR#J=!pUQm+2wFn<$s_yE35KxQO} z%$S<7D!6?iez#Zyg|?(mgXotK!eQjuq|MwLj6cP|calFqU(vMtLi8m*4~4#1S5)-1 z5c#Rbo%$}|S$wnql23NIJ6e2g%H3unQ^o!`ux>V2wJXNBN5NR*cR;aG?-PN~GQtUb z7_~sN>DVRvITs{8)dD=FF1kGTLGm6+3-&tZC$93Pg?Q_44fW>1Z|XvW);svdmj=zE zCro(}KNoFqPH+eP#FR^gSw5y3Th7i}oiVNV`=e+yjA%bUhc5sX_tg9@~ z**}eSAG`UVm5TNfj4bN$pW@h?);CG2j}weU;1&E|)=1)1T#ON;AnUUVI?>fG?5{rH zr@pno&X|WH=N;%!Vdzld=ui>pQ0-VBv2&l7Ip*Tu?v$nPwpe=9>28ASF-D7mu|_jc z&Mu9^j-nsov-?Vaq8~}02lS&G!1bpD*JoWwKZ-Z{yIG?~#yjkmeiX>}BmF4exP*K) z$o-$>`;mSWZ*(Q!6!DKG^8Iv(XBqWPK_3v_ss&s6!ZX;d&M|T11Jc!cTO{_-Ddl6n zv<1!j;_@ZT6XFe#-N{;K#TVuMPrTz>(MajQTI)3S8$~u1gq%J-Q_giq8#h{bw}~=^ zM@hN+!I3TC^8QL$3p{r>^IFAACrgQ`|Z6RrMNK>-OgPsl@rV}UoDyNZKa?fyG<$|33cHMKJQt&q3_*&sr zyfL44aQ6IXc_U|nE#4S!L@KyU*@SdUWU{#K|bmVZn+*1Ac(;{!8_cUfn;$oi+$BVW}u)^I5p zV_d0Vw2`A=XXxrG=wu}HG6K384*e9NClB*NPikH2Khbezt}Z(5m*}uz$iuff4WGA~ z^W5C`&fGgLQ{iQM3omD4VtVg(f3ib`8M{_S$`OpjQ*Vg|GZ1ZB{tL}+ayl*qHeQC zsQaea?;7`8-qY1P_LM8Co%M`YeLHRAHo`i5Ybf1~IO-txNGZKq`gD5)Iuh{*sI%zW z5|<`%mcIRBnyGKMrQ=`Lx8saeyV>Uoc1pQVndJ20mzBIn5tb9y@gHxrt9ArBc^09*K&x%n zIDe1G(o>_f+TYVZmulO-`=%hM`C;O|Ee`e`$wJ=G)@o0cX|>-Jgf@SRENbW(=B-~# z+>2UmIOn4>`Q}JLc=NYu5#F!+v=cpf7hYA)ILaBXV(rXez2mGa@R4>pUEA>F$YG2{ zj4@8y&6tf*Fy6RE!9=4#@J~ZG==EdW;6#pzD_4Gq{d2Y zCT08*6c4*t#l;(at$(!D50X_H`!i?LSZA{E1m9vxhF(mXv`L2#pJHrVpjo9qEY{ZV zhxZ;x6aJfMJf!$fqHz)V@mV7Lcf*Br)4^wT(?sJ%>znr0|095=ZWC=dR9q7LY%F6x z1{@d-E?fgnjDnxB&teZ&{OtU-Humi{iVw>jU6t(|OS|Ih?A$t|=(djkaK9m(HG&cB z6F97OgTe5{{IG&~+LNw%b?_y4DR$Doyj~!@)ZDu{R|ohC`xT*{Cx51n(W(v6hKF?F z|L1?7a$cLnAI2~Fo@;_$S}b}wW$DO5F998S1OE}c{F8#Q#tRDKUr)h!V;xXz^`g<2 zSxeivm#~@Ouq&I?Sy|XeImVjLABs(TsQFwh&*RPK2zef5K0lAG`y0prk-1k|xcwOa z5Zo62_N3w&g2(8+?M>WXcs5RWovC}wJR28BoKxu@x1Eg>8t7~Y&%4EnvuO)iXTm=$ zxa|lnuhn)AIbDw}Xe#(HI338x2`=m;M#GGeV`s8AS@l8bcL~F@`7@ZR9J6@0p+RlM?XV z_iH|)1t-2jRtpamoHz{+ZEIUm%ba@)JTt@ke-8y?jqVD@8EFc}8>tE=7|9Ak|3H~v zEyEt;F2c2h0fYpCADf}+7S0r-3r)9hCPJP^TDXBuG~L3Jk@9TeQUhz!rj925W;-h! zI7na0ea_##h>aQRck7tP{t>(iW1NxiYHh?kOPWOfIO8DgIYb{H2Cr^-xpf8eAJ)jU z7^$y}OGB!RPl8deAo&!GGY%;T-YLlWR0X4rFBO!z%!SWHFb0Z8wHAaxpV9EH7%f1Z+-39?^rd`1Ku>bl@J0u1dylDH_7=8Dcg0X zbrbXYkEJX&1UH_*mV7T^GXeRyYe~<1e-Y1$_Erx$m!P{|JR+ z?FX6i`DXe?cvuR3xPZ0@PW=WMx8{Itw>;Y;OkOscINKw6e4D6bbgq(vPUA!P1M_gT zz9@Wf2>sNbG&PR+q}XYt%RZs>VIffXnUo`PU3gul!q~Gw_fsjGJ?-KAN4b`am3>Y;njtw$spcY}++q)Sk*aq^F+$S2>+GZ!iG zU*@t3#u+CRj5Urc7-KX7uNFOX=P&yJ5*~TVlAEV3xheZK0lB%~DzDBe|KAG68DA(E zZ~PO;+VGe?$jXg`#|ifk(k&j?fJ{uccwm)0Tf8nyo-MigA+j>v;)lgN3k}Y~wpi%% zJ<1mzmO`297;_)GneYL@o%a|k^O^CmWaulz4OD$3c|?wOR`~aavw201CMmpo;%ppp z6h3k@-z~G^0^>Q1@k~dKrXWWZ-JIE;D{@pl>pOC_A^S>%)_l;~*>Mgn&uwN7FSv4` zKx8TW6T11iF_!bdF64mZ6PY9Rj|2atUuBH%v-}+c=v;hgV6O71>=lG3S#r8V2rTG{n9*Gt9^wvXpF`; z{*EvQyX%!f*h!16MXx@dx7+=&g8A#L_F|)$jvX0iVaiy4WsW`$2i_Y}h86g0&@0x% zm*jcyFuOlo{Uh2q1{8i62^1Npv4$5?R<_5KY39C%>}$)};eLv*IgM37>8tCI4*_WQ zsmvKKMecDy_^L0 za-KN5ms4aEK&@O}Srk$nk9CG`zLhZCJ&>MU{Q z8lSnZBf7a6C-U0d*LjJ!Q^?R1*~eiWT-Idj>=ilZ)?>KqRrN`NQOkd=eVnBiT90YF zS2)u8#>+SI&*y+rC*fI%s%}nWjY^9(Y7~qyo>nm0cuK+lU|!0;PJLPkzVd&5Z%5`p zvTl>lp2c+R)EvNOXu7Elj(!hKi!LU6KAGr4b?vmiZ!j0yS76>PtVw-B3;*joL=^5W zzru4l-)pSH$k|wj_(q2gg&*!iKYEL_a?!b>u}_k-@~oq#==g(UT_@v2BtA^E#tePs z%5w4fo{hiZ9_;NOwtF3}@-e;x*aLluo!e*XOw7{MoI##?J3cTj(v~m5Z}QZhtQ&Kl zS?DDOdf5nl$lA;>)xR-p&vO2vgNai`#Ic{VQSsOl1>#fSB^~|30Y0%dwIO4BoM&*D z#u-{ds_uW^P8|2ogmcfK+~e4eI}gk9@Im$fVK*UB_ZRD7o`<5e#yr;B!DEly51gE* zH&(E|F6By`Dtlj?C!PH1lr3X~9S}I^Fr3t>8ruVj!yiIR9dPgf%&!QMCsrPXX&!*ve?rS z-jkB+*+GAb{v&u5tFTrJz+J&G6Y8|VoeEq6GYXN-j>sea5$ll}B;N3fjp z7u)P>=dpi)-+SgMqx00g1HSQ^*M{%A{FF;QLZ8B?4)H(1+ZM*nRW+0~@MvQc^dxEj zCQWp@8ti`K@xgeKJbU*K^n6Ai$=L29UE1*8O_|vQ4}OH_d|8s2{pr);wV#a%_HVnc z(ErJG!Tz1(ll*T1TYx3tf~wclJpXFqq%Zbw)jVtQ8@jMv`Ix`)|NU!)&%_#Q?d4;h zi7X%U0ddlP?f0A$fQP7d)LiO0SSfqJoR5Itiay!QT8i)hFXi@U?M9y0@~pm;!w-z|(tnT9+xF*dx zQ25Zgv_2WnL|&c&y=>4v(L5(RoAgqSZ!4jL&KX1eIp#Y4Qisq=SRv;ZSsw~6gr9^r z$o&s_aop<$T}&C9=WqTTpQn`TTMMlows~7yb+4k!wEt)=i(L;_qD{8o{M{ixWf{+ z%!-3P7xv|DLy22^F76rP{v~m*oQr#lxW7x>&U0}GiDM&mxr~MIo*Jv4x6(GTZ3xi( z*2;wDXAbDQ&3o_SjFZrW(C%xM6WEt@8tciQh^{5i&+&Yg-V302_^tRo{|DcQTx=;2 zT5}pT`HjQ43Z-buz1%9ru>^%#{=Z)2Q82tC{x_2Iw{@_cnhJviVK9O!WsdGuOe z2%*PF@&Lhw(-tnA0vBdcp3H;q1P%iaQzC=fvXGhk4fXYOzb3mB+IqI1jsb$EbYoNK!DyNK{a45q{pr=Z7TWX}>Ppg{Rb|1)t?pf1!RttMumvXYX@z zy{4^Pk2)vU;nQZ{ucq(0w?M{5)32RZ!~T9irYpV`km)s-dA5VQ(%;AUm+;ZHGJT@e zkM*Pp|4KooKMv1SGJQ=i&u8JnqhgHDNtbfpyH3ma%o*a})@7*wlP)2C7QgAs7-J7m zWcZ9Wd}sr4N`}9J3>SWIW%-zY^S>X-aN>mi6kWEJ;VC^m0U557F1k-ZhHs(V73hf~ z7hmRCefuLB&i$A0)V4DGdGaNmC&N!suC%pfQelRY-)nn$?&Tc6;6^q1?=IFd?oA5u zPe~r?pPL-wUqPItJpl~J>&JL*E3e~_DN?WB@_*^`Wu!|z9tL6%!1qelZb{FO2mkVoY8d|*i1GKK`JGA<%s_|{C7&y?4{RdMmgZ3-qB(-n+0exqQF zakGNa#!U+Tvb-K9xLx_nd$QuYP}V3;Sayp6AM<|ZGg9~0(Z9;<=(M8GJ`v{Wz<*uH zW9CIadq+ZG?{AyfK>vlisQAW5S@O+v{C_DPls7Be!JXeSSE_?2X|CR$JamB6jqN?s z=YjlUPyGe=Uaet0Y5{OM;dl>hScH#p9&9-Nx5BxPB|X(xrB$@^I3BnGF(^ zN;#i*i9F+@Ql5Lsb6jz%JolC7*y8QbzGEQIsr+A_b0l7$B|5!B%gdJgOyHB_96@>6 z^*tmlthkQ5Q0uVS{sJ4j@@44j=(Dn?5iYs~eWv25r=7WfFmW~bIy@UkK8-l+RX#q> zUhyl;1!X=6KiMGP$~=KF6y17Peoar$dgw;*@(ZWQd*zIS>|sU{FTD1z#LKro`YxrH zr`r1NBkQ~4%8p9TZg)1+{1*6U(&TP58+6o%GG_0;%p+}j%AV=z3ckoZOx8d&eehlf z=RcN2I;yl=hWRJRSj%_8^6k&(bCd5l^_4RIDD~i*a&-1(w2zyBmmdc|Bu$>nS@T}T zTEvq)OPttWiBJ3o(sb|2TZ1y5m>it(`1p{FBEjirhny)-*3K-q>&R^UsDM5Gy9ypaDhMX(N9XZ zO*|(rD!>^Pzul76;FQP^Q&#Ji%8}uuJFqiyIJSMajyd+L$dH<@ ztP6Av@ed|VWQnv9W#_x$2`SGR7XUdN+R2+ML3MLqTS1{Jtt6+@r ziGtC_#|r+k{MelMpUn5n^=%jw^1{vo+ooW1TO?U|zTBd**41 z9u{2US@&?B-#1uOyk;TaZ2S}RKWQiPfj5{BYz@)ozRn!rHRb@dPQ7+Z7rpk?uIT=Q z@t5&3H1q0E|0}PC`0wOf;Z3ux|9;Q^{?>+WMiDRkA}|+R*u8wreCpbEF8B;_b0n_K zT<|U8W=mX~x!^(Kgnru21%oawA2UPZ+RO#tBJO61Ycm%NN-rNXS>oEv1-lV9QR3Rn z1&0uKEpdS{k$v4%tKVI;Mdp41`aXZ|cMte3v=W&6T}FOCHeND67X0qP`&s%fI5+>h zp6}Yu|5C^|iTU5T@ipgwGpW~SKbiw}$oQTy+{ir8%RF!$b3mE@d6@@3pEWzC zuHWM88Ichf_sxvE%t1o=#~X~f%=^5I`8vj&J@xGG8DE+Id6@^UlQ|&sKQB124jhpA zpBFq>2Oey;aNu=tpfN#kAj$X+C^i%_{~JpAF6MvI21kWC|9fBNea!#FC-zjC=P?J& zu;zgEhXZrKq~Z#h_o+Fcvp7}eeTg#fODx{b98ZjF7v)!fqCBy;>F&^TEkLh#G((#7(Q;5@g{kM*0%uj zD1$kmS%#YT#TpZcm-R3Mc!Iu>{&`NtB^u8v$oijxPGhx#aYnU*(Z(MXj4@UzD1I@1 ze%@D}pv4>=^y=;3^wIGbYr|*!uiTppjf-#YHJMsp*#kI%&ip+*Pwp2GU0L?2PN{u9 z{Eyr_f6oZ&AiQK-yv%oGz23YFz})+ZQF}jMG2SnJcxFXSoYuF7HT>J`F%drO+UxBx znUjcXpzUAVlQMtLy#i&?+N@(es-4xg_F3Pu$G3IB0- zXh&OAW+Q#QElKOEGY)IGQvextY)bRaT!B+h@64UnN%K5oi;GyO#o*Tr@j{=zsJ_nTa4S(F8>v=A8R4jVw z%fy|qM_~h}#iRylo=vvks=@e^s7ukt^}jcKRsWEqBbvguIq~<{ zADfAfCsEdM>QPpq&AJKQh<&1u^NDMQhx_gAjlSIJ5{qxNxGo#qD>^^pZeDcA{SU%Q z!s~?7i@tLIi|{mIE5Wzuu={<&gM{Y^zwQ5h$?g45lsuI9d-v`APnMMQZz@@z_^|t- z#H-z<{ZEz5>)%o`F=>^1ZhwEtoc_%v1CyR`Pe_`;dn;)_lw6{ zNfUW*;QdI+yrh+sQD1Uze_u&y(o^n-R4D&b^7SoRCndMKuAlEAgSmJsw-}_g{Csb)2-hneZp{xqj@=ZR}Fx-qdBa z`|ny})swcwh`VXaJ+x`BbA@|v=F|??b-ug973^#7z3qzA%NHG>`~&XnXe}xsT6<|V zdoIoPNTZdqHSX#hf>9g4R&xzS{#P1Pf7FEr;N*Nour!O9ST0;LffRF7M zi`&7wKY@22=`kZmfOkj1Gj!vAUC1N;;EUPcspfo|i!;hg=O&MxHSvxub>PZA?iJr} z@1SsH?e%wTnXK814^&h;)4FMM^0~`G3j$ZD-()RH;fftxX`3eXkvnnP{v(Bbl>du= z$ozIH?Oa1U^P#T-+IbM#3Z_ow&{+la@)hkqY|#pJ_Z-&jRmspwd8jrn7Fy{S5?d7q zt=u#5?k&x~Srj zY6UpZd|%m~n%f4R#xHLa_LDDFkiW^olN50Aaa(-E3O)YAXD9Zbvlm{}6rwe*CeIGW zV*vOe`p=;_li&ZG9kJZE*pQVEpU%7e=%5dNEBYPy7BwnLi@GLOiyED%MdfMw$nHVf=!+fN=pOB~ z(S15AD^Kadtg8vc?}@QL|^m1eu#s*8iP1bn+Qc3Z5puB0_l&-ixQ+`lje zQf^ml`9{)5SJ6l9JDP3(mUc^<*Xv@&h}&nj zZwhU%4#er&T)Dd?Yu5ZnUG0P1743rDnc^EJit}gm@g>Nl1&?jd1@5Xmu`_oe^bs7a zjry$?9T@xqr&F7Nwt zN$!0F^y5<66cwb83bAnLBR#T8(<1So#Qh@pr@oo-ew2Qh zTyDxOIlr7v-b3)buk7uNWyq@UJ3r@+UAWO5x9}DBibbp4a|ll^TH~HaShZ-a`(DD{ z{x6qoB7Ew+)*UzC<&q|z_d3VB$MZZ;-klTNPXOl<5|XY*7W8uOB{UHdl6$!~O&Gj> z^OOZI%wF^oa^)rWJDfk8D*eOynBb7qy>wclczXc6#=rKf0gosJ)a5jvRt-Y^59xf3z3df+N1vf{J^r$7b+F$k^jk8jTf z?y{dehd;;Y_gPxIh>JS}x%YuX`~Md>)H~=L4&|qqIJD0mPMMMPM|;MogRzzN&w0n8d1%*Ykx%GDjXHfG@<;f`dT34LoDR)?KPn$Smd8Hp&UM1qT6fxa z9~~47uM2_ag`$%>uulnNZW~UX2=caLZ?-))2OYfMA`_--wulV)Ur#+KB8)zJo4ZVo z*gIr4z_V^gXS@o!s{^N3GhSuTy6BHz>g}_RTKZ!n`s3~3{Y-HCD0}cTEgdq7K7Lo% zM=qn!XWKfA{2m#0fjowAvv3JlJzv7IndPYC|3Y*(F zbjV$#$Du>+7Kje{u|V|3J;3GAhvXkXnX%BG%oF#)$Aw-p=(7ZSRMlaHANVV~TpPDi z3ya{qgVOQ3&}Jw3_JRXVj6dh*(JR{-6S(s&ap7jSbK$G*q`|M1bRJw=lDN3G#JTvj zk}LYYR+8Pfw&cm=Z1=q6E8L=oJ&Asm9a@pov}luiFQGKe9OoY>tC~ILZOGSu;Md`l z)@9_qwk{Dj>Rmo8vj-P1=reKN{$-CjZN=GhvLiGPy6fntkjei>Cd-&-bMJRzP-2zH z>4_2VC|wo%TcxXBhfY>z(Y&nRimrMJTsjR--2=ay0KfA$;U{5dg(jppI;>|;@MyFE_&(V#KrD+5{9|oMGyTK zI_Psri``$KhklOUx(c23aKa?_=jfqdpo7Z$7wDn?LT`O4=@0JGq#JnumiKQ;-at3~ zSO2d|{*B)HX42E{r;`51y{OA?+*b}8RJFLvt?pG_O5IO&x!3(pvbO&7+4}nI#mC(V zgr-Hu-KFTM6M25>tezU7bV+INDZ+mC8P>pC7SS)C(e}U72dlU{_zS&rgtX^fX%~0n zy^}t*!=lc=>F_kV>cz!Zo^C>SPgs1&{S>-tsimu)QaT9be~V9HJt?Bh?rhlK=P_s3 zNjWmV@W>dIai%wqF}f9g^C0-u0Ns2I-7JM}geHV;eohuBdSQK8=tbyizoM&-S&h(D z@_kdj>lIS%^ianA(A6Q`Izm?hJ3&{A zkRwZ>t5E1ecw7*4b&&I59igSDKpgaSP;A(t%Z|`iH1Qpwtr&sOme5cv&yPc2*_68i zxs-q$lKK00JN5Me`kHNvsVe23X1S9qC0-l%l+YLc2&XfL>IhyA1}~q4p1!p7MjLUN z#7VuJ(A}V>ow>)*taUDSR#rRbmG+v`5kKU@)A1{&aHkja-8raZ z)irupM3?Y)oQu)3rO$Tg9V2d}kN(ky#--ne5GQk)W9a`AyS(g9VlH~R$+iB^&~Y`P zh%kd-Kqpm%s|dFf{FiY-`u>-OjxAf_pVzw~pISyQS?d?AyG?_zYpaVP)qH z?p2*%b{}JWcIdW6!^ve-N`S1CSU+L7&1weZfAK^(@vP6pY1(k9j71 zEbE(n336Vh#qP{(DYkq4_QXton&!`d_QjW!oCkY==lTP|-mtR7RrSw>VAB-pt*czZ zIWXeahME3g)`ol8yTJZwnE#X5bN+^Y*2cm0~p@Rim#$N}~3N}yS*E{p;f z+>G(n;KEgBeJf)ViaursCxXC@VDzz{w--LfI$)J%kMN@R`Iw6yL6$t@)cT4p`;9#) z^F`!}8+|MVxl(P(jAUD+k{PY6<@H2nY(!?fgv@9}w!DPQ$k!q>Z$VzLUhv^@_UG3&bF6m>nyTIKaK$#XrDvet;kR zfc*af9b=RGsl}Vz)6mgVnPVU2UtglTx!~gk_Q;WgnUn5My|LxQvSrTSmG+!-ly)(< z9X(l(RQ^mKnI8c!kF5F|y1@@&?>KLQmrL7ZU2!k}tG0AC*_#(#?SQ?baTK~9kFGM1 zwMBn}3tjCny4qoMwZrIYhtbuJp{pH3S38ETb_`vu5nZhjU9Ayatx@S}Z=>5BO1xj` zYVV+{iQe{X!cg}+mag{4q(wi{)!zAE=xT-V@5hFvRSiXFn}n`*1G?I?PHp|`cm7yc zn+6{jUF~Rv$;Vl9ZrFCTb)4AKNt;g~UpK+OZ-IYrrtMqkgQKit*=_a+Y0o?8YR@Ln z4*2)|op0&zJNWng#r@FL4lq^++|%ISN4e)$`1eud!C)yL`Cw;$*Q$4EIMHXB^RY>3 zfia5O${2~R_CN6NWc=nkShEOc94=w}E@l1ZGS+W;vVN1s`b`w;H^tCQ3hOufp&8MG zWGzGH{k?4wtlxw;OkwY^n--4F6I5K-tJ>LpP4b*%{I=FnpZoRj223)5Zm4n;3; zcHco-IO{K@J!_?vr{~Q(R7u)?(mJsI0=~U{HZ9*wvuSgaJF))49Qx4Nv;s4YJ;LNT zd}lKUyoI$6`CobZ5Hl^DG^aMJoH=*!*|frW2l-#O4%*z0=mVM8DLz75@>n+vqc70O zcI83CzzzAVrO121YW2Qr$g^gAA@nHeL!rlT_`)z~|2Fzctx-Vtm!bQ841W^eINvb$ z+0*%t?v0(Fb$?2}$61F6%uBKv^VhCCS@QaoO=>Os^(&8;ym96CYVG=lMO#Y#XHjj* zMLBPlbk2FBq%=9lJux}gJ&$>olQryl(C=R8V>w|qp^1>N_$BE1CACH){()uQr>XIA zG)$$B#5S@$ZIp3!&$_77Wm<=k!n5|#Ug0zQX|KE=puGYQ(q4%_M0?StN6GVH+8jxn zg$G8_R#|`UrL~Xf#2P^(?cGm%kF(Aq?Nxg7|A+Sev-3Rn6(Q2zW3=}e?L9_&kDYC= ztS3l&8{t**|G|1xaem4*EeDo8;G#*c0j*GRDHx6!d6*B07ri5 zOhZFfA+l=-`b@#u^9{)zU5(iKjbp8PjI24d^;^O^^0BtQT4n!AY<0xMe?nN zoey`oEobi^n;^??8Sy&h^o-)X1@&s{C#BUccN;_?pW7j)+as?#u>KK={~bGLF3z2S zUd^|1w)w;$t+qd9$UUx2_E^@0@pZJ*-t4{azC8~Tf8@C!Z~b$@-V?=I?a6zvBik9$ zeB`-M;vC!q74B^vrTPDkz7cy|_AO(KD#{gm?fcLflC^kz;;dL1mcki5J*d%vPjflX zkr-RO(q~Il`cr(^Ka~EInPuuv6?`xDFE+}%KL-8jfO!tW&KU?fZ*>Q8Qoh(Ch-~Rb z9VeX^H-xzBR2=qylqWtjw=uVqGgp3eg&&`}iZwoz{-w+t$Se0|pg(Q6kp8rlGjLPD z1Z?31ZWk(@neu34e#m1tz~ z??uqRkNKw9rAawbo@L*(G(Y8XPZND3`_mVbS8!I^t10`AXk#hkCTW~yQMMo3_#gJL zA3K7!eODy9C;i@uw4y$}J(}+FgXjJT-X*>r&<{ixF#nG|Q+-EsJomS;jg9C3ZS9$! z!8W!NajG8YY-5+^*Ia(qp6LqeCH5WS(^K}l8!7iy>`vslfoJvYkL;PSZ{{1(#l`=d z}4=qD<=+DU$Xj=kp`-FFi{}jlcuxgjsFD24; zDf0#JK>BMB>GH3Sfa6tvaF-DDo%%uwwnvFZ3GuSWDtrAhHf_sTWtFjmJYv7}CUEGF z%1BjZ{15SB8}K4f*6x(fuj26eqaZ$i6pS_gs9=oojDq<8P*BeK{nY1)_&ZtB(X@R% zI!K2H+R)$a=Gj)s7vq2MTIw@SnCg>CbQ-Rp6Sp9sAQeio$_X9sv<7`CJ- z*e=SLCBh4{@yjT7rn=bg1^C!48LJW!U4EIfIR3ynJj%$}eVHfoQddSsnBaJPgDdsg zc~kUp^HTJ&^U71teH+HNUHNtm->UEQOiy}y(?&9qzH9prjeN<8ntv*N&^A8#51M~+ zT2{8=!`-+$n)_sReB#QP)8{!~&VFOxw)v%;Ig}mU8K2YkXn18zW*Pd7gSC`!YcHq) zIr%mGLFrI>SQYaDwI4J^_Ji05DnoB_p_^6f+N|64YUkDL1C^mK)u1m`hY{DCIQD@W z(OYECaX0vqc`55=_p!!=FKM;+f_yGVR|Ax9; z#`D|lcjO9uC(4BH#+q7j)-7!CGRY%r$n$Q^%`Zlp{+@$vP6IeS+NKWgJVAIY_=8R$;w#E}kd-DAB z5YpLqs>8l`3u*s2KRulE-lWst&7>uwH;diC+44J(&V3G0Amq_WGytq(mh|ag-3|K`7Qf2GB>@GwcoMKDFQYduP!>szSlwb1j1axYpnUMBHTcj zPxv$T8r6g%!VJQQ{*5KYgb|5V?keC5@}BsRdqm<$clLnBk^~9ZV>A)JmvGYgxH~TC zI`?McpCH7uJ~jt?PJh8hk%7{Lp)n*`fInSHjmR5zY2!Fga z(&XP4+Apt98v>szfZye#^X1(xcKAah<^D8$3bL*+jlP9$mTqI5reX`T51(T4zJu|a zMt^s(_BOB4BR*`LG4D~f%d6PS+|aFNd#=Fl&)|26alM#%pU$}U5XiXp63Dpr2FkiZ z1@-U4v+RGezY`e>&C5E1?0xpOnfssBwB1G9*Vx)c$hyEm#{OEqxsH0oQ@1VX;ZLH2 zU$OWrcM{IlML(5t^8e1u)gQNl-wkGLZToG3{p>`Sb0 zZ%@o){rLpz&ji`m08S)`?afN&CWHE&DCta){g2bc|0|G>wdyyB{{unxKjs5(Bn0+9 zQafMH{N{3ZDd8!?)airQm)^19g#_mJP0aK660&(-$Xd0G{Q=r}HFM_!%zF>AhHy8y z-eljFyPY~W6H*hNaHleFPVL;c!&KI$W$$D!VPQwDaUVSOfJLw6oEs?9Yz^JmM-7^j zHixwg4{IEXUY~_tx&MiE%l7DK7rqzl*VZln+QHO;K2$K)cpu2#)tF7JS+0!G`pzXx zAe==1^rq!|pU=wko?;#I)XX5Yj)~9yOfHD`3STJC+n!sORG!P8rgy}A=c*$!@iku@ z=B>*r&#lK_rGq%%G<61IY5C+~{fm_PQ@YFo%67r+P6|1X3OC#>;NYuXk*93$>O!H3^i z_|Vk;KgWj$tZxqUjo{%SU;rQXtGF1WPQhs7-wOUG_@FaSv!Mgq(Sd`|fw>E&l(|}^ z_^V*vl#}UsjIxkPOC76!tIkr%8S8DxeRDm)m5tukw=Od3au4}xxGzv_>RBiFSZsgm zFV>c3pu37rh24-x{5*)?2G#+T-XK0aME*_ILmI_?D2w|{2EGo_L2byKdj2W0D0NCM@zKU`-d)@yD0(^b_6-Tsx_TTvTwWi3 z7q?5kyLx*H$nzw=2x~((@q=hwS!~aJ7m{A_+T89oTJYw0-yc?j-_+v&4xbm$8O4jydzE4s8=_d@(7M zWh<{3BkL$vKszpQb3b#_{FFZU>DP>F75&B;iF|tnJU}P!qwUPGmHk;e?{^XJh>Wk& z^78y~dAi?0-WdEvwkJL2p_-5_Eg|+ zR`78J`D9F_PyedQj4?h`Fxq%u!8q`42xC~lSmraPdEo5Z%3diYOxxI6pzprWIE!yW znMXH4D@~mJlf5L)Dt6LY7fhqv-$CmstPLLLzw2z}+3BsVn|tR?>9Rri8n)e~37jc8X_7lk=ZKH|O^e z{i4>z0yOnYd16Dg-7;43uJVruXgj|9o_V zF8H z5neWVlQv7*Ao8#ee7S8|FSjXc8{bHNje1HM>(FJx&rI2Ko1c3EkWJz{-wO|M2)>a| z-q-QYnb(GCK(XtSb$#`JWI;OrPwvhh2yKw{z7&D9uY$bCv3r($)0iK~|9-A-&%gUn z((V_2(`D&f3e|DkJq1XL$J{+7W)6i#SQ2Jdkw!N1N$8NPpN& zCr;AKCXkLlOf!9_^MgO(kDB>EB3{acK<2422j4cZO^p4H? zv|4N%b$H2zb&*zk_-?t&UVP}C#=b(%*2tZL;#)?}+SG-1@Ya^PYG0e_s@*s~pS8|V z1%!!x`#nf0=7p0|6y zeI(R-Y;cHo-z4s(;7))B&bxp8u8sMU=3fmStedZoRXFyCk+p{I7&(}A$S~I4>e#XXat^@~*O{Z{@ph_-3Gt)i89S@h<<~BvV%&Epao6 zyT|3VcKQEe#SN3VTZtRx^1o}vV?=yASVIVUM_n#(_3;AGMT7BFVqCn$9T-+9;k_=j08@luYIC+8@3!sF9f zSCaY$@@YYg5)V`npK&R^nfB?~=4l@|Bang?6;fYlkMK zos!1+_#dSSzxAOX{zxZu1bI^NTOcy{)o|;6G1~ey&vmr@h(P)Ro1xNS;kP?h)q}6~gBibBj#bicU-xwRr_FD-%2yfexdE7brlS|$?;phTGduO& z(w=?Dk{gpv+70OE1Fj=Zx;8@Y#5fA z!FtKka(sy&rH|@v*UmIt3ZE;!EaPkN{OkFezwtx8_SoO_+9TQcu<8Sz)4%94)ZI2U6gj_$ay&Oq;ZELb#TSPs4Jvw>QS=$MdJIi|DcfNpfhH)MiKhnODlr@So zv2xB8KK4zTjdQX#@ng6FuO7AaS+?3E;M9?x)c1A1*-Bk^((ZTl+Hc14KA!hUyie9^ z>-N|^!HFi$bY~2d+!C4|Mcsfxw~s^H*y1XACF8Rm{uqp`U9;1+`#@T#N5*zvB=j{6 z`gx*t1#^eK#~6Oc=s zd7A0+-?*}MMM^OLSoGt6{Pw^6hc8s~i~kICF7Aa8zcRF=r%5A$e--erZRp8ASKnyr z3;mjX1sq8oL?;m0E_$ih*vs7BHxgWyz5wQvNBTkZm2?^F0!{fe@quT)9y;>aq!&5U zEORUIy?2m$9x8^HK{JOZ)6W;JbGN^KNjqi!&~`j!%udLd9k84I2(1bq zQNCKK_hHID_GIe{;gxB~26VE%Qb$!D@CZb~Yv`$o-2zeDB2ns@_!@ru420`8gF`_7w9v+0%Wa(p6gveby0|ul|Am z4+Ue5D}g6x%lO9`gLrP5DbK8}sPqJ*KTzhJa#yhEWj@-vDF*+&gxQ3aqwT!uF}Nasj8=;)l|^RT77Fp#9A<~As$O$up467s7hNJ<-$;v_db z(qBq*b5lstK-+v8hrCS4i$cHxQvbed&CK4jUuT~q=l=22$J=N2o|!dk*37I~vu4ej zvy7Ix-MAzAw(~E@KsI|CcMs$4hA+KYb>A_dR4BdU&w)(8q3j`2HjR z{r>OP#-_R2z*_Ye@_P+_Nd_XBkLUAk#{2`&S-c?Q z!6@!Mg87_eq6eo#oh|T(LNXX=0lXglTbcI9w1z<)my_Iu@j!70 zW!z%Ky}0;7zW?&YD?$GYjQQo*eCZ?;Jpb~S zMoyf00PQ-rzGL=tTiR#;w!Llk$CtIv@4-50_VmUE^rw5wT*vQkf&8 z=z`Yy`1_3c&s=6U`&0Q*1i9P`}f4^W)Bq_8h!*m z`KhtX8{m(+`O|lv^Vs9<%OATfcFNHoLvDBxa>K8$Tt53bz?nQ1a3J$!|KCNkd!B?H z{%7E)@o#Dl{Yc6^FF@`&2KnQ*Cr>maXKsJoPZUV!X#4D!p7D{E#hdinIDhtc23 zKLqBF}M-#0v?;@P>djjHoa?f?BJNlZEdw#xH&{)VlxbF!#{B(rm zo)vQ>^E{IkGSBOfd9G734|q`y?M&-mmU-~|BKRBe1eSS1+F=FDJS*nn$cuJonTLMg z0K6-g7+31``d7fCK0^3Y+d}U&Pkol@2$@Z_-3wob{6p<`F3}Ob>LudVA?-j4dcc^S^iplvDL#WW^>pGm)ne;fZ^Fn1ySVqC1i?*}07TvQJoBY5OR zpTOCB%pDi46E<52-$Pm#8ZR8~UOGelp2I`6;Ng#BzrvVV{XYdCgb@SwQl32cUrU6{ z`&;~yZR%TDtg%AuQ%QIAGsx$!LU(l(x+}6VBVP{K7-BzDU(L2Pys`v)ZLm%KFz$jh z<$dkEhWiZs7{Bj=`$?Dc)@`RW`~qix>goL<*s6Y}c9vwppCB)d+ZXs3{=fM5?70J` z;2ZRv`VZDv&n#-#Hvdz!kJ$TOZr=g*48Cn(8k2t-@{zq_j%-kAey922X@;qM*PJ&; zd*EN-INVoi$Kf7?lf>OT4|CkIxi2H`t=}w~#eF9850I@Y&Z7Jde9PG*Up}{z<{R{> zzen1=;9-Pk&>lJNe|_l|n)i{mL+1IkEc*feh0KrNAp6v}ve!E{s$VyreYLL!dr^z9 z-)YRy+H+aFV}99U+|5e;q5G1V8vuuN4n*He5a#u1%_TF?@y$bbdym@pgCD;6273S9 z6*IICbu0Aq7Hr@eYX0m#x%YU_O5?JbOBt^=?4QK2&u7RpWwZxdgS`~^;NQI%ao8Js z5B6cPXQ10Or;(TNxD~&Q-_`48)=(Jo8m$YyfZu!ZP@3hS3(eO_#8aEU1HXSkYbV5= zF5~_P-$+h_Z$wT*eWxGaez?MTIYn{=>O(pzqXzmd8eT88zpM&rHd$G4)_Kuumch){jKD4OczXHCk|CJ78t*;fUo}9nYix-4}Hs^J*fYG7IB2zefa$tcnZah$+*9kafiXH zXf1gO>hy8R%l;H`l(s{r#l4XX=PnJ-H8KviY#gV<`H+P3Uc`Z3SAt(XgB{qJHoV^g z{z9;?k@TZ9M;1e-z&N@EPc80wuUXoGJ1}Xy@VZOD2frIf+I=;~M#`_~pB80sTDteZ z;D?mTmhEd?Q+Y1oaYSpF7u$Z`e+Qk&8~)Q;Yqcr3nfz!G54WT_|36H_!Z41YS)G zz-#>7=bc5eB+kVEAG&MBjd!_wvX=hjLj%7B&CWOtbI+pdzJKiW)@P61dEv8UXXfPx zCdSfC0e)GOh;(Yp@kSoqv`p@@0j&K|D5!|5u@S}H;{Bgk?aGL>cW%~u)*LW1X zY##0W!unHZf4a7A_7@*HZT2V7xg5L_>x41NDm)iT7Q2Cq0i*Lw(Me-%0Od96ofo{e&5^hxztm;g=S6`PfH4=cPYy{`32e z+}1jeJH}o*a>rfwy|uW0j&ww?;0)NS2y+{~2ewYr?|!WPwPRP_26`{Ty!7~@#V>sZ zHmbi`-#h!^_btBvARg-H7tg%&jU%rf8(LL!?twb^zW6m@zrE(P14|b#`QInioPD5n z@kuWkmu!8*xTN!qJ62x*#vL2BYI&F$@%9s3yhQ#^ut8c2sV#>?Y1pM1x}IR{p}*xoWz zKfU=e_-UD;{Ydx$e<_Cf8a_T}uf)0F^YOd|7zfbrAP+sV@{}_$m!3O!=TC&*q!x6f z^&-bjiMaR5xEjRKc;~nuP~7=*B^if2tY=4Cz5Ia{t#6!g9&9?$SBa*FYM@htj)wHu z%ZL}_p>-R5dp`d^1{vP=GXd|sxwwP}T{iP`j{BB~dxwnsA>xR~i+pEszCjuHJmRR# z*U;adYli=4Jo6(b&fbFGBUr1Eegb2uh1OZ$MB0~7Cz4Z0PWT4i;h%@)E8?$D;r+*W zr?O~WPxAd}chY|z#@P6Kynh$(zd(K}`%UPh>QOe~ zNAb@$8jl{r^A~tX*K`nb9AwvJ6n6ye`yZ>t9hWbnkERg59?$C}V_yRJ)Mqj7$WGvQ zu%kX3Wswe$-p@ezcb8F`bJ&gm_vc}b#{Cm0+c>cnzVTp7g>N)p{_Nb|**DQoXnx&_ zvd=}`)VJ}IT0i>YAm$O=y>=b^s65pm&ga$u|AUxU8=?361^Vvwu~TNQuQ~O=O9)S( zA5S3sOZ4x%(ZBCT_&4a2G|&DT{q_Ig{WO&S;?kuD$o}d2l{n*!=WaX=XBsoL@Xz)H z^sayV!PcW|j~)Bs+M3ow_h5`Sz4P<;uUl@+KJwnv@5kEm%tedOIB;>DF%O?_XPS51 zezk#k^Nu^N-h;I35O?M>WA=<%V}9i+@RNr)V?WaDzvJqii2s)xPrv`n6~^r8$ZsHz zf&Bjo+3tl)PQZOlbSJ{Ob4Q*%_Aw<#{ALmQz(q#O7qF%)ENYl}6=U&NpyzAPwa?-l zYs-;8#ybAa*5A+48L3Zgn7RDaR^01T_t5oCpZxdBPX!JuTYtZ@{+73`Txt9s7YP0K z`cJKzIq-h?RjoPsKsWHItvThPTBO|$J{F6`es3aeYpmw?^@uxIQ_FE^JKPC&@`0u2 zbk5J#EPbc}>6e~!BHDNb>Ds>YI_3nD7ib=*cBgOjUf_4u$9?C=(CcLJPG_TDLtDI# zc9}k{_P|RI_0IkZ{qHFH+|8Y>kA3$;tw-Va9=0@fbHtMmBV61FeL3<{x>{q|fs@g8 z6#vSi)q+wq)aj*tpgYR{^<`MoAwBP%O~2Q=KYs|AD*z-H~@Xp{H5TBt>ASd6c2v) zX{0p~H;lBmNPf5#@z36P=KYJ#GiJ|3Jn=~4l`o&zI=>S(vBZ-n2$$Q9 zna|<58_&1#Sg^A`{4Y4~Ut=A`Imky}zTG-{CiV(n2E3go8DIQ9`01Y8t;hBwzH(fx zqvdf$SwXm>?48&b-18*LL%8DdkhW}jhz~4}X!gpnM~OyfzK(Kn=hS|LD=rsl%a)7y zpmP5TG(8pl5_-HxYWFnKor;HQd*q#p=eNH57VrlH{funZbzcayj^lB%deO{IK}Wd=_TS=e9?~Dr%QMQ#KhJ*H zdVSJ>tWWr1&VfDh{5uhT0^uJbjI;IFQpY_eCoS1|QO%O3_H$r^3V#w8)m$*~!ZKs% zcc+6N6tlOQ zIeKNwAGaR;;i3yBKCrp!CP1#`k(m~-pt7xOA??;iaa=J0<3 zF5DjwcNqQP6PS;`iEkGg7arJ;bPqrdd?)zKh2UBAZX$nS(YXgchB^5Oq@y~}-8`Sb zQ)`@i;PsPH26Qnmq7Dtydyl37lkBx`hwbT$sE70I!k5VYln z!0q$&i?Sc)Upnh@J;p&7#=~}ui)|PmTj86l6FT%otV5^#L`U*3{0!!=Gj51McW}}7 zIgB>2!)IyXpVz{FuZ6#)g}< z&$RF_wD7OB@PBFH*R=2(TKK=U@FER=+#zMtvrY@2riIVa!ppVr3N8FLEqt*Se!CXF zTnn$!!p&NEqZZz*g|F4ZZCZG%2pa)^KLYPrxxHqMZ~{1B7T2F{0Ac9?}~`O zJ0kv`i1>RW;y)e{|A~nB2O{DhjEFxN5&v*R{38+Zk4D5l77_o|i1;TW;{P!s{^^ML zZ$-pE6A}Nti1_Ct;(rhke>fukClT>SBI17;5&zqW_*Wz1k4D756%lW=6aNn9pC?4b z*G9yj5)pq!MEtUd`12y-FN}ycBjPWOh_8=`UmX#@E+W1qB7Rdu{M8Zhtr77Z5%Jq2 z;=3c_`y=9qBI0k1h<|59{4EjjiHP{|i1>6wd@drs7!f}e5&zzZ`27*_ABc#*DRV;_r=!|9C|FCnDk>h=_kMBK}}R{KFCPk3_^j8WI0kMEqAH;-83!|Hp{({0zd0=W2{g5q=fnr}5oH__qkZ6X8=3UUuZQW9R)_jZus683;d& z@OFe-5I%@-2g2(Reh}ex2(PB^$h#WhdW7%A_viMb41{y|{#}Gkgx`tpKSlULgsC32 zsK-73T4OwmZ~gf81K@Qx!vFY-*N%N0`R+pa354&(_m>g=3BptO{wstJBV5GyM-cu2 z!Z+gk!w5f*@T&;F58>}2Oyxj#X&gj3>+tOpKNj#-Bm8lMO~AMU;jbe67Q&Yy{20Rf zk^XFiA4QnTIUV6g5Izc;o`&$l2!9XX>kvLj=~3oVgdarMt5&GN2kciGut(=#il?yt zM`^_0e&lx^P~bNG>Ad90+rl2)KW5b6Id-(hsF?@Eni}I403FAZ!!w1a70-4&{djJ~ zW8$gDvkuQDJoHEH0x-Bb58-3SjxEJghv#HGC*U~|&k{T*;W6;QZ^<#->v?Q3o>TCg zisv*ur{j5k-~W322jf3|{IQP1k5gFvs&DF7rBm-Hn*Y>0#nXpt)?}|<_2bgj+ut0& zn%@6I!hTN18^(XP>CcDzd$*lhx1!FdsXb%qqQ!M*o@bta*2yQFc*@x;R@Sd;xcGu) zr!G0^v~w=JHk=M`nhjgwR-*LEiH|2zv!|x8?L@)MmEjQ#%BSi}b)|Yz{itr| z<5`KP2~P}99MA1|p2UMLXWWjb20&^61ol!!4G30)PE~`!Nd2=09jnH~gHBh|isu%? z_XiL*E(#xolfp~2g=!DgCe(ikU)46MeW;Cq0zXt2dd@iWETi_MrAz8gJo$uE7N5H4 zw3^e6v7ary^@E=u`%3RUUdHdIehcS*ZxV{Ovjab z{^iXlzWCYy_~MHrJ$d@KK6T2a+0`#L&V8VG{SRgz zdFQ|1HNSUY=d~Z7{6^Ekr!QOl$Uoh7)8nt4eE1vNX8z`1caD5>$yh_fvJck%!)2RR zUa;n^$<;k)pSo#qs8E;uO5Lq{*ZlKSw|{VT{-d8d^7yBBjIKDCJFw%a&+JV9)A>g} z`s}4k{_^`D+;G~fxd(1NWBHR0o?n~3@@?JwwqN$euRhZ9g;OrP>iW6od;7lmiJM=^ zt{MOJ9Um!XM?O2XZ0hL`)Gqp`yWcqWwR>Jl_dodhXXd~5UsHR&(K(eG1?l*&20iWs zjD#}`(`YsJGvA;HYCryy0rhUm&(x;$r@m90(w}-)@%*v%Bd!1QS6_VK^=~|Kt%~RG zGJPw4_v4{Ilm8cemxBA{yUJ(Ecj_bbXP0C0D>FVytI{jHP&xj)Sp0~AKw)B(NUYK? zm+|Ty#Nj{nuF6yI9NGGjCuIG;)+&jI%_%!b|lZqg8w--bR1amh`9EpW2xIP7^;o zFHw;AAN>*Uqdx_n<}3PB_z|C{Kh>XUE~7sqp2j-0*$D{I|0ft7gDuuj=irdl(>WCDwqgU@dSW+PLw)^C)=*bZSMN3> zo*$cxPnjd}LegASGDqX7bTYBoOityJBgJImnpFvNB%3MbvuO^-Gl^6pUQE)WWr2kR2g+fPqG+Ko5gIdvB}Kkvx(A3F_q0U z7RKYbq?ygfN7Bj7W`AsOunwY%o>-))4o!D)MbvL?axzz(ZXC;JOF5H47F2PP8ri+c zd^(;pv!$Z1CUz|gBl&o7WZc}BM@m6AGhRe3(`iuBO~SEk5fpQyBJ(%qQ<*UVq#iXJ z$xcp!$c@9L(a~hyER=>PQ-uP^Q!w|XisQh@9M2Yt=Dv8USZGjqOCqHvL1{r(i8?Y* z5|*2Rm)3N4^Ip}oe#4cf8@+t8kV+&oR7QSMf{U$fGYP$Pb`0gNrmjL2EvzlXCv)i} ze-J@J%I-01){mN{f|VAwh-S-oleFSa>-<(l*4=T=#-g9n0pG|n?_f` z!;q-9jjyOVp_-f;NtY7Ig1K$muB~e)vx#KdjE{^Y3k4HoFb4-a%&xV4YkUyK@+fe+ zv5-uo_k-vr^%t|c%8!=P>Bf9Ap17jDf0tbqodk*G-qc945$HBu(b5)}pwZRY+hL?8<6}uPRWNx#gL6>3qX7$MaXe{mY%zOM=n9z; zG=IWOrzTTH!(LXVGJE6cRDv>YN!BtKFOJ()HCBORtttfJsH!Vy zBe;l-<+kn~bHkeTqK06-NpN2p6vTmY@qBzTSxn{w!O|FeH|P^eo4qMwQh)+xlO(zT zQU+XJ|8Az^)5O43U$XJ)NG5ZG$>fB2LrWlTJfUc{9ZeNwI^v0#TByC(Z)(Ainn-3; znZu>TSh8riP}*i5#jIJ8vwa2N{V%i&ZFsQ%zdMWT9vk^YKg}1;%g0a^UG^kNVvZCUWKCsk~L# zml{i#STZAs3`$Ve%B6D2bSjgyM&p3Ls-%s2#TvSL`@3W9oz_5Kf2Y+S>lz416WhAA zt5;9bv)<_0U|ewX`sPV!YHn`YLV*izUN?zq0f)_|m>4FLliB?AssfFcd?{1J^k~vl zVr~OpxWW{5r@l|qVIEDOMec0DWM0fzd#qt|Ytu$2>KY6@I~_Rmb zC``PnLL|o3XR?q$vbj_;(O|<(PQ^zs=y(#`n@aA(%#AS$r~yDr8Sa3|1OQX1G*i(; zSU7&rYi(33gY9h|iMANk*9h!nyP?nKQ%Q)TK$t(V#K7#o2$3!c;Y(488p3{CZH{B) z30Nd%5eTX^zFH9OC?NzGY77}+lS8BvZDR2RAyddt`DDq4#+j@Hr05JLC%bSOkOgaB zDxD7a4Bi~qlJcuHi86qY-LO6iaWYI(5G_-=BK13N>V(Czn!s6naj?i>ZbNs_lgLkY zn(zWePx}8|lZ01DG@m(%8Gi!8E)8y*3{uE(t)*z-(j28==?K+q=ttRu- z`8i+}(pl=%B%z~Wfj@U>iDh^<_w;S> z$!>Nb4Q3`;%0s53l~g(j z21y)-u&Xa+CNkN5nFeDTvR@`^Bw2V)81cOz2ccsqw2X9WBAK2>q9Pbok(CSzQyVU3 zR563I)JqdMrqoeL7C>A51D#vDy1T8O9-ZxpqGea5s@p9p zagz!bEXPJLHDWO)a!7TF=(VnD3}>?w6IfReIphkC7AOLaj@Jhzy{moe?HlN^1_pP6)k}$2i`;R8H4xivZG}$L zfx?0?W`}00yKkEw(GwfmZVkq^cIs$CeU(7(OQ(kO@%%J;H&%(nOw4P{LVPr7)}t3d zf-}21J3AY%+|bfM!^kPjhAByx$frh&=(wcU!gv`gnByZ@BhU!GW>o=wGCxV&Yjm34 zCiX%|+DC#vz2&pHqy_1#*@BLUzYsDOVL{PixLWuIl`cNi8QF}H9UTp*3=KGN4c5Pi z=Ay-JomzxIoU_243u?uoqSrqA5#7Meo^Ut&5_%N>eQocD~9mX{i7cnWkz?RK7|v*%3*T z7K_s?027g~941iWfFFjS;6zqfww9q>p#^OWTERWio1{VneA&vO0X;Jma{)pUEa5@i5S|Ra5vq#cSE@v!= zmGMbz?^=o1YEbj#Er*rbMu6ZSf{!Ofg_wAAmBWF%G}iD|p)?sb)A-U< zh=)I4rI>6Ap@$BU(<}fJk*^#k!>Nol3S~ASp`HLyAShV?J{mI>Tv#r_0<)JxE1pP% ziDN1%p!lHGpPf9Sxb-2;fczX)d2u3nU~zsn8%PG0rL&`PBFs2b zdz!7OsW9bTV7h>Mg^lRo{DBpcL)o8$jT+EV0y=Pb;^V?ml^D#A%Ok49n`-cN@*F?f z8mtA7S?11R|E$@PR+moi_bj)ti-0|wR385!xkD`9TFpNL`1>KcVVK^|X=RmYL1eNs zS4J03UPkWpV5L$n95ow4W%#LqtFl!zJXGQZVd*W=Y4Lzv0ZAU-``&1&tUR)uE*g$= z`Fz!*9|5ldl1m5`c8j3Y3&z@2W+ha|+?=nT=H+r#k9%2Y70^30|0E6{WHae$t7Y9) z8>gDrwP@=x4~Ob$GKonMC|LI-8Y!DYKOdiRbM|Twpj~X(71*;>IXpP&aiU`vvgO+k-L6k#Po9*T1VE}^u zL;+|;i^_Y{_GA?ZDUmfAE0uELsM!#z#jlcyAIzQWuMe*U@vtyJRUoaTrD&q6lnW=S z4WR=3_@q=InZhv*&r?0lWnfi@y{dFYXsg7>g|7{yQbac{gwDa47KnHVuqrE{Djq>< zS(r+Fd??#cD#dlvLWmulX@QuB0IRYBs^SsEmW8R*$A_{FrBYm*TLiViD9ehIJlfrf z7l^3`NVSNn8b(l6m70D3@A>Bi8N)Js~3X2MKE9BK$My7VRq=nz4Yxi4D&DIgT|~K5D1%LP^|u?UlA=`Nh158KV+DVp z8rx`8P*tHZsEV(KW^tC&J`GSP^}>PK%55Jo_qcK(sCtJpR)5;#!&#N+GgI$LaV;l4sWd`bFeeFq*D%j|XQM_~1-NLhes{)Py3g zk@f=3foy-LDSD7@ytW1I8`v-or_VSYoyd{=GzywM%(TW2geTaZ?7k5swc?|rI8iLx z!h?Z`YRqK|DLy>Fbz+togo?&&D~97IklVVikR2`J9IlniX47HaMNd;r2XW=ID-_U; zDIpG8OofQqdUS>5yRuYTbf8g$4#{d+K=H^_4gt)KIH<)IWMr3a(OKAlh6qYlAu@qk zD?w>8iId3GCY2!OPh2H>0RYM&365=xkYWAClOzBkZAiKF*nn~)g*e4|XD4;Ll^ z8qb{|0F5Ad&y%SfE!;^h7&#Sj-Y*gnm2g1_*_k5|Q`WNklBuzAxGW)C#W380(pH0B zVE#x9>Bv|Pr!+0h924W&d?pM%U&3l|^JlIcu}WxX!`Blp2tgNsg(1n-WvVTmFT&$R*Mogd~!t`&9-nnpjaV(t~|$qtQ$@w=E8ZGg?1G+ zE(FVOBAL0p-BigO#;VC&8dvrRIspL6NH_qQw~s*u&hDNRW+62;nZ==Fd(C1?>l#k3 znIa5AAj$lzCx3Pv*gu}qk$s^>Lki2K;r#44idP^Oio%{ePQCh$z*bCq${tv>6xChS z4j56ccylBTS8)qPN9GR1rh;w&JLTj;i-yEWcFs_~sKY_J%pG0$C_KCzN7VckosWNQ+MW>v3Wv($IvTsNaLXK4Q7-Gwkmb70)K64|Y-3^A)Q9;2VMg465ojM#v0hcm0D1ps7%SL+mDv>~y zzyJz9B)#!PayP7(v1bB32Oifl6TZ&jiL$89_HGiD9hkr>jFo}!N6-s|$Au`%D!N)5 zreh^&hPAFNpb0Lh7g|201;>ASYNwY7VkTsVt0awFdGj9nCTOkm#gxM?iAG_-12YQi zI2`{=rSn(QT?b~&fqTOHM|oCI;Ai`r12(bVb!Fb zfKG9^wzi%9Wd=a;yt}0(@)r}DidB#v21T2KUXVV#!V(1mq|#}dGY|+z(8TYHwUG|Z z%UFg3*bqTGiVuXnF(86vYAF*aiGh2B>alLg-yUB48RIKsL7pI%KRHQy4>6OqxY@HO zEzx)s5+PiymCMBrX@w;M*JeDP9<|aXmkOxo#;yG=CdD*%?;3)$D!5^WCmMJ(6Kg#< z3U{X<7jE!$TH$n+QCn4tfI_azU6rr_ zAfvu8F6L-KF@{xwWL*j^;jxoiaTKDnq(35uQ)2=$`4=vjxQ(EZT$z&{(J)-$fnH9R z)molNNDfU6ad-jFs&R1yh~lcIiu8P(&Ww!bvl*;@HIIfpojrX6H^E71f8W56L04qJ zt7WE?(~|6Lv0_6*z1B{v&Dsi<-X3e;4tL-KeQ*`&@^|QUfXPcW7(&_ppNFTxyd4+4 zn&abOSI@j;3-sOUaqOB;(u!5497*LzO0>fT>p8eLbbA%tiA620Bp6O6m~KJ5LMq+` z{aqu*o0(RE=Wcym zdP))A>dED8a7k=7R%Z~a5fuZMgrj{QL>>V&XvWJ2N5qb-Z(X51p0uJ**?x5>TpnNvr<2P<@#oW(xh~^A`3#@0s zOT~-<@5PWAKnD_KaM=-@EX!33?svldJvm6vNkg_`BS7=MNxW3d!h>yU#2X_xq3yxh zDPE%Ixa8Nxf83~4Xiye$j8{U|B8aQ|e5`lI2D)#8m*7~3&WJVA#PjLt#&j0eeG&}_ zd#iAVi!+<7TeHUYLM{b5z7T{d>tu(YUPv9>Pc#w3mq9c|(#bcRgjYSsJ-xsJQ>bcL z4k};g{Fwzf`W!?7TwepfA|4O=!BkK!CW>qnjb4BpWJ|tP&?Fp~k1#%-1R2JHbR*sU zvwLVOwri+Q6ModRT}4t4M7{lOlX2W##U|r$B2Ha^1S4$7zZ*BKxg|x>j7~Pv8qh^q z?(B}}Ef*F->U6w-ql37Hr~}Xqi0&1j?&Y;&S2DgA9e3`k(GWaQYNfG}OJ3K5Br==f zsib1dP~;%`+L~+9gu+}4fatOy;r*RoUDA32Oe;SD%`GiVNJ$K#ScRH*= zM(OS~WZC0h54zLAHRD7U4lZ$-T15%nYFJ)ix&hh4R;#)Lz;M&p3<63Lr(xE&(_IOP2scDqJvdPAMv}5EO%;M58Ac;vSugI2PcCsI{>a zO~J73PJ>IXc#%7>eNaoNW`XFG>cYI}lp0K24>Bk(*wWhz*3?qAp<2*QSB*dmblp>8 zVh=;>12Z0Vy@qqW2wwo2&JKg8QBLdxifc#cJ3v~Kxbcy0{j^8HIy=;44+zLz9CkDv ziom4XWwA^^rvsyk%!r~MN`2UwZMB;2#I*|TeLW!eAntUqw(c72baCcA1e-RK8E-6W za6OWJ#fPv3F+kcKCIE>8S67hk1C`=CKzN7Q1yJpTv%<>O*zKXMZ4pIxjR|cQDSFnH z$O@|9>?$xjmOwDTd1G8|tBZ6ZZu6?#X1WeW0hLMDilh>xLlF#ia;S(BcxnWko2PKijl(cbeZ zn~<^Tjos*`PB1nu%=u;!FNr17Som~7wDACmCb4_AP^4pi)518rAO^58jepUdh@=U< zzI?L5p+5Ly?CZ68o#dph!W}jyZQcT~aq~LXuG3XTt_FbHZP^_H*(Yp+iEI=nIdufk z1i5V|socTmO+Gy7n^D2%HVI-IjSGZ%IwNTZlJi{o*!wc+Xkq1l#qnU|&L548o?gd8OzeBsV#pp3IzeD6Z|8vA zAvlszqBuXxoFuSY+M`ZuK&=EXh|1B_cL<3O+J?OprYpJJw-AJvsaK9q`c_B!I>VJo zhsMWnD;leH2B#$-$T|RlG{YxCt*r)NU|!`o!Ld>>g3<{b#J!l%SlM^TW=GM+v>ONA zwRc0(zL9S@3Dt7azIu){UO4@N8%Kn>1B@}~!faj5sjch$o1p=-KNcx=x}kws?;tKk zAEK3|meyH*+DVZ>?yv6dq8sIKC!eiU)pBYpDu31r(z#c+I<5`^xCvEk+UN`6kfd#T zkML3*i!-``P~ANUYl9MBuS<$XyG3Xl+-}6xaw9<8T}wJ~zTDHfkJ;8kvT>5S2HH~) zZy)T-<|j~4BdN%#ljgJ8$<3Mav0BR9RLpG;;kv7@$0iv+4#xXqHT1%IL zdmB>{Ne3bKbHy$X!6kx}Dj0?Js)`NJMWDX0Dw-^8aksr{_$_~^roe@pWm{s4)T>JN zqq>N5K{gP9dC@P-z1YyT#?M@jV_s({Zx$|sr;^YiER!B$H4 zow1>+;HrGvz=Hj5>0X4-@S75QWmyI4MjqJ=$o zYDj;C*f|QLqlUfR+jLrL@q$21Pj7eBQh_b=PPSa$E|``IPSCIkl63=65O)yM?X}6N z5w=jnWyH9hjh6YMr+5g)C*N=;DCo0BMlo?~1!6Iaxkjz%7hGitX5lr)XeD*ZarI$I zcL{0%xEyc8i(^xP?9y&`jrQCh#VJRVo#JS5;y@)h4l9}HA~H>^RKl>lWuz@O)V^Ja zyM4QcEL^n=y?$3;ua-jztd-|rxt74;vXX5cb3=OvM9P7iq{8Nqzd}kd$JZlr+u~Bi;rP zG|(qsmTl}bj(k%nBR8D3q*gA+c zfYa)(_6^KmbsYntl&xgdRp7=Asf1X&LGU34C^;h0q3U{s0V-RS@b==xD>|lDS!@*g z?t3i@Js^V)&!If|Il{(iw8B#ux7e~1gk%Q@X$}eoOx&b7B2AQ7G0`em*dD4w_R{=R zSah(YEkr*rD6Q%lx+j}X@M+~$iL?WP>3~>p@y0`eeS~xXFtlAyo-Wl z)(Q&}tHFe?qzal%Lsvl&g*qfJ+mQnDJ2{0xk`jT*?SeeU!ly-rqK5f~G(byOWaR|c z5j5pcf3y;`dnOUn&i4u*T*b+p>@Z=di;m~+nk18@Mxa82_hOA=K}p5&v5=%%%|!uD zy%`o1Vi_1>MUR|FKxG@iKeu&sw)b^(S_84&bPj2I=KzjMkT@u5MSa|Zs*Le|)esxp zm)h37%b5a#&W4FJIvbG!Irc)z=XBBq=U!iF_Vr5E;(jB9D{yZZ@)U|m=o2$z_L+8? z+O-O5X(6?XwB<_;%ArFfc1L=Ta(jhABBwW>m~JZRNTFS!w{M_FJ1sNFtXRX%i#FU? zXuEql861I7qD2CZYw9QT!}I%36l!e{l(nki698Zjj*9eFn?`VJA<9@pLwg zJv5kNrmbV5xH0$AMa^3vT?MKZ!p57!3sG#%@OB{Yld1F-fcFrtJfzAy5n){xmbVIm z_vXzVmC(e4w3C->Ujefu^qgEwCJ#c58`iA1lhC?c@4r#~&!1VG46!kjvM=`*e`4u) z#a&BI#)!ZGInHaO@+hH>P9Cx1CK1%m?Wr(v8F2`J5Bx#tRON($@+>)Nz`k(XK;JGX zlZnu7z}{jpk&a;sbtj4{9i$1&6;fgxZ7%131(MApuqTNGu_QQ^u-YC(T?(eb_2ZO=f39jPT?M>I5=6nO ziXc(Ig2|%@A_hd@oa6}074(HNa3aYOhD%js2mzvZ3);ToRlL^rWs$Qp6h^QkGJ;do zayv|Tvk+>4$>d}fb{%ZC0Y@FAC4h5A+AD)9+3n(<`mfaXc<6<*p!vAYY5aI;nM3Nz zK=cYz7J1I0(kg3NhEv!zsbEz4XE3#d$L2`qfYsI&8}u#TBw9i!@!~9qvjEIp87sw~ zs_e=bj1;4jFnh9{YQHk#+@?)Rk;!g>R0ikn3fOQbBQ_ONIq0FgXwaG{r`!F5s z@9rA%Vyd0T7AIK5GQ=^O;M2Km3`G)>JRW<^x~WxDIAWI=OXB<>1SP5Jw!NZ3UxY(f z*>YaPNPFX^aZeu~5%BDHR*=T(j@^V~iT;$Xol|vs5kB-pt%aX7aSf}1Eu)~XNXsV= zrHdnD&jxWYi+!FsS?R(}%sL0wwJQI#Z&Yh}odOz`v{=Lq&R9 zQ|otdc`DXo&H;CJCsDYjNe(zZ-cIL}u7Tki!KGhU z+7?&>C#tHu59@eJmL?_+6i-HwQ0>reBRPaLVZK@%$A=DhLK;jaCrsF2Y1jIQ{a^`1 z42|-0hPL98xdfxDGOtwUOAtX!#MT5-Kco=|A5Ypcg>%9uah6^0LbaSG-j|0Pw>*48 z6ft?C#XP=xL>|R!iR}gz9K^l6wZ`_ZI5H9jM6xHjNha#z(h5pkqIo}8QS?(GGKkhM zWgKBjYeX?Ih#g^)v_fS_q+%`GfA}ZI`t{YJjbSfy9&O0qI>)@^)-S#=ER7wsrlJ0Jlyj!Mhc z_f1-f%0|YUg1aWB|B+HYPeWkq)?RdLKG93(JIN>mUxfmb`~ebEX%ssXup*YK4}7Ps zsf{M?myz96bnr$n5g7U<$x;h0pq1Pq!IR2ZKqjbHAoE6DX`s*gj%_-t)7V*6f9JqZ zP*Y$Jx~fENeY<))Vgow0G!VkQ7&*n#%kto}Q3)XIv_m7iYD5JvG@%FJrb+>ZCiDQr z&9k~+kuoTTr&N{T6v|bYP8BUdTMS3Y!>J}*jSbjes>pSvdR+$8&>yy9V0fY|!CIh!?(C5ZQ&Z@$ENRgPl5~ql`Oub?)Nqfhn|Z z5P^&j>i}#8Heu*$Wr>7kX(M(tt-kkh^9@u?Ad}HR@ef(PjU}TE@$FWhIR}RSwxTK(+qBp(cU$?)O7c6yC~{ZfkRg>%kdNF874Lu=ZKMD z0mI}Wi?1F?VLCB{|0Gw1=+;PKwKcsc7|;Jf^~7 zD<4r&--N)VzA6ua0R_U;v%IgQz&t@nmMNqe2Afrvf=s?*Yi zTMCGy%V!#KN(Ej8-&(cpck1j$%{VWNo#l9O74Ov1$l)s- zqlgV&3;S?6*9UQc6WTCNC1qKU{d7;0NdrbUlDutuGL>k|X42ENNs}mzVB6Mf;*BEV z9G5N<;Vm#}_$_!3eIa=CD~zN#4P8e=0ihSL7hw+ZN^8;O;828j$fhV<01?PKNC@r- z>r!-J=kZM$lqy(P0~U$_1(%Rg?XoX51|KuTS?Tk}z^`<(1bqG|MT|@f5t93&a)CoY zc+h!M6IHz5*v*^wqp-k~cfY$LF(_QxBqA_wBTC?&obm!HZhm%tZC6ONmT!WX@~_g+sAdIfydQ;_Gpbdp<#ty%TKszb!@#VEb1 z(SYkuhjEnEW*Jg{YU!+_&Y-~tH|mC?$dNZI8gzTAq@1I_(ead#LZub0oAFHfPPVG_ zN};M82VR8=9l7=ZbF*%|aga}YLw5r=bKO=?k0vlGT|#6+UGOgp(Z_PM(tL3~hP7=V z*1z5A*wxc79H-r=FL4hUC$+kgP#_xAlhW2 z=3N~wB@)R80mujRNF2nvrnn%qXavaBQ41g`K5Pw zqv~2A5PD+$)?hpAi(Pvt5t&F43MEA)wGJt^qi;9-D;zh3?Xm8y7Oq86dr z^Th&5EPDNAL75}SB!xra)mh>{GR;0u7lNgnSP+U0E?OF<;6nu~F$CAz|CyQOx@O9qNv7bP6u)`;anUGZ#@AN+a4#b3 zrdk1a-$)yJEW+U=0O;~nKXtq+;cB_pCX>Mt#w4y)bT7~Ni-DcpY1d-J`7pWHpnvh4 z-gUJV0vi!Kz0{FWs#zysz2)A#HlZ+5l6aj#2FsokBvp2Dr_0H!cSwRF|EVlsI7mZV z1$X2uPDn`~7tWD~ReCcZ_iLJ)<3)k`RGIR%l6)%%Ssm#aN-SK0+Ca#OzI7Q~XNV0q zF(R-Q`GeG}%xF$dB}Ynhw9_>vaN=F8?W$VRh#hG=FKr@DCW&QKVEm;K)doOja~OWu z$OPX|)|}1;I76)?hkT6dS~_Spp3P_Ii!jFqtER=D=Y8!+JO_h;B-*7=^hOF;I@QHi zr&CI4lF%vSk|WYD6Z3%W?bo!pNEnb!+6AzOe#HsRedAep`{s^LTF59EmC;0_W&j=w zXw(2GRW+}RdYtjekdNqZR>hGbbp8eCo+o6=~NijfxtxC_zST~u2%zJ9D5 zcR|`N4RIWmPr(Es$BqG!Ko0IIxr*I`Z7`J%1OJ);_8)Lh!p8}Fx^9FW&8|VL`|OK~ z`P5}Vt*jATEe?LzZ1|ATTZ8oZ;CcxY#Ix1X3+*j|z$&eIL@U|#!D-SYU)*gd2j)S= zu0^UwqP|{PImA&_!LrGEMcSFfO{}Pjsa!hQm>q3wA#(5`MpxJM=dYZ-`tu4l96je~ z-O~>wh`Iq=LP{M6lc;{-?`C&r*S76LgDrwS9%YQLoEB3wk53CzXZX|lKk1DC%0jRu z@m1ywq9GCm&r(kA1(46ltg+GXJ8YcfK~&a2*aWw8&`d9DiQS3d>IHg)OTOaD zgGOEsfs&ZC$eO0|Gt^pp2ZIX2(;m3` zMWs`|aPs5@%S41Nb|alAZ`%r&=K+_0DsE>tpF5$d4V@)jFD_c7n^UdfQE}us0Ja#c zu5!IDvVsdms(fn7T_Wpd_i}OU8EZ9ujz?wd!(E^ftqrOjMJ7v#n#C66xlK*asKKqXHxXamp*O5p6nK*fS9p0PY;w<# z?(r0IVqmH|ju1)n#1uWF0lOj;<7O2*I%3Km-R9D&FWKyg677V%klC9}v;NDVFFSl+ z;8VhrCI)wLW1Knx-+naCD5n%1xKq1E*+e3aYp!U}+Q^I5pM$0!eYNipj=!%AnT3{O zSS#9>xd=iM5-I^;@}Yr|GHFC@Y=j9_w%UN**{x*jAsm(oyd4d$?HgDq0&T-lsTQFt z?dn;1&#DC62v%RpWS}BUG#IN2R)*aR2$%u@(vRX2b9OvO=5}yvok@sm=Cz3(sch*{ z;G!il!Ba>22659RZUdCciqUMg2%8nC6_w}k)y}?+^J#A?UqolO6S?-W?JKvXpZ#rJ zZQ9Qj(KH^U9P zxChi9Avc9^xjI|+xR6zu0%{@4k(u0hxv~XE(J9QpqR1jEFd~;PWs-q0;FbBnh<$O~ z8LzjOLX0Eah%rK1Tl2ddzb+Um!h=75hJj{k)ZGF2dn$tspmf>=N5K{wQ~4x$_t44@ ziq@+V6z{H)oq~740%41oK(Hbz7$OZ|)se)`pDK%Eg5Yt48=9yxW{r%FoAwoE?)g+w z_2DME1%!yiIrK8L&p{=Oq;W+uy!h!ujz(deT;jQlRv+Ea)z>f(V=1^|uDCO5*zP zWCr{*n<0nkrtrVO2VgJ9%?w3Qf={fX1q%@7aDxgAi!*eUF;pw1%tQv#pzB^*eNvEW z=2ZoGQ^q!!84tvIT$%zq4X&)&w+3vVbQv0dR%SVDo84mINMWq$DkIjsPA>=-$%@mS zvHmX7!w%!r8w^^K33Z)!Iyn}{zP6~hn2+5$9v1|4dnWG|q2`S}8(1NVlBMb#nJ>Ya z{&Wf6cdnjHqVHhuIFU{bUt8z>E-=%-GZy{HJoclp4?U9X&(rlEslCa<+VH3~qF5`ND{QF~6*l)~aTp){z>;xHsQNln{)w+w%LXBf zkJTosJB9DX!qu3Aao=EZwb^{_wPwt``DP)W)vs%K*Sl6P1V)YFyMHt9)B zqq$NeV24D;hSksl$W2^6@6J0jg{xaY)5%G#^pPQyJ~_Dt{sssth9Dbp;KBmfdXT5~ zy-C@MXs?kp9VD(V#sJ=VYr`AU@Y;I4UInUtQLDeI4il$9wJxe^gE}e}xL|^d8p(LQtr{M!2V;B=CP4YH(>p*8F~ufEuhFYj|P z5!dJyzU%@~EM9_&#FEIDfo+!gAgLr591Zvq(S%%zo&{LIe!J)`+8v>>jp>%*yDb9+d7B%yuLdzu`TT5`};Z$`3EG2 z;YUdFzO6%aU$U!e1#y9euWAFG{heN{vA~7^<4Z~>{9Q0{$i|n5_l#YsNMqnj)`nZt zT&b`c@+HD)=Ae4J$+TvzU`Wm?o#ap{v?x=;g0^DrAx;AKR&=u%ElTZ$w^`4(mXQNl zQymb*8eUt82V|-<>ej9`<2gDV0&ARAt#$ZUM$8#aBu7n0kG&lppTueN|C=JvB|t7U z>KA#0By>qC94Vjhn=(6Sru&Z~Vue+OwWcBCcW0Udvi$`#0-WXhRW^>pt%xBZ-dOF40 zaB&06*GOs?)noN{SzFEZYnob`HeA)bal`rzo36U@s`bt5H??dWt)u=qfFS|X7PJo^ z*o8DiY8t-SB#Z5J+&akaZyS04Hv!YYv19?Z2*q*e$)F$1O_K->w-^MAQ?4mKp6VBKhSBfRTT8Mz>JSMd2<% zvy{$)0P^qcl63THXLO6%AsK~*5reTau6N1jAsK%h8CzT#Nw7C{vA&E%FKPpL zG=})4fpT6h`-G6FQ=?`*CAsz*?lXl3G$MH+60tz&W(}z;G&t`9RJ6&>$owu6IqQ-c zSQOFuteEaQ{tLFWf2M{`H<%Z->>kYwOm|0o^0Q zw*FoCI*3zM1ARC}Crxl+_Y6IDVcLw3jG)0*t$}iA5(<}Y>!Y3B|e|T=Q z!c912R1DUFp-)#_%Nf6(758+NCP(A9kle!WaN$eBl-GJRI>zK z3w^agbLC62y1-@!L4j+NI z6w7&&reGE|DGZlI4%g5kg=40agJ&nGD#1b^YQ4LJO6-McvCv|Pex(#tjyQ;k26ytI z<4D0zdMFi|82^?HJ02izW$$uYVzWpbCdVlT_Q;{m$ntCmr9(W&2RR%kb?BgS%gM#lW0z6Cm1hU|HsA~ zdX%c5v&TgrRZEjL89g6d!5M2kxIv|x9RpdkWTIUzjopgIlH)5!B%`|}%HROQdwV-= zHT7{XMv5hgz9`_+oz4Z+1qS=F4TP%-ZWr2f8i9v07jYcJFme>Zy$`)f!xlO~hX!ot zrd}Bi+{q-8Y3#5k_tE9WUUKZ}CEmWHvx8_M()y_)HR|%*(+I1)VZI^8omg%;aFsqf z$Qu=neOA_u$r0*6Mw>!>j8BL$PNvxfY&2&Qg$51h-aa_*qgLD<8|c~9Z;&`k;;K$! zCaAv-g3gRP=;vInY;TvMb`m{LJQV_Nq;?yDC=Y0+GWi7=&1WZ-pBwd2wf(BXnl)<- zNhw^7oPbmo-HB0_Twl|{j^e!UO`Y9)2 zxXlsOec>KZZdNIC2KN)kKLZ^%W(h;1fDLMr$Ai-uVk0_sSfwPfe*qpOEg*QWQY^haM#oOiRoziYf8@>6 zf+NYnE*!k*9THX++E7ro^_8Xeb@8FDp03_)78Hv;ZU>XTlv4H#NU9L_WeEi?9Ypvt zuwdQQ+tH~z$Or|mrN;Hk=*7&o=+1Y@x3Hm!<4gs3`K_2X!H5hGd!uyuBwwIo=hFr_ zL|$42yPJ=tOm-zt;wNw8!p^4vjgwH1@$ECBjcFfUmoI#E?YyB2t`|D`dWU*Du}BLL z?R~`ypxFICV0s})8nh7#b z%cr19G1Nw*_DN3odMf|8@KJK0Gu9E?j{l4hEZEoE2w!Q=<60z>HWqY?IdsnQ`ayRT zpSUGX;QlPm;CeoiVP`l3M1Tsaf~7jtg6IxntuA=bfgC>8~F3X<7q zNY@zxUG0d5Y7Jp$M-nNowMrRTmgMmQAQTC7Zw2vw+yn*-P||x~!3@(6I#FxO@up-Q zu9ih(MbV&`pWbXXHo~16u1}Y2s*x$)D6P|2^RZ3LZ&W?xz~YYAEKuZ>&)U1K4UI(p z`pfb#>INiwYJAg1MooFtrTp?BXiTdxP+JMZa#618qOzkhxW*g$8M%1#aB*}Q9g;J| zd|X0|D60orsvQ+^TUns!2C}o1$$Xz|5z z!eYaWSZ#%%)Zf?FUAE0^qKgt`<%Qlkc%s1&W><%zP0Mzg&e%ZrO;&dwXj93gCr~I? z-*UYV#%bwk-0}jtVOJ_u7k)RgO*R zu(ulsG*lSe)EKmbV}z9@k`0rUF}CO{zz%*Pd4E)%(uI|n@G(l(q2vgZ8JlA0+7iWI zPiH=WUQ13+03ke;>K&(Jus=4i15UxoMU@bPbuD$-LLA3?taT(!aGJFz)Y`U#4bVw> zp+q0RLZLv;f7$a!nh*!q7U0hsh-zlcf9rTsCjg^(>jvV1IM$)G`87LOY{`k zA<$r}pvrLe<{6jip*3o2KxawW=gS6#!I9Po=JFr~y~ zs&^2FyQ#IkaD@*$n#x!Uwm+0~gq_Vm&fsPNA*7J#)4_GVN+6`U-Hb+XwCDzRdZHBG z(eo^vcJsJtkL?=j3y|E(B&GxG^p8frwTHZ$l2N-bc-J|?&KA|QVt+*rnsWL+t&=*r zh3rEF#cd|<#*OGwW$hxWH@UYjxHJs%2)?5k*t|+;2=E(R1A&7&ryMF%FiILtI9oZ0 z!bL>6f#a>1FRjzgPU8yq_yMfO18uWxA?{6#3-LKAi1MC~Y+S87v+5(fP~n(xuLT3u;M{eLgnbg~V zWO8`AgZLlZkK7Qe=})ele}JB#mi*iMhG0+9-`R}?DY;w5Eiv}iraWW{hgj^6umER7 zMYZjz;=~bCD=<#xYw_k0-@RWYBwuRrU8m<5c0q9UH_UscAO?XC^HwEab4%_vMv~Yj zOh5pzJ=(}I;^IN#&>Dt@4M+l@K?I|NrshqJaL)k;>kuw%eLe5?%93%40-HEE=moc= z`ev{vnr|c559tQM;6xB7gQ}!6d?77$kY0xO%lrEV-3~fM0&J3mr@`jCy!uGndZz6# zI@RUy3e`gfclGy^>10qg*Gh)&-f1$yc4;H5Y%}Q866vuyoNI!v1>2fUK3e&baDO!P z^7QV(23vApQ<$9xV&j+_D<10VA3;!p_HT6+ejQG+!&pq-T+q|s4+Vb_BSn06P{N6L z_oq_Yn%qnF@8WZB>_)YdC;>={@u-_*Qy8X_A4#1M!r!u=1dhuwvWZ`di;l6pF0IBu z;H`W2T3tGA7Mt#VSoRo;AZH+$3h0Rg{U&VV3Wd}Nxz7QaaH!uVjz+f@(@!-mrDzA8 zj7)Eu{^3p)94X&q*{6~vb3=Pj{JMyEJHPEPDI|YLyq$l&Vk05>L*niHUe@WwKP2AH zzagT0cRaeGq768pCGOy|^5bBL;GHf|Ws+`mBNdv`4^N<7QPIIkgNhDH=`WglL;$PY z*#c#U@Khe;!&vWqTv1V)IC|S50QNq8N6-gsbJ}ZNmc*Hm_3od%R>obFNl0sJ` zh-({|?@9kCfd~i2X)XfvaRo-LJ0ewMJaCMLqzxR8A!*&+T=JffwC-`HrImat6h|2! zikFNJ#YM&km5)wVjzn5?Q0ZE9P~lqi2I{WEBPn>b#ZMMLVn&0NB9uEg)rDRzloU$t zlXH@$!ruagTpGuw#$cm`y$Bd2%cG5=cj?*%Y{g*24$}2+?7X~pAP_$($zqNOqCJtV z4C%G#2;dt6cnVJD4D{d@1oF&Hs?_TP(#RXN_ykRRU+31Xf@V^qi~7s8i!zEE$o6+a z!z_*fO6fr`1M=8oh&Ev`@Viy#iay)==LL+@Yt-9%vnM z+W_^H+yRg~N6^y3bP$XR*PFtD$R2Ch(aD(rXWDiuFe_K~vGDttgiUGEW^In`4n(dN z(+>rFPr8<|m9xHd3FuE=)OfCX&wXxKI zr6U?6l}=EdR8~P0SFE-YR=Q0eJfWwSr;yEhvCSh4d!ZA6u^LDyobs{<3t^iH$xknt zVI&v__Sl-JSEkP zV(BM3H?;u#5yP8=1Wf=fK(ZK&h|>$9)gcZE@wqFf5TEZAf})b)z-A&fk}T+8=$IuM z3ij}33>PnXE5>tTsHTH&;@1>Y@nI~8BB=8NN1 zHNL`lASkOm*%m-0Fmel?Of~G~u0m(Wu69(TS7GGTV5oDDYSlG}$TIl`+flFf?J#bz z8xlr*;vPpFWDeT&_amc7+1U>BMa(u1g&cfT3;1)3s!#)10#E<}f+}A=hb%{`18AA= zV&b+DeV*Mb6mgSS`_(~ z8<1=FkOT<%AOH*5G&VAXGM4*7MW;&T;qNb6oQf7|2=F!>g&*NkB(72n*2^zPYSzj? zic+@x?D#_Nk5X+hm~SKt&Wu8h`{=s<*TW08SYCFRq_!kb427mzHdRK8P)Zy(Z% zRbx$u7nsnw=*VsiEge%%ZtE)1R7`r`aGWqO9NTOp$ZcaO`eM^!kevI8rs$`Y%_Hd}NEsj;f7oQ_%h#Ft&@ zwI`v^n%qD%;cEqn9D#$5G&$je&OJ&ftymPuF)HTkGVQ1;1z&7fYC}QH7dnJg3`djk?Ds~5B$ zuMLgCLE1bOstUVT^BmJ#MFz1bA+Ix9CT(0*R*EyeDoY<#fQD5_PenP6YWe7pRhUuq zmTg$O$I=CtB@+t>+>J>Mq_wFwa+T^*wBUlUfr%-Y#3asc9G#1fhE{>IKF-fm+YWLbG8EgU?@X(aUr8FJImMc9D<0OII;90c>W8h` z)?MV6C6FpmijMQS4PJJ}VU#;&ig_r&V?C4At*4?DD-bZZlFOPH+J=w%wb?1cs6^;@#F0DJ%Au8KedbAv-{VBelmNr(^WUG}) z8w=#sc@%_AOl)Xswi0{sKh{TZ-3S0sM<)!Mu`)R44N4(62hdPaZ86w`)5Z3zB$k|7 zvM`J-oN1<83N?~@NUT`B5M;^Hi0*)L1X^0w5euJ4PO~6JPVG1VmPp_CVNIXiM{->ZT!nGqw% z5GG6zCPNS=2!bF?5Ck!T3_*q@ALeh=byCauJhSzueJAH z`+c8%_Aw_%mYybgdtALM5MnE9;b1LS>xfG7ha^^xv{8d)VzHb{MTt^HA2fp9)M?Jr z0b8JnZr_O%Ne7(r;LT!C;5$B^91DC)nBvq>z+FTvj7X(Y6)+UZAK^Q+^h|D%QO*Co zRYP=4YTS@)36@!Dkz>&2j4RZWbzPzR973!9*1YY4>U9B*{zcWEJb~jYvpRJFkGDr_ zvTS=eW-?dRf+oFGN-dSiL2#pti3R5THQ3uG2cMgRwSC=+O=t34d-B+k&Vn69epLka zT|FqdA*Qrc_}^zFLI^HS%Y|bvpX2+ZO8eGKG%r-XCoAWnDc}tS)PtxQmS0tt~h%eN1$y%7hdrO68ByKCuFD_rR-e8&{m?kyS#@!>A(2%9I^E+0OF| zZ*gI;?>F-@I2`$Z7N0TQ)hfIXj>lRXY3+2fAy`P{X(^LDhxFBEmyQ+5khdDgrE;QS z$h%ag3?gkY(btdB$WO*FmP8Lcc?9>7Y;HN^XgVZ6s2RuEqIgFO7UJGUJ~$iSgpS5t zot}d9?ISc<#+2TewJY$(5QZsu&7V%rkY6Z6*GXSO^I{}IbQ{irm3MTY0Nn&wI4X%7%#uS?h!{J{-{bQZA(IO42Wl1V+Y|h zr00V|Fxd(o(gGPd2^L9pV8ML4MM@wA3edEb(j?Sn$Ko3G{O!e&h%0$G3{(KXPpD zPuPfq_$)b-!Er$d+VA&%}BuZWACKKx3i6={&$(STe z)vu2GPIidDW6T=zU&L`AYDRfxJN+LmYu9gF7YxRfq@0$< zFUFo9dfgQLRY;jLqrfpWgH_Gu=<2emLOtNXcVqI$n06=5oukbG%m@jnTM8)I9iX!^%*t+9GVufei+z{{kjjzcL>>AAsXi+bQsiUXHz7sa;E#!Hg zIfmoF&pmz3d7DC_#XkA<#o&l<-@Zp{w5-V!JDAa!nCAKsA``k}+7?9bChxb0{J|bM z#*=s06Qj)D$BqX=I#GEX&3<}PemLffp_gHUon`2Vasyh#(cu7gzNkJfbW|h{Siwd$ z`F<3>a8inf5a<13%ceQLQoXp@Jz0zuX}rs-#*V{A{nHoWf?9uwe|po@jKvS|U+kqN z{ryW=tBlFPH(r7>1`3`=pcBTQ6Nc3xkqgWpkUS3oKa4*MsREVZ)~-@wVm1L?xz(6P`2 zzd>I=Y%Is#@qHpJ2bS<2%F$w>PVO@g_LK|bI0BW1M|9vUB~e!WN#;Hl<(bCv^ce5| zO_3&Tqf<{=Lr3Gjt2lSuBty+fEd{diZS|Zn89J$ea{PfsLC$Bmkp7?%xK+Cm()^6( zIW5qlNif?NwU1&S8q9Gz%c*fwuyLYtHQtBSP~OxqZY)2E2DA8&KiAbd6{?zAlwWgs zW>nk^fNJhOMa{IC0^?(Z-973(c0N#09b&=n<71JEzGINF69W$UC{sMjoFw2;=GeDU#z~HS31}R1 zl8L~m+VSuHVJJJwJU)J+DMGmWRJ>%8)C32~6D+vT%(2cFO!d{MdxApD|1CJ9`4oJ4 zX<{nXUf;jA=b7!N* zwJ29VEGfaWo^T2n9XWA$$#Ez^s;qE=d^oQ^c-Xp1+#*#z%~!I523Kmg0 zg)p}##n6%AK@f5>430+UfiYlM3buPe!Rqt13>94G0C#0K;|zS~DiM5W~>f zkYf*l8yy)h;%E8B0)y6zJc4f?{cDw>v@Of+t1$cQvewS$U zYzw*ey-yV(3%O$&xP1!!&;@Oob}YFbbHcGp=_%3=Sc+Fe>5v=<+IOkvoK!4HdY})V zDpY$f_U})A(hy662||635Z}T&SAF5##F#}C=bPL!72`(l*)|&W`axw zXP#>gbjJ7(J{YDYdT9X`7-g^nH4bCqgkuNfN9ZSu22VL8f1Kh?yK?ctx=ABLHOH4h zJprBr%kL45kJYFTw4KTaM^8ShEgoa05GU)H%2tSs>)GbTBj<6E4N+mnc>Kp7X@M9o zs?ygd#cF&=W)m%mail0;Y1OA7OO}BNUzJ?FX5~3Q;T?7YpC7<~`HtB*HNn!rrk)Um zV6Kmk03oaeA^PNPe^jgx6?4e7X--7*d{K75>W=xMC7N7Xqm8!t zqF3JcMH{W)9!T!x2O2H&Me~6*TKjwvI5@s(zNnfP(=cE3&uh`ZubUsxR6K6dM42vd z#db{>^?IBTU3!I%`(C3PUYZ@z4bw%ZV_x-iG2ocjI$e}Ii-Z_;l2)4ShY?}6EM5)m@potcn(F@XQYek#`j>vj|YDZ)pNTVaN7Np$~Sp(AR zh^z(~aYR;uR5>FnLF$|?AsU>n3f#9jBg;X`-Hloq=s;KsDQ*gPHGs5)uZH>vgE(ao z?G7P2WOLxmLcKP|DHh}h@$H27ezVZUx zH(ns!yD#XlLn3QYCp>jzxhN7nSq7m7ud5ZLKijAUsmUqV20upRbDNQ)Js0+B#B9lH zf(-`?<`JkZlydcjMh#GXQFJv(Yf%L09NMv{1E{7L#;Yrq!88;vYKK#IvE&4N;FS9$ zr_v`m&E(XooJM@*dM|vJ?TjLIO}ku8Kvku#Mv%JFNOCOO?WzPC*xjWOv-P4@0~rv| zq|WkyhGNvps6C)9LrJ<=i>-psri(RMBH9@bGp^XHZ6W^9Ud1oJMEP&NMBxs`BaG`W z)m-K9-*~Ck2qX1gro8oCrlBH^FbZ5wsQPlntGQf@L~*)YMY2hvHbxy1^)TwYT#JCI z8I6Eac2t#XJsR1d@k-4_xz_z_IFK2dexr5UQ0cGMj0(WstF?$82vlAJK1FiiTCLYc zD(Y)A*8phKOxn+@(_9b{E;9i&2k61B{05 zB8yf0Nt0RM^(QTw^bGuoBu4(k@(o(AMyY7MTZ`@mt++>P)+qduMlGTfwCY|Jk1kLu zEq(VX9JpVlqXv}H(agAu@xc9BwN%vnfEL#WMS%ykc@#p;gIYo*XxU$sWo!SU6tob9 zdpF}D=2tc=egjeP+Zp$QQqGJtYiSgL-iH)F@UXU-@W{i8U;T)3uYW|#BLCGbT0}Q! zYm1iB3R?B3W^{qpJ*I6T{k@NCMh9qFtG0;vjjfu|4%*VHMG@`>ZUgOU)zBV{v}zGj z%M+T>0@nu6X3**~xFKJ~cVC|h+LNj3HD_U#|Xwxei>U|4-DVqjffu}Aw z55B6TTe`Gi2l;P)OEYSrwyj5tt_SYvQ6B2v);17-_-)Oo0l(~BExH=C=3T|_dRGhR zGQjt=)f7bUdzy=I)%%*O3LYEZ*D9rljt{t~e4y27WrPR% zwb)AF&H>Hn9?_RMDX+n_YuSxP4Te3)7BLy z+TB%D_rl5vbWAVTdlrkptVMx1Q9mnHh@n{q+=pk)Lnk?#ZmVbGwn^Uh&yE;aEZU+{ z%Hl-jTo*Ctx`r2v26^8!*EO_Qw9EU>xvs&*qCdJ-8(Az`<^?FM)&)ivq!&=t?~5VT z{W0-quMVZ#nnQ8hA#W=WGuju6n!_SG7mMM;HuOQyLKp5w7E*}iu~FR!F}6hm7&+Vk zs5o*_|6&T_HX*yn7>^>Xl?6qaJ~X0u?!UVqJW0P3G=RUcV^t?plSF8XRiAA-4$c z#TR%t;rXBUJbaLLwHKRZa*ObsF{!YKwp#d-aNH+vyGZgD7lR?vR>}BbnC~qXJOT8r z#V2|4b|;BrGfQze9gbJ0Pl*aF>1E@?wUlUtQS6t=&hpCVv=&JiXD;l=Ru)iSsw1JZ zHmxg>&uUf_7ia7?&84Q;%CC>ph?8Ej&MxtmctzSu8E1SEqr}UZ;}LKRQM>6> z)L}B|o%h?B=`o?=nKAUJJn)Q0QQZbph-ra5j9q2?7&IDlgug<4#Ss~vBmk*+&_Z5T ziu{@=@*Ixqx6(&{bZoj9CCUKExbs?%o`u0n}QQt=M9z1&jLzK?N2D#AgJ`)bqImK4a6VK(0g z+;pPg&qAD~@(GT#lIh2ZZK^tlh^OXmEk-#b=}poA!FX^FfgsSIC2&QPEe*vmn9t~i zNZ8C3q;6DjliX+`3y)CMV1=K}HU^>=lx$VKRT6tdp*z!yCWPBq)i{o}jq;A;;1{%$ zBOeVC_VujGC_S&RcsoiXA|~sW`t%v<+i1h+g9LpK4KAvXnGT_;N`fZ@#}N;4;h&2lY&58M1a#$xI^8A)fe_nHi?-$P*Pb@Pl}pVrlNh z;z8Mj^I1w--Ye@TVGokbY;ec8j42nyRE;ueSx+Mtsbd57VigzS+&CN(w=N^IxKP<8 zm?hGaM4XW-4Z&R9gl3*JhB}NDR;{BqOI9A~S2Zd>!9rpYm9f)(>(F}VqMu!#typVH zy_qG{S)7-fMNJ#@TQpQ5c9xAW2gC1CS_nxlc6niFhLXN=H}VLZ{d>bj@-)oKpsWIy z@s=CE3dqWOQN2glI`;^%r(Db$tPr+EzY(J7 z24U;FL1@cv60;lYgw}RD{NF8{MU6rmX%x2R`=GB`Xb}$yo4-XkYaYYu+vCEvx)tHJ z30ra-;`g-BTb_pO)510c8vl&2H?|9{xn0=$K?gx&o)g-l=Y(xDXgX*mXe+4ed0{W_ z5L#u2u(g5?bO=551)*(zLDVP_Go(YFSV#G0ZnTPX!gE~u?}*nrj1;x+5MMkdc$R!Hhh`pthquHX;*2sk*l;h z!&hsr)!%A*|F@bw?K@4c_)c?H30$VQJE_%Xs zTT6s4;%4a1Ws$llnx*TNvvg4trQ0?fpxf)`>b9t8-MIxy_RQ0@s(E@?#6sPc6|38G z4%Z!v7U^2fB3(qs>Bh3-L6_?G{-wGWk)S)oNxCf|QP+|ab&+ssY@T~w9m&b(c^2wbe|Ef?z|>N4G4dztQNzEXFO zT&;`LYjnNg8r>MaUe{dJx@}ptKCS6SU2DHl*RyWY?a4RmTHejNUIW^7GtyYA&kQu` zTFkvj)4h6F*~5Bx!Xvsd)S^djY16|ap4M#*&*p4F#yJg3`oP{vbV(2dki-CgsV ze!!kL^%-e@)AioJ=@F4Vy504*F3R3Rc<<@XstYXX!TKa@g!KPMhB0w28(8Y(~s{o2_%cO&1GnMqr^$D_dwYO5NPgK?HZfW^Lv}axY4F1 z-e@zTZn25>hirQC!|?Z{&DHZPD*1ahZOeN$V`RXl8K2sWysvDosPAn?zizk(?S}A2 z7_Rb2L#vB4?C~=VW9SG&3mj=UN*5Wu5tHA7ki&W8gpDa1E?B^gU|~ zdtj|0TGkoP+VzHG!`X&#r5TQzpQCBYF!c5e!`+l+h`dt6m3p}`J?;uaL|kRq7F8Nz zbB*DOxfA~WY`9$aLFYq;W6L9kUh{}yM805%lvfPB_7y|y88Wo8Aw#q+wClqQ?IQXJ zyAgM?U9|7D>*5l-h`ZHp%)8AlO8;axdhWA}t_SV9>o0au|FGTI&}w(pwAl^UU+tp% z1-m=?O}oC~O}p6hzFqHn-!6JTvO5|-wu^>O?fSr{cK3!a?0VT3cBAb}yFT=#-6;RY zt~GpP7g3r+PtnlcIY1o_u_)Z3ZwYs}n`b(7W0u3c`Cy0Me6Yjaf0#p0Sm+S0Bfwki z(1#a0+`UU2dhAk%k$#FJtm_X`EoaqoP*$yo> z$1!(at|MAp=+GiAbQs1(jswdsau`iFIczaEI}WOda@s2AIQ6UpP?+XA_0|KOVr0J4 zDGqarJPNVGiPN(Ze zXIRe%PJ7fxPQB|Rr#1}Q*$=sooqF3R&ah^u%NFHwiKsa)WA$8@Saz^Wt3TLf1Y%rT zU5raK#kw@t;o!x)Y>n|QV^M-DEOD93=w0m+8_scQo6m8HwDVosmh)XkRyyvpUAFdY zm*^{W**6!t^o}BzXe@E*i921Q3>(-Nxb=y0RbK4^0-OiqPw{7+DZfEZjw@6v-wxzCdi{W$J z#-h#8o9}it`P_$8l)C4~1l(fYUiX5*a<{Ga3b&)@H}073Dz_M{b~{FHaBG`ybUT*a zDSn*jBD-JIj(nByMFJ!A?(KRnh1N!H2dk(?Ptue zpBZUSooQb;%YN2u`+7u>{`M74XT*0hoGJ`fupgA;p&w%QJBLjHgDzWO;A_J<+ zj0|9IE#1|OJu(o+7#nJ&djwNV? z;{%vtT!FvHfO={i890#fK}-*3N^2M7?+~W*nPTk$f02P0riU^;jOjwAu}lwVisb_Q zMFx&!x`^pfOpj)|m}wl-W0)Sx^f;#ROpj-}gy~YI2~1C5dLq-4m?knknQ0Q!Q0!Zd~H=}gaHdL~mE7m>ZzF+GbZmRs-_8K5yA z(X*LuWV(rI8dKWGNB+)bdLGlwOtD0RzsSJ(On=673)7!7y@08QX*$ykrdye2GR~UT+ z)wDWfM2d2z>3AWR;`6s(jin9il%D4#Mf~~iLw2{FrT8(Dj`B|(htHQd-4x$1 zE&Y-Huao{GMUm7T6KGcrpE&Ou$_Feca){&kjngQ-F+#OpSQ8v4{}uC9J2#O(ZMu?I z?bk&9(ag8fe;V^u`#4cPbB5Ay75+Q>;r}#p-|}6|w+jD)nM&S@-@hMz@hl~;`i+U{ z`-u6fpP0xmn62bhzci8m2J=;aG?BkPO37RCpJl%4hbGFOKWE?auk43^#sNy+O8?u; zkFhA9|1(#~tNw6e{N_a~zUn6@@*iQo>K`Zaa}QMVR^|75<{xSizq*6=EuVC-;;Zq% z#PAFZ>^Rs41zspQr8Y-0GU7Ad|O4^HHN&itb-%J2PnN0i!sHSU@ypK!F|tMS!D z{!`4iN?+z;B_CrEzB^9w)wpt^{`Z(~mH#&#qvX{%c%ppDv5Ie%KW{SMD*n5VQ}SwD zJyHL$ILD9dXC?n4^R3EP#qmnsD*sPdqWDKy*zZf`Tjl?)OZP3GnV|Sq@-t4@cl?@} zZ)N{&C+<7^MJMe${Pia*zE%JDI`ggU`+Cy8iEE?K7dR^>PTRK>R{Uym|h z&BrE|pB>AUyqa%K)pr)Ma6tMRojb>IB&nQvwPEk#Oxx<&glBzQxWKIAaps{UTde5>)*9n4qjSrhZ;N#-Qu7DD$oC_d4^f z?DsMAtUReY=Zd%aw*j={>a`YV!Tbp6(Ih?2LeAG?`v)qh_;U&$YB zp?~*r`{vg%-zxmv#{-^e%KW(WB z{|pQHS58v=B^G>pqT*YvA2cmj{1^-Q*O+frzlZiCA4pO9t;)wQ&)7Hr56rh(UwwQ( z@_*lt{6%M~@U8m4wDpQ_wy&-2gd;^i)5PZ#pUjtxYpi>uSoAyPKZR)y({iTuOxv0E zGj&~~4vcQxDrYtaJfhh}2 zSzyWnQx=%Az?221EHGt(DGN+lV9Ekh7MQZYlm(_NFlB)$3rtyH$^ugsn6kkCk1cR1 z9>migdUGGWX^JNK{;}FMWxDN)#(-(yO6UC}o3hpO=@$x#5?8&VoW=a1EX99``8M!r zFIo=sXEFbsar`{yAIbU^KVF{sNAk4ai{g>AP3b?2^<21J;qSeQu3-MVjCU~qIq5%M z(0Ti$&$P=^(jJjQ*(FkJ_?&jm#NbkYDXpY8hLI4R5+@5Q`#$|&*q8S57jgRca{B+p z_Mv?flx{!s&#>T^GXFoEp7HkI!~AHLAI}dk|5)b#dtCmMG2hR8XOXgtnZM=CKb85D z%-_!Lw=zy-`2mSX^E(k?)8AWc zAC*s$BIdKP`Cb3S*!-@1lCX7tzx`k2H`%O$?d9e4&trR198p9?CG*c`{`m4xHEDi~ ztuGYLc>7(<_WUK=YZv2t$Jxu=hhf$al}_v<8)vWTr^eb#w5j}Yv;I#mRM^S+G{STm zr|b7j>zO{vw4JG0K0adn1yi#eP3QU^#q>y~W;&0w0x)W`Hh zwtG3_KQeuS>HAFknTB)zBs0xt`71abKWF!HrZ+Ktmg)CQW4RnHWxA5-7N%x9ZMMf| zxiib-RqU^U=@U#}VfrCcvpgTf?LZvU)0qC0X%^F8F)e3$3)2UfzR2`brXx&$$99;W zuk!scrg2P9X1a#yc})FG|A*-nOlz6m&9s&2CrrO(dVGNj?;NI?OfP0y&h!?h&ok{| z`We&LIsXjVjz@|EnI6sbRHo-Ky^!gLrX1VhWsaBG&Z_n`Ap212lc}AE!9{&)1g=mw z>SN=_@qEDW6)1*R-8Wq~OR zOj%&c0#g>4vcQxDrYtaJfhh}2SzyWnQx=%Az?221EHGt(DGN+lV9Ekh7MQZYlm(_N z@c$hP#2_j;xRUV_`A$%J2hIGmn>V2J4j;XzM}P9Y-(-gn0ea6({gGVIUA~`Zxs~6WWt|(1tiMTGn<;Qg;t{ZUOj;je*8?N_o4dR-w3+)VC zJ8{+GYQ*&zt`~8=gKGfS*SI2VLOTf8B3uc$R^r--D;-w>u3zE$Ev}n!HRJjlu8(nj zjVsK6EUx9aQgK~?>q1=RxUR$XB(8p3N7#jSF0KHs+i^XEs}GlOAY5E&xQcN73Rfkr zyKp^>s{>aru0dSFiS*-&#g%|-HLi1UU5M*4Ts62FokDNLHH<64C2U)9-HhuUTnTPr zD{~8@3zrBJ2ENZ}$2SD+4Y*#!CBlV00Tnh? z^fX~#hHDG1J=28oBCc7}h5amCeq47=7g{&IUg+_x_GV|4nq78Z=TGY?LPl@PgX`T806qBE-oyl zn_ORZZb7c!n-vPO3FK$^wihEXdGF27EZR+K2>B>10-o%Q+`O?a+e$Kur7oX83xO07 z*I$@dxXX*+GW_1sTtCsmT;bXJ3vY3u@E|FOHYi2jv(4-G=Tf8reIQx6zM>3&W{ziD zabZc32Wf?p3C`XEKj|OmInSH1bE4l}85D(V)}NhO;71a&*&x~3c_qFaV8p`*Qds1L zy|Rmna|`^iN_J6+bVe+Y$n0V-)Mxv=UYJD^BqtlzA^m=D9_b}QuDgn1WI@UE7VMO? zC}W$~m-`Di=4T*%ewf?i^X6rH$U&JTA7;;o35gc%mel8ka$ryo41;7B9DgEQRLdkEoj6ZT?A=ikDsf>U6MsMhDw=dJ5w{>?x29;=;!?^|7C=MkBX3!iNN#G+cS&CTq`6?Tk}xkQ7E@f;F`|s(o$DR!*1H` z^Lz8T^;qrQnVX5Yj8REm3-gN*)MinOwxju300xMe{M!ziFXo}Xq|Giyak!NH+lV*`q03ka*U7gS)14fD~gLVPe zA;?^vE8~$-3?Cj(;noW?Jw@JPUm?n4u79^@XS`63vSfHvk(a(QN{}!wS`VLhLt$aw zhC<}eqzu_qjOT3h`b&xnWU5)rTYzS`ID?Er@g~ZgJXD(VycydelL51$NJ?$_zHOVl z1>?}B62yK9`zS2(dkVa}LQt{SSA>e_ODo)jsAtNok;*tF>=k_lLK*{Bj3(F0J!@j3 zd}JdOwn8d{G9c+fsbP$^DL=PBHrJAmLZ6ZCU6q%Sn;#P7Sk)fSsMK#VWHpWFU~09 z!Iek6uo}09=)d|n{5JpYI3e^krN|S18dXQ(PR-cuEe@7yF~bn8@GA7P6U3Y}Zvh%2 zkzj~Q2pXbyf_Ta}@v{t3ZHlL{xKHS2B35roSck046G@>-3%c%B2NjKU=?JXsY5MJd z32yqw&M#>}J(F(k!evd-vvJ1pl8v2{;+`K%D6iN!Vi|?VU`I7^nt9oPxetY`x zQANEtADu~IalU7lH+Nf(-vj+*sK4rb^Rh1VrKaoCx<|aB8=`M7MoFUWHd5A5KL@`9 z6?25J<)B;hnVx@V%g-%ETfbEfAI7k%ZN(Y8CS%-eGeqDyQdsv#`t80u3cvOB_&lVZ z;5pOd%R}pF2JmlNqIvrbEluReci$uaY5NH~dAW#Mg$36|??qBzgqK#yUn@o*m+1V~4ojj(F9Q znQEV+pv&GPRmB+v$TE)C<95Dz8h!`fp;VVcVw|rJOkbbag^fmrFsw4~7{fkfC&MZy z@mD(yg>o~;cobhCn2{%uTaGMmW?>eJPMr;dRWD_gWrqF3R)BGCMu{&Y&ts8wpW9G? zF+B5#KiSQKCw}d~$OoedYt0=Fl18mXiLf*eR3$ncf^xN?3Stv2GA4!0(BGg$k+%H{Z zYAu+Mgr~ z^|#Y+!*3$-TT?~_WBX24UzCWnq1Q7F^I3Rt;jUnNGZYRL71M;UXJRk~kzF24tuWU3 zMNpt8JR3To4X#xcY0v2Y&od%dUgaGr`N-GCZJYC2%A?_h36Kdn87Tl6c`tJX$*;pK_PXBrq<4I zpZNVWRMQbEzV$uy+x#&OX44Zr@usJ*rcoM9^!>EZM(y za-4#Pr-v$NogS*7Y5ETF3|u_o*6GNv_o1uqM$*z$?bbXQzTE|xA}|AmWh*L-zi>Oo zv@+(iQPseSv&?m{m)UY42reQP=%#f zybkRdz8uOdR4p`@AuVtIJDO6ecxlI+!3`_ZMZvDI|%OkXAn zr*CX!E}vcw{EG0pbbra2eIYijhM#4~m@Vfm7Y3Br|gLVQ8qm z;+06Ve2U-B+@6(zfx^uE41az}o{Ic?2wP4@c>s`;DL$V$w)*`WN}JLBpgjDY($U>P zHjuVO7OGzV`B{9^ISZzGk33-X2UEX3s|4c+W_|%?`-jocG5iJn)&}r(UmN!h?B}<$ z`KD_2PE^5Q2sg||f$o*EFOwZP+}XQ}s30lSIh(rBVkEM5wqLvsiE8*oTk+Va#K;(l zy-|qEqvWUN-cdibQK4FIk3wpTF!N9=8K|A2JjyQ`;LQE^XS2;7u`LRn3nJsC0$qge z0ux@uXKWJNW@U^2%rd)9@fPAx^8{J8`av>WwzM24scsJ6+%?A#gP)VH+K^bzE{F0`lY+q67ROx|kafkAR#$ZsbxQjfuSYkeY6JwtBba7K{#NXo71>mt&2$zmf8y{0_pa} zF@wlw4pMnA39I8E;U1lXh`}g1ouNirPWC*aagLmEj1H#cP*=P@$4ra(;{n^n#|P}# zF20(J)yIbq#ER8#4?-JJ;tiHM@ybC$KVhjyd@~akJQ!v7C2F;+pFR|^&dIT(ZxQPo_O&tuMQ;8a2w#(OxZf_aado}PxM+B&dpq+I-I9&7ol>7frE z%D#z#mO=c6j;Ag?gq8jJ5LkFG1wp-)xcU&KL(Z>4`mcI;>-uO0yh#p5-5zoGAyhs` zZ4l!9CDQsb1zhtP{kHc}@KTh+Et}6bSI$T3a|^cT6cz`&uI~;}Q#&-USdR9H%J~== zyiba%p$L`aEvQ5DU4&&5uLrGGa8U8Wd=-CI{;CEml!#!<4AOXTmB%1)(D~0i?Q}AF<6QH z7_LL_%kg^;{g%Ho6F%f{cXWtx*`cagqWKYPt2h+Cf}_?DM&M9s2jYp}9g3Fa5wcJZ zhE(Xt{*A{CvOVDO{+KcE`-igbe?Tm76$y0pK)_a%k&Eble(2=GAr`7SjBjo@jH(0` zK`N-Vhb>)V#`zU+sy?F1Qr=G`c{G6ortm-ErR;r-v}vAg7KGu$WDHpURSO~iFs%~w zJwzV!F$2iMguolL&3y~mYcssI-6uU26?w#C3+15RTT0z7dwCmP8f&IQTisg}v+iHh zIHnD=c`8t&Q-F$C7WiE(tl3S$H@zeyQ3d@@vwMGkA=NHgZ(;9&*wKWr(`B*S)nbFF zjtx!Ny|FSt?o8P8U(9eHMYs(z#D;4qijC?Xh7bmao{vOMw8ou>b8xKjp2H^z=7GbR z`^e#V5QgOtaqU9cYd#e#hw|cy!%^Nu_hCCkhVJ&0D~0)Z&(-(A3{Wl@s&EJB~u{x__OY zL#F8t2jUU;a5ka;Yvt%&b`=(Hr|A?Ptzhv057z#8v|PjYhzE`?!ec`ZDw+5mdTZV# zy;b-ftM{*qS?|k>QM-r8Qwx5_dKxr6eYqGzD=cQC;JmSzS1-i~BqIFD?o)i@u&%nY5)r*5`7}v%XirQmQJY-QB zz`73#@|Xo2@#3*X{FDigmwhz#Dcp`VrY=bJ-$(_l|Iaj$r8Yv89f#FWSSvSsx2%NX z%HuGsxSf`p@_6?SBoq=kEb-QHBy?pw+ItzfA^BT}D4_7jh-@v%+J=X= ze~#x0y>+)AkHhYVL_?EJqsTFXhp-RDV`zY}l3YatTig+^7Svvi7k0A?`P;GE@-UWa zJWe&`qADIWo*rw7TjJ3&L3xcFvvxm0rDwR4e#@SuA~3e+XgQv<{mJ8zg8Qi>pV&D( zc)YBAqTTfTK0H@Iv#@0q?kb>qg(T2$EVLjs9&Z*e->%$kc_oS|{&9RhrsJVb9D~Xw z=vZ%*i=_^5t?UPQ3+)5B>y4!srZm4QYs*ycs#+Jvxt`ygng@c zX=$doGNGUlHPq)VmP?oFxxM&v0=h=Dao7+6W=^5MfL62ahXOL&8&F;KQaB~Fmwwy3 z=(p^ReM$C#bM?iMA8akjgM5hcpeOz5Yhk>orSn_Y3RIATy`Sr(mS-;T6QD4C?}#WE;~L)e6mFD4H%s) znk(OO34FJ>dMWj-x#+EBFd4q!LzU~7`o$kkRHcgZ_puYPCYXb#k<{?FA;c~eC9L~h zj}<>IKBDm?>Yd2BLjB%N_ceFHiyRXa(juRD^(0JgpcNZEUN{MP^(LgM>d0pgHUSBp zo@l|l`1T~){&7hn>UlkJdu4z{^~-5x9`W#XyjP4FOdOL5Vr+;cL< zjn|Uyvg;_lv@(!~jS}cCP<->zx}2=mmSnqaPI(G;Z4>{3n z=OmOOaeopjMIBYuc1$<~CVfX7Sa@`K8%K2_DrZ z(=tU~iC^526gmhK4c8!E_d{lFQ?9&_G;0%g1vlsYVJ1WAbyJte#25#$6iuZ{|RM9 zx%}<_nC^$JB=-^AvnFq8W)3akh!2)SO%QzoO;wP?pq~a0@-dxUD-iE4-zvUX&eIE? zpHq(}u09RjOF#yTz>xx^@cPqoO8jIl(RNyi4_m#m%)Z!%0TQjvi|0=hj;-R3<+Rfc zCG6Sdp=ohkT_!-MV@1*8ZMDZS#&%4LL)^wD`t6s$l~uH>APec2&)me-D-o#^mWu0FsvIQ!-JCb}w*xM?Lu4*c}@Un}HRgiBWziTkl>v=swbOjXTCc;Zr>S{?CuGVe3! z#2&|O^bpUFvCc;jD9gr2sT#5ni9C#`MYY=WByL(|6P;eLTDKvtosVni2E6&Ac@

=KZT^x+Wfl*LE6SS3ITRx9JH=yHxf*EBnVPB>hQZR=z;LHQ%zj%T}ZOc92th z8=Q9K`mvFZ8t~t*76zVysBi+SWjGb9eY79(537YkZdekRBThLeao$YXk)vzCu=HLn z%a?#5^~wv2yxTnD`BnL}U+0}wMHncG`)CUa{k0zC8d*N1&+D;`*y^-|7H#n00i%0f z{<_@_MYa$!M5|>r#?fTI4Dx=#`?{2VF zXcc;+RuOy*g9U<;?iRC z63Y$4HKNxU5#?ID-mCR$O}1r~Mz1z#RO(GeOoh>8>(=vzZJkC;lrbDpsi&0bt=Lr? z5!L9d3+u6UYOzE19&K|(r@akZBqAEMLA{I=*@kTG$5q%%EA3^*h*9Mz(;DmnN4Kln z)@~cN!zzwt-L49)+SN9_Q%~=3MJ5b8ToJumi`JvX#UoyI3!=-l_@-HnTC1&pdZSir zgKnb+JC6g7&ahr*w^o5&(ZkwE*y;{@V(;uS`%qYm)72Z{s?zG5)p~dA@~{p^t*hUbJakxd*yi4oM#5cH2z$Ui;HWbi9leeLW5m@r zKcUvBI-$9k;Tk}O zm4&y2CpJWFu5fkQN9@Ucj)c~*_$qg7e^^4hJF(ZD&=Zl`H7mVwdTN)wF+90{+Uf>Z zrE4gmS!>XmrVZFew4!FcN9%L+P_(r^En@i4fEIAoEN*diyYdDGkowXsHCl4FmXg(^ zrL=^VIa-mT3R{=fW@`znwAE`vMxC?U-R7=zcew}LEz>K*D%`cMK}U)fnB{&`Dd$?2XQPXPu+b2pGe*yl!`F zr7JeR%iSN5H|(g4te;sIULD>P<{ELk$|DBD{S{%AF2BFeU5?trx@6qaAIF6Y@QWdl)AktMDU?(-^`Y%dli1x^xvfL-xzQCi^3n6{1%13WZd|u!tN7PdX_z=@IuBZk1M>4@i61_81H#n z@wYQheMaG5F>ZQJ;Y!AVPK6&}Jo1XdFETb>RrnLe35<;sIev@}WxSg4$&6DOpTjtf zaX#ZMjIUsv#rRIfMU0abm$2f)YCdTb=D*hhE zos2P`ll4pdP4Qo4>|(4XDSjm5BN<0CKAUk2cj6D_vb16Amgech5ybt z`U-`er>O8E=v^YnpTIc4_@|6xuTuN~W+{^u5fusz%{Y885p@ z;V^u)i`tvc1Mn^q{2$49-hm3A#(0GB1&s6NDgG}RH!e`Pig737#~D{HRQ$g)t~pBK znWw7oQ{xq0%y{$33a@3nXSu?ejFVR@yod3;H46Wpart_MpJ$wNw!;5p9J^iNnafr9 zjrj`4F>WqW_zcFWr3z;v zCJdAx$1?6^d=BH`>lJ?o<7IzPxSDYVy>kZnzcB8E=@w=RHy-VQ-8K=xv_&vtyQ3^+(AEx>iF-PHJ7}wD|fY6u5xM#k? z`Ha&SD|`v#RYgPEoCn)|Z#(u0kl7ByAT8SilE8}HY zOeEaNxQ_9Mj00HTB7XQ!l>S!6Cozs#tN3}0N6uIHH;mhUrf@6c-Yp8h&$#jeg}-B* zo~!WU6ct|2g$k!KuKOQ_w=*8OSm7%f_g$j!ZHz0b6n=_v-gOFp!8pEJ;e$_C;ic9n z{1e7KjD3uAZc_YP7^l}N+|D@Vj|zXpIQ2G#-DfEMDUU0BEaR#t6+Vk`(=!V1WZcuP z@Xd^S-&MGsu`!F@Hx=S@#<7PeJnKvq-sYtWpUHSQMd7_BpWa`E{3DD-vBKXn9=KlN zBU6>Ws#=B5WnB58!vDiK?i+>c7$+F?j;j!_F-|;L;c4rXzNk!vmoe^Oyp3^bmf}}1 zj@z#A-HfC26@Hs>RDr_cETzAU@nXg;g^GU;<7#?87xDWQ;}M_2_cHGGEBrF!tP+L4 zX1rym!b{ex@KSdvyoGUFg~GpJ+;J%0=S6teG44D};YS&#EL8X%#&wKG7$?Un{t+8g zcmc+zGv0i-;{SqiE#q4l8%HSqi;SBYf6G`LsrYecEB(vj6kg9b`8b7t#W+CkEF(RC zWZZF*!p|_?e6qq{F^)@8coF7F)LsvsqVPJ#aVr!qX56w;;cFOYtx~v&arYX9dl~OJ zUE%OeN`Kl}3ZKL{dXvH#jO))+_}7g0Y*F~njB_#+?qyt;qi}ed(%+P?@JWo5ixtjb zoW4`x%Ncj>R`_v0@OH+1w<`Q=#%;GN`~c&5 z|5ErLjMIh`jyg|;SHt)O#!>%P{Edtg85b~){8I6M%Qz12$df(pV{Ck-@P~|tqg;|d zW3vh`ZH~gnFz%kK@Mgy42P=FT4>iGv54w!Vj4IzbM?rxDI=xNPd{{yhjyY_%juL?c)lk zFdlea;R_h&yrJ;LjPu@9_y)$!j2~wl@wVa*FfM02e~SvwkNsH`pEDQ_;h{a@QpS7w z6uyOV`bP@Cz_{}t3J)``9#r_~pL70xt?=25o4!-HgmHx7mf_vZxY42Tvy7KH75<8G zT$sW~U7++Yi%@tY<2^GJ-pROiroy+H{^u(EGUKRdg|U%X&PV1kK8A5D<5Xa3pW_%8 zGLAe%$=}U5XQ{$}W1NcfD9D~)Gd50C_|SBvFE&x(4UG3N&S9K*vf}SyyeLWGdd4k` zUuT?lisCQGQ2GOmS1=aK6n`7zh*K5*9pep*TNuw#R>s>H z_cH!9WAUB}zm;(e<2M*5Gya%yI^z@aIDE#ZGp=R4mvJlO+ZgvUev+|xUxoh`;~2(Y zF-~T@G@tFyIEQh7@g5fzJ_rt<2J^Q#!2N)mCxR&v9#;uIcW!%fSn6cekyHy9@~{x{=v#&b(J{fv)eT+8?j#;uIA822(RV=O*V;orkJhH*RNWX8RW z(;3@$a`=o7Wn9a61>;u6>5O|B2N;V175x1pH+-f1H!?0~+{d_<@qrg1Jf{$S=ZDMj$uW%cex~pW z#*teTK8JDjR)upJH!}Vu@ciik5x7M0A7ot1_)GFX2Lqv2#XoqD!YPc`GVWxY!#Mg0 z<^KxC>5Ly>T+jF;#+6Sh|B=5``dv>coXB_s<86$q8P_ltZOVT;<3z^)0-irti1c>F zKj2piXE9D|*6#bSVFe z2p^1dC>fu|cpl?C#xabqW4ws*qlD)Pk<9od#-c~*f6K%*5wbjgY~pz{B%ZNX$%~Ol ziH|Vxo-~P9Gajy2c&mw@QTQUprEe=-W8$!Lr2IoBo~7_>jCV8smhqk|6@S4cDn5Y= z&y(`YO?>pvB;Le0IZ5Fh#yO`e{7VyWR`_=&-lp&!j0blrtO+5q=!gGWz{iv?`1j5@ z{2$oD?Gdj^h$eDXBjU$9t+*bq#rV{y62GyVvlHI7g6DJ=gy%bVDg-?Ko9 z2jxTgBya8yI=)PWZ(>s($CJuGA{2fopXAN`M6|z%;%{Q|Pw__)lYfLsK9o=L=Kdqv zk3{k&CjTVgLpS&%Lh_+}k~jA&(f%cpH!=Aq`4PIo9}$u_`PAPW5z^lfKa2g6|K+ep z7TxgQ`QvalFy-f|caP1_xFssAYR(T6lj>Mpb*zst#V3?c>FH$o+gRSjrhLLU`A|N| z8?XcUtzdZ*oANE=0F#ZUWd?=sf&HZk)|Bd8LO#Vqe*@hqd5hnRiKFN1_6tB_Df5xVK zpOhYzH~kZzWrrEK)?&Ba(rDqQU_^Af)Ip=bSkgIXBMC;i6cmRH#^}sHCW9sA!~^n3SlLn0UuT z#U#T^N`0uzOOj%JpZ9v6_slbEW)AfG{qg;79nQSxd0u<1z4qE`uf6uV#QD$L!Ot^( zo$=J<k{xR@9p4MU9UX)yN3L0ntUcoKl;NCzW+_i_qFh;$;a{ccJTGa4>6vad{3*8Kvy__ zXz+9H(D3(_?}$G&>`4E9a?iyE4c^9$xn>RPj3g${?_P! z4W63((oPZy*`N`BZx7+i4FWdw*AC&ECcg|{WWyiuotzy!`*WjzH+X9Jme>wV_gQtc+_&L$_ z{gQU0SH z?LDZE`}Q7a-`)f5+k2pWdk?g4?}7I1Ji-*`)#GnHLXV%C$N#SF9~$ZJ?MT1X&nbVi@zmtkM7+0ypYy!(7xmWgsmV)s)UZJ# zd~XNO@d+5Ofb&NUf7xZ(+CXa<;JqFE$d}ZXkhTf%)a2vx+uK9<#=mL&Y}4e&;)`tf z6T)|P@aulBw%ZQX@Ttj5wJvPry&XKqV_nI^e0v$NA6O!E-zh#`l1yCLf3I?ch272jhXjQ$KiWB z_;to({1bR;@^Sdy4t|627%v5$x)ks2;5QkM@m1idOYz?R|8K?KF)vM9*+MHdi`L0TR48xrPoqvcLz^R zK7Ri6cJTGa?;+25;Hk+Y4X{C1M0W7=jDO-Q%2Sh%>Y5b+zyyjoi#Qu$d4(HTgAq_}Sz4cJQ;NDgWMAm8T}ZXkXm{bcOSW2G8-t7+;Lz zr-r@Q_2=#2IsO>qk-<}w&uH-3_FwY*8-KOsf9g`aw}bzfyoFCqzCXUmhCda~9~wNzcVoOa(vKSU zaQ>q7^LFq(uG92;(&A4|K0bbL58)fX%HrSKH2EG94{Xqgzqf-QZ2s>XPfb29|GXXi z3gcHEsK-xDzC47U3*XzpZ!vzT9X~bsArbHG;QPr90={Rk{4`BIPCxp?4xZ!jF+LyX zkDC0F==ssx!E^jR#`A-xCcgz=WW%3|$PS+4{W1O@JT?50f8zF6ZwH^jI;Pn75z14O z?-j-0+re{w0n9gm@Tti!m3?D_M*O`UJm(|8`~={s;g9&o{Q% z6xoSv&Gy{=6OhbH@KzB8u}zO}=xKf4m+1`n~k{ z?=+s8e5WY>-VT0)@dFOg@Ttki&mZ0nev|PR+WnuJ{49Kt4Sy<}KQ#DBdu#lM7*7p5 z&Ry^5_`Mzc?0uAf!j7Msd|du`JNS9VzivD=`MDAYY|x0mw}bD#uZDkw#h;pd{Qk?^ z!B5&x`S0=gO_Se*z-5C*`1FScKUiLH;rn>wsqq`qZ!9HfY4(+re|b8O%Qeo*Mp$f7j^vy&e3bVXzbzv@U0f2*ZGHTmv=&hg$3 zex31sEdNoH?-cRg4t~Ay3FE2B&)7|O0FC&2J9y4tg!znc{M7I-cKvue_({j;@tgfD?JNWwJl>fxq zZ>Y(~=g-^0&oX}C*EM`<^6~x8+riH@{&fqVx)ks2;1?KwmW5AEK90Y)gI{X=S__|= zeB6HE?ch1z73RM}`csqdE!PD$=!(b=p7Ud2zAW(6@W=U!`%ig$2!Dv4zl&$sPfdQK2A>Vz z+re}GFw7?go|^o`i1&8z8JmCX%Xa+KMV~hj`w!(oF5PK<$|3ZGxvJ-=mZ602=Z4cJQ2!6Z3O|r-nb`KYaIm-rK=*{!Yy237(pK+<(s7!E?S( z%>N0Vn*3x9K70PX9sHKXdj8KlSb1vlqoU*YcJSqQDu0CW)a1KG;d?vyk;Xq|&p*`U z2S&WNgP&#mM+a&AsmafX@~^jp|9Gj!e-Dd4HTk&x(A&XpGd^>uhEGjCuD^IY_|Ere z`1=}9O@32!{N4_p^SffcSDb%p@~a}=+rclnSHu6?;mT8!kL%Cg4u1B}l<#5f$JFHe zMd5op_+`egv*V{G-#_BL9sEY)SN7HTQ$#0+x-uw}a<= zzk`M-Pfb2fe{ToR`GLn7PfcE?+X@>r;_vO?IiK+Nji)9b*WbMzJm()?WjrdYKWh0J_&EOF4xaNNV}4}t)Z`cA zi){FV@Vy;8=TF9b%HXNxYtit%J;Z;#=3mUe9O7@9eB6G_`I%t{&-s}#Uo&`W*pYsV zgpQ`?$dpr1%f7ST^+2T)4epZzJ-VT1vKa@XFG8@9D zCON z9KN@M?`Hfi2Wt4#?}0C} z;Sa+1cJQ2^{%zx_;g9f_$p;&GZwJr$?7I%v@Tti!jPj4SgXjGBm=7QErzT(CDL;H~ z2haKP#~DvezIznDw}b!SFg^dntp6)D`PS(E@9p3@KR@Q{NBpVD$M=742j8uqhJTHP zPfb3qe|bB2t_N_rg-=a>c9eeJ4t{li4L@b!QR;Hlx?TRz+L^k>8OcJM0(Y5Z4O{He(=kFI}j2VXN(`7$Xd5k58fccb+4 zcJL!lReqZtKQ;ND5%29G{4peVY=|@d|l0ASzS2%xY@Wa2O z;r~b4X>t72up|8!N9B*VgMU{Vh~WF0@zmt84{Xo~-`l}=w)H9QG@iN?@9p6G8Go0h zA2oSP{%rW(4xZ~{V7&|+KQ;MTdxq#|d2a{5tWJ-=kA+W7J`Ug8!7n%d662{$@!k%8 zh4F9q(D+l6kH>F&JNQYB8voxIPffmWRDOCp_&HORpJV5bn*7o(`RV8F;JF?N)+a&w zQf1)P8K1x4t z2j6vphJVS|^!TaCuZwtZ2ftC442JKfUdmIGkK13o9X!{6!Fn)Ae`@k`qVm(*!OxQd z0=``iQJ$LoC_Vkz>&M%{Z!mtC#h;pd9Di>I-|2P@f4TA0!T zqw>q!!E-&I%PjuXFE#mzQT)9f{N}$ZKcT0lA2s>0(fRjw@a2C~zT9|f z@{=Oo+rcmRhw|4OPfb29f4v?2BIBnTPfb34e(-ki>p#-)|Gu9dKQ;OKDF1mo_`#nk z|GI@wO+Jplw}T&U{0if#$*+&@f8Gwh$A2_@yk|uIrzYRYDj?7mksW-`oprsYS1kWh zlb;{)-VT1*Zpz{HQU3RK@Z-8Ff4J4(smaI3@9p6Gc2oY}mVVUarFkK2&=t-f z8a&r8!+K^&KWf-<{m1t|ZwJ5PD;oY6-Ib>%-#tn{ZwJ5jYs!Ceg!0tn<*>pAjre;z z_;-!3u=0zV{JeKp;j~f0+zc_qv2fy0Z zhr@bt;Hk-Pm3?4?M)=+ip6khBeL3*d@JINosd!H<;>HfZqP z4xa1PVf{Mr)bIx%-+#OvJlD6o!+2`)^|B9a&U zdG8rdO@4^%0~<8L_jd5B-THe7pBnxMKkmQh?ci5@OY`4tQci%UCLiCwyd6B(_dDS< z<*CW{>MD_t4I1(HcJTeq*6^RS>yMiJ{6Ob;ZwJ51_&$9#d}{KY@I^NKLHOPdes!gW zf0pso@JIR$j@qBR9X!`F#QKIvKWg%8Wgpm}5x%#B=lX|N8&6HXC%(vrKj6I`Jl9KH zZ9Fyn5&z}*A{+jI_jd4HU-2E|so@WPWt4xt9sJ0orr&v;_4ujD_lnAYZwFsrqx@{+ zsmZU2!uNLY{pyrobBKmdO@3g+dpmfpCyDhXasH^u$K#K@9X!{c#CnwAsma$w$M5an zhcs*aJK6n%ntYspyd8Y!7UlaIPfdPHbo|~9{^P02zhn1*YVzYE-rK>ClnEB$d;1}F z{!Npg9UVXYVF$lxp80QSj8TBmB7knYV*q_I(ZS7CU}w@|&XhPrV&H z*Dt-(!l%yi_#zwrApYJCp6i`HVLUbb5&t##A{+jI_jd4H9~JAR;`piI4?b>x^>*-F zPxZjlm8T{jm;c@lp6jokW;`|dxc=kq;JIFF+IVX6asNMW2ha6gZ#SNr{N^bCc{_Nn z2fNmIYVw^T-rK=*{n+=7rzRiwAM{wQZK;Ux>#=)JT>`V zon$AnK_h%`2ha6*u|6+&YWO4kxcu;T@QZByUi5PWPfdP^>;oG#!uNLYT>p2Pg-=aB zvroXYytjiN`v*<`|C95C@Ttj{N9phF;OE+U!`B#3O@5*VpAFyJ!E?Q0tY3`qsmX7W zb{TBY6_FkM>d!U)RTe%q`E61By&e3jope3oM~$Z@-#LVy3*XzpuQC4C9vXjY^6~Sh zw}aoXtA;<#cxv+T^S8HyU$%$x-{JcY)8w~A`Ir8%gI_3%$l&`amj9^9%j=1-L033` zXz*NL8tYBt{87V>{NFn&zq}oMUwJ_cUwMrfcxv+UdNXX$2;bYmPdq^RJ&mU(KRM#P z9sJ6#Du3#p8a_4o#wh;Y4xa02V|{HLKQ;MTQTls3_{E24__td6QpLU$4^bZv%J^C295B&9en5c8vY4({M6*j z0n0|-+rbYpzQN*8O+J4A@9p5%T&v;BGz203)a2v$x84qZ%Z>CLh25_IB_+S1bRh z@zmtEM&Wxq`0i_zzr}cJ@^Sg??chfm|7)q|A^z0l7f0cHJNV5{Y4~?_Q=Xc9{Qkh( z!OwqQ`9E3s)a2vm4{ry*@g?PVk#Y|4rzYPh&jr|^D)+c$_{O){_=mZs$#06ocXsgg(m@Bl-!`6_ ze0==g4t|mGl`Q{Elb;lo-}Hwa{MLgs{65B0lb382Hs}iH4-J0!!OFiU{WMU1Q^St) zKW-2CM>c5i-VT0YU*(q>PfdO@VA;rfJNT`}Ut~Nr`TkM+wYP)s+h4=4FrJ!xeEi-H zK4bj%4$5!C`0^8# ze?<-y=Z~8F<|zMoJNRBhls~}IkDC1OD12`RUt|12tG`f_?-yM^-VT2La1Fne^&ivZ zrF%r!pevj|H2B3Ml^<<9HSEa08R-{*4H~?+gYRCg{9dCq{iw;W40Mk7cJR*`KiGI` z^2;LL+riJM(eUexrzXE5I)B~{zTXt(=NV5;KAwNm+rjschG+QRXFN6eZBhKa9enp@ z<^O0rHTl+v_jd4ej4!Lu^rt3YjxVy|PlfY`2EVFB!~cfy)Uf0HZjHf35Zp3>#_#xAk?`zi|HF+u4!UkR8{Gq{*GJe1qJ^$3O4t^YDL`La&=ytfB_o`yfk;=jf;`9)FxBX0*k`3~jp zvGA$M501k3cJPbtQvP}4smV`_cy9;)@x97d8c$8WQ4c?xe%>D9|1;$WGXAEs{IvY^ zo9yi1pOcBl;d_y#A2s<-_#zwrRLDQx4u0LC%HLo-HT;o(X2=H{d2a_l@-XEeHJ+OM zB*nA5w}W3|{F}y8lV23^-VT09Zw-H&@zmspM7+0ypKW}fv3mVbliv{W-VT1^;Trzw z##57**V18wu5kX);1?Tzk@3{9BmHaSgAE$Iw}bE0N5j9}cxv)10-fW%9sEk;Uof7U zd>nso2VZuChW~}})Z_<8^*3(^Ke(Ur{m#<#rzRh_AA39aAxA4e#&~M-L!$7#9sKNp z%752*YVz^>TW<&d+zHD6)Oc$0^YBGB{HbvM(BLPYsQmAYr-mKpf4zLLL4)^p@cjlU z|8L`|$*;i|+3*Lvw}b!SB<1^_t?5qDbpN}uH;ZF$P*}<-z~cSy&e3DCgr<~)AXk% zKQB6dZwEi5P5BdyrzYPk;=LXG-?*`v{DHTFpEOVTi;bry|88{r-VT0aa{GS=VY4Vvkd}jy0;wI&HJ;$y;)8s#nj-URp zgI{`!@<$p^O@492dpr1%i`65%2Bb-@QZmDaKQi-xM9cw}W4Em-1H`PfdOf zzQ~3@70w?z#DA&s``PouNYk+Qmd|?W7lMtvw}bEbk@62&_|)X%_5*JRKkj4Y|6)8f z`MCb*?cf&~-+h9fe`@ma^M|*C@BWE~f1L5uBhscJTeSD*q$nsmaITdpr0}|5pAN##593IO@OV?cld|?;O(RJ>#j# z$Ioxx4u0I1mEU`!oHTmVy_2cc}=N_p1MB}N+%kbK;L033` zXz=|HR{jUZQ^St)-#g;H9sIaMm0!l=H%-2ObpGiNJNP-hm4DWFYVu_f@9lx_qx@#_ zrpa%`7uoQqLjIZK?BHkhRld`A^!!o7AIBfJUwAwC&Bh3G_rzYQD(hwVTh4Y67 zKhpRba+rjrdLBl`Mcxv+XQTw;IgP-#) zo`2KZ!PgJf@EGFYdqO?cjTzt^BzbJ~jCz z(es11gI_a2`A_%N^rI%fFyg%({G{`hA7tTElOGuI-VT0#mGWOUo|?QRUpD=`9sEM$ zU$*#DlSjWgY|s_X9~%6EIt~9WU9{=U-;5W=y{$`6mHTn4Y%iF z)a2vw1KtjP$sNiMFrJ!x!T34zhaLR3CCZ;=JT>`cQTlm1_@#1#fN!()-=ro#BkI5A z?cjSosQh#bpPGDmRDbh!@Dm?WexdQy99dpIDcsHy`ERT&3J0qasOKz_5bm9@S9&%eu>Q=MNQr=-;hTU zzPE$lV*E%;KWg%`Bo5f1!FxOS`qwo43yh~G-zhr(-VVO;b>%NLo|^oq5PB|rZwJ59 z_?wKUCOmN1z$Bm~ZA0NNBgP&{s7UQYO$L-hN4nFg?hJTPeDC7P^O@21M z$c8@^ksbWU?=!?Z#7+Z;j5sw}W5z z7v-Nfo|^pHsQ%{d;1_gd zKOaQldpr1kdn*5LV1@-W;`|dS<(6P_7MLAl)r=VH%)$nq#-uw3i)TSvxD#6L;07CrzRgi|9Ct2p2qK! z(Db7&#d|yW-o}5^ch{g{ed{=yt4Sy=|kNM*(_^#hlzAmZZQ^St@Gh9B{puu}P__4-cn^c|}{@|aJ4>oA< z-VVO!$r}Eur1I482VXBAY|!An9sC^Q-%Tn{4S(>9@I^NK0q^bLdkxX>_pDN$8vfu% zMfGQI2j6&_rN8mi|F{g=0c-y{py!8c(%HTlI+_}&hFc0&0}ji)9b zuV3Nq;FnKU{x0LG$;bT{yd8Xbwen9IPfb34|Ksi8>r={qU_3SXxc`8+gKw=@{>wFb z{;0{v?+?5k{O~5_2N_RIeq)sXyd8Y6R^_XVrzRgizj-_OOj`LL8Ba}qOBBAhgP%N2 z`3H@sCcm+hWG>mDE1W+x_`w$|zutIi*n7)IhU#D77Qf4m)h&zqIs zYCJXhZPD?2JNO=VDF2mOJ%7~XJ4fMrJNRBplrJ})ntc5HV8vfvCNAdS| z@Y|Lt|4-wo$*;l}+3*M9dpr1*_bGo!ot{5x_#^xoQTv0pgYUmm`3mEy$xHJ_*q{-< zw}bEX8|7ykPfb2fe{Tmr%lLbYrzRiIzv%7Y`#quIzhXQ!`MCYb+rh6e{y)Z3laJdU zydC_K-)Z>0rJ{=VGivhj{m0wE&wg6@GmWPvA3uM1JNO>YSo#}JO@6+fzHI*ScJND$ zf5>=h^0T|myX1#Y`;3dq%KGdo&lCUF z@RysOYx-T&i%nm0vE)@t3;g+Ir2i$l&zHrpx=To_181ooY5G*t>rB_1?(rM-zryrn z(+`3x#_W{>r5{+y~uRg8uedmy4>`m67^p4;U>fJ@-&K} zUNp+z3nSep($_|M=4^@YSLA;6w1$7T>G`G~hW@IQN6)JLNz?00|HbrX(|ca3{$q@mBe7VMF z_JtDye=_yg&kMTJ^pHWS>rL06uKE({iK?%qu2X%t=?|u<-l6lKcLI)d?SBBuQogK6LdGTBmY28HjVrVz0myMgx;x}{Hyo24}-@2RQ0|Rm%ZgP z=ML2aO>e$i^?1|EO;0!7d#T!QFg@@d)tDa#`{VpKcrOXf`R2En=KMNkk^r!Cy(G*} z1kL$}Fkclk=cB^>ThN?e3-duib3Q1{j|I*7uQ1;lH0M*p{B6*jpAGZzL36%6%nu07 z`TsEA9yI6E!~BELoL{g&~%&X2jr?yBqmp!wqv(<|F|3fI>eru!{WjrZ%|`TktG ztq0w&PVEnxUSF#k^C`if^C@-iteWeKVZ9C5xt_*(ra8Ye=5vOf^EG3A0%)#3fb|Zb zxt;;m?}Fy~Tn9?M51Q-6OEYKCeVa8srP(6r@^;minCANIx0qhlrgp4vANFVK-M?*m z;3&0kHN9lCYOH4hf39zW`2nFh|6haYafhq_rKU$6q52Nfn|rH9|03}0KZN;;pgA8A z=0Aew{70A%1)B4r;Qcr>-Ie0%0&G+xd_I^z0-Ezfyl0y8Jz#zY*g5|K=DUFAd={8L1Df+= zU_K7$g?)8=F6Q@u=KLL)FC_Te{2`cs1e)`UU_PdxH|hL3nBNJS^EY9>AZX49g!zY{ zIlmC*RbXwJul`HO0a%heh$M|+=j&H|&6VRMb0`pfub3O~q zhXKv`E-=3sH0KY)d|$!e=KsR_yU<);7xT$LbG{hNPXo>QY0zI0n*9;c{}Gz~AJLx^ zn*BA=zY?1LC(&OMn*A}+e-xViL(!iWn*C+bKNgz(U(w$cn*Cj|{tPtNmqCAOX!e&z z|6XYJ-$j36X!Zw2|7B?QPey-aX!b`&|7vLVpT>G6&|HrM>#sm_eHE;S3C;B`u|8zr zZM{dVR|(DaD6#$`G}l*r#dPO4^!|YP-C^ha?U-*Kn)AtH{&;B450CXspt)WN)`x=T z`cLuo>idN zS$*Ra)02)-d&2a3vtxZ*_;dYQtTzMA^<({-&vl0;NK{fc==O?^{k+|UKQ5Ig68^HSZ@lN z>q%k#Drl}xh4q}Exn2|2hl1w%Pgt)An(Gx|{1!CFXJI}FXwLUgX`1s#V7)-txgH?q zzX<*|9|q>LfaZJ^n4bfh^KW3iLTIi>IMnnOOP?CkjaEOq+VtkDHGR-u4Ltj!J!!g! z@gJFGHoeqzzdz{tUuC+t z#piv~gH8A7rSV~Z@?oaAzTiyLTyJl=X|5l*$u!qf#QKLw53X-G!1RW_H2xDzFR|+b z^JT%G^I_d?n(I?wJrUTsUdUUfd)}w}+xJklcfLpUsiqfMdSpy*v-G~r^t#_``m8g( z>}SgFbeQgs>uZ0*G}pVv`s9cY*C)TiG}n{I`suK9ee^$@=6dZ|A02kCe}1BAuE&n~ zUt#C`t{0h}^Ss9Q$EKg_qVc`c^m=ee_T3M+4I!v8|kn=JhA(SJX+UvGM@T|bYT?%bx=&)-ZhxmW#r^wIdt zw(~K}^a4BHTGIX1*S8mCl1m5Utzkhh5vKYb1Z%@n_gO>{+ms2 z`=Oqn|C*ltJ=HzFq4C*ph3dYh``@H`i0OLMlTCNKQth)$50U2}y~SUuSw=QvKgBy<)QJ_f1bU`#()D`mWkPH$A^r^&b87 z_(xW$-p_PrvmYT^j%yb=w6KkcbSl#GBfT=xn^bq={{9u|?%xdI?M!=r)mi`1k#33n zuU5S)`6ZG4k;wk4YMi>T{Wa1%9Tnn}weO=k8=qq%`|wDQj`X>au8VX!(wC~v9?!za zes827jr7_`zpOgDzxN{hzassgNOw9qKRvocdcQ~?8tEe=JtWfSM7lB3mqz-ENY9V- z!bmTU^xcuZFVYW3`msnq8R=&u{rgD&A<}O}dUK>di1gM-m-Wv-AA3dmph)+L^ih#M zKGG*gdTgX8M0&F7Z2p=W=_@0>B+|c$^c#`>Fw$kmgyYYK*FDmQM*7>49v$g(Bi$D1 z8zQ|l(kmmqHq!r8oh?s3jr4AMU&z|`iu7TTK0eapBi$6~%OZV!r0VZqz6QLP^3?Z^r%RWi}a*O*F?H0($gdT{YcM`^i7eT z_$6uUlFt(zU%#qLc<%9OtAIfL0e6EnsmGZetKJ(;rwS0agpC8L-zI?8c&$aSdAfM~ybAx{6s!C$>(PIERxSH^0`$$x5;O*eD09XPvx^jK6lFJF8SOopQZA-M?Uw;=V$U+ zCZC_n=RWzM?RU9+9+1y3R>&szCBEuUxP^Q?THlh5<=sg+Nie314R%BNKNw}~w+ zpZ_<`!*vpFN9W@I3+b4UI9AE0Mn1^P-;vL`@;OgFljL*0d?w2$DW6vPl*&K1i|v2O zL%qrpV@^3^SYp_yp_P@1-ihIr14pMa$;Pt=j+oJ!YRimERyC$d`ix4>OtqbzN;lM` znyY7)a2uCuPPep~arB6BLw6*s;(kTLE9S~D6HWDr)QnVhdnT2rPbF(o5|5hnz(h?d z)tXMFrVLCZGnrI#rlF-dQPtAkT$7IN4Nb|qRH89Cv!y+gsA_L$tm)9Bwk;*el^=xe zqBWVRu1}>C#XOo*GcpNvOSdMQ;~liNrD_`*8xz&_?afovSl8N~NY>OGn~>Ot)J2f? zW1_7oooH@pYf4N{HPqE-iux3AZFNn_8HKLdTmhlJk& zq%App2Z4s>8ab&oiT@>#=9cE_WP3W zL?~S3Z7r>GFPt9hO(w1ooG5Pk!)>kOeN9{Qv9^o%|UlU zA4&C&{2JQi%tCK;%?Hr(7(2`N(I zq^e0z8;CsCDhV<@QQIH^G+b1eqQR}UwLCFpT4HK@Qtq~;oU~wXOH54`i^wF%r?j(j`PN)6$B=PERQ~%v4jAWcy6ZlvHz~ND1l9l1TkKOoYXe9i%Vfd@_$GEtV5aNViQ2UP^=n9}JP;Uv@`V^>L_ zImoOtq*hgvXl-ecDfXR$&v zj&)Nq)70Kr^njcR*F;RFt5L0Ovnpl1JEa#m zB%I?k4}=ULS4~ts%@0Cdx8u_;HQ$b1SRd57)Q-or2=_k+G)>C9?Hy)*IR8r!nVGF9 zdOA!*?%nP?>DETMTi3OZCj{B}(U!$gx=P;jt-K za|ty?%_TePUCd8YyiM;uMXO)P=JX21L!za*@Wxo+)j=VWm?D+GQa5evD9>K<mz#;Ep~iUr{I}K>eW;9W>Y%FV$bdDO;7p`?XGnG!2LWt-RiA~1(f_+EcDVHCMqGV50?upLSYbz-(G ztmhy>wAL$aCTRSzD^Q;$vR6@Ki_{2;MM8?g_Ea{HhH8qQ>f+CI2reJ-*`qik)h~$} z?&Z?>k)dwz{CU=TzEW#FD)Zk8`BW*Rp5*L2t%LnUHfriJI zLSsWFS2vUrGWV#2_LJCM9I_8orJioRj}|D!_SBS|A`kN|Elr79X%vuRG(0x5{UR;( zdZ~ctVrJD@WWd&BLtChHT7!ay5yvzJI5hjZ34v| zON1157t*1W1D*h_y4Td&Dz~qiRAXXVL#ptUGT?~VfgOh@<1a<=w0X7Y6S_YP^Xq6=+fBR>h~4Bn+EFQ6c)wY1Gd<7EB;w6!1vABRM#hK}4dx?|-_9CLlPsY^AdLX)Q}3sJod`FF>5 zEKPgTrX#I5a^s!e(l!N6AnBAeUrHle?zkm|Q%QN6kE4OClAnO|yeK895pFs&^p8A+ zht{zWbHC@-l(g1KwkmqzN*bFvzLc_68fhVxrt0D}Uu3CI$z9lcu8#L%P6azD*@j;5 za8QVLSjBgII3sL8Z<&^AYm$!x2U*%gJU}U}1-HrAwQQyg%nF1pJszOHDewgOKBhOwp@};7(VTUs}Rc zwb#_8Tp`_otCf3IGc&&EP3x(zX_+1xzpaW>v<=!^nM=~s8|oU{HM8j97peAd@lWnwer>mhZymsNW&zEQ6Et)^$ zxLGyG9XA|vGF|v0Hdon}8$|IYlsuP$=!Ogu<&N-{Fz?CUMZFjhvQxM$EW702XNol+ zgm*}F@&t$X2<^>2GZqP<+FrlyfNOJ$6i4Y)TX+kWsYlXw9DAu6$%Pflv_ySNraABR zLb0ez7m*rEF@MPxiMIC){V=LqTK(xQg)r?)MO-W3Aye>`uGGJ(OD3Z8$D5t#g--r4 zI$x_`?LaK7ey{_cn%2{urxpkW8B^Xl)wi`Y=T%tLDIt~37WekT|H_Llw|}!n1(^LR zxlq1k+-b}j{E2cuY6Z9jtuh5+gmz`AAQu|4{eN(WvAyNzO#2aqVF5n9ts&#?98wZz zZ>k;NF}LR_x!C2I;O$R;sx93huZE>=+}a{f>{ycjMx8FE1N zEmCY+R?Nt~GTk5n`=k(`hN%P4rq?E2BWk4Wym4ki-akp_g(7aYC-rVBegN2A3_At{<>)V4}jvvB7`8)ajAe3Vn8=NY|c z7fshvu5y({X%c%{t+Ln~&k)Vn-#4m;eT(a~WLrbASJS~) zOJ|p|l7FH@Ly;m5b@4gLFV`BSAB(18%3V9B)?xlV| zxW}4Z9%7C=R&9+fb?D`jLG}N_)>NJJ(300h{4RG6+z*05;}O21xx{`AEhiG`GPxf% zNHVs{^S|vPe6>B`yY2wrl;=0JhbWj%q6Zj$$G61vhD^QO!*J&qJ#N&nu&c_Ma-V1# zo~li@H)e*XQ1^=Z{lZm>P)Xh~t>|?m7i(S@W3$T>?^t^X9G*&7w>7j%x2;0gQ`?(E z-=$%)ql}at=int%kzJn6X0OVWM61O9Dk_KJopKb4{AxQN+SZno^N<(aipo>vP)3Iu zejLAw%F*qit*SU^#^DXx`>L&Q*XB`~L7%LGfX7s!i&nwUdEUPFb44YIgq@(X+nZCP zTJV;qERhJuS=}bLw#tk&^_?PBE9u0S7~ecy-Zdm70o$A4mKb&_(?dfVCbz}(*{Rx; z^ozsUPDn@ODe26VL`||yQ2Zbb8aR0g$;xQ?lbfd(wGe1LYLn=M@2RHh)|uE1R7=%` z)d_jum_`pZc@9oY>yMx1bW3}r_=K`A|ChRUDH8N2>C@dN_lbmb(2|$dc)r&irkYx1 zSJqmU$TZ4BQ%g%mN^?BU%T+Z4g=8|*(omMDLSvUq5pA_ixqKvGd2G%!AVEZiGP6OR z+|ivmbgf88I)+QLCjV!t$%H0`{9mQ-60Vj=0ES9UFWw2QzWIhv1J>SJ7!TpiM%RQkZlb4hIiWr|k7rG%))-57sSU6#L6^7F6UR?yw385QLw z%~eh9&H5#r+^`q9B_ql82kAd%PZDFLQ`XoP*>_Qc^c5?#oSl*vg3X~MFk`A&@~*a| z;*vtKV-^&36H>`3!btgud>O)NO4n7&Ehb-8wo5J?V86ArW)jV)q6dE{si3sBNN?J4 zEtTPM9yvZ3v(H4P5K`6zF*J9$rGBp|JgD$EFPC5oiiWW3@C%N{v?O|y^t!|?BG}uT z)5+SDT(1pHF34!tAtSZ6q~(QkQ?f>y1%;M-6Y^23K1HKK&?YxAxma4{Was^=Co%6Q z)B&cpq~$`9aw5?zr;=uQ=#oJIZE^~#rDh%do(@eT(T{pfWO=*Ds8eVu%Mlm0ggc%^ zx=HRRGvwBWLXCj_#=OX1k-5tfQ>DMM#7TnCJf4<2Lrr34LkdGYR@rY>6_RR6=xd1@A`UL!>VBD7T~*yC1#P0OuF9OiqU&#X+mo$1NFgQVK{!`Z znUEuEluVkqsIfsXG;+4I;h9ekE9?lrgnEtsY?m*7l}iqtg~dxM)Ya|s$&Xvn!>Q6d z5cb$mgBvK~70Nvhp}`MT4Y(UfA8(X_>XZ8k*8?A3gEL0{$Z=xWNjA`&p#~>X1`Onx zOjxV9hQQz4NFr&O9B|z635zwGfc--j-m&VVA-eCkm3{L(4$~SJdIt8A94hyX^9v1< zmre<`K}C!$wQ{S-6fwxJV}h+16`tseL#LqWxELc9{8G^y15+{PsOlLrv>!EeH&kFVF zU@w-E$ci2A$4k z+kC>XBzHDSLE6fdz*8Oijo^-Tj7uo3Ae4-c)U_~pERJTpQ`zyc(sJA;Jr3o8S$aI= zk}t1+iCkuqBzXLhx>X{3wn~gJ(L1kHQl`lWpWGg$|1*15MelP%vsdxAPGZV6w#Bcc zB<EoEc9T%w|_tE4fUD>R28C*~)=^wE)2(@dC)xV~%Ew6tnNQ*vgNge#3TQqf`N zRljVJB-MA+Eh00n7u4+YOm<$7@4}lYybaEQQb(8f2x5q@tpW^=IZpBbx_L^|O?b>M zkr|{B%{r81hy3_mgfYt5h~>SyX_grI)_n9JcJHO5lbo|i z#X@4s{aNOjaQD0XNczOq=h+Sp$nTfWqQ%dld+E6YIo^60*udu<>N3!ZHqL^cMnzu# zt7BbbOBEiIi^o>;XkOof26;J|bC;*@gghzQ#g12?c>NGMEfy3N`M#mqx`b!p9)ou- z*qlQnx>SY7^zf{Kc9UXm5icoOq{b5UcrV_^CES+fhGR89D~R(ibl+RJF;$nWE*ybQ zmvFB_nih0m#!bYkUznYvu-er@kD~mOgsNTp@dI<|B2`)JvhA4GjC-k-4N%@;7gTiZ z-kBSk(9kS>BGDSpoFnHXyvJiF`;=6p!;t>GZlS0ewfSNGF=oYLXh|-$>(KERcdF^~ z?zeOd%prl-ci{~&GN-?ai-%pGOiS%Sh7N=bPS@J8P?#i3QxL^CsLpq@szMyxiC^x+63@7wiXh>`?tfg40SOFt|1e;IWuFy_n^8S0L9+Kv2)-~_sb(Xmp=!)1O4)~3X+;tY^=^7&9Jjvo zOt!jA<6LZh3L7ODZj2~_a&d6Jp*y_+(v=s@vH8AChg=K_cc0sHp|=a5(2Wmb;hCwx zT!Wb#(k-ran?twL5x~V!^lO-I4P> zd|g>18(zibLd|#AF_L%)ke7SB$LG7~UQm37Zgm(G)Uj)DEAnK6t4a!AbhhEcYq+h1 zdSl1C#1u5bNK230BJm0dbD8CkCQMUjXD)$}!wG$tR{Vf`5=EVy@{hd0QFE&cm$Q1z2U55$ieE|>hq17)nO|pN^DX;8cupN~gA$9c3PLbEK zGHkE(v0J(px$tqP%SFNtEbnX;B#=2}vr2DBFGTWWviRoYx=!KJC= z40OC-&LjJZKT=Ua6o3W5urJR_fI`YY8rH-5|4{g^)+FCn^ z4s%UMJ|W6-W;;G4%_OCK?2LC542Dq^jVVr*OckLekE;l0PZS0j+o?!P6-kQ4P)*Op z#mX60$+h^*mOA113T|P%-Act2;|p+4PgG-cU}2+6Z3JkBD2QE^j4Cc^v@!zyl5!xTW~r(Ng!yn}l!(k` z5=NS7J8;fFJCRU;v8p@n>ZUm6DPf1tH?5{QynXkBV225UunxItk=x=ki zW&4>{HRSdc+cX_?cd;!@6Ia}neL^c0rlnSgL8s&q2Jh{Q2NCM~Vb+woRAaj&A$!Qn zE3?ovl081j`skNg=#Z*vmr8hh7!*>H*2(Znnlsa-@0wtRKBO&G*&5mnh_ATp9XX zbojzdtDhm6w4F1Tg!#KwN9kxTElN_S@9^!E`sTfO?0C1qRi+(6X@*yg`SZKxME}(BC4Aj13o2)c-?9H=ymiKGi6l!OfN`PaHcyPKWGUJ4F{a7r$Ec$ivIg zQmLp19u6a0dttLf7=%;&6$N_hhk{jdDW9O^3>Uuh$eHE7rcdIzsK{fG)d#eGnsd&3 zna3Gba_`=9J*v`$O%k$Kd(D;KEQ8-=da%&#ndv3J$t@^1zqRgIk2%iZ;#=@)Bv}&@ zOB&ciYlNOC&WFz(=X#jo9$WfVP3VVP&>JX@Vy%RfKM8j1!E>5FvqkI``T?h<_RuQX zq~o_^`qr%)gJ)guVN^2Woodj9@5GAx6?Bx-hoXX41YrtGJohK+>YC7$P}A<;npxEi z^&q|#%H=ULx_1kF)e$Dem1jYWxRKo!LD2OkROD(JnvTiYF?WxY&oVu>HI3CuzaVrV zDyZ>W;313gm4Hl}mD^QZ>nyNjQ9L>}$z(UBJ;M728TDKIeM4?X;W8+)qfq#u;XTaK zEmyPT)`HN2liKp$tZ4t;3Ku<14Q2$9-yZcn%yIQLeR;suW(krN)7x9g<&qJ-a$ z&MFiIT-7+G#18!-6huF~CzSeSJjFiRMsg_;ypZCSkMt?Bh>yO4>A;J*obTSGoEhM5 zU?HHqw=r_fhad9;vp2jUt>j$V3$oZJv|I!l<(YrK8YP~cknnb;$k~&3Dn1$o6*PZP z5)cAA)*X~L%ad4a&TF&e`wot%vm&~|;V7}M8qt?em$F*q8t;9iZx3h9r2RB@Pt`{<0P}3)#GThnJ`_o zF)w9(i*a^Pqn!eIR<6NqB(EZnlzBN?{N!qAv5lSNt`)jZxKQnEF;^99l?gqR$jVfvh#7U0Q|!qtKTeH}=h%qs{NLr_sk!DqCN<8xHUeElUJT`I!GHSZWo1- zz?j`AUad1vZEI=L10AH-unqdkwAinB*un5p>AGSjyf-WiG)(Fg?OuL6W#$MD%WbYh z)FsDfQiH@%y5ghxd%pbM1H=}jkS*qHKSW$K?MC$vKQpj27>e;eFr;y#E^%?p|@Q>NY`U z36X1TyM8sJQq6T4j-+NblnXRmEn(_`yu=bqwm^vEr^D0p-TjcX$X{wGsSz`zmqU7J zW~{t1jt@khUxsS8)d<8~JY#GEguKvXh;8AqpjgD>Gc}z1AvXe?hby-m&DTW}L#*yB zj}9}0mUL{9t_SKJ<&NkWWSG26Grh81a;R=vc)w^Z!+^>(tc}FY5csMN4iZ<*= z59D;LgBB;K0BQ|>DB>zwhDnRYAWgSHGLl`=COyaoPNMm&!(C`1WU=T7B{VC>&eEQi zapc8{vTm$Z&O;G^nDIPG=!}t6hS9<sl9QQkdh!B5Ojq!J`~w^fs2%; zQd{SgBPDk|s-db)9<8wF%5ZYC2UO}Pve%Z}Y@@U&g;a)V7{i4godHR3NwlGvqM0Gj zD5Yt8mClS3AHMoYQF3Og;O45GvEooMY&ZHXv^$Ls&sKT4ApCB}EN@%#Y;OA6(a9MT zbOKvRmLlc84w8{OL#`4O-{`U~WAq+V5EzEp_|%Wn!TLvJT?%%P8&eyZq^-=#F}ryd zvFFu$;-ImEd(t+BAG1T9!peT*oRmcmbNDPZzr$yb|6*dT+ zxCV6;s?;e)VbErgJ&o|Upwjj*+PCnyPnVfV8j_8rs+W>tqmosr#v=P2Ew?nOR}_>d zf^evEVQ`u|2TAwpBA1(xdG-#jTDa;s{UTw6arK42kClfcorS8UZFI7_ttDOLs6tte zE4h@7S91}2_S&f|e3Fm|Yt1HA6P_B1;CfpL(~3XVIt;Z`ur zYVc!fPnC+(pis>dtoY4G_YSQ1gI3PIyaQ$JdpDjU1xrK3bfCk2-#smxJ`SM<&( zaOkih4Ic%wgdUY`_>rE6x#psdna~24Z9_Y4*f0sYIwkWyh1X)ooNz)}!EfaQ3w|3o z;P`^yjytY@;cv$uhxdv4wn-keZ34W+!16?DKzU-?F~{%p?fyF*I#_4R%e6_Id}z6( zWihaSLSIwv5Fe(v%gr5k_87PJH*;V?Ry}9v*{2UZWz>k=^$@#`J$ua9#HiCN$7QdW zd=FjvyMxe5L@94ljw07-Va!Spg%K?3SR|fB{Ypl6yO({j7?*;IMY_0e(RjzM;oP4t z5hyt4M-R!xfUJR)sMnH}<# z{uFMapdCZB&a^joejUJ6Ly==`iTu-EqVhF=ML7X zQU8vt!FF2EYb|s`D-2a%{Gs<#I3Za2N@i`rzz%uoFKd^D^?cA}D2{Fp)b)+n2dcOq zX4BNM2G~>HJGAT@Oe|y|k;u=MM-3QoVtKJ2Pb~cL*#0tQN{k=dUphDx{wO`P3x7Pe z?C9~*kuZI9i>!21cl2;6xzT|reY9K{MTOxll|UZ ze%o1od$DZiPGw~~%YS8MfF_##jk3M&{Giv)WnE2o{AZ8n*O{~UPGabNqzPbwE)JNn-LyYaDxPBoWZ{>Nj6_4_D!=E%S7JN@QIF5jo`Q4^MR zK6=d0clz+PZ(sVx{^$O@{7*N&G`M{Fu2uhjd#{JTI`Y5IOd4~1kDpvKeCM5xxVY}R zs~-O6zaOqmKl{?a>sxy#MoiwL@ztYN{-@WP^A_H8!&5h|e)z+or*^vXdlO&1X7pQ$ zYp)%Dcb97(Z@Oo~vp+xbfp-oV_q$6EIR7t~w14}9{ri9AxW=WoG_LD*bk92mpYXej zZaV1R(;jP`^;WNj_B+4b=ew6zR$cJY^0U@;zVI*K`taU^|Gd}BpFQ%_u;xd0I;U*K z;b)xs%!^;ywr$*_PhI}4jTe1?;6-0-`q$zSuibXX?n|yu{owPf2A1vm;)!z~eDU9{ zH=Ob3gSzZ~+Jvf|AOAf0jRP+J%6+qr-f-RJhy85untz&zl(W~s$zdcjE@TZ@befq;+{=D+P)d!xt=kGqe=sW-X;7dnNe)!Jn z{!P1Fv~Aqa|9R5sxBjx}q#pAtU;Nkmr&PcFrPCYtP43*|`IYYuK5Xx;|9Ah$@t=M8 z;<2}GnXrGaMV%KNyyx^stB${=b>%0w{_TOsw$!hBeelZv3|n;0#jRVf-)GqR>u)?| zw=+&SV^jy#2|Ks`k0`p?*K!x$2wK z-g*6uf8BlFEwAtXE(d3869B}i=C#~D#%`1Mk<&eMr@%5el{jpo)Fn_v0%s57^n`SS19O*#AO#RLAFx#g>Wd;H0@^WPe?sZW=m z?lo)0xexU?XYDgrKJ)s%zd!Tf@h|^!_~+B!{O>o<`}5cTb;6XVCSTs%|In9yd;Pe| z^`HLapqe9}xZ&`BzV+(Ny}z{YTSM+Vk}Wv-|RC71MjR?0##nKDB$_uyVjTSKPPT^uND7=C(bL zxcXmLz4O!5uWM(2>DF66+~?g}uNgo0&3SL0^=OxqZu+$H-(Q~dWO~UD#*UhFM%{C_ zUwg|7$6c}M;n&(;x}vS|z=^G|%{}}_f8FgD(=OO}c6I7s-~0V@U9V|9DqWy>pi%FCBQ`r$1V9 z%lTW@yxR5D`^$dy?Aeb^|MSOd|8&wFZ_j;j)NPOTzwE{PhrfE_Pd>cs{`o8JTfFPt zH-G7bPGc+Q?R@DkzV^YS7aK2mB7NHL4m+%_>e)+P`{=S879`ib)pOgAzP;dYw|Bn2 z*VlW0cJE6s-H_V#p~J48`&RnpFS=w-`uF9FuK(i6*9Yv}|2sP$^2MK@y#BezdT)F4 zx4RB`x^w2(TpQtG)`Cw{u$s1dz3>~ha-KaDP*l^o9FCKjEwLkn)>;By~jlVawf6x7^?>fBk*5fyw+;rZnyZmbT&R;sCTiI)e zT|4yYx=Y^t%KfMB{*SUPH$8X%MH^mw>vyBi?z!voohH0|Xwx~vpPawyiyt0x?Ew!~ zp7i5B0}tQSchI~;_Z{<8%_E<@bmosIpVIT)=P#^0{?OYde1PlyBbN|IPa! zc=^2{Rc|f%!>tX+{bj$?e{kG_M^AZq&}B2PKBM<5<90jtfg{iTq}SyW-=Fl>js5y1 zXWmfp{oU&q&s}rV{->R^?6LPR{QSCqUH{mDzdz){IS=f9{(Fn-Pn!Sd+b4Z;;O7V4 z`2I164gYEK*+Vuy^WI)(4L|JFw`xXq-T2>%qyF;MV@|p2Pwy=GRmHbIxoOJfAD-N0 z>#jEp{rhuAFWYtY#9Nai5?#k#@Uw~CZoK0U6aRG06JI3fyxe%e=|8Ogbk4{=&m4Q* zTVuOFaNy&={YmHEd+faE^_M#zcSw(l`!DI6>CylDzuGe58&3_o{@VGUjce?6!?f>wXKCk}8gzI1ETmFNiM*Ohzp}XESaaz?`PyKY` zkrQY2S)57jbMt4j_Soy;Cw_d{lB-_0Y2y5UoO|A0!=8EgqPLo!ZT;Q!gC0NsqX&QR zi?9E@TbEs@Kiu`utvCMUTZ^9GZOV(?_uJ#u2iEWRVDdNL`1F!7?VlgKsA|;fcc1d< zr1C|-Pn~?occxYxJ9WkJGnd`*?*CJoS-myk01TI!ba#)Gk}(?TZV-@$NsX577)bnJ zl%hyVBQTKB2ukMw>CTZdkWNA3{hse1crV`T^PJmrb9lrF$31_@5QQf1n*~oSC$DsU zV#$ea3lslLuykGAPaATiE2=3san3Iu9b26=sbA#7_7XVLx0zdb5{A6(ISD7teXV^1 z7yPAwD76& z=_)urILf{Blg=&(jF3@LZ`h%#Ys@J1#|=5P{dP$IEA@@ z6o6b(8&Lyg*3V&UhJ0|l>^6VJL(tLE9eE`sTwj2xpRa;e$(i2QAQc$6ot*$!yf#>I zV9L%`OO=q<5?JH7Z@&vQWHHAsoGKjojb4WiU(n|p{cM2WdO_6!82tM1fxX@dcO+m; z(6@bI`wp+1gK7Wfd+Jk-Q*m;;p~?|XL*iM&Qdfd{A(CV}eXh^|N|3Q6NkVC=oq>&X z4vxJU{0OQ+oct-Pyh1v%+!FESH>Qx%2HKmxDkBrblxXC<_5Lv~2yFO!&fw(}q9$HS z#I0A_=&fd9CN5@|k%{~+rWIl`cO%GP#~v&Ez#~LCY9l4b|2u*dWX9#G&?%sGi*!=M ze7JCzlX8TBjT&fV+Y3}=KzB5vCVWwc?D_Ctu=5J+im1hVjZ)RZs@qI?kq2sI$yr(U z@nMeKt=_M-KMgtV-y@D)O#tmg9`!#h58}+yL{xzL)YsRHfqw-Qy-c0G)H4XQ5?g${ zbyuz z|G_Q@p?z~SR0XJwL2H9FXCDmj5zvebH*$@7S-q29DZHQ-xs`1#k-?`z6T!uhNNzi|lQI2f@~~8(X@|UvTD8zU_$ErE*K7uK*Z9 zq{_~h6a>9B<3;-i#$6Z!Z$O&$@DIbntaNj|kb)(8U+;Ag$sbchvI zimrM#gXD*nQs0m4Q{rAhi0^MY;4pg(;~q6c(xqrsmY-I*X7MW71J=coT*4iD*S@kb zV^cOhYtDB+VA!UO$mW4vVha-se3o1I0xpfyeV;aRN~z-sjhX0W-n2)U=KQV^ATM~k z8{<9(1k`m}d&-QA#L&|4)tzaw6OL8iP_%9qJbRY#?Wn4mJ53~n`Mbfwbn8d1{6k7a zL)^l0(lOSCRh*Mq7LXuoY$u#NkQZ-BANJn1@G@0V%mMNzXuX*-tCwn?w8!-MK=?)D zVB0UU-O@3wD2GQ`t=+_qXLnk5H0O8GtZ^MecGwCdS6lJ`iFEviYYiKuI=uIcM(#i) zFomS}0%UPAg2Q(vNQ-cNW15*wLHU1Ulh<_W{VXLn`OR>Z7$lSRKYrHyTPj35v49Z>bJaogTkl3ChgB zDV8k72xM`vin~Qa1iK!fOJm?y6vb}A7Je1Als`-`-M4XUz3i|6i#m3YUPV+pfF&A*&^oOxD(i&d)^E~TvuS|`uehx-847RigStsaWOBJIH?^j!4{wqGxi1Rw9*b( zx>9IUBd_Y5&>eSoO9!2O#)BU^?d%YmEt}NQxz?=qHggELEn>kW)>!_>&&=em^l=G# zdf{bGp`eLw+CB!ybDw-HvLze0?ne+BH`_V+uMq`P$eqk3b31UdXd0AJ2>zzGyAx2r zY1%N5T+fI<;YG}*wmBEpkLVBuA*o3KKKU$S0%ua|et_ZYgHJ7Z~F^d|&?0Y%S~ z8K1W8MnC)Gg9`%Z6@QEhPd}~qDa2!nsgG$EfD`;{dS&_^xfy~S?}EoF)R?KY`^u*Z zadGm{rE(^?15o@yZwivC5UD*%wWjbt-Qo!BPewVtcjemE;G~#>()7ipQe;s4uuKMr z*@q7zN9gjcMs97-U+!DxATHX#y1ZV$S2;c#{10f}Bj=*x1MZ~0WxXbN_Dq>^cUV2W zEacsz6KE=b8|*4ZwBcYe6qWyNncvI)Px)n^dF#v+dl6+4j?rcWkVT zWGy8C-PE|nGPKMlplt!KVB;OgOt$Tas&GN(Yf0~>Lz`6$P!rIYj?UWR%-}0Y#oF0l zIT$DZT)!^*MAz#J0x7Vfk;G%vxvsLST$Osg zAjj%Kw@VB;8jd92aD=jks2em_DK^) z2_+2dnS8HsdHkHf3)(~1AFf-lg*!}!Y8Cbh`Lj;uU6$DB1c`ghvID^0JEL+H?_KiRBa7c84U1EXjHxIg# z?3&35FD$+>T!RXfPS-}eUsAq-$3eI=L56%mdRf6ho6$;TP_M)x zL(7$gz3sOKdLPS(q~gttbd+n%icJ?`0rhtw0xkWmG~?E1}3%IxB#5{ z^hnP%r(mSstYCX zypA>zw?VO^zxcE?D^IE3hfH0r))nc-c-sH&cj;{{lEO&=cDe#`SLhloDy!i4DUm!R z=%jxFdT*}dfv%Hm&0@hEz0^aD{g>5)flpi%(kN+wMKMPYvc+JGTkSem^GesQA0%4E zEZ%O#5pNEa@qp|VbWBiumBV;k$d#k;18l3j9?*?fZ9j?rg8P?waACg?eM|= z$F84^aoUV`Hl7Mb-<;@N#}=E;{Z|e=zUnU~=f98h>wGxBf)2%6MdMaz7+z!{<(1+& zs^z8GQ%<-r_UQvEq2p_EFI0*Zui*QYpPC*;4rVpZIApP8sY!0BMZsT{y(hY5`8~{F zeXY1<%QNq-k%fTM4P*>C2mF1ErI*~KBXD;@CyWXlxdgj^ASHyn?0WghjXVj1f*+HV zpHKhU-YGlKX<3u5;h?MAts3_*mVk@CgbVyVu^Ho#Xp)&;-yuS&UaVB55$}I}kZ&%c zSLMw0WZr#lxooD#H2H|v$HVo|E`?m}=Oi;ns!Cl_b}$k}=aVEj&J)QUXB(g@sH@5wjGyl7(?|C2R3V{6nYAFgNXAE#dFkm&~W_cTDT*IR0O^Zuf8iY2x~g2 zohMxZp~u-n{(j3q)l6a-ocGd}t*6$FXsn1u=(66sNUZ9>a>Fru$g8(xq4EmkJQ7vCoP{&E6uPa%52ifqJ802gowI)!#I4mYTRV`I zuFsNrN)PDa7gi|`p`~6(qM<=HxCkpe8)b6LuRh`}`4>OAIU(u3yyRO7hzcPrJA0OT z zBHjANjb@U;mHxrmbbn{{6(j1I1H`A_k8a-t-^#ypD=Cn4#KOq?>)-6}hzmv~Mz!{} z&nLNf2~q^!_=;9vbTnTW@AXTxQ&~vbI~tBH>r0_knYD%yJ7}_hIDGZVf7Eiq(x_A@ zLT3&{F4UXC**RC0-f`MHg3{CzzStNkgKq!(H)CTAEA#1-o=hU@c<`fF6>ppw9E@}U z-^@Fw=xf*0TUa~zsvmR17gy>=TSw&H?$!63ODW|{uqpJq#x(j`UyaFUD!L;r-Ks0? zz?t_mcQmp4Af;V?;LFN_ zMNsUC8zu6yGfX)pvx-R)YRTiC=&lnNj0hSnyK)I{{Bcum(Pa2;uOVbVxT@qy zqTsExZ9e<@D*}j1u;A)IyY3}MZ>S+9jKMiXw60KJMc|u?omsvC3lJc}!+y!I#aP_E z7%GXFcoXdbF=UB1ks=$_QaA*UUvOBS*Sj?I(QoLuZKYy+oylk>UXJBNuR>5#G^cR- RSkQ;n47h0Bum8W_{s)pt>eB!K literal 0 HcmV?d00001 diff --git a/tests/test_metal_dspark_capture b/tests/test_metal_dspark_capture new file mode 100755 index 0000000000000000000000000000000000000000..2d881efef7cb5d0f1e509cff5f6e4934d06cd46d GIT binary patch literal 901160 zcmd443wV^(weY{+d2^Y`Bq1RIk_qMlqM2}$0+J}0NumM?1PN)a+BO$1CK(6@v~mlX z09s_wGRXPblAb1j7cv^_1uLogakG?*1(CMroSuG%0HR5TTf9LOobR{ZnP5Ql^n3o# zIp?3}+1b~3?X~w_d+oK?UVG2sFV1`wsg$PpGvWH<&Xe$%Y9gdmB5o3{q~xBQiFe(9 zSJ5;{z537i`0Au@!~~8xMBo(P^~2(;vyJav)~geP@laC5fs*l8Qu6TZMGprfc-O*n zLq?kFNy);8A1PmWCFxptvvyjH&bbHNF}HHz!{2Cc>CphZXM%$uas3Lg zZw5+AX3nn$PS4-9_{IDs0PlAZff5oAHofPQt9;-se0V`+^@G9GYvJuZ7Jzs9M*##S z-uth2UrKsO$%8Xzmdsu>bM`|Impoinel5Ih@!o&mg!f4Gqm|XON-C@8%%|YB?L8lacTceYBz`T7-tbFGW>=KVSs+a{{;q}R zxIchjYLNaU-kbiwwxTyY;RE}qu%zV9oZ_64ylI6q0^ET4Zh&wEOZV`DU|e8|6SQM+ z(8>I-srcLBJ=B8?@OpxLuEKCXl0fk9oAcso{2p2vz;A1?of7X2uLn+#54<_OC&sny zO?WH-uPq_)g~Y!b-q&<2jNn>$7ymf`F9=Ra{2F-DZCCz+bX+;F?7`VWD%Zk``Aq;` zkW7@szne}1eD9HmW-nMUd(OgZ;Uxs&^&lbfYw){*n*@URl`U9Mws=n*#0qVGy3gzYE^gbleN|odGyX-4jS8Zu~g{p1>5>)2}`JsfQ^C z+8HQo{13u&^)O-L--TBY-on{)WfXem#n{`8P~!~csBsAXWq(p zz2Wuk0TlF0N`io{#IGI?M-bk{ZC?v&!s~keg6yj&+DpEhGP$tet~)0OxG{ps0|9<~ zWi$g4PWsLO1o__^))=K+0m6w?TUp{q2jg2Uk!szbfD9&W{MTC|)tksZFQ-JQb*q$; z*%6#6DiT>u-0c9KNSWUJ{R+at!0Lff>cF*$#6_%1#R;|}A6d9yAt?(-=Jm=4uE76cFZjyz z2u%gAZ-+m1_QPd!uLnA3q>%-*0NjdRa07V%ukV7JH-EwG@5Vi6a3WZW``MXDxvs67}LI+VW~f^o+)QL0^tNek}JaL~bKia0pdHc^ZZTLc0-km13txdOikL&%sCrBSiy5t{Oq&9p?eyS$<@m`I3 zPf+e8c`q$d8%mWLvrV^M3|A^%@|zbczO#8x%{6VXQFaG;UyfAW6%Tw}XE%8Q(`D2d zf2B@8=~Cx5>im-YHYIf?1?!AA>hw@2>0>4RDWk4-R2n=2D9wlCk_&_49) zcK)3*7l2u+3(Nsm>O2vw^B2K7cN*>NsdIo)=fl(q$&7hZPrP^`TXhSL`}j_9Y|_;0 zDMs6qyj2E11H8|XH^6B7*X7>|mJhT!=}P^(gY`d3{g3Nk;v}SK2(#JKd-w z$@?VVNqYz4F6OE3PsTaAOUpISX40jdzh9YfQR*rut*5=ulP6_( z@Mc}9E5UnfussX-PTG@z>xUZ(9d6a{zgUr=wryQ`-^FzC_4_Wi4^Z3wj%$a8I&h!h zI&nW7sJ6Yf@`o2c8|~=+pway5jctt||5K>iR&hdkCP%yadM&QLPUY$w8S3hLCc;(p zR3BH-v@loEdA@x`a}}j7f2ToRnsgLKYlRLAj@Q1^kVZT$RkP-3TETAOxx{l*P1bVF zRIr8kOyVMZ%=jcv6v+U^LnVRSK^p(twK)HDDe^9P8N{x|k>qA{dC-nZ_C5+8yGB%WxwC(|_ z`-Gn8eVFuaeV|upqSxpT0i`-up(tqL~-*G>Ixy2YT=OS&1JVD^4-v!nYY>8lw3Ny@ar zVNqk+`0hCK;RIuKDNd0;(E9;p9;3_@&9tG(tj3H;hqhDb533p@{UCT7eURw{DOKNuPIL%?%$l?7^22(4OQpO!yVm9b98smP8qlMn^p618S8{>&oJh=(7y0t z3oZ^PJXq!eYkd5*G(Uti$r~K>rLUzAST~IH_Bcnk#MuwWHr0hE6^VFcY;270lZUbYj3QD6u#yF07YWChPv_e{z;rI8BGyFL3Si|q<9c}mt-cg31=pBhK<2YP=`gy1k zzyG4$$RFUHP5r{7XNpgMml<)Jw*_pwYI~ z?;ij?6s6-jQyi`Nd&0*dzr~Ly{^4rr(}CX0OYdP#pT^o=$QnPDwSG$TrTGQTtokG!EzEF8!RW` ztUy@sm^>j67Ca{B1j2&Hka4LOyd1-F5 ze~&WyOXeM85T8Lh`<(cEd}OuwDtzSl_%--QY4Kis)RFNA@Yy%VcjB|BiH~4SL8Ob% zz?ZSg$M2y(GxNXmc?BO;-OT-$({41ud(7UgRi+J!@Bhc}cqf3zRf>$@*R>;`->Z>M+m`Ch zm!W}v?>g-LnI9EPlKUQ`ja$7rOH^~t5@gY0X}9oHbE@))jH#64vZ{AXZPQ&7q^(j{ zDss`*<-co)tBq@D(|1l#N?leV@jpSi@^)bDjWgOCLfIzT=>Yfo2;miR-g^vO`+;j7 zIOWproMvz|=(3G9@MiLl!4odc|2^|uW4@axqf<5n{6lHG=CRzivgl>jo0w?U8&mdQ z{o#%y{=A!yNZ$(n&E(nbj&3`&*o^$Vl{MPpYFgjfFy~dfvbeLg2%jFQiY=NWEBmru zQ*R|4O1Mm~NhACzJ3=+}h|Fr!_TpF6-q}gr6cDT^Ej1N zv?ymYb+y6M9nkW2`l*dIM#8U4cv>L*s)T1EUz7e@!ZJo8|Mngyc(rfWR8`DadKgR3 z<)wEq=65podC)*Ew2%YO-)qc~)DYFD?Ll?<4E^8YQJz9==lcihBithxi}`Bj`{$8S zMg_;EWfNnh^>N39sV*0|4ys?{b|^sjFmCgn6SR_H3?CA^ujXgjQ0 zyP(4p(D4468)jrIxM44-XAjM^4kWwY!w(jBfQl^>(fOP^~3t-5+x8G0QDU=V~L8o|>TSBbG-bt;-6v zug%dsEv)OiprK*9^E^mhbFIBWomz4z+x#ku}52JV{}W47!;$N;A8MF-IDi zBZA8?=13BAGuTNIRC5ftLPlvyFG;_n* zpK*^>qwX4MCUN^KfZrZ$gAUDZ~oB)+}t zCy{E-Z=Tky?mAPVui~V?IKNh(k*}rnovc|hB3@%3Sm|vr-K9K3Ovw2s{9~lt%rE`L z+ZdO|N9fN8`eUA6qxDzClDE5`D*hU0$NKCj)`~WNamfT#vzocIFifMr6MbgVRIKBY z{rKXeg0sunJl>*P+{)v)BylY^=TaJe>SJLo%99gJ%MGT5la?Dy%L}HNNy{_RQm48m z{2%&mVVKFNYwO?r#Uj5t>K#)e9^*^#Lp9Hx_~yqnPZ)koW?FS@X4;4%{2BNg@Y{7u z#y=7ME8%~^&!mr2!KnJ;`2IjW_N^z!7*g*nNa3ZF@S zrf_BQmclm5m_oIdkSJ5juHr96(Yn|EMu_dy*BizxQeOWbUYpOe&d9#N(gibi#62;u4-?Eu| zyIE77Fqz#M7DrY|aDB2zIkGyw@{h4IpR!qlT+An#i#PJELFd#xBJFBA-wEt%>Ad!< zZ?@5Vm^RX`eTQgc9&Mz(3jS)qWlb7kzZz~S>(aGw9|Z1~>>n!h#Dne6M|QUlMt&K{ z{+W5VNjDu4*&+hEZaU~6*VFf1*TH4$fdDQQ_$@2JC0f-qeHz4NIJk_u8kg&}@lN2r z$2h#p7@Qv!7{~TM`iq4xY{hTcldydqV=XwQ|It52eW2zy>El~=)QxNYRQLyd;iiqU z2Ns^&4<7xyJkyakXvfywCEG2m8~c}x-roFp?sjC!B=cO3p+`G5PjiO%tL+T66{;fu|* zYXj}6GsXMrDd)TFA0W65eH*?`xownlEm?T6k#f@R_9RzpoyqKz`r9Zg`1|1JCk20G zbthp1|DlVTUtpiT>9uP6*k(C94M zcB&!Y{_L`-q>pCYXMZ;GkG9YLeVYCCCe%B zwhf67OS)_S?99h+h)=8R!@3xnd9k`?+kXYRJ!|FWtzSh_s3Iva;uRioXAH4*MFZLyL26`YpP?h4;1ggNECmzsvr6`lJ2P`&tJ`e7P-2 z=(&fMFY#UKOa5)L&yabz1^n8d&$n-vZ;*4$wFy_sc2QPnw*C2O_SP%;iC6MF$d|oG zok?|tBsjj<1#cP&&5F(my=ze-PUuMHiJ{W~_mjvjPlv!WLN1QG-h5ed$^VO;=(|N9 z-2Qy7J)6EsgEt#}X<^R*y!<-H)9;!btv+NRnIm?_z{woR-j(jF68VB}ghquH&p=1j zl@*z5nIpr-Mm9WEnVuNI1PqDA_KQV!i)>p8|ubV(o3Ch6l7hTqTIhCd`YFWm6( z4rpC)8cMi-5SLD5uP^yl_UV6Ro^-MvEJX%7wmaR|$(k+Y3#do*-d})E`wRJYbM3$+ z-{|}7=BR_VGjC2qRvei00^w5UvdsM2{z>^!@7n%^KPfFCsq8>S=2qh7;fYC`3GZ9L zIE9+6d3(~`FNUcNzYbTP4%*YGTYN*2jio*R5p2)P!S=iqY|o2^KfrtW(s%FoWY7IM z^0v@Ps*?StYeK)$F!T!jyqe2Z^dN9{ns!b|`#?=I!D~$$>Z={wUnw-Ho03aZNR1tO z<{NYm-@d1CvKj*+RZqFacIUIe(-)KE1& zwXfDXz4D<2ztQi*!7ZD5d*fAx%tt523;bJ`EkAlw>5}!X z$dJOD=hm+OovFWDXr~9>y#_pF2KGN$;F7JIoZip;#gD2`_ZPG?vu^p*|87cfZ^xzR zHs6WwJ@CaB;5eAMoXOm{U~*&~kJh3^{u7y2WLa(Crb5%e&4noms%A4|^F~C1`=TE1 zJrP#_V!~kdF|?-*+`IYqS1K_haiZFOKl=`8=if+w)FS1>D1TeaiLl#o<8c#kci^&d z3a7PPFok#$bj{coy~}s0j9VSPf2Gcup_x6Ug}r42drT{P%}DeMQRo+RkMJ2Yxb21W;7glDbGIlR)@X%j(hEEvLcf5 zp^5ycL$((tSd*ND=S3a172KdkP03QR?pNHl7YV;a_}y35+I9$kn9Ke{<}rRWV^USC z$GLHbzvpV5YqEM|KWCphyg_ssQf~4e8f_grLYnIcS4ucVbw9CdnCJY>Va=JeQO+Cw zB}ARSusP8qXI%Sh`!qDKv|T*QIbw>`0}mS*qQ=%0CAQW*7uj96B(XJRxoUR)W=ePE z<|*A7@T3yf>k|*E;)t787FAVMU8EgHrc_pD%3dvv^eLLfy_Wr&#E-*khw;ta&5_;0 z|LQc0Zxi%4Qb(6qrcW%PHiY`dOV&^}SMR#xBFS_QpI zdWxoM*0Rqt%8|aqWOhy=Zl(>T$h*S#KewUZF26X=!rnh3QY~nMk5^FEuJyBuzIbk# zDs~p#*ILJZPvn4k^hq^+GWWSD-J%ZITE(+=8v2P!l3mkwPt79Jyg`XuxJ9dONN zAE;%h;@5Lktn{JewIEBsO`h;j-+@Pr?_Lihzn>br!(?&3%bv3Lc&ukU+Tt&cGcz8T ztSoTQm)X#0PCR|jIP4%?03WRccl0AgUcnRm?L!W1%+?$qcT9A&W@x{%2^|Rhsr1+5 z)Sm)w$8Ugs`NnK=70FpnPnma-bBD18w=8!&Eo~BhcTP(zn6`Apq8+TmB1d5-Tzs}_ zUgor=XBVA=2Ft;v1i8`*eh-1m^bj?@20UVJIYfQr4T5)0RiP(Fd5k>r!JV z7ujt}Rk7vNmoiX|l5Zmz(`*&%8w}kDU3~yueFppz(u96yRg?=HgMM;@u(M4jgMN)-zo8T_+1^1tW`-_wj-XpjZJ|f{WLA_89?w?V<;s*g9u}^3sw$bJcZL}5m z!p8~SJlSc;XhlB&Mhdzh7d(xzjF)e=X2XMozd(xzNv|?!Ey8&QTxDD<;Ga@1{SGy@ zB3k5NRnze=jkaIwmV%`s5AC?Kxej^g?4G-twLkld-=p7}p*`kYtPegWdNJXfC7Q+Q zqOVU^Rc20IdV0}3zkf_AbeAG$57bpHeGc8tr!Nl;IlQqNx*MTKMT~&%c2MVtJ#O2d zv?%M;rNb5p?agPrY8bEctY^lz)KkWIp>y_>(no_TYBEcLv?e;s!qjNWMRuz}Dz;Q; zEm@6{Z>xmXOnt75*NNbGrKmo}cnzv36*$ItDde&!-YdEbh(eYQzjdW2cCSfvkcrk?`k zELWAQOxtgptl1;dHM8|b%7`p4{old)|1o5Z|W$Vj(PzZ$XSj~B1v^SH!IHz$WvBiyd@^9`*zBlMlSK`rVJl;7_tUgQ{Vvulris_ zq{YK}$xI?IiQIIp%#@2R`Fb)_J!?^KnMs45ddp0-sjtXQ^^C;{#-c}V+LQFP+$8v& zenYdL#|BOIp2z#BZjt>m7d3BZ|J`jtPV7I}-thP59L8RC^$R&mI^kE+Pr_f0Z^=O~ zXg2gkqVtutai7i6+CD?I%0A6q$G)3AWqK|CZ_ReIHNkD}sx@rlI!3BiStmOm;EW!= z)iKh+T(US*;a6hY)cF7~Sc@kyMjbO8t(^sq)s7;--u< z$X5s-k?^~$ca6RHN@cJuoqQ+#CTsQe+9Ps?z}}Z;rd(wA6xuUa_#|t!e7gqTD|9Jq zbx)f_&S)g<+i=gmg1g1w{Q_Tje{bAnt-cobp0!%~pEJh->DTlj`!+)BeeLW$&G3K- z*1xL32W%L)KwQGX?yL0hukMj2+Tt4Tmp=Kf zGi$$YKC(3wds5D&&74bbh4;uFvuAG^F)P5k+>=!C=d6{M(g6Rm5|?vme~D@pUiBw< zRTk^#ZD#n`9_;PV>-(_P93RPEJVtqLk5`^?3Cc5ipz^@8JU;kCfs}>kAxC?B_u^B| zHv^xtzG?WB_Z8stjc+2nQNHVAYDR~OPh7ss!r<;?^=t2)blc}7?S{}M#MhttA2$SPgtEukQ$S}vN zQKsduz66|TQ?$2FSlz2J5sp_+fK!?|Hfa>)MG5Ecb%q<5}GbL>%dg&5$#s#F;;T!qq zOEU8gN4^vKM?mV;}5)R$3Km6E0^&f_VxIWk?~(K$=+_gdi>KE ze>>wD%lK#DhC&0wpn>7gzzAsIx4;@{>f^ly8W;r)oP-8)BUO@vZxmyew#<~IC2H0a z(0~pNeAJNF{1tM5(1KzPr$Gz3(85e;!38b61JC;d?Oe4iG-(wuRskbLh4|Df`AdAP zYZ5MfCI5?q(1Cf3XC<-z@5;=2j5Z7ML91p?}RqrIr#1HnkaXG_TC9Cox0c2{mgg3+$m)_ zi@Xz>J~iXJ%eF~bXzfnu@D#Q!-vw9t;=l#UG9OMAe0N#tt2G}}78*V^F;G^{5BD$V zMZ4ACLA#Xuf2LjFT}!(;(A#VM(U0~2nsy(9miMoi`d?4GNx98m_4!X|mv;V-((ZNY zFNG$)LA%!}=c3#aPAS_%yV&Dip~vf#ZIiM+w2O__ca^=0 zc9EmMtL#;@i)63wA%U5`zl>|r$c6*AR#+KI`YDrjeR)wd4T>q8}`4e6Y}PR4a|E}Q`` zj!0c&wQ@ep`E}B1&WP>9Ba`>*L zuQSfkT~NmU7h6c|Ks+7XSDKutdpdrrnq?38WuD1nhVP2)y4k|nByx?U%Xy%j1Il{# zZzC)Xqlf0%OOR_Su>Ty2d?4qfR^X@iakX|Z7jIkqyX_sEACA4j?4Goc^IP_Soyd^_ zyK`h-GvnpC$eyo#q@y)KH(MDupXi!pKPYnNGVDVyqNA5{QQ7BlwiD~eULlqKPY1US z(?GWw-M`4dZ7V}s+E#|*!f@d@GtPpGXxT4!v|19euk7cwnmM;$X>A#Hq$G3r@si9D zT_u@)7N1@ev-r%SQFl2q)#9^@w8dSEayOMM%G*`4=+5IMi|%^Nv1l6S>klo9NpdYy zN%NNu+y2nf;oD1>j@bSpa@5?VqqaL^RaY6ZNgI84oPM1MA8h})_Nbjc>G-(zwS4?e zd?&sSe=YuL{1*H(_=oV%;-AAmhkyQNM|T%@E)?ssw11SzW{nwye8yVb#JZ58*?eo* zn``fvWUg59(~CL#MrCG`c98v834Q_eT_y;j(vUW&$##D>wl^(CM zj!Ah*kM5P8ptAg=d*L%$-(KkhnFH{I>&-(Hz_LKw~KZNjK33?b7?tiI{c~s7jl*;@h&4> zY_zYo_m;d)$)kU+ZKKpB`0T)5$jZ2X!wwz$)D-L#<*Zd~vp)e2`}zOW4*fTj!!Fb4 zhi|h({{v;N*BA8bx7wj^mv+zxJ$C5Ze)136p<{1(j5Y@B(C@shoNOW|QUQXGjYwXZpGI0HNJM_oM7d!OcHupQJZ&Q%g_CbH<8rkz{jJ4T2kMh#@J@h5x z4}HC_}PpG~a~2K}YWpg8m^CJwzDi)8W|f zYuM%M-p@F@`GmSfSI~RiXlETsk6rWrzK*RQZ_U~D@ua-tAGfp59RME^+ZUl-*)Iz1 z{sdh1-Q=*FE!ZcHR@>8S6J}tin3Y}|GoyX5_~seh;mYb5s4&n~_yqmPZ-P#0iHfgCk_6G4PGhoXzzXN%^{(hV<;>VKzubUnA1Z)!av**o1$2l%s#rCTk#~p@#-W$o6u}i@g+s9ny z?rqVR?6d!ljm)il^9#{aew>n7zvni$+y|ArhBxrd1yhpyem&8e5VVWl=Wi_9%2{*A zN{zb)>pnKu?r$((o_5r1F}o$5J-f%kcMk0LWbCk(mAf^Izfb%=>imN5FVGKPf>Ry# z6pKc0+*p_A=xV>Y(YD`|=uX#ER%R$NhJVu0bc@Pr*E7B8QN%A#I+{hFq{phP)P7>i zA~+^^KmD<3L)HN0xqyAfQa!;ZYn+_T%DR>jy{@QJy}V%PSB>btom3C1#PWuZm+mod-A_LJ_Va@rxuz2-(Byu z^-aO1{QnntU|0S>jmMLW`vvYW`=HCy&}B7zutkf`e+k++46W>dR{o^@-1grc=Pame z6Z$Tz^U>{2==ufCnqL>HMrjGExF8feXD!x=OcD(1&Ng&%k{-F{iK4aMtqbb3NM{rJ zI7wSuxTfgcO0SV7c8QWUta)|OyH5nlIk30MXO3i?8aZw&^Ry#gjopXy-^zUi^31xU zH4YpqOeSketYg$k_K@aqM{9rK^TD({()KNnOR~6-x7pwC>r0uTj;z&v)mZaLN7f|q zM?132__o2^^|_gQXjWA-5B*_XA63)FxqMv`wij}a7N=@ng*3tmK(o0=M zp91%-2TU7!_?qyz9v*d`H9~lp@W@K)=)i{AUK=~3qeOKJe|z7!!=T1Un~%$RCTRn~ z=LzoOS6HyeqYui^vHDFRHSLUB8|O?{>P`*R{m=hE-AjQh^<8*Cb-zxXs|kxPB^|$H zFg9DzNjv;p@RV{dQby7zQMZ&ar~S>ge-QGWW~pjopDE>fU}g~R4cM?P!h`?6Naxup1Q=xOE7aRlMXVk?Uqz?so` zWIySbYs9BlVNQ%WRn5j2k3*J?(}c_oO3&oMGSfPHAm|_Y}ZRi%NrHSEB8%Arna)C zi`}<|JFUlMoYHL>QeU&_X^glf8@AQoCO4+Ruz6 z+(me_-fZOYwnrs*+dA%5tzYT3gORJaU;B>oJjHkYbenrJ{oL^&WyGJUx-W7z*`bGI zbU0P_iC=^_fBGojaegUt&~f9y2H9Vy)1P0V^E~m3iOtfNpFZknmA+_0FW(MLiG6#5 zp6EXP=H%wJTB7?5blZY|2=_YjdB=lt7t4fCUEFy!!qnZt8GzWtmttquK|L9D0~@4& zjzfb(;~kms^_tUfTAI(myUs8V>)2a~ZE78Rs#4*P?6Gog#LhoKjj83u`IV!ebkLu%muz2PDdsK{bCjMSbQuh`=et(?`bKbIIoz0JbVqM;n zBiu*qv*PErbJ(sQ?x!+Er^lJNaX+*PzF&tvXE*b^kM!Xs|2C=PWxn~4d2_gOdFEu& zB>oCvu`#vAF*j(-ak*2e_jj&f4LA*Lb=>P{?TAt1&e8U>(CHcEJoJXl6|HMC?S#hO zN0%DkKNj0+_CnIeNIlsdLE4kpUW~x?aXMvPww?){YK?LZHE=yS$n3; zn`0)6uZebN)4wOc8(Z9>>y^=fmj|3SYZT?)cN1^FIltM5j!}iywV0wDEjRwt z@|K(axvs^vDz$Las2s;a!CIV&%vu1bw=sc3rTwP|FjtJRwfTY4rfZbSE8}=XbE!^GvcSma<=1UCw7W{&T;KBEB-=WV< zhnWxY5$9QNQm}huK`%ZZ+Q;36EO3h{#vP09)N?!~GZtN`rz<6s^ZBfG?8_Ttz>Brw zJnMEJ?6-eQn&|b?>#>iRhHU{l?^4FAobd|jSJx7D!|oTa9j{WxtDNy#%UFpGz3A{$ zwEq84Kc=Er=Kdab4_5Da-O+u%)4xsj&dNFWQd;F)W2}^O*rm0MRmv3`9kcfzg*}Le+PJfk8iBW6CK@2tnJwes%wvoDd(@qQ+-p?;9bPcoUvNcEE%0k ze_qu2_8SZAoVl!tSy7a4-n0Pz@2m{!{X_@0Po__PSP*? zdgaX|e(v_)%{wz%o97#9n%6N{pLc0~2>O6f^a5e%2jKP7%;?W7-0v`X{44t(dF`a> zNL#q2*{;k7?Xl{Jnf-OeO{)3C3|U7_-X_{qw_IyZouIbKUKhEf=)%lM&&3tW6YGMG zSQBGHTt)kMgU9s!T@%E&+!yd8ejMU{ZMSwfuXZ7J;uH?Auv}j-T zsHZj_t5;p3a~0d{DZo0&T$&&2_*j#BFSyc_kiy5JLJCg|c1_s4YI9*9)6X|Pr1d%Y zf)?Z6vZ}f8nN{AxXIH&YxHc-rI+eNClA}Cl^%34f%)2nY568ve`r=}7akveuUMzea z_fy=xxb3IvT5Nh$VUiwQI8g6X$azxX1j2U^&L%yJc*4ZZ)xVzjZ1vKK&s5((aZB~M ziEmVYKJj}2PZ#Qs0MB+Oyi!C;=YpRP}TU+ zQQWbA>2m4OH?`=D6N7h7n93SpteI)=+TPTXGM>bKPWl|hJ?mZ+bIa;G4t;(f`&1`7 zXi2N6(`Kx}jO_r`KN_Z}EGu?ZBucLt!V~tj7)#w_F;B&sOsB}oGLomoj0+@?NBXe-_TloFPK_7+u- z7O4i$cZ2O;oAxK$YVdrKx!%cHWsBfTn`Hg!4~`KNR9`E4LK0|73fDF-**xpq*UJ# z%d{D+C)1~@sEiX!<}WJKOx7K|{UNf0nefm>DKlEQ=O_1a{5j}Dp&j{N#^I%*^u1;+ zFr!B`KTxsAI0xptY-GO`HdQyv}ziT1X8)08pIrNJvdZB6Wm@ zOmF7?mVJ}WbL)K-+y_@jvZt$8r2b7yDz;0Xoj7XpycWYd4vca88|=fi{#K(e>BkeV znLLesd9RDNCEI`NYE5~BJ1MkxHT@><;z{3%%xlz zCSV}{4Acf%w}F%Jq(*e)R?713TE;Nl?Arot`<`3foxl}2em{A=>k_`XuggDX5p`ah z-UIU@yj{+KhC++Tf<+?l3?|&DYc(gKJs*3a%CaTvGq^7{1sUN?ctGwx%R1x=b+oLA z%4un0?c5w#J3WV3KSdVFh6ZNVw)Z_=v|xDWX<5A z!`MdVM1UW}cz3{ag&$0(FRBfG5aXT4yjaV;Xc;}sUZI)u(^Yx|d&rs-dPK%}-Q-@O zPyS*nGROsF{fp?WW!}nI=dcgy<}Q!SXPMVBzh$0(&e_3dvUh==r!t1^I~*;t)|`DK zr$yEy-wV0$fV>u2zhoVgwMFKgtS{0RCyjarc%?7e(Tjaxz!I4KyOIRkB^qEuKKemGyooZWw1Ir#T}T&Kb#B`sgsaL8sQo zI=}YChI5>e_*$ZpsJ*LIz= z?7K+seU@#Pa?q0OgWv1!Irr%2y~9ZV=DCNI9VdCT(VwRsxmcnx9wx?xcgL3UewVI! zc&~GtVe{-#uq4-ndfuX-P3tEkf5{%q!hWnIRdua2bWzto=Qz%OMEd9gbK!GnJa7hU z@;(oasaCZu-i%Fbq{AM9?Tno7^yo!~(4V}caOEsEGT6Qd{8aLuHmk8Gk!QiHXq;JX z`#|(8;cDAH#>f2qP|sM#Qt~)o+;(G6`f=SelJpWYXC?uhjW@qGfnz%IuDtD)Ls-rh zW$s=8U%AUJILjHX+`;?9Y5y*#wgDFZ}?p39wY6zy!1}`JdeK5WiIf(8ul!~bLOI%oNL{QGujjzZoJR5 zoA%|t7sJ_`4*U>rIsGB+{5ClE`u*E(#YtaD9~rQYgm~nh!XoJEUYzid56Cz2&V+hI zUm!3ST_zuQRsJJk9(g0czenz6TbR2M;B5u}NZP@j?4{AX?WlV`*F)ibA;!30q~9=l zWq+=`t3x@9w<9lT=m)vuvx9xuvYjhz5A0lN`x5)^)pc(+z~8#f-E~Jd41N<`TD*5F z{grxyGVD1XMNjZUd23cr$ml>P)~OH3C}EGuJ#u4jIjnYTLlp0%MKhnyafWn`J>z-K z{JW6lr0PoVG>N zHuO8Zn>IA#yeY=oMcX(dq+OAm(~Zn9=_aFXCu!TP*cC?GblMf7z>BtpntpEkFxa-) zwCh9daU)&YwQ46}a1S*F+Lo(+ZVNThUfO0-D{N9;^2-^om6Ts$-O`X7VsdXbncQM0 zj_xCC@Of;xV$>p$x3Fn;EAD{MzAJ58%bugBZOf{a{gE(i=~zR|(+7DJvO(L=R7-oF z^7P-NJeO{V9-zZ^?lD-{_jh2=c44&gBw|;#k#xBOB4_3I>j?*!z>Dq31)`(L9JO)d zPU8-F;9NZ!`e{cWWY!Yg0!#E*Ijrk}_eJ`9dB1OinLfn^qFe4_${Vudw0QRg>g!-{ zUdb7B2RLPdlhiHszRtPk2Gf9p4_~P_-ZyU4hK-6h)dIE;@TM#E4!CZ;Bd@9V0`bVDV){g#Schs@=UK+jO3X}+zDVI*tO8uU+M zN1tA2=bk2M?%|F_PTutzj{ZP&wdHA3zu1TFr{ppCJ@Q0d7V822lWC6aEt)0&sVLrX zWY5@g$v?I_O&vLwrn>T&o8)Kh7@`?x4m-wbHJgwTp5#p|3Fk<-U*PS>9nGAlFn6Df zaJ%;=&bA!b*{#rO~p9J6t43V(~#xcH^eWQFY z-)`dDt+9^9(DC$aWN9_F(I(?5X=dubHr>BcJMwLBhkn8uCw3og%)ec@bol3L&YI*r zWuL6`vvrTGD?N5y7mBg#3JI(eJu;aFZSjWIn1@+MXW(9>jN(1jeJ?EDv>5C8OizU247LaoiywYzXyZkYT z=aa-AzUbdJ183lKD%{g*<2>v)qVk3Cs>`^us^WpmzOU_`NO~85@8`BqPPXX=OK=?E=9&daztzAX47?-sS z%xuV&n+XTx$`I%v)cdZC4`nYeokG7C(Es_2!(_%|61;mNa%zBgd$|Yjsk|G23{s*+ zI=9jfPihebQ#tq9qRlk=V7{tJeu;J;q7M(%zR+-pcD+Ztq9dXUkP~Xojpj_NM#Y{Z z{~Y<}$UjehpNKvM@74acAv-k2w;O)_B0NRvu)wQ@4@$eG{pC1m`%3(o38l%oi)JQw z@Vgi(p`4lgtkI^^H;z=&c;`XR$;$_pC+9F8>EPusmnO3$B#Um%{1|7O_;!4GAFcnW zr#b74ca9yk!M1w&YYifkx(2=0P*1;XpTT^f4`qMP8(tYZOarj*_~8;|%qfvG1G0zw zhQJ5Rxl_!q51ChEc=zxoa6i7~uIBX5{M$5KK(}M@w4i%?8rzA7aek3c#~Qkj_wC4j zonZsq1=!=w4Nr2f<@XTety$5hoTHwd$4kbme@$E7v@7MUon67OrrzGEIVKqCq3Z2J z+&z%>fc`elmC!kQ!-rU zs;T=Bb)VBC55A18sCss0pIjgz}Hs<{E(jp{Lm!j^ zED*f|JU2tucRTp7=DN*VlAB*BVt=XmQlal`&Qr?iQ+V6^C+TY`i(b(4GnbnB;5+J4 zF6Y;Ey2+%Fn+R(yGnMYi#pX`dyb>+R=V09z z9p(o(H}r5q3*pT`vwP;Un1(!Ech6p@TIPhT9G@GbUfIMqCy~<|`Od{UUrxDP%H^Ur znyKptr<+x49&MV*y#p!xCiqq?*Jf0%UsYtE{OSU8ZF9qb%jHK`lP2K=!rjXglFU<9 z6s7XJ2|s}*Q`6x0R%AP8Nb`jUOwHUAdEfjr`m_|)D!;AJj!gUMv~YXf0(6MHbI^#d zhkxWVCxsVz*{^lp9>|ll0@6;!-P!!&L%aigkLHQ)$D7V|FEvy_ zC-*=nuf#-on(j&T)R*#GE$fDOw!AmWv+5zu^K8r=o-pbO$DzAC$+?#hTQVH`C1 z)!4X(OQCTM!ppj2;~K>FBc9*%S%XY|RP}c+SrXlF6gqP3x&N)4l@Gk-cx?P|N8XR# z%F%yJc*1bUo*$F`6T)|pUP8L#ZO3ET!ySDbZ|5XQSPggFOn9_}wc(Cz!UYmG4bMNW zJooD{Zr-iz(wGxtO!4jm)S16HSm!S4tbgDw*W+2k9XZsg{h07=!yUV*v;HT9Zzr8P zHOJeo$H6NK*ohJbuOWm-Nf^8)5S}dI?BN?u$h+5H*BN0>FxvU>fkFTdiAaex3K1UnB&G8!P;jA zA9VZZOQPIay5_dCPMEa;?h@862k!&>gM3{0`cQ+9C&I@SeA()qzRuC|m{Nt0E3NQn z%2c>Y>4o=%>>2+(lPVl-(hB$fYg+ZQxH<8mEkjJE!kbKbVJvy^&0>Qk9F8Wwy`ff2~L97 zzvv0K?$&ho%n;o@>(k{&hp=9RsSx)wX2<1cc{}qtaDE<~Uk2xA!PyhUUFsBkV}LLC zNWFh#ZTf>lwG1Kcop@8rDwpP&S<0Dds(MBCxE~nn4)yaM+zC_2!IkJ{W{!_-Sj8H& zDkQ`mVhVAuYAYEJ?B%muDQ~Sh9}Mp#tQ-@J^mo)-t9W06HP^6r_%-WOs+yTxk5kBB zIry7#X$j@Yp~zgDk;PQ%?Bv`RXD4qVKb7!I!lyVZyb1X@B&IZZ?~AjN!_x!hx(J7m zC$j&!+jY(XLO2fy#YP|u8-eTZ1$t;LG-ht{^O><^5ipmXvSGRB;)YBW$Vvj^L99N`TBF5 zw;q0E`TFmLh7``GA0G;}y2IFKpTh?3oZjc)Io4O+qxZ}Rjc~`&&iI)5$^F(nl-xgN zS@O6WnhVE;Mplmtjjv8vSDHL7bYOKBX<4M%ViqK4k)B1`?WEmK+U>+|C%yq%A5YqN z(#Df^1LyAJNxy@%J4j0*eh2Aykd{tbI%(;|(}|1SqR3Crsxan+71=4uSIWByHz9xT zr7!j)>wY-Ead~fgW%7G!QSyiQNAP2ozZ}6Ebj}~V?6IYo`Wx?ciET)8W@U0F??V5Z zDdyn)wJ$gH1AZTDbsrBk7oLlXZ?API^UB{7~ zdh#Uw2z1I_7|%y4&b`Cb-`B((e2)5WvV$jkw8J|~liSi3Cd;|s+3y#p>^09xJ~X9} z>qRO)n(y1vW+ks7@58bmr0lJjmU1X&8c&ObBlpL8*-*uw3x2bA$~3;MPCk6HB=ekh zbMoO&@NbALJojws$aDWzR_%-3JMFOUm_T^>f%Iu9@21a9mN)1AgZoyEdXle=ewo9b z<~ake@+){*nZIE{JmlVI4<38L;{%atd4pDDTI~JazY%;Id7n&VT9L2fh@WRo;K0K> ze#wNE^aXdc+%YJ-W&9FFSR-s|8D-KvF}VSm_8evXoO`->-|QpfyKwJ0&IAu5+n!^6 zIj@DgyWkn4qT?IJ%~374N4O@8+RYhiU+gmI_jA}V$vbiaqtxdwz6Jeb3}ugKu?NGs zQ~NP;d*8k4lIVn`d=dD6+a9xH@ZHS+yWj(NavqpBuV>Hs4ZT9l*LsB&_h1XuH=rBf ze6Hmxy#jWI{ye#_r?Za@VLuznzBY{gEqC>nVsoUMJc@o+!Sk*hej9EadW8Y_ndlY% z729w046VPyANxt|(RA)v?=U6!B1ZkvmR{#=uv{)Y${T@R^j=+JgCY8WNvZAcYs`aG z;f3e{a+s5%3rJUslhYV$Mc6`E_MZ{K{pV2pNn=&41zW@lWdC$ze>=MWVaWTHq}xfC z_;BJcSyb$P@0y}!uXn*A_5^=JCVz@F3IAQf+?y-W)q->SQyZscztnIK@fh|GhiYSz z4rwVf&a&q^guHQ>vhzdL>&Lo7YX)&PwtC*c8F8VJ1zYixLZb@m`Z{Vv7gZPMsM!Z? zJ?`PU*w5g^iAAx72U#n=Eaf7`2n573g}19vyAjp5r)pe2p^HN zEIC`#IlnM_Ut;_MdWT7j%S6WKZpP^@#_LY>4teMu0{6vc(T{iGTnV$2^VhF>KV$N% zAFsJ_|Ht{CfAn!K{k;ht$O@e7{by5VPQt8YuetsGiR3+FelYnDxGL`Rg~e1Qhr_pH zSc?j{cVELeO`-o9`|*|`HlJ&;(OFMDMYicFMZ3$Ai!vTeb_|-C+({oyVI0;zlEQr{ zRXl@l8*1Z{l4NPcj~_F)l^K<$H<0N&F~!(Km5( zu~%9PZLEbhz&&;?bg+@Joy(YCoBnIk*9Ph1jJ|f`)NwC16cRrq?W&DVS_^$(GwTM9 z*i}q_lJdhZ_+M}QEVQN_o$^xVu8h^@kNw+L;*9b9XRGH4(%brI%_mq-|5sSNb?5Ts zN8gxc=wY|DVF&(aY+ZJpl z))0An&wLtL*JsGN@7Z%Ko=-?O!%r=^Q`kd`e$+hVwT35Hw{^|JIi>v0s@y$E^qz{- z>>rZfivCpWYd7#)jAGYc2F(bJ;lNU#hfizdJWlwuq={X|zVc=3pQu-!jF@#r5w7PK zwx!ehB_E_%~UyTL#CjcribR_Yu&C zh^=T7dxc|R%Rd&ql)&0owR}Cl<#TY2CBi+FdaTq_Mm@RO&Itl94>)J5Fx_k<>518?@r{(tpkSJ4=B6m_>_Pfx$?;a%Ul+xU%^;O|>V z8MzD5u|Rnu&^K7oH@MM5tmPcU(24bp@)}Vp24Y_(dYwJc+>OZF zr{Fhx8MmgL_rJBt#a@2Z_~A{CAHB6nv6o*pVR+NdACt~re$^eMmypgreO31Grbx%z zn-V0fhBw_v_!bFk!<)tvo+M$@@KtRhhef*gB8Q0nU<>tqz!}9KsLw@x_0(6oihVrw z)l*;TD)#ZzS5JMVtJud=Up@7ef|HB->Zz|3oLtmbPkp7}MI4Onbfz6`gl*? zsPCkuzx%J>vv@v0KKbveleOHF+hp>j;XcA;oBH`S@fM-z7XzTB5qfa*o$wp6 zv*vedwjD>-*mWp4M|<|H%)|O#<@Wt4P_7=IJ?ne+u~9vH*7pMXZnGDgWZsz@(Wn4R>r`l}Y~8)8@%{ozd`;?`q}L zk&6h+cco*r-|}57%De1gDQ6At0>bi5=#T7y;hg=JHGb>6YjYw$wwH5i#gXkdbEafG z&ll`BqrE8_%vrS|^pCQKLC!_+o#L7 zsStHHn|*kxmC{b)FX;LHS=N5QcN5xgv+#Qd%Ds#uWq;V)WgP}E&Ilcz&_UAd!2wI% zRXUltd@HRk0N!43{K)j;_=uN%Q1T7zUl+@Fu6EA13T8V0R50v(1(f`f^CwG>QKl%5 zr9f4tMzpg3h+bYBd-7knUQ=(qzjx^N`}@qj{r-c2^gX4}U&GV>Q>)b>&USa?9~hl@ zXYD!TRr=P``QadI;L3j5Z@p$}Wxib2b--$0vrxl?uG?s5u(MfRg!Xfr=IyazCX%1{%=FhT7GB9+IVNk zx3Qt@tmO++d>a?0SUdq9KTk4G5>Lv;*DPQ9!jxtkk6LjmdO9|~P>)Oyz))ju%#-r@>eT{Vr>2E$~PBobSl(*e81t--?3l6=wY{{cFmd=_JxGn1KKS}$B{^VP}69^7_ zchdQ;l3pm|Zu-D-@?wA4>;ZOJA?0-TwmH^`;yDw}+s1jc^giJ2EW2UAg0eoO-LS3C zzvGECIp0`Io5uK>wq?PXrO+1{zDvuB2FU&Q(ymCnl+od;Y_@eS5IboT^L=g9ehb;_ zTg^J5?4_0Vwapx>usw8{Z&@3@SL=V7&HzOlcFqC1<$Cwub-7*$ROLFk_0QT$&U5_y zyKZLV{HvqzV-8h*n(w+h$^ZZKJvbA(a`w+<0ITJ`X7-GF=WdLVGNjGjNn5#|XED!p zJQwmzV{XRxqRLvvDRR$kEo(k$^QLY462DLK`MExtJ)hX(UQ1b_FHFxT-A`QeaTVLvkp2w3IZIHvc>j=FBPJ|+F+z;B zm!T8u5!)#X+xfnoF6H5UWRO=GN>AS98~0lFJ{DSm%4xI{!+7t)`(R&EWkH|CIjO!t zNY=kbqWeYY@ilX-9q6Ea)S{@xyozltd?$)`pv#}d7uYn($NDlp>VuDJDc^(nUcQq# zNegtI+3ZB|*Z8R?afRqhxTw{ScP2%WTr8<4fZ7Z@paeu0h^ zq>&e6L|!WGjhmPJN6-m!hiZ-FZ;IZBlFz%eGg9VVKhkIPr;q4Q9&9;_M|cgk0^P(X zPf!;tEMF)US$B$$NOSqub=%I`q>r`9pWpwJhMjfzzMOycbZ>24+JQ0lP@Wq0t^5jE z8`9*h^G5!kMwR7Hp0#Jz`dL5b?-+F};gf{ho>4REA;NoozWrY= zvo2~kd;P4v3$6V>65i|c+wU)1bJ01hZ~hUSV$$9xUjCi3@~*9;HZreynKd%%Or8Am zqdNJHe@R*4n}YKizMH|CZ3=e1miP3ixA<27wy*h~CHSKC;-|3vsZo8;d2&{tyLhf& zH>*$Hzh-5i&vSEbzkhF#^Mc5EZSm8iI16LHEps2royaNYtiW5`rTlC9h1pi~khJsP zGIyK-@9)UtB=(oCp`BQ~U|CMwm#p9`<9coz>`$xg)A#P2-;eLPjrA4PKgeDz;d`OU zr^V=&wD^fp$>dM+;wMMl1#E_%VuR!j{mN*`pLF#47kKsH93x$Pw-2A4gZ^W9=JDLg zb2ew6zQ>MYU>~0JyR}b0z^5-0e{}JtQQ}MS<7DP0%0B9(_;D@mlKAlzk!7p)NTIIKW?#|iN#4jDF zlZ=DLTtq$K97Ou>*f%P=?8W!rvQ8uVr1xgMrX~Nr;@M<%JyXj0AAnN68Ox5Aa=gaU z?c1k;q@v)wYq!es4TiVbAet+e1 z!Uy~jKaM|V zo9M^);!E*GI%BQ$_t9m$KNGrO<$Ct#h@US1Gzr zp$-cy{xM$s5ba$%@>%v_Nc+n7hcS{SXB!`$m6!35)h_hmT~ChMM)*SwWF?D^2b@LXFoM-^|dd}x~}t(jHPt7ca4 zx^=S(n&6L6_-jYyf;%7HtFC=<*2G>fsy4AGfBUGSy!o?YcWs+hf<7fnUKrI4{Ewq0 z%gdHq7oE#pn;%EdT^^P2P#wNn!c%nkD#BSywvH-Ugx`m929UAw8h^XW13=%Qzld6cD}3aC5%+N)3j4|I=HSShAL0EHcK<~r z}(_z`jAUbkxL66t4 z&x?~9kDvW)2kX+*E^j{89K3Ge6|zQDgfB#{j92B1@8AnK1JB%g2LCf@{!(N^oH>Z> zH55O6o$%9a^)IKFX~TC&I?(lnj8z#+DStACP;FmY6Z>WEtQ_p25`Rd(um{S5lvMFc$1i{oZ|f>2_?mv~c&d0bg=fp@#Vr>mWR- zp@N+S{Hr;l?%H+7v;XbRHTkEc{8vHNAcCy37g#l`Qh#y5w45#rwr5!(JDY!obeLSS z%A$-%tdxxrs~Jx+PZCdn$Is*Au{O5l|8f@lc_wi;Tzl@$t_`#)dhpDg`|dl8r);N&QafaCuW5PFJo)!bh2Ppf02dm!)f-Eldx%6zFK z??#sVN2nYBPb%|uOOtYeY-I57t#-?Egs1ay#XPxVplnaYfN8fDZOdhxRn3~j8_a7K z_}eZ&^ylvOzy2M6ctOzf|2f(Z_6vscf3oJhN%(H^Z;~&yvW&jy9+Q8fgR1ZUCGX^a zQp?SMr$kr9gEB02X()N~i08jkP4Oh}WKZ+~sdxV&@75&p?)3ki`rq^(G^Xd4#bfsNSvcl~bC!;=%kCLd?z?Nu zosTZc7#g`dWATSeGOoLCVaA2N)a5C@3ERTHtF|4p!bK}N2O0KVy^X!|z{qyKpPU`; z5ZS(2U}Sz)UD{wFjF9<)KP5CUkQF*F&?aUCE$btq0ljStR($C zD{ot!#Xj-dL*YHNmzAVH?avF%W&LX|>F-)+ZmaaA=yZ$zjB_nPp1}+Ka~85sNBXN2 z>_2+bnPbP|->KJF+oulU>BvL9o9aJv+f;uZ4|<^MDfEoLg52-egS>m2yqitltsw8_ zkar8nyCLM=BJ%E9^6oSGH0cARjVRcTpITBtkuRCRmv+&z?!%dQLMo_lemaA21^3d;hyq02{v$2#OoH`ed+0?k6*ec{j& z&ZPgFe{%owb}M&DRZ?!Z@#(oE7Dm#iLZR1h%Lu=ATgKSe%CpDB8+>z~;>=#$^7r|C^IbV- zakj9~7aG|OpSdN5@dV!|PO$#dfpW1W zL+-TAbi$<7`Xk({XKnwGHfkkz)tyJ)$u|JuahN^(!q)=NrD^7&;oiLd@{Pn|ek$kg z1V4#%S(jztC}+|$g50$l-)AG%xBd zw)74Z%2;d~bAZ#yImB3`J>}_?wd^M%8;X2>_O1r&WGz$d_NI$W*07yY+I$)F%DhR& zz%x{S$vM{W@};bgH{~mRy|-_dJI5W{!+$f)oF~YgH$m>@k@Cmh$fAqLvmkjUbE`=$^nde4 zlK)a3Q*Tcmdr=$aWTa2(V$DgVEUWu6q&zlBW88aLNR^ooWoG$fWoKKLO1W9ki9J81 zR(ct7WRCV6dbFbKyhYjBh#nib15x;Y%{rXOm-62Y-uEIm)IU(ThVm%9)3(WdC24hU zaDUG|@SemNa&3FgR>Auoc(;Q0K6tl=_dn$x6UyZt>=PdUWY5DI{xu-HX@}m)O1Cl! zc&2f`SJmxFV_!u9gxeofJ#xzeF|-J1*V zc}&KzYxCs)oa_ZOHf==i#twhW+3w3Oy0?>+o~>lJa@Mr7`ZVBQkuPJ^ulW|t`SpyF z>cH5ST4>obX&1Ur>oGvSqewlNOg)jaRC3>f+_i9W3gwDt9eQ2EIEHb-%AMS&DChYkr0>O6`g|D|EEuyUpzj+J*>c}di~o_ibBnrva{tQ#U#q$Z z>uxO=^L{HnOtoSgr_Daqma_we--sXc8*S;yJt`&auX>63OZJ>V;hVnZMYHEu&Ku=p z4cOOG-8oc9_zl*I@8OYeht;gLu4C=sA#{+ww9wZ)lsazOJoz7Y4fSH#?oI=yu%33D zZxz+x3?9K9lH94wynjx6*7pXNvfjhqx?P+J+rb_($(Pg(jcx8TT{ve;IoZj*inmcV zDk&SyvFl=Fx4^EuRr|XqXFPc%I)6#NRC70#BRjW|Mxnz@#tjA%Ia2) zI(0y&0*_7?$-EGqM4xvq(E3Q(sO=glEbs+G0fmo#XZ#oanCJgr`;5Ds6F$oopQVgl zo$t}FM-%;Mn_T^z)9DAkl-uvxMff(wJ&9yp-52r8j^o!uJTeZEd+b>URrkcy4jHiA zmz+d=8{)$b-w+tGsQI7z=hErMU9_S9^-kS+=p}1+Qg@`@glJcbfYp4%tE6nUq}`J- zcMG-d)S~Wd>W=)^ekZm`+s5|~JDv9DZR*b3tQoWdpL^trZlDadj7a@ScdFHY#9KQf zDQ|Vur?-4ohU_n-Zd}^3d!Ug0g+)E*O`q4p*OEHaqOuchZjkT>Q-@UhAm1yfLy7SL zYYIW?QNgf+<-yWOm5dLhFI4m0p+5S+#yf0I=|dLL|FK47u?{5vhp$9dE&mW!cf2w_ zn>5J!u7GkCLLc-?56vr@7v!6be47X+MXK_te+8^Fqy{5ZMajX?a%6I!P~~FE(Ma@7 zYaXfMUfs$hS+C>Dc+O&N2*@4k}H#Vf1USc10q#Nxm$KIc^y74QdQk7 zrE(c*;^))#$W<-6V}UyJ1j&al4QNgm2N za1YKB{mz~=;BCv=GP`QlycK6y`zh1ubEs{4r-o|y42E0Yutg4yw?#s6J)8D4w zP}O5zFVcIk=k9mE_t3U!Li@6ZmOCqdM$1335^`HKq2*s$3Aw31r`^bZv=Zs8Tc5(a zV#a`toXHoR>8I0Yqi=Wg-O2oH8TGZ6@7rnkNc+t<((+#tp8af z_lneZ^%WL$E=}AUQrp$fd*i($wOs?eH{Lr^+ck;z#(PIb z`*6=Y|HbB>6w0H`*|_c9Si_i6deydj&$uVNam1Tt8(UmdHf!OeZF?{Hb@-c$-!A)S z;i$6NrB`n&qYrmjlPIGdn(kxn!J3SouOxjZ-{fd3-%0C^UD(1m$bEg@>1em?Kcw(e zm0PV4iA7JkoI?+4nxUW`Ik`! z#!?o>Fu!MBKUB}_`9IrA*44NCZ~sV=dHn+TS_cd4M=4tcJ-C-(e|I}|zkmOL>wdA} z+?iKzIPcO)8+!XnW9KB7#yX^y#*#Zk3iEx|obQ>}7fcS?ftPojC|DS@+X36RDUI!< z-X0n1+wfV6wP6AM+-E6~4M&ESZcx0(Ul_aY8f!zI|o(#ebY;1Kl)^RQ$#-1u3^_TWF$>&&?mpS98Q`FQ+10iGnDWS(X`DLg@*ROW19XAHE`cNgNT!pVK@ z!9T6p&9?^)CGNZKcEL|;mU`!v@_kO>pC+y8J$J4h6_!;zA;8^|*r4LByO8;(wPo!j zeK*_9ms?`PilFa;ScT8pD*NH&Joju*JmcK0f@8fpEOrKYagaS*7C7C@)%~S&>H`nt z^ygjiNrn$=F$(`L!(!=z4_}p*dO<80u(ry#1z9r|KQz5FPriq{d}_J7O!$iL(>=0W zUJ1S48WuZ_Kc&n>z-{KiP3YC)f>;87|FBq{!7m5@M|_dbH}g~MNcZ6Xby)0x!CwIW zKTn6>#)JRSu-Ll>e=YcjPluoF!TOW}7=z5}3x%9DG~v8mG~`5k;E`KA7n5tNz2Lv8JXE|CrS zzSO#D)F+86IOLBNNE_iRvh#nF9NWNuuY)bPYwWt=vF|A_JCSL1_uIqBM;RmQvXbds zh~B5OuW8vA+jKoGu*>~P_DzQFMqryFYtz<0=hotITQOS?8F$>!Tv3|nJMX%A8ljwtG{5|5q zWsjxqPl{1Vbe)^r$L1S2bzjzP!p>OaiC*5aKWeBy)7uWMgzw5b&$1U>gCD?u0A1a% z_Ce!6?3&u|+Aer!L)#x;uk;t$5B`z43UU32Tgtua*ghwTGfU;<%eV4vj&G==zvkkvO#D@j4%PVU(;@8fH2x~bU*-6#9Dh~guetb3tx+;xP;%etZZEwtNy{JDl?Ia-O>;Ji(U3*Qk zeF9zA{Cmw(?Xy$=6ZW6lpr7XNU@T=XX}aV?kOZ_JC(U?8+hjOpE;lLg!~cgg}ckkm{X#k<@#8{e9iv-YQY= z5zF>5Zn2?_2YmZ~xY9?z=4af-Txpk_1K`^_`*>?Ok0WI=^SYt23huEi@L9#dK(M$r zsde%0W?98G!S=;V(+Y|!n)fb_wj59_YZhW-o5>e)R-lb@1Nyd+c7?FqMS8vH&;4cB zw3Yj6+c<5cJqJ!9Uie3UYIP2Iv!cziVJf}63H=pK=#vfoSVLcNvYn>CSJPk8g#N-N z^xxA~iTt64erYUU)7zRp(uDq;CiI^f`hJGK=6F|4|DdMt-Gu(ECiH(b^qma-?(chP z`bRZ=`zG{lo6x^%=);D-w!T=?|3=fdZbILz3H??>A8_e?12p|QO>Z@!|Dp8M{Cm>S z*Arh(p4!wcHRf+IJXYVy+8|}eS2~2e&kwiYe`*c*|>N7FG6SP#($#QC3G3kac%QlIioxtK|O1^k=V>*-DC%bU=* zGW5SN^iAvageLTtG@<8Qu&UP=8~Ud8x~vI(qzU~uhJKKtZ(6U3H<>>-^0*1 zt=C0O=-W4;-)rd4F!W99bxsrd)=lVl82T12eUo~f)`Z?_LjRnh|0zP*B2QOI`yk(r z(tlpB>%jT{Qm?;&u4%nK6rsK}?W>yB7t?ng23Pf`%t4}z)unE^eQkc8HK%sbf*f}o zn#wq|J>$?IYlCiI>*BgN?pRgEsVvl6AlD?8VF;>>9v256> zVZYx()&V;Ft+m~&gB~Mq!B*x3-_vl{3bZZ!j*)jooT^%L;0acgsVT{Eq0wa*Y@%Ku_Hdn1W*2L*AidwZEMBckDJ?3;&ucw zP6XM%SeqN*KTOO;Gh5p076tYn-j!wZJ?Wh!9!F%b7oawGXe_@2=gfT8Wi^2@C!&mJ ztYt%g>&mr|%eRxCCdAfjja`&ZeyG z7}BT2y0|oER$J$<4tG4rQ<>eLH!(eRo^-whoz7Qp>(AR}WS8_l50I$X`pnOvvq1R1!v(i>G|` zL`!>^;4!B^C4N&&`+VXLvQ}_P{LYs40OHFzYg&xXwfA2bt0nv+^GETwHKH`;2M&3& zwf#HnkTUjHbQYf-16Sc(S{i$qI4Rpw4xS+2#h>yWO9T)3XM6Z2w4H(ldZxZgfOL6ZdbYLY7RuVe>7}vd_+0YgT+*1Q?pt|dk^R7p*8YmNrLpq1 zKGpsekyf4Ki&f9^#n#YH3f=97t~+TJAA|d`#?9wl(r-5Doh7|5>3wy&@+avtO?sZB zmyuqk(_6{=KS~pGVwqOjTkrLkJ- z-`(f?SF%pHub|8DSoh$+a`r{`3BT1h? z`UIW+rb&Oyr0oFr@vy-?>Fgxk@TxczgnliWYX_8=`TzA9i-o()3=!PJ5Bm_ zNxz2lYjpavCjC~E{(DKklk_`v`V%JoMw7ll(yu4|dY!)7q)#{L>m>ay((lsgD^2<} zCjC)KpHBL8oqmr=pJ38$Nxz%)yLI|vlRn0zFO&3Hq|egnx0&=ZlfF>We@*(Yb^1;6 zF7@F|YfHszkyupVne0mvcn;^6`By{5+oTJ8hjf9IkDx%x!EXAxABI}GT)ORdMSuHz z)Bjxue`&u}+g(~3t1)^^B3{~F6@Ni#Oxo{4Lo<#v**_xRcMn!C9CUCI&k~;H3kQ`m zj0fd5MBIQbW-tUWT< zxyu|?SQ9^kHj?%2cZAm`hF1>h!t1@C;#=F0CbUw2 zgf?twzh}%q`&A<2ROLH-F8dV^o3=a^JUtEz(xfr*F`qoRg=a3$Y##U>CGGI8wzl+_@-F$I+P^l|mIiF8Ju)Pg zOMc`?J`k3?$>+}8a`Hg%UxmK*z>rv*s$22D-g_zi+XCA83EZc)CXiEC%KBkb8O^{E z{tuc z=YZeOwl@4YTIRz!jv6zRx3{Mme>|=5a-Am){(1$uD_y}%XASVU{6}~=_pkYbbw18k zx#iU@cjBkN!q4riq%W%HzYwY%e;MB@a#z6T81NBeU(h>FzHOjgYWn#y@Z2$AzTGT# z@#*jj&a*oh{PV#tr|wqMhTkM%>T@;i{TvC?HdoU&&yq0hcs1>KrG#m7t7&s9Bux8U zP5XSEglS8wX-lt>Fzs$N?d~KA(?(X)MqVXh+S6*<)A16f?W?BkyF$XWlhw47VI`Ox(%&-o{LFZKEi;-sz1 z0zRtKHJ|oY-_na&SDX@69iga5x&d|T&R1!>=aTllAwYGW94f6BYkopH+{!qRrDvDvhd+iPiy zs3R+elxNygfKv`;qDx!nY}!rN4>DF$^SkbY?8(jaSZy_YR9wR#dqI)@o}}I{I^VvH zcNr6__nqh4^LUrGQN3plwr}EH{HEUL4YqIKUGgQLJtcB>Kzt!>cLVK26CE~Ly9e6~ z`Gz6m#rfz^&$pQ&oShNSJ?#g zo6su~iv%LEW$-VAj&eO`dSKaOabP7dpZ${EUtaYXcjnCL0c2daY3}1a>{-nD4)C3Q z8aQ8f^>6s9lW)V9y{rvt&X?!2&p@X5P3ArSK(F$D(0;v3d&+;_WdEA{Z(=yX*v4^6^~oCM@c2iyaMn z{jnocL`Igg036B3`n|%RyNr}hc}^KHvMLfAzsiaYCmbalg_q)gQ1h=cv{F~k2j`;! zd;^Am{T}!a75@A)9vqS1ut)1(toVnWzQTV^BsOl173(GZ2?t9I|8nWu+BuUAt<=l2 zz!@-D^KW<^{++b`m$>@BuJzAX{PUa~;lCdJ*E1#&{)B4>x%zAVBRu-I1ZVsCM*m&# z4`}@dxccwX`ll%Vj8laF)9C-S6^jpqKjEb=|MG05e;<$jKY%mN@Nd`w|08Xb|J%Fz z@6h^x1D>=yUjie~p#L*g?5OZ3T;TFQsQG7m^#2DqG18U)UxfcI;ms}N{e2$&Ujt{J;otB){MT#!zolJN<>z^={|?1J&v`-k zZ$|&kR_uAGw>g(^{;XDe@5$ni{c-4ZWR7I(0_*&n=Ska7r6Y( z^OXM5e~Im)|MlRUAYJMIH2gbh{bkIe{QtDpf0E*#=S&d(yU>4^6}wXS6RthiE&rPT zGLQbFz}as2*RO|vKhj86JF}_FF!-+FXJ0m|8u|@ z=jy)({zo!Z`7d|%U!(Q!rTFJLMZ$j%`tPw~U4=j40+;_m&0oehuK&*jCq}v||Eu7? zOZaCyQCI&}TK{&6f7rtwEjmG zf94g!{{!^@z=|CZ{)DCdl(yp&-gQ5auk2_A519u^IeL$@2f!0Q#qUGT6p<5l{!M$P z~-N!xORZGHA;Udl z(}#E&8!iHtAg3hI&u)(%^!u?ph-=@Ea6ckn*FSYF2VCyNb%%egrDiT&+v?wXgz-fO8H3W`CfB-nujjIGq0sBcb2(! zyE?vP_|?Hjbo?AR)yuD;r7ic0xO54A1+8r#cJn{eM4IbM=~Km*?;9QW19g2f<^QYJ zcEm%M;F+0WzYNb&o-|j_`wh>Rjh-(72YL1UHp3q8p-b=#X4(sxH#hgBxq9Aics^_N zd>UBb<@r*k-PS{w;JLGn-2pv2d(vDzD-6$vjh^=d!(N_C+uFT6bP1ko+bN!hxzq3a z0kn^%U37IFWB4vKy50u-i6@co6_IPN(aexk}gPJv?1K3k=UI;Uo2E4Dekq&y{)h%O1Lfp3k14c<%M2xq5~T&mr&; zJqH2*?_EB%-7lYZ+2 z>EkXkq>+f|2gHR=y9lb8-!> zw9yB_iIJ|xJxk!az}|4}QpUHB8O(ERreDes{4uQRlw|0Zxs z;4i+~D75sa@oMB839Iqjhv3%Zr-o{6$J2sS7m1CnvtsLnKjEOu|DfjoE;QQz+0G;2 zd`P8V^R@mqc;f$ig#S0_|BV$}Cj1H4j8OhT|Bg!k7d`qf0H+H6 zqW|qeEBfDpob_7&8dv{YwEnjUPCfe9Td_I9pK#RWe^B#Z@6o>koGe%Wx$s@2^}pZn zjm;Il+0JC}ME|RV|8e*qw_@XkKjGS_tA8h@|8hes{=WpAxh8*Ov*ABQ`1AitqeH`N zt@jAUKie55{7+JTPO|7oSE1oCcvTLU_uZlmHTJ<>J4%V~J zVDaPpSDf-Cdj~|m#63aW`4TrNtm9~_Um>nU;wCo6eM($^;>6C5$caz&$EbJvh8*l< zf7(g#CLARk#9k%0T*kKT92vujyo2y*C}v&pVqG3$gjIQ%gq$N8syxd$MV04C zTDN+IQyLo^@Ws9+91ZXt5qfQx=AUn9B%eP8ryd)|UZThP2VEHxk+BOI=n!&cOw=+y z7Mv7hr1)a*i44LOqZ{*Vi||DMsN~m?P?gTNP*r?VYL!|?H~IEg_{w(y@sFEt`y_5h zhK@7&wnyS#Y>bn9dyP2BH<1+|>&dr8g4fQeA8AQmIA78Js=OPkb$|w%D!WE+$ZN7y#p;I=GH8%H3t%_fn zQBpfLqvY`VwzgYd9~OHiik`U%yQTgezS6`QyYH2_v5j$J_flnd*!d;$;=?_5chYvB zBrlJ^_jlN#%EfSP_e}65zbk#c~Ko30L4dU01s(xfdDQgYZcg8I%brQ}zAezX<;L za-i{TLqDxkn&KaJl7)XeeA~_!^9z5%QRyyTN~`wjs^6p_{hh zBgLQoU-;)^L%uJzU-%Q2dZBI5`~w~vc7r1}j2&Za=!%R^+6Ec3h^?`%A|u<`sbu6i zFN%y#v^Sl6v27xQaK)vr4PBKDUm9A;hv!7bSZU9)oo9ts`p{0u35YFW=L2w6o6t$y z@|56o#YbIzv9-dVaP%^3%fGTXYjvg7U#$x~l5S5SP4*Giwp)|)JnQ*V2Rz^OHd(>0 z_REapzB{SL;VLa#&8@@EHOQ3lrVS4nZ{7oJp~``p6K(2{RA8U<(3!E}ChL(xdop`t zQX3k1hRuBaV#D(`cu605i$ESTC)m`XrpV6p&?R{G?ykn0uY2UVdJZr=r@}|{yass0 z%d>qCd%uS+!E<#_+lQV9JZY|;oea-0@DV*P2EOg(xwe=6rH3xTGp~=*^EaL}SI?G) z=OFlqo&$i}y*!)uwV(FTC3s%d&t6A6v(=O4>M8xJYCes{LyTaIIq#+u3s@0!y|(%dzb6NL3z zP`2}oM_xjg)x}B|nM-T|w<)h$L+`F%HBd($_3+6uW$TZ|7oUMIb>$#%vRCih2B0_J zdx)2M_6~3WdS|hQWl`Vq1KhJhoW+_`K5JNS5Y}r!jB7peT;EIoDt`J4cp~o=;00cJ z`%09bt|eaN{SnyIw%-#cGAdYq;(WSVpSzj0LARc|WvAN65}!-owEn@ zS{qq{BeKo~zQ~i1)qk*(^|6uFowTO1x)LY+tIRqj`|J{BB*(PZKTsZ|Jaz$B%81y$ zj>pK^*MWWTQeSr}Ik`>-X?zo1F@f?g{mmrGe+%Lw`DUuiliu;$+Klf);91Z&o%@5H_F=7VXao((0!|Pr{uj(~=C+f9_m+PkqM)zgVi0+Gl$zI(X zn%SIRmvX9XPUvu6&|XM?Hq4Xe`a{}J@%>GP=WO7&Jc+u}I@Nwj*XL7tmZzyb&~JFs zTs?amo|6pE3BXUhJTD8W{QR+>_VGoeh4HbsKO0V*Ti1sX79Y#lT*ht@-s=a-+JD#? zL^#G=P0Gc&glk9Y_EF|<=kQ(^Q)8l2b>H3GUT5;;7&^@SnGM-a9KVWRdcs3|T?lk| z5`Nj)LfP;aV?!=!O>JmPoNGgvuv-WAdgLYY=&Dxs%VvM`UU)X;6)^PP@-6+cH*a*F z<~jR}?s52t?&9b9&?j`CC;xEFb{;dje+6t>#*Y%`mhn#si|#TeUyJTR!l~p}!OcY_ zQr2V*W+ymOch%gsI#_l1WNS62OB%@?^OOf^C*A=ko~!0%?9cCNf4WQbXy-`29zmZp z@PtQ0t{!u1--ex6g!f=87He(h1ML{6uWxN19?YHE*eW#j?Sv-myacYudL9_*#(2G( zFSdnnlyD97g0qSDc-ZJsUSNN^Nb7Ngvh@Z0>P3gJ)}coDA!j{st}DM4@~0plUNU|W zAIDk=ezvn(@R9#8Fw%=TPcL6=CE+OH0wq7&S;2cePuP4Ly>vDy~uyRNF2xwOrXHrI2p4)BtCcM15?_lyL-$&)C%m&rbeT&J$D z_VXapuJW{NZX49f@HwA!DIX=k7rcCGGwkAEV_TVR`Ujt(>kY2tYd7Gtr_hD%tkgz2 zS6)5!OZ4stuE;wB_**aCwQcNNk36wieDs;2%LG^KZ3UF_k+64OTf47^&Xp(qtH^5x zuE_HN@AT4j&a&evje5HF{l(Coz<*+2J@6(k-DT|aLEcjCWC)JPQ|CYU9=)l9@%z2X z@7c})VEm*{m!E@_pIxm~U6wkOg%3rihLgHHe4uF9{{xgVoPCBV!^bH@>nTH&t&`{_ zH1)@YChWYSX!4v_fsuif;eo!`p9x0^%YJmh`4jK)dZR;m5%suHhvV>5^|)T^@JHc? zoZkZ;JznSf z0Qo~3% zFftq;4EMz*6OIxtQ1Y{#U+^CP#PtF4cePBEgNIxne4^!FuK0zWOMt2^%P~IqK*|9+ zpj(yegAcS1qKYQZ848Sy#0MjNu?XQP;ib@v56DGS_Ge+fppl=)dwh@U1LO~Flz%z= zq#T~yBm8omOvNwTX$5rK(vz=C8PIu9;_Cal);Fl|bDaP%G8X&C`eGL0lWtpjXn3q5 ztFbMWdY;=({F3XuO}&;j;-@~kZ8-s~QvA$%(%*WJSA4G`UTpjlxal-)sr0GBr;cJA-46_V={9w?S9s*P_DwK!%fJ`=76X&Lbl-Nd zpZ3tX_FZJ?<{Npx1Rm!}*te;xeUpdIwNLs^v2P~0VqXRD3oqU70{imQ*e89b&`knY z%F6`cCtkV*MfNz4JZ)cC^_@a@DY#q`19UkQ?4@rn)oia zJ?KH3*jOLupquD)a+j`;y%qnk(+!wy#x^H+NSoL~wFj4jFZH)!hpvyE6@IRh4~$&N zdh?aOSRUa-ef*62$emYiU64G;Rp*9soiE8Nu`e5Zu`d(&AWtG68hTM5!_F3EN1oG+ zv>^5IXZ9LA$JpBdjKAo%KiIoV<|cfXq%S_()&E7U|1m|w|1g1ab}D?MX@j4aHW-^} z8?%g$>Yo=H&i^QyJm)YlGLbfDqAzxUaFp=wIl2x0g!lMn*XKQHgBx{_v5oTKX05}! z!Vfub10R51^~KLg8!SF&{{nuJ@)mne@UxxQ1)uim&%nqe+TcmP*bc%`!ciqZ+j)`q z_%p5#kUz9h{xSGTo;N(B& z^}!nLgS!MDA1nk$rs9LCzStdvql62T{A}k|-WkuY7;;d`Q9I{%#s<-CE@@)xqefPI z6|#0A3msOwx~Rf>#C%66qMcAdx|Tr*p2%yTC59)H-ivA6x{`sRtU_L$4(Va;be zxMJgFz`4+?e0~7Ai?oe*xHdkZZM<0Ukr4$(W@F=QU+e8=;1IoJC=O`M+|G)@&GoCzqoN$zI zROt|Qe#?8DdZ>NSk8;qc!w2wF<$(GrIy@x&@PQ2syYgv6js&z1UUYphMf>1x!H4cw zzzAh&BIWH)!coGzE42^q;5|-TsC|I^BWa0p@Vx7TNm~9d6~C}E2Pi(Mf`{?}ZTXR1 zoV7)_$6Oyw)IOM|Xc+$kBh-)a)SJnKql62T4q@jPyvMI}ebAqB(5S;ht`Dx%I$SRN zkaG#Je!7uAR>}eL@%gQ;55{UAj1YW$Fbo)>PEVxXmlBQ=u7O_4!64q_qg@{$e@df# z8Rw{SFj~t$Tk#7!y@BF`@vaUdr5tDkSYA|K`Y?9(~J*>N;%LzC~VC^}%2*{}{MZzP|yA4@z7e21+?dRplVd^}#^xgQJRu z{vQ}wg%4KI{}YZ9uB{LqIN!{Byx8@@Ig|sr6J6^d;~Z5NinR{!3O{`CHgGfasvP!{ za)5kzouGYF?Lj~7gVzNgAN&~@S%VMO(Ek&T60T74>Hm3;_i}xJ{3(s{Wz3`a_0s${ zD}MC*gJB8Mep38dI{PjM&g7t>-fJIvnbXE{Qmven{@0NWjVMl1LGc@;;R*?d3ipKpF z?~lT}q*C^J(Eouu!Qd_t+z_}Sje9Ha!mk53ml~Wq1g8}^tu)RYg_Estq6X(?!3l#C z);JZ0&;R6UJ7!9HJJQ?f^m4;vt4Y63(sN1A)#>ASXFtRytG0{%tiicTaPq;)*EpkH zoGoYB>kZCW!RZ7}CyjHVi*s#P`!R!4COBQe>8f!Cxj5$)*tWqb6`XG1bkjKfU7XH^ z_ELj0KyZ42(@W!YlXv`7^MSQRd{H3~n+pVDTkZQm^sfipE!Z+#Khk3%EZ*Jv$bCWXA%GK!=jrW8FwcqgFk`^-Vpacom~t%j?O&wTU#{z0yennBldf-h;HY-8t1jzx;6)Bl z&ky)wpAn7{4nimV5Ah!F3N%Q##Tq%$50aaTAj~IGLIUYh8eV=wr zedo9{aLPSYyA@rwa}79efm6{8e>Btnc$u^cxyypRHxe&|}DC|7Ldpy@|zXsT!?g}dZ$sFoa_^GxaSKGB- z_+iIyf&C1=m4ayvM_?|Iu?%qn4UM&_dRt+cOZD4JYnDll@qG(4%8zXNRxFT;baHp59puauHBQICU%XU%>-8tZj{e6H^J`#EkINRoDII^a7io6*0 zQS>YTSLAg9KH;THDz-a#=v@2i3|$VmVqY8JLteUZ18o0k?EAZ+YYwi+O94vxNaz_k z*G~4xbM4z}=>FYZ+ZO}g>ZO}8(9ZMFneUJ*th_;XDc>P$lYVad3x?+x@Dg1<1zzXn zS#Z8RE&1nk&l0Eg?8M^gG z-fw~VUb>Y@_G*tj*S_lw-GktZeJgq^cbI6|Wp>yqPZ|FvYEAq;Kd%Se}xFgU*=h`QIr`T5tuGn`T@PE8?pSG|IPh($_ zq3Z{($mtsOdzyiJDgfRXnxaH5y4cb5I3N1kinB?f(QJGB zY2>Xmbi0haKLH1O>Hd~uzvPkU+IN?s+X}wew+YzCOZRQA{jP`3_2)c8_q37sIIxqK zZs{5JS`VFT-*iLwDEMOEN?;o=-4kcp3r}O;RfcZ4k#`p`#Y@L`4gLxAv-}%n=xzsJ z?3*X_Jc;_gqJzE4L#OS_b|QvuHn^habYPvAZkU|u2|H;jF4wsV$T)73cmK|n#L1o! zwO@s?hwR6a{VMxHje5HN>}=$Xg+}VbDBxb6gq~M*w%?_1bjxR=uiDa0odq4_Nz=a0 zb6Odm=fg*AC;`6U<+-9eHe@>=6^Wc&ryFUJpZOxs`3XNu9q&%M=++fjgtQ~g)% zXSxmRmAJ3)6J$L7GabXu=Z1G1_=%3Kfwq_TMZN8xPNUO@hAstM(a8^7Ch7|`pZg#ViNOX&Cqa2UIua=!d?;@mwzM+mDuK;-2lPafu)@ziFc(_!$WtoR=K|9dKi2<%kvRItS^py(B^*uF>ps8W zUHeb=Lv#O+;UTiFAnnnRnQz9~<8>s3c~7>p036Y$Aue;lJZH4}3nebkIg_+YJoPhCuOb80p3kd2V+~zS z!_F7vnaIe3R&)#lrF$hT=R;b>%ut| z{tud}&oDm(ZZ>C_#a8jBE<*#6*UsTzFsdGj>{m%E;T|@bu-bztQGR9X^!3#<|W|;w6ut1g8Bw zk2hH-k`is_6mYm7T;22dC}Caqv@LHT&n>U_lcwrkA7f8F_AJ64>dK1-uc2PoiMzp# zum^A~`veygjuKu9o%WNIk#^1>p^^4R>e_s8#`o2H8@_?>6z!)c!BKwtM*Ha|@Wig! z!k;~h6WPl+L--ReaQPq9{C{g`C6B~LSu6d-=w8oR=p#e5?)Mly8tSy}6P1jxbA`xY zALV%VT8clTgll@cHVjfW{K{iP88}sjf5TDuchWZ8=5{pr_F>n`i?dFHhcj7`ev%U^fLO^f1rKx?*c6=28`sRZ@xeF1K}v)rO@j# zDfPLXQ($OC#@FDCKil}@UHC78Kl$Ab94R*q?`oYsSN!Q`gg1^jbX=2OfVx1rH=aIEu+ahBY zRhOREvIdBZ!L*};{jt6xgK(5P14TwpSH?jt;~I}GT}i7t*YMvA|3%uCON}k{o3*dn z3;&_mFw`H*5&nd^(@XlDeX=f^tH#G+r;fTT{l4HxztaZz5Kp47eMHWv<~m!u>Ux|) z+Dp36R(uj;i)))D&hEVmF8%Qe@4PEUoZJn<|AuA0q-euRhdk#Eql4f`y*mcH*{j2_ z6uZhpmykIxsP2xEvn4K1*RHLG=cn)zyAA-$PvMzr%N_PEU4rKf&X1TgDQ=pp=UT&a zuhH`jpp=hpUT-{yYU21e+0KVk$d|(4l!gt@w{@}2atKzbqnZTxg zsvyoC(_KecbU%C=S?3s8f+MoZfroe!vc~D}o0f{pcE$kJ|Hn+Z%rJX>l>%mh1U1Eoa!=JWlYu zCr6#V^43RJ&;5p{_*{H>@+|+#+q^s%24y?biI;l#60m7~ zd7e0TjTcG~ z1`;p2-wSN&m!-tH_3u}N-L~T=_RRjQEPUA6K4i{~m7a#z&BRH6AUwS7-j^O;uI@vO z?sK4tq5BNr*F1@OctZ9dgq`LpF59^p*i`rN#JRd(L0IJO^~g&2e04#>=V9T4j$)gP zseAA)zT4fLG4&3ARs5FZlEd?p^|<<>1!FFuZXdoSS+&9I=|egbHHr(g0I-J|6fBVzupR4rGc2eOLU*pmL zh@3}7fA%tn{-x+6dDE~)>u)K2!p=$LJ^F~ze-*g9lGNC%i_yP+l^(Ag15f&_Z-9}J z=s(gQ`-*Us@NSR(ZxSbVSF}X`N$CG#Qo{e=!n3LW>xdKoe*|6p0gwKRjQ(A<{^{T; z|39Gh-v_?W=dNecJ!8DEvHU2QV@k{YU#_FAI%8<;;GG{ z|F2c}dCnwY!?eYH< zqkpc}f1&ID*;@Yy_~QS8z(?_)@_!|`Lq-2=N5&?~|CQSR{S|(m(+e1xi2f7(vF?PU zglj$eHz!W$Dn$P^=s&Pg|K6w3Uyl#NPBwJ$DIWa;M*n_V|4Uu}Ptp3f24DOi0@fF^ zE{hJzhDqQb@!N-sBeDL*=K4w6hJd2UbN<~K8zwP7J`%Hi}-hO$|-w^wi3_Yyl<^%&$AaB-0j49_Y^(r;`AP5 z&o?-m1c(1Bmh&H9*{`+U#d&FveWSs7T5$N!L^=PN5S+(coO$enFgU*zoD6U>G>+}! zLG0OQj zhVc2Ni&NmUFEBW_3Qk9GI%=GmF3wt?eV)OYBRFS)bC$-r*2THVZ}&4e6@pU)PLal$ z=;AaF*hL0slHl|Nr>Djlsq z6LE2FO14`WoMD1jdP-tDLaH8@vmoX=gHq>#PU;Cv}K*MM`4#`(L8b5qEE*5Dixoa@24UgNy?f7pBT zxT>nQZ+z`@7|sC)=202c6mXtJMa}`7Fpr9srIk1(N@!?SY6=J{ik7=H3r$e-*0YVW z#F8d4tH-oKv8?X963`692^=?inXmGc z=sd|S#axZ&BjR})JWs1UdvzYxO3cuBiiu}2cowTXyL6t+R^rnd&mQ7g2A*Xq&sLqM zs+E|g@oXoa<=|Pa@@&+3X0#S(XgseH&r0yDRC(6wJhxhllQo|8#IpuGYgC?WohQdb zOwxE(5zq7Bd0ypNtn-*W#gQ7%GU9m=JTIy|={nDIo}xwLSx7uDgXd+HCq?IJfp6qB zo+pSWA3XUg&lH_!t(VwM#?rSt6e7k}2z{}Rt9;Q2)5an^a(28iEkJZ9qg96X<^Ja^h@{e5qscueDA#B&rp zM^&DyIuHKW{E5bM6EwQ_E%|Ge=e*AIY_M3O@mwOFGVqkCJQX^RQyX!w#`7!joC41& zmFEYY=Z!YvE{*3W;`uvx{;u+Tqx0b2hOHXUN#gkhJin+spX)rP5OJf%bCh^0!BeU7 ze5CVy8zQdNcn%TIIq;lQd5U$O1EFHJ#&dvp&V%Q?%Ckr3`7}&itns`>JeR<8N#)tD z^Slu*rfWQZBc3|&)TunL>O3E|6H_#v*NEo^cy6dX>vbMWdvS`!vw?W(!BemDtkQWt zZ7)8k@#GQDUGUshd6wxs2_3}I8qad#G2qTfgP@FY3w55KI*17x&m!V+0gsEy^MuZ` z{yuS##`7fcxPiw_<(aPYbm%Dd(s*VOPb=`WQh6rnJl}Q{BQ>7Kh{p>&UMkNxoo7HN zF-+rmfOxojI7jF__%7LBVW>bw1XUcshcoqsr4k=fQs#uWCFI#1jRcD3vEz=eZLpp4WIn ziKiQQx~V)~IuE{ouF!b=h=;p3)U5OT6ea$k@w6nKen_XEnhw)>)^--Z(RiGQXCQb6 zsysKt;WywNc%pVrp4@iof2(P3cVr~|Q3xluoBBU!nunRl-w)v<;0{DMxkC_6t_|Vj zCLw${+y@a(?j(eh`xwH>ordsXa34oFxpNRs?p%bEJ0Iag;XaLUau*|<++_$ScR9in z;I2eCxoZ$k?(+yI_eF%q!+jaya4X?k5Ph!2KNI&PoV`$j}RZyF-$v0#lxJUTiOypT588 zrsWq6KB`wAxcl)m=2z8Osn@qkekdQpt|CAF{aQF^l!yLbz?Zmu)OvYKSI`$pexcx{ zdIiDVjHfZ5vvl?_SbATpR||wSt(PlkdcB;0)p~hL&v5za_4-!J&j>!M*IlHW0o<71 zw_TNbZPn^U@@(2(HK6hBHHphdZ?E^Ye5%1q^*Rf8CZA7vH>F-Vpws@qDY#ARbpkZK zUS9+2?KOnUPp{WjEx)h8NA)@acjWK#Q~#Hc0y^mv$+KyD?FWr-uW&V=_58cp94()H z;H7%K3Ag*be2$0z)I+J44RoqkAzak!IIoABLDTE?3b5W@&Rl+ay;8LNHi3`o^&(tv zK0oDNNZeNkI@OEhiS|0q_pc1l`1YdzQ)2&|t5=|8)ACsgUaHr#lnx$EFOG)?(0!7@ zQh-8hBh7)^v|cko)9W=2SZ^=-Kc$|ZUawwSepA6m^?C&E&v+X1bD}e#_})8Itry7? z?RA{*Un4=IJd}48*fXPZ!*6nV>3z&!%WD|;scu%dpWwNdS6`)W6-r)hq}~W?TDNYX z>2>Q2O!>;llim}FT%LOU>d{8juOq=r_3Hq4C!WSUZ}nCFzo^S4P_c9Pzq}SN7`(R; z&kruOePg_9^nYLcuU08*uw+75)Alxk7KXh$`adKCR@=c_n)wIiJg()}fb^)G^>CN) z`3<1|^5TCLN`8UT1-MPy;WyA4+W~ua)DDSUUV1x_{Un*31wXaJX}HOJUR~%6YOs_6 zI`y3saGSQnm!RqG@CC5a4#<<*p(~dsy*t8Q-f{ZQmd*~)yYd|HH{Ir`JH!-*=HbsqD0ux{2bTy*5&u zU>fi6|9oIMR2}C(`tG0nneY#(Y4C3_3MXz% z6ztc=80?NEM!VxM6R#Nsx*xUHH>V&UWBhR@4wLDgWFy|MtwDXSqMn&XR$MG7d-dz- zK4rT1bCfnV&PUq%{ntdNwF}*WI~{Zj>}hF-wxKAgbn#s6PbM*%_(wwi#I~lcq6pZ1NPTZQ(60}wbYmIt)*d2J+qV%~)%%3Qa>^8Rw+puD)Yp4hM z3+@PKG?LZs{9$q~!~819Qetvuo+&}B1ZiK{A9tTvXt2ol zH;voQA`b`6hxz2~ehhQKKZ-j|jT`d^aR>anxXabJWZ!;QX5>q@>tE)(U5y+12XTY` zUEEL9xE_BH*ZtqctyJR*e-M|(i+@>WLq|oM4S-;1E(xJ{lNa$XHN&qfGGA2LGCJN5ce4lloq$;8&B0yWAwkAgAMRFm zEE$WwJO+K*X1_IMv>kn1EHzF!^bD^sTGO!4667aQ2Yi23>{t?qbYVMS7b!g@{EGxR1>s>04f_@UR|&Gx$M0(X z6AAc70m60vcM0+%lpgM&SNMNSkRPIO-T!lfJb}XRH0)RBo}H9?Ajg2lK$~iaFPiQa zSHxZ^w}s;g3&^xt;ZDP24akDuVjF>Ps)h7VV#37LWS7jCD8_U zLAd)Oz1p}s=sYma5~Ww~SRF(>iM|Cvk$9#6pEs@vG6}1L)-YDG8&3p{nb^~zHp)c1 zLC(ib3x|<@%2~R9(J(mz{^h!V=`eW&{MoucYnYq@KfZ%glKFM}e2Xx0G1bv6+!9?A zjQeH;cehBvEy~mux=8Ky85A~k4HTJB8dGR|WiWfYba3ImR|fBkWXBf_=^*;RZY96?*sY8gLvBqX-SG+q4t+MG zEcC#FN_Qh-oV)M- z0~lxf9+;6+_E65=x>vTGtb1sgTz9=}ZP^3E?RCCoU(_w!QCqj9AI>&~mDZ6Csj`ZS z=F@Jir9BvFm*WgXvC(j%5RVo7uw#n7>`P%dUmzJD8gSH!^>K&YTWzqsa{+f(WGoeG z3(agoiZc_>8{MMT2yRF3UmCnQYIf>?8tANBLqp%yLPpt~?+VJ!Zz-&EOa7)V%D18} za;e+qNMTV~AMkp^h89C!8z3)V$SV`_DuBFNA>RVXs{ryUfV?(9Ui&q9k^FU8(OQf8 z+%?SOZWv8bl!kw6?BT$!9zePQXWEPO{aA!6^F;jgWFwmp=*zqYPJeInh{=}dP}s;r zxSM_i?0Bf)Q-iw|CyZ!sSrlq!=~D@JDqR#BZs{oy?hLx%m%0r$-7*E=>p)g?SNywe zV9y*4eJ2=cZt5dd!f!yHK9E(@Hqz;6hvO{QQ~3`{5mROTcf_OdMQ@`l)a!3r*hz#@ z8&+YC94dI%Tpw?+m$$~9djf1b=Fmj50w2WO(J8=EpXoKe{)W*k^&K~=q|hyXG1;Li ze^x@WXn?DSksOiVdR^aOca(O25Bbr&pLDIto0XuA_EH=9b8WN}JWbmOg6Dsd0Q8Kh-$V&g&=3F(+8PN+6oqwNM& z-_C2ghH@&pR&6v#jTFpBqo6<4b5vb3L)S+B3tj5~UKdSQ&Z3_rpr1IQpUgl%p>h~w zm>21NWy1?zzhZvg8~R^xm!;>rFwbky%d&QI6+9F%Y^~O4wTz`ukLoRe<5V_EsXmU}v z{&49|U^Bkc;pZEqKXfODV@VQhVz<7 z@VJ226>&_kuV%X=-XPw@T8r!z&Q~V+q90Ubuti~*W5LeuHNj@z!g@E?%=+13!$!h} zt%eO72^*FIn`MH{N;O~{5m=uVh&u;!sRFV!u57|9(4iWmdsL+0el&8aY4fPrPYfVi z2Px{5DhK1Fleqlim$Tdeu_hUL-P)<^1sIT3{&M%WF66=H4M268pVGB47F3Y>o< zoj(iv`73N0&QwE3{G_+Re+TXilmY!w`JU@}&~74)%G*pmZ-@H}ijh8*iPpARS~+%u z4|8XU+M2B!CO2UGID43hm8S*!8RYjpY&)e>udMxpq?f_}9(03rZ#}RQ4{LMSDV(!H zJj8h)aj0+AL$8T%HTVWOv0}BJ!MJDAM>>JBP~9prN*C$%yH~ehX^2v{U}-Vpw%6)K zd!4?Rn^T%2kq61@S%gzw3*cVE13jj7`0?-`XphcU@&=904b#{~vf{Nu5E{U#ja@IUn%AxP^tG|mR3(wIyArW*NR zEt;BzahLi|1^No<_76dj%bkY4q{q%EXENvuQ0@lM{gFSHF75oAK6zHi2W*UZ06@rA0xc!*sRZ=o-SHj_Tbk=b?>hIs_w>~i*+~mtSj^X z=8Zc4=RdEzdF(=6$@A;V86&m%St?`; z`KCkfCZs}Fo|tXlyzXI3^g+<9psjPZ$OpCg*}8DcqJ!G}Y~6jpo-CboWnCojY?e;* zvvuj#ThG^8Z@rKncWZrZ+_ML<_AV73t}hoR)-yxm;;cIj6E-OQ-bZ>GveM^ePfRRG z>Jq_~b7DbRR77E2;<>|hZnJ-@OGUX!x7TvzU5oOrLwVmsc~5KQeG}z9jWx_6l|JCxotCZJI3en2zhw}PkJ{byq zDS^&LV}29_eF($+C>HZ0Pw>TJeiV!OQ7q<1(U>1i(&k462Jf13H|yd;H%omfo|~A@ zQaYrAG+r4Hm+UH2Tp#IfpxUO?hp4~S!%t_u9ZM3>ALG$4WBruAx)Ezq6f-wwzbuZiPCGJLk zv|i`qDPaS=#cc>{jBjAHcXHfhO^-f;v8x=IS$c|Aq!a$Bx7da1!`S}c(T{tJ9YMDs z{V=Xvmj}I@cCojRhX2u|Rkt^${j2#|L%jEi@)z)bcC?8n9?~VOtIAN`G64_v>dRW; zX^qDNk5b00z9MtBDEl?EA0uRaI@pl)*uNzVRb^0?stMeg2u=2zSuj+ zOmZ7%vFEYJx3+vc&W`$k#?e4L1S58BE7GW-m$?}lu=1bOJ}kWocE^C3h@%{V}#auPYND+rV3XP zdvmn{)&OU}$&9JRe`YT)5#%e+2yzv|FJ%bW-)oguF^^m!uY3WyEUzk0ko%$jrz#`F zQiC9FM7z`6EIPiY*gqcsJzRkI7TJPKVWOg+e$raF1C+U-iX)z(y+i*!rk(|VHyKTP z0^#~mn%yjjXOVx})39kYbfxIJw>Sc>>1W)#-N{%&dmndMc~<#d$mTlg zRKH4)CmDV7stnV&mamE5dJ%Qn_S!H<_zzQ;wfy9_vhBjGt>LC=A%?v8t?lvDw8uOy zlsT#~53F!Eq+#9aIKPkC6&>&5quitOaRC1JgnI2qy{bCJ(|%N+Z78SGXMXY)OAvm_ z5R_L4n(5SUWej;Rb8<*kwpAA1j^A4D%F>kddiaTZz`NTWdjMmMG-h8h?Zh4}_O2@N z-}^JYTZ)IUmpc=3#y)OeO0NoZHqzo?D8C8i7btDh)R4-3R>zq=R!1D@aqv%vJC*3G zSQ^GD$2OW1-svm8fc&a9S>;m(lTyB$kjKsKy~HZ)8b}1^hen5_rE(?B0K&PeaAq{>q$ty~)~NG>2R4 zQG!W|!Z?9G?ej15%?(%bt@(raZrG2evQQZ?pIslBg@19HEor2Sq?_#_pK#OkkV=%P zN-Y!hr}g1qb`ZbU(|bEr*)P-HC#*MBW8WBkr~rGRh4dZCb3?@Q;4OqsDEM0%&~>uZsnVl1-%&N24t z>4NOK!XlqR`rqMs9nU5_Rnu{X3+}G&+Wx|!K+#{cID7)f*#oDJO$&V3Vh>tsaRe>G zKL-|C9Kq=pxlNkI;XKugYxAoo;(TKJ z7iSLM()11xEpEi8$->tHg;5MXc4Tdq1CF zJw}i-+fNTkCcZ&}Bg5Fb<^tB1)xBTHuO2So|Isr-au7Df=uuOHHSd+)>+>z`XNK$q zE!OB+a~}IiwY|ms(*{;^8S|?0-kJGf?H>>M9O10}tdQT}78rz@OPJeN^v=pZ%?vfw zMi;5F_wxMmc29!`ok8SFyYc|WNT3h)lucG+Os< z;1$6;yX_uZ%^tU+r{GQiedD?LbKeQ+Wy?BMXG>P%dlnf--k+A?aE^{|zX<9uabW1C~w$1wT&6s!Dz(f8POe3y0|dr`NR_}Fhv zw#o+#0mp7l##xPtR{00;-kOND=`rT09c6Wt4?_Fgh7FFlI!Xs6hm_w0jHkoGAG%XM$m+->zR7~)4*s3Lh3L?oW1D^~jZ@>4OZx=J#rFiqg%1Qr z{Xs##eprz2{)Zq}9}^tszY!caP702y?*#|$Q*+!sEjYdj{n+L>BRK4#AK7mICdfBY zuj?4cx4W3qc62kR6(uqG!v~ooH}pf>8e|8eFR9s0gaifd^v znbs<;OK6GBJ@kO>&5o_@?=8pNB;-BYhaY>`ca?eC-^F(yn1ifmV>^l!NdK166~DY- z8>+@&-hi^7%}y$++GQ4b3%gN| za_CVZ>QW6IsEW7B_I54pXQ4x+AFmr)rB&l!dsRlFx3-rzSR93@TXhZc8gG&9+db@6 zyGGdU+db`7n2YY#>zSg~^Ah&I7vbE;S>uSJ8OvGPuf~x@GnWfkS4^-k{+q#`62jzj z*@;Du8=TU9%N|uU%ix?=e$3)X9^#TVdmwWx8G`clE{(yrVvh3uaEHMi1UHfN9Ig#z z+3~P_HtLCgXFF0*zqzPe^`1%g*{E9@)$6Uv_BkC}+MnpwDy`}_i=2vj%t0NV`^Cfl zEZXL2z08>%#ryp*&R|ZG>BWlmdB6_XZKdtnbP_AH`@>_wN8i)mz!-RHepm5`!BX-? zJM3d&-L+x6tEim6+Hs$Fbd9A%na3F-#eg1}Ne@E@qEB}cY0po9eWtOr5@Ts)wlVD@ z&1?Ocqh_CAzw!aIV_!zDfE_g>FFk!qZ$mpKF2G)PMjZCCv8R!TG}gvh<#o&BWLsLRqZBk*h2cMim>(af{Yx5amGBKX zf3L~vny)HT$SdjEn9LU;v%ROb$A}#u1DcnO$c}SF8QqSh8r_dw!1;o;+rq_-f=}MI zIIBa5o*L2uX|F}vnMivYWOWR(u5>k~o%e@sctejn33mMFi`D__!zXkWKSF*J4;$r& zjLx{v(M6t|?JDn%Lq9>gLT;X?yqHKbtjE5o-Znm%BUfR*>hC*L4tQXg9Jp~f&c+2v zld#t}8GT~J;dq&B)=1+}IWc~iJSuy*Y&M#YbrDQ?n77o1858T1jic&e;U+vGHx$zx zGfSIe9)Zk!2M5@yuuiH@waUE_2Wy@6MN_+qyGNs6;r+1mfK}du|1!Mss8ue)+}8(n z-hesi5wwRL58kKkPsMXTo|ahOXQIt7^f%bgcNgqcowT~Q=_)Qm-LG$^xi{8Tm}kEX zK4pG{Idw#>o9GQ0Mj(Bp+Xw5s^`>>*#RuT-zu_&qp*@tgJKkMfgLIVeu02GWXX!k< z4Y~RB&Onj%1v-oDCrt$}l?U&i_in+OG}mZ}#+q?`q;oHkbT2~KlCSiWc2VL4&^HX} zC4PbTGUI_4bn7L?f@ikZhWw-5CPz*IUSoJEKLc(a+%Nk*5;+lkH|O*cAA%e3QZMl! z!deo4X@Xfc7tdU#}a5{aIZY@=rz^R6)NO z)-0rJw4SY+h5zS~i*==u+|5?miCnD1oylEgm0iePV3l3TooAIj$j!FOp5$g&WiN7< zSY>Z=pRvk5aIc0OrF2r60OXx3tkNhlHx8O`=v8`JJN+ zZSiu`^O)NCmh~oW&6K5{!CKF-X5#l7QCDT%v<2lVgpE-ASU0@_e<}9&6hGEY8{nt@ zOlM!P)*WfWX zWNg6vkajH+(&9`{hD^uP!Dnb%4&0r1I<^^>_Bq_&@bn&lwUOJCA$e%a1xTmQgoL66 zE+$*gyO?wTf;l~&?qjjmLtZ|3T4Sy2h4Q!pJL56pp}hbnJT7=V@EpS#Gg>Ek2>^c(p52V~ zAK0QaX3{^V&P_u7CN<2TyBm9s*c;q~ww&OFG7T6}8WW&}2O#|Np!ff2&(^U7@8;qK zu_4>#1Z>k@8lR!tatG*24(6ex2bnB0zZcqa@HK1NkmnN8A|T^P)H@3G?2P(!-8&|& z=*OwcoUpckp3T}?WS$$+ZLlrv;s^fr3m@QITYtZ_+ix-KZ!!5&tWTOe#XGHX6O(Ve z<(XEShjmP6kF+ZpI3KsGRodkROs*Z{QdEQW&6OQ)Y3B;eX=gW?((X36rd7V+l2)I+ zV(|ISSm!+BlvXhc=Pg|`W46IAhYubgo^Ox;d19PenD`OiDcn1U`5pL*9~*@;Ha=3}W9}u@aB-$PwGiiwdZRrTRI1-nd7@9F zFYh&NGl(>wf($B4%Xk24+@Jg#_5c+5r0`%c8QBM+rcwOz#7u#Zn;ZeTui z@W20&JqDM>Nk1Bb!5avEckHeJ8LOZ9?y~D7}%epP`7i8yM%I zQj?*#K6r2bF8ruN`loJ|#kBvUpL@J9@v!nvOmLtEs}c$+R^}-wa4Rxu*Xjd8*nxRXZBrc zEAX!GsmB90XJX8RtmH?LUMBYb3UD5E5@GBC7Pv8aB5(oDoE8KLi>Tc*muC&mT)rGn zHl7?jEAXrwJi^^OuK??)&`9T!r?Ecx7Hb7MUzUjY-(k%_WgiZF>XUe2KPdrt0_;5T z5PlRoB4lPIedI;;RLWgugntw04=8-T(m?Qy*ZB4+d^nqcdJ*3`&`;fHKze>s5AcoD z`0NTF-nnUf*`U9n_phFvMRQN#MC_y3<66y1LS;M{pF-cp zG%W6OfMiecX>Cw|J&MC<14?s&p^KQY9Ba$%eMIQv+Ns3UbCIJdfd-hUaks|E};W$}q4+8Ru9@$}}}j zN@W*?ztjb3vmPSVEg5NNlsO~+Zd%_)9+kjIH>F*7HT_G@_@;e%O!8ekhU}PR7d&ow zTH*1+m^?7MEe6dQ_nzDF>A>+gY& zpTIc_yzec94N}UE^GFuxE#~VmhnZ!{-1fO;CzUmt5Gh84Gkch`A+o)vF_OlpcHvHu zVb0E7+qHD*+QHYgE7db{9KHoYUx-CNp!iCOmmz;cuJdE!6$=m!`-`sBJ~MvrJ{c<+dtDbEyRDPe+_7lm1HK8|d_ zMx>RA`vd5lDZLNU)7XwQEQn8MM8a_XG8gMmQ*FTjQJpWdks_1a$8uX3lwLz{^OJ&YZE-X9m0kjDDM1fwbw{h%m^AzK=MB{@PS; zAPeQ21jyOz&%!AFpM_C54qy!tFwtEp$5UK6(zSAI!+N1WHH)QmuaySKBccgM;G?lOFnrnZ3jHY$&^^@v873OJs z!*kum$52n=E3GtRE%Ez)thYx)Je~P8qF+0qe>D(a3U#z+4OjAC1>HMP7FX-H$H})U1>HMD$&i8ps zU%>X0KHbDwI6BLb3x4=%4{I;OKFs6r-=e(0|1R<<$6h?0&t9V8b|{+%@}js;gIDj@ z4Xv5{>vYN|K%#E}XdF@{Au7X!TN>H+s*;j&JQYc^FYXxc%l)9>OuNL z<(LllEFQ>3JwF&gcVP!gM$iJ02Mmdi-X=j>8WCC=6Tw4g5R>4ZB#YwuNkbIZM;ffS{?b6j^_B)GZXnLVj6mBBM;i`9TMk8=Ccsw5TFT^gq8#7%_J_pf?8P~V{XpkEvh-_lZi z1>rb*Tjs~{sBIM}(Y%W4dI3DtR=>fewnuzw@AcuzSmz-mxZcuNiW`8o8V=bHLmLc5TO^=O;?Y)d`0mpnzT<(m%Kq19v0)FB zGF!3YsdyBf#`#Xj?KqutO2L@tOKXHytj`wYM{Q=I^}!6xpKxxE>Q@Xpwbv=cDSdI6 zY+yM_)K94X)HXcr0BF=U)3r14I}w-Kq8Q`XMm$gBc?b{n&7F`HrBC_RQ#uMgNa?@S z|47gE?{le7()@f7>Y~#9l)l;l{#4K-XwPWjCo$#jnK8GqAFucq!Ec1$oynEi_}12=`oZ&*@&}P(6#v}TOPtIP(JEsRQDLrXil68 zH>WYoUkU36T7Q&@zU_rht>^aEuQ2@EB1_v_Zwo$Z7g~3bOXV4-q#GcOQCvT1wBq_o zqZHRi8mYM6Qg_9rvuuBU?w95llNg()kuJ4A7F&I~p|4z0wx*6KdgHJs5<)9NhxT)(#uW!KW`q^0#O z&bE7NX|)55^oZ6`e!$fC9ZRgx?N4?y(qo)cuIWtgm<(xubWWMR-=^}pAui@9>sev< z#_z3353w&<R^F`nI>U6ny43g*Kv5JDPyq2Fwl#!~5i9)C1#fAJ%ra?NMNw`-})r z4#6J%1SM_E50`xW1ht*bK4yEgeuVVkbrDru+eUJ31AHA;mPr^?& z_iMPMACM#MhhGU#pu3-ZrMaNf7*wjbe$vNqF?T+`!4NHCP9sl)u4ir!!MZ22>~n+_ z{M{&Lmc`2jWkz`){8)P>(RhS!Le@uOzbOpmp*n=2PG)wbtlG1xE?heoRfT=na`pVv z-tZ2s#W6~r_@-T}SF)^rD?@eNqJ&X9|Ngu*zIDM~u{Dj(OOvizCK-$WOXnA}%_qp_ zhW22xvUkF;XTq>|f^XRsz0_>mFJCuoVhNOpAw;*Nosq_Zk5OQFpsKJ3Bqv>Bi&V;#lCnQF4TQ;2Vu#;4IHfkxjOC~-Yr zQTQGpKJ4Z3w6R26^?TYVqGkV{HjHT4|KsCYiMHhTG-ZDA%Y+ z8QeC6^YC6S8ty8DlbelyOOcy_{Z(?8AbbE^oVm;;_a%gr`vSt@*7pa-na$mBH{eOe zQ$7#x>005eG5RFgrkf}S^~1BEIiMpZw3pe$#5rzE+%x2bq|}{7^WP}WO6oHBo%v}m z<|MUZE0W%N_0=Q~%ZemAhbt^?865tc$9RJvjEWl&>J{!3=N|r?VSIQCVU!>U?uXh6 zQ3hXA-qgkIo)wn12CvunaFgJUuu(z?+-KW8yS8tO#I-~3Ph9KUA~ChEdt&O)QHiO( z?uqx$?)hwIJUvmCSc6bYYcH}-k!C?m#Dnh~Yb$Qk_vyY;G0OiHl|K{Vu>E1eaM*$6 zNm)6pwP6KoosCCYn4Of+($H#Wj?lWJ3!CTm|x%mo#yt$lZ0mh9-pDHMP8$qzrk{zY!zu`wOLdz=)yA8YZK}f z;h9;M5?WekGO2R~Q-%5+FZws;3~i*Rz;_4x-D`;#^*31uVSQbSPyI4uOlIf<;$4NF zk&T-T-sRvESXNSHOXwx$1~Wigs?mN{Xx`Ek(4Ga2^n`dO!+&Ek{%Z{y^59kr~ zchaGjz^Cpr78l}qx)qZ%u}*n<7~J7-{{|N};A!khWzG`hr#l*C+-o8~eF=8`vQaLF zuE!(&N09b5AAGk6o8Se1Idne;?t>h?1<}zL`okT|(Orm+zR?qIB1dOLM_=g-H-V$q zQ$5jl?gM=YN54jN^rcYH2XORDL`UBW1icqWuTta{Bzc40g`+F7MgMCFdPk1_lfv&K zxq=?X(N7V5mPHmo58~*@L08&z8`{)4uBd#KM-g+5hwBN~0=E@hE8Nya4V?vLE@?Ux zqnv#&cWqh3%%w5q3o>IQhdMvLfb^6$3p5r}yIw|qGY8S#C_>!7^|+V^lN zk4oqe(LaTMtUJTxxZi-hQB6AliRYR#QoD+*R4!MZBi6sU}(z*_ z6WN@mJe$bos7^y4)BDk9It_Y6G?I-vKPR~cDRuo6v=4MXtcP@&ysOb}%+hp2lMAgo zl)O2)?9pjj-%5a7{zkO_q+Eg_7rHZ!(j7rKUX{^)*e|Rxj)xzhfBbn%uiRpY__ly=3&M8h+_62U z;U(Z%0vdRgG$tX>Nysw?aUMm!laTN9oZGg^$cw4*=205atM1r71Z@)1MH<~HjZ-U$ zPfhP8@MaAMth#NR1fEIYNm^NNdmnKSro`F6#n}(6^YsK@B)+#w%wo0MZZ=F1R-Lo8 zTXog8W91p!xs_LJIV;cF{$u5Dwy>4uwxKJ3#kW?!+H!J!gL~FCJ?ES)9N%6|#kW)R z%@=)xWyUvHCc&JS63S$=+u9^*7ivFhm(FNEYBOp-4?OhFp4yJujoO9UjLv;fyHOia z+tJw!YCmc_np59^PDZ26NFSmh!)VBG5ajn9>WVh$z6|>DQRonxV@zfY+5s{gLTxfH z17Quo(U5Zlc+ES*x<|I9Fp}dCn`vj*JQMuU;Ee)rqJqQbMI%gat5L}3iJczZ*CB2Z z!q+0sTDYk@J?6~??Q39a$1zC10{-we6c#ow74*w+FDv05^BzU|R7SMHyh(7AR?5(o z^R~}(s-RD|ZE-o(kZl9xe8(n_!CW&(wgs%Jx9x|nomzR>c5`Kot>?-L=;kHchLsnf zj~5%;FdS`Ifp+s)Rj0Jw=PS?IrlT!)qV3A@tz5g6_*M>WHVy4a?sBCKuh{7Ow?A&f z4galeSbuVu{o=l%_Dh|c+wuQyJLV*Pfqwbl)fb!l;Q!4&xVE|N{=)XNx!?U&$G)5- zd$Y~_i_TltHn-hh*k(5OyT7i@T-|Il|FSl-x$XX{{cb~2bKCt@+b$<*wXu0^{@2_e zTie`rf8lv%bHDrR+DtQRwwZrbo7voUf7O1sA*s3T{+ex2y$k%x zb)lHfyB~H|&nvwTnsSaQ)*`;iEGspyB*N0^S*~0FC_D@bG7vW*}^? z66S;dR{Kjgm2*w^&gsn3!Ya9Q7t=sHiFkD8ppw49!Fh^L;eSNqy@hjAl{ky0pTBre zPeVJWL1$9N=``-_#r=)(EwZef<3ZXu>p^EQhEkYz2IK0Vp5gJ4K7jm}0H4H?*?nfZn!|uAb+4 z9kc+Q#+}*m)@g(8$)}Z0o2Al7K4zUpZ3r230L`h9c26!1F8?{XEQDOBE;|%AK=Oo( z^C;`X@9&8^2T0eh;=7e{q0hDPI7@=FYOc^(e1oZ+BQf=`i1Z)BTWG%^CWz&$EHb{A z*${wm3M=0s*nNnOb89l_UX65v-IwSY7WpdZEkG~djP!B7>oWW%@*CmD*{&M+of_jA z?S2#wXJdW^{VvL)=Hq1dCOUNMH0ZY)>CSc;^7N-L)a3_+)oNk%@3eB9#|fY?w8II6 zokv)?E9lVOui!riKMoa1biVNm_$%S3b20+XLD0M2?KmevXDp};biTrjw2W}w*h=M` z%pBB#&R)#^hb1k&wZ$bfqTAT(~I<9hEO~vJoLYd zws=DD>@m3I%@MA|q&i=TIR^X$aN@c*<|aNIHurvhoQb$-8)dwgZsNM_|CClLXid|Yjx!D;KjgCFHtKlJ zmIMFJRlnLk$N80=@b82_4F6e7&dwY>4gVOO9?TrC!#~42Gp2eo{!e4Y_sh^FH=I-c zKRqwhd`97a_KZR@{@qrM@#?>`&6G1<6GzzToY3X*&1Z!Er_Tr_<9~(KuK&Gl*xc{_ zhx=V;=7_{MoXvKU&QSh;|2$=Kcjj1){|Ux!G${8!u5qg+=;`TIozGYy*b>U!-F^+%i(wq59jbG4v*pRcn(kGFz(B0tosxWPv`I~ z4yST>9)}lj_!$l_;cy0rvpKwq!+9Kjfx|CxcoT;=bGU%RJ2PPy zui)@%4zJ^|$l;9~ewD+oad;btcXHUy;UW&d&EfqVF5&P;96rS1&p7-AhriFXX?#bc49FFF242P{89?Id793IW#aU6bt!?-uAvCNNhcq)fy za(Fg}(>R>Y;e{Mt#NnkJ&f@S24zK3$Iu45*-pJusIs6)jw{dtUhwU6L;_%xX-p}C@ z4u8bqLmd8$!(VXtD-NIF@OK>kk;6Z8xPrsKa=4nq7dc$Z;p-fhIqbNH8~cKH^SIEc z{}11@6?uR>Y;e{Mt#NnkJ&f@S24zK3$Iu45*-pJusIs6)jw{dtU zhwU6L;_%xX-p}C@4u8bqLmd8$!(VXtD-NIF@OK>kk;6Z8xPrsKa=4nq7dc$Z;p-fh zIqbNH8~Xyj-EAHhv^n9!_iRPpqa2>f;h7ws&EYf-r*n8AhZk{pDTlK-yn@54IlPX; zB8N9}_*D+S#^G%o-pOG*hl@D;Hi!3fxP-$Warh92KjZKh9R7;KCpi2ahkxYo&m6Ac z@UI-M=I})h*K+tehh+{s?%~G15ZpX2G>;2d8#a#%@&$bN-aIZej|+LtD?u_mS`px_Mk^9v7PTk#GhB-*zR>Y z;e{Mt#NnkJ&f@S24zK3$Iu45*-pJusIs6)jw{dtUhwU709v9>bqngKsdvhdiZu}^R zr*e2Ehi7v*jl=02UdZ7^9A3)dEDo>W@M;dP@688;5st*v{eR_Y1Nc z?pU6pokenYx@RZcO&o5)VGj=ba5#X&Z8#jp;SL;*;BXfX_uz0J4iDh)U=CY2oWS7` z9KN5!V>vv5!w+%z5e_GFcm{`4IQ#^MpXBgU9DbI=&v7`D!#Nzz?@Q;ZHeygu`EQ_&A5ZCyggL}Jt z^!R}suBUIikzV;fhTG{r_;RdWOxJPmCB7~8!$WcCo;C6lzJc#(OzKTCcx=1d*+pSUjfl@WO`0n>Na+lyc zUvlxC@ELOP9dQM@tMHvKxp@}(XL6D6Kgiu>k$+VDe$w~kZnDVVDSlt+q~iKZC&-;= zk-s5#x<%fv(7mOGY%}gl|Kq#fet&=ZQRV)0{SMpRxc`@}a4#uG+cC?%yVB^MG`icC z?%mRP!LygYPYrjUQM|o$pRpPDpoIysrx>&R(_LV6U%KG2EF`Q&a>yU1$>odqjrlgc z>-VL-4Vx^ucTUN(yj$blYjn4OiDj1f#AYU4M|t)edWxn8`fO$`p*Jm=oC6u97&1$U z7Ai7140_P?e&65KkxQG>H6z{gNHEk@;-Y5GSHWA*~k`*VnWPOICd{l=jlBLjIOoE^8vZDA*9EN*8d?or;e$EZ( zBk20K``vCr2cW0-D>~pK^@85hoigL#C%$3inprXN8xViH!cYG;@Rho2nEK}HgemXx zNO7T-*4yuZ23ae0u=W7$3b-{g@ev+QXETo~t z6H}u@omu)e+{vu?e;ie^+sOKMqJFC7_Xv1lHpp^>}OXdxS$*Ypd|B$ZC`&QtT{7;I6?a)GJD0bcbOdJ$;5c6uuL`^$?y) zJQaB?jqa!mr~d`y^b-H3<#ikG4dhArURP+|(mxc}4|ml4@x8apaEBs8p44CJXMKP> zI0f9rX~4aOM%=52yX%d(!w&b0-Y^=Zo7x?P66Rx<*3%h4SgN8mRx|6`|d zSNU$pm}sL=ckt}l0UnY8(Y{7}y7ON3Q`r)bUp(@SL;hCW0s5KJe~EVx;?o_QZy*lo zmac2>!B4tMJll~k>CYk1l>Y`mPv}1w8xWWNDew;EXY|jbyT4j1`a|J<(iSCLxf4Fg z6Z9Ks&r`Ucope*Jqp#E!`b2l$5$!b6c@KAHQ#ysfH{iBJJ@j^J)Pv`Gixp1lcrXck z#7}zsB=S@1>?^HP=swZ{xF4Vml5s!ipY=(q_Y>e#?jq6juLS%8ush@zXkd0^(Os98Re;T zpGp@G(SR^&mp{_E{#OMb(ls;7e9^op>Xy>KrmFok5@j)?f0~yLm4QX`QfK9#Aw!k) ze56v?3Ehr#MVqVb?;{OF_=D*4ub>|Vywq7Vy(FA4M_T1b!MEl{@74K!k%H`x`Aw-SkNw)HTg9iV;FRR^rK$M!&~|pai~w+fuHo}CR~q3nR!by8)fM& zok!R{gCF{bkMx@szv(!GIf&ACo6^H@-+Dtl-S7U-cH=7Sh1y=3G1NCR+$=S@&?)Lm z$CZ5iqJb^K8E`eWbqWnCbC^LC+qDDoyDE) z%VBp+C~JkWG$sV}j5ypy3Or4tw*oEY(sM6{IXT(v2A@2-C+57-Bw-Fv6ecWxBN6xM zZEI+lkO3Q(iLz$6EUU|eju21Ejb%!_k%H-%tx)+}8W*;R#f9Vmp5h%|!$cA_yQs{Ia zYCP%?hq_o{&p%ad`Ts%vhAQcLOFjOieo_CTe!*J(1|kirUthR3u6{eT`U$8X*=efZ zom%RT{!$0<^7RW-%JIkbyEE^7j9UWgYe2n?s6WQeCC<2)SP*f?b`tDL@inyBcDUD# zPG~#zzTewuL&g4Rce>7gyJ1tN6OA$6lB<@Vv*LP7SKyvT`&Mb=?FINv%PW+K%Gfg#Gyr zeM~9q%wh5oM?-NB+$F8}7e$GY;DPOqHrY+MOA_;Y|Nl z{iV@p;{hlW-9b(6VU%C3yT9}%cn-qP-yOXVV`=iRcp3e-cnIPW&KPFId|AE2d0VaE zqhlJs{@oq&LCCWZ@}f4M23Ku!qjJYH&7=4>zku~Yva{OeYn1Z)NV#x*VbgctU3L;S zb3e)MNz|3v-=y?iAL$ABsr^IEzBSV|otOn0=36yYX#WHC87E18(-c1cW1&V<&1}S_ zveEo&AY4j!j*=(rhvEi77ykTCcB(tYr@Oa@gI{UG1z1^VccLeQMt2vND*c_=17VAI z8v>7Ey!Q$f%r&N-tMlpKRg^E;5z>+Ih@;+*<|7@1?5W*v;!gR0xCNrUy`?CPZ={|d zX#2re<%%`4b70;yV^H30=!KKO6#b~cy7UZmGl=Dhir=p>(y_{)qmJ|f``WN(z^}CyJ>EBJzmB- zjnOQ%=#iO}VP$g0YCI?K1PXGh zU>4VUvRW&~AmvV6{O2juD%57t9l2;9qA6)t_=yqVk3iZacfG8~QPzfRrxSPPwSbLl z2^-f6Hm)^noCj>2Cv2RPC>x#8W=`0PFyanM1O98L+P70zNZvluw~Fg4mBA$$mc$1g z@NSL%(TWv+ho__!>jS&Dp5@*rKJ`n(CTcIGk7NxL&EZT8bzsctru?dP@(usrSaefqcj|k^QPN1m{6kCrsvlGx!e+#rYEY zh&DKG3Rman83uDr7R@ax669Fel1ycc36$!Q|6QwB^^I0p86H(4Y&6}^F@aJ zm8*xMkX4^xNo;*9gaE=gKiA9Lk5a&Ty4)}?~Hdh(;S`lawy)__7?d( zouo{3hQ14dvBa5SQOoZTEJBk?k(Tm`!=Lgnp=FPN3ul zd=_CTD9dg9oAJ(NX3s^LRU6_hRX`SWXX+W0nZ_%c*ZcyZNJIc_9>JX%KoV0`btPsk1ak00Hy{5CPJ#lH!=JWGR zb~)8-KM(t9A^VB@ldIdi+f4|on$F}cut}L?hRa>h_i3)1()-`p_h8uf&5C{ZgMCji z`YCoi*nCQ{^?x_1w!SCIPxG1ONV_ZAtP*=)^#4lzKcxYn4OVnC0PRIK7Hb6f4WJVr z?N6S89VQ#=K-p3<0ykIH80>eTlQ&>rGs)Ke&c+5v3(zhUw*~Tkmh|;c>@CgRk`;R! zAib=`xpPU4TMu`TQZ8TVrb5I1pW;F%6qou`654hg+ITG5dJNj!hCVeKeM+^r{9Kr~ z8AIP}*i?#n(M;%~LdU#jx3&(|=La+=Y-)pH3rb+ijz_|FMzl2LnGOyA!+1)XkB%zA z9Jep<6_d3m?b((@nVwH_d%kFXq~Mn5*3)tzuV`(O`|I}kiGq6~Xi;u$EzybJv2mtw zOSEZGkQe4d=@xUsKo)6Poav&>my%vW{tKRt60z2eej0NMvS9}xkEh-=iMa-gjP*V2 zf=QA<%W%P74rnIWLd6DS?iGu(pt|D#TMf+JOdtR(I@A_ZFy?K0<#r60<&%HOvy$Oqu zm9QiMh1{SZYm9=qNdy8YR>TdhY^@DYY;hL^0;x8D)jWuz!9oPv=3Ygqf)Z^dKtC3= zwJa6aN&+r7#Em_JV7~7&&%H4Qw4dMi`^WE(dChs|nSIWjIp@ro8Oc+^Ouq1Rqseo$ zr8H~7p*rSu>%N5!Rj9{1*<+}u(HW<%PT#@&ZmR<_?VV41V*`2caJeO}KwcQSUotU} zS1tav0kaO2>>jtt3td@Gn@W*$su)iToDz67GS4m2NAuOL1uk{(g7MZp3wC$dRN8NZ zUfnN7PvT$cK;VHf`X+&g0uPPRHwb(&@WnCuTHumf{N^q|TKK5*;SYpolJ(|nyjH0zP-C%1X5BZ`z6jpE9A3@T9bRoB zxGibBk_Tm{~|*fxn8V5dpmbb~lJ5MVdcoAG%A!CXk?mbGEpdXYr-c1`@<3_?+aN}0$ z6~zsKqhq%{vsw4lypBsIDg*OJd0og}n8x=F^H1{^2_9}R$Lzmb) zpi}-W?d@Z_`&1wgGb;8n~*g`zon5iBJ%o9**69kb8`h}*&{wSN@<~auO^}soK)mzz1ulv zUXES-2e0k50=mJKoP++nGkn}3PuIZ7-=qglo{N7OF0)Hu&d2eAIZvkt=KPFsPnW>S z(s(tY*b3wo$E)1p7}kD#)xCOLZ}1bGMD8mcU*c1Py;+$(`ufy_Qy;_xMW%Ry{xj=8 z@f=-Kx|yF6_99`?Pm10sX7U_gdiv`v&w(^PSQ-)zJ}+uj?asyL&@-nz{z`?c!6{ z*~ae{`jzId*8Ph3#=@sZ+o}HJva)#C6&DF%)e$q)Z!-_ZG z#qu4AJC?XJ<|VJ3#5z;1nxxU`Z9P`DfyrXeSC(TpzRZ8ecBYYD4)?@HnUjU0ta-4R_zt80# z{MTdfU!Tf9#!|us7GCTzc(FPO2X`b~VByOigD-m@*w_L|82s6j@MmwEu(Z%#34>32 z96oI)@NwpXq!(Ctw#VSv-T;D&lw2Bkq#Qmno3z>Isl4nFD(@s?A$5LS+6T{E3D3L?_y~9<_$sjQ&yT=AKL{*+ z@jD5FkA4h3`Z)9I0g2zoi7&A5)sMhemjZ{!26XwQP9Gfw4@wJxp#z72r5>g5+PT+t z@cZ-C>Cv>EJC*f8bRG@80y^hU-k|?VrE}NP{^wOq1MT-0#WxMRQFN$j&isz!-t0dm z1pU9DpY{-kKJf3uE~TjbcWuAQx+i~EhfVNidD2#)H#bl>Y0Guu)7ERm2Zyc}9~`?% zeA*PI58gHFj@otfN(;V+z0NXaXDe6ZLS(OL=%f~)pUK8AQLd$?zqgF78IA4_TZ9Uc zxnxgg%u~unemYc2ysREuylBtS_Y7 zxx6sRE`1HU^qSJUB`$fu;*$*YfT_ zHjihVTaSUEkC_+QS-fu}bhXUs%QI}^#E#~N_F`pUBEny{^hG=RLU@()=?m<0#vf~E zc_tp_YvH{IQRg~ju4ZJe$ElCR7hDn;{(Suj+IyV# z*20JGLk4=7aQ5ezH^>Lz-FD=u9p9Iyc4i|_LC@CbxpVCNl`*1|*y;7>*{i8TlC``t zbL3az)KGkgSHY$4}M@MCXowCOkl|6@22hqRNmm)Xr z^w+ch_MO~_e(V2!?@i{7m$Iac6y!#+P5c{WiEcMy7q~v#PN)%H-OMrDr6cfq^{5m6 zy|_;}VgGHfKzR3~@a~Qci3K08LIwx?=)V}FaG&qfipzq~nNDhOD)r1@f33KanotRT z79W_A7d}Vj3jIj?eL_t!|B(T|&bECuL!l*t6GHCWrSJK|OYm@8}H}n)d5IXWE z;S#Ta@t*vSg5ORF?TCau8%Zm11P@=w-$$6#SKZM)yg=Wu1_?zXeWYABb{W)<4sS zk6&Uh#J)AI)tr{K(fn(=xW(qNJLv^i!mRgZJp`}(#LH*jb)i?}npD=>AK&B2CH-{= za`H^xmk+ZO$g}ix^Tx@-S3j6&$4MPav0H@(*Mu22kGje?A?4}}D|7as0gt4&Uc1KA z&*c|irb{n|hh@C_NnVV(=(MEmh4TMW{$J|+@6CUSUm|(~!r^l^l!CjeG{vUx)))S| zqivB+(vBnUWOOrU^wpEYY-vMu@916X_!s83gvoqK?bJ7K6W{*Fa2 zABSA;MZWhT=d-srHvu~^mu=cTp_9FX_%co(Fg9}!8U0x zrQS0a_Oqv>pO1$R`kb+N5c-!hAhdGgfPnvVWRn}NDezxEyC8V^4TZtanSbzTo&4vi z&gfgNnx3WnQm6Km_aZo6?}ld6ALD6LU1IHU_3o5$^$GYNPPx_l?yu+X1>ehW1GmAQ z>CE+V>Yky}8n;5LtEvAG(rzvc%pQ#_?2$G^;@;dU?o|G5j8&bse;YS6u~nHv&Qhk2 zG8r3mSu^tJ-<{IGiZbZiq4$CF@;2CG-!yMbj#Go5h{wh}QeI)J^7bS9Za)?Gy}JE} zvZ8TwTg6?3Y#goId*FhP{J&Rjf}yPyUTbDvIq+UbkG0-w7wYnOvrn-;v!+nj!-I=W z)B)s}k2}g9VoFGKK#Dp}q2BLNU%@%=+l4yY*EU@j(xsXBrRbTjA-?RLc8y6-@Vc#m z`PeliyUWwdJLedEjd%1VOMP25Z<4Xdw~}X0KGr;;_*h)7kABLxE}Jd=Eo1H`pVhQm z!due%x-N~6q@6_C(j(0iN*I?)+Fm};8RxHLE=I}|2nD(7s*mjWk8;K|N@l)(k; ztz?Ze<4_FVG1qsBTp)NUeHA{Sg6O|=0rM{F=}z|6�f+>X4_;>MMKt>-`1Eb~f*T zNq=UeACR;%W-DpSp8d@mGV@A$7}~x+!+y#YIIXg$^nedO9i1bK$+Ir*vRTZLo4soL z?r^8@VVCubRatyj|M~a3t5Y@{4uACSVej704!a3|I+i@1o|=)ViZWF8&w5nFs?*i+ zVlSoa>UiYgn^x^!e0tsC9iO~+c*nj|hkf&WEB(G5kHl{;-#z2;s>4ejeEskeU)|w+ z(bN2=c>;8NXH>4StZA#HO+8N+wVavhMVW=(X*~+lru0zw3V(l@n#J64`sKmS_IzwbUhcPzAS3^Z>vw2ybPeld!@$&tZ^UL$5f8x3D4H1fyjqxb8@ z&H_e>bjl_;xqt-wQ2vmky5+ zeCmP@h_*_);yCn{E5_E8G&c{qn|x)TB@=!~a8~%F3c^G_c{K9)B-1tGVRKJgL~YKgaC1Y=GAh8A0kP|L?-D zf_IYjc^0tnL)8OWH#pC;g>{2{vWnX~_qFfrZ0)LA=ERe<3y4>>WQ5+lBu4j^JRSmm zC`R`JHuam*&*8D|Tr@nx7o&Gjr@`>KvR4w-0UW<>TXlp#ukc3N^Az+@cyw=adAg(# z-dN%^m${HpW5D-V@IDUw_p;Vw&x0N^YC9+8rXZK6Bq)(pOrF&VYN&xfLBDr8$IhYu zuVGFPp&q_NS8U3Utn*%U2YKiS^5NI51lGW9T{p?N9{!XukuxN}&bMEC+rjT6gw=IY z8-&NMj7?0Cc#lT(pP%zj_-%O?H!N$c(A1}rY!|jtUUV(~(HAf6Wr^){T1fD5GjYTg zY)zuzWLgVO3Quor%{rD2KWT6iA6@MJZBcu(w9thH2h&13C^KRUQ4zI0+tgEWxesS|22L}=vn;CR=6xQ`N-8qrd^+P$8h^6 z{2$x)zuU;K>$4ub)Gp?K5&xx6qWX{il#5(Zu@~JzEqZ}{=mnM$Z!i3&#B=?j(9GWh z%&m6OxXb$4+n0HEi5|o?eFSS0=Zg4O8=;f0k$yXJa)x!zY}prxhQCU9>v?F#b!Wc4 z(Cy5_OZYE)p3v$FKYSawQZvT#CX9g}dlMQt6W*;IeB77|o?UtMRm$Jjt;SEf!1A+3 zbh>7q6`VBB70ixNHPQJe^Xx(LlzH|fekS8Ax;Vj?8~9dypY1FsxNc~8YaN%+(<@11 za2XtD9vzz8x%gx$?UQn@VcrQnyqq{fBPA>V?ChsHHncgPntiPb@AvG_ISa}-3SV4| z-L%PnA#y5b86^*)Pa`9F4fnoIK(ie4~d`Cx~{kKC$>Lu(Pv2{mhL_gmcmzE=O7qM?GWBDHcowY34 ztUodzzE4XoiO>>sNzjsKnFITeIO|mz`N`V$GGT^}u%?|fc76B>&=J>ZQbU@wBbRaR5z`y8x{|o8id(WT) zHH7~d9mwNf^{-(|Y}w<*Xlf5;uxB{U{^4eoSMZI2zuU@E^jP zh>pj|IW0P#%amOZ+5dQ&vZX(M?A*}*X5A5dy8(Q=9$cHrdAw_;TfwnEbp>S&YPyh!iqz^Fe->1JSG#&e>OmHjHu`#Of(q|$E3QZqL-`v6f)^jsRo~6#* ztZ@Cg(9FH_iI+|OQF<(CW&B&0btSY7Ic!wH~QN4qP9V1Ib%-`4w5{@?&TV_&{HMKYd$X zUg{|}Y`cy%uejZ7Z0F3mt19qY)|9C2+z9eBW5oW+qwJr&C^lrw&qBhZHf+OyR||j7 z9!mA97`++0Gl`Q&xcI}!uhyKG{_f5Nne&RX0V&jb@BE>5J@oupcwJ{+Vo$(#GV58C z9`qtF;0QfB(E1e_(R#e1^=R8)%r9d{ zC-%GB8KbkM5#0YN<0T+H=gElyet-x0k-#K5p>RB{-wr2T&QYH5FU*~BI@hg<371Mqg}?UF?9lzjyF)^&AmwGO#2=WhDo z1bq-5m$@|ge(8hc&=l&mA71M~FKK^r$m1#<|AfJ7wL#`h7#oHztYL#F&r7~-<38lf zsZc5XC~K+g^M=uDsC(x$$zI@)R{c3KPTBih>5X-#nF zy_6;2xqOE^q#2vl55d8@PYd*9@X_LI%HQ`}%gvf(rE<IrMjVA#G*vOA#jvTrmF&o&P=C z{I6rKv|iU6dKJ!!?ys73Q=N24q^te5d86cAj&J@ga{jgFA7f_zRpXm~Yn*?v{G+Yr z-}Y0He_Ne@&5U0rVUlMUUAsxQ-}%?XzW`zK&nrItsyK^wnDEk*%^OR=RdvlsUFuOA zzL2>1GN|Q(6_bVoS7f!Z<87SV*YuDq?+|Dk@d~wal^>d=vn7`w|=(ULa*ud zPU-R6e>SgB*mH#4)-2Hx6p${fHre3+Nbm-T46hws&$ zGk`PhS-m@B)AIe6jgI_~*uvaF{>xKx#D*Ym^$YDb9Xf*DM+cwTTlqKiV)_1lc9h;J z`c3*)cqX%cAj5=ZeHPh~eyu{D*^E4M6XjLW-l-CXUjhvie=~cj*YL0UuFG_A{$*#- zzJ09I<-`*jC;Jq##%vcmbLik#f*Z``hdEc3iT^wN0RB?^$@q)$OYwh;Uxj}^el30( zejPq%T3499TsXGoW!iP3A9w`cQz!M4KF+$pR?};T`}{Ws6Y2xHT~@#!#;)+-i&n4! zIZ|}CW^5;<+B@qRM@jSN$bCS}8}r@Z+y~SPx)secBTv;hd$8@-Wz;L$tju10KydHA zOYINHs|vo%e4m?dzr*+B=hO!Cjm~Bd-!eaCZ(4X)|DU-J==68@0eNYIqCM&+m%W|1 zrQ~J4|KhUqwl}XZI#FzHqWyo2IGbaU9VAWIWxvWdG=0NN@}60LzC9yrLU8);u^&Yz zR8C&gabs|ALjQhAoCkNEZ%d#3?tdn1Ds5mdf%(vZ`LKogpqLL24Zk^95mTTa^cCpO z`+I^1i`>Dl`f%p;5=(4C)@PNwZLtmi!madt%9bGGgmGgo_`7d zSs(Hl!1$u_R(Zv@qm8y4xlv?B$#)fTB<`Q_ulXiWlMD{Y{zY2RAY1)5&}6apKFvR&5euOa z&bVA)#^ou(^B0^G%x}h4a7C<)OCJ4`C;9%Kf3HIiP5DJRcHo;4x&vW>H%IDr_>;cj zoY0%GdQz@cGwF!Ro3z%-oAjcU2afLCxF*Y<8JxU)-p1x^7@ap@R=(}B}n+9hxg;2tjBTnF3}xTi~>61X>TZhW|#g*;Hkh$e471-uIWU*I*sYux%@0{;>Ck8b@B zfkVI{x85P}bHLBJ_3Hw^0Q`bmZxi?y%DM#|sK7ITXSnoV1-=#dR+oN7;Jw7(>((y| zd>i4nx%38s?*hKdrT+{JZsb+XC*FLwUMK$+0WWgvkiZWBKj7AX68Mkc*B@i_(*iFc ze2H5>CGbPQ54rU!ftLX8k}k4t(6Defa4c}FTVEz{JaD{Q7YN(|xPx0?DsX4u&Tf5)z$w5fZheu! z>A>l3Jw)Ihz&+f0pujzWd%AUhfqMh@cI)#6&H>JGYrnv&sLv|qzrbsN*SK^qf&U2n zM`TTbL%<=I?k@0iz|XmKy1*|0zu?lT0>1?Ol1nEE{0i_Z%zuIVQC2^LN?cqH&hw~i5b4DcAYb_qNXc%WN1?*kqRJQV$(z?TAF>egD|alqr;`nbU3 zfycY`5rMA-zSgb3HZZj5TF(Co+>g5TWBv;~5O^T-U*Ms@Lz({qUkrRP^IzagfiGqL z3%r#)x4QKQ0>2LYI`d!PH-O(@{tNsT@LSA(fp-J%X8sF25qKi=U*Ma8Z)W}rJQa8< z^IzawfNx>`3p@jOhFjMP9Aa*UV)S1Heh&D#82yUCw-V=8=D)zV0pI4<8w9=!_%65p zv%qtJ=eYHA0>1+Mid#P;@N2-YxplR`?*qT@)_)NABjAtRdX2#Qf%m)hYJoom{?x6X z6!;+ULF{z}{u200&O!obtcBJb=KHW)FOz@AfsecO!vdcKKIzsE3j8hbw{E>yU>AI~ z%cCC^yc0kJxSmk z;2ijWf%^gX^XMA{9tb?pqpuZsDDY5^zFJ^x={93~_Y;A!r`wF3-4y~C0vCGp7=cFu zkM!tK0*?V6^f!6@9@o1O8e+2%cM>l^290Cq`^eKU#1AfkQJ!uKVG8~q`(ynYa7Ix?-a=QUu(?rv_=imJvc%&&Oa}b_28p*c z5|4X!RM`W#`-iEq{2#0Ivl918;48VKLgI4X#mI1FG3!*{(&1IFG4EcB(QEkET3!{M zLNdJbyXbb)rVX;YaL+D#I29A!1MEj#{us}bCmk`$cLadRa z>#i*uUiVn(@aBnDOMdOh&(t%EIgYHYEVWKq%w@}kbK^WX=Ajk4PUV#j_hor@Wm;+6 z(b)x;j_Zo+hU<>&u`W|X1BNUdt_I*P!1;0K;j(buab0l9xK6nCI3F$sw=iQ}=3kM= zx8VKH?e}cOEVa*B#ddmyGL#Ymf8c zVsIAjH0iYDv535I^mQhEo=M+l{s;FS?rq#I+&^(UaQ}nbg8M73hCD7PqfFd+xGY?E zTo+t2t`n|3&WDS^S-8`r(YRxz|DudCai8GoasR=+hkF~h3-?dl4&49Xw&4DXtHEt7 z8!q^@`324wJfz?ateUzP7}pt=ZcfMDYQewokNu14-UV`}3-+|7yK`gE!?`ZBq3Jut z-UfYxePWFQ_q>gqGn=sa+x$b=LrxI9Y6c%if;)l}b&D1bt6Nk$%)4!D%@K6``wFd^ zT6m#-@Iu3hQ@h6Dg+>GKgBPkD!x<~qO5=~Q^k9iIut2{rVfgz7TDrf4yczFXpSri48J{6L)UF7Q_1t#LYC;E}*1<8-RP zGe#NyRQ!E2;&f*TTLQc!PInaeb>P?IbUT5^0FQ~&K7nr?rJtAb@b}#sr#%w(5b#5B z+7kE;;5Xv*w;uwJ10ENrzY+MhQTi!KhrjQ(IDK5gmH{t|(?H8%e{=PYJ`h5wj z1g?zJ?+CmXcyFBkx4;vDC&uYr0?!|1_*3!s&5zUnkg!$2tK#$yf!_yyKTf|c@Xf$C z$LVbXFB+w%OL_SF7RBk;By0`vnmGNcz#jpB6sI={JQa9qoZcw#IL30ErC$)ZCvZ0p8)-4_j^5tc`#JNS05fCuPwz}xe3|)wdz6=&+B>l)p`sx#Xob-=6>5JN=&uf!D97#Vkl79QK{!aQQob-d*r1!T; zFZ`>N-#d~%d?eRN|9dBWuQutsv`N1`l0GGx-nz(1|CEzHu}ykkoAjF^>0=2O+5c(g zy!gU@Ms)$rADBG}Y%(seeOD#e`IgcdPpBI98*19_6Sv(5XOC)q$yRL7q087C>>k2- z>W$E4p~=gk$s3``t0Oddb%Z8YL6f2Bx}I`nepEq|tDwn}kI>`=TNBPGx4B5{6y4Y< zda!Sf!M-^bo02%}o3U|ryX@oKArif>@FezA+SqJ~4VSTF!3N5?L!|dg_I|McQp~MX z_N2I1cEj7)V#s%fJ9*|*_GGW+`w!Gj_QB*V(`x$eG`3!^5hi?kl~Nld{3*gSiib9C zHFs^WPnZ^KmtzAPJ>R`B@_x&$q!S#FJY~)=#h3C<&n>16qiD-W+BAZ;UB(#&?q4Z# zY+J+blpOCoxr21X9&ab_(r(;e5jJ-pZs(3c%4-*A%N>YQ&^i0v>5aYc%NKZ8%Ko$5 z@fhx?G%qnQPl>C%_LY=7nYsS)TCr*D%6yl*M!6%fV?Yo1+Oem+pEz>2=j_U`qhC#V#rQ5OeQE1_=iA`sXM|^yCj5c&V^>r2Ntx6&JyKWh%>&28j`LIclf5nHF2g>7 z8tehSF0A&MxUr3n?ZmOjU54-r?Bl10Mo?}YZ7#x(r+ux*%Jk9GUfYVC1Gk9dq)Q6r znskYw?#53JEdxiJi{$J`9AoBX?0k%2JY&hZy1DJZN4I@~du^gPsU!CsegjTM?ePDS z8mKwf(KhBEr08KWGA)UYnUFI97*#!X(a96@GquJvANx1!jeOqjh__ytML;< zuNq(Olf9fV9n08`VT?yJ*2T=vQOwWC`Ju8Ox1AOoXo}ozbQByo6|o_W-lJ;jAE8UC zl09ecP7s`{iB%gaakt~HCF~+xD()!#vVgkDUD$%h_cB&OmmVif=#sSWj!4)eCd?H~ za$yI{{Mt@`B`ui0=a@S+NABfaIX~IlQ~GP-XQ-}?-!K;D{vYy~X~L2k7eNcpm?NM2 z>@5+xb`fPreUGxwSw|VO_!c@h)A(IOSK)KEHOZMr$@J%$yiRM|-Rk+@(K;`9+yMQH zpSOd1UgI10LxYg#&a6*3-d@O@{g64i#O-Qy!}Gg2ceX&qH&$cM)10C7#|H#|#15|; zcUd}kK9;%(ea~bKsN19L^BSD+5rj+s)+z5R-m0;-ZN0bHanl1Fn8|feE!`AS3{nj;< zDs^Ifqzy^He-QffUvTdjxVJhJxAI{N5aguWmxq`E@rb<7_ zdMa;22=1OBuL(~yulN+2E$2k#?xX$m%_p3P`Izw+`%tMX>(5{>_)*nSZCHr=dE|Sa zika!^s3siaer~x7R&1?h%|2z;Z0BBB!RNod=Zw(_@)W$1`6>ARA>pmpVVS=f#X}nR zV`Fp%zFyYOzJ@*)JdioRZ=jl;NuOkR)$9Un)MPzvS!3l+QSdI7_?vsE+X^^WUOhkN zYH!vx3Eo-Vt_~-1CIDQLH!FV5+EXrTZX7m`oP}+0cg;D-7>)+_8nCFcM* zRC1qprCC48Z-HX{#Bc5j{{udnb#$BHXPQ|*LytRcTuU3}j1>1YZm4@v+4I50; z?;pX>VI58Up{?)}JUw%r9M;}`DnbKtsgI1!t;Xkk9=?pZ*!BDvu8AG+8DllLgSm%A z?p$_gMwig_~pM9U;MS;PK*5Ut+FF4M;l=;(^S8d@7 zcq^cXXBi^oW_;lbx>Cnf&cp~`u!VW_YOg?z@C1_*)a*Luz@$O68Gk_r zZ2~_f{vC{C6Lf$20`541SNNuv&!5e`&T2Xy*TxJUfkZmy1c)0RNC~ zIdrh;UGCWj@2a6iVRWHF8|!55abL2R`;yT`AM$b+vgq+j+-c@MWS_hCeSfFg_tVbX zVfc*YhaEg_Brl<}f|sAM-Z*?l*A_lwH(^3s9X_Lr;WPdk2^#`!Hhc#45T4>?!sWcR z)4rIiP1~|=G-Go{+v;gs=9z8VLVYIcX4AKFAHJMTjPfkcx6}om1wF9vER}>cwc=SC z;Bgwx%C~e4b)=7^E!C7U*Wp_#&c?SaXyIGBh6YB`JWd)(`xw5Q2bMm6#Dt}UY~v?~ z9yWeb=y%3PzA(Pr&;0+!zX-0%J&Zz!UWVV?%v_T5u0pFsc5v2|6z&z5yElG#ZGY*s zSyw7V2CGItaRmIA`@dyp={!3sN_a$w+ZhW`UXD4;ghYo6Ee zOeK6rc_!x$h;E_K=gyq{mUc*=Rf#+Y&-Cu6PIwdH!Xr$EmvZi^>k^s@{>nNfeK-ji z`vHIW;Sv6DWNmWzF`3`#k^3dOgkmG>((~|53R)iFn|2mH*@<;2k#p=0-*ghbNzRL( z+2`ljPr(cF4o9lfpS&MP-M;JJuA#q48yLgi3lFvJ$8Fw)hY}f1-XS>|!Bu&OM9#-V z^a=*2OSqf#9Pl@j`{ZQ(I6*(`!MzgEGvvF|XBJ{>Hyf4EbN{Cbti4;yOn;WKW< zm%9DX-JDTdbFaH=;{onsT0H-im4ZWtm*KmdcYnfvQo+sc^d^Jf=o2m>&&izeej3|y zXFP?^=xXG;?h%}sO&r0EQqGT7iQQ_eH#LN=c$M>}D2Fq5M&A@Y!2Fr^V6v^YShiIk zV+-&947|$Z&T4QK@9O=gNAtnpr)+LrsFrZ^EW`Fv+1*jwY-UvkGw4U z5AJTPD^jP+=>sEM@;*xVN~Oc(9qtt$4Bx0Uwucpg{-bnXGp~~xy%#PwYg!l9v~u3{ z;r!UHB;R|iW8iH>Uy?^1PJx@!#zN8#2G{qOSq5L-369L1LD*XNMEWf;bP5`?pYb@C zdiV%mv)1Xm=vpKEz1Lk@(~bRK@Mov&|9YnnTj|P^HQ2dZNYSI}`2CsSo2}-hjpC zyTbnz_P_c&oA0pPN5guTCHKlq=*k!-hxU=C7P^y4+eN=7y1z`|A9`0XSK`4R3Jo^- zlsNhP%_)2K36X30@9wm^65UgUdPLssO$yBe7q`=1&P&jz-p~}{%lF;1qnNuPqw%B; zx3!6TZ6t1(ddm2!(FNq09J-MCBJ}e*{>!>wOMVi*2)vfI6%#IPiRk-U?$|lTc?(nb zo6_yZx7h=sOhvpqsM`;xqlVao_!bc4) zOOCun^}<)q+U+Br(BVST$IxdY-^!ZpZs*XhuMPbWx(@FPO%S^EfeA|q9irbPuEV1{ zy!gM(zogJX{^{FYLAfL57_x%7$HvWDP-Sw)%PC_tbu;OSD|d4kJU`33P_6Ig_+=!` z?nv1?@g2T6G4xLpCVcOejLSI2=L*K@a>i>cdb=?}#?QI;r|ie|c2OBk*8NkA&wqI({F9!5PG0g@Nc;?Ml6iMX?)Vlub)STB zw|7K$_kEu9oRk)xbTnm1otxe%2#W4*zLZIx_ZdGqG!^~;=I^O$!zW1p{BQV# zHSmohF9>fs;A02BuOfe07Z{_6KCf%&Rd7hwJh9=C_sn0Vyt%T@&hoE}VO@0edXB!( z$Rza7r^q`Z=k#YCEQN=Qrj;~FW-Uz%{UMUCtP^tPP}b5W)(^QC%z1-J*3$IQm(&v) z_#J=0iO5B7ynQj(O4^`UMRubM`ERejNqxzX9)My%N2j=;x#loBnMrpU_Ti_zU=4 zzF@(g#n7$K-KjaBL&pw+f2VGb^EX1z66PhW40C>`u`jy(1quFga46h2Hh8d4TrhiH zyOrTy-XQzc74P(OXx4u672K#Zep=`w=JIH%Xi-w&v`jOW3E;T}vQ$v3w4Qt+pV$O-2 zO%KrAa)(H2=tBBH>ik%w4$JWen|36IY!fDPa~$J%1><=+<2sh{9Rs}^&3XICK1I=w z@14pUi$de%-GyU}wVby#b&Jrr_Z|UXQytmnu~fC;PMoZj)vlyQcPeK?Q`Ck9)VVrE zO~_WYy9F*8SYCrONGv}RZ-jrH(#C1O*@d)lI-E@AFx@IH8! zz|5HpE3r||lb3+Y8CF`O=(FTJZY4N=6a7|>{mypg&~)y1h>t;bSoqx?-r&cfI`SF2 z@aCoklr@Dtfw9&k;AC@$_t=G#npV3K%pKmBquW|4Zzx#_<__=Am?xs^uVobRuNPcUOKo4xrF<2R*jiP6db6Qh%M za8K($V|3cC7~V@zgENI@W)86TN`1jC_)F2X49?i%=mNbS^M2D{Xv+HNJ%iGRqN}Ut z-Ic66((E8m^4%R4iSK41;b$Nrz`XYO}8k`r8c6r7Ci@!u|c{Ds+#Q{iEw zd;F8(!xq3xg~?O)_$RZ+Kbbv#$yfIHqxqM!$5e>>-{kK?=53q*&L7Bs#kPc+Dbj!N zyI$HP{GWu~L|7^JP)ols;zt6bOme(Db=C1v{#7$G( zmRy7$*@qoN1~v^o+AQ?D#EqO!9u2+RL8lLU+7@lfB(BVEbytiL_%aI_guBkO?0bnL z_kIek3$H-G<*D6m>NRLFbksW|yVK*9^3OZq@k*c7?cikYj1RN-lg(a}yt(Sey+Q}-3{9Lr29f_eo`BAc(2yb znk{2%GGYR=p*<50!oN3jzg5$~#pYiZ#>=5Kee73vJF=!XvvmB&?(#}0(`BV_Z=ZAj ztL!rg&oCL@PIQ6Nc9Gjk+}JKacaV8^7EaExpJ$B|Uiee;i|(_EUa*r{`_e=25H4wi zMx@URntkiH2`imBr|F5A7dQQ@)qmgY;L0JpRPZ3}9YbAm>Me8cPYHXqC4H_J*!F|x zC4JI@C2!K-%m>aXPmp$}1!Hx1T2YYul1v+G;`?*!-E+;{y0Fzh!4Da;?Ubd+*8+c9 z^4fHs{RMGtX*YD%EA4^SW-{-365iHE3*Ieff~=X?SIOE84o_Hv^D*}3u6g=3`P|UT zy>3bt~Tlq z?tkXqh&hw#d#ShRw&jkBA9`bSm14$GC4j54Z{V zBJ?HEdEpfuE_j_n-|fZDP2z5Sps?vR;))G&n6ft0Z$+Lo7ift@{wy9dbg zb=r{+j+?nznr;7E(sfoFtHHJ8DTRjCrWl*glu#^rJqC{uD#`ZHMbd>Mb`5Zh;J{2wbWVGaSAexV{0mXKpO+N3QC7<~% z`SfVXXIC^IV++drKW*}P{Ve(XvL&DHE%|JY=3{I^Q$nw`$!Eh^@|oC@Pq&tQ)`K*lQV{AlILX~ausW?kMLtFAmZ^>tIG#_Itni48+lh6FK zZnr89C&BJxZUa%nPrRg4|IkIc(^arI&w z$=rW`r?vcGpV707EHKH@DMx8UGUfB8+lG4N?8(W<*^xB=B+cjjN6(6W@8G+x54?TX z(&6Q8^6-#HEqCM&p{!{9zY_m5;+wee)}N)84ma}+zPpKknVlm%0`FP~-~Cx??QoG> zg!krs3;0myj_tsEQ*M?!VA|fX?c;v~cW{TedppG4+XuOOJD$6@ST z#DzSP_2JZ8vyev?y->7j8gBo?)~veUs#%}7)oDY6YIE$TY1fU=oyVXfd%5?dmi1=o zH1sUAU*K%kaBmmCycOWkt?lRAD@Y^%9>gzY41~VQ_dL=ib00i+v=83Gx9PjI3+#s? z@k{X^WgRi$U;FK~d`thUqs6RWs>Z+UPHi&vsJOsh$iF)F7=-_$ zUkA%uae@;ZOd;#n%Zth@ zrOu=2uV5E7I2@~X<#$rEMNWuk{d3{6XioxuJ6wBQ2VBmb^OtqxY|d9(2H0QohVdcm z#-_T3KW(Z95BDbrbSJCRQe~;7mSrtD%y|$@6wL)+7f$YJ-_H8l9Kj12 zmn#0(!3Td5u@M|bnCQ&8{EEE>-j5^xX3k~`Zj~d)qvPM5D(90HU1$sbR*6msTEV_>0qY}Ag&9v{renZ$Pq3_LZY|4xGgu#!xxI*r8Up%ZXqjdPOBH9C9 zAJ``QmuaCX;B__UPQ6x_rPZ`=F88KM8|9qU0{G?wyrC%Z4*v@t3Ry&S_{`ItQJvQ2 zR`=oxpFJ7ek@$1TSNwX?3BG?ue~SNwX=`%m5&Y;oQp~ps;hVaEKN9CY(hA@70ck~6 zsE&NU!?)}O@5aB{!fRTKDeM0XTJMeFZ(Xo z4_?cdY==kpLVHD*j!ph<+3TC6yfbgOusl5-T%Cvv*p<12ULcg=?qbd~979iE{ZODu z^m1|sTsb+Rm7kQm56>|5;K$InTk{*lFO3l=lfNj0uFAcnmZ7BvciErLhIU5ZzBS{|`mQLW7W`(N z3n}R9WYMkFau2@H(HPzWfH!j9Sl?pq>(8WrMWzw|^js%QY?T$VccRel_mqhsLST(zzdb#M|1N3Q>&;OWt@I(7J z4ZVt3Q(X3)oHs)^u(L#H8Ec&nTsg?Sl?mX;OmwW$o_2GmH+?Z@ded{Rc8yQ_11HyX z8h29GxplmS_@z4`^c6ZZxx@V}-c$Svx*K4uccP2Ua5=izo#bBr!_ z7jyhV*Evf!^PP?DHZ&W(M112y&b7Qiezk8ob3Uq@y@78p^@E0)H%y}ZQ&0FJ<(=9i zi+4leGLi%vJMkTu34f9x6uN5@e1&RpCN zoWr|76sP{&;I6Fn{nQF3c2fFs;swCdC=C@}(BL@woiO{~=$|UG zev)qYr602y*9r77dgqGPck!Ay_jIv)5(m49@mCP%8t|ia}KOUM+{!J)!|MeZ=oAe9kJvs`h>qQXTPSuNp=qpOW{ngNuQIY&h(LtA@gO278?u;KzTl`jkIrC=j zX~?uW%lzn~;rRn%bSLUAwo}hihqrLj)=bWZNjlLdS5l{8j87z=uhA=+^PZWUJBL@& z5?m!(Jee&7uB%||BXvS~IxA%|a+eqIC|8)Reum_*Fj8j|RA03sK7HPTHFoGjb zrSUGNLZ))w7m#vJ{pv69m=?Sy@1@RlW5b~A=50y4M2}J~>#kpPOYkXW-I}W9oAdul z`iZmuRh<35@1JJ=bq?|`oAdvI7n@&5;BAMd+)Uv~A0J@9Kpb$;@TBMxj9et|5K6yi z2NTGLc}0H;Z!EZ(L71#pdvCkI-ghnc4j^ZGd3)RBckV!Y(W}iXWDPUlI|thH`IhvY zYnoll9Nhr_CG@WVJj;%ZZ}z%`8o6uT;Qmwn%)Ob|PS#WqCTVw2#t!!9ccceu7RRU| z5=Q;L_7@B=?_2csJ~Tezp>?;7u@c7g9T3>`;JOFLw|hEx+f1w77{YQ6y48@J?t#29 zZ>y^&by5SSK@0s^ys>(3dS~>?L+aPcn`pczCQN+ns~f|)tJ6ile0ikbs_Aco!?Y_0-C8zpYmC9ZDw{W5j!L;W ztCTx)DVu&{El3O%pvTYPTX5sEe=zq{pe8_Hn7Fk45gD`QX3l@G4>rwt!+J1bb@WYr zFWL7>3SG$eWWK{I12qi^q8sqdtVdTGz8W3@K7sGXDSXQqux3(jQploQi*$zc0JvdJ1bPuI`ZF+ZKzkNnhE z-PC00CG*5jJh7D&ejt8|9zMztH@?Ykk8RKil)(Gq+tO^?i zY;@rI2173D*s^@;4qtXG4F*-u68&EHF&!egjNzS$nNRB$4vW~7E$*^t2+ z%2-3*opQ5lVzt4Ug%{a5S%KhK%Nv@B42b<|kp4-UB<~&*KZ7@qC0#b@Bz|^5PM+x} z*^6X6;Y3Z3`;q=Q&K<0o;I;bOaC;VUp`#n6PXsr`E+P8w7XC5!487bQ$!7|95Y0zu zr;HJMJS}N%AWbQEC4bJkAngjqs;ly?q?tdvI55Z0w?Daj;y#zUEfc@(d2XT2^woOr z6vlLldspWDH?H%cPlxW;D34m_J4b2neKCH)(I_qkVrhrE(5~;8IPmbB)^gL=^=X0= z+7~1Ca}K{Nby)yimNz07DC;&a_^_JqFmL}&DjjnuoLHWoA1F3HG;MYr;ZjcKIh<`O z`f-~D`fm3fzb@Uq;r8baCmr8%c*BGH4+j@LynA%R^N0Pp|2o|8`jNvkhH?*hX6@kw zwQ~IVacbv-0|wgdFDxDZC2wGS#l5Ko)}bu;v~y?^_`%zIIrgysjL^CDBlOTu*du5dp@)1iLgyVCp)Wo* zLZ3gRpyrbQfgTedx}0|~M&PNyY=VDxQF9ZO*ZSQ_X%h)dwznSpZ8HUf30a=A!{}`v|e3q*)@EB z9T%wioO998!I0Q@2_7tnRRezZM!}qeF6oQ8yY@}@_Dpo`o7OTpJy3)F`4-s&pJk2|mi zh^%|f>E;dhB7b$k|0VWPtA|^&tC3^o(67aDu~%hSu><71{$1!zB;DR`nm5cq53mG3 z?9K`8i!E-P&b&8mV=au({R*M`5TpUKuV{Y$ipj>~}K>Hy2hrS~Mn=Z!@e&`E$tOZuPnFX}@isranfvdi3o-o3# ztzM=rFLVDF zW3Ao+en;uf0-pdrp>#*!j8sc!eN*XIC2b>cqtcrMP9UEIOK+5X`jW?3+)VPBPx>C@ zbE(uPf%9vwifAY8z_bc*QMEaiOQz-Qz zpRMHcBKhng?H;99NZKR7N0eSJa3c96TH2O;29d|rmY#e#-nI(vAK!s6bbGtaTw0)N z8rBvEWzKpRE`}%04uvlPM~k2h0uN%Wm6K2Eco#XWyHq$tgKE_ z8+Xi4Tp7MEapl2FgwKv|JU9qj6MM|b?TP4V5=I9vvErA`XrH?D#%pNPsDK{Ex_#4~ zN;lBfH1faxQQF-X{-iT+s`d=7l=F-74%f%X_i}Ds&beHTJzn5s^N?I)>r=YhwDGCI z<}MrI1^ija$Fw68ni85rd#%Ko1@H+u*7Z%3tgc2clw;il+^umTyaMaKo`ela7O?5J z^R}nIaXN13I_Pz@e@c9*OH)U)u0zwk^ltP>6h1(?`R%u=<8J%*lE5%O{zaolV}HOy28tGM$*2i%wzNurDdhV znwe|O;8QXCtKq)j+QQOdb?#W+XoX4z`uVVSZ`SW^&hMW*@>{B#gPAS+-lXCXS zH#Ddj8uX#0InXK%^S6%qw^!mYe@vR|N{8>3Z{|TW^Wa~=a{i}yZA{C3Tg^g$co#YB zUXhM1=}aH*w?n&Udyq?nzhOMDlJU)?-@aK2O{c#ztgbUp^;f!(bCPn7+>P(etQz0I zx-7C?ka3rC`Z6C{ugiw7XqRPM5qa!3>LWb5;VUjcX9OOEXR6>9_>Zh}GVLq*pGH3Q zoQn;Q3e;?8ZTOP6j(n^QGM0)u$r>SVch@r?KVdF>${0Ad$w}aB3dg1#+fhkhPWq2X zuP&P2qdRt^(L7X^qra82Rc79khj$9|ae*&bjr?rhc4zOq17%5C^ybTDzWBlQC{7(@ zts4v89~_H4)fW8zzM%9)&mPP>>MFXvh9dCD#~v2=HX0ngcrk1A(m+ikxGa32?z1}m zM$XVrzK%If-QiQ0PrloMrM{C_0Mo{FsXOI2Fvo>vO@&XE{mL|S$7#rp0(;RNSBnfO zFuLPnfd&7i9@I6Jxi0%GR|tN*6wpmCq5lIm`H)urjg)_T`L|c;fc)z-I+#XY(R`wL z43l`riFf?}@b>QUQB_y}|CyNpGYPkl`$duvl}u1E55twF2JImLOIGky^A~ zkOvZoJ_e(fQM{2p4VR}+GlM8=9_L@UTG!^M2xpMB265z+Sf zyuKFd2I@U%hW_pJ4pjzQKgG{eu11dV>Sj`hwoI z@xhPt-Jz~g+#@o=3bla?_L*D@+J(;p^cT99{X=Nk#lVgY&9K_hgnl{Nj+L~dYsAzA zhAudFj)<2t@7y^e+HPb|?i2YndDln}AUT!$M7HaFpw(>NxliOZ-lcoeeIkZuJmo$S z^uWax*d;{m9Ax7l+l3O+PPKI~_l%>Xr<|Ssk~4w@%n{Mp0+$sT)+caM_?U;ig6=HO z?PHyMhO?g?;1qI8k!WOYV)=dE(RG;}n!7RMFIZZe1CgDyRrJTeKedeP(d0QCi2Q;)eV2N<`YE2I`u+Sj7+yEwyMdP5pz8zHB?J@MKm6|K7aDG7e!P9X724s0fA?95qIIpy zmW^*+wtW1k5HGKEMr+N84k9p(+e&Tg~AuD9@-Qv5!&Gw$+ z-_x$XtUX?6kIs<5e{Q<-#UkA=)KUT8d1+tfBK+;&38mY3gZy4UxNuk8$!Om!Y)MSo zqPUZD^xvmG=u&dHn}6CEf5|3-O}eYUY|&O)<+;HkVA#q$g=X%I>h%o#))ochS)05+ z=DwWWuFyWt7k7Z;$gB})<<7jFdBHgT6^`~?YejnNV~*`z?+&hlUZHa-Zo3wIf#>yWM&9SpvHQ;n>pb24|4M>JS#%S7P;Ys&$hTRCp_UrV6XDr6By~iJr6ver@kbYt4~; zw$8C?-$i?H>Z|b3#UmX-+lWBqV$LVD!IO<6y5ezWpV^zOcpq~{#G9*lUvoypd#2+3%o!1{1)dZ|}&lT3TuUdlq}!PFs-sBLC&I2iYyMnt1hRt`PE$?k8Fp;J)>W zOmr=oR^)Nz8%lhr6@&gZ}y3& zW9Q!@{?mt_KJStx#_!HOS3fcR+N(F;k&S*b6*;W9pS{1CfxN=L+NL_`uCUo_OQ@M} ze7Mt2XFYc@f7I?I)^qW=TAw?vwOW$UrL@bofw}PLub>$@4nM+qCh;R)_!0TdvCx0% zKCxeuN9(_N*ZQybp;ohb*ZR-9*8hW)Z{WoGpLEh{k!;AiGcLC787F#1OUJPWn}_aE z$ZAQPkIAW-{;JCKcy>kKTX+{;iN0?(?`|VAy5T$Z&nD8YSm9dbJyEec2b&eyq>Fa@ zc&-38Z*lkHo09{`vaZ74&Iv@gAFS{#=A-s#mG2<-YuYounD|1{)LwMF!Pnqp+c-C| z0s66W(Sj{gNDCKoHwt<7dj^CDao+Zmla;&oZ_Um+aMjBVu1PHoMbNaO&w7fBn7;?m z?G?oZA}=WP1|ka;<_01Y-C12Obbv)?V)Hgu?`~tmP!v#nV3!xY-vMNbh3MWD?nyu< z!hchb%Tu)#-YkDoU`w?%xT@_K_Gm0aBKxl^ue)%|^T2oi^%rjW1h^hBp*z@q{e|7|_letB_j8~P>%kd&4>`T?OyHsQ zVvi@T@cuaV(XF^H*43GkIevNX&0BtT?@e1?v-(zfX@`sF1CLwj-K2Mu-c^o#`zhtx z<#~d0$v@HUsVb$6^|a~sdxzO=S~SdVlL=|lqG8=X;=i>AduHyXO#bVhXnDGZ@vibT z_l|>WGdKg+>B%jG&q_#QjG{RG?k{Bne7ShrmA&`r*}UyaukO=hpH;G26u$=v_hRrn zLAciMbNueddklZO@=N#Sv6f8*->SA|1S^pP{H~;W+49C?x4RF!-4ci2m96fnHoT>L z%Y)w)k6U?`6}oAec-+cf9=Bqdb&~Vrqs#FlhfFip^ocu-Sy!9;|A~7Esrz6)`bKw+ z_Ip?QU4_8{WGajMQ!j^pUBnn3hj$9R*V8P1E)U)-&+837w1ft z;}{Hl#Rr zaf^8J?LTc8M%np1lA~H5pe%PF(srV!xsLMD)ioj&!_;~RL7;_DP(HHC33cj)nC=yAcPJ;hO3>{L6bs15osa5XQCp9T}px zfB7-}I~|@PE86-l*BNH-QF;r>KiH?7v0&`URzhFWHqgk%HwerDj<%{;I3peHR(z!}vdWjj)mYlI7uZ zHlk}^DZPR0!KH7n%nvn4Cw(Jf-Kf%W1=#x4mElKJG`_tQnTE5e(6IvSk-UlX$7$bI zJi_m&LusyWO^W=Icpx8}8s%};jOsr?_lI%TLVLn7KDTrhgL%;6mwi*r8uBu8t(CQA z4K$K{w*7gz@xkM5*pKDX9@0uW$c|;2Re}{XA^1^NX@_o|thF!1Mv0A|JmVWl}jBx z=&2JGr`#U)aRw64rhf(pY#&?2miYRo`v-Szhi-5-X%4h0ui`AmBRzcO!?qp0kN-3$ z%(piCTe-ALkEQ1Ip^6uuTr%XWEk)zSo3V;kya5 z7$4!7=&9t!EmrqWGmqmJ(9HMD4RfJO`j*JIV&Gl(0X#Tk_`hkxu--Oghp$)N;QF-+ z!Skz4TXa6^>;0VwzqN0ovpMov^*()*p8SKH(opR~;N3rZMyqY*$9qQGczUu3aOSt4xYhyIu+W9hT+Sp^ zdC#(r{cqs%SLl+B_sQY^;$Nf7gO1UMr)0l)$2P~$5ou*k+Jcz0OPsXIn6z1^3D9a#+I{oY9HYR-W|Sfr-dG2BkxhZ)vQCO z$R7@0x6;eLfNaYCUB2!My1oKy<(=zBOSf&-Zj;{wL4>kRKUA`nu%Zbth)=O1 zP0Bx?ybtk&$L@e4)Q+6^CJ0ENk;~HNO>H6--C)@L;5vVL2m^LbiaP+nfJ? z#iu;VjK!xjc#p-W4Br1Od|LLu!>7~Q$CsrT`}j)P;YIbn*vA`sH{`Z)qoI4~rN_KH za9Ycg@W_?7SXXiHU(rqQLia%z&6@mCR`8>(F4=%KHxeSBF7&>R&kyN{^N7E&BC}~= ze`6C+H{_ggD(ep?du;yBkz4*9+uNLz*1|gIaq0uu<~i-iN00noytQyIwiuPSWX?z) zkTkWOxstc$;^5(Y>^Fr?}$}o2^uus+HNhy zAveBef)|-B2j5HVh0iPkK8e)lihpDEuvTBgY-E|dxMi#JI1@VT^*bJGN3WM-c@}zV z9?7&+_Tc{Ls<2T;FZ@1qVxl{4!!Y7~+=;(=&2o4B`26LoKkx-2A7&VRNbr6uu|6&^ zwEK0!KCMH$;{rL|{aSOjZpCiSOB>th zw*|b~qp&J{V0s>~5*-#@mR#Xo-qBC|cao9Uyx5*Lk*`3w zh3tTTcJvs@Ro`r@oJE^2Vy=~MuNv1BXN{>b za?ZKII?AB!)6KfF)wWs1E@`0)Klz2&t-d)BJKU3;^Wg3Y!zzN#G;tjM&>8W|j$(l>J95|_Uq&+-=}PO|r_6Z=L?-4(-w*HU)@ zWqvC<&o3Mr{InI?=(5L4J^_RW>{*RNt}uFfi~Ot~3%3DVfSar~MbkXY(GH(A_Hp`l z=bm%n+mpiIq>gd4dED+1!HbV$H?-Te3wd;$Rpm3zpY=%0x#zu{X9L)opSK*F!dZkw}byG)|S?HI<~vI+o<<5WIDBf zs%V?!CU|Atf52TjPqEGx!{_bvZngap&tcyHSg*9&&JAv)o>6?OJYw5gKSlb{VFkhV zZGFIrwD2>0*S9+StwsM?PnzmtUpcuO+9i2{vy<-}_oNke)0UU;v73ROYTTVk!4z;T zwaVpB3yED(qGwtkp&Dcy%p9LNs#DC=$+F)^(Q}rz*KWoFnqwdB`>6C?o-O$%4 zthewlc*Gghk_9?%nup!Wsk&C3IhF4J3+GeGEB(I}SDK@>HOwBnXiznD;V3q1@>f{t z@^0`JHU4=@JGO_|C2Q}c&W$b!{~S%8>$S$Lj9p`-V-(zNyYakkxiA2o{y=p5gV6C0M%SOnnlywpDK2PW>hM4EBk+d;YyXb< z(^__%30*}8to3LIY2F3X%z0|fZ$IPmCTUaVjPLDhr)P!fYn3Z`!}p!Ovfi4$Vy`~h z;-4Wd8^JAU1N0KE%J~%e!~?0D@3l!O|kX9xhdAk9PI5`A7@UM9&PPe zX0JrD#0s}>gP;6HPh3)B>xoNTC*j+6Hoa)$xb{nId47t zzCXuS?j>yHHeg2t?YJJgR>4`Kz~b@Uce0ODao_mvc5vO`Q6BcPI`c^z{bx;Ukx!G~ z5SP94N3<9FA?|{)lAkLIM06(Txgz`<+*hdc;P9*QU2ACbxo5qZ`AAQ{F$?aS-~ElA zp3Mt<@%7J54z#?7EkNnMo|y~o`)>CD&SgEa=)2wj=VZ@j@}$L<7cC1q_IX*9v5_*^ zo30wx(KGXrMGLwYP{t+v`xIwe+dVcOuKtB<)yJbQUC@DTTf(ACPeyTftu2@DjP8Gq zjKRmZ`6m`1f5UsL&+NAteEc=?_5}@$N?g3rY*BhR$V=#{G!3!2L)G@AHS+Hh*)C&7XUjYXS3Ur|iep(k{zw?8kb~ zg_<#(bAGS}AA6MDl5iD!IiV^3Hui4z;qS0>q!ofc*_qd!7rc}IbD6uc0X>ywZe^~2 z)twlf`O~00pMc#hW_f*zO`qnZ(qIxs5Ad^ z+BAuNUA7>4m!Zlyi2VX`ra8+LJ;!t*ee}NM&_(c6ZOVWyYA+Do=_W&)IO7c7^wPc2 zv?~_Rig>s15zKfRJUf?n;aP-zD$_^sEF)&0Dz@x#tjSS-4}W_V+AG@?Xm3dj?Nu3R z?0Xyq7Lj+^_c+>9{EX;Yt-ZhY_9_3hduojRiwnA@eW(KVprUzSkICEap)SX6x9!7z z?BBRHK5_TZ8S-H-e+=ReWQY6r-}F3%el@%(W>4=$mE-K`sja8hS*SY8oc%xYwtE5_ z)vsuM*EszaA1ir$0{#G0M+BN%0<1-IQ{k&D#!czZh&DLoJx+NajfDSm#x0g6sNKmi zWA!F^W5?=OYPa|EvD(IalqTrk)8Ty6*U*I;hc4U{Ll;C37_x-m-nM zw|Sn-^fRAj_kXoB?|(s9z&~9hUNz?vqjGwA_PQ(l*=w(eXPvb1+MYMD@Rv=>uo(Ey zr<_>$-@-fdyLYbK#CsI}r-OY5coK!BYgu4{1q^L^I&bKi{`o_fvBw%qdtJ+{1r?&Z zz)&*3eU5Q^lqZ}n|JTuTzNQb3UxGjVM~o~0O@go5@0WaiZou#NVS@zUDxaPeoK?-` zsd01#y=Q@$qd9LRw1BVgBw$0@ioK|4PC4aR2jW|z`M$|F;dJy}cd6(M=#gvP@Spgo zSRBXwx~|}E*6K<(YdHIork##_(vIFqyrVgFxo^vJTM=~f&aabqU? z*@`oN8i!huPl-#nu*&Vize3Fre0J>r!?rB-JyNrdJm}IpmA|wk(pc(>e4zLq;(J_? zy~KC8lj>9E#I>wK-*6}PI@QbF=6tpA!3$m)eTK`u(Q>&q<~+cB&GVc@pQ-)WNKVF= z?0BVgZsNgLvz6K4nom_$~^ntx$E<+=FvRl-K4vV zcCa7b1S~6at>&g7;5~ANHy2xiRM~Xe=Xz^%cn4fE0-o`rQS7khJ2OPGcGw+YS((o0UpTSBcwlR@ehUn>Jw<`I0HK| zrAMZ;)DbsrWBn^Q!=W2H&I(Q$0$sruRE73{WA^{0W3Vi<$G<7>=Cgxt*2{`)_UN!H zgT8F)wF&sn<&iw(fl}%z4M_Kt;jDM5;bCuL7r;NS^^6O>)^j1x*pPe;uIU>HtxGNS z{{);`6W=FlylA^qRC-ZgF?HXQrdrw-@4cWv;vv9azG3fH=JZMf4#JT8R&(#|Pc z&kt_ct9MU`ecN3d)+_GjOn@uY#+tpCHfS$TcYlv6qkrj)McJ$^KfPg5XiM#N-8pN~ z5poAX$TMq;Kh-Kuh37b$Vm1GXC(Kx3ySK7xZC)(~Dxa%^LrXVLAx?}KSrCp=n{w5tR%9Uv#?1@=O3jjUbC0cn#+yp5a{LJH#XK z^Ga8|rjhhUEAm_7GiJE`9m6jVO`h1drM#?f3-o%W^~kmf0r7p}LC^6AA10q;`#c~0Kz$@>7lRh3_7K9&#q04-R>EL8JTuzjtGaf1<++xUH%8S?8DESK;siI+tPcwnN|h;H#RpTlkYwy7m)J z=fkj!`V4)84-uW#db;_(D4j;f4ezNktyi~Aa6P={ioAk30}HHMeq=q3u7bF9PPu0c zyuwGC|768(Y>4B%CFSv6mzC7Xw~4NT`FY4Srd_Ucg1)?fpFM$1Q=q34k-;zZ1&)>Q zT{g347&v`duT0y`+O0K2u(Q5X5;;a*`Kc@-RGU@4J91fsJrw47Qg|EbCe1aeH|^DE zTA$C;wnWo>pQpVPO#@e>m+Rk*G2~?180B9z+lHuv^FG%cTvuvH0=GLf6`&!L>m-z>FN<{kOmpAYH4l6huU!!FNEY9j^RC<6Z@-y3ROU-2Ob)+b!ldx?grmR@ z(Z74>cMSi$NjxvQik)89G~IhxY{4^4g)bC;`vm8IcjFU6@yCe&%l(J#{dO6+_@n2I zGP=hAyB*)gmW8|3Tc<$JvaPF%X$w43(cCH4%o5VFtw1E3FljLQ#4lx*1ZFg>W^~Qs^*H<~)Zf9eX!>pV)v9)I zYPN$@;!j5~?(@MZg;n6xd~m7?oSN-QsG3$(pIHq~T?$S;l=x647r+&NJR4su+^v%A z8eBgHf7V{?czyWL&|MmpXEzypUSro=R5~sMKj<5PZ2~?aN_3A>A~v3L^&a1~mA!sn zisC88SDp_W3H){$yIJ2Eil=t11+RR0il=v#D1MgW8C~t@EhX3Elc3oLot&r8hpiN$ z>Bq9s!FdZVGqzLn7W9e4qsNxM*%#uDcm9Jv3|6{in}#j#@5}7H?B9DL&jB-g4=_DE zhV#FNZWmw2T!3G=fcGO+HlHW|8Rzld(=&wa;Y6Onq2v8rEvD|P&J9+g6PV}Myp#`y z6`AumKXM-%b0D-8wyV0-muV2fiG6Y@B2=9@)}f-on}d-I7dn2>w*O zWGQJkkfyb~8$MP%-2~FEA?;e^t1*(Xcuw3Z-zh2K!w&*WE2aJocY6Ik?jq~D!HS%? z;fhe#Uy?#ydu=}!6`aAY;4C)hnUdA@b>zbr zH~;O~>k01N?g`ETKeu7~mz!l3AFU2VIu-{`8rsMH7qk!GtM(pY0b?1v|0N#^U;6Is zcjdC*rE|oGu4TPsZ1=KmeVhKN|A!c7`+mIC@bxAwllK(zWQVWj-SDjCBZ9ANVGU>Q zMc0K?+EPJVuVg)I)4ez7Y~XuXo6(bsXZ~nJFvvfL{)jyH_>|DG^{!Am_thQ#&-1%jK%LL!|?njWPK+rX5ane*@1}8^mUYJ-+VyV zVd%E#x7NpFvnN4|t_U3{yDYSKcpxG=AEo(o{vsQnBy&!;oxM}Lukx{x5+03@4I8(T zP5PtwhY|f-cj;d#)9X~_4A`_N-eV-6`EQhb0N-+?^Gx$Wcy!!~@0UPtRvp3{DJ z>M!`J?@pWQj(^4fPCGj|mw4LpowlQ=I{kO2KTY`KKJEWbADi%LbGq+8k*vXULJo`QCXr`RN(A{@h0)Pge!oca8J$y2!L_vgHX#a*86!WF*2V^iTL;;CEy zU8>IkLJ&5zdL=bWB#1(`~NV1 zgo91MN%HpB**eK@0r&VpJn)GZ~kxQ5WeyY*#p`X4-b~W znR)Hsk2^Qqkja@U@w{gKkajU?+44~Y{`vNnayGo?F&BDk@NtM29DEp>(gnZ155J)8 z@YdpwWPg zImb1~+&d~cc){XI%!CrE_JTJh;J4~(g#WAM{R6)1OlmH+rCMvG2hy2TomI-Z$F-_% z#AWbJsa3*>E#y5MA9!rvXJxyuJ+Nf=K6iZmb?u9GpW656yw%!tk$*$OlrsD_!7q)U z>k4K812YEgrIE&A*1|os8CbmIbKrsOG9)F{+Ps54DP8k5`t9R>fr!4df0OL@#Mh5J zt28p_3@h;?&p^gyP+0SLiff3O%hHS1pesF(d~XB0E0IIq26jEr%?`%e%yC!$D%G#D z&IcwRQAQ`SSv#_m>_oJ0c0S(*V<(@3-%9M!X1=xX(sAFVES3A#3Vdidr)b$gAbq6wvTULXkXD(yM1$%9)%zDEVg~$puEJdXx|kj zG4T7w=j{u{v=7?zh4wu`c`0AfKE8dSeJ@;Tx9>nq`$Vf^+xIuh%lL}+{XM3A2d*0T z*2g{e_>L->va^)?c7NuIjE|{ZT$SUf{WkX6&Uwj=Jt0}Y<$w(zGCtGEvhdcxqKOq%1~X~ znZNs;Ia~ARFVETY?#AZPnX^0I@0t0Vf8(AOLuZkhkl+4opCi(9bPmE^J#q4V@c#dj zcIwRXoCK@6l=G$1#~o#zzZah)*Wq&{1A3ha{mg=XF2+VH8~BNDPGSAu%zozW_quw0 zZ)}5Roy)h4_}mChMwY?n#)dhDE<6Tp9ddxOMOQZhFYbdHA|D*n&cQwneM>uYyMsBN zh>wXu`0~hI8hB}LoRRMT0_kb?xM-CXd0sBY!faTknoc(Xak46n`o*Uyw(z*?|Rj)C<} z2YP1CL+_f$Iye6&?p+&ECi-jOEEuDkF|@cp^zmZ@-$B54V0fSd-_|98m(cYD-*XMB z(z%GMfbV%rn;YK3r^(yDOwHOk=cf((OIjM-7nF{>=@hsomI|)Et|s8R3-}k!x*eY$ z1FF^oOZhgjX>$Lrb-aJXxVF*f`Vk)8QBbso`6!rG5S|LNTYy>9HJ`()j&<)U{?&Rc z{Q0MYKU$ZCJNwMKJP_O&K;QbCzV!!p)Hlt~<<$S%CCv>F)2ByTQnP+QpPrx4(y*U3 zXA^yTYe~Y4_r}?{^3M{@Z(mmj?SGU$y>pWPl1q>cyx@J4C$rnkY4(c?(93d$yjecp z=bmE~mk>`uADcqhtB*xjmyABPE~<~^UHVx4^QS^bA4?e3$D-qzY3ifBl7rgh8wkJ9 z${*Fo?rH|ko>^a#SuDRb^9K6kLr)vOrXzZi`0_mg9q7qSK6JHvXIm}r zPqbS0j6la@_U2vT3%}p<)SeNX)t-%9k4~YD{dw}+@>8D}`3YGns*|mSuKd52pM;}= zy94VlocUq@edbWcAu3;CcZW_h#ux8!`iFlYqgTzhB5zQR=C^DgK8Dw2Z|@!K|u{w{fC;yK~N`Bg=}k z<=0G*9yHsPT_{`mjx6a%GP|Vf=m3_SIS0<{MVdOZ!marKYI0>JtYiGIxvp|cKF@lq zd(5+xkxEYj6LuI;^o{q>#O!O6!7dDc`3d^6T0N5w=Pk> z7Y4BIxWd)=VyI-z;%w}~H&Yuk#i#b~i+_Ij^r{OsG~Bjm?v^R8{v)vKo;80_{+94w z*RHZf7j4;xe&{S$e0_Mkd)H-)F6x#q;Inz}tn~!X=DmIRq)bw>keDv^PcuW0P2;UX$=Qj8(_45$$hQ4iz>Uj)(+w}Pz ze5+q_tr9MtZqBi%KQlk(f9ZPm+IL;;&wKXV;P<}f-?e+RUZt{!elCZm^E+L8#hV<3 zM?8~%*;nt~JM_v9%CWtR+<{H2Y%G<=n$m;~DDRuB;?q2<4#FSJ+tpwpkC-FNd`$5YIcWad~>p|5%eG==+v zz?~IKcQj1qTP!}9{tPY%Zgut^Q%!r--_z^hqWXj5$iUgv-iC&yUhW)B3U8$ix=&5I z=1tfgcG6b0tFmYI?n>l{YIom-dGJkjymPh#J1t*0+K*NABbWA`ZcGlph+Q^wBRVDr zcwfU>86Ag@^`2;tQ73i7zZcf^wZ@>QD(**LWDjQF1E+IXnY0gE8QR-T8)Z|FO& z_&kq&CQ5leaq=XQM}BR4^B}wL1xJnjl#?f(JkrTXznO__&>e3zqfBU)9#s21qG8fS z1kih{-p^=v1wM{q$D#>5^nP4m&ZvslbH&>>F}eKrL*}9Q+s(^8!6x|F0?ryVVWZoD zeQ^^uVoij*yXu{<1H)H6Qqu<_SonIJS%em`Ge2dQ2ihuUVm#&$skMQs9d^^?d z^B{9DdiRO;vf9KWes6T}3e`nFOXI*nXCA%>o|iy3U(4itDq-Dcvv=9E=+SVY@i!5SECdp2!9${5zg^}Wgmt@JM|HfP(oM+aW5 zZ_&*Kfn#T;jhVG0q5pxp}X_`BfSAWy$jkaS)+F?HvgT# zcHX)3_NVky|DIfY6?K1;He5+tuAoidV9lHsqK(1dQ|`Lj3)yc;?_OV290;A%Ul27jCta(Y#Y<> z(5ANjT5GLv2C((gzc4r;Kf~G&>-3~d&GGoELfAGkZ_S>RWT^h!JKl8n(#Hl^@%86% zNB9qD|9FTnjL^}@-y=m|HRLbJ{M_@1YCPx6yZ zww?T~lqY-nn+egUPv6JbtQoY3IdNULd~-&}<`wmyvB@OVc!a=@iQHeo9lUma#$!2r z{Y+qd4KP0J@fFVdkCcXez<46#;}snTuEQL-UPfIh-?MRQAZ1#HkFmnrz=69@^b~7; zdAX0heDUb+_fFTBEpff~TfgG;j8LT5(n={DU1dOd0uovLG&hc6+#F{L>0{A1*8S;$q{ghFuIkPrD z_%?A2nBYrN!;%5?UGwmh-gr_t+046)@H)ypAp8-3%N##w@?zgG(&mGPn0&}!l&QA= z!h|Vd^-*j0&rFyUepC9}vai33?g(q`VfbC`nTe-(&MD(rr_3mqLTl*bO7Y&$@J)0=_$S+5=I7M6RhG0>Tn@}Y3(k2Xx9O`A$9h6iUlbgrI%1WVI~g_EEQ8Da6IqCJAS z_;&93YhH~n8;#X4+H%;y8NUF86`YAqrEPz3j|y&ecsIs>`Ulu*Y%S#8QszK8cWc~@NnX_MN1 zHEkFF_bBiaZCpthL1q`fe;E8%o@q{=MZ60J8t;EP-!9{uXwMzIpD^FJpWgX)3ExEP zt9UQqK8I}Pjc{iq>6)|WnlL>)g3!SKTP2YJ#8pSB;NLbpcq8pxK1epI8R4sl8~C%P z^rTslz3X)zEhAjTIH(WD(=<<@1(b0eyt46YMn8UK`ize=+P%Fuo??91Do+yaQ@E9K zgjZJknL)Tk(;uxd_0(JHv4+UjriS=^_)q!bSaQcSu)R~THe;D>#{YJ`o3YFem-4+z z?`ABs!#DAMtKQ96W`~#aK3DH%EVIMwc%Q9zGnU!7JE)m{gWkXQY*E@tW=qo%1ccoz8Q9jnJ$9Isf{=RO) z^zb$lqVFZTfH>T@O>HKO%DEC&UfLD&G%)8C-8n0e0du< z9YGH*yi8@Ds?P1yxyOuCDtrGm?_93=hAtPH+-&+|=6XSJ4)ag>wO%RTN%E=f;{P-c zd+-Y*9>s8{XHCv3yUVGV2fi4Q%Yi1dcoV;@CNFMlqRu_Kai>?ZiF zeds;W)mLR>Z=t&f>dkoY5fQ694C6RyY|8_t$}~g8vLl*OZd@q*wVv4 zcBA7=hiA(1q*STx+p%9bCuaWk;^Q|jGV<*(`tslWE1drx`4lc7lz$k-#pB5rY!m*= z)SqF{-`M_0w;^8V&kJn$?WU{%It`7-dioR{hxcit&(LsJ<0IB zX~ti_{wZgy_iC)+cigli*`_T!#osLEpA$KO$QEGW@Ha8E3HnUgH80w{jbPFNe}imc z)1{{;7`il#GAe}=@HbDX4b-)Vy5ms4)i>Bi;hi-UR0d)dW(iH1~+56*#xXw1e^ zr^e^Rhh-tb^ft9Y?Kf@74p*8mBYX?tr|?q$@{B`S-(B(_a{_(hQRtv#SnjSf^zk_S zmDWY|C8`^JaRItf?gx-Q_-S-ki+Qf+xrhg!+{H(^JN~acMQVrkL%{zLjEm;j6w)LI zYArt7NxO_R$$!!V2B>AG#i~`Rydjeb1Y;BXszlUY~EA4_M%gtXY#TKFo^rcmPkjL8TOj*sa;3Fv+d9d zwZTK0{M^bv+c4IJ*zs8-o^aYZ!8wk6{`N*&SAKkVX{e2M#hJc<%L*yeQV2{wBcJ%7 z_X)*&3Z9~$J!O6WrEOz}C#mo-_nq{(T?<{vc#R(S!mapHXPow1kw9db*5UN=!Ea$t zE4Y6|9l~?@;%!Yt@3xq`4W;`Da=*#;2glb@cgGK`7H^VGnvb*;log zxttXq>&)?D)8}l%cS%l`PJWHagRQ%h$F@&S4PVMP@#-3*;j!Pqft&gEnE93-&W-&B z{O{+RcyE>CE|m>s%{vpBhcyGeSIFmA$71?OJ1tk;cmtCswoO>$d1`z!x|2QIJ;C+J zBKr5388_LzXNT{mzwI*uE$bNj(($&RxXWI({Y5D6gTUbs`+lD?u17p+g~A>8lGFxx zc>H*s7exOOv0M}XV00p!jq**bnLeLyQ(Q$suixLWo^t1|JU3|Nxw&JsZ*T{`#VfGU zA%CO`o~aUj!-*@0`r$z~<@z|MRc)PIK|ee52Z#32r+9P#`$i;&PUO2kP{{-i)b+eCB>Qk;3yP6$VjlBPNQK+HGE4#HvK=_jHW9?6ei z9$WWiHqR?O6yCm1Ucr97p*7UO`0Cq})USDdjPaa|pJeQGc6nW`fA%J}{<+|+9jlvW zj17(E|IOg(j!OcM`xvv%{63+>qwsYL&hBNtsy|xqP5>w2rS?F74Ga{2)Pco6p0($) z^1e%4G{oJa{7K=z(KdSyCxtzOY`E^J7CVA4ohc@KAIo-E(JhI!iLN$a-f=nphUS6Yi0fBgA>v9C$tL^DU#&!q4p^soJ% zK+AG)v2?%YXm3|0aad-NM!!3Vb%K4D}CC*x`QvKvJd$B!beRg%`55!m9AY_lh$RLTx zB14c#l902KvCZid6hCp4J<21LcL<&Wow(sCjw5rOfTwU|f=yAL;=ii!Z@}KXe3>tT zhrE&JVjgFYtoV2E6+h*9iMHKI*pwD%DIo0li^rcvpX@Qs4kt3Nwcedin$1&Wh5I{c zBh+`pQ)GtwIcaArE#G4NGQw^rEzkVx3Ec&cA^xI3A$SRGYwiGdn+OkM({%(sY9{FCE_X8 zq4*8`BfLGicqU^vgE5@WSYE}LeiM1}N_6>G;3K7v8S`qldm*;7HqNfK?TU;&r1W3d zP&Qs-E!_81V`dk8N7cgq!H)HeG(5@QxuMLzZ+Nob_wGHr;{(vR4e=o_>tDx) zgwWwzq4~R^`G17wk8%fB{>(0K6Xlt;1o`j~c(1#T)Yq$iX@76xyZG8etVcSJwH~^$ z2OWap^&DR1QHPfFC$wo-dRQC3GMfvk1 zvELZC`}uDU|5YxvxZ~}z-S7nO9N zk64-e%(e;8gyb*bOaBC;Q~o~s<|wX=AfI5OJ`E>s=+Q##1S#Kth7~M6Sq2WLhp{0j zcGo;KT{z>edBC*e6WV!j#wm5fH;E?ej`Ka+T|s;ONe@2@E~_5xZx;an_8(w-0}hnd z4FKNM&zNcKRo6Jir5bvy^uyrRLDs@z(nPy1AS|c-ZAL%b4;qvT4f<;Pzykb2$$#Pr z=+IGz4n?3tI@2J(H1d;toO)k|rWknnZGCSOK4uLpoIT0A=fis;&qU^0krwEVtzQ`s zeC28URpCcGkNIoTDhCF4hd-yunT!k0mqt&}co5h7_1Kk;M#qrE!~Lzzf8pWG^jK5h zR_4`4VBgo;d;-4Pi|yWyKGtac_X>U)>-xoUW;3oRAA6+t+!gU=h9OZ>OcrA~M%(W*CrZ;q9F zK(tC@@hjdhf@YOhr#49LDKq^{3b&Fjo1z^{Lk&4zuhC}}l#H+U;p;(mBdT9#xBT)K z08ITZ?UQ)B+Q+F}`Rh2ntxsqEHP9Q|-sjkr_PvdrH(*CT7yBi2D|4|SZRC82e1tc$ z7o@Wt_0@f5%Jh?pDN0hIg*Wh1%?2#Oi5}RMuS1P6D^RWp zYa5IIDDn9P=LJVRWB0wnDc7X$I4^jXlRk@d(N&XPIWm~zq|YS1ZK6|7e{8Lt^edDO zZ8G_vJwKS_q?eN}m>s?td7!{*{yooO{EXgp7Wz}3TpsqPB5P*2cV)HMZMv9$G#>h| zHJ^F2{hBSDTW+b$w~Bu@T=SCqX`nTl)52fzccSs49Sx#Eto_;>6P$)o)@JNFwRZp3 z(aYtNCfaWFLH(@AQ0VizG7B1&7OthvICxqgX-9VV360@>uE*xDB|m6A_>?+Y;Z@go z9v}a*C%Id8qAz<=yVsR@cCFv8@ktBMWA3cmE1h7PvGY01yV@YWyq$Jn7xWH#tCp7h zv;CT59pFVMf4G0PhjkQs*i@FhC6w$%OfYfr#e-@lG<|qa zNN2j6J;_^C{^L#`_k}*UWBNOB&Br|#gDbRM?bN#mIYNDXOEA;-%lXc_rarbfX-7xe z>!5U6my-TcH1AmQ((ao{dyX_4M)rB;`$-FtCK$DitU<=M;nOy@X2Ov>ky(Jz-ZIHx z_+;vZQ>KB_hF&;jhM%G@>eIKKK5eES_WF?-4w*h7V=%8&zuhO~5vNbjn0E4g2JHk# z)TjHSX^cU%Kh@E+QRGXr`*R;@c7LQ_P6?|&hCd(G`gvalwNChQf0Dvo;Fdk^wtd$< zz5PiF&vp7!==5h9eGm=`Z|^ewfp=prtA4vb$f!<#zSG+#@;G?-P14$HFWb2GE8&X_ z*xMc(4`=kYCoMdYcF6yG$MDIabZi^6@4l7uKu7jo#-4L_*mBx=Z7KB`n-1wJ_vwcEA}l~SAADijhlxL565dpRo)zH7q+x7NEhmT_O{)PZtj~Pew2NyKiK=@ zo6Pys%cgIc4E%h*zbxW?_cHvj`MO%&<;T_<9R+2t8Wen*HUZ;i(K35|(s*z41Py?r4INN%@6Cx zfyh+mnDlefC!n)hEgeE9`nlP_WF74uv7~x;Av{*K?9;HpIfe~Pr90E?p-uL*&hjU= z&Pq7zjm+2Fnb^&wgg0NXI&mGCs(Pv#sV6@E+2;qyyg`c!6^4&ytY2l-9b{#_cq|Y5WC4H!vImUD6t+ zaf<4@PXbrr!D2#N4wSDAThILgY4RJGL*I`iSdll)+{y@l&&j(78skHDkS?u5bj8sn z#`fJk8QL*QZOsl}N*{Oe?@jmp$77}q{PzHD>Ug^xy>H-g?~v5|gKO)@ z|Ex9%K4sPTJ)EUjpRe^lI~)Kn4>2a{t90Ezr@x)hvDdhpWeWS7-8t}VS6VHGSIMVc zlv~JH?<=#6?kS+Mxih^0*sJ|2^9IUPxb-KrhqWPE9(Tjc>}D<;q^?8gnvRj@gz)FS zoEaTuoLzFIBxC^J+zo+9)_~t<9>pGf4P|-byt{H--WNKdnc|T;^Akee06ONPn$;_+ z1CPsRwRj?P_NR!uE3u#BPNvnugN`cjA3W$>KOod`5AiDlEl2Z*`a95}%f?SzUbU+o{D^={!lw_NGKC{L7p-!l z|B43N`Le?|^ZzvduLLHVS6=2-6Js950ge3;WUV&OL*v&m{(I-zeR$2#m+bI$^zp_| zdzv*jym_O7Ue-*Vg^@g0!dc1Q^!7{5UUA$0o|*TvUP06FVOktKL|%hO{aEW>j>`Gj z;fa(hIsX_ut-fE(`;j|KLdSv6uUPj^JZaCdOGwZCQx9^AJebl9<7GxYx6;^6E5WAlCQ^$jenQr`sgWracCF3yah zM*#NYny_iJ*N4%;E#zOr`m6l)xz@~kaxKU9c`9{Hq0Y(FT}~S=XAPJXqD_YWn{pl9 z#>88om7LkMX#Wg+366IB3H}SDG&M^DAn`eWY z&6E}3eKxZ73&f!ZD{~*8gl)ibqlrnH`%R)T@J1(3 zPf$rTty;FR86|+5*xfPgIbeo^a3ZVYEg3 z*DcVHJ#$^qEaVCH@Zy*&SCJ-Ob}w^Pe9yP}_XPA=yp+=4pnlOZ*?;Rl+0KccjWYjb zgeQ}~6xiyUU4=3Vlv$J%&2 zNjgOfouVy|KLtK62bL=@X@}Y<{yerFYFi)DB&+F+SwZ|@rqkFMY4<$Ztv!Cp`~P?iUFfw&7rLo5 zFobbP4L386@L&rMLKkzP>%#v-KmHv5<-2Dhxc3QZx%7V`IQ9{7(a?7YMGM}1PiN}1 zPpxy9sYZTQ|4%OdYPwK>FHDuOl`=koeqlFl#`pK8?9}jXr;ZjACWSu~joU67=Yzh) zLuV48H~pbI*x4-}$bCf~?l`y4rFBE=bY4$58B2R|zr$G%c<(IgO6_YkAMJzRXwD?J z@T}%pOI@~HpB8Qa-onS9ktVt#SUu^aZ6-})_B&|dW}X|XTj0IZ%=a8`c9%yqt9na= z_E)kD@0}X{0srP$SzR}sj`v}zfie}dqaXzI(%sQV5?bc{XNtp9#ru>a7J*r zmo&*Eq7&{H0+Bnx)jegJ%UNL$b1IHG^=;AwhrfbznzQ3fKjgD7+05BV=6h=Rcg}bH zE16yMVj|znKEp|S4>lLvj^ulg|L-5)J1=XkjeGVoCu=n)v%;eJ3a8RG(dcpcqk9pqx4hsvyHn%#K(T#$IS2| z)85SR5ZZLnY5F$1w{Mx@A31sB$ZPnanlZo$8#U(b&zhtA(3#;A|INIG7dmcuq4aP! z?>DoC{(O}Ui<7*!b@gCl`b8cn_h0Zo+UsGx{r{c+xrTasJo4R;3T;S(Hl#xvGN28a z&<6ZBZ10QS!rmJ`jQ#?DQbukUoxr&QG5|iSu;+~R?SQPhKPtx$xtaYK?vfPG()d`# zr!HiON_R@w%l< z|EFvd?eosu=gwUH06M69#4lryUi7trvQsEqYseAqQHhq>wO%^G68pZJ-?d-C9Nrnd zPeb>O8F-&7nhbpR*!L{6d`+mv_jg~87#V{x3-e57EvyW$`msvHLs$ z^hvLh_upxE8tpz^87_G2lZ$TvXJ&yr*MmdXflJrI$6f;;YwIzz2AuBS0ROfb9PA4Y z4&lEf{!gZy6v|5l2h+g8KG-Yw1J~>`)F;3}ohiXb53;CjpAd_4cg;oT%-K!toi;uW z&f&A6lCdy2*U#E~g#Kz>;&5EE#_AKsP;`1Aq1L8nz!~X$I-PW{N$(R%;e6{*a75*5 zKbG*FchP$tuI8J^YVH_`j*>F#j-po@jt|@=Sq*rj3mvd&)jHbr$#ZQ4ip{*d5W$EM&%^ha%KqyFC!mu~VM zh0xP2CY0U4wZQWl;CeOioe7+0Fh0{6pT0)Uu-jwTCm%K)ZtmCMe$^E{CAv zi_2e<-XQY3byX5JdKP_pQ$pFXgTp1G!&juDKlY+0#dih!%QMl( zwV1i@8WhZZx3b%d4B?|p$@O{o0xwFiVfhXEx@M5od@$c?(KqE=LA@vD_UBIDy4CVC z+zZdK*2!}AHagJt+3@ZcItpx=$Nsh}t&@VW>{1VNuA}?*bic7*yp`|3qK$o;cO9F* z^Lyc&9X=-(zK(5Q+G*hHChej^8@@kq;Clt|l|5uH4uL!95EkxBtC?QmnR0A?)#Asd zENfg{l@$2&PR~QD?c>tzqW{Cw_HKHqBI$DI~+;G6e}bAu}g z{U4tbobO)z=|0LYz((?}zKcIK=_Tg`WiuJY$=5S%I38bz9Dr^Tyfr$!e;X&KvQLOz zQOA1D#9WzT!wfy3@Lw={*n!zz%0Ju>ISV}1x4#1e8!uCO@$wDQY`nB(3*lumX&0s2 za4T`(_6vumsr(m!;X&5K$_s8|9ZLx}u^00toNX%#YJVsXUk1Ynm(mvH{r&~#j1#@m zJQTkCOFVhpYUyb-HbyUYl=-?fH#;>ihsTFHhbM$Ov3Es(x@k&qa?5P=u2cA40<62Y zTP>Y$>s}1zMqFYr2^>{BBo`k~wpQho1rAg$ugOfoUe&Z=faZSx6U?QR*6U+~lC^IB zPn#A6X}j6Ka|GTKK_U*fQrGoV=qKwx7@WZ;^J)q4)cs-me69tbOM6-8qbt96mp8_d6m0hJ&_ZAEE%`dD4)src)BMHe=9aJYjdI@H z)RU+8{9fN4rf;e@=KI^Ey3nbnb1qeMqx`Jx%aYnI-dS64f$;^C|I#?{SiWHD_sMgX z=9FsASeL@tZTPUv)7|694DECGa9X&Pe9g&H`mb{C9QvGhSMe_T%R5TRf0=%(t#5ho zG2S@D&*`&q{T}|6euL{?zn>;ge)2(lorCx~H{rV^T04&Z7Nsp-7CiHfWlbZng|hc_ zJ^Rwclzeww(i_XZ((n1F>UWCMP4jK~rM<@#=U4Q-z0Vh`*wSouD__CLCPeLAZCcierr@u%ua);>cgiM#l-${9^ z3pz7>{xa>hrXtHK1w&J9ng-$`J(R`&b5qnXPILXu+B2a zg}`F;=S{{>_9xLEjpJ~hCF30iH>&~}*j$)P57!6t-V+`jdWH8ygH%@jvjLx1)|>YM zl-(@3z^a;*1kRs-C_SZ+Cqs#n1O8 z>D>M9uKf?!b*%9$#z}24a|GW>cmDh|qa)N}^KNCYM&~#>^aP#B)}E?t)B)$m zo*jH0-(A_mTF*LDtoNls_`UX7AE(|@em>vh%i#6v#Orh8c)gWkJ-m5zXkT?zW%jYD zJ&ST(e!tGh`lCGAbYjqsa=y5JD)J~g!4dWzkIhOgIxxAreQ;KG5%s@UXl>bFXw~nJ z%(kL4q+=Kvfql?Lu8dGk`;3nYI!pbFzVJ2AyYM@K%g%#sfVO8kXL|n3cL0|Rq}Wuq zvYzB$Yh-dyb|?d;wL65q3ElvI7{Ie+Z{fkmtOdyqCXY2Cesr=*>=5#$_nd3eP;{_T+Io&s`ZGu= zbf^#MMCtxj9cLN}sUw07<@hniIQ|^NDay2VD&wOK=(FEwN7%@CtmtRWd6hM^xx^a2 zn>GI_c7=aI1O83hze!L2@ECiY*dCwXKbTvE5`tpFCas zSbDKyl%?|>&I&-sbU!0~WPVeeTFy~?XmxQSe%C%Bo|BLqI=YQmMyvV7;u`JdIfOp` zQ|uZmhTwBe`&SSnN3y^!_W0dcRsp9zkGZBzM`=qW&}Cq7zDtjTJ%USeSicbV)Xllp z&9X-pYOQ&8q~jTu{VKTM|MB$j>=oQqMY&VeTYE>B$*Mcp|Dxoa8*?YuP_`aECELSO z%%^C}E5OVi6#V6J5B3r;(Ndh(qgVQg_sXwSb`R)M7Ip=_{|IHH(4IUit3~H5$~kv` z2>O?$eE{mZk9_4FVjcTkn}}(@bZO|r7uK^G;a5GI$dzgrW4=@=y!U>k@Q9z2!rRzi z`DS}adBsjc=XT-0TX|#8H=mLh+JCj?l)3FsnwL+!5@>9tbeu8Y#Rlth3V27d z&kR9r^O=9)QEdD^{CUCR^*}SH?FC z4vg^HTHg&m!aIlHCA+4jNGEjZ^JMEp^xxg=V|$bBU1eEC^|P!K%lTGQ{%HHyGx&Pc z5U;B&&((jcuo*DE`tJXEwxxXPk1}qe%i1?M9lF?H{2|z9@aSw3uns!w?jeZIwu;t) zZwpC9XC+%)4qP-Vb|$1*Wx%;xy)q^byc4}`1jfGJ%^KVCln??=pEBt7(CMoQ>G?l^MfTM|6Y+K^YpU2D1cu8Br@ zcC>NaMJqd6Fc-gsz`O^g^Aw@;1hFA?!=Cgb-{NJ$yBXNwMN<}Ly7=@4-z~bjfA~$( z@pb*epZI=< z{;L^V@nGQqXVz?;^*sy@l*aRDaN&T#g;ZyphYKtDp77u}c&M0+%bB0fxNsLRS6=SG zgE_#22W^bG%Wpg8|9a&y|3~8Uz;Dfenevz)KfF|OZw7c4=_xupdS|38XiKMU>NL+y zo#y#H@K^R(&GV)4^SmP~Je_gXdN$+#gM4@Ve{;sk;~(v>^IF_3v$);%bS8E|(VQPM z9>NE$%YLjs&4I>ySAI%SbT(^lnumk3iB9)yqGjH=*L1fv2MuMp?%KPhW4vW=|Li4~ zewB0@^Y1ez?PKoQMT>xS9`jU<9b^7<{J27&$I!kW%45uz$LD!#uB-Ai=9X(ic$xp( z{qFhNV8QlfO7 zN%pWH{JzlRne8+az48$B<2>50^P@4FSlv@v1Mmap;TPza2f-b!Yt5^CEk61=dsE%n zGax^h7JBe7Yq$;i-GIJX(Kobz7wHSwvNWF&>M}M{@Lp#vb)KOWTKb>PWh>ukjh|2Y zb-Oo@?^3r*Lvx)!Gwz+|a3iqJb65jRb13<)^SXIbdCcL!_&je8#ouFkqxyP=(H)`J ztf7mU&!DF}UiYP|J8C|4-p9~OZ!QmYT1QsgT;35sm)kq$ay)aXJ+>o8m(O;V&}YrX zVp7QxlEWm=iq{`OT2NStWh5 zH+w97*lX#_o=XaQFPycVmxk|i7j)UK$cbIxU9QhX?3}IeEhbyG={^Q@$8vngUi~pK zJdhJ5Ur$G;hMbWm$Ceo)`>e62B zTI$n2q{TTo)p;EK%j`mSth4WpQ4_FiO-M{mZ;TR`Z-$Xx$b_lP(3(U1uWjZ}Q*f|Eq zd0ym0_9oX$7u9**G<0I=M%^i=eX|4T5!yRbUK6@R8orW}Q4+j-`Eshv8J*-r_|;M? z;d|}(M8ESS^GMmQnn&h4K`HCD8Mv#@NXNdM`J2K#PG&wQF|U`gclJY@c{Y1zrA9x# zYDpUMb?t%=%y=wBZY)>)D0JeQ1?fg7);>(Tyl8m5biXWf&*~G{vNR^2^Igf#EAb~e zI<0qyyfoI+XYoxu1`R*@BDQnld?1fuJ5M&Y^9ZrCTJh=e?B+{5+szL(VmBw=02rEn4}XF#U5nj3JElh)yLr#!GbCFZojBWhw9;+oR@x~YcMJ5WzSS3gYK;s3 z18|M|48hU;XL!C^F98<~JC-bY2mce+t>hl9!G!URP~A0>-9jUIH`OV8w!>dcbzR4p2(Ixuc5SgOnLHSGcHE5n*(1!^@yn6P zg^vfp(U{Jy_e84%{}FX5PrCeMQ!nxJl!Dt6&Hps#2PRE*hLDbz-n~_RA6ejYHh7%_ ze&>ScoX?ndIzC~E(5XaZDp!}(d5uR;`Qqc|1PNJ$JjQrr5TV@pVHs z*`1C4niD#jz;7GRGfDA7pJZhfAMLL_OWDVuDTO}dX8b{_p({=3Lp{|_##DU?GTvGj z1;AYTk>vzD*cmD}SmiREFL_qE(^Rf1!G0gwq5b%3rSKzk_4N-!vsC9XmC2Ewnc7kX z4LM-W|7SS|fw$Y;7&rIqN2asagFg)|`O5sP9NE4P|H^XrE=2La{XC0DexVe8u}3NR z@;<5Ts~QK*{fT)$Va)%VvHvl1Fpasm3fgj|tveGH51DftEzPtgki}jvdgxYU^VpvJ zAm~lm{EX$-`P1I2u(-D+gBW=8lb7q9P&Lmj(9X?m!) zo!hk!IPc!qrhU1zo|c)vSb3F;&+kEg8}z3eWlI-6oSk%`)%SDyy}3BFZ~Vf_>{0y( z&UzL4*@TXl4t;uM`kA7QOJ(a@8Av%^%39sKX|!Ftd5EpJNzR&~_A6Cs8#kAvZ)}D} z4lM3=L_U0LHl1%jw`rLD8GDN_Rb_5`wj^ug@nI?WT;l{f{> zTe}}=+0?_{v#F>3`7x{RGtL;8??M~<756yauehsAA7A-C`-v1Nq^MDEex3Yd*xs5zKqI># z%lX1>f#AE6sU*{Vnv!fc28cHfjnsL8I&jI*@LNZPpR9I$2rGrNxmNgb{)<l;}NerwW)}jYLCvNoSq|gfi;uxG0ynj-U``6xludm1W z`_|q&smE&j=2<*1@g)yTCZ@sozgU+|?_-^-I4ecBW%s?#Z=EN*Y0%<6Fz!u_K!4j# z5A9z?`jC(P6MozMDNgW-s~-&FZ&g2{o4rLguFaX5st#v5#S6U zenfL#r7hT==4hN&1h9XB7u{(4_;a|o68&u!eGK`CyGK87pr5gJPxtkhbr0?CM!PG0 z<8N=~y&LHN=Q)*==u&CF`0!8XVxF-3wxZ`B#pVfxnM>1p?~}B!|&jKS76!r z-7W3^nJ?Uoe3QrAp-c2F2FIjR9A+LgAGM^9@VlMgrThvDn2-L}3ysW&%C-G^#{Pvo zbIx)7kNBB!fGXMP4-?4K!_R4X-j zV^}{WtfSGar(*crpK8V9)R!ju!ajUU8RwLmpP9hDNZncjJjeP{L|>XFTG2u10m|om z&s#0%h85^2+xJp#Vkr8?f~4h5V>vTKJ%zy3C;2A1e7vV~f;I|&4wSgOKd=WGoxYfS z`g_pJmkf~48EW-Y=MLTX)11#^@3WVpG6#ln-rU^b#{Fw0p=bkjh-Xe3Zkg{`Es4YG znKP3ozB`1on_gbOX7VWCZZjrtjF%T`N*wm*C!j^hKpp%W+tkhpw#qk7z!vV=k$0&a z{c-uXXr~uXVk~ssEAx@c(B^vjeuc+}b_zz{)P~6(>}|n}JeB5L>cQR+Y>@FoTW8I{ z4-Gt(@7+DUUE9Q~wcMxD`$wu~~b)t2NuP*uk`N$1TZ(*0r&o0t=&}Q#kV*Nbp#GxF@ zO$(0);ORTz5{FT=})ApFkCBEp^^%K>_8rgCd zb+H~4yX^|`RG$6p*UQfG8UGss#Eu6R^IseV%*YRVhq)X;n+NK>CF8J#RK7Qc|7Ynx zxU!!18QDQ)2fK1b9(hCCF4|d+4Q_qSFk}C;iPvG|B-w596hFfA>B;x}k$6AMdTJ;k zh6K;Cw&bcUl3P?ixTLlu>f5ZJN8-N?KFuW0B5%9*UCm>y;)LWW#)Ice6ZgGe@Vx_x z;|$H)hp*FX;LYftorP?H9d=2mZVO{9J5dnY9qdwlRRo;g4Glj6O*v^DHfkNpw>!sK zgKbH2blc;!aeDkZc(Bg|11Hqy8$8&* zfr&?O-Zgp@`)xVr!>-~Nfo=v#6_e&F_Uilb6Fo3;L@2@7BSa7hJxD^R$~S<1^cYjua`#w1xAL{4AS2=DO%Jybpd9 z#`e!Yw4x2Obe3wu+WNb>b1(6dwe@2wFWi-!#PfJyUDqBOLM%z-YR>UePcO3;?gx)G zhR@MH>GE5!SxdLS%<~tn1uj{d{ZHWN@go>hvrcpeLWlj}K42Hn2ZK`{T=;W0a92L& zUB*xNe6wIiJ_g1-372jJrnxx=o){e0w;10=V01>M2d3Vf@}?|gJwu!p;6Q2Xy;_D1lwP<9s2A8Mog-(t8+JN~4#&6vo4 zVl?Bj3OaC&Yzx~H7=sL_PaN)Mg1bRa_E20z;f7M-kK|sZ8n=$6dj@z`hTuc*6?7Z)O(K7{7^K7 zROh(aznDV|>nWPEL)cBlo6BEl8%q27?w1{v`em1mPem7UR1ba1QVM;{ApNK88CAN6 zA_3W=3o=GmWQ{~*jv%sn60*7cW&yzd%k6#eof5Z9QEz@ty{)+j&efWS( zxfNYU{%+%ZNpB9E-+g(~b+ayLflef~wVhY$t9Y;Ir%%2&4j=QRE&sRwI{Zy^zFYI8 z@15t(OXT&t-Yfj|@ApS4-`roAxNCnCx>At(I=`FByJ_s>{l3sz`awf}IE#3n#3eNT zrlp+E-GJUO16eRT&FVXCTHW&UY1|FOy_c`brkscjtvE59y(rHQ-6mg~tMy#KbN^wp zbe^Mpke)A?wF*pe#X6P!w>HsJ!4?_Hr9sL{W|K94I;|;_gkX7HqxBFtTW}=k(Upc{R=9?Bm@uJo z(lY9AXm!u03N`@PBf!*N?IB*{wQVu)Ht?=+!@7@qD6kvt-pV}`DHgI`hO?1%DZTAz zj>aR~xrOq_dRfs;;FJFUjQ{4{lkTv{ahAp2%W&Vjj&dXLCv*ERJARt;0_{oXc~4FV?@( z6xP2|#$=F5Gn^umraJ{DO>@pPX)ZRgaje@?*6&!>@x`p?i_jIuU~l(_yFKPh{YHJf zsyp{%C8B@3@zE7S@CbIx*dB}QD`uU0XX>y25por0x>ze;SgBpYZ_^LOi?xE*hL^G5 z{taFFMRc_%`OV{Z1wX|EOy#$dJ`tm4{j<=bdr9@)+Nx09Im8Go%_F8C@c{qK+1Tgq z4%OXHs(c$c1D}AQ&I80T^z|GV^mQ9pn_vxletD>_&F7yZzq&T=5bcj%yKY{n?uA+G z+xz}=+|ctgrYyFsMr}aK8Sx_mgnnk;sG`-5j{w1!QW%bvu^)cW5hkG>i&&-oBDGI zy1ND%zYaV`HlNg#TwFJb_6L27vm?kukwHHxQrinDo7AORcVo>#wp!LQmG}$(l3yW$ts$0rxq{pY&EU{B=5W5EP8E@v!n%U?sAt z6aR^4d}W|D++{f|!QMBbtKC?VXg4Hwu`go%M8JXiS+tLF$xBLVsX~X=y8J$QP0W8R zznSsLOX}8AMGTNa^!^BXnAWV;Z>&s{#-BEop-W4T*ZS?8SA`BOnN(#9iOKKQL(INY zl$A~`T9I>#ypqqzy9L@sUc=zAcI-RjvDr7%Htz3e$w$|YFy^eQp-N3$*$gWh^Ih)3 zyY0Tq;G^PTYAni{+NM0lzNBQGl_z`Ea;(R-1cbA{ru0XOQeeY3E_ila!dP)CQZ^bqb z{_h%vPBh5XDem*i>{6LQlxd7B1D-YxLvHoTe55k;XLW#cAhCLSU2NasG9) zxcsTqF_yFHUYXZbhBd2kuSF(WoljY4glQA_d= zVfLZv-R&sv)mHuJh~h{c_^^k482pw0p6F(TIW8Y+8Jk3*QtBoS?s~;-s#QvV%9YaI zLZy5o@?BE;SW7BA?fP3+XIZE=8eNSuDIV+&!Jr>ncgKI@mb*nT=z`YRVGnk*V50=f zboP6&8wDF88262NuyVl$3&x$~9;{3-a7H>}qX+wmV8|s}WB>GER|?h(7&!b_4>m=x zWMJUy`yOnfV2R8xrO=&HrQlvr=@9Y@NWrmcr8(p)O(9=tFY>WJM3Tu@ znn*rrb5&Bfc7@J^bPXRZ^VjW#KJC5MUpEN8J45>CFyDzm@Q9WXzUaQ^OYD|WzGw?; zfdjJ9ma)F*(TDt;74z4%u$METIB?`%%6>MPvQzm@<2T(GJ%_qhu`g51o=h`#n!Q{7 zbvq4m<-&$uQ`R;-(0gq|uggg*FKkTd%kyNOd#(MVf_=Kl{<<#?`|3WQ7>a)0 z%NPATnQvCXM;~DA-;Eypx^?0QeDm|Stmx+jlzE4A80oNV_Tqi)#rx_8(XP+N=Gyzl zo@Vc9^wl-*_0{#KEqlJXmMvl!sG~dJ{b9AV;7^(Z@AUPsfLG{v%9TmsHe5QzN}05l^PfRcfqTysFe+z zajx*}fFA%izCL$zK3H^?IgvaT+kgLqVC*}_U<2syM!}e~7;Fr%7fm_o-n`2h(wV@X zGxc@A76Fs&-Z0mNRq=nde!yJ48+#_}a2qh`6g9*kR(+yxyMf(f%3a~YSf2^V7QZs( zJQ!x;;ntvvYY321kF-q=ud zHF@PGFV{H~pU0ZmM&8vXFUQ#%pT|1cO`c?Ot?Nj99&0Co@sb=KHT)>csrT}wJ4v4U zDXDC3W-M4^11K-q`z4Qmz3-J54{I;S8XQBp3n^Cv%`@*#rQg6V5G>%qpkIrC4FM*) z@oIeASih^uJKN-OzhitJYkC`bXObtLSnKh`7f#g{*8OhEW~=OMuPo_rPgORN@d)6H zY`%$aUq5#bQTjz>dNH=x%j_cd4vV^sU&7wN68Tm3p`yu4%>c$=EUaR^c>vm9b`|6R3O_)<=ao z+f>;%9UF?)j%C(ZDEj3?j2Z1cu9Q9=Rm!*=QOejIQp!9WP&%0Xf2GX#7fQj2Jxalw z&y<2&%}T+)h*IX|UrMiK?39+l=an*t^-4#CqW@INI{3TNL80j1l(K$4QaURX{a?}) zY$oz;*PhC^-Xpcuy+-%?X ziJSN><992+%lWNg43bIhL8I+bUy|$B(bL-SB4btl;EV^G;A2I;(1XmUGp-AJkd&z= z4oJ(I1lu~T_R>D|!|H**GV#ve!XLS!Yskb1c#$!$Ca;;kSNl_1w&DY~!k^JnWzzJP z64EBdeb5_ocP{p)zomVlTFy(1>%;xD|F{)c##=_d1~wfzdOI-;TA#P<<|>~p|BRcM zXX(oK@eLOr@XzsJ`fki0Wu^zK5-blM3f@fjU<(8@JUP?3%7fje`jFvT(e33ggsx}q zr6?zDWvnJk=jVt0DXMl%;k~@xp?&UtTdwm%{)^Y@+*gCygTvR2wXgA4EDF*|xulXS zE>c;>C&ldJrI<4mYTqc;X{EH(nD;ZC5#E14_M!R!zM-_^Vzr(1Fvj#N6dkRU_J)*# zXWv%}o)1upY|=(tm=p6ZMo;Ii+o@yF(@W6PN28~64^aN=HwXt~vDA&+3B7Umy3(CS z=JFq&&G%gDliupeld&=xj(k=XQ*ozPu7Gl~;TU-`R#&21=1s56>6DQjN&WdC4Sz;{ zkMN6NH%PMXoYcZN+JpNQ4P^a?GP?$=)?p;-SE3|4d-NG37iMW3>p_N?^j<{~~pHC*{GH)#OPg zmkeAGp9fBDBkw_zm*ez{&&zXp{}Pjz?PSI0feQ)n)kP*R)9HIso+}H?2ln(%*N;i^ z#u&<1oATVfTxYF+7awBu7k&vupXCR2ei+IcM#3!y&-u3(!o^s!qb8vdG*^~MyFy)n* zG{d>tr0LFdQoromf&VN{lkV^lE!P=K(Q@74Bl}-G9e?&R_D7-P+)X$MpUm}t=l3$d zC;0t>-$v&A0@6WzyPExqD(GgOuXJAtsZFdF8yl3N!_d(@U+;ZYiU~8ysykBEg}!7u z*^G_waX9bG&qeep&4UdEb{HDM{eOnXV2^{22Av*W{@LWqzwl^*OJ5T`_*uXuzaGvP zJj3yM@P5G6k0ZmZ=tk^tvZp_Z{q;6}SMlp7Tde3Zb9|h#qOT>`!PTZC`fv5#w^cq5 z;Kc#&e~JEs*I(%W49?U*JCB3A!siwbb|>^x-xc3K6xQ4!W0*9{*>BQJ=L@C7;QvaI z2i_+QK(F5;HSf}oZ;PFz{zRDnjy=d2on&3J*L{ztcm9()wf2z#-a}ujE5&x~+81Kw z@fl+sEbz*!4JxO0$ft;PkbMZ9VSBbUOm+u!4({;`hQprE@1H0y+Ox&Px=V9x#@~H6 zVW0dG-$zcUK7+meOly6RxE(3x4%!^20Dc1w*x3DdjGiHVAjP?zbt(LKfjY%+D|t_L z_f*OXz6QAHy~Vj{;jM-J{7%{_nfFD$tGxPD=9Te#in6~q^Jdk#^(dYtG9Pv6JU{$( zU3_fnAaPCKpw1%Sz;GewBCVh5+ZoQ8)M4bw7hbzisSm!OJ=oS9lL z{XE_CpS}^8=)d+rB!9SjnZ%^`&a_EBXlNDvPIan~vj;FH6M?z=n>v5!-o5xeV8Zw9 zjFn55$pa5(Qhtc?!2MU_^H{Ts$U9qktb>;Tozx_bJg`m>$gSQ)VXKw2dJ{bwFSlYK-x=up}GkEWaxv$;vbD4uy>DJRaatcq!3d-6*(&nc`F^c zD+Bo}ll{#s_BXqld)#z?lFt9?Zdk?Y(|P3hSbZx?vQCoc2ma8Fv!BMde~%sEaemPu z>|Ze^BY3un>mgdN{FjkAWdA8A)qnYK`?zPU4q2eEjTi^r@!uw1yyW;i`lvGEF-De0 zK953!AJZ5ZxjDxPgX>!B>wp=V1{}QCgRKFEU9L{|zyz_Szs&xI=<6aczlMDA2+`Dj zvPpN+RiDInj`yo=53kwdCaF9T@1`qYY?q&0e_*QcpUY3Joi;Zc(fJjEGqQpJ9| z5QHDP|Cit zV)#q1kbSfnzw*enoMqmLZ}}&&vpYT9vpWSniFwt_oZT4+4s<@dBb}iModKLMXLp1T z0chYTzh(Ai&3Dm5O2L_6(Xen+h86C&uh((L=Zm6nQMeiWl<%ryl`R})rL3Ooj>o(7 zOMMK9520tQ`Lzpw3%K~u7M)RNUT^aJkp2iJ4R^7KFUMlx zJtmmZFMu(YV`%$>z(iXmySe;{JaBg=c}sw4Jr%O4`0#u{zW~ym-L5Uf>-hUEUvE3yzipUW&Z=-*XsG; zOZPcMyL4}0U1PDmiM1!5r`QL@&})rjZN62v34eakmfg^$PwBtTO6pAHx6#Ve83*K1 z6aS++U=358po4=IpHFvU=85+hhz$&`Y0pWvH0dot?l-*jQuHqP<1PbNzRIb* z-qZW4hZtUiu>nf%=RM(nsdrc48Sp2I-!9fgdEd~3wa~vuc)pF_75uX4U!--#gF*a^ zD*VCbRrEQ?w{*U$6j@p}J?#~%E$83w-c?fx-x80WfX;)=zpit<$+NhVE{`~qi^*60 zYR|Wglk7jo;FHVwhHDecavlMX#P8Pw(|AbMTk64{S2^i>^m~a1a|DyF$C|p!gZ)M@ z3y`}X79hTneqLby#G&YJmYw@8Ly9$!o?ns9zpvwMy;G9o@3e{iq2L_J8x9V zT+A|QigOw1T4>gG?3gv_7WYJ^wv>@x_oZU4PF^yQ^Ic7}^D21n8~)U{rsKn34sV{W z_#>n`YtX^>Tsws9K8=h`7jG}dF-eH4tZ9Io5qjJZvQD{ykQcmqYkZP^p!cTf+ zh5xqweqxFq<*b8f*&bxlSWHpwMVKO)?Et(qiSZ~}cye6P=>a#cXwD(G{$bQFy-{tU z?x_0J%001#r{I^`JB)miY#)YKww7fV)z7xvxUF;HrJOHpCN>Cud+=iJMc4XCWbX`{ zT;y(W1s-e*A%XO~x@}7v*2QDn)`Ca9MYt!T3g$J%C?@i@_ zcUQ#cf!o{2+X~F(AL;G<<85FX^B^)oq3l_Vul7vFQQpMk^Wu^;1=0pK`#a+C>5To6 z30Bn1gBO$J$O!GD#A1@*p9St~{AA~V_xT>~ z-HA_|b&`;5&PtvS&bxFZ-I3mC@QyLoSSNb_Rqrb55${@|baKdk%KN{Zw2(U5s(kN$ z-MK8$f5k*nyoD?E9`a3kg7t1DxSAPo@8J=@^r62WzSsBg2l#h}po81HZW*4zSs&^j z7CeppiDcp%UKpyge0}?>jD;`fW6CR}z{vUiRN?%3=(py5v z^OBW5plnd*offD`5MN@$o6K?ppj)eBr0ym^Y=Q+vK`oHUn53;g!~@~s%p z-3UI4zJCbJ<+(X#eG2xT%Axm4FMZA{_h-QhfI*jR54K(JRt4;<#9yIFL8Z{J?daXw z)2dbqy~*ahi+F7f?}?XgRog34f#o>AHn3Eezw*6h-habi`PQS}|JTHurNf-VnGl~j zKe^PpqkFhNF=eFll+hNAhivE4bssnF<$a~J?O~IJ0IfKREKqIYH2io; zH~8w&6#P==|E)b%Lj?V|5xrLHtA=z6cLFx~)81Nv&t4(_b*7qFWu17g^i=6N_~0-< zh29>`9eh*zk$6vseu>^o8I?QTD|a*H)R&YLYkhrReiJSGII#)ACv&b@`$#bz((uF& z-8@|NZgA);h5g|0y4IE;1Qk0=tUiz z#d!7mR!S?rqenfS3jNw)6%Gq${mkX37t>zDPrK7r_-w!BE<6NWvVeP@xx*jnd%$D5 z;`#89&bnfMVB)hwX{*tXpqm4DP8K|-FG8!w0PAXaP?qyByt`drWS(b|*9J^Dc3*rR zxUh)4BgzBc?uyR?cUP0QPkG>8MSLDOwT--HV4ZcegDTH_$8@wOXp`}K^3FI%W(Dm7 z@Y3zykHuN+YU@79c>Qb&crM$6@MZ_TR~|22I{JcM@Q^fT1!Jl+NIKW7Z`ChxZL42d z6T5qa7ZQJa1h_36lK+TO)v=Rzg!}I*ofxv;@&5mr^tadgMrv49?+yE{Mj^zmO&m(FEA z4IS;cV=%?Qt6LBVk|FbOs6omlNsYl*xD|`j_byXkzOp@pS~&IXWg^Vhz)h| zv5xx{u&++ReiQA86(iY8bS52|5^#EY`v=$XO|7pj-SI^yw&2KR?t1(&aINhvX_jKs z?(2^q#UXT`<7It1Xv+j>3;p4YTU4|=!hLs5_|{1dXvJoAd=s%^wk97b;=lI0ld)wz z3q80uD;U17&DUq`{KUyC0=@V3MCRGcI*e@YZvT1Vec8`Z=IvEU;k}!C*w6F4w>T-h zA~CeT?RBeekMI6v6RB@Sm9?+(-20auDLAdDdq4Ig&b=?Y`LI>@^K&1_J~ZUsW$ewY zE1EN*?l3g_I-hl3ULd$HsS+Gd8F#cv=eyfR?Q4E)MK=B%*QfS*I6D~PPO3hSE<4a_ z)m;q@`}`(P{`bbG9vhu}7yjVc^-_V07s@`^n>F2#uRV$mKe%J8QFlKDA2Dd-0NVU7 zU^$GZ=*(Q6MT33=etrdKBE7@A5-TUoWu3+FrHc58D{?sF&K~LXd~07hsqSkSjcye| zw}P)=qan7=iMGdL`c;IuDEMgBF(>R(^*-s&hUOI4me-${j$8M;ef{Jf&aNQiLXYZn zu1b2qV%lxyT=zDha}~z;$|>gF%=s+#r^>7o(xID(19xC&5Btch)7&_4^N_c|A<=)y zuGf2G`6}NPuC=lj#q(ZaY~1ne@Hu#0aa*o2XZ zNp|bxxbcKfYu(7!tFephOtuf8yB;CFu+d9rb3eT|UcaDDmnWt>bK1uW`>7W*@5k+9 z#d`N}Z!9pKX*wo6W~_EXw}gM|!9R`FA#nR#o@Kif9?6b62KaE^t24AQ%lVV{PKT|M za?r7vlp9RB-Ssn0LFQ;fl0)7=e)|hg-M)i7(t&Y@7f{n9(f&nt{ZfrxuA=U z|1@|05zC`QcF1gJnP?F060XSiE{*=LB9`Sf%){u2V#A;ZJQ3Sh%XZGvSPL&mMPpy2 zKOaB`wi9}k%iN)Oe`(pgVDEk-1+k?Kg5o1nq^ssxIxO&Lz zORX)bVfcla-@bgyYLDOEr|8iC&$*RNP5t?eXLC7DJKQ86garoSQ)tStDX{Q+O5$?Owyld~Yh%wY%1@#&1rJgkB8O~|UxjWw(&J&D9`+R3A zUVpk%-ag+Mj>8yf4^?+3o*&mId`4}>CmrGozGY-Wc&z$&S6u&6I`z-RvyMK_Bj3nr zmK!Tl_+9SBifp2;3UoTrfKuW{>Mn-=w11x(9Cc;>tK<7uuKOp_o$ti;?`HZZyW-D; z3#?=1-)LYN&MeZs_-bmMx$`KSoZHV!XgmAIb+Nhph%pr|bj&4Xy}2x=Y$PCk%)Ns_ z~<>iqlxpJkUW2sIW{|_;ySA$!k z8#)KNzuJ{a8#Z$elsj-2O|6`iN!@&#{-imAtp+CiQaeNo>%DI$nm!pHs&(cZD7=PvT{y}n2{&OtYp47-kbQvW|8Uv>SBbeieAiAl4=J>!|<+zuT7 za&yKrMn_|Htnli1n=+y?Z;+0Uufv@2%yF&*E}Y&(dWbfuT^kK7)A_wgxnIDfSO`49JTnZkH1s^U3FD?Q<#vmt`*x*YzT=uT**DpN|av&U7^XDz7m=Q02F$pwkpGcl!PXZu(x9d=7Z$anb{Aemj*t z9mx*crQ^W6s!0{E?wF_BTuPk>z#Zu~uc6<_hA3Sk2GgE~;@JJw z(|Z;LH8wi0r28d!zlpnIWJ7O7COlqaU9pBJ=@Xhs-VZ{fhdG@d0k*D#M{p>ktkb6%?hu-i6Fjr0ze{}V*MS_{Q z9l#P@Iky5>Y@KxHS^P3)54n*3x^gk)!Bxq-b14(!ua;N-B>u`eTtvC+$lGYvQ>Jr> zwel4{%vxAYx#^TcUS&rvuKYkLV?BYir?CU+u8cHjZ#pzM16rI3O~#jO9(TXu7Zk+~bObw4{GPgE zfXlXij4_U*Zysz# zb#|p`m=$fg)~frEwzmvJW@G)UOb_3a*_<=2ga4Q_@;V#8Shi?$t|hCb#-zk#BfVk% z+@1IMGv8XlS?CBf^B#Yed4`8gxu+sIrCr_O~U&JOq+?BX8MDFfl}JISSfA3TPgE<4 z_d1@z;K{ykTOXdS!m8?_k%9_fu<3 zF1EA1{F_W2K3m`32tSftu3Ra!qEKn0dyb{qj6+6I67;~tC&OlAp5?Dxb??X~+SbG! z8U`kR<*NHfnrHl#lh||Yj~|iM#lLn4>!+ERWv#?4Gj@ssUtK(pfoB*VlLU|H7RNWb z&`$Y_E5@Q*{_JoPd?N+EQ9BBq1K!a@8?`nnsl$C{eenD^-%JY0$H_{)LeKr+k>Kc& zeD*2Suescd_9D2;nuxsQ@s#o`bME!=_;wk?Wy#}vuAQhib9r6VeANf#uthJxU=YfCX9Xm{3w)2ztJl4%N@+615 zzI4Xs>+DDs!x%>!*e4Hca4dV}IH}o#hk#k7Qd@ zKIZ(zMt+a+`z620{LbW;&F^>?vMP2(?XNH6*%=Xvjwh`hBK;+_?`qCq=sci3OXr$; z&yihxK%dgv`vb4#KCe7@)~m#Yt|hKa9x+<3=K0e>#5$g8MeD%TR~Y}MGAsIDjCo60 zD7u*#Qj%H3lUDGoGj!dog_C~3og<5hZJ&=1(sSVIia^hnt?ZFL4<4=v^lB;Nc^&kW z{ehMM&o3~aD*}C5YS=$r1^%rF_*zPMHvBB$GdyfN{zUf#`k3c(^W2;Bnx*>!=DC-7 zt|9*OEcnFr@VV+hPhjW;im4CJ+l~$RGi0b;V|{jWiQnEkB4B@x&B4S{n9lhe`XoA# zCcK&~{)PXh2kR}^M8UYn!Gk3W#$KLy%)dQYSHZ>#hMecYe1eq-mg4-wgB@Q4Y!t9* zoX_=OhXuoaT7PXQy3>R07YwwhnD9YyE6Wq1`4gf2q)ii;o35l|*_T&Z z5{h1@bQC;RDY($og_JdIlUDLGyq3Oirfzell=wmF_a^>F27@!iN69xZ#l{Fa8&r<_ zwaVbh>h}xgfA4*bh8L$hYrX$v=6{M~s~+$qt7#r_W06(h(du)JS5LnF)87^P9}Mj? zd;Rovx%dB0zOBBBNB`E01?%686YF0oeOsxNzW!P%eScCZW3fyrWAnID#_Ca}tb;p9 zJ>P!AV{YZY`35wqtt9aq;~D9E)#f&0v$PHKVy0tvLDm-CkFq{=x5i*>e8QtDeGl2r zl?%LbtRIy*%PVuc%Amu!GJ#iywGiMu(P>_pIV#gG6L@7y~Lu9%Bc*{He-EzB9PR+*dGnlxwZ%Om)x;Ka>vR*pX25kxnpHjSTe?re#je2`y*?tta@nVs;Y-a zO6KT?%z^9{wV(N}y|U_&kxo_mM)@q*B^evdd-$%e+?sw|db4DbCdnko1P75xOnw?V z_ucGGrTFR&WcZ|O?y$CH8@XWLweFmpP9J3hn<0OSl!Wk+2VHn3aOt4B%ThApCj8Yj zhkbz?e^|z}C%o+G#sK5n5l=FDS1jJT;t2rvWP(=U4f!3gF|>IPFjrRvUl`w+!2T*& zEXE{by$G0OZ&!Ap9OJ$km}GC4&jVu)wgEG;H*5C_@?3d>Jn(!sd0T;L9bXooH_?>= zUNL!@&iMGe39byV!Q^E)CGmNzsWHe0&zZb*XJmXH>ux4_&j54%F=zw0u?X1Hf|>Kd zz`&E$z@9R7AQQ*eF^c|>_o(uq4U*m3`!j<6kZ0ucYeUh~;`4^lAI7^{?@bLw)8q3J zsfWCU%41&q{al~<^O!@?qagnBLEmk^Q@r^D+#^;WctA3Y8z*8IGE7Z?eX4+&(=W6x zzzMCX8^DWQiN%s(c8tCwhx{~W#6vMXJJ*?C=6u)J`((6Y-C$8mYd<`4fkGB1(Z;kTx+gHVNo-eqs zg!FuPQ*APPp2+aqyYfsw<#$cI%=OZzBig6sSu{?3GnetkR%Pz8&+8xBmrtBK?ailm zf|rReQCIJHcoIBByov91>jZasRJM~452vlJ9+~5ONq=JB9bR$E&LU!xOIJ2Lin%`& zmw$M{FLyTidz<_Qc-X)1WS@|+`a0i^ru-i=e-oLYDg6nNA=?EK09T6J?N*C%upUx00WDx}klUx#2Z)Tsl@y z8PT^b_$SykV~dar>)I+3?Y4s6MUjd=Ma>m`i}nmlIp?#%spm8oq@9C)^@lcet9tIT zeLn2g@s9e4-*-KB&bgkA^i5#WX=R`L6Fl3s&(S`7-4oCuBp(R>OIRbKr~gZN*FPZJ z8AzVi`BC1L9aOTFY-0m>e#pFs{6Kj(HjVP!{jYmXUas?BUY#oA)``uFynvOlBv#i< z>ZRf#2$2zIX8C)Z`Hn8rn7>zQy(Bc9nU%7&MtHfLqk{QlFzv}aLgUQOy_L# zJ$&xGy>z(i65wx`RHf<<1?ug`jlY)r#Ppf6Mmy^wl&$@%c1*ub=TSv zXrkgF#NvQzy~NYR@?!Rgh%2ab%ZmGX6kFzB60G&F@LSIBHhxq14dHj>7UB#*vpxz~ z>svqWVXuY1-pI4wot|Rd?1M)Ao;aW%pqIW#?9Yctoh6~V@uX{r`^nw%1K;qc3{3JR z4YYkn*gmm_J-0km_lXz(^AmJ4*-BdQebVzPXpribzD(LzvU%IqSeG1q+pXFE0 z?~73xMSHe#|IJI>k%Ru!vZlKoVYtwFb4 z1y3JEEK;3|eL7#ye&Zut1CJj?EYd1;+tu*OQT~LMe4Zy%_$JRvyDE9sZF!UbxM0ol zv;7HgeTvSv+3Ma>D18^$AmWsMhR(Nw_@mY6w%r%>0Om*UO>WVdnnA=S)pIR+?V9<| zyRk^oGjtz@bX^;nsfjgdBS$DEsWXWATF`TO)1aXJ2IWos(Lq-9ZE#IzwF}uhE8%`% z>5rq>Piy4f6zPz9uHw15pwuoR{wTh5EkWf`PXC)r+<2tr@SAd*Kp5@;vXIX-Y|HKt(dDP<6%I8@bEgG*f*1l{3rVQ!-ouZ7| ze<$nc50xe!N8cmJmZy$0nu$!9j4VQ&QSj&v>~rGVJ9tlaepgS`U6T0Hq@Z68LzeZ* zysk3Hc&_c*E0Yn59%e1*eD!r+IkmyW5;eLiKkyO8Jh`qd||{z2Zm7351+x`i|rhg9)I zBcE`-sxi@y#UfQqyvXEkc64@6Tl6*pT?r1cO#IC4-zDXpYNdR)ROwjqmD1psW-_XotVydN67hd7*H1c-k_9L_I@!TAL-IQOEj zA9{$`oh$rxN4K(X@H%lZ|81T4Z18FJEbirL8GO2Zw3YakZN9p5xNmJwfcFyl5liz6 zVrsVL^X_1OT{7*u+KbcKpSW2^C;RGJX0fiOQs48|iSc~D`9;q9^+I-di4+`b9+Y7# z9%tJFsylFGPf3>jMM<{Z@}jS9?`Aj7CXU{9&CvelvDgkKU_;mxs%x4^3Z69$A#WJ) z5&Sl@w=l|A_lZAc*7iFe$_{o>oK4^4$Y5hvSZ7(M8N0&AW$yY2GN;m^DrHxwNeDOG z@524S>&u9jgfq{FHE7zEE9t=E`y!cyFPO-lz2TqOww#tCy`OsT%`@+1IbFQ> zD%;=7az5+yo@^MB&(((;Dc5?7J2r7SuzXWK)%m0Mt&sOE@!4lP^?xTKg( z&7bN$UFD|g-Bjlp@7*(0AMdWW7(4RrJ7phJoZZxOh&+9_yXu+j*7Jy0Pw)78$eZ^y z^(a>9Zm*uB3shf+-X#4qGPt-dMoXHXxy0rkkR>O@lv^U6$2m^omImd6qdhg^0b&P@ zHk2f=uO^#RrZWM$(h5CMd!`=Nh_6Rzc?zdL_A=#mH4p;lH$oe~CetF5?#}0+wkp`Y0!|r6xP5Rte1?Ako zKz|+#CVjk2vHEo8-TzSO(d&?NRDJ|_yhv?B_WZlgTCcS}pLvu#^CWQL)RhLF<=o=I zA2o39Z^V}*hxhUSdM~T3ZG+@E>}BT8So~Dhmm){Xo|QU3XZhcSThx6Mb;SJXy7R8y z(f@qpvtUKeN#}~nOj*wOz_(tLe*-#A>^vj3n|0Qh)7yRP2Gb_>C%SXAE!*Kt^$8rm z#-v$JsYw;X=RfhObH5ooS0vw&Cgw7qI0qH%^%U~|RA-6~Ge=R+Ztw%XCz`vmn|r3H z;69g5jRigfSOw>a79zh3_l}nJl0Tdq@212uXCU_Qy~w7ZefX0k`{)|aA8u>6lg<&{ zi{HY%c4GJk-Z`S_oFj^CO16X0*dS+!?jwI~Q!u<|Q+NA*p5I;^49i|M9pBAA`|ewI zDdTmdAg5@0mCg~}w@hb<ci&x+el=_vq zz9P@~;r$AGMQonCajqry+?D6)oDI@G(W~e*75SXSCY{dtpleP#AC%+V|3%07phEUL z)n@b^VlRCC`Jhdox?}L+eBZ;B_)E0*!ndN+86nMMp?5|oem*J3IUsK?XQveri~mFu zF^mrE>~0^Ko#UQunQKKQ8_8BH9zW9?(@lKO9n);*CB~=om_B`qF})Ai--#Wl^_K08 zDgLUd-->6n4I0Y%*UCwQz>WVDztoQyeyMLK;TQdj$FI5I*X1$%D#$I;IkU+gepS*} zE19uHr!~Bcu{gsUhugeyH~`+s#`*J8jK%a*jKz431vF7(@gej?_*KGK7`(~A=Qi$n zF3&Dc2S0OMAC>R%e$2=5b?^Nc?~n4n@LPL1Ga0YJlvQ8N{;c54yXxBoJQzrWhgt91 zoS(oqQz$i?mFfbe4+L_e!yevB#3kU6r46d@w_n%S z*FAa2JYN^r?=_}x*^X_}4CjyE zV#|%m5CjB3t}rshx|j@cbOiT)VW-KVA0KtG*4OcSjQ_vncL(*02FZr>OYlju!6U%T z8bz*}FT|{mOQ}!~0wKk1Y3|Df?=AcvO;wJ&|93UvzO--uHbc+|61(sfAc!-M_>6tYRx# zv6$bx{HS-9FDZO4w&UWqHuFt|>etm8noFHo>|GuOr@*bDFY$Yn-@oR?@QO1q^iO!z2F%5)Eazhnb`+TK z>V3v;1HZ_EjmvY4&6C(wl*zMl+Hq^17dzv94}MDA`py~QRUU4wqHeWMxb-2=!mYoU zG{xCOD%^7O6<_6>aVrPhnrCn;$9cWI-fU+$_+)rHcyxX{Zn<$%8^9ak)~DcI47VzY zXV8IL`QVnya+m!Q7q^akxb-vdY{2jM)-lEhT>6i2Yov!;ZohJzCwTw=1#aoPx!@Le z@fqC8=f98fZ3}cEMotg*?0e#K-V5LJ-uR&R!56(RKJ+Q}iLD7HCaaOJJ-_Xo z`t|1b@SDe0pG}>k60Mtyv0D@CXJ`h$OX#2IS0*}%*7rpoUd;q9Tziyp6OS0>!G8i= z_@)>NS+q}eo<%A>=L60e!9(BsIdxsm?^qyX;P)yP?EEph%SV=5|9J-Ya!ve%jx$5K zEy!R~J}~%^?wlPfb4uEmAMVfDP0_yS?QT3V;pbr5CHy?cq$y5{NfkqGGGjG~F}sYh z`ypdE5x@OQiA%~JdZ3GmUF7Onxh*yL+!t2(Cr9%Wi{z_zbZ&xu93I4(>pvVO=4`(N zYhryMu=Xf1IF8Om_n6g6zW1Nr&7wFg+P^;T`Si5HvkrN5L_F&yWWRV?^3rhPToL=A z4PV_wPU!YUIicGXrz(M;VpxqK-GU9E?JR4~R?-!DA#-+RMM`K+vHoAfnH5rlW1QWI z5chc{{9R{u*wbF8GdoW*X881PFyHNS)9~t~@0?$${obYLh1+gJ4@xq5JA!G&(OUGE z9Ll?Vkr)81G4aZwz(l7cWA*l6gMk@eSk_On2OFqz+$R7cn*ES+r9;@$R2s+~7-9XJ zI{@>E+idpv@c-f6g9YpX5@V%NduJC9TcWs2srabsd&j9$b{oamlw5bf$anD&DQm3dn}S{kG_KKlb)f)~T%M2lXAq4zjY<_wM3q zw;r3k5cNa?3E{=ifBf0%niBB~ojArm_)c%T^#OQ5zV8o*@e>sfuom7q(pp$G^5BQ5 zK>cc8jMm@o(Uc?LlS|8qzrlQ~ z?V_dEn0)Sch|6yP7erI1nLO?s@#thP=w=ac`M}l(Dla_Dex`S>VyRV!J~aXx@I3Tp zovo;n4Vd$PS`W;RIsZqSMQ0U%MCS%0j9pWHlJUa`YJPMcH5P+pYBFb0@y)Ir8}h>d*o(bz;v7(d3&J)6S)67Azh-{|+Mk}G6)SUUQOg0RNokKmW| zorB=G`r1|F0DKT@^ebzB-I?Kq_`LZHPyOopT|nF(?8ZYouiM?!*M)hAt=}^~cWoW= zbBWOpjl1-%`2QMXowtgIb^fn1f4x<6hkZyXc>EXhKi%0$`ZsI_XE82DPOXxh+6y_g zH!@ZqWUao)Tq($2smQ6s{bJuSh!1HLUUL*)qgal|;5CYEtg|egW!BR!aAj8cYfVRH zT?oCr9eMRiek=I4=zUCPeVlO>&3Fm8Xnss)eb9ryAb6E!ry;Y7PO8pAx9>(~J=kX1 z8<1Bc$()BEos7&H31kh71gz*U76jX6*Sn}sH2-O6dWYxk=bp)F8AIlg!Z1$gf$>(f^OPcaM*% zy88dm%mgwCcXGdI5@JgRq!a-W#WI;e0Z~vut+yr$Eg-cNQ4tUc1T{$e$OsloYyq(~ zbI_`fK}ppIkf+~b?FGcDt?gqVSSLiQlv@s$C*SvHpEDsK+J1k}_xJr{UhB+0`|Qiw zYp=c5+H0@v`F&nPx+nLp@c(SIBb`IOeUg`wyl@z4n~nURYu7WclCteV@RIcDJeNG~ z-m=y<_YSE%(zM<^?R}HWxH7j+$NOK(jD7q`=9X|i+M|IeP;E%ok(6PzzMol78|9yp zRWppO>lkTnOrvc3A+LVv#O?VQTk)IEc&B=jy!t81Ps+5il_gjU8>_30T)POlcKR2} zwFStvwQJ=2ocz8quw=yE?Z~xLT)FmI=+EU5`SvdGzh^?Yp%1hd`AZ0V=h^ERE0-^W z!@2IBp5o)^-GA|4GVNc0H-f9Sfa0_EVovvFPG>NuGnvy_%xPlyEyzLlbmz2W$d31c zBQmx*Jpf&X8Y76qac+WcMNAr6N zX|7CHVApzSzf?Z0>$LwzUfL$5fg{mH%u9P-X1MO zSDnlG&fK4Yd?$J9A@9HT;#604^1aaya_vRl{|NqBpQFF?`WgC@&Uz*7P~Sy!=H4~U zf74#(zaZ_pPmpn#Z(xR92^3xFJL$S77hKO6T*p{k%a}}NY$l=8PDH1@2A$SDkFHpw z0rqdUzzcTZn|De+lHC~JEBJtHE1B5jgxl%VA)XrU0d6+&OS5~SHfES0P@gMeh=|`aKvHc4DOVS4SLWm`x?qNeKqH0 z*}w378pe^{&KI5??Oe`0*-PCW`NAMGcM9$2plk62$=O2;9$X%99?!WZPl5e&FOR<0 z{=z8U_2Qi?SLCzC$Uok-Q+F}naHn&}Z=eOu*A(+E*Vg)xkaUGAY>R&8@C9F_n`SVFSFTAQv!0s6)g$%>*w5JD&SCNYL+}Ny1(M@G0AG7J zt8C_R&-1|#v2MU^avpznkJ^+;EOl4bIE0L)*p#2A!~U^y;s$<;x!sWlewOw}t9b|1k<7`Mf z5BTbZ=ptQQ9x$9etz3S`*!TSlzgN<&Aukn#8}0$t72!k0JSNt_kRASle%OM0*QHrQ z*Q&qZPxJK0(2(eNGifew$hTK{X)h`tdYk6!BVO7sl!jg*-te%O7FQZP_%M8gGTb{f z1(BPk3nB}b0q^#B!&1IWKr4sgC!3HD#3$S{Ao=z}-X;0QA3YoYy~+IWvt<4a9^aVl zwv4c(L49?+4!tw3+0BbM5M!vw%{UU53r94l^XV>r(_=OWRvP55Wkqw+Fd73jN2Ys>p2i)&3=6l(9w|2AY#i#%3`P&R3UGoFo zafSbd(au2TtzzUFouiX$Ys)utxzK)``IgiXbhaY7|L*6#D+fsDaOI7Y3Bb)J)%49 zjVAoaldqwhR6l6shPCjVnlI!zned!dYuHmc5PV?m8t(q;z+zgkXX7`O17bdOnyWp2Bia>TQvdZF z^qvh}gLjqx8f6|t&o~6WHE$dEFMjkgFd}{wP31gW2IrtMIS-Y^dA4kL6Mmc1a-r)y z=sO=eFMu}{a(+KG_Iu8=)p4F}8Ryw<=RDgu&a<7*dA4roBJv}WJ|aIN=_5MNcG&YH z(wR-sPEu}hhP17&;+`?$n@#*BZM~7T>=J%?{7%5XcJtc|Pt(0!oNa5kmS@hk8JvQW0+ZyENLd|PB? zdpxbs9^?Jrfb4Swe*BW*YXx@1OP7sHW2M-wmy#ygN`C0CpN*%z{k2%R$J5eioA7Dw z8lw&0FzpXJBLr#V#e%f?B0=WZg@VvsAyB+c--_p*TyQgEeiLJVBQ$UWwD3)M+x76a z>!6Wqp_R#;nd2U^XsY3JFAc_~35_M--@;W-wHY2m-x`X_FKjvTba{O2G4=tu6Gi=K z;ylTI<|Hv;Tefpf;{#->8sJ90so)$%DbHhkT{yRhti>4(-B;FrE{}Tl1kw%vXrir~ zm$`haD}9jeRg8a+Ih*%TBrX}F3EVyB0vo`N14#VL|xt!O_3rY59F3k^p zGPqzsOaH-X~qWK<$1P zzVT!%ZnopN{XKr~l;L4@q;+;!zf3x{tIPlRwYS z?wG&7>!-Zfyvw(HbbJS`o=3Y~`8?D1v!5)T?`!aD`0`Hc0r+w$=h()%@?aahIq`+O zIS1an`V?;-){!?qgj|@scPrEWFGK&b4Q1LlG6s7#7~Z@y@|Qv2GS?0*P0ECs_Dse` z^579~KXeDS^`s90v_xFnfw4XC=5yhx$vLR|RFco~KTJL=Mr`s~^-QMhql|-W7uWDC zd1QhhJo_qOH}UCzaoDx67NT>`2GD^pHJ-> z_JDVvJUr90rEGt2g#5=^7U3gD>>DH3l%~h`mvV346w6Wnv)D(>!X_hK?*#S9M_bQ7 z_v|)mgXZHCC==g3a?_M_V@KyF?m6^;<;V zKFHvw^sbJ3g1L5QZ+@+%P2Yre3@vup6WoFvY_;zR=Gy(d{42vk{J?cS{083WoT zxqL(ZTQ?%lNv8R`*9P^e5<1-r4n+5ffYp$+6)Il9KKkeS*JPXhzp=zEXMGrA==tU$ z(6fBEGVDi}xO*A06>9!Iew!;B{F!=W!}0uURwRA5DzSf`ZT<=$??X$%kILTW;ckn8 znfA{NlJ`S&&{GKc+ebXZKz{2Ium4rR6StW?_Z7hy@z_gQC*(ULeXp)}Y$WgT zN!XP54lxQ#TpJ4Ub|yN|(amPxBEMfnYRc?@&VL#kXrX-(-^eb%mHm&7bX=uNOrTwd|ST*~mDV?kd5Af^U*_!Y%J{zN#ZWw>{(x=8>v-D@< zd!vs{v=-0G03HOU0#p5qXY~Yr9eC$eBcGjCzT}z6(w~q2K{;`h@ynCEEjlaebpJoX z^jE%pX?NfM=xyCn#;fiqbw|+oZuh0V*}qGd<)#gkH;VG7c3M0lLj70tvR=3qpAnU1 zAq({DlD2&2l(d2%-;BG=+C0-&pxAW>*2}LX&~tVg^(+Q&B~`E+gvEVSn$~8Mh84fS(3xFPwc?2q`&LGX46t z>U+sc|1Jx6ST(683!@25`{^l3kK%TTX}^V5rw3y47%8uW0!`lde0*TBTYWshYq zc8(_YOH^;V;n%tL2KsyN8SA}lUWywaUm)&=X>t94a_s@$H!Jxj!5kUMJH-$9F@0u! z3=zLAH~1^CANKHAV_?4hAW-^{(R)l{2&BB5H0S(#-=r;ZF}!8 zjlqy?gXV$i?n8N+!ymz?1QUHC!*}?tVKs|hO-FB?IYn!kKR8Y{m|!of_n7}`8!DQt zLzc+KALk|JJpMTN7!Qi=f#=8{XES9MgP-A)CmliM{(8E{1j9r`W z*#9=MpV`Bm?PS#C>)QtEP<_G2q*Ghi0xP7y&V3<2=xDPQcIzPO;V*X*` zLD%`MiDiLcI&$3XJ)E02G|had(O#o?<3$;<*6(xAA?2&CZP*^xrC6KZK{hz8UG!%u z^&Ej-8sVd-y`#+w)mQxF;HhtEp648n8Cw4IG^wE^qn`hX;I>vQ6+QAN{ z^(wi4r}K1@VO{=l`*7%c*s!molOdC79PdFVn?B!~$o!ZYBnC!BpkU?>?v{NK{p<;T z3;0b(Pb))D`*+&x?&oFO&wyjeW^a-vd94x};Lh`Y+_T7@djay1McnF5wS$p?q{Gya zzbKfI)YVpW&|{ysmJVsRk>9O4kgK0Db#Xr*Q0JVX!vP{ zy@7n3jotKp@>yCFU+o^dqr&CuPx4M{b}{zl6SIP$1+q=gOba~dwLm)8G1@D6B*{0=`!d)$8~2?@*aijD1&=R zTBiR~ov8qwsrHo4gzO@Eb$KW{6Z=$6QJ04{FgEJ*nZA?KleCwV1xYA_f74)>oD@OVk!~v1pB-jQ%O3a_=RapyX>6TdO2$^YrP}8 z%*bkMc66*YdB|A&Liq(p`>x>}$Dj>k+&Q1L=XV;2AL|%vi0Sh!;~I|5!j1~Bk)6B_ z{fBYe89Fc^Rz|Gfelrx;HhpFnWD3a|DS6@f)43N-`KRNP721^=)4Hwy&CqSXg*$4c zznL;-4~X?2G&nYM+m_Zvtn1SU)vRd?q_^<>8rI>KqxgELjvzP&rz@<+uf>8^*s**g z-^t)U-$mzIvzNiI+ICxtD?d^5#Ld?~_6zVH#MWwP%omEy_-b8k+4{WL;F24MFTy`} zG4gDhktZ@+O7z@GvPFK&Hp%PV^qkY8yY8APTlRF-kvYSP7v18E6Gw^hS9|)$R#Q(g zV{M&xQ7mvv+iLBN6r~VLm~Ru?v1{qwe_j+jadX>h$-haQ?m!+g@9r8NJIuS}H)~m+ zIVU<<>vR}ch7P}{IybhJ{{8BE%*Ex~E^QoKb!nx1rhYpt=ChVe^yBA^JlXq+J#DKK z)S47g$4szVPtGAPY4G+b+B!8MC<`@tV_#*UZ_TD-^$Z zIWe2N5VM&W%~vQ!^VqLjL(_=U9Qvbm&B$KXpzQDP>L4UOJ7 zr7=#$j8n7Mo-tpuhBjBgD;dus?n2%%z-yc63;SbPku@Y2p0EJkS$N*MiPO;oiv|%R z`F!?P;nzjJo(Ft-hJPt;;rjkOS68$a`3hUgc~*bs1u9w<-zA7$YkNQ1ck2xcx57x{wA>pAMPJWd$a3#>mFU?>)KMon9uYTE*A}A&zTHwdbG%w+mgHZ z@ihzJ=QrxiweAvt4#3CqD>#oi#ut2eKXnmf;7!Fl3JzL^J?->w_OLhdSH5`>`ZfGy z$yZ~QxA|kK(9Co{aiieZ$m$0qj8~;MazlhM+g>2Ok# z4w2c%^hPe$+QnTGiC$)W+i6Ag@kQ~y7bBY+I#Dd|Zi;($icac!oT8I4Jnv`Rr!(GZ zjQ4)*G#`V{F1?u7jC(n>--Y?Ph;iRT+^a6k&qa*;Vm&XefM2Awl<3)4(Yl;{rtRFZ zIg{~TRAL>N!E-}7TXgqO!h6{D>u&y)~=iujz3;ePJC1Gg0fhQZ>E>A z7VIgArExc`d`E&MzF3HM6eDw|mxmi3ZfhHoPQNOA@E!E#bo$jozqMDk9=$z1P0txE z<$8t>>Ac1s`jFmB&$%t?PkNS~b6QGx&N# zJLyVGRQHMH-!OP{4Rh*fh3@MMZtQb8ccJz#kB1hf*E%Ec+toa4tTjh(1s~!e z#oEIe;g4;Ho+V4S-htf+K6EkZH*VD4TE6Y`+PZn4o99#V$PSczCx5gTymtxSiFa3u z=hl2Bw(O?1)n;xizBravV#UO}%U-BiQ^2^&M(_^#;UN=G^swsx#P65?$P2fUpc|Q4`|ISTXck=CxUY_kEv12iAdA=(6ZnpaL z)~x}{+aq_^r;lk1ZT&3B8W`-Bw>+(cxPrr2`^y=FlQ-@TR;`=Jx_uz%Tf4@rcicN` z)@Jx}Kk}swvxc^j_8;&Pjlm?=^u%t~WGk4SAZ{Y|;}x6WYvM`k;9WbZukI|o4IS0d z`!{(n*~vmrGB~83R_LJyUDTz|Q)_${YdkvEigze~E8jNo?QY6WK5t?Dr(E#l_Voqc zYi=gj{5sbBw4qVf^2`>W_&V#qyUsI@;E@@u<;Ir3*czTJV*_Or)6Y8MG`&H&A!Ml# z<0`wu`buPt?R~L1T#h_<8S>mHWZfvTZUkAk99j3*-<1rfu|LciInlz^roZ|2cmwn7 zD-A5PuP`vne(`YIQ0>{c_2a*0o)vcn+t`G#6JtD!%UD0)J4t*MA!i*8tX|Xhh~?~= z;+~sm@>@n{B5vKrNFVwd$XSm4W#irw=8i9K`A;aPs>B-7TG2T!Ix6<&$T4VHb7Y{} z6SbU+SXXkl>psQY_Brxtlbh$O$}>_l=)Th&NqFyy^bR|Gf;gM9L&)ZsY%g>&lyZ0G znz?tX+&_8W78+cfDs#7&o=y5P+O77b@+_V35K#XE`ewA-mZRTAr;B1M?rWL2Y|{lV z!M-Or8b0?`tNwNR6=@`v7ynD8vmjIXuI3IfzCBUhJ=Vxi|G&fkR&BAvYyP~!ty{_0#yzS!JI45FuVsZb zww%7o9~5FzSr7!0(`X6u7scZf5m=G|Ef!+V;!U{7awyCK7!yQt%UXCN>{&ww7ru=&-r#W|3~q=S835xX;Vp? zLs}w-SWhQ)F6Hvw%^BqliKlzSorj=@{@9y1&*LoMJW3PiQL1&GC*Sboqx64~Kc{6e z{wqrJX&t2h;z!>D2S#T*Uv*{J*RXapVxwtdtoPAxjp>J}SH)#l{u}eshAzWlYsglu z361B+9s_rGqiZEtGsWvxa*lP*odaTb^-hgd!Lxou`DXliZlYf4YIS@odrcPn1$nR` z1dWEC4aV3zc>apR8N|P~W>4}hTca}5mPLR1L*h`$Pszvg>*$%*Mbpo1dL(>u%6qro zAAJ3zw^Dr5emW&M^<2d+Od-#Q$#q`C)*vQ-IbJIvOM}zKfmof@1s3S zX-^vM`3mh>8vORTYDXIS_gB@XT0d>^(H2YfuPFKcH^8lRvJrmy0(EzkBcimUJA6%c zy=0vhbrw^X=|>mFJJmW#+jRbIMageR#MYfrMk!^SGSU*PX(Gk ze6pnGB5;yr-$7s0R_zbCbk;w1oHVU3lFg>kC*e!7jCis7`Z|5pSY6F}edfDYymvaE zbLKl@ZeLFhMVucC9*;Py1&>CY#{>^XoRxx~ zMx2;nOT>9Z@S}+HL&2tqvt00lh_g)a{fP6B;JXp$0m0o7r&{n25ofXBn-OP`;IAXj zLc#43XMy0Bh%-;{m54J}@a2eex8RErXO7@=5ofmG#)vaZ@ac#nd^T=~I5P#;Mw}Uf zwGn5U;OdC;ZNZfhXNuq>5$9IHO};5e!6}0>QR&Cr|KrxsxM!wA{%OJXr2z z2!2}b^b%|-cX|kZRPJ;WY$|uU2!2rRqzS%X?xYI7Tkdoc++FSj1SgDge1hXRZ+<&) z^eE?~V04soTyW$l#}T|_lyg*Y*eK_);NVftLBWBeoCAXWM>(Gg_8sMXB3L@gX%Q?M zWt8)QVCpF6uY!S5&ijIGQRh#B$D_`>f=8py z9|aFao!x?;Mx9-PEm7wWf*(bl-w8HFoi_zPh&pcwz8`gdE%b;44w*WxH5->7qkU}@Bu zDOePBW(XEUooRwOQRmx&8Bu47AacR2g2<5-f+NGuErQ5IHwhvm93>v(?`QD5{gm$e zEIhfKHDIKL&6T|}tvNx~9M%HaO`rc~cGIN2RBQivdEsM?=q;z&OxG93LfA{CkCb`# zQpJuVKQ^eUHLANzbz8wVrHdS!_6jx-!{oEazCe4O5$Y)i@7uqiUdw)rKCDBI46_$df(#`&SpRM&w}XjYPCveC|=Ue+ejirT#C&VjL)?(DQ{>%?*+f0(_N^vj(%a=Z4D*!MWb ze4Q`7cL(+PZjGQN2=mE_IBl6Uii%(rA-$)d_HexSTt z@xL|u|MWA5ek!lplSLh}4P*diEBt`F#T%K|DqFZO@)fkWd!AYL@#)CVsbME~-h6bz zTszHtn`x&4H*^1m(qgRtVeSRi^MCNHesto!c+@ezpU1rzdMEj?$SS;j4>o$WLuWWY z<$dMx)C=|gIq%yer0HAbk-tsM)R||0M7r`xKJ;6K{q%ezwD~5s7|n;wTQ7Iy^DR95 zg}g~RQ919KZ!_(8fEu$T?daXxq{)9J4xT#Nk;>uedVg}k+0UKn%-QqPeVN1fK6%%s znU}0133$q7jM0w1E0=+i(ww^6mQy1eQD_M(o6QkU&fVLJ7?}GjLH5j zUO19iaa-Y6-+f@z<{z}R4IO(!*=F{tdpY1&-xcyZ9?0mIjy=VEduxYpp&5%aL2BnS z^Ji>+%dm#r`0FiP=y`d@_(zf$jUYAmbFCBIcD9m{Ol`P-OajXEzxrivu;_e zNk!ZxqIJvatmkfK-O8e$@ndcEJ?V$~+!H$k_kXrn*co1!|NYH_ZnSHQ&9~i6UB%|x z9;UA1bhk{ck%PKpXY_Y7b*aBe{N4|KJv!Rik&f0Ar&jvf7oG7*p6O9p*n4JH)9$iB zVM`RqT7qr!a%0!5BR&GY5Dn1Sir`RajI%z+7{}x2*L9bNUAxW=@!GMY!<*AN7vYX? zL2U2Rchx?jeAeDE?rYgB`91 zycao1W#wnp7Z+RzBIZcOD)iT>E&W?2n&eSz#T z)s+Gt0n+}&LsmmW0e)J{eeDh3cdfhsYyPtzd@uh!AC4804|g!H%{%skXY($(9~|Ev z9&Qf|H|P#?vzNzQkZiIJSi?EF4=%vxi}L`-I7ijYISPYEpX-m5(U*OE;s#V|ukeh$ z;$DAlTP=Ti-5VN&j?8!7KJl_tcb}Me1H+VuchCQ;`^521r|%Q%ez~BxPrM_=8mjUV zeXj|zXF78Xdt!r;k#(0wLTBxewc$neiAT7j33)m35bGf@m$bE|;RD;4yNUs^*tIRM z2(sT?=C_6_e=~W53*)tG*_&Dne^os-`RT3FUGGw=h^W~&Ut41lyf$YA9apB{wwFi$9E>q)c3Km{GZ-oCt2>z zxh`M79NIpv^$UFcFE6c}G|`dtlx<$x2-4KAqk}D{0Q~aaz59KjVJJ!*<`jTV-aMw49bDq#3*n@G;&uE3|v9*6fGWj2AgZ}XkLS?14biNXt^2j<;n@!)@_ z4%=$hnN#wb@|Age``e*i?d|2TSJcLSP$O-gp5!OYE5?4vp3=6}?wK6lqLIf7ZPBAGpn# z`Hnv9yHWR!2=Wv4lBdwvP|szI*KtmGEA{H!<_~C3vTeWOc}pO>_0;`9 zlF_p4fXbyDjmLxJk=$L}#j0Po(!ZZ>#>o>$4{%+ZZ(V+f4gT-mhkD zJ9^s%vBp1hZs6PQ`qS*6-rC&9ieby&c?J8=)y= ziVU|iRNg(7=$jrtbQ3&)e&0;Fd!yP<&a(f`bK^~6XR;vgCK{Mw?_kWNLuh{eif1=I zFz2B9yRmzkm>c3B-=;4wZohVO6>UDq&pp#gKE_mM)Akh*%c`nsVrOf`#0>C=Of&?( zY1TjaSf_EXrPJ4nC!*TRzD@gKPh1iUGp5Ng zm`a;PpSMFdg4i%J=MYoMreSk z=O3Y(Sed3z_|q6%QT}?jeAWWu5<(w_t_8v22f(j@leXUjGIrtUl;#Zu9 z%^@G#LjihkA^Pt*=-KC@XQ##_cWwDPJ`VW*)*}n7U1H4+@GO}^GPQh(ljl44L3fg= z<%1I94E`0I#n&CPKj3Wsvz*a4XZ44<^2LkD1i!rLvXGHGHhOXg=aP-w@iY)wAu+-1 z7q3YA^!5A-zRA$lG31mZf9Meck47%JZqg}PLo&-Td^OBF#(M(qM7yGe$E2G}K9r2` z8uw9)FZja4kz0nD{NOU`m8CH^X=nH+9eP(|kl8W=ekRGIpFLcc}-S`n-m*D%KQWm*yRe3|NqC1C1BhAK|>i?$@v zEcr66kS|jWF;aN`m*tr>;w^Rbd(+*!NqMFWIVsWiS|iKs;6A&=bK!;>;#Mb~4mUu@ zLnO2KNLM`8ZLVLwky)lQ|CG0xe3Ds;@ORmUEL5L@ys{D5Cx!OnJKl`!B6)?iaPkTE z9N%RO%>87*`}m2kmN947{Qj=piifVXTyvw8w3WY{MOsEnAL?yHb}0ehetstxd+ao;VaL>5{ERYX`?KI< zl2wI=WAICz;nMr1-Wt$0)!n;p^u9Cv0sJVgzs2B$5@{Y9i8DXHVXPxOp5H?{3HY~yBi$FEyv34J zJb9$X%e#fVnyZz1hb&Ub*{kXD`#D1vVa_@Sk>?g+Gm_0pXBLIuHquUjpZw0y7;Dmw zPC#Uc8sHJu>Uqf7Tct;^*ZnSatqWY&s{TLD^Upit0Gw*gPN~wq<%)@w$B+S7Lz3UV zN&c7l_FZIBt>LrgxP5(ubYok}CdMFZ+sf;*C+Xg!R%qF^*<^8k&CR=%yyBJD(l*h+ zr{vMP+E@GpT9=%r{m=uKxHMt>*};AMjQ7nQ-mlX;$})6AUc~~W&Fy=l-=a>@-K4*_ z4ISKpb|QaAr)F&!&bZ58pyOO)?VUK8eOUQB82c*p=$`Y+MQ702=44EQT>Em$Un$?J z38$~=huYV4?6XE@aPP}7dI#-HuIuyVhlf07*0n3G7!Grp$T2&uywN8cr&%Y>^BX!x z`$J;lAiK>ma?VZ@yL7Ji{&#x+$cO|oN20VRF=$pyl>f#d`EhW+3%V`5INN^HvorOf z&Ba0YeEME!9v@Vd@YWltF^-I@PuNq;V6|;0; z)ujigSk9Ayz^=5$lCI^wG%>{{()Qo-CRsxV(eWN1C#To#B1kr(iQJ3|5o{FEQSM`{)0Bz_<~G{ArECGWg$R{fK#Q<2L4;6Kma2#p+f+_L8J zUVP;{(5lM5oB!e+!ut$8_pqFM(I?FJ=t#*sz}HYl8fQi|2kQFv4=Mj0UKvxpGVsrb z|DaED&hlHL)teYU{m;Pv23j0W+J)fLq|vSqt9d_)-#3)D;Z&MvUEjw` zXNHn~PKl?x#*fVM<}E&0>jJArycEc4)tuV^EvvjIXrrLkj$~hk@r+-|ijrq8HT0Q3 zi7^jqJv@c~CdPFiZTvQ4TZ#O>wNvY`q9C><Nn~L)Dg^(hmZ@UCHx93G9W<*bDa|<2PY1 zVQX!R`dZVinu!l@ue&qs>wB(!^udY0dH&~jUe0rRua3DZX(Uwx^)|-U(=)vqm~&B)RJI`+k;fWCG7V*rgBl zLB?9`U)0+@7wXCk$oRw*AIlk0#%1G>%}zSkU3!1QcR=S_H8%QQy!gyLKR2E4 zsYm7BW8mlWwN-a!{{!>)Vj|b_8&ed1bPV>9hKz7So}atq{rPtAWX0|}&P=XCMxW1b62IZ- zDktz)eKiL?Fay73Wc6{2@@f|?to%nMeRpXshko8b`@Ts#ucy7&(e7)J&nL&}hZ$4l ztK(P0d0&;2l2<-|Kl{JIKz2(2yV=;b%XWuKZZz@a&tI5XyB0rzG1&WC2B*fGhjEuL zYtdZq-tRo-nf5$Nv6r2ghVA(NyxKzZF3axCyAuN~_BaY#O4(P_T6SVuxS@`9M!peM z>~}~PDa6j7s=UaL%Ddh3DRpHP;>^QizK7k@a#DxsyFEQF!*-fw&mc|ZgwliQ;EVNj(Hl0ajy=%k4*5Kg|2VSX z-@LZ#{1doskPcKgB|O~N+a|K@~lnm|0!<=-;G(6S?hytB~#%jv#L1P_-6eg;G51<-ju`wGO@Vy;vY*GIS=$#?qxVqkzeMeoO<+kN1=c@lD< z#)!EZ^yQ={h>f9i`2*{n#E3vE{pX$w%KMNxDI3~H)HhA@6+5jT87}hu zbYuqP1Z+?5d3~v%4vqC{^t2Q3OUc2!Ms}O z7l+U-oQ}E$ZK|K|(qtF;!8PD_h|32x$ zmE=2Dw@{4ITw8M1+vZ)CeOt$OeO=w+ccgK?$(3nO(=DhITrZ>zWsBUq5SIP76${~e zp6J^pesqc_lS#Me%l#6QEJtzunv88Y!yd-kbPC@Y_P^6dwfT)ommV}OKZAxu8+uk= zgZtm6wc~ysxEJkf>|fzK(OQEb{d^gy{bBYo9PJalXwq}-7Yxj>pEEGqepZn0529NT zSb%Rc>=nSt!T`cdVK6r8;EJ zz69-gjG<)i$0<|xdDVB9m$s5L@y;5c*7pa2Tm0EA=pv}D_u05}s~Ij=!o=$qKwE6Z)SDWMv=X zhw^`Q#BvnlTY9H-wKM(S=&O+vxO2?Ri?5rqN5K9poa1aaH`+WSpjRymAW+%(C+gHzyJ?;~_q$St*2Az_C*1wI&IUJn#y*dHFMhiRJP$Md zR5~_6(#-rPeeXewxpPXt`kT_4PNhxdJAL0wY|wW7s;@tO6kFF0&R>`_F1jnG3v;%_t6<(d2r_hdhk-)4Sm_}#QhPt+{_HRb}Ve-{xH+RsJ-=mfO7gCs4n;K6MQ> zvo4n{%&u+TK+HIJr5R`4-NW74M=G-1`;@UwjG^6$gTc7-QJS%9_weZQJG4*dqZ*;h zSF6LrWed@~tq^ThD>eynoxHW_3etvCj++OX1&5-|A*8DxuR|a1+C&-f{ZYIdVBY20 zuXKC|UPY^4<(=%mr`x>TI8yKOE_p}K8_9L)>6EyGJZ<`@b*TquqDrCV(qxRMd^-ld zH18BgzZ-oNt!m8F=05aQI&ZNc?K@YHxmRc)@jnbC28MyeEBS)>AIW$hZQBFznRMv6 z3v}HT`tAmw>CQe`AeOveR_lk>2{XSu9rdj*2~=Ao11WB5B9PNB@lhbIbEM{esp_R8mhmjQY~Ii*eypW)ym><^ zFc?3wC?ll4WfKGEoph_-@B+$n`9rpS_q{Iv`z>kWf6D8>W3=;Y=po6w0^Qu$H~&tW z#^dWAt;l|-IgpHFy2g87=Dkn86P>Lz?}$fQ?8fFD4gG|B1jjqfQ{5rhk^db#ga4%% zn#r|)LL2Y(8T(v&e47uxATC&6@0%a;4RX!SNN!i=3ifQ|%INo!_fO~A5Ac2qamm~} zp|b78{1@&Q8JJ;z+d%2~Ydh;?t~w|dg7Dy z(->&}7M+)O_IdE#uI=&2KfJ{qi$XI7#N7v~Ud8bY64&-{Y3EQd4BtdIR2vni_m63! zacl|Xb!j}ye%13)${?+s#%GUqxKgCv+DVnKAlO!__?1ow%tA6H-F(9L&NuW zqpjL6%CR54<5WD`w|V~oG^{;vz2E8K?e_*|*!2b~p6wT}@ikYuH2fR#iiSO&_%G4$ z)6kOk>;GTV@G{YGwGSTeV@~>+n*ru1XH6D#!p`Q4$$vO0LyFIP^Rk_m6NT&>;HxN_ z%7$)Qkr)4njavU73$_j$8OWL$!PjdDY2N`a#*T1*y@`P~4O%J+=Cnl4#>*SKwa2R- z!gpVTUPYg}k2YzCxSjNT>T!Bt+mNp7@zEQw4OsY0xVq6P|1)?)L)yryiN9gpxWYFm z77T@I{olqmHCXhk?_F8cz4trI=+r->EXGng_0C8m=gmz%)^BL1(fCp2*q6}`t;hX= z#k8lh)->{s^wPfq6h6*&pES5P{Li`4rJYO2TlwGaowbJN9Cq&wx4wY?Y5myx|A6nQ zo_zRTK73DpfM0YzNiypZ=tSe}r!U#igZR}xp0%zAfZ}D9*a$Y8I~THT@hJIoZvw{< zhKgevT!?)LSw44RVQtx>S8KD7p?<_#rLwj~eV@NjWjB{mmz!6)!&UIl=>FN#C)mkR^yExQtkW z>DU8^Ib~k~jVVsl24emwX4P2cAvXNw$RfLfX@6OhMxRG=?x~r5k2GRRalfvu{z|S? z{=7j?tQkohp%J>HhJF=w>N&fKGRkR3$yCKGzz#IRo%SYFzV;HhI=+hgGD zI*%?M;#upr{O*k3H25AxzkGa~-Il!PTIIU;kF}6L;V(3}Ndq_1-%NbPz|s-U96$F0 z1mcp1o2tcc3+xK|C)j$EZ&fXIst?JSi_%Fu>gbC+-BkG(d*6M;cl!UKfzS}pl{K>L zMw5p8Z(xS~7X!6^U(H;Zz?>P++_{Q5bS3*V3c75&>LK2z&E(hG|(ybMe*$&Y!gYFQqndd*;agByxH{C z)BE=2!z1w( z-K6m{wuc=1TKc86<1W%%dt|nKwU;(a`LHz{hW`-Lv0ihm2I$^vmSc}6pLk6Z`sVce zU#ZP7dL;Mvnsnl?Vh8NVA1v)#jOo!Xdt_0*+IR@smtK8^>SG>_5o9iX4XAmkJdztv zE|?5XCxP3E;P@JFeKqto0YAO*jKNj->2Wt%w3Au$>!RV|ue1yZy&BEgeE8kY@#8al zgdF1FYu>bSN7yHWdk<*opFuqQ%mKQ4u(x-w{RPx18N3nPN*?|re!9ZX`J~IoVhK1b zV=g{9$-hf`>bj3cYm;Pm`SiH=5#rb2-Cfg-bjd5rnOBAayE{<+YVdS7?o1bacd*f6YxU-To?XEhsQvQYdY?Y*VV?#$ z>-or-p74iU+or#+E|h0~d?S9Oo{kyt@NLF~d>``e$C6{W`&k#!-KTX1_o<9Qkg-UE zzSFT+$S&Fm8=EW79{&RR9RmGsg+4V8SF`Tj!|x{V|43+dFSh1CBVXu!t$Zy4`7^OO zPkxRzxNXg`w3nvd2+-|$`$Vt4X)=|G`iNn4Eq5C#rv)RA7jDG81OS1JY5cLUWN|t z+9;1T1w#%#i=07?&7I-W)M~yJt%W^WyV#?(80n%l&3*sfqn#f>2YWNL?wq2xxrR1! z?8A(+=t{b?-Y0DxF8|NAKk?qX@3IWdee(>u>M9x)T|q~lt=FZi&)ItO>^Hz!lCCZj zT}3^*da9S9t4!!hdW`D3j(#NdKFwnnhq?CEq(2Kyed)aj(c)c)peda%JBGbhXDoF; ztazf~DIxFda&>-dM;=+5hwhN!($Gs8t;1GlSoQPxT}Rqberz-kDWHDQ(|bl&%CQSY z2fIp8Ty3hXJ5-M7(aAL+00&6uOIX}b*Dj?ws10V`M5w@7u`cJgS??)->}@_RIscy?}l_2_7o$tNh1evu~4cc*Y0#&A_jRtdmUFA;xUb z)AQI^_^hLzr}CutYrjD;zTCZ2-4T*$_vfsN=4yg|7a?nG13!{`v>%g9t0YZybw6#I z3=c58Dv<$AsoVnle&$S-(J467Kpx3yKL=_(E~TuhvF@CDowO>yE9cSQok<%PK0Qer z*UpYMu1ok{I^O5TJvYPLbFqYXXSVTL>TqSneETO}TdINLTXoepwk`uVA?P!N?Nl)- z*5k+C7A^kLxcuN%wzZvS-p_+=ZRh_a_OrnOwl(#$03EXc9djvt`tX}N6Z+*b0<>?U zxXPM`2bqV-ySL;AB>r^TJj8FL<2-E0MOI zHq7Vl!&mSDxYONh=toOC#Rmzu_^m9$AK)ndJSwM*@`9ae)|fMK*gnRUzMFsh z!dGfw2M^sV9uBHwVciDNy@)_ zy5K8d7S`43p6y(-R&YNM-${>-qD#p~qKEf?J9BRaZIFMk(bd>5%OUQad{#S~ zJUJ~JMBCcG_WnN!Eq;#P?#gOkZ~nm8;14tLFunsv`XU?Awt(7;&gU}_`w4k82hP;N zjEprsv)xAds`63J`#?jV?1`UApNqd{P5!cT_mZV`A5oipDw?zod4993hw_np1bdC} z+(aG4@D|}(ci;w3R_-qA%zhYqB?;E^GkJ;D>Lfq;C3EIRY!AU1q1pg)%9QCjF>A3e z=3C#1*c`flI>*l8PN8-1fV!lOAer~K ztHfU9jFaeBJnv)Lv$mHt+YilXzF$q*2S^*qn$gEgJBPH&?kVHroVQzBZq2?8-l4p* z8I;{~|D($|o421nw%|{2CA|GA){gP49TU>t&;K-DP_P2n_1_B$o(Fb&si0usppl{9 z^8AB_f`S2qmXvXSO)u6Db01ETuUCu3d|8$rTzjdercvPVN|b?&yMP~4o|Hs@waGciU zV~j_eKY!+aX#4O5Yx2bles+Iaf%w6p{#Jaf&$X3>2K1{f9i)3_yV~MQi<#3)IuV-$ zy+w8f=>%)BZ_4L=Z|@uDFX8=>KZN7*t!`z{bxa^@9D4W6xr1vA@A&ZkV1eX>ltsbX z9UgtNnC?uju>tBb^R9=hJJe9N+IrZZ)kSqYNgY%0X&lHnC7`>t#EN&<)J!{m{pmD1 zi#$uxPH$^E{!PiYieDMtvwKwhSb%*_^8ep#`=?*f_KtP7(ofwLRE2&ZetQYL6MfQ7 zpgRQ_1KA%>!%wfv;rZB^;gui4dx-V7O1whr!1;XZ>Rz%#>kKV6Q^s08KRR=A2>v9W zvZIXkN0iZajPp6Nb5@7rXRFIUlre^%yRwdaFEfl!n!Xi%eaiR3rD70iEGjz4Jkk-9 zY3}&s+vSW8HnI!FM}pvb&s-x@uZZO4!E3Q~L%*gxvwxCfSCRj3_$R3>;V4<2VsQK& zzVr9%Slj=JPlUsoF8ih8mne2a(ud`{lr8<{b_28Q=|IWt4bl9~laO(T@XO-ou&;I$ zyFgevJ!h-ZE^}i!AOoxjZX*VC`KcJt*~mdh19|-t-Q_11*`4>oYR;x)+t-*nvh5M9 zX(Rs+?`AB(=2sRdF!r#*wD&15!`|=tAPS$7Pc+V#GWHet2>cGdBiTVVv!fsQ<8|;? zpYC1dj;B5BPbZGyBg@`&;)BlaJKe8W{SfbJMkI9a9`?_~_2}3eWM4zS?Jmag|`=@+z9@=S!X zZN{tJ=fI4A)mx8|ru`eO`^E;$eARd_GUJ`}&C0el-t$S5KY{!Tfv%rTmVLKL%e3bhm|@R0@L%vv zsg}Jb1v^G3>>1o)b$u%K4Se7hVDpLkV}}`M?O)1IMf^kYkEB!IO!>)il|M&4zsDHc z!^D$Vz!=S2sXLU21z_5NtfdxdX2H~hnVZ>#hktOr5qSONbzX% z%)HM}`;(dXAJ1~v_b6pZro4?hMry2#zgt%O-0Vl1^dQl{Hb?>d`znRX%Xjxny4-ncrS8`ms*ff?52k*FGVD(H##Os^r%x|=3A?Bd`=}oq zbpRXn7msP%_O83knR?FFFXN2;?VPnAhirB}XYadl24A-76Q2L0^g+#W#ScuzzA|!h zyIm$r`e6H6`=MFb6F6^N0N&;hlj2%_L-^(LJBp9ppZUf6h8vcVXJh|x!#LnGgToD7 zfi`~D+<7pwj(am6?r9BE884wXO77BK8g;}N!tZJ5G5GVxFIx@!pSFmfZ#l0ZLo^TU z9$yQtwC~f>e|)?G`Mb4)E}4cdd3-c+0RjarQFPR`I{OU#ZpnN34==Q9Zz_3X&9Xq= zn~Tsb$8gSm2K#Z+>CsUqH@}Dv(*(_7XpFIu4)ml~m)4{Ey|kZbtY(sL7Uws;w8xb; zP2c9&cX??ml{Q6bnf5F%?cbCJe>*k~9>?B?;!>9hPSd&%oZ`g?s_^vJmpMZpl`a}@ z_v@NUS@P?eVszIGyVCowy|Tny;%L)X#fmrj>(ia%xzb;0*L5aeuB*R#b5=U=>+CIj zPA7UQzkT1eh&F4Cj!g(VjkIgB=~vj9C`h}n7Nj5J1;>S*D}mBu^^K>`p01lM&5P~v zY;ZyHDxd5^U*);3`daC;J3HE2$}6O!7ue7^D98uJE9> zgZK<)-Oj_mVjg=-eZPZ^5}tT$AY;8d5IZ=<{6Y{jdJ(TzVyYbobZ2J$^;Q(Eblbm6pxxdjQ)N0EgL zoi;8Z*@JPN>XjM%Y|BK8vvy8@rBgOzmn`?^l&ihaX4X0GI+~5|`{Y-Uk+nv{k7E|` z&KjfmQx=7ti_mvV(JM7CuA*#xr}Gy13v;2X40{Y|iACUg7%?-jSw75ow$S$d15-jR z?6J3?BTWtDFYj#SFW*m`Dbe$hzP=FjZq5R`v6a|wg?2uKjy@{oz1O}%kB)@Hqns;p zX(i8g`i7m4`M;n3tBu?tGjkojlC6wuZ630Ij`q*}vzwVC(Dw65+d!k@-mEBS-?N%N z*`>c%sOJ*et@2cs>T3qi7Jg>Ov3nh(Jl%mC^z`~m$X~?WKV{y3y?>B2-G_TyhqPkS z(EWFs_Mrd24xf?!ul%?3PU%7PaOEjRcaAstcZC*PvBl`je(uVx=9hwAE*-lQdU@xi zPmWJTcm7J=$Y(9?&h6`4^2`;UUam5WeEBWN@Z7EC-ifR4#>~C{(J;EX>X5!~!T-~I z>2GeNuG`bnm#Z#|vcu@*=;Y?E+hkdd%oVjM9Xmt6c_yYIG5hmds?pm;4{tDslJuZ) z$i=rnG_jrZBu#{%iT%*SvHmF`oELWOhaN;DQvPA%uJG_U=P;X3Jo9rg*t_n=#&ryqxxH1J-&Me-TKALX+t z9c*?b#1`19I1$i6Ieig-YrO;>6*%Q*6d1}}hi6b@Ev%BlzK~?OJH_m~U28u`L+f#{Ou{Dc!lP|dNm$liS+_zi2UGkmeSK+ys z=L^|GNMK7yAgd+dGtI19TK8`?b2a15X5JY&;GwnoGdO4S0X~w=JhPA5QpTB^y581> zDyy4UuFi!fZ3)6b7X1-F@5Fm|zbw<1eAmWVwAKw*#S=4qp~iN8haN#2bbd}TELw{~t zt?x`3v_o_=hB0X&kKzF(uI!Jg|K7ccmDV@AB?xy<;-&vAa2vi48$HMC9hH8jrz&hj;UGRODY$8&wJ zwJh+xwttcDwdQKyYl(+^uVD{1Yi{;!?wWhyByZfIqkcSVz0le0#8KiF;WMQ^J^?O8 zlYc86A7{QaG|^ts>K^+%(Z2=b&4b>S=|xFQtqG?=qSf?7rvz{Ms|=$;_!0g)wW774!LI2hX<0 z-RIe{&Y(S$_ha(th2zQVaa#0dr6kK zhUbrXZYw3`8uX<$C;D5VJ-SA3kfost)i4ny|1U-u5<(@o#c zEGt0$niKUO-?v*aJ^fa2dEyUN9DEY*@3Oe?qPEC($k`w6OrSkoMz`A?#VdT^IY68C zcKA-;s9fmI`8(fL%3cZmC=Pl6*{t;}?c0y+B04@>8x4*+S=X4jqh393+c>jJJ+hCg zjk~F9>p$9td<0MF$WM}O?D%~Wcl#K#WEt$mtlIh+eI))~$M5|=jxy%Nw#1*N_35kP z&G01dV;FQi^K}8VBmeuN zRO`;g=yh|D(I@k}h&{+`emUIReDtQ&kZexH@{25W{UV#WS6+6)qc?R6$yZY6SQ7Y1 zwzZ|j+pf3ZYu0P`qW5Nh3f+OTTtlna6TrRV3IxBty8WKyv|{-&rX0xQp3H2B5z8KR z0-J~9&E z%NTzC+{TGn#9YBnsJ-L1Xn--qRx&M!jVcXW74bqRbitO`72n7}?3ia)`VhRI&2~JR zdJc(BnNOokomqCoz-&7VR6BL2=3!*r=>^v0Nd;E@P=5IZj9(YaQOxNE?&G`&->;%Y zZfuc`I$l^~x;Z6Q;_c~vdbL~T4scL^eMINeR>)s;y6Z2xLjI!bkzwU8x{&(U;TLMk zSl21m#J6G29r@uKXVQvfgQQISIa=wk-+c?TQpvfG1paIB^tbavf6OdsVT^O2zlZ&4 zh}_l6JTK$? z$P8>b>=n*@;)0s|2Nva~-yZ3g*jA{QC^cQEbkHN>1ECYbp310xRMucG~|j^|nbYUWA4 zeGl!L@AB>Tvxyz|6SPNJDLWTBl4aJceEU0mqyJNZ7g6^qo`8S3d6#Y9B1rpg5~M#j znDh+$dIQB9u4Jx{W6oc}+#d@M7=wL!G`zt*n>dnqTDMWJMfu_t9q)M&|7KpTux?I` zb)!6FZ{x#_T)xSyKg`K8=qk)wQ#>I z!#dE&o@&Q?Va{5AVKDZ|1mCJ;o$SRPNJ$INU+=ZMp_4CG!Apw232tas5~$sM1O zrg@^hoV8uBL(y0HJ~H3mN~V=!d*|G7J$$!*H^0sNw1-&_zpY(VUpt+620_lj9|Pwq z`*q4L3Kq1eER})$oyoYrr8L=xj=+Ru_|h-LLtFu z4Mr=Yo}%De(AbjMXr(1oMm;Ymo|abIDyhBNW4I`j3@S(nnQ)o+`&)ZYh=|&LKIf15 ztUYV5y{^ymtYybysVfZ$ zm+|6Yo*iAU_*UWalCC`Pn>K{Y-|xyp7e(6-khh4uSKRqJLEm+T7v8BJr_LPvdCJ{P zxeBl24`Nk&%eYVjnqf<6E;q{Kr`tQuY>bQ&XNSDl9pOzMM*Cii!zgfUH z`t}CWakQhk{ST9uV_$F5Z2LNsX4%)6G}FFXDKufW`JZmjG^uotUriI@UhxaPpcTo` z41C1rq(DQkGhYJFusta#J=76b50&5-WOJU-HIFy#WZ2_OnrV+UX_h_Oq}ldJQtnro zxoH5pNq+b6W3MLG%NP9_;hnnn5@H_5R`kP$k#D!4f7;ToV{9lp5MGEJ%*Xkx#C=z? z^o&_M`VtGeDzwITaA}k5;L7`2o4G?FWZ)Zo2fp7aKL6t2gU>neouR&A8)^0xZ;yio zBiUEL=flkZT5KbTu`*k9_8|WS&lYDDQWJBkgE(6qz)wegs{hzLhksmO;naI#x>H_z zNGI^*_%9W7`4fLgp6qL!vqQc)JG|4CN6bA%-<5p{l0tAnk1dYX-jnqhQ*VsoO#FmCEmTsyO(&!B{RA=cBA#4{FSlM z@PeE0fhXbo{s||2o(k+i238B~R~fTnL4O&1ukG6F3hc|=Z#M9a+7FRVpnc7?>^rp9 zUN?Qtl*zW&nKaA(iAgi?FYXxDD>M~ZZfj+7C)7&pS^PK-ILtkH)6Fp?-roLFL&ux9`s7|^@MAecNP8C zUUMULU&*?6`MkR}78A%DuQidZ5xS%Jd1vwe9P*t!*`Yc%7RBV1E2h>yPe|uD;fSM` zub!}b!AXDjRl+ywJD;{?XYpmQ(%sJD75AGWGv7J(nWT>GN2Z-`@-pmPlcw9*Ce=Me zR{{gy0tRLP1Ji+lZvq2Xpr3bq;^ag2ReocF0oh<5b&7sn6?E~^ zArgzBQF7`g>M`dh{3Lcmi}YO+b!h)iRHmCH>1lr~XZDKX;NRSon?70nWQd-WB@LUk{KEbFN)p#U1v?i6zqC+U%amKS#Tp zS@*;_t2_NJgBC4#-xsuQK<>G`Cq7F_b`r4{*7fDA1<#$3M}6U&D^l=p=9^N23HCV95ubtG@-!CH2& zv-8`?gpCYX?aBsQu`A78mAye<|sUhTXq1MXYp%^jm3@+Z~ zw2|cE`?ajC&Pdt|lb(h@fS&C3D&{o4|86XoiH0t_G7iQod>`7C2fiZ{zm&YQ$$Qy_ z(O-g};+InCli*Xl?r7Q)Zj3Z(x;=vQY~DNklyly?iL&X`@f7x^;5m5H!=q7cI4lapAMe_j{l9{W`3*rZ9twjZJ;%F6EvoZJJ=%N^og7X^1v72eL7F(9JXL-UFPYqpWuG)~YYSzGEYlXD@d1 zpH}|u`CcjGxKb(OnxS+7 zXD?FuSm+z2U#NTb8C`LwQYhP1H~Yys{FnXD!c+XMc70Rh&#>d(8V_(K-D zWa;;um?g$HpSu1$-{Gf+(x%qFnwSl-k>^ zA4WG_u9P`!VO|BZud{}{i>8n!^wWNxbKLpWnhZ`NHWBe6TZTK|ln6FxXF;N!8G+b* zwL^b=ipN2FJD{&E=%f{^LHcjCq4S7%qZU$R0yAZI)4kpb*(^}+Hxu=a#lM?pA7xvp zy9HTl3w4K7H~nt~R?rVei{1Jcy7h|(?_O_&SP(fLYvz5eOPXS=*L&Q@SIzo&Kf7tZ zXvfF`|C4u{C6m9($mBP6w>#K95`7%89@a6}9ZA+$*#wNk|DrbGUzGUfb9&WW`!Ig5 z7VG?la-Y+w=GsxpZDS6Fi)oBqy0`A%7@LNN7&~}Zt9f}9+pGIo`Xs;fjM6WD2p;U6#t++v#Ms0eNY9vy-|D&|YgFSN#Whr1tAyP!da$|Zbs6tR z)F)eqD|T|9-qt{PK_T);evvY48qguhrs2@A0ih`6qE8r`hLy&qVZCG1u%EW~Z%Pk6 zyfkf%&SkB#Y1ot*YJJ&??YRqEhI_DSSVCHbO#?Oz`>|n&)?w4|7&Z-0VAHV5wP}Ff z?7G&WH=bz@udneOp6dAhWr?BAo%dGS6#t%ptvX<KcO7EKR(jyh?8x69X$@c`CPs9 zIZ}67$keI&z#rKvb+1eHoN!^8IK<&miiglkYgAyLVBcVEy6`52c;7|D`_}xXxG|d& z^v!=2{%DNj?oWQu6lE+ej3vpFQrGeld_Lo$9;f{UtS@t&fZctJ<$q{@{=k#-E?U{8 z&!;Zg9^v=$xq42mQxg_+hOo!4f=%W1vXN5T!e<7Qkt7}x3O!Ktv57@PWI z)(E)FKu5lpK1JwL5LLkNxG+kmmGh?0Blb6JNEK7$a@Qh9}4V;}rHS>ij+F z_p#woxks7XiTqbw)MK}&AS*e2Y}ABIL}wGpe|2ZzG0rB65215q1vKa6*(6VAlZ}b^ zkm)Jx$?Qw=wJStMQ47z%xVP1KEx!r;^7&yG+1SWAqY>CyseZy|sI2xS`JT7vK347* zuHy`G3ulN~_?}Cb)_^~H6dI#-se87x;Gc=zP9MH)`i=fi zDSfU@7-y52Bt-$|UL&zkUJ#6+hurejYni#*@zZ_lq%3kKI0{ zE6%CvI^Fk0=-NW%z!iM;Ue50Z*Ni+va>I1{1Ne5W)kxakh8>9bd%=ui(rTRF2YxSt zm*0j?{%|&HM7*G0*Ervt!#66gwy`zUnJiq+9f+FeAF&UM&N*}d`oP*9V!hO+Xu!Me z)G0onHK=jRo}>qqXMPi9e@)r>?75BT8$L%53agQg7ua|4z370!f1bGu$JpGjfR9N5 z_glAng01U4A^yXEC;9iSpA_my*wl-5(O>5twruud?6p|SV(5?RlpSMYoirc9d;KR4 z-R6gH!Zr`QJ_KG1E`-y6Wv&$;`Ol>H;E$U4R_~7TUvVcs9iM{lMJhfRY4|1e#xJQ4 z_}&*8X+LD8+=DP<0Q#PR#1`ri{3@TM!>)|{7_=bak0$${L(l^4ldL1O)4BUQ!v|f4 z=E}}6qd$1w54#_J(px^k?+5%O_q+&wP7Iz}vemyqE5yf5^yF1m^tEOm;T-4h>-ZqO z42_5*XJ|mK(16^tq7Qd9!uz(&wZgyp^fLc)%>ytu5r2pS_#CyPoB4e&Yd`}wUi~)b z=EptOqVH#~$j!Opb$XmksOMKE;_no0p?1{oneG5LFW{w7R$sa6tj+()> zF{ZDa50rvS7bs<4%TWqF{<4@CErYKBE~f#XQ}GR&!o8o4to~K(b+HK#V?!{iojxt$ zPVe!|`-i}^2qJvk&@#$IsxBk)Bwmp}d_zDlwYm5F<&F z=rzW!bW*7F2a*+K*vYe@J?-&ELo?Eof;+Ev=-FrVPy5yURIBl4M|AGdUDmY0nmGNE zzk9BI7XI}C=;6yp1LJl*6bSDIRwvyt7}}a)KQYINU%iSu=J9oezDa(p`$o8X6!_em zCO!@SE#^RHr{jNc%5&e#*sC}%|7<+-QWyyL^yE&{x5bpxeqruon;QsoHk-MQJN=e( zr{9g-Ctt>$erIu~Ur+AzTkt2RuCZQgtnh9Zu=bwL;EeHUcZ^}in6OuVk>{6zQzP$T zjE4VWe%lyd5#M!ckyq?|6dE!?l#%!K_1^85G0q^2FphxBI|1$pnd0D}lX4bpR%`^C% zjI77~FPf)*G<%T`(I@TAUuPe%m6%`8FYvr89Cq%smHs*1{&=o4wo>Ymj$C{2>#Ub( z*#ze*_z`=2@O#{3@r-LcTM&lp^@bu-|ZHh`Hn|7OUFc&A2n*zZ{zq;%B*bVsU7Q z`oEM^vYU3sH`2_L^e|aQ|M`*1FrK^JGJ;RZE2S6tfV_HmDJQSMmX71^D$iIeK3Cq+ z+c)VQW4_^Y@A8eU$=l>LoN#m_^RYEMG+6PK9+*%onkK!7;t$fNle&0fR0wP7YUbYzS>`1_kRdCWFxHmc%=JF#8Q5VI-X+B5)BTrHiDOp^?CJbx1jhOQ+q!ZhzK!Ml$L?yBsmFhLY^d)?J5zu3;%9d6H$SsO zAAKhIZVNQB#N!)E*-^QSy9k)Dh^e2+IC7z5%{`56-$q~0QghGpQ^ppG`#*pu@pFRb zEj)i{p0%$?CMTN!WL?aU{G;R>HN04H2-q{>w{^dT#+|6E!L94+Drgh+Jy_OJN!|my&4$&+p;B1)f!JXJ0j8Tgv{v~-aR8* zm_h`Ra^-o%F_3RWc|u;1D0hln`6|j)e~J%r=B z+p9Up=a{+wh_r)tt{en!(;6S8Ih{c}lDFwu`@`qzEbd-kj;U{Vav6M&>hqZTE}^dZ zk&oumre*1I!%e z+K&R4q6aUKCwg#?$Fd^~p`M-_}snCI*lPf(X^G$!%>F|Ct{UqNhG>X6KEF?oM;`*j4`pnW0d zo-=1@&lC=vMY$U&1K$PB$+kan%cRgJ$+S-`do^YMLO&f{xax4?gMN$rQ+46SPXzyv zE}Bnl&=$@OC+~sEvfnVYD9fJ9I9wl*m0#y0@}c|9G``U|rjU+d96BprZt}4CH)*;( z$t`m_e}j|02D)#SI~Kg2V8amXo;a6n*fc5hB0Oarc>H*n6^?H5gj&IIY^Ou$m!4EC z_{7~`ErY|HMLnEFlQ@fdIg9q-EXsWiOL}n@Wp6L*^=={##)tES6AtX?E|ON(BXQo0 zSk_p@$Y=vEzTX2sNY+Q^>UEsqL-gg9PbY;o(YGzLN<+0tIkwNEJ5uL!HnR?%g>CVr z`GL-x@&bz=$P0A#;Q4`^K68NQzB zfyWmH=RXduwabBr?&nHFbD!xKEJm&)ze1hQ1Ej6<;N3jA(K2+r^N6obe7D&t@zN%L z<~4QaV&Ap!_+?GsUD|uil&X|9e&~#BzNUH#-r5RX9~3O_tR+_6{;|GL6d#Wm=j7O2 zWGBnQYkacDnusltIUnWQ+gL~WLe;YuN{*Xte84m9SCr446?@o6%DD$t{tnG^@SStZ z52F4rjN{D#U2zkaGk%Tb$P7nDLD?=pe#X4fdcJ>G9EtC{WrZhC@+>@LZP6EB&U# zS5WUd*0Pwf6z@+DqELpu=GrIDa_C15@5Q@rBfjBpp?UgVar>$@_HKKUXbk-``cd>1 z8xy)eFZ z>T4~EF{TRSq_nkjA!CaI56r{P)O&|2wqUR-DK+{mdSdK5f|iw12cHr3@LzEOwt90r zp5G3SKMS1+aB0qtlR1~Ua<_8Y(KyhRG)`b%Yj3_h78y)?$fRb#e+Q7OwIiok*W`IO zpS!(9!^f+SlI5~44co8TB0YRNb8pV(k+XwsWr9oVpz7)Yt{sC{`GB*K?)!cp-sF$a z>G`B%sb8{t)!oCb`z-3#9L3mI&Dmo(@&NL%-Tho!x!No)mHEp=J8ST))BYhhQGaWaTL09|3BX_aW|wZ z_j^+7Dv9@NwZ~@{d44jXkoXMqnbTs=+6jflu|rh_{#xpZZ0H-B zLS6}a%$5z@t6bkNbSTXej`s6}b*Jee_fFF|I?zZy`piO4*n#Ve;DO&lgIS*@=3jek zgZJ8w$YjBGzI`R&py2<<9RC z@-;`Bm|x9prwhB^BR?^>yTIK9Py3ns{=2+yOTabr`&Y^!ufuQD6RNLIdYAj$gGcYO zLLV|GjvPzA4_VAL`eWv`pg19$7H(wN-=aRvx90ep<{AG|ljhn}NM9pP%fj1!@OT2& zCj$Sw@aq#DNdpH2`=`b`5kBq5R`O$Pk$VEiKO?rrdSah!B=*UJ#6Eeb^!^uqO5BbY zOP!dcqB)h|N+I`^mZBe!UQe*jeiUv%2YqOGjz4}gHV?D%{D-RY{pGr|WNzQk4%RBN zg!~zW{u0&){0m2tJz;|b4*}b(ccczIv+(0FE7mp`*dM}s);h8Zd5mynB;On%KH|ZC z9g)8{srH7f2d9PKPs$z`pX>==e>V7N6%5@0Zkl}~vvOQvPtI^dtl7^Du?S_3>`dqH z*<*ssd4BI4XYG@iKjDU8cUFltWu_XnfAKl$sZV#Xw5I&J|^(wd>0Qk zke|Sl>~45c;=ZrveS$9a`!{i97WBx-r@*sX>JfhY(4<-R118P2?tbTR(6BCHI12Z1r z*OT}|!bizL*^7cNf@h!dvp73eQE$=f@!1Ewee8qaYA@>g4S4zvc=|5*+3U=SFZRFu z{uh4zz)z=-nB?H<0m|J9-*dp*k8k?g=MaymSFhz0e$96y20HPF4low&lhx=o`rmv0 zmj1oOU-UO)`hdMo_$(NJxB5z)9tXS&r(@vsQ#=cwJL>y8``y2R&%$Ac{$?Be{1|)| zY~`_r-D&XCGoAbltu1&Vx?Bz4kf6aq_kC~PCt&Md@f!8kIA0O{!%kfLoak{RuCuTe z+tWpdrAI)Q2Q2A~z7N=H3|U=p_DShUIvh36S#}HQT$e8q4H2FOfPw0-;31#n-IKDh zyAnH%)9{dfVi{DQVU0@gk1CgC_(pJ^_7b$<_<0Z^{%9T06 z2((dWSwnB{$PS)G-9KV1>#zY7k2?2T4vZ}!KOqOYooCG-am*UU3zwijkzHm9^1BZD zF5YGWaG<ziHZ8QQN0R_OOC#{H#Z{@xeIyf5H+@=A@FK5EQox?{eC zaXNCteER~P3%w`#am8nr+)I7dnD6H7@MqeGr%TnkoArVKmoDDp5l|D z%%{W^7u`FRm-!H0W}nN;^mpl__@nvIYL)+#-@_iOu_rO)e+u0E(bK=KzpsB?Pw3@O z;fX#ahPuvNpTa9u@!h(k@!9?E9lqsLvj_Au^zyIJ%XVN;czN~q_~rus8yvhNGq{0y zE=&#_w0wb1Uo|!lN#&jMQ_H!|{DGOWIV37dQw%IlCdBXHWlw z#d=0Y!(M6PD4^RnWo`gI8_;90pDyz7{FkArv$mc{UKIK9+C1)kx$_2OF|J}E zY%ayGb^oRsnVn>eDqoajHQq|Sh1J%<*O5mkuY&q|1Ji|8(!s%*vezz%wy=+Rd0+V3 zyFS~`xmI?t@Rpm=HDsu)-YX9|pNV(YK)QzCVt&^u4_NRi1&?>&!x7^58obyKNX`7C zH;`N-wVU2xB6@=r=nYbg-eAgN$^KK(9YphqdE!Y+<*Yi?hyK8KhBeTK4ngx;G%m1M z{K>ESeQS*7$hQc2Nv0z&X~|lfHxVBAbs)j;cSiKK zE!KK|A8WJ8<9vT?B6!@}+Dz=&*vo0y?aIDl+n8WG`8xLtJ|ju!G4cJ-9SfWKVOx=u z9xdZPeN9cl9;_uP{Vo6TtC}WWW*w|iY(}1mDRyu$&x&RGQGz&?^D~s6Og{D_qBT#r`F)j-jH7v0&(I;{3udq63{~g8@5OuBjUiWr z9v9dTy7?M=L~YY2$wk1q#!buaY}!SBp_Lx32Szt7y92w7eEWSj&#}wMv;XS;uK`{f zE+1W|m}BbGPB(u$>$7RuBJwiq-@AFwfitqP_?zY!xcH0GVS(6tq;2fu%_9S`oyvo* zyrYzF|3E7G+k%~U#s2u_Yk=hnt><};{h3*}Th9p=yY^x$D35+(;FsBLBE zYtypZ>0^QYJbg6lK>vQBw&~wXO6lK=O6l7MrSvVNl>V+K-O8OdYqkE+$meU2MS&lV z9~|+QplJ!`+()EPHx{c3bbj_-jdiF*!u&Q+;3k!FuiB(b(*u~>L0MVmGL37 z?viy!ppnF-v)5w}5ZX{0+JNoS3uDVdKQD9S-k;F^{+B(WC~daT=3Z#&htSlI;G0jC zfq&(_T-#H*mn+%+Xw|^b-nGE$V;P~{$aE9~I>h{Lf^Pl_nksv!$V*nN0UV0FOqxcz z$)S5b=y3#@5dQG7g_hMQ|2OT6>@h2?r*eYr=Q3{W9y-BC(d0Sx}qfbDesz* zADOZj<_43Q59wdb|3Bphy_z4+;m8Ev=}hHJdp~q#p*3RN&_L%6;Anf56*FbuKPUJ; zb!jg!|JMx({+0jQPh{)aBd2t+_OX1Am0s(0?8hEBabi^wwqZv&|I7Bne0O(A@ON%K z>T?_45YKO{d3T{dxSe;(?~zuzSno`Jhd=mVD0|3M=<77{uYkUuhhDZ1df6&ZpUOJuYZ~;m+B0o&6#5#!z*=+zxESbR zZANxDvp4#NC~@(oL1!zWu~q14!Liv>p~F+r;imFVGR|kQU7g6dZy@cCgQmYd&I@{6 z89)R5{D`%xNM&BBD>BfUZOYt{7p!&1DqZr0tm)&F%kOE8k*r&^Uo=+oaLsQeYh4MR z@8|z~kM>>aqRcSHTFF?y%UEw?tiu^=KgPP%)35ST#@dImZu5L|@yC?CjWK-6Uh)S| zztk5!eTPwG3C<5(IIQ|4$ZSd`#0#0RF8P%mBnul4T1-;P4rK`OVUHj zPUpzn8KLix*ItDW6lv@5t3wr@RJ(=$#7j2%8F;tNDm$eod*Msu!=p!`y}#oZ*?=F9 z()H+3lrG`D(t7qx=;RLGJSfM_OC$Z==Ae#ljk9?1fT02@F#xsGS0i-ns=N9 zX6t_m^+)bO2ES~F$-jYpqMUCcxmKh4fz0Xj<9T6tuoU=u8=59Nhw~W|zR#l*YlCeS zesgQ{{KUgz9fo$t$+gE%tg6PAp=H?BnE0|ro-DIYkrZol6Y}d{u?MSf=pjZ;<=Lo%@;d-h3Lqs-z_|m?9hlU`N3CcM>OKi)Y8Qrj8nQKjU8R@ zs4d9l)RyX42rT%)nIiDz&@6l(IExy1zNR3!p0)%7Iy+P?b8IoN0~n<~v@?Fr#v^t`&GIKO^wC{~1S5hzzjtUCN6mh;L6t7lw>`#--@O%Fu;PLKjwwF6`G0 ziqo{UCcDEvfqou*t2hxKEn4;*d!6>EUC>zB1tW7B)lf61V?~wVAtee<%@!ZAk9Cd# zCoRKgg>K<|u%CY1$ozJPb+ZNs&gilge^~vxClJ%wq-9kgwvg{?klWS)zm>&Tq%+M=NeW&pVbma4ebDj~RQ#vOha8YTl)rcmgTl#K*&XheY4o zD(N5f`H`cw+=b0AaGY{1zWK?j^ffC^ENcoKjc+#kNpSlS;PuEn3-yny)4U6vrgWF! z;e&cN*Lf$tX%%P1X5g-he3c)B|G^i=T~KIrvu|ju^k+YPksoo(;92kvDbf3xtMRFy z$0ajg9p?v2UAl0I3lH;z`^;DCTIqIu(K6<&6?lkN5>I?->YCPB7O~J%qP5IJN>$HU z{^D!WhBBAV*i+ZE3`>C}@d@J%m@`&O4fvPmocpX1oNeo@*#FR0c(UX{ZBbzwOLFdZ7*35E84`I##nb0|{fnY}S@k{1R z{8^^zd%_A=oZx=C@8k`vI$@3Qjd&t20xVZ0^;o2HnZ|-VxYDn48Squlx60c=Y{NpI zFM1<q;v*Mo~`XEmK{j`fujTwQWdIJggM%WZ`^n`g-o5;5~0Rx|QqHSIS%XS!qAjom#i5 z9{R6+ZaKIG930RYFpKq1%)u~s4kBK6-Zzd4LJu-b{VT@=`?zy4iTVX28uu8UB^NLB zWJG<-pEPsZUSu`4)0UB!H}wpuT!A}ZH_*2J=V-nx-@!eMInjPvw>6kD-I_hO*c#Kl zpE17E-j+f+N8ZStUDT_6@gU>t!smMM89I2|KjN#PJu~|-|E%l5-QWX!!$xa)EaEy7&)KPY<%!F!?PYCKmY`K~PVT)2qy%M^<4#)Q&3?;w{F_Y?%W6~{2*v62!Ba(QtneQNqqK3A z_AmOVHEUzd5_2b7_30Td-Jsv_Y6tv`tCe|bUC;yjA#cdg5aufaui6S7G4$i!!NEi5 z5=1}7FfQ?Macl|@bd14%aHjh45^X#D_9kOfY~}-dpmNF=v45C)j-4G;9Y^z>IjZKJ z&cQi+JD26kj-u?%hEcm$jbFEP&37b@OH`a3Im%4AD4b4Tx zb|bBxtexu8K|}-PkmUKhb#9hkwuq>5NZ5Uyk1RWBRe5 z-?`ZM{X4qjt)5eL$LZ16;U@*ZN5LJ*(Tw~IdbK-wR%p-7f!O^s960?mb0&D)HrenK z8-4g(N=I9PZ2F(mfIaSOUsvyW*Ph~(|D7|(Gn?12hg^p4s^;1)J1(_GM<-q#`gj5R z7yQ5WDd8Ra>Mh#ydk=UbuY&na#SSNW)D!-MF@6#!o`TmCYf1LRhAr*8CYt7neUhJq zT~boGj($heJ+anoPiz|RWb+u!_uwN(nX;tt{>e$VZjc&Rs6eV`UA!0Ygw_36NHM9k7{P-d4q3Xzc%wyt0d_sR)$LEGxnkch@cQ5hH zCQrEaWlwkoeQ2rANAJuzd>?kJO}zh$C)~g|xc|hlBiVx;$;)2Hj%1aS#*cWyW*^D8 zI9N&@GoZEN5q`kDi55yODS2EQ`;hjM*i`5J-Mn{ngZMOQuMr)EFA!WGaLZIuMzS{L z{e?1dbfV?7p>wDY`bY)2UBq%_~fr zZA%8CeZ_~&>Sg+w=;J$X`_sDG&#+%UrH{XH%S@t-Gmb3#=agyoTCv@GlAS)yaP~)kSgy0!nKQ`>*+^zEOK z>&&<2B8BfYH_4=hvoGJ`uhWGhG9zNg}=uHYNd zZrOl{PHgD8>~Ya6*@Ea(3mtxQ*@ou~4&O%?EW$%GzTsPgp zi=Q)x!iDFdK`Nv7hTob-EGxInPbt&P*`pnJdh#sz1HLaOH8LsYr3G5h3N6syrC2V9 zh~*Mpe?=%dBqtOV&ANy4_z>v8WcHOId7%zylz8z7G^#}SKFL%bWME$%6x_Ksavkba<27c4$3oo&PqmEzN)X+3G zB;KN-Ca0s8ebm~8@7dVui`RL7jG0^Y%m#0Ehh$Lo;*F{AC*X*|{ce5tQ+Iuh>>p{5 zbyD9m-2Oe+t$*s9`rb$H)^kT(_jhxh_|4+UO+6!>e!s$hBcE{lt^S&R(ck8=+e|-C z?c?`^@2r9R7!P8 z``XwO{>b_1|5;DIHN;tKi?tNJH@xEA=LEC2I_Iy>3-RUC+6~SkE{#Jk?sfOpL%d7u z0rMDd_dVctw@fQ#5_`balo9_YJJ3DQ{k_oHwmI;!YaIIDo_{qqGFiqSM|5^|R)WrD z+JB@T(YxQ9G|R3+KGFl&a=w*Mmtsxkn0sUPOa_*bWkzZ^ze`@C`gz{{gy?CyJv+zq zq+|k;gNIFhnfA-1o#^1j`{@4idhgZfiUp%{s_F2 zomBL%3EwW_ZM$_oM_PdlS~3c3_27X|(#yTj(|y@9xR6d$K*qAHHIIVdUNeZlroPGD}4+dg3=k-)F`EI4=AOd_mghzjAvhn4E|i|6it1K z`u#l1*O;*m6g=Lm^1#&hl>*~;D`g$-B)wMl7G+-@b5{C2-KnCxoaHa6dtMWJ8aL!R) zDs@S2uXClBJY#p!G$katuk0=)+xJqZ%CwNbum^P5JEx;#&7k(Bv)ftohrGj)e zzw!KD9mYMM$V_$S_5Cl}TEHvL{#DX7W9LwJIcNWFy7!gV{*o@;yFE6cd;jVfqzT=- zfst;u8w(S*8|=qEbbH8?M>D>6x@l>^L5y-6CY|=0E!1I{vTg zjBnnQjvsfLYr}zU$Ajc2Y&dGY>Cu{492*kO`QEJPV4nS+oYa<%_^1`$EXRK1Z0q0x z?r}Bt8$5e?UdLLDcK6DS)HnY~ywQ>6&jpDL^(sHa zDPIqcHp;exzFw{T!EXJs?|_zEsd|Px^^}pJ%f>@lW#%e zn@6HQF!M(IMl-L(4^cWX5F4$ujQAl+CkJ99l(J6@CoO{q?5fl3HyL&@-ktlRiu@>p};ouPeDvGvn!?^)|+{x|1YoyBq~ zSH0J<7kOph*pR{bud*5W^ouqlTK`@Bri8@9C{~DLcbaZLgl)-6{9dI`Ot;n=n~}CL z>R-A&fbj}H&7DkM?E!t!>Geam*B>3<0CauC+gXy%9Sc2zx>rkgeQOWsbZzbqVRI)x zN6~YGs~*QL;*$0gM%U+KZ!8=T_$(lLNIS=u%%aYj)O{swe2cbbpg){W{QI8fUMQ70 z#vLfN_$oF)TlK$|J6;txz63qhjwHt?atC_3SJSZ7(KtA3u7OqwH-5%EYCiv4`RP_H z+vF3U-_74jek45*(>)6Ci;eU7HTFYSBOZe9g`2WFiSz9o+K*E%;fD|Hu9FO0w7V@? zxSnkbHw%4P(Ho(eU!av4LxZL6{ZaF1SF(>b`rJ%=Pn!M(Vvj2Y*Y7oHroDyrs7T6< z?m;#+C4N~`!l&|2%Tm^GKT@=W{1*6(F8K-PLMM(2ca2cb2p66nq(1TO=)oJ;J>hwG z-enG)Rgtg#x`_M=`HG}WJ`N2zUM^o2XiUV*xnWjvs12G0CWRwj>z9%3zK&Y@K1Dv8 z>r+B4>+vl`#?k&Nc0<3h!pG*O`H^kDBiZIeZ0@E|wwODUj-kr|#}8ON1_U#K!9nDE zNndM-ulit8;9$f3)p?m0Tj{1Bedx#bv}5Ze?|yS^P(J);=Q(|zNuT92Z=uUL2Fwa> z+gw@EGaS$T-aF; zy~<$izp#$hu;6mm@kgvpVhzVqF7d2BjX351F#ZF(J0nejkk{wjeYG37)A`{z?Fe@2 zNS$*L{Q)L!;$6FWmu@E&>5hr&oi_bLzY|R3%%gKd;x3oavcMCmjHh)yHY_iBp`XMc zueZ$DzXMFl#!Yasi+Or>`?Xu%VDFsNHjMWL_C{dA;n&mc&qk|ko_)Zi1@Ws$*w}DBo(i5%!A4_B zp0#;o9{25mAF*6waPjlQlljpf4TU$D%6;5^^hYVV7T35{&s+gbn1~Li7N5u; zvawmnu{P^nh`CJI=G10en-^xoBMk^VUcz%rA^WAq$R36e)a=(!14E)2TgVsAXl;a_ z%G*Sq>ePMfPiMf7!IK6^jqeuvzkK;A!?UiWykb#32W?P!c<>CR&yXI4hN#YmD5E@R zSk#M+RA^c58p*&5t(2%@qty@_bHxeyR?jwm)T5SW8k{H2`Rt?E?wrJV#h{RHexAYk zQLb#Kmc9y4w!w4kV7-0+d0vov)mHjHJ~ueVyX3PHaB^hwlFv+D>A6AOYbL!?62HI; z4zArp9n!-csBmk4iIhhG8vUZ3Je@D4sf_*SfU%>PTgbMTFK%(w6@&%EQFW$OWhZ-3x_ zg?HQEeFL`ry@mI!$&wjc2Q_ykLw2kVowytt*ei5wBJya_wx^vt0FJ?joxBGi!+wCZ z))?hyYB5g5j50WQ?l%mM{*k#!oT~>ge(gDe4WFATW~TnDjuhTEGdIaf8E;Re^xvzL zF?y83`|KN~_lo=S)%L@U(w#$BJ8T! zw_oNgf6_KI8Tp^$FO=UYUG!fuCv^)*o;V+e_So=lPw* zuZMoj(_Ug9RI@imtX}X6JpMeR>tXl5Gn&#k;gX zgNWf^c#(SOjrfT>cqcvtpF8m(pC8Bd=Lc_a`H+jLTRz^0nkE_j;+^Uj^L&R%x!;Ub z_lqh%L82bZ15JkKc$PUSe&!p&0hH04H?ziH1qb3OE4W*shVz5=qAB3jc<^e)IsaHa z5<9HBmt`6HR!g6|Xd7^sh_8RbJ?G4%y#Qxn@p1# z1TGe-FZBIPQk!z-yEiZ3-fd{uGfsa^nWOPW(P;HEGPj?9A-29n;=#(eH<&pZskK?} z3$-<+gt(6-9Df-e3m&XxduqoD-pS6%&{+KK+y8;Ri)@7r@6_j*c(MreD&y>btZvx{ z@5Oj**S5$*zqGfBULJ=w?cO6k3HhzdCq>8;jjb7^_(#Y=8OvF;x6{zEY#}Kr>*fAOVy~Q~_$$~|yqRiZwV1dp4-m5@ zX~0#BD>!3!GUhzyCZzRGO2;r@f#2KQ@=}jbD?G~gG?wA)tz*U0 za3>@}k7l0$0 ztDAUl&i?LsMsqn&dF&@TH%9h`y_wd?y0# z+8keUHS>QJ>oA-3n8muxWZ$@wc)iZ~uZs>qgLW}yN9LPnA4T3PeryKyh-MbSqlgyx zCpr03+_s94sWbp1kyLB6_~o|2ef@Fv>O=GT``fApI6nK+n9oY${zp?nhh}B^i*%Rw zT(1~#Ag`A6g*48K#ZcYDCc?R`7qW&Dm1@1A}bJa_%=V_8mL&Y>^D zr55(&^1&%<(AAhVsdwqtnY9UP+x{J? z+DzPKcQtFRn2QzQp=^aVVAoec+)6!9<9VSMA2FVz*oEAZ90ew|$7Y`em|!+y%7(07w&*f*Lq-JWk!-A6T*HJ`%TmjeTnfrZP#@5{jN#J%5N zt!Gf|X2FQ!xC=%UOG|#8$Dqf;x9;@#rw320+J--j_~b}Zu4Cg59sMh3%$M<-dz#-5 z_|4`wn%^Mxk-3|g9{8*nIvVlj3_S$gFFXN%NPMIz$c05;hZ5gvYUZ_dQ=zYw%Tlo& zCpNNZE40*)45Wqg_IPu@Ql>qS`PLff>}Pl};GqwBg}(Ht{NKJnL)$JivIN=lf8pKx zV@zCU(N~S>wDk4;vB76u`ue*O&UsR2fC$h33h#fCzDhpe+;t|oY5nC6Tpytx?Wge4 z26ocX9p42n6^G6ummVdW`*y%7yPvYc0rj|x zIv(1}-YI)lANy9MjQ0x$IP`a#b$pH1aY$L{C^9D5fF4`UDDVct7*IKw`h_X(OS zT8nKBbP+r&6Hk)Iya`VgM*&?VG}?*DY+??$XFB32QlIGeYLjN$PmyXK%iv-1t^CwO zteNV1oP4jl-cDYY{iyjS-LCr=y-0R13Z|oWgI@W+NXTB;>Jn7 zcedMiwQ-iaM+_#7ER#>soU~N@2sVII5pTiJ6s!7iKQvap1U1ki;s6buNQ}7~dDi!m z#R*?4z_mh8{)RWmU&y&3-;-arBPl;RsJOf{bV|PrOdPl<=yCl+J@iL)2}cS|-@g9N zyw>&5Snc~SC57^6PkUkxw4>ATP%zbN?p^e~DKD7T_^QzX z3IFg{a`_$y|BlS*4d2tpZ*ULXYN=0#uQ`cxhNqa3;CZ;$ZdR86DEM|Ab#|T(-&`BA z|Ciz5Tl9QKcC*>wTaJB8sgwU9`NBiZ-IdToM`xC9-{`*oC+{8k2j@cG$Kh|xo)+zn zYnms;6cw&1hNohdXrx^H#xLooD5x3JNkzd?M9$G2z~@`S&EgIjqIKF+?E=X*33_Gs8> zS__pGzxQ=KL-R;aaJUQeGx&BLvH)iu((PBkJ%_~a6H%LB8njOhTr!h|Lb|6RD1w2U?Cx05rJw61^KBCXE_kjMePh{BWA4hTjO_+1%JCh!{ zggqeB{(%0-4_o%$k_9Bjt$juGT>lS&+bWw&pI;NKd3(Sgr@|AZ!5eXYpVJ3ksW1F- zKlE$f;J!53gF7)TbQV=i3*A| zk5fv2_9}&@?l$?NsWTYMbjI{e#&!i`oCe=F6}_HwPr}i`R#-8G4^g+`oO4e`Y;##) zoQ40Q^{sKi$S^DXG|X|2Kk^UK6nS~ z#`$;Fiq80G<~m&7nLPF=wnl<|-H}^~&8YU7%+jHC4JWQ`iZDiluZyOIYw>R~Z8wh# zE}>nXop%c-n>%zwe|@Ek#+5TC7SC2-MDPaIzxd7cpqX>%%jfdnNe^nyw=tj1k{#Bb z9h`#gzhLC|%=bLx0~KEEDl^yS?ZAJ5eWDAGOg~;2-*uuta{~br;qt(Y3;cs_&FAePmCyb3Qb1Pd<#m(La+(r5sc|lln>m$ze;p9N<@1Mj+ z^}05DR)qIGJ(+ca2hr=@$i(&lQ^Kcn`a86!lJSVXOTKjxKjBQGZ|b-D`r43@{?I;R zhkNo;8-|SXYppESV;uA+5anJ9V4-36eNBvGQA&^dXZi=MT^$|xHfYtN=)p-^i{aI+m$hnWBx=()n*cH zj)HzXx81St^m-jVL?L;iXM4CKS+cP-_rHlX&X{$U>(!N~^PLl8-;>A#=4G4q0OgJ` zCe`K0H_}Z^eevFgKSl;6nq_o-+%?>ZOsu))!3ihz3%T~K2G6DY&9x(pp~Z{;9k9pu z;S!Gpeap3ve#1G(^=9tBI(|d)yVC9JPVvOKs}0(tc=$SZox0CW{>0%UE&6!9Ym1ai zdIb3VGx+if``T*KU(>$fLA~rXz*;Cj5FX6EFIBA13Wsm&!U6cUcdYj>F>+S%kx%fg z_Nrf!s-5_D-P`)*zQ-*KpbeE4_Y``e&pO*U_ribIm8rZ=J=OTRYv0IWjSGn9`kHW+ zvW>jMCLg<6E8{NLHhwdC+Do_t&pAUMF2gR(`h^EP0iSlJKok70X9dxB$0mCC6=Qqb zlRVG)!BlGv@#0eK-se}(tSZI-a0S12`Q>_Im7bK~BgEb-#ok!9tcuNASn4^r9NmHH zRi3GPS5~lu@~owSg$VsSb~AihCAc_Ov_HfCA$?T;A0Ta6!1}{iG;l|XcnS}Eh7Bxy zkd#-c_zhRV7tWhyElT3Kb(Lhpx%OOOUVO!3z87ALhq~I$`yP4D8HsqSZr&XxFEw-# zYn{nYcKJJ4^9XJGmj23|=X9@e-n-v}Z^*T$@m=R+osILNoin&k7JBvGGQG>TFXf%? zHP{YZh{p)1PUyPxw>SNe?!B4zC-E$Moq$q!j!Q^$SVzsvc$1f5 zk0Vw2LFAkBT)uT*>w3%2n8Ou6fe-uH`a3PpkTaVHd$Bu^{t2FQu54&>U7k~S?qu2j z*DHR&;lIvu>mEdU?B+o1d(dRVAN*;e1Akis;XhyO(0|?QE1p1oQ2bE!EyMkOph?s1 zj`P%i&a2=m_Yw?+u5Yw1^aTGzel>6!zsm|s=Onp-QnQa`-bRehqOqF4UCfgi8JxE-Y&Z;I0Su!wA zJdBHj&G?_*+SlRd57Liy$OJlP0Xv-E8$MdxG+%o+F{D%6J19HpzvhQ?T{v}O^~_4% z>z+^lc|#QgaOHx6OM*qERFBf9ltE$E)~u#qdJMFKIt#tZ)$L3Nq>*BiTioY zcNLcet!FLbPxlD!hM&vhJIQ@oX;1N~e8s-H%zxh76uHas8I}y&=W%@5e15;v_BQIc zfjX8`2j8Vo2Yciwvw!xU7+lQwn5%<6H_o&1iyV%h67hOEX9dC?d}ATcnT5@B06Xkg zFTpy7w7JF`texP@Q3G`(;NuxR!+&RD(-7H%-z#A#+D=J^=)$=0lnZ=MHk@Zk?bFRuK6SQJUzH^#j0Y^|yu_{m`W z4oBi+%>Vt+p+ZmZ${UNPbbkEA!+Cw~9d@tcZ$6^$JZaHl=#ZcP7V$bKa!1$=&>?L3 z!r=9&hI-Gtk!0v9cw%7UsfocP)@ITcU>sc0xvU5MGj?XsfPM7m?Y`JZa&PEia6)7K zHhpbo+}CK#^xpz@XY_OE&h>i7-nOnkbELJyzcB&VB}pan@dUmm28(GUiJ!Ua>GnX_ z15Y8kb%nk+XL9%&V7!Go*#AxJz~%U)C*b^pUc_Gm);~xpS|m8{4j;GBMz>h!D;InY zPIBz3t63Xjd?otgqyOjw!rlASgS_|MF%5D1be+4FTDv3o$!Kj3ur8^0E`2=h&NBu^ zy=l{K#n!2sc-=~u<2R%9{tuo1H7!p1z{gJd;K4U8{^9!M8NJabCgX3@pO~t_KECPu zS?`0awfHG#9S11Z_kf9>{?Tm*2ohRbldfpwB5EO|C#Bp{cy`||MkQ0ZO144*KNlp zyms4?!U@x_E4+C6;{5BT|Gi-G^jCT=pYZ1&9-RIpzq|My;g|E!#_8ku{h^qB4xhh| z;l5$gk|gA3_SeOg@+yI0uQ{8^G@H%2Zl4m!3cYz{Fz$P1F;n&&=-ubMg`>m z8WECx zIB%Q=cXml{p1_@2Y_z@vcaHag*U3AHJ3rERXk!JbaHo9_^3P9+y^cP(6&;jfEw*^k zV-4wtzN&wyWypX~JF%rXhlGhM9ZXd$=?#~mM@$bz>aExiSH`+ zYNfHEyayXf_?{GNIWp{iA@MTUO2yiVckc@Ti`ky=e%VrvwSv7_7p+~CGehJh?g3wA z#jpcdS#$<=0Bf+ZdMP~=UC+0(Sj%~y@FQi(9g+2yvBp`U$g5VY6@Q<0{C&WKofSXX z_F2Qv_I=iYoo&jx|4`!0D2c*te_uLB;g)15^N$GPiGJ@iHEEuP=N!YS{x?psY8)a$~6 z*87!2p38GRPo8-1t~v~SIQ75DyPxyU;4?9n#P?jpfAx1aykMG3bDtnpf4&TVlllne zkoQpE>aO;6-%<6>>A+tLbXq)ZXz+z0(S?ZDH?#rx6YXzUFvjuU`Wy@*OYefgZ2JM` z{A*$GwJ(RkqrKtz*mDwl=p(}%_;fEJFX|5r2CV`ao#|G)}Mo+X*jdG4A& zKzTG_T-%f^X_h?=)X(OXW33XYqW6nYVoV z?~L1l7wTpWF6G^2ynBtf>g%xaXaQ%7mj7sip|1<`XDRke(eg(ooW$Q;I}B{9-#W{V zcIPgFo}xSd@i`jW_T_W;0&|BS&Zu^SzghOh?%Z8Ssxh=Pf6)01;qgfFv`=UsQJUb< zkI)an$1r`%eEF4vH$#*{GtVZ~zJ5&e4J;S)|DVJFj@MYkUi4r?X9Xjir9;Ce`CEp0 z{2vus{$A9-4?W45$KVy0lwv1m8GCo`C_*OaMPAkexmi!-XT6Z4B_k8`abHnVQ1=SS zW=`jF=k6fKZcg{qx%Md=6Y}R9S~-{ZfmfleNK!_nAD%F$kJZ@T+iHA|-zMzmT6%FW z8~4^uBF2=;RVDc<(Q(h#nf(=DPO?CqA4|!5jq`c|d4C$J@iTAu8Ressc-5Sn^X-qw z*Irw~{QcCm)!s+mHssx1GJ`Lko8LLjx%r*{rE~K;{!`D*^zUf46|43RKGWJ(e5R-W zH_j}|DLRu_;^Sf?k(1P0!Fbg2LmR`A8Ng@m6pWvg1dH1V>DIwxn~>9|Ka*FEtW-XeKIECA$vY%>m?z!@ zTX6%2x%LR!UV}UqN0tocbf@|)Srh)kVIQ^|KKA97`jn7l3)_)Fyaue-R^8K-QT(UX z8lUjj_)8&+SS&k3DY zZ9SB8A$Hv1_H=9XM600k*>r36+9YeyK4cff41y+Di$3DH;WwNK)7XQcd)TLiidZw} zZ051|uHt@P{y*3k{s_8RJ%G6!PE0mA648 z+18?UtYu0D&-vD(fSz-#`%-%p)U|M4(pdZ+?zZ$;_hA!c>LjAj>@?b#mcoCx-f4Y! zcI%xsfM>VfX_-8`^-jyR?o0C()al%Uj1tyP@&UoN-ZycU)jfZ0>>G)8x@1x>^bK0essCB(2cK?I z{pgJo<+_i-M>)$QeuI6OIWamW&N~S zAZ_064Zc6tncHK0YxXqg(_wg%Px(I@AFIeA=u8jdK#>12dClw@hm)+>KAy*L&)GU} z&N0Imr&(kEBsr0nc-GzJL>+yDyP0z%CxVaK1HFRBL{5Z03BE~kqDiUd&ht_2YMc7i z)=ssBOzHQeqB-UH@G{tyY^SXDrgzBShM$o374co2$i(zatksMacP)L~LarhE5fA<{ zf*0gxD@&2DD4m3SWh#9tL%yQ)QsgU2FGIef^m61Ysle-Gv9`H6G_x+kZQu`MU=_@lsEGd%wa=HoeFAwf4+(C@W_twnF- znL7w~z6G!MHZhrh-`BeG4+E_$|CnW6*+0h$zME%V*$h5ogEDk4JimNRrqV|Lq?Ed~ zR%&!Ua-%_}y`)O-`N*!HE4){9xEa~t0#9mMp~qW!X-Q!5p7iQG?;E^>CR@C#ao+_> zN*C{`;oWZDK}(B}0o8ZCo8-Jpa%gLSeQkT!yE5lpFNfC7XRLd=-d*avOLb^(HFBiE z*j}qIfs)G>@20(^zSez`9VsSF9dq%>U1tp3@9jP9o2>7%z=6`|fCHuVz=6`|fdi#K z1r8)%)AM>@p%1XIk+l~Li1y9o`4iwkcWE3dSa0Cqs{bW8kR0vr1`ZAY2M2(I1Hi$7 zE;#tqg@aFsiKF%lJ$);87ft5SNzu$ z_I}CRjzS|VkRzoaV}SO(laROd$RzH?T;y%gh+Xi2vYUm^*!h?b9{LhwZbP6IoO`;{ zA+whx=#XS?r=de_&>{I46{bK}#&c%uMvGpe&!R;HMtV~8i^GzMbyNqP5nR2U3A?Js~S!PYKtQ0D_@-BR3j<4&yi;6S+;4w{47c+uWVrnYIWk=5?it+= zQxM%|WVrn*lLk%i{Qs=Idwf;J)&IY94wnV!hOmpqwONtu7W(P$A(r0jq6Ux=%^H8xeJ$4?T=jF)N7bG`O@hO;6&jU>z?M*vvFQR zo^UM%Tw~qdn<0DF-36{ai+bRf>>LB#dR@z8=SZSVZPr3Zs=tuD9&zpH)OP(8 zbO7L5{=mh-^3voTI}1~GX#S?Y+PkHmd5LT-h&)%C+9FwD4{^VZywqpn*dlCK_zds= zB5fO*g9g=hu{(Ef@(tM!WS`y-yq*Rg;uHQJj;-ny@(LK2@;ulN<#{vtCJ%hmd`u=! z>tlU6evf(mA}31Oe{$hmiTzGE;lzc?wljcp3w=;OvnhljJ~s(_d!>{BYz^cX67|IDm`+v+KMSQVclbH%rRn1Mb{;~(%zmK!T;LO+dy~DzCxcJ zAJ1(2E_WO{(|!-|cW5u8!`6HQd0J;Y{rxpyFB_uvs5NiGL)kvHrf6O-qy8z`U69kW zp;zSey^fqd&ZSq{cIW$I>Z)Gg=FVjcX=e~+YTLt^mE>iSSI_!|569962R^Z0J`H~S ziqyANiZ6m~f3q5JezYx|Y+= z_~0U^?HW?`MYdF}^}0Jvd|x(IJ&XUVFK<#$yhU_&fd48}J)I$ZU2Q@0e^m>o)tw|@RPqbApvN#~5@6Yt#(CSNAkX1d0+nWlVz%@jWX_|eR7lHQ4qc_lvr zk5#gMR5nepsw!tp3YH;DD|YxK{C|j1g?yB}7#(MJD|*up)xSK>DEKoSKPP+g0p3p6v)fr#10@PPg-}zAc~1hxjiVFwem6^Za7#@r7tdZabW(a|!Z!%EEUg)BYm5 zp5(5Tw1@0n$t1|HZA}*XAwDmavTx~I#IZ8Gp#nY@!G@{0I_>yjD)tF>y0ORde8JXX z3BeOHtf1a;e5#1w$@=2NmH{UwKs(q*bK!&7M7K;js`W>8bL_Oyj!m=`*>ND_%5l$V ze2H(<4-@BJHsye8BesB*bRGPkw#3_Ub=Ln) zT)(U>o_$%o@PFl-rG8>tR$p@P%gRZxV!!2^pXpau*mIxk_a8olTn#Ro^QG`TVm{-q z!5o@ARJ%zw`|+dy5- zbt0+!)kYh8Y^r_O&G(baUZR*pn>Y{lPdER}Q1Y=Qw2iT1cM{|5BFzu*8!daRS6~0# zIxoxa3Jn%1g+v^TvdzH;bcGObrbMG4euT*QcI-l#UFVkqR z(>w9DDR{@WF*oxu8E%lS*qrKkTKWlo#Hhp21&yc2x* zxpbhV$T4OgT$vSP3N2-=T}|05$nX!8EwB9$R6f3UIwm(kLz8*_A=^u zxSEFie>=E3R=E1*D>~t-V0$rjg{xi57E(5dGSeRyPgjzc<>j$n)FGojLtbip8*zBM zo3g&~WoBL-oIXrhPqj6LeJ|Fs$GeQDxu-K8+4&{Yw;4Q6Mdxv3`Ymb~e5hD(WkmQT z958b%nSNNw)orto>7jwTLWhpO+JO`Jz~XuYcV`^`J}7wq8oQ-0s%r#;rFk(qNA*Wycn zjd7;qTadH(9-g-%LyuyOliXPk|EtgI8GKRwWxbLdT}{lG1uLAh0!92kE9c_t%-SM3 z_d$0b<4?R7>TVf1SACS6dlk=;Ltmjxa^pv1rUc{V+kEn($hb}8e8FSI3Bl$Ey2`lT zS+EqXV>7>ZhJ6(D17zGalPWX^l5x>>FLz|zCf0@v;6KPLTRO@5WeFWJ%a`bj&bdg| zmyh~w107zyp8Pm@c#S6yBU5|w@S1pecpPo$UOcs>JF?Z6655bFyuzd@_Ck{;f36PU z%FJudfC(i@+bKKQ(F z7OvEfPla<9rK|pSK0JKp5N9sg*Ajk>>tR)9@M~u7-{ki)xb<^nkFBMdJGLH}94s>U zn6kK*{d|LuBekxlEWU^52y1+h`L#!ds%}9hu({LdS#b0@=;?<%FGkMLJDRtn$g_%_}GtUX<_RX4lebiKl5FkPiKV>ebLEJAm89vH>>j5LiD{c zp(-2QLowTt_qpT3;tm@4{qx;}(dVxZ5`QKb%|jl+26iIPA8eT63q}_}(+?yB%^Ksk z4BnkjC)`|m`wsLy-9uCG6lanR&D@2-V*EKlI_*M40fkCGISCb;Yf3AL& zxbFxtC_nsupUB~LeYr!#iYcCmbo`fRVPApIS1?xYWvQxQUZSjfTjyOHteSgGuwvHa zVDx}hCAqZHd@tMnEoswS*1GRg4}G@YKe0;ZS=#WqXx@wtH;49Fs}-wxfPp8PsQKQCI5Ce|69s0)x4(Lv)#N0lvilQ z-US}Zds*<*Z2Jbc?5ir9K>Y`3-)~h;ar6HV`I3#JFFA6vcx?uLJ+dppAGh5|-`c(J zbAKYfaORt1-1l#Fzxi|GLAR8KD9f;?lh@*YYZve6E}rwX&ze)InD7-!3(;4VvgYQK zX7Z~67Sc6`@~tT2XKks}9O@o7`NX7RTh+Owa?LGxnf5T}m8h3N{Xte*<0)WU4PVxM zg@K13I*>N_EqsX5?f&uQ(0vZ`8r?jNyILX}(umpR#h2|tJ?-^KuNK_qpK9Nxy%KoK zeq!61_OX@ER+;*FnsKW?L+DTZ+G6yMchfpzasM&bi8HP>;0*tz7q=<}23!uxyn>rb znS+x`!OJG4-w(x(o97JlWOU?3@P`#IZpc7Kj&d%u!|!NJ zoQ}-g%}sE4Pz~SNg+F539B3)eGI8-v!%NSnFN;^PHW__6UN<>MUmnICU&O!MgbXp2 z{T%HbN=GsE;&o)?$==kPkB;0*|5^sY?~s)PXRL5G{!!5jt#AhVa#1lh3v}h{XiL8D z!KK5Uwz}xYG0p*5=*NovV(+u%GytowP|(jKVd%zk;rg9n6rI(Msf9i2PHzMl55ITUc7E7mfPxGKvk_d zF#j1i#QAl_?~@Ombj+&Sb<(wU=AAgvwD%!=NB6_1oyT89Qsl zt$9}Ecz$ZT4BTs=?+xz$B>n%bw@0b{m&Rf6mys#l?UEX49GPbBjMV)vy0fGHV1n7p zoXWUmOO964_d%?~hHgUvi4Sv@}49uH1xY>ErHgYoHGuMcu!;Rp}1fyXK4u$lia%o`qj!PVgz zKk=x$g_F_YTY%F`byq}opu>Bx`kD)?-_zC!!3x}dmA*N8JaH1-J;dk8bKs#EvS)1_ zKP4ZTC*JP1#Q7 zPk}ED#k@Nvp~}EY`us*K?i@)9`ut?Yfs8fJb@%rcnLA&?$?h4F=<{ij#+^o=-(vLn zYV`T%xkKo6*8XyE?0M=7Ug-0!3$i0Ebylq5&zv85y{j&N0lNH!@Tx5P0vCqg0EWUh z*;0%6@8Cm@y@YXh;GV0?-zK;{GIM!>%j@h6*@z5CGG8S^!Y88^!X_zSGTP7^m&Jdza`ud&w~z5 zT;#et!TGI}Id*~@gC9@0 zA$T0Ve#n1Kqyjx6G6h{}l%p#(p)2VO&Y|+|!TRcM!M2_L;713rAv~K9Jf0-KPp!M> z?v?$D@op}>eaByr)se*u9z_?NbV76Q*b%0G;jCWd+7Z@xb_An0p5Sb{8^iZ{`fBV5 z6C69jweA`1GV(n;g0a=O`)ZyYp&r{vS3AOP)^MYP(U$u1HQJC2`Bjsq*fUKkz4Wu= zyU31U+V8UW_F7y_gpbB_hVB2xreMa^#ip>1F?{AeTy>9qiu0;-G&&#Af^ADa8HzuB zf<0lQs|-II$LX>6J~6VL;DTdT+qKi}Db<#Pj1qRnH^4+32YEl1gXQ zK3@W5{~FUbc=V;7kw)%Zmru>{3z#2lffc?qv)5cdgFPK$YM_5My7q?7c=KfUj(CCp zZQc#V-Xqob#XH-pdwizMud@Q}oC64Ou2Hf~{s8niWZtRRFay%*@`Udh_8#y=y5$kt zcGiq+yV}jG)pwD(ge=BW=gqsW^cidDIJZz&I&uVLMzKjw)7uwp~uX^2*WI zq$}3vai{y>*U;y(?4P@3)$CbDOQ%Nap%3YD$>?%Vs6EE5cdMnV(cd-X$sa^}mp;CM zuGV=Tw)E(r-#ZzT;D2IPC?2X^!b+F4PTZ1Z17S4f$yM%gr!e`AiQwZJ>ptXiwS9=T_s}+d`uKrKiQ)P9ew(^`{oy~leY%xCNp_px!!3K9 z7*cN849Xf@nFOu5>ZjnLe8E!rj%lCyP`+vNdSBS^3u3Mqo!iUbl;Gqa3K*aIh}-s+ zwA~9nrr2HbWl_K9>HK)|<5~5R^^_{tuYNf!Bz4HupS#2G57a?FXzsE z&+tTd-}=2AXYRiXJcO@0Gh*OKOe!lev`ldfqCKsuT;PO0XX0xF+%xK)K0GZhW)ex{HLmu$)pdUKc=I%;BR(n5OOOt z#(nTHoqNv3CzZG=;n4BN#!D`4c%5_Sn^PiQjFsr~z-y~N{IYv5`@6tGymylY?e=ue z_j3+&zwTBM?4`?TE&8@=+uTciv!@Bn{{qbA<0SpmyMuDO#&Uno}0I2>O*#^J_& zzt@ZnUq<>Vn3=l69C#6cjd!sz$CtsE1e5DcS(g2Fd>OoJA!XN4ruoe8V_l*7*yq;K zJP@NQMSbn&_H{1(bIvEG+A~aFIUi162@>JFclJ`r6mU!}X+H=Sw=EH(%%C4rjoDgP*pYJ5lK; zYr`|-eM0+MFMb%`7uJB?lr@vr1YQ;QV7)CVnVD#n%^bqLBeE~*dz;nQbjOwvrG7oU zPwTPPJmSc&{t9`)?yJ@ZE_!W!Q(hl)PMsJ)+Us4DE4^c6$<^$6#b&wVnop{B%UmA- z;hWFkz7wPD0R2&!Vm<_JC$>W$(S<_;8q1$G7M^|dQ|$=9v_1qbaLyWPJz!&-T}W-B7-{k+S&tm`vV!je+^dCw1_jwwaHGl&vLCXV}eM4Wf1E+>GN12bP`3 zkZ$+v7(=>UO5YFtD^!J?zjxA>M0i1_JrlminZ~8ENu6X2;txMDv+(=b$QwGVRqKKrY1(u&Q!csOlS%QCVB$^X2Jl z^2v$fD|wyzifq*33yBOIzaJKWx5eo-TjKY_(s);VU;Cp4Y6sjpvwS*Zo5mQgW2{pd z^Az^Ot|hLdyB~%usyNe!7bF1Vv-@HH8tLFpA?H=43g0Q);U(X7v|F6GsgY!3ZP=Q-y*`)l3F`~M*hoM{_f$BP3uj=ok% zzs^G+O%7ElE@0hA?yWEH7OX$u4>oQ01#1cu!VOIc!FF^}{Hl!4ulCA2`D~&$r?5YF z0z404Pc=9i>IM8HPiET{!_VTW{CAOu_G?~FGB2w0L6+u*IEW@qwV(Jr{i`r>pW?df z?m*qMadClZNAFbDdj?^V>(oXa*d zbrf+l{peYvkm0eV)G*&aW!}Yewoy<0T2G#EMf9os=k>is*d-o<&#{*O9J`2YG-Yo2 zX38bg6a%*cbVTpF8{)oOHq!a-M9$pmyWELZ`7QVTb?$ew$kTW4ck{z>-^JEZki*`h zzI(4*zDngY7~^7KQ-|$^IwdJq7@%bqfFVm@%uK3(f9 zEA~!zYukSye+gDIopk@(&Fs~=W1rg6K03Atn1Kfi)rR}sK%zC;i@otB^;_b`5FoCrS2kT`UjMo4Rqb`{^V}hp?D4UH(i!N6 zgGr~O8!Da4e3J^^A?SZ7<9|DseDGk;3eLhyjzzzg{^rlcuiN;kX4zTPQ*8eQgV1+0 z=NGUKp_IM94EFxU=g=ne(4TtmGd}H03MPXzcG4`R-kdbI7tBwvD$kH7T*@KU9z=?% zn`ZY@s`l{8t-E$~zA%Hh@2Z#Z=O<{L=Ic4E8Yh z9t=dkcknD21RmbLAqspA9GC;aVF5ax2Zy()D>zg`TN;0v_I2Xm@G|2P9R5Z<#o=j> zo5wwk?W1*fV!JngN87ip;h8mO{~#-aaICZm&T4N*ZAO`o4D=JhO>-rh9ptwfqtdm3 zkMb(XGrFj&bH%SKHIg-kW`z&AG2lxw9h!ffeyRP8g!H6ArPk64wWqqoSzJb)O+$&Z zxZJABCeF{GLD`++B(`#9a!_{m;%vo_BTk}XD_9R_YzQRdEBt^}6(t^GfS86R?gDu9 zs`PA~Wglt9vQK|vs4=#;{<6kq6OT(58$8Z%{LWom8Z2A_KEnA& z!S8c$sar;PB6krEffpXjaB%6=qU$2E$q%6~qS0KFX4)4jh3+^bVB(Tynfw$x!=%Lj zaMKi%W`FLUyYMv;d>x)@-M^=oH86@Te9wxA*>m3$zvm7<=DT~@d)Ui1u|Dd$MT&~g z#`%bD?is27Bj+`UcGHe%DUte;)wCXBL#>MPeZ>Tl&CmC+p})QHHUX{s z9{r))?e**e(A3^ezWU&2Gxr(nLjP`+4b7#$bKZ%kL-fuD-pOT;1)QtO_gh!AOrS4} zCqOKTR^)xz%(U*lrBv}Ihhs}5)@w9{Sg+vhp&6WwV;$2y+K0Bgv0mX@(Bl2eu^;VY zPe|?UXldWJh&I&br}XV(`TfVops1#AA^bAcr}Mqj5r+bN`#-=JUa(iZU^xF>Ua+PU zFBk?dcotqahk1uS{RQ|D5!W8DfV^b zS;%=PuCYS?|AhD!HP2dSBr9!)j*<85^Yz=G3jQc&BW)alKD&VrGx_eOinmt<2 z3c=l;#JR}3El5AA3{HJLGyFUF-6V5{JOf#x@Oxo$tZ(RZzzqrf{KaFtwE2cL-VyZl!^coW|h+HibVq-$i_zia`1^xK>R|9ASo zD>E#8J-+X{KU8yZsrpaXkI_Aa{Dn_{Q#a?-$b{9XFmTudGkMk=Lh6fe5Y!IkGs1MBk##S zsjhrVq{BanxjBR!rTY;w>1!@_5)XEt3Ad<+zn%HW3i@h^2MKO2?)EkP<$g9&(@$*g zlDl<|=@aT3`Jb{A%zX{(pzy{b29Z+nl@?&E{iIl0!sAE4^Z5J|*q-FSCEeazHFZBF}5yepeKeE#FJeo~tE^iF&k^eDJ0}-g_%~Xyzu#e1ZsMnX1I~ID zG(c+&?s;=~>~HJWIsQ+^PbwRG;mCCSr0`!EnHll?SL%@&^Cxy|xelM8`f;q4&}w~N zb_81hG@fNI{i3se?BUy5?-jp&w`hX*gcp7{uI=;WH4lQ1VINc+)6uN$PMmQkj;YRF znEJp(_QJK)SKYuZUYygj=gfadnd*A6Ol2?peti8z@gJxDO3I{z*|=l!{F41%>P`Ui}&zF@r=)0cRKLirE&02 zyl|KL)&=ixQs0>4>7?R$!kv8i{UHm5T)e+Qb-=L_Y>>jY_lW@|zrQ;8i_vL_G2zKg z6~q!hj2tWZ^$@rXE+g*|%bfQz8;M0|VvD0Ed9lUsMn{;+ zyQ}yq2K~J5PJHos3AsxH(hntLYVA(wq)+Yb6k8lzFXAkIF0s-zzOH=>NbW=DR39}a z#Tc)4``TB1q;Fa)ys_e!xIdq|y~smGtUAUTBYN~xS2WR@s&Qr5r$~`)xm%4h`p9_b z)yLTraN=V>5Q@Ea6L-wqWEG&3?v0O+Egq3beC$2MM%I`=m4>0Ocp7eYeb77_j(|_1 z;hH?fGj?eB5!Y7x7JW1H%ewXk>q1xh-AUdF;93SvW@%4i$yJ@=KZs_3MIDuQEt^c) zM#>D$g2URc5Y4WSuiu4c*HY%usAyL3jY79Nn^+;=zBqafh+e^2(d$aaBU#~dX`+K( zJJUq#Ddd>R4!vd>dd;#|YyN2ed!*(*7`{`=|DQA8noIo6Q>4$B^$GYY4!EA9!~;1s z0zW`}3=O~C?!*PDA4eJO)t4tnIP+{rH$!LO*&gH)ql>t4T4c`}A)JEGrn~d>HNGnw zXD#z|FEA1uWb5*~zLmF-C!aUvdHhjxGl6`Q2mcWKH78e*SD&tail?VVT3+hR7v(o6 zJaBv{xdYnJr+lU|?cstmeJ*nQoI#A1e~V8lF3WlNwt;L@{VG%WJiu2Vg3#$6Q9GM%+Npm7EKB6%_$lSYZi3g2EKOCm+QlMggl*zh|1q|H)4qzxc#wjK4W<{K8d-UtAc8x-e+&0)qzdpx)nWcs+JU#X-LVUmec< zbb`ww7bgD=9OTO_IK0>ahn$Z1H2>+sLH>z?!(bN<;8}|B*1&LzsRfa($`_~?os{w zkY%N_k$wdIXinvS-!YcCjvuS!bM?XVV@(omO78>T<;OaX_FjS>!TY^tFSoGd>ekh+ zA1gYi-u;71FXDrffVb$e2>B#S*$23EaY4&vnhl&3| zs{WitCK=Z;mK?j*otL4E#ak=P{`C^?ZUW}nsJa7PB1XRzdV`!1Xu2t?wG({vBoMNc2*Xe`!-5;s%@VgxQ zCA9&(vAQVEc!tfGV5*&uS72qpn;VVV^RRI$lKcgke8etJ1|17LC z?BkkW#weOv;=+2WY$niL7kkxNSSv>Ud0>6`1bV*qStv_4G?i|DLvW(~`$*NdX5cM+ zlU$)Od`0EJaFJ5*?#oISvp$fvA}h;&qP^tjOl&5_!1r;E*9z-?(@ll1j*q|t@FT6N zsV*7MJtRT)rK?_VX1Eh1#9vbM}i7KClcK%DZ_}}_8yy)6V`_doJr$J+vPs7{BCI`(^ zCLBDLU5NjWm9o?4(1r4A*aLL^89L6Gt6rQh!#U@`U+ZploCE*g_-mtUm^m;k=P7=$=-O{jgQdg`jBhSz{A5L@bHs;oJ-F{XXTs1KkoJ&p?gc; z%CIV5TI&LFu@g$PFZ*Ot zM~wY`)RPa&HvjD{_fuEzTuWX>^_(4>{Bv5=uI$j%+3)2=(F-=gw+Fl53Q=F@p2fR~ z>&6-H+f3}20`-CXCU~6qo;jP$7?a)iF4ue17yqt+hAMckh;)|r)Tf|xDdpWg+DnHQ z)+2wl4ddK6FpeQBG||>L?l2dudeCMoyhMBHt)@(Mm=ENvYVED_zVSz3|68$r1j`uT zj}CJ7C(D_SZQ|e53p~7SL;Xln`K$xmP4EDnAv~t~d{Zzn{tV2SU?Lkz1^1Q(Hg79s zd~caF&3;oUaD2l&r`j)?RDNEwptYIM+|AJ5P0-*B{Jd_&&#SvxA3FQIOMb!s2snT9 zVu!!IMmw_K=x8?uCAA3UAwl>Lq}>8lz?F1|qW0W`WX=j|)N?I?Rxf{)@IQ~;Z3 zKjxY>IXZ`!2f$TpEbFE8XYgIVyv`kwhnY`f)5jL3`ym(loc#yIJTUxdPhX=K&mMf% zpGx=esvjOM3tsX&)AuW&WrO#XeZsd2F0uAXjpOU?IM&3CLp0Pe4t%{tqo$9;6er-* z`#1o%bPx4YlZqAh)|0$)h*1e(K72>D2~QHrm>;4^?a zSMu$toGVejITHfiCXp{2i}Ie9zhE4H+a(?9ZajeDlzDsdR(IAD z_+&NT6pze@zH5>>lfhimt`nz^wy!$w_yp{vUM}{H7fJQreqecszFw~UF6S*C#WyGJ zn;qwX@dH>RKLFul$GxQZ6P)1O>@N0!D$w!eheJO*zZ2&N;4|kzHr;=Ocld5WfaiQ` z)BS-2e^@&A5Y8~2WXzx1R-A9no3tUCega=4*1Opjy4!>Q9>uiZGZccgK# z*3@MCF#ZSX_c;2kd8|ze#m2BU*YXX?klvor5b{pb=c(+&n|Kh+X^=Z@0~ZbN6njj% zVo&O-UeK+h`d#Zu7d!(j^bOs$EpM0WbIv?8&aq2R zU2erfqe`xx%J|OSt(I!vhl7T`sv6r8`vWVsjds>>O?o5T(3xMo)rtyA9MbEfkiwyViw-V-i+W*|E7-& zM5fb5Za1r6s5-IDI}g;Lb48pb3Sh6RXPgb-*;Jj~8^L|rXL>~@+~v&Gx81qgOTV>_ z>wL&xgimI!usgf<%k2iP+4g8s>FC0{TKK2tW)pdB%wrkzBibCt9C?0q+DmX^9y<5i zX^tMJzUpcpp=-J3>xbrBwC~QL=H>fd9&4rivY#RE(YSV;d33(9o3iEcWqsgd^jG+B zn6htpZLxkRFCW|zj9TFhqnGrJ1faeAZ(}RJ8`%W7<^R2RiJ?9mJb6<>4N&^QvORejrs8_;`y6=yLPMSAj>YLFWvb5e7CRX;I`?vV@n!JUCHE< zStZ{&bD&sl-1kk{1j=-tLgN^>6nx-bK!a~pzq>k@dp$>gB_4M2fhv!=wy;x?&euBqP}Bf2W)`!Up!_A?~2Eq zULJ}m=Bd{9Hu6-Kt@a;+&!fXd4rC6+s}##>Z>K+5_FL||&D`0cy~VEY zl<}SUeCKa&y<@7kh&YqPES-nmm=A2S&>2(trp`^tPy9S(mGZSf&yYVrtc>(q@|(C1 z{6$tkKI%9#i!8Cl>V0AOZui{eV|-8Wmv80oO&`&hl=AMYN@;heQux~|CMABDNs0e! z(ky%}b?4eO$V`)wnF^7aCLuFTL}vO!#V-H6Hw7Qe=6&oRTFw+3+%CI(!g~zp&2yGXJyn4tF7@n6{1%3SRs^dBM&A7!-)eFb03b%go#-lL(_QF59-NDidAq`SBycW)cN&yfz4i(S-4ex2~l>EE)xRV!A-Y_e6)g-Hu zxt=K;+cgnBH;%c?qg~k(5_O*hYw}2Eoo!?sAt0`{Z7@IQy7YUjdgcWiFN!D@L$jOQfH9W?YN)k=)YF~&me zpLpJH1bfP)TLeB{JyY=*wAWITmk^1T7Di5E(>3~8$wlEa#P?eENzW@(S97TM4cv;m znLVkZQ$4Tn<}?G^7v1g=KZXC5z)!2_hja^#1Am$QhgTF6-vKx;!WRly%SJON2b@QT z(4K_W*_}<9S85%$c~`ar@n*I4E1r+IZAtf_FGeR4-Pgk_CxG8NPu~EoO+$}KLUx?? zl@S}Jr}CW09?16-_8rZ(Vn^qMs*Y~9#G zG21@Om_9&G`tUwvgm>}5=I%S70pYHM7L`GlI=~d9j%$U z5IhtUtI5Ds`?_fz`p`egYh~<3$i|Y3$3TNl>?Y3_hq}N`vS2lJRWEQ?=Q7EHFH@#= zYk`gGFC=e!e0^YPzDwCk%C=BuWJUbtELZ0VAg5XAJb}gWa$}l(5q!kJm-e4xTxth6 zuG!Amgsa;~ucA%v#;s}{c3q^E7zecN+~fXz%hh{;*;?Z0;X4xu!B@KY&j_{+g6*@( z5n>)l|8eeb#}_^NY-*%oJL|1rJq&c{UvKaG~&par0 zb~@*YocTI8cJ>^>*vwbwGR@6R<{M5d13 z%A<#2+z-IF9oU28F6^^R|8ndnJH`Y~yRi3D)&}g$fT{dVvw*$U{=io}e^Q55CNusP z-dDY@WeX`gPFXyytR(NKmlp^7Bk^q**gNg-rhZL)S)a4Cb(pe$bhPEn)sN`6&e$Zf z7Z!luh|U7@dPig{NbCgTG<#YHjMMBt0JGM1d<21WCi32(n}?5`z`c=@@syW(_wcb# zo_2WGd)(E!}=ycRnx=%iMX`$FpcaaQM9`0~e|HC;DF7 zq}al#9lGi^^5W%~UclY67s{_fbv+)nUSkI*^GQ2s2b`QtpVpGc0#?u!FBekwbIQa+ zgp1l23dG|h`t14evEA?=>Ex;Qa>k``{=^;28jTOUsbTy-QXcrWS}8QYN-2Eu`=rFa zHSyl+^AeY|-Pa5KB?&#IH~LH;^cv!4FYm`*r9T|xPSODPg14g6wqSc_eK7Zc8yaw9 zxc~i6>`~$$&$>Hv3^vhBeymTG?-1A2x}f65w-mz@n5#UWyujex3mb_6lzTzs#(k8P zaMt1m`XpUhc7!R6Px8_KAy0cTvI9(V^X?|knZI=VYB%pr^0d~lXjh&@u6UE*ccj1~`&@4JsM75M_xVv|$|!s8ar^Ikt=xX$SGzms^%D37_WhZ{n=$;? z+>bVCiapAt$#$km)3K+YKgL;o-gQfev1e@Ro3+Rlke+lnN4lL=#zFTo1c)BM# zuRnXZ_WoLndj;E&e_M-t2iw|vu>SVDRIs-mnG)*`%?z@#8UtTkzaBl}I6lMc6LUEO zm(cBa&Sk6CPxa@lKHU?X>Tc}|B)+?zIRIxl-!#{|`x|GTC=l z1$T4kdpfwScy4d;dkNhACGC5-`y?iYrM-FJPVFcMPfHn>@YGp9GVMKl>wI{6xxvK@dzV`t zkEbI{S-M@-sm#ICp&p(hiyJ&G1yB2ljvgIfi_cNr0tZjmGZ!A7rqOSMqr3YX96baL zevo+HwQcZr`E`G1ThV){Z1k>Px+7k4mdU9abxy%Mb0%725lWEb`B&-3nIFgWC{ zA(LMtehh1f_Ke?R{dkF=VrG1s-<{|zMFLgdY!Lz1$u2dhhB5c z{AxV%I~~rL4$TNv8JgWSAguftFm!l&jy+CuL0&O5tC&mPd}s~}C^IoB3jO=)^Lijl z^@LXj;F*c=&R+0PVks@}9XUgsA<0ql_i96y()|+6$Wrn#ji-Z0_pr`@@0^#InE~!M zv(~)Jy7QtdH_azrt&eZ2Z}ZLG|A6oZ&CZ^57Wo!ylDF=h+|NJSI3 zfb;0Il~I4z@SlPE(RZxK6~*!~%CL`7$J|xGIQ`H;QBP}tQ-*yZuI#)x|IEPfBd$-v zZvzRBYdpg9DF3zod}z`X`>;uq?LU#q?@BoSNB%qYxr1Q&^}ujCu$%@=uLHJIS%;<| z2VJ|Qz0ir>B=}jZMKy_5rN*?M_wfzouGF02@;7-2+6?3(7h%5&e33Y3tXF|E6Nz=2 z*;vuuK6?VP(^zDuw#LL@>yci;=sfF{R{E;`eii>e!LXV(C8G&g<92hwvDji-O0y*y}ol zj4xb{orB9+b}MrsI@5hd@>c|pw@i968Cq0cD0VHl?8r!2c9~!U9>>hulWni$c|H7l zN{fR-t9ahT+9Nm`ygXT`*pmMhxAtr~XYB#j|9N)p+0yk6Al9BO|1WW@Oh3A=J$zGZ z&-bMKH_+X3?gIi(M0+jl)5YA_QSv7e9V&K|=+GVS8ZVyN zrawA#$iCUkw17j0fu7dDik{Z!wb0};(W4uu*9T2X&-rCf$9F;cjC7po=s4-X)1k+7 zdjRt!T>2t;4m}c+$ITmI@)9E76fO3H&ywCEdh}-wKLO9KTp^lFw*%B4WTiBo05?RF zQGdp8r!37*h%0kt!o6|&%ogita`=$D4*k{bt8h?zp1Ma`?OP^Iu|K3-d`{yRy`EWq z1Ms{axK0PY(}44J&>`_`^QJ5T{zfJ&^Jh2Ku}_mqzjyiqjb;3whu%II`E-zf;!@Vd z+3+m+0OkMfm@P6m<{3Rv)2zrX{ zuu@y+Yo{1{tzdqDHY7t@y@|OhesS}}*hKm=cxbQixVcW9KT$_(S}4h?tXZ27ekRSy z-%Y+*3s_6VC#6T7hEH%mMiqOkuLO#HvvlrwP(uIXT047T>u6+uVH)R*i?NND!()nx zt6X$hnOSQy;58cWE%?f*Z_m-b@S&FRSHP=kX9*tI7F^!4mORbP08*`^1@4@9{6l)! zGL?;y+$?(rdJ}oyH1A=%ao;m#%(e8duaW07{Km0i41xccbrV~=&J@)6SB`HKq1M%tEGuC$>WWML{ z;~u%nENrH9uAmb8ex>Z?>CU;M*7AhNQ2HSp(z0Kd>3&}aRCUA_Zdd?VYT$365@^Gv)shgaO<@`}fMg}-=*Q)ef2#1nQUTE@5b zXQ@tp75RpTkbn9}pWtcuh2|-f`97J<`IduygHgs3t?L(TZ|{$fZH_(5^~<$+S8GKr zH1I2Ux#K&-Jt6QD>9{ypnz*CRnONO7l5OPZ<>ZS$5XZ>y2gy9Oz+XJz8>TGHmdq&L zVak{j@qi`diROKZk;Xf+W#+;cPS5EcX&HsBmhVQ7_#=u5R$G`5u7eM#-uvK!`hAH1 zbLqQ$fDI3LH;Z!QY51nY15)g-GA54)XkSnG9dJ|0dwSM8x*tchKi{O;_LoRo!NsqM z?n`wRD1mj{3P+*u525b}^mzmy@G2nr-lkCfYhd zTPKkPPP;O>e2e89#v0Kf3;g2u$O7CiA$wAhFV*wcG51t%U_DvJZ#Morqxii*zh2{h zzzGRCi=XCxN*n&M%AYoUUP9vHe4aP@0*j->r)ubDRc&7o8rx6f6`#dcnFw6PQ}W4U zT{pUcXjk?XpIbIq_03)>_qy|6^>dVmoG96`#m!4s9=5e+e7Ys?$wrWlUk!I}#I9to zT0X9O1rO=}BlYDsVhIEyI!ylNk^u}`N za2eG}^OFbq22c1?hBr0!3r2@c44&HDKNxKv5R`0v&mfonsVlxv%lvI)J*Yk4 z$Tl7gZ!+s=wpl+!kE0F#b3YQ}8_Ivl=bFbfQionC0|uhyM@(6!-7l_8Fw)+y==s|! zXI^^7mCLViC1p4SvoFv+641G_SJ+SVe{a0`&z@IJa>>=T3!rbm)7K-+llY15kXgXE zo+`Y>|?DqUq}H$n5=* z*#{s?4n(F*Mz%~r#!N+KPs1MD-N@{sQ=P@rnjV8rg_)%M$Cqo_qY2k7yC*ztSsA{2wQ~|8$Dre*!>p>tJl#W= zY;Ua#jm^?nM~EI%?G#`s8XZMmlz9?gbk5@h>^^SUC8|FGzi&gs_~*O%7b|}pKC{rV zcw~Z`pR0W2UC~+_|J8oB^5EG`(0eg+(Zsh!^QqAMDfssl-Jxym)r~XfSeA**{_37wtvK*K@h}0eAN04F3Ci(PpiiZ`Ne|9Rm-$ zWiI{d9iy+%X5itf4LRPRG53Pjz$bX3YmCM|76qO`dD${~Jmf z&wizhZ?95t;BO{Pu`5YEdyUqYGs|xU$8G`FW`lFHz`dF1F*l>f+_VIoG%jEC$F%JIM!|y75Ie*6Ytg$SaXLN#{oPT?!I$_@_a83M4`*U@` z^h?-&HAlatFGhYGXjSgy{M<8j3H!WylKtygi)XoYH&a(~M9lyrgRHUcGW=@I;BO>_ z``zZW_e<(X_ROWurly2_-%57!pCrGgKl6%T`zhdE@y_tzi*+NAMN;ilbd_yKE(u0g z_-1V{zcg6STp^ECMasVrtf)?Zd8a>Pht|I@TCd6;-px{*<4QeMuPJ0pJp6Y;Yf^bR=nPTV^>V=Z2Gt!WSc z-}o}-UA%09+Txs&(^iiC40%_nK6ohE&HH{Wc(I$ZQ7QuuUyUyVcMel_iTO^7?IVVz z-W3k{;%&13GR+s#WzZJc`96CHNc zXrV>y#gds8_gK`FmW{7n;Dfj79)}cr4fw8gRkBlWVEk0!ZQ`wKCY`vD@=W_3=FsSx zlwY8EB+t+@y!9kFsq>AJ?8F&p_xX z8G1^AXT!G(yNCDW{OZO?&VSzcgPdn>T%1#WAmy z+mgA&7c1QJ*G2I2Q1vJ4O0a`UK724>Ro=$0>3|h!0v?q|CIw$a&dSIBf*<+7Y5tt) zI{TFy3@tm{3w|xW&CfSr_Og3MOML^1FOV_r%8h1T*8HbE*4-{I3-k`Zd#l4|7E@PpR7oG|t@VYkaql$$GWp2fmFjl|cLmh5FRp#3wtw{w zo<7hg7%NAg>In~P?-%@Nd;d$1K0n~nhRp*nErY+wCKKIgy%KMiX)kB3{tVkp3bq;9 zc|UUPM%VCtN1jW!C&E8!u?uNVy#4V@$dldIhOWhT9No;3*@iN9t^eizTh0047;yCV zH~NYf%@OTU7uYSNZlY-`&0goWWy-*Vm6Z7?dz3S`=G*Y87kD=IVb*rb?4N&kypbQU z5!dD+%T-HPLp}osB~MD0`+D z`D(7wqi=>kXny_FQF`2`chl{EDP>H@%yY8cNP6A>2boXj<8|+Z_(XG@Zq^2$&|Jjx zi7S4G9jBjFa0;ILLishps=CR+_kI1BE?n=(cJPAJ@B$N`L2}+u_(C^+yNSLJUCw?5|P{YH7naaxzfgIBrv70O4JlRUH1%_~

u)o1 z=A7>ECe}7&StG;k=|;-hSf-RUVk+sj1DvPqCp*)m1A&KM*szoN^!Qe~;akb}H)(s2 zmEKs5EO&6lcfhS2`|ECA9(n3(Gcu`YQ+lG#7A!#5%P)RmL%QK{S@uG=e!2ER>F<2^ z*_G!y^SoVyJ9NBD8IRz568S_vN%xpB^S)Bn<}WE_jCU#pHg_mRRtS-H*7ZJLo_iL) z+RQo^^(8e{BfC8ZZnpb+H!hGISE&DqMyC4{{7}#JzQjMHuQwJ+uJiRa|B25&`?VDh zoBQ!927)u``2MzE89`n$GN0CV#i(u^<%`tP#z}ZgJ?l*!=|5dN^5q%>cbLIqXV>FDLu=)nSejCOtzV!|~M)KI>{5ScGNj&TC%J)l7bMC@}Odoy*?ti;~nH+>@$_f!4-wbzNP6qo?a`MbLPsmxk-NzKfK)irek|&p8|Rt zN}qmCelvKlZwB2udSCj(WXc~W&)CyTyZ!paT;j1!0hXo6gLzszbL`&Gi_RmQ!(%Il zn7x|O@J-F@wbapkhe(b5#y1Q6$%{ElJ-f)4om6qa8P6)}XiSmM7*jEQ(wNSOuKmfZ zr=i#=;!2F>UB8vQ+VtNY-$GM2-Toiyc9gkeyNj~2zSe-i!@gP3S;RAB-@wpw=?}(h zoHhPc#2-34)_vg8324!w6=FYuV^0yUz{tOk=K+rtJ9~g9|6>17P(M?Q{43jn=-@Wx zu_oTC6q=r`6nvZM*1d`JC};MxC-_uzb*!Q=7PX0&a&zD?Z*E`tSgsuEY-=K6)`Sb}MA5yOMz6Jc0 zJ?#)U*dc%U9h>-H$P?{NBNaap4jZ|Nafm)EO?jF<$t^c!j7_xpYg3kCYi$H>d++gb)y2Hy=yA#RdiwiR^&LAVVOyJIT?8Hz>jIwY$yLu9 z_+{E!gS8jd1e`q^xc+N=-X56u{7dsrhW&HW4U(M-d+a+lsz37m0Ccc{=wQjn{VB-* zsptV|=mY8KU>W$C^axj=f2~6Qx)=RxI{Mco=wE%%zjSx4g5- z?_0LCCTO8%=O^nV&)1}IMi5wSZ7K||@+X_vLl34{m4*Dq@yq0QBH5~J_hl?DXN~-M zvNeWvbIb{REUOMo3YPg(xPCfw@gCO9ZN#Wu$IqOdVU5b?eGS@JbU8~@;4EaJamD5<2L>)KDm64f6MYw+9aH4f3Hd`cw)R;;MnkY(h$Y+JOce822`++VF2K`NVC zoV-Eb5N&#Gi|?rIJB;5@n*F}}?$eGcTG#`xd zeWi@~J*B|l9i`ChKa@f%J4q!2>Rq#rEx!%?p93A-3O(EcUCc%wo`pU<6MDHBeHhzb zVGm>5(>sId$KaOUVZ-AmCWUu$PvRqtv!3-tcbZ7{HGTco#o>F8ePZs~JH{B#_pa>S zS9xvD@8h(2capge_M%x1AFEWqf!QWxt4*VP`+n@+v;7s29*Sprdq?8RG`EryS5khn%E7t!W$<^GQ9;+=xuBetSo@&1YzB#-&)gA*Jo+_N<@ZywVh8Oo> zz2W@l($nx>)t{pIfJci5YfjZi!Am?i{`qQCFT);3dYgD~sd(@JaB3htI2j(C0@?9+L$?o@1z+s}yiL1O-;+jj8blhE*r|NdkhxM6eWOaXC*W^QXN z3{LZ>8a^!E+XnAFz%RmY9lv6K%HpZ;Va~~pL1!7$JdS;k=O+cH`jg

5Gft!?z8z zMvsHH-T|+D(dEAjiLax+ST6tltIL1C zfQQb-l{XdrNuZgUHvSJ=Hs)Jk}P?yLi3w{-!*5H~S!+`0pur@)hE{ z&|p-27Tl;%3LO?H1!wb>j=UEBTe-pGy$zg6l@5H0dHW^bkRL+_?*+#K-1C(-_)QaI z?WdH>x3UgiQ&hZtLz>~c$@UsIUpz?rGl7R++MqpPmG4=RxE5C)n@(upy{*R8(cKi=%{-CNaX;I#|g z_*75$FT8hTAE!+C22NI17~YFb5#C!r_u7d3Os>P{r5@aG{}=rC9r*9OP*wBh%;3q! ztYAxdc2M{dZO;iF+j;>!_(FK_pi3ne9K!cWcy%RhOO9`0{>6hw!S8CD9Q(h=gC#!* zXC>dA<-z!E0Q;f9wudRle&m*G{i_B4CR66f=b848;>(!Bg_O037lMP|iz{n}KTCgK zN%_YrhkjPXmDe)Q&rnuRUaTj$&H2Gp&KYI{dndNdXg5ZY<0G=0y8ooE>IIgZEmIu5 z!;~GQ%=nVc?G zQ25+fa8YNTW9(T!9v@Hk^J2e>@FS^U z9QywV|EKy>r*A~h%;$eSYvw8VK^0?sP%xV#zQG+J;HGf&G4iw*sl6`I@x5;TD%D-2 zymb4^Zr)1eEl{3A|9ZdJyq^@A;Kpo8LvDS?4^LdAbE>2ZkTaFegVvSKBA+|tvj6@d zcE$`_v?+fx@wu8A==R2@E^rMeGYr{quh7Zl}Ad(^LXT0It^|(dY@HwJ+_L(M<%tbvWUI#GuEI4Un4e_ z*-`cxJz3nq>Eq$yr(A#F*SM!gaQX{(KPmk)X;*%GBWIs?#l@x5U6SR%d%>O&VQZol zPPm)+BU9i})GJY3Y4}ipy95)@Tzf!w38tMXJW!8LmybO<@URs=ImL=ah;i2oUK3n0 z+-Ym*CE;9bc!Qtn7WVyh`A;sRtz6`elh~pacO$@m3m?9{|8(H>9{ncXf?_bv{Dpgm z<0OM5o|N2@ncew) zkGv+g{_k8EwT`f2FO9S+OZb`h*Cy=yS(3y1tEeM6BlindW!c*g9gyw7#O%PARCfg? zG6(9*9%Llp`RTk1gSm>6R&1S7e1LjjcEvw4VN1(xtm zV#x3hwYQRYZlqkX3M>4nErLVE!O`+fTI1bgv<+X1r>lMYoclVG?ca$;8JqU8{)6%9 z`?oN@%i#Os4}XFmzzgj7^V+8vN8py=27fAY9B@qmu4B1(_XyuR`4;h#fH!g9iCJew zOxy6RZuC#{{vm73f@6G>dusHp-=3#^-E*(D&uIrRJEDraQSA@szuLQ{sgO9w%xR(b!AuCM8Z zj9+qw*6%FfDR@av@Vh=4-OaopH@NdM6u5uPIO1UxxQjai254@awS6*mj!>szBrqE! zyy{2%Z{Sqw;BKya&y=~hhJHHplVabY`Jr!G$KRxXF~;1s=)$0xQy0JU4?cLn?8y%6 zACU|gUCrKCc0v^ls>;+G3>~a>=pg#ez{qJ}czUDz-HRwIx6VZ0LFam&{a@}6X}`~n zQzCeAze&@5>5ToEd2^&Wfmi-*RF1Ix%xZ8ya^t|24RSGxEJS4U(S%cX_fA?SNN;?HcNo zNp7M{b$sBD+EJ{^g_M_3Zu;WR*-G+COx=8_+LA%9*ZZk*0xv2$(zVv_U1{!XOZ77q0OVnoZS07_W#oM z?(tDo=l=hmOahaHdqQqRk`R;(h=_oYg3TmB4IrX`g4!NXJT)4vhf?biRKVCah*YN1 zYQYvTRx%r@SU?z+1Bk6yYX$MrR(l9RWs+c(a?5aWzVFZ4d%}R&e*60Vet+!O+Iy|F z_gdIQH8*8ra z=o*MAUz;4%nFfj{ul;gj{TRC%!LjzswJt#hT^sjny5prUpI|*?_A7i_YbVCgtyXuj zCf~qXXe3u2*GcRxj6RpW@mNwuO*Lr_;StKys@|(-Dgj;rxH>=`zgherJzkcv4z$ezk&C1C8u^I zcDkvP>)1L%PgK^^$gqLvFWmQK543}|Y+M!#$kr_2ZMf17@ zPmFhDmq+-HbcriicgX(v)>J3|Ve%#C$S>+7_RpJkU*zBT_8`t!s0g*t{|&1z_E+)m zQ2k*4mfGC5mnw5N1w6&Qt*u49J>4$qU6@nYdxbae70q9cYg2tU-!!rbb5{f2KF!)( zI2`9)*NdN|W8Voa5q)y@U7^wRzc2s#rpg)bKf2|HW-_m%DI2abFYz4{%NWO7DZ5x@ z%<=yulxdtL_dQHGXJ0Rg{Gj)wBu}^QrajQ@56L_ELC4TxoSxfwlWwSc`NCPIqDLbLt4+n6%_dB3g93RlIv!Msyd ztQoWH1+IN-Bjs9K%WmK!=CSFJ#rSfL9q%MFU$2v=_FwMK@o@509_Q1~tDOBlLxb>d zYoAl^yvy*HKtF-Mo7eefUo)pU_GbP`>`l{FU(pa;anNI9H!>gHn{V%E&Oh|EPUl|# zxzc@S4eyxwziI&OlRa>*Jp&kUY=d&_UsE0f2DG=W@|8UIV9ai9c6@Q4<#`4FbZsL~ z$AWFml#;jZ+(@q{R*r< zIl%bb_2C~e&d$`BUh0l%KB@lwui`QoJb1vxgIl`>e|42ZTL<#K@IY}I&c=g6$`nuK zM2*iMossIV0~c~#T=>0fA9)^i;<%72T=0q(aqg7sJDTj`!b8BM=;+zFa6qsME*!fp z3HiA8G%j?+C!{_ehbPCrh4ImMPtgDWTRixo`%WwG7(96LLI)3^+beyi@n9e2iFmMw zXW2PF9dYpBW1hF30}tSv=fnf?{0QUr+XP(r1K-fR?J#ME{Tq`KH;X#|Kf?T4!F+$( zb9T%W`M~T)W;1f1m0WWRv_o)YI{rlf>Kb?E+-@nSe#l80Lw{X7#I_zCELEuz@kC@BI2~VS6gdXgatY$&; zZP5PpAOl^o%b4V{^1e+?D-_ zZUc9?w$O=o(9QMT2mUiV=$_a?OBQ?%dn}Egmw$*4#cgrE1h<=v+>>S3y84D@gYOyk zZKR?#&c1S%eT*@Q`+@AFtQncQxjs2`2)X_UYlrF3pEK=ivg|9E1Iax)EBaO1pnZYF zv1hy3_$NNrF>WuGW$)7X^Uhp&s?O`u{ArI@bD%cF?Q9aCUo-FI*n%bT0i7c)-uW!4 zi5ZrFi*bB4?I;>(a8hv!ddC=}vwTngRa`x-*?idKC?3VJbKsu4chHV|Ctt+QnKhMU z{3nsa@8`OS>l<8IZtkr5AKBD>dHI!?Ob%gn6D|ztb)2_^YuFUJc|w zl4+)AA%|@D1UtGo-H&|9FI$xlP5OgtqgzD2@KE@4o^a6kmx6;XzxNp&WPcd^5e^=v zp2PEztAXpjjDcitjq3w$9V2ryU!y6Dn6f+7O5@Zb_9NZtT+Yv;zy$*9` z#Z?y}H)PmL7#jyRIsXe7KMa2Vm$SxLYb|ic*4I6Fr8~B&qp_U_j%aMxrZ{81Q}2>r zNPfw`8M`CP#O}znzvIrwX1?XDV~HO}|Kt9?@%ebhj7h%T#~m{cTRpv&b?3v8&wJUq zZXNYib}^cdht2zWc8|m|cRn3$#=xy96AE=z}q$M z%vpSmYjBai&LCg3_h!;utqBrnuJlWxNrBSlVykq2vE}c_m^gd1335=ae2M?LUz%ub z+~;5P_OPobi?1urrnf6L5YyDP`9%jd;8k>Bo6w6bzsGh%TpdYG0o3WH?ODi`dQ{ocI|())9MeBLK|G2ShZ4g zCnc^<44DQQDHHi9yFW3dLw;;eWUJxGM>+O>z9|}A5B+}c!Hvrrw9f@i7forTOfV+9 z7wzS4Fz}aW|J8lRtNq=66(Qf=`*u72bGi0zw@iDvaal_=d>(R@b>~IK*GOxx74Enl zolqH)UhF-drH|dM6urFZONYd@)ZT9P4e(YxbY0T_q&JGv=Evo6YqmVXChni_x} zuSyTS!5GSRrn$aTi1=)qV|%Qy=-1`09raUu%b`I%?I)q3@;4O?iQAgg8oJuc_K=Dv z82aMs(e5;5dG>MOs=bUc73~rK7c3o0EMwfo3;x5D<=6)j%8o*p;(D~}DVN_ca1}1` zT9<0yOaD!p_d~O%&;i=iTrb_K6AQ&N?d{vTO|oJqWN!F(kCJ@&7pyVE%)O4FD zISKv<^s9w3oqR^|j ztzd$luU$U;{`%ALj1i~G;A5ND_Drp3mtfYh1Ie`~yZZJ9 zzAbz%#0F9Ec;|XOGj)a#Yv`GgjJmw#NnVq-thzi@hV75?rVaKUG%0fu=o#Ha6ZmyCXh^Pf<@a)>>)gw8cdyGlGA+(xHWS@;#l;qIkH| zxi|kAdP%S!_l3-J9rk^xH`kQD!I>c1e@wl3KxzD+T9Y@|Umj|%@rDi;CxuSjgC#usI%foRwu1k@jG(bu;%pi7d@3Vo)2^}GIt9mZ_DoIpN0uI8 z|7p3GxOLR2!mh;!eb;|paN8T8U5d>x4?m#j+sUC7*ni}DoxYS(rggJL*~=de@}H2N zfq7cL>SBLGEBhmrx%k87+V8t|hy$tb(C&QO4=sHr=b0_e+Gx*&FJSQgpoNmzOWOFvHXtVTYifw$h|CnGlzUSI` z9cN9Ur_z|c2u*&1YZe#l_o1xcO{}@Y(C9YwQ=8Z)_L;bn4xK)U{PiRHVPb(G+sp%A zM5mXN=g{eFdybp8)Z}?XWiFjAbLn)MOQ(Hv9bS8O+&RBHpQ6(R1y0<#IDSbEGx#`l zXs`%-9qrGzf9I^Yb1*i_6nAcFd)&F}M5p16KGN&Zy`Mx*dq+EA zBNwtZF#Dy@Y|-=aDo4)jB66XzvW&7PiKIorn>A69MEY~x=>9}7KIv+HslUo-B_`EZ&e$(X}DzB>%P zT9oO~tL{C6qC-YLE$SRhg+|3?%tco?|IX*%q9Fln(nX7odYt?$^06yRZ?A?Z>rzA8=FK-Esmi1FFMZnU#x*9v6U z4k6a2neyd{u>VXvXotokYqrxp^d6E&q&wZ`b=sW{oJ*z<-TN_pY^QtEheP)=?dy|H z)4eC@dz|h$HqM#$FE z8GPvi-gE_jpnY?&I{{~OHg}x0o5o}5CepvRUN|kACdS(A1?@X7*_8EjVyw-! zVsB^+eUNM~I#CH63&(CDFHZBm>E?agJDX?@p~v)7Oh zMfbX@{Ojo6juGv9ksonpoS}RCs|VvCTcvJFS(6;4%&`TzmU+8|`J2EzUd?=thwfcP zoJ5!I0q1q>U2liJHPc2v_RKBdRXo=#uw9Ypg zy2Cd`>k8AWW=ocxI~RKQL;}4F_jKsp%~_}E-OXpxyK66Z@t=Pt(z}-|C;zYH;|uA~ zJK{66(>r)hJH2DSA(7slbm?8|2bG~Xy;I)V^zK_{(YreD*U-BirNNcZyC+?G$G)DS zbN}I{hlvMq1NZue{`qq2ApiQ<#eR*?>Q#gN;rdIUcWJ0OSYcRw&RCEHFWZKrp@iRj(krVQU5;H@IrIPw~Qr zi9X=JzSHyMsnqHtz&$5;|EBb z8WX!_HhzL_$ncx+*~{VHtmV8pH4XR#HVlG~W#C6LBoOwK+QacD#RpJzo*NkmFZ86P z&-J9$RN^CeVGnC~EAz9uED&ye%nGl@Cos~lW2j|FXa94qAK)#VC8>9}x_*F*@X^*j z?zh~0`3Kgq|0MrF?c-igzI+3VU7JiQjQ$ zn!U;cPQ({(cJn?`9y$ThyKlL9A1KeP`!nq+Zr(e}Ll-JNb9B=fhJLF3U z?y>G)AAv?!DTOxrl#T=+9t(s=ab{|RcS@Ul{nX!|0IQOfB_D5>A2s#zAAnC(@*VuF z(a};5n>Ff5){su47Wqs%k*ACwc2D~O$`p5TyY|`0_dT%5#B|O#duN&UJ@InsOYX+E z;*9cM_FeIEbOPUU<3d)uc~7H5fL|_-ulrf2e_>UP3a-Q^O!_*(g~sgEoNs{_H-R5F zf+shCFH_+^Q_#y@4<22|zV~E&vpX7H$ydG?7-`lV()|`OPv)D9+Y~crf#`K+K8cyA zl(pZrN*VVFN|^`2lGXx4NKHIw&S;c;FZga*6$r(7+zw))ZK=&_duE2#-kinWaZKo- ztEatO-vGDf5o1xfX>BNvdbHo4lO0@B>F}LVssqlgz`kz9tfXC8F~@G@Qsn~^h2+0d z83@bn@XxM}XAou56EuwrL6`n{i=`#Xt6#vP}D~#4_Mp{NvB2te5?r z#4=$1Vaj%qS6{1GX#8h_<8$~&Q$NGLF|j^#{2KKg`k!r2N-SeuKc?(|$gA?^ER=o^ zTTQ2L@R|Q&EVpXU5qycv07q&c)t;l{J3e6EWk=C>G~%QFZ;Yz z%+Jco>o@h0?zj4z)XA`azP#E&XH#JEm|cX)yg%yDQyrGGVX2cNBcb<}ekukat(nBwn5oTE>- z&mYw~1AoNPvwAjx!_+Oc-RYkgh$Oe z4>|tp`IO{)#og7J_NV20$NmmH*cp!%BHs1hZxnYI{XRTm!%cy382*B6Jha5y)5N~< zrM{Qfi*X2B*03Kl7Lv>K{0sQQz2L?qu0CA(T*Rkd|324xuBBY7k&DKW9-D$6J3Q(b zyg~1ld9!PXJv7+{ug1E0(^9SP?Uk)M`zSgSd_MQjjSGaYVvjB~L3x}jj*sP6BjP=~UyTta%mZB(W}0S?Exp^mgt;`2Xr%pX}J71$?`v9v%9D;a&Z$ z>Ap~la38y-UvZwU-eX_K*flLeb{0Q;(#^NXH~bKrDDlIek{`fMO8oFK?y>DLc1@3R zk8aWM!v)+I%1){nJjO5HE4X2e(Kv)Mzd>i=u7I1nK@OmR~djtGp zD*R#!@O(Y|;yV1v9exr2UMnC5O zRS$%g^^(qsG1WQ@JKpubVGQ(r`KZodz9QgiFX~nAzGwG3bl)252)tVOE1H~2AHG5l zKKf>GFZ8yQGWLAl5s&uL=Q)QS*sXa9@Si2TuYW}NS1~-bd6oFE(;jr@{3qXDNFCXt zINxyUZim~e8v(GMv=bw1+zTHbqd8Ylnv&-T6 zBgONX+ZxK$M%Nauo#z|9V7?W+nLdg~k6`Z2JTk7|HLwwg-l>#vze6eWP_2}C`Hs@z z@C>EQ+bv3=`_q+z^H-4G4_#cDUd~z}snrWEx3HfV=h=1Os^qh0kk7OSp5p4?Z_xXi zL;R;3b_YsRq2vBgb})@}?4lf<**mkm`+}v*;N^3+cMet;OTWL&==a0r4nMEr8}ggk zl9V)4^KPYHGuvm)pIGR1)~?OKu=Y=+ue)Ogdj^$O5SnmDmAg;i<6piPa)ajI!925` zecJu@V12vR3Td2{<^zMkP8I2jNlCkWryRMV#Qi^Zkn2}Zfu8Ou_REG;@My2<>rV@? z?&>$eZ`YsiPh+iVue!h=fw!!#@8^#IKSoZV-5csl&X5yUZ1fL&?)m-$?NtK?HdJQ4 zB0O!V&Dx~#`>m_TOy%FAB~2Z%(||WV{kWI$kcp*SI+TYf`NT_04z;y*K&JCKryUy5AJMx7P!1M)Qqrl)dP}^VR37 z2b|qXy^WNuci~#+-K&jfd=LD8SZ$(A@IBJ4W7-7lK112F}%9} ziZc0AHmELX$+_wR=O0ryNSXeTWIbYZh?g_IEyZI(W8@cI+}T+Z&oXP`wPrtN5Nl#& zm?r${_HCEE(#y8MMa`GaL%p9l(|x5ocT4dxb@r%x**}r*Dr;cHmVKQ6I`fE+r#p`i zk{9pa3BGw||9Fr7^`*_XQZKQ8U!1Eh{Tof)8>m~)8piy$t1}mq_5Kuhd`KJ5^*(t2 zFz=6{uHH@J3~kN*DV|Hn*Zz@UzqtlpQ@coeNNdd=(pSbO)7}L9I^%<#72~pU`}NKq z(sLR=&Q=4*B+H#JJ{QD+GvkzJS14ty0!o4XZz`n^qe%Z~;t5JVOF?ErKI>N=vW0Kh zHaxd3P`U> zCycKj=c(5IekEGMSWQfv(G;hM}f zgsT@<8@dI>yMKjLvPU&~h$u2x7`*~|hoQ$NA&=4jjc!~b$zxT-QSzXlwpr@^EN6^vJrSX z+|B!~@{Fz_-@e4nGyhG{H5@g%27HB}CD-8ZBpb1KOuBRF8X8<(L$#}GC{fxEImXpB zG`PBkYEm2AY_7jDRD!Od1)q1t6cT^ifnHj&jPT>XfOqjE$tp(g0N%ZC^wvG?+bL7b zBCqrjw67UF@ewP>iCvUw-yAQO9)fZovFVz@lPovh@D1^D=_4eool)M)zBXPieFSt_ z=R`Q)&bP<9Wltj`HHg0slJSY$ytQ6U1kkVfGhmtyahp~>{L3TG~$iR&G7-YN_ z?`b`Iwe;-mItYB#I*twPboJVO`Hpn#g~+Nu$By{@hlU2XCIzCW7!zcToxYpTldQ4R zpOPPJVce#9l4cfh*4&P!+~6C1@`66jq?7%^{4z(r>8AF9%ZrGRzUfb;w<%xUX@-u@PJMJ*?wXUt(sjFkSgc=5&WD1UrlFP&6+ zhpzbOXK)V;NhUc%XMv95GjtRO&{1%}^=AAqHa8EDzQVIPMy!i6`f!WjL~!KEUFbXb z!0s$hVNiZSUzxAE6eDLZLDtY*{lJBT@44xBNt^kOfjLj=LVxp|-IuZ-)HsPVzg(N( zH(OnO3_R*}ct}&Nqc5p-`Q>j7U+HOIOe$V!bQsLJc;qrumTCJF%YX~<#3iOI!|sz< z*3aRMi%c13t|XQfJG}5ml?l!W>V)5EE#?aKmPA^qWP;wC~xM{P@17<>nQTCqIAi^fB5gLyVth4aMH$R{ciY@B9c%U$ zP!^y+*l4w;0-IlH>&cEr$NU~JaC}yBs7`QtTT1BotPY{bZ5>06hpga>?m6o3(=M&! z(4VXqZ@bUc>Fy+-H4d>pgN-$>uex5j~g#A$TSX~5su&`5N5jo$gh$HvE!HK^WCrSFdH z+ROeSZDqe9Z1j+aeMWYj44oGIsqW9|Lkt^K`3`Gs2_ zvX@tEGub!^4-_-lh2wTScsIoP?fBiwZ@bQ$RZ|a~y^LJ7+}o?hoW;P|3p$5^vlos( z!~Msczwj{UFx<+;dU3Mei{T>}Va`Lnu=z*sYk=<*pvNmGPxzgtJmk-U`*Oc0zw2~& zzv@-W^}cBGsedCDY~=ZrP&4|lmMK=a!Id-DG&{cMD^3wt5IOU%Q!%5%Tjl0Ao{SkC z-kOs!BWJE8zmPoX@P5Vp))O%!XD;J@I`@(@pXA=+zF`u3M*XaC59UZZ3gOOLm-h}J zPx}GFkJWD81*&Ic$UOU5H?K%}MuyC>SGsxm$}>I#J?*F5ylj0B|14xd6ZcB=-1ulA z=lw;oE8#(PO5rzEO5r2$04LV4`~Zea{@S=K0-WaVx(9o#47*w11aAtg+*Zx6^lOp< zOYob_H8xwBb|ZC6Iddvl{)lJ6yU~4-S3UP$d4Tq6^5zLQ(Om^qZ5PVp{Jm-n>) z=$1F}?pJ8RsMK~j?H7!(fdS;Mx4D-t=`E%7?@guj_YI}l@OPzM15wTOEO5mq8R~b1 zjtsS?DA+>ZrAsdI7`n96SJc(9yW&}L)S$~8di_884&%w1#Q0-A>7JpzN%@SM_JbN& zBQ<-oO?=5?voih7%*Tm-J^ixn*MD02W%*mt@hIL>>ws*3b8(Iz99V4miFb6Hb?WGD z_!i^*75Qmn9%tS3;;ftefsxPg2{>eh4PU~ht6Daml993JlKyqC#+HBSzSh0>b+&qx zc&uT~zvY;)F#4_gdR>}gVtf&2_ z#4_gmVakkr4xGN8SeEVRbF|iL>Jcz)6ixn=X9MGLc?KA5uJ@sPwq(n&*pe(mtl`6z z=wFKCv4)8Y1UzaSF9^CRpQmSQw~exapO`DZd|EqiRC9(^%HD67Z=DgRGPYgxz6Uz;4EB67 z+4Jd%?mP<}dN#WB9Qs9E;yHQ53+nJsZ4(qn=tzQX!cpjb+!s7i?wQ(vFF1ODJo+^Y z9l^J_F6F}5W&H{45#HmHZs19-BNKd~FnucJUiCLj811h~>WQsE&zf575(@AWuabQN zbl)et1nB-B&;=-7e5>el8FO;46&`}V0NuNZ0bdz$Vu#Oj^#SAA!_{8PWp4gfS05mo zghQ<1r4zW6GU%?M)kCQN2N;=UMhBH*cfzj7}ia9`5G-R(Xb=_p&c>^VTR&^t>_<9q8uizxd&Y4#6wt z!sA7|O)iY%{ts>68DG;+*P8y_LFOE1t-<>+9)g=g zA3Cv$rH^h@p&AAbLxBs1p8I}k#u^^vzvsVyI>O^ysJ%ALrTo8YxHoc$;M`r?kI>p4zVv`=&-3nn z#}8PxXp;T4|F1PsoL9YUXnMX~N-AEZvbY_@Go~!dz9g{>SQDT6r76p_CA+o12Rw>T zNj{gH-Z!xf92K8>j6Bm%%ewSko?qnB`dYY|lUSerJwsV7d51~g=MtUhnpg+E_%X0P zn>@YmODK!`<9!T%eoy7<7kpLx?*e$FXhg*8>+gdH`n0D1!1W24W<}qS@11yHKlUKd z2{qNj11rVjv+X^=*CEDNGNI0;x{+}fA4FgCMMc}i)XBh?#o>ca%>8Wp4(32}QcWtm zW%&mC=?`>?{i;CpuitQB`gYX=rsp-sHnihB^L8Woo`1n72k^c4T0fCjmYVf`1@X@; z$3DWFyz5`ZXOO?JZ1qmU4-df)bq<|u^`h{@-P?PT*Wcxm6(Qx##JFSyx!E#rx@WQ8# zt@5!+#9!e5pYkkO{9~oSd6QCLsnMjF_U}nWvyFW-YwlBX{&oC?NK!^?CA>*%68s!Z zUral;b_y=xd-3ymEz@`R{54>8ANB7+|D^GJgL<+@NaRQ68#~gC-r+4^Q2b7C-{iil z7$94J?aW7*_q8_G{ZW;4=*eg&!x?;CySKSe(4Gr!$ z8iAX(4?Fvz&6{Jya}i!uxjNzH)RCS@b~jpUHm#G*Ec<(I9Ecy0CwNLG6>St!DW<{X;FULf-$ns&1(%d&kCG5 zv?kLoqyHV@VX=m4gjx=OL zv!Ra-*b+&Gi+w_S4gTlb9Li&8LXPQ=qbEA-=4FsNdIRRq&Fi8(qc_O4Tim=<OSx!?bd*nnSM7dIE{-oE zPvf5M;vBI#+hdCy1^1-y@5@-tqTIoA?D1*)X!5>m-sSvwaLd7S${4>}DZ9;-<=ICP z%NXN_DVuJ}at(iIf6tXuZlEj?&xe8MPa8aU;tr=m2LuuV46Ac=K@m-Rz3>xdNA6CGGz z6N_`ZCm7q_c~_h!=)5)QPi5<`*lLH6AJ~Ji4S(@VW`!4XhG<-th#*TG_I5J<0CS)D z@(Jk~4$zJjtV!43G~T~M>m*mNQx2ZhJ?NRV09o8Wb6~K4IX3*9aXBr;nt6w(F3;}? zgbPyxv%90qQ7nsZd6wi!53!>|GO|eW;3RMnTer;aEt@sB$4GcX* zS`WNbl0L&cC6hi2to|My{VHJN$E2$#1;XRlH@TQAg=L#KA!ztfZ{jzhJ3e>3;K``=tvjgFK#TpM4rA|tG|3cCg8 zyYkLGA3OTU2F^bXQRZi^nGe~HwifcQ{C`qj#F_Gv=YXT&sX12yTUP*Mmji2~fw@Z7 zsuiqN%UP@LxKr^0cii7?Sq#~2K01_Y?w504#J%tS^kvhy--fM4^9*>#tU&lnZ}%D@`*36b;iWAo})VjTE=;vCG}Rc zoOUY)C*I7_4^ylq?0iM{FPt1YP=tq2?*tv*~2)S&n%-Bb#@%%NWN-)+?fee{<>TRPyADd*mkN_q0E7^CyxYm#N=z z^Tz9a`Xkx%EjO=9d5@E~41ZL<^SYZ~q5Q|J=ts=`2IQ z9}s#G$tVvORFxPV3THl`6M1pjkxKmhq=&f)`vB^QUmmGM2P2+2mpqlN3q-pllo|P( z@l7S~1AGN_X0yXLd)djRUUus(j$Oo=@R*#@?mw|>pQ|6IjU7$@uc6j` zN8jA5OpG0Dyut+ou+aq8Ot~NZH8^KzT(ms806RN#-@PKLb8|(nO+D<3=kt!Ut`oq4;|}E5JePp!h^=u5=IBLdjSE+vCDJuvs^F+$l3yr!goX4mx$LkFC2` zct@^T;r#|z=^9{#D~qcqS0+~mSC2KZ0#7v7iz}Ne!xP>4C-%g!VPamx4aC4P{~^w? zVk|V&Chd~VO=Q(%_Gm0(qiHRjXE)u;Ij+!Rt?56hXPs0qHgr)6ZJA+(r?b8+q`#4I zz#DYgeDA(HN_UW7(hq%%=c;fu_YcD#uVrjvU1>);S4U6u;~CmRbpOG3_gxu?j&S!Y z9;L6si;pW4>P(^z{~6@gsnvKu13oTJs56B+6VPS5b!Mv$bVhXe$RyUDPl6Ntt3>Gx zaN`Nq!SWf}+(&yYi~So)f|{oi_EpUD?}~$u!^>vf(lvO;EuP?5&*MuqpW271u0TiO zdBp6$ynU%7SN(_E?(3C88wA7Ga32A#@Q|5V-k13G%dBt=xvIK}^J(~J(YqG}Pcb$c zL)Pu<4}M9UFYMr+WgqlB%@_*4q4Y+#b$|xcdUsjWDPdlHMIYp8&%Sw|FW7PjNyM{YX@BI5Dvz%}xD-Ct!3yq4 z%5L3uJU01ASw;BR$cnIh%Y<8^v3nVlxA@Re6Oq`4sOKUSSP2aB(O6TYAV-<=Kalm4|y$>aZ4-Gd)SS5AIbAA__}Yw;nD-kuo0mD9&GaC_VXVMbMX5+^hb2*Ka{%oJym^#w#1-CF=&nO z`u10?U<)`u4V+gS{F1k}_LgYO#{ekwO;=HbG;plJAE;1szeoGV%*dIW#+ zz>nhdW9p8`m0o|b@8)j7Kf@Q}WhQTLH?!w#`ZbL6Q0arAk-zXe{Tk*E{z32X5A*+b z^MW7lHL;G?`rhazU;4HFnZ1JjNyl!;51QC@|Cw*-?H$n3_ zxro#zhh%S2!u!4(&U3z(dS3fDE7nJx&dL_U)Z2VsuoL;tJH4HE&hPEKLm%4b=HPk3 zX^e-SBgLHW1AmQ7W1oP1>qXeb7P9Ym&CgnsO2zvON1S3!^0T@`BScV4KOF%rJs4y^P~0gqDZ@;+N0 zh<-MtcgW>sc^cbEtiPI914DQ7ZgD|qF0iTjmyE}{s5|r;w>Rhv?H1hq1DX?ulSanV z+%snL;`hi&D|N5;JL~X-_kFpp(>uM&!B5_SFODWJpuFtUdAE``lDs3Nf8lzCOR=gnroYsf^8O+EtbJ7R z)Fi=n*YfB-#z)`Pe+KZr$|HPNJj2vUKV3)v?L(dSNW)xDaox)$IsZB24Mz`x>^4$+ zG1+!DbtOyuIow{-<`_{5j7 zp^*#zl2`R*$Kl7JEfK-*hdH5A-i^L(MRzF1!|&J=s+@OOMx(YaY@+7zn&n^W<(twf30B9^WeoB3wKxw<&{b=vtUh>L-n?`i> zrQl=fW4(h7qCo>$8$rWm7qSN0YV6>^VVirrGfp0ehcy4Wy2wT^cZZr6f;ia?( z+zotpBj1oNYnGA4GFl6Z*k7MLtBp3V-#dHOrljtHkoRX-ggT@m171ek;aA&}k*m-t ziylZeg?5C$5Dd`gh7sf=%lV)M)u%cRmmC=3-&m|2l9yvcFZsXtPSL;kPGl2ycM0FozoZir-!DPV>W`dd=ofO9A2|!z zx+ZP#0;3m8HnJM@nm%eCni-E~#vUHMxTrcixPQ;-KB#Z!>PKXKZ0NcT9WT{fFJz93 z;a|oM2wCe0@F6;%W@u)4xDHwP2gG?w8&Ta`H1j_8frVp=i!A#4dLKt7jq+}K!aKL~ zPP+TfFOnVEElOJ*d!*B{+rpB#?B>{ff9%JWbh|IuRPV#@T{Je={u%Q<9XOm)Vhwfr zSr`=U{`_r6w`zDIXIGS*9|&n3ay#@vc_n+3qLOfS;VF9=Q;)x2Jk*c0-pO-ot*A%X|Jud8}Vl zC(due$}{}N*_RUCe@1zR-#B@0@ZV^DMo_LGW5|s+hW>Ns4cE27CZhoq-R*?;&9DT<~7~%h=G-d`Gg*ioe8+zGXV8 zSresHL>nHDZFrrv%Go@mv2yDU z1}oE^Y~~r>J1!6v|2bR67)>5>;b|FQhRfeOp62g;8H@hZ)4XkhcDM6$=4~`(#VTWN zUPvrsZf>P4UuEm?%S|kM!hwaJl$1_aOBRg0ZbeltAIZp-NkQx2W1}L=$$uG{ZNJAnC-Uh3 z(l|4g;wcBv+ZeeU9{oK0dMot*8saR8CVdT$UW`?U;nCM?+~Lu~pphH?Q79g5_&^5q zRy_JO;K$%1?IzCb5-X8M6MNk7=-q}#FD^QRM`zdz{})~iUk(8C2LC^E@t^#5`Cs?I zMh5Gzz3^f7gBMpM7a1D%2=v0xUii0U!Q;SG4yojxriD&Cs_!s2Yh64jBi+jTR^vrM zpCucpviB`35RTKu%aCW)7en9v8XLMp{-E%`2b2mv8Gn8I1=^&seO@W>xJD`TXSGt` zE~KtHoD^4eo<_WEq$e`=98s(lGV3-*o+*CDeWp71~Hk9qK)<=isPf4PHtV^N`%T~PJ!uyIn)T42;LR1DU~k42 zSwCEO);lq8N8%l3vA>edK1&Y!ExCLTd)Ya}^xEEmGd7(#UGja9-LrvrzmZ?jW#_fQ zOV?WW4-Ovx8@3GWVF5>_)$Cn0Twq;Gtc$_7xa}OyIB7mpSKe9Lj0~wgdFh=}S6*0Z zzL9)suvyuzV?=jXU z@7mG$UU!PSA(~Yrgd7_nOK%TDY16`$G@b}c+8Vga|^U6b}D6N3F$SZYnFANeRtbnf? z2s}#M>)O^Io!y+97|RjtHf#`X_;=BxO7^Ih}^aMSHIzv6j&Qv}y+l#q{KE9L3Y{z^lfukr#4oWms+vd(3AY9Bky>KP59bJP92N2 z>%4-%2&I8o+3?8N!iB!e6bqwAk3cLoEJC{r7CC!FF>DSb-)qnCX7IM?kCz00&c6zh z*=u0mJJJvSNxS5`8?8wSJsMGK&S;NtO>O@fb*Hly2+;1vFJr^%-2XWLASMC-c!+;Q`F@!^lSxe*@Y178<_uYPDq)%m_C8vsk-2MIcl-;71TT%ZfjMX{K-NRw*$k{G2F!8ZR~(UOO-1OxNx$i~ zO*Y2UKZwb8+{hkU>lr(E=GM&X$-%+@rmpzGU*Oe(LBXPdLt@NIPPvux70l7seq+?o z;BC}V-AV4Ze$Th`Jb~xPA+D{FpmDB`qtaZ#sX!K?V@kNAFKJ=HmZzdn6pJvt=#5o6Fv z|7M=(+dFePKd^~?^SLKeW@;~e+9-GfDgR&VXJ0cF+mTaqwh&v)3aWl69shIem2BoU zjkML<-!4lT*E$Az8vn1C_67K#HI$fsu1!OLf8GS_j!Q{hcqiXp6uu&gK4bk{Pv_Q^ z$f?SEbpD9oqJe?XZ#?PtMILLz0{X0d+J)f`(XW)pV(YiCwv_&mxRb5(NCz-is<*Jb ze`yVA4f`v1LNjY_*FHgd&7#2z^NThpQ-zlR-J?3%h7#@)|+*oultUz9^Dt} zKDBiv@b2rY`?S__-6JPLuh#pLv2%l;`h3{Qkw&~p{mr<~9~SI?*X7Z%@a6t@r9{&& z##WvCwC7Sn?3X$D$Z7^gJpNz_afMUgt91DId)}zzDb3ZXIopg(lPEtOLI#mM3ZGx= zd!eSZ0DB+lhRt*HxNrmUxi8{gN3*EZ+qkyS47z}Tv+#u4$-<_vX8=EWZf@2L=)vW@N}YJ-Sq1m z@~@RU`gmNfa^=mnzH8lmH9s^ssQnCI%{X^YZZt4c-Tjfgv=r-FA3ksC^El_?RKZLi zFr~e_#^KBJ^qu3ky^NfK4sl8na*G!k(22`;6npTvJ(tdpipTsX5Cyhf@IP$3)-jgS zDZF6Pp7!&krv2by19WD(aO(b1!T%y38E=DmzWB1>ojh*^o_OZmIxASsGrVE_rHp}W z<+Z+0d_rS$4DE4rf!UlN+SA^TzGU+yjz6AsoR9IY+OWu^tfx$>m>8q!PbK}Tpnv7a zVZSr5W!mt?Wx;XOvrfh~nD!1H6ih|-Sa2dX%m@Bek(zo9-wc*hZ|m`x+Pc_!xGZ=X z&+jl#UpH=vF>bYf8~7)+54|vZUpe(f!Q2T$77nkJPCwh8p>HD_e2`>Czs>!nwH4%N z*}7jr{)^;K;eOcmO7eT!x}Qfry6@<<+?Vj39nvj(Gy40wf>ZM=qR_SVzHOD@u63}H zHR0EdgZRYPm7To%CC1CcxFs=uoHa55m~i6($)-n28&8*Wjrtgu*TH45HYER<-+T8XW1Ezdl)1DXO-@1$U zOu1=KwtYSAQEZSrpS4T2)bV;VsHgR|=!eqLy~VheovBBvjVkTEkzZZ52C58J(x>FdCnP4EvZVgU;Q?m2a_Er*UTfeZbgW z__pDV+c-ZN9rQGCS9{b+=$nz<*QYVo90=N9x=`_0$q$;BtNQ*S?@{!io$kA%4LjTQ ziw0Meu7TDpD95h=8l~^2{V>qh0N3NU9c(1h%c+GmaU<7il>ISf7Q{AT=dpJ`kGuhNJ_pdqi&AsL1Qy-Zv`*fW@ zC)9byxoluBVDMzl@ z$n!swE0CL{tH8g$S1?h&kUWtnU!?KOdB}HJCAarZM_;r6SwV6{`uSEUl|3xZ;aixR z%6?5rHT67qotjet9_Z}1?RRSIMN=A?`_G2Fg3YW4`|Tv^z}L51!MglsuD(6L{_3&$ zPhUNNap;8Z)9}Bqk*kUl`ak!q{?FlAJUrh2d;eAc;RB}s+j<49lmPv^vv*NRO>gMj zT~>K-^{sej$AQ>W%s(zgowQjV|cYwdm9PajL@Yh<6N)I&R)i^JHd zd!PeJ&;u`YA(?R@hTR-&)oW#^{taWNE*{fRTv5vTz2Vh!uJLaw!zLYD;Yg(?{L-xP z{`LDW53O50#{cp-;-sys3_bsHMd*d?<)P3`oGGXEOuHT|x6On+&{q+ zKFOJO(HT8LpH0dQ9qE`CI`DZ`=!=(ohCaWNxWrr71BDkJ<$RaJ-rfl^RR?V5owJpW zdsFRKp8WEOW@I4kp?|_2%BKmk(3jAgR`ZRX*#8qVcFUB|!HrY-$94Sc`q1IGS!>Oh z7&^&0ZTlybhg+aWvZaL2OswnZ*ur~RzZ^&Z{T?y4R&qVW^?j}zxJGcLQ>OYcE45}N z@nEZydKvrnDxWp%LTuPObNwB?-`iYkxlmB7zXSc>B(9NM%e{Fu5qOIE1~^~Gz2?eS z^m@Audm}tV<$JzCIrmQa`|UdaM(EoUbJs14CG~EthAxym_*Y|}?%_W2fOEeRy3yd? zmvirXaJQ+m7oM}@0q1@`_jM2a!Q=<{rkO+5q+0V$7ah7VIT+u+-Tr4M9#8{*Vhfn- zMBQE?-|x*{r`mk3^1XfJ>ywQg`!}#-zXI6192gu8ELH-O74RnE=<5#rKA5qr414dJ zvG?BbK*q8d>opJebq}O1+rs@y^b*xpht^8+=2|Hx9xUtL*6rjMKA5-cb}RYK#$^^h zjH#_94fA1|Coi|+}B#d^QYdfp^vk>gqpf`4t-gR4rfa?Fq328 zQt;WjoVWwnh41}QO36g^2~9Vt4i$ad0fQueYMGDu%1MMt&jrX=C&&N!}ym`|;1 zxAoKbc}a~Fh547M&8{OarTw2BCqjr^nuL5AKknw$un!Z-NcJ=^D4+Q zds@(wSKYkJ^zM(POU$)5yLls&w-ESYE^i{9O?8a(bD2Bvf5SM%kUsU`Zt#CY73=n(h;mX4?BqDjXo6e(y2Q zJ?-C-x^!h{BHfveJ?&ML9c2s-x2A@UjKdam0(S1>xUS@?iFO*?$NvD^jrHap{{z|Y z8M_D7!=7=Q-f`cT{ATX4;ggK$@J05;^qxY+CGz z706loc5@%vDVsmoVMlv2s>lh zcGl=0tD#p3W3GS29wV+1_Pu-gk8Jm~k8A3<_O$Uj@W7o#*wdPE#-29r8>073UfiCR z?@yzh($AW9Vo%$`x5r{n8@ChQ=(gz!wF!I_f73pOY#DaocX8tLw2*MCWElAPHgXa% z6q*N34YkZ7wu;}ktabI(A^$+=(Y+208c80$LE#c)4%)If6T9K>0|)P5BUFfwL9E_W z8kvzAlwV||HZ|B>&s7Y)uOywM`{Ymr*lpgfdv7SRj}-ebJaSMYx1iZRiEfQmmYN24a7M?qpwCdGzCv z<MK5-N-B}d~`VXd_OX)lb>;jDh{7`)S-Hx zzh%|+Iu|M|{uCMD4YjNzzE$zmQ0suCP|Nem^MqQM3*iX%rT&N)-&bTe@t%cmId(o8 z^H?k`GyxeyV~(yl^Wzz5!6TE#hs3jFD za|6-)-8ws|(|D7GFH1rlc=Eo-0?}D+o!6-&-!IL5+Y~GM=kuBCo1mMdbGRCZqz0e* z#QEnd`d+;i4fLfBd_U6a2_O0#+hWFo{Q<+nBgc?~PjIiXi;+V2(G_EZj2&{vq;MoX zX|rHMaZ%Vq-Knz}BVCii%^8#l#r=<#*sKIhQ~+nGpeSIsn{ZqB^}%Ik-P)= zA01%458#V*fbm^!;Qf)j<>22??2^r#;HxAVg1wli@bYHSE7}{kO;(&$_S?cY-fCj{ zn|aV!97w=F*8C2Qze928hu0DuB(bXXk+vc-;@qo zelEtIcS79WC)3#cUPL{WM}YTc=GD|0ce;-3e$Jzh5 zC6>Y89_GDl^Iop~>%=nn-ZPYSQ`rZB=&usXCOI}>sgwUljV&2a||5MTN^s{tB@1w7hPOAx=eF&V6*X>MM6F8ih=OgdPtn)$!4i8yTbkbiK zo8G&fbK4^O6nmi~Hodad&a`iZ1|DMkWzQ;n+k;(>?w@t-W>38r*E?t02f5dHHO)YF zp?}WYWVPGe?IkY)ZHdoK66Y>Nu8fbl`(5Ok28~G00ZZ1AAq;(sy^P@H`FYnHu$ z@!UJ1B8(1gaYdErQieSh`XXJaVti^`f5G@l2NkD_qCs;$c4*KtjXgBT`Y5(Rcq*J( zq&((gp;GW=fl~VRW2L~w50wHlwMv~K!(@i57a{4X|5b#ahE$OLAf>{u$d< z$pg3PU&sS>O;&IL{9n96@3}rkJLTVF__picBO4mlcj0cb$*Sedw!3}kIa0AVN+Mo= zg?}Zn^qwX*a6K{l?S9x~;bZ1X*@L8ng58N*t(idge92ONj3$8Ghi#7rK85_HdkI zr1;ja^a_OYxnEbC-L@_}5MIX_Mo$n&Vm8+eTqC&bY<$CzRYFe$N>^t<3q8GRDxuMS zY{eT2;3MoA?m&OuP)xineDqscVkxCb?y{P|n z^G;EB1?RA6{qq+$uTAey(7VL|p4r4{`F>ldAMhn%`z@#iDjHyy`1m9&w0 zc$;^AU}6tt86SQ9?+@hF5#P7~fA$&hD*aFL(re^Lh9Dc0@8D&Qy@fo*sqb^v`o)5; zT9f;3zn6RJY8*V|^=jr~9A3fi|>&-CGClV;c}Nriu`b=Jn$y?k@66(}<2 z#P;D^_fkjS)VQ6RGl@B$$lPBG3|s>&OaM-wWah>R$+Y^~`q9F6mJmd8em6w%Nh?U#VTxDP+$=>!c*{W}YJhtSIjft>e6H zFYCW&_MLwJ>Dno6-*Dgm3Gc4}h7_A&L+w7heDe~DtwZTGgF7-{i-(k}T}%jTxB)>@-Dbc#pyGHpc$1CF$xfbAx8zK{86t3hh9ep?PS>F|X9HM!k9fL^~TjXEsUDJ%-bwA_P=HmY$@c$_Iuk$(-Ukt`# zgXhtW8MHqgJ1f3@$lz3loyUC}cqG`&=Kch@s&-^@FF5HzdUBHTh!xNMjm(X3=pgsP z<8Gwdhke)R5zw8w`DvuuH@n{Cvp#q8lSw5D|5fq3vd@8mm9;H=F#(B-Uq3p?yoO4frbz+s!}6EU<>2WK51R zJ`MMlmqw-lZ?i~;Tj7^`jrBL6)0Q6&bQC=qaF=dHhw{BWRDSvTEb4vugXMWG*{t>YS<$(~fIk7xIMkK>2-fXT z)-d6dZzi^|+&3=sh63nPtxny8-`&mrHTZY9t68Tmwg#6+S?ew?`hH5VXFrD*K0_Vh z+6nrx0(%`JU$9;h&d8rauxDrkv~(Y9o{#D4C(9~B{piOwWF2YG@|F74;$wb5Z$_E=C>HfZ-OicU9n(wjCGd68- zVAH}ik78#M`rcSi`dIS%rxgWCTYItg@ua8e8NF;N>zE$C_pEEnST9+rqrY*u-|{^D z3;*7*uf!Vu1KPax8|6{`dqqEM{Uyj_1@o+fMdw+Qm4A`pA;|~p^R3B;%bY&?%4e3= zEkrNEyH;Rua3!*H(awzE8q`1h@$`vwtr z?*0dNpTsZnN%TH~tix;30TuwK6W|fhg;4QD z{^P>~;rg0Ee)UUqIJVjfZ`q!;N#noyfJ3V@UHG|$Z*&7Ub*8Ogq1KHfb%MU?xruod zeeFv-J|!;?-Gs@9j_!jdO9pQwe-(64|Ep&1^`E2f0AtWs$*hOC7Y#Z{+6GS*jru$H z>f5Z*)_S$uKN=kb_!dd>*}fsbK676Ozdwxc?qU3PQ#n&6vMMjsyaydBW7M*pb42jT zUBEY*KgbUqsXs5|V_zUr-#auFyt0N|8GRKRRbXXurofV=$TkNvez<&TT~fwE^k}=X zQWc{%*S^mkkDatfu${zvf22R{zCr1ZU+TvAc+jVP&|Zzl({A3BQfs}@IdpZ#<02t1tEaffTc%!UZAu!iXMvS@q$VKtzc}; zDtKtY3WzPqrWRU3-D(dYwc??zC~a+PTf$9t5)_bI78m#X{moh{>>${l^PbQ9$9!h3 zxz97tJkK-FJTp_UtPHv^Ei>J=c412!XlKio_7LX}tZCfojgI#?FlquOO~45~V8{9o ztuqELc|EWt>{%z9t>Zg5v%C`*vVQSYS*W=o?_%Pz>yGCke}&LFBn8p61kP}ut7G&C z=n!JMvjlyqXiC-mwl7u9Z}U=^wv7e8x!)b*EDTmf^KZysrt^aF5NkwmRxsTI&g=yz z_tD>v!IA!=3IA&BDc5mlx&19*Etzov`PEnTF-RGs4LrL>SnGeV#rsxYZ$2Ad@iyN_ zkdF*((?->4zKjR=;3E|`}H^(YH<6>|YULe`(8-x!aTxAGnW9S)7yf?Ow z6W^D3*~Ppt!J8L^DSRN|Cas}kY32=uuoxN2`@XEP)2#1FP3pVWelUPDr5@PWtEgT zE0Dva$MyfKkQ+V#W`_Qx|5dN!w!x!I7vTwT7<%@r(J-T<> z$PM&&2=!b+xvEcPES{HaHv<#v$&zNDrxts8-y^TVOV*wXdt*O>E=cztG{AccjrC8nWFpatEKjdG(}VkN z`g0F7+vP>P8{qMxuM(!U(+|+s_9|hDSkfMiDlGFttOtt311DoA}f>h~r}A z#RKSz`tbz5pHF37<>f19zcn$R%FHKD@3ku52=+OQoL1Y}ZbQ3e%cAp;13r!y8TsMb z5@*xC`1(8#cc+d+b|5TUcx$yeyl1R6@%Q7-&atiIzc@c0_sZ2h$&w$kSwFaR4Vdt+ zzNFEnqtLHotgBoa<;FcuTzmC}v8}zD`}GX{a^qekt}StkneU0^sgH5mCwi#zu+6m1 zTsCwO7#NgL>n_=np!-9v>e$2_@&zRn`t z?D2cB*O+MCVZy+HJpdyz%k;$KGCe##p#6};q4 zjHW)wv5!8sqY1h5rj{1#=2?GlSwb_8pB&| zdDPQ~dX^|YjrjDs7&Icdu+2-sg>7GoZ)0!Cm0P#xt~@KZwgm>q!E<uD zI0xPUPYL&1u(XjeX#gi@ot(2{PB5jZcmMK}^Vzr1{g`UU6myq$^);dodpsHWGw|2N zopjSTwe<!vTsLs?`o=PJ3NBCzU=!o!uC?9?wd(W|2Sc{ zwxoCG<%@*fOxQ5w#9#64rq75pGcLmYvBViU3^_X4npe@Ay+NfBex4-Gm#L4?zGIB} ziebpm%wc5eb?29?vk;Oide-mp}6W=}3VElCK0aN~=J)rRT!_(%>UPT(;?b&CNpN0D) z;f7yM>0&=mnUc$7yQ4fNoVCgi2>;8{_`uJ-G8A`;?|7N^W;(Dp(-D1qC-m~2(a(25 zPtU&0>~zk^wKaP)dP`Y*GkQx|do$9_Ccgauo!#!g0h#eV*5}YlWdDi{$fJLN@4to) z{j~wb(FghdN(XEGa9kG7f%mURwppLX*?zuX=FL~>9tZIKYFXFUUL9T>{gC|}bDuK2 zvm9H++HS}joWp!v@EZxwV*WkGm`RuWBf_+fkpA}}FHC)R_g}Ii_jzH96gFOYyGLet zVc%2O7=@)p?()JGDy+ zPR~cXEz7ogHmzbEr#5|!xY%PGmWA1e&>qbmvv-qa_HJ%f9P>osdETDR&4ii!W__C$ zxnBR_ZsvYl_FZZc_FZbM%k%81o{d>|+Aa7Wn=qzMxOb!JTd{M!d~kJ~eDy^>^kcN~ z(<7JTdwV*q%TCzS36e%-Yi^vGJrP`)0M1l^JGX#CH?yb1eRFv?a({eVv!_Ga4{Gly z9$Gie-P38vOVyihgQdTIm-Vu{pVKw6GWCJFJ%hQ+Znw*83e_gkV4J-DP4;uNccS%} z@{zx&w(Kjf)pDo)**%@Ekr$r)_qtzu`LoEc{TvH>--D@XzU?H=<0{&@Vr{S+|8*7s}weajdXomC#!jYMLH})A{kdHja*}atb zoM4^ybsD^Vo#qe21@WCp1!`xEeVup@+Ra$RLV>#4`Cn<)p3ZCRnXtzI&hf@I_^}&V zV-Gla4Em;f&ZHwg0Zx1j-&q}+)U17-pE_~qs)c;w>VA|~X;jux=#ly}2iluvrKPCc z_pl#&#M{?lefo8%J4Wjb_8|DKE92cH=*XIm&{F8chqyOzOK}h3 z*7qoOitvvtv7#>{NAA%6FFN9A?kAuAT&Zy2aJli*BVCNIcWnMK{Ro4f8^%CC%3S)f5jjwM!rB*m7CLfg3k}hp zu;j6xkx*4t#jkd`b2w9N@b-kY7c81`j6C5G`tv2l(LCPkIJ+mT{F#vrUfyK#=0HD0 z177p!hiHf9Og-~%Wol{Ni|p5`zFIHOX<(vzxmIv)E0M;u`8#|O-ApsQWJOx={p}yQ z?K?zz?K2jWUNq)*XuMBjzRI_E=XgaU^lxk+IYm0{M(E5j=u89q$H_TWAG~XA`mCX> zO>-kS(J%vD)S9FM^{oNA6N_`d(9S?z1LG5;&3gA)a=+$uBF%B2O}cl{>`Q?o{yuU| zvrBJum#IJ2vi0@9yCi(e&Qx-V$~p>f6fOE1vXx7VG9y2wulGZXL`UlQ7JXVRpZb1` zKLfflGDr7`y64DrUI|;e4O7v_Fy7|u7`n?=_viHHP8%bCti=8nIM#FTj?RSd;+)L4 zC{yod_~)oew=VE*s=4n*Wgi^=3TK7_oE-{s4l0Bk)CL~b7GBm4o|Xh}J8L%+FB@U@ zsU7IqQRvyeHv*j1X+vEh=$d=xRkBBA;u)a%8L}N%hknG)dX#-Q?su%@O^e@D!*9?( zp4b?0j=xGdYXi=amjli#*bi*TD|VJpM(q3Ud!;4J)2+N~f$n9UdoOEvy!4=%*!}BX zRW^t>==>!($$n+m$Ut<|Z$JZ2iH1P`V#@-~0=`!uR~6&01wUe!V!OpXJX^4heg%44 z3;ek&*1Ty`dllcy*#kij9ebn1*r7b9?~Nr+x%^j4oUr_t-Lu8D!l^abU819}8#A^b zKbTyY-OIYSiv8Yt+0z}p#BSck7!x1+@iP_Dq50vT5uQW%0>ba4&jnW$J2?T$13ww( zL%BmPwYQPR!0^$#hU)Eb_OoV|fFD2l8}o@i*UrDS+2k!iCrnK$kk&R z!*a%Q3}ZT)vAquYgnKdb${6!f#=hj9gTw#JoVjV35#WqS>CkoafzM?6&zY_5>{pw* z{c|qp`~5qoXvfbtK_A&uvl*N11-mYM-_*D^=+FN@+&4vAx9h&C`e^~@@Erl?^qhcm zXi~sAI6dIJ1{{rSyr#rCc`z8w(b+8Khj`k18lNSc31EyjL5Ct<_`3>UL^xwUh42@> z@GS~|hH!8InzsIVFZ`DZuO@sB`x@Zk54`YK75-$vIeI&BYfGGB;%qN&o#JK_x4gtT z3Vw;@a=*+$;qpuC#`MiwyZ z5a6#g15_XNXpO?1#%{c117NQBzKS1@ULP4kHf4t1F>gc6Ju|NlUD{&*b>hFUHD!*T zrr-7S`EoOkJtKu?TuYpM`Lj!$%j5&2OXOFVIKAZq!yNg*^kVspYft&JN}M46$%CcQ z2|JSp#hevE4vJA1^7?v%gY-poN%0L6=r3oHB=ZEJw_oT}n?Ckt z?v36Ce?lJle#5j9CnfparIUH{U=^@wSVn&S^#;Ou^XKH-0nW_7L4Mh&OV@XF0DH_Y z2c7s@+?t@-YhE#b=u0c+4|%Enjrk+$U&XzQTZ>zRTa8J;kd!L0l2GheQ|wodAMv`4_r4~8m=QQ6_MLm$C*;u`4sr4-Je_dao*&L~`Sd>8 zS3eRSO5F#%uzH0JhwsfOjW*m~%3fTuk&)&?$8?^fTyocN+KS(>rHie$G{S2pv!3ov zU$macqRv^2<*X}p=9x7FZlcuWzP7^rS^l}@{?Y4;?7*Ag&2c1ty_sJIi@8JKPHqqnT+C<~; zwTU)P4?3r21f51`ZsRP{HIeTG;n?#OZVoj+rT)wbI;ZF3o(wt%(%MFwy0wiq4j}zA z63n-7VcVe_5HZtg(EDJgf z)uefbyf+axKIj;^lJiX`k@I7H+D1?IZ5wSK&^CH>8EKZ|t^(fz*cPxCesUH6tEqc! zn`rYIz8M?j_nE~SV|{~{E#D>X{TSg+b1P!|+5cf|wO^ud>2|`M+;8d9;)EHz_RC`2 zTguXRVp$(0l*M;TS$vza@cnj`)BlCIzPxGCNI8=!-@I?JwtJnyp`t?j$X0MDFW@|~ z#-$S`e0`z)3E^hGa9>(9l2zPX&JssuIm~*bAr60c?<6NVDC>$J{g3M{Y z>Bm%LKhxJ_*~(h!IU2{)@gl-%!f%gpVzi_7RD69sb^P444f`y7GdJj~-^MU$b|V*= zvWB`a`7!B^b?3gE6Y-)q(c395>tBnV?@iTQ*^dBwKTeb z`_q*teQL4uALP+HWCq4fz*v3p@Y-Kv?{dej#(IB@Deno&^V6IDV5_{AzXC7tjlU*iXk%n9{AT!Q_B#wUO;8k5cVtxV#Z@yllXO892&wILJxUbQiN3-s2< zQQ_wSV5agPyxcCNe9j=7d{(&lu3dx!|-1Qq?!2W5l{G=)4R^E>E$B{C~Khc!& zjVtW#lwrntxOjtWdk{khVezBEiGPkrYv&fX09y#?6W%ej>j^mp$!LOsj zHO&Fpw|~lAtITEKKG@JS(G6%}Ky=&F@2nIrqKuZl`tYC9&;H&9^x?eAW5y%a&wfh( z*#BM+t}~`H$5YSrM+@!BC(`XO@(gb`oD^+K=1q>u_s72`m^1?Gv4Mh=#_^?5?7>qS zp-J3>J+mQgVV0rI%y;umI~6b7HUHr2WboB|gU6G>V|~w=-|4Uko2GKg&G%jJ-}KsK z>N-Gt1O1YI>Ok7lS)V|=H!|M(KCmYcogOG2I)}Jj-1lPI{{CG<4}jmxspkN+v>KcB zjq_3tH>Fi)9RSal&mCvJvqRQQ6Gq#Y({}UCJYG)Q-L$2Y-9ANS=Tf%%z@1qWo9M$F z%ATul`Y?w+=p2IjunSt&L?7nRhq?Nut>fW0b2NAPN4^-SeyoJY2#$iQ-a2W_6D**e zvhM?Dn~=Ni0&ZbkdC(dpJZ>aRdqOeJII9ftFze67Cs)w6*eUK$!S)9^bi2QQXU6x< ztIeGty1(uVao_H5m!$CS4mjT%Tr+Qv!8_x9aIDdF--FJN_;QzU@)qGHwD%_Y9ZQ@W z+b*VEHKb1XX z-7VMv9L+r=sR2jy$=oxNgq$imws(Nt{A=`aBb)3ftd}9z}Hk@KX>Y9!s(_Y#*Xh}ykR1E8{7u=@(Va)H4a)dZmMOE zFDR~?!+14t&x6KIzGP&bLCLig0Rx z!KXAM$4~WTnU$8YeQuy#VPuC|zkQbsvR6>A<^TU`ko{a@dufZI>-gxyOuHJ^mPC65 zk_#(CRtwz!QTuo!uXTHNU7$UcgY8EW+Y=sQKbY7a+F;s)A5VhcEnz-zXGHAwz<&>3 zz)A3GS?~C8$_*GeCBwG_r-meOs7HYFAl`WXbcj8X_N?&n@#dlS&HM{L)gEMpsP;5s zPJ=*BaZh-H^z+Y%dQw-b<+Qlc^ z{Gj;c5pda*e|ngGrPmJg|MYOXFrj>Zp2V^FY9RkeaKC}PhMt2{jm)J2#;A#QAGp0F zx-5`dI4&?er3svyqxiI^vYNm-(JJAb`KLXaLz)w!V+GLPCNqapQ-qs_hE==sq=b3m z&kf`UcP{5#V+Ru2$o&aX?rbbF``Ob=oZXbahqiv47GHvHHhOOOQiabBL_@$BJ!%wv zD|bPZL>p#n?j#q2(_P-{#@J*Nr#mF7JzSae-^7)Kxi#F(tsX8edopY|w~afu&?lR@ zB|Wrg*Sc%$ci;AC6|nZ(wE(**6F+f;y_tAqqz&f(_ap3ICZ^AJ`;ia-FP5$3U%S~G zhzIxF^k2Q!{yG2j)2*-mTKgqGzt$G?+29nuDSyF8`^Ut`k1!_4?+qpmceqXBZi+y- zu$=kq-bHsGzKD5sgmD@Py+prF+c>{x*^kk-L|8oR_eHQf4=nB_J`ooG#{ZNW zYyBp{enB7mF8;Z@$n@2$wdnUI)`C7BLcd(N0r$y%UBJYzt1@5>Qhv?h810SmZQ7a> zuqTk7?+xNTn(ro@*?L1lIZAKlGWGiTt|NZR=SD_9WW5-$OA^w@9|%N0rJk{&4uuVI z3;3QL4^&{JuHrS#&FU-4 zkYQXNR{cpS@@s0u|9|dqsiSien#=7Ce`9=*S(wL#qW4YES;{CBeUPvD)o{?o&v1bD zGPe|Vprhtid*WTY7PVElv@;l-nTISSo-Eq1m9^F>=DOq=|LS7p@(!zp3j*;D@b%k0mGG7IM*^J!hGK4c=lwycfQ ze&F&^WIw~RC_}idy>anQ@qd+RVCIP09(xwA8dT-Ec@jAcgOa4GfYQK#f2WZ)Kl?SE6Rc%kX{ zu50Zf)TuttrOpTa{t~Xa+tT0SHnyEuH}x(Eb+5)o0DHEq>ewpF=(xf+(1vA}-a1sraW z^=<6UZL2D-$Br>R=b}2DHBXdjV_SyUv1!x|4~fmff1o6K;AQS_122)Kqk?-TxC$?w znU9Tkc=NUPooS1grp#?)r*00~<2qL_T~1sRV?BklNok#*T3SEba-N~DQ_xw~KVUge z^Ua*zZf$R8->h)x*g$B(wcut0_3A#bl4}DK>2pnZS6}gm=({5VMalGYawvKF*GB{< z^6fOf!FX`r?yhV2hh8?`?*32Je?hTmv*M@k8qWU}dM~x>^tt-(PTJMluTE(S=)V{L ztK9#&!}!nTf0-Ar|4aD?AC!LXQ2q-P@BVYc>BR7(;Q4CP?xwjz0^DoI-u)%eNBm#I zqXnnTzm0KnIgeuceNB?x#T$FQ{~ilw7RIJ~d}e);ol0B_Z5|S~llX_WuMhKUrzzbD;D2It zDsNJXuNz&~c57?cKCs257di)I+Bz}W-cK0qS#SPh$=u09dq#Wz-%hc2@-G_*+O|F1 zE7cZ%?ACe7Qlr~?&yvng=N$)ysaxr?aJmCg;g#q^1fOhdLt=ZtFKC+PQY;ScK_e^C zZPYW)yBOQujM*N>LALTAb#~8d#HpLLgVU0RarD!C;H5dPH0=Ggq+x9LFn%iMT`$ci zO2ZmTZ?D#7ptny0rt*=a^6~X9)-J+~T!oC3i~OYhd-a_&tj^W+&G0$yLn#Mu+s*{n z&(rQ)cE*+OeXrdn?fnU96WZU^OZ#iD{U$BA)>3XlKT^H4ZzS}iQuo5q=6w0|BUW@y zKafKXc5&PDlGhJZGId(ZN*0g50}VccOIYV>KUwb^H{`iIt6`+;i(eZ$k51)UY&utP zzT_vr$JPcvzB&}GU(~_A_XF4N>ig8?pnuKbteD1F@{IVOp_$~Dz1n*{IRn9&HOcGX zu=J(oU14y0@B(OBAbAvaRNW)9D60zG()VuC-o@HjcaS#3i^YFa8jch@!Y7qIwZAn{ zxE9_w#x3(5#aR^@oC$h@xhWa9mOJS7&d;!0U@V*icAQD6xO1Mn7xZ=N)IAh?rzND* zxOwR&d+EL^n4%xpTb7VcV@5kw-mPA`8cs5A~W+j(AEZ%M<-~6z2 zvsUuBhXp(FH^D2nOJHJVm%w@hvroo2wVea&E8#=Q1Cg^j2a5Ea(w}>jpx0YF1t#)N zO3mGY(aznS0_*jCPutPXMc()H zz(j@p?xK#>@69bmch#{ehp_iHOYS26BEAFHnY2^*{!Phf(zI`~_(oPc^z_PJ_9x_j z`k{N8=je^D_JN6N*W6&hyyem}Y-q26(aXSS#o*R3$}g+&_aoQI_UUQTXdW!@0$!{I zM?0Ry(W#dr%RoP*djYp1;WH2Ks2prZR^ef&Eho;TduP6`8MUyhs1JJhVqwr`B#x&b3Inp|7`wHLVo%k3}m#l!{nzw z1HAl;$RC^E?`+<(D^1ocAjkw*_}WM>HPT z=5}}H?5MiHJqS8G_bhQ6bO*w%W?y~#w4?29YbE>YQc7*(rB*j=A?%%Yj=BGUb|=@}$o&=E-_fUkUE}TK zo1BE5DsE(w6F=3?IS_B>#6QB{*^V?<*ELQf%^k#-;l?BpHlS`Ha%bao!e=BocNx1S z;%-WU_R^MAzB|&Ergrf1cIfp|s~flR{a!mqd?fw`?c2zEOnS%4qhG(-4E;i8_T^U5 zuiE`B``l)3{+Kj^(=hh9{e1sbvq>;RBHTl@E-A|2Higj%@x0D=xg@;x91+{WjEYGdhT951l=~}OD|gB$z0(~ z-9x)Q_jHcE8=5!!V{|zi15xbC)6JT>M=+{AnrJgRV%DDJY2e`z>ld|=`p-nOWT(lw z2xB{??*)8+fp&qj(kB=2##Z=~M+_aym;e7j!xmHkM~{Z(>D!}WTW{f;xhonrP2V03 z3+vmXVYN5&?a{DuzW*5-*87hx4V%Zf8yZ%2MFJfu{D(BGiU4I_946hAzOCn4`KX$Tn@KA@m-IKxuk98(n~zCbd|!OSqEozsb?%yS zN~*0q(%V+{b!pyN-uTFl$`R+`jUG-J`uN|(t5c~iUlc!!%?2;PsoCO-`Ua=w5T37Z z52v;~%r`it^p*Msr}Q1xH#nvAwGVkX)dN1ySpk+OGV=f+yqRd%VVSABV*rS1##fwEj4 zpBWkFmGL}z65DULlyQOf`(@Dnpbd|`2}J%n=V7g;*Dwl4gK-tPUMkM_13k?AHwuNw&; zm)qOczIDTi5_AOU2MH^%^Xw-4=~v|0<-|3D+e&{YIt}y;XY&rc#P;rMSy!-+JO{ad zoOA`~JsKyu=}bIwNK?F6I>$m&=fX>PD@b>}Jhe2o)^f&%t--fSr>A!Q$h)WX4EL16 zCseQ6r*uOY-zLH;gQ1d)4qGd zq*wdc`_X!Sdop|QrZ0caw@v@hNzl%kg702zH;~TPrLCN6O)MaN!7s-+W9Oph<_x}g zZ8^Hko46-y4QF+#uz5&(pSvINwT`U+or}Ac#&!nldg1p*@OTFHNAxTY!gJ5vXMTusjQ&Kt9l1q!Tgrw*_8QWu?1&(LK5P|zj$A2N7`Y%V z7}eN>c@r{)x2zghC)XvfEamQY?vX#m+1_%>T4rn}Dq`$Cm20iM8U5Qu=+b~y!K0C7 z;pF7%`YjpAxdD4Ob1#HmHmo*pw4&VS!ak>E>X1Iil||6uE(%760|Qr|@6E+;cClkK zEGLEWPQ_iMaZht?LEiV;d{4n$7st1dp0+J`0Nb%GjLAm+7qOr6fay1KCO+-aSf=n^ z^tdOR2V)yPJ@_Pe#rY93|7HyYp<=p>uc`Y&7xtex7ItRi$++(q7v^DGnV6c?ULoWHJvR{QI^Q`Dfc&$yq?PRkE1cT7JH^{;Gq4>y-&`eU0{1aW|6RFU`@|$!9A51Gq}hbJXz1 z^vIFh1t;!7fOnoq6%KTZe1C^aXFjG3ojW!99@e)eo%B7VS?HztoBHkRdi*rtfpk4p zUYg%44RiSv=ZiTzW@J9*s?qgq{dQdYk_H#5NoW0{b(^ZSevw<`E+L@xcVRT`#v&qLs=7zsOcY9hzpK|X^ z3iLzoz%bW1XCs|i@zAZV{Ufw1-`+*oQQETt9X0EzbtjJJ+djOz1Me#I)L-8l=LYCI zFRstF&cxO-20J+$IiL3;m{06=$kusXZqEYlfZ{)78JgagGM{}9x}WLN{jdMlljjMS zte}4UJKtk_MQ#+G$3AdF9RC$?V?|fyF8G>HUeVv!9hAExWb8*{A42B`Fs}~=Sc7p! zkNAcY0q6|>3G+FAiu9V(JGes~nh=YJehEEbo!4!8X8$EwzWvQGz3sCDK8*|-%R;7v z&vbt@a~XFil>*c$IbdnK!ntWN2Zuro* z!=rj&U&Wi_M;3*m%}YYqjt3mGX1$1Up3E7`chF~TDdDUlv=e;S*(=F+^#cNq*<;7o z8AlU`|B&M1o{bN_&OMqq{CgF5(2K*@Sx6IyKV5Nqyf}Qb52N~^8{Z&KIHUTc1Bk7{ zMptq8|4N*~r5}y$N57uej(1K<9EbI{_H|-==m+a~_G!^$=$m!?GkmjV#~!_IH@5wI zpt*aQV-1T42e-^yC`;%!?`50$Hkr3f7*F@!S?(>zwn8*T=hol;CG93|3x39+fOt+6 zeb#v7+SED0uG6O-h);|ppOri2lP(8434e|mns#ezx#Mf35#HZ@5jcTN8bhA?XXk*y zdvIN2nDzvHWW2bW-HcloU?uz<3%peRM3s;JzLGtp|55qy0F}RWq+34k@R{=2-%b~7 z9`j)Hh+so~@ByQb4fp>a)|cVdSIL@Nc=&g(zMHkip2nP9##)j$KGa8}3+Jtz2_2zr z_(pcpJ}xvZKZW~`B}?hQGde2ZgB@VS0JRa=$W}TLr+-B~I-4#$(>)N0IQ>iFw0=?? z`(qy@;`A%T2~QL!-EtyMuO;q6I9*FzE1X`Xw|l_pRlM5+P9Iyvn?2xj^Qu4#{>H%R z6VPIvr8G3y$PQ_dz1_gwG;E2$=l$SmCOo4p&aqPIfM`!i@=h>Q8UNf9qQF zsq#=w#Vg>Aq029$_cAgkHW3eCPs93zJ^VuUZVClYvnPao*?r)z(iZ~raoA;)vnM<@ z*SiZ3yNj_4_1*d%Db{USk_!t0T?>J0xkK4~zDOT@zN>F*Reb#v(#4W$DjugF(vz9B1!s1Ax-Pw(_7}~)@W5nLCQ;zRjw*fPK#%6|hg4fYk)u`O5I zM>*S}cJ*kE50pOct8J}8W?(4Lhuz@g?FDTZ{C*G;b4J1g*HuD*F9x*1$u2Yh9d!?gL)2E{qMxR{Cka*-u5! zV$PVcrZwwk^R^E%IB(9Ux2$!w_8mq)kYSWw_r8{k?*#hU$^Ea3Hq5e|bYLO*;T2#e z`|zWj8D7V?(q*y-tNTXQDh?a+=2bzb&Wp=pPgXp(R&m&wpIjbvUhv|2T%@=jk(G+; zgKpjn@7Y=5JtNB$o>$`hhZo*GL*d;c&nmoQiBp4r3_hbfuJkRx8b7VQHCXbP1xz&; z&*hUR;qw>blShcV5T86i+=ck$KH@IKC;v`d3!lWsAT}*9)X0GUi~MpPvfj`nE8WNK zG{NKBt#P}yXHCW7RK`82G#Uf11~7K_UFPCp{;e(^zU=kmC*a3~gfpX!x104-P43n- z`|qzaZ@f9=@V(+5msgedsHylfRkE# z9^zWYQ1bgK-v65~SqGWN$U4Y8M%F>*F*1%T^GMD)io9cFcl0Iu(3gCSE@D5r2;F6o zCjQ5RFdH+2X+qb)U+x<82qq`~dhLy$GI9$|J!g-auDf?Y_yBjxrEj5~>*%WmYX|PhvhSmpehV378gH`PR_q);#@WII`fcde8EhDnEbhQ};mf+eCil6vc0bB1 zI0D^9SBuPsF2c13+I$6a%N?ZM5&DN~cYocYJ)PL~!2O+KX8~^j9C^&xx7AoH3hg55 z5uKibuHdO3qg&Wx6>a#~)p1C6H}6O;VhzhWmiuLH$v%T_q|fp!_KWTc|CIA+b>Kg` zczDn{opBq&d9;(z+!(ag++72|h%I*K>pSo{t#M!E?Tr{T6j-d++0d|jXyjJ(DUxaZ zc@_s>e4VMUE7d-^fg5-uxZVZ+|8p=!=b3VL-umSZ=xYtu9KH=2a_+ocMtd(bZza23 zXx@I8xC_nO?+|w(+B=81|L1wjp60dh`Q?iZ^Tj4^3&%3X+7lO}5w$MDrc5ET|um4zx^#p4Ntxak>a<1@* z<#1lttYeJ5ocK8OQe)QM(`Q$rpY8>IwI6cl#hf*iZl3pWC)NGzZ|K9&TTSh?=Eixw zetKit>Cczw{A?}fXU#szu}u4NaO`Px^$R#h)pV2eZmHWoVZOZdcBb7l0b0O+_0LP| zKHWI3ZhD~Owxu;ZC$4QDScJgr~LnJVW!;@H#EzB2|LY$^eIk! zJ@aKo&?-E%u{hc>*s&1XgPKmI-R;Bp{lC-QewQ+(E9}t4UaCDM6W^h`-J#3(ma=|6 z)QR&l)t^Z)nV?JGKccPVYS z_qUzDyY$o9*F_J`Ac$ zX+XFx#{Gex3}|;xeNx+dcBQtd`*dWe?$8+MzUYm6r$S9`Wfpr6+zXo*sOu8_^8J># zpB zWfzO5G>}*CLQc!HCgw}t#AYO%2`ys$ENn)Oy~f&=_o)mmAZ--f_2md}-CJXA?`p41 zcF#M0OnP6I(06fPSC*)}yr$yUtp9yF;CF|%Y_f`WpY$laP{ZE9e;zZxEYaPs3 zZ2BvzpU?$Zn*h|x7SbST}i^7RE2ksm~Z%dJ@~(x@FMM> zO-H}i1-}WMINu|4t(o&WS`(9z6?GRkIIvy&o!=yVP7fouZui$@;SMzwW#EF5$u@er zSo|gk6uHzDh+GU zdxfmCw$;F1`kQ^gqhtYfA`_dvifrzo11{3vNLM2r%>mZ-pKKXqf6DsyAi5ZHKL;|| zA@XKpTY~NW#6#5I>9Va^H^-9(^?RtYxrX-zItPpEZh0ubx$}2M*UiMQ!mZRl^4RyM}G$Pgu>`bKOk3_@bcM4>oh|h$UI)Q25y{_I%l- z!5_AY4^q!A^78gl#VY2KbxXWR^FAM&B#r3u7vY7ojBjp z$r?DcEoY)Taqld&9Pf&4}!0eNn@D@0YBLG*8VwGku)Ryqywg9y|rvLvjN&qNZh!99Xf(6g4pV zZ5MkOF!JZzf>#-<_~+7tw9L8kjx`nE!k$@WB)>b}?9DSK(*xic>&BW{z1d3z7pBfD zYT2XDrX1A^-*)L>Vx0|L>|WHV_He;;Q5;!2E*ZEw6x{`F+YN2o18w6C1iL|d9Q6BR z%HIzy(|q>lNvJ(-9mqVXE{jU;HvPFM&Fu4k0`2zuG=a9%H@EIn?CP3|{`6_1-alH6 zoL$=@XV>_5NgSmuCq!q#joJbiM?Qsr*RKg63t8zJf9Zopa&B&FE@x(QUA;^90Pk<+ za6gNW8$;Z@JIO2D*g{^vZI{zF>uBq?l@F?^NTF?-gQ6RnL*~9)##QfCydUSyWG`Iy z1BHS35(gS4eU9{uS|6O;A3*nl{$-EuTLe#c2C%ISIJ=kwd-B*j3U##i?dbqbVvmpV zRnA8pT$<+WmmZ~i(GMXHZF!uV`<6!|>auj^{|4o(x_4`1+$!MDdJqsMqFKJa~X zZf5Uf<5@oQZ;YY#BsEuMlaaWuatCp(=J(59KKv=fU1-hnFT}OtU8_9ag&noGP04;8Y0a5x2f#Q|D~5zv_ky3Q^203`Pa)D=;QK3 zuK>5dwZ`SG88sD?&_#jEt@fvSp4;bF&(g^Z;w}Wo4#c&BV*;#_5@406Q#+wb7^t99LMbW}a8#oquvu_R==Z2{~#p&PB&=Z0?R zzL*o5e;(b~+TES|^~f@!8{&PUCowPlFNACCe?zzr$3DO@2e~ql2FHfD`>mU0hmD<= zMH=CC1%A!(v-CTY|El-T8H2$!701yf`Q>*!U-`pYmmk}}oJ@jdq~g+W-EiY^H=VtI z{^m#|n z=N&uJT9@Y}>|dI_ca7z0WVt2ylILcKR%c z$Xyp3`UcNcSv`Qk{p?9?BAxzyU4dkxE?)Va@Y_?*xS`W8KDVw)>MFj*t*iW6tEeq$ zQ-|Dr@n5!Ln+1=T9iCs`R>@tLb~Ze?5WI||j}i`)%k~TZV&E=Yi*B^Z#~nYe8*!Qo zit};DkL#2WXK*K>%w*z(S0*22izer*KK!8S!)~K1W%@Yh$DM*F2 z=9AChBd|Ad_@5A0pf-B;APdmP>Mq{>UY-)-d^}y?twHLE^YP8#SRQxEi83t^at}RP|BM$ME<4wC6PSS*+WdJ$WV{*+%m68n3?3 z1&axxDDA00*Javu!kceq4m0mokXCa$aV}Q!UW4pmen{L@zs(8r?*#LY$Cc08!K@A% zJD5cKmwm`H#x92bwAjLYVLs|zv8>4c$?OAlb;nh_EsZ`^aF@UE{$~8SL+`qH*W|z% z%{SdsqBmjd`TrMeG5$B!b=w|y@%wu8aEq9G1H8I5UmxU-c(b;`4>QOAJL|no+%tFX zdhbxX|K579#TLoi%Sfbs=kiV8_DHb5Fb$OLQ8#E`R%8`6ug?O{!QhAbp2l}ZK$CV& z3!Kq-wu~SD9Z1uSJKoPHOMi|YG*OnG^!evYG&aAXybhG7vd-1RwE1&a3F7f_c%PNide^dB(#UUVtxF?2VwF^}Dje-M8X{+>|u`2J8d)@Nd!Z)X@o zpA(A{KaKcM(7n$z3Hh5fmHIE)`CGi-VExdOk&(v-sO}@JbpTdXeBwvQ#fH8?Uu#9* zSj*{N-rsp+uKRem@!iU{^u1tR z^L*$jFqCbn8ShrMr5TaMFI$U0{}nd!;E2u?YPu7fPr3U}e9 zCLjK1#A$p~rf}ZpYX-OBU5&)I;PyGZ%abpI^fN2+@c?8B_KGH?GuGfu z7&v60V{VpigtmdBM(%jmYrpP361~X7m+nAg7L)#KhevDeG&`Gp7c+*?-Ed&x(hb_gP)t%H~^JFwqt-tVl|dXjgoGF-UL z627zlR6n`o%v|=JvY9(m*n?s}=FD5Yes<%YUTbfBvDZsCP9l80yXI-ZXXUAXIdD&G z^(15!>Jx3FJ~O}m)Ysnl3+g!E{I5?MbU*iEXip%FeT(je(;k|12)lF#C-@K(-NJvE zzG{3$gNFjM%e-+;T-VPeO)K01_C9{xMO-V~0VjOim`0qB8}TyaFVTlPaD4*2f9lCx zi*!aLBeLfikH!H%qkEnY&hiG>VCbC6AE`T%5@=j4HuF`SOUgmtlgs@VVffVfY~~G( z`;$xK`Y=CRXxv}ENaLRUlgrmchaL7hMdJi_(Ko%Lncu7KM$Xe2JP-2*3}aMe)>FV_i3bP4Q|Xg|jp(h$RQhhc z_YmWbuXFh?vXr^M{x0abe}BDo4>WyM!1)esiQf@$dQqm%>da{ejKOK{e#8E2;nK&5 zSD3*W=0#ZZ4gV8(MLqe{X65I;27^zg4V0U&bt_?eymSheJYv#n&X6WHp)8uu8#k5k z{UZ7^654LY_{3R0R-S4NmW-o0Et`Jku6e^eHa!$I{9I=RHu9d3&I(||ogGMo&K)2R1Ov&O?>uYczi)b3^!k~uE#h_bU3#t& z__Em=g8vtE{3mvVqDMarMUQ=iJhC&SeXpACojH%OjeaG}ix=s8t2qPQ%$&jh5pjt$ zRkmhk&fxzCasHf%L3bD@m+l~^nDr9;wKd)8&Uz)w)n!$S_GCtW1g;oeIN>R@I~|>s z;Ddep_<_I@Tldts?A$r)%-9-xb)Ti`?&^-FHRqIInZ__T`(u!y;ZN|ym zA#+Zixo)0v-IGV2X=gn}y;;2g)QY_1$){J5QINBfqncI{4k+AiuL`a2IcXeF<+{XC2Bh z^82@c-oo3~&H5~-MSf2Yxbpko{}^9lco4Avi0^ZG+p#Z|-_s*6uCo?*KxSXT{Zf+O zKicT>wo`=ryzL<0;%!Gg-qymSlC7DK7ZyJ(J&d_Ga6*~xE;RQt=5u~Iah`t#c z1M>Zb-dDE3FU^DB_kfw9slc!LT=;z$tUKBoe&?L6ON&edHvfq{pmQyPU$qCnHH2%= zRsCGf_Z$y?N01Hfb7g_;;r9z|?&KTVfIJZXGv_Yg`Ezw|7kT!0FgVJcrn^>`Mc-oW zQ`+xJbLMdLCgdabig+6(3hyZ@0w$aIe~xxkDIEK(7~#z)F^w(os*NT zsMf!O7zbpi8m+lXXdm*H^zt?SnyYqO4fd&L(+nYvg$^+>&4R5pykXA%>>A{D?u`V$ zu?K_ad3wV?!=uDoN=S3Qz3S#aVE02iO&<20MIU}c9*qZZXlY+}wa=BITiRDi`#hQ1 zoDctOt$R*OdbcvtJHSx;2*rHYKTzykD}Q#ebB+9@5@(ov_=fsbpH|`wlHaYwDZ<|s zV{JiO@;x58+Utk(sm2ykGPv{==qI3a+tXRQ9QW2Pv5sy(mXb%b7@PbSI#((A3EsEJ ziwpa4@H`WT|1@#ZCn6u2GfI_e1L2bCBrj(YXLMTNYbzb-d+0z}$B6f6O=EZ@I#45* z`@C{Ecr{+~cmjO~UaxmUhtJ(X$e$nRp7(sU6aP{0Oy^CSfvsd7g?)o|NamByS?y?O zqCQint1B>VSdGlI->RekuKu;hdFp+V`Ih63wd$Xzo)KW&=zA;39J@qJ%c}* zdRy}^m%r}8R%9=>B8HwY&Mx0#Z-Vl-Wm}7%r!0+8SI!A&zu4fKH?I9DQ)8s_o=NKa zxobk{VDF$WS`z{%;hU+4b-N!)rY`JB{!( ze;m)P6M69b>rlo_@(uM#zM0U)T6_m}8eE`^#*WDMj6*|T^zPsRcIm!6>7JV;?;iGU z_Of^5w_D}BP1|p!9G@0k3;c_ZpJ_eM`_)v4fBup?B&0XgI7shN1%8mv?A0(=iZ&4M z>sLyK-hFXl{*;TiLbrUqO1|WM_HO=i@N9j(sc+L~t?P@S%Mq+u51YAt>SxyC?~=za zTlyKltiFv_KxI$b3E+5wir#1JbxJz)-UG&D~;&O1Jn-}A> zMiXo$f9nk^;iBfa_IyNdGH^_I^d44v7hGptC!BO5daG7zwf4AFTnbLKLpYa&Ylo8! zmDZw~vl=(On<>7eIia;^!$G4%Fm_Oz;XMyBrb-W-ib9Ny;bH8XR4@idZ~D)=GqVGA zXUZ%aJE`W@%wd!2li?r;cqv*&_OYAgTStw%EK68@k6 zzDu*Vk5NcrAN_agI* zOX*+yTJ=j$SERmJcQ5wD<0J>A^eP@Y?hn>2;auL$eJHC7YvGy2$)ko@1;r0f0oKf2 z^B(Y-*&FN5ge=?iV{eB2uha{E9Ii*MIx~B7t={<+OuTw|v&}7!`ahul&m!%A^;x9g ze2@}_M`9rfC#Z{)pDoDZ87+{I$rRNT$Z(73=CG?v!X(M}cL zmEc5RYN=DA_F7gKzi&#T_pvI$=Nu2-ChUGU?7U;}V5WWgCF~6-S9$!hO#M0C?PKSO zGw|hZ`%fXBKkZx6KpOq{`ogfa8`1^p}XVPx9(r|$M| zNxzA?qR<{KHA`x)4r$us$Xs*OgxyHy1rrAKRZud7pQ|Z z&ItPj(ii)6tm|n%f1dmaZ9Nq3Rpw-Owhqtp=8r!%rjA>#F!M5{fIE0>?zj$wr;qP1 zyt8JG;>>bo=clq#(8q@#4NOR_s;ZcLCVOVMe{~jj_e~F;j+^<|syy)K9QdxY`i-#f z_u(MAb9XO$=9jnG;9og+F2%P5-#y-M)O|ubAu9;lktfSS7EypN-sPh{nus43cCi{$z7@{mh`S$7Q4E1=mO-57-y)m1Kpc);W07J zkYookn~cpIXQ;9RJ)5RVPnxCg9!&v#U##z}rj_ufSZ{r&HI?WaTW7s*y9?ZjalWFK zITGU>7qnr03|ry+ZoFwhcsyu1R|C^};^OTr=PG>@A5XEI%lVE!#eQy9;32b*K5y0V=~77->w#>}n(Ys`6|(>6 zi2O>}=Rc(``f1|n>k0i+PqTdbO!>|+`hA%HV;#kZGa~vH42}VVCcd5Bfx6S52J24# z4m*at(z<-^xjg;?cYEZGs*C*ud!ua12L?t?yE-^jwv(%tQwDTcbqimc2NgR%fL`~s zGOH!GD9x*PtAErVB9w?ppuAQ8?{e!FV+f<9!eE_8_n18oeE*_7!7$(Tu(K zJmOBH+xrNYMVh7f`8vzMoyiU0b~^Uf&z5S9m=SrMdVV;sl<)45SNUH2RvF*vk@a5v z8}NSzJk1)2G(YDbLmx+URdZBne@fWf{D*1B0r+7ycPBUGJ-4(mZ`snOyroN%XXiH; z;!+>XZ)Q)>o*hc7ehznAsB1Oz?(kgZo!Wv8_XnY>xfP#L))DYTGLRqk31OzE|+Q1gAN<^{?@wO42X*EB8^5wp>2xq{o+Bv6J;gzIR?xXIh|J(9Mfm z?t$n{dFsbCcf0MJWd6HHZk10vC*c2)k9 zS=VRVok?Hp%P7!`Ia$~(yYlC{UX{ z&Y`A;*jp?}T~-{}G-4d-o&wjaf>k3lUiob;OrIqSjG`@Nw5b&OiVRa0XrZeYiu(dA z-psbAqCe;t+ zrbS;!>p=I`<=&lZuOnVK^Blf(w&zXv(;RygaZ2|l={E7kfzkExuX@0*ikDOlvc-Ax zALwm=%vp-_#$Ddq-bviE=#yec0>7xd_1VvAAKSVtd}h>+WZ;|phva2-ZLJO6fcuFT zdfPE@N_hVh)d$THoyw(tvv){)<`JH4Wi$mi$E)^!i~q!W4d1_Y6m}w3oq<8m%j_AX zIp27%IuxH+0A3bcQOx@%&?g^mRv-K8B0Xht?Tjf}u0f-Ulc=7{drug4b*HQ%a?pAmTs|90sju!)<(+>wrF4q?JM z;JHrkeThDwNVA+Q+B7L%>O8=IJ^BK{R_{)HrJc2*86490Nqp%z^nHZyAooIs_l=1=4L7KdFhlQcviQ_B51Sx$&@8JBR*fE zd@Jk*&ljvz?;qJC{XKR2xY05eU(2@x-dt58rL37zYsylV&({kc$EzDJxh zt8IbclZ74mNqFfn=`7icDquI$pOHnBYvLc?1onI=82J8O3m1Ik9-ud z7C!`T39jMX0g~T&11h;`E;3qp*C=Nsa_1Dj)#h({dA5;9|L+>VTjU+%XGGpMKKFka zKP|G^_+29#jo&l!OZ=1Y#iiW&b;qyS&u?oDlnzDX|B6YI5m|5i^vExapB7nX{H~Fo z;rsAj!?$ow`k{N4y0BkMnD&(|=$@fT$KZ!;t)Cb3hOYRBp+~#8gSUG?`c+ph0%vA1 z$Ai%B(m(Y9E zBEJ*s9{K6`N%)!gsrWtdJL2cyr{Q0M-_457Mvi*s4fZaac#-fza+KsOwc}puQ@!6d ze%Hvq89yU(xAD^>cNsq|Qfd4w^g7o=m&QY%#zCjXLa)kMcZ`8QhTJti<+{AKt35K= z(m8O>N3vhv2L2o9XQx=pDs)b+eZQ)TIVGI0T8Mie2fU^SlI^Ei^ObNOPOwC`Qk`8A zFfx;YHE=DY{VAjsA8i1xDU{z+^mBgT{gZbDXj723g=ix(#O$`{umVPhVel?+*HG2~ zgO}V1EVu{uRbXPmndkXw-gqu7m5zP`Gr!hYRiF4_oi=xJ)u=TBJMSoA};&S-?r-d+UwY zIT`fz4>z`T-WqqENu#yE)0Aao>Yz1L`8x5xb?J4^yRVNnGR>XL6~Q^*2|3-VXK1L< z9=WWolQaH0V~-QM{znyQ&<4?Zl#- zqUXpRH4Oak8mS^&`v4ypzguKGzVw@Zed3?z)<+qRDdWMR_P0sDB3K;N-P&c$@mk(> zF#Y-F7)SJjeFf-WTI6-oNhka058e5@k^ke7u)URW-DG5vaOCHNu@*${CH>Yp^IGzy zKd(14uY0-kx`^^)=kWW)_J)B^i`=obz#c^%Zky7XL)U?yqrlTL@U@gVgnT*Ql`j*^ z2vCOlxRrXe-tcM5$VaWZk+kD~Wlqs2^(CG1(wR#yYc7QX?;oEOppEDWXCse8%a0;o zc>2P1*3*UdmGox{whGXxq515IE=LyEd&OoxZN7VGCHuk|;N^z5F1LlpwZPZNyx$I) z`FN(5@%yiKj`kycSu_+}QSlt*7R&y#UvK*z(ny}s8i2C4ull;)FiK<3u&UxMTncHq zkM*$h1uKZ#P5NTqze>a*;f#?NAGyMo{@ssjksD4I+M7MO;W+t`4UzrJ=zA&S!?=Xp zaq;sBFI&od?Q&cC_w&Xz_ciUmP^Ad%uD)6A^YqTVMRq}*8*jh= z@9Er5&v+f2KR>OOoO=v8*Fnxbiky2AJ=qcTWXHijcU>NgY^5KJ3XwdYVdl!(m%j|mesxb55P$KKhh;kjK?>?<3HE{9%_9e{&zdy z!*D&VZq=t*r!WSGw1y7fFr>O$32zF`Ep@u{&e9(rdvu3*=W0`b_s9!gJ60IKTjY7q zkAkN?DhKhldp9GGNlrUTnCRaRNF(}_W#Xkb>Kak}0TbUnQbT+l-#tydE6*u@uNObh z)R7sf^6GjFe~{*ei?M7YH|Y`Alg_Vqv~zp@V_a) zI)gW@)aEg?@mAhfttU^AHJIetNsJxmW7dnm^@it_))(3L!jF{xTGCGfW-W4mxN3yL z%^HOKH3ps;k^ZC$ckEhiQBOvoty5@VkrBzq&m=sfO!u~BMEdYQ@SOSwtNtzt^-mdS zYfYo{7n6SQm((wQrgWW2XHiE+Svx0{Z-ph}4*@=zt9scPv;Q4D%7`Qpc5eOMRDXwr z`s)YUS9|q;z8ZPtOX|PbtN$?R{x5Im9v@Y8?)^QP023nD+!Hjx&_e=Rs$3$X&4i!? zXr%?bQ$fLF6g?KH)q)fd)F4_JQE8zqSZK{`v{C_a#2&S9DlP31rHa*71K~20fVTk2 zfI9E@xAxwX0YUBOob&#%KF`|E+IwA}+j`bzTa=MAYe-u&ag`MVI&F9SXzXQvB*$wz zbA9!F8{e0p50ZU9c}OVBE&Dy*Z==4&pIBk!kLf?t?|gf%Tka0tv8tJYx0g%C7A-Yj9hTIgf=dzp64A z{}E*bkOO7l=lHFU7I6P^dO`5$OXv;w9(yLIxr8%FGiTo3RtUcb{lRzx-_~HG*9nqW z4e;uUVEo_7tH>%^bZjPNkUsa5#QKTD_;)AxURJL~UfID<5*wtmPb_mDzH{`&@qBsK zHw<68y5#t@5d)L@=@Ru3JXP<%sx6hTGnIDVa3`(UOUtpZB5j8GuF6Tv^3rnc z@ub~MS{?mYZ@u(!AYf^)#fO1;-wav!?C1k{*Y%geyN@d$}R zbB~(mEPE*BL>cQW>>hnTSkH{9>>$zKIeKR7bIfzDeKybkCLD9iT=$(bdDgcj24>l3 z0836WMpjE=!UMm-*7T<_SMf7O>5Nqe#;hZ9vlDif@1$}wh7WpFA7o{3>l{CHDjB*J z+Z^A);!#U3j^Bu`YC%^XLRTI^uC^jSJ$VslzGCDiI!kSeQ~qH70UqSLG}dFqyBXp^ z=w7@V_ZReYWap4jDe^?$9N?Qkw}M3$dF0GXi|^&x@TA{*>JL9WV|;_(3hhF-`N{9Q zr0JX8z+?2qE`RVTA3h)9%=qxmV0V1}&O7`+PIqj?w-Z;tMe&87NZY$u>-lunNc!nw z`s*V4Z3MJkjcw+vb$yPXf`dX;u1zN&6+UE&l(NuF{9v}QG)PHh9G?MC;VEY@MA0lD$h}j?Q_n-ZZ)_Z;izXRLAWj8x`R{P@mJ^Dd%$HTaoB9|Q95>6M{sos ze<<*2%?|aG_UIX31w1SnX8q_7$ga5;#_%2Po?>wx3Ef4$cN+OoWcQ%`v!FL@&c&Xe zY))FcpKMOjPgZ0vcfawOaV)YEzqTG3M!np1GH5W@J_=5}*Jfb0eZ;^l`;dV}&}I!Z zG!k067@E2W+8Tk*fVZk0-b!qio!13?SJJ!jUU%Nd#+%iXwd{1`m)^GoT7>2r`_rZh zi}M^l{M4*#R0QLZ0m1mDzQOng)(O|spU=_vZ@G1N6L@5#MIYzdJBYVdD~>OKcw437 z`SxFl9~!23w*6A1ALt z#E+7H8~0)DB&;MHqrSCS&i>rY9^93b=fu}=Z_cpM{;?1lp9U2&n zFX|YK@V}x#Yp^w%`&#ie*aOX_K+>8S_ci^2_}JH!0`VuWVePA#@m~YYHRHq58vL{w zb0gFE?<98?F2djN+NjxK{vEV!nf{%izEqnl+&#xXF>M!*;D0f)RP%-++uy_Ck=s2S z<@(W&edvSldpO!IUK#;znui<7IyK`o72IVWu3Br?xrRr+>)~i$BRsa6_rB%fXsi6o zTfoUC*O=+QpN)+39&p2xIA`rzJaSz!t-MS#~q=!-iLL z>?rXz!>gQWCLTAuT0p%Dsb3N6QN7^*T>D+p#FPIpu+VQi+(?wW`))cjcw-a) zVqqV9ycM}Exm*07j*s+*l&QX!4!+Ik81Xb~My`DMo`<7MPrh7Xa5;8YcU-O~57Cz+ z51cil5%kkH&3k$F1k)~Ta`*i!PxAg$-aprUzZF^^X}+N@^jjmcbph%9yt+s>(bm=A zzUJX1hoI{%;2is0?LZFEH+#SpCEsKJ4Ll&(l$*pMgXl}irXDVC7xXxs=VyL_%@<&Q z3w@@0&C-ykPl0=hGaS!SFa9YMKF`SV{6$&F@|E&uPPj0%Yn9gEvDKv`;XBCy+0C*k zkZa?v_@($I%N9KwJp3U1IPkGmRpIj4Wy|mE{MND~Mi#ohB;;WW>nSU#V~lSl7oTgF ziQ7*u6Z7ne^tiGea2I7Z} z{K~PPA>L+W9QXedj~f~HdAU|kn&jGV3@o%)8(3uj%D`NEB~Wb|h#cc5ZtD+UJu-Y4 zx%H+i)4m5COQ!vt_rzzGE7LA_@sEK&bPAbPmm<^ZQe;|PicG6Zk!f`)GVKmizkK_4 zpeNJ*;L0@l9&UBhZvm?Q(sQc2YnWMW&S+9DCyUUgPx*;K^iKhItS9Y1+mesP6lHPV)X#`t?(n_9fGL zns1Ox^qnWu4x4mW&bcz}Ll2iC)0&gGcA2&>iEEc>yOOwenf4DC=gG9$KX>Gp_`{QF zcY-THraiZ+YUn!TTC6k}4j|X$J7V49wdVd+;hSdnTV9s2by>7Ob5SGH^uIqxrs==A zrN~9u5)O{N2;j~Dw=OBeoV^I&rw<*Q?&4IN?v(QDETaoJ*Yc}>O@3YCez%_Q&SNY* z`L*s}%;y3(pI`Bvct!oTf_N)7lw|u6^wGJJVPE1W-&@=$p!BhvoV^2v-n>5()`*?)+yFcjYc;{xheOHN8LaI_5vJnO??szmWNl=0ekm*A*Gt zXu6Mh6SiB8kNvjz@ayp%)B*f$_ZNO%U(>6YHFwc_hQ^)*D zGD7-wFYEr|=L_fy`Me|t&NKXg-wIjh$ppu~2bUrf&fvXIDF1FhWivinTalf57@0GI ze(gy5-JRSt+O9P5yd^wicbIQh(e?($bpiOJ(9zx0Zx!wT#N^|AvzoLP(xQ}=MxV@Q z?3LFR@CQ9SeKQ~VAp9Qi`@w(G*R30UY;9nNLnF8x)}UVL*j#IpxO}rOx(~SDByre&^vwuxk0)`+$s*>t;AEfg z#twAmmCin=v5b*q&}xb?{GC)iJ^{u=&+ z;o43)_5k;rNBKs1ECC+xuPL;DV$uri2Mo-&?>8{dt}`&#zE_ZbT40{D?Yj-kvgaBo zf6f=L0eqP}+F#T$+?dxPd<=W)%Vg08#^!pr9#epx9W|Nw5i@VhwZCW5^X#h)%(t&J zu)x0Dz(V^{AZtA1cAUYUIKooGorEh1=My}6wB~ec!lTHe7rD=53~^5$?doMsxZm~1 zEF$j7qu#`8#iQH<8-upX;L|$d&CE}0;e#^bW$=OiDNrk!q<;$lYxOLdq-S6h*;Ol< z1S|vA>RB>L&p^qvroI+<@+V#ftO3?STLGZ>P50&3OQuO~l_HnG8To}=iAsLKOVK_Q9+yZdhg@fp8rk$m;i$hO zmvW6v3GcFEGwodWt2>y5ACA+40bp-dMaMW*# zS4VJ92uJ;Im)^!coU51eGoM@>Aic-xRNo$cQDvQNJl(9l`ws zT=XGFmW9(?S=QH)WtSq$h8bB#-&Z=a=WgCt8M}vJ-%&5gp4lD_o1u|iN)?4#~CllewvUIQFVd73+jueCS!N|Of9 z8<=ljY9MEu1?iXX3esO=4L;lcwt-pp7z1;@T&DT8ce*3~vQGHT0{G7|@S}0oY8Lya zH=lf6BvrO;WLyTiWuN1gIlw^H{0z*q`vZ;L)Em2r@I%7o1W!I~D`~fvrV;n#laKg9 z*Irsv+%BK0h}S^lfnsbN=z1%%PV3Bdg4Cx*5ZVX`LJwQ{*H{g-StkfR)(A4*0YT_* zE4Eurf6@g9kPg(kVa-4zW7gMHKpV)s8v46dkp3D3pR&c}V z53TEbLYio)5A^el@bph9LAVj%o)V67N(sVE1y?T|{Z>j4ZUMMe!jV_0o8M}1zZ8!C zEG6i@E#Q789DQ3#5N;2+M}(uFO9{eBW<4YveO^isPBQC$;pqQTf^Z|i-6LFoa0KC| zf}1NG<5EfxZUH#io3IL57%Alx2sHw(uYmJ)W(NHwhd;xOCRtE5Swm&@6H?X$R&xvhzxC=Ns86|3CN1F;DEH{ULm# z@6A5;PRLYnXM<}4=irf{naEY}eU*Q=R2k^37yKQ(WZ+TV$2)+vbo_Bf2C`pW`gnJ# zw_iQqe$4QDzC8eX)mj05v6ITc)U|?$`%STEoO+)QJl?L$v-3?_uAOUOj-72_w%yY} z_V*en`)v%eYcw)!6tb)anKqI&fQwn{cJ@~%s^c z;3Lc@_hzMs8}a|{?yoZnj!$eobhZ(i+yMPXitzhqF}I}e4jI2=k^M372>m17as+$h zeV$v8-;&`+@jIA*;-RzjZ}a{)QyhXXj!|Yde*0sbS$UffA+SGS{au7M>K$z3a^5+}hJ$rqed3rBH#7`9to{*{XE`{fHMKZtvDU-}otIijT}OdaT4y({oU}(t6HSOdRF8-C zF7=On}#@NHXB3%FB#s?WgetCXK`4PsWKPu24&&jVyKBj-^kE{4be!h)UhKAQ+ z|JGUPn>a%&Ua5o5YmsB-UG{PdXYzoi>(MjjUCv=9@z8UWwg_)? zmO6=tmZOb{4XTGHGq+JMWZ>uea29Ph_>?~Uq59Aro7$FcLjusjiG6>Iz7sy>yV{iR zwtkd1Bmk|P_}wIZCw$6x>VL1ivJZ=g1mKeszx%Gf6F%j;Z>4-!_kQ0Y$o|iLSFP`a zPxM%q{A`n3Pez#O}e zfw}gX2Iko%20DIjogZsq-2X<{KzNLB2jL1rS&BY8J4K(BrRcLV^jWd1&&pEu8SB`} zTl&}MmYcv|FWvpAfjRau1GDY8fmwE|fjYzW#WqGQ#(mT+NUy;U+qa(a}D%Mh-^$@O1-cW9oqGf)40 zUY|Mo??ipJmAXj(dHSsR$npB`rX#MtOyV8=XQR8MU#A^*-mgt_bY2v_Wq9zA!^gUZ z$>^#AI5)X66g3ZM}WINiPL(EY%pIG-(CDM1|6n#;nyKp3fz4M|;`@5I@tf6JCi}$k6rC(%Ij-p>K zhJQux3!}(h>WyxRml4<4{CU4KHqGE`kp(sT9eigOU&}bv?{nqxKIdJN2eLQ1*B#%z zDR^kK%*8kDNx>s~>pw_rIFq{6NZyjK>L-0wg3dGjb)?%6&F53o3f#27k-Q-_l8-6h z^>ou}4i^upkqk^p%XHJ~4)q;UBRQ9nmhPr$EVi}`7-HH({ZijKM&F-oj0#&48!vb!$4->kXTjR; z`hO6fBP=JxXy+(F+L-~2qPJsxp^YAnOm1erF(4-xNg$sW5kDJyr~Yf4+txm9p1m?fVQYGUuGsck|72fjv)< zy3P@#uCon3$DSoPjPnoX`OEj0Jx==q*i7Yj&E9c8{{d@4K97OV)&NJtS2qCzy{(YW z8~8Y*tG-~)HK@UVn)Y^&`|KtDE5SS@wr)gN{c`ruVEG4|I+h=-v?9iiMyDD(TJn%@ zCI3!kN0&{X8M+L zaQW_kNuw2<8SfvA4|=(5k@gd`@tw22#X3CSw^EPCI$E!-r$4p+u036{AC&eXX|d9g z<=SHuC6BG0w=NrmU8;0%zHx|sMx4j8@8R7jdB>QC4B^}QtDQZVNx9A*%p>EE--8*H z?LoP^8|ekcQTgszh7W;p3rZG!iC<$cdl( zEZTg`wK?uEG^suA?Q7uIf%EP|*bQBH{4^2#uV~vb=yeKwB0V^QzPng>+Bk*w3O5y8 zm2k9k3T+l{0k{ivGH><*NviYy}7RHG12IRBKz>qobv|wIoF>|INTG`*i~~ zC;4I@+Lx_cOFn#pe9-=8$p`HPJdEE{dSW9qvI*LKp1zacb3J1szvr{2Z0-U9di$JP z7~{1D-^>0jP(A|5sx`z98T%#Mt~crI|1*&Ne+IJu&%iu;B~bJ8$KYF?SI7ZsU-Mx6 zomqr9_eQ);*hE-LXro^Ou6)s%f&lVGb3V=#aHpQL*QP)5&d8TZ;J06>2b99=tuw&?tUbil2K4xM>|r)#hBa4E9kQV%CTKJ^+wUl{pM-%95+G9FXG4fEcs zcKlQezzy+m=xI;ap5x)9lNq-yymwX-hfbz#d%*Qg;-F2&I2|54Gl_F}tPi-;!R>|? z9Nux(Nw4z#Enr}Q{hEP= z_N%~N^v4p;qWyqyDPahqfN+#EY3N78Bhxtdif%OVOyW1X{IZ5~X*aq2GKTmX@rC$= zac_pk%^4i*Pjdz*69`?2Po}{iC!WJe@&ge%Cy^*J)@Ah*zOOiPF zfO7lLo*yT1%+0CO2yl*lu5X=r`BZQ}NWSOH$rpgTHHmZlva7+}l*GXY^!XNW*Mr+# zbUc0Md-M%o#7a9E`i>#n%Drh}_u`Z3h*bJM z;(jyHjAJkRD&PgwN%XDp)V$?VlU87lH!$D+u7P>>SOatIZyT6nk1;UY9%W#bJ<`Bl z(DNnG;O8qle4!`TiQn(Zb?*9F z(-<83$WFx(5T;$-`kQnXta|v zSNbEk-PG5St&UwH`j%{M!e7*cA1F4!iW|8KKT6)l_`h}g8f=k0*w9ds{ohNIw#iB9 zF4Z=)-8jbzk+MexbAZUiZn! zcM_dpMWT0Dzn87_ZN{S5Dr^pXW7#t7n2!SYtXP&&qPxG-KPo(Dp}99I-KEc~$-|MM z`S#_aol-~kXr7VIoP4;UG08^#|H+Y|XD4x@S?9cOSrP}$LW@(WYiSba(Ch+mXMl@C zcWRf_g}dwK&%Z!-{5AORzF_xe&f{_>SF-OB!{5DZwehRUzK!lTr}K^SC;}cY`||Bv zla^;^8<=bNG%&~RZeX_E)xa$KGy_j9>tdxfx3tTHrCdwV3spg2f zE3)-5<}c;Y$86`{G@UOE4$XY+lIKqAz@78I4)AZqV7wArbA;}hF9^et z)mb3z?V5s2m%R~M>(En_Jh>am4nj}DOTHJUTOYlQTx+GQ)<5(M?S5BxIAxpp!4c*h zZ>F6^e`hnkoT2XOjQM{-Kf8C2h&LV`5!YTarS}0oE&M+4pE-Gq0A~x=1TIbY!epEI z=C6f&A6!QVr*W4}zWZZvogLh2aLa`|0uftcxJx+m7t9Be_uD!CW=<4nZqCGEiniQd<{)s#O^bcLL}k2LL@x(6tIsxivgi`T~XQzz{!(fy-`kzuD=8@txjCEK24V2=Gg z19R=G4a~EzG!R|~YR)(w88(QJOK3%3zC|c=Z9Ca*WvsFNdX5kcVOPcnn!sj@~S1ULN3ofsE5@fs8*bbJpltw|?58-J#USTYGWz zPa!nfzV=cA4ajye_8s%@KGeM^iIW_1{2v{YIOGuZn96&OeXegE`)&caPmHV(&av-S zgKGu1n>stX(^)?eeV4I*5|iDA?v&h$qCb;%+voN$724Mu*->bJ#<>)SuQ+sQ&cZO?D1-JS7j97;U0le1qhFBMB!98=!BElOG)^ zTCCN3^v91$cj#2Mo3r*eg7nGByrm1B`!KFf;-Eq3PIP&hi_^KVU-8`f6X#Afoehun z4TgsqJ}NSNB>%m!1vuN7-ZL3|9tz`y^roZ^V3#wlU?|#+#*>3&1^NmxdUUnDae>dxM|8xHM^byrR+dgI> z_x~7}YquK6`CkL`?G~WQUkB|!LbzQ#01Z4tUBn+Bk^WP7;tqoDhbc+nk8PrTS59j_ z?(v5&g+HE>EO+_i4e}Cylzst!-1S>WF7M!7k3U{@WmKwMe%|16>^ireukilK_+zEs zgFl`n{bc;{(`4T5{IMj7Yv&KgCNE@t+To7@@W)Edg0ET?94a1(op46dISGK$SZK|Lg>Ue6`x2+KQeCc)lj?ekp{`wf2)q zJmBhq$K|t?9>886qd0WVp3~{O&=qyemFwY^I_4y`g3w@@Aa#mj^WYzd*9&4R)B@{) zvQ6qGZ)MByegNH}|A$KsJ^Pfy7hP0u*@^IWhUk>0fCxGWu?{=Fg&;FH3FjH$9Z$O8<)jN< zO}gMD(mB8SWxUzb<;@$_k1lUsFFK_?sl53eaMvLxstHQ#LthUQj(({o2sZ-U5aH;f zYJzZ6!JQ)<{Z&m6ZUMNngro1O3Bs)g*H<{=)db8xM{k+-h(;g}WRaLAWj8UKj3aa0KD@fcvX(lfV&@vW@dsvb{ao zw$R-ZaiZSWnXcdSUaH>LnXc!+#U)1w#!u3Xx`Tg4d1gVQX1|iowK6x>UM>3ftIN0o zyTs_g0y`3N{zGyq{a=~%i4@q??l)n+QMpe5kN4-I|4kbD-$3-gf#`n&(f7a78BnXXPH|voeG6vs^suYwK5Zv!>7G--Tz9 z_su-({j-G6vZlYpKMc=;4xeP-zVJ5#SL9jKE#cvd13ou@;EE#hkApAbdl!EI`dv{> z{%zok`JQLxa~fx)RB!p$y18T2#lT!U(?IV3HZa@nWMG!v!N4!qo3+dvWP^L>WNw;m zFK^oH3+uif?ldyY7YT3g)X zxz*$+AIq-{%&}J*m}~#iz&!is2Ikw#3@osJ3e=eIfVS2XwAWbAW_>Qx*^Ar;eeEPX zOVAle&JGP~g3fB(y~o?|>DS~qbIxhRN3r)TDD8`TjYxcTAghW7{ImiA-Z` z$LPEWe5rfoG_JkOzTqOX&tH3oKiH61T5>EgC`uaxv?~mh@80T= z({3jC3uwntWKyk1!{j3x?ggD*<U^dGQ*wzM$6`x{{c;W0wgy{|44zsM)tS0}l@2EPdFX~u_MrMT{^gI=5Q7uF&x zev&A6!Fhg?n_artOug9LODa7nwA@sK=wR5)~ou}i^;E*1+%U5s4{ZYsD%!co4lOTjGwcdu}aaWz4Fwi?_# z;izvlLAWj8?gDpmTS@hkkHooS`ZnIvH{Luj9sBSWaJwnj@rO9}P>JZIqx>Pq`O=+p z?w|7^awgmGbAf$3{iFY~VGnIT+1{R1dnn<4bG;e!0(%N@C~c8HL}Rb@%WF(pjy=)9 zZ2Kw$v+OGjEcjCY`B(W5;y%^|Quk6EP4V?b=wt2QsdUS~(7+sfn1Q+WPy_Sq3Ip@) z!3GxC1A&_VW#NxF1iigMc$Tn~a3`V0^#L~Hv#!Ag61^@WJ_;MC92=-oXFl|d4P<05 z_EUIGb=a&&!uMu95}GhF4LXV;(?+46N6pS)A5Y4&I|R9T+58#e+9{M&Hv$oO1j;?YcBq^+AT5{KT(Iu$Gmd^CGUbViTx4fI{H zRlvTSpM7p$&Qav!A;v%7-JjFTrN0^EZR96;7s2N$uhuK?{*iGjXPL6T^)Hm= zbIZEID~o(5h{kBUWS^gTU^+HMCAbRuN;sWqG7=Z#6!U(m0Kc11I-`0_b{lJ1e$XQv^XK<#bjRY(IT_2_8heJ1 z4)aA0Rr(^)Ro17`53<8;jXB|?)4<&dZWFkcyzo-qZGA64ocO?s>)#x~c^#e#&P#tv zJmq_Er1^KS`g0Hcb+12tk;btRy}6fpj#yqGst=A%* zdAgQ=X{!wBALXz0%Kv;&|5Y zl#ls4t-sFlCtav-FC=+c?h0*gSEa5e>mIpky&GR_e|YM+U|2N ziZ_0ffBKld$TE81I^liSu6=%|Gi?_BPFh zhYpp>?!Nm1V}CyC{L5nZ6z@M4S@s`YKA1?o-@2{KbDVo0s_{`jDDTTmd-P9~_vamUNRZq;rt$6o#*BH~xFggxHLa9?Ja!@P=5%Y&4tyt@2=@@~B2*F1B=`y3tGZs_tp$b% zw-PQPl=;*9;a|F~gff&~@4upr{c4NL%bDN#u-iCW`Vw~=T$g5LM`!i5;@5QRqB4gi8EDhr4nyd5i zXQhQMC6CUeO+MevgLcRE2RY;J=5vmdPZRkxwfG|3%hZo~{`3aU7bhBhkwmjEqVJpD z@kJ=t{O_B+o9i!yN0m<|Y0-DMcV~9q^5}cWW}v=r+QdII8E5iYKgnk`*DJp?tGO0v z+V{!%p|7jGjE8xCU|6Uh&)N&zoVTK|fHtXIm62MnYw@j__xlV9ox%Hax%bIDZy6FQ zeKINQj6y7uW)K!K8%9l2l@XkTX*S+Su zuAYd@;;ii^*}Qr7!M}gD^dQeKYKe_E_s=DG<-AkCeQlT;$K}R|9ki^2YDQ*m}v6&VA$0rkA|J}lV>|G zbn0*fdI(UyzCYDBtxsFrLs1oPogG*{X`sa!_;1aOGIlTe(xykp`yzh&Wd;7EmnApw zam+zhTuVGUaCn603ei)5|B}{_X5!PwM9Sz#`EsCxY(Wn%yD{}S#yFIJI%!9k_Qsxe z;_6SnA8gvs-txz2|4RPTX`a^(2tBOlQu>^HmDj1-C3!13ELv*Hs){$IbzanjPH4)k ziXTzD8~skZcb5B>b05a^CeC-wO&dM4#clg$x7|&}Rq=VmXV50Ky*F{jdhlW7v;Ifw z)$?YJ8#cSjGh)q>M`%Z69rOBXi!FDmD4;W%bGA4Ke~7)pl|N8 z;<>YPma88mTf6d&-Y>RtoBiyiP+tfyx#Z!i*+0=he+ZwJf}gKFMVjl)SDqQmFTfX} zK8n-6Tj3$2Cy?oe_G&9af4x6N=LF6R{fTG26ZCY>>5Kzq82QoLkqyV`9CU=q>&f#% z&-3kMPyT#oXy{p*%#`JY&{hp{h2#iBy&A}s^eMwQ@6KMQXX+{i{uf$EaZ0Y zV7QVx>fD0bU4sAfM4dEuTVlds>SXXKb#3@#V#9ghC2wNY!y`@2RT2J2-><2$D$@K; zRb;d;v!8k1p?Ge;=JCVZaV=GmGkJF>-&@REMGLEtog0q)c*k-{Ir z1wH(FM)#LKMs_v9a}%y4O=muI&(d-4&E-FT*2>$w_kQNSr#n=14nqFN=rL&W7b#`% ze6ljG=3U)eku1Y|Z|?DBEO*~i8PcU@j!C`GdnaM;;XX_KHwRh7_XAyAU*Tq1@%psf z&c*1B$FZZLa~zv3Itzacwws|@{*T^(oY*~nVp#vMZ(MRs_=nx-tNm7FMOw${=-3HS z(q+3>6E8{8vs?aXEDm&C|d9 z8*(~xi3#M5jOj?e#0R1${Z>T{{?=MQ_tfxzXEV2w5$q~cEzC39lSdkTSIl4r}ppa!J)A{s}CnLPAh1;{Byu_c0tb{O#5u9Y3rw*rbm~|EqsizlYK0?as=Dr zE&3)S?>~lOS5~lpXHWy}9}XR8&-EkJL*w{>;H}bw{PTl&fcML=!9;fz?z;=YzfHd4 znIC?d7@Xk!L=p4TTh)i0(epo67%Qq`ex-O_VRR>S#`(Tz*3!a&FUQcT?9u2>`jLJ% zw%fchk$U_sif7!ze}X$$p^SSv#y>|N>`hf08K=1E*9AjD0qT?3NgbF!C0@1g9nO6{ zF{ej3vBrw$6Ng`4XU+3<(b9X^^oG}WYW|l#{W|)s9DE1b69ZR1J21WpdwCS+cABw& zo8L)~Gz~-Fzn32Q!Qmh8I5Z;^BF&W@BC-3xtpay>k5J`D-P)Kt zuhrZ+K8Cch9U`5f!O_gGo5pvDBqs3<^O5GOJ49B{2FY{BMhJ{=eii;_%{{TniX5n% z7(OsQJjxFJ3A!8GcBYkDL)EG-`8+h`^5JzzEOE~1?e zrAO3X%}aP^DS1)PTFwJUFHeitfsc+&i#Lx?iyvIayBm3DKHp9vkBV-gV%l+RGkx<4 zeYlNsc9KrLqdS1(t0KsjLA(4D!)C7ZGrHP3?nnL|~Eo&ayoh4M{qhZTj19ahBSebI@PQ8iO#MVHhSs!s4BD0rZBfwva1MR%FIOBGg*KY4< z&Og4#J3-~m{Pa8I?eHe{O?j2EZ!|CMgN+u~rXCqO8;i_-sW3RRq@aqvaPFl@ecyYh626f=)3lK_UhH}4=zy=NUk$twqdzO5 zHT9=tSpZw7mO2?;qYj4Gvd<4qWo#vrcs64_nP&?*raIQA_3Dg#ldk2QKK_%_zCQ$d z*&a9>8410`;3x5g>@U+l>(39J%lC<~=+(<*ho7KdAL2Phzb2TodOEgYEcQR&icW7A zyd3*q5Sw7S;H}6wLFnQZLB@ZYAme?bAaj-*1Rvs`KZ0x6+ivjr_O%8U*pmbo1>@f{ z&xQ8Yg0q70D+T8SEuyz4ScI_`6kzq(;r&cd7s|v z@ZBfSh{j@TOQ#|D1tOh&);QU{vO`5*qT@>JQ^BT17V@!kXN`^aTM3s+FM2lgKiit# zvZSui&_SOILVx@LdKAB!Ll+l@{=hTqW|2C7ma)BhV+YHI#OFFe^dveeX^)@dBR=s) zY)NAir&I3S!lu4gg(rPliCszkTz-qe%ay(oQ!nvz63=*6&g=e>sUQ4YSyki28Q;oz zot-%RTsd#FiQ|Kaq)QG0#doUf1(dIOgy=O7-yc5V>8gim@2ADn+o(q|`+H>fnsx@h z8Cvc3v1Eku2|)L;HSiAO7R&m1;UohW7Gg_n9}i#7K}N*078X{z`LxS`;+}yHqx#{a zaq0sUZxmDRktL57zQ=dTa=TZC?mbDlvgbALG-W@8?xGy|cP258*sT+q(DSNKZ{iuq zrdDiq)#q{A(_W7=dgf$|>eq&yWZJl=GBoWZW&NFW?;Gh;{<-Va8>nyRN+%w$LK9E& zE%veGY@}(CkvpB6>909%KdFAH{qyIqFaE(Xf8_myKk^|o{-K{dJ7LGpx6YM(p5(J`8%>)6J?>xD4>-ER{mVL^`R8Q~ z&~)ZqJ(hoKcDLmZ&hEPW!)nSMNgmMshfh%68p?Q@yr=jhi5vWp=tg|Sn|$#DFZ$vK zH-le9nGb=7m!eDjkq?&o9UFpqKr9oRxm$XqB@0{nMbBM z!ro^=R)fE{6M+a~f-K7OUav&ZgnzB>ZcyUIiHT=FLW#euWaLkofBxBElI%nNl- zjQIbpu5SHbNDpP6=H&Ul@*vNBf|RMe8i*S`_W0Q$)<=x44P*o(7I%keO(e#c9ERUS zKZCFrqwt~RtB*P7A@r*0qwjPGeeBlz4Yv&WB%{y>wq*Pb?ju)OTS=3C+N^iLKk5q@ z{2yHW?}ew09jU+ft>%(mTzRbL9m)DSbaoB%UE_nGK65p1Lk`FvF%MhC^v$WDi2=sTZ3gkg@M@D+dY4WWNAL_kqs~wXzG~L)v4pCE}^FRh99Pv zF@|PLf88n6M7fi&XI1Yr)fcqA9y>;Jn@Mgv?xcMEOT)-$%8>FkG2dj|Bo=$9Fd`y33cMavZ-;d^deHXo)zLkH2{=rT&di7=A zS3eor%=wR@y$U*xmmbodG5H?kmq>nxK8lSE#>1R@lAU{_AaqtP$au!M$JPAb{?;Dh zW7t`3*hD(BQnsN}g>;eRX<&9nQMB5^=YUU;wKdtQwHd~TvN-$ti{n3lM*gt!v!!#N zNep@eJzD?z7~LJyqyF{GNMe<5XmnLhNcXTsH|K<6jRf}6Mdt&TD9(O&{HifOq0}Fa zmIA94=RfX!uO_f2)OVcm*EO)txQlfk`L`NB;M@-N+m|;ntaYF_8E?1T@2cFA-?Hy| z%0-btPw4XoAau$)!s|_xHReY*wyoiO=*^s~xS8~uu}ja=GxMGUoR8Mt?lnO9hP3`P zk3BhRW6PtTEq&=+@?$Pik6tnMb%r&4^9_vU6Tx`h_3$-oJ<+N7!+|d`hF|kTm%Lw_ z$=(9y1vBpUhpy+2*(0$it#kIvoirVVZA5$%y}#U9$jnr_}3j@*1@fisuCo;zsu zjd_2G^RC|O;=cEL-fLhCuV)WQ^1Y?JhrZBze)qj~dha2NeM=6lv>|UNaONbshOriW zi1j10mnYxe1#UObjrcwG?r`d}itmph8=_0-2Yif^wP%4j_ch9wetL(w!Aj{(=EqkG z*OzrC`o3Og`k@#4PPVN1=DVbO^F(i+?&i52Ya%p?^vU1A*4^fd$iL><{|zmtg(fX= z>N7`WWclJ&j_w<8D_u?VpME}Q_XGF2l zTc9<`)?q#2XX@>PcQ@ z<8YGm)$Keb-tIUk?Q=&7&&cM4m-5D(8;F z##}qvcTOl!UcP(^`AylwuEA^lAz$xKA+6Ic;B5U%k0u7~S(O_)Y-lcrJ&~e0?x%w0 zj%*SgahL5+oxVJax=8NtnV-vkEbhV4dxq9>xxdnRr&Rq-|LmEP@{Z{2>0tbi?mLCN zqxDsf#*R{#85)O0*h;_xf@%MuKB4XOi|9jR9Hq~SyP&6-J51KPLQ0=UcMfOWa(KMy z)#33q(9$&Ql>;p!A_w0Y5qW>iMUjJ>I)t0nAy1w_j!0%akrvjve3W)ZpFoaa&#)*J zu7&Q+*gcXRl7IJM%2(N@4$y-)-xQ7g@nNUU`!sgYSOY$$_n{NX$zv|+Ke61hjj5Z+DI0KkD&) z`qLa69%$Zqq<83M)%iUuesJJb<@LVP`k@y;EWa{8bQ5s&;k-~Y@}GMe`>jCdjY8*v z-#&V9pU{i$a}zf8XkV{>oZ8JMuB#_9zHR-^Z3ZwpT6jNqiUWS+^|u4*Rg#6B_joH-4%A;_ji|D?VEq z#P(SD$Y)C}pvK`ccN~65n?%>@pPPuU;~d9xe3QZcx)#RO=<42_iPQYF)v;%i@hZj% ze4zWhaqLZHZ!R*%rAw`2(r+JH-)C*`G-uCN0r`vWO9Ll)Z*f-0=dO2l<2}*bpN}Q- zy!_-NIZkdg^ay>dHQN&WKa?$-FVNkIOJ3oBj~m;Lx|hhNz*dnhW_fYd9eY%Dm+h;3 z^qtA$X})D1#=jk=Yp+NRbzVpqggrG0Tobsa{yu!6^!F0C9PF)m(5$gNs;rP~Hm^*r zff@WRU#O?*_9gjiUsZET{*$mlcX5U>0nX6wwciY>KN_}SKhalO$E*J(^6b{cpze&Z z^Z|RT29;6gDEw>w=Z9R>v z7g`%ceOUvDjACt1>n71DX_06(@7<8*tix0=4;l?93Xt0`frA55+P}=WJ-SDwT z$6N6zej|Jq@hE)rFahZ;9`4T*<#jm4(e$hp_UtE$7l3E zZSomfFDgTp2?l(@p-yCd{lDixkG}xuZOR1kHgGLcwSAtCmO^!OVB2R7rDBS@eQN<%3a^Dd_N@x@_`5O`z{!lVck2<@T=qkv{*|SqRC&Ak7yFP zkPUy~f29m;HSO{FC21OG)qy?DuSZ`AI%&YAq-n2{(%_xfqnq6{;9|FKO2hZ^y2>+o z0vD2|wkj=)H0=Xf04|DeG=`rv`b5XD*25au3l=NAEZp)0cYSisOM9AH>Fd~<@!>=0 z?A=qcxGOCqvZL}}3ROPO`-Wr@LQ#BWe zW-^D-cxkLW|MQbYp@$i--q`7;4JrL{1@+zUwxN^SN8Odj;W@r?%~@23-5x)@DmyC6 z-b3B?Vo&W`6^#Eoyz$aB##DT(F_ldX98SLK@AK)KD7tNH_kW#dWeXWufIVbn1NqkU zcJlA%=HH32mb~-i;-I7dJ2vG9I&%!~oy=y+`!tc)K>yy#8h&@mJCUCJC-x~ek?0{t zzm`jm_dQA9GG8@)TQokbvlHFv*Y)hxFnOnp*8y~de7T}I{G!Lv@m|uzM@d>uq2qT+ zJC4Rv=y)e-C(`lj;ADqIr!coQG|c_ahKBuN(XMFt2zRD>v>RJBK73d*!lPMeb&vMH zGq>6Yz3it=GqLgJ&?f09$%DUXuQ;~Q)*jX%qYHLj$lk9s$LCSSovprw%W{11541lN z`aL3gNTG##SB5`9T}5mCzCeZ-F;;3L^N-`?cs+9bFF(hhbE!4x7UmKEzvTFJw5J3) z{sZLnBab0tU0chv7kRh+H|_HLuf)HKJdf`1ha2AesxmysesC@{f!ww(ab)Sn%(#iR`kfxWgmIi+hyU5=Li~W9?w8((ouK=5mw{}qDEdE^ zqW@bM&qEqd@iphsBsT^zmMVKBBHyHgla0x_tN4ks|8T18hhL7Q$o@mj z*-~Zx-uuRc_i&bEA9Kn5@X~LQ`D>wX!}HWHD!nF%j#ay4_ll<@%t(FoiE^-1bcWJimgE_xkpIz3+$Z5 z*5_@X#s3So4|BTHjqUR&wCM88_D8NeT|9&TI~ezP#-GA7mcujKT^oqAGsoLNB|~pA zHqa?}W_$0G+C4e;OLOq6z%vcn^MwuG`V)s|@=|yvGlgfwE8-iC&5jhhti#5WU2-R5 zB-+a$&5SKLFKxOzwn{U02{elwSg*awH<2dV3y|j7B{fQeKBtf-ylfIdl3`TbEL;mhx6^#FMe`4w6vshpA zc$Pck#CIdzdd_k4R-O%g{o$6~ODiRztoT)neSV!uo7Ujhx6{F^oCgEjvjX!xfN z4ey~{Ng7_(PQzK!bIAR1zfPF7+J3U6hC{;_ zyEOdo>T~r!V{{d9BllSY68*{_ps^7h_a+W)jZ^##;)ecGCn!lF74b7 zO>Fq>=V@p2>V(-pEZTXA_!rTRHwTRFOtBe|1Ma-26mtMg#;t~mp{4&M&@qdK#WX7bcNvAWaQ^X%@E z--Vw{^X!29LclHPK8^F))N2X;$@s_^n8w`^ujB-q9W0{f{ovS^GVl=i1kaGg8(3&y+J#*TJh+ zKW8qH^>@aBd{?@84y8T=@uR8CI?YjmKXdDkt3 zOIuf-X$_Km)7(kzmEWRHHURKC>ZAUX?clX-tLy+|;MH#1biPmd8s8%QY~gG65kA+? zsQq=+CJ*0zU@f)>M!+M-`Rm8 zm3?A=wNd6JZa%87Rn_|p^;fc?K*k`%BXz4P$+ua;}m+-{7w%Od@SFX|IrW18LJpyA}Kl z!X5ZFm$OEvKAfM=UeCOz#FJS+!^fG@{$t3V6ZMnPUG(E2(o$*5jH&b*-ybAReJ0yS zw6z<1-s^)##>mhYePQT}v+MonH|2#MpRToa$pweMbNfX*TSE-q-|o|79#xU?Gh|UW zYmm)7OgfkUKeg-oo!Y0l)_Tj4#Vcl~$l|x#WwB=yby?M}*E$pbBE4qGx49<$mgtGH z_$%nPKiy?LG7wv6Wr-t;tp^<4Rzm$G)7015#L;0N9riz3m;wKqb(KceBUtBJ$vr~y zhc`oS&CuJyYMl%CJiQ%&-VQ)-3ukvp(wp`^iRKQvH0R?VN4wU{z0l^IL=$w^1l=`5 zcL!7G?!cGP-7g%vi=yX_KtEb9pXkz;WY(1~9SVOL@iVDA_O`PJm-du&bk~+8Q=SOM z-=cra+P`EyYs{ngMt*s<+0eMU5PJEcj_mZtJ-zePWH zlfT&~fqz?aA!nW|vj;fu9p*jF<+;no$Yt&6h;4NIhNCntseVI$7l-$EO7?N*iqR{* z?ETokKK6;6tZ!<2S`|H5ySCU5@(g$Ph`+HrK0I;WZg5|5&Ei!29aO#_mKa;>3Ho2Q z)_8C;J}wDu1!~@&>w_Mc^W^+0u~c(l*{wZ^)1PbIc{}HJj-R(D`TNxqY_0c_w(HuH z&)swF*Jg3Y5BAku)~aM{1r|B{T}J)X@8a)X#NSKd?<3f*7cp+)3-P;0Tfu{g2}9W< z6`fTTsW}}QVZB~7vQ<7w;N9+6_IG?X=G%efV{|bzXyiTcPTo6C7cfVv6P*I5ljg~L z(Sgp>MEf$QuI|Aa6~6h6Jo}luL%Xb@CFj=8{X2Q~K4@Y;cE_M*c3(rIvguBg`E?(3 z8lv-*-k4a7$tl(m*7knx^w3=TFexM07qq$``vg;TON{=|zS*tv3Bnrz>`>W`!j-Xa zO?>FJeJJhQL%*wyx+lufW!#Ad4L8!Jy)E#2BmS=)!FacWjy-;2{WTYxxRf<<_6~f? zT|NJoG(DL)f>rW$L(?DA529)24~ySC^Xo~P_T~?hA8(&Oe2@57qUjTLR{Q+n6f~V} z_o5#6qNjqKr4UVzzu%$h8Q_gA4Nc#mICT67dzR#TXFi+#20XB!;k5e##>5%%7lX^5m>za?O#o$hEyE7qef7|5eR&&rqXF zzCqixXYT8QwDm0Dx)%6Sb^k0eNb58mGHAbem3dIKlOwAGr`qp|4AS|KJ_Vy^#x^>1 zn_(>&*O@iP(X2<%FSiY0ud>JQWt|;oLsNkOK|)izNpf8DKd+6ZAmiL-aR8qBo*cPj{ zEVbI&&&Lp4Zj!c2S&|FE{GP8l=Y|Vd?f3V6eE&F)IcH|htnY2!^Pcw@+SDa|A8l%? zX8xc}?J<2H^JDG~p?&W9C*Apww%q|;`U>qbYXERQavgm-g|Bhni1*Kj%yj{LZ3_8g z`py`g92D0c_6i!YExQCeyQmo-(M;JziB{GEPpXIcFtldTr?iWDzSi8e`aDzxes8F6E2?;p|LoIFOBW#(O`B1&I{er0{fsrwvdj6-=etzA6dSs6 zO6P2}`R2UhXr=Ql*&}$+wcifjBs-$u{E~~Xy#j|hUwCgtzZKab>-*>fLi2+ct;pV% z&G~I?XYzb0^Irz9)yETy#?!Vke5~zkkWV$!riL%t1E`}Cew0t1c3@X~|I@$i(H7qC z;(ePxb^f{^l-}Dka&7j%$dku7ZKvEk#_dV!G5xuGH14FVRP+ACrTB5$7CsTM?0J@J zo36f}ShS;|+=@WgQ-4i+{-`Jzr_q1?`X3 z`K__GWHg|XVJ zu)A%;tp%5>-V3Y+=hG(H88L^Zt+MZpUE8cn-lD#y zDeM7PgR{Wz_{x#?kHABnUHCKQN#1<{xU5_`5nH_!X9RMU(%v9Vd}*Cz9BBDYzK;^h z=c4GQbuBs@;v~N|Z?o+6_y}7*vu2vs460B5bRYKW`8D}OHwFLJLldK2DOU77@XFjn z0NrST7wDc``O}oYj0W;=yY;=bvNxr?2}@J^3z0?sxma zUETEMfZD?T$Rng_USsq=NAfIX&!r6Y^#`P@tcM8u0AGxq&I9f)czYOpEXTQkc+oP= z?dfGq^U7F89{GG&On8E)yNquQH)Z4!r#00#OcVzkGk?OFkw#sHvW%bzO%>Y!+d^^{|xi5 z_x#T=|3spJc?0KL6HdbXu*4X?@3!XkJPGr4J-T7OqRaIGGMGM#-t?hg|CR5;U)a-j z1E1Z#3k&gGxO4cP*d5XG4=BIehhshiYqP_@`qRX4cxY*KO9tx#{*C#3YN*ZV$Ol$h zes6K+4_n-Ou%VH?{5C7f+9P_L?@GQ8?X;rRe7Ey08olLdd>aGHcM(=4_WTk)HZBT0 z4Xr4&7VP>3zGcVx7Ss`j2jS=J7u?UvBY7&a34hz-5uE+2+hh$(ss43+_0r0JS9-Mn z#va&RJ#5KV@R{JIsg1L%tpy1y-19%IFBku}wGD zj1@ibfe+0a#g{XkD(<@$t%%P*#>;=;#_eP0lv>{`cze#35zG46-(%<40e`>Yn}G?L z->mfFd@Eia>wkUDmCc1UW2FZ@z5J`a@{PWZJPOx+?SZlDX7PN5XBy9yJX3fk@{H&C zBG08fBX}<6`2tS~&v`rpc+TO;=1J%2&(nt|iKi!z|ADbhg{8Jtnqm94qr0as5%hKN z>Ba~;A%#a}({@!^WBXJ~c7P*OwJ!io88aUk;asE6AEQs5Hk9Xf`gA8cc6~=EH&~Wo zw-6SW4Y8$9YGa(|myNUk!FY=IVw--j6rS?XWu;p;`8q~!^X1G({-4q3+FQ5&k+Up) zuDxLE7bf32>W`l!jx3pc!KhoW9k%tMo>srZ_>w#bUk8SF$Ib>E|IiAf%a6j}pDBn0 z?PGVlez?`H%c(1PVa>GaK+grY46VyHaCLThw1V}LWTldV@~8tGB>R$Ht1hz+*3zbx z0pzA-7sAVH_WldMoFCz5^DN3}>QTL^xp(O%D`DJi1w7+<=JGW47)ShlKR;`{b#Omt zS2l5u<&VBhds~_BU;*D#u-olT`P;@?2dnT?AQ`$9S*xTVfDiAilDbgo4<4k>DZUHs zYVc<~{w45FxVH-aFdv;;ukH90y5Q&alkxH0%9u`JZ1EknGJ$f6`&qiD}O^KSz8l%9HRa^vCG ztf)iU6lBN>%9&h%pK@fk2==#=SLi)7KMA=FTCpGjd2VLEy6hnGT$evM|H#GogrWQf z;G|Ogv)5RA3vnQ6x zc!u$JU|^`M#GZ=X@a3=K-;XlZ`g*q&`vxb;u6xQG=%p!ZHe=q-H~N@vm?>iZnmm{h zED2_?7wv|b4s<*QzFrw?pH1F;&WNZ#=hL3igrf*!>1lCg0W+p7$?Gxr)49@7K5Mw- z9-Wzv{g=ppm7jEN2@>}h|Fy7gp(u7CB9V3nGpz(P7ldj?F zi@~oRlU#pCJrc!pyXjSgvW~8|!pM~_%^H|wzj2rN88+2~Uu3;!%H2Ow=U_dbNmhsY z*ZmEz{&3@0!iOoVp}Z`N&yVOYo-Ys2=02f1*}^@#Hv8!u*==JV;Ln*KVLh}0{Z||K zSq<)Dd!1c%u{EM|uCMXXajWrT&Le*GLeKC&$61_Pwi=Jk#ZTLM=)ZiQ*TY-;inorm zn+UDak;X6EU9|Nm-+vjO=*1_R_-_&4o?L3TkI%512n~R8KDeebBP z>+Zd$W81x>I`&ZZcHZCR{g1pi@!rMzYrL-|97q1yJei3WXOnv#EPxK1F@ARbCE739 z4@{WvmZ{t&U5Y$`&H$LzxxtQ=$UpV$hs}j{!sB|bcXfM$^+@0m_goQcr^<;VR{9d* zhpPWZmroFRw~^WlpX7?}c$%%E(w=ThQYfR$ok3m=IknFY}T6gdn^osna_kWpp?#f}U*T#FWpz<8_lQR}Q<=S(}_OS&&!!6e_ zHcOW5YXNVH%SPHmeZiIPd!jvB@8C-@ylSN0!H43qLH1z1gA*ktOtjfIT^V^g6mCVw z)jE^)B4aSObe!Ej#1}qv8@hMkcRzj89oFvwzwZOb?mld$Gwj(*clY9-SkmJ-|`F2(aqH&>*|zI`(Q77H`%3fVUTE zj`I8AZ)iBO2k=|TTBU})u7v?~iiC&w#^!GGpU@3Oz%TVZGF`SN?)vyu;+u&_D0Os( z+g&%OI6H~|JMp6F*9^2aU&3SdT?<*qT;`q3Sw(X*fh zkxQPbZ)N;bi>+`dkg>8okmQsEa#qgwr>*=4eG7%G=syExjlHw1#+H!P@4MvB+u`fT z+u`rXE8`h%HLm%p6@HL>50R&c^tGg~C4DXFYe^Ta%KfqxP9~4aG5@Iy($q$kC4T+( zlkS`sFy|547wpa0LIWc6y>W&vi0;JbT?|k6H$z9r!+DBfDleaNzQ3ic(5R|&g6!LU zQLw|>0i7^=LYy`K*-XxZaxV1u>^XDB{G-t)Vc@;OQhRx6hAsJ`m3sgqAx`Z`IV?J~8KkQ}2c&cQ;b<$Jj_p;|O<^xDih>)^~GZqJ1Ec zXzyn{-zj(7ydD0J&fk@-d?W0p1EnX>$3~NUNgJnJwDxb%>u5n*>4}1(%5w_RtP{}l z-NjFyYZqTJdZqH=8)##Yd_~GvR(2wveEDU*6B_$U=J6Yhy~e$hF+2=xp}TFogFTc( z;O!mo=p*3fmhpq^7;kO|KRdwP%<>F)D}0r?JO(YQe4w=BDIVD`%>mZ8^WN-zf0b{G z`^X0H=#HOj`PN-zb6C%)9q_Rk6KF@02LporB*r8HELUMaS0g=g#bA32Mk(qZ5I}rVmR`SZ~i+8hrV~y*}?88hbb=niBoDa#vpX_#IZ*tnJQXpPBi& zk21Su2KxMn`b@Y5Ipiq){LA?AL@kY!uofyMB^{DosW7BP%3A-erso(+8L+g zjMF0E;j#0qXEF~I+cEkYgbz#B)jIEG_*V=DDT6#v+-%94;`ZMawbrLAcGWF#; zIoJ^-5++HP*3oF?C@Ge+UJv(_Vb_d)%&p4$ z6=`OULYFt*^vTqH<1VodhL$8xAMp5<_ttTqIs{*TkTIVe>ecuv_*Hzc*dDsDXzTy5 zE*H-zWj#KNGX9f{FJI5Qdg5DPV`=ni^5I88``9B@XDDt>DZTvypY@2*-Bd4|b_AT> z0_|IO^PL?J@gGK1d)5xHo?Ql9rYhZ|kxIY+KcyNI!;Vyq#Tjah+SwG~Iw0q=j`j_#@yU?a~!sCRC%KO?!pz&MA53!HYo>|Z~aBo%6 zS8V5UPgEyzcQW(WikvbT{Y%@!*ibDj?tq^4t3ak$RA|{-23WTAk~Q2jxeHo<7(P)2 z-LYM`h=sbpAOZtF00NBcfHI0Q6YTmPqV@r z`~ARZGix64Qt{(m-*Ig#a+#+n`f16v?*&+g$MPaqDc^6reErF%y_@$A^gNjdSUvze zZt?Pj$fG@?Yq*PI3=fyEKua>66SqTWXmcfPzlS}5qkL?kAe;1+nuyQ@F|>1JBsNEr1zAp*Z?nl zoX6m5$kYFX67dnonJ3ELygzYdtX#>SE&R6v*{uZ}FZQLBgwfaJp<|i@k8}4L#@Ppe zcdhT_H&c6#@2#E~{$L#bdKa)B#a}sc)id3?@PN@Ntt!Be#VG1KhzzT`zsUQa|0i*z zbc3QB`&Uj3AE1o>jLY`_LALh#8uVo)%V%{FIB|&a`tes?o0<4JMBDyb{cr_(rsFqq zo(owYUcRbed5YbiwnV04TL#~K4!;F0f#FW9{;r;r+rse2(D=w+`6M_{xP7ell*2tU zHtM~Jv=T!Pl8g>AmQQ;M`1r0bSW*6CL!+?i%LguQNXYzJ`KY<)Sfg$@NAe{1)t%6p z@|L;WJ3q+Cm8%MFNwfdSSZY3g!Fa{@RdbE+CF$u_6@1*sK0rRfKn>&gAM;&vmi<@q z3C0K0SK&&r2{WBRg!(_a2w$m`@i_VN#`qRAAp0D?>No6dW0O%Xdko7FT)Oz2id`E1 zO?YIRmU%I>oxKqG`1e8EV{|S!dFaZ#;vS7B7L~Dg`m(W|=&IEodII~WJ=jO>$$n}g zY3!db3Q>>W?qIL*2z!M`y}iQN-PD$8Z>sZ&=|`H=NvN~!`|#tkndecSSEryKL3gHm zdH--FHYU95Ufx~aUA$9ypS&bopUvHoCG2CgSv5bH?@M}Zb|AS8nPY56XW*<-(T{BssQ+_bse_6AtROSJt3mR+DTbxD$;Qk;dJedTWYp}jxdh4;FVaWZ(%asuof zXl$SFj?=loOlKf{mfoi>P}7)6ol)w%B!K^?uaBztFVGqWKVSSefqaj7`Qp=bws0P4 z_NS(ChX-j7lD3O`cVC@?kEtca^~Ib~)_t4euLG9Vac8Pq$0+0=trslpvGMh>Nq17} zKkUe&&9&U02M<$Tmp*4W->0v4BM*ueecHXe?Af$cf=~J<3)-)++RS%`vxHD~a$2Kk z-gS=R9+Q^p+-1TP=T0y0iA9rvsS03g5->IqSi6k9`U%|W=r{NAecm1E$GPvcojcMy zYio7H0K%oVTK7a9X657z3B3NHG^k3PdZOKzO7=g6a<#y zeR{Prr!l+m5OaDHbg4Mt!8YrG;_2@DM&#F$sVxKf;k+pOIJj4kVUFg)hHRJ$$>iHTVVNeFO1Zz5nH>Eq+cr z`<8`WT+MM_A>S{||GKL<%@Iv~nf|>9&u+*oZTxjIJRiP?ZgD{kxRC+;Kf!ukYnKYv z{S94RoBgb9Zy*#O)&IZbf9-3{*X8@iN0*{lx=Vb7@7x>*9p3w(e!2otwiOvUu&?$hg$Kf5v z;2q7=t;QJdXw2jOZ07d)zNOJ`Ql{3Ux0o=^x!Hv2PL&BWoNt&g)0t<&EawJ7*%dqt z&fdxMWu6f{nLIe-5)QBWG{1bzclX(j7!FU4!{NzsI6Nf|ho{Ej@Z|poI6MHjQGc{< zEb{t#o(VIYfhJ6M2AD9-$v0uDbB+m9oLm#?PU$Nd|0|e>%bAa<%*zz$>*O%=Waw){ zA#yRe3NA-=k9`AhAfC_wEK4ud8sJ>@jTd#SKJNm z;rSut0ND*RQHSCV$AP=|h}+y9$2tdiOyJ)8r-?)N zhg=V@b?HPm+{9=E@^*9}c{eC8aFfS=Y0#>}*VFQ81MpcDLC4&}ebOboH&9P7jkt{o zVflCq;cHHQt+@Z#v`>ARBDw^gwrrdb*8WKF+*jAIM-^=;ObmCHTlhy}p1{|dzVP+H z1Kca^vk%Pm!OI!%I6SRp-W2!4+pI@!5uTO{PZNQMAaD@^K9YcwWbm{X_K^Ww_Fz%? zs_wa$J(%v0?TF*6-=Q4g=KUs2b81bP?kqDQvH+oAcPQs#QHXCo!dcb7@I0?GtDl31 zx%kJdyMuKEcpC%{MNeNtcNo-o@;;yU9PqFjAMMG!hrq*e#BYS(2Ejw+naaC-)QS() z$e!e}Ji9<`gD#ry{CxXtzO`p6-*%E!l4);bba^z$00p>W?a&p5hhG?hM6$c zx!8m$&V?rY{5#-VfsF=WRPvj_QTD|Qeh#u4{|VmyvChTMb16^wDIQ#4U|Bpbvl};i zb;r3rEC$l|b^|ZUCmhT&Jow+lMS=e!@GTpdKhE4Ok_>z6rJf9Xk_UfCUge8Vi}Bs2 z?y|vU@J=}Tj+Z|Q4?g7|`X{{HM|xzC(O(S{4UF;J_f6aR1`mWI%lHn)^D=Ov1z%;N zJ)I-Lkr*#asTtcj7QbT9Cv zH+a$qJn0LboCTirV-1vG7v(;ArToT3!Ixv;OS>nBc7QKO;_&4K$`rmdnlR0I-h}DS z7DD_=PCJATxj*x4;rSuYz4#!$mIr!l&Y{dsv~FLAOrg9Ox~CpzZ6KJvuG&Anf;$_J z-`KM_U+00jJADOtjIU;VHy^*TS8)?}r;A=h3WMRU>Ja{~QZGdgyD!yK#cC3I;e71df;YM} zFq!ulAJ=>*GT%YwHqS~p7-YVSDXTLOa*A0qj`QWT-8Z!2SS>mM&GU<%UN40@$&hs&YKQ15FdR27f{qDGvbfl>}4*d7{`#WmSZ8eQn`NU2i^DUaz2@NZv z46{!CjuTWYK5Ld3E=k=Qy`}|GdpV;(vc3)SL@OKbUxF? zapWlvCMwPRXF7Lz^}9UA!+(>O?%dX$#(k8m*XB{qG?SLx!u(DK z_bbp*xH2?4lHIyLEr>35h4$kMF0hNr!X=M?%A4M0gt1k*!QJvTMZ+MdS_m2T<$APsd zuyzPo>x3U22G-imSuM*kZT8(PzpNvwCUH)v%(`d*@{MdfkANHBJpYr`dXJ?A&Oz71 zzVX3Z%-Ibm1Nao~v@%9IBN8IiKEk8Gtn@wadGP^_A$Wb5^+Se9!xtBE+MheZ93IQF z8Xq-l)imc_;&oo8gLxi>%ox#{1bH#S{!S=l*`v_WX16chx7MHCR>=9aZG;Cnw^kR( zYTJoj$=X0lTP@*=K+0?5F0u|5@;)0{Kp8WRf*0GmMuj)*^zC5}2;7iwe#ts1PA&3! z?J`$xdW-(C?r}C8@a>7^>11zR`U}DGLNDJNdSQMrjDP8ubPpCV^h`E)1pJ!)Sm~?p<3H_vtYsgp?tzo^ zBu(@LnNDv&7xnovjOM+K@d?rwmma0t>|e59 z%Ts`FnFQz?wjoK_iL@a*^wRl=GS2s`;Mv2&IM4Tm?Cagz zu`Hv07q}OX`j(21q{ zuzTN<%$mAt$sf#~{2cnRYv~{AKee2<2iVJ?0e_+#_Dc^&29<{`-6skTgz0k&ZNEt& zeJLgs@2y&rRL?zUtcP=*Hr5SO8LyW0rQtj))D~bZ4lM3Tzc!^_a^M8gS_7;T!DHQz z9$>s1y}22CN?Jx+GwnR1pFc0Oe@EL6zm*uSV|-iI2gB_fL*dAuuw8vOIXrYpYCU(O z9juC@gQ?KLrhPs;_g9E25-uG+9C7{ z8V~stmK}iB1kDFTGgF*Ek}HFy8JQ;6c^G~wda1N#=2$fD6Y*pE|K0dB#!|Gd-K1qZ z_s6F(zM^>_6PM+U&yx6bkJd>hKY~o!P3zD_9WDe1YN2&=W9#h{=WC3u`X{>fH~P5$ z?$SnmOE=*#F0S29it~?~jqg1}^F&*_VVJz{lXeATzaRSh0rJiJ;QA{udNJ@k`zZUc z7HyWibUIy$1QLua!97X!3BaPEF^`>RXbgT9&|mjp&3_ha{(iuGe_%fa97qKh(x5Zx z(3u{#<|{Yy8#F zoQ90jM*PSP%j?IP0_2)bWTsB&!Xxz8r6)Pg_kah{olV5)4723y6<%C}^07t`9a-+h ztyi4pf$^>O;(n|+L$flS`-syz;3#xvJ#gL%?u3GWUsJR!x4sCSc|6=38RYdr_8jj{ zj4703<^}i>4XHMDl}48+98?-zq!9R6sF3-dO(>dnMwuU7hcjwZ@`--4NlC$foU#+*edl((xj5F%^dy%~t zm}y17kPf{-uN2*dToEb6mOvrxuOKYy5n7qvvItt7)TIj-Fiw({T{$(DC*28fugPvN&p>Z1j{ryCX3zb|;IeQ& zVH2`2`u@iDNmg_PW8G5Nzxc>iMo+fN7uq?N_2eq6_s+5V13fmi-q9=EGPpN>Ad&e# zv>JbithW7S>@(@F!POMM9l6JCQ;_;Zt7?$**ZMQs>MmY=#@aGNG?4ymsCEB2ev@0y z%aqv;ZD=m^L(5W4KN8^k&4ym&IF*E=Wg`r1aL*BNC3^M;;$4|dcOHwD@S!SGxy6WqtT>gIg{ekfR7EN{nDgmI;HVxz{)G6Z8T}@ar))IvS#mW z(ZG?c)65$6J7e0B{hE;ZB-0*1-cz0DFmCFbQ3+h^_bF89rpD8ozS&Iadho;8S4EaWGLtwYmsTAp=UVf*c^qPZ7D=&+0*J@ z7qo`o!T0q%BX}Cn<%+iLU_Evi`tvol)6k|A=Ne!~bnR}%F^0mw>0aFJiW{o9T<1zJ z?pDRYZ-jf3y|^mHY0SaX30~Y>#d$Pw5^-tBe4C-Ec^sD)y$c2Zyk-=!E3qx5P8ue{-F~%H#c~anzW9M4ZO)Q)%EUUoh~hHK~Iv+5rv)14(U#*z#-?J;knP zCw>R>0vT-0=Vw7reKX>VV& z@_S2M8q-|m%2_AZ^sMh_yJ+S8UY@^_M{@XvfWN!EX4!L) zkX@Yr7yVV}(%+t&IDZyBT-YbvGM#;gTK-+?pB}1_p6;xJA!un*0`xoC_fn*<@1-`* z2TehjTkT`cSbJ5Yd#FMeA^lzk`aQ`I4=~?deyeQ-a}7+HbEPT1(x}z~;@`-l;WGAk z3NF0J(4$2z?J@bJyFdn5Wqmxv-VaWUW-c`z_mby_$=N7<;(XUgE_SLrcFFjVr z{53u7w`bq&jzcx$AiiN4yY*COzqeMvc(6q zepGt%NS6;(knZvU;+VGyq?aqs@B!kOt9itI(Ugbr+7!2^L_Bkl?T$CWX5g(&kXNaHU2F5VM%rV#u!Ua&|&lh zX-*Df+LyTzoC-#zH;K)8khzo(7Bkn2FXsQC^TLfTeCtfTORunfr3}t}M4R3jA5P?5 za$XOGYP$(D93SBe$j|pR7h#GHf64R{7xM#u}?*WW1fBfvIl|vv+*O zg@vQEL-3=uUe$x4#umQyU;IDXziuh)2~Ydad)SiG4|)3bqwumQyey_~*SagdjTh5K z3)x%ihPTi;Zsl3Sv$lWff-?A3^$px@Mffajk&IImNN+3Dp7e=n`|?88!B^oOlCgiv zScs4PmVEA7Cc}Bsi`!0|V0;GasNrTUmFYZ1T3%`CL6?SRI6v0+V&B1CE-yM2?|O}} zmwB@He9DU1);-C)=6|gTQ=I!vnBmCwH8$TTahGxS%9!QlUr4BapICG`ylE=DX$rh) zGQ6n*-ZTl`H1Sr(+uh@W=e{>r@F?As_GHtcwd~^PHFL5WNJ@ziQonNIq$ul*` zCF+ai>2+rke>{=1jHF5DI}QA8E>wSW3{Ob_U&j)sxqgBAMT z=pT*G1k&@(KN-$1yno!ZG^0<%eEGg`GzT zc~f>G+6M`V{$n3~avvnexgQvE=O@SU(XaiJJUVg9C^K)n83R|3mE-iMzwmj_#e~eW><5OBXzeO>x6WSJo`!{W3wNKRb2 zZx{M!!NboOBjNtotL&v?SQs-A}dIqUE{?!DjvD_I5LP};}I|Z zImILU9)ovlPQT~HZ&EzCcof<8SCa8Yv#wMKKP*xR-->8m3mi5Os(r#u(V9ZZdB~=< z`mXUElzeB_@7{NzzTp{*^nI2`o8ZfL^BwHGa$lr}wdD-D61l|Q@6nYBv`zZ}?eM!O z?UHOIL;D09Mf{_)f`*<3N85*Hxb0Sa75OSZ@Ee%R;amUd zAHBbDj{Cp(x9UXZKe#^6nsFSQkUhyUV3zg$s$Byn8JIbY-&VoNGV+V(JwiJkc?!QA zj9bUo-7$KEd#-(yy$#%12HdO#pO7h}=YJ;mpz6+bI>~z>?LBJhz~+`Qc$aU%kTtn1 z8okO2KgIso_@Pwf;8yE-WGz{SK3rH*FCMe&l;2l9Ov@uD?{ zX_I)P(zG5E51dWfdd>lwINCOkxLL{%j4>v0`LRzRZMyOYy!^oC2I8(XWqidggTB5( z++>x3T>4gg8PJ&bNE@#-XxZ-gG-#vb@X@AxuPk`MK;lLyKeX&~@&*&f3ph;+yTzRA7U%4eWE7aa#k7xO6DZ zNoV}`|K8)(`1Vx%`^=YcPxTIBUaH~ML93>JN{_l(LFQEREIBVmJM^wN&7J8<7Ki z&|5!rHvs)jfDZRS4(N#-;Iq2|z4u5CkJ-jaS1$g46#g%}7s+&|@c)6V6|kX~Z|=Xc zUf#;{L!Nu_=ZvAhp+m+N4jOX?52_kt@Avl7*OON|j+Y5lR-xb@xcm)$5gqv-r5j$G zmJ+fKVrF8r{L(^wS*S6d~KNGVf$d_H~ zHD5Gqf@aMHbVoXwDr9fz4T`)q#tPGqG}90D>wfRs)6<+<-Yw(o0>zC7=D+&h=!chnJAC!yKy`z6X`ov@`!XP5eY^O7p7Xr#*5T|>g7%fuHFSYNK@PHXC0i=X_Kzz?&> zVK*!u#`uk7e!IuTv^4^+@L;Q#*Qc-J%j1algV2EnKmOUYNE`kSKVqG+(`Ogn;f_%` zc_k;*l!(T1*Q}>Y8te62{qXN8jgEreG_MyeMfVL|Nr1){5+|5_lJ=flXPnf1CzBqp z=bk_xdn(T8BFM*F%p=Zc%E@xR@0FuEq-R-9+A-p^rdW4r&Daj^kAQaVwqAM}ACG*$ zAU+o2PTwA;UAF-@uC6WBX`>(bEs^ee)%b|kN9oS5>9hET%I}=(=FMS@j+Ofx+a~$l z`)USx_tj9>@gY1zdAM7|?h8&;E zbE7Zn!&k_Y4ow4$M8eecB8Mc>3P<9w`GKn-=1Ku6@ZY|?VV$99tc_vHUj`OhZqoN!l;FZ$#IJt#9_(=2Rij}1AyxM>mTAN36% z|LCmnu^j%%^-6l^muEG41c9|{f?`xDV;YW^DW9t-Zd--D5F4N?HXj1gi z)AR3KW_)>ddiuZF=>KB2oPsawP4vV3BU}~!_h!zuzHNHWXD__nh118GFVW(4=rd(s zRd?~4Gxlz@e_atsO%8I7JsA3Pb|hfIOR}7cJeaPd49UA{XK>CK+ux7-BdAL-q_s=T zX7pC#1lMh}$(0c}pAOv`$ec;`c43-0*29A7FPZ$=*o8?RkxgNaf$OQnN3OcBaozYU zUHehiNk>^%74eU;A5Fv;YodSCNn0EAci`P!wC^uBjIm!v&QrUtrJQC1<0soF*afD+a(7K*cirPYzwYa(Tks?Q1aF}~+JBozPlic0 z%5r`RY`QR;?L=;H?E#LUrw|<%Pcb^k?)NO`UGm750AGO7zngb-qJ-yYU!Og&&)Wae zo>%Nln&gv1$S21<`NY7pXE#(dfW3TV5$(BQe?2Uf=jYTRo_0)U%;?KczytBeSBRG_ zz#S%@{W&lGH)ll1`pbhH*|XYMnCe_0c`FTK8jw&43BdT8#1uw+H`gEf-$ zy?YPgyRM6NEj~_2JB}-)-i4iATaf?up1~6r`|T$^o;Zni#P%VVnD(MSP>AfjP$4q+ z-3r0qZ<{c~xz2>z=l}FQvDkiC1V1-HWSKMSzcXMjqki3A6zp2MFUTDNZP*)CE!k-7 zjaINmlns)D-{;9o-ooZ6)P~K-j8~Vs@5^}C8J{B3DErLvs}k%TJ>0#&zO+^SYyVSu zxPx`NY=2siAB`OW@~QMKbqeWkg+k=-phDI*yBPDx(ll_=7Y%Z)9m^z0x!J2VZ2s3bvu-ZG#S;%$vz|TF5Nq!Ok_Bxg?|h z36nS1NhWW6zFK4($%o1l(qV{^uM5A0!Ypx;LdlZ?$%dtoxND)s?| zy~8cn^$E9vcagz;xtrpwa4Yq+7WN~)f4HmO?2D~RJ0g0U=5)*fm;191OnVG}Dju)> zvQ-5&CH6PAxcuk{d0gDb|2^-rxzJb~fiJ!Z9yHzIxBu`hH*YKHMy4cQ`zYG)innJ^ zH#P?u&UZZh&i{I4OYieHXpq)pM(#>+<*wfncd3z`vz_t0i$}@dujXGmW`$Q1O6Oo? zDd0hL@n__X&C@XNKaIcgsa+|)^pz$*up*jWPo5XOxhgs>KWEVQn7qK#Jo2t0{RNXZ z$2t4-yh(pS7mQ2?%q=JH{iOfeapO!b)-VNkiM4D(u$jf8WfY(<@yOX#><#D{U zwrvBwdjx)@QpRRal z?jwdKr#fHo;;&FVbT`Gsr#MAk{3OLgZ=Vp1LfZ#;@uptrMYf60c5=P=(aH}!5Z}ln z?k)EGW&e2tI4!@#YXA9!qQlpT9@CaTE2Qq}(B&@b`ny8Po^^tdvP45-I~MdSuq=jQ`a_ebo|XqcV7t~d1g;IN$mrdgQj1l(Vhx{kpzXnyk z_Y;e*VeV!!hhJeXzs#J@U~M~{I~ZIWhhW!d({%Z}`U7?v6-(akwiS&meLH3=dTRSU zI=!2JM0YE6W`MQ@6^j0f4#V>deWovlK11USeGYJkr0BBFouhKJ96S~_di&F249XX6>^>@R3uKU&;P&!0LYPkLK9bTR$Id-ug?!{eWl z$D_}yPSFKGpP$tjl>6-!z>U_nZG$+AG$<+DIw+Z^SGaX{@9?Xf7s}7tIo4X{-(=p; z=KbuNv8}WFX|3{%6=JQzI!E|qL7U;3AB#43;crp6@;&BQYufFC<)MDN>NYpNj(EvC z=tnk3;034CVb}Hx`*lzEy_o!3b7+t39{5l*?G`->cyxF!aqd28n$rYs#OSbaVy1^1 z(>?mC|3v==l6D#Y#pYnI_g{<-n|#2;1oDk0UyNSwIxU~tpW@4aUQ;fxGmrd3$RC@N z7f;LY(rfoj>T>c9ByS8(o;^Kp(p%7L@&cn9$eTsp7`^`KX?aaPVD%O9^(Eh-(eX4H z8#CmzQ_p>)-vM4EXGyp1W32aozqE0H^y{Uf$(hb>=F8P7W;y+t1JPvd|0%5x-v;-Y zlVskzBs&L0m-~amDd2J{IGqM=V+*?o+Ke6MF^?X{+uI8cfS=tl8-Q7P$j$7p-vdrw z%QJ>2hvyJ9`foh%%%;tT4rV!`L4vcxii6IHj_&s2K2n_4GOPh#^Ws_+2mMp~f9J)$ zPn=|euT(!*|7Y*skKdC94KQ-?ebB`;Y`+@rORbMU^O1E<#ifd1x1H-M$};*-=0UXW zx28V!6cqv+I}`#_+Z4hFe?q7;I%kx5OQNxJP5iX9dwLpKR(580^1td6Jw35#CS&~- z#{A2S{S4+{I&^eem^m@~Sq&#;<5SyzO~}~cize>+1Q{3ExHXVnQo%W@Aa?Sy$6tYb zd?=7oGN19eX~~=2^6`qL?)ypk_{|vIJ++Q`NybM1dFp88ZnCyyD=K0U3U2gm?z0j=}Xd#-Tqg+ z_)n#Oqmu2NYRjOq@S#CLBcmQEl&@>(AF^ue^j_gM%8Cr>9Udxq`KmtnnC?sbS>djg zR#fyZ{fOo`-MNZ6kUnw;Z9AO~zVu6%-d#=}S4K^DN_kg*)CcKq{=)gVLw$kCLq0qI zn{IyPQU0UIANkln>HNWLWYvnevzIB-vG`KVS%6?@-x~QF6HLAB+1L)D4C&ZJGY3N} zB#TLgiqXt;;xxXGKm%OeN}6+&IZ^(Bz|<#(Hsm_rjY}7On?PE-NyC1CFaqBTSoqMQ z>`vxT`&skI^Di&YV#10=dD8cUO9ORo@QYk4_)Yxs!C~H@vq) zGm*i}`Ge1sX^(q)hnTIc>>}g~M|?4!ekPs6z6kVFYmbH4tjyrKlqZFUGt-;j#J~0n zJS(W*rK9NTffdowrxb_$C>e997xyE@nX$#cnHTq%;yijQ8l-<8R-B=?SG^k1_u|Su;+5a4P+}rLVDt`d?^<^an1D%sGQbN{6t~(8$l$Ap~6= z!l`xCon`ybQT*1>C1@lvW`w@fF)or(D-_b#phEC)mq#<}6iR0RJ(12pYhS@yI^%5C zxZU=;*-pQ>IMLF0y@7a0h;-TI8d?foYyB@88}Q;km6qPp&u%nyy)=9T8M2N!YLC%T z*8Hv4^$xdDR%A$@aIxAqsBc*NPi=*){jWPyj?6H0HN%O16@2^;boEC3_Y3BZlE>Xg z;CwOf5$L4)-wwXtt#il7oUh&D=4~TgvPKhU4qBn@r_)oNJ+1JidGz$nHSRcOI?FwK z)OYw_vaIyEZ$VdF85G+_PX>M2qcx?3qG#Gy5T3_mO~<6CIHSCDlg50Co~}1(xz0s| zqHQicO?M74&pLN3db-BTQ$$$RO;0JGbkWlvnDlHXpU|bJqY{WhoCmpBj8$te4Vzf|u74fw2_qou*zSc0= z2-f$<*B8$$p5Z*dVw{g+D<<2n=Lj9f;AX-GY{rVPD+^;UHcn>{uou(+;~uX3x9rGT z#<@OwWJjibl#%L-p@nJAMU0W)@@mB~p2EZPy|}508>zT-XOI^+QE~7w;obl*u1s-^ zf6VSH*NY#mcte|WoD45+gyNvhLD|A{hP}9}Y+rHOwJUM_uuturj2*ttl{a9I+6o<( z?N^c3xFeZw=^BT!2cU3BX|zfq^rlE5a1(*fv><1XF?2i4@l$6fa*ED?$)_GLHX}kE z@ia#`8>2Y~WIIM*568uo6KDSU=i+^?yu$fCr85`OWokVuIr&54WN)?;nNITifw(;P zxo6~0rJV_>hTn#OHO*Dlr}5;s`ESiGf>v3XM= zW3^8qW4A{kbM?FlbDgg8#LwNE-xkR7&K|huw%vX2TxS9LF2TxgDXTIaS_QptKj62I zF{X|42g=@ORgm>{Lv8=g3T!tU0x7yTAlh(fX!yn2)Q-kL+SYA5)3!qQe;IkXSI1-Y zOS0J3!V%$y!ja)ft#xsjwOlcG^KveG!DP}4e2=a9bs%$V!_LgDX0JM-pRMuGKCxgr0{y9II zC|f7LGlBTah?ktW57>5TCh5%cJkq{s((vCJp9U-}CvBujOLu-7p9Z{aAnihvmSW_) z?lOU?S4b-+?J)2yy6`Lh4U}4@ori!u(ed}llke603#He%=}Jprjvz;hmF8rYLwqLV;^|6livz`sGu1!FPfUQtDc0xPRF^C61Bef4F zJ7M`1Zf8zqC+zJjcb}=9nDfc%?Y&%o@M}2JxQJ&a&lsNPpgV_vug&1iX2J%}qgC~R z_H+JBwBLts;Vk;I?FW_rev;ul4sKxxnul$4FMR;Cx(*r?r+eIurrPKNM5{HnF#psu^7Kl#3p^X)(Tsue9fIdb_Wza{CwzV? z1kP?z2oBt=5Im|Y^SybvO6}^%63U- z%_UD|I_=Oo`5X&+`$pGOmS~jD01!rxf-fD6$CkP089FUJ=(^ezk4~p{hz>WrJ1qQs zZCb~cKst0f-O%Y>(CO9mMRf6bXmms2sQ*BxSFhO`$bwF1ok6EpzwycHR&YnOF-EJ0 zLtD2)uRE@ar`HdTX#wxr(dV|K&uxcppH8p0$IfJzl(B|e+1`2Qqe!f$}?+uFV9gg&m)rcr~B>QH@W|QOuW_w z`NsE9n)9${%i2o3E5maa8-07KT)YDr-s2s6h!ZU|aUSn@ow)UZS=f=furYzYJWrhP z{xZUDSsU1!N7^$cEywv{d>Zh%oV2h>OEEHI_kY0f2GX8V{_&#sC*}TEh<`%y_qp*g z9Q(NEJha6>#|l}StQ<^vPp6~f)%8GN8_Oi?!<=m{T449h>nfm-QK(w$EX z{-hb4xR>$UzuMyi_hgZt##!1JA4qprGOpqSQP1WghOe@}h`sBdW#kjRs5W7yvxG1i zdLMrt4EepAcE6Pc?Z$4;%>64__Gt321D>>Ah(F_H(y!09ee{3&%PtJuMqca{4oaUh z*VHE+=}DjUpUGnaiH67MEK1C8gZ!Gt&Z3AG#J6)3?Zh^^yPf^0PcU!wv9@%ezk7%0 z*E~<~=={jFggV=rL@1uK3;q=2IdeEqqJOK;DgC#Pclmpi5A`S6lT>^adfbX+Ys<)F zYh)`tFvF{>!Jpbzh%cj?KDlgPal!PJCD7{0tl!12s_^gE%9$Wn_nqr(WbVYT&eOPf z{OYG(T!G>Yzsho+^x|?ACw>JSJ>kWvUYB2GI;*|7{=}(`r}Lo}U_2D3w^)bHqKUP{ zT^=85<3H(e*TaM4f9t4;&vD)-{;1(k=|=Z^^k3pX*zb7x4gaA{@94Xm|71FEdFk?t ztoAf9*3H+dz3>o)wC8^nf|q|+2=4w(Avm<1FkWZW6_AYo8T_Lb`F_3lhva+Nu%|kQ zp@}*}aF3_YeVsBTmz1V3-*W?do|}Dc@vF$y`QjUal+I$>7_Lp}kbE7UJ}kUBkk;{B zE%JSN`c~N_hROmRE65|eiSWh|VQ1sW@E`cU6gk~0>)9z;|M^==H5wMyj{gW`gt31>ucp-OSY-mMpmCi zoM>(Yy5!P4*N$Nx`KOyS?i-0u^JwnnCN0bPTYQ>Fb1x$;3j7GBB)hrtc&c+0IFT+| z^mmN+{~wfY)G>ug; zV;!?U5x*7epVYggIR&IkHp(+$wsSV&CD7l`^YJWyS+dKQ&Lwt_M@-%pEX32on=?6= zhoAd}_*t68Gm0lo_)Z^Ms84h!NQe*n&4P>L=!15_%a@vafbYupVNI$_eTD8`NgN7S&f}nJ#?aL^Yzt@^>yfX z>#&c|IKAYx_wSU^)Y)a)tN3liqenC{((9xd{Y5CZZ728bb+UK%OXAeGZME4Q5q!Ah z&{xUOErHCE71iPTLY=)w-%&x_hPk30z%FATxSEe1O=Gl;GBmGRQ_;WOlA$dh`We}q zM{{iHd*{e^z}A8GO!6|HQ9 z;*;-f;_x3zUoxF1$S;1a_lcgp?~jNR?ULT&ap1i94#@}E-Dg~v6Suy0R)?$al)bCF ze`Mry=J6~~CR<54#+I14c3|4Y7v;yt7Iof0IU0w1$fq&(d;cxr+tu;nhvJm~+%vH= z%zya&C!gf*&O*+Wz|(^H_!kT$9IWFmI{B?Px=qf(;xi7LB*|0R&KsIX`YOB9GWrW% zJu`koE@|$b_*vkV+vaR%nrT<2b2Z^%XsXr=SD3hTXQ~NPor_GEdV5%9)7!`+~byUbmmK}m$)Nnk)N|U7P3BfmIK4w0qPWWy7rTT?<4e0u$oE;>}_`E zDb;D`TWbK}ab75;?eIwV%&^L_NE1$Glwb8!(LksK;ITAA2eI*<_PDXFGa2k zVC!@=kXzD%P3=C;A}m@Os+Vu&3gj<$Z7tbI^i4kGbDWF4xqN`KWCJ9)sY|!|hf=Km zm->Pwd4sG4(nrh=SPPnw-`A#Cx9=x3{r=WD_P9T~xOEr*%YNbv`uN7#r_jf~Q zlGl{cdA5DwM7NBulO`IeeW7&^`}U0ch8sVd_!{Dl_2sM%bUVTwu|7{{dktx#iFt+A zf-o|75oLE4hKdi}h9C8zROAs_VByQiYI`<@E5UGj}X9l>g=UvNo} z=_@|5`V}wfIlW=fsIX+6FgkEECf+?UErnL}sepgaft{9J%A9C!uJppGgxG4C{&Y_7 zS==$&YK)Cfl(xA3<4^LR_lVQ@45bV=pF2i>^xFL`ub*vS@%H~1x0$4!9HUIbe_l5E zGo81|r+W;9t8=`6{~q^m#cBUa{(R2+_f_)M`E$H;FYjc_54Yw#!a2vP`8m@Y@IxfO zE$eC@!(WTfUe6e)&4u9FA$XVOK)mS3=$)S2dF@uikKQ{6ew5tQDrZNom+vMb>?ItNz(9Vr-=XCEiOXc5md z*1B`;Ied$LTgb6N;!iBvVQ93zr883)i!=X}o@dX9`^Wg$V_e9W9n1H5_x;EZwP>yG zMbLdeI$NQ8M8tDMA7Zo?I2sY_6KxxQ4e;_tn(KplJ7+9nX;r+R-p?JhQ8IB2{>Q@j zQk=|rJo!=#5*i&a`n|}{SSQ1uz6iXFw|$j=jC_sl#Nh+FLxc0rq=}zPPjb0e<`M0y zv3}HkSRS-*E;`x6%#nEDV6UtxURgElu_^w7xN*Ff+Cm%zG&19|hcj{YDxw`Kb ze(lM>xxl@Apy*6!y5o5Igik0_v{|}hE8xk#*XEr<+aDtjcAk5y@a=s@*+#D$D_efK z)18OCvOnZM(Y;g3wofg48F`S0DLebjvdx-5R(7^2d$CvcZvGR0IHl}YPc8dq@?=vs z;}0*-x4%2VrR6(_7cJMgk7PeOi{}`=Oy1$?-PL96m5#0-ACA1!yI6cDayRsGz0bDi zx&J@Q|7P7}_RVsg8J^AUM&jGqOVD^q=Kd)(rnxq&bH(g`a+WR#&DR}G znkRG4@RT!k8O~3@GwC!W8x=55`{$dr{WIg2=QA(4&d1EDJ8rCZp)D6t=SkgEu5$t3 z;_HJ=nC%psFv}Ta!VKqJh0uUx!oJA$UA6LsmB=~#|9tmEmF36I-5Mzy-Sva5VY-{< z213!h{K4{J=X3URMp=HcHV{fL@D0K$Y`pT8RQ}t?UCoA8v$kv~wzf!TV7|R`aW|x* z9~fc{TU?!%6TJMsX^&JlU}Kc(JXGoGlS_+{iIJfm{?naiujxC)8@mbO#q(*H_yHLhWBpwzVFgEGDpo2H~n^fGcWlUy5HZ_H@LB#?}vYX zyQ%jk#UYR9U*zWbI^Wy%j_j*;%;CH8V!qK!92)BKZuuW~X;`W=_6E+CT!g(caMg67 zHLM)^C0hb_k15MZc$jnMHLUL_>qF-He>%HXQ~$8b(D4B2Ojq1U;THVk97^>i43*3j`PT}tC$%G*mYYTn-+QRr?&@vgU&ewe9ikF_zL}1 zIl?>L+w?yFTW2?1c^-BAjynTJ_|dPQZw))(hX&+a7Vd5RTRePMi>akGe7vD8E?cU+=bse?(IfO!TA#VO(l*nK!jhG;{1cluwL3Njj<8{WksC@oY zzEqW;L4PvJ!>70X1JZZV&UNXOgMRmvIW_IPzq=g2#y5?8=!j35)6?>G9Opi7?vjrL z%EG}=Fgft0wCUEWf#>+n`ocNQiQ~f+6{M$7&fwGAeZLu(4|B@GA5w2XoR{pA z$s;*M@&r1gz1Ugo<=pdLt*dl)_rJ|6I@VWo4<6jxgq#gEVU80rVYZWK!YrqU3Aum6 zg!pzeVY<^boB!$C35C$M<0h28=hN@TX(^n9KU>#l;OF(_=$|eN>rRf(>aX&&GavFx z@}I^;c{<*m5SFd&r;S-GJqjGXNS@QjG?sqs!}9QE(&NWFKJ7U4?B}G%<|00=^P}FM&v^8|UC*hf;j=BNF-%feC4WFMbj9zJphw&dWXxq~jCJEeMJ+mMKj zLlC`M2>n_TIy~-b-03&>sft%?O{IG-Bp=ECMd!C4w0h{?1EZs!T&+94lG=d78Ti0= zTpx}BWE9MHRs^lkW8C}d? z!PuY5TzM^la?~E_J0D?R>DxTV-U$_V&9oXTIBSuCEFs;sJN7;)e zf3~xie6FuX*T0Q))>}yH@YYYi^vc^qK3AWT>5M(4JXfF6V9LvKenUQd|CJo)?nxuZ zBcFuvsgl2teIWc%$o^QmjsxYEY5#y3oTFeYVt3bd#ob-Ew)-BlT&I+L8k_yd5!!Q` zjU3|YhH{-JsZ+jP=Z$yA{il3uJy1g*jEok$69>JWr-jgjKlvaUVQK+7cY#wcliK2=;6j#$`Wosr=}eSFW%${*0AmaKO!@&mty$w zyoVoy$*;VEfj2!{%JYd^2Yv_+gcn*LxOj2WS80C_E-ep^^Z3Wzf6H=K0#hztXpNq0 zbdX04oaH*b$am6*X{sZ<_{gMXI{ze1IHo)?oVe$dvRs@vVDe+r^#8HC~6Pqc>Jpk7AFt$8y(+e3PG8CuzY;>HpWQ6Un4YU7O-t z$Y<6G>s;&PA!x9J2hxInq#s%*Yu!0`m+w=plZ>xfCl@jX8-Tm;JiKiFD=GM*2@``m zO{n|RPpy-T^rQRFUMuQrDr+T`wX#rag+1ao?GHb9b-`LEvc1$kQFkXbu@BN5OXeef z<0T(9aOiQ?z|R~#7o9lt#n(P%ALoRRVVCU6eWYi@{>rMJHrQP|Rg8^v5QWgST~Sf3 zJFxvqV=fx;!DHyk$LAPc7~Q9#Cn@CTEY{v|{uGyvrMGiGhT5r?e${PR=iHr$TexOcTSfNl|@`7@+8R?j$&^kS||FV`*$RV@!~gF@}M4O zU7}|vzt(Xoq1J)c=nB@4aDs5H%4pwv7Vn?B>#7@h9a)mn5)58wA?<`Mouj`!pnBjJ zojN#gLmls#I{F3w({1-SWcSLV9I_(l`doNzXFU1_ zpQdf$s`)O=Jj1u}wrIMk-=*o5F0Pscd_>ap5&Xv@)_oX89;Uo>3_mepO7KAwCI#;| zVM1`V3FWup)ckyymlxhLZFQmX+0nHBKcMmEpP=#TcQSY@8N4;g;H_`IQ&)B?N5NGg za8=l!bbxWSzMEwa9^-aaL{}hdFcyiho17fwF_9U{N#Ft7=HJv=HRbZYyjccP& zLLByg$if^ur>Vfj`};M|=*C>x>V*bY@X71K)&$Bp^ffK`EOYF@v}ln-U(Yc4`v-@S zPcW&qCz#CVJFM$_;-qqp9Yrs>aB^_XN#z_G9A(P&3#OQI*nbkL zz0(Q9uqOWQL_d?wS{3gtIvPj5X3jr~Zw2nRT4oI_(;C=nX$^QQM`tl^l24htdcDs0 zTu=Wr#$moze(b{QU}h)1P7SUipIHZg&^pkW-jv{G_Ljoo8s~#k#XmXzz7#&?&gsX* zix=y<9vaXui^uJ{9!Ao~mB5qZH!Lyu2g++7yVrzC!2LJh?fcJ$d-X2`xK9D@XA18B z!M;!QT6YQS9%0>Up#7ih7liveDq=XV8PmEBy0ZisOL!0HYrkaQX7Goz_jAsIXz%B> zmQO2k^$cqm53Lp106YSX7VRwca!ytD{`uL4b{6oiJ>L-WibgB%pQu;*AGHBJ-*%u# zG$-5!vVYphaof0qa++V!&d$1*o?Ledd9=SaYk5bueW7a`KA(8Ot@bKIp>;_-+^Mq7 z_*LHL%kG$J_Lh74v9|wYY!>?kze^csZ-ac*UEALyu7R~5-dl>!%>d@#PdIsBS!mV) z_j|ePW-8^Jb($Ld6|n298|@v_44oAI7abi%z7zXOhfa$37B8W-r!}ZN4sQy~pH!El zdr5HH&Q@K>K13(P=Zlt#_ZGd;`cz%Y6V_cmbW&X@=Ip)Yw%wO}qG^6)VlA7Z>coS$ zATu}XIgCw z#h6Zd(J%NF`3#H=%I%~V{edxuUi1rIOS_TuBHe|xS;U`;UTpXVdht{Qy_iUO(TnjW z6up?h{EcTGkvYyBi!7uFIY^Z8$@2dvtyuQ0Xoch3Qs?Qyazf3c=!E1TYoQZYLnqE; zj|HtL$p(kN1`anKv1V`N&3&G_s@c+Xutq6o^#rE8jXvO1~IJ81} z-##U+2)E(TiuG==66{UmhUyO44cYt`UC+*jD9`H?`=Xk=j zBCJcpR!j28J+hm<5_yVn^zSL_?DJECi(I+r9^wMf6i43DH>mxALoZIqTb?6M>)~<6 zMq~6cWy127A?}(HeF*b|Yfb+C!L8&wA#ZW$#RJN3@V@dp`>K0Q{^a1@Zv9VDF3kTW zpH#o&Lw}jc-!J$u`66X7!&s-H17R6V%K+KO7#YmgunZ>hPMJ?>FLHgRN6>&s`HSd_ zS)U*2K10oEYH-;1PRL+xpU+cXmoBn~32#?kFm#`nl{YU*>AWK9nVzkn5lQDtFK?DxCvHh18oQo zuF-c9y7e*KD;V3)=;twbL@_&r-!9uWe9V<(($(HsVUhQb#T{CC=(S=kkZ9O}c#hk9BF!;0oxg4tzp&Ry~YV)-7ogHlOpW^`UT(l96fTOHSA7!BA@L3 ze|omTVHWSk4uG;Jbf{^r9YDWeWe7cE0eV}~J(b+I0dDDm4!$&C4Lh^qmVKtY?;N{{ zblEUtNb+Fi7;BjLa$jlHv~0#OYeIgl@6B8G$zI;quxwfuaV?uTS3HHjEWO$IYsS*2E*pzZ^y@=(%#G-& z_u+fr`7ioKx_E<|=vM=LFeA45pq9=W7hGIbd8IJAv>R zU?)L7{lle=E)Eq1p3d5-O;?S=sc_P60{pilZ}Yk_2a8Q_2Ef#H)v8wNzZ;qh`m9sMmV%eA8~ z1|EnDkKGfrvw`*VV3fzMxWhVxPf7db{T(G%rgg#cczi8r?bYOVu2W;{(PfQZ#C%nz zp9}{c^&fi?2YzxpVPGZkvLk66$at7_Dm(s^-~sT$LjEUkI^)IIHKvV?Sg*?+8wm3J zoUzKlrerzu(|8y5K-8RUSm5s}#{G&yD3_M&T zHY6d+h(B|1Irg2jZCyp$16KW!855z$6VSnruRFYuv!RR&Mx2Rdl6?$DjsiV4^?!7h zJi&K|ddzL4Wy&wI;;hoe2|yKQUqd;DaVi3*K+S z)Zl8u4dASM&38)hZWAU4SDCPHu*QT*!4(Rpa;9D31Ki!=0pu#1bXk+Ovk$yX@cd|+^=Y#|0dFDzS_K)aE$1>c3v7)>TVLbkBS=jvu6WD9#D zbu*In%DyNpi_W9G_{$+Clx$%l^D}{Y8qa)<180oIZ}v&Dg*{zuEYQE~`5)?E_Q=LU zeGAK>Yw6>$8-7I}kl)ORqEB962A@4%{Pk4#dP8?)2yQe!21kPX%$}t>tU>@@w!<`D5TlaH09oe)~53KKu-wDtU(8YWxEv2VbB}K<6^ZC;5|us}rFu z$-#%fP1@6kW>|G*uHHV^Zk%n|;=j&g{E)p&`w3@a(vSCCFcDeCM)VY!KlGYBmDGz* z_-)R7CI+j>EBV-d_SS1iQ@yf#xq)|lOe|>N8lYzGp*oBmh+zG8;@{>R$GgCU(i^=G zq;9Uh=I`)+hGa_64blEnrXwfMp)Tud(lfccoAJdz!C1z;h_NqZ4hn#S-C+=_=6YxoT{yc_KuRjYW=?~Jz5NPbfd0V>g252R6p~HEm^g2fX;$D=ODxPvdPzc zoz<;XI{y9iIU7AcKF3bj$_$XK+{Ye22Khucqwmi{7N_-mC;bxM5bqPFpB8j+BJ0cN z84{M??2#Vfbn*&^8ko6$h&^VcQ^(E33kD?bP2jx5=g9gubH3+`0Y2&Pw)O>nju`!2 z-{5TWI&$H@!5*`n^4AmBh`i^7j6X4WEx1McTFKJGxM&h_(lw0$e$@9XC>NITzXVPY zE^=h=j*Ne>$)6T1BHw=SvyMMZh@O(E>UhDHc6C(dNq2&LQf$>Sey2IBU zyJ0-M*SI>?LZqDbpN31-&#?Ue>Tk6MW5B15K07R@y?ulO%g^vnT2C3s^^acT7yPdl z9sbyRUxv?*hsT|Bf-C==ZmRyRE07;s2v^|7LAM!y{#mw+rs7 z|FI2)?otuVMd+ZmS|iW1-@U+rzux?t;7@v`wtd~QzZhd=<|n~m6nVnD;M3UM36{JX zlL%eZSHQ&k`z-K+bDy&GO)bPpKlMlQs?3oIVVs#9yv~Iot))m_@ZU`t=9s*~F}vKd zAMzd6QB8NtI^~@E+WyUL>ks5J`l-t0=s`VoUm_dpgg@reodulqbvs-0#*W{zg6!uczL2;)Jsk$ag|V8&xOY#GPvZ z9^kle@TRaG0RA7TKN1H=(*K#AI@;9WL*z4SpyYgq{cjDg&`hODs&F3FBf!D%g-$Z#s|4j(}Hz9sV{}1C~^)u{$<<@UO|AjlTtu22U zAJiM*b*p&94~*?)4U^BhZiM2~e~a&HYzfP2dRislg(lYAFg}!p4UGJm$QH=)XJ?u9 zWC8ou9|qZThdc1Op0UvyX#j>fQ<8mTx-a|a41e~6)N!+c>%`z{@`QPb3thOji7N%J zHHW}9vfhMXEpXJiuSc#aTz(62x>H=d!ffD6YhoE?!~FCoz{Uyqmfe5@-&Y!#Obgyb zUcqLGTXtb**%ZUyIsAlEZY<+9*)3N}KJgQEcl1pY-+COGfIow~4L#9^eE5LZ-AUz{el777hcyKD)hLp zT}1qU1+R;C>nw)q5lrV$UU)srgem{WVLFnxxXthu(r5NWpUEA$H}qz2*#~|*0lqtt zy=4-!_zRs?5}Z2zeLsii(EV+KNiz?wZn8KAy&CfL&;J-xcNiVv zwrc|Q4|2wG`^-T7<%C(88E3?0;v?*iMf?2j-8EGi_;lxFY_;u3$bUO9nFI|2=DzM=# zI$b^&`URhI%PY@@{^9cdf?FcW|B|$~)1Y1K6IFJ0n&YqGQMc@`NN-3DmrV)&ETZgF zq=n0E>Zfw@W!pb^@5ji^`&q+u&P%YNxRu0JA`{{JmMWd^s{M=hNl&v#>Bz?qw_0_ZNtdp0GkrF;&&bU6@0*b&nzQi8%ys_#S!DZ< z?)~w;m4Ci;H&tTe=Ddfqu)Ary|ZIv-Oi2?JQviVquKhz&F9*Wzbw0b`3=&Z zK(ZIk;?lO};Wo-?+)dtJlXrhaUfNdP8{NF~)CPSpd83Ee&K=}#z4TvsXSsQ=Q(j=e z06dUjXyB^F3$$R?o?!P zGm^4f2J{`=_&}8HD{<<|qArbJC3yTx##XTLx+{;#B+c+7^#62l@cWzO4;P-qJrm#~ zPO!fgoTj-*a_drD6}ZpAyAIEB7XAH*IMp9TKCOkj-Ey&=<@yB|b}1)({TH{~=VMha zC3uHhF2uL9=2C*+?^5o1%I$H>ePGHZ2Y=|6Yvo(K!wBOuB{_J7Tehzwck1-5WybyK z!FHzWTWcNVB!6pPayhuF1e`S;+%*jxb{R5)OVQ6v<@~CnpV9oRqdxgVa`-66zr|Cg zPkno;z{p0$`S(#~`HB|i_5^*m19kGS>xLiNNt^^A1uc{Z{h($4sr zdS0QPyWDzyN*Tv5KJx_}9w4ofw9b3AfiwBCT%tI0S3LQExwDA7#pIiT4-n#jzp=zE zF!|g#VDWn5ZZ!F>aPqO%mJ@daabcecKDYnXk^Kkvd;)$GEdAX$P+q*Z;BB7DfRpro z#D(nz#0fs0(|D5h?kVd@2);r+<4ipX!E00x_)qm5aO;^FSr2Kie@#6HsAs5K&m_ud zJfbF68K0kjCH{+jKIeQVb}}nC1XqtK4#W~!w*R&aXRa< z>B&Gnyu|9lvDzc11wWb|mW`zb&oyl&1P3XcicfRqD%-?y=0_ppf3`yAW`IKGEz_iP zf2au)f~PA4M|;h8T5y8sG+NVcP#@hLx1Yrfwv9+cgaipuWQC1-$di%GcouI|1LbXpSm{S z|IXpl5`(W2AI3BLNc;1b9oyuWxB`Ekv+;wcIos}*{R8O@$HHY(gU?5leTB3yLmk^x zZWB6D&Eu19xmQWENORgu3)V%HdyX`f@wQQBf@pJcaBVBJ_h?51Jx9+$T;q|BVO97p zZ7d3eY`#nRZXDy3xku?=bPUVt*{{~q1Pw=TQA1pSxXj=FXrJyiSJxNb_I~pJBf1XVpnc83GRkSpp_8kh=%njzUizWDx43z4RT~aXUHwESU3c@cUX{1p z&3mKr(sxsDCtY{*f@`GPFLm?IQeOVo{Xv+D*78oc3I>`kUQ z{mG<$(aBPyx5WptD>F$WUh*>G|739CW1GaoaPNxC7eo=KeaM60D9vS0@;mE?`ys$> z4qxER`C0V;1H%s_1dkW0Tq@@YopPSea;ZV_Zjt52QtoZH+$W|S{$ky7hxvBeOAF3) z%eB!z^-*wkJ!M~W%f6?)tSRv&Wo~(V@0 z$-&)}b?7K*Jsp4k|3aEWM~P#eUZIXU#ThzE9CLPnxW`RC(M;l)zi8;mdXvwMV=m7k zZY^r`S z;;`$j9o0#rkZm>&tQ;-=Z-0?^z?5K!xiS3&mVQE=4h-M}#D#$>(@tVA!R_O{v=@ef zW8gMtF4eCDa|Y)7CVxV3iRo8rFvcxc*;y_%_)(W~f&s&OQ^!10E-iS}h20-@mP-r% zrAs-@|0K7)nabzRf3sV5HgTUbeufVt?RReZlgBgBoT(Y%{^upCr^tb$x7>Ozi>!yV zUw=(KkJF#C-FhxkJ>X%%<121GVv3eCubX<<>riG>wfIrs5WIgq zZM|l|JRCVJ^fN4xOIogmlAwVA!~n|Lh!=BDg{q86Hw|0CtQ*SGLocwfl_2Vo%4BZ*(=omN$NSXCfR;vd-s|30E zhQ5K3y@@mUpZC*ve=o`^$1bo0e^Q4$*c-@Jp{ws^XMc3EKWAUGWv62Ua2-6I($$}@ zvk`gz5qt0R!52O|dhE^5MjY(;?3PtO-TU07n?8GM%pX5{>yft4vUO&8)=+!6hkfqS zaiKWwX`BLHyVz5?cXD3*x{+1mLjJ=`_hoxhcF*+qTVJA0Z1qP`56Kp zmrT!H2I($@7pEmk7Q*)csBXo(?~qg0G3xWl>if_6OiZ1B10h4_Ie#jbYEzVz@UE7A!uV?Qc9}*tV z=lzK&Yk1a-@u4#OZ+F>C?6DplZfA1m>Y2djM%riW_MXb7>iGO3_W6bLWLx!b^lN9d zwO|%=UZcH9)46sWaM=Q!$i^k$8S~Z{{9bH2-ceG;enNh98}XUs^HkOwIQHb&Sv`EU zzVAM^xfd%b2T!QHix$1AbtE|6LtnkGj=beu24H{1w{tdE%VAPJzdj-qhXc zlu1PXJ=2qT>awLH%)R5=DJ%MN6F6MB;Bsqb z$w+V_`&IfbedKNUN|#=3J9(M6tyacByGZ|)!P>-s`vGsw+Cp!dxieblC}PN4MV*b< zRgL5x`1Dw7K@)dgUPsx&#sRiy_eSt>rS^!8=i1njQSX?ySVJ0jY~Gx4w;1;j^-1T} zPJ0)!9yCtYq=L|~>F9SCT6IpHXWH67i}$VCwm5Mnl?%YU{7TQ7t-&MI#;&oeB1H{=%4P9 zk&QKXyOQRMWn80tPxok=hV`FW3v%*x?~+shS+@Fj z4R!o6N^~qCm_eU~Q^wQZ^iiG#o3WjVT&KQs1k=N9e;@dYjJkDNlNFThL%Gs!ymzy3 zvB7&B?{QYBJMR^%ton*n&m!x9HNSLO)u_^C%Xq4Jmh;@nvtm@mcdX8}t@W(7eaN^w zkn-&ON@j>Ck2Zi5SH?8^1x^?>3cL{5T z7UtLB7qF%XS&C>*Qz7H)Z*5MrO7`OiUT1IdrJ0i@+oJ+(`8|RBNnUG&Z$Sb1t~t~M zPB3W;2HUCQrtL_3qaf7p%>oCFUZaR64_emeFbelh8LJs8>pKN zKlHxEJjDj;u47N4@4tk<;niltd;FPiXwFvO#!S0kkJQ=`MFqeJat6k1WOX6$-0j4B z{>w$Yb5}g?Z{4wU-w0q&`5xf=Y34t@>;m{H$Dgkk+W8@IBTb%^;41QL>dYhFAOAAs z`3rH#f2gNlu!=luJM&0q=j3^lIHfNnU42<#!sOt5!V>J6jxY`nv0k+QnMa&p;vrz- z2HqcHZ75#vf{TZMi&=_~cjEOf7;+y4QQH&~&9y z-+I>GrFy51_1Jw}!n>IlPqyv%r`A^N@78A4Ky}oX(rOQnwWXAOR8ID|`nkxsCdXKp zt-zLr-*e_h+LZcF_0^2oz;ve%x}SOe>So=A5Tqfpx(bUe)^bv zYk?1W6bC!^O!8k`pG#Z?ab?Jxa$~ICQ~cfQ=bmG&o_r|Vp13G`#|z+#nxh@VH6~A6 z><_$wI^Ee@3Ye4E3&5&(Z%)TkuSuZgkcyiJ=mfD`!w5= zYG;8n!A0A$-{2m1!npIe@9OHfwA`!a?r5dogXrfyznM~h5AQcJj_W-=%FkrIZQ=d2 ztLE+aJ^ku=+L3$JxjV*Om9xXF$KFHi9v-XSXPY(ajGe~{4W&OOJ+C~U ze;A>;^bam_(g%A(PFk+xYsO>c`<@Bp?>Z1WA`;L0WTzSvWo$-R!+SC_y!OKH7`WJz z+?1Q2N*f*39>cqIz01_p|Gg>o6Y*7l7#t=YXV^~K;E3D1*>_M^09ZHQfmnMn-w?HJ z+j%N|*7jZ8`&I-U9Ll%!hUl+LZYGZ({@|e3zq=+XZdGlRZ+C?!;i)Ln$~_6S75I0A zhbw{is|AkWA>&rPP9FJvydZDY+PC0)cS3KLaet=;udOps%b+h$Q*Q?MIf-VO{#+kz zU!nd%lfz@u*X04`i1xAm{z7^qYX!Za_<%i`gJR6yD2MpSd({ko1-=)|YFzv;jK>d> z|EU4Yrysu6|AJus#64`0`LjKJYriQygZ_obdKdlgx-Sy%P)dKktG>b)+{(Ik_DG4r zvjPt7SNvt9{}`M#-QX?V$(0a1m$(KC9#eg8VO`>Hzh&skwf+|M_vAg1#Tc4?xU^$s z-xIVW6CB^+^}KOphKKdcy*o>=p?}%3O@8P}U+75^^dvEYp1cDeAil)R1@z>Pd^hWx zb}qm#?aso9hno6WCF_?>J{0XQSX{}w(xBlBu1_tvush%I%X>DT5B&Ew@W02``=)$f z)JUy`J^n-G`Eyto*_81g%FQS4(*$Tx0_)THkMjKV7;C{s>`RoE*n4c~On48ImX~Y$ z^Uk$%fCq!SwBD#E!CgoA0s8xOR1jr@PZBNbzG+qDT|C+u`wOW1t*BHRJIhcwZ6Rrz zw}!p=3g-;K%{-CxQ1jK)GvWLnkk1G27_WaI6uz7JoA~Tl@aYy$ z_gbH)n?0t$bEtte^$B>ZCwS;nZx8%kSc$(Q&);IKLmSv9_3k!lq}|am%wBxcj>|ea zhA$9Kz0g|F3_iT+>f9Yuz*Xho(b~o5@*ZD*2s~iM`5W-eN#hfBit%ZRvgKz-eB5mQ zOaC~F@Wgm?e~qVGXv`bkLOCCgbH@GSn3Kl6jQcYL1G+aSyk^C(8a(#U$#|@Qy2CJ= z04~!QsI7GPjST8&Us43D7D9WVH(qD$dKhEV9~Z_g>-T$K>vICmX908nRXES=0_Uf@ zaIUsg{}tFd3D!NpVHnmoaCh^wR@(@{x@eU0{*eBsbBBTz@6grk%Ldz73!sf2-D~47 zF90VBMsk6-a%j?A#-$v3bR*}+djJQg!6)_vX8s26*bDd(Z`cT&T()S=j*L6vr>Bp! z7SvEj>7oTY1W#kMC+9okJ9vP;y?dH0Quk=XE`%d8gyzl(Wy53{AuFE^WF#nDA z4ud~@|9}pi1Rv3-fR9YSEjfn+58q7FBI~L-6|6nvv-|%y>YC`WxUyI05Lr`MnS{e#jxj%Vop&DUbLRt*!PYm(tIv^tYIPUqb&chJTxaykYXZ*4#+M;^O+5+UopqD%?H}HOd~l8LcuP5i-O*1_zch)XZZ(MZJjm79;NRT!w0Rl_KdMd z@|}*H;0wmV>2t1~)Na)qS_O^Seyn4-kNxPoU!CYvHS!hF0<|YQakc-De6kO)LeAYK z@|DK@7Vdxk=S9|o2M11=7da-{rf?R?5Y|+bw|swkp#J;O z>r-V{b%1@wZ=j>!9~;=Q)9O~mckFT43tpPF{svw+O8h}@Mh-GE-R&Yg zzTivd>CI0%hId3WKGZdaunOE?7L}H>t0`9e(pF#ZxrQIo9$xdSIM4USheE*7hDL+q zxB9RP2hO)WdkJ=njUB_j7|6aS%E+j;cFL&koDlki^1{(y<{~rQ-?42&RG=Q%+nRY+ zcRP=>Dh=;<3_ryEUOHRiz-JfuCq2M_0PoYv2*WUa*xMc4*CTiiebO3ktxo%M0agZVofw9H;mrQ-ncQ%!9kheA^No^>V=~Aj6XDX z;v`RW%C&*4(lau6tQWlPrN{>wksXzR<2Mt|0l&{7Ot<{42Wd<6d{Ppj?wsU1n|ICq zB*r8=Y315>bt7$icdU|9e0OPJ*4=s+ad1 z$qc?9XGihgirhPzFoq|Arx#COp58p&c#?S%d185zc>3_9@Lc`I-W3f^_s7`A0+Q$%I1?_`Hqt3 zD)*iCO`di4jVgYXGUS_|=cJoDQ|>bL@gGiG)0}oqneDuX>-=zeeKCr*Lz&>9St~Xb zzr5n{;tk6;6i2OitoVJNj1?P-tBHSc`NranD>fE;@7h${NO~b@*7C=SGna27+*oYw zc&0ecI_C`LwKV_@PWLbUj6Hja=E6UC>HV574^P4A(ZzE;{@qVm@wI0%ceC#d9G`vn zfYW9_?i+Ib%Mi-uSwO*5_f&wtN({8n!reiNqEJ@Ux3m5~)aoaxXBvYUnt0=h0ZJcQbgFv#I5U>`!!O z)$h0?Cd|v6+1n~P4(})#)ZOqhHSjVm@G`4rakmzH40fM={>HlFU;5ll8?2W;J$Mex z4AiR+I@=tN?(1XL=V9p1ac@s-=1vQ;m#|NRUus+$c(%iHTKSpm6|le9Hub8~9ilrv zXzV9Tx2^kKd)dCw2?i$b+`_zMwGwP&?tOKz%0rk1^~&%Altk zCoUM7NxY{J{~}iR)?Lui_dL<%O}7T>2S7uckZ;QVv(9swD8`ouCIe*6(t4H~MXyq(uZ7=>MJu1GnJahHhg=gBSj}Evmb;*ExQ~w=2 zRuOCES0&2WO{t&kiI1;{iYtE!_|;x-C3hpC>-f~ylx z(Cx@~-pxk_bRjg7`KYN#TPNNXAQ!hKx8f9u`r!kqdoUkrzl$I4~_ z8ee~eF>eAM2GjR2em;&JeHLSp1x?G!8@D67N3`T6^>dNk_`%IH>F1=pm8mnq$-=wf zVL-%|%WGE}jK0uJy$2e%k78aB=mwz{TZVaB;XD;bQg%yMtC-?Zx2Y^`~1uDvzm6 zWqlZ2TmikG1HGTZp0pX7PzO(ugA6^M2f1+hb->x?zsA?2TMjmR`{zhrHW|2|$i2mr z*<((|zfk}jXRR#ySquA4&MJf&W36_>n_l0;-Zm0G5jYo}X{?Uk(m0@U^atK3I|Dd( z{5~ade_bMe;u3}uOfZF&b#VN2KZ@p zfAj^vX6~5h)z-t^?OUi9UfRe$E&7e#ZTkTJ!K9DuY2QS;_Q}osqtYKncDtD8DxRS{ zi9BDx1HZ-dJWma>pl0vL;LO;Gtv>WCX|btKH2_mH_20BByMsAipV1pMV}Edw$9J#+ zm@zmRIH>`5rhC$Kf4eK+_~@9;-kQM6s^1T-n1tV$ zLDaiyDYoCDA1|iazBzMuOd-7-`ZCuOS1URJk6->6dXJUdzdM0=sjgl_71*R zPbTjqkH0n>`oh1*f15si;ISU8{L<|IR{PMk&8cwoZ_>k^`AG5NUFgev%vl-nqJ3q2 z*Tlo4OQy_sGdeWQou7WN9#>C7H+N!OvUWVle~xMQE~xn&`w`N8haA}yx=Z9zD^ug< zNG=suo}0HSRlJzwQnw?QlDukVtaX|80Gpt%-)HQ6zIWC_Q~QXP>O5qH*J|}W#9oMc zv)MPS^!lfWe)^$Hs=vQyW$L@1b_`R$w1(7AbfXI%t?*1A0F4z6nNzXL*z;#Xi?6Hj ztqa)W*>5AqN(mgRgdV>S9)Ty=+gtRuCq6uZ;fd0#fomk2ZU!${&lI6gGI(Znrj=r^ z9ihB|&_{%7`+dfs5d2iwc#fUUUd!|sIc?T|AlrsF$odZ}bP@1X2CtWeE>LB$34hQp z_<{$>ShS=~XZg9~Je0NQqPDl(`X7hZn=zHWKoaL?1v?uvY;>;_HdyOnsU?X3j-Be4hUC6Jxs9 z*VE=l{Obw$T(aX@{!2CzvVA(0eT44Emc3ph;nS2y9mf7d6wixFP1FdaFpH6%| z|0119z&*b!n}|O!=H~w-&pd>Nl<^MhYqTDHJ=kZnwjQSLXTSkYTlmKje?(nPMV65X z_R)BOFB@YE<{=ln)`wiG8M#!HkxN~727B~tHzSwYj9jW2xs(sNl+v=S!gQCLauf>^=-h;CtUxr3#dQV+3%O(|HFgKQgTPr_gN!sk3Y~Jg-pw0&o7yl_WhRM zYWHUc>ckrcqvD>L%v|}it-4#lW51!kz8127&5iT`{A=zrf%>O7>nPr<0vMkIjK{$v zPX4Rkt`H9fPJ`dy+Z`B(ek4Z1XoJyry6|~C#eRqM3h`*{y~1Uxi3|VFtKalL>{UEH zPVqlI_#ft{4!vl${>3AFm>4{lvHF4#JZ$#kxe+{Iclv*~%L9JwjsI`)fNTE=Jm9+j z3wXdS|4TezcX&YH=_DRdbCDat16IKUKJAI$eTk=s;Q=>&3m)+9E<9kk9n0{5@w?gc z)K1ZU93JrQn7cc9!1dt!mGFQUyF8#-11}D@bMiQ|+J5c6nN_nsJIGpuChpD#-k=$W zpc%(9sCV+6*X^6)e{C&mpihOTd+UvPcdxA|$Z9<^Z}PDkyC$`nI#pi|d?k3~vYE71 zB>n)NQuY9{B?v+f=_7LeRC_$@0Q=c(;kA@Zdz)EHKcODsli!gpp3LMgNwDEHL(O42 za3B?1g5+Q_d4%`07Rt1)z-vD+q5PYiY#(Fn29WPxd;w>?t7TK_MQ;=hPZk3o*A0Fy z7QPODhfDBzhP@}s@72vw7uRROqiV0Y4g7osac@0#x*fUil#FF}oWJ%)_<}gAyV-Y^ z!%viZy2qzm@tyn57RE3Inv=3W+fKMH#Dw=F@6nvul=mXRWt~m zr4>JXjBA`3*JMwZarHRkD&D3AS(jv~)-`k6Ec6Q_8N;`+zdb_zEwmE_j%+s zjQ#pl5AcXhNVR2~b#QaCeVfJ(*zw^naWk^;4%S7}01G`QciLfJXXb_ZO2-~$19J^6 z&u6c*Aler=CjCT))qp#l9ln=7-0N|5$Bp1PQ_p#Q>_YOtN!}W0@dJhy zCj?uG6D>CVf)*!ulXr*du_?Es5A--GIFx*y{eLt88M4#=iNyP9JKG*&`0Xt4X?9*W z_C}0G5HId_h!<+jQ*u(=%1kd&cD8$X!mp5*+*IMo)f<{ zi9UFPeMsv{`(+#3GN52y*f!Su$8CM>7u_=;-;|%Wz7@Y97!WVAhs`_|rE zJDKn{;!lk#8fMNq!4r+x8I@qp^?0hg@n0#y z(k^*?9kCZ|POyeIeQw%C+AePzV$aD8WXB;xm>Xk}*I#}n z_EH9?_ZSG|Kv`0S3MB{!E)%KkB;nL}p>0ck3RIm8AJZ;atOS~zwMfNYqC4lvGLdm82d;D{i z-$(wGgql>9>3gi>dD*&LpBgx}0lmY+p^g!+9SiK$Ijt;vemvmDz!BPk6Vde19&jL z5G>Y91LtgXzNGNkqMtR@TJt*B?w&L3>CSjet6 z{I>7A1=$_=EL#R_ikImNT*fjlg6Fb6fqHcI^~s*(_>L%l`E}U%m4PR|JoCAAx0eU% zZi|3Z;BxEmM<&%Lb%9m+S`&Od3asj!WIx^odr2;=z6-3%5AxT+YQG3r-QLH6)lpw| zJU`QG?6QtEpdZ8bXvE4d1AAw3_t8YY!?5()`F{_m;)SokC!XNc<4vrsj_PmDJV|Dw zc@&(kHRZa(Y2rz6`mf~qpTOyu{}!AMYrENn)0y7HR?Srda>PhDt%Md80He!*)vj<_ z=ZuHBZ}VHj(97TKpIS0>-EnjvIl4~wy@v0*)E;OX{;{8x+>gCSlg11F{5=yU1^<&! zcPn&!6|(@lx{~xUo&}o;AHa{&%{-U$$nW8=fw5-qsNfjn4H4RFpJ9pTK;E_At@kxlO^41|nqx2Z4$Zf>okZnF+uo1#p< z4^yAwWK(p2_lDD~;VQ3kHSAAS$0WD>B=)1GylZc?W-NC&Ad!jXD8J8d4S!;NQQao$6;2NO0{O6KRGrZcK_5SWCO&sX$C?d{O%3-(VK2_O zr{u`Th~{sB{-3qKk3D5M@0!aS6^CulA2{pt0~5#jAH`wk^CstfX7MhYmUj+v&X0L) z88Ei)wa23BnmIGI>jQ7yyPrhYW$|DB)5khzKu(Vi317VSK}_9o_OaU6zWeiTb-TWb ztxF}og0mMdqARH2uDU8h^=k@ks1BcWW}G|FxaOdLILO{w@`HHJ*Oeo0zHyPx>?X7V zGbNHw&ScIdJ5B;t1c%oWu8QBacH$x5Iv;0Osq<4GGGrgRZ9j4a&8qAIoxb+=b6!dD<_z5B1)SG$_pz6I*y{)KS4NzP%M8OI z@@1Wg>(3d6G|n*a-=A8Xi3_H?{yz218HS(p9qy0L#;teH##MAV8&@CQHe!s<#|n;K(3a)=iG@wFcQ+MFcUK0YZX z%yZ!BO%o;r_nENoKlG_s!#y?OzGoaaXPZ`s&o=Es#-+1O=p)V9rns5T*`{)jHC%ka zjhq*_2Yt^@>3^8lxJ83@lo1DQ?w!RsrZUbD=s*2=?)juVZ)jJTzMrrgUmbaV$^1X( zFh(%<$?QdReyQwe?1)JG+|0Q9E;-BCUYqsJ`K2QMH~b%}L+6+{tDfAN`vP{>o_S$e zL|Ok$#gqC!R-A+1#&~2YroFK3FtB32v6J15?QvLm1MzfzX*CS*{EMHg-4MIO7`2Hx(j^p2iQ(ribf8?B}?-l+Lc};pI z@6W7EtvRiGeHQhW^%;~e+$bHVcme4|9y#sedck}~l2symax=ct+kl(DjU9SA`iPtN2i=KrWfZGG*93g;fDpxQAW?`;xS_#dy{ZHa^gf^&%kGX{9|N$YS#KVSGKBS zXP-j+#>kco+!k!5ud@FO+n8MJ@z|HV7-bh%McY#gy4ja{eD-B8`t7k*@%Ffa9`^W{ zK>Lb21D8w~9cX`&@X{X#+V>D%wk6Phh;Z6p1MPiQ1TL9=dO`b8!jj7i+8JUBy2nyo~3()YZk$5ZGvRUF5myIQDp#jEyNizMOD5|2bPv%JSmq z{sW3<)A#D-n{HAcvU^+46v!rzKBUqIKW)X~7sH?Jr#;{9xUxBIliQ~mx{$+qy4sT( zpG_O!l|$0U&h&c9>(KKHUS(?_n*;4(Csr)pXI*mP?(2b5@!^4rJNC`u9UE7pw|x;D zRmit!O2CBjVXkx5y7y4H^6z~@yfXhQ*}Lv#e>|st!r36C1?+3qfKD1 zu`&NW|2pKWj%}PT*n6qxwAx;AudVGB)xEaYt3SHmwgRCl?9Z2{$SyA__^PMp?)R)- zt*@T9dhNidp0&?i{M5SNrdo9at)8{NglZ(d%cHDhg-MSvE?@KsixIWLS@8Ox*b}#hmKhP811HH1UW*)CfLPni2 z>x{wlr!vWE{~LXJXNWU5e=^~z=R-KKHFt->>EC3R(%~KahW1I&jT85cRwuXBM0dL^ z-P0{^z$ok^v02%CmUS6;EN|0Al1W7cXL80Ld+3r3?gs@|?eSr|vP<`fQrFY);Wu(e z=$3>)XfR>3cW`ih%*577z?EqS-M`CIZ;j-spXYyG!57!(@c6pZUT6R7V>q`*{r>v# z$keTEx_5lm2yB>%%dQ$}=VS)zrg(hP*KZ#*+^W<0qki?>LVuUu(XnkNI;~mY#B@)O z)=EO*V&%&!$PquWCv02cGd2}}f7WO7P;1!7^R3WFw_5V0R}atECp+`q)V`kVxf#Q) zl4IWfIlqDj*?cy17rwA*20X_EE3^wf*Ti89BU*-xq;(^@4}Xl_J+0F=la{^gl^&kuwS-l{TjtsA8i84ce$ADJQiA~xD36JxD6>S#MV^L%U= zl7hLwnBYwK;NubNQ_<;x>a3DWI$}rYy$U!y%$`biH~BxphvOjGjaTlKd{g!|24=;- zvWKsYgU>DXaR_9@f=2M&kYo6UtvVAllyC73vgLQ`%(dg( za|3?(pHOw2eejP~{apBmChkpY;2#d0i~ezTr(MC;uw8-9Gev1VLepf+mXWq>bVmBJ z(TyL*&d*5TJ$c#azukcCFn2|2JUSL~2a@&_O9(l$(z4E~uPgGt5oEtAyO^v+S;nR} z-ILl1PAl1c&aTv2WOo{)Tzu_3wYY3YYX;{PnbXHIq?@z0p<^|=u)7VO>)slGm)^4Y z#vMD+dugw54}OI1U0l|-6MWRf+W&abjcuPSa=vHr{mByW1aZG}<6guzc;1p5+vYKb z|8dU0r(V(?dqM7^8+TN|+t+$(Q*-I#ADJWdwRItOC|Bj~&>7ZN{_PFon-zyY=`ROb zp=0cY+ls8vkqK6P$NZR3$5fsnF(ETnUdHEe)#!}=eIo1 z^TZSXC1ESiJ3PC2epNNPasIeaM$cuV4>L#2)!6A)6}6==_168(d}nojx0>&Pc1!g+ zw%WHm)uWoL&$crN_1?%_dp1{2n@v6g7ub8G_>m)J2kQIU*tmdSn|EScULEU{&#N45 zRoyqLqny)0)t*pB^@>p$)hkDJtmFGxecwGQV@&1fj473)JN6OR%Hxd+Wz66#&Gdp$ z$2ELU_k?~!dyO;rp5Y0-r1xu_w4J8EadyXS^3CMA!NdNVfAe_Tb|~L`^3C^zw(ET% z?+ZPlf91Vn33+ejsqloFfo01xo-*Cqo>IQM$aj|~v{~=9yw`d{PpHfTB|0K^Z!euU|O!qch`JN}=^PbSXdjA#gzw(4? zRpw>#@8tPyxXe__^lV$Me7`5(?>(U^z3=0lF}p)$-XZ@VdERr&*x)JK3f-!FACm7w zPv~a7xAM*yFHo7oh3@(h?t7E}`eDKwtX-{tAMbOQWVKDn<18`jcuF4l&c&A- zz{jtH2UdXx=7I+%g9maJmF<}6NiDw)IyiU9+_oHK(W)a%6OqG&T4!2yP3)yIZy#d6 zJ=nQhWOMb5HfZ1K>};J46#n=ac-brw@R&M*h92h;swb4Eapg`LMsjuU*QWlHz|f2J9t)pKGs&APN?FH{9a4x%k@xmRN&z3c!k-qYy z135Rlj=r+*u3WsZjWZFUhv{oGxZGOgd_SmfaHsiZ{#WapKD3N*`fzx*6)FUWs}E^a z*S39$oX7MB-mE74UNy5Vi+9n@SDB;c>Kxmeb4y#=oD14^>YK932^%1sIXQ89(TYHeUCKI;ORypBEv~#z48y#DuM01LZ`+f2fnj_%^gF=|oX2-qh7boY(e(^rGrGr8&dP>$HenrT0dc3& z@96xjkwp}oZfCK+w1%^gC!NVYR`4D^OPdj6wR3^(urzNV^iE&kLbwF_D)`IjJ#n#b zI5tv-)@0lLn9n~Yk%#e3;r&Y91uvuNyTZ%CU1qEp_f*ogmM<`A{es1$Nv5rNPcUit zI5cS~!3m^^$3EJ}+V&0)df;Fsp=hGo6#P$Q|Is`f{Vx1N%M5H@!52l`C)2`Q8N7!6 zrDedybw1>jo#~1Tr=xq(8fl@9$UFmx7r&`GcJ**@VoWkE>)C7N5S5PGZsYTRECx=SJS`%Z<>?rMd*3qn;2;RPxb!Urr>_-sLF zi9+~_MGE05ZdJIhAoOE}@F)uv!k;WqSX~gBuMmD`oHCBZ zpA-mPr#N`AQiazPgsxRMWL)UG3UkMWu2zT**p&(gj0;_%FnwI8L}A*v&@_d~<3g7z zOc)m`RycKB=wgLakR>aeFfKGvVbQqIc!gueg~lqpXk4gJ;fQgefWo2ULf=sce>+NH z`MA)93U3$}8mVyhxX=X(XO0UEQ+Um|(D@2yj0>HoaQe8=xrE3cl5giMxL>+w_zvw? zgujo2^P2}w&mQWxdUsUxuFJy50XWs1JU#qItYLQCp6I zlP!;D%aLlIeOznRV{LKh>5?5C6}{>@Dy+~lXy<_e(RSkm@IClrKkM<=0mp}TFZw3D z65^9#3(@(52gEMC&~_Tr1$j^t8?b!y7&_nv5n_4=4w zMtLx#2v*Y0Bec`PcZcI2!`Q}P=U&pB%N-rqPUK+I-@-i|vh8oK5RK$aHSbGe;T>{C z*V9^?ID0$jLhJ$=D+|5ZcKRegkMB~i{>Q9Y>=lH+u}j>>IU#ee`Iq2ejnfh841vp! zfoGc;L-oCxw!J3Iv7=2m(2gM#UH^bF`$%KMI6HY`+`QQ)@AsFS$oD?^mht^E<9-(5 zxNsWd@s8>k0v`6HJe7i7_(J@%I{%c=`Xc8#+wPNIBRQB4eaK?%LdUjdEy_W@oE&_V zfAa%3+9SO`I~rf#(YE|r6&&N<0M28^fDbzO-^QK=A)CC7@dcs#6vndt6?$3!3Omqk zDEu-Ix=Z2FK#A`h65$N`4433Owy?z)KiTjDLp}fwq@;w1ysLE*p7| zy36@~knarM3-!%hujU(=J*55pyZpoL{DXXu$GJE_`o1{Uwc_+oW$-x$hi}cZTXU!z zo-~Z#wU=nT5B{aA9aubeRtp26!|-$LGqz^lc)sIzO7v@DuKg$K*?=A^Tu(Cf9GOjB z><7x{aNg|{bv7}kjj>Vl@KI2AQr(;7D?2&(bLxB!p8Y7}b(pbgV;zMUw_}Xem#n9E z#|L(7T73JC<%@4`6E2BabbH%}n~LkY5jHbkb9mwyqg+OmDP#Kj zqQBq6?S#icpI)-+Y-bPmI&zj1d$=CuFTL7P^7#2D^3{7T2LDCXJA9BUjiPOmF2JUv+4pI!YKE|B0 za&h1$_QS*UK7x1Q-8)Fvd837dJI}C;ui6KZi`~K_Jk~e!%~a(-%$i8I`p(^mta2i} zi)U6Lu$UOUj&jJ+FHoK{-8$zG7xwL^y==5_1mV@hD_@Vp5)dBzE#f$z|mpokKRAyU48r1gbBe<2z8HrHuIg$ z_=aU3oI@_@dN%j?{7X8|<#KidSyl8K$8I={^Bz4p@6n6%9{5;U(g*u``3b{@SNw0p zoW13?>F1vs@Uyu=y4*jEw$~6J;M?HW+s4=r@ZBIkgp{|wR}c#FEnOOR8mkK%!Dqmh z{{1NbuKlt8NA%zD8=Tea0RQYS^4RiC-Hmv`!x4BB;nnTbbvZnTnJawDHsV`Wb^U_) z_luaH*}6vxc|2qPA@w&fN8(wU69S=>@Emo?_n+ibo>Jxrp5B?Go|>aY%u!PCY|`_Y zqmHP4&KyyfGe;@z93=#o(!Sc7%Z;`tWaHy83toyoSl}0E=e7bK==|1f^z1pJM%wWXiu_##aASqlmz zKO(R1p406X!t5TW+e*(uUu?ebIm13k{Nx^IaNc<3TJwHSx{WQ_p7I{)ChuH)_>W1n z7EJc6Un^aHw!XV#M?ik3jx@K9G?V{CJ^kH!`eS=a8Tw=Dz#hTW!MF9|%C+h<u9SL?YC=?gHBEi+&?+m+Fo&%qeGwDD{%ko!0^lh z>rf4Pk2&~P^Pe8b&;AYPm3v?}46gTO2J(H2Qv1|k3lZtV+>C3%S!ONE_wW|x)Qm^W z*|z2+!}Gw}$h!s1oE;co&cKBKw=4bfSsvZhi@VV>wUFxHRjs;)fZsahfSA1rgGXY`8dzKIv*RZ1y%Ifx3QnAV@xu(<&KUqv2~;LZhqn5 zgx15rt@s1^LI~HphkCWXyY?##`?8z4k~L4;t9@1HW9x|DUJt%i-qgJgFK$lCvJ$)g zgT^vEhE6+K_Onj^SNq4gey+!L3vGwjRlWX6Zrr49p>5%~(Qe%6Zuq9q7-ZQ`IDMVc zwXa>|1+5$donyEG_+(; zL7@I_e8Ua`zpeN5ZIzwHAoRHFJxQ%ELfZy0#_PFfK|K7R9(s>&_2|8~-g_O|h@Ngx z67QLTLj}AKJ6(7(F*uAqm16UFGd7Rs@;;QgamG5&4$qaJHnZU`(RX%^#dXeD{N21o zu9@kK+rsy=3=Ns?{<|+^CtB0m%(6><|`nLs$anMV5+!l_0jDksx9#i6$4(h`mSx9^nq$i+x`$aU>tQ{q533Nk1+pj-*cU!c_q*O@;kwU4$&j#+Ghq} zC +@|SPkYYi#4b{pPwdWrN|tZp;@_mh1le%z#g{Cy{~Q#En#bVkN7ar!vVyTC!& z_ZGsxLrvVXgL{m5eKoWxEW=e*4(^3n?rJPCaW9NDiV_p|!dP>FE-Bm#V|}57eFB1e zVbs6G#Jw{s$P%}uAz0Av;j1&5J)ky5ZT|SK6(xukr3ol(WhV!`PU1~L7 z7oV{n*Ttv#gqO$0Ep>d39Me4GUE~4QGA3EUYa4hTfNaQp6v$;Q`N4bSo=e$t;)8C9 zzdgYdH~)SXxld%7u8gG-Cn@H{Gtqz zh77S38N#k)h=RFUCZFF9e{!~w0l+2b|2dKe;#b$&KIX;U9dbd%yWk_P~hz z!P-VP=P-IHKHoRgu;kirW$hwjLj=txJlm90w{ODU@MGjZ#{IetD zSZb!*r<2yHHEl3ybK0g+Z~u4qkoI8Pv^HffBF&J`lXPp8alcv`eI@&c(%JJUXDX^~ zi;TH_cE`Zdh|+;xTTJ7@Lz-uOnNjyIO}vtI#qqRN>M!+6Q2&Y7Mcz5A=%wtzQ)dqL z9s(Z&=|ee3@E^`#%RG|$7Rs36KWGAYD`Vf7p!s~`gPo1Gr0nznEwVaOyWrq1(_^f^ zO)N>NSV}!>>H@syPX>B@m#RyGev^W+`T(HVuNQIVO=K~}^UXX?>U$slF!m>=fHl! z+JiOVjAg;&1J;x!H8ZEV=O@4uk|FDuOIO80|K)$l{O=)fbd}6?a8AzA$sWrN$Th?1 zj{@#b%e4fkJI){ivDN79oN2&L%lub$W&Q&dO-SHAOP+n$sIp3|V?$JVA-b#$uA_Y; zC~r9BO)|@q|JN|qoTaKO=ux_gb<=vsW2;w5U#-hE=A5sSp(Bcpe3o%vr*MS5$KX}~ z<5^9d_;!{#*g&0RKdJZ@>PXyQNRxPf++gB9xGe4-;W53;Oj%pWoaVQw4{|YF3)O&>Cvy5;S zA4W~g*(~-Y`#2XQKGpm*r8=f`;BUx#EHlt+pPy7wlNsc#of_=TgWgmJ2W+(SO-*K~ zH<>w5_&RIBZE#L-RD3%*xf5^_c)A|h1^J+^pd0we-aUJ_RhQsp^Gya`j-`HLySSV3 zM$umfcS;|(;-$#De!LXim3{pksbfbIr;L3y@%mum3>lUCC{22~k32qTh~QbZ4IE<( ztIJ9U9GrsP$7cNJX*Ku2HR|!-;@D>Mn`G6;iFy*SAO9B{NxX`S)ngTm(c3HNKvui5 zzk0u)`j&hfV;&5jwn{kd&%Ba+l%V%^W21)0+ij+NyFXyGVyLR zWeSawau0$dmEiKhGO-1=-pyR0=#QKu5xh%Qc(?dsob2aHQh2w3IB9c|A!}9|vSyAc zBN%k_9>(KgGoO7rX-Y;g(iV{>GC4eFxAsr=a4`?Fh?8}dX84xeGhAJ^aKJ&E#dD}k z@D~4l3a=9M`Lu(x=bLJ>u)SV^HgYD|q>TxRzsR@K`9}Ie`~h0w^>Thd=2YZ`mX*?X zlII@jbLb&O&K8P5DEKu#X!Wj-mpl&bSj$he}b zi%k19GHeL6pn*}~L)Q>=H5yPx+g_0E3Em8%`Kj8|>AwrLw-1HLts>_!}Q z@Nut&lB=apw}&Ak5&tE17Fk>3G9^yN1$q1ROhevoJCA={-j3HR_OQ+s!W~h}Igx*A zm~SF)*W?E&S-W(g(Py7gw&>^w`Tsup$Cz{6TZjEU0G|3M;@Io251>y(FKj#aULnnh zYn^-Ri1W`q`@PI7r;EO)Jj#+ex5`XcY1g-=$(&n$u{=3@W86Rfq#5VuA%p0*Qjt?w zgQ>}+AIp@?ZQSu3Rywd5{j_yn#_Qjk_$mIn=F(=Vi-$I|S3Th=p!EB1fgd8<361U~ z{N5G_?T-L=Bf;S)a5);BW?f}o40l-ta7Sk_{5}YNZ}>MAKUH#0q`{>Bk5C@_QpS7+ z4c$ywLD;~5yfLn-5y<3S2(|fIRe1gUeZou6kJ74sOaEN2ZTs$cen8`s#GNP#^d9R1 zzn`U5oiEX`o|PTs8{Puoiw zuOjV2t|e+_KYXM$Lyw*xIgGK0)yGM@;f-SyOweytFiGz(_@^NobiGtII6J_=m7B>Y zdf7tCDPUZrpOx+uW8g{;HCD1`QpPV)(Xdn%m!S7F|It=HNKk34|6NSO&Sc_AluC=$ zuO&^|WPuK!XKb68vr2wgqHR0~?LCw!^fyUgq{cZ(zlQwyEV(NE&Alj_Ch4!3-?TIT z9|kmJn;6}$;*z0fV;TD~;J|2b;Z|^B6!eUB7F(dAXP4XBShw3GJ}h^1P1k>A$6=#@bD59sKnE`K#11MztYE z_mD31|MKtCFKd(d!}u}Za}CUwCUd@=H)Uj*FMb)hlK%)^{z1Vw{WS&gucu&wz5ytE z^kQdr>cu#8clgU->Uj6XIGF<}y3o9zn{ifc zk@$??8x3yTgG;Nlor9Ze*$bM1JdqxSq!uuDMm01{d;_R?{5-M;_iifL10JUE-DcqX zE)!=bQ}6#4-|Nh8Myq~J(nkShkA~RXrGJI*j8JLudVzv*`cMU9^}!0p=z|o*_smuN zr1*XJ{gjVr!HI9+)j~rBCz_$5ZTD7Gp>uD6W;)IPcUF+KTm|FxOa&A43r)@+lavA5jpzQ;_qi3dZPPD=50mmCr;l28u?tNoi-H!MZQ{3-;2z(gYRX(fW~*Se*Zf&{(a)lN<8Hh63>M?D*xNWiwvUiKaF^y zJ5o+1@gg&ceMbH%{lAVl*;9UD-tDwuJZ-s+HvNpYje{`A>Co%Vs&h zA!WNRv}{JN-z#OYAh_u)d&&0`-XWwwmzUl+$XCF#ntM6JF4XMQJZs8go6n=44~GY( z=j`O}pWh4!8N6nU*B0NX;*{+wzF9pFl=LSCI7?>k^LFm*4bc{k@lNgTtS!^Jd&`oH z{CU{@<&H9B7xj&oIgSidUG^AZ@c@a-h}${vK-m)Gx!ulzmIaR)&qL&S@c?bXk^x!T z8qWNODI9Cp-{8;p&^JQE(&@v6v_)|0E_mF!L)JZtPlp+_Y&db&r}FqVN%81jiVr&U zozMsLaAjW(RPUqrsdHu+wbe5GbSWL1B@m&g|(`4~GXufqJl%$e5zCg>lKFMxc8tpXb{^2lN0 zB8ii-{B-v%-tTQ0vQLfgheBHyioJHXX42XId~?3D#oKC+NYabJ#jELZB1=%Oaq^E9 z`Qtu-ghrk>`DU}pH)XxX&o>X6 z%)y!LEtdJYi}ITU-zakfV_t)7CUih>XB%T>JTo394}F!mzN(KTkMPkHg@4an%qx5} zS>fHXi*fK#=*T^MS8m4n$1|7l%z}@m!$+Anl^Mf#4irABo^N3;4n9yUbFGHC#yD51 zaSkpW*oY1pflgPZ|IAOzA5^qla4P*$ z+FhCcL)u;0hIUtGw9@X%jH}S@7julcbT+LG?WQe4M^1uom(%Vqh-*u`KjK|*a~$w; z+P#%HKkdHRO#d<3{g&i6X}62#tI=+6n|!?`-&JV$YRPBP?(2A#`dx{3i|l^N`AW3A1$?{OylzXoJAt!)+Wi~GLFV;R1>^O_3bN-VgvPV(qJ7-}^u)m_CkHdlYsq|s_zBR}d zFM}uYoIA|s3swJ!(QAN04;)Ox`sMqfFsl$AqQ&yVXS;G2Z$d9h& zB(j$C+{Lw=0$ZZo^*K5@nhJzWh7YSl8)5+5ZCH51tm@m#CLf-vDH|4~UaG zOPpcjGuCzfK)ewr{MuO8@e+3)9-1!eIM~6(CR1%&Frd{Qlc=YwK1tN8_^+(vTx}kv zU9`s}>iOn3UcQlkz5?lllY?`5)gQ z0`697pISWK`Cg;nfCrT9&rd?R*a z&NIt+#WG(oU~LlH%rMozu`JJW{vyJ_sRH6y&ylgkH*3E56nMjed|?NlYKe1hkN4z; zXq=%XWLSKEu@Q&w$WVMo$~}(ZU7>Ax_#k_nu!oRj@fBG@JWoVx^?BIa!DEly51f`~ zsb7G7UCNa>RrY~+PZs&JC|kyeeL&!#U3XBY)$BcxIQ$_rRkLsAYHJd;nM1jk+E|z9 z*VsZk9jbrtm3EzN+5Ene!$d~6A+H4>w*?}<1tG@;qdRlA0{XScL|NbpXX$dPyRfDu z<5w8q*+GAb{3Ctin8}(rcC^7^uBL6!9yJ#Z^!0>{mOC7cGsZ%bR6pir%6fW^Jy6d1 z%iipt&1L;y9PM1D4bOu-@UQWjw*$Uj2UTC^k@+cf>InZ6ylrCKT;)Ttfyd|~n@QS# zNE4ZE4f}o*@WFVFJo^v!^?XJj$=L2DUCP~ccXkfJgCF79Ul(WR{PV@os?Wv*`nKIR z#P{iKfxeyOll&h5n}EgOf~waYdA_y8Nnae?s(IGqH*`@r_rdc2gX@LP#Odp8rDI-- zEFH6hIBCE3Th0kULsUBoulF1-lQrNt{TTFCCDMAChl-nEIjXbaCS$!*K46q#? znxe5^EMEUR-v}*jWd2FM7syw|H%31sLAPS`!~C}oZEx7>%4x!%(Ru1H?@(XQBgiK* zub$$Yho@xcOq*h~zhp*s&fL|ZRga7f^c8YmbaA%3C+g#vzLN#;k063D-SNvcn=nmqr|Ll#qLK(teW&DcCBRuF{;IX!4 zTojr1baORb0G&n}Uh@I0do!D;TTas$h)nR#0q61>p5C@OvnDJ_LLp zjDC`jJ;mzjaSdmLeqs-V?0t~?=>0TNa8Yn-!z@SN4O1L_x8^%$ukGRJyDrnwSLo1& z%pT5{BD*`)e2KMt&ZQXhr4-**G7nOmgMHk+u}Q|;E^{Shh%c1B)dGh=PeSYD{)fDH zZCromMImP?8@~)N=-Ybc>QSq=rN!b^b7j4Mu3Rts=zXFyOP?muZkZG44!cX$e2LZP znf=|KG^OicM|oIqhCB^-U*|dhH2yxp(E!pHIA60(*&zrwTnPUd?)^BsCCe$V&tjqt^$e3@$w zeGU04_{JDVv9+@Qin=s#4#PWZ0{AzfZ%yY&r*|yR`SKj$^!DI6Q=Z#7H*}umtmzyM zd=MDsWW4bmf1RgUjd!xXl=5Y~YdVi|)>;|oAVO!ivo;hMAJeX`n{o#U+KUf4iaPHhb^ z#FgW%3GNfp&Xseoz-U)aF8*~aH7(*pjpwAIwMTijq;_99$;kI(nXLeqYlw+pSR&I}YfwTzze{Ehm_ zT%|vEJG!;<^)&|nW-lVND!P@gL+g#cUrXO}Z-I=BX8GCNHLUMT9|=!aa+$%?mklbs z!BYk9N`If?UnWnl^z-yFW;ol)LLT&H0%l z$hR$ZsPEI%ARoqW`Z8AE2NWKDuQ}ctJiL-P#lzo%hYLNpi8~MY-z7YpIGKNHUbgaZ z&IR7!@$+yW^igCVKM&tRxhs(qg)hFzv-t}cte(lH4czBA` z>mL3ueZHJ@smGJRNQFm=UoXpeC=C3J*KZ)+&#x~pW2{-mLh=Z|o(}}isDoL?qHtBl zH7d@b&rtaceto}+OVICAFj1eTV4VI-1!MJl6pYdDR`AF9^)SKhvL9cQ72kzoqd05s zTlD*w$5|yhji$ec{8eH@rWL&ZJjxTne_iln^rEZYk>FqZ+a_ln!gD##>H1WqHI?#vz*DTDK1 z!|}fr%6%+Z8G41bAlzer{FWT_kC~Iv_0gAnS>TQxKdLq0ogH7PX%=hU0&FVwaQ5VI z9$wCSSmZrjXrQBLt>`Ts?tcM1T!XK}i*e*@B#!+mdrz}g{1&>P=!1ff@~zMm#!zHy=*f+qmra`S zmBXODa>hZ{FoTH~TKjk6{j})fchxs~R+`^^Vt#j8?W2;j+bO!}i}LNCNRzwKtjwbx z{D0QL8$8mc7i`%c{;%|Bv4Loo-2HaWe=Lo(mutTm=DStKTD}XEZ?B$jDEUrPUn%4F zQV+f)-yJyG0P)%o( zU!VuN`2Hfl$Z6#lF7QX_pyC%vt^8tvpO0)ec{MmCJjCGDmYECGdXqz+Lb*@+X}9nY z{wx2l#n*)~?p-~d&GEa7yqEKlZQzg4yYZw6AE7>*-iEfGrB0i7boIO)Wbl#Uq}$nN zWVdhoZUZ{@+whPz9k2^@2=e8UCcH%2D7@sC{`$a6t|6{1F9}E~9g`2vwB;ooiMxq7 z#SbpwC0#G_l3!BKw!EYh<@PaoNgB`U+sk;#aK35FOA^Uf#W!vB!9xCf2`||RFTod# z@QZNrD_#N*Da?eIWCr;Hh?BHHVCEJ0$EV<_^rww<`S8m86}8R>=H-Pj5np!_J5PfZlgT^fj3e3$OERcrwpd zldCCOvxsjt{Q>qE4k_ennl|>Vhv3_khH;(FH#su1MnA=z@od zlljwD7Yw+zbj)OlYoiN(K-`@Y*G3l%$SNH(UgFy5f*pw)BXMnX!NJ7cLY#k0WL;Nu zYw33vZTT_XPvjq&D}LSY2J-vZ<0bmB;CE-GUgi?bJo;z&M>~B|9Q~^H;4|1{^tb;Hh=@7|9QcK4dB5$CJwv@4%813FC|u=c252TU$nAbOwD0UbpdqW2|< z-j`Ig-LLo6x9WWvZS=nFO7Dx)cT3zaeWgI^tUHbbkz!B6Fx_*h?cQ5g>-$ra$ zf(N2I#vu>L7(Qv1@kjE=T;BrBp$v3DDWft&>3wnf?Zk^cOb4E&Z=`=-QE^H7%L-!u zQ_!KWRWM#(tzeA)I|XC)3I)Y4#?|${(nKxxWdFCPAMatoU#u0M@jrfVZUS>$d~>hM z)_Tesz*%JGGtfM_UqEDKS*JR$*7@*1^2q#sBdCMWl5Gj1?})wLxC_8o`-xR+Ki@Fk zuYY`D<(hb{=NfGI_t|2@YS`CaYm3dENL(Fl|JIhA{afx8D2dT#p6a~XvD(_M%L&%_ zw$3m7ZcpH9$8YbQ_wuO59m${L+rPhW3K)_PhPhjrX30FOFV`0b8?0wp$I&GV8qK5UT|ilUkKJ5KX%xl@}XD9z7w?Dk4#FkRjT)NtEXJ>6TzQCgK(NeD=|Hyi$ z75jfm>FP0U(_~Fxyk-e&2;Jtu-(zp~OzfRVS*NK-$pUTW-N;6)6YU*DTq88xXKSbT z_q^Wci|6)kDxQ#B;hxjmS3J9SV{zZ)=iIj^ z-_CmrY2O#$k^H=SZtn}l^LjU{d^yPzc(3FASn=HCRg_U%{7CPb;(5t0xF;k}boWoa z-~GkVzU6&WXS;J!=eYZ&&UH_mqHVmR$g*+&;_uv>2`dPRWmbV z?!T{J^NPiXDF2Wj*Sp0_53rP&hdntCj8d@&(sw)j0_ z%%WFwu2RNE?&*{HvXuU>10TZ~i|OFqAHcg$EU_a;fOjXsGxWn=spJuV@I|cetmb^0 zi!;j0=A@0CIpKjV)!@nj?iD|1i%__-{*DK>Oww#chZd}MWOmeM58^HhEdX4hev`Cl zg)262rEQwjNAARF`;Th3wJ*~-(HE2Y8OxH#rYub{Z{c;?E(6Mns=k@m~|Z?Y#-=1TwDf4yZAYxQ3;SNbzo zCeY_{2Zzj+Nwx&l{$&2uHci?ubA<80yv)7D=KPR36T!ROC*1&^Y#G|S+!Cy9v;>1+ z3&E*{;1k~pK5gU9Kfxu*SBzZttd>~55*%oJv}E6!`}#KHmp7XIlW!~_e}jo9>EPnC z)`YN?mV}RAp3r;terQockXFBzJUbYVKH!JQKS$yXdVjTj#B$$aU6(=7>AdMD`~UTR z@h{Uj>5r^W7(P0(H0#Jvi!H3yX3eg#S+eiptjnYn?wn$b!)W8p_)EA2zoG7STJ)%B zE&A3tEqZj47M-VAMs^C&Mqg{!Mt2U^M)!!)M)!=>Mr(ovVX3?qMygyss9pC5O`9_bUq~8hv#rMe4*JsqNW))JIzBOp)rq^-KuY=KPH@ka9b)mrs03+)N*}i!$1Nf_6)rH(El=Uf*uCQ)4Vk{BZ}2_7&3h)&4k(Hb?Gm=`wTv)2?;_?gili?riZ5 z6V3TE`uIBd$ijc^7$~sJvwh$q=0{+lHtK#YG;A^XzlDDuXZ}y+UaniYmn(YFEAIA- zD&6}AyjlF|fH#WwEPA7O@1i%0yZ3yvxLeORian{_+z+O9cXxA)aeGo`x;=pl1{_`V zvO9|Vjy7g!_2L&~ggK6rpf%zfaW(A}Jh=`&Ds{B*e+%~zS%b7u)fdqM_JNNv#}@}^qvXBw=Q{=-pdXjfrsx36 zs2~%EKCwiWYg#1!lek|5|J3&|-cQpnlS&P~CFhs3$a@5u_l+%FUk!T(mkM^>bEiuiPq);w$R@7*s)X>VkJBS)YcTPa7w zAMLf)z_72ifTGUOu;AI9X1@##{0JP&wFD~O@ecSR{NjcS`TPDc^O~`@Fi$sHB1b+z zUq@Oa!vt?WIsVALlNRep?6;?^U+fqt(DrmI{+y!UXKLYL*G2@m4}e1l{}(vaEua;L z2BjM~bifu$nUVBIJH{wN-%5K1bsatT=<;Qb;Az*+{>ZL*XxC`rPsl^{7WzQ=kI;{e z%r)V27Ut}kQG=jkd91VU+#qzVWv6Y=$^L=Rx*%v?Ffyr~{V5^nwxQ$+BX2lsv+dY( z5aB%mPnf1z!<^8+8!Z7GYbo9PA$OS^vqfarL9?bKGv3U+s|KgnGF~Ojb&(&xwzTVV z(v%Ixh3YNuMR!qRWpe{J>vXcWvA%EhLQd4ob#LrOgiV?FR=M7=O;qBUgs&w{z!P z(xP|Vjzw>~lXKrKPRXq*PFhk`>{#+{alfAL7U%S=Dt&^_@Z|r%lV!|vxc56LAgNsV^n|dFl&s4BTP3UBhD=ss&UvxlimZAbTxtfV z9){lC4!!f$266 z=Ql}+r7wpS2Uu0e#RP!pTOFi={?a{*Tx(2OwcqrpB=G77G(~psFm=={C1Mi4s zZk=U5En{xQGq)OSfgcO~ieg>~Y|p$}%)DC0yb5N1NL&E(>M-ZQqL@q3{y65>VcEmZ zyo_RQ#SkCG+=>;*+>$vI$MdtyuN=x)313Qt4~hQ%-A>C!|NNR|jV+(YJQ#4iyDn=7N{cGoQXTQz;oM)*(iBzFn0fY6d$ z;C>;kxBHKT{}5UTDZkF$nEL3#*XAwR;+{YlWb&x3$P{Z>tAQu0dNg7G?*}eTfbL2C zCZxURuC!Xq?+&(>CsW6H{C8fL$r1C?u8o)oubQyr9d{J+tMIB*Chnv&=XY46!X{}c zMUJx7j=A%?&W^$lxzKd{iYeUb%KT0Vh$_F;5)zgg`jKM^a<=r@4og(nUG&kv+RSn3 zx531TE^`X`e?sb;?qqb)<_6csKQWJ26AB3T5_INCIpJo)biyB&oN+%zxRvlAp<|z> z;!b_e7cWYD%-yL^b8(+O7m8m_TC;ktQ=jkQ{V%#-P8`qs zao)cz{w#T=`=&m1#kqZs(dVn(4UW6q-TQo3+_lfC;wZ<{Jf9@riQ+MQ7lkbKLFzs3 zRl_=zf0+6U_ukZp-MdpCa1Tz_$ZOgcWv|HNV#JbJ9jL%@~4XaWr-4!Wsx=%4aJ1o|)j58AGPSi;XjLW#aQCoP1aIi!8Pv zIRoBAJ+9>Uh5fPEv#>!Z7>5rZ<4pKi?3+Caa$cs%=E!a;vUz>Bq-=)iboUv#A(*45g zlwK!st^0+4M_d8zs9}xM@B0P&-w%Ah#Lxy=i+_!O%bwv~wCze~a@y`0?rnYpA5h<} z0vh#-(^smvSp69VWAx<;%Gtru;KHro!YFXT%^2STF5G<4w=#P|k;iP{L;$!Eh&*=n z{lcfP1D0#HFfVdn4Z7$tc*#o+t*6Mc$87=Guftc|$Ybg7mDMKCNV7&Np3#CW??!mW zCV0jh@QiwR%Ny{FL0V+?FW?v03qC##oxhPiHV2OH7%1>y{f>c8fdhxY6OqG^p>}@= zpEv@aK$hBl1paUY{*VaI2!;+t!Xr+RHxeFE4}2QF5DA|+&9jtu20aWuv0G#?*~=_^ zLU@G8t@CVwN`?`+we&h|+)c>VoU>IjSS0NeePus9q7St8bL!yZ-%@6iS=aN_^(3~; zL(sk#Qa8E(O@029>nq-x+oO8*-bV`Aglq7Qa*cm&NZFhYau*+XrkYZb{36 z{yW{@!zb=aeboIuI$bK?=Pdain)5yM;CuN0_sAHV-7hTJ?4E**o`H^il7D@T?B;@w z=i4Gj=Ax4x%($!R?DFM~U(LI5_DR}>ZaaFCB~tk_eQJIfv^=u>AIJvZhkWF?8(J=H z6T9Mm{U&v}h zpx@67%Pb#?%r+5O?M`I1mmS*1_a3}dR+|DH7g_D(0)viYbFSNVvSpm?r;|3Hg}-iw ze*Xgc{SIy4LLZ#Oj%Bmj!lXSPA*;QdNIRh4kEQ$~;#bh`$BKF(s~uvj4!Ng5zfW?{ zuh8$4@Pk|_AO2uNziY9i)}8IK-0{rBO#c`~Z)J=`R{J^hI}N}2c5D`*jKg(|-}TsU zZoqzXBleq2>^ITaZ;F^R>DX@$GG{~%5?hAo{oSl#*l$AX3RyessD&c)1QZSFy4ulc zUE1t4{I*t8pT{hrVd$boi?UpEkHjo=blO2$DE1f9UN+N8v-0L1DI@J5X%X08z_$-C zrVTREtlFHk_Sj$0p^scl%Qw1WIkcIj=-h!9(}v7F z%>Oz@XmdWH4@9q1bcD9#VK)q+FObQ0=P`!`4#JiqF#m}0KDgF^Lzs^eKa}|x3SAh+ z+`o^$QZ@?a{SC-|d!bL_8|OH>&%Tu3yEmo0?EWYDp2ZH~*GqC3^LKALSNz^h4a%1N z-c6^A-@oaMvR&V~cuVom7grTuGvJTKDFfawo|iVjJt1wNdoFsG0~_{S=I?&y#|pwM zLIWXj$s5e)HLr>XI=*G;C6WN&0U+9>1do_S6C8?=a#LSGKhUV#T`ufRjJ zSKwjVEAR;IMV1~V&qrx+TmM?L9?%Pto2}wD;7-_KH10+FK8;lK<(hz_H+vp8f8^9 z3flqw5?1~RHiWHWLjXs9>`X&lmm%=3!N@cD7tc4OMY-zP>o*Qt^%$`^wDntp%v-6k zf6vaoDcPqZ>sPW@M|`icpj5fSiZ8O!&I&nS5%Pqq@o~IV2s8?G*DJ?d++aL`7 z91fpu2fvQM{t=1)9UEsZTF*eQTMatzU?e+S=?z^w~W=xDOdJu{~DPg zO-sNh&dOCG>73EC1k~H{X)fnElHyjcs<9?1`6(gfUrK(;?qbMK3;164zgQ`6ek}6S zA>$l`jWZB(-f9|gQoig(5Z=;}I*z?8ZZL6f703QR$`hZN+tBUg%#{yW;nFi#ar#c_ zU&_3bymD{G)#RtGoPjF@7bNdD<~KK+?HvK^$$!O0KTO4mU*l2qzngKmh4HuQ^A= z&}Dxzbt#au=jfUXRK1e)F8q5j^PlzPO0`Fpa-=-j-^6~g-Gi#Tc}~(dvOaw+c?D;s zy_(wJ5u-0<+$4>&ENbt^HvY$c*uD0EZQm7$>`A}3C#|4IH;-m<`C=`aFN}AI4+r{y z$O6Xy+0RrPWsK*+HhW_u`G4E}OfRuFHk3G34|6}0s%v5Qi~E_nQ!m-yAwE52y}O=r z-)7&5JlFB8zP)5WQ@kF>HzJFR|2N5Zn0$R@Khw^e8;yPRG5U4EQ#v!Ju)6dC$U z^7z}9NZUT;UE==Ab!{?8J`20~Y zPXE1vvHD92;{QWIIp_C7pC{t)WNDPK_w{6d3pCJ*{BA>j518IGH4q#P0!M?vQO<77 z3&9t*)g${}Pn+~a<}3KQyCAbD2b#D{XkrL7F&&yHW0pf)4t^PBpQ(lYd%OJf?HA69 z#3YwbbQb&XIEP0W&OJAHvafeJogsqb33aZFpUo|_jGLQo89TQ$qxIVmzU{!b>-biE zXUX}Cp zob|?@ZSzYx4^VbYiq^B%76Yw}%`QQnv13aKHP?db;FI4%ACwGb2`NV(Q0qa3vL3`b zPziFA3)yV7MVmR@vf6PA>p&&QOKXsqR)-MRjX2hU>XBPy&2bMnmwi2Uvq!Np;Y(Vr zy-?p$cs1)ll?&k0*TLtn=Xo9cybRq&o?n8eAEz!i@ciK&I|mB<=;sFf^JD|={)GYe z+-tzS(+#+9rU5_wl>z@b$AJ6i?;Lo7@$|3neS$Cb647yDv~jPqX7<4RkcuemI}OMT zi9H*z8#kzR{X}d>iD{V;iB|2xB+e*Sv)A}`(w@IOJ&1JHovPViyoI!XU7j9FdNOggl53u&Jb8XV8L z8(61lNX?8$aA+4!v6eB(T$@23t$Ty@y*+abTNgg#>LxR`Rg7JijCoI&kZ&zxretsW zzm)M&W5(DBTxgDA8DkiZthI=K3rFvKOdxvaVu8@_B?7U9I!)R0b!%vt$eSlvrxD%s zL2SQc(JB0UHr`%*n02qi?%N4-2=8L^tsvYo+8x(ix|UXT2Wx4Q7ry3~amOIzZYF$!KD8FRnCK!;FqbBQYZ&k&)6J{ieHn9^nPTgdCn1VXKz0KSVokL3cic-g_7u!mq*g2HUoQ+o^LSAtUiQcLsWMMoQ0!$=If4 z?PNb;QIuAH0GfKpoUf&v8z|ANbsbqp4Vaxd8(W448;6>&FEd}c{|UQgJ7lyg-wXCr zcFVs<2%g32A1fHA{{_g})tJrLELVkTJ?9W^C!9n6^kxq7zSh^p2SCs5mwQU-LyF-s&!;18ebDX(z5`3g#xqgAdO#WNh}WuHq2kDIOKl?Isc=e*f4*1_m{dKZl@`i|1a@ir1{_B z3dZTffPx2S!GkXt|0=?B|1BOYxH2AKlc;nFj$8>JusbLGKf;F(I~(|rLK{T3T}au9 zSHK6`Rq)}sIX-Rip@z8s1|RM+@u8vJe~%9f%x{kJjo{%CpdTL&s<>FaTEQ6oKMMXg z_+UX#vmyiAkbwh`fw>E29=ckY_^Uw2dZe4@8Ol;TY3=>$ER~$G-Ui<{>;bMEXN?uCGc**Kpw<$>Pl3ivui2DQR-YWXL;Xm`c~HsWLS z;k>)JM^NPQ&(Zr5r*!bxJG;C!_+8vidonil_2iT1pUBntIrV%-T*HKZo-Ed&oLx$s z$JT3J@p~cXDdet=zU0lxXAdiT)r8*4zXuQ}>1jah2|B#_qOXktWNP|Fbfe0QW&Lm| ztaPenq7T_D_&t<9?b*kZ!P)H&%$p1IhSKLl==;HpK|b_(khcXLxmUZ^??ey1SY~q{ zPmKD%=-*fV7UNi~YBTr3Ui|Ln{+>X-yV^JEo;J?KISFaVb?CU`#P%ZbrhTZr%A?_1 zgg~Amb&f57cY?e#=I!mhmQJX)Ep-@qOWSL_%X8Dz_h_Gmc3Nq#4IJa%A)Cd655_TR z11vKOz^8vf@3T1bR>Pis_Jn6+cd1NVpW%rcm0teVAlYvpqj#YVn-+>6tXMsZ=gm9v zdB*=I&&#nPjYDVtr(N5OjW0HxvaF>GlyBAU%pDiFiQbKWjPCg9$H$JEzwvq!-*$%v zSjhWyxG}b}K5OIse&X$s3FTT|o-aPn;Wwt%!YM-;*1i+ibcfJ=IVyE7cPu@51&Y9P`DL~oRCw6`He|SL_|4-}08VGF=dtbUh z+P8qbr`h)``KF*B$p5a^x0m02D11orueMr>7O?+U*6i|uDy}zo32@e4bSr1Zr;gCl z#k3>zJ{NJ4KB+J1_>VTycaZ+1kxrbXm)uS|{xFU7osPf$p>#~Pea(YX2J^u9N2z7z zPS&Qd-*CU`$f|qoS}(CF$+rcxc^d0gwaf?S+GtPu9n%_jHeFEuGaJFBYR>gW3vNS) z{%FR3M|>%KYjrm3PmB@v8GUtk`iipBdTBG)(pQ4V@R;4&I2-xk?*UUHGC-eRX&I<1`-w!Zn^jC&E-@q?)ml8+v zG@ThW&^-&*is z!+gtFg=4=PS*2U-BXhAshG2WEWj&+7veZ#?qvn%)(*ksjoj@@D=ivF)jsC9&KQrY^oWn`7Thmnl@!k zp|1<%ZYBOD_by4>Bws1{Wp1|3YhzAIJ0-2|vNWN$HO$3JWWs_lPX>Mqga^MJYThqK zTfgPGnzkPkNMAsc=xdkn8-euIVS&S3zP}1YUR?@Z?ZEvJ8aUK;?XDW!K|WJ!ccG+> zu&{Q=U2Qv&N2fdx&b^2>kHr$~>C}@u5u00jP72dz9-<93T{wqNztqs5ocF%+_tktq zneQ`#J#q4V6ZdRA?BZS#dv;xyq29Z1u~!VAZm*~XUu$z2znS(5>9=ohbor*t53i^x z3-sIxOuG=}h4%aU^Ze3I%N}XZ0eHz>X$J1SNgGAy5I;fww1d=7{00Ba%;O~wdx-Rx zSbw^hhcOiY!8e|+dw^pJakE+@()~aLR)Ug|>?|Xc+ zmAdYv-G8=J9Usg4c-|-SKFLy5z0c+eOfqn$6JwzGmdxo<)D0-}_DN^}dvO)NlJVIH zeGG)xuG?wdb0|~%r=ep;GQXy<_WfMTN_2;wr+>ru&>Q5KO+t6(Q^q%xVdYynr*XA8 z5_~UXtSuVT!W!j4f7uEr6X8pod79z!-LYW0nwzRX{vuYkx*N(L6*E^?{tu@~LGW+b>QeF5x89_a^> zSF&WR^EKtuqy{{z>1=@?8}miZG>dMfeD6`uBSp|M=FHJa^z$_v-0dE0QNCHvC9HN# z3Pu(cItty}7y^BrML$~kA28Ldlkl~xwqv@B`52?W$hT5on_2JEl(#=n>vL|Q(M~m< zv{PjNwtX#Qc2>shkk}UJN9L-~5#_6udLO0iQ_r`o6k3@HZ$KvNDRork0kMg#IBepF ztnI5Z_(sVj)Zqwt{|)r!D6+}Na~Xfe^S%{LmuH@%b zWZCoRb+V@WewnMPiuqYh+#vN2{C_AIr{4rTH%rDpUhmIy!wh-GwxZG#_1-|yIpwZk zk;`gm>*iSe_Y!6inofp$vtqSg?|wbViT@$~kqloDU-9BQH;M5`(JkDGru5eKT2FmE zcSM&YB^M)`?c(lX?rzxZo#?8Xj-0lE_g&HW?t4|MdTp$$YGui56*b>Kx-W~kr;y)j zBt1&oybGRZ4{ChpwC;R>c8;w>U`u2=EKKk{4u^gR~F*!jSayCu1J531-4xNKkmM0X1HS9Ms3j9!Bd)JDEqrfq5G$&mZZ>65$YCxt7tWrd6(ZF^cEZCxgiwm&Iw zl*{*oz>zNBg94F7mI|DK{~v*~T)ssrK2Be#V60xMV2u8#g7JDwS;$Yy+tNQ)LuT9E zR%R3YX(O}kZX>f{2W^$vwsgZcudk(NE_7(8$ZX)2$ZWgJeF(^VmwwwuUi0$pnueC1 z=fH8K=rGxsw|6v|HFur zK9;sk%hWbaB=jR_gz5y=h-SH}-d-*;Vp4Y9lt9lK##Qez*t@ZRzW4R~LEb~1UA~#< zN#4loofc%}bo;Oig~&fMheUYapW4lNAlc=cJYVyz@0{*@U|_KKB)BmnkA39GJhBgZ z#*h&2VQ{PKqX88&^FqBiqWANbhjI ztwr|n#gLw0sTxi_YG04`PUBoi$ozncnpfI;ef=pb7kOn8Z8znggUCKD$S*a0ES|Lb z$i`~$UHn79C&!dukIo8Tsea4m*&T1lG^c_Ym+|Nw=;VxBRa1=SJKq^PL|<3akI4R; zZbR-7`9|z)G7eQH+zeE5&J^pO_3WjqUOvJr|N8SY$h_FDv4?Lu2VE8VJj5(R_@>0k zcY<5?Rz2V%UImHFzE@w`ln zit@;OQO{3Ac?9p(^NL7l_*T{{G|zRvHRvFl7+6c$zTvwdL*_jNl)b6VGtjZFvOiU9 zSKlC?pTTz3i0w-Dn29e37KW6csrP3La-I%it&P2@k8>Afx_Mu_(&5(Q$9R5nh1hbM zOTwMUIQx?&@2|2~b-mpyvfyF9m3ezmLHvIy7^9b`8+1eB1%J@7mRX&{eE(*AjJ5B0 z-DchawTrl3s*lA#jrhoR=&}b@_`C3jU8+pK@0vti*1%s?Y1~(OF^zi=f{nX*UQ%hX z`n#kxpR{_p&&0Pz_NsCgd8IXS?tiT-6uwX10p>UB z%zv*@5Sbr%rTwYRGe)%TQQfKSIWx+_T9lRbPR%2_XYAy`zE~S~vkE>uk?pySa>V8! z{hP_N(oZd!9&CI*Y`44pKKAj$=X3InGqAfy)}ijhK0k>)a8AqeRp#2`vJBS^o{p+s zgIGVwW1UYk&y>j;tcA4{eDIIAk;dBCJl0`ZGibY~`F_5YdfW$8^?Rg`=LUI3UK73G zE#L}*}rM_KMT8!>A^IYjZk(M%JeMRzfW=GZITJp<3r5us_=9qP1 z9YpyTdx`ZFqdwe4MSVo}795XM|4G!_0lyB_>Z>_(Amvt>`upnF}eipIIjMYokp3(zG3mcg-?)n`LsQV8z$*v>o3;QqQ?& znR|#ATvetdz%?I&2z?o&qn7~L3k*CBU0oi{eJ=!w`z7Nc`2R9#QnyvWmCzJP zn`EZFXr@&|t3;Q~q)p3ATJ|Jql6QcamwO{sIZNA=)5}a_FPlnhTh60qIggM=fAxW0 z?bbdW=o!R&A@oJc?q&9$Ah!98!AlL|2f`y=Mls0~4_AzUb3U52uV#=48 z_3cMo;k+K;M55k-XIURzZH=kRW1q%B>M7%Kh;bGE)qy_AVZXr(Hp`w|TYR03vocaX zf)DqxxP0ve($`VI4Ymo5infDyPVg>Q_GLnkz)j?UtKVhxaDcV~ntlmS5#F&)cu8|# z4{H%>E#I!Wate^?{cE*ipB%(m?Ku8jLtSKz)W!eM{o2bKhP~_3RcZJQseWrIca~hr zJM5EKi?jyJw6oq5a*sR*u^#xtd{?d4-*P;^`-ytp@PARSX>$|Xi!8~x80sT;tz25~ zAG;?jpIKE?;_`9FSY6G+rK_54S-RMWPICtA49{we=Akpq znY$(bTuYx4`r8U$+G@4ct!Izw@g8pP?G3Yh>Q= zJirrl@j2WR%=5Q>LqY*oHtk7bD z=xq}B`GePtRpl*}@)GsQW_j4M6+KsJ;YliOf|>R;X+qbZ$v@A5Z=ZF? z|1-fiCD2<4tU*^3`vh~Tr|4OKCGR2HByx(#2_NzPHSdD!LSJ|A{w447FVX9TkI1>B^?W*uTyDKX@1WXEk%UXt%mXa@;c*=%UW>sUYh;+Dj)A))^1?kMEbYqccrF5ui+KJP{C)!beuC$d;H2=`6X5IL zd5_@#wILyh?(Cn;;EXfj34$|P^Vspvwhg=M>yNn_Z)|DVe51uxwTwC5?an_`bdA@% zm5)SLpj$>;ZBgYNLh%Q0*P_!GE*PMZp1yG5fSKg&N?LTR=8dvzzKn4E>8G`{~2vfaVV0PbrFQ1hW?Ps>Wb9C6YL4y0*4_o36xvN}Cy zn2X2h&vT#UdpXD2vt|i;e3|Qlmpz-#wY@yq?OfdJ6Z-0iZmYk@Zbuz5To*F33R5#O zvE~Jy!m1nt-McM|7U8OO0+zA#| z9uhai=e2~q>Lh~VV!Vb__;yPS4M)`k|)&6bCb&_ ze7$X+W3;L6Vfvl_pXiKEM}D<-HfP=z*=r)}{ou~%d%Lq1X7f7=7+MkovNIB8;H+w}}>tCH|?1vCJscpUdb+1k{ z_h`&=`C{mg=_+5$#Wwy_{l)ez^}kwu*`xaE6!^RFd+E0uhGu_b>U-n zo;PrAco%RQ5MP#_WaQjD{G2uy3p1$&>x|P1`fsP zUl1qX&SM_P8oBV?1n}`8-sRh0NqoK$A47bxijUQ2Nc=D(Ue2OUQ}GV{UWp%R#5;(; zoA)!B#y{m**1F{%^36ov&(gPo)AH=+`EjI;-3sg7O&V9h0a~Hb%Z-SwX6m7ht3Jz7kYhm zU%prMA$`Ld)rxBRt*1cxrMrT0dRGNw^&1p)=+^<)TxdxMq0fb8-uLjMPuDT$gx+1G zk9nHsICOI(baNxmCwTq@din{^Cz+!sfkHc@pqZVh3-qn3l)mY3XJv&88n@<==!#6} zVSr659}K!wIL^llw_(@DFVyoDw`EG4awJcXq1Abrp6(G}K2%^OX+&`6<` z^?@$mX!gVkO)iqU{2%JxKTfXWsuR6!E4DmV9NSS6Co#^AZIMx=(a({h$UmaiOpi3t z{L;)wwsFFB&vegBXL`C@{bMu}2q?iJUhqB~p5LyQ2rytC#_{sB;D7*aq6BZ^<>LiE zo?-nK?V<&<81UP5h&NHd55Mm@r|RCi_tx#(Jqmxk`KWd4-a2*a)Tv*mPMtdSY3K9* z8o!6}`vd$u*jZovL!9@|c$aVv@|kab+PicE_6olVd`GW#zVs&;PfvW>J9`T8ozv=F zYELWjveAmXk78f6Y!X4)a*tYp0-kKlr>|1X>Lp-|SJIEJzPMt!y<9q?zHebZs z^8H!x^sQ)@Phw14f1;1>RQ&Drqv}q@cOUrV0~jA1v@^0<{I*$8E`I^rkMf%54 zUj%3RCSaWZ`p9qp=x3Z0fBgaYeuMtceaFwghu(h=YtkqG@y$KwX)XBjdq02s|M-7d zuYK`9(%L8so!{4(J{`+OMeiL6z@CoGwNe4p#x`^H(dHq|K|=|`m-zEyZGS#1Al)Jw!aVUI{4uyaenC(c=iD1goiNa_Ri+_g=aaKZ@_glg_Ms6*klUoCW^SoQ3-?AN>~iY|dFU{_6Qd&slx= zMrr>5d~>eEe03${e&Q$QtACH@D{qrIg66BQ@%N2S{T_Y4I+%OT;k_T_y$1X+=M~iL zcTrETy?^kQ&wGF&o{)TShVX#4U!y1Tewv?j*5#w<2gB$OhtV$%p??g)H&+fi^eb40 zPU(q{c9`*XoTNsgx_O?Z!yBR8sYCZ!jO@CoM;n%7;V81 zqrUy{K_fh5gg*x?{)`d+ zaU=W_M)-d*!ar?<|Ai6$86*62M)((v@KZ+k*NpJ58{ywH!oOpLzifoRW`yfTdssHY zpESZhVuU|qgn!%!PS zxqjaW|A7(yBP0ALM)*G&;s0WU|J(@wHzWKvM)+UrLF;kP?3}CI01<_*YWmUrmXBEhYZ-l=w?2@o%KW zI|J>r>)4q{LsJ62CDe{-%`pn^WT5l=$0I;(Jr#x2DAJNQv)HiQktJe`iYk zft2{cl=#Cb@gphm6Djf2De?EF#D6R${(+SELQ4EXN_;sbzLpZ-Oo?AfiT~l0_){tI z52nOFloJ1NO8g@!@sFm&Kb8`IIwk(`l=vr7;-5^3e>yq-ztWoj%)obF!1H>G{LJys zq{N?1i9eSTe?BGt*_8O_QsSRaiGLv_{>7B|mr~*{q{LrLiGMjI{*{#YS5x9&ONoCy zCH_)M{2M9p&S3lYw=N}qeMl=$mY;y0$m-;^A`0ZQrXv0lCDn;B>Gk7S$&{_@*z zmJxpWQyJ&Qzxwu@3*X8(7ZH8|;XCpDd4yj__%Oadhwy6%-~M;ltVDR@E8l+eA$&iB z@bw7ajPPlM`w@N#;l~i(fpGctjPnx0(+Dph{35~=2p14OkMIb62mCpN4tXn49qZPJ}NZ{0zcxMfi1uDIe%2odx7`{)a*1UkJW) z2%kmxB6#$>2wy<>9Ps@X!Y?6w0Pw$r@QVmjK0lA}3kaVD{Ldl$Ji?FS`_CZ!9Kj>c ze}(X~2pj4I2bAJ?hvUFs0{`7W|5F93zuawK{ZaD=a_=^O4*+ffzZ!lk_#MFSFn$yG z-HV@#UoU<;@Y{zU{SjX&3)t>D@Qw23HTd=5cP)PF@Ouk>SK)Uxehz-{EAl4p-F)*( z{5IhCR{XBR?``We&y`o#jj9UKlL~L)NuM8S@WNMr+E5s*S6}NoBpD8 z=iz@?xRc)hRN?-QDxSV+p20pJfB$&U{Qa|uu|sd|c~6g%S%3XCS6tb1!%gnHZ@hNh zTQV7^gXfnN;=d-o2`VK{HYkLv90_`M6i_u}_H{BFZ< z3x2!ty92)g{HE}m#jkLh_`M&$o%m((%j5TH{Jx4G^j*%U@ymda3^Ip(lam3%GH7HObgm2< zS_Ta*C)A#aUR^i3eyEN>0{^Hi^t=9s z8=du6UvpK@TdrNV;mWsOab4zZ&isG7<`bX&wfWzzf2(rm{{K1t%Rf4K-O+~^-tvVn z{@*|LzIXqJi}yda{U;87e(*yl{_r2pf9aS1^h0fx|E5CST?mbV|e)7oIe(7lW8}EMQr~l8}uliSi^4UGtyu2uP@%&xEAHeeze)Q+^|I*GWcuKwNbgp`* zHbQ@XJ}$rVKuE9}Uekrb@!zfTkd8oM8aM&0;oqs^^*flufBIeLr{6j9z)zi5<@@ph z0r&CoAS2%xlTQ!jOMeQX#;41r-wB@jitZ!2Z|FYZB98tvI)bA=>Pz&e^CKMer|GBu zX#VN^T=h}E(^LN_N~hEDIzRoc>zUD@-f2#vzX$mL;Hl}-=`@_K2_3JWx_k!|9{pWE zH6H!0^V8oo-HzkRCk=Q?A@o^w>LdOa3iu7iW!C9r&M5_@2-Dv+oe!vJnmg!EBmD== zvGn)%@_5s^QQ|dSx_+Dk@}26L{%Gu@KixiPY^1;INAn>4>3X6$ME+<5VeX!%cg=5ls&sR zisUf;#{q^^K~)b2f;=%Ae8h1Dl=y%sXR%nX6wBUH3%_NrRPdHo=4$=k-0ofNqNE)Y z4>sd>AAbJ~zkk4w^Kc$ObsfOZrE0_P{N2>fi9}s~W+V}z=st)b{kgg!@C=I&U^s#3 zpC5eyAL;J^|IZc6eaV$~4X4m6AUu$#3Fjzy{nY6s!TAI9Bpg@ZN3}%pG!{^utwV_Z zU*`-?)$;X4cQ#*Zw(3Q9Cf}T0aF|0^DMr$q|u0G?OoN;sE5&ldNkH?_iChPG7)m4JvMjY4O?Rf~atugAPO6l(d~?AsDj2=r-PEwrsEaGS znE{uJb)U*ZBctw~ZM&s}R8_Tlb+*`Olq&Ns8ee{?*euqqXzRsB>6743B5#&UjZ(D& zENHbw;tMcU&~VJ}ZaKeNZ8iNS>eDq?EY_xq#YOjczm+y=xj~K0m6{6fWU0AOs&IYp z-q+vPtS%NSI?tI_VZPWzLpMkdwGdsO!Yghd2IW3c-4#KQ2_f_KQo-@@m@$eIy1kZ~ z4R;C6w5*zGP*lJ1wLuebyA4pl^=9tgCR&hFX!LuHX1$nS@|wj)(`(lAl}4#pX*$_j z4NZSkKldaJj@UFdi|99TBH$b-vraZ{rQ698xL`q?#$l}5GRK(#KbylG6T74wUN z5pO*@&C*i7rdq+tdcFpd+(r#m;%+LJTds6vgT`hR1M}8^u%-Np+gqtN-PvlbR4l;p zgQBIllAlGVbrzGRc~ms^-@vhNB(pd*ATI~nXh9IkawzfstEsWtGj@nC%7=vHV6fk8ehFSMtlJ7 zo^z=(RD{nCfs;D%xQ{9b^{HOOq-rMe3e`%pg2~CxoH}H~J6S51EuS$q=M7MP^_GAT z1o;&k;}C;knrh6}OSLAoJFe=2HwzLKHS#5(qJp^%&A|jwpM03)45cOg|E@}s3nbO& zE@8%Bbm>2{!sha&a`YXdnqn*sFI+$F5=H{Bu6^~KcYgI}oT4*bf;#Hn9IFqDOfB?7Z){6jW(2Rr} zd1C2pGzxuw)7WgZsj+cyGB-6mm>V0o&vAPTr3MMEh0PjwfhrcmePgSd7W@6#iN4L{ zmrCVTiJl(mcad>Rpk?lIvlGJ-yT`2*TXo2E`PtcGxrhOhMi`>5w^doJR8LkmJFAfW zDpjY*!gImNFM}OKj-$zP%B96(c@;oS3{=hHETnEtsgD;ks?6Xl_0j~6sdZ-6?dv-Oao|3k@&83kJXiG7)>`Sl~8+`&7&4hE?{WFS_ix)fo5uI zFc5F_0n7uO_!7p=rQvRcVQN*1nR?gVR>Jv)|Y7fnp>r} z#bt;|CrR|Dw|cc!^dL&_^2*hD{zAlfLbX|CF;pkeZ$E0t_ZqE{{Lv);M4Egp3C zZnH*`jIbw){l2b3t7t|`&!I;G@DDo^@7 zhFJq8lCB*kP~%`kOduhM>@aU5LAy*FMj!O-EiwwyApx;u?Z}v|R-l@*bO(v(FiQp8 zHlx}qaX5j?vq5a9SsE!i%GWyH`1y7`+?XlS@&#L(rFkj#&b?m0jm;=V6s!vl#FD0w z1R<(j6-aCs1SouMM?Yp6VNR|BiS0JulmVae)eMq?rB;Ck>w=PK4ymj#6CgkA05m$H z%ZFVfaY6cajik_6#?BLo`W~bc2rl!`KI)3A)<~ql3PvXXd59<211Y#ij?p&N`hZfp*Be#N?^GQwtgoVr6~_ zn>e1{%8QE}oJkUVH-skQNTn8xoC803HPw!jQMhiXi>2*^Di(%2B`przC4CkIsYGIZ zT&iLRU;zSuJJv~<$q1JZL;=h+FkHM{sQYvl~q(3b4FVlJV$OcMGU*jdQezXz0POqWAYp|s3; z+xWt}E52wnR#KG0puW`bSQkP@FVrt=!f_nP!YEsVv?T0^m{dX-pj#p&Q9AKD5cF>y~ol<-F!xOm?+ODTft>FQ!v!i|@#$lxsyS$Z^B;}nI z&Qh+V9&Jx=Es*NU@f2*N(=e6kH(YhbfVUBeFA zF0Z;=te5k(p!G+VTi8dyu1%?q{{S8^E4EgP&j9guh-R3iw+m`n1FeZoe&Wvf!r*P> z&InZ+a_yw~7&@a*cU+yUqwb-B*F$0nd(rUop2p_>w@cMM?QOAj!0*KLOE77!R?4eh|BesuUD>sx z-&l`DMyS!9E)7zI3idsPK1vqwFE&nvn{GD*&^|US3;bEC9UTmMndJBxg%)jUND2`S z$jhud!q91bg<5KI?`8@^uxq3-jH6R3ZKF}N22&Yp4rM{&HI}12MQdPxKvBoF5F}Y+ zA%e;P1s1Z~fm=wnE#=yxXh*6s+pEn(3xRPU3t8!+ z@(H~?*#$vbWKE|^L#~}PA49kFbu#f|aOd)y-D^!_SQ4Txh}P0lI#V^|+L`KO=zzX? z*gVy&Y)YmyIZgL8x53pN^}5g4e&IF-%C_Pp_x3RG8Yzt+ z>6TDk!W6FRTr~J)66>SsfLy)mqX|4Rfh7~&CQTS}L@mrn~a{6&2sGjAx~R*HYpF4i><${CiPipNtNQ zDjI{X#?{0m!E!pL0ZQavGBRJe9YYrGR{;iH?&N_r4tset>l}j$Pa>B0v^v`O3IPR( z>9jziVRsyRP%euOIS%gO>~T3iTjbf*!oQRhYbG<4@8!@810S5qD5(AEh?-F1`e-lE zovcpeTxmh3@!B13OU{Sm{29a1nH)91q+#>BnNj&xd?7`C3QN5_JO@;ix+X$_MD^9G zjS?Ro;4;y~7jV!Z#TaZm#uH~y*t)M#oomt=uUfTQPHHYDOg9ZAl+Nd^r5O_-4qB`v zh}mXzhxvz+begr*D8j_#^{*j&Bx*+hW(pkDVhb{|OZVtBtfeAsU>y=;P1*@bOGTVT zraI|_usCp+o?NiF0CSUQv%z*lo;7_cg>+=BhI5)8 zW{$;$YQ2&~UJS4s+Tw{jC#)0R`S?xXH4zv>ur?yu+RSET#%vTaC-fUZYqm2`B1U9L0{ncFcQ9Zp~V)G(+6B^!QM`J6q1K&ber3MSA9Q@bPjPd2@1Wxx=+Uhrjemr;zZaYC+=NwNYaxZbyqA zL!q?7xqM5L2m@Hyk|?`jFchrzdQi+@{90^j%)-k8Yma#d-7dOXVbR_R8eZYKWJM^=hFt%U*}tWFcv2X{l&Q%*e-U z!iHGjTSRly=2c0!XuOJ|6%-$}&tnx8=P(rFtg136Azf8=B)b~E9kH{-$c;3p_=I`wbHNl&D%38!ViV#bjnKOeei2PAQN09Z8A@7WEd z8hdMD?Tpu|70B#IRWK1f(<+qMn=X}Vu8+q5j0{%M&q00Q_IZb z#3EV_e6Cd%V~ryclP?twqe2D_)pVR`s#VYk;7@n$2;k9B)yi_wpG><*B6ZP8 z5h8hYSBX-vD2my9tC7}@+NHENp{U`V9OyJgiy%+79-YiMMPb%DYkkjl5m@5!93(I@P%zG-A3;nn&@{arf=lfG@Mnz5i>l|z^9q^7#4vbCvNb4!H_1RxcNWC zpjl(_*f4k3LW`Zmq!#~W2I+~(7V_k<%RMx4j1MHhhg}g)wBq1mJl$7ZhVNt{)&iuQ zp{aPzUN}QO+1L-R@~WW#1DZo+G-KiZ4(?USORI8^tKO7*TzONe+vY|76>Yi9)r;g~ zuais^U1a!+@}NM5U8Vl)JbPp z-nsX{M|K>z_0G}hkpb~ucaXhw-+ik;k8m@Wy;-t;DMD1$ZkA1XS=m`=$5}#bz^IU; zJi!KkzGOB+UhG6VEEf963A&uS1L3?N2z?Z+8=LkHWv6q>?P-ka=?T}wtdRzAusnit3Sh(_l%~#JE}IMF zI9Gh~As_h0Jd(>zWJiXNE2nu#Kwbg+ut0-HX0wO8z3kbh(xUQ_UvZRI`)1VHh<9}z z>*mp4YUnhf8g_fdO)K24GHIJyDOku^eW(yV0wgq-pG};MgiAiG64cP8$r7KPREu*E zofZEPIa8V!%;aIX;SNk3>mz68FF`=sCUpC!jL|Y z9XJe!;FIHU78&w)7p8eL40{zkibXBm)KDxi-)v(A8W$A&ivOV=6HQDjLGt{M6y)xcoq391 zS@FbaI-t}iv!}<8e z60Y`R77(EX!=?f{0#y(XtuVV<(DGFQ?Aj)5iK5hT*W;ts1QF?~8iVswS7NRJwLqKk^pX^4oJsFVP}&T`eXYwM$u zgf1nZM3Ir=(4k{Pa6elkB%uKVnE`wtQ3hVO;bvK_T6!Ez z8GSWK&qYVoVjocRzDd5-tipqBX*L=oBcboZImlj-7dYiN#ecVVQ)9EXh-116suZ?S zJ?3RSmz^BB4_<<^gXTbNuqIzGulALzu(KWXHB`z8`WW(D9WZ%-JA0 z{PaTV;C5n&7_kIW6~!RkOc7r70tfX%1*TBAQJ0%BD5$-i^Jf;|2*SnzTwnvgA`u_@ zb}r}|lPvqB(i@P2e95;9o+KlSF(x)9L5A@l-6)6u>>k?79-AIFgdaU^cL5r~Xx6`P zGLFkHNYn%I+yqE4!iM}Oi}q>^l9AQ`i^WLtK=f7%3z0elKj3H}p)MLA4AVh33sCcl z+H`9fUs_dC>aNLzqmQp3M85M@Z8f1%&ClDP5_LFFxd>xrohH}gx>iIdo zs6wxgB(HFN!3?l3WwmWq4OtEPb;4NPE)0K}CQKiXilAfB1T-b7G&Ghdc}|a>YccB2UOp>I#bLA_v-H z^00E6&V^jsvy83BGE^p}G`i{mmlz~uAj%WE&xDi8O+vD{Etk#Zo21J&7{hS1rJTpz z2Xq}ByL-3t2OE5)W!h4&1k2D4Fr`aD78*chNE9MvFs1^A1c``V3yC%?>Ex*;eMEaB z2MppEjXG9BE{1-;(cr=?UM^1_o-zP+8%~GRT_PRQz=^9yrqpF!W_`gfTS-Y&3c59` z4`hL_CtA$x8LW3mk99NpSNS`-wD?-Sv@>JUP`r_N_KeBIv(+9a1Ky0?rY zS+TRdMfOy?!NFw>q4m{2ZE znVnv4kmSxpiY%gvj7HYdr4mu4_p5}YCZSrTC~~SXOi5awc*{yHWwh!Pb9ym4!(^5U zJxz(zmoeO0kfhTZPDI_v{30&@iYrasI=M%}Mum*YU|6(Nt10^d-odHt+~o^0DUf^p z-J`aEOwCyK-Y|FU)PyMKNh``?HE67Tx**;}kfgIXJX>hi@p#kEG>0%+hB0IOqWcv| zr~1+MU{llMkdns7{IO1rq@m1%VN6-iS*Qt*b*!hSYmY(|00-WxIar5JSQ!)BC{B&k zDM$?QOEa4yO> z4_S$@4b0(WCPGk3*x2Y1!UfNeB!Svigf`4Z)h3}5H4<+5N>$vOeLENynYMxxLQj1l zpw=vqsOK9l>?gLP%U%vSHZ+~rvn6!` zjbz>^CvkZsu{KM;*BKRPneqz-2ZV+12%9e25xMR+-!3hm=$0l@Nd@Ui}M_Zjk9rLx1 zZEp9QDMCqI^s^w{8P3t*?h-MafLRA!zHKT@jdfjoTh%iAW0B&+O;2XWrf|XfG_5QR z*kJkTg91S9)ZQ~pchlj1KVLm-q%>Ak@uVHVg;%#GtpNeJRaLfs%!P0Q=s~kZK+(w7 zNBX8`EY9dQLw%1S>=s&lRWGS5{reVaL*1{$t!hg_?!hJPI$snT-0kdZG}~pw8xbd~ z^+jaVN6L0;r1ffbX}_*`BTHHa>boEPwzP$sl5A=yJY?YDlj-(iTC7wQ7IkxUzD%1Y zd~aZ}4ogv}hYOp5y;)p>8jx-j_fc(iKeT-x-!Ir)MH^g#9h9abFj>I5 zUtMu?f|kj1fi0&gw4(*Ve2Xe1>2^!f0Z@$)3|X)8rQ9BH0;5=J<3Jy7z@-buqm`+c zWTH?z!?IsQc!?pj36a(ddvy^vLl=Sa!mMbiaYwlBb;aNDy{Zaa-r2S$?oehp%G^pf zk>Ijz5CZuW-eL-LCnwcraE+`hmx5{mSXGPER$xXM`mCx8e5U>azZ>7SLj9C?mT?aiEs$a}JNQ8} z)pp`d2a-V?oaZDfUZC7g#*Cez4MWGUsv_~WX&XXq+~6_?yF0o;(`~uTTFoJ@3Jf|8 z>^6n6YtKhxktWn-;qIgZ8QrLg7>W*9)R_r=y1Bo_Ss;gOx#9>Sj=Ls(JT5j7g?cMdx_CoNu6?8ZCLSLp;`ki zms|020LwaD`t@$`p4+3Gmvq^^ju9tEGcn?@l8LS~GsH?Q46FM}4rZqZ4vV-uer(!1 zJV1K=;qfseg%(&lPr-66p~JN&ha!fH{tk$e1NXVo!63`XsmVTN6@?`~MpRNqoMh8& zGC`*kdg0r&b#Wlvc2N{S`!tG)B}vshSVZaTUkiPVNf!`Sf{TqZT|3`K>E+jPLQ+|{ zHuzF;Fo?u6i}54xIs}=Ko-3?Yq0JLkH&Hf5fP>UG zub*segCVrv)3p6zXXP;RwIOLcWSYtpUbLVI)4@hoA@fll1AB}<00!e?slq<$U6=+0 zwH~TpYx=IsXvNaDkX;vnD>tDKvUZaZQP&gkq3d!aA!=KcpD;lOzSZox=ZjNn> z-IBn>7jT|DCSl_=UEwKaGQR8tAvr)onuCS|V>xBmNE0PiOtcCXD@Hx>C_`4U=wL}Z zGBdFfB4jyHp&Pg9ETEBJmqZ6BNPf5HD8*BRv6F!WK%-XV9D9r|xIb)TJTDokq#b4? ztHDAnrw*#kAXj6NOoQ5M+mI~jgOp;Bq(xwI;D8B}^gDd+R1`HlZYWc>f=BjJa6&;- z4z)+GK)Yv(Fyo}Kc=Kv5)l`E?B3&9@xM`A1R+@zh4StXfj@dxvAX);bQF6&3C^zGR zLaYKqtn{9fil}V`#OF3;DK?qChfXvd&Q0Rb1&LaUSJcKMq^g+cSA(#rlchr=$AT%q zcFIg(%wa?c+X?GcoHEx zNXw<~V6Y_ef>g|=2tj>&w(a&oXkBi$-?a7@Pb_Cle9DyU%dI62tX#N+OUYnN2uzvt zyhdtI63Q5)krg+s!Tr>c0@ElX#|il8Ae2sBP8i5f+aSZf@X+M=F({LX(P6~VY%!6} zV+suerR5IRkn0lhL?gQ=*v+NA7|PI!GMaBNO|>m2;UhmG!DJUjq?H+JY)!@{V?!7` z1(`^Z-JukTk{LnDbq?sZ&L6_<%5UVXAZ%ah#0-r}CN6c3z>J6~8Y%L`qKicj+CO9& zn_{5dJYY|o)(1mbrjP&eSr24;M~{q*-{VakGL&e+=fT{;V~3;y1Lrzen~;+)Mu3KFJ?r%OPdKe5N$8gK!W)0 zx!VpyDv#rd(>YDeA)3x!EOsxXMToVQED=eEg^Q%`rhK8K(IQGeGS?2)jv+X#@=;q1 zsiljjmaP0aDp@&6H5gAW4L@2;USqkuQo|@8Pt|RqyTF&qf;3!RF(eI{ojsZ%(jlyK zQXnjM&=%U@No7Y8EnTr80g2fxX#0v+@kZUZCC*RKn8l9BEY4%A?J!dfuvA>C!mfkO zHsJDuv;=VTC>HzV<}otsDQ!=LUpQ4-&zmDnoGxnoDse(t8A&uhboHbA$O4up=!MxCxD2^<+qt%?=uS_`s z>a$W3`880g-~?X-8}87}>O-?QWQ_fo@R7|(dZlIPph6-o&LG^U!BBw2Y(GFnP^BQk z-*xb9(e&wCJ`o__a5kYv0-XG)xtm_6nTJk|O!lx4Iwjdlm;sj?Oej>@xR zipE_#Rfpp<^K?GxE*P#6UV0K%{f?407wBaZK*csqEu;MN{4(+S7Fa?P6*b(3O*)yz zx=1B-;*5+Sq1vI{Msh7l=Bv#Gd>DjJrKw_Z(S;3`aY2ym2P-7fHQLV^#z>%2$q3nH zTBXicC~Sj>uL-1fNIekVqKsthLbn#N>$@i+KH!GOfiPuMqD&03BTSNZ zgHIvSSR+j`e#4h`<#|5fy0bsWCS_iV_G1mCY;Vr4-a_Llfx4h zN=Ns2aM#D>z<&cr%t2UV=AsS?%+x)@*fvPEXBpke_(q2ZalC8fJ|jnMR1$^Yritl) zwpFiF7Z@5EL$l@+y>z})+02RplROL3pwcHh6R;vyst6q0p1cSTGaTEl9lAR2rDGeqfu<>NIwHHIbW~wp9i8pt}M(IDTwwFgs~- z%RnKH%E-NznOB6Ky;=Za_a6q^btP(qiGUG=`#MFK2pB<#J8Vr4C2dGdhSY^%2<>#t z8tOCRkgN zw}qKHoTa^McFq}Y-+orqsH|Pr%q#FX80jWHn&60$3;{FbYl|;2C}EOu#1mFb4wdNv z);+8w+v}z*XASlTX>7I1GSCUWXt*fX3{%$nEBRi9>sK~{(ceU1(qFZ&K}&`(^=$8T z$(qJSWSdNO!C=LP;?<((QDxsHWn6xrbQun$)YeUJm3(1tOTl4T?I z)1y0G>M*L3)V49tHBV84h83uO6_v8 zG!Gv$G_um?z1FXEM+JQTXhn>Ig$T)QQM)-L=y(_zR^zWn26NYOGBIKJy#L`nf?EqqNG~tA~C|>P(ehNo` zUzV>XM)JW|E^_TKYs}7S42u@k6k$OJXc_HHH9=D~71u@uf{k6>>AQQD@q~0uYQPiC z6-HJV5-D%gCbr=K}qWNvDbwl~%fLCNkv*$-2@jC8}})d>txu)Y=1sn|JTMQ+(PRx*NEe z8}UX*4S`YX5>f$8!M`oW*pOr77K@7wScfLF6NkOQW1|z|IPG2|b=xxO#lrB2Hvrv^ zk*q!C(R4i7M`j7d96nyHH+}V#F*?dh)c6#kl)~tGks;q4#e*fH91lug*LMUrJv?@w zP0NJoZZyAQ;SE*H=qOh2W-93pabB>3XrGIc4-bZvNF*P$V}y(lG#puYpj%*sl<6#L zbZU5Xko7=mb-Sj-F{NgV6YN-yv?SBy;XW7}b<;|~Fq)n4rUqbN9NI%kNkocJC@DHA zbwaj-nFzndICtR@1Wl>%TVKJ zBx>U>C>>_`$n8flCkw<{E}t>?i{}J66cRAcV}7#tp?FRm;^;J?&*(K|>Zk2Av#!e( zN35$a%EUIbBqTBAfk1B2Cv(i*uA<2N#!^X^4zwinWfm=ig?4|s#LLT8Ed*(i3-7#q z#UVsmL=;+N&94@s?8e}7atwB?>qW~AD)0-81(Ga!<9R`uBWzN_p?H^8^v|xc&l9yr z={p-$#7erQWnK+p5y~ram|D`Bh;&fFVhU@(ExZtH)nq{^8(g$BEWw8gR$>Tl_y2P% z#T~mSb){H>cTzm-^a$}NWyUvF`*1I!%BE`px9BJvdMv`>Bmm^{r9nNsD&b;<*CtE$ z2xkcwG=`UF{Kdh}?rLZ;5`0)%cIaQeW;R`8g}_F{K`RYply24uSZ{?luU#mNv?Si+ zkil}~1WA{TJZN(25+9PF$Wti`7y;4HR>843ixW!H$JKS@VU^w-$o+=q=5k4(Hq|D7 zBd6HPK@~@NhLVMgtqw#^jIGPyIzu+xq(@*a@;xb6o8DYpDbBXY*>k8*;KaME?Ydgh z$d0t1mNpTWiZo<&XKUmmAhS7iKWuMe+|bsX!3H>E?IDMJOzU+VBvW<)?CW2T zgzm|OD!hGjLnkd{8jj9*Vl+_;9c!pm3zn{`H^e?h)}F=N`5?^qfMJ5@d%!h?D6swa zEh__!L!kL>TLFTqEB{AV-5|au zfc*#DlkjoE(cybxM{{fn>puU=V?K3hDU~;iOUf}G?s8(pn6*Lre2jVp7tHh3(raxk zR%D%)Jf)WW^58USk*@%Ev;%Wnw%Y;KN8C3HtGYOfDnm9|uP8f{yo(i8vs5b=`>Jz& z{lpGF#29M2@${V!uko}p45!aIM)Qm#v2oY3C8X7HFo~KFe>eB!h7TQ{p6VC=M3gbH zeEMCfo|qP<$BDznKbe&P&O)#y@l~a?@en}vSt>}q2KEJsjbSwTHk<(5#$^MA&u~8l z&Gf32Q0UoI@eAbiG(~6&A_HUg>BJO4_}=N(k^= zRz(D8x~z;KVb^_wb}{3UuUvW1$IBsb5|b8L({z4@+Nf_kDjS`C$2BJ(NiK%l#wFC` z?#YRuR$W}g41)q(cyyU%2w`YTx>aI0x{rj0Bd zWh33e>dnl_k!K5T>8zoAqb{=SnF6X_S_$XKy4hvUj-5QI#wFw^#Vygok(NB2(cSK` zGEAWyF4s{qC8JFEjW4V5-;V3Jcpd=(bUO{%_Xp*st7xQrQgB^14MaBt)9H5{`oYQr z+AiEvYn~YaTLpwpWMFSJ591t30!<36-uO$1Bn3{gH&=rKCkbt5n0pG6P%#)Dp(jYv5n6bEWtqE-@BtexYK{WD95cPykZgQFf{vieh9AQB;+9)murL{Rx zFexVcX4}EC>rO|#P}S(54lL?`ALW-jhS`sivM5yeS_znEqh^=RLzt)1EtPBDNxwb{ zIM9LH5*Pb*Ws8A^SY^a05rc1ymM}q|r>i!GirS4|yHNs1MwE!NbC!INQ$Ntx5$d!op@}Q^TvUdjY{I2mn8a>(KcsIx@F| zTkA?eF03~uc0l>kqtIf2xENDs$ER>-CT;^%%8I#awF#RQs1>#6@U6kVP4H>CRBxiO z`#_<6Z2$6ZwbAGmniG6HE9L;|YIaOyN%~@rz@3Wr3%5 zc$zQ-qw+LV*1O<_U2YVONBBuZK^`JjC$f}6jN_C!5PYPW1qvmzN6{%vYgS~DWsRuS zTa}_U24iL28gVj@+vm;t(u8qDm@p>DsB3;#{WpX|M?~o7&oIy|&4n8Pe=k+A0aPxB z@Mzq!F;y>;cMl_fTeMjeTYR`gJ_PTAS#e8@6|F?sF;WNC4M}$XbY2`|!^aU}Y*J** z8Woc^<15VE>!qS@!#n8~5MmOiF!L}z+j5vK&?-7RoVk1%M*~7) znvB^q@HAD#MdZZ_#?NYn9HzVCe}NCceh@b^G{Ffzv5FdOK$yb~Dljar(51*wt+XnO z6-a~X{sd7fo0Ec4GjD3Bn=Sp=XN|b-Mne zv|Ma#PmbEQaCZ+D^_YEV8pg%QI1UCl zc7|{$0&qz>to}bZ5P~nZnv4ES^3Y>7lYPwYT_{TT9Kg1N$ z)$?vM>)z#Nx3O>S-W^-rUf|hGXtu3xzQX`J2(W_+EPz?r>h4frvI5*yxKjrpJHly5 zsfIHcy*u}Ab?H}FMRk>CF&q*bH*bX&KyBjkd3QEjY~0xoo-QpJxzA1`_obz6@HaqM z(FNIv0~Z#+)`L8?FBerUqP}L!bdb2WnFadb?I7Nmhj;gy~M;Qn^jJ; z)7#j0UT<>%Up^HYL_)n+`akVz(A~@tF*DTm(}oZ&(5j&|ElaCt%YAfvpdM^sj6<(A zMz|rk^j6OF|BeA-4F+WcX+FFVv@A79m5`9+_%QkN zwHv&;uc6}#YQAO=Ji+K?tM zRQ3Wn=v|nN0UsN~3FR;-)Z6vd7$9SX+rc>2#brD24S1LI>bj}0TF1MK-YzC*cbKCd4JreO>F z`0=)mL;e&0}xy4mgRUD@p3?e-%05tqXI zDZDv=93C?v?@WfAx+goKK!3;;ab&56$rDKLV9W?Y3}FPAQkQCXxS#T~IeG+pDDLiE z;O`u&XbxARdq<2|T>7wU&)yzY4qXhiPMGQ6GdwtbnA57#jd_zhJnVDqaDh*K8yX;Y zA-HD3-MeE)?CYreq3~Cg{8GO6)>+({HxHMb)f)B- z>(z3(mrk*6zI8Lp*8ugi8uccIy&-q^ww?Vu_k4KQ-aWhb?ECPCKD>L^?tT4x=X$7p zPNGY|v<3CU2X-M1k(!1tHmPEJ9k&j$``bR=|1H2YaK6}pEkbhvdNSw-YpWze!z~8k zV#saoG`ZiwKS1yeDW%F>5uQ3|v4d^);tJe$QI0ryi0)J`(%D1Yd|Yod`>@_b?xX?Q z780_R#;;jz%`SMYePmY(3IkDFiMSY3pXU$Kr}{e)pLJ0YQ17%>_@;qH)po{$ivdg@%TpRhlakOX_z z6zkha^imq&qdvqhneu&o$G!F7CJU%%H|99D`0a+A9z7OUo;!}6@^nmj% z+$X{6dA`s@A)-QdNPm5CoAaCS1aa z0b@H>DGG0Be!y(fobS^{Jss+H%@kH)*qDHjL0n;M>TNNQ;QpK8F}PE}{p+LIu|e

dr0Egi(8Mu(ox-WA$#I;bQzp2udxjpnvFhe$XHj9Bwm~_xgf5Ju@^Y`}>`chv z9egPa)_W{qe83W8Q_Z9sV=C_H8B>0I z;S&q~KO(nT;U*k1D#p-)u1{B7s~NwS758+NCP(AhD4t-7_2jKm%JrzY)pU8DjJ$Dw zBwrtjVz!`bp|3t_E__QB7t|bqoSq#SKA!WY4+$qjAK_~X3|8Y83#rYTw}_oAZ2BxK zEfR>JFMa$eQ0V)sq(CN!Dc3{QBuwTr$ubnXbg*62WG){QM@P*!s_dvuY5SlFL*0^j zqv+H~rNnMH{a0TtWr{oI!2u|A_zD*sl*8DKq1`_^v5 z%!}|2AAz_OD|l0;U>;Q|43{NEsB4kJvC^u+vlCR67(yUweX@l@EW@-|v{=%vw1O%S z2Qks$PChgoCHN^1rJ{+6Z`rWp0UE9RT~1Fni{vmlPBDb8rQz8SN{4)n4{`)f>d-*d zmS3nG3Z4W;9EZ30Dn4}R*bvxLt6|PfOG=#+BN+S|(3CA1l19>+Zbfqndaw!+%9kY2 z4GDOKEuzq*xJfV_VCjMYZl(lZ2CpLN8f6%sg|o}A9AtI*NU<^>-0RR-LUrE7HWD3PHcG!z2>GI+zJC2Rf*nT88NW2i(I9HTLU7dTHWtBI~H>BUmax1{A z_0cwORMhubSvRkGXn+}Y3h^<&D1Dqvvm4lGt`r)Z4VuTs;k=J(@t*AD=&=ch#90zo zO%^jl<7E&&GwtACaJjO-U5e5v@*?q+2}UEm+XzCrLo=7jFUVZIx}^Qw=#RSYH#N3x z+vX@v;cDapdT-N8iN59*uoGIGv$-GO5EVrHK6tj6uo*g7+$A=^ZgT4qC2mjw1b$ zH%||aB&UXP@M3IQtSpSKpl$0rgN`-v>EY4gu|pmdi=$x&lQBpods;v>!LcNX$d!YL zSOONThsOqUrh|+`^aeDpUq&nDVTLaTzmoEg`;43LF0~2ze$V_6Js>5MCM0Q#AsD5UEC0oWr2gP3e5zW zr`1bPr8s({(fFh$d=o0(FJhdW%w-3&hw-12Kn44Ho8>FbxnB#Ivaz6B%%O8u*AIrH z_{1%Z1mVwe1~>AN3_FtvAO>_$6fD)D7DRK9wYrQYE<_J3dT<07T0k7jD5y2w5_Ty{ z6;!eB2MPKJJ#OCNV#5&zS49O(0x7Hw3}6}f!X<T zOZPZJu5m;|*M`{HkwnUCtyV?lrN($c2t@*2JwanXZUTb^XdT*HESO>XK__a-{)0yw zR}CGZnnh)$@t|2>-S77G!JQhePglcKA9K7;*QrWP~@~vI(FzF zDvA8{w;jW%8Ib5{jGI0&ddj0F6_*FWV_JoQ+gcb_i*j8Sog9V1HQvzAsKr}k6i1UW zF$G;LHcCj3@B+l%hC-*3{Axyxg4-<4qe->jUR4>@j z;)~_PV#7^YZ6%O2F+M)hw$6N}OOC4WqIb4WH0Z+Y>QL&mZKKI$Cr9q{M#jOLP9{Ae zpt`1L(EX#)Zqo%8U%S#*fiP9}^r{C~)tE1-W&CX~@K?_`{Hp|#%Ln+Zx zRD(c;ZGtKz*q7UZ^=9}md;$XL=e}l6O;pkGS#wm|KCdZXwaQf_t6g=2W^%wUOJIV; zWNJ2$V06=KdvS#iJDSdh7HofL=?FWU){!AR1SGIRkxvKL`6_{g>JASyGNPqdc+e7+ z@Q#+};j~-CO?&p(^teTGYXeLN*y$hje(wZ%HzlKXF?csegrohsYGr>#b((hiKBJKa zsYUi7hT=ApPvS;&rLqn&H3}YW3n2}II)Wc)2KH|f4FR6^5(o|YoN}T}!6X?x;biSF zvLT{Y^L13PSSJhWIjCH5k8c@jB5c~PYKU7C(?a8%5=42=M^&!ToLTjeoT)e_oErWC z(=#9_QzT)NR^5#dzH^{W&9AlHgr*p)?!=mDI9c0vM4;Rkw1!EwC^V*ob`(Y)lS!lP zM}foB9mIcoI|^g$ragsHeh*rLUh*FvpN2ijL~aBNQgXM9TVnjJO?Ai=4zbuBVFS*H zntIz)$H@^>&l;!F^+(gl_vlxJDF!XyO@7W}7X(*-!@Oq&Vi3k*-m2tlZpq!oY!Tap z1qcAXM;p~gLSv8|T0__H5h(=JiC}cFbJxB;xaWX_bqE)}zMgk`Rn9m?flVA7^nzPb zb2B&+&9{-8hjeW?7zpB|tw`p8FVfN=>1BAod}4em?4VNsut|~(gUxq!^^vjlOxs~} zsw)^PbPJg}HZeh_leT1`l?>gzv(v?}OB-QrtHI=!z+-c`wiCJ*Y-{d}@hS$w?a?vw zGn)q+Y^i-sF*~?#i z-G|re%4xG~y2o+(eJq8YRx};(69@WT*v2&)rCD;H12*AMzt0?lZ~bnZYX+p$gULqb zH%X9!gr(7j=z5+URGgzS#8wu9QTwn4_T<`!WYDv$O< z*&&|FZ9a_6*2e{v+ydvW@xd2X=F`K6ydgBuAxwk=;}iE$4=2{Mk62>&99&M&3+6l! zaKxVwk-(iiJQbWp_9`p2*r8-`H=>1qnH5PRPChGbgLq0mZ;LLyzne3Ia*{$m;X9lbqrxIyY@riV) z_(WP%ye)q;vUV&oqHVbw(YDNu=snb2XJ$+AYKun|51G+mr3mE?PIaM`i;_aieQHiJ zRQPv5Ay>w+sd?CFVJ`v($?9mMv@TP-fUOwx*eSaHjh&Z|OB8B=gFXL1y`AdqKnQl);>0;6u!;uACjtFs;i-4?fG zl?zlRom7|-iZxckTDKX)Cwgji3fb+IZ5}b~g-!s*YG9!_I4NP@sk@ByEQ`gk4cmLLjD=sC^yRnMPv;bDkRJ6h){rDl|+pEPc& z0r(?^HwlHB5PHDmF&c?8GojTX4hiwOt00q@?-jzL!Ej)+P?{|^Of+=N5)}n|csGlV zhoyXgj4TXsz!_JXm3iwLOHG>;;Ez&3K-Yd~nWRAH^aA{{B=ukz!%~4co7#aw_igJH zpD`#-LLo5TkDJaXakZkyTS{%oQ}A?&tiB$WxMO5a8(Z+*Lbp@#1#7W5URC2Oj0bE< z)yXysmoSif@MLOGuXYu3gU1F?j4@47P=e{)6qRat3XyHnO%0%21BYSU;8!G!_~af( z9Avg_`p1dUtjrC-d=axvz@dO1T?6sdQWUBHPY_ucU@Lt46e=I34xnXzXb|^pqP069 z88LjZQ1^rr`(P;pV>0sS72Y07z@UAj(q(h{L=o=-gfzdXer-&cfXAoArtFEcm1i*c zC&j+xJ*rJx*yuwI4^AmdtXTGS`@{W8<1;Bpe514 zLWQatpBN$;%Y9MN>0G({$Fr7G(V~XH*oLFM_w-z-+nlokD#3kddwKYbHJ`U~tiq zUl|4*b53pRYSC0CJ?t`~KlFh!aBT;-NEqa$|3FO*Q>1bUPETMb(n|_P(&Q5hXFO8L z(3>_rLpZFEF_DM+W!Jqb8CBbkl~!a$Ro_8hbVw+%E~}c3S^LD7U6`dOq0d{|LpGx=`@NhNU(X#C)McLcy?frCP*J!;1VDs!yXt zI5>n`h{}9hBBnfb{W+BtC7}m38)*WuF$lF68-}=R^W7VeI9a`5w0NUyOij_|si-Ra zR?TzFSQiOoQ9@p4j6_Di>MWO_e{}|*mVrT4LQ6$HeR}yAuvJVM#@be_-(u;4%a)4; z1n$P93Nq@{=($>TDK)qzYG7iDL1GDKw{d4@p0@fmexcW4T*l9P-@{a;vG3N3Al3 zDO3~BwBL)W)~jEtb3?lUzXh~S!lMh84l6Xl^Q$p1c1AM^9%~{rkuu-h0S6G-w>_xD zu$@eATndnz;9=WA&O@fNhs>R6-Sazv(VA1N`G)Xg&1sOE4buc{)rO9dUluD@pcGx; za~r(u%)=;m-j#XC;<29D8rDpLgRuW*!LjMAn<9pbWYrANzQ+MnX5o#(*V!s4Et zyS&0O{>S>ga;w53GzXc!N?1&H)TkbX`o1;&ib;E6I}60866S!$f+F%FeK9VlUUPNPg01yNRf6{R4TgA2z(Q7&inx;(DYnfMv3?XygU>O>ZQ`yxr07d8oSgJ zUPJMEk{$_6q*f^n3{A2NXtVX$ITB!AJ68=HR^}Sj;8|?88pr!|h=mp3rnPfb-zk)|w>UsZ9hkh$mGh0UL9*p36N1pEVg_n zElB0T3#2(XDA6nWnWh{&XQWNQ=NJSM{O`mm=k6ZKMoap-w?ke{p729anX^&$pov>;JUW(m8RLJ9e7jAIh7`jbO_OQY0)?M{q1! zC<(T-(JU+efe%-ojaVwy$yE?JHQj_vZxeHWG)PXaw}|sjY;xK0TZf(}?Sd)M1rAoZ-+}p_=7J0W1#5Z#R5GgJO4~ z)TFbDJe(wR2=ex%UM{zR8iKvU;V3tK0|8ByPm-cUwXvYQcK-V%z0E~gtz}%^*<-5~ ze3ZQXYQdLTTxPLUfjdVzl%S4a5IfYhO>i85#Tfn(n|8#AoW-5fIEbJ)?U%`?lI!DS zy-KGQEY$(a3BN}4u?TKRs$K$ub?rbEI^rdrzs9~T4C=E7;esoe8Iphx4^Ml?vm?WU z+3E4g4&Z6BL~o%y+CgijFR#&pP8La~=kNI8n{Lw?2Ut|mndhP!363VInhjaG8-@l- z#^(Bkz6*VANuOjPTQ#~Oh&I?|+dTr%sZ7A#^paz3pPF(EhK9sgQh%6->vRlIIPirYs4d!1bXbDu!AF>A#iDGDyK@?1 z#bnz1wLjHkP3{-RP;Z$i+36Ds6m$rn!RFr-brbS?lgqO23O=c#+P7Xc@|+T7U<6Zu zLAU&&W6&~IHlcRn%IwjRKGrta-A3AP| z`O0W>ngm6u&Q^xaK)b9_$OZ?lfLRK$b{T%>$T%RngQu~p0*>B0G|>-tSUG*510EH3 zI@ucCJ2tx0pbC53Y9MZZ;>+f+E2s=$k%M94dMbP-EC4h1^Rx`7GCw|?yJyNEOF^pR zi@wCy9y;jJ+>_%IOhAcdR~JcQ(=nL^(b?qG8M0$QMX`Otj+3mw!}5SJiLyJIl$lg* zjx}654C_yprbIb`5it-4VEH0*oGBHtgB2L6@%bp+aI%7d5dMB)vZ>`OnZ@<=lprgT zysJ=!<*)#rZiMr(b^=c)O*IuAz}JhU(oav3jM81e9WVZhfwCtFG$wz>#A<^$!!mKd zl5faVIkb*2z{9*EUk7AdOJc7~1${JKnFK4rU3}B|3lzBf&pumwv`?x6wgC+nriB16`si3#R#xB5b;LahnpV~0(F*E6b- zb|ir(jT^@B6Vy2=XfY({@kQ(?+F;Ofx=JTWzz|w6v4Vj&asPU1ZC*0&f`N#MfVuwm*IGXP8)~n)K7_f zlW=4gi)eDmMLI+6hR;vu(p7P!1i{c1Oq>2Z4=+_%QH~899*_7tCQ(+M$x#xFAsiOS zCOHOS;18x_Et6YTmE@qc`!vI2AvuaIp;&?~L4?Jcj?ZZX2d)ytS(bL8+f4&BkS1g+ zf-`jE2{~yaEqk;QoNdXmFNbShi@SD2PAzhYgGfupL1g$eN+f0Y5Ktu2vJeQ+M$i61 zlno+x?ufI(h;55_m!&kmkR0P++sa9<7cBKf>W(4A`adBis3-sMQoNL6>uXyiNf~T0 zei>|$$%``REE1M$QaUR(E}bpLwtVZX3tN0z1S6$VPKRYj*Y3~;nr5W}j5xOm+z-L6 ziB^_~gn}_rO+GXdH+{1?a?)m+-!il(2V}^!gWQ&QeB{{p=z9|i+v28`Cg%UcI&lgo zlnU&h#=nd;U%h!6sRY7L7k!aXkJ5g8y=@x zGw_py-3iQmbOkr$uBaocMkQZ^%L>e(3GJ%0qWHd^;026aPHbvzM+1_!q6*Z2zkc7@ zx)iQg+NVB}6*2GbGUvdG$#0^(gwWB@{6Y(n+ zrJZ8NB1rT~NaJg0S)ljGcru)y66=zpeW^|@2t;y)k*nC$YXM5_N%?i67NgXqrHj+H z0loP?z+i4bI%E1~Fot|XhMm9#l2J@MpamGUNDrKplI+QY6CWN6WLr-+J!X~xp_q^u z?{gspp#{)j9Mt%+CW6WpBb}i|F0{zaB3+K41{*b+va}}Cv>|elUTJjH8y@tIP0_ZT zT;OHuZ?v)^3DOvK4iBgu8*~$ByY#^Y9$2(A>z-V|WvgPCM{ZX_Qf(9gH+&`A0s#(B z@V!KVZwuCaw?$znazg`L+eDozw7Kk1x%TbZwwrd5I#9)nC~ee$&)?nNu@R^wJ>bJt zg<^Y=4#!s-LM7Pm$bE#k3+uSt@UGvnil{H?*dy|$GP~BL60w$$w2%rn35ga+L z1)UJQzcEZIdgg#i5JadCG&qE4g4+SUgx*GmT5@oF1o70#5-zN>5<@+wk*Ape`@no3 zQIx6x0%NB@DDrr(wjN@W@hO>79)*ax-h2^T&Jkh_6q@4b<&UHw#z9rOebO1kMKV*Q z5>rG`y^_|aQs6cayH#>9cktLDWnt&w`T_joGiDJqerr&yj*;NkdXxa9Or%?m6V!cH zN7Jf!pV3M2W+RAEMoje>A}RTy99o0py!mG4Ga2XA8yx%w&)nj?_;&Y&Tb$S5KJ7Tq zZCv;G`<&-D-sCthbMTdow>!?m@1U>Gyko=JTbu{qIr#i7&bgcD%lVtuJ$s9Dk;5w>W3teLC~pEzYU;?s(=F=h2(pr*Cmyx%pfM@lV`xDkF6KY{q%; z3J1U2U%bwF@`|mF^Wqf`UxDz&buR(r>{T~CeVy~dRX0C&o%8BdH=n=GdFbkz<6OG> zCIlZ@ztM4?Sib?mGwb&`&MWKb>#1w-_0enS>*Lqp>r2xa#^x@p$y=>mR}6iL2K; z&eK<~e;DCsufF~vJRZ8{Oy)s6U+p{(W2RA%-mE$~rhrcnaUnzGLGXczgaG52H{oZC;?HukKj)D!#q6>!w%mcx~5BFXQp}?hUWu@$~Mq z*P%YM>tCXm!E;x>P9TTM7w|Yeyh3k}e)N&6!OWwzC+KDJ;cLN{kM}-{m*+n2g0&KU z?djtB^LV^6x9)K~PA@!^dF^jNeCaG| z^ui}DW+?Jp`5Ex=waU%(@_3cgJy~6M8gEb5Zh8!l^R*3x^Ys(G&*SZM9n5>8&e=R& z?|lxRUaE7tfv6Zo?~ChyOy_czr0lb_A3e+ckrKASlMCSCk-0ekt!Gw6tK zsJBy}qqoOCCvm4gm$@Fz>2pHlGyL|fdV8MVE~vMcKbP46R;#x+@TBT^^r6g!4DsO1 zPh{3pt-bV<_&{uU=AUO?x)PoK^OKxf=^uJ3iu;U|HxU1f0~|%e@?w$#FJ{~jkB3)BH-e0 zOZ=(d$=pltZ~TtLKlZ!w{mI|W6bSyYbD0e<;d%aC=J0tuAN{wPbuZ%i#FsN4Bm5V? zl3DjWo)4bS^iuqp^O<$e;d$8;G1szn58e1|J^$cbS`>#{21im*F0M@f(>(uY4L2k3OGymLV_vhs?Ss z@!^#}$lUYf?f;V0=U&WQx{Bb>{%L02<3RiDh0IM);{Cz}0eIp+XC^5A(tpmZJB|1U|15LU zV|bqaGl_rk&oZa3;0*sfGe}un{PWCudVlmUGV31&;F-V3Ji-9a|D|dwf0=nALlBSs zm4d*N-k<%e%sxu-&?}ic>HXDLGPgg1_ZMExtb6Iz%v)Z3m0B+TK9%_tKQU+D;9S6z zec^_iUwFH7=>}v^fB60eD8^8H=`Al@`QldRxoaM!zSr{*y*z*2Lswkb;+(p%_taMB z$s0!;=k*)c;rpc}k$u0E#=`BwS53m&FWtX`KFa)TN`&^nqLN9GxB!fV*$q`u$tSwjS3; zQYsmIg^yNo&G70ZdG*?#kVdvZoQe}eDtl4m;S5 zR{nr&8j60@?eTmWcg{KFT%=MM!m2D!$?Eehx&TXqQQhGq{-=gFkjGvoj&|>NpiFEc zf2z71lqonT?950b62?HuNO1~H=#3B^tKcHtJY79ASyWdhHyna0HR(aEejtq9`jtCF z3D-2j?s^(7BK9M31=`srA;Xy%`fmug&6;xiXy!S zp~4QK{C{vYhx>cU(HfcpWy)P~f0cRN*w2ti(5mC|=ccNqVqpqs99)o1mobgzD>#n~ zyMPkgM)A!9ns7eHz-D%h{pwJ|M6o^{eBO^_s*rN#Ar%a~aKtm$6t17mmvdwgupgEL zEfh>5?!nEksC7$BL3Ls@3$9yg{s{Zc$>9;k33p|)6OmO$jxWE?HYadEc)x?IX0y_N z9Go#|N+We}r3kLIblCYBcvaBj8X1paL@_upic7BQtBNj)s^ac!iEo5?<=_AaKo6qL zmWp+%E6^=BEa-ZwthLg~#OUYy{ysjB$M=sj?VNp{v-WHM zShHvEJ#%IzJVHPgFrFpzfoAqJSi+J044n~*Wv@W(b_rLRH=3A@N6}%hFvM;fg>V#< ztfkRvE=Pn$cS;bt5bk5CbKKQ8n%}q#A%%BxY=904?(12eQF>NE(Ryf&P%!h@qCb5G z$J?%su^1Hi_t0PqH8R;?t<_wJ1lbgaMmE9*OAl-X(=F!E0i6c7VeU9^WRf=Iyostg z7b9CD@mDrxi0?2GHZ+QEe&fbwjRJcNdq`;tW@O$rYOcTrOZ%9Muj-gPIxXnsmgPdn zYsTi0mTo{cx15xp8_Lbd+tu|$8JA_O4d$8F<#2ezj0f5#Y+gZX^E-KeF5H7;?rgA* z+A^wLppiOdOyyLZv6yMBgnO}y3gEqQ@RGRY8JR@|0MVn#BFcO=zeWUsD*ZXQ## zZWt}5-Z7YE9z14T>Qw#&7ZMAxjh$4y9D46u7-vt-CesO}!ORjIEY8Z!!mbU{cj%}r zel|N{&W4|(bP$4;R z#;5dUQOZ3KrCf>8%H0E3{FWputuYxc6&<78rN=5)!?DT@E!@>TSE-rvmAi4iQnkyK zulY>nt~*P4JO8Xy;<-wjmaaTQ8OoKk7EU}js7Wh}l&hjhsg8@3Ye`sXwU?;Lu~#T9 z5h{~dt-KvKC~eJ+$`xM&>D{KZyxWzl^G@X*z8fxmZBwpo_dve)D_7n9P`(G0KJ7u! zKB!!)A*^~(xnmzzTKvPxwG_gY5QZVFfUpU|76=C*OxUhmIop-H?-8X9Jfd6)k1B21 zqe^dpu<23dn)#Sg^~Y%L+7!)OIZtyX%-6K~1)7?9vgYYpuIaTaG+#=F=AOM))9co1 zam_(Zi_6yBOV(*BWxb}YSq~@a>oq+lPt(%!H1BXeTz?2@t~Djvl(fy7Z`((jzVu_w zz2jp|@Bdiy4t}C(>Ql{A_L=6b`&`o+KG)pIUufQWU%>^6uQk`~otm2Wjpmy6EnI;b z)V#Id!^MCfG=1O)xQ_Fq=4$*|bNhbLT%lhyZ?mTB-I}fqYI;ubx-;<-BrDhuGQ@WZDv2+RkOdYZQEaWtxnY4^#|#y z;$S_d>=50vWHD$?)K%U}T^m@btHHB%Z~Gcu^R}H1Q z9SXTPQ!F>mWy)xUJr(gXhJ(bde)bS>#K zD9>lQ%K1`vhx;MkSGtNF){U^sU-CPQ_j$7}2$898byYPc^auEVP*{?i*3|CQGj`pT<9KY5L9!(LUZd>Z^SdfYzE z@c2|*luyfv@)_Hv_*}7j_>79FzNnfRJ|lIOPc_c+OD3cc;5PoIAF@Loecu)B5O;UOp#P5K00>p=5SV0_f(at|zX{FP( z*qD9QO4x!|e_Im1{|%Rr851_>;qe{X?b>bn^{#5;2KSAgo4hr?oBg*$-5OmR<6b(> zebRXM$rIeC#JZPFbT6OeUNPByDijcxKguU3_E$5XYM7*$OWyc_|7A7`E-|Y-Shr23 zG&|wfOLNKlUl*Mdhy)Q2L_84jK*R$P4@5i=@j%1_5f4N>5b;370}&5IJP`3f!~+oz zL_84jK*R$P4@5i=@j%1_5f4N>5b;370}&5IJP`3f!~+ozL_84j!2b;o#D?if)Y!0Y zCLbHdd#>TGt1#WV6&r>JbWNt)p<=^yt5s|m@5RP=bcc6rIEoo=$2Paea0s`9n+z{< zFhlcBZqU(vU9n+!GT7XX_oZWca9_C@((PZd;VH~}a0vIK!4ey$r;KC6Q(Du z4wE=MoWmnHJd#7WogJ3g@KGEd&EYW|rf_&Hhj6PqEV1F^IGo4f@f^Y}@UX;&7jT%$ z;X)1;fWmniNmEFp2XqF9G=1<-cN`5E$46rho^Fg_X?u@G!9Saa21EE zImGwAV7xOqJd49L4&hdQSYpFxb9fGiYdHK1hv#w_;4q!T3=Y?Fn8{%lhd~arIb6qK z4u`oMp2y*O4)I<>EY}7zObVL;e9(SZ>Y?NDh>2A%oq$J-Y=!3)IAYl^KGR;#A;nLi z2RXp9{%6u($>D<>zQ*A*Kau@24kz>W1ss0KVbn0iJD9_z9QJYO{e|oYa(DuV3)j;2 zG!B!_AbucybXYbA>ldqTHbW>sFHi9Hw>TWI*~cZJEX)c5!8*9Iw?bKh5wKwi2{TJCz{oz>s z=L1xJr~I?N`D6PByRo1BE$N-|@7;|(|GPid53!xbiLv$DFi3hD7slEL*-qooSo>eT zCq0cnW9@J6#=iW&q^EIbtiFfsPU)xp@W=Ugu-z&B%puYzIJ8gqk7TFuZ*2MQVmpnC zW9WH?$my_{!IEA4&|FQOm-)I3)^Y@ADjP@Ur6uN{@&P)J?&T0JNb7f z+iBi1Hve;fBfXP7Ps9E&LD4*Ato}!~(>!Oa{d}GDG>;i;pX?&L)BfGgcAB@0)!$~2 z-pT*RxXJFc|9aRi z`QKQ3Nfg;-8kIVlX+i4y>HvJtF|5(3z64{;F*QCj0cWR%{u>D{M|CPm& z-f92OoI-Xd{pW0V(%-PhAD3_bo@96O|5t2x%Kzc`KQ8~{L{(q)PA2}yHopGyf5kL{A+Cc`GM_+IqcuI1k%%a*jRmd2HBm)kEs2~ z?lgbe%66yz+p|CEoyM1&+3w`u`H7@=+JCRG-6{W*4i-_>k)^8Y=%u{X2bY5xDlZu0+PH|cAyQ2m^akA2zh)IX%K-KqVb&332x)n>NS z^{cV_=Q_4KwXb@%JGI}xvE6BY`a0X4>h~$zo%+9@*zUA{C%#Jlb=rUXv)w8Gd2Dye zei-Gbo$B`!+nxL$)lU9*YF`O# zcj{jcXS-AWC$ils|1;U{wEqfslm1q=>kj?D=XI)|lmC-@ab!tQPWmU_{o~{RBepw@ zPw&1*dME$=<^4ZS|5>&>`S&fhJIx=yWxG@V^5935zmxqXwmbEY|6#jR`M%wa{>hIi zf2Z^x|BCEBhyD93+nx6B=C4WbG=9Cwb|?LF@S+}UA5Qv7-;mvD{?POz*`1Exb>?;I zgpv0I*ylHMej~lp@o^H{oyPC86kUg&<>0@QCjPNK#CE6rkDf&OvmDaDn(a>KFU@RE zaL~KrD1E2=_hGwJ|GSRuhd88v{ch4voI?4Z?4ZAED%p>4u)ocAr|Sph38YVO(BH#$ zr{lYIH~N*cD1E2)v3&L)+Y8w4bba-j-RK|LjsBEFDSxN&FY$1)OaI!^h8(N99TBVSAy& z_OQeDD%*A$m+H*z+?XOk!~+ozL_84jK*R$P4@5i=@j%1_5f4N>5b;370}&5IJP`3f z!~+ozL_84jK*R$P4@5i=@j%1_5f4N>5b?l>2e!fw^LS6a+*dD#a`SoQLc!gJ2@u18 zHJ$G#*?%ehE`8f+7^0OPSxpE)gY7P`hd+^?>`CU| zh(kTzABg3MI*IJlIiI=9iT}a6iEQ7_ypZiToAHwr{(W{#PyBK{mxufktExKben*I6 zE=yZ5{R9H|>@Yc1Ir;a7|Al|?-u^4|)IH&-?|K`=^sB{XMuG zFZ2Fe%<(&!uVedhyuFCocM9p7P2RP?Qy_0#9^`yzKgFu_w?^;p`sY!ffKkn*>IG!y z{r%AYVt?Z?QOW)~itFFc{=`h7iK;5L&tkj1Jyf%O^!^xqykI_d|DDGETh9La6Z6G3 zf609ra(qxdW7SOC@lo^l(f(4cU0Eee=cshqU9A3j=4TraJ_-_t-IE?0Yn!w>44##u- z<}znj z<1m)@)A6kTGuPu#-k!$c1`coH@E;rwa_HjrG>OC697;be{jsz=X^#PpSH|JB9Nxv@ z6C6r={(^a!!wD;>ogK_!GKVWTOyh7Phg&(kox{Iz*v{cv_QU(k-*Pz2q5o9MZwiM8 zakzxTQ#t%IhlLzo$YB+S&v5uUhuTU?_W%x$UD7;Vm3CaQG~T_wxShWd5AP zpEw+6_BXNWAP(np_=M=#uXl2}q@Sh!HEfQdI3{C1kpL@>sWGrxx8WF@WV3AqhFD<< z3r$W4n*n~9<7=#{X(K-rnTFV~Jzl)oPsgg*SN=Gj&+MOKmE5=Up@DzE4y%BDy7|ZKER5Tqd8Bb3hX1F_|LyqqJ@|m!s{Majq*3uA`u}hK zD>l8E6dvVN0Q+{F6XIguj>jP`v|WWwfR9-D5(y(7h???U(o$eO7uFE0SHOA;toOpY1J+hpKZ11t*6F&^PKI?8taY$9!ul+%Z@~IK zto^Y51Z#{-X?wvs6V_x{7r}Zutm&}k!+IgCSHpTctWB`~8`dvi{R!461GKQthjkgO z=fZj(tQD}{25_Q86fTWM#)8iw^=SYLv*7gptgd|_P;Yay%`!deCE1F$y3+5u}1 ztOKwbUZ_8;iLfTadLpc6!@3F9O0UvuVeN!9)~8(aVLcz#r(uotD_6Q-8I7=Z!wP?A z)O|Lr6|g=4Yd5U1(aH_a3A-t*9%U%+263tXX5`I~d|vjV3Fi%T|Ov_NLg`r^>~KvqT(7;r~C`M%a*1yuJuBGrtEnWEO70WRQw6FN8pLMsD8dE$d1$ip;c%Ls^hXAzDKPc?Fw; zkX=S7SehHcupn0j)}9|MDo_Ea3KSbcEPP;HFciwgLIV~@dNr3!2;E+~RBK^ElF;>?_2R$xnRFb{Sh)M?Y&5{OltFMGJSI5Q(3 zw}gsJ*F(rn1HR>9V}KC;n;|r84oh}@!EalNGedc6x8!GFOE&jxZhkg2gOcLBU=Un) z>Y|eu2No||xN21(5f0)LbMu1BGxDK9K)N>D@{&--+B`VEp$V=VW0lRzS1(&^x^dN( z;!tn{_Z3Tmn{qRu1f!C~s0AAeAL%9X{(9D0&4t`#lTNupC%?~aK7H1aa7KX6L zutzfyug89(t7G={ zs_V2pYiy;8(MQnO3a|~D8JSzKHq6iyZOF|xJ7?2gl3$#W9bBB3k-Nc?<>;gXfyF0E z!=aW8@r#R34?=egZDnjjkjUH;s3gG+g|LG&EiGsH39A?GI=W6>dmeabOuLcIfl%J2 zz}kX>P;sazgWeYa^C&H#US9&Cq57_!1D{RT%~47}p%nH?Xf+;?%D*gQOR#99U8@O( zYJsRqKP_2JSsl!Wjz=XMstSaL>KP+nYLl-pRE@}2v%Fa8QjI06l9$8I%u{o%l^fZ* zgw*a7 zx1zfGHc%Tlz~o~K1_zZub5oCNxf`IH3aIzB{9tJ)Kms zZ;)2a6MH~fLoZE$&*ts$#}f6T4KR2V6m1A>4(6`Q2?Zd1bku7)fBC1b^dn{Ki+Vu) zOE*;O>oJhPwt*?|S@~TYe764!{&J%$2Zpg?iF}=FLvAVb?Q6~H!YEUXt0-gh?-_1) z!LGR;lP&)mlWqEOGJIBiHyKKr95^LVoCp1eze4FHFl|96;%hQhCah2sT%4XBUC7gpA0G$n;tS! z0t)89I?S#D8`Z_eMs<@L%Jn7ICww*5r?&yzq!(r6!!F};J?G{xFGAe*`!R0c7w~6P zN5}d^VtwgWE}RL1W68av9Q%|T9ZQ?kYi^wSGBeqdu@Z2)%-)+B&Uv<*2dZK?HUp|1l56`7`U4k?lKCZtYKO02gZkB9 zQUtv;)C8tT85`6!aQasBgc}R~9h}DM8Joa;W>5KeR7o+6cF#bBirW1!!p#_MM-4{X za8w|purMzeD9(k04aUmuD9AW4W;6UjR+rg+pij!D{gVyJ2GsLW!9tvZCkJ5PV^;+u z2edQwP83dl-~=b2`mkut*cEMqb1v9VUyO&(`m4}GZP(*xWfyi)!*5K060P6F$rq+T z-C*1k$9x@KRIquZzZr~%B>n|2*KudUbP6P!18^z@vyJmd1l`fukOrg&Gg|0IcNsp4 zrbD18urXt#5A2KHh*?VqseXzM6%?*F&%>ky_rxf@Frx@J7OgMX6aar^o71XbVB<*V z^iK4qWH>5flq;yJqjQU(nSop2pn#cS5NDBaB2a)sqNLgpU94^%2Z!m)a6+pp?!(WT z$KeHRdP*QkBKmc;oIBTc(-MEcv>v%(T{tKnGcy6q^pSLCO^muCmE#s{T>=?gM zy#!kV>aOvyzdnYv%0I=UsQP1{7RV^xlAo!<6QHrIg##v3upZ{&-;akQ2|T|64>5Ha zjJ|zcxL0e!*zq;ojOraXfEp^h4qLK%7UHy3<54*xYn?!irAWO4sb>`D;Ld_W3%W~7 zPu(;DrhWm{4N;(vfPHCZEQbd*5u8N^b~X7&C*-N`Aiu%p8BmV;pK<8J!e^oT*FxIr zrPxv!HM0VRFz4jX02?nKgYB`2g1pU>Po4neWU8GLM<0IQL!vbg;l8MC*b_dRZo+z)zJ*;#qyDRt z_)Etma1}&@u?%;DdS_Az%p=(N4H)YmiH9`0n(;II8eH{q@#w&DuAa@K)H|RUx)l}MF5jgX9;LV>4$4x2 zafW}!LQ^_*an@Atjf2`2!np@sxqzcnRm6o949GIHe?3`7#MU?%Tp*DkHt0eaF5mA zh&Avs#@g0~pS^El+(1G8NKf|PDYkYtJOyS9_Ca!BirFO9C40c`d=p#m@T=JJ$v-W7 zSpD-7#Hs!}T4C%R>5x?W9-FB0BmE#3win_zeuJgk@!6P~RPV{LZrKz10kFf=#yoCD z)%R(M)v~A3p}$OmJ-%niER=c)gl+#eiv!asm@%ynsGdDH(Oi`daoIP}MR0E4fv6o4 z1HZjD6+Y{pg1`5yXPI4Y0FIEn4CvrThUbUlS+*k{c1JfhE}qFf9dB{<2Vfm|3maD} zHuTZE>=#Mn(y5S9U_${6fS<${aR(Amm&0b;%%wgaPUY3zQ{f)xeBg0>o^Lk52oTk9|)Q!L3F|uSQ}iIo4;!^gQvKnu7GQ~ZE$?R9)bPF z9;X_Qnnb>|7wv=JF}3fd{JYLU)IgkhI)k0GdAb)+jZ@4sj$M-}bE>P}nId&jcki)Y zeX+;J_3Fnx;cDa4Q{f8L)q6o7Q4$cW!K=cbstXb%6-7YucHx1D|PKYl!keJWf{NfA=-{dacW~) zVAdT_5ATERbC(Y+(f@&|)?&uPP54>A9dj_HoNxJb{&K~1sC{n!`kaEIk)i9?edyE< zI#{^;98gu$VPfzwCNyMz*4>Mh&dex;3n#$<^jag6ir1%8`8n~oB$s!mD+3xGj+W|+ zY1lESx4Ek7{^_H0`v7vQ{uL*4J3hu5kLjbH*_V@leqT5W0_vK5WzKKbP`$h_IQ%-C zCDxg9#eutEipRs~=%MzK1RhgI8hpa&PIGu)DCyM+P@on}gZjoN6Ug`r@Oma!DTb3Tf*}rPOWkV#2T5kS`6VC zrXSFzuA2cp%Vn{UZ1Zb4Vd{dZ81)A{-yb#S{dfka{W-{b-bKNV`#|6-%*ch}?ws-a z(;*gB@5f(m*$)p1+#%Rd>-IbH2r2WMU>SS_50<`1@nG3i!F~~c3sHI>8V5!=W0r<6 zw4Yf5PXC(yL0=BPov2p$gt-=+0p!66L2$%3kM7S=n;`18UvP(PC@c)9XZJTJ^}$jc zb~(y_AWGF|m|FS0ShA*f@H~DSoXul{+I0$0naG0c62Y5yqoDd0vk)IZ`d%5{Ki(gY zF1+5t(Zh+mD#DvCOI%ME8&plAwPKGXni=w7!jYepeE$adR+>3hzK=z$qVHS51qSDi zl^pAh`w!skIK}M;{7yEHAHddU4p1(<9HMU6-yAjno@h?x)$<2HdsAKeZB$hU!g&*% zj}{bcfFDiFQO`V2lN0M=n1I9dR{dv&If+#D2g3XlPbu{b_2_}-&VtrQSBL|}>vId$ zM-ZdyKiF+_Z^gNMqj^@{{E#`XhwspDuEk6HRR`e@Sn!+8jR(Qx5;oKx1g#l+1l0Et~jU&uKz;I$j;7(bB%*w2Y(R_iFf=JkB9mjqM$%9V2>_3lqxf$s91e< zP;p6NA>>K^RNp~H=6DAR9uv(=^~LJ5gWx0$2FxEJgYGBsTz8-kMo~B)yLTr1kc7YA z6cla2b9$^n+sp!-C0LUP*iI+jn2C=1+d)ODb0!=v?bycZze4w&0>1*mf%u0<;jIUc zst^sG>w@`wm0I0-Fm&~U*bBgU<9!Fi=(_tfcY>z+TdZ0Slmn(g*W<~&qwnsYg+KPe zhznN;ODTW}K3-Vj3%A>6!Rf+1*l)MhQNO+GF#B#6x5FQ2!N38hd~mfKMwiV6MeFf& z3V)e0e@MG~mU$IFpdO!92tN}B;2=}KLAt~DW4eRzIr{K@?GR4)pNGKF{TxQBh0oEE z1|-sd4}qx_T+W7u^RGkr>ZKY6WyP1MtiKgTDb;f4Uhvsjw-@Z~QFG76*_J8Z6SK`S z%N3x4P4E*Fv@+E^8;*)^ap$&NgXs+41nJ0?(O+ltwJk1m`JqtgZk%xPEb@v&*>Vq9 zYM}dqi8o$jg2~09InW-n0_*T95B!R1&V4!ZbB98;_<8_MqXIb4jLdh2W*4g$Ar4PA zUziQOem`p9+`w{u?19-h%d9#K8+pwkBiAr)m|dXi4uj@lHkEd`?gI^Z)CC;%#$kp0 zI}`kzT#Tna1?%A&Qzxj}KE(#sHiRd#*hi@HIdC-;yp@~1#XJbr6?0%(F<@RtQdM*4 zN)%n;pi9x&cpeEC&ftf&2Ow%^Z4@NfeM6LH_EH<*djH$7x%ET5V$oiShrfBTRQ1on zYud%?Td>w(bTu>w`noLj-W(KNkp%s{S-4K~v-}$@Xf1rg-YCgh2S3t2l*9-0+ASgY zLF_S5RQ{%GSU5KD6!xhkm>R%b$-IgNW_4c@T~K=~Nx5YR`S)S#@?b92IET7&RhLYC3(Xpt=c`X3Zu!I5^>>;l^YxFJ7wvAwY0;>|?6xC0 z^}CM*m)whcyASp`90_AiQ`Fy&H20L*5_p%G-N-8_{RDStw<*PIZ0lgIidTI~@Q>AVC7wAe$dt9`CYUYSXWpA8u;b@DTveGh$m8%QL%b1S z5cQ8EGu0Kz`2}!<76*&WOPBOJy?Q7ahDPY);06dV<`jhT@oLs%kN^}({R@&Fycx6X zdjLP{@4(L;kQsCjum#dK>z-Kv10Vbh)^HT`o_tB|(W9V4#-U0z9VI^;K6@19wOO?u z1p_JmsuoaRC)50h+QqA2z!O58V8URb?myCcUZ^fR8jhv=@l><-6BwerP;_)_r`|QzhO)v+3kHkAD zS|P_B(3If1-$!0envZHc21h5{+)tlF_%S+ zr(hM;ttmJWRd=S)O(6F{5`7P2`|7*dt-+xbQt!j2@;YoiJ$GVjEi8a*;ak)6~}GLD9W{5maaMuI%zz^=$yL&s&?aXFglxV>;1so zq2>WP2%jSk>pYHK{BMZceI>e*`+_|9%?9ohfb%lA5_BB7br8~r<_!K=SFGMRjy$P; zI!;A#NBz4)S#OHv=)s#3a^XrmOas*~DHhL$;70^%Ki3|cQvf|7?JNpy8W8 z@H~_Pd=YPgZF>qAVW;dT71cmhBB9>>o%^Rv^;t{c8{O2N|19K3|1KA8{6jL?^m z(jdlinT^o=na;dcpgx$tR(&&{PcQiV97i;D?E)BHUNf_W%%KKQ!nTG z-ko2pIu=0NP@f(T=afG~-o0kUEBdf%GSRbjcrB?k(7%+bCS|DWebQ0$ZyFE#GSv-H zS*mqhP~9*gTiqB7a{x84caHjGU%07bAOZW{rWqSl-+}y7C=ARrF&+7ls{Eirb?w0$ z)%OP%sizJrR<9l%Qje$750I)o6`K6NHP{QvS|;*s^Tkkg(>QPwMj3vYle=sQwBH+XQ~fotX>)D}ZrsBT{N^RffZsqUpYRehpUNf0 zc>m!YOO(gFWl8-N%9H~wE|_T!W1E)TtPSklZ*)y-oSfI<>DA0Fp>|J&*1u2Pq+V@@(cv3%b-Pj; z_8j!3)W$YMrOd0++w{Tl)kc@6-rwM=)GGBxtvtFz>+{qby>OTBu)9r5>Ck8Ul6>Y8 z6L!N_`e$xfYHon9G4m2*3d5J{gW7<%+O?)MEp=K?RD*X=Z}hfmwFeApp;}LBOiZIb z=#7alO-YT}=I(bjxay30ZP?ZRjy8LBVqQvxVfcpiZAMIm)~5GpJ=zY}yegwd8!)Q$ z9Y#W>vBTA+=MA|!jD$F2D5gqZTCTUa<6>gs8ok@1x?LSw;-I@*ON;4nx5C{KF^$@Q zUXBU523>7)D&3`3?s8+;sP>d=4eqd~%h%;zDlje*E+sKPjC0dCJ%Xh zF+Ezd)~(G>f^u!!H@-qk+A*n7YjO3BZ`A5skhW0^Hy4LJ9Z@~rF0B%7d>+zZk$GCWZaR;KX6Aaa^yz)>rQCj<543CK|(W zhEeaYkE-9Z&s)2fuQR$MdUpA^WwUEy>hE7Sqc^HyTxoY~OVr?AU9lZTx8ZB@ z^~Lx)z2&|te|>bPx586CuG%-PQfob=QX7KeuPn{!cBc-yyJG9&tF&^xd3>8zt2eru z+;yH-_YPxN@Ab54{q9a9?CqY=W%PkVdiLpYwYqA3RZyoPcOST{-Pbm?$6FEA?y2+j zxl#xBYl=$iIc7N8R}Fdh`};lHj7Cq7r{5U%^-fQ&GpY})bT>?_a5os+_9*xCPpNiS z%u6YEL)|8J?K9x+_Jlpdo<8r8H@VJRAKmWjhaFZP-5j0L5SLcz>u?XdQ+qwhEm29; z{=~khtufHE^U%IANOYPE@X6?|HHb<3vTA)Uiu1>Ah)f`pjs@DdM zZQd?_tG~+M>F@V9kFScV^w;?YJk9PtqXL@Lpi$?k@btQCJ-z-x&!D@`UF+`gmb*G{ z(3irVI`)CSyDMvJG_7>?@F;Lz;#cg^mFTk7kK zH0UCuMiA=o*f0Fv|66qtaF5tu^Yrje3Q<(Wvlr`n%noo_=?? zzbU%QUp-;i)j6(Ft9MnqhsRYIRYs+;rq>mAH@k)=4NP3q2t|RN>#otNwD8m#=sR(# zI#Q|e@O>HF=!2W?WR5+E_;Kd4gNff_?mUM06Xx=T#6!#j%u&gd{sRm^p>HZwWlB1IOQ)%u7!vE@mD)gZOsl(tP4p z=DY&pZ<%|T_c)I7E63-AApPT+Yqk)d$~>=(xRiP3g~V4g4_-|C0CVjn#4j@^4HNe= zm;FjS%)ACBp_u>P^Qb)S@Sqd&T;^Ce@oCIS%w^02KC<7=JUpKG4d#X^#C^<7_}mku z?>e6H>)xL@fw}Hr;$xZ1W)Yvs+?P!JSLU{(iMKPCuOxnhId&EC=gjeE5_{)U{tbU2 zPGC+tmv{klat`si%rWN?U&7p;PkaY+LWp=f^BU&YnRk?u{Y&OlxX~E>F=+vnC+z~_ z!FLC`Fo#wV|BbnqxsQ3;8DyWdg3=#iK8|_KpUIxj zoN_kt)y%ETPck>3L-w7_4Qq&}o=WK_77-uAJlhM;<3jycF(>(m&u6Y@zLnYVll>{? zI_8g=dzd{dDZi#Dihm5ViY7jnIg9y9=J*)0Kh2!Z{5f;&M6$=6M(Nk#GsaN<7=BDYyPcT>ROZ*A*${EB_r&Ics2NKU=o;H(sC37qDM&^`5$bK7h_hH0s z%sY-G9%L>}A)da9^79=}yp*|nA@O<4Z6^}n#%!!0ZexyHMLft{cNX#V)s%kAImAnt zlLEw<%>5a}7cj5RBEEyUJDd1b=7e10oy^tii6@^y`R&LjPGN4|NPG@+U5NNXv2P~6 zOYCLDFEGc2i9cobT}tdflk%(IN<5Q!Nd@sz=0te*9NS9~b6FShz05n_CH^P#F!Q&} zp?{No!daAG4f7$)h435;rhf`^<$s6^n6us|zMDDiC*pSI9n4=ar~E?pDQT2nAN<8( zOn)(Rl|h`#yu(Xe!QAU7zDGEk_<82;am3#+FPTg{>Ccp3`C8(c%tOoznOEkJJ&ieK z6LE-n^%mkP=5QJDgA)H@;&$ewa^juLMg{Tsv#C65t{_fk4%ZN$!Q6BUago?-iLYfg z>WCj_F1w4khgsc2>^_I`Th>gxH**{FeCGCN$$l1d{By)*%y};m-_P8~{2p`X%VhVi zq5M|ALVOf+H}kp7E$w8#f;s6Q#E&x%!4E#zK0XsY{JMlZ{x6h%Trcqv%z2*^pTiv6 zPh8Gi%lrg$T;5WfM8=ObmzUoe+5 z`va7ICG%|NLfvoX4|kKB^Rc#g;+vQorV>BNoV7Rc`^@G05RXge@-rX6Tr-{Q$1+#! zOB`gLmOy+x^A6^2<{su>nbT)b{KO2(FK<8MQ<)o?w=kFOPxd>RdzoKm4j(}FUzyuy z6Yss2(r-D8cm;EFGI2h0-_gW1%>Bm@KgF!(62HYxsJJ- zxrO;j<{svq%GEORPz26H;|`OIPF zyO`^kpJr}h{*bwc*%hMv)ITWySgmeFfU|IWzJ(xXWq&jX1G5Go|0dd=#^Klk(4FPGGKNPGxRjPG|054l{qvT*o}Il=5p~ zPGat1K8;zuMfq=GPGG);IhFZU;PHl1>C8Vd&+H`qp<__j&WUo>? z=0%(3>0q9BJaI2`5AzUn&Jwc6{155F%n88L$A?wnrDXpza~1Q=7=H>>9NrU%^?#Om zDf8#d9n8^X6hHoXia(nB6T2^={Dxj4p3YqQAL0eVQD>O?HNumKOPRMY*D;q}LG~Az!{?o4 z>OT~ob&kpZWlo(-9DOO3C+8&M1mQH|qlDKHpUgb4iC9xgW#I=e0kFd<-N$C)?XLvR@AQ^(QwvO{1FZDCA05mp~Yh6Q<>^s~z=n zf6u?6e3-wm=u`F4daK=9|1WLz^_GwJ$7Eov|M3rxu74%;0hm_}mtR=&-$vPw)Zc2y z@+ZZU{vOr~i$2+=x7tyk#rjIt3yZ$lrnlNrp8&@=#(jqM!lEC5{euhg3QK%s)XV); zcz+f8Ul{GU|MP7Ax7tx(b}*%l_hX@6SoCS*ArV-RF@LKa^>V)!-oJ%sEc$IWz15ETCf0w(dSTJq+n?2r`hM1TuwGd7_V#DB zqrU!6RDS<#@}IEiVcKji$SW-Ikx?)A-{Jju=zn3fqyLxL{BO0RKIat5e=_GUEcyya z0~ch>-)cww0PBC|`U{J`-`0MucGS!Lf_VQRmS0%(a5>amkTHL&9reSRRJLz8e__$v z+mF?b`kpM(|2B*A7ZyF-E;K5Cs~z=ne*MkZ<2Ky?-S`C-3{Y>iqdo=B>(Lg^{u9P+s84|v7i831?Wiwg zeLCxfMc)u@%1888JL;#wJQw5Q{h((1WfpzN=0Ay#cGOqFybZt4qj`}v% zr?OsH^wVtPuhowF-oH}*li)j+Us&|B;T#ATSM1W8{7tH>V+}>@A=_tltdZvHZgD)yzK@zrcb4>aBLvS6@vwcm~7N3uAmN|5E${ z3kImS+EL$q4cWd0JLWHp@lg*y_nHeb>aBLv%l*Q5|1j!>F+S?!Y{#$Fj{2;dDSy1* z81=%UPqX!(Ry*qDeq_8q8TG=VuQ$x%kJR64H}k)f^2hs`&HR}~-+*6W!C(b^WZg;M zkx?)AJLCP&s24^%*1r-~T#!+3wWD6{kH-6@Q7??~Q6Ga}V8HkNR@k@nf~4Uhdb%`?paqEcy;e0~ch>-)cv_+~1A& zd!t?$<756gw*7Clqh9U@$NR%kFD&|*w)Q7>ygqUu)XjQ*1lf9h1lf9h1lf9h1lf9h z1lf9h1lf9h1X-?+VEMA}gO|DGzyc7SY2&rPIR9T{oBz*ygnV0g5Y=B;>fg(0BBTGU zcJyDuqolu;^}?cWwdt*P)c3Of{6xxMSoFg-|6A>-m;3keetz74!lG}pD>%W$0H60ky|C!* z`CIL%m*;`-`5@E_i{75U)sA|3eh8l@LcOq)-fBmEDwiLhKSI5*liq4ay*!_U&nux` z*hz1-qh6k8!snY%FYKhZ+EJgz<;UltP%rGHx7tyk&U$=)3iZOGUurx4t#;JQ^Hun~ z73zgWZ$JL6cGS!BSonMv>V-vb-~U!S>XV+M_J_}NpXY7j{aW*D=x^Wx7tzP%=&j(FD&{}Q@u-XwWB_H8Rh>Nw;y5Ar`hyYJL)T0e+}z} zMW1TZTkWWCXMNUm_8+t8!L8V-ugqU=ZZpVf|fd7c-a??t_^=%?BIXSJh#xQP6BBA@>Wi$2wsztxWV zdBvnJ<^Ee(^!EPKYDfK2)?dZ#U)V`+wWEHu=sAC3(I?>-STI;&iI0qWdHxxnherPi zqutzpHvd`es4p$0`oGBK7Z$y}{#Lu0KkJ*h{N>D|kB4&Lf{f+2+EHK2@jqj|u;}gm zpVf}~IDX!{iLW0Bi$2B7eOLZgJL;FQej(RiSoEnjz15ETO4gq&{U@{N?f#SaXh*#~ zua3{JtEEue?*-EQ{_^4mD-jt8zZ?&VocPHuVW>WsbqMuFj5xv!p`q%-| zEB2qT=vUkHRy*pmSf9vxVbNn9a6w*SiI0r>3f5o1$Q^|#tl-}7J6|DEeEEP8wY zW3{8cpY`vuURd-MHvd`es1N->`5(mP7Z$z!_b;p6tp5<{FOvE*i@wK}f3wAo`r02! ze+KJ?o%B{a>dSs2{kL3yVbSlvFR);+!V(`D_40lYd|wE*UtzRElFEMmXSJg~kKZri zgD|kt;eW5LXs~z?7z7u@^ z3D#d&^s)E_77Q?ds~z>-I=%0~CG%HijF0}a-+v(S(T;j~zY4x@1@jjceFDV81sU_V z+EFj>W5M^cnEQ|4&m#I5Tm2x3?dw9rg15mom;@SoCT51r`jj z{8l^amqtPeG=;r zl^0t*IMeybhzzIf74gnDB6g)u&szs0uyt#;J6 z?M3?U`TC8p=y!EZA7Rnk$3Lqb^?j^=m-82P(p&AQA7uS$oWHQ>?d7-H zQSaNE>fgrs3ya=<{lIERy}Sg9b|`2H-czp&`LDf^N9t#;JQ`?c_WTc{TneVR>gwWGds7S(?W*I!ul_V#PFqrQjr zbGZJ(qVKTP-)cvFZ?wWGfJa4P>te%k-SqA$cRuwbyl5+51$ z#*w5SV!bfhvF#i91r`iYZ?&V|2NNW;y~X7h#%-wYffW~I)LZSSm-oTp`{A(t3S)dz zk6&QH0QFWo>gD}$_&zz*3uAoL!|9E=V1Rn79revARQ}I+{1(ResE0rQXD*_*+EFj> ztHbx#VgACRue7yas~z?7emi{M9qNTeZ@>OzwWGd*-;W26{hNAW(O22>x7txJ@6*Hg z>tX)FqBm@Ms~z>z(#U_easI-hFSF^bcGTCLO?r6j-^^cF^l`T9hgLi4<^6y7K0qwL zu;|Na_l>k4s~z?7zCe6`AnJuh-(V}h)sFh^bSnRITz+BED_i-kc2l28`n^TZEc!HK zbo=YJ*ikRy_UKrzJ{`UQ6wWD6%CyDQuM7^-+E6v<@)!%AI zy}W-C-$#jhVbRAMlmIf8-)cv_ysr}9Ux|8QjF07C;vKEG+EFj>x5W2dqFz|^EAa~~ z7-0TZJL=_qnD~B7)C*&L%s(AgT#!+3wWEI7Wwig#;r%Zx`enBITkWW?xq|cuv0hm8 zv1abO^0(SiU(foN`TR#%^vO28)sFgZ)_*sh>Mtz%Qd|4A+EKsyYAU~<%P%Z?`}ISs z9rbHiAKH)d7Z$yJ{$jPGK8N*_ST8L49$WpbcGS!JPeZ)_g+QVYK7% z8xJck$f&p4Q7`YeZD+l(=qpV1F1^)`dU+o%z8@F!7Z$zU|5iKd<^8!6Q%Nr@`eYc_ za6!iMTkWWq_w6o9CB3ld?ell59rf~l-Yu*b7JUnTfdvE1-)cv_ywA5TmGT$H_~`!} zTmNmfqh8+s`zGs!MQ<;^)sFfGeqXS$kn$H6eI0&*1p_R<)sA|3zc9XU826trd^O8& z&);e{%Wru@K{#HBc<^9FCETsH}F+TdQ)>Q8*ztxU>xD(% zYs=qiN4>lsxu5mIqPNc|h<(J!Iw zNAkDYQJ=&5r#OFM(Sv)<1$l)fJ~HZ)yD0w&oWC&IvHte)$7)A?9_!EL{Dno|W9vVy zcGS!Jyz%|sSbt&Bm)pv3wWD6%|NRZ?g+(8-`QK_sy}U1cuf>$Vu;`mz-~?EZS6JdB zqh9g*#Wk+~2!jdzXPI+%#+MDth7JW6OjSKP$OMGP18{bp@|HJcNVYFlUx7qw>wWD6%$Byr3$NCG4J|4fo zf&u1lwWD6%-;VEdN4+q{$NU>$#RVDlRy*qDeed}Gchn1GeAL(2+K<(adU-!QzAqm2 z!lF+xAQ4!QF@LKa_3}P>e7`*Eg)u(nA7{(oYDfJ_KfT|5G2j0xEc!y*_;0nNUfy4i z@3S}k$M3Tjy?y+Z_-IFcA;0hbTFzfs^hpp87vvR|_{gZ2_vNQ@{=#U-`iBkpfCU-# zRy*nw_MrL)Ie%f%vwTEvwWB_f^*%UIu>QiLhu5Q;3o_Rsix+EE{uK;>`Z@(YW;(RTb>?Wk`(kn{@rNz7kZ^u0F!S?#E=KZNw~npaaVEP8mo zzqufK zXZMffZ?&Ud{(eLQ`%hT(Wl^KcZ?&U7W;yxqK+a!S^!EI%cGSnR{sPtuJL#=<)W@;@ zBREmS{UJL-GSBz-yWKVi|=+uomGwWD7CUI+gD z4lKX0==*H#&uT|~D4X)1!{e{8=%rcX> z@3i^PYDazE7RvvEeW?DzqVKTjt#;JUyNL8X38WVmz5V)&)sA}k`zZMLQqcdxqA$1g zpH@5S`?pg5(KARdEc#{S{*mL)YDay_Wu#xsa$qi$a-PX$J@$pwWD7C z9t{3{7_7gr=x5>=STI;&v!h=Ae$2tF7smLBu-dO*TJ5Os;=ez$C4ur67JavE|6A>- z-*G+p@3%cjFD&{zTmNOXqh9{r%>pjJu;^24=MPpp>eFC=z_<@{{=%ZqvX$RzM}6}> zq`!dm!lG}m?LVs>^*Qi>2*!G$IPS6K9Ew(?u;s9)Ma z`m?5y|Aa-q(zgGtcGTCrN%|Dl3yXe@O>ecMez24DH?Uq<^!EPOYDfJr>o>AqSoHSu z2df?R$?sDBUr(m`3yVI-*8Z$^)YpDM`gb{hVbR;mZ?&U-8|!zlURd-{J##@`VTq57 z`qGan|2FuJ?Ozz}*nSNIK43vcz15ETvd>5le@21&KW5Qa+4K@0?Wm9GC;gqQ7Z!cJ zt^Hf=sNeAo=?~!fyRhi(^|#tlU-3QZe`Nm&i{5AR-~U70c?U*SeDOX&=)DI)4822u z0FmB1p(OOOY?4i~WRp#HHxP<+qy?!e(gdW7h)9)=fKfncB1k|`x=2Tgly}a|IlHs7 zyMf<(f4tR8_PghN?#!J#ckbMoxihLAd0yWO*ZacsW6ivN|5NSA7kMqie_2HGteG#O zrJrg?zH%11e$?Yil4s349G3W?4psb7Bj1|%P%6JzqaDwGDs1>cjl60{{y6cAsr`X9 z^9SL);D>qDjy$i2hU=qY`mttS4_~z-A4uz`;d*Mwvu6IO4Bi>OYDb>eTf_C&kY~;O zE{#|1$n*MaxLzCbteNko@v0qpUe68JcSD{v^ZNZqwIk2#zu|gt$g^fXRtsOXBhTx_ z;remNvt~YBD}Plx^1Qwrt~ZA~Yv!A4;j4D!c|AH@pALD}%tP~-_@EBe?8p}>C-eX3 zStZY!dHw#S+L7n=?{Gal44*agfqC4+SMA8BHjv>TqU(<}^Rp!G%zvsK`BHF$LEnm% zW%#U__s1{r!9St!k7`Fg@O{a5C!RI>vo`@j!s3}3Y) z&+7wrB%U?%rSJ=U@DK8;9eG|)5Z4#P^k@vNDT(vDxXBfm6E#^0arAFP?z^N(srJ~mwP zm5FD~{B7;{RXg(Gy(Rw_-TzrLAFlDL9r-nVCBLi;oqy8IkJXN!{n3tm<$;plmre4l znNQCyvX3+Ws&?dgJxp956VD%O=KE;zSMA92`kA<%Ci1MAzky%igMUIbJMz5VCa%AU zJZtpF>j&$F_`yHOt9In`#L44dOvld}{gJe8^!fUk!Q`kUjD0g<{65mmU(&9B)sFnq&n17Ic-GA4fO?6V ze^op3YgS19FsC1B<~wWQvp?FA=k;g*CZ09(f&T8{t9Imhy;@wq7SBIx=JoPJwIl!H zD;fU)2nTuA%r}?8JFkD$jy$iAi|geg&zgBhLHGEpcH}qimEr#l>4ZFM=0mjfSMA8( zI4Jqq#It6;j~0K`jy$jDi|hMh_^g?~1nn~TK^>~ukq^8q!-r+31<#uK7h3#PJMyKj zOMW-;teMXtLU)C)+L13${E`3}K5ORn=TFs+e8}H2{Gr6NW?p~(R_(~=xhwg;-2XwE z`P*9lWq-6I|Ma2cTT=dG&AbKD1wW`m6@S#o^ZLuU9y6Xl)@Uz}n^u0QcI2l#li@d{ z^k>a{kQTmbM}GSY$>$=THS>ctUbQ11;3wB-ZktQSpEdIqE&i$JP`zL*cV#eu@^pYDYfxeaSZ^o;CB;HD0wN&+CuldgLPgXgzY~%WM62?2mTj z)7#4ME%f}(nt6XM|EqT7FLjXoa*97|=5NDwfgjYNia%=PkA_PgE<<_#NTVIk-#qR3 z*`Ik{ZyncP$Kz+sd=BW>!w+f9uhH_aYDb>eqsR5>k!Q{PS?&2jwIhFY zo{WDaoGj#7Gk;G%f8>vL7lFtI=9LAqD^A>n6zz^zB&5rz<-ID)}?%%ALKd$9p)sB41KFNO#*9nHtnt6Tx zc-4;li$jwCka*V2@6zg@svY_5M*befNB$o1g*(gmvu3`!R(`8?;q|uIStakkD&-~_xGWcJh7a?q9r-@_BtL?9*31WK?boUu`6)#uA4)uH z=Jn%O?Z}@deoPs8{H&Qjt(AYO9r-mdKns00Nrr}SgZd~4VTeo$lhsvY@*#LuVt z3v1@9YuArzNB&+F8Ga(ye@HW5K;zjT?a2GplzbQBSu^i|aRK;2jqz9Q$UE9g{wGR5 z*35557(UFacI2yfko+^^Su?+0;!a+*BY&3o@?GTlW6k_#?fj{BuYuiBBnLi}#xSu-DkU*Lm(LKT11$d~FV z!@og1YqVqjPlpXZsF7Fg$Uh~%P^e6Q*39em57mzRfW9*PM#QsbUJqZjBOlUF@(IMV zW592@@px4*3387!dLCcznCuhW5ly&zK_PM zcI3lnNj{8t*34Vv;X99CwTt)@AI$M5?c|5Lr{6xsj(qj(*~PvaQ2McE-XFif2mgdB z{-}`;$|3nM;#s4;JZvej;RiMHsvY??#E&4JHS+@m?c!BC^8T=J0{Si_o;CA}G+wnM zzm)hL#It6;xyGw@+N z`kQJ;eo;Xg{!QXpGq1NFt9Im<7Lk0O?lS#ZGp|2?s&?crm5_WR;#o7V_ur~^6fg@)L+>&HQPJJI|kLM?UpE$!{f|HS-7Y3w-cTsN#Kc-4;l zLE^g+&zgCM_WYsRkq@aR!%rffHS;U9_Dj`{e0ojEFD9Nf^Jo3#0iX_5{81ynxvu2X zh-ZyO!79Z5et9Im5LL~o)c-GA89wUzu3;#o7VkAGC{$oq#%elhW^nJ=J~KdK%1B4LtGBc3($soL?YcI21# zmi!~)Su-D~@v0s9ef=e0I$WkdYvvQR{HNNH4>Cx;HSw&O-=*b$)sFmaqvRurXU%-9 z#;bPZ!=ohs0r9Mv*ZVJ3JM!t#lK-4|*37ro!dLCc2bv}S6Y;E>*ROxoj{Lkh$=@ZO zHS>X5`m1*2Hz!EGa8G&uSTnD`f1uit-<2f!5aL-ge?>cf)sFm9SYQ}^`xDQa`BGZ^ zRl5j(nB+%u_@tS?qUGORid}?1O7hD&eA3K2^zaos@i!@%fBj0+8D=foey7^Et_{BAzw# zdi_zgBmaW<6U4J-UVr{j?Z`WRm*GDko;CBB2KYf8s`#Tuem?Q}d&~3B8tusH^?%il zd?IxGqHi_gSu?O@oTCb`99Aj{{``^ znb)7+R6FvUi9bp_Yv%Rw3#uLYke4$2Tg0%hwiD!-enE&5XV{HRhg{_hjdntA>GRn?CCm9mm|6qe_YHS_xUSMA7mD<}Dd#It5z zpMO)eBcE1L@_UJA&3u6zkO=TW9jf@FM*eI~$zP`Nr>xPA>8FohQtil(Z6Nu66h3R_ zS7^@01A}YDYd~ zsO0-l{8=-vKYyuqAZRSu?M{KcL!?_n#v9yTr3*-sAl_@<%)J1ExtnUq5;NSu?+0OFz|) ze2$MK50CL8|FC90MH|1S+L0eSU-H2eK5OPfwECNBM}GSP$@e6lHS_xWd#YW8zew^n z4xcph6SdD?U$rCew^H(#h-b~b-u|!J zMf}%D{s+2#{Yf*gk3UiE$e&#&`955JkY;|3cKx$I+L0f-N%A)0Su>xejsH>Y$hY4m z`4u#O6l><`@^#*SR6FvWiOteMx_4^=zz`;N-+ z2NBPj`9$sft9Imrev$k%;#o7Frq$n6JMvFq;u-YaLOg5cuV~kgY8S`I;#vFRRXg&j#OE}~^k>byUVm5Z$R}pb zDbl7o@vNEeQ%oeEGyPOM@@WB*k0YKn^H(J9q|)fQ{q`Oza7882mgdB{-}|U zf`xmX z{Gdi&wIjcg_^w9Dvqpd9kHUr@)X1xL%a@2u6IRXg%Wt5fu=Z`h>`tzG=NB(pl$#){2HS@Q%@KrnV`(WWi^i3q5HS_8IV1y6q zP{kiL@{0ybem3!}(Ow?5?OOSv+K~^Bmi#v2Su?Nqf2nrltH(+HJn^iVpQn|dsvY@# zHpyo)$@F(>ivJ7T5q$7ZsOk?I`CSgl2NKU3{qguuYtL`09r;-!CEuQS*355&ec%T* zhOgR@Z$3)$7UEemzXrd+2mc_i+L6Bj3n!v&3h}J*H;n&0E&r%?#P=hfHQF)#uizK>;2-2wJM!r>W%$#HXN~^IkJaL@+L0eH zNAeqpXU+UB`~n~RgW;=oF*zMlT79r<+PqlstDygvV;YDYeGjSPPd@vNEG+n-cB@BY$aw%aG z&hXh@asEomWM?zR~SK{Bkn*I!Q#iC%^ms^Cjhje?=H%T8S zoqLq@57;mLtCJo~x)13!q^FUdbwK)mOFGB*QlBT?jdZRLXQclmU5a%6anc@0`hC)sNe@O{3hu9@ZKVCu<^D&rUX(|i z-x;h+lX!t{*m>laG|f0e!52Ltm9>PPY+97nsmwqshhFR)l={TSQnByh4gUJ zOG%HKChZ4Ej~&um@RwOv?JxAdq?ZFn{Gcrf z|DjK2_?tff>L+P3{V;!^4x#gn`3vfFhV!?f z=K0xhK0eeu-yY5nh??jB!}<15^L%9=chwE&rgT*HKFGDm~j3y z)I2}hand~B8IGetJI_Cb^F^WN`JizAEYv(d7S4x_n&-Rzg>>WWa=p0gq<29FGr0M| zv)T^^yng+kr03O^_NS!%>q?F5nV>(fZ-Vm!qUQPk%%nHx zlK$gJuZIdhxcMz7-Ja|?z6g09KZNrYq2~FBaQ-9IJpU2Shk}~tL&5%W)ZG7#{oSa! zKO4vYpyu&CI37p%$IAW@?4L)?{qH#51~reT!TJAC^Zb4|9}sGu?+526Le29J;e0=+ zdA=Wo=J_^oeh<_qi{xF>H1vSs-g7eRy=J{oCJ{iwFfpPpYY961A-XV$k*Ik+NPMdRC}; zy((NE3pKBQh3ieB=Jlj-{VLSFJ{7L#gqqiL!u6q0^ZHM?UJ+_uj|ku2Le1}I;d~IN zdAG@=DO8Q`;wD%|7`9rBw zNH?bNXOIpjy@qt^UU~j^k)BBLxlVc!>3~2PA0AKMiZrh;IGi-Ex3`fruOE1cG_R+K z>mOoz@cM>zNQXR>>Cv0CKRgJ5n;*`Xh5kGr)-uw(J_W8Pf_7dnthC2c zdbTC~!Y0FWkRCwx&q)Vum+5nybe`FAf0@e3{qg$RxE?l!&+AJnACMmPT-vi&kl|UNg9L*0t3=xIMC#_GcR>d<+Ix~-^i1j@q?=RtW7z+xw0}(c z23p9{zPg2hVJx210q#J)D(|;l9Ksq0|UXsY4w0;t}8fgdVB30!6(%VS?TBI*kmFd}qbZn5+X3`sLNIik{ zqgqlgCViItHd?U77T(F;X`tokn^v=|u7$ zMY=gW2jTUQN;(SaC8+0XoE}$i9(v8)@H3IuPm^=>HSx_O$=wq;I#C_Vc8R zkiJfOJL&tRUqnj(|485KBz4Z}^7z-Zle#eJSh80F4e`hVQ4yaunl@>AmZrCB`jXWC zyuW`nU8IHxFEiVNq;~pu(X>_bpCNTt=2vL;otpiO)R@%bb4$~iYKr(c?FFTF#;2}k zZ>{Mrn(n9R7){$XJx*%p@yyrk>omPv(`lMMCAD*Z*EIXTntr8e|61=^L89r|GAf z_6u@9A9*!hTGJIZT~pKbHQiLx-89`>(}SgU=C348Pt)`YP4CzAc}+jiv|nv;{Lb)- zXu7PXn`yd>ru%8yrs+>Ky++g9HJzsE`%*j0lP8+aR!2LZnl7j5`kL;k>3B_#*YwAl zUa9HLnm(ZEGn&4w>F1ixRM-9Za%sA_rUNxyRnv7e-B8o5HQia$;hG+xX_Ka{n*Ko3 zQ#9Q!pPyfE*!scNAGQIo4TNnFY=dEoge@Aj7}#QAGs89nwm8@i37Y(rri23rbj!(kf%+ep|(!S(@cqhT8Z+gRAf!8QT5iLgzA4clWU!!`xB zsjy9hZ8~hJu+4z&BiLrbHXF7%u+4>S9&8`O_6cmC!Zsha&tO{s+d|kD!L}H-C9r)C z+fvw;!L}T>FJM~%+e+A0!L}N23sD1Z@95(tkXJo6$M=|CxTF5V!8IVY@x% z-S)6`fUP5JonY$>TUXe+!DfKXSALlVw*Mv11o|1ew(Qu-(5iEburNb;L+h~ME+&W3 z(ml9MN}|c;2scK+P&i*cosGjywjL(CIm(m}IoyX^xGBMIwUMz)oA4HI6P9;Bui<%f zSB=eqrnz$HzZhX@rGe0b4;wmtB(g~i-|X;c)B{1 zC(0CQjgqNof3sV>ImHxZh`?hO3I7&|%{c5W0dqnWoYW}8{}M=oH6hZNY&TjAZyyev z@wik(fc~oUP-H_BiL+#H%!;%)$_!Md@oZCcqWyJ2sxgn|-*6c6<+8VMEO4hx8%eq4AR3R*=B<~g2C>9 z1C4hj6qSE)zeq5MbdOH9n+(RGkoxi-$tAFNM{uS)2PUrT{sTDxZ$5DUgwjhooA_Zm0i{sD8!~TXK?06zF3w8!FkhRYcWW!!;&PNFTz;Nlq(_unyp#$cJE#72Pf zLZNCLUIj6h_;v__2#*TnMT@YDQpjH)SNVHnv3J~QOUUPcPq%Y?|;q0S~aQC!F zLV08NQQ%0|xF{oB^X`|p%R+^w9_x6cBR<*U^?(!;UK2WLkHl)NjjEKf$|;qB`Y3e3 zsMLp{P7ehfUfytPfJ%_fXrLODmxNO|nFmA$fU8EUp1KDiUEko73^m`3UAR6d>r!tY zQ!m{A9MDiG^WJor?&16|K{$pdV$qXfBJ$qpzGF|cz}-5=W=)1-@&AGR-$VYN0`q-j z;voymkVqpuFd2O35zWzC@Zzz_RdW$FMVU+9*4vw(OmUmM`*^K>VK!$kQ9Kx|37$7b z53dXg5knkQ{(RlEaYt@;pC=EJRa_s=o2a+rONzs19;jEx$(xPu6w^K5us3$Ltzj_PZcNG~E4XzTb(#Bi4z8EFyN(LxEEk>w^1pC;hMtKj|>E!F& zi_mR!NAi9Dkg;_7c?;~Vl0rIhw)Cv$V1me6FSMCp;}2bd@>#-p6EJ z&O9orDe_c%f2PCW+z~Z#y-BSN>Fcd@@@BG zabAb;2q>ycULGjmI6U0mv@023ahZ6lgwZZBo4d}K4@bExyE!2))@t*tHNmc=hiq(u ztI3`mFE3Cpk@Vu~#mjA1k;X(vGCbfw)zY(43!%F?%l*G;mmYS0pcV<{yU1`pcnd#$ zFMMVBF7Vwv>{KK4ap7ye{9x@XQ1STUX*4@rbwem2U5`rGexkdBgY$vP*VC=)BMTI7 zdx|&4!Na`O8gGb(Mgb^B#bYzKU)Z5u4;64%%&0nx88Fdkwuwq7H7Lk1^qA^E26pcc zFc|>8Om*vcBp5|QG~_`}z?azot{d4MW3b1Wm4-fQ8Fm(!p3NZeoEHae+*Su%xzI`y z?IpuQ+mO*>fy@>ionSCVSZzx2;%0-!AhSKzb%nbb;n~m)w3$Sm0`fi8I~>@erabyX z;Ric_HYmLyfg_DH+Prw$lM@pmcJk6tq5*AR@S`c-Y8#G?lkNv#vtkHp91Ol1I$YQ2 z+m$bR%;mF9j48n+nmm=V5UaN$|GwRhp=l4=bf6UnZoI>+wm578v74az5*pcD#|X=2qm`z4^;c1Qi8m^xsLSg6XXwR%9x3-&$#8&y%Ca#*DX?&daHT_=4fLOL z7Xtohw?j=@^g+OJXeO;|Lb$UlmtV4kiAau$F)4*~2Ch`@MGSYSMXz0+`Y7u#(fCbO z9ItKAWryhf+n>E=IWx!q_ zXj6yg?kH!Ym2$ewF{0aruYK-36TN8a8ONKI8Sc2^n2mPNj#yV^8*UKZn^4?bJfbTy zh?YAPT?wUho!m7$OoyDW+bsUZSmfig6f`)CXL zg)Ff<|x z+RiP*4bcAtgBQHG(VnDty!QjZ-D!R};J__6U_AsYBiNv9F=4YXp2B#qKFU#Y!%V_3 zpF@%9;^th!Caq!qTxDGYi%-*`Dg|94~m6V12qa7L+Dc}&JpA+|T%>v_CWbcdj z%;3}8Yr>Pao?Reoi@|N8wxf^Ufcj~KIY!o(GqS~*)nPO$g+p$XgHMEdvf1L6yoo+{ z50BpNY#fYsgc>HKN+Jw_Q?DVYjzW@(t5vjZisnt3;i2n#bDs!t=e z_0c}C9_YUN*klJ@A?`mW+Y|Bb#Z?Tbf@0Uc%g6hGB)v{0y~LeYqH=^KJ)}SpPEm3D zS+)(R{;mO=z6NsA?~r|L)<~#Sz;h4{FID%0_gJ#SLrlH8(q^&7;3yvlR{w`2nqpvt z7IYW!cer!l{XiIGJj7RQE}>sV%Lzo<5AKI%NXA5X{-<4tue1mJE_Z<6B+uWlJw$?b zBaQ&W-|?GanAs5v_b|M3bP4a=O6)3ZINT@VTbrVd$reXz6V|=7alhyaMJOb1*F+q3 z1Q)B@5M#2#6K^+q2yAV#N7~GZFl@`ywQX{O7`xO8cI1GvqZf8EdD&$TZuSZ@L9~4A zFEp$b_9<(jz^}CP7B-s^&VyTYL&Ms_p>z>7d_8`lVO^3%Ta|Z^jIGUb^p(wX*W?lA zz%f}K0e6kSAzB_kyLqdlpF_i#FwhC=k(^-aY{gz4KZ8LWXQT~oZD9^*>T3yAD;UIQ z=$SAK`Wg(7fXVUbW@y!x(?f>T3T}(`9;Rp$jElpwZGeHuadt--Xb}DkryJTM!AF#R z?!Uw&Ly;hVf-&7TxK9{hfEILGQbDkkjKlmG*V;BO*4yUNG7c(^)XJSmKUgXfZH z1C}W=NuY#?#Je&6#p*Krg_58D!fgeI8zo>xIbP8GR6S|&zt2tf?Qror2NQqWEbU4Qw zqo7$3Xt+0FK1!5N(O4m{!A%S<7Au@=w_oK+bo&YG07+ImTqsaZ7!u%AvKbz_;3WVX zoPtQGS!=)BMbn7(qr4`ZJY8h0P1sWAB8FJS9giX%4|kLlxV2%S#(?~d^CJI>ncL5h z1ml$Y#gP56oj9MjNl@Fi!5mZ${$#bFcI0{o?G6~0;U>V&`W zzxWG<5&cbeeZgk#P9?o7A@jZGbcpDb(w6pK&1h$M=aT}MgZ42^u5ttUJiAN zO^%L+rY8E0>M>BK5CbK#dWSJ|w<^C3kBo@4K`Cjl#YB)3vNbk#U-`w%i*%p#$o zCIHd1Kn^pEu$TeE7D}rPAJ`zR#E$S6Q7Mr>)3t}c!u5h9eZdQA!;#6bxyQ}x;Y7$2 zDE4TM!UVv0i9(Ctf59Ij%vhAe0BtP5q>t+-%*_1cDV*`O9XL*WXT%w(%MgWU14jG1 znMhERwhV!PQ(gmtX2m-7>KiE5+_cp&8{rMfMa51be5V9E*`U5=; zkOx}|TSG5KYc!Pe4lf4ywX3jsqr~I5cc>+{`+8%bcJ3>BUD4!?sTrA)A`6h@NViD3 zy;l|(C9gd-LhcSmTMQKJ36u`L{otTmxVgGTO4}ziD?lY(CT+&MR6R4;qkK-0JG5uG z$2MYI6?tZ`N4#{zJ9T01=h*#7;HY~jj&HfY?#ZTPsL$HtGK{IZV<(Y82nq7gGTdjV z^vPf^l#-YgGu#uUqeqr?PoE5Oq5GSJ-ZP@+SHs7d3~h4Kp~3mv#42ERGX_zsU}BEap-%dd!Izdnw1)YDWn zYxpvXhY4=55$_Z!XTujHo2U%-M7I)l<+?zqU_&PW81!q)14Bm_2UQsy zqY3RU;u+aTW`Gtj8V%xnUq{`Ac){2iF}Hq~1c;H^6pzDyl}=^tB)crovIdO~@H&i= zIrOx%!D9t>w20(##fP%JvM-lS`4OtQ@i3wTo~06Lyo~OK7HsBwLkT2H7tyInC0Is% ztDLDs`=54--AgvPU#LVuAqeB>@FjF{Vd3K))XpFi!s~NE20Q$#_H=pMi9-zQG{I}s zP?XA2#&xjn0F}PtaHw~YiX|KH%Gq$ouqMqYiy$H zIO90Tz~Nf)=|-iPNNl)2!z2;P{mwm-YGS{shubZg`B~KaIaGQq-y+A;@Bn&i`K|i< z6S0BPW0;VazI*8yW3fiyW3u#t zBjpSbI-or&x^(Y!y;opPfB}uzl+HN^&WY%R;}P;Es78zT@!iIBV%5l|zN4>WR4j^S zU|+iyqjvF5H4Hj#eaCis>PD`uw#vOs6D{Ae~|%v?_TX>X2#_$=OTb{ zaqw*9`c)56ZIz$kn znG~Ts6Jd3l>qS&e!Zt27Eh&Ql*~uMr9Xz?<&TO$TD}Zw9R1@AjprVO2dAPCTs~Mur z*keCf#}?H;tnkCQGY<#xFaUv}U40m!z?{zt-pyD&^Fi{^84?6EW^Dw#%cp~)ra6+dxf#1hCA2gO&6g(m>U;o{&4cVA8iR}4IN z@7lAcw-SJ-8$XDNXC@DG6i&Nfw?;c~!X7*fmxpo$axkE`R}Z~J#%(8_c6+Qj8uE*n zoWm6ucKXPvljz+7DA{GX?|#L2`e8oz=1C!o>k?lo5i{?=?Lxei=?a+KUyi*~}R8nBV2CBZ|+#+C(&O z8Dr-mzA-Eq!V(L8yU;f7eoQ#j+LZz_wY?UJ=;$gnWy`eZEg77=RuV*iDuga~?TW2? z@Dk-IrNleTP<_EnqI2$5QCa?rx*(lHF_DLRFobYbRXoR;u&);%cwDadny}LcszDYQ zGY8ctD0!5|3~#|Fy4{i8J=D6gjI8KKbA{^eE?;oOhXClA<9ocji`)wqpJGfLzR{Dh zt8nx3WP?`~6uvl8h96$VZAH`@Gu|c69|H`u^uR3=yO3}qG8Yn$bIj41GvI5#2H7Fz z{eaXY(nct`AGwDk3?GD74WaEN3cG)mf>LH#XwXq#{`Q)=>B{CeAwp;ZMun%wT zn{xagW*W+Q3HF%UoRF?v#Mc(19i0OmJ?j2YEud!vIrPB2Spcl^aX{3Tl-j(v{Wxi!0((A(srOY%PVu&c@?1h8P(z6q0Pj2arl814zTIe-I1~gD{_UJ&p`90A6F0+T zO<1FmZ8ct^;3LjZ;^V7hN^+|gF|=I6EKA@5S=!~|o~h*JL{V@R0E)XD<1gai!-1$CsN>?uzKHlkEk)Q)}J6j$JL{84EVutzH zJeD)+}nx@;hK%G-;0vCyuJq$US?N|QMoyv3_;^0f~D4aFeF=vt*`x&mv6JD1usqcl7M_!&*kClB7iz3JpMzK4GL1xUQF#rE?4*T zb!IF~EniNYqcZYwW~GdrC=b2S1=?COhz{qPjC@3t;ml@yNHUZ7@}V=HQ84%pl-HQz zse*YPWXYpcgqu#VTURb6pNyo?urYQGh8K?#iz5>5y$6o3**}@R7~;Z0KBIg zBJtHe&qf!`)zd+gJWwZvySFhhNUMOjXirdS>%lzk(cI%=`DxwL$J;H23RzbbQzVH; zg1Hs;q8(Iku~e@BK0w!QJgg!qAq~V-Q_fM**nR8asud;f@5Xp|XSC1c1niW>HUeyh z@Q7Umyb|owNM!_$OLBo2HA+=^KscEVyy5{HMo%X0oo?-6qsm&<@N&tXjvs(I{B6RkN1iAI=R`_gSP9pvtGo0#LxyUF>4 z<||A}tqg-U!6OXz+j|c})c3`dCov{VG9)1nZ-*{3(KO;bKFInwsM6EH6p;*-@MQ5u zhfiA0WarcD7zSh4WM=W{0~N&>7?T8bB|0L{s5l4V!m+?OGCU#t*ny3`P|<@~j7&+c zU+J-cUMs zv?xKHKEvKA>Fd_$r5ScLx=+sP9uYC}6KbZ$2vI%twPWd)@xeQHLW!j0 zD)%sjBTRsScaaRougD{ZBY{2QOWZhgQ*StS-?b#5T^|}N(Np9`=$7|&TyVpIfs&#j z6x-v~fjjOymW~0E+TFG z2fUhqZQrpb3ykC@n<>Olw+@^R*tZ+oXuNk( z;81%}utF~7Cnz|>o_!uJGu+qYlejA?@EAn(0a-tFIlFb{@pLGzzBkv1D!XTs1niZ% zx!jv&@OPNoON?FS^n%~u7UY_oTJBgLv!21h*Q0C17$p)58rVf^ggj9^nLY2=HNp(< zvA(-%#5i1!(Lj0>qah^s8LV{=KBw-}Qs`b{9Izc~4~dXXGVUFdy>5~C=BqM#7%Lh2 zPG!)ZePZ7IJO;|iha!(I0x|z2KKC19V&btWAu3tzHKVE<)`PfL$d$*)sPrvxSBIF< z7M=z1bs5;L7X*jih>Bd4Ili{bjuY=d`3!SgQ`1;9j0?g6L>@JM3S49{?h=4GuUxy* zYn>hzCyI}b@i0@3Zx7Mm0I%$N?{9GJNL&V9b|eZPY*HeJn(MT~a|#qxTJybVHi1&b z%M_X!`s*D?89Iw7>jX!Ink6tDv6J5Ogi(UyN#=4^9*2#lu~e#!ZYis_7(3rcq*DOT z%29Y5ajOUzVNwmNIx`x!*y>Jj*Al}glu+qxajx>#DkDZHLEp3Mbi6f$?`v_z!98C< zfY?s241aJ|ro1}R_ZXxagQT-&rm}tk5Ozv4wyhTUv29|Hvt!JBrsBrNT zsBQ?uDeSzMdWN?R38S}p-53a`1-@q{-j8ZX@9S|$U~C2VW-;qTd+0&1z*{w*E?vdz zz+MbJxB?kwa(FRg-J~Twxw*&5V(CS%Q@j5T4^IhEZD9o=qv!tIUbE37Dc_%V-x>ba z8{Q;uWrT_XyxyzFFaqBBg`>x@?rnI)y4xQ3!OQ(eIZc(@Z%{+Qy?8xVHycK$X9Bzl zEnW$AzS<0zfye%wudMR9f*s*N@C;+@)dvG@O^pxX(B-M81GJIi?ZPt>_`ekF*qNNWiSoHADojl21c(a2hv;19ojL#JMJ(`XiFo^)o8Ne6#%I!@0Q+kzkDoqJ6d>+hhIyiq7C;W52OPw zVx}i30MaV_u!sw_wt^OohBDn6!fWV|Hui?};v_boWw;BO2u@5pLI{nDvF>uDr5<^( zVp-RXD(9jIz?kuQg3vieP#MM+c8Fa=@m92#X!+V3Px0XsWCsj76dE7A?AyZyN>iw< z^L!ybciq_>VS`62+;f;Xxy}RfbrjBP3vRYrTKGc3#2kv^Lf6g!Bse5m3y0Ut;AVu< zG&w>}YoZ^%^n;?LlgZ=eDhFffq2g=1+Ha!Wsf&2Fa?1tccRFV1ZE>?HbI*1$ru3Gx z&qA_zDfi_|GH_>bRf6;zoztbOyvKM1hOgPEsjsI4jgN4;cuiiLS-kAG3*6G6Ug1%m z0K%^-dj=i1+F}fA0nlNpcD)tF4PM(q^@LIVw?> zleC--7O>M+ltZ14sRhIhv2Jxu=6->;fA+@ zR#b!U8r>EuP7Osh4>0dHAGvqLygz8U>~9Lz^~~uJH8~5qy)DddVez;(I7EHf{gi0w z0*wY4WeI7P_f6<&^Q@XH4*bmE)elF^^@OqH7A$+jW3Uzxr-OSIgpMQ=Gy`=BgCm#I zc9l1D2n`Ew5!$Ma0j9=k-?ep^uQiR|!oRb&zVRE8aA?@VX@SJ_B_qr+BaAQ<0B0Ie zXHlsMWi*T;7xV6TGr{XcHlr5|o^x<6KChgxDWIG%Mw{8BCijU$!x2qfg46lb#DAa! zl}$5bP}*3t5GOp zMzpgzTguwCY6U?@nqX=s(fL)oK?6UJ-$H^tehaQs-{ZG>^@2Qqt6vX$4P~DaJWkUb zb%x*&gQ-r4VQB68nVJPpg_2{ZB&&~PF zBSZFT(W66)mYv(Ua=q@_tw+~xhRz+r!kyQJyN6tzJA=@CM9FRH4KG)tXUu#Mo)Prw z=p~+B{d`9EjTg7K82f^}McTWs*Ldr$;@o$D2zZ?Hng%f+4^0Z_!zF@R`!d{(sC>L( zKeCa{+iu*_x~8iS8!8UY3W)ceYa~q^^H#nvYC@R~=~m3Ey=Dz85j59l{h@sDXS@Zbf8no(8UCU zTB6#Cm!cuzV7wRJXK4P_7fk!kauPE^clz1^5+E5g%AWLpHd;)G%TYJxx6U zv(kV9-JG3Tgqd1%f+NsCbG%?uBGgR5Vhhv^4%Ovm%_wqW zcjB08#%JV*yOVBKKl(hsm}j4#U&(7AGto~);J3{1+eyF7{(gR$;Xgk=8*F{Ze%>$d zispft{qm5`_)pGn%acPUpQ-29;Y!~FUo1}=dZ5XrCcPK8D10Mjo}))$-gyV_*K zp;HYzjV*i5l=tt}yX!t2bYa%;5tY)G{xyHZu#Wkk-mvtK+cKre^eUsWG%uXoV)Cp* ze|=?L>sS!dqub$`gRjnQWQpG~pu_mD?v=UPGQG=$pY9yma<$V(d8^Fr)BD-_+pmK^ zzFcor)r!ZeS03KFSn*Q37nCiT*|0gd`;qs|8`hPa9v16g=gIm}m7ms&?@_Y$w9;kn z-iobzJ-T+J{YdS#6?eD%dUC|e9kp_m{i4yN-a9YV%$fUs>q~nZ-d$cku~7D%sil%% zTaV@Ws9Z#qBfFZn`gqLD<~RRJ3%P!){lKY+0$c454+-PW`4(;n2P{dzT+DYg1V8;?9$%zOFjb5&eF)I$!39IaI3q zn6sM(r&6<*;&`%izbvi0T1$L`D;u;lv!CH#+ePMTIAf76RIDkc4~ ztkRj{$*(b$!|hfM zY*?VciSpOl=DyYCZo#4-9{X_khZpiRk9=aU(|+IMdxl&84A@&Q?w119znuKx&s}LR_~G!DOUIp!=$?|@Tz}T=8|LNJ!{I=7hKpP@@4ENxee7)I_|vq-hw;k!|zmla-`MTX+g!R zH_g}KJ5#|=n=L6n`Q@%BPyD|+Qs%#sN6&oLe!``TqjJBy=jO}5Di;W9mAB=j(9&DW z^*U6t_^#y3xmW#Ns!`mM0yXmQ>0b8Nksa4PnA>t*(T~y+e%(-iOyLRDYkcv0zhkdE z1*NZ8-sOXm(T{gknegj_g(Gg3n0m6vchBm6b-vN6YPW~QHaL)VYG8@d-_5EW6F52c z@6x3&eURhU^Qc~(mzy@VnN)h_rX@qt{_=}34{jW`(>DF(r4|n__u4SB{;^NGy=s0j z>Al~3`6VZY=GZtj;=bwF_RvXN-W@sR@bvm6vsJnA^N}SJn{N6p_EFPu;X9I_^?rZm z?9#=*zVgrCH*20m{Gv`czJHO%z4bxlvHf-glcc!lXcWVCV8>1tV>lCW`PWuB{P5U~mToiusE5?-bbA((L@1v6(j;uMr+=O@)Jk~tmEVJb<|`|cRzC{+_20(N_x<=zwrOeO2BwTJd&|=MkKcZ- z^f3N#<9$EZKiB=c#K?ym8{A%V|MbQA;b%7;>R;%=;VF%`A1_<1V{pxdLCtRtDt@8$ zrQIir9}2uSKIPu*QiI|qtg3b-?5oYsi!6Jb`%bq-PxsvE5S{J*u#qLNhHYPH`ebeK z4$Fow8S`rY#{GHQ|6V)q%z(OQ%Pzn1GTWj0z5CuAz3onyBip|Xn`Jrk*_uo_<0?ljc6M;>*A={ zn+jYR^w2;2;>(vstd)wkUO!;zmB{Xer+zeX$<0;~V_W9$&~<)+dC$f+{dIAl&qlTk z+!=bZYlol}N~-8{04Op9>H3K0deTz3EwIPdPblVzy@m8a!zn zetqBfUxsyA_kLPZy#^y6UW=~u&ykh`>;G{2=;k8DGG{;9xNy!nUG982%x}%QIa@L{ z&U4~blT~F>SJruW%Ch3N?9D{85i) zvyauNUwD7VVJ`~Lxpk_+-kSGbof>yEW!kuD)k@tTAGXtUFC|OEYmZYpeeiA7y7MkP z-|&6umIrGsX&ssL{`&y@N&W$)BJ(KKk1?c~W}XAhNVR<+@~vs-1kH#zgzt_$P*dJfFg zt=~SYy?EX8HJ2?eb!Smn!*_oOD*X4=zO@?W3Y=3dN97f_)|c8ceoz79rdE+vw?B;k zb8f!cSr1n@HtbpB==^twxnJ}UMBI%!;zgCQYv^vw{GG#t) z`+4dwJ8Jijxp1#~%P~c3<{Vp~&Fo1ZuKL?i>!9h{!x6U|2JY_o{N3MXO&`{)@6tn` zPkQx!a?e5~M%7q6xOrr?X9KRpf7-NpWWcI+4-Gr)=kp$2*6{QD#Yb;8U!{V!#k3{PEJ zw@Jgy@#SkD`RGB;)q}HbdtK(;^wiIOsW>K2w|`Fddr>$!Z!OAI-jD9_W$xeZ zMEPfrUHZ|s>&D3F%bx}I{Bc3Ze*bexhhKeJdB6Wpg$k6(u`B22-8l{+_XG_&6R@A$a! zvRO@kxi_ffNBNpI4~nssKha=gxovafx8?5f`!|7YvpOoZ8+>Yyy;ZK&U4quVY+Ewh zKL^^5PMi?>Z|k^Y_6KMFew;l2ueGCp9-Jxc@1MI`9q+xYapc~B_`MT@CT44PKVo^` zIcq*TH2!syM$=8_R$o7nc41q&oX28<*S+ddYT1x+vn$o;b)`?Ev(K`{j~_Ut?aPa2 z?nMUOd@&`VSMiTWIl^zJ_nP+5Sij-=J^g#H4ZGE|WRc^RS}m6yI~u;G!@mDY^hjH< z*IsejgN~EpNBo&QEjlS*vs%s1w3zzIllG4OACzJt>2clzzIpC~Z+ zx5X((z6q;4?BE}FX5XIKwnL9@74|=Czw@iqZU>8gaP_x;k_$Ar_~)fH=d7pC)Vxuk z;fV>u@@|hP*Js+Dg=KU0NnY%?)^uo2tt;K1X76YiQtja3(c?$VeK*U*#eaod|M1)T zV*+cXJP7_{+WD_5)LMLV=l7HQW~+>nr*8JY8P>M%#)VE>_u^8g=&3$stc~Y?|2N(Y75g?ic>6XKKIVpGJH)F#gHY z=OYiSF~(h39TruvY+K8-W=#U~Pbjiu=I8acwocw=&i!7N4cmrfopbo*B*VBL+qCL@ z=$m|(e%w2uLZq?IPFwW+BNa9rZ_(h+VdH=&VX3D6rV6`CR9o8NyI*3rM~to8D09Vc zf{)hh6<}-fU)1!6U9+68{B2UgquD1$HukGHaQcrgzHM1)^tx81|Gc=m+}%!7d)2Wm zUJz(7e^4i7VouW^`F3Y+6u)vn+adRNRgE7%v(o%kn|IzEIN|$yvxi)La6dB7kPH9R zt32^atslN?*Q5W;W!=7=@!h~8v8AWYtX;ERdg_pT`OehKcWXrQqWir&HFpJIHrrAyFWGfvx z`Bq5Yo=uK_^s;XEo?W-Sex4=x!)uoZ)IK=<&%>Sm`)lvR2}53gaCO@^ErL#^4LAL1 zJy&E&Sh+dH_KaISVr=@fam}XJ{PM@&ZdZT!^Iz6%w^lqk{Qf&zH@?U*qOR>^qvz}D z-E3I>k55m;F35ND=zno>1%_W9Jk&BEpyr{G`*Yf_|MpkkPFpSZ#+gj1xwe+x{_^z2 z8tt|`%k#&Pfi-^md~l;r8z-NiUUhPi*>QC4s*s;y=3vu9}m@w zn>r*%@fwHvP2X9(?8A1mrnGC;^w;_8 zZZ7|OMR7;>*~M;-D;9Wa>HUHGKOa$`Yq{bdy#BLGwfaRbSJ^-Pi|uuvJU(9VUj5?n z^PU9{JU6RUNYfb;+s(M~?udF*Zyu~aan8DlJ#sHR(c=EAze|;|yqeOfQMGY<=8Z|; z-m&wR_fnqD%yF{n@nR*iubXF{KWc&hjx_;WtW|88N`#Dyy&qV_KPk)OI}-z2ELvXg zhx*3&@h>)vKb_^_+H4K#*G-Li_0h~NKYh^0dMrotOsBH#J@KGg@TMATJ7(*D;7HQB zpm*kedbihS$75fuZ8!8YL%#A4YfXt6{c>2QupQHH&&gJDz`IM!*UZ_ieU8|;PmEXQ zOt`aSdy6cQ!5v1{KGD8P@QbAv@_!rf@P{Q^0}2hka(`Kw=)dpR9q6|qU}NonhV1+2 zNN|ot&%XYl>EY$8I@TUHyH>$-&vy39QMCTQ^@;}V8u0$MYC96UW$n{C>CbUZ6W4W^ zFsbjVG400X`*BD7q6TX#mZ+Kc?1ZJYHq2|^Kku4;TfYq5Is0@>(z&w{4{f*7PM%6C zR`H4D^Fuq1n6o~a^TYFIqw_32em=0~lnwnR9_ur@-sCa8#!k%K>)UD*jsAJpXRh_j z*S|HolPme#+c9^$ zuOD@pf&O{a|b}WodACzpO=8RGsPf$b4nd-m%knwzBOXQ8#4! zi_BL`%>5<4O`n@B+D=JnSa|HvZP&8&*|Ds_l94+aG&*o;XN5m558Je}^Zkz1UT%xL zk>~R+UmMrYeP>aHwA;ld*6mnl?*4PbmQC6I?yYUB%>|2PDzo^L2kRpf22_7t)PCXU z{OyJAG}v_OQTsWs4mBKk_v-`s`tDxa`P)nbJ}6SGbBE_`R!{xLcr?uO9@vlJ*}VP!QBcQ2Q^(B(|Omi zSA#5d4p%+7yQg{T{nL?~-Z9jvbMNW)`Xe_Ut1_^7{dKEN3Djh2( zEN=a}^W4yXzp6H5(%p3zv&HQQd>wP_{H5>(hk`pzxHqRswE_*QpE#RX@J{X$|K`s# zHu&X*<7;jYda$8lmt3doWNBOEs|_p5hg`1L^7F4(4=maLVq$o?Y?Gt&rw!_Uufr!t zYQNWUn5DY$;qW4JET4C6VEUq3xuGrp+Vynuq*JG>70p(%S)Qy9PL+7oE6DLg7wbLBy5@&*3$ASb?7{W-s+>R4uXw-lnda1(Y25kIL{r}T zg9Z<3HYM>?*tgAUS-zToWyl{_3f8ax)!#3+f12>w*UtxzY*!_%Y3DNw@-57Dq+qW3 z{azW9Kb!ab_qC-iPR=^$z+Ww&WIms{*_;dir&P25X995mu=d@|c?rwgZN;rj-m?*5 zhbU$2rY#*O@A49}mok(k@9*1n(rg@3lJi}@Sy31B(o!VvFKfyB%QUxeW8+-z{)2w_ z{``EN&+`vF2kgiF?xrvS^G7a-y!M&I5uP@9uC!`9XGPU+)9wbSIA1;9K47OPK?7_A zvHkczR@EpTMWt!7Niw;eMe4HK4Be6FK@_O!%imz*OyA#ZJa|;&XG4sxHTx_czZYbt z6!@~~HkwJ?gnW4Dxe?)f>oY*A6T`5*83J)(z;R~szBK(WMq|%p7{Gzh3Hk@!k6#uh zXY$y{n(PR_N{u|{MLyC|>UxTstk62sz6n0S8jXb_IG7aG>&6vGUYP{9|KL+?oye3P{!Ig-_s_q#ryX`9F?DM|)8>yGr6sGtJR`$Rp znbo*ed&#$^Sel(a2xHmF$@s1QXXq4jR>!1fG3qD|&m50RI?Xk~Yk6p=^N1ClJl^IF6`8EwuMIvxJ zz*W#g_-^B53`#yuh8Y~Yr!;yjK5h>+J6TT;ML(41EM#6)pNYEf_Fl5zFQdtEb)iZv%vpKpzuL)NEs zWE14`2zN*sCwLHiDA`tFZCyQLL7l+=JS?nVhV!^-|84Rob+1Hq04Iiq;s7{*&|d13 z)N9$Iw!*XdmUUm_!SeBPMt)$h&-F5%hQLX+#00DVYjt{7VwS^kj~wTVrh`p4^a#_b@s_8o7su+f%`aSO7Hz-guIgum z3=~_K*k_h9M?pJ^ z`*#ftyJc&PS?XROIhec>3=aSskBin;dmQMvVq$?)bI z>GI6qIgJ0+2%0reSeK{du{olvSJb%&j>Z%9KdJXil5%^--<-ZZqz~@*l#)Jc&likA z`Cat%1G|6%QnA%Y+7k>bLHACx=L#fi%{j8mxT!p_sY6|d6+M@JHfWT7tvbwhyXy2$ ziuCEip5KtiGdVs0dT2POZ!K$w?qo3F_u|rXWpr)tGLr?dw*&H>#3IgUh zCDLaDXJ9zPf~L3jNNxa&>9X(&HMbeUK0^Gah{nIFDea}KL*Hv@K5|@ZDTJMiy+K(N SWuZM3!Xl@aAufqZO8)>eFBACy literal 0 HcmV?d00001 diff --git a/tests/test_metal_iq2_live_index b/tests/test_metal_iq2_live_index new file mode 100755 index 0000000000000000000000000000000000000000..6306bbeaa8cc8f688cb5d3f6e1569063240eab12 GIT binary patch literal 901624 zcmd3P3s_V~mVe!Q8@S!g`ytS1H%K%zh{mW9qR0g_Dkw%^l1VZ%%|i{)hyl$6A4nr4 zQCl;%apxhk18N?0YbP3UB9ml6O&okgqh@AyXXd{GY67$`@=_8_|9|Ipw;~#6_WOUk z`|t0&)Z^AYb*k#rsZ-~iDo%cT>3d(sIEEjOXBeKV$as-8A;wrFo>_PdhR4%p&Uku8 z&OFM!^Cy3NXVzUQf+H#jPTq_k=iT{Q{@$d%Gt-^+LM}Z(N&XoO%NABGb0-iCg(vyB z8uR`hrCf;8)UI1U1vA}>BVsTtUG{w8(py2ruN;^*V!N0rV zJzxGpN%;apN%^8BC^)pe#W^l`v)uhh>7g(N!#5Zf78@2-Qd8yMP;!gZ_*SYX(_Ouh}!SFnAJR#sM8ay$Ew)e*`y5Mz2xW1tD z{o&oFV`2nD;U)df1b6@yHyxKOE_ik!k;+hbkGtWy$%Ik*{&eEvd(S_& zu(ERDqNPLO{n!o9g9N3A;CBl*in#GBsH`klzMx=P!BBX`Zg@#KBvW1Js zrjG?)C55i9>AQS9p6?W1+d)cWxN)w+-WTvkwaFVJb@^f?GTY zt_$y9-VbiZlFEhm$31P_on_qh+}W?^k$Y#q(h4*-ZE&wLHX5a9+fLFg zL;8(yrvu;cV?9J1|K*=4r(H1o`Nop>tk(^k^_wSo>sRvx>&=x4>rH{PIO~+BI=t8> zQ7;XMoHWo}7j=CyGv!3%iBUA=qPHqX+Wo76`dT*>)@pU zJ<4{z$y-$@)3=keX5pK7eA}g9pTuksEPZb?Yp=itIt9`8%QAjT_z?K9NCSS(3cx40 z?L*-9Iyq}5VD_omz+kw&3Cv0`YuzwG^0#pIz+4EJfH#rw>~!*0!j)hVO#L7ndv`Lc z(~aX|H@rNQ>xi{PMfDVCs)9gy{&;CzT%fa{=Ep1U|w3@(i6nt2ux~w@0<7U%du{lJqos4 z{LNGg^OEdMFHg0Lr_!#AXS}XA9pX+sgEC(V%+~qv^~s$=h>e@ylrsf+`46*67=zQ4 zE<{~~$9cT>LS7!;e}!)_zI!^yv%I4T7#ANlu~P_@zt8y{+ae)N?O6R|wuRvS40$HM zrkvwXA+2DWSlKko*9i9wOSN(n~bs669+i;cRk#wPWg1z%vSrRN-Y#LcWTx@6Wb;=Q{ayIC`{9N8* zCbh-Wzt0cq*GhN4=Ad8qdHQcqzdnY3>h$#ApnjE*AMME>RQ^#ca-~ ziw|q*ZIlL%c(azi9n!#hZ?>=RO=-a3$NFQvSzg~xX`s)GwR9TVQ~EIWX+faJ4cp4O zA4iwR#1;56Jlq4m<zGGB64+#!9;KjL^m}nP+wwK$A`3L-{PE{( z%fEQWqUm;-b137XS*%1Jn%#{h-*P~3jddE@PWAjJ>LFMyfJI}^@a#d{`Ko6uM2v-x zy>l|rvtaSRH5QNAyHjZ_9=G;|U@XEh7Eu@r9mXN{AB=_d?y+!|Vl0-@ItE(B_Z^H) z>;bH`M8kkhbj=3S?Rm~p>!w?OH{DWuY-o?O$b-8Va97wnbwo#;Hc#tnZJvEP7}`fPM&9brB=+dNf^u4VhS@{ZMiki7VwsBZhX7( zou{v#KwozfKLPLh4cZ~c!P|c6iGy#xia3qSQW`6)(?0(yhdHrNvMsCDvMr6?Y>$bt zJP$v<4ERf#i@&c$ocMkXU{KrJcvkTx=A67nEEFug;4Av&f+dbyyOsEg%xB)`Ee-gt zx$c?kRxfr{5BWmRNdrW$1)xvj?~pw#H&;%FJdy#qBpvcen)Sw#kD5tdk=rmP#o~oJ zshy$BwD}{aGhEJJn_{7{Zby0!!f&4Z05CYnG`#g@9ka-^jWwsun^mYWH@*@$TPoy= zov8Oqfw#@7(gy3iz~hi+pygi^EYQr_)%zNHg^+l$TGYj6Xm=XqCe9*{8P?mMG!f`84KP0S&rTodlr!VOQ`EPF|6qEt3YzQ9zN|%Q zYTDe_u;?8P?}dR*kq9;sruQ4B}s5ygJdxUt`?4hw5@pmhfBN z^=7A>{;Yq8$ntt2b2xzOHz<>dc47k*n(f6beS*L3EEn!rXl@^t=SZ}I1#~{kZeBwF z+bztJ&9$FCTB|aT#aJw1?WeB_k+$*faj|d17;ypSKyTJ>0GOSAaJncZH?2$67fIaw8pe$;8gS+bp=^FlARC_^m$9Pfx-;)A#~BZz{awdc z>t*5zJTvkbD*@wA@$DFk&XcUQA9UCa8a`4nA~(KrL~gG!vbAqXWUG-=s!NndG`&kB zTg{wGU8+2)>3bovHPwezfM4gS(J$D`g(mp0Qy0;;4F{OzlAyM|@}plirerVuhBVG%SF-*CprJ8>`k0@WDd#f2Hzp@G7*8?F;^b<* z9p9h8cZ*w7^+xo`1~1m12D&8L*@p4`4dkO9e52`NmRS$hl!(>(26kb^nKblcFvd)J znZMqC)!CBl!}>XVS9R4nF&7WbgFeg$F|ID>I=&YL(cI(11~h`&vR1_`#LtHdf@K!q zV0#)WVxH(QM@Bx#@@8|Qc?{-ABjyO?7|#w z3=%59ukt8et7UnHNLF#E*O@l~bHg|c;~vDuXCQ6-w&qM(BnxWfRK|`TXWqpqskIUF zz=*gX#_ORlDS2$9lzatk?~jyPQ~5w+XCTY7qn%gK&yAd)gW5J3^str>F{{wt*?uh8 zP|Yev{w8gO@iAs8v^@1~&jn{*7x+%}@b;UXt%0AxETed~CBNG_k;)Z)=ge!txHLYG{#2nq77G>JFqTK}4u-M3+c-;`Qz8E} zcRKS7DXd~G=F(Dc4*ebJ5Ru1%q#OFKoJZApe@JB$kqhVm!l@KRX;*XPQdq$I>Z5^_vOe|5$jB^K(di&?%m-#(CY&k=_a$L za%>>v+)|uauGv?)6FUCnfiO@0_T|d`iU{WWs!+ zx%dFSmFXOHkBzyL&f5WdD4kb-e|Hvh1V7~k+= zjhfC7sr9j&&R_MQehG3R$&Q!8rB*5v0~w?ZVL|}QbD&%e;zob$4}qUm`0;g{@T>hK z;}HGmL2tSPiH=e~@CYgg-sX#Piy_~QEh$Sf0#9|dy5X&oge0sn7DtGwmBz9Wa*G;e zsm{wZpa0F-0-5~uKD4=M#j@*Gv}+66Q^SWkYEjN{(-}^;KP-aAi*0oX*N3(&R;4Vn7aSNTPKp8J&kb>BZ~zW&=h{l3+HQE&SH zQ8TGzanj*cV$@;ZPc?sDEk=Fn|EXqFsCU$iBMXyW8W9?`=*XfZVUiH_>?-Nib05Z< z>zH7Eo$>PzbDSZlQI_QVSJ3z0EyBKr>fl0*+unBO^(=j= zH4HS|^X3fwAJ88?FFe&6PU+Q}D57T%E#JU*(Qi3hXrDpza0l?~c{5YLpT2>dBUVS; zD%+2;M6*3_&eONvdLMb~eJ|eA-lK+R{fY?b+XLWD<3O|6`<;%!9Eik2bVT!{3-Su_ zi*t}&UR8i+D6UVw*L+!V!}+Urfwf$d>rL0EqHkiro8`V#V$T40nKja@$9bvM0hlyL z^cVvp=1A&+I7cbT7x;!~lxXo1=%~D;IB5gs$e2mK4X>2MCDEL^h%wV~zK#Tx>%siE zf-$3g^m(+2`nX#T!)%=hN4e*P89clfv`#oBAU@2EOCMyf@9-^bVxfP=Jn6%Fuo5!R z*@JP8KCIbPJ`42>NBM7oPtVRwy;waw$}!<7z36vLbLqW#kQIkV?L<7^xGE{LdRSDZ z-*L^y2xrAaL=_w@PHIM4OpJ`$j(A5U#>q=mXB>(%@A76_e(S?5y=YIPpmZcaHm3Ic zr@KA7-R*hH-JV@?7;d|HDV^vd#I_T#FjhGj+nl4=^O*~kmNAHf z4+Q&jrkrunrksb6R?%2!pt&NAMp z!hC+?=xFoj=!c&dFHHI+8y5fP>1_Y!3f52j5_A{zx`J6A1N=7T?Z^P^|3JSd0=HDu zI~cD5$b9I;P=f!+s?}#k=dajo@>OI%SXRCH_xvz3(T)e+lQKNW4A}o91D8~RH`=~- z=DonY%-^D&Nj0lq{Tm-)-j7EoXdK-SKlAOkz%dqcISF&)8ZRYx`E&jx|B*~fvMe`z zTQ)y@d$uluRcy!DysL^ZUl)9A-QKmkB4V+RL3=uZ`vAWE4U3GAoXPe-jeQ5T^RLK% zK}qGkQT|bTxAzo0Q}LwWnT97755vRRuki{?guu!BqT~238sk=j&{-#(y0JvWo>Gau zr3!mYHTIgm&@cEwzaUtM&xpQKtMY-moIvOcX#HlhUP{qbMYbx7pK3iahfUNoQ>zO5 ztVia?HynK{LtplYq%WWLq<(#}Dk>8+k?D6rvol2AF>7`q z{ubiL_iWI#5q~Jd{)6T*LVt`&X|)hy#&hyvQ)?Now$Oe~KR2;~bQx4`_D7AH-Zq7` z2JsS#Ls!4H!ZM4!v+q zPGoD%8@>ZID(ELA z*1D&!mtdcu8KE>6S3cEhg-$@9Xlm`7EDfl!Uzw;+Xqbh4Y4OjWYK@HT&^%xI6m(Wk zwRXfpw<$zs^~_|gCFM`GmV#a>U&pbE4cO<&<&fXTi^e%fi)ceWBe_m!RC?*v5?XwUwbijnIu4sc#7e`Wku?c|KM>5!u=gnd4sN9Oy67 zyE#!d9djb6bP2(h?I>5=^|f&U@?P??;Agu8I;J`u2rba{UPhU6;M7C;7%Pg83my^d zge;#Ay+jxCsSQ}0vIsZxsAF3pw*_yAIjS*oV^l=5rK65&4CuqY@C}+wPOUbg-!o&5 zX&gsOk}_8&R80bpp+4P+`TrqsO~pQti)VTJ(peDoA-%Ihmi_?mh=)3kJ}-Z_&l~T< z*rYaIX*`ZS<>2wyjPd9Uy*^pQcqlO*68bU~G@2HQzQ;JUA)W<3S_0gmAIY&1p1@x} z>gd)~PWqyErqmkG{YFD{K=9|H?_NUvI^fnd0`!Y-MCiN|5PHfShnzbGYp{K_^eVN9 z_}vvQGHc$-u~ltYhe?it|7zal(#1*hR$i{U0vap?E(XYzYT)-AaG9@Q^DBTy;OJ9Z zpZnhDk%xo;)kEJqCppo7cC34Vr@n|jCz?zI{#fVd_aa>ZdMW@tbz)5;I02};2sAYk zyt@!%wxGBmsqu!;4}CK90laNfbRf$44)D<|s1Wt(hO_bXtqNnB%7PrRpc|sA^PsD* z0pEZ;qMrrDg#<^YpL93uRGydVr%MR3eTlLJzaL>e=qC+j3Q#}E=7^u7_?o~G!3GKX z09wb9UkcpIfV%_qN8>@bmjm~W=&!{ne-iS)^QH^;izw3@qeMC1fyI~)i-CI$_#S$sV=wLm#UvcB!hCIUkB;;wpk#VQ7AlL@C9_6B-l5r>ag!?am`!19r-b1(}ev0Ck z+71Aky3>*E0dIfIY0H>m1C-i_LgA0DllGGv7X7_qMibb7j({+ zeDu-C;)*1Lo7PBYnH}wqa=rs>Bn!$XTGO)e^ld588Xs_Lyt>`vrDFl|c#SO1CphwW zG0+jgM&HS_)(;*5TyJzj{PqPPTF(+#3ECHQ>_txFQChy_n6rTGW` zd=&f6GPG+k+GRw45Kmr$Hob-ZBc68^w4-zLM2sir=75dGvTr@W(FNz4(=Y)-F=<3#lXC_}P5^?$EU8aNAC zqier2FAQ_FbP;F-bJP)o^`h`8d2eJ!9I|_k5pvW{$Uw$I=p>@pfN?vs?1Vg}hK#p@ zSDUAx%tgp04uOw%gg|!&IBFeuKsb}90;;1GM+-3Sa{y`z2Lb_hbJ(poE+_ zELPv}_ctVYuex?;+KN8#E9xiWFI_v*pcjPw6m^i!m)6D(4Q$7`td;g@<{Ip~v8Rly zM)_B!0eS(`)T}UtxVcq0=YuOn6CZnq^fJ+MNWRWLjs&ngQbBuAzU2> zeHY1t9awkEFz*{N7G{#uKtt=mZ%aT!;;18AwV z-YPLBoyFQlvL1|wwj=JfO$1*lCSFM}pg)JqC-aq+;47EGSMF7g=G!d5r!|oH%3D}B z0bAxP8{G97z;n{D-*I5QBUz8?q_({RzG6lhqnocFev0D9vEDTf;wvTYw)EjU>Ni@e z@6{fXGYED^jEHi+19Q-xGUAh1tLfYI;Jrkbv{rlCL~=$W@_q>S)LXbKW!_KliT4l2 zo!081xO>)W>VMcAv#4L=Ms;ijtq1F|_Y}bcR9OE?V~=Wdkga>)T&E$LDgms)>+Z#O z)ASoSWmX?(nrM}HK_BKZ(d*R-(tx=Od=|Wd_z2{SAjlm?Gi(^N4pScQ3WIO1oQr+o z9PAIXuusgyesMNzkh84VKg#xMk37*C()cv>$^C5BPC-1?>;<1VkDnxX5A88MdrQ>< z7wX!N(5q79;ffjs!L^)fap5KxUZ| z$}E#3m}SCnW&zK#IKUsWs4RFMMoyIiqPVfYhnH^Wt8)QCIhvDQm!;2gkOl^Aa-cS$WF4c;}+MK^sYLo~AKl%I~ilk}{6bd+74q@7g@?U^^Gw@6Nj4IX&-u zrQ~xmTvMwK^_D`{MEhD#9V+no*iJ)Ito!>Ke76(wWg|lQ9c=S;?sskOcV|8CV52W` zzjM0ZQ5~7kO~nS{9qMhvJnT@Rov>A6UgRCgbdCQ`>^nOibB%u%;&+Wd%Kgyz9}W3| z@dv!2;~#@@E2Qxsb9?+J()h2LrSDPSIsP#ie?7)C2;(1*Cjm4t1~iZe8W;;2cpI?B z@d37nKm+4J1Lr^k>AoyV!Z!?K7PE?v;vzY9H)ucr4Se2^Vf`L*0MP=&9F74kq=OcU zKno_&!iV5_AEBM~tGuG>0izx;bWGu3doouzu&z-&e^2JO$3O=ZHzB^0^xcRzAr5{m z$J;4Rek;iDq&Q?%inDhRht8Yg(TMk9?6ZK&bkN#|9WhBiRDV8TtOpHD2VI=MPdO9H z*#L7oX!FBkKMb!4<&L7g(?LrYo|Fb&yARBEDhpfWbkOvL-20d9q_Uv3>7c_4@NKyt zTB^%Tiyh{{m$}!xv_{%98!?NaY~fEx!-#qTGL(b^&iF?WTd=_6~!7 zY}jqueG#;LWX;_Fe%g&nw|*b+4`>(d{9mQrd(@u~nz)O0?@`W#a(B`0J<2r^jd^Gn z{+qXG>>g#?sjP=~Q}184lgfH%7d~3|gL?< z$zqTTXkATcbLJg&%V7?K)QYt~$N@PAGPKe0fiur7hvm3^#dZ(&0p-B2AXDAfPmK0d z?r%2@{#L?zeZs(RiG%%hHl9A%h2z1CRnhC!YS@QizmB>H8?ipoH%b}3F6$U<#Uk>= z=(Xz1YMaUdx!wSIJrlZo=$@1H)mBw1(t4yzV3!rEn^iibMWl;imvx<=`C)%XXPx({ zZpc}X4{x8NfqdABeJK1qWtk-bJ~`xvN_M`!5NRN*0Q+C~Lc$Nk()%DAm>mf}TAY)i zJ>Yj4yhQ}x#a?beRKhk1xrXw|9!PdTTF?Gttg>N3LWbS|xuzKYp9zo;$WE#T{I~#9 zYcJ;FqsxE4zZdqyNh3t_tfjEuVh`8{Ig()ajmxlNye!wT=j$0KwMGb{8sp|5T{G0O4Z881pz^I%_pZdG8EX%&lFvU1G+=T;`}&tExq|1QW;Wh=+;HwLl( z0>~zv=({fT>rC*$o-eA;=+P&=UsUhSMA(PWh|qy>1Hy|4?FcU+Jb~~s!Yc@`AiVmZ zG|(?Z*cjGjYX5j%qYfMi`3!4u6V?SCr*W*u-kdvaNLsVv=hxFZ#wVpB?-=%D283Cl z?;1)YY@{^SFwjq(BCE-@J@w(=Drw|4_d52{(5}F z!L~;A%H@4oawCPqpUyPGw2y8?-J?(&$!F?nr!g|r}KMf3;7tt;Ja?*ox?ZR@r{m! zIu4*61mo{WlUe%)R;o{rZD`==-T1=mU=*ddtuLNk4S>Ti*6VpZ;he z#);~teZvp=p}&W^qW?)h^g@(v8sdk(N5=Js{m@^;d-6jc>~r6a`nI`gtpoHYKJSA) zABV9PZHrN!`rbodH2$Ek(>R}Z8s}eR-xZ>xJh|T6eMD;m+6rBR%XgpBPWU?ASi(c! zAVBY+fd0V?dI)dWr+whR&%rNWuzd~N&6lW~bOnRgjUKEcaY5@(cl0)Y(VTYRi&+_6 zU-V#~8xB51zAr?(v|l9JEdVYZqb0qlgn!}$wm+^qA{Ty&$#Kv%k&s(>q4CTZ8QQiOR$}!AuXZL2arc+_PW7S(~%b~ zz~3O0CAoZg!fh$Ylj~{^wPfHss;e9RzRob%7ZC>G{l9OQ^bzn$ID$QIGIX4ieOORf z&19Tmz_|mwr?Jz)7u$iktU>#}!#?}(@R4~0-~5X7lwatQY7aeX?t~v=H}?9`_~sfP zWqw+SR7be|qC1?8InA&&_ttSZYq04HvHD1ZcyrzvQ7M`!AA5F-FTRuDzei&SZ`mBI zr^|nY^i!zwTYP^Fbn+c=s)0X6)r75EYciz%o(CH>NBBr{9LJKAydYyZXPt>tvg95i z$rk5_^vzjklF=t|K`c2sjC@%L#|YcCpYmIh!9cD}}}ZF13Sbs-rY{KfQkk@8z-C;$w*o@r=zd+pp!Z8D_J2-hV>#Ddg6?*@X9i zo6`eX#0uF+ufKf;h}MmpD`#R}Jcju(1M_4$=1T_jbLluw!pr)(9|7)Ao*E6B%?59@gdsi!dQFweWcUF(}&p#s2uPe%F|u*`A$B6U)|uLkW(Zx z4W7r%kpH5SSqs*#&AmGoy%Wbovkp4fx+K^f8TtWp{0Qc_6?1$x<~Y5JMxV%Vp#L7) zU#7|MDIk15_qlyORPeQT3hr}`0%R@0b%X4xuvwl+9+-P)YkwYJJmA3rM$H=&Xc z3O|ep^k=y3ao|Jra#hgSh#otA=32jnzyEjW(=XBI-{4%^yMy^Q+8RoYeXP=q`bU9W%nuIbT2h+Z|&W3;T_r>z$lBv&M^!=OS2MmKXe-xgaBC z5_eoMc0wmd`M&GdHaU;{5-D$tb#2b^b*^#}{B1HZ zN8&Gxo7{|f+8fFyb>MM6g7XM?Ckj$)2yiIod9^M`8h;LZNYO`X9Y%cKotJ^Uj@2Pi zN)zO5?C(2*Q6@o3UK`9NiQ}Z?Srkr?l0}4?Se*5F5a-a;tYR_rhcyAX4+3`inke{Q zkR2_ARUE)MnpxlrWP>jWWD`k_7rDq|bbja@bok#KVy)lEyn=X!2K+qImMnyuufoRB zDFlLVN80xNgx~Tl`1q55cTRlt@6LS-0ju;!|B{pWqkWZqI6vk1l06LjgRvMo{@H%; z(e#6F8P37!ZmKIf&`;QlVl8#?D#Kuudksv!BL!%ZIIWg4Aue}-mmeF+IRr8>C}LJ5Yqc+-0wHx zJ=Vz6v{sQ_lEO~tY3ZD}3h~+GD~l%_HlwSM{it8M0Heo%djxp4PcDyf_$~T^_LS>} zoDV+GfqgvYvGY*KefO2uU|S_SB*7zm$#;zEf*dc?ALLG%9=BniOSH5W_~X8!(C@Hc zl*bTrlk$s^PqrXI`VS4_2%OEi znQ^9MwY7oj598Ktg|QuH=Z3}iz%DnkMudzSXuF2T$}z~PSQ};8B;584`rzm7&X#Xr zwwa+G-&m_2zJgx~`uNVBv#d&{G3%lKfqnH&4!uD z@4XCp5wJac%xjL(nw-h`nNh}aY|)C%-U1@m~TDK z*M7(>ui(2dL1Uhce(rr1Whg9S1J_}j>=hL8y++(^@hczeS1;f@*e_`gN)HTgApKeb z`ty6}JiC82(@K5$)eBN9^+hN2@;#s_@^6n2BFz`yn{C~|MVc>xZtVz9;MoWHy!Tm> zQF(-@3uj*CIO^_&4S;;&^WkUKi+bW~hBr|Ebb$sFLZu||^@@w{DXo{lyDniK)?jZ% zzNt0XQ{@wX#2zc{0r>exu!(AU{D#>s2};{#@FtS0Ad}?;;Su-%^&%FOb%+HRpUta# z`Vb2=79l(gm(;^~FFjquyp0d7*`#qI?B0~I;uOv!2CVso<_dh*PlmB1(&@n_E}su= z0^hHJKIb6jcL4R_4QC70u^ZofhIwr7+C625Z1Y z&{pr0QfqG@n|uXrzYIFP1UV0SL(CPfe>>U<8aoYLYUr>a_*P>tL~ZmHv}P6ZHp2H} zES>semy$j4eB;|YBi`TCG8{)j(o`Mp1zIJmyEBUy!jxgEz1 zGdp6Xfg1S2q@%BU;2%SNG2P&GR4*_F%SGfgt5pRvSUR-Z)k<}9ytG@;$8=-+PO4PV@xdzIk; zF9UGS@?6D1tWhX;+Ke>a8{vSCk$KhFc|Xbiz`v}vkN(#hyQw}pds}@>_QCp*+2`tG zaV{|x@7L_oa~#g4aQ)<8ME%~2Z)wl)&$Klk{X*p(dnGIMku46`7@JB9rB z9^wPdLC~FAx^ziF(3M*HbxE+#CvU>OyfF}XVXe4|bvpq5+ixR}^m=i%@Q;`W-va2o z^D$n97%xRwjoo|1!CgbgD<9)ki1FHhu_7OO(&6j4VgFP=MnkWR^Ly}nP}{Bw(!kX| zXAA9}nX&9fOi7tMR?Il&#s-X)?v{^^X!~d37vRiu6WT*#&q0{zEjTu)ji{3f9GdcOfmp#;S}_#`mrKMNZ!b?^fzzb6Fp_CMQ$eRtf%Z zEOG1oNC(!z;wICYsl=G2GNpsZ_XDUqHIDV4L%(zkdRK(>l_|epd}#u=*pa|5?u``| z-&mr6KEMll0dMFB!0YFU(4Q%BzJte|Wy4PGJ;$Hw^if*%OgyF!Vy8sxuZu@BYj-ZK zqr7bg+ElZevqq<|7TW7VF3Gu8O!3dbXDJlg^cVv*!TTG0deULDCl- zoqNHPuTx~d=%>i;jy0uhuiu^>!2e?Fb6mi&om`-KN4+)swR&6j>-9UcH~0mr=VI>J z)0pM5FxGYg^UfRJ```)06O1PaPY9kZ^}Dk7;rTh9C-LmRP-E8!e%VoifA(-8ARG3i z>=eYOA)bo-WTYczZZH4s%-73T&U~%>>6tsqC(nGh{F|A-FaOibedR4PTgo5Py^&p@ z+m>z8HD&wg-pt;teJ6Y318m-I?e6ST+CAA{YWD)qmh5^wMR@F(m+1&k)UrLbkmcxn zccJ#pY=*PlvD!DX+3+3NF*v8ha9)X%SUEy}Kb*0D>t_C$_c;If?%4K}xmW|_H8bY8 z<~=Sdek1(nsLyTaPx4XM!MCXk^!X$Bry8Myro7@BE_Xfr7#p$17emH0z9U&1QO;az z-dd=~xd)6J`2-T*I{}%i1AdL!fQvf{WVm$C2=;a|+*H6V16;~eSM#~JV`To$DP#1a z7YpuzoJu;`_R(15+L-54K7>;fprW z`ZWwVs!~|68hYhjX>2~7d&dPdd5|kYn=lW%P|n#E^L=E*z;lDfxpxF zyPUTY%Jk2nuf`!>fw~Rwd0&iv?n2v1E^p)l9cyYgZ9NB>it-LZ26>>=)sOYjhcvZ} zi(e4Uf@*Mvg2v~Zps*R3s~;Wsz7OenLn5L#doh*QdPk5>Nec{L9&C0 zctVvf*N$_3bS}r427M@KhrXwAcq;*Y&#ALS=uyRIimPNhFuqHL?6*e4@}@$56pNFS z287|Z)B)$jefUntDJpO=eV*=lMMZHW>hMy`KZ1F$kBP86@>DU-gR@hq^VuG%f7^=U z{nTgOXL!rrK-}YiF{b;ThH=Bxa$lk!yZ7>z#$enrNB38~ZEDp$k25J~?^^U*C&^;y zTatO@Iu1lxrY34ITh~L`MUl zi@%{BiqqLhN+0EwS=1+4f_RMjg>a?!WhKj#d@)YA?}YX^6$OBG65r4q_R#X;KS)+O z+x2nn#;x0-yI2C6q%#8Kn;tmwxJJ#Z<8kp{P$TAqiys8q+Q4&(AIwKzl*{}e(6$)! zVgu%deZm-hF(+omu{ag>kQLp6Dt@ZKo7V`lzioyLat*Tnb?B{W-qKj7VIMMpvph7P zXWd!e#m>vH2<9+bKlZZJKcxZ-CK_IQ%sElE(;@E$-tlM`y(c)HHLD+Ei#aE^rEhE{&Gf%H#JZWAYfK6mAr#6cR4ExD$ zu?{jWt@jCd#=u5$5jK)U*hns;k4{22XygLaOR9G@T!D?mVfTwF#- z_CvPpXQ)5G4@eL2D8&I|sBMDmF$Aac9=7Zm$RBLW)>Apz{t;^XWWZzh2(wL-^Y6BM zP}#}!4sCR1aHp;t?!GTJ-;cG(KF=lwmSjq)#a$F=)8^Tbzi1Dp#D2^W&HC$PUDUnp z99`ItP#;~xT=)hw?y|x1wl{%ew3@Yqitvf`mGlbu&X9e_qZb*4{=_{Bw`{S#?)Iq= zM&sS9BAav$@+|PmnJluF^Q33Ny^I|gAMq)rWfI1c-od`u@_;A*f{$e!@(m(vCN7-i zJHNI8$2iElbhlR;;$&N-xqA)x(ph%GnQXXp2JfSb&R>x|(B&&I40&O;`Ox!Rz&Fps z{#fWQ>2U`|D~%n!djxH8*_p#^e|v()D8%+32yvi$BDG`aop=}v*&ar?7s0MI9gp0m zFtvO?(?PT^{nJp`-UPr`*p{I`sO%5HdD7`@c?1vj74?w}>rbME&M8!ZuAan0Jmfsy z%kTP?7Sb0G%&MDskLQl}%T*S-Bfxox&SfhxcU73fYRqL{%xRp-Ug?i}`2@>1f){w7 zLLT?)=r=gMVt>wTYrSBL*F#?5pdYN$TyMiZY*qUj%`@$Fn(yG>y|(7P2Jp85aiHeR zme}{8OAEE_M1Mt(V6s2Q3(yn%7*CH75#I})Sf3CcZ@?ZC=g8%~<(TT`20z@B=8ySw z1vaEB*fU;*&A%VA9NCt;gt76Qz^O^Ua4^WMDGI99oYmrcZjC0EuhX1`4)i?tismEO z$j=UA6$`j}#9x&2)7$GcMeT?OQyIL+Js2_U70r2+ISbut3}36E^7KBfy;f6*^0hwO z8)6ikc^l7}dF-)9!mno}e10IiB-7dhA8<2#!ROPRXEBgXNIp%*nn|)-7yJb$gI;OR zJVwndiD;WY+6Mg&?oCUGzsd)y`_VSsA&YkT!cI3Xo)>tzZRgOo1wm`%wh3sLf&nkI z&5Qqq<}-KO7NT9BaWBdF)UNt=#DTjP?`m5*`-R4fM|;sWo~_YPd3s-n@v1}lHR>G= z=?dPwo#)NuCl1|5a_m+3as{#~lDFW~Y-TtEg7)2N+Xn18JZ)Q5&h*cF!GPCz&Fw3xQ%yMH2=mB)tgL4c@?E8D+&vtDBvqZwLY%B8V3<%lEpB5sHtpG39 zLoOg4P15+STifL`wutCc_&^NMS>Z0Ds*D)(@T_-v^}v9%-C-EE+g%{t@^J z%9HI|GYIC_@QI_lqf|b(%f5`V&Hn9w?U{nTv-=xAlI0&vmwik0WH%bk>w6x|7?c)} zo<1n8KzhcYw3j{_TA6+A@dbx!~%(nAzN zW*|7U?$1JK6@QVV|9Dp=>4prr`z;!B9b_)E3VH?u$2poYhPdJ1ydL@|^6`v8dOhwr z5OH@wR+=!sMvrrv$TKHORYu(Fl?eR->1qpO=6>4&-H+i#@O#J;HMrwgK)5kR+HdES znXmZa4oB=6?Khm0%4686v(S5IU~b}la@#0QwmGy-;wrX5M%aiuu_&HK@i5ojk8M`i zQ!sZo`bn+3(VvNE3+Wf-viQCWK6g}>%GDxH`=w1NdjfkR0qK*tAMg0Qv~&))T<>EaYdJ8(2I;hr;Rn(glZLkc>?* z&fO9*;Yd)>B%qLu^gRC)wyvD5pjr&}*3JTgALYD=s9F2c4A?_NW^3H>(MCl5NsTle7p8f9dC z)f#5c=S?Yted_Rk(psDuBwzS|SBT&lO7IR9c!(PE1@3+k1dA@vb!RhU%wC{mEy;kL zjBql>_L|5h9>kNRkXnC@um*SXWMEGg*Hf2M!$e~@Y&6ObO*ykxSsO}LDWgU!;wVE6`?$MOqetr@cVSi3QpOR-!d7z8euyuwSCyn2tS-W~~1IeVOk$W3z(TDqUF(1%}w7wIQ%<)yn;dh z;gU05vd1~>t{L^%=Trmh>rCFsIwQ3{%;^$E}Lm3DC7!VA-~8K3QxO2MYb#S%5jC>PaH_u z1UW9~yH|4dTzj{Y^bX*;@wC3{fe+SPv&co68S}^fl5<3ZzEfdODMX)wx1BzRzNWI! z3tC<_vANHF$Zn*=eqAH*4u*BdXLwV#frUAEtVdMlIoyZ6$t2C43B9n-iJ0s!z>^vw zvlm0oAYb2Tz-ml~Jb*O*C&}p`q@l+FkMy%G3Qe&u3{15zh?r(y6v`0i5$EhhOt7TG z=Z@Aq0~h6xux&RZ7RaK11kF-@GV}=<(6!&&(Y6*r&6r8HiX|SJhK*g z6puiBV0A>4IA={xbWL+Z0cbKh2K-(P+0Ll2UVDbO;+)87@m1*4bgVTFW$=Gx2ELl- zqpzui4iWbpG$Irbdd8f}_od=YE#{(!e`I1#5-+l0zt%U!^^WqgkarrcxCi)g z&f*`2JDqFZYA6MrJPtb96X<7YdOXrno6lLcZyIIU@yU2g{d1h<^}uNsZ`9+12fCZn z6EL6XtPJ@QhJYr&pA^z?!z-kLc-cTuNCWwPgkD>HW<6x`Gi;c7#R~t1GoT~s(9`dy z)jjjR^y1V+DdQ*arwKnrJS9;&^i$;j4Do5mHy~g7KzcDXQ3{YgNQVEQ<4qnO)5Cv=C^s0RKJ&imrQ}2@4RvxqMf}l3 z=>Y1i{Tbp@kdHb!=>yYCz{?M?BPkBNMj<|);=n5f@!1qlP2AE=_g>$wQzb^o?Oayr zs?&7v>Gx~bP&<*zm}(VKALDc_*3LlJEtJufP8Ah)I^iN^g*qL;wg!y(TG1x z@o9;sX^3Z1JXLPz7T8BU?acgNsI$~l=c50)cDA5Sc{~^TH$)T#TVy)=pBPin@*{ZO z?e&?AyDjDa@w2kWXY%LZUx#sa-^tXDZJ*0`16pYRV#8jF^z#Qk!#)-E{5d)Dy`2L( z=&a?Ne7as>$Ga`XKyy^tNY zWBqdJq(s{nxbxD(OK5I8%2#c^2>+qQ;0bo{gv(ev=EX9LynbZ5)&*{wE9BE%q43y& zl!vh9Sun@tH3Dm&2z;R1hrYzmoGfr=J=O`43pX3Ec1gG&*y-lu#McvKJ{}1^&cK(| zw)vYR`-_Zazr?ugml>a3&xGv96^Eui%(LtXJePg=-($*O$FnHZ%RY+dvq$qnb`ai$ z;$0ZhOaB^Eo`a_zWj@EVV7S7*be3ROpd97BjCT*=xebT6KPz$e*O9j%^rc<(STDM; zf2@HoWGD88=YSL8^%o)Hk%OFIE>Z~Q1z)W`GYad4H&d8j6Q!H4rI@cC?7V{Xtsy;xaZ|{2leAVxNcr? ztPZ-FqNza*^;m=I6$-P0SD5QN4O0Po^#YUb{rao!cst@uN|EzFWbfDGehI9(vcJP` zu|7q!B5f@m2Kg%u;dVSR5rtYW$XweYi?Qg1+VouuwL9=W8u22;FThqf8uGCsFkgFk z*8;6ioU2?v;tISY+5gHE0d@cd>;Ycz5%7kOz`geZ9$NDXEYrT3v`G7TIJ^1BxP{u? zNfp}h@e8$m0vGQPyyLy4MOt6>_tO{sdcN4qJ^Q=ERh8N!Qla*e;1z#aP{f~HylV3m z_`IF;TD|!S?5!uCU%mNZFGco3^y710YO^=?*;n8LcSQ&|b_MGz?xVNlda2AIXlH2P z5^dO~=d{BDS7|4Yux3y8@-3h26fev2eOz0SNbC)ALg z{2ckXH(@m7@5AVeBan4J3zu#ljw{iA!m6~NAv}dJaP@8#?w~V1vD>22@x$c%y2v-g znpC1q!oAS{!3Q3Dx_Wm*7~luMSNA0^G5d;NXv3nLi^>nLEhsh6~{pF?Mx$`3bYVjpZzD%H*ug3O;Qe9S(~$E-%5sPKFZUrS$v zv|k55?eBu@b$Gr#lfi4_Ax%K<*E+VM7&q#r@6UIrH#s zx%T8aL(&!QLG8&e5svW9zVdqXxGVorQ0@piJny6+r69igXxu#A@whTA-8uK4IB(S` zL^(RqFN?6Jc|*pl@D^Tb%wKOe9(3;0gU4auah_ya+(An+E&TmXKLC6haX%T!v?O1J zAbk~U0t_CusVh?Kah22T(?+J+r>vwpXyeT;+Vs{oonn{X-ijFJks7DpSh%gRr3n!!HBruGZS?ZJoH4blly`5fSXbLGAFn7y%&VgAnmAD9k%V8&w4p7SofLf~z^ z!kWk73l!|q4ZuEUzeBG8KSO5*o!1kvk5ynl>xF%-H}$_9AOgl3jYq@Z|E6Xe}iz+m(^$DaL&4ok8r5Q|5_7QV{1^}%szuV0&UQH^^*?< z=>uj(_nhW152A?|LJyFJIZ3*JIJR6HgRy3aD-oytr^>zmOh7nm5(`qo7qJ+!e;j0g zJ#_zLAn%tTUypoBCnEipk_8>Hth^Y@xN0X=jIFotGpt-vUOhS zTMdsR9f~s~KUGmtR&* zuv9_1sG1O|q64(G4F7W`4)OzZRh4u%MIHSANuEfDz6obsTbfCyUmXZJ-=%X{1NtF* zmYm<~FY6rS_@Bs@MYc5o_6yP0j`4Hp9cE!%W@3CE!#K^rcuj}iAp?2`*Lkr8=*Jm& zOc4vUnVaiR$IpJ}i}ep2`6Bb1&%a1Ve{X{hWDOqL`!7V9MG*_MHnHdQOuTzdd{+Ar zo>H9W^A0T4`haf-VlB$Tx%&!?(;W0)V;Jr-gwN*&_~>j#Jvo~Bx}1Xr+MM`jwbIBU zZ6Eqz4#r`_^E#ZDVtKjvwxK#CN{ja9s`9n@BcIc%&|h*H=q<>9Xa{Umui|ck1{M8} zD;7G3uh1^W1OJrZM)(lC7AdVTq3`ATR136wQSVaKLT%29rP>K%rM3)ZH)1^YVq9{N zrtc~J9@1x^7kv*;8T^$tfHpROHh_E32GGG)jBOdld}#h}k-x!BAD4s;Tjx&R)sT(! z32Ilhf7Axh2YhDDfJ1&2^EaY=;x*^K#;?69dZ1HYiMdN-wfzfcOC27J=O^N}0Lwb$ zcLs3QZmg%j^bS?GuU>uT5A$R_Y)dEnz(0kr%R#KC+vtB3whyzk;aw+WS(;0|u)pSm zH}JNT z^2LJJt+a^mdjt{oL}fg!A$0eicoDL$LzZ(tA>GWE$QQv+m3S_|ADZ-|;;6k1>#%MM zoDz1*aJT+6(sm()(|?Bdt`0AWg7hW`Y+TsoC=^1Az#rZ>=n*>ul|DcQUt4`boFNZFP~%Ul`3-r>QSSf0@RbvwWkog z48Y0os+|fy%RL3)&jiB+IzGaS$JXLptOqv<>l+t1P0M4S3wcn-&tUIE29bGsaVnQ< zh3=95e?SDz)D%x6eT6bA4DraCuqbhKt*oz@L^KiPy0f45|7&OC9%<+(YNo)S9{qL* z_xjd6ivQ8#{{I$KhR#CtRx*nU`UW-h4QA*eHoy)d>%@ZnEdJ0nT*-;F1VYb1>*42+ z4amQY^g4$?a}Pk?z5sr67~|H|{`C9XOxVlUPfcu+e)9e{hP`}!N@7#{PmzzkeEl@! z8<3BEdVOkQldtr_wg`%|#HI%je~98-V$)Q_XHlF_tnVZ_%-4JvatP@UcA&oVuu*)3 z`b?;=7WL)VV;_(DYEfT)J@)aauNL*?*JB@#`f5>MK5#OjzFO3m51dS>uNL*?11A&e zt3`eJz^MrJ)uTS#Cok7`PC3l{=ZBS+^N>&er|QI7Zb@(AEirgL$CJv3Ikw?0Leeqf zZfcyjP@4mB7ar*u3t&U~O*YQ!^P#f+p@;UY32E5B;(zch=Ja6&ync9zppT4UtZcSes7k?t{{S8vSEGqi zfp4w_p0GD(HIHmaa(Z7+t%_n;S$~=Iz(#y48ej-)}-&!v{ zI{Iefsb2@l>m~NtZk$~@`mf#X{2_lTv`N;{cP4Pw8=%4N$qaW-vw;iX!PeES4WSsH zFTwN3u9E{;WLL|_9GQnW*_FNm|0cUw7SiOyk`Eo+EX2ts^n3Wg;GBI+BhUI5XrynW z>73eRsr7lBDR~M{FZ|8GZ{kyNRxKUtBl$4Uc?hy~X5)d7CI4>*%+X+y5pa$r$cS?} z8_-v6T$nLBi1g@c%-u-%;VF&8C(&Q{9`?`7=%oIqqxUoPdkpBFCX{4=BqSOlC`o7{N{FCL0A&mc3KohhQ7pkIh**fI zp)Ex$T)_$oiX~h{QDB9YU2w6mqJq7BcL|^pjRlPgv&5a}{W<5{$tAM;eV*SRzd!El zbM8H#e)_qm3<_B4tkO&QmT#ps1i;$~jvtv`93S6h9hCe7`{^9{&z0pIP_Vu8y@Fxq zTcD(u>_1uXGkJ>qSO!#i>YT9sC-m~#*o%ANdS7+Ry(fllySLA@+wQFkWb7z|{xY7< zUs!GTu($g-?!f56J9E#OuhF(1%@1|Vfh+r|zxAA{)&7<8ZvB`0T7{a%b=ycigPpA! zBGmU+@ol%U^Kam=e=O6BBX-L9#=@stP3zpa)u#Ji+S}KXtPO)$yXlS@Xw=((h41&% zkN=;+CoGyBvewKF`PMX*pRj0Fif_%V6pJUobC`BFk#%eyr-m3M=mwPru-d|zjtLfV_J7*h@6p0cg{{}y{r z2(;R<4O_mzR>pQ2Qm;yS{|P(tS@$DtYBoNUu^Q((FkjT({Oinio+8a8PvVDBsq|}B zl=ttyg|#gEu=!KgN-{sC&QOy039OSnn}0&RWgdC-%T4>L0Y_9KBd21UPL~fvT-VRt4XwKY_%0|f?l0%xOT8lBC65kMd9$N) zhS*7+nD6V*P(WKPYkg~&CzQ3cQonYXY87>cuKnBA>R;6SUwfy&q76I!fNs9t{tun6 zX8~2dPHq3Yx|01IzyGh>zIpt0obY1|RdJO6x?4&A|MWk&J#-bUpUVW+$a&4I8TF3c zf>LIwbGK4guHu=?a|O?tJd+rk@xQ359^2)d+j{1F(xy+^_zixa;PZ2QG;5xz>tqjA z8F@$k9kNe_s})t>YZQ5i^?HmwSf>yiGihTzcMN^QzqRh*OoAWb;c zRLy;N@GTJ&mbDlm`q~T8iTQ|4WQJe zAJ6+BUs6?JpShhLHnq6aqH=o8(aGhm25$mzlkrfX@ZaW zWqi~JAJvn;2l2mrD`S$@=zMLfLnYtgr(S$3LSMqgZKq$mF^&!5OWBJ^{#R>5XdBy$ ztgV!_He;^cxEmRl5(eiy8ME#tZZE!SGiI`iTSRy_zI_#6bS5nAfxOFl64I^eapbLm zPU^zd$Xe|S3>Q1UM#oCxNQ+S-FO~YnO-t?(bb_3rS}W7MU zMx0Cv&Bcmx%?ZtZ(?rJ$K2%a&;Loo&IWv6!CgH) z+F9on(8nIaQ_Gm=kH}iX8t?Do&tJd7*Scyi`d>pD%_aUx@Hx|C>fiV#?d4hc@w;iq zo+Nnn$d|a|=nqRA1-HGxF$SV*`)ioTw3Dont>(TkA~V?V0%^88CD@0h%5^1t%8jJSpH zVZt4csU2}Y;hjF;yWcFd&d!>$YVyum*1P{Aywm5mKU%or?0(E|{t28C;y&WL+?}#? z-i8rt7}vbQ92sS%LGJvhNxuCj$t(O*a9+oE*Dz_sRQdawhsbKlQeIcLv!nh@98wK0SiHF!nn#_95SioC@{| zyv0Q&mUmycn!RNB#o0;U%G;NV(!d^o#MV^1z#Q0YvUk)T2-Hu?&$Qw z*j^i%Us3IYti=+(XPR`Hi*8ACpBRx$`XtYNa>P7fEA$i_ByDI{MoRi*pw~~}HG=aq z@#4E(_^coLkK&omGn?lW_CEcB9VNg%JQ;UrpMH!_U*Y@yxoby=FU60SGB#26Q6|NY zE2)>nkFOd(CZogLyeAWW{G-ZKPmFN=^~N=;CZB+eui!fuyyX{jIBBeUG}1u$&?F3*)t=^dpdwQ3;gzJ*3fmc+~T41$8d< z^Z85DtPEL4zk+qRGEaxRQ6I8?c$&ZC@K>4RxY_C$`skMY(TC_CtY}J}`p}7sxd*IG zRq33&f!S51eIM*p<8L+fmXqvGF@MhRhx|FAW&W(tErCuU_E&(nu+tXHnp(;|Jre?H zRpa{J(dpAM{Wkt7dfLYq99GyH!?|=suA$EU@HX@sj(g@3Kf|S%)F@EB_xxNu2C$d|+~3=KWSy=#zO* zj@U@}A@*|R(6<DbIqC>Qrz*`e(DPmb8Y`!D_u!ymwZ2l2i5Ehpi( zqw6^Uo4h&>M8{8~C5r|q`7ew($XzoVGOa$3N!{Vkt}6I8e*PT#B$Doa(yI`7v@^cG z?o!4){u|f0?PlxFkxi>v8+tY4!nC!K)!m`veErz*@Myz%S$ENoXw5j=y`#vYOv+x& zdhR_4rdzR|?9*!9nKLc|hf0ovW()tx2ZPh{E+XME!Kd$Zmb-}Bfh&CEzKDA`4~6w) zb~-pR=STQm3A=X@Nq;7p@6x|m0e@Nl8|rFplKYQZ>cL&SC8^wTQ4}m%RP<2UChCxr zYYP?*527Rc7Ic3N`@A@*@%Smv7Fg${9{<+E>ELw-uZTIKVtgTTrN1hBd$IAai5YbbvDF5#ys>Mo~OsKd8NJka%p^i}CgDSy(3P<3Bg3;Sj40wmQFWuVCt;9T{btswmG}=nM9sd~d&?Y!fz|SG0Xn z|8Ll?rM&xvv%G7?0-9RP41MGdlh63BFH>@p;fyq^$%xE>U8|f zP1#n+&f(6Gf=f%6S>*AEm9i#cwc<(UN#Y6c_<4Lh)|!soFK4lyX98!#b>{5s`ap-` z`;P6j$YPFmA@_?6;%<;{@b@C_Jh+QH4|7C z&C_@AJuP(bveIP>(pYz2FqXWX8cHq5*;L!+?CEm%Or_u2)E^r5-Dm!pHQD33WZ46F zm;a87kX7zW9exM0z*e00@=vm?yamv+!vG@az3k^E@ups@2KoQ>6YS+ zx%9JYn3H&uam`GB$3=Vo*2Dhk-|+`#20i!BQGc*rFqHqBHT5mRcaVM)e5qCCv_*HC z^b;LadH*kIC-+G$GWSl2u8IeFSjy56(&j$NhziEm40UNeLLXPNv*3-dp%J^$4@ z@V~Po`?Qknt2vLUoc&nk?8gdPep~jNIGktrO@O-b^7)x>x1BSp*ZR4mcJ`Sy>bial zM%m?ej;ip@8#Vi(IhjKucVy1}WPauq_sq&X)0es^#W!wa*mv>9pRI85QuaZHeV1%x z?L08DiT@|3gbN~@)(I?&Y?>zUQaPv0I$733wddP?{&1*!@$69>ZV%-yy)Zp@+J&ui zJNU+I?C6VZ`q|gJ9;PQZn(N_ZuXRRxusJwa`#tSxgYu4 zW$d>?q0gWnR}%_OneH-UTBeQ}~G=MTcv77ahF(@yYPvft=xQjcJp6<>a=x;XvN-8*a}GPrDuY>(%y6@Gu8uh%W?RKz*mlaRN=N(r6d3x(deH8cGBt(l`=ugDn{Z}Lrj ziamRA%irg#b@Mu%z}~_lUubxDeCFCzgD%3a1|KSZoChv`zXSbUzq>vX-#)W-Q71`< zhINZNb?2;cZyJQ?XL@bBk$TUSb3I`>-)&Nh^W9`V^0yfmQs-Pi-E%&5&}izSQT)q4 zkNscC>i@(?!~9GC(;nnsn6vto_WvO5zx;QoP2>Dy_KDW953m4#&*qWyk#nMY-roDI zJ>Pcof8q)LPaIZqlhRX32dlYaohV=rs=g>L{n zmnPLi!?}4U%Rdr}@u@y@wu6&Gyv)neag;sjnL*B44RY2h<;UsHUUdG2l@I6N`Zb&z zFoAO_ujWkC|G(cE!>MyBe-0EEzl7fxh(AO4v&)hGe2uvOw5K?j$2|v(Y0K$nrq)JQ z(_S{5HEZF+vR+lj{UQ3MXPdtD)swF7U(Vfup={QT`$EM*`bTM<14VLQneeEh914{3 zlbDkOm$Jk?Vy~!s#G<58uq*o!kAH>!h;MkXt6tM8?b{La$>>$cUa#5wt7A;^9%TbVsGtv>4*tRdYw9;r7p#{vc~Tpm!Hyaq>*t978XeQH_-i~)`&XAgam&g7>f=CYrNtS-)iSvB6YwKdq8C@<<w49A;q}v46=aiF1!Dr>1n_$Bv&QhY!+Ay;l-|*kHX3P`h%$p$R@<{$; zZDjG;q*;(Old;u=*81MO;iSLh$JE=BN1xq+F&Xg_j<=?!l9$za8Im7c#L@3PKcw%5%n0!8gpk}|^dvx&=Cz0ddVBGznG z&C0h|aK5JKkgjw{Z+L6goexVNc4eO2&&gUqW78Vst||DRPMdr=#dmeFGIEscw$6$y zt4|aD75UOf{f>XZ>|f6;tqF|Y+*-@NmU^Mbq@MldKZ=xtODQL^mrBlCkh2yJPb6RQ zJceGE(T|~DuyiZuDa!m1YrHvAsC}q{y!gewRQ`>3a-{9WR@!{&7t9>BBB0M364`Rz zQ0u?Q*tvDX_niN-+t;=s!n|8+`n(6khpAR<&7>(u+Hy*u=m+s*ezPsTI7g+F^;Mf0 zzvN5}6usq3FP<`^YWfHtbHKhf>dc`c!f!HHd?%0mJFH=@^)coS?neh{ON)Hzq0}*x zrptZYwUmp6+q?9i$b8yC{#Dd~Gw3YNkmO8V#{E+}Gru>ejQJkc)@@@?*cR51NxG!2 zZmx5mIi7vC%-(|jL3;Tcn zlTMuP@qe#V2airBuPkrdtWyCx6?$|!TgHXxB>KE}hSo>&Mt!$PQK2sw3MhQ^JLbRW z$2kB0+Gm{QobXw$_$+1g@_dheJzMBU-Q?=$98Ev)CExy-72)3$=OmJJ4PVF4KZsxV z@JK&I&ar16RGkx3Ke+!QUvd)P&FtH+)Cgk$dg8Vw==${Qs~ssDIv}?7YLAL0j-S zN3Qrf@?e{Yl%EWzM%^Rc&KXXAYoI*6?XxmveIaGzyf!@oMXWC@?lt}D={BZB7{IijN6TzfNbw1^< zka>pGV5GV@IT%`mOwJRknoB+!j=pK>k!sG>txA&lI^IL=BGq^L_`gdzzKi$t_HwR3 zdQ}qd&+*eOm8Q;Q&Dph_I~+*c@f~(F zbg^uiPrZx(dLw+T7wtaA`f)31DD%KQ*-P}CJ)!?QmbHFL_2lV`kFnk*PiIW6?lt|y z>fY1WJT{^K+r&+-zIOUOIo7+hw;9(}_nh9F_@1n}d+r4fZHE@LuXt!Vv+{Se+=-Qt z+qwlUcV#8yrv9FG4fkjz;+eNTf_LRL{nxN3Uv#FO&X|I}J< zi$XZI76b6GjKVp;M(eIrql1um|Wds`dRQe2)*Fn0=#;tjMtJmGb)2I zT_$6@Fy*a1k$b}dVs8&coR&wKN^BlX<^yf>daQr|6! z_vUj)>boWL-hA#zeYaM;pXuS3!uv4KJNII9P73+aW^df4?#y9aQ+DyjyNE^oIJSb3*_wQKZC5D+Nb*LsZP6rLyCT(ulFl;-$C-+&x{qm zq+XY^7-ZbIwA0e?(q6BXFTL`$@?ZVQX(iV2X}|hA7MEOo%|_UZr^%X7fQ~z3=iP+U&6t&ugimbMT z14XrSF8*ELeh+oWZ!^v(4~!-+jADGxxPFKp*Kjs|TJpVfBf=vRJ?5vRFZCSuD9AQk3trrvAdXzVOnZ z9e8ESp~6`~I}6ykLs@Jq<#yi?-|8<@tkn${M6$*mm@LYi17BcV>CVpo+im>QY&c zekr~wrk#$k)=bLISA_Xjy>{B(?rO|*Ew)B`a?kW*X5WdAbJl!59zRcjCy6JSrxi~M zPmm{-F&p)~WS_Y}`}`nekf$9ipO ztc0|vW6hQYPLB$8e(BW4z>-cU^RDN&ei_=@i{JhEI~3BBGP8p}YAA^pmPe1rMe#*L^<0lX3Yc2gPaE!B%axL=`5$3sMF6~}; z=`z!cHF1%ctJin8oOdx ztSw`ct;nR8k#V91Y%w5{Y<{vm^U*>JX&Sl6Gy}Wt<4@3Oz zz4g#i_%5A&f<5yxzdaiKCFtsgwGW!_uxn|*YrEhLh4y58z0_Z9-}fiRDtsHvw*{Q5 zj_p&E*t1kYx*U*ybNoXUy-J7|yi{;I%V75J+L ze@(+*YL1fef|C1Q4|~D&Ns=#OE9O~;?!&GP*i{R!RD3oFpGkdFdH;fkWo#6jn|i1d za#vo~%Fb3W*$xBAU;oeb?8?iM?G$KN{CmX$?Xx5A3HwiV&~G!g&=<1o!`P|PJfoPl zENFMeh9Y!jUa9oA8Lz1Sa~9`cww%Y5xjsGb=XYx1F*7X^TjE&{yK=C$!EJm7ox*Nc3#YkvA|jFq;@J^=o$vyQix z{Wy{*+g~vxR>?V*g+8k!7zmcsC$%ft-YUDKHrTmjL0Vx+W%`LF(Kh`{WX?it>~QIs z>=o$X+={*(q+TH`cXC`W`g4BSWgX?b+73={sn3B^_%8gTzqRd0+AQv{aHxvErUm_F zE$H(N{RBf_c{oec-=*m3#h*{bQOwy#>9$1^pI7A1|X0B2Ay6d{oLDbI)P1#xB-s$veKX!K5X3Jb?4t zj3MN+!SoHwxa-fHZTi3D`3C4(=J|bP&3WEi&w0zrKZLFgWu;B?+VKyC&XkS+M7LAu zGNEJNu-jjHL+B_EQg;3m-77+e50-RnrrRWRoMo%ZBIo(2vX^e&)6ILfDXZDKtRBgq zc5|gJt1rU8f;$YP+&(Yi%XIiz2@lp`SKp_R`yb0S{yw5ybKm#>Yq_3>OmDfqm3LLH zshgxcl$v}|&G$<9w=CB;wV=PI1$~jBpJeD;mg_59&|lnwKF83HHS{gZ^@T0y&u&4V zYUs~0^exNvnJwrCwxEwu4wN4Zeamv)uLXUN7W7{m`XWQ$vRt3gfpleyKzl%^_TDDa!%Zq8dz6V#e zr;I_O^rfY2xovHJo;9_8&dg43KQxtoXlMGNLFNYCw${aUaooPD^iy3N;cu0NRJrw* zOC^_a-FC)9(pGXN#?pG#mkm3$toNJ6JU~HRd%Jgm?jvu-R>lNBHZne4PgwUKn)_VM zeTU|6|Mh=Z;28NIwl;BgGXKL|ohoTxWzE&-R*b#Sf7)?9)j0+g{?==1_FAgXH52`; ziqq}Bz4Tb6p0n^6-)?Fw0{`@`c6;VYZbVPD2k_oY4%OUy?4fzQ*JS1s&YxcMUjD{u zdv)K;$9$-g)3Mm5KUZ4wV*18igZxXUO-{2b>6cZ|$CH1SH$g9bzL|{oW|5y{>~IU; zB65}|dIs$?WBSbRpi^z)uP0vmKGy=9c--;AM8Zw$1;f z_mX%viwxER)aMR~vcwoh6S(`beKG#=ma0C)CHft*OK-+XiOZn|fMH&3J!TVL@@Jj8CtnpKjqBnGY3x zEc`!s>g-W|>ORtQ*R&l&=1jYs z?-lGdEkVcnd(Vv36aE+DNAb5csVwFP4&L70-cB7Rd2A2<7N7kLuELpG7JHj-lD8!v zJVCn47*zhF?1PW=vBcjSNNdSsJHVCyFRugnzWy@G-Qc<)b2jar6mW%4of`XgaQ-5= z$cdA$_K}z5uIXQ?LuGBqFXSWP_YycSfiLSMM|$#!uil4B%P zB0qHmkG&1+h1Yy|i;cDL6d9|4CyC$Oy!h5}=-JUJ5xtmWyidUn&Jv*PlRU-4HzEI% zF7_VQ)k^spX7IXs@UoGA3VsmzR~h*Wz~lT}XAUsaEC0c+$mgCrzRNk7Hv%)@wSYVl zOzo%kF{^xF^lVMM=+*XUIEGh}!4Z2dReU=-6BO*={6WF?&UgjG&RC!-YmfD`voE!Z zewDM%DAR7)OxXNZrd>dtCpM=+Ls{Il!Qa==-*iho-eZeW#`kTpX zb$67-p2TKJhck&|oVsi2^~Ls*>#cVyJC?;NI{H-o*N?cGPQF;pWM6Cr^`y`}VCec2 zSNR#ZpK09gyi5E&CjLZ;KZ*F0bbJ>RKi|X`N_;u-svRJy{?1g5prfI`_1@HdO+yhO%uKcPjR!{l6<8=R0<_UKd9zQJB`S=uNO90$z zZ0|h3EcQ>rVtWcu)f_?w8oNypcj_{}E%ONk#x{5T!|iHTot;{PG>w-A4ej^AbCpEdCx zO8h0nU!vo;oA@V8{M!>xru*G;x8lqG9ACs#4k1Rf06jv#Lw38Yfb!} zCVqp&UrGFxI=?&rCWbr-rpW$+P|srm-<`P-Itcd{$}*3;=9zn>iZRC zF{!_Y7@Et8ll3F=f48n?)@gNfc;@pgnsr)*{9BkAviEMwvR6^}C$XO75qQ=OvnZ=_ z@9Gfdv{`dimAbl-v-ep?z`c_Co7@E|bB~O5&Ug9rK~{Zya4cMXGwns#nGLUe?iA_5 ze_46=L%+t9)6>C~wne@N&9_p%$(h_tPEx0~cTVE{6`qFE{7a=jBxNVr*xZZn;)@Q< zi6289$^75GF5l4pLZ5;9t5o`_%6Ir&);Bhpy1WB; zx*ry%P2_xV-leW@rJ#Jvc3vQi{rnf{OdFT&ylrR&$4mRPp*@H#DlKM^1~>Ce=C^!2+3$2wHs zg8yZmL|W7@0q$C{2O8U_7?@i&db1GrdX^0HB!dIogCF?s3@@K7=Nr& zc)89C2LE{lvz;{xa>ohqpxh%ojPuw0!3H0DtK9tR<~#Az5Abu_Drt)vxfep^<8R^{ zMDB9<{0w~1*mp!fdFCLy%(V08famrB^X*o#bAFG1!0N%8P=j9veg$Q>hB|zvgelK8 z)c3bZn7X-!y7?9fQ;*kBkKZg|>f9RY+#4iJ{ai!+JXONfr8U&0lO;^OTSL8DC1L8w z8tTYO2~(feP@i5QVd}mb>b}b)Og&jcJvl+b)M+);X%|bF`mcuiZ>)r=t7@pLE|f6! zS`GEuXbDp%)x2ZH&Xq9rR82oCCVr@(?y5PK^_>!?KC5}%iVc%6b)3vs50xKG^I4ZF<-H&8l9%|lTJ?pt z_Os>OFqbw*(ROfpfTQviw363?rpy=dUFy27K*?8>Kg}nvALS{>**H-0W+$g3aYEN# z!LXAFl>c+c)#H|>CE#~(o(12Vr*s|;JFOI5wv(h_d&duy`bXtC`2SnU>F6{m7z zT+f|Os*a)0eSmjWH_)&7jbGy@UHZi8eY4-5&b!pJ>b+gSzJYh~n|i+? zU|+|(q)R?)N@VYV_(JOLjz`ns)qp*Ve;CqVyd51H`8RWf{uLdS{`t@zY#CRlBDY0$%;kEm-U=gb_@O;*EH9_jNrw4VsdEQWmnD>jPp)&C8xe|N<{&pAQ(uR{M-R;-Kg zCtQD;tH0(y(W8GSa5kN8^xp>mfYyJMtN%8we^~JkJ8gvj)9C*+dtik>;RP=LicU)Z z;U4|{;EXZ+o3_AzUkBy?Q(gVHX#Ecl(Dlx*z{oS`|BMxDBpfAN=<=`A{QG+J{|1~G z@yh=%!+)Fb&vA}(^?zCGzfbWGJO32^&!hkIR_t@(Pq=oV>;GJ(f38RW_rRHM_&2=- z|5aN5G*|zZwEjC3|2*e);lB?3*IBV`!k=)|=mJ zpV#^?Q~dLs`-T6@=>M`6TO#}kFL3$SY5wmUTFL)+fHT(cZ+r&+L$vX9wx%yuZ&LQHJ{!hcdi`L(9^?zFHU#0lxIadn* zZRo$vicJ*$gzHar^S|c5%A@}{a5fqKjjP}v(E8u&>c2|sKU(n*J0pbu8|eRr6&o)6 z2`_N@R~)PKU*yq$FgRme{a3($Uwf7RrQf6cze4LjK=IFW`U(FX=)c2?^%eev3tj$o zn*TJ9{>9+Lh*$Z48T_{i{~Sm9J<9*fwEkTb|FCnc@ZX94JFQr*@F%=IQUByC{l|Os z&jjZ|!@ubP_^;CXOTS0y|A5v%r1NVI3kZM0l`j7}&3}kT|9_q8Us~$w zzXbkswEic#`Y+M?H-abi^!LEX$LRmD6>A_IB`o!)^jY@vuG@jm$`0xC$T&#KK^<{R zz!N{k??KK)krQ^ZT{-t?IiCvu&*A^M6?abbnFLRtee3!oAJ-|}rlm`0Roza7KpZm(0x9Ll`uN9k5SY$S#cQ21T z(t;jW-|!PYa*wgAci8w|+GX+m4d9FKrU3uVBlDTc_qA!tcLDV+ z+qo3j(mxmTt-|;grX<&0L4HAf0|a7HSa=bWP;XB+~59U|%3(K(T@ z>}5Qw#a_le>=~(}KNSxg^IzB1yEZ0#wLN4nVBM_y-A%Lh#`aT?Cut^fx9}uvUz%<& z@z6E%Yi;i#zdYuNbNv}7&BdRc;UoUc1v*}ywQcNYJ#-142ihv0oQv=~J*8b1o+0>% zo~?lQd3o+?XHW9bC3t3M+Hz)@YqzW8M~2_QQmx}pz*$~?hcazBSHz`D@ax*%_F*^o znI_^~U%p~^?uD1=_z&PUUY_;s?TCji!LwaQ`xSVO@Wi=#K5KZsZS>p@yvWP*+m7~F z4_$(1Qno#dadWyS&eih)!*iq2a~*K7m*>W8yQ7CL!Eh;Mp4|YgOPAog@))~} z{~0k)oU7|ahUavn>olO>%kxf-bvl}^+w+yKFL`*ndP=)2`E#Pt^Ag~9Jc)9&xU>C= zhc2OKZGqys(-Y_F+0*bG0Us$xX8}L)^1SXi`!f$+g6Fecl*JZShg&4GVF>!0rGKS%5Tw&I`XY#09f(SN@cdrkNg zu5|fV9H;b8_UOMEoKpCUuQmxS?Pootr zc=W#koNQPBY4Dw+^?%mzjZG82InK4AHU<6@g@4#tYIJCtqV>K&@y~J275<0GKZlwB6aIv2p{E|(WmT8Odh$(t z&%fg7@-Mr7VtRG_`PAyi`FF62bp~@E=U*-9FKY)xzI=OvZ)4=!gs}cbUHvNGWUR4w zeDk+2_$K2Hv9lL);#c`&l)GJn>$=)sbP+tcqchvN5#1$io37Hjp8&3;52qI|jg7Qo zoA9^F4->IBfW0HJ$IvuR6dKM%P&CZ{0V8qpL)?mG5RMWKVy}{0alDc#{UVW92cIA~ z51pg)L$c6HewcuqeVHmhpAN3d&l9w6@e;u)i;WKWVll!|!u8N=yEOk&LnG;Y0Gvi_ z7=5np>wl~0kue?_+mL|{$GbAdYZ+e&P6{$o_;)EX2v?45PP47T6aAx-Ui(7TI^9Cm z@d>HbY98IB+Yj)S{{rG4H{HIGZ(B0;HHoTI>E2_~O?Wfu;CGrQ1l!r`n(2P`}E5hQ^UP-JTJA^5)@QBg^b+&=QU*N#^wwHQ-R)QC$NhL%^}|(Mjv8Bsom-J9eZxoK zA^ptHYT;HnP=FH*RL`>7r;x}$T`4FFV81> z+3h`a37$PpRQ=62Jn~#UM;V^ejGmK$IxiVL^ZMBDdgu~7A3VwSq33Q-oU7+)hUXK``cSRbP1lBdM+r(dB!6zq055su$~JFJGXh{xxQa-+;Fb5*Y06KR^Ihcq4cOAQzwk|DR5Jg> z{&Y1zcO!FyZaH=H&O=7lpTU)&RKkyHu`BG^K*;6^K*~$&7GfH zL0G2=Yu-Ha5@~X1ko}7JcR%VVyzVjdh4@PJmN`F}qdU{X%k|UMM)yU~NPe0J?9i3_iOMaRTZ0VBkm((zwaG#l%Ymna+U(`=ua zx4HNzydE<2s$8RYqFj4;xqiCA=zbqGqWfZCzE}4{X*T=U6ZtfuL;rMp7VX(kPn_!y zsXxW{GY!ufz%(z<^w#!fPkB!8yts{81O29sTPuAWSI=_8v&!(i5~$0y=^vciR;A~^ z`f4AKAwDeoApFh$P{jXkSwEMs_*nYp(uY2q_r?J-_aAo7A{=9^Ci&t_!u7*-{V3zN z!Mr!bRG&!cXLMg{z1q%x%%sUdbh!3+Hsm-t#)ebjA-+BZD0#-@leGn#WlRxv{%&mO zMqEo9j^mqaLq1`*4D9sCOXxHvY`ADYnx`HP> znsRlY`$O=>$Ge30AS)JYXT}3r^wU?hv-b{)#H8ONG>ut86Lz*MnmlJ4Fw&j=dUs!J zE8!^NTIdC5Gw<=R(W9cs{$h^SV;^}d0KZ1jA*^*+FZ_`6JaC#Tzb*17A|GD+@wNCk z)>iOyoLa%h)>XhrZ^k^meX*5i_2R}^#q zT(b_+ZxSEG0@|i&!Ve!z28LM!D{WUS?qhzbt@6QB#s^LDGJKHdTqXGU;xb^Q1Rs?6 zXy3#x*2Ix-g0BsW)wfpvbydF2rEY#GU5~{Ez)SM(#o$ZZa}n?>oz?It?dj zKc7Y1#kyWQq7EuDe9DOzf1UySz{}^0j&?~<)s>#I6LxY9-9T_9T}yzkA3>L8XQwvP zxpiJ^L)QyjkyixV=%r(QQLaax*epJZQ!d5c&fto@xj@HDcS9%pBoCb{??*$I39iU% z4U~M9@KfhJJD$?4XO3zAq+J$yN#Kfoar}6J=qc{t}7lZOwIhgOk?$Xjlo zrtzTAgq@ERO`h{EFfxEVJir%whj5gztVb7|H+hdY8XYRSQ;wT;$Up~GjvKWOe-nPl z`3rCf^pc00en9?2ork4;5`V;g5d0kHPl8W=eE}F5L>?aGi#2tF9%i;4fE zglnM}A1vfOzR&eR56W?~4!hu|e6UaJFiZI1gIj^qT=~1D9Jf*Bc)ja`-P#8?3O+u# z4j37R4~F?-*Ak8rE>!Y!oJqXLKX-kA{B3O#`CzT@lD}Sft1MxwQGtKy*ai^Alvf$%`-oVI6d@#}%>p?h5IB4V-@*dye`T+Sun&nS{ zpQORz9l|fy>8$wWIJrQ#EOSw=Uf? zELNG_T$f5Y&&?9QBypUW9T@vRW3E@HmR|CX`yRlT@)Ubs@N=B^1)uuv9bjYvb?^jVYzN^e;i!_I8o7@^hSY-s3f{50Jkvlt_cot`BOo{3OLM?8JL(A8fkT_+W;lKYJ|E zt)J_I8QKR2!4(~T0!H{Y7Tg~QM+sL#FFt7CJ&tbVH}(YeB_A~FaFXi-bQV7QgdcML z3GD02$FKY5ARk^GT_50MDeoT(K6LK^BiJz>TXzzU60W~S9p6Pi&T- z>H2_tBYd_geqraYKvf3dq4Fa6d*2Z418JXBc_*)neScCk^#6eo(q=4a_8j3T;i%Ff z>^Qu~DTmqzCzB7Fb@&l}Dj!fjMTf_QA3k^l7j{id#lG_?;3lWCTRJy6+ix~0L2H@@K8RWF5kB; zP3432t`Ej*AKa*D=>G#Fl#j8Ln`;S22^T6I!p7sE)xj4-2JZx|-6r670bkjIzxj6lM*tWqrM{v4>(_Q04T%685?F9yB zsNnPlr?OTa+uNJ6mZ{JJh zgs4-17q|J*A0=O%d|*VSpO#{1IX9V>Vx;FV~+40$K-)V~LR@|*2wr42<$Sj< z_9Nja;UILv|2y8}UEFfkpFVDLIXl_q(?#?75?slTbwH_aOW_ei56Q=a$)g9Ti`0LP z*?}X@q5444F@FrsesC&V;g43@A8!*^DQ8)*_C~(T8XQ%B%M!k31Nccg9u5n?T<4F9U)Y%m6#wU3sC{*~EjmopzIq87 z8H+Zy)xMgpXmXutz{tU}*u@8ZvFiv&30GcdZCFmZUjA~aeHi^$==DvDz5o7(!%xPz z*N9B`UIqM>$Bc0o7c4);9_yjYF>^(y8M;fr6`za+)_dum=x3iAP;*R2$dh(iBYYV#i?McbMr+eMd z?KgCLfm6M7%LDcYdL4)IwQIvhL-#58V#5c(iC((JNp^#WF09fv&yjXne73{LdmVUz zm+t3ed!vWWO(SWSh3+rlOB!ti4)fAAwXz@b_{^1ex1oF9$a8?DUb@9Wd$~uRYu`*m z_ZawM-*RAgFWvf7d!2{Qm3O_NTVmwh4Lrt67Y*4nJ#?;p(k_dAe*|Cbn+a^|r8_s> zzVs;eNt-KlHye4^1AWbOiwh34wl7Csq1lh*+7~r+lfW1IE(bR9B=TVAw)PAUoonBz zhHgB#lIJf3e(9xqtDPP7(7E>YFm&gDEA~Z!AA9NQGVCJkbMoeKQB zmu_*`?%|Q=+Sks|^#NDx>j8Y(OZRGfdx(e5mFG8fUBDH2#{$=Q>85wI+j;0*`+mmf zqGtzi#lCjH6<)f>~6Ub>+<_Rq{Sx_)u(`^3=2x@-G>2F~%) zP48rfjw0_(L)QSl$omR-qnEBnp8bhOo@?J`L-!Bx#lBB~S9$4%9%FAhioE9x-A*I# zP2gBB-6zM|n?3Se`&JsdSHPF_+YB7;rTaGDe&0jq`tx2xx6a6W4tScEZf=3S(nIIk zH`mZT3BK6364={IxAHi9)=})c$d)Or>A9I`0yL6e#sRN(#?!yk zG3;1|cLDrF$2{OOUf$7@?ca`~(@&&@)B|B~MW;5vN4#`DpQ6&|ab;VMV-e@|QNn-0 zl7x;Mz@_hc#QyT1_~x#2`+=}p14LR*@T6g`=}*0Fbow4V$!lK&r+am3C*zS^=Qh4e zANO-$C**n8LVmkg+-h<3b_$E4RR*>;OFdp*7j-%ZgTyZM@44V7W(!Mpq=Ws~(rC$^p`uuBv^50~g z`>!ZP#!m*bqkH?eOnP}x)`>&CMj>Hml=qQCfG z^q-by#!UhA4`}_@8~qytdMt3G;>Y|SFcL@qIP-smqlBZWdfsOe@7jL@l&ohA50P~l zaSw&ecr(r#uYDDC4}54m|b4?6L&;7A_s z4}62i)I)0v-jemTxz1Zf@-5FPChlBM`An3nfrHeV&r3Xg4V_Q9pV-Lg3a#jv4}8um zqqD5d&T-Z$8M#gdaV^VbI^W!KnMzp7iL3>$PWxRQxz>8ZXFo=N{^z5#D|O@_vDXI= z(WMEQiwGy|C{yQJ|8z_j0|@mlNWV50845**G4SLZxF zNm!RXZOeY-x%u@`;#Aq|W9(_fo;lb zoE07$E(E9A@Ne1=|1R2wyImXhYa2!?{&`Nh@MqoUc-Dgs6aIv26E95bUK3#5 zYb(9(_4%2jBIHr=b!TwmpIfG#8N}R@tekG}c-n4fTz@B-*{o*bm)4mPwRBUVVhImP(n zefZCTKk3~E991X0uXXwfJSk`20VC{*8_!<3ZwN;T7s7wA`VS~|SC&(3?2&x42OP2C zebwkqmdlbjBcX5;OCtT?Auka}wV$=z4xm^p6vH=`5*Z2l9x@a5r zgC}LM@eOT5t&+j|Uy;E+%dzagTq!aL*HS)}pX#)XIzubz@Blc~hJWKW_y@ENJHb&l zY|}O@R{X=xBH_sa=|P7xV|n{CiCp7+>rB{-#SUV0h++u%Nl9n}8WDKGn z9psOlE;0y5IWtgXoa)M`(=u-L*wUA{>QfE>b?~2~ZK*Q0G_KRW>LL7xV8al9>;&OY zm@~bk-PtAcqPePn9CnhSleSiHq}?e1O1?^rah7(+&Rl0hH(idy#BJ7Xw&IiMTU4Ya z_U>&5m-cwEci%)ZbaFNb_YKQ?qk=;_`Iu+G%(mKo_yA0eH(1JX_mxhP_Ld;91H35wj=7jdS(MR$Ti=PEeOW*vJZ_+>Uj!Ap4*4gFb+VrrI|1$Vu)1QE6c;#QG z_nnvXUGh;akg?1W`?^-~%^l0EB<%K!q@Tw9@(Gy<-Rm>e-dNFn6!`S{j@V0h58n#; z-@|>oi9JHUvG&f@-Su6S(fv;F#dq_89lg4LD`!@)&Yka4KUV`=`soI~xqZ5+ghltg zN0D`|ktH}H>l)x;o+&%;d`)Q#PbsmV7xu=;1RC__d-@>q{3G zL^`_j8%LIf=~*fd-RsETF%!Le%ZmA5>) zdNz>O;>&cyGX;3Rm*=hdcAum1Z0fA?-ylz%>&uS}&x0puU;YG?@|8$u>o|M3hc2OK zZI{HJrbnP@na@7s+c*!MYr~(7yw8oikAPQr<-ICr%H%j#^Ihu9?ZB4hWgFkzG16AT zl82@2_3-55gr4W0kf=AN9fj9gL*KIe<#>3xzLIuX{QoR8lJA}dN}fsh%FLY&;JfJl z5U{0Rmh#Ole>P#a?)Z&0v%kv=9~9bq%)YUqN8vS#Z_*wJ4{yEqjfa=3`^850+n|Y| z`*fh>pM>r|%Nm5Rlg@Y7|1EXDf;dGY|$_^abLCztM>k*xdG_j89kolNR11W4MDyMU9>ee+~cp_o7=4asYlWO6s3QT(*e5p3XlH#WDha=vz9^h zACEqgHcczE{yHtfP6~7nJ!te_2JW^b)%O}?^lx0I`zscB(qI<;hX4R*#`Y5p#RHB3IEGjqox04+#&w|61w;jkN$Iv{@t|x-Ch4L(fa=n_~O@( zfuoVH`q%e>J5ltft#ti=kM{pAg`el_07gcl|44sqJK-qd1rIm-zm{(*56QT!H~Non z*8ekjw$%SazKI?ipo`D<=-AvmZe69Zu^ilbLhSvX9@Wt+%f$zKe-wf`mfb#$IM*rB&+W*%n{5+=$7`X`j zFY?DK2}cPBmpAJl<(udq75#Uj|FUNNrGMDc|2G={UjSWvipT#GjsCe>|7EWKr)d4p z24DO?4EPZKQ~s|4cZlep}sYAX2MkeCJiT>Dr!coG(74BSvu6Mf` zTCwq;;2g%rWshkYoWH}kO551M)oHZW=_4hB`9G0y6*gYwkG&%@2ycg8ud%Z1d%}8s z)h^;_i@3Wu<%qRJukc;r)>`&lgZmoaylaX!x;Q<2_6&owS#Y>lv4VSiWxdw(F3v`u zeZ9e1Cpg?QQNcYEg7cJ%a|7!j42~l>nc!qRTa5{n0N#iVXan>id}g~iU6buI49+~kIS!oTG)}dP zvpm@zXmD;5oD;w~LE~KK;zV27eGSeHf>R7mvBtSd<1DsOO$2+o<{oT+gN zU7U>}yTRb}5S(&w$~8{Di^Cbgdksz(!5IP02#wRx#W}IH{i(s}BsinM8KrSryExCb zws#qv48ge&oC`Hhl8e*1js1qf2@1|QaK>qz!(CLlSZpnBWB=9QaQ+qdYJqc!#`(#` zvD(@j49>5Da~U|7X`HWJoa@@!&l;TX1m{X{uGBbnE>03>0vMbx1m|jSuGTmoyExai zvmZ7%p9{`pa3*V1ZUW~fjkDRsY09u` z49-@;xdohCG|oB~=hjU7CWG^m;M@+*?Hb2%alXj3uQfQ&3C?VAW^0_sT%3xqeYwGT zQgG&hGf(3@=;Hhww#OTs<$`kuICp5Adt98E?d?$pXNln44bI&fXP%3b)WJT>;M^fN z_kwe;#<|VKncl&U7@S#xb3ZuuYn&TgoFIGT4bF7Ic^I6BHBOa_v!J8h%iv5AoJYZV zRO4Lc;smqo;|g&l})*!_9NA&eIqo z2WUKFh(`gB;^rBm^Av>2J{r$(;&}@^Z@GEy)_GdBmnDrSiFn=t&pU3O-a1cNd%00N zlbb+1?}6t%H_z=l&qHCde!lk(;`snPAGmos>pX$s@(C@SF2wT*cs_CSgy}r5hs!_f z=YNUkQ}BH1=JD5g3OdN&YCJ*2^96XmaPydSo;N$n`!$|c#Ip}P``kRuowWXr|22QA z@feBc@8J2no9B|w^F*XvuJJU1Mt8!cSGsvl>pUz<-lp-?6VDOw9C7m;*Lk)?$(uDE zC-M9Mo*&#iKk7UWMa!>gJV%M=7x4Vz=BdUkSb)JeY^8FgmTH^T~ zJiohnmg_v07EM1&8oP(EtpBHTPlI?uM-WlxPKg?KuFr<0o} zLFc)nt9;eZJs(RXp4-54o15njod^F}Jg@QeBAz(##JPF8=sb;G<&zpuJn?h`Pd7JD zgwBKSpQ|;VPQ)YJ8yckZeAi9>PUC4$JbjQ(A9p&fbe@9l@;4ffAMxA;p1a&UMtU~j z9eApCPM-Wu?*CTP+-~nMJjWrN{BG|5LDM|UM9+N?J{1042q*t;gp+SWIQeM^9|He= zgp)rG;p9JzaPlW3JO%zE2q%9g!pWb7aPl8V_+a>R5l;Ry2q%9b!pUER@ImmGAe{W= z2q*tJgp>aQ!js|u4dLV$A)Nfz5KjIEgeSp&1L5Q=2q*t7gp>ac!maS%Lpb>#Ae{V9 z5KjK52)Drh0^#KELpb?=M>zSF2*imm&gVckbe^48_^_e0759pU7o|Lz?CAAPideDuSi_~Q4+tzMxByS`olpy~DU19sOdKzT~YPp{V*v<}T}ZE5>h+WRe+e0&lRkX`|N8dY4Vu_q(Lz3Yd%dRRvkSab zFKQRktCoDe>&5G31D)#iCj9H`wHY+MUate|?d2)tr`KzdmfuG3QN5^LNUvJ*nhXE> zdS!#A*J~EA-d^;7O60dnv>W$m`DKBR>P78BdZp!eAezw`&;aFXq`O|@5Qg?TAoj2O zK%+ePy9(@?(YfKbguL`N>!Ib927apB2>3t2UCS$h*R7i8)j=7EFx2gUXglr#O|M&D zV9HlTp7fqb7V^~FEl|s|H+ZRjcfco|Y02|K0{{P_E|>P)&f)*^TD&gcy@Gh1;BOb> zouhM__~#TaYq%1KFtqmp(Vn&iEed;g^nb`!!0vVkP^R6WoF~!7WDk5P9pvW)e+^tq zIeXFBv;d_@YllYo*SEuY&|2C7dv??gsX|_QJCOaPyv~82+My2qGh$vbbe1e!$pM{o z>}UAbx5Ia!>FsbBn70G+q;}{kNC*zyE>$y|4%bq zNJnqa60PjJDGikUBlyF`vf~^l$?0CuseRspf4!U>py}_s-T^cR5XY|twB&hGAa|Ma{9&v(}~hcp|dgO|oi zjte6V4%d96!?lOW7mO0!k6Is;UtEMS{s5CxRJteGi1%x+qrT@+PrH$ol}UWBeihxP zO!t0H)5ffYNL#=En&`AHqC0SBgKj~8XI0%~6SJsH8hFIS?B%t{BUXx&V_KUE*|Z@@ z3*T>Z8A3M^r zTpzcSMV$m%APX$q@-XIrH;Oyi9e3;v;*!n#mvI-l;|{$++~j{3cY`}_!VTi~`gd_Z zb;pgnLEKyaUECUX-0&O34gPm=4YzSQlU@8n8$r(Apy9Ypp#ySW2070_8^xlHW}uDC zXm7eJ-qd4RQARK;r~YE{?<$-9jR)hRt!n(RA!?4q>W8CG*BJ}LCIZj&EDVbu;ieaA z^xH=bQHv4Q3h#fE1+RO-64*f3A=)}sSdRSGvCoAK3wi~5V*!I$UpsQqySE3)0Ue*|ye5|u1(>>&0 zBdh}bp6+*c`dZXr^b-^Mrp`mUPJDZ)Uw|jwz{)~kGkn0~^hST5_f(??><8A4C2=>M-SrSaQ zTGfG`ESs4vP3w^kt!dcjgVg2JXVKQK`AJ9@wgYyN(&OP@4pJ8(JgT{Q2Y>#0kV-bM zU+rjD$gsj0a) zPjgZ4mb)ceLyqaNLpiWPKEOFitekWJ`#q)UUiQTF+Yoje?*AmdIN&(gKjK@i@hw_9 zAbrsixWxm~QxTrZ)JDiT-3DyK{a?U2l{&9yX?mPJ@p?JJPNwBUKc>MA#9iEU*9FdX zW!HKahvDvv>{{cpu#>$+jG_8`3~_vvBT8E<+n&G)&|#|KKjnafI7q?uoLJZ_POJwtW$ ziMAxvh1=86Cw*cipEy%T=pwb(=WNtKiDetigPy7CQXF<&x@w&1xk{hsN*k0`pEW3s z6Ac3;pE^pXAe2od3nLu(SnZ^ahCFrth-s5rb>g?8z zrs-|aZ`z{&_(9+O(O24`uUt1K8p9g7T(!RAGt8>`j=+AvD0d$kgg!L2y`{+mZW8)Z z802yX)-I%*rqg|8CRHSs7+)H|wpR=&dH1CO@5ZtNbMC%H4usuGf8pWAoEIV*A0gdo z7YQ8sbWUaD&N(%bj0^1RrRL@_S5c2wLgM6NBTFcLx7AJy{U1d8U&?Kgd*5v$;?tO2 z;>F4e-o0lh#@XIGr{-5akiV_rrB@C$Jg`u0xLCQO^1dOChM>wX8=l-)-!Q)q&Nf9= zG>{ItWfhkorrlOcdnnSb!WoD%qv2o)oE7}AW87X^DGcWe6ypQ^_Ia>gzOZ}C43>9J z;qHo@1yX&9nT^TtWb#R)Puz0JXWu;w!q>&k$m~}Ko%LyM?!8RPshoMZxboyHB@I65 z-!#MpRX4;g@L3lt&8zGM-T>IpGRSK+v2X=KQ(hWViJQ-!|w2Q--G;U-cP#b3}EGY8-)mMv>iOxw-LrYuG6S{ z8tF3W9=`nwrP-Lb%p`X`;*$HJQyt9cT^vkF??C zAH?GO&4IVbp^%s5#fKXwz8KLs4dD|5B7q~Ho_Hv7=eT2%9O~_s*Fa-HU2W&2SZ6>% z*gz>Lll&aeOj4i^VO`&w4N`U!o@w1U3hB|>Yb5z?E$RsJ11#!r@QKT=UalD@Va*-V56~5;oN4xH=nik3H_W$E)F(d*m5l zUp)}!2_7%-dLxbr_SNigB^%_+SZk5J!uiUyAoPRk95ycsb1c}|Z6?_4Myz)W%&d&WDck*u2Xu@&RUa3rFutgPLHOypHEw-pP(jt5P z9@_0lO=r11w;@3ar8yL~1GbvmjxgAc@#D+aWeBz-gWHZEZaeV*BiM_+nPfXM1lxiC zAc5YG+m3YD4#Pmp3)N@~1+paD@hrC;9h9SZe%7Du7=ZFq-n#s#K4EAt`uC)Bq2KV$HuTs*u*Ufl3PIHx%{_!Y(1K1Z&$4kgIVZYe%|J zjdM?=^T%O7e}yf>nQG`ru<|kZufm^-GN3>5@41RVyNonyt}yp`JKSIJccf2cqO~op z+bORy@L}$(P+PN=DQYvukK=ooTys=%97BHfuDI385K6Hb0?-gJk4{LMS zDV(!HJjD4Haj0)KL9dB#4fy(durhZ&!*S1Kpi+mjP~EC?D(30+yH>YwyJ`eDFqOW>k zE+pZbaZlx<(L*`bmX$Xe4GQ&LteGGaPo+Qd;C&Y3NTiEp$C%idQ0Pw6-K?Fnt)(2c zF|Gspjox40NB_Bu{zJNp|4{qC;?1({UTjP-`pu6IU+Xu~7>EC<-$-FvzoBt9G?T_$ z>NmB>2W!#HT#UQacdF4>NVk6sgIumQ_a;5|M>*3$pM!Gy0*4@fk@paIGmL?SkrPW~r7=203zbbe84iXl z2SKLEkZlrVPU|U2t_w2+HckH!bP3@G@L~UHz6bVVBwC|w)7NPKZk~xXW*Yrt17po9 zW_SNTZ^Q4+Ye`3`?qpwCd+b+1=GV`k_96WGv00x#%`II~dH>%_8{S*-b;G5tryDMB zU0E6O&6^D&&wbHwdH<<~^5<4os#{ANzCW|SAz|gIh6^9Asazdf(lF-47Y$`Ie`{z& z9YQf*jnu|>lY0))QpQCnce2r*b{uU)V`+MG^V+%(*Q6E3mOO5Pu9#t4XQnJk<71>Y zKg)z{A>VB1-Iz@1%A+$J>sCH!N!SIN6||L}7Il|4KU*1XnYT-upRK$V*q>#SuB?m& zp24zdezr2(+W1_(wek7vq{dbCNl)*>+Pgw}u&GKK+r$j1&*WZh93ZoJIZ?$<*h<_nPF^Glwmk8?+#wx zAmx6oyfm&pdhgP-R`Ei4?_FBy6Hn#+z9IF||BmvajL~lS{#VQU9xrdO5~r0n808JY zd@>UHQVyL>!2Bo-`VfWr(Ll_PI)ZN?=0^iDKN^VnQ3B>iK3bmChp4}Lae4^)?I85WWc14<^iM1L?dMx)E$N!S z3;mPtp`d@xOmH&RhxgSrSUXE>2SYlp`MWvquFdAP7XL&^}M9gIOY#Wx`cIACCXbV!C|ky zvJG5YI6pXE#@yaA^R)2&8rqMMaz7hj$o+f(?cG>AOtP{3YMZodP^tdxf#(<+AESF? z@4%ksGs@y9WRJYodV7)s`vZlerB;PLU9FCXqz;AIl7FF3E|aH5m~3{#^w%tlEhF;t zt86=@phAW`q})Bo>x%(K2s~y7}M6-g%a=$`)Un9N!1FR?`PfJT@ycAyemedCEQXM0uwuO&LeubuL zZy9@Y^%B+q$G@>B*5g03XXi`mxu+!6iSRQy6887n6jskBUn;DcLq02X7D{R#)c;6L zj9g)mc)QcwEFrmv+&3BjJ)DF07J0aX31KDB3DVdyvgV z)TwEyq>eKN6*>)5UaMN3{Mu>MY5l7yuITS3Eo}YiZ90kbCPx?wlV9r$SJxTy zxJc%z#XPXu*N}yEtLx;u%)xd1j!=G&&L^R`hY|JKfqFT+B-4IWuk|P=?=w{)ayi0} z7{UrmKrpu>|NF1 zzxT&_ww8BeFZU_P8T+`sDLp6XY?#HxP<|82FHzd2Nf9;gT3yGsT3tz?C*gSt{7FP# z%Cay{xz^L1urWb?9{D-fTGb;46EELo$m8;cJ7p*K4XeCTvnX9N((T04og6`OBKcGy z9oPYtm)j#;UVwC}P#&aR7KQ#@0{%N)SEozrUPFgM zXQreM-IV0GhwS<{6^YnW+%XL96#kug2E2dI?kYo<%J)ZG%1Mt-;=REBWrTe*NAxHR z%K>drd&@J5|8ja?kiCDDNuGtYKEuA+Y3$$aU$Adov(dNiEbsyB5gF*+6kNPy+G@wX z8tWX_Mrv{0y?s?=U4| z7oPK$CBL?)!{ms2@~!HK0;_6UVO2-2vZ{9&B*hQ!4Q@faTMLq3!~XKNIUD+5uOdKM z;+1su*KqHwX1q5&-jlh`4D@uI>g(aazM0Fp+2E*oUdpQWWiH;9mPq+zcjhodo>Z0$ z>`Bk?!GBMY?~FDUbvbBO;A-@PTCFWKJy;dJx3hdDgwMHGnXG+fbF{?~Cz+Huj1%b7 zf&U`kRnhoo8Rc7dgZMt!kEXIv88Dw+6`PCyTbV6cq>H4RogtrS)0BuBl*#EX6ZNN6 z(FbDWf9UCL&*b}M+WUl6rdsS9qYo8hFEk48DcPfU%csFx0-dO!IqTB~xfah=6ImI5 zzQrj2if7b=-vQ0bu{~anak26^jAL;UODO1}jixsLPQe*2_?x?H`wP3<%OSGG71(~1qy40Nv)VssafB_fxWeY)p94=? zT;bUkwL_N0le1<(ZZqFX zay%M=ckEs%S(ERwxU%lXoo{Itb;|t~*X(f?b?Ua<5Ju57~y$0HH8tVup2 z9gi)txN?}e?rP6vMY{~x)9$&vsCJ@+`+a?Z3yW&UOX^W$E8rDHwPSHUv2%KaKk5CO z*%4L4C09k~NfA+mlO8%DD~oDJOX@DxrtUP>YPCI|E2wyisbwPkJ3=Zk8GNceyB)QEh9O*Hz|)nRRUuIH*EOXq13n?M_A^shUK z{iOPya?w!(t2>MFsH&&ED5~=#5nmvjb)Fva8~kE}RCfk*`|6&#MMs&TuGZ+K)bw0b zRMqLR2*0S-nBjM6*}weXjAF=j&IwZizu_bi03C3RSbCY z#1pgbKaoA_U_=jFSwt_}q$3w>mLv7Hs-^8pA5Tdt?bFd=tKH~Z+7o_&C*put2JeiH zTWxh)eM+BzKL+$QCmx^mPQ;zI+#?OPbROTo)Hv*(tn3aOZO4XtmySYy!z13Y%}E_p z+6%ldfY;zTJZnsc4YnF@uhL!~l*XJ{??&{r*^ivJRTbJ3qmWkBQhQ?HgJT@3(WqoR zcdz3L*2q);HprCr2cRO9?WN8@;_y3-K4 zzi~XyYK*n2-+{MrEK{5JGgtj^tE;L%+UE*vaI)1^(LX(+>N4;^tE+_YdFHx0+NwT_ zIP|}w)9`_Dla zEd8f(J-S!&-+vD6f6mxeIs0L_h^l7nP#H{h&YlHDPJ?NG?Zys{v*>fpk`c7LBBx~9?;MS~F%LZF7daW8hcmU< z;ECtOMFoiC9KzJ21}{7>DY}aNoV`R##=VtWF_(H*P0*>(U{~<@|@_!hLG4-;YYJZz4ajxsFLLN94!0E5AwVWz_2;#_H%$oA$qrhfP^bIpqU(6)6lQ$Kiwxkf~OU{jJ>JIW`w$+|tV z+~ym()ArVFZ5{6~!rUa{ecMN$_&GLL`a9mkcOMqAWn<#xYNX$2^j5kU*lOoU%o|Yl z<9TVN&dp}oqyG5;&go{Ek>6#wqZV~=q7Kfn;6pu*=Zz|@da(oC!&%2shbq*e>b%8O zg*rITf(Lato_B9)Njr~HC-i9x%3n1e`hYq(p+8$thpN$dMm?O+g)OK@74)bCb*Y68 zIFqfaqf=|gap(}~$LoeRS+)4rp3_M5wvMW1i>m~6tF1#`qb;gqgP+5>d8os&!QbJ; zTy%?G&qv+$JcIr3c{ulR+&Hv!>LQl)t8rNAv_%ru6=NLFykT%;L@@P4UTWzh29K=Y z@`jgAH+W`M?YFqn@Ak@?aTjyVzZ>Q2S&@iu#avZ=;itgw4?mUk9KH=@+4!Ji2I`4_ zXS*^`zgehT?bdOQ8K_$p)$8r?j+wW$c0AgxO_uXFi<*gg%tRfY{l(AmG}>maUgrEb zd1Nrg8O%xS?N}M+0S9Ovxe<1ow_Q}cT&>+7o&-Mnp8gWXz$1@$mG>Gf(waPD)@hp8hA>y% zyOQJFhs=R}8MPXA)Qr6J^eMg7otQiad)YZj*vrPAMj_Hzkz`d@E=p2uZ37Cw!FX3; zwYtvu1Qh-~b};t5SoS%*FOD?$WcuIfo-=ubS)RG0-73&1?4>Xn>zVBs`&PnXu2WxO zwmegTy^Z>&<}#Wqb;2AyI&^Zxw>WFSWpTW#?4ijj3jZm@{P+OvU(#61!(YPrdrelC zgWNKOywWa=w7&qEZ9B3dQN9H-pn2KQyd+ng(Pw|A(RcqToG)0hK3ZNQ1s1MPvbv;% zNfE7(_6nqJN7|DitNoC5jkhuDWC(O40D9a-a^OE-v<_Gm{Xlp5W8^n>k5PTV=!yFr zz0~n}-s+Yl^b@ozW8i{|$V6euO!7 zOudgB02#(0eWcq9>pXk`+Ecy{{*I;)*$3^x+wR+*@=Bz`!+Z3SX`ZF?Y%vrR(K`b! z>vMD#IarwlUMdgXKW}>lYtjOvB>`*3Rk7wiGU;B7^hy!$C!ONtF`%zb>LY)N_cEh_ z=iJ#x9tfTp?N%4<>oz`iBJgsI=tK$-`OlL@X8pR(kag@utXm5C+IdxUKYG8xTt5b4{n5p}2A_noaoy?=Dt`w$ z_KoB}d*4voIT?FTN!Ww3W}WT7?@`)UGr@Lvx&GulCi+srhXCgWs%(dqW?O=+tUZ zR+8zlh$(Ql1P;#1hrbE#whk#-U%>wju4g~2jeH)9C`4P%K{~z03@V-DWwQ179dqtq zFsFy>elON~$SV-8E!Mj2P#$k!PdFnS?FD$idBOR??Z+82S|}Kwf%E! z`fH`;SrOd^*s@N47~(kfA;TCHoOR`GhW#z3o*5XJrA`dUs#(j_OK@z^^fhQykJAz?{9rmaON~X-X9tB~MpxuH!s-gbN!p!XXGWTK8Q$znV3{axbr_qiW`Wy4&=e>RDZiX1NLz)<_6~7yLNtwc?P!&uHu4^ z4m7XDJOAFg9??86ifoPvcd_97EPW@i0Bu6=v?#q{u%D5Lw*?sIp)%8@0Tb~@^Pmx@=gvg1YVu}0p7bvc>iL+dl(~Rhj*iz z&gb9HRu{y=zDf6#7oa{2bM>B}(fhfN?qz)L4wW4&oUkADxw%2zPIpT990*I=zn^F>-T|H(zY>77vy_X8D@1I^Db zW3Q9u=ysfK3q<}Gv0mNfWo{(7SEDTrkXdIq4}?8(NLr1vAvm+|RbP#FeNQ|Ry3UR< z6S7igAU!+w{)%xP^%26@11$Do>J;E&oH;EHljc#o+ZW{yurFE!mj{;*w-|28fT6wt zg~eD$MaFuT&&B%STdWo6eA)eoe;8{9D*I^QBcCP%2P^jikAa;h9>PySMP38;K|gY-8R-Qp1Hd;$<8yGn;1<4DL4Q;4 zUp?Ywa}Vj@zzNvnTE@yFRXB`Kk=x&b3JA5c*1n;hypg=r76o zFy0I9z;_6kcb+@4p}!o0e?4CQG@PGd-?fA_2BI8C4XpgA!K1wSnO-|d_7tDi2F2K; z*n>8pH0Ky@mva_jZMgySU{~?p93QDXr^rjKfL^fUW?4etehT@M{L_oPj>q`?;-x8kkXvxR8Cf&-U+<7!NtLK%S*%_=Bhc^!^C@zQho;7V1RFg^MJ$o zWrG)-H=L<7J=LC={%IRoy4Rl817%PDG?e3R9CzY49vEq2PyAMHzlirX3OD%haI)za zMh-%sPDbBOLLax{OvD#kI<_+ea(g+?Z7)q^SD7i(bVothmuRb)1-|81es2!B-j32d z?^}cye2}7ozb*>)lWgmbq&T^xX|Q@1@-EqzqV|KotI1Q{wa-i4)kiAd)xZ*VZIBXn zy#r^E5~z(!PS_K78Cb%LA9!J|=05wb+b(WH@y>TI#&7*S@PUIkXMy*^P5P zf!>A%8DI|Wm3(gd?7~BQjV8s)G11Hs~D;vF{)FvM{Ja*XV*@xy}I5KS=S1qxE zw0He+e--)x#*bbFUiO5c&CT!c^)VHi5-sISYS)g!EI1!WHefT-vg7^$I%i7ngY+~u zAPo!R(;1N{oWCrl(KwrjckPw^Ft&vD z>?7M}KGpAUNR!T-vCOCXy$Fncn^}#t>D!1X$cVm=*p2>rz1~0;{F?;Gx!s?IQT#s( zqjK!T8X|ObTV9SQgmPqSQ|5s(%fN~^FZQBy3C*q(#_U80QuhF}LPUCbZ$S=v#r-i=@=*YV_&)az| z?`SP=$mmbya0Iq)Z_Tg-DxdR$cVB&srgiScL+*Vl%+t0-=eLy~Mm>qIqQ;E1#2@>y z-X6`#bmr5De(iz&?TLQwh5qi1u^Z<;jRu)?ige#Ok_6tw`M*G%?+Z|l!S<6rUB+2B`d`~>@Z*{G zu(q)$t?J`=Zlt`xe*k$@VK1J}XRpz4CzLG?c~RU&;MMzeb6ckVI)(BHRp?tl8rQcW zFFOBx8Rzhbe;)YDv3{ZRwsV2C^MibD-X8KK9vX|O9;9znj@j^khl5<)=LZA*(5~$j zBWUfB2MkG|-X>vM8cA9jQ@}%K5UE{Gi)s8sX@n^aepHWeVL9o@y_`b-1eG{A+G*Vj|KsO@X1ALy~ z$9ci|Xzx7|&i66}`>fkaW?HfL&Qt^TJ(8-TWgo3Adm>k>6%VZThRBfdOJ|xp!%R+9Eg6&Uw#R4#@XA-V1dWoR`@R`Y$>&O zBY3E-&ci3YLVRlPRnh#cXPB~7YpY+mkNmlh|NnD8SUJl5AmwN7<6I;61C;N%ABwgb z0@(rFJO{SG+?=k}<66`)gl9YLIm7gAIM%TJ?zLiMM%5owj6Q6HV6or&L$ zxYQP97{AuQ&4qgaj{4><$coaZe48j8P7mY#m--*+x&D1F^~ple`lBvxdNA*+x8Qj( z=rL$7HsaI7Dqnlz73|0J=TUez;@Ov}HF@~n&q((pgefZWCBF;GO66FC=SkFFtLa}5 zp~}N3d%6*4v7l=UQMOG8uSWT(pHbZ{pwXQ8Dfq9ogoW_11kn1TO!RFpbZV8bw|0Hr_o=`7oypZlfx1*Lll zW7>t0pP=7jeVWW5Ze~J(#QG2pJUqPoqc_~&e54>tIq zPcEvRe6g*2AL;_mXt>jgmjf?e+lO*!Y4z07`W9!~yJ~551C8{E)=`~+sqeeyTcO*Z zZfQk&jC0C$@$`<#kaeST%Jls zL|XxD0`}=QBO(g#lhaWTjJLg5$1S!Az%=(68l4`2J^C>`ZOr4b@5sO9#{RmC#|!za61Cds;`7>HlU3+|9P_JDjSR9h(7cMu;()$J z2D18U%%OB03F7uYKzTuv)iI<`{pb_z`6!;r=KcVG2I>Mi(th~4Xbaut5~R!moyMTA zxF4+SgO9oMfz^ft8FL!-S?IcbLj=}6_R30x75`vV?Ul)Daivk+g=eh2(r7%wHzBKH zv0omA@=zV3P$x6nTUqPxY>3v*MLDq#Tjf6gv@JR+Kpx5S3{d81^-5RW-^x&3x9~7( z=Rcm8#Up7|iz+{ina%J)tf_DmS|PVg-|*GtW|{qpzb zwd{UlukRs?gkyKhF9xBbzzXx zD`6~9C2NJW9Atr1qnB5@|FCs>)>e&mh^(0?tzUh@1OX+ywAc3;dRiQ;0vQKTh;Ms`D_^&`IN zOuamtseSt!%bb~bhjN#(>@b{jE#Bk6pG$Z%-kZViK)4X^QhAmEe`{kF@{49&p=p3%Jpmlik zvwouuk~BQ2Uu3&zk0jsdXAPsHTS>zuN%Gy@5jT9r+Zk4)X@I9$3f?B0!_V!K996UTVGsrjfo*6x!j)&`kvJ5mx^|baP`xI-I z5i&@(|IOWN_v_Y*6Z8qgg zZEy2pvwa@hoHnGjhs|#ZYwMpssZG25$!!Air?d&InA&Z%SGU=#<0iEUf(>5tw`tva zeR_>1<9Uqm`~sez5T0Me^S6Fac01^o-L2B^@otCw=5$;AL$s|2TpHXQxYfUOnbiX> z4Q>ux;NXF!?M5tmljT3wCf3U8bE#g?h2^N%cGN4z-(Hy!SUnV;GQatEs0==H-=v%jf zevd$}1i`p9WUIP6P=)6s%6L6kA zskCaTUn%oUhVKvG0>2G>EBv;l&G8b=CAXQZ{rTDVD(}{%%v_OJHOHQ)xZLyOMx@8v zti7>}+SQD_XztVix&^d;7&EBNYtX*5Z(L8d3wlHA9D1hp9X%WT+~dG9Ja;m>b?Xdt zpayrzQMl3Hz2{8pf*QO-;WWDA0cm^-8a-c1#ycG7C9QjDtQdr_kF~I?&Cf`-jOOX_=yy}>iI?}g`y;y#v@W2Hw#g*&6Ng5Rz92mlw$;Q|CpW(B<-oPD6jt zWim;pp)cq%c}S-TG8wDUE^-3+VP1oM(5w z&D5b-TR6AkzxE++*-S(_*UKiAXxGVRFwv&CWotzKgOHDVjYB%>-V>P$T0E7D^g`1i zlF4l?G`E~y2Q6BoT`e5txQum86Z)(!o3Ivs$XnBwpcWd^0FCs;Pp4_}BDwhLG`;;m z^CH@RQdaXJt4_Xd*@PnA^|CP%?K;`~4qdxmHY#XtIb4&?ah--fruQS7vvWS^5z$CC zG=F!;4dZqF479^7w7?db9MEW&rfa&P$%W?iJiZ{8&vlyCw+2Bj6-4__%B45tLU+bd zx|0bfyJhqZ>=$2Sl+i!_0+g4oHm{93Z|J4MmeX4J{Z@w6zktU*o^?>(#q$q%u8OC$ zLX@}he7P@E4->6B(ZUo3w1f2Alb!>WH}HI+KU2Q~jgNt7+x4LB#WT(8*B|#BNqTQ+ zZL%5hoDqq0Jn+pv`DtG8XBft^wF|Wwo%^76qc)ZXeA@t+p$h&Q>k@iHi17vtNwaM%pgf#;vK+ZAXHE)XQ9@~+^NRD^gOq-%+ zoA8_f-Z=24avU`~0bzPu4M#qYZu0BC5^+lrz5;Ppz|Y*|H+vRne+Q;^9EtR+@f_WO z!lGtpf_@hMSsv~;djis@GNKJ;kAt7KM1`)Lw0)89gg#xdCFR#bw#|_9Rhv2zbJ2X& z7P_>_wgb9$WXW0E+MZ68q+35SX8@J)=|JF8aI+Wr#{qA7LnfROS__t&+V%Wm4t zzpTx?*>-=`ez!X9X50N$+b%zCnepb@{I9t`w&G^n{e|b5H~ZaR*Jhg8O`G{wwV5~D z?yuVKR;S%;yT4}J*)O=yDgE8UeHNe2D18e2(nYC^&b$BY={~RY1!(*n(?E;*0r34i zJhM+nWm`-E8t;)_-}QHa+#lNx^_+{R|~5V&RskU z+9AZFGY2*F4Gzvze1_+_8gC=cP1WEmntmqbF+B~OS2G?8PH3@hz&#&+#B_ zob{kH7!xT>JA-llPtWiKDu*HeML6$u73HP!Wh0*ZYzoeUXfk_w>8jKEUz1OaPMhwg(V3mj zI*r;8GN3a%5iPW9a%oSr|Abtghg_&GA9FuciHDE#D667}_C}pUl?&(b-Aa|z>%wT9 zCBa#>0O;&p;GuIQraLS${l~Bo?H9%*xoW9J^+oud2&b^Bjgljf=s36L3Ht3Vbb}*^ z=s6Y@-`A|Z4fLvYNFV3BuHd;7Jsa_ivt5_*9MKZb=m@5GI2%(BdT0yX!x2Dq=vE!* z?ON!b4i)kYp)l0tIKq6jF#2~|70%;?QW)CdXM}kmtjZg7=()k z)Pc^)NH_;U?|L`joCKY*pt{oe3Nz9&!uMfI_&J%Gr~{q7nDL_}E4!`5H8mXnOT;ss zzo0W0boPSI$;=5El+^+L$>y~$(0>^s;Y@J!zl)A=5pY`#K7})-bBURr=Mwh=KMI_> z^37SP4@S+p2j?MHbd0v8Zi|#3CP=_-S zr)|TH*V0X0x#6GE$^`BDG^QYpBRCt8hO-iWOHSJgg|hu;(^$RaB+gl!!r6;DoVhq{ z^A*YwFS+YS_4u}@Q?^%-Zo<;jw#@&Hvdv$566Zfo*-VS;Y|Tic?tdeV zaY$nm%JB=z@iEfC8KCXe_03~;*?tWPwaYnIg8j7u-z zoXRm<5}rFRJ#NcidJ$(XYwd?SI8Y{kEhA#Qwobvzad7+zU6#i$=D5T@xZM7J${yW=@ zpYa+y)IsNj&W^r$M(BU~j8HoMS6J))-`j>a``!O=zl&$CSbW2I(@xSE%Kz`5r%dn8 zT+8r3!GUWG{Qk$~-j;-kz_FgoikABdxUGN#1RNsZZ~;dOxU+z}2smEA-38oJz3q)ykEfI2>6hI{~_QX z1$0>4d|AL(1l)WL&+Lis&2TPY2G0AR!nYU8B`enhy05d))0&Wj z{*daqyr|GD;MM~67jTe(+Y7j(fTIO`tAJw#+*QCm1l(J|2?9<$?@FW3G6YvZHX9+l4z)uQzo`4q!I9I@n1-wkaD+MeIc#VKx7VxVAUN7KH z0(JyhFg{0{&RQy9NBYfWH***8)B$;KKs`Ucf&KxLUx!3b4|IzbfGM0^THGhk#23yj{RM1Y9oQj|IG2z@H2FO96i^;DZ7_Ea2}2 z{Ih_o1^lakYXy8-!1V&YC}35Byg zyhFg{0{&RQy9NBYfWH***8)B$;KKs`Ucf&KxLUx!3bPjte)(g~FTTf_kd+&2i!8xNvh^;CrfFZjKAr`ogt2v|hl!MQ*-d(C0|k_7-mLBk6rXkjekw_mR5a92ahm3p7U(#>D?u_mS`p`sTQB zb6mK&kAyQA__pKbK9b%S1lu@4z>@?#O~5k*oF(9F0Y53=c>-P_;9LPO7Vt6wuN1H> z;57n%S-`Ifc)fr(3D_avo8y9dYWU4@;o2NYm>W+J@FW3G6YvZHX9+l4z)uQzo`4q! zI9I@n1-wkaD+MeIc#VKx7VxVAUN7KH0(J=a=KBTJ2X`z_)XpOLdR(&;z9s>;60o0u z0|gu^;0^+g67VeojuG(f0=`4Qy#(A(zykzq5%3@Z4;An|0=`$kV+8zwfFBZYx`3w& zI77gX3ivSrKOx|!1^ldl?E=mhaDjkV2zZr%R}1(h0ly;P5&^$1;4K1vQ^4BdI0 zzE8jp3V4EmCkc3(fM*CeOTgIzep0~m1iV1NxdL7+;AH|{DPUQ^YXtnVfL|5xdI4_| zutUJ50^Tm*9Re;F@W%q)E#S`u{H1`u7Vtp<9~SWU0{&UR)dK!iz_kKCE#P_qUlg!< z4Vzrngh<%_p*V-l|KW|Z+CA|tX_c|9LjQX2iF={;;(j&#F0&}Qv*FyG@@{;GOm`}t zz97-P%xT0+-*Ls?i2tjeaHFDc zl!y3TY#F3Bhb}#ei_R2-_@!juv^5^3_U-I#t@LBTl9Wlv)zB8^PztE!A zl8=0UBY(3+{gpolE62!RYf(uKbl1*N?uRHplRw*{{z(25i@KZB1C-`GGww^j@m+6! zygz*(zdv2S!*&b4#b=9s%d6;aJj=Da(&(Nvy4#lS-O_nEk8qzF?mlzJ!#!wb+=CV+ z4Lrh_<)7{X!yQ`LlHbCJs8;C_H%wE=7x7#2z5cG>7q>O9wcy@4UXH46EqAZc-2x_N zFAp4OPrHbE>@f6@P51R$$67;gS~GPOWRzjBmlG{gW@;4l;QIT0|4>IEZA#aSbWbAP zR0-+E#;(Inxws$LEt@~Plb7!8BH7aYiMXS@&qc;|kUq|a?tS`h^IF1dTC=kB)~wfP zl#lA*q&_2cmLJA5-DOok_r=L6xc4JSp>O5$FF_wc*T3B#dYN=1Sb2c!K%jCD^p)rZSkUJ!P|m@6dZ# z)IL&B;ZfYl8!Rz(8Sb>B@Zgud@SS_GqNgzzY4CV*W=??~0|onOWF_LkRccOV`;Pjz&czZJ?!atlNJbkf(a0^}bM4qdHx;#-kr zC`+syFUR1n!8p__PGWS2VJ|&>hB}nIllgu%YGkpD0a=5eag#nrJ{V?aI)!eL)^*chS&u@ z>tYuV+c$IhtCh;V8yofw{71vSmm4`f4s_g!c#x-4pRC47KFfXQ&1|g0-L*MVU>*9= zm{i=MmMWP!J+rxahpzuyk-uJV(od98odn&EjtP_v2%~nnk3;Wrwj1YRFWl{APy7-2=lEFa3ZPTemww{;1oJ*i<@^QD)X%<$&rnV( zGx6Pra=P0&Sow;_2~qaJe-W~H0%H@|E}E0|^n}jh&h|yHJ0_I1+E|eo0eVgn?ji-A ztkK(mmT~6U7ot2oYz{+UA>9*m(r8jJ2Pln_7QLB@`}Ec~H;>7I4YQ-HIbI7J?9dV7 z$+)zT#~UV@_A5RA0 z^?F+e+3`CAh*lTrNv=?gFVmS?GT`3Ar8rZA<`iE&x2nA8rfWm zOY=e#3oK+tQz6|8Pj<5}&o4yDfFFbMT!3tdcPgjTeW=N(LlWv@g+2ewZOi`;>Nkp~ z8=$2AN&N$R4b3OH;5akZ=iuJplm*d9u zyE^*=j9U`wYe2n?s6WQe`JT9!Sdwwab{gzT*#)%O2KX0^9%wuFeZN=GhTQ&Wce>8l z-n`cCL1RpS5}@Vh&;0eeDHEB`5LB6C{PNYwAk3zkvJO?#;4d`wi3sfFL8@R{q0Od<9{0D^7 z{i!5}F$nK$3@`L#WTQgRM)XgiFy#c=%MamCWA07?}T}EQmX{a1H+YHw788%3+^{&fmC;*$MSSSmA3HTzD}2@Z3ECp0wh+Z-`gF< z`Dt8r>ytP1iEN7r>(y_n-2bra2p$^4D}HY-+d^}@i^(e1A=@RRdrWhH7P~uG9%yFj zK+KZ{Moa24Z_uPWWFNLJtW=yInU$Z)!D6J<66VUwSkRm3mfML8|M!j z=OL>`Pqdi__9Be9gVKQi+PUpp%{h`c)+F2yQhtF?GAvIH+ZoUn{i6*lI}BIehV_Eo zTg4vhB0up<^IB>z-bZrplFiXfj`U#JG207@f?-=|UhoytxgT@p%lL2DRb%_Og0@UH zxsnb|^k&&1+g~Xno=^0?aHqYvS8@<$b~wl4Uws=H zbPxZO!~gN1A6(~Xjze;zHRy&p(m@K(!?Oq4Zw2zdjJ|Lep50{%pPeC0wX`+i4*9$RSEx_sp5g2do zaX`=beCS4Vp-UrPW8TMnU(T+P>v6i_cJx8Fg(Rf7q z(GO|-!^nE^^*8DMtriE_D4IVJEvDL_qW%X=Qk)$5s;BCpy_nYcKku|5>QU60^dzV? z!f1Z$&7v$v) zxtSn8Gy0#8!Y*{S4gwp!bCff`d)(_0w5xy@|2)IOIfj`XXrVd3k%~s75;n+LeV157in&<6ui( zpuRye+yfatn`2(rXbg&5mh;`ZW}~^zg0YA4{sD2wCVa>J0Ob?-WLqhWY^!U2D(W^2 zbsUPi4ndt$pc{i7kO9|?^PQO*kn`R;nxoTR4#hj)*`l6=oT_j~;|17VC%!4#0>2<9 zbe%Jt`f8Z+0m|&ecXyG@#LG8lVfZ@I^D@vQSwP()jG=p>OZr|g^T0k%gV~&*1%nYeRgb2a%|mdb-ao3 z_=+(s?#gTrhYFkB1f5YcJsn*ki_<;59Ch*Dj?@KV>rT!#In+$EL`ae@gEJZ2;HNP_!4>*bs$mYzNSZkM<{z!48v+4FN49r~Nu- zoxyPxI(Z59)lRnd4>mSbS&4R`xS`1VY0_7+H#e}iGng(6u zbj)kEXzNgYen4}=>uoS>K{;&MfmqninAWC3)9xWRjK|ZQFuWXd+}^<0v+Y59w&iiA z=hA$hD}6jx@=5jSVcA(&x+2Z@b;skWl5Z+#aXuX_38{zKC{wg0!89+d9p*#X7W1IH zSghq4yBD7?rM-y!=gf_hvDQtPi#Y|^u+JcmC*Cs21qO?X^*!u@NqG;n953wUfM$X% zwKNOzbGBGrjSQd6zHdU8jsbN^8u!imgm7#=NT- z^R8mdqgG(vRjSRqXg>>OFC3`NyC_Yf@pLh#8-_H8Nle99P+x_9Ey@;lXcb!>B#%Hj z`JQn|aoCsW-`DQ}@%k=AeZ4FRn8WqWFjx|tu=^R~ED2l4zu&^gfs9JqP800PYShV& zbxsM|GaNcaxEO1m$4N#D*})tGo0pR&J)Kk8W~aT|V70iLr<#c{`<=jdda4ZsYmnCt!|ig(B4cUEA`dvv(vm**MEj*dYVjox z^*)N<1l;7IrV;*M#JzcZmBsb=KhM24$-N1Skd*{U0t&f7WnZFTZW4h2iWPA~D*>tv zP;9}if9TOH4VCF-L+34Rw`EOkjGotB<;@$e@>=V(erPT% z*ub5<^xHbbGA zR<8ZdEVZX;WsF|=A7{RWDJM*swUjAy?E}7%Ve=%6d1=}am2Gnexz zV+wQm|3!u}bc^~*J(I{Ui8@QaR?^lVs&CT{(V4oT#~$c1dZO8}thtua_X+>n5(hqd zmbb~lJ5LK7cmZ5nA!CXk?mj}7q92qt-b))>;>LyNz>QhdD~cNeN5^hw$kE+EgY=BZ zI9H` z^j}L{`h)MgxKu}f{{MkXZE25#OT)pJVc^Xbtm8$j-;Vw`%IA4iVNkK3E%>7@9AWl* zAN)q~Ab8_wCKX`4gg&lz_IT!tT zNBFoSo{Yfh-=qglUx0rFE~`^u)<^MySx==0X8nwCPp81?vUoMF*b3wo$E!ibF|7Ug z>inuP-N8?A61lHzY^hK6_hx5x>FHDB&U_FP6#nN0`p>NY#B+2_>7nN$VOt4v_9;4r zf|0P72s>ULOWLH+Q<1REgw+;t?|M&}-)W)Mk?=ne4qX|0hI`Hbjo#}Fy!(ry|H39? z3-UmHQ9w_@4~T5gL(*o1R*?2Z+p|nZbQxD%EZT%Cxd_zClWwmdRGAEyD8h4m;S@W#;1~>2*BLr^O zupap}^v$q+qdLSdz6#yTvvX2cO|{y+v4(g%PdjO*TJh$)SiU21M-zAIoa9v#SZ6A9 z)40#u!$bec72Mi1DA+Vw>8ISTJwj9JsKb}fxP$T@!bx`ygq%@b9T^ zPTYgPW!@wzeTcTyM&bmBBV+$jk;~93$?xDJMZtF?{|ED5>~F%7e=ntrJE=_raSB(9 zEzwTa9IrjHI@70SR1j|@F59YDSt9GHO18W5?dGk3U*D>Xd^quPm8ir9;DXESPyXqw zk(rS?nDCm*?D?cQ5c!wJKk4t2NorFKPUKrJbTm`BqI+FqMc!Kj-MaF0(+WRvnd=+9 z%rRe#$~zvXbkZC}U$L*z0G|ZEwH$s+1B*;@l6J|zM)?Q-wH*HIsQhCrC0tRnE=YQTg=bq1&$b`9 z>W}vZ7j}%%6)Cdkn-RK!HkPFeoE{n{a2J6)g{~2}YmEMyxgha`CobzAqo07T%@A zc{#&W-f6}{>im?n51x4?Jo9ScW8jtGtH8oPKMMc+II#4^6A}g=y&OLJB=c&i#P8w6 z7g+e}N8zg<0uGN3=!(moK58dzF9e1TBmhf2%HXvJ-O$eO&sS$h(su4t)`!t~)OQQ$ z+&_AQ{;QQ9w4V0=NmbX=et%JXO$w{$oPW|8x3jA93gd{{ie$irRkL z_G_$P=kIB^1>P)A+A8$sHtHsAnJhkSohUvyG(miD?5E<>rZ9c*u32}~p5s?r@ICBx zmMc3)xf#ExpTYSXYDJ@-`|0{U62&Eia}O+yafigm`8?KR(Ew7|{iNdKftaUb1DHj-&~a#!s3o(gZ4sR$B{;R>LDr z$2MX~cIjaI<#VL-kgk=?;mlvwx%is`dC0r*@q5N>nE;JBhU^Cqme=3%#+RL8uh5kk z{~h_lt2VWk3q>!5+$g%tjnVb_>JV~8tm^+u@-X4>235}f-fq{zLOsb+n~@JUE?dyw zh9A<`vd&bTF4ASE-Sn9^R8Kn=@aIN`EME$pap5 z<7Q-pDr5xTRl{_Dc>ScSVzj!j0GgZ>nncstJavqEhr1(Fx!_OW zKU5oIFJjtT1Mhtmb*@F`YC`6Eiuy==!6kv=&o`c;y(ejJ4SeVUWS~`qvp>hYNen#) z-t9!5+Vx#|YIhFu6!dIko;%mhUlk)diQV1{huUkXLz1;@W!C6nj&0qQzaP7Sc`xf% z4S3gD2ku3F%W40u{FY03FS^x0DLbEZnMvx%d~_wzK2C|uhaB1@>*sycRdnU!BHwrO zE&8!vNgHs;h(Tno-PwW5Svylg9>Rpzm}LCaPzgS=LGu_Y@+XZpY3>?oH`ZfofZQhf z&o^S#vMOXYQ(t7ZIPguzLh3q(d_>Rbe1G4TP4=o8bE6wR^PaP&jv~!TY} zGlVeddkHH((`4{ADtjVFM`eQDvd0#cJqJ<;(ZAD|A~){#*RlWht=x!y>;Hc5P3Dc4 zvZRa@F(>>xai4I){@Y%G@b1Uq-5nbe z3qBs3PoZ^w^k0loxW{*C#T7y5Ob69Jm3n5fzgFBqjavzR79X0L7d}r768e$$`-JLZ z{v!i^m1Fy=2SQ5(C*G#d1(!ZytXI+A+3S<+QaAJzJP7d zk+c#=@Nh4_CQRxp@y3uw_BaIon@N)z+JT?rPG0QAo};uq?+hWo8oE(+xjlz}^w*Iv zG&BKxGIH;_Bw-3{m zx63;04Lwad36GcXsf15;!Yd=;9tppL@H?FFm67mon2%L637_eNFC$#?&7tokpD^p8 zq^lC2c%}HJE=Q@$QMD)O?iihTUyQEeTXbHcYugHbZ|#UHY}Z)ZP1#a6@w4FtT(sp6 zd^4Ap6b1&^m3@2Kb^H?>-De1wbwc7U2To)kh;p0PKhuZH^6mN9x5l-Y)3P?2f5$Ix zwt37Vz2Hii_1>(9;FX_v`Ru#S_ljJT%3Ay5dpv_kf5V~NJo5fZ6)U?Pd6u1R+FT-h zbw#QjFLf-#ZWS6_9cJ7->Kfm;lxs7stQr0KJeuBe?fM1%Tyg0Yy6jSq8LvwvFUDMS zTGIAg#`bs^n zY-vMu@8|>S_zH7d!eqXrcIcV6h41-bu1tf5)Pik3+8aBH#Ov^V!>*oq!#f z%Qo%)NwWP9;>$RFK)=jBlxv%IGXBEzs)_8CB$N$(hkvKmb%!Qrgf5r191c_?_s^&Y z?-!vHrdz`em)*P=rhLR zA?ROjpU|rDeFFZ^kWFs7uE2lejDp~mHx&jyWB$RTb?{%PI-+m6W=gj5OPxAX-V5M( zog11>e~hI~wTU%D)w>hN)Ft5md*U75cYif^KlolT6Wj)O%9%qI)IC$BHEf4gS5f}~ zq}^H=m@yJrI8oXViTmpoaVPO_bFAvH^P9MVi7m<;aE>y4lqqB5Ju)>f+P{i2=-Yw! zgY)t>*z&KNHYdla{!heXV;(8*mKNn5ME2czChj|R`;4-paj$6+cLB0-v~KT#3qJDy zPPqvVZS`8y@+yG$J9@0$Ub|3N#G8GJ-C5Ozx(*&(Y@!Y!$9&XY_7GD-q61RYaU%78 zkNV2`;eESM=lEKu>qNRV6Tb{S^L50Rz0-`C^aQWl>YI;UL$bRfy`p2T(bsrKUbfh` zZR-{pi+n42TFHr~am6R%2KnfxeCvuC(%&-XZt{7Kc1w73T3^QE_(<9bq%AwvG_I6! zSxMU~`a0w6YuP_X`G3^uOCR|~(%BG{? z58plN-T&!PH{nmklE+h%GP6`srpozQm&#amwklrirIcM2k34+K>b(okZaBK@)E2_ zGc(;Nv(P)aOJUl?E(%}aA1qhXnLAFuRHWGV(}oX_{c6Zd)#lpij1&Ei3^f{BHwv0J z657W*SwA1a-sJFLeYauLp^b*G6B_wr^wIm(!m-7!%9&L=(zgU;&G>h!J)iPQSReY} zS4P8GJ7)7;LwHYoSvO?ONbT?Jr3-yF=?ACT_w09g$-fX!Xl{vDZNkQUiT4;d;ZuVQ zk6x5w!wW5u{nrxN-;FC9y3%Zx*C)4ct@E9VOWtlXIp%VNb8{g>Inz>as)v?D%9%(JV?gL-$ zWIZcky;?+`%#$UX59XTvmQCd|RUkUFd>$BjS@IzI7SvNS(vyF9weX=EY zcI;{2-O<`pxzvd#Y3C8Ia?vooby1A&D|tK&{BVr!1#Id!@e+r}x_iOUOka%NMV2C8&W0{uuq<*<3r9{=benJ%D=nj$E}RKeEny(H-QWBglte zw-Q(bcVujlasB2K#zf}dFWmO4Z#($iL|APHwMlsFm9dEl67R8y{fH&DKEMf|Hw-hceBKHIxQsl_#XHqwqWZL1t-&*aqx8$Qhh|*=;kM_yoq2dU|7Fh;TD`;%-v+K!kFvZ8qu|Hh zfJRP(cWVP5H|pYNS6zFJ^7nMBu@f$~{HzfjuA5^8C(LmLb7E9=bpFXaTScBS&z`~0 zVw^=6C-`z3--_>Zo#h1A4GnLp;}Uv$GieMigX7GjBPAV+PnXd?DM#pz(8C*vBQ#RN zt^{`Wz#JRefuERtttH;?*q?J2lyMZkxEQ->lmC3=RL(L=9zvglE=k@6ERw>9Ikv7e;9 zWgSPJE^!r~W{om!eu+9r{?UBTbK192z3s^TLNm@_JM}|!L&mv3ec7LV0@hu#7gm>O z>>HQ(p+^gW8);L-{(Om7Xvq0J4Gp;#8q)7aXh=4DX{9c!Zy3F~^u=c%e@jQ6{g*>W zK9{ib#MWJfga+IkmzFDW7qD+F{rxfjowY34tUodzzDr9ki_j8uNzjtlm;(ooIqOw9 z`N`V0hcH7&Skq1$yFUB`=m_iDA-5BEu<6H?&<@JGr&ZstX+cNioTAhB=pQ0!ULZ|N zIwI>{l#V>hKf(2o@smSU#!m`8jelM%IxvEC;NX&F(19}Oz#8bllf+vVr2`v;4u}m| zV(5ZsT3xzd|1iUJBfge=kMO z!u19|guMCzwiQF^Kiafc*6wutZgEnU=+N9DKkor?g+l!1isw}u1(`S-gQ%~;OL*ag0cp+UVF|^rr=u}>Mi|Uhm7YXeSmTQF8x&@ zDLV_?%5rRsHhSqZkpqRMm(Vw@=jLYeE{o31KNp(0H=KAmX z$r#I8qkf>wqH~n__&Lh-6ECYpnNfaHA$uCWamRP}&E$-w;FOdd+1nX-9DF?=eBHmk zX+?A$7)RS_e}98N?C%IZi4CipJ%lHP*I<2g%5~0t${v`M=Y@TgT~$(_(~N7>cJ3PTyNkHX z*gtuU{gW5OhK%_+hVZBjTQTq&;qTc)sahSQw_H$(I9IqC+wY$fU`_wFK2_+FK4d<$?(e&)o7nwTp|1(jhav}E z;0v7I2G5}ko_)6-t^14lW$ft0es?Ehbgndl`#)toWlo6=sf^`s>3f02p7g^**0N1z z{E|WfN5}77V8;eADbzr}$Xyr4{)@7bw#Dh){6B~N*Am$ulRn9O&vni_FLlay9Cc!Ea+68( zuNY+Mq|m~Bjy?V{_oL}M+a)hn#LcAsmBih-n*F_X!{Be5?IjJ)nEL<6rA>VHyGA>> zvQd#|8Fi3+mylj?rZrB8&DM|EYE{vm=w41$d3vL?-w$6c?XM~y-^d)^=x?Eu55U`@ zw@VVaQ}z+yTQ}6Y)du9g+@L({3(L3pi0-7MOl9P+rz#=c$p|UoC=lEkFu7^K5rPkhPrQ7qwEC^XwjeJYELfmqF->0IgP&lbQZ3Fwg~<@>sxoT zcYg)_T~SC|+51w&$p#n9|3c^gkXHX|nJX>V_4;mw)1&*VCfy__-6f=}`KD>J)j(Gm(GWoqwJ9mqnQ58AjJ`(j9dEb>v@wF!|>d zpMF)GMQcNN+3BXurQoW%Zn!SuUhl*%zkRc5g~FaA?6#(hj-Y^a*{^gm_&=O{@1*QI z*Y&kaTmgL}c!$ls(Y2P2$3|?dSI&6J9wWBq-||Z>`&G&{ckZ-R1Ak06)ZfpVc=-i( zGByx*Qm%}3+0t7Z*DUSQ_#3|0bj$?K{Pmi>nOl|}v}|^|E0%-+hspa;tiUSdb-ouc2QZ-r+v>jyGSm^mc;PY3$75_x7T@=Ph^Rnp%3 znRDVVfQE^`mA%x7{Hwa>3LTt##rJ660oLgX;t7qDeF|A)c8Z-jbkIlNDdzGcoU6*h z{~dk+e=&Xu{zCjR{NLhN;y-|2gI|tci_e+X6{auekFNd;?K*V{cm&^5OL-`artxPJ|O0e`L1{F1L_9d zisqS_r>dPj*tQ!o>lAHPX0N`6Gob1hm)jqbS0#K|Ky=f;cG>UqUGkjTWWFzO+xz*J z`6+wTQz+B_XYKF~c?`d@x*;BK}1*iNzz}etHbp?4%!HvSb0sZ?0aUR-pku81pyT6Uw zQdZAi0`s9A^I;qFK`|d59(rqVNlbx$$XB30>*Wa^E^-II?7^AW%Pg@8*_i#CI9qJP zKX)rVm$KyyeirVL)lDnj=Ktn{JkG${2LTWyOp#%NV}N- z*w^Kwi+D=@UluJlZbSYW{;@xl&%WsI!9yq#t z^SW$%g4@b7{rhRzP;+O``t$8h@nvHtP)21T=TizTeGU0kE-2Jn7r68ofwvdwt=rvt zw7}S%ZpH4jSm46p#E;d(fED z?A897K1{ne*G|4y?Wv5VyjYj+OT5a%)Z?(FdkOpn@E4Z8NZ_NuM=k9a_$2U2OLrIe zH1KIlcN6#<;BPFQC9sP;T`qmT!12KGE}brLJK%OMohoog;EpbxByb9Fic5D8I2}0M zW%{$UHYWJ!-0pp^f7@)0grO&uLK?sJl>^07x-4-TV49Fz>|O{x%8(3FK29)yY)u` zuLNG{)^!4}23`&SFYr3xb#DEEz)u4|?bh!K90Crx_1gkJ2mG8{?-lrY;OE`?p90@b zS+~3NKLnl%Jk_Ol3490e9WMQv!25~6->r8DJd^O5=u8E^2lyVB{wpxJkykmFcyryl zTK+8nUf|X*3j84OgYattKMj689iul2yom5cZoNU^hk+k<>yW@pftR}V9|e99_(ivV zO5j>RN$q+OI`XA zfnNoF)ukU2csuZRmtH9FYrwC$^n(Jw4*a@HmkazB@LMijCh%V1y)HdR;Qhe+U3!+l z?*qT@()SAdA@GMTeV4!ofe*U$41qrZ{=}v45cn|gVV9mJ@N(eg$h`uu1YYUVw+Z|O z@E0yUN#LWvM_qb?z$bxEy7Vmqp9Vhd(l-hG4e&QEeZ9ag`q1Up*9sg99P8FU5jY+= z-mR|^xE*jiw;m;MN8pZbJwo6V;1svMLf~}Zbhj=LxC?L>x4vB9uE1U0`Z9sL19x}p z!2;(3=eqR(fmc(X)y#i^*8#6{>0Saq4g9oAUnFn{IONiPfu93@&ZWBx{5ei_O7XlZ$b&|ltfrq( z>(&VZ4+I{F{!ie`fiHLK7=gzCk8x|4z+-{Ox^>e5;Ol{}ck43-hBjT#`9Fa#p>CHj z{{`*~+?V+;@Ic^!%zuF|1-_K|FYx8Smoxtb-cFv|-TG63Uju%P`7iM6z^^m^1%3z4$63HT+q{)@n`0>A3kn*@Fz z_ zq}}>bjIKObpoa(_j=%MwrOPFLJZa-Sx=i49!0kMGj=&v(J9_jifm47}Jo;XN(}B}H z`YwUH0C(}|83K0&?&{HZ2#gK%*6toXP2gPMT=;*1F9E*9qi++qFK}Ovo+R)<;DH`J zL11j@wqkpCi@?~^ZN<*+CV>lq3qAUJfrkSR_vmW{9tAwgqkkgs7~nA;eU-ptfya9E zD1omBzTTrp2s|EmyhmRl@U6hNdUS!nlYl3A^yLEI4t%>uUncNW;He%xSl~N=@9^jW z0?!1V>Ct@!Mt0eHk4N_scoy(1kG@D?F zfI}Yb75F*e=R7(_;OBv#_h^^EF9N^l(M=x$zXbe}N1qY+Rp3`S|0D2r;O!oLQsCEs zU-Rf=0>2LYx<`K{@LRxddGzN3?*-oL(T4@z54_)_KNa|W;P*ZHBY{5z{?McA1U?9S z(4#*T_!HnyJo*EH4+9_e=ywJF0{9D$ep}$9z(+lLFRn2#`@(#ukVabFg zEB)J+VVwx;r1XN8VdoQezS47AhGi3$t@N*3hFwV5g-YMqGR#kyU+FtqhFwh9#Y#_U z8P7FgarVuto>FzDVrV}<@>FkzaGYFfZbmx{~cN2EE(y1-O?j!6zr8~9^n?u+frQ5a) zn@`w$rM-k9yV+@n;_R(m)sgJan#Q5uson`LdavnjgQFdh`KrOa>N3?hX!d{5NAoZF z312mO9b~afY|+(Jt@G^J8Xv*^x&=ek$GG|hL$`94tcLhdW@*$+FV8qq7q(9hZUYj5{CKWkZ&R1`L=#RQ17K zjPv6z#AV~o$92Lb<2vBl;(WLm-2BW9S${SCN9cOfntcRsEYE*aMW*B0l)#o#R5S<-3AV*z>L=<6){Jd3{1`Va0s z+}pT4xHoaTaDT&X!~GRkO&%AQQzq_0TsH1}Tqj&It^=+u&WDS^S-7*L(YO<&|GbBtGf1i#&xQto6>Q2 zSnx0WWB;P6dx6~Pf<0~7-a#?w;at6KX!>rkw-KB@wN8P1-UiN@jobQ7{t@gU#|d6F zfsbRs9l?p(1@nj0E+`w~-7&iQ7&`s~g;sS9ywCx7p^?O?S?BOV*8v}Z7pfUmVE9_` z4~(*OK;rZ*(8pO1#Xr#3(w9lt`vv+-3By0|zNH6BSW1E6YsEj1V(C5-cKZnZvBbka zaC@A-NWvZjelSk^1>O$4Jx+HQcsTIzINeR)sU!3|k`DjC)HvNm!WIE9iqjbazXtqT zoK6#X6!54xoh+yCiHW@X|Ql^l#v|fZvMKX9OM#JT^{if$tfiH%L1C1NX$~;}W(UczK*YBJf_| zy>a>rfv*R?K2A3XJZpqrBkAxD%!<>WNZ3l?m2vuz!25ys$LU&u#{-X#)Bh28?g(v5 zI{X83DMJ}9q_t1{da*s z1pY8i|Bt|vfG5T2Z32&BEXP>-Re`$#ceV7(0&`a9z*tLf0Tz5d;IgWBu0t1Y=n(e1 z2SqPlHuS)7i*a}K_RikV_unZHTb76aZ6%?@9=r7Px%XeIkmE`_s!c-g5^pQg=-GGg z96q|5{rEwgQyLWZoHuA^Y}%kIZ`PoS_?$tdZF&zXZZ~+)JV(F0J89x@^m$34N2o7% zi=c;3VxKxO^Z?&w?Axy+e+et3Z^d6gxa41aw*5fSPbV$B!u+4tD*fH9(hJ`!>1RgL zmwuh; zzq(cWVXe{&-)s6Il78ojUQYTaob>sv(hq2r{wQmbr0*L^A3iq7N&kB%eXmyOFKm_m zU?hFlXnJd~lYXs}KC@N&wR$cs5X{v$Mzh$jJ?6$0i37a3|$tQybPMW8JfH%LX+1-XmTYq z8Jey?|G=TimC)o$XtLxZG5N% zbNAs+?ii#zZ-Om%AWlT*?02U(bi=Qh=UpZH&vM6OxV_T6#K1fy?wJI8EajFk*FRb> zHjNp~ce!hnI|ADWbf2#rdrF%)a{e)q@e6a8p5#~0e?RB!%CVzgLwUvcE-QU;%Y5XV zPz-IFwbJRUd69hNe&qLJ=&whWedS@Nj9>Fl?sj1hW>aG~Ro%#5i1g=|;O3`<=a44+ zf%0QZSN(Ch)HOX)SMJRN$Hb2F6Z(_AE$1%79)W7?0lq4Hs;!9|+u+zvoQT|I2*1ES zetPH{%B`i%SL4UizLsNU`lzyvZAH$3dx+ztO9~a4bcrFq@smSK!O^B7IXe=^n0XmH zA7dELSaPmzb{p`~ZJ**^nf$ymPoa zlRIH6q#p+%lZk!bpb@$~{mY%VN4&pTWA3hn-#?Pd-N}NJ&mDKhrUPZm*tm!<{PaO% zkC7g7b4FA8J=V|@IU6Q+rWxk0!XxA>{rfiaqUC*s4TK9FIOyiA9d)?^cQO)J?zlaa z5YRs(td1~!r#ooIy_?dfz+1WRw${C1h_U6L?BKu+%r`0L;QQ=TfyZWEpW|M`3C;H! zriW%l((EOTq}`2wDP@Yy?VBbnIrN6{lR~?UpBVbP@#Q|*D;d+#jO{4KcqC(8%={d| z{EVC*D*tiYX~BWU$lXTA!GSXo8`9`Ksx9e~TFIU>cP9wWRmZALD{*(?t|x3TE){p2 zepyW2xy){|T z4f^aO?$-JkJd`_Ne{2kTaE>mDlbmxHBseQ;s`Qhrr}8$0;O;5%8uvugicg@~a!yq4 zJ~~L>e9U>6j~IWk50$#I{`B{PAC>LZrun$rBHss8%rsYfHSPrWbIV<@VrwmH_8GHg zJNLp0KJR|d8KYC=DR}h`Z5MnG6W($ievk0X;sFf@u`&7{z6Q>-ucMCz4`j|C=&NRA z(I=T+HKPC&?C{!8^V4wc$k01b|EOX2or+ zJr%O%#$ofwS=f4aM($z8a3r`VXVObqBg5`axre}^()+!u%=$@w^AzhRep80xjBV$T zSx0vWex{lAGxWIA#y`jM4PGk!$$H7R6JsRz$U5UBG+glWQ~HyB5nI;&b-g0^IXKb8AJ_sv!PD=rlY=|h zYa=w^QtBgPGt2m#&%>877rUMx!!=`r+=5p!>u>Ll<{lQgbJ?L8okBk&zi;P4n#o_v zyERhY75I!{=p)i(!-LAWX2FkT_Et0Cb;fa6b;d+|Y^=T=XLu=1*&0wV5yAt$EK-zv?z=&RA}RA znS0!q?B%{>bkRq=+=VQ9yi#|XxewXru6f_zq2~Rx3wIelW7$y$k59ep&{@Gt3*2@1 zjErVJ<1^MEp{)*|!F|`%<(){_0BEz}GpL8~6nhAl^VUxLVy-o9%f8u+&G*_?N87T# z-?lfY&v@Nr`d044m$Qjco@G~aUEo>J0}IcxiLk~NJWD-1PW?Ig7T$HBkEAV|C}Z}I z@GbM2`Id~(m60@0kw((4!yZ{(s|N1XtxA zMxjH0f#2N9T$1yyLaRh}aMqL*?iH82H-30+fAQ?M>q-^+iDTfu-2W}>i=3;GJ7a2A zG_5dWN?TzP0Mh6F$B*(i7ChC?wz8@;*R=6DUyRLXahXL9a<=o|`t=FHjnNS{@TJO|J8 z?k7&Thj8H$O5mlOyXrcH?gxKmosvEjz7Tr=fB2DM{%~Y%a`-Ws-@FAtpMDHJ#YWbp zKfyOCXnBNh+Fkf~GV4-eXvwDz-*g(jNzRLZzt5BGweW(x!;$LrXJY7I^!>N}n-SX0 zw~XQMg@@Yl<2LWYLx~J0?~t61;HtbsBIjcwdWGoum_g@(zggTTC+o*4`e7gLrHGy( z-<>|K5L>%}d?({pQU~Y0&{Q+81xH^XtQJ|FHE7du_+BIDpG1y3W(9{ZHnPUcyw6mf z8_qD^qI39O`#(xMxSswd6sa-`zdV8o$(YtBg4pb=SOg64sirG$~Zq-DK^tB-qaAf z0~U~n`K*dF}Cpj&%mpcbX5H-d57=P2;SXJ-ZF<3deAU$ z092QE+H&&dHvW2ObPj!1zn(Wy@{yNC|H0jjwMFV|1$|&-OWsEbU#)bQyu;n%gW;Q% z#`bVYpw|dJ%*^Yg25;|WW=-qFnpVNPKAa!hljM7kbqu_X=u3uChcn=&w6T!1W5D%; z<(9!$cY-4`=MlEvJ)V9`44r|-9ArFXJrllu-Fm0*qHE11gn8X%)t%Y@1%GzS{;zk+ zkX5ccS%aOsg=GCnBHr;H%G?v_$jT24(eRY=E{v?DhYU}d7+M6*NPR`O`x$kXyd}Rh zlV56RDQ%T^B%C;D=55qd$a3-y!G)Cb2|V+s&_mAlEX#rxuibIe7IZDU3;###)H6cQ zFcu=q9A{4UJ=dHJrx;%GX675b#S$;~@>eM}z~E4As=Z-9c=<&W;|H(&$3QU3D<6Ws z39rn0fvzcIaZ4N%JyY#t4sHLK^a17?Z$gcgG59&CUkbndqcd< z{X(zed;|KDQgor2maAa_^kD+JnQ`c5Oh2uS*oVW%neRGu@7y1gN*xW4Rxl&ai)>jU zx}88DdmDE(W&bPH{)YN69_9^LY`&NH*JA&xzqR=e%Y8JgciHgX*+0o-43k4gNK*sd zNu}+gUlZM57Vrz8v!5<0@Hu?O_$!EV)_KZ^^*Ye-pVa-Z(PfOIJ@@{XE;Y)VX zUd~I-8Fw4<22A*1o64iAC1E#pp##0^tV8DBM}fIO2!qnHOmKPU5F*8Ljt zlkg|OYiZj!!lf+{eVFC2f9fK<_>1WD9_Jn<>Z?ycw~FDt zg||EX%Q(OOrPpux?)9GF8RkU%oX)Gl&+taipzLhoG-NvC^ z@unYzuEYC66NGM^Ff@fb6X`dJ>+t9fFaCx3$6Mn3(|5Xpa!1SwWCe4NjhnZi%H@og zQ^sUyr=jTA>emro*woW<_7gfjhK>h5ROrUL@TM>09>d*7 z*bTUWxR>CaPs1BLDfNS2l{O3A5WL+?n8TZ9gw{mDHcC8r0nzszr(Z;W2OZqBbM9YO z!9VG7=;S4jRm9KqCYg7KfHEFK~Qvei=<5Q ztT29Z=nj10nHnC`U&vIy?WI>Q&v~9puj4FglhfKPmH3)*)H{k|TQo zcTpvMw-25q(}p=xuOF z);zJ{k@w7Bro7p*&QABQieX)J^m>lI(8wh8&nNJwBIopC9V~-~i>8${NoFlg3vG?$ zE9-=uIdnl&8d*Q&UNGkkCRt0HF`kV{I}Ff4hV= zCOUTB$Qa2ux+Cz3$oiFR#=hLt;e7DB&{ zgZE#9_p)Az-cR&%(uW@M8P`T_dIfy0m^W|VLg?0K?$q4Rpks%@zcY8n`5T~T33C!w zg*m^|&=Xz$yaay*I27&~8$8@2E|@c?&8l!WZ;<`!CGT9~(5x^1CAgOy`po!gp~J>c z4IMJL+!@;Sq@i8u!iVuLqwry!eeWB6LRmzgdx$i`lW!-_&lv}Yj-`g8JlMO0$(^Xe zM?3f0IXZ>ygbUuk9w{>_gI+=0m(9QA(2nMRoNZ@Jh5y=Q{w0N8;a?cpU-rpFKUmXF z-eX!Y3 z<5i63m5l3X#&;C-Ze)=0cIaKvkMEtz8;e5Y>zb5)8Om!oR-3S8Q^VkLGrN7AenD{rNqv*A-_yVJ})t(jIz zLmhlwso2!QT04JL}yhqQU(748xVD9j~5?)}j zyrECPHRk?Fi2%ugARK)E}C%F?!FS^r7hL>Uei0`&XUqAZc@`f439nQ!8pXq1r zcRH3ETy-2gj_&c_DSP~dISrHGVWWHeCGcVM;H8d}r`W)hu*YA*9>3%(d;HP-E7)Ty z#QksbcOmn(&i~&(kpGGu3Dpy&|KNAMv`P3s3A=@`GVaOiNuR%olYgduoClo8*vh^0 zqKA1-`Vl(*-}dO6%GI8;v$=D=NcIIBoatxMo_p-&-Vf3^^KysG%O8G!QS@Y|pvCe| zp1g}9v{>}S!skw+jP&~hkKP=|n_BMV!wP(lj;-9SV(_6h)2?IOtTf)P?cCsZ_WcZ=H0SkDwq3_PEpms;_sXfvvI`=* zu7NTh^yz=Qi?_ZT?_Pk{${wz7VqM|w$OhlJ+VG36X3di~LJq)V)KkyzmH(S=b}j$S z8+=yhMeP4R<33kjzq-KOwZEUZX{z(0!RV2F*fC^c)8M1cLcdGh$ob?^-_0F#`mnq0 zp-oxDl{poL<}@L9*CAuedQ-?d93lRPd+?6RY<1Y@Ubs@^FHeUXxQdSZeZYO;qtxP} z)!P;ot&Y}RWDxLWIx+}%onL6*M;y8LQ)pdy1^O*d&0bTlehZ9aZ?F5%AjFnd2a>@~@otA4&4N&h(eY$>c!VQ7q(ws|ez8};3+V7MDRByYz{8&5%} zZl>>NyL;K4p`A~

vH+K|h8wT9Mg8(p0l6PN+*8Fv`|y@~s+8v8CZ|2i>V4z1~F zzq;3vHN9D7V?T0NtdufcRtop_IrqQHK2t}=wglczbb-=#k=sh$*e*bKka>3(mUOju zvBn86e3bm6`>diD>}l3M_N@q)G(scN=LF5Zbsb@4(`Ge3G40aE4_f^9%?PdWxi87ILHhe3 zG;Ig@*LN!jo|6897u=68_&WJy)3~e_W1Pji3@yi3#=Dkx8q64v>tSC*KD@Ij`ibw( zt$*d1xz)SHKfwq>ZQ8!dRZ+;Oz8 z0{bdio5A66>u^5C-rO}$zb0Q0T6Ge7Cp3M3#BS>VamAMQKr6egd6X-79ktt%e1)bz zdJelSgHIM~*L?1GK0i?HwF1@Hm(2)p|1pcoC6vMw3tN)M5Gr_pJ{DZUi2Tz9c#?yrRPeuT$u|{n)ul-0cq*Hoi+-u|W<~ z)>ith2%e)2}=oe z<=-F3M|jl%(3_iRTVCSDc1}O;9*{iAFCQE?bMyX->~|zxN42>MTuYu*FUn)4|qp5OxZpymyUoK?JE1fGX4 zvE*$05wT+toW75EGSBWczSx?4doM}jghJ$!Z{O5tDv#OodB3*22iA^v;@Wdh{dxTF zfc(-CIVKT#CJDJT8NMpUj^0P+<-NE%v5jQzzrWjBcDTpL=|&cq;OLa2G$M`idDCrE z9ddR_GIDk#&6}k8tk=lt(eGV+*Y<$7&nO#O(JGHP@~GjC+yRsojsI8Te@c847vB2Q z)Uu&wzQK2Y&A+UU5gvheErjp>G__`^$SuNq^S%XqsB_15;Jt~r${jGR@7VV7zn(j| zL)^U`;_mIk+`S#o-P>|EMmA~WZtimY3-E8lAC8};3Nt5qf7WH4_x3J_GHQlS=Dz!n zaM3kJXhY24(k@wpOS-6>NnPgu>(H1g6`FJvV7TPax4r{nK`C@r1z@c0J)7xG_8u|AS zei>sR^i{s!Azd=}!E;A@|80DmzDw$3KOBi)hW{ArhzbAfV*6>nrT^9Orp<=_7BJ4o zng%>hUDA4C4~E>mJyI8eOm-_54_DOyzLcftmn%n+WZ~JQUmcA*#XI()*A@k-s@P8ch;9BOxSmwnw z$n!tp?((adFJq>E-tQ{zPKq%ygp_IOd0t<8Jb9F{R!KdJg->l&PZg=B)RFNsb@ZMc zw~uvnbMn9L*>@G`y~q@@e*I-Z#Y(C3Nct<-N%arMsy+D~)C`dm;#vP(xNO>!fZqn! z7S|4!d-vR>?O7YY+}6kbiZ_gpST{G;&i`p+9e8*!IiNdO9TqD~Ew(IcvCDET?i2sc zO67fLWj3y-D`9aP)n@UKdiQk7vN98#<`L< zB*440;Vh8688N!gGpnY}7yd5(jj0b5HpW}=i%0q1Sy@Oqeslt&2PjN5Yx!=GJ4BZ= z6M9#3o^!_-yu-%Al5Y0zSYMkWcp-gK$^TmT;Ey9Vg2jZ1&Ya7y*lXbZIO1>RY^LB= z1#&z({=KPkK50R3TkyA1bVASy=G{hfhkIh#Q0OQ7k|~Wr_YIBk=tie?&~#2s^S`IyNh7lFy^%cr zV*E~_&G_pdcINdjv8|LgZ#4gsL(fIxJ}W-uh47(kOQLk`p-I>i{G+MA6|Dnw@(SVz zEgOhG-n@5bp+B&o4=(7$_uu`klYJNL2d|g2LGb8aXs_tfvB}>ndwmm>ciK(8E7H@! z)$z!H8O$a00-;QICv&FZ1bX_ahXakGmyK`Q&=uy<_YrGk=kblfv%QT=_=-J?=5mB~E!6c`i9LowOy)iMx?w zehwThyOjH;Di_oYDT!tNv(8Qp zB~S6s&UV7YW*5Bzyv`5bw=H{4Vi$D^eUI3hMD?+zZjn8wyB|g$dwxV8dp`PD+>N+F zxFo{Y#Htx})XPQx9->d9eEvtwgCE+*sqa?An&PtW=DZoYf!(D-%UJ7t;L2g{txNz% zrlDh%_OzKjrSbDwQyQOhwP|?DA2_|P!2h?jyG0jkoo95hdzjFI5jO<%5@!e>w0>e)QBrpMvN$ zkeiP5vz}O~X{T>?X=9M^9oSPHo>kg7ioFI|YYO=m-9bLM3tu_zG<$RR(?^}KXB3?< z_C`xHxc}&G-kw12g{i9HDb_ zOzfcamBb5xr%@Uzyr97&^gChpztKM}$-c6S-IIRIVO+=2$LO7xMBiiq*F{IgUA*Vp zU5SI;#MrBda~=3m#oS)w_D(yEu0G88I{E~oBi<oEhgN>}9ehMf4T(BKnFl zaDNT-ne(1moI8hC(h@(aPfn)}jz0O^c9PNgCp6nji;(&vOCq8Q2IS5m_R&?QN4^)U@{(?T{eT$ynhsP#7ykX`jD`8a6 zK7lO{ZFp#Go2P;^r&(=A5te({tp?{!FcSNr|pl-dq ziIy0e?DW^z66|>D|IQtL(8dJjPD|SU%|p(dl`!E^YteO$gr5l$AN%Tt@Srv6qF=r; z(r;Dtx4~iBm5XjIhqpCGVPBQQn=Z$tT%1+Low<}vzp)l1h6>Q*XYwt$@##O9dn!;J zpf5~Z+P+4{tf`6fAMArocHXe=PgpH|Q`b%Qy^=z``7Yr*yed##pCGyc-?Tb(rQvJg z5zJm5=MwprF%UW;X)MaMNT-<3EyD|l^ELIT0*{iwqYA+_c)n(OMcaI|&Fc$P)7MKz zP06+!@Tcv`ma&{sYAsr6!V9ux9?$Ud-AubGGr6-uC2b)8o!EwYm8uQ_C(>rA(;L9j zC#`WKT8BjNf4#JwHW*rXURN`gA4+}8Z)r62l6m4Mp4dtXKafRxm`C>6*+tB)LgrWj ze8&G6zF;Nkw|cmSS=p4y8p>Eh-koxDs$;dmnfZh5-0VPbwB-#=Lk7ftHAw#? zO^|nwiJ!@v$C56GbP_+OAUDtSlk7z@9`V_ZF1OyjDBZhIVDGftJ{9RQCi1>tafgNI zg`E5CrF{y!VQ}RCbNkl*U$oEEllH~)rqF217g|jFC_fnOdufxjb1m)5r=8N?XkDXi zjedV$pYHHw5jwP6c`bW3b*zcdxrzv#t7C49?nnCLBzLf8$sTQuW&et}(9zA(CxV+| zmk|9og@4RFLoatm@|g%8MDr2aDPzPQPji}ENQ0hb^JlCJ(ym~vx+dRBn)b6x1GD^m z`;#liA8@IeS@^Bba|>-YeKnCWo#@_^^}x*=d_OG^x?in4YJ=}QrM>sZ_ytF!xEP2< zCQ`lay7q~EkG^3oBmGz2x-`KF?TZonIfvhsy3B(v%NvpNlr_@}KCIz8%-eqx%0?Xt zCsw5A2a1gkO`B0mxRjHHy#3$Kd((G&@A_5Q-c5HtcQonbwxgRKI(Rg=;E}x}>;H7r zKj@!F>t8!|bm~Cv0ne&AnxIyVy(muYeyC4hyKV2Xv0v~8#+U9kp|4^_4&GxWz20YT z{Hjcsnz77fuY3z0Zv%8W8(o3$dk>*&@LSxQT3{W?hEF?>Hh~|!y_ahb`Oh#tsBV}Z z_%V9~^~3al&xh%}Bg6EiCx+>Z1{73Z_BZG;@uACkhhx+r{|xpitecO_XFr`j+Zf%` zuTNFg@q4%{1fDYC?E>9^+%}=0_(*X~<=7Id_%wIx&ysNu$EoTn=$`lOd|T|p1V2CP zp{oB{-Lyj1>`MNr&iQsV-(STAsz2jgG;}Z|_FaMp^I}z>pS@l%>o9q-mdwbp7Ns9X zo|`a#)RCMH73qiJSEO76b3U2*L7WO%t2g@lm^Xd$yjFZ}&kh0ngxsEcF0r}mSI^_! zp$76SwQ}Nn>Oi$0S}>3GK*qQ8Eh?B&Vg-|b<_f0sw&3$OpYET2FA{&@d?+goqw zxdq*@g}mfK#&sj(dZ48-(9~G?70&)6w>6df_LRywvSwZ-wbAq~a-5g>6SZj(-9U}( zA&u*1F9By+gEtMPPpZmqXq-ZNe+wEu|87T5LEg~t5tKKaa)(j=71W~$T3(1Qt^i%! z&R+<=zFeR-RdiR^NPgUb)kkFA>&`Z9x)1rQ6aFu-ms&H_no)%uGmCyLj*Gn}(~9jQ z=k@PFZzAdTf8DfcDtdrL_+fW$=s;|7!xZMdX&Y-{gzj%t(0z#?rTyb+%Ria-Qy9l- z>kEPnJ$%9N2==iV^Kg;UH^ihYmj3Xu7A#~fkiFc3#-?$yC!NYVaQIp!Yr#f;i?u*# zrL5xvvsF-Z9TU(O&WAqM6UI8!IV2QObLdkV;$rLj^@r-u3P52d^>S%0-tt|i~_X!SkJ_q6%&S*2Zs_eh_X zEpS(XlcsHh?v`FIaga9!E+mgg-9;`ABj+)<_taCaKfAAe82v-fVSz1I;s`(TIXu=p ztIf0m+I&@0+#rD~zi1ja%&p{2zv?jcxo@2chN(yCRJZ@GxgP(#nW}o;K&7SM?pYTX zoHr{)U-=@op(9lFDfEQGYuzpOChTEI-^;tNrrk@?yKtx0?AulKua{!W5v!_acPFnx zZ{xheHjSAplvaPCEq_73&RDB=fZtKNi@>LVPbr-NoSACr%t5#tEIoTF>A8Drb}s&Q z^4U&4tH|eN;=inPdrA8N@CQn_5%_E1uX*cF^2s2N%WyZ7&mE+1M?RNGeSFlXZDZyd z^4UuKt3W9Fa8=bxngC-?tJ+Ao2>RQh>=+mlawOaDpo z=|vt_<9_!zn{ffLCm(bB({eEN~cwU#b9 z8gE+#4~%U`7`nZ^W-iTB)%EL(gED8m^B2Ms=Y+zS(Z@y527&w0_soSH)Yg1+3%zuo+t19`Et$Nv3uKxDI<57YfJ8ZrCTP%s6O!C z={ffe+7eBRKF;R+88#&7{IDmfnsx7%;?%}LGG|@b4%TzVX)5~f;)SaEg{f1v?3}xg z^Zwq3Pnkz`NzB#z_pPc*Qk!?pO79w_X%a>TFGJTq zwQcI+o3EoyBLaE|>-H^oD_u`p)5!nE$7pv?_>+#j)z>w+O3p9J{S_Y}-^;mmIp=aM z_IQEQO#=oQTc5JMrj2X+o4ag;7w~5zAJdL3Xi8`n?X?o86~HItS~oUMuriEZDA&3L zxO2mNcm>vdJpmh%Y+%!G7w$}d{cPO84bbao|CIVt7pIO?hEMR)zxDJlXT@{t>DMV2 zVzUB0g{SH;WeTv*yExSkJ!#RW^LZ;d%s6?cTv=VQtEs=Cuh(2`r}+Zass2E9q8+Ht zfS2QpqFr(*V3!#%;Z`DH_D-LbsU5{L6{s2$B6Y^#XF{L-Y& z;;zfA9)!I>7;`_Xe9@4seta`unxIEn<-Z$}m0vcrmU-O7T#@`w?1DG!=9HKg~ zPzxS^Nq8-^OLRV2l!KtNtQH#71PuyHnnNwpFn?>Ae+MNF^T(vQp={`f^36PGVjg?| zEa!iU*T*#9x78%{hj)>~?iK0Sl1}sSemk^#h6lMs_#4La8X4a#`t9q*&~*Ac)5@53 zrWbd;b52suk-PD|S(Rh!S(in&3o`CfPEY1T%XQiC72XSNDo9p_@hBLdYsSsT9Kts@_6gN&u3PO?VG+ue1{$B&r{pD+fF zZE_Mgn-b#PZeu$t=_^S8A?elNDP7LTZZw*Q%69a(a<aXl zUb>JqdU2q-0bCY7Q1@7qelut2OKxCJQ+H&YWhM7Iu++C?1u$()m%3AaJ#$=W)+G33 z*;h?NcbtaoD6kjZah1rB0;4-F7Fh6K>Ooyond`F8a+Bc4ivivEBKkjIlMiX--&py# zpMU$&k;=awBZFz=70oA_N3p~^NxYNb1hDi$G4f=XyZq`0TC|yQ<=*S@^q-7FG5luT z4#z&>`JJTg<=BAMu`e&Szw#D*bX=ol)vp?fy_s*ZZ*7bnzc$uRSQ}@zS?jgiuJzg8 zwej}n-CV(je!L^nQw3|m1?Nnzf_8D|0s0Ghm;F;{*=6({8=4`hM-}?zXg!utkA|M3 zryIHud2>W~IrGh%BSPDa+`xMxuM@XY^Z=r}<2{kr6E^>0z0>EyiE^Qf&_Ce^XcY|KaW35d&;24vh)PaSArizW$Vnm?&|(D?MV}JHOCVB9trw(_K=f%aS~*I!l0Hwk)HY{R z6g604YYoy1+EziVZSBPbY)yz4kXsIy=KcO=?-L@T?el&<@B7F8%@$pSLxYLFC%j-c@767!pQ;hJOv)^XX$ANySWos5q+ zZ?M8UVv*ltts&xdEz6gWZ&|)#{KI_zI%DR$d|%1;`2(yW)udPRJr+65&-eL^qge1K zSvNLpg)P2Yd^fb&)?N4q;OfiVxJmW&mn|CWmEoa6aM;2)jq*%O z{7`+et{@c0+~j|n`*L=B!uvR1+zySiW{torcjo-=oKOt^iblIP3o_4F{mwmW%lcTWxp%gZI=BnAAS7KcIv9NYCY!aE&dJ-JeSl-N z#9@83Z-UN$bV<<;?U(cyKS#&L8qGggac`$b@kGUao$Hua{i%w_n0;n{rs93f84-WB z;(g5-5&zkW_cLci{1$Y=dKSG{q*0+?-#GVy9ztt!U@Hh4p_bdVnKppUIBX@d7^kv- z3TR%EJx6L+`9-W}vD?@ADcS7HUSB-OkDZ2m(fHYKZ_l!tTY&#Pi~E0ESXlQ(!Y(|l z-6B6CUiB|enDvhCCt4KbzV-5SbS>#tq(S+H5g*1`6vc-VA8tjSGXKPct1jRle8H^c z>|ObVe7n$!Jf?gX5x>ZaJgWF5#4o}BxA}+l>SX>IWG%ArO;kRWZ&R&^=&F1g@oCs< zD?Xk0bSqM2{)r1$eT{z-@Uw%QRDLtxZpPP~^4&)KHY;+s;tPl`up-|y{|pFM-NQdA z?A5TAFTbB}_gj%l~;BXawfw>E5Q8qMF{utjLvm#$pz9)!3 zVMT6K96!=kYplri=AVRc)sOfm%UV>;dcWM}n{7p|QNCx0KVwCvDgHe1=kfn-{z(p3 z{Q@}#`&K{lb8Ia7ZRDAW=)k1kjzf-S-BCHR;Gy(6q)ER$AAKhM-i*A2zSUp3;#_Oy zp$z6DUuyor4OX~X`rcmt4q9`~KJj$y{F^0z#^R^XzjUebyL0cg-RtDY6WwE_<5-K$L)RGAY6+Z=$zn{ZOrLL8^!?|&i?2lAx0-jaSu=W( zJN3^Cq)l7tS?xbjzB>z>71^YVcgONf12=DQ_u}hQf~;jd`M;YRjBr0#{u_)(?a?aV zLG0JGXM8E~MWhK{biAQgkYigpH?a}^v1;+cty4*}^SSGUJo|kE>_MEj{p4iD?)}>` zGY(wyV!dZ_bA16kt>9nXg$0b?1L*b&VuF$96#9daMGCWnk%``nP7gZ3g0r!C8>e@# zv0*3(Dn78&kKXS9Yl=na-WBeN$4&$PP2C<}~)|~VB7h1~lc@0iv|8@2C7jJzQ{O-Tu;;o;6*8?W>hT3kp zxC{Ay$g9lzS@4Dp(2TQ(oSJ_&_|SZ@#}||TKn(lnR!k@J>P)RUes$k1TYr7u&0Alw z`d0dZ!^87WpI7N!q<4|tS;qSIQ_6M9^M&S-f1=k{Swa~bfa%WrhC47V9`3+oLSR}v zyz77YZ{5M}nR_Xd|GFkxzRuyit2~Xp;s*_vx|Esq%O&Gh^vJm7%dL}~A0JzW zA34@E<4l{l)0la+ssEq2myo&-=Av)(R%^d^Rlt)U%41DsaewMu_}3-$;c;ZA;CtOo zlIL>RL(SPA8+vFD|Ngyu>;Uk9O-bRSgtFrp41OgCnzBxpl`-Y*!@8nwZT{(v&7nY-EJkNiz>=I@3$EAlS!2Ik8K z=F4}6S|Myg4g{HF)-sl`4JaswZnuG#iNr19iFf|AemG_4@@O5^@*rh-gOSz~-A%QW zkFLIHC1uYi%)$@JKIl+u(kV%VI@iv7fVgNhkn+?T_E;C>^=Vi$W%p#A$*L>c-8Z7~ zCOn=v+;3bI2j>Cj=`_2%7tKo6jneCFr43_-njJ>2)f&no2*Gx%_wv z{5bE^?!xF=>~uS*s15pXXf-#wPV>(j2HkD;zFhhl+O%Hwom?h*fuD~DU$W5_?d&D| z=Pz|O>K6;6XCS5D{<86y*{gjQoYua4-dRKRwC~(r?K=~lvQ~8TU7oYd-lOyuTK{05 za?;AH7{-3@Ny&B{xv!di8o~TAFdO*XJ#7D1G_ND~FrT+E7iiy6_vC(=uj#0Ljp>J( z@u3soAR*3bYK^tFe2=lx_{5av+So2<47+o*_8nAj3Vi7a=3CAVaxTF#bMue+ztue} zyzOS^oUrERL|@m7=k~(a9`g03hrjb)-iv>|%e&~|Xs`J>rPutddhd1T1g(`$+n*P=O;*+fP zI8%lV*050Rxk2|_T|yoEVBu)H7V+Qd?OGleJn4n+Suv?!)7n0mS3Zoh8uZhdePNsX zY)c(~*s@uDo&NiGehmJJeDps1E_;#B_l&ceWmiA(M4v@@7g#g3512LHvuMZp=%Vi` z4Zm<>Nw|4VaacBh@23Zk-NN%r9`uCA9>@qDt3D5z4nJV>*C^ko**@lQ@N$^F*O%2k zct$sT=_TUj#F2p_KPCPi;bHuryh7MOey!z^b2g!C-z;5~?7^jPugDG8ODBC3VeOcb zqC9N>JQS}ad ze>i6?PLn}(7K3r{xjAGfW37d`W^Hut@b?cLZ^eEr8+d@L0-aJVZLjFVUMKuGPa$$k zdkkT&)?TS*zpj=!sDnpvYrpROQV;er@;{X>e;9_}-ueJOKUjzUl{KRF29Rx=I*<{k z5X!%JZ(fUy;b(a56v_xBamLo=w*hPq@^?*+l)-cJcJ(#>uma>UXDv>vSGIDKsr#P> zrbN3+5LY2_gBB;zy{t5+c!>P@5DOL!J2Xoq4wZ8FWc0?w;epP4V(z> zMHA!c<09gRXpiiQ6c=qAqW{Mzo%ShRanZ@QNI#!=$A-ya;o>#kJ0=Dr@*#^JF`{#( z;$=EN@G?9AU$6Y1X=kfk>gYyKeU{>s+s!^sI`K@}mnT~c$H$iL_T6t)4-CD%9lpWY zq`A;SPWid?N2-1G!;T$&68~vTm~XAlw=(zJK)y-e<@`4yB>iKt`>h|}q$8?_U*FGz zFxjMjK2BZ2$E6x0#C4Ybh(g+Yj5h8!X^Hj$!YCd(iL0GA)6ROI)f8<*9z0TIcJTcl zJc>O(SKi3U;BuXu>AIMRw>d7y|-z-0lmv7&HMDH{4UCm^9RfU zrM1zvCffgP;7PY1Fkyzhf-r;r5sitTYTdZi>iTK=ar^?B@t(eM9(+mPhVZQreAoUH z8Js@+zrZlO2Zl`hR@Duy&r=AU&o!{+h1uuVvFFACWX#WrY8eJZIj5a(e`^9g*>iRh% zt<+6h*emTaH?5*qT9KPJuUA^3oAx!*PM7P?33t^B)I_IBbw z=n=u3Wdu&%S_c%KlTi?lZc+JZsfm>&HsBZRX^+sIz=H z@!?kFO~p%ymspYC5trUd`lhP&oR>lNmoDzt%5yI9bGiRd@yWy|c^~6>p8UzAvj%!Xd0c(p9# z=mYOle)M=%|I70W*#e%jH~)W2PiIqRZ+gn+y*E8&^8O#u)AIitJ)O}$zBI|$$5*ho z(WCdpKHl)Vp?4IGg`Uw%U-aU@tD2ucMy|Ncx`uoI3T{Rgx*xu1#=zTWg+AGa9S8GN z10n0vMgCvn^Fun~9OBQdOm7_6-`E7y4n40(WxaE<+st9cW)Pj@e`0%^b<$c?3qMYN z5ZgQ#j$HJ}@5Nb*_F{`saa;O~!~qFcwlP+6)?OMqoQrKYkM?tXjn&id05(7T-l7b1 z7X$lLoxX(p54*oJeLyN}DgJu{S$5F1arOKAaJKVb({~T?T3fPCco(&!6CQlAT~B+5 z3}KI}EiD-Fe3!laKEX}deXUKcu`Ls_{6FU5t-+ZhMySBV!x_(L6r)5}IOfajfUrW~3tP@rk`iW$va<-w> z3O&d7iQd7Txxmzl%pZku8+LPkU~Hw`7Wit5;;Q7q={ev^d{}&0cH#4(kKQ^H59>dD z?=DR`qg{BS=m498TFr6n*(gfXFWFLoSFO5vxi4GyC(lQ@*Nk2{lb3(`ycpzI_&N2`;Gy2 zc=Do(|7PFWg*gM5FJY{eZLch9jIl1N)?Q=l`JpE%1K6jVd1aepvx;5PA`gD@^RZif zeIRzYCpqWA-4n`}PZ?8pjS5}KnCcv0jh%{3(^P2bp#j#G_q?%{J1$(A?vynuG%l%c zebH)8589}&8Zy7MUWzoO$jJ3Dl^1>Wd!`b$0mLC`|uXh~Ht$k_c`DL#xk7d^tx*5-_UphG|pX-DIK}Z=zYHUJgsYMe2VY; z(z@VLkt&{Xz*3%|bL1X7C&4Oo_WpEEz_P_}bKy&tv^ zzn!&p7CxEEp$m=STE@X=$8a)Z_=%qJz&%dsQN2q7<+t`LUg+oiV}Pfdu~P=!9j2Zm z=yvd5#oW^JR{M5OS1a}Yi#46#zf!zS>n3Dn-G9JcI!`js78*Hko8ymoF8c;iT>qSU z#z2b|M;u%0C+YvA!|`SGY9DAK#ct%gzSZJyE&9*%q^U0Ul@q(*U0P3YcJi&`zLfkf zV0i%_yJ_gDigphOB|&4!m7YKf_wAN@GwU3hb$m;)@vn1bV`p>DICY*C`a!PfHobE$ zdl4%C7;D0p_^$I6(5^X8lmhK;=0BHqcjmO~EKiEv`HHcboIV>mJc$3wt-xS$mQ(dr zlb^X^(NS+hx^&8-!7lh~6xUn%7dqk$YVksyH_gFr<#b)E&YVj3|C#ftEjv1r6^`ZNv5uyJYRX)Oyh+;h&??cZ24bRlVmJ=@``> z$8P=1wBM2Yo2V|dhrd@s``@A+@`wM?m^gIz@$7&1XDkdrr#}$g{vdSxgVFU5VNM## zoD>r>I1M=a-tr^xPM)=Y$AYVxcbpAhMF*_;Xa{Njg;$yL)EeIb`sH=frp_JT)7B2( zO4HU#PvXXJxo!2kZN*-Fti?Y=J;o31Tc*6UtD51*I|H7JblxrUkT-7QRQ}6nU6q~W z=zX)3tdm*T+cQ7ToB~e?tUJf-m1r%o(i^)mKz^eqE-rTT#KoSI$Zb0tpLb|n`>D|U zPU&jDkI#k_bUkgXt6G3%Ja}rwuZx9`70*0JS1Y?NwY{DFrt_hhq(&@2#&@-Lufw$_@~_!?)fAce6K)4gH01$%cYZ zko9QEJl4nX3O_s}n9F;O^B$x3QrEdQI$vl%No$L!NsVSGp1*5?du zlfTM1Xj<))O_R}M*7!EH)_6A|=bHA$Y3`AJ(@m%U3fZ@L>(9Vh1iZ($>ffxHB?t-xrpD74NbSCJT0xPoc{(PMW zM_!HVTno(SpYwY9Bi;QjT6q70uCH`=Z&?@{SNF`6VDo#}0+j6Qp1JV;uXhdLT-GCt zzuxuxligd$lhV7qXj$;FQ)N-cCdy!Mx^j4X_smBYFYH=K8JF?zlbmgB^Eq_5<`b;KlJ`!A5l2OW$!|G%W8{nHawx%Ii{!|`4-vk-ZUcg zHR=e07t=QMSEskFEnRK<{2f3YxmN(wWZE@pVe~FTm2nXJ1=gA7EK~Fx)5YRTFSvXW zIu%T5@I~zfqC4GecoS!wp_?AQcQJ7Frn5rcEqnx<{)F$1=3R6aVV}yh5jsojwNKT% z>>}pmsK1B5y$tV_?FziNxEJqL87b_090eDVKeO*~w7c*r@wFOff9=iF{%iA98~Yaz zd`U@>gInI?l=tB%^bowBv0pJP=6l5pAAko;4pa==p7&XP;ah1fba=z9hBw5(8zh%j zcsV;qIjJ?HLVje)3UqgVWL$J|C#^x&$;4db#ueMfTlnKHU-2qo&3Mj>Mf{903m&gN z*Ba>iYBO!i1DAfE)%??cbT{pK!|}!5>U$zRz<8G3|F!P8{{>+l|8$Og*_=;|uG7mh z*G~&%uA3IeJn7K2Gj4k0Up6Vjd*Oc}<@Cn?eBK$~J!9oo-lO+^$wKrA*$$gT3}St68foH*1wddnw-=PpwtVKi?q@os*IKe(`hB zh{JoB2RBFkd3)Tw>}*Pvld@_YmLXiF`tQhc}@vX>LsOYV-|vVXsrQ z!fVb~iyr*YmCbIzIRyGIPuUdXuO?Jyr3e-xp(e{9II(8kI4(&?_fW?5nNVeTTP8ap?lUD{%mXs zl4aBBoa?R0;vJloWm!$EZKq=sagFBFy_}h;NGG0T71R)K1eayQtfu4CS+OiJBl+%O zGvur6Y4Dt7C3kM1oG3md;{=8wfAtAAx1VM9`uF=IQ=4mv8`zlt^3HPk#^w>BsYBr_ z_=0li|AG+uLi}k=d`CJ4%L+Cd+32dgkeB(gJd-^-?8*#(iQ1LjdPU??)&nK{(T!f zMfiCQl1z z3@eK#lPA6t9nVZ-1Gw(2(1OJay0W;NxMe7My1o6v9h|MKU`>Hf(w#rpK04ISe!k9P zf8U31JYez5z5}?@c|QhBe)I^93R%BZ5Nf}9q{azrnKc`%$T`H*$)j||t4oU`YjhS? z>HMQ~#jE#{zSk2;BQ8A%XEmgoJfOOBur&<=kJW;|2H=qm#`-pF`3GBt^w%bA>~_vZ zm$-gl@ut%44~#F_UNgRU`-<_w?aRmClwj>rzt{ZE@sSxv`DNR$?g|?HBKN!gOgu5o zUXDCh{T=OVCfmnJtF8-1o~UE|o9`+15v4sEj6C@$`~SqdN(FnWt$6k0_$+^%`$LE~ zV{54L6tCu7#8XdV$D?%4=qp|EYR*YK#ks`46OSOzD_!yG2GSd>$e)PMnBfhykGLW{ zWn$mvveLfI@at99t*=f9O725PWhIUbJxV^UDaz2vea6?!fXV3H9Gef(OlL0V|3`5) zEXLto>6*<4O!F4q`e&%65Li`Ej+^+U8iAG!zw%QwyqwZ&rFOBuIQ zhRTvXgM4{>E^m8YD96ov6L}xRx2p2%%*P7i2l%JjP22k3ec=~tZs=+SM~{g zEa0`Syx$sKc7M4m3+P;i$=e2h?}M*uV7KrmrF88loXLmb)zk-n!)9thk@&Rc(=GQ$ z`80lekUdqV_42C|JP)s(mXkMkV4ii`|61QiS3z7lr|h!^PKzZiaI$YfIWfDQ{kr*S%Y658$4FbciGGa44yu(R|a-7cWVw2 z?yLukBZ-U^`Kg@1KZ051dn1z~?4dBu6YRf|ZqhuHd(!?GP3vzVVSgV_&G4-iJV>Q zms)o=FxGCJZ00U>ZEDW^3QbIedG%`r=SHI~dN#57h$fMz-GK_@SDJP2pVf z$`?pAVIK9-FUIGH!oUIi-%Bomo;;+i65Af`JR(>nuJBQ=rp<=W(Gm&;zb@y4cga2NK-X9uc z6+Q$_#d)lzDl4wD?LyA+5m z2~914rYfPSIiC2+s|xDUtDvdNp{a+4Je1A_aD^YuvAD+}y)Mx+xb7nSS^Kf$jm3wC z?$W3@x6#=18oSF6YvR9tb3G(VB53x zZZ`HT#gjYNL07Rkil=rKD}IjRX`OB8Ew!%4CqYvzd~&|RSZt*TO*@v44b5LT$=FWK zU)U!ShaOw{=GZWIyz?LOVW`3*+ca!>|4{1eW&gn!c?O(0dw{9-MV$XVbf@Gx#sc!f zg}fiBbmTnw&$xj1?(U&%4-erP96sL9(`@R#=KN3vI)V8CjZ67pSeZVb^K)k!qw~Qv zu;33Vw(&f^(GS`A@5;RZtHh4$@)KEb2=_H>tYg1Mj4 z@#m3d-vfNdZl4}Lfy`K*g4U?7NfxzcD4;u^m0ta_^>&%3!49V0yUyJ-yeG4`1B;w`Z>}w0FBN zG#C1O72Ch;46E>HRWQ=NBzV&BKK8%hefVB=_6YOn%ijB6@}cm#@9utAHv3&VM|@}& z^Cf+&HQjx*PwhWMKRfs1CEItIv~=E+$dhT`$-9wRN1q>hWh-+yV=p={Bm+x1uwKo4 z)~b7N(AgmOFgJ5=h-Bus(ZTS~p+B;odwgp6*alCyjr-~je_)0GvEeIbzvcw%6ZMPw zQGEEH;O2vSc^-6aG3kp|$}lp&36t!<5*D)W{?VLZL}&WiOSNx4pz|<%Tl`z|{>9(apMzZU-8e&LdMAD zgqnM_-#&%90|D8OiwFBV=Q4+iH+FCDhfUX)@3;SeI&|&?zBO|jXLi=3uV=p8>_?^m zF3A*|B~#eFWQrPSCZ02&|6j@!UfO+1maw?X)0@9CHu%_;$P;nYE&nc6$P*>dvhI55 zV9by7b58?wFIhr%36doiB1^pD$`S(uKdFC{^R)Q>F8Bml;?2e98(Bi@JeBeAqNR_W z&)6AA89H}bnS-+Ev+J&gUr37S_dy& zQh}LJeC1x~rWpEGT@A>8HN5|m?>dv3jcuvs8tH*_CRJyZGVb-Pt{piExhc6)G_jSu zhvR~e?fX~h?pX(x?%wB(tGm8!@$S>x9v!!un=T1#te;wn-zMax@$)>P3~*rjpsgg* zFx*RFuTODgEpL0tT%jLp}Zkq0N=FYUikaUDfq&@;Dh&k z2EOl6UeXu9$G6YG_uSPEd z<}>g%!B_er=eT)HKSFeaLUuYW`&GU%6;wy~bxObaW^4brjES+)1Q$ zP-U^#`1x&t7QenLSoC$q_ScgfdeYdQwq(Ip)tAedcj3Ely60z2)%|6j#10DI=(l4m6ow*;Y;=2xS!XT;(LSsC@feOKV#x0 zI@@5)ye5b&<{i)}{m22cZovLgK4AHFHfR4E@S{-;%=3Er(Kv9e<39sm9F@7u)ysKG zfN_r1zjHWoWFY%(`dGfkg3P7HX8@u6E%)ZnFWhZ()2saWaPJu8tZy5+6IAE3u#4S% zmp3K!EpR*&94CY06mX2)+vZeoEWa-4@MrX*tHzTz16&LDiM?=r^MUS}^U=HJFwZTx znR}{7mWuxxJPXI@W(+Ux4}bi~;CB%C9cT}9@!PUA_yW3~(0iUil{y!34fwraSyTNR z_%wO*SIHSW=l-;Qe{plY_rj8*n@_{*kP_iFwzCnu?gIY>v+u;G$AHQW;8MO#9G=|2 zb3N}L(yy(wxo)IScN7$?WjqRJ<%Fl>>^5-Lc-<*{)iUp0!@ru3ML!?A^rLxMw6o95 z%LAdE0ko~ZXE)jifZB!r2yY#X8=TG^rK9(@5k448b)6@sNu0D2^g+5mKqx#slo4~Vg_UG0t zR=}EhBkhSrPaC(kINUz6Us!&)I&#t9p{r#bvP$;`Xgtc!QM%eP^6Sn3ee3Nfx)MFx z#kWT93^=2%c9r}?8(g4Qi$(tR&zHZ2&)*Z!j-JfqW1na59IN^LiB|KTk?45L-n_@Y z_*>mi?itBh?K!OL(J8dDKTm#V{nV$|`iZqvR3}>lU-|!f{UjO{-p_<<_TOg?qaUK{ zE9~yjY4-BPJA(G%AIRucbFIj0l%w%2+lP;kb=li{3;X!#KN-V5dz!T$`N*THp7^x7OgZwWcpmAD);^6Xn~Tb3W}E=*DuZCrA&P>B-EOt$cfi z^dsq=(si_hOU|5wXZ9jZof&ou{=XVM>GA97|Ld-=*qY0;!Ror`CzO#4EPBsPLWj$; zZ4Z0o{;KUphr66T@I2yW+&LRt<*U!***EHPe1*n$%1>|Y5cyshz`WzJtMJ88!JNg} z*hQ}=H>68W?cW#w{K)B*7jCS-WAVJLQ$77hV%I%;!Q$Ml_Fm7f(#4l--G_eY98X-G zz1_QO(&9_Hug}8!(RiJgi?`cR+#%8efNOx1*XvPBe_;cu|o8jfLqpiqu@PWS~bLwB|Bu|X+ z1vUWFkAUL?>h3~^r8?Vv;G6o|IIAojmhsOy65G@Vu+>GMB^(Ra!n^R_!d$HW{&*C9 z&=%zX6_qJ${X5g_#4) zJ}-M~i;@_N8uwbWiKg42FUbbZdGjRuT<+&`bcc>kv5GyvGr=p_?fYWm!X`|NlE(!9o0^9Kz?Je5ARu_2HrFEN!Rn(WivLSeR7ERwak^#e%P<~Axer;dtBJ@;+3A9D_V9q^oI){}GeE4br-Y#I2O+g-cfn@4fyg6i%hpW6I1|DVcts@>guYJYIY=L$ z-gb}M_TQ2pzW4dIpEK%I+g;n7yU8oPu;!LS_>6hMyeHVtnK0FU)`aQyCKG1ZkDD;j z{%SA2j&Qv#Z zvKiC(kX_WqSe}?_Ekf@qNt#bdntxzCRA1H=pd{0Ex08wcalA2sk=rtul<>y1~5+z1mAKi`b?XwPVNr3hSnIHJ)A4 zPv{)1XjOA2=hZt~fyvJ~*eolx)B1kNF)I+fKO?3g-3#pU>$x#^ao8SlU-&9ut)5>T zImn$s*o}uf@ta=4+N`8JKlJPt$0qAf)T=s7U!+}O_CU*!)5T|Y@l9uT=}UM)lKmUr z4-#_jBGKdzO^&1-T)NwwkEPvv;E85@4bqsRp4NPGHsl25NtO_P#FOw-)Fl2TT*^MI z3Ll)pbA>k}Ui;svv42U|zUz0%D>$CzziRx9AH6Xq%pEe3!!!DX8?kZi;NFhoKgM4# zK7n5wz&a5akRhk99kDn(K2o*&NndQ;MTeK|M&8W-82-Hud%usyVD}CDt;mggm~SQs z9>PbcL;rm|W?BH-lOe~Euix-wT=^05YpQQRU6wDQa$W9yVK1`#KHtFnSCB>Rf``^{ zp8mkq4>qh>@MiyvDqmdX;xP}UPXwN^-)Tr+;7z@fyD%ze7SyNj^QGqJFO943M6R8~ zIv@{tKN)95J{~hLkh?UoUj1F-@z?#3J|FUH$HsIKFtzsATx;25z^$M5+0cOe3~N8E z!Wv2!9m#_j?D_4e`at=Wr+R@HlJ8 zS8gomns#GO*VQ);@48^1HRP)|UeHxJL^e0cbq&B*jQp8}Ek+A+zvxPJ#8F;;z+WHl z^T$uUab%bF1-SpDPBb|WJ>kaFc($SK>-+TVNq)N7v6H`@@?HFxLwSyKj zCa&+2Z_a4nyrcHhH|G+nKf=(*MDDNP4qhie{jmbMekM4+4jdo$#pcidc2fO5a6FOz z@r#dx*WoT+Cs9|@HyoNur%cPpF_!&TXyBd`-Gv%oe(qy0TQaumy)(6CZQq{ztzU85 za(Kq1@R1SBuh7$IVBdS=<*c)k<=^eC+YD_n*K}PLdXfBxS??SrUu!S^lg2!vxp@qE z8^N*l9QFd7*EwEeg_)D(S7236Z9ijk{s3iEXw0n34gG~U2298$$#xU8Av*w#!&o!c z6i=`(aO3MKH=cNOj2|?4v2PgV$U*0sd@1(tC{wV%Zo(vchY1tyUz;$&eqH+8QZIbg z2TzNExAlR?^@Z1Q|JV{gx*u=o2y^XW|`zD~QvU!LS2;nKjcXfk{u&F%+I^=%d3B)4|CQP-*5rY3!*3#lg8gbQ8LjRVu=7sJ7&J}}Xqnc({5(ob~b<6h&Z8)ByaRM)(jO&q=jbAg`@oUp&e4GLI_MUi>@nNex=L4U@zfcZzwaRKc zJJg>t)E><-Pg8G+&l)OQn>EDm$A1cax${(Ca>q2dy-T<@eVJ+c|4zM|zRa{s_+F`Z z)0dg{&Ai{Pchi@d_6pwT>D}~YroEo`IeIsJnTflD>ghM?-Sj0kKfKT4-QdH}BkQsR z&aVTX+VYQQJ*zc`39r%%$lge}d6l-RPxta&?fX07LfVSV$z3VfcZ@DF^YJ%HS9^Dw zFx7s;gcwAV^{m^xCR-$qv_%>Kp9 zndmkz;Yr{jpZLd4KeD3a^7(p4UX(1`3qe7Z^} zSmsMJvfj#Nj;t4UWWCSkfB%wDo4d}Q=)$D2w9Mcj1sr^ZHeE&AuB46Fb=^J%xvvbl z4?p1GfI9bE$a-VQ{~1}Y23c>OXP3s4WW9Iz-dH19FZqnJUWzO0$+w5vlnieZ9LLnx z?{PNpkN9mt4_BgbRvrj}AHFBs7eYUxj~glf2xH6Tc{?p5_tEbz{mkII=*N-!GVLk6 zABA4t1W!lMLkri*j8oP5H|pGD`YD;c|LV8qYP_M#g(o+e_L#9A2rXg!DZl0`78-KyIs9mRbJ_xb}L|>V62mHD=_Cb<1`~-bym< zWQ{bGeFk8Bkv_nwjr^hJJFusqe@M2wXruf`$fxx?*f~!0Wp?gGURsO%pgFiv@Dg@C zgDpMsV;4HkRAiQmy9Q0 zu#NaHQ+tNPe|xt_x(&%X?=E!k_a0>h(P^kZHqfSMKfDi&zoIP;+{yO+25#nm^6SjQ z4}6Kpy(z|Dzy7(I{@AO&M&9uP#|5G{jU(inW&Cp@D;U`d4qW-B7jH_nk5hK_^NwsI zoU|j~u(ojcQuRc`m#(6W3eg1e%?80hU3;kO0nrq&icYj95f6Ov0jF(^Tgj{}b)kh#S6YXgC zDD-9KA6~X>xAqL0jLtE`zJUL;EY=S2TGg9x-jh1_qEnx5^0Ego$ckLFR65FZd${?Y z+<62#+YYY~3}=xhKezJFHk^5(cmJ%_S(&oYp}Fq*{H;xnuKf7!l5i_<4K{6IZ(bo~ z_E!i_;!K!m?X=Dw3|uV;~mHC}_yyJ#D})aj@FRwNi% z&K!`FXoda@ds^YWi#kN-^2OUS1ijl5?lzP@dp`G@Z2!*qTIz0p)N1wztEV5zjSYA7 z)Ba>!cptXa?LV_#?tX>y)tr~!`g4bey-#_PyR8vclg8|l-#F=iC*6^G*y|F{-s@q^ zW!gUUbh5A7ZN_qjEtyYaywJ2c)5u*~Crc;q$nt4+nVZM4PfoTk=i3YB|8%>!_cv(Z z7QX$&d`q=2>HQ7-KfpK1_$tR+A{)x;w{Bn@RuA+~lh3dACA1MZEl=%ugOi1?PFU-E za(olIlRevgp$)7>^lu_GptiOVzlZj=%?LKHr|(NHE})GG_N12`e-X<2ICwb3zTck(f{zG%n0G`Subo^$S}&ITKu2xp^WCst2iz_+QMf{;HDsNX=j^Byh;Svg+r80{O{ zfp770Y;?#U=|pC#K;LlU>R|z7kj>f6oYShZPOhY#9l3+U`)E@fI)Hs6hlEe$dYX@p z@HDqRX`SrIO$hsep*=S-d~gQovpmi4#|zu%c$yDEcMUc88)Qs$Gj#G~z4xP4-=pSxf@!P;Xcr5>KflhZ^7JNLGKI_Qs6Fxi!-?h-}UdF5HxeY#Z0z8Q>wFmkeaG>~i zU0fXai8Gd!_aDT?L%hw(&psQlIb%4%_6>6IvWK)UZyR}mUwDej_1E^`3E2!DC>*tr zCf+)&_cz9?aMDa#SvPlQo-EtFmT}tXOX+gP=%daUy@vkBpZ{munqZGKJXP&Xupgm) zZTGrk^h1r&O#3Ei!kNbtY@4F=jV z(>Ryj@(JP(FyHp}i5%5WJjOROhvwX2?h|NZoSF9_#)ny<*27oTJLBLHl?iRC&gI1Q zE$Y*^@+Yo*f5;!Yly6ZT1Ul@Uft>R~d9j{mjoW} z)+7n6vl7wI^a)9xILaR75z0G+Oo2|^$P~v}bDcn@aMuKzqx-G@Q;B~A_U7fwd@(ZQ zO+1(KxO-%UZzEUyl;;ItyNj?fCD@!t*!~w^AeS~deVS>HWL#^$8&8@eQ?QnI)5fXo zMy5!&bKJBGm6mJKe`$86n|8kW*B8DU8AI|#ozZ)l8_lP z?Iixq@*^`q6T0VV3wJDQjvu6SE2+~*T2j@r-7j*dvSf@{lkVitg%)%_y3Sd>Xk-@r zKJ@x7c>@^s_u8A^X|=|Omirxf@WqWfgHb*GsOS**4)HFz<5Pvezh5C^`(qO(+iw#} z-q1gy+mlOX(swiH!|C+pHT3CMSx;V#E`J(6Qu>%aukv~qVLL0D-RxiI*cBOjNa??@ zp=`L!TD0%UhV)M4^vW9shCU>G_@jZ?b8{9gVP*PCPu&{si&(&!vS#qO)Bm>tA#2FW zHIJ@wXnrR!NuJ1#XDvNrXrSGfk{>&mHjPLKbZjgQ>>H67i2d`uyW@iJxQ%gPKl5Mv z#`y5z+u`}U;rV}r=a2D*R=wnu_jAfKa|!FiLyQ64b)>dl^DF0j6W=A*9%4Syd90V= z8+*_ph#y5|70LYS+j9vW{-0_K$69-dx2-bYGwfl#zH5FEZ(C`;rQ0dJzq!0^h543d zOD>F-E1viO|1BY{VwuGqZ~Hs_aFtPJkyQevV!a^*1`@Z=r9F(#lBk2`6gP2;!!17hxwz`GK=IYkaa48cwyb zAt>}#KQvu5hhrod`M>+;&}(alPMwUFleK3<*5k-`eyS9?ndU zGxcp_Tx|mPeXT7gkh}fZ?(OJfjn#iY$1h`TzZlMJ#uVgYkMy25t#S{1LEjn@tZyzS z9?s$}XmrF%%Nt@1Wt@$DF_$v>a@G^OAZTf3UQYI~^9ilN7YxfL{yX?m2Qrm-)%)N# z%gR0=UZuWh<^2+PR#{bYz1BUYrkx4)yQIseXveZ}eU{&E^jUeu1;<`Ri7CP`rH&hb#c~=^Q-LohdhbD)t;H3Zx4A=OuP~M;<%H=yUUWw>N44{ z(|tF7(mp!ak9(f2<|9{fS10|}3NH{I&iI6g z5?{YFtjMxd-#2<>tF{U+Kvf2nhx$F2-U-Um0|ao1}P zQ>J8qr*oZuqW*!ESNWE@`Bsoma_rN{d(Ie3X3pj8BzW6f>Xi9SH}AbJoJ}LGrYCvI zc@DwLTF2o(N;<={FAR-*%4z!@Zn-A?mluZ4ano-jU3}G~-!dkY<))XD-a64O$6pvq zbJJ%l9o}T}Hx`Bx-1M2G3ulKfWj&B*HT{9-Fn&hwJ_r3NPc{$xQ<1eZyt^`*9hk1* zAN7a+YsqEYY`<q0A=j!D~ktW`5^g;cs$i>XT>q{+oREk|goiWI?xuhN0-6wnz@AG_)e696^=7aux zYe80B>wA3si@wAz*@?dBOYT}<>f5zpyZVQ67(46tN++0N?0g0p-ss5XZNPzD&|B!O znwxXa4QPzDLl@!P5rH{A=27@zV`<{naBkrIM&GP1?i6YCMTO)nTY z`94GAF4gp#BiBo3EWNAK9?A8hn>}O5m*TW%HEB+JlsCy%dmdH3F)gRsGN@(3=i8HD z_vvYmW8W3(X-|SZ&uve>+nz^ggJ@9oFE{OBO~_bQ{Z4yWJG$+u>VZjmONR~@kk(c+ zY16td#BYk-2Ces|!-YNYq}UUIL;l~}M@$K)V%wm7_ida9IUktP~i8SeYdgJeG-Y1eSJ&b6fiLoNw9r_F{B-k(Z!0OA9O;*TQN;K~&_E3e4 z2hL8~gWR+MCQP>D6*8`36+(;s6vCVOC`9Jz1ddV2IjU3V73cHbHRtEZ=pJNs?xi@|dyBoaR zH$nU;`&REb`{SF<`O`_$w@v|nv45Bp@&9=ee%N9=TfAk*))^fIWq*4}s2-TWag%tN zGe4=n9ec<7(S>Op_AYr&E-3}(5@0W8eeui2mm0N({ro?D=!@Un?yTQ_$A3Y5Fe3{Z ze^=c#lJ`=O_flD(q)|>f7?m zlbP=lm!`AMOSK;*?;++z>7Vz4yY^*U>JwSJXuVm>H>E?rpIP6TxzazbN1uEd^$2Fs z-csV4=a!f-*`7-nwOepxtN#KgH-VcQ!O?7Rbpva;>siaqS_lr!TJ9L~j`YXv$SWEj z*3-ert&B10=cG?SXSGH;gbws`bHK@Z;2ycOYIi;|R+a42u)#To4NQeM-Rz-F@wLnj z3~8Aif6i;^uXxk3n@O^_T(~B^LicU|K0O0pE0uxgzL)MlCn=)}d-{FsbGLq6ea;-?-bG2(^Kkr={TOY)>tGBMS6Pn~bEvhT6f!61BjSE?v zFZ9gq8(O*xUBZ#>!WZV`2eMOrBk@+`bu+fo?1$XEd*Cs#tR19FYlp8y zb&0*(?wx{Pg)xFP)4rTGcJlAd_ka5_z3aaQ)$wK-df(vV{-p`^?W`;E$Y%{;4z8&i z|58l?a?0xQd$xN+8*(-OXWFwxv-FADDqZ)_Xm1C6>=o{2naciVR~9nc)mF2SRdRug za`WlyeWjMsJq1-Zcc$lod%>?V@1{(JzyC4tFt0_+<8GLlU5tf;)O845(=qa#5dGYr zHKVp3SddU4fp&E0vhmYd5x?h2 zblINf6ZqE=pDuO#{71m=tO?TXHN=Mlm)f-nybFhl`{Et>>HDNfACv6%r@l#^(7d;t zw4?NCJ3c^XL(8*$1)((d0Q~cd>eZ*pyB9pEPwyeG;A(?DBJdRP6^~n{Xhi3tRZjF@ z@n9!krhNH8cNK?^gP&hB@11zU8DmpP&;C<4>lA04`45-syyJ5kpV`u(VjGC= zlY3i}LwRW){O>sQEnh_mz~N^MnD5(?Lla{y;O5EkL^}HN4`-irj=vL{>d#qGe2PV? zR`TrPp{z<`i0sjnq366FB3ATavtijfS4TdKr*?-e}*+rctz?BER_R>Xew#E+)EqCoz z9tS4bH2#86eRetzr$3~l*co7-D-L>3KPSKb`i+b?Y#hoqTFt?$<nYw%o*yKH zZgTVdkMi(;HuRu8t9VzrLH3}O?k8Nyn9Qn%wm8F=LEYt_a0bXtTL+I7eOVnPkt$%% zvSJSSNK+oo`+Be8K6pR+(IdI>fz}zG(DW@1e9Ndy`@51~l3Sd##iVIHQBHbl{Jpz} z1B>LZYWV1$c^-Hc>k0PoVi+rxhEF5MF;*q_Ea2Z0@MpxNCr0d;Hpm`^PKjLT4FW z=;o5VMEU_6Tlx_hY|%mZVm5qT^lwpDl>X(rXCkzB{Ap;4_D_Vyx`>O1{+&>~;Pv-( zrcV3RI)|BT*3WAH$t7RR7xM6hsWN^~8K1zvu$wmh+h)p6w%>E>_>~D0><`4_wu{Hb z!e8RxGx6}7{_q{_?3N7Vz9JuYoIB^zy5My>uP2)9&3m$M<*Wy?cLsGO_qCdi_Q7v7 zXOf$F*6?hiE@xezVs8NlqQ_s8CcYwEZFbYPk)}R-8(z4D=ccM=WbYL7J;zkNiXa2@4pAEOidvV9}^h09Fb9~O>Mkwa4lTTPnlYp6#u zsNy-$jPURPX;agk|2&C#ALy<6Et}*L+lMlzxhKwsMz< zGoDP?_ly8dFb1hfG2E-8MiMrMfai8Z9m_&2YuYg zLaBBP@3%0A{%o~_i&)-UJG-$l{j3a>{cq$S?e#F<{{JrjTt~g#KKX7)hBu_Z8&cs7 zY4C=0cmw_$w)aJE;p`0`Mt^}nDYI@DpTM~SYXE#!NyZ3h&ExoX=*>5V-okzicS%ZS zX?U#sQx9v13U89_XaAv*J0!V3_&_eUP-ZOu{F2c0$DDndI_ww^bI0dV&L*4w7=3Bz zxW){Rhq?6R56zg0z!O%LybV7_7pe54q?@wdzclnU#p(Ap^#y0BkT(mJPycG|e1bj{ ztQm~ozaq26!5_v0XRoxvpCR9z$J`wBTZOCP_ZH#LIg8rn^O$=tEXrz4aPGa(eA4Z+ zibQkr|0&x<=e%5L-&n=@6D3wEbuKHwbQ0{@Wh_T$>qpEIo}!|`WIy^iLsh)cjqYG)1tF10sig1 z&l5zS^fGz>6Sz}=`%Gnckg-oLxe=O~4ei_j4P6f{%|ed74msA*V`vUI)4xIfZ8bF5 z7aAPOe+m4bNI6NAmkbT2K!bg-SMCR`IcKO(K!Z9{f{z~7qK*pc(0Wd~SN8N$(R*;(Y5c zXhh{|KbG*VKcn|LT*Wt^)zm%;9VKPf9!0M-0w1_bGwR=^k5xw(WyrQb`Wc;PVm!R1 zckR>pm4CM{oJjsZQ;zEX(Bx0-Jd_)Ztas);)8{_=yjA-MHzkJFxi$ss=s&^KO8u?G zrJMY_Lip(~O(?s8S>W?J@Omxyoe7?2&_C1ZpT1_D;lSh6Cm%NLUhdc6e$|!T#k&JZ zH`N8$i_2Y_S}*_9?VPRY$Y8Gw+iRUobL1DvUCD+w+=foq%>UFaIuFKLO=}{|W2 zlinhj{G1=}hhJy<<_E*_#x<4(i@58f(Db8gv+x@CbI4vncX+5@7}xu)RRtk@`>?-V zJ$-(_<5>F7+_yI=b#m4H)~y+xZSITG^-h zvupD=z6ZaV_GEC?gJ0LSuYk1E@taB7CHW41!;DkmcN+MWJ!B6JK|AOW7VS$xe)3H{ zwxDv!V^fzmtf@>0etMVhp*3+S!B4m0J3Bnu8vBFMdE=PBn#)FWpYmv{X&-l5)Ix9m zP(f%VVW4AlXn}Xhr~4>B4;#t5`!4y^q)!+flFek4CV!dc;ISWV3Zk2YZjBD_Kc>km z*(b!VsC@%xVy;edaE2aG^e>z}?BeW`_3)~Gth1m~ebc(a!CO)fUG62#!JD(T5M91O z+9jzD-fnd9_6wJ%sr=`_;X!z5#f5h;k0sfS?8SVJW_OG+zU_1HWiXsFmjjFPe(S>X zio~xp4n;5jmQEhGn!6i}jnVV%rLkQan;jaLBjUmxBjUpy*t?=X-8?lkrFjl|*QtCj z2G?ENt>%t5buR|v+DZx~K%;^~>*C{y*6OU%;DL%2)#*vts~Q+y*4XcVg0Zy9YT;gT zV7=wP9bWW5z;5>M;M*EEg)V-dB;6VFDfYS^+%o1}ntZqix2MMZy`&v;`Mu;q(WHDI zYh3HzQ>~p1kAOc*maQZYHYe1ZX66Uw*BYfKAN>3l^35fm{_|!#>+d9J9~}%fMa8=t zu_w(Vk95RqYra1|ye#;59AnZRf!-WDhV?%0t_?NX&rY!4geKKS*;^LS&UM>w*t*Ba zPs#8z{O`b@4W03Qn(5ON`zjM=*dw6}!6pAZDogNR+6(?2%vWCoe?b&}>j9%b_y_O) zZoxl!cYt!jxfh4S*zuUYLZ`6S_r&<{X|S&a_M`Nh=uGf?dc%L8;7_u5K6eWKVJ_Uq z!RKMo8Z^_ zMmcwG+Q`#;Ubk;g^PcLB`u;XjR}(tbOwOf>Zj_u+yDGl@qTN-w=Nn%z`7ccbkL3%d zexEpJ$@Wh*XRJ%%>`r`G7U=GAWQLBpyCXH&O1|b~1^rh!XAV8~KiBat`pd^k$$yc4 ztF0fn@bT_A#LnrT=})xZQ@_@4aNX_qOXSH<-jA=dA7AH2e3wLPC(_^iO?m_Of_bGkyLx`hHAwt<@hYDa~?d``k|2Zs-qvO?G{hZc{s;1&4D)-&5_yjCBW3 zbY+ekfpz7H(}6|l&tAq)_9xLEjpHbuCF30iH_Lsg*j$)P7uP59-eWEudY|`1gH%@j zvp$bo)}8n7Q+Au=0;_yVJUD+EGJ)z6ZTNz`l2Wbf3wOt}zVlh%Uz_if;yLy^*6Tqt z-_H7ukDc!kY25wpto;u*b*}N#87H;H%n^KN5dVc2*(Ob~Paz$RtxP`9-{k+?=K35; zzm3LcI)-i`pF_#7vAV@iW?=`zZ%;P2@o~FlTUo!ds0|)FP3``I_LiUvB(X==u?LNuE!`IpiN ztvyxQsD1X;t{wa(zPq-!Rn0n6toKcR_`UX7AEn+Rejd-GtD?L12N>!ujI5>Byt#1V`9=JUS;S|G?BjL$JlDwEu8fdn&sgHf2yEOK`xdePs_UNf1b_OB1HTWr>^#^8XnUG{y6ewe0bDYW zVpFYTJ;}e;$mFiodE~s-Rt;4w|ihad+WP`>^U#K+~+JY^m0h^%Stpx{+y^H4| z^kdK5K+TTWBTSqCS zKO>bwhfX1#EZx7n^Grh?b%e2@96QPw$DU(2NtxCzWjwS2efAse2wNDBH3O}iUSSPw zE3`)KWzGK^yTX5<0iV+LZ_<-LILclpwg>3JJ30$d3lSl+%j(tKrCoUn-wv$*!8~8=z8tvsd zfIfaLc8xW|@wuk`Ylx8}Szr%){7x(@pIw*DT+^mD+7kA48{%K+&|`mZ|FTThFN8g5 zTb6aR?2&n#VT;<4j-^_mSI~0?m8AveuHmjK%AKsJZtJRl&ynL z$@cI9^C{X=3(V|6!C(I9!gc@?EyZ~~a;2Aeul$N+_kb>?V^`4o&rmi3?a8*%n|02j zgmd?YpnvJw2cWJe$XDJW*0I;IiJ12LJe-r1ZQ#N5;47|8pZ?=b&lg0@{^pF9oT=3=9IY|M4Fw${glwyGU+&h^RU5s z>><1(S!fn}pkvbl`#sQDm83?75H#+|vj zp`%aW>rqL(uHtM*|1HO6!1(IB|L5A4hEacnag+U7`v&Jj7we5b1p5pwoy`H(NoSos z1ku^vz=dzCp?RXSk}WO=E}9iR6B4H~;M_{LjL8Gym{f`!?a*81L(ukdLno=Q9T6$G@2o&h>{{(eKR|Aotc~mR-<_?WEr4WnV@- zBE$X}IIeyQCu-@dWZqZ7?Q#Ara3IT!{W1FcA@dhb39L*_46NjiBU`>D{!^^Y+2!1| zmWA({Xq0P5o5)?XvZML4@JsM5SSg(+ADzdK4XG#gq#t>ft`gqO!VWK*vN+Abr#EEx zn3xlMll0D2NP@WbkwQc|i|jv9`N2_qD%saH6iMz1qmJn){h2 z&b^`YUFN@nu@w&%4sd2Iq_e(I0han#?fZxrzRJcJn&odzfgJ1j~8Alx%X^vCEQ1JcHHi8u|Fi8 zwz11RH+Grl&%j^VXEo24#?JGW%-~GMRqNS||GB3-8S^U`la4WWb(BJ2UB~=88au{ZGGEs*&s82{zBx9}U2_AKr!lu28^U}1 zuk||TYyG*ks~W$$2%CGCu}9a?8tXdt_r{F<7xbrNja{uZmc$xMvR~fV{)l9|UuiB_ zOR|Ug;rDqi&+MR?=#__{ALlZ@IzJkss?FUyBca#okme z_6*4PCkIv@W(~JPzw6OAO9uq@?;(8_TbAbY0CgFgDR{55mO9VS3N8I#=dzV=w8qXS z{kq+q$A40{LqoIdPZ{^FbGQpw*ExI{nC5U&>^#9^$J4IomB$>8kIi%EQ2afrH>$6v z8Ql?j%^I4;eEMD8@%pbE-BI(Y^FD@Nx^sD`%R0)4naew3=Wijp_IPfr4)JMMgvc?KSR&FhTkmd zo8{6s`?1H;pS_j=?71Ye_rh7*1#P8`fPDdPIVqbfBBH#3;e?6*Zi^Z?rZ)Sw)~tW;0_q}7NXx>M@$`kw}d^k zsmN}tEOf&~uNygO9rDl{oTGghSl&Vh<^k4-eD`S%cXcyPj=q^{p9Sp@UVk5$W2@q< zp$i*jV4UamTGi}LR!bMvdER7nV(CU_P)_@12hbz5cc#2XbctkqB_*Sr>gJoXwUjxn zi<}6*BIMckJMM|z^*Hm$x*DK)WWM_;W!)YE?&vepu`g%-rZJCGna?TA>t*bn{V>El zo4vClqaR&WZDZJcjK&!Pw5j#LjBPr^mIMujpzwKh%KT zoOlb!%9~_2Pma;0&+2PE_zAjn6?XFsM~~iQ8M}F(W3wb%8@qXi{cxGn&bMi&blmOG zqqKZe|IN?mx@*)!G4EH0)@C4+6%x$Yoc*RBFNWK zpS87Sof8vepu=;gK}&=Wl8KG|jJeO{S!<{k-8Ti@_bT+=$n?U{QSd|GQC-@P7oA$} z*5}CUX@;-g4eY?j-9kF!QdfT07{B(mn&U#Uy*1=WKFhMd0zcAxnad7)6PF41koQ`` z;J1hQK7fwv@boPEO8UL0)acNg#*LAVon+@d)#)#$x^7`i1lM>Sy{;hCRVFvMnm5** z9lsozT=;kp9F6MSdQY@U@Sjqb@}$epH1($2KUNBE&oKX!?JG^1WRE4iSbFyk`F*5= z&l%u#CitBNo^w89!KwI!^?*+GK&EnZNuAedMYokNtn6aKQD{|1On^scIOlG+Ru+Up z3HZ7po9xX%f6WZE#qrz8^K4T5(5G1G1#N@0XDRy_G$qf2+>Ad+1$3nmeJDfiWK7kU z(-?29i{Zc=`jKv*;=;b8a>Ggwyg$#0cf+}n~$47`O2t94GOg6C#v=eG7yA!mnXXM1wa}Y9BvZCc~Vymv8z!x7=>Zsmi3yvg9ab6=bG<&yhYX8xk(RW3HaH~H<*pPrO0TKr%}`~}v4&*}HJf`4jMA&73gOpI$jF&>7|*ci#D#1F1gm1c&v!E`o-39p{i}eLyDVZKRF`wQhD;0 zZG|aYnxK(G3VI%q58snp&kH@hb!6x>_7-0#PuucTVfvP1BNOq<#Xh*LXGnWD;mU;2 zvA3YF#3^Xn(d$U_*509gTl<7QKWf!{#u)?iU1;OLg5Jjl7Ib&^6}NcVS4^=lEp^8A zZpKUWPBLEIRB!Nd+kr0zE_v%9w8ByzygJpjgWgV_WOc>dUjiRnU+iN}I`L!_S>;};P_7RzGw2-n+!N~c_06^Z_o}7hVQIc1${&>D%gF1*awEL`YOiMvPWs` zkoZ#8H$YpVuWIKLTF;D!@HCz=+4G2NgUoZznSRI^Iy4~BK7+Bs-f@%S$BS-$ZCzI5 zCx)Jz0i6@x9xM-pqgO7OQ&`6yZr~#J@jrf%>_;`imLume|#RVVb zzj&4TzdbI9Q{yAzr{;f2x8O6Y9Q^r&ScbDR3vM@d9i4aaVaKtEo1aMhrWpEr{#EQt zBv~V#A6*cdM%;q%b?~MhtkdpR^+)`k=9gE_dBVk>mLlY`JnRO3Viy>Bv*2v*QRg0( zvjbato)Q=pKZtnas(bMbquA^CQs=sQb0^bU&)D~STT51cPTc5i@quT3#4$L_f8Uhe z_iem+xu^HV_ikK1rT2!=&2xBO=1CZmKum*+|7Kk_v%ht=;;iJ)&lqsM*E&~rQ@_Q1 zVBDJ;hW@sn8rZ*%^Z^h1C;V!?iMIc-YgYR4x2hZ6GqhbcuwUi|Cox9*rsJPV9EWT0 zBOHsZ!^asw{D^LPg|=XOx=G`-#)th2yy!{WFFuR&ndonG=wracc~|;*BmIoFd#0!N zoO@_@PugAPx%l=b-n)_hf1Zhq;OX6ydv=D*VNB*=!=OA>U*(cjyuW3cxYx6o;7y%|{jKL;P;%cPYQTT;^kt^=t$4 zp>plNo3(#2&zy6t{t-Vj4)DcrH*51h_#NQxvZ+qpQ^GU2Tdo+s#@U`6`rbU#i6OoN z9@Y5Og}d_yOA7lFoK2A=1YrmjxI z&Z_wF#mq$=?VSX?WKp`=bJ}5ifRpWufYrLNak);r>!t=GLui-rcQ3VCW-W8@HO*rs ziZ`=Hh>>?8>t;Obr;v3tj`dUkpZjx_c%1suNMG28Z!YGXQqvQYLWd6D9KFK%9;p2VmRN5~5IZ){E ze&0T1boyfQ>95wk!7t>39L`XypE`Hwyq|3Mh`!HWisXa?BROww?r`J&wZcH8o;t)c zr;M^pJyvs%k#)?O$rIll&X~G+1DnXBe22}Lyzy>cps~luzdr^oLI&#O-`J*hPYS7g z!z6sjT|4sUDo1}D{w><+#*>%;U3bguRT+f?2B!(uEq}yJeBX=J^dWp#9zn{)Ax4H7}gF0&*?{nQpTo8DLlll6n;_r9jp2P zbfw})6Cx!%2l&bU{jB7xGW6ZEz&m)yM#eeY^0xT+Uq;yw|JR;L9pIVYjnOuOwf&%% z_?#|~f9r-LZRrU=vRi=a*pWt5mC;ElfKIeJ&vMNU&Sg7#@ zkJsig7R=8f`dSYksAG=zJaSQ}f%eB?A6ZEGvRuwS7hbs~yv(X8!e4pg6YYxOQU68h1x9|*JIv)^+B{zGEt`lf zq-@uC{s;6QT=^^QGqQuq4s+y+QRIzizi@X6Hn{4_k;eWRB3_4)lVrEaF6;E^$#MOW zc>hUgMtvc?o#$v<&Qn_?x2S$_No^UVZ?k@$iTyVCG@Cr>3bpQcHIG$_6Oz5X)0ZaZ zd-wCb13e}hn%9D_)2rakxGVCIJ+Q+r3)F09jAbYCL%aRmDy|BH(|e)eN1!PutiuMa zL-}@R+D~Fzk{q3&v7Z^c4!Iu)y4FBH*1KiAl#%SG@BKRDym$Oj-Ya(2;Y}{=h+yD^ z`h24cYXv4A!Fkt68}{20&WByaFAUxElPV_7RqWLdE=14gHgmqlw7oL}leIX@ucpkWXE9YsqS;l9!5gjR+VhlUeN61kdlywuaz ztcCl)V~yd9v`@PHc5K$t?Jslvg{y%}mS+DGIC}hO?8j!EfXDog+7F%pc0PSDIOW2H zKkI-y@-go+e!}PFf*JW381p1tlFY8TNhUQou5U5EOUNrz9{ArUHV@fx19|h!dk%l- zc#n0olf0Wv9(D_Nuiy;uwG`j#k~<$>ZchkUKzbktvE~0QlsqjZ~uTqWM<&3M|ovM^Ey-X?X zo^0S5_79ZO|4B+2qlroxzX_y1ohf5|h+Y>8&U!3RIt<-kDfIsWrL5u6N~w30(j4SB zQk~;w|KcWMSWnZO9l~xZ-dpl)`v}@MV886B)GxbiY%02tt9t0u=}MuGIi&xRJ)>Or zP{biybVJ7Ij;zrGnS(nh7Q`c~d$DIYXLxmIB(nc5aCZvl$mj1_V9l@diAS*?o_xco zBa9943eUUV8yIyx_sS(uwgj7!J$uQniPDCgNirzy1IAUyr|u&Ub5` z^u2T4c?rLM*RH(ZJ-yqk=E^L(DQ;{E!ZU^?+WiA!kw zO^Z06yBWP<7P4SQvNd4FjGEOYGq@XwdoN#+O}Phipg1v{y(q~E%$KjtHG0nFdC(p?4J{>@(!Q)XqNpt|L@>VOP!k$ZbipdIF`Y= zpun7hi27j0`U&T*4KKWV+^nI&2Ur_g_b*Y0Y!DH~f8a9RwUKO}39VM%X7c1Wuok_5 z`y@vA###fUD;7|{eB;a=lIYzt?tmAPR#N^L=?%2S&-bnQ1(7mfqAT@BtzaeVFm6)W zlxp#wR@voJZNUZulg_2~Y7g-aw`~vcZawb`H>`WPhXT9N-W}XSk!T_7rP{k#m(ttX zGBq9g}*{o9F;t{8$xuv)bt4f5VTEt2ooeTKUrI7ax3=ekfk7<+nD!i2e2> zbm`~N)gI@!fZrAT6caFs-&*=ajGF4Fphe3`_1?zvK+Rdi2rSAbrXTSD|H9eWr|%Bb z+()YX5OM}S0VC}DiDT&LGsN%dIixDi8u`rXKux>Hdz1X?+WX@Jf?m66L7?W@Iqche z`w!9Dtw7IvGN&N)CNUnVYXCxB^#}Z(@n_BL zA9{^`eRh8!@`KXS-G1YP_?K1Lo?a#%VB<2;gZO6rJ*I7O`p+68wt0E4Z`9k=pF_~y zC!z71z++_dDUAsQHDhVN-?KC$j64(``jdRMJ&&^S-70i9)=kJ(tD2`1f5Ds3T+Uge zXLI?!w|{dE&+Eh=yq@MA#Qa+iZ^`z0n~Qm_WPN3Oea$|eCn0Oj8FbaMx#)z=<=maV z@X6H)Uf)~$kmtFtx;Y$apVnL&g?r!HhfFMZ9`L83%L!i3Tl!zc-P#G>{^q%e=Vscz zjJmELgzvJ^BYQSi9B!W$f;VbEKLl@Uyp9xJ*NDv4#JGis-Q9>hrhZRc7+Re-sCHEt z`Kt*(EyX(VV+SkdZuCZEhiu|9`Se`uJXaoYp5wjoZ#BVRXEFzV!6+LGD`^E*D4RO* zpLoVs23o^imLuarEu*`K8VY-a>U(qxX?=&mfx0=gk8#P4k8dtVht|5hg1koNKbqgf z_+-cTY%V7TNFI8B7(GmDR_i-jrcvWhn~Kq;rN>V=NnSZRv}971%_AniQ%|xz<|JjM zQ;Sw;-FN*);b-LC8i;&{y!v4iLecz-vDr7#Htz3e&Oz4>Gv=(T5lT&5*;FeM^9 z-45Sn@KJFvH5SEk@EIAP< z<|l5MKdB7sX+ybN=Kcwx`bmMvkKFv{mCxD|UpmBC>U&QDQ!F0MWfjj6+Ntw1zvsD% z`wgpdM}`i~>=laeURC*zjwp`Qfe(6z4uik)-xJ*oGsh((EMt?%Q%c>$!L3%@rYfcM zr$j03%~Q%Z!sn3E$0}0cY4<-kI?KIkqtVqklj6cE1cQEP-5vYAQ?6Vv=z`YRVHdVY zu(5)r+527Ce8GkbmTrIU!hR*#Fu}N|!G+Bg44jdU*x{`M40t1Kt z?!smXmH-TVeb0r>5UdCD%lOw~i;$jIp%l7Pq!irqD;-XLE-5%xp)`|xrHSM#?MpuP z2l*XWDD6Q$X;XQ8uxgFYgLDtJ6?<#GfIfY3owsHve0P@g&yk+vL*Wt4qdk$9X9`2j zV?B{()&d7)Bh3>$k+uiCoE7udG_#j8slaz+Ib}baO4;fBX7HQoiJV1U>)4kmU{9tA zJIxn6yfsa4du#SSxBY_pzSB0=-`{Uzec#JT%Pwe09KiEbp8Ib6vXr>nQ@u4`9`@9H zJ~>=ohmYP5AGsSn_;u^}5BTQiZ&{Jgb1CyS=}6L%8SKS-*o*hn45eM4 zO~?whOgJU9ufbE(^o6Hp5N+9qF5LuwXhNS3FTg(3$~csgvX&Z)$-55t9DaubHQ~9` z<>@ykym&>1{{o9U8{($w&Ico(RL2{Ae)DY(PVIY;IlJ461o~Ljb;aF+FI775u;Asy zBFO;GIG4J1zg$9p0k%W1c@C_c{~P%I37DgIW6xwA?gSPRj2OhKPxNiC%9?UlI55^{9J0k4 zQ_h94RtE!HWy(!=%CT<81Cz{NNz7GK-*xErz*Yd$x;>TD(b4%9xW0tErOJb!rpD$? z2t+oJcc;nAvirs6u_ktsx4`6O+VQb@tdqUu{mSGK3p_TDwG+p9{Tx`t@S}9Q4Su2a zNOzJvbuFoEZl>+5vB8wThVn1C{A-_EUOc9w9BXhq<)%}v5}Ie;old`jO%crJ!k}ME zfc*fN=*GuxU8X$icLRABnY<+Xz1Td~^iJ}|k|&;6w% zE8Bzd@Rd!;H{Zm!Z=kb>DE%TlvjAJ{Wubib4)eQRyo|kpW%8>W$T`PwC3Gs+6SP|H zACsrOp?ZGw`&CcMB-%Sz zGtvoEzCY`u)SPW98<2txMQf**S!3MMqLlXfmD0!VN*R|prHq|dDf3_{9fln604ei* zOer|grWCw6q7>XZq!j!+pp<#pukEwQ58N7W zYIC_sQ<@7&8yWYZZ_L}h)SL2__JyiAFEOz{_tXCKHeji58TlI6OyuZVVi>eOV}+W^ zJt6sLR4{MSl^^394nAbr`s7xo*~hYZ(> zZZCf!bUkw~MG0vuV>MMeKR@hG5w+tNyqDcC(Bka3W!X3IU%Xc5zUs{$9KLR>eT~0j zQIJl_B9&Y*Q)L;SM6-{VXwFcmeOIbZE3vuKyq{)Y?*8{;AFA`=8%jH-sO_wWADVsz zB9|(qy_YBj&n{L9o_|*q@a3n9F~e;diycOz{He>>ABB!{ zH{ld~GOPc=??rx(@w2)p8h!Y*ZKUe;x|yXSkYzXIE8maUkkB= zt4#y+-|E-0Lp~4SMIZNnq5gx{J@tPUXKJ9G$G~0Tb2k@uC-hU_72o&L2H~t_(scW) zp#G=X$CQqQ|0_ivs3Y}3um44A-lZSk7CTA(2{Zqldyr8&$+~8*`yN;CY@$xBePn=L z=xa4a*p40hLbN_O-cQCZ6SFd2u_4(aJdC{Kj zCe|JI{xS~EyKyb@OMDMGq2e_5_S39tKj(N8%^kFvb^-hb90+0ee`nk*=@Nt^8Nn@2h^+)k`PTpJDGMFIRcs`sUa?)|Tl0K$EBb zId}052A1ioE*%f|qD-d!BJ<_QgxrJ9{0J7+6HH&x74SR(&Y22Q-Gij(<0}M9wvQ@h ztRm(=F|tToO-xRmd**(Caw}MsTM!yTdxdPK=F2V(KR$ zXC)(Vr66~uB7dc^znRYdW_NRso9<82`Cr`)t5|(Hj~pATZ*5`v3G)1qA3AaNllk`Z z*byG(7a7j}6=O1*=MZr{MC+CRA~J{UKP9C4FaK>1_l(sb3*@yEWj5U2fK&o=T6&#+kTJMoBUo~XsUFLmaxU~a22wzD?7N>l* zWP~cE;A#o!L$2@mU3}*tv{7eJdyw^^{Y~8=ao%JH z2S%%{&_PF6aOx62h?YCkEq6BM!r*2jb6Uh$hJ4^0@9A5y-#Lgpp>HYPiS}t<+2GLJ zLHbS)wKv&L=K1U2OSPYO%chWT);-_rsqgmCcN1;R!6E7or&!f*@YDAjpF#IQ_J{sB z=bO<1ir80H41ehrvX3_5S028Ov&_5kE&n8XcBi*Jo!5Z{sD0Dq3t@O&by|araqge-_|p2}5ObbuBr=lb=2Cah(9RqtAZ6rSnxq$kMXuX|GsqIqyE_u9`CVmU#3ebRJ~> zO;(q9lV@-zT{dwhmy)mg)t+w~C)t0F!Y7~L8;(sZ-F^l<62IRLOyeP0&vs#3RZjXI z{r#skROiHhL7m4_%rihY->eI;f;RM68S4amopSwgn4+B_uJr8 z*%sHHif<#2`N_+GcdH#M$X8yD^gi0K-2HECW_-WG{l8x08AE@}c)iy>=kPd zi-Tqnk=5`p`6U|rR}aSzxgFZtX(w~|Sev1@+z&k2Wz`;AC<~kAL<|JKZX5?1mYWB5U8;{0|uy!g)iqL(BeJ3BYQ(T_`fH| zVjNQJag6;Jd@Dj<#`;p41IY7|mENbUU+0~cI`;x829wI`+aHoQiddfdcAjthsFeTm zZI#it+c^UoKIfrT?a+$tLjt$Y8{ov>JK6af#>dF^X3cWe>Q(fOeAZk3HDiK#SC-sn zagI`SI(wkiQURX$7q7{P!q5x8Ro{&2nu?ou*wr<~b6;fro{Zl`r2RbX3LNILQ0Q4F5FuM!P&$ae+L{zr%BHx+KbT1&{Jvl@}d! z?yypPjzh?OTDxK16W)pE!uMKqb~(-vwaef$OVKGGN0!W6_@Rjz@Cx%Y4Okn#t`&c zK!u6ZaP_jD@YS|N_AC~D)Df#8jQ-nzUaR$0Nji-?0UN!^Z>_;+FOUB^Q%$V0E<9Iy zs`MPi{NjASyGK*OH>Dqm_jKx)=)IItxl`S8izuhQBqmzbbp!ZKwnE2-j8gIoa)W z5%8$4_(OO|S6y*5FzJdTXsgk;pqqnvK3(vrz6h-z4{V^}LFx9>@a_(Mig}(*Ub@N4 zv?XhH#HLxmeGBE?xC^KUI?I2*YXSYIBkn`SBx2K`r~9F!evd=<@}PSm zz9pQf>vuOXo7zLq|LBSV+Q!3|ST1qrWs`wQO~=?}<)5V9F;qD{jWC(DJX1@jHno z_J-C>5$#i(6yIF3{9XJPOz;)MY^Y0*cHXameRUf4n@DG@7|C9uGbzv%pWVmZKllaT)cV@q3tx0%3yxXk ztj7}ITHD)`Eybp78H69jA#|T(#r;KFOw0s)7LOOvAI`W%M7zV>ch`t-o#cR4Y*xp% z5-Vm$!jXLbYri`ITh>$1gXNx{!F$_1{WmV`F?EfvUrQg!Jgmd;wqBvXEWS76Y0CU{ zX+rReZM{R!@cc!7LU2uw!2b5vt(twF`&Lb+zBT1mOWE1?tvZr>N`9|_WmATqeQ!q7 zVXNj>XWyT3X!!C~?9FV-ziCp;6K*w(1?-R)yrnjTq`fj`F$ zN&O$p@CUe)s{g~Q4zyY|*FeKQ|Aj06yW>-bjZVG`FS~ZVfxyKJWgi^E+ON;i9z~}g zTrx0cKLsB#XyahoYyrz;JjKiIWetnY{2u)L8qVxZ3hwDqHf0{`EQ&AX#7|t4$r*R{ zNN46)EhVJ7uVEazRT$k0zJiU0*gD7CABpN$VdA3Tqglh8uus+hggYCW5*=IKAYwZ1 z*zfeU`5w-$Amc)hYILqjdcacJZRT9}HlTBT)7+ampTquCv2|QJbR%)#4(#q7Ix^=J zCl1^K*$3f#jXRcq;k&}MR@S0;-usM=GoGD32d@Bg#xuppY8rQqspR^j_>5{y z&v(ak3}Y1?x8Xb={@4?GM6n7|11l$z4#97zh|3y3ao8sk`>=_976boH$_aNkOvT7QRmY{Ky5_)zQAnDK;9Yu(7!tFa63P6!=9cRfOUVWXGMwIZjwW73PbW8YG4gP7Y4uRWe^DNt?@TeXdH6Hj> z-m5XRG2O0n-|4hfQVu#cn{pE=_q-_wjPIzPW5761HYA?-?wnV7H!q%Z4WsEZ+;Xlotf5h@AlpQj|t`-fVUD9*pdzVcA*AdI|r_95+y^0Nk9`IOn zUoFEvM`I0Me3w)-_Bs0VK6Id#xO=jnv$rxbcCD?IY(nh4b7@aB=H4eu@eiZy-xO~T z`r1y6ImOY#?rq}g8NT^aS`dC==J!;-Wp%`F@1K9@|L5Gwrl$V<-nF^-JLWjc9>6#{ z>!EXwk24p}dgz?vFt8cO{hHrf+_{o299;_8J>`2Yp$h7M(6rk_x;Yf z<~#0O|CPLG3_gvO#^y8=htJtpT||3}y2Nk~bKjljU3;hb%%AQms7qxp^@KT3wMR1N z&U~lZk1-Y<^PQA#=DVb0zS(18uC#}$yA#ig=@UMqA;l*h?(x56WI}kX`X{+Q+P|~9 z^v}Vw&OSawzTS<-iWGjAxUnJ|sjC#7PBfs1xRJVx;eYL4yTMUM=D#Mkf3wp}|Gpd3 zzeV&dbM9eR^zv&(wVzsrG?K zqwCkPc^9aE(7&DT=TNWs&&SY(S>~K*qP?E+QQlDUBv-CAbS%jp z#Q#H#={4Y%=!VXLepTVfr1jf42g)6|OQx4iIfpUi+w>>d7Hk7B;g{MWTG+<3>Q6L% zGCowB%sEha;a>Lb#mlsh@I%@mKc|Qpvvk`-`7K8FCB7fu6@QWsOFJ@6L$TgVvxoDJ zhw+a*+lzdwFUb5Ie9s5U!(_QdW2Ky_M6O;+Sg3J>iQ4qEvD}# zCe1s}8P81ncHsDzn=_tKIvTB`#;xNc%817NjdXTw9p;Q@rhPqd;q>dIhiH@9wbQ`T z>{m^iYHv4by8S1UCfhHWG|7I^q>1(xlcpdi7l8*8z=w;#iwnVz@yN-AA@C*m+X_$8 zx5v=@=Gb(xI!em~e3r50NI#M+x(6S&uory=8%;$|YjYLj&l$y*LBO?Nq_L`Fem{0~ zm&J^Q#&{)g;h5r=YwQ=e@Z~Cx{PUrq-)XkSU*+#o`EqY)8ahoLbEog0@1(oZ<#WJ0 zkCGl}_lA<#(~;~@D;)>kRY9tFbw^#@ri3~VfIHG}UPZr=4Nx=k?s4MZ+f zitbRT6kNzRsq9qW952(+fs&ew(SH=*E{AbfY`X^J;(O48rnxcgWP2%+tc>l&Pipo% zP8QFOb^eBH^H&?BcSg_EIWg}t(3>3F%Qm-MHs#bOo#j=(2BtD5*m4z{rRiZurcR-Z z=#%7X>Wmzl*(cwxv5{>?_e=18BX`BfhTe)yc&yUM>wj!4?0H18w`7yXVa`2P_n~fop7jnHk)}dRjim8N=83hUSBBR!fEzADrRZzy3s?##i>UT4<1SPe!NS@EowH z-cS#p8_YL{M1cox5m+Cwg&zm8l?d2m(o?!AYNK>nY9dcd~Mu^T1ZKQZI($Q?6y*4dRSl`_`TN&6T(knYMzhW4gFgHxf!Y0zYR$rf<;D}F%{ z>_A7b1I6yCD+ai1>qi-5gMaQh`bRIqj|>`XVCe2fhcCEklyuR-@qyd15l!1J9vC<~ zI0zYXP;F{(wc2A@n@@nhml}Ks_&{i>!KX8`f}h=mpCskX|JtlzSpUhFY%Dw1@da1W z?kj2k74%~|{rM5R_xsLGHj(x-XZBeWH!bFD`Wl~e zhJQ_>b(4qxO~b5+&aO0$v?9&dSv3u`y?G=u8|z5U8L?Y_sG_SPECLWiN5_juFIGdyhCJ*5emMVvj))){jCe?AnuE_cIz z1|N(1K3@sWh%eRA24nX}*O=zQJ{C-Nf9UvR7xunjIfA9y7rU@`1%q!1k1lj!{|8Lt zYJBFz$B<7;NE^}P@|Ciluwztj=VyGchGG9_j^&5#6YpU>5*IqLS)Uf1vnn~@i`rtl z|0|()D%Z$d5jQ0>@s?Aks$J`&`OJmd^gH+eINn#Ag!ijWo3Znn{sbbAD5Z@LDIFS! zJfM{Molklr>*7{Yy?Z^+=Daqsc{=%td}-2b`FKh`8|vD?m+_9|KXX=_HS3&pQM_Sd zpt@!5)OWD!#rml=Bou@)-2B^39iEWBy#;j z#yrbkxqSJUM%vcM9U2BEf93M~#+YaPmE+lS9E2Z{)y=zcIP0g0m}RZREHieBTu)6b zkAY_x9up6b=^4W}{Ipa4;)=28ls_{V58p_HZ&Zy%=YV%K(nhU~GU{-iSsy$<$~WT! z@^P|~uF&&9cqBM_B!_(p^=lsYqCE%hvL?bWxIAU{Ddybk;p-jz{Vv~^95%3zRbBUh zH~5u%_x2Fr;wNU_dy&UHY{%zGIG;lvc3-1sorT=v_$714uj{v#25b!NsjrkR5V-4G zJpj1FZ@ABybv&Lj$Dz&6c~xM{k$B1x(QEKpGN1U5>X0v(_(+SP<@gN8=7E3W9Ze=L z!@eyxk9D(?yar&7FCBG(1ABq}%al*GZ;UO^`jU_I2PQAszBV=w{FkrvJ0_2Mzzy-I zCG<<{3EeTl&mARx;_b%!wqKRsfBRM3Uy@PA9rDL={G8=jI>yL6I%A3dv}_rU?{vC7 znzg1e)?L@XWt{gs5vVadD(h6k!)|;i>Ti~AzxW%+-)vs@2RQ$dVn5G+@mw=k$Yy`! zzs^%`Qp%Vi&tS4d4QsR5Y6W=r7nRvU23pyw;s+stn7kd(V5=U|f`~0K)?7RDk?>QEJ z<&D4a_PX3P8QM?%##l3joX3Gj>R#$rcmG@bQSq^3`C}Dr77u-!_R80+EYliMJtI)F zc@lRUdy|LgET8CqIK!IuG3PI~@Oy;culY^ocRIfee#g?0Rk15-e|;6t_Gr%KkyZ_t z{t{@phBFvC4;Y%Gb4~qjl3jdo|DxLm0k7gduWWeME5wDaBCbp}FGpMww5)8OhFU!Ue3 z?2$eL9cjJDu>pUE47F#1C)8Bv4Sg}% z7y2BVgNdauleqq@f6;+K!mFv`U-)mjuq?qQ3zlYgcVVf5v6m+v<8fgF1e+ijc5I$i zUvI$*1xvIKyD-0CV}Z@!T(1lB35NZ&?mA)yxUjF507EXQn~u%cg>fMLLE(eqR+jXD z=1+$9lQvFbZn~3B;0&YE!a(GDrDNf-O2LK3ZltW~5NR1d!)xjL->KW2DJ6c8`u#Wl zhlhbP#7D_7FvZ63+i$BJ_iGiylhyCH%>RBZ4Tcw|*sr_)i_L$|POBd9B)xF~abuBH z;L+;y%Wge6`cHpr^xq$7F?;>=b+h~bPQIlhp6C>}%tDpvkO|x} znNFD`x6H3pCX;%6#7RLWGHuG{Tdbu7x6B-sK`wL7(K=-s62^yoZho=yk?#y{G4~bF z6Uh}tN@yBO-c+Ke=0K`Py5>99_EU{q&~lwKCqw9?Y+$qGPmvfG zJhIY(4+btBRCifQM%;?On&$9S;Km=8G0lUQt#9x#zMb(T6M>DOJ@v(kCji`)33>ys z&*_AX7mWnw=&Ilg<2xHz>s?XV0T^Sw1lT^oOsorFjQa*)ji#J@=7BK>JAv&5rt|(c zlREMQdEoh8^8Tqj*72O!yvdFXU}Sda*zDN6NsbKgws|kr{&8#`Yid05L9NM4v40es z$GV$M-YdWye+=3HZY%-zl3?b1Ffj0B1F#oN9h^Iftz#_xA#aoNW&|SNkIfrRf5;0d z?>g>cYF7X(Y=5E`S3nJZywtB z=Z!i2IN$Z({W5B^{%~n1>-U%5leOW}Ia!sLUXr!`(zCMGUD^l#vO}(}t~)4ohwTCU z1SE^-?61~DRZI+rjpOjy;O-yna8quNx2mt^H=5sbw6)c5Rd1&4qiAE5H?28`xR%@S zHEdu%I^<1yYpiErOF7Tk9)C+A>3Kb%4+-!uWcb?dJkwA4T@x>}TKaTY`?NfZ#))s9 z$C|@dW$v=i9u#QFA4=?)l?nZAvlV1-HYrT_wLdNR9`F1qs{E+#Z%sgKD z^DjpK;O9|ak8SPe!be22<-ft5;mCeIg9HHt2*woeJ1GC@v-`d-*+8$&Uvnl^nGB`X=R_QW1TqmIogM> zdmK81cYhu8?w4KOje~B^aNjM~yQ~SlTj#!eUCg_Cy1YA>cQ0|@ zokltJJsKb3E8u?7%6pxC!GptDzxuE9vwh1hIEX#yAaic+4&I@CvoyO#bj?S6I-WDZ zojvFphK4>qkbKU)fn(12rP=43??EHoy92y4j8db1#SP=haaO5>+It?v zzIvJPCC1k@8Xw_Gc>Gvmk(Q&|Zh%*g^~N>l z@I0y1Gj&e#RS9$EXHR`?(UYss^v1pQZ*;zGRAS#&5~uVtbiOsjAFV*Q?X{>k zFfV#vHL@l{$PtQ3Y7b?;7WG-(IMg3{ zgYqW+=uj*2S8z>dwe#3JE98D)>5pUCPix@b6zPz9F6X%^wzB?JcGCsxFttHQX4|wAo z?rp{AQUaP1O3@)Tl{LpmNfN} zei2hX+e&TLconnuWeYH6NdI$^GHU;wtfM!|OgxSON02Q~9%nQSnJ@uaggB$%(H+?5 z#JB&!d$RL8daCY{#Fr)!{c;%4BC>hE9E;!l%eqQ31 zc}iti3nqSPvb|>-e921Xq|rE}iYFTWg!5GmJwnk~q>70bp4u}MncF8MdK-qW_=j62 zerD}Cq`Xt1l)t?U~&gC%TTow~sbFR1MbNqjQKn%-cBGP6^H7UY_P* zr-s^EiC@|7sX2@L*7o^$uLnP3X?{se&DI>=9pnuex^6pyq0e%0+evac{b^krd2sQEch%@^C8IGZ?n zb6;DarfCAUgGtyBwgze%CzFC_jl;N zGe^SR=w8Ed(Hx)RY&x*P^!JEh%wZHZ9+>nFb61PXq3_Ptya={Kwg(5c1X!bB&pEJi z{%_#77nrkeft(7C?F9C=V8-?UjP<=2m}GWmj{z9CC%N)9Q%?2>VBn$T#qFk?Vt)Yx zC&vTZYRb7V@N+h>=Yh4((3(lKFIeG>)6P=)gK1AB(f&U9oicDLWyi zTd(+0Y(3+tXQErrWxV6W4`q&_GqcGXM_z}H2F+Ol>;h9h$$riK)+O$@#ApBPeka<_ z4U`vu*Bp*gJE4EVgXi6MF93GH#f=wEcsI%3$-A0A)jLGxrt94#`w92m@2EcBt+p6D z@}4+p9~14p)DzVk(p1k>r=Ex0da`2cA@8C8QjcPlenpnMC^`Xi!wAYFOmhrTp-dRK+dj-G2d-ZvGFY%-cswnbQxIvR2<$ z$~t~eDQkR}Qr7=FN)x%~Noh_X@_$MvW1lCj=N#s@`Q;tO4y8L|kDznuerv@ak{wFt zaJB!6tZ!l`l&~k3@O|u1=pD)6`BCgn=G>&miB(X-{R{MGr9b}TRf^T8Gw!8Zc%pybwvH?dhxE_(f=IeGkelPvH13Oqy;_GpS{T=I-TaH^-c^?}UPp2>^JkXp9MQe2bcRU& zakrd(U&dk14;>tumA{yCK+V@UZ68eAwI4f>^Am@eSFNAJ{AyeqTNe51kIw#{!TQ#n zq$(5T*ZqMxwtd~Nnqq8N$5^9x7<(CZVV*^^@drx!+FVb|F@AWz!Cn!a=boHvi9UDd zc_wFrv`_R3I!$Q~XR%3Vaz5y%C!7z;wD0?}^L$Vq`<-et`VO%d{`>i$t^an$pkbls z!7}_MTKnQ#(dCSg<}uGbBNRKIl;a$bJC}2l^NGcOypb422X^-g9hsZyoNbwBMI;-^ zRx2Jq+a1%_`JOYT8TPx3PuDSh30T)LeFE4&h#jc)Ho_fK{8f{_70+lJG?eqNWmATN z8~-bQsUK1NQr}L%FZvgYU-Q7P%cJ;}o0YF~W>a1KT25cB1jZJf*6=dM;xu<0e&deA z0q{;X&ihU>7V}Rs7PB=L&_sm^>N(e zz8~fN5#ASmYcFRu<28}8>Z{qG6`XljeMf=&eaY}J>)jCNC-4pV2;Y!ZI`@&sd(Jta zyT&Z-I0u9u<52op9`$2Pw8b|h+sO|%i80Zc5Nv~+#!VZ{8bjAx9`zA%&f|ZV@sb_) zJERBbi`pq4&e(o;?Q42WU;pqeeU;39LSJcv>XS}-(!S2^?wozWj?PrBzt)(|g5Haim^MCV2N25PSV+If-A{<9Ce zX`s%FBy#qrOW)6rndeop{eIk;=j&to{etORhP~OOsrGB%V#|%n5Cp|Tt~gPKXdBJF zU)X6f>BooNtm+zmkMRH3{3@tlG)Ok272uO(gLS~n8bz*J=E5EaCVinAdEp^`ZHwfK zm~QeF$Dje3BkW5Va+Qg1kY;O53l1KCowtuUJvfMSBsw!ZntIhP$-cMqtogsqq>1)V zNHtGRekT0=zu%85nJMGiiz2sUW0p*JLr1;ob`#?+xk_^5r#$a9&)WZ&pQ8K^lkL-K zgA?B>#ZD#m>p)`i(e@SN;?4jkRW{T1Fh4qXeiVD_0NT~#Mu%@)&bN&2%eWZX?O>tK zN<3R8xrcM_*k<3!-;)x|OZO0O#)^#QTOTo>PQNnjF}%N>|HyK?Oxagbf@9+??1}sa zc_K@@^S-AmqtO6@ix|H9${HS-1CqB3w+i^jAyZI(QTg$QwJS~gy z;WcgI`}%GsWizSw#c#!@IQYL47gN5>u3Xx2ru|xQYG8M3L>6_Xvv=7BPJvq^Uf}mI zzx@lMctt!B`X{_fAa(F6-PXDmtUIaj>OID8Gr#bnEvqw)&6B$b$ji1eJ8)}(8#_bm zUU1=7EwWFH+_=Iz^Nip+7q>Rh2DMMP)xfi%IVMfCUndoAIr)mO@{PE)DHGgUU~ntb z-rZ4ehAo-T@OJR%haI@(#7UKHL%8*C@Ggp5WyCY+#I0O#OJ#{+zs$j{V=iv}(mfmS zC%$!*@d20qC)^t2;+E5|O#An||NjEF^xZ6Qi@W#?ZsqXb!}zxQx)CF%H+%Md@Hy{` z?|DCb(EH+{~ZZ5!XO{||0 zsr;^>f1+P$=peQt3JGbIu4Jy6ace zbveJIzSJS#D_ykvYIK(mEvNp;2KTZ|{DjUkLs`wpVAI|=_`z9(Xql7Y=a}5!AkJ=z z_8q(3i6@O=k|Qtd6S?{FqC;kwFtLlYv>8R{Ffze!wx0nUDRUtrwvY34#Q zpLnhE#g7Nm=fl7>PvQ@;dE?3Z(#>1cB~N_05!mzWD{3x(?dG4xy#3(@hXx;DEY}`v zua=)BNa^trs@6{~c1sR^2%kNsJ`nw8M+nV$6KqO%`c zMSO!!8Pd0SOntc(Y{+rO`*d(j?b$@XuX1R^CjX$}^6icI_l)OUcRaDRirQv@mnt{W zEx(TPhOdp+oz9Xuv+POtIB{-N?lJN|Vc*$_Qq9)A(xJbXT5;z7fVZC7)6{bVjr~QtGmPw+`c3;aJ%AE#qm=NtMR1Uu>rK7 zVcoQYbWL`^oLyOy7`UlG|9{Gv6;gv^oZSf%_jxV+U1xUK)83>rJ5Mua`1Ef!-|hd4 zQ5EsuJ+Dmry|;}FddaimP2M~H!&qbDiveJwQ6L~%pq>G+2_Omhj$O=vIj_vl?LsdT{Lo; z;w~lOqpI(vkT1K9Vr)vTJ78oz&UeMYz0lyV@!qn$ju?-x&39}zM~5Nn(;lB`dm!Rb z3eRgNU-a(ZCQY_wYl4nV5lt3-P&@?1_SBx3VuAjGSWE$Uz+U=KTv6$;Y;J~2jxZs(8amu`|GScO-&u!EBnFi-x4(B{3 z_uHb|UG46pyri)KkCcZ+3y^K= zXSCgkSAqS@jY)DDFlUb_(MI2-T$ULF={KUQZ?JZpF_L{O)7BV_FnQ_rtFd|5nZ;ku zGI?qC*4RA8ZV7p3n7qV}*bN`?KfFlyq+hr@JWu1c-JEAK{Z<@`Y&Xt-A>+r`IcHP2 zU!r6D=o|fBS9pc&4lBk@8yeJDyas+r-#G}5tN#Nu4#0=9M!&Z9pUMv|#^=prcP2&$8 zQ3@XKG5=HSPe?z)W^e}IFmh_SmM0c(Tp9yMf0OF z>r*cLWx>m>P%<*B=%nh*bNX&%)`RU7BA`@(Hr*BhfM^lU;WX?Xb!JfHtWe@-tn|b6t-> zZEnx})(^Uc4vvK0&9h#P-0I4_<#&%!Oz_sXiAnT9&rs{z#0Y$w`$p!nR)%|mBky+T zXfgFkW*BJXSMD8j{XTyT+?9J*cz-`3@-xL ztSc`!a&0McZOMO?Ycr8+SFe)qbM*Z>-<@OXcOcgmJ96z@@Xw(U*u)wCJ(B}9r!fZO zK0J%@PPf-GR}Njyu=m~S?CB{!j`EuLFPSz>`v39v?(tDp*Z%)AGl9$mFhK4JnuORU z15%2Bh+>({L@uJBfOjoX>;a?}@mfG37_33s%7~Uq+5?EKnFp=*7?gN9fV8Jr&Z!{v z(&{;ei#j2;O1b6X()`|^{X7!_qV4y0zQ5le^IFgBXFvP8*4k^Wz4qGIF;?vX#b@mW zP4|YTv!Lm0XgUX)CWhaOM~7S?($*KHGbjZJL~ZWe}dnA z{I2JB0qJAXt=`F?HT2)q7UXyFo|nNN(f3x;T$!%OUgxF#hw^D%r~S`+X`7VB7zr=L zEA;KFO6%svbIp-#2X8a6MOo`iY*FTFjPzD$Uv>V1??mfsknbdKt@i$FFHUv+JoVn_ z2l@6B-v226Szo}vl5ZINlg>JUcBt>dIdkut=-;##`WK{Kb%M;p4-Cw*zY7#z={xDV z$LC$m99+d*T*;hFW^N{-(@sREy#k%qJ&&$fqXG7BHoyzE;G1_sK9b!S-zy{c<=ENS z(4t^t|(-Uk$=2vr~aDxroeuo z^Ecpv==Ch~F5lMrk>W!-pY;^)^31yvw7ix7Is+nm^#SP$HP{yY(D1omrkiF#!^>7A zky%gBaP5d)0roT2xHLQ;Tsi<>F!nb3@*(53le5Z(j=O)8@yWJ-#n`6k_}D!fqil0u zkYtSm$XJR^`9(Y&7_TI5;J2ae&NwLlX#L+yIl@bo^$E|{iU({?g9mhn2lN2X&H&#s z!MmQ|UoUt7HmF+H{*;R2aFn%OcLMEaZ9nB~NCyx2`g!Og-CQ0poIS04en;5%{ea&q z8P<>&iy}?;02_+&p@NQyH85n0zo;L!;NA@(Yv@|_m+=!`yZ{afe}6-o%Nq*qUwLU8 zln=d4^qTO}URD})C-4>d!0H^MD_^*KZ?aKY*kLsv)zokJNl&ZM z)yqc=c=6D+%X7oyG$EE&Xk9NjF%ZiaJIXuwCwY3c)O*r&# z(3?x=$no%v)c$)P-n(*ubPiYE$TQy*n)M;|%`V3O7Urg>c~@ZnYp3$eSvKB$5ybXvM>1H$`#!G)rt2!h+PccQ|9_3OaUKVn|hYLiLu0;{f9= zy8SKx#g8@vqvA)gbk4J7aSke*^H4dQXUl~*;kP+0AG|IAzYD?hB6w3D&hMwk|G;^+ z2F|lB<~-Z2oM#)ydA4&n&(;H7M1DllN90E&eMINk4tjn>IvxuWI^Z@*&aag2jr9WrpbZn}V zGpu2A)HlP6vh0Q6tLWiT(xeLsFYoixex!WNxyI*SFRe~#16|&BhnH5XG}a)kf46yQ z^G$ha3E9?V*Qyg7h^!;nKcaIt9dXuk4S&nB|AlXhtz3_%5%lOP`ssMuwsYf^9#88|+cci$uCa3B44C%QHbL4rN{}{J2|{C0 zLGZ2uDBh-T#q*BOyMZ~sp1Hpc9Jm%-xCY*KHN5RAaO6sGWin^xxQ8s3Zus1bgRyCX zV@de8#;Tv%43D92O~sYxwH|u9GBNfD`+(euqJHe=Jjq^Yl9;fqn>nZPAu?4Ra4p|V z;T%O7&trVuIJbzb#TgCVSJrVZPdalT!|;#Yv~~Srmv80M2kBlV`1hEzc@ISsk}-7N z`Z+I8CV6V)_Y3YAJ*Lml`0qS_;UMoUy`!$f;9R0?bT|*bxqiym!!~?#UH|g%Udnp0 z|B%G50y(d4h~=!`OB{xSlkz#QR}hx$(Nb0z{>$K^0j>S}3}`7kbHKqJ*TYhJ-+%fX|8`LX^iJ0@~<=b``EG0`N5T^ zc&GJWv@)XeJMif@ynC87_beV|F{gV-d(z}Du)oMnT6i3dF!P+g}p#6KI{ zzPvimJwtq6p4FJc^Wj`;$b0;P=!tc?*3b&-OW7H}$@qweR+4695B7C98`@9%klcaQ zZ_I*M>igVZz*(JKdjNThgMC_c_Li}EKD7r9oo%zK&g;2tUqbd0L_$vEujkXMH}gXP61^K7MefXG_`qgAwu{Yh8ek9IWv^baL`FYrbR;(1NaNh*eh58e}i9L-s)iKcRPZ-V@BXOFa5oOPj6%cML9e z+7sM>9Bg&$3Fg~oU4kBdMt)?ZZPBV>(4l+p!YFvByN>7EkMUl6f}%yeU&?>s)Q=6! zvhOlb>)1cus}>#ZqyItp4Zbc*!(a4uS^B^9by*t3*G2k$2!8~PcgV**%^pqvWpFpv z)b7Y#U+t6dWirzNpu1y_dA{4aJ2vGba}xcZi>(M>7}@ORu=ZS+X$?7_UoO9u zf!vu}I7|5I8QhP@`7V4Po4&?4y0U43JpdXJFZdQ|!YvEjj|W z^I`VxI9ntcrkwAJgYMqlLEaO$q^U1w>At`t?i&{0H+@Z^(b^I^;RYG)HKY+Mqto1+RB8 z2EzMffcSu(tx)j-_Rv4qzb4o0|BWSXIXrNP!RKqv2cPA;m1RHrfV-C=TcPOpr#HK@ z!N=4i8;<8+voz?@Z#DMs)6uWS<9%?+ZCj!JU*6blH89)$wSlsIe>p9*_P-RFZd`}| z#{bu}pf+bg3&fsto}ukueI~s3_BL?*5IFu(x$w;O1KWk1c7QcbYt<{vp~0sx^0$w8 zLxIA!AA9}(A@JDEX3u?TFit%7GS&(C&Pd;TCO01CJw6HRlK(=C!cy0ULcE=c4tR8f z*|#X{Hzhr7c0lJpjSaMqeIeh-F29lekIr~pqkX-fo*%!-!{Z9_Nf*OcC)+{(dzo_+ zDkBNLPRX=pPdED>HE+o;*vd>BzZbln60>GM>7|#DuGrJEuZM2C@6nqA1F-;CVWOzmW3?PuUlxz%&DQGAm5C;*jhi+ zSESf=`&P@ZB+zqqhZl%+QLmPeUWfNUGIGeFsl_hzDy9Zfg&g6d;d_ZyQw3pdD zTl)uAw%*}0^_F}WTgthAwujCI`X_B*eaP*#*z6NNlQkMIS53?s*eB68^%U0ZE;(B)~SN|9B{uR<>^Ek8yp2+{?KzyXjh&@Maf$(ov7Vf1_ zQ|QxP?3UqP4;N;XAQupWEgKcIYjwD-+QzDYtOBYCIz0Z-6p=wpcZZKWB%BKs%aI6h`zq5UXO z`jFQCM@(9l{UcyQ;JUUqkU_)ngE0I+xVRG=o7R-}&Aq=e21BY1q65|4m-0lzpTMUC zlYOJZxA?7Lbqik2KyRHnRco0)I8HX0U@xopnEz=XDx7RUmdM2)=S659f1IWA$Jq(b zkw4DwDYJy}8BTf95mfG*lyB@<>;c1nCk9DB={sWVzRGv(f1B9P?BULKvg!);Z4-5< zzTo51sjWuChl1zFyOE!`eY34#gga}v4>1&*8(A`C?m^-~H~6iI6@g#|a@_2loSQc| z4L#IpuTi{lL|**p?c8%n`KoIZwue<|*1C6)4Nhto{n5M}lARkqdeS@Eta)LdqY<9^ zmgsZ~whqy0OCZ!Q#M+!>Ek1PT8R6#hA|^g}+pM19=CkB4z`jV7vU|O$I`*%7|CmGh|AGZz%zlRO` zCOR21ndb2xbh7Dlt%=ab%pfr^rUZ&+ZsBg(7tqgs#%~_K>F8+{=xP5>o8A4qT>C#5 zW65Ukk|uer8XVxx^M2g3$ew!<@{vW{>UH&lk%6ScyiWe&VAkn%wVLxgY~ldE){TarX4%h?kF&AsZYQ4w9a`&q$8VeB^7UWwPHT1v z_T^)+&@Rt|557_&3j{(Vht=Ckpn>dB$3s24x6;Mio7wYld0>{k3f`i9ApH-pPm>1zEiwEi*A4-ta~+|*l1EZ}^K72) z@OV;6_xiecW`O;PHU5O|OiJCOa69V;_mZ?u|A#tL5js=-37rYqMfmFSP;@5tsdmR) z9=e0Mq0ie+^_`TSq`jmR-*m`_O!<6t#0B(EdjSRZ@dYRRW6omi1J&-BU%}X{0w-i2 zGq{&J4*=b>7RMJe#+`LS(R@O*#n@=RRqq*;r~j-)>rA~r(|YaJ`yutF@@X8;p+3Fq zXJD>fVPLlXbpx~P?gnb#^$KX}a%gJ;G&UYuy9_(yrPvwAL4%h-i(|1fj=?v>-A@TJ zf2OUc=xD^${QQ5TqnR;5M^nEGkV^}YOBF{t*gdi}=<~HkONgca(8@yb-sV7Ep<>Lo z2YlNPUG1a&et1HFy?ZMTUN60Km;6|>OyA8sZa*iU^V`Ymvs>Vgq?gn8x8c{wjplr* z#rjn9aO_{31J?0U=CgF^1@Zf4LC^DIlpSSkDihQd-+E4aUJ30xHC=7_$G)lM+Ycf? zE2a_wPq5FsF_okvieH$vbjZ#HZKGIwS?e9yWk%Lovtwhe$wS6kjYa%|qkUI!j$_c8 zF^1;1rR@1#2I9v$hVwhZ-5bX>9hrq46<#Adc?0?n^R_L#Z$P|)Sik*dD6VbB%x=gO zk~Pu_B6Fv6FPidC$0sYiJw2{ck+7_^`Pajmb zqCJq&%J(Z+hg%Qh>!CV=j4@-1ALKXUK`Y`|zLD=_ai8ykv#i;R;aBZDEX9?dC^~WT z^^d>CxChzGGC1Z7$7g)Kp}t~uL40uOb;B3npSuuwHe}?9?AB5}cadyS*t$vbdJjG4 zwd$_B7Rr`AU3Fy7uoA^L`Vz!ZV*b^h{_$T@Pl@YJ6U&zuvTtaUm9tU!m~S(_if*IZOB-<`(cai8_TL_dDs$dkQ)wzGYCk~%ck#g)Tb8_xCSbz#4FXnUk7e6BS# z=!*m*`w_1>gLuuH?YTtpo0kx?xf?N?iP3zCVlh>uRUYFVGV7W05_VwK7(=+@_aSK=X=ec%D zTd}WCYbDR>&z!)NHpO=dV%OT-kMU-;7%^Wf*#>CCn65&#bvk0qCI9&?N@ z`0!rpBF4a0#XAZPT8usIxM* z1f6XLkD7^nmBPC*w8cE{J0~98v|)WP%Nm-(!$~PTL}nk;8@X8Y&s`G9US@tfa7Fm> zW%0ciBAXjLQ7rEsihFhfPZ}~$;K>-C_cHI(nePzuy%#≺KN}UeGo3UJ34ZgFY89 z?>mWm)eZVwz`QTi^TH|ci%@H+o_$l=maxyXnL9RTGT#eIt$j0i4z9N1%+tPw#8}(E z#L(e3YlIc|)hsmUW6G>m6Sp;Up8J{a?r6rB_eJjRlnw?ne1D@S8kXUtY0=QlKAPAO5&S}7gWSo@y+xK)`Fcy@ep^z%6BAK>WhbIM+q`_ zMrEYw;r8|+8T4z4559xmoI$@@>9_XER-?CPg!G)%TB&FFkj`uDqz@Ur^qk+S{$%9n zIj^;p=d3f`wdi8;2Cp3p%`@kAfzFh1v^nBv4?6XJ4(Uou*7l7TUORYw9W-@#itg(R zuI)R@n!LDwWg>ikM!j=AaY)lcOYZDy^~&?*ErAZ=iiM?E5UUTjPrkFz8ggZt^;iwC znYcftjFq1syl5i*ee5~v6hCXuT6x_wJg4Sb{iQ!mQ$*j)p*tSXx1Je~DWY%YFoSuX z!aP^z?k)nqFx8ytc02Ue$m#?gRFtU z9tBH6rNk8+#@b)W92~!HN3dqqMAq$nLEp+1X1(LyS+h37m-~?~G|U>>PTIf2OEd?Q zSksd`Sd*<_Mv}OR*pHX4gRhAvt%7%bNPP{b;ce)sj^6*7_mZ6~^dvKe)YAq&)S-*I z_<3TD&tZ*6$6ERhu z{=4fubOevgVl6ke{Ds!=R2lz9871_yfjCWXP;MAmD$Kmf?y$NVSz~iKHiuEja~C7e zRUzxfkaeTTx|PVfzx%FaIL-Y*&d3QDZZQ2Vw68O;z`n-7KK4}x=GZSBY#*vU8@K** z;?3q+ac8iNO^7&g=Ch=N^#i_>8n0sHtiyrjE7~8ioSjqMa}&G$meHArTemjam%aw_ zmSBHbyQ>u1@f9rjIpx%pT0`2VbWI44N<11l0xpY2E>wGBmUBMqO8y4jruh+f+s^McU!)py%XjB$@{j#jKzsEcX;W2 zNncF6)xIpAr4yb3E$e@#z8USdCBSnshOM~VGI80a3totQPjEDR?(0_L>+~zy4DIv( zBIzv1RKClJ|N32cL+u&yW`6qrFZ^#qM``A#|2OlWHQ=qJ_$B)jNn{xD&ET9n*N4Eh zg4+`H73Gx)`9lUqPfM|?tlS$_2u_%iA6@pyh2JRF`;%o?Lupc;?ujGw+$9@Sq`Y&Dwi8PnI{HCp?Q zF&=HSE9pn>_&NJA{cA3gj}Ia_&~MG@N9mU(WLN$ey0pQ|NW>blQENi;Iq|0$ySve~ zlB}8Hb;~%%dgmPj;&=5%>vf1Zz1uXMErz6}nDI63eaeFu9y)1Cs&L#fa6vt4pm0AM6+l-Fd5R41A_m^4VZH~Qd zCiWi2cs}d(WAvfs(NWGq+AjXC=ReT)RJ((;yPEP#m&E?n&u=r&duY!h+7qHZU!y&X zg5Nny?FgZNe_d^=_tO?1ZL!??zbjuC>v%K#^4HYeS&oR&jx*qEvg@Vlw5YR$x=cU1 zG2iLdaoVQyZ=8>`;;T+6ql_|+bD#7NDFb?YKE!%cL0XbEO>19zc>SVX=jp7oDsP_Nb($!62&lg3N3jCis7`Z|5pTwTt3ed@bcymvaEbLu-{Z05@RO+X zgy8O|vt01QsPnks`%!0^;JZ;LF1RD=JRCM{rHlxmR#y)VW)*KI+^lxIF627F-r} zW(htLbu|9XOQOz9!H1&G48hu{Gfi+o)cKC!yr?r(@ZP9%li;0EXNusgsB@#>%&2p{ z;Iyc7t>DzCbG6`Rc#T9(9HbmPMUmg2hp1s9;gl87!C=bp{D$ zMV*0ynNeqeV7I8#UobuDgardpr(CeT()pU;(MqRG@Gxgk1@~7v#e$zzI%f*DRysw3 zpHw;pg1al7Ji!kuogBgUE1fLCcPpJ;*+hfj0f=6S{Zo$JbXQ$x)nDe3FXEEo$1zTgz z`+}duoIeTfjydlNei(D!7JNVE>=1l6=4=<-5p&)Yd^6_!m*CczvsLhSG3O1z%`xY9 zf*WGaZv|h8IhzGviaDDEUx+yy1fPpJuL-V=Ij;yl9dnul*TkHc1XsqKb%OOV=LNy# zG3VEU%VN%Rf{#G|f=i%(!H1xK!CL5Fa6!!Zh2XrHvqtdVn6p~&&X}`Oa8}Iux!}y0 zQ!h9z<~$)dHRdcAoDy>$7rZ{^EEBvs=EMaj#hgb3C&Zi|366_7O9V&9oW+8%nDda} z$e8n>;Ds@#R&ZF%StvL-<}45#7<29y>>qRH36{s4If7*|=U&0$m~*#aQOvnhFfZoJ z7R-t{vjmX~ZWBa~oGCbx`!iOeWb#(mnwD~`LRLOtWez*s@n=~ zl`e8*+AHWNK6Dj7It#Y#d1?55`_R|idqod=en#>$)LB#G3t?oMiuiZw=a>p)l+)NV zR|u%zlPY&0H@sU@Do>a^8~thVa?^%fyDxAZ z_FN-hj2-1*?}>-Tc6Az4X%~5E7j<<~W#pT0v&MDa18AG_8`Jg!6aJlf^07B*;gc=- zvLYTLzwoDm_7sP(kI}Y~v`Mb300>BB1I$OwA@7b43@4%Yu9?H)o}-5HbFr?FaZ?mOm4 zGy62BmMI&=scAZg^;KuEpuNw7=<#a1{E6JLI$>u(o@(O$x*a_LA85I0C)SmEOCB`g}J<(JQj; zYZ)8%>&`P}^d1mzICntYPdoIk8+|r%KYilffI^j3;S2Udc5IsNuW8Dm9m+ReWsDl_ zG>%Gl#_)VE`IN3_YjCBr68&&j;pn+(KaCuz#vW$vv1T;~e9txqtY?J}((Pj8)qL*? zus;d?rxBA^d{h3V+Q*iUzwB3%H~W;nTNs4iQh6ndD!=%q@@~Zc*6jb&&pi66ylPJ= zb;veQ43w?#L+%!DhOSk%#=h8B)GC@%dFkXyrbnFoIdjno^X(k-ZMK~ST+jU%N?Xf% z9^qbKJ-^4Z`q7K`;!$0JbGR2n@1EdYvDN3+o!IEr4xQmVI-R~9O+QcX-|)Wmd*3RL z{B73qUpBb?TP%p`01xa{BJg(Ha-w~=}O?t@k9f7sqWbnLYi>)EUB zeH)=qt8b*K20Ogv|8`rk zm2cK9>j6FInRUzZv#(^GVb(3{0X=7$b<1K+D&{T`ty@-CJ@+u{Ru2749BFUtNk7!* zp4b_<|FhM?&hYEGx33>`on2pIzCFX#Rbsx)G_pH zhk|3A^*O>k9!0-y7!`5tI@c!Z$BvGy&){4HbF^(HGQr>hcK!FiZ6z$<#*3=E)?6gJ z&y2qKwX?q)WNxMxTHN^@fxZjuGwDP1S=_;PmgVH}EIu^XJ|{Wg1js73bIK1)I^mJ@QghX7up_=6z_J6tBb7dc5~73S0xHiA#LPQzZd`TyR% zZn|GqKH1uDm#n07aPrA+PUlXE{@CDVSxpV)K=zm#%7BjmX@Bw|tEs67KP_lWd&Bo# z>F)oU|Lg~UpZ}f@$5P3MTcB(6j{V@-yi4r|CpJfh+XEv_y2IS;9^eS)s9HEjVaCymCy-UnK0a{+swxm&+W_QFRyz;gW!?*&f6zmobK)u z6K`Oc^6>8Ye|DcZ(dFcQV%;wn^!ACjq*+5%Ub6g(Fngvm$FL_h7#Uf2X(V;l4p|#s z)R=sPJDQM}lMk^T0`o~*Sr*y13EEW*h=s0gd1;XS<_fP73@JMpJW)_IU+nt;+Nb~YYk@~xhdH{ z5GHOwI9cuwt1V8MFN{2Z&1p1tz#%xA|xaON}P zr=GcXeASuu_^+Lr7~hpRQ@3Md`9HnGPIA@OdtJUh5!^ls-<2-%A70vJqzR9tr)=`l z#*n6d9Ug2sMT{@+-MilhTF!ZI-nBzz4uJ-Bj&U<%F>J@(J5=UCla|-|0BL4i26EpQ z<8rh=ahqP5wL^7$-Q?-h8eoi%@{RU5PQ*;ewXreFAM!kXe>8A$q7QA+ImJRj^5z4d zGJ71}ca*uCar+M6>6^v=oYok;5PX2{E=~mho<3}&;e#jSHRY@D_Vzb}yV~2!W3Q;4 z{h(&rJUzuvpeyEn$j-9%WZ%g=k=KDwg2tD|10lxr>mha_62Jvn|;Bh!Bh4HEo2|f+Xv)V+Ygs< z_Y!9=lI7q%GVlBRn#&^2n`Uf%`^qFA@NO&emiGQuGavf@JM-SZ?`99bHJ-mQZNN@1 zNEsV}PilM_2mG@0eio>a-oiT4Ki;;U7b5A)yAN)-)7tMdB2>s?eNW<)BZE(2EOC2KP~?0Z7qGR zIJW$4m$2_#;hWyJcO3U<%4c@dsN>Ut!9f?r<+q9qw=G4<}9UzHK09k#)^I^|X(8Pi@S-+?bW!x=y( z`6z32qATB~^DXkxV<*#wJ9Iibmp*J(bjH8iD zu9|d0){x9{1YZsF4&S2*yc6yU7ao^xF8NS0!fV_|ExzE33`cGmX7b~k6!Xf`oSU>$ z{F4s6t2xMSodG}7*sxcU-MqVN;yq|9f46)s^X)eoYt<8~^lT=TU7@vn`#(vS-(C&+ zE&A8e;FNJqV_aXRr9sb^sexDmre0*Il2pHFOEP51muabdnd*pqaWe&{!WWf9QiLXYX*%g1dYlq^Y zYb_UTTtwQkmuHcd)!LVOn~_~g8E-$oZe5`oZ-eHzKmG`bb^3T^HTiuJ$ zblv>iNz((?Q2Ap&@z&it9>Pyo#E2^Mcx|Fjk>jW-Rym5 z_ygmkxB}zZgD>f&b*Rv8yG8kOZP^l?af)G6&>Fy($@mJETKf`*+9w}jeJ+tKaEiZJ z75J-|6mL<6>>ygN*3|m9yS~$kAy;7EH6rCl)W^QjYug@Erz@jf`KwgtN0|)u4V7L>)JcYgUohp0{*Rxk?spnUeh*D9;x&4ZY8g1wOX=1vPd;&uUvXP z*+&VQb@n6AEx=|Zo0ZNiYJA&CJI44Fb_K^+leTmLB16;x53yFyLC)R?E@-d&W9nKJ zxT?*p*F68S^B6FuTC>w?v~Rg|V)YSZ0M?Myw|^%8OMLq-GO5<^S$Dd9ttZ{smU4+P z$lA8-s@zGs_oz4Ps_Z?Q>yS5gOUWxEzM6S}uM9u4@dnAG+^C7blEAJ9Lpa z<$X)1_p9`dIt<>BSFr$TbH|?Ot<)*Jne^B8p@S(qg*(x|qEoXr3}@cuFVLCRSbHZ< zW*=7m4$?6c57|BEm5b9=6SWw(f){-ILK~y8aZd1iCucH_x=}p|Hz0W zGDotkCoyQ2PL%(~0r_!ozYDr8yg1k1>e-pTL7Pj0?)mgx;L0%OVON>_Es)#1*rk0* z(|9RO{wR`1ksQ{ zXS^KO3GbAD2lecyzlVyDi<#p$`7inXzXa*`Rzb$*4WQ`mcjkYt{VeeDKei8%{u=~$ z{sZ0f7wDdgJl!*j?pccNsTjxN&*n~wWs}3ZS$jTWZTWcRgz&n*As2u*7tohDdsijE z6#bNX?OZ~-;(Zl9i?1v^;&swjVK3JjEc;9FsZ|rDJCtzuZ~%Ib@XnHafi88kcLsEg z+RPcQnqF38JM!}$Gw#yI;F})L_$}W`&i|?MX5lY*hSfM7-$nUI`6)kukJPW2C-DP~ zLlgXW5qanIuo|C)H$|)QpMwYg792V1xMe-ad-0VTa8+e5@UMprO2fSot6F%DCMt1OI&Z5BenMEWa;Yy`K5g|1A7(z{N?VoyT~Z zG}@KI{mV#y65gluTHaUjyGChiPNWIf^?jmjW;o^Nlzh5-;?OLQZt=ld6?kmKi-DXr z(cBtvS>?S*8wIuIr229N&-j%rEq!dH!Oy};%z04j;fe9z&AjfRjo)EztC8O~c4-?{ z9K^PS{L;XhB0nXIwW9^!BH;pdk_2TwU)tS@Z@$_0*Z_~WrsNFSL_71s82vFm$es9e z8EMBZ@YZd{tQX^|IJ%iY@$3uXtDWga`9Jsa>-}KweX35CuQf?H@ORpD(%KA;{fKYj z^FtcIo$1I4ir*0Q+L^+6)$y)Z2Yzp6?pi6UIg`E%(#}r=Y3E14tKVoJse6*&aT<*>I6faTEUw;uEE6$l>{44JAA?8cxskA>zoO823+IhS-l)7=3g%^s6T+eSzapci4*hiYO zB25K;?vnQx+QH*fb~JEi@-bxex%?*a8;-7W41d*E^Uwpc@LNV!AGe^Oe!>0Kf2*eN zF0SR#&ueMlHMH|;+Ito4z7qL-a)N%CIaR&}es!GpRXJ$|m2>y9{~HYCwg#}9jcvbp zN4WGl6Hoq}`?Kp;;wLZ$dw=WT^hC=r?($_Vy4Sn+doXmSJ&!W%WyhvrJDy)q-^aYm zvCrY%v4IwQ9DQ2L*jLk9c5GUtseyGyz7aL-cSslMgPp%rd66HLceCeH>dGp_nTN-G z54)%3qz}_~J2NlFcA8_)AWh{(+R}$DPcQC${3hm+`Zlx=Pqb%PvrCche_88;r@41< z%I>xs8_FK|6fyek-ye3*I%zysvBx$&Yz>KonUA9jb|mSiY*kOK`QrC0?{V?7o%&X> z7hD~{w#+zI=!`hN&4$E= z`zKjVmCR#m-FxbCH_u1p5uIpXNNp`e9z^$eOSZc!?{M>cKpuBZ%CRr^(%vOa`Clxx z8h^^KnxE>?xV%7{Wp~uryw2E*o{Y_qG5UAb#}1nz|6exm{rjTmfhw`| ztz;@3WgaWZH@+L#N_0k|&l0PYc9&Vs?Ua=&_ZgmruNt?gH*Vh}t@=>=x)O^sUSpr<|Fz^-{6^UV^}MF`+BS{1_A~AkU40fynEM$Fo)>}Vh3qS2 z+j-DRHF143w&VCt-;V+Ys8jfU6ujNTShq|<4%8e$t3h90Mv~YVN|!&d-f1pdV%Bg6 z8=UTqch`UJxuCp{ph?-#KB2y8qF3y+eq^}lckzvvoPh1=J+CiQs6%tT96jwA{8Do8 zk+OoYWP=KL4)Ko?r|K7V7mrznXNbQTQU- zF9BCv-QvW(2;E2%-iQv&eelfZ+5A5U&TFh{fv#?$7^TFd;@w}(yBzzz&hN@y-Qp9{ zIN#*Tv?u8n)X7*cr@o2>?p+9r|8B*@_?{=ryCn`!^<*;X7UkS8G0Aci*KfD64dYAB z+H_*Pxs#DTs?Bd#yZE4a`2{#6+|aY~nz7%~qhsvnF!sWI&HeBAPPq14LHfBFsQqE~ zF&ym^Y%uBh_G<=a*{>LwYc~n<{eEt!^JcdVK6r8;2Fz69-M%%NoNrzlhQdG-0nUfN31#5*4YYJGncxWS*> ziY|icir+UlCV8MYb0RrH-%2hFTG^bp%x_JWuPj{bw_43PU7qoQX)Re;hriH?v(_np zDrBwgr}KeRxvFa?{aUHBMDR$%E6_K_kVfpQem}u(_>Mmue`EZRC1LmiHiX7E&d6C3 zuGE7Tz`o7&Gs@26drt>a50q_)zdj70!+y!KzLqnV|I4G6qZr@PJEg0g>iq9n6l(+aSO6M;mvFX@1wuacMuFGPE8>%sS!j z*Oi#D(KGi3A>WJN9%Y$O|~`lTWNoqqWR6W z-!(ADe%ruoo6EUK&$720C>d}J^DvtE7{$C?j0{+nV4lppDz?d2#WFdBJbQ$BQ{3L8 z$g|?lo#*Wpug)d|bL^LZoNt{t7`;53-@%^jNAg?GZw0@5_`TE(+YV6qrmVT?Z<~mj zzvb^!cGP7OYYWI(-gOD|qc<$8vEu%B>f70a7ac3V?#69F79b9=kqv&#+zR(sQm^)H zH7<+2v=v@kj9{{-rH*Qf5`7S`p8`*Z7C)(|reUTNl8clU62_Ms^`?tRMGCdSb2kT#rh+&thcV<_AlPrCXc zna^FDC&a5cg^*?&*AdAV_<*bwfxqvws(y7Y8f!a<%kebl;? z$(g7!aJeiM}3{RE-C3ImD%VPKYhwt>Ve z`I7h_sdyjln*;Ef4Dh)dc-mYdHawzTX?wChT6 z@O+;0_(UaBwj^c%g`p2;ro zNIY8hjerjuhiu$!XLQ8Pu_;qLd9Q(4_BNo}A{j_=Q{;eTftoXNKnwDJ2sW1s8bg&%%NT(C;-nUkXXCd`{)jBNsDis4*IFYj7)Srnx0{;r(<4>@G-p%s@zkhZqXT`(x-<3gf?Li(b zL`f4Zutqf!pRAwe;9Swc*#)PkgXujx;*lqBbmyYN%t4lY22k}Xj%Sd#wg=0)hJz9K zCc2^8s5rfEhlJxf_F3e0aXg2+X59HHCanX~^Y#@6|1}dKIm#^`4m$_r~ z4ta&c9#8zIaQJC(N&EHxf8+3C;c%@F9`A!D{m^Cr8s)6Xye`<;d~x{?r({U+d5qBeFJ=5^mUPwJoCbkXfx*i|B7TbV@&jh;B($2Am;0;Zo zku?*4Px>XkLGfTXT<`x5wyD9wUw!Y&qVB!lIYy`c1!Xap(y6yaKj`h{WBmqqnvEY- zo_z)F(0Z(Pme8KAv{mJe^U?? z-A0o34-ATjztgYYk1c2_|EGSC7BBGy;-#Cq z#)ol-P>uYU@aflF2fq`pH+hD1iCd+CxNm3IxNHgo$QJ=e)V68xh+Wjt`J2E^5oA+$ z?@c^Sc9y-|zr9L(y`9ST^|;i?n%L0sA$RlPTVeEO*)q}R-s&u;R0}6R55y~ph1Zuc zt+~P7GtqjvfoIkEnt@s`PuBfXXQbL=(j~Z~d0Ka9wg)tO1~i)q%@P|ikY;S-@>ANK z=S#SEfSj0K+T+}6A>cs~x(Kt|+^c|}P zS%O%Ci-{$efjxkjQ}!j`nBqjOA?A-_R*i)YvEeU47TF#QeXt@#pGR`;sfB%y5HY2= zU)NTDC08nc!Jwb57)czV5xS#>eie7=IeRx{RML*pOBK_m$bM;rOZ#JJ3-&w4*|MWN z&tU9ao>pXc;lIj--@PRqdC&8yxR7-DsM7D}{q(y^xD4M`xhb7ypjiXu=R!)umCjBZ8FIF>tTEFFYXZ)ra?<)FLM4sIC)IHZK*S&vCGJDeB$Ba#gv622} z;wuIgjd1Stb1y(3A$fRrt@v$`J%|1YrqPFb>Qo<6F&CwicGl4sc)F?b*L&aXzux_R zpMiz;XFylh$gw{)X~_QuX4y#twSHd?txSMs#zQ-oK|`0aPcsfZ-HmT3T=*RRmO8&< z+JsKiU|Hvx=Q+qc>=9*Zj*CvjZL5QqbiBu=W}b=V%6E;NU3p${uIp$+4gcM>U{izk z`pi4xye{NjlCjpi3ihS=ep`^(wOd$ws)f_UuH8gAid|d8c(L}ZJ1LzwuQ{YgD8Ga< zj?Z(;Z;y4^{+Ges@e%8R-{TqhJ!aD9p7gyJW6+zi$bxThpJ|{={LA9oJJ=>tHl>tp zLb9#+ym+(etEczvDTGJjHzav{IXwOzev`caD+5_GcfsS|ho>m6Y8}tYH>qpeunRc< za~JbpH+Qu0VVb7-GPZ|2dlvoD+EGKAYmdyeZ}rmVDId0mgYX|>IyQ>N>VWROW_k8> z@`=~%M&F!1|CRbIqetf2H=A_guVM%6%pWZ6TU2Fr$R7EWuQnb4_oY`~rTUOuu}+jaPndOBvn8*eiwkX^J3Ha1tDJ^Cg1I|TgQ2!4tVm$UBO!|!_U|449l7q;dreJVS?$&(|#N4er%!og>G7LKkpFw2e`DBgDo<1vLD zW}HTWn-`;lyEe)ryMti|pGD4~#_yfs;?x?x6|O}*T)V)-wV#tNTodj4?;h>^65QC8 zrFG{7zQqln&NDKf@JhO~-luFGF8|N9(>vo8?HA5TS5V%Rt>f#$GvO6@d&}2CuTgE9o&R|7QA;()&cm?il9VH<11;IQ5nHBE(9z9{{Ix zzU&D0TAi`f{jlPRhNpzRv&*%GZJl{!eF3^dmWx9#X0;7lo@F)8;dd2jL;11MJfxiZ zg-`DpT`A8#Pk0c_H@ean_|>tT_-bfb^5m!2`W}0ber)I(oE<~XScQ!JIcb_h3%@7H z*BkH|*#$pDrY2^;mEBfGY$-2&DsqnQ8^|E76}e1z8r;U2&+mn-UbBene|zYi!k)j% zE&4TZz>B#>+kxMBIk)KYp(De8;rZFua*M_feV}4yFRRxNd!4w`AVOO>8MtIzAkNN_}Sc|vAU0d@{yv0)n)r@s%>JOyV_+2@V z{%%XzxbW#o*|@fKwsBp^_tNpcFz?xU=AMfOcz0?WucQuFRxGp=URxdkif=X4Ue~sm zu?d5pVQi<0NwFF~_V!rGSH|UMTxDC^cIy2+*w(iFZ(={2F~GK_eior)7NKJyo%gV~gL)0{j6ECF-1b}my|KVPA3E1sF_v=7nvVW3*h{NY*@HP> zQf@ix*^`ui_jJL5=soN=SKQxFuY0!h&04|zKzt`XI)*MKABnK{e>1cc|ICLY9h;7~cT5pByOS>MccaYoRahTiu1&9J+rx&(7mcp;hpJhLnvUm3IqySCe;B7wivNb`SDZ!@D$> z#TQ$%H^bK~=J7$=XY3;Zd}XLFrMH?f8`3Af6TMY-h0bHI`$xZ!_q&Dh)t+oHjWRpu z7Y}=iv}Mp3u~p(PaK=gaE1vf$?OEB&n(YT?MDLeV_CC@^vS#%4(#|BU`i!*kZy?*P zth8p|4DV20*$gV~ng8fw&gSi8J(euC_ zFBTOI95gcgpFIDtsi-csF^fIoCHTcwaH}CMpi>A5wIhuKq{6L?@gPiFZ?-vIDtElKx z{6Fpu1jlJjKEiyo`wM681-B2LyP{CM;1}~lMdAks`df*yKG#+j9?-A8Y>@7q?QV-N zErg~Y=t67~^cL9_q!X;fzFBM`5A?on?gP9(^kyU>-|9B@T*m}*#-Vr5ymxS&;T<2% z4;D#INLvuB-{Rpni|Mx18XKT4LwA|3?odbBYU@FNPB+!@By~*1r*RF)hFZVEb=@MI=QVG_&256Dt={n&yK3ZkpOY}$p3${?Vo)~+dJ3UMn82|P!0Nl z`0bVOPV`ATiS86+4rG5kX?%K(g6CsrhF5+B?;+OTW8xKB2S)L&t9!`~tuwT__#hYB z^7+x3lLPQ4`IH@Iu0NrS_9L9nk)5+Pk~m#m{-Mk<{M?mw+AbCz(e&Vk*s@pF+Em`N2kZp7=j8YUX|lJ_7#* z-;wMfo7v$H{fP$7G5d7yYHjU^o$OC1kKiNA-gNTAuI@YCuUGpJ@9IV*b?+Yb&*atU z*lT28L%;7Oz2C}y@=p2FqL)#&Xud!7b=Jjp!wb&lT@LBWXMA&(e3j>C+r^S~I@h9h04Hcq2P(cm=lYed9OdTVPw>IR72IIFEXz z#~$Tb^XwQ%+%^NH$DaPqk`AARe}GrA1ITAwc%^f^DZJX{)z@rbj{QGC`Bm&^&;Myi27I{*Qid%HAGW{%7}EMguW91+xgb@EgBBt4<~lXliF9+6y=kx=~Mx!_%7!f0nR zegEI!Qh|p{S9`ehGVPQ+6Q%6mfTtZk2WI|j-g=HS?cZqKH#T7CRrCFXk<(JXS-H06 z`!UkwPawYn`4k+5ewqU(-znP7%w4|y6YBN$VV9=tww>o&dQ?hh`iA$-56sx&X9aZq zY;x>{CN0}uU|^Pgzk&aRZ%VD~J!#l6x?s=X4y&uvv2Wl5HxHXn%pX6D>v_Q?SUQdxUU*IzlC=t=3Ta3!Mh{OYqdA8&KKr2 z$9~+8F?0pZ;;LBEh z%=3SgJ}4Sj{J>P~E9B&D!4A7jj`TtN$K89thUQ>T;Jk4W<8~)8DX!! zz@I;T$!gmBv_<@U%XtMEqGjM2iM3ZUKj1}c|M7__$lq<9bjc99g+S{yQS|HK77Aje|_m2D;5U|wk|-o9K$*L8SKYNr$1}HsN-+21$woba)Qmf|7_)gCT*Nwc&C%e$OJU7%{DSdWZXM0QK z6!dLuM&L6xm($ZnOIZw>h(a5cobCU;xuZa1(Zcv$5ZY3>7oFf5?&elZrUvQn%-uO& z|I6CiC$|l@oc0CMhdJAOwPgO(Ip!jJt7qTsPy4iX5TC)U+XeVn%wcb-{9mw9!V`}S zWUhAv;`^t%{HxgOo5rnV40C%JTgYf@C^1YN$upXhW$*;a!EQXm0(&R(f0Vh~O@H@r zw&GRA=*AzUt&H<11Nok_DQ)yix^U?J{Gz1rqu4@*4vkAn_F!J;du0Yc-!ReQtew+e z>6FdfrON#!`NuG=%T_BN`1cC+Tu8fBp2|{vEsV2;pV?9DUPmZT zci;v+z5YV-7jySdh4)|Ye?Xe>H8M^Kjh2Ux|X_b4P`7*T^40W(96-u&0V*tvYMe4 zwJ8HTL%%sDrXex=3tMZ^+l3Fip`jE$Xdd$MEf7xpH|Z&y2!j)Q!G|OL)516}Y~Kq$ z2uE%SWaMN_4n#!urW{MHzb|8n0B@%D?TSMsaI`2wELV-F#TEg^}lmW0o=ux@GHzsb;Q z*47r@89Csgm4!1nXY(OGk}W*5kJ?(nnVW{*)_E$c+$&e-LQ}Q`jX@6m5kD{Hy}MtQ zZTI21owI0dYc5M9XZpg;9sCYGf;Q;Ut5iOf)?W>n(A`?Bgg*WXq3_rEw{>RRg$@6WQD_T*WdwR7uAhv!_2ua(R5(XG9C7a!fi z7;tX_^=N;Y{8y`OeQcep`-rjrn6XyesfYgDzFgm#GH8eJW(;%EN*=`nN@Cl&p7}|F zr-=)>H&N?A|DK7xFI=41yTo#E_SqIe|JyswibwW6GNN@Pai^Z^oe(ehbZu6m^||cC zr_XVIm$LRw^)TIi zFMq(@Y3i$d6!%k}pvt?{``?*<#xRBr`#DF>I36{=j*KN`?P6?p(;sKQTbAl+1Gg`t zj3n=}TITJzXMSP*srY1Q>W3w9YbW=)`D6#rwa4A(*|AQ+J(D*{ULQ8|Qa5jeyc@BZ zi&i#3Kce3V>$`C29{MjFJ+;mv4~M_Ud)d7tOU&f?6Q0}4h`9!Ssm;m$R(L1)t2RlV z_;}g4#7E%%55V8v^hidqmfNLRJz@&c+{p0XCeYfg1dWv+Vzzh}_7$ z{2n;{xEr6NnzmmHDc=1*l&%(fH|oI~3E?Zv~em-?S2p zC-MF+PH0@z7TFFt`@@|Hw5QwX4!fgxg^zI#(579TzSB1<7rb-+%6HYWSAsu^gC0OO zYdcN*(rBOX_;hVFW7NgE!o(f*>T%n~nO*9UeN=7SL0udF);{DDcuHq}l4@h;?^9#9 zhdE1?!CuT`8$YLy#NX@uz5j_Ab53ka{At>rzAVuKPvTCVraco}o+O=Hc2@DF!Nm1> z9s6hF72=m^+jW=qm-%Zdc3dikLFY4H-vW2!e_xz#-LVk8?oMR%$^6b|4>Ffu9``mM zzCJxHn^TGWBFkLA$QJIEm!0tN^*zG!mDD+wBtDYu?V&{b)fRlsdhPq@y}6%(cZ}KD z23K?KGm#q=S0MP!^KQHtrgX1sZ4=a7Lkncr%e#$#r z^S4{}N9+@suDvmnwDakQ(p>vfj=-B6rHI7{Z&Z1o^X)yXiDPtrDfwMJ&6c?*0|;_>HM2+kV}^ z9DAdI_zVH_8Ivsh>W|<@{Vu;I_MV?WcRHGf?Erp$A1@Z7sONz26neVc)R|+CH!#<}6sUIU zPR)bJy3>oS$&-q##-aQQi8h^rQg2W9 zZua4?`+EV$8PIvu9m&#vswR9u=MIWL5RrrOPGV0RfAM z*&rnoe*srI?RVb@u2ggGBZ>c7BIA#R;kUDkTAAZK@b6)NXq@!TD+fJM_>1|4MY8kE z0FUIqpW2huSZqQcX#rRIYERPOQ?8u`@0{y^hsa%R(0K*tN4Q&jGJAzHe|BzN;e!hb z%^u_)_`-C`OfW8C&T^>i;Me-q2L`2Z4_-sr({RD| z@->5xsXE_4NBVzTXY(m_PVw;cMxfg4?F)9`d#cTQ9;!@yv?P+aguOZ8_Z8$HBOLg3 zFg{lOJTn;AxG7%O+>tKteTBK)xUaoY^LAuX_wd1KJ;H0n&)Gi?mcYBMw0+swsMXhT zp8x2t_(rxy-HEpz{?~B^);qtohP0uEuApCNcQ<_CZ2teq@Ji7%G7@7ee(my4(mYzd zg|u4o)kV)Y_U~Zmk1L2dMNBa9>jy_1z+XlCS)I?b9yWARXg@%^=6(g=)*fZG>|E$b zT5|;N;2Xi)f#*~A37$}7&ou9H?HPi!Z<-+e`Ho2^?xlg^4VOae^&kv!~kmUYOI?pOz8-e1h+>RGsX_ z?n?_r=C1bI-TC`d+w#ztv_*FtjNvSjDbsiQw)`W$dDvU~QgQfq!~bNDPpy5gvYw?; ze~Nedc$S>nPG7{=j~mFnAV8f}&|J5YM{>t8(nKfP%URhCI~0AD?<4g7Rw}Iw+dJou z8{xZ+JNT{Vr#;L@_-*}y#`@{RGYE1H{s?2PvOl2g;$Ts$%2FA~-`ULjTS}9C=nykP}-5(%(C~hm#AC#psXL`r>1OeFu21 zL%-5`v4J$5sXvsCO_j3+s#mlxsC-JljQ@Q~Y9{Kr0eVc*4$<^go*lP|y50TcY9@O`(F&qwQn&n$G%w* zo=|Q6XW8F2P%5x=KqJaa}STII`{vc43J5} zJ@<w1TJyoHQV5kP;g+W>@@e~v-nN6+q5Y$n7 zK(Xi0V{H|zz1bQr$|Qpd5<(_i=J)xoy(dJxwa@eWW1h8Vt+m%)m-l_wyWZPM3Q7-k z%+*6B_yyUVCv?q|&37{F%T1bTmzgxno@i3+DM-0rW#*Ov=qCBy#gD!EbNv}vz?BK! ziI~T+75%VbB%*Xkx)GaGra#qF8zQlsA4mJ1=FWD_SxbnW% zR_;&;8Tba@4Sb*H3I6Cw2fmBcH*6!#zT)k1xL^eP3i#ZQ`Co@^1Tj`-iO#0+U+`>k zRv|Spr#gtU)dBo;Y)JK=nCtM58!Mf5Q)yTAwTEN?GmigKL6<*qoc3g2$>*30(-9e&hxyZ?{6g?udy*-vhUDZd)@SxrcSo~3zKHqwn;PXMw4dPKR0Q* z{gg=yfb(g!!)HcSlJtVt(6g`HL2P(5|zV;JR3^$X^=Mm84NHJ5Y@?G>7eEVs2P zITVFgZzgV2OG&EZ|2M~q&4FIb!w%vW<_w#fXgR)oDfsfWV#n^smv1ieQOT<3; z?&6Qa=*rXFZ>N})d&Ee`nEf1kG@iAe`$)Ba3O>U4pza5zvFqBU|6>dADJ!s=;2)cI z^Lvipi|FE|LnIbMljPJN^1eAg;U}>dTBP@O(}wo%ME-o%Ub^_u(21c5@X`~}5wzl$ z7(JL#VWI_XLYCFWzeGeucZd98<2Z0>xs`&lAT2Ch4p;EEQTjnDT!A=ESf4!Zy!!+HB3ohmZP^P27u7 zUnY6B;dLZ$=)qccZ?p5;$b^jySl^WmwqjSByUJ%Kf6Y1PD86{4inu^`ri4tpx2J}X z$Awy}dWT}fyfe6Xr}K>@7vI0b+Uks?y)fygf!J-(lf7QWoW}Rxjpgz!Lzi6{2jdmK zU)Gfez9SRAjJyftJ@3Nk@4!#-OR4lp@F`w*BHt3sO)zP?eF^Eglso*CbKbg%x?iUq zU)+m*V)<@A|BJ9cYboi4P3isP%^q}>LnGzix35h6U!J`lxKuinHr=z;=kinC&sIC! zck)et^6XsFi%nm%>@1U)X=j*}IBzCRxBHt^wB+;s9}_;0|B|hbH~C9#eMEQKp*zQ* zI|;wkn0vO6jTQ2fl|H|42A;V#v%}C&*VgCa-pClR+mU}yemZ;#IQ|!YTluZ!w+VUL zw1L)`Ezp>1?qG|6*M9=n;m1bmUawn7k5*f;5VWn_lQIOk=s@nG9HsWQ!gp=q%=~lj>IgpLXIycXJ z_b%WZ9pzd#Z^n9>dG=~I|7qplPCm3iJoUqF{yODX;cq*ipKN^YVs4e*iEWS4 z+u^H}R>23!_UC5wlCuM`vD_`E{w=2+#R2mo`%&L&-2b(*``Iuhl(}T_?!_h#pIvi4 z%Cqls|Ig>Y+Vd?*s)&?9KbLH2P&_cj8_<}(>(-Nxu~~a)l$W(fuCv*?^_(u*SqWp% zIvrU|8U4J$j1^l5rHtd-N*UK2rSpmTNh%);y`!|Un_q@$`z#QHKRudnYVB)?*$^9H z&a0afc0I#IFVPJh9SR(z_XwW6&UwerJLn3C!DbzfBu@-m+==SMbAqmxp>I2muX_cy z5a^;i;K@42z_W<%GVg1k7xG!r`mW|#`W56KMmJrqlsRl+UInwSu!fXHQ%Do~X+O_7 z?)+*^N|K08M7+qBVa_|Hf(^d2B=MaYf!HE_hyM5!kAv^+gub?*lUA$->A&?2okzqQ zwU8ncm?^uP?(J5}W`TBZPP99QGcV6R>bB5!3$oM}+7790`ritypdXGFyX`M<+ZPYs zz1;}0AaXp`%zIiF?~buvf8;*C8rHwN?SbZt?-*I&k15+Knf%p8CcnA6-ND`w=;Mg> zu%5Z@NV3MrCSWZ77j+5$qQpC&(yQj$NAY{LSm)2w`;<;K*N#$eJ98*p6kjOaTlaU2 zO~d_+9lWd4yu5^M)+WmjZimf%PT5&Cw4-HUwfi)YSLVvZhQjuL0w9;2W!v<(dy`@x3Ep(7<8zu=s9 zV;$&8#WNe-1aQY4Q=!?wu51EE(oVRCXiv5dSMK6Iy={T;{6gfB{32!8G@wJ0 zO~a9)142>iMISXb4J(aJ!$!xZ;UM2WxFtRGz>>5Eoy%Hf)37Bo)cU*?+jl3n40mDE zu$Z(On+9wc4r0R)t;eR}5o{VB#in76YtsO|+4C)j-t?K~@cJ5$hi7>7nOUM0&>nCZ zn+EK{-8B*13EuY6U&+w+fQ$R-s}rXw-97+Jev{bV8ryGJW3v|6a<>8h(7I7)&U-80 z6#t%pt$JXqO7Wn+jvo6g_AOvN( z`2;+4e|IVG8ag>45N-pX;afk`H{Gw7=vVi)rK6Ybyp6SJ+hD=B!Yc#s1|Dul?+%@E zc;j^YYs^>2Z21Z0*!}U5UQe8Cjdl`U@ZUO`|}2#o_Ep8E`2_2$@U1pmru2GW*cv~V-W4p8W(0- z3w2(b1uv)dVjl@N?v0yq?PWjwJbG{@>?Fpf{+KlaE;G=P@25`@`V@JzTc3a>*}``D zVll2p`m~-im(SG;bsyK~wb31}y0`O_Ze!bz-GONDW#B?PYxg#`JLvzQbI=F+C;!vb zDDkhbYm&W*!6_GpjzPmEyXXP!*+r~=d}T)_VA|9JHnb-t>im{E7O>oiFKCyI?c|zE z3=R#jt`dwq!dyxY5MjKT#2MFK^8k6oJCA8US&?MyYj>r3wto@@$D^A(#(qk6wKpYY zVo#Y_-@^XVGS{`Q6%Cj{KgQ4x&F5YC7!DP@XWD1EIMzWO@e(>`X)Y4~Rj;R;r}EgJ zE(>W+e~KMX&3EE!))8Z*&Dikd*ngYCzD1jVA^ib1JgWB)b31|mii>*U_7vo0pC21F zArsNrMDkzV8F+%TiQ+@(Tv-XtIej+C)7fNm!Ve)mg*};lNxpW4$SCUI`4{!Jn!d$v zJimN?*hMxqan5K0b{$2L-ds=OZN964Ih?}=wdB}Ej9(cgbYtXy3-q82O>od2dO4;usqU9yP zx6*uYVhJ{kd~-SP=`O*2@KA>TcmL1lzf*59_44dRZoctL=X*bJ|0iLaXXdUuZ(lPg z=y7fD&!az@w>I$4#BQe#3rxS^^OVxZ?HzU*^oCx<$T_QbDW z%74XAIFp~p4wdnwbN>BojMEdhPw9$tsp5~zIs3BcY|w2o*}tmy8Rw} zyUuljz3te6h`$%iC?>7O`2*nhVtD!O=;V)Pvqr=V>UFL2&V{_A`uaAuraF^_%eezl z^W4bZD>~=U0q6s3cZBuQH$?;9YNt){`K-Zzx^+*}1L`xsiMszo-FfV}P3RjwMGp#V zkc}7E5AnX}fWd#BxeLeG+^>L-Ndfm;cX)!W8$BWZ!+$6F4{V$i>PXnsi+0gp=N`6f z_G0X{Sj%GQkJ^+SV`7~&AHsY6Cl1}#2X4YP54=7CUJEXS)BBlg#Yf&tdKdnviFZ|Y zod1eD@$tA6d@oY*!AQd|sW*N}eZcp=$jSR5FXtYF83WMw3?#NtkKmX2Bpr2Slo)ae_zK3>3L{G963WHa)n0Zo|S#Ls}bI}WsVho?&Hh-4{08Nxe53~ z9Kz?Q9o@_uy{rL^*m(8ZnwuZ@SPOrU-H@jSq*YUQk`($V&5C_kXoU|JivHx-w*b?^hsAtPwhzKR;kEL9pfMTRm0{0w^A?(VJwkI_ zI^}ce)K1YS;3Xd#sdlfYzTz!;U3|XQ{a-8k#2lBF?95xt`AdA#Rc@J~QS6D|aQ}l( znxiiNAY=q9t>s_ql0SIeIcf&iCYio+M~YH#=~AWaYlDjTik=Z^=*?zuk@-pe{nx^*zLH3J_d zCw}!B?wH5d5&9~!+?PJQls8FMw~<)4pZUJ3)@o}S!k zdbgOl(tcs?W1ABQb2gi~o;&@Pai`yn+$UegoqmJ3)2}CY`ptjaX={wv8Y8^h1FZc~ zXK=>&Q+JHNVT=iT zW_saqY)RUvZF-8I2V=N+&MAe#vY1CVN8rk{$V!buu7*^XB44E7-EuS?A^SLG81r zG`+x{qVsSe^Htz^EN`7FI~A|v_hv`Sz`4}wtqlo22fUuu%z5HD6Js{d{&(=H&;s7Y zcR-JBbpMy}|DB};4b80gx7|F0&&kOB-2b9^>PNGe_2W%te*HiQ5cb^Yj z$){fW=XCqSInLO=K|9iUYY%>f^%C7GC(pn-{d?KXD1M$AQiWBBFrI}V;S<_E$@=coB2*oZY(c!Ke(oo^21J%dXrf-h`PT^ZuJ z<7R_=_A!Y#TNS06;7;2 z2Nt;hga2CJw@ivnt4Xu%H%yvtziv|Q|0G=~T&n$Ye|6peA>5FSuQ^dHoZI2TMK3 zt%e6)80vd?X4=Eg+)?OBtEzkE!I{3NLfp$(zNoYqzmujfQ_&+=o&7K%8Wz?1kn!OJe5-#5?NS0s~@O#pgX=12Zf@{Jl+tT+Vh znef}X-$LU~wDqdn)-~18CffUN{>x`Y=ey&`28q!S?wlie;HTSn?%B&S-_hPYfb+1! zA7$EWfx*8lUA((S|7REGje}{d&ha^B?mr;y;5%2H18>tBAE`NAz;`5X z)3f%6Pqm3$>~woMroFw%W$-;}&tuxVl(yzYK0Ozb-)fE8yUA~l-t4YZdcvMP)9@#f z--eDk>vN&mYrpJHC&y<%@&oM=&tJ|SVCFE_-UwWZ9=t%F=)qkkKihu7&Hpd*b^a6G z;vT_KGnq$ZBD;>Aif<)u?JniN0H3rvg}Ax=uEUnRyn1rX$8!gARKeKzxz2vEjRuDSBC9KY#cJ~Y;`VC*&)OGq?mD|dd!}$;5Z@Bd4&MdM$+mrNofP^cnf94= zzeo9>=%=F#R~t@z&|AnqQx|UhMDP#kqWQ!IZQqgye-BX=Ob)Z?2TADPmb;6eFVzy!b&6{2*B$ovYV#h7Zw~7eAg9 z+Ctx6sF)b4OUki*9^H{TkF%L|co4S5m*odKZ^{cSTAml^?7{Q$oIvMg+)e)!|M&BI zjo(e=!=ujHKNOpbIaciOWez`6zZ753^uVeG!Fj8owRSo1(EZG0Xzua>!D8e(@+;K& zJV4qym$+n}+-Mm(-nqnAC%)UPl=#Hm{>*FZ&%?fJ!O6>ae|JgmhAGu44Swj1Y`&&? z3SQp^T|XyS-dRVix`ShUp(s8cG0w@cImk|yh8ujc$C`jGkvSjb+ka&pELOP3PZKh+k+AVkstL>y&H1NNh*Wi6ix`ovvQA z`#ZVIB=gC&Z!vS2Ygdu3V!f*SS_@;0sS-IU-`cf+u|xFj$?G8hrvi zG4>rn%Sx$-&xm^Xuebo)yty4u?SRLxKxYD6nzQ3%&SkFLZ5H3rIM9_ejmM6t?&d3E zkrKX#Olk)FcL2FsJ93KkyFG8^bGNr>_+<4_vRu}samTeUNDtr6+?%ucet+;-nc&ho zthRc9YbW4U-s3E!`@Y|WH~9l}dLHQ*+L!EJZTE27zJRthM=|zQbM_d9?+tm_?tbc9 zxNPLEQ%xSS_-T1;{@_9bT zQ*%i>@S(lQhmHcjosr{+cjd!ZQft(X-_~Nizm9eyoBD>PkXMQxvt<+aDsSi)I+ErINBeogy3_QCd#7m}9cUyU zeP*F23|w!v0$IUBzl8>~KJPLA+G88N-|C1=7HsF+S8;Au*`FzM&g}X2O!f@te3xlY zam(MNT=1ZMW2)Ix*qgAk-RH%}RmgIRlf3F>+-+0_|ABl^bYuzR(LC1lLH;#?wPfD5 z%pdCCQhgrxZKO9eZX6xjG-H@wYiWCPI{t5IMuX4_FVMEee+~*!Vey0_BpE+^lSUQtuG1usinb(5iglt;4kzrp?dzx>} z@wd!#mOa~~x%SnhFB7L_!EHZWm4NjL!2cfn`b0<4zyZPjnek49PY1D;{198@p1|?X ziLJ4b*e9Ebeexq>pWHw3-d{XT+>U45*e9YnRp3e?_mxgWKOnuHV4eLa+=veP$gmuL z{AO$(D)RhCs`LHjy0c_X-_TChDzcdT8HN5*)(89xN0L2Zg9G;i+pKq_9zC=0;}I*? zRs!q~rku5otU(?lTp7VT$B2)3xL-%)Pfl724a@q`wD7x0*#qN~J>l!m1s|<~Av?iM zmH9HO#uoPE3^&-C_4r_mQ0Dkc>HPi8i-XH}{^Qr2weQRP2{#106{Xhb@uk*Q_0vxq zrI|y=zVo&6(b%-_8pj;Bfny^@J1@LdzKvtpFS~pqft{7s&JjWJV#TSB{OuQ$Jga|l z;dx^Prd|zs>{FYyPicR+y2R<<2b2j0+rUT3V*jEx zi+CS+d6#D|G*huV+SxmN^z+G)*yJ&Hhry@4_ahq}{8f9%0JDcA@a>8E(V<^~Yn8Oo z2(C55uiZP`>B|nj*BwuO&zMAOe(m-#fhQNZc zS<8Cz%pL%*H|yS%noAC1mwoF!=e_Wt*^7Idz1Z*Kh3@g)gTCNwzLmzeZtb;nRWUJv z^sQp>s}FeC*OT)4jPEX5)z9ize=jkCh&wQ2IlrF79}+%F4$58>dyA{6YkhdT2 z^tHc6JfdE`mW}@(-Wxv9i9d9Rv1p&HL9fyO?h9V%-%I>Oe>0}{*z1JPf&qA|FU9Gx zz`Jld22MZ5v+%iNLw{$#`xo$8IPB2h9D|=9g3p4jud#;RY4A^HI{8IfTkt}3c`Z1Q zpus`6JcsfGY^@TnvB4VaE24kciEEz|J&we67Peyhy6CX<2+t0`M88@sEp(}15mlZW&Z%b?~g zYh;3dRJ|<2H-htgFF_05;rX5S;#)=A#1HHH#eCb$T z&+XLdP77DLbNxE?HD?JtzSNz|-;l5VYpy3Sm(ILr+N0g_UsIl#|1p${en#e7W9yW1 z=GDA6V~?+SclJB<8bc*}(p+ob%Vw^*d#Z0@uCWuQP0_gjG--j|K$@VRl}QDuhPHFQ z8I3Ml{9rrnt%ryGBm8#?IFg{vmWLQ)7ge6!oMxRYW!_W9<^?0rMxA91y}dR!IEc0% zW-RNm0Thoq=bH|U-A{f(4z!qO%^z{hn#2p2qCb&cW-0Q!4*D+MW;}49yhML0C_9dv z?Fq`T(}ZR__fzfT`}Y+(G5pk~-u*i7Hm{os@0DWjor~^h9P`6jPkJB8&gvd_bj$2< zX6$-r4|<07>wy*ey@qjr;h4Yu*)bmocvfGfG1Etl`D}O0QyHftH_W#$<+;#%njcqu zX34$OXN~#$oE_fb`|xzBTGtDyuQj{Cq`CI_q~AXg&o;aX^sowky*G65yU>-7ylJmL z0$<+zIlM{ty-T`u@g5I)`alo+z<&&guK&OItm;XQA9R0c?Evb0Ok8o%y)${4_u*v@ zxV%h%mp+O=ng^{`{g3%Q;IW!|5<~uJ;N}mW{`LKR{p))|FQ0}d`j{B%I&*ytuT;%@ z>yO80^}Bo63m=<3pr4_ae}-PR1B1fLYj(u97VzKT;I-MoUoy{y$$`U`FVN|$!R8^U zymMY^IoFvlpDCNOB2Rgz&a){+_Y%vgJXGyTiKbk(erCWkrL%?n5gy+yf?Hp1PS6_g z_}aIy{rReUD5Tpug8W`Oe%%M7J2N|x-yi4fBRn?zk?VVqZxL$%{_W`Be4(us=#*W) z%Fu^drNO22U+}Q)UTfh#@U1aBz`b#S!|;={8uNMf^gmpzXJj<&l_rh?x_wjU2H>+1 zJqG*fLJ!Y>8j@PE?Nsu@$WPYgaqr6=Hz14I)+?~El+>4GjXuG>@fFB}=AAz&*36n* z4=oWrYAds1>X#L~BDS4;?dv-Cs2=dURkqZE(`N6w>tu5&ey#gEHOTBFV^sa3B&+E@ z+AXZH4!?ptLV5SnUTg9+#jR-iXXF?xe3 zizNF`MRyR*C+3MKEtRwC5Fh#j-&xi`A36lhYth)iBJn5B_50>%+I`cv5P3fIAAw+`1THY3l(6gynPvtpS(?b4m|_;w94UahO>yB@US+~j|!D825>_O_IpBVT>b{h#h4eHE12F=>C^mh7KVE;FLH0wbBeywlQ zzn7HKKLl~y0|PuLrEgo6(%&tl+qlzaoz@>3`BW{kDDcDagCqVDG%X>U_!V`FI^$-o zUy6XspfcXe>#3OZUj&6B71W_s)zEPe`!~QlXtKl0kE)#Gcyn(B6KN zL$bRzyjr2`Es2khy+86O?zgWQnBKGQI?dTb>L0MVjqxF~?viy!ppnF-vo~T75ZW{` zvV*Ta?+s|X%*gbTDkD|$u4VG~AO{@I8;K!6TBR?{Ai}Hz0=jd?uhYoC z68ide^s;@>%T{~(RMkUY)1a?4o@tAs(AW5d*1{XW#Xt{hE3(6xz0o&BiHkQ4I$H&e ztwv7^j?J119iEB~H5?yEO;=Gbzo#`?vTo7-ggjjHTg6&ef#(OQJI|wim$s-gl(AMZ*6%Xb z+ZgLG#@dguZu9i3dWf<1VXWId-&ph^b#G$~AG4SI-qSDj8BgCKRT?9EiN?8&_HInM zY*BRZy1XNEtQy9rD_%iCIC~I_#QI zr6<*H;Xm<`jeZ8+ZL8`|>B(OBEcx*0QE2b~@r!K2k4Nc7^e9RfQ?7IadnWXE1#?F{ z%$Rv^$^SLP-8cmLZW-Te*@6$$S%FX1Lz|WNQZo0VcB6+~uzJA-K~I_^lbKAL;#IK^ z7JS8pdy9E{H zOI|m9dV=p?iOkXI=kH9OhrCjJu6M$p_|eNaWxq9LoCRj-e<|%p?m`B?bf?L`fqkN! zcOtn~llpGQ|4dU$Xm@bx=rn(Q3LGbVhWM5h}@PM%s* zgDpeL&}&ecvPPaPvrdr|YwK?0*UzyBt8eHbMo#6~um7x5ggsd6)JTe;&8$&5=?Unr zf#(wi!MBGx^kCgN)UI*??WS$JI0;l&~ z;_wxsBWJ$1;8e0hBi5f4e39>nM!c3faZv~3lrBkQN7p;@1>|!2mfBbVEcn5hBJkx% z1-=iQMGZU$&JJ$mTY>?d9jcc)wiws}j66twJL4y=d2HuWXTA2YF2(5LT7kFnvjVI9 zk2`uoWPnY7r@nZC_>NR`VaT{=T!t>J3|-hHbYTVSG!N{COHrCGWSWzu_NQr_|v&2UnV4Y*YNz1T`&@G$~4$_YsncwcP zZr0$>gTl&kxr z6}O+~ol6}!mQ3p>jJN;w|@@29-C{S{&97hccRmj?h-s)r?NRtnfRtPoE4jayPuM; z`sd((@R@P(i7!cQXsxh_g`N_vV;)kfdsg_1uT2}mTsmVhx2|x@Z_FSjwfA%8<;s!4SKR&U z_;?3T(wP_08pTYO|7j_@Y(M&U_Nv1^&$_jWdlGq+e!`zLlYX6V$j3(alDtYk6g#xY zZKIj-3^rrOZ%!%oN;GbFPidz8*O5`Fo&R*_WsoVyw+oz8{+>sshW0Yn%6H*!;2%sL z?d;WD($0;_Kg$}e?+VA#c;B3lzCA5mimzKFe_~iNr0wkek@Sh-3{U33X@$}uWDGHT zEA*pn?y`KHQ4*OX1{<2%fmcj!dbH77Azg9}XIlp_dQ|qz=pR_`ch{YV<=FRHR z*{hf{R!c4Tm*<@Otl^w(>+zFkUfam8=?kw941B=eW-IbU>@n|Q{ zj;~3~ZGtZ$=1NGu_S{1d(mWn<ek9mLVOs$B(u~9$q4Qf>`l;ktmsa3 z(FWHWM+Lvhz9E@%knc8^(Fb@Eol)-9nGAWrGU5R1|5D+{iR8m6{FmQa_q9M4lr1{- zADSO#>$v88H@RR8JgW}j=@kfL~4 zDcs4z`P{;$z{(gDyz5m*xAO1mEA=h>thAr%POV$j4*l0Yw+!3@4i4!In8o_5UCqJe z?i@tCRtVY4q2k9z2cZWUrv1Aw3ifg5Vmj>$Ml|k8JWDQK=*fusmaR5(+g@Zfweu|_ zFW=oWq`MQB`>;HwCFUxm$A7f5@|J-hCFlD+mYfiB>x_dukyr;dbgnEv=G1u0a zUi;!<#@B_<8^CAi;B9}4uYvZ=?8E%Cu7~%65AY3}t?T;)Jxe0JjNh8J5VY3 zBi!%J^HbQnHp9p4SqF_+BN~xwKjWVL`%o?%jMN*LIV`#>dj^AZ;NKv=iQaFs^nQ(p zlZAI?zP-#h^ZBOqe$s;!yEOBcq&N_Y)9Hh!iqu-+KL|$o#?{)t=%d!GjWtWmooLm^ zXBoP&*+;+O)eiX?S1a?>I==_@L*9^~AF`mA0{oS$SFynr`%<-R)uU1O7^Tn3sB4qeX|{y%pmUJJ`?CA?@o%$M5xo zIJ+YQ#Pi*Ltz75edPwb=oBgh9kso zh;OrE2W$BjwDH6Hv4^TB?-7rQ3-J;CZ5@{zYS~SlO_V*$J6k;A*5^Ip74)HHLq2+E z&fy2JTis3hpFH74#=-q3jvdK9>`0#XI(8&$oHTyS6E^41{+{4O+L!^Y6^~HIyonY{ zE-86j8~c#fsWPd^iV zeAE5@?5^);*l(TD$NzQfOs9@Bjx2j8b(+0aZ128gr%yASebR}SECj zkJkpmkHb^^c+jW6zu9ejOtWuGibnBeo?fZ>qe*clr|NdFI&U|YwGI?KflTJEv z2zGez#*O%)wVAa*{_B1xqg!7d<~tg5+oR5!E_dssQAfPUtaG8ylr5*;e*O<;jEW(p zecK1m?d!QNug=ezzJ)+o-(4DRTVtz5eO$H(GpuH+rjZrOl{PHgJAbd~6p zY(aWM!=3>yWe@V~7ubWy7DP6m(tCdF>OSW}r<(sEy=Cuz$rhxZJ@2b*K@QVL=~m}; zvjGws%ei`syruA2j2-=?-4JIM!L4r=UTqqjJM< zO~Xgkt@A7DG;{W72cA|Bfa#ACcVqRLH1+CBm?OlrHa)eke(T!JzqJwinQPHfs zIFAp84orqO7@QaCfJTWIk3geJg*?{eQVLmiyJuyP%tALpRHyn`hyt&bJF(`=x5iMT_U11-xiJ z$^xF*1`zrp=(eJ@krYi0k)_*f_P{d~87FL&#o`lh}Q zR@p}MLfy;eIPsf{68#?G^!si88~KFWZ}r#oi~cr`*>3uIW*;A+k6HuyF&@T#L2I`a zo7VNjO&Q1c9|UijH(VWR28JTQP&4qOSU|1AazaNQ<^LK_*w@CM@CVLM|81>)eXz6E zy;)1qd&4VUUmDEb=A6HJk*__Vxg?9YG!DJE+ud8^7bR$ChJ6p??Y;-x?$&9iPGS%E z9(Bb3$qsZMbbmi|wrw`NY=cAp+w-r%MkdSnSMI;=UKkSjCG*kdb#QY zQ}-$b#%q+a4ogYDC3}mqFONAZ{hsbr(Ou5+m()G4iP(>_Q8Z^W*WO~=u~TaTocFi> zVRUHg9Aj^>Qg*EHASq3E@1dx6ff z%l2ZhQ|AWRUf}yOiOkf@ z{r^W>3)X4Yb4k~ZnN8c}oc+7$-d9?Y^SX5J&x}dv-oHEsX+rm|IqPn_u^?f)!G7#R zw}(7=6ytld%XWi&^A2a{#^cF~m#|WH9INu+1&2sEF&b1ED=N?yMzrnMY=k=_` zD0i=XnfB%#i#Iv4oIHD&oBtg7*qJmnPX_xq5PpvzH<+2uDygj#SoJGx|Z{_>^d^A>rh{Rp!$QI`WwK}CfRn-*Y7F6 z#BE>p9ng|0wKL3Vr;NNNV`GtTe}{baV*_|~b@& zfmoT+34z!|rDgDaN+)xVo>KOSainGNfL(2x{U*b%qfOBRwRxfHjtRsrP&y7@Dy6h} zo>JNzqLemEmD1+dNOjh2dOvxLIq%NUKB(CG>9%*!`kBAsJgc+VxzwxK@7RmHcwkJ( z;QW`_jC}lAn-Q)5o_yD(kG@{>x|8aW%`$H4`95)Pje@e zS9?HTbb9^J?e#~;HvnB9@pcxcbH_rDpzhVuUEkURI$fK)BiP&>Wj#gD4X(O-z@_b{ zjIPhe-dH#wfcX&gknfyaTtS;NY5OX^@lC!p1O4H2;@|f)@l;gj1b3j+;j7pPZPoue z?s!$)_)_##JChur$erlrUP?n|H=23otl0pq5^ii~T$<0fl%I}&p~=s-Uv%?dCqI%N zi0K{$_{FAq{F?d^k48KM@4rAj*`36BcQ)UTQ!n9%5ACj(3|zFkEm^ou+<)p9`m&-o zLNh-@EBl`xoao*kbtB)E?4yl7H}k#qrhkFh&y<4e%T3Dt?yN^;Qf_n~vZ*QY%XcSy zD&Jn3(!l*l(NgkT;4`}9C!7nNI4ayVLOa7<_;&Q*P3s@^yfyc72hQrr*M40@ zex-ax!dIMxhMX*yFAFp#;^o{>ksNA+W`RlJh}ZhB$PQme9etl7pUsUap_YyKmLlV5 ze+j#x|Fyy==A`+NZN4el<^*i+rck$-JCjbJ%K^s^SypB+6Bs;)d@t$Cjqx?_O$r=t zytgJV^AaoF^yAm`V@KMF^^$jQ9~YDl|G9ZipZ|?M%V*v~mvI8Tk*!mkD=V7qjz{$4 z2KZPfW(K&`*EyG5>GHNI%;i+ZSW=vdf4}0+5U+;*jHLW;F6_Jly~<$iKeLV}&JQkQ z9Uo?G5^FeydWmQCY4{ochw>lT-4)p#2zh)o znxMP(6l<#!zOdbmwRI*<7<=^8nt8-P6#w0rlwo7TxhfSrpMs6Xlss$eh&=Aw13zN9 z#NguRhhOlcKNeRVmLcg;wkr8Y6ocLQu0`ZvloxGhQQKIHR@srJJ{nJhiF&)_Q=>PKNrwq?(Q(tkco`g22K0J7a(#J`ULqpW&6Vy>2G%V`HMk=&4w?Q)S zLMtVz*l4xH#$0hqzSXmgAN9y3nFi;Hb3W@hwmYYBUNI=-o1bTJexxhgSxsMsC)?pU zcCy~S;P4>#s;%_5To4@XUHnNYI5{GD@h2wlk_&>m*Gzh)Bz}QkIJj0z8`8raskO{G z^+d@4e;1vYZJk8#aiM%XMGubNoaa9dPu&Jj9ht*#BR@88;{|wuU-Ga`4bI8;Av^x%u=liOoIS?LC zqK?+#O<>>aQ~U_Z;1h&z<5b4{|CTZb-za0gg>NHGSq3p@mm7SG@W0Z#{ja_O+yB}^ zd26y{#@1oYUFqPRYeT25fClyootS_;TC{DUa|gf)_^{LW0A$z>UDrBg*cRhd%qW9{ zr573;{R4C3$RsoDvlzej9KnXqO%*d!|20pUls7Xs=}H;z0HyT5pHjxyM=89|k&!A_ z+?Oxs$K$oy=liqw4qy)+$X=Yzo}9rxpBXw?)-(7#cf&l#_tq6*SIxftJZJgSwxP+$ z{}hkuYTlW}yA`xCleVs6AOB{Ewu2vWo{T{YBH&trCn$%n6AUQkj?R)NpcA@#RrdPg zCoXI|wdQ^Jr9JdtF(-6)^a18hcUiCp8Jb_oJ=X7W4%<)MgQxfn;@3kz=4n5%4{F#O zBUUeXg`Q@gj95MG6l}j_!x2igwoVvbQuf~B_E57!(wIi^@`u@@^L*MG?a~Ev`?h^6! zPr2uuf8%=r&cfpJ#LJvl=FpXEO`k1u9y4nTtZIJnA6V6h3kU@bdRJ62F8J10Y9@wadP8}=@;6*jz6pA+KABG9XhbN{xst@x6oWS)!ebNZU%vTwaE9VCFK) z_p2QEyMxrBtt!j3Z=&q)=G|O730x5`6XYA%e;~h|v|BO5BGAm{+WW?vF z-N~5qn46H+KPesCRL6$6G}+o3#gDuaJ_UYnYs<4eLap#9%QTi@?5$(O({LvuxU#_T z2o?21upyeVB)j2a_ypZ2)5tkncPoj@}k=Yu1fE8)F4`@82E z&E-AHV?WUu=T+KD;7b=C%o}Uyh-ibx@VU5hAUL5cj8KSZB8z}hWWpm zb(qC^RIn~H**C5tUaxch>!JhDpgoM)k@@D?$C3AnN1H=CqM1eTD53@aNlyON?zf7N zsWbv3kyLAx_~o{ezWz9S^^v*#{cY6)9H0GZ%x4vG|D!3PBNdtcBHiUZ$2uvVCNh_@ z!IQAfz~&P_|A_p9;djbawhw&V(YGUB#_t&M?&*iYbJyQKmgV&2Li!?HYGF?gG;xDP%avvb$SZ;mi$p`(%_b#L%H@!wQNqb z2cz#wyCU?#P~?lmId|U8u#a5GyWHV?m{hW5v$nMLA^-2!Sz(9b?e%0$d$G33tTA+K zMhdvbJ&m|6T&fR)v z4R;Pvmb`9l6?%Oi_7dos-twiqzE*nda_&?9(AeW}|L2v?8i(nN=0$XHhoMo5m7H&T z-TUc&N!~d6DthGbGtPPQv97wzF>@^%`e;|4^y3a+`fwr-+A7*P30jJtD-`|Mi_Zcu z08NeDnG_O_a5~0fo^64{`er@-)BRJ1etpiJ?A`C5DskUgZ00%7zK8T;`gMAZv+eJj zvMl=!lV;lAGiio>n@Q8{TTQC_sHU>!Q&{_QU|=$^a0U2%IryEp_xsEB42s<>7*QN| z!H8mM$*=PS^jP@TogP1Z_|%&1_``@#jwIzeHvZ7jKXb->9>2Mt^7|pbS^P%vJ70Zd z?k1!MJ}HKdM!Y#gjsW)yPQf1%A8875VbRwi#J8H7`K|h?(ATP^so0Ja8(Fj!TIxpz z(!zOroVi~q(>@P8(i-UOXLvE-VK8}xzVxX4-#$Y_4~;jn1ljX{rtHrbo4C%RuNu?m z($_za3qIk}*Efbc=SiIb+IaqFc>mM%Rq_Gnt~1F^H(cSswU;^4ehM#bU?&~j@tyEe zap){^>5-zjzY92ZJNTw>Kz%LZY-i}M&!M{=C8_>6`&MjjZ+{1O@f!KkIA|+-r|emM z>|2pC%I6Po=p}#vse|tlJ`TfZXcc;B;Go3!| zrcaVPydZrhe8?5<{-C*X&i0x1WZ>PAe-K~ZE&mPW4jtxRILZ?=SF{$}8t5W;RwkY# zk9iZGDvkoWNNBVZli9=^aL;rFb|dYHe*fI0+4fVUn#VGDn0zZg^$2UGwthyw*IjQX zFU$U^c_-bj|0m=x&RNQlzZ?T!bniief6*S&9bf)(=+qkc;E|%CjY-4;HRynZ zfA}l8e2;^F$7c72@9E>0Z-j8GWkWK2&1sx7JjILz&%?cT6zIUN>om{aaGaTP1sSo|Ts>JbaD3)B9PL zy`BGy{|TNYao=tt&Vcr##9adFe|P-yfM4_QDZ>wc=4{e&(8P3pCs-Thy_b|-buT^! z|G^nT{~N`(czg@@AW!%UIJk{+@Nw4NJlAS0@c)wk)U(dweIF-Jd!XU(-MlsAIdm=G zUgqYlB2RPER%pc-bkjZjuH%RA$q*wSre5KB=;eUdjOrg4f9OTZLsusqwf05dM0A0F zR9U`#r(1UBnZUrqW1cSAf}ihdTpfQXgI?y@H}hX}{~eR&*z-+_9Dr17p}OMtzKUmP z9^nZNb76iC@2*D{;LJn1{WiGg(4YeQEuI}3o^Su2=Xl%`QhZO*@_2=HPbV_Xj%1_L z7_0AgAV=8)JV_TPe;Ua>-UrS;pwF`Rfc~&gWZ38*M{@s7*ql4-B%92%-=jbB!5CiC|M`XeD#fG0efKc9Uk=Z6?jIKT%454k?AEeq{1R zQ)e)i>5S+o+f-`+VcxRmem?368-Z0^u8 zcs+69*mCB?f)-nW;lUe_wG{8n4rXg^p)a4xe>*#IL1{rMU2V)CyY9~?tF7lFUhWj1o}+)m(8 zx?Ezq%rbT0DZjxx;TaeDIn#$lh`vj{bumBTOrme;1T}N0b+-H@=_ZIkMwJ; zEY_m}`V)w9F9opBxc8pjjALO+k9%kO2drBg9r%;AC;0x6^vBocr~hQFk&7bdNV%YH zZE6vB0egBJzIgZpYki(P_e}TKwNYYXAEmFt&AqOS@p9%*bX4C=;+xZ^r?) zM-MTUJkhg#+>tEV*tzb16Kk9?>nvB)m8bKa6JtL=kq6AnHf=fevJ!dFSLHoL-T?9p ze~b)DG|TAvxNEo*nOJk}kH(+YFXYsx0m)^g^F?wsG!-|E?=jd4qOp@N?I`k;57n5YP2x;VN~vQ-)1GcC}W< zovv;CE97Y};SN0K41Kf=yEN-p9`FQw+LZ!L@V}B3MBg2o;Ne${?QKu;JQoC0tp?)7 zrP#eMsF_(k5&y##{NCc1>xor)Qi2Z>dv7B4#3h&i*rQ#GwfCLQT<;@+A^Q@hp%Yljui0}9{3C!Sa>ffuS)S7u7)q1 zTVX9s;<04~I1T*v>& zWMq%PhWhv!`uNzTU3dQWqaV_}H}n1JJj-6^N~Q1|Q%Q4JN6pJ*lb2y%POAFnkZ;a& z`PMzH8!bO$4p;sRKJ4ck@31_B&u%X9Vs{|@6Flb}+0f*=Jg4s5Ax@iD{D8xMUEsET zKIw^@@s)%o8~)%;7yh;d!tY$-(0|?QE1p1oQ2bE!t;l`7C4RHX@#Y8lH5S4*~c<(BSvS@7|q`v=FYjR%VS^vhJ1W2{%+J47% z*uRP2QXB7`omW*(KUVN7T~?RZc{_P|c~yBoBCbkj^==DUGB8d&j0wSJ{7-M~>+tg@ z>Bo9x0-Y7W4(IpA4;Jm7r@fo_e<|)Al%3Ec%@60gaO#AbnN^hQo=^YRhbRW%=J`FW zV9`V?SbRf|V3Oy1|5eIeDx6zJb`np0(w}~?yU~p${U5xSxS!X&_f0EkJz)`lx<_y? z{9GRIN$%6i_Y|MXSM00L{QKRzBX>GJ!;)e9JdQ7$&+m7>%{@2P4YaY0Hh3?EHrOLa znt5)sf=d`5b9LC~#(6e=k;Cv)B3@5tMIhY4I~MYs3T&PO*kQkPDb_Kh&9&ZO-FRn? znrI^dAB}p3|IWmwA+is@SL_ya7lml{Qz~=e;AcGRj@I?HxhEv|F>pO0!2RI(;^XtD z|9ZZe=M%IiTeCI1Bf5wWe;9gk)qBLENaDUR=6zRdb3uJ-w@L zES}Q&;iC`a^|^cK-HN~Ypx*POMT?zu$HVK+dBu;mMb*CQJ@c;1R6Ls!8Q zU}AG|jVG8yI%z#}5O783vL5u$*qK2C4$z<9^~FY#dqaoO-GdX+> zFy2BN?EfZq;4*yD6L9`sFXFEO>+dBMEfk!0hmTu;wQjM_H_!hRoaETm*RVFk_)7G} zNB_|WguC~t2YK(hV;bT1={k2UwRXqwlhN87VqH@2Sh6bZjDOrLY4@ zPt~iya;dDZ{oPEL-^8}T5R#6#~R!ZeO3QZ%isZ_c4AB8dl)9JbTC!1 zq&HoT9x*)>*ND;^%=3KetFG_c7d3P*yNt5yDN|kDmn=QTP2}l((^Ws^bX&_Q`zB?T zlAFEp>Ln5SWsR-5#7};R*#9T=e|XI$A^G%p6FxoobqMFQ7JTPi#`pzmu+pVpD<`e~ z;x%}jH7Me)?qi02W!q!jH5f^HO!Sv;4ClEOde=U*K(rFfKMeE&6U9+}llxf}nLYeR@0Sj91w-b3T&-D~}f9T$Zfe)ws z*C^XbnZajbEQ#-##eem8FT7xyOLHG3Re!qU?^ODnpz}$61aruHXs@a3`?~Ll`D7ki zpwr@MLnRl5L>D4n-;hS&Pqe>r{%FU4>r*h8ZJ*NxgW2|S=KL#R@To6`!Q;K*`Pg$3 zd+1|B9r%8fRCGY&uY?wkqW{-0))QsKTz}6BOP(cp>(|{iUrBvOHb|e;2K?+^XyO{i zy!TGwd~Vl0LicK3!TAK=bmn~NTdJ=)8pQLzn|UK9FynUMg|=CP%P5;m*~`RLUyqGP z3piV}?BVf-zAnhGQ0$kYWe<)&jla3}ZeY{VJLlS?+_{UOr#OTEIE}yWe(~J>g1N&F zXJos<-z3{V=URGkFB|vPP9nyX>QyKCs?c%I(wY55 zU{10?ogXKX_cG`80`i)NX#C6@epC7ABwjM-=6w4T^0n8NGJj9Iw%SL?+m5`uOJ?xd zbMu>@b8dd~|I@koP5+taX8LzL+ltkAOU|~o7oY9vfBxA;IYnm^OMGl>1aguMRxlp5 z{Lsd*WCrk=y9DE>Wx?_XPq*KRP+CN_g0<2#v*Zd)mPpB1SSUh#D`p4O%V>Y&job5$ptK705sth^H z%XbCFzHnz?Y}@lz_+nEh&%V*EbEoRewPMm;%Ra7*)ZCe&_VC$2&-3Z`oPLyxHsK8d}0f7`t-;1%8`}IXVQl}Q#5&}vh$4?anBEd#%Pdj`xkf6taj#vNJ@EFq%C1 zZ0ns#ocl`MxF22*{3862^}T*wI>xBqwI8HkL(CinVx?vt1F^3uWv&OAG~GVOq*->n z)`vXn%kP!aSQ35k;tMC-gs%(lVO%I`hG5%%hvH?8^bv1w$;%CjEmgVap54A>fgp0` zwX(0Piz81udByI^vt)NywmUM6xHHhe(j|=zz9rk)8joCi5puMJuVhK?gWVE&ZDsh_Xuf%0=h=0I*221C{39*v zo^`~%D!N#4UDnM^>A`>OZ@06CDL$V2Sqtasxi`<;WohZTKhN3L!u70WN(RsQ*1~|E zbF6z(dlb~Sa9+|_{2uPM^jP;`6J*-7##pn`_{Ovp{=4l?>%+6#?z90syX{WP(olZ`!`KTYI;5Ywy->?cGXyuQP`adN(yh z)h7Nl(c1bv;v3xa0Bx>Nn{GSvhI4Ny?LO44t%tg`^$>0K>0MB-Gos?M9;|&Z?-#VQ ziFOdu+!LamZSK3)FmBwWtzUL~|ChY~4bGd7c%NzbP;C&4`=i>2@;;)CgR~K)jaJ%d zp^dov-uCmXS^a6}5br)>-hH;AFjGFx59LL8?<3wj$a_)VYvsKb-iuTBTF!cVkO3RH z6KBQWr#Lc|xxh!9^R^FMXci3so|=J|dyy9fz{&10j3nEoo{3@9oqzFyuk+84A+5l# zzKDC=+t?fGT)ERM+WS=8Bji;U@+uGVs=XtKYtB10+C$+97r=kQ;~{&pg4ov?T6eA2 z$|z;+Bp(oLt9&5y3#=epGjw_Q|Ie&HkptX$lpP+s4=_a+0-Z)XO`xt!Gv!Fea zy>X$}YD0sNTm7Nbm0PhlD^8ALdPyg_P50U$W8a9MK@GC9e(X;H(&ioB;Jag-x%JVG z+0&p;N8wG5)7B__tRhFCGd+j{K|TWL)y?b~N0YFj)W)iZ~a5xLZnjNk?N*~*E?SCmdd zzM`}Y`HIrZkgq7c9QlgUE0C|G0j&c_=lxvZpj75g>Rq~Uwke?KsAwS7Gd(reoMt*Vt`N;v~CkK$996)~ZE957? zn*N|GKY=D9pVPfm1<_^19e)UTYli3lG4t`V*3hM!E9m#S5Yrl;-^#PDY6hRNK^d|io?pHuQ~5^!q?Gz~R%&z}a-(z1_mZl- z7a+TSvhZ%v;bvrm^F66)g&uF!Wu<{d`_gOjysuIQO|~ejb;|;!6Bq5PrED){(9$Ag zKpVQsCOKtE4s8vvukGk6D|5d@XAz)o|XjiTC_A~M`t)*q9)g$8_pgTK@nX4h9OVA_9Qd*!- zsVSbZf}u^s*btu~y8`^Z!$;v6;5p2G-`;KSmk#45WFx}E&cUu73Ug)k9uVBKB@lLO zT#*L{YG(H2*Ngc}hYwKwkEm~aC6M(t^|P9eLQ9F6-XtGf@n7@di6n134vnltj+BCo z0owOw;=JA?le=W*Aa8?4?12ZA-7I{@u19>(!Ap_34Te^5?&(g4%w7T=+DjiNN#^!B zbf^uyk&jVf3Up;0XU1-{=q36rTJ#*L=og126YHoRIwQFHU6KP=>s`2d!iB3RUAWrd z!qrp2m4Po;ek-_2$ZvlIObNF3fP*?8IO~?;t{Zn^{V_i4_rUlr;Ot}O=MZw#gbeqO zynCh$_eEs5FS#<@myqGU?8(GIreIt;EO(13x_v4i!L#qSFbthzGLEr`ANAShIr}a$8IIkd2u$BR=v2VYbBYD;hBVBnG^?)zQInHzI zomnP1$GMcL&As4}>fc0OQMdXUW0H|yIqfZ{Jcsgf;mv=(O6;FU-55m~9o8T8|0m;3 zETA&48?PwKeh?aO;GXYjY=U*pK3@#$l5GO($C+pO0`&NQw7q+LRmIi+zjF?d6T&_B z3laNW9?HpUcOXmD-xcPv+=@Hf)cd_ z8`@&8ASzbc8WNO~1iUnaOjHGI=s zcRhSxBDx`-nfA22wYTF!eJ9gyB~`z*CTKm8twe3cU`I zNy@I>g{iwVf74&>(^AL0M0ONJo+(Ldk*u(fxZg%z>N|OS5w-605B9@&ewBQa2fk@Orjn=iv91ii$Nc`0 zlO^mwxp1z)ekYu8;zH%vUj)uA^g;d59`K`rHT|Zqrr)W36UH0d1!fw56R^@;1U_pn zwhT0LA)mqcc@WR>F$a6nyUxMS-8p!o6UOOw1M*Vs=<|XGXJltCwoVsy+L;GLq#x>T z$@U!A&O993KXXubk3%D(qZ6W|7Y9X7JVlIQ<~Afda~bE&UQCUgc;1TDE`hFYhpvFf zh{@QRnb+u;jL7jSE2h|lwGX>7$A~Qz-H`CBj*hGd{?~@y2Dx)~7kzeoJag=CxZ}{7 z_U{3IhxRf%ZOzNb(>mkn@3p{QHbm`FYu<#1vVCez(Y(&0{wdjAkkfOZSLF2lj+{Qh zrB~W^=Q~7Q)eBtTwQMQv;8(?0pY>RIDWR@hJHj^Q}efEwNrCyY2X_ADeS@yY=mSVxG;vEOHs`y#Ov7{J6`BeRgV|6i9Z}D?^$e$d%RXm9vwD0(V;R68=n3src8vB>WH$J|=Y5O>-`XXDZ z)_UEYCcZD5s-DIF&6ubs-Xc1Cm;WkLJ)I#uthS)}zbS=pA5seZ-&6{ZJxIE>?eotW zHM+h`I%g!Gc<*j7`7*gS)AgRsH1$1fruYHCkLKJ(dJ{V4<@^XdR>AsF(KyMfte8C| zSc)vI*x^&~{~<;d@=?liWQ%!i=uJOS|MEGb;LmdWobW~T{G95jGXi||eEslq+8*!c zbQACD+w!R_;lF6WJOjVa^NX#+7or2X?P$KvC3xpKvg}LH^(1$#r#)ov3MN5*ZR@kq z5Ak`akbO(vB94{e4dw8$2sTW`)#<nUiiQA%~C(H zEvqlN_+{lLSg~L7&Cm3!D@;5f`~Ca(AXkHn=6or9kC@N+>!42qgI(at3-3+|%KmmpBY(BA#vYqyf5Nl! z`>V}Dcr`qI6X(HN-25cvV@qfsXT@$J#@Pj$AK*7u_E@jJ{=0Qvk=+#?A66|w-H~wGf`n;nv6SeaHqHoyfDn$2^ z5ma|GbQVuH*fKt{wq%5lxaUlMJj{WS>;Qk}zxcx+m5vL=UQ-J1`n^e0?BAL+*X}6% zY#&1Pxt+bf>9VU%u&@!ov=OhpgQc}`HdW{x-D1D7Q$c^jdOEbWlW*iDTNToyrr@iL zZEAnKZAyMF;OAMoO)cZbA1YY+&m#v zDgPp!hdBlOo%ge)-_3FS@QI0(%zP39EZxD;IZDep!>RhE$Y)1r_c!@Y3=QX8jpCw+ zPo>!z=&bmGR2H#j$(D3UFNZE>&;Z2@YwFho^1ca8bDzfR?dYcO_O|Yn-mCXz zGnPJFmgKxYp1R_z;N^bl!14!@Y*5QL<8|OUy`B0OQD5JQ!CSNP!ISgxm*qTy=^JNz zTG{WA4{FrtKgmY_i5=z~^0GdW*zmrQ*s^{RV!~F|d3q8$(sb!RL(qQ~5~~XtP;>Ac zK8K?Ffqc*Sc)+W3SpRD{f2q0j^q*ejHG>aP*Y}U)A3lseH!%NR zz3$~xDX*nmZA;gZjI)%ye|CSz=s`}K>nMMVa%7hMjvn;*M90S?U2@ER`F?Ej|72o> zJ@;nDytxWr4{VJ3#&qz;;2QsJ#;UwOGxzvRf-{_{bYT)ZG$o>UUtB&i_+m*Kx=Fgx zP3nNd2=o*7&|~F?h)-FP!8yScd$9|*D%vu7V+wqQKI+?&RXp8e7kLxuzw*5C3PzhX z-mdhz-J@6EK+%Vz0}5Uns9O)s%5T0tKRMFA1lb{O+^;Zh&6hLgbUV3o%<1+D`l)k~ zroQxnMb_!SLC^1`PelAPy4;Gj5*s2A!sc_#^%tt@YkW?k$NEJ$=N+5D_l& z21H^HbKcVP9pc-)r3V0`D6wIh%GeK_l~CC{yQ|Mhv^WhvlXTAdq}T&o_^mQ?As^Xv zdl~-?UL=QNs~K}A4!S-kUnkF7Luwdfybe$UAJ@Br?JVkfxSEdqzW`hvFI@flWnFMp zunkdHxZ1sJDP`j+GyQS#bRBsGULNa3Ei&pB@7$elRa?{8 z_hK!3uG@H;6P@wM&M%oh-Tkg7)Bi#3f)C|OE{_Pmgac-dCDRXIdu97vWO`_zw$P#D z??c}n9Ypaz)wq%cZ(yCI6#Pvz@XxS&lJ3LK_wSs~*ZCvKDyQP)W+z4u{D0(`SQuB= zaPOh(Yj6_#&0+itcJcc^{O;nXJ=Um^nRA!d;7fpwagO6#kh}a=o_8Qak713I+*t?z ztIJOe{_cGG&3Yv{x{8=FOYU>d3Ka4GjGT+FGi!@{o$q${G2Y_6P*2Oqx$2|j+$(sN z9Qqn%k{dr5H!T=1-;N+Jij3Ph!52JHoDghY)m_H*&Vr?C9h>w0)9j<5A0XqdpR%Qw zE92(aH-FKQaT{412E%`lSvt?iBI~cW*7fX^S#F^(I_Dx;Uq0$L4sv+)FUXIRhu3@Z zFfz3#53i4xhbPd6?!{AEx+7cT{1I(P9{!<8Q|&uUn)10igexTeD*GcgupPtos=DnC9iN%*|<#^)s&pO-%Py!6HAg|l!aetasNvnbv5w{zj) zmLr_GWM518HKCVPk;$)_xqpk_E8x~okv(>lWbN8*C~8elUE7|rAy-3;O<_>I3%-KhB3k7sCM z$7K#KrO?h|SLf4N;Ukwg`F+VZIM&0e*j9+XH!f6ZqkAZ3JMunvTv*&e1HXU1XE6Hw zH9_Lf1f%)LBiO)B=KF*7vwgwn5@>o=LeQ)+e#_w9xpcyq824iPJ+cp}pA zUzv-21wLQSSfhUK$Y5Tgtb02aT^+1kcvY}`?$luPpj9ckbdUL7j{R%W#)YhPKcF7^ zY@L5{rOvao<8#rx6&-Fq?Xy-ZR`Vo(>hdb^MzP&X!JA3wWz|dY^}e0>vfQt&Iyf;` zG_npB`?8jA<{rA2z_Hb-#18_m6uUHsGl%W$pQ`T9fJHmHx6YVL_tv-H%b$&JuP=CN z5jyxX@h4)lq+Cm^N6w5bBQ7~S|JXs|n&4BWG0XP(5bdZR*sfAz;H6}l>U&s|nD_63 z1CkAYsPUqID*k>R`K#Rgb;>W%yk^+na`TFnS7^ol4m{TMw&1Bb_A zM(^00*%^!b4|z_UanVnbnHSc714s6El>!sLQsy8*DR|kzJBl%XS}8E@C`|gyGj^>@ zpo6lRk?e%jWTPW5gFoE&(#A}5#+m1oeFeBXmBM>=hF(~o1E1G3PM z75l~D2Qrp)6MzM4QzOqF9?Nfo^9->e=V(iN zpo%m5_51EVDBRPzTea)x+^P08w1>^1fb(3jmPN!>StFWCvwsB4cQD_aUynWQ1J}jF z(YI?hNY~bxcj83T-uv*8V_QR&|G>T*B37gB2Nqva8{*UYzlyesz*YSh9mM$W+Gn02s&X@3;1H>R(CycKU6`&KmKRe5+z2Keb&7?$y)x zdUt=4{{Pb3qtyOO!*KY^=v3}@NsBa$PB(W(>i!qq*->{W!R%#DXWX(SN2}=j5a3jn z8EFLuS{EzFJM@!FA7nGES&maU}I4Q1#n+dX~05Z zc;aXo9wa+)ioJ{R>05_~II(bq2RXpw6m!_j{}&dG48Gv%@Qj~$)IGu}=E+tX zBD>JxJy?Cqh1CJtIw@Fz+lT3!qsJ3d*4;ytuHnE#F=Wr!I<}LK%oA^STi;cWrwtcI z|4m)(QOh4yXKZ3+Mvvd_*)KF^&5@(Wr`lPaV@|c#(9cugOMNl#j!URCu#!IilofZ5 zBo%#rs^UP#nisnJd&|t7FX0sT3`z9)^hm>Qqt9I+`z z^KDCVA}zI6tp1OjANgx{U4AgS{CV)IYGFqot^+3| zSLh7M<+QiU(dV~Y(&wkGy|U#|PoH;a_!03j@jU3@2f%u*@Dl`A$~9L+gXy; zLj23v4gB9xh<-&piAMY>Bj{8SQXBnZy3X{WU!Bic%x7kWDy4U9bM+|h=xZrL7Pv-! zJIKR|!-rqnPVKi#&sxg*a01yNx}#69VO!rI``MMhK9BJuH_1NWCr(ae_8DEPl-Qb| zrW<{8jKe?w#QXR_dUk^S#!eua5S-slnPVroHu%w`F9$z{uOIPW6)8uLh)hFQ8sq3n zjp#}`gL9;;XRxlSN3eajKls5xYzW&Df*&W#?^EmU*?VQbV7yxk7wq~IvO2PO!2{U+ zr*O-uQ#Z$+b@3U!$h9MM#zt7Hy^oI)O&qE7fz53CYU~J;96Q3*?iuaJ$oK3B##ZC* zt9f>WI&34|?Ff5W!;KC`Tk6Yd+K>$S9h0WoUo)xf2%jC_MRo+!ez(21*WzL#d@!yn zZ2vbl1v9R0HiZq0;WPK)s(KwzoL8Nr(fNoLY+LflQ2gnW>W>vhwPqEN8@>|Vs8@M`(Gy>k%A;ZPybEaY0)L?0%@I7n!6zo-c_gQFSH_yL8 zc7JZiRl!n25B-);;aTNRaTnzgbcyFDgesp$cC*n@ZzGk?s(ro$%03*|FL?ar#7F~o zuFI$9^kC)(TVS~_-Rw2j&1O%Bm>TGx4X(YRE8aYo)EO`ERr4;g2&uj=-q}&r>oaA3 zofYWd96*3`jgnn zMevXwr+Bw1=yK{?6>SkahJD49hC*OErdato_OHlSUK#qDbj7-ScQc z_AH|%(<61zhjh6Vbh+o%9^=-#RnpbyuT7r(L9}=2;~VH|UFYGM-ktQjh%pKNC+CJ@ z@^|=|8FwhQQ7L14LMdb2pmaEXzDikx*N{F6uAhnFn2P+Qd(qG@hM=Qji_{!ELcPVn zL-EFU!iO8d?Nz_ID7^UA!q_eFlA*(ThL7Lj)ct|lW{!(|;4<)=#2JP?z@yw}jot0@ zt-J-kD*f5uqy7WmAqfd9p9aUF#r@v>-a+UfaDPtN_?Ko_mF7Ev#PA6CxW;+_xm;}@ zq3wONO`kqmH6Rp)xt-135fQRx0 zOXEAHeda^?rp@WS!iHZEbH(V~Uj9#eI{8Ne#-~2wwmp@$d&9>RyGy<->h~g@A5VEK zyAJruUo~kAaQ&iHvG0r4xcWqJmHbhx#|B2s{jK23*>j&9p6u>hSLZr&|DV7^_^LA_ z29Cs}vXVlp6~`c&XjSF`C-gZJUnAh2QTO!WndEyiX^Hlacl8Ndulk5>(JNd74$t8X zbcl20<=D1FyLIS2$C<+hzW2=N%OXEyT#tgs_Uv(yXGUKhv1gBu)HAOQqs|MqFB=kU zxW}9?s%(HJxLdOFIA@rN|5VwOLb?k5F$28?f3s6VkXxZK?t_o%+;bj2sl-(YhdzFI zqU7TGzjE$;Yih)cu@ZeAc678eVrK_=N?G~GgEiC1FsdPjCj(Fp|cFWB$)h% zDa*D`#+SjnmQuEeGR^0RzSd=$j|R7n=7AVhsp@MFx3A0SpL0Gj&0c8w%K32m`YqaU z`pVgFx33G#H#6-+o!_Lc+t(?kEXUp-U*`68f+=G!D89_?YXN2IYe478>Fb`jzV>tR z@JUi+s4i!Xqyu{MHIg!q9v$4_yfttbOxw<#sPvPyVGDVg>dX94>=*HUVGY+RaLbCRsmIp=fli0q5{-mB_shGWZ!Qojz~r}bEC9&zL!`UZKyo)115xZt%X z8}s{`bLzwZ(q8ZSJn0>y*Ivny7S1AI9=-6a6?tCn@$EtwV-R zQtS%+#`J#;sn)Z9=6E=gV!zIR!J{UTyE*xOcO23DtIucp>%eXp->jr8emSD`U5 zA5%yj9H(vOV<~0NlBYB5QE1T6I&^NvvBH66*D+++Lp#TiVVBVNBOit;k@NRYd8RMC zAj_Tu-{VZnyNhxqZg|*>+FXedC=^HdLdJSd8~&WQL#cyNBOb z_;JrcWiDmft38NZiXUKQ#qdz&wwNUllu)JO0@jYk)>76ZSa;AL zY<$rdtS(Fl*Ec2vJJ3b(t1>>n+AHtkvx(lE%KqF*@H~V))!^vFe!x%iWDaN7&)}*2 zcaeu)*Ss__FRJr#w&o_w{?Mdp_T!(Ye-$PjP+WK29jJRYj;^8Ki{$@^ALQ-Yv*Ql> z+N*=wpDRbtj{crFkBk#rXk|J06TQd61~@fXzN|2)I~dB7U3%pG1Evot_FqZA?GMEi zV|Oz8)p*V?cFeI>Vr!mvGKqKu^I5Cu`){eIIhSo_`WWJ9`q8t-Aj4xzsb;>nGwT=o|A-F0sH<0_xc7?%T^T5K=WS({4y zd$;_*R6dLH+u8G@Uw6CtKT!Tc_LPwmM*tV?)3wdDVuyQLJHLedC0NaI(%0Wq3_Q3)ZMfeJBw1s<*c-PgpYO3Zv$KP5%m*g4_xe|3XWH2pV~@|YlWBjA z`Ymx|2oTrRE1Rz}um9Zjs`k3~dExn%?D4UH(%I;SLrG_$8!BDMe3J^^A?W{0jQ^cH z^1*|B_i+|paxD6_^f!MVe%;1THQO$ro?`ni8G^o}IUmeEgi`kYGTHl^m`j_?!$|7A z%lNb}DVU7X*hw!`>di@WN5SF*e9K+fkDyF@5P7C* zf{y3G;a}7h9IBu#jlWd;I&pA#g>eZEQR*oUPewG#<{zV9YCkg}BY8-PwX$68sV;FA@1@SBVZ>QnV^!u5=V!=}oGx(^+c+~hBqwKi zj^f7=CsDB#ta~#z22$`9US(BAiH8^gZsJ3%JtgeP1+96B$U$ohtb*HFQ$nQG+_^MygtNvL@qU!CiZ7pJ z?`t8?1?bn&C6g zkM{YplJS={Hk)`{3K=}kbo|a;T$&(U0zSg|2f**MaOrqfcrtep%J%wErh`kTmd%LB zCVw$~5siMqq*?YzrO+K~wuwu6p~+9RhnbZ4A8wjw(wxuTa~HlQfv+RetULGhwgyG9 zh3~sBV)oqk#qYU;k0ac@?0xKIn^+&UJt9TLXX1QBwz#R<( zryHAt*3Bc(dmVHb^e2scu%|V)2%An_*_VSl8>KiP+CxqLZp*hapg0;l z?;gN8?}XCqXfd{xWY))rr$t(b4`TL*8N=Gs7I6gd&r&Q9Xu{YGfn~J#s`0WJ$qsT0 zclId%UGjS|Ckf=&ASauBYT9@&=f>cb?BA-sY=23dVg3oc@;T_bliqlLQp};Z{j?)m zN}|4GHLZu(P%CvFK{0`3^Yh(%HvMVc_vkNPx2py3yZ9b}pUvE7um}CSO*S-_{?2+Q zo(|DF>v<=SJr;1Ta)jTytYs2?VLSn1NwgvF%Vwr^_id$$H!1oe)@w91Z=Cqwk=dM$ zV;$2y+DBe=W4*$+pv61aU_Uy*o{-wx)zYzZ8EvS|PwiW`7!+0XEregD`gE>$I^$4) zZ+8M;j~86Tf0q}m@4^d)!wa^->*h1>(5Jruzll*+!SWz*8|mby?@ASVsCoFwEzVEs}RC95W`cMC~R;034XpFD9W7w;dIn((>(U$)2 zChuYTtp6>*;8mWZ%-Om4&X69I0ps$2r;o(P?G9s|7uNlz>ZhN6RZb&-J^#hGrlx?iQo&*V|^Z{b6a2M`A5K8c;K9! z!5@jSiO*O0sq50^=k`VMh|kfN2z^jc<0>_!iP{v^(v0q&qhEJ@CLYeo~tEj4pmsnXK>XPb2m9ou=g${*6}b z&97P&oA_zpfU}+j_0U?qd)^!#`^&lw3C91)_(|npFC3kLpA`Npqq8EO|4JP)%HQ)|C2ea;XUDn-;Hbg8hOn_;A7Ya z6~}ZeXW*PT<4zn?ox3pgfr;#e&r)A?12=eaPS2b({{>~L>%}saz3}Py`bpwHPW^S1 zJxSRJ!H~EQ)w4Lyu@QMq&#r%zIrCHE`3hy){oFM~{?&IdzJCk16#D`E=}wc!n!&Aw z4sK-`+#>ELuoCR|+>TF#t^LG@SdY}1U=yUNN zz9^pYnd?p`-n%po{)rdvQQx}Z{Y~l{b3C6^JWseYLVkb9LLnFL7pM+6wiX+t@a2A7E|&N%0M;I?;_z6I`FsdmAi%k>QY zK1D1-6I&cT$%`$13p&Dd-hGgtV$d(@>BJXbl#sVFApKA>rq=F+Gx}5&bKNDjIJjQK zS^PX=rE7fM`xcPght8=!YD|hTUgh>RTYaQ&S}VM<;+OdP23%D8CO)=!L=y3__YoUe zWBybchQ8uyxY_kV^Jq8%K8c2_^A*q7q2c>oTkX*_mws8-J^=r_)9>HNI|*D%p~-CR zNvyb{OZ*4X>>lcllw@h0mplPI~Q16K$uEW2QRvnr-Mc+ui_d)czx+<~|s{^DzH^%6w}s z@i$MEK4aD=;Hx;`dX5qg&W{rFOPk&+C|R(acO)T zar|m8_3!uUyYzLGyzi<0?Z~py*+>ICYfj~V-#M0rjvwnv_kGWg^#akR^gi%ieykH{ z?`7!G;Jx;f3)f!R_K@qxiq5J2SG)8gJ~#zpTkoHucr)jq$I2#0 zeiuEa8+uH)Z(#03kJpi^Kds0l6FSF|YuC8*@+HRNtrcefdWCm40rTwgp0U(%){3rW znvaVOENIVND>NTLFE4IBhPiE=WZx{HZWDdUC&X#{~9@HCFk+OJ5g0 zv>5)$xF}cIGwL7n+nhhBGuFM9UzXbKUG!XJD-u`R9Q3(Ua);+Gn9G!_ZWQ zy;g7nhHFUGw`Slie3M+EG2EqcV7N*tc(+pN@=)w{(l%se*-x~W{EUgsq!{=<&hc7d z-EX?7@YRVCcmRH+mDN>iCvp!-kbUXOzcw>pYo>6n)pFwCe~wRseAqwDpJAVSE*%~> zPUqL3di|3P@2g*ovk{+#oqT`C+``~1$sx(}E0<=6Rxiy7wf5k5G&|I~SHH|q>pGsd zP<9LZLK?e#3W?nx%hP!CKhvjSdz??h`gos)ql26~QRUN+taj+v|JJABCD%@xO*@`X zgT^kOhIfojF5B))nQ-t#P9gq3R_bn_Ll?@gW)INyXXrd*u6l943}>AKf32t0c@F%4 zrH9f+*Be5C$lD)CjjgQdo`jBhSz{A5L@bHrZoJ-F_ zXXTs1KkoJ&rF%=?&a^6C;DDW_?nw{yJ)%P5zx zqSm@VTOCU}6(5GJZV-xN%YKLc|n zn8=1w&b{S=7j_f;=jkJ3lfH-jp<$o)!+Eb+dGIT}JB2>!-04|!vui--U7gzoo|&5$ z1XExUC9fLTyswn;9W^O2bCd$dYV(|C|IVcH^O_5-&4K2whxV?624~~vbuE5gJWfRwlLifxzy+EKPcvb z;YWK18NGPk&@=v2x`$W&@Nil1lHZxWUk)uByx%@Byg+b?byR2^cevyDY1}wOL!INm z*Gn{N`Z!#10zSQu1M&N^z=O^{-sJRg6nzv;3IF5!rnc@-xtY51{XPZGE-+%Gp2yg<4ycVRV%y2ThVygW8?yFw)yY9NZ=!APu&sE}{eA&$>3ik)LV_pI9DYK` zN1}}qeEkKVQPjDdZ%^l3iSo^v5a{*-@?~RD-e&m=#__j3#`ke>cyglSSMUyZ*a;T$ zZOx+J@+rtr3XZ3l=Ttkzq$&15lcqs4Hv{*Zfd4|~;j7HY0%+z&?iNZkG&7y{WTSj4 z(XmfH<#+9qIacgP#n^RaADw(@I4L1iIWq9$IlX`U?48_4JG1wtp`=dF@TOxPP zOh}$d-pJ$dtOxN4NE*!j%x_xP<3n(%AAfyHVDH8Q zV1IqPx36pUgqOf4|H?PTBS%2r)hV2jWUgt~jX}`IzT#uYC*Y6N%fr6$J5s%O99SNq zuU}Mtw{JdxZ_c^Ci7nT}AP_!w`r_bEaFTPgd)NmmN5_{RPJBPjJ8^yhK64&q)1Ak7 zhwl~ycphPGx-*dA4@>7hpEFENjQLaBiu28Ri#9~lkK>EPdNexuXoYQYX3sqf51 zpAdcCMV{J>_D5IBw<;F%tM+sL2$@DSQV%U?k5azWfrL)@be?1Bj3KB!2KcIC_bKkI zVkNxx5&B^sdv#R_;V&?!4a`>zd53RR2Cyx(;Tw`6y*;CeFmRsco3Y; z2y&-w;DV7|Vvk8z97RKHD<@A;G9ukM_sM& z*kH^#^UwsxEP5J$T z^~esUUD-4zS*La8uP#3jshfqIu#r82km|Z{u1a#_$@PxJ7Fn?%{W0gymsrG;BWB_4 z?9B-7^l#SaKx7te~ewewc69><}FgCz!tU;QOoHzMh5K zrr(Y&X&7}En|^27AAonx94J;>hCP+C1(fMLg~l;qCHTO-fCk^fzqvAxdp*Z~BOZ6N z?LT*pA=@5pa5vkYWBMvMId}F|)8{WM51g2#6gr+k`djw9Kl4m*VcD%Kc0^>w{> zw_@?0qh1Sqf1lc1LR=rW%|U7t`ui$;pF6)}FQMmc62CkIFJ5Tr{yNh(_bSD*J38o3 zw*4>OQD6Of_Z;t(@}0$ehjW0YUW~gvOeiU9Y}1%|%!q-4Yg(7$%G9;>XpnRy|7Om$~FGNiwm zySeFO6umaP82{*nPJ0hghdF2noa*S@m!9Oxe3KH9`7U5jdV&=*FzqokjDOSXcQ;r% z>pCa8+r()oKyNBQR@)!o9BLik;5^yLQe--O5RJdE&ipd}^YjjPA*PzP-WVDl4Qx$2 zv*;srjv&iXwo`eN`~y}N@od_?{0nxRd}hLdjnK~o#!@c6_ zNWT#tjg?d1e&9^4694{Rizr>HM(52y_$azEeWrR2Ke*GlP0ej{2gKvy9 zaw~l_-`JBMK0%ye)v<^HjXc-o+^%d60atdtf->EmYkecQ5t(rP5`4rH@^mMCZ+j{6 zEUTj(1-kD{y0hLpKTZY>IPYxl;sZX0SVX*+cdSGHq@A)=XWetioHMDv>AjAwGAHvV zne&`&#L1cycy~|j+22aSceNg!c@gxZGl>@SC_mpi;L?Ge=n7=hJ?KCloQsfoyTf@h za9#`VG%&qmSokNrYhbRqr48{Fosavt4SDe04(SM8;4B^_p9v4f+oh}WUh7<|@*Qwn z_)!i%oAy5%7G6vHi#g{Odi3hpFQFmDJ&&U9#Q9>F`{xzAOY-j$YBdyqUjeb@%Jbaq?UaLPz zyi9d9hkD<@t+pXy)gF^XL%Tlh8W5yD{r>t;2hHSGEK3X0^44=VNYL(mm*l(TPO&b@0kb;J41x z*F$SF(PNU49cO-H)W%t9JSVXS@`Ho}$8*pt=Z7kfZ?(p=-!WeHwdl5lNJEDeX-Tx$ zr{>;S^^Y~n@q5g%k20qBkdxlO9U0;8_+WGQ9dSy+m$44QObf~z7hKLh>Nw$C#|IA1 z;qRdP;wyv?F+ctSoY9ZUr!K?s@sJ)QTV#~`lj_*ts~esY+4DLw`CQHc!>gkCX_4p- z&0mVW$+e^TX;bh}Osu}h6E$XEH@#CIY5_iNjJ*iiSaR_=XwbPE-t)zwE^w19_z&u; zUf|}gWs(KoqD=6w0XC|?l)MA+^?{|zRkn_@KTu|5Mf~L~SLX>Jr&;Jcf#vaXW4b*F zK4Rd@`%g12wIeuIfoHzOGQpjbpVMEXqe6G(z`>al^g&nedDOr%*B(gf$lU0b(2(Zfd*sEK zuMy0HVrQS3ud`!kucWT(1#a$Irn$Mze8Y*Q;LXX`y*%kV^BsL>aeNzQ4jsL9FKsMP zePo(r%ui=OSK&`cndJK4z@w*FK}Y{a?o*uXo7`{s;34|11x|+g2d>Ro#oU(|{b9Xz zJvad#WOu=XH2YWJvV#X{_7Gsn+Os`y{m3x86%#Mv{w^Y;rJ z`@ke%Z|GsTqgQm&gA4lt)4yE%mz`q*r(M_&rmP*s}_N#9mkcej_>yEb0@HtstojjInul!Z_Xj9WZO_z()``XCdzmxqjsMN!%ML8Bcj> zw~QSBlU9d!)p4hTY_rwuVQKwS{QfU756$`NTZVmDcRnx=tKE6{2hXAb(eeRP1};+X zD1EPK)ICROow};@f_OQmcYJ&XGj?a3!}xB6#tu$SA&tk$sgylS9t&7Ocf4Fm*$b43 zhX@z7FBFK!NA%fq;bT@OtkUc?j7#JExjUAhYJA{LHRFFmdEnayrO^E2O5u}_k`nvY z#CxmDPg>D_dvElYWb~Lm=reuMYlxq{raya?{&0{xNdw#q-iA)wg6*C4!Q2CGXuyqg z_2wVgqr^X+b$8A7_is1>&RX&2ez|g$&8i)auH#lh+Uxq#3%_|~Lw8f%b(S%&_7QgRFuc584 zP)GN}r@zb}?uJvSx%YEQ?m-h*#o-=8DADdfNK zWU@(9?MWt0u`e)b2KMxG$2hCYpRs}%d&Z`|X-C3=KJH!jm-C+Fz_r9hZYvH1TNBZF z{W&AG_t#e3JJ^o=+g98s*xu2L^|${;g1vSBwAc{vV2G975V+*YC($E5#%K7+q&&{R zCG_|>_tFQSobJzks5KFs>S-MaB>nwK<^Y<=wqG3P@T6?M>+Uyi_t9QHxLD(ty@osT z#5bYge&h*`KjK@$>+uE#&c2oK{*=K*d{y0Yy({_`-hXJyGVFW1ltJs1Mahf$S@$wn z%fS6=f62u5WcpWf|HPUyD_mP>nSOkY`B9&>SGI?KKTYhrTbY|FZ>Lb6Z9fbQ4gFL8 z4{)giyR*(8NQQIfi?c7}KO#A@%Ex|FPjI~#xSj~E2f+0taGjVJCBE=y;dm}Mo(Jyc z()SE-Tk+i9=JztVYtz1myT1gc!d=}p?BVXSZr-oS^KiG^&8sA@JMOM_ad#_qG`}A1 zmbuR!?mp!{YftDb+&!5Ro{S&u`SkY&zM*~#cOT=waQ6|DrrPUGnquEgDwsL-a>3!5 zz-9(8nhva{0kf;IZ(N0+jnCk(p?~~EkR>as<@d|nt)-0)7kBX)F#OvJPmA4UXza7` zG#*D=vAt<-`?`2~2kmy^DeZk1+^HGk;Asit5}rEi2X<7xbuK);#^7S6{f=86kEd6e zvJCrRmof)WzvSU5vbe$167aOY=;(ooHTWFWE^+X*g1PYUG@X7M9Nm*@aP$Z?_+Hi- zJRKuAfv4@@soojQv;0Qhch8#^@Vxe49TmWMzjud&_EqxWVZbT;8u{_@7p3b&f%|6K z2mlLH-YtG&YZ0y}5%y>v|pE?44l6*8Ll_CA+A1f1Y>$g25qo4Vn5H z@ncv+v}gP_>&MId6f@(y{B9!uKoam3e}K;a27E+g_2fA;mTkY`=DkOrLt|O?U){WF z@jc1TU=ValkaMbEe6Lk?Cm^@E}t`Ln*EGPQ*GO%nf7{8(WTyz9P@c} zSq)v>o$2uR&Ahv(w^h;T&mGx5%b`bnw_?`g(_-rl??_?oc=W8bBZ;-6h&#L;ntaWy z8;#r-T@4V=d%l@ECnM0EcOj)LVtV@{_D@*IUSy!Of#adUO*T%Ex6<<^4OXHE> z=}5+OWOk_1(CjN|Vdckwp~KU2?P;0|@`|BZ#a!~{Lvt9S%*3Q9^dG3p?}aRt2(Jvl zGn3$*z2Tw6Qd-j|a+)|plB4AB)s8Hs`z4x@rQ~B8PX`a|W1RutIWI9M6WniRt@%6a z&P%S`w3v9c{rIN(W@MD(Y2o*pojvIb$+uXOymhBPV-%g_k?*ZLz1_SV@pY>$_FJU#yAqBQ0K=`H1-@Sc3}*q$nZR@gu$|62Gz~fE>J=S@ zPV6SZ&tfgAPO>UArhoE2zMBn8`!^$tcCS-)zduj=nN@&6MH|4N&Z(S*By1Mi&mhkY%dv;OQQPcWO} z;^bv!yxOyJ?w)$alxNsixaH#e&A`>$)7nD*`OT79ojtG{g~#Ar6g=L;Ue_sPeBpBJ zEL`SXC*u>H={_U*D;hk0ECX6pUMO}oxa`PC+4f_C4R{W&G?6@q9@A~ntn$X0yoAVI zqQ(C3S<)LskN&KYC*j!@_lYJm?DMHV#7b>A32ulcqyEg1P8o53;>ui^aDSXWv)zhv z@1?sA?RWbs9Mqm?KfbT_`nz|q6FAR+4vA-*KWzo@ zH!@+VKc}IVeVR1-z1tUPDCPem^!B01r$hXcSF$e7gJ*5G{+<=yLHix!;L+^y94*tj zfgRKEf9|B=d%Cx=;dhRX{}FZNhr|0_$Fnapd_Q`M?yyo@=W3@Id#&L6E^SDLwE7Tp zRs7=m$+5}wWoTZ%@PvguQjoVwkaKI1b4w)WrrE6qKQvcq_D%F3 z+ql-v?Nioen>ien!F>euQFXt|xZZ`===r)SSt-@Tjpp3jea6!>R@1C;30F)61jFP_Zi8jQ~l97kf+YG?yMGlcgOj?*(Uz8?%C~Zr|+3|ny`au zzrXapo^u03=OqVL`I4wb2|MGU+U%hAM&jj{}petSt&UA50i?QoZCL)*+z`-m!J4)z|M(V zTe-0woXNkp_r_xOx2FS_r+W2RS$i({R16L$5U=YW;LBOC+nyUvfM4hi=rex6F5iJJ zzMbRH;~Vr-^Gv)shgaO-@{0BS!k2v2sq;tbh$rkxvW#!-6KPKVZ^<`2g#6ZHeS@vY zRhp+P=DR6{^DT$^1*425TH8O^(J=rY+gy9D>zDfi?`o~6fd=-#%N^et?g@dPNXOkq zo_uwkGqJjFB*)0nkCQLnolh$MAepBI_)i6wer(Fp?cDe>=0rT;2jq$7eTtFBJF?kc z0bgjH-!sxO23sxPjUMwy6cenbFdm+S8Aq%v+GP!(<`cp>g z+(72gL#$C*b}!mfY`qd_(fCQ;yJKU9%D~|czA?ngXefgYH*m*h%YEtaiX2;XuXk$E zjibom&4$Nh+l_90S6}GLLxv6M)E6Se<`S$=L6hPi$9S%$pYJK9-?d8NNk^1IQ@K~r4z@go@Xh-^(1>J92m2hB7s*}H;e9?K+!G6J${?w6; zjs1hs;gf@>whjnJI|c?NTi-gwrGM&*FVrx9(iv(FIX_7~xGj!c_k zuVuW-dkP$r-BmR1&7J7m`+v8omuY{S^dZr7RZnF00m$qFktGKqQ>GwWrXpjeA+x7r zkL_t>cG0QM;%QBfL8tPqYwRYkmm{w?BggO9`^g6BQ@VF1Hrk54IOgi$OUtebK0A79 zQ1>K8eVHS%X$%VbGiJ^7_g%hplyg>ZoqKj~DQRBH{MAQChE@;D39aT_-|DHH=UbXZ z%71*hRzHw1WA&}!nX60j-K&|O5IF%IA0KX2HstFbx)ggyZD{<38tW+0W14+2I4_!> zL|&A65?^%A;{@#CZrSCkKMB8YL&K@|P&faJ%AbJGEOaa$DPEzrzMy>MUExu>n>So} z@a#tDy_mUZtS22JSYwWkz42&A`2t8*|+? z=L5IwR>yAEl_z{Nuk&8QEj=8&S@XfE5uIhI;klarS1V;a|5VEO{;m`p(0%b*tM-t3 z_8QUB=`~*g$8G@E=7Do_!M!=?G1sHVT(<(8Gc74moc zavS{ouY=!}`*Qz??^#1>3eV^SyE*^1r7GdTDR9m3C+46Qn0^i0ujcL7^u@@JgRF|( zoS)lLn{dFZC)vN2wRo;ucL#MPM^q1VWssX)esxFxaQ_>f_H61%_ROQsrpAN=-$`-u z1*_@-%qxEFr+|0);gP|YYDXc9q}gfcDm#yTAsD^SH}}P|i-L8`74k@Br0n8gc~!b&<8@Q2MieDoce!Hw0i&hQLkxRZJ^Nb_*$K_uv zeKHA|s2*Er5Z!JOb1&U)Q9@e7^h+P_w4+7~En+XOonvv2MRiH(#F`~Ocq`|q8M_U> zYh9J>)CU+pU3jB->-s4lolkj|eV92kx+dj8%_Dh+p5d)c;H1tsBG+z@)9X3kI5hlf zZ|)j_ugXp=8tG3x@4s{{@4s{{^PjmHNc%c(Ek4!Hd@I-PW6~VEH|d2&_Ud`y!wG#> zoEX!0MZ>6mE7*U8XAgjG20}lBpraJ%DHWa#-!AMK-k1A}Ya_Y;eeDl(w_LkCx9r+$ zayMUl5qrq}%pS56yIV3>BfMPu0pi<|xx^RC-SgM7zl5qj*{~KnsN}=D16IY2{2C8h zkw)NAacoNPCFHCT*kAA?A2idSJ4zJ9Vi?GAI>Ias{ zKBBkR7@O$v9r7IAEz{oX=Dlh1d=c#lNG6(28Yu~lFNN2!-(mc=wD%EAd+^CLbH`7* z{VU#;kFMmMqT-5;X@<9D+Aq5KuDsKA&c2f$zQwhVuHc&*XCwSYc)Cq}MXre`g=alX z+SL|w12CTl?B_BEbC`?k;bGUo!)7x_*D_aMUh(0CYp}~W{H&%d6e+?UQ#BC3a@s3~ zuPx#KqwuhZ&-ce-Us6MqyV;7VQ%<^@I~VbHjBP0qjY|#?A4?$~rNa|`M1PPS?W1MF zX{Wpfo~JsE$g1bN^&X&}#&xhi{E46TZzNY+w-{cwerS&Y;lF>yY4aZHN{(9FmwJ!- z!dJO>ny(}u*}FphF6XX*I`+l2@6`TbpJ403zQI@-`cxu3tfPPMgBJ%}bo}{&7u9ba zbWth%O*Wb6Q`WEI?J^x@tks`kn@PnsBRlU0uHEQ*zVFC$#3_S+)L<9VoOt`=SCc2Z z?_TIyJj~I}9GPtxW7qm$=KqR09~=XY-u{MUcKJFP`3~4ErLK|L!O#D3+cIU~!8*!v zss8|HZq2vhQ~P-~_F>j`%j}>3=R_kvU?bK(Ol_5PwG{iu;GpyZ$#NetrXyR8y=MKC zZ(cw-_tgPshXyovS#}@BYw}&Yjbd=0#czs+89n-X_=D!xPaUPFeR?;;Zd1ybV&*x; zZYG`a|3K!``FPzsAwJO@r<=9ICo~uFeB!d-V8`ii6`X?SzEE~mu(Eb)@Lk`4l}n#= zWIK34E4;wOXONsX48G8V-(KP~SXon7zlt2!gsj#CESizQbjCrp+)?ydvTG~**a3g` z@-oh3tVO?*UH8lUpX3iL50Gxt7>z!L><8?`SN4+U*xPgM=iI#As)sBmnN4)9dM_yt zIZo@ccyO7UU!i@Pdhl!JkLU)Z>t`Skc!hT&TdUy=S+g)DdIzVCrsx%LnFp7Qd^Q(v2rN&k~P>4`d9 zumoLiMDYt7GYpR-?w?z~O#7hpceVTM%5zUfth9>KQ>`9wZR-!fwk#lESO zwfQ!sjIl^5u=x+A$O_kxcGdMhU!L0rUu|Zci~5oqs*v5D0XI8*eHxZXjw{svBqP)P z0e-0GI$zQs(bpS_B-i=+nE%9QpZD5*_nQ0h$_Ifn8TkHoTpmGQGBTgmcEzY}7~_l7 z&_)wHrjGTdmh_*l9a(n`SA(aH+~?GnZr21qlg;}Lw_XADM3YyF}X219r_uDXL@vSC!jO4M$_;2zVlX%vL%J)l7 zbMC^~LS8+2;!EElb!0xZlVo9UK_L&Og;POIb-;#`7 zn;#X-iM!<5-2P&|nQdR=e$%nNvri4Z4Wmylklzg6>zjpc9lbC8VJhX%k!S4bB|V-! zxsZ5l(|~0O@?gH!&RqLK=tbud&f>B7&ugz{EPPY*dNp-KFCkJRzwylif68*sQqL>$ z6y`XnBoK`MJs9d@=c#FZG!yM8O>A=7_%eD|5U>Bbu>}lc^82NYo zaNv<@=M41ZU+n)0>SwBve`Q+`9V}HIYvQd+q3Oj+!MCrub#Ed)&Y3;!34Ydopto1h z&Kje=f|hQ3f$?k1&-Q$>A;ftBt#NN9a;Bdjdw>ON_}#?s3Vte^gbZ3GJWGBj8=Y~v z^wK`Qq>cU`>YD~11e5J zn+3%?j7@%s;E7B%B*1|jk(dO?>S*HDG;3V7)MKH zxD@+I`ulX1v5~Ex(mnz_&a%G&Pxa)g7Y+QfY^}lC3u^?7 zooZDS@|(afi{HrQ7A;&)vtGxsw>R8~B;C zGptb~_@Bk^B=;U1?uVcf%BH8vN|Fdpj(1hEsPR zlp~K}gPXT@@H5x`+Ra~#JXlKpMdU9YyzSauwcMejj3?+WF?1F*%Lvxy^sa}xHtR%qJXr-Qr`=!Hzjy0%ZU9ynd96r7o^^!cfg z+qiG1p7nV;{9EyRqQFgkZpB8C!FQtQ`I6hJ!~RG#0H+$$iPih?fs3RX!$$;he zRc_2Savd=U-F(07eB58H7(ptVUYxQ~-w;iEZHw=y?XMcYp>&%<)8=N!rnla+Rlq|( zvepJWWkA7PxOEbm6Hh;(`CyEkZ!)%=I;Ftim{Mr=J*CjfA4w$x>Rqj4r`OyF{?CUF zz5+em0A0*OAD)XoJO_HY9(@?wUSTg|+tWKk>BrEM1HuO9Cntw@b5G*^jI)mQM0c7< z_BDO|_Q>#UCq6NE?VVtZ=XzK6?kl`DZyM;dc}udn5B7q&4j`~Hd(?-gM*m5W>*!-eZDS5SdSw`^W?95;tux%fn6%?M-=Vu2y4qKHc zcH~^tur2qZy0XC+iDyo6?S}tNo8mjVXF)vkIrvn~4#^tq^RqA71Ksfn!@Jl&A{EaZ zW8jEx3Z98)-mh|Sx4&DicO|EaXRbEyrrUw|GUje6Wvf&M?j^*PX>KJauA_Xp%E7r9 zxZ}uvw9EW%p=>E-(SDA;qI2MZfgAACf)^8eqY>F6AiojqRSa2sC3mOCY6_7nsvJLw zH$_MAT-hnc0f(mx=kJ=r98VcA%JAY|tT&whT-gfm)mY|gKBzAqtT|O51uya7_~#j> zUZy>b^hWXE67k@H;M5>^a0)y)6X& z=VZsBvy5w=z&^a;I%Kg{C6qwb<`Kj<-dP+`R~`<{E*9k`?~!1M)KFX{Pza_=ehiM4*&PK z{C7707km77oK^WI&xT)43dQ~b{KbDiAkXY8qYu38=GCblYl+tNgKpkY<%tg>U;o9; zdslg^Et>b&+`M;`2k&MdqznH&1y80ywiqC=@O3A5}c-S^MQdv6-m83*Ay#;!SRo;@WZ9{Xbh zA7cM0WxPLB%J@r_B42&qr0ma=iudYWt!1azECBCs1pntl2Va36Zh-gBgZIvbPUgUS zug4DQ@Lu^nBw;s~ts;<+`A{9>I-j-3>UREGa~yEbx%Z~bi_ugFO)vT87dr1Pvx#F# zY_kIKackw|nBnW~1H-?+-r>8qs?We{54!QGMEEbfcXVH;Y$0XfWJS5*z1S4ty>$z( zj>yks20kx!;C{!4@ZZDm-$kLy=B-)5riScbOIc1(_!8~N4W8IB7#@5cJb1`Ok_(RD z`y@P_O52j-TbO_G;4j1PY8u6Z*)MT_=|7-mL-u zrc#z=%ChXO@ny{6QpyIY3>@4PSJn)Fmj1qu^4=7ha5!p-K7T#CAz=|_vilcXwvX3bblb@J{vz5u+4SI>Njcb_u6JJmj_`@hK7{W~813eMu)dTx1la^x^| z8dm+I{neo>8IgU^y3CI4)$1;($DLaF65qf zrSpl0L>kZAJ}=Ho9sMj|pS$#v8xKa?J9eJ(=Z=&g{mcyOPQ@6BvUbXs#t+XM@vA!y z1_q};` zujTCXp18Plx=XSQcrV$0QP`Sng%fTe{>U_V6!q3Bt~7ioz+Hk#r>{P!y9Cou7apub zryGGiI&iNQZklGrBE-1s4X+8V80oat=i+c4HoT!v_Xzv`yylThX)6!8qX}EI;%)@^ zZ{fqY|DO)L>gYG|78HYV&d=OC98XCfWBq?BnL9P%8`!2Q^{(P^h>uLIao&B0cSQpR zXNoRXJR9x_ByQYX?n%j8JEteV?~~W)*8htOqqb32?B&r`#ae#m{YMiHJdy11{@+qZ zaz@_8Rz>MM_Z*b%z{KppmsEEJCIK(?Wgjw<@VqtuykMT<1r%GS6(67um|gZa-di{z z(u_RboIfyfihb7hF~nt<@0~j01-iRw zDrI*W9?qWEtTQnf?r`fjGjD+vypt3%{6p=nYw~33wCtotSl2#Iz01>Ouc{=B=IcK5NX96MU0-YW>0qbfQV$k=tJy%v4jn`f4~nz` z!`7$V@4iV{nRPmP7@g~R_J6rQq~mrsPKn^f{U(jKXGGenIQPR{A?*)4cbf#ZANyCd zI4jaooE=gBzT?8~FfnsP8yfdt`L8*ajw_qk5zRGiD$k3Jq4__>I;FL82LB!XHN`&3 zI^vyY+d;k;r$O>l;AT%Yq8;!`u(hdED!GX=)$xHp`i^2%E~R`k<)$z0oUJ48XC^P* z9!2VXSKr)1+5d@aFP_HsQuc6snVDm!A4e&3kML&sxmcr(96S_$ z>Danc?Aw9=Q`knun}6)9W2$wWf$KOf&eAV_&5dS}RlvPo1okd74BT<&KPX#Q)lD^;$lewYH-wPToWY zv$swOZbN?i0&N~g=H%Yz@x*%xHu$Z|9^}cV`Rk`yL*ni-P`m`ilk2FGFLPgXtA6NM z{n51spmPnx)<8`8k{)5*X`p!W+Ak;8kMXM!9&5i`dI>V)w*Xye0>5m2?f6?5doFw99ln(1MWhjAPITTufxgz*lk9^Z;mC|DI{5M@09zybDf=cS$yV9GS}78<$=m z&@*$mPqOLODUq?ro0alYPT}q{+8PqbGP-$n{$y;QX=YEQO8M-kNM@ByHVt2B>G=@)@`r7*4oe$m1P}FMFUZtYs7QqFb%egF0og zo9To0iDbJcI9p-*8jSr)DRj9(Dg9fn6uf+pRC24{5sf=OXA8;xg79vVXoQ#mQetzT_PFMIFQbdHwct z126t{FlQ`Ogj?zVx|Qbzs`z)fVMt&@U2gkJmAUJKp5i{%#-cu+Gtcc)m{ZtinJ@1Z z&0mgdQ$2@o8rg)o`!RU?G;4F=u+QaP-xohg}PjlyZ z1m9A5oKL@@a`yWS4Z^>zb54D=7ve90egc0tpYzRcnK{j|-{7Cb-ZX9X7Y$Xc^VkmT zM&_Y=^Y0wP`G@|t>D=o-H@NRac*o3tm3SWh3e4jSV8F2r%CVoPyaO1}-nPnD@!X3s zyQ#(T#r-YM%lM~j8`WBdrgfuHyhYGW?(BXT`ftlsEl{`Z(H zaej6Lex|q)r^5FO>(QyOCKbFO1b+&YZ6k zye|XZ1uK_Qr@-I4?KQ^sE3m$3u<^O;%Rjtk3^VKslcw7xr26;2i_2i};9eIGrl*Jg z=Te8Zmh!&vKyeyQ#e;89rg$nxYy5%OjFdn;1ez4+$q<0 zw2zAmPXUvnqo?9R7Z(=}f7gqATz3K&y5bYk5Rb!?W8cI0=(}C{-~TNhEOp;G%z2{* z4;G#0-~n`dx&H(nd`@{H9<=c+JLjjP4jz2L^TyNQLGWwvKs-OnxcxQ(7vAF=!i9HC znr^>j(rjCCf&Tx7`2n=&)R-yqf!Tx1X5>CAspdv#hv3M_dvz!L-KymO89NWZx43RH z!N(1`@Ux+*=-JcQkLt#L)Yr?1|C>!fJMGXJ8?p(A`@tpHT6W5gAH4`K;f(kUjjwO? z0BBxK3FmEO5s!2~wEJW1wTas@jJPesu-P6q&6itK%o!Y2zTELUw1}RxpzlHpb#1+>5{qMP-&b{{UU*q25Ui@=N694`HI1}zq(|jYBik`OkdJkyDj##?7*8ZMw;9lFDkuaRS2)#2F7%%V> za~V0|Y4nTGgRPR)ENDLSv0C#1@80Jpbb{uEz2h&6o*f+eJ8jVZ^(X^dnRXGW`e}IM z*T`fo#g{UFEs=| z9h>Ouu!){=M&R%q?Gg82f8xwRTU=Y{L_6q~hMt4|H#_J|?4TtJK8rn;#;*te5Fd)$ z;^aV2nvL9(W!Jefo6qpQbbB6nFIwa5D`(k<8I!mlNGoM6$kZ(jN#Xs-^#@ryOqaj; zN%l2a_H^bza*xi6evLM0Um$Vp*)BHziH~-T+lyt{X8d_)E<9D|b!q;z$E!I|8{&30 ziO*X#XV5jlp7_9@O`2){mej-yOTfiAzM6KNHN@bg;uQ4hV2n=jJ^goa^|WU5W0RwJ z6o*fPd+y%B!uY!F*bCS>v!;@a|0HtwJzUpueUmGT{KM!MKjacF9OL|1)eG|M7k;rH zFi-t5Ypxu7K69x#T}+-MkM_17aPuA~&%vF1d#;^pd0@hoJsR;P88kN6hY-9B!7M0?w}P_8xB zLFTItxl?lUf0?`tv!;^mv23(5j16<1{T;Wgud}8ycBQc4Wz-DXt1)Tc zULKa8&?M6*^m0o1*JVm+-=!wau+Jwwc};bCpY_C;eus8ozxG#rLm#wW?YN&EN!Gn( zKJQEqsd&5fh>O!d$_)L_I0vsv`Hy6p=~>7jn?0efE=~_6U-HXFR2RIsT>Kpv&+38lKDkF!&=JJfM2?Tk>T*7EOw&lW*ig)iGtvm)1^RQMvplbtdDSnrZwNPfwG7`r3O#O}zj|J$99H~5ybjwOB^{g3XA zW=wk9qunu!(dS`iK3K~lqxa+fSEf$BJ=CqEzRE5}^YK5XEYFrK=*$&m?tD0MJ?BFu zlqJr`qvXGq;4@Om+-feO(A`?*BaJR&$YS z59yo>{D~NY4bXwawXE<=z9R(JiL*zWAP3dSm-v78OB1b)`}~XE9&q(!@pZ+i^mf@gVw$=(zv#dQ zzJ?BLJ$kVvT(h~Z;Tnd$N;a2#{@;`C%k0Z~&G{YGoXJSc&Ab9nM$OI0b{dCoGajPP zi#VrN&%mFd&!1@@mskMNORPw?#=_9r9J>Hq7o9$jyf{tHbMpqO9`h)goayGBqdatC znx8Y>ydvfGx6GOB+W+dL)$e=5>s*~!wNi8^C9X~knFbjt1NkU>05PS*0c=lXtKrB; zIktyyi$*sXfJo2fxkTa0Qq{y=*9Y1g#A13-tPF% z<=UUQW!lS)%UYu0wa8W0l=F&yUg+;ip&cKYG~M20 zQt89K-d`^6BL{t6=-b2!r-zC&0_@8~_7`Ucj?K*qY++1Og2>q`(!y^rhO(V$Y3LRvKHIAuJFJN4*F~-! zbtB($Xb^G>cq@NX(U7>U$uA8qJ>5lehNAW>29I+SJk@-KrA{CvcV(3a~#M z?eBN?o5o|01FtBAS7d-UdSCP2-@O<9=zqdn>_bes`M&GN>*xE?708*yOBc*XGtcw6L_2G-8qb_;>_L;{8d~%gt5%A4>E6#tdwq7GTzAlhTf@6 z=T7g3UW0C#V0pH_V+aFtO z;*t~)honOGAw$_i{kfAf4Q6hdD8STLN#t*ibI`#3mNbz_^d`Iz&vP@Y*JR^PI3(YEn zm$9CE7oP`>z3^V^wo2*VhhnRTy@?;%6*V+5$KFo4=$vR%#VNGLv}O0u&?eU&qMor) z+YgLjUir@-G*+YVCsztw+a?ulnZle+X6`02hgUF{m!oT+h(68ny{-4L7r^?q1$--J zA7L*4W*+dd&1-+APX7P#xM#Wc6j$HAmTwE63$Z~|Jl?rJ&rF>m#2R{LG`&7=sn=)H z))g0p%dq`X-n1dUeI{j2Vm#{|KeX$n^s3qO@Db9x<)nJoL#p>SAv2pf6r9q&0>072 zz5c1W-Xz-^tWo{@oc1e@wYyU@8Ax9bc2T zG+Y#Jsquvm6nn!*uk(c2e_L4et>6xmp>XDI4V5@^R#xwd4a|@_D3pn@rTQ`-*fE{OR4YB?%sAKeD&#c zuYn1l>wk7Gd5ZZYIMF`867iy!sNQ{zWfYR@$deC`HbcL%@E0MChuKj%zn zbCNlmR?G2}%f7qT`m`=!uq(ev-B9GS7NL#qwGcedyH(oR2; zv*@?RR(qk>bFcONwMv17ph-oiFK7NHGLM%rpA(qZOQF-_q0^4eP5Zbl=mAQI>9Gr) z=6vEkoD_48ypw1)b(?u-5Bev~$0hv7%n^FfX`;Qs+DWzEs@c=iDrfsSvfIUi1QBpGwK$A62VSMzfmdeuEQBsyf|)1vO76lhdj#(d;r=iep# zTQnqyO}c2&A&-;amwfCCh=GG|a6W$QjFE|-;KSY!_zZ-`NA?4A0raLV(9;0t=*kbi zEXA7HN;{V`?s`Xg;s?b%kL_XgqVDi3=Xxd^82{yXbM|hdcPU4q7+!GlJYD|HP&sh( zbC(xfO&!e@%5H~-_|oi&-0L6u=h3zvfdjkF3`ob*x}qoX9eNRSD-GE(&7R-Q>DOM) zq%|~?IXDE(Xx{Jazs6~%)&-q36BrQ9{Ij7g+4i-hoir185zYL)Da*1aCzb(2qM5%V z4;@yd+GAZ{Xj|<(Bfowh`WiJpJHc3>yJXi2WY;cX)}|Tq<%zQYOgw0Z#v*HW(mnJZ zl1HRF-Q{!IeKv3|nL>2$LHgK9_pToW-OI4A@}8i3&(QZc-E(Z5Gwh}0scpYBX>WT8 zsc>ra*U`R#6@iwOR)F)VBJucnub)c)>O}v#fhXO;movbd9^enOZw_`R;Ec}Zj?+Kw z(Zyxccr4vS`uEoJCuGyaSeyCKz9W)NSwAPn+H5cOg|DO!lI=w&z6~4;$G%5ioaSBa z=G|=alD?Mab;^|SSewARWYg2d+WZR5TRqCempz*~?fg!h=JB1=#@ZzRPSHHpN#L=e zd5WVd8o1u*IJ4{_w7tNe)7B2Wi|*BHeb!9h@0w=pO&- z!#K!Rskc&O1FuO%_pV^xE@%EGGLM%rpA(>aml7w@rF+16J$u)ip>Hj;F@QaDD|i); zwdp^V?txFzKZRXAWeWe%JS{tg?ipIg_eAUbW1u^HQ?#xyt!lPp*|~F}cTXhHyLGt^ zz590F33_+K$@K1ui(LGt---0@CCkbGlze<49ePK6hE941&*`Lh>^CIRyJIfBYkR*k z9H)26JC)vDe+s>;_k9h$d;P-Da_HTYF1=%4&(OK=xak4nL0rSV{-J-q+%`C{w&T2j z#%JY#(iXOys(3 z=N#$o^iecPJkr<@^S<8EH;C2k&_&+sq>H?VOX`I zbn#61TwEUf4rA0w7a1qf&1=Y08>X5x$G(zOI5+I;=pu9|F{bE-Q)yylSN2;o*ssfE zzcq{f)@<-82fWG!zw*GdeCT6uw~Qrc=FE%4>i9X=om^A6MsekGtzmq(;|EBb8WX!_HhzNb$nfj&*~{VHtmS+;HI4WL zHV%f5rQ=63G#Ckx+9U8M#RpJzo*fm8EbyeJ&Gn?#RN^D}jb7G>Hs)t#SuoP}s1;d> zPhhlv*Kq65?ty1rKfoI~OH%J{bo~Gy!AD#BxYxV+@(-+M|4IIV+Q+?$eE9}yA9phM zO|Bo{<=o?oW%jNna9_y%E6l|^ssp}?CjPf8(|o2p;6!}k1~)INJahuGdAZ)r+oe4C ztl(g(oA(dpp$nCs5+i3Lw|%I*V&$ddr{(0C|DuE4;LC1*x)S@8!JNGb-P>6fjMd^r~CrO9<|NFK9f%5DdUHo zX+J@kVlr;lJ{$S|d)J$o&b`gvS%&>+yj=Q{+wrY9sXX6)FkX&M;CeSM`=8*2UAM<3s$+*oha~6!2_6mIi+^QwUqHxn%SB`%(`~4aD zp;eU*yi}c&#Tb_E^t0t?1?nbl5IBUoxA>rIs8-b^K)pd_`?NezUVvS*N=P4 zlxNyMb<54TFvj8^F_$hsm{>lVj1(go3anctMcV6kbVzaO{Z_zu02G%H)_uje2LBgN9rEdo}=SC zKCt$}L+Cr2eLnQQ(vj!d7X#0Vxgq?QeclS@XL;pSj^5!WN00tP=2Je1&UsC__U6y6 zMdEM3>q6(g=JlbR8l}>@b2v%i#pj2xD1H1fZE$_-^{I^OOgVm(t#g8gI#JEm^ zcX&dL%yD>NWnd+72Y+4p+URFryg0D;#pFOY;v9XtdEV%@8Tcy>^b+@u^I?(UuQhy1 zHhdm~M;&$IQ<>mb7JMojK9vLB<$`~C;9)-a2#=a`7IOU8^C`*qio2^b?N7+}j{O~Y zu=RwGy4Qa9H;TK9ejgsO?z&(k0)Ig^9#-PZG_h~|DevaxGY%2U8vY<-A-PP?zkom7 z1#V2{>dV!ei}=)Q-{V@#wU}!qa?yCw!&C8NhesWTH|X6mUv>?#ho;!z)i^hAT8eeY zPT8unkD@cd=W+k+_+aEx_UOVBmB+c__*kx*9E_}DzfRw|Q+Df~ZfRwnZZ$s2T2^pQ z66+F`h4%+bZ{|LX|F7Ql%Z?pdz_)7}(4p@g(Ie27<`1_D_pxjGAI{U&d+h5NyQW8w zoy8BIbn`9p4L`&tO8oF=$tilcSDa9Tgk#6Xv@5&ly!1bH90Z=D;A)atp2 zy#w^%X{-&-1ST|Y;>*Y9d>i<@9yq-Yc)b?5y#{`9HT+^K@O%~g;!6C<9exr2UK=!4 zwlKnZ=E#YOhdrfualKM#T9s1hmtQHgY72XB;>Go(RriLM{=Ii@f6~;+v7b7n zj(Gmj`+m6n=cYW%UUX_XJpbr@cW+;0$}{YTPc4V%j}p&kZhuUf+UVNColNs`l0rAo zNAc)Uz^a)?#&x#fnQ{J+QpWv8 zoV9|t%?B>GvY!{{+4bP6u#CBoAbZ8YL5-DJuLw^Y>$28s{C?}|F*EtMXi0Nd zU?1N2^kY88m-9f0FPsDnr;^tjy@ufOL1Y-o4yWaXRmq{9yfd3QR9{!QIQ}bAtq1kZ zcnrNy_|AG4j_+{4DR}R!2i%O|8y`{jf(y^pr>h5?-9)`@l&y8)`a7qq2V6cty+2YW z_`cGuW7-7lK110?^46Gdur4^=H-OpKsrLe9@~3Q6T~f(>8k3XS51j9&?n=t^53lu* z(IHM_EL)4mhR4b;y12WuCZ1*1#H-DI%wX2U$S}?L)$Q6WStQ@i0~a-4IuG?8=1ljM z?%e$vA5&+KI^SL_-&NK=ip#r-|2p%CkEc72Pm&k!U*=cZ%zN~&A8o#mvc&$So~|za z8$;b$lr^x1G5_u1%*FTg{#18-NRQC|lgEdDf%gyazLD!`vzIfpHTT(!PYL z)WB=%9?>4sYO{y*mGQ~2-v)l2@xjiDaaq3EmE{_{Fn*k^298OVJ8FC`GHm(fD*rO2 zjMb$|f&Fnx=|dIipG`bL$!E#POvq>b%fq(t4cmri*91#fA@BSKTJ!6=&7)V;y*~P% zYpl{UB&#j;tojW5kiCytrESH;nSd_+Q@(%wt?(1Z*N^j5>;Au!*%aSR_N4LikM^Tm zIDl^9pz8-1m)oj|MN$1MI@v5IrqE(+4DaNc!ZnmDpQ|0+g5ur3LMqv#8a+e|87zWc z0lmYp!;_K6=>LmuTq4P1Rm4*bGZrc2Nw@GPbPTJI*&ZR4FQs$|w|cs#E%kJ-*~J>{ zO>_)qNM>H!zEU%4cX;0Vi~M-bq$x`m#J?QB4ak9Q~1o)HI%#g z(_LM|Ao3+!jUs=Ut7{m^z4Q*U86VEQ^bWGaI-h b~^1QNGbN5?-{){=b}@lm_c;Y8kjuX2m!=4i_mmY$0 zKe6drz>_RD-teq=x%3f|)lMqUw`=0%(nmm-bxwry?cVlHZrRhwNR8sJgPe7j(G&c@ zZ{ivuAY~YzUDmb#i5&Tr?@j3%+6iBkOw)xqGv9gXNDIeG`#-IO5%Gv;HF@mhT+^z7Brvv=ws@Kx(NF1*#%YoEt=q+>5c zR{c44#P8idEVR)ZjAcXXkTtgYzg;9*V{5=u7;0tQrg^+Gi#Tg;OLK1MjlOvyKWEa( zeqmmjBj5B^`@rQUU_$ze)+3JGk)?d_dOG=ytfAUgpo{o!??7t->#l)k1zMTYwt;;D zt>}tck3!Fw!`2Q``qS#e7r(3g@qxWGQt2If;G>_;JuoDh!OT4+$cB^9C>mV`1jqrJ&XRyFX$`t^+1Y|vlk(2Xs&+f!oeMG z`aRMXzGGm{ld>StGH3e*><2YXMjmx-g5Pg*^)c|Mos4U9oueSD3PVdw615vBL|Ok=Lm!VL#^?zO}^E z?`;PX>jRgsQ?|&I<=K4`%Yen*ls#g~(7AvY;;ZHGVBr^WVP@VaTmvTk_!FjZCaUsf zXDGfKz1_$n3&m?w8ADSKJG~Uy>7A~gY)Ilfxi~JfVBZN`?PsjxI+n{AbLm+6g2xrF zr^1)h?X!+LeA$WflxyF@d}w~El`<#)t(0-NiS(>f^(ErR@TirKi3g)E8F5Kq?}#pr zz9g;o!nRr1-m(83U15c#FFE@w`jUSaE7*t5-Z`g2_T=!<)p6U7pWDO+W(_Z!zADbT zY#FgM?|o?Sdan7b+wbP8^7&Jt%fkz4yY{Wbhpk+rN8DP}FFCa2TMk}cKzS{D_>K&R z{V(IDeb4jAkMDctxp@Q0lOMo3)}lY>I_AqAa1HCVQFD8nqDDZu7e+S;CGbjYD-$(Kvvp!%v`f;w8|hp5me!=iX>`tMz~9)= zNOX5izInvQ#>bL1sNOH8?`o%H`ybO*_8TGwN51H0WY;OsX~CcBK20AwutAmYu+|!m z&u^an6!{^8AOG+1f@Lp%2zgTW@``OH8zX-gdmh=|eWigOq1v$QA&BmCEtGpB;(Lp%%KjqZ!Q>bZB}OAw6Z z8kvyuo!t7x%po!Z|9si_cRA&m_NQ)nGw*(d7L4rPDW^Tf7#kQs?)r#(>5@KFO8?$h zN`K!~n$3Qx(jLKBgmf0T;+G7ysh=Z5h0hMP(s${Si#&!dZS@!RaO|#lmK-(sLWf@e zN#9{SS(6xl%qQJ5v|m>~5iHEppd{c9h{mJa@ zG3HtLRpRQ|7SKoIhsIn!4&9Q?sCY(PUaB>;EW`d1m=XPSe4g;zVje_4t6f@m$Sv2q z&CHeP=MAPT$NoIAjQPBYvTIElF|-oPnDYlHn{3M1$4M;9cJw)yQr6rnXxb>6Y(b+9 zjK}2}V6>&dkM7x$EyF@fvJA0?4^*OmDUQb)&az(u9yJc~QR~fE$S>8<=&R4*+qrfd zFd+ScbG|y~1T#+JlY-57D*eLw26lq6^GGL)UT;o_7X;mu&(p8#HjTD{pAJ`k`LwQf zbW6Hb%HD5;Z=DpUvSYL8eJ^z6>FoJru;-JB?mP<}dN#WB9Qs9E;yHQ53;O!l#Ij9L z9HD~=wh4!z_i=vO!53yvNjkABTUM{qsY1zZ@rtUZc7!n<734Lr$paH2mP zp--jUtNyx)V*)kaOl%D@YwEB|D8Ns=O7;oReZTAyp!IBZ?zL0y- z>OtIZL?^I@@n5U@+0f}r@F{m?inqvFz5HLqHjpPjK{Z?GCtQSrOnrv zG~F&Em5iish$kJNb1nFA4R~=i_%Rhcxr#leE7@b30{%>9k7*LV_+89i>VC%W2sHa8 zWShD8P~5T$_^L(B0&2xMEsI(k?UYz-u*K(*6^Ufoj>~7NRNMk z_S!U;^8c#g-pC<>b9Ze&Qfqto(!H)dPtzX94_LNnt`1o}uiQKAJo@xLHE zQZyp!^AGUD1N~alzwi2lOtWHd$oEb>us?ec=!BXZ;DMFm@!56^_}b6-N+#5~RM#@j z;)6G8J>TT`Y6lKGe9(!xpKbq;InbQ^fK+zN@(m8qALtVMRl(TbzUjd9ovH^+*S2)5 z>%@8H?Z3(Q{2M+wknhFU`iZ=<)U5X_h<|1|_7T3MZT~JlgZzcX50AkQ_rni$4xMcE zV(`Nwtbeq2G&tb$!$rSFpN?Ia>~R;d$2^DYTCP&A-dz5!ethrzLmJTYN#DH1_~PZ% z^yN9v?X!4FU&dBz{-NYKYyN!uEI035)pK>!Ic{D+dE!yX9_emgKjpFBJO~ZA(~ZG^ z52iUkBr4nFba;rnhj#|=`0wg7S#rK)^j*+ztuOuLze`*O*&$b&JoZk=)AMHNxc=9{ zf1+I-*}kts!?z~rWw%=GXN1aKy~5g09b4tYlZn5;|Nq6aWbrSQ0_Xo!3M}n4Df>92 zqS?m2nKk$EIsZQXLe!hyRtay?ngl;b(--u^Tdh~l3_Zd3;^*^Prtj|gYrya>>fed} zN#p)5^<>Y``6 zhl*Sne}y_`ZW;4h>=mdVTejV^t^T0X291@z(FEMIf7CqyZGN?bcrL=XDpx1Gk~-28 z$?j$ye6)FuY-ZWtbK^iPBTw+uhg9@rIBitGm1$JqK4Z z?CTkqxE=5T=!R^t1Ve&5(O9MNczXsGd4HlAZ|E6ta4Glt<~Y);L~klZD^iiCzup(@ zKcyA(e}(MKzK+_JNDI8!7j@DJ_6kl%M;f!C+0e&EY>6bprK!CJ|9ji%jFVuvpXrbI zSiXIRn^&NEk~5KiQrx^e~kY?(vAbBEFD$9qZbg~Oa659leq@(jnBjB=*X6yA&(o! z<>^lxI&w(oRzNqJc-C6~A4+NK$4cRce=}*iy^-`}I&ykHu$E%tFS)*8yA}k)QO?Nq zVP96@@wIKn_9RX_1dqb~n_WH28+_ly7$F|SsrY_&sl(IM2I1;PXoB$krw1H7e~J8B z^4dRgW5z51$C=ZKt}WGXDHk1(j`9^~Z>OJ=i{mTE)3{4M7LF5}vop5HA#hLn{(ivN zBa}OMjy*nYA46Wfd6)C!-FF4^&5Ykol+80`d3KM)GRF7;%4VA~&dy0JbLEs-EoGf_fUz*=t3i`aOT(Y-W%dcb2VM*J1VfS~@_COE3tQ&l+ACg!{B(t9A!upz6oSQwNj?LY-#c6`hTa*4&w*HE( zwjcR{JqX+Imw;qecrj;)#$|~pd-Df;-HboL+-JUgLVAY1v||}-(zVx32y78wa`igp z;932Bp2_o(#RD@3g$9&k!_OI)(~_;3w|MIF0-j)`FeNy zUc{O5M!v#6?f9HafUS#xv5SDUF~D3UYt;(Ys^zRzx7@0Dfm`l5b4drX+dOnA)!Z-T zzKDDOJ!wm(alZ*$iEX{` zv|{`ASdm@6?%_QPGQyv*F8lPm?3FZFv2xn27@T-B$39B77P0dc+p}OwcyHNN;m@8E zz7pg1^U2|{zWg>HdY7oL_kd&f_?Im8rL@)K1M<-IeU?n~rM4Aw{{;DCeW`Ei?A=D_ z?AR$POKGd*{z}TKd?{~k;5_mz(B5M^{NaNuQ^GA%wP(}K*s~n=b%);du*WlwO{`Z$ z2mkKU)tktZFYdwXl%Hw8@8*Ap{J2d0wwrg2-lsp3J>PQkrYP?*@|NI_%6I6p)$mze}Ao>_O7^ROw+u)5&FLFL0^dg#G9x13QF*+2^d_X7i!jglP`1wf> za~*nQ>WJqZtV9PRo;jD8(kffS_Y%sC{LT0lllMNpf;yks;hXt(A5$;8?MBBgVs(Q2 zz1p&}JN+l#+2!iTX=A$SUoh6w^c@^l$~bj5X>U7C>9k;MAMd#EyZR*fZR?nFIkXV? zKly)OWEkJX;AzV;4ec$Z?ARz^(N|I0au_>Ga5ZpuL9c$fmzo$m*my+>24bTLteNrv z`fG5`(70H6Y(92&=Due|Oy}l`UYmN@7tiAzz03aL6wwKb)YSD<#KQ6SGAqDAYoGW; z$6VYSl8|H@Qkz0V`ZtT$x-M zTimC7(2(U^Apv9&WH{loXpzuNpOOHl_;G7Zal#{SUwYxbF|m8F!200 zLYk)%_EpUDrv9PF;AOLJ>=C-TtJY@D?{(6BUSKZ>a`$nbE z2Ep+4+(&^cJY;5;_alD&5-ZYyTvgr7`852qXzPH`amGeth#lP8eP0si3p=={nU5xL zj5A=~*10C1eZhJj{kEUyFl7Vou3kSbqkY6Q;(2+obMs>_QQ*mHtAtiBr~DxErvFRk zN*%}=Mc;FuHJ{NWbj;|sZwA&%-Wng_{hlRnrQz=bPk{dnA6mco=fpi_ANOsISqtN@ zHLQQ$eByAH*s`HKMk=`R-F?^k4PIYc@f2*L8>WQE-s|H$9p92NPi~txbp4UWE0rIN zo$X0o(CSMIU3B+|_Ex@k4$m#V)X+IRxA48L=*yzMlu%cmPX~w68-2G6G@#D6&7w{T z^Xe~pKTok*T6XzEt^1KgJo6V1K+dT0_y>SXk;7fA&<<~Q+omHOQx29@L=KOth{(50 zxFs5^GaBFGoAEM(Z>#e{&vC|1@Dgj?@Jkp2u4Uf-eXOes&#=}HIm2(A(?`6j@?vYs zao#Pw*cxWw;#T;#^pMBPV_RxiN7v+q(P_n^YdnzyFM1*^j6v(*ap9Jg`S?^#3Ae52 z9cHf#8`3Tz&5!1)jk9F5Un8GnqmKe!zTpLTzIneD1SqcKscG{@|4GX3oK={k!`f z@>&YxmRh9uup99`l;@xSb>EG>)@IG%ybP>8C?~SnD$;qCH0j#9>%#24W+8qvMH0 zdSdXnXn=#qAC`nPADwvoNI>@X3&)k082B(a{Tevk7hCxoE%185;u`R|#=Btg-X+)w z(0>m$`EmRC_lG<9eFyy!UAjf7i{Ce?kI%{I6rUecci36d>o4@*aAxQO z_+q@w^Yl$4d*;m`j8@;`S>8GUqc3k{-k&Khxz~3KB0f?G_j6W`+r{`U;5R7 z+X_MhNXI>27&5WzZYVVL_7>=9e7;P*vkF52@>=0j)0hiCa}lje3d`Q2g!lbd7CGNb zF6ta-#rlZTS=nNkdXb_~H}ai#`a16v^>yB%51n)KSy5;j7XIH~MjwZVI|u?eR(8iZ)upb1fE!W;LcYDONceg)ureSSJW8(5`)ny^7Z2?dc6nKz#&$C6ua=d- z(CxfiTo9fMY-;``xtWiBe})45`z;if+CMSb;oe$Clj(Pv9Y>~Jhdaryg$JhG{aL=UdO%k7Psr2{BrP=Sv?S6oTnLDCPoUg1)#DvjwcHKx43pFV3JRXo)z*zQpt+r{|kyZX;Xysz>o z-xbd=b<$4M(SJu%=UvhW*Hc_~aY@eq1M-G+FZz~zCU(Q)hEP|s#D>JO$5hta9-uPT zN$V5K9#vVcU92+J1p4oS| z-|rJ`&$EkZuhtLuqvMP}7w}x#)e3F!=C&^kBIv?VI|{U|3~ z%Db_@TCpvP@$gIbgeq$-KO@FMtEj@^0r_R8kuo`ahbpwg~Y@fCYdUl&K}KNUqG@+OM?}bEGB;ZDKgt0 zPruf>EUNDhqh?+f`yd~STqjp0MD=Y}MEl)L)c3mAhqUrUgEuW)7GNXg3`!_NP|_)fo* zzq8eqp*z15U60+J^PPo)75=qVIx+G666CA_$XSMdA!h}UvyiQ8Qisepda)!Ut3j{n zqvoN7@n~V};n53=Zpshs*>R!|>YKj(mH)5p7`A3z*9$b)3z*|#_?NK*Le@G6e2C7c z8k$)isYe$6A#t8kM^^U{&Agj^VBwhJB8&e1v9BYO#(0;tICR!|=VsnXbKm(zk|Vpt zXsdKZI(IKFyDcb*%WjU%_wE3;q?`Sr<_150@1n7}_T$X=bl`Ami8ajWXTOkW_ve3g zbgPCZa$ZZx*}<^ZAvZ%G%>3>2#w7O^{N2kswmeeBHw0@Lv|rC#;4Nje(@#5_;3+#9 zQ%|6#kL)9xS;u@fP4>7-^&lw1*Oe5_;L zA6cuM%0rqeH=c%fpzkoWQ|rSFTeen$#hW$Xl8=xF^_=3)^R47NHdq<Q27n46m@yFg`Y zn8U=fCmdK9OxgBRWUcM+>!IYWWghE)5^Q(QY;^eZDCP#5y{{adf|2Wx`Ks9m6up&> z%U|@y30*dNcFBU#KUy)B%SSSLi8o~JdvtVkDfurWv+ePi=R_X;kj9y@6p!AE-p0t? z@aX5@*Bhb#mlJ16H0f)2^g?TIKf|M|HSX}};n2u+|Li9oZRl<~^j19jb>PR~BJC#5 z>>?|XM-zM8@aXM^M=vZoiASf~3;qEwhA#(!d4vC-x%f|hr~I#bU?ZLN*G~8_`@x(o zdA6Zp4?!;s?S+3!7CZt><&YW~ZGjVys+PG~?czZh=|!mz4sK5v9y$Qd4xrM>$azYE z&F7RtV^%4J{;VWD20f8pc+ruLY;bC;-_;lRXAVRUrhNo!W>RkXOB=PPfiB9S*EyjQ zj};3f6F&q!bgC!i&1vPNIf#^fr-Ckf6RmbEa#SS{>v@Y8@GILX#9|1ctUAKU}A1C za`{CS;mKE2hQ}2JV{<)LTdk-3$TCmG?2Vqv*-f4?vn}sMM+&_cADQmGv=m|nx!Udt}k}N0f0cXE@tlF;-1Xh>(f_2>PE z^*8;8^^ryYqxL8NhxJqb!}{HWk$nd`%y>S1pu>!3HR;!l=Tpq}Cf0#P)m=kP#lhHF zz+!DWHVj$=S9(I8LY<+WV`nHIn9XNyp^tB4%lEdQ{ncmze&fK%Vfj~<;J;9@FnaZ3KWccCb{9P2>=AWfb0GO%dxkfFw?%7*g&O%+K@xin?0d_mTJ4hW zZmh-|ewu&4Pa=A6C+*RD75MkB;vLy4Vjmov=Ih;tUU142)CKn={$AGbAbpw+uU}@3 zJgGgxHMRY5>P}}Z5TxBrUv>uzwIFu8L%<|8oDaCCwV%+JpXG=-KCoek|=s-WP+e zR>M=Sg~#~8*ZJUU02)$twtq>=-S@PQ^`)!~3|acOl)Fc_OTQ4!mhF&#WgdAt^O1OD z?%LKJfB*1MRlE(%K}!L$9{SD}U_~)tj_|(Xh{S3t!apMYmfJSj7*Bt{L$>2a_Rv~S z_DWIa*32vM{C`AU@q-WH)q+96qJcwV%t}tViSlL4(bs;X-vy!XQb%FczpcLV@qbm#-Lk|_4B(c5mEp2v>v`#G4NtzIZwPyg=g0XTT3iya!ioQhwXx<; zZ0~^!LYMN*pYZ*}da832e|^)Z%ZGON-^>$zdwVYD2R5^BKKEGiOzovl8x3zD zmE0a+Uo!>Uk>hhV5L?U&seU*O|8wk>Z00qUw9PlbE=wNYHWqps|F72bLF^W+^N8u^ z+B8)1&+CBQ@yRI*ZspsLL@th@&saOx)4gpua;owko;Nb|$e>{OH=Z>6T#vPGK7H0c z?Se>`*jLIsI@WGrZ7KaBaVOhqNe41ls<)tgKxqy3lQsJWg>Hpr*4(Upg0z}PhAhZK zHkkPUYxb2rAI@t-b{)^zq&1%N&^<4iH2xdUF6q{*cG7gxwDT%T+mIpCx=EhEK7zH9 z|4hNTr>#i$8M;qy(>VzK9Nl+myFvH8b>Fqkqx(YLr?f2x-u?Y_pW0Tgd*npu)mncN zc5d)fzaKj}(x}fnfN@_Pm^VB$;I@lme&huJT4@;#QLr|<+q zCBzj@dAHKxzEn@+f?Mwg35=(gN&#q#HKRE#o81;qY^L zf1~7uTfY_h;3FeXtYbb`TR-xK-r>0r9})B1&llRtGxTkr_JvH_pC1@1Ic`N1b7+9a zDh1bv)t<5+H$CAwh5fmu=ucCBHE&YdGxH{Ce*qb5W?I8M&E@Kcp$*DgaPuwg8iNHl z4`_er7Z=CaGh9&ri!QPHU$T$FUS$0*yTlUZIPi3mVBPfVH1e;NJLX7Su5#tg)&48o zeYF5IIHdgyf6aJzPi_n_Q{D5Sywqgt3O_z?X|{G z=Z9w6d(fA>dcNb2CmrV!-c=iZVbXN_36m-&#u)ljNxv%SUpaEv?+t93HhfSPx{P|( zv5s}7y+a0vQjk67AMF_K2mh)_P1_b)p)u6kc%;K=>#d$pkmt9Vr>`5g#2B|)zYY3} z+J|14y|0{xqEPO{p$kS-N~fP~n{hxkc;5?d@ZAgQD#*{Wb-#@K7s#K%{qW6|B8xKe}eTsYj<=+?=WFf5&et|DUsV^9lt+9n2ocR4OC})p9R0te!9-_U) zTU&};{bEU)zf)E*<;W|VzlZ6M(Mi+(E#Axp{+&xlZn4@=s=tw#ijsxm|ET33z|F9A z=n-yUou)DUfIQaHMn+S6693okw&yS~<@cHPyb%A^ZMuu2wrDO8TU%IxdwXF$$;;(sWBU{kRQ! zyCK!uCtcTuGnhl{|2AaNKlx5jcNAki=k^|cqD6bicKeZ~YC~`r5WGA>`(t9Rt2M-ZE!o zhWm|Rc5q}*_Z#yVTgk*vb#>O~4(wh8T#f-gD}mDr;I*7})*lRx+PX1rWZvqZ) zgmzWjqrD&}e+&6RU{PmKI(g~PpwqW;`CIJOX`GpVKQOivzHQ_e&LJOw4tg56t37Hj z`etPJwW*9Xhsh0)E>t{L@`L8((tdx+dl)@vx4Um?-%GtmhE$ZUg4WD0$FBeyrSGTy zIN06_uYQATJmsv1hYd`!u29)n>eY~M>VS8bQV!4Bb}2X@n6F^XoT3_dVjlF~ zxPEWsu^#SuOGl3j5_J%TlK2q(GYgsLg7dIX1Fn^iTsMXBUTwYOoEcRx z61i4&gV?gE?o*CEoT+onsblV+O}LLb_tal#otJOc)uztIggOs9_btV*-gaOjx|lg$ z{DG}dN$=&CnLYIX3%Me|7;JRqiq}3oS+4k)BjERAuBmZ|8DJ5R8rFiI(M6utp47@^UXYaZnJs@ zk6Y?rPwV*cIhFLSg1(gFtNMCpANSD568ak5h&X(c`cIqDJfERk;gDxa7F2t~# zgROdqL89RL9N|M{p)r@`)B?Anl=l z!XC<}39`_a(3>{%jZEzSi5a_LYIxs^SM!f6`PWt91Ah(1VlyU%k8w`ho{8m=R_Kvz zDd97d>bp9&@IKZrN6>%2ON_1MT=#SRfa@Brkz8q%seXr*QnQ?Ru+`ptW8Yrow}yWM z8}{y8|3vTiSFY7uC@9w6g8pwZ*C?)~zPy?!JjHwioUi9zbLB7kW2X&!6FfxaJHAOd z_fGkHojU&}=-U%>*DUGq_GznzE|lE&H)Egf;XZn=bH5zA(dgcnbML=zyQ#Aip0nj% z=YAgd_4oeCz&9#F0`DZc8@;#X{vzgor)tIG3mUJHE;5QC|jv561rJ=3Pdf+0%layyoVO)w>T$mzZn6 z>gJ76-U8rU!FK}U>A>G?r>7NS3U;O!PyTJd$I%k*m=NX=pZTw)P zGl<znNs@QW}Y+cZKN(;*_udqrejZg zDP@Nkg9B|T;e+F`1)YeU`*^NPxGK5IxJGe};2O#`*b{52OAUF@SB~Y~j)9&SxD_d2 zkJXR=K@$E4M_9i^;Sa`kT;rs2?Dj-E4esNAfbGUwbC3Ul?Dve_gX&?=xJmE0?@N9& z_t@}R+&g@ceewAF_#tRLdVuxlU95u--!Ui@^|H=J?v3_=t|S|}ve1IARMTF~ne?5N z(Cfk^YhARGb{gB{MR`k$;TPaf3;0v!=`yo~dp|O7(}P!sf1TR##4_Y8eY>SE?Uc=* zZ+>3OL%KiN24-qc>;R#4TaejbV^Dg~kCG;v` z%=NF1M~Q2MeeX{GBinuLMs6j}yf9d4Z! zjNKLRFKJtOSvW8VdUTgVgGP~uZ&0KJnS-`0%)oB=2f)GG*a#KkW6;syDUHrZ3CS-q zT9*=PY2Ye`-dB=N)_qbq3hcIQ*1azr-9?Ih7;QSc>K0%eVv55j9<{&0A81{1mCl8V zh(ARK`ogVi!0Y0x!)*h-;nwGr=Lxqm7s3(jO9N3KzOTq`;ynxAa_oFG<{cfW;fcr? z8gtps?VgbuIyiYkSUk(IdCInTEg?pIn?uV#dCyt<@2*r_Nw>~j_^Hed#_nnwa(66(N{cRh*?iCbqob>#b{xo@9p#XdNjxxNm%NjitCX=qC5sZX4LZqfG|tXQxg zb>REaHcw>#=hzlA7VHlg9v(f69DI~}ja>&RbRS(YHpti^clAc1Y2H@_8;Xm<9_m(| z#Tf13jkKgwCK$^m1%`ybdl^UKuoxa6#m}gkHl|>UJdSi+=0kaV@ju$jc<;p*X)oiu z)WG{gc}u~+Vb~>`Il)&+G6Z`uG2!J6qF1yxZkwz)tL(Q$uD!{`^f&XMvDlk{f2{c( z8p-!-FEgN3xpenka zcs`fs_s~~Kr_~J3-VaX4>-MLt85~Z`lMd?Oth2%f4)YSUS^f?M` ziO-Ffa~GnQ#K+w6Rd1$_#`XK1<3n6H##Qt2k;)jmn-j|zyPGI`M`hcCu^SW1COY$> z`Adff9$;LDF|G$0*TzoUt1L6FAJg7NjH7I?nxT2lm^o$dQYN0`AA-&YKO{#^qFvB7 zA2jMV%8h&nZnqSl71p|q*m<$TA6YSDm$k>S%Q`WxS$4@uD&TfYWttnQ4{ym0oyZ$}0p<#U&Ia4-Sb)4CDyB|GA3id``;`LVqmJ>_wX<`F6 z5ToDjPwakt%%7?IJ+Xp09|2v*Z_0wP=PEA`bf7!=yz-2|KjCek%^-IFVE9Dc1EbsE zWv#5Ub`ig|xo+O*oy6jc_O}8npi{@^jAbobMJ#`>8(%1S8|OKodpU!3ZV%SGXR_|? z$@;e!ZAwS)loXPUoz{T+-FSQ2Pdbc^Q9FAH@%<#j58U`d_bkF5j&qE%=;tf>!ANiJ z*VJXVugMNZ)^LW=6U332&2}Y-!NpA@Dst(mFdtzPkv1$G&+E-cw+&4ggwJ8 z=+7IAiMNH1ej95{xTVq zUJ_~3M7>LVPdBea-$YJ7RB6S2$ofTR-65y0Rs8v=_D#pLP9<$(9$I+khbHz=mhsWo z-q&vO>WOb$fIs^Tc$NMqdFdbIM~5OClkebVj{PBdic{bBl=X|H{p`!n>D-6483}0 zvuB(1D2}|7X^(4haNbtCs8h(Eh1N-4@n)W*1FabE53A?AZ6E8uXLg-<|LMA^?ca3Y zU(EZRfY!~ zX?ii>L4DCw*@nd7Ax>xG&-#wOoA|8niawTU z9%#pfq>3%_@Aa;!M(?_Z@oIPRe?Rzt2>jQ19f~goW3j>W*o*13KMkD*-`;O>u+iyQHhkbCx}iXRV9nTiFf`8S1sx zeUW5k$bNkh`^z~R=i?pgH0Fy*JNVW+q`j5@Wyd;;^rZKnvexEs9Rs#MS^`bG4!@sb zWT-jir8&U4>SmK4oy_`dF|=>7wGMxU;oJG=*!kA5V~ojR#;5VF^3v#3;B6M^2rKe( z{)vxGYy|R`z(#EbC(h3OU1|uutRRpG?}a`qy`O zJpKDvg++Z1yuUQBHJi0we=9bZ81P5o8T)&%9|8TzVI6-U$r@gT4XS@8wy@kcE%Ake z=)kN4y+Sj$vwsc#9q3`!sSB+k}vF6!LUq4w=8SYO%HX*-jt`7qXg84;!Yd8FFXg6!PkMW8B*YkJsG-nCl z)j##^=FabCqkGR_EtzlZAZ)(947wm%;n~T$FX3JD9_@8v8D!fB-uH#|u4Au|Z!dA% zr9P=`YSUxH$p44Wz{w=T`^twHAL#-9pH52s>8d-}=NXqeB)ERTs)w;N3Ewf!lQxdL z0jWj7(zbloKAyBxJ)@T`WgXMY|E_gK8S5o0Wz07Z^k15%f8pO7@t0U5en^`)ezQEL ze=qCLJ|Fa|pw`+~be1(m`BjF8B<*YHZB02)=Je5DKC`rb0eTVMwSq%J%aNUn-pLOA ziZLx*z}Y#B9{~i6PAJ~5^BuUy*TNb9Vea@p_5*9!uEE5eyXU^`$MB1M61|V$cxjF` zv%R4*JnsSa{X9uC%So|cURZrG>+m{sfCa$mM0f;rAzXZJ;K+zzq@iYTK>ZRO?pSF> zHf+vXukn9%uS2T~UHG|?Zwvr8b*8Ogq0WsXl_`AYxtVzteeFj(Eb{WuO_+S>=q_lo zWbh-5%L?eA{#VW1>pxz;FZwE(wc~Av1|27Dho_209pPSmn>EH-t9A#*po0M4qF%r4 z9}4U<_x14m1NiP9z;8E&Gi9PH^1>}U(4jI$t(!SV1fSgbe52+4-r<7{XNCRj3q%|G zgs%p#tf800K88jWSQ%~p^A;^ew%M2dS)zrmcVug%aTcvqx^ILA@O ztopw=2jV{Ol4aTtP4F$yzv21FT3$P)taj2G%4d0pg!~f%ODdrYGpwwTwG&%fPnVEv zX}`e#z?wl!Z*;te7^61Eq>XVx57?vWee1Xxm*RYE344cSvvp`YKFd3}eAaIts`RzD z6!)R5AbhA8`OAmSp{o~Ni|07;Tw_L$fDR$5m?h{-MN?|)l2_H%C9U$)wy}(F(Ic0{ z=6h=+B@;@P$X~F-#~Kls)tK%AW_AOUpU~e=fsrAi3EwvMl>Z9=e0Emdses`|7Lu z=;a&J&3IULe`(Ch{kcuWtxuqv#nD=k@{0FQQy_F;8;_U;KN@pGdx`Yv@?IIfLb!oq0L- zu2Igqefa<3?d{{Ms;>P1bMFo0CgC{=fe=grDoOAGB!GxY9#A6qPy?1$`yHTk#wa=z z(Xm32AUXzPHK*v%f>WT_A-R@93#eDy0@79-Yz3+H+uD}!pxm3FKmp0&!TWoE_Bkiq zAUK`*&g=Kbe(iJi^Im)HwbovH?Y+IQpyj?>Cizt~+Pqsvo8!CS4cs|fu;7_%k>_@p zx66icuj&jJj{AsHJ*${U@~;494`O>@ct+tc(Ln5f(^|D}7v3|LyUxO`Q~6ff-*|a0 zB9H#_@F#+g!027qW=!|0Th3(801EGJB2KgjTb1fb@J^S`Lf5dPJeRJ`q^!-I%zpM^ zt53H5o1MrK(5V9UDGV&(Wyg}rs7m>K8UH{T*HBk2GKK20%FCS<$YIjs`v0}a4F`dl zp}*{9ur6$fi9iS2OI}E^+x{3+>Up+eU7nzgJVw<&>-XRK|0U zrP*!3#Co#4&F86QUfvJLYw(h_=b{1FkDv?Ey$5l4Z}E{dtt}hI&q}v%BFyNn&|6uI z1MAs_a3|hNA^dW$tZiO9UgI36+Jbyjtu|<1#PsndukAy++67*o*U2N9Noh}l_bt#v z(Marzj-;{PUmmiKRN;?doi=SEx_$7dcp`Re6VVH?HaNcwz*yh*c^P2y*Li1<@~5(g zJ%v37*=e;xr_hZ>UmA2papS_ASMNn{R23+TY7GHx;tg;1bhqZbYVL|n;wz$S=pc`X z&c!p)B|;<3zvdSIEy46i%PrW}+`>8&_bD=w=tQokSDojdAK`mJhB5} zqlC9s%8`9HS(AQ0{_GsvHsSO00dpZ}{mu}-xoNE~~NS$k=d+OPV~P>$*oeG-iq-BkIS-@4D^Vlg*}Wc} zLhZt5uGL-Lt2}w-E@+9$_-M!9Pahxk^8A53%KLl#QtFmmveSg+MBb54zkiE=6kL@Y z^PhY_$hmj%zv3;IbFR+YJ7fG?x@%2ak}la~N}4rmANC;IkcEoxAL}eamiK8Yw)h=s z>b4v^m+@{!hBtUMskc3gdZu=@HkyCd1NW;fU963bp`4#<-rU^c%5ZB5*S@vN4qj5> zybA2xxx>4B^d|+nBmUv`P4`eC%P~(bChj|gn>~IH_8OBqKMWk$H_RQ6gNofrDz9o8SY0|NeatONhbjvQ6rXtQhu@1WhEFps7{zrwmRf&Gc1 z&gQPQ1}ljk{&}EF%OCdoXYm%U%IN0xTkz5PMfV)fHSN=VL~h^I=PwiH_F3UBZmG{- z^3!_#Q=cagM*G*P{k0+5kAC*^{m6{G)!vWH$Uxd5e7KQzK;N@gXixM!vYaQ!H6+P# zJEnKZv8_PR4irj`%Z@zC`Q^>Xh9^I`q3OWvu}!CESj8^9cq@vrlI*>I`Z(4}eOM>) zz6d(hWVtC+t~n(5@;)H(>i>uE@{d@PeH?^yE1*XA3fCizjpHK7hySJFR)FK4neg z*4{sFozbUQZ{O%m+}OZwO#@S`+x9(~x@ONf-f|mAJ%gxciQ+Sf&uWT6BZ7-k)&v)I zT9eq$-jXY~?kHS&R&MPC3_bjWAu5m;R$^s|Lp0=cE01T z74E=J@rOSc=lltJ;=Uiio)c?Mjlr)86YXxNZApeO zj|huhxgZ)}V%Y^pZaZzgSkS$8bnSJz7j^}8{s?>w7eLS9Dd8b?OIsL|I5;``Asod^p6uJ`Oyt?=cUTHm#}M)6Mx0G837;*X>7}0}gU8o+dRsDKu|BmusPAb0-<@;?@pH%OkJ)590!1^R(CR(OGUB$oX%3kL0 z4F*mf=~@9{?Q{*^DR^HD%yQe)=9;zoSi-Ui^XZykav{35xV^2Jk&kKDr_`^ubf+z< z>r?PbI>QXL4Vs^3+SogiYW#kYPV%9rA^Ef^Ab%pX#hmBJiJXxSA3Y@>Sfy&1iw zti2iOW|QB3fX?pl-+;`1hV?nL64}3cBl75b@cmcOp}#t;Ecz|Jzue8*FdCPObKw0O zkZm?(a<-rE^}P8i-QzI6U#aZ<>MNtmqVKbxWA0OicUEDm*qDR7!8yzZ!EX#ai~09G zU@zTl6Jc6MNdLRo3wu%NjIKC0@{AX@TwxQGw{K*T7q(Pk;}n(|dE5(oUSXA%qdQt( zKyMsl{Ho+L-eLKS?>5$QS_jk_J#kiK9(lv;&7f0t^5*3%%eV4cSFw&$oBoBk*rOYl zh1rMD9?d?pcav-OZe}Wud7|)v-k#3Agqi$ieVZBiC;f+W%>B0PyEOQ=XqNbJ(I9)8 zXJeK}y9NK_6UQ|P_wF=(D|5agpSFKlK7E-aANo;k{H(~;_}-pQ$Fh_5bUKknWovGn znL7zwnF!8QgFCl_LtkP~hx_IR-pc*)oy`6pY2R(UyKGd`baziDJ}^UXx{Z|n`YqPW z?tV`1$jXd|oA$9@`NLjU#v7qFkp|o34L@f;M|&q)k0~GdOB;DJwUImZ&+h5;j{NY+ ze`)%Sm;X}oYd^=rUis(X*vsCY&d=wMY+~QyUEypVdph4{{oQ}}XG^9cv%U=eXM;ZCKMMT&vAAyJuyGnqYmHPqqL*MrHby#1&%YOK~K5l5N z73Tk|9F?2=uX43Mc#CqhpYkTY>{kC?+Hn-xp?#gx(2h^NeccPuj?UkaeCgWNX)hw5 z{e~@^-EADF^#*$od^c6{ZW46lrS9ncpcC)oUdJuPJ%ZcNugodMKf1(3tOP_Cb~3ch8Om2g+ZTm7xqT*OnbpK^1+cJ(U^(C8GPnzqeo-D zLEac+ojvbgpd;DF=2i6NAH48zC(h9EMzm;2$j}~_#%SLAG^Wy{F}D+@GPZg&<`%Dv z{BXE+fNKxmkrwnRYTq*s(ssejnaG|o?JG0=D|4>LUqSn?l}~@JQ8;k8+W1+Ki;S;# zZ2oKd5e7duj)Q(wy7Xfca-jBvwJ-KObmX3P8lpX6$zyquP;G7XulBfe_+que+Y{Da zuxQG0@`OX^&zF=%2l8IW**#(9?-SYR<;^B<0rW#O;8l-)h<0esG&Ap3W>hr2$bPNr zYxMGj$)kI@R&Z`BnZ~626~2gWrVU=QA~U#i#|LivPGk43ea14lnOIwCHQdRxT~-6M2=sJ_Ici9ofjY=+k=n)b~^Tnb4In1-ehvJx8YVO4!nEoQ6Jz z@iu42&|P-8KW6}U+Gw2@UO5muP|@ZvvPyuyrA6+cOud`opQF}z1m4Op_uZ)ML!)2j z%us-{LqX0#g^+_%;9;HMWu4(^sqnV5b~A~}>&!m213fziJ^S$W0B3bls4E0rbI-g= z_Aq?882KQ;d|jB`raIA&*tzrAhvR<78s4;cyB>an{_(`7fb;1ql(RPA99ES&N1*)H1I_)ZlmH>2b?wX z3BLl~)ci`&X&Mwf-ru0S+X802Vb&qQUukYpebl2h3U?a2 z@sbUIx#Dk7{Dg87E}OC%onK|fzc58_2teL@`2%H@_}iIe8#m%{_Jw68~({d714>iQ%4kBfCgC2iAn!T z{pyQ3D}o#pqb}t24F(74OE0gl@rm@8vq+M8g3#N~^~oA+Z{gnP?eHh$k?+Q*mpkcc zZ!Mj|n+L0aO?(;o`PUl==gpsYnH}KF{OjbGjkg8;u)@8-}}b;dP1Cv~d+~Idg9?VS{i3arwA@xEx$2 zt~)LRmx>GF5|3Tidt>yiCc)9kDH5o7&jX?6E_2QH*Pv^3hpl4WZXpDZMX@z zakxs{7~E*wNZc^omAJvULAZgqd|W?V4lWbd9hZSi#f5N*1@s?x0(TU52-k``fZK=L zjr#!iK5iH89o%-@o4BpGO}N)_ui)0>*5Y2mt;VgwEypdxEx|3uJ&UWyJ&9Y0n~$4| zdl)wxHxoAlcQ2(&!vsL7aSW0Z^gEjGt^H0zn^`NDqhLFDST^bTETbU zP3y7wZ=AtuX=XkpCIq8Z%V*fz0hCjEpe_i*pXRDKCjdi5)=@szEN>H!blAs;3%~#utpr@uC{GZ$Q&9r?I z?J#Y=i#(%)QOU|3(kHiDeI?CvNqKaKLxQ{t&mvs5=SOl~{(6M=HIIRZQujeG?4ZI% z!}n%ZMB{f=uostRWTXYqF`ehAlH4_#w&KUP_OjKM7I@7R*3$#%i`Eml)H$26oPCAP zJhO(Ne4QJ?rn||N!!Eb)Q9T|kpVT+>Q)#!Y?+C*w#Q8A%XJDwh#?K2+yf@=Nm%haM zr1i>x(+YmIu213IZc6mv1o|>1=zL7NkGJxFH~(*@L_gi05>3355^b3gbWY6-IxW!L zmf56hCEp3cvF9n-5^8%&{h1$hPA|kg8FUV2c8a#?k)m1&Ii z4PLf?m%I04gnKNgjvZkChq2XuiN2-V3HNZnrAy<@TF2nEUl!xuUY5R-%laUxEWX>z z;@gyk@3*Up{x8A}=1q$h%9%p>=6#E`eVYsp4Y|VpbQ?G{FyMUqC6`W^@O4+%hY2_H zZ6M>Lvz@>}@|>Yl=!2$#v;JHR=-cc6N@xIdYU31aPRsdr<@O2KbyP%kX9DAijRtd) zGwQJ^j6qLpq)EeFRCVkXs!r+C6QoO!PW{tazmu#dV}!*BlU^*5!8f=f-IQ!cV$1kf z8-SH;l2oR(Wt@|y{9fD8A-L_-IZf_XAk7Y|lk5$wh~OZ2=`9|=oY=AoV?PoH|HcFg zjO}fl`7x$bnRz#3cQ9qv7V2u0or30M-Z1-zf+zZ+WBJGx(5)EotK-f&9}Y$I+ob*R zFnbAUwNDVsFLzSX%1)a1bd*P9&bgBIe*AWvbFKQJaByrYGN<{bAJdTiOh3|OD{EyH zXdF)`N=<#g9_PepN8_o)hGy#cg=t$>)<-VK~ ziPE2=w^Lm1|0#2xp&n@ByBqxU*v>Wg3phU_eVp`Db+$L7B3j4&>B^Het<3p1^5`8h z1LIa;tiE`7?XR)-_~W*7oGH(yJU_kZ54OtNJRJEx^Ud08V;}5$)yDAc&|4ozg`Wq3na1?NA@=2z&lzNs z&kC2FtP8KGmaJL8UBBTv?kf&foiy#cgSR97aik3LPr~i6rVh39D8r2PXz>Qu_5k}9 z7x%yF*(*U~#B0qw#y5G#US)SBZvkulYv`ZfN8Jy-6uz!{b7^}q@Gxg$SsN=Xi!x<1 z}&*DrGmWp3?QW~nJNE0U@*`;|LEzfAD!m~f4{jeYwk+_lPF2JRybO%vUK z76wGOJ^ju~@gmA-@2e00ua(-*Y(yWg18@p1Yk{nWwCr*l7sc5h<5 z^?h((AUY#ZHfla`d${k#w0-BjqYi@K%c zrahq;XPi}rc$oFavXd)lTkI6~r^xn)J=>&p^A79P5#~-1-Cy^)xci3N<>|H6I#YGY zo6wx@(0F)fVi1lsy6$_>`4M045>Eaf(uwxoCBJ*Q^Huq@eX{&P<<1@Qk#i=>zpdQ4 zT|RpVx5*z}?%X1O3^0+uoO=W1uc~my$zNUJ+<>3K9A)>Ouh@8i5-B6u6z2KMrcIb$^*S~Px|WltzBYnsn^#kuD}<0fA+vd*CA zGd5d9E76-i(}yDLRR>@Di&LU`L+ z&a|H>H}ux;m+(b>Qyp6%mz_-cUfIc`;Ib*d?RtBt*ADYv zH^#m^seFH)Fz=&r@{a-c_4oDdx=hW@siIh2tu+%z<--km4q%oBfZAV0V=N}gq>B(;(I6QbPN zSZemOXOug8DSsbr{U|fB1l?@(-0-CepBspVfH8X1DEe0Jf+&y1=W6bxm4MT|-pOHX z@`=+O67?RgO#W};O48gKZRS=#7nfZbc3rqI#hqK|lg-?c9$GTkk1hM{UwgC)So`g& z!*0sNe<@)9ig;wCjplzxz}}pkKHu#}5&XYQHX6JK^(yh;o|}GB(B8m5{dDX5bI@Mr z=hxbTJ{z3EH|0MbvKxs{9A!+9-{U3?ceqXFZi+y-q>B0M(`h}q8(ThfCrv){=`D2k;jYZ9qm0uS z=q37X)3$Jmy@0kQ!(yJ_7s2j4uy~00WLV7Ne`$!(=A+?D07?^*xGQjk*^Kh|G! zVrqv63&|(3#qf9Pmn@>Tq~WLo+7hz+(FVzZqM@UyyF;Cir`qe2>!iK}>zvoPvonTV zl{YVYnb9{~Lj41&Q*shAa67;Dzo}Qe(DZxH_4d`&sXi~D&TsksC0uj2y}#Fn?dOu~ zrrx?x-+F8WuxHDykFBzdjw^f%ZCGaMt+RFEu4U1_ z=!&liV0RE29w|=yr`}>%@147_xLt*`t5XkqQaqL!=VM&gPU>c)qP;)*9Rt< zKJOVU{t$iZx`}4ob{aB5Wl3{AHX|$axg{y~DaX!_M}`Uhluhf_+M2 zGYp;EVXe!syAkKn<{egP7dxGQX#0jRZ~4TS_leekv!}AVUBTNQFA*1OEjRN@IS7<->e36H12S#> zQdhf$Fxs=h{O|8-f5bm;U%UVJb+h;KFB=HjwjrJ`K-Pt@xNDw!s&WhILz z-h>7p#U-tCwVzCx(fB}@XT`_3zWBAF^XOEr$EI@y=S%+M_t@ItCsv1|&5OHn2G_N_ z`T=$6UWo$EifN1`&xrpSnn`}ytG$!Q83@j-NnQtsr7tz_3WM7t>o}(zNE?eCRo}>b z%Blso^u3p~kFz$`9i;I@nfPyd{AigYd{WudhFX(^YvB*axn;hoIIB9FGeJ)T+jx#CM_dMqA1^qL1>K=;y)05I^+`M#Cyma3XOwkYQuS`m(F{7O- z?+!2Bok|CcPn7`=^Z||XCxCzW32-X`UM3j-1o(2k@mWS7b90^SeG+dlHwT%$wsq2R zXs@km|FhLY=$~vTiYDhS1D>tYfmvzgbHv-Nv@gw(Zq`a0KS!_=e-pe4dj%%-=@r;u zVD|Air?F>XLk)Zplp zBke}#rS5?Zx1x9b-O57y5VYx?Tdu4>RE{nw(7knYx42w)uLat4m;#+O;4UFmJi! zg$?aBFnSpntr*!6Mn#nk{(j^-**-l@8qI^{y}*mL;Ar==I6CbTWEtp(bT8mmBz$Jh zu9}h7M$*5FZJx37mHZI?aE`I_P+IfN7{|b;Sg0tXbM7&4#(XojG4R5CGv7{wA8p{$ zDd>mhZGv;wvHPhL+~^Yuw=UE-X*LmFq;GKL1;V%9$2W9J>1*`O+}C$l-^>H0=MLXV z;vZ^9FpyKfA3h@8)3LjtZ|G

WR!?*l*I-&3Z zoFCiT?9_PBJ;Em;Nb+kiQv^;)O8g+hng#V%QoX% zR>tuq;tv)uKXc?BDdh2ty)OM^j?~^U%Ik{{%u%;|Cz%7uQ_38{(_3@YN9Je=a};Cd zlAgmH`5g(?98s4wNAdO?MVYHBGasN%?0eFX2`Ei-c>5??)arzU(gq zcI4j{&|iVk2S^tlQ0k0VWjWIza<0B||45v9fvPv&T&6;N{;Dw6yhDZ2=IttsG8Yq` z4>rVxO8U{cY&vs3jk&)P7?{f0pW-@T!kFQ~$74FY6n)UZFVN2Ic|3061<d;g@&Z3qc2w9w{ymn_=(4pId8mroqE4L)xef)U-|JAm3JOK z{BxtVMHBT6>qM9Dk?)?^5s=@i;{v;m{wn|3dM>o=9HQd~3T_uaiDA z4hK}aF)pcRy8kt5+{tDt5jV zA^MSUZS8dL+KH#aE3ui-CY~ydzw*>Z;U~0SMh$Y%iP7GNCWdG`DlW2g==1t|A9@2A zo|UH^sX^~C7yoMR3%og=KX6{T7k0zodRMA9$F(G}Uk$bp!9L8@xOy&8Ygv2`Z(~l? zczk+^A#;+VKfEsZZUHrCf4W$mfeHL?clzbhbh+nYyOVasci=31L~UW-19y_Hqm6)1 zeiNVk&pN{Kxoi34FV~~-x$AC!9ePjw4+YZ^8C#bg(dCPOF6Bg5r(wI*Jzw`e1$+iu zwkGVi1n)Fw8b_rsGC$aKxkD?b?-3v8sn_LT!?mc2KD#z0*gD1-Wm|6P7^B-aiQdgE zI5?{P7;r26f%rlQ)cXPT%KGl!uK?_e-OR15Tft|ps>`u;#BZ;TZL-^DP#=N4HyLJ z>!j{|?cV-*#&DmM-(>Xxdn&u`!99GhUo237FXj7IEanVL>8}UDrsv8sG zA)%oqgYvu$_u(6M5cq9_9@{Q<7K6~^ZqQ@ecR||*F~%FXXF+)QLA~TXqFtBwNO|vj zWD|P2K{33idXMDso_&Gf$!OC{pGvWLybYVjD|jEu+^Bw3Wf_6F65X)}{t|tsjD`A_ zMb=n+ZQg>{%nb4_KA5g($c^^DyK>}Afwt&N*09X`ELBf5_2hv+Ur{)%W7hA$qv!f8 z4xUqWo`rgMBZC%Os6d~J>^A(ie^xhb6=CBLmlM$t$WuU`Mn{N|r`nA*N84r20~c%0 z0~c%E=fZ^=i%V5q0vBg}*LKW$RsBowcnBX*!k9dw#w6Ojz;4^Me~`j?6+ivaQGtU2 zkrS7?7KI=uN2c)DkA5@P;jVRL6`eYEuZ&r=+0Sb4Uyhmhag*^0jGgdK&BW~<_`hj; z^Z2NWtl|H5ca}~9S;*drSqMlvsO(DwOoy-u+9;qPsKDTiU?doE6o`;O5YRYEDt2YGJ=BA>d59gKBFuSI?iI)Bq6dymZU>4zwfDg8v+B2&+mQTKk9Stt*TqMmQ!b| zs#C}q<~TO-{24eX``*I%cesgrc5sg|ud9MKMP#~s72sZk<=%!;6ZaxmqbN0TFM>4( z=#s*{2-X)$*(V^l7eW0?P27tBS4&OYi(~(S@a*cJ$L^^{E>-yE9HiP>LtBMb!Ous1 zL%r(v<5Por)%0iI1A1qDTZ`IfG#C7*9HAp>4E{o z6D=Og++buz?#R83{TX7{$DgF+KcBCA1nWKI%Qx~pLOx`^|ID|ZeEH;?XykiPDJ zHRkNyfzNiQE+e{u55gyJ+*p&=1WLCDx=eNP1oGoMk@L|+s>Zi&9Cv)u2><)is z-A{U}kABjpoU@1>Wqzl${wS%cXr5wM<$DaA!a!1BNWYx#MW~|s>Y7Z(K zv8g(8fA@4$EWk@j@Uv=(K~C+!K+B(F8v zxL-~5R@pz4!JbDsQ&DA`Z_Mq#ZtqtXS=P^Mi*49{K=Z6FH|l=c#4A}>98FsTbx2hI zVQ(9KeNfR$*@LIf9PB;-J_gf=a*jamJ&}1Nb$C$54F5r6!CM*ohD7{GjSh7-*phQH zg0!frEbW|wyG)O;{x+^OwQ>>ltga36o;@Dy^^b1ttm=bptTvQ)JMXd1s*&;@&U=Kn zd7-BEzf_e!%bt^G2)`pd-M2mp`(j{DI-Qk?y9Rh=oNrNeNz{ib7^e>biv4;%XWm2> zQ#{|y;2Z69uCT>7y?4Ba_x;+Q z-uI99@=88S>mB`W4Ama424^gDpX|G$Jh_QE%{@Osp0G?=$6T~54*D%jjW0Y{g~ zTnFdm9G&d3Y>QlTGyO4y`_u9)LF$e(h(KJG*Z`!C=!WsCuFQX+q6vvEe>Bg2Y*g8$ z){$YVyf9tX23OI(p_F$s<&8JXlmA;8OU_c&4(V35jCIpG#}mt!NnfqXGv-_{<%y0c zI&u_vwMyX#dym1bAjY$i@5Q&X%)xrPFlw@^Fc4v;4Cfw%$0{ri%*`-uCy8COr+ z0(s{XzsHOZ;PFEK|3P_l!-OdA&!lfP%e=mg72v?!vc59!#Fj4q8>sLXn$TWb!WmG5 zhh}zDGI%NH8KEnq?E40=H^sP5MZTGAezTQtj*|>MPHyD44f$t=LV1Biy z-ItqR;`31E7j{K8zvja~)womNc|416_N)11m%F3I*QVTUCOlR49|!DaPpEdq>E#N> z>vMs!N4;AFKFbKF@nO`+oXuumvY&H7;!`a^Q>tRi`aMP7BU!;-`|RY(>Z}lN&HbU? zeCSP8XwW)4-&ALXdsCPL6W(nG)PN5sc$Z@G3KH0c{?&x7&25iGE@XIR4eQwHtzE>#$5c{oi*j% zMpKTmoBs?^5IZ7J&Mu8&A4LzsH*-q-n z5+`j=zDU;0HDv^Yj^4+3c+7nEsiY|x!AP4=+Eb({-n5Q81(=7~#L2o!6MReV8Lldy z*LT0o;yF+*c$=vItj0P~f0A}^_WX*pku$-yBA3ZGnF?Rj_u={w2$kn}n95W9~$2Pg16UWm8hGI%>`{#SdhN$(PZIx_aH4sWddCN5 z9PeUhy2kvc)Wfan8n54?V4OZw!B~BWf~m}_TbL)qm@h+_H#akXh9D;o_A;N8U5Yxh zx9{pQuITC_(|(T(8wNl8sLjo@R&t)3``*#L6LN$U3Nt}!e^7dOT!9$#JFPS7iNvCb979Z}3V zk$HvSiL}G}Beu^%v4)&aFFNo}9fg?jK)c#szrD-Sk_T z$SJJBRA(`+(_*SdSPBB=t*i`%G^rOXVGS<%O7Ynd({&wfzt0U0Y65z z6B^xK_`NL{+8+t-MuEf8;BpK&&ALiSEO%K2aYtt;{5}MJZ}>MAKUH#0q~4_eizttM zDI>mthHfG(A*|s)-dLA!C^C6RLQR3@YgIRUkMPp7!!_UU=%4Ggtv?+t2x@qSxMPLE z-Xk61_tQ1s*;37Sv>>$M7`&*qQ<%4AHF0lhJ~U-x|7a;6bWv%n|D8{ZJC7&$ z-=~godRNkTTu|77$3i9kcPiPhs&TnhATBx64U9Jn1^xDA{b4n1R?#TKmS*`>BN*6lWm56h3+ zs=afVcE#7(1v{hAu3x&|5RQ#tDC-1v({7LlUF;uLFiU&MHLD7`^h;@}v37Hz4gq@q z@)hbBtJ)B&FCks%|E1q&T+$}-hjBIEbM?%ZMsvQLHDzR(F98|(1O6jp{hosH`ep?a z^mi0Y)HecUuU;(jGPbmhdkOCn>^8M0wYVeuqwFJ%=PzVWd#LeTD9@vf=LmToZalAL zuekQdLWO>0K!ymLm$sUiNkCQo& zste6~%#5>Y^TlWUZv2^Z7jBtP+cBW2hP|Mf$P*dSNNPc|rd2`1#5aJN$HmAV+`Fk{ z4|tfucbkFl`%RpkO1*z6z6YE5evj(cWPKb^_GpOBUHVse(VZ$SK_8`Hyw0W4YHV&( zFjjXfn9RJmyq}bS@4o-$BU*6c2Y9v6P{D~NXlTp56+U$C&Ctvq=Ks4X7_WCxFhTF6 zV4{A#f=PNu1)2Xq(XXCoKgJ`3I|#i9Nd!N8h9;XhQ-~}y*~FO$c^+os1~SoP6HkW8 zvx!T!*rW{^P5jNKDjYb-*voy+KfTEwGwgS3&}07yUWGBv)g9PFu4FnnOPWId1pOfG zIYb{H2CwdZt9dE<4>mF_PUuFi}6LV1iz!V7z`@!8rY>g0Xt7f}+b@ z{!9d8plDQcK?w6R7TOgD4U31CB|y^>;b%$cG9lcpY-4^J{<*~t{4M1^PhT;=l3eA$wZv6J#Png5qLbK$?} zoa9R*pJA(@oya2xiE|PsWd-Q&UwNO>JaCU1-#-X#ohSC%o!k_p=xiUpIorYFy z))#?`SCb{E*C_eNpU5ZQ%X6^8QFKWK6LgD$@p^NHK}*j8ZxuOn$JKoR35`5!^35ib zZ_0X2fNvf%%llEmc>Rci33|1H?8^mW8y>L-Ub&I*0^u=2wn+nP;fdKM4Xlu7lh$>V zXOnM!0k6z9>0u$yG6###h1o z@)+l@+VTG8`Me2=t_~nhKOYAlg^t`u+-qiBU_9?)JhS1W8SqgxZ!&lE6F#b*+cOsj z>?@MFCVIm8aSko(*MJT$xN@LCc&S5=y2=>Kd0-cOK=KJMlKLlrf6}kQFP}2|e363j z`T_;x^!Whly#f?f@UUfm468Uno<480lzy~2*U!=TYdc@F(i@{8ZQ|1HffbcisnR_j>UOc?s=apW!rVH*3u%?OvZ@((cR9>Gc`c2Wa`j zik5>@>oYE<-Rm>{lXkCfLA%#yUZCCUGp|6qU&%G*l7+rGPrGT0(2+3k{ZiU}3LI=n zyQ_E?+$;iKO1u9~T!41}#!SB&?f$FeH);1+p07l^ceKbSa^aO|w@>n!w0jWGQoqa5 z?l(!hly)1szy;bZx~fUL*J~GO_xh{R?qqNg+P(8idSHNd=aDZ!yI)9`{!G+Se1Hl1 za|*KOPr*3-8KBVatF5n}fo_XmhX!O@S!Wmfu&fCL_6L=jWzI3nd>kn9ce*X5(9V9k z5}Tv&b7azu%v*^swIvl!>Gq;yNa=6)+&KHuJ*nJX@w6>02V3IW+FoKObF4iAe0p}` z$X3|%kX3vAh0y-4#F17@B7XP`pB@LG%W}`iLpNX-MqZWwIng7Z7?(KmBhIJ=@)`Pj zoc`miCmEYbf1S`Pc%?|8@pS zzcl1&zPGVKvuS+eZwWKlcfBl#ebTb8bxdE+$E|)|ivC*MTJ|Uk9&i?>6#FYW`Y0TD zpU4mIVMEoX-ZE4t>; z*8!!k?u0)CpwXwIGl)JY=SUOyCP|N?Z)LvUN?!2?XUA_>(z*V7sMpSn`v?1fqO5ee zvxN1-u&b`+B(awB;`z0lA+{vB>+|-MuxS(*5J&&A(x2iEov+q`yH*@)@`><9XueQw{FXJYOP^3>SyfpLwtWD&ks zCw9Va%z0)RuQ=ul2CR)@n;EJ4H%@QQScF6xI5mp63}_U2@y%KwJ_R;dkT2}uQw?#> zZ3&*dFpV>`giMS7a~pB^jts|lq}=1!iaQTW^YKCUBw-gJ+2Sv>gn6Eh(dzQCw}ZzX zxgR(^-%>Xh`?{1XajNWn37%~7XH&L}5&M9^LA!n}bz08e1Bt^QLSq&CX0Eg*QIj=@ zd#R0ei5nZ5X{ST=Z;7<)bn~VUlpH29x(#_P2)Qj7`7H!FE)?CFyA{x{MJCDyS2##oCm7$OHcxt$Ewx>$N|AYh`{4onnzj`L|-3A+Q9sid~3+(;~S$NlA&9%`p=Zxi?-KqapgAR&*&_5C^^vE z^EmQ}%&SVid2CWn?&L{E`%9uzv#QRcDS4_CHG=$-5YMEo~Qk`5n)l z&3=%xXo-xc)Zqc@Bz<2&y3}m}Fka0ebrxWKW`>=9Nz@(0WB=J5dzmusGRv4n9^pa1 z1qQV&BQ8vp5k|b!<$jefS?TZhsW^P^Dww2CP!OBHf^quY3dZVV6%-rN5b%01_&o?b z9|*n=KtCzKp2D5w*KkJYzwCjKy$^C9eSjtk4HcYPGu_d9%_K+fEd`DlE4w**ugY@t z7CN*htDE!HsIE>mUz)-^{#0YWl;PV-=0U1+fS?n^3&XA}6SJ!#YK9|qf#_Q3fpL`+2pPj*6v}(S_Huz*p&~mG7 ziEmZUgO-L{d}kspzO&0h{8g_E@}GVs#ILD-j@Kba`WvP)!8iEBq(Yl)k7A#MP1c8QyQA?`lnY!dg_g}C{|{Y=|dm0XB>l{kpf zl2y-}am>&8-OEOtlDO3u;$9{0xWv7EA?_pMj!4{&3vmaDJ4Bp}h0vb2sDt$L7qm_G zHU#GUmhz;AR}WZr8TZ~L=ytx5`SD))7}g~nx{v(H$XfFJHqYnhdjazudMkdPSM%Cp~kzq!*FMfm2nOsbZ|Rs!hw7iLqQ=)rS#Yv$71r97J4IlOD>o1SI+p5Y@u9|Za^cFuJX`X8>`N=o=h+^d&%Sqi@xC88 zmzFJO8I(^p(vu4Fi4z_p{)6@%l5=XQ`dI4FmHOD_+}L?q=Fof5-!hhCNSA&e3EWR$ zK6%;Sw=?tW-ii(#PEq?@6fL{74B^XC#$djaJq`VVyIPjd@1P(uih{Bi;mUh_!c&E&{de9jw5lpASm@MZ4LbD| z^^>_OMH_&0kIp;a*#`1(HVgR1XW()Zk3AY-Fh?wGZL^?m6h;ps{)GkE&q z{u4TR*pQ(5inZ(jPhTJ4=?|Lyc$ze!Um5W97oeH&^!0mJb@qG{E;NceBuSTYKf6K@o?f~{z;o! z@NmurcJc&xcr586`viFSN0hr1IZ^oHhdirqFXG{e`WfhAOCJ6%`I0Z;;jQTxX=~%S zflm1M`jXY1Jadz1CG^2SgP#fd4a5id^`&J@HOqL0Ji@P^1STn5G|PA}Qk8KH z@j|ysR6c`W&rorR`lAXa>5nKFuRp9{oc@r4v3ik$SLfFwQ^4)=tFOt5??SOroHq9@ z27JuptP-6@)89w_Dzzcg3f^yR6PInyZT^ADKDxvqaAyTI84g)c@q(t5w*e<^d-YPIh1qi{^YH>=pV%s(e=@n{Mq1+9Y3no;GG>`sp%GL?Obds_Ezl4;XJ&Y z_pr!&hR{Gq;Y!h4+R1ZrVZJ z88Z9xv~Ml%C&3R%ljkyQ-p^x;c#UU?ll@oX6aUF9i+9=mLCzP)2RmOF9pW4!IDK@` zIqY!fme?%tZ2YK&T5^{!)MkpzI2pWWlHnYHlC z_`wBz5MI*dJTEDuo-KJv7s~Br@{*1`t8XviC6oB3B`;}5J|Ew-)CWbUx`>zTfS2G4 zM)*Y%`4umLhfK(Vmt=+bqluHW7+}_A_{TBuRQfZDboqBU@Fn$)N&8=xJEaozSCX&H zKmKKw@iTdZ2K#}FT9&bxdm}01cfD)he9=zXunc60u*AbE2@mU^7J z$k>`X-_H1f1N=P2nE1b%wA0WEnIBc?d+2c3`Qmi+x=p9h)6{&Jcb#X=Gx`4N zJWbJ>#eB2zPw0QrPV|8f&An_|ccR7P??N;vST^7P{a7;_fFdFeb9D+s*9v zTWE`^`zcxRQr%DFBbh4!-S0;7``P0q`mx}5H{Q?B?}7{ZUm@SM)c>v{-#GNY3*&3( zfODzWHy7!EkC-u0T-sgk%$o%<^bm{NE0q>)p(DjS-zA41Z zejBl22_A^<7>_(4W4OjF<8$)JT>l6-k}}W%jWU$p7q5>YUhHA-08i65(m!vjxMY2U zg4q8QWSvF91bwZ7vAS2mIQ?}6#V^K{^}eT*w7BDa-kEx|n+1QdR(!@^{odR}=D7Ig zUX`PDmojQa!OI=Ln^N-JGhuU{d-k_;o{P4|!4~z!LfUtrmEC=~w5&}J^1k}v_I~lm5wAts z-SO<%`J+H|!PkM3zJB}mexBDuhsPtAzD3+=TMT>Pv^dTdcs5ysEAsFsQI(;M>iKwh zWzUd=ie59)N6r}g*k(WW{zm4t?Deyg|M4-8Z7#K?7B-bFcd(~>MhW-X3|roQTKSa8VwcfvBl`-G+iKe+!zc!jWqP`%)=`*XrmgtdeRd!8zq z+VgbL)5*VcPwjc8sHkUs(YoYk+)pRp>MrSdwrEz*#-gz)mF}56{Y5i+HWc+vdC@&4 zWeo4lr2SlUSISH7Sv}7cmGo><`Epan@?OjPk)l~C%P6C!=<%M_MI|XOyT_)CbN5Mm z(EZ(@-W9#mX1H_HX1Z@oo8=xiNn3YUp=I6P1wXkr5ta~=%C(q!U~V}+)VRlE3;zDA zPBxE{Ha8Idggn=S^|_5{E8LsXR=U5{k}F=aCPzFpwQ2qgTZmbrQ6sv7m$(l=pElO{uY>j*SUa}@dq}vi}8@nxZd^a|ChWI^V z%wm>vu2RNE?&*{HvWWh#1s_{67E{5yKY@2&TH=Nc1@Df7XIt@)kVYQy2Vcng&T`JD zxj3V|cxL*@;;|2Ht^!x~aj*D(TcpC3)ptF#dAw#TJTP~;BdeV@qd#|9XhGl#^&79n zC|t3DD=pKcK5{2c%YS6Bj`IKFAEMt*q@Am1XMg5b0qs1<+zO^nWz4g=%$FZ%_hECc zPoc~H*^OQ^Qa5(<9J z1E=PJPkbx*w3R#m1eYXV5pvlJT2jSQaG+sM>7Er2^lrj0Zw&h#%T(EuzL|&8fCoa_-}-%lK68oMMb4XyYyTOSl!kq3+gN%M<>%qPG^oJP0c6-(~cyhUZ^!B&|rFF&>{oE5OIiX4}5??GK~{k%qsd418j~Xr@`a znsu?0mV|G2#%{Bj)|Rw->Y3O|oB2PCft1^py?o+R;uiX-b+pm;W3*e^yv`CABV(Z|=pN9JwV-cMlplx_XyGe3fZwc!tH;SmeS|0DeKDD(e*?&Z3Td%0rfzwT}` zf4zH8-?xgs?z^FA*Zd7dyXU`E)V2FtMP0gYDDtFraX*~a)!oH0!tF^dc6);7_B}lR zHFq@k9j(jM>clU|P;(r|Lufdhiwti1A#}@=?!{xohdiL(pkBezjOps-G zh>1gATB0g6Eeiih+%JNE>iZb)=jfO5Wd`4p^UK-fJp|4B!PZKD9$s}S^>ugr{EhB} z`ER?IE?DWFL3nAwD)%fx<$~4j#|e9TzE!k|@HfXD?u1@%71i^+*D=~Xn&;l~?ik~K z5jc~OlyVomptE}~p`MVG-r2orOy0V8C(L_e`hpGcl@0DsIFmL}`Um@%;E>ei5Fs3X zvWNOkPkYrJYz?Uh2@0u*qh3q+-xB!9lC<`bU(eT?rZ4!T`?Y9oLnb(K2)eO_ay0zW zUTY1G_+ASt>;Mf5ozZ^AYtXf-k}^Za7!4=d0ptM&8OiU1y0J_7Hs? zWsQmuy!rCz<9m)vrK!$U}7&`at-P(2sS@ zHQ{p>=Ip8A{h?#|th4S|BXq5Khi%vKKEcqs5NKW~GO3;YDPicg;pB-RZ!6YjTeIgN z(t8Y^Fj=!kIH7-?EI}M=Dck)qcbOcqMds8(v!)_5-om`A0;g9pUZu=+ksrUewC;G^ zlppJmAE$!%)4=WHtiew+Wyl!%_*08z*z@%HbZg|WQ}D3U^l4A}dHW~Z`w86fm+dlN zqh5fHe~jEEai1VF{=gYo^^AP@1AA^0kRf-Ho`4LwOCU1jZh^>+dw@%rACkWpA)dJ> zdg4ClxXc$PeU@a4sW`0g1Ak>*wNcBou!!%mg^7%pMw=bv+Y1iVGya^LN3Lw8kKxX@ zz1&} z=MTvXc=i{4id=d)d7=B0q`~e_kwgE54ElP?LiazBL;r!?T8YehIBA^wAIPEqLls~vjQtsyc81F}mK0r48SI-}d{*B!F=ag66FQ?q&UXb=1_f3QQR4h!p z-(8tj;(j^parYU4ooCmgkqx%c&7cmX!9MCG2ONVHLDx z0sZm~ZU2@&sO0Y8e_B!_q&=TXySN+glkAC+3sQd*`3kb?wS_k|)g!wnEj;9Y8CkW& zlvU3v8HDnW;ZxX>5>aY%)b8*0dq>-GeMP_U$QYG!rZ=B4x*vM;6!=xky!nxNvzU1! zb3*3Lm3V=gFW8r5zR0}VujW;B$2#U!`kV@70tX7*oJwvfO)l;c@@h1khmb`)j`gKMKhOT0&&c*gR+O6 zc^S>ziX}dpxfLgnxg~Qbp63^sU%8aA6uy)M9}@ljryZ7cf%!Gv8dp)mJSdW<_#>Rm{E7xI^T5lOm`~rEdX<&99O9&24(45-!fpLdAjj=z3{&U7=e}nC`|f)e zu5_0YhAdp=oqL&FDfK%sy|0eu7Hag}--KpD>Z5t<(&o&2qh#S`_gF%IlSgeqrdYvR4Ln)ZqY?Z6jo{K)=$_PX zZ2J4|^;T=euR^UADb%rq|CV%|7+I2jZR9w3)!2pax}%X_g;$+0aVLX0zug)gFRszTWF9Xk3?WP*=**J}!YzcUgg-4j<$jiM8{uI>yIze& z?R%XqnxFK9yM3>wqF%ku6}^_U)IC3GAn!H2R~H>}JmK!utGVc=UO&V8Uva;dG@AFL zy#HAAP0CXDO}%Q1@_HSi&zHOF9rwDs_WG%)bFULc(T?YMK2E-4MI-nw8d>V2wENu4 z2Dh#FIPKT&-D!`xccwk$9+0T58&P0c_wvHi?h?XS0^gGY?Z{sz zxVPfR=281;x3s;Hb({AXp8?n#mZh$DSEjz@KEe2Gw^$?E(XRHiPue!0x&Be=eUW1y zFWoc0@J7bzF#kI2?jNni$aslfdIFov7Ohn+dnQInzo%Q%&`V!*EF0JM{CMGmP>t6B z=K8r3H$3!Kc-a80AW8hSHv2kXuTztN?BH!ixB8x3Z z&VV;kkIVUe35^r|CQc7lkUJKEat320_Ra1DIWN;_bL2D@+Pr>Sa*jWXdp$@OUs7@& z>`9($4g`C{N|P&VUJqeUQ>eG9d=clsh+iFM_=8y;?)?XQr6!CV?EfMDfA9Mx6`w>y zY^lm;rz7m2%0JPp>y5pg|B5yJz)<#KbTwmtKIdU$jQzs9i8J=hU%X%V1EtqV+=czZ z?-DnJc2u**8Swpr{qH|~zg$lnWRJjR{w;fkKcj7z+q>U#&oFwO`eqH^$hUt48uen& zp^A&sS1K5*FIP~`4&Dwf+y*WT2N&Fo@vY#(E$4kJvnLdJ%mz*bfg8cdV^`iUd;&XQ zg=UNJBKK9JiyncOyz0=pi!6K87L@ZQe8r7CmH}T`Zt{$DYn0*{&Diof!810(Gd938 z>fkLK;2Hh3sGMKJFR&MUaS}S;i9I&^KHlC>;Qr6I_j?u`H~^lA9EJ?F^JDnLA@~Hc z)XqcjhePm(BzQ(BbSMfQae};2@Q6C#bMS>I_{2$`rMy$~s`M&#DA>$FifAzP=nHDs_T+9&$TUU)<=XzjPu!Oy>?%to`WXQ}IPY?%k3eJ`hN zbpMO`e3<7i+LG6x=+BnCvqfw28lm}%+@CFYx9IZ)9~6c4^%mLtt|@9x&xZax-9N)8 z?oFHH{u!Mvjqh_8{tV6e8G7(D{QqZUj7{#B7jAM-LPpO-$3D)#zDIU*LB|VhQN!}k zN%v>o+j#o<=N-Q-=``aw?LxP`eY_<~`7?cXb_BFMs^TkTgP+4baUfszl(vaoaWDT{ zZpv!1HZQW;0b8_woOwMOS*16&MSraeS?w^g+F@k1!^mofk=0HhtDQhrJAtfr0$HsN zS*;FPtqxhOPRVK?BikHGenQD=pCGG=-1b`1AonMxtoFy01sBO`pZp)lY6GF)zaN}c zF$kG$9J1Qo$ZD*SuY3RDi)FP*&~cH~j?XpdI5y|nt;d^3$$mO%^J)0&Cg}ICq2KS) z_RaLcaqL(&t1Uv>^9i!rYe}>N`u#-euOoj8{eGgb2eR4$#_E8367>5x_xuX|J`O*~ zlk(vYHuSq@OIq#eZqGY@KQ1dUMlo9$BazkafqtjsH{XuUBAju!j`6!5`^^p5Z#rSW z$-;gUgZ-wEIg^3?WhwZd*WS*eHft{B-+OJBVk&fTi zD(druB|HLMv~Yg5Yu2IId5-qmNejpRLfUI)T3L4ftV89b?I$e~`wRH?@%gm=Mw(Td zncfEb3p(_n^JxV}8f%2<3HZ)N2mCd*4*6eM_5dR-oHU14T!zjad_Ha9tb_coU8FYi zOZq_cIz>llOFnkPF!}Kwl{v z1@ry}WWU|eC;ZZ2cWI^XN&TaHW9n<}zme|+><|IHB$qLN@1`?F@849fY}xPMbh7A! zn@%a)_1z0L7u~bKS9DF^KNqF;{h+8My{~(0dO!Cp^ehK9>{-m;z08j#gz1EOLejzw z%;yctMkD@#Meozp_}FVF(nqp4vNdg#adj78)8+;(a+uJUeY98Le%dSW0PPickoF2Z zM0=5?hs*O}+8jljg$BmZRK)>=oj!A4L=d-v1clh|3Ly-IHWztGo*?b5gI3A^^DHri{WETBJn;Op4%>{jV&C?32@x7)RWu6Q0sRtD z@g+8dEn-6eN3M3Jp|;~dc-H{rnS%4@8`7g)b?o&Ug{^vo*c@8=EkWj8uT@>})hg>( zvR6lZud<-Deu))dWVbsj<$Oih)2@amITI50B>tJtx%b@6I{6OzdD7x^{IUjd)_c*2 zNY10L;C$KSfUiit)!65Av<*C(Z(GJ&D5p~l=PjsLOFt>iHo4m%0{+|zKHVCA9f|!T z3jaGc&RkqL1HF!Kx-2Y% zGkTVwIy*ki(>1LwIez)FYHPBRpAy6VuH>hjj)wg7JHD6wFILK1itK0jDnt(C41}Dw znrF&|vKK*kOFQZ~^^&*$Ol?Ej-Y@tL_5-A>M2`H>YaK64eX|4aIpI!`9A+?#PF z`DqJh;3j|zl2=nQ5$msJd+!8x=f7g3AF1NRukmpD-_1DO%6Qy@yfh5|H$jG+Y_l>> zHsXUw4<=s-NiBrq09bc>M}&m zo}+8dRrN~Nd-3lD%zxIC*Q-6Ulq2QI{wDT|?d)HLUv^|aS)cAoUcp&uucr2Q;AfL| zNm@tJWbemT{>OgU-S(iZKMfJtlYUPlZAiB+9?jzN$5}LgBJUEP1oQ)u1&sf*pXqe8 zF`oNd?2UEGxUOz-zJlvCwq^Q^&;YBjn!Q1U*EEf zb!Hh~l1KJ0eGWA6Do{p_D&rpFWpBXyK(XB^nP0`>^G88^{wNr)zolTDzFtB6e<&#D z{QlGDiTFEN5^d~#J>JIx4YVS^+mPRbrZ(Om4336?qoLp^XSYhi@P%#l$iCN;COwh) zD)>1ht1ve}6T{d`n!(;h8M9n)F&Dp#vd`4Q{=Lrv^zGNq%A{nMUvw7xA2^3c8O~id zcyg|HIh|pGO?k)36IZsh`Y^v71 z#uf{$jLRuSp0Q&~2{+e*YT=VVLLZb2WeKZ5A5iN-6J$Mzb)Zt@CKs~Va*I|x)w0}i zE9*d|$V)4bmzIYS*M&INf$ET3WX*9GIG1xhcC$IynD8a7)?TP@8N8Zxp!IX%)7Qc0 zujhFc{Jb39MxI}Vryr#*H}L#%mmU2Ce$vB$pXM5HXFmh($~WNdfd<@jvjM-p)qsDy z-GF=V*wOD8;~7}r`x0O3rK01+YNOs{&FrDsVU^L?cj}QDlDgMpH?CLf`bpT1lG3vx zldRgg@tjetVz2QS(q6hWJ%n`DovPShyqUDWUz#3HdKc2^?*`J6k(*`T!1?kcN$0+h zDsZefX?I20B$eYJlrxD%sVQjx6(J2CZHr`oqkae$v z?lFX!g!i!dRub+e%qD!uevRdXA%rP}p*`!03JF7#E8G>pH{?C}Y4_0NVeZ^sbwx=M z*pE?9{9eKt#|!R+lsnz;68|D05&PH-;9Y@ytey4lQCE~qkn!;o{~aNheH*ftcGoo* zuci6!Vl8dryf*?e?g(VuO@uGer&eMY6J6wK=F)g@Et54dXg$6OI(p)FUE_>Bt@)hr z-7t%m+j6c7T@m^?J<6cpms>Bd$r=EiD}dhhN9N0)D*Nz9M9KYW_!PvhFp0jEd9#&q zn#f+5efSiU_w9_=B>FqjT-&_O67fadlv&TQTwcLi=K6LYZ0{$q{Z@PsF|OC5_gNU% z4gwk1&H@?NE%%%R_c$W1~)_0;pne$>tkhRY)R%88hIc;~*_Epwa5n>lO$k^Y( zH+NExMC!H~Is7GL@EaHY;7%d@lW{#qxCy;Fo$wj`SwqM{uZ|%64|4TU!k6r8IZpU8 z`B`^VuhT`Dgnh|#-P@A$u|J>2{!EZ{4d7UU?7dlrZqld6>7rDEtba5S|F1wkY}FqS z{|ADsf6NBnO9-rgWTtjSzv=2OA-qhOI5}@!$wTwrNJ77_N6+6&$mMxHwrUys1GMv2 zbms%;y$7)&JPNMY+qU-GMx7f7nMp6YGtrwfQ@ck_#5OH!CwmF=qqVwy(9{Fwd@bYL zK&fV}ZO1xl(2T4Z*fKoWIMjT7jrq#`PuMM6BcomZUa>~07(+ON{OQf=?_Jw5-+LB2=GkdM%8rT8{v0ld_X=Goy9d8YDP{dw)ASCV z?WjC54PWzxVcx2aW&LXKS7|4%dJ^Y6mx%~eS4@R5+y+grxeI!uu;52yf9pmpK zy!cD;;I7N#0XB*CF2Rw@;RAN(F8`PC;o}YlK4jAdg%6Z1I>V*-aLpC)A@nl%VEu*o zFvG+L8~^^L_)uwnbF#I8hjqXJJ{(hVar%!6#_C5D{3ZBcK~J+H1KW^+gOGu_3#J5J ztz7(7pkv+F&GRBzP8Az zIV*y>7Xlh)<9yDfot>Fa+~J4}YK7-C@K1Qr&df(biO1&*?=J2U6uJCc(FsWiS0$=P5V$Al}E$32!T9>>Kt1T?*w^g%-h&|EE-#7 zTjVhEmbKA%m*>X&-=}>R+G(Y|HgJr4hin!PJ{U)&_q7xc0iXU3z0c;%TQz(3*%O|F z-DQ2!>P%1k@QjML`^$d&SiKi**f>x8V8!X(dET_WfM@)F^87qDq*3V1f3s_wu<^xZ zP?oi9uJWyV3vZlZUu?_bpoKmD41zoM@RdV9XTg?VTp?{lq;v6b~%8}AEZ8rZ8q>CLaT;`#PS0+C0 z)PiiDx%itEd|XC8858N#zpFCi^lud8EV_c+M^iF@F)Uy#`!lBb;Oxh0zfwk+wz0Xu zvg>l=EWQavk51&@_4pT&wIqBWw6S0p%%a@iGS|_qnLF|N8f#f@c60N))#3IEdq0;? zv$ScW(wn*C2pmYDPtl>&92Z$KP)-cxxG00YrF9xQ*k&vI|G@Iz_Djlw&h+>`QmfmB z9yY=1%5BAWf)5$^CJE#Dd#hH+KCw|NNDJm&>RpDfLMzW(NVAYOA>5VwbHs(Tvph?h zod1-0b7>#ZFUl?!m{V8F6CbMWlCes3mF@1OZU5;MdbrR@x%a%(8eh8~8~bSTzTA+W^bCXM2Tlq%vYt&Q9Sc5DZes02^2mIU)}IaXq<9fV1|ZTRAhoc7&HLpdH~4xQLVV@x4jM zf3%Ulo%CmnbmAnvbPVbE!!*)&IR5gdvJpA<6%R`p%md>eWtQR{tW9CR;eOR&zA1LC zhuD%1mzPDFZQ;A*E_?Bz*Tnt`Ia?!l3W{$TIcrlD8tL_w zxP0$TbNM!o?vJfA6#gy#)nvbc1KkRrx6REa75>b8-^ZNMU%%P80l&~4OC2c_XX}*{W_v5MN;g!t zE%jDrH*s%-MXR#|wJ=xXWD^cCq0p7RLcIINaW4gT0@QNe{l`zO=u4V^C3vuAwq>Nku|Ev+=@$F2 zJnWER*xqVb&lqA^!<+|V>FpFJ<1s{QoMe0P*@2FO?qMiv_F z^6yPCWaWD#ZW?iqx%|7#IP|_H+$rF%n(p%dj~T~)mL*m96F1!D|J00gOWY*lhPnLP z&A6K-ZanQ8=kgyJN&Ba9MxXlxs|$HXj;p?(bCLolx%`s_jwfAU0b|B}f>Jl|opt8KYILIa0duH99EJIH5h?Jk_Op%&KexT|dka_Xcp zk=%=D^H?mQp7!0j6S1ke`}hd0_yBFF?#MZO`lXuw z$6VYiV$Z4VILP~}TkVxMPqkOpfUh-qj9;<6Qu^)3PA>nX*{v$8%Y!|41Jln%d!haQ zK0Lp=!?H`-vkzW!Z@PhdAJRtAImAy;AZO8L6G%_M;3EenBSnqE6sn3y#E;SvKI0(aaGh`=-tC+ z+){~S&WPNUW75s%B#t)Ae@;ee{yl94uec9UAZ;Hakowz!YqGcpB*o>|V_p8bf&y-WN{h(XphrJRa3Qdwbw)E3U6@!2%i5qTl3d_Vey^#%HlhctND*~1NZxB z{-Lv@E05$;mx0>3r}+1g>%+Vcg0Hn*!o7{LR-Zr7>O02&j-^~#@H<~XIfFS5iy!Ig zVU#u8;zQ3|Z-D zK92YC7GKpKn-r-xHFpv>Df%t`j*Dt;y7vkv+g46j|a!@BE0mgth` zm{BhO!AbZQd$D;bxjdKC#|oF5c8 z(&c|ZAox33Aa(wYg4{czV3PhTAij4J|Et_p9;W23%lSDKS@tY?ovi78Q10^in4eX| zxz#`L|Dj;KJ`{Lnx{N_Z&eIaS@nQ1oCYHX?M{>z$P^nZ%xPhkC3WF5)3-pq&mGD`YE;?|mR^N`W2 zFO{!WM0qmhK6CnHBK@Rrg|@7aF{Ev;2&Anq3#9GK1P*ukUlcgZ<)0%ES!9X8Y54yU z$Q)m+;^Xya6pYiKRxnn7O2GuZxjgK@%G=UE*2~Cjf>*y&?PjO4`sz;#2e(;st;2 zAx>l`k&{}Pabt)R8c?;?<;%7hcs`Oi>0@czO?Nu?K_^$fqv+M05Zd8U1A^%ieqAO33mG`?=}?K9<` z{m4Dd$RDNKf*U$ElbMd|QL;LvIs{8xPZiH^BQ>I_UEP2w+- zd-7;IxE7Fm4%>`AHsl`ed!i19CW+kR&_(9iIn9uH>XCW!0x}P@=yAqb^j{_O0MA0- zgeEAN=L+NCP%@80pUSt4yOMbXHd0=%5Up3sb{(cX!4avyjBNw`C^)P75t%J8cKdfA z|H$|yNfi;^O1Mbv#~KkBhz2z>^;0Ay|b}ri|55!LySk}i+X;fjYsfaJwF};4d23gh32{L zcLp6yp}cb?+t&OXVvKzgQ1+%aO+&}J!v0jTUHyQ3ehS-F1GX#KVT0`JWWf`BEA#f4 zg30=i3dZUc83x^uc)=fZti@I*_h>Ud##(2>`8%NY6W2ravG}JEAK4CF_Mi%X7yj_I zDl_1_CP|ky@Q+nmqW;(OY21SlYTV89p-PL>KOwE@xYc_)9l1vKs&W?PG<5628hL`= zP52Es^&xp5g@#EzFSka{{jaqXgzuC0PV<{Tng4!6L1cd5<@TpGO&fY)kLnI>*Qwza z)}pMecWNHdJ>w=0@Wn^o}Pk?TBnQI6Ogq<^z`R{E(W%Y%*2kL`A6z{fs*_&BG< zI$&5&kE}yIfPH>Ed*Gaw=a-plkBc+2JA2xxdi7`hB%gIY%{)^kYp@p9Qt-h)+C~~{ zVZ{5QZCM0X-B z(@cASIFZv{1D7R_;LhE|eeclfridIt`@}}2S+IXn{uy+kZL~4-?%tlCf(;v05_4p) zS#E?`?kKZdp?B|?aX36sYwV)CsQVL1lY26HD*Q;&Zz5g(-&g+6`pq`}FEm~13*90u zR_|fvx!ips&E>}WisY}6xk=l*l3&&_q#Tj^o;T}q9r5B{>{ZrNjQVgF74;FBU2r@_ z{U=FJ27Vu|)m3rkK+4@?mYYhv^jm+*6uD6Tl}McY>(%f;nd~Q_%%4dYxiLhQnWzUz znXEmDKNKnRL$l0Kv&>&n<`lC`d`=i;MqXTIjaguo z^J+q%Oyj=nEvn2!{ZmzDg8ng(wf(E!k<(mm-z7e@1n*Bm7ZnaVMdpC7*c|u*$JeAP zFIL~EAbWv%7@;`Gxv!1l-9*;oG(@uFJAzpIMW?s49Me2{ip8o+7+stMHP8H;P{qwU%$!T)9J#=>u!EVxR2KTCMbd zHFc3SQWyV2_iH0-81~MKmZjr2r0VTO+*xul@34<&Ez%lPY-hbE>^^x8VLk9a^IfrC z8~b=x{Fi#U{(sbK@~osbB1>{EhWf}|D;L-MYWHM?9b7Qt1iU9Y65eCY+uI!F+SOd1 zx=Z$%T}lr!m`lOjIVx#ha_`McVY{!-`o;@a?(WhIqgu+F@waCxI zpa(zNf^+ES=iU#AJ_>vN%OyVa)8|J?SKEP0`D4hTyGS%cx*M^ z2wahm|nGAG3+w|}a5aoU2SEC-Wt@3$8m{@5 z^FEPxvGZsa!8y%;kG$Wno_$pXK_1x;^QF!3OUu4|*3si@f8X`@%c@IVe(o5nt)91N zS(7bW7aP$@&VZfbS&dN%d!1&^+FWp^xmPLuZG|swvD#`^vq$x4H@El6*|y3x1i|yR zp1T{XPc@HfZ%M2O$M?lC%D&bTSrKLn`6AR3TVc0_)@oUU8nvu}jq@^xHqP_h+_=g! zq;Uv*sXD^tvFBgs37&al1M`OGzMhct&#gS6Jpb5B?%NfAi|pG(mlNCVS@zt{?a;hb z{3%w`PK|u%7%$XThe1jYjO)aYBm;qPI!hw+~-4QkD0Rl$WGWG|T&mu@ic((oPv^W6iW$ z(uBqv-=(PUMw)5pY-&&E8Sw3!uK0f@_$LK>CjhI_)xlKQydU9v z`S)4uQQ7=k`b5&dyugC6p1l%2(-^w%8(z>FS=R)wf zkmn!4@2A1`=5qpa>kkPG{G68dF=RS+lt-w%_m$9 zotm3Bb+Wj8i<#qH?*3C{=LF5W{_&_vbjuj4ExMv@IR4=6T1>{gxqUU#Gv>|fTTI^0 zq{YN(-e|k#XT3b;Hqx{*@|4Z%dpqf0jftwn=e0MA?=`;B`2OF>cKfq}xUVTl&4=n; z%}WDv#BnP)kgj##gHAWy>hzpqE*_;n&wNwhV@d><*svHIU}{QhbO1Ci+g>- z*ADHn{JWgi)FIP#E;D;VT4ttp?l0(RLsz!VubJ7WVM2()v5k-1 zh~1q$Qijj~Nf-RimOSBRo||0$cQ46vgf`VaM!)m_V;#`x$gkGUW|nN0y(Y5W5ANJP zr7L%poBcDTtLA@$F?n*sp!0E0ZV2%^zbT;4!;F6Y#^$eqZ_wAiA4lVBe`a=nIG|EerwKLi_)E?f7ibaRi!beBJt{#dB;#h!2Df2+UPzNP+GsxNy~*G_`J3%{3s z%eL4nvZ?!;nI#Q&_OA;cv-7-$bHkqjCj;?i>A}`hmye&*`V__yd=R`i4c!o1*w4^_ zlcf?LSl5gfKELD?^hfBSfkW}|M!~5>eIfHe*2snDvKLa;Z-|f8=SqBm5g$wZEEON8 zmq`3zBVNv;KBD3s`fnwEm=W(FzKHizS;jx*S=PGcAM(xpyq~6T1*hdX!1LotL*H0Z zCb~)I>pf}mjNF8bwLYNtS1oj%)4rWJ$G-RQZj0sqS2X*?w0+D|WtU{Ut3s_)tEXGM zVpkM8ciz@Fn-1>^O;3dZU9Ya-sE z_Xe&w*PIwep9{@=;ITQ+)iURV-kqn9`I_e_baNeaa~;pec>WT4`X$fDnWM*nLOY|O zneC|y^vzdB-?Y7ZeWeQ;x8m`b$}H$%kWH%?0KIDmJ#<0qCP_N(szxCskA0(-Wbw_MhdO03wHT$XHTrq)TChPA>v%DX*O+q4&YT0D3B<}%WMd0odhT)nRR>vDDF-?y_aSiFt@ z@%)SbN8YRckMzs_Px|a+^G4~Tn0o%pol|8z|Kk6W_p1LT{c`{P1AW>Wyj%lKvlm}4 zcPjeqx0-h{R+lQ^0o9$pvqh~?Ej{l4YFarf0fmrlX$jR{xM@ z0$G&c5G{Bg3eRuXZ)3oKSxn*&PYVLHpiPwE4SsyQ;Kwtp7qlAzW)a}s4T}j5_~G|G z=TzNW_ujgFyGP-VosU|#?yXa&PM!L7>eQ)I|Mmwy>zw@a_rv!a^ndQ#fAZb*{u@}6 zKJ^c;@4ifH!RP+T7w-Na{~znMul`3`8)c#M`}$LqC+_`*4*3nxA>Raj;LXqnu0$x2;I2&2hdBrK$lqB+_%g!ZM7V@7WXI{n zn{PUDXXd6ILvM!-D*Q>@nR(BNKi=Tv|K=?iUz$pmeEbf~;Tyj6+kYf7Ep(ZIEP3~x z9QT_l?l|H|Ze>~W$veOB3iP^^_Nyv>O67YM=Sy&w{lf3zN9V-O;7NTyhxiKmIpMg9 zc`1wcMZB}@fHCbpycaqC#-AtparW%Oui*D>>?6`WE{71NJ6T?No5*fIZwxa&fgRa}waz<#`&>QqTt)wzs%CNFCX~%nUy`S-isf__v1-tR=x(CX@AZFe`wA+LiB^r z=A1Q#Z+iaFbJjt8qqKhjzBxByzPi!*&I`m(%vZmM=e4)U97FTfZ}ayJbY}(YFdX!6 zntPtb`ylG)v%{0m0-7me_* z7~x+t!e24MUpK}KeKTy7>ko|Ze=)*;Y=r--5&jD!{NIi6|1`qi zHo|{rg#W<^-(b*#JEVNx;$A{Oe2Wpj)d;`C2*2A1-(iIBGQ!_ugg;<}w;JJ{MtF}A z-e-jGH^K*u@Q4xqh!LJJ!qZ0hm=XS7Bm8j*JC;A_ea^l2(1Gt@{5rS(O9JOQ?hE?I z@nb3RQz`NJl=$N*@gGfz|3pfBF(rO6CBBjpUr&i|rNl3%#D9NE{F#*ahg0Iuro=yz z5`Qiw{?U~9$5P@SPl_CH`tk{7WhE*HYqNPKp0kO8hG+@vo-Dzm^j340RsA*QCU+O^IKZ z62CqrenU$9Z7K1$r^LG{@pq@h_ou{fNr~T{5{F5p1Po=~^of3aOCH_K6{Kb^`ODXY}Q{taViGMaF{<-A%KY~vG zz0VJQ=XpGB>j#d1AtnCBl=v$t@mEvgUrLF;mJElkwUl^gxHJFP zq{OdHiC>ozzdj{?LrVN@$?@x;{9ccB>uvuts2=7JsCHfBdL4>a&oX7WX{?m+e1>x=Z{)Y&^i16+B{@)OO z0bwe~T9jkzuQJYm!nd{8zWw^+pzBeDzyB}3{rYjFJBRQx!g+lE4#JNkycgg92I0pL z9>n*H2tSJOZTNlx;d2OIK=^YAKY}pj10ATdhX@I!0ccO#-^gG4V`+Zw$_ip@?_PvMx ze(_#<|3iiQKdN~8ruid${)YpO6a4#zUi*_Y2hfLHQiJfx;}KBPcJ;J`{5t^y^r7i<;T8s^NX2> ze(=_>d~V0xzwzmHM_V8JO5-2?&3`%bb2nZ2V(YKI__eQgGfAkN|e&^Kmk^3K8`o@l@o_+s~=YR7D9{lQGzv=2X4u0Y1 ze(%WKZ{4)8Y14+!_5JGm_rCW%TVGq+GXAzV@14mv`%1sw_sLUR|L?DV=5t#bKlW4C zzWURL=imKQ{YMUe{b!C;e)C<|e(e9b`=b}v$JpsZ z@BigX=Ldgj-R&Rx$d_K6nEb8BANuRs*2Vw+*&l7y=6-g0!}7Bauf5?nANjl2zjE#` zD^pMY?azGYEB|x(XrWVyN>6N`+s!*KmTvP{KVgWoV>2x|?=cnIwJu@2AJBAAW`$Xia>C)*moURESub;Yn`xPGjT|YG*{jT%V z-!a{uc`P4aQ~G>156s1*Hhn-!+||P|-AZ(4R*7_n2ep@9*UC zx^t_=HqxI)PxBf5Y5Hg$pg-N- zX|AKc4f1f_uAcC5!GAOl(x0v;nnUD|MiA!iWqQ~Arl%^)*XcV&Fpuyb!>C9O(|;Ub zNEKA|us_HXqrrzAS3rpmh;mMp8r4$8TWaHX$}1PWrRDkhpf|s3XQwD>$Haq8_&tc< zKf~|u@#8$4PoTQ?yEcYc>-Zh*;uYmACo+g~5;Pq3dlLVLd)01%AfFIQo#nV_ob+!f}`hShHt+iC& zR%{LqER|Y?%0T(#&Vfq#RB51GEtZzI9_K%2^vF&(`0CD=ms{;dDf)S7sZe*u35{E> zmRsdQrTnQvt6ZyYTdEaH6&KO&NOp9r&l#Q>^u}}f?3g!`pU!2+M<)(?x#Ls0>AaU8 z<+qum*#l#_c$%@%V>xehVmNo)fX+>%0?*Z|twyclsHZzuJ5_2F z7D|fBxk9yAE*4rP#}D}++nOyzD8TtbyV4q%FH|bCg}DVjKZu9q4ull2>$ea=*NX2u~#I`vwmJh#F*xXotq zzKun90c72`u~KQf#oFnrgi6)IYz1G7Zn50d!G&6jzEBo(trDf}b2c{JCd%5>$y$wa zNxkUesMm6}3Ch$}G#BsNxP8Z-N@cmvY0NsOXPwbQ!_&^_a5nEujpcJrYkFeF8O==$ zJLM&+!*bItE!A5qZmoer&Xr2VrrTO9xqAlP@$vz;RGmYkEP}U7<(A{GHOkdfXtqU4 z)UGz$^?I!V=D1jjxQy0mGcVm%%|!`obJPfmB|$|%_40D5(sUBgd}J!u1s;adh`20E z?PklJEx8;6(vXuXWxdc^^oxr6+vjd<+Gy0p72eFY=jTfepUQ({UQyQG9}y-}(+ zYICJ#lbT??&?sPNC^f8T(Ui-d0)G;DbE@3L001m4)HLx0n5q~c%aR|2o-v9mCbQe(p}1XX}M>!)u-{FI3t*9TP@IC7ku@ z{p`_%ZlgyE{Ovaw0MrO~do%1b48(To89Q}+cYDO z47!C@tLmNfX5En;dxEI@(D8gEVAAL>BQEde12@+&sRAqK-V z)tqaT>n&<`JnOxa(TnKv3$1c z(lN`drPD3WVGPRMQO<+W?mx>;Zx~i36 zl=j-y^2s)d!+ssg+)qMQ5dz$b+b98`388=nVV+p}A@&dWO=FYMrY0u6>Dvoge~b|^d#rDd@} z@Ro}Axby7_miN5K4;m|0&+h3$W2s$t3n)|bM41}Bx%kJD*E@cE#sg|^YC1PEIyUBw zkDJ4uWUZPi72U5cnm)QH8Hq(J2?QU2U&L&cb_)lQtix`NoDR&caH5j-Yw zhq5zXJ~uHlIW6r_5D&UW15?e}okttUvqa~LQVQ)d7z7UuQHh|qFuqMpPLF%jGe@BC zP!g{ZdH9$&oqfO?fg&rw!h$e#Bs;ya$%AIZcs75?o5_ykOfs8`g(6yCr99gxG*-~M zu}Y-+aqn}Rh53@(k0ApRoI9G!oQtPhaw=l*Q_Z?S6QFj>upctx`yeM2tQqSlni!3 ziGt~w5}mGFN)D&;WYA-nRZt@7I#B{84#vX-5`xGs^EMK6$`sm7zYlu$78wQUl7LvU zPGmrz3&otJJ4i&ASt{U;8P!*a!wFoT3u3#?(n!%&zSi-^&$sL0#!QixFDc8*3sUSI zd%Qs#n^BA?SPvYCB~2p>U1yB(_c*^YpgNPJndS7r6;_7>eG^}ILyFLZ#tkZcC zXqU`OOrFX+wVd%FRu+~DjiRS_TH_)IXOaZp3!#ZPQmF+a=fO{2OLgL86s{NQVrjdf ziiP2BNs9ybNS_5kDv?+nmut}fFGAq&#JVV4FQmoNcA`peBS7#^peF*+B_+{RooL`D zjWxX2Y%e9vG%=Vi>4>N6mXf^*{4g;x%ql36be$-fEmysH+UZsDPXaMEFbh4#U@D`q zT7p@VcOt7$EGCI#I?9s0ZT07ar)0M)q2x%pAvSVLQ~>xsF9;yOtf zfDJ*V(D*J;qA^_$LB;YZ*4oAw-aYX}qp^~rT5Dse>9H<^Y?7#7*o5OakcCmU25Cvy z6)~xVFhH+FNTN;zpaXbunk~x(J7AXtB_!@d7!QyqF@gKkC9x<0vP;sEQm4|kh)uqL zIG)B^P?#qSG@MO`B)w4E35}KPKxe3T?xD!_#&>&%a@k>TdVEIh;Ezt_#%QyiwzNiv zw9bzDiP-MJ?#x?WPEy`k;Vk7!>e2S})&i-n98bYU4QMEq2Am9hT{P+(?c;H0LbZ6) z3%x;_>nGd5wF)+?)OA=Ej|Op+aa1^lHM+; zWev0{Hu;IW;|qg#kUJw(X~@-+=40rNKHYJ3vaY&^23{4F(Hw&smrWcQ1j5~e_rCXd zIw|+8pou2a+&NwEhq{5J0=wuq*IqsZMk- z=yj6gXB1krsUayuI3TaH?g&G-^%ZKV$-SE?48g9F#xRamWI|qf^ z7$`f6lib_Gz^kM*f}~eMbqQ0rs&moc*Ga68rVDcQs*fh{$OM*5bcaN}limqgj|I>L z?~ZZ}eSIYRSh}DU?N$yR>qK#-lEjB<0>=cQ;w6h=ZuQ|MUj*f9VPTZ(TVwS~6dfig z&-F%|pI>5P)hsOhfSxkUgAizxwGhq3OhXPe*o-ctAeIJUlgwkux@ZAx2ixP6k(|at zXiQdzy|=7Z1{ja1KXIa|y@cZkFw3U%s131*4yRj&_p1PdE_d?48i&0;nstsr zg(nfqdse1(7l#B^35(Xcy?Jt&t&ha3m@aPqiPm@Dz@YT;i>idB;t%J+I`hJg>x zWE9o@bVN-kaRam$=uX$Ba;~%>(|GM3w=Ji`ar%tm=tPbhVA8Pp-OQ+bE548-KZT`U zVSXMbilr_h6iCznJTSm11r85znP@f+IB1Y!47Oe4i8Cl{-Pf$mx9Egdy#{X+@y^X< zGhupZAfa?VZ!OK30CCV_IYG=eqr1#Ml%(6Ntws?hCU0;R*&|UW0x(nHpcY$@kzKk+ zXJIWBVFT-u7;DmQNLni4Br?@WH-yE3dnC_-pc9c8v3(ITsog|CECd-{Dx}YjV((am z+(}xs`U$k*<_SyXg#j#N*aVYT>+JDj^zhRRsBoQwh`7vZqVsg+;h5 zAzQ^H+HA1BkY`PwN+BH?tK*cWhneHVVy#h4A}`>(vf&?tvd-=i2)9{YSgPUBvA<^V zrS+tlA_;?)WpM#3o*X_*t|xaSUv1Wq!s?ZF^Lnz^C{|0tN%$1Ij?z@Y>{m&SN^Pa4i)H{j<_?e(CT!T7b<(wu{j1=#MC zykfZlPZL`6$UuQx2T#rq0WJxE3=YX55Khc=0fI#b>E*nIg{4XoI6E2fXf?#k#YU~z zo@1Xw9kP%#w6s*TBxdB}HDN<6@GYXbY4fTiTr^%q(F%%>+83~jigOqWaaLKAe2mj< zwW>!dg67^HK)uE+8g198(GH970MdN|91bH#v!jwz8U}~_Z42@YGKs>Q2=uEy;G>i%vI8@VdnpLZy z5x}4BIuXF5p{kWrC4VyQA&JyQH${l#)mtS>!J;VU3hicEJL;6u-h`rtcXFW97?Gm@ zgRGh~0q7J*Xumz!UnbtNi74-G89?!D2#XbpRgjSZiZ%tKz%ja#fNTgUciQH(BH;*{ z_+zm?(TQo9%47t)Q+P-5R@{3m5iCS z7#^M@u*BneNQ7{))+rS`q!mj9F3n=0GVfL3PuN(xjUOMG8ie!d%KX6C(L9`0!3{G! z(ZHjb_7@g`AQx`%eZi0>qPY1##31thtyfJ=b9XJY*hx%k@m^+-o|p`-wBh#3Jveog z46J)#=?OfoU4+L zR`SUjyea3n5~osk%!|A$+H#q1lnk^TGf{Mr;V;UAMnz6^SImkfGZj8wOlv_nj`#-#W|6TWv=f{S`cijQ@(S842{yf6XT=r(k z`laN-_*R)RoAR=goY=peniF8;h4v-6UId=!bc|j2RC|Jq0 zDY_@T{9$7i{`3fg=omtX;fS03v>A;Pj^BhL(f83I()^T{JC>Wsdk3@moN{^^qdGt3 znwT}x01lQ%P)-4i7=+R^*vDmSk=*8rPd?-W&zOgExv4B&8z5960r>V4;8{kfQ069 ztc8=2aLI>Nf*QIsS>myiYH=Q-v*JG@XUhwMnfwbk-Jz+Y1LVpa-ZkZ`a0-ROjb_6gbR7^)jMn0}o=N22aDptRSN5k>l_~i71a8f!oIh}V3a8M1e zmeqFM068+~W%Kz7?}&H68^Mr1lpQ(*ci_{La1|NycNlemDZX*hs^!iBc(h)@;O%=Z zhPzAudIGx-Twe4k->lTU0^h9>$F7YMtymT0T)8pVrX4Q29D;9DFdF?4ENXEj!EC9> ze6x)eXk1Y6EB=RiOf)gA1j+9|QjoJxcI7EvWyKGt>3~w7%+609&P{l^@dLSG+_sS& z%6rE~a}SvPGLnmUt0$MY_MF&ktk0pLMhb>e5{~v`703Yd# zDU8NGRWh$*2+}DEToR}&VA9)tG?y;x1RV6ZO|v%CF|fotf<#(a++`IdrcY-`jIK}> z=}`kubWw5Q3lR|$l@b8hRj!(LZGBXd(3J#~C^AwUJa}}3F8(1Tp#d#9TeyV&WDOUE z=&4@i3HQ>CT9XM=Vj16p5JD=m56wzgDA3J05H#3VIR;9Yi@GD2`*{vAoZ;UC>sgGY zGGoAdF=PhtfkYYj+=i27rRVH(Fl7wXAw8EIRf_{a&HE;WcB=*tw&l5KjEsc72j?Jr zMPA^N-xUAd{*BE|+9HnWDymZ0M)jDF^;~v(>_K=5&JLRcvB8=`qp~tkslmEWkpW?E zBkphsW|Qq(xB6bll|UEVNfac9pI%5E+)fM;BbGp_q7I}Sqqk)9FXn-(G2i+?`9XD#ztz~>^RY|FPCPT(VrIp4;E_q$INo4+jr$J@QP*MMiPt@Sx5_foY1sGPt@&YrQu}>g64D2V-I*EL>$Ww0F1tx3>r{E$G+F`uge`GJ# z4Z2L*#+mFbm?xIKVxut6*Hq~BkyL!RzF-D8kh0n~tA?xw{W{^aMmL+z9XUD*pGCvD zp~+!%!s&zK+2dY*a>{n~oCU}bqlkf+`H42to^)QyFqr64-i^7&=He|l~X;L zSPh9bEa~K_K_a5Pkpl*Cj7A+RAs0ix-)L}M7B82l56u{WdOk^q)LkMS(!hy}MrPDi zU1oj3E?Y@SR0_H?YXD?{uP58g>{+aLVTz(J>Ig0>q00#uDz#aRj+7F6ihOk;T~tUY zy`|k0G8*Z2zq#)4I|L*+4hK3D&0x|Jvw$bq>FB9uGoq-4QX3BDZKI?|a6!Y+116LUGiIk})Y6$qkwsLI(a37LR3fVMewC2aBvh*uMNTz_DM{-S zZ&|6Oj8@%ZPA^7xn9Ne4rzvszI)+;dl5|?biKrWyU&Q5KY1x-|xJSa4fsDyuShQ5H zEBgW7!Kv=p=}SB*kbC^yqmF<~%|!Nim^-?|g_DLT=Q~QGELMZY>Zc3hO$13gi^H>p zW*v_=gG_S>qh%N~#xJ^Ek#wpb=?pfLpM;b&IpL3W%qe)9X?@YOl+e#HBzS_Es_IxB91gQfjJb7`T@x_Ji!oB%(OH)oU-#^E*2?( z(SteDW#E+&-Ix_eXhTUZ3|hS?WI>YNaKeeWZyK}&eKQP)sy5p_Gz6LDBnC}CBil^hm)BIK`CKlqeloAJVTNMYFiQ7FdJ2ygi6#% zxZ^8Tac}nRU|3|@icSbU^?`s|vp|v?WZ9mXWE91MB%S(LB9koW5H?>AMd9@6#F+3V z9Wt?>+=ecDJ>=NXbYIVw)Cn|_d83?K<&nhNEd5?*RG^VV6h-%Bn%WB*2YnM^3e)9Y z;rk4d^E3;`XNG%YeV-FbWMbn3yOh-Uo6sBOIaCeCqPQa{#uHH5krpCH1zO8DR{jo^# z;queji5Xn4o~M25mQ?&qs#jg-cUDxS0pxbW)Mq%|O5d~^b; z9A23i3m2nU2Iv8^ML^NW)<^oDXDrU>E<=5bA?y~~baOVVEdARSX+zzw#4Tz|L2kh% z?Kwa~`4KI`D16xj0@!uR13hWTBNoDGs@6sRbAjS z^%wZvXbhY#thlon1{X;utz6Qu(GM&ss6s-3{$S^)F#F>0%&|dtWNI(n$fR2j6(DA6 z6%);9@QbI7;nwQQlc84|$9mA@M#M`DF2(@v8%N*?P z=mt%9D=mbldw7p7R` z-AVStVhTkMyZ2O)@d)0Mk~+5V(V*UMZ6+^`cwrFpGuj*tC=AQIC$5&atLCM~W9v3H zS=T~>+-6Lkuu98wZ0m+=jd9N#E%T+NL>R^gfN;9V_Ss;g>_{0IVlj)kMz82sU1Wu3 z^(97cC3VVmwPD3~g=!VFTyMq80W9lq>DRl#dv1?%UeaavI!2rv&BTbqN+!C}%n&QJ zFs$w?Igrf{9TIVO@@U?}rOnXmk4{b)DYU@aeF~Oq2_3FIIT$fq^mjni&h}ncIv8Xb zIW;+;tfH{w$B0Vmh|_GkO(y7cLN9!mwk{6j;2w$sXrD$gu_URQho>lggR7yBG3f%r zO7KLpLYK~WPe!lv5sbl{P^uyYLY3HnzbK+V5%F{;<1p82Q?ebR05GWeP7^(1ht=Bdd`4 zD35_XMjrr!aj{%wAN4Lw1AVRGUGr#v+*pwb!vBS<(k7#UM$Gz~sOI6DH|* z_}r-|YIxjGrffxz?4{s@f~Fj5k6w{>&lF+CL0$3Y)mo~n29rd(G`w)rB$=!{2NfFp zAR8RBfyzO&1W=>ol0i^z#s!5~6^2;pJtq}W#|ntgZOT$?I{N^fXgZXe#-R%mwG^+Y zjYmjTG10FEVKb-82gi;EQ-JN1nZTICh!V)L7t%Rp0B29VtJ~zew5-MLMuaPfBrxP@ zwo1?^Ru}x!@sSbKfQ8g9u$^*mb9ntV?l5dhvSN^MN)#3J_BZe zu(&4^dphODz8+keY5SNdValU?(eoBq*M+KusEMYCVw9~J-VWq_G7aAa`UvAXW9q&U z5!D@WMT;O~Z^J#@4NoFO2Wh$VT@02)UXY5}6d`C}_tsrL2(8P__M6uJ;)&&qiBFl5 zeYv&7ft3rFa48v#34tkdf!9c#NkSQeG_vBRHMpNTQeYZoM1x%q~ptRh<8ggACo@iwE1iQJk7eg6ZQAYC(rm2qQBz)v2 zB$(`?h_o_8jjhSpWNZk7ryvt4vNx0>Q8FV)xy}LI*7-xYUHOf?i!%ho42?=AE_Fh{ zjEE^3De~kA7mFUWf50%5s}w$K4jDm#*B>4^;qNX%wI+gH4bH|oA4aejj49Ck$J za2{K2hnZ@CrP5Ljb{%ZC0hb@7C4iepvDha!kC9nVX?r64!l}|m!5nGgbW!71i4)4o zNTT_va}M1WnQ~D{X3N+$>7rN0XE42l#|BPr+B-0sorx{qR30LfL}?c0tb+1T!fxpg z=7o+#ab(dQujTZ9Wy%RqpOuoxuYodd8G;y%WjgsKP-iFLZv(sdv88A{(=I9`(&7vp zJ`IKfBxd^oDuOCSA^cqj-xf`uzU33)(aJNj^F>Zo$}WyeN0=7H(BXWr-ee?-W0cNJ zjg97`lo}_yr50ov5^VPKIbSx0A_+-Zt=+eMdE+vU*cBH_I6nwMNvXQoJsK&up)bOr zt6C@jW2C(a^LTucj|fEeJG+4K#3q_CFkB%2$q3nFTBXicC~Sj>uL-1fNIekVqKst+Zf(tl{9*+rD4~Wv2FBeL1P(Kju`%TQfbA%5 zQXw(uUEJ%aOfb2}bidd;W@SV1% z_PDsIMm1Au-~$*$VCYvOOD#B))^dkJPby=CKEyCiW%X^XvpMlN-wo<^4Vxt9DHh4RzhZ%52dsG-GDJ zK~j~%G-KLj&lc>k!WZCUlDidL&BX00?HCkmm=Q@fg544p-^Q5c%j8(0vkV?IAHNLR z14s~smenCP8Wj_)Ey+8=%pA(n-ZeYt47YDTD{55Mu50EMcpQv$6CX`*#7KsKS@N~T zml%{WojBqND<+4^^Z@G~R+8;uPmdiO8X4>l(%5R1WuTLM(Qrww8K$iDSMt3I*RN~@ zqrZv3q`zukgO&_o>e<=rk~NKu$PSt8gu!Odxgb$28{hG#(bUm(w$a2MtUZ<9-j@Oj zX!6wooKnGAAk`hX0{a~4Ropk6g?f8{wuc62wdsffv_pR9NRQw^b5_P}47J99+VC^J z&@$F|g%{6+mf?5Z?pN;y`=hC2+%cFE3>TsPU#T_8$f|sbtdJobsZ&$5UZ4X?Bq!_s zZy-zL@La83!8Qo3wZJbjV^X;h3&}xOXb~d@m-c0O`Q*-R_&M~%gMDFdS-VAhk*UUJsl3>c!9t+ek;%hz%d z>({0#DT2i<;eRTIeWQq%S8^aNjNt`$@R6FcUJJt)ZZ&ve>b$@oth`f0BiFHTj3OJn z9`@ntp$~F^6WTC_QnGBsetLX|OC3g4lDfBhsazbWRVypBNkjL_G-4*+$P#XL=}Hpb z0#kCyQa4&V9FkD9vA>Sk-mY(wI z1Q3g(2`9`&@oLZWQ#bhV|G?!ShT372n#wu+h}L137V>@xHc*f zZ0zbz-`%r}C!}jq1D7HW)_|M2F>id_5E!*CAr;UR{5xWd4LL?`vAEcPb#OX6b;uh&IzA8q!V@o`q7#-|9S6h_yJ4Eg3L z9xM^%cu@MfzAL!==){9IEfc1@@xro&H&ij><5<0$siZr^dBF;zy)H^VIvi3Wk$ljJ z5i&y1aAe_uZh;X}@>$gA%;@+q>w(hhc2$XE`o0(^*s&aGN%G|3J{TKy(@MZFo}Kb$ zhG1VD+Cxc6M2b)-DY_|jLbk(`55T{|bz?Y`9UJj*!HV94QfJFClfp~1z5?4L?*(49 zv~#=X-wbE~F3;EH?7zOp#Um#l-kfovS|?~r);`7#h-mg+UT*S9GB_@aV?n&!Mq`cj zlj8tA0i@M;&~KP!sBttBwQ(1e4zqmZ_M@1S1!66i&zSqgbAlWS37F?GKiT_GJf|*k zbeqs;^eQs-({`I#&*h3E*3}nfVh36hl9=*9Ah+m~Ip%H`)ZsKC7)vEtI?$5Pmszw7 z7TW#k5ic)WwGgC5F1+*d6^9UM5m9WDHNRSjvKxcz$uZcmZj>xHsK75U7D%$_jpqer zj<87yhvHpY(LcAsK2OvhrSEK15i9AIPVs6Gi%?#X!_<-%QKW+k7E@RQZsvtxyDket z+2Eq3VHrMDuo6RXoBy9%Ep6XPsjHyf#^~M>tEkpfS8W<1Y?&c2`1+k>JD9DTn?Q>Soh5RtRiF9JJCv zM(Jgpfb~{*^V)^NNK4{<4jC**PLTB2$b%-QF7Y7=iaeFFfDsT4Z514wvpAt7eOz5f z9#-kif!uFsZmyREYEvEZH*$)t98__XXDC^?*y=#!#MrtFt}|rAO?m{@BHxvAb?D8w zcXh5!&YnYk0w>;OZP(M1Ms}qAw6ux1RH7lHH(Mhg0h!IA`(Yy!+N*5K`%8#-wr({Oah6QhY*=vYOiTCns~y&?87vd%2t z#s^`(3k(xP-vzEIM1k$UZ&?{=90JX6#|jWsJ@Ky#e7@p;BPK&JEG1PmwAAH#i49{^ z*C9V>Se5mPlJ+vt4au@7IJ>HhRw)~G)j&8Cv)-lHQu*ewZrm#AyEMdcR6Ye`BaR&d z0zh@{9!C|w1^Z~a?FR8R0qj5Eo`jDRj*lLP9nH}hto!^ckNMQ4rBvP=E-A-&xYLOd zW7Y=g^D*ibT`QV z8^dVwZ8!nAjmriKpW%KAn(0+3v3|1`Uxc1O3Xr~ zDlC$Xz0%Pdl(c(&l@Q>$u8IiI^jH}|!k+sEX*A-JuUvUBz{??U5|b8L({z7^+Nf_k zDjS`C$2BJ(NiK%l#wFC`?#YRuR$W}g41)q(cyyg*2w~_*x>aI0vX_L0W4R%%#$qu6 zMTiv0gKH&xOM!w*D=JT@wDD|@7~aLip_;F)l-x;C8RRcco}y@(iLk{UpcCZ>M&Rdnr}k!K5T>8zoAqb{=SnF6X&UJmETy4h2l9XkzD zjZ4T;id&+EBQ1G4i?+?mNSJ6oM zq~N-28i;NPrqk~@^n;ZLv|YHT);u!;wh9QH$iUuc0meC!1ez3Bz44b2NeY}~Z>|Of zP7>OVF!vNBq3~Ft*Dh2v_Tb2$nI|FHsX=%EF=KJ%S_|4zNP;R)f@l_&AnFO7+~hO| z{6h>1IKqNxv{76RN^5hXU{Xx>&9}p{mhA9XO!_ew1JC7-m04%A!!=Yc*h= zjhdZ04`H54w^XTnr~UdU;y?#(OFS{CD_aaS#4003i5Prqw1f%zJYBUhRMc+#+Km!8 zJ~k!FZK`@E4Q`#IKzwzFS+TMxh=PeLyoMGwxn(H#c$zrrn5x}j?*pE(8Z;u4#&R0r$huM3wLn~3|)k8Kk8?cQi%@S zs6)N%M6rNtuBg-c#LMb0ATv(B-gii*KbD6|LQ65M75&Rxgpovq1^`YzG%(izMoQxo zOr)~a2JFs0Nwyy1uuSLqtF>UW z0=1&{9KI#kw+TL-DmPka>^@LvAKSmYTlv{PIC{X?I}9vsRrENbnPk@ul+&2I>4w}E z{Ip{W2gT5~c<`R?Uye?y7O!v$o)_fsnA^lv?x7EJwfaiv1KGudJ}{0}tyRFriisus ziSdL#F{bb*miWc7yRyJjJ3K8If>C*zD(jta!!GXm;}L!mQILm-)rl;nkW($n90)$r z%mRgy*`w$brZp?F$g)P%8|`Yz8iTR2VU0Ll!0q#9eQCltB1{+)WYjgktNt6pp(7&n z^Jf@nmgmC_fWMck*Z`_jLU=T8*_djS$h(J;zb)D=>y7 z>xLvde>yLYvEk#0Fg7VNW{s*zoADK9?u~Lux8WUh3kWfZQB^D~5g2 zWf_G=d3bQ*tq#CgfQsaGkLcNB_9~y%?eTEtW@Vn{63uNue9NP=ia3fdN1~li1{e98zLYeuU zshBD?upfto}bR6oBmvfwe&q4n;1YOb~nZnv4ESk zDx7e_YPwwf*vB@x-^Ucv)$?vE>)z*Px3X{T{_R`be&E?eXtu6wy2k)J0I&lJEPz?w z;%--9vI5*$yjKSxJHly5xsEd!{X6z-ap_lFL3NeqFdPyaH*J9yKyBjkd3QEjY~DKv zo-QpJxzFX1`_j@@_!}Us=z?s-feQ;@>p`B{PnA?HqQ2%Tbdb2el?D3X?Ev1GhxhlJ zD57f`+bv}$Ng z%hD>^av$9ms0W)E$+=2FO_9b})|h#3?)Q^yCx{ ze3;ZS5p*M*J;C_sjE!nE#5mo!gdInGaczQ$A3GW3mMDgOXev^~xq7^SLz5FQj}0TF z1MG!3zJs|upVtoq)3Ako{CHc(A%6=v3_lXUCr9#hUvj8uO}W4#RhejX@0T0B%hSL18r%1Hx%$Te-t*+PX~!q+pd!a;Oqov?*cJ zSTT>&NCNj(bh8*OO8tem+t0U_kpo#*9}vVE-dKrS67@NK+qSudI-L%IHO|KUefZZw z%$YBi=G{P#eFz?(*t5Lcd$i##F;y|0{1R7`k^N=XWJ-}_K5@=bz<*N^Fb zWs}?Azr4x4-|a{6!!CvQQFv1TIXYoN-kS_L^FVe=fqtJW;>c1RlP8eg!nkKe!z~8kV#uv-p4{)?A0YUKlyY^x1Wz5b*uge?X&G+2C`X(;M0aYG z=$K71I@;q z7RD3gYn9MXBH$k=0*(6f6NZIq&I)%4y6s903{ZdHqgY3;e!{Srosdv0j2Mlda92n^ zPe=$-J#{YFPdFG#NP@jpd1R zK46Klv2N0hF%|dpOejCT@QDTgACcRva1#z06=P^Y*QYD4)r{ZIihH_BlcVu$mQFIo zdh%8&<$6@yYI0rC4$y`1r zj*ePr*4R;-()K|UhPoy5M$xH}N{QWY`merP$`p6Zg9A|L@D(mND2K5dL%V-!TxrP- zH7i?(XyrKsT2aJd695WBm>1z4J_2zmR`8}w!91!`7%oeSP}d@bW3^p}XD6sCF@!+W z`cxZ*I0e&U(PBxv(h90T9K=L}JNeLXl;Ed4l!_)MzGcIX2WYhNcR4-TERw_IIK>dU zmWF3TC>;uQKFAR`sY3%*TYjN(D0mVWaao4VSMkAvM@PV(dL46aT2ktq7{TDzfTnE8 zkTjCkbjzAk(1TTpP`)IAZb-l@Y!QVf#Z7|g0!t4Ba5E+N3V0Ps*Q~(sESz0o`2efS z$4b?O;9iI35~`Dj6Z99O{bS<|ElL+K*yEy)x~55+jFyj4A?R!4lf$_&b`0dvl8JV? zjNw))OZBfhv5fAP=s*Ju@BQtxE!4)N6zL&J+M?j6JDr;-3k>#E9f+$6t{2*Kl8g8Z zx&p&6aumU>53R|d7COLy2W;o2Uzy2pCsQm{u)|(DO_vu(*>QA&#`eRxVd90r#<`+2 z>gwFn9IL!xz9IcimRkW{t&g^OqoTgg%DM&BLj%mHQ;3g+6Vk`YG`oq7=4!FI$)I^+ z63+Xm79Yq?j~|_KNSq~c)nqX1yNw`}J2Z2d z{DRCkYD?PBjsB?Xeq(d%)~$}>6s|@tLN50WT_V!2QBJdv*wUUA z?b=nd8Xw8_Mk>#2A_5!doWc<{N7VO)M^I^Z0@6NJN_#sA{4>yTW0o)s4g|64X!*pX zIMxjsS`0)x3W$U1Hkq2l=y*~D>#(g_07HaFGU{t;sHdi|b*t`+da{_K(GU%6maaqR zb|m?5^9T=8kwWAC9CY8MzFaQ>gVz9;6Kq^BSN}8`|$9dkX%~ zY=n^kP*Dly`NN=aCt(~!4>cP(C+E?S@=$}LeDqB`?9tfAgVPzRCzCp?QcBprz!;<~ zAb77*mfjJg%s8kM1CTN^^rxYj%6?M zv`(VNM>Vedva<^xz4C%~6`lE%Q;Za1jG9fXfH3(^j8@gi!~-C0>O1HqPzsP`SfdOr zhNJiRj8AH=HKF27GDiG#E<2n(g#VJLU^i=Xe499TDIr#wH_!#*Q1_~v|H4sx*p)hA z__G{;jeI1V%47nF0bLXYt7K^8&>Uo$EZx|}tdI2uPTxWShZ7M+wY*xw20^)sD)wCv zK^35ftb1JS^}*Juq<~2l#74gXEMrc%WRQGkyP05sgS^aXS5;nWcol?L!_%z~H0B?n3YT~c(ifV&=IOx zR8|@fT8))`?!W*%nc?PgHB1dK#|M;Q7%LqP%G5%|Lkg^`c)0)_O6Q~#2M?f<$W?yF zF^rl4Nqxq+=@X-88){Nv?D%MN>Tg+T|x1T24uphK#w7%QRmz+MvH=_bY! zZpuO@fuyO)$+3=g<}+P#RD~CnuYICH7iKSmQl}joO)fh<_MkU53Ep%w=?Mv)%G;^+ z!7i+_f(ur_H*5%%n~Q;W0M$)M{Ucfi)&!}ch{F*b)oHXnlztZ}B5`9(w57^$r|bY7 zJjvRWSVxkEK_iQD>||Fcz_>t56v`Oavdg{*+ZvR%AZnlrS%wBi?eKzTRP9rg&RK_% zKskiYO_f2LCr(mc0yawqzSte_f;zZlX5 z^z706q(yS;08D4c=^yoe?<6@EC3|wQH8)3uBZInXWotuqnog=Vqmc%wMfM?vcJNjm z1&_9c(BfX5eh(Jq`!>qr98Y@*ga&;?I8mnHdNz2%$vRF|$vU0Mp)n=2qcHNAOd4%J3LH-6ApYCiQ5a({ z?J1PMJqhA%K7_@vh`MH1%2;902%bI0~L6~-VmymCmB~KS~CG5}@AprQ!WKLn!e8J_jV>Fnah?yW%aKBGYsEJdr)F zH}Mn##WC)6OKgS0T#6HEGD2J($5{|~`yzAnZMb6?%j@Xu9Ah&eFRuOKUdEbL6=OY{GeZpE(BK2HiN<3`nU5la0)8n*QO*6eCh&hvy#% zR)ZVrgW|WR#QW)eUquP&6XN~!yYw)UkUk;aPahpRqx2`l`{{S5m75j4;dOoT&|Qx8%PC)TrjR$})&ygkqh<~$H^*q;!Qz@0uc6C4Bf zs>}7*d0_GMp@n~$6-gsb?jmi2cuGI-h%UXqn=^y*PeQj7r~~;Y1J@*0f|c8naGY^E z-|SaFW%>Zsecakd6JV|VF#$H*%+=VF02}US2CN#V5@}TNiFB#>L|Rn5Eq^q!PAoE_ zZMhrKw#<#_-PBxX=gM#>i$@I)nbBY%2;~kARH2oNl0wUUYECj#`1e2|SHX#;1(-)+ zF9No}>Qte$E>pXJtr+y!8M=9m{f19WTj^7hEanIs?}=?4SZ_q9Am42nQ*bC|dK?!J zkW*|@rGCT$qb{@J!z)9Rxseg!nbPQ@{A%r@?a`)dQ#oju<@BABeq=R62VHzC-lmZi zMi%&CpYV=&2gpWZVrG=Kh)2*Wutn?TClBNLI#c#Q?U2U=sHfBpfZ92NmKIif7^rZw zD13LE^k%VTp>%RC(7C>e2rRm_X)0VUmT(5MHx6YFWWp~QwyB{?8LxA@rTRKBg8BclAoC~<0vu#0Y4kMG0|?j9h$kN zxR3;aOV|TG5%uvp)Ga|2n9y^Y?W<8Z?ZUYaA2hViQ%cP!OFwDcR0D7Y3`Y(MH6iqX z$zwDUXJ$gHL!8gyBTqplF}Ej#MT6n2WU)L~YMN;1gd-{nrs!@K9S=+S02vwg;cPK( z6RYsnHI|w-DZn44fPik?&@xGpj>rZ0Wl8G6CWWN}b2haDg|52RD?VdToQ6VRauAn{ zPvcfYkGGUMlBeM5{#Jc6D{;rjoHn-LHia&R;=9daalER=R~Qf2lBz>x7A|2R_uxd- zpkD1NvPz}SXU=yRNlqE(5(X8xU6%~}~qQQC~2eNL$TQ7SDu^CulDItK!- zYkM&*IuPRzu*g8GlFo41>uS+d zCOzyjqCX6PGw>=0k3<;crT;)p4O65_8U99KC(=&}M$+UH3uir2$m z`(@9)Dj8MBj+ItqMb*FoUvx+)u`a8cj#>M}_e_|jC!x<<+D$y+8~unK)=o#7oa9ay z?j-~(ivrb0WxnoEkGfEBgNCIx6vTX&LPEi?bfsFvPQ$YN7phOAL^!j93xFzoQ6Z*0 zb@Mot6(ykuH5+LHu`vj>7aNASYxA`ikT_YrV6=FnYs}2h=BcPE{8r6#%tQ|fWKlv6 zUyMXXzv?cRpnr7-pO%3^RYFTeJ_CCB7_e1L874YbtlwhkZppTb1q808qzW?X)abcd zbtyHtDr#V2ia}xt$EI;zW`VZ)HGZY8jC z2%wHGI{v`Q=z=#Wg>Ww4AVt^3%s3AA`m>TOISsHRiai`BrV9cMmPc5uhk6lY$4srP|!Fojz8ZJJk~oJMns&9tljORw)e( zO|l1Qv-Q|H5@23ER}CCi<{H)Fw^`=M0LGk1qqa?5rk#V?^=B(59O~D&yB;JYurAt} z#o+X^bbD&;vvU|;sD0boxmf~bP(U+?tRQ=A3woeX zc-T|r^rI(XvKPJuo#+2Nx^t@Gv(QY#%7s|CImGK=6$Df-ls~9nm_mJ zy7a}7?;1U!>1qf^v9YZ+O}bN?2xbsZs!Rel=4w5qLnCP%D2$Xc&KLq`WWo^UMI5RZ zjA~~LvLt3;JAoaFtws)(rj7cbVH|EfaYhRqQnL+joJEJwfOc_Qg7S69joX;ubw{!| zF$6IMC+x{-1z!$^sgo`tE1j0-0$i|ih|cm;z+bT#q-sgcq_U@p5qlr?kTIiuG$oc8 zc?1s5x|BDf>C>i1cp>bm6Ebfp5zBzs(+UgWB+@G%9awDn{!x(1g9Ao$a8ROG^fOI4 zcFstffV(dUB>3NnQ_kHzl8u)1b#I5fnmpl$?A8dswj-3uq8r1=q*Ug>7r@X3vUL50 zFVtgKw^8UoA=dwCsikw`8g=Y6!8MaNlN-a96QoF3sE*-8vQQFiX`@+I{R1DaJ{z%A zYLJ&6@+Z0xnZ8fV{n0dGstJb|q>RHyIn_X$74n5!hu+Nak358N@@0R#S4x}DTA>lp z!>6_q()#ppMoc4a22zJr25^Q$XN77OXZ~0mlFMwkV+O_UV!1_U6?r&G<`CrVX}w(T z05t-8heJ_r`UV1;DxV}piE3j(dF}l7OZq++Wwn;^)Q&z|wcw-V?Nd&WTo#c97xK9yV_ryDgotzfARSWfsgqK`#z zLsIn;5Ugtls?aen>HIbJZRuwE1Mq$o%nV7uheq?>vFzCBa5g_V-32^Nmgp^%M<-~l z^yM{L(8(gn^!yzhi&||u;{b~)I`dpqBf-%mRkI;0cf-&?$=F=K(08G)E$NdiWUEHE z^UwynY`aGQ+VzgnbfVh;Ku6%)q?ebQ4sN+W;XvVs|8~=Vd+;CM4Idp9qxTDI`_znOv5AK=4Tw)xPzrk>`{s10$FM47vr69EFx?Ec;+sWkkgD zZVJ{h3MDRzwoYj_^3kYSB!F*Yk6R>KooQr3(>uj+Yts>yJY`;uTv2Vn4d@=S3K~10 z39?l}Ot6nc8G7gm*i3?*X+kvbH6=!KpW);|E$bSd#;!1q8Ody~)vs|dwPe!J@RJI) z-voxo(8(o@)B>%p0qr&iHgOx7oF4bH(J85>nfVRa{Lpbz%vVO6(pckncJRlxD%BU6L$M3vJAI^fK3hm)<-m0#mK463lltp?)G zC%$aHvVzJ07C9IuUZ28e!U8a3KTpeWD)VEbxd&zpvJ|8`zUWJQouPvs%{@6j!330O zcJ+`XHXV~$5S>j^R8^JS-0wlPJ5RNtsF2=2+9E!?6BjX-bq+7!d<; z0G2N@$C*+QJ6M6C8lR8CxL(CyF;Mm-fyU&|m{@IKGAtAKDfxy>l|$VJ{jruMmZi9Uri=~gTEmou zVz53`ce0!U$JIbk2ey?BWza0dX`c3}$%=7!A0_b!#=*QM(zsy^KS7<7f)+!9 z9$&k@#H~E7TwW}yZk|?h+L}Ymj2HpIlE>)9DA>A~g{c810kX&a#J-1_0 zT{fK{N)fDkO4M_RCI+v;S?N?&}rjPo%$(pWf6|-Vi8R)xkz`YJ>dE2 zT)L%=lprBz82&sDFI8AkP7EEIjJO&mQC6MFQ4)+H92UtYIR;@&`@wXqWpc}ElzgN1 zo@R6+BuB9&6icvWh_G1G@i~p)z*T}c%hE1%yJ>(1(u8b9aE5L?At!C5Wsg>Zvn@IH z<#5AlY3KIHsYNbv5NXLchzy@biKGl40*XXh76Jj<=-EGrvO(mI?QvEZu^kcbw3Nmd zl4Bg~SUJh{f~CGl-7$n%|0l!*_2eI3ikDJseQk>*DT6J>FM};Ic~J(PMZ$7TN@vB! zrL)D@mT%p4VT(_TV5C&a>9FkR+8x?J)2vj05$866`yseB(aI8$P%uWS$%jVbmTxvk zPTEZKTZZ=JfDD;-khe^aj~p8xeQ!cxTimqL#Qc9)Cr;snQi1*B_?NNft2a+0H9_Mg zi$rvfs+{`{c@RykjDQ=RFb6Ub`n_*1s52Dz$g8Ms!{bzY7JibjJAs*xZoh`y6?J6Q ztQP8US%DcepEg6)KyUteLzo+o&X~R#j3M8U zVJC2bLLsId&;pEFqz6t)N%rKyi4TtjvaP3^9y7~;P)taS_qh;)&;n>M4r=^Z6G7#Q zkgN+(ZSz426+7P)&uQWdHjShQ9XJ}ha?#?pxH(FVd1Zj*qhlbRS z4Y~=mUHaex4=h@mbx$wio>MW*BeyFdsWys$8@`flfdGdm_)4I_w*~9IJEAZYxuF5B zW1?;q+FW+1TnBb<-9@`dU8v$kls0O>=kG4>=onOz9`NCoK(W0@hvVA>p%NT)6pSJ&@YMKqRl>|uFRa*td*s4j~?36oMb1keT22o4|9f=&qD-xwwpJ##=M2qM%6 z8XQ72!R-Lw|865gEjc(of_Ub18F$55iJ_j;EYM7VePF%{C`wfTfw5B{6nVT?+X%7A z_>@d3k3z&;Z=r-O=LoR|3QckJ-bGRnC{OAnh4wo5Hi7BF}UPzU7HoR{C~;5U5!4(ElpxzFF>yz;iZ<2<`z%@cPz&uzHPabDu!wGDSW z&LeN9uNU6F?&2NJ!|xb=?hfbDZS>{xZEK#n!@0`gmu_2g8L{u8?`Pk&=F%O`#dkfP zdG-$H%zL(9xWjq$cK7K!oNKpV${_y9JI-Vrhv@k6jPvjf4t{sPaEtTQ4O<-Ng&Q8Z z0pY7_UIfU+n{Iph7U%h!Zh!0+=jEGjzkG{x_U5|dyn6F(2%cNJ!Ev5kyAHwgYxg?N zwYBv1%p39b(Kpi9C*Fv!FTRn!ev7_7+;<$oD}5;E`8RE+Oy0ELah_P`BKX|8+aJEg zxwh^T;K7@>Blzl@kr8P8;=^ywe0$yUn)5GboJ%*Zxr)c-8!v>JN# z9>(+KzC(og!kgAUjmM=o4FmnUH5Bvkx@T{4oab)c@akUz!`nBU!MBTV-|!mVo_qTv zDAbFa7Afh=+t<8|Z!hk=?HV56+Iibccs#Ld-M8>~de_BUP@mbgFVf5Kr5j%%kb{*g zcsxG3OmB~V`gfQR3z-cB!1 z)HvN!wKb39?Wy{0kKu8-zK(Fda~@fPVPY210k?^x6w}dvtm2c|4w6UQZt%URiq%Z!fRBkfF%S-*^xfd}X#WVl56R3 z6I9hdnOP>p*Ph56;^?2wtUZJGC!Wl#y$0}?p3Hnd$Ny|*-D`yA=Q4M{isv&=W!6#r znV%QJF8+L`pVFWGQpSA+&!@hWLA$^7B`NtcUm`xdpx&HW1|llaGeUA{l{ z>zN|KKXxgz?nOK=U&Q^&sp2PFu%b9+PKYuy1=2<*1 zUCwNv_ZRX044zjmXD}YTb~&?-v3xDF<`TX>jps!?ANzV{{RM*hI@mzuy!soNHRti+ z;eVgG?P|5k=FuCUM#Q7fWu9TkEB_(0<|%x*_S>1;p2YLa?+D~YJPFe? zzms_w4eZ(HGlLiLdhT~KcVEKu!tZ9#?=Rths-~BJ7eFuI^SAzANqy;s%&RvM{KY@a zta$=xpShB`?J2xpxgr2h{^!gT#lQNWGix44{KMbO-1ZorkAG9*U-)L`%nh93A7_Rs zi>rT}SxfJa{z+!-qX0brCz*2$@Z6uOrt+tmCo=@`*qZ-4%6&Z}=l_VkDEuYqC= z#jn2M`5Rx@;ynAtN2%}iou!xOZaI6ymCeqXTl>#!ah|$$%yC}1bq&71dh6|&B;Q8C z$KHnE1rA<&+q#!GJI`!5^zatv+&kA&_l=5EJ$^^2%w(=^&trH<`)UCGrR-X1#`!mls77Vb`BT zLFpXLkL;a+ZCP~z4Mc3|gmkcs!Iib0E{}edI8C7^b>IagO3*JM76t5zeQt#+KJ*q) zj{LK7eIhD+pMzIw9;C)A9^z{}QRnzLIZIdma8TiAto#AlG!^}*+hc_aE_-vxwMex% zf;Cxzk~J3Ebn}!3qq@UK{7(&UsDPbJ9PHlbK$X}+{#122s8VoB*qN0^B#eQIk>V7Z z&>JB-TEz{v1-f`ed(zfPz3i$I_&xkyejH( zjZDWdq8J<)#}!wN6-5_C)o^LG#5cpda&Y_upa;-q%cTa@73h}x6?8pS3mDZgyk=Ef zR}Gk8rAdsEzu|oZBJUhSeR`rcgch|===R56HByq*M;rZh(?{A>ZW(&`M&|kx zWj_I$8F}m@=`maI*CyTBNIq7cZJ?b|>KSH$PluWzkOk(mLDkWCr1}5Zdmr#R#`ORH znK^U*C25EDS4T`GLAXaxz(&VIx=3kSOHl@o!*swv^Aa;WwRuBZS z8w9a}sIWm0HV9$`L98GMe)m21eW#PD;rsdiuHSY2uJ89uJLf*Hxu5s*Z=QLcXXeaI zFozTS89E~r%U*%f?G&yvZ!<9)kD|li=nz-iB!r`&d;^VE%Qzr3x|4#?g>WBBo#W2F z(fr1F2r0aeqYyeIxU1*ntco*>OE*Dlgp8TT7X9HfINo-4jAfv}zlH`|sFBGAYpIq& zAV{VxG_esjSbAV9nAKtq9nfiT8|ID!M50?>3 zu#KHib~5zd`7q9&l1r{*D}veOI9QySpMzZ+#P85iS?byBggF_0iqb)3df^@~n3_RL zKV}=;wF<2t8@;&!(lad{m@9d#J0r-8ux6qjYCL8Qgbn*)_7AQ=MfhWR5bE z3##C`J8L~I;|AChr9Nvu*IJ#mVcI zsnmlGO-)*sqP85Ij22LxI(ZYIS~;;P2}1#@Exc)^nmlJ$r>YYshXzz9Kq%X6Ri&z9 zCIDpuF&Cd{cE{$uR2nxTt-V#LirbWHc_XCTq+BUYkiR>X-g+l!?^Lc~u#4|f?xtp?wKOZ&0N5e0 z@%Jb#;U48$3pN9671%bg>R#mzwD<|$dN>B%RjCkEMHb-|Y znX79lb9J?Pf$mCLsB6m?>aO91x;u4mT_r5mW8(JJJ^e?6CQVnyDY{m5ims~9(!Dun z>#8EC>#adu#hjzN>(0?VEk(M2v|LwdTXns0t8R>(r)z4v?n>USPiekD*E%lH_4OC( z?v#skE$1R#uLawEk*<0x^%)hlx)y(}?pk%N9<}`@J$lK_x-s0KPfu^sqm4UtSL0oJ zbo$-;l+Jr}S0=RUwEJ}BBaG&U^={p^&#^vhLUHX{r zGAg_-ZM)ZHl*GEc1=C%|+<2F&U+U5am%6l}r7o4T!lf0gaH*jjmo}2)GCI$3soD~k z-cjN*(zd$v_N^|X_YW@5`1vj^=>nG#bBRlJ-00F%Zh~-sbNPDjcB$3RxU}?VT*hdR zOB?TT898scd@=9C?+X8M`G-Dose%!gFFfkf>PKDf#b3FM;V46^h&DVWF@|@^6vMTB zis4>A)lezZ4ZUKzp$g&*-{4|H5A9>PEA};1>p_OMZmHo}bqI7tDTb%^XakOaL+?1& z@HekARL*+Cm$u26mRM*gqs(w6gbcMdZ201@fN(X2PhAIb-e`EzZ#MMWn+?NvzoAn9 zZs>J?H&m$4(6;v(s@?0>N4#zo8|5|<_jId{T(>@!>sE=Ex{bKY+^Xa%x6yl@TXp}% zt*gJfRl`kgV^yQuSKH(^)O~K%bHCdk`|UkeD2oMKX?0EzH;m1 zU%CBje{$k3M&{$HsK2Ktc&sEamGddsi zxl&twYRSXCnC3@(?y3%7)Y@l#)01EFse)I1b35Pk8F_E{T%&LKw7~)2w9)r`M#Yd% zjehEj?fwk5edhCYed(K#5ao9p(f+CJF@8_~RKK1)&99A2^ScIO{qC34EA)GMH~SNMD*S3_yWcZ8Q_3~JOzE*OwtHdxo(AK)hGRKc_>}?2{B$p{H%aN+ zA#4w<(;zJP%tbr*G^JI{(_&-htF^ENulf$n`|2lL;LxN+4^QpUZqjbl|Lm$Yu61AM zx!zmnyTSjLsJ}+n$GBHbai1{Ned09tNwMzL)7>Y}aIcx^J_Ry{%P*xy?Zp=T}y|JzyT2qGp@1Cbht)Ig*LA~g`Hfk+KRY9LYr zks659K%@pDH4v$RNDV}4AW{R78i>?Dqy{225UGJk4Mb`nQUj41h}1x&1|l^Osewoh zL~0;X1Cbi|zoQ0X!}LUGY#8syhP$7_c;7Qxx|J$63=hhhOt&Y+hUwO)*f87=ZEmN# ztYgD)JG9C0A_UXMu!UQ)O*ZeMhIrxLZj_hY++?fz^JV0$3jB(?{!J(w-rP7X_KIGODsY!7977~2%KaH}~ivEgNGk6^o; zE!=_*OKf-r+oRZ~vOSvZF>F_|O=Ej3+vC_C&vq5t6WE@}_9V7=e;cNIGTSw5PhpGq z?xFuwwx_XO$2Ohq>1@wndnQ}Bl^&MZ@L6ns%l2%x>)HN}?eEzJ*k-cLV!MHDHrpJw zLAJSUH?qxRo6q(fwwu`Ey?vO!LbgS0i`ka2-ORR>Z5i7T+j7$;g-rt=)NbHK7Ax?G zi&ZO*!y`_%Dn;#5skm{U>7UD%!Y9xJ8Q}Tgd(ykd$?nDWShk0-Ka1@ZY#(MDmM>P#OhOC!=OxMHSamYnb8P-U z+PKl=Op*Vp{pbHM{D2ugR&6n3P6&5VN2#8E2Ybi_rE!XO{~;sfk8|*!&wlC`)h_X0 z!+z@TcJ<%)EAe--pZe2X^+R7!{+!a!{_ijQ-~JWi?4cG>+}+U;j1fY5dyN-}@{6)!&ew#;slT@37w~{xipZIsJdJ-zokL z-;zGTq5T=(k)OuTUGsMf`)OR=)t~x3>1jOO)&Cj$o!W2ZIO!KSahP3edKqlpNH7* zRKC;wzns4x*-!JiUCZAUMS7Y)?&=RklYd`_{HMf_pXO=1>K|r5&Bu22ubM)7r}7Q4 zKfxjWE2om4=ApaBzi}G*o%a92Sn|`nc~|`l?04Efzn}ig`TvpqG!Nf3{s(9LvVPr6 z@;kM!>2c(DYM+m=e;r~TK(ey8+L z*qiiDLu1+K=>3 z^M}&?$?r6OA3NZe{gnrj->LrhTSxw>4*kzl=J`^BaytG$WWQ7Y^5uOLzt7N52xk+Ck-W zIzHyH->H8%jQvjS|19=9&9AnwpRRxHx__=_zf=3VgZ)nJx1Ie?^V4qjJC*MP_B-`| z-?HCn|HeK~_3O0%7P8+d{Ug}#l>RB~cdEa0*zZ(-;a}1J^;h)wv)?KIJJ|1(|F_xi zwEwhEFpJM~X5 zvEQlugY0)I-?!{{s(=3rRR2!xYd-s(`qw1(JEeat`<>E1o&8SxulQHuuVlaO(Eqz% zr1Ck{{~;gX$dX{b1O}t9k1r3s`OC-uyXH8*ll@NlEBF=tL4%aOQ~U>hMt+~e{!L-O)BgQ}{Z8Z8>dz^D zC;j1Hkl#sP&3>o(!@=K?-s$*#*SubxF!9{Kz5U(!6X~6fkEhu0G=9IN=(_d34)yoc z^k4RW#(t;tTV{~{Oo#Y=Gs*9C{&Fb$6CCt^h$FpI`q#7HssDYK{rfq@KV{Z0$A1(1 zPjt}FokRMA9sH-T-|70nj|rqtaL~_RKz^s=dj_D`LX&^VV%4^R%Kj6;VanIv2A19$99Zu z{GTa&3fl~}6>RI+wz2JFJH|G?mi25i*jBKuW822IkL?)S_-k3uHiK;i+d8&wZ2Q=b zv5miu^=vcPRsik>gKY)dI<{?W``C`LjjtnpLj&11wmocz*s5E| zAI~<4?JBmJY%AE-ux(=7!M2a>2wUH+6mJ6C6t-*G=CKX4tz+B5wu|jnY)~p8teF?Dqy{225UGJk4Mb`nQUj41h}1x&1|l^OsewohL~7vw zpc=RYepttQ;^n@0Dcbn4_`7q7;IG;Q2w}jQ$@hc&brty$^1KQsK@*JFdrA4K=EJ4@jP%c@$sy?jQxw4 zA7=k_Ghayx|4ur_C-oA|@ld_Qs+w2menkjkE~`3rmJ>Jhn4GGd>UWA6-~X(B@!tFr zF8>bB_Yqt_cpn6oJH-BL9Q+mRKZ?s^umA1r&t$#bA7=j+_8)26Kb7onXa6ZSe--;L zVgJwVKP>0*ynoK(cCvtZ1LwQ+B-)=>vj3mDvg4%a8<5$}nqVgIe{ zx3`B{_D|j)laCin$6kNW@$v99*Vhxw-C#Gu6Nmvuyu2Lh-EA7<3Q%a*-AS)gLysMVz$yxNPln@Z@-c4Q*3vz z9boG{k@6qQb{5;y*k-W3m~9=~JJ?>z<$QqoZMJT1mowPz$M#sZ>(~a^7O<7$p_=(7 zws*2^W&0@G^SOR|m_K3bUrpuOk8Kj$b!@BHKE(Q$xE#0h_8zt$u${*H^?0@gY|mqR zIon3I(oai&EbUI(^1$G6r@x5U6dD2GMBKHdD|au&wzkKW(19PR(=^8a=C zJMO+qZngej&C;ZB5&i$W`a938&7|<8N(HcQ$2lP`_U(8a;zHY1*aY~%<1(^DY9LYr zks659K%@pDH4v$RNDV}4AW{R78i>?Dqy{225UGJk4Mb`nQUj41h}1x&1|l^Osewoh zL~0;X1Cbht)Ig*LA~g`Hfk+KRY9LYrks659K%@pDH4v$RNDV}4AW{R78i>?Dqy{22 z5UGJk4Mb`nQUm}0seuH@N*=5k@DTZSu=os`EN=M%HXc@dP7fFSu2MZph4HyMT2ME! z-F!Yzw#O?C;@04gpJ*AdR>1lfSes#e2G#*s$6$@ql$HYP?_mwWdO56rf%R{&w!+#8 z>wB<{z&cM?+KI4kfwckF7FZvL^+j0UhIJU$?_iB_DQym{i(yTM^%z)BgEbS@B3RFd zwHDS!SX*Iz5!Mk{;c0p;7S;ql97}nj~N?Q)= zxv<^^YcH(d!8*qSWrH;x*6px1!P*Jy%dif@`Xj7Uy^tQP$*`uunht9otQD|U!CD9F zU9dh5>zlBSzzR=PYH>cLC&IcO);gbZJp-%mSFU4Vt%G&QuZ)x^W#qtmQWNYAWLVe2S^;Y%sEjDx2tW6MI54N@7Vg%HTi$}gC_Wn+0(sTo&UC#t^0C%SsDLG*Jp2OtD^wcD36_OQiy`Sr zTe0!jHer4?7ni{v%qlAj6hY#m$S#IfxuLW;De8R*to@6SPrc%K!u*XO{zk+VpXfa=Ca~a$P*+%HC~pT7t9H4%MTX7u7)z1 zI}7ZV4doEJtVr5KSy^^g5pD^UntKQ=S1Nofz~%!M{+kvK1-9iozu>oRW!a&E4cm&c zuXB}qeV)vXD43(D_nWbh~un0Qt z(k!eL%r{#06u`lAW-x0LXtJPMp=p}26_#yW7c8>Tx^l?{_TYtRrY_#n=YTgv>2i8fNIi3iFH1 zUefeKqtD6>t}MvPFSH~%IqE=Q<#Ez*s3k-Ava-{H(4|9L*|i}^U~UPN6X-Bu2WMMa z&dJB7AHDPFI%UH-P(!=48`&HP6>JG?C@v0_g-W5fg+&Xf7t+8QYUt_%;j`_U1C`Q` zt$;leO2>my`B!Ic3zkl_X*JDI9S~IMrzWde>A@oCj#RRtYCveHfnDUQZ1OdRsuOuS z%gdB5MMzthd@}6F0=3Lqu!*hP=!Hk4^*SUrUX7mv*TXh_LwP>*e~)VBSofr6s3voB zLDsh7@=!oMtC^#7u#5)7J_t}>hwA3PfZD)OsR&yyR8BcGHT95|UkF`PK)tOM1uH@U zA{gR})VCUpI=T4;1u(NHhc53Yh^ytn-5{=-&S~)3+YEn9Q7zVM`Fam#eQBMw>6eufzzEcwT%9SEt zzXx606Bsu9M+}>nEuLEpF325*p<18EAYJ1#;IsA(_*;vv+=C94ke$X()ecehfU4#L zjFO6>UNb@^qWI3BT(RpOBPF21Zl%NQEU;N!Xlz#3yCGjgSf08+VR^>yglf`Dvx;Ds zalW2#^OvU~?BL%pY#0=_P#;LBPu$9dGfAjes@_Qz`>-1;mNu#9-8gm1&z=-eeF4dg zUJeO(a)Q~#InZ<(T`;)~;;wSsVIR4QU|yV6UY1o5aM*R9yPyHXbS|Loc1y!kfAYZm z2j&#cF?V~QZ9fICbWx@AcA+ zg;^D{wb~U5sdv55xc}sXitPOiKU@EbpAGlGJ0HQGS(p1OgSVIFf_}Ta> zc2Tw6aHghj+QQM7q(IqV%#(`wEV{IK>qLJu8VynO!pnQy*)Wv?$<_dzios0d+zCN{ zbS}gJ@xeS7y3w7E_oC?#C=G1Rn&<?co6Pe#$-(V0N-xPO#f_z# zinj!yzH-eeRWPu5qH}sJdP_1K6*0;cRJGChrO?cvTHv67Sz!=okZ^iXjKiVC+7n%- zZkPgxX(yZ*tAfZoyt7l>>VARYBl)!wGI|FRId<=HOwU*vbu*G(~0e^#3H$zQ6OfQ;V zmJN-wZ1TZeHQi7nFXKU0_ufJfQm3mPoz8uLdJ+S?h`FllT%ZM@$*F$WRR02MM&DQf z9d%H>94qZpT{(SIP8LiQrWa;~3d;*9^Y200<{2wb0P?cc=hG)2eqTYPEqCD{)OzP^ z`0T00a+q}syO2ix9W(e#&kU%lyD$LE{h+E}n^6w)2zGt}&W?NIA&!xU@w2WAuBy3s zbl@;o&g3t(Gq=DYIFZ6%WM&O7MJ2OOQMDlVj&VKa~4pUspJ zu`><^7lhzx{&LqW zn85x6L)ATiIi;l8xkEzrK&YyxF;u(xIsPvU8z?TC=*hmGWou{SvtY(xA0$U+nN3n% zv>WWs9&Ej}U6?njpN`$E_45pbY3@KTjJ*>blIq@V3l)B%ALPsqLin!FFpu4XyOgAM zHi!DlZ0H9*z)*wcaWkpBk4UJF*-D50G70wh?2wr$^$Z9HUN^G?(UY3f^%6F&er)KIci9gT$HlutLV?0!7yv(rFXav-pe}>WwwX&)Je+jfV8 z$lN7W?_()Yd$jpbrGilOQ_T3Ty~&%rDwGnZeUr zQCH1@Djmbx34e}z%&c)L@uW%QGjnJk{EVx6j`Ht32T>z&=IIP}(&njNK()*=&p38Y zrp&3XdTo}JMcuaBCiU@dn>VR%XT#OXM|OwnR#(n}KB7E0(dyL8bCiC_(t!GYI@I7E z&~~rK{;}o?=nVC|EVz!Bl`~PQ@jZAiUNjdDmv0gv2p#6CYA$5W155CKVXQY`!nJ?L&yHqH!IW~kmGk(^<@2EQ zaD6MUxO8IZ`f*P>wSx{8E?);!%{-VG+>H^{Ktynm+=hkD&MJY6CcyyoS`(9s7v@p^ zIr3K|l{e-o0~#HUmg?iV*fFSAxv1*Sd6QFn2U2VP9w&3%16bl+`lv_ebM#Nnhoc~% zu9`1%ezSz?+4)f6SK%zN&73P%HNX^)htbJH?L`SZrcN~Ygvp)e_4D&gwW5-QW%X2vciFp&|%#?UxhadhExrb@*;`{@#e6;lII|fNoBAcea8{7f`o^ z=SLh{^#TYrF>AFr!VAoLK%cs10rV^v$3nEdT{vMHd2A{~j{O18_b1JH-(SFSe+05I zbB5L32m)70Rz76+^94UY9b!@KLjLlXg?LEd4#9@nuyEOG6l2Xy$<9zv?%jzX*PRQT-QUz(P0!D1Z}!;6&X#@EZ==20`1u z!yQssQW8*)|Hhou2P<&csGXCT^mO4XQ5DTCn>Q%>;Qc;lTfpbRUCs z8_g6O|BYE}qVG@<@aqoWFFd!y^rptTzEM|UHcnz)O;$@ zoXV>w_k#AO`W9|hHG9K(6P%A07Z<{hF6O9bo~Ox)bs0>+VS20HT3}8hRny)u|NH^l zeU^G)Z*yls>!Yi~fwE2cCF(s0G4dvM7^4^AT)xXZtM0waoY%v5=r=du1^=2w_yZXH z=5yU5m|Vh!`bE&1;rEw<;t-yc!IiC@jqUP9rEu*RT1IYe5u9r*f*t%}G(uDmmS!~ zIzGngI|+U|f&=jvkHWw1GpRr{bZ!h5@fB)y%RbQ6{}+1!IB&dT9~fPKJ2~VfshZVSZ0KdN7wy${wKcF7kw*-D>48TFAeu8*w@4$Gg z;dAog`}}?!?@Rl^(cO-LTH$kYpb-i5^?oq5f=k)Za9-b!uU@KgP&N#qvZD@0Db;t= z9QYi*aSrV5NpsJZC6+1P-2^jzZ$(JN;7*Y z{2}#K*xdg%Ua=Uw3=e+yZ0pV0li^c2!0HE5EPB$x`vr!2Tx%iPJ*cc%$3ZmXy8_NB+&)6 zSCZg%1ioha<^dZEU@p~iAa&)cHVKY+{83BYk_0^yL|$voS$i%shx9u99Ea~X>o8A4 zcyG5K#9#h-5R~A1Xd|v&2Zz5OWcHh?O9H~!BX?mCH;D@?n#EB5&Af| zfdZU)#i1g+n)M(=09jJ6L)6vRW0K(}{OqX5&+dEhv;7W8(oLsm*~KvM!B1e#hd}Si zm((6O1Uh6Ks#Mz{@}uG7hfrEuRp%ivkm4_D0rgoj&5x*E>;MOz5aI+A1`BoPQtNr4 zy7W*umi~rk|Ksn$5aoret4j~cQ=c4c-XXB_WKphso0q`1sjHXb*qRTcwV6y-*~E`3 z*DnpJn-8T{#rya1L*bfW9{d)G9e*dJ*bPkyuKT^?)uj2Tmcwv#!p)VRn4e$cXYGF> zh&d-H#*2LFmBZlV24aO9LtZ!x_SGApsvgFWz5l`1iN9z;aP`Asc)Q5ODR7*_jYiN# zn+f)Q24TE(%fa6dqhtNq!?KIjr-$XL%Tln2>aQs{5mmRO(48Z9KosF->@dRDyEQnJ zLh5bURDUD3o-w!~2|CwexE8)meF1uNitr8itKY#?Yhcm@UDBl<4Etw{q_-BkKUUm! zIQ}qDrtUl(4y&A;QurYx2d?83Yzx2zrEK`s@ z+a+MQfaY5Wz02WrZOQDn<&>vr>qhk@D2K<*JiLRsYI_%FBxZ*1Sq800J-7@GieYnV z1t%N|PWrGlL34X}84o#@OP*cJtOGii=S7I2^;3+Y?Q{ImCJTN!? zEtjQhj(|=Y&oFwI6++RjI|4@Mx6PvU<7cmVfL1qHs<`(EuHrW#>}VBMCHDmd@S6?X zBLL@Ra3$yns@746ADT1N$Hp@C;t^Dn>boOU6nE4=JCqHkSdJdNAt4{G#KSaD{g7g* z*%17QK<(%1!}E%vC*&OkzepFApUK^9P=LQefOw_brDe^)hy{;kV5hcOQOMzlC#c z{!L~4$W8h1OVvdy%=>z(R^VWc_jcjDf`fd0Wd26lpsrj2Q+P9~&K($4!-Hnz=I6iz z_rmJiLUXK3DEy^K%^0 z)YV79@X}=_3yDJspoG^Sl~*3Z>QbFYm6yS-VL37um%#)Hug$9+M=8$+b;k<4GY(qV zvn#A+dtrrn&&Vq)Ho+Y{F#JLTDS;m;gX)bHWvb^WXdCK3N5VPf_mK9uS@4DruxK*T zvvqhKsWi|(ld5K9sXzOqqvqcYol^h=aitjc3Q5wE*9nhYGjW*^}&3&p<^Tg z``)$%g=%PT{wWj&W}28z{76-~s6<`8&t~=2KBem61IpBnLqqDJRQdr@b*Dm;e_ez9 zF!qcF*@R1JLG|I$batXXJsK+T-{@@`$Ik)tv#K9M=0N$)-`v#I$3UhIS*or-hW4P` zN${s*;J7L+&W52Xpl&<{W)3`)`SNJ6K*(#?A2k>Tf`zJW zCGJKkUG7NK*#<214nC)68_L)5`Gv#hcTV$Glxh{Soq%2Q4m`Z4Trx zG{Cg9u=HoRxUj6to{=<$uY`dul_vqya?^|8%SDa&j0E`4-rtsnutkvexEs@XfLw|G0X`n^WAP`xKLCZ0)x`$oOt_Gt?8+UcTrY%WN zEJ&#`4BxokXv9=$UHX7F0CyQLuQ3L+5u-+LH4>_gR#%^1Fy`tp65@=pm>PXmrQYF= zi;0PA@is>FyLz<5QFp(V5!2)DgnJ`mTC@?p5+ibry1EXmc30H6D~)lZ)>El9yThJ7 zU!SYXHRgs|@htE2Rcm#=&Z#|mX16akdCcRB8PM9ber-t-g`jzw0gb8)#h&Sbh=xOaedI!r474#jj*?WTAwil6*92r zfUDD0@2i0_jk$-Q%DR1ByAOD)qPjf|z9CoY=)$(BjDf?(qkXlI_OO50(`dAK20X*Y zxNmS?a)VL3ceT5DdX>A`Xxy#RGd!!-U9~)=(hX&s-nZw7yWbP`jC+Q>W8UNjZ&P%) zZy0u1WpsOVN^@LBwXerL?oJ)_BzHt5)%p{MqLRD(DFgoG{+QLhGcsGIuI_cWM5hi- zNo)4i_(qf4v}Ub!%CKu(D{0gF;r{S`%(gbD#f&WoYhhpge(k2C2hc^f?~M%Wm0 z74-QNYkY}Gz5bz?f-z4`Y}53{=(_0EDBrl>R~0i79jcD1@r6Qz{wg^7`lr=5224}YU`l1@$ z)!K-=#^~^L8skxY8XTH~{=OxHS{oc|8PG*&wOXCFBn*doTDO*0@9Lg9>Kc!3HY)X6 zquN#Htv4FHEqax^#i;W1`up9zo?&;tzb(4YUpsBw)jOp{YjV}P$EQ>oHAc0ue$W+m zx4Xt>j7(qO0$G8b>#oykwD9hA&~wt#yHu&EKuvoo^LdFO=D5Ygo@DYD97-I|JU5ki zKjtdtBbi%|BmY|F&{@PAn45#dA?ERX;>$%3e`p9Zc`NhKwZ#9B@YfUXU{3!l@vGv$ znfO1b^h`IS5;xKbXEAj2jqkkuU zoOzu2-^@$?LH<$ZWajBm@3?=KGw;Kk#(XUEYUbZDr!#M3UeA03a~|^}%tH@T`tLE1 zFi$;<(ocAV{D&|vX3k{JVE!X>Ci4T#HOzg?b}3>xEc4OK@ywf; z!zr(C^RF!Ybod z@}~h~d#GYAXKufn{LRb-&BU)TFS(c4eKhH7A0j@CIqmPno0tdxLHt(<-$wibv+*eL zH_Q#p^N*qUEAbgSrPOi4PZF0hx9%Xmi8;QT_&MfPFA#shJUBp{xRT<}d!P7Z=Kdk# zt;`L>#D8N>{f78u=8Q4oac1LN;yvNS0_(RPetX9Jr!n{H_^h8&zhlmrMqI|65=&gi z+_EQeJM%yi@jJ|mk0#cRrSuZBh!-*|eBKb!JBc~&PsHn)msb;)GPl(eU&Xxm8{+$! zm&C&}hY;V8glCR{c?6cX1@34;|5D}xxJLy!hq;~k8s=oU8wC9w%=OH#F;~Ft2I&8u zxs7?z@syqcKR=;=J@Y7B$Uy!Bb30rqKyG2~OC^4Vx#}3=&zLh#Cf;Ke#oxPzcsX-@ zHu2faqe0?xnfo^qU&~xsLHrq7B@?XuIQAym! zocc%N*O}8UBmP$Ww-L`fk?zc@ zeiFsMb~f=5%p*&POU1v8_-^L0(}_Q29z2hD_G->=CGm00RXd1_nG@e3u4P^_MEo>! z(w^|#E9CD7=9uG&mz+%TcQT*JT(XM%+nEzjA-;(@_Eh2*m}5>O9%0_jyvG`fzjYn? zk7cgG=W`)HMa-jT65qnylR^9>^ZK)hKW0wzJ=*9%p`>xh0Cif66?@Joi+J zKRK5CM=__&BramEh$sFtbLSq!e`j91C-FzjiSvl#PowyU<`W;uoVXY92Ikhii7#ee zzlgYzxo02Z9_G;g#NRWoKA3pXI!Z70Fyd308;&48k2#b|d?Rz_vBW*h^{a`$XKp@~ zcu_jVpK=EAsmvL_CEm*1^*iG0m@6}hTbPG85WmbE3KIXo+_sT;;pvp#l5>cUV_scI zyoouzg!oG4lrrM`#lMC4Rq>xk{1tQa`NX@QLFqMINW7dmwUYR3=Kf2G&tq=CocMO; zxYvo_U|!uvY@A8)*D~+L9P3rry_#FiKR_1Z$*O*7f$o~`b z>Tikn%%Jppz9(MA9R7)T3-e+ZJbw@I*E6s75I@Si-ACLf97R0J9Eu@MJd4unpH6%% z^V}@rvzcp{@wxzx??bud{}Xdhi1-fX;jP3^F~^@r{I-O@fcPioF1Ysy%d_ye6o37t z#78raRuOMxj=!F`ih1n~#J7td?p(t7I+^QlA|7InyM=h>*_2-0ZNw?egO3oeVa|M% zcr$a(W5ky;w=mzwY&=Q+e&#Uq)b*5J0lX^#^S6w7_*vpS<`Dd%g8r+SGoB}YjJfLt z;vwdmUg9~wqxeJcs|f0kW^R6uILK@a5nsvN^fB>2n3F#t{)jmdp18vJcKaer{sUoK3ug zIW3;}W9ISQiKk~$d~@d#r!t3_GnkY1B!2;O!aU+S=2qrsm{-py|FkTMzk>N7<}v0o znT-V${vzg8%+1Vk3(4Qhyq)){ zWnO+N@oCKQ>BQ$V7o16a8*|2Qi94B-1H>OQ59bh14N`iY`NYZ08JmgMF{hUipU1rT zT;iLV`%h2 zqcz0SH&S{n*ATB@UR+OH!W@4)@eRx&=BJqx?;!tY%+1Vkc@%&BUF1K4xdNU|)09#r z%(3?p-_G2?{1$Vjs+0UzGAA(K&78{oB6BA5 zhsH!){2KgS$qew(?0`M=B^%u_c} z{sx$jVOH=kDc1Ly%n8irF{d)OFlRD9%N%BYlevL;uL7=r=EIo>m`j<}vy}d|%n8g7 zFsCv<&z#Bp5p$UNH-%jP%o)rb%w^02%zt54U6lSmnG=}bW=>@uW6orrSw!iDnHMoP zFt23pV9sV9VBXHGp5yW}Cotc|oXY$Hb0+gA%wguZVlF>(Dsu;O3G)E+#ms64rQgb& z!2At!D)Yh;ia(P%l{w6OC36Gwv&Gd$U7*7pE2+-5@c|01RTJLUxD z%b8P|?_kbkewI1R`~h0P`|t^)KH4%n8gP=2YgJnKPN&n8VCH%ni(6 zF?TRe-@@f*K8RWMQ2J|`6PQbwQ<)!N&SV~B4l}E*6n_Kr66Ox(qnQVo^O)63l>W`k z3CvG0r!v38oXPB}p!CAb3z!?2)0jJ$vzZ5&E1A{HlzuaF0`m*ZsmwnB!$aRHlX?Gb z#EV}ceI|1a^JU0xrPjg|=QzGx&z#46J9C)1jk%8bdFHNHN#DopdyV)@=IzYW&ZYE5 znD=AOd!53sVjf`5XC7s~h&is0!q-Unw}`u#2JV+HvOnFrE{FJvBLz7}}i)Uax~nEWp?_b^Z04(ZK;jC7EHDf24k-!u0xhneG_ zr0};eXEMLQ+{FA7bIm^~{Ng`QdcLQK*D|kVzJR%oxs_RUQuqPp6z1sjp?tHUJ#~@) z0OlO#)0p#^w=fql-@shL{0ei3`6uQI=7TSw^tLnSFo&71W3FU=mbr@g8|G@}C1HxM zhWU5QwaixoPu(4UAA63<_XKkd^S_YiDAo2n`QKvhVb(4r{}A&+X4OsMQ<3+8_g6BX z$vl_2k~x9-UgpKjuOiP?DwX*|X4Oyee<@raW434gA_}kO!ut}HQj3JgV@+Nmygl9I z^~_^U#ODe>L;PpvinoYcg`-Y4^)Cs}ApVqj8}qD-DL>mUC;ws0;d9P3^=An0d$!4& zm{XS#hne$EAih>OgZM7tjl@qdk8B~vI>^BfUV6a~r-&2F*EXI}sZ=iHB<{ZJU}jGD z6dVBh)G73R`+jh(!onDJBCI|P3=8ran;-S7Sighy!lLiz2t+S_?4S3x_0P!G{u$ZY zKO;;3jPY5^e+rb(TK-Ko-U^K6Kl0wmJN^zAmi)sOl%=)*AVqpTMeJv=OCF34*v;gL}<_ha1z^@ICg z82yQ`7U=K+3o`1he$>nTTX;Vg>V+{p>NBRo2Q0{_xB5{p_j}>}U#J(x@ThO`!v`$L zsJHr2KR1PP+BV@dxZ>xD&cZ+})l>ML2_!+K%S+uNVj zkNVW(D1HADsy|`Thiv(``cW_U@8SJ?SpUMJUu~;@s~`31L`r`qr!Oq}DqH$iKkBPk z|2>yqSoFiT_G|T{UhX%<`wuby!lF;GwO^|r^|c!)|6g$W!lJjgAFChr71^Z!XPLMh z>r+`TEc&^&@z?4{{q{>K{Tc8b^DiuVxZS~AkTL&OKkA$QMEcX=JL-ku>(A!j>PP(m z>yue8Ec)e?{6zg({ivT?P3doCy|CzG@e3?yF#lFR>X%$WzU$yS=3f}Tn)zRaUtmFl zdaEDxp)1MvU+|+|7{i0_z_d}cMGmE|%zrcb9^FMC! zqh9Wp#`~vHFO1f6|J7L8I8TD2_>bKudzH=ck*1s@@NBy|1{;hu0%l-fO zJOJv2Mc)N+;DU_lTm7g{eTaMkIG0DgFowtU?c=A_Z>G=sw^?7oEPA-z++5I@>0A7$ z@A^BXfAd00Us&`ot(&B``cW^>JK*yVn15l>^YAxOeybn#@_Yn7FM)bt(I?r`xB5{p z&r{&@6{r^$y*+)aANBJ51wM~~dSNHM)sK34UIU-sK)tY&-s(raJl}!Od!Sy}NpJO| zUY-ZR=R;60?4-B)Q7_Mr;PWJ?7k1KH{it964{AU7{0ZuXMIU3^e^x*0<@pqRUIq2S zq7U1SKdT@0@;nPZ--3E!(cAaG)sK34{so_hLA|i(?d{L%N4-2RgU`>PURd<@^sRo> z%kwq(ybbDwMQ=~v>PNjikAu(Wpk7$?_VlfO)Mq|I`wySzLA|i(?deKzSWQVI!?cj^}?dJr*HM6zKiwvd=aKEEP8wTRzK={SdY&mpd=%=1o%B{e>PJ|Q&r_jZ*hz2oqkfF_`1}>> zg+;&0cKloYs84;A+8;izg?eGp+mC;%ANBG)7e3#GdSTJq_rKMTdU^f}p9e#|u;}gW z&+0eJ{}|;TpC2>J&n$X-`tp1k`cW^>m*Mkfs26t9Tm7h)=h5)_G}H?_>8*a$%kyja zJR9nTo%B{e>g9PieEtpf!lJkL|5iWh6CbDgYno4bVbR;qKdpY$XR;n{3pdjj7Cp|P za6w*U36G5WO4dKMJL!edk6L^Ft$x(kus(Nh(hG~e1meI28Pm7=QLmmL->b7oFD&|S zC8~Got$x(Y^M&}lA=baJ=toIDk-pWB`cN7@&lk53>4ilEh-s(qv`iZ1}gZ09quORt^-s(qvH|rnd z_9HC%44dBSM}6W+l>Sw$7Z!c0O>gz1K8N)=^SJ()MGw_#F34*v;gL}v=J40sJHr2FVCam^QqW=gfTqon{E5w>PNjizlzVZqFz|^A^ZXh8cg5nN4+|O z>hB58zc9Km{T5sLRzK=n&LaH+u76?CcS0PvAY=MgKk7rjCH;TqQ2vEQUq8c?Pw1_F z)XVd_`1~%GUs&|^9wrtbWwT!i7%^`!wfYSoHStTm5GGtZ(Q1&t(>UJj9I) zGUngvM}0Df|B&^dRzK>SS>MLj4}?XZVy3<`eXAe!gRDQA%P%bYRGZ%FM|~nc zuYRKRpUk4S*Pnz(KkDUqcYOXG_n)xn8*S&0RzK?H`FMO@9`(YaAI2}Rpjl({qh6k; z$LH%&FO1>M{u95zf(G?gKk5t4r}ndE5$T07Jn9o{`M3H}UmYerKF^Qo3yZ!9;=lzN z)3^FjFVFkq^Z%$9#_*W_IDUZz4eG6a)F)j`>0i#{uP}y3{pwBd0gLFZe$;ROjPwnQ zDSctlFCqDa-s(qv6YCY%pRnlDZF;L8_2aBhWWBKHu?)B%ud#$jMt#EPl>a|){)N$x z+5%W{K}NmRkNRrXujKj@7QOxWv-(lr$@;4xG2DN`qW9U_kJXR*?k_0+x3gYY^oE)H zo%y%=QQyP*gBMWx!lJjI|5*K~?_>QrJpK!d9$r^rE@;+R!Xu+T{Y%P!D(i*OkM*Bu zE5Fr``iiee{}h*BSoHS($LdFYCF|c{y|C!3AP=}8WB#pv)W>{H=`Z5^3ya?V`gD|+_`VTrzrq+E)3=}hS^cOtzNhrPoPS}_$J+M4)o%llIB{VAvy7QOxVQ>!2KA&uS- z;gb0)v*_*jA4qugqh8+6g70g=^o2#A0O4>!USsp4Uf$<|?{`7HFowtS$Jokm^`pMm zP5J+qoAkn>x3?dwANBJ77<``$rY|h|4EzEM8qB}dkNN@VAklYg6zPR=8|L5Me_Q>i zU+pJ-KI?@=4~LhzpuzO5e$=N$lm0^13uAapf32rwNk2BRf^;SRXyI8+Bw?ARg8#cYwkNR%b2UssG`U($4fQx}xcqCGMc)HHT#(K5Eq>G|>_z&^ zST8Jk%mXf>xB5|^#`+enKVi|2+S;Ghk9v9E7QTNA%P%bYeoB5KeXAe!@_sISUl;0y zMW12QTm7gn;P-pY;_?fN-rjz#e$-d6{y;9ju;_bi<+u7#ziJ89e?IGlMW1HVTm7i7 zK7jNWvtC&AAzS&ae$=Z2Nq;iyg+*__{%rN5zJ&ERvR+v9_WHB>QNNw__i+6Si@wE{ zf2$w$OOh!6@A+x}3yZ!4zrccKjU_xX>gx|8{TS!@;C~h4U{g`T>Xs z7i3J|>c{lueRBAIIc&ee7#`Cfz%Q_%LA}+FdU^jGzK;&|!WbU)qxc0DG^n@wQJ;Po z<^Mw-zlAY8>bqvc2P~qu`cW_MyTkY2VfwnR`0)LFn15l>8#cYwkNVa#DE%8bePPjWx9P2Z)F+)q z`mPLMj zK>2@y^DiuVWy`l+65k(*=?jYaBj%4_4Ct zKYKpug+;&GR(`7=^+}hJei7@1MIURXzB7HRAN8rMf0oaGghiih(_8(h53&Bsd6a)) z(O1~ouhozGVHjXAte^8QEPDI(L#rS4qpS}tr1XVFZ=b(d{is)0Q2H}iFD&{2TluYi z)DN&8-=~WEUs&|*Hoeu4`uZy={ZIEIy|C!*^Jl9c^%YlaBj%%lmQheYvO?#_*`Gf)y8J)LZ?i zm-p%Jn@W0N(cA0a>PNl2e>aWw!lF-xaSaz_OyBB9y}YltJeATH7QKD`ZuO&H-tXJY zdSTIb;1^iXVER@+>g9dFucT7?!WbUwFVEJ0Tm7h)_Xp$qgt7jGMQ_i))sK34-|%6q z7Z!a3et`uI=HKc^y}X|o-&c(33&U44|Mv8)el!34K4W~pv6+8n(U;ifAGH=g>gD~% zFR)%%^!1o|SdcOQRzK?HeaYV(P3a3`c&z_HTl!W%>gD~)_`YS#zp&`-^GB=S)bsn8 zj}tw!=-1k=zf@ZMsPBD-j(->2r-A7Ui+;d1{#gB}m-j!Ha{9ue?}IwP1$m7nJTmI5 zpQrRM`wgWpjDD;?`~1i1N4>mn8s9&S`4<*_5`KXN4W@7Pqh8)mjqj^Qy)cHy^v7Vu z1sU~LKk8For2K!t`4<-bc3b^h{itt$h4hC*Vwk?L=+n&9ccyRkqh9rr{$WmESoGaG zMSzU?xB5}v{Tk_~ar(ldw~s$oKkALwN&kCJUs&`5w*J%VN4>lc9N!O)IeM#x>$LR};-k!eIk9r^LU3*aZg+*U$%fHo+ z`g+!%$Maud(Kp)a&+13Lyw4rq?~e5^Ec$r-0t=coHb3g+{qHZcUKqpU{%?jA7tve& zsF(M}D<<80|${iq-E z();08^8K&EqKDW2nhWw8OL%0|%lq%~efXHZF!~c=wU576Kk9w4l>XJ6zOd+%AP!uR zF@38a_42-beE&YCFO1PLMWTp&VUkkc2&ZK!AYgx=~$eLL&nwY_Hg z!lEDL0|*(@xB5{pe;=SOjr78z9|s*SqPO}{-wZEVL|+@{Us&`lw&UOGNB#P}NUxxu z#QGB!{h+P>tbWv|?n`=j&8w*w7X4hC-s(5gKY;X6eE;`)X3^XGZ>t~m^7j~SQ~8pkJXR*{#4Sx&U#_dV;s03WBOJ<>J|U} zhuc{%?4-B)QJ={Azj6Hui=L~0B7LhL_44;8@b6Jz`GrNlJqmNbGyhgU>Kjj@^6$;* z3ya>KzSWQVCf5If^}6dW%g+*^4|E+%1C$WAvzWyXE`d(Z8S^cOt-~ttf{llJAeqqt~*z{IE>U;k{ z`hf(}3ya=<{l)4>z5IQZgANBJ0Wbp6HVEYpm{bKwA3mVM7)sK4l`!o3W zXizVV;W7XA>z7tP>Pz_V*KA85y|C!}ZTsKqNB!z+sQ!Q2jr78zFR=ArRzK?H@8#g% z&%yi)i$2A6{$TZ^e&i-f|6WdCSoArz{9FB~Pj4jsA6PFe`exhyv-(l5?k0U4loRtW zEcy!AgbVT-n;-QRtbdWy7Z&|`+x*4qM}6;ulzuXg|H7hAwUyuMM}6GGq`!>&FJaM# zZ2gzjkNT=-NI!@BUt!T_*z#}nqkiB;(w{k(>Q7koYi;}A>PLOjOQcU>y|C!l+w@jH z>Z@NN{k5zY7QMaywfa$C%lgf%7Z$z!{K4u+efR5>{%8LWb8jACWBJC74@ruOtd+7f zp^|klN!eu?W08OD;3H59UDt&k*9*0Pg`N?B3~g_5NtiBRwLx$oDL$C=M_U7vg3*OJ=*f`y-Ar9V>+e#BAD|Ca3+EPUJjrX2i6%wNTP z!NNaa)qkcO{Ng`p`!Uam^e|0uWL1ql3h4W%{;I0^f`w1sQV(#np$`oH5#~p8`6U=~)ISsG z#RCk!DF;8LptgT5k3R?&{tk*4Ji<5S;LCk#xL*z7Cs_El{iYoJbu_^WUEIe8zF^^B zQMUW+H|5~VeQmhE4Sd1E-(c}gIrwtF8}54pU$F3FEWRlRU+#m${czw57QS8om~!yt z{y5wx2fkq8r&#uza`5H8xh>2WEc_!@`D@C-m;32(UmffhEc_;x{iYmzxz7&w+kr1w z`1bleQx5*JN;>}MQ#k>?VBy>KmnjEd?$4|GoaPG_e#K(i1HjRSJ}~&<^|k$pxA#Aoa`5H8M%>>B{|gqrz5mFRgD>|x;=V`l1q(mX^1mquKeCf{ ze<7|v1Pk9zKc*b~uvpEn#(crTr{R6S_%r3;pX;poC%FC>Ec`f&Z_2@M(oOTKV9DKPS6!(R~e!+Qs z{6PSciOgZ>+A1dxgh5H2y|B97+f9hoB3l{!qi*L%o zm-|-NF<-Fo3t4Dh;d@IcdLG}DgWr0S4!^S$PjJ6r;YV8GZ_2?Rwq5flGheXqyJ_q5>^J4$%YDGO z9~kxv7XEP>m%#%ZZOOr3cTC$K&-M!z{&ma!rX2iLr!;>f^92jPh_c;hzbOZQ1M}yV z)9x26e7pZ?%E90FtG2&C^92jv?thzd@E4xf{O&UU!C3g`tn@4Tkb^(zqUO^z)++o3 z3qPI01rKnvp$`nc+~17*oDqM5AxHWTx5_V54t~We+Wv+d{(^;H%d+2;gCBEE^Y3E5 zVBz<%_@*5E75{2}tGl%O1q(mja=$4DU+$~M{nhZlVBypC-0A_2w&dW?yj|ZfO~s(% z3l@H&wSLi*gMXbS*rB_R`GSRSkAItT@N1RU{6bXD!~KGVU(xcvDFU#${#eU?Qx1MO6_C((Ty z1q;8BmHtgR__?h$|4nwkVBw#me1Qiz+Rz6EKQUJGuXFkn3_0R&s^x!E4!+#ij{Do; zf5E~pLi2iffMLHW2OkTh@Y8_(FBsooeR4l znR4*Q{-pWU*#ClsZ|8qg4*mw_M>Air@a_3;Qx5)h+BghdFY^TpzZN`C4{)@h4-9_w z)0#h({Vy1D#NSja{!KafTh3|zlbrtr3m<0S0fzmi9Q>2aA1?97SokSc{u6!3!H>GA z?XSap!NRx0-;{$tjQL%dFIf1yEdQHw@LT__?JvT7!NRxmzbRMtGe4X6KMZ3m{NuL$ zh8+Ap*R}nPnJ-xQw*O5z_*0o5E9Dv0R;IE3+_CFY{!(TAuh`%HB;sFNVl!HGhPV-wbU$F4)_J=74|58`Y&t$$};oJ6` za`5+c)BHD>FIe~!l>aO`)nJ-xQS1i6M2S2Wd=AU7{VBw#%_@*5E$ex;C zjq{&i;TN;`rX2hh%&*)*$Dd%~U!i&(4{)@h4-9@I>fqq-i(LN;#y7=<>2ostofsvFBtl8e=favgm22hZ*qs`zsG#R!tbT& zd3;k2{v76i!F<8OpJVY&Irx!9wEgFpFIf0ZEWRlRKbQGsVs!oyEd0Y3-;{$NLl-|l z_bKKJ7Cuc&s|PsR&<6&8CiD9;Uohkd|3rH60E2JJ!N1P@$;=ll{8ftWSpxl!JeT`RAE0Son>s_M0gOKccv{zkEj>{(^;Xj~|A{to;>4}QQm<=~&Lp!wxG>F^f}eb{fupD9=NKdSl9Fh88J@F(F9 zdhny{H{{^2s-pSv%oi;DC5r9in{x1bRn`2Lm@ioPUaS9M%E90FnC35JzF^@mw8k$@ zIr!cuHGd!T1q=UhA?*R+XhR37c6{x{i7)de@>L56$n$e8IwZTj|e~ zgTJe{<_}=LVBv4D(!VJOe`FucpU!;2!cVdIrX2ip@tVJa`GSRS&tIBy@Vze0|C;%N zh2O%m-;{&DE?M($Fki6n?fh@b!LQy|^DB4O@h4dL6|L|$<={tUXnrf^3l_e;|G<=k zA4eO^p_|5h!NNaj`QMa-AL-HjSC}tY_~kA4n{s7;f6ZSh_A?f~z5hARkSqHKYyLs8 zpRw@0w*7`2{AEKm|1ahX7XBs6|E3)L1;aJ}{w_NH1PgzG#W&^PHyNq^~A7Z{>;oJQ;Qx1OARn7m8`GSRSuU|0b z;P1Pp`44r|@h@2TE3Euu%E2G@pXRq_zF^@mvG}GO{A#xqQQ_a8`GSQXZt+dI@_%8? zpTqnp#=@^<#lI;Be+V6%0o{+8FIf0t7T=UB{#}}XSon;Ee+7TggP&;nXNVyO-&;)c zFEL**^pXDU@e5N9{&D7)Ev4g6u<-5nuPFzAT?uXf56l-Vd^`V|a`2BZKfJrPU$F4) z@o!TO{<+fH{+HQ)!NRxgH|5|LFQfTunJ-xQwI~en07o18z~DDwej@V)Lk_#`_O~eq zztn@;{->BPSokT{__rwse@G?G_m+n1U(B-Kl!ITax8|?s z^+SS%ZUyCOgZ?c zlQiGU_6rvNSPDZtz|n?2F!=6d&Br-2n7d%!9O=Z^WSE^VBuG{+8?GI{Bnaee>?L93xA!Je@(e^|BIS`RNT*4 z`1byzeTE$TT_ZIAKjsS-zP*3Il!HHKwB}dtq2o`m@B{YGu|DMBUmC0VO_?uP_{*&D zGv(kfn4tNYyna)#@N=#8Yo;9hax*kPne7)W{79?)X3D{jnWgz-nJ-xQ_WnInuI!(! z`HRGU#=;+I`M;bY2Y=2y&ELU%!NSkA+;7UkFZG7zpJu*b;oI$hQx5*jw=}R4ln{ws;#hU*q=ifPug>SDvG3DU9-_iVTQhqQN{t|2bjOarSez}#JpUr&1!rx`B z|1ss@AOA@67xMX0f`!l7H?RCN<=~%Wek6yVVBrrTcj5t#HuQnP@AI*?KbrZ1AxHUH z$clec4u0ZB&F{y2!NQLsGw=Yzep3$q3g*Aee8Iwxv(m3A2fxr(ZU6hs7c6|+|E3)L zVaz|me8IxE$FEH}_~|>f{r6Bu8TF4~;Sa+f^x!Ajl7oNlQ_XM4e8JF1_}lfLDF@%X zOY>8hFIf2Y_@OBWzuO+opUiy0!gpKoZ_2^n^||J6WWHeG@3Pu&rX2i92Q>dQ^92k4 zq?Lb6x$^%}&3~Y`4u3CW;YV8apD71_*-x6^l=*^%-_2^jnR4(8oz{F8^92ikp%s3n z9Q-9`H2)Rm3l{zaN&|R+qYZsv@FUJ@{#xb>h8*eVf@Qxc2S5Bb&Ht78f`wn);+t}1 z{{_vz|9Kt$5sZc3#LB;>9DMJen%|81f`#A0vfq@0zwR&1Ph!4c;SaIm&y*|oU(@_4 z;(o@$KTKhW2RNGki7@2g7o&qeA=}D)!T5&oxBEY)9Q;zuFW}JOmxn{}O*#1Gng0;; z1&6ZVl!G76{6@?dEPT8DZpy(gQs7SIe+Tmg3%^?#@(4Y^(S|-S_ zYVl1u_^X&dllg*$Khxrya`2-nX#00EU$F42TYOUvelGLxb?WdFEc{^>-;{%Yh55Ca zFIf1+@CQBkiKc%{IrxPtY4>+^YWoF4AL*wBy?B7ZH|5~BX8s7L<_m^C_udfV<_m@#;a934{X-8h_@*5EnUR`*j`@Ox zZ?8Wx<=}_Y#dpvxoS?&Bu<&PE^|vVpf5S7HUyb>Kg>TP)nR4(CH`e?P%oi;D1(yA$ z9Q@qon%|H4f`xD2Ut!9@U(!PJr!!x$@a^>rrX2iQ&uRW@<_i|SJ^y3M!SB;X^N%xM zu<-5m1Ew7Olqk)=JyFM>VBy>I2c{hSMjbW(apnsa{t+wvnR4*U(FPajc4EF@;oJQ; zQx1N5SIr;Fe8Iv$XW4Jc!SB{X^WS2=VBsGrM2@2eINHz$20y}~`8${|81is>*IDI< zDF^>tqUN7rzF^_o^IxVM{2i&9e~(MYpJ3rnwaQOZ4t}?OnjgV@!NR|eKj^_vw4o0S zew;`1W0@})a`^v%)qgYP;8z=@`Gc7+SokZ*3_QTF-;{%YnECUWFIf0X@CQBk0pFB^ zKWvD$|6}G0hCbXs)k;669Q;x*YW`{F3l{!$%m1bv{NkfE|IQ>Ge}aWy#IoO%gI{i} z=2v6BVBy>SFH;VFwU;&jIpzx%elg2_Qx1N_1kF!lzF^@~w_H8I(S|-S_>-7Fk@
    !ngY$rX2j@%Q^g+FIf1KboAw=A5#wg0_HDbzF^@GE9M{nrX2i=A8Y%6 zV7_4C+wM2z;ICu;J*hhU1Pi~#ZT|L)eA1w>uqwCH`123i{#}gA9o6_c<64ZT4JNm8 zSm5Ui3*R$@%qvZTEk#vW)n(k8@$-!LF&@Ua+*Yms2ID@AKWDsz@qdgbZPWVIUeNXx z*{*R6;~2&-GM>qJ72~iSTK_oXNX8q<=@sdfhqlxCVH86ZMM%GWEnLOIQ!G4r7`gX; z8qeLW?cdCJ65}s{AE5Z$qvby^-p9DWa4kR0xH99gy;{Es@PkyIFz(8@JmZm!D>7cr zxEkYwz~yN@3gcfGhaJ)8|0#G*v9JPRVPOSFX#KGA_bOaf@b%&fKPz}hoWi{rUsG15yxK?+Kaeq7X<$m_Lj8}Eh^39Cp{`#L7SEm62qJ>>yJg2tCxX%RU z$$cg`9}rm1|4U^Yb*I)J&bUQUjo)N^oH`&d2J4H!m-Rz9uLxMqBf|Mdz;eD3&O-r~ z^G+~74lMKEnC}Lb`E0EJ0haYWSdXLhd3_G%=YeJZ9qVm?Wjzhf{{xou{cs)-u$=dU z^NE1v{2`q811#tL;QR<+IUnL@#&X^R&S!vJ&R@WJ7r=6!1GSy^IDZ6K&KJRXObYY)OgP^OSk7<4c|pK(9uUqS0+#cIa2^k^oVSDX zb%5pk9GsT}Ea&Cm{4QWQp9|-K0n2$`IA07{&JV+RFTir13(lVbmh)wBo(!;@7lZR{ zfaUxeoYw^`=XK%yA|>bZjIf^^SoVu!e><@3U&nb9z;fOM&aVKL^C@s12C$rWf%CqmiQeJIw`0?T??td9kj^{-g(3M}hgasLdk+%JRm)WEV{8tZ$3W&JMJ z3j@n~V60yTmi5V4j|?p9k+HrSSk_PDz7k-$j|BIx0L%Rpg*GJtx+00?YbL+-C265jZ zu-s>e`xAlXeni|S2Q2r+VSOsFtUtwhyTEduF3#@-mh*XWA0M#Xw}<=vfaU%^+!qNf z_d%j>0W9|m;yhl3`8;0SF9O$Gjw`L@4#s;~j{9w)FZb8tz8PS-Pv%p`O~z^d&y2l2bbA2z>q1}d&&7Rzz;d6T zgK?A9I=$gOEy(4*taXf+QA3wLhaF{{%lbvA?*skB)>@AHtbpadD%_6+EcajGzA0e2 zPYU-}0n7bVxX%e#?rXyRP{4Bk6YeVlmivgXe+yXlXW={uU^(vr=Su*~`4PA;5LoU5 z#Q7IWpU=aQsjvHV%ay|T2tC*zYNHO^&xfbE~i_#ERUjKeqS_}{>| zJiF%?#u1FmRn+d0_2kVN%l(1_8OwcpD;UfDfyWukeTumM5aA*BAJ$>K?++bbT^Y~e z{DJebpfBfPy}?-Sr@(z8kjs4`KQb;gMTf_Im9;!>TMbr9nrE46!HWxV5e9iEFA@8$5_&N%lkE&rDBNX8c#m%6IucRb4exu$Vd z#>Fmc+=Ox5pBi^&96=kZ;J?0%53~I*ivHhPK8^7(&Oe(Ohjr2N^Ni=bqQk3P744o} zj*sSyC$N8$7*FE#{W9ZPFKP2uGQQGA<8K+qaepeTsy2VgZ(1JC_&Vcg#_2<~+|BsF z`#St*GhVksW89a7_>%jOaDN%_mIC@bbKHjnEcYFCWGwd=y}-C4`)>i`A?%)Yj1O>l z{LHx6BpqI*9@FM~TWI&zWSsk$mbYgdR#W3t#;vPs{1W2{H8h^f*vS+Db zj4!3>`1y}8jGItD21^ zXIc6aHNIW=3oZG_mi&;$2x|45v2cMJ$~}4V5*p{Zr>-S$Vd3@`?qT6%3wta)T;n|d z%(Uc7ExggfyDYq4<2>_zw&a&B{I7)z)$|XKq82V`;mQ_%%)(79+{MD_79MWlaTcCr z;h7d*VBxndyxhWTExgIXpICU0g}=7&cNYH9!lx~K!NS)pd|NFmJ}g|$!jD2f$igEu&P$)KS$LU+cUt&_g>P86RBaVrdFEHPaD5B6u&~p@ zSr#5*;i(p0XW`E*{GEl*Sh!>z|MI)%EvZ84HJ7_z4R~Sh%T$+gP}hg?m{z(ZcB#?r-6t79MTkS1dfk!f#mkeG7kN z;cXV)YvH!{goX8>w>Q1d)9awO54}!$6X;E%H!!CKz1j47 z==IW@LvMe22hf{K??8G7(L0#lA@sgL?@)SQq<0v-!|5GC?@RQKq<0j(W9S`A?>Ks2 zrguEO6X=~t?<@4aO7A3kC(}EH-q+}zO7Ap!r_(!=-dXg%PVa1b=g>Qs-g)%Sr*{Fp zZ_xWDy>HRGklsb~E~fWwdY91q4!uk1eV5*4^u9;$a(Xc)y@KAA^nO6^hxD$ZcQw6h z=v_1HhQ( zVwj^tv$oA0&7+#e#yY|sEn*|uyS&cyju9<$-L7nJoHIV%6;db4Inb5e$>m8+bY&(C z451e1%JgJqv#@>3xTd#aS8%GirYy{TE5j`*yc%tVhZPt9;9yBz7xfmu0TM|@6ddSbpBN!cz6E`KYu5pJh9 zA;sl!1k=cL<$4`j&Es}v+6K6@T}i3w>5hbyoXj*WOm^ouoQa8b9priyy0DTzIpHWJc+J^tVA7(o|~&>q~^L39r5t93iw-~WIG4k zA|W+1ks>wG@xNG-nU$H~%<(wW9k*@|MY)?|Jw0oHqhExhox_u!n&8UE@1}m_&GnR- z97GBKlX^hp<;4;^pw_vhpr`(D&MEO=6;dSR@yCmxjrzb5hHjzv7ODsY{ zENb0sXC~#)(5?w&++5e=b$U}1ID!3d)&!x3wy%GxCx?>cEhXekM?Xir0}%=bsZ5@o z3N;iD3F$6pW{%reh=_{Ancz+B&t;d=ByOtLL3GCe zC;j1ZrMnWm6i?Y?d%QEjD2T`m3SK8adh7R3kV%Uh-&$FQB;mqxvG#p~5{W$K!H1`%~!aI#NpD z;EJToH3tIMSN~87K;=XACzY)oR$uxW;Qd6ZfU?sZ8T3z9e@Z}^Nvd*pIjBgngDTO}KLTmY zO#w2%k(5dnqz(!UQKgpTj&!8;cl66~Qf(VbNf+$dj(*Ny4kOF@kr8>d8~#QHfYCb6 zgF4WULkuf#5Q}I-H(?nF+o`h_Z&vQZ&inI-s37-mGA0KPlbmNkKeQcBVoZ z2H0_ul%7K>fe!Kij`W#~GR1I>BI=C7%{b(O$02rY8Z-+H7u8rWjGGEe%oVkkDukQ=6U&IAm|C8mJMJ?R0PpDoDT?Je>wq0-#)D zHBbGm(5g4l$)T2S{wmTQ)NQF-cWDsb|7=lzD)VmUO@Dj-7c0C2-6(qU4MZ`W>m83f zooegk?5rFr7XN=R|96}JC(AXwL>C8m}#+sTb!p9D(;o-*<2%qu0OOZ^YnsQc1uB_l_XgPwsR05y46c#CK&wT z9H{#edATS(i&_N196?24jw>&Xs%DCg>fn7kSm#eMlSgn4HNVJdsLN^a!=a116lZ#p zg0`E2Dd(>fT54EzGo{e-Ad49OA~`E`f$yi`;TBqWS9hf+jzc()93Z&>p(!4c5!g zsGA0!vV<%7{0s^uQyuL8Ed~_jCT>?cx8k;-9ekY(DBtMm7>6e<)fnjGk>R`oGjJG$ z`gv(I#+~J*oJ*r5NkI}kjSV@|(wXo*cKET&J^!;S3u4qK~eOtn7gpob>lV zb{1?f-QWl{(BaFYw^qK`F4ujVWLKt34S5=6A)2=;{ob0vG_*%!Iy8zy74Lwo>@*C4 zcw97mNrP-Ye^Us%oYb4Qoq?p{AAq7c$VH){DxF;YL%nb{I;LE1n(TJbXdNZ1pc$9K zn8kKh%4^cl2r&&+2WQJfmevVr1kZEjZw4y`7!q;}J)mrnUoBa9~UORyqyAba<*Es2@W1EySP{%6*BBWQ>>RxPy;C!i!uY z`j3(OylJSoQ{ArgRO%Bap;ecjm6c{rN%t==JER zPs|#i2EVz96LbukRV9^p2BapZ=jgU#?)!JJ27w2%80JiWHs#~d0!^a)GV4AGySF)f;pX*h$M1|nOQc7vrSF44kIDU zZANb>iiCYAhwBEoN)$4sOYN_Okb!9EV`j&i(DC=9q00@dTW|%p4{kwIhxGtMQx^yU z36tiWQnIr${TeJ<$w7_HEMxY<{3@BwXFf}<0*t>>Dx|->Ml|LL%!o2R>IQHYMr8u* zP~*zfKn@h-t^Yv{!>0MCO#X<%Fn~TFJJoB{98?nLRjT>dm^ag;RBWUfrENyPE8CMw zQ^V9Yc4yHnvypQ1QMKF8N>vZns}_KlCWZqdgFKXyMa3p3MK9Hro>a2f3<}ao?Nos}Z=m5fzBUNryvx?Rb% zLW`!0B$?rC@YZ|DTOef3gYqTN{ACG++M0c!(x? zoLB*d@A%6xAk~{fbr@=n_Hj|ol~J(+sh-Ga;YxDmqi1lJk zR)EDF;;}?4;Ip5$x%xReRu~SBpiViNuBa@`@`O1Y%AX0@RJFx=X{fImHLYk7pQCf; z0GexXPyps+K+Vy-m4t`3sX0}Po=&bL7p;p!Y&&Qna+=4R=16pA6BQq5Km(EIprnkE zKdL-UkOY9iqik}H`rDO};2sELfU@HICOBxm(SsFg)DL#_uZ7PP-83FaI;!mR|B{?T zMS}iBYr36yZf#QbA&aTo>`g_05L9JmD)rp4oLMcca8Nj^teNQlnOZVxh{69W4Zf+RmHuCK z8K1=FCnaQh)59DT$|@+6)OskwADDJ_SiE=rOE z#fuIdDl8IEG2<64XZoc|Kz^x)!OEjdw3#y~kx(qqA3CMc(ldaZOolm`>Vt}YWjIyZ zh!p*SR+w>LBZii3#bgmRsDM_J1xh-(Xv#2Cl@%;>Wm4+aW39+^D41A+!mz8$nMNEc zL6LHmJsFnYu%k6b!q7kq3fv5@N&Z70Uaapmd!;d_ngn7LtMzASQ z1klvy-B8~X13jfW{FF(vP1!9G#kBSZNupk{Wqb zB}8|Yho+`8oQX7mNTyINh7{-4{cSWIvZ=bGY?Vc6!SAb%OutWPE%eLsP?n++#gR!- zDq`xM(RP7sii!kkgInJ{Y7oWxsB=#qpR=Q27sk|ljJ{c_Ze&MiP~DSDl^}{gA?Rlt!ySYkrg@%c&H9ZbPP;&VKWH%N#!2;?H`{*+MZ0wq#hc(%ET9Mw#qW} z%YjONr6WJnFa586srcgG9N!=KYoI5=n;t(f(}}`J<;c{`B+AL;R+^K?uQ(yclTHl} zZW!p*Q7Jh|Ni;ge-?-^R%@DP=5-mbF(O+tO8JG~CkWB@vBRe^smB7VPUp2dv*HWND zi88$kacVq}`_d`NIR>St5)FfzS=s30QJ^V<@I|#_^k>eD_(~ZFi-Spv8qf(j^!mFk zsCVLZOezyo6H#HpU8-=CEi?EKpNi6-)^www(>lISNY&C=Rm#}uL;l1zlRQg(f<#0I zZ4mGiG2l^fZG(I>b`H@s`DhjLyAh=OryB;@Amk|Za= z&yjvx(JChDOlUgDU%{E3OeK9LheK#R@^n)_RX;~rX3|Imwd-}z=5M5_nU9IOYoiVH zjDV8P~t0F>_ ztzfjYmAQejY-x#^P|V*vD=hP7x2VnJ$Q$>gJ(VcSbTy&Y^HUoE%ic3+8l3xwKA}`# zezS+Vs7$45&|FoZhd0%flU7Ji#eODhK$2?&oroON;K!sPb%bdD%Rt(cgLMd0$4em{ zR9n^dp>&L59D~_DHEJ`ULt=bEu&~Fy$@t5gifuoEfk4$)kguR)DF7Bn+F`0jLP7~u z|3k)j)HWm|HbRl~sFnl0III&vjaAzSDBVB^=^kn^VY8U+OxsX0xGs$pXVVHp>LAk! z2Vd~{ttX*uMnQt^4z-XRd9f8@M~SX^sgcs3cJ%m6r1j3SlFFLDRl{7tXPHRk8(g!e zpeXF9m*(QVT}Gf{Y%8Qt#TACXkP`iapVs71sOd!LbGV*c)nUoah79Mxc(RuUXQ-hg ziC6386-m?xN87>@Bfp@1?kD1SLAp~jQJDSp5viq3^9Cfa^Hu;2ias8t0W9mJp&He# z4v`pW(2N(5|P^ZexOKfa4NH8*ZlM+amE@~c>OR)T=VSa`iMc8abdn>lNi_R2g1X15t^65&AuKDY{y*4) zUS23ysSY@X#!=8ikv5IskyzF%{TU zKhwDqMt$e+NHef+HYx5G%+ePP9*4#x=PjgoUQxgsoo{K0fE#0)0V|7wO!sTWPPs(g_^Q3GP}0hYfNJ0Sx9r%0S#TQJAGRb zQZs1{Bu3vQohguJ`RX`*EV(DIN8Vd|z!FRci8NQw2QczgfT8N zEIEM?vzH8jzZr-~iBIn*F>fH`PmtL7gX{IpapKtXi| z$TG;y&H!7M*H;KGM)NELQ&!t3>jSN&W;Rk9n#STpD*G^X79_yX?iOwA+t!nuPPdV0 z3>7|)f_DX~iV-D{&kcsIT0YMNTDpt%7XG>t4n7wI8t*eXP}{H|P)#~ws?QW4PQ*zK zo~$G<&iH{hd>V!i^!mYIyA^{t#?O#IxhEwxiPDQY&%e1VbYPWCNu zaj4Gz@VAC3(o|LbI2Wde(c$r_VrOxl4mE?)X_*~0pQz+91~Y=btB8L{9VGK0uk)a38;07Y7HNV^kh>?INN!A3!; zlPnGBnES2#y9nnK_nVZd6}N%+7ggHBQk2M1(JDdBLmw0 z)GXjWf?loQKP*5}-5pS^C8IST%$&T`OEde_iog`;0JC1@;8YtdjD88*zyl>g2&i|k zxlpB~&}wWXR*z7?-DwC2`w5uRujYtA>2$QK6aZK!EI(@8o8!}nMj3_ zv`@u9QR(c&PVf}~hPvJspz8&6g zgJgWvbUsJ0$NXX|Ab?mYFR7%`5pDrWYNLeOsY>lHJyqr3edKIIsZ&=kYA>S4SbWV2 z+)L9bo3zz#IlWWE=Wc&p6*9M`$~ohfj8#tHJ2^@jl3xW zYGYFOpPFn`kgD-w9zXJ_`iE~`!a`_;Qj!#vpH5y<%1?>Y&`nintTi9!NUF(CM>&~d zHh(YaL=sAeV>~}=ux%=+OA%FcG=wgBjD~PtL}61gM}>!)NK`DUX1dQUT+T?7To<2t zrH=AnKous{Ej3KBp8$2bBLTYs0|#9!)j$PR@=%+UYHz2zmem09u>*<5SP#;8z;KU` zE4}qVonTcA7qY%4rV5gP0P`#C)i|g>%;FtioFA9?V3(3A$6#sm`rPusIA043Uo@|x?S=Mts_H( zNVfxnz0}a7gO6POd|$a+z&lZ}FhMOhG%IN8C8_dHIYmknKEh1Bqe=o^xa=qub@OBPy5KQ-Xh-#Gu`}D!KBAI zeL?7NtX8Cj>I`0!l4YXS`~jhB8>I zbZdentD7d_pE--#3(gK*G*xP(Y57q8 zq6&u=N~(cSjK`Y`cl-_3iuvnsr0R)5awwh8ZnR+i%`^MyYCciA=f3qG{sd#@59=+| zPFOk+lV*O@%FDd_J={PX8cv-@WzEKtAJI`7hto>{!aMBH!8gVA7z@hM25&#qsHBAq+IV z5a4_@Pe|=c`$&C^jrc-BX(7Zp9qC?V5FI@QlaH39a+!Oha;A<^fhmjpW@9MNtB?Mc z@eHpDD~d~=AhUJU$c18~JTvE4t-#|5QL`&SVvpujOlJf%&`eK}MX;={;VDy0JuzGC zH)bjG9kPRti|F7w=4NmInp_ue97?LA7BKN3>0Z~4x;?HIuKA*!`YBXUS;O~g*n~=z zAg6BTt8Y&r<*TfmY%|0uJS>55#3X%SgmHi)lBJy0{>bK&FO6oSU&xvx#(A3>fA^Y05a4?_>@wbTECz(#S2P>KUlc3sG$D0MipNr|-GBYtw?HjcP zu{iZEh^TX1tH@a10PM#hdzxaync5%KkT>v&O(bkib+bCCqBYGRq|>gNK$Q+^2XPQV zCuu;&y1YTeXqz|nDfEnQ&|<_!ZaRfjlP#?!ZE16#@_+stG*ol?Bl)sPfvDUPS1T6r2^ zm_F`_)43}#R-FRmXLh1KYoNXC!1os#d+RMhV*9K_IwGzUMo(;OW%Madof@TT0Z&7{ zLe>-G$~Hdd&-`CW6o*#C=;A$A%?Gx`{9i*=d)d}%wXSq32<5Sx>Lo_GGLyZsFItiz zWl)t{)KLR|fkl$M0>O5l-UaNhZf;4l^r?m7YMD!`96U|EF*IXrdxZKgP4%)>i-@y# zI`Dx9ezws@*+9RbSj1&BiE`G1Z$Gzqm#Q0`u7d^!NsU-;uTBIFshCAe545(GIxIiX zUU)iKw3(BRcXVYV2T-W$+A?_R*WF@2-_kZ7*_Oyf8_dxjX@h&u>;NT#mMT3IanV`L zX+)!;4!4H1FP*~1(~vh#V)!gyBXl6-(JUX5ZCs3X)GIA*$CDIgT?{wQRS^J}Nj#D5 z;v#AcV}zYt*HCSX4&tm(rpc)wA&H^@tD|Y~A&A)lDpZ^S)Nhr4WwH_5E?OXws zt6q#{+lp zJe3Z*jtQ^~+ic9xx5I(gN93sl7~tDeYl&#AjLR{uJcG#n+C8?_Fu-3y2Ye8{kT$S@ ziO%3@f-KS-wF6zKzjHwmn$EdUGE5K`2Fs^G)r=h05R14P=4UIlDWb4xGl)qo%_=Cg zT?#9)UF69PlzI0l>@EQsk0$1uMU#ki0LQK zC)Yw%@6!)rhuT*k_&tU?N&3{OtnBvAgzPL&5MQaX9633ZOmcG&d0yU$4eUvfgLTWs zm8g0RLFl?#eU7@rZj=FSJneboYlkE)YN@VorAne}YGPn>f@}y;jfYHGs;R{evcpR) zoeg1N5ZxTRK%#ylwXT{CY^u6L^r!CF1^8R3p(<$3E%**et*GJDP&M<27QFJ&rh^u| zqvey|%vxWcGr(#3;B-$bI^QBapl3OtK9H`;(l{GAK29kdqb@;H(n! z9F`hX%F{Y@b%aka5p7V)CVK;D)KkI%fI)dmW)Ss}GuGKCHDp$uhMxH76PyfG1OHGN ztOs{kV2skLBO!V0RR+aqa*`aQ4-2QED{l?B%bHtKJS^;>0g`~zLu=#>5YY-n->_2t z933Zm35O@_x#rEu>I4^^_N1oIYBy*Q_$@Lb;9EqU`T^hS)vFcwt$sbsJL*{}>X7rn zc#eojhpSGcqkrxC1sc~XP`QylMb9^O@oeQt3d>-`S`Iy(c}skC^qucaxlSF7JHaKT z1|;b&O*^$|+AON2FW=azF`YWZIHKCb#^vQKe+_*lcs{m;aFXBQ9YOMQpvyuq0v#As zF^D^Z>V}M4hV?z8V+@$51&LBLM5KZ3-dKbHD7LG zjvl59W{h80*YM4PQ_~}_2@~9Ki^)gN+>$P>yD$zEx;2U(;n23{V6 zEfLs?{6g11MbywM3#9EA%95ZY-7O664V#c{j79qjW!xwt*M9_Dj#fj>3DLBaqrHok z#UwtbwpCNW(TXwDGkoZ4GVK>Zje%;zoH^uaICW$*orVf=g0v#^GCpb~=q&^=GKhj+ zpK4hcPNz(Uqh15)W;XjN2eAR|6zznYX|h}V)N}~QCQ{`dy0=3Q+0fxKY8gHsUP0&G z1lg{xYdxCo(5K`FRf4hpTytkWq(Lii6?c&Kv(|y+3E0qp)op6ITA;0Zf)J~uRD|G4 zEIQo<8$f7r0NrV(?)kyeBHOt>qQ3V?)=CA}!zr73PXbKxn`_IP>BI#G-M$@*S96BjM)k(B!QA$YIEs>ZIiSE%B+*8HegjKi#a9qxSOgow7oC zt42nCPJtC#Ei24={yAD~OIX;0Khx&|EGk9c3evZ)!wMD(3oA(fg@t9)+l}Slhuyun zc*TNY#Te)RbLYwptWY3iK!&}M`2O*G_YZjC>igkEcit8g+jm{T%^SB}^R9cULsWvN z^EbVYo_}L(`niVd*T1u(+Su^jJ3lMDrrOblKfc`~C#hYj@820dy!2zUm#rLqKBx|Duzm)Gad?tITP(`QU?dhE&ScfGVa^Yn#*-Ae7hG~-H>$Bynl z`g^Mr9rsKrz4EQH3!1)G=!at`4?dQ$yV%$Np52vMzhOO3|1saUxj(mP%SXDEyRYKf zPYV}Wb@Y*q`~Lf-!myE5Cb<_E*fVwE6T@mGyjx=O;~UC~lrsKDdYFXTyboUpFue|VWm#__2=9X(d`}yB{)u~pk;Rw(GVr6fW`4D{c8i>V$^*_SKTw{@Z`l4F0a4duf?A4vvZA?!K=C+=?a?75FHz)S$^zyPFrnD)%`=Ne6d2T%X``q7hKS0i#MHH)uR zn0TmAQoZRn>QDKD@6=?gMQeEceBdn(Kc(dGh-rDQ~u3 z^7T`XTrRSDap#GXb~M`BbHx5bm#0;J_too7x4m1VOV-h(QH9qI{C#4l4eM7;>9gr- z#+Em7ht!+D?~C?L51#(=ryu`_dU#!d4~u@Wcjl9`-g;u&+r=A2)k>&cFtgR^U)!xZ zR^_|vof|EkRwr!RJLBKo`uC>fAGUV-z4d}IwI==a#&cB7nn4C%azMZKP%9FM(Vc>mo#0~V#Htfyf*ODzpih5x6^~& zE^jTH^vaPhwzqwL{`RMCr2gzYI$%h`&*krkJpbe6*SCC>xO2hPv1!gW4;Q~QWo_hL zhfch-dfV2&2M(WCv3PRIBYU4MQS{Y_cfTl|-goDV&sNzLdC!9XR*iqU>Nf+HmD$`c zX?Wpx>i2V;>A9@goWq6xo%PRu(c$m>_T24ZPrS2o?A8NEAKLN9^(G6S{IzrcXHNHA z|Kg-(XJhtk{(Mx;4F%tr{m&>@+^pGytKaeOo@&Fs{_4})=KcIcyE-wS_bFYtOXg$I z?+tkB()+8lxbkF*GX}_P& zSNNnu=f3+gdauZRaq-p`F?AyQ)^w$fNGMxrd((L}yH0S3I|U{38bopFgo> z{P2F|u5|tQ>HCX6>3+Q8ilsf4Ho0%G>w_KzGJ0I>T6AT>>;qqAf9U+FORJZXE_^#- zR+m5DUj6+;Lwi@6u>8K^uU&lg*JbT5&o5MZ)J13VjcI>xFV`dO_uWTUt;iX+?UnMG zcdzeQ^2V8Ou1|UY!=QoyV(x~*hskN^*{_NVB z@9Qp~FnhrFaWhVyyzR%oCyy?hU3#$&V2T*(!;C8-sgRw<;FE{ zXQX^_s#5LVqaRrC{o(F|KY94AiWN?}DqU$9xu9$RLH*twx1r*-SAT3YYTflR+b*8{ z=u}ew@n;)^ec&DaVf#%Vjr(<4mBtghb)Wv~oT5t~nDXJnpPgyC^vROdE5G`L=ly2~ z6?`J$!r;V@CpCHX%;XD2_qyLYe|@n*mv5|XQ~OlC+4nY^@JOLL2hOiA{N{Vz_Lge-&cDfb96fQg#7l#=4^ExF zU|847(LcrR9&=yKDvve$;oCven;tE`U~=hiS3Mqm@A?CSzB^y!rN7$#n%?8L=N~E@ z^IO=ZoWjpN`QvRr*L(S+M-y-B_(H2YrtN(7M1iKoCaf>EruC#pp8Mmi4{vWbbMw!= zUu(5<$oeZU{P6CHsxjA2?6^>~*N~_|x7|54yy&7MD?a)7w_&ka^%7qCZ|{Xe;ggnn zkKR6S-PWJW3?Fs!;n$|utQ<3{_L5J=w446a15txVUs}HEOvca-YdXB})o&jb=yx=( zP3B`OzF5-!!%L^1$k^L5r^=#Z6Q17s;!{1BpIBJAOUL%7%YJ_ExV6z!_bqJG_SmCe z6!@p}n&-xknz8@Op)Y)0_R#_V_Iu^U_`$6g?y2f| zZHZXA>FSUj>u2N~|EBnro)ORgT=m`LwtrPUxb(fFKbKxwIDT#IMMt)j+I%2t?b%(e zr}yhre9{NKFMg78Wk!QX&er&*RNZ!?7ou4y#&pUBiKYJeTxt)o+H~*5KtUTQ~paUO4T6E)U=N;Dg@2C+aV%Uo`Bc za;;N(TyJ#!`)&Vp`>XqZ%TwKdef3+!Ykz$GLD;{Os}0XsHT|yE*Gu0xJ;>9ncleom zYkxSXp7Wj}2UA8k4)>X#TC4tw**W8fmY6cL(WE!e{9fYnw@=+s`a;6TMV$+V|GaSB zqxX5Iy}tc{6^l=NS-AGa@)utIZ&3eUx!aBv_5V-9<*AJay*r1l*}t&YzvF)!*!+*;zmFZ58MZQGWT}{EpPKc{*z}F%C#OySXi3(g zdM~-(OI+db+9=!jR zsZY3`yM5b3&&)geQ;qC>)9PlI8l2E!`QSSz)hN5{TC+VdzqNV%l><+1?cbzn!3G7Y zZg_h2nexLsF01;`T|=@eFPY-*+4H+u&1=?--yAb=P2(RMOy4o{+LcZzj`;USF1gz6 z>p_>+)P1Vhk*QsxOI`YG!i%N)#+8^GHzwoa!uo5Y7Wb@OtNXl`ueCe7*K_9Ep>b_5 z9vo5cmDrXehrP6^-PreIyZt<~$(-(2yAMgO^UP<9?%&vTT>5hD znA>v;Z5x-{=7D2d-@p5(^o*KS7Voa`R?Z)LcP~8So&AQh+O`jWyg%~B-^HhP`gmH; zstqn?PA;~Z_Q$r5)ps= zb>@;bSFgL?YdUy&+_sT-csgajH0{{JN2)xNe&)cbo}Z^Rf2QQsWdm>jw&>}z7e2qS za(u*^Ez6FTFFU2nf7MSMfA+79%N{9Nv;OV7{{HuIZ`A{@W*)Bk=Kik3YA$=b<(q%~ z{qK)+N_9VKRrMA(=$Kb->Uh_)Gv=+~DeFF~bNAiLqYn-B6yCf0tv?P$UXFR?-|0^mD%87%ds3OAuW#IOckvZ% z{up(A>c4MJj4kIpH*?@W8{YnG?KfBNI=^Ji?2DfcKDua4h3JT!OShM4)G=r6swdA> z9$T;d_CiHVZCbhZP}ha0GE2O2x>a}!_wt&ZKKZvo$Bw^*P3wNLU4d=8E8ab7TKBZF zr*3Lz zcPiIe>H2S59HVQLn|!=mTg6j{cS51y7WxXlutkYW!ucj zZ9nZ(_te#fH-;p)DbuEX(e*!Xt2p@J+DBg77dd(Cx0A>2dp_;=x2kohzA>)zh4xtw z|F^}p>GK)e-hO>_`h`NSO+)Wn^Vt0OngbW#Ubw``;_n@7`$?DE!fuRw zTVg6jwf$$n>b^66%&1*+XU7+oJ=5UVO2_L~nzyl0>i3-=`txYf%gvRhnV5IN_LdoRu2#(HbzR!doAt(5{km*#_IjC74Qgkun|o^g z@|Nu$I2ZNxi)l~1v}#6=B@5o)G-Akizf4a4_1A8VQcrKaR`=Cwv#YP@?Rn7{Z16FxOHCVl8(EI|I=*B=?^xy==tJRcbmp-%e1R= z_pwJye|33VVx8i%WBXSvGqw2DNBaIY`GXh6d{pT2T1WVy`eAp+U;q92U#4_>xa)y= z-&L94ZpyXflV82I<8YmVi(Vf&zrgWXYin(5S@5&)G2z2v7FT<7_SlW{-+no^=YK0- z-n5|g*3`;nN-Zn$@%#6-`sBSmi%&mS;+@3q&n!K#^;oxJrYnu@xPBuQ#@63r8w$1Hx<>jAB-?J?8$nM3*c8~t>{GOiMjCkz1{Ykyuts8D!bLG`>XJ?PuA2#mbvtJc0`q8V$Tdr;2?1PBJ3sJ>d z?OFQd6IDtsU;0&>oX;m7uHUc1kAKEBc_QiUxFvIH56&)E_R=?tcGNt0eoN+_mgi2z z>}=O*$D3cRsWtzT$HI!eAK&q%n7Db(IyGGJ_k=D}%G`a=u{JGoj|^(-{b-iw*>*W; z#jE#fa?i5FUTLE@M8`z0p1&yfO;7Wdo0?AFaPjjp-~F>_*U29qDg5lfk5(U>*SNL& zr7NGzD_^L5($!<%H;KwvvHz<(rft8Pap#IR-uq^7`8VrT{O6kc`CsEAXViG6*_=IP z6Z*t{e!0XmamP1azvJav^&cPI?8`6yNZ&AQ%**TF`Z}k|(eEBlcy-i!gKnJvqI9#{ z3x(|{I)8bCb8Y7@eeF=%h<)2*w!FV@YrCSJ0(-k;9k{p9@R>ck9qX60Eq0~*&7zH7 zjQ=}f#lPM@ZA-O1yP$RFHe-+FIPVxceeZ)iYyGh4&hM)9`|C!##C~JP{@!X=+ldBUA|tF6nFp8x^=6##vcfq z@o>fQ0~%bY-*e2V_h+>)cChOG_cVHL`FAZ3-g&0X2cse$t5Kju&#V_V&3$RX-^af` z@@Dv_MbE81*}c((ZCk4~eY?e7wU1t|)noGRO5rQNEc;~fg{6KvvawN-cTT_AYj4^1 z4YK2__UpZB>7Pd;f6u%;tl&GVK6tZb=Dt@l>pZji$@WVJci;X;?;h=9&Wwrwq{y(v zCw}k$Xx%*@ztFm6%;6vZPib@h&IAJhaGc2Fb(15tk(&Jl z&N6dGEb*phgo!zF?j=`qH}bl2zFsCu$lSbtLO;Aef1dC2JyfwQ&&Vkl`V@ZDa$}jd z=Bg;X_@&n!Ms=)V(@0){utLzWC*kmVJRPi!q^A>Tf+#oGXu!Gz?tG%s(-DwmNVn5) z;-!u4S0Aq@m#PWpZ`tzD#5bPfG>r~g z$kM0J@!R&87<8Ek9!a&(w+-(IQi1gT&}XV@H=_6->MdDIS(%d=-jhE{@=N-S0bSmW zH|N#qvybdMVo;W6ui76Ds5;aHhZ7ylJ0RMo?|f~iSM4y)6Jt@C`!MPX!081l`^zcF zcX1<|-#2K02Q}&6R}lD`$Y|LA*B1`1kEWC%(Dd#DU!pzGW*$U7yJY0kJ24s}EZtqt zcGx+L3?Nl`n~{Ofca%=6L;r$EkPa(b(B}ML#yXnY>bNTNP~t%Nl=run z57k4+k-OC(8F=H=F6}+AcD04G9h(+wOqjlX-qAoR^Q10?{TntfleX=@{`fFzP409E~Pi@3TvjURUv^UcC z(qXv3mvXLd&Z#Y-fw8@eHty|2OJR~qhuhwUMe-$!ktarej)#(%Gdm&PqGA1Nhx6EX#evu5g=?mC44ps1?V+4GC5()wNq6Lm(7H+%6ul zFj3D}VS5sn0?g6K(RCC|&Cu(~lGqSzl~(%I-!29`+}>ZC#1!Nd-vOm4k4D_B}N%@Dxy_VOc+gQ;@Tq)G6 z#~|(^FkY0bExGE-nqvH*S)sfFNIYeiGkFMCq}puvx_GFdOaViB+_{SZAR5e{2QlGW z#?YVn#&g{_EV2q&f9@IR=1S9FeRL@XEf}1UR@(16HcPAo4+S$cFMJ|=ztNZR2y>rT z+mQTT5BEEmXFtHTia~a63;U+k8N+|r>o3n!zNwt*P1!bHwMm4J(gt-=v8Jkzeb0wZ z`#e*4WE3dz+8*fK2zV&1(59)DTtc;YdaAnEVi2jDSJb!sme;HMnfuno;-5Ak^ zo<+}-R(f-or7k;SHr3AgYq9&nu#k`_l?RTU z&OOo?fAPq^c@N>LbduhVsbD+fbr0s&x2|$H-JO|2KZFZ$d358|MdfhAG6EzMELa2( zQN&BIniD1mtKnX6udEu&-iGF*5A>^VsbeogG!ixuwntWOn=m=rkRpuJqVa9{K?0<| zeu?G*k-O$LzPM9#!DHt>Vuk`=9rCbz0j(6U=Q@?FcNgQ+DMf{d+^efn7ti7NW-j&9 z=^h@`K`Y^u>)xbBWnJ2YMpVV)WgEKk9TIuf=v=tFK(`EKC4qbNnMlvx=%L zNf*~kGKwaiDL{95WR;^uHo+9dLYu|Mxx@k=#iBZ(wYvjKeIZ3(Ww^C7vj88YLwVD# zk|E%)+3|b91JV^8kEA+!0gs{7n;R?geqTpLwQdVp!1BN-^Y@azNSOVp$~K<)`9#pB z@5j$a&T>em?)Ku>Q=@T3>xl9YCjonMxtYxn4Jv%Ftl(iDKoEM%otcUh_2etvNrC?1 z6*nZRA6%Od(Dh^40fe}Yaeb18FmFg*=1Hvbp|dyd`9R(NBpjN2mT zUFbaqe3eFvIa5{Qwl0WMXJQepJ;rN%wnSP0IQoQC+d046)GPpgY{)NNwS tw_^rWaM6a2l2b3ppGhUjloMjEbt_NydcIkR0(Ssko_K>YvWCjR@gFM2fWZI& literal 0 HcmV?d00001 diff --git a/tests/test_metal_iq2_midonly b/tests/test_metal_iq2_midonly new file mode 100755 index 0000000000000000000000000000000000000000..a1277392fc238acbdad53ee2e8da3da52b69592f GIT binary patch literal 901048 zcmd443s{ubweY{+dAZC55EK*!!$qSqc#BmdBpL=tqJm;X+T2bHXf)s`22EO{BA_NU zj!7NkIc=qLYj+dpiMq8rq)ZMF`mG%Af zbIyOB=Vf2s{jRN+s~j;3+P?BYXO7 z_uN)EM{>XZGd})$);CfD#~dPXif+5B=<@XKz40uZyK3KM-e8q#`Z13R!yr+VL;3TbI z0rt&Eaq)s>6~O8HU5(%QBLR4iS_35{9c+5vCtnMJx8lL&pc<`0xYYgtvP} z0N&Oh5J~s{eG}e86+bDjxUaapV$m`RUftfQGXwBu1p80YSHtKJzqoi|S@ELf(p2Mj zHN1^;1MpIU^e5^5^ar+O{ox58I7o%X#kXb`Wf$kpDVP`F2F!N@gfm#Wj~@in0#hDA zJDLYNS@snbe>=Pf`j7!$UzE>P7|%aQB>4O0y!bkP53CR1w>#KQN%x1>2d6It-lF~! z@cNa5cP({~Ul91g9i@6+G#-E59HemoF{7f1!}d)$pQ! z9e~$2%_RNZbQ0iu4?VDO`SOK}R$L7)J_xT52}xgt-xb^>62z}``SQ|-?<;+<^lEr% zL3kNKq$K_K!UNN9&|gVIpuKkm;Ys?t;C-Er`=Q<%fTPqMflSiI&lv~=raXQ9+Q*;z zm~x<NUsU+7eRREgYYEXAHT0^4#bL!uYkHHyp4wf_}vgh zO48pA@4>R=3rp`Srokc=Ts?)dUD9LhWXWpBPNMldd7RSJ(_JN}^+%f~M%pFjS- zhnL((T7HQ4tU;np#GyfX7;(O$!@C4dqN9eR0@xWMo9Foqv{u3jO z_6eLB;M@Nv^=9zMy8y|}VoGUW;Q~JM9$xX_!lmPK#(`J){J_`pT?tR$cOjuWA*C_e z_(0+Dv&!c$FJ1od_}gVE{>ifCt`*~NU-nQ1#9zLwVw|x+OYZm|EL^^VoE779`@ILQ zz`v^>d}Vrwrh?bE!=JtI!O|ty0-ZhHcm=co+_HXf19<<_cfrkFwtV4tt%xV ze7#@aBlqk5Dl5>q?Ebx~)JRInxZN-&LN&lc9l^BFq5Q7vOYe$QokAS{&%c_&R=xPo zem&&)Cv&12FRvcZczM6>X*;EP9=lgP3;a*TXHVHF|GTz zGaFC8n%DTe7U4a&Kff`?G{8H~6y@#t>8!>Slg*o=ls8I6c{`Rld!5Q^#7}Cf=WKYr zec0xX3iXS&0%v?pfx0WDLhXK`f|e#xKK+EImY)q#FC1N(;%OsPH7TB`=qI+fhN>-U zjI&oMXK%YLtnvKx(8e=aA&tI?dSmBkQ)AZv)!R<_)L+;=?dxl%p4C5dy$6A39_#GYHD|9;&%Y;lCaRjk_RyNbVT8N*w(wiN{p`-y zfu+$`rpC+bmFH7kd6`b{?hR9wHeD5cqHEr)5LNlCNfmvpo4hktCJjsuRh6lvm0nZ$ zkHBu#)SiuNU#aWT2c?^PHxVM~+!OGcEwkm1hN!qyEo4rakX0Iwyw1>KpC1TAf%JYfQ z56;V~);YUPubC!omAX<)YR~SqzpEQmHK?vlZ=D8-UREKoKSGFjUHVRNk@kjAwt;p! z!M)b%?Cl``4g=RX#xEC~a%gupW2hNp*rvyMNAdoo-#_Hivfnp;e!tduvDiczow6a| zA4=ObkL9NIg)fAv^U+bzk16ZV{&44Uf9|N`(zk+tBkvpzXRiZVXw>Z&cSoo_mYRmG z-F1szawv;COSAg)2vua!oS9jd^~zcs@lfKWdSxo{PZ+N@`uJ1EUAtj@;jwblwpYW{ zaetKR*=1HmU3#q72d1m8V;e@}AU&_|}E1qA{A)y%G4yz)$u2i#7tk zz@j{nm-V@Od-VyszVsI*FVrhj!c`hcH?igogHRZCQUt74EyQC^=-f2xuUJ|C}K0*8m;!)Lu5(m^Y z6rR)uxXhX=8n+r+f#!EWBjdsoY)4a7%tH~<@)T&T?;ftTy2mjV%T(*T=k)~dgy6U|?O=?w0q*E9)l&nm!)iCV zoywV+8{*6?T@x~|;gMS$B@t?1j@g;n@K}~(fm54Xt+WXXdi_OuteRULttRB8<*w?y z=r1~{X|6YDfBO;Da#nbPNx4jn6>D;=#CI|lZO2qg4|Lc84Iiq!ZeIHG>*jU25?Z>K zCA7FSi>=&p$lkRgp~bCPZHq03?cF~~Xvqp!m8@AsHu{CNA$D}QI)0kAZQiFmXLOtQ ziPx(fN2@G}TmG2ih+b<+JXd8+d}5k%j9Y6>+>#mU*qp6-nk=ekA2c*Zw;hRu=bhzy zSLWC{*Ky@pnpx#&;`^g~x3ne8;i6AAhpL`z=u&8B2jlxoO-<_L8+*I*%otT&Zmx3F zsrOf%$fh3$GG@-7nl84U_wUIJS3Mfvtvv6aG>=E-K{xZkoVLF3BHx>Jv%51~_1blt zXQNeln3KIjb=@-qI4pOuX68vUb7a^kRWwsGyT>p`>X{>g%NXWJB6H;N;2haWpLFVp z-fDe_Pv*-QO~uwTXPhHcQN7F=zInWF&hUN^bEev4^Ub(k6{Rz0@}T>Nb*no|xA}Gt zS4Cey7xm?O<#Wv8`WU?uepMvt+9Xv}oS-WAclnDZGB;d<8TS}9A(yo4^Xd~z5>!mR zW_2Cx^cS5@bhgwp4_w3}8Lu1SoSEYioSElndryM1CCe1;YKvAyO|x8LKMK4o4A!t zQC^ikFHcJzI8(EvTVJjzOey!)nQl{_5hk^*q{BZ+$}RZ9U$lpDseg$6w9+3-^-689 zDw20^#HpgMaCUCZie#;5^A{CQQH-C|p zzIE0*XIUTNO9?|Y&#i>!M>J0uVRS}nMNCHOxI)5tgxd%^bxZm`68|&te-N}QPuf|KowTFisidb0)+g;MXrqiNRBH-} zG&Su@db&V`>P;_$XK@Vt_uu?Qee)vC;(XW)9d;(Gmf5^NUKKL$9mc;WJJfL!8c6#y zbI__PJO0&Q)TGBZSF;}1!rvW?<7jBhw>qI^H1y@GPH1sl=gb_<_rCSSfzkETgcdXF zOqVWeo!Q;BE26h6+}WFTSv6*9syB;yv!6M{-Y3=)$=syhvY2~su%DUeC7%om2O?)UVTdE3mJo^Qteu*+%m*+DN+&9;J=B zw2}5I_^SbzHEEpV>u^h0m#&6;KXAX$hj`2Mgd?5LMD%tJM}8TqCdoRzLpL24*}@84 zHyrU#>FfKRYv8i`Z~&Jw!lw1$5~V5|J_+J77F;HL9hYmh@mAoz!#KRf7@VIF7{|^( z`HO@v>?Um5AK$!%u@)TD{^Xye-c!pO^r=mItEV)6BK(8CaMQ*?@DJg+aq#Hh<(iJa zN;`JHQQT}{-8i&rVsqo8InBtDiROj&d8*3!q{vjiy`Qyo)x?V^4EL-EgTI&t`AQk9 zzJAyVocF`kw)R~)j%4sD;~P_4y}fOav*q^7{-1YJzZ1DoWXCf@oGnr&6&YkN;phRX z$Va(q;;txFWCp)R{jQ5%{pZah;|TpYptt0GV||Mw;So{}-WEZ*rO0>V%9mugz|&S` zt9!nDbOvjT$2X{^MaHroxy43Vsq?JN=l|v36U%q6(B_6!4_<7fUE63+wJFwDOF7?V z{}91#^y~0-%I%?C&8ig_>nSJg?o6y{sWzE?Qhys|1%Ds>{FLC2tnMOi;6Hk0QDB`?2~s5^p=S zFyqndViOk~T9lzr))Vhv<9zbeo5S7fmF|98ndTng`O})H#Ak;6)V62MfW!lLL?`}c z*rWMRuZc-~de|ej57#_n^kCcV(C zGNqqS(cgPD-RJ&1(AS5B4jFUlOQE^`eLgGL=hBz;!M>Do`N#uO&Y(@?frdwOE@oe& ztjPIK(Dz>~VqYV5XoFlke(NvlTyb|x95med%x#X}(;uBbxw~bEq}SRLg`WFp`4Zoy zJn!El`wW?fyTGsWnLJ0ce1n{0u8O}>wuiDpvz^b(akN}{pK#@U7w={7QEgH^A@R=7 z_raUSL$h(TBc3^sz$0`d^Q0YlMfk-jWS1vH;29wor(A2kth(g?d8@AWTrld*ab(dq zsqki_FD>jDfR|V8eDW=mv&9EYnIjIyz{MQN+Lz{AEb;~42#pFYo`H@k%F8k~Ge^cu zj;MR0JS{`!)M>^nS&Q(Erd%iU;~Znw2E6xZlk{gU^Oz($v4t`-K^PCKA(DqQvP%B>3lZNVXhjQ=$m-A!yI|U ze&!EzkQIj}K1;mBwI(C4YH(s+h4 zSu-uJB+MiGMXjc$@P6R5np&r&zNhAx;I*c0wH40h7Yj`2Ac|E;r2~5Ao5JtfQ`F=L zT@3Q>1{Pyg$k-MhX3yuIZ}E&F4j&j8rPUOUPpK)qfwWb|T2r=^P&GGYpw=?C{DBO! z7GrZfoIH=ch9~)U_8aUmo#cD0(QfEyQrZu+7D<<|&L$62J!bfJs}`MKo7!sc(6syo zZ=|_HeiT11IRe>`u?l63M)K~YGP$hG=ieM2>Hd&@_;K07jDJ#t)Bk*{YW^@p^$5R& z?qZ$}QJ&j@zgLC%a)JFH^!r$F%c9=?c$Fga(TTAF|Hd_IPmC;CwY4T9q+ryNs;$2> z4R#Cd^ufE+fQQV${wEV$vUHQn`>DU^Co0taIql4-Ui;)-gTipMMUH z!wB!rs!#9@EBNGXniWB>Dy2BYegj;cT&7zrHXUeSxgs zYR03}l2<0QSjz5hIW$X6a;TaXEBmY)XQ$U4em>W+nsy zBu^sn4pj}PYg})?cy_$%O_qA#VM9aIeW&S4$;-mS%BpX1^xsc6jX=zFD#}qF4A|wPx|{ zfd0noDt1$;KDRhkO<2DWnHG7noHByn!6bAu>+iW(&OX6@oyA?Y{O*=UbOMgCH7(s! zoV_;oE0Y|f>t?VoE&K7^EeXR9+8^1H5B~MR=Fc zpDuJGF6mo=L0_XMG3MjSA0)K&Aah)+oR9tzy_=u1In0Te#mfYCWh(J=!TR=U1B|`o zSv}Hw20E5HeS{vRCZDBD1vqs|K4T^E@dK|L=tq_>K`+rxzO;e0DPM4Nk2ta&xozO) z)Wdd{HpVJ6yLiN5dog|3J!G>zPqW!v^m|_F5xehjc}Cvq(JLpzW28?XXa2tlu378@ zwRBbVN{)(=K9qM&$kMO#PI##A@I%ITuY~bFPEFoxvbf%2PuYJwwlW@Vu@|S984pY| z%boOP7Bre2OW!jNdx_`6N6Wz-{Yar#@C1Lyh{M~nH0MWM)158p+Ar-w2LgXK{q-pI zCxcu2bqY3-UyoH=QhBdfpt@BA~lkmH9T0;Ju)#Fy~WgQkd3V)cQvx}Ez z%vpVQL4^X7iw#<{y3T;BjB6SEQ_7{?q9&I` zi5#pdyZ)`-{wv**zgpy>y|*@2BM+V3e_NyWFaDx;=(k2_k2x3PgO7<`O!#K8W^vWf z*QXbkXUtxGdgW5Te^Lo_mn^mi>Z*`Fhwhfqmq$k&+g<_PjngBoYzdPgyMU5B_|ZediL|wUltyE1o(AD4I(Jo#ilo@^d|$S(9F?c8;yubmdxVQdKh z-`WWb_67fWJ1s70=kWjHcFI`gY36(x1J;@C@bZCWsR!*dv^1+(v)H5$M$u1!a+a@^ zD^ERSpQ$;lX`0z~J!M3em;UcccJ`h`)@X0`7sWAG7cYWFn4`W_){FUf8+#)+ab)*G z7jo3I$Uv_7=p+(VuWP6BJc~SKL&jTWvbk@j%xUBjpKeO`4MKMY99uFxV2Cp2J(IL( zOh1`PW~yZ^>Mt{C&{Kbzi8c)sxv7@1=wK}RSN5Lm*hoe8%UIdi%>KLAf}A*bxTEfGzi}FS)s4?)uj+!MJHx2bUf%7i|oxN%L5L+DjE|CWhvhFTn-q$l0ZjsZVp~v92<o2)0#m%B1fvbKq=hw;#RD7dzn;VWgrD+LDnb7Ve) zudIfzoQ1DktDMZY8Nin{Q25I8tee0#_{!#BeZ}ycZ1y`o);p2)q)utu6Yv!`Wn4kN zLj1VI-(tP1@5filgKg>NJLxxBtFP4_kuwDL!BjKlB6?@ho+ZL3S*zvSP4Hf!OIfS? z+9YyDJ$c`Td)5`)Ee7uw_`>`9<1TCU)wuVq)zbgi9P_1L(?%TJ4y_M#u=g~>1FWon zi-#Y!CnH;TJ@c4dXsR4o{ny>ouVy}a%01gby)IDfpFmE)w9_b&SHO<&pt7a{o+h)kTV+DKN|MxK6#>TQ2jm9C*NhW z_Uq>3yF=AGX6({t?9#j8J+jB_+gn=i3-B)Y3{~_QYo(JBFxjDN`>_ zX(O+lG1>#aIdmY`(HWlaNSkX;G@}bW3{9qBkF`8--Mnb_B36_0C7Y@GrHC-+OOd9v zFFg;ODEw~*gjK&3ZFRoX0Zysrn8XQ`pMdPB)@X@1%h}pTJ9|y&rGz(6fR`#|v8Cd> zfz032hR$ezSPeEw-IosT`_WS-Yr4zJcnp81IAho0lnlvRfL^*7opC<-)r7|T^Tipt zN8~-ayx_aNeebZH>%n&?gYW!(?;@O;rwq6?Ey>io7+sU>Yy0Z3!smy#6=w_&eqYUZ z&mv#e6B_TZ&94u>+Z%j$vhN)>`USyv{@^>QBM;rw@Mzvq?_TENK`ZUVR>{05I+Pa} z|7Y2E9=ts;{`theG5(bMw(&na=v&4gcvp{qD&sa^#(&IL<3CBpf87j6r|s+GpUU_< z7|$5SKb>bZG%yAl7z+)Ig9d&Jtnv6^-2e?tfCf%M133{Y(aAT8F-u)zO4JfGTL(0t zLjxby&Ak*q>|>czZOKGrpf zm%Nzw`4Q+q;x)vd6@54H2IBB*Bi<@;@mnFkP2$L^5?3z~N9Qf^6yn{CeLlF{3a!0) zFg4@b>MsGtCTQSR=;FQaC|5%{FEDR~Hs3t*?eH2XcbN9x3N5{Vr?dB|?||7VWwAxx z3QfO1@4L&kNm*#^R_O43d|SQ?uJpy>3zTI(yr2KwWu>n+eMDJk`2FdDvSL3xw7ehf zR(uETQttnmc7b;_?Pf!7FAqjPHux*reFR!Qv~Ko)J?$puG=4eYKcQXP`9Dg#*Qmb) zn)n9oUZY$M<-S3?*C^K@G}cGEx&7$o8f9ChY#;4reRtV5DceW8_-I|B-D|-88to!S ze^=SB(Jr!;loi@N*iR2)$YQ~L!y%EykPBp89lh6IbT}x7`HGz_to<=Qi#zyQI8YbUOd%AwB8f6doMXt$XhVQbM>or@jO(NGwzSslB4k+u{e~h!#O&p!; zC`PU+!~b(M@`2b%ZNN_(P}9=IT)gSw-!*q(Kb(A>**#+g_FMLV-N=ywyL)_YBje?{ z$eyoryt5@cMp@UM*X+RWJP*V~%L952oo+g_Y8uBSL-z{96k zMn8OJ<%HXu8S3G)E47DvR_5#|UYWbEc;&6_#Vc=n#JO?~_VowWL?_m)QHje|k7<5j z_1NZ;)#I9Ptd}%v@?J?3o|H7}vY#*yoJRm(>j=QfX`SFkf16p% zJ1hOHebg@>TC6626W?gq)}&see1yuZm-3Py)-OL=W$u;ojHzbrmmjM#PfB^okLs5n zuQL7Qd*L(Mz<&8dnFH{IYt2QmZ}``Tedjzr^&``^t9`WhVc$yKmf{(}JZ4XB__Jux zT@L-9Kpy_mlf`zPOx#@4P?)UDzE1jEY{-%?WlfZg;=Mx8AbDq~2cGc_-)phO52ak& zm;OECW01;s9ps(jn~QvttYUrpXotY~8)>mii>>L{C;p#{Em6`vM!xv1f8F0(-gV16 z`seC4N?n4_Ui^h@jQcnI(D6@A#!pdft>T;gF>u(=|EGTFzos01nMOZ+n;-fgDRZs9 zpkKe$54~C1K_6UR9q>cn^P|7h4;_EYle975hkol#^BE_pTlNj#=7%nIrTm?K=<_Mt zaFrkW^9HWp?uY&e@5K+jzt4Ru_3a4K+Ck{gyn{0h@MQ7ZTuOQA`#$=T@rS#umOUq2?8+7yzA?P1M(L;n`pAN@=U&Ak7_kN1) z=40v>T|xhKqmy+cEoRfZ2fKEEv^#s>M>BHUKk8(kI|M!?zAr+%vR@S1{Smkv9O-nJ zE%+x+RLyBs@$>Lg%uK6_p4T~CLi4=NQ7W@*w933NUS-O@>JZ-xFLd)HTd-Yg^>%0w z9}Bl#_FOZx$Lzq$^q){G{aGq=pUl(eQar40^wl8mF6y?!TVKJiW*d*t|GVIE(V$1> z3~}+dvFpn1#CCEEX(xT&OI|x=JK(8V4?WC;ShV?Omv)6!c|OM^%Tx9#Cfmhy^LKlzSutIvYqyQ!9M$M z_{iMIH$N9W!x8aA_!CwD5zPVsZbl;;V*y4kJ(FgtYg}bpecdgerYp~@b zbJd|b^W`}w%oej-^4YU{EPUt0e^15^Z`nfD(}%xL`flp{obN9{CtrY5HU1PUCvM+f zo$KuB993^WWJ++SX(}@#6dA)m<3yT8Wp?Tr-n2;4muH;Fq)*afRAx$?__7F&@!n5< zXxf%JM0qaYpRrnx_sJS3wpm%%(xbK%cB>bbkN&bA-FK|_@i=FO1N+g~DsNrW&GP;w-Wy}n zn&KHgsBdfrdw*e!&0w#$@BJm2PlK+^n=QNtZeb^~NF(Qa9FD-54w3(4-MGAZI`iUo z=ErT!lUtcDx#;I|I8S0S^mE?_?$tas1)9@%K4uPm!W(pcM!-%Iphj?d$kI>6` z-R>27Yzv>=_&NUmU(lx?)90VA<374X3(Eh_JMXyX{PvKLx; zTl<;)zdMaBsAmWIE}QEo%`WKrSH2g%*!S3;MzfnmKx-3_V)(|Qf9O> zbK^iY**xBvIYYvU&P+3*eK==*MsW_!rYe`BKdc_0D%-HjS100oA$GJus&XIaXlB3{ z#0Fm;ttN>aZ`Kly$oZjD=7)NGS0yzU$#{q zq@UP}vX%z;m0nU)_z7@dyVta>kFN=j>*G=9S?h#{36CtNjxKzd9aS;&x{6h=@V9r3 zGYo2yw7FgEndA)xpT{itmss$}qYp~avHDFRm7R=R8+N8Eb*BXC{^!4=?$yAR`Yzn7 zdS9W=jl@Nll15lO9G@-dq!WHFcuKkFC?okZs9VaIQ~zQ=Gz|Gpvn+04pDE?~V5SrA z=G%H{3${oYEFYUnSEXHnTQm+{08HZod7r{N{*JOiz2ieCnkNx`xedoUN z0=8ALLkc{>SA55$F64NF{*XHjdfdT2S7>P?_;de3>=*18jWJ|yN`4vng1f{!?{)TK z3+|CTbLd99=;9@R25CJQPb1#>vTEr%!Wovezppc2$Jw0AxhKlkHr7e~aoS_sF}8Dd zZg6@hcDZ5IW@OZ8?**PlO(CbUHX5?Y5bwS8!H--0dp^T%C$fkk55MeeS&Hv^$wakB z_IYyNL}Y3kYr6P-Ydoj*LFw=7_H?PQ(e$LfoB87dpH-X{xv|Qg_#QA$W7At#^;DgO zHNN|=s>Ok?;3v8xz3tfhbhyYn@FEc8lNp67u4)J-ltQACWaVPm6R@ z#&cxRs;%Nzal~4+)yJ8@Bg+Zdt8iYm$A@g2h3?2San|Qu#J%sY+S-jyD^Ih#yV<*V zEmd2^ul>w;;yuJUBW}DKqI%0Y+tzibYWY&PABot&`Pw&?=Lx=x)9vn=^mEt!l#y_O z>b;0R(Vt(U^X&Nf^hW8+Pk!QTk-lg{ zFW(7GiGO>%p5Q+HhnbC=wFLJW=(dUQD9eLa z27Axw7VlYjlgL%bWQ8$2x@mxIk&4ORuLij8FIs=keiiLnKzIO`)B{D2-czl@TsN%S zV)qkvY{^}9ob!kS*8R+W4&U`-aVkS}df3E`^Pvs!{c7|%Z!o_HNFQGE?~yuQ;F}Ma zH^<7?X3Qi{(k~JhA5+^P<_2wPmot_6VAndrncHdFV7G6$v$vXkL=Ju3iGPgjDLUYFQm>5j z`}&N=FYp=s7=JDq|91S?I?%BTJqUe7fXi5LdQOY5MfAf(d^YX`XB)VSU3m#U(;{<5 zy2^uP9T_rjPMR#f2HKrP|8{^kzPN?gDx(1}7dR(PTICz8QIvbvO}cYbUZW2kqYABV zGDSL@uK&liO(Xxgx~XPEO2Li|sReIr7*=p&gQjGx$3m0HK_2MRIInE6FmCXy+7A3u;m=Cem^QdxWN?djzM?oX;03GL09VONzTANpSgv7 zd3`i^u~wXC-5!Ad_HW4(yRMQ_w4Oeh^ z^@ESUe9CmZE!@)RQ060!7_WYV4RN>!``wK%1)9YK+5#ORQk!rdY2yh+ga{9>Q%MH+fzmeC2ynJiceRpM^DK(@pq0=l-C%9v{-A%!dzMw_se%`=cEhG!7Z zwhhk}yu$Nio;!J(->+`6>yZVCdQ`zseLw;Bq=IS0Zy}yVekSSo={qZaHT~&|)zhD< zxM%vViYe1yt@v#E?<)Rv`YRQCrthh^J^42UrO7)AYLXiY!jqpV*qZcG!KCZeoEMT_ zC^(+s};t%jRo` zKEIEDstX;oId`pl`+NuI9vC7aTl{{OOX&!fs%zgIeF%EO68rX@PDmvNLk*slhRnys5wI<)Q zrlu~p-qeFUCHtaeS*x)vv_aeLr%av=F$#NziaiK_EE{pq{&pq)%ABPzgXiMsosDAS zTU@(o`!HmY3h;a@*#6C_Z`(J5=X1>UZfuoJf-h~7^=mLVTBoUjHuTERWvjVz?wt#0 zijXT}8<>af)ZIbd9_kL*C6E{9VY^Ia&&Rxjzsvc%!skaXaXdg@jVE47-NpF4FQuQ` zX}ief^;)!VUG0|br;w>6?+s*->lX+5aYM>}d#%#a?@LiJ)tsS_@j0c3c#Bn_AANk^ zO?sG81IMk==CPj4ovk9%J60`QS*n?Ad%62VWCt_x(JPbZHF3^Q&gJ;C(T74i^1Y11 z^P}l|&6aOQk7~ZRY^7lb=DRFpzjbz1G!^;LT$Y*9s}J>N_4+5h!gtA9NF^8D79~Fv zQdyQj9ibs}8=3cx(#f71?=Is!xH_ISSG_3p?^spVEPd8-!sK~5ntL1=FW*=v$F_jXJbU&(w#@nnj173;fzX z`1?(sI^frsfPwroR2ypR<4N`C$ZeGE)Q6^z(apYHz;^7v(cKMPk>d~Xu76#^HxKss zC#|H;tMmI{UWB)c4QMp9h%8tr^3HJL^}1Gh3fl9r7b-7ZwKbjdVzZDD&V&c#?z5~z zHKER?b&=Uk4Xm9z18b+}DC?)lB3aPjKJXJd>V+=;N<9*nvyqZMY_jA_pXBT2BhoK| ztGr)Q{%}SF=wWZ! zZ6;g#7U2iXi2y%{_U?t}3O|@jUsM?UAlkc>d9j&!(KK<4qf9gBrKvP4d&tTT-I_jC zH@Vm8Ge6&r3~~Wk{~~&8nYS|5+3Z7lIm;vSS?0COZ<*(xp|k!}_Ab!#Y{sy2ud_+k znzOHFH_3YBdo~9iklQ5dm#ky5w#dAb^+o#Plu^$Ruk=MHda?HmSORmfw}-u~^v_~l z4HO#Qc*H-+u+x!uiFYR2CGQ1J-}nQfJ>9*jGo{LC(uU zpK>NgZ0Rz`WUl?{&VY@Q^_+eFOxE}rto74b^KWPEzYSZ!t&Ls7bCC&hkPWhzp1b+* z5|J(X=IMFHQ|9G9Y$6*qo7=pvxLNELk0ImAdOw6Or!?8%cS#HC_6{0UY)hu3G$h{*^kfwn<= zf1o`j)Gan+*`tdsdl7jxL0k3`;-UllHe2?+(jV{x(F5EhabR3+n-F_UIq%x8VavXa z{QkCVhm6*^fvcU0^PJ28{=7uqN*_ z;Fw}ldt%M_#6~zBA^6USeWy<^GJ^i(9)&Bm*oa{Jtb{4Nd(x~XpF*Amufi#2wdXz2 zvxKWX2N@spBf~tC8B2MGeR0qAefcq|p7G=to3WV$a5nDz+5wJf$h&g4S2l66Ey~=z z0KRgTU2ql~uAIU9<7xlT#U2>&6&OrjoOdpIp7;6YA?%Oyqnr-zplFe?lXo}LhJc+p z&imIN$QTXs{s$okx+h6HF0a0oKF_7^bC?UzdV}`m zsBzfdbl``0=hGk3&ToTrr{BNlMjq)a=_3P{Hq0aE6jnl4ck&1id5`zTyX%H~L|-5< zS6=2l&)42RJ=`OA1o-#Mxoit_*9zV?@Qn~dS!pE zyc@ai?TX!uGd_FShplN{XTP^~z5NUPyEj((+a*o{CTaKyPT^GqcX;I9lbJ&p1v1dGw&A$g(PHfBV z`nYsW*KDF+cq7JbUl3DkKbge$+B*Ai(|Y?!bfE8PPuTy6jr`D1Emb_xd$UvJz;;3GAGfkrkZN)QeNI?x7OO{Q@%EQXI*ND=H6k_+$Q!|!|>}F zhR+YOOQx(n_<+0d1)nQ-o~0t2h9i|EffsEHHT}%~L9lHLY1aqZqei~8 zYeOq>a1S*F+Loh!W)C&dUfO0->+Diq-p^;e)>D3+ZC71Rh{?UvWO9q2IJ%F_;pg$? zidHK{-omHZtvCZh`>wQYGkcD{wymj9j)%hVr4#>*m}wq`Y|!>o)zq1*JcD;A&!wB8 z2k5Yqa|{;t{ayI8U6`mm3HX(5CtuEhh^_n{J^silc(DVyKy)-26Si+}HO`O+?CMF- zPbc~yvlj0bSfab?_3!TXnE!KU3`nuSgmt%wO0;ddc zlDehdSFmetGYvWN;FWq~eN!fE+pf4%E#UhAZ@ND&6HLmk4)c5{B~G0x3= z)AnBgZ2jC=ni22A20!R4Y48lh9&k7xHKet?TL&ejB~8sm|sm&64*-BzHKnXKcFUpIniuj-N#Doy*+h zedgW~nqhO;J6WsTfsF7tcVbCATjFtnyC3&9Vozc2J|5|8d4c{MOIt+0Xq4sqc6{!n ztdy%IF8ie|ls(FxNGE-a`vLc(A9{j4`&*AYGags!#&>8}1a}tbz>)ao)M)RP&SA{+fHrj~CMV|cAQYiOTAE%Tx5ld-TOB?jQ@uIq_cUH<4Zb6vXp!(hpwr@0rJA z;1f5((_*}f_wL}`Q^XtSxBfnP=oGKqgZm5082YMps;R_OGp)Z*9shS_k3U!Bi*R^_ z8J=N*cUa*eHslNLe$jPLaBFN{&(~iP*F%Rm?fwpX2>G-k<0F0PBGK zcdC9{mlYcAdjo#`96Uwpu)wQ@4@$eG{quRG?du5_#Fr%HtXz=PwYIq~ITV}8PwVYE zedA0ak9!`(PCkF={G@EgBMrQq=8`0qge1|enIFNnN$4Py`_TqZcoJJ@tZVXwZT5|8 zU#=6Gv}V}Lb+z74H>ffXB7`mOxk)9@WZ%^Vo@gR?1S=8+o~r&mb#24G*0eA>tLeV@TbdTdD&i*MTGIlhdvfr(lQpkcOY}Ke_eF>K9*-M(=+HvA zGtlf_uqL`LSJ&MOx2UE?A?v5+M5`Bf@XaaY^m@LlVV$2(xg5&npf_5e>qq9ARZA{y zTEMvjDfso!x*?b6pV&yA#N&zgu8mJL&stZQQoXzGN6=(S zD*PThe48tz@xr~PM$U=6Ykm@aTC!?MqYU@d_kJ=b+)=$89U}J})D!B2ePb@=yR$e` z%UtZ^A9>74;YD8dYuz^o-br3QdG8Opweh(JxCi(S%@Y;JozB(I*DZ!l?to5SjE?j) z+>zj^EzvwXw~X-YdS`-X!vmV<>F8TLVbl}OgYM?tiOeTCD4A*^4z0GyScZr zM`KP*GR3+NQ)k}8!8-R*XYIYO)jXOx)|pM6+7F4}G}gI~I%|JK{ATj0Q**vv^C);l z0y{zC;5CBy1c`&!G~zQQo;7w`hunMpRh?FIywT1Fmj~*sdE=hfYS&3SGso8C{KIRt zCTZtQV{6{{A^FnIo5?RGU)nHrY)zE&^;)~cr;V){N&I^fzh!LAEyVLAo@KOi8}`w@ zcIN#L)Va8?&PD%y?c771#&|A_s*7JR&|}cif5(_Y%Qy18+7&*9yDbZawpVwCPx%o3 zb%3+`er5CR_|Ui;&?Ea7FMBD`&+q$yeJb|+S%t>EoxREE@UvNCJ4fRefbBkRgmQF` zn(nZU&>Y=1@&-A3ts_j1?qp}nOy;n4gl_E3y6~aEKHD3C%~$wt2JmEk@0^JJAD^Ul zd;oS38f(#~BRy7lVH@9!>sM~~-axrhLZe*OC{Gu<8f1re0y-(P_apAS?BgXew|y0B zww}g+Xem6Q37&A4wPVh3#k}Lz_mJOa&uNdO-~DVtE%wdc|K-D++lwQ07CP z`-X-zt(c)Vg-}lNe#*NWc)o&TYP#R4H9bw04&QF80Cy2Y$DfmVMU+|H7 z|HRt#N2h8ULEf9Orlt)wnrA@?Hq#XKqU>?sGu9pI=RUX&Q^=9^=w=p7jj7wf8nhuK z#2sP^ac^iVo(k->_thl7w&8p*-b!3Kry2Qgs@FDfzXWTp;qUM()~6J;AgPu|A%A5P z?&L|0pPv+p%(W9)OrK(k|Yo5MMz2eQbpzk&i>7OOg&ecVAL?TA*AH@etmL z?0@cN9Xmh>_JB})1j6tUxb|M4kJdt?mn1!tu_)=oA?os<(iSGYkWra5A$?&|x2~o8 z^sw|W=c1$t^|yCVM|OU+RlEP!2UadmI^>+6bVd(NKdXnNA6vR+>p6Vhj)kt>dJcQ* zv4_@f{a$ED!9x1+fl!+}jD7YweBjRM1CE?yedRuS&%98pdl2o6jb4@%x8;GP!O?4y zrd-!pFeNmiVoGRiMf{eMq$#08D>BK;B+nkbJSmg>O!96f?`HCDCVeyMZP5Bu@}`nE zmAvb)yH6$m7V>T(FPZc$=z|A#61 z$URjr)WrdR0KU49hMEh`MaI@Gy1c03z{b*w18-IpxFgL42j27*94IS|`an~cZ%`_Bq~|AnU{S9J?hlIcH6Sy6fUGXK9Y+2ROJ8i@Gm`L6;5A|Gi`Fa^ z?wb0-PwgL~v&}LMb>*=SZpv7kG+mEze;5+(4p-rB8+~Hs`4nHv2twJf!%utKk)8V9 zN&a!@l(R6N57i*|UejP-19R{<)IZVzp6t<%wU#8crLIU4yWiRG7bPDsFG@N(tAOK0 zDmIGm+fwgK+QhpLN`H`iplnX^(dao`Ef$X4ALC_16?-oD&4F2S__iYH*s0=-bK0n+ zV;>V<7g2ET>6G#3{-d0V9eoNCW!?K#Du2RG`;-;nvCfyUA6Od`oQPz*$vxfW4J~X}y_nyNhcnsP09P7(@ zE!^D$&zKMuTQ_BqYP#84Gi|~f*iZ-JmqEXu!-q-kkrNmtK7Y{-=pUmgdt8e-63&_0 zkC5939#EG=CoJU)!T;O#m|eqfXa3&?AGj5JVD8esJ?A&{3ejKb71rH>FVMh%ZUFmS z)7SJ0_!;_h<-DHGJ~o8?Y$*HMF!r~c)mx3vk#6!R`dJ0-UD;Oj3FsAu5N4oP_;-B2 z(KEFCl5q0JRVUIoXT8@H@3T($g*~m>TW7gkaDqDmz39Dq#0Nw40W(rM-_@80DZ&fU z17tHNMHi5!9!^SStQBz!aoK-bgZt0Xgfk|q7z@6LWyt<%$o>v=|6`E%%gJ|;FX^$Q zpSP%(L*7k=jb88aqwER(f=vDdc@qDd#5p%ttgGec^e48@$$Gx-4${%=AC6WCWc1lR}XYniY}^pkhAh2wDlnW zb0!V>0bSK{xtn4={{JFRcRKH$GT)!{N1nx;=L?HooD>e& z=iGfI<1~x@tB>O@Lwr6r+vCY zD#5v`hQ2rIv)-5VGWD*oE=($1wIXSvd3n+j%0A9`yv(>1l9ul!{Rh$~(2M?oX9@mF zo1u-(&<422Y=#cDGqy_@^Q-fJMgHa>eVoxZZ=XHoxw-<;?ZT^`C}TcA`^W&D@o-+WC=x&w3tXJl{_9JVt)o0Ijiu_4J>@Vr{K!*PeKF zj-iL$(}o}Tzu@cg2J7h#IRmODd0yjP8?vm-r7rBRCGcj^?TAbx zs`Gy4&&ppb(XSD=5WklGCdAO+B)UdSsI!-uLh{w++$-hF^w2A5Gv9aWX6%WUbXh~> z?mhEqWL=*j=f2}WH}f(1X85Uv=Y9O4ML%jD@p9c`tlPR~!A>~@8kVys3EuYyX^szg z--7;B{A;&`t4ZS5Ukc3#jIqE{pM}q<#~vqqTJpp%v3LqiTarxdbCCy=O}DAnkUq>P+} z=vuBkR`d-v^bKzG5Sy`s7&@_mk)9}Y4d)6IJkjVGWIg;4*+BfuM6a_Sn!6r(`+fM$ z0miMN^`6&u)UcP|Fm-H$^B-Q@q1elBm^QYd^@rrMm)~#;`Nib3Pv4L=wjsj#`i^*s ztFaB&6Td;?+SrDv#Ais{G{e$p-fE)K^P=CE!#;eYMnA0!}s5S4({*;8a6>wbWMvP7A1S1NCvAyiwmN z%V77Pzi08hhkWwiRVQn?C#S*WN#*&FC(9J)+reFgqGRN4YR+5O+|k^HCwj(GY)HQ> z;Jm&m*04Wx%AR#}Hv3op2j5c75m#z*3@z6k38|_j1>Q0wPW8(FsS$c`^PTV;@w4Xt z)a+?T*4TG6I7j>TtjxpOe&r6n9Vk~z$e#5b$HMe)>{;U(|Nh*&W=CD} z<W;n}InxP-FzovBbDIm$R9^+N;I6Qewn5Wn=Cp;D={%37@3D_#XRb zUP?yKJt@tKeouw&MUE8zu%mgY8ORy7j(bw&ohM6I#$BbQ_cWo5mz+P@_z5%xJ!TU^Q!h9k9^?5Pc=d1NIQ}o( z`1@CFy}?$sb;63Ot!;YDk!-%7&y(_=8qm(!?qU27jNzowJrDdNYwNZ8p$#3l(SG=M zKxe9A_vMYs_>~rByL8W}X7~*44D&hR_xF=`GT6>HD8u=&f&I%!JLP_3fitQ)GSRQ1 zMt88@{xL+ol7;VP-=2X*J@y0o{zK&WzvYZ{ud}J9bvCW(Qr<{+ZMfD{8?JbC9urR} zPY6$V(*dQ$)P{#OeW(Kd6@D$jSHoUQ;C#Ttxr;qv?Az|w!fma2qb^;_8wEVo^dWw} z2hdZQ^p-i^1c{9*;&|%%=ikUe~UQ- z-N|?u{^FY1#(l^(@#9v^+q1ko$9i%eJAstJcP9qBN{mC~Zq~pO#^QM1vJ6k213F#S zT&XTjsk~=uGw0D_CQz<5?}3b(ya~K}pn1ZV{wwd~d}Aj(4fz`0Qd78*@6&^DH|C{h z$o==iS0rC(bh<&CgT1#&JK>4RTAXp$g82GYqbC$!TH&v8mCBLAcd>7)#!n6XU#vI7 z_;!%@M#4V2KKQ=_bX`km(Dmf8eZ2C8KK&l&IlgJvjs0r>HB8{Khbp;-UAL9-|2BJY zEZ>#jKR1xDTJCGcXSDy`4H;>m@Z45-jiyXF z@QA0+g7(ZS7dIzkf0TX|U8LXyeHi`}!hi5daS45x?~-yRm)O_2zcr>4T461WN#?zY zyokHXn(KN0EU-CC;M#CJXO%M;7hen;vi4)(gg&ALT4=$(9V7JcXX;?A1{!#h78`d1 zzK;&2yUO7cw~`(~I!g<2rA^piw`jUe^k4blJ_|g~u*cd94uL+3Opjby)*Pu_oU#{O z-jKeaO~o4eW%_6WebfnkXJIdIWls_b&P&5CPB}wAjU&$izQof9tX$gcrw!7VauyN# zH+(}_8)K!eR_0pNT}zu!Qb#Fq%FLBL>q*{q&{t7+RT%Hw#81+I2?3NFK)D13B^HV+Q7nNVh**fI z0b!{YMzK&$Stf;uciVCP}-=(fZux?Pe68Apu?>T2~hRE*k_dI_*f6VK9 zX3pnZPnk1k&YY3Ha09wF_~J&2pI>8RDS4E|7}1wVf8&-V{}FTo?4epI<=arNL&@S? z`WdP7ZXCI1^yMDWmonIV4u{AZY{hkvm^@8eEVX>eiRijlVnm)R|9F=w<|cj2P5yTO zpCtS|PV7tgS5Mbw)}`5uv4?V0vVQaT=z4)Q-rq)Fx@C>8aoHj4pGq0cC;u7n?CCM% zO|HrPas+Ywc4ph&f~!Wq5x1!kEd@A#pcDyyZ!TdjtN=>jvH=O_S zsE47==5HEhbL}BHC*AzK9p}Ao`E1z>zeNLUbW6Uid2GS6qaI(de$y&G8qG@i#s>Z8pbhj=ylcll9uDqO5DC z?Vh%7+Amq#N8L&KH0hRYDn~s^dXLYy@7QANqE^$_P1`fq+V>0TJwCtv@!~ZX^<#eX zPhj%N`3dP?|K;63(Iet`SJbgNOa^z%RP>^K$KKT^i=tS?{{?nrpD4&G+&x~3CZGfHPgOm;T zmC;f@DcJRIa5Z2~kT0>@OU(LV{}_&09QSgZ%ePNI;YU8S7e~qx9n(*U=^I=>I)CFR ziKWEx8lFv*f3!)7<68P9iQ`))jtST>Kl7Pb9DlFs)YGHfcT}n*$=&zq;ISjmu%W3wAN}aXSYb@Qr9f^z23(UH1Yc=-(>5JtaK)me=U%cwy z{!SSTw&Kf5_|7LV7F^n?{LY+@z;2hfzpNGXL% z{z1xUAu_7DwuU~J{`s=yNmh!iqhG^1T$!gs-pLrxx$~8yvpAy`OdiXWWKKe zw>W*fWz8tyKRpxUlFG*SS(5(Q*nV4o8P{*C*vI?gONm7aW37~rv8Br29KJx=I@ae% zoQ?#?t!Xl1r6gzr7zqxYQn;`qpn)`r)jIFKRa#p zb=#)h)nl9Lg9c?*P5WiRpQnwVzMj77)lt8rFG*jxS>dk+|4;BwT=&|vdwRS!?Vt2l z=hwqeRQNwCd>DM-re~)ux~_6sX^+Zjf$N`~mQxQuLgCkr$^bhX*{iO5b=rg;uc|&V zFRN-)Ugqp+kp)|)6<|-n!atAd0{-)G!P26o)BEZs$OJ^4d5E8$S!@FCD1#|8dkw{x!2D)#~-6^d0`zW!c|EFPq1h zM9SSydF4RQw#V1k9SWJp-#OL2Z#Hfhs@lNX&?!6@CT$FD=mH=6>qj~vqY3+E-Oo6p zG0)lVKZ-2cq5*y-JsjlgvQm&+Vc9Z)0(quKwlp27UN;$%neJkg+ObDHTt~ z5UTG>s^`BvJGuUUWByGMf-k9_zm2d@Y`u^Bp3uhFep7i?pYDOgrMpuKtIBB4+ZhXX zrG4+Zys!!%F3sIFx&JY~RjB0oiFpu?#N?dUbNE+tNbR-jj%WYZo@???N%^mW%s~X0 zXV0-JS0(=Lg30Nf=2W$|lI=A98IpZX!77V79qybE&0Ek z#d@BJ>G^6!~a zzg5*A9=^LzUCCPN@ss-6138!Xj!V&1A^R<+W-Df^szYs&;RG>KUgoA zob{_U<6Y8AD8Grm#IhppMfaKV6B|@}{~u*1|C3s3{yQbMDjC#aX-h*Xo5wx>ovM$M zvXeE@`=#Cem$F+EE4#D*ck2ICcJhC`xwOaG+#4h&OX>gR`9=C%X+LLMcV0XF?MivR ziS9^XUNeRFXQ{kL3-Ug#Iq%h4@V>Jp-)SY-Q}`aMC^tkoF{f(zZTa5B;eRK`;^-T% zTbTM@(|KdMZ=OG9Pp`RSZtk~ej9ql!m=fQDG54;RmpU}GBz69$3sbLuaBk{_zQm;s zedD(VeOGNgVFmM+^BrW+clB1*&O<|0ygxZNm>sIxB(yM8HB;y{vQL@SSJp!{=h`Fw zU~-qdd&g|KCplyJ6^$}xUeP$Cg>UTEmcCHcC;k@6CyRo`TNvkTf&L7(#vdH{3geH> z{^lcFAggsY|1m=6pZyJ!2gbEd9u(Iixoup-rQ{ekw@=J4*pEu&JwGt zb6jTfC*+^JCpq|ul^)CA8M3Mtf=P}`PCkK*<>Y^4Wo|ugu}=K%ZgYq$#Ndq(DQeInOo{Rfr3`I=vSGj1X+`?}Z1 zEamz{^6!U!O@1TrWp}3*mtB+g8|*u@Tk%)sf^xy+f=QyjM{Y>Um=c#hay&j1u~%$z zTrhd|-TsWnX5$!jC71oN$71(gzE;VJ)CJeRDYQ?KCvg-TZsJ^Q@X9A9BS*%ijeK`( zlZ+dtHO&adWsbb{p48yXd(h7}fs?OI8?)v=z8M={U!1rO z$>;czlXu>g8hrb%)Z({G(#Aw02x|GB5O-s|eu-CY^43ZgVdVRZ%e$Ul&3u)QkZ8H7I zZ@>S#fYbjjY&-;m|c|UQA`JZg+#pYDm)3&)2B(K^ZVqZP0 z>Qnlt?>XDTi6>v?Wf?fiH|ePX z_F4_F*DCGD>B6_@ybCKD$-DKb>>DtVeJZE0r|JLS=RCuyeJW4H<>kFX+!skalZj`C zGvoOtdH?O3;y@<+!GR1FizUX8Q+bN{amocwAH^~@yswyw}jy1S+ z5%c5q<;uO@dvBLL$4k%n-n#86>tEJrzL#U~U_VD3NBj%fa*g*m2YGjMkoPzT8wOrr z&&ijoJEU&-dPl;`)t!SI)*cZ)>xF~!EZg64&X{NU=6x-5qX)S+$ooS-`%^s0waR%{ z|E3R+XQ^Lj>psA3^PgSlU;e+w{Ly($_nl!oW0LLB{u{Q>Oxv17xlLq!P7!4kcwY9o z2^^oyUP_EvkI(7MHR5-ZdFBbQ=S_fpd8GcaHZt!b$}B*c$+OkO#`?c`BPoBWkBN6D z6kpVWXEO39cCuz9QkT_!8B!nX{pU>{2li1xewVB z8AGmZ$G26;z7N?=k-Zn$&5-?_>|;W`+>L)C<2%+otl?h+BAb5bgVrflY7WO__V+5k zJHB{Zigjt)&2hQS8^)&u8E2E1vf;3A-%`f!Wp50!*Ra2)*w9GX(CGNPbMIRzW7xHs z@_$a&0vexQK<^9LU#3_2((>-_V5Ouf-A$b}t*lpo81-A;1@rxSYC%O@@wUcV z_f7N*T_<9-vU*y@j@$v82|7E|g>G2TrZjBl9 z{va_-v?4D|o_?k;=f>rJFLBJO^QAlcs1&fiY8%flX*1$--}N=hn?Ad2)+iryz`iDG z&!Jq>?=V+T=K2~XCyt#wOa8}QNxN9Qt3&@u%%`2?T}1_$ z!6Vp1l09{K?w`?)`Mtq~%=fUiZU^6lZD$RcluP10PJCxeJ$apd74M>MlueH>*ZHXoy0D%hJ!%7%4ok=%yjlfQU%vi6C4RaN`OR!Z24Oa77n zFlPBu%D!Q5-8rv!;JbFdA8lw=T}<1RxXXOacE11pZ#J>N$N#-eEj%`vy0W@yolV); zl;g4KB6%*vCb8#(3$#5_H>x{_a&vrvz= z*R7s?^i8gP&e`k(FZK4vRw3R^u}>mJcl_(Pv{{ zy8Krm=O<|Q{&^`(sty(_OIic^liNVuv6%N-ly%n&zwP1@a!X(cQbXcNl4mHic_KfBi_sz zNqsv`dwS1jrONt3+Qy|#y2jTG-c1DJL*-etzZ~Wn5(A;~yo5mVQgpJPP}zLy z(Maq~Y7{DGU){2JnXltKxml?EJ|FLQX~*|--l)0kE6}Jcp7R$uZ`_)1@bQ`X^vm{e z-Xt?r{yqIjJm)`f-n3n)yqUjg*?i)i!W?e1j-m2i{$^$IoOj?nB_~uqiF(7k@XM}a zZal7QsGM(G%HlcyJ?HT~L*>u#PGdfPKFfJR-%$D6Rzg`k=Q}xX&_7gul)Yu=Q`W&j zq4J6b4a*jjCvi6S@iXdFA^ng$CUp0y;f-`T-xQWhn`2#Ja&?wf{&fedywsPNyjadV zbKb=&ujKr1a^8dUKJaj!=;u72^MUaD6O$)foDb%Fs8zm*{~>OW^WmJ2w90pJUOn0> zmvY-pzjBOn3nzviV*eqvZiPCm-jycKVa?f1>^mHnwEJ89INrgsWj^(O-s_F>HD0>E zjrHT}l%dQ6cjH^47ww7t-?yyI)61vLTGqzeN1aZYQQm!4&+;C#UU+h1|M$q7R({j0 z2h*&5+;3BEF7Gz02l?GtbNAvOJ-jXI@xI~VWzWjr@bXWrnBKA3=pYytXN2)u=abCCYNOk9U&g=FasqUP>dELGv z)twt~exXNRL(YeL&iOAk`=n4GZN81G>cSkx)WWN_-rwfF;0q((EqbBxMMcx*PTacZ zf(L@%U;KX2ce$gArWanlwTOGT!<L2&mPRl=;xLAALX4KedPy9UGWQFcn7(+ z*F)#oP5KVWJ%XO2yl0TLQ-{@kAIJHfNUzv$OxF9vyTT%Nu> zxV-yciSXWolv;=^$b z(9_U48dw-S+V}0EqhsGLy4?yEeQdQ%y2A=4ee7>lbd-02N39mg_hG-xpX=x71m0)h z2mR?WD=B$5b5%)Qt=v-D+GN^bprbE$k)Qt8Vo$`5{@k$DEIHR|N;)pLa^6)kwm+j! zI7r{|>+H*@1I5&ZF+AV%Tt8Gl*Ykh2<;<&Z{y+UANuKNHAlEvOW3QlYg}bqb*S@ZH z;y(Yr{@4F*!+@Kv-Z1FWi5q(Q3M2gz3M1Kxg^`5pP;Qpbn(-6Q^*PrB?6^0!pURmV zuv#p?r#O!|Yr98*UvE zx8e2%@f&VylCa?pe}fIKEu{lw4bpIYJNtiYW(?E+%*|X-kFEGsAs0DwXb^vPnz#eeC)I4^Ktk&;yB_t5;z)gG~@_yB=XD_bjH9d z_wEtID)*Y+_TXRF?Bd;nqUzd@UG1D-*DUfrSIYaj|CU#OpF7u&Qp?I8AIIL3_@J&` ze<9DG*5h?M%QEwUa?zUQ9qxz0Fui(st34vS<^76(|fWr68h zqV_MHQ4{xYdSA|!oCM@B7o+fh8y0CQc;u?Q#0w&UIBSc%TaY|-teHLi_wH|<*HB|^+|$z0m)$kJ`5J8R-X5z9CDW7*5k zvO-lKvp)9YtFzP`Sv7j3tqYIzeU~EdfV>C52302a_p@o!1zFj?f~-P+0d;j_?!lIJ zPN&d@EMMaB$+RcAmUGY_0lzHZ%d@lokPz9xf3E|L*=y|j;gK5Z%j@!O@l{#&Fv?NJ z$hxjHcP_;4v-wxQ?u&1_o#xmj{&@RV!*>g`MV_^H%bS^YOTOV*%O0ZJ{Z^zW*I8Gp z{tv(Q-qnNF-unlRCkAc5H|x}v_CA|`d8+SamCe$x&4{;o_Z7j{$RhfKMCwx>_uz@# zgJt|Z;*rHG6ZgeOXe7GLUDM0v9XM@Y=52ybG5W+VZ`~g?)Zg6O4=qRT@_Wy*=UnTz zhk<_>TivvdLES&>>ih5dF1T~yy^L5d_vhJ<{E253uJz&CBKB3s_Zjhgvs6O4{2=e< zc!w%><&ZD9bD%-uHIsNXCtfAkP(i$o3}KC@iB}2nDj{Aa#H)gM%_LrGj*{mErT2rb z_M%(jrCvnVEU->Jf?r$ks}fm>#B4Azlm4dk(M2oe*(fkS@l-N;*IwJePE<6(j)xNa z|JVKO+G`W+Q`oxZKWi50n4S5bu>aNv{W^O)VDS6z>0(Lum$i-IXl?r~H z{f2r!XR-ff{dr88>(ldoerGi@<|Kt84|~?bE+67=Zn2&~`24Jg5Nh!aq&jC`Bz4|S zzhC#avkKIC$g+KmTWolvalU;&U+Lq%=4af-v(gUv4uE&-tmCcZdmO2g&95ICDP|b_mOWCisg%gzi96E{XB0v0V(|!`87K?|e{E~Y7SJdN=GyKJdzw~q~ z&40h)A3VZ-0Y@K>Jftoa|&{5|XOpHq+j zL&M*}@bCJmhvr|Q`Pe>ggL?d%41b)<@9VGmpVa(T zJ^r5y&n&-nhQEgEC6wudv@JE}Z%p6Y!P-FG+2|`ALRs=&1kA6qhf>dmFg9dw(>SxY z>Hne5?-!l*>ij2#b#>m`&w1O*KH+OZTWM0acKmO`XWGVpB(FX0~s z(r(vD`dXb{E9oIR?b^Evz5lgc6K~V5H>!5c|Gxj223HbFSJob;?RtDY{!8ld^REcit}izH_1kq(J^oNV{%;Ncd4|7! zyB<)Fzh^!Ee;EF5hQEHh&a21Yt{(q;hQE#BuiviI>+v_M$Nv|@-`M4^*RGT5@muxy zUoiZ?hNxSV=?du|xDNzUe%r2(gZcl`uD^z_e!D&xqP^6=SJiJX=H7J(tlVn?JcEQ8 zt4rH*@3mQ(){N?TbJ7`yZnO?`w-XtMwqqO`U~bU8*Sc62;2|350KrcyWJyOkC8XyE6)T!*6@5c(7k zV~4tH|MPuVTpM{GwlOxB$@?(Zrc&;&vgWF|0nc98KeK%|wU0rGzwrwdhb%Sbn!~#t z>%LZYpYHlurJB9)c)qQw$pwFYN4q)mB$KgIeFON=)u$>RYJ2KV&MQ*0NN3He_#kWR z%tLx?=3_oo>1ml~GoCA`c)8Kmy@UPBXHHA9OBt7yFvgR2mQ&!DG2a~dR)_i|&kobL z7LvU@r5x=GBYMy8fb%<(e*^h4_PGXnfWv)WxKhsPXBso!P;uxMV(U+2i(fgdZ6EI+ z+Y^K4;N;(SO^5~WwwBFI%Co~<+a8yCD!}^1>Wn!4!(=V9hIO%@oENw6(2mwN?~^`= z=Ws*^YXPb=hDNfo`OeH|T~--4=2V#RjJ0^^6Ruthy}Ud5726^{WMyBKXUn<|Z+_m?Vx#=c=6OY)Z+>mOfj#mIE@#}ceMqkY>*B(Qm1%9xG%@Ie9TJ0f&@~)x3|f*7 zVoL};c|N^~ak|AdnGY3v zEaE?8#=T>{qV22lytfpyg{`*Xd7 zZ%y;Dx%#0CBh{pT;rUVGZH*|5_$h-SwcYIJ@k8p^-?3R@b^@%zTv`};gKJW^r5-#@ zxyv)Cyhk~V92p}?yth!+QpdJ~mG>`yhH`!Lwe*8S4m4!Wrnz&RepBQeP|v>X%PLs( zM5$MYsY~*&>CeH)+K`{9MsT#wwC{zm87 z2U%Au?Pr+5b@t#|qyJpuAo@od{da+5|6Jz|sML{|{&&wqKmWN?*V3HH&=h1XqMZiL zZK}RwR`tNx*_eE>tLfP=MpnGRh(DJrxh7^>RZlilpr*I2nf z%U)-+X}4|0e73sU+4Onhb3=H}rEYEU_p#UZ4n<_FQAxkG*7Rfj$(Qp*=USWZpw1nb zQW#lG%%vO#kk=R9<+tS758q<#D{WaADQW3b{a+q=73sc6#WY`J4gI9>%{F{p$t(RF z?B^Pr#ku6qH2EDRzc=~4b-s!x`DG?QQ}T<*FVgu<<@}$8v1ej=&dhma(aO?)`g9w( zhCW6xhu}G+dHkH0aPDu%zo2OYrC$|Bs%d{q&i5}@{YIzZk#?OLDql`=tsLLm-B=iT z6F%|%G}lzW@l?_w!Io>gBRn|NVH1J;>buu1CJ~-zvYbF!HRiVsJ6hT6==s-r_9!SXw}Uj&^Skd?(hMLO_aR?hwK z+oqlN1}pa#x&DpuqZ`*`Pi`hB=~J6K=Wzav$UM)#T*gDvcBD;68*2}hShQeHybXOM z^V=VYtj~kN54k5v>;D-KR3KV!}}9s2Kui88K=52m-USYOkbV|PLIQa+!I?k zD>;|G{uk^KU5Dt;AK@DQGq%p+9@pCWqu~{dmv^b*J7GJjZKhulX*eu!8t*Y`J7|DvDxBS+S?#IF|4Jo?Rhjc+^9 za~gc@Co-Rr=~T&is=g2YiL_k%qoP6QCFoDntqs47mgnJghcUy(AtmkX8%;dcD_n;2 zjKQx{l)cgwZSJgro|OLx4`=^1f8e-}Z>!w;>ef4n(}&38UbR>5Rosh~)$m^kRgaHF zw}{?l$THjL||1Op^eN_d0 zmH1dfzg9uNHeAy5Nfqx~kztaipQ`9*MG7TN-&N7piYOoH&nn)wA_L_*eVoi!_m?#D zN)^m2oh#|{NuO^;dXtuCjT*nK8xA_%IhXb&F>gVgm2prHuD!@1W2MVjmn!W&kMj-s z+6FZiI?>dYeZySdG{xJ(X$MB?g>GM&j@(zJPiVt6>FZiUAJh3-&LvH0PifB9K~gu< zoh0(4js_GBIt`%mehypoSaDY~@GYF@!F%hJuERm+7bS=Pm?_%a`57wxkE(OX|F^o{ z()mHrpz|GcGyil_eGFsnBb+PS8MhoHEq%AtxxnO+Rn_!Gw2@^)N}Ah~ppyAQcSpZJXpr!wuiyu*<3;%scF;oZy`_LsC! z_QzrW$@+N*bY3lT*;I1Iar&_{d0Q&p6XdVL-`QF1Y}t>b3+s5!z#Y)IW8fH1GdI(Y zc@C@jTI%U`>gfa6CG~L|W7zGztw@-9S_zFhdI*E6QXEnXRg zmO-;vFWL2tRV&#uXGS+DPxg;vgWq*Xt3|AmoO z_cw_Aj&tpED`L{hc@W;$!Ady}z^3u@`r6`0?H+&R@FdaE+PMdelw-{vkdw-(h&At)36&v;I6^Bfn-h@`sB2H0K5|qQ7>xwm)CV z4?4X?{+du^>>4Z5L*$bV6d3s>a&K$p$e2ZVrCpu_rvG3qzji0`J81hyx%ThW_Gc;i znNGUMUx)qc7?X&6($(j=_G|gWJ@z*NQ+2+ve+Tm8wEg{D`*&#j8!GvXQ$+rH>|bw1 zq63jndXX!?Bu&}h!(;!?V8$BxwcC+@xP^*;Ti5>W+WzmsNxyRp8hQ@=Se}ajSuj2nI@^^@Q_Th2ue^uN6iIN|5J`njYVgE~fD=hL!R}OOHpRVkW z(&xGHe;dp!Bfs_)8Js$fX1+x|X zs{X%({7KsW*IfHw()K^7jHU`~;*>|c-k4%+?)T>IB+ z`zI>-na+5TzXSVsSdlA5KI!TKZvEHt7kca;1*Xc#uUUuuIBoy!uKnw@{lk^~pfg0| z@5KI{R%EcqC%wp(U(!a|Kiy-0KQLom`_~}d`;RL5JXeVPPq6>(4b1m0J|7PeG{?$^9e&JtW_+R0==zkg7tB!wfcGq_B zvt~>2w|1U}%6#!q_|;gXW42vr#y8iIr|!Ls@7?j|<6M*b`Yi7272Ma0elu1KI@cQ; z9tA6792?rqYr_xO*pTM*;<}6t=Rpg&pBBXRvD;w>_x;E{Tx-{dbRR1+o3!Yxm2s^{ zUu;a)tV7J^d@XCES5}$f?{DOZ9QluNSiVPMd>k~fmwTziz7)K~ZW2`Lh0G@s`;FOO zc2Kd4Q`cHMmqUrq#+jb@T*9>y6Pt@ki`}R2RmyCH;$yEgMcFG&QQrS6n&}iO+T0lo z{duUAXG^CeY5A6MM7?hr5Aw~(0mf6&xHkWDUA^n0iPc7{wxhj>b+hjEH068HIYs%1 z@44ucG84V)IAXr<>SRCc;j5E(j(w2&@}wuvjptz_vpI4ko=u?-cx8Us*?!)`7n9jB zSIJ~wgx}ctzL6O%(02X;E%(Zd&$B0c_+m0I>SD{DWv<_@jdEWVzmFqFZ2Ss(y;ok> zuD0we;_}7hjqPUp@SFdc#`4@)K4WBlY;4>Im3k42WoCCfkd;2PxhuG8ar~dShrfs^mUSj0VHMZUb zJEq|x$KiGPVsby~XBYB5BjU+(Z5?Q2PDPHC{f$s5|Cr3z^X>Sv*;?CQ+4_n{ zrfcUpM&^~sk#;l&D&-%O`Bs7bhKDa^=a&PN%srkw*Ul6pa|m+8&hwyB{xO+P541n` z@Wo^v7^Hl^-;?LsSxcEnIdwyh*qH;B@{h^neaB-SzL?C=VEZ8Pe8!XK+9~%^k=YtK zVrLL4+`D)y=5cwVSxFKL{s9mV<=btJ0LFXfpf0+CFVJosv;iTRC4FUa@~8m;&TW ztX>da?x)cT^c)VV@!Q8>Ylu^Ah4y2;V2+0(#mBA4lOmsVz?FYM%ijl&j(?i-IG9h# z7rC{wk-J0Wwsu~3?Vqjfx4}vL?-TjoWB>P7WU6gmlMcJ`4`}&Mdh9O+)7rIvCUWO#` z{4{5n$UjZ}InDf^$R}M1zw~3)IKD67n#7)W#k1sHcJ-u2<)Ys^%-{M-nT$@mL?FiR;b4~ngkDlm_{)ldO9qc0=1SkLK zZ0(H0cBz}SH)`9{!AkkGfJR1BFNwFR50mgW4u407B7+Q1%_QLoI*k<%^MBA#l==|0 zA_=6!qyzY?+ha$i^IXGw06De!{^ct!*7YGmTGfY%=sBFK>T`Rrsyhg zbdW9`T~}tCMJDkIOL-kmF4yIjTppd6Sgz*LO}TxDTzM}b@o~%TeYv(hRbMmZ_BXlq zYTY#{x3{<^yj@Mlizr)J! zG^aoMq60mC?+{!o=k!4B_cXV>2Wr22fS3A}2aWKKG0pT%v@8156#LNdP~=j>QZmU{;Z+N8~rA#9INaF+TRq3ehA@x@K(9ZRthxXw| zuu^Zofrc{hA;TB>nsk`-E^O33oTGe5^Z0N8%nBpFrVH|iY99iw4_&kmpDFp=|3!Wl zK4keK`$RrzX&2fDE#K$yVHX(jp?Hk(p))!AWgBI?&&A z@I|(Y4$`HUx;}JPK2#fCDTkLtN3ry0Y0mS)EBDY2=!p|wg3jN;sy?BE_T^c@bS6ff zeUY^ypLF;#YxBQb^R3mDR$nzQu&cX0i9A_HSlwz(`YX)m*Bfi^wfe`~Z!nJg@pRod zJk2~`mY`F{hBh)}ym=oqS=9seY`r(TZ-RZ=!&f&pY+zr*+L*-JI+;NwGt(JqWZs1= zxkugs{h1>+hW)gm-CXnS^}KuBXq6Ngb?ZeLfo;=r1xrdAIV~`_uUJQNLD|2rn`Ci9{u%FaJ{@?1L`8JXuHN9^nm zeZ?ztP*Z!ohc71czGn85^fOyLd9Ix&DG#wT2RUMA2k28?nb)P*uY34nGM`RW&+9Wi zd0J)~|9Q|dgUAs(n?UdP%A7CbjG*)4xl(TFjzu1GD$BfcDyPYF=TuIS)^kB=&T}4p zF>q@@6lR2bGR<; z>;q_j>~75*mPLEZiesM@u35}EWif~K4rx6X#JJX@&yD@#M&IASiN3d>L%sSec}Em< zuH(As`xCUjZ-3;P=qP3WiSN_Z{M>EK4bo1}sN;7VT@rJ-cWr=5naA2qt>{W~MsQv3 z(W{`$&n@%LsXfXycYf|+(z;B7&Nz?0SebN9v)?f9?nj?R);zaWSw2Cj-;FZ`ULes>VGG&Qb)x1 zH5@TLH_JPNbmw)YC&Ni4k9VTW##8_0elwB!-= zfwWs@<2?FeZ6n;l_L+H`ac7Zrzu{Ny8oOid+9S)2(^zBsVtB;%`OrA8?Oi+Ce7_#6 zr!gC*cCzPkKO5%BbK@cXr^NnNBXc_RJC0bL3Y}wb^R(xf%w?U`8t8XCd9Iz^jm(Kg z=6L95UYYZAR6YHrkB;$0kuON?au_-H{=&&V^QT#NzTpp7P;u z#)k~@>if`=YpxGL(rz2r$lvOy=9}zG}C$C*d9fW z*e-D{hd*X}Wj9sFR~p;Df!43%N4e(K@gtFb)=hX#it$MIEoYFY_T(Afr7biD!%bz!{T#TVI3I!w9} ze!*r}3vYS?gb<I23O-&3M z$}^FUf#b%Yp^m|9#gpmW4h{7o27P>ynWV#{7r`$vn9g}L&W%AX|DUU~;aL-dNSyZR zMv+Ghu7w6!11tBgNYuytQd1R!B_;;7(L!R7=}ZtjvA6;n$|nZ-zDP0Y2zjg>S{C^0 z@JMxIYl}PP%Ajvv(MUgwWg$!2-6i1V-ZK*V4o9r+&hJm1$#9PM)^R?MysJF@n%f7- zeO2U~PrlTT0_aw+oYp<<{6Jk_nP%<}QQC>{^#m*B+6C&I!Pm=fomj``>XZAb@U;gk z`r1HsT`=!Om-n_aJo?0EiP2%h*Brd~+Z6f$N6g>KK6Y;ppR4chhOYrw(dUEC@$y}E zt{rVyXQ%7mPQ!PK_=$ft&|AEG^ZVHc(6@+pxquWRCdzlwXBvmY8g?UT7s z+A{TLhpc5}E|fOZlGuw)wWoD`_(bur{s$^`IIWGT!zZaj>!?H2tq67rPt8f;2|Dj6 zo=oSj(9l5Y@IYVWP10e~vL0P9uW=r&F*cOs(T?kEka47H$2Ho9KZ!i_{1N&v{8EQ& zzeoQhU5DlVB=LxRFZeXa5j^$v8E9xQb$GBZ@)YSX>9EqD<~+f9^tc-X^bf7mFXf|R za9rzuRLKiEHdOUx8ODaA*w8`8K*l6021j)a7Au}iXFfDElo$;4Mdp$YldgncVsJO- z(Zg;Gy3mg6Y}o9^;IOvgHjzgRZidcu_3xK<+(fnG$K4q0*D;tZcw%rJG&GzT4EIH@ zAsr^2qx7dazvDdmh0?!q2;YPq+0i6c50<+z_(JQyT*(VMmq1ltmTqG3iPQsZz_vTw z7<{5*5LP^y&QNG*BrzE2i-bsrNiTw5VlasF=!eFJlCIQ)IvZxXHhidU=qvKj(*rur z)xSsTfy5xqxzfa-W{=jNBY0xa0U8=j3`YAR?MR172aNtq&ZE2C7@&V>oqicdNg8BQEDRE+<{zKBKoG&TkJjwx_(c%-zo z>Pt7u{7oADN=7S*ONR3C#GH+=s9FKz2n=#^f+b_4Cd>OSfWeNPy^ zJx1R<&=Fp~`Gf3T9(`_KD)&|K_YLsUmu`a&^zz+zp1t40=lXZA;oD^Ny$J2*UbNw4@_!ff~ z|K>xbUc~Md?c(h99zNHI6Gx2i0DInDnk0x)1(>^XMiw=H2Oo>ugAM zZP=u3_)z4b=Y8nI@T+_AOVS5R%!AI)#7XK~Qu3Hjg{ph;&BliH(hutx{N3=>tk*GkR`Fyy zPeMbJh`}UZI>OM7Ql%t$k4IFHrJ=PG6`RZ%i{bEX9VQ+V=^Dr)H_}1f8ymC)4Q+4b7ySXZj)?Nry=< zf?qK0IFBxLW6+Cr0ChIV{Y{ngLTy89kw*-I&|+8rJoLxu7|8ue)%AHg28{$y3>rd1 zcMyX+e35w4VbTGmKh3c?k5;%bK>uO6kL&iH@5Z1)>p#^)CFzHhGMc#hSqu8eETS;%~L7fdP-56lA`0|O!BL*Kp z`?&gv>*0CmN7gU&b1DYJSlat;!Nd19G=v`$@O202FzM>4qMv_Ea~`FB==zTSNp<>v zMxJW#)EklWijo&}UWTeRfDBa^so#f(>KMrVNqB0g>*AlIco_dfLzK-p%Iqo9VbWn` zL(qAG^C<05$Dj}Opw5QBBTv->+Nap?sK_G*HZ0My*SSUuR{#$ck>f4hh$00 z#$dG8f3A`jbb3N12IE{CMoK-Vn$wf+-erG9@8l^7JbHVl+{kf`cGOE(4sbqtOw9`66p z&?;iEiu*t5FzM=2u_5Su!Fe>_jX^)^f$WK{ZAf)vkgsj{P~;JV_o18MSM{)u)C2S* z>u35m)gScHG1w`1V(=z3w1yb0;r>rLOuAI*=l;)mw1*o5^iQhO|8G}b4=rz#lE?iY zdYTwy7#kv8q#o!P{2dYn0?sn7&jdg7&)tbp&=H>N4bMa5l{N(1P-B13`3kTF ztNdWO|A8HEunPs73^rL~@8n$MWrMlYVD1r2Q!q_6W`@F~DNNX4ZWBxpOi*J=jhw$^ zYCmq0{8r?*()lGu#uk%*z2s+*pP}=|a?W~)jaKV8`+0-8N-$YqvNUG2i#ZZ!uQQlp z!E^xAL1Qj-G0Wral?GEJn9g82Ys`5rW@>_M8%&{Kx`64TF@0UkWew~_2Gd_KJ;3zP zm@aZooGL%zdst#oDioh{gyLKE$58CA6pD?dLa`%9D7vcu4n4e7p%ivN4mk%yq;}YbYx>NlD@~Ib-HAyOkMAw+gmyqb-(Pa>-uqUq5ZV;{l3V*NQX%W z;1l@=IgfTw?QGeQ1Nn?`>)M%&b(EYATFz%+r9OTHRecRIBG@7IcnEd$Bl@u*a~L!4 zjXPr>s$GgN&AAqgj0H*?5RV2r9&eCWDtlS5_C~JD8XVPsWZ}=@faeL6Zklty$R);$prIQ2 zxf);O_oTz5#b3e9g?QV{5~BNc^OW*lE*v+RN|j@g^tzfrr0n^$7&Tk z@+?}@RL5$n;>mDsgoaKQMy@*Pi(E%KOuF<6Ys+fd_3H0a?bF!5Mz3#L=KVgX2lC_@ zw?uRzcLMZt4)ct=EPHPgdz^&=)j(!@!EZ z!O+*dd}CAW^E~=8l)f}4!SMA1EB^O_ZuIh%rrP`rN9MH7(07vdDE{Su6@49`PkH(J zHn+1qe6D|A8@_a~;$I8saxdSa7PkK^{>gn+^fdx2`WiwPc=>K_WhZ#_x&FOp`2N#X z`xk-E^75^2ZD)G;%zMaX)RwMIkWQi?Dpp(2Z$7a}*6MoC*7``tI z-^b9)y?lE!?T_?24i#(Hht-DfJ@8USyP(6pd~da}k9+uns%$fz2MphKqwmkq0xw@q zmc7-(=a$hN!}mw|*#qg~&`ksKc@$$WuZLjv|bN#!{@I3-v z{96WX>gB6$Z*TJOx%#d!d<%`fd!arr-(4N;IUYXOzp&ve2QU8J3a#OYwaxjR>}$^A z-vGl`23E@b2I#-Md>uR6*Ln20{&h8czXL1&jfH;V<-07$p6%gt{cCIZMuQc7MbN)_ z`7HJb^zgad&{w^D&AZsSXYtQ(`1*hqechoic=_JyYIpVMbNxF(%%yF1 z0xSNtgRb%No$6)}_3*j=eP#GsffaqJP}|G*d=I;shtKuzQ^S`CR{V>H&hzrU)zkjb z!{_RI$MBu*qV@d@z0J!vtG9hZ&-lh2~>MZ>q>==&Tx z&dXPFuKlS;pX=XR!?zE-`1dY!q?d12KfCHI`W`ZTJB+^9pyzq{8uhogdGxvEH{bAW z0Wbb-g!b_AwJWec^zgayyxs7vH~OA}W_$Uz4zSmH_+0-=4c`jz;@@&;(98GHKzr_4 z{2OohmKuEvpb1_+-fQqrpxC|2_3t9XcQ<(PZx-|9dAbIvz>)4__J zQ=nhh@omh$)naTEbduDy4Cg8+m{ zJkgWkbRjSFn^$*d+H);cEz1Z1O|rdiiqX9WeV&WI8L9o(yL>G<2Hhj?=z~O*%|Ez&j$I>o*pB(SosbrX&A#Qs*@-?OKn2x{YKw z(tnD7OW+s(eh-b0`4_}L_N(EV_%{p6oFLC<1=NF9j$?cg`)80R_RmZ*&rNaIAE)hq z+}K|er=JBTD|yWSK|@jOk23#9I!rp8sONot$9a4C4z*-}()FN`A-b+0Z$+|s-i)%w z>u^J!d(xbHj6Jncc^1rcMvHvb8Wpo1X@tlpU5R~?cY!PafR=xU>kD$uC$FZFkS$Y*WT1lCOT68WUVjjSy>=KE>44qNa^j0GcgxGVHIj#z(b z>2a+&o=Rd6!0+iH_Frij6_&LtY)1$=d8R zXOq&A;UthpyF6pB28(NMyZjZLv=dniUY=B^FJ{Z9>GqS1=TC57acyz?-M5S_KOsYG z`2jlJYYTgFNk6j8*m4L;Kflr2&;NsK?i$C>NsBG=ta-)RY}wVuelk&i!~7`NG`?XL zUnQQp4h>Xw=s(yc?MQV0mAnG>VUuV4DBqVJUZ?Ajj7h}inv+t-tOWuWVt!--^JZH~ zhenE)V}0i&FzgSm_IX@E zTKRH7`?4E-(vOIr50R(ZUN7TM4gSo-AKFTl!PVC2Hn9Y3h&6!4tP`A1I!t;Id^%3D zXK*WLGdwCz@XrP_uD6z3`#o|e={T(eqvG_vj?=B+#INZhpEZjUSj#w7Eh}_9e!S+T+^x2}(!Mxk7ZXj&dApEhU~|(v>}3A7tO* zR?a++4@F?gjr`i9$nT(ixW)D1sPruzCZneGWAYBT-)GJx{>gSr2Z&>y!uZUya zYXiOR^(FRxpg)mVHwP2_!iwm!JD<6w2Fmvu<{sty(;{?>jM^`>?qu-Ne>M==*jsGu zt^EZ2x(2GgbTjtWe4=CWpB$|#0u5zhZ?J|pt^CT;@X%KcO1lg@GFmkd_&n;3tj-uxYm`0%0Y!ya@@ z(mvGE?xg@y_l@JbzrTtO(u?}IK4=|38D8;W8<=rMe$8&=57j;# z0V8$4cDMFni;~a!Uy;u@iWB&D@dc4jItTef&;91ygS1*X%%koc3Xp>%$K1 z!+a$_=v0V&z7=JyZR9SIPde<%FYzfKo`=`4f~rd^T_3hsC!S}9w(;Jy2I!M>~pmjX#@nJle0=F)`iu@hOrw-if`tYjup;*ZeI-^8B-{4N* zTiuZ&pS0}REp28v=ej)n%9nDFFCp^8mreP)Jfg3lYn}GxMuV$;MeFJ>ItJ5^4)#ZS ziw@FZ_6!sq-CZ3Aw2l&wFP+INA7JEfLjFAMi;PuNf4WJ>s-4ImiVs8mk#vzynmxUK zbKm)zwk-F3!N`551yt%qY<#>c9zQdjEuD2cZb;rX&pkVKKVRM<_U+zZ!PdVYN4O?? zgYduM-`wwa85;y6?d}9r>P2jPb8APt+`|{sS=mYL9VOqE)W4s~{Zsrpf-LcCKlJJ| zWS(Qo9`-I@Oy+97A2HvgxOqV{fA@ruxyRV~4s?W9=FK^_?91=+#bj>HReQO}T>L%o z)Q`=}T$AyM_nEXSYn@#=?ik@7qyHuF;*$f-_3D2*PnA^>*QFk&K~-YwnotdeUP1p|j}fYjg=lbd^A*{A0Qn>GzwK zx|ZgQfvW$HnQ?cTD(|2p_f@Gw#pFxd8U=k%@u+pG%d&6nq2@PSUt=+QzNe}~4|(c> z8;8b5=J`fu0d%WZ=A2&k6CS>p%)PzsT+emaM!Bzwjd@01C+O2&d2jWxCwTZ`@_XC$ZQ6E&?__4&z5i4e&hGHeD&>>w>`Rc?lCeY<`T=(=lGZ3>6N*$ zzuoI>GJ6hC^>45z&yD4ujm+c75}98?rOadH+;^Zo(!&??{mVhIZ<7=C;4zlXm-#Us*F- zuP$t~0)G3T`EIQ6EV6Fnn%oaWhPU54re*E*zPpqAs?_lr@IP#=iIS&9CJE?WA6|8P1*BA%BKd| zwa@ACK~L~fm%2bdd`#c(YuAJAAoqLz-3eZ7iL4iVn$uC?Go37GXgK%%;r>V_=`iWQ z3iVu|e5mcD`NECy;X!}-p^Q5F+n&Y#G-ZF9lZdS78jt;lWn6&$tYr}U2V;+vP3;aZ$=vVg76ReK^4r71KJRSe#3ZLmb z2n`iuf3ZJuAL%gZ@Txldr*ci~UnKTNeLPpPPKG^^;;HKooW=gd%6{g6;ET>S_8*}B z?~pc2{mIhy%a}&Rf3~*&X7CdKGU$h{{da&}7pLO?up9q7bo{SV_)KRaG;}5QU+Irr zMLJA6u)5CvL0l92!(#s)>|a%9zl;y-*Pn4F{v+UvPWQxrlCeKS+h5_vf4a6m1YY7l z5W0f+srZ+H9V+%SZ{fzjOvk^k!e=@?prHxaKfxdAN;*us+GBqsu8IAnV*eWKA6RF< zJU7(0Uyl!iP8xjCNgn&-jQxGI{W7*w@t>saZw6lCpA4T?ln)cZANJda@VaR?gTss=ySp)*pl$db z>;d!~gI4{)=)V&E>(GyFscsCf)cOw#o<8KC(9k4eILRORf^?X4V2%5nuG>wr;T0c0 z0CO52S3Rk9u>TIvRocf0{g`T3#oDHKln&$TRo zm~YbUTMTBsVEE5O3ICZ8%t{wi$vOywc|tI$U{W>4b}_RuZPweV`|Trw;lHFM{FhYZ zEOIdywXv@>nEM2i4kle=DqKu;8+(+&%o7a%#wg+67$WC(7c;i4eSyK;DVX+P+H1^B zE@p3Adyv7*5X?DX&e53bT+Cfrb{~T&6-*wOJdK&)Vg|Ld^9*L9V7i0pt}$a=jFoL? z8_X4gVULXx-Vch+BU}t?CR!WJ#e(7gA0_<%Logv1^K^T=slf~r%m6S0G$!B0bnIX^ zFqnaY;a?yn{0l_nbaydNcd&moYax3JW(b%e8q>+e^zUe&FqmAyTnOeujmdN|-*mK( z8%#UF6oDzyn4pVckKjWF(@HR-z>LzEBo}ktIre7;(?l?1z>LutpNlzgj=k4l5(IMv zm@72qR0mV%FY9dYG?-ehm5v89USqy@F>iIYUo)5*!CVdIYK{5I#q`aww;0SZ!CVXG zT8;UKi+MW7e%@dX3g!kdH)za9E~bC3z0P1h70eVcQ#59`i+MWNUTH9U1TziHG>v)F z#pLAKw!yqDm|MWyqA^u2W?7!S$YA~=nA^eJt}z>3OwTTMg~4nV%$;EF)R^@yW@8uo zc7u6NF!zADM`NCFF|E7WHyO-Rg1HyWy&Cg~i&@^)zRqA)2xbA81sZdoi^=I`PcWEe zf>{D)iN?%xF`K*DV+>}YU>*STfX3YEV)}QtM;OdKf_VtcLmD%~#XQgc$_8_rU>*hY zsK%7Km_a@4e1n-Pn3Z5wYRp6zv#p2S-C!mO<_R!QXv`HZW>8PNlfjG^%u`^V(wK`~ zOl40y(_k(Y%(Gyg)tF%}rfV-dXfR>HIA9!&8R%lR^|F%;X0Tvh1oNWC^mZ|!-nP$R z`U&P`FfVINu8XPeZJ#o0a=QuUPhkF}G3{K;vOc!EzPF=b{tV{N8q>tZeA?Ikhr9k)FmHl+Q)3cb%(ipwj|?VJFmHo-TVrb5oA!R7pS{~)EWx}3 z<{gcxaWU-I{HDR2k@;Ip$(2``c9p^PONm0P}&y9CR^#3+#;s^Dn`C4CZ5v z`P9XHSzxa>m@ftMIhfBiW{-vO+$fkPV47&mco!2NZ|54!)q+U{ld3V7x|q^?%teApM-KnzlzaDJ7c(}&ZelP)1=AKxTaD@GVvZ!(2?jGjFzvy#*O+cD z=DG%UZ8JR{>m`_Tz?`Eo9bL?>26m0XGRQ?RJgeJdb>#-SvLZG7byKc|Y>Q&;jHNJ)eA`qsSLJhWz2sE65i*o_wKKlP~mI z@-KwmK)%o^`9j|$KLmZ7e4+1Oj|Amr1Z;{Z?$rt)h@`WBEpZ_jNAGAp5vHyp?caMvz+WW`Xp5d|wR6vG{pj@<+ zi zS`DHpc-`p8@BLXbgG-K`^E}V@dA`4I->-kH*Lv^Ix_{Q^^I4a@_Ff1l|5t>Q-;8kF zJ47~t?r$Q!A4UE(gp&{bJunPDbhMa!=;0{xp=&bv(5C}K;X`NWejw6|QRGARW%424 z2k`ESWSH*9AvqmIKIBY>KNNd%_TKDlxD=Q8jq@GT4sjpwE{q+t4>%VxL~F9y@EhQ0 zA8-oYuVSRTm2v)$)4He+Q9R@){`+tOXtej|{?FW>g>u3r6Lc7RX|8haLtJ{#axeU^ z;o8dSO6Re{)w5B~^=1mfT2PMoo+Sx1_j{H*fvHS7FSbJ{FOZjq|7Ez!OGG?s*8upt z;M&Ryr89x}UWJ#3^Hc~shw{Yz&1lf54EKKlUlPi3whQ;_K^KXCk%&v}3WNU&Tw6It zAKtF_UG3_Cu=efp2F=|r4`64z0@Y`P^4#sJah0b>JZjf9V?yBZLe+I0f{G_f3i`VKl=EdrhP2advT->$=;x!d&>u)DwR z63TP8Yon{YFAjTwn&`F+%pY8i=H)vviMGNJ)`-{d*YS%8rrFOjq zzmHf>2l{_RxM~5N+Eos}eY-Y*=5E)^!0!I?6v}hA>rq#E>k*II^&3$9#LxEqH5xQ3gTJf5 zo*A7ReoH9J-Dg3rvPK|2wapB_8t!^o&VQoU@v^$A{Snr_ZGAy=w=Eu+%GFRNy(dzH zGTr@l8GS_jj73~(-)-=>z_pbb6UhI+=;lioZg=p1c~`n{#Jz%a!SL6K>29ODH1WS$ zUe|C{Mp*m))`J#>y*v6pBm;K#1MZQ!Nj<-EmDh^=sGcqG=ZocK(|>vKzY1Pn7xfhU z_Wke+Xl?y~Jv-`$bfGMFKal+-o}55@>W5?S9}&xXi2mIluI7VIx^ozQ`+oQ{XzqUa z9GLe5%A|hiBa}(+j3$llXKm=+r-(;kW10tDw2xcM(6S?zES>p3p8t($t7lE}%=h*iG z(R(=QtBBsqNnb(qcqe@sFr9CrHro3+-~H3G70-7ywuZForNb8|NVfB1bvDOBz0I+o zDd+VP-H+N7R8&?J0f+) z)>MzfmnU}wtrNoh&~AU51Mgb6&c$N>gfg}jcD-EQl~qz(BmJQ-xMQ3#R*{MNudR$&E1y#iiMuSrx^bj(FvFjnE#1T+p7eEU}Vo zFZ>4+_aJA2M!ecqIzlUEm|umNs$@?VkR8UVg798+cgpQbxz41ew@=&Mqzwnn$c!c1 z9>E;&W@)E7(+<5!+Cl#+?Gk6&-Zx3x>tCha>`dG3CTVG$_@{OL*qOG|P0||vRoZ%I zTC!O;@EN=!`}I%D)%E0jK6{h2jiBMUO^FSBUI9MOLLbGVk7l8d4CrsVD_%}mQJNjh zsz_htfIf;LKz}$c+N{No9;xL^tZ58%x=~*eHW_%fXGvK6Jx+Rwi{5+ONUaQE9q_K` zlzvrMj?^J8UGj~Kzt0)ZSLaOM`$1PcyvK9L>;15chIjaG8c#dLmDb;xmT0phSQX^E zD!V(Yib^nRR!>&R?`+pg6&QmmpdZ{87eJ3lF6;G7Io?B3=pOQ&2&;zP)BVnNU;N_< zdLlzN-Em0PDc&21M{)4}q%s6H!v}Hf-q82?&s?@szxITTODrzCC#$0S&-dy=8fzV` znUp8#UZfPANC0(n--O7%I_d=J)LZ>aZ z%aiZ1LC2M9{p6z`cEdZY{p|4hu@dwgHm(r&t@p-%+F@@Dk||NX-K-^hvOH!mwQNE@ z+}?efrp-Yc@cmV#V_^#Nh3$Y{r2KgJ=V{ub2#;!Q-OZoBOw*=P{A;e~!)e+h6z+ch zCQX}2;kbjI$Nw%(yPv|{&p)JT85DlCb+>bD=*91WyazM}+7w-C#WbIk3ifiXB^oX& z)M_!qp9*IVEx@zMG78^;NZqQFqPpTuz@9vd%z8%1-B@S2rF9_BbIHKAyCo&?F$;Dm zA2#S3_>-T)sz?T~-&2w0WlhTJiLjoy{}W+Zalmm*J5TXey5cQaJ}hg=GPtF~veFTr z&a_h$*8*(8{a+Nf#vRwQA}h|C)Si#9lNm*jj~Q^uxQm-v2c@tPwH2O$2g8%?Tt|RXz!(J@NAM8NSa~JdrH<=B%h#m0@_@_Jm0qy@1;26KzUt47X-LCw*ci zpE$W2WRd#oQ#LLccW>(qX$xw5m4&s-SBsOMukm@lqFL?mNwaGExj9+#X{7vQ;@-O76Lk<0lnMqHjqveP*dIP%&2n#eu# z>Lmpi*f&Y7t(jNRj*TI4N|~M|mhI}W$Ao;{|4Zp9IrrVhBR!4T(Z`9jUAn)^oDnuBUSZ+>b^Q}e<>INKCe-App%O@!!oWT5FNcoyR=x zqF#=pJVN}ihXcF12l<8`@1fB5V=>+=0O|9x^enSW5bJ#BwD&fQnq*3hgpJ&XyXi;4 zjz>zyM%=BKIjV~d{@t#yXO7l^ znR=R=8r6C{>rkcuj}txN}c}ZO0s%XlCHOm^*renp&)#$G2S68`Od0##EL2q%I&kWDj9g#EVw= z?l9t`yFBW5*d5;Q@1Z=J_miyI16h^3k3xh#+KM>s`v~J6mua*;gJhXx58oOk+A&8B zft*Z)Y)_of4|8z_UP+3Cb8Pjv2jV#Ll0DeC$oWzp=82gNdUK*p=ajdTE?Gl8xvVwl z4RNC-!xwRopN82^Su;S^M*kC8yA5%@T(WWkdXfe`@qnI8hn`S9^hvBUwZFdgh0Z@= ze%>GQUv`b<6??IO^N{7jme$Mx2zT{ghRCxd7tdy+4YY>+He*cH`;fhf$+sz?;Fsy8 zM=no(DdO^j2%j7n2^{(C9!gh{@4Yk;=&c)ngzjN;KYB=T|g*w<*4}^Ilju+y3BaICEYOpy{ zb;>2Iwa8xKd}T%u^q?-E&5yzyi`!Y)?8{j178}?g8*JEU*sv9_VWVNgvSG7i*sL5K z#u14P=zz4dF_$VMTjR|#Uxo}d>V4y4CEqV%7s(sO%zA7H*&<)`Kgl-bI|zO#$~_rj zdJ$s~k9YDec|75`85?WT8EjGYpPOg1+BRD>5P6Zkei!|Aw7!SZh1-xImGT?~+W}k6 zZATbv$D~PB8?pu4k^Q!fi(uY=A?9z<&M&TZS{$kda_@ zC*og)KOJ?5LAmrj*YluVLLT*3nDe|H?k{);`BR-}ZA<$YRMs}c!`xY=zGkaOXssAO zPV8q&{V~aQ9OZoj+fMnk@U?%K`bWfn53)hBw+5J}!`d8n3g@hl4r%rv4e3@3~bm^y}h==Es+9ITGyzEK^{{jSj0(a^n7(7}<=#SzfS z;n3GK8+25O3&*~|ehIoNfv!Wi9`QU7-xIp(g}IP~Z^k{<3wjUrxF4&!tkGZeDZau@4t_cK+&HpX>@-njMUJ?PIR=nu&*{zDzG(VOL2 zy;x>2^yd3VuIo)S#^Ha^8!61CH#E+M=Fpf+deeY%uolfJz_?4gQwLok+5T%7_;R&% zAjz>O>X`-lJk+}t^bnLU#ytdDw%%A0`7ld-%A08mda(J=_G0s?4cYN*KIL&9aolsUAfJRA;Qrh%ua;B5+cPU|U2IToffw#@thbP3@) z#KZp6LJ#c4NH?s}{?$AaYs?J##|9s3s*QZEu=!f+dXmwvQ9rV;tPA$5!1MO=r#~UQ z{n+fDKRsQss^=g*oeXaCZC8G0Rx`D&zVe3#vG2v;4Kpx&u^YsLxm5sjr; zt*z@DKUkMh5?lU+3|TS2w$2{0EQ61cuK8IGcnkjKLGChhAS;i}vTa!Xpeb=LXlBq> zdz!SpuKC&OXw&??uKC&O+kpdE9?8n;Sm0SKkLG8q^URl@Z!%wgAur|fnx>Rz_hRi` zEj`#$D@|x&y7UDFS6egJa=kaIPlH$PdD&wV$})P#2=$y$RudOf-kkpP{$`(9zclBd z-XzU+#p-?vcT zw^836sP9hH_ZaG1i~2I%gqA4X7+&AqyuLwdx~sl4u0D3}@{A7gLVfRDUgHx__5H3n z{jvX!`l61}PX7L9>-#RRZ?GESs&6pr8-n>{B;=(k7T@h-eiR0Ih{F6R8S|q6#7oBf zC>iskWXz8eF+ZBKgXfe#$nKN9%Pl9S_UCL)nC|w)8qidnbov zb6(;Bj9s-fW||TdGwu7HzeDLw?O|;94Rn^E^aS06{G)_=of-OW?&)Nv!hP};7FFRuBt(OYa}@A)z@@_^Mmt;<8>?;s4!0x->;$l7^&cs zVY-4(htb}Rx$6`QE2^_dE7B_5&mMTj_;^;19*DgIYlhD_ldXh3`dY`GDK_j6l#G`; zl=$>9+a8j-mgGqRB|ZgAnI0iqth$-6nU1s5);bssThT_q>>K1N85;I28|W`8ToyQ z{0@zP1eiXyTA{oJo}oE&ZKpJt9AW7kd=zt z6O>W#<&FtTZ7+Ql?TL)Wb8KvaA_uH4jfG3cbNc86)WxKY!Sk>9S8ylH2fvU^iFYGx z)r0+&$=F{}rB$VGO3#<-VdqZjJF2JPPM4Uqcfp$rXj98_Nt>t-DzWRPy;i$2^|jMz z)23HPIHLbLWpT%kf2r9l&3r9do*JPmNqwycTw@Q+<06@(0rS8*UtKQNt&Wqsn2pQ$ zLkavIoj--@)&`+nyU{LtuT&^)zOmmA7J#7JFCq`0xGk{vDNl*vp*(K4TwuAmwKV zosBj*80s&h{u1RaPl>4CWp*6jZg!-Ao`UCT@TU-cIm^X3<=BLO&qntSR9--N_Vs4% zs7~hfy99n*+WZH_j(x*guk>8X*MNMx^L(d95TA%Ywa5o{K;!kkV}P;<`P8C5$h*=5 z`dtqH9X;cBsqQ8^9ND#6uYas7*`OilO9}jzD@JH`>b8|V!NB{`q$M-*i+m+ z8txSSo%t)`ewEipfh<)Wj5bw~9G%2_frBdu`{vw}P!g68T3Q#=0ySVIy)Vc+I8IjP zAg@obuXY;ycLx`Jv7uG(+js`}5cY_4^ll0+UXr(4vG2yDX|GLzW1;8+l$UaEA7#uZ z8GFV(XV%PCv*wl`>eGFY2|Z-973FkFvDbV8-WPS97O@x4g)35DTheuE#N9<^?Ve(@ zW?5y{#;!4IcjzS5AMXurL%Q3GQ(wdW@{W0%2Vt)wP+jJga`mTh@7z|rH$BmxIety{ zw4EC4VZ*+e!~TZOR{w&OTj$Fhye}UcqMYoXMB;=~+Jb?wButRRZzA<#D z411x~upjKraOEW8mP01^b7oiNXFS(VW|jPzbyMo`jCSxlpm{xZ#w#%nRuzYFEKXvH z#R+^oJtP-|DfCV|D9EIFB$>33tz&Hs(5#z00J{$uMct z?l(E+PBdxLA2B%|pK8)(JZf^}>F%*T8ez$u;xpFv_!5&NpBWmj_FqxDSBE|A{wqrx zCQG>A*B7{?v|$qd7q0IBysETe0?sG)$chLcxqmY+qIQhrsO~W(B8qUTji<1-Z0P?&X~Ref z|Bs#?QG~F`djG~ota;D&UsG!8F(YCtXvzA3#*^4jYU;0)9@DYLGniM^_P3Ts^>{Sm zGla7qGb4V1U#629f5qIsu75%4F{W#5(0i%%{g;&1c7Hs=|E;`;ye0dFogQnbc&A&1 z<*|remM6SI=bRZYSDfqiw&nSVO3O=lesW3mu%~{0a?brf=gm1BkzlEe7+{%l^t{D% zw8>JtymQ49BT^~`b+cI-w)j@`hac#PG>EGp?yPRxEsfiKDxQR&3HrL9pP2Jb#2+jL zN1H8KJbgfge)QeBd0n?yj*s!K7>Dx4M7(2}m!4KJ0C8VLT%G5b+{~_*kh%yD(RS$hs?=zm40cXhE{#ee!gl<*AMsU2!| zSSj8l$#E6`&fiFM$WGSK=@GT#B}e1-FdODA2hTzlOhadIIl5O0ICvKQe^&3Op7|i$ zc1B_c&!pmi{X-w+^2CPDhBpp2CEouEE-N7?@ly&Fh5w)$@q0-5X_Ws4CcAb2% zVM|xr8R%TAqzA3A)Go;f?c>ol=7Hy;Qai)*7^ao!Jn_7=v>0jZBbjzg=Y{8GrB^Wz zKR~op+*`RFbE&r_$FI92$LaSZ$Egn_N6TJGyRcuVLyo-;XW(FXecTQ6y94=-r1wV4td&! z`qxf^JfIDB$j>&kp>{l;(GELgVH?^}3ppxBTN)q(_EfWG>)z3J0y0GM@w%>4ZUg?c zXV(+m&sN)Na+ITO4UH&kyh*cd_P5#J7-h3<4zSrV7v1J==gZD^{)+wY`8fA+LO-fv z`VyA=lYVr?j3pA*6`8gL|E9BLM=rUoa zco*u|zd8xuiaBZr!yf^EDExGibNCk2Wy^!MS!gHzo$bg*`{tl+4cjN$W}$7l)ULND z*=G0bXnU-0r(F9lCM^f;n2k0(_oKh) z?V;4U?hj8!Jo=vgBF4a@PxMg^=uB0gcgH>!?ETu!-U`lN9h$i1cIAterYb&<)5R*G zcUUtX#Q2uz=c&-1p9K3%V`)9c()vPu?rECWhA>CtF3EQG17^d%j8+FbYCu`;{3*Y+ z-I+2Ed)fIZ*vrPAMhWs*m15RbFGN48_W8WJ>I3%zfI)@9vDoQ(@pCt|N}5_DqJ{#1=@)@c23Ej@LFHl}c-X3!fB z_Lk%l%v+kG^yw{G`Y|oAaG8&3t(7##EO5;+4}fPu!$K`~tdkmY%vyh>!CGfc#gsnE zwtJvgct5P(XV$i3Z~V=N%~}QKzDBfpE!Hpx&>uE9yiePm19vxEN38Fy=<`#9b+(iJ zB%8gLtL=CDD2vhd3ma(ejdc~~*?&Yl%#YT@Von{?`3~UgypK94c;Rt& zcTarNh5q1u$2ux2kq-}lxT8YzEO(r3y5drLXTayDUFj@xusQ{CsXloBykjHQq{Vtu zB40BO^ixRoVx*0w=<_v4yT>V+ke#)2{FKk}US>S-yhr?$WW<@(d2Q(zeJ90E241Oq zsWcyc3H(0~dMI`R;$15BQyzdHy5CRPi?ELR^`%^w7Wye=cz#oQrS!|rrbN6K#NF@y z$|U#;e{sgm4J<8n^ND!I>%OX^(jWCcP!GL5X{zFDT8BTcZ@RRuG#PpFHhk)j?`&3< z`k+73A%iCH&jY$4>t88t8=; z=b1Hc^5>d0fAR~>S^)X^X00>%3(Z;}`Ola&Bm8qmJad~4!e;N+=l--omwWsrtXoQ4 zYv<{72AMg&H)8$K%ee-hg1SY~xzbSy%3ndpzLD}nedHUjFsz$mQ3nIoP42zPSZ31Vj3zC9m`Uqh zd9N+tqJD1QRdl8ddxbO}SgVIcTlWoP9!B!yUvKGg>DF!ii~L0M7q^$vM;6o-yFvh~&pe1BsJ&AO| za0aWH1Yc>b28AriLFn}*@itw>9b>OZpC*~7JFcA|2&)dT7_XwMBibS+|wU~*iLYZEvf>&-!;nHCz-VS?I$&#ECmrxuQ3EFMgqZ$>kBFw;E$e{O* zEMbNFyxJO;=gak%@?uXp@x=)r;hnb+E4~>Q&Vqc z7VP8Gm>U@O?cMV^<{8{BILeAYJk+|rT!KwRxi`%7=q?tVpQY~v7NAe)ofhRc8ul|1 z>9zsmJXB5=y7X#2e>|w~NOMqW1B4C7>U@*oypu z)jJSxq${3{$HO}}SG)qy-*oF&LcC%~kPat5j6JRutSVB2!}t`r6E1)yMjalZL8dC} z`~v^59{FIOZB4fB?y79aFY)>tyvyE=?+}9Vjlj{(LzNKxJMz-U;rtBy-ev5vjCvf? zv8rP_kE+%M1NIQ_DLt(X%CJYVAALZ1&eQc)@|R$3xp{yBc|MTuBUR;>dTG^=3wFYw zNYL$PP(JZLtJIp5-Q5T0IIT(fuX$$u4DLsSM-(DmA7}chPCx%$q+b$Jk-w7a$*QutBW_Q)IJmxrN#Mf{ z+QY1QKE{9Af%|gYo#S|5 zp#!(^FcTX(0#DdjE0wL4LE`4=O!A@KLM zcxrpU@Y41UlB)JLv&6ldrNq7O!0Dt!>f`dCtx0=zEb*oHy)ajEo_%-QE^b5d&i4Su zZ})rPLx*wB0`GgvVS^69cF?y%CdlnIqYmcKTEpkI&n-T}*Jx6#5);jAQJ%Wk9s&AT z8l$>Ld&EY0diLqw(W}pGLEe2nM;@``@GTH@AsKo=>3P0WW^>$HDB;_kf3%0hgCJ_B zH>)&yu)}!wx8o@63!P;fi02F|Rw?mf{};*~i{DY_I8%w3c}O&+4)n^+q=7yT-t8DP-H~DZBw)8;#=tQcY{k->JjL|r+&fkS<7mhgh zsW4B&_hm84BWNeZtFAX-EpekByZfUxmCk(Xq1PVJZ%^pC7xdj5V>ixy>U9dq6v@7Q zGMi6j(V4*lvI5!`i{H5A!IVFH>2FUx_kmu@_J0v*){Tchs#D%A&MS zBd%MoTm6{!(=;k4RHbhLXf__>H*h4l-ax19}4=Lg;V?1H&D#Tkq= z)DDs#s>d|=^>E+|ogds0J&L}s>!RvG>w+?1NZRrUbLG*~mB$3cp)-gX@DGc5_|yND z!_;PfYDc)5hA_%Ah5MnZiTlCoUEDXS!?+)!-pTzybqM#n;2g{-^xa7G;Ry8QaP(;! zY;`KWFVZR90&r$fPkpTeU5{`N#PdXaoEKb({@yR)d@t-0_+FlF#@;*AbeLbJ)J7@4 zg4e`@UMPq9{co^)CVmzb_e;3i7KJeC=W^InlJj6ZUxsf6KPW#s?|I6VPY}XLuG}&~ z@{)vnNDiP!m0}+r=Y1Hg_CYxDU*mn(MZE@I4MALIJwmxGrMYC{62j>FS9f1B^aHhr z_t&4}l$Q~Xv$r+DLL6sb;a*+XQflid#G$_W1wP3j(o=u0iRNcL!_?)VQ(t|@{ZRE= z?&JUe+z(cda6d>r%zd0=<$j?0CHF(oS0lmu5$J>A=!-P;Nhb$@lz05mSB3xl zEH><6hP4x`oC3#X8s|HSZ*s^;s6`&ej@~e4N{(_EmyT{dE*+ zs$U$T=~z()=?S%;`bMO^3mWy!G}oE9KQHk+u9o*A!55SRbZUL_-e=4_y^5OI_ zuD_&zBL^YJ`|`s)Dwdm>bQ1a;5S<17|r?FrOv1;Xo4KhiU5dlG0g zC(ePtq%ACjhYbR4FzQ6#_PS-t^=VGV@2J}X|sBZ>>M(;ezX-_UQUTOAo?n9l&84YJ%@rv=n^?fLtE3aOz zy#9i-?E$X5x`Re?MC+(vVA6fZLNjFh<86%O80VB5G$woyMSQ?gy)Xg8vuv?^<1=f;kQTF@<^G<_N5NtTmq@ ztn6=k&03SHmDT9AU3kXYD}%-(1Na(?IYJcbLv4scn+)tgO+$daIofqD%8q^5TIczv z9nqP+m9f0cK$X_D)UGVe`K=7Kbt4a>e!lU%G`@AgUa>is&P$W5nkMQi|5N7|3k`?K z=0@JZG`@GjuxG-scY<%(xm>zz+mBzhu4nh_2Yd%!Bp$y@iRp#&H28)f9^d3*UD|}b zo|ZJY>GW+M?IEN6?tKz&2Z#8!{fqk!w1t6B8^f4!9JOm2?ruT8S+I%SJ<7RFq=#J{D;&o^l@`3n&~1U}AO7L)%H!pVOD;qY4q1LMr* zHu!7dvfygxLQgtDE})ZSn=YXqq=zR!y8;=J(O(9cDYJc;vi+_XGIF+7Jn?2_K}PRk z?>v$FVo^pXwlw4IS6<2RH!aPebGXu?j^WYI`H$C0(wLMXk)5MGQhcMI(~XbrAdQhE z$#-8jDNYw8mrPmE{aI;|AL4c%A1zD12pc0sz<;*;v#SPnNMAMl?(|hb9ny0K`lja$ zACsOF9H#pe1v{zk^gjvf~OWy~+2XiBHfMN>Nk z7ESA9te)O?tykZ z|B1dw{O9#u`?qLI0$c{%Jh-(#_L`FbmjO2q&Nw`|qVqjV-eg6OcZxN$rUGggWMMJd zwGQoy39!~=M^-n>vU9E=Z*qRe3;o8Np{x2N;$6jl_ezS3u}_wQvA(Xxr+(HX*#LP! zx^s{-vT?Hz*NS)&E6Awt2)V@EU^-|EU9=xKEl`~d+S8zsoKT!ec)mD^X?dWL?vgBZ zf*g^)lMML+AH7{)Sq}GfC#G4kPI-C+{E_hg4L)qZ)7X=;&XlyLd+IdYYoa~jRv7&L0sC?@&06?sz~RdqAV- zi>Y{r1G%JiFO3yx2;1ZeyE=*JT{(R^=o?&gnqyypOi}zI(AT-?kRd(3&v+fQHK2V9 zpUS9*3=#bkJg;=eh5Qih1ZYd$H0T@U>4A5(RyS?3lXe?uG(RKWGMcByL+_?plP(?P z`nZF|g4PSPN!Fyxcwbeo_fqJdDJq}%+uhsAA1c2+f1V;*JO0r8kJlZ1xh(KzVjKNB zU+#0!E{!C9g>ZfmU!3!Dv>!U*=1aOeJ@moNli_X}bivINlba^+WT=aF5j=7C7x9JW z+_lToD(b+G11Q%Ho)8~m2)E;f3SMl(d3O8TOdEx@g?&5zYaim|O)Bzf&zrtPYsZ^d zqD^!1R*&-2P|j^s2kL9rp2&32{Mu+vUJ*}dF7AxyWS8lH$PCB?Ray*O@ofP^=QAF200=c@rLH*&a`2?t)GBa)kZV6@ubp4yExM&8!o=Q z<&G=x#pb5DbSn*fd7WtgLB5dvr90!OULy#nI(f7U_REYt&uF{zUxF`RZCxLAPB%b< zEvL2c_71wWA0v))JnO2yf#*Nrxi+5i3Q@P<`O;vfRS~Tp(ZbZ%K-*2v{ps1Lmg4#R zP^RqwjgNuo+jXG5iD#PEZ#v;QmgHX7QMTyuoE?dCJn#)ZMHyc3XXy+jp74EG5q~f2 z4IBtZ?~2{$3Fuuiy%%=B^QZUwLmTod7QPTY=ebEyb2hvo_uKe}JonW%{OA7Zji|Xl zMs~N%TXxN|7|$Eu@bCB08&Ul}0_~R~hvij-6ZT)`u>9b{pMky*@irpfMucrGx@vjO zg%={uLeLPG=P?mwPDGhSNb@ktorrR$639YwX9us z3i5cmtq-Hohjr*T|K-iR?><}hvt=6kax40-7T?NsUxshx&}UQ8kK`}meR$SF-@o0w z57+*;_F>DB5w_F2hTDFPztxZbd;75{<8$ccf0r)a(!u|oI=Jdq-~Ep5=PkYay^eiF z8Ma$C^LILLS#_)Le#bWRmfrn-ZRUzwHuHCDGjH|X?^W;CX58w#-)r9$WvtNOTATkq z_s3S<>bu|ZJoA>`{eEqxf!(s1zgL@itM7iVdbc*?R^Rbp&(nEc>3z`nIi_Tj_7?E{JUqDV+|nB|-rFOs``pqN*R%WFQZk zdn7AM{OF88u)5jXea-^skzD7ORuhe%vjEL?eyNzpqcbR=;rv1x-WeByM$hNr#4}CP z5%w4lGvdG1A?hW5uIc(Yof)pMdg0tf7HCJ1j?Ns^(>FLcPw@$!C%WQZ#<{6_oJDh= zNy&8Qfiny4Gbv-;G~w(;dRuyvrtxz;$Qx%p=nO_Gg}KgPocql)JVvz&{9l6eURP0H zs^5J`=RBK&^B^vsnTW=D2AYd!LwP*n8E7t^4Wcr68=dFBh*!~Vv=H^8tL{ahhquw4 z=XqWSE!a&H&g=xZX+y8`r-Pd|(@CQTxtG7o&Awrt{l zsOkqF=TUYtCe*GDO13 zaP+^6Zg3HB+jTxAv!%01Ii6>e4gx<0oWA0Co2&P1HHjL~1uH+}Wyf5Y(+hOL_I!29ykNEv#zN%b05y1&{>fv z`nNL9^hnDtT2gR^gwFazx{zx>=}hD{JX6IpYbzZ6GqwSoY0x^x6TOtPoELW!v6{zy#IUq z@Rr{F59?h#bHw5s&Rcep&QSiJf1Wa{A9Jk0{{)lQ>G=JRE4@vLlYwJBSCp>w6|kRx z0|gu+;BWy)3b==WdkHvR!2JZ=U%-O}JXFBR0!|h1NCA%#@K^zl7w`lD6Yx_4o-g1< z0xl5nQUR|J@M-}o0$wNJR|Nd3fHw(vtAK3+t`P7}0q+)Ym4H7I@IC>5D&Wrr{H1^o z3-}uWe<$D{1Y9TJp9I_>;L`$b67U59YXWv$$8EZR{jrt!Un$u{r!JTTT@Y|b0S5>; zNWfhLjQe)mbShfFw+lE{zac@>zogWtP z6ami=@GJr63OG-|PYHOwfENk4K)_1{yh6aM1*`~ooq%5v@T&seB;c(Awh6dGz&i!J zTfkKU{z$<41pKLhKNs+q0zNF@Zv_0EfPWBhoq&H5aD#wP3%E(Z7X+*c*l``V=|bSG zaiLBAAG~fWN*)&Q6ami=@GJr63OG-|PYHOwfENk4K)_1{yh6aM1*`~ooq%5v@T&se zB;c(Awh6dGz&i!JTfkKU{z$<41pKLhKNs+q0zNF@Zv_0EfPWBhoq&H5aD#wP3%E(Z z7X+*c*l``V=>oppy)`bl=7bMkw-qH13wVlvX9#$ffO7?$C*Y?9JYT?z1Y98Cr2<|d z;MD?F1iVhbuL$^60dErURsq`tTp{3{0^Tj)Dgl2a;C%xARKTAL_)7sF7VtL${!YL@ z2)ItbKMA-&z^4V=B;X4I)&%Uhj@xu0{MNW|Yh1wE@YcAXox*qTx5kBA<3h=;aX~xP z$-4#4tZ<6;2JCcUs^BV-y*l(FSzGO*Y_4~?IXE$LEy>% z?fXdmZjB4K#s!)q31i~_tNTcJ2YqW?xHT@^+DF0}41C*hYahw23xaKYSin;RJVU^< z1e`12JOMu?;Q0bxB;W!8FBR|#0k0OYBH(obenr5q3V4%%w+h%M;9KK@c52M6apC$L zNthcyEZ`{uo+02_0?rk1o`9bc@O%L;5^#ZlmkM}=fL9Ax5%4+zzarpQ1-wbXTLo+r z@U8a?nh)++p6oh{H zHVHUQz@r3ww}9^zaHfDC5b#3+&Jyr+0cQ*NF#$g=;3ozAtbm^ruvNfC0xlNtDgmz% z@LB=CEZ~g-E*J3Y0^TOzHwC;yz?A~tBj67OTrJ>F1bjfie-`i|0slq7e--fG1zaoO z69Tpi_>_Rp3i!N$FAMnUb?koU>?2)2pX%#*-A4Et1l&=;0Rj#Za2El06L7SEZx?W^ zfcpqILBInAoG9QV0h6Yx_4o-g1< z0xl5nQUR|J@M-}o0$wNJR|Nd3fHw(vtAK3+t`P7}0q+)Ym4H7I@IC>5D&Wrr{H1^o z3-}uWe<$D{1Y9TJp9I_>;L`$b67U59YuB;tFegUB_K(6jZ2k{#oYn4+Z%J$Qek%Rz zy+7`SI)FQ1-1j0y(VY#qbx`)FK*+ zdggVa`z>(q#d&;hdl_ZZH{}=MQ~C7$>^V*|s!in2!*{;?IZQoGKEB&+V8(!Sx(LFy6i zhp30ipKH>-CV!eq+s)~LYHOhZ_od(buD2WSPyd47pYFcHb{pMEw$!((mhQ$gUB4@h z?n$G&ZRy@EcU&GvxK9mtpE=Xv9y9~)L5q@-k1}TZhr7UVhgP2Czc?bQLsrC1^AyTO z`nGc0-}U>_j@I=i+&jnXQQNoe?lrnwKxWn|W3n~l0@|@#m!Qb^4cNdsLT)-Ttq45I z)>*5F7O5~T3Ubi?e%~9~DCAA~8j$Zv6vo0|z! z*|o9CQ?9&rz5^O~&D&t^uNWkJhn`^421!9B$8aZau*9?#xYLfpgJ1E&eVD(ooGI|}Pud4B;OwCY&nDBLYa_iKEHXPJc@Jchfjv=4-aA!8;T#VpA|Ayj zIi7>^oNW$LS97{iod^E|^g$Nx2mLLbq;@}sc>FFBm;6=X8M3)UKAoy%Mk@D}CqBzTAkrYx5;zBlIXU9e1duO9oEQX>Hx@mj8_?-`#GKPt;MHg1SXN z)JM@FjQZtfGB@~~WF%QLFzbtk`Ei%Ieob-eX)NkufPNYl4cCAb!y-@q&ye9fKco65 z*a^2CiASG1``@VEiSYZO^LwC2p)bWN@=MZTL#$bgm4X^C_FqvN94l!dxUbn=_guu6 z^(L<466DgYu>D>X!hseWi6B^MLHzxb&#uu*B zGiJN#sLPv>5fl35w2Oc3#~20~Ao*zFWjOVobmA(WNq#QD@6^V#Ky_LhF9X$+2-~F# zhJF~;UtHZcQKL@MTJWYuDC-^Uc7f)hrVz6B_C+qJCnZ=#$OJH|o z)U{4uofH9jehTg)1)l1ncLFW@*XLf0^6;?ObjA|8C+4JHRxt;th?17PnU4GPHnp~9 z=EH_rQP+I0#m!d82*t_1xR|FKEy)K}AD@zy_&%cpHM`*1VtGqn8JIpv%n2?XrD&S=b+8&$D?y1ghg) z{C@&@5dU-G7lB4Lm(tR_4rxo6UM`_~;mK|e;N^v=55bQ?ea?e76!!s6r~6P-(S{VX z#SDA?iPM(1SC!u|0r>T8cn@Ar+ z)Y}kOY+n?w$IaV!b?*BZw{@jVLdruyu5(T{@elrf`f zLLWHC?Lf7{75){%>HbvW!w`h`(1(|JGO|%2=%Y~e1pa;WGy1Cm@#*fZDpxz+<9>)b z27NpPb)q|{$={FqJKG+jzJ)k@@hsjQy$fSU)`(OM`dc{+X$j|#&||*ryu*1@lVo&b z8o&P49r9r)vmE@QKA#HT+2?wG$1}~N#6CZT^+A@Wv(H!Z`Ww|^_(8Dg+paA>0-L#; zcsB=arT&+>?i$tEc&7f3Gz2wHbIHUE&@kU>w4?v`wB#Ql{$=raAqOM%a^ozdrMl7l z>rVKT?`&RXpqkD7Fv!Ah-^osGr}T99_DIC%eK-#*3)h|KS)kF~#noKjnXLi0{yQ~rCOF6i$-HO3Whw7Wde zb|IeK8*66IE+td-VI@}}7akJh@==F%>2b)&C13hqRfswcb#uy6sM-&C(7Lq`d`;J- zBv~KSm;zq4w6qRv!Q8Gc(p1$Q_ob(pI%6IfuKwssds%P7{l?5lZ6|s+o=>Q%hyQjv zS)#rTgpLGCtkQjNw~@!EIgwMIuAG&OWLspcSHG@x{=+T+acB&$zSdf~4e}^oNY$_o z*(vFrV;Wy4>LHtf!{O2jsEHxF-9l7WqqVc@zf|VG=k3rtVcXwS6p{}ik9*3{a?Eo9s z5jL(9Y@8o#oIh+_0BoFxqUk-+XCBy#(BlqD9sXousu5E$sb$SQ-5(CDY#QHL^CDQgXP8SEG`X(ZJ~KVHS)P1 zbLLCI#d76<=C>#J zcFxc9b%w?Qnp@PRY00o9RzAjbQO_ctyX*kI z|2)AbFYwD7e3QXH1N6_w<|x!Dm!W?&#$1B_(OSI)`bTo>tpDu9Y{mv>yxqDUyc!UW zvXALPbGCV~%vT`S3*hbteK1@Q9Blf4LuXv;O^mGvz$dcP4WK#a$i{FRc)(@jTo0xN=D)jv=IFGSL+Q@-Fli^jr&`?6cpi4w zj&F*#!7t7a-Cz$VUB&&JsIwj4-9<8)*Kgk9@C_vAFM}S*0vnfL3_So@683`Eu#KIS zO|HG*nW!hP#*4;nu-jgnhPcNwIC;ZnnLN z@%V~9Ebhu&51R&?-2$1>ay)H)z>CxUy=;x~-nR5bVH-})m2Fy%!FCe%(?s?Y_a`^> z@U_VZvrl8%MvSl4u_LuO=swMLv-|%m`yLMazJc5KVA%I;eK5D<;fAB!*8fd*+WG|4 zpXM`5kas-#tR8z`^#4lte@cTv8^&cc6#Ydu7Hb4NGtem>?N1(u9VQ!l1$E2L@3O(( zsIy&#OkRY2wUVv9!N!KFdFU5P+W}=iOY%xgpr2H2#QYH~vHhM_J6$;BUYjqez(0Axx6QD> zklRBtbfFkJOMIvGBJr^Td~~lJji^f$^zw$jZ1;S#c@EDT?_aXfXERWyNoE6SDy(Rytg>y*~nS>YOK$j>l+v z?T^U+=z=)q+Xd`!YpJJJ`h!#7{*H3&D6zZs|u!Q8zVb2VOf%+(%%y`;3^$OCPv z;rTAGu6W$d%!MZpJ<^M z#$1W-_tE@Slz|g(;%;)torfR;yC7@psZZhCdydo~E4!(U&!CN>Y#44L8*@-Ew`>r0 z_uT>2xpxB_q|I>k(|zcpTIAn+itqQYkv*|)z}g3H2YWchf&Vr;WCpvfgeadwhGO@1 zRP4PZ+lBEuTfLsv5rL|m^C3_ryYO3b=9W)U-s+L=x8%(&XUrvMDTph|*+5tQiAFlg z`^Y6{G!OkPIlCSF6J=?u;YL~d6#V@ELzX1YLz4XwkeA_*n>38$sTjYV`^WBip3G8g z4DYi^e%S3J`TO2wrzDaIZm-Zn=v2;04SF zFJeA;4)ejwSZmP!mWH*4YrWoPD+iOz;vMmYkxXNd%MSwa&VO=h>`K%leNmh;YOAcs zl1XdO1;t_iyffx;O&;NsU~d}d=sq=dQsYNbs^Hei*-{Z3vNt>JkpxsY0%Ddnz51Im(toK z541>TZ0teLgH2kl>v;>!1>^re(%wBj>f-wUfA4NCy9pN|SHdL;sAPkpAQz(`n;;Ot zOGUi56124eiv8fN6%+z#Z2)U|D~duZ0sOStwODHL615VbwVVEEJE%2nY_<%o|Ah=u0I`*%TFUIYf)FWl-#mHWMe|OfZnVxR1tR~*JW6?A-Jt_9P zRNt|i7Iv0h87l;7T8jSasU`+o-i zrGL|;{LgYsX)kBv2%Lj!K$oZqn&)>X*5(GBIaS1)h|Bj>tt``eI&82jJp~_c1^o1S z$LFx)wMc2r*SRHKmXX)-j((H-->%u?v_mOmGV;xTW+mGy+d*f$a zQ?%b(4c)rvSlfyqaqV1Zjs+5(qR)~YlYX~DUu8xPwZJFAZ#@LR^)ayaBu8kM{(Yo> z@LvzXe|?~TjHSW_ix+zcUaUdk;Euuti!XZ!zU*yas|%zs__JTbpS^CwvLm|{2A}pY zd|Ey55$1x@3l`7z5Ioz z=gbAg8=~*i6U`&6!9~6^+~L!lpw_0q^pX=n)}~VAeg(9%d0lZh$lkjEnb+`hoZ2yC zoLb^NSuobu+KN2**YsOcR>+JRkPt2ykQg3$BmXO%p#?v-@iwIVpyWj1*rem|4)sC$ zw!rQ7Liq)u2ddyB3rJgVnp0FT&M7*^Sg6hqt9|gyE8&^%2R;N|310<^e|`Y|`Cef4 z#V-{GAN>$~^bzLOJ&Hdt8eg#Z>IdMfD}b9OhfLK4(LOo|9#oV7LkIo?ta?P!EgtE0EQ`y6n9m!Wzn|hxkI91G+aJ}{>zF2q zG?oWCQ#xfK6RH~fsAuumN8yoXqZ_dy|JOa-=TDNZb-yS5YkLmOU)H&lYeGfrcT-Y! zT=MKRXv`t@e(+#L!#w_!isPIWdec(AvcK><8$0g{C6{8~C|PFv`h01GeMOQp?8oF0 z=M8FN=X>78CFTZ?^A!8T`sIs;x$r|~B=}WzY^yF_{G zqwJ>+QSYYH*;9GpPvAeCF3!|=d+Xr6hfwDR_FQf3xgMrIiZ5IeEdKl`?L9(!>)=E8 zum}1T;hfJgZ=$qd8~ds4C)rOy&+3bOh3=SDiIPct>c1w_T}>U*J*s9h zDp~otvF{mtOFs5}{Czm~h++0z_4&7Dv36!g5(pEoai#UMB2)3%8>GkccgUYK#-^#s zbX%J}%d6S9N&fjtlC!*qJ)5mBd$tzJ)L5vlCFCPHXY~8)x@W>eAm05m zc=xCdi3dI&olW6D5cwBl)HKl2iVbnO$Aw|&Ob=&R7WK^Ke66&HGj%0(E8RDiX%LH314(a^;Nu~q|q6NEyt$Giadi~;LBL*N1vm-JMRp!e}!(; zJO9dZ=kkyKI?x0SO$DFqzISap`iH8+hj)i-G!_D5EdPT`*@d?w*Dqv6YA;Y|v^nedyV;rGSD_bU7r!f%O& zR}rp!3+OxL)5LnHbT#scSB`J%a*($i-cxlT+d^4Apl~iQ9)k6oljr^02?ym{gI-$6~08Zl^h;l7mkY1zh!)@8_ zLiAgcJIv|!e+Aj?I*)xxFI;J2y$_Q|5WEVK#~99C7y7lY$zrYj_A{Q5q`zukVUaEO zu3l~`c~+cgdn!bJ&I@O|$*N-odaKaj+9t*=!MQ9jHS_XZPu`q!hCGnN8nhw4c72C_ zt~&2RQ*j z?C%5Y^Eum_mx>;k*R}2b*VvTlR-)+4s+B2Je?36Q*6mCm|nhEJ+|vRwU;naYBL6Tqk-)KIc^S;nVv&Q)ITP6nC#VgvxZmv%_XhqwmE`o;_C@lDv<_tsKS`MZ z%GB8S56moz_pd`4^zDedzR zw1M7BQ(|eSk+$Mc+thN#WhHH|8X6sE?Mvh1yyzhR4@UbkKz{K!@5SP%?}QIdQD|;H zZ+rgHOcg$YSzrNWO43bL4uzvu$TnkEVlO&krBX z=ey>Qzu8%nx$$7ryKf%!@BZ+hkMPHm$m6jaa`T+AxlY0N`&B17Cu&lpFXgy3DeQ;8 z`{>TaC)OX_{{CABx9>TAFmQKZRWPvqfs}1kJ7+#zeQ?RWFCAPGXgD}Va+-g)O|^79 zwy*JcZhTa2>VINv`<iau z^K3}@pEAUqPkCkH(ebO};jA4$<-3mXLHJrXv}R-ti=L&6KHKy^8sgruJIYHwPCU`v zGQYF&VbO;}GmHEIXQbuP$7Z_Xg>?Q}rt{t8iqV!9p9U@VRg6v)KJ`KdL|c`vG#R<& zipjO*ZEeGULcTg@$%7wSz<=>c%Lvo{uh2eIT7j|A?@o1K_WcL+hn?q&4AlbRcw|y{}R-`OL^J@|W*{Zc6X<$M!oTay$5_ z@E=gF!f&*GdSoKL>R)gyd4%R-d=6;+xBXQ=#9auFp?z6jnf^H?>AZj4#dzMzdRE4IHIFEYBPaNy!+$Hdlo zKeB@&WCUa2*FCAMfm?e&t8qQ}0b?S5xVn$~;_FfTK15hU4`-uz?3GDrsfzbtO#b;1 z|HNIO8Jqs1Sg)ibbv>?)7cT>)-v2GJ^LHO}DFqPH>RVnKJE6PmndD#}#*b!qe{dh6@s%TIOH9 z=AY)-z2vER_G|n+##ypB;mb9AJA9w)E+@EdX?RB&m+0xGq@hoi=^UMTbfBzf>9Gph zr*bZ5-iaPwL>$pbg@u5l=c!R0+QJX)xz;lOSM<-h3(7c(FD^xI+UCEIeJXbum51ol zg|WPclb`6b!bir!&Q_f`%dbEWLVJY)d@tu)haaKdDvg8&Ynmn zac_ zqc4YXPQV&u&%zqhoMCqSp&g5XTWM2F|9qKWG^Fn!OG7S)hMe;)G$fz1v~sU!XcKaC z^+m(`mX2%?9r^t~qjaQEVW&vf-HC_>T$h|(sJM$bx7Jv`#sBD9#=T(ho3YvORa$a> zjFuovf|f*xv-ePRy{aTXt!+;cX6XoP+A*uwho1@^VO`tji^lC^`!O@}4CUR?sqa^J zpd-4c80~xH53w}ABaPPcI2|b>JWfZd`6pa|)cP5bUt2#tvJ(H4C>_`krvu_Gz`qQ5oLzWit$Kyf=s^WBDj_5#IB)~uBd%ftu3#5lP?nno0 zcrD@3f%>?t8alAS=|BgH_}}OMk`Cy7+N;lu(t@jEwBYh@paqM`SG3>}{4Z%idgMyJ zzljzc|1r{}7(K9f3BFEDY{%D~3BZlW9@=ph_}lK*@TfWRE05JZFVXob=h3=TBR%1f z)I;+rn>i@_Q$J{44#S^JnDA8ptLbCmrQm`5doFet?o{A?>{s7Gw_-H?N1JwP?asj; zkgT#KL-R%Y5>LF+7x*eC7hn11;18i3D_7zFJ(Lrr3)~yEX)+_J_;0h%{gd{(nh%~# z$^Gs9$HVBW>prCOb7YwCs}GTv;t0ob$N9|h_v~7;Y`9yLzx3lQs%uUDYGl_@@P}M z%osnp%wg|o`NpkZJvY-GOW~Btj>&6BdI zI76rqufh5lEw=<-6nQ#)nacA^AG)}o3x9mT?|EP)`@s5OY#&J8lI3s6N0y&RTfhhE z{nhlHd3n`SI&3=*x2?F#&Ui zUM>EfGnAS~6U`>{&J?GJaQWwvU!9!;U)o&IoOiezkV(CFKau7(LeGB>uN$40=o9dr z!Fm>_2Ls3pSoCOvWd8*n?g%I>c1M7||N0$)=NL;zv>2VVRFCcnq(!>hx`}s)_EUV% zAe|zu2PX^HnE5;16X;x@8e~cDG9NnkH+!XJ()+7HUK6GdwGTQq5IVLQo}=S<#p}_z zznEWEk52l#+rXKVr4jCbkMR`GRT0yJoJHRYmOkmueV*kT?f9if1jonk@4!(V#PrB( z^owj=Sp6@`O5dDp{=)y0=zndHzM=Xg_bu;9ZDMrpgWNvIK51{7?c*=cdE;%iIJ~?c z`z|LE1Q*nv!@l%F`1obJ>AP*03|pE`oisjcsS{_D8*LhYE!!13BBe zWh`}wn@jzd6L;IAobRn22Y=hHFB!*~4Y~jJ8XKSUuCwB}LY@`WLHYiS^un3WI3b;_ zZ`0MXG(C2fQ&XAKs`dxrtJVIR%4=Kg9PS{K55e0Zw@cT)Q|Acqt?L_n&U*HJg`8n7 zJ4zokU6QvnQswQ;MJdG zJLDvNRLr^=9gCCeOU3o1I?#j-xytWETU3tNY5tz942L$}O&xW9-8{fMyYoF_i;u{- zS<8#I-{R~jWWN{`t}&;P*B{Hn71I{sujb;KA6&5iOQQet zI{j~8u5?`2n+KH4j-RjEbT>rP^(0;07i~`|?<#!zZ&CDLD*qTW`>zJy{#zUUm%u;T zYX5CJ9{aZ?`tJnemq(cLY(my<)9sD^JHo#ZVfyEnPa7TXqJ2tu#WDN6%(-H`sYq}( zwkR&X<}`l2^ZlXjT>7>SoU`+TH7tYiFXf;A2gB_8md5%8^?yan~{94Cs zo8s3UFP$->YyRau>yG`ZaUz$HP#jPU*Ed={(h}L z<$HC{T;SZFuHKpZ?DD-H7a92h>B8J5os7&v=@1mHezME62M(e4(Jf%lR{qHvwS4bj z_YUHS-Z}J}tq(lY?SwtW9;S&oB>v|x{aVd_W)u6F?^0eh?JZLn{!P#@`I|UPy@G!= zcU)+~^Dpd3`<7|F+`~FuMLf|sol|Iy*(N=6=%9E!hq?SK?p5XC{}MlhzZAa=e=&Xq z{x9&W@qdnAhhK@`fX|)Q73I{~zi@KxGqmgIVDJdOr-AZpc>bBLGox;FAb4FkwJ~J6 zQt3LbXU#C7>(ac!_^PwB_VKei= zVLse9`ugy)#A0)ApxAtLRzi6HSYP-*1G)2hzDGJC%krPibEO;pvClE{DO-2&^Kid< zv~9)f^zT!92iduk44xoIUnX4_$-y}vF)tB+%mBxnw=UbtT;cE6j~VDQd8`r0GtGc=|LE6^K^F0_{+}N& zH+lV-)%@dpXbk6~uKtaQrQJo^H%R*@$(S-C+xd4VeO|PcF<+PD=AKx+tzc`+kJ+hN zh@PA6w;Kj>hbGCbCCyyYT)S(Kd$rFgD!mnHv?sc6)SglKK1v+L{UiPrpNDEQz#+~r zmSvAU$8~-YYW1-8KE^-Mh=tII=(wC|$7K!SV-}ne9@B=d;EE)TOA-B3q&r};cZ zwtqjY7)^aX&0g2n?Uhn7Wg2BvmvBF&#AC*hPxYb_vuTmnlnCB3)@<71Gh+m!ce)9^ z(@}y;#uGou6ahQnPcHZ~Lhx94-LW2Xj^N4?lUwOEX9GXg-*K}CIBrogc3wEE{i(|T zNuitZ6K6+t66Ga%&FRFe-cLRDd&~gAp8$X2F?oUy0w464zJiYcAMuzR!N-7)VP{D2 z7rXRCgV($x_#wvTA)k3u@Jis7KJ&WZ zM}Z%O{};R#c&*R8Ciu6&zxA2F3yuIsd}h1gCxD;unU@4V3H+qbY!!SHW!;1fRPapT znO^f}!8Zfn>^08`-c9`7KJ$#=TL{0!Yc>kL1NaWF`6Dp6QB*yjc=LT`z5Xo%UgR?o z!S?{)<1@b#{9Ewrw~6L4!Al5V;xlUm-v@l3&payle&G9kW|iQlfuHu7hXv22yt!WU zpy2ty^S$PN!Ha+wdCji`-vfM)*W4?33GfoHSuFTI;QPGh9>Mnm-|sb*f?ouF(Q7IM zZvo!oHFpbs3HT+inJf5Z;FrDTCxTxEe${Jk6TB06r`OC8yc>A8*W4`lZQ!@PW|rW0 zf#3C-a>09n_j=6_1b+bhf!Ew1ct7xdubC$JA>fC+=DUJd0rN-Dl1aTnJp~GeNN9P7fJXq2K>jEA z0^kdL#t6Oy_!6HvB6te$6rVXH_)6d_edbdOLz}MT{-5B%)NL^HU+_@iq0E25BY;OR z{{^21d>->(@CCpZF#iQ_AX2fm*9FZc%F8<_usZvwuF`7d}T@Jyen6C7b~MiR}R1wR4&M51|4@Xf@znfWjH z7T{ZaW~1OcfbZ~`KMI};JlAKQ5d0kQb3XI9;1_{k^qCsLZv(&WGyg02UEp_pX0718 zz$oD~?xnKW|03Y$0UkN@2e9UL= z75oM87e2FCuou4Cn_%t{oCKVdU@8SaNjslJ{~y>n>rAVArnH-$OElGci%p*RaQsbs zJ?1BhpF-M{1aq6 zZV-$P^rq7j%rwD;z=iPtf(HW+PB7O99tu1(!CWbL1n`IibGcx2={BKz_Z`9L(``b} z?qb0uz$FQ0lHl>c;}gsT!IOX|C725ZUjlqdf+-d}1$at=xj^uhz*i=i^95fEd~Jd` zSMc?~*C&|af^Pu6A;An4d=v0Z3Fa)pGl6F&m@@?5419Bf2@1Xi_?857x?uJ$o9;+3 z0|d_no||Cu1hb#nG(W-g6}$*|QG&@4d=KzF2_{SM65u5XCSCA-!1pDX9)j-&zCXcq z6Z{bHLkT8T@Jis73C1t@QQ$`tOrqemz-tqXSMYCvf16<1-UW^TM-t3&!A}4`kzkDA zCxM?#Fh>MG4g7S1IVAWw;O7#|r-EMuev$h>g0}!~Nih2bzXbeJg85MJ%fK%unD+#~ z3jAt#u>;z z_5{x;9`|8#;r}-(-qKh+?Cdy|_uzg$%9+gn$&UHG;$8}TDK;t;m-{Yu4_BGE-WgOe zy81=t-HVB4E&n?1S0z)(fOmeAeN6U^=eWJFv&$LIhHHI8+y}knQF&!+Ze?cca+?mD zF*Ax*If;~$=$K!13=0qzaLkg9VOX@H*4CC%fZV$&)bPUTNEWCN{Qiq?`-B{F;>^dj=r6fX;r9lPt^MIKsV$ zd~JTnCZ9nr+3@6evxKm=CzI?xy5Y*o(G3q(jBdNu)1F@!^0W2KXO6Sic0A5{$HQFq zcyT^l0*-m;Nm}m|Rg4bgC+x`cWMiYV7cK|a8`lTd7uRono`D7oUpU$sf;$rz#GQuA z$Mwba!e!uk;JV@hxJ2B--1T{XW{+>x5 zGfZ)Y7F)hl{+^*8bC$y1E;hRq4}Z_w9#g2W%wqGZ!tnQGddz7GyJ>>uOXcsmDcR&H z>>l8Il1*R1TY$GDn;gO8fyXDCEWtA;nCF!j{+^l1rl-P|053^4-37k{{8F;%B6t$; zq+}BieDegeUg_}n+?;F@6m}o*eaXfn_+{Xilg$_Z0=@+Jl4SF_;9DkGzEu96TawKY zh20N)f3i6w_*LLnlg+1srvOh$HXjSVV}fy29{!#?l1-Ds9s+(S*}N}!C-BZ>vsdty zz*i=lJ%Z;>uzacfJ#&-I+X`C=yfWFmA$T|N?qu^%!Pf#`n{0Lno_$lg(Daizb*KDjoiwMakwxg{=i%n{1vJ{4Vgj$>v$X zHvr#|Y@QN)31fMQ$2=*xKX8AK`Ga8Y>g<`~F~0{EKJW2*YPYRL7H;WK#prE&B^R$4 zy=T0KagWOFqh~)~f2TmYEQ|i-Nk@h~<^E$|KYxWTephD2V)I#_WEt( zC)aWwKazV&BbyRV8M!Sfdt{A2Z)8t)=ueXbV{EYOMg==efj5E z(e%HJroXXM`fEC+Pl~0#GM2u|SeuD$uW(Qih_+2?hSU_&HsukaZ9 zQ^x9SNrwv?dDJm}@9^|hoc*BxQ+ z;;Xz9^Ga#M1lls5HjSfg7jj1d`zvFkx~)yV%tHU&vOzj-m%oR9X&>xYG+En++psZ6 zd81BsWdm_KGUuQ#r)2@Dc;_l(=V~=3Yvjq2X?n*AqE}P< zex>S~6RRtB^T09baehF5a<&z)xT-a9XIgX<#6Dd_hVu0FD%aBf+#@OM~Eu|88t&ViUGX z{W#Jw;*p$@6YRQ;&D#V1pRKmGYvK0~WMMm5IQhh9(XmOSY>kbV_-%~GUaOCh6Y+6J zQ~kc5F!g2BR$*^ztMK($*z3%Tj`j*)BV2S~uP_=;V%Kn5yIsSa$n~)_FOf!R z|AK!WWlHCEvkl9LY_dLfqOG46dEWZ6CwmcNI+?MZ#28OxtV@}n6PTZ|`$LuAt~)Ip zXpPx6`V1U69@8O>+fnUEmz;W6*4X1n2gJphgqrcJ@%-?m`msKdcysPGCSUaVnO}S3*md_ar zYyXElX4tUwmPOFQug#G@1KrIrx^^yQsJ@?Z&e=d2v-uXCn`Qmpk<0M8+nOGoM;Y|z z*Lj_ey1O;=zod13@VFWJmvZ-Z?7XJ5?1cugpPO369fLT1WzN<+*#+)G=Ip2R*-Qch$&IHWnDK92y4Mz!v7g6JF7M(Jo8^C2VhS@)8#Y{)|gEqyvF{Q=HL8%J9D zR9|w2-hsjXH)TP)=7>J|WBsE(>4&PWtUts2;74_LXX8TL4`SbYoWxn)?#|T1*yollSm|18%|33|?5JH> z;qzbKijL7y@)TZaUJBp;MR><`So1cwba=~Nbd0`+AGZbFE9hh4f#&?4q0XE<`Xtxy z%qd1kP3vj<8Y`Qk;9U~&H}!LFDdt{z&HTj6{rOj<`e*mKyeW-40pOC}toQ+IPnFi( zWON?63)}4LUAUhyoCxmePI@_OWRtH~;XZJv{4W10yMB`20*CbzzpXd?5BO-;(XGPI zY`cC&9*(wg9c|PdDeN?EZ1|Jo-rW?%&)*QP^_em^wheM>kM$N_V5c6uETFHO-tYvO z-{7TVK3J!Gdof0`BO4tj(Qx7Chx8}?B3;&Djc3L1^GE1s5Pw7m`~*+azt*QeDsL7QX%ikYl?Xbw^a+GHDid;>8U(SVWo4?AN5-aaq ze8w>H9%=I7K{c*<@MF1WJ9FT5rgB+z&UAcqtiBv)cqxr@sS|zwOm!L4$GJuE#h2;( zEAeA^(J_9(UBZiEix(Zk{~KR$gn6m?)0tOo=L>i%pr379dZdr_Ga@T{HYcjweb7Ti?XaK6qCHEowp*D%$w2<{tKv z{n$%J7Ja~vEo8~@%6-|^9&*4}_ja&H-P_rxZMS^J@`F)4ZXqwxS>fgTtT$0Uqjx)> zQBRm?Wt7k8W%-QfV`0Og&6dxg9^xsUB3$>aqwPz)+_o+MIy*LBYg;32%lmrUHdCK# zO`Gjo*~8b}#5m8gzP&E+EXaYyvph^#YX_dC86Kzkq$AVGzU(vqzws}^RoP(_ z9eM_Sa}#q(_gzJ+w0DTEDVf+6m+g&jUfZ8O@#VTwgM8u;_%Hk4T3>XpMmA&WRn@pJSZz}xJ`M6!!Nc^1f`<${rgPzM>-N3rDhI!r$|D?GcWM12S0Qt7Dj>Y51E>kn>Sup>q z)FA8GBwwGFeTkxt{=xxw`z_B@&Uc(=irRqa6N!8jowHxi4)s~J_H*z|Z+;LBZzWtj zLK(bN)K*=uNE!I6bxM8sJ>VmIJi(@4jSDu#)}|;wrum%{voFyrk`!B){s7=sD8@!>d?j&nojxtc8E?rP zzSjQiv)cEsm)ZQ#p_UJyaWlT^_D!}q7tm+T>v$7o47$>?V}tF+hOy3xD*C|gEqNcM=~Bluk$2O8lyK8^jzRZu zL+GptCTQn%dW-+;<#tW$#hO;dyFT0>+mRl4i**dVjmb-jsKaq^Q*A6E?ND%iZ>7iL zt1mUWXYNbbI^VVQTUz8eG-fa3aVqslCVcI>Xy3)xnx2IDeHFERIR6EIKGpfJf5xa) z-Xg8RQQJaVe_E;IX9FE;Co;NM{`n{aPpNldw3hC(JY`zsW=m%z>q(1zM4gql^2@gQ zWkv2FAH5@?Hi=Kfp3}~w?B(yqoH%5!Dj6M0=g}T=nf4|W0 zFgKBxlp_nx^>|wrK_8|eo0*Di#`e>in0`2Xoc(S@_Ky9SEb7R2y}x))k)OR~nPfYm zA?{{yJl`4Oenfp35BmlzI^P?DYta8SU+R1}$sP^sT|T&#|Fd%$!;Hut($qnBvS_>H zYm)ut0e{oGf+H0V{!lcSeCqQbj^^{1XxVd)YG2ELUys!*kv%PQ9?-kJ>5+Nh;x^jL zeF^&1ADK>kegA}Zlwun)9#3`npH6YFjKyuDo{Vq3GqIRFGa`eTFQT7U@n7qH9r-D| z0=!n+N(on6B=1u?$qpR&*~8qouywz8q}%d3XCRd65brkX_D$_>^J%?EK@OCPJg5tD zp{~e>x*_xHjy)}(ecR+%Y`xICIkJbRw@u>j@)b}Q@#0S-(|Z^@O4QdJhHjO@dyBU_ z_T%KB`LRD}`R;WI;p5DSl)L+^YI>YEdT#8>ziz?ib;)7l<6WG)d###?Yr_62bZgnw zW21M;x>JVsxm~TF9XSGh693{q`$0>~GGcF0J^7#L+8rRC=x_ z6H7lt*WrDk38GtX*|5yW0s2jGqda<)7vE|BrAPMj&)n(_%SOy$_6pXHjgPmWDs{&z zTE+zGX44Z_wmHD-c6$aL`QnbYIcCPv?2MINk00fW(<1+{Vd8r)Wn3;{d@g33E@HeU zBe$CrX8fXde=5ISZWrI9Y2820I39}esGrApVOvjRB;~C8Js(8I10O27@g}_K^SB3b zcM^6LZUpW*c;{pA2Fp}G_*J#JihdW~h6#)EroAJTv9QM!4_-j>zR&0v$?u?p8@J7W zW)=LCnTkwac`P7)u0P$rJ0u(5qEmAfhHY=j?&A3Fv->yhHHFS;=Se40hU(nKp=O;BEwGL_h z%ZQx?%%B|cLKOkp$$rjzc>bC&N7X<6FZqP_x5wcVHgx0@*1|VxzaZXp$a_)z zzKr~}E-*$hd0y{GEjXk#PdYq$&-{7Ho2PYlc5qc9>ta-{7nK*VB%f0sw*J$zd}^-DOqN0DqYC>Sb>X zj?%2X{$_Ex547vomUiWc593`%@nL-eZ(DgnMNFRi25H2TZz0be zj6;-;WkupV*b9WoCaU=8s9n3LOyReL3-4cyl^NfIUP#>4_FqQinf8C&ZD&lye?4sf zrAIdKuZg|C&dDSntm~%tm==vXX5}Lt>0MSNmG-Fo-;#zk?;Gh|Te}UBtjOS4nulU_ zSdKr;wj(X#+Az({OBlzC8PAIt*U60UBwZ}DYuH#F1PxPUs>WI9s|oVuNY%ZFC2MDON6wr9o4 zDb#ZwyvRIXwzbom>&a|sgs&@?P95)QOGaJJ+|T8G@O;6{nOsj=i|&(`gUh*|>=wzh zbRTymIDQwnQ-%J{Hs;U_>^r0+vUgbcr44WJ<3I!XOkQ|>>jKJ}&Y8eu&otl+Yr}i; z!fCCmy{XoQ_eIdgrFuijlWJ{vf5<$MtiO?Us*dvNb}?qyk8D66bOmL$F~+|>itL;J zT4$s07e{TbyayhMH(dgaaG?7Q$#vHLWDRGN9_(n!2Bm{NNbT1}A2}Z^usSKN3y$jzr!|aE9e+Z}pvS z?rzl=+=9O>V$B$qyE!Tg^e5Q&n})^L1o1%XL&@qIp`-ba4R^z&Euj8mM0?a$uYXn( z@_@Ct0Ox>Br-~oL7RrK1&LGfmfJck1t|=_3~u%c5R=QU?<+e=2dr|+lZYO*>L$< zIsYtji(|4bwcVjl|C=q|<^jlvna7Sh-i&5_;_b+mz{%QBeZF1u^hU@Yc#LN1`L*(& z9pyIg-@d`;>9d6M-^YC?%j>bx*4F-R;$}O2mYjSl=A+va%vp?Wlyi)tGgdSI6iVpie z;J)}MXX)5SH!m6cXuR&4f8fh(y9c@L0{2eh$nK|TUDFEWTM2bLZN1J}3?22)Ea>s@ zO8xT>d3a^O(`SDfHshN(`zhe8NpG$O`EDis!<@5avPLyQWBjzu?+LupJirrf8UPQ; z+wp4SQRo!(c-cJPS#BR_=dXQdqw|y1QpkI?mey>ZT$`I1ngi{bx*z_%4f|HDLl@hB zy%;asFW*UUpWhkXYx?slro88?TB$OwbDe?Wfmb5_X(qkEa0$Qy(!jc7#9-C=uf{W@V4v*xxwGV8q7*E;+U z%n7d=zQYOcr@fP?OJSqO+Wjf#thRjM)q>q1cwRm*J6!$>{mp#fp7K<+J3E|Ynr<8$ z#$J+bgZg_fH0>GkZyrz_KC1qM7u=06e7*5V+tj=cW1Ppk3?0W;uABd6>?X9Srz(KTARV)scUfo#@^bRr(ZKpjjTEXy%SB}9n;&|LtN?7?&+krbt~m| z)Z0?NqUjHuL~pAUT|vKxwQC{vo%@Dr{hm-Q`ek!M*nh@u#N0CaUiFr2TQ(}b>5YYI z=`cB6-F}o!nvF`+L|y4i-V7@?=MXL)6`ZOsX~=ggpea{T_EGOy?p`l4a@uyQ_~9qf z2ZlG(9#r;E?*l(YPd|ef(LQGk`83QNXAWa;y$HA!_*~>AwjIn1zht=Jb!H-O)1l|4 zxLfWiX?=mX(m`&btWEUWSa=TMlrwRxNsrZ2vWE=DRpqooGlcU&(uvxK&nLG%pMCLstPUu;1D*1D^CbD))t*nk_I!54^Rc?1y#LcF zpO;RO&&>9G`nKn@DV~qj3C)bW*eRclC&}mP_I&!Z=d(VZkJSyu22`hf)}ADv@$LEa zZqH{`JRhqgni*NyDW7F0$!A!5J~{3AERN@6bwx8HRh{ygf0BIqx98KVJ)fTtPq=_S zsC`4CU={Bdf#*$wJ-QozKzb~~>Dk27Jey^G>6(0Lm!x%C3Hy>S-_&TUOx*GDIbC@V ztQ+sdb;nNqDg5uj{-r1Tm^Ah?>Fi50;HwhdxIHpI@5MDrH_&Nuk(R{rJnjPVG(Yazb-!>qc|+P8@J=6wtJ(5Q{=&|A~5 zmkpTCHns!&Z^j091l!vYY;W(!_I3)kw`Ch6pER<~U5S4x{txiSqg&*z5DlYqKk1^XX3f#{qoK&>*o~Q&~M>C_gzwh{NBrcB>!JG+?>yTWYLpj zAH5N`_g9|T4Zm<^zwdKSSQ_+cvilhAx(>SY5c%!K?nxc%&5j=+XQBOq3$&I^OwwBc zQM%O{a95B<|L(=FU<^cG^}UsJ8Q6ozM*FbMeA~V&O>ysw#jn7Bkafg{=Ow%A_*VZr zpS3+@>2ERPe5h^s!_+1FEabTCyPt^FMX;^Q;^eQ?xrzZv+~f-9x$Km4cJ{Sx+j zQr!DDMdd_)<9~JkD4&w$t)B9XFRRtoqb|i=$iD{87{ve4ufz0KoNz*PTl`-(xS}_- zlumow3V2C>g0my{ME3*G&Z$|y|Isexmnr(qex$war>3}$V>>C)?jcmBt>=ZQ?p5Sb!CIwymWog9RL@WN zuX?JEjGwKe|HRZ?tfNn5{PT`ollbpvPoeeenMGAARp*KHSGbomtSQOaF{XzzNBe{n z);}*UpZ28UcfobVb;A|jKL7sitd0NKJjDH!H;fN>u4`>r_`TLf)|I^(A=AUtW2xhD zmU=v%rCyJB>5!B+RyzJSR_5adc~h5mak?x$)9JeOGQJ0UyDk0CLa%#DPv^j9>{w}^ z{&!E;rHk;Vq2pXl8&ct2x=^0pjF>#+@l~@HioZ*FW#-RIT2nkJOD6^1SXn|jL1Y4w z1C*pi*K+MUB+I!HdRKQ!)W(?Q9m@K%G3US3r6)U+Ou3LH+bC`Mc^X`MV)(?O5*)8xQ=GtSW&ztgZYqOfsrA#k=v#u%8ij zjIe?>A3EiwyPCj{hU60Lxi22okXteO@L1XdUhmtg^ULhWH1N8Hd#8R+ucbA#Zyt8j z)JEO2S^(d?k2e$*@8E0jQ0zq{!?*J^rjxs=gI!$l*=694;@?TW@*7Dfe1D(*l;1)d zMdlbJTUHj?x(I@zO+b~bO4$#R9i6dGz z0{>e3-kk^efd~5Fg-(3^-QPx?yP!XKC3J5aJh~s+D_J@^`8##iH_h?Sy5{VvoE&iV zTK0gwnM=q8BDua^_D;iLFFKmYTLADz(KpsNTYLR^^sn|b z@=wf*hDm1^xdOb-H{09RnUnOQjw0`ou1Q=TYwH#}bGrRL<;Gm+1^u@ZsNND-EC+#a*33dh1_d-lKkpkjn4VF zZ1x(y{nQT{X5TQ0^H2TZiyZ$A{qlLoBn&SVhL>uFmkPj3>0bdnQxG||=u;TE2K%N1 z=Xf4jX=rC)UU_Sn@U7@m?VnrTI*GFetu-ZlOLi~@+=Z{4dW^HVyXd1{=rc+tjK0zR zy|I6_@19WW0NepN=?+v-#JK1a6++3*5%hv%NNa;0oW6_ejlWG8xaSmy)M zyE^wo+X}5Q9`Z-8aaeo|b#~^(K5!xL3i58?uyXce?El@&E1dAOWap2~zLhHj-}8ji zdN}4H;)TG|I1SZ4(c%&N*e1@ukv}cVuNdhLq8|$w*QxX|a_0>lZSmSTH6z^q#6fRj z%EiRF0{p09Zm;(FXB|UU-^BPva4n#bKR zrIS9vO6oL<@rmVwEG66C_srwoIlPil{J1>18+C}vlTWTE8J~Zm89yW6)4wI(7Wzi~ z*FI#y3HZEaoYm2ObbPugQC<^L*$q zIBwh4^c|9;ROudDP_iZXl*&G>HTvfMzoUNQ?te9R|L^*Toqs*U{43!8KX_5U>B&^y zc4!@$2Oc@wy17ph2OP9KDY68+FVZ`N>i2?hD)}(4=uh#+)U!U9FxIR3-M5_S?zs}X z1MD;XyuEGno7&x7^nBY2tzq`tImMmNx6*U3X-*w;bR+zi=wC5-RuCKCg7vAjvbC=K z{c8qWyP4=t)-EGVX}43xcFyOw=Y(n(CpyCwM*Thw77wxSTMY8wHzoDH^|ws&q)r+% zB=qdP>+hY?<+1QBvpiiU5mvb0=L|3G8!DRgx^vmI9?p;(p@qSG-dMderzdjd;f?Dg z%i%qlXn##9LyxEVZy_@V+L+4RvGt14_Al;@&RKs&Lw{Qwrd@@|)(UuAV-osR1-$99^+_Pk*4Zr<7&Eq+KzE$RC;^hDHTx?c2>FddV8@i!>$El3~ zr_pBB=@nr0NxU5Lll_$TI;4UBuG${Y@51x#SiY$xSG2*-l<@J|x~J zv9p~G&i8w{Yhrfucax`h3@4UvL4vbXI8j(T-oUJE%w-K_tRe5watmscjK!IS=emXY zq3~pnKQfCwAo{Ce`X_yw-aRIME^i(yT>m_VZR=c4?s>aqa3tT!STIG$Vl z;KORZn|S+gTE(OTO=(p*V?w3Yho;SGAYA3-ox$ zAKZBF-h<&qzuGym`40z!BVRk%{L-O=Ge=+tJg@FxsPk004 zKfW%JPZKAeyTg`>!Hi}$O^>o-HWUt=)rDkvFAWOeA+3r z$Fx7~GyUc)r#59rvDpt#nfiLMX<^?st+@0+X=3%1GEeC-Z0pa}xHly` zwFTM_Z}00$KTP=f(Lks6&$VqUv}Rizx~rdC%lD_rq1unQ7Y!YZNWV*Xupr49^8J^K z=k6yj){;2|o+UZ^+0RW|IO#w^kE)#g@GC0U!rV`0eh{b1uGPUI_D!E6zbB<|P>&FP zYT=+AgI#R>ngzT&)Iy%+o`RG?CR7`Q7A#;r(D?TGt`p8I^Mo_L?+xdC#~1GMBY*Ie zyg)GJu9V<^J6mrV^enPr5BnuAdt4uT+yrQ8A~ZD#eucaL?AzKZ13Rj;UbOQnS*^Bj z*~j^rKXIKF$p-3lhEz4m{Utcd8ocpb`lP1vs;|-Tg||oL6yyyJpFnxzDR&&@Ur0U1 zLd#2##T6rq+x8>j>+{9V#;Vhu%ak8Bu!d-_d&P;ijd!yD>V^L!^rcph_ROhaA2XMJ zElp0kEZ37XMECXYKyIRRyFYK+I1@R*68t7#VPsEIY0C`ey=@z7VMn^J_;K2Q6>WKq zc|U`3oVBhv+%hl_ZkoV3He=p2)-hKlW-e8K1Xv3evli$qx45-!s?MadSO@lBj-4$2 z2Rp01JO0XKjY-`jfV)<9v?Z^s8;6K6kEl!cEkpe5NmW+x&#!f?J&01tZXRp`1I` zCWjZy%`_K1jc(`!r}ijvLi%jj?b0{l3`2ddcVBJ0??>)}O|5x1Iki8%A6KK%G4p1U9@|^< z3h}p)&ld7oMLy3H|9Qvwl=dCqcL)>wIq>Ju?U%`?H+h_oyN-NrCVe;Z8LavQs883{ z+|{JnMEp&TIY4}85AYtxd?L6FxXm#w$|sjR#^9!rkNT!3`3zNk$Y%@rtRtVT#NUc7 z9Hrd{yw5TJ66`DHT*+hJQa%I71DngaHCw)Kixj^+vr9RzTbN@g-uaWjO?Efk4e}MnvnBNNSPCnf| zW}Wgmi##sH{g`|fk-k6ql&C)BvxR*AL_WJnyUQ^vl=cwtA;&BioJKxr9^)#XbI9Xz zk10Et;(CgIKBXIB$o6*HxwOEkZC+Oz)|~Y(TntZK5NSFe932a75PS|}#at+0ZS*5Q z53o+|2d7K(!{~1&hFh2)de2XKES8VRe#X{ghES&iicPo#oF? zOLT_7d*>9~Ir3T3)+;UYIG6ip=#U`uL!YE(?oXaA&1xN~IqOAtu$enfGm(FnE_Q1F zG;_wY+ve}$zQ4caL*`LqI&<}|U8`!+ou{_XPg~V=SK6xm=ZnuyY1w}cx+dzepwq}$6)eq9{LGUL%c~iB2c$Mxi>K(54 z*x&2ky6(ALjy_)KSljTCR@bLur)}e!Vb+$7c!6L(`(xUX2Th60rM;fCS;g=Pg`TTh zr+Io?xlp0!yTE-~7Q!pA?we`okmLi~emiYj&dVo~N34fl$NQ%|khL^xBI`Od-B15E z)4$vmFKnh?XPk!43iK47s>h5Ozybf#tRVEHL!U0>t>h;9+CSr>+N$ks!=SGl{Fj~S zW(PvGS;0_knj5O^4KK$XN%y;z306K)QCTsnjk(qaK9zF5+B67UTUarw!I#7vEy+0l z`i9BaU|U5T=9eLD9=0y?1`_rs!kGJcl}kqDox?Ztr44$NSNY3Pd1ER@H!zRem@CS^ zc{{w}fM}Txwy)Zl2MyryKEfNIU6T3aQO+KHLxb9&LH|;keI3#;e;b&8yA_A|W7Aw! zF?y%InFnpmgV%s{|EF|aV!OSqHqjs6MQ-w~$U&EMR)F{0q1|&5*q4aEVLUI>_~y}X zpD%@`)8DzC-m{LM<(LxgN$MWC58t0xJ*Am-S$n%MoR;ry))_p z*Av^1-9mlDqt75tyDV*%6J|eJpC20SUdsP$@@eE=Y}15L?KakiPk8Gnz}ld(bf}Zo z2)*6i$b5XCx$psF5Y=NutaX#Y`~8#Ar`n8vRv@gt=--cdM_nf| z$D7B3M*+^Tz_*Fu=y{7-qnCziTfk-Ufo9<9oa?wlUv`yr-ZI6fE-(8@6jpu9Rshq+ z9Mzrjo0;RHSvSBZ>%1}>*>N^|N5Ot%$2HnR3PyHZDp>fhdQjIa=DN;VE*5?~9Wt#? zBmV=o`H)us#_Qj1{_S>5NdE>-3}=&9JfCFJrU%$3eOa#e{X%b+GS@BcqS*rTV%B%p=j*0*_<* zCx%W4AM?>y(3|CXiOz{fxck`zPO)zpA{v?3qjau+cvVi5=5B}aS6@0$g-ktl{g5%U zebVWFRt9wB^^j3}-zARZ?49%IGn_bwL*_-|yiS})=!c);bmo|c-#LET6~!h#7eB|A zlIC)JtP6ZuO9Q?>ODFl#mVQcIPJ!1=y*X6-Bj|e9wW)3o&JS-M{>N26W`4YOjT7D$ zfPW7-Jw)s37A_oBw{X#@d-;DebLKw&FXsP@u1=2%!YlY6fKT)De+Kg?K>vx?4TPPr z!+(eW7B?H)hP_U`l308E&>r0(f&Z+(^~oW6U#PYWzVpH)<|6#n*)9$r&Pv&;whz4tI8ABObTSm)gO*vma z>r|2aqaia=KI_wP`S0)snSA(OwM*;WS@O4r%$f547BYk6zXBbV|9Hp@l>dj2IZghb zLS}&c2SO%aK4X?Ee^tozl|L_JddpwH-cf!fG)?}E=nl&NA!nxYXQ4}h|DEG)SwH_< z^h{c7j`Xv2k5%U`I)hVNg@+!TWCRU^Lgsw#Cp5s572$J7(XH6xb4Tc2lh?=l=lCUF zm%F^nD;9a&v*mAb+`;&~P3oRu{j|uHlz9a%Iu9#-MJ{G)u0(a}j%XYqnDFGWM98<+HzxpNahWZ=UqFtw}pt z2Bjf)NJs9F!G1rJeSa4F|7_#{y^uTPa91kHeSajn61mJ%@c}`tOVa1aZY>+Zeiprb z-Jg=qzV!7)gZ$`e$QF&C^Y*4(r?!sz?{PTWj@5;IpZQI!KJ4AhO5g=wc*5*=^ghv? z5bv#*bz1oD<+A;TCegND?BF6I6_V8U;B?ga)uu zfr^4p38FO+sin3mQb-{BG#G6;ikA|<4VU&YXCsOll-OE>*cRLVluK{6eQE+$6QUL5 zmczyT-k;g~gotRL=l6U4et+!O>^(DkX3d&4Yu2n;v&O_*Wv$L z@6&moZbj}g|HOr>zQI3<_}M{DD!+xaTk!v_eBb2#oA4;TFXVlp75S$5XF#~>ZvIJS zuZFdJ`F*6_hptQcmh-;cirlRCmAtRCA~%_T28XNg3(Q?ei*ulv@<&K}#EMK;zQ=ig z+=@)oJAS0A*0Sem{z(j1{TKgaTZ^k%@0Z)8*;Ztt@;%G@v&eXQe}VTG$Y=gZ30M6R zImVTrV`I^8BhO4g2PXY?9C9@4j>?gR4`$3IPWtTy=rif}X5=OGt^UfD7g#foWHKN5 z(h3f5w8GWUM(=Zst$Ajjcsh3e&5}Q3@zdvDw#@k5x##PTO`rAkt>4K)KbgWhY)n6E zm-aW)S+8hctuZRE#aE9%Il_h0ozFdtAHki-d@dPR^K<(wt2q%}N}FsO7z;oAH9RBR zl}9+wBzeS-JR-k2Innu_JfE5W^*yxL{Li=M|HG8;$P*(&iKne*tqqxXih9>Q<5c%p z={VM5^UyV#wOS(QW3sEKzp65QzTMI9X1>K&qTgH1x7Vy0y~v&VXA5!HtwN?gRlYYH zn-$rli+9KJTnBF6;_k)YP7bn`^%VSmUNFM_U9?7HykCoL#`(DC9Zl8(;~4^TCJai+#SBg8O6GN4H`+nOA3O&GD;yZ{7CmdvDqHn$@?`4;&tz_kCW)cM;!3 zd}kT!+fOLhDbE+0PyPvBUu6kpYy_q|?;YmAv}BkAlL>)o$*``U@Zb8w-81)7CjWI! zuza1v_*Quudnci_8JvOZ@Z}UB+r=l+M^T!7=U1`;{)%MV)jjv=IkN3)zwXmxpH*u$ zOXCpvc93XK`#}0ezKs{H^|PAHH}Y*izI(~HtIw8i{mf-k8E2K-(?b=k0|K7JI@$8Z zVYhn#yWL_}-j%KH**3f-q~#*-O2(}iZiR1IAsM%#N5(B*VV&mu_}DW1$g!rm+_Z^1 zjhR=Q`u~M{390*V9{NUawf1{g2e=zJpEZ@m{i)(>SJHb01w!djQJ6v>^KI4U&(={th4R~Z13tG``@H%>@@mr>|!0etM?w~yuprs z{Pfb<%!xNKH{Qq`Ig7dS26W?JM>jqb-S~_}%%#SzD%xL*=&y6}pi{Gg&4=NcTcMSg zpxso~(Kk^y_PVPD!^wWZh}NhLUMunk%BW`UW{*FzTkp)@4RKcFE#4cLFB_OI|1`u3 zVH0vF$Q-kdv4m|vVL5cW9lT87-Qqpz&X?YG+k z_5#9e{E!@g4z(tooJ^>5?R*D#7mWr|f4r7G)`j_f8rDwUJ4t7<>dN-^jcB|HkEf3I zyF7}63xV@&nqAS0W+m$ydW2r*leag$&gGjn_0a1azN7Ry1zb&`AIn^RJQ;qR|4H|l z=vwS-JE*7)`Y>oUFS<_i&mStj?tQiNGqh>F>N~w$^a4L02fk#ZE!x>n`2HI@8}-XY z(KC?JZ-3c%%<9#?k!Q7U1mCP7dfJ!Qt9|F9Q`U-(zRPo-*?W}ULhB#wQ%+m?6+_wY zJuTU;Bky&yPa~NB1D{H(% zL$wzK-E(z`b?k$MqwQMEe`mLAMO^T-7rtl3q~WsOJ1;^PeOGDt#W^M6 z=DEdT*#N$m5j=4#&l^1G2~XUg89Y&aAu=6)z~rw{zE5*}%;DhWD0{EZt9|f{Zuru( zyqEKi3>4YK`@4ik@qh9fVFUTKmPgLnjIMo)bXBqkm%hCsFI+F3^v#5|qf3hNvGuDh z#g8bs4YZZ8rr~TVe9X{n!oni$+e$|G19d3Q^Bn6{siA2##sxU z2kYfaLuWA<2cMfmniy*>%r)zxbH~?4gif|%Kb8YLz*T`xsg|}^^kJ_Pew?omxurdZ zFi&f*G_zk<%N*3fBe=C+_g<+7dl~tb%8)+{!*6f9AD}d}_*Mr$#ctY3%9M_%9)5iv55i=V`uStx zg^#PiM+fgZOMgHiZ9YL8-!pOOEeWG|c$asz^A_4!@3WesZODg5s?7f;{T&{?e*#RR zVZAp%v#o=KmrVOD(zhZ{%b!~Xp=`?Ac(?T*_HZgQwqp`D7P8ff>+IlMb!>8UTneu4CJ)%QdF_a*hM@ts{C_X6%l(~*zO`?nvpMov^&V}Lp8Wmnl5ov~(A__~#|pNx zligz-x<5apHUxeCx>a?~xG1GA(YNFg^-B@oqU$l>akPOW$sVK79pe|faia*!-SHjZ zUGo5QSojKeEN2j^yl0rl{x^8}JABEZ`y~7S@UPM3!N+LB6S80Y&g-t9BjQTkxJA9< zu6E-pdc_sFar1k{jdA0?LEQgSxo#`}uIshVdYdH<8=zsS!TLHfF9m8XFB0_K0cPvL!v6=_udh2(vZ$0k2#Sg@&xY*3zy zdA}I>K<}=e?J4E|7I}Zj^DOy?kY|V$d0cre<^59he|n$J`*iew%KshmKEm??`BR8z z4fLS$xca^y==}!bZ?GcwEC1c(eVped^1C|4<;vsg1efZ4Hu-0BcZBkzqmyo}>J{?4 zdPmV)`PJmP+KSw%_iylyKJX6ZM~_$a6P{nn7VwO{`PuZ8FS|ikK3)&+FyHRF-CZY` zd+u{7)1{|fTDPCaw@XjEEY|Jmd`IaiAiM3e*0b(9VYPq7|Bjx{X&+yjZ0zGJWQP~k z`(htIhI(s<+)*@^5WVzeFAco5`Eg|Aif>xibMIf_EyzOm!55j6cUf6Qp%1rvWCPmN zK*;)ZvHuNxen>~0%lq@IG8zZ=H#PyaLoO^*S${g+J=Wmoe_?x@ecD=F3qMYI0NXql zjy&|p@5Wh+_hXAu@y(1GNdpq6w&imU1V1ZB^RVsa(SDAvv3mL)z~+bFTa;n$Vql-D z)0bHAVfPQF4@hGz#eZ)h%MSW_T>Zg5>_>b$eQ$!-+M0dJySN>l@Zc-#dfJjmq##Y*Gfv>hGu1X%5o(rzThsBp=7rp@c=&dvHu>MnecWLrD z?ZT5p2iP3sK_}fitldYZ;ZtZS=Q3)nkmeynH`|Ir-yq*Q5BnXQK|{_NBVQJRL-pz` z@+YLX_mljcXx3|fY)>1>moM64?SOxF^cYE%{=zjGlfA@EX6;+JDkEwaOx;FjL%$D^ zo`e00wbI4S1vccf9vgDb%zf$E!(*IXlYbofj*G%R>fgY=;~cQJv+p|^*x|{GEB=#x zXBXy7V7`*ER<@(Es4>R6tXj6!trvxADFfK28$Phzv023~X|V@C`32am{&paCxTiVi z!QB(eS3nt4c8>~8MUUS(z#2ORo2Dty(t`u6t?zncD|cSHD#IyjROs^LzL6`Ic>?*l zRvkmA%2IwOA@&GD*0QHgFNuE+?%g7<*lWiYsj1OkVpR` z!x!b>HRdz=8P(sPbmhw?ybJoopWz<%IaY7=211Pklj`IrNH8Xlu6x9)@HJPA-Y(kD z1Bn-|CV&IAKN_#|R&k`M{im0H`FT(NH#D9IjdNFIYDZo(dY>;oPb*tvgNG>J_oa8i zqasy2mjg?AqRx?f?A%0ajI;Nra{`tvewznhvc&rt!;8SxXU1?mW4M(2E8Q{d0seN@ z*4g-EDu*sKhHDuIpB=*~jN!+7#sl{_WkmHZiIm^kuXvH4^N#_ZZpKa-ba#|`j-lJZ ze-(2}%bxZfo~~Bv{ggGG;GZhqrgamtvhF|NE}bWsXUCX%XS?H%cpm!(QCx4Pp3%@^ z#WBa$`U(2~_^|v?+v|OxiB$W?q${l!e{0cyHWH`0*jG;Kf_G^>!P&_@Cw-{}UBL1p zK6cZg)1t&tqSr`Dr6-WeeY@q}tU8Bg9p6$9P)-n#y=Hej$g%c=Snlb^X^@o{fM zhIGoJ!7lh~6xZAM7dqk$YVjhSH_gRvJ6Eg$;k5(uVCJcFEd%sr8~u!aql&@21bqG14&#?~dL2xoN*6?>A9h zXb*p{g!aEnJLC`lqtS8b?&I12?$1~lfKGoPy8S`u_y?owPhd_O!kiQn>OBtRN8lK9 z;?9NFHt#$izKRZ5^AWZItNn|vHRq`{zNMGJW^DD8dE`+qC^|Z0BY5|sU z;HeeAE*3skJeL!?c3oN6w#i>*95k)=$)?HZF>8FATWh?Vk#kLZ<23h3 zzv;%)e`DCU*@H4}brJBM0Cxsvd<@7orHK038wsh*Y~WZ4Z>)$vWq}hL(A9zA*EY+i2{yT_ zWwZPt@M1s2T`*SCvxUKk&ICPMh<}6o3UnSEc{Q$c9WY-s{I?koclWz&(R~ZMzS`Zr zbx~|w-LsQ}&F^9hP;#Jq=A!$))ir=~Sr0GyR@eV^x_c{mQhS#dEek$&rYy?XOd0G= zR}O3Mp84>SMO}+1<7)nWg0rn{K8Ft1{?fDNqtVwaqAdsGmt1o?O1o#Tqek_j$G7>X zH$DE2@0Qpz^!OXTMUQ503;I8M9aW1i=Zp4}*0GNyd%u;v_LIt(&;9eVLdEPWY3#wn z%>In`;ozc@{K}#8*`x73Y%TIWT+H{vp^nYpd}H(HUEx_|#tHUg>w(Mi8v8LLgIJVh z`f%PQp=x~WQFe3u_1IK|rvzHryE%Zr!;Vo_*vKdO6GC_Ke-2|;HlSzA<=YwSUvwu% zN8YvY!m0S%nZjOXjN@;Ia+2u_c)lZJp0(EWAMYN!hx8ok)Ll>?jqXDkec`A57JboO zk)PqE>|F?N*=_O7h9~qo#}xG=--5qZRv@7{)DZ+Prful2&Td; zXxGF=(Yp*)#$oIiSZA8EOwn^pS3n2;7hS#xoeHLO_@ed#(VcEFyooc;(2e-s7V*8Y zz~$1}7OmF{`L^&8Y|`O-7x67Ri?B~cdAqbvmELQgs(0B%%*j!I5C8otyjQj>@ZREH zyjNwUvhQ&mTtwbs-{W}qm>-L;)j0cWZ=dyFo3GmNZ4Z1+`%wApK}GYv(JOD8?+kx$ z{h%NFH=fOp-92=MeAvq$gX9C*;r`=y-A|xjwU_kT(|bYXxO;kn_3SzeRA;HX|3}_7 zUvRV96`fCRaN8|8R_pQc_ybTK5qNGfxE9Y%L9VjsH^u*0yumH+QOf&p6!M?jZ!S;R zC7TGroz$zZeoNloef4X>?LX63uk#({3Hq10ch}i`q47)jLbb~mZt2Ar#19N@ugD0& z>lynMLt`E*UUWY^U{auB=#Ko)@(bT8YmvhnZZo_g2HqgKw8G2TIp{I1W>m z&X0_XPVTfd$U2>rhupYw`#1}K+~q4@C#)IAd9jF}F=oNz)#q9RJ*GC(wtR5u_gT#^ z|FgSk_gjuH_Ez8H83D$#?Eb&*j{9E{=JQYI$XCtz#OOM`ENjDcfvok{#W7DhbnT3r z-uRbI%CKJeUr0H<@&8S}8Q(o)l*M;id^pWVf6pA&#JTbKe*=gQK4$a3UEbYXj3&h>xMYi?|Mfc-7wIKd`kPO z(KR;=VgFz6=#gbh)1`NG#0?qjXX~Bu(=gPEe8RhQ3v0X{{3}!sk&mfAy`HJIN2)iF z2VHuH@|P4x8cIBo_w~My_kEtoe%^O_6YG-a#Wb%$-*6Z9I#nyZ=6tp2!4F*-eTK)o z+46WcXFtGr&Gnr|pQ-)WNOtRBqe#y2{IyX{I z6d#gt0z;9%`UIQX&old(2mO&L&9%H6*qHzF&og{um(|6|xFPTrd_g(%f9B3D=@=}_ z*oW=PU4z{v^JRG!dvw^98U7NrE4%G0obmLmDWN|~g3>*uyYpR&J?yvG1@O;n-9_Qo zy0755JS<;>>-q-68&XOFPeW7d;#$TxUQik?p#29ndWsHU!*PIpYCF%ndy~(Ljdh=} zUE|rk=`IiNF=6bN4o=yANodo4efz@f+wR`9QSV;P1bD)&%-Q>aL3??+`+IaL?MtIC zN@s6-dCrpXwwfEeve%;{_4H?kFyf zyo>*;R?ZmSv7!VUj{cq1df#5kxv~D8IePa@GgAvGHW+lk#ycO$fJ0@SC{51fNY%n&oVaq?*8bg0=mQT#t=n^*!EZ$tYi3%8J3cZ${W&%#JEnC7jegN@AL3i-w;|6}|3LeiDfY+2Ro4Y0kJmB& zO?s+*Kyg0`MxOW)`~SRml?rzDpB4WoKFc4qBLCpM8Cye@r}t{kMf~^)EAkh`b4FkB zdavf3#E&_bxQF)$^1R~pUfn=^gBAHb?=xn21MMTe5}rJvZ*y5`-)8Y^tEgpsP;#Hv z<~R2ZJw!ftP0`!e%t8NvN$A}in-9@UXCCMOM{zb~n$D(poi)XWeM8^if7x74U@q4f zf1Gdi#}}<7qU#4_eN&&0HH?8}8*)f2e!D5-R?1LWvS*MlkI&`3Bqo&W=Dmr$58zu> z`E}-FCGUs$r`nC%_U^smmuhb6Y6V9>An!VG_<%xiTtg`QuP|Y%U1h=)dl{kD$DgbB z#y&mw3SLE?+ub_UkL~0)JZksIP;4L=@NX^-OmJlZuQl~P>yonj%3WDN=Q2#*Hu!rV zd{qOxg+D39Yd_&!J`78#&+s?o5b4Ch#bt4RkCoU#+5}Jj!d%uh1}@Kqq1fDDfIWd- zQ{bl)Sc6{^8$3}=x@>08GkE&EUK!ZU+^so8xU+66j&zY%ekuzI1+&WcMkYqsLt&gJ z+AYMJIM1Y>xSvPk`kaY-DH<1hChqxY9JCrO@0n;E`x4Q(r=xMnXW}*x_g&VPwU-t* z`@tPL_k&ws9~(-|aPM{?zT;}&Zk;jS%G?YNI$x*#8<_K=<%TF@19N=z`(b@E&ky^w z+q7*J-?Fn*I|ss(w0|Z2+PX=lk#)3x32}OVZIUn2`U?7B@O|7{5ITAR=Xo#S?C^b7 zk)gpp&{sc8_F-GB1tq6j_ zQ1}*Mwfdx&4V)kQxAuXK?X=|o{~%BM+KG{lS2XrxI~PE&%!LQ@BfdyJVfH&-bHDvo z>QI?4nlQx03PC}08>-r(#J1q^{7XJH78FwjW!Al9-eMNS5KgSP=sTUo(%eRdG zDdk`9o_QV%4$vhgvA4w?3Y>FkDGnzjan`M0THX2RU~9KcGWQ);v8O10a2-6WW?SFL zaMFHOeMKmo6ATyNhiV~qBin9u{7_B6rf?p4mFrWJ9mo4%`r7(cck$lNT3$*@- zt&C$2M7&35yvsP_-RP_hu(w=TwC`!FxosMEfpY)w5#L~Bzyx#m@P;MbPefr5obm~9 zh(10DO|?zKhUjVhAO+nu#(AZY+8C=za#BHVY2*Oww0L|CR~1`_cAyU&$@vWV$do_i zkMM)}FP$ObJWorXs?DztV0|Am;m%y15}x@yEqx{szw()n`M(Wa>_h0sHseFE!SyvX zrPvDfr~HQT)}adaqII56IR9_-;Ro2~D#-P5R?=J0UWyNw80-3c>X&Wlg!8P$-jUqz z036(zGbZ4Vt&3}6?;d`w^MIAe2GTiRh`*DD(|s1R=9#%@m|g$uk~_Ac>nwb$uwnMI zOD^1Y3|f|7=efSYonsa0U#seG#@EBVLi(2X{6cH1zbdmn zz;oa^`K^fW9EOjFic74SNwmAFyU*gC{P$bv{lU@Jmlb+no?K^O)Z3`Dxs;lp7_dZ3+pnfps8!1 zsRt7t%-{mJF(1vv7Ylc*BzXqcU4}nvKX$yaoY&G_8Wk5b8hc)2*IQUp6hjawK3wnV zoo(nXwXVk}K~pSza)H8FbgYD?9V^C$7A%@*bgl~)^@+rx$Ckc1Hq0IG{D*uPs_@7* z4O`wnmO6XcfAmG31!vA4V48gy=YNmfDY=fZfV^-i-^VH)IZysGF6O(tdkEXZ2|R{;^-_?Fg0N03h?OO_CK6LFf$yO3ig(~T!?CULV^UtOj(7SE~MF6S&Z=bIx3?xg?cq=q|o zpr=~l-P}4@L7%T<`L$o-BI?-^i?6tb6^TUuQ!KlH6ksU_)@jUVt-ALHoegpib2EBU{>jaX z4TbpU$e&ryJvt?PVxuSA#(i~1Kd{38-1t?qUvrA}iTXwTC_el?xcT57o(Ei8O#0#v zlwo9k6DHfc2*6ld~77!WAU-!&{mR({~`Wi#DAADM#NhSf>tQLMfapyNmEuB z|1i&2|Dk*4wVt_#uXZp-uPt`oS21?4UE#b3828t1ci!LBJ7Yk6cme$3TGsb^k42Al zEn`UUVZ9G<-gmN|fGsWdm8b_ea`>gHa6nZ=3MDdYpvnZo8%Do;0{_3d3*d|CP53yhJR()Q(QVJGhvpk z^JCF}wSRPSt`&y29?ZSb6Y6}~gDU~?y$pMt``?uc86#H^YVOf~`()}41Y|!h9_;U& z#~do&*uA44HeFx7-~JSJ=-df>Yvy*&>})_^Z{|y63gGIIDePV{Ma};YGKH6RpOGaj z?(*~&tcndjG8K8E0C__GU8;~LN}y%k_0YkXAL-|w2IyY0gzOR|ODsZ`c+Hh11_qv~ zf1C5P`2H^Z7+K=&B^Mc4LhC%0@zCOBk6gsq8AusAcUs9g-_4Al2IhMUIp;ag+-=>I zrSbDzvGcC+^BnZ6c#WUupjEwV{5%I;>RsdKIcQMt8b8lLZ+h4Gc@EmrJLme$_!%5{ zwm#v$JGT|$6S45phS>@CjoKEiOXKI9>2CcRKj%z$Vbb_HXSxfg#?Lv^UD!2#&YAAE zN8{(5>24cq89(Pr|1ZXmXs{7HX}$erwodX}0DW%p|1a!KCs^LfK=sOuIAq=lp1zAO z?F!lXL05;zBCj97FK8RGwd5n&U*$k|{cp>te;8SAQFr&4hl_)e*H|O^k(obYj`$bj zU*iC|`A`dMl;;=+&*3{c+cU`AJF0c?qNNp>3B_0Lhi;0YZ`IX+{8z*G`=skkY7Vxg znroy7(wS79Rm!}_v!-_BMC7KFO3}nN@*a%~K62pG(!H||E!%s*8&`K@+mgL!w>>&; zH8))u*i=8I6u(W#OXKEyLYd&e^g&xmq+yt~cpor>i#@R}K3Ka9Nlvl0?xanM*LaPl zebg@)Q9AoKNdaG6-KgOuk$LA?2_O0f(l3K-jpHevA!aN~FItVR^kVY84eq9~4tX2g zb;CE?>1#8_J^d?HzskAD@(g^>PjcYf-3vaA^FAiF@WC5C17Dxw$UJ=boD)9MJ_BFjR0qC!ijU$C ze%2eluToyZ7r=L2aWDLR^$dLBUhu(tJ_Fxll$ZPk@R9Zz_@1BUz;~z@eBxET;rnmO zOaB7+{?QA*L)RC*^-;IezeObme`Oza&wLi%CiqG}zY3q`%@Qhs@M38g^uoIzK-I#jXR074yr8n8b7}+(Bijt2aCSN*#1_sLr)sp*DYPR zP4(rmE?>c;GE~^%<{Z=tEq(ZrP9Y8XP&!g{|tO_ROT^Puiz;G#<^Di&SAVG1KEF|kL7DD$XsfC1`x{M za&P|p;`fYhdbJ-P?j3`i^=%_}g6e!0cClOT@}`Eq3yx=k;}me53XZXR+mZ&3<<}(x z{)}F9^*Hiof@|SEsTZzqIn+IK0eaV5=DCHpa8LEfQt@AdXWep{9WUqsgvde<|kQs*MB2fr6DZ>oO_pC)hrDkXE*yqD__7B|;>FD)s${Hys`^z$#5el#zOb`F?%c_6eifVTBFZR-#1sBIdbE2;nY%bMyR zqD>Dsr)2(!Ha$1Kx&9z?&KBDA*0T5+?-n_<@~>iz@7T_E;Qt|Qdh4b|jPWz`^2gNG z&AhyiG7mtf`+$pic)ABTx=Z^ZFQsA+n}$7XI{WS!?7wGXkC25uLcdTM`$~S+27c(i z(U;L>#x(oI`RHXiL*67G@AEIT#uW3Oj6OD*uty(@t}Y3EY;9B@%eVBg`sXhNu0ED9 zs*gp-Gt<-uyjlmfY8}LV0LmZL$L?+d&%W7OvwX&PhZV49&Y?ZA=xO8D6^GkL_6y4o zS4STDJ9M?^7FO%t0F6i4IZ9VsMtb zi(l)fK4$&2MLJoVwNz9mTLWK-XI%Y%sgo6r3h(E_HT&-~htdzx^%Zt^=rnuz;vGTz z@DF74s(DuAP0G>umhHnw$hz$9?ZG~N`ZJ^1XHT~lhtY38QW>4(Yb-t@M zuk$@NuC)fAtu=jl`tZbTo*>`uobzeVv?8r})#IfH&GKXw$X32RQ~HsNPU$+@!6j$T z!83c2rp`>e1^-`-o{abn^#2VvR&2}T*=TiL_6%jD0E@o!lF{L^Y}>~kxxZ?M(c!LO z4?LguGVYv>t@739^Bfp;4ZcF-JLRXhHbK4@1~Bh<>?(XQR4`|8Hg@rEQyMZPr}n3x zbcbK%rJL&SSTcXx6i@$=*mcidxFm0zz2CFDbjg+54xk?z?uo0jcX)SCTykZXd;wp; zcSnscbOGOOBPNCWd1C9@cX&hn`0l!0`ln>0Pm+(9Eq}(psLva+NXw@!{@lN!3bYRS zJq;Nt*bKHF>u!p>gt34<{#^R$7I=B=C068l_`u(hIrXn}lBY)a0vmzpzkuTd>h3~^ zr8?Vv;G6o|IIAojmhsOy65G`KvDHPNB^(Ra!n^R_!d$HW{?{n_pf7#}&|P(a_bKSf zZT5<|YbH{~7%X877GQt;ZY+MASi5z6#rOtjvXO76+_>-y^9G^+j;U^5WY;guJAk_v z;3@K-8q50LMlSuryxaj9#+RoRJv((Nb;JY_}L+8l;-I+W1*1qUjBU(Jym}5_WW?rxVrR&+xU03KJSLgmc zF7)j$`FH(3%~vVx;ol5zp97Dd4X?imo_`~Ljb??J1I#`zduxl68H*bCTC<6!+n_JW z2F`i&Wcvc{=W=w1j!v6oE7{~YGUECC%f5Q=y+ihL78X^&|03VPrj@DDFVG=WAV?mea$c-7u>>)@jLgOjX* zv#R{{^~?R-Ihbf~2L|1zCSCIu><&ACRd7{w&)r+WI-<(kcX2LqQ!U?|?Z8ee){eGg z4eiJQ-gEWI(HF4GW^6?JCh=&8o^qb;%rbMArD zIjjue!&U}(yMR$P1zF^gUyd_*oHJ3%^Rb&Jkv#Hi+mna2`+jKD*iX56;>aVNjP#os ztPQ&2tR|ERP11vEzehYwx`-fpZ`J!LaF^rbsCQp9LWlnI`kFJUlJz`sj!jGs|2@Sx zr0=(EUFi!oBFE-))}Rp^-FEDY8?g~Wv^s=YrWy$9A_Z63v)3#$8F@~O=~X%Bc-B<@Ial4x$!1LBLw0c+V|hZF zwHUqs%(7F?7_T7Ar|+XS8!9`>ck35BW8B0c%e#CR+Xr9h)bl-K+~A;#Z+DDuvEa2w zobMh!EZBsLDD36Tfl=7Afc-dqSR2##`dI1@tQcd?wsDURvRdE5Tl0e_&QHT96%YH+ zC(x&!sHUH~Yh?4|59H@KbCzqThh8V)}&+0NB<>dg*NeT4z%d!EPFe4?60wI zT3_SYE&YVfvQGdnnlm}C-q{LFe$K&WTWOuvlO-ptK=8iIn1&25u*51vJb1m2dD5{;mwRk*L;0i?B6o9?<(FQI5zTMHGamA&xr|hhfL(?j6UH;Y+O6I zx8vl~`0K?d@S6izCjtX9bDn+5-Zf3L^h@1xO{xzD&2 zxp5!!O|IY}TtOZB?_Z;@3t)Saa1#0YEl=jukC0!}d;{vTeTkLp^X?6Mk=+mY1{S=A zENT}%xR&$uho(Kyuy*0w{WGh4ag|F(KbSEAc*g#qA!DI8Z7O$RRL(4{&p6;qD=1hN zSKo{M{;r+qdw zAV0&}59{zHPtEqlR)(={WZarPDXpRUbMJWL-OC>tV8zv4%pKuB0{(;E0d)z!`1oAz z1RfS=C46;GVb^tYa=WI@8P;|2Kr7*Eb1v?xOpwh@N?il+6(fIUV~f#(+%LLP9dVRb z5b)Q>`~2}!=8Wvpz5w^1)QKkNqbJt}-F8^G~VUu?mG?py=t-bh9I`fF; z=2Y@Ff@AA>>;*WlbF#(?GbeM$0sFGM6#GeNL+Kia zAN9N^+C$8^OSd;r?#H4Z%@vIC!zM5G4Wk@6Xt2r08jLap`!7wHY^#l$yI(P3qWxRx zZ%e=YF1lmPwMUV6wPz-o;#s$hXWTMv!k3WanS*y(-yRcM#CHz(h|UEq@EWyI^Vp9` z6Q2Wf&b^=+uO{SF`1#8_{W?PLe}tmG78H4)@6pzzz2`~ z*}<`<{t)jG&Ly{V&tKD8eA%e4h5^fAgXdJ+Ls-t4 z=oDc4lXrAzyDPiV|I^>cR%3es_m;Zn|CVpn)gPDy``3Y8^4|}^pLpYH!U$`2$@_<)f91K> z&2tam!h!nxLpNQK?P}7*>lg5y&wUPAj2qF;DB?9{FEU}8J(AGi z|38W&19(>*C1!3IVb*Ia2gyb?-PU@{;Ga3AJJpKp->CCw>GlHpL2Wphs&N7@po|-l zm5pCB+VN}CW_+9h_l})2s4j;KU$;`*!Aztl$!-Q$}>n23sWx{m(mxK}O6D|GRjcYbxvi-6NlkBY~ zOtfD#VH$dZbNeiXUxc1~j=KbEe8^{hWT9AOqBvxucx0sh$Y%qP&wR*dec&50@C_ep zX0P!tD%xu${{K)%D)igv>V(<9m^l;O=9N5&JmfR{!;h>exqN}XkryS)cKMPEYM1|| z9=`s0neW@lq3!r|l}@nCmu_UeRm&Y&FMLi}FD2CGuCvFvFlj6~_)G-{U!_gg(zdCz z5xcJ2CnNWjA@})0v%o=A*1Lp%KO^haAnVQd?ACaatfziytdXpja!y$<)s^+++e0*v z0&f!>|4?7Q%h|v`}pBl6Y(^q%PIE3>e-(A@*I=r^Qiu0U?9S)Nt@8o2g)((8T-zBOj#i*@S` zN8XAx?PQHKgnb5Je2G55sg3-h7C5k{qJK!S{|@Z(A0eOCf5Ogjf-kFcKl0K#R27dVDEyv4{jOZn$ib}+IH9Q2lNQW%foS?^MI z^$U({Bb>A&->|lD_|nb{!Yq~|l!k@VI|KvpFEp@Z*%c;Cx4%jF39{6`J>yW?cenhvQq zL;E4ne}I0`7@IS^dj?HL=a^{^;s0!lwFA6X^$s-O$({SrsV^{j*#j8FePhd{ zqs*}Tn)H;;W6;?Sc!gl_5GOyk^3OJmd4c{}9bMP2(>)>Ij}6Up*XM6Owt^w57`Pb<8CNFAbc`QmL!K<~DcyA7rL8Q(I#b;l3J)lzr+kE~{Y zuzLEDyx4F@KkZM(g%4m`-Trgy)$Z3gU(I>xZNG4M*k345abd8XBoLm>tyNVe`4~a+ZVcd9Q))H`x?@o zFlia~@ZM?Az^$aMHfd@0(B5g_|9;X|nKWdq64_8z?-|cHtRCpUPCmcdm(oVyv^=%r z3{Hw$#;@}|F|G;S$-W)F&_>oGf}zXwo9y1R?7L}i+l*lI2Kv6_jyT$wXit3A@fV@I zKL8I$*!TN{em&+(EfDQ^m!;Gr!++*nl+lTBHY#>P_4I|LP4N_l{DDCIM#`Pv5*M;^ zz1%U{H?$Mq;^o-rkU!Fi%v6EC;ncLD0c4OZIrBKDRb`!CMLRq428R#OrZ{u}2Sz4@ zPvv=4U#3D;H_y#X7SiwC@(oLsd5b7WdG!G9| z^$C8G7$FFTVLI{cIdQyME`K>$_K`E-bFulLaQG`t9w160WPgxM7h<> zwao32-LB3fpE!+&ql|mmpXWj!>TB_G-#~{}`x~0cupeaXoB4<|%@d;0qr_QF9gbK!>8+edHDHHyU0;9rUl#)=J{n7(b>#I4A03UaTWEGa<5l&D=bi#jqD$?8 z{stWA{f90t{`rhEmX-G%-o-<_&B~u>{|(rjF`Q`o203{7^}lT!d4XSe+C*OQ+#*5; zhsb8|K;h`8CXF*Lz0(-4!ii+@vTp9qJYBYT9pkjom)hlw(H}WubU6KwKmX6RHPIeq z#;DqvXg^H*+U{}3Xihx1$g*#SCY*UZ(f$c-Rh;(3kFkzF${3Z59;I{HTt?r8i$_fU zRAVP;;_2^)iPJcj-ts}-?`OX4?GrhwpZHJG%p96~hq+Injd5na6O0eDF`5Uit#`)3 z3Y7_Ms?P86uC%C6-?`5i`JTH^&ktQeT2u#t4trN1_rg$qtfyJywt%}xu+JK+GHy5J zC7SZ)omJjc%ImJNc1Q0LX-5VKb9ed&BdkDQ%K3o(CC*x`QT^mId$HZ6eRg-`4a8U7 zAl4p(S%W0778$~tB$0Ji61F*gLXszrvqyQ1@{S-=pc6MT#Yxs&r;sV!H9?O|@qa4u zZ@}KXe3>snhP;{QDjs)_Y|I~!D_-V#5!mh`Y)lO{=M%R7%@;_eO-`R?*@GC@n(r!2g~? z#`ZoFrr0|PC2!~-(e3G_GwHh-^x<^+@_PF8Ypf@yq07GxA1QrIpI3Rki?N+`Xm-70 zS7huVrT@Z)vf*lL@qs5AGCJY=mFokc4+$R{TYx<`XVDT@Wvuelt>wOmg`6pCHgm-A zK*&m1wf0AA9h!dwm?Tf+#Iu&3F(lCLOD%{UOq)g|20AvC1`do!3dFv1&)&EoJZ@85 z*w6gez9~L@^mcguUU>eW;rXMzq1CTA=bSyfCL>}&M zZTcGzXQnSV^=)TdZ3g#!t*xh!yZzYi?d)TX)qlUhFJo=L7|v|Q6y{-%^sYC(av$pu zr8Oj4-&w(XIGekm(Ge>yKfxNpI2-#?9%c09tS5Fs(9+EO+?=5o5n6*U9-71Z@8L@w z$W-E0;tSbU&LQzC^@YadmGG>xs+4-IdrD0^6YXCTFPoyB%ft2Ae!tOY_$|-dvJfx-9nV zbl;7ixQ`C^ZIRV;RV9O`9F7fL<4b-K03|LcH`C(7nL6}+_=YxQ=3nr zvpP1zihREiJ3gKq9_=H_SI=w6FF*5X9g%Dwqdktl-y~b-FLloI#8mh+xcPy*UaO`| z$pAmgbN-3?2UcF?Tju7wn|zXEe}=s0jIk8vT+U8{xBaC~ncs2qR=RLDjkKDc;3?-h z0xxU59RE?;AHFd@H1fwz+soZ@O?+2;Xt*0cn|Se66aQrYP_`RCllayNZaL!ygwoyk z>l6=fGWq{LAe89Fmk}?V9leV6K)%)VN1mhj8U5aH^rt*IJnTTT{iMJblP(SRJ;LjUMEqGL_T|=EQ$h7^4JGQq^_%god`yBaN>j%vT2dSe4 zS#_Q7(Qz;NlDcFk`jRiDYeT7T_r@LSpH%yH#?FTQ(g~&-JDr+w z&CPii1T@Clp^I?dh`?MQ^CBj-Hx{lq;@oN&}Ss+u*ngP+#RtH&R^i!}>4 z+Fz6|_M#Xykv>9%MJhvg&u*!`jhp&-@;k$m7!Cb;PyROx(Oa z3e)G{Vg23%k3)yo_rQ~CPXG@2e{UZ#Ih=-VgZACGa~|l}{)y~4XW5nu=d2#Qr`l7a zcxMfz`USh-k)2-*v=kstx}M(nJD=}7;-!ZX{+bvo!rhV2&_bdup4%ID7xciIXeR;7 zN_bbv{R5#J#u4-L{%!am0Dmpmw=l16iM_tEXaPP*0@N3K{m7zx_dK)Zkxdn{lh2aA z>|k!!z|gN69oSQ8&pK>Bk_mM_|9HP(jvE(=3tP>S7 z9ymK```kE>3DNPX9>&#ah0x+jh47|`LS&wI2}dF4s7{?%T)=nN++QH0dyv(+qhcw3 z8F%Oo3TJJl{VeT=N9{jbrQ6HDFyyIr)(Adp<@Cv!)*64dwHsU7=cNnvKXb?41~2zb z@P3?qt3NsWl#j)Jn^xG?l2FoEMH@iJ$A zQhz)4j`yJp(>m-e@|<2;3d|+IUd;O9jmBpiwTAutKYi$n-`wu3-&*)Dh!19DLF4bL zyGHU}D)L?$>yvcK$)LPU%H=G`((~Bc>lf0#-tk_tox8@|64hJBRA2{-y{G1nX5_9X zdGIekW)=@J7_wDj9WbvHvr5033 zpUixhv@C;lUYdOud5kc+_f*?TA#$)Me_e0q$wWyof)M`A2ey^%5j5x^vPFI zk6;$F10S4kWb9jvxMJ#%taT>}Hbft(UINsL*}e|5rvPzE&y&&p(#oAD*09g+2WN_PS<1 zUz@RN`SM3=moIy?6B$YI9eGwH|M}XrvY+=Zi>(hbPJ8>1YDHG?ySmaTq(yb5HH_W) z7bb)(&KG*-^$jiCjV|F>_m~&w76fu)tU>E)l26sro<{gf`_o?uhc{k^T~}(?5#*kx zxt9di)uf%;T6Jkaydnu7WMi_drc=ls(z~RA-)>}qGVIS1S#v3_WxGS$sidj@g+nhm z91LI59HxGXt|3o?SJA-|LT4Q)UmK2|`#s|1H!z#FAB(plzcphk-Ts!FcON_^mbHU) zY3<@Gt}d~6+r5+F9is(nmVFIve4Br7x$par7#R4k2AJC4E<^7deAK@zvA&&kMLzkg z0nEWQb>m*CNkmRrGj1PeDK_S5{?D?n7R}NpYO8eJKc~GN@Uhppn`H|7n_bz+Y}2e} zBdg>A7v&bv*9S^1qk9UfZ0=0Y2ls+sW!^-Y3b#KEJj@Nz^0*sjW*1}OFm)Y4*K~qB zr$j&ZWzT3YJX11KM_zo`A4JDo zSiN>tRq#>ytd>k<&i)i~cO~|7+{v_7bkJT2{X+*G8wZ5j@8Nw~u=#l2&_FvnblLdn ztcc(91iEZb^C|r6h)oi#y*{X^b|0hihp0`J11-hJ_o{Pca|q>niPyghx7 zJfV5-HqwsMr|tLvoeeF|_7#TG*#qz|D5_VVs+|4cNqs7rTX3~O9}#Ft^z=`+OwowW zMXQ|Xzv978zAXDz{-47C72rhU%FnoJq|c)?puRuGTC3If;J6L+|Ni+-8(ua1CCk2% zHs1V6caz43KX-J<4}a2G7}=>7b5^n^zU>;bSKNBAd*=PjS7uKBc0%X~c?}))W3GEC zy3WtCCs3}|`6rNRm3|f9$L=Z)p9DX@X5KsXxHHDCCO+pc-KrU|@yy%aknUe{Xue&3i7dgH$wx$oV+!Nrwon{d9OAQZcs zGh^rxzn5RsOmhYvw&UmTUVwmAa--=VasgRMgw4Npoo<2JkOvd$9V%6H`_%~eLuxc{Qi z3fEp`H89Dh@dZNl+1Wgt{*aF1V1RwDIOsiNctQP*n;37{IFxO&nuF8i(}*?T46~oR zbuP5oL|H+;=d!kbo_F}cmYhc?VH>d0=%QYSR?GM=Xz18)H<{ zM}F=sG4-^w-y|MWRGJ#f@j3P9L$B@Jt7h7Fp3`{awi~DJ@RiBI2~3C?`HG{+_+V zfJO4xt?-b2^F8n^))VaE#V}T;5hq!8KVwyL&o}t@6#QATl;VFw{o-Y^|JHx9ofAJR zH2mH1ZyX(ix48YiAK+~iNSz38wtzrDkEnRF0Z%d&nwi7a5gwVw4^ zn@=W6r)Z&5bk^gKL7ywZrOLdNf6s2md8)G)9D-4Dd2cubn zklR+Loymrm9w$zC7(_nBN8#7HvwRTqo(sGDmbdn!9cRIPJ8)}{U+evUzJ@MzmeGZ- zF8TH-`XR+`q92jL79WN$=D^oQ|3`jyhW_QdX9BeMF>yJxe*!f2A@Aa$dkDn~e*3P@ z)M=ku=P*;u`dRHiz4VLuLO#AQRmOJ8_!$0$-L&c7Kbo>r?7eOs%_dB=KM;@GAs!bC ze~E+7#KUj;!*{T=TRM>YihSI0?wm{Og4gN1o@f%fULBRkb8h3T2eNl2b*1#RnvVCu zZ!~9;n|apqtfww#UC+H%;9Kvp>KKxANRv)r{<&YSOd)S)D%d ztg3DG+F!{uvNv`+{F`lMcHZL1-n*i<9M$NhtzAQU?C)--Y{92Ke}s48XtfCw?Yjvj zdpqfw$lhOmM{Ya!Ty^-;3uL3;KB7;!j&ik+(TRQ8ff4<}W#-)<7LL-8L(>LZO`7ZP zp&rSgde4Psgok^H(|Sa_-1~emqIJQ(QjO(I+sBxSVNBg@Xf@UTD>SDuTV&cHpM6PY z%wA*CQ|vu%y8hLgUE^XrX=b0{w6h1B18qn0e60WPAJ;Q3Ypl)t_A@4HG$uLE2Tm1E z1vdEU>Y}{Sp{wW*@k`w~w1T-LfB!X+D&FDoGnIF*`_EPUqxN`~#XAJMA zom(~DGwjQH+KR5qZELRE)&{pNmz%aq@8q;K)@|z(J#BS#QfjN0v|4ws`MBCB{S@tN zg+nI>SEo zQ^qZ_&`Bc;rP*D4-^v{N^ED1GPV?Q`*^P~<)8Bqq2Ff`j|8Q^p=j9*m^)TN`{@D`v zy!^A}|E2u1<%{H>8>qM2C*KVz@P<@)LmIpx9o~=uZ@_=Uj=tzEoW0?r=r8amW!CND z6C9Lc4S>%o>^Xb;cIeGFhTO`240lONW@&h&{1Xprhzf7A?Pvd?kvk;0Klo4{woqm) zA508Qf5h3Rsl$%(D0h4w=WMd+k6V*MCpBhxJj|t?;F&QMDUm+Ox|n#MiT@|@rmVE& z&>X$f?``S}&QKw5j!{1StF`kf`cSZDGJgM#%oYcK7zdoa;>P?O`Q}3A=Ahpivj%=| z5&nX+sBJ!vx%a}Ntky*Cy5Mda%_rSHt4MS&|DUo=bj~|-pF3mqefXg65ueB&z4&WA zWhYa%=8$9DqY^E%bE9;E#m;>&EY6Sz}>`&?yskg-oModeCx zhIVd(hHiwGW+BJkfE?@SF*FC9>)#;%wi+7j3k?q8zeN5|qMT&POMwPcp}{`bEBAxe zoHNv?ph2A}!AB2k(H{GRp1JCK^U*nTc2j$&4Ua-|_-v@4FAUA~v$h_iy_%Oe9M`12 z`iMRhpB_l4x#`EyjC4L9xbc1y-zS{R`PQM(h|1M|Ea9Ga(0d)NBF$$twU0taNtw0B z(JPI>2kx@W`oGf0s^jmJA=?7!XLO#4@vukV+Nbj?|6X4>iTv+Sj_TfJ@+Wm3$qPm{ zoHg&YY9HaJqeAOln}VOv9>LT~{eR$Hy2(8X;iub7D7%4K;PVFX`gQO-6FkqLf2Pwv zea$+~B|3Uy$M0ORl55@(imvS#Y^35z;k#=k+88?wG)-u@Ez2LXjebT-ouB-cH>=PKTg|1$-mhvUr zNEft{d17CU=5F>WT-||W_KmV}CQdkz{B|R8YcEh=X4wB^PSKoRh+pdH*mc)qtF2#L zV(t$3A!9&&NW%<$;SVmUl|3QD%O1KQv4h=<>6Mh^0)e>vQo1UKsD- z@~gCU-5{&!aGuqyH04`Gy{G2)=T6_+wemCEgU`#Y(`D>!w4>{D@ZB$b9NaRF1Fh4n z)55XrQjc=3qwCJJfU#e^opf-~%0A6IuFc;iJ^0PC3wq<%we3qj2YxN$t}Jly`>>1O z>%gzD+Chh~_&{p)^m5;n6ALSsJ~CxR!`jNk;3s$a9$Xuj8vJBCzO%!ZSYw~M zB>!?hd-!FSaG&xeR?`9Qv}lCh{J)9|g$V=a#f27nmws}9^7FBg{9fOspP2Z?u_4(^ zMrrbmbO(qt^aYpq-V)8R!SU^n}BUf~4!)x~ce@y?h}wIArg zEo0uL$#3`I_RN@{L);0M-?ziRHRk2}SmRpvo@(uEcm(`evg|bSV{<~i>1KXVeyve@ z^1;t^tlHBXKUFAqK%$C$K7pf|^kVS~@R zdt;6C+?>l|oT-hnw=ATc>v!C=?RCjdS(n1k@V^6pHgraMiRsf+J7~g8dn9xrxa6Nl zWeNW5UhqHTw)b4{Cr9D8?l<~_HwAyU;GeWNKsn*OE5c#yc<8HL7CMD>zQ@Of&w~Aa zU_Va3iOvMSr#Jj9g}|>f9cSPl;=+9rd>$Ri-L=qj*&ypO*2;VS)5URnB078K(0q^o zcd~U>8ET)}rv8ie&G$anHuAZB^AFPWpVw`h)}1OhYWv$kSqE5C&E;IG_(sJAJJ-f` zPukl!;tFF6Ci|tyv^&Rb_h+i{qnX#(qEz7g7tT9{538fdiSZ_tKFA>uHDeO+wOIw$xhynt+O9n=MHR_ z#A_$h-azuIwf^C6uI;!AT_}4`3)z>(r{wO*v2U*ZT)P8Ke}A^!iMC(;ZQ7;1$3*)9 z+TPRV3ejt-J+oJTKW^HXWmnNA_?0;WsX5lsNgy|z#j~z=aUyzXa{1N+#2tnX4q(e( z!95At^qK71;4=rUoY6+^5Wk(8=J5EHcTjnW;0172`EPy9XAz71TTEpvR0xU*%BtB8y8pPk@O`X})o;jxH!$#}=1 z%^F`SIv4uVrS*&W@5e45dWHWqhLu0^wEY%47rmW%kb}T$@9!Z=?R1PO-L-&7tI1_FfN0^ zF-GQWn#+FACS>#{efVqQnZ={@D%KuttgjkBWvz>TT6d@W<;O90?^^%qlD*QcYc)c@M8E| zs{I1xNH*U~oYp`pd$Sw&7I7+9XSSNqdmmwZmlIa72Jk1(sf%R|kiYDY#-E*ZLdshQ ztthPJtgQ6jdCQW`8le%L_hI&GbdKZ9IzeZ$wWlf_wa>oU)q`)L-gN`4rdDL7GL2p4 zd+oD6Nx5Y_9?z3&8G~Ci*Jt&b>#am<&1)sWL$x*487F5A3}iX;y)*76xU%Uqe>lSV z;?`NnqpS(K*n2!#nG`rOqks5lWk!JV_vBkGpX6JupS0g;febE!T`nzku^ zAM?ZcyiX-wbnp&iL9&2JV@zm1y2hlT|LdIwZ@-na@g@!1<{U>pIK($L5}{b+1L*~I z&gAfD?ulYwF&5dNW1}y8cm%u!UCMrRf~w~kXu#;ZcrTzGZFdHnSAz4BmK4&{>^q9ppgP)Lr?;!R6tR*1snP?h*l}KUJwF?Y7iCH8w$37p*1^-VgV(hAE4Ne zv9=YZRac*XhY09%ro++e4Hd6nk}6Qdevc>`mhR2;Y)jLb2UsL!UB3{8)OiVw9!w zA8ZQHF`dsy8yVjuyNZ1jA6i}Pu-CCqi08y52HM^vmeG2yXk4Rx+y~Ide~evY%}{)< zss9>c@G03IRx_TWEw2MJ zYf$i)CtcX9z(h-NUXNVoCEhDnvFskur3~x}djB!XMxZ@8Rz{QdS(LKxz7_hHp>+Vh zwTyh_wK9*rj!mS4&l?{7$*|x~*CukE>P4TgPzvvTL@7Mt4pMj<>nmSx4=F9%W9Zyo z{Ff`dDNU7}kq}{S%r8q?ExMh@b2GYd2W^8_D&9oYkM3M-#_Ef^`Jd#mMaT)VflPVP zfzJcp4_GZ@r~T$ra{`BN)R;21{Yi6jIiC_5yGXLf9lfx@dhCI`BUxxBYoMdk1BX4( zSe2{kC0gsT*LZn{GsAOwaXuq@7Pr?zF!b;~O1+$w{XFj$IV& zPycOXPNq}lcgXxox07mKALyg_pu*d1TfV$H2jD32idCnziPmQMmGKXQ10&qJ)^>xB z@J=hdWbgG!(g_{w>y(Jm>wR5bKtNB+$?wHW==kWEYBwklZj-&sU zV>6(C_1{0bwxwLYAEDnwm$hz?4PC4={t&D)xO6riSSOuz)(}Kz4~f=+Z)K#Svyv^Y z1}>Tv-4ha48F23BZW)sY-ihAs1;)DGPt~`jNfoPVy#8gO)F)OQVXjLSmh9VzZ)1Y5 zTVg)GHtf&no1gH9%es2Tm<^Z{~jZPubu&Dc)rd|uXN#3MMD1sqp9g%dTjwL|9p zegQa;ZTkKM?fr=H3#SGiO-l+q${9ztd`tXiSzB_-IcqH&-!;iBt{rVWXVJ=z=Fi42 z!8iX=={))9Jbr9Q-LNNJ<5{{&csCO}ylBeebO)c_2KVxV-wXbMbV9?Z;HO^Cq4AV8 z&!a{KkpYqoezx=qdgzO_U7fLi{Z|f7)Hb$nHgc@Se#WU|Z|MAK<6lAFiU$h^*s~VW zUf-kOKyfUO1{YckE+pIIU0hhp|AYr8z(d7ke46n&9T%1WbL8btJh%;*@ZcnU?(o~r z@h?>#?o9#D!aYT2$LtB0_(RfZ8=P_8Xu0FuaN0QU27hIr z)i__-InKMsSOviu^sDBx>3`@q{k?m_Uq{FG|LzI-qqSe_P=N7>+MdqEE-0FF1N|X< z(7f!&{L>hyzxU=Q}6*w8ZWA%I+bJL0w6~GoIKY!Js zUqz?&`AGVtqt9Kt=s;kn_jzCS9eutwwqK#oW2s-F^62xYWAofO7pFY+x#idpw(?x# zb@tc#2iB}=`0_Gr?x*!Vx`yW1>3u&prtc5Z9(RuI98C#adGt9S)%@n-XUQ6v~Xoi;lw`1ANKbm95 zlXl(Xj^lfL+o7S^_B-_Z>0|f`u+ztI9Wafd~W_r{LpvCgraz*uUH?U?jL zaA+BA)>tehl`J7SO!BOFU8}DMv#xtj-Xc>r#a?LARQq0~;Ojj~ktb#vc)I;LdghH> zGo^2qOW*9x8cQG6TKcl)lEm5zdu`{Z;QQPKUA8N7Vi$Oq<8u++XDfVzZts?^*{<^# z&>c(hA=`NaF+7kHC11}#r{tW9sWs@7d5P8*XWcHUS7m`)Is=?F2k`A*@S}melOJhc zIf*t^@hz?8Zs2=bhqTxyr*ED>fBA^(RjzRPjlXSp;KtvEEkAn+I0J^Yh3J3R5mQJ1 zEoBXD3bNZO3*B(xTSiV=hdlH)`)Hp8mbbuxd4M$_-~Brm8IFE(^vyK84`U|0{tht5 zR+VmdcVQP882fpV4_TYsEL~Llc~j7dr5kmloYu`+&?B^Vro0Api4=S#C8K!VeED*! z%sGyn*s;G7ezn3Hmez4j^t;b6j+E`Hab&#XlrnD{fjjz)bnL4czp0Gl6vlHh<9Zcq zXWtJo&Sveb*yzX8m!%+IS1tU&^v7c4#!|(PLMN_Vm}+!lt;2N4i-yNb_scNntUiq` zOMSAR|4MdVi$6)*^}Qu8IdpoQtIy(_cpMtuwh7xgaX!L63)^|3v7LvBoz;v_k83wy zak|~SwH~`U@fM(4^uf1+DKWbA1--1tKSh_W!fu}F=+VUCl-<1NiJ6i?jom!cesYmh z&t240I_?hWQEjs)_{--V_#430?=uBQ_n+zdYP||vH0*ey*u0*T6WZ% zv`n~%ymu%JetQ`27Ia*P7U4rpyZ4qE9U6bJ4Bkz)^ImZ1ko?8;t=aU6;OejAHx-6l znfz2-y&>J~9lsixT=;kd9F6MSdQY@U@N4*%@}$d8G2hLwuTlzbPcqLb_7x^gwuh2V zklwvZejgd&b0&D51%79P=j_jze>Og0@zAMwWGY9O)P9X-bX)nt$}T1xjnXR7sHd)X z_T6r)EDVJb@pVHs*_Vm_niXh^<9d_(S)}-(Pqs1&+xlzGQuZ-uN}dO~8Gn!p=t=|n zP*2s9K2=*ZFEuX)0(0m`hV5}-=cwEel}oozaIbP_sa$zn=r7O?t;bg=g&(1-Z+;w_ zrEf-5CQEu|a#J}pq{Zz2$FCiDhuw{SbM}6un>i`?vuC0uyE8i0;a^|voP{XOguV}Ek~59#wC(Dyem2G=td)1fWbg>+`3;vutdqp6X)_%c}Q zMGxJDY#v>c9}K-IS&+7Rwm0RSGK+It(uje#AaS+!2~}|41nu13K0M^C(aVof@-hCR z%&P_1QPp4XQ6JIZ^_5!3kCw?ad+p_$KF=-p7Ufjui_-90<~l;GuB(xSsvdQ0bx)Ev zovYCIvAbQEi-=6@+udb--G;i=ubCjQt{%)GZRKxefQJu?S+9u z6BaMZ9NmA=oSo3m26Vhs=+o;n&h4O$Ykf&4ikYhiw~Yx^Z66v^+$8(?VWHQ{Q?_m| zO5NHBjT}_i?U;P{p5JzH=*4XXq0d=ce6>7%>kCC0TTc`u;g^ejaC^6q)^5U;iJ=qk zKwpVd(73Dnv8HW3LI<|>4DCN|Re#PN14EOcjr|IHoak5BRqKi?tk1JpS4_39EOYvG zKK&(nCmFAHiZ^(*?ZAHqT=LctXoaObcy*d<2mJ|olGPP+zZ5?9Y>AIC>0EQ10!*~g zk(=A{a4V^?n`maTc(lz=Z!hnLm!)Mjp7iguL@jUWWZzY##W# zp1gBSUb?+0HV+(slf1J{9&3{Uhc3U*v(^pT!NKr-6|10+=tYHlT8MpM=&G+`WDRSS z#tw-uWnDeA75b`rE@S?wKZK_V^vT|pF8#bAoBT}sBl^&x0ZDcyeTBVauHwg^k#2@F zjbBA3bWV7Cq&yIkf0wzZ?`5yly}ldIw?vP-D5bA_<~haokoMsH6Suv_y!0?Xz06Z$ znoWv>Ms`7#^90*{{`VwPNv8WODKS*sgi(Gl->?d68IaG{XmWSQIPwWC?Cn&s-bJRJ9 z<-)+$Zf6CCC-f)YxW2vk=HUV6xb}7R;7q2qp3xulu$DZ!pSaQ669O;$h+}Yp|B=Z( z9@+5IhdezdJiOtd$vxJGemaNyWuC-AiNrLR@K@`q8GWn^6=x;??##Zkz1BstoBA!z z1LNG(F!Zp)6KjEtJvi|qVoZQuKH<>5BnE*F^IZPxPWsjv4zZ`jR6mF_h{!Wv@=@o8J-?< z9;DvgsP`hzgnJr!?-ts>KWous+SHA6b_UI%Pv&65ne%(v!W!p~62{)sIB-p7>%YCB zqwnH>H_!^$kV^K?ehX5~Km|@nOb{QYw6nhY`8W%QZpi^({i9MTo8GrXO>(I<)4!)*)tR(Sf z<_IzJE@j?~W&RW~kH#>c3gL7AS0x^&wlvTd*5R8<*r(L^+(gbr>el4rKH8Ri+R`w| ziVQ{%P(J&6-f2QNEJH`BIY_xlfymnn6IM5jW6u!Z$pfY~$v4U2<2~);)KU1;QsnS{ z-vMNF+G6r)Z?l^(UZ2YzYPD1Q4xRT??561Ztfi<-O9A`M%^7Z-zg858)bS1R%*n$o z^B=1zzMz&dGkM~>Lj#eCZeG7e@+jY7GbV4Wn-^$^FZk2b&?01@PX3K;YR|-w%GXaM zmX&Kq-m7x7$Kl_ioo+mdanNC;$6*`05UZ4#vErvdKRP7#aOx>fJKMiVVAed8j#B zM0~w*p6Z6d<3o+lC5IYbvWVkmtuXQWHd!Z|mwBoq%gIM>Xm|&^WNv1@_JcOM`x0yC zTPItyD3e2($Iw~Z%J$@QmX3Uu8Y_E+8Y@#n4bO3|{d2uTYd3k`yQ<2O2j;4O!Q-`g z^abPdciLJ9AE;%F_pZDwR8Rfmu#YUD{Gx&EeJ;9gYj~MeU5vl-hUeN>s$Bdg=UYG2 zx0oY4&f{Cm2gPoaUZ8s~>-Dm;NUyE)bqNX<^gV=9aMIRBUeb~9@c*8o>FXZn=1>9{WC!-bK!$PM@A!*B^=ZmouO0 ziijb>eY7r_s*B_neIHy>UApMs%%8_%{|!FPBJWFJHST{kj#Y{ik`v=o67#=#{I4Z` zyrFrA@OAnfcr#|LVpw2@T^6X`K_AOb~q1u3AOna7xr&p z;t}k3jkICEEoFb$bgnRT(@(0HG}BqD@5d#2FjGA9_Ws5$kKd2cjXvNTGkcK_O%%R2 zgYRaHz}~U!ABsAn0W#o^BlDwECY54Z7?|H!i7J-0Pe`g zyi5NHpKlk;$j87KC*jf@U>chUcw%r||Du1FkT*+t;Q!&+JY>i9|Y zU)Dhfe#97$Io6*(z@`?1yXoMr-<3TS7g4yORQMygSE>4KJpHP7$0?;xFI7stiwr!| zE>ueUM=Pa|Mk%HLMv(fnr;Pa_dR-tm^YJ33LjsYZN}>N3DrF82QOb8OP?{TvB#~+# zH|rO3iD5leW7dk@RJ^(L<@RCJukT^mQTe{?vazY?LYBTmn=+I_AJa(xDSJk_&Y_4y zw&;S4(G^)E9+|_Bte$|Z?!}(roIjy6Ba!{z2X`m4kNobv^R2sUed1C0o}}D7{1|;h zyuyoTdjrE~bFN$>WlOOs*|V0sKViwn!w){Q=kSDuk;AuK%-J&ib{sBS;5md3$ke;g zb>#0h-jnc$K?}OCPMAICk|yXxLVNo~#h$YF^MCxz`{VI3PuTIt!?W=>(f)3Ylm2&+ zJ1*h3?tee;SFaonFZ#pby!gF`8_<>feDCyklX*9Vb-Y*dtc@Sk#wg~U3xud199%>k)ahQhP@Z1xq-XoYjdOS2Xf!PV2<{4 zln&PYC3Dtc*P1!Q3VKHe9+h6Ud?oAW#Hzg4Cq7EN0iN&WOiS&X5pG5MRydXkj}A;M zjHnH!ub*?y+BgTfe9X*2!8@57n)e&|hHMZK`oG^Yowbo-_hz0cFGQaF2G*h%aGu04 z-)O6^bj3oxFW)$Gh9r9T_4mRHNmo+-1nJGx#n1n(>k1=x025uQJ8lIlnTK%`iG|Jg z>zbYYse%mv_82hLS8Iqf-MTI1-8$YCZdea<4h43jeY-e^BFRG5OS4~LUP^Ck%Tj-^ z=Zx~lds&ff;FF$j=h?h_${7|}_EXXK(wz5ZQ*H$QWKR2K$4{|eqCRT#-m)Ec*UjH? zceL$Jo3re*O`Egq>Aaikw%30Fb}Ntb&4K(weR!?vL>;G)zR!h?fpc}(C*<#Icwh6A z{0AIYoOAVA>^D-~Z;azT(cm$pJ7d4WnQ5Y#b$mnkFwB(8g2!dq7jj?Qi}|lKiTSUT zJ{fG%G&|p}t))%GsM-7iwCEvHy|bawv1L{IgBS9=jJk2Un$@Mj&^Csa+l zK3^V)Tv1lG$8UTP|FkN{)7``aY*;3GkkEv`$J7^`_A|$bdsN>2YxOp@rxm*UJT!hI zc#Ld5xgoKzdNlR-dzNN~k%z*Af0(bj=TSDHONGwHnu~0;s%aYW7rcp0ZWj{eQHx#6z+ZJ05Y-QdB9(UE+=|D@94RTv$Yeweaw9^_f6D&8Q+@S zAKzu8M|NwfINCln1aH)OehA*ya1$xKt^t{?k$wviySo8-Ozj@OAhbHKf6b~e@>e5% zT8eez#|~D)+2{?(4mreS^69?Bxvy+-?i0KT?=-?+XD|kS!6+LGD{Tf=B%3<%pSZ_Y z23o^emIZO4LnFF|>Wkt-b@5$7Ma-WtI8Zx>`q3{r2?SnKX=tCil66VRI&I+hnaUJ}-FEPvX@_uk9=fdFBmF$~J=z=ncKN?}?>4s_ zspw$G_hZVbO}U@7Pu*U|S=D8ia1m#+?BY<=?k+#Ky}m?S`rebS?%ntV^pf_i--T@+ z{NFp8?+CtoN}GB>J6Rmxb!F)k39=azX(Wtg+-_bOzf^|_RVMwmK*&#mCEd>n6b%WqS8=E(Y@m~VpDD)U3P z%x_hO`Lw>=Ewg-FsBR)UtDA2tpSdNz)Jk9Ke`|m#7LUgA8SW$0Q~PJ0=Dv#a4XXwg zgj#2G4@G#ds{EQ`iX+wXVUN&J@K^qOqMKpHxOAB0J;quo-zE<3X2or)Qc8PDl~Uh4 zrTioO9a7p@MJhb)`Wr`QS*$u5U5z~{F6>^xpdXrdCw}deyHhaeg67y!7xq)ZMhlj1 zA9i843N}=*412!|D-~>rV4O+r!b$`KXQU(6yRaV$hFqdK_D>gfonXCyfx~}uVN(T5 z1O~qT#f420ES~YD|7);CNYATK3f(DI3hw!p4kdpeDL7W4G>d$tN#rZ-MLzZi`5jj% zjVGV9u{PD0?J+NV#k|!`tmRBB^c{PMvY$_(>@=?Hxn_7G7x1lhtjiR#Cew(W=HM=G zb>q9<>V2DbjH>H3bwl0q-W%$AT}`@ZRDDuk?x%3yYr_|1tkX^LR)2BSQ@wvuAhN%g zC$c}0f0n~Xm%~RMKo5S)I(Y^E-2aXh**}mn?~)df7G$y(?_n+8Q$3h^eLgNbbZFdJ zp#$}v>c)ef>i*Q_0J?M|{GkzjIy@iyP&55dM!HOVguI)8&*3^6s1D!8w>-V)gcq;K z^pCPQvmtIum@^-cPj*}3bBsQ}K zFyHHhEdlnNVDlVUInV34o(1OU-Pkjkhi?LVSTLW~s(>*k_W`Rg<*s#L%+EMvi*i%W zg)vtL09$CvO>@dIZ^r_=+mv%*H=)}DlMbMH>m_w`bp8ddFCp(%<-t#nL%Tcj#swnl z$@?FZmu!<}pw9ktdm3^ZH-0dCZ+S`s*t4B8DGj*#C6%r8`NU z`Z1|&Zl*7oV*@CEDdk^v`PchydGVNza?HW8lp8_0N@$*WcN*;mcClbS7Y6-W0?f$e z(2eb}bz}anCvOntML)L2<}s(=Brlgd@x&^ZC%$y1x-jqeQ8ry=Z*$9%u0K=Rc>2S) zXmStpPkj6OIctd0FTyhlvBh2$%4h8`zsrPWtPL!aUu8e`Ifg5tQv*Fg>(KI%d9rhj z$L2H+p}r@S(#AHW^vf}&^j)h`#-T;&5bQ)s8SgKYf)fXn zf;XQl1-BZNf`4J9jLW~2-bCLiErHJ~WejVTjtE5lsg!x}iPFK?iIp;cK2ka-5cyxy zBy1+~ZP%L0H=ZN4%Q;6X_S^^gct}^$`lZg2lD@ZgSNiMdOC7Nl&Yr?LV|Qy9XQB1| z1Y6=Zu2o!jab3;zJbjQz8X7z%RP0G`{5pDCTQ<>GrH{^hv;jVr?+HB0c-rH;um(w) z3gUn?Js%gc&Z@ey5ACpe;IB-)^LOw^F6$aFF#y*SiSd0npY|Vj0ZV(w$k)JTAV=2_!=U*kE7Vx-3CTa> z7RFh+^27YY!G~=7CKsmvM*UHK2= zspN_xm8E}@%sO6@*+ZfFjnX%*q^3&qK5_fqXD{}lS|96P)Z;SMo%t}O8QPF$Fi^DjeB=d9alW6{%#(9_4Dr*jTa?pwDA2cxmnjob;nan`z0 z?Rv)YAFj>!LcS-x)sZKoWzuZ2EZO3DU+PwSSL#;DFSC$2;_wA&qVF4qBAm zqd%gs0pQOYz$!UsSuoku#saGntXw=481uNQ*V;D!sKDYip|S$c>hw9hhI!=9=H$(U%k)drQ5wv$#Z0Z`M{ptgC0bk zC2x$S{QagpXD=^uc*`L8kmiVd^&1kMJ#i&|&Ckj;Hsba>axhZB-vD z??PMB?R5G^_&A*R<>w;$l;pyO0Xqr};ru_tW8g2)QNNw&=ATc#{0rL#I`p-R3qKFI z9yk?sJiz!RrHhp2?mXXy*xVSNPoE!tR59>c8UqtwD_)GKNVr?87Eax4%$Y zz;~1)5B!DH2fcou)VxbOzA1K++7o8{JJ%qibdq_^TK9vl-uX|yskx5~@ILxlbuqSM z$G#9PkIxwMV4+)Hbx=9gLq0{!gUnWRhMG*PKz0Xc(?%DQx4wXhc9@@t9`(~!QanW3ms1#^H~$k*Ph~mNul;K{Icoi_DN~^wR^n5$!i^2 z{Zs1Ao~g~!&r@Ch>05z`{%Z|H@`tmQNlbcoPn+a}x@OVuWV;+WdjNfM1u$oQQ~M8{ zvllM`CVc-t`pTipKc^PJ%GnYJkw1zqQ8{dP@ ziyN)|F?KJxAYXUa=RlvtG=bedHx74_tpbHjlX_x*uoq z)IR4dz5&46DW`sj#+Kbg88a^0D>9In)_ha(o#4g0`<)96KZG!JU1vwt?~>19vZ2ez zp>^sH$tO+Np7y?K==jDl+2>Iv%YKdVa%4izfLWn`X4z3a!L$`!q1C{X>=sg;gCrb3 zWMC=wL8bK7e)F7aH>r$?$*FzMoDWcL1*--Yh6Yhz;tN%W@h3wcBo_Y|{D9qa+{(KW zVl>o4_qq9%pk-2tA_0SzImS*ZtcL1 zz*ka`#VK!WgH=kw)l$+YT;K7V`Ogt(qxPW4r&0E=6xq+IYZY@tdty}2nUss#funKW z^e^hO`8x8<`>s5GwfkHLKM~#3`kT*V_T;XxZiox+IRjmdpSgH>QHrBe`AB1Z z%o}Lq&8#ec8F;y~B)c+RfMeJ;MCZ z`lim1xOkF-14C3-=%Ax3INuUKh?YCwEq5;E!r*2DV_HmKhJ4^0@9AH$|2cv@p?@jf ziPmX%u6Jl|ivAO?`li?k+<*0dtm(RCyOD3^J^%Z1Isc8<0TI>TD=WcKa|AAHcj(O%1}%bNe9hk&aq{MRK18`7*`zeBxFyv=j|&aS~m z@Ke64idD9Fw3W1ep3@)i(Jr+yAU=ei@%*nG_&dPGhjwU>y6B|qhxB{k;=LN1?eI%S zmw+a_{K*DZ$yqGo%h6bPPY7o83t;r+Sn9q4nCR%)qz-=~58Rza-cn$iPkGGyRg&G9 zPswh%4Cd`u=EzfB%zmx{WF(EvmG0c1Kq^_ZOYA?O0ZS-zJFvgfH(UNUp4gJ<$eNCi zjN}W&wRh_8l$Bg=%3c{CRJmP*g2dt%Q$Y2#?B%CGM51P@jS&oD1=_CAFFb$>TUS*i?-~8 zE`3J(wO3MmBEN}No=rbQ`DcaCDo~uDBSRFQPiJE2>`I;KrgO=N4GgYn%}KU2=`DWF zH@xyn^e*`0UISOY%Bj5G)BE}kF}#Lg1C-p)d&2)>_pHEk;7=CUUgkw<-@v0)(7z|R zznkk?u1wk&ZeH`KA3viqufK5}ZT9mo?XN0EmX=LVYsIR|#g8~=)hvQ zY4Ik{<4n37;!G|jU*A`KzNw#N|2YnyT+Kfmn^=bZ7F z`-tb7`UfJnDW#sbDrGF@m^8`0igW`ss|Gt}WvazFk;zRZq_e+N%+)E&2C=`ZfqG7d z_rC2-erEFD>s zg(cj-t9)?n2eI$M1J{#Rqdf5Ln%F#W`%Ut80(1C>W9Qlj?2o|I=YC{@JlV78U#*#p zC(p#=bK{aU_)-QndOPFsX^;J}iB`mngBz>j*a)qoL}QYq+UJ7%>Oa}JD(OSl&b4C9 z<@xZOB>Shobo{2B^I=l$O%Hn>@9psEj=ikR6ZRbEnZ1%1gYym@NwuXn8oZ;A)z|Uv zv%b5I?}&G;Q930M`lb84nlz7Zw3mC{`%n9_ME?~NN%0o0(|gD_sd3hO>ELR*&pC%j z{L+K|e)J*F;~(JP8GsJfblovLjlDj6zrcSM>l2B@HyjnHwmg0Ns*Hs%=SpbtUf#iG zQr!;z@6EO7hg5qEeg6;s6`?JoeW^_W{Z31rbAc3tN#*tLkH{NNEKmJA z&$nZE>VNpR%IM$i>;Vmb=ZRJA(25;{0{6`8>%`wX)BYOz$H?|(&a&6)cjz1W%(wh? z8Ns~kO7F7RM=3g;)6Y6o0iO65ugQ$U&asOhB{|)W^CSFo_!th)Z7oh^4D__1=bX3LNIKulU4F5FeM!P&$ae+LHzr%Cq zULNJSf=79-%8TzhXILpdM=Nrl=5Cnxgm>b(@V!IYyBz1n;u=e;eS_N93{FoiZeO^E zxwEC>v6;l-v9{Db0B?n^ir<#L-n5+dyyVfB9FG=e`0yioSmc%;C9N_RB8peU(G+m0tRS z3;UB`1A#%8o^xR}dbivcnlAndP4X*+hSi{VYfY;{DfA|j{Vw9Qnz!PmyHxkGWMEnL zGX|FI@K^r#l>2P>EB|`jeg2)pUr(hQFXRXN6H`VyPYHEVf5>(&UH2(dUwo#NQn$yI zLMIu}1^v5Vq6rB%CcZC0;H+IB)m_i@j=&e}AJc#OzCgwv_ zZ+saUTRiZX_G0wn8|=l{`8_MC8Q;-1m#0F%IA=IHm~p$qPcNgshM#t)uJGA@s~va% zxMTrmKXa!)(szMJb;V)ukkfU=9AM(J!wi1YkI>Bl+$RVg)fb`FV}ZpP9+Y7}3h(aF z7a8YSA+mhHkaCbd<2bBlzEr`tnr`{wl4D56rty$$6@2HNp zk~$f`CwGr?c#c2R0xzxkUNp{P*O1PWjMdL3gXgk62yb@dd)2{9OV5Xgq}XfdQ|&?0 zzGnTac9~;aMdmzpPH?9AV&ZR)0Jk+CZdv>zPV)idw3R2zRRR3_76=e{qY+5@@o2YDt$YJKAw!N?JDfJPOKQ| z#j^eBpYnayIScjJP?sL>JYNC(>QwAEkC;JDCSsowo=>tOqTON6yKBI=PI5pqHmeic zh!wLd@mN03TJKK8mh}R5^oPcu8+^Fk(`Uni_$h0Ay$|(7<~hhb3~%oq`qScvGhd|4 zpWiz#cyN1<&`aDOoO@nyO?=>R`&(A^0na0=Ch@&B<<_A^7e2D;*ub;$yZ0k5!G#ZJ zHXgOA@3?SzX6w+0RZ8!?*&gen9H0MC!Xj`yX?$CQ_IJ0BKGe8!O(y;v zHz)UbJkuZGOsYOlu4-wvs&9mb?f;1@|GWKDi;Ygc3vak~y=36xg|ZL!WKP%RYK@}P z4=%zSb=Fhx5rZ}kpw9b%WznCaGjq8Y4f-|s`IUK@{;lBNc+Q$;o<;GcocM`rvI2+K zu|_&0*E&>6s`DDgpj(B}t>7!zXo#(IvVCP#zX}r<1s~07qg#1;pK@kHW0GUb>rYI_ zU5A~ve)=GLSCDa`N7dR_B|Ts%^)_R!a~sgPI{3;N#@>wi9M-2wtdr8A8;AqfvZqJr z*qpPRIB@fkx4qepjC^2Ywu% zQT1uI+ovPxt7yLs<^Jf$p3q9gDo6`FI-Yb8enZ8?i;?c@?_+J50WR$#4yK_=(Rhcw zp+^bqo9Jy_R4RLDmkWclWe2_S+rbIG`G*yUeLS%b8(C*D@aiF_oZ&D9dxvP%gS3q? zFuHJ_`ZBjqel4*H!;=z1%~N9f6F#kZBU`WfF1#l()PnAMjQGMvFTKr*obC44&-kXp z6I1QmI{FIxsT(tIdPiR|-(B3h6qxoj9Ty(cSMNc$gnygCKlN2BxP2k_vRw*~WJetf zyny$r4Q}wc7nbni6YcqcJy$iv-IM*CwUq*J_>1V;$XaNNV!B^MeWEe@?G+ZC2;$N#3Wl}%0U{IzRy zIYB*~G0wJI>33&7bdIrfM~A+2j`1IXU60(a@x9I+E9t_~xuCw(f9AO3Z1|Aj3CXs` z`gPukj`eu={m!xGKkiunmb_>TKJ}IQ<{T4;&skUf5#wHbS`7Cv=iO=CwRVdBxT2rd zSJPNaJ!Q<(>@3FI8E?)2q%S(gyIAr1Q|;1@@lLaC`bcZ2iiLG?Oq=i-4Jkh9P>=r| z;e%{6YTuHW_VqcfeGZ;=w()NAjhtpVu_A@vrEaXq2EJ8>PA3{rOx#GF#qi&@uioIO zBlF)F+rFoC{zR(%?U?rcl=jK4c)M_cd93_f4J^%`LwXQjP0cfB9Az_d+Bt!`vwmD1 z9m@~sQ_c0xv81d!mZK>f_DLUe&R|e~j;DRm^A{N3>hWKRL3`R*%I5QxvE-~S+N!lP z^`q=Q=f<{orsi*&-SSj){yH}Ai_|{o-%z!gw(pXxS)@I=KIEV{nRiU>px}PmDSCV!-xdG)7`iai>=RA0 z@27v1mqVW9%C&}$CEF=Hx6-FKf?J{++6Q{L!jVbqwzChEGjNwoLvG;P{G0Zq*n+JG zCj3%8L<|4v{yW~Z$@ow?`#|A^`&hRZFVi~0FzO*cr-2@0LNDhe@{JrC5COd5BOXO?{raQw^79?vKpjeg^o?l=BS8PS-xNsD8@VfJ`t*|UKQ zr?-)|QYY1GtAVB4ub4E=-fYqg`?n@du{WAD*?!TaN%jjSO+`*F1`o!850`-#mx3Q- zk&}x;;7c%A;Yt4H7@A)nn@;^5J+tA^IDD3|sTwy+m{1shF8H)~53{m&l7 zL;Znkz33;ZBjfwAtGoP|zEB_E4_r8=_~q*R87_R0$|L`LWaxLgt^QYe^?6IVH#8NU zCXcbx|1WXU_cP>kz&lTowzPXg$*k!}cBqk#1MjLJRlK_6u5NQV-)sSQq~FMnCmW)4 zi6~5K8j550H<#ZBC?C~@65O(S-o?AvK&gMaUEr2s-EL|WCw$BcB-$Bm+9z0$xS8bKZf7`YBs}9mTqx4mE7v5s_@$bd9W}Z_nua-afSH^GsP@{*efCj65iyge1d@t!*$xhey z4nFpPv0)zy!-LVeHs%+eAI$6GslLc#F|^h&Vz?dhFGBa}Tij;iYN2;Epm(tk_2vrN z7S+9y`aHI(?ayL&fcFUQ{F&EZT?gx}ZUS&?Hp7>ZYFMC7VB|kS18(#ABVC3gK z${N|(cmGJOvF9&{%Fl73@Q^N{x)S7PpXBFMyT;5V#eq+?FQdN{@57OuFCstNPFWet zd$a9p=u7c`$8MBjPd5GT$Q_fo*WQ&Yl+xD|NqZVQkj~0Tf%c|CgVUhJ>Cj|+$>wwR zD}F%{>_Eq`1I4bXD+ai1>&NNiSQ`7(W%!XnV+{=5-RSUzbB9Y89gq;X2OH7U9q9E| z;KE>k=1~8ZZwHsEK9;rR6u5O^upD?lXsN-cnHL2c?#EA(a_0HOMZwSXOul4e*||>6 zpH986qyE>@j%l>#8hFXo*jyanCd&0S`_!`n)w(ZZ4|*{%E?(z+s_ZY@hlK_Qs^31@ zKCFs)QgO0<^L(!9Tm@VyT&*YCH`|Q0O`6jsaESGYsiQq{bEy+)Z?k8gIWc!Jd(+qW zoIU(&lB~HNo*Rc)5$#=RD6k?;H(AvmQun3;WH#o%%JlF&lgU2QYWR=YBd@*jOJ$2T z`&u%ZDosjEHqu)b%-i#zH~pP8?1c_PGavM3n0t8G)CbEFvx?b!o})eFJntWjU6-?A zKZlP+eV?xbXT+EOLLH3VA6;Xr3wu{E+5Ms8lU&#z1j`i+e-Rh7KLL8%PdX|# z8*?vzZj9s|!1k6DobkLl*{jg8_PU8tx0#T8@GDSv)20ltw0->4dm&H?XeppKdwi};3f z&-~#2DgK!dkdKp#2*)zSpC(9sC`@ z|0Rd@>uGJSeasvD(mi|oY~bQ2X571z$2jc3=Seu9Mjm!w#xB|J11)#_k~2E|)=ofg zM^c}7tUwo&Bo77$fnNPeiZ5>zhf%PxKA>f{Bm(Re8*{ zlGr@(PrO4ixA6Ri*gWRVo8-L(%<-k;Tj0PxV81iv)9mlZmS=v+M_O{Z=D_&aJn&z> z(owm*4%`rbDy0uJpU@o>{hU$aC*E#?Z^!ifK0Bs!eo5vc&X7Nm>t`>=(vg1gb?vdl ze_FN-$9Foz&S9>pk9F4dFX`vK&jqUCQM<=*R*B%ZJQ4LbLr?jI<8L;v>tpPHNwsaB zrPG?RVoZL+v-VT2S4y8gtCap;OWKQhXz+}fEj6q;oSeTM-O&mjAIsRz-&wVT^#}4d z`ib$GKzz?`#P=LbT+&41l8*Ky&RI%)&jjLo20Y{M5zcn&9~j<5e9sA<#BRiq9K<^R za6jwre&TzMhF^ITF1@FApw9i9@xj-|nkiyG4m?ulQa8Kn-{OyokL~iu71BYfsIz$J zyVO^{W{a|{VVkcHRBxHc*~Z?KLE6hF`XA1;rhd%+i>+KMxqi+yh3j0dOs*3d$g0>C zwZ6WJdwWD6GJ&*esPvb>p&QwQq5XiNIoj9Md#>!_1Ns!-(;s*h=XvG8vvv{}x{A0m zImBqWk^9dE6YF@I6{!YSU#I^YO03AY=<}wMKx8{Hq$IP5C#~ULd+54Zizi>fnIlVy zZJ&z|(u?5g8eh+*U96G51Rk#O^=c~Nek1gh^?@cI_b)S^YkYm0Dp@~W2mY<`d76s2 zH~h@!F+8jWf1(F{eawBSx$n(>&Ei8obKlF{R}%ku4t(Nf_*{jrCouE^#ngxA)nEhu z92siwI8Ugt$QwF1!WY_)&B4S{n8ChW+9W!VBD|U+{)PXh3+pY|B*D_{Mi-VS7;Aar zF`v1xu7Zsd3^~??c?2sGEXn@Zg`HReY&5Xz*@x)DjtYkTwDzVz`R-lTGzuT=t1R=Zy^&%F=T8(y4hKkq)5nCB$> zIeiB_$!M5Q+*o84c(mHQ&iziVo@wtd_3RHEGHd;`^=bF{KK`w?ibwy(^abfSIU2DpeM3JR45%tzUWICXIU}#8yV}+;Qk$Q$68;X z6H&QiZF$gyNClw*0Y?>&hP=DVd`mG6%9-B=p>OLTk&P7-^TMZk5kMs3>i# z`3Ao0Yj>rdklrkrq(L$XGQkmK5|e)`a?S&+O(l7%ThctzHFsMr1x7A7bdxhCXVXU6 zz-G#yA}KC->`@1v4qQ5@&axDL+=joJ#;`AN;}1)pc88ZeTkoTPJL5?ny+0anUGW5f zyE4Hc;B~p3u(8znb6}3H3ck?4vw%s~cVq`(^z{;8@0)UpbpeciUk~h0rks4{fiVVe z0;>V0{r=aGI`Ra0;Q2oCb}El~JSjGBk|P7WX7bYQ;@G^2jtsELIm{?{Vcp8_tQ% z8$o->GjjP&*feAF3TO}geZStDMohcdym;C}o@8$InYW+gGe49u6g~3eFYou<{aeMG zZ{ZxVTHo?c8KwXkrqahcmCuan0nH0=LUZaC@M3R#p(De*a0&S-_K3%#dUm$Gpxn{1 zqkDMo=2`RcPNnq0&y>C6FGl65r|>3r~QDh&S;+|7mcCM-f9L7EWCqJu=Ha zL3^VA9bR_lo_u1GOIJ2Lim`8r$v?XA=X)Bwy-j`{JnY~1u}(-|{U_g!hTQKnev=r- zD{nnG;)+|Nz8>4#FM^MVX3KwrGsBVne8^9~a`ooJd{=tR`pVcc>9#MhXz~+onFh)r zBTxPd`rleE@pMD`iiZcEpX<=ETFQvN?Z7`FR5>mmxv;vuEI!meuy=mAtWSPpS>OBv z1xXitJ|y{q#(^mppkKYwj&4=US+*|)oo~FWHsbePi=A_xYa{&wFzK|i&%FuHcI>pLxb- zQYrK)&O9gCXQ2~*t!}m}(VWYn^LcgF+A(ON;vq!ifNH)}#l%A>Eo6;|xPscZthk?T z*fRebXKj9+Yc<#1TvNG*avi&qI0MkEk9^kV=8t=XHo#wRGQ<>3$tFNZ*#eOxjnndHb%!ua#w0{w4Ihjp%}9@(1Fa&mHK3vg0>+dJh_nzF9+T->Q@G z1uqa&G!dJPzTI#qu_v*4Hq7HZ1?-(Ka8+=9F*+^(z%I_ed6hGA(4U%~?;i4NAB=le zcLo2`+H-%_)fdTsHD-@uY72f%BLk13a~|4B3{KPUF*YXijEzZq0oH1L1sl^Xiya%2 z)+R#QtM9H&4B{Vyv_5gP{h}q5`F=+{)7`5Myn1;k$J3+f;I2S*CAw{nr)N_Msp6AL z2bImWo&HSoEa&W$X=jd0irzMexTKZnw(H>OqlrbTeX-Bx>fUR7ge&3kqlraYj&8di zUOCzu*Obfs#4^v6IVsZ<=iHq$<@XDpUwyte?w!xj`L9gorfV^H-ya8z?=*rM<^z#J(%%Y z*mHHmV1MXs%I7Km=wK`IXK+n>wewgzE8=`$>5rpXPpjwL6zPz1%^8~okQ3t+X$vG>Ef7lAVZphe>Vy>Q& zk6eO2J)Zb5ZM<`iF@HO51$MEU_^|KaDN-Tp$!Jw+sE*sawKo z#g;VRCEXKKKF3OHQh${&_hkz(Wk~;UhBB)Eeb9?H7MXY)eUBkqo;l8FIx=A*vIucT z!J~Vz&xvpE<~`Z@9X(ZNN#aYBgnn6oEbEqeOJ$Jp9NV>9CM^&-%3RR?>e+5N)xpFP zHM%P02C}DuIifN%-7=e1hB<(ZH&O$gb#zwhMp^y4KCw7-wOcl%vg7z~kz^$Izu;&+ z_<6ZoX06IF7fk%pXc_d5`XR)!cgw6)nIg)R$9#wRQ4el^*Ddpi$}s;k8C%6%Tf*~E z`atw{G4~zcSDW1L4|d-zBVV%8outt?q>3jR{*?Vy_3@!-EKjYp_Km@lQ!=1J8?COx&$Y(zoBiYx4N~! zTMZpr)*ZgRi1mRv?02Zm@tnMzwdBupiKjWx6S;Lk>gxT2iE}xGIF}{F*1XMIy`Qz~ zD~MtF2sHKpaX7#55x-w^9y2d9z+hYrNHMyI41Pi@2EowoZONEIo7n5u3)$gSQgEzsa9T+5INO)&+rDE5iZVi96lI2*HhHQKZg=8r z;^@tJZGq~>ao7$fVnf(QoXtt3;90{^@(O^D;M&gG!e~$Rr{1JFHTON1>F=UAo1Q7* zA;zw-(famxja}j65@&w+8B^&{i)2@*j0@I1;=sMYYfFf;DR@aB@@v=Da-u9MH)qILeZXEcJj*`a|}Q(e8Jr>AUls@7B8Cz25z<@k=1@P4~Ov zTe066%XcntzjGO5Lk>%wH`#uZcQt`(iFFpci%Vl%d~eZW!(O(lrjCIQs(L(lroQN zlrqQvPbu^NO{Gcfhf$hK3>u}Au+Nj$u@Cc`{PK=thtip`$I!WSzO~{H$quD`xLSXW z)wvSChaC#NBLzG^j@`-ZoAfxb3Q9SDf%ZJ=PxyG1V)bdyyZ5o;w%N!zDn9}|UZT1o zdw$}vHfyeHUP_*M2DotQIs?zJf9ArUG;q#u#Fr$C_woOFKcl^Ui{v=$W#-IS{8Tm< zBS*@fmAoKp^(VqDzI_Ydi2BoY=Uu&{=Un77e_7Tk`-*NfW!d8c-};^W8_;Q@e(Tt7 zR#r}$euAF*$!u^P2l*COqyXAn^ZA;{u`e<=bzDiMe-eKU@UWqb5O=w zPae-_+Ea9tF^af$gDd!-Xzto>&Yq%y4?A>f9PpXI%Ggh|82Mee*H+RiA3WY@$p@jx zGJ7Bn@W1djf9U9^38A*s@^bolJ(BB3GuWYh&BwI17#{S!wSSIwWJT-C0PtvC5m$rZ z$F{HYRa1=(s}|V3#$J|f-^#ss1^z(EUm5F)k;V`27uYMJL`8<%+A+7vneiqEuyVfB7KWaYj_!bagN&$cf0-20^Z5S$swa( z>5Ctnp)ZQn7tloY#b2Q(!mlFw!r)CBKDROV^SF0-I{2C8_^4dM`%xdq-@ETed4GiW zh2L7snMHqHKv}ibtj`M0xa)hL0{8n;;9=H#A@)z;8}fI2Lso@{#_j{Uf8^4ReL(mz z4yK*uQ9s5cTQpa)o&0c<>63NN9-tLtDn>EK(DfdQ`iMCD@dwafvg4jZ+Cp1YPx)}h zw)^z9o}_)zwm$d`Z9Po>DQ%?=`ri8g)YjX&I(uKR<1{2<8yOYjvw2!uucxgojJ@c$ z+WQfH#2xL;`yTB@c8O_k!@V&vnCSD*cs6b38+F_pdoJlc=<}*$v_GP@^6j+cM9f|h z>`w+yzB10pUK;03%&8}m=NUuQk?62$>>1;piup9enzeo~$e@<(A zcFZ`hif#99XPjrpwEKC}woE%@(lq<`-(bs)$`Ay_L#{9~#O|nXf7=Mo{lZR@MLRy~ zVr{PGTFLXzx$forqCv7D-2pyHHh2PGS z?0Yx1*}bX3ybMny&tpYK@UOo!o=&?m?Qiq`4xW+a-Zy1;rUpkRSP}fJBBT3zB1^j} z4(qpr-K^Da=5+JFhK{zBhv;R-L@x;o3)JUPppGC5XO2X4)GV`seY!q13X zGe-y4xwy5CZ>xU7tq-{uZvEAyN%l5U;g*xH_$ptITUp@Ne1ltA_FEm_&9n=^C&Syp zqhTGm<-|#?18;;|pMiH#+*(9D0~fb8j=4l@Ln@nX-@MGhtrIS8-R|xU_$~iBPXB;Q z{}pbHbaBgRSC;)W@BjaSTl#M{xW!p~2Dfr~_RzoWzAnVb>A{+PPkhdM;d|a2AM`%> zqW8szJ}Go^SDcB-YUFFzFHo|z=5v&O{o%X#&10+20sD;Lq3_?px31L&O3c}Qan__-pKn7OF*w@hp?l2fMl6-YP#g9+^>-RL z4qu?Z*z?c-9UMxtx4OQ=y?_bVRn{9C7x9Ddb@jqw;sW$_*1Mab`<(G<#^QGF9s7{> zOeY(i;A3DKC-H~ays?bKU);POot7uQ{3l?qu&$`FobKkI!?^wCW`_oc>C3f8+BZwS zHuTP=!JLPCMj9-9eH$96x~p!I9R>i~Wa`ifQ@vgUru=AG)hDF-9B<#!2~ROGGb`Qq zy{siFZyJ4+M|tgOIWE1b{JxReFEko`E^lbXD(zirgr?_Xf0nOiCG>lSC!?wO!lzde z-=K3Zxo`2vx^gR6pX>B@0yw7ntf$@69on$l-+!oldn5k6W7*f8Ky0nzwwd6i%3a`= ze}eMR&5dL3lK$qLl{@i4C(ez^JxuLVEf(ZK?x>rw?CyYQicAKMR|uWW}0~~UO5by=#*qE%|*e6 z05iU@%%21oHb~_-PXI(T>mj8|hq9)r)R#Rd%=|ZJ0Ok_6*{t*7|HHdS2C@c7jFoz= zon2P2OmUZz@ln4@?8JK8CID23-Wn>5+}n@Lk_*_xnZlSPw79~2Kku|2gWrdXgqAr?~r9`nTqVka1TRdSq(H~cX62aVO6Y8Sqy@<$S`Sd4Xl=3W2rS;6!F3mUHD<>M;+f6{f*N4yIh*m3O+fsocJ4zw`@eBr87-FF)?HE>%axk)ay(h zXO6gZvIlfCAGmyAYkiAG9c4Y!-B+=~`Z4;{2yDRf(VMlmqFgp$_Wx-)%55q5eVr-m>X^HGb#Ww?6b$CW+{1JWYq-*1wO4}X1DAg`j--BnZ z*aBqRnxMX*9>)IV#w582n6nQh$wuF#T(;?h9MjHBdqYPb$Uc^3s}IsmUWWaf*gWQl z_)8y?mu|0#&7<#@kk`%RB@yqyiQVwOJck$Rob*fggy*T>=r8s&nRY7<1$$D|H{l}s zkG^yErV#h7qyK0d?Os=Ot@cl1cVL|9i{FD^(sz!4<7#VH^#kz1%+asR{aSBuF+Oh| z!&ARHe-{$B2fOjG)939zzSo6uh|b@0_B*x?`ME^thx%RmR_wF-So^JFVW&T<%-`
    4vP5^88A5rQ08O-57 z{jAH=&zyMZ=Ps%GefyAr2l!aX$UDXq&_m_YpF*hd>3ZpBg6|}=T*84rjgH0={n1#W zKV2T-pg&DNjYsoOm*=XN`kB7MX-{~vy9x^zAbr&~g& z>u+7ZLkf?6ufH`O{jAH=?={_yOI zWF!4)^faH*pQexI0s7PZo#s0FL*NGfd!PD7j0gUsd652eJJB2>e%}{}@Ik@^AW&e=wx_soLRCSSF;wCmmNni4RC}PWg?hU-4Gj_?+>|C2wVQ zu|DW6?%UfbN%}GIU>iP<;Pa30`8#~L4Ck|Gu0!~^)NJ@%dXV}#k*Mp>j3goy-G}j$ z{#@M=c!tFbFq}a2CrCbom-Kgt|L2C~vE<6LhEwPj5WY~R3Fjzy{jKv!hD(R&n{aHv zhgyl!NfuC>ZNxA7f1}g#n=Nm}Zxt(E`Sf0IrCh32D{Dh;({HuwyZlvuq1`Iis_tU3 zT=7eNsqkByF7kS3YmHL#fvuHFn;%!o&6Q$nVcA#j7K+tcwY-4BmWnOkYgf-U%B_}P zl^kJeyKYG{7efD0%a1s9}uQK4&^VJL`<*$Htw;ymNNm89O>MyFd8N4IyXB_quhd&>Zmr?^s|&tgYPzju-#swsPLvP3es!T%@=I=|yi#sC zftp>eo+(z!CCb#UHrw@jtE7w>goz_=G%*lej}jr z$oPc2f9F1_p<8eG^+s*MZ#K)-CAVH|6frXVh81nYZyJggNvD{K{XUnbSa+TYA-@(Cw zR_&Bu)n(4NOG|#siI5)YA-X+toZTyGUlw$Ou|QghI2wi2Or(Nf@*<=A$NjA zyu0GB)EaABn>1P)?P{yM;=5aS?p<_`KrU=|rS8=C{nfhPKob~=#|PbFt5x+*d-Lw- zo&#ah0~mM#99Xshp^H{K<8N*DVRGW)-Jm!~`(~}tWB|cIGO6wtPX$kXI?u{Vv95Z- z*+#JrlH6t;P2%zdBSYDsu~kDIcZ7tk6j$B;YOUoi)aqrwv@O8vuR`oA#AnL>S$Cm` zF$%10H-WTWQ=Dt)L7L6f|stLkxy#s=3f8*IU%@ zxT#AXbdZW7P~L=8R57=qJD4EqRREJ-p!B5w?3yIKLehNh3TFIMF8yaVm}rFoTUbkM zV|tO!K;nhl$6bLW;L+(oqPL2MM4O$PDdZ={CXaZ9lhcKnId44wNMXjCoy*V7YB7*# zRbev3!2o6r5(=$it2q?Z!KG={Z&2swA6~OkqfXscjZeS2>E^M%&Aqd8Hw>kDr+RrK zh4Hz3*R^AoS!<_z?%!8+l;~!0InbzSISIx$%S{M!ND&f=!89q4ix^t-m<3cXpgt1z z2}txhbaIlqDjiZ9%6|ED8-1l6w4u!Xq^Xpkz^%Cr9{^37kx(K}Ed9-9X&~qt+l)Rn zIpxh1X2(Vflf#cVZhxuVB*nF~O>?|N4GVGK+@Yq$!C-cxcZr11dge7e$5>iINgTU-daOF4eFpR>2iWv zLj;m48qBh8|6--sTy|+W!Y_9~S1qYut(Cg^HDA%Gr_QKX3$+ytgC?eN^`ZeC7Q>LH zm>q7pTEenXKdsb6!`R`T9vmR;XrQqyQ#Kf_w1^#Uqj(lgS!)Qao~R$r&rCux3}OCm zS4u81r~x(?n?Am|W3=43Rzj~AifDzi%UG~ttpi?@LNhx%5~?@%v3k6`HV z%-3qCPGMa^>`*H>b7jBBAn#~?)|)F#&Q8rpPZZRHuF=I*hjtgy3+7m-bHyTqo*9b4 zqr+4usI|ts$*GwMZ)Wy5guT*vjl^RQc{BNky-`?kLM*HbV}97{jZYmh6DIO=N4?qn zXu%}2xm+xv|5eKKjbdXB{Tu5~D-F^rS=y2{>p->q3`2N9dG?2o=0-TC@ zrAB$Ng|16u2P0;w=`Jr|l|W8Vkx`yTYKy4Vkpv~GQxiyP4+JQD9i<<$j4-D+ zfu#0YZic|8ay5gbVwp{#!Fr%1UP2}-%nT@R9u}0Y=n7!hNnDt}Qzt1k&tO-KRDBQH z2^5zlSRZx6)#{{DU3T@KpZN{X^!G)y3_d;kYj!bI7 z$VKpzS5uuh8HekIxZ(8D<@nNWM;#%$KX) zBFx!Dg#K7m0UMZw9z&Rl6jnmjIAKErm~rL$Uju!GG z-ps_T+QT24E{xN5J#A`@4QrbnjT3QPBgd=N)imRs70yzxv=MF3Z>^9T%E=t;)_{&; z>A>m0*F~c)(JqfW6ROplUg!<;TtC?cu63|k<*s7~ZLe25<2NeBdf5A8%Ps69VArPH zz<&S_nUz?pC1-$SKSVc7v)hHOtbx|WrXX{7d|~hodS{F(4Y_{O0u0^Jrw6Xi*46OP z!0VziUSindvK=HsAUZsF_xk|5caoIhkrj5)becQo>z(`zd|j|yVd%13*r3-<12y z0!!!V1a2YOwv_9Lq7$jcY_B#CEd(ZkEM#S?%BS`AWDf*!Z|o=)AT=4WhFm{s0ft`b z>t^DQkQrvUPHGKAJrJ$6rR=8Gkn1Niz|aMKOYnKB8?71EE}hc!JiXK00atI->q>V; zwg$c~dIKcg5_(`ge70v=C*di`daQu1cm}IgVH*1SNDi=cOY6b)Fx#GKos_2_>#+j5 z;u*|Vg=y&PBRRm*Ev&EijEVM7kZ;DC@(d$YAFjpV5f}oAOad^Ei_1R(~x5gHmA!t zh^0f=H1kBZE_wj_!S*<1r01~^n$XqZ@hz*B0mdWdPn>CLui!v}I^5b2pJ>y)Ik}6d zmI@uH>8<`jLv>S>23}8%OE_2n$MPRWjRP{esH#{DdXlTDS;FOX!UB}ay>w)OaXW!5 zI<7(ty58x6H3@rtH0u(>22UfF_q4j&_=*69i0QOIs$+K&dsr`v4LJ$!;p}mxxZv~b zYT;i-j&+k6DffD4hJz1IWt7zZbj(aBX#=zu=+4xp3$FAa(|PSaw=L(xasG_q=uD21 zr&-v7VP-VG6<@@VpTSbExVVTj#nKir3M6TuUTc>5@Br6|X5)~91}TQH?HW&#K~e9% zW^J)WXT0jQS|zQ!m@vI`kVw9Ox0Y^9fH-KenxbZ#$z7Ho$Wa)3BC?uz__+j5TXFB(3;3i%f0O4Pi;(9?7#H=tLwWcA!G0^_v)o zg&<={MeNye>>Zns1wgA-KZQQrJY{LTD1e0wo4yyz)QJ~SP!CQ{2;0)gBt!$Qi=ZHL zCSlrJ_N-rCTE_87@>NWu%?8^GdDi@y6w;BgI?ic&m^n@@*BaF{@)Cf((3Z^HJz?GO zF2HXBuZzG4g7p!}-exu@Gw!QaQYFiE6Rgm5L$+DZ4oz-?^+;YMYp*0>!>8WD;cO4b z16p0u7s+!S#76OCQVw$gLf&|2SE+H+uD=PGdvw^MWg|ESWu4t) z5N@-)v{J*d<6zAaXzOV+MH&Vx%aW_7WOf|czn;>OeZ56P3#(V!&Fd*%qgXEqryE!t zC62&$&3npjEn2NKBiy!(s7SiIP$}Y|{W{4}xvkW6(GB3I?60?IXq@Ed?Auw9UHBqm zyj(}woZVXQXe-d+FMZM}qI{`VkaTx_)R>t&(W0j#7_D$F-_j+b02a2S>TVE*g4Ny* ziW!Vwt1Zn1L|K6Ej_;Mq4aAzzmPZB(13P$feh6@B0OW8;A%SpWrV9`(Iw(?{x3siU zNdxCFBOa}Wc)8f9mD&p&b*MuTl82U-ik8%Z0=y<{gav^|GZ~QZb>M<#;i!5%=p#ymQ0*Qn7CONar|eFhv3BUQBL3?7Uo?S*KE zAg+=`_nHul7M4k55D(@FL@B}1zIcbSq(ensDMg5j1F4yvdvO{t$D~1Qij}bJZn8BI zUj&^F0q($3Z=Qla*hn!@>ExvftrjT`il>kmb)`i3+*fLiV&L0uW<2H1dx$*vIVzh@ z{hMhr5@~26Xc@4WY2?Vyhb}q+NtzY_r;Pmj_ra*f(OTF#?Oj?lIn2Xz^Iis z@1Y-r)w1k+t70$3WVRypGblW7lGq%FFsLaeOb8c`}1N3l?BH?#Utr=0dK z6#55>N0}VxRLTIcCJr#jrb!ckPH{x`+r#~3MnLJjyJY|+i%ErI6J$(4(WYP=WrD6W zAR9s|owhlxNH~Ee{zPg(bZTCvG9AJG4Bkwxh9nDmP z&_j4tiqc9^S;Hq49#_&-?v7*O&*L9d>Q(smt9XfE95F0Wo5-3lwA)YU}jZhRj(GLC600lvpb9W$|7X*=) zf|Y`sVt69XA2wDIQI9Z)jUj>Ap=DP9KZq% zUCtcgZa+u1skW(p6jvPe)v+0MF%n%}r@BRqmpTSbq=nsn3Db(Ot4!L~b_Ny-Rv)QE zfB+fIg>effBN37hs{|#wG+7d}lUi{Rs)Xbby zM2Kp{wXC-52FUS2FF!Xo=^ghDd!rEP!};N(2naqigwJGDU_;$l%UCyKf7i|b)@wpj9x;N^yLgj zW1lLW*9ipK6onxP3X>Fe)ywYp@lv{M6L8SuKF!up$H0;t2$E=FahFY$nm?N%3A!Rp zq@@PFF+?SdFH}TKR7wHhV7Y4AwT)59LYERyrkJESa^%D)!q3(TNkpKf7mJ|KpR3`j z5IxnaG7(_9QEM_`N-g7i5F$urYNA;Q3k4kJgWf?Gn!xz%;u?~oJBqoV=MWSXb)XLI*>_Yc4gfXpn-trv8Y0-17vd=*34<8U zVeyK*FetyN{=5BKo7=QU9Me@&wXjL`M3nVHerEg;LrOa-KWTa zvbPntxP-IG?wvb>IOIy9<10a!vtf3`>4nz8{lri)5*ef^{4n3VkEnWKfO??hw-4@ zsDS?*9@@*Fn42<`A3bgN02;$+wtwI|7d)dTU|1xPW%hWzJ!dozaF$ZCMaVk~>8 zdaH$nXq{mkaCDH!5DgJV>7bhhsC&hIy0wlky(%qt&t!;9R90zhp=;Mzr-3gjVAeEVr~UAtN!oQt95YYHTwq3WqhwB3}aF z!xYj1A3`&psv*E7uH@SMsblZs?h5rY4~V+!3}UAW3_EI4S5X)ZNhntVK!4ZeqszU zi$)5=QzHd0KXYUve-b*wv>6Ouk3|cRp+*q{G4m5`WIgHJp<=)WMNoQmJbb;>Iek#g zLmotnTrp8rH^xDCQddyaPVtY9m^`eU<_eLJ_N-%TS%${Mlm`1q6PFkW-b4XnlcD=e zIIG+wB%9l83c*xJwrqnj3`bkaMcjRW<%EQLxAKQCqoXbLYPbyT0#mjYWT636MpPj( z24gB+OkOjV;ZZoszvEGF%ioU8Nys8AtXLqSmn}>9yoY+&8 zD@^D;P6g(-cL&{dPcR^$PVn#0W}+EPR$^B01Uns#)NDo)^-$`=;k<3s^f)eP z7@nE{yJvAbgg1I(wh)oYrwKQ0OEcYAAmOT|;JOg;DsjNtV=fTM(XH)dMM0(bJ`vul zjv&-~>U?G+u>GB;O>#-1d&@YH4LiHq9wSB10BYMNT8@8B(dJvd2|QR+CU|QWQPa z5L1#jAl|Z4%aB&xV$QBccbLptVWcT_`Z~m|1xYro(M&Xq%r_G9&tGlnF3uwoJ}Tr) z2FIe6dR_Ss@D5IO&)z`e$$&f%>>hOlWNIe!C!^f4Qxl<_?o9d@JgU!xOK}(vN3}l^>q>;iyF{ZrdEYw709oy;Y z+M`GVAb_{(4p#9AFJodGr720BfwV*c;E6c0)P(L(B=tj*Nj$+3QrxsOIh?bzFc*&$ zz-VF4bQyR>q8qc~C~YXog+c3Ag)B(2J5Dqck4=M?aBN25(9~wXhs91V1hLBUV96AK z#q)yA-R2KKO&mCT3Ht$pi}KAwRw8VIIhxHx2ucYXJ3T_V;2DxMP}_>ohS_M^G*n_n z!YyB!hI?~thhmXwD>)JLGzLOy%?e3r&}DmOl2H{4l5FY|nM|@^L)d;j6h-rA6Jx@g zbjZYhdKZT5^^jvj(|tQza;MNp?u~L1mnDg{TLz=fXh5TcIEwDcH1`@w4hAm5^qVgC zir#0CUZz<&J~P}K>-(HYCKDSU=(kg!uAzxPu3RE##y)f&HFG?`#}+8JHt5|++8A$6L9OG%ePH~sj;q0ZmU{me=Jf0 zxVf49KV50|z9d&^&e8IW!iXeTy0-?RHDD0Z@w&40*5erQGBh2#n&X zjRSqS0hcZqk2j`bl8Hj?49k8I;U$L9CPdaK?AKM?23rK`3;(2*=6%t&*A0K)534C~ zd1uF#xKCMVmAjShBH?A*AO!j;qQ#WxPEKmg@ETcHE(O&Juxb`*t(?)_)Oyk zaW@(RXNzm@JjCEK*`!rS8g}}@QA=u&2w=eGAcr{?M`j-ybVsKT(v3{I_fQ2Au2u=v zjD%k@ZvwYA*B{T%^@LaFJF0F)+753)6FIyA8*!{|Cn@)7shRs7$c4&^YCt zW!ytW3#7!%4t~%~wTpPug=UZh=Q+uW7btg;Gh=sXqtFSgs!F_V+JR6TH@ME>?vC!z zbXzX7R&$7(0z#*W-KI!!o%v`i(uBG^!ku)XqZ>_;K+y%uddS|Y)v&~^uDNi`D~akac%H!n3F+pw|ex)u`T zK4ZEE*I!*=UpHKAjC>smTH5 z6@?`~Br2IB&a&$^xuDYtz36S)x;jv9d#DPa0~*D}lC)|T7IFFp*Fzs;(iMc4;HhSX zuAT2-^a|QIC95o4n|!G_7)0us#rcu9@WR5#&J{JQ(B=u7o2#_v%x7TJ%+0Bw343!2 zq->Lt#aU6r+aSUNbMj@~CV&a`n`XOCoA1!#T5EM|&B6)BU``}g2_D&1>MOz^imSl4 zxnr?VUV@UJp0md$iIm!1n_#4R6NS0_xW4k79AL16G(4W4olV>c#QLUe&@Dpi1n4o% z8jwm2n#Kclz(LxZ*H1RK!4WzbY1;p=yLuSq+K_Y6la%2^c4%W0|GZPyjBAydfx^bJ%0vhG@NOXvT^mluSQawc&2N_5KG+IT@u_x$4 z_`?p)^HPw?+GRnq8Z0GB>Y~{Uay1qyG;F<&9m$eE%qb2@S_P&64wx{>zQgBE#Zbd? zL%Fh*Jo1-95DJ=ds6Too+C5W*87GA$npbP3t~yK_>C*9{U6XXO@&Zg~2ozy(%myk4 z(NaK-no9v;y%`q_VpTX|WsjUpL>(I-IkzcKv6=kCbfW2KVFrgTNYzrjqCOrYRi(t= z8idWBEgu;_5l#WNQ)U8ViV>xdV=tt0&JfO?dtbNN_i0^=`;919P)Xp((`@;$Csvn& z)A2O58x=HQp|uNa=TdDsOpGM%$nMc;uh6D)#`DQ^(@jT8?Fy4qGZV(Sn^_jc25oV& z5fH=J-80z`DujwGQg9+uKaroFKY2FMXaif;o`#PF!D$?hE47x%2u}74xB80rjO&c4`%Xkucf=L1 zg2>*6d#oFt#E1^_a_zeimPB5di`f(-Xkh=&eE|rq%gz3q)&G*2<%~%{nbLi^wmD|3<8NS#SS9m71b;-)otkULgkk}`6ffR7Hs=+xtcf%3EuGW-jV%uJnt zF_{=0MI0{{6X_DB&?rz=>0k|oE|EwyvU`HvT-uAF0<9?H<%ZK#$9fVziW3q}c5y`7 zn4x5AIyR9FQSc08qDA(GQZ!0t1R2-)hwkhABHFKlPR<6x&W%pY(5PhMQs)TFgoL4y z5>KCUvFJhnhc06?40M_Y?0K`sV5G=w`JbHiK(=@M*!a}L-s};>h!(ycDI7j=L>e#* zu7kA+1xbS^Rcm<&LB{qqRy`}r__~^#LH9X z9J;MCm7=oDm$7To#i&fqV0sCU4V=P^cX%v6n^?Z7GDInf(=5zc2j!8B-O?Y<3mu8# z#F9HvE9m{oj1!;%Db#V*pG=G*^K2^R)#JbB-Y~$!UGx% z14zR615|`f3L=7Chrkz2pT6Z2;qk^ZiVIXuIW_Edaa=l9={SZC=Y#boBT<~7bar}t zY%Wfzak5)Zu*fnbbeiCEzU&M|8j?C_`@rtit*ba)nG5ep{rV_ z_+wa&d5nlQxtoj)>Z7tKoR)C6cnp#E$=lNyg4Lq=f zCaP+*51Vu{jSbOC=)@U0LBg~{yNwiD(uMnKYZ)&_5L0Q^_fNU-!7?rglKo(XM21G^ zIm3_yDwjye9`h=5zCvLWB7r54`XP-#M2j+(DV!70jcXjl7pCP3$-ah<>jMy7qJ_y5 zEf$OVBYCu}C9!KO*e1Q=wZ`_Z92rSNqQsMO8<4t0tb&S`Y}wCMlzu8IgS39T8Y)vp zBg(`eJHn)CH~ADI$r|aBk)RDEAs(I?n=8=1@5D4&#fRsNaj)f{a*! zZ{o!G+}Jb%K!wsGE7$1QNt6Ve3d*JPPV3A%dt$=$eac0$#+*^nv%ZHFdznO*N{U zN(Uc+5P_qgPo7!`Cav`jg`P~tf|;;xN9F^j(ZHPbL*H~Zr*YV;>B7vMttqev-4oE^ zsS}eU`5BX21_}vOM&Y&0vSRcc&F}+1ZTHG7~LR)m;}`pYK59K za*eVCup@6|N7I`fpGqA|yyw{oVzw||7@iBpWTM$MMI`FFg_YT&TWH42af75Og=xmL z%aJWOU`3$7Cp32}xSEOERoXEq)i5KHZUnz2EWQny7U<+yp|cJicAuaQIs!b6-FG44U+&(c*ICVz&yp;;!6z5m}DG@ zgq6@kWqN>j4;#r2xEae?gZ*J1TdT4Tbeb<3_T`#kDq6p7Blapnzp@jIekTf(eyd{* zS_*`#XXmI();u;MI~1}P4x2rff=r2Gg210fSI5xVMH73t_EdKJKnp0M%U1_*N(E~myP@z`*d>g@sA9vYz4rXvo}4#k}#JAwntSsAx6)EWb7!_Rm_%UI(PQ9M(6 zMlf`{U%eabkLONs$6!h*E+XT1DVKnIjaPuAn#K$gnkg<89U zZ4g>(fnVgtq)HAkz~ap)Ugd@J`Z{vXTSU8e^fY25){A?gPz zWq@yBRHGH9GeIvz-F=Qn2P`$Pm43rF~yozLrZ^zcy7(@w2oe z`cI{BY!r#|N&%#WF}&aoUs8A0YhlE~t%eUwofpP~m1pW`6gn1xQDlSH!#-R+@=M#9A#MIy{DT}i@QV9N1ZWDmU| zeDp0&QVc^E+t3fO3j~X>ka_K0=yC`s!aHQE^t%Zrkav(M?)YtL(bmqBm@){;ZmWR^ z#ehO^No969TV6tp8Ir8@`hfK<-BE#VkE3Om zO?8WWBnJjX#F|75#=S-*+_+O&^9FZ+;=tJ{f(ZF05sCa%PA7mw8cjG+ zF7g<*RSNv#St|-|Vx$;+6`xy&Sz``XLo8ZMQ-lQrpl$Rs)dfw{RMHw1DmHd=XAtgL zeQ0VIJ4om3ny{o2SlpghHz(uO_ILaD`3}rv{OwrNj z&|t&eK~8AMQFk;tbPucITxh>B=~Rhg(#p2Y#HRc(TTgbSR8vlfuS`J$OJSM|Be_FBF88#k(LlxM`rTVN4=2~6Vnnn?WB>rW1aM3 zVQkzRhHb~l)|v8nKA!AjvxH&}AFtM%zIw`-m|!Dna*9w!VGO;P$Ty{UxI~oWLD}p2 zuHfdzCLgg`nK0c=6jv?0p@x~5!0O%1CBq@f3pNlPbW!uMk%$qA^n*@}5D7uz$if5N z17oDj<qiYU?g3T%&}7kE{F?`|)+8PEXa zi(KgJzrM%CqaYvNf^nf*CumF20mcr9X!c%RZR)^fNi0a0+h`EhESVZ)2i*ja)!yN_ zVV0rA(Mi(NO zg>gUWrCv$Iy7}Tv>_AIO7E>Px^%i|H$K37mMdvq`O0smIC1IemXdNuF`_m&{Ubbo_ zNUL0U=H)96A<`nE)Fy9!wGibn2G>(!xMSV$EjOqjE-)5Ivgl2g1!In|NhyaCU0Tt< zu*NY@)E;FJY*Y~|`Bu*GY7mQ1UXjDqlGa3Ig9;Z@SOae7g4fLy*bsE=1A zT&?KZWW}D~tl)yi=<oSDSkPSB(5m<}-Q0moTH2bUmLYsm;N5%wB zyvy3IrzMT-NC$ao6LG~Sky@g_dl} zFMwnHYf0#yU9KV8H+OWhLT2IUjwePJwa~GSMzvt+sd*#pV`QCKyo(RQ{16x>h<*rM zQ;0&}|G={{v^a#8-;NC+sCwdG7x)6h|5i+f;#f+iXjrMsb)Ow$)XR=WR~ZHvUK|*BKAc(V2CxQG3oFi`aYegQ+7Pt^)X(G zzAp9)3WI~)G(xP(8Q$Q=sEJoUn4_Q<8JAIX3XsFDlArvz0uJdl(u_) zjS%9wu7(KF^w=0-#-7Ip?P4Z1U%B#NfR{tyBqlBLrs@6+wb9;oR5m(;foo1al3g6P zjZ3I0+>;YSt%kU$83qNo@aQ`05W&!qbgRa2{2(a}4;6;B8H?2fR3TO&522OtE&~c7 zt*Abc+9r!VZbTQ8fNFuYQc5R7Wmvuhd5WWDCc+ncfKHSj9!1D=%Qd1}+6B!Q1=O{n zv!wgv%0{|_)tg_GBhMDxGFT(!#zSP;3k6i8yc#W$ZL?>%I1cipI+u{647WrJCtAvM zCib}}DsY8zxL(J}m5d7EC%&vEczQ^uCCdm2pxbH4zdx)uT}31HlZxxQX&|~Gm`=ar z&<|D~uy)~|TJy{Z*eW1&A_IG)ML6e37Fbf?^~P@^Bq?yxy}20_IB95mqTExEgu-Kk zUb|S)*ux`#W|@R!uLfZOVy5ECwHBiqu7+!rN-dJR3E8bs54um2ag| z_s$0GQNn=^+?IH1P&c+XXoy!vf)a7~)@TV6?0LFr6R4=)1g#qL)9#zuomiHeG2BYEwQ9!P23 zmjdxbK*ymkRL|5ZZ2xlT&9THA#FU7^6hXxipz({QZf{9f2+JN8L=g8MX z0+tEgp#?XH4J?Mhz;RTWMd(Vqem35-DFHWv^|z~4mgUB?NwtYSMrTv`3-E0C6Ex7p7)$XC`g~RK|+MTCD}26_^!u%reb&L*q2&Zn`12g*ffl!a+5(FCL<& zS1QWCjZ7_G@eCp_$l)=!iL2ZrFBWR`wa5$di;28o9IaZbh>aB!OY{}viN0b?(O0bT zOJMhaLUCGyBN)}EsfylS! zHh?OX2p)}FHl`XrMfWhuw|NnJ;u+)ms9*y7LNgdl&m~8ldevyPd^i z1xEuSYMRLGc|@A>aS^#+h5W2lDPX!Q@fY|2?8k63LkpbX6RT*!CX_kcpaRF@DqV^U z(@MK~stRo|a>K1TDJV1Z)~32C;|Sc0XYvyfOF;mGD{Ic~gxDurh9RG|TMqkX4>~wf znBVgWC%<>MSr9IgmD8U2=`pgy&g0Y@99sO6z8JjXFBP$`E%lc9I65;$FzDMedAA5P zADGzB22qr(Oy|h_5uE9-v=M#h{uLj62Ybh*N_qalzW95g%zV#OO#2P&M`Is)!Jlr> z^&jOkesfoP($3`v`>?3T>_gKqE=H!v&{Xh!$=CJz(2x}C)#}ar`lQ0{nHmn`qaS!G zjTzP7=c+$>d%tQB^7z={qPnYi-)i2EIT-g1wsyFCAAHcwyN^C9>RJEpZJ+tfj`iR; z9OBp$!J!DiCF$_`|JZN{wl@OS21Pg&xr91B`lzwb;bFxBZq}*@!U?bGa_KXl+2;NP zQ%G0OyRE$YfScdRv9-1MD!s4lA$_W_5?VTYaVNA%xO}s zTe{ldj!p$Fm|**xhj+NmJZ|fmGjX)ZA}85>b|_G^4{dcl*gtHCQ#{6ZOyY!c6cpy|###c9u_EkX66>imcHo(*X&m@4sbwPQMmT$hiLqH5 z)#!*xx^W3Rj`-r*1QS1YGAJxj0{ifEtcnZuWCe$(CgC0%MMwwOi%EP(3Uhp3KMG7@ z3;X!VzK%ow7H~Lzq<~M2&e46zk*YQ20*ge|W(w1VIMrBSLxD+v(h2_vP8_mH0P&u2 z1d1#M39!SsH7x>#)ldQmr5APT#IwZm~|ML*R|G^-v%Fbx?B_Oa7u8+Odx!;**>< z|NB&dJ_6#(i%FG7G@*}_(}_wbPjM|tq3-(#>P4|h(DC{)-LGwP`}R;l#UDb{WGY-9Zzpg~a+-t?F^>h9aQXK>H{PwYLgf8YLtpZNGE_U+wwaPYul zAN9`}3<&s)9}S6HEgfr)R;|6T?6nV)UnvlWsLO!QWPRd1woG*L~S?ldi*kuz8*@G*O6X-Q|`P;nmhbujQU@KFUL%Xt3lwrgF-H$&%GOhMd$?U@R0t6p9I# za6(}0!YW1Kjm!_2O+Nq~Y+pd|zstg+w5DG{dj7_^O1PSiH8Jk2n1>C$Cz&K=N#iaydI}b$%$OsJ(eLKu*BF}H<3Z6;+~#K z6~`Abu@L_wcAFJ$!Xc+(h!zZey5d^R`2B3Sr>itM8Q)F+G*hf6ZgiBYhpUdNo9m6Zl3BpBiv(3C9&l10**Z&h;&cCacD z%9kY24GDOJFQU+-xJfWwVCjJX?xqA^0k2~DniV*nMT;x09%gg-xL;ig?{#RdpgCEb zV7wUpAFYTUr7IZjanVcN(qv6W&xce9$J)fyNMW1<19`M$qFpXS+)87q@l_|5(cKaq zXn^B=u${Jp`goioEs~@!3VyoNxrw^KVPCa@gsR|np*<(Lh|i!abPS_F5#0OGn+$4U z0}Oe}#|k6F3xQ2?MOoCQ zbLY;Tj^Y%qMlL}w4;)=$+OJVhvyv3I6#>1x33F4{25`*MpB3#oRI`$gR zO>$1*h`J-{`@&!l=(sU!7zPKzRCTm`a!La0h8-;dq5}maL3N+Z z%wh~Yse)B(s~*4*QAtK)O^JHy3Oje|v8X4D1(Js7VDoexI`<^ydus9J7Fqwc%l&=hf|09HcV=a?I$&>oY+qh5~o3J>Ed5mwLkv67%bW^`b)V1)?7(y?MOij*B7O+UO zsP>8CMXA?Wl#o`FOQ68J2ox+2=-lz?H;wU8VvO#U%KTW41g)y4OBh1(EO0PXVVNNJ zv_=`G6i06~8n4uZZ$c%)RStU6mh*mqx!Mtr)!o{B{ijV1P@>orKS6h@Ty4q`cN@S5;X`#tT9i66op)lKr>|3?86l_rQV~t{-%wmi#|h+PF$| zL|PV&mBoWrV{OPC7(h5RT%WGQ)BtmQKzXOJ<>QFVZ&W?xz~YYAEHLDB&N_MIFdB*C z^>>tE)D1}WG~{MLjGpqSOC|L|@R(L%;I>wV)uLS2MQ2B4aE&+YGivb`lj7(yCZ=$R zC8UImC~pE*s$*SATUDTR1JzkN$<{=P&^YrWp4_9*fiMA^J4(FW?(?rOM6aFhqeGCS-#WQwN))lqllgF+ysSmS15p7pmh=zjJx4wbA}xr{2fRds6v+^!KkBd(2S}Bo6=$L zC=ytxFu18PXa~nhD@(xU$;%j9^j%PgxRAU*s!r*`OH5*nl6NQt0%gIb1-cC7l(NJ5 z0Cp`kIRS-4Dm6RK$n11}<`{y4Q-~^2gH0=S$3nuudu(+iOK^p)C)CZo~kw=+6Y*aMeqtqQi;P)qa_)gjPe zTVcuw_vLnDy%{|WpMpTfd0?4S6IEP3YmRE)=XI5mr(fdqv=kxVEaRBN7&i4N`|NiNMVH{pAN3`RRSr^9ThYp(Xtyn?1@TwN6+(c z+AS8QJ%3_u%A&b-0Hy=%^p8frcbcM`l2f}lyqgl?_@HiD*Gtc6Cib?iq3%40!Wm{gBKXG-ZuQRE4oG~Ry{I3nFa z{kQj{D8^p;QzYjPp(p4i|Iw*A_>)W*#<3u!aLc$QCfM3khfEO=i^CB%5sav%w>@>5 z95MB*X)51fJdeDOzg3tL(DL5o=Mr{7aP>Fbdsd+aK@Rg)C0}z(;Wie0Y!jBC00a?j z)EJ4#AUU*#p%EZb2xt(&>0r;^g98Z9fdK1JE&_W!@Aj&aaf$+)I5_Bqu%zZ@a4eZ` zBR3D}+Hf!s)Ja>FOo1=j(g@jQc)xslYBn06Qvk3@k`9B-cXjoVvGq*bVRWi1loh&% z%$}H@Cf7+@w#Z6`?%vtsLhRB;SjT2Cxh3$}9Io$ytp(efdlI}#fN+0w%<|0c!46w$ zUsK%9t<;3>##)9(`iBjRZU5FZ;rHPbJDkPT%>@(F(=hP2FjC}oND@xOM_;uo-pU#B zf0x&j`ICAlQ6W$q<59Q5t}x7{B#|Z~B;In81!2fB@`>Mti;l6pZtuWA;H~@UT3rQg zmQD90u3(I1u+xgB3x482zYE{EX0yCN;d8(y9O@64WAJUzO>)hEly)%L$o!`1AK_FX zks5ov;8d~_ZfFlm-<^>j zynzJt zqq9zM7TK$=))R-4CESQs{$*Aqi#Wxsver9cKotl2fTPs`OO4RC+2cD&1B- zI$0+c8OgTNjbvNlM)H2@uJa3JM770N4PP>&!AlXw9h~YyFBc<)*89|)WSH>pgF&u> zV^d4;(ZXH?9Fo=1M(JIqbpcy37_qZ-{Tl}_pPaGMr!-m25jNfv+d8n`NX|gM-y&0R zGG}H2w;)htZZf5Q$^xTq)Z!B~!&8OPQQ?`g=%W5=?V_FHW@^&~SeWGqpwfP1HAAOu z0$1XuQ4}N#g27LE$GyYks4+P^M*GyG=oQ#n_U5LJ;SN1h_dx4V*an!V)DD2!If9iI zu7eO%xZV^2L{5A2p-s*OIydlBfm^xuk44-^A3mjiAlgFT9jIJ`ZW0URo(e4!7-ti3 zCD>xg+gswVq{D%#T|`;8a7tw(F|F%J@_G@xV(T%nDZ&7*HkKOjbVOs6DZQD%X6Fap!h3xjrHjg;=!X^M`HLy^E@^SfOTeX%Tvh0y8{hlKdtRai(O_6lLqU^uW@Dlhm=6Ac}+L_@(J-pym+VJROXBM(Cy zaK@Ep72dkWQq!gd1X2nJ=-Lmhla%P3UWi|oq#hh&SSv7RQ#(-PzHPnYGlb#{3<6Vw zxaoWbS1WqFrPPr;1y7gA>g!>NJ4WHOv4z+zbUPJau$D;URW;tidBB!coousk34+{1 zBvXTWwX0AVIWdfCOlpe48q5`Dsa9jNNbHbrb{O>PFLB~d=%jWcnBHjgvSbov`+Lg{N6z7Di~*ubp5q9nbXL~qxPBwDC`v3hpx2T@OR9r~3e_|LF+?(! z`(mQgrSkAk7A>cu#S8)2hNJKcT#BMqsljIXomkCQ8E8@3m!AV)sQpo@Ee7-F9I871 z2)VAE*0kzCj6cjO1FcFr!{yVP$3$lVvz5ws3i0hjMzMOVnfR=L;YCM5V;FGEIkl~; zRa2Stu*-n1CLCDMaftebvNR={zp1@C}pA3v-$tM=hdt{QKC+&KM-%*2% zi99+kdmdHks5%a;tST$21`Y?RLrRTxUDb5V)+fH~!mK?hecsA`;t5|XNbIlna zG3BZ2&#A7c34KwwkuH#sL1?{%7~-MLcW*%BWb=a26VVyXywHP11VJ!FtY z2}PYTG8yBlyHdjO)g63R0R~kmJr(5)=;dR`R&iyR?AWkDkEIJP+b$LmxEqri$Y@hz zL&PO;`2#fL4YVQDr@)9_UrJwb6@UpWAr`#o1 z<{?YOdS+|XPDLwL5a5nd$eKI?fU^3 z=Y;tdff5WUZ3fgKE(czEv>c}WDZZYTHdggyYm`YF3l!CP5sWRJ+P`P7S2~0Lu|7iR zMj(JWy5t0cmC=PyFbW|!fI*6Gi`faBE)HfTS#la+X%u@n(@eJ%8Z3{oSc`g5WXaQr z?tlsvT3XhT2tVbou_8u6?f3^oBE3J0HGSg z%$q1oOrXm%0+wiY_z)-z1tdzkmxgeV;mm8u1H6dGJf>q1xQDn_gC%d(}6hhT`!oeWfsw zdZlzQbjco|&E8|@NQim;Ts3f5nQJsl&}W$=1CTkfPHmgIOg{&6=+9S9IW(@T2>pwp zZLeT_FiX@0Y;T7nnPrcnj9XOwme>n-U5oreci*I;ET!u2;KN(hY8Y&iB+9N(UL`}_jL3-xbXKR0Wj913U#kriZ* zeL)Kfg@?msULU5Qd$Mr>*5p-~`e}AU7aKqygH^1j4hzvtMheEGyD85mB%5U}qF*oz z)+3J0sdV(nA#=1KTzZ5Y8Zt(AqFVNXJvji>^q{P6Gx>)-9um9`qk+JKX)%aw=Y8I9 z)PlKRw`HJ?{Ltu$OjpA=4~}iE8M2+)R4{{hGG!95F<0v;9TrLBRAa1`NyZR3BNK)& zFOpEbU{pI}&?PYg+X?JYZ6!Han>N~m#5iJ$;*1tJq-Go5IE#zWkamflg8FqRjr*A3 zbw~3!F$6UQ$NVXB2H#MIsgrI(^P!2*_Y&g3@-GS~KtF;Ni$SW^)J!USni#S7QI9w? z+NCM6#3&<-veu=(5zn7BJt7ohPlJ$oOPN>)#GY1E2`80a1q#7pD+tnpTppr8n!-V; zUh&s7<=8nReF8DZppf8yCrLRE_gFDn)7PUN`fB=wAMslw{MwIDCW~$yBhzx3f-i)j zTWskLjzFo$u5P2)fkLeRvvSMe#9i*#X+nG`Z?-UwEhlJ^@K7Dcv1p+r+|ow3tOf@@ zTzxiT#cxokAPQETmbDR~2WxFDtx*AuD3Dg!vfp^FN_(_nE(ak~){8Wg+BDb)9uy}|1 zwK!mnmwqF zVcs%Pa?mGKC>#(#gU!EL8YUF?rjX~@6?{@f^>4juZkn}T(WLW$d|ty7wfd~|A72@qh};~t4#XF8eC^iFBq+jN4ZOqo|>_hcJz1A2t4 zgT@YM!eW&Y6Ye8XfgVNzHk06Inv%?WO{vK|W;lD;%DRPTuq%vXMl#!L^J@}JEtxbl zg0;fzHwp0=Ik}{fTAGrs|!A3AP|`O4^Xngm6uE>^^5 zXkFGQWP=0u#;inGdj@gm$T=XsgQu~p3Qn9HogPFutb#t!fryHGoP3?`9h=x=P=!5i zC5Stp__77;3L67h}VW9?8iG zCZt5OtA{L!>6qMt=xlQ44B0iHqS!uRCrMV|QGLLeL^&Ky#!RX<$C@r3h7Bf5Q=^=Q zL=4pdc)rLSXKF{oUB3+BHk}tHnn~wv$&p~QglVqcNMCr9u~sWjc@_h zZs6&psivX}_=Zm={p=j+DBWe;@e-^UsCd#q6Z&UDtu}}=JQIhMenYOxk#&Rt9_1Cq zIw0p-QhQ}87@+COBv>i#5}Phqu$T}*gDxY62{~357b&g)BMsF4M9&h>Qrv-4#)fpQ zVd_E&*Z`_KT~2}Hej=Cy+scPB>=qKaPY3p7!#F~ql6ph|XQ_%j^etSV*JZM3zwceLPOfmHp;O@hO9|bGc4k6_2o!|S~JMYF1rG+XH+NcN&`=m8;1A^+nfxv z5D8koh#y4<3|dcD`9yCDgcJ3vI7FH%uQ9}Qc?$;J{o`?6(`zBaG^PJ)^Gujs7eFJo zV^&=^ogq#UynEyvyF^A%j^64#K8#d!k3sPa0*}yX<4BtZIdN|ij_hI)O(D6+W~jsP z1^HaMDvpdG5MAN48O-yDQiT=e}p+agKoU`q+=U`tG2l)+|^vRsqe zS+Q~LY$>+oTX#d)64N4-l*%|A)g4`XKpR+^l?gBw+$Ib^gs>*sSRxtb*ZX1d=pv?m8-$hCvQmUw*R*mxO46N>ucX4NJU|HC?Q3MZ5a?C&SOjI~_7c^Yd8 zl9#L!(Pgf3?mH67CABd^Zgj#N>O>g#fxDnCP+~N%qq&WUQ|)=gNy6>~WGBSWk!oMjo5_av7>h6fKuWEg}-SLReWGm@9FGH3WDR>Va8N+ju^n27{Zy;AZ77Ft&5 zJ#wCmmZ#LZwB$gmQwsvoT;b#@KJ{9GQhQQCo2bPo4QUzTv~567K@2ds8<5SI-WiOc z*pSg6aG_?DunuSiMlI6AAf=>xvT)+VW1(*A>8{7jG9VNaQqu!2L?E;R8p=V9A8R6* zTp{TUD{`q#VHW9f1SM>gG-Yi~rfEavBD>PWgf}+gotULI4ZvtH%jI&Q!YPI6)&|bv&<14;Y=6ZgzqMWyDmE zA=0uRDWEqv&O7hqzLay`e2;_A$n$qQuifsxdbjh|?Q@Rv@*NwW{*d#^9d|m;>-_VL zJMMLy$KFS8UwGf<7w>k?|H#NIcRLsFq&JuD-1ySn&Q<>X`kfmuA@u|F{=x?~UcB3R z@dHogUcTEo_rcvS-0eJZm;2n^&Kq}K%pv`myU*o>j$g_-=QlX`-22*X&a)eKIL>Pu z9@~K5S2tb;$cs1K`P^;Jt2f>Coo9OMiTk!UYTj=f6x8UvNTj=fg>FxQxllXb15A}Th*4CG;FzOwnQ z^S3!~Z2l~G@UGqX`S!a|5P0>q^Y6<2VDsw6=ikgZ7jM{j6^n+`U$}MCbNIS=>j=pP`4D$8H5*KGXja9$x;83)af-$A28ZpZ{@wfBDB> zxe+-^{jUP(mC`06;!0^FJw4}dx`eMc7B@bPucwwT<5PHgwtnZ6__|czOgP^<-Tw-no@#)3&osE0=NkPlxBu;D78&!4%2(7kYW(+hZdVs+E=_tD@n2CLQ6HGETZJaHj+B}Y7X{-<-BsMRk23|)kF&0ox2y@60X`&4ezoA`d=mvVQ|^V`3a z8#xbvi~nuzX=>{KF1JdE-*`HAl#_oYx9J?7pMEB{=?#Ft{!H#CIsNOo&DRLcujlT4 z8{aQIo7+t3=e{9?z4(n>Kjpvh&7AudzMuVO4*mY(H>Kt;eUtd`ntHmzPj9NHx4uan zd+b{|Zu-~0mAjMhy!kE4aqU|w|8u#kIjY5_f1JDXD!w24?{hEasQhcs=Qh8J?S0Ii|Y9`_59{<m{};JCpTqN0|02gD{Oa%Jp19#TBs}p-?j?r2^82}s&*H@!|8wrn zXYhUQ4+Qc>d=sXZ{vdZA9qi>-bAvD9@$o;*-Fp$=FZ^K+kZ(5csC& zm;O9=kaApjBX>VNzxhV)-pBF$>YKTZm*33Y_}ZJ)bMg0u+!y#8bM||jE10scyyvc0 zZ+G5)4~nNhyuSvDA&TF=@zon%+u^)?%M&#A`YzDJE4N+PaAmu5?!Eozb~w+zcieH_ zdhbTOfBU_6VUoO^em;3Se!jpz-?)AAo7<%Z)AzSKZ`^q)cWt|K@vd`J)};?^d=0rjL__~|mr%dq?uNYjF#UY$!}$3M z|9t$OjW2I^p1NoAtJ|Ho@0orbcs{ZTzps6SO1yCI9hXt!y%%$MxVCj89v=H>{~Oz# zOCR0%%68|l`j+hv^hkOIgrjt>?%vhN%RAHBnPKVDq( z8#59EdtuFSJRKPKF#Fy(*b**OOLI8=>8MLbow-Bz+&?ilE&vbSv*X8hOzwC{J4vZ* zh!s9j!!^TeGZfWpC?$_d#{T&wm&f z(vJ9WmMPb<&p1@8ldDB@2;n}J+r^OIXduCv8)o{T=KBptEdZONxRR%`HstJFSVeF; zgjc7lL>-v)7IAScRT^b9T3qdt&-=6~Rfw?*{@B6+xv7paP0miVd9NnlXcX5pyJU$i zuuWA*M0zP5Z~JZEnLErh<3^0OuNtT2Ac%}G>M&j2h>DFdP{2KT71AwJva;Rvg#5#h0lXyD}Os(AN?q4nnF+7zza!~ zpFs_=EZ|`53v1Nyk*APyl%JPd9MRxI4j!p{kRh*hly4J7o8#rwJYD+3KNWt)$}dn% zQ_+vMeW+N$opTNa7pay;u_`N4w#HJMF2K@Y)OL7@|Ec2*7qOR#quoOejEOCjPfb^V zF$L#@oq6d*!WbACDNUh?JQ1Q3Ra~T7q^oCUe061V(;=vGi@vDW4@a?E-|}E6PP3Oj`I z|KYU)?(d~QYv>A8C=bP<8uPk2#E{6)Zs79gmTD!xGz&BiF36_Km?nxy;aIK}o!Oy^}l9p@aJO+tka9{$LTs77dU6fSA-Pw}fjPlCC0T6&5MxQPF4QeaU zEjKLacB&R3)gfN{XJI{61otgx~ zv7{jigLZ~6qRhHgP`fbWEIXQT=W%2h+S#YN{R*c}V0c60 zLEpsdNy*K>h5JMBLE^?nYUE9%Ur?e5Wx_H=8Vlo?hA&p5hPZJE5;sv?Xw-x+L6a~L zKdFY4Is~mc3o%a^%@@(kA|3q_4<3dUp>hO<#5(n{vE~G%cNz2S)S@I6R{e!GO%@NA zOC;KWKSfX(d}bl6ic#k%Q!;TXyu6^Ap{E~SgWoE8KXrP3f%5p92IjDu>!ub*A#jIS ztOHPw(hyKj?%|sxvQX6-a#}pgL@Vmx7?t1=*Y44?7e zx&{BsGyhQkm3|tf4&Lhg{4Ch@bHMk6lbicRXT!xOott0(Z?K;K znzP}tr%>+G&V~z5gT7yOZrJ!M$onhKhWqio`YXa$Mn^0Ur{P2X^GgWqs&cmm%~f5X|Z<(rQ4_|Dvo=Xd39 zZtcr$*t9>Fd;CDo*^a};rF41+|951uo0ZfZ5XNK-gl^)+w|DK&fT#4-~9jVy$^g{Q}+M8 z@4e^V+?%FtXb}`CLD3-yf+Dm`nW~~jXLQ`8P1;1$Bu#GHl$yB+!gPq4AVZiiVZt{| zW|XOsAqW%1OpGANkRiy7FhLLm&-$FTR=RuY@_W9|>v_GN*Xy~bJ=ynt*8c3Z&pP{@ zv(JBDu<;dNp!XGD3Y}l@4fXp%g|GQiY6pD2`T<{X#=m?iS%Y*gKyS@~h_ucZoFVOcsoopKM`5Qj;1xG&e`InCRQkutnfzC0XuYb&!y7+6Kzt#AI zZ9adf!twjs9KXpA`<rv(x>)#p(Xg{Aqsw#_4`v?R39s+|}=o@8496^H8Lv@&Uh|tpZ}@}7Z&JQNzZv_`?-YOT_h)?J56u6l=eu#DNPTU;e!H!oY?_i?(Bfsd}P4aeq_Mp{v_bb`$@o=zC2(Civxl5 zl7LfR9tfmW1e~mOfzX2W0bhK5z*%~6z@*<02-My{`ECt_vzh~@@!^24`Qd;w@2P-q z!BYWe^9zCS(pLjc_UnO2?i&HqJ`@P|3K~i8a&!D^N(`^>Bl>f z@jS{$EuXjStH#&jA8y#oKU5=@{ z*9i>W>zK|S$JgEC&~_vk$TLBc?+-e$oq{Gk8Vt;j22JdzL8tJNpy{|W=ww|RG&6n` z3@rLp(2U#=bQ&6i;lZY$v*>q0lX-VAl7D|N&~SgybUqRc%;*T3S)IYq*pops`cyEG z^K>xM@N6*9{cO-ldp;PL^L)_hc`4`{eJN;`{xcY;`)ANBd4tk7f~Mk~V4(S(V8py1 z3@ms*7-{-67@*T&k?gO7fsJ1W%_4s&V#1+7UN{t)H8m6{of>jlb_u1NR`ydo({V?R8|4}Fq{v>4PeGy8T|5eD;22w)3p_E{LI3gP38ZzUq?*AL|I)z}v-E=$XVXV1rsmTW-^iyaPS<#fZ+M)_G-2N&U)b4{7WR*& zg`JA@aB4gw?97@OHVub{eNBgkP2-VaU-OY+r*#hfJ}K-^pBpx_7leaN3&Mfvr-#kh znc+bEtgvyGgq?wC*o-eD{fe+@T@wyDv9M`dPkd=OH0z46iPwjnkvqb^(L2JyiaW!h zSWDR7(Gqs1w}$<7tzonA58<@&KZb*STf(VLkA%}}o(h}xXT!Tse?IJNc_Hkd_hQ(W z^Ky8~yjR0c*Ff0JdpkUJ#yd3aop2~)IJ`rJKN4IVh|EX}MnbbwB7vHeh%YZ3@n=UO z!KJB@Ky7LyI6W=mUy>F{$(kJTZ<-uQnKdM|q#`k4`vcUZ)!$cdPFr$xf!XGHes zT^!l7cxl8Go*&sOw9$A$@yi(-~P);sP+1Yscne(+kPD}*>^_#U3W$dU4;njRYm&~+OMb3{(CCz+kM!V zQ%YADu)klNLc9z8PN9A3M#|qq<5bFz9Wmu`2j7Kf(avp8UwYbJhIXq4XMXcDKmHm+ z%*{y?{P8J0zFU2__bw0?7y@Banbw0?7y@Ban2Z2~%$?3prKQJHh3TbH?({TC=@B-U z>E%;y>OOr&<@5dM>2Z3Si8~$7(!no13+JYM+j@F@C*hqXrTgeKS}qObe5zCNP3{8^!OYN>GY}< zPQNGV!IB;#X||;FXdMmd@xvrNT+$p#>5)4c(&I-;dX%KOk{&JTF_IoD=^ROqlk|8= zPmnZE(i0^;Nz%EJ&Xe?HNl%gV`;yL=^i)Ytlk^9Y;yD1+`*cYcN_vK*c*X(tGbKGs z(z7L9Ea^Ft{!r3$C0!yZJ#t7xdi=+do+s&2Nq-{g`I1H@&6l)5(q)n^m$XpQB1uC|&N$D|18q(tzNV-Z=JOhF0tKBp^?k4nym%A~am|(@<#NuJv{KUR zB>lyY*xoGZ#s$ouoV#t>wIAj$-Mrtg5a)%N!B?l{lxb3_n_E$zp%YM zZ`LgJyH98PV6%i)NkEK->y#^WjpWxxA$Ks_C)P( zw~tw$$lfgWMDacL6V`LTWqbd3e}{d`80)#8vc3M~&)ClWobByTikAZ`oe|?=M)NX!%_>&h|vhZ}yjL*Zr1v`}iNRbN^*~|7*TteWLjIzh--)`MX{0 ziR$+QgZ)va;r`h6{&(`Ro%?^=+aDGCfeG60<$l(4KWlq^D8P2^Pi=3%UhIkH?+AzW znF;*ADt7KiZeRZ9AnOw?|MC#qx!=3Jem8pb81YNAeEvu5iR%A@FzdM=y?yy6!uCY^ zCb1`KU+1T?KGE_|nZ))362z}d?1`3tEbZI%M@(jWBK^O_o~V9TPhowc_#Zv>+x>qo z_C)???#TK?`^OfsCyH;&G}b4o|F5RAJyH8QXD7BNYQO%S*`BCxGQGBo7o%M;9-%c~w zp2+_VVo$XHIAjmjCt5zwi9M12f<0NEsQy#;VtbnT@6dOChyG=;CyL*PVowym(BmB6 zMDxF=*b~kFp<+)of2W8&QT%@>_C)QgRP2eiuNQxZ|6hKG|J%i$X!*5&NBPf*-Jf9l ze^%$Wx8L}ye0-Kj?|+Wds@6CJ;VQ(2#xK!3(0wkPsmBKAbv-}}UVP=fMb|Bmu2 z(%AoL3H0x$v;D9H_L)1dJ<<7to?Y0UnLz)6*b}Ya=^5XyZ=A{YMD63(`+U3oA+aYq zzxwKT=)e39{f$}dKhgfLViwzV``Xb3PB%l6=03%CgfEZAZKtQ3j9+s8TuGNmS}kdV zq+2BIlXOhdj9-af(j}5sOWGjm7D@Xg9g{SpUi6YKk+fRU21&O_+9&Clq#4(XUeYC! zR!iC-=@v=*Bps79;|9@7xGS|Mqz zq~&N(CNu7HQ@XBsE9-T-_7}1fOwUU&eNlqx`3a_1B$ysgFug9p^ri&U+Y(IgNHD!8 z!SsOy(?=3aH^2Gjc%~bw0?7y@BanS)2A5%|dzqpSA^BoiWiUjs*u}_n@c;mlO>^Y+M+T&tBP3${}e_ehx zVy_eXJ|26m*ncGUZ(4qs&)sGDd{36o$CCd@srM--@$y_I_Gu?GUn}-Z;n&1|rA&WV zc(LfiZn@b;mv=7t#_*{uZ(dI6ChzHOmv_sf+b-|A$H0k~_h+NYO`X`+iQU^C>c4S$Y`ea|k2ij|$olYWiPx`$pLKa_y!5^dT|am}(@og3el$G3 zZM;k;FRy*2{1hpFA6ZTrGW`IVulptKl=Nju2PM_^5s-E>SyF9B2MT9PI#*KNPU!Yv zxlCUp>5Y=!D(PlPUzK!F(tk_3--$e5vn4%O(o#t;lJv(ipH~R~QPP(seOuD6CEZop z>;94+E@_^mx<2F!S4w)Zq_vV>E9qGh-)7-QB>ktPUrXxD<@woP(xsAKCHgyLJ~qho zW=S8E^i4^3m*tx$=^2uqFKM-;x}DbTv9>#HkN+e2x+MLlr0+}Wm-SxT^O3^yB|T5l zwUWjqy-Ct$N&h10bCUi?(#Rf7^$A7{Jz)6`pDzLb~$9p&YA{-2e%$n{&{;KRF^1#HvZkMLTI z=@0#(Z4T1^)8YT^{J;C3yY*J>|I;dMlP{_Ne-^(gH=5b;ZGuYRosK>s2JdvNhZuOe ziMv35()C9&O-3LYfn)@d5lBWL8G&R3k`YKoAQ^#V1d{9Q&!FiL)E|`M8NO^`_qpG^kTG#Q z=f(qc6Vu)2^K^QK@ln}6`Z$Vj360e>-b7<7jZe}zK;syV(|yL5OXK-8#%R2n#+zuo zlg7<7cGCC;jUzPf={LU9Xk15Q6OC;&K1}1^Xnc*vVH!WDF)d(xyU{p{#vB??qVX&m z^Jy%j@uxIiL*uP9ZlUoN8vjG%=QO4|q^0qA8W+%bK8+X9SWDv#G(JY-5RLlofrE!$TjL9aLOXHHs#<`ou_v!b6Q%rCXjaN*eWkFvt zj7H}bttePq8ar!kSr+^s^AL{T~Sa{y6u#eYYQsfvZ`W*WOZHEc=OZvBB@8>*JaN|$XYD?m$jmuyMd3Tg&xt4fQCD0F9>blTkL+yy6|eRed9 zR^n4iN{db}D5C~J<$7$VuZWY%As!tt(kh zHQ1&|WG!D^K`zgzh?SI=Q8WK0JH>fXNkvgq+7^`*7L_gEaCT8FVR^JUqgBCM;e}^Q3{I@9ETERUO%*1L^Ge)$ z6jV}b_6_V^C)%9&~=B<-z~p9#2Ej{)wEP%i?KqQ$by0mMU@ln+Dvs!2W2(> zGjq&Ni;K!=!((zBQ%6F_3~VQ#?~yNbOoPf7i@eJCbvEXmopU-Z%u;iVJ#!ONH}DII zI)C9OXWd`==xQSF##;(NV(t-`;YhzLKSD(9gEvn+3Ss!H> zCY@u(ucmAcb}(gV!4#mi)ZEPdzLM3nnTnd%d}T$|u_zPmqRY%@KH9&mC@C$ap26C( zRpx6dE8e^_W`6Tj`rCX9eT>mxxtew!<&~?W>x)WO7RRDgJ_7ZOUw%C2H~zzQ>J$E` zdCBjX)`!xlfK9LMM1MyIrqkcePtcbe1I4r(tJ2KB46H7xrfvH&x4W>7sUc8Vu>PA2 zw+0;3bqR_ceig+wy)%vejtoqrn&w1*5Unbu?Wa2UIFPG9zaU+08}jyiN}ovd7cW=K zZH~;y?;OX}v?EKydppwK!KWx=V8x+FYR*n)tLkZ0^^j`d1-R$gFfTf>3RQgWaIM62 zkCPiU)Ccxc?<%^+T;i-THwLL*FJpd&zJTlQnn+XoZqNkK6Qsa$c()1cgFytACz)r0==zi_-zKB^fZTL`LDdKq7A-F?q^8px zpw4L#7FEIp_U}L$^=k{(Ruz;+6D+#_2B-m3cPwiDC#Vh2{5(WG9qK(KF1b5IZTmiY z?Sfw#w)sb>EV?GTEQ&>gIfWKb1ToQ}YgebZ{=<3l{iGqN>*fQ!+ zn~pFo`=6t&wA_KeqmS=EEymrGLL}aHrLYOTf8}liVx6XtiNC#h0onm-k-X*iuQI7QHGJL_C|&) zkgbvalBXl01r-&gMbWAfTG?n{*`G>|quWh>Bh>`l_CwpGGG0C_sMx4^G_|M#UGSVJ zEqiRLXwO0I%)FS2&JP{nM9mPYb~iRfn{L9Eq`7Yj{jK{Vo&L66f;f#nflbt4`&6pY zs&!I)MJ~-7?e{b=?@y{MUq7+E8J$E$JV7tl2`;DZ6iL=c=}?M#8y8Lp`X{ZRGN?T2 z(b8sgtKp4Fyb4rC*Az@_1BWK9fot7Bn$IW2$}3j6$6;E7JJO85qM#BJD_51Ti&DH+ zxZSFv=$eU*(~FbV< z=$K*`uzAWF^CV4)n%k$)@_LiX8h##Y(RfeT7cHpTP`2E}r&42CMk`FLd=>S>Kb}Hs z5=DMBRxxu0?R|Tw1giM^*zna~33cOTq^6l&iI!|0raWzzVy&Ezbx!5RQfXeK(hI7J zv9M^>qRpkXrQR}?x_(j9Pg!UiLCey0tg8n%5%eOXTbukJrc~$@FU4YZgW;sLv@I12kS)2fNd$mG+~>);&$%81=ikwp~px-JxnZHO{JS zSLWIss2lP)R`g-6=$eOikZpi@6d4{tt$H`l^hK%3nSPoy{1<3CpPoq@>LT-x zbZwvJnjKaZ7EniFht&nK)oV++@^6rD_ux}H0L9DAe|Ok+_4}9#jo*x&YRyf%(BG!Z zF&}PhX%X^X|LGm&N6(HFD$0hA2}1qij%%qOA;u5LSofO@Dx>3W{2kgtXT1WlcaS{S zOp_n=)7H@{IN{-@Y1E(_T$V zI~Q_Pga%zfy9+wNrTT0;i=8{JFrV(IyH4{e)nV{6w|n(ITGt9UaxzQJJIRl`ccQLq z3u-!aJ8H_V#rU-5>Y-eH_aoP)2k^JM19_w6WfNPnPj>ROv+!w|3J3w~o+PLe@HuL*u%|)~4r|o4nE%9AqZl%nVBy8()t3%x>>M^Z~nt@%`abJ~J zaa}gFi4boek!bV1Zrq;Qm3-Fzj=uNoFLXD#QCdSv3upsBv3tHZLu8vXXmLD+<|r-U z&ly&zUngtdqiBGw4{psP(l51)%XTG4(beU&1AHr^QZ^t_a}`bY^jump=uqDLc2`=7 z9>pT*dmeMS{meYStFH5_bpBo@QS(=+lpT9P^ZKsX(MOBigCIWOiY_Z!SyHyOGlN51 zRoBwF+%{T2Xo=8rlZf*iZ_|lhrvM4T{L95FUvhZqdYG;V$4zl!-n1c7wL=j?p zL^C}>nTGnw96;gbe`ZK2&uE50%FuluO4+g*rEJ}~T)u}C_VYa`!VkfSy_LCc4=%$! zzOwdTEtIXX30)i4LcQ*&xpxn=&#f`Avi}{WUJb{Ccj9l|E%4w<#c$1?^5g0~Y3@tP zRuz|5PVBnA+Jg`6Xah^9pQEO3PwE&nqM!lyZ*4thdU-(woj56q(pGDtQ}OJcTz@J2 zdG+%0p2nd@hrOlw&kSrB%nLHB=B_=r_4Yb>8-Ev_xyI)($J=eA9@)Q zy|mBo&Y}71UKH>+dWp4eU$L)_x_Gi1-F8=dX{PK`CmMX_wj0gyy{M+wWKxChK^eSl z{8OgetCLD}leOo49FgOQi2B<9$fPr`|Dmbvjc)(_di?FVfsO?HZg+QUE4X4NZR%QEU7kw1tIQ^&n@dBNQtNPmBuKJS||rR;x`tn053 zG+s}FKt(|bRrkL$zqvakqWZn%$4z@K5nPJs2aOn$?{LiGS*91V$ATbP_w2^^eH#d(L5T|)>2w*mg`Vh~|?chvPb z`V6W##=j8)N4&45ncO}ackL_gq{QFv`wedH-&d>;?P~%!9b&HE$K7i_p5=Ds&7=EL zdoz7|uQ7G|(Qy+UkCvCOri&)-Ue7&F(*x@&>VQ-C*8Fp(+le$S`%(WBhm`&T^T++% zg+;B8&k#qeR+UtkHz-HPGuUi&euQJ|2KT7C`DV9YPrqrqxeTZF>-NV5EV^cM!~WE{ zqzR4tQ){M6DW&Bx9F)isL}{4ceP6)Vc$Y?}cIPa*kc8_u<&_(7OpiI}npKWof?Y(^b2#z0SqRi$_OCR(vuJhsGul|)s|eo@ z=qeDch~K&v-h99|GsL^jl|^N8mfGBQ0B!2uz{*L-jlVm9_O9Pu=6^{u{yJu@i|Rq$ zpi6KtpD=cJ9f*s4w8y10gw>o-2OlSv_FOa z&>2UgH!!YHj|=4&c<@J5u|Pk*;veV;i< z%6;x2TD$K-rW*RYZKe^;^vOZgwW8D6)No!pNX}lGaZ---xsB&4+Dn<1Yj>l+?bqx^ zOM9EXXWMM6i}%26x6XP7sC*q=Vxm@N+Go>RF@S~J@+JC{gO^h|dS>*i*>Y}6DqZtE zs`S(7a7r)o>hFo=4zk2w!1mJhK%I;46;pdGjIP949=eL^_I)MuBj2OhlJf!Fjf!GN zGtu7}n_Xp|pghvqd~7yt^@pLMV*~5_*xj?y%d9&XjlAKYiE|j&&n`Dj2UGJHKvUU# zrF&v%n-e(ZZwFV%btbx;T!llQ@>O(>sh3o3y->BijRRS1BTUU9bT*WtRkC7(yAqnK z520?wYwmd@Q+EiTiQ+RHd@6bcjw9*B8C_Vro3gfFok|6EUY6=}w^FO=eE$nHx%KZj zW3l;joUw3EmYU&1a8A3*d`Q+fvYW9(Xj@lkUOfb&tFvi)?^dqe{X0xmpvFV%11*iU zg>;qI+> zm8H~|YCD8C<)%KH)_7dlGPh;ZmWc`lMucx?^@jYCBY$SJnEGm8EOhX#RcZ zYC0UZCrJ(J4x=3_-C{voHndH1!(ruXOAF}|DV_XtuZ0#)^z+SIly}(O^tb;SM@DtG zxhL(eK(}a{)$EqTrS#t(P9dqs((a*UPHVz;hbiXq!`&t2wgg!u?q=jENT0_7?R2F$ z$F`FCsyOSjwk&48I$TbCGS<*Vo(8%FhL-QOu0Pt;>8w2h-1Ajnu;9xKDy ztox_{gzY6Ny8lYJ>}kN?x?kXLBRQkZ15Kf_-FaVLPCGuj4Ay!CZ9V0b+8>Xg4Kj9B zX3G(J;qc)j*w=c~c?9i9aaAp9-p}Fw5x0w{$$$ewbTDaWVeUHIJ}xv@97*fajX2co zex7zwDO7cH#i7OK-NW2_0k$42>X~o%6!->n-Qn1`meAhXbyH9^aZ%;Q!(-+*M{=u@ z<@@lFbWX6Cu1Deyica#_NKJ{(`}L&wwE38}qp)|v%}DZe}~rr>NL?N=~KwNYVf!Da%^=(@!O+uVW7&~bu_J3g@u)L zA*7JbsOkONZJ0LTZbOhs`ek~k+>DU=NM{5=DuTSrRc|5_BuM?Fm%v|)z3i%l!O}N~n1aCDab!qD=u^Ih0-)T~WCD*mVV!CDv)_x?^c0jU$ZS zV^-6w-Eb`JooS|M*4!Cvu0x3X>F-3qdXJS5ze0ICKcn||%PJZzrE4~Hp8y?~(V3uQ zIjo~pJ~d~G$I2@6w_`by=JR7ss%)sgaaT5&Ypp%FDWQbU#8WrWe3@%SHbxf_xcyvr zba6Rt31vaiRq3*-63TCGJi27fVNvtg(e6RB`RwS`)hjA;qP$JUX^PUdG4s1)>`s_@ zkme67^tO&n*RgcUe-1T#_X5v7xwPLkzn?=*>T&e1d;W&M4Nu{3|Fh_8%Qcnp<5rc> zm8wgRckke-J)Rbbdv6x{6_VwX<4RU?N5fo0H}ZA-9fj0Cguf%d#or$HZ=3tK-F4SV zzon$2>gB~ag=5}2o{E{Ecd4X)TX!r7< z>z3}@qdA~CyzzwMwK0Utbe^!bif-*H)O~Rkb&zmw-aLJR2`w|fJ05qoQ49O)@$ODd z=Iz3@ce^t_@&aZ}J9?fC zUPm79=%36pI~JH^$M*EH@8FE`hx7Mjk0hJEjvCscQDGFxMTh#(aE0rFrn+D)aP_F>`+&Uw|~-dDP^)e6&ou z{zNyE`sq|!k$LAtJ~}b)ok#)fhIMQhf7{-}-#(gxlNEB!&0Kd9RqBYt&5bAV64W~h zetr_ISC!?a$5zMuH&5;} z=%ITWSK=aGjQTq7kf3`{`uNo745~ya^{7n)-}Yo~nHw+-Fx6(#pN4AM3lyz3TjrX; za&!A!*X4b4akOUsLOHv3qfEUQ(Vy`RTGAU*(mP~tx(KsMACscp#r-}m>(Y7DcImrC z0aJIeFS@=YM)&Mt+kO2!<19DB?Bh%JQ9G{+w;tX+&xG8&mCTh?Mlm(EqUG*Boc0RT zBmIM~uquao(ZbqHp%% ztkT?C#|e-7o1L^;UzdNtH{jbGm{aEr_(q&M|7It1le0O{=Pw-#^f;N*ow2k!|NI(% zM{s&t+Vr-R=G6W`k1uO9*za4C))VZcdm_@>d?WrE6ciW@bRDuOSX~#aamJncP>ruO z7!UP@`vP5ou^>eyG^a1T$=49>oYLdZ?+&NujD^B!1HN`&zi)On)vI~0j9Opz<{jI7 z9f6@KZN8=emF+arEyeLrPwGHQpKlZ0?mXrjPo38t%pI6k6C6!#PYDmCh3kFIDGmNs zU&Ev=;ri4O(q_ja&gc%~zP8DOk;ZUMus@?IB`eDrpYAvX|gV zX7YmB4QVa@oc>9})3-Q-sjZW%`_ns8M|bN>?{WH_@RsmUTDUi*CR`V3nbez78>*RH zAD*$v*Llz;-xyVYQFU>DFmE*2m)?|7=d1CzPwDbC`r86qf=!{$;AUssKN#xr4F`Lj zcuN1&K4*vmGO))$pfk`IuA_Mx3l34py2D+&4y4qkc88k6LxH@}y|<(;88~WuQn;Rc z4@ZVW%}!frAT;cZhX?n}X>#iK+Z1fwp*Gm+H1Av!8s4csSUV@TCP?$PL*E`F!TwM@ zG#(mC8B57&N@5NHI$mu70Dfl@F!*$`&oGrdq-{#4~fpK5O7Jt8QFw~D~`v!e!V>9Evc)0PP z_HbXgba|*Wz4Aw)`WV4EmOOuIU8%71-+q- z!;yHPmKx!R(=~Z;+Lmx%YIAUtZzNdfbc8yc@zg#at(t?8zS)CxBYl&zgfW%aVdqcy){>YX|eUbX9 zn}+%zqcIyP0`NIIod8aD>$VHs)P~ z>u+a1PKP$XR_$}djVc(HlzeeF*g`0(s z5N;JdM|g|yI^o$7=``2gXuFPQU$Gsc-$3HN=?{0rg1AU&Hz@%oE!eH!yi!c99de<<9!8}roT*ne#% z^DN=Qnan2(_Z-B$PPpxR%(n`cexLa-!egg0_X(TRnLiP(U(7u9c=n%h4)ejnGos9= z3ymBnw~6=k#Z8N_$zy+g!iNf{HM720xLx>K;f;5){ZZk;dze2GuD_Rg z{}Z|V96Y~fj5$|$Q1~k0#rLuOKH(AJSA}~YV7vb$F0c1N=7WWu$C-aDT-?cgxp2c1 z%=ZZAyv#fxyoD~R_^5u7xm^D2e=#2}JTl0PXnZmsl%;mzHE17Q* zu8uMPRk*&I`6J=14a|F;%KqAfPZw_4$o80U-OreRC0u_A^ZjbSnfVpr{#%(p7aqTz zdGFKMU)CMWrwPw$XO0Okc%S)Z;nL5T9~N$!iswb?yTihH`!XN=11`VmFy@uQ^A|GT zC7csueqOkLBeQ=2>-!#KK1ewB67wSAjdVQ>{x1=Z??lg}51oWZ!rj6z2|Eke?pw&^4+tM9JZ}-(eB0;AY`;+01eos*d+?V-J!i@(qzbTx52=nB# zx%`pCnU50g%w_(OaNBXrn}mnwG5<+;{8Z*Qg=-fwPhQOBx17a%xNyb~na>jLUBX-{ zJm);-YlJ(0!u+6cIG_17;f>3f{pYa1#vO zoZ8z9!Xq1)$As%PGVk(3_CNb)%tr~QUB+A>Tu2vpP~P>zjXlgA!h^#95?=Zo+rJWy z3-5L=`&;r4wjU!Lql;B2|9s&EFEIa7IPU}IhlCr2Ulz_BWxKP4%kTM=`7q(?FPN7I zH-5#uUbvgSL=6A+%6{fH;ZBEnKzMd4v+qajuk=UEy9f^mXA9>>+5Uau%+<`xh38Z- z|4g{JiuoqZU(NiGaK=TQcgnB&63H!wE~ z7yg?0@4`cLu^aIk70$VjIrTjD*CzZu;m!B6{S;yI0P`y0#dIYEP#3Xe(t znDFjVE`P~qtpBla+ti2~pDTpx>Aew%*PX%(c4U55xO6)6xNxWN^n5Nqz7yML30LgQ zyja-Th51t9M&W0KyM*5po}a<_rxi&2cV#|axIwsBxNtYN*9mtEKOkH@gYEALZ{CMF zbs3l6vM=*-!p#RU|3tXwd(2k~_a4mLB0QAM{Frd;Fy=wwv?G}(EoXnTa+$v;+;uGT zS;Av;nAZr$Phf5oZab0rQQlRRmHqsxF^QkEL^yr`3d3tpE3^$7sr_c zE7)K4rOf*a*VHhdDm);3p>XFFY`;Z#-c`)c3Y(uZe=b~251sjpF|$`nJa1t>N4Qh? z65-fy*nYS0xNwi~@a=5>RJiL7DZiM@&%cwoKzKy>*TNO|vHfY`@Mh*OiT4Z`)ASJY z&Lzwp!iNeE2%jcw+ByG9;Y{JHh4X}eE1WOGa!73u=xx7 zpDUaxTqv9;e7SJG@J+&T;d_Ofgr64f5dKPdKzQ0Ju8(<`{U0TqDO@U?Cw!T3zVHpg zap8XnHwk|%+#!5$sm#CdnZo7~_FpEPDSV}Hp70&Q`NFRY$Azb_=JK0_4-xJV{-N-I z@K1$J2m8NCI8%7DaGvnrh4Y2q6pjmjE!-r$R~gr*LpWD>K=@2y^CyW*MtXz)7P-xJkI{-3TF!E3+D-6ESxWVhj3i@QQ;=x7lk{7|06sg?5`C6o$P;4 z;Y{IEh4X|5Z*N={^_A|_&-HBQg&!Ag5`I;vxPH->0MAf z|H4he`NHkOap581CgCrIJA|{=b9n>8=Lwsq*#AYsnZh><=LtU|oG<)O;kdA==JK0_ z_Y&?9&J!LGE*3UVv;V7vGld@{riZ;vp76hg!_RR385`JMA$&48Nat0bW&2seON3Vo zcMD%7ocA}*e~0iE;m3rt=?-wT&o_nV3FG!4_$w4ny^uNk1FNC{=r(Q(ypAt6}m$7}maGh{1@syp&UkBUo5uPvH zC)^|axp2m#oPYO?tj`zzzHp212I0ELIREX!;m4Vu7G5O$nQ(*f!52&YIywK3g>!{3 zC*E@xWAf>pc~~AT!iB=!!o|WP!llB~e#-g^;nRgn{Ii750MDTHQMf?Z^mF;El^fIC<+D+F##EPYQyx!u zd5iMK#V)@nJl4WIs{ADLYp){ZFh)XS@k&{y={knf#^4h zURm|+9=&acKAg_}9}>N?>PINP7{G|1ZHHd(huX1*m!GogOZ~2VLT}rlFU;h!aX%H7 zzq0C=P<%0f;or7HulHNw{wwH}kss}cUSH}CVCZc-^m>0*ujrLkPlrX@=xsandOsKL z?}C42)#nAa)!TOHGv@HTwdwK~R{fyIzvhP>`fWtdqv+PdS%sn+n;TRzV3MTADPYZQ&v57bKL>F(8>>nUhkj7{d9=GGVECX3q0|+ z?a(heiTzI#|H`Vbr7|#p;or7H-zWMnWd4;^KkRA0wjFxC-w*fyq5jIM&-Aok+YbHU zkGXCih<|0(d)tp~hra7P)_;8<`&U+dj%WR~?a=G}iMU@8^;cH?BF7EiMEq$dBdUj~_H3f!?-5pGoKCV9SvBDPtP+xin$` zLvP!mUn2T^(JQOIb&@Ne(A#$C_5MxV&*`>bIllJJAmO|^Mj$UrsE>~o+bX3 zG3}e`Z`+}76MdfOl~q4OcEDixx9!k(Q$G^69qAA1uZ(H%KbwvzFo2=A?a+_WeigQJ z=nwSDnD$Nex9!kp*RVcE^vbHI&7(Vj;or7HUn=_b^c((_>8I;I9Y1J50=;dAe&gkA zyMcZ~uS`E({e1kO0SWZB9s15I*!CgWp;tzJ%zq5WR5XC0x9!mD{l2*W7kXvnhkm+e z{k84TFR0_PalbJ1%Bo-D*?!t~==FYL++Pg6vg%tLsss&S)Zezd{u|ha`;lG$!m4k@ z4;oazz_LTH_bcQ6W$2ZWAM?KnVj5I$+o9L{n{mH0^vbGF!w*Yu+o9L{p>cmS^vbGl zWBG)C+YY_nKaKmTp;uOYjc5I_?a+7L#`A{zuc22~eUHb#ZHHd(&&K`Q&?~FH*t7g? zJM?-#H}3C-URm|CJnc{IIKQzU&6|CG18kq)0Ndv`!1nnKuzh|5Y@go%+vhjHdVT}- zE5sie_NyUQp!6gUFC@nC%SoR8e^v{}zhZx$e`THjK`9fwkp9?q#4mgo>u(mlvg$iM zdfN_tx9Bg-V*kpjANR!HwnMM?-{XFKEI(z{cX|BVcIfNw;rd7ESR8s~)qD4!w%zqF z`qxC?C9L{I_(21bh4ja^L!a?`w*6*r_OFcm6cR(-I(JaLZHHc;H^B1;sK2u6W%oDX z-?l@q&nMt{1?ZJkpY8E)+o9Lz8Ss1q^vbID`nT=S*U0?ic?js06X|U`^qWMF=O>_7 zPNcW((ChORc-{hfV)=W!0y7mY;2hev9bwya@Els*iitKidwyK2L(@OQ2U)y?6d?JM{Ye37$uRURm|t z_GjCn*XLF6{0j8Ss`vW0?a){Mk=H*w?*hHD>b?GLJM{WI44#jHURm{C|F#|aHt~<= zX`okDz1P2OhhCqz!SgrJE34k?-?l@q&*$KI9q5%+@AYrnq1Wem@O%&S%BuJJx9!mD z^FMeV2zup2dfN`YJ}-plhoDzZq_^$RXWz%mAI}>>ubfD4+o9Lzk??#H^va3!wjKIB z@sH=3pjS?$x9!mD^GePM19+jw4t=laAK#Vr%BuI)-?l?PAo>;iv0hpA6|VQK{%t$-i~hv6 z7j|O3vg+eCTmTsLx9!mD^L_gt!+K@ak9z#ucIZ3j@bhoe4`98r>PI}sFSZ@}&3UX3 z&tSc>>g(}?1|+DzZHIpTNv!{a#7`OdvFw)8h(YzX9r|X`zbtxX)mO88LT}rlpD~yH z-zV)yS@laidfN_tO!U`^URm{d9=&acezWKc_mubvtDeH@4&a4WelYYslK=XBS+5K` z;+IdyY#6}M+ji*nc~CqbiuFes`Jr$1EPvY$y*@vR=SiVgR(%XVXg~u0wjKIKG!eE( zr2fj72LEjy|F#`^a~A7oO8k{o-$_|8fZ^Y^L*FU-w|C?EE33Y7N2s^zZ9DY(yeyud zb+@1L{H*G|>!0R_9eRDf7SG#4udMo*>wT+#+YY@xkBjGXp;uP@3{U)QJM@Dk9KTcK z_)l5&c^?0^9r~;bSYIvMZ)Mecx1Y8h`fSnvLfXG_BE4;gevazJzq0DH@q-2=3$6TM z==J$!JkN~yDZ`HTH|mL>ZHGR;oag@uslT%7z4LF|UH_tQm-?3qt3HG3fB}s9+ji({ zB>y|2S601u`)Avs9~b==Ie(z6`druhR{ypg`kV@${}W~Yl~teT(c5Gzx1YkQ z_r_22!w$VZZ;t2BvHX-(Pmlk)19+jw4!u5~j_1{(S4Ms;|6%-~0SWZB9eRD99nZHz zuZ;ZAd(XexcIeyJar_qU&w6FmXL{;y+o9Lz=kYu}{41-zh3bF-jQZPl==FJfJbw?p zGV-JT^tik`Ac5YtLm$70>wmTEf0dCR`UR_8`GkMl4t@9AtZ$md{*_ffo8=RF+YbGh z=#9irS@nxOdfN{Df_K<|mgtpLk9oiVUTEb9Lti2Ki>3a`utQr)BL*<^wjKI@(a)9m zDXZSQ{@Hfurw?)cuO(+#e#)v3d)kj}hknL?SbvA;l~wP!wclEQ+YWuE=ntF8{*_hl zJ^r!n(9aV61+xEFR{ci&paIE3D?b?e=6AXNd7@W_9r4fd%)f1izKdQ~hP;o<{41;8 zyZy24(D#b|Wzj3EzLv_y07m_7JM>H6XaD<4{gqYkz5Zp}p>O^#>wl*6FRc0jIHv&& z|C%2Rea#5#&k?;c?1=@Iw!8J0_le;BBJi*87g7BH)d2(8t-s_4L$B{2!TU&{ zSB4$+ukiS{?a=G{O7Q*?=#^EUjvq82fq&Z$edlK!|A6*ig^?frz4sqze%PVc_oLu_ zDe$kX`b^4)0Sy1P9eRDA3f`{*y)yE{KfON19l+4rcIdOd;`;wR$a-bfd)tp~hhE>` zg7>+=zq0C=;0Fyz;NP}GpY7xKS=^q=dS&EC{k_|7+YWuB=u1Seta|VL7q%Vx`T+aC zMD)t4U*w6OZHHdpSA+N0VE&a=pGot80ld&-hu#UX|1pWbvg)V1>aBX)4t<8`_mlRg zta`_zx9!l+5Peki%BrspaRFe|-?l@q@8`k$dNBXW$dBdcU4Lvl^v#ob{!bGB%Br8w z?kD1B+o2y8{kb#Qzq0Bb{Gb5|>TlbjADP1bccA%1{gsg)_3!X3f7=dy+YYS%SkB)l ztKPf(Y&-OwqR)~0f0R}4-T&Ek=zB!}viMg{q_^$R_ly2a@vp3UZ~bjM^dqA0693Ao z_ntqn?a=G{obY}p#9vwUS+rhY059~|q1X36;eAlhDR{f}_{n>Wt^?g@({}tw6S@r$w ze!{0h4*DaudMnd9=&aczI`Uo|4uUh%BuIaU)v6Sm*@|X`Bzqbk7xdEJM<0I z!9?B?(JQNdo=0!nq3_?1^_PiWS@ki`{M&Zu7wymb(?zeWdhhvX+YWt)=x-6dvg*C@ zv+dA#i~b&ozq0DvJoUHj&~Kz0T#)yT2rqwS)zj$!cR;d`{@8ZthYw)=nCO*}A8p^k z4;qj_Z`+|CIgs_wOZ}CRANm0rF{s|QL$B|1!~5OPew8Qm_+ja7JM{YgH@puHdS%s* zdi1s(`sPEp{_n{CTUqs8yKL*_8;wnMM)r^EZ|pjTGC_xzJ> zhrZ%8uKzKz{FGH+=kagbq1X4};r)2Nk4ywjKKTnXErg z^vbHA?m2&G+o9L@_u+khsK2u6Yk2WZv_IPpy}s`c@Bf2dS@o@+`rCHsJJ03%(__f4 zURm|VQ-9m;>VM4o-BmBF`X$b`?XT0aL$B{AJXq?ltolCupaIE3D?b=|eV-xTZ;1I< zh8^o~7k&W#or`%o9J`4!yoV5${ukf8_~1e$apf{%t$-tt;98 z@pArF8TsLVsVDxn9eRCV<2B-6S@qN{aR)H`+ji*n{f>CwBkHe={P6ExezqNYeIF#= z4+*`p>NmOGx6Z$9hhE-#D3zDnqoksta+_(1~___yuQ>-#L{iC!7`q0grggX(QN^f?#v@;^_Ozq0BVc;?@> zLm$6{_4|unS@r4ce!{HdS%sn`|q|LdVRlcvFMdm-+>=AAc23|4!ypQ_u4%6 zuZ;Yde|r6%JE-2aL$B}eeKe2t%BuI)-?l@q@B95o^vbGl!Vemdp#HWUdVN1I-WQDJ zr%XTH`g{G`cDMd*+qZHHdpSB&=; zL$9p*L63ji4!yqL81FlVURm|t{-bSo_3}Ps|4E`3R{bK+`4@eEGVIW2J;Lz|(0v;4 zudMn3&;G}@L$B{&K3M!KtG>?@Kidv{zr4Tss(skMvg*D4AKMPSzV8|De@6Y4RiBL? zG$2{%u|u!#hrV3&%E*uPXN*P+s<-XX*FDMcdrRuCta^G~?KXPb4t-iT>yIX9sK2u6 z=dpakzio$pk?0>3|H`T-^EUo%JM=UD&i<#0e`VEs_dm8B`o*F@U;Hbpe!#Q+wC&LA z`?&9s_$#Zv##4XW4!yp=8}IW*{FGH6^Tgk_L$B}q#{0jaS5|#{VB7fFcIby_1xH?= z%>M+_G_)UY|I4;RukRPf`^Mp48D&DB?>T>O+o50dF8e=7{41;8>)*CRzeMzb-Pymg z>g%Zv7{Cjy{9x#ZMSqd>f0bcJ{hK}Uv+dC9`_z}o{41+I13zd$0{^xhdVT*o-p7vk zDI-7px6+6K483iKUfMS@jj3{l9I8Uf(~D_tCrYllRf9-n;+R{IEm6MBZ0_ zo%mN)eKuyE2Jk{FKNxy_-#y-c5C6)rXVDmU=noBG=xsan;mJJzMdDvs^&+3p+ji*F zL?5OV1^$&)Pp=N%T=iktF68@D{Kj?{{ZHK;YZ`O~?{41+|hDUGPUH|*BK2`4jZWUI&cl&MIq1W#j z;QI!szq0C=$^j^Np~nvWrX$$@S9G2hdS&EC{F?&whXye8wjKJ}^8JLDM6axRlz~C@ zwjKIKqQ67*%8B&09r~E)@09o{t6suC;or7Huit;TS>mUx`i-gE*59^6KT03rL*D(w zzq0DR{%t$-W1_!U^va3!wjKI$(Z4|lidcTis`oy>ZQG%5p3n7vRP@TKpXJ$p+IHx> zPiK9NEI(z{w|L&4VB4YB?``1w8>qjs>W4h-&$dIq^e6295ZV7KtA57hZI_>Ihrap( z*3XvtS601u|8Lu&kBfe1Isc@r`d&}`Y&-OeE7shx9!kp$@fVHGFh*z zdhhuc+YY^c-{dukzq0CUJljv(4t?)NuK%Q&>|a^+3po4}>z{3hKJ#MM&z1TstKM6G z+YbE#(YJ|SS@jv7`rCHs^?NS(z6;u)vg&8y2MtITdhF2a_h0Zm7J% zFEu~x(054w4VkQ0R(-!``P+8r8?Wa0eZ4d5l~rHr*?!q}==FOw_(7&cLv8#S@nxN%ip#`AAgGVxuREA{ZfzKwnN|l4C}8Ky|U`P z+h5xb{h;XAh+bLs-s1<`4*iTC_W%Ago_}T47kk>DZHK<*Mb^J8{*_hlt-o!Dev{}o zi(Xmvy`KG_ZHGSp750zojA;MLs&}LV0$%8`LtpqB>+$(#=#`Nl+pjv0-nK(O`UdN7 z6TPzPTRiRGwnN`YJ9y;XSNiYDs`t*nZHK;MnDw7Y{FGHshga?ZUTEb9L$BZG!uPr` z|H`mq{=NG@+YbHs$L#;h>8w{){d7RzMhsx+Z9DY(JvMxw z4SHqdhu*vYvF*_7_uKG2H|UjBUrc3S0K>m+hhD$;hVQ>YuZ;Zg-|yM}+IHym`*8SP z9Q4Yn&!aLhfZ^Y^L$BYH!}sN&S4MvL_uju}+o2!dmA`lPn;`3zRqx$@*>>pl`*rx9 z9sDb+J~NFfK?4}|x9!l!4rc#P$oi+O`Z`y=Rd3s&cMf6wezVxWvg#fDpaBW|+ji)~ z*{nZP^vcMO_|5U0f3oe+>-YNb{XWdUvg*623=ClSx9!mD_x|a^+-sc}|JM{X!M0`IH^RKLW@AF5t9eVw~ zBEGi>y|U^HJ@aqdp|3rS>+i_^Ls|9S_G8W4k?v+dB&S;+b~W&f|N z`o$i-ZHK;&UQmg=x6S1FS5|%8Gyk?7`t-%D?+CG8S@r!XR5%Ubg;str^!hzZeBToB zSB4$!cZsL|wjKI;^8HJE4-3O&k@T{S@n5vP6HVJZ9DY(eOr9*)@}dty<64Cc5vkrdd&|z^bHTN|92@)&?~Dx z*Asu+4t>uTc059~|p%3@5|7GG|S@q+d z`rCHso!40Z7tt%LKJ0qm>fg3QpC7hko=e z*8fPae+a97*wcPBKkU#C{D<`?N&8b)eJRBS19+j89}K;IKN;Us#`05!Jqx2}`(@jq zuNYzf$4LB@RiEwgZ`+}7`iS+ZqE}XZzDIA{p*Nqh{uDWXsjT`^PyKB>^!mMMd_NlV zudMowp7`5#=!d>!|MVGaSFf!4LeKq+wjKHflllA5J)&1u{r^SWR|iIQeBlOnDH^N| zPSM~&iUx-SNO0(~NjAw6*|57YiUf)~lol<;rG?^JiWG`V(Bke;pcD$Fl)iIj&dJWs z?goDE{qb&l$$s~oZ|u$;pELUSw<}s(@l`qU`TX$e9Vq?iJaOii)8_A3ALYp3_nGuRn)>gI znV(t9|Ee7M2b)R$A__la=0Aq-0x!e?iauiGZ|orXZz=yVMmfH}8Cv{RIr90ubUZ&D z-#=sKXNP$`ybz=RsvP;9A>t^jPw{7r&(MEAxbZ@ad{vJ8vBWo1_!%=_pZ{0o$j@z& z{uib6W6XSg{#KPEpU;P1N%3dQ{1saMRprR%^W^b-c}zdX%s;R7A5=N=H%^e@4~4`M z-ydP-KhxizB1is`$&!zDa*)rM`TG79RgQeG>5{*J_>7tFrR85$F8E(d{w3%qqW@lm znGe&W;zAst$&o)87D(aaTTsp-pD{ke{G-o*t8(Or&6E6}iO-n%uQmTwIr951ko<3n z&zSjo`K`*4Kb82~p`C}}XUzNsn*XXC`Myh}|I6}AK4a$V^9QON`R7+k{x$NSG4u8Q zhbl*YxgRAz6O?lpe#Xp?hJFEFhyye^@~3W;{NJFQ67eU@{8TOfvOda@-+znbuYvCb z`HY#b+mBb}$iKZq@<$V&G4nwzF2opqRgU}xzes)r)xV6Hua{q{9Qo&opRuD1KV#-s z)yi*Gj{MBWr2mb{f5y!3DTAMu{;C}LeThGT_>7sK8_HF@5C;(8Qu^;j`JXY$ z@%z)?pDIVbA1v@d-BXnQjG6ydd;h8&`E`Di{43=QaGiJV?{;C}Lsl@M2e8$W_sKsBEBY)OY=|7$ehUE`q=Ih_TDi{6}-%9%* zQVBEvy6(RsNB-Iu(*LH!XUu#({;C}Lw}~Ibr7kr-A9m80DCEf^)z>a3My%Do6hH%vptO74aD}zdw@UV!kRzezPo+ zKZy8@neVHOU#oKD$K{s%0OB)dz8-&7j{F_OA68n%pE2{#Xyudam7PVg(Gry|#{it%}pZAge$8r0IF!S?keAY)f^6!_E{7%GY%zQhn3&0C; zfTE8W`F)#8{x(WK#wf@9zZGu05F=leBfn5H$-hs0#>`(O*bcrbNB(Z&7wjb8A7kck z)ZU*eN4~v{^xu#8jG3?7-&Ezu@83@HdlR2A^WSRWSLMj>+g|d=5uY*h<21f1NB&Xb zuOdEU=KJ9fxbRPaqK_E)nLA4Ve{^$*bcrbNB)wo z(*I?|XUzQD8ef$oe{7KCSEBEaG4pe4d{vJ88N|Oz{xfF&YpB=pLL8vzBS!wjFzNqu zs{a|I9MeAqZoCj9UzH<2w@LExTu>}O8KXY(y)xrJ4!$ZEe6!?d4i1ojG5n9^Iw%C|Mm#UKSq4U%y_5b|p!8$R{LJ_RF8qUhRgQe`|0KT!@fo8&zP}{6@nXIz zNB&gen~Beu`F$lljjzg)|CspWiO-n%^EAFHNB&r^Y{Kmo#AnR>W*T3WBmW@rPZ6In z^UrI1RgU}x8KnO&iO-n%FfA-D!~u#vV&vZ^eud65|1(B8rhgdRcp*lje_mqt8(OfWs>|s#AnQWJ^ZR1`E7~6lK707-&kwEsdD6x%r5;uPkhGA*T;`l zIr7KmlKg*&&zSjo|5KGC|6qQ}uiQnZKV#TZzvY<@o;3z>ODTB>!*rpD^>M;t#m+kMLiSBR}&elK*j6nf{EKze2Dbd{vJ8Rm5*fe8$YTYyA&Z zj(ndo(*OR%XUzO1+W4g^NB+L@l0T97jF}J9!{S05py(q;{=CYPzlHdWQI6@qPvfg{ zmOA)@*jhNCDffre8$Yrua!Tl9QkkCN&Xh%GiLroE&i$;`C|hl{}%BXGv7z! zt8(P8=qmYnf@S(MW|h@ z_^KTFjr&M`H1QcTU!T8J<;dUHPx2=dpE2|OHUCvP^0W7s{B6W%%zXX(SLMhb93uHw ziO-n%K3e*#a^%l5Nq)v|^8GPpzP|rJl_P%%7??rbio|Ek{M%amRXOs3(G{8c&fbAKuM^@-1z z`6KZMT=*wI(MK%8KT`4wQvYEpVU*+de+Jei;Kh7Zj{L9_l5Zsc88crWKTze!pO-56 zUlX4(^Y!+lDo6fp;_oItW9IAq4^@u*z9*&sH;B)e`IrWHAr4UV5hFi^`0t3%80E;- z+yAN@`8`ib|BLsK@1HUAOKSN?l_TGNM)DgHpE2`w|5Z8iJD!#N&xy~N`7gBfYpNXi z&Hj-5>BMKue7*ms%8@^h_&*V!G4u8H3#uIXJ};#I7l_Z8`D?ZBN0lQ#^_Apj?kUrs zG4oewd{vJ8q<xo|Ml?;RgQe$ERtWekW7EZ%-7q$ zsvP;*b4vad;xlHx{{58*ATZqq? z`M!{bcp(l@^bsR}D)GaJ&lu(Cx8D9%<;V{!D8t`?_>7sa?_X8r$Uj<4^6drX`(w;} z{r#(Q8Hw(f1UWJDg2C?kL&92LL8vzBS!w^meT)a#Al3h zjKA)`Do6fp;(PnZ^kbaH*XD0jIr1O3lm2)1mVCy{f2O6MDi`4ol>E9yCI2#E=G!3- zcp(mee^fd0J9n1+&xy~N`MEX!RXOtec9Z;Vw0?*&^Y!vel_Nj8m*i*YCBvVFA-tIW zdiYg2@~;!WDDfGy|2kilBR{ye^nVhqpJ2>lffe8$Yz_kXE!7sqRr`Kbxd?xf*o6K1}?|Hwy?Bj5WA$^VJ?jG3?R zA5i7Ue>_z3uM(dz^Ii7Okv__i?>}7f|0O;%{zxtUk%}DokH3-p z6yh^xev%e`RgV0y*^7saxBpc+^6$@;{1e1y%zS+B%S4zH>_>7r7sKfDpE1fY{q_1!l_P&2@ykI+9qT{F%-6>cRXOrk z?3Mm^B0gj0$7%0hl_TGKzvL$qpE2_fYV9{wj{NIKC4UL=88iR3_Wh`G5&v_Ne~#l% znE8HM{kKn%BmeD1$^V!5jG5n4Yrm;-GL@XUzPG zkO%NW9H8hUM*hfKl5ZnEW0Yh5d8YZV%8_rsEBQ-_&zSl7HNGkr{@<7Uv+O@%<~P&6 z-;s(O`THJ8{yX9`X1>4XzbZ$5_9v2G=5v|;jF~@Jdw;52g#S;;4gI7QhI;xoo)nErbIN0lQ#jQBqjpRp&tDo1`a@h=dcu_wMNN4}l-uZho? z`Fi_Zl_S6B-!lFs3^M&0GrwmMNCdbL2Ppc8k-s)y4k7DJe8wop^t%l=UWk#e%8@@K zzvPc5K4a!@#UF6tALOfYPAA1(i>a^y!BlKf-DXUu#UmJ%0Y^k0=D ze;@JR5}z^icW8W7j{KCu(*HvJW%@H_{xglQ%8?&eMDi;UpE2`iXna+U{LDoqKbZK8 znLkV8t8(PeBmM~DGiH8Ojjzg)e~|b`h|ie$DH>muBfqV;4FB)MXUzQE_yaEd6QJlL zMt*QH$uDS>>CYJDn1B4?#tSj>RXOr!5xeAa^!~*uybvQ_l_UQs@uwOkpE2qqe;)pT3;!Trl_Nj0r1XEYQSuq1KJq(i?Ppbv{EcNP z{fW<*`TF^DsvP+%%1eIE5a~Z-=D*d-e^rkBB~>KOzOUr3AU+6qHIr8nbB>xxU zGiLrQt^QW!$j@C*@^2HLG4u8LFIA5GN`8``D^$Ke#>`)!`LD{6f3T6{S0g@S=IiHI zsB+}LZY=p-iO-n%`uYV`j{KCSl0T66jG3>`|EO~0uWc^*^N7!w`TF_+RgV0v{*u3k z_>7sa&mX9A%bCUJllJ|na^#QfA^9o9XUzPqTKS>Mk>9wVJAHzRG z%Rj0d`C*BYzl8XVng3RczbZ$5|DlrqEAbgKKfC6?Do1`~isauWK4a$V{V!FH{G<_* zpDA3vKgP_@t@*FYkw0>zX11|i7d{vJ8 zeWRuSgNV-<^^u>VgxxGhesAsLGLV z|4#ZpocN5HKNiA(7h(**Do6fX;(t$k#?05pA5=N=A1{*rrxKqr^Y!{il_US)QptZs ze8$Yz`yZ+t`Tdts`a?q*$Ilouf2xSRBmSx!`Ok=NBtB#2r{tCnAP!LU5hFitgYMtUk zeXqoa33n#^H{n@?s}G^@?UVYw3HuTL8G`OZal=2b9nJ}3G2{a`UjyKW4+HpHLBrEE zoGS%%iotm9A?beu!cz&iLtGr%b4R4S2jNu0Ul6`ecphP|qf&o6;u0_(OZX(=l7!z8 z_90wpxYVyiI1q73SPw?nNZ9MLbU%jiyxd+HykOld35N_5` z;x~jN+ew`7OBtTXo)VX193gQ7!p{;U&H!l=hX2r&5&q`$hW^<>nSPi*5c|>l#{7kN zH_0(SA7%s+_x6UO|BcoykjMVui&{42lh=K^DWDsce_OFp=d4wbkLVc(GwcO(27 z;sHOs5(!6>{2Ri9M@c!_$3gel{teDcB4&H@j|sDV9bC7Day~B!?Gqtp`$K523NhQG zLi<~Y*}fLqgF?*qpwK=RVzz&U_SO)yJvFr7hM4WMp*=ptY;O*zkupHE?VODW$#xFlh; zr-b@!PbqtLiTV7nI)wQ=4YdD^`fT4hg)rM|#`6h8{OG&_JnsN8pJ#yQcL{zOX^$Vz zlSjlm+lN4V5Qy2{1KMXm%=Q=1-UVW|XMy%Jgg)8RKzkgB+1>`) z_dv|{JJ4Q;&?oyJXnzDT+ZRE5Oakwa_C?UX6JoaCg!Y0EvppcRKZKa=3!yz8#B6T| z?du?B`#ESY2Qk~jLHk{Z**+KA14GRAzR#B84e?O`BhdlzV5 z3^CgeLwjFBpX`6(`MZeud|k9BgP85bpnV#|Y@Y_#DWAxK3@jcQzPc}(zw1C zF|Xgn^}>jGJut3cM$GGzaXm6(UXP6Hs}b}1X*{n4F`q|*=dU2<^HuOXOvHTNC7ur{ z_;lVQo>z&O&#T1q7ZLOMif0LT-XiM6y&hv)kt z=JWgTyhy}+9whcH5cBzhXpdK5vd4?(3nJ$81D6rz^APtEKC@ZAFI-QJ`n+B`6ZB0G z@B3TU*VtD<>`m>B7KE4Nk@b&(a7mKm`L?Lf=hxzSGl=;-nFEBU4wwA9g!fT<0?*e) zeLi0o&-+8n=lK~3PhBm;kLPKjoX^YJO8BishW{$zgQTAw`aYVA?E#AXb%E0+j~I! z5{TJ;1fCa&n9l=5`xio=>|vli3&d=%0_}4kX8Rj>ULj&WkFYu6I@DeZBfOE?W0MH` z4v^`C>(!9Y>(PEC+}|SQe-hqD?=#0IQor#i`F<-BP9b>{!mA>rybs|zLnKZjT#o#o zM7S~G6@=}Se|Hd$rtsV&Jd&`tj|>m5CvQoZ&lgN4%;)W`CCujsUMI}wDdPEum>zt- zVJ*Tw4`q7vApDrV544ws`fLwtAz?nB0?!jcIiDADgK*e*nH~koNO@;U&(?%*o27qt z!u}-xjG#&M-q1uQz`$Na4LO2y9jrNK{yQm z?}Q(Zmg(XM^r>`zUt<~G8iWs)minCtcdjV0neeP~5`Rhfas`RMCA^#T z*AkvlO3Duse)f^X7YL^keo1&O;lh<=_;yfu>JZ*X@-Bog6Sfmx02AgI-*JTF$p6)Z zk5c$f6Mj}%>fa~qA12?|e}oVAk=VP643AH^#FYpqB}m+i@LIzC3HK!Z!GxPZKM2D& zk#I-oS0J86*be;~#7hZ#lYA}X#!~(hVeE&X{$avx$^BHqXFikiON8GN{~qB5B>#i( z?cP%VAHwHbNSvdpjQ`Xo5*H-gndB7#LpU-)P{h?r!zK++)$mpgUza#DyZ4ud3sn>T zWn{Up#A*7SG;GoICrO-z`AanUPn!Ik#F*6LdZ^(H)kS#Hc(}xA@tLK`zt`~38a}Au(-Nn-cUP1DrQvrP&RoMeJ#uNdpoYt6_)`rx({OhU zM{9VvhR11ms)lE2c!7qOYxqYEZ_)5h4gad)BN{%X;for+q2c=)ex~8q8us#aejj-> z{IP~BXt=tD>uI=&hC6Gxhlcx0oR+@^YIwYcmuPs8hA(OOxrV)JiukAbS4hKUG~86f zoiyBA!&VJ{qu~`A-m2k)8vaA#wDRPYhO^ev-lvAkYPg<;yJs4JSfOe&fp(Zr<2KUjP6pWRZ^VE zY7aJsM4LS7bTlTLtX)kubC@YMG}(h%uqoDNv68S;tKjAz;+K0pxBj_PWj_rukp@$e zDKy@0GDMn;VI~Min5~W>%w&qQnM_f23`V=%6l*tIVhtgd_}DO;E;q**BTR;9W3naQ zZU~7tM~9`W5pFd>ayfe;orp8qLnBQ#gFB5_Q7WjE{|y z!iczdgE1_uwgEyfQWw3XhX!km%@Av`#uyS!=7>nUTOAjkH6q5CV?Ztj5F-B$#8v;H8Ec{ufVTEwQ1-c$+cW@ZtWzE02S>9&Jf9 z46I}5V6a7-Lrv+#UDfw{zn*ZDk|^|lQg?ZIX>W;wFwl>ZGQ4KJ4{({aBR?bNzy7jQ8u>>_)pelrq@==8YSwXa5q+ zws^>rAC!?}4Fe4!27I9yAn}n~EpZTYqdD3o`XQzWbF3lM5))%gE7Y)f2#q!wW8>o- zg$Sz{jG=aO0+n4t6S!u(0qBNABmA(LqD`T8cu!XFJ;WHQ6hz<#B(e?4A%i2ap-im< zx^Yl+x`fUR0h7%H-=?7H3FbyW9Tl71Vl`3zPe(02E)2bN(TKNNp++#+>=4lyM?z8g z2lYj)L8NKcr3 zanUA&%?!nlg(?T>2AGO!kYtMAg9^vCX>}18v!rMG60^sL_Z8FAZwH%2L7=m zKn98p7nQro07Z(PRAIJ+I+(}eAVCrh;b!o_JjgXgg<5!=pCKy2FfiT-wXG*5S+H9T z1C8zi2A&NBC(>#+{EZ&~IxXM1+WYMY%-N6cYm3-foF9#Twj{ zkg6CG$v0iTh<|z-w)c2p!xxdRAE1GW?CJ@8$?&qY0>Pv_H5hHlv7wGkh-D49C20JC z%Y-(@+b!<$et5Le7K!f}K4&PD(EZb8>T(_w${xxHXH8ouls7gH1&&mW z3Nykt@BE27BvfeXp^h=yW8$OTMo1CiH=#3aq1ddoQj;=Ld8N`*4}}g0mG&^S>7jtb z&l{=+Xare}25LdMNjQZk^MJ?z@YQI|Q)e%v>U(tJq2-&t3bzMkTk69@>W23}dz1iW z-ut5I?9cz=g*`bAi=K27kzJ>H#}*e2wRMEm5)Z}V{{!=X_xXSF%yY~{CkxAvP$P7h z44(6drs&Rl(QR_HTtrJz=8_N9cBdy(+$w7yx6Loi=By=(2ZJTnwPJM9N~aJpL_y=v zQ>Bd#Imtb~JV;jYeWX>Q?&3Ev3i~|Ju8xwGjpr28HQ(c!eCZ$Loag+3^||QX%BqM9 zD0u45ch5~^Of>jPut^(Z;r3#fXe$|@0F5?6JEV>WH#L@fkJFSqy?f!im1-o<^@j{) znw~q)J}4=q5@$=-b`B}MH-{S1=PdjPC0!Y&r-#!_bGXnyOE@QhtLHS%_TP- z3W$e`+WVZ6;T4~WyG9r}iP_xo#yluWr)=ifs7Q;|wbcYpNewyJ1YeUaK1P0^ZUX5> z)s2>uQ=!H`LRyE)Q}0&Qkp+r7mtu@j(3!VbVhrIh zC;-K%=r;5Cg$>&E&;WOYjGD8U0ppBjt7vr6fP(Zx52;Q_NAA7@lMca?sCF%v1fyVx zfjr2GTr|0k3Ak7E;FEApq=P2~>;Y19FD5Pys;@9pv?0N4a($)jvBvlShec1>l!R^n z^tADvK_!~5jw5N(b16y=ue$A_tI;66^qhKc;t5Zvl26YePcqTL{@>n!pu8vSLgz!m z=C*?`vjKcJayrIfi!v(%eKazhR$#gggFrto3dXoCcKC8(lqB3uhKI2sV{|lRw(#&+ zgE7QnRf-oU84L!QZIO;I+(`(1LnoruB-#{^@2TBk#}PH9>kow=H~?c%dO-p)4K-Tb zXxifA;vjVL(@=r|V_)#l6l1X_TC7nx1Y$G6@FfhgIpPf|Y%)S`UJnLl73TzG%|Rwe z4N>VN$v@Bw7o%e$+r(G+;*IbCx+TG6wL-!xiHO}096Xm}va+ddz$)0arTrnD26s9% z)!;S`9{U#TP$YbPVTK4CFOQFNKLUwSq!Q72jNB1TMJ3K0XNopMpEw*_b4Q+T^Pbvm}baZ))PW9fKxS&Ly@)b3}B!%q%kc zZd)knu*CQ5p_YChP0#1)=zCC&fpI-G4z6mJ_~AJDtgOsXa}r5blRioK#$bD>8_dJH#xZQ_o%rx;ui%PU5qm z?BZO{xEnqYb4U@;gTr}*_*gYFy73{Drr$n5HP!;fk?%lG|ARFQx#pZR=^++|F7!mJ z*{;+aP!gwAs_EC5@6&`_tmGM?t-gMf)ng{KtDu?Su3&0K& z!!9oaA{1$XVw19>9qLM(89Y{#0(8s+YvGul6_!SX!Pt3pvH|9wVDW+*HF8O6$GDFG zI;ZL3f&-`Afb9@$jNpKBv;hR^EN&C69ewo% zv`<6K5wg9Uo-E3&E~8Odxa3BOd>pis&CyQD8|Tq^=z2SIqhPfov@ju6;$R7!`VB#I z6p~bYtzv9b44*pri*gxLQ&MQ8bHf$_EW{91k3t&jBR8-e=Y(Uj_EZFq4kdvN6_OV(*p-}<- zAX;9kx`TBr$)OWdx3-i;TOx3kj~$!;1L8~(utE!_i}*Lx99SO+feeTEiNhuIRE(TJ zpuM0zG($4RLI0ne5I@NU{4E{8Uy{$ia6Cj3ZAM%HhR^YrA<=A)ggOjsj!wZHTMDOw zlA)f6@i&DV2VCTx3Y>wZxcicm=2z&KoW1Rtx@5@V7>kGC^*0{u<4P^&o(mTkGJ zwvLY#YnNJrLv|=Ty5l618z;N+uvd@?g5|-zfS{H*r>yw`Ps#D-R;v--gHvz=f?7jV zI*As(9=?E}PVr)_$~}>U{${!Q%IfMgX$0AEO_qztfg!j=%jL0?wz~Q`Acz?TdO=;| zV@(|`ILqT@Fo<}DTA^wSvcphc3us!wB0fX6*hH9XFhByv$Do>_Wou3k=~GLn7HwTk z;U-uYhi}^e3z4I2_9#P`(F#<200SEM@(hrbapVswPZfkCz`-Lc1V{Wf#e~Ksqceys zAp=4UFyCmy6>88AHYE7sV|d*#9tk?4>~nq+5f4Rzd<1K{tx!)GV1X7)T4VoSI%tZC z1E*-T%3zO%PLsuAhteFo^YB$AVIgU>Tg+aD5FG6C0$thKgwF>&hHkUnj0plzl$mDe zx#MzXv9!Vf=_o$UFz3h6k`Y4;&QD?RO)RZ+ek#lOaB@F9G}a#NWq?!`Nf|EILji_x zKz0v@Q7TC3@Hia6g!BxDg~|9BLILR@zwB^a(GCFrKy<>f%7Q<3Q#87ZfAA?aF~M0= zEG!&?zj4ramEE`)_&Q)cDXf8meo43i%M_U;P(p-aZH#}hxeR}yAK_SZpZA z$Xpc@A1i-|mn&R>DjAbpK7jRO)RX88i?TXfz-_kzSS#i#>1u)r!B|lekkAwhc~_34 z;wOd04oR@6>tQlR0SC%Q%$LHS7+XXTR58vdC?0ZQEqZ2&vm0Wui7xaJK?5bu0;_3* zEkU9?k2zil)A~e&(5EyJg659sdU_t_8Y$78hfmOgMT2l!Ji*uyhD48nRhL*r2zh+0 z%@}Ti@6{Zmc&RxpGE$ty1{2OP#xNKb1UA%7n2+LQFB%&JR;a|_W3j-?c6usbqSGU6 z0}Qm-;6s6O!Vn9ulEu*Jf=vKccm<)*veus4#L$TLP<|6>eEP^3n{cGe!5Cl>H68^z z25OWfsM@elV}g9fd6A!D=JqlSg!Rf0Ch$V$aU0YOVTNS03AcDaoDxFfLCY4)pPUUw zqGIuh-73BX_)Fg^{EEh}6aK>g;x80N^f%t|1AiskLhaEZ$+1Q(TEwSjjtz(JGTs&q zjRvaJ<;teW`0#KTS)ymuY=I_(SQm+{ISi1qRC$sd8WL)SLeXH22q7ip;xb(^Gm}=! zLxBR{u1Hs?_#rgWkf98NqRl|V0ZEG$`)80e!XbPj+92{VeQNj=J`Y^j3tG_l4ULD} zIc#pD6C&T2aM2uwl^upllu^9h1s{Z%vE+vJ*jQdk9mgZgyWG(f%D77h;)z>C(mZtt z!Z7h+aley@0F7WvANZNF>jy>=Yt^l1piuL0t=b{N`B2%7!=?6CuWEgGOI5H~#peUL z3d;C?t^&w%ErhJ08=)l}igmji0Xz*9GIuP|`*u%ify2D+IM5jP6udwbx#Lt1O-hpG zMSQ4JAf2|6`Gm<&O%0HRuf_^KC}dI$$AV- zVN>7EA><0oY35HBm1bCJuC54>Lz7~lXD^MR$J8Vo^+IA90#HuYS3`T&V=#=lZ7H~kHBAcGj7>&O$4aE+&o1(PUGXk zMm`k&*zraU3qfT4i1ZrOmyHAu9JX zL?ChujOo)ykap0+GSR*)B~fu)%A-)cD;&DQoamhVu$~51!pThN2)LXVm1#-Mh8SaV z2>1&FE6~v4%q#UEtzJyjXkz?cuodnkqW6ONPRt_V%&mh6?P(bE2Z8==xzJG2 z!Gop@F3E&p7SSd4kQrdWixz*RZO>7i5c`SK0_HT$5(^shp|AaG&;yol7?9E;OQ`5QJrOxaVAaSlF9`)){0%*e&O4u))9T#Fjfp>|zzC z33g3GQ7TIrM`WD|RQihO&}1SNOIF}3Z^M~}O?RVQRqH&o?5rl^#O{hTkXa6*#-E5Y z!)i@zQ?9+J)mFQ;Uur?O-s0bTA(CUC%I%=U3og3=mraLDt-4?sn z#7uGeVUV7~x8hNaN->exP(OoB2&KMr4y2mc@0-weN@nhhx_^ht1my?hcv=lWo0LCj zdOr@wBVASnxtXe$iV@M45bP$q53S6jPU`^7Fp=m`hu*gVdW!V1<8&oX6^I3fE=7g2 zt{7tWpy}$O&~|w1P>9CmPokSH`WiUK;Z9BCLrS`7v1qH)-LFRx)iP8#)bgW(*!e?O zSN2AmB8;J~yTw^W)K!?KE{lz^5~1c7TGDWBcBQlW$N42;(=NVV9Y?xAQx=`H){L_& z6HsX$!hEw!Lzn7K$2J0UEUaI|k#o*D@J_^>8!w1AK{Hxxzjs=~iA^J`x+PzyQL!jS zeLXoX7U^P5l?d~+oc#)nduL-;qWev>srYzsLZdomc0Snd8uwrUzgd z9Q2JGPjw$fZn9b8t?8wtOcL3j&;#X^W{m_R0LrUVMOb-2Llax_P_g4q3^7FP z;tsa4Me`3E{IJZ-ML~21z%!JqodNPJt*_u-jLkC-OkOD?ym$2$n%S7suxZRegt8B( zz1$=iw!2vy_gu>fN@JXoX%rQ*B|$s-M8$|DkRuF=u2}j`1Xy;9YbBg@IUO7!aCP3{ zva7b@fvXyK#6+LTMI44!7HpPqJKE>LXgD;K7|6AM?sgiw35=5?u5w$XIUMqfu*Ko< z4Ci>Hr6k%)042LD_np5OS3S(1kvA9qgo9WYB%(;AUK(QaJ*`4{o5T`}) zy?()pl}MP%g|TtxIN{1_M+(T)c3UK(qNCK5Bh#)`GAOxiB#7x#@Lf9X2(7dClFAmO z#Fk}fzF;QNdCpByS^kT*AiYCjx5L>RqHt4H^y5r8wTm4dhbrz0PSb&AP&6!&gXR;I zJj!5(JMZJ1YGh{(wXG}zE2hmHzB;SR{fyWNfQdM+m-`5*Nu(`E za*nx+BCHmKorN&=5{9$ANRy?bZPrTvw->ta;1UuumDJ9cR;k3l-9gEchYh%OwdCs0w+2X z)q0hZQ*4e<`X%H84V1VcV7-GI14T~qtVT}a$`45R4^xhWabDOJZU);-*rJhRHEx37 zAjzVjrPRSvXA!)Ju~&8??nvH@VBg5pHK)kuo2qN?*nYIV*xd`;F5N zfY9{&ke~|9T6GL&rGju18C;F5L>z|c@B>Ld{;+;IKBnG>sA{fzau(|e+XuxTR$wck z_2eFDc^11$@Y9rg0_5I2hep~I0kkop`wvYvC`iS4F^wNNRGrf|En{J7c~asWm7Y#o zR!UEa^3Z!#V5~Kr;Bc-r@biL z5KOPa22CU=7DY4N5f&vTgX?&NbFbkS5@6-3Dc zZBnScjd8wO1H{30B`RY*n8#g)dmLQut-I>Dt752-bu=+WlDH(8Q(-U0LG=+!^$Xw# zI-GHlill@z5SFFV7D40mt&6HwlsIpNanVlC&G=ZHl*KUu9ENZST?p&|_9&z>0@o!u zh!`|VRT&|)T?0Eh!19RL6(+}l9r|he-k~LeIT^TeRuY;MW%s{r_?Q=T_m_LK`bg%D`@B? ziA8uWyE2rCXqvsi(QdHAr%nV=wsmR2y8XD_l^jx0PVqTb?{Tc08Db(CHX+5rc1Oti z+RUl5u3RUHg*@y5!}S*FPMFE&r${C_&gn=(XJ@4%EDDDaC1}&9J3A$HohBaBcUGu^ zi6AV^aH`SS7OQhW#6nMKnHobx^VE~WQZ4<+r#V82q~t1RKZPRLyTDo`-SsOn<`5Hb z5%+B4(oKEf*x6~xfN^~otVB(b8(~`BQ@o(UfrXM{AQZ>r)rC9G4ok)Kb?Bwai9rY` z*j86su+HwO{d6&($lY_tdJkuUGV_P)EyONX@eyh3KVT;Uj(ta(qG91S$xMER+O^zs!}4E37q zi93P<-5_cY$o8p2*{L_i)=wOBZ;lmJHrF8uaFwRHoQGxbIhgViOO-jj;2Bgw4qMdH zVHsyVgM+Ti)QB-mBo+*?i_r-AqS$slJM36thIOpxsT#2k*JU-39>j3)$=P7FuEFor z*$RcOCDsAkp!E<3*(CkhF*)lNiW^;()x+4x&>fXtyUvNZ*K=7YCp$$hQv|}?B=-9a z5fL#sln@rL&YDrv4ckFHE9A&yBvj@Wc&bAfu!X)L?v4Rx-B57pjcCY)nPX}?b)DvnB`Fg(q@CgNI(liV$cmJ6yurKv9pwQrx8* zqA6jQ`MpXqUwF{_Pzyks$05;C9voDo;1YZ>PYCTx-5frSjW|+5Xkm&|K9apiH#%|( zCLJx#9hK1evZQMlJ*!%QJO;RZBd2A!2sL z&D(=H6*U+x4Ky`U;1POOTZxn^hMt%%b{ezf`3~OwMn$xH9d)C&b4@ObH%cYZQFEDi z;CwG@N7)`13)dX49r`IyP-(;WV%P*q6*m?$)794}kkVBaLDm@@6dIPmbi_$|*B3?! zjxU+ZSs4#2S*=u>jZP`6ju@wHK%`dyedRE$Mw}V~MzEn_Q4OKth^?*!wU$^uq4-K~ zi*uE`Q5ms93Fe+1uj7MBxIK#-4$k=kJj8K&W%)x|W6J$72#1hj?N~VIQiV8sMD3GU zW1J0FGC5~Kv1yLB0+wsjxm&B)uwb$0Qx^oG#X2{lZo#enf@t%u69@chj@!z_{-}nu zJ&%0?VN0l+g^`K2FoO^cyJ}oj0>$oMH-fI*fDAI(-H5Sm(t>(!&S8pou5PJ zDK@M%92jJDbLCn)Exr(YTLkP9bc=Np3|!nx+o8(u3LL_P;OU0gtqywXPlFG^ zFy*PI1B{Vkb>SKa++VAOD=xXU)e<8k+E9MOwCT%kp{Ll{!L6XMbcIAXZ|Lf&FuE24U9$?V}zOnb}g>^K=M%a1RGiO8iK*iT0%mg?k7D&=Cw-P#;o zsBUDwcAFR=Ro-$ZVa?N{q6L;7NNp{5XmNlo?O+tNg%M0MnymN*K&r~x(tYli-D0QM z!Zse>mPkb#x+5df4$myp6BGziD)g|33$V0=5sij2-5SEKb4VLoL)tir!)NI_Arm2u zro|9^qhhR!TxqEX9;8^-b*9F-C;~8K{GPyf4iPklafBU0*HCPNb`va5uCYZAk3?2L zuLEH4!HwH4Do~n2Yn{!5cyzj>Im8NGD|9(Xyxg=1d5T5aw*{50mKL5ULBeFB_|UaC z00|C>*4*wkGdKyMG>s3DR!sEhOFbw`I+$E4SGgEV_Z7G8YR|;DQzy~4a>@nbImH=f zTbyKyso74(q#n}PEF_DYa$jzefttb52+}LMG?hSE$GCWg+icX-*VBR4N2IB^IN;b* ztBGK&jLI>pJl)8h+C940=zz0=On5hX9(^DI5smKi1b8GjYP*I|KBt1hGo8Ld{$SjM z(A{_HR!#ZB8sZ|ZhUxhVZHkV#Y155Mt<);Wvs~=rLW*@XhM1z= zxZ4S;G-y}2lqW#pt;(+6$=ulxmRGy^+<@sMPv@&eRk=^!jUQrPz3cPN&`FXOqb$}= z#!#!p<|bC6EXOa|lZ?M+H}bS^C&;xY0RfgR8&jC*HMpV6YPAh?hvTpeXlLt08(#y% zrKpuSdKD^(9_BFD<^=fQp&Iv?vqV#i0;H#xSUT&$K{vYbdVvJ}Mrd8NNNz5=L-12} z>|Ek4)UYaONiF!m@Yc|9YABj{KyzRDNY|0(zN6)kzt3AopVK91(&)LZHQ2X^cBvce zqP^_=O0;x=L4)+NgcQqq6K2|6o92oF_Zi&k;flF#u$DX;%O24U)&i2I;G6|vBFO~9 zK%Igh=F&=6IYYaEpy1{KEn6AD5?0$l{~%9C7=M6%N9{P{_fmpkU<<7P3G*d`%n^f( zuoM6-ji^>sszVtKtH_0!9d{zwUSu`8p+P?fP4Bto37JBa24ghKCe?XP90rcaa0y!P zsgD0Z2`YzXNT7_d3S*2ka>_$mFtrGQk!HGK;K^Pc!==p(2_7!XfZ+`nyPeh3hIe3n zp<_@eeU2DsOB*R`)3POa9cltgOk(n@X8rnJF33@v;2NB4N^itP3M^u;V0=VVo$rHCE*{O3M4V_?BMv@(cGM?qZ2XD9>OIBUX=)k0s zp_2&~wS=`1o1!7%V7(XCGh}Wx0(N^~je*sMI`2o5Xu?=B*dax6D7OLRB0Q+#i4%5kVUQkN2T`6l#4DZL>!E#1xuXDGa+)bio2SI11t1X4 z*|K`AS`GZ%J#66mu(mH4M$zfDePJ1c>qA(F?fS5`SB-A4T+mj-0%s3J)bNLr8$ zYQTq412d(qhL^~jayt|@m0%YT8aJ;V`hKbhrX@Y57k-k$I&u0wIMl!rp=iv?1|^%w zODA5!3lX8}G3f;by^v}aFSy=Z&Ar>ptHdq%nE^3eh2U96cy_`oV`eX}jPReA7wYyz zAzYWd@+?mE$>^0^!uR}}<9pW2;6CNNKA!$h@W4&eW?yeq@t^e#ju*Ms;$f*p6|by6 z^xyE5c~kaodOGL1agpzv3L)cjw~K8$dGL^ukB&7Rl)T|(&O6y=9sBF^#Ada8PtSjs z>Cv3u4i?CnuW;sR{d)~4Sh`rdRu4C4?bovR^uG%3{dH{H`s>b{&i-+*)c#@Z8cqFj zP~AI;HJ)U?(7R!opUW)I6?|#@rhJdK@xM|R-ABrc$*UicdR_VDz1%p z#ze!NwO^K&q+D2hW6oEDYg`?B<>t6sAN{D2ONR+Sled|BoiReId_-SXelk9Ix(p~vTwF8B=XdZ@&{#FCx!?#{Mu@JDeG z)&t&cswB?48@9{*YWT}CZ%2i747l>{yJM@Ctu7j5vo2_GXU~$#*-OqCIrC!j>jqz* zD_wU&_k&qiEc*D@oL%;2pV0T>Z}T=KysbXnoU*A|>&W$Ej`ctL%jqgLQU{mqHKSYi zLS94txA+fQRIq2eU(Q~y(fnAJ{|;>LQR7VSC102LeqTaRMAhuStv(h0yz_+@cT*qr zEZ3`H?M=ZOefJ!kJUVFB(fGAPmY3TwfA*Y@-!<+3$<+;x_@f_dZ@;-eXT3kuk`N4H^a}boHR81z!n?MW}a7J_vB?SW-RkQetdOz zpN@-%R4X&)>iVx9=E!+;+J6nb+TQDOXyb!&OhUMkXb;`XvxN*!*y>$oii@-MQ(_d0XyD<3@KqyRPi=@J~zB z$otv8FiSVTetQz?uG)Ea{?Vy^Tg~sX&VF)tTAnKd8+P6MG%CYaCFXxsGh*Pj(VzEN z8hUP8c0=i|CB8g-Wx`iezx(BH%Pa3%U)=k+@YpF&Hh$b|$C5ffoo#&UbxfD;-M@SE z_CfHI{c}Qo*xbJ9;W|gAobWHOr2WZlS*xA!`?&j4`=Hoga%C)hbH~~tQ|=zR{BZlp zezk|*8+Krn<#?f8#ap(|J?i<$!j^j_+MV^jv~cZ;ul5wavoPW8vt11eRb5+f_tjBZ z8;>4wcu?hhrY!?9G&$QXU!#ZDjHNwtIy$cUiT(t!=`x+>eSB zD{9Z%t5u)gkLUc_cbvRe^%rF1^^Xgo^CZ>J|$K0;GBk!BX(*wSK zS$=GbOMlM!r&#$XW3Qhb->TfLL4mbQUp?-&rgp`^CNasS&L6N$@Hu?@&$cm#7j3pb ziP~2=wnyKeqkWJ1=Xla#%tyNh^%?eR^PB$fGJi7v&>vM=uI`v*uT*gE{VSsyZtnJ7 z>ca7(_Fl6U8tUIV`ggin~#t7uCVsMgpC7x4k{bG&8z6G-K*!y_9~UZHqiHR{}s=(_>O2XT01uIYXyq(;8i?8vn#>TQ@If>u~`U6GzUB4DD3p z_O|H__f%f6JpPB8+e7Zov`(C~y>6qVpH@z;7SMB&*Vq%Q53XO(z4q73TaEoU(y;Zj zRaJJaE3oCCue+7>Y18(vw6qbZK<= z!DH_si)TkZ2ygWKr>Lo)B%k@J^NNSFhJV-h%Yp4{dr!$YY0T@8=c}9lIi>v0<0I=V z9p0hov-0Pzev){qZLg8z3f&7mG~sfAe}C;#cl|I^-x2+*?zk4V^po#@F1M>)%%z9@ z`+vHn^7lhV5568X^XCoyqn|y9+B@@ht$!km=WZO|Vb_SuO)}N1bgF%UoJ$^D`t0Nd zbHe9i*M)yMW^tz$2TM=*t#OG%7p@$*`?TGK_~}i*KCu1m!oO$k9{PFwh%QU2?0+)e zcXRO(Yezg-TV_zKb0(&a9jUQZM%!^t} z+8AC%wETC){bK8@KWX&kzPz^Ay(TOlcR1hvJ)L|%8`OJxjyu7D!6UNWC^jJP+{6Z{ znO6_2)#}P$-S?0FZf(zQzw}Zbc?YX{PYI^;f<=Vf01iyUNZE>vS?=AhFj@niIVx^~_r{o=3KUe+yUp;Bt@v|>X z<4XCLy4^BrPijb^GubD9Q+4^~TiN;#EWXNm;88|f(kTCsi(9I#^a}s!(#*-XX>=}S7#-C6jkT`hLg|D zN8@h%InP+SN2STxN3XJ1eA;no@v0AhYrB`7}N*x9qwvR|EE zHIsjytpVS6xq9J`)5#a}GjQyCWFF%F1&j~guU04X^T7mo&W8%U#kpw zl>Jc1Km1F$*U9BO@}{qcQ` z77r?$$N$ptC1XxroNu`p_v+Q;*L}yv+ZuIS_^fckxeZNMZ!5Sd@7Ygp_4u{@phhio z&wUhD@owph=Dl@a^?g(`>`K3yoyO$azaspP{lh9$n!deMjzLTOet-J;-dblGjQa9$ z@wa{}K3%o2!VpW&2VY#vd2QI+;=cEqg$`SA+xPafc>CgfPnvE>{N%xpugA_?SLk%z zAD@?ReXRe@nm_Nm(&v$}Z1(ftz3#K>_m*EY-*@qBwzq*kH+uJ)*|_QLkA{?~HKgW< zJi&ka<_&CC!ub8v`Hv4AURLLm{|swpCI9v`_CU_*Qx@i~*l2a=?CoWDtuOs%=SQDa ztgs=+;Meau1zsxADM!_f1N)tu`%&J}3%9pzT)+N;%E5~kU7B&M;;7kAD)nEsZNtP* zHwG52JEG*zTei$B?e$Y?yBza3O!|JyuDCnL>lS_4FMDVv<5%-G^gG$V+>31Omo_eS z@65`^qcW#9>rj7YO#Z7St4@f1(7Ms$7Img{t8uSd_ujV#Kgif9>+3>uez?E+qY1~3 zz4Z!xQTA}B(Lv*XE%@!)QHFe*>*r~?F4^x$2W#t)+&j9g5AV{W{NaJ&Gg~K}p8VC_ zF7=A;y?(yQkETbKM)SuNviNn&l-O#)(Do^xeg3YA_rXEQNxS#fxIMc^hj~?3_Zih^ z(E1BLreeLT*D6#cWO4Y>O7rtJf3<4O^e)FbkGWZJN}rhx6W05es9Wwg`{!{_?q%MZ z<%^KtLvIe+^v!?GyBr!bBC2}Xx3f3C=u&3dChO|@@zdAs$aUq&mA}fI2!CF(Th;tm zHlIFx4odf-TOud%XJ}9(F?fuPew9T~e z*XiGVv?0&eyY^(NGxWxuzMTTLo*DJu#xea(hZ3gx1huL(ApT+Pr|-(ePTyPVqGj~Z zsylBNuU79_yI<>vwfW?D`QD!{Tvv2|vC=o*_?62sE<>~2l_J*-+c(8Fu7GX%lvRPz z-X((UuQTqiQopQs_FXl`9b5Bp{XRRZ{l0o+OW)TKW$(9%&)cr*oEsH0hS-M>d;Cq# z@pV(Xe_pkUU&*h3GQvS=Dd2qt`q^xm0&Xv3N z-Kj$<@1jylZQ0vn|NdKEeQr+tzV4K^V>fp%_bg;il@W8MJsbP9#p1yu>}5I+_Wtvz zPp8n-I`t0h%XZ}9?toK!rffNT=6SP(l2dQ3?2wTBcDdKKspCrg^~tU1gosH^Q=g1| zQ=;$oGpD}|`}5%V3%?z&p0`ZzW5;hFt&}C#+^>DYXH=B1I z23|hiut*#g!Xyd|Jewke<$HLU4lSgjcZ;|~t|woL*o*MqLF{Azz-p(~BXzNt~X_lQL& zz2k1LD)4Am(UBwGl$>;^RnA*?CVaiwwq@p?E)x&T`)^R2vEv&*JJ`Nc!NbApk7oL7 z_|#oZ`Xv56IC|5NcXx^v`890Ssx@P;w;y+6d2-WZZ8JAIGpXLQ6;D!sO!;rrkK5+F znEW95ZN0}M4*6u*G^ovygwUv2cZXT49$b>6-ok+grtE$CAh}zQ1_Tr;gCr(rw z@}@zz@ga>C*M0S9xV`x3xZ1mC4Ow<@SH=gnr>$=1%kjyxiW_qm_sh9-)UzB9S1<3k zVE&BXCq!i();sTa8H<10p^W|FunPGq|J(MHA7_sHYe0*?D{Mcq`%1B?trq|O@kq;t zXJ6f^Ry6!>=&yZhzxDpL;Ht0ZN8UZahnZHYhJIcl8N_K&46N4?o}wrG#k&#tXW znLfNz`?@!#rM7?O(=1=%_P_S~tL@|sMf-c*O8lyQ*T0*XUhk`W=abI!M^>|a`}5U? zRp+)`Gs*DqWs{hxyX=?V9XU|6{IqPt9-TeYs$|6(=B*FDIMpz@)faj5CnR}2U79UJ z&{$JkzJ6EB{kuGPd*M&73^-LI>2>P6$P29&4100&Ot*PE%sE;myxTJV@}kJp>E5kA z9aN+2sD2lgA86Ecdb6Uv4^-TpJ>booD|OG;Xm&H!-}gk9&qtg%+~q;Qlhe_?GIZW> z{--(jiyhhVd~J@2wYFc{-XvvuNuQ!Wbne#s%Ic|^?-#y2=kd6y$l+S*n>5C?YVWW{qm!o*4CT){k@Pg z%kG8N*|sl#VxF59#y|V^X0V~=yy_AET}w{R{;2t_fz7fuYd+MRnl*R1Tfc1^_3~lS z!@c&O%`&{>=gTu>T^~5!uTGxXo7xw6VHtEW*ZHnxYplC*dTz6jT-9GjWb6Dz?s{{w z6!#gmGrIrJjbHaWRoSauhdllEuP@x9c39T-)2sI|RWI*=?A{ z_LR8&x7K|3M}w*XgJxu_vT5y@K{fL=N|})TS*=CYcTB1=@#OZyQy0#gde(AzXPc6_ zYvg}eC+1G_tleMa%hIk^leVMB?+-jVXm4QT_b+i1(hWjx+?l&lTJL>|AWpT!$mj{%+vhGY`rMvZ8Hotj3GW2xd zjW)*Or;2CX)@RI|VL@3&UbBuJ;``^a2YnV8e&6uh@NqpmJ~=lfv4~gFg4$=!56RrR z;la);&#xNZAlBBoXNiA87MClyyI{`VZ6XrZx8426zPf*yFFiJ`xLYJHXySFFchVmX zk|#8oR{2epYv=rO$F+IXX3`r|+mwu(=FX|QX6J;V_g+lS^Fx+g;Y0K0=oazS!;iAG ztlRJKYvYKWH3}@NSop`MCo>m0cXQUhxV{1Pt@|%LsT}gvfZ&?zbG#aGs7uiO+_6P$ z{-evkdK>=bjYpq&XD?Q>WbTL!?hpXzZUH|Xj`B$B-pP~HAZPR@0qvmxwRx`6-+v6jP4f-MH zgI^NXUvGZIf9LW?4Gwh4{!fPnZ45Q4G~NG4K$cE{reFKFC~|Yymqq;A3>miNMZKt+ znHp8=({TE(!u_@u+}o;PQ@^#xJNtTH-n;jonxzltdvNKA$>DLG=gpfnZ=n6xdQ(pSU>!0#wD_8J)^XRm<*M-T>D%plHf?+DeK2I&v5@IA zbN8?Pano8OzTa16#b53GRvv$KdD+{J#v+!V7L_O)?Y+3fg;5dhUzQ&B$HET3#=ZDMx7KO<7kjSQ&n>Oy1EYN}b)lIoGO9rB7Uc z+wQ9;xeI#r>RmPPvo}Y)XWorDws~##;4HV>PM-~9oV=8o==;kypYTkQU^+Jz@|wlzQ3`&RazRU<3>_szVo<9wC|29t9HYy)$KP863VnG|Ht$h?WdL~^+TEazHR5`x3sPGVAlPyBRAwO zw(r`S!RLQUNv(U)@9MCMuS-VX?=bAQ39&ajdiOmL;5#Hor&YbzJ*c+5M%`LFUyr|N zFP>@Pi=ql(=;Yh<33;BLK+ z%`2LkFl)ojvX?InnVDnsFZ%~%t>yc0xtI6F!hsjN4{cfdBdc|H<}cnnYH<2l7Oz@+ z587AcdiLvz;rY50PL8!d*fZm3`4c5(?CfRfFgP`8Q*R$8 z14}epJ#5wbo|gR&!gpu>?c~X*jDbC~{5N>x;6bA<{yd@WxT?$FC%^kmi;0Q#3;8!) zXPY$t_c~Mje1F)r{JW&2=aUo5y)1j}&kMJ+CuI5b(CixBhkFk!JElw3tXVo1o&V#W zDHo5P@4n#6=rOhDbsgF0oBCIF7HOK*_nVrNyMA=+{BPqL#a^ws^Yrc7t>5Q=*V!Lm zTimdz_0-&b;dz2i^m!Q+QEkkig&BJ+3d{B95Kf$b%p>puuA4n)^rp2Af_ygDpaU6mxs-{w4)l(ObC=k<)J4L5DbN z`6(N_5o%e0?#~j>m7UbL>j`m)x}Y6_sA^-~jp|Z*_=wKSwDB3jh_PiN*ZgT6z^jfa z2s~#0&1+|iuzeu0Re9RQsVXJsw<;fgK$a;L80AXH{m;AFRLp}WmvOXd0}v|WpLVsd zlrR)C6A9@tiJq|H*UxbK-Wu9uQ1W4!&XaZ*2Q+UdYWCH>JO`mFC;oxMf2I1o)077d zwO0a-$EqE;Qu7PL3Ol}FPXY-FG=RICe^|J-?=U=^3>3OI>}J2Lh6k1a4TWQLEimBc zm-PPz3y+`fHRN)D2@ZExs}AxS9W*T#Ha~iHJ2STO2HzfJ z(-9u>DQ{0%n?f*`a#AFclfv26msl=nMG{Hk;m&0SD#zTD$OhZdR>5U4&^s>D6Nn=j$pH&nU}-b5hR=xCTwEtYLEhbizB8ea8fnGktvV>)727Q)D%TeU<5 zLW%D*<^0-*NQd8C={AGW$RFy{JpaZNaPuxcj2r4@4$dyKtwKV8Z1%(LlVI1tYYhYc zVpC^6Z2$o6Lu}}=rd^Rh693{PpS@y%W8JQwi+P6lL^(xU;L7g?L+gq2bCc=sN8CK hjGLn)lwZXvQ>5bVb^r{7`~|rWi8|wRi~s-t004UBLg4@a literal 0 HcmV?d00001 diff --git a/tests/test_metal_iq2_ssd_grouped_mm b/tests/test_metal_iq2_ssd_grouped_mm new file mode 100755 index 0000000000000000000000000000000000000000..abe4063ebfdd5d85b0519c4624d492f55c115ba9 GIT binary patch literal 919136 zcmcG13s{uZy8rs-8(?4nxeT`fxq1OH&D4a%7(h+I#vCZq+5wWuMTZbA6_FwE)|fg; znbHxwB{N1@VKKE!(zcm)%1hg7?{l_*nuB91d4W-E{=fH|85~h-pXYzhJkMfXzO~-< zuC?CvuJ^LOkMD2%`nxA%9K%n95QuOY1Tgi4gB zarOShOw*f2B0IZu!NOUky+wWD<@kFDld}i7(~t7b@GvF~mEXp%cvfNFZJ>SOMF!j8 z)q7Jb#wbtY>;2t@p8#*(>;?IA3rZK_vIA zUtEyAAa_<@c&5!#VLgsYY)~lQ7lt$Z>}+BF^5*82ma22-&F%}YdAki>n!OR_oqv5{ z+u_lA5V*V6@B@3EU{WBoeGot=#kZ;W{_yrVkO4eL63v+_!t<1{ z|L)!wcjC9_fDOL}`#34@49@|_5d+=}&Kslec%S~84PI-otp(-(7~X9;+Y3u#j>9}BC?wr{~Dt+NSVTWfY6Gr(zrV|_ATlnJa`SWMLP}&#X({^|c zBq-kpzh2xZVaG3b{`}m<&*v`4?F%p84lluu6y^V2c);{7`m1lX&EL~@c$EJmcz4pV zGwPl;I%ZGU3+*R^!j>hN6dd#F;7<;wY#XPouI;Z4o~D-xd;TF_c6g<;=h7-T_Qh>@ z8nF#tt9@{kcOECsBF7R5jU06HQto_vp7zCUcsUXqyaam@<^K&jp(VX1ytnM|F5BTz z-Wk8!J=;>**}b6d32&9V4ZjENNKyWe;VsCYKRfsNYz&yBg1+m~EVaSQv31WV?+nl1 zUhaq>uh`(tDiwr`zVL!QZSWlXmfAVP^K$^Q$JyC-K)v}p*F#~4*WGkmP$ORF_}ST) zBkLsJO_{9Bm@s~_jT__nvvxYRPn^O=$2kXW5a_QntWk_<1j0c*9~|)`?RlScPqseV zCWFx!pDyuaHz51uJnzNUFK3KSGsjLzgRDkjEC4Sw=^U>mKg7G<1Ia(uYtMiC`O(973mDNZZ!p zig*Z~Id6Nf#_p4nz@3bqt?8yDb{xwv$}?0LgdhXJpGS+>^HE(gKU zPVBM`q%@QpZmT?eO2Mr8x$_qfpGZgX%f<7HN{3G@UZ@7~7Zj_92?sQl4u5L){8E&Z z4o`D>4!8vWX(#whv=BoDUiXLp`0NF_bMFN@Ww`JN&}?w?o#5K=e)EsurWMbh{l~bc z48OCEy`MYh^$beyoL4%5hNU>qDr19Ci`K1vt|w~-4~?_ui4N)4-=4qR;l*AI6N>(a ze>AEVKKtaQu^-R-btE^>950&J5yQ{xDHiK*%yH3oXS?dJPj}N_ohs4aDs$K0N)zd< zX`aVSE^NIdPGOO73QI#(K({~B_Hgf_gnNE zbf13v&U@GW`tG>9@6>zrZC`s~u4(NX(Rc8_(>KQ5eZRf~!tYw^nEUh{g}xkXy%v4T z@9g_1(cN8Z4SKtA@4k&^|eVdUEoAEM=uei>;NS*@x>-FU#e$umPsutvxaM*fOZJ_dPapQ7{_^d2>H_rgtb_)jvfktN&}1`uU=Uq`c~D%2WhN3e!b!KAm6Qv}U#O{ss}|#uMvx zE%UKP8Z(W(Gp%JD&B-b$^IPo}rj6??Kk==|U3>ohY5H`|U0Z~J^T@c`jp-h6XMU?j zv$#nJuemYbwf&gyIw|wLg!1nY)+KUr8qCjB#A}1_%%AyI@`J;-_V-kJap5Xf-QdlM zgVpV0TE<1TY*RJSaiQ>w5A6*acZb)9vI=L&> zEQwUBl1S_4cR((Q?eu8InjV#a4w;LhF><-Mu{F(I_f3V3&K08WF|B-vp&Rtn#s?c3 zP~O6^(RHE#^VT$$H45|r;a1icW{w?J2zWoy2DLu~xkU1S!oovNf}!*%+u@_*Ty4DP z4ZtONNGK2U(`8^?gmjOeZjvM2UzkzN(0 zd)SeVjn<8Hq`!^UJ?Kbh1?WKMM@t_H1po0j5KpuMF9BAYXAyLu^3Iu=lRcWi=bzP#5#N67?dHZb!Q=?4RqWn7(Iv zy8aq?xzPWrARQ;xs7?ds8SuTE;Ey+^DlBIay8@OQ!Wi%|;#CBL+V{3?2+~D4wRLVY z1Z}R`+nClVEGFl6hwki?+TGwod;1GT+d2^|G4UFC1>$S`Dml@EJLrMNTWBAz+CJW9 zjF;A!WKF9zNN=?UA_VALdGPzwF1q#uF1pj(U36!vTy$qkU39+QvjOV#oIB zdh_n5^cOx-Sboh;Fbj;))RW&M3s5 z9NW9E(k54*T5gjm`=R}%Hniv7+I$rB=`#$pDFXblyxu%gr07s$P!oi07DnTO@#l7VoiiwEAQQCB7KD85NlK>=qLhi59}Xj8qDihC-75j z6#B?VA9NPBUIGo;%kogxMP*lRmpz9vqM_C0Ccxy6zlS(+U}lMS^ZtWA9)+ z81{22_8j!3qlxj+XdhyD6!{?JyS}4*kl_*9r`Q*A(494nCElccRm}9Z_OZIH(9POG z(>>r#r?IE5otUgY1KH&)=8N`JHy>!YK3!oshqy@!93UgY)~7O+2|Cu%sGnkmeu+<9 z!+5DLD`=hi>UQ+gdb=O$!x=uo-9m8lAoBo!Z{X|TKZL)72mQ+q+te3!LR{|{?ge`# z!Oph9cECDfhxPZqu$KNYtOW#XoqhZnHkj>zMR>MSj5%ll%vi)G=kXu?W4O-|+_XD! z+-8UOZeJV+4Ee+HW)Q5G?KmdeU>fb?-P9M>sv+RaN|2gBi`6-0^j90 ze=Xs98t1kv<}HG;&Uwua`NUsaAqqC<{cg=N-LeMtos$cYtLva^Uw~Ymj5EaSua!q| zwm2JS3FR++mZ3j|I%h%@OMZn6UQh8l#L$s0rztFA59D9Am3NI%Sk531`G1L-2P_8N z3Gsj8d})OIQ_m?ZFL9oQg8YxOr^xrM&8B!4{guIDy(t-Ia5k?uWgt!y=`SYR&z~vr zI#U_u1ZPkj=C!^`)jg78KC^p-kk9NMF66ViA3)rC2|60ZR}>bi-`b_H45s*|!ZHxC z2|5PpASSHOjqR{qVP1LAnh1eI5F^~g{x}1jVKsELZRcyDSKSQ)Pq7-Dyzerl#yjClxN0!sxRl)Cw8n)6Zz;P5%hW>)CNvJ|uUDqKg{gg_W$N(I*OevxcPpn& z^i)4ys8#Nb4p#dC)+Xd-Wm}ZPQsn9_A#0QsgJkOO<2Nfe50t4FP4HANi(jLx?fjtCg`XVd?_FxI8gf-HGxc<0dw~j<$oswkq2SURAyan7i?OI&gS)@kV7B>U8yg zUzv$IuNACQZUioyfyd0*Vs&h!hx$D7KLggsQT^2aK;5A(lN%q%U!x2fAFO^D`Lk%Z zFUnILIXb;@b%;j!6Y{GUuU0;a{=zYr3y}}auT*LP&ww=HKNRKpXj>M;spDNT8fiR< zwe`xpYNOhSF};g=`IVom&rWR_N9|XjO%0xp zM!kfU`_xfFop;oE)lKTNcs>!&-B#{Zt5Am_fBX4|>Q?AI)hHW?I=>)%jXF`)JJ82Y z_4C#5sCkslMmUJ@BDJsHrH-lIEp=Ko+S-xF2u- zS~s0jj`Hn*Wil%)yLnZ&C-{C0<-x~xgV#xr-ws=t8T_mfJZ~Q7W#0B5_wPH~-Nv_J zI}`YK9duuTuZQqLybt{SBJ5d5V9zoGw@a zUEm|w*>*EM$;*-_Lv_##_CzHO*U{X6RR*~jYmzULYyEFc)fK>=l>mE|3^u3LNGBpK z;X<^M&&TT|?*G^vNVcvc%p$wAk5_lyhZG5}orl8(5$r7PL~&*eIIA zlfd&Nbd)DVOx9JzEY~0xcjqz7LQ(LM8d1oRR-FGNx3(cH1WuHfEPX;32)o#mu%R`G z{I&j^>`0pkcCbi|YLyGtR1~1y3O$_i)k5B1%OU*%(hj@WY#JYA7KdFd7kOv9*I$q) zU4iK8VFdDl1SPSip0TXyp9oh$k95Wq_%j{$#d-zoQ52`Z7F9Ts=`iQ(XN}%J?lkNi z@uEtVR#89ydDx%My75yFx^cxtPrjkbSiWoi9F%DR|3%o8bf{~NfG=f5rD|@&uK5oj zp4Yr<{(G=ZDfjQ158ZD4Z;AWI4Hs9cruyudf5l7eUE}7~Z#~*PiMAi0-n8Sp=40Qi z4@`nhOrlY>L7ufh)`3q@7E#GMUBq?r31Vh^VDW(FY6)}_Uy1Y`6Z8FgsVMW@()Vl6 z_2ixS=^dfXWmwpul%d} zuR+PhZ@WEGT)T0nx@_e;%Cdh(Da%d`R*%i!mXx^ivN9N9=E@J0sS2jcW-8^=Z%m7hVLcrNxe zWnN^E`iy+JGBx5;hecja$Zo_#^A9+lQ#9a~nTY{HqkddPn$KgSuFRG!v& zE>f<(TJ*NECOS+#{V`8<{D|?5n?tqA2OzI(D%-5Q5wTCX1n2tmI3LR(tJFe1iN<-^ zgSx9B7q}oD8#keGE#v?tUt6!PTUW0xMSic4--`TJ z^-1KH;fz~_`oTDhZS@<}%hsSi+BoVjTZ8&*?e*7uqF%N3uWG7)u=)$N&x!{17Qje$ z>Te6$C)@kmg0`{fi~6lY+qiXqRlmAsyE=YdliC_#P!bN0Vr~vr?*pta)L#Hr1z-iQ zY_Q|APyGepRRCTCU=dEs08bBCg@Cmiu;v05!A}O9$(T#SnjPw*bw)e<26b#N{04P4 z;7=0Ze*yUWFh33I=Kwz&@F&^fKL_}e0Y4q^GXQ@|ANc8jp8@z&*6hR_HmE0|&-8Vl zp}z)Yet&~9wtoZg{{r(vV`@Ox;MuD_zc)99}W{hbcq zuB?e@Qqs82pubPh-&yo$L_gJGJ27t!>b$TW$}{0c$kaQO?Q6EFcdq*s>!MfB!XB?e zorBfeAoK7#pB38$ndi%M)-1O2V?mbLy5b{c6l9)Av8Os^#Ro!q2xJ&;#XCYe9_dw( zbt<(fw+4yD%GHp0Ji!O6kBz@|V9kzUQDRT!K*X;=URqg|a%+t=&A76^O!=BynsF_k zW_$u;?u5+e6Fx%e^XUi5exgC)`#A5Sq5!SD4so^b<4c|phbbxVwsx2LTx_oQ1@^+0 za}nOkUlBLLZv7hPHRRotDMQw9%XHRmN2q9eP}L3M<(PeKmOk zE6GJ%0lQ)pVhLocI(OI_A*)RWPcsb`>0@CFjKjIx5yR;vh%M0N1R3JNk-DbEIICd; zGb0vkeT?kx{y4`&3`bC=$Yi4&XR8OvPmo1A-glBImyXbFM?FU!I)4Ru5ccpzOCW1P zR$;J*FLb0~Kb!AJ!yaDjNW&gJ7wMBCHip5Tl`)58fDNjth+E;0`tRK~dvR+K?Ax%L z9YP-0j7BNo6F7ujP zC==pP!(xgd$GnW#A^)_%M!OGoSV4Y*j0T&=9V_*I)p9uwv@;bGl#Xwx8)d33;! zMm{tXWGJ#XkgO&-wa+gTXGcEdvk=H*qamv`ARUZ!z9?85%BghZtCS&6{YkQ5+pfl1 zp5XNzp^)7$Ki^<&J7Pq7vh%vLS8Qtw8?Dg)q$FJj)_5Fnu%1SJk{hQzD)?Ch`{#1E zMv}K}a-Pia9*q-oN&Z=~$5=4mj&Us|c@K6+ODVIw1UYXnU^-(;LG$V~A2F;=Bwnr-trv=fzk)SiZa zN&L997H9bJK|Y?f=XlNu(yu|{-~S$XLaqk2cd4ppVf83Jrwn)|5<%I&Jn1a_h)r&uouLj zZtf3Q1j=CHEtgapBk*HhuXLap3k3rqt%=ZnfAJOKU zqC065wA8zP0*#DC{ZIa^z8&XembfsD=>nlsbn$M6t4p|PP{ybI75ZUQ;Ufu~K%ZA% zV$*6scN;-RGocrCd5C;Z`oVVvJ7rELGwu{K-MOXxeD_7N&M(hev!amC8^HM{ixr(; zo`D}1Izik{ci)I8JU=Ui*}x~ zo_Ogq16b1|2!h^*v78sP%cQqaOnO^7%8I~0E+GF3bhmfmPx`fMqw2M3d**M4o;JXh zO`YxLHg$y?U+nebF4ZoSy$0U4$F95migq93F2NPZ_>^u+**}iry?-Bn>sR7qr3%Y6 zoHO3wA^pmwwb#*Z4D_5}=s7bW1NBRmdHe8#!hM!`hIT@)x>W8NDi_V!@*w3QkBmTk z1NR8VqAWBV`p?Si&{3`{libD?FA0UsaN`c;GY7V-pFOaBSnV^l$}+da;w8{;Wak{dyGW$p!_R&a<2nkT{ILM$`y=EP zlxx4ky@4+H`HOjP?a37%djIGetiB#TQhA-zjLRx|FUdy~u6nRwSo4D(%!I~*^i}9c@67GCW|2MU9MXAIK2lB#Qb@P37m8cihZ4K(X?mJNbD(=vj^5~3}n6INxT!77j?^dwA9}>xK+um`` zaGVUbcYoO4Wro2xd&u_gGCjmcF1p8M|ir`}*`3E8p)n^nIQi+OO0g=|=4vocAM zk0D=&)I2NZe_*o$9oT#kciOB#b9*jf-CH?@_aj^!lWM%=mTJ5_GS%40 zBYsQ_9|HUdV_?4;DfklhM2%uPb3D@rfj-Eu{)Q;TgZBMEta~@+%CXL|yv$r6V_Cg^ z7L3t(t=~m^T4SO)+!^b973)m&HVZx$f=g*%z;wu@xLz$IKTgO^YtUWvxGGto-4r|R z`r2ui=%EWZL4I9NKHtBgTu4J^-S0?4X5ELhLuTEA`>Z6hZbM8yp$Cbs!0%eiY%*&o z@=h|V^B&s6UKI7pte0Q1?H`iG9I~n&WwbBr(I3gG$B}Qvort4CEH`{dG3fbQ#EvmC ztP{zqSDxV9;#B2WQZ_E$nL8hV}$I!!jZ;2R5{X^ z2aO|*c_^neWdAD-m;=mpKiucbuVFQd1z9`9uvqA4a$-K@OuFZzM*Y_5@JSKgnCC4N zuTxmw7V=?+%@o7V{1(NKncqb0Jg!m^n=iqbB%nWuf3NJlK5A*Gt^#!%s-Eor2J{*i z2R{_%0eY7?#fxRp{>R;E-L108xKBL^_o^Sm{pyLhXFWmRQ#@YZ5tD}d*QvONoigvz z*u!(_UN(!=s0Q2DHeOg;e}fU@RBW`4BK?4Lr5(sey4vj8*C1QhizJQpkN&xR{nXRA zA1KEC8^Gzg#PqOrl0AJrQ;fFXoe42q&m&$Ei2DJ-23O$XhA@+}(U#z+wcF6^J;?hD)EDgN4*0fv2yU>?+HCZA$w21VKkIs|L9~Ay<7-F%wmU#Uz#|MiE!0pCJ1x~; zB?|B$UnAYasYE-oTZ-k^h!hLi%Jaa7Xz#fJ_WkVjc0a6e_W$?q)}WskiUhqNZVB|( zrS744`%`De`J6Lh|2T?sC)?s#kK5wBpU2-ivTLPk?zxpJufeNS-ilSKOT1{vW!}T! zi@9orj=({0;8@T|P87CC>?u;R=&(i0t8T8*}~&y5c}z zBG4JHhb#j^913T)2xuc(GX-+2u>&M@dY??cB~2OVb*?7~k#$MFTOzR+>ncy0*U zTVptB?HSx}J&W-SD)+3-4E7G4vuqD+z&R%th}<&HiL<2VVAs9o>6Y=5H*CcAXFoVS z>%z0gJiTfcEPKDU1~TVc(0OETVSWoCYwi`vh6ESebe?OBB?hF~^qP~RAn8fyH7B9h zTo<#%m*kK$0J{tAe-;I0bh*xuc0uRp@(#+d)s+c#PeQM`E@6o;4?;WCCmqEi=qNJ; z9pz`}D1iqczd*0)f7<{{?WjW)s1oh+i@j9|AJ(qQOL!To|Wxs=)-Ag0T zt))#^Khl#)ueqAQ5=%dZ&W!p*lM6+%j2hP`d-dy5kxW_&+9*Z+D@iQTA3DudUTnC= zyBOYliTQ4H<$T{-!ud8Gk4X5qJtE9A_rI=IYcm?AKK6+Z3D!Up3G^?uaqsJsb{s5z2Wy z7h^xW;5{5y{jDlaXRUHQcCMHie_6?lXM)_ozj;G9FEU)=Im30F{W!C3zoWy4bldLL zxHh)J|JsIoT_)^t$%H6v$j>jI7`=4K3CoXOxR)4Ws06=wXMs&$!rda_o)(pDMVW9n z4mKG_y|+~AotcqFw;lr9fLwFmZJKIHneI&D7$GJ-~4#{4o z$6&X?GaG)mi(BQ=cx3}KT7Goz#pe~kq&tGGxHInPyBuXWvka}z+%8*$G8%iUa0kqW zPbta-StyTLe)42ZpCG)6umHh$Iz|7HRAFfaZzox+5d2?AL&linNJGYG!}F*I19hZh zj6wPJVui64=aOO;tLYj^@*1;^4zj~hoV&t!30L5h?7$CrCppr< zJCV}BEd;o|4Bf!i7mvO~LSK%49DO+8Hv@jF2OD)AK6F#D!g30FP_75;x)}2n$fO;m z3QL8^-Q4ERvN|9i>fPDij+F{a0b<$8H&s78q@OQ=Z;Ev1moT@udv%=79W!LD3ok>T z!ZWYS@2lvp4fz_KX`mHjOu{;mT=|F)hZzzm25men+yWQC~r$t@T~BtmfUc>GAN@ppt+}cAHqot*&ORU z54rs2Iqt(}MsE0R?3HuB&En^5VUe@K&0p8a!J9nZPJZKf(sQ5w_SebZ{uXxe+;6Yo zdF1Wqh3B=*_#@`J3x4@l*onJAlVm@!Af|WuhWO+D2kv&=EammBZlZZ6l)L|JfX;-y zVj9f!!hYH_NO%4jPX8frCYm`ZcA^=K{Q~BP=!s~C@FyA3d7WCIpZ`zS=}W9pJ3l{O}x>Calw!m_HVr8_qC)ywm(S&s{FfS5$8Ad{qkbXUoroeoAxV zJO?J;J_kSFdk#uy4lu^M<^gMe0rcU#cm6l#;oqUxf0vGyu6yP9e_d<%E$P|se|!77 zjaHm#Dd5Md<9lEH{I}5_KmPXj&U3$QgKf$|OV9L~tDfmj^m6vzb2Z3LFR-EA<}uhC z7qEYv>7@;{<~&yw*xUbCG?VK@GtPUCaB!v@fli$0W-I1`McU|wXeJBu;h-Cu3kTgu z0tV{Z;8*G0H&Z$NnfaocNqneBhgM;P-wy9W$vjNZVJ0!?od{C9t!9!Kc?9@^1Y6By z32;IfihHPJtC{SHJObXiGS70c@q6$L;qEN-rq>;Jo2*sbv8z+XkUt=Uq;a|m+&41$ zuae_EIU%mYJq^ky8>c3_jpdl4`(8))xlkXm(-Tdx|HJckptlfMJ{MUNG47WRE;Sr$`iZe7jOAz)8-%g=Jc6}CdCNc@-9KtT z*`m{nPe_ms8yIS6#o50RWhAo@oWH|vAp-o{uw;f>JlFJr^Crrgt3tYe^0JSK$*V`cbfcJQ~k-8PLOODzl;eo z*^(>eC+PehjJ!=2lN$z58nW6USFHQBWhWRX)gdAzg#5?JT8~CTX^C^ZsCYIz&gjUEl z9Ok4Qa_(u!7XPY%T_S*u+D0-UV9~SF7|Ty6ua}{`AIbsSQGOidt$5$ifiI77k<4O) z3HvTR_v`Z<_&A>Lr{}(RKCcI!j_04!b00RU9yr^|WRMBN3{5DHM7d)gsNe1k;0C*l ztzM8}H|oTa98@Y#hy?D&Dn+98G?&;ghdX8(3%l}kuQ(IQc96dhp`Iz585`23H=CYe z#*Xc{Pq|C1mq0$nyA03`f>UY*I(z{`9KR?6OZ0g6xr5hw3`!$O0KTc^%(Pc!s*Ny*IG_VQ#GH0BE)i{3+_E z0lMa@r$C3n!h5-Nzf{fWy8!_n+puOL?9JY~X9j={=smR%!)!+#tRdAAU|qF6-(EGN zo9aAkuS0j<5AlEl`WEWR1`2SlgYUH?P~B*b>xOmj&)|pD2knRcw4Xrpp%jDGLxg;&AxMbB47UCFRJT9! zM3=rod7z=65c?aH_BEMeUz2piZv#Hz`XgfS@lpG|F;2`Y4_@HdFJnNjjGp;9KMND+ z`9Y$0_#O5K-hSrTCvkX&yBDMO%kd2JCcwlz(msgBGqMNXz9wpO_uyFT3? z^dSSk2hUG)(OrW+kMq@V6?~p}wjS>)ICKYnRsZhXtIJNbZ)aE!iGh#QsHA|M13!%k zWo=daliI5eB%QAMD(Ots-;&N&9ZdSA>g%L)Rfm!;SACP@J9&Jg|Fj8>?8%9Zfq!|d zF}D0Im1&e=elYxIU2B`wCn?CMEB7L1YY>8;RI=berAf^81WemM?&I(r<&UX>>XVNI zDNlY{qC5oOUJdG&QjljURftDWkgJp<Q6>}_i;WWeNg8d z;GYYALU|DJP71*jl{Tcc}n(o(zy_CUfpu zy@*+a`Gp)1iG8jJYgI;tbt)s_*JN4!rsAHCk~KW2raR|DKsQfvaULBwJLx+O@Vy?T zJLM6OCnj;LH=pjuZe88Z^jI_6Ge@H?4AgCXk?ih&>2_X2vb%4EjzD&aTV+pT?X$oe zG9f?M_(6mH-CPEIu4D>w$I++(19fWwi}=L7;MK$K`UIX^54b{>2mp@>1g{AK++e^D z0d2`4OStNO337owMvQy4JY)zF-bdsNWG5jx9`AnA9T)`II(U~x@Vui8evb^=Ka_^w zW0E5c-F|{24c$J~ktY2gX@XbJk8P~ylN#x+RRbT`Of~gq};h{>EVV5S+$HIGONWht`OeajjKrdU5dvJi$&=jQV!`+)*ajl1guviBTrl zoM4A?;9@PljY}(RQBQ@irX++=gq|vqz8XqzIUXt#?WQ%cn5n6oo z4G!EuXFDBf;OW5QB`U-D;>gbz^77ui8hOHNksYsIoSq#67(CuVwBcj(Q{ToT2zUr{ zZ{G(i9DM_iDo5H0k1V0z+j!94w9RisFRmFVBV3;Bi_5!&i|D?%TKfAoO$DHlA&mXM96c*k3|JOyT~F@_ zn$W%n^2ZH?YY1HkHxcMw|8)fNLsK0K*8I+T%z*E!-PN9G*D;R+9dio4M0`Lf#~MU9 z(%?&Bjx^ROgwl}JCIN4f)h1FrQehcS@fd|ARVat;REY5ovk-?Gl7txVFbi>jAwh`c zhKGeX%rHubgAET+jJ3E`HWNH|26*q&;K9?ui=TqL_ax*!S3%x$?7#2m{7>VKIs3$J zl|hf?Io$y0a-<*EF^M?}@7}~hFDCss0=n{6=y$PUc)x|Qy^|12M9?MdX9nrAkZWvf z^lYLaBj8O!!47$=Y&`I~ZO=#F0zR?OvC|NQa}BmI;>BG!7uw(>B)dJydjml$#E;t{ z6Y>7ot2S8_?_HDp=x@M#MFPKShg`%5IXxHTPk+3N!UVqB4%vtg#dDinc{ECpEB%QM zYEB97@@wSepEtp8T8Fq5@%`|&riqT-m?Op7z>wH*eJM6=-861ZXfZmBwK}SFCum$5>eccRs*Bdsysf*?|zldPPy=1Z9q$Q03 zuWdfX4KM73&EORGVBv^BvPJr8_qnnUble-8&vF~&u22W8oKI1yXt_MjW%cI0BHiX1 z(F!@f{egQ~HoqwOd$DhYKB;fQfpDR|V>u5)5a(t180&Zvw#!41PvF-*O7A}5PS?>j z*pF5b>(p}E&F5j0IJ1j`{n9QQ$R5#|rA)9%ELV$a2rt}S&1wORD<^E&1~w_XGls z!x!^MccL%hZdN+yqD|*q5!?_0wQGEt&L%Eva( z$)dr9XMtO?Xi(u!S)JU#% zX!!V}-u2Wm+$|y>|92R-&znqlsnQ4Q8QDobehJFr$nT#rNk{h}bMf90(Ty9z$c>yS;!U)Uyf)6@4j_Ccl-iM8z@K+%f*@Q( zQ7s79p(m4FOz`bV1pB4xDr^7^uo=|B-?w!z?(V^so)yIDef={Deu;btMpS=P9 z*=YFB_QHP_aN$m$2mEV==b?)<Y4Yr(jVViUIi8%X1$k%zNKjhz=&;KnSS{u#T zeS9I$VvcBDoae#r3)v%q53TR`x6=6Us{C-Nf&g`=l{kW{BQct`qGXTv=a~7 z*$vu}o`-K1HWl)hU4U=)Nqj@&j(Hj0>o40QfxoQpygXy4iyxeQW$-D!TlMsP>7wrc zkS;2n{ABm@t@fphJLUtv=v>SP`OIiOXf84_C%trn`RJt+_{{E@k4#P)iEZ zm#l|BCjkDZYw$lkJVl}Z@%pk8r|=y*@;%Wv5AkhaSu6ZbQs6*muLkz!lQzH8dYj)V z40=V7p&y=WqN{4#lg>r*x#2uH9*W>-GYP(%MEIXP5IP|H{D|-=!Ulvz2+tspJ%vM?ZJ9*=UmzeDG{%`X*X_YH*^2H+XHba!s&-}@{O&lL1+KdcY+Lv|aoh4Y}9et4FH@AuK1-RTDlFi7x>=8Sw%ErGa$ zWcNe4i(8I&Y7F9*q6V#v_GpH`59pfSyXP-CU?T2S%cj-VNd&_wQ zvcHgQNH!Rvf#%Z*ddX$$XoAtVJ;5Acl;Fde0Uu5fa1SsXvsgEVu`mzZ zyS`lq_JDfeO8#1^a|m_ncvjMb?-FJ@*MnWaR*&W(!0?4ry{Bz^eNPm8Q?@$6hAOUJ zMfIAT>ODj7Y<7q}WJj>oCI3=@!QfPn==W9Z8=}`$6k}a!Khiq`)j}*cyduQ@28|F0 z8)*M50Sql-0u{|*kDLm!t^Oia&QJv-}wI` zG6`%Iq<1l+^b~&zI^l56icp6~ z-P?Cp+#jh@?ZLkH0bMXr@=2?k;+QpU@G+WO!XJL5qrJXBUf$h+`;6|I`q$6bzVKlj zbJr&0t_<(VvfMdEV)8A%q*98s3(~oK3BGlQ^@eX1b9@c!&OKPA`o2I^^NA}vX7y&B z+o5-LfOnXI>vhyg$2f_gLEcY0oPQ7mqBX_@&Q z=cPS`vQ_v_UOz74A*(xILN;+C^Kq(6^Rw7mje2gy0K|<%e^x%;9Q*R%BHPFdM(o475<_9H1(=eT)!f`sQQw zqmJp6T+8vpl^)t*Sc_uTa{MxQ*$Df(G~%0_>0Cdp58k`b0N3csSG5YJNK6we5_6Y| zGwYW>8i#Y*FIA#QtY4iRm#5&SRWNQup2eEQ%h9%VnrgX70JSs-iEbk{hr}HHrRkc!{3zj&m1zpe`ZHfi1B=Jh!NjnmKL~wBkw2; zF>3K0h(h-xHOQ!pkmm1c7ULN;dxqImPzoo|c zEOQUNRPGVFb{vZvw#*}ReWFX;niNjgi2Gb$f`*3j(j#7YC+#BI7bQMWTXc-+<|UTL zHKP4@XgAN899M)nfz7ou1$0TYvlZ*R2|N>bM|AQwrkgalqCip}SIf>WIgx^S^uwAd zR*Je?E?b)t-SACjv|Du9Iw})^_Q84V2T3eGdWH59e4l{slgN3gZj}eq(RmyOpV1`1 zp}Tct|BApKiH5CWGAGdv#U81{9wA(YVvmGkkGy8zBik@1CddO7xUWU~Whm?kb=WhC zfh?0#3?BP0J zz6AU#i}ID>`1WoHD>=|%%^HclQ51-E_hlo}kT1GXabj)=^R44NiVm5qSrE5on#gpsbTe+4@V;~#_)cV4OA@z3QXIX*XE@s-8O|e)+Tl~o5X5Y&dN$jp z-cfx<$=KFmxoq38$m&)lqkMDcbDr?oZ@{|=6`~MxKHjfRFXuDUxd=b_@P~MOq)|l{ z7-~fmm~Nm5vR}J(6xGXv{jmw_QnwKEiTmPp^Y{`j5Z?vCv;FwK`)!;Ruj4z#B@M0k z4$?SQvI={t)D`y#QBQ(0=Bwz5Yg>GRaMl#*liYcCEz>D_D9`z(^h6Qd}w zsrE8Al%?9s((Gjtl%)w}k&kP}{Ws>W)Kw(()$m`|EPRJcr>Im+@mP+Q6uWS`@rWhM zIh`wFpM)s2Z$i{C72-_9HHb~T`;a$~K8f^?h!Zf!k-#Yz@hrsK5$`~JmdeBHl|JFk z%2g2(ElXjIttg)rzD=1DzE!yc-?pj3w{TifN94jaioHaQUxx2gG8ew_BjB0s3;z3~ zHOsLtV%-&sC7?r71io#K=f}#$nMbkyohdGHr$7U-C$R@TSV{ZetyztHaB~ID<4W-N zIILqlXvkgYb?Y(5|z5O^-R^z)GL);aK+4l1(&P$Ql@w;_Y9QIQ(&L9o; z6Ya$TXe-b;`W_Z_C!Mzdc3(O##~s$bW0ZW4F=AYYzQY*PFh-0QUU%F6!I?BH?oPNl zIG6guoddWxAs^=RAxBKRJT0ae$S+~^PAJoDTtTeFBSG$3v~5Zt^TYfU)D zer*-LSx9$0_En70UnTy5xzJ*aq-PS(4FHe+I8Aiy6O5w){u+0j8{aG$*{olY+6-AT zR5Dwh$;uTQNTzy!4$jggBfIgQLlfR_($$Il&AC`B$2@EUoU{0LV%zrAxCr2rk2a!! zI(e(V!Z;EBc@z3qKrSTN@z)^uV^AjwGRQu}@%`|^6 zcjKP1HsZ?%%!PO-R;UNwMt-k(kne^S%uOf)o>F}K;Jt$Q1oG>d{WV5f%R0y{Qq-kB zFVcSg!rDZBg}oSa{gMUU?_ykdzYBX@X0Aj%Gv3K2+~Pk6Uq`(r)YB{}?cRoZG;UL< z##jNHcMaex#1OI27*}MMF*cDfwpTG90t~<=x_BOAJ6oF`w{xji=()@paXUS~mS6kl)VRINyh3+*e=Q$f zFfZZYQc37R&mZI`mq|i@@cuzQQ05vs;hWhBEBebqU-;&Q1b#FhItTw-_dJh^0Or@k>>d{=6fH!mjQWz>It+7d7yqpYWMhi)FnM;E$05_3(#w*56-`6>-+d-ap??W0BG2> zYeL+om=Dv-GmJr$UnUPFdUnuq586e(XKf-qgZANe;Ah&E9@k85Am>Q%zcn0nJ5iTt z*0gJCoU!+LNbmCwJSW|w0)AWipCw;{Hw_2Pk_`*C9u@3a>Q>RxbHhc^Js2MOFw&^15k>oF9 z&4naipbgC{(c-V5BXvQ3!W!(6p`$%(*A~Pk5Kq2PIgE+s z_^c4){W8P@?fXIt9^L_3C!FGu#{1L)F6SY8-9%fu7lQY1b?0#<BhMwLjK;9%nYy z&pAKAzA zfqgvh3vrO)7T*2-BYlr_Zo1D;bP~x(-_(o?$ier;v3{IJqnZOaEuxljQD@jx5qt%r zno6|--;xvIK4LZ#m&Add(MI)0I)w&4ce)St9$;au=$&iTVd#9?S?;=_NP`dfdBblw zJW`{25P9<71FkgEg-wg}9C48F-&rUH>kq7qkJBJ5p z&tV>3%AcL^CJP*Lay)B32VXPsOZeD*>H9kq0e>HJHKzeK-4lKQxFw@sXS~Sgj7h}5 zT+$qWc7%H{Rc9cddLkI0kwQ#?uPiEoi%mg$xOKj5W`M-a+I12g+Y|r~0m_|43uI z>sW*&gmDOuA|xX)1g^0QcH>~46ZE2Q(Jm6}R)N@B#lwy-fllcT-O>a01u5(cp0JO2 z!M4H+cY7qB3ZwggDwPlH1>}EXlU9t2SQKJ(&!1uZW(pe>$23L{=vfaxKBV^WduefV zA6CSvADtfeYNAJII%p!@>pS`GanexSYne9B>w9@de>P%DBJ67A&qgaNU9@!F7ELMYW2Jv*LYaTqfld9bTK0b`_li$B7r-4z|LBRlpt<^5{)Rr2nO!JLb*d0! z#XFvsiX|aNyx*ZO+A_sbux*NE2zXL9&g=F$EX!k1m8!6?up8qzHl?5-fpoPfluyCE z;5E?KDBlKNI}~l^Zu7K&ZgdqK?q7lahVxAJYA!!5JBp2{nhlv2@?-((5PpZk>3hmE zy9=Nv$oso%^XJbn>R}Uzdq876KL+nVL0=ga7hgLGdTIVkGmIfIhvW+jXTWAP!+0o$ zjW+Qi8K%dWu|Pe;SO|Kha@;X4Spz*!sE6`>xUVt=c?rgl19_MD{&hL*gR{EFxI_2% z@MQB_!N>DaH;xOC!Uj<^H^H~JjND<8~!N6R@P9p5bMNfqoJL&ts_o!#ae39MF zin^)T6TXGT1iOUpwOz9Jw-4WmNWmVs$a@)n1szkLX2d#tBl{xisDYD-%CT0I9`4uQ z&k9*S2X=`zl+ze+Hf0cQ+JQ$lLvHh16LnZ##0~Wznk^i7Se}hJJRh`%-XoP3VcyfD zj>yf23lh@H;unnukD)nz4g3Eq;F=6QkQ>6X_NFplnnQZl2wD1bJR=@zKDhDLplR|v;ernKB6)Z zp1?nD;Ni{5oZ`2R#}vjP+$K5E0l|M9^R)u~S5a-|gby$D>UiP^Lg z;Ndgq*yb01cYF9jzMsc~zgtIJF@KFX_W)0N0dr0?`2g_8IX{iwBQ60w<$|7Cai$TR ze&{<7G!+frJqv60e12|1T@T+0dot_+BE$MfAJp@-h$5NqEc6!<#zs(E53DIZ&u5MS z-4I=!0bN}K{A`pF{XCyPi{J?KlWK>ZED{Oy)5iN6uAnZ#??hY;`bj~ZT=Y+}Inu`{ z{fbY2zib8U0d$U|yb!q01@30hAFT)Bt_JR}VZP>}{`Zjot+#BrUqGFXD0kHJw9LbP zm#?;%eEj(|I@1;Ng?+fgpy{ubd*@Co;I!2Mm+ zA>Ko{BYlk0zuN6W4&1Mye*RN79&w0h!naOd^WaE_!!y6h;L?d?nN5R^@YNMgvZM+EShJvj>-Yu zMUd|SeW_{AL3hQN%kKt$zgZ2s8^(Ki3@2V7)Fw zZW7v}pIodLY|gqI%u#fHNkX=r*2re5jPyo5PYa7?zBxo|;cNu8EhJhK_3K@)cKdoo zuztdNMd#-b9AUi}=!jrr?gUz+dv>&51fSOH6|9#p>Xg{m3+ZE&R)E$boM_D*{JwzB zB?T*q0Z%EU{RjSh7<%VijB6goRfPEl7kxmP{n$(evpcQJ`BFBhl@wlL1`?ET@}!NNFyi1ySF~C*z$W`o8f1 z)FD}(=D#BX-?@OS(bkObxM8mrz5p7*9yLedyqGmZ(2cZ6Lv~jcL5|uD8K`I$Y!ac& zQnZcfc0-<$LdIJnl4{4I&IQONW_<`-+;7B9D1A>^a@3BdX|Ps;??Rez&$DAVGtxPE{uzal_*Tbo1@@AAQ6%^k z-I+L#_fSnZizi`?Ix_KpKQk1@kT~~HDcTTUom{r4nbvPoSweFGSQT1Vv*f8|(FADZ4BUum5L&G=rvrPiNl25#nV8DJ3nNQ#=W#B6p z!B_58kM`Rnz^5~i_{w`YHvwDVD{JiiWrOFCtY^k~N3tIENn=|JzM@5)B0FC}`WU6Z z#d%lf#8(RJV>yp@G;efP-)lT1XAtZ|Q4-Yiv`oQx<`SR8Sxs$U1@9%gq_f&FCXzGi zPS3gRP>FMJ_) z6lvkZpmUhYM7G%PR@vjw3#ULo%z&Pl4t){t>=aMJy#-$IS3Bg1R{y%0G$(&#!?5y_ zV-0tF(@EY&$00@De~lpw&Hzn;#L0%UX_UR^AY@S5Ay+$ zt_XI0d>hk}VoG%j@FrR*DC|{%(#o4fx2eA zJC}j_<_yGWV}9)aG579qRaaT#|L5~LT+W62O$6iOl>?%Y8jutDK^-)CQ({L@m=SO`GYN>6vFl%|yw(bWtkf_gPcb?xLpV!`>eZQ=|_S$Q$y*9W}@&#*+@KAC2mwdUnk}nHa@{PomeBcJVx$)&b(nx#u^>y>e?9lw2Ira8f@OMpATFzK# zbFk84V!kEa84Gqug27!A-Eec7Gy*fLp*f@<`K(9yqPo0aY>`QM@J;Q>g0puU;73Kd zRN{~ILMLk!_V5Wi${cM2-|X9yYi;+>wb_KHQT zd?moo_DZ0(^p%$=CrAr&1^ZRL5@NQ!a*Q@58bf2#NS_Ans4P}vmEMZwXo{^+&$q!d+U-|`Ow!PiOxt(Er;Jw(b{F%}@BN~Tr{77)kgyr>rHkMh z=M!FutH+-!%E;X>@$m9IaXY+m$j%L(xFeppE^l0bE%UfuZdFS>_0EFVBzXD>K8&B_3mIDa<9B4 z9$6*pV#>Ze_x!)WxpVKG?)lHh|GoJq-Hqn|o$wpXKjrnD|3v0(y3GIJtMflx=6~5Z zYrEyz`A=m2t;}aA^PkL@0t^fW22z26A;7>NDQl<}?7A5kNCO6r0|Pk$O011M3UijY zSc_GAtCnNHfB_gdS(n@R9dv-eg2Eb31Qv3Dg&Dv?6|nFoIPWd`S+m$TwuUllC?j6+ z=}=zITiC(ACjR1=^S<5>9EiUP{|mzJ#@~P+e69Oi#V>j*gtv(wT2=hYEBN7gi$4MX zY34qkwu}MR-rSp*aijW+DPuJ-Fb25z_y^Kek|qqu`H4U4`98f#rS6?)rZZyRkWq-v$2< zuuDJxS7G-$^%nyZ--F%jq^ly`_h9!r=^6yays(?w4Q{TJwpG%4VK?iC)3!-kFYKbD zbp;-;Q|>je3myGKX|I7@Xe&u8u)DV#A4I3cJm-deLW@Bc$iAAgqif1L9y+X}$kxK% zAKC$(0}XBOc!T%21djoeLeE3@U$O5YXBZWH1x@vXdSc`ayIx)!`}Kh$4IO7~(s;hp z$c2-^#pZ<777OxW{YR^{(UR%|59m{4oUTjDZzK-U*Rujj$bhxeRm zUD9aI;@`@DDRNoklFjCL{*C;XAeR+g%par;m-`($2Ax$a_mi!hTU2zJIh(d|4n@yX zr&&_a$q_wNk@KAnw{_-E|Y;r=cUI?9)SY8<{WXCC+^9Lv1b52BU>}>kwYEoCk&OT#SC`C3y5A7nO4!vYpT_ z^c4~q|0LRWO6%h=!uuB*xNW)Hc4*sjw~f=b?1yJ8T(B#b*oDit`I{HWe+%-C49?%+aC+cRXra zFcJCs!;3>=s}?JChmy^?)=74{jSwx(faaJ#M3kMb}313Yf71^59tEb0)a=oONaKCQh*h=0Z z>6ug2+$|it$VVi-ga>sC$Nq5_;V$ru8rCfwTgR>7gzK$Ek#D$deBM~!^Z5PTZ|R|< zy;E#FmH0Y?S;w5ob$u2UxXWSu`w)h{^azoi_r-6lYAB4yMnF5`Eiz;Ym$VvbLy1@5 zGf3DO>H%kbPxo46@lsCPcU{{=#~_h>Vh>L4CwxPmcqO7^C;gBz{=vWdzG;N(;1^xL zaqDPfyF(8b9qnuNy(O*GEAq~seU!SSJv-1BvM}%8(?dr;H6A@hk+q6$b|>X^fG@?4}C9bt~VBp>kakLUz2_?23|e%Ki&T?>Y<}=d4xW?_0Y%MHl2Bry5-z( zBR%y0pss{}Q4f7OX?yCSZ`0d)V?Fd*;zbX=yUu+p^=@l_K~E})ra?<+I(Si_Rb6Aa*tkU=bYOMJS4g=0=sfv z6xf|fTlNmLS&b(26Nf3yNlT(9qo z@Wu09oL%MKK01$9m3PBAb7A8thO#odK2R2P@utvDS*KOLPWCs(D%|xnb;p2Px1(3H zfluK7W7;A2w`83i#V`6cF$Tq9Lv}KTe;Z@oN!U@+%3ZTN2um=aZxEqmxOI7Yxom{# zb+v{$bIB)l$-U~XNaTyS5yb!PDVsGKorHayc{Aa0ru!?Qk(KG(Vc^~Y@iKSu=wf%U zmbnL4_zmamf1o3CD|vn+e98;)8P&UQo|aox$X@p}!}59<*f(*-tBe%>TXA`8D~)w)Qu) zsS}$zE>k3aRTjr&FR}b$y!nHQimSIJH zl)A)K*K~RA2aV5KVufcL==tBg$r6h!SJ$UV$~Gp8tqz&9H>g8kIEw!y%9VZyOzoYO z*(~wfiPz_5gRiq6??$=jhI>5v+(fyWyz%eLdJ4a?Zq^Y`xrOb}B0cFp**7jP8q2!4 zll5^2>tqb;B^Ul&4);mi_Beh_xjk`e0x+lYea;&Cf;A*Kw&z-2NWM1KvTmP%d5}B& zUVPuX%Zt;aT+<{S?d^%v-Rt>UJpY6A-|p~3LNj$=$3p)lWMZF%dn>1&e(;qf-o=ZD z=ej%tnWNZTp-3r4jv-us$=_754el8}rPp z?MP(pX{J?$&T&|NRcgcft*R~TSL>SB<{#LI;3mRmh(HfRY%=z8O{P5pFXy=1CGgnh ze^=wz`n}C??%#2@YqzcCdS$(!^>WNY*Z+|;?#O6rN9|CN^P3&{*eLlWaF?P)?zf}o zTx=clHLqtv*G4+>(d+-zkdnM+i8^_$+9$74Rpz~*u8Eo5?cbrU{eS*H0iABU8j$zz zuD54>6VNID|3y2{EC1ha$6Dt7B5&z*0GFQvmu29=CN(JUC1B$qu(AVKc~5;j=HLB~ zEa?0u_+1wJFPrVa^$V&cuhLgZQ=^qB`PkB_R7361Bp$!P-Ud%j!UI;XDqQ2*Jg-s> zus6Vuldv@fs|(*Pb?ITEmndO_8*2;SUFA+^L*FKkHIjT{X!>T>>8S{1#9qFxTe*)w zoY7!w38xJunr4X)wWS^B3~BVYwL}S?_k`sVws&cGtf>lmoAdqNFw&&hGHb(>5yqjm z%yHrlvt=4_WBPH|XAt+$EK2zt_=lCj*s($`Um1(;g~-vud5e~NG~>VvB7-mG4z|$o zMzuF@x%KM6j^vcDb}KDkfqy^OaYhXIytgaU(93ZS8Als-edK+m?f1Fu-~8ksUBhqt zN7wcTC~FpPv*-PI`@GZKpX&Hrxdr)yy#yZr_&{_t1JNzx9$froOZi^LiT5_xOWpX& zP%Q6=gt%V+skXt3*96CTanw2XI>BLrBTK2{6gte-C83i~6)D*Ct|~mN-(gUOOP}Q( zh7!X1(4JM?-!3tskH;8HgU8yX`INUaZ*9n#uGF32uKV}@LfwlfSL(a?Q>Ak|b=Kk+ zUP=;fQ9pFHfRlFcxwKQ#y+|4fA4lDi#+dj~%)Y+Rci6dW;G8Mxyk#ciKTY0x=?k*R zF79${f?k(2>;-yxza>ukxD(jyiUEFbCH|+L_;tjyN9uR&MJ_3ORvUa;=wy*eA098d zvV6Ue8J&anlW{pp8SU6?*sLkr_4Ld`J!9e7>#n^=xEK9V)^XSF@E_z4T|~Aja!4sp z+AF$aQWtc*4u8;{Iy`RToGY+YOZ(-%_&1yv^*Lm1N_Yw3(suESO>1Pq=OxVOTN@+1 zcnKfJzro{A!r#6XyUqK#!?N_xb;g^xn{zq$gVLpqby9yMc2W_xb9XK(xgEJ&-%2Ai zY6$nzLtKs61o}euM(7slL*IecZQh#0X5vOeN%BhM!rZxjAK0GeCi?6h&w~+yo7A> zl))$Yl%4lpe&gTx#V^T+{8HASt$&|7IbRQ9JimkIdF(f18)YoN_@%8y#-a_rd^_VU z`t8w%-i}XS8{fD_?d>=N+&1AJ;M)#;e(E8)i>2XG7k6HDKXso%1|T}|#ps!xqMqc+ zK6NraM}fhV2wMhtz5LVHOpRy2U1wN_*yk@7-PB6XRKLm!OI?xIg9T4!AK=TczVdh_4}a>;QdPYIlEZj!7_&5c5Rb7wvgu&*3H4v zr5WQ1lmD0Ti;k%U`|8q{qjIOx5M^J+9`Gr!b?PU)qaC88pQZ1g0jFoc+3*cnE9&{D z=qE6C7+z{b6!z7voQ0&10fsn-nXt9!UJT(2w%cW2ww&?3%Mxf$(c5}NvvgpO+}%$2 zU83tFviN=2B6}5E5#8%M(pM=ru(p%1v)9qj)>#Q3Ift=sM?Xe*%E!QUQm@SO3B$O? zZ_pY19DOdC|D))!9fQX%@F4IJKwDC2(~D|=C7@edL}%kCwAn)2MXo#>ooS&t1MQ`r zwAKt+H%EAjr-6QFF}}xWH@dim*Gr>PUM}Sv(bV!?>`|mU?BKtBP+nsPJVwR0vPlcH zHTD11(x!pGuWYKSNhsJ)YXRhO1j{VH? zYW9kA?AyWUZ~u`n;p-(;qaQI5-2!;t#mv`q=F2CtvdQnJT`%^WuVUtFI`g%Lxe^_E z;o--tQUB67Ccszb{vLV{7Hk~yUdQRKZE|*2?6bd3ES;^-m0}#q;%OcP0U%AV(QSy-$~tBNy_=- zjLY6`aWnWoJMvZD-c)&mQ=5am8SE(fzz^_+FM!S6pMdKpV)Jy7iTfQIZ?8oi+In0& z)aGw$v?|8^)==e;k@I!QK&A26WZ6eG*K_o#a;e&wFk0CrXI5<4 z!S@rs<`b1oF^0f`SVK@jA46~f@}z>%_;1IbMR+Fv(PN(~dvffvWsAl>Q#N(%(`D&n z|5WzX*jLN`I(B>6wz1pF?u>t~U|Rg9f~xq20{{5u3)aWIQZT%~GI2}XmV!fZFBg0s zx0QBoE2!a{!PmsP%)vcSt-M?fEhqQ8r^h{Cpm4XlU)*yAN}s2(bN>|XrUKQbl;H*i za>xFq%f%nOrUoS+!_MVh>;d|onfPwZYiexrTJ+~+%wgQK?*+29EFDLI&mW_oYR49e zgq2jPlUJk1SkE3`0*z^Z#nxC)I!Co*<8&+c9+)@L2^73{0GewrdW{8?i;V)kT(L94 z*-kGvi*jdEu7p{ZXp^xqGUJVTWABej~>^YHKDqloQS>P$C3eiW8N!AUzxiUM%p>+C0nD&_-0kF-q;sf zq>OgH>*@cR#P?!qY3GZq_0z~Io20$;N%pTO+GrlFgjwJ#znHB|l6&tMpqT<)5z)Xp zJWAcisM|^1Zn*^X!enHZiJbXZci?xqe^>ZY%53YyjMY&5<dmL*LcE?mJYVynkCA5+|6f!@@v7=g~xW}=B{HOWvt0-YZ z7ORulPbS@^1STI_ICsG`RkQ5C_J`08M*JxY;wLw8&rj~=ctO3X5r!IQhv3!{N0+fj`FLp zv(yg#(?{)N*+!cLC)L9vw~!WF`N@L~#*U{c+q(N!$7#wHI({E<-Rly(x%Yh6@CDS_ zGu&I|C2+gQfKq@(Xu(3EclzP4H>l;ufxQmSLZ#Cdu21H^*aT>VGyZP6`!n{TDqmaE zvcT-72KLUU+!PXMhn3tp^nK?t2A2XPfM;SE7 zGQ;?!}cAvCfRE~ z`%`w4>_;6hJ4|HLh$WIyMe|F6Hx?Omtmm`_=kJCTXhs=Vonoota?tb)ds z{XT_nFfx))k&&b#Bl(OmItXvjt_EA?E_t!;EHaXgroh-)s@XEp^MATAFc$h=mywia zo0DZctdzs~!d{JD&tIvtr%VFb$QF@h-$#Fnsas@)az+Vxi#$alPakpYY+b`-A2VuL;X3&c$z?g^tZ;y7?my9(2d z$~JjB{VVJx?`3|B_xd|WFqaaCd~sWUZ}?qyE1Z@UD_-%T)Bhy)~8*+5qY3{e4_}9bWMWKbAmjNAb*@5WV2#}qDAIT;%=o6 zZaH(L>z{Ya9HDoB%Z2XY(vKUx1t4vofm~}0pWdfPlWw1B7yZlmOBk{@Y>LyqpEDkk z_D1FWq^oP&t$Z?8GDdn?fAw?9J%t6p)lc}mZ@%ks=lz|+FOV`9Tqd6Hn)nK%Q)~ou z?UsAlCg^rEbh`z*Jpj6$JK2kZu*+v~e#QIEU(-JCTgfQBaz0mFwZ6=c6?#Dx8;~)V zc5n_`+`260r>)ClzCpjcw(_+)@LQ*`v+{!t{a%BY7U6n{@k+Q!(e*ig37_B|c^k?Q zoqP(O*l9zr<@SN6~m>I43!z2yy z*uh9t9*g;yG)LgACTi6&l3wDoTdQNHlfK&jsk%fT)v-xa9U5n>zUcMzMdt_FB~$hu zbif_xf=?2gXNk}zLZ4=`XA13h6n(*T;8o7dgDr|PmA(bhH~2f)O-o5Wr-fL~(>G*< z^eX^4-Oyyspy_=(PTyvRF4OyFpkF=;?V@kK+T$^wc>4AL{rW^*p@&PqYFhErc3;ii zw;bj17+>ChkiKckvKUD(@za^F<)mL`dAcsgM{_)-X%5j7hxd`$?;N^ZA<6=wx6o;J zDBJ;|e^>gphBJq^Z;Q(m>mz>HR%8z`9vbFUpbgqqDoyRVyoInyaeg}zcmNLDxyNAQ z+Ptm3f+9Y*Ly}Wbo z*r4^=|MM&LMs%d7ZP=(_Q_ZdW(97j_rQTlGt#@dTdM{EhIMsQIc6>p7we-Oh!nt}A zx&yR7{)BQKoS@r-GSiN1)wn-_Hm(0Lc!l&L`;O^W=9lQiiR~yeZ~S@Ft|aZ|pw^G% zokDM(K%wOa<> zu>F<*T?ftOFvDjk;yw4x%wYg~(Q5clqT`v!|7z?U7_nWDpKX{_Y2}_KVUASW0y}oS zQsEy6uXcLkU0?5o_fzyJ_#S$qGL!uPcWt7rxk)wUJr;-!N6w5*-*$~COH>XWf$yEm z+9W=6#{gBAIqVprmT!VaSc^?8@n?%a(rx>3MqvWmGm(M+gV+b$4S(n{&g}25wPmbTlw04YUjf)6Fi?*8*Cd9x*7*iF zLa+@je6Mj|VRP?d*S0yDa_f8aUtqKenLFpw^9jVgq}nPDFa}4)`{?%jUc%;{{?D0n zCu78a%K_SY1o~Ol2AgL@&vM#6cb6k&2#qae93j7)8zsNw-9+Bap|%R(cv2R$v@)VT zM&?t(jMU#VT;969k?qjW+2cg-qmA{qlP?MUS&OVmK_S5PpOgpbY0H7CiYX=uftspgbQu-SJGJ+ zgDK=0=H7$XVGqf>2d{JQ!R|aykVjz-uV=Hy`-W}=d+_J%wQD@}Ki;E$*^hPljd-E4 zOdcA`$a3xTm5nFpAE0TEWuSOg8lCI%W7y6XFTy5e9KZ(9pB5|U} zA$F8n2#;5KU3s%_BmZ(A$}7J-&RBfRSS-W_nVZgucKuOL6YhH3)6TX0|NK(dw#j^Y zdrq00EqYwE>lxyn!Qa5Rb=S#*r+CE(7vNLhhncIul(cZ0#jV69PcCP& zr1s^7mAuV!3>l5-&8ot2iyP}o7n@?ua}mP8k8$;1RhYN_nR#|}sO<2KW|ziJe!^tQ z??keo52fa*Z?MlxRNQH&>0* z;!>2-IQyLMmnC&5#_4gEZt=5dLmq9&rwuc9U&l+VtfsW}5^e3PGY{361=foJ`S>TknO19nMh3Qr19x&|#9QeBaDf2_x z%Zny3?)i*=9`i7s`4|W89?LsQZrttS9>5o3Hvk%>NDZ)WW*pY4=KQ;m`#i1A(8pk| zQXcmb{XW1L9$4~1-2wXbKK%+Z2jxR2l%M4d>9gfZ=vm^=5`UKXbHoRmgY(~C^2fR? z-;j=7;OiH`DN=_CTrGG|`YrvR&L@3ejyof|I4)3ychWiua1w995Jj$=!R6W*HfQDmFAR$Q@<7M1n{vd#$mh_nqcwM)0w2~Aqn zcWYfW-;{Re7UTwT;%peUaQ8s=1IAmIE5UPgihV$M z&FHE{cT_W`Td{$^fzJpofv-~-on_R0fV$5b0`_l_H+-Jj zHCkNtIk&61+^*pdZr5kM+x7jm+x6SHYxFwkxX^DND}4FlpXLeQ0i2sG`@5C)u;)6A zYOF(1f;eBQ9SOj97V?zoj48Mcd*X^q(!v*XuB=k-dgx8%+Z^QAl?JUtVc+rZ(@;>P zM0RNGN0R1Y>|w8~vfVWnzOes+#DWXpq`}}m$<&Z81zR?VWVgDqf(vnM`W^nI7 z(!NG}OO~pWOV`&FTF1XK&$y(quGi)1AJh^i{%HK2OQU0r6P6VwRBo=D2}~v=g6}QR zc6Oh}i$B#GxhHbi_yqj4c%>zYH0{u|yens|oCgmPI|uc+23+r)OZwAU+^J- z`P8}r&Zpl`bJjepI-d=>-RVa?{(SIm4i95J$z7Rm*mJ^x$?ryl*L~|7UMIM$Gc>$T zbU)@Q+Tl?fx0z_v62v z@FK!(Z`dBqO0@;s-pGy>zmjSjgnyX$)l^#+{(SLksd+~g=Tt+81G|;yRo29CEyD2* zb>>xg>fA}4)jxf`YDH$MEt@*kpW(kP)wYv5tMA7@l5px&ZEsYqpk0BK-CO*$YXJT< z@zbu+_{WPsD|N#$v3q^BPO~vu@8{3wx$CUjHTCuCWzx^g)T*5OUa!`qpSPt}?fMzv z($A5E7ZEOf7?oNTWP77JM*O2ws|Mn~Mf|s?R^5(2PyAVWKQ|yB_4YIGf1%D<-Z~%r z_w{odb?Wo^Ku}%uj4-DTNB@pF1(t8+`_n1^bZlD|3T!WG_fJ0w{@TOc{Vv7QvFW64 z8_+4|7Z+zK;m_~xYj+3XZL6v$^g1CaR&-W^GKvcI)!{&^k}b=(>Q-OD`^wwcSy+c>Sm756uB5G=UR7 zWAB*QPjTw|$7Ak&L0#r{wH0pHuzJ_%o7wZ6tZ{vhVDB^19(eokO9CC42GwC@pD?Pu z97XJ1HtYj;dGNU4^%NbC_XdwE;AM+z(mGqyql!|nLQxA=Dq2B}Vkr2L&+bvTXiC8_ zO)c2-x5Tn%`5uh$Z5p6y1p_rhK`3z%#6|Ky|D(jRLcSW(oaCF?$ERujI75>U=_G6= zaX0f_Ek|p5$fhqdhZI_?LcII=m`FYyBg*HD+o1LDvOP$i*5XzVKNWE{fH@#(3ng$T|W`x#M zQ>8j*6eBZDP+pcZ?#KGRL;cu;JEr;UUk-0(#;DM`8up+XA0LO0=HsYoD;h=FOJ`Qa zzg~0B<8Q^U*hcH&Zz`|XU|)hgSJ!utx91a-8FAHo3iMYt?o)h;(bMC6p}C%d7E=-) zh|78Lfw-rMPryF|{|RJ;1EG(7LW<+|yf`z?Kgpf$JboYIg!Vr>(twVD4>|(A=m_|s zBXIq*fEU($LuSW4pYdSa$zICkx04=-+mcZpmzMlM+-ZZF++pxb_Om@07ohy(@TY<8 z7uNHx)t&|O;`Z65$DJ|wCVyt|Nj^Ad@%ppqydCsiy8bNk)`O2MU4M(OPr(C>6k#_ZcOOOg?S$PFUTeFCj6ZF>j$+b?;R-kyQDZ_}={r7xSa6o31uEh}SA!n4iN`q=Y0 z2RCKRiW_SPb)59^clay*4hv&q=KB&|%K%(CuY*s|ABA@E#!2`g;FP;C&XY>GV}};i z(ZCvfj`{~$X(wm2gRRAJZHe>aMDF+5kEg`%F+Lb~U_t@citS@ zjZWKz3^feB495K|I!t0mPRc0m=$djf{KpW|9#TX1`*WxE0(5)W9_3r%2}}Ay+JB=n z=Ba*nvi|P?4~&^}c4Y1x@0s&^e1(v!e1%7Ugf38+n>T=buIU=Sg1mW}EBEyboMW&5 z&a%QdD;{}Qwi$i`e1%@P8SoYUhVD0fhL+#qj`)1Z2T9zs-l0WznA3h6lT_)dGhHtD z02_fW_+IBl2SfM);}Y5rtE__r!G-VvvRRYD3rJEb;u4u_1-}WuoIlN;^Jfa~xDiUI z30=ezX#XT=e=EHI!O;7qgj)%h|5W~8GAW_^T&oKkU9NctI1_vXP5u~R;{S*Exi?p2 zQ0ARAJhpLS)=PCi;y;A*!+|9su?N)n$)9oNIsm^kuXL7i2 zK>lXjSl__>$}n5G@S-ZiZRLA`t)KJeOcL}1ysCL(n_@Zo|3XjXz;EJ?>$c6p(_az< zo$uy3ECYT-o~4JMLRJmkp!@$SvMiCU8IWHXU9HTYoA2=bx0}D6n|tN`=9!G+9eh>M z55(oIuQ{AN{*?=>`|rDu_tnV@IgIxvcp%I8&d(I0vm8PyZmCa5#ZJK0La z1g$)74j=j{VfE+`Jk#5@u!^zQ>od=c+e*Fj%@4#CE}S1X%s4M@Hfh%~A6uE1LjEPc z{J+Nk2k=E-0rf3dEB{{zym zB|))kfDd$L9h4(_6_eJIKJ{YP_WCb<%iG~8FJkS=Ts?K6Yuj=@eLgn@I#&_i7OXZN zV?X`1UxcM~>CzA0n5gq%x3!@M{ugvzcCnvs5?Oj{vU3-4ZP2o^mQEpmEe1CWZ%1f~ z>A3y)a)ghzV7k6`5^WE$eir;e5&jx}6aMS*Z+vwAO{{%DT3t+u=98~n&b^Yp#Nd17 z-$?#;gAsY6DOvUqvAt*f6k4~VYg;35EWDY|2{(dIO?)TNhZg>*alqEPRqWda)r6d~ zmxq7a+jSybwSGc;3;a{juie1geWKSt4VaNKQYlOM%70=#@;JfM5+-^Xd#5j6zp7es zCWovmG*>-8FB?6+)2xZUft5}DBW+E7eHsexd^Zt$+zkcOaSgc3aeZ)~!5wJIYU&rd z?8Q9voTkAK5na&+&I(8TmR=CPl$5o1*3$L7<+Fda$?Qm>9t-tMqn;eKb+nY1OF6l| z)uYg}e0du9S<0vaj`wNC{k7bS^|sB%{-%c99;b26g&vI8?oeKb2GMbPNtQaf5#FP` z9}vx*nv&avUtx-k#NWFzGS)b-TIW|}2uy^!ZT8Fgzjl08;c$2ql_SxoXWVvU*SGRE z-e~c>Zy{;qF2t#Miqj0g!2-X*0Uu%wauA&-78d9Xg4b}iu(vY=K7;IsC!q~QzfAZ# zyMej>(Ay`#H+z`3hSsUCZ>r)fUo$GT!FJ#4n-tFSHKS7-T7O13XZf1j2`?g?b9zly zYD0kSjZM+wS5h1LQC}7HRa0Ma z4d;03tERr<8qV?5S51AzHJszAubTRbX;T&TRa0LvZK|TaYU(SdO;yxaO?}0*X$JMx zP#^Z>_4X^|b9utg|5Mr>1a-@@Vu!4{tI z8K)sbda{7~`dWl8e`uF8Yf3ifSKi>8qgo@UY1Tfa25av`r6mE}(koKwly_F15~spymYt-chs@mpZ^Eo(AbW(mySt3^}W{ni$ z7rD|G;BS$O74R=QEXDBP^6`sI=v8!JxM#nuUQ^D`jIn*4AotYLZ7q**r{rF~Q|Oz4 zZ<4dPtCqw5C^`&sA3|iE1$+=$`r8bw(J-MAxW^J|=U&bl#%hNeX-^0hKDvdq+Z#PR zlU?v6<3)btpLq!x=cgt#D~x+0a4&SE=!YH1OU!`IupFD3C~?j#gQD9iokm!^V(u`I z?jUJ2>g=9IC0z8NLVB2^Cf(&1Rv#YcVFJX=*K_2=dicfTdy|` zZScVL{)4~WJX0lNXWpQUCsjYow^Ii-gJ%lhE=1$QkHZW>K1!S=6R)^9DMq{ne&w ze}&J$r}6pn`SAHS?NQXwYJb0`lZsovqD$@FQN>wH%K3zkdl%dMIJZ5d`deD_27UW& z-XO|TnogqUy9Yjn>^Glqrpn@-vR35Z(w~6_zx})E%g6MUvt6j*RhjQs=|?$ATq6m+yGuqY2oC!=a4uvM@n)=>FRmIb*h9969=F1}{fxM?>?h}t6G$3#cS4b?gj$8}W)Cc8E)M0* z&2Z*ffz!Fo<;tb>@~NYmxsMi_M!MF#hcl}3(ujMwIqkcyD{*qau^pTSeGP7@Dy$`c zvZvhIyyOhAe=m4N!Uab2bl8k?jgx+Y6XVrLUEe}}HfT3(jo?S3b>G!}kE+oqVO7&5#$_+o4m$&TTt9AM=9J z=F^OOk*Z-n8l7iwo!p_y26jYt$UPOFR@7;)LSP7eJmnvzzTa|Xgu*6 zVFCA)G}jaV0%dcTz+Q7GXQ9pG7hMbswDv061V5q$SZG1M9V+nfA$2fUVR}3H7a4a0 zx{p@HU@rwvjNyL(|5>V!Jt?gw#-ti7!vD&r?GtFn8O~TcXoGu<0+Rz5lr#sbm(q98 zmR&Ly^r=XNzswk=F-GmccNX&UR?Z}Ww0Vx-rSvn5(-6X}v=@JJ@PawbUGzc5Qtl!G z|GI7ndt-#u)yi56x@S)F5$c$RpEQ%?%zA{lV~kbMJ!N{F1OE}m_D#kj0l(}A@-O-% z%$xlw>T4pO;KD}gYE%vR($7=0v4lA0qEPBHfp5HX$vc8BBQ{hw%l!Jg^-$vZ7d#^{ z?;S_>jN$AN!;{{En0sc#|r;$?x0$G1Jcu z#=e-hdIpCmlMwT2En$Yn+)Ah%B^=hE~810|U9Mus2Ea|c7QND{X*)PX4 zj%Vh_-6HAQsbAvyLJvzE3pYaCdps^}jhGD|)9Is3^s9NfAk0^p=wI_n!4tS4HCqdu zgsqo<{1GE|jXt2LPO*g_>8DJL;%?1vRysO zg8S2-Evw|K9VimcN8qd z{|)}An>H8x8vj96J#>1NGNISpr^^mjD~G$7EPEBGGVRF>A~LRlE?`D6K#^#_gI7o?s&HP07t7sh!&&OX4M)Kkn|f%mYb z{22SfTqVFFc>Ww`$Jvy9o;mhGzqB4aQB$=lrc3qJ-8b(U(wwE4?P<5Hj(PdsAm$7@F zF}s!a7xFFOTgEqnyH8)zk96ELzR=a~G5t@*bSL2_Yqk`~Sjsp~<7}e$k2NXd*Z^LV zaeUJ=j=r>^rq8oi#_>@drk*MAj@R3>pDr6n9iLEc4ej!d+fmQBH2}Ze$4$m-ALG)c zn(gP(R>y4VF)oR$N$8-$39JL|L4>}9-l(+YAY*?HK8>^|^k(=qLA-s%w?!EgCvg4> zuE006>_ma%dc}?3J~sFy8yC!MZf6e=Tr7J43UyE zAH@H7+U?Ej#r3l0B`@@cj_cnhuhe71wuj`E@{jg{XGYs2z|Uhz|2cTF7LrhCiFm z6+QZT^l*hwM}38VMSpm{7M1@dJdR%~QI-!E#!r?zA@$$-mjCYNb-V)>WY1jw7;dyZ zbJ%ZVDmB0Ig||3kIa<>#g@zal_bW)ZDrSeGlxOlzPm#fFpE_)H%-`?1 zwfXCNZ*7+LAupc9ScF1rg?>m|TC@oA8SPJ_pCjY+u~FKp+V^qJTf#V9)Z5!XS=w~O zua{lgK7%zZZu-|<>Awcw^+-DeDvVYo(XvZ!4ZHXvY6L?sD~p zwz&7PmaS$|55f&@YU@-(iiFJU)Ix)BfW9Xb7i$NH<{=d{B^MAwmU1aQK+pK2 zHZBa8()*HU2XgWp()4_bNXjTu-ETU}TSOtGm2%|0h{v&qfs4vEM0j)l;w&{Fz%LPO}d&)lQ`a(43e|5MM~6gKjid-OMe_DNgU zuDeE$xh8K)%DW1}gD}EpPf|A5nSORhY0QACmT1M|?9H1YiPJLc6kyz@ z_&3=UKR#bRA3g)0#;5WrO;Nlrr=ZVMgl)J4Y-hI{B9ng=7voUi(XQfsku2T@In8)G zc=KQ_ZypS0K303)7=ondmUDeOcwTB>92!#qN0(pC;L{nNa$iuz+us=iFPVJZC9Lc}P9Uat_r<(FtQu6B;Mg%Q>8^Yj_w-lyzpJuZ>o&l z-k$@1pf6~NzpRwMi+?rqSEQQkdF(}NJo6`Q(AWD9%$>X^&*Q#E9``kjisqF2CNAt5 zo;H9RXRZu;FL-(3kZm=E2h*wxAHKD=(3!WUuvlGDxa@b!!*Xq_!)pGvGVFoJtHbV8 zO%8u`YIC@HU-K7Ac=87BL58dMH={d`YimJ%G9oN`%+8nhPwk|=ieO|8>!yY9HrV?{!@#zT+9ofyd64cJMLGc{ag#r{}c4b zHZ3AQlCq)`dB=!4|E&32MjE0mqYROjID@~%VTiNXk!MY<)M6J^8e*Ru>Jz(vsBi4{ zS;4W{tHu|K!i(CQoL3ADXVTJe0%NmydP#kQUec zkk*HfcF@*q@ge&bB5`tF(4#X3JpasbWb$?0(>5$^(|z+wQViM-EtO z%`?PWemp*WvWt01fq&P>;Aa`EhFey0C;dC#?L90^{SR8GE)ofK!+<6+>@V7coE^E{{gT^)L&N1=inE+?mg2u2rnXjE$%7e14wUQ z5;oaBt@roRzQeke{t7S1N*XI^%)oo+gXY*-hM4@R^dS#hu@*zPWziBX_BV^@7_ef+ zKK4Ddd$HQfVgeSt?VH8=l=5U8r44iVmo{`SpE1eLH}uYb_nx5G2g`zE!wr4%f4(#< zeEw4E=T2bh^xlQ_->Kz|t*c_{H7)Jb)>Sc9>~xn>-+sm4GElWx-d-FQzGrdR2KP(>2TcYS6>^7Q z68FD+b@{}HW3>%8x`X^BJgcwR|1Yxti`-$e8T-fF6K&%jU?Staj8E(%_ZjEr+jkG^ zMc5JK6VD)@xCH+v5xCeECN^y&T)l~F*KF9;Q(FE8j@p2&x>3xX$N;2_W9aNlxqhVc z<_YMQ2fKO0MMk1!jLJ*#w9MjmemTB6Y#S1rj#=p%E2!7A^SNf~z5p}4YsuJGhY z%Ua?cv7P!mV&j-2V}I@=4%i7jqUIa>yLDP+f4hdCOuLe}>$MD79cPlm%x#33vhuNw zZ6Qx`iTe%rcvb5thK3Fx{i;Nz{IuYLZDNzh)bRrSs8rFPBmNX^G}D)t=t~f1gFN1= z%)@p}fZmsY#Y#B?T(6#^&>{)I(>1*8XKanhs)p{Wu|s&7((mr^QYSoY zS2j3bXkIxt2@O0?pD($`8t$ltKi;!k+3UOS?VnRd$u;+_<1Q&Dl!@GD$9AyBXW;W` zN|Z3Q zN7)-heyCxeViRGTm*4+A93W?@%ZuDNz`N%Eey6tKhiLw;<-v!pX*)E@j;@Tx9rJrP zo0+#F^mFoDkiY+B*){d${gfV_|JZBywHgAmBz`y=fL26o;9cA98MBQ_AF{4?<3q<(?_6v|#py_VqyYd!ENWt*GDUWvKm z9qjk4q3j}P$c6;&R#EmE$_}RN_bEGsvOg0$Ccx!>`X^<4hR#DhZw*M<;Gx6Op-Na1 zUn%yzDwg<6emzu~)cawBHNxK~G#omcxX{Mq>LCZRW_$H8XFc{cr40dk8v;7stzNTH zXxN56@;)cJfSx`zQEyY?KVn+c-pOnGDWSdfx`SQyy_B?X8DFViXw);vg1NsQmRV_- z{Cc3f?m6Iv!KJAgB1e&RFpYI0cd5kQg4kO4W(IJ@w~2O5hmL_R*wBhSMd1&jRakFu+Z&R)#okch8*+9I?D!1(FGtki4jX*8Kxp1` zGKMClv#E6MwS5_3u%4B1jKA8KA=pvLM8E2F&M&>o4c2$nfaJN0>tfa3ei<|1r(Hx=QAwJt@z{{W zrY`6H@&x#MS=sP=(5>6eov zx2m-t_u9EX>aVm+X6?$j3%_Ou_kVBHChU9s>urkc)+P_EJQ;koO^LKAsau;S$hnX< zNqY|8;ckz>M*AR}HAyvE40?Lnche8Fk8}RNJZ9K(zA|R9GG_jhpN#L;zSJJ=12=iw z=el0|NH1`EzLyQz6m}Asx{gy_Q!X;DNBM*f5j*zqL3KMZ?Kv3^)z^n`c~>Fag$x0b zML+-PZ!TSr?gEGYpiSKcv`ct*vUX(MSiq~vxRuE8?7(Iacu#2VK%MUt*l~unBk$U` z(l^0v$UmH+;GYAmodfU;f=Q1Zx#Wj|!62KgpHNq&{*HKvD<634U_HI3Dq*5u$l90` zG}vH8zc6{of>{ev)gabTpuHbB*NDH-vxap1Ao2><&=vXso`R8elr$#EVa&Ew2z?-X zp?==YRoMd#oNs-&C#*WJ*+Z7I|G^_t-~-9~@OJ8I=M7=q#w+w$(P-s-5^!aqJ+v>> zvLJbZ5t)t1CX7C|ig?yv5_|@e(N>Y{YqU716MI7T8sI3O_L>8171-6a`v|{|e@lq1 zVvUM?mvy|B|9}XwD-dA!;r}K61EaZvPoLqZPdUhcP#;^xS@4Jt|L6D*POw#kXu<3xegy?tElwzx33~j#`${tcs6**OA|t`>$E!( z=?d-?R>+z|uh7ySuT-4sr&N@vCd(@MAH;u(Qn8u;eeyq)|6%0e-=y)M&i_dAYbHyn z!haV3xk^PX?;$qI|5*O>m5ROmw@*|mWZw3JS57l;<4m^o*dNmMR)Aq$R%)t3=WGsk zhYjZaXXr;qKgB8h)V0X<3e-Tyk(-qBt<0hDfm69l^pdkE}Cm-dj1uejEC5Me2f~#HXTj_tMMV@<#T^yR%zfY*v0R zFK=R9sW-4kUf#;OQm^Uzd7F4g>qubs`nVr$ZoqTCC`*i4B+B=x`wsJmO#d>W= zzHMfVWL#v;_%JqB!UfKY&tXHN1RJ>6DwtC-djWKBXj#SJ1>-5>BKal#LflznCHV>v>^aUB@`)P&4ok7EMtRGX)N#qQoJ9S*A*-fsHQMuMHq4mB3_gw_$ zzTm8I0=zD^806fzA!bAPh9Pg|ZFumlystE0bGkCX{FN4!oIY!IGrTV0ZCQR*GP@aB zj^$T;+cfxIoQaPc{HVv@b<(&p{N(UGc_;7LllKcHJnus#%KS?u-29=|EAJ$-z>`X( zWex2Y{<-@c?Tb8vet=I;D`v}ncva@XinWBbR>~SQ_E)X78u(YiCSrfhI!+0(Se0P> z2J7bK_X%yk&f3ks%TaL0#HbNX-jfG3v4B&D(@Q)4R)FiX@@c%_*vs^bM9{(HL0laW;MI>R^RN-L{oOBZ=%f_ zuPWtVa;{IBW^@{M?zogxZFKg+O^D3yY-QaZ&s7^w_$!USW1lq3P#{Fhi_cN$o zzB(&Y^wFm=Hp%SMHgwHo?RIb>e#3 zx%2j;L{ZYG8n7)%AM~&X?&SQbY-=cT+h%*@$BN9?Iiq?GbiWFDCuhj72;$^K1GM+AaNG$8=s|zNGvr(tmcn^pV}t|9wp7r=Ii;q(64O^u4>K z|L-xKZJzY4r2p`G>2KJhqYkoo?x`H>Q92qHDy~FS>@09HwlzY`Dp_ zf^{u?2^)Ma;iWk#%e`iXpc7~7RMe~w#RiX}*jhdWKYsX_yp0j%YNsAq>yk(QCsAIT z$N^}BJ|}B$b+V>2;}g})_-ri`SUqPw8s$tHU~7z5O&z7ICkabBs&$h7eWRM}jQ_oF zXCv=k8w0U5_Q2TAAkHSOVw+-(!8wL`6dKtLD-o`VwEKGf>k0ePH}^VCau#bo&MHs7 zrMQvF*m&O6#~H;Po(61)?$DIZOv2GC)!&D2SoY+ohGoC!`@^Um%i=F>@2fg_%TvEE ztG8M3TDgxC*;glhm3KtfnSiHc_TVD+V4=Uq|7z7n(;=Tu7KwY!O-pkk1840E-zMBu zM19h(?yx^KSBvP5hc-~|hGhetRns-+k4XOnZT0%yV{r8iyPo~`_Fd9tlXo~{y+KQM z{_0iEDuiVbR*PMA`d;qCouy*t<($aokfBPuh7&L8Zov&_yyi1r5sX(cZK!0t&g7uu z=^3wL#;cg|DrUSY8L#<_m(HW)yr9>6c(Ai}o{zvqXZ;H0(y!>(cKWrMvP_Iw7Goy( zrsUVP8|7?dtTA2cMZFEv{hX1y>+1}`9ru5MXB(#bI(^7n|6P5pd(5tVC+tQz=<=c+ z&_cYoNi^h9(*G0O@^vji$<+UCic^Jd*+||lgL>$zpQ!9>8gv_H zrQLE50NFZvyqmd?BQP29KyGIVb}W-rCEaK+rnmcqr0?~MPTy=yNUt>~rI!TUoIWln zBVBkF(#ObYcXC%C(p5owBL%PEmp3`)O8c>2Ha$x0)keC82tLQ1O}Lal?sD+0e$ZD`zehAJ8E_ga|z8%n#+rFfwLTF!))IAbDtUiCzx-SI(xu;TlQ6$?~doXua%D^ zUl400=&J7cVaexN8#l`LuH*|NANPj6^wOJ>kM$sH=SKPdL-H{OPefhK_p0Q>maV=P zvFD?&y#UXDfagEjvsR9YxR$mU(6c@vTk1#|8%#%QT#b>zqh^5Q11=b zH8|#)bza9z^^oC#i(e34LiwI`{l0fy&+n1Hv`79VPyPoz z`916U{YcNcp3x)!y*=_rd-C7y$?sX$??-yp_2eG;$M?u@^yI(Oli#zh-;eaH>)ams zGkfH}#5$01xYd*2v##Hd^sMV)J@Q+7(eO2Q4jY%e&Xc_fg8*msiCU(@v#rh|>fengAVYgg5BjYrSc` zX}q+m(5c=uQofRH(bsMFb*b0Oxo$7#A=xXjiLrq@?9j5|uFdHCRU_z+M_f6H$ zlf_zKQ{^#5r@5+xCjK*Fph z=f5&69)J9T%ER%^^N+b{GZlWQUQblA6M8PQ@|A$*_p`JO^UKW666msGXgrZw&L_Xn zd{vzHs(>dsJN%L`o7nP{d34^{>9+Z8bS?12&m~@HpINxy@OjS*#rS1?1wwD=CoL z1d~(rKDx(WiQ8gw{)Ms8`Kt>nxG=en(Dc7j0(^g`tz)!rZp(jY79I zr#!PAy)AD$blCq(q*LaofpvMM-n8V*aO(1=)5q$!X6N0Kjx+r=;m??zKO(#hUcoit z+sw{9!i%|UnogVB*WcOMj{j@Uk22m$X?CZ^9OSeHINt_`35@-dHp`fOL0Uacd3NVJ zgb8d596ZCk%NbPUD90%$k#U#t-p*VLjO`_@$X|Bj625IZcrd5U2+tGPXrXCgRH`Ydf=EpFMa4Rbc!V(WZ_K5hnYHF)q? zHu3UbJ3`s^6JV}wb#~`7^jYR1mpIO;TQiBOMag{M@XJwh% zSr49+e82YO8$n#jN2LA8oi>SoiGR!!pDOXU5Pyq1KEV^e!V^D0;`4~lbH_)^|39;@ zoQdU}ne?mLjV1q7-D{wlI7xGiJjdL5Ec_SquO)C78@Mj{B)hYn^|$(VZG(MIr6I^gUr^cISWNm%huq=y>C)Tbw0@cXk#R`sr!EB)_Elz?1H4{!8W&KhGV1 z#uLBa6MsSC? zej)J--SO{v;$QH@AC~z0iND_+|CT5IX;1t+62FxArSAAQJn?m&_+1h|{r_X_&Eu-7 z)<59A&p|nW^8w)if`}>L98NfK0Mnx;Dw-4CM6tpFlWbBeP^?JQ+-Vb*nAX*^4fUp` zq@`wqH!4ld#(PEZ7K-J7rdv7pyx(UHhaLI3zxVxo-apRgS$nPTGp=W?z4khLK>JMI zzDKmL7408T`!cjI6petl(IYL zGpFGq8io6ki1x+v%lKA9&ev!sdIIf4ksmgq$b+Le*Zq;E@Vu1s_q3kcBys+m5BW5I zv$=alig8%jk%M}gds+PhDF)5ose)$)+UWif+DEUAS0G*we;6S91g0b$Ita?jyi=C3ziv3zYUAaji2!%IOYW zHQrRizUT>@FYNkK$cn}{MPl*4EFD|udQFVefxyzah3fr9U0>AEJGrqri8(b`?}6h( z`29ZEvX0h6G6rSXKlMk5vb!F>)>Nk&5$KO_ z>IammZ=&(e+${6~;lBod&6!jqWYJ>8pYD?gVdudD%=1(6KDAP7cx?*y4;$-+KAy^(?@a`eKuS3o?(AR{2HuF09+#oGQoae^^C)a?{T2o{6 zo$zZ0YcT>p6!9&d1#R1eGz5=VKm!l**WM^D*bwqa1C`s9kRfO8?Kn7P@X(6SJ}h@FDuUz$2&D z(E0~Cb)4t0UAHp6FkNLdSoZ=)^AF28$iL0>wABqp?fMPS-T0;x`GF^8jo)z0Y{$Ch z0?IUZBj@nlhEe4;n2S<@OU(+=(m=B-Ltsl=JsoqTj03IJ*mdj5A=-?le66+#a#-E8 zWUU~9|DVK;-%Zw@!ZEFhaeP^6uV_!=nC4k_{HUTmieri!JN_Q;2gfn>OEm5&p=Spu z7BqK<{MClbN}Q@K#eW#IUR(+r>hRyppX|?S%j}N=?~TT7mAow!ZFs3Y9KE-$5=>9eO z-o!sKXHmb@pXBrC0`Ta_Mb1T6LA0N|z; z_SVRc`ePQOI-7Ka>8}Gr{aAO5>G%KVZnOtAPA8lJC%$0z>-iE4ckmCf}|PB>AOIV{)lt^e6c!+meL*EC;iH zvEZd~*#nsV$y|Q@=a3)E?Vl~}|D4+&$K*%qu_S*p?B9%gy-7aGHG`%7T>eax{h`2= z4-xhsfqW~sf1I@c2)Dl#lW*69Nd6YszeO=@Bp>A>DL;#@Bk_(Ill|U-mUWYb{Q85C zUme2YpCs)+$nC!g9L+m*pw4Zuf16@lM%jgOoRnY5<=<_x|5soPw6pl{hx{WX-=W7y z`}cGEE17(|{u9aH0sD6-#`h#2<*kEc{3DqC;U@c!0rRAgU%wafH*@;~r2Tuj{YRPn zNc{-O-v#@3VV|DlqwJFME4h4&$^N~-IHdi%Ait2?e+Ba=%b#7`{+&#|UEfCX_rm_Y zim{dCqm1t|@%+r9>*!GZg5ah6DFtR9^t1fm0r}b7{vUv0`M-nP{~D7YsjnsZ`(gin z#aKh~Q7)45E4lm=f|v6Dd0?gp`E}bMKaJaeSlYjh+h4%s+w~zo)KjkS2ff$KKaK9&w(A*;KTIzJrTyYG@UykZgeF>wSl`S+8$0)6EtVzL zpKqa#&g)O&yq=Hq`pCbm743R~u%Q%KTH};}b}`$~zA0>Q=)F)+Yr~bGNjRS-SrfJP zumk6P<9XDzPeeIUF_xoDI_qI~cauJ`CcCUS{e3nMzJshs1%H1bkL1vIjAiWYBK8@= z?mXZrcJn|z_|bkQi+wZNcg0$i)rINzgEo%O6x3yj*i1y3?7j(Ksn2-KL-nzYAL~m- zL-czYjnqdn8mzlO|45_$Y^!%enVw}FfpycL?sdd7Bb8WBd9CgK=Y93kM_;VE253dN zZ&ubjnD6-3iSYey=%YR(y&vJ{^ZjOkw${YgATLNei+p*@)F$KkU-UWUay!VOc!q(x z%`%%c)7~}l`D9LL&Sc`f2!FBjypU;w9I})8Vx3uL?-tr@6Q57!Q!O=mXPNX{+W4)I zcP)wAcnP$?EN@yXjovFF`F!$LLMK=BjI@c`o-Zq-k#OIUw zs9ie@ndzoBX(yeZDVARdJ3j@b{POkB*kEmniO(l9Cq!F{>*i*rHfiT8LgpS}=T6WR zv&_i0T3ZvJPiFg2Ee3XWG_^@PmkOC1g`Mj`yP9RzglauZd_I|qgUPJIJN>Tr$NY$V zr)yVf>s%psg|PKm(6$Y7kH;K!Xtm%4AMXF~{U3K+@r%$>@PCG3YLm836fz$Zwk`lw z%`z`XXnx$*Kg;aej@i1`Bvaa%B4lO>J2OEq;wSp)c+9nS+F=u)@c+2dESkwYW@?jm zb`>&5K@N?hVW8icWuA)Gel+pKW3$Zq80}RPpHJq3 zSnVw0`Hrbg+G!CoV<3m@j07z+%UsY=+iBwS$-LP~n*o`0J^K~<9`C?Bf%EOoRP683 zx|7adB#-7^74jzEE7onrINzr6b;Cel3}EVt`MTj2;oH>xMImah384<+xF7 z-L^9odtxNNx8S8Y`aCcO+S$72dB~6D>xNiiL;dsod|t)m+x35u{A!%ns}6Cz$8IF#p-?H#rf2m4?Wd(wtl;U{H{Zs>hrlDI|)eGy z@c29Qw}AN`?IgE;3FICjxncTGz_9o);r5pRNAZ7|Gk? zm*W2dFpD6c?0=4U$^IvxXEV3|u(baPZhrw`>R^AJVk{>4D7&QmN-qC^$^JZG!leBR zA-9m*zfH(B7Lr_tJ_k6m{{fPJ1M+Vu#!Qlra*a#c-;vq>w%{fErvS51^sn(S(*JL+a_WK){M;`tzrWz2e*O)ZI`}YtEMM!NXZ+BS2^~kE12%M& zIx@MAYQp$Khreq4i*%rzGp3=>J|vm2-$ngZ9kht|ThJo!^ngWdA6@j@?~qIX3s8Jy zzn!PLgY+E=)QNth>yN7a4RzFSXHZA|M!LM?P5o9#IJ`4@ltO)>Hvx|I-FR;MZr~}d zJ3x~V7uIiMC{KAj&trb2{~7AW@P6A$c<9*->I_OTGJ{lOBg!t6bJ81pE@=j`#R5QL!Qq|_Rn#|XobD1Szh8hKtiK0xzxxAE@k#_W@E@Z?%uN_4 zm{VJ8Rl}XeJi$}fkN0^O#uKS`0(II`j45{2h(+0jat>n0$7(00_W{9M2|3}U135uC zRhJ0)g^-U}P7txJPvkb)nS8t6g5-xHwxOyKMDkH~VI1+X%H@v+FZCn!i3ONC_<-@s zbkz5tyyZR&ls@#}KHTcd^Y%KZGXg$DsK!;4T__)gjogRM%!l44AI<~wx{zPj1@hCl z53$mRF5HJICLia2k{=BpqE+KZl8-Wt3+@A#PwOD*!#BW?592e051pYSmirJWa7Jg+ z;n4rdbVTZ(kd9c)H?gX5m~^0=Gfw&t$9%9#9q5OBq+>kIXAXT2@zR_b3q4ly1^;E1Z@WHjR~l4kZET! zBlQP`%x58s&XEP6t<5rRUA15npHJqRZfw2zxk;b2bDWU5K-f7Kv>rd7odw;sZ%lkX zne{z16?UF6wMjb%37MIYLv~IC{nac}>7`vT@%dyvbvLu~O;ek+v$K#n400%*DWE6J zGUp{~TTFaDnFo4nZ(*L1%yoUVLnc0-%u{{Y_4-0n8<*+O zJ?IbW_ejVgJ3~RYnq}_phy1nc+xt+A@$L+?+3^gOxXQ9@HAG=gDy7P{c{rR4$})zPvh(- z&@k8?hCM6=<1N~%7^hLEV9zNUdsrt?=6gYQ{b`dv8T(yA-wEJI-&dgboAvb^$l^2$ z^`!40Xk*{VU(%6-{USPv@@pK-0{+x(!lrDrQ{UYWdI>))cL3K#QrOQEMbu#_Y5 zpp;)eJsVZ%3D*xXJrQ~Y+QJd%smOmi-%LmTw?&;3eL5ZeX-8S+$d@L4(yu!7AB^|tWK3yPdPZqCVDs3RST&`CM8!=xvI`Q^}m5PEI`OZVT@fi5@e zIckNT2z>?WY2EuD(8hjMqE7ntFO<1o4t=djpY)67Px9+$;K;AD`KYHc z_fOEq{q+gzNCz9&SUb?Z`(q|u4$)_ugs#KDQ=c6G9c|X7@O}63s3#w{gEo$#8+CGA z{~cx7XH!i2eB=5_e@zwpHq-AQ>lMMz#x?Brjcb!E8K;H9_Lsmzwyy$>GTT1Qrs4T@ zU!MAGcq2etiu2h}Q=5zj&7Tzer-jTXK?BS(7X)etOyk)n^ANuW`U_K=v~z@znImLo zgI>eWm#4d%u{^z%$YXpT+8X!gSk%d}J{o0;F|EyM4Lt(KbpvVdAM+o|2Cmg8FVax1 z8O7&Ex_(Q>ajn7DMCSSGxDwYwdrR~Q*^v7e9~^p^@Sz`MP^|9;J%XRlhxRR*5C0TC zbVOTYAEHqweTYO^j)7w)eZF`eXr&z%_cwnFnT=&N6a41j3r|ThyULtJ24%#@!FQZQ8_-`na?X)J}2-|Ha2Vh=_dn_S|=0w_qp*Yak zW!JX(wnbI9Td`|8zfpKs9%4!J#81F@BiMBr?$3|Y&K+q5J3@8p*J{`k3mnO)kKk+W z6Tnj+e?_vB6~kyHt_MP~PT$;0t4eknO$AR~DDl|!PZ>|7ehAdr1?%-Ls_`MpE|j-| zpD^V(_S%IVS@GJrLT*Pja?1jFb!3B`+wc#P2R-kCE|mIPLVq^&L)I@CD->g+CE*?V zJA{X?8$q2taLv<0HQq$og>oFzkF^wzy+JY#(4W?zzY6kL41&0Rjmfj?D?up+KeQAv z@TY4c9)nK>Pn|!H!E(kEspo?_6A^<%)p!bJ7s^H8rx-kmW4z~su94TIW+mYJa}72e zhCGUaVdXw8AbE(vTu?jifu(bo;Z?D})RM*E9T9_iZwg`%sm~%jVlf@m*$*-3ryA2x zHqd56-q=1JZqzhq|Ld}G8G*U^^=AB9Yyf0Y?oI)o&OH-A|AU_|cX!i$5D|Lq-8|02 z&^FC9ugN(mPRJRKc8X^z=t;92SFqO4)-YE(#QA~NM5I3nSnAi_pggD8|C`5STp?Om zKm(uD*G%wr0iN`A0^N(B&+bp!Y7r)VborX zS^`V@0zucC`Ib90uYZG`(!VOfrvgj<)gz9p%zV3XpAYmE;rTP0yB&M}1OMwDj}dWS z#p3SJYe2oXRh~bU$e$xLA7U>QW2hHmK{nOj=J`;`cyRv@DCMxDoyg%E$f3>1A>>xD z;HkSoJa+vg_XWl^xuVJZ-tBj^rtoG zr!_0}!R-o?7ooRf^02-Dm2>Is&nX9ZA7n^-Kj-$gX7~ub8K`qS{2#9x0Vv;=bLrXP zMow76TuS3SB9!71q5q0;O>;|A;Aw7AL3vJzxol7m^p!(@trz#P4s_=o=2BXZkeq91 zr@pxaS^!zTxoujMb`H;P%IhPtfsIeW_Y3eew$6g`oD%xSMr)s%<|sM0(fOJ5eJ}K# z1f6Bpx4XS|)TB?&rP~DG7r>LhpMp*_^BsuM&Y1Y5J~}^>z5_zvUQm~r&l#(|YvPms zy(sv$15f_BK?j)mCUn$R-NC=5f^VbHw;q)8%NNg9owPM3ebT>$g6}2Z$-h;gQD(le zowY3{KIz{Bf^WIdmk-*ifp2F_t9b2E@WtWz1oQm_lLX)6z>|M@pcXS;m z9{hVl3pe+5n#{x$*5yI=T=XX(!!Y3`eY)203Ux&0o-_XYa^gHxVaO~YCV*veW4f>l4f9kh!{RfylyS^8cor@O;8@A9q%wupFc}{s!w}r=G z2jhv#2{NWwxH}nc`NuSPMdM;-7I6!6M5HQ!$rtrd9<0^u%6^02CslFl=@34 z@0zg~90#7opp?hpMZzNnD?ptOAqEer#`7q

    y5z9eN>-S+3?7cRh3+fiBpvN7}ZY+x95wn2X$=s~Yo32g+L?CLbg9IXL#dDt+vwo!d;;@_gM# zYbNI7t6a`ZV9Ce7fi46;>*ux5TgZKURrQVZOym6&;lXzd)CoT_;p>+uyHKu~NBSN5Q5<`b zA3WcoKf6Ict(hqA>ybAk=Ma-;*Y|_6F#s7X7m?r9X*>pUZBvI_C;xUa9<2XCo#>k> z=(BAoyHIv98|?ZP9D6Yic?^0Z4;pN^j5(Rb0OONvC?$D_K?$f`>c<>XZRIgI1w6}x zY#xKPga_YhP$zOJ6Zuw*vJ2&-b9oG&$FUc4A&&v{R|op?;5g*57)f6uuJgNW%3w2!g#R$2X$h6Ou@LBgR%?dIA()ge-OvsNiqg~kOvJm z?2s{-#BI2rv~490N%Lzp~#Lm8A}Fh$xhit>QR;1L;vQ9K5{7!TI}pw6|30p_oG zlwBy>m<@LQE*yJ@%NX=U9yHi6U&dfKx1k-$Lkz+|PtF!GNTWR9G02cHNaHbRO?bqh z8L0DB!~k<|0Lm_ui@;AYXo6#JvWx-rv-gqnIXPX%Aerm;bSHUs{U#{IAW7OVkn$ja zZdM7%!f!0iHEcD|x z{7mu?gYQ6hfuH4JBIN<}Lsn}UgG3&KQPmhl~OA zXE*3?F6H&$@(wV0IRAs*Mhqf^4MrEr10Dlf^DvvcaGQ599*3@jIyd9oxEbevlwBxO zKV;GSdgcXbDcBo*qz!+k&{niZV83h0X1v!N`wdxE1#^K-e;xG;@m$}+)pY+D-q{GA z2L#W0wB`5%>(8;Par`=DCsokB9=Nv!*r@`$hOj}v261d5j!9k&FyjQ~dBU^=rX|NL zW*9u91&m8zo*|4K7(2)02{|7|az7rY_E5Bk^7h$6#vai=pV}kP9>LpZ;28Hq99LYO zwRZ*P0m4KB6U{M`B_=pd+bl5C2onoTEXRzIm`~!g5`h^{n9jg-=9pm;^G3X;3Cu{s zbOELd#|)O3pX|bN;ZkcJ!@Ed6{&UO1e{PfmVbZ3KT^a5_0vAEJe!%tPxE6Gb z+^IPM`N)f{MA7FtL?JJZDDXAk&^V0LEx_~hQGYi6hKW3;@pii#a{4|#-n^ZW>#=;i z(VB;yD?0OBzXqK14958x)%YD{7s@v9k^Bod_QuL_)*ox!hH;iC<-~G1Rlrg{{tJ}N zHIQMz4$9+H77hrOlA|6e7Jib6%4!z3)_uf!F-GjsC zk7)Q)ZDaEnt$E0wx@fY)t{-CZBK7^C&RY0Ys~Y=IcA=cZ?g7Fz7mmFVa{fxv&K;rW zSor+qfemafh~R#`OY-2y7SKe&Pxn&L`hxyPiOa{k%<{Rt8-StXIB91n?A#1HVb^!S zvvYnZ$#v+jkX*!A19jG6o~u)h)hN4ACVvUD0>@svj6duwY_RhrN&h~upd=BK%Udn9J3AwXCuj41Kao5BsiP{tspF`{w4HA5r zz*0;mg8pjev-Q>nSpTZ8yWkrQEa`KB{$S=S>VwA#8~7raK8N03@FfFF{tp1HF!R0E zSHs6}{^DO7!Pg5|($^LALo?s3{#uNQPx|L4_+o)2|Dr&5nECc4X_hQ7fm#!jKItEwpNY>OSn|&Ty4KA1_C4_J^i#NxL*|n7fzHq5!)f5jhi^b< zn)$A&TCIss=Ac{f9TobHfKD*;O}1+LOnkDB=-f;CJ^-HjXbl1+>4J?^+Y>4U<0UU%uem06h8kI%tBKFU()tW#W_i==@A}t`Yi*K_kt4@7lBi z6QA@iTktIdp8R_nw1t^(cYrqI4*uOI_#PGd9tKs+d=rASStfnbzu|)KLEy>18K9Iq zzJ6ccOj}~&ll~68Ao6nx!*CI8|GZv!m(*Bq2`$H#XqSi554lluN6`22t+ecmpXb(A|kzO8MwYuIOO zJpR5De18B>{#^m3-0|`K9I6G~LEk?GUk&i2uNstc$H%wEp?z=CC;g-IGx_%;@Z{gO zpp-j4zOCU}`5p9?3BE6dzRy7^cYJ(b1>Z-&lYix)lsi7YvF)^zCO#QY zIzN-0JB7Y&pp-j4zM3d)qlr)YS19;40Z;zD2}-%+x8}+K`D29 zeE44j-vs(g|2-u5o&%ozTMA0Ku9dz+Ku& z)45Tct9B~E-Pp6BgH3Hb){%H`4UgGy$RQt6K~I`xUhW1TaIaee_n|l1oPUW$q~2D@ z>4SE%ttaSVvz$kJuend9BO4u{x>@$Sy)@4q zY`P-&S^-No1%bX{=9`?z`ezgK4fhv=nq%a%ab9nqjr)MX+VxM*mtR7iya(ta%IqE> z^yPF@A4Z7v)R)4hTHq+xegS>LY}2ECU{i$tH0o)M`#mV`8C+w&7xE_XXw|A=s1zcVRl0M^oxI(^rP9mfvT>P`OfF%qHE{7L?;1wZ+> z8Z_GHpB?@=^iHTF|CWN{8t`~(4p7X?Ej(cLfI9-_kAA3G1HYr_g>@uKSBoSnvS;DgT(cw7x#Eo`{TL? z_df}H>b-O=7^zPt`MB3;Jnl!zAo(b7g?-ePF6CEp`7cRdAZIw*>Y54pcs8*52+7C$ z{eYnu8F)^R`rb+Mac@*6?wJ}u@=!uw5Y4L&d7U4ueL40y@LNYJtd9Xn%|4@Mku{|VC( zp|?O=<8c{)Iyo+zqDR%yWqD zsSnj1h!Kq=()~5slJFijy2kh7`O@mmJcn$ekLzwwALCvi;GEbW$;7_dVU%4c7xDjL zO?_D)^`6NdWu0O)A5&;6Ugh6sH;>Zy;ng9iS!kB=*poBHO?*}&la;Osq*n^5Ls zkNffq^s%uAKi8v;jlEvNpE~$c2!AkEP6=Fn9Ul{Gfpy{@!11_Ga23ifl#9T}<3u?U zsviRnixc>l12g4rF1P*?9_Ur&;J+_RX8dl?@i`6$P6`FN(B z%ik+_sgLFXLwlt^3ES)NEOd1mxBVSqM|~}~eHPP!@7|CO+($VD_gYRP9Vl<@$$Y?b z<@5|fsJ_AE1I2uikYE2Rp6Ed-FKeRYQlB`|zmrp_==U$aLUcDx?GVqfWtnt6fM3$~oZ2`G-Aw9E3XR zUl+ZrJl~4@UYqj!UUw8^BqK*D*6o1t{-n_VH1InadrNeWIi5eo-XlGKO6w<*QU4Ry z9Sl6ppDjr?>>V%ct^W@A%}rUpxP-lR-|?9E6AyG*K%LRBH`-!&<6tk!Mc`+;E9n`F zP(4NPl8!%snbJqZ<0Rx4LOxl}(({Yh@qCE#eBT>y3BnYc_mEB7nOM>&qm$2pYd zi%>mY_(L)OH!$SGN$JBe=*Z?igi0TdaUV`I9g+Gsqyx|QO~G@2Uy}}$ixNfdS8^S4 z?L$6%3d|HCzpeuE)3^^_jBDmY1^3}2CLj0zl6*X)n2Bc>-zWJf$4U8FD)Zqcc;&b) z1BUqk4BKn`96DmT55E9MW3cXX?!!Aw2k!qR9eByALamHY6u^x+8iVHJ~a*NaF#o)yi+Go;Uwe3V^MewKy#@VUu{r+^_J>f{`8 z5IU-xuw2?BeK^Q{c$Ddg)E_1tc;0mio`ao7I#6!#f$P|A@*x|TB$-S5A^!;EBL`^x z!}?)A_hA~7Z`UW0d_04jiDz{ukbIQso!ykjV{y#qIy&cu>g!Ctj6fUtva28O5AR;+ z+RS}fCUEt8xvmt_k&JmX*4~;Q1BCotkYC7sq4f{> zQn!o8DxT!0!G|=95liw>#yh>}{-2$+FB-wt$9QKh_~^Vt7&`Anf$qT1ca3wCo>7g^ z_jKmtxE0zC@N+hk<2#>!;_rKQuL9V*u77?G&;mYsHweBrOxJ^q*UoH+)am?8HV}q# z)&u_4W*gqNX^TvJKAqnOuy;q%vn5id^o!2VBoptNWPafrq3iH**apAwod9}=z2x)B zq<83xXHsOFw6k2u{2sE%&Xb_C%rZAN)9AhYlFuizra60;3+=@(2T$YJ>_;7~Pt4b( zU2(6olp}q5UFhEjJoVKs(4l7ir|5pN2z?~#DUaR(y@h@-U(0MnoxGNL6J@zBqIDX+ zFYnXov;E#y?Achd{XXCg$TL5;wGwr4_`e6=+x0yoychRAq=I6&j8^JGZb2wj9BU9&-N;pfZgee}KoyRNW0hn@+_ zzCR}N(jmsiU?FE3+G%V}0;N3i$$8r@_c#7L7Jd$9IrNe#7i12_37Nx%%v4ayBcIHj zA=(BLpRXU3P%Yk6FKwiAGUa)qkk=jbpjqCvP%YEM=aaX=Cw||e=dO*m}Sn2)<&85e0DaA@jcVD5j>6a>^sy=HStLw==@ARd@uB!1f6Bp z7uNxKh4)XOp2p#)ppD1NA=Jrhqz_T19Hz0?)s)9RJNL!<=8ebiAgfI9Hy(culPnpl z6~cBmcqo_O2JK_EUF@9=L_OL53TWfFXsDCpZ#Bwt?(pz?X8$|}`ghXKis#0L-$B+h zsH5`%$uQ4*7fiCG?U};%#o#eu`(vO@&9+abcQx4cW~it5KLpy?_L-=YwogZy^d2+m z^2NM9-WPKRaCH4dc}~xg^~Ev8?r1ZtsSjEfd7o&KRJEiDUsu0UkU`g5RZsBiGuO7#{nJpw8hq?+>>accJV;+4g$FbwLx<5ud9$e0b5as4Akt{-itD z-y3x#zYX}jr6&8U>DgD#qs>*8_XRkFC9i2>a{S z^Yx17F2YCZH$k1FVE-tKaUEqB%12H1e~UV@Kc@xkpAP%?`}yJ@a0mPSnEm+n0QkIX zP4*WG`#W>{6J-3?a{GS(p8P%qIv)Dj`uas+v&nwE|3t?BMIQgJ7(P-z3hEpK`^Q*} zPf>QETvXB!|94Qwa)_?Wdcgk72K#@2%*Oe13Uy@1d*JgvZ?ZpD*dNR7r}G<&|MT2_ zH}Dkyw?Q4y&+IQGERX-cfhRla3VHnBWcWz^HBjex*gxK4yo|C7W!L%!`yWFc@fDH% zUKQ7sR$u(xcd-9$X1`ru20rf+W_66~L3F&;qKg|h7pW`-3_&VSlE@=!3EgY(HNg+Vu|L^Jbguw+j0c zx&3ow{Ij|J4&W*N!Ju{hurCW6m=Du|ueNAa{hY>V;d9+|?n5)igYOG~I;Z3M4Db5# zL)nFL9QY|73XZ**(vBv`1B*uUNHx}wBV-ISxeYfv!UpK611*13=${1ro1q`Jb(1li z#Pwe!Jm!#JL7mx%VK&|$kFpD8TdCY@;PY-L!Am~=1k7#txc)7!1MlC#brtt9SlTq6 z+jNrY!2TcUcnCf|WHBm82g*mmk9(}vC{GU7UbOT3tG+@TI~RTI|EGJ2K0-aiHXEWX z7ub(cXTGOskHoAQqAd}aa>C%didp!MFWs;8uEazpYmW%bPQu_j6IuAq1YtHy%=fqt zLSVKKrVTJ{IHrVSb}CyH4forzb?NawXQ^_e_Kd%tXTA z`yW~O{s&=3NX)6GT1$Z$O_%||4B!~2#7ywlnhMNN!r)sVS@;$R$>}FCr~EaKxEFFD zVN!uf<(TdglVj7a3C!Ju83xQSj=4)>A_KHqfr%%~NMJ^COr*r%9l=!s)1ENtz@&4G zU1HV+YCi}}C}A>y$>5kkiD?z29T%7ugqaA;M2=Am|4xVJpxlln3=%L2+a3{nGei-j;WBC$t^TZV2%;y5nvwSn2#mqPz$X{U_K+v zFH6ki)>@{(yiS-Ez^veyl@fEVwU!|;YY4L%nAIFpC^0#0 zv=IXHJYik}<|U3^bu3d~f({2iFTbIfRonHHki1!f#!bYOIj z87eUsL$pAFaS>)aFxxq1pu{X_tEmE$Oqln8d5>f6mYA?m?WVXVw=ZGd2j+c_iIDWPAEHL{B^E)uVbIdl0X%?&H3Cs?{Tm|MT$83_A$c|c$!2F#se*p6b$Gk2v8#`*# z1?Ek{+ydqn$E=Z1*R#-JSH)hJ81(2W)WcmfeGZ8c@nd{vv#+@EFeq^U|MiYw#4*~)8Yl@A;Po) zrVYnTm6+G!wDtmXKVeMY;T$J1crH0qVD2MKIOO2_oOJGXNla0^)MlyX$9>W60v&*MqC?P5G#%|k zGtfR9bRyb`PDMM>`_WExCfbLAK8SXrbI?w7KH7;sg7$krA4faU#b_t`4BCk}c1=@+OMmy1$(4GeR3fhU5pq=Ojv=iNg_EgZnqn)UZcB0$SPV_yrr+~hXcA_7k zo#;nsC;BnkouHqgo#+>6C;BzoiJm|^z6(WjUJ=nB(N6T=XeU~Qc6@h<=Aa^?7tl`h zceE3|igtW=h~@CXh$6p_;=e1(VfsD}<*AD(^2`Q05PNbpU6q{?YC_Vt_>ajZp8J>kfKzZ@p#9e; zWB6XjLQp^C6YZVPz;jD6L%ZNw68Hb1p04X2AROeG|F6hG9qs+e_so51(i5RNF@_yF zoqI{o{lL=yEK@));OEnGZ&&Dv&@Z>Ax=4K#+VDTiPV@gP!%!#xXGueubkhA|J5Bne zU$H`8DsZ%iGZ^%1{CxUcbWfmNKgIOn`(J3oHRVq8eT@mIBOUU+fNM>9xLd7xRXk)+LMxFHQEtF+{H8JUvek~RHHUdX}tq1LD)>pvq z-@F_3~H5y+}~KxgYGo*O`vD>9Qymr)(E{d+8XzF zGt{-m-W|OkG5}@X54d;e4tAbLA5*{ENDlN_LD%8uv-352FE8FjCHmnyXybmkj5=RG zV9$>FVWdfy><5}ZN!M?{Q$N&zE;8#{N%x>e==rFp9QzTpaX);EI@u2=P-guAozxHA zOgibmqb!_%lrIOe|9$Dd@)f{0j@4n*vH$8IL%vTl!X!ubXQ{CLBa#8z%RwoRd~w2k zoD`?AsHgtf2HH4Ie@C7CziTtfWIOGpZX_P`*9PK=p!1zwUyrg8&Ch+WQGG10f0^n# z^ZJ*lz6-B^5oNkJXccHp5B}dj9lwC%A+_F6uT}l#k4dWg+8B%5v&8E598t7uR+YYw zdL?9eX$hU<6>XS7-$}OO|FzBVw+?<5Se4zoRb@BjDSc0wzWX^#oU@idw*3Az)xVG2 zrElQQLw$R~ zn&Lf}_zv>JsH0eIEg5DMDOg{HJIi55>zr_<9I}t^Nxe(kXmJ`e5B`a_1M(Pk!AfxP z*88yryc6$C&O7lAyyN~a-esJ3_#Jpt|1aK6oVWiScoY9G-j6wN{2h2{?)#tqR&n0& zJMf15U%Zx1EY3l9;BAUJ+-_6sMx0+joM)qt;?YO5(MJL3Z~9iet=9`BIc=448ZWli z-L!z#)^8GGQj9Jmh8y{+a%B|8bgi{Gd^*Z=niPk38O`g9MSa(C!;Mn3HN$^J7p*J9 z=c>&TMyf#xUGC$!APeX3Iz`~{Z4-&>`ZrOB|L|oU5#Vat5C@-t5idG$~YV;lQFVS`gGV;9Kq_Z!GM%g7?Kzc^b-F3#Z)-z!?0~H~R2xWzUka7|UZYrqkUwr;m1H zjBAIi(@!60hySpSDBoNgqhg$6jw{6X)~Sy$-vp@6B->pnMsgD+PYH0|dJA&Uc3k;% zD4QQQqOUzmQXv;}2j)eR$J+ln)L051+k3rx*zs3Gji+(E@xS8uo1w-cYL~~~4mBR5 zc6@`L;eQxvRJJdU~??C3DPC?xaOIlfWU}~B2T4j0+e(l2w z(o;ZZ;+GP(6vxhV7oG!AUpO+jeI)(~*q4`XQ#wW2+9_>px4eCr%w>Ii-#)LDW@F3eot9e(}KZSozT_^a~|OmP)<+`58{}=Lg1D?KPYS2bND?wC~G9z zM=Hj3v}dKGoR05*5%z?{HYv+WC>Y#0j+iGimLoqNz%Lo!;-+t1;J&WB^Zupb`1VEK zdFu<|7f{B%M0w{ozYtECks+nw@%T+g`GR#txJ`W_e1)QvZ^bW;&Y9TLp*|`=zah>$ zZI2D3{4|!z<3+=aI2JLDqu*Qd(3WPE$u5-@aeW#`iHMske=+v*d6tIMwH%|5xRyMMWwtOnMS zJR9U)fIOQT7#OG85~3la5-}xyI={w+%t@lSQcM#<5|U)Byhr@Lseib+`_@A(Fm766 z{Io*8x5ij$gR#QSiN|g7oM;WdiMXzjV<&J#iV-v#`vK$l7#fN(G|S<<<%i!4jHPhI zr6;akC^v1nL7Ji-8obf^+92g0hX!rj_1d6a@p#^ENJlLg^H$bN_utHaDe7i66v zyWW-QOp3-Fc^KcOcVQlnR)cHtZN&+$Sm&bX03~k*l{+6=6dmL2rBb;I>eaS$-@=^k zoUUkZAy)LQ_yduc&ph5f6Rfm04c2RMY=O>T#Hw*0$$Ip|ab=~Qz2Brv;jyj;ADv%h zA1#Gne-~{(qmBBo25aPK)n0pjyv1GFQZe3AF}Gt4O?4?KAH>?xFU)zXpw0MOe^>+b zKI2A}ZwyRZO!H7ps8UX`@Pf)Vilfv={f>Eu_4^^{qjf*!T8&*Pmwgm!>Z9$zH0~pu z_gGHD_YBHq$~`=5lvIN?YAEt#D*T?>B^YaQ1+h{!75A}K;X4r5A}0ui-WrfFLJjyL0r_)&4$rj!#@i%%8_A*=nyn zADbFqV=oF%Rzv0z%}1S04GyHT%y+Lv&AWwTx^7H|Ji7L}muO3;F_Neq`=vxfoyIVt z$c>>ykvFMC;hPhbz6p%=#-F|kO!-6KFbLLpgontVErn~DO#@gcuUwgb6 zfwf1W1@o&P+L{2<6j*=o*f75axIJkW?FO#3XuiVzl^G!z2WRt@MeVW1V)HEK?3=jW zEecTjx-o~1z#R4h=CBc%!*VcZ*)V6#wctFWD!rS5cMjH4r8L+0D-&Kr4%J$N65`dM zFXErJtsgaeUVoa4g3$kz+a$LyXc+Wfh;qJ!a}dK_7-Abw<%9>`I5ASeTy*Hmt8+CSBg_~5 z=F;4eW12hg9wgNFXLCmu<_=4;^QE)si!#KL=8mV>+!3k&jN_+TD|-gPe$p-DNB)GP zzv$hQHP1NL*2OAr3)YLNSvKumtfy#gdlI~~w#7C7n)bzXZ5yoLgS8H|eTlX|(6$lR zwvmXdHCbsxxo{TuJyFh|!~FRl%w@Qz8adKd{{Z-U&{?nnGu7^NMX zKosLTgB`ck$1)nMr!yLf@j3!y&xJ8K9Aj}9#^gO1uS4A!qgrAF_63fp7^^DkEoip_ z=LdWfjMb)C3#oW!yor9@>Zez?RLXB!Ejo=|Tr(jiP4xcI!Nx4kBheluFV3b+2t)4N z8ltqRY3VG-+?WuFaU;jeUW}g`7(bM|cn@{!H~f{nf~LxZwiq`*-QO^7VsIY*%eYa) z#kisKY}i~n=hC=24?VaRox2q0T^c)QF;*zI{~eCF)O-6-9=CvBIa~GrD`?dmf%|t?MJh&NT{uf8M75dS9g+{T}ww{Hi#xUxk=AUO#<~_QvOCxqf=QY~_iGUzc5df8|$K z|Je51)f?MZod`Yl?$yxMpIyCCaq(*T>QyI^&pw-G0}hMQ6l%ay#rjBkbJ{dw0Ozov`;Gu=fn?t%SXbWzwzo zmQl>!JzJNKkS8#F+9HiU+sOL+1pm{ zEbMIydqc6Fj7GkcBWIJaJ_<)Zw8#1=8SA5P;F7UEO2+yq8SA4YtdFLO^--zCURxQM zvUp>l^VT8!ZeTr2awrGsd=&&Q~!Y2f?0za^@lluNj- zIsto6sQ6*8{zMD>TH@CVKW5|7KAO_R$@Xh#KSo{p$so(pPY2Q7O-kg9bY=P3boGUy zW%Ae$$8mIijOl~DgMy5}aZY!!^5BN%|44OXf1r50+N?ORTZ(&{8d;pDwk{4_s%W#K zZ0QA-hc-B$b&g)XyfS@{8d9u4hr09#bbU6+3Y{~TSEaW{n*yrryK;XZWKLDXiv!Ds zyPG|Rbujj!vCmlSgpR;67j&t`&Gw9NQ(X$`l%-!n-dB)UF(?H#GFd`SrW#SaU2Orr zk?t{S%i;xUt76+(e+~PhS5#aBoI6%9_zK=LdufSkTz&%YdPV#1`6~AJS`?psn5bG@ zxsa$*TvMzXeevAR>8d#GkOl8|K)?S$&n~9*()y+0y@w0&zeS;HP}`emqda~Mv_D%5 z@^ai0w09V~&DN{*N1N5Q%>mk$WOjFg&uO%K)%)sb>=%?7A)1T!BhH3sm7T5Sv?nqG z$MK#J&DMHVNj!cdaXhkJTiD_>M&bB(yeqf`)`MTD&ZK+Mwz6B8HXZvby1KIDU3GPd z74zH$Yjgb~e!mr`81ExC*WuHx=T&2>HKe%4lD(mFMcRh1;L}@g4)er(KjWF^AOCt{ zllstx7~9M!OL5wUSo~^Zv5t#YJm;|vJR4-m#dWLa!Y;*)9N!t!vn~6M&PQQ*788E$ zfnPP9(`Y}c_gkzNNZ{ z>hW{Xe9-i-wjPDf{^&~;@wSx>GitCmK=L7r%Re5W`J+EsTM*j(6_2lN7QSnQx$|$( ze;fIjV+k&Q0*m@k7JL>LjO(S*G08$E5N>+qGoTHfu9mKBkE)5OC8%y8}&)XzHv{b z7%2rQhRh%8)5FLKD{|8dJuOmePJDvcFOSTQI*8-K7t%H?i<}vC@A4F5bWw_tzB0ua zvnIvpX;Jl7_}`!-`0gr7+i(&(7H;Z`y$XCwsA+2beEvW0B;CuGMj z?We^AICCi%DK}#gpBP(qR26Kh;WqK{bi9-Hos{>Fxop3T{!d``oyWd0#!xBtLJ!e1 zCkqB>mx0}goM6Xc1GS4duAHvyX2+il)M{`HKd#Vz6tm+WU9>olQl5bGSc0k~74;J5 z)1~1W{ihug;xzmQJB`qH#<GD9Pyw zTQtnw{Hvk4_-|p&!m9pS%Wu+hXCrQNwx_!1Md826ro(b)4sd#Mhd7P-8BQa6qSN#6 zRHrfPey3-_O#Gkupwp9Q8SQ>BDm`~b;28IUWlm4N5>Q*8_(I7+3-+`VSCpKeuHyT? zK`0lOoX=E^GuCD(uPiw~3HK+)W<|B8yni<@s&bU-ITSl1sy&s{RL{oPnNdzEC##-P zr8#(3m7E{18V8jYwZGw7?R?_ulJjF!qaZdrDvNLfRZqUPW$i^=Tb@sRvE=-474MIp z6}22~)2*#)YjMqcIdM&iGxmX~cTksXZC!f-`$<<4wURRyrS=leN0o^MCGBG$jQR}i zO6)^Xzk-%p)Y{*%wm+M=wB(FpsXcFPs#hg0E2-?TAga~&yr{fohX?&OCZO!Sc4g`F zqIRV}(ll)TrSZ11x^_F#S4ZtmUyI{MmmM1P*e{RHpZH7O{BNRqrSFdFoj&9AwRGp{ zE9sTbw<&vMSZZ0{cJB1^n}f;{LG4Yz1FQz@>~`DIYqtfKJqkJj^)LVO$o%)B?oMBN z`f7R><8NJN9dU1NUgYNV>QVk>#y${Q#rr+79}xlZxak54h40uQ~f=(k-CcRk+Ie?Y}Ca;I`& zil=~ZnX0EA@6LaN>XAEH17}55j#oXk+rr&gw^Uq4E;t81!18FU+PdO0`v0=ErG9CD zg!_`JR9s3^jrIc{Wcj2Fe2C@CF;%U&eBGt%57Ln?NB|x?@~R#?NvP&_p6>;2UX+x5!Lwb=c;kOLiJoYrh5K3sd{R@ zQ$6^en&-b~RL`;KkJ3HWs>dDuVfyV~RpSQyx{mYsrlz*s&D{fX%Q6(>z(mDU6n!B5 z-KmPP|9-_YKYD-qwwa3Y!Gnrtbo2-5dTMib`OFr%U8Bp>gQEAPZ|~I7y>}VbCQ*CS z4}8?hz2!t}_xpJ6!%1t9331w4$iHdz$7=|bbv0wKZh-CQ3Ny-TwghN?S6&=c^H6}M z5bZq5eI7p4z=xViz`>7mh2zR9*GA%Zf9^T>PzfI@>ztlS_)v2R82E6maBSJeHhyI_ z$fvEazcLf~03T|QKU?8L<#-&!j~e8{R`^kgJlY6f&LanE(ozg}hvx2c$RWy)w=FGl z&*NQtHCC!`>8|uTJsaWM`C8~2?=;+-TDfbsxZLhdt=%xi-kmZ`X|nBZRgcZ)kGN-=&Z99j0D#V_~Q!ck=pS(@ZlRyaLbLz?E! z9-w%Z41s-#hX&(WF;8Vb&|#nhK}S-agQmll%~Ra7;V0gm?a6_E^Woe1ZByN|;ae{G zwIkC#r&DwHyzVV>YktKyHsQw{`0&)fTe+V^-#jjDULL1qw8eP_Ym$OC%5J$1*o=9b z^>xcIHq6pxxTBdQtJ-YX%y0J3KyRuzbK zmA^IjLMU>>jy&$Hy78VbejPBmoAx2}O*&#VCR>}}dyY+w%tC)-YbwSG`W10&eY%aJ zQ4DWk-&FQZFxJR5Sg(eL++&2@H_ULnJRJANh3iwX*O!Sg;X0CL(3~~GdXF(OZJ05t zaJUg*4XEg<+KRDmxzgS`@>Z60)Gf?#6XqG--L%GBD%O}s5wnm%Vd*uvPCAc0?nLm= zwNBZLZraw-7+3gzSidjD*fu4_c=vB9Mj6(=!SHzXTJg*TOVOf9d{*Avm=IakPo@N*>#@Q zOS=y_x#t$1=|X?7zB|=RTLU?)z1Q6ut+OO%tEH%f{xe|f)62Av-BzCgEZM{Uf5tUw zk=2>Rt{DS*Ym|F&>Ki3&oOJkq*n9W5s*1L6eAZff!``5H;3%eml(F%YiifQt8}NvQ zg?6&LMD2hYnp#$xvhj#w#qyY1ikihk#nxSNQLJ`0X(f+oSuL%<6ACY z^Lgh7Y@>k}tIO78!OeyH&z^s?-HUJ+C&lUa!3|p#r=k^7Q6IgvLfcWxqYL2gzeZ%OXVL>@@)(}_F? z?zscz%rjp=X7AB5e_E?%9$kiYORiWu=SB4BM>_H`F{?_;N$7x^90@>kHdt-aduOKsWrcSgU#F(lyvtAZ;=l+6J zjpu_0n4`P?Bl2@#-2$6Ry9H;#xU-j+M>CvPFe3^3Nlvs~r8s*FtYSF3v|t|tUx^iO&d|Je8>p#j&;*Iyzj*Ro_NQ)X${_KJd^Ey7$J~$nS}^x#-JjNGEROfWm1$mej6Sp>rRG zPLHR{omlH3uOK|lu-0vf^7sOq@o0ExFTjMy2TyZ6U%qjm=TTeqIkHVomz@f0xDt51`i@uR*U zGbs)A8&x-TQUUfHu{Zbz`f_ATl&SZSo$+BF|1kLf*7yD2?b+7O#MxZ3qSwv!+2@r_ z;B7tt<8U7IP~rn8bFS%zzU+5CF|+@312SXK#x~SD7WM3a`gGblJhSlIF|$or+rPjb zTUThE)V6cK)XY;Kgt~tDfME^_$-Mj)!~PcIza|7_^3j2r6{{Gpeyc_1nOv-6IyBEb zoyGY1*8?(to5uK=;XZ{`Sl^u9qgJoPILe2m6uCb)VvlC)=!#1Ic#42QtKcMtQx=_*BS@)@Vy_o90~b z?fDkw;ncdiBI<7%bJYLD2gDD=7sMaLC&Vu;@%)Y_eyiIy8)=AfgZWh0v0;x2@&WdE zu(!h=Rf1xf7~3@`JOZ?vu}3we#iDR4dohj99a+~!!rqMHvnl>!{0jV(8&}`E17`}? z>@XqEN{b#E1{<3oBmW`);uCOT(>*aCc1GQ4eyDJNBi06Ex)M5-WrM1dJ_Q+8$P@*S zibum^!ehqcgU1(-1&_6m&NEsdt_yjHI-TjLkB5Am4&A`It7Q9U&@+Tws9m4)VQJl} zd<8PmGsk)KygX*H>rt?urSAls=o32AqV$GBenuc(0Wj>LCZ>aLgK%#C9^O%hDIfbe zX3+kR*Y2_U-aX<>&Kx{*@yx?BAJ2k*L;M4C*JB+O zVKWy`$NJzatQE+=!dH7kY0#kN4T+`a4r!pq(0ospl|VxudW?* zYgc7o!h_i3TEvPYI3CPT5pUyZ!R(Ry266CIQF*h#ZmWZ-?fSi0eoArH8XvwBe8G-cbp>PlY2;7spT5Q!pV7__cAU=mtaUy- zqXO%A!uKdVYZRV`@QlUtsDgi2gcN3}?Afg2tTs6lcDBF@Jl{flK$n&I0GX`H(@P?6602fNxs`s}P6I z{X(~WZuSAOMpJBhOcZlPnpInL3(ZDzRJ$mXEz)f6)UK&dr<;R)J5fDtci~$gjD-Y@ z1FwF_#!L{nZmxoFcYZGqsSR3FJ$+eGkcsWX*0#e`dMD6no<<&dNe=q<)E)j-(2+OMwQ$#`7jrl4J~dS*o+93_9L=7kyvSy7G14U-uK3`Z&EQVV7s7u$ z*$oC8OR(l5o2E?&OZJOmypi2tAod^UlKo#0?ECQlM96;Pr;D(KqyM$(2#2mC0_VZseFh6*>t z7%W`ae+oC$NER;cQxz`my_q=#eK#0=I0$_?5PdoTvN{>x7pZ#t7O)xAsIOJfHTatl z&W!M|7o3Uy-lM?2ml+pO|NIGw*n4Ljwy1eYWl_4-)0ej)AL@7LUt1l*78UnPcq)0A{D`kaUx6oxU+zOX#0MBhMRFe=6@3_D41qtj zKNo%1%J4-Ry%E-3jxfPXqdh#q;78xTdi#>b2GvLOSDsaW3GEEq+x;O@7^&jeD|s9A2df_&HZy*Y{^rI5$j&JP zb6zm55dv7;X5>eGmX5Nsk2~pEu-l{h?Eszn3wwcEcP<^oRW>h;#tGG*`bMVh1C9D- zoW~~qO~j?XD8l@;0?%|j_u-+j`4QTR(x-fDC>=o$7vq=4AMv^O`&=56i$Uv)y13~f zVyxbb_j#bl(4Nurd*aLdo$;5kA1~f@ylZ&(XS`xAzW3AUeuQx267nUNbQdbea=eeB z{<=i}o(MA@LfO+b*kXa#W}*2;wRz0QtuZk{Q7~`3uU5jd%ZkG zYcA2I__sy2=Nuv$;iz9|-9;{yXOu`c%(zduA;vwz4K_v!H^@j6ZlKXyxMa)r$9BJ@ zUr@T$nA6S={|Mt2>(gYWi+<0ZXwOktPr^Y5hZTSLn){m%su%S?7F%&7Kd5i|gGOf_ z`H){M%um16%)JkF9yS{8v^wZP7q0C?xjbpz=1J=-*tU1{q}2g5;v>=>BY-;YD-&+$OVqc_~(vA^hJJ48(aMqjh zQ;C)fYytM`J-%%u&Xdzo56ri5tW81cgTSQw42eo_i#__0B5mmL*mo4)a$|qJHU@23 zfOUsxhb?~kd+zu}6bIk$j6_?Mvhw6w7K1NQ%e7je*Z!2uF*oTkKWRK7*m0KhDk@6> z_%$MkohgS7<>irJ@CfBB+zLN z`b@YX#y{Xfciy^Owd>Gn_)PG+b0hv^j&byF_^tm=D>LT4Ap0|@S}dd z-d-Btx?rz3F_Y}2iB}z?w4(nl`^CA|eI#=uZev{RoiOZ~FzlV+TXw;h9@%#I%eqzU z9xV=bci0c;ugBa9dm4O0&;j4%VqJO$dp$J+@H|Z4_Mz{+{ZT(=GrH9Oi@PW4!q84H zhqItjRIhQky9Mc{LngK}qr#brcsi)g*VHp>VHz!g*+($Q4Si1r$7rKnS=ha zT9`h;kLhppe=%+1=EA94in7x>_Iqb)=F)j-0c?KSTdP;6HFwNUBRgDWR@2r|&ov*R zD$1~=-VrUMOiBJxD#nkd5*Me*O&Mu}B}ExBW6w0lOG)eK=7w1xXj zyJwd4Xp*vI;O!|(f}5mF?BSm>ap16&iNXFUw~z1oOb0w&QI-T%IYVnNl210PqNm`& zcaCSuFVpww!Nv}h|2TNu;)FkBe`Hi53AP0y)VW4lhXS#ynogFsm}YFPwBkB`P9w_nosMz{JW^su6WY$OvAJM@U4@&;z`3Z z4NuU(gu<40%-O=`JrQ6_WM{IeUf_j=sMl81E2f2We@4X4vlffc6-q7T?(cXpzM(VR zWXwXiE7 za0kPE4K8HBbnHnvA5%E)X5hGYf=~Yya{V`rmx0%BNBU18?F~WrZV@se4DV&&{XuYX z$Wl*lNpy?_2i)-zy$R7VHu}IFC((U~jJzSy~2%~Kp!a4D~XQr7Xo^MM0dA&kkK4;yF~wi!awHVexP@k=!ZZT zeYydCYECLFTiCpinUmqRfa`!805=hCv%SUbZfG=s?OLN5l_T6{3V=MRcJms z-7`gbQTw*@b+-@Y->`l1h}NKeNdFV%M7vy)+UBWx`n7g>!b7__nA$5;v=_C@1aDZ3 z32(bR;-z6cc-ti1OT$?3w#frtn$#xucxcsV6K{V}yWC0ig#!x9(H?t|ZzbA<+98H; zgSPktyj=i$_R6;yAA+?-E;=+Z^@MFvh&& z=&+XtJ|Y^m&9`0}+Qr)@U)9rs>f3~LbjqXpF%NHe+QqHkqhF+U`PduZGqwhxT}p`d zpKKSB<8)^n{pO|lz#XqzbIox5*gSDA>T@1E6Jch8hOi=yQOI)? z@|=e_4wpN;W&Q;bPrLW#RGEb_>s?Zd!0O_4tC* zsq+>bOZ~@!pHm|jl%)<_@DskZ`YCnZyr1D7OC2}wcxn{By&8jWr|6q6`UcC2Z?G(i zH8&#y|KRgml1BYP{YU-M0sTjPM*Y_u51rXl-%-C&zfhl%-3Rp>^%3}W$f+ORL$?>W>JebQw%_+v*z|J2&y&Uo|#+OR+M$>c2f)dAbl&M^pU z-5lA))`tA39s8$RHb+jj;N6a}u?U+YaO7k={JecN4Ea31xp|l85w{TjOAu!X+=-i; zPo4zYm%!AI!;yYD-lJ|JzsSiGLH`ZzZ^FO%0Y$ zElV2v?k_Ae8^_&WS7t71l$n27nc3KPf7NlfJgu?s{;GX9FKv<5xHkW5?vE{L?7P3P zp4m9={<<>L${JuMyUk&r3*?C*}MN> zcH1lM0ZrI3B{=xsf$tIiA@z1kyDd1kM_jMn(np?muierpynhIr;g_Qm^f}GQMj*u4 z;p?@tfIX7OeyM>d%0+6**Lo83iuiaNUOzo@PH)zdhr$grV&TFbWoy(H zvPTIs&Y#0~D`iUD`4O-sfvs9|@T?tS$d1Hvn?tAn7+ymEg)>DjTL_y?_;-ar`IT)_ zTtP&K-I@w|hkCl|3MP6M`a@Hf-vWBsTBHyAF8mvPc@({Cc!zD*dAzr&kEgjpC?0HM zeg!?Wo^Eml5*@r%33`iqy4l6io}uK2x|GAu&*Mk`PAh{wP8j*2AHIX12EQ_2(80Th z@Lq>9!>Gt0`^Novzl?XXlTly?L1(=iVJAU07F1WVudpI54Xz(sAnar&pblhvG5&i; z=9FfR+J{@?e~EY}`wOzUAlnPFlbIGeAoC`;C+k)%rT;QS;IZJL|6R1f(-zMgs$cE| z<#hZ+^Xd36fFB1=d49{Jl=~wm-41(*C2gWoQ#MCVPJx?=FcY6IoHP-BnZlkT%3~`L z1v`Y3skZrK*H8tUh*PPE( zWa|@!_lgB)QniKWVW)C5H3{$03y-DFTX+FBE3gT|`!U!L`OLqJIv!7*hxg43e@gun z_A8t5z8UY4_|IbcTxY+r_{ZqD)~xnbygwZ1j6bm!cF&3Uei^*v2Rr5eQ+uIC8-@Q_ z8-;ZIyX^$#tN+eE6EcLB-;nTI5`I_0f0ytF5-ySO9trCC z2`5T;poE7?_znr*CE>dze7}Srl<*h{KO*7r63&$H6bU~i;b$d0OTyU_o-g4=5`JF7 zx`bCqc(sIIk?;lyZ;cq4UqlC*P z{F8)FNcfb5&q(-!gt>%kui^T!fc>$>_+Kf>ME6*5NMk|5O(ony!od=5C1KpRTR*0v zBz%j6Z4&Mz;jR+yAz{0O<0YIZ;eiq!D&adMe3yjpmN4$ksxR|{5*{PrMk?ie;nfm;MZy~-yjj972^UKE zZ3%CaaIu7UNO+foKaucf68@Kj_euDmguj*Wj}k7I@J|vxA>mUJJ|p1^66O-Fy@u<@ z0>0gCoEJPg;r-WSMec(V9wXsLBs^ZinG&8N;in}0tb}JtI9tN=CA>(&&r4XB@Cpg9 zmhdYQ-XP)45_UqzyCnRHgg=w;za+d*!UrY%t%QG+aJhtklJE%$ zpOWwy315&fmvHShTt60CH_i)<^8(g}jq?Kk1>e0l&I^t6LT=-{z<-HuoEIABg~oY7 z?5W<`I4@iq3)ghW#`S_{uKa&#y?}pZDxHW>J}qj4X}I~F9__@IQxNca&6kC$+!gr`XODG5I- z;aL*SmhgNDFOu-{64oWWLc*&h{ECD(NO-e^T@r4b7x*v38s~*;I+COtKPcfb5`IL& z<0YIa;VBY+O2W@dc$S2-B|KlkizNKKgmnq8knm~=zarrc65cFfmxLS77q}npSRU=M zMe;XYlN0_H2{(~&a|s7YI84GfNjOr%H%mB1!W|`in}p*e+*`u^BCgF4mKP=%42|q63CnWr&grAY{a}st+c%FoFB)mkzc@kbO;g=-*vV`*` z{HlZtB)mn!TP0j1;q4OsP{KPU{IP`hO8B1=E|u_C68=WQKS;Pt!p9_BDdArvd|JZi zC45Q3SFU02nX{jA%|6xNd`(99TP567!Yw2mEa6rXZX@9+3Ev`Nn}j<_xT}PFNZ2mn zcnK#;c%X!bO85>5-zDL@C49exAC&MI2|ps?@e4y{&@y^i6pk%0T(k_p?`kDSQpN)9{_Icn>$K$;EfO=f!&~<1D%O?w9II-vj?j zF1{1~gW zup*3fpBnBybH~FyXja^V7O5m0V$AWoyTEXV))b}r?6#3j(%arJO(|c*ug|yPUBAn= z)~#~j-Z@c@vd;B)uhHEC7UnDtN^quKKs~mpU3JUdacfyq@J&<3pGO;IsLo=dMd*x2 zf)5(r?|WSxrL-wsE7CoQbW;?hYqPDzO}V%q*xfdNbSE#}+eK|l_b1|x@}3tM+eZ92 z8N9dq-MUqTS2Sfs=}lSOohTpGp_2MQiPj&$JKbd^`bKx(XG!16&#eYOg6{oxfAB@{ z0QmH7!2?0YFz_wiDKiT16mB@VR#rseRmA^5gr|QS1RFy4&GA<&OL%Vc|#P&7vWAj@()?B<)n5CNBnf$Il(^m*MEdR zc=b#rz7<)7ve@(vdJOIwj77a-6-IX$#(C3cs6+lc$?JCEslZd7%QESXx+wZzz_IT7 zYo5GJ)NaU=^1Ui(fyOc6hTx968{d078+Rx&7N20P<~oyF5UeVBlw5>LyT?0U)%|w77Kbc`t}g+XD8k~g*X(x z4fu)fydzo_Xn)6@*+h2%SHo?JdU*S(e*X9&PA@mP=Yw=lI>g6|ke|EG!Ny8K4>F#E z`vLkO9ruI&(U_!qKZS7OE)oy_eS&xJ=4s2RWS)V0f)`yKaKN@=)&Y5H$tN1PUe|Ey9 zD%@S0r36)B9F0uD9cn3xRnRBa)ot_ge*yCM)|>bfW#nV%K9$HOx(Yw)mmBe%iM)2iC2#%;KLRsUp1~<6FoHuN)bKK6m$jkdX}k zdobo-!Z-?B)mqUTOf?SEsOKG#G45NUX2f{)-mw|pS^=%qKmhSTPIus_PO1w7(F zzgW(@+ppmq!@vW?9~#0^+gHP-F=2*F{KMed>f0>Pm|fqNfyOzM>s>Vj<0Hs8Ey_rJ zcl|x`&_Tr5ZAg#Aed~3}bie!W<;FS43wM7xYH9Pkv4rEf((A;v#F<@_G+ zG|mpet@D(b!i_{Z-TfS5d?w&qABnOylyzu3aYtyG4=cJf`a{S-1+q|u?1lW@!*uVxu&CEI+&f%E^^HLtEIxB;<1C!u8=RA4%7$%QQenCcV^aM-^wdv5{Yb7;{Zx@(sL=_oTtAzj z-?)BPCclq)OF?~A)LTRSF?Y^1<6dG##~s^gkSj&_r;f1^?gh<+zH{IAdl`KwYjsx#5m8=Sk+$e0FCD> zh-+c3MHnmkT$Ba$>N9RRabCD|5BK%@LCs@{xQF{w(RUQ*BIM6Oj4@HxM+Wh|wRJ_e z;Vx3KzDEL_S9pia6^qpLGO(+(H+#} z?m_w8bq_WEjxZ&7m+y{VhPgC-P%_8(ExH$R31^*b4}dwOnQ{u=f7ZmkZyMO`7%-dAVY^644GbVb@lP3CSqG^7Xl#>S7NJ?H#v?A3jr6ZRa4FqsBF{i$ zig3fh3x9kkJJp@y)7{&5BE0CsX;@i!?nECC8r@yIQ;c`!Y6V$bptkw~^L@(*#adraN-cKSUF0mxt&v2p@yAsolM0{r0pHR5#bO@54oVgOwR6k2Bd4jogNz@A8*F?Jm)fv6IedFyGmMV_R&)?g zaR7^h+{ywbQ)1rESrY=;LVCexNar5t%op+Buq#@t z*qmlex6~#b812iZgueaq8Va++I~KY**I1#CpG5oEc-Dc@ilu53+G4bVwM$p$ng}Hn ze7R~3+WJiE`Rfrk(74;reQ*3W3!@oqx@3RF&8oZXsup@@ke~?F8>6324lU_-l|(w2Hm1H@;;)5 zO)N06k*mSiGw|F4dM`Y|cp%f`zWvR!-o)H`6zxRv_bh0%mLT3Z?4g|k?egr|2To{V zv5+N8X>3p%?m-(qmt|dhNehl$l=aQpI?Y<;z}!Q5e~maK6AlU&>vy;$Tgi`PYwgSw z)NLs0I0SVaj5-ejZwz#y4FqqTi)K79>%Fz4qtjjv#XA>`Jxa7w8SZF254l^3Z;A@w z=46Gft!zzWHQXphnJe+#T?Dg;@=crFdM(xEUC<*~VAUMVp=guS(q3>L`)5ymgJ&-| z6Xg_T{CBeJ-;`H(MW1ukExX@*Q&S_;t10RE52Fh|7|7N`M8*}BL z%x)QieG{}r1+_EEO!F1#H9vzt!!Y+mVjrk{xMxm!wHL-^pkc48TZMUL%4O^wU28Y2 z+@z&NuLe_t0am)HXC_m{l zbCC8R^jQV=zUcpz-v5+#1+Aap(J=HE$=FbXWNd5DDID!j9)%nx85;sxMpmn}l~tKm; z!~Wl|ly0DLMYuGk($Keep^xuGUk^v0r(#UqfidNlx3VrQ%Z!0<)~(tJz336}qM$>s zDe$aAz4`&^gbigdWI-`x*;X55XG~K|u4UKY8^#l9J~*rxI&KeO+(qA&_H2t|Eia_` zy-+ySrue1!b#-jdEnJf3|Eg$K^yI@Y@O>Ch=ihV4RoJo%LyVgUG zS^~YR(4%+Jeiq7}o8ZyAC{3b?bfMD?MVdnu#xWP1DZ{vy+m_Lp)TNgCohYZ+GY(xJ z{u#!1!#yBT-`S|IkHZcfu2+WYuvbFvXN+>#3&_34A?AUM{i)k6kSohkCnwf9`RLDZ z@D$Vcuu_Y1w2dV#OJH5 zFke;hsD=6A&iJgQ0o&Kdnt4`xX3rYL><#y{{%9=BDa1c{(QX?df4yTN2V)lPUP|jl zYRA18f8MoY8~Jx8SN*$;RGc{(gCZoVf^T2~x^-b-VJ4NT??`P1AZ z`o3LvJ-uQ8&#Ex-tEZIZ)4#NcdTLUM8mig^{{x>Igzx^@vcQMCSouqz{i~QsE*JNNN1w?=0>`fQ*{*M zJR~pbhhBPb=35^ z*H=*2KPqqC9@;b2AdgLu%h)HH>4P~}5&M0lzsfvtYzyut2j7__9(WhLwu<@`Ze{zy zJOlfKRL7@LM~ds^jho%PkvC1^ji*sAFK-Y=-{n!??KRwcH;_SmlBb{UMjw?S{jMr=(b_M8_w<@j+BUD#~M(c<`Lj&KCI|VNI ztW?(3y?h4##QA5eF7sP0%5VdI+ZqFY^YR(<@YyhgmHDi{r~E{tF)G^A!)K(2diZRs ztpT6ig7za`Qs^uLytLVRJuj7^Jsa@SR$DpN2>db- zd@}&^crxa1_x`b0&$F;37sGir(#d1D3>N3T3w}|EC-N+i9p3qc=Ike~(z?R5ud0K8 z=85PgnSZ7WT5ifuy#jsk0`$RZ=!55=4_?ArgZ8&zH~}3|@N6EP8-o{n^&+0d8S#a| zj5F}%2Z0X0ZFI5?^IcxbtXO@>W{Yl79Q=eD9E<()meAwQm|8n(=7l?IZiYJmPh=~{ zjAMR|8Lx&rW;_hKsgBmM6Q5^OmeyrCJGv@64Kr2he zf7d(E_zn}}o6_6@I^;^)75vxy2kd)Yf$shu?SH{0V>{LZCz2g}JX{B@4Q{2lt&NpP z^F6GaB3MXOGWe0!3_57%1udwG&P!=+vIMjU7E~2Y@4Xy6$Me34biob|K2N-dRE>4- z5v7&g>fq-fXJ?DDI{0(sqO3E7dz0}Dxd{$FUA)sC&yDrM6VbP4R&^Mb>D3RZPE?;0 z&^xHEZ$X!@>W+FT&8jHRaaZa_p2T-qS&Cnk2K?(_!U0vAuzoeJ4m^<3!Y}s@?7eK6 z6}onU67a=(gxg!=jx#~=6YoRl-4k{g!cLeKymmC^nH*j>@>DbE&~K@(lDb~5x?zmJ zs;T=(rj(-$zrCTk=stvMtqFAUCdLWqs?;xosE**jSEJowD;8qh1TuaUbt&_NiANaf z`(w$fkX4jk#f!kWT&9vu7GDAVq$$braud0v|o|aTR;A9_Z)eXrwoFAvt7?doFg8;VE9pc zR|m18op@+{>kBy=!Bp>AR~oJN)td!U_Gby_v7ZoM+cM>ODp?8_i8P|E;xQZW@ApzZlf> z#HK`-4QuZh?0NO*&WaNfSTVxA6O*i6x`h4Um(gyOp$^_J+T@CfHoJO_ga2ICEoQO^ zx7DvxVr^`)&lTtnc{a3dj2`e|Mt{e$9O#iTh#PYYv&Rf%_8RmBm3afz4?6Q2=*(+@ z&w;OqUkQ``ybSvD3SerBl|+Lcy%KtKHOAG86#iCsc*3MtFN0pa2)J?>WI%s+8#Sf2 zNCJi&@B^lDIH7Czx;MaPOJKD_QFr`PnV-a-$BAeMkA2Wo@4QxkXmy+D{THWtbo=MmjupPRYo$5;RW-Q7^ zbx9`|b$y6j@X+7L1&@s)7j>#c8+;??9k%cMa0Pk~&N}BZJ%*`O*;spx!=6+Y_Ge2Nj`h{-HkUNo9lUT?f?dVfsx~N>Blf?M`mHKrx>4 zy69s(dqF2pVx0k9QjXIMafA+pIFU%(k(0bmS(LmEI#T8sveikyP5{##m`Rz;HJ{mIS9M*o&!R&F0#m{*KXN78@-*xK?3oB~4E~I@atQ%=B zvwnUaZeU&E!{R0*4Xay>%+X|bK-R9Lp%-hOioQQch+cVGmCK? z^_YWq6>}Qm*Xx@(kXMMeb$L>d?)(UH=_AI!r?5yv=y=dG3$RA8-Z7BJLDvtu!_1*a zLM8_pV-PnIJdE|4{Yk9N{m{>HA25=ePP-n3`)x?BE^@c!?`-18j^>ZfV(DDMqhHsz zMH93I=_(!37O>AryU;}F)_E79GoM4bE8AmDr9wY}{=u3Edl6CJozT5+N14m8=BmS* zYXizd;fa?BLqE^Ei27EezB{3Z9>E&uCD3s`hj9~V6oT*eVm;D?Uj%4yidhD?T=-`y^06AyG!WFulB{< z8Df|~BVA*(a6^p;;bLu2--it8BMujFzUiZ1I-xAjhptWgpI`W}3;vcEgPLxqtC_O@Re`fbLwEksm6L{FZyGrXP?Io8nr#qhFlS6Y+Jo+ zPpqSFxF+a{GSL1z+LG3d|0B9Sy!A7QWf7Y~8|XW+va-CEoevV?fzE3`TB(oyvFTD0HHO+k8w=HThz z1Ck?OfllE@fydInX3)1ojQt+kC!V+zhWPN`a4&&I<)v_UAP${z5Z*1~gc>{H#%RI0 z7T9xSG{ZeZSicIn(WIa5gdf`LOeJKfKln-PS!@h~eWQaP0gt>5fAn+OkweUNB!G$Y zUV0aD>p0}r^qYF;rzP@c$hY9pfjs|FnujgMCd4DUmFN>dpWvpy;-PCqp9K0OH+_|d zUW@TqFctKvZn_RSr5l5`qjV}UA5y#mauF^At|-e{l;tej7xa{w2R>`&1$d`DFWPG> z0e_dY#FG?osVti5R5o&>pbMy|%SOCoEafGY`sn%HI_Tx_BOBcU&}p8au*-l0aSnuh zw|eJ4v|%Z`)KAZbeXFl~OkeMJU%z^r$7sYSUa7=);F0Mqsnb=*{ks%`@l*lRF zyDkfV680S4xoaZYIj7G6?(Aa{{nd-oLZ8!~7S;U$^6wA-{%-#+@Tc$@v~K`9^qe9m zc$_&ybkTO{t#t+QDDEhPO~>BM@7n50yiRrSo*f-V89&6>CRzr{7TThNeLLQ>L9f%a zeJ7&LSbzIqU)~q%dJER~R;=@Jwl~usc3`S5>OG{X{%?e*e)=BmI`e3(F6xQ?->YG6 z+<>zZe`oyH@VmINJ!Eogqc7Fvq@x(?{wXKG_lvP7jC%D*!TxZ0k_ll#jrKPPYFkm}$G`?M5xEc~|l zuoio-`t}Tz^6YVgJgvx+`o?l*g5BG`40&Lk(DOO)Jlzep@^am_U|$xu+7CA7p8Ot= z@~gnwckdP7>&o^E^74kgQwlo=Yh!QOz6CE>k^Xi0`eVId`{r7Am29WJNKk?&reCNVIpl(GFDu$9 zK^ZWG+MD`ZLppDw-bAmDYi*tD=ZQNSah>PtMrNR2)}Zb=-Q4|bk=y4i{Li}E(u(xF zVd`kSB7bT-;s<7T%!tArU&IS1pxfsm-wTu`)p;B0jCP}SM15PKe%?Bl;cN@%tU=Vz zPT*hVE&&v{WVHPv_jjPnoln+<7Nl@4^K zpJz)9)?;uF7`0~%_6H~~^;r?>a-gEFDAJzMR><~>2z`ULl(Aoe`Slxv#Rpgx+uICy#31lf{>!Km50AMYdQ4uSq=18eUQehV!_7VHV%o=$VbSIzW!$S)o9Lsz)@UOMKE*?8XxdIz{PZ_u0(8s|Ps zC;2Sm-_~58e#os$79kwT+;j^odWOd1xe0cQmGu%jdUA+912j5+O{eo+UuV3K#X4lM z=8QKJKefWy1L{ighWKLNa@DZnjJmoWPNYlw-;vOV*1(_iq}8C2?oMNe-dEtA^n04C zXid=$>vPg~XznrLc@DIVBgh^oMA!q-dGtciE=EJY@^uuF-4V?Rmtg}z=h;W+(w=;X z@hj3N_a?|qvU^=D-h+)i@FUSDAzz|DF5Dm^4K9^GrpC7?jYaPmpl=>Tdo6FSXG6!J zby>8C6KFgL_s{y*+qYtzl}~Wn<6}M2aQ%7!{4x#mSvuycg-G)$wD-1(SaIG`1YL*L z2$Ml4|2c5;p*zw1{46l(Lj~P1Z{T~LU6?nfA?~!69rUMKD*N)6y2DZ2EQHHnJdl?x zHuEl&#tXnNn0ZHFQNA&q-8$A&bK)bcX8sw<6bC(*&Pu#{0M+;IDIon^uFnR@L(tWA>eMqnE9HkS2J^K)UR^&~yz@POL_$TZS z)W+(uX#cx0rhA|q)-!i(Pw>q17VI6^u}6>qeO>X#9Jr_TcIwy5$I&OWjxJ^TM+e>f zeFe0#7OaSL>@_}t{uFM7XaDm8{7B!XdvO);RSwD24MDmJTPX|nT5Lo6rh&63# z)K~rWJK;a9p}!{9uccAHB7x2>m0lGX6 zMZeyr=-HZk945k_&OFDXJT~ZU;FaPO#p0g=ee4V9ZWE!qHGv+N(rL@u5qB|L2aTnT z?xff-N3^(mmf{*cOLfJVS+RHg(KvevY0@|=fE$T^roA}gm$7(fc)!uNoZxjK!{t3L zlBf3}j_700xm0?~AvI3akMbe;L-Ox#gdrJ8w4uQ6GcdOeZTI6!kcfYs{W-n`ML%K> zDsKqvriFht)~WcGk05g z!w$OF{UkH4z;^17Q^agFeUXmm9oua$#vH#$SvlVefZjk3lrl*UB*6bBw;U+FfgIQ>%K^Hd_P$PTS#YmM7L2%F7Ua=fOnkZm>5?p< zcDN=Bf{f94|07v&Wiob?Jn{g1S!x5n4mH>F>wXh(IrbieAMq^kpM6_{OU0ISG~!e8{{U@Fd`b8=_`UD>7OoTUQLI;J*R+xaC5y5hCJ* z7(sBq!#ej}JYq~LA++x=zJI&`dv*FAlFi2+CiK;}krsuaxh7(u#?6R}xu*1xJbz@m z=Z_Z<7JP$mqhs|x2y(B1#| zyCKnszn^!0$8Yz8-|hpiO~m(jcaK+G!yZ&!GzWR+o_c=c$dmZZigHtXmt&1*0lzxX z@7K#;W>DUd;H^md)(vfDL7UM!kYxI3v<=2v>2hgozJs)#7@HJUeLG2vy@3c9gY>=f znBr3Z%XtllY{NROi24|DOW7afnRJ6ZSKJ`a7=(+I^7QDFr3`xjLT}u2{ka)^V@W(k zd3$UxdY%WrwgJB$+Elm7I}eOR-BJHIsvpDo4)GJ&u+kaATGBNzKf3dEfA>jeV3eN) zdV*z&szX0MV^Nl^!8$O{=2-_KZQ9G<8il=lvzs5#R@cXGsLNA1$%bv;g}PPKEUbw1 zaj)$hN56V)=TecL=p&q;tibunyJSO#@i`21uMOKk;Ps^6;|!%>otc-w?u^1Dfllt- zNYBmd*Z#RcW1ivLfDn}X(BWQsIpp~k=(_H4Nw(M_Mlj|xuRQ38w17QxC~VTCZv?<& zr|BC3ciaE|8-au9OGY{z?1=mo`c5Fw2oq%^-QoK0YssdF=7Ss2sm~gICqQ|V-I2FE z1=veEjPcO0y|Gq;-6i&mv0vgs8`3)HW~-xS7jzCL`0V;}G;A-7FJVVV_Pcw*GdGGu zy#FBjljbm|#}0A<+MY1klYW0xSzaXiFUTP5?Z1P-ZX3iP<0RUJ{&gYjzmQkZE?@pH z_&2ow5^WCqFKUyBZ`B+A6Qf}pMBIeEPt-S1wDGlbUhjGhJX{*}F|E6pVFNEveJ*H0 zu|l?@?e?a|aOgNh?5^;KKm#(P#zr$2XX&GmYdY;uO)ise}zfE6ictk)H+WHVpf^drTQlv?I->bk19e zeGT^Pj7xMD*h6a1QNBz+qJ~$M)z*!qbV;|5P+CuY2j43AHsv1`%5pJolu&%F$dg z+e>-vq^|2-O=bIHv0k(h zuVGAMzrH3CPa^6<{44aycJ0O4eKOiRCkb`M*%w2YDDZ;tPjdVBZQx&qu_DjwC!&)w zz2~bU-dK0MPKdYjYTY(UI|r`to8$It4?pyo@GF2T{5HD%BH)L*3ctNqJbt^~ek30w zL8CM)vDYr*Rk;0Hz|R31`B~snpD}!k<`25FM!c7@y9aTni52-!ShzH%;httOEue>^ zyx=)8J}`#`qyLA{*}5%I%x{61-;ic0yMItE_E~3JcK^Iz!M>)&GN$dqf7PvGu;-}M zluX(qNJPA-q85Vx2O(X|W2MtJcGJ^U2fr752b+7b*P1a3HezWO`o@dS7-4IEO<$7R zmGTw;+-b^s{@LD8oDF(Q|C{w-*g!mie5tRUOCP$lerelF*?3>yG6Fc_nf3c4wlA+x zbnKCzAzK)_A}>T_Yq(fXDeSpUM#QZtb64hL%UJilFlUN59mpi z7(=B0G)KGUV?9%X^~@v4FCX=ti7`iRHe?vNB{)lc2z~|A2XNQy0l$-dM=($4ARNg! zI;WsHW-r;9Lk?Qdb_`?rMSNEk3HN!p4!F5+)8Q_H>x8=yZa&=kaCgF;3%3j|zG+=0 z+A@1s@!wIei=Dwo(0j^|pP)x|(%JZ(@mAXdF8^`|ZxZFORl=_Dr=#@9A%{BJuulU;SIW)A*#bX~>XL+uQgbQ0=w<0a;K7hIxv8sOeuKtP^R8 z_nS<*eQ(_=u_p?f8*lqxMVJyFtQ{!M5R<+O??qDb zTI2Dg;Q0db?{S1#vah2~ZT9>pBey$G;4A^-ApqlH7sdm_cz7ZHAy=L`kuR|(@?W}` zTqlz?*KfDto7cVy*@TovRpM+B_IIZ=#%CjM`UXD|&x`BoRvkpYY^&%X#!evk*Ikc4+3p zgs0&LyTAn44Z6s$mnUus;_gA*T=>JjE&+QH8_2(}H(%evg!S;l`A`DRMc0sDf+y|) z#Qhp^-=#gKVB;YC&<3UUJ?QgIK6*s$`n@rGJg(*l8nfP zjMxTV-B!^_^h+V=30ciu33ad)T;)UkVn_SfDP3xZPazMH|JG5-vAW~(K;8m0#}|Wm zGq|HK;~StaeE8^IO7ZA(%szUPVjumkVqby%o^2bW^wFAP7w!AAGamo!*&4Q~jo!)+ z`*O%5KMCKbBq=-<>EzEz;w5ubK8*10WL~mc<3k9;?z9AUrvnKm4MKPyo&?OmKM~-c z1j5PCb(0m|kMP_i9x+$tcHnL8m>w3*bh|J9^MbQlw8z%4r(^Ztr`f)IALQqw@-7IM ze-h<5sql`3e+B+k;Wol&fzK+uJ>hELYK2D=t^ux5cqHMgz*iOChOmk>Rh5Sm_5=1) zc_`rk-~g2e5pD_GQWb5O9|9br@&KZT1Ba{JpKx2?wko#}ZU@{>v?w0v@FD(}YuiQ&fJ6@F?I>DnCj1 zA>fBp{xjjRz++W@jPOeI%}R}z6J7(nM&pMGuLE8O{h#ng;EfvphVW~^uW9@sVFTFE z_7#8e-nNO_#KV^lklU+>rs_|MtB171eJeEcoOg=mG2>Z2;mQDd?(?lpifo# z4#Lxcr>p#ZVDN@Le>TF+*7$qmHwSnQbTq;XfEQ?dE8*9`U$2>YA>qZKFV?t0_yyn> zH2yl_rNB!y{wm@3fZx;j2EsFt-wc(nBRm^;w#wHMo&!8bE1pJZ87ZKhKyj$f92!9IvsmkXP{v7yol{*Q43H+tXXA#~H zykF%r2ps>bV%0H*<`Y5WS|dw}oJI2RbQ=^lLlN4PV})*0iU za5vy?82^NO0{6uDC)@|P55_;?{=oe){t53!n!7cAjPR$xpJMzI{v7yojDNyk0)L6| zPk2A@evE&@qkuu2J#2nPTMnD`9BErDB__|t?#fJ03DNy6d4;U+$Xa9iNECO(O9 zJK%OEK9MkNpiA1Dcn0BE;8^JYggXOwHu159y8(AI@zI2P0{1lWzY&HlT?uUO?k5a; zx)Rvg-9tDDILX9E5FP|P$i(j?oC2I;;&%{E1x_{b6vAo1X(m3H@IAoynD_v~qku=5 zcp~A4fFCmP{)EQ@k2UeWgdYWd)Wmxeo&Y?-#Cs5)1U$*ayAhrWJk`Xz5XRc2WV(rW zBs>FnhKbt0A66?k%SimFE;TugkJ!D!NkJ}F9lv|;-Q3B z0C z?lACs!0(y(6~gZWzi;B4@JGNO;rk!LyMcF`_&LI#0)J}arwM-!{JDvrBK#%rmnMFa z@P6R^CjK+wL%@ej{21ZyfWI^Ga>9p!51aU5!WF<3CjLF)lwUou%T|mIPcT#03N-4~z4ApZm)z)+ zcfaSJd+xdCoV!cHUlIN)#(bNwrc>)i7!jK8HR2TXKk~=sD}>#p;qw-}Duv9A8puD+ z1TB&deHb_N|4WjtA)F37JF4zixCh3nDg2+J%#D)vQo@%i^I1uYzKcG?)y1w?gKEaD zdYN_ia;&+If6?bv;VGm-J8yzs&X_&S$i&X>UkJ}|6d8}%DWmSnPIp~e=WlgBY{twP zvqHsEPpmSZh>mj+=ThdQ(Q!SA>#59#qvOzBaVKH7HaaeqxKw4{A03xTT&6OsqvQG! z*H4)XqT}+2%Twl^(Q#)JceXNr5gq3t&ZErA=(xee4OZsO(Q!q@6(Q#p9XEow5z4$S zI&KVcW0ZMSblf=N#wl}JbX+lU#mf9~ble2uCZK;89XE-%Ny;oGE&?ZB3C9y3ugr0T z-N_2u65%Sr^NkKnM0%FW)U|_nS-O_ZX)g`Wfnxo%^_}%GV`P3<`Fkf znf;^Vs)(ynW@dEUoy6UVZeDcUeB$OSvv+jdy~N#%E=+V>EpfHVj3th?>m_Kq$>cwV8g~W=NZDQX<8Gr zb8|u%_S+VXQy=5n7meGDF4-!|68p*?>S}8A>N^183&`CMe3Bh`cc&V^qFh}byvY~f zOSV2c(R_%w(6e#+9Nl_l-MH4rYsQ6U*t*Kgkn*IRd8~2H+RCQZD;sOsX2&^jF*w$t zEpELUQ!~z$7qc_hmVu4VOk5T&8<&IYhwHyS*8~TQs2!(@aD#Ck+}XH1Tt8eUE)|!A z>xpyWVsW+Z^|>!`9^ZodJMLB7o4B30cX03H{*7zHeT-|zeU3YXJ4*f{#%eImgF73S zhwF#S#HHera6NG@TrAFpJ4HTI%2-5MIL11cG0$b}bN`KdANLM!C+19d+W)#^vGq;WBZlxFlRpoC_C=v*AvW$HW~a|L1koiTfDWhWj_}ecU^^owzq~ zuj2lW+k$%u7rcXr(*=3j znqBGW?H#iZF~I3dMczhW_ShN)>^VEoGn>BooAHN`hny~O6#^bF1$G1`S{K!hZCz9| z*12s;;4nP?y(P9l3$)N)XrYOuX<1{@LemKEg%)a=Tx@o-4~oBcvdtVPX@(YS`d0kC zLv7}0iTj|~{6f;<@BP4L4wJaFVzXW1@b{+K%=0Af=8H9bEB@Y_<4upm{fh9f;>~j; zyp{0Qcr#zZ6A4d@H*+OC_hNI8l!d={ZoHW-aZ3mD(ZSiL4U&K8^ z_>p+?q=erh{8qeaN_ZOKY4PUQ629wV^N&&={@%Od%|jCRIN`_R%`YUpi}0>^vqQpH z623Ct3`%&x#pctJ4}b51cyqtRH4<)&H(MpVhwz?w^WPGlL3l>I`GJJ*yVzVR`SADN z7jM2JaZeL|I^NtR;SUIZ5O2OM;p+)sA8)=P;YAmlizFZZ-bL}|D-ySc@S1q@Zxa3w z;s3;&TO@n~;Tz)3mnD1&b9sr)d{M#!2oJECn4YVEY<);TjJi zzq?QP;x*&;PP8%a7T?~w`#Jrc0+D5D_?ImO9`>|Hj-T=Vbwd;9xQf0??j5-ME;Zq^ zFWo+IN`U+LQRpd+3dZyqwLLClRFgAzRDD9hsEQs#MwRs%J*wK`FE34*H4%PZivK~{ zi)|73@G07s-6FngxVK+J`4T7eulUu(OZjD|dW)>+hLpt@>Hqh1%YR$9{6*pXw}$gq zoJhCwKV;>!a1FigzS^1}S z%U{whzb%}9d^rF1qvu=spS1Fi?UsLNxBQ3LlcfIh!})`UM_KuQYvn(;TmAvv@(cYd z`Q4HHw$WDp)mHwTZu$Fm%fB<6zc=x6_J4`BE&hwdM|c5F{xPRZok@cY*JX(Y^1i0~ zNfqF}A<+GvxcfagcT`I&wjz5DUPj(vS1NdUBY0VG@-lGpMsV`VFiu_>#>uO|$>4PJ zz=sx2UIk8G1x}W71SeN-O+2mMP^ri%I*?P0LB2T_`Q|v}o8yshM#kA;H@?P(NW@;@ zapb2=Ewd#uTw0E0X~c%eIV-sPLH{N{SMvP~ZIgR2 z(PfhLeF|B_SBMijeU(zp68{(C-DM*?wsJqJxPMT0#~bym$B67xZTS6`JIE(!pMS36h8{j=D34E9zgFEPk{%~DnFGOnQBa@P7sYel9poAoZX zMzImt+iMmbwB#utCXMJnCNqCQZ0SjP?fmzkXIFtI?DVn3nW)poby%_!N5i%Zp%=Kv&+=bFy{+_l zDt-d}i=HbzMllJ7EvyF~PZ}#fdbFSBABew^do$!1roES-npYxk zUyF^J&YvmYu|#O=Y;VAq>~nZck+r=DcQl+}R?G6wwqW2&)|=D=eYI=>aIA6G8SEO)?6PZ^<)0tU^EP=T@0<7+ zNS)OGnvP5LZ`Xc`{~y{<_Ww=$Vo!D|b2^2&oy;6hVy??rpBJ+}!}>#YKP)>fFwhyc zZFB?}I2o29jo4Au?YCLBjZ`Cd&e%>6mD?;3;{nCV-dLUv^6p{(by8kYd@k6Y3~v4Ia{gYF1`iN-J$(#{|tO|TT`rcl*)LX zPU}R=?l#@`Ev|C{$L-+1gnM7b&TB%)K5!7{xzpQI*2Ab}&Hjruxx``ba6t1r&^xPE z2^~$y^Mu^W{AjX;I!TEIBCip#)L2&wqivqrl$ET<`{C_7S$VJ_ZiO2J8>b z!9eurA~1=b!zh7S*;8elWIvU+Ap~}hQP%V)L(4w_XN#Vw*go3F*nEsW%ty?>$cIW> z*?)#Rfsa+aRdX%wrttS(6+rVo!|?f3ADw<=6np}wvQKvDanuqJZAr`x3BDdloxb26fs0}d z7t!(mjV}0_bt&toJFVJ97w}fVK;5?#|JmA4_2+9p&7X%abU`+4Oh+e1=z=Y*qZjkN z0ig+ICaS7d*1*hR^cla}O`m{INq-0P*a_aBQ;m%yXoVB`E>8h=$7LLZhM2?N5d1@| zuS{gLwu~_{(7RkrzINz`ESrS~ZPcFy+%n- zk)iD&yBsYac#>K^$T<5|O=m3o%7Wt~?^$?O;8KB8EjlB+i_Yj^4-(vJ(HWVV&e#)< z8v$qHk^WFZOcXx4dih+?>|8Hu{!(df#57Ju}RZ9$T@8FS>~lnq_-e zTcBCs0}IXa9C4jdG)p@)PWu_@mTZ3kVlaqV-y(7xDb{{Kh62&{@7M!`cbKyPklEs4IX;3_#g zSbItucE!c^#`o{-&z}mV>pny4G{K)Z4E&4zZ`of&uSRUfv@8!T*KQ^VyvlxY4RIEI zl}R-2TuwYP>WIU9BQy za|O3yyDJFpZe`zD%{p&~ev-BAVO{HU0Of|*$0BLOE>k=EnJ4e6L=XGfWJgZN{#e0{ z&cghAotkE<;5$MySvDYY{Ql3ZH5(ruvsH4QgJyc~6D!_EywC{c&{CGIx=jB<;8*r3 z8N<5?BOBle{$_$F7~Y#KdQ8@LR@lBoravycFZ~I+NrB6SZbHucV9Ck8>`Tf1rJq`K z({bn~(HB2`%#)1O(1N_fk#3D=vVSLI|Lyo@``_eS=J2;dLv8zEnRlU~ivKTTS<%IP@Cd<)OvwEiPs8acmSrOWf0 zhWOAKzr>feeV=X4h^)EOk=?N$TTF}Z`^yS}Ax+EhUC+Be!9OV9W=B@1rUBq56jNq7 zI^MrSw%nRep)<1exvpOrW)_e}V50{8(N&_i67{Br;1#E`-V{YSb3^1!QFJa3&i*XT zP+M$BTT@Z%5*_177SiuJk@&-VlE_2h@*Y@zVgQE)=v-Y*T zi83BpX$R+DY&W)+s#Eohfj(REK1%RXWdpG>wIb@mM z(=yr9>Uq}({jr@XuJ_r;fZH&?q=Yt{1U99QCFK1vu)eR(rs36*Xq}ly6SvkegK@)l zJ~(C{^KlmK7(o1*wbs~0_L{-OIUO~D9P9pfgWUf*=ZsxpA0vCPWm`!0pJdV<8OYsX zq(6=a#+uNS@-B?*rTaBane1N-%t(9Xos!RJvy?66W$5zK{RVxNcO#G_80o>rJm0 z?D97$HA2JCMXAPyJ;3D`A?6QS`43)SgjU`UeiK@m{Q_Q7c0)9b37@I;F>BrIBfpom z#+y*nWDb5tJ*TaUk5aqr`x&H1XJwb@MoB-#W>+LEbiagUeaTpFCGBD0L;AUyu*AKj z{dE6}+RyQ?)_#h=ka}fpj1#<4dgiwaMXxjRexXyL-vGa)0$!-wX75-8ewYbwW;(nX zJx;5`^5M{N`nwI@JN9GJX=8=(YKyDJI5}IE3vb6;WNg7cY2H6mjc;fV^P%5>odi64 zRwMswek=1G6nix6cigis%`1~RO!XfkPYZY_oxTfyO?ZE~gum}y!BLV9`cQDNF6TO{ zoIO_ERmbF9%YR4G%Eqf{gL+in?X~EW?erIY3C7gvpG$iA{w4h=!!~3jowT6=xQ$Lb zGn_U^J7s>=oMOsM^-p3Q2>vYRzwG-hlqc~kfNSa7WyDKg!u&pcj#1FFM=gDerFlO| zGdkYU_P2K_(%nJZzOUaQhwK*#@PQKH2laq2)D!+tFL-{vv8UzGZ<`zs?-%lJj@ZMK zw@o7N@)gk*p~atvr}qSQlxVMc6ueai?Jcz3@mu3P=B-YTrn}e1_)fAW67J1e5nRU` zJ+piAua~ENZM@HP@Gj21nJXsYg4kabymfV{rAwAU9dgc1(|(4gFNA(^4tY%DvefWf zRL_2C?cIGzCwRDo{QVg-Ip4~j?dW0QE|(rh!Ryez-~_>2Lc2>H#~C+CYtiTyE&jRw zm*PLnKl66GPi({-<*cCX*f@9#s!nvgtU7K0cS?R4f3eM>;rtAAakOoY2f}$i4A=cW zzC{-&`}gQLp?fc7UM^vNe#|^gWxl4sx0~!^{w%vcbwA9vi=5G9-#^Jb9uCu}C&IL_ zZfBTp*Y`1aJkX(nH{OFbeG&HxnuUeZAe2*2+L<0AY!@L==y`(9W9{bWvuCog3*lHTo1(eDn4jc>tIk4YT1y%$-$ zyYHUcf05orlU_<4(&o;0i+#eo`?b_bnGa|`)nAD(bd#i!^Y(wEODpU-9d+=ig=gov zH?Sff+CgmY2H~+p=t)_RvJc7rmm0ncxQ%**7OKI9nCRl!kKVU?mE2cE(Frm>&Tr|2 zrBQUkj%Yez4RoWN7lbw~`pAOs8I&*k0&^7R=Vkl<2@J`eCo(+pp81Q^w@~)kd7c%q z?28s(&*B&AGYRAK3FjR-=bXq&P>=UAMXa|pVvVVwOFzXE_ z*-NwhU(il)V1&NiQBt3&>4o=bb7U{=&DewImPYQ9*e`deUu02X%_a?3gqQBEI2wr}5 zqNg4h3J!|%9T*tzE4a7EieSFecO^O_@1AGjtd4&Q?4|nKwV&btRQtTisbM(>-1U^k zU0Fhh@h+p#VL7f3G(Vvx%+GBlkI>{>Df2Vt!NOzd{s;~BE^%TLRp@BTuARkG*h0L( z{VS0=X@}TnxQMiW(f?BYEnWZ6ZD&q}{(4UTOYsNz7v$_O_hiB!Z0RNMF)bQ5tU^C-Si83BKELrgxb_T{cA0{gc9ZR)*W~Tm0d=AIbhk9Vct^66W#8%;!|* zbqe!48GJX%$9!A(uJniRPUVe7!Ey5L!cpd0^lf$9!Z_~zM}gOL>umFQx@x`?Cwpa+ zJ*C5uj&5k0YObcuO=*g^3R`wbxMFC1BXT!~GHlBmuce*WvPUg+WN16BZd+PM8+2WT z$kg$kw(zJcSo?0?2hWo*YsPI$?ht+Q3Sil7%jgh(mgwU)0^`>)ZuQ9TY-bJ4!M;O6 zEN6$>Z*6!3ABS2gXG-n$oz>Jei#vfSwwZ)ewGHnnwKF?c+7q=6@2SuN4f2MPEm7O> z{*-khy#6-!sTS&M+0C3`Ke82h(B;$_Vve6W2Jf5yvIko47h5)0J_3$}HeCXaP~iR6 z_!ez{vWc~8!;Yrdpj6m{l=Hga2kyZ28M&VFo)o;*RbIM(ag18_XY~79pcC|5RB<;y zVcMp&EwN_m-($^`SFzLjW~`a9GnV%f)bL!9sY@Ab{7c#kY(ZZNuVuJ^M4}HLD=2q;^3m;~kj3apbzunP? z>eS9t3$Zz0D)#~w%nZ|c&ph{H_k%pvy4)t~^84Rk6h7H8aIw6TC-0&NE*3tq(788J zN7g;wN3V_NO)W?20R_BA=GMX7YIt~${hoP~H@i8%>Ni1zmU1$VCmmU@dmK4S3LLK2 zSzqfGvA!~G8N6Ma)8VmbhpxRxmvw)p(T1HCvEg!BJx^p9#bI987?}^o^uO8SZO=#U zmUXO*iRRVpE4&@q;W|?vZp_kqp1cvV7aF6TcAi%MJ=sPp|MeSuwwxv0|E_bKsjP|| zZEJrIX){#LlF{&yUC1%Gk!f(zXTjeU4$k?M(Vp+{S!3AWh^J4vq?I)l1m}b}ceioI zmi?xLcR2j~4-WL2ix;W`F2~|VIe*0@U4-F}0n=hCT4awW_(#K=qDe&=*g^u%$9B}7Tjv>hW zq<0kZUaiJ8Tc!luvEC|h&-4S(?;-44bq-yu|79{?dR$WSjV-&Zv!*k*X4*%N`bMeK zZcD>%pJo45?m?57+j3|-;RQx?(+0X=0oWADiD#hJm`r9Nl44>I2Sz-cX%-=1IWJ0{}?Ew~3? z;C1%bq3O9%bDYb&4AJu|^WDlj4SJ5Jx{b>yhj%uGKk?nQ^+v8oGyDN@>Q}o1H;qT;9ShTepiZd zO+CxM;%o4o;PgFVxvjmV63Jp(hpGstaec(So~)nebdpEuyN zc>~CoRe7=hjNOO@<&3?wTX@@IqvHGCSZEO*-am_sv#6Uq&5|cbTQyAGTWpRcUT9Pe zQ>_Dx6~Nt9^!b?md}E&-9yxt`n7Gzwkq3r0lQXE;KlMSI2tGXmEh6WfGRkS4JHb4P zz4a2porFijFNv%Rr|@ur>omr04{~mjcI&T7I^QL&$RG!)Ycu0k3e7>DgKAQ#nG$ZN z(DJFwtJITBy#n(?$tU#IVEmVHt*i?TFTcyzyqUl4l(;m%n}5Hj9HCW*gKw^(Z)1`N z8wJC#JwTazDQ`S5uGi)rXBi(zzP@T>6R?&#t3>14G%fR)=I>2ek3%Cw=#5yu**``~ zS$5q%-k&C}fHmi|@m>w(O4@HIQ~H#qWj}c{f$!*Y4)fi;9A8&C?yhn^iL9*LBr zWku8cMz?Z)eTH(zb(NFZRZb1*1Qw77)o(}?tl<43;5>MqO?2ZAi5!c-^j)Nrb#{mL zMb_k7yCj`6OE{N&`=&;yE_UbV!+P=_STEj*>y4fIKKxJO{L+_mOfu)06walo&{eTU z#2%TG_u|?_Hj=gf!4BK90|O_`<1DbDex}7!UL~>#;D~JM=S{cfHqP1QshqRJdEO+? zXXj6v7x{jb@796P_SrS#x|fka87VAQ_}0S(AJ-(*NkJm?GU>A8~)|? z4bup`Yaw*^r|B)@*m~{n+02V|)7mwzm_o zy)Cve^2j5$x$E%H!oLZBB7UwaanEwz*nf@l=KlM$TgJ`C-u*|o3ekfB-w1Aq9bM5s zcXWAwRd7T9+JEl9qzPWJo%2ZEzi#+>9_Nuo&z3$t8@KN_wt20;R`Wh~s8ise9eK^? z7{8-m*MfKMqP#uWJ!xUTdE*lJEc9Q(1tRa-^K2t%;jJBce=RVUlSlqNh+o4T2!56C zm&uokJ$P)i58uMK9=jeM!w9FZ!GDZ>M8|)0w&CMj#$O!?ZDgHqa27MqheIRorY#xg z(>KoDzHnP4tlRSAbH3A-J19f`&B2FHx}l8ohx2TGUfQ7wJjNrNEk4l>{;wKf(JATn zRnuN@q<89e-0v}d&A(Re7=-?-2mi}kaRL*9w}t*I1Xkn?Ey*Wm+Zt#|XN=nEKGpjW zxN~~?A8*+$<)sP!<~-8X_h5lBjk0BIit*W3;7`c9xdQl~&N{f9^)QWfaT(|NpJ2QE zQr62Q^FAN;V{9kI>NAAYsoUu+G=54MHSAT=&N87>yR}n=+bL~i{&X9ir>5^_AKjSx z&%1X2g#1p<6taK4u&BOK+B}Kz@@1;w!8o;Ze3Gh?b3y|9pB2KJ2HY~!QiH!3q`j80i(u4Zs z&4?*Q>sD0O3VoOG`rHRfIumRO4U=8(HkMG22cCfN0ZNkfUcN!j9m3100^hatv22V% zJ1i|J&o>@qe+`AyhQRMC;R%5&WZfyT;htPG4*Y30Inp|Pj;lJM(KSzNpZ)sIOqa;Xd@JXiN!ju( z{C8=u@ZVF2TT^P^snY>}!mDyiT-~>E!lTNPxY^&v-A`O9aRng8ss2~ApXUEN zzR5XCZ0|b28+vU$SlDHww3h#J@74A1sX<0w`0TqNTt>6@GyTuvuYK5B*S8{DDSckA z|E2oZh12@Pr@lYnN8`2Ie~diA8=>L0NE^VD7n4SC*+~2u`n@|F`~w^K!494{{oUU- zxpzT+5MJws?a=5>aIf&vk;&gBcYQOJv+|lD^;ucK>I}|+*{mh_0)Dq6Q|mMwg-_q~ zu(wnAa$*Cn9v)r0=GnPAwy^fxq49^dGp9AQX(i)*m$XmL1-y-&0jx-!Ppwsk%>8WE zFX!Tvpd-CdzTtmQe~f&|b6(^;hy6eDma``A-UlK}5>{?L02_tOR7&dA%hdj8qp4!+>gSl$AFHnQGW-=gjH=Q6%hrue58T5%$?3ts_R=lkt#%bkKOEhbA`M-{L*RFTuNB=6VObShwBc#qNL?>$Y3G*d4-)we``w z*qyBLA@)8EoB1w4b{m`xUm~HS7QL2dDeu*{tTi9u&0fQ|llFnb^cyA-`e^`kk#gS9 zU*0kCK}-3drP`sTT+mYTuK=3K1D{&(lMlWI=cYr$Y)>|t^wYJlqSHtGHsq-eEU4(5 z%w2=*H6?rt?_fNz3tc(=ICpdRFh-fkGYU@_d80?Nv46GySKiKQyQAaAp>>VlCV zc?!n~WSig(S0g*TU|3_L*o-QsykqcA?VWeH5d^6+KpW7Xcjjo^Ce91}5oJu+`a2mm(LJMjbhrbi#{u}<& zhP(~g#vsPAfO(zH7|#LUM%&`mY2M2+29O51iD^G3%@x2$6Ki{=!&!M8UVV`HwfG5| zN6h`)5aZE7!Xp-)AWILdi$@GxcIV+%P`2QW2#;9G7JkBCS+fTjubLP&V)LQUa*3~C zZEE=q;Bi@4&kQ{YW0~wpVSYt*m|sx??5_l${3Kj{4Ls-?c+ipZft_i{NWcR*Uv%EI zorYWkUFOFYjT>JSYbMclk)8S@ZFmbOea%HTO!5goxsf)FWq!it?0~PN^*wXZJBL;> zC4GdSoK71oe)5^+BqQrjaK`?sD(Sw7HO{4(^HBAaHL7$7{9YPuR z0$(EKu&x+Sp^XJL-Nea$wdc0M#@;IxcRcVQczauymvNr4=*7@-*~9er-_A4c<6H8h z*HqQQ8f}LD68u*ToE3!Uw_tr@Ky0mR*k65~wwsCUWMCPTp5h|?zQrKt!_yKUUVqzUTjJzFMcz#hu77Y^kKg%jtF-l)OkCjshZ<4X&pT%F zJL{oP`odQp(Y99JL`(MHY>n5na^!g0U-6pb!HtQm9o??*UE4Pg zT5DF~ghp+J*Hs2R6C^$I)g8f6E3<^ZJT*LSO^mmOVfs}FZ>@m0H6|lpRlu7rN2Fey ztp=O9)XliD7bN?O;p4mc7TEao4c4CW2E2@gPD|gPk~s^7(Es2bY_|1=^(f+68Jo6z zx%W!(597O>@8Ak=pgmD|1Fp(8c%{M1p%L_59(sv<%N($0Qg4diM!h!jDb{mzd@*TG z(2jawCIvXEkTYfY?8-Rsp2K-!hY&dM1{mw5li%!bbl_KxaLZg)RoIp^>iFOHmvvm_ z4(hDDaJggHB=|omi=z(1N!#XlfEi7eim9WN&p)>xb>4HY`$^S6Du~c0# zB_LrZajTF~Kt>lDBhT-SDeHKZvLbXyWK6<$I~!ENXk&HQZvMY0Q)mnoF1H{?H2@P! z3j-5Ph?QnHdnj`adbjE=2*jE4-%~rOW0XiSX7@Sb0Y8il_f2PFJ#%bm;Nc_hJlg) zbN_z#|Ik0(PWqRCZS^TOm%oAjQNJ%T_R=Tm=W6;lo_^;pefPG>oG=00%kde={j1@8w`j9TyNqfF;Lu^xfZ2rTBt!DmxLjJDp%L%(|6 zwv7B6oNXBb6Q(OxXbU9R6zV zNbG>;wtSVSR!loLUTt`=XsFS1NX@h_cmv~0M-TtO*h!;z+EQLCx-Vgc+pemX*^S1x zpz+p&m-FBi2)*|pyataAyQ#&tLwV3?edv>3JH>^@*ndwjN3~5bM}Ev5LHh)A#OD*t zF^49Y7aW~no;#vAaN*y<$D{`@k2w&lMtQ1=!4cOUs^vbtYaMZ?-=3}l2|J6;1JIO- z?-ZLIoZDs=mmMmLT{W%TR(2fQ`U_;}16AOqKxnz_*&2qt ze7P8HqwTJeGdl*L|BRbX3^b%68S%-YF^3X67d7B;1qW`EPgY9fLN(8@6#?vUA3DaK?=R zm&Srq8f( zbyuB+!)x!f_!N{44!@ZCCQ|PN>c5C~l!D7k;Kdcgi`#yS!0U^}s=59gb(xfh4Xh$L z>t1mx)I6W_S0?@~$V;spXRB)B9J7FNEsKx4%x#M+5`F!<;G0OkJtsoVbKwIl!4EnL z{d?oeI_9wMb>G+vqw&6^kKq2F(wDbc_j8!X%C*J5j)5*;@M7+;;S23*;`hxHB|e?xfS%2M%1WPTLCvSIRymrzEy?Q$*-a?WFI?`)@D zPu@`D0Q`qR6TF+I;)p-=IW$(atw&`seg1JMew2h)eG!^I!J*_$zd(@o%wMB?LE2F< z*WtP2z8FvSZ7NVbQkgPtcdd!{RWC?0r#_Et=*22<3_hXIT6c=P33nJW_VVtl?)M|` zU9hRO@Maab`w?V0;#6SaIh0l6?5r;7(dk~UO!WeNc>()h%(Z%#@Vm;)mhdsc$9RvL zushvmx<}!zvY89#k{{b!3k&hLQqES&SwT53lKw?y_K>_E68@0){Um&X@CjwcNjce+ zaUt$n%K16@dr{7L(jFJ>>DlRCNjaNIznSyRYvkQacrP*q5)Kg#A+t)@?WTFJ1C>5bjXs76~U(PNL0xS;`qi8B=hTlye{X`%})3q&4+ZDwROzTA)yuDp|Emf;P``R*}tXXI6Vrb$5 zfAB)ayA<3Y;bF?`U@a7|H##}9yVxfW0Mli8KIAuJeI2Y1dB;oSSTr4x_iomKYI*O^ zRjAjFF~m4P9fcF@n+oSwY?>LXilDu-3g(a66v+!e&OrYR84`GY$dfcJ_~oXu^v+ST zX6?uhwxi=T7yftIVikC9?wn2A@7s;Ozq8|0)=^suYxSPpE1FW&##ir4UJ<+}dBuSX zh0adsI4}%Z6Jzr6?aAHyyg(O(of&yP6Yi{85Exm=Dsp7(fY#3(K6Q?pC=arJO*%SLDcPafxE`jHDx z@h_miw&coU=!8Ps)txhK*_tm@XuFPZPDd@Y0{gx>6B&{`!g}1!-k$Z^srZrW!Pk-T zsc@w?q)%dB2d6t3-*(0qUGc(p#&yowoI}8;&{Rot<`8x{8`3@Clc+JR<*no(^W>Z} zHBkR*Xt>6&mkl;DT;4#s#~VmCyn$?JIdmkA>*``Of1;+YW^9PH76P8ixL*wp0@iA4 z#4zX6G{I6bxHq5u`RM@@>u?|{+qqCgs%G4W^36I3u@2rNEc!oXYh%0YZG{B?@Gf%Du{;Y|(n=Taw}ZQ@VmOxw zeZzcSCi9!ixSePKr!(GeTXyBi^OaeGo}}oJJMf*ktERQHFU#4^$Gl5DgIEvI`?97h zZW&Dd6OWIET?W}RZSvNo@%e`s_yyFbcjuLjlJ8qIQq=exemr0m*2>uc}A)U1@ z_bgWmd_36DebYr2Cq5Un_GwVHtxm z&XYlG`L&O<(P!oryVo-qKbeOz=*_lmmVCmq+sS*rB?H>Vy}Zc&%3JV}d5zT7zG4#c zX08U;>R2OTb)1p7I^O89+G+G$?J}IJ6O7OE?Y@p-ydyG5`C5SmOD9)>yRdlx|3cnn z{}fzyA!EmdreE681b;cwj-|AtW6+d&1<370-W(BHPJi>}h~ReSW2w9+@+N6l2_Hbd zc~9h3`R=8f^*8T{{GD&%J;{3_9?I+fo(O#4;pNCBMC2Sq#zABkiiztk>!9BmhevOn zrTQ5KtP#Pn1vX_4h~;e%;4u$*1$nbPH&&f`65Y=rFvYp0NN{9sQrQCMi2AIctlg;j zm$6hEz2^J0)y`h1$E4f;^i=T3J6>}KX-wu-`0U-w$QePJqh9kB(i|kslZ?aNIJtAo z#qXXz`HEsQvKCKZOG(ypWUdPw=?yMNPQzqJa>GH|(g#{M@#o&aE#UQ@YZ8ql?hk)H z;?FB?WqrJTjq+`GLBG3HlHj_Q+S;)#wTs3+$p4?SW*+AMV*bzRsgi1lui?K7I?c)d zIjkcW<0rJP%cp#b|BC+_HrqnO-=SS`>^)9!kLZv{WRHSbTSrQ#LKjzdd$@i5Dyp-baBu^{zFJ}`On+#hIkwx2}#^xBK z7{eCUDL8XOgs-Q^Z*`H8z~1EiJ@4i0w)qaAUmOI+IkTEukmbo6l5fP2S70=B6}EsQ z^|8kGueBS?z*q2GhPPe?yg>8(#A&^=c3-So#=En_BXtNr4;~D*e$(_>OY~&!B#q!; z*;~q}L-v-5;$uYr|4&}?Jn^6Nn&*oDsMjnMpE)ZKe}&gPNBo6e^DObJz2*S%>)1cUpY1jK zi~lpPnInEB_!9poCqwBEykgx+%bp|h*`miP_bze=Cw&z-wBdwD&^pL#UWk4|D>T^{ zd~_6znyn6Fr06x-9lU>zUt%}VM<6P}0`>&+VD{y*@ZBqjk z>_IXI7*LEOe#fjE_el zE8%9H3QT)sUlMtajL!0roM(~S7yT)b*%x_z!9h;sG{hE-ll%6dTLoHZ|6axWKjF4; z?lb=<+#b$u=32tl-`ISdcjSGdDlhM?muJCi$wL1`()B0YUztxzcmUx6%6wesiSbpR zM;>g!tU~v$d?^2hD)TpzZY1H6%6w457ZAPx``8+|1wu#Co^KUZp*^=%O z!j~wsTEf!^PvhQG=SlEYUr(N7?Cd}%mEXX>8?gT^>24-`vodd!@La-km3gbq)6-Xd z7kSdTtKlqPzJPxV7(+>SAL09yd6R?}5niOs8+4vNzH00O^A=K7E-+L62>%{Y=CzXU zal(%)^J)oWN4mOEnF5p5-z590|A#zoRaL`zzueD1zcMe8bk7ieMwwG2{4C*Tpg| zKa$OU8LS8Tj32mWLl z=dj`N*8OHC=N0bNmez}GqtI)KHRDbU47W4rA=fT zSPKvT1DxRw({?E7U|A#5xq9+C!$){96&W7we zW1{Q6{A6f^@Hkc>^UyhrvsyCxG47ga+oVi~V`t=hGv9(&BHu6SZ@WG-+Mzoo&-27x zvKX5BWchA4GAkmJF1Xu;yM(cMi?ov#Qvpa0?0E^B-=53UG+qDcskypaK zI-KzR4soWag7+@|zC`3Su-_E2IV!h6o8?dRZmL#&DqD{uuZB&4TF#hBEy(uAsU0Qo z-4D6`yrO8vFE-_!OI~2f?Xc@H5&5r6uO7YWPmJ%uYesMSl5stxeY??i&FD_(`=qVx z`)+W76C-I%cca(9y{fcs}@*Ku&(vP=0DoZG1Ep58>?g3Vt<`1yiq zG<`p1lCN{Ta&!#fTgsEQcLG?u938lJM{XgsU1BnG6oKjY|1L7X7k`JgT^&W+HgV4? zXSE1?rwZ&v(YHwgYw_R0cTc{5h`x2QmrViQDz{`B6`TV+w&Vtp-o2q_q380zJ$WY`#*%&H{ZD8_PsRb6 zlHtF>7dehTjIYpvXQbmhLnHppe_1XOc>{;RnHzzX=YidH&e7LU_XXfIX~T(l zuPJBLm3C$Rn=)$HySd{xKaeo{_sRrizDsx|`{hdZ%RPOSflSCDFMG@?))KM-MdiTl z7RF^fVMRFQ))!U`pzM5{oTFMEpe(!BY&{ujuBUu>_05YZdk(%EJ0u5yLphU9O2rqw zcD_A?1x7vTPd0MLI=3KpW#gpX6HRoo8p?LZnX=ww98VsN9~Bvcb7^OHn5~V1+354w z9MVR^>rH&qr>^tZEWRV~TF6*U=6%PqFg~6HJ}&qwG(2(^>n;ZskwG5-tma3~Y0laG z%YbkDOQqj|P4$xF)O`Xk;PbJJm&j-f?Ci(?0T#c>SAifrn$({!HD2jP$7eS?c{|cwbJj zcfOFJHuN=V!*S#vW;!xDt{wJ7#cph3f|q7TrmW-T(I%P4mScA?6d%VqZLhURZV z&oAe!HyoH-X76v*pW_Yd)g?D@59W*XtBQQx`&FCZJ!J>)shEuG`g zTiE(N+fvIOw#cl$#{B&^I|iRXAAQ8V%YNwdeWO%B5Ev%SY_&V{DK4w%?$6x*k{4)$=yPz9lF}9wz63#T}rh>;bEG5kyBlosKBeYV7#MyqzyZ3~>`N-4=yoL9S z>OJ870q9x?j7HI*qKm;gkUeCFCu$FQo3+-$UbD*DJC^39#2F`Aksr&YJ+!L=o>D!1 zuZUI6_2A`!HhJq(R+Z@M3wADlz~ z#TilV2B2-5+o2IB;fsCoXk3eo;dgNDB+BrlqGKDzx6+ToofFM6aBjiQIBgHhLmI7X z(XC#Qm77T2pXfHF_}j$39Q4b?y>n3n_q7Jolrksf{ckt-T zHn;O{J5FQ+$1?T;6Jwd*VLsr2K4{6&_a&dK3H`6t`d1eIm&iZicUk!c8p3}p4*!ec zpYVuQfUg(eASRn-o;#^a#^XZ9<150VOMh5=`h1)|ey-zE{9oWl#vw>p`gsHWT;Wj7 zkvLb$cJj7zp(`nVB#TI{)1;EPOo8{y|9pUA@jC$9F1 z$XJN1RzgQRdeyG>IJ42wy}m5}YvlQd?nXSNko18a*cd<$H z5%c*mykoKB{3&}tX?>)xDgHX$o-BW@_Ot!f_}R>lz?k4uIX7-noiAjaz%HO(?^)|+ zgO}uA693p6cGQ2up3fZqU)nIBs|`8+8>DVvy;s$G7Ord(?`M+2~XtxKS_TxX@7h!sjWoNy1@1+wUd)&q@0*?iteeB~4#w2uU-P@KE@F z5}rnQ8vH*=e;a8Z!97d*G~zh}Jt}F!{Jy0UzLNMWmHCjQzl*ewj}dTyi?M{$E$uC_p-O0JBl7V}yZFAy$7_7o_m(juzX4F9L-0z8z=WUj;5Mvx6n3d#a)Ur^ji$vQ*}6!3qK3}dRotnnrUy5?=5KA z-d7~7I2em==Qq=KC)w3T_epzI5T0gTD#jG1ZIM$x@`etSC`2CPYji0!@=BJsOA7b3Ek{V*X{X74L zAKTKuGsf%gjBjyoa-UQ_;Ky`lpxe-@j6d^#yuD9HK5c4+=8v>-3vzQ#+Snp@VvJW? zWUNXan3l&_2_6=_9F_}T8qG7YasHG4q0-bd%7v#093XR$51fRyshvkI#ir0g^fKy{ zA^VVqn|m^h8%ejy#(f7mXwW&s#g>J%L+TY>J^UKo zHL;H)w&Q?-gNn9~_%CgLhyUH%9^~G47;Oh9S5^E9y^(O6vuQK@A@sQ`$27&Lk>yfH z>tJIUWzcrvYi;msv1C?}ORBPAC%+K6)z^9SffFg8g)B; zyM5na==Nm)b<{D2_Kev*$hhzXazndqJ2{Vzk@KaxscxM6C1sA^)*Cs8WdC&13g26B z)3TCbMk;p*8*dO>+9`e;@s>PEvj0b<0~d|ZdQ<~Mw>MeKGD%;`whbENWB$u%gWTPw zFg7zu-wGXMBQ7sjY!{~Z8|mjAd@s9Upi#3Z&KOD>$qx-brr`GB-{H?l`V9U{x z{XezvhxI+luVFkH80W3X^!EG!e4p>vrxjUaJr178eP3oLILfTXjiN2($)ZPY^XDb2 z;nv-s=m{vl;M;ugl45*h9o)`XeP<2Ba}l12tQT3s$+SPn+1iawrgGpy)^I)R;Ja%$ zjWzsu*LvU`r>qFyC7JSDvPV3NdxOYW2dHNl|0=$=WUYTm|BekPFxs}p z0u$-}-}7Jo)njihlIKsvNnPA4r*wk5LCf!P`88E4E^##8wMw^<#txQmeTkIVUx|Ds=^=M30Q2X;4* zuM2jMZZ}oG>E;y(fyl@PeHL+J%VRNo?7OnhX zavQRT$R*3&OT8Ul681To9M^PCV!WgG7~wI>xLb1TXNLXu{8u8p&@TL53GClNKg1sX z=fe`<-6wMY-GjBz6P|uAc>BHK@%Mq(pTwTjmpv)Qh+YR`M_^BZ+Od7^)WG($z^m|p zWk1?ZoO9k(txqlMTlg}s5jT0(*si{|I~MD{R@ze5-4X7qGu&6?)ki4u^tEX_w7Yb9 znNx*#wZUT>Df({a5_`hB$>b~KT$P(@@qKes)hRdf_Uw<-NsA5av-Mqx=%*~UyViL~ zuldBq#TK8q*mersv!Ur(3&z9usT+h>`vf|&>F|2mI9IjMma&XeD|THJHdbtFEncn2 zbxGfY@V5s8Zwh?`;cw3;ZJ2J|A@Wpr@~yZ})&JpdSMd#OMfkNFy7OzL?yKO}iX0$v zhr2#t>}ISTVSI?caBY#{F!%F6DbI-|##_z%jbUVt+F* zgS8@h%nn&AtnWzQV$aYwu~(S@OiTYnrb+WL>m2J_>+I{Hb9H|cWbYCFW;mYt8_vDW zuJ>u@7}|TBvD0nF#(>DCjG=z+Mhq#_O*ka4bfy=+|D~4cv)V z^v-FBxdIzDcY4jZ$Gqmx)?V!MIFOym_nL#~-?_8B=HNPHYu0k_`y#S(&mk+f4ml!l z$2H)!a&(Ej3&(ce&OJ`~g0Y=#z`BN`Jmh6X=aV*i&Yl_&n_rx! zRbr={cM=#Hhi>9%Tm- zFGHrnH`&w5-OT~)9k!pZe40M_teON}Do)Ex8MXce+92CUl&En=ZU}A?=EWvr&92 zYy>l(I}~_t7~cYCrgcxH@m^-sJymqsW7v};_8$KI61Z1nSHQi+QMgyyl+L}!F~-7t zpL>sEq2W&oUaPb2*WT`yugy`T;0`L1_HR*X+Z@ytmfLOpG@kn#+xo}u z>VLV|uors_LLZ16?tfknEr-AApC5In_pFo?zSEPocCWKg>MRZ4|B<%M;axBNiqv;y zxZgs@%6WVo_5h>~6P#PjSPRZggRWA{o5VjUxFKBLqm=jA`Otsid5gvg((aU~x!OV6 z=(&1B+U@*yuD0_X!3mOA@4ek%zUc?>LQNPi+z^Eq1Rn^DJ8QEHa6N0kqJPX|#q;h5 z2Tb%-^xszSU3}qKtmav`;U_9BSVAmSu~*)$w?i zhxIIS|5t?9{mb|T74#&dFJ-7<&HHP_u6XJymG<2jG>(Q zEj`9<5t?wC*uReG`O3PA=<2<9PuE#ssS}(uiGTWhJ=p8;#B;%dUY zf-YTPnJ1;s_E;xtt;=_z1G~Uvt@L7ek>W*cc zRciHFCB?sh|MD+itE=-9brIuH3SDl^ zVI+TdUG?dV@3O7u8#SyI+GX~nPc`tYS0*SkLBjB(SN2k7yoBL3uk5SL7zx8i7Fimb zgy9jd%u?nzj6n_SXJvn61PBXnVVT{AeTAC7+GfYrZ0WmMvxYSA(!Y}QCB^2-5}Wy@ zg!d8NXEQ$|yxpGMkUA?Sunc~~?a1p?FS2X>YJmqQd>hTru-Vrun{B=O0oH4t;}rZ% zxsNs7nWw)?E@zh?IOv4L8``*v|W1fh}b06NsSXSh!=BB>DJ?9K(F0uq^BGYN<_13xhX3Wams+qIxG-M(!ll^o* zIx`hngi}>f9pNU%vaG*qK0%!o_oZZ~-P!+gu~qgv#@wyaI@VH7WITk%@$`rOiuDG9 z{g4xraWW?d>Iv(%vHunH3*(Ky<{6Xwf>*EwWx@Y>2K+*7L5|-eJO-t_0cfKgc~=iG z?Ch82Io#1%a%D@qWCCuw*t~%AKneAfc!l?r8Q$;G`~zM?E`U6nLt}iKL!)t{d}3>G zRh-wiCauKt2ViPdLd&?Ob4q-NjH!^eH-&UI@gY)2=+=4)lo-cK9kE>GV69#%^KGMJ0f8Jk|XcD`>@ z-PN7$)$j;;gTUvQvB{IBhNnSu97|Qr@8SH+)gaD-m8x0Iqc)}td=!nO%IO$$1PmBkZDlh^@|XS$XFT;0!@JAZDco41Ku`ogE% zAMa~Nx3YpWh1ev0YTFPa$bG)(Vn5-)HXdzpihKv{%HsPtZF0g#XcC|ETLr${Hy6k{ z;Vjd*7CjikS)`G83D=Ysn~mtg=14qwBwoTb`-$IgGt&qQ9|T%}JK40wradKIrP-FE-jl5KTki?=Nr>)lp6_PS)XQ|7(y z4a-JmJmr^dyR_4*`HN2f@fQUrX8LQO=W2c}_nK+`PU31ByyoK#tbhGK-G5l(e&aQl z|Aza2!kwkk_6)y-YaYdB`J>AGlyCr9Ln%+fHRwe=xg5Tj#G|7x@e-~hUZ<&wpJ4-~) z577F$J_qL^-IjZ>{p~`wmony3hLk1p3}VaUw6sSC7Wa z7Oq3)F(~PQA?MCPhRfsiIM)|@#)oMEyPC2<4Jlht9;O9EFGHto1HZ>&tD3ee>`6(y z+$WsLhT&A|)A%inr)59gxFCY3vD*XfDP^jcwvMwsxayL;f?2%^)Xh(;C*V~O7M@dX zzh0NPi1VB(-;E4$g0r|R!D&;;?fe^WE0~+dxkk6kcCO*d^Ln@wSUMSeI-WE5MK160 zV*ZQF>^XXzPUkDrcD;AY9wK9>9w;`GSu0|v@&fWmo27iaIl<%(g>{~+VOGc4CU(W` zA+3&!{WflAB+m71+#8WNU^PXoWFHmDN48{Uq@k->w zHmD-rZ%kCDXfy8<+YHXS*yrkP1pgp=LyG?c$~lsbydU^ac-RYRgZS|1xz*nSa%OivKU#Pxf!Z z9|n8~{`)2Uj-sD6%?qyrI(n+SdpKM{Gfjam6#Dit^nZ6_6GFm|5dN?C58LA9{^46& zCyva{k&NB;>yc&QZS|JP@JMphWy5I;G*Z#*$!dBraXIY$Irzza;4lBkR@F2Ko5U&5 z5piKVBz$LNAlpK|_sV!nF$Y|VA9RfIyW=f8B)VRB=wZC2?N3Si)nT3I3mF4=i7DJ| znZen}rnMCNl2W{8etbs5S+uc!(?tE=VKsM(f)6eMXVqC%M^)aehV52av)I7jBEfHJ8)y3-2Hu!;w*hLZ>nPLz5bL?P#Avz@J^R&cPuix5} z^L@6ApEiQ*brP9wly@lSQ$Mif8{uJsDis_(dUyf z|2O>b1Ke{J<~h)nv=;_T(PNHLmpQ3lWJ|~QQ&sjsyzf9ecr$0X$LVTFXyNW2d@cHb zmCy#lb37M&Co50IR&nN;K5u}3#WVA7*#xh%=&hoaGoG1$?xwGSWwGl#)6u7+tZr$e zwiyAP7jzV%XKeAQobcA2J(C-b@z$!W6{pRkzk|JFeCF(4zLVpXd9WkacW96@FT8u+ zrW3{cXp48X=i#Sk?;g??9u zVym{3?=|3k$uKp12{4slQ_a;Xp`&dmdOQVQ-z~NT|7T$s>Jf&qNc=6>)v6A|)XXqU z34ID(xO6TsC4MC^H5Ztw1g2)%5-X<`HDpx-Qx^eKOOlpk@c`WL&u1#$amZ>&vGr*f zi9Ks4a=b2VXvkX{73VZ*d0s8oTU0W}2R-QOiEIKkA&TW4r6goLXUlg&#}@ATU8xdI z)wc3n$VgzfOUuo=`bjvgV>NK)%9C(LN3n#@mT+cA8+=PS*JG2Q*#(}QBfbkBE57bW z?FeJeya}4;I%i(2nE)SK_{}a~tgnK6(1%8aO=Q!M<$b5ry32mYVLro{S$BXL{*mba z9=TQMI@SX8!ce}yuC(Ypv7d1s-=R=nu7{IweS9b4ZGl9gu`e5JRKOFMK%WXN zIfb}8h?Bj%6FOFCx{1WyO59bPuSUul3wQEoVml?(fArUkrAlph)t=FCfVapxu2tsA zYcKJ2e3eEDZ?E*4?T52`!M*ZbPdaRI zlW*UChp~T~!UTf{O z*Dho(WB0!lL*Yy1?tWJu`&~Lmd|)=~C3Cx5>lWkF_zy78&i#04_V-L)7SFm9FUS5a z&&aHt=`IMqyoEI!+GAbVD4Tp5eVIpJFK0b#(Y-g=Y><0co3WG1@3^HP804J;@1f5< zJS}u&ohQ`FeRT&vu|oe?cd6O0Ifj0sdC@$I5C0R`d~!R#AGyAm%*Eem!^r$5O|}0* zI+}g=J#zyQo#_jgY2Unm=Rx?k_;(txIx_cSc+q8{ePxqFy9)x5IP&`N)4DVNuk!Ip zHs^F(**kT{su&xo_84MpIJlK!@*gBVjQH4a$x`Qk5p;jaOV0n-^&c7#AHEU(a0&Xo{wH9^x&#{1|B(Lo zcmD4{pX1zi$}MNC{}R8BeoJ*uS?+$n@T&372#=*=27g7!X%Ccv=cyfhFb8MX-i@s~| z?$Nnc2;RDPXzf|S&gVUZ5)j|ZvM0Oml_`ZrN=UW#xclv2JO{@bD&E+$JszK~f4|@U zbH1T-C-ALVFL7pPE%tiW%jJG#3i>6PV!31rJ4U9cab$|O|NoRJyo~#VEMak%r?>dN zgusgF$PD&&b$a9MXfv_td5{M^$3-b7gyr&(kH& zf6>p=;IHyUKTm_J`Y-x<8obnh(a+Q1p#F<~o(A9aU-a`dxTXJ`>ofE-sOYKs!FS!d zWh5~XN1nI(>cMx7*b@Dg=;xH>?)ODMr!05-B>FjJx!X_C&ne5@zKecNS?-QU^mEE` zcZ{{r&#B6P1^oyI8-bJb?SJR%q__osc%=XTg}>=U%UfAgeNR>*GVerB--YLMR$lAB z&dTS3BhrHn&LblxOwFq+Dw^Dkj8nPn{=i+*!M}=zqWZP&qiy#k7k3l0jQH$Lk3&Pm z$}eUQXn7(sSQ2OEwSPZuxLu#cnJUS=$mC{E|042o6r%|IOW0k?+3@NW9_-fO;}AbM z_yB8IC-Qm+aY0*=ttB7H|0)l>>vw%t{ZEkPZt3Y6{gaYFWGgzNADMX%Ys6>JzvuwD zdEYj4l&7JCr-_}M>ltY79hDw@%c2V0gpw+EgEuAMx4yL+`LBlOe^Rb9sd@O8YORqS zNM}-YRw?^-&&t|ilaZU!DuokUsCzImu%hE%WxHnYTfD2on^<>M>z%t!9($B-wKiQ? zw7z~?8F8DCm&Pyf1havGnS<8S$m%n!g>Cd1SiF%pE7 zX08p5d-_%C`)cc4VDc$#w40+mUJ7fAMT90Y(+~3e1 zF?phWjJ9X`_uH8M?JGagzf5e*(f+|3zR1c+VI5_fy(S{fhok_J#g!y4>mCzL@@rSH<@4Z?u>B z75)2rO#k*>G479hdYt(kS8Cu_zNcr_Q}8zRuk2IKaq~0t2>z)byE}Vk&4)HRkl((k z^%L5kzHoiL=raL4x|Q`l3g=etB$6IfS>iQuez$>(-`yD)_g!fFyQvO7iMB6Wbki1n zFCV>pDL=KLZ*7Nu_qsG&{lS-M_M9K$^XSs-j*ok0z4>3<(_;86IurWae;so~c8>Oe z_^S_|dKa?)f8?Dyvphe^YAWS?sqAsxtn+sebL1*wj%32Gv*4fE@Xtl~XypJu*_Be^ zotxRuy!8%z9NpL(FT=BjQ?`Ma8=&u9}!F-G!xg=@k#0zz{!J2hN0Daiozf<;+eMWD<|4}huDLa$1|Er0k zQB9v0#Kh6q_f02$2C+CQ^I5Bx@++l}bFF@zXYd~x$bOeOR;;lAYpIDDK&p7lvHbbj z9{?Avk;HIsAL!_}job;U^I7=CF2Bv29{fHqo&}84fN?r7#_w%;1~67!mn`^mD({b{ zZZ@zM>{DW3eeJ%USvO*L9m+a))3w}FJ*-Up*T7jY#x`SkaXz=iL& z#erwB^#nii46M|-h%12aIZK-A|42-ew_Z)letZ7&^?OU2>%Hfdj=T0GxDGBAToXDQ zf$L7-Kl18ZiRsb5aviW#Y!ink_v>8C^QX*f3uCSu=F=SoBO9Pc!E7Gs$uRpLVAgo$ z33%1A?p?vVT91W4pSk#>by>L6Vbyi{9!ZMch6$~JsW?79Q+aDSIbBE3+GU*B5T$h#*=`ZHnE{36do2IQXH=KeC&7FYOyW6 zt$PDRkMeVrt+t%{x-&p!C*K)x!S&HQ1HNplRejlN4J@#$VXtcT->7&CU%n?GjGfHX z!&bX{uGReUM60=N7&acWH}A18_}MKlQ=(1sxgv30*2` zldXZTApdRr|5JYwjtcIlf;Ic^vrcCoqWTqnci1#zV(}I*KH>uzyJ|jX5ot&CE#HSd z$hz$9y@P-Jj3-C2&z@;53}N4XvL@qj``A838$1cq8+;dSXz=}Xd`k^6TWk9A>%%W@ z!$ie)=bTSC+lsX0S5J@~G{=)uEMNI>w(KKWow9X=fhA|ofiruN=9}5}HsXIZda{z% zGXGazRk0k+wQZa%w;3Nq6{Fp0~dK<~tW`nda#? z48QKHZ@M#oi@n>kv+T|bw{&11I?Iz-XK(lJoP6hnhZPI>Ydp8t_<~>KxwYWpP`oFh zF1+0vjOY3A4YEI_8hesryli}r_@X{<(4wr6vG|96fGN=0=l876O2cQczm2t6^ZnTf=AbWe1+ZPU1NUjz$!+$ExN9a-1r3%$gT?qC zf0RJnCUm#J@RTk!m=h-)-E#2R4sdD&ZAmaS#zZe+wWd6#|l-o3*pFRXz7MZSkmt9&foHKh?BP@dnmRu%;3F`jMM{LSPMrZ%nl7n^Q_b2++7vu5mny43x^^%)$+{Q4*awd?C3aFGLAg@d#X7(`V4;A z&_;Alx_NG3PbxYOU3wnu%uzevMt(1@?Q5Nnooe(T#v*?(=N>qn!^)z6#A=|whv}ny z3eKXA;&Pm*K ztrs))Y`31i8y)+F%ykC8F6Q{;v3uOH@1Q<>@5^Jq@9Yy}cYSm2pswt~T3ZefGiIZC zPPU&iX@>ozNwaL*q}ldzlcw0;h~ZnVefw=|`LN&`&Rt(jS(H}ieb%E`x`tLC;oVy) zJK68E3mS~xeWJZAOFZVjm3Wo;H^hDvHah>YhtjUI^i6OhN6r;Q$sR6)yAaEaqPQp38vN+C!D+X;` zamC&5*OxPo<;;Wj>V#VcA5tb59;&?&_})7QHhEGCIX7Pe-@WnfjC$nsPI&JZbPe;~ z-U-20_s-kjFiyRDe9;wr`y2G(a{6)^eY%u2^O6vK4E~*V*VbIXeoMyTbv2oXO&jlD z7~I6WdEg?xspG=EQxe+XApklp-$qam$GM6IrZ@F-!OBL zImPUOPC-r=pZOzYI0OI=ne(xXyA_@&`XUYyG{tvX zip|-OqqHYk;(6Y0VH_$K|I!@DKdg!voPu+OH#-Si^A#Bhf6daq>rbewemu#0)x;U^ zo)Z`14w=ZonSDZy__(%nZ^zLmh}TO@;Me=3C(;LG$QcauYQhC^m)M6&-iR`Kyik(AJ*J}C|GL4xq@i;@8l}?+Mkz=?g8yxuI zv7XVQFF*IOmoFM~_@k4?V(9cqtM`8Em))@(oH;pksDSkqe9}4Y-E$_PXQe5=+dJ1A z+`=9aH!j#r{e$Q`-PCJ=|3u-G3IEaBd>(Zhfw8p-e*w9egL9Fzo9k#Pqv4<|JQ2n_z%r)w4m`WQy2e+5sn-* z%+yP_U#Cs={WX)O+OL{4#olhxWcv--Z_B*!Ssy$t4&K%W9@iIM$NgiA{MdfH!9%RI z2a$KRXC|5AdAE(tZkx}NKF4|jJoj4t1_f{9InP;FH%j*22CvZ=C3~->OngH4C*NN6 z-@kN}{8CMfaqPbP;^6;z#P(*r+-SvpJvf(lBx@)>o!^sHT<*8d7pyJ#;DPrXn8KTu zm(D8;-om)WUsm#tU}@m9@M8Evrrj5qs_Z9}Np26Ildd9`jpphM`ZB0QjcyebcS8 z0(uY(B&)@{WtUMVd9;qQW2TJz>D{s^l!@0bqpXno9CDx=;m!r*i)PO^X@)(T)WCmn zNhFQ``bMeX-%=R-0sXvZpnOy_?VI>-;Ln=UlWs+JuhV(7OnVvM(-@AXi%#GLv~d-( zvWaWPIDTiwOpG)7y}kE;s)=E%I%m^Ar616ap`F$^1R49o?^w{VTqa4*oW}HevQJX3fI3c_F`Ke$;Dnc!LvL`N`K zUGjcCe7%?F_O4B%M+aXbrmJj%<-SZK>)p4+k@Z5X9Z~r9*55li*y`%prEZ@@ON$H) z(t*LHjOh}_Hk~oz*LA~GnXN}`j`f9Q$He_>yJ1a_#Savu)~!aK35b3K9r}~Bf%fx$5pg{2-@o9 zc}DKbv|o4e=Nps@e;m0l$G({7Zt&$T;B*K(v|ybEo$8w(^368ts(l}GG*9$~Ef=2L zWX5A?y(oAW^r!k-uT(FMXZ2n3pXjiMxG<6d8@=g92J|Mpm(`!P)!(!EehWSh4XowI zKJkSjvK!)$-Hn}I@m!|jN2J);jmTRa*gdh;SLWbvp}Plav2V!ET!GwHvm~c}E3o!@ zGV2}(-l7@BV%@ymk++7JaiSv)VV{9MKF1so)JE}8i=DowV}D4qyBVY6M<}NCpYd~? z=*#Kcjl9%={Gc`X3H6us;8Xb0BR?L-#+iZ4lN%M%0WY|Y&yZRYNwh>Ii$T#Q~4qtj;oZ(BC&_=m%0{Nyvec)Sde5*z{MPG#{ zZ=3%a_H)aevHg;I8kbX^W}j>DEX{s|d?zo}?(BU(#oobx!&eQ?r`wTy$v~NQhud!Q zspjQ@vB9Cxz5L?7#1m%WHwh2XoK4`HnxA8zl!XM-AE*!X|2ET?9DAWjGwoYR_ajUF z$1@IPeRnE;%rWeV-S9{0u-si|_~TLJE3J!$@3}hqGq+$H<$eI!gMWtYY7xJ0@w<>8 zF}X)~b9elQ{4P{Kv>yWgPhws~V>8K<9;mf=qMLUadD8!62fEPBo2qxAZ_Vi#H*b>a z)%ZdOShrK)$Gp=NX55WRfp3LUUrzDCB};Z`&!EZJ9JB4=yq{~KJHTu8-6Hdx+PNE> z`i-V8djJE8CA?TR$}D@RDNpM>1fFe&SEvs;x)3`*4U!3;93GtS>gRvj z;MmHK?kWwn(69bxEP+UpQrb*V3QXcnnqu#vzGP3qQ~a%`tnYvHZ4CR8iw|<&NsrgF z(1VV*+UH&P60y{or?;(0AhJ~J@EsF^-^QO-a6ia5gy)LIyKOLbw?*7-sD6jI-(>r} z<7@eL_#vy=AE=&jAU`409-VI{8<>(Qvh}7c!yXq~2K<*%_P8lS#wwK$W%WB(L5I}?{Ff=_S9lR) zq@R|jcD#YfhV2s?e2mx1V`Ea?B*zh3h4u{J{khNFM=Ja@I1P_{m2uVfcn13y$InbamRRi@w6J(7_bh>T{=dZq zvFqJ!+X=c!#alXv}j;23Rp1M99ip5{aN1(i&&lGB0??py-Q#0Rj9H5F37 zeFXOitcwo`HxI1yhdQ552+4n*Ia+Q#2HyJW#)rYva6!LN%RN6D-+3GJ*a@zF4zBL< z1pB+VdOqz|v(~bq{hDMi{eq#i%$g!^hC!BRW*?xktDo=aj zhtT5>LZgz=qj)Z#%jkc>qTbX`H-3^PpZR`-Jkh!AmJjoP8S8CqOysD2;vvd_>vI3l zo6UU!tbr;=0ei& zUwuV+jLDhP9J?60)_OONJV&O;w$F0&Mr-UwrpU7M+`JLW%eR=nOgr1nD>m=?LO(#p zkbF_76ugADHHE?5M$&`$bR9wt`QegOWQH6&g?DrP$PD0w?s?kG9m`td2Pof4?evkC zTD4@?bKI#c8N+Y#o%;FUg6>DxIjiT4%#vwaG3Bnj(R+s7@Z?YX1ef|9d9ZoC&R|r} z=oTK*zkNJQ?)Y3O{okV$y6-e;n!S@$@`m0K-X32xi@BT09L``auV7BUfj)UTw*1S8 zkMaCae_Ah)WSAX4F*zxGp@TQT^LN4X--G9m@&=dx+G$TPF>49>;Q?qs zcO7Z0SG?+!Z=zgs?E%&!oyU3+zR`vaL9$*{R(ZnZC1;U3{6E7MjHP>tw>@Oav+dJj z%C$a-x7C`mEITc>%;jy;VT5l3W6PMgWxRJ6c@;}6?s%KL3zp{ z4Et5l7~j$Ub|LU@eF)zhaG><=1mMm0nKRA3zE#S+RKbsxe-Pa2VlA9Np7_{Bq~-L# z#n^}A;X!Heps)51EF> ztMB~qCS))4naBbw(hT1*^!I*3@Wr1IuZlS0L!n=jcTHmOFZKzZoW;Bld}+)C%?JPW z{4M-S$6#Yf=Ewc5O@HObnd!0SdoMv(8-RUZYx6PWZa=l1;<`RiBwR^|>Z#+MT&GhgaG4_j^*_)Sj82 zvim*h=6@sp#fisDc9o}=*X6KZr~7XFSW$p;01!inJ+jzqLDo5 zGJ-{}o41a^jrrIxV*SqKuFjG8@$t*!r+q}l>e-6?@=Kreh*bL!n5 z_>RtB>YV41>4C_{z~)|8U%QVsB?J62-+9NWFS}3<{v=uUVz*uu^(4ps5_!*|u{73P z&Q1ci-DOUj-*fAJ-|c79FstcNe)ISpfS0w5C616}s@f63VUIauztL^ibWSxx2q^7yeo6=|61-I?9&^l1|BXg>7bwtVPj`;}Wbx7=KjZ;k$Sf#{O>!|)o> zwC3!2{)@+pcQlCyvG!|kOmG@TTbuFg)Y|=f*DiN9dE)KH9u!Y}Y1ZJiWfnXt-LB!A zamci1kauWTpV0X{FYr0?we$zA2T7D|Lso6@Jv{z7U&>+mi9Y8`JG{2cw{zWg%}=_0 z545v(w`_vx#?NPf;f;=5-bz343wj5;RdaLx*NQ}AVelf9Ur;pH$2tl>Y%ELJ63QZLp5k$mrr zNA~4ZTjP09^+s(wF_wYbCVY83$#z`tcpU$(<-OxcwimeLDR#$mKVuLM8u{Oh2b~aF z*7u$9pgX$bk$!M89^Ob^Yt7^hYrYb`&^)*vi-$M&_9xw*NIw++JzOv~l!0%9_T69N zJkX)tli735u`Rcs^LpW)ZqJIs9UV&FSKpm;-P!05)$dFqPqv;|_?^kKbQsxTgbPj3 ziePu(3%HPMZ|?1@1G|#mz9!o#^yMDO5>4lUH{b{B^X@IgAOL>b@NWqZv-T!jQ915L zVvrQ^y@V@kxEL zKSm(u=$ksPcq7k;=e~%H?m<@Pj*3OZW!$biC>-5N`&rr#kNSVM$gk_u#Gt3z(Gh&s zJu{|eTPyv!)=qqBH^~<2fAZ#CtG(Pe!T)adt^UkU@G|F5C(qb275F9mX>!E>m&wFo zOX%F@EkClx*eGbbHYxZE`UH%d#LJxZNiho?f5*G9g-H*4k2=Q}mC@%?`d)&5@k-Zy^umyW!bfqs%nJ6W`sO}m^0S#%nEd+|Z- z>vhM-cG5A;cw9Lyt^z+;{5`dPG$VKYj34pkN6+Sm42Eo#T-k?*7ywj5#~eV z_$7JHx{_%J-Mj|!v>siA%#*|Kh%ddkLiS|VyOhOQ=y@6TgVa61x+weeZeSN)vbjD5 z-9_^My_6{*{FOb~;G_?ntOlR0n@9KBlS}xH`YhbLi~m~Z?l5VZ{TrU zYw7o}#Z|kCk+G`epN0?45qw}Oyjf-sZK`kE)kTB1U7d8+>sed9S@_MQ+MCZ?l~tkp zw*QcoO{|s5qD?=|@}HHOU4=h=2YX$!Hq~a`w`9q~wM!O1+=+~&{Puh+Qn;yhmHg+u zixXrk-56^tZ9!J>pRko~jM~#`fY*X+#sn?S7kcLR4KCh^E#Xkl=x65^7v;rS0~=~m zkJU1sM)*tki7BDby7TetNj1WPv`Tmot_1(V zgZ6d(L*d)`e|ey}JOA{eFgA4g`01>O-}5N8Y)|ts;_HY{m$`Gkj{ZA30dffc&!Ar# z*OS0qFw}ovk|RGoN}lX7sqTELtPOcW>s~c^-OOp27@${!=U4lXnb`yI-#D&bbE>+# zfs^L+2h>%+TEUM9JVktk6Q8{}qI1z|C;F~5ncJAt48KLiUXSa zL+Dy9zWc|oW&U?BaK`Yy;V(J%e8zaq{vKyt7&(({iX)2e(D&xdX+2^|GKzhm7y_J~7c zQ^?PIzXv_Vp)>!%GM#taB>K#g4He%&WBY7P3l?U2h`;0Dw_+6~(+@v1V9H-g3rA+!|&f8 z6MW-Wj@PukU7j`T_B_k= zeV)#@rt!_Ge7l@JOkoYUI7FWe|2OTrYvRQJfmd>7)1v<~i6uD3jVIUw+;az6`_>tr zlxlyd=koJAOX*i(%vxpSjPicLrLMoqBlJl=jW3XD&Q9jx%!h0g8;jTrNd(`s&ML0I zYCZIZk3;!-t2uDFVj7_X&NTbEo9BX?zoD%F&vVhOf6f00aLIf4VtfPcF}A3`fve@b z7chA2w>MEIw){^j=luZh>%2rr?NOgQOU!q|>^F(Wj4MkI=J}lO7lN;0?o~76%jqAy z{-+K;uH)T8>g4&HwlrQ-r?G$V8n@0W)#3d-@Il}E3D0Ubz#f$H{iLb+$3d%-dv4&} zWAJCmQj&4o_`cRA`G4y@`Ob--U1HwLv}aJi6xgav`jz6`#DPZcS+g@zg zTBi>1TrRmsx-9zZQDg!0tbHDzwc%*8Y>E~(MMods2tM8eEY;?vynFIEF3>k)`k_AN zD=(%W>YLV6Z41+wy+Dnm=o{8Yz zVe;}A|3q-?ApgZf|4J%e@Ww|vQ>T4uox@Bs`m@G=e9>3)g+gLss*N{j<8$~Ie$!@t z|8Ck&vp;md@v=#i?N7wxwu{Flz+V#KGfD89e()Xq>=q5+z9JuYoIB^z4#VqoUQaj~ z%X{*!=d1^^cQ)Tj>uWW2_aSaHXOf%wt>S0%El00Ux0`^s@NqkN;wysH({A2x$D(O16=`)9gmx&9$;SuRWFQZH?{qzq^*U)j#2W zJ^uxx$4r`RSCLBgcFMD{YySH?a>Lwn)$YqImXCh8piiidcD0YuiGNu~L42s({P%M{ z+}fLg9GWr6YSLO?&37b&>i|6M3dR}o4{1*EcynIetTU0sNhA+hxn!L99qg+Qn-6kq>BIW_*ts^nfuNp-qHBl zf9s5ID$mm4Rc8YK%{VuU-m~m0ddG^b${p)?cdWm5$1>K8Rdy$5tY^AoeWrJ;j!jBq zJ&m$jcdxlyW0ZZ0akg-mh~(H4W6ZKwnEqzj+4SkcQ;cnH@7S{J=iR#fsf#@Hc4^C4 z;Diq`bo=Y3=st9o-G_4RK_4};P=@W{`8utkD;-$$<+-J^2Om?Heq0$S&y|1L!GWSv z$UoZaVZD9Fl%FL3yiVNG|1a{7q2sTRf3D=aJwC;5NP{<|!y7W-4VmzUEO-O)8@Bhw zZsF_=AH;q^JSn5Mi%)P+3LSu$RrquIi=^{tzcuSH-+Mc{S3Kl8_G7q9QZmcx74!Cc z&>^mY2TEosHNcmETRiX{&r(aK8RC@2&LF&lxJ@&C#l-ccnWYV-D5VZ0Pqx zWVS^3!+81`lQ;SWMfxIVQCoc;bMJ*kTP?}Xy%$DDPpc-_jv-?lYT?p z|EAyR^!rq8c#yG=FPa0+Tn+Ag3mm!%T$+s>dnIzLW5>`MaH@9$yxU@MurD|`g!htp zKZSNuX)g^NOa}-1;IAAHt~qC@kAZ_aQ$ma$bWz7YAr|L;umGDgXE(KXy82;oj+hM< z%!R?Zcx!VHcFpH62kwKhEl&dBEDb@Kn`W*MGnq>l?yL`4-4Nqw`GA!#jG`KAm6nclkmo z)c*_Z=-Zv9eoE(o{6J)_v+gaAo=rr?+Gte_2(EE`3La-X>Qf8f{~P~hoBXR%_~}a~ zmEXW@;CUr*{U-381)OIxKQox0zDCb*`r~|0F>JzK?$_Xc)%$u%b`_;wQ&+@ZT>j#W zdc~&>bGD{Eo4qnp$NXo(B@FriO<-J6AsA%Wx^~eZ?Mt?iE$IJPC)#SXcC%06+74>TTPGi9@&p6PZ?}=R>T8kK6u>A zsn*KX@r(Aa0d`eBtFwZ1weQaT*Cv%6=_-(pj$Dz3{n(G4l-L#QFVDgr*KBCtGbot% zmx{xFbch7nlwLoSSl}a*99Uk-SQ`dfOf%t!S^!YD}Tsd90GT+AuQ}jubwf_H|@wxm5WwPTe^Bx zWpZHuZNB?gC8h`VzeMcp(7D!_pPySemi4Q-{9Nu+KG$mM;7*H1@XdeloM4Ew=+bk7 zH+dKB@1Xrcd?bI+chP>653b5*GK!P0WIAw+V@v^Tli;nf;r-V*Ih}n%{EEWsI1_Vu zssl6ZfWm*l>;V^MpRR>h#iM6|rz-0P1`ga(d-1Z3JO^%$ZXvw<19=x_IB>h#h1+jk zo~HIU0mClV#ESE7W*tkl8`+Eb63)ImJgEJlp~NycgLD~vQQhyKclJ2(E776wuJuxPVgbXkxXYA;9%wd3zPjrD3;aBA~h?5@)&F9Ftvw_DBaZ|PnP zXyb6dU@|zWen>AqnqsZYEeq_cxTiWR6@OLJhvB0Ae#fAtjn70RF2DD=uvP41(Y5Y9mF{eK1pHaD z>`m0c=Y;QOn)N|AEFGn{9{l_|>dm8`-t*=;`gaPrj|~Q&qLN*W_>&e=M>gVynxBmi zEeSlF2u<1r*v;`{SnKodTvwz0>}30o;H1VVf6I}KbItZ|ZE2HFbju+48S!`E&j!yZ zpKj(f-JWXFY8PU=O#j#-t)R++rWEjH|qP_ zPFo%5RC76(D!x(iwH>PxdM@78Sa5;y1(W~MRK}g>j{A#qmTJ$}oI#hu*;nvkS){wi zkr{gDu6A&+i*n7$a>lQE&K!E|57zK5`pf%DDgQO&R$t$B^C!7*h@I0{8BcWFGrlx# zaNQmE3zW%E-jA=dA7AHoe3!&)r!wB6)caQX&w71T`$gD7*?YQ)eQ9D!?wp$N`l>ID z`=`!)e{tOG9c#YLxU~0}Z2yR{_m27O!OS=N*fI0H+Ke&Bu3}8F{K~OOK=&u{tl2J1 zgbz(F-+GX|F7TiOU-k;_Nyuf+5m5+xkLPRdWOT}>!@$_L1$*nr{M2zTQM@n zG4Zu7f2g7=$Kma_p5X0<|1j26*GK6G>W6rGzW95(T@9>z*F;z5s3h;?HStRFqWot! z@RR*Xyhm^x$A8Iq-QZ@8FCCi;bLrxGIqyB@@}WQRo_LVz%7519aqGJCK7qPhB^OvV zGZMi0Gmr_?mUzQwlvPxTt|#wGfWC{M?~~?x;x{*#GkC!3J!)`2TZYdI?`yLbNLRHGv_V)O1|uSVxMPUs0bldU~f*{FT?HLe}J zneR6Ax0;|c`ET#_v)*f;^=GtO&d=ld*(%ohx3$*i#H{sJvbAz+X<%P%O?B4M*#nAl zob}$JyNRxBdbvLo;e2t+Y~)dNg2U`R9<59%Ixw?e=wM}55$(TIXl>hHXtnGQ-)iLT z<&KPyW6xRU$Ou9DXM9x9Sz4a;1h0MC$*&_{b{=d4^gY8q%k^je5&4pV6q`!AgZyia zOzz4KcataD9mUv`-^TiICjbA3eBr?XXhCbKDT5}o9?dmn*#Bb7;O*B@c9kj1?v)Sv zL5KKA1QU=CWEa#qlde(R6UDw_0FfDQ8P8~?@S!Zy z%cc9*oH)}^NE=~nD94ThN6)QXLO{K%f1$c7!d!V|AXj@Fi$yYne6fV`%;# z*cJW>5BP+>|2sYTy`$`PVtYWpF#Mq0tGkl=VuRpp67NT7OLhsxc9RW#=3K4E(u)r?Ds?C@iMe}7#z^b2E8*_va0&)6eH zYpxyXSh^L$PPTmL14DwhuI8>P>Yb|H#=HB>RNMZclasP<$(i{ibz4}cWP5my`4n$? zl{~Wt#rm?z&D%zvcqz{7k#Bj4_sXwab`SVcCUynA{}gp2@Sa>Nvs33RDmZt42>zF; zeE{0}73C^B1RZ+`lbn?s*gs!$%G?em&CMrX2|Tu1I?m&xu)%ul;k+YRXdZi@ z6Egz)J@8o7+c8kQ)?=^s@(y=~=MLn4M(hAto`jpCdj9$7__^3yUZQQ$*g)p@OYnO! zB{&rLZGsPi&1ztXLwqU+BFD?X_3HpiAPukHcpCSI}n)R1H~$*+uW7#x`7_O&IE zb%b{gv6i&XNtRCN@aLJ<@yI8Mp`)KCg<2P67PVAb$Lsl4dwyx?=o9#QtRr66f?P-c zt-)pheD&SmySAmXX+Hwo#Fw>iFc!YpX8a-8XK?xK8RVVdv(6rZ_-sG&g>S3idE&E@ zExt*1mB_+(s_!|dHmRr60s*;>A7!}@NOPCM;Kk|C^ zO{K1R_T&VS0a6Tpo;f<`0TvB?ow-)?ZAJa&x+7Y-O)NU^87xDeud!h>Vrp<*00F+V5cg7gDNUOs^b_md|)$RKsr z+Y|F&qcY}yaBLa)t@*!OWz3J4wN!HNx!_88fcR|buJ8hXNIGr%N%P!((mZ#9zp~G2 zo-d1?=T}mLbAhYq*}%Uf*Ma|Ir+*Rtdrs4Nt$hLJBWim(3%j6r&h@}U{%E4hJm^nz zAb7XtCl^I-h34iM9CYY#u4@xr;KF@fzmVpjZ9$Ghd$*r}x9shoKJW0aOHP8h^r>E$ zyE;lKc_+hsxL^m&UjWZfhWXc22F#mc%UqhvRheLJIW~k{Jn!&2=WG21J65&->tbx~ zC&6C+3DDTduzw&1_MbDJUK*lh+*d%0uvv zF~C>nN24~et^X$)U_D?Set>>?5Zn=6YhLAR@&5NcE35liO-6@K4Xo&fhI`=OZRnd- zLjwC-NjG83(tPftEn_nU?{(Hv=NY=-rT^_*cF~8f*!g5!ce(TU0c|@xG{^oYa6fqt zcanGV9KJ}N=5Sikj$K zFzdSiDZ9_qO||beX`1~brHr*kDe}aPCO^ae6g_i3zj@L(You=uVvl7odo4rQb4h0J zg|oJcQt^H6gD%?_Ik69Gm*aC0J!k9g884Tv*`xay&>buAA$#c>Vt61YO1_?pPRTtL zvv#0U7ACQmW%e7RdKKrwkTG@U=pW$QKUt3oc&8}R^XTP_sgbs{m%D-XK4Huj=j7Dp zG4z-B`904sTr>Zf_Iu|)6Sn-ECEyMi_7zt!~guKGVPM(LncI3NH^0})UI63-ex_vghLwJ1=d5*0r!!B_1&Nq3S=k;1m>`gXF z7u9**RCHqLMrTn^`(_8wBeZv>vUYTdRD2~RqnzfJ%a>Di&T!z~F^@Bu&l$|?73`gTBg8zLy|Z$oA75RU zihSL;yn;+VZ-JEy} z+tenybZU$)J$#_G@*{NVM(pNUjvl?yGIsL;$L2`}HFonX`;ls=pTDP{(s8%Jk6OAs z!T($D}SmV*Z40Q+JWwyhVFYc`fg-)S?DPEq4w04_T$B;>fH7m zc|F6d>pv#%z+d}>bjGEn=Evjx+S}@k3(5AjJ74lyj(r^b$na&?b$gTRgnP()`@-P2 zhxtB$j_a)H_+>Nh)+(bzFE4GFz`H4S;gcu)#njgKfQj-2ucJ4Vgj|{Yi@5s0cyo6A zo5SuGDbGI+7D+z^? z@O48r`8W&xH9K%5j^8W%zm60?^chxW$&sPjvy^=do>J&RZpI&^7QWJsK9sJ00#l7; zDDW0toJ*d=KQiqcH*c8gjZnP|JKfFun(EcWh2DjCXg|JIDeDosdecgHmfB3A?tyIS znJJw$@Q?%M{C}oBfc##&8*p>ZexR4T`D;(bORTT=?!&)a@7#r`_3tSEwMIsivR-s6 z1z-L~D*LM7pt(Q3=sUpt+ra)B=3owUaW%ZB~-?EOXXgzUjBQ_1>nQ`aC8bzh!<0iPbd*9@DtOvDH07+130?e2=W! z1%J8iNI!7ixvx$8a;XC>Gk?+gsux?{pYk5~Pa<{8m#)l8m|zXr!??GW1oll^TAej< zXmRCB@XvO1yfpaJi*wKF<&6!#gO+%h8nky3Mp=qy?%7)#hTPDTg%e6pszer zT#|TLK78w6I6w5%3u8l{!Xuxr$=LE_S>~2wW0UdA#Xh(-F{Hhl@VcbXu{Ytb#3|_5 z-tTbd3;jc%y)YoO=cv{EDQ64}Plh+`fWCxs0+YrYzI`Q*0Ud`#5C*Q zF}4gGe}%Gfri{JGfWw#f@T`4<9&j*xPwguBBYIKEt^>qAFnrZlJAMayl*SH;FQwwu zIJ$4L{VUNk@DL6r0F%~7T^r;BvQcK)?*l`J2e3Z`tgv@1RQ&i;^39Bi#;+m^J}10A zSQ7}zzsvjw_Oe&%zuFx{TjIw9lme@M<~h|)B<;`p$8P;Kbm@UUz0fH!%`T6FNA^LM z^8|Z*{+*JkB-8yPIVrT;N4#-(q|OU8gG= zEaDb~Z(wbThfe!iO&{=kieF(3=Lwg3_LU=-6=FB=6T86JGD}7V`{DbLG%~Oy@wC9W zgrUS6SKCW(8pmG8+d9|PpF5cvJQM%i-&(d}4{@WnCImM5h+{C^|KN=N4{rG01D^iV zezM_#8T}s*eXo-Lb)KZ+Bw`v&`;&FW+`-mJ#aSu3Gi%6=UTci(rhbe2z_>Rx4FByp zJ+OZb>BAoOPx$TdCfokU=CAPMZ`E>9VrZM>kGqEkrvszUX5*hq9EbV%5l+O`;o}S- zenbmjqA%E<779+Qeb~Rii$uvl!@0K-{jHKQ20VetO^owq#u@GRTu=YX`{{Qg{jT;* zyQ_ouZf5*@va4q>rbO=9DXs)2mDq49Uu7)pabBA+>8_4~)meu=@rJtJ!vC(o3gLIR zap)z!UlTOjiJZL@|Nq@@kE`FEsD=c6>hFY6;Gastg^NV@=m-3%; zj!jqcGjL#C4EM1%f5`6ucb9#`X?sR^4tL8fU|r*EPd;Psoa@98-_9D<{;$cqiiXav z8^7o7UG>;2r+Sj!tRjv=m^pZzc$Yd4RLlSN8umrt@!tT?3#-!WGqAHNe*6OFqJ{oW zBfnxvrrC4aZhU}K?aAcraPvwFoPO&}Yytf;{_a)QzIk;HzGir=WUbB62r=>|LpPJ4 zpEBsE6nZLQo%>y*);Nu&ow2YF-?@NuN*zy34=Gl+kN?rJ6fu_e%dN;T^Z=D}zUR$O zbi*oilpUW_@A5$8^(6`Q?Nc~2L_3A#X-x7>a@O$yb{c&b{v0TC)_&h-$mop4lrvt@ zzrio$f_%^KUW(*|17kUFZtig7{VPTX3iLAnRcws_^~a_ znJLq{J1P*F?v~|sP)7Y;n=xgR+_FG>{MbJ~1}{PeI$iBP8&6^ieBG_nsyg(!g|T1du0tOxkFlu_Gu^!RlxO6rRQobF?;Yh0Gy7t1 zTiftM15f39cmE*AHt`1KVaDFW8N(eTz;nh?s}$IjD`gGwD`madagNn=0KQUt<&;PT z{{#GF|K2G1sv3QFIC#g}@g#80wY>X$JXce`f9X#+g-yD{2EL)$C8#Av&ss2-f( zyCCI^pw-Hm(lvd9-nfPtpS+B(?lAaZ%sZ-^^6h|;(I27T2WDE4(Ziub(P$a*^`>~5 z+lNgJbv%(0YJb`yj+^Dg>wDHZ-c{#mjyyy;azp!@*d_C`igX^d!#$VSvdB7qD4ROD z)Oi@4^+?sOBJR?W&r-*_fuW9dX`%KfxYz!PL7|3cJv*;xbmW1%1YhvDp%7RwKL;6W z8|y#|bKLsq#i8BwKMwoIV(M2HaQ30L2W@J+rCCy z(1T*PE!3LIe=qy>va|F7)@{B%LFGmL7o{H<`9be6muJ%FGQC$f6{Y z^v}o+syo7wD=wgHbkF2n71-dK){Qmx&k*rC3dOf&x5-uf2-l}4-}Oi0eH)(9Rz?g7 z{zv;VMtzaoqV~Zh^(9Z=hJK!k{Wkb?9c7u6?Qp-Vd2CdikX*%hcx1fsDT(>sgM9Bm z{8Yp9_TlUFGJ6rF*PVfEfgQFk(7X*8%TDBncl-O)UL6LfKZb`NhNpai4tI+V<=dTY zugA6|IodDS&yA%+VsgXRb~BE(Zk?W=IkKO=_rs9$-VyRz7C3ac(9JujJa9r|zS+&& zOPGJ7`r@vKSqc8h&JX8q#T|o zeD4C^k!6<~`@$2}altQqTPpU2v{3hL;PUO9r`>89pV@YFq;Nq-NH`zP&kS{~qx=cp z2R{m<`{(ajk+w>mrJBB>y&5YWAd0M;nI)E)7+$x8XVWRfbTNOs#FI4_m3?@c6^+&JIs5|`q29x zbo2^k3r!hz3wN*JEbz4o-|C8c9$Dq@@9Y^gelj++B}y-@KhzfazeRDEetabS0w(gG zCEDq3vZK++(EG@0a@LPJ;OP}t2-l+{oet1XN0S)J6ji7ceePDw>X>QPrYf} zVPHeN!t-zR2FBgUy>dy^t-z*aU$^X?Y0Eb5zyIgE_D@?9*?;r-+%1#0ZGY8b&pv!W zX5E3VBY(H4o`g4w7x$}AxUuqrPWVJZPtTZgPt`j`*Z=&TsrZ;DZ2SHG8}T>M`EJdV zzBk65m+)&pey8vk&+QLazp=kCzIA^)x{{yvPJTCqcT?HNd!*3X`{%a&U?%ZCiA!i} ze#<$ZyBWP<9m zP+1bu7!0hRaPHdhvUZow8x~vvZHVrhX+t)MzX1QdI^DHF%qMuY%ATQ2egh5Y1>7ew z+BeY}BAvE`_T?LA?vO<9o^v;AA!!5kkCEO)U;KREy0av*m^|^7wxd>X9dsBsy?Vy8 zT6?;rSEk#_JCnSp$Wwo{hq%b?+X~)o<6Yr~^%L%)z;5*McJ84_wvhGG?VZr2^tL0} zf=8BJP5q+-t;h@Dlb-M7*}VIOJ1ny8XQS_>JMVpudKckO=8Ru<{8al5`lB(oR&Bep zZPB(nqhoi*oNb?O#++?0;N5(8y#C?XtvpVfrF=s$oUcC7$7@LU@WaNyy*iu|diPwj z_H{kNcffJQITy@g+ZgG#@eSS+51vl?VQd@34-?O9qYdH11XC}Yy{>HgLjJc5g#MK# zL;p&F$!L>uzl}+W&u-FG`y7*EV_h^Ax-Ex(r$EOSL(h}Z6((VC_Xd+6^`!h;eY~b0 z_hZGQe>?Hf6+`eacFX7+Wc$mw;!NO zKZ~yRbAF5XUByo^0aN%jFeYNuG(8C~dVo~#ZKw$}4<|-oc`h;ihzIxw&c;4 zQk930Gw=x*Z9haDL(hO>zbCP{G0qzMbbX+?$Kzcnzq%go5FLtMyKzyVc~d3(_TIt8 z#8FsSi=MYWza;bqF&=4a2tr-cd;He(L!WCBo$M<+z3B5KPxEKb4TMd&A^7fk zc>G527}9zMSOsys76#?$%E74mSVG`QJ(3 z>uBr7q4+KvJu)MgoI)Gb<*xl{OV;c9=#i9Dbp*vQE zk-s|d(^9MxKX$MM+>PFj?2t=bCZGNC*yqY3cD7pQ5Y=9a=JJJ9P_*$?vq2YF~7U zy3(n|D@6a%e&{`YqwG`4ZVyC;Q`R zjPLR7*yh3i)`{pu!yKLBCvKe%)fq;e-7$5*)7@i{TirUXs>665_i+v++Rng>LwCF7 ze^EI)!Q%^J%4gHY6z-{T>%6Tx(5&Fzh)nc&K6T*{rcdDWA@El|jyJgVUr~K%zvO>}e(L

    ^%Ij$Iu?gT6gmC`nGaGMmj zsZlB8sZdIP3zhPX@Oh++v5{1G+V@wE&T_x{XmmBsq_}xMQXc$6ba(8RPQ7a7!52hh z-EQ6@4>}Cyc?8Cy8&?u#P^&I&v?1@N3rbOZn!WH?7E?0_wa)I+k>77JKm? z_ToLw!|2ziQ*uK4rkob~Y`3Sm<8x2*Q2O#2x^xHYLkIeFcoFuYF5pl_TBmh{vKz>+ z!o+zVXAXPFbpV^RJ2&&p%pF`9QKp_E)sCOmx*1_}4 zyR#WLc~>aU=jOq`mXUV}dEy%%xNVvG(C_1vO*UmI_O946X!;e(zD}9e#71{beB)Go zf$l%1?kLs0)vZgq^;C7^frpRtcjlY;_T@Qyh|({@b4#$rUJ)u{@35%Pv^w?%>f~3M z$2rIFI`~w9Cur?^Xndjki8o_ooQ|JgJ@zMTo4dJFS7TZMPwN8zHMjD!Jb)dpbxuk1 zwa`-O`@-{Ndpk5Eoj~;mKp$1+Y*Y1+G;AoModiQ;tp7^sZy%+M(Wex+c$5OWo`*@9 zhvP~|u>Y@=`Hm0QywQ9JVXHC=mHXDRl4; zrNdZ9l|nzQN-K$_w=bqugJKMSlXR?J(` z&N^1)39Mi~?Wuj(gQQL^aX>oP$AzrZ8ZR5nIIRBoD--YhP5hCo`UXslfMQ_2YHxaHjY-ow%ShXS`>@w<+jXBe?aeA=%0|vhOdZVqw12yUy!1DXd`;e5 zW{M6&3iz3xms(p>^t1NdzEL_ z~r?pa_k#<)>^A` zUu|X&4qrEDU+`Bf3ep)lq>?MXqq@K++3e#bn==&Z-&JbUO72`|-p{aSxzAqgLoGgV zk$zmRzC#a}nsKpDrIh|oRSKR>Q3{?9SBh-XLtL2Si!Me_=dRn?lhD)4(9=uN)42yI z|FxTigV9*(M(%{)ID1`b_HO3#Z?4VvLfVtw>d2GPI_b82Ruxlmms{^#>dA&<&uh{G48 z6WF(pNGZ};vm)NjdlkMk0=~pp+PU|rwXW}m-4m-P5ag=4W1y!w4%xu0>(}^$Tb4R& z_n{T+Q80#TXz1Wv;@=?iw%q0ozPiRK|BP~tv4!(;jl0>WPjX^a{21Rd;qN~w7d&+) zVn6(EtuJn; z`hTV@!yfj9GDjA8guJ!896u(>8bvBr>}Hdu+c%js&8{Hz%D(OU@8UG+4j=Jyov{=z*Bw5x z|Hbn0C$C_C6h6+~gfsBTZ2FMjulYU3?+5(0Fy|MLj^^97>{rylH*-DZ`^rc|#A*p) zgED*=KAP(pw68`nVJ2G5him#UmJE9oun|6%^1l3B#GlS_^Cpnj4G-b|KeNV!(O(sN z@oR4R7|P{ec%;DLuX%3%2=XPrcIPWU-Oh6Ji^$hF4v)1WTd>2)p8j*}uXpmhnqQu5 zvEs|jaVqbKzm{PKSD&=rtU|{Z~@+F5~!5v6D2OF!O(64>HOpp=5Ur2T7<912t*<_)p86r5BIqFN5IVz-ENiUn4u&`V zZ$dEW`ux5{ees@cCe~f5{W5TH-i_NQzr=Tu6Kc<3Z$HCo@)Nfs+1x>!Z3kFyz=05U z|Ba>dq)Q~*4?~y2k6+WK*4t{{Gxw8ISNToki{Jl0yuEpRRMq|eeC7wR_b6rGVhzxT2Dcu?vrMyFsZ0L+(AP!rv8e;ZHGQ2rOMQLACHO^Jzt*?2 z?4MDG;U{0;befkLHvhkKTp-j2V; zA8c!rnF%f88f=4F+!Ip{-7YwA>dCus4i z!Z7oO9YPemuDhdlY?3{9x;>vcKM1bVc!)n~M)$Psd4tEd1kX;P4EK~UUoKCWZgZk; z#x5o&n7$$_v@KFPz5+nqgQVwHla^s05M-?Ong5yg9+fdMIpz1v{Q$LAxV}$Cq%Z9y zzEDFHdotufV(|yD1NQE5tL;pTjWlBFr^9Duz;9*3cV)qUWplon!}(@sbB~+uPm=$y z?uJ#YKKUcZ$Ld=a$`L0?LyYc1+16zSvS96{hSA% zJ{~@O9QO=#Hg@Ks<)I8$rt$;hZOohK(;x64rJo(z?S-C-(c9pG5;iP}H^{EhFv zWX2U7`;PS7(B45tH|^zX|L$df=zLRmNSr*zT?416t>8gdR&eVQJ&2X-=au^j<)W<3 zCg!w?v5fdxbG)Z-#eZin{Di)xcqcli-MG@lxyR}|NosF~-HYdMe=o~^&MTWnzS;MD zFG=4`(s$Er&A~qEk7in{U+1UqH$8>y1Md(1as4Wh0jfAxRt$g171EFH!md1e8NSS0 zu`T~J=DX9?_1)>yjhI*6jqgqhK88p8?nq{6LS|T|`tjX4jyCwgfg=(ub1rMXiyXpQ zT@;vqO1LS@3K#C~e&}`nm)>@Kco*wawyTO&Hg|-TwsM9$9v|K9_OV=a2sz{NKf3Af zk}f*5K|bot>u)_fq&G?iSlDwo5){nj42xVQ1=8Ul}`xf$*4{j-4L zMGND<0S=f)nT4di$Jng7^#o!|au+%KM>I_Qh2q+~?RU%8#LD(MA*^!i%pT9PQ@naR zbaC{7eYukQ#oz1OKCg^hU#|TMdE)OM!3I!il2dl_lYF563FR)2lLrkHZV6yhl8O#Y zv`jpRWRm$K+!$ZG9_t*5KE>IeA9}Vz^eoGMPwSicQrwACm@~;F`Y(PWurwAsC(9nk zJg$$$&LMvc|Bd_)l)TzV?3S}kTBcnES+`i>vEs+IRZVW&aD>*W%gWOZM4EyL4}0!;Xr` zI`*Duo?;(VfUh-<^~F}hI_&v{TXuq%KBNEgm6T89kKxK=8HX7Ctn*t#6eno!0LACi zofx{iQg^!PUb1ZHuHxL}8C{y>mH_t~jvkHN1%2FR(v`1rDzEqSzUm={*8tA?#P{=_ z)_;|ESKx8bCyU=U_CU@zU0+oa8Wc&Q)v?=HUU}Z-q`3SX_H5x(z=TATJl%t#;f^zVeDC_tA!h-hZPr zGvo@p0 zT#;#UPh@)YWZ=~QDCX+8g?;hwYNDMJpuMjrq`xr{8~$o&^F+lT0m|2)gYLO{2mXeA z6+CGyrYQF!j2F+g2U-d({-$*9k#R*Q`rWvqdHdY@%c);-quN5vuL`%l1jp3J3Vj={Iw z4+V7Tn8r8Ho@LdtUlJlJf3$cpS5;l{JYS=mE`SE9_wy;d>(8274kkI z&7~ii?eybw(lq7)c!Cn?vlw5UnIMb0@%X&BBu)N|zPl1S;_=DH{=g{9VIJCJk{lSO zbCg(2l1#gtwXgA$o@)hT=;^t(1aIyN%}KMbdDyj^cJGJDv|qf-cVCaA=(O1S(L=z5RWDS;GH{iKKW7 z6ZIbaO=hC?HhPF}vi+Rj^BvJkAM*SDJAL=Pk9}u3cyM#)4MVf=^`ZVDfnzwINF~1E zY2^);uU9XXv9RTQ09?F{ccgD?x2ITKF z-@O+|F_=_d-~N!ip~Ujkw@dsRhGu@xw^c^pZo>yOdeZ%i+rSkY`j*d}(aVj$_g(%q zjE~{%&7Q^A>Sg4OQs%jITla9uh1EA&_)!W^7Zh5%>sTj&xl40nX~+dXDsRSQO~uXI z@5!2?xi7GPzl-L+U}$b8G?)7{pmG0Vjz0(Y{unK(IAmz9@inf4=E|0DA2O=qZtUg# zLxz5ud!s#?tGGZu=HI2c)6b64T&2fouF8w1$q^)@qCnzwD*Gt#@nvkqM%&;G}>cIBYX=x6ZWc1i?4C z_`8VKYX6ItZc^Llq?49sKX1~~UHZ!Rp7s74`U(zw%KQJaOJCc4;2Gmp)+b%VMZ}bm z%rlj?Xgs7lm#n+ov={zBkhVQ62%cOb$oiTKd>UNQ3NKJ+;xt^mFa^5WnucA< zL%u}rrHslQ>y?{DIrSwi&05{qi{BV4a){W3tS2|-L!1n0XyOO%9qN)FV>D5CJ}TJ} z`iI=u9`j)w^>G(5Iw-~st!bf*q{F*g3pN^8M(wAG`j;ID72 zx>LeASG)9d4DB`avt7(b++?Rk(ON zc(XUpy_Fu57s1t)q!kz%lw&^$?QWMBndd3wWt+S_`_cG3*1|mUdMc0gwlqGEwY!qM z6y>q@9*oapO}#>%pS+`Gv~K1*9WvT8{5N(_o{w{MS|G9qTDtjXu{evJBf3vAUOu~$ zbuQh5*3A}duRL11D)?XzXh?>=lrfbLlKh%ACQS=n-6}Zg$URZnJv^8A+rwDfT0^oQ z5mX(Y@Q&90M}lL@BOiMI-v^!~y17a5X?mgu>V-Zi4ZTo0`XOS|%;LU<+D_aBl!VOk z{dX-O|Fp+_$Q?y&8szijd?@_*`HIZlZc6s(wtOenrzmP7x9J^y7;-#-O z>4}ka?~a8u`qIzsleuGI&m8jN@Q%4-0XaZ-cYf^2G`uuovB9 zU{3B5>FT846P~)ql+S2Z+>Do9-P@64JgBFMy`eobiuS2Zif=AnzLx*O$#V_NurD)E z^5ZWU%kvo1@r>;_#&|5cwsX-AkDPx1Wq;7;MCN#*};d(CS2%J;j) zW2kRwt+l)6#NRDG(C3)aE``_%op@L7uKiZSwI|-4yKmr~i#eNFR62cB!+vn~RG&4t zz#rJ1T*DeqJF~S({@raOcJF#%X)g90m#6o!mom4%4zj#lJ)o=+o?91PJ{J%Fo zjp*oPyYP~y*E@!E(L(75OWFI|i*-iPVF#B^ntPstjTpGGH*IcX-{vu%qGfmTEFAPF z*5|kOWmT{6wj}gF?6cTb`~`vQ-q>3c7-hhS)lX+a8F? zS5e}kV58ZHxUPJ&(^My1JjOi>KE5t(nML+C&^ogOg(M4ts!)lJT49VGXW^mIelAgr|z{4X20C^3VFW z;;^4d?89B0vzYXzv?K0t7>B+?xawZ|#vB+~xI|-F+_SWi*o4tB$&r?EapMV{*1nOh zS7R65ni|=I?0SIs!bUE=%5skN#%nfpx->DP=uf?vdDpd%75m*=dy`0$Pt!rI zW5((W@Rru!YSy2|Y9DL+M4qL))H;$LwUYF)yw_lGV~+i)_fChdl5*g&DU_?C+;gTJ zX?(}%KuN=&tZy>$-SJm>yCfOEhGF!X@~$58j^R_Gb1C43U41%5u%G76KVo@=q=(G4 zpA`vjvjh%6Iajhl7*RJ=!KczJicYM z$8Yahy6^wv+)AgWe*VeRxpZou<9xd(ib0Eg)0y9j zLEABx$?Ty*#i9MyTxQyvw1&<8b?<&ZzN5d3f{V0&iBtYyZ2!7C@1LoE;J@MOGkxDA zUNa=0Tt9qJBK%M%_#$}XF-h>6_~oKw67ATw-Ws}noi!A^xT^8;3TN9wD?&Q-MECHO zg#*l=@bLiZ75!O z&dD)jmSg{iZ>=@FZ;pL{XY-zXK;db2OxAnZ_DQ@WKEyHWBgfuNxl`dy-M&dbn`TS4 z5kW8JJ_QGcdz$FfiSI-oP|_ zje(i)$yKa_k*tR^SQn?WJ}TjpLlM?XcyXOC{l{Zy{%~wMjd#pudG|KT806n7VquJfWB)JEp_V^4OeVJtMpcayF)rugL=`rzE54Bx76WO{|n*NqMK|y4PazWO@2?k0}^~3QpT^T67c{1{k z;@cH7?uu=<1HO0xa?p4$rk!*zRpOP=y##>fyyLs#*|E=G_jLYhgXGSbUtKJoozB&y zds*+5%cq?BBwt?jt1yc(L6@u8EW3X1^3-XR5q=V1O`Xo6iQObWx$!b|zXb0$aaW9V z=q>PshgKMV{h6jv$^r4-;!Tes1ZY6YFx#ny$FQ`0;N;w`TrloZTRM@NdoEyIDpK zSp*JN|K_^uZtTs#W#9!Xe?a&TcNiV^?kF@k!hR{O=oc>O-l%r`V2d~>yU zSkb|YJpI=?@-)8EpS=bSa_`CLkQ<&MEhaZ?htBopn-43^#O;6%Cb@j>gQUgw$?-gk zUdHqxr_o=RFQz)WxDq@4NRFE`w*|Ky7|mY(bItO>OHLfU@;{Wzcg`~q5X9y%A-wuy4RjGua5d4rzk z;DcU8jEfh!pDO=9ZG$3ZUVwkm!?_Mef-}wz;fif(lo?!nlD4nr|r!{;Mv&! zD$~{XP%eI^4bUIsBQM|h`O-xjzm}Zl6$TQM4S2;ZGqx^B$R>1dN^=yPxga6OJVV3A zFPM{>SB39+0e&w5{(o79UYEOJzkrU#Y@aV=&4@1jlQtN=KeEPnFYQC6^`R_ye2ka2 zRcXaa%d*e%(%w`WbW7{#bT937(lo9*8>kl@gFmeXHX+BA3bLP?*#8^&8QZG?$al=K z?6Cc!J&Z@%EpBYqCzXz`N-^t2ZHajQSAg$Su8FxKZc1L-b;piVyOziDnG3b)VefyC z_thq?`v*;%@vAodDR=G@q>c9o(&j~i%b+SbG!8YWHl%C)~6Zl1AMPUg(9 z2s?buB6N%aI@>LN*Z${nzCQ`nst2} zQ1nN2$QDd=e~*3h3VGW}b8YFUi#4#5w11lNS@yK} z^6W3!NWX9LxZ5&5kM%EG>9s-*_HUR_+H=DM?k~x$ z;STvj#Q}Ud<_`}@*CHQF?5Cy6aBZh^>{Hom8e`pc{V?Oa?J;9Fy(n0dXK2_J_s8tb za_r}CbM4J$biNn=mrVOv{)^_CxndtY#eexzJ}JnUJ}$_3M}XbghcQ~S8Jp$9v(_U! zTH%A0%U*sL=NVtghO-!p~yo@K-(O(ibr2w&>7`Na23CcbC6@64GdE@)Bt z&}QO$p5;qTA&z8U&iRK1ICl>a-*W`?Dk1swnT>sx=IT842V>2I@W+8h>R##=@BCZz zQSq@o`nW|hNIh*94SkFD%GRtV&l7?~!&rXJI!7w_G(YGx7X z_1x!G0L|J+Ti0E=csBIR?=v)PGxkIa{5{QcwR!G=zh>2Lzj^L%o>vh6c^Y)$a_C&0 zzZ+@D1&XN;&D)F)_zQTbZ6ke=U7>`?*TejgFVQ)eSPBz~@6Y}h9ynI(YMkg7_M2W> zj?%^`E!*zorKKy4vpmrl%S-F7w2?|f7t6EiOHo=#X=(O8FRhc(Mv!(9{=HtBr8M-X zjhB@>UwCN;=aB|q(0D%nyYxo&jOW!}FZsSu*{2=xFUH(T0ux5ylQf$%`8zW%9sdC(}H5r<$e*ew< z@3DJ_p~ac@U%mg6&Hps}W!1wv$!VHJ+*o)OXtetLqE}C`{?p&J`X4CYZO;1X>uT@+ zt$bU36^(w%j0O8&kiI-FNZ%rY^z|`8`u-^Gm?MR`Zi*-v<>lMrYoK*F#y?b4*NrQYYafgr+L3q--EYv`2w#T`$uI? z@XFMx3^JU{6L@9V3x52Gj`7OORGD_3z$=sImPz-@Ojnsa>hTjN1)j*Xseo^>mr}hl z)hYvD=K9gPWp<=iM*Lp>B;~{3nYG2-*MU#OS5yhorhp*d-3&gF9->aL5Bb6`bGXZj zy}ygOj+7OZ8eT&Fv;Llkc-~S7-?6MVEFNP^A^e755xmB-+IxpDuf1=$c#cAN4tO^w z^4Lj{WwrMYw`()k%4Q)F%35pc!*+eyrp!ZC58{&usEnIzI&S(Wv|_)a11-?ztj2htep zd8B=*G!yHBG{$`;X;D*7HuI!02d|K}gEaa3PY1eu0eP(Ro#cI@JofSB@p)rh9^gHb zmu*ju&l~0P0ETyGUtb)b$DXQ$KX}c&mua6DpU1wNLf!__Tzd@Kz}lEc+B&5f|6tNs zCo4&N&eXx((D8MQpg-h2rM!#Eozvp;hS4AL>XmmH@!{h0hR`3zd%5z?FLwsS=Oxh} z@)j$PF-s_PZRSTa2f{}I?BxT#o1Rg;`90ht*66=mJd7JBVhB9U3P0ymelw@Z;3BOF z?WrqR7u%958iK5?;3xUy6UXY_n4Fz&-(Ks=*fAg8Tlug3SR=?7%ob#PZWd%7X9_aM zHwrTU*9)>Xt_5x~_Vk_J{(4x~E597r?aJn|?pHPy_rSmF`|UC|=RZ0+lK-dC3-VWv zo|eC2^x653jP9Smd~`SL%l3J)y6&LV9kzS06A&*V-(T$s_$R}QRKtrr8N_CTyMNHb zjh&fnt-hGwFn-U{)|P;^dJSzqnKsrZWH%QR*K$3!hC4Wqj$ng4!dJMvmgfRrV0Q?3 zauWC?A{(p__Um3}oOt^XGT();h~lhGgf>ZLYht48j! zeJbqM@uB*N-FG8;&KaJLbSr6+X{De0lzrmr=V%|c?up0{;t#a`L+laZ)7L2P+6RzE zp72&H?@A9U-b%W$-aJ2H-b=Hep}ZTLMtSb}*Pl#YKEC2^ohswjNg8>6D{En_t|`>@ z2d}QBloL)Y^xj+9{vK)Oy`1*<=JDP_@4c|vP``z=?{7 z5Q_t<{SuE8t1CDoBCeqPmKFE26wL))ckmtNC3b3*j?Rqd{1Z3MVzbGh&PM#QOw+fUDNRLdyqMIZzKk% z8F%j-!_zTs37}(=FTgUL)p9m*&0JT<uU+g2rdQLDl!YiQhBZx&>i)_0RS~(&ivALM%QFDCbre#b>opw{fxL0RCzNBA5 z;v1hK^R2hKG?z%;C9RA&rC%WPEhYYF9kOkg*kygtVk#C^iZVvG5%E=k_H+TCY z+Xnh0J4bRq8nk@``_j<$G02t4nMQ9d-fSr8P1E25*l&{83X+N4gnoKIe8CXodzEMZ zH11vQN+H%~R`dQ;>?1kzlOC-kF24z!AiBK*zC`?FQC!}coR>C1zf3;7NtCi#oF&Rm zGb^r~>R-VGY)X;YMaQKvH=U?T5mFX|jq-!QR2 z>6;&(B|c_oE9^sV+ z{kMWI{#s+=ar8O>Z~5JEMzi4wQ{hF3Gs-%;8U38-_CI(}dVW_<)m@U<(xf3@4k7-h zS7wXKz~i~PYp+ZeaZT6@@~@uil~Wr`EKwt?Qm#+ABbi2JCV6FEQW^FDI$mcpc-EC! zX&Ytr?H#FAk@LK=H3z8eyc_WIQ)ul_Rc-8%BcE7bsFaYz+U zH2Nw2RXdU*u~?*vi5DH05^=8T77@OUf>#0qk+m7`%_jkQr%sUXRtb(IUywd;J`qU2 z>je3J6>!(4fE!mc)G0g)|Ay9q2@U&(Bs74B7IuMd*C3})!{4E?z<2m;&XT_5S z*Z18^?9QbL4Xv9vH+YS>n4epRzZh^#WE%JKG!Hm7(%M4&$~Ip^f9_lR%FlaA{D`Ib z4KX!aig|ZHLPIL;y2OjqSw!5d)^Wau=4r&dJD>WVvJRic_jf&uuU~g~hv$K;v0Y_Z z5yj(dyIXbp4}2BMiF^~vjWj>&YxsJ-8)p+kZ|-X=Z`d^w-N7hy2T*eHkCdZ-IS+m>Ex}2?eIxG;f=O4M_RivANfPVIex_-bklm%-~CEs z4r6JRq>1-8ceSV-^6nJUzEs*K=^os)d89>^_N<#$%m0=9c978TKo@cc*zT!@kG7muIJW?>*4|9%)Ctx0Cn8->WV2 z%zHU@C-1%a?eCFx)O*rlT^<%aK~of-R{`T^xnnR+0d4~;CHoG{)yt@?)Kn9 zpdWf;#-BDw|1rUFfAeM@xqRqs2ocCMtn=)1=4WVI9gr*-h0_wK2r?eW&e zb4R?JZok62nm^UsN9E4fyXp30-n(V0k9SvFj2(H8erF%k?48up-PA+u4%IWxt>=EP zo}Bo4$h+@*>QSuH!{n*`e%?_ZV{+3|)E^yC(GbHWyB<3`!aX1hkBBL^P&5xePU4mZ z#OLZvjd+0QL7nZPM9$Tulj1H~@J9>yMBkB2^;hgHx0M;a)Rtgj0cE&{0~{2SsTyay zdZ_^PBujBidG}+|B%A+3ka_z+kU4!%kiEK9kbV5NAbb2RLH7UO1=Gr%Hw24yry6h! z`g!1X{4jsaF7F_EDBT%*0GUhoTPyyM^icA{)%h#D|Eic>Uh2=#Lm_u$u+9&ncQSsH zJ~viDHTN&jpL+tyA1_v{KKZ;S+*{Q;6+TDhhp~?DQrqA?Kk-?swby^ooQR+K6X{x0 z7n<}Ody$vE)TDENBeo=YypR3YJ2`D_YsAN)FEe+>VyCjY3O-W$tn^#*mVBbMMcsE$ zN6enC3-9V3{V#?;3(Uzo!msFNQx+c|=+?`~f}4?tWN+#BVJjzDdSyYoZQWqnr2aTt zgAwTtuT-B{zQKF~1_&jx;fs#l$(7!&y%W|G&$pXg_o0czT0h z@IB$&Wht&tQJ=e9JarD~lSrF`Kha$HcdfnF$=yp?$8NlvkY#)zzT$h)b%DtKPm?39 zk9+oTn^KPUBU*Il!~XEkJwKv}_z^|drA7ka*Z@96Ph#ivxcpd1IS30p)enfXImJgBamYmdW}`35 zx5fJjXJZeP{;j!QTWIX?ZbM%Yo97h#T4K+gd7g-Gkj{xVBGb$%#upnn5&xh|kMIx5 zvw!zZhksBB=bdWv>#VKPx!>;}wC*!^3>t6o-B*LXL`!#UD~|FJ(ma-UK0@*HDW0@x zg*TU1Wt0+&|8NsAjP`8p5;<^Hp6lB(!*awMNmnZxKgAo7%55LhL~8w}IZ6Vy~M1qw9>efkW|str=U!+Q3ikzgl1FM{IqmZ%3>z`WL^xX0X1_ zi>V=-A{0Z!Cd zd;>nw`U)`?X5D0!J4520XYlOObk=8{Yojub_hUAWjo$k)+VAkb*0;`brZ8SrlvQ8N z`K;2Jch%QQdcdCn4YS^k;6H(F$Vb?QESBF#3GcanK(`N{-|h#59b*~&tc}?*rrA%k zx5V4Y4mX`KkxvM^!BxSvMeH$Ty*pzzBCbDvf5uCC+%n)E`l5EqhBLn3NB1>5uCL<# zG*;i&*RDtRl{Top_21Xmt2(>BFX(ZI7weRs7H6}0R9~;BuX~t#;cxZ#L+ps#`+LE0 z^cUVGuD}1jIW7hhWBwKYO`oZwnP=&9n}9Qs=hep8c*Jz&AJNxCalRtI`M$oxdug8E zBtPm$p7i`hmJ%;SI=j-0`HMBL$Xg9lzvVB|(SOci7a4z%G<<)K>idwmd0rgfZ%h3~ zKM>dN)uwN`_A>@%*{}YHE;q(QKtSOu3=gp=#zV9Y%UXgB|c3QGaIN2`0V) zaWZ*N>8xYX_I96PX!|N_*>T|_{7B?8{4?rRyTtq6z_aH6Is?Be?D!A8oKEJp5QSUTga`;Yk#}#dD=9}1TEi9<;?VgJbuW1wC*LU+Mn@7Db z{OEd0g#Np6F=fl_@ulr+>esBP!p_#9eCo{M?6Q?L#o8M5Jip)b`+8PvUFF&i{nNS% z0Nr(!WAE|O{6MX%cNx1i{GzkhF3B@G&s@8OyaFq)eQnM1VrP8mrGMwzT2vff?yarI zXoK3PwbjV8*4F*`9;y?hftp~Gh=zZo8E{QG8oWZI|4%6up8BMu|Jo5Fox{(6A!raj*DviS=4@f2HKx(;f3lSr9IZ2uJ*K4)OC>eZihfSxoy8i5E-+riCg%I@ z8p^WQdbY#aq-m|IY(iwDQ_OnbzC-7m#0BW(o_BX6jXOTgTr7kJh|f^I=y7lQypJ@^ zljuWyUL|>7dwI7Ul_$C!CG9!R6*ZUFdHKgNZ;xK?;^5tk<+8nPtHoa%eCOd{?!)~~ z9IW;FIyf-aHt`N>-y~w_e5V#bJdA|SWJ9_4j$4!clh>N zE4-uF9q-<(F}3GO`aQwL4T}Or17+Lm1h!S;*PTpkt*X{ZtV@-v^2+~_@`kR}AJ4qG zcjb;+;KsR8x!;rjDd)~^oU$zY9{ykO|A02d1lyx?ep-6 zXx8)ae(|{E`Ju$QBKAQWwz~7|@|pAU%4aH0RU$vdu&M-ZKnKuvf;D{;aA`rg@m*P( zRzAH#|1ZU71!&e7zB^IkJ}-m5%V&o(?Je@z`7>jNP5&D6-JZW4T9^FOlWTO|JMYAB zC-SUhlea36QQ_1hzw{=}rHi@tPuOFkm8X#=JS85h)Jr>sG-C_P{^{+d4OBVq6JQ~l z^N?!6ft+ay`t$on+5hGaz+&Pyn{z(we|UFqAI<=Yv9d#FXJ-sqsJKh%*r@7z8RScE zqZpgw>-HF45C5*X^aOD5xA?l9$G69L+}7Pv_`i7PbIShMohvC{ zDYnm#-^)^+0N;U4xXyJ4vo|Be11#k(O`Y+UC>>i;%KpR+gA}!*`*7(ER;f z{leB%_DEVQ7IR(18Ys$62>1KDTjmv&kt~mXZoT|xcEo&Mq(k9;TV%V7y>pZXl@Z8%~H~31Ka4YIc-1ZnewZM zCa`bbMh83$9q=sVX8BgsN(YSppVk!fWBmVUv+%6qkH~K@%Gfm(CmTDA05X{+AD~za zlJisXMa4D)U3GL~NBj}hb=cGK{gS@Ba#5yzuEw5qwhvu^bX&is?QXmZblhG{l5gynZGx+kPQFkFlFa-ti_c ztvz-x{5vapQh@{F%(yDGmibDH@w-i1B0WT;CM#muMe9`bNK(hc1xb zVNq~xNmyg?D(g$~&R*8I`rlpSKzbQ_l)Z2K&L(aT zdgDPy@7tZ!*NJ(E?cd|RbafrFbCEtn>q+A-c`N?E##sJV@o7i@H}i0#<}TlE6l5L$ zulb*8e**jnoxurw!|Itva3!WI$SQ(dhJ#vmqO51hv=g=m#Lw2U~d2UQ1{;wc((P9jb8ZiDwuHuL(ey*0eA(T_jE-qM(l^wvNKs5ZpDA%3$r`J(Sf ze@DD~2d|oDbX{qrxiO76ANIZ<(+R3x`E&^1B%3yJ_n?QXU*dm^r%HpbXVE(#7WW4v?h zWz3a}mvimr8{9KJ#m7`x$ToW7XzYeAb@q>0a#VH1>2ldpd(XO$@(TS;(I5 zo)!-oe;-I}U&nN5@%y^Z5cV~8;lev>u?PPHznl48#P3wntCFl<(SSAhDYXUuUAU)# z^`rg0nKYND%d=N|X`56|be;A;<)!^aX^fH9h3%y^C@s~E=b9nj4*a)SdmdU>=k5I} z;CbwQeRC!6nfo)~@5FB{^Zx5BPTzVcmT%;PZ2Lj)e-QhuZ>_(1WoG?JW*tjA)OW2p zbMKn=ziBV~Uyyd)CdfS08klBJ1!`UCJ;}OAq}9g9qRE;6m_ zkFHpwe$H>!g9|oeo7ce~NpFnpmErp`>~wT;8r$i7LpZg(J7coK7ak{kFs5ti-t&_p zd!c*Lh1N?({@qkJym8a~;o>!x;rq};TnTILZSX_$_}#(p_ThWsGaB#pg-^WQt$PA> zo4y)-SPL)-awJYG19G|h+jVc`Vv+47e$`-^^Pzn*By0+gS_Rug93YY+LlBJ&$=>NnWPO>)6YG z&NEiHd-xXC(jM@FXo2|n zPZ_Um_$nLT;_Cj4PrChQ#x}N(o$J+?bYiKyyv81QEXAh$_B!krt{`sUuh`r1Yf$#l z`u{ogXkDtVojhMC9Iz=79MApTE9btS@LQK+4SFuGyy1FaeE~L9>|#bSmko;dN1vHr6E@cH>~i|o>3Y&crSQ_ zI@~)n1>u{j1>uEDfY*7r;bGnr-q;I1Sp$C{JmLC)ApesW!yE5;I{t91JoqeDeualO z7I@{4`!2k(vVVA^r|YVujhdrrX7JvW*4o2y^%-9_@&c-hIK=wCQDavZy z>1;GAx$Zq=liJ%2->?*%Q}?|%Cmoz~-(t>Gb_Z@-x|qAa;&9Jx=*7T2#lpL3_ODov zu1-D8-pE*P!%xBBot*>7&$AaX?wVJX{rEcHeM>!@)APIV&4!Z?aTZZKH2~b3Lk}%@MlZI4=Nphs0{qsGQmyQZC;km zy3S#J=d#Z8z)ku1?7F+%u;8vtOXC6QE_o_~r0B1peB}Z!I`Y_jcji)^Hxr__i6GMI4pE zd%!QNhm`X_AGsE@ydZ3sjCd*Z63EtK4l$ z8{p!$>0Vlm(x5@2e^+>EvrK)75$V>Y*Qyg7fL*U(S-Jdf+T*Ne8vK@K|C)CTtV|E5 z<=dBd|5w2K>;oS^XYg8{eUX|#$66SjF1ej1;`Q`G?<=bef8Q55rBUyqh5cVe-FRl+|tldQ%hP~sm@z={K74Okh zoLlP^ijV2PcWZhn_@yBVPuEzR5Z+kpwsqIFuH892^=i@0d7e&fC+Wfk zYWFwbjYmTMx3yPe`1NDb2J@Zmn}l~sXWo1B|2@*T6H`TLwb)}-lD5U*oiuwi(8a6h zHa)y5-rcniC5`c%NBPyJe7+rwFV9+8NuKDx_R6sMJl4}Ix)NS%;`?jmYeeU z-N% zb0dCmpLZU*86DFR`w)HmHgNN?;MCY2)O{+kXW1Xdo)sfD_N?#xlDhXZ57J#s;9300 z1%lx0UjVxbPyd(ambwdGeB#~?9nhB8x#dFx+a8#d<@ykxoMo-f;CX+hHRwZr0p!HG zOl$D*d@rVF9KiSpH~x$?!+UV9gKubGokMa5R^O^Q>z3r-l7p{KrrnRS1%do#`QCnO zZ0ZNL%WMCdp?;;?g{FULc4wgU?DsP!CD?H0u;&h3mQ>I<^6sJUX6?1RjWXpl(Y{}ppF$0~9WW7#&Pd3_m ze$><3s134Fd5E&NK0eav$vSk;5wX|z&JhoR)6`G#hwgsD-kN^&rmoMtyctL3Rg(9= zUY_Vj{CiW#`@qW+eTdJSN8T2bhpn=wx46ON{GeC=dRYR<@oaO*$ zf$h8xJa~uP6@Mm}Z5Md^tDZJpz}nGV#GeVShYz;e&jhn=L!a0Wf1teChxLQlhwi-# zXMj6hI-YHZ$=8{n_M-Cd;=k6^LIcz6=?039{rFk6_Tg^&9{}H&u{p}tg?KIB{NLER zm^om)CEq7wkD&2R_Hj?Mx2L}xmU8o z%!y}IO$>pDPC{3NEsS(_H$i(Qbhieb!Y`BGQh(;O&G-_&*p2(~@bAL*vEewz(dA8Z z?0)PK;euhLX>D1oJ?_l|2drJvnq|57kAScCGau&c4&Ng2FemU{LBKt`+e(6`B-+qdO0U*vM# z9|!GjhGtCq12n^5JN!&|j$Po9l3M~x7V@rQHty=f9gM?>4LYZLDZEv~!Qvj!?Vix> zUeN8d{%zx2ydnFo3Gj2`Y4&?EoppI z_uss++iW0u4Fje7{{FqtTy{1*-Kci^jsL&*g4&$MULf|I^8{`GVW03`-2at?uBo&A zSs$%4*A8queA*spoM_cL=FqICQuuElXM_IS0}H+W&j%j5l$c#Q3kegCy%;)y?Tn=} zq9@YBr<0FO!kXxBh*4PN>QIQcGuB}pU2M)Ra{Eq9N}TSO|EJM`=G#Bx9qHvaaQ+d$ zj%#E~xbmd%CEhwNqnuih$S&CGo;Z3J>vm$u zn*NBFUP!uPPfNd^eC5seT;lI@Y;f?)r%Tw&S-OFWH;_op%neWVH+={Lh&y{cA zJJ#I?MU36DH1QkUJqV4-<$nl#ptyDB9Dcga53F>P#~8|Xv8fOD&vunxV8Xcd(1*;P z^UXQo6P^yG3w4OjbHB~W!i#zLpJ8M`y#K*BqUU*bFYRsQcl{TCok-n>!C}3rU;fP{ zeAC6LeYvj}845rBiJ?AghcT?VTpYx_pML)saWVAI_e=Bvl&)6_~yv1kj zY4D_cLpBEb-dkS!N32)b?ujR^B>i>L6)(%R70tJm_AY5tfNtH2o1A5@kE>Vz=aK&c zY0`P@TLDhwe{=vg(#6D{Bep>4t4nit(Wi;@X%~9S(w+-)Qwre=h(T9+;(}cDO?{ND zfon&Oo)taE4$e#T-C~2Uv+XD7?~O;%d+EFsH$b*P+zr$0+5u(TXL#>C#5+;;$O!Tj zKj07anf)}^{HIj)v&U5Vf8lGj3vke`S>~a1^orR3gQ0kM6pnBh;ext|Y3>f@7HbD9j z?-677Mc(85+r)n640pPdR+p=H8~BDhg}YInxP8;DKsk5Ta35lF zXlD7si8J>S54t|V8e8HIq`=2b--h2j@2?85AL?}0C|%g_jPQT1=AJ|9*S9vJdsvoe zt$82b;D~n7pWpMHebRG-M~}#(&6*?5IaY&H-@&%=oz3Vvv`?G-$$gWd%~5Faz8T$0 z8&57jj34*m16TDZZ9G=?0-TEkNgHDwHtQ68{19!C&9M0oUQA@n^xgXzGo=Z~s*HH8 zCgEA$QU1yw$bXo;%jsuZZ6bQG0CX@Jek=ukEEPRi7jz%or!%n|x{&VZM0%hHLr)({ z48L)d9xOht)BZR;nCMmP{7(LK-nsm$m`pb0r^saRWSYn8k;$rOT4UKC(*nf6nCQ=& zwwb$SpG7`=%k;(k7$+8K6n;HtoNm%DMz z+1FY>X=_G;Pt)utD2H$CnyV>ii6-Xv2wyqT#p{ogCz@S|e)-Tmai?9H<)KXk}ml8JS$ymv)HM0X~| z?oqf3y1~69&DB3DGvy&OE$NV%;9ay{T^x$c#5vWDkc&g#VQ$P`Jj!-ba+1!HVtDfa z<=`o|1eXn=e>w}uu@BGbu#Y*GvG-4M$9yznvy3$%{TOv`3B>#X*!R$4$%I?tWkT)w zi1rp^qxsf%yHTJ1LyOj!?^+$-{gm&<%4r-+`JUR;$G}YccmvbzVgu9cWCL~XdM< z)E#7ODk6L<{6@dFtU}s%bh_H|UT-%gssqhryH4<~mXI67Bn#xyWlU4e5Qdo3b|C?C1 zeQ(*kMDm-dV|xE^*@*+g(>AU@FbBF`ePZ3>Hh)Sp?=OZ9H}A*RL*EE6#v7y`Z~RF( zV3j+TZ^Zj)+~+&zSZn%x@KxJZOL67LYM;2}%EB)&?g7rS%o_8RhNqrTzocY&PIzF^ zgrRe=&z%cDn{4=r^yVTxcM@-r+q_ZydRIMXHS4asChC?xUEfHbYDEfu>5C9Y$%%bu z1>Y%TuB~}P!~S2kJ)pCZf<$5o^KNt#dM)MkJ~e#k;tHVh2=tXsz>VCM}#+3TTg zZ4X5GzUI21VrX;yN#4B9?Hf*hyS$74M?uee&iJJz1Z;FW9d{b_(>H*eZvp=Se zYS`e-Q!?{Z$UHT9?Wy{SHMnUaxRUuS;4b9N{k^tneW8CWDX<1*gA-gC`2q`e@hxHuyrFnUffMJWPdoCRZJdq#lXsqF z{Th67>j~jWmnMYMSTofL#Ek-9!>jKWk5h7g-D38Oqx^2MHTB}K@Nd0!W7f*W!^4;J z-_TsGt3AWRzH-hkjoP?)>STlxhk-`UpTOF{n|j9 zH8{2o$HmqmJbP6y_+s&q+$9n1Y36s!e-qz(F1)!}CyM3WRdLTc)=7Q$j&)MS^DgGS zn)yy=P>Vc^*nbX_#(NvNYB2B2NrV9 zw23=5r!n7iimcsJc@8YM!pzg|xx`p&S?KMEroLhibF1{+T#8I@FI9s$O zmVZ6zN{iMW7tXzK;MzL&)c%ROuP^ZUac6KBYFS03^p=z*&PCWVc{_rN&vv_GA{S{t5mGlXzvNO-MeO_DF?sUr>xH9I)>E_8E?P)J>H+e>n zC!Bk8QF#7EZ4a2eG56GPa*-7l?k;({ZgC#-DjmV6lm~~5J=EP={SSUG@LR%9I+6FT zlHBLptvGO!N#7R#UPIfx|9?063A=YN7Ha3~v3&kJ7g^MwYro=^*)#$@7W0t0`0G04h~P)8mL(|7P`GV;9I)b&^zv( zHMALA(U)?`L#)AVr2Pe4qB$4`O^4&C~*_tV@%h8*MyUnfxAB8d-dOe+mKNm z<$plF)~1D=WX6#19AG`vA&a_g?x67*(0F)`Z{DZ=4ZQm*?{1~;*z#Uta-++iYYmOn@f39w($9M0G)1Vl z6ke*7d6nK_`6PIaO?}WgoB=;~I{aKEylx0yHwdp=0k8Yo72@GE_j~b?(^|OD^f%W& z-@qLEJOlIXaRz4C&+cs-tTP+;`~AX=JZpT6Zfs1s6J|aOORPb{JB-&N@nZ$>S^NDD zEN;8oa<)x&{U&xKSVm?dZr$U-K*=Fx`)cX(b-Q&$zbkw=kOD`aOKJ8Zfdhjfnuq%66|GVg&k#1Woo}(dj z#eFOjm#tcG82UZIk>I%#tksXOj)IMy!-}~xTrvwhmG2zx0J{R*P}?os$WQ-&!~X-w zD2@E||5E-#1Kx=WUvfSXg@+N|4BRwhZ8B>u=gP>El0Fp?*+b;w%X3`!a^hWMo5-5p z$vE`@530S5)FYX#X4Z@Y56l{PU@QMMJ{ETp8hs!%d>?%io{7>n>!uk8a&H=V;C0Hi zagVBe$C$tI^_sV-74!{T1i$5&eoaaWFQrfV&R@On2v5i^%cs5qpC=tW9?mWXhl4W; zpfQRCs`1#u_~~7h(f11rtkvdy#&jdNMzrq`<8gp?MH6WAKRJ&nt2s?FRtt5x<8gx- zj{xJ5Tm=0%T)DL=i&$@I{21GWn{GHzJL?8?sJYw?R`bq6>P_(eOBX6x{dUr}k8|;J zE&nU|g_RcSNSi|1ouoyxi1l<>eko^cZAz9^+ev!tI# zq*wkI`_g7zmX{lx1@7q=Ue4HEhpZKaW(wEc%iPYmrhoX_UP<8^aF+0jnSY)S@m)_= z+ai5U2KWVju%T4=rfW)AX9s6F8DZ`7alZMBRcG@2&>!z24wdYbd_2F5oN1j>ee90A z%MVZf@RHeqmp^|e(RbM&CkLh+tJsB!l=+M@xAXgKN#d0|?)6`}ZDGQdiesqQN=<&> zm8QR4I)}BkTZ%2#ZHB#N8u}i_csBIlq#YbB#eScNamJy7tb+{3H`83~- z=OaS2qZ@cldcD{;ExuXEw@g1$neQa)Fm03n+p3~|KMOBAs*YmnILv+0cT$J;TQc~w zgtRC$O|&n?m;LxzCocVH1Y;fWEm$1zWj=24|6IO137RAt?W5dPsn&h8Q#8uZn&X-8 zDDe#gq%{uQaGLeuC8H&mX$`1et$|pZHNVGd?$|rNe{4E!JLM}rx#YJ+A6`A%D!JrX z^#N-7PLzIJ0-h?m**z=2lD?>|IvIIWR&ZC0U#8szRm;9Egwf#4fK zr%v#-pmV$6rl2!VaDC9ZRd8L>hOb3Z@2~62YXPbAq5h==2tBt8k7JJXqlr z3hu9PjumXFaPkGeu5fY%n=71b!Ott4Ou-!$PP*Wy6;3a~k1L!Wf*({k-2}H*I9&wC zR5~ewqbi+%;K)j+vtX#wNfaDW=_Cjat8^^EA(hU5E(Z>*bPfp)sB~Hd%PJj&2Kry= z>=P`mboK}qR673_%&T<15zMM|z7kBUbiNSmUg_)-Os#aHf=QLmzXbi2&Q8I$kh5Lz zV903{+#hoOSFk1Id?NUD$oWXHIplmO_<6{AUvNjrc~|h$kn@h<$028n;0Gb+O~I`p z=YIs>3ORohd?Vy+7JMz_{8eyM$azI@eaLxPa9zmRAozU9`HSGQA?HQGr$WvPf{%xs zwStd@oaY2rgq+oaOGC~xf=fcqD!~Ur&YuPE4LMH;-p&3OT*&?xyo3EOSj+wwoD*^$ z5u6ormI>Yva{eedBjh|JcvZ+*EI2LXJSccs$oakCM!sr%rHW$hloG6msSXjtDuo3Jwc7HG)Gz&TPSfA?Ien0U>9mU|Gny zQLs7pOgo5u8|qW!&d%3B+eTaM6~E-wt1 zqA!&^QsU`L6+4dd=%8vA>)R#zwiS3ovPkP?>yS}=$SMiQEaE0Y1o>fV|Li8_4 zKP!;MUaeQZ4@`WKv?F!;dpui8(MROUE$^e>--JF)@nK^1_M=|SnPNETd?tD5s0QqQ z24@XTm2MN-UlKlnG8+;S!@W%#GVKE38uYn_zc}*@2YpXC`OMDF>R8%oUfOA$omd?i z=H0YW@n--BCca|YzGuu|m?s}+lNL7F?l(JAUhAiX_7o(eAERv}Xp>~E7Ie+Aa&20V ztj)u-Dj1iSO?~^$e~D-6o5lG?t;O6Ya8}^MwyR?tq(AtQcg=d`oFf@MZ3;T9Y1}!3 zj&l_DebWNd@n?dUNeGW>opjIm)|Mf%*&|Lu`!|Dpr-E_c^gZ9T?EC4%GWf`H&H_$_ zmk}SV|DV&pL8R4n8_zk7)qLr3RjrMj(;S|4WS!DM9GxaVtRM2hV(+aAAjhlivL|A! zR>hwsz)v-Bf8C%Pu`9~<_YDWS2NtHEKGNCK(^{joavkYwt{D)1?wU?-Zx~x)_!qT% zbIKV`1ioEoNt}DMvR`LP?%hoNzDt7073uab7#q&(PBwM)>L0E@sed?ub|^1}J{!KD zK5=hAuIeiB1^U7}HdH6nG-S{Yl^dlx&KT*eJ|oFFo#z`Ur*u7A11p@R$cNi=N6t)q zu)K8=`Y>y!bycI^_hh5rdeX-?B-@o!R`b2x&-o<#KarTc!k@A))j76o{H4DVznNe3 zL2iKk7Aq^cK;?x$RdxgRx90qxerC~6l~sF=q3jnT_>NQoi~YS5Nb*msU@je1d++2Z(t@ z*LCFnc%+N9x`p$__2s_lRp?&j2PD76z2Hg3zu6D&kS@mEM?MWf z-EQVvh34Jv=39j+Zk?i$Cw4>6n9$XHOZ|t|HIpx$5&O|d;j~K6Ua#d1jsyK5~7j=TB=MUSPldvfPw;wN-a^afYfrS ztpf-NgEfd&j!;qK^AN-#XQM5*1R)MTp!D{lc3^ON@g9%aOaLM3a=p@yZomP=u13cY56?xg#|9khk>3&(+WJ}*JT1kFzvdONC z=T33^3>H~i2|j{evDM<4tE|0NHK?dxsq zYx53$@GRcN^ugh+!Txs7V6E;jH@ZCbf@qT}(hB_KKD`o~FZ=hV*$ zp~oj~K&fbm^K`|X{?@!g_VT(n)DIk)?_53c;&?|-OuT`9DpSFk9n<~&BRz4r&AEDF z-7n{N^~BratUjtQ((T3oI@1{?=)`(KBkL}Wh2og;#y2zJT!3$6a>k!vG8d;#JH1R~wM0re$X=m|g%Aafdy z9555ROnkw0Ch+|M_Pc4Ze=d|?fS(o)%iV>m(j3*JTosngHhQup6Wr|rZ;BpQX7|;GPw;t=J*uf zNXKzDW`5NeUYDHhEmF=k-lJb9PW42xtDpni|_Q!VsBbQF}M(T zVBZZ3`@fFwN8it(*Ho^^)$MNucBR|PKv&d^KB$g9PmAFb&P&$5_wK^x6^>7iY^xk! zpZ-6fKTXZOi-@xo8D!PU|AhUwRyvXO>_5>OM}O?ht;zSJ51?K7Kd<>O=Y0e3b*F2o z_7{4=h2xu7IQ!^ay`Tl{qjme3^6LA^LhfEtT#atPJv8qpJavV^=wHm-dX5x|KH%Lu z&|A_Cu3|m(|BvRq_sE?td}}_pm_7ufo0L+=TcnERqItkBJL5TDx%d|5uSe)O@629C zHi{5AcF^=${gH~G>V z&)yFt8ZFKCsb1>QdOSfH(cSq;R!!A1?~c-6IQfgo7j0{GwrcuLNFNvLqui&lO{VwPHsrOE6tnX@tkmau(iQc)$Gp+H^D9#n>IIb9crh?SJ1>F#K ztA>UfoE(aDOAH^I2o7NH-$lK9ish@BX2%P@CkCUplc#qRO^P0mH4_h^{q^7ccH#r$ z2i3!g-LsdyA^dR{V_CQLrj6zF`6!R$(@8njRK98Za)@PBUOu+1wRCI>^9W7U8@y@G zKiOC(aIa;wn-z`}Tm99a-N1`{IDOD0`%{L8+wyHZ-$EZP&j&}}3Cs>z+AkDjO=D~D zZ}eI4c{gyQ6d6Y9y~LCcjtk`+4(<@9Nv(HrDdZ;MG{)w67-f%?bMDS07zH z!?*RXg#OVVr_JwQW#(2sHnq$Ted=uSPI$=ZN0;3YK>tyDFys2b0qCIb!o8=CIrfwv zw0#QJ#7Z?|`m%3J{i~e%ISbP4!@!5u@Q_mGFhY6>bJDz3{`G0tGNxnGn1Nk!CU(VH z$Q-hfJ>BczxoqfT26qxO4}8$%gKDnG{+fGRBIU zec;jp9WTK)8MtbKPC4<{JeEXzhA8HL!8>WNLG&kr zjvulH=6*8Lhj@fn2eW6N{majHDIU7ca_xi_o7I;jw zs^*~yd?_C;y0e;$+?z+zX&fo{;M{xy4ql3@yq;n|SK0Za|(rw8Sqobe|d|3@X zY?+L$V1acce4=@L6X&ymqx0;=h62BeN%2?ekQ_wk)#`=b9gfYmV#sCLKOY#AG345} zyM5bj+H`2NTim?AlUE0Q_FQM{XM2EA(_VhMp)rFED&NhG8viPOztnmTm{XnEaplrmE*)Fi1P#C$68rW; z%5UJ?51~hOhR?j$8SB&J8`)AiF$Ou?mff5_PWK*l;#_rPHo$^AcaKw6xbi0YCK&Kj zM(1ib;S*q8bei;`M+P`BVeHx27vb~X*SC6KrFYb2;D)k_1xTM;bfQyfQ*blx^X5Li zTEWhc&*7;#8~U^EvKMH**EoB}jz_`lnrDJ8kaZGj`-0uQ!3ocH#-*x3o zJ?L}3-|Z6H zY)`g+2TaQL;|zJRIr3Wz$0k^d#&$*jVcMkbSNXQWjaORjOKmjcuHSl1)xzZi&2Pml z?OA@!(aEx%@$J~V)uG>}fKK^G8zcyFZx!>9$S!>-U;_i~sclJAZ`td=cJrk;{7yf%hzc_f(8y;b(Iv#j^2% zy_`M!Ia>~_91~do6?6e`b2VcLp}Wc_jqRTTx1USMSG=$6SFx1^NBo}rD&*xlgC&3Q zzfd(+yhA>B5Bu2fLEhyHSHMf%>H2_Hsn7Uum3OjgnxUWfnRypK2HtdW#((gw==@)+ zYzp>*9SqIE-PhMfz0~i+M(S18N%%nXPz(NRP~i z;gvFARdvtfzi@}<{b%}}Y(*b{pLkyT2HY0A1AHBIB;YfuJy6}PM?mF&>DF} z;2-!&{4DPmtWIS8^gjjr8(?uPd3~8rlSjW|uzwx-&x8AvzmWGsc_t`t_1QeZy1ow= z&IrWpoFXr`3!j*2_U(4rU{(2+4}8s+)~G$V8dz3+>*=FX;po^{Zs0d|C2I?+`WyJn z9><#dbsnCb|Glj1KKghUYg-EazPU|fzdS#(CFqxG)>8I97H3C2wnc&knoOXsgxO zhTjL1tl9%UE6P6I`LIXvjT4YP`R$KR1M~PA$;U`^)i?EiM&7tR=C7F475N}*t^SvF zcKkvengJT0nBpVw5rtmp4J)d?|Z<$HI zS+03uTl61@=clGY@1{fdWq@Kvw6nx^hXYL8hh0@ zGvEVLuv>;!A61rBSvI%yt5U}9z*+|5yp{glLO&QT;Gqod^wBmcklha zl6@wfMH6J;3G*E}m7JYwooK-uRb&Tzk7jAIM?4gYIiS$D1YXS5iyvI(-?r!>- zQ|4pJIA>CteS@3#9(gMNT7gybG*2my+R?nMqtB8%YHqeL_u5ZJX2=}9#rfDGGvt4* zdG9?k1U^vpHPVk}UtG9qooC0=e{hbTJV6|clkdt zx3%JdswW5g8=1Q&!T6nf=+{xsca7^}_OQhRZD``h_v7s&fCqJ8VqT8>e z-hQ-oHEC=7Biysu@EqUcekx+Y<4>kOg)^=76`5AQTx7m0^H@9XOsEpykBsw8+MtfL ze&iIo1EVC%THo@Y`nK_1Nm**82e=hYg`v#y{7ihYB_A+861hvP0{UHOMeotPxb=ql zEqK+u4RPmf26?3?n%BPu?$>#*`$G+^az==M*w6l$Nm(PG=l{)=SNulF0`+_K!dn|P z-_mE?uYL7Jaagi0-EF>)dE5`I9{|=f*ej*P_0imx@SVOdCKWv)cs~W)?qja&$3X{b zjo7PxPex*d*ci%}J+R(sE$fS|{!wIbx;NfA|FPeozWwY;$Z3t@R4{wA0{A(ZNlHSpm@oMc^FbABA21!nvCi{-T01BXqB8w`BB# zS(V~1j=@_*Tk{t5sb;nVlc|iaJl?7iylC&|0V@t~arRz>MDheT+6U%7c-He`{vQM8 zHCLkj9Nt1PN;7TIS)ZAAY4)7f@47j>#R2m0Z*pkbb9f8dWUim0y`nPbE`-HjTcH59 z=aFtn;ggdMO}5q&Z_$nWCB|7%#r4~3WWy=;4V+D9=R3vz9b;6Vca}Qvpmli>7!qvg zx5}EizwGSXKgiq*_O>xIZS^d%sq0Kv3udOkzdtR^IXWTO6TQ}INot)%9iSw@qO{z zPqK!hxmQuAx$hsFeZAS6Kf(mLf?u$^joR; zTV^&yx~@w&bg^bV%vtq29oe=4B#dj*U$+{-+q9#(bTblN9@U()Vbq9DT{Mu2%F~ z{;wEfMHS;)e5ZJ|^X=altDzIPbIk0E{-!S3nw)DV7c19X?A_5cH4a*%4KznvXpeYk zBR@8R-cVyJx+u!JD*kIMPSd4sy%(D_&F*T_bUU9^@beyWf%QC3@yzDAg-7|5R+fIX zTQbz<(p}Z@*7}m>p}QKnQ%vi-{y}H&r%{K_!=O1Q9DQA$nH&A)-XQ3E;oBzWxt|%Q zLmz05n*C4yo}(6c>nwlOth~Kv^QQ2fzON@XXbXSU%^N<6tZSPW8Jc`tbXQCgdwHR3 zrkrnY2Y$_Y;>L71KNikr$zatk<W}hzAKi=3Fz4-)=hDHPeIQLKet}Djc%K-nKaF=A;rIS zY%looRG#A<(MR%ZZ=a+6ly@zoY$=5PCY(yoeRVr`M)%ey`dfAqFx z)fCPBpt2bqy!Nrm>u%h7XaV8?8`|J;)>g3p9PLVPt9hw#^H#WhInCMfLu`=pd30C7 z4>{wWDa#<Pz0m?&x_tb}qdf7mh+t8(7x4l#EYQA+TH+i}93g6I_PAh`!FY z+cQSNs@6<>?!j2a^9GdCzivv|dzYA$_#Y-su?tK}ypnH;{}GG#(Y)0Mo=F6rlYr}X zz;}D_Ob7I2zEJFbS)Cs`C(QnGd9FJzBDOU0B;o0sfWfQyox$^|H)F;&;P4LCS$J@c z;Os{+yqjhprC)+GFR8(?7JfNp8!EG1{zmVmGuZ--6q`B&^BDZI-b-YK2equ z(Ad(6f%8G4Rby}g^*Q)~y9Xa|@ZbC73ID0A_a5T#01q+TPi`{LFXbk4DB~DI|_4cE)!oK00 zI61=AE1`KLSJ+GH=PWU0&WleoqAlP35AMq~eIY)L+RGq54YYl9KJ9(iIV}1r76Vu? z5tH9kc0nG2eTCrhDP*9(;CGqVJF5U+@k<%MLxZH--*@-IAo8>qIHPKaPu5*)aE124 z#aS2L2Y+hc5|6y$c4sXvF>8=wCy}aM#qsnL*YGo3KNBHopEY*qC+vLKPx7>l_ z-%x%t_l=yG>H%-Na4g)%IlgD|*)#knj9Kj*XH4hAFm~=2jBWRX`^|p7F);iQb|H*S zF#O~_XXDxa7whNT17`;Xmu`_QB*_FE=ZJlk(S<14OrV0ag01;Z{*{HI{}WnfAA z`v1RSc(Gu3p$8oAVNZJ5n?Cj^z9w_pAZPQ0WIr6EA%*ANec1xbkzDi!*eVL9(t(>s z=*9PuQS1M&{f+$w`O;<#!Pcucc{7<8BS)B5V`89{150`SjD{f&E;qiZE-oilUvf*l z>Z|$gd%!F3Y3`$q$sz6~Kbv-<$;dXu>$-UKR%8PfHWTnh+t2qGNElQ;7N&6fNY51^ ze;`oly$hbbm*7|5qn`$ToqNC23{U+cb+MM>skaXac67^eegivo#*Qk(9!)=V9``2A zr$24!tLiIp^S?)``MA)1(#*ZVf6?n4*cnRM(yyI6YYomg?%W%0oyC7^TgM60`LX#d z-dh{^Cf-~7Kg4@$6_-oxWP|sz!F#d;{5Jn2(X1zc6RmR+V@U@dgs;xAp51Ljk||tHxgap2zWo>+v4=KV zf8)C&2yN=<-h|U6XF0_E+f~x_wyN9H{yIZzB16Z9+$o1`h2fhe%Y>iX37ygc+d_YOCKg^-=5*l?9i55J%QyM0HeWZX&dYOoznG6ybFp{{?r5Icjy>C+J==jjo6Mdi zHli=i$i`)-v^T>OcJ2TM~JNYlYl8vj%7+Pp}fa8@NU#9s8BGj-|s)& zn~%d+H_^vq^zSHh5%pztKTTXi#WD5IMLq;ApE)d3Z9|)V)qdhQAe&m95EhQG@BtG3od$l9|Hn7xf9ar5fg2yVQ1d`t z;&-faXbEBoE+&>>BJu!YPT3=YF~x~mP0Sy~tQui(m46Ae$PRzP-=9rj%!BZIsz>jU zKujs_*R?fX(UmHnb;Wbf4kC`wK;2QpxboU`oVAxa?m(tcaEtIxj=f=^v-gM57vy)$ zvt5=#OzO}YX?DHIgZ?)uGF_AE{Jw#`GKM`fn!R&9d+0j!G^60to%n`=g)hNx zt^9YEif*g6tiGmy4-UuP)LNgcHO@I3x2*zP(()den)yvESH7#kcV)ZcTrZ&y<@|Tf zf-Tk3^_h3Xd0oW22y?A>Md+pYeyyL_wc9v*O8KqWwOgo1v1@afFV3Fz=j0RTHI4ir z_2*N^nK@4V&BbkYeAcTSHe&6ud+dPSV=`mz$k;nE2c4OV6z~T3nflsf zrWBJ+h_)4;7j8CVb@{%1+2Ba*hD49A0LS0QGtT|L(w8!04>lI|uRb0F_Qh9^Q+w>A z8}eGRHSE*EaxPW3ba3%^F;fogR0S6}K^GesyuM{~HaL z2i`2s*m(TIw&7DVk^@oV;A`JBa!1%_y*gjs&?AL-_^Frc?!nIPz4ilWQ#5!Tb1QoI zee85KKYhrTjl}}ya4>uK`ElMI(y8k{8l6p-ktdrT=RQL08r-{U#B+#Vd4YYU{jP5` zUf~Pv_YUTLy8T?M_j{O&DZGm}@3LBW(l_{D>fRkFdo^&n6L%&F+Z|+d$Xea~pJhM5 z8mRxW-TH(v>_$&x=&b5a;4l0L8Lz_&W!cAXbMAf-kLh#g+pG!YKIh#tqGPvuITzvG zr?zG8<5>egYmoqaCnB$qT(k``Hiw=)^)2w*8~EJ}d}<%A;M}{9XQKOm5HPz3S@YkZ zFZ8}rwidqZ8OWT+Z=?@SUo-6AvkqF19ppK3V)T1%-k-=54A%govdzVI4}LDgyw}|= z22RUfugr1bbOYbzSvd_|F9fIl&Y>+X9Wje~g}VfUHopa<)h11`e`QkPzLCtw2K3rvx1T+3JPajjeHHWfr_ciUUKhr8M?ikz&<6$a6D!eRxnb_r93+ z1y3ItUWpjXf&+i%g?XjYl+Y^nvgpY#ZuKmGm~m`s>z`E&olymieUv<{p@rR(=<7|` zjO+m)LQ@m7-%4#PB({{BKN&hl_YEYH*8p9nI}Pr^=kuotR;QW7_5Zho53)P{COzlh zNiTmbJ!c2$_g+uWxuMUXz-Rn^^{w=r(R~&a&FExxdaTpgI}L*L1<$}GqkIWTx`$vo zWdpCK=Zw&O1k?{bb>t4();yV5)9gFRGn?pp4&!>Bd6@K2=~wfO-X`1Nj8Czf0bl>q ze4gqP-!C3dF}@t#sqP3#wR_;JqP-en+r(|p2wakH#`OL|0p9m?YE>lj|-`*e1x;7-X*Wx>(F_OcWo>dG&Vgk8P_>H-T=NA zkN1sp>VZ^q&&2}Xo!`e>Xv3iuv+bAMzC1=Myj8vMw#LPra{=Hpfb3K;DOO>}-dvpj zopJg7%p0<`|1WsVrQ$I)&K!8m9C*w{jA{QZ@(KO!7y;6oD6X>h;ZgQs?Cveu0SP}b zXXnHaD3K3>ORLmoLaU%J*PMKckAi%r`$K={@}NJe`G{v74|gNh&gSmJH?RR3pgR!d zCyEaJWMP}|)tX!ER?4skIEg)v>e2b_Z&UHC@rgtBF{<#x?7Qc_QF&Xzwr3~#5*pB- zjc@#|556(W?#DjYS#d4(m@^&z;Yv4euGIDcMLAx&ODa_omYa*#{e54SiV#aqncanrF&nG^`eEqnplg|342b znz=hGUtepzm)>&TywBQT4>R^SwgV@+K^xJxcIq!YU!qCV>@O*!J>c?SYg@_c*ILlA zgI%2UdzGV}eWV6H(TM|}Yg@zT{O?)gzsv7lw6yLcYL-pKUY$d(-7M#cY$TsRUZZ(F zOdI*&7R|Np!1bRg-Br{UeHglu28u9sjV$4H-ra`m!9P7v>Erk0ZN&SV z?+JNUwIMc#?w`)EGq_Wz3LH=!lM%$qUX~TIR#A3K8{`iuc6-W|g1fYqdBdz(Tfu7< z>sUemjC{n0tqg6PGiOg{g?7WYO0Lj)?sfm@H_C3$2(5BygS)A-b$Q{iRpc#Wj}coX zv<@F9!LM-MVfwSOlQqi=%xJ&gK;1{k8^oE>)y?ZdUTKH8(P8Y+R}Qvj-3jhcS;-8F z?wj}IVtn%sF~$b$39bXTU(eYwnzLg}!YA2ZgmZG1lD7L@PR?4=_OIpS^t@tF;7|O1 ztTrd-@+%e;aeqxG&JS}RPM)VzgT;PXoakS2x7VLkc%p>POt7RihW>b{)Ma)fzLj)1X&}c3SXC0e9uh7)(2cw{O?5 znKf9+y*s7spF@08!8tgXdw050UjcqyBRu|hA5BbLqIwtdoKq)v@2I{2ckc{#>(iMR z>vsj;E@8inYGd^}zu$GcJfWPa4tx$}T|_@HX5padw8T%c{eRBMIgI_s{XYLFoykqC zN3%D3#vx$)_?6FQ3m3dNFCjmRwCb&&HJ9tx6NL_`xAc+hGkpbh|aadmo^H%d&d2}Dh%$}KhK{dIw7viU%AbN zZw}M7Fduc6f&h>Q-Nmd()EC#`Cl>8JotQtWyNITS=^V=S)qtUzu<&jLxLY zf`oJXnuvWF`@ucL`dcnsp>tpu-#WaPZ zlC?fa9nDSn=Sa@EFc`j2Ti!mbG5Fk}b!2;)VrEhmzSmqT29efcQY)HAJYp=* zS)XisDC>iatgrBhpSj+BzoDtu4tXFOT$^DJ0)9<>Mt_oFFQ)tf?2}ZN<|tO5VsLy8 z-ue7y&h~#`6A|T1m;6%kOBA~yX2bFe>K1=}lc4s9Sr!JMFdR#rA>=F6Lb-`6_2@b%0^APgc^dLN~ncoZ8*$irt;bqxj_nT(mK;ND*X}W!iv@h+7k8L!0srCtzvIpEU|IlXP zAHbF50J0euT*;3&23McD?fsWY)9g=3Wmj<&o&Wnh8=?7RJHQ4s_jYNW%o>?>Sj0L! zJkYQE>SU*KFg~pNlXfp$JTS64F|7E*w?TUb#|)2(-ua)vQkDx#6I@u@KtDy#4599= zz-f!kfm#3ZoxdSZ`Wv15Mh47&)p|c|=(Lz^R=TbAUPhkm31n9wn}Sp9pE}>UcZw#O zwac`Z(XRBc-?rZ3qhdVMedz((PMKnqg0H`!$xa=p2;2Qo{EujC?=9UQf)&-p3Pnr2iF)N1J8#eRjgf zX5Sy4>74J^QHN;CJ845a+8$%?me#U2i^&roB={FS8QYteo3+ZYdzn7youjMNnq6w% zasO}2JMN`spHJako_UvQU&6a4*0t1K*XTFaHO*dba9x_+g|ff+j&ZaKd3#f_<^z5SB6e*k;|ktw&n-> zq#;kh-#CYPyO)?0H}Ul5$>ceSjosgP!rg+kiz&0FN3eDj=_|d0we3i4?5w%-U`93f zW<1)_>ZdwhgKrewrMon$i7|xTQ=caA^WhCv?V*<~;^$k@H=rTvdv*v{-^BU=7Y#i| zhbKXQH@4y>6W}FJ4Y#=GKBu7=9(ARBpGkjG@KN@obM4BB1#dsQ*q8Ng8N6i)e)iMR z$BCzhM;%|k4jZO1+QVt~bF7DWpfzq=I*(Skc`uPyH-mCB@!xdwRw-|)zRj>7ck`Z8 z-el!r-{a=3P#*ZLX_OV6jqXEnsf(0O)wxeP*^Li0$>m=+;6q<5UNqcd*EN^AWY;y> z@UAKLgYJLn$|Cm@N1L%KR=nX~Uv3-D6#q)UW}0%D4*yD@X1M=mgOmmS*e^0OT% zt8$VH_2hT;!kfg;E^W=v4xS_)J;w%~k-1!Wf7BNivnPhIHwNR||Bkt%Kyy*g{9T=} zE&C8W!7bd)t(Z*J;@?@j9`5)H8=J>B_OhbQW#WhN?VTW+Kjz1rW54UlcZ=ws&JN-; z2#;^i!v5z$bW7cSjEoYT*wmA?-sKA&o$TPRTz71mxBL>;_9U{9;Z`4Fm_$X-Xic63 zCx{Mq;u&VyyIKEJtleJ5yAR)rHx;8BdyvLL{84)HJ-#W8j7z+5!raW9h~Oj7f`(2Q z6%p;hy3Td$^nbZ&tc9;#w1@Jko3)G8`x5m^7h2Cb$6ZIWuzer@1~f8fGhAiTp z)fHn;nHP*+1;1MeU#WfZW9rs-^0&yIn+aT{*msZ@DPykt5i$syuQ37$`M^8|o*;|uJ>Rzlwj?Cb}Q4i@s>?O(17N1DTv_!T*@ zl4VD`1*3=ge~9s`kK7?MqY7KeM%J}53tB%z`g8BBdiKaSVEfD57M;~k$2;)%2JH-> z->Oe_sl9sU*}~536mqX7>eC&#ewVKwK>0lG{wZ?*>-{gu(|x#iw#v&V58i*f=@0z> z@4++T|5g5O-YMS?AFeX_@Xq1A-W`GYR%kvvvzNPa7xKixmy5@417F^D(etC@;hitZ z8uY5g-MQU73tk!N^5v>C&y(E%4bRv*O6x7Ti* zyuchvw$XYHgK2|ay|&AOS2T|?*+y*KmOF>}u6&~;!WY_D&`KSK8NJ*02g3+*-CLya(JGKK&H9q?Y%xEt1U;^1Y>#TgmJSh%K;L zaUy_&+Zc=RTjKz5l+XFzeA9=$4$h#x+Q=7H;3$28-mI0JbvHO*A2{FublXA3Hidr6 zcXt&ys2u%x-Mzq4PvPindkXO@HmA{V%K7KMUYY*cL%W3A2hy(SSIu*Oe)mO(5J8p@ zfmVxvXX-h(bnf3__G-$z^}I85z#}WOr{iZMJKuVKm+`v@pPTB=R$tY1savo7LSwQ7 zjVF!q2%lfXdq-cEYUl8~8DF%<)z^n3GdzL17JLUEK_BElrx+IpSBwg4tPe02!9xtb zGl1)IaF5y*eJGiS=4s#RYr_velv3Hc{kCeq$FQ(zYmGN>G{vgjmto;+=d=|M&bbv^ zE7i|kt9ESNg-5qB2i%)LJJK&xeuDazYs;^0KXZM6xmMh%NB-8lLf@G>=!f8@gf(fP zjN$=BknK!leImeVcmVe%>Ky3NF??vMb8wPPN*l#t~S8^wfTKujj49_j}%InB#ftP?_hg`h}ji zB9D09LLO|++;sOo`a$E|bq9`m@muGGe6u4biCcutl*YK8b6GHXpm21U{ZhM^{t8xi z0~7lgt85w%0++w!?lg^6Hi}27PpRs=&i&td|CBI?)kpCoXC6-(TSw-Sy7n+Pdl^sk zs8g5PX#}>fp^gaeQtIdIx^G@~<@xYr_SBbMLe_38$_6RB8JW5E3ijrPZV7U}3zmMt_ywcqx0&O@@IAbj+)K2?-TXet@8&{ct^r@_ zbEJnA*bV%uPogIdEE^Tx5A6RE`0dQN)aLWFskss?H{#o)ef=@ClJ?{*?BUCh;hgf7 zV?Uf?zdaVZk#+e`(rIU$_#CD5{T5;+o!>`K7vQ{bSU7c7mvN$*yukHd+S^B8XCa3y zJx#2T^XA7>z+QD}A@!s6H&ggVa>b{K+bMeSWIt$+dAfHP|8T~BeqBD=*Pf_3{Ln7N z^z>T(C6T{cVdj%~f5XC>7xhK5LwtX@GlBjj4R4V<3RifTXCHmq)9O2Yqk4h6=;wS_ zDtRUFqd4e3XtTx(^v_5C1jiTZqnV>N){W|8tbb15u)EbhsSc(aoH%zsA0#|FkXU=J&PGK256 z_q^0ssP)?``6Kd)WJlg8e&{O3p*%Fk2;M^La~S?chud# z!W~;oJE`_2lcw2knKa#ggEW&lNx`nZ2|MZ!d1}!;KLhV{Dg)U8_#H#P0b2jkqq!$|J_HWWSk=c&zbhvP6;3af|FD7xx=-7NdB$H4p8Qpf10 zN3n~J>5-(nFyAM~^cUxTyP0oAyUAWh^P_dU$DO+wCQY@cn>59qYSK*B@SA=cr)Ls# z1v#N~$IZn))(~0AR6jDR1Y}jj3mub$EU_K7k=(cE%9Zvr?-!CC-#|OZ1gGq$>rI_w+>RtNbld(lVcK6(n%V4O1I;{owK9BA#_(G z`@9JMk?F{C&=t;j?#hbnhs&~!4ssuOVH$NJSNSr4FNf;(U(&s@MF$B?2sWoum*)Q0 zzOzX~t!Mz(<%gslcYqV}nPN3rKeviC=T14@uj4+l3mzKMf~r*$XoTGTIG(fXbj;cv6AZt#Ut zs1MrP*lW*`efdXt^Q3$B z#p3Yq1^-DNA3OWr!0cgFGxk-g)0Tm#;%*~PPwM>@_b1ey=g(jh}e}bYgu^#(QtO zK%@*>jD52eSkF7{J93QQ74TO&Fa96$(#FIzEYJ_{`LIN&F8LBepj;} zBlJUi`es*-`x5lO0=JpUcqh5Ne=z^@-(O3cAA1uz@GP7!7r(lq-; zlcw6MO`2k_GHEVwJ_@)U3H*)#j!S^&;hcTf5{u9|`~H!QP9t#c#6-WaO~s>byA>u) zvwum7UGmsvz{bycZsO?=FB@JxIBc^fP52Zx&G3bnP)0t64e(U2Fu&!pvB)UDrm;uc z@JM92hi1fw8=%$eh}%?OnCSTbO|+uu@7I=M2Qi&F!=|R81Yf=ceEAMx#~#3!ZxZrR z$*L#evsc2sdy@P6uXxt*ySeD_OI&=F4Zae7{o1w5`w{)tS#u|K-_5$W_`JI|7B`VM zKx-mdBX~#g^DgH9Ao86&*`Yc%7FUy3qL^9-Jz?>2+9Qr$zI@=G$!GoD*J{5}Uw_(m z?JO<`E3NG;cDdgK%zS6s-AEnV4|r~qmuh!4X^Ne1Qr$yzBQS6SFfaxf7!3?u4-8y~ ze%|qklMmT<`Hcw%WP^Rmw@5OZPqK|6PmHXYc+nSVn#7O&@^E|^o!Ec@CZAf)~ z3O;J_LEQ&TW7oA)|NH0SQu7#to1Ej<_=!6Ez-qv%nz-v52Q<~-H@yGhgRzma~}*{T0eZa+UE zPq46EX(#91pLv((M}OT(KFnEmNf~$8pCOh=M{B)!^7Cl7aQn*f8g_~GGw^l(x&h7U^KcH_=T1_#eH2B%=Kw=M}%v4-OY)myI-13d`N$gX#aL1-dA6Z-ByWggEnC;wbUQL(BLh~uvcXpD zN^@8F*!W8wK1cDzBQuB#bZovf{RB;jZ8 zGr_3%i#iUYv zzDIY{Z1biz6r!8|)SKC`7yF*OXXlL@g)Ps|vF~}?oB8faWSB+dPsgWAdcAGPgzHD4 zTk4q@KK_Jgs$6?D>mXeEJKE43$j0Pc`#u=afGKf7{7Cvhlf}xm9{EwmnK`LRTrB0UadUpQ-32Cj_JYxm!;CTSz^M z1Li~aqrR27{};;cXZ471+T1z2W}7^Gb`5`&ZQtkqpUi*N=b!5x7bymR&RtijcwmY* zpfUZ#Ehisiv-aR9A8U_XXS=mMuSIrtDPzz&9i78F`uQU>)?oC9N*Tv(N*UKJN+%QZ zlTaMw{CGJ!*Wn1dzJn6}Q+5b#G&);hI^%{Sw-Rz71m+XIz_r#{>5er?i z^m|Us662dsT_2P>^mH%U)Y_L5vmx5c@T*H>c0E@JU!og2-UB#Di3^>%&G|-ig06rV zY}S!T{NP%PJ5il@PTQeV6fD`W56K zMmJrelsT+tUInvTSVP`5B#_4R(*b^Gy7Q|wDfAMXhzyW>1?v5AtloaX=wyETDO*q7 z^~h4|sXMH?>Hi^M1^sYCfm{D{w|>#!t?P{t3nJ5FjeYRYoLy1Y>wWIy4iRySPxaqb)(nnC!2u&_+M1S{EK4WoTFF8UmCwx zi*^1+xpQ=?S#|^EHZzCXi=q#udu#oTuMfJxM;JT%u0r$j2DVviEZI5Ln){sCC;NJ! zd*d#HgUs3YkX>qQk_TuX)Uj3_s8=zX<>QsaZ^!HVv^7h8t=v1yn?T82#nHVlWcVQ5%_O~Vt|G%UlWVTEhc0KVCM ztAlS6M>({<#`E<^$L}v)xB}e6UdE;&%`SJ>L~ti~+e?2XL)*<>+(%!XI87<`L16L* zVtZ?B|G^rYwZNA981N6S>vR6RAEHgs?=jd~0&LAtJgCdjV|U@)0@hO>%rm<9I%2>W z-8%CqefHyx=&dd~M{m^vm!1Ofk*kxB!9(kB=kcw5UI?$?X;t2N9^ zM=#y^G1lVPY74p*S{ZmZ@Guj-J9x^WjZ^GPnXkqP@)OFmJK`f*#6GJ+$;$ zI{V0yVPR9J>SO=NR;hJes^@DLmWe}L+gtGv+G>q*?F`n4wQ1Qm3B>!(C*HT_H^GhB z6vJ1_t3ClsvW0E&#bR95^r;G-%kS!iT94~Ob+m@7*7ZEqYHa(k zI}q+2%3e6Z+P#PE4*Eav9Qc9$$^SI5f%sS0HObz@>?s$9PJ+WFyNCn#>?Bq{zOubz zFm1{K8#W!|QY1uR$V&c&FG?eOwz%pU4w-6$CO6>}*$K!ow85ocUy&0_M1cOKPw zve;|vYj-AkHhLF{Uq1ERybe`{pd$OG@tk5W7tFR zj_-oI#~LXkT0(r5<|6iA<=VJ;dLR98XjpUl40b%_cQvnANesYa#)c=;{%8c}7Il6= zdLK4CD)%^ZJBa_ptwbi0fUM-gu~B0(5%DIH|LV@b)9@yW4e<=^yj`)Ga zZr;=ScA+o)bJ6jU_&_zBSS=lG{%hst`$TFwN?XZe=~LgD(#WH^yx0g}dmla}QevXEF9#tYrcCM|H}MF}6;c5AA#X zCl1~EN2g$$$G$$wz7|{wD{JE{wsTv!vuO z%^qvkeHoS6;<5amw1(%|r{CCrk*+Pi2Vg^MdX!Wmm)rH8~59MQQS6gs?Zz$JjsfFmS@!-&J+I0 zw5J2p+7A!Vo@^hq_q4y2cdy1|a9655$<6zzDHj)>=;A5Ai>I~;KLIZ};7HXwk@AYS z!ea%duiPWvgif1s6S*r2T)Nb{3_qsf;23ybtyg3cHqh+Brm0rI(3;Hi zo{ZsP=`!}Qrw_1SWOMn{=gli58&Ub_YWPFuG-B+;#EV_{8~A6lcBa24oBLE6yV4=y z!4FARlxoM12lt$4t}!?x#T(jrvxCo0(LbG6rHNL}ODDzmq&jV|CQiTP@1A8}jDLL) ze7NCMu>bByg0*`%D~4>p7~Gm_FPq@RuU^3&^Y}V~-y}cQeIwjG3ViNM5}lT9Xle0I zXa3@p=f0PIH^DEzG=O=@3)Z&rWR23d1Rw!*W|}rH69*ha5cnpt$Be*^3Y#^(~f}$r4yH5zvQ)h;yvNL z@s51TIun$>d2Kn&S77VuY;d>&BU0^w(nv;;pCt^XrAl^1IIguH;iM{d0=_*+ge-BdJF^Z=JzgSTEtNVdNQD zr+;s{c|r0F9MHcFZr%X$BtxmAkH2&C`jRI+sD3__M9f8=emq$`r`RXeIl%oOuEs3HuFyN{kU63@F{tv^dgPqt%jCz@^WqIIF713W1W9ac}H)5Snn9~ z>~r4b7+aJ5FGr_#jj`=p>TfgH+_U_&v4sLYfG5#&f|niq-fw>E zT#-ypHUa2mnIHK_$v3KJf#MKwWHd-l>zJ35;?!4Et1QJVcMF!=GjInqU+6XQyFJ0^2_j(5+=7G{=_MX#NF zo?LknaSY@eQ4*6^#LAr~SH6jIjzilh1}yuvwvCnO+V*PB@i}Jh z50Ey}&W&B6Z4Nc})|^hG9m(7DTj#?$bs`r#TVJNBZ%=$NbdT!unEHM|U8Rw8=R)#Z ztcf7xFha z>uaFpbB@>~Usm)CVsb2QSKf4RhMUi*mLXx7`togRy^-_JjjFx{Krx>tV)B48#a}aK*?t z#=f{O4nIiNM|^b^JbakGy#B?I@H+aoaopf=g*Vgodvr%?DZH6=d-Sw=Hcs^5`1!csPsv2 ztzE)ez*n}!Q~>90x0WV_3Xto_uTcDXko3?b;*xo?8j8{JP9nZK@!iHJG!NbtNV{dp zW!QI3KQnCCJ#*Vtjwnm041i~3^EJ|w`|c+2de=}%Qw6c=4)^ni8}RXn!Y4;3B0HH^ zTj`fQ)*x(&41bhkf6O|{7iu+Uq2#z3#s@sj-lcr*tk}yrQo=p3@^`445RCrWE#H;; zzcG%D?OWm|KEn7lmXl)~8O1lsKHaV*j>JQ5S?!bm#c%CH))sy7wXF9r)*ad9q+;M; z2=-cov1$2F6C+|!nKkQe`Zp-U`dKOcPozEY``C`mpPAR14xwq(r#$I~*NN>2pV(V%ZFBXa=uTR`$(k>jPnJE^%wd+Tv4>c%vJTd)C}Wz2oRqe9 zPG@WlzytHJGx34$iY*u_^CmW|LQjl+N64}gmOy7Tc=)fl0GoVSjW2A4#vg~y1h_Q3 z<8b&gSMGKr?Pwh6N@@mzXDX&%8;umw9x|yh(BDDiYA29WRPFNY$l-2p;qamAqhz_P zOZC=UHcAhFg1I-mc}cfWQ?cOEI-Hgr;%$LSkz}g62qsSxr*T=l<3u z;M&J{@;oUG^S{qtEVn{;GmnpxS4G@{JC1C_q%$me%gPx&<`fDfHbesmQ0?erc%yemJxl3Jq^_`Ml9Cg;{^ zVDZq)!?~X{wQ}U#c~3Tu$qdx5Zih{1N@ZlG6+K#(8>pb3$eIq}5#$x2$E;t&y~?YT z!$*@mwGGLhTHR@S)VTZ0FcF z!Z+*P`@C~_cJeqg9R7~GzTEeJ=DpxS=f((grf@c4XS>&jkF9Voup^)21FX+M&Y7?B zV|Aq4>-koAU>)g08CGxcS=u{|tfB6=6}|lm@6e&Ptk;Rj==G2(-Ai8CeC{?XhWn8UItSh^W`!-T4t6%OLUNfd=Kx=9HG8^BSms+WLJ+Y=8 z_O~gOhunF6lztpq-GRAhPXCGdyzWY8t{$bY!UJZ0Pj?JG?auEa@-;{6m|x9plMB1^ z$dAo!6MHvC(|+Z?zmNCFVsOp;9-$2KI{ZdG;nl0XJGjq1bn0F!yq`I7rFA-bgWn!PKCHBd~#6Eds@Pb!fByPv6ZtN4`oEhwuJnkzU zjDA3RJ;6HXQEfFk=%YO|1I<&hc^H=+I9iqyDAAoI6FY>ru~v~ea2Yd z^QXNb*d147^&MDbtyez-)L)d=qyGn&mh{D@edhq?_!xVvw{Yi`Kap=^zw^#ZViI2- z5-nDc=*ZuGJH)g6sVgt*FBt#BHfy=oPy3_lBFo&p?d<2|P&MVs$%~y+Iv;K-bo#fK zcY?uV>_^FBKTw_dd=I>Q%x^C^Q?WZvaCZ3V=kvX>$z$%0u%Grkf^4+gKRQD?nKLA| z-&QT@8~%6p+BE8@X0O#juPwO3>C0x?YrRkY!kC0>UU&N#+b7?5_dyN$v3>IWR{NyL zeg6vYV|c0PU)&?(z(+39y>L+{E_B*|UKADcrN* zGT^VBzul5@aJ|j|bkyVLx8XNu0JPrt1qtQX9L6sDCl6k>@iB82w=-vPz}*+R$9FgS zf{SP?iMD>ycHWZ(!~{}X1?;c(?86S8gm=f>GyBP8D|yKRVgeC&V9X+(HpCy&ev}-P zvncc``|JxI3*NDedh@q5kN?uw-af)!ZA)GM$v)lAKHb6oY zL!9`*UsCQT&^=%JlKG~CeJSyX+P0lP@Q-|VMQ11e(3gxw=VUp0jgAj|e`CkCqAxm{ zF@4Her~ND#fVTS1J>4I8*Pf2Dr=RAx_H*Ovj?Q`aFYIUSVF&+qHv984_OoECkTq}pJ5NgaB#?dpT_$bY=uN?thW05^XVUU;yUMqk0Z_Eg{|n`7CbCH z0=hh4Nj&;NV5=rqk#wR;p(+_;Z>! zH__mB_?y1yqD2p$puQ?-*!Q8o6WAj$+-!M>F?Q9oi|UfBGez(L3ByuD5pbh;S%Yt< zq=YV}?q4yMDr^8nqfWfRfw4v8$K*f{@LTgo9J3nH!bRv$WS3cl{H~F{i?$gE94If= zpK-i9g`8~_@37MZXFK;(?WO%+<~cF^RHweZoNwz^j)e9~u=h+tcQk-pEKsQ1&sM%(6jtTjhQ}b%on+1 z9?m!&xnYhyfZut(v-G&)GfVEJK5NWBhj;jZ_Mzz#wXQuWuQj{eq*-qI z_;3dFdOPsoJ>ZqkeM#>=fsUo_0<=l%vrD>k(H@U^+Jg_DeG2~waZvN`&xTK@MQwXiXLlfr+0-^T ztBBO^wfdgs-uPb7SEb(@60KuRCW1?ZkB${vQT59TT^rp@zRq{?Jt_zMu9q#f;Iz)S z@;2FAie78|O*t|<$rx2W-)q%8Ouc#K){!m9Ba}Cv`q}~0d6xG`VVdls^ag{_8!Sd|kYMx%BW6qXpNQ_DA%~bJo}@&0)$V@u2mUTrXFoaw&1-)D z;B3(+uO;8m*R1O-=h}x!4l8dnCYSzvB*{&w?^Q@GH)xhYwdH;r8Mvi^l&2#KBvh5@8|AoLy z^|gJLDCU^@)ZpfCWqsDon?)Y7e>d-W_Ka*S_Ar;Sy{J>#1KU#4W1Qo4y`Wo_2VVJD zDc^obD*RiIo%giE&FgOgmP=fm@8g^^>-MK^p#s-l>zK zZzg@rwcnzTW*z9?>uQ_+ZB$DCUQ?dy_3B`Uh-oVtmM~ zTV&l4a3pc*?3b|z2(K9&UW4t@EB%VYzbSU)-k;O{;SHW}18vsR=00%gesJmm=;rff z;NN*K*VaVt<%+ittmquxw-Q)=A~n1RnT}#Whne4X;LX>-sj`QP{ML$Avxg!ZNRvp{ zIe5$4{?m}>FZ%lH75UhPorM3F?T7hp+Qp%F+;Gir7N__g^;Yt|j?#bBJZ z9@*j8cIX=#h>JH0JUas%TZW!?8fVu?@bE}If-%;%I8PIlz zDO<{3NAEE!LYW?nbp~U-hq3;YvG!!F$&7WACwa!>jI}*u-R!x3_Ggs+DP#D8v*a(H z-uE=dn9 zJDrmg$As@9??hQJT1tAT=gr}1onE0|ro^-QLkpye~F67s*aR#ez=plNK- zS*HkRu-2)X6hWI=qY~27;9UdH{*2IjJsfx&1Z5y7APN;wRO|tT9-{G5q&;3J5E@%U(VFo_3$~!qL*`K{Euk<< z3yPRNTWVu6G~EG`c=p1^u^0Bc8pUbaR*}(ef6o3q^j7+L=P>cI=UD5sM(u&e$}bq3Q*lkj zto9YlgbztA(9}%H5&M|uD0tE|WJc%?^n?BM<2K+MkLzX*_GWeYidU&$_XVQTO`28( zqPO#21$Nt&;BQ&MH9hYFe_No_vL(IOEMq^i27M??_eU#kKhHatI(RIb*3*o=bm=<| zj+(L*6Hg!snm90|S4jN5t&INBo*z3}(;|F+!Q-Tk&P`7)OIfqx^QCp6h z0{D7#j>YzmyQjH`Jx%RhLWe(B*=(mwa?>hw#d`4WC*-UC`J6xa!nph9n7!FIG*$x`%`+@wp(nLe0*9n!-Dmg0I`eBK1swg|laXAR5%^~_{D@f0;cedZcIjRP=vYk^ z&|kLE`!-mYpxdsrqWger8~NpZkoCcVM*dg!hrjaP{{W{>@;a&4xzP%*rCsrJ_DM{; zk&Wnb@HyyU`Xnbc^Owf_X%2)lg2x{jv*fchwfBS-F8!SQ>AscSf7$2OCB93Z$c})^ z%M!XRlwPK>U=J?yOD_Yzw(_p>b`sk#&*y8o4LL_`Or|aNL3hXCCmHR;d7Z0bu#L$j z#9RsKtUdP-gapS{$KF;|keM5le6M}St75p-4VqiPJ_R@xnZ~(Xr5hQ?iITdc&XT%j zbib=TzLjm*U`jcoC*S;LV#-chUb$L+5|YEp@zqlwWFz>2aBr-}W3}vLFWTUA?U3L% zST|%-{xRRJFQE^}B+^l2g*09cL%;PL8x>RkE>c4@P>)Vi|cvwl?$%1}v`kH@U z@V+;ky_I?DEA=hTS!q4homzLQ9r~|zZn?7#nE6j<{$nth;KCr{HFnObg6A&|!Vjb~ zR9O#Q8tm=DVhZgGM>OuSJj*Vg=SgkxEq~I$_DH@}dxUQpdwE^=km?O{;dLwD*8fWc zFUxnRkuk@l#2!*vhel>>@MR#5N(n7}2zMu9%3S6G~ z%h8;ja_9V~?EiH>M%&OrswZ=hWGmUp#(}TgEds3j9+JO=@I6m%8ETgB+N!d1>Z2HaEW! zU!c8o_=MnnCts{c=#{VH4w0R@9b`3%+eWJNR#>B<^O}6 zcB=nRirKrpmp<#ABJKzsdpYMW!5e&Gs?cYvl&|wk&*E1ke;CO!SJO5=F;yq`K~u~= zXuLlR=YYS`ALg|l)^PD5WN*c|#1Gb?3(fIeu?_{{Kf_KgqFkNy0er)xnicdZL4u^jXu=(-S?IlYn1RLU<+pZb|V(n=?GoDU`|Qu_edD89C~dB!u@* zObCBEB_Vup8qXEhrJX}O(fu=ccUD3;6J4}ryPP zC3%l|Ok9YA^tX9*R;a0tIxkZ83h!+6gqt^e!Yk-QQ*{pe&gkL$@LR2;{9RADhH-HJ ziQ`Aoh#$#juj5Cu%1JwqdctNM`OX!=akMcFUMm^l2f$6dPh0zKK*p#TQd+lt$lSi}YqKl;=o}Bv3^gE6 zG_F#dm1*43Az!%^`)@z#Ty+iahC?;HCUQUO9_Dhum6=V$V%3{ukr;sL?5-c%KaW){vgU5i%-aM_CG&_A8zX$ z6%t>H!j~kwHO!vDes#Lt2+z2AszVo>M>%uzJUmEsRBq(0DS_y9Zk=CHrykwo2>A5m z0OSMSpG0bGQoyANUeF9L(AuR~F0I6JX<2_wsAXVgs6{;MKJ@W{@PUcQ1_QH0?eHkc z;t_aMk?6~t5N=IO2uFrY4z<9qBJirtbKqU@s7`p4_T=!XG;35Q{PTWz=a0dG{Cs%l zM)vw`0frupy1UtFZJ07>c>-nEs|@^Yj4L~iOsaDz^C`bp<@(+b=M0BO zYwTwD0QllNF|O|RE39Dgi3xQzbt6KOEov$<+e>9Pk8%!SMET|GyieoiRzAJPo6#;C zRJCMd+S?5N8`_WS`*`)eLjI3@k9ktxFLe9&T3r9sH}(B%DqGL~P<+{JCw_BYtlvYO ze%JBe*eBe6tG}jS^tXP*cGJ((`}kw}s5#I%#zXioXzn)Q)4GnhDWm!RBhXEK_4T28 za3}%})q_8Z1=KtwGt~M7|5tg!zBbl`x6q${Yd!hLfzDhfF_+@^mgM=;^Me`N9R15n zzSe;H!gS)&IQ-&%cWvz$8RMO)_Wg`Eehs+Ct~2&@9IpKVohe6dt(|Wg3H)4 zBNgcHvX`iRo*SMJKTWY`W_q5KO+a?=H%xnJ_IA=^?7>U+(f#Gs-W%ADo7LX5l2|7t z)+i%$tDTj!11|2Rt&f@KPoO*bNwvHeJKL3MzwEa8B55f$XxS+6)k6k8$uIZ9Pxoa^ z<6f>@`yu$H__1sNl4Hi1v%fjX%e=_MtT8$8Ta96rX*bJ$N-1!BLg{0HXi(|2K=d)C z^zBil^m8TYwqu4c6~3{v+VM{80gfwRlXOTe~|O$ERKV(^1o0YWmj1TE?oru3C~rh+{A}RW@f)Zz86;l zQ~6$W)pz|aRJWdY2SV@0&Vrt!yh7TN-Cla7mptQlQ8zgxyRZB%WZU=Brs_12e|tCh zuy*@M%;M=j1eC;pFhojP)(ozxa#D^ph zUF%J6DdgGj$xLo)?<`*7O?Uh^F0c;G;~rPzzrnMY=XK1*Fn6sC)85>powbfFhr4Fo z{Eg(}XHr`~5&DPL{BHis%E#YdcDQxSVWFG9o_rIZkF(f^e^&X}u(=1eR&scT zo4-c+Bgki;O8l{(n=d%XH)I`qRJHhVsIL!0TWv!uY|_O1!TwTf{5i4>{kIKv@|}BF ziTO`m#SoJHx&nP#ejTa!b*Qi3RXYQn`qj{At$aJ^>pjXZblaDI2fXACwKK$Nr-Zy( z<72@(PQLn44V{`ZJJflOJ~rYPu<;#i*PLfkeq-X5tO?onFUSj=lE=OE4|{s&u(#KjJ-&YI^$~Ap zaSC@VbPMWUE#394HQ<=*bJvQ`oz6LmpBq~BIDQdV9QoYr_4!yE^ZEt;84y3@J0}*; zpv~#DeI4KU2H%>-{^3;O-**qH5319_9VnHYRjh%x>VGA7yee*d5&Kj-6Pz=VJK2|e zttY-Z8V9=O8hDjxV;gW3JpZEn6wVi#{0v*PtNfkhM^XY&-J?)R*<60L=MaxZG6e5$ zrk?yxI(c^%-|wVe>>NJ4d!=mP;@xeDqV){>Me67I(pzqWXHF9Dy3pBU+Ad!C9rm!@ z`=h?acV+u%qtEqx?`Nie+%v2cT3>F`G?aVOv* zCno7E3p^&`MQ@mq7;1xOK}q3=*ZNgthp)YozE9Sf&Gkv4ruCdH#l~^uHT;HtZ-qN% z_w-}i+#uWLSbXj#Q@4OSlRDVTLDsCY+It1lz`^s$_mXa{>0I^E_`so>@0Vw%U5+n} z`f(xs*wM3No$TEMMg?_-|AK6%&$H>X&dgivWpseE!rM02Ry5NckNC%2QvQL9JCYAmnfouylXXLdtXP4d8+!ucz3@hpBG19W`mLebl7Hvn9QO@qU$D*a1&yhbQQ+J;mBOhFo~W zjkR?;PZ)Lh^Hp<+fhhUACV@SRw7}!Z(D`J1G$vKAyti07s}Vu)*utYH{I&3gSSa7eu2b@D|snwyv1yshM^P2IQt(^TX!WYPesIlIOF zUyjZxBeRC6uQ*oE!5dT`89Y_#Go;7iA!>6Sb(9AWYw_YE6xOQlzf zPt3ATuLmITl6;6l%*1L_8~)W`}kk#-TuC>-}d*LC~r=b&Dc65 z*cA=jxjOXuRq((bp^mZGqs6O!>D&R(fgE=79)MKa@ORN4{jeCPVn#uORpTESVrcX& zVB^>%Q|)ZVuQf-w;d4{P%+!CusXyiQz^1QK#@k0J{qLoeF(xZT_W7(><%;|A-K+A~mppN4 z+vlr3Mqb)O|D70&vNI}kc0RGm`?H=UT5tP2J#2hgngh=IT@|-}S6P3( zxGRR-cY&>J(PO#yO#7PI@ZMX9%M{;+2l=Fpv3&Bd6_Q-k&KUP0M~K%xqPfX%_&|ZX z7QaBAa6ord%D$W9=C37R?;M9ey@p)!l#xv`>?-D2vP%;@h!_q=7O95cNS;_hndA`8 z+(`~ObsSqS3Et|;Atkh}Gv2Lr+XAG8)thOS0KS1Z2u!Rn#-VSQ(5y5Vn?^tp?-fp@X^ z`k%Xc&TPIHKo^#rCt2p}B@SP?-t^fr`k0wpa8>Z({J`Ti*g%lMs^mwTM7?tKDHc4A!(G zxqSs?@^dmgmh<*UKEU5ayxGW3y*nh6Mc`Md7ksd~X`dC=cx>0VD4Bj~Z43VFz;sB3DAmUnJT`4{|ENl@pXzB7J+BhSNw2{;iFH%M^lIyhK{QHG0_oC|E7qy z8NQHZ|AMi&vP{+ZpA=J`X)gg6jxa9uc_;HI{`fnSW}5g*ojb(q`oW+2!>3Zb^vsBRU4%Dn?!-GO*+q#r=__#NDV%H)L!dB+nt{d-HoW06~ z?7(c~o=Yc4}h z(0wvB=-Ikk$(;FMzRlTIc#e-a4H5W|^!$0yh+wsZa-;jZ`ix+Czw%g5q~p9nTQPd+ zqJuYw8$Kf5pfQ}0RyGU^E_Y)C7twds)7gp1?%bSMd;{>mo_Uzbe9T~8rn7EbN4#D~ z|Lfud@Sr`6*|GU%+sCo@N=CbpcEmIDkx|48{NtVc>F&4kv8mJ`??#fXVUm~I3j6pw zS*u&;^!2wb>*t)=p8`C~i2L7?6l$H3=Fit%-m|R}l4&AyC>uB)-wb>{Ip-hI`C#Oo zNh)iE9(VLbCQm9%TEkwAnUiXlZw;heJV5hQ z4Svi1s5x2VmgiBfwbJBKu5m=zS2g{E2m11Xso7Wc1lza=Tl{h2mu$hX$N8 zmCjyWHt~#{?-9@DQqc%90x9ZeXqzbFmaUl&{c>`1O?%w^Gkj zc)s1s88Mz)@C&&kF%X`^^GcpytPYHNsdZdvKkM$x@VPq|m2>A1Wr=H7m$9$!!(W1Z zrnh}be_XA7>`C0Ge8BkQaR28u&K&=mz6dVjgKrofrC7;1w%5I%?q%{u(^v5$N1k!? z%~f4>fibWa4}H8VPcbVUx%AOk9=uh&a~!;seXdZ;r(Vu1fCKQ<$fAUhWQ3D37PIY6 za7W*)q<^}9%J8o<+{qq)|J3==z23Rsz?u7hNH3#bC+9fBUTDhF?R!j`X5VGfRJ+Wi zDfaCq)qPY~Gv||;`$^!yL~!9M==)0OJ9h8)m+vzucC&CqaomL?ilwDZJE3byXmMu-jcbwSB;;ryfKQ@pi^zG5+ex)>fF!0tK zNcS_c82Iot^74EsEjs`91s*!{GGj}SKmQlX#*8#^oyA`@rZe)_F{6Udy8Lz5C5}ER z9pDhp|BCE?lE2D6;M{d4yJ_`R4qjWpNb4!Gw85Pe_Kp`JOLf9$u}c?==l&_+)IH2M zMFZ+OYzBTYg=&VFCNj7j3ohpt3dy(*MCnmFr zIpFGax%LaRC;nY+(hU1qQo*qV879ZdNp5A%)YcmEz3zNFdFl4k=A9IK<-cHmadaui z{&Ez0(Y*&T`9*6;JiYw=v(Hx{2N#Qn)+A&YdZ~n7z5~5n4}FZ_m&LD@v!z?$i(5$l zjlOiA7x<@zepJN1aZEp|LGJWT9?h9rty4Z*apPp)8|?O7-x%Po5rw3Yr8-kIt7kIj z2wsGyBHr9VNmlveet4|T5>%ka5C>?`SYpiG#h2*(MgR&1`tqI24NV*0l9_^&>pnaRGo-ibCTwaOffAc^Kh@-jCB8T=xt^ulzv2Jle?ex6-M1Ty zGobY-c9($quX&2a8Sp?~b2(GSIsECfNJqmHQ}}f-H_H1cA*1a3oH6(=I)wh$NN(}? z7Vg2G@ONl%8|BdB%=>vR*I1DMW&imR^DNo-r{rl3H1fTh_au1^U(2z}-Mq)h6KvY@ ztSE!7eSqK1{5X3u$k>Oem-lt{<-pg}W&dH$LqDKA_Ug1pt##2ij=jLYsVv97*DaGy z@CE$txT4Fp@O8ecakaly0>9)wX#NZKx0y84o@>$+`zlh+h3ZP)`zo2CeyArn#Krj= zd3PPQ00#~!MkaK4P_DfTI&^q=j{RqzJ3Bog#rG61@0?*ha15Jfd!pIX7^UyFV@KHo zK4~vb=V@g3_!vC{LH{|1FWTKwPM(FRedLt|KK|Vf*{WWiJUr+giJ25S!iz=pt z?yT5EJDVHFhqQMse%O!p@KrIm(T8V^rI$(5?PQa}+fABj_aMC<-*wJQjuM|(>&+_j zF~`eD#ZP}tSq(CCD)LVRpF+bcfyv<^4o$V1G~GUA(p39XrS#{3Qg~{M$rn$Z##p8@ zrfV77HH>iza^KbL>pAx%951xOiYeSm+lq6}JsHtWC4o^E`$pD;VZq2?&H&L?Grl0| zL(#IV8TivLrC3XK`U!Mzo{<57!AEHoD#0&e3SY1U(r|jF3(Qc?ZnxIR;ahjH*TTq zmcv(=t#_NKH--7%dAxI&`uH-my~jHt)BsHw zA6;N8Ur)u&-Ur@A&N23auwd&W&h+8LK=l2Coy9$F%9s(Myt^lDrSL)gx&fP5BRC~` z`dVLy7nLy{@psv`F5@SfiS`rf^7LwB7sbwzbaCbC2I^FxLTZoB$n7)cO8(bUXc;F{Ks&6Ln&1>Nw&+TyhJH1|KA7UhV;%ANAku2NT z0QbL%HO`o&%jI?DNxySq>}SXFz)gbHjvQ&x_ARb2QWoh^@bew$!|k$?{_?V`r5d`BUE2TJ;*KzSFrw_qLw>>~YgPcte@RJ%wKQvveEh zUiiCQo63LFPC4h?wQgiewj!SER^X+&zo87DeEe#y)J3ju{BOzATEZQ84lg)df?t~T zOAmAcJ?&<1a+m-0^dS4)(Xk$W1^C`}C(m>A@oEGnWPzBJ{6g0diUyv^ZP5Kh=JW zKC1tZk~Ym_{*fzcxFba}g$Fsq1{Xd`$Sza-hU<|F=ghDcCh*+6O19xF`#a!=r7UP%rTKPlX(!rF6^ z-9V{X$I|X5MrZyA!EX<+bMET$*th&yXM8Q}Lsl5xe(%lrze(Ow8y}sQT{ek+tl(F) zyfXXPJ>+F)mu3HexGKk%)mhk*!ExeYj0)Cs{`Ag1jy!*ieyqbLaBK#+gZ^Ig$%AzP z`4QJ&6ijmOpgaaY5`56>!pURHr%$0=_k8*<9Hbb48|L-BESNve3KrbjEtuf>_OFV# zO9j29_!#lj$G`g@bv14*>Faqfc0aFq@0QDg*0UDzr@I9kkms^_Pj;VXzNh$9z5?IM zv=8sEi!5@^49kY?^EhYOe15<4?ONKnl{S{s2Ja=&25V%ofwMI_SkCx>)ghl7=h>W# z9Ktyz;`JPx5eT>Qj)gsE20qUL{IFlU0`C~o`U-Eba*P9`U(!YlKVHx?@^>0O4UtC9 zz2di^yC}r7pH-QQ2WxoN9j)ssvN~k z{xJOFx{ruOk-&XpzOuT9aQ&l%{Ds2vc>K7U^L%l!&a38~f+v~wvKyEiVtmE=;-mlU2ZZDM)Q!A< zxMRB5?bFTfTx#x)a!y8b^BMD!eD9LSd)|9){}yl0DR;8wl@qU9>2l7^DED#jX zQ0L4`sdE6DJjj{BjkzyOeKPm?sl#~xKXR+5&dPmm>PX_IW^rEU<|~feJ^PBIci)_| zX6o!KTJOH)inhBK=R7lY>kqfw{hJ?#?>;f+H+Of8*?RZlyfIU6&bxf-gE==(eLwfX zsjqflKIWYt9-8_jzeW6x^2>a9!_-my{!+j?$CayXVfnh(gAc!Q@!{>S zgAbH-P=ZU9w8Tftvs#omCnS-*}u< zv~#2p`{$>`US~hJnLQ}QT5R&Nk2UZd_N)4ang;d@9U-DM-|KUn}gP5@;`y` zoBE7<0C2vI`l{>u=EybiWfLfyO_}QIzGUrV%qQ=L`1(mF+j@wyX_S@9ce4BT%Omv5 z8dZL|pZpN9|2y>mlJd(#I@9Bgo#{!l-w@4dF8Iy`jPWbyVBOj0;2Gv1pS!w`8vd1G z4|nI_64ImMzkK5&o}1x!M+WE0H)@0Njk*sXO5~m-YdJRTb3&43@Rf=lA>O?&04`>D z!u#b*Il>C|VqP?NE$9r9SGWg!l@-MgU_<`p_yMfJ$Lf`oP|JGWoxxnr@q`~KNol;)9 z{)zY3%p2~U-#P^cGwgyc9L%sE0_IU4QANO)duox13d9*#@w(-G@sRVkI-_#6`GI9 zP4VlZ;H7V=zF;(f=WiLf<=Cx^+rbywW)3D$_HUGJC9e89d_0<<+5F`{9%K0H?Kv|P z`z3$*BV$g|Z>}(_oK^maoE>&A0Xxp&6dy74n{Hp_!fqU?#&87q z!RJ#&$79ITI-zw$X-q~x$~c7|#d;TbU8)qixmYPY^CD8M>m7pk*g$j;|34%KaA$=@ z>_rbgbXG8eE*%;?-rqFXj1*<$(OjbYggkTZ7+$4}aVbsjv$ z=DgsO;G`|TOWEIyEqJTP?0@Zn9%o;6(jF|k#R~>zoX=AVZLcqi;5&kdZ z|0wi$(e*-bSTc3F`iE}PJ`3MNbbImGJKVAdREC{p>wSSyTNVXIwQaV-mzg@*_N{K6 zZ>!E6E2_O~`Nx%znmaSp9%nY#=Q;I#ow1;P9yUG6b*X8U+?#G4>ez^#KINJ0N!Utt zX3~c}Q#^U6><)7zo8T*M@G#51ns2Yc9?L{Kx-xJ9C)i zdw0gQcV}FCchcS;f#Dl{TA@RSiRlYtB36T0@ZuZ%6(_#>4hx1@W&lyzWM?m0HBy$vz<5R(Tz|tnT@1 zW8H{-$F)gS&0Ciow3gHUv$PLA%~$*E8^`L!kHJSh%OiP%{Xbw*&zLOiR=bNFyH#Wm z{ur#yis_|2$!)sV1{?c&&KZ_o^> zjqs~(z{Tt*AveiRG(Oqfd0u=(-==+i>jQlYo6Vv1rNT7IqE!k9asQBYe4hG6DOF{$GiG zMd?-8SCYZkiP%eibbMJ|6}Fas*jW1WzXiOK9jz|+Kd07WLrLR1Ph&^94;#vj*ic4c zL-{KE$+_51a!<3LWS{ro)CZ0IWFPjEeb`U-VL#c2{p6R}PkuS|5!ZeKPsBc_d#Q3; zmJ@gUQShxEnV*XspL|Vo=<>}K^m}cgweXE>a|hw>KO*b>iI~iP?qgl|m;TmuZ>3w; z_06<`Z)aQA)kDwtpbXlJ%&)U1SM!a&2}vv0TFEVQu^XLlzL!wuy%^i|b9wiR57%QG zoaagIndkAAO(+UH*qBnD?R|qXc(O%VgG1*udNi2gLiP^L!9I z&|Mm>x$6xcT>pQB2ePBRZ}8wV@ZdA>;4|>xvo1XN)Ww5?#Kh6}^E`dZ8VasH_SqAU zX7{~+$o;yf0~^|PYrW?f`#@kPKb`}7uLE=mBBiT}#;7`d(o>9V~7m2YU zIYWL0oc9hNMrJ_fFzfx1xV2w<81u1>hz^eqbo@||gQ~2|p20mE17XL<6?<@?e0q0& zJ%C>da)9c$P~V)Dz}8!Pj#YaYUP{dLTAjg_{B;{Lk?d{9;gO}-k&>`6#PhyxX~dkF zjlB&Xu?HDYezV9KyC3r*LtlZ-Z6Le?z2~faNH(`K@S!&7MrVxjlHe<&(HY}-(H8nF zUbK-^{0n8t#5!6DpAlaDDZ#<3buL~#>*CdOE?!l;c=bGZW$??j-wLl{_S;{AQ^Ku1 z(4h1KXWo+BdE-v3zje<13plwPoU(-Trye^54)(kM;`?>C7I+8__BA;8 zXXr2rk4A1uRjN6!_9G*T&+lYS?P$fe*@?_A8(GOY?Mda>opLzqG`KHu z5&MOj2lo#(4#gfemY7N0k5_~Ztr8m7osf~Csl;*@E~VKAUEkEH%$t1abO!Kd;TP+k z<}+sFyplZOS}M54y1gM&_N-frU3(Vwz%SW32D|l6FO!{P0A*^k0XkCs`Q%BySG%gO zF(w%MmDAoslxI*r2_9p8xP#a~Pq;CPQroS>xL80XUN>G*y1fz}Z}6V)Xl%lDbf2^1 zx@?=^`f=c_vjE-rW^CP!;rl$%4e`vhrxja!JHDpxWZFkb)o-l{T2EvvQCoUfWBrtJ z&70b8nurbnT&ov-&0k%SvSnLI>K4u4q}O}3H!&}vjm4ptD$?2|E9@lhw~?26kJG&g ziovt{V%jz|2Mwz2W$xU4#5ZI=kbSy^_3b&}AwJ>l;n*tQA+MNmDK7^50sP-izR3gM zG#?Yl)B4y{jo)M8Iia=+_Mcoh*J8gDPB?L)a_mchb31)dKePw@gkVj->8t5?YH!ad zgS)^?<39|nG#9?J%!Sz}+tAa@g?t9%=RrItfjQWoe(D^oap&OaE*PiVEyzoaL(Y?( z**UAf%sNrpWoPail76VWB|CClJM%zn|I9(teGZL?jt+>9Hun!5SWA3P+6ldC=88&=v3~8i%c!d5sLs2<_itMHQQ{@i8~%7_p@yt31Ey?92+`e{JZk zzdL95(r3rVGspgScN{v?z8LsBw3pdsYyNlgw9drz_Xc1u8>058HE+U0**>+VXkM?T z{t?+-kkfOZ*OPMkXqR4T+nw*R)Kxv-b*Glir=5|MsqGlfJVf45@|sw`@ZnhaAN`+N zEuRK2enslr3dI+}Hu9?5Kl3j5@jUNczSUxqeyVh5s+US*D-ZmerSB{IYUAR`geV^Gp5eN)q3e{r;2tkgLH(bG{V5 zN6csZbvTm<47PwPzy4^vU-tJLGnQ04i&S&dnUG$#{nDEs>^3;?Aa;S7nkU)j3wVy% zX1(xQ&2@j?k-yq7V~dUoqrIoU*l(CL*X}I&avwtVxr4pFNwTYswy+Vux(2Vky2{2ln<{jUoobh{ zQ$c?Nx;wP?E54E6)2fvoH6CAOY*SV7@wMgW0)EDHo2oe5)UH!(Q`n;#I3qNdeHWe2 zjjb<3^4>?PJdZ3?a&tngB;GESEf5^5I!Ke4TA!`!T zeG~lU*U+B$`g~v|-gE?fd~S3gEdL^%hdBcLo%ge)-%WS?@QI1klldeDSh|Cw)0Nh6 zhEw$`k!z=&bmGgiBeoWJ|iZheH=v(Fg5|YzJ>eYwFiX z^6rGDxldzv2fFD;$=1End-c9-#?ptYlbrX5Q&)V|;3Yb+{DBnLvXO7b>%h~Ko%(~Q zukS?RtyzWO$@$PC=MhZbINNiK{SNt{N_LBs^fdZU^aIWzFX$DD4(uI@F6a{?CTzGV zrYE5zO_KhThyF90SY60~nuA~9b11s+!}pAj2fR9)^}m7hmzv9%{?nbjqs(W-wSC?K zemx@{Xd!ZpnNMdwd;ooJX8vRKPA{LJaZ|3grE5vXnNQw_r@v$LAg9fTDE}Mf$Sk`Y zJ?N=1j*myWBgf$TvCjLGF(LNck22=xcHrxQjZxp22;LZ6rOQa>mS=j;=AM+jr4Vor^T}6*ppzb;4H{^E>Gk690^pS74aRvHQC4`?i@2`N*c*^LaLS(K8TzfH8OBpzCuopS;)_vWGFo>i`Y#alI?p zUQN9iuBIdZ-vq9X60Ux0_$jz5*p8*HaP{=E`IL>M%=E{_(}&0#8p~t7XhcSRp1h0V z+la&4Hz^w!UuNdj!Rft}<*TiU$Z@P?FP}D^qlwOVWapPmpXPoyCe!~x?Sc$Z5keP|G7+p)hJ9l{Tc}c?OB^jTWp7^}w0S0 zZTt_h#`~FHduSkhJu-pKoklN$qc1^EkMO??IYaMg-u5HUwhZp&Zz}BVPZyj=$LW6P zLowYiofY15v6G)nzQM6>R_%)==zAjqVH@2;G24;%x#Pm(4jTCVE8YE( zSFZ6Bf5smvL>|Eg)>i2CH&09ON9IA(i#&d_#&|7*cW2TG*H_%M1$|HV&=fz*nPfvV z8yuPmUUHINr0W_SX8gk8_kf3R_#Km`+OK`3ewFn0K4MUQ@^J6a-c^0LL&S^*sqbc&StH9n0n~5 zP2O=~ooDI5=i=ykbhufx&swcm&11c(Wjnwd#dfa*Z^ojR)z8D%`<_5_9QSLh4o=Lq zt*nE~60*vk;~u(K!LbKZi5~=BDRyZNXAV2qKULkI0gDcFZ=Erh?yYZsTsRHi-UR=V zIq2XE#Gi=Gk}{Q8kDM7>KwNTo{=Pcmn&4BWG0XP3l6KS&Y*(pK@KUl&{e7%S%=`V| zfMmmmHD2^j#ox~-|6VtLx$-MCuNj7asP6ZaS7Jr~0X!BbTkurudv5*js%$Lv7ty|# zSgLORJ>(mk!)uP*EMA+5UytmH@W)M4>04*)``n+1FP!=2H{AE@w2wnY7TXen|xx@ zu&wG`Qnlt5yi8xhc_r#yLj61|z2yk7-2q?LeTBXy_tm8ryoV1_hV73phwgKk*U0*- zxvM3#CY_jFvG}ri)YD#%^lHIv?vc(-+AD##>?XFIX@945V#?Id6w_{6%lY)j7Z-m4 zy4bHJRwh5zb|H`5lcgI_Pri`S8nCwozEE;{mY`q!QZze85`ov?yA_(w&~vx1rE%caY(S)eOVrY-rt`xg#U zTf5||5w9OdIR|8+A1n5Y!4G6C>B#YYkY2oJ_BW7^(F?()y~xQQBWFr4PW1o_)}~he z$M9Hw8=Pl|o_B_}vWO0!o0 z^ItLVoL`SVn*gqhhof&dtdg#+Gw;NSroB(#Bm33|!XIMa4G^nQ_XCSBsSWXIJ%5|F zO2JheZJ9nSwI_KqR<73+Rgm2t~-is|=@#%^cyR62tYGs?*2R;if)#&k8Bsjc`3%s7JN8dr_-lTqA z#vY^2fy$1L@6^4L%k-Y~cfogKcjvvUc+c=g(NUNFjt&5fw2 zGAA)^*^(nW=zAXPaCK(r7%i|3cviF}bA!#B}<;hO%8c+}m3Dd_O+!0ENd;h`<)@G)53?ZWCWwACh9f!puXH%E^rrmVY%_+R8X@K6ld zleUgm$VcYsvbzlehB3RFqsP;R3!~?$t37J@qw0)JwA$$LyR5h|YmOW}KGn|b8gr_> zn0_7sUz(Tk?g&rVz)Jf3S}X1xNh+uvhaOT5%yVrVR-e>;!hMxO`+Z0a|KvcNU++d&>y96tQoHep*Zde(f_hXcq4k&V6lEid-= zv!5M)?L5Yh+$8&empD10X(x59N@8oCO*gu8sKY=1!29?>#_R+ejGaI-AvnL4GRICZ z)&JSpYyF4d>wCOchHB6wLKD%IhB~@ZE4q@-;OwdH?r+-B&EK)j>pxhB4dF$P|4>i) zeQMqP;$GQ*GT!wiH*NV7vO2Q3>X;`i00m%qsxZgeo(Qorw`4atz-HEF6n z+oZB1e0h8q*N*Uomb;U$DIO0^;{y*9jcKO(ci?=DPVhmrp54WSo+luq5b2K_1 z(T;6PJ{gKX-Nv4<(NzXnT{Q66ho2kSPH@37tM;$_6bo$)zX$of2(At$4S}~!$Z*kF zoM~7v!C#pue9tNykG)Foz6?!lHbD1A(fi(%~X0N$v8hbj#)Ik4ik-gyy zc+=jaD_-E6=3QhxQhi^%vvWs}FO~UqR-lt}06xw&N|q_=haQK_I|&=6PdZ(p@EyGr zJdwWm8ErdjMvnatH?KwCMdlWMz3b+w5A{RIn-YlbcJuZq51OxM-T$+j_YbvK(Jl0Z z^t2IyC_JS$GL-dMDYC&%^jFy}DwVRn2p-bo6z?_#T~2*_gSLnr!@golO9?O?x=i^w zhUS%54cw$FHWdbt0OpHzzM__$Km;QL9R z0M}2(aOC_1vWwYw%Hy0Gwn)vvW7L}qJQQzy6MVQ8++Os?g~7SEl|*mF?q5)u5ZwQ5 zr|v^)n>j8`0GEN^Sk5qP2Oc#E*05~}2@7w9uS$P5_^4;#o9FQ?d=?yMPrS&9GX?tcJ0gs(a?V&F(jDk~}QpyC)r60LAPaDs2|ir+(y z>BGpRyJ9kF1?|tv^XuqRNere7{K!u^8Xc>CfvkayE?%H-oxP<`V*Zw^5MTBt7-1E&sNVF*ZPV_$7udd zT%L-RV;bQpUkqt>UBj-JjkM;d^X;hcmzL=)C<+{xynO;3rn9 zzaVMl`G=O&nfd-6^L?um%j8|o%dS%#6239Sh0|le$>G6ib_LHyMm`7r!Z;jXJjUV1 zeXlTM%d|bDf|;p1(1F*TrYy%k#u&QF;7fwZw@g{K-5Os8@0w59zfq?7Eb46y*L*a) zbu}R>hSOKhe!G3W(R?%0ey8i3)OGth&XncY zZ^f6neZ9<-W!pRB%iO*WrA&SG={z}o-4@r^J}w@vCOv(=qyxt0E1n)5+~Ev3aLA=? z=T21m$=dKdc^T@ej;E=A1e)fV9_Jl`p+xNcrXLc|~Wq!xG?u-T>wg0MR6D{itq;C)owJ5oPuL>~ z9i=khb&opAJFYZYk$%LTJ0xOo=m+YTa z#q_8%tRZ-#dWZeUSCa z&lp;ho%gPwEa0btRia|MEBd!g=;TbSLlsAP$^qJMKKYbe>n+ABZBXA`|SmHoLk@H~J$ z)!^tl;i%-v9J^-FNj#PRF7nWB&C6ltMRg8lYi_da{U%MbpZY5Ot0d`d#dX)+fx2g7 z?_&BrNB)oaLEfW1JMN%wyvncrxf=BB$nS~s$T+cuhHJo|$bA+zzzP1E1tosn!BErF zU3+=|FQyME_Mb?<;|)X=V|N_-)hNy{c22hzVr!n+mP9;)SkzGCa1Fdgl8T=3PAJx71U=R+A@O3Ay=O^t}bxCGLaIv6erBT}1KRo^{JNP%fEf z8E`8`M~r><`MB>^4spIajx)FVE_b5U2Hp49x!=tuPv2eX=C6+XF1C*1T=o|A-3qt- zag|SFjAg*45!(xO%2Tb{#cuhJR6d3Bd)V`%U*C1}A5#8o_LPwmi-3#v>5k8^q91g( zHeG@IC0I>&(%tVLWv|X1`=s{HVbKM^3_Q3?ZMfg{C0WB_u{RbdpYO3Zv#FDB%mOB~ zxBKSk$#!-|+2b?quunw&_P8ob$47lir=c4bkWN82 zR63jaCKbE`(7*W0`}yR92RoN?7G82J`nB{oZ$5tA#!ofd9!fpM_MexBzN0xmmwgDO z?EPi3_ctb&Hkk)M^*&;J+LsheiZpi8LZz`eIoeq~*JIVDyRg52GVMW}YwD)knMyT3 z_~X_>Q~1WyR=S<$miKkb*B$Px?e4abs=PD6Vh#8sSWF@v#V=j|fx%AZJ_dsh-o2Us zf`M0sq4+hvuK8Uur+olhHG;!dh6P_9T}OXR(|*>jn^Kaj_N7A~+&#@3Gfu`rL~nnW z*zo0}I7c*)xns}L?0-6Ss}yh18t0y0pNKyrb@N98V`xG#6EhVvaWVT>_|`@SlIJ=3 z{toGtr^UkCs`rOm#oMLRa`t>E<3XM|v@`DPxrv*&lC`ITJvqNMGZ8tcyx1zfhczWY zTF;$JV~U(L_FKFkVXWfIZR~yB$p2#W>&U!u;mD=du=X7I6=m0}-YoX7J=|^Ov7%G? z|0H@3z6vLb;Y~3-Dsk~>5_q&TA*(0;vc_f;k4s|>9%nj!=PoW?CR_qO!ujum-^eH% zV*0zmkM@+{IPM~p?X@-2!KEV$CWmB`A533FqkfZS*#nhAcNZv~LHrMspK2GFl=vTR zdag-xzH-l9_?iU14ob7`-I;9lk6;Vmxin<<+;_(Bxr2{I?q2pz_OeZ^kH&7H(q$*( zd_>kKhMIoQc@5$?bS~>Y%GZ}OjQu7Vn}gQPJXi{zs^P} z4v6+p6$7LcIPA^1(Z9_~Sm;w64gPQI%QsJMlrx{xD-GKVd0m zL2agDfj|?o%Y6haBg?KFC7Y4#Ah&X7kMjRPes|`?Lw*Btve~DmjgNC~^k2^Yt?JA6 zm&6(7pTH}B0bRciP8xdS{jpJp-u_HGqNOD2OIFi*hz&Iy;rogSB%5Et661TYA>PiS zbw5Ub=yn@owiZJZ@xBM(=TYu6*pB{vTsAb9{=V=|JRPEUHuFwCdo18wxX5b_Zy!rv z7>}>O3LZz^m(5J;?t4n%MTWkJi5W>H)+;!>XBubYSjTja_MXjdtXKFJw0Q4g>_>02 zC#3eaw0CY=KpSfFOnqyFN1YynVh4Q-;FqaBo#~yfI27R9J-|1{3ohpQjJ#kVyx>K6 z-7MxE`t%m#H!;{MF7pGoK@L9{Indz;x6(#DKbXn?TEAuZL8)KyC{pd|@)S-n+Eee z(oD{krC4TtT<`XAl{?3VkASn^1+|yyK^TN8{RQ>eQ&wBPbV{rbN+sB`f%1#~YUmG~|E^?ANYYbm* zLtdyE$oPO^DXHY6b;!E?-FZl1EsTf%ndsn*^8h~?!#o7ow-qf!patD~aW;DBng_;n zIz9Y}cD92{|c&Y2s<(Ks( z25+q!r}rg9U*@8t#rjd=_Tvim19)nE6Wt1)MLfsk&EK*%e@tG@_rhxuxVw8V@}B&Y z8mlLUy8M%vn?1-;x*s8nzUE^miNWqm;r5Rh{+`N5meN;CJV8YudwMm9z9TeVJ#2`{CzS0hiwV#wgoqgbWe103YC;4wlw~wvyN8L5P zo%Hq&r~Qs}$L0?2HnBbmXxq7;HRBXNsZ7>)_2)C{>pO?bEc_d-=-+O(YS;18z5($A zi<_afX7{`~JoZ;jt31a4$@od-U@sh!fu9uqD?_qEG5?h&WX7U#-P$MP6Vx=CwGvuw zD$EIC3xLM6?S&UR>&M%CTkE~zw|~eSHt?SC!t2Jh-AUfjJop&)LB%m0#@g=08F%8C z>fD8?4@_h)e2Mz1>$^S{=k(+`^A{*n-B>JB*$daj*H04vaq2%r*-t4elH5$(hx#d; z=UC&~L|tDYbLOWa<|~wG_jcD1`B#6N@qHC;89Gqx2k_@~I@os<+?ws+R+ho7EV~?7 z33iJ~k3oOx>-{|c6aFI}BK#N5D<(YiWcDd_wyu%++Y3&8f?PH`!Nq&{qIkyH)*Zty zy71nmaqv&PaJ%|;8s6WezA?u$NyYPoJ4N#QLlz3Scz=WHfMey@Acb!q5d%toe~s`L zqtg&$A|^N05KDY7a;)UnJ>d3s%>!}88RztaZhPbPEpX>DwF~|X(|_=HEwKnqY;p9Y zSZwiI(Ge!`?h1a2K|iOv6JLCeCx4+&`k`b@t=*n4(Wk)mQqJP%6DwWgJH2l{#;Q1C z>Z8V_7~?zKzGkS8^i69;Y^?Yt?k=KkGI_{|;RCEOqQ_p;6-{(A{0=@7NF@A6q;kiTKz%iH)o=pOuE8uXq|h>iVF? zXgCBuiH7S770=kA;pMKab`O0s^vk+-0Q^6le%~Xn4Y*c9liAvnxa+c0;y;LHw^2vs zrT0(lkp!!vm4{|fw_yOW$X!z}BCoV|SXv%1>sX8UZnP)q)9y$Zhb|RM;UBr#kB74?g;S{_g z!=0!5_^xc64a`#oFcKVO>+-t3m3NaTpEu>j_@n0LM)FM_{6qUinv)siHD#!u;_2z3 z_Sev3MV~HTl;522!11BXvVU*zSpHR6_8`HTK9{%i0b^fwxC2(Km zzAU1l%0r2O*ZD_uWUbWZjEBbQ#p2gd_%(PJt8@u$ngr_iI~&76TA!-pOD zUG$i4=rP@%&fJL}uO(G~jvU8DPD)4Lx7`ER%kn?xRp8*~> zLg&}l?*4q$C+ZjDY{h3`6W`xBqr|_pXF&4&!ui>O2j}Mm{y)~TxQk$S1x8UyMkk+wEmvLu+_`!mmMST44|@9+EjV_);^nR(`!IdjgLGiT16IoXNd z;oQ*47xl{yom|QPP2}CmzL5GZpF(2y$NH+jg{;@&@tua8ficdg#vG(O5%b({T;TaI=**XV801%| zF^%8Jx`%O*PV}rv(_HavyI*(H_rS-bC%z`cRGHNq-;+N0)uiE9(G&1vNxWz z@e$gr54rXX;&fOF9e#3vbLly(S^1{ukGp+`=-$#dvz_W^_^ss^Nyp&YEUs8~AP9+&R)bOXnSO-P@oo$=*f6?@}J z?3N#~#@AgMt<#u`*ay1a<}V;$zKWXb0tvAb3?H?0>xi+RK{@%L?DXH*at~$o&NN_E zwX=4w_s?oky|P17=G8bX%DP}Za{Kf4TazfSbI+39#C7A0_iPjU<#M$Fcr!9ia?hMi zrjMVs?~T!Wl$ZRjf`_VjuatC#_SCO}ua)xde(j|r3meeC+6HlM9302c6`HAQ40o6d zSN*B86d$R;~8ddEfXWu>Y;tKEh=m-ih}2_9rVDkDZd=lnX4~xwc^# zseIPK?RsQ@&JcD{dA=!}7=H%FOgNDZrHXsYg)i(T_|LNsM&GpF!~W2;U;E*_cU&;! zmEN5|n{@8N9x)}-?D zngOp(hv%+?_pXHpr{U){6+bV0zx{JxKhx)3`V054g6I1_VRF%Q72=2+%S zt)HQH`SN;qL>^{5jZGh0nC^#MkmT(@DCU8YM@7BNx_IV*)BaSthga>0)3Wd-zcYQm z3SKsJpWZWkv+xpYuU0?4X8ZAzgno#JI{JaHmw42)aggE!e0m#uXiEUF|_oyw*TVEqxB{^B1jGW|r>g_p{DpqSx5Lj+=L|M*|L5EEPqa7%T9jNUhpVm-^v%=$(J->p#afh97A>Y zYuq8!+qu|}zd&F>XmsF#+&)$K+nGMSk{#YgUp_*9IGvoGd}DiH(a>;8MyO^LcWwvx zR*6!bj|oqz(Y`-=D|qbYwtnK6_d?p^AU|i+jsz7$&2sFJDr(NkduGlo05@5 z@ONEWCOu_!AuE-0VK~fvT`S$`2>8o#vecKY%229%TJJAM#ECx_p5DMb7$r0zQ9OYwim; z!*q;3e`;L`zB#W`hj{u&_#!di&2(71-T!aT{`)=k7uq+p1%9Lpp_9C-{|}P40Djj0 zx>VvZ^yI!%^kZuy-yXM-AIf8HrOv`;ug-4>cgRS6X9nvD@#j6jRA;m&Yo$V`dLF+z zKj)9oX~ZK9@PhW{DjFVtrV^UR|v({8{*>iSde| z@9?df0JeoT{IA-}9An>%a#x;w=cBg=BC*00;^uXYEKk9Hi|>2Wux^p2g_={-+{5@E zsNEB3w>eXi8j4-ZyjjmTq(jE{jK%;vNt;X9hgUp^6lgEVowk7s2cHppOl!pg%Bozv zjLM%~PHVwT%IO=rYhCNa1;BcL%Dd{bXH)RbsIR1~=67R*!A2SyD{@a316 z(7)4nt7W)Ltjy5)lfTnvefN`|zQ^Kt97%oA%@ZTX3VQ?_(H%^^vT2TUPU*~FLt!A& zFa=#^EqeqZm9==TO|kLhx<+Diomh}@G3U?cJH(SCX5roJ%?R(bZ_2PhWD0c@baHxz zYE#d{lX}9Kaoez0m^kl{g zyR)@lz5+~h##B(l`-Zx$(@3F+Z?#MCPD|$Fg-Xrn4m_Gy? z&zK8Gt;mLvclC$_;Ju;;u$AA6ZUWwlUh5V)J?=T~De)+DE<5==@F)L8{r`-`<+s75 z)`I%xVxFa&#{7!8o#TF=Z`bb-ALP2_w51c@f55i&9NISR_H0Q5DLdD+JKH@1-Fai6 zSZ!JEMDk{nr}Gr*$CyRX1NQ<#Z`+*%wU-xgujj}+5@|Qrl}z{gkn0XJw43Yd?qCm3 z-kp7Qw0W{%&_sz+`1nfF->~2PAI}6A2M)B_^_=VqZRjb-te)~7yd*tkH|w@b&G*vX z5u~~FA;+IJrCC;s}##^Z>K%E?#H~NwmN$EJnvxtGQP_n z+H%La+e3cvCy7})mvv(ixXESBn87!7Zc2XQ=gF&#ZDn-8i)}+M$H7W6bO`40ZrS4q20-b3h zI#V$^(*$&;@#svytX|E1&4_#ou zTWh2vK7`)5m-$$ERVNr1(nnQxnx{khPjELkZH%(6%`L+}dX87$cPYadGzE^gckD}# zvpV0nWOTj@*^?gQ#EgE{D=&*E>yka`L}-T*F4*;^)?F=@n((^&4*JoPWm=2MUP{x|c( zefYlo4o~XbY`Jtx@BVMqWy*}`8%{|mqxFsGXfXP(aM?J{VL#a!X}&dn(P*ua@k``x z=!?Sx(ANj-&j_#O{l;+`1NO>k2j3WM^j6wvzVU~?;eQclSY;eyK%cEg%H|NTx+})0 zf9dXA=Z@f7qZ4x9nXf>1(sy+i5YMtM+CD<}ooVf?_by1#K?B}9ugp*M0Uu5*BHkmM#8Pv4$szx@Q?(OjZ7ucXb2H-3zE zXK2nT4#mF1yxV`d^YIGkU;oP~)8Fazv7i6Ze?I)fm-safVy^TBxKAvl>ExhwxpAHV zFWa5x3c_4|_1He<6y&iv$1a|Xst?px3j7$U<6z<8> z+9L3;ho>tZgZ5hL3Vo4id2!?-|m+@MgA>Ep1w*uw6;(`@Rxb@@UpUl2G6%+e*)LC(ag$+=2=5%PeSwT z-sYUkG!NJEu51UA&8llJ|39>KY3)H<%$i7i-+-(f2Yu^2eIvX!nRQGmy5rQF&ka>Zuqz_4rRfcbisEht8#%`&dif8_!@b_e?7QS z`31mUNh}X8m9M;&`~(A;X4xuAy>}wpAl{wh1)G@5n>)_ z{o~!=jxTz2OGcz|7xS%fJ=eo^fOS*=ok43LCt(fLx|T89?5pYD-Zj#nbrkkS&No8a zjL8@~CTnT0S);;to56!OCaeeTy61-mm-%ilQcve*Z3z!)43+?kFB+7S zxYI$l**f;HH2*1n{}&jC6NPG9miwmed|(_F+i`e{|Kb7h@}Eqe;&E^`h_=@^>z*TG zT~JRa+OPFNq8`%~85Oq|%CACY)z5*H-=V%klhQpBX>uZYTY)iwjX0Yw7m&A;JjoEz zqV|OXiS)>N_I&h+e_97UE}>uQ=UUsBpQwM(O&$IJv0%{KV@l!qA1Xyot|le+t%>*6 zP?&O8+udDRzofE`>Bf4dJL?+aXD{i=UZp=AwrftFY&ir8R0XIBg?>>6% z_v}&PAJ4oyeH1p)9DdAC)o&Bm)ahF_^-aa_1n0^x0u~rh(65OYKm~mxQxA~0oU;}; z&?c>wwI-hjuSh?-8<_TDw0AYf!tMm-jbD~~gN1zwn0SjxyZRV<#q0bY)VhYcuBVLd zhbfdELp|5o|CPqJp5qe$YXGC$}kZ(V8`^BR&-5-EM!~f*J z1ueB>ch>m>>2Th7<+^_X{zK`JwMpzZb%xfvK z`LsO?+EzTbH~Bpa?QW+2IPE?KPDQ)>fW>KdtA#xaEKa+dE$nGvXVdOtOS?}}M&lc& z-HrBtoOXY1|JyyGGii5r@9=p1XfL3>v-yVFE!qw9EZY5nNz>g&Oq%B2ODdds3%ZI4S{GSAnxDv2R>~pG}gXU&H_Si=azZ*U9gfv0F+V?Ur`&88GtO311cay5X_^ zOs9!7dJ@~4#i zWa3N2k6{kcp7EQ^AJ6hr%!~*4-3s*-=l{2^4#B1&cyUH^G2?#yTkS0`3 zBiXKaTlq=yy|C-uj(OzTKH*y}FW$#@)yEb?<5})D{);cSm^8zUm^9tpY|?D^5mND` z-jN>j-}tf)zPLZzlke+zcVAbhy4jyUxNVBZkN9rIoF82k`+<=iY0Mpuo-=o(Fn5%4 zhquR*FPV9xnfs#apyQ%KukH%!Rvhape7%yW?@@TCzEC_#ED_qOxnmhHjaM7r5>JjX zeUQ&Zy19?%%O*e9J!?;!r(!zqrz@aD8^WBLW3t(mNtYR+3$3tUyIe8`~MX~=t zLtz(msbpkj0GXMB?CgpRC6>~XZjn>O8Im3)f3G%lDcvt|0$oZzrov%7j?zQ>nP;GP z&Pz!EqpwAE36?%T;I@Q$=qmJ}EAMJA_F^{)e-3j|U5Zn!KE2KR_=a*H+IO0f0AuEFSBXHP3_ReQgJ|DSO9 z2kMlLCfe1S!kd5C*Glqey%v~oHr~?YrKZ2yv-0kq+G6sv+|f2)a{mOli|=Vw0>9vd zbXIQ|?L%I6A&)Id%>$=ep_iPkg5PjO4Fq=(r^vUKEUdENI!&k#gOz za04C3%-oabKEnSs$nUFKJR15T|JO732#-cSTmR04xo7n`a}T)w#p$_c^?7_hSN~t) zSebU5J@@cU%{|}I+*2;zONRFX&_@b%(iM71g>K-xCEN!Dorw2Z*r$ux*irH)5+5pd zl=x8nP2|J%zw`KzeY5GA0gn#@$xffDWM|}~@Z@6gqm9#>1W#(6^YdiScR}kJt#PJk zjRQVCe#~+Q!!x3#(ZD=@%ycibun`94i`*w(?1`KuyV#+VhB-F zh%<;+yCz{hjsZMp>qrUK_ zOb`A?;ASr9>d8s1qfR0xxF4g2J=Q&evZNV0cii9C>toHGU9ojEvA-~xbH-)Z#w(FA zWyDo3y|}{6wb{rT_4j&w<o{?wPU*9+HWxDzHJ(I^+YyI^dU`a;acs7hN$R9IrVr$o#f;#`ou`Oxn zTjl7vhtPA&rRQ?i+t7!`D#N{*_7elcn>V9bW(v|3YWc3et*YimP+Q!9ef8!{(eNdhJN=Gm=lI+M{zI>-stO)QP{( zrum|56aQKF>~_@C{d7IYu!Cv8zhYVE8G!*aQv-M9&(PY;$dx=#u1KaB*>blNdDhAm z@xSpUwsNIvS-OcCo3S(nxr6=Bp0JfH+fVkH2IcE zB}>k4ABo#WjPIA9_-WwIi(6Z>wh@{sT-tSQ8T;EM;ALHxPKyqm4?o%LOB1i_Z_vv* zxJ%Cs`;Zs91NyWdu$4RT#ni=806+O*_uxtNDveVP<9#d*T)fdE7^N@KLp_7-?Y;1^&39*5zuXMDb2|=DnpLT)CSJyietNTXsj2`_MaLMiqNF^Vn^VEa?iO`bP4;sI`#5~4C zGT__5#Pdmtk;Xf+WsXKJoSfA;(lQ)dE#HlP=#MBSSbecCdCS1Jd35=u=z zoU}T*e2e89mZ%H->^JBF+%F+}QfX2~++WAsQ@NJ;WHG;)`0oto_cZN#iTeS^`SKTT zPSQX+>b>62J*bYhUsp;6S{*#^ezbG9gbm(%!H5S-QK%MLK84yWndj zc@G0SHluSSAiMT!#18R)d#rh8&#NwN`Q`QV;cvg!R);Z{JkcF8^Xb=y;+s7Fey}h6 zo#kGFtn+l*GJ?eCcCS6JU({f@A&h*rd+nWi1cCcbZuvJ_Fm}hz0oE6pi`!y zTc)F9W}vfYVvp@?bawHn&f;lKkHM$%t!qB5uXmi+JAod*aDbA_ZE;Aq|I7QekC^Rpg9t$kWI7u#g=Wzn=XKda`l^=)Sx8dP*SG=oth6z3fpIP`=GBVe~5qkC> zj665P!UhV4%x;F?%NUDhzAc{5fai}Rzc1|!Z)>k!ygvq>S2_xwS2`TMfK+FX^Njr~ z%}u6Wo!xm5TY%X=Vcr+-`GD(xrQ`v3_T>+FZH;)dmG28~=4AXG150flw0qR%ne_^F z2A0;W&9`&T5u10LXE$4&U_;pMywbPhUc$|tJiFP6gA*e<%kVD$b+^Uel+vHKmD0bz zDuo8#FloBGhcs@l(fo31$qmrh_0ZZ(Xl@3yH=T9Nb*y8qy$hN&>zHHWsbSzKDXFQF z`!>taMM_DZ1HLyYzsb*Y1M;paDgR}B&zdUI_|KYPFX!Jj)%p&|ULyG<9r+MA{UWws zjpwtp#psWH@DJhq+@?dm1Mzaw{SPr0&#+~;Q&xIJU2o2e$i8ulkyqOX{H0sC=WMS& z;Y7M;0cF-V`wo0F&4X_MUe}9p#jpK1_^x_;aPXN!L(oMs+zi$#J3stvFuE*h#;(et z!3M?(eWWH*`MF?KZPs&p{n@)U|9!5XSJ(HctA%-9{H{9|b>904s5^^wMPkQ0L>o3f0KvJiMBXYPz{0!}2)jD8PINk5Fgqchf$Wphn^`2Qy6G47IO zvsD-8l)Sp~-A%w|s62G|USfIZ;zjbVRvvUH{j=jc(9U7`m(Ran&UiYf{jO8nJ~2S=-HJ?6tO=>&s{=x%fvNcGPIGL+r)n z(;e=ys4K4+TR%Su*{XXS(%lI3u6b3uQ#WwDp?J1r>-GsV3&_uL-)0QWT9f?GXdHnV zenz$)gC=#p5xw?IK56L}e!eSrjUZQLCl-(Nq+I-2YpwXR)>`J7vFSnmI&UpGm1@3~ z@1~eE&rK%%jM2S1ANc2(Zg(9S-u6(%`3bWHxfUxN~@a z{*zN9`M;R@-TX~c7v@(^y*hv6)S>Jl_b_|NUhHn^T+PUG?FUG1OXsro7bDM0Irkc> z{p9iG*g>Tq-XCzPXY*@5=tP>qNA-sjg3q956=8n~`kg+L{rOXL_NyQmT70-G@>+76 zUsCUxi*Fk~d2uOrnE&yEOLQOc+kW~cetZL%w|2{RYc1?m151i%Pe3|R32CG}G`a#= z$9{+L+tS`gFk{u440Fd%ru!W4%18GQdjX|o)oU}1Y|D0^ws5QOoH=Ii6ohZJ_R+8L zP4%-G`64>4Qd`k$o>Yp=`X1?-wwUX|`Al#>gE5%SSX_q;yA~NXjWL?aSY311KgV2+ zUB;7V^_8JWDfXD!-uRVMUm0?3KF^OL!y-vZFP9~yG)1|at&B2Nq+{CZ@b$~2>^USP|WQBM6j*c180Py07UmfdP)S!wt1 zJ2!Z8=IfM|9<{tXnZPiGrQ-!=bN`fo7jgJa+^zP~X@vS^lgkFwxy z0c9^Rb!EESZCxf0I#@|w9(fOO=GJ^0IrV4$8~ZSGyJPmxzctqA57>yc4|AwiYqd1@ z5ol2B0_k%9qEGK_HujqB6JEK1{9Jc0c=mWeW0&Km&|d?$b{oauK1bgCVSrgjUx$3q z`1&cM^kkCW&2n3m(x>C*Klc!lPWoTyd^#VmdnY6(P9&_&+K>|(3uvfAPF(r|cATEh zh~voIrz@`r)*PA`d^f4rq6KR_-40oB5?Ns4Gf2-Hh+OEz??vJ>I5`s+KaU=F3|;LQ zxHy3hrZWz*F89W= z2cfNeSG+A)AuzS|1Ul(TU|J{YY{7iidPQYVugx+tF4tXZ%U5b2l=ePg|DRW%D=6$( z;|-@j!tXKk6Zs@9G=1h?AEnIAw=1QOUs4Kg<|{>4m_&N!TJOK>b6b$BCz$7=NvTb> z=x$q~&Gw{jP4lJ473(?0=ybnF9_oKXQp(G$*PBYE*Cll`&%|e+`O>nb=6<}YKF~}S zzQ664MbMXw&ZoIuF{+z}Cq?S1;}|lgf%)bT>EEp#d5ro1oqBqoS6*woW5_evy#Hd$ zT}(OgWoVxKrL~3i5X>*VFU!qEE=h+w;g{UvejEBM zxpfp7BYo^Ko(-HnNoM^+aKH34?=GB8z#4!_E`5X4)5TOzhWjFU^(Id-C9>SR6Z4^i z!{onZ^7Gu%#5`z4{+$PbiAIxFNbY%e^?aN561$+LG}*al9y&lpQecYiRZP41C%Mo5 zc}Q^g+vvgAXR6VItBTQm%d>WGd{j6m?vk~+{n30g*PUX&>Dk`dr-0uE(x#ojPeAwj z<^)?t?`!=qk^F7Ij6J=))3&2?oY?WJz-2l5V4>#DeD^~5MduODk+JCm&0ft&* z^*d<~oA%rOEj4AcjDFaWXZvpW2yj^k3N)#0R$u#+*1;DLj3P zQs`}tEqf#BN1WNyp5TAl4~*{>v@yqMub}1Jy}<49J^7#Q`{eNu=LIy!y`IdOetzr$ zj#$F)W`3XNr@V3KptYj2)VFe3GcJ_gS>ex3MNVI*`QkQYq{`_|^jx(U9+W;h5WHwU z7!J&{t7N(TE$nkDCmv67s@V^yJ_>J+DBEFd^8JKQbgKRV_5;DI(La|xYxPgg|C2Ah zbOLMl{j6sa*4C??1s8^&xBBNRj@3WGt8f|6H^F-slcu>JkuMqiX?uY2{Xfmi&I%)Q zRu91c>`OB}xpGSHBjeELtJlx)9r#mE4}JppJn8j^x&>RCv6X+=BdE2feENisf0D2H zz6JV}Jq*nE>m-^820HF??YAHb7n zH&B3lVs2;{1Wv85K%M_vW3>mbz4t`DY8>x)>$o&`4ei}fYiwkzC(Qc{be!YfflQ6- zRn-Q6_zYQ{su?`TZQy!V|KfXK@&B96JK63pNY_etD(-UN$nak1_q|zz^#r%Szdpq3!PSoD;@~QOnu$Ghf4Wm$%x?_89DYaBoa*+Z?1h!g zkv~myMlo-WI*N~F&A|!53V(X4WbS_E%$>xjeVm^;JHs4R#B&b6quhINkYAN-5$HFi zJZlUNq3>X`o5Ei5!u_mExEp4~`o7zyzS!=VGn~5npbC8q8{Ew0eYZ~i9}Ay{K3D;K zDDZiGw@lp)d_6JALw((;J9$=oa`_(rn&&#~`qKA)#WQP<5mh@vqp{D`Tq68Kv(iPz z!Uv+qDZsQgko{?tgcyl(f2t21? zuR94(oBMRocLH6|>6q8{OV@*@E0jVrB}#uaF>)vO?KCn!mmt3tzb6Xb)aH}eC^Gqu z&PqvdtJS_8^GG9ex60-Ycx#RN(Va5#F_jKjwX9}suF>mQ|690Uc0TT}R*WF!m6WBe z)i>H{d%SMR9o2o4@f*r?TgX$L8?ot~BXy3<6uFMbKDoi_)C?>zE}xohtTeLUa0vUh(zUgthr z=dG#cKG+Lqcyery+6~Uuqg$;Xo^;^*_MYusz*$pNyTpNO@MjE`4YB2ojNpk~PL1@{ z6Srptk50=DHh|lGdu>EbP=E#n`p-o%zhc;C99V(eQ!P*TsQm5pO?pct` z+=iU0-yvOteSY>uJF#|r!pJW6k4PmmhZ{VyHU;OBnahA%)6Ox za$+81w}8BRln3oO33(b@>4_`Jzf<|pTnn_-Q7_}Wi9G4|(H`DqSNucVWKa{O6*R z=n%rgwHgn~O9pF9)kfhC9n8JrHSWk7S8&|Vfa zi2PlWgB|SCWw7p4Y)#nDNc2Hs{I$02ki*B|;bs5x$>Y$5%bhbLh%+>OXH#)-vOmMf zVaeV$WbZ+K5q^*JEAyu>EI|%)PIeS)mQg3hun+R93BeM7Dsnh$VJUKWb{}Wt802nx zTo$titwQEL&Tk>V>-ja|+tN7Pi8Zh$-i=(|jjVmf%HIXV*HK#>D}VoJ zcPoEq17B|C@AW(vSou4h=Y3ZGPUCrAT>g%7YF_2Pk(c8_v3J0~+08!48S?iyGWk-;U3f4mISXx6 zDTNP9l|r*cN{3yE{HB!#eutR#XSAGvE*v)0D2>7xeZlGTmD4v~c?RB3E2e{M8 z@r`?3V;r#O+#AV@pShZkOrHNru;bn`mpGQhHXC8|&P7)5oYyn_yX!o;J6CN6U;9`a zA5TX9B728*_wp=FR#zF>i%k*P+c4+Ki2O_@;q%e}?YI9E`TI8VcW$WW#O9pfv8LQ$ zOJ!bA^b&2)4<6ak7a80S8Qg!U^n&;BeG*-b=eyG5TNr=I;H!~$_05vO?3Y*>Ed4<= zyA#=US_U)6f%}2rmKX@+V?VO_X8wczCX$zJ@^aiK6Z06u1?2Ts9yGWvA@2n8S?l+e zBl03;rMs(K;I7koilcX! zynm5re97i?jWo07?U3Dt$nGh~ZmkLZ9zMPVy=o5Ee#n0C*CD$v8w|d3-NzX($?oGu zc4ITq{a?U!|4y8Kg=fib{cm}9eB@2aG&5hxF8dnKCC2uc5Kopdju0cxy*D(vow%UL zxzW&~&O9sj<70{OWdB?27s+$^oM{~?KaysUZ!*dxjsEA&!W!=P`-2iuFBbycWHOA4wJc(DMqO zOZ*vA*0Ihk;<D?XWhFxu9> z^SD2Mu>9yJXF2yM#z>U8Q@%8QWM0vpuN@5Z&3rh1ucP!3&)+rIJ;WTn>LU;TYJ=0s zscuF7C%_bC75}t_+SHTx+3h+fu&CP*i}v}LX5kv$eQ3?gT1=4i^2ui z@CIz?6i)i%k{@17T?ObJ$FM~!?nZ!T2Oqvy{_f%H1KLfz1;t>T{xf@r<8iIWnE%(M za;GM81KU)M-c>vf$&rcm-n(z`u6V%E47wumY`Bw%xN$SMC#8V1(fqy*tl5@-+2W{m zh!cBum{Yx+pLzdL-+`Z`db0lo%1F;B_?%N+@zysE%64F4cHm2@y8=_dm)f!)9Z7V4 zvanyUK=A^~oKuPq&;ZUZ{VVUy=@mJFK7OLGcjP$xtZl=I%P`K1%h33S6E69`zt>5=Fb`pq0M&f19A(H66J#hkJQIqThV&fb;1$LPnv;%EBP8H2#}BaEDH zg*M}M6zz!*1P4nf>&bhsjAViCPMb*H*NqHkA8X3#7z}sY@+TO#z+JqP5;F2b^{wQc z+sRj~!YY48i||nO#z^@lnYawP7iA~D6dP-k4tV!yy#Xj0-x29hZUNJq1`IjsmxEM?i0ljW@SXq>MwE#$n*Bzv!wb@xQ@S zxktMN_C6JJZw>AA#wXpqSK~w5G>^Yd`(pID?e=~_Gp3fli{7~ZpxKih)GHz#F#0fi zUwOV71XPVFHy%EC)Z>Hb+kGM@!Qshu_PbY*SLvLJzRjBJSM2|Ce@Oe?HcpA~#r-DD zcV|UfYdQDBT_J5h@a{GVZ2$1z(XyOKds%Ko?fa(1-J8VB5pSs9f8beTtTnD|Vs#p8 z>J%&<8$;uNoOw!fWeLyT`jx#+=8^b$wqFB}$7zuM6u2d>8&MB*CEP|RQz5;HJe5g; ze)Jv1s$4++2J%f??3k?t_EQ7PbO)2hzpHOpXlKr9Q=wSpRDKNX;pd3lj}}^uLg(b(=h4J_2{!qinohvv z)BL4V%pnPP87N+Y;>oqw%9pu2YpWitv3j!B>cyI?H?{_1%9nQv>rMm3lh=MZv3`tS zjp$hW*4!u49O#3%*WgZi=TYF&X-o_N);;f9_&-!0ouQTWoA#ssYT;4AnTN4`0nc#{ zT6n#wr&C1txx5QaNp?v${Si7#S z?N89fLR$Z>AKamfeT6<~pGdxYqO(n=uc6pRrO4&=O6lLvmBN=RNu{^y9m%*;OQu5~ z*Fh)OLNC*xo2jhHui@-|Cu0MZjmf!hd~5*U9Z=UZ+e+@^ZgWarT9i$lT#?Ec|C-9w8^KWS3-j7aspFGG`t)*i65R?;gx&(c~8 zWyJ3~W25$Fa(?#)_5wxMP2}b9e#hq>4zHQQNWcSpQraLpZz`~gE-&TF{b|ImvdeM`vm9R{9c`xn=#FH zf6F(Cy=mGS7%)7X;=>0SzmXN#-U9DU$K_oq6`1r+LSW|JKjaKKTRZxbvX_ z&o?OB{U!PB(17-~mGAPu4>()U;GM<&75~@sP3s$ZIu-=-C(zjmN6v}^F`9nvfJWpe zAUW`3-q*Mge2A17NG8?!h!W`LD(L4*=;#XM!^B8?u@~1P0RBw9XT^_L)!F$!#g8yF z$r)a?F^BjOIX)*=VeB&hIVMYdK0C>LrnnKpT}8rq`KNbdQ(;XidLyqI8^nkjeiP_x zoW`$)wngJ}fQ>DCc5OxbvV)BIPSN{%=v}ljg);pE`Asi)>*Vcmdjhig6{qF1=FHs% ze513$VLEridpNz2RNww@aT(NqXQ|o&0tK%_I7@IgBoO>o6y!|?X7Y$}h&skW=9URyZQ1=nTb-t$0c@?@j1TynnOC8@*KWw85W0xDh{M+3Fex`69ukuDK&&r1m0s zrvw`BA0Xy3dcsD}zG&ougUpYQ2XnVVS(w-;8?>LiI~e{OZP5PpWrnsg+*j2yYY#h zgHQDQ9>Eh!v`5^N{fS;f_gi1+L_g?;%HBi&lOJ>je$dhdx8jc_{B`3Snup@PI9bS( zdZYJXJGMGol9~6?+~vp$$r^88InzA>PU7c48pvxvr*5e16lq4UKgQZ&p3a+};a`*K z&Seaw_vo(Z7ifd_1rl-3cCk61_=DuQzgVXGmhjIzWz4C%uS?^nJzkB0+7S1%N&Npu z^G>$=vPrYtmrR=B{)*JZ3`^j}IKP^9tnheJaSHmigQK&~p8mJEdRnsu@X1j;iWBGI zJ<0S$-fMdrKWEld((#`_4}Xx~9DbkYmkIm?_Qk*Q6EC!J|E$V|_#Y5|u^+HZ{W5E= zY`223)R;aF%+p7S*<)cp1m^Kho~yg$RqheJE1N3owFfM$LghH)E?gJ!tpIB*+0v-5 z`ZcYoSeH~NWt~()deonZPRaQ{-d8*e`K*;|9pxv!1%9_bGG|2d-8;zF8tWM2RgT^% zy?KFwrJFUCe2?X$m2P~PbKP5P-UZ&8>dblj#m{xuBpy{W<4>c-|=_xU;8s>L%-wfX@AIBG@f9Gr|zGaNS@E+5m(Q?Z}Py`0`ffl9{Neh zi|=u+B>&%rXEWWT3Eo+%Csh}`9c8{hQP>&%;@h5nA-fT0PEQ~gPoF!@bZ=uU)mQQS zZ|LuP_l9bW-mrBTdPAD~7`X9hleIlGeiZ)xnzzPS?X0r6E$tJ&%HmdKgxlrth;X~T zvxoD)>RsRkz>EHuu{$zN?2dHz%XU0|OI_YNmiTe>KYrdfJ|24vPV(I`7H2W~JOW&) zKk?W=n@pKJ_Yzyi|MR&t$=djtf63dU)=m~*SDYwi>>9inBR^3M&PfK z#ZPDcUzhF6?8_#Z`#VaxlaZL4x&3|V3-3U;6CQ2>ACk|HaZj!OLw`m-AJIN8u>hi1 zIW_6Rg^{({Za>CA@^>h(xJ=Hmup*UX93_*}Ev!&5Y+@Rpo)*?mutAQwlU@5?9kTk3 zq{vQd6Dw7U?WD-s#LzX-k`R)oxPiL^hMlY3@ z-!B}$v*0t6JXxbQXzWZ(fgQdBKj`GacLU#t&USQeS}?%b)8mEdLH1>8nhP_6ZDpCk z{ophugr2=QHS!WTl(oeoU19yGe?eWI49akyLWb(Rsbomp*W{;0 zuI9P9q@KJ#ZAbf-$;)-q67#^R<^s+Cq9y5m9q)m6%>~~udD(8~guLU(rMMmKZs0mM z3|-X}`JGF(@1-^+E&rC;Q|OOuYN(WL)r*A_?CXSs><>o=1-$*HDfr{~xC@H8A|1Zb z`x^H__F4R+?}=~m4>9?setV8Lnfjxf(KCscE}D;e_5DlxSF+vd9+P@MekewVO>=9- zC(Qq{-F*hUD$eXfvfV9^?JnCr#<+R9QkokA-{}8F-YIXa@Q=Kf3>(e!jgn!#xQ`CL z$7gIb_ap?P$gf7?mAuB;gpPGyvF?fOiT$M)b{O_~XZ67@!yU0pI8W>EZ0?Dbui#1K zRSbC*_Z5uW`JmZX;&%P7SD%h&q*z9m@k`^^iVpZXzv@BwViT97KXFJV$UkH_wsY=W z)A=*C+fV-`Yx>ZBPu66+pYSYLcLPg~%(d~1=Gu5ha|=Huo)L0sB=L-j7#EEb7cz9j zGujVNC)ifDJF_n`=U@vdb51-Xtq;uJ{#o&iRDTjOYaDYK>$%rB^PoP6|Fv!_mJi8r zeD&}*2_UnaP;VVC<$dhSxEc*J5j*hCR(Ydt1R>&wkdo4alcL_7TeXHsin<+uW8-I-}T!0Uv;_-5hc9HH7Vhz10npTm!F3E4w#?4nm#^L)T*xZZ#ADfgh ziSb|W1dv?|(n=Pu;Ea&o9Z#xveWZHtd30tohN4s2*PlA-c-A*H*3ZjlT=1uSAK%I| z_Ll8V_VDmkv`_13mDAnU=ZxvVON@`V?(T@=@TVGI#WC1m^t9~*Zq5k*?J7?e-ba19 zpGEO-@8H>d=dt18p*$Bb&Z6^_TNjRgi912G|Cn;?kkRpPYE9lyc}1jQp+9o8Fe%bH z#}{G$ZTo<>;o;c5+>^mwgGKxg+?5{I-3r0KriYEs5_ij(|F>j>ze~F&Ki@4pg}Y}K z_WtggWAd-`6Ss~sCF7A#$aj6`Dcjyjv`euWmU9j$_UBHK_4t3}_`SYNAW!RNhrDOs z9u{nsoq=(x-h5H8s)_xP;vCMy<+!g~Kg3HZ@5%0bcLMY3#fdANfqa$+Oz= zI%)5-^vO?)AN8}d<3|;1&ovc(?gn3XhrfHk=fuQc(hJ$#$=ppV*(|%6VjG`*ek@To zci+KXQ`o75v!{{CkMp~iAM5uKtlv$nxueMF7VJ~I*(VN|xRRcnZbSe20sSzsK+tV| z0=-BkR{`^Ik>x&SVU-5vkBqZ&dYqNhUHBc047g`}gwR@^xzHu4mAZgxl9x$7mTnHvM7*JFEcLr?rEe$3L-zDm1gb8kam zG%$7X=vhvXxFF2p9-gjxoIg$eUE!=7^^hTbwzMrKOTDaV#lGE2Re$yDo zsf_0}jO*3N=_$x*&*r9m+y?9bMa1+tgiUi5@gB~IIS1?vnN8Vx-uV#wlg48r-!Wr^ z9dxc_Z%F4TN*2#cEnXbpjt{52WU)D4Mjs1&C5tO^yt8IJoBQE3M$$1y`T}#?)d1i+Dr zpWw&e5SbbbPpN5!=7QKw8<>}a+@q^=@Z(aPMUAv`19;avf@wY|@zBkW?OZUMG8!w4-JT5br@GU4);ILc)~24pqlbD0 zW#ehw+?)6geTcc0itd=|uIlFXtARUdjm%^Wjv+JZn{7-6D>LhWb;wL;Kr-`XBU`fE z8%aB4CiEhi`J&0obf+igK|_+6dw^lXswwq3mm1l&e1*|p??%2x&6%B0EZAMT>jZSy zE)mwI={n03^+QjET|60!uGt~?uzN@!k?r)5-)r|k=v+F5?x(7W$IpB5Z24C%`PU6T z=?=g2fNy%jKghl%_?^Hrx|=&L|FlOJ*G=QGbQ9&@AD=p{nScHUPRmb2eO-5pm)i=3a#&Ya`zhg zq2ykX@;^=P6@9*AFS4FHssVn18ofA&)f)K#ba#-&X#-dlg^1ntevt8-_tm)KTGZzSx0@6b%BY< z9qN>g1pPg^5os~S*PXQt!K)+>#nf;Pu~*d-7}5{KLnh!ke<9F zK0}ARW6tT2ckDML%DXlz@0#8yj>P4iU}wv_o6nMW75-0=cSU2u8<2NTSb4|3o{@83 zwCPdeLEOl*zM*eE+cYd#-F{I}_}Q}g;$ThXCCIz3(0y0;8tYTM&GuJ#uR&Ivts`y% zc3R$H4@7SLz{r#=w=Zdjyn{|8?_^NQQ@Mf~4*!&XaNmP-V7`3DrF8 z>zZ2T>$2w$W>NF5vk&#^Qj=z^_IZ zv^q_rV9<%?gMK%VMj)5P35ULr=ovBs+SuHy2r~Yn+R;4RTKdxn+u5$THnJ0>=hjZ50 zXepQJ{)}?cHDuE$M?aHIZkSi=~BQ}I8>|+VGoYhX%<>3wZgvnkfx)9DzEx8%Km;-;@1fSdp zzszR-nT1{M2KeZD_Pu9vHapq#OFX;2^<&MNL$=>RjFYJoyv;LX7K(k*jAtk&dXpXa zCZ*tgwo=AH^rp4ISW*)YnmZb$-;2H*H-{qeIc`6((Ds&RwrrZOwKsRMCr^reZ0)pD zRR_GaoEVGJX`G$LezAJMH?zarioJQ~3YCH9*5hBd{@$bmneCq6$~eKHi2~ry6?11c zvgb`}$Th*1&3%JWXrr;RUr^)j;8Q6( za9d^nV4`njU9xWv_oVFksHbmFQ<`s2W0r4EL!NI>v_C(1K3eFjslQjg+1v|n&mU&- zP0`Pfkg=K{E;Zvt-@%{ek9H$_Gu%gPzQGGP*8I_8YX9JJY-~hwq#6+&`6Ao-uxr@}B(9a=((8$G9FQ?+suj{_KZk-@{ka>l^dT zD*Clgdyeo+bUr*%{yptEdS}OnEWi9X_KteLAG@z?sY(6eH0i2voEx0&(TP<(^z z4yz~ZA4i!q_kPAxXAr&nnsVHI|8ySH{B{ogum6IHS2~mVMYJm41EpFM7~17KPw}pN z#($+09Bd<nnkNniB% zPoJlx-z)B}?zBIx-*@=mnK`P8SRtCbUi+ou?qc6(j@UUTR8zzJf^Iyb$e&?i-vm-# z%gqB1HI6g#JK#ckng0KT`QaPz#teQJ@XO~%eCq1g`Bn2=3oJ9v(8{#Wk?Nn z#VqLg2IhoELytA9s7RKL!s&S$)$@M<3AN)O(5a0j+;;X2A(y+{3pUsLGAS@K}J zGu8>ehrAt49(z8oYL52P=OxWc4{BUOd`JARZ$$Z4A#-ZOX3fK1d$5`F9pc+kM!qOs z9bVa5=K8#)_Z@t~l*xA2o>fM3e(SRP5B}KXXS$D`ozI-#y6l02kD2^*_u;eine)eJ z&Sz{_lBYJx7w()sKIjYIN*^^xe*s!G;|N~AA$kVS_bCPM_bO!^7Aa+1?p8V~6uV0) zk#pmn_cvbq@CiFAyfp@m{_uKTo#*p*V zRR=?(vGK1CrsRYNlTJR8qdR*Sjqkl`&06O2CAHndrG>KZuQm4ln(^LzejRn_+|1sj zq(vHcC*|730cXYZ0>8I*ZGeWge2mWc8L=$kTTVgM%qAdUQ7r40a8% z?iw^L=vH1B?8;ix-Tawgl(}U~<)B~``Z0O}?cP~gbcUX=ephhl)?W=C>TVt~w5mAs zIq_*#dFF26@7LCj*@tgSmeePQV*QvKH$LhIzuX5(eBn;ea0;+|>>8rWZ==IVcR0^n zxZT;B6n>9)7BhzG>o&{dzaZ6mP<6&*=pCk>-Ik6Q**Zn<9p#{#iPZ5Hc~4tel@^qe( zxv*M1J^p-Uq4UF(b;;8=lAP}v8{!wiWnQ_s1& zL$%UZ^4uJFQRAiiP#^jvJmu97pR-X3+H`?$`oVmnl9*{yun8%NG~+HrgwSiFDf z=d_vk=-)uvyp+7e{-vC+Ed85E*?Y*VWDR4!+tVA1yY&7niyzV$?HA3Q9$&r%->>9- ztta(v5_f28>@&en5peAviS`>7GS`$pqCKS5#%6F1e(>*xem(r)X9Zq1)LLDxTPN_x z-D>cdbh%b@<|5s_#^8zgj!MDRB&E>)RZ8i@6{LSK@dTxxbw+1GKN~bY;)>t!ZP>aa zGu}-J?e~ZDkRPXa_JvB`!~pmJZ{eV zai41W|9?80;=9S8RPo(pTWH3%a1{OQm^}x``laI>VChzDvYB2?p*8pz-p6kyzv2Ay z__bhLP`vx+NTqv}Vuy&KgVkVH!0s^O#0>N?`oGJ@C6Yc?LOfO37E*x8w(xsw4BODz z9wF6PO4$;=;_IHe&ewh6A=Y3oV`KOT-DxX2+(*-#*wxq~8Y;0dT;;@?&|QuY%V3MO zHC)BHOx00F(lqzU&a>2Z&AA0H0*qc*FO^^JR~au69O#o_l9JU-k&(vhIoS>drT3 z@%8PEp55V|BinO)Yk{~J9Lo#xE?c6SQD$Gj??CWSFGbt0oCnDIkz zGh@s7U!_B^FDM-rihW)wx{c^fc87kXp50-xXLpd_%{X*ma6SnguhDko|x?@I!&$hv@RCitDDkOsfhH{1x`=K7Kr4dAZ1 z{q;HFmoCT+2e^|?{tGL{dHPL`+6OP!K@+l9G@kVIjx@pH>v_PdSVJ{!#ujmVez37W z>#iYvgN=-7(~y3_Mr=imt;jRRkeGcn>QAGev-rt6j}PsolFIJTlQa5hJVQg$NzSlY zV52yKjp8G06kMvihjSQv8ivST;oH+rtc!8<;Wp8U=*XAzaNy&m2Qz&IVVw&)XS||a zjGp}%x`xK;eoF`U*z~KU4b)?3&X@9Vuwlu;OW6;qo57u5)+hKcP1YX6990Ki>&rcR zNx7XbUpDhfhC7K=^OvzYfSlkw9E`nW z)K$TcMs@M*C8^6VZ@L%XJNCb$o1KX4B^RD!FZpk=g8kU+y?ZL;Po6lB{@5ipFl%`E z^p$YeWy7d-xo;qYck^4ty8QuuCH_EH2Ub}f;a7P4h0_H_Z066Rs>Av0Cut#{V~5bf6m|=S+9*L%V!TU+s&e!)@)w9 zq%2pwt^e||J%@ju2!wss@9mF`j)^NS+h7;IA)9(9lcJZIlyxsC)br-(2@6z{7?={_pXEO119vaz9tQ4SIJt@E!`#2Hl6x2y@Wa1orrvAkj9`>8 zkL2O=kKR|sywe{$-gsc*@2-NOfA)VM=N_HAPPO}0FOaYICFA}#v0$UW>Ktjn9@aR^ zsj0Gh=C%g!?D_gr#1%x(yz5lEvEgmD@Vd5kW5e6l)^7C74ZsV4$%glHp7*u38$EL^ z&+~Yep7{jN4$oCH*fSdB)Z{TnvQdb4w%fUP1TgIfh(ESi*f5ndI%KZbc%O*IzPjG-{#lz?m4pHr(HYrv~}Rv&;WYZn>@>w z^tw{|_ZOw~_f@4?q1Y=*dxm09limxj1f)YFphf0l_adE(ymU<2dPIw&J3-+q0k zaZqNk2^){%Ej10v3N{pG2jPM3&LrX;-R_(^evo+!JfEXK?aJk@n>_Bi$sZd1Cuahh zof_5CHXne)B;iv$GXr@ zyVYMlC|^FdAs~N_I%RW^U1(=o{0@T3tM_N{UR>$l3d~i zb@`9J35p|hEWtP7IPyMz7Q9qGi1sY_>?+QJV+Y8kU-x1oxS8Lj{BU-uZp9zrHGZ-U zJi+hSv_PbWK8@yC<#$e-7+jc?fv-Wv!gBl)`g2aaME(iL{eb)uko&*K7NB_XO_I;! zoLKZ5PR(%a1=!wA4EW-x7dw2pwFg|o9JS+aU4&-<_m>;V5yseBf4`YP_fk~&3{d2s9{YK@CO(4@9Yhn8Y zGd6*Aca()aE0~ezI_u}HxtbNG>F=87`rc*$-P7hY?5_|dHCSzl;P zFF9VM6yAwiIbNmo8=;z9$#3cQOLNyYFyEv}M$?zo&|EWetG|=h)QlV~)OsE{P=eop z7ymuq{SIXm!@U?eZ}^)3al3%deR=f~FMhy5&S9s!OKkaC$rt36*6vMhz2)?@C z=lsAl%9Z(g-uZIxN7i&lE^ixI5biH`t z!Zvga;^-K#r+-;{dV>p2)0(ws&>H^JL2C$i&>F&`HKuHe*08P(t)b4N(8*ZWP_&*k zZs3c3dhq*qJ#NMt8dUcEA3i?X7i`d3o90sYziJ2@K16Wt?d?ZvZx3C%*VE^D|A2cB zSh{F&@VCzYwSP&`s&@=d?-!YhFIuIvlpe&(Caq^=a(WuDCOY+kN#iY%^fcg6bn1DN z);Dr#dKzm~bgG#+(@)E~=pEIA)Baj(b69$Q`u8kpPZM_p|Iaw#iPH2u(8c}0`V+*d zd~sS@>OS6n)@Os#)i3rmqJLwdk-`zNKyXkH8W_}`ez$igWTxf34h<|44IIcB1Tvw0 ztDu2ZqVc@%27Dc6e8m&$U8)-yXVJm?wO2VPTSs}3vg0ltbZzeUiY#CbG$-@%rMIlR z!DaLZyu^7GcQ-F~VfuFE1Ey=^$qgNAo_U)?yzgJo$-&f@+Uuv&%5t;cuf+bDxTd z6(<@6@6p|hD^?u)&78UKl8xxD>=&*AVQcp~eYo$U%wuCVb#S6BJ?H0&?% z|Bv|=FaD8y;5;E8Sc)4zKk^p7aJJEJX3u?U;lH-O5cB6Hs-R8UlW@<`^u@HJ=q!6V z^`+kDwM^foSE}`Z?M?E3ANiBUeJA;(kC0A}OdX|NjNIW*z@X?|dRf^3+46ID-d>@+ z_QpDY7S6V=GIE|B#b@W*#U6}D$z$f0F|S2mf&5O>F5lbrN8L93+N+}lxM_c{OBuNN zl^9nl~Wn%9Sf)3ZPE%N!SpodP&jRYN$V5wrKd5^!e6%%R~X2cFP(7h zdoEgBzx2NH}B(|t+iEhk^IQt1rE`_UxFNsge!93r@#6x*uRS_mi!#vnR6YrD;*d3(J$)26Py)%86Iiw1TaJxN@Q{UR*!r6Q2qH6ZGPGDbC0ZkW+ba-4zFpISl^TC7c1xs+SL) znc(4)Abz%RhUh6UJd*Py!P*JR3O@wIACe~9=Z5D5pBy%6d5HR5xq#Nc_)k|p$-c=e z`=xl~g08mDvo-kh4_rKQOz&0zm+$kfz5aXhY3pC*Ll1WwKQHn!{+W2>%kP23OR&G> z-34oH2!&(3ksCn2tk4%oY(w`Xg*ya~TKjW7Im_><-`E(z9>nL@`+4OqO;a1RR=0o? zw9X$r;;!?J#Mctn{+?$urhzrioKEp{sh%fYxI{8a?HxMqIeBaR7sP4Y&&Ago$2Mn& zZINTFJ;@`_1;!pG-CgIhIpW&ve$bTV{dljeV16Itr*&Ry(h4GHrKd5*TIY9}v_6JD zxV#eOJU-=i($d%YNY?o?W}UlshX;TM1Q*itm;I5Jx*uP1)a-|TgdCLhUgGNgw*U*` z6?7k}im~w4`;N+@ZnomPgC|`Xd0ymOjFIA2uunR|SjYy6_zMq*FHP+!z~MWrv(Gl? zgx_UdYwZb!#7m~a+MlPw<5e%?dAad>MXK<>L%GD@P3*6I-~&JVh5-AA4E7P3>?b<0 zzs45lHlLl`*5&OKPSAU6lAr24UbfW^!#{8a5rMue4VH;#g%u)a#Z~le$pbB*OdE;_srZxW- zUwuKDFBC1x4&B`iS&nR3eAD-2f#eX|J7vI&Wbn)=XXW&{8*zf2^t38)(FkxL2TGlv)I4i|Tu z1Yfn??8P3xvv9vFk8I}s(=c;Z##}QW(j83{@vnaWLwYf9%8S1Oj_gwl#{*ke0ArT} zYvX{qD)y?C>{Tn+t8Twl_5!!xb9Q4A-fb~5lxo5&2p1C$-jmxnlkiq_E#h;a84E(u z&jQ^BeR^yEM(y_syIAwvj>}G5k8omJ*YH2`^1>f)w46f+tY~YXOZY%Te)t3SWgp(m zS;;EPsi57m!O3lA=e7y-J@W0~ z-jjQR;iGG^!|@q9v*~Q~S&j#~fbY6SCNPdI>{o;b-}mt9Y~pkm_vlTE?-AMU#ot7H zil^S`#a*xR^hdnspS-wh754;jjoe42p4Yti$%=o>a{kWTZ;*_f{tFIbPp~JG{}{Lp z+Ssy@_W{8dvAl|C;e=8nL*dN_WFjv#99DE;}tC&jE*L-X41=$U$f@Gai5eq70wd$>`b^@rK5Lqg|AIM z^oti$MrAoaoGv_J;hVg^N++Bu*Q1g(XdM!rNG_7>f%htfd%ra~x*eT$vyN}=VLwXW z;f>ZSkG0>L*Ay7@TvOm;T={dr3a%%v2QD9%hwJ`ave4%w`{H`x@_f#YH#rkShlzQO zHe&ZyyFTk)}H=@RlI>%I4S(KOxiNX ziq2wxSww$hlYuwzvZ?Q`TgtZ+9~>yz?3K}K!Vf?nuVrkKU1>)yF3acapW~jhn}1Mm z>v-9zbor2<&{wUC{Z(msrjdvL4Ds?jtnmN`>@P{nGlM)+LeBYKo;u|L&j=46oyOkt zDb@u4DwRKnwXuwSu;hH_oU5~zwPig9*_x+P&Q;9!dFR;+nCAuG?rPuiZJ#~K_r!9| zr_N!jE0Iz79x~@IZ(Zc_RX2O>o+%&PAQ=7z;TUkm4Vn2B=VHIU(TXPFtE%@|HnOy} z#Uo1XQ;dzquo{>+^ci-((1Ux1`DkH{@dnIJy=(H(C+z2uZ$lGyhDjTASM{bz`R$`- zV$aKuo?8%oi9%n`L>0JtCFw_*H~n8cSI%JeD5}r<)_le?kTE0M1|D~m{%LZQ^4&`R zl*@e|Xae+SD@!kOM=>!_b`uZgN_BT&7U#IwxbDl4!p)HVWUw+r9_BQG}pYM1e$3CC$IQ3;A zFN+1T?JT~(3=SnXy15fLpf2#XMV?aTHCVj6K(<=qtwFo(Fr0{Q$?`$)854ZLL9C_d z@lKY#$KNZl^+a;|(Xo}$<6|nLx?85TB^;|a8vjI{sWh|R@_XCA;fn!kNd68kxw=GWs_?@5j@>+g_jGb}h)I^n^ZLBr~?`DV>Ow@>Kq z3Eh?u?pf3NKJ@&->EUg>gHijwR}OwHn{mr2Ryp)W0uL1gmwZ(nv=4k0qfPIli<-i_ zlAE8nudXI7~lX z{KHqDBK>vN{mqhyr z*9ad$pM21x)clydPZih(erKO;{{_03N;7eL&o*byre7m@50yR`9O(>l`}LDS_8(M+ zf0+M|=x6WUYiu3u2$r3ry9YbU-tMP+4?8CH=x-a_b-z2u;M-fkqpA5a`4*F(xHjn2 zOy(lUT*T@!!qT@WrF<~j->q+M|Bi8%t&bF*l`e+KcTS<*nRvHMpD)8&@ngH z7TPlz4}Hf-S~%jlbxNCYY1NX5zN_ zUKUO+<2>RI@adD-Tg6}VMcdv0e;nYva47UwDCmd&tv|i@|D}ypwN162n zAFV)+IlO<^qh$pe+iC2-;%k7Rd6X?F3@-vUHUHxA*cWvJUvupZJVUz$cmD+Eq~N55 z@ibw^tTq*fPg<$4%6CzD|1|J{lbVBA4dd5ig~2^~;|5z`%VQ0eV`OgVG|6U6Yz6sg z{{)?{u0tf zF})%~$tzys|E8xsp|pOHGNrLk+L)g9n9}-01}Kev!P@k+Dy8*^^d(JmSIs$O`rN(i z&0R0%?k9{R{oc8bInzZOjRENj#Wa7#?^`(E#GIc1%wV6Is%kH7nOS?Ovkf_INhj{6WSV)q z-QY*<0}CSWA|KP5o&X#@b&|CV%`OKRle}CJ%%o> zEw+km!xIFA=X*r%XtAFBZ}9o7N*A}R`PjuR?+|bHhQRN3!pM*hNryx7=!xid$!46! zJ?OL0qgqkW%)a4xR%LWIdIpCnBO3BD<>c>B`|)4koBhd7!6G;^{|Mu96>k(?$2^Ir zisf-evnWs|-Xy-t3X2yLJ^m1$?Erildr4DoYF{9J>;q`=!nNoqpmQpE0e?DmLasF` zkN=3z-9Wt?sY9}?1%?;POB5AzzJB+DcG|pt@7)VF`MZU}fuCFv?vxD=cqwg%UTw>O zuR^9Qd?4Nw+!6gmFhHN1M-vY(7X%knpUN5~J}}0=FXbPpx$$HmYpv2=ws?;UPhMNj z7^LVLcy_JF`(EquzR}VblOw+Vzo=*6zo;j+3B9|tdh{>J#6H4kydBhJ`Eqt_O1?`i)u+cX1A^H72{r!oRN#aCuUPvbkwx z=29ltEA!I~mv?h$t7JubcQ3`eHI$}!H}2VspDRO`bX(Bgw<^fJcj4GRk)JT%vw*`H zrPc_ypT)Lt_s4I!vQ$JK}o_-}keRt%y#b4#8SJ?br8q zXvKsv=Zmag$p9P2}^z}+H(-R^aHTXF1P zl&6liiw^8moS`@Fxs>q!%ZfAf#*O<9ahr$}?!N-x#r^yT-2VV+zg8N!e{6c%6Ye=! zL}}pu(dlX6{#PltN@?K!Vd-hY{iHoj+@G-@s5P$joxZj$`802DgS$KEIrBD- zv`du6-25Rujk);_X(N@k4*Z#(w#AE8s8}GNRLyuh&TS5Gb@N5Ts z<~yB6FVQ$NmZB*Kk=q!)8yfvQ^m+^U{~GKp2`7CejlO1pq0!YEcWCrTaO8%m13GAQ z9{5%?`c>e^tVP<5o!KX?bQ+EAaYLhbeHo3;i!}T3V^0_M&7|Hxba#CP!j3IiK? z?7#Lxhta`YTlt*<28TTazA(5K`Ym4Y1TfVbUwqHL2G<_dBIahTw+_bQH%VV1?G1X% z3Pn?R@lyC%^~K<~za>X(*L_gv-@Wn$pNzli{w;0N*lv;!JiZ_w{Igy@aQ7SetKjeC zPYXG}k`HYDQa(5)A|L$IjDHe*BDwIBCz8EbQ)`2sydXG#Fmf=RBUtk@`c(XOi_SEV zMY;I8w_WP9oU%;phaiW}@nyd;v%)&G3Z1oEGT7TJ9%|=;Lk?nF>?7ppXAP~L+LXEI zxo5hBp0hH>&MU@M;^yF*GRG2s|1aO9{NB%!(`)$0!@l13q6*)kV!q4JYs;nnMVDHK zs?m|x8EZTclupc!?Balm<&$ya&z7GKrfExvJgTmH*W6#1_>G0Q*x zM6G|~i4gYWTA)i7*gNdW`ARR&S$cEc(uexcmtBZWuWg-pW7D?U7C-bb zI+SlNW{=|QoCqOPeA_p3(|lk1sIk0DI)Jv0{Va2SjIj+>Jm2X1;r#aE4CK5&9M;jc zw=4e5{EPq1{9FIc{P3dxR{Jym&HUN_X8tar=%FJ?GoH^JNt*Gj#{a7Ee44r5ie3KV zYW933A?F-mu{IAK2FYcre73JhZ>aZ<B3YOK(5ma$fjbAOd{ zbX)JE>kPg=5GoG_tdZ$;1gJxKwjgg?$Ue2Ds-oOVCP$p)KUtLVVVm&Pzq@%X_O7*s zq0#a~$+4qilZqOGm&z7K_wJ!&a%7Bl7e3;i5hc+%5Pz>T!`b5Tw|vcR=3j*woHcOn z9UBP!q+PnZ>(uze&+rfENmS+b(jJwo?=S)7~dT=PEf zAF;EnkwyRFANP3wDDwU>?~H%+G5@%af2hxeoF9a+t0La&KfHfz&$ULjb)|oo`VIxb zA1jiXhaB)~H8kZ$XiSjxx}=ozZE(nh^MZ}pciq!IF_68cY}kswW#4s4yW|V8UeX;3 zt|=f+Z$4s=%-h>~Cg$iX>lixJzL`AA zJI$->_td5DDSXEc`+ie>-B12GVbS)+AU1&GgR8=0Gp%1sUMs%2Z1w;feT-jE3OuyD zw9E>p|HnzN=VuMYQth>d*;i8MkEuUBpYnXleSOoXZw|5l&KPvoznLfcwsR5h2kztC ze9_6w`8rFVc?q-upZ_l{<6JWv-H}rZH)C7OvXwuai=G?$N)hHY2R{)Q6d9X2IWZA@ zn)?cst1yz6A%Cav)eMfTh{b@BzjZ0y{fJ4E=oLo3S@@Q}Hk z#ZRCg!ConNw&2{CC{{RM;mm~IK?wF%xKm=b!u=G^O868mQaC%Y5_k`ut8h-DLSYNq zFZjA0i1<|K0=-wx?q$h7^}gYD8&R#Y~J2KlUV z*7}Ir&z;B3O8d^{d~OBu)103#o|^mY;;A}cfXAAjyK1qS%ZI@YiffpAOS{IPVeX*z zhkkm6!z%8W zfrD+GX9Q~|duMXvfSKxU4;AEOTGs}-hmc##I~S)4=Ldl)o!zyJT2Y{SPTc$=de?R;+mo=p15?)G>2hBmCfh%u0^y!IEePiXX&!9A`luov%# zau*JH$;)4N@8e0v`4nZ?-5bB zmiV6#ee(*Yav*O|uq$h7ais%ZTOZt7#oDzFwXi4rs&NpV_^Py%^M1>C`4~4p zjC4>7Xc zUt11ey|FYm*ukrqbodp`-^0{rWYV;MyT3<6aPNxI+pYF9@^8VWqIjXyKWh2MEO?3y z$Ps3Hll@+H)M33I+KfkNwDU(quDQ?(dIA9yOp+zFVdQq4n$J&cg~qM(~sN0 zx0`dULy~oEK8ra-|8H|o`iBfeH2o&d?H1mgWpDQvG+2jbj-H@)GY7M3?jC))+7_^C zR-#`;n>gzzNa$Z9ynm(t=OA~wH=xg`f66u?a8*8%@|sTzy}qv#-s87wnxG38P+o1R zrCznoN88kv_u!2eR*mRr%Xw*S8IWoVxH`Qp-MqT?QrF+OcXQ*E3>&`r@sq&!mRlB% z&iCpF^$Lyd=GC#7u@z7Jbe6k6cVYK(;Bp-BSp}R{0! zJf(-x@)UFcp1YFqtGp(wBmQs1gA+!Qe@Ere5gqaG5Z}a|Vd6ij3}tl0ze#)xaQJP> zU}m|txmEvm;zPiq-k=mbOP}}D!Y^;*%0F>dr*UTfgTUBc=(eGaTYK7rkU`I6?dpu$ zkGvV)eSHpN&0}(dBnuUd75|`lx$@jU6g-R^wDVoJw4-Oc{*hso<aYn3;IE}QZ`?drpsJS}b>6JDAY{-YZvKRjEWsmJ7bHZ9Nh+;F_)<=c)-K^C*n z&wXIaF73DSYIBDEzu+s%7z6VU_=?~C^-R8EtGtOLFM zw(VApy>LnkbN|usm(ZE@p}*}X4|IK-W!Lw6_UgI)R$V=*-!oSaW*j;r`!w|LEBLD7 zwEq9-bNye;w`h2(|M&i@{zC^$|F`zFt;`VpyS0CDX-$9d+-+8-`g;rCbNTk&W_1gl zvedtB)~QnqtLR%LeW~EC>Z=`nJU|~y>1(XjSMH;o@scQd>OSy*AAAr1FJv$-*sxoO zu6mvH)Gs!A>Y_2tC6(p8-y2=C@S3tsW6?=RS2$MXi~e@ORb}fBTpnJxW@6ckld+q- zt}6WeitE0Z{kfk;O(>?d!P0R(#JxNy)={aQRIvVrud>KdDG6B(>?srv_9dZ zSq0&PANLG@@?wwh$K$a}yqPmlXyGy5cR3R1pT<*lLTBE6Tj@j~JMz*~pDl~S1L+L? z1I|!BOyh+<1Ggkh9X-(h$7by28R0`4Z{Q!-@vrN{N8ZBj;GC)9lf2V*U`j=_4SXbB zO6bhg`Ycx$KEVFv1oH29v9YxhcR%h9+&6Heak-=^f6~gXS&2Q^YJXp&-##H|jl2LI z_Aa=8AoqI56Vc%#`1R}c=~x69<&3(eVnuNz)W zxc=ThnD`KNnmJ_utUVt%W2x1CiJdyXef=-4J)mapi6Prtn=accqW=Dzb*jx@mcDm@ z?)vmXkNslw*slQgE(Zq30gF|@WF@poYxFf2e(%d`9E-m9Z1lai-<#K%WWVMkTz_v) z<7UDukxNutof1{VEwVC=Jy`a=iEYFe-B-{!*UEUKrP1OJV|JpHxEAib&$Tk&(04O3 zov`k}KV~5}4<<_a{xHxryua63;eB1Zgg>i8hO@aBFw@(>rQkEM0=om~h40kidU?g$?`FRZoapSPd7%~EG{$mXlJ4+kzH6-L`U9C^>6@=xmCZUNek1WEwCxRi z?|G51{QI^tTMKpw~({LzcMhB&jQ+ZA$N^Wr8GXU??1C$D&MSE%fRk|p+u zyzIqQD6RqcVJ>gNo=tU<_j8#$*8hgdvLSuyzFn;U4HMXh#_PB%@U7;?*k{;c%(ONn z=D??HW`3UtImaGE#sEE~9qZr0y?{eML$bH^l0WqqeF^qWw6gw>>z!TN-^b9LUgjQb zY!0?tbf17er~b6i{!jSUp6(yU&yO55KJS0Yr{6L2-6QfQ{+HpES?JSVN!l^S z;7B4nd~`Cppi|IupNt!itHO=Njlqq=4aW`fIW2WLwhwvbM9L-y`yAF*w2(8_AomY4 zxPNeh{Ywn`V06bdPD)2_Pq@?Y1l&JBcVoQ?bN@j4dq(d;`Os(Fsxn@A@oy%K4xe~L zQ^v=+c&a@25VRjX!hUoH@P7RE3+&IPV-9%2jO{J9=sv^i=5=LCipDX9UnkDrSlh9QV@Sg;WKq$ zW^_Xj5L~w%o_!YPrN^J3--|pQPZNhO z{~gGup1i-S{SfDFlYCDumky+{sWjlUF8|-rGnQ`W9s0)_@KxHF>tD&ouxo^V?_T~R z-F=jKa+M!K5N>EK5ZLyPePwIr6;`6 zYtt09iS;P@rgIGGGHmDG#it+VgtfLxN3tH@f=|MRLVU;#;kE_XJueG3Ce~aXF1rwX zbf=4h#t_HdplB&P#{}j&AHCr_fPt1B6F58Z;!j*vUtMI2O zoDq%zyYX!b2g0#deDuTMW5co4H*kN%=jy%B;~q)YRpEWWMl0)DwB`d~;zQ`p0qAhs zx8Tpwsf|smjF#f74Sm5`vX%4w3(>zpcG5bw!r4Ej!ueoSh4bO?3g^I(3MV$BG8(G` zC((x3SlURvu?1H2*eJr(A6wA5jB#*ehfg%>@Ty>0+v@A}E>u+XDKXo_a-E8$-^I5-u*^_h)zSzrRX0^SnHFa!+MZ$hpVM^H=h;++=Z=B`ps$x%DyZ z)O&e$kVkjFH23W@EaxxhG1oVNH}MzZT83xaPk-S4^LMImmF0xaB@c8zmheRne~fN1 zW5M}=q2aOP@WG!F*4QQS!TZRH(LqKJIm;i7<@#S1Y{&*zSbP@rG1k=|jpvaj80&=( z3~BuyWE`=>VrYDfdq&l?F&kavN%)g`JXCOy`$q>E?}OY$I>`90F!26R!3x&j2=tQ8 zoN!l3JOpPkj@IRD;Vask(oL3~RnFU@H-5+1^f&XMu{fBv{@C-oIFkBnQ{j}Jvg%6d zDKE1)A7jncvNw@l@*7r%Uh*jECDSkAoV_zDqr%aC{&VV6-dXAcT%P|j{)y-=GEkx zd^(Txel8=A=+qyf=dJuxGK_ssxOzXMlm^WIE=d?29P!!yp4P|`U3oAJ7ZDsV5q$^0Nv!@(%~J3{HHSO^!J5){~39eWLo=J zv-h*6Q+dxPZ69kmJ+7FzqYKUn8#p{{Imo0xF*?2T+;`hzt+Kt4g-)+@weutQf&&jT z{?cdFdfS6uj>5n2^kzFTACiwL?}>Na@!B_sd%^U@otvH=I(H}Na}3;)nj1gwF2u&C z#@xNDzLz{2*LfY|gIzerRrB#Tr7?EZ>1mAJcSzf%v|S-*c6!9 z9WOb@wVlh@9)Hhrj9%6OS1;@Ixb}<;JY!sYMwT$1d#6-JjSMYc_KET$H-KLxE0v8; z!QnHEuVhduyeJ&BaKDR#p4HfcgRH+NH)x$|&HR_*mgBVBek>on_<(%Y$o=wXgq&LZqthmaPcZfkr#K4*$L!UbWQF1>IHg1K-oX;0NmWS@sg> zzi5TZd3TI<=zb4$d+j9eevfo$*xyCZmQGe3Z???~BIn3P-^h=>{>rkI*wT9j+rX=^ z(H|Lz?SAfC1*y4)~w8~b4r%o-L$X<2=w*37@?v@N>zRi0M$X?E3pWBuF?%C{n zyRrZ6PMh+OJ7w6?vC|&#uxD>i=Sj!WF>2>5A$6VvJ=l@5w|CEz=)>`jQBV5$Qr}Rt zAK`U%z1r9H3PsoPhS4(YNZgJ425vMi(u=!c@G9YDq4G6(;6h*DnksN~8M@-lh0qbs z47VdcZ!W>!7I*X$>@ne2qQ{Wg9DawfAJ0AnS}(Xe$y{hoa`D}@UJyC%#dX4O;vE+4 zfBxyk`SA5-n)WCkd2uIICw%&`D$Dr+`xm`+2cNn=#u|<3+;lSgRQwj^Ax4=W7~4ZV z%^iK6eYFx-kA34p?z7K3#7QznF^ulEPFk@jrIZ=!v_*EfB5)%bal2)@=Idz~}uY#LzRi5);)_mW3- zYTQmOoW`6_W$v#92Ce}XrT`~bBlEcGHekfSN-Oz-;E?bWq4Lei(cxov!l!em>(9uU z)xR+K>e+3+wUnQF!2a*q*3;#m zshiP$u~&W><(q&Z*=E>KcLPTzhG*WnnK;2-FZfh={tb_VheF&pXgchBduFTq?|RCL z4#&86g#C(5wd_-mvS!}(TT?ZMN$A1{N}nkiNsk=%V&%t1!RHS5l^;GFEGw1Gxov^= zPsdo4_rI+>*3!9Ysk5Gw9Xi>gdXcumg8@f6J3x1nId5e?+N%9)y7lW`wSm(~hJldbj-CgQF zY}Qm>q=axg>qxL!MEFzIs@ic5VZljX{FBoZ*DunW@Quuk)=&=XLhHB>aXN>6$H)=# zBN<+N9`QOiyWYh2jC{to@^`@(FZ?&HPwJ@`zdOcqFs#FOe&hkZrI&QQ`R*C1tE`AO42t?U zAOG>x?Q4H=yU#bUzg1M+|H$qY1#P|9>kYJ=Mc9D<6q<3kE9Vj5pWf`_k7QUQ4KAOL zE-c}e#z5G}%l?vW-?5AHYu4Y9u2xk1zcsAFVXwQk_?=F6kAW^Ne2zR?YoF4OCiHbm zX`l8>S~I#&A=opx0o>Kfo@YOO{h+ZbJdk`_;omja$AJaSl}%myp?||WTO$LEPwKzE z_lYAvc{}RWKUMd>j(U3`d(USt+1KbnM5wz7yr6db_OkCwD{IQpUe}gEugHZ~ULQ!y3^#1iN$h+_&o__adJ{ z?jtx}(c7BezN#v`_yNxSd>QjA@Tc*=>MPiX*C7Ke1Wu$$WeN&gFw zO_+G_Xe&5bJh(&rYVe@`SIyk(KgpfI82D8@>q)}GK_~Frp{c@A#|f)%3&vUN)$X!! z$RJp6F@G=;91iR=_w~^GBi!9R!oA&W-js>0E(pi>AVX!0+P3kI2zPRqP)B@szwpsj z=Y)ft3&d9S58nWtvWAa$-UCM!TKS3K&?lF}+Z@XK(Te5u{=5d{Xm9t-mW|pzk-NO{ z_$%!ZZ2Kwq0sZN?8>S4!v+Dog9fA} zwHIAlUng6-w4ZSQz?y?iZ)Cj37^4JZl3<*W19olMZJjdXQqmV)!hT`tY#rOho#j2a zfc2Y)ssiotlK!L>hL4rNe+7^^WciV`_)cNZ)iH7eWC$_YEJ0o>oKjnt`CM&X#&bd1 zhAe(Z@gw7%CH~rI=~bnTx-Xavutx-DHKwh=%zj|<1N!?RFfvRy;aljvtIxg?dmdvg zp0S?t>Z|(br;eFsJbOmi>+k%#7gk^Ao{pdT--HjtADOYu8yD7knGEbfN2XqbFPo$W zj`QnsH#GAaDZ(#db z;>tVXdPiEvxcl}IUYy@@-z^jWDjaRzEu+oxUC;(>&K50x^b+{F9p>$_;n-EJamTTh zH2vop)~Wo#jM*LN9vGTYJX|;s{om|_&h3JG$6@O%n3zRad4K1X=}Q@X_raeAJTgW* zU7a!6RkxfO+yNBaO(adY2wjz_)1jR%o&~R=NBMcYrn%daY3?B`wNB>RufGj10iG)2 zoWhJHwCqS)9TPIXSjYcE9T)SjT6hZm%c`hwn&87E$4!0L!Z#!sGlPFQ%h0~Cwi5Xx zeJeZ}IJ5-%t8YUW`(@ix&U5MF0oo(G+lFtTzoYrjh19G6nZ4!x*>*c)Vm(;Vo}#HC zuk2qaYv7W-=aK>Fk01+@-GjLHsn=z>d&_sT?1{t~*%fjti+M2n*}1%zLi~kZUBC0% z@mub3sx9z0Q`H8YimHVwHDdxJ0M0hsG4U*rL{e4BR$secw{*fTkEke*foJcVp5x@O?l z%9`tXZMp@y(F9*8syzg_i8s7C)7{o=r9+x$6^ zwyV&sxr%)z?if6g@I-+pSH0UC_kHx|$KY(27V&O?M~mhYr@hlt$ZJx4IhVHF2fMQ;!8;H)T5xM+9BQ3tO@C*S zSz|lIJ8Yk9*2fO<4l9wniFf!fuU^?n7XMJl{=vm-j0xZB%LTOQ2>9zL`zjYlxoHoO zcAome+%|t5`+5d{xoOXm)}OQ|S?}rfsgK$d2p_6GbTe(UmJMEH3_Y5s|9(b4PUFSX z?FoAStRH&jKh+)|X`C_U@1sp>zy5cUdh|cxC*gSEP1UdUEqnIn{kb`(?=kAl?sdo% zYU>`oM7FwDdHl+a9*%3>(SaY)9r~Z~qr2F5@Sj;-Ea_%{VzD3ip4!sc+Rz;6^^(@jm2EB$ z_ekUYtL_*1W%oadO^AX7_Dy}hjySi^ig#g4eV&ra>-A54oZulc*5D`4Pu1S_BF04ZNrt^2UNjT`eBusJ-|*SMC2wLR2ExbQ zd*7}EeCLTT;W=~At7#kDFD#z(#BN`-+Dm{AMIiueP7}NXd z^A~_G1ZP)#+G)&CBhG_9*>;c0n_=WZ))qKrf+$X zB_6$d(DQ{W>IfU!=H`Xncb_zO_NL|$?#wxp`~uvci8u7}f?WGq z>J(ol-5r%N@$6Op5AlCno*em^SBKI*C7i6%+00p-&2&W`e>QUYZpi1mBd6zFW??>e z6 z`m?O{V{ipH2im_L-e!Fs=W>MC@#d>!kHZOXtm?UGM(fkajbqH;1o_N&P(Jg!oxPm)0rf^soFBP|vO&&fkf}QT?(NlBX!T31 zVjrhAeV?@0eH$8soI~i0rq!I?6qvJ{?<$RTqWBW;OlK}}ro7qT=0*NP-@#tSzHK!$ zHSH{Ct@ZG^_H0kbtdMqV{Eto>9~SJ*FntR-*U6{t)8x~aYvqGKrWijzG7?|1{x8&> zcBT^~kLuRiIJIy(urdvpnF{P&3k+SunGW{mO0LF!ynCiY-kr@ihswiq+%uhcNv_^> z8zuSm>+F}^^PHZMmAQ9@TiLJtd7sPU4Og4UgKqNrmpISS*@^aJDo6Pd%~jWLYR0Di z=`$V9gdhAt`1fA<63Xj5$3kEEC1C6Y?@Z^VMWe!;TkI6f_TxWJnIX< zpU!pSe|Ou^ojSBH-oji9&K?10h{q9(JVdx{;N-GW_&M(G93zo#C_GGE1)-s=ArcN_+H=v?O{xZ{|2uKQKEqtheeFI~Ml zokbLK-msOsyUpXZ-{1^_aJY(hlfWZux+3=jPwd9Mh+B@k3%9;+$Qgluc$pPl2Oqgh z=fB8^9|Dj3Pv;i5vBq>xb|d{y49*J2=ULG^)MtYe@+0>F3&Jsv5a;U5_Kn=-#nltn z1dK_>bcYvLr#RqAIA)#~cc1E;=Hi%Jytum*2YxyVp4kAL=`6TfJ}^=$95YQYgU@KW2KTr)M(aO?W2!tHa}9B-W1EL# zuJr0C3`22iORam(>o_ycr0p6rXBub5wC_^WzmPKqzlrvj%cno16wf$} zG=6@hukrPc&A-MULEv-4c<@J+i$6BQ2kJ~%=VCtrkIe1BAvzNlKh`f2sI8s)>RxvZ z7po24nXt}+g;S1FCKy0|zAO|i;k}O2XTmDqJF>wm+nusS;1A(|O&OH()|ZU?tISXY* z0C>&4^D5rMtd}zQ10U*HDBX%CTTiD^*hHmr=;M-=#A6v2JO`GN+gja9| zf*d;bBKDW?pH_Htg)>3^#tJ7Wf1P``xLGi@2E9vU^x^Rn%S!#(g9^{FZm;FM*V4Ji z^z-e{-e8VNkNy0yspbx_7vD~N5%KlJ&jrtvT^MqT7&q{p-UDA+{M7mO8{{!#c<(Lc zdOKWvUG{p$oo?^r)bBow}2lx zQ?rd+Z0}dwn;Mj1ee(Z@y(!wdR`#Y^=J=dLH~XBEi+s)}Gkng+H9lu6<7jx}H5JZ@ zkNwdi?qt2r`VdX~i{@t;cLJE>E#RSTUi?nQKS4ZmK8yI5z4$*W{xRZ#0dU&-jb8lk z72iPoBF;5{!(V&xTNMAG&pC1vY0pp7i=u>!4TR^`k{$ukF}S+K^1O zSK+ia@<#QBq}Q%H6s8@(+s>Bs|D(T=O#PGbj6H$Ap!~;JEAksyEAk(tO^j1)HR^IClgz;CNEA>DAqdf{Xjoozd z28_AV$0>bsg^8C=S@q}ngKGwM=zpF11G=WH(UbJMg+7lo^Vlyk+{|l*Gfe)%3TKFX z#^^%%4HeE{`HW$ie8zO3eCG8$`3owXT>KLsS4OAp$-1cMD{z3-lAQiy{;$56yCU#G zG5!U=zTUtfed*@)H9n30au-QFj~{&dg+6_!)ZWTnt~a1h@FS1M=Tta3*{?63$(skO z7@K$_<@wed2wz&hw#@c%XZ}UXOGjO@z9YjqV_xTXlF#GT_{~{!Q(gIUO?9K6Yk9Hm z(w2?5b-3qoYjCS^t8goDjksmFCvcD98gLKd>Trv23vqYi7U1ULYH&B>=HO=HZowC;;Rg>O=iia8v%wlL`U?*cKgox#Mp;=S`;4V?#tpKq z*#TtyS@w^4Ujh6)qM7%CW8G!DORjixh*OzA(J-##h>@EM~4pv(W z>nS?2J~i#&`}1wzLffa)4%6nFC^N<% z6|a0o{`7XMujF|$t&D6qBq^)-eBz~heyG5u)Bm7-En}gf{QIC67gyXE=-#}_X#A#1 z&f>BSkF*#(ru!Td#CMIMt@!b6-EFm{4O%mk{qz9(qWwex|6IUaF8G@6JhO+Oe%%{E zr#tNOVT0_s{3kV*59y!aPvzad?#wZqNm^PnISJq^ zu`YvqyBX1glj+M$zw;6KKHA3j&3wO>5k2-sMl|_WMzpQQ?|eGX@3et)+ZK>7LAm3^ zqt7#FYoPrh^=FaaIa!B$(C-|~%Zw&^Wk%bElm9Ww&8UpFm1IVfqcWpow`R4kpx?;z zr;DZrX;*X3_T^=&gEqAd&WOf`;Am64uir_KCozU^=tF!gZK|S8|6y)*UZSvMJHfNvuw-dT;tX7;>SEpnIjEXN`*|T6Fj3$-(wv;>~(1VSaSC zlW`C~XYdsApxMA|YAyN{R-b9}N^k&pYQs!)PAdqT@kPfp&iu<}0`rNE25XW#>am&3 zK{xb9GzZwCs^_dw|CBsENxmfc)IZ(zJHdW3MqG?I$;Fbngn<>wrldO(YvkJ-cj+Xl zPHXFUCtKycwjo1s+o^k+*jAXpUe3ytzJV1{e>Gluizii2tg+JQkHmq$vA!asdmCqc zjLi(0cQf|*Gv;sQU(M1}(70Jc?VoErkq;dygs%W^#TdVOY|f>|p_G1`v0tBOFDI|g z31Wp6PDXa08F~9W!djk0cH>H|CU}RBXz@ZBCA2dHc zeBqcv=5J*dVdk%p`Kutz-fKf|^n2CD;I-h}6pRW!4>D$&(;F_dhf+UxkWD!&7&=iO zY?>-wvj|(i!Fuc!`zM?*bzH~Wk*Rs44$4nAb&UI(T}T~fuE&TrxVi^1WDus@H#~hM zaExfJS;zRMZ0QjDY|0j~*T0zlrTQrQ(95CgDqld`%NP%HCzidj;_|6eIzy?o^L6rT z&6xi&hr($m`Jbt;(I4IH?{Na!rjkTXe(!>WvoNl;kUsL11bg2F42IS$q%VXxF_b~e*eWU+-C9uw%&L7NwYVI9mSN|yA4#Lmy zcEbtbwrt+ysD69$CXGoOV?EJVmeV%5GKxNUP8&E0JJ|E%c}ofmZf3okFzr;jVAp&D zuQP#H69$fF0>=t3syq9TiJPN(CYbOoZ%^4YlYbo~Jx;$QpE{WLaKT65?#;}%!UtP@ z(HdWv7As6! zCqr)*Y3=e2e=$=1SP6~MIBHz=)=685#sb_a{XSqe0pE2C;}*nC@LLxNj@yXSnNW;7 z&Z@6!W{uV%ux8#K zgLWnd;@G3h-h=Lsr1&nu=E`SB-!*IdSa@$7#J zrW0Ar9pR(AVWROiu+7-ZFXN8YByiEB*_J)IEEHbEe8sWzpm~!o9$9zL3z?g(!j;HP zAMH&Wd+|P&*6x4E9;esd(If3Q)7s1aJ!*LL29NLI+w^P4Nc&awvkuz!Ja^iUR~USo z>X+a}eN!Jzn7)+)BYXUv+;932_Gjk^riL4M$}@cYYy07`%>;Jj}E!{(MEWuP=L0b-?Nv_kYzs-pD(%J$t{RJx`6Y z?@w>f=+X9l>FuEnrakz{Ea=@b)&n*pVmJBzz42n4G+rJ1ozyrL`^-3HL$@?e@ho7d zuaEm6-hBRjw0$G(X-dIkZMl6t--1uI2VNnnGmRMKVuW=c&dhJ=#rBoy{rcs__Js6) zk=OJK-_RH85-eHv6-I9MpU{dJ@Czjn>NN7 z{wQZ+mNhEq>zflq*U#XY#Q{4zJLD`uKAqhS+t~8KJK6GCPp>1p4|ZW)9cG@!`l1t% zZ(BOgxx}&`pl#`6QJdhRr{=qR9`6< z&?3!=`36qc_;Yg(vIfP^T!8-S!<;=eUOsh`)@H+nRq!w_4Qu$Q68<$elKOu7P^F`L z5?afh4Si#N;8|G5gM{xB;92SzB>W&>>nr}T3!gzB?PYB#?qFA~t*4Upw~3c~7}?6yDF!Eb0(!>ug-KQ}kbT z!vF6mTX`98c>=2&E%s3Ga^kx;qQgtufFW!)U1n>&ci^*Nz}}VKR@#ECEO&p;{x_BZ zpThjH|GGP-c6ehU{v`GU^qv2U7g1ZXaRyF{19o59AU;qybPWGK*zz?4Uk4}olKIMKc^E{F7vO|OI zkz1#)H#`U(=yfMeljJ8T3PJa%@by)~1&Xy`E}7j+5i zu6a678{0P8j?LlU&=71ZIk7t{q6gPuzYVyAmyT-OdjqS`()nf{w_aw?&3kfr&f*L^ zcdOr?)U9Fp3epnH^(^it<#l^#dCNk}d5XTyLT1@=r{(;FuxX2xWnZs&@YqOj!DYZ^ zod3!mSjA;N@m`+<_YM|)h`xTQZ$vizoEgZj`Szv0>4crO7nu+2?e4vVZ}4T?P44$> zeV2uVo0VR(cMRVb>b=ySHH#JQL*CUoug*#M^nDKBtK9F!7xP`rccYiC@ALQu9+ZD^ zIp1YUcfYZ4Iz4zVaK75KJF$4Q54(1p-Jc77#NPvr);RTkf4o!7eH7F0TeIxm-rVc` z_n5!;pjeGZXZB~=-AL=e&DBA>3*SZJsd&pL#=1`?e9qpgu68AFf2<)bmZ&i6N;0w_ z`@B-*O_r~HV~qYYPcb zw*3)tw8zEwy=DK4_FUn8H|5wL@GTt(+V)BCs$5(2u~)Zc%Z+U7ElV;x-FLhw$iJ1Z z04E!WimyfyF-pkzXW6oNc2kFYc*Ui1J5v2d@ z9h{Cl%wsCgPA^ZA_O)>K*O7<0ZDsyc&+A^EW6HxGN^h?=_dsr+$C%28k1EC2yI6aP zGkg_1QZf9K&hKL?$A0VU^v%#Y?4i(~Pcr8N>tE9DVot_Y?r$nbyG`Ejr{zs+e@`#( zTVDH3USO@G-n4$?dU;<>>qoWh!qMhZ`Sc?;;&c6g5Ba#e+nyJ^ejt(wYcDHa9NCJK zJd8`*=juFJ?;FQUT$&Xh>-wVC2G1i?xeT4oChnK~;vIBt@RO?p(UvF9vTxt%>RmPR zFWHqS;;xwHSp1CWpTU`wm%iFt{kQ|coi*|6z_8?{=3QZ6dsIC*&6hn6J*qyDd#I}x z*iv{OdF$94%LZvY84~@?i60I*f+y8Idzj&)cLYBe@7DR6(yXaHxD)gv)~0yeW^B;y zuj^rVjIm&jvExq4)VcS&XF)amQ+6o!&q>RtdGqqk^zzNtm?9t8UzL_mb4EK=-*sNT z8Op~Pe;Q&ukOwr&pA7urCxNXbaG3;tlfcWD%}*ndtkoUT_wiF#(Li(7X7Ikw+9vFO zZ0b<@Cmo8?83m1uXF@VCE4$)u(RM5Qn!6>NwX!GOt+5k*(|8qk_f7BJ-RIKlkH$OA z-F)k-+1qE2gwN{c8=-K{FzhISueYD=n?4!1ztwkzbL-i@^$OpXd4-=)hL0nTR<{3R-@ejcd?lyWJ%4@>RaSV8h1xh2j=NU%`gPG2X9a-s*1wk7z#7&F$kX+EuGPQ`#GgItyYiV$RihzhmhJ z*+96itP$OuPv)H4#kN**u0A`9yK{Xa&vSpcWr5yl^6Utt^4&wet>jzm=KDGMnr`R6 zntT51=f7Vl^GC|;rOe7K`={hDnh&o0U#$0Ze5^VT(zhklKgid)As)zb+Oo2ocxxx8 z?La3d!T;x)wA_aHHsW6EmkZBgq$O-)7!p(mPfj{@S?}`~}aP;#-Bknh&_=+^MzsbMk1MF6N9o zRqoX!SaLHu<+zCO@`UKLKx>`Xv`F^LVb&y=8~?li~}C?S0_9g&!iz z+3bs=SDtV7%zgb)ozXb$Vb-p2*c^rbU)Ih%KC0^O|93Jh6T+T- zfdE07fD1?<#*GOH5Wu=btgZG}QEZK(k40)-C<>!+B#_$(Dn-7Q^`etmyV@H`UPT2os z!fR8)N+Z8*)hI{iozRWn-cfaN7rN1q!K?OEX1fvvQO zLsw^nb8aba{|DbjKbGauzr_jqvFm)^a`SncblQgyDH{fj6t}+_&nNptK8D6p9X;vQ zhyPQ3wz&EHhIHp@9Ab34Q)ZO7vW-3c-p)U0zvgvsIlQq^WtE*_%GJNDiI;tfe;_YM z$7k8o+&rEpO=A65#Pc{s{k=S>e~FvN4@l$iSiY&t(s&-+{Z5;#xZPTR56_o*4x``I z=hF4+A=U;S8sq zXLC*|dP3!@eu{TK{o4kvDk){8JGL2V>U-q~=lt+LoN4mac`>G(FZ0crGWz}yJ+t@R z(uQfiKFm)9MwS@fQW- z@u}aP0phEE%>8IQzb=_Mc=IpGg<z>gd}&WDvS#2nTUYqXl<3q& z__^7GFI|g_DY~3l?_gXHJsp1)I=+MTO1akh zQqm=JuBm(-QNm>d)}2e+qWd@#I~rg;apYZpL+fmG?0bHEn5;kBO1zcep71vmqtB!N zg*}e{hBLH`ALn-L(D&hwpOPoEW#s46lsU7LEUv4F+Z;v~-$xwD-I{)m{9Ot>Nq!FR z8@_=x%9>+-g!m`ffau)L^UU{>kMW;Kw_~^HY)h@-(0UE|RCd_dpZ8fspJ7+3EsR}| znxuKLbph^#?8RMHd!J6OPkw45XSZ{X{89GyR*=_9vu2_)!rW7Z?3&H^-}>N7qpb?( z+baXf$u&(|(y?=W;bwTxk6$*RI=^Z)#CGto_fKWWpX1mf_;4RfYPgVpGvE7NUVMCD zIC6&-?L~j5@bpo?r#fpvw!3w{z3|q-u`R@>Zq;|Qc5DlM@+#ktF`sg``ETq@AoWpS z_Ts+iX%Ds+vo?Ho(u2?y=dazsc?3HNZ^BO6PaNG_Sl$zR37qyPMbpu1i#hY#jnhmV zo!2#ZXKeYx`^g)<(TcvnC)t)dX6XvrI$F2ZD#C}wS>&1QXPHs2u)re1ym1|}+8ftB zf$l!3zi0L~YtolE<90(!X|!SJlUN76UiHZ%l^@X0Iu~LlI`>p#mU{@x@-t?+p>SnS zd`ZoW=~@`C?P8qP1W)L`#@$bsMptW|{cNkqqwQ+V;XvarU{i>;v!U&i>bkLCcoDSu zW$LPV^{O7s>vS5U%*7_~c%u2&l(D+we^g&uTR9k;TK^Fr1F5d}x^-o2xQj7iVqNd0 zt_y=F>RQEKK-E>Yoa$Q9Rb3DCHSKy%Lc9K&`YhyZB=5guFGY?#QuzbgReqiWMn9(6 z`>#_warOba^Kgo2Ak%(yha+b`CJ*g9HU1vPwJ^?VL|L>o#1*vzfa<$W<0g+@R@~Suf4Nr?}hlOZ`>8*1n64^ zH|1N$bth2dJ@!T}=DrB{iMbAqd$tX|6kP$IM*NnM>7)3^$Nz-f&vNAcjel_MdGN9o z^dAd&pE}6COmd#}fg57L7od$NGvHn5HAq^?-^k76yTfnRk4D}`&X0w!KlU*OV~-x; zt%rTc8NL(vId+uz;^`fnp^i+5#Qd)y4;bfV&dwUMJjYvqbAj&m*@2$M294xkQ=(_G z=Vz_ttjK6=3;8a_2M4doE}Q5N)a)uihe3DpY0Fn48?JJ`V^hxHf8>w26sR?GQ-Bd@ z&7||6oUYtWH<@_i-&2)w24$4EaW8bs80(gC&S}a}x>0Vrvxz4g@eGqL#~uPizU?c| zWqlQQj_-fW-_X9?&)RY0*WPC4yRK)CD*WRliNq3L+Ma>*LP&^zP!M|fw<&U*CvX4dw%Aai%Y zW37*YhqlaJD9ib8?#nj(c0G5Q(4Wq^vpM#^SX<%jWzVty_zLwV?DxR*@jmIC2K-sm zv1?NnBxTH=^+9ZgoqWRXSW3JhtdsC~%*eE>JK7yzA&%(&wm#4VHfaQV>fhx7L-)`+ zdVE{z1Ns{o#uv_AIal>V zwgG96+JNS9k!itRW**!59)ORE_F)}h3DfvV zVa$*HIgzISM40GAVe%~}(sUzXr=sc2gms|lRl2(enqI}dJ<#;QRovMFJ-4s&#pyQ! zO&>-UYcHjd!Nzt-wRdGgyQ!=tf}Zz4r&;KX?(_rn7eAlC=%t7IhlgS-T;}MhTKOyf z^(*vmz0p%9u4uKwUt9SCbYtZ5^Z31t&B>aGyID`e_=Gw9k<8tURC}5^A=a0D6Z%#B zky=~Gx{M0ugr^R5&%$Hf#nh#G?|6?CSli zu5$O_%+egS^zc0c%)Ud} z=fcxw#Z9s0vVQ|Z-^tlDK>LBt4-dKbg7(%|L)4FTBG|Qo;h}nai+_6s_(0v{`nD~_ z;VRxE+*_Dz;P`Mag>lDw>z0ymQ12J%eVlpMUB1$%`M#Xlp~2`OY+C=l_ibE|A)I_$N7e2t=4%I8SJvL7MxG#0L#+&(smmS^K%()|)iw!Y$)12o|XuCT8&mxWX zL)Y@{%qJ9^`YVr)={eVz^rPMM-&1_rLg~d53*+g>pCUbT5&Y{AH~oC#e-%E`Jx16# zMIFm>>MDJA$2>FcaKzToC@ zIcY{y@766Yt!%NP4`AbVRsZg!o8YD^C0%0ui{fRUzW&PN95;{Aq)DuQK|GJs*I#)I za`VV1jpS4WnpNKojOXFdEoW7%znUj|ud#s*pKW^?`}V<(aw9mhwz96bm1g_}$PsvY zpZo|zxHAvmLalrY+UxWJ;}QA2naBU1=%jV2gB%?ZV2)5a;=izSqz^X8x0IUHARom! ztiPU@$vUj}V#}o`8jEzlGB-)Nok~x3C+t*u@-u9QQ|ZaWgvIqFYYZZ@d?m&P{6Ex}_hajo^fdOHq3vw7 z$2U6Cc3oa=<-Qd9z30S+2y``;zWe5x4jl%scIZ&{q5k8)p$|`ANe}J4&WxYd4Q)>i z|LZ04c6C*Z_a|?6bX7%eZROv|UpDU15u8uK9)@T?d}o}guxt>%Ri0~kB6G22avWW2 zd^)TFd6#~YUq|2C7wh?Tc2uyBihB||_ust>>C{=53Bv;F>E z!s2}>`+XJn|1FlSgUw@X9c&(B>tOR38^^JEWak{f-Z8d2{*vAJOWwy9u?Js-&az0f zfhJ<}H1^;g5BjC>hSA}X@Z3mjzQXS4U(VV!cz8fEsSO(=z`6*npR18wOq^9`UUh*C0n|1_0S9mXbB^Mu$&8X>Sjpv?*b#s$OhwqXt1K%}32T|Zb z-Is!WLWh zvrD)$3p#z#imrSVzw~R^AhWoW<=Rkm-$651bi(eVUV-DRVeDyf2ENn2jQiIO{eAcF zX!0xEkKD#ri_L~F!dVZr`E2Z#n~A%_|G8s#Z``6eoyeuM`|m>0YVH8o{~fcwZJo7^ zJ!zC9IXx3!!9zdCx6oo0ZFt}DamaQz_eeg*7?yD?=gVA~e;nV)h$nKGFS<4G4ExdQ zp?_zOTdnrEozH%>BgotcvelejgT9C?bNIChJ*P45YVO{MAVaa2*K2QRKoA+(h(AR( zt;c6E=*8nL1NXBQ1o}1d9=cJx-U|JDcDJ#mPuZsUJbRtsTQA?SqpV`_@U_U0&V0L` z`ksn!*AjLrzWr~)PQ|xR5Oylr`$NM1*L=&I?(VXY@a(6g~E#K){1TMGT_JK|?N!Pr4# zle&KFE2MweuWQCJX1$#BIPy|`*4y=GSK*%?1by{B|CYh*HI#3j`*5$R|HVJ?hvB!n zVbGe(?jQ8bWvRzLTdw`Hb?l!t^CV|vhgpMO`!K%xYW7jJT`s>{%GM9zm$e78!fjU} z3;3>iZesmMuTHCbrtEji(} ztke9Me~J;_1i##oWR3jf)lfsfq<$k=d$6wmo!Q}iz%egmhug zJKT5R_g64}-qxQp3@qC26aF*b3m=~o?lbUvD?WiI4Q}qFhJANWYS=joyKP<@F!<#I<91{y5-?Yj-Z5)X@AO;k*y+ z|HQSj?{6TViM+Rs4b(?CKk&n`J#KI6+5PsNDc$Nnn&7YhWD0U$@`mvl<#D!E4s#Eo z=!XTq`hg9npKtjMGEC#ex=&(7I}6fves;K}CC$tyyiXrCK?9OE_=XH!R5J!L-?X{b zADZ!Nt~KNK6v}}%Lb;q{#oD;Qz9~_Bh3n1x!30~7I66PhOS?H6`NtTsnmb5k6IQrT z=Vx1A8q(L=IHqsFI&-jeN-Jq~FXXH&&c;$1tQiSpA&X=Wux8}wM#iq(Pi15QaT}mr z?1*(bd%_*})>>9(_$l^6YTWw)@i(!~MB{k9hekQJMAccfmA`ey|LX#0bN3{A%arj+ zT6nX@|EyHhnHW{WsPlg)yAM1+F}nX*WGQk~>mX%+e;<3v$R%W!WThv|(zp-FYfHw_ z4V+s|yRl}HI48BM!uTicD=VfiHoYBl(#oHMyzM;pux!E{$ZmN zp=D!pc;jbiR&!CiQ)kq_&$xRJJd;D+rN8#4PDHn-+ON?s`_b#Ny?>?meh%KequZNi zhoUclkERHpZJT%WdlU4(4t$a3&t~J_8whN}C(iqXq1K%H`&l!Ru@!X|w?_lBi9aOQ z*sZ4DrGEl_Yb(p31!I%F>iS}VL+P^)`U0E3nz4^;{~W?P$bH(t@u8l94>d0=o2}jR zn=-E1jZ8Ow6zsKwPw61aNE4rlSE{s*L;D%j-h4+E_zN}H zE8j&6zLEFwaqPjbp|vWlYw&ydtrKgjX?yu^cGDiE)s%@%Z00KRp*h+`{u}vf1Yxie-LizpAY|ObL)^q&md#l=OxnE#FQmFpQ zdxPx*zB{>o4sapQQ~Jgp+Yao139=OUAZPPJ0ahMwISdA&&(c5T;@_W4=kUN#oiVZuTOGK8#vZAY^`{}YS*0WX5z&j zOEUAphUfOv{>Y%fZw7`JYfT#Zp;3B}a(0q73BFtfFIiW{ip2Lp)+DJO}m_Of#|MTlpp879`~HjcRqQkQ`r|xT(XY1h>#S#19?5S^H*@p! z$!s5VmI`j}Fy>OBg&XcKiqFyKlaIR%;JFQq8PSzOg39S5J(lr_liHve;Rs+s5i5ZUeh)0e4B zQ+vmMikw$lIfj3FRren~jh$WR*x9Dv-LoVPP?y7!v(QFep+h4*>8GYOK5QW?P5m!_ z&;<6)-7u8B+e00{OEz*UI)w9CJlZ(lN!yO@7j0}Ityj16sGD`5W8Eso*H-qTZsI}7 z4e^jU@0NbmeHGhd+?nj+wSHivFSb0243j@cenyQCj_mQ_d%^$GqVpD^)AxL=t@TBB z!UHV@%pLjrg?G2~MJ6%FNB+v^-M)@Yi|%pOa803ZQ6uYf>{7kAIC%3eS&yIRLgC+Y z@aA2z9vM1L_;($=-p64_NFV5bo2cvAz!<&&;YR8m85-wZ__cKh|AIYZ>#<2+hk@B;1j&yKX*m|k0X4ZbL7xx@U_xX%Cl;t4r<4q>Ni#}R~e zXvc)M8j{dfi9WT1nuDT!qMC!EeY`m+watFlAF~?Wx*Og4KDu=ey0yc&ZXQ0WTx;1) zv`?fbHr-Yq>p77d3FC85Zs@$2!{R?zZZu{)ydTAuk=&5(lRSyI`2FD3_rC}4wd2{e z;}B<^S6l{1&UfZpH)|a>Yi=##h_0^#)*d<`zX$WZaC@gdD6XwMgfGdX};Ii|0=|D$ttV*Ib>B)zbWgsEQ@X67B~ za}x%9jxhDB!i;SPu1a|VZ8y&VVcwi%qxQ%>Q5s!E*fU;!3HyK(HpSHgj1C?NUB>W7i3TdP_6t}{ zyKAjQCUx>?#|s-xn0P^99_@Hxg$ZGXb`tU&MwsZzq$6+1$A{*?q9Uk<~^nX>5i}Ivlt2+~PUxU`e{F1O6ygDcF z-(mQV%au>8gLyj1tb<8h|FRo<#;l9sf8uLlPU1)1E0$yLxt=|T84j&Tx25t=m7L`- zy1xRrsN~kcJFoX07r*&cAL2Ce{bkl-{9lagwk~n#`%?UHkHLFm-Ll26_i#qM8CwAZ z@c4f+-rK}EbDhU~pY-_8#(VL#NbX!lqU`IeH@&q-YWq{mK&?H>ME2#_t61~;IPF;s zedym)d5=6+*6;?ptnh) z(3VY2pDp(xlXlMX9an$G`;YIw#L480_pWT|&+vmL+S1p2_Stgv&F{&tFZn62&VHC~ ze~&GfoKn2kh}X}J$Ni*5jJG=M^|P!{gVEXF92IWe_L)hiwa1S!=k`2l`;b<9<@ty4 z$frt$tp(p3nph7Weqmo2HeZduA@+!$xeR^#jK3FON?HWpjno@r0)Z^l&mf7#Ao=l%xkN3M;GJw8@t@9*dXuolK< zyo+6I*mbN_AdQxK#Mb0+sN$*^bj6-MafF|eeWxWsmC|_ebV>zA6`$xCW zbsq0l-aD)<{ZmqX`(orNZK$=SroTI^EoCnHd288cud_xT8quDDt*<$2R1SdGf5>i# z@D87>*ZR_fcle^)=zpy*m0T6B+_4|F$v?W9xS{)v{b1@T-&h1&``gU@@Lwj*O5zMz z)GZ-= z*c@HZ@ehq`#jkgoxi4(%ZQl8g5p3&-Yzo=e()H*eLxaE}(3xy((VAyJn6Ouw|0k@& z7%PEx2X>}ibg4-POd(ADqdZ0Po?bJwjqdUj9;fY2y34a2yOf^=eVP9~o&3zP-yh4^ zjJcw#(&%gGhOybUbbR;i@{Le8Xw=vp&DfISy?(B|C;-ZLpgxPp|Fy%FwVPAJ`OLeR zK1A*Yd`nksVC{;rBNn*)3j8Da#s3LC-uf=Ig??>b)t`0#jJK&b^APpA>mcIU4$SwO z`#Wnip5$IDWWL&MuIQclr>5&mk1t~0DIeaM$s82(F~?sU^oz?{20eS(>OpHSy9WGv zXUr3)XQgR+-gkTC>1(i6C{MDD@(h0k&I-T!OUmeq|C@S_zl(D*G$#;XeT&Y8(;S+7 z2s?EKC-e}J+yXwvzp9>+!6mfWnQlKPj_V&HP6yhd?LGQfOjrlG2Tgdi@omDqc8rx_ ze@Q;v%rnBr{im+Y^_cdEq}wfzxH69RGrs4=*ecusR*ale{u6XYQi6=DW6k_R_9YF$ z-!qi+F9PVPuGY*O85cOm$hZ>tGcMzLopX|md;D*XUXzS_ly@s}{rtZnP;mLBBSUPIXp#&TBCAWa>b^L5*9%9Yd!mp763q zOkD8{aUxfhH3YfirV71Z#Q#h{wwpdad_s>^q*%qWaf}VHnE6c239}9(GTYx^^ttv5 zyvlt-+AF{s?tEYB?6fhDu}>g#q<1!FD>SbCLXk_H{nrKa(^kf|vhM(UAjy}^{>~F? z{CCeTYq)fdvlj6({#|~q$v~~yDh0lSkN@xvf5U;d{S61-#U6Q&H4FHEvj=dN^H%;V zfiGU+-#hRO?Phoe_yS>xGF5BM49@_cCCuZQ2y%yha^wzniWx7VUpva3Y{n}&jxVc5 zvM0;_3AAE-;owuLcN#t`wGZptr;npOS?it>(>izdI@7miy}IOt*^9z?FZ!pnrN}(~ zr%~fb;8NuA=!StU;Y{`+-(m~rSg)t1O}ub%O? zEOhD_WSzX)evn(w%ets%#%b!=FR`AYKgQ%n@40)K)TSrK=gbw-p1QMo5AAv8AZyvB zl=q_o=&jSbqM<_^-*sRpVN>8q=96}wY39w>rMFHoK6*n7oSmfle~7H!1HJtmxzL#w z-Y4u-wD2Bb9caO;Kj#7LDhnDOI#vCjrT%$cwAY)bDR0Xu%gd*{jMJ8P?O=ybpXg-2 zGml#Q#_zG;nKQVRyT4AO+tyiE3^Tgz{^#PlZQa~T_E)=S2Bi7uL+tl|{WZ4S=pfqu zUEVwEwu2q*_fC7m((KjitYv+%*`MTmDcSGuzUt_TUs@s0wr_GE^rTuC;x8K`I z^#@MdeyR2h+UD2T1KQW3_N#H*Z#{T(H#D|E9q$X=_S=tb@J+`SSQpqnQfKm&v|Y7%Nkyz{u5`OWA+?Q-h_R`ToHGpG@yHmifEHfeE*nw zEEJygSrPC-?K|e&KlEz@=b<(*r#v7z*=o@EcRc-o4Yf{Vt|`WMg)5XZuYm>8$Jv6ef`nV(&Q-Ojm@&^PPB(0Q)k@Kbb@bW165x|*wQe~WcL)YGJ4 z-dXZtH)+%#v_rgp`KlkbM`GDk)X%k<&HnIDpLO{%Ku6GU&63Z^pGDVSLry%@MN!q|ek1YI4u$^8%cQ_WgP+2Hb5;GaOJu1jO= za>yOKMEW`Z@jPiH%UF{imvdFJpU{2JxM2Y=44r4ffIlZp{zU8}vqz~)bpZbac-hNY zgc+X}^xDA(`X_v#jANvGG^X)O(DUD_# z@0U=H=`-qD;r8=+}0>ExVaA4K0wz z-hSBk^h4_?{O-^J>(V`Y(%Cmj+7{++b}@J3)m!=ehPq!(KAtQnr~N~Rj(6a->e028 z(x0zzhJ^fv>IeBf7D6ASGjlcYO3?_ekUAE&q&{S7QFZBhFOnR>3ocXEM(uo^qZto{2n? zMIj#6L#AEAb1~0FJQwm@z*EAbvq5CrN=A<3DdHK+qq8Z;@MsT_&WAaN=WL#{ct-Jz zgc{+xe4cVw`BDSrs}>gbM% zpfv{_F(ZF8=LB=sm&!;z&V4kb-%EPI z)N$B1#Q#U9bX!Pg=^XN71H+4;C)JhjPppc*;ZnYT`lutb{z5vh?p~SeNo(r#hEu2C zbjte~FTBACf2C9SYA1YKpmdBmOLt`8`<1=EzxGaSo@u?tgf`Z!eYi4Zb^V>JC!Qud zsMnxS$+WkuD+5EhH}~G0fsBRcgp$iIunI%>&7`g2U2`As@p-S-ANN~f^B?Q_hG$VO z^s%oAyXyG7&2_rxS8d{!%bjgbew6=b%Ky~v{rabN;rpMO^<>Fi=WY0#`&dT)__O8t z-B_cHE$8JO+hT1$a5?=tFRA|cVk@k6Gj*x$7YHvdN2Q-yhL^CV}nm^$_DAI??3pcmAa)(w-R3wf`A27EV6j80R1Eo-3nZ;GQc z<*K0PA#Qsc++7Z?%RVR{5KbZ=<*PJaUZ(sH2855UJvq#@FK62yI8_+u@PF7v*psrs zmW>F%-$hsxK56TwG2!hJ$e6q z%6!ge4KnGdgOktS(!#HJ`6jlB+om(+n};XYcTv_U%GhR&E(vo6jC{aO8Q)G1ukIp! zLS45729-tg2Uz>=clpQb8&k%WXB)okRmd4UVa~Yr1!hklBf7KZl(T2KYQRG|z3|5e z=KHQnS-7zB`s4X?0%K}&IJ<9l(y^G~#}4^HH=WvdoAr(J!}Gj$klZ;xBYgMi>ul(+ zf-{$5TavaPyQ4dIIQQT#=bqf@9Dom!;k#b&T@rlP(>};u+y__-*UtUlviFX0Klf4g z{~qDKZm)l@ODv!My@v0FYD3d6)5vowc~?-z6v~>+ebJNPwIsu9UR(a1^Zz&UJaLI{ zP8Dlc>w5IKGT+x@jGy=Y$7AbXNMXG*@B@GP$g1!Ber1d^IehGkYINV9a@(tyKd)^T z@A>}RwgLll+FFjrX3YLh@0HqLRn2>{KX>Z_e04$I-vA%*=WNw^iFHR}Gg{AO?UFxj zBFqHnV1qJyiL= zytW(UC(Y4&ZktcZ6{dH>W4)~CINoCqF`t{`yVuO4-@oXhru*;bS>aFJ&FE0^ESsX8B`g$#Puh}&E*VO zCtjW#Z!qyRF5lH+CFGf7x1=(*A-{ocyforPalg|q)(1J~^3cyX z+oPboKJwqJH_9h}+Q8Up=OvYB?c}N_$OAd7vPG}$<3rKskh6JKR*mcy#d(qXAM{fu z<-VgbDf>RcIBWgSQQ@iIlk`{n1)=EANSjMq*)_U5O7#n|_M)Bj-YW?^hHvj(o*d$= z03stk3268IdVGLB%;)Un)`B0e*juo2MO(p&70L60?IU?o?g_RtCm5dR zPp$be&$a%H8u)JCBKS^qVGZ}6{RX~YP-@tVH zVL{sOA@j|;QbGFtyTA=QCf09!x76&d@Y*7kwwOs1 z^PO#fRgijq1^5T@h?V)aw_oo=mf?rF+>e~>hMnX?PWq6M#zxs@Wo3mki68Q86y(K% z9M&y6_GfMKbGMb?_ZZJy>$<|H@Sm}MANz#5Cb#`edozUJJS?nzs40Ht77J5WhJ2eY zo<_Wfp!J1G3ollG1-n};eU>dyPF>2V(?sMeHcXi>E>}+#c2ZkZXN7OTe=v;vcCWFP z-3a|^J#g}KXO-N59vxcMJuJV;ON7yO>t&Dmpo@vbc@ta+JIu?|qTa;S{lD9Wo}Uw5 zN4RL_$3XdPyNq{FcDS4{#e0=_o4Dh^`1<%(InY<7$p_n_%k&#_!|$_~qRX)MobY>u zJ&r#qvfua1s;eLWwC+2ND+9;NcO=uk$!{gEtnY4Z$fVs5|7=(|0!@kTpHaP$Ig(RD zDc{T;QlAIF=UeG*KKAjdzF+4%v0S6~uP$euh^2dew+)R94&OnXuKIh`C$SlY#3^L` z!w}j{V@#9g_xa%)Nn>HJeqJA%_P)j~Yhhk^bwWF6|CXWkuM7&8y5|tyPQE`*h|{6J zf`h~5gzLXX1KD%1ZsxV+(doP|xVBsLAEb+w`LP*#qQ?W!TQYRl3;Iie7ka}B{_tUJ zhIVX&W7r5BA{e(H#3MRezX@1l%1E=98klZB2)s_d2-d{i3-8FsQw>fuM|;-lzAwq= z!>Lv@hdNyon;5;D?ne;&wCQ*Li+iB6nQBB%1n53 z7`D>HnfPhQgK-xBoM~5)SD5mpM=c92Wck(#Z{!4+;K-G=hINFw@>U~J{0x;t0qpm{fdEU_R9vQ+6@L~*e?RT_Fm7s=tusc+gCVk zzY(0~l`Z5R|FSM{5PjI)dM?Bry3!v;9_{1|-fT0E?$|}p%v^Xp37Ir;^$^-6&)$jc zaM?@L?Rcz6H1yKVrS+=cPssnb^lyN5cdgiZL45jwZlQ(>KQO?a62bnAdj!*fJ%L%k z6kr~(A8-gT6*wH2X*JBlj(X%}<}RYKBGHBHDA`%6$2`hYx%V2FVc%_Fx_y^{Y4%(L zQ|&nh=HS=46uC4V`7{kVH5GYP!MI}z`qA%<@yXZGwHe_F$(HtkvpU4nZ0Di9X8V%E4>RT~Wj~zS65mQqeyPvcOs1`A*OAn}7jdOW zTWQx`^&N&VhZAHZgek zJecbC=hBJt(QiOT*Un`>*i3BL70_eVk+SH_BYt>4V4p)<=sSp9FtU7dnXy&3(Qh+( zUohDpJ%c#tzS_h0jSL^U&)UA@Oxd8R_Av09cz^FqUo?;R#>*x})A`r8F6$nBZQ3Lg zM`MAXlb5lnldKY@E9ARz#iZyPFHJT!%`NbX+Bq2YN3$uX#6L1TVP*H|km-}mdK~|y zKdC$mIUre&9kDI(EL7P5*sl%2et{b6fhojKwd<+Nc+SF5S-*y_^V2vB7G3Pg*$nu# zPHW@mB!!vp+?VECwnA}NQ7_4)PwO2W@F(~CBfe!vw*v1YZ|u1GMff>}uD%fZ&#)JQ z*F3oS2g<+cl_&k#xjgcSnmqn?LHHZQe=;f5ptH5h;PE={buj<)wJFAqWnKaK zmuhbzo_wRQLJ^l}Psy4ts>0i>wFAZLLUcBV@Z9jC4k2>NfoQFM*EI)vK;ra{LSzjs$7x6zc zS*w7YDhVxNdG8J7Ru3v=y;byj0Rc(ZFa93wrpA+~=R|2~oapZQ2=zle)_=AC2_5&-SN%kLH zWo+0~dl_;k*UG7ZuGgFKGv7H@cFkL~k>-EoOPCRhZ9vEWc>{E)ydEUZb-Z7|lV@et z9Alh9UvdB9&~V@j=htMGa;MOuiP3ECS^DdD=I@a1TxZJ9w%54zs5dave%gh9hE8*< z#&frOrm@Flr}d#6$-k$F^CsarCR~1_3|rydO?bBbBf?)JJkNwX_MF0xgOhDkW6H>~ zm%3#=2pmtJYL3XG%Q&rnG9o+_dwUq+rNm=h4l)wIi%FYi-(z5^{VfACx|CDL?)a^l zr-&`?#(1SS{n!T?))!vs2ha3JUJO9zb_*l>Y|$3uuyuv3ZPHl0-k;N+f3a^)fHQyP z`<$9*T{!_8cQI#n97a~OJ~}yCh^?+MO-rg(zdIM70QW6t(f(PhX#dNDHR;@Gr8-|h z9k1qo)h5y;F$R-8`wiNR{W0sM-x!B%o7gli?2L)h?J2~U&WMje0t+uzK4uI;`g+rz z>GlQ03-rsVu_!0q*F8Gcv_-l-2ABmty-eq}rQ4(V9@nY-D^&i#gz{&O56?y)D*g!K zUvZl9m%8QW5YM8F^s*k&G~ShVAdoirBx}{6aQeJ&K}YF!e{h}4AFcBHCY0YaKCCgO z;`b!}=+l&ci(CGYr^$md(&vqDKft@vehPH{-SMOC<2&8^cgFf^`vH_a1bq;1`(2GUl(^<+qBrv zlf>g&djMbZVSiG%9s4f2*a|nLEWB9rrLt@D=8bC)G3K$*<=0LKMIWV%0CFG*eu3ZG zw}kVTyJd%tY(j6q_t-P(Z9~|Dbl-hH@Of5VjaPQ)SZsrIcIPq|;X6NxFWZBLFB5f1oM-!=71vKQ z)JE`Bo&VbQM81-by|j-tLmU9&^6 ztphb9$x(2h6mlONYs}%VP&5M>Cp~+g>4yyaF1J7C8klC!F)-D>!-XqEr#;Kp z(%x&EMs2#-GQ5>$-%9vn3VU#(6PE9VrQ5Rzd)TBab;44;unhYvggro5HSJetZTd72 zuw;|h1{U=TTlnl~19;c6M9PWK*QwY& zN`HyI=~LN3qQ49DP2Z=R?+kl9-{%U)*fPV7Gm3AeEi^FI9swM3iaxUT$7VeI-`JY| zB*rR!`luUy)tx@;f!yqgo#pFNZnolso;n;^In)~ChfXC!8?nugATy^vHaU6=x@teV z@&LN>5OVb}^3#(SQN}AqZlbgFPf65o9Cv_+NSDNXtax{rco4c5??(OEBOTdU9KHyi zR+@v5Pr&-+5QNMev1prtO`~^&Fo!g?5@udrhL<%AxHtY%^!B>m+{a zeRjCiwdo!tU+AUFd}lJx6(l@yy@PDr2Bp#31MWb-kN1Vx4{6wI)ziw%8u|*toclCW z?K;A-ug2d`m}v4@+D!Je-d7VQ+VIvT$j&;cu3tJQJk+i0SBUS75mN2PyGR==)4Zae zfA2;cbf+zP&?eB$yZ}DJBz(z^kMN-DBb4ogjD#*6+kLT{=OP2s?F9yA*z`7@hox7$RQ_`RA% zTS<;@>l$zqDX)M>VZ+IDqHvU5z$08axQm6O9tAwY-30D@;iyvqk8n%CjTMgi74QhR z8r(U;QO^P%;a&$<2u|;gzdFm_1#Wm8hs_3Ucf&5r2Ui{QhXcnecBq}SM$h;v;Nc3* z^M|(}yB1#Ditn&_mc@P~bQk%Cjr_>9bNK&x(3`QjT|e1^q!WI!1#v%FuD#w()64W@ zt{nhgNWI*7GH8%`nK0)33{10QKi7Y-KF7dZXtM$ungT6NhNdP#Tjl5sc&p6et=JaX zd40h5CA=T;`V$`;Z{9%Wvb!0-^!txNi_l!lDE_I~Vn2ruKQ;3j#i3}!xuNLGBSO&) z%oDDsJvY+!JKZ|G2Rt;zqKz}`fAD^|Oz-#tcyFJe_bmIbydStw?`if9y(3#BOSUNv z?WVT-1Mi2ig=G)F$-C#1{T=VJaS!6>x(K^4l}GWT6Jgm2Qp5!@7eXCNP z^|>hnI4ddBd0*4tnsJMt`>@p{{V+azVG2E*N=W2P8&Gm1I5G73on&} zTjIr=!aOzobQ8EcJY1RPt}_ge-0I=@zZQ6GHSxaY;rLhimtO}r)5FmZE%fs)aM#6g z&fK+l%3};r<0{di_)BzkmI=$WM;e%82Mx@&&lIFxh6~bW!vty1A%e8^U_tsW&%ks$ z$3Ts@D&egw;jt^=waekT%izT?u%^SY-$c8jS<&xK{4{#k9Ng#eXv-7b>os>Iy&yY5 zJSv@_Ihz-tkzdi)FGG_rL5~geB^^xt((J>){f75a?E}0YG`yN_@8iAQ@M@O*5${pM ztJ%~mhx+9*AC(9Hv!{eG@#Jm;>3;)r?VSc@*zWP8dz!~{El&xL#ZxuVEj;9ad7`kvl%@RxV*aV7NAu))+Z%YGghxk_<*jA_OfsQL!z|Fv%T8lZ4mcj?t% ze)XT5@LYS9fjRb524>q&2tpG-5rigI8hnQRBLmazh~g6yfw2eKxm4*udSq-Z_-c%uEHbgkP@77_VTlR7n3+vdUhmqToyZOI% ze54B~Q*A9-QDbzBT8oW)Bb$1_hWu!^7G&!Z!ppq6NH+1WtHH^RcgjT$LD#Q?D~aQv zL)vB+xS}|Y^*8WEd=mj}!U+Kr^{kul=H_CY$vD%@&1KrsJxcc>>CUGttyR|d2mfI{54icfOFHq7^7tF?hbdcp zbnbsB*W@ddT+`S`HhfodZ43FUoXtSbrua4Q2aWtnw_oPH-N-o3|K~kwWZcPe?K#3E z*Pb;n$6jk-uD!;<47(nve;SJ%;}W;^2d|q_e-OF#S68Ne3p$oe`#JH%XO=6|zTx7Z z0Ds^VGOap6rd21%wCV(zR-GWzsuN_|LQ}si`#V5SroHCMH2EIxal^j_RQ;vrRCmd< zIVLREzQe#A`*s7f?b(9x<1K>p+sy`_VSmj)#`y+j+BX=OYF}^Q|6ZQeALNMk%Cuw;C)>)E zY2D(u6Ee-Q?;R|bwBUnINsj+p#!*mSJpFuEYq{^QU7Gx>G1o9-3TCCaZ~{fGHn?dG$Fbe{aG{TEL^r)ipT9ufFQ-F+4PV6w6aqVB6G@V*L=e7@9Bo|?+O#oXA4i?9VE>v z{=JEQT>?H;cOlo9r6d^rl|IyY-T7vtM}C3%pmaa_}Dt-$lJ{BJ6#_B9xO@uO;A{JbXgER)d%A zj?dJ{j$63Tu@Kp@7n+{dy;se3$dikZANq%8c;UM^a+v)Z%p0bwR)yaNJ^w)3?hHE>e+ZAq_pOQv-{218LUz1}mH(}ZK z;|6Bgj~SS0hYifI9}%RTzGuF%&kanqml`O4>g>K2IL)$?Z|!lG(lfDE8Fn$)XMP z&40P|xDDvpQMd4Z$c!5^?5~^fO#3DSv+NrU%(iD5m}7qx$Q)1Qj=`*n<5|vg56_o* zF5vOx(V9WljK`5juX3KrblyFAv@_3|@sR6}so~v|M?-n95|45YY%8=Kgiot^Z$mFu z!Jk3igYbdw6sVF+0tSIq`j$-6H!y@r2n#&<^Bx3N0IQ&_ z05C$?IxoLYGEH);0J#Lt$S>qdMDhz>iev&w6OsIaZZ$58%Pr0hlH7Wc^gu4mcN5av!i+O}w z0Nm@)Be+e%QNLMU9l^<_zn?nJGV+r;f_p(Y z>Ntx>WlO%uroW&1&GPDqjCe*k>Nm@)Be}q7$g+`Xq9}^te zvy%8KqxnK?IqD_Z^8*iuozz0TmVgV#amX_2wHn;Rahz-OfqM{~d@CkT{4FiiZ5Q}O z;3EYgv)84WIUm{V@|BIH-zC#tMy73WWZF98Cp$@|<=LMYp3JlFebQR?ZAYf9^ZaCq zGVO`DpDfQ#HFiaweG6$+=FPxyl&7(S_*!dYZ!}@>yn$KvR}IXxzamJxTq{U>eaYa{ z>@OO~nF|J{f4)rfYwdIo{AE4yn+5QnCF4h9uhl%(PjBgZUL;Yry-2@Y#Ry*h{l{_vDk0_oc4Av?l+Ad@AL=0vZqGW8*;A zjmSF9Ggk{zp9(=}BOnMpG;-Hi1+-Z$NS{{-(%%6==&upmtzs16g69$r)VyKESR-TB zR}@1V$h->LyGoGu4hohLj;yPgLb%{`M+R10?#RH3s|ZKdRa`^3;B}4+thnBhfu5|Y zqppz@WS0IR!Pa`H!I4>ewI>MKCYg0eG(&uiNBr*iYbUsgw1?((ju9qW8V>zz6`uAf z;1RAI+;4@WoB|%6WYbS1uL3u})!<$fj`l3zQM}i|{ZcsEwtz>t zUEtOVM>`kr2q&2(`+g#AUce)qWY#L-X#WBp;mX1NRJc*#c!awN+>eB#UkZ4HTLNyG zaAU#o2)7!XZ2F1xRRNE1uY;3KKaqYb;1O;YxCexz4-0sN>xK+iC|nsh9^r6ZWHb*a6H1T26wY?*MQ>@?safC3U?hi9^rO@ z`+ve+4~|E;Zp^!X30%Yv%_1k`c3`F>JHHI>0wX)+|K~h8#)-qVK7=$%Z`QH*M5cn9 z46YrVgGYv@AXmYMlz+cO8R*Oxdr4E8$U4?u`MZUWFb8~y@#OB*ZuKqr zf15{XkAmY9TMwPR2u*H)ej9S}`=>Itr0tFyzaxHo;33*ay5$h|#$mqqBfllXkKlJO zcjBS5bhrB_gtZqdZkGKa@5jdKU4H*GyOnRH*<&F7e*@`L1Mvj_HD=CW&g~HP*t^_K zaR9zJN||Z+?T@l&9{4r* zD+7#8<# z76awWx`IBqoW8h>KKTNDb18i`ojFuz-sS`LN(Ow)1#_-ZUC%&qRiJ-yU2;}&Rj(n% z!IWTeEAvr>tiv67MyURJ_SO~p9$JAsFVft3Bj@WqgzY-=NNoLd?mcW{4gb_qE81uU z#y^DpdTZ3dkBZ~_kADa|)^2z3v)jG+S>qqVp51%c!LK>&;Df#);~&DV9qHn+txf*z zS50CJkZOMbtu*!wj<3entwN{tq7EaeTQYU<;%I-wQtFs|Ft&aq@XZ4;bI)Uadu+oh zd>~7yXCXL+?cMK$wH}Ub(0-|<)O9L2h4pg73JH6hFwunQL-lx6ajAdnJ}0gXEF|C7 zgRu<{DU5$ecEjHM#0e`TY_YN_JfL*ow(gU##`RUf3Pkj`d@zIM(^~P3* zhm4N_-Y~X$KVZJ2?~MNI$v1K*;%`sf;ow)v(Ouv^isP_5`LAx2-x|lk$Mnl^aJ#_mr%g0AF}_83v>g1uz(=UJ zXug6oAS1m4#npZL7YF-i6*mqTQmpx=jp(vEUvT_k>K4JTlSKdTCtu}NKpl4TUj|Pf zd$=ayMolz+gjVF2=ZBOZ;VSe;G5TYp{EFmb+L!j2OB(t4UYu1@|2+0@h0bBnJaPWQ zi`&cGF+J<9yt2m=@q?z?Vfs+zKZnn9AA40LLt6hC<6cVhf6;2*4Nv?tW_UvU zQca!f*efgit?Z!{uT(?lRmd?Dm$lr&nLMEBI-i5Tm3^3TJoFslUlgB(v?uV;a-=1; zLG|!t=2q&33_Pg~m+|igpU{T#A9#IJwZGr^0Cdo~?Z2yZ!Y8C#l#s4*U*`A#w9+}< zJxV8hLOQj-S6=Xw{P6+ksdKtpl}`ACbYDwISN+k5@yPy@(tSnggilC!O+q@4&LSV2 zTYM*S03CYTal$cIcWRz|SElr)ADwq%{nSRvdXfJdtTs z<4bv0yr~A4HQvZ;^prvSeHq6)`p-spNx#lM=)|u|a&%q(^Op?7f}IiMX&zFffVF-%8=rhkiFHtgTNwAbY7d zx+NOqU48TB9;a{Gz*iv)D)u<|UM{|heyV%lmB;TpaZMh`-pFpZe|IO~q0yj=-@7XT zkL<1cIJRLJb*YfNC12G~`l^KgH0^b$-*}DZ6T{AP!vcpg$5%)`CZrqXhE*KQA73FE zm=HGH4XZvdVtj?W`(qncN*;dB+CFbAu?v52 z3wOhM=P}fk(Qad*KyXI!CmL>sBk%hXlti1x4_M{6BKN5cfvKl{SNH_(%U#$jbR@!g7R>Fg{|C=To|QbU{O43b z{xcaEL2tK?fHnp=GP#ZM#<}UCh8Xg>hWE3vck0ey-?r9i^L>n7IX)iM{&wkXvEW6ggT>JY5=GYG#m~B5KNL{N1sq1$QKHXj- zcp-Bf=KJ&4mp#S*1+bY4zde7)L)-(l7x_FLK3fBv24CF@3=Flx+Hc@vkFMH+F;`KO zzqi)bSAMpMdnFi$v_4;6uXZ`RBvknE%RLG|oM1H=I~tv8>}bhD(n|iF%8m}sS=b;y zL1Ywk!ap3KjKk|4Ie88LU*L|(n#s?zr!Vtx(4%KhPxWxLkL+a0$s!L&+X;6Q zxN|)m^e5X{@^NGwhmG9Rk&kD@am=$r>#q}Uh>L3YsV-z8e#$Jspim*!+ zZq`ZAJ_lH5#C|OMr>ZA;w=xbHPujZcoHdwnxy~BQLsxZNgZX7-xcX6NBmIhgRKEXM zfe(Rx3rQAzj$dOpYZx_8u6d~zXd=;n+j)-sCHkY0yfsJuTi_8RgL3RG2A5%PHjuLb z49v1$F)-VH87LjI13%|_p5;9E@O+u4+OyTTD|!XC+V#wf&*r__v(iKyyow=tf z&H2!WL0}bh6Vco?^Wb&BD(YCJZ{~u7`j%b5op725uLD*=Csq1p9z3XTXe5%xH)Ewb zLF8NnIy3xK0HhtwoKikeb4rRYek0%3Rla)pWDj!omgLMq#?-@IURuIGA9ZbxTBCcl z#{Gm}b{;tIJcMTG!sDmA!2gVYI|{wdf={Fe%W1poh36k<@xQ{|1nyel_|IAVvv5nm zT`e5{I*a-Vw;J3P!tuYe_;=x62RB_f{&^PvBit@ODO*)aHEAwD1Qk!+4NrdTGwjo3jPf65!Sy*Moo3+v^5UBmTzZ1n|3T_ zoXx+yPW)`*YyGbAWAZP%z@^1;$bejDPP<fBlj(*Hdf!oQ!oN=v%WUnz63xahK5vx%Ov2itjh5?P{(kale71pK|Sw zNTWPH0LrFILXU}Wq_g*!Fzz}skpD6eoi9jxyl1}C?0*`VYQJls#w4frp?%)GwdBJw z;DX7{hxuX|1&Vtt_Ny-{sZ_{`xVlGTGu=de`hLBlyf8A;dz;7 zIZr$75^&{<_7nt=FB%g*NHbNjzcH&Z@a*S;y83NecTNmyD*M(cx*Vh@!*=F1&4Q>dD2#J;+|CJyQ{o}fjQma+CM?%ZGcd;DPpaHk;AGqE)>+dkW+$NyeAm3!iohzU|)lxce~A(3gEb&UsY3a@oI# zuS92A9!>C*yhPhI(2m;c^ZpM-GimlN16ku^AoJG-X4?NU5L>{&Z2KJpbL_tX^JtI9 z*o*c}o~wDr^JMcJVNV+R(eTJ@_PwGT&3g*(x48VWhJ9(?KAq{juMuB}Pw4kHXx!|< z!TvORa8iKKmH1>f{Ly(IPMkNeTctm*{-whk8>pkl8=H+h6>pr_hx1Gv2Om)GaQ+<-qa!%ONn^TZ^g~OTbVQz!?}B&npPlLZRO`DVu?JEq-gPt#g7QX;Z zUJ7kaheoF{-!hebGVUCi|MY%7oiTwSgKp4ytIkbMFwWr=NXu3 zk2WyNJ_o3E0jbdB0rot;#q$Esa-Ms5Ji8HDJfqr^>%1@Z)BTARv*&ZZs=HUr$Zcr3f<0rA;v{3^J@cGH zqrIu0XWwNRTIp!t`Qtcfl)f3xzkFo+(6v>Nx#i$m<2Y#4@6hP3I1U$z6c$W{1J^0t-xtxwFs=GvDTxs_|bz1*=)66IYF+#g z<{9hgGtFNexjDJ8V!?NIL<+jsE57lUdr1Dfo6^E#NzZu>`{X-` z%(WUKcU!N>R{A=9k#FU+1xBw}fgQ6i@W2x*l85N*?{52Y#w<1GMs;)P^Ct3eWN4QC z4be`4BYQN?=*F1yIzwZUjk^EIk)f00IMJ-L-?uc5L$10q^rASn|9V$uLe%Z#}-8pmq=K=0k3`HkkYnJPr`RverOMp5vW8uJxBiak3wOzB2 z>C)}tbq+o4`+@f7a^4v9B>Zi(M}9YJ-)7|6+t@~jUmF>2{+G^hO0y3_BZmsD+P@~9 zMSG{wzwA}+>-71*2y1RGkG32sk7_NM!iNJJg?}IXXHFjF;I;_27hICgg-J8x&0h=m z5x5=>PW>*MzWGydy&T+XaI)!}4}t6J;9dtOo4)x*zp2w=zIXAw1TK;kD$zc{KnPtn z;0b8hnS&L5Hr|M@igTO-$4hq{WUT#UU(wb57exk6a^xK2?&ihNR)J_M&CZ}*4*tt( zkS+03+H5~|#Ubh;yL$!Shpr~q&U6K+(VtTu0-dL9z=$nYHsXSQjtu} z)WA&pMgz0#nLv#huR?|u@nrBEMqmDoC+OOCvfF~L&XoQvl+I)i!Xmx9zLrtE-@^sKa!>M~w949&C_&@UFIOGuZxQTcLT_x1F?xDaZbfksUd9PyXq?PUDh9{ds+S9Vh!!-86@x z1^wqi;02T=I#WMMu76^}GVPBH%&G_6(kJJmQIYw2{_Szef0% zxYMbaCqs8Tp<_KdtYN(E(cvh)Lx*NB4C9R;v?sZ6yWY{oIjlGOZ}Pg{-(yS*^lRq( zph?kSFq1hycMnRrOLwb@>(Sjhqdz**-AaQ?x36^T`9tD2BV(JOZ}pQO9VuE|qSDdSdRcobY?&c}o{M=V5#^j)MlFJJIDGE>8Qxe#ZCV@3YsHGu5;=JTf9wf1%-{ zT*F85-!F9cwvMLUO=j*-ds?Nhojt9q9ziCNcW&~#C(R9N{3_Z$iv4HkdW50tZDu^e zc~0_UHn$)*jV#Qw8)*aS)hj8>)ydnu@pd9zCyuw<+%%=65j_W?=Sz6^X!=6lHRdic zko7qRrrYBTOtZ%tm}-wPP&9q|dlyu{LgN0r_1ogsFPk(@o$~Aq-v4gq=l=Kn(Zkd` z&F*F(=l>X(VOs{W|96@GKg&J_RQbbUw_G1pe46 z+IQu&#^WA;_!9VItz@~&AO9jR@khaF_~YBZaOCnoi0ko3#FbHrbK*M;F5Q00t>-r4 zcf}tKiU)tZN_bcN@mxIb6a4W^9Cw00o_28_f1C?{Jjq`0RjWcJ;*r))dnBFx04I3j zJ>qwsBcBG({U?0T!=;%=_?Lf|&I;W$3rOSf!MAz$_~2gCE}8b-26EK13B}{ zK+gO!@ZaNun~D4H*3akG?;6s0d@zG|!v~*Zo9Nsljpwy)$Jw{!XFmDFx*e_WFE{m0 zvnLvuZifuaurDz%)4m9(a_8cooxvktt=6X=Mo$DhUlV7)=$!QWqX6Wlf2E`2w~C@U9|C? zlw%f;;!7{=6OKI0+zhyD!F?zkJQv>sy9~zkNEC!0khjP|67tO6Zz*&`V@jT6w zr18`HAEWUtoPm_2@tJz}Xgo{r9*sW?ABe{5;1kh!xgdQB?HU?y6{PR#1fh{~LG1LX zc!05rc;m}xKK@IW<}X)0Jet2wbV7QmH2-yQ$J6|1;f|yE;ldq9^FxF?j^@u0?l_ts zEFAfY=9RzbeV}mEX%>%gqV1D~qkgk^lDr}M&IkWxydj$JO}tdz7zZxUjdOY9T5x`F z$MeRo!TmG5af^|Gti$=ySLcm~ynj0QN9BwGkLQh@CXBsr26AqffjQ3K4dl*01OMOh zhWxo2n;tL!U-a^R!N6Q+qk(zOZw<_Mo&(CqHn$A<&ohiCkEgTr7;k7_*8t@IO38oT zlf3aTd%2RlF-h-9`G1PL4xp7iTy@Z3ogj2qE-3lW-mN<5t5p!XsS|{T$_1gDDC_O( z1`#eem~g?<2^Tz@aQ1K24JBM~1mS|C2^Xv;Tzgq-2^YMGaKTB03r;4Sbq#e>2^YMK zaKS4H7o1Kw`&Yk=H~V|Md5hZ7po5Z_WmH4RWHEM`5SZ*3*QeU21uR8wYN( zaI{e^k8sz5nnwhGOWF6=(AKp)!ifjZ5RP`QHBiv4KCkaRY)$$0Jj=v!&TrD^r;j}IwUAT+D@d&4N0SWl> znn~bzgu51;Z1-y>gX0nIE^r5gn+lFcxJST!EZk+_c!YZqT!(O1g5wcxC%6xUn+}er zTeh+PsttTnwk`42L>#a8wWsSX;(bcCfqNZX9693RJ^2QHi_*-7M$LL9?Q3N{EMwk< z)cvc=xI$-?(Se1|3$}X?3BJ!0>i_C)p9u3)-ao(MALY9i_-TJG`rm}1{|!X{8;Jfl z5dCkU{J3Ad=K8Dp-(=(JuE6+lbFRs?>Gyj1HyN1g+-)FxxDCvA?liE#X#~m^_!j=Q zVLaJ9`|;a0Ww3tnWUIajzuNct{x;8S>|gnUx8TQ}jr`O59Nurh{++~n#|G9r?g(1b z=3pN*5Z285jmW_J@P(}8+%!MmXZS<$SGX_yzF;VRA5b#1hCNv-!z=W~eXRe!e@H02 zf&EYS`R-qFCh>2|u-0A7^EhjKKjhBwCpV2{jmLn>vj$oA!+k>WU)EUhC~5rxU-cf& zP*Pe?5ciL)Q4HYc>yt2Z{O{p+ zL%%CaNWTkw3IFr0bb7HzO6ArVyujoVaK(ve59KkYxg!h>iRo%o~Ga1tJGAbnoZXot^Gp zO6J<)NuGO*^yG(MV_>cmHZae5)WCe_zYHvJRvTF8JPcHyZ-utj@o25FzRmnxmb(_Y z3;KGS=hr;i1IgZ@s&?pXytnpv3qJkvx$c;A4(}6Kdo16$_CQWx-LZI7dmtyU?zjRz ztcDK&Gu=eXGv-(s=kEVV8ENP`)JXUY(>f{z=m9gd{G7ZmYU+&@H?zx;?uacZ_Z;!5v~u z?)r_6(eA0>$54+$$fUX?4U>*&co;N%g-63t<*y%TFjjmf}`Q$02hT zWPNu3#3u4EZPMe@dni7i<_9=;cydB*yN&iyUOw9A&adE8Og%$7X-lV`A^i{hxuW%t z4a{{uG?2LfU?Fwc$N26|p7lIG=81ae)n(xqIl_5$lKX4$i!h&NeCRcL*Lii&YX|L-Oo#;m{ezE(IsL2n$D9j9m)uT5yjDNB+hx1$P&?mBK*-wLIdp zN5HKRj`G&>2=^kmhrpfKR#N%oBXQ4|ZYG}oN%fK34Q@O6y8aN?9+G_%?J0lAr+n#a zPwY!CbPgDP=B_o`XC68VduZ*6eCet7kgJ~xo$s1HFLZ7Mj-oE|hp6v0e|fzL%XO|Z zFvpo?V74>Uz``%}pMRBm5I+#q?qSt2LCtw3rU;~-6`SpzrWMnV) zQ+Q2n*vv=5_hvp4n$Ucte0nit+646TgoQ!Y@ubX4p6;&Sd9Kx!YXc}(a*bKJp#{l} zWBNF|8|SWt&>H1asmD$Nu_3kQ(p{hZ4}+HZqE-~6LAwg5kElNUO_Ghxj68JO?9U|^oJ(ZF2iw+7}o z&l#BQ{EvaM(ZBd^yV#%zYkhpIH%nvPnV8wNJj=2Tg->}$cC;WleyF6d?A?)nql_-F;EK5t(=>%NObDwxw2k~E?PB;``PC=$yZ55F6+AA#js+&K>;8$n zd|qBNlX;QuO`u3}IL_g{mj`<5JcX)mAgN9&B*N*<- zL!*6>{nfrmbhULf`cY1}Ye#PQ&>V0#gL@X-zWnfV;Q5Keq##dSAFXi<;HLW|NL z6Hoc>zsTGjtoGbVd)@6%Z&p9ZpblVk&Bqf9&L$M}W3LiqIp_;wupTMHjEp1hfH#GsdM z)ZHA3O}dBrvwUsj(pOq1Rnxsr%IfQ`lhS=2x`TQAqpmz}r99f7*vPn6^&A8pX^oVR zdtKh59o~AY(ti6b>-GI-X-!hL^JA|bvR_)^sXsARUjM?Grj5)P%v&>M6Sn>IQX#_#J}y~r3+-=Z$B6Og>F9Ag8Lpg<(XEhy@{+VfPY$3);2Z5 zL;DBG?!I#j_UBUTG4}9sm&Nv3$$Koa9nEhljq51)JGbkj&|f5%Fzek+j?dOhLKcs;;c%U#*N z_y>Mpd}qFoJz|#AI%w>u=+Lx?*|VBJ_ntkuV*Th){IQ{zR6P6Y1>wk!2^Eh_3Pm2@ zR2zQcrLp1vnL4h*wJCk^R`gyp%NLK5S91MaEAj(+SQ_9i(O3y#>DGu&f0h%^@WuBH z^2HDDI5~WzgLRZ!PYG+!;(^nB>_c(adu|&{IC`PW-y7X<5;Ck0>yP_7d(Y-yd^+_P zf3kchZM667n)t`awvUh@AHoBBkjw9*>$VN9!KbQyi8^nzhBd)C&Y`VWL&M(*ckN@} z>!6x=Jfntv9{D;azlwE#uLbN0k&>WQ|1jgi)`h;QFH`o4pHGT7zQ82eDc|xn6$O1O zi@4W$N|U#DHI2Au(`St1rW~ZKsbPb-0T`}6XqJr_Ak2QzvE#1i32G^unInjlGXn}7# zW+F>A^WMb&rZ3K9Tt{5io^VIcS@!d+6&*-g_ua>xtDlqR^An^i^BrgfJ)MCp`0RZ` zM{XXAEU3Au0|1-%~>1dy~^7;6Q%b}ku!|!il47lM6=&FXj zuwHw%jj`Lvw~@3=J^EJLf8|^MTHMuT(*I#7t~7R!xXPsQ(da8p8f|aWPD$H&iCcz) z&_jUy_5bJn)A@wOITSVV&V_-N=|e5{z+XH+O5bhprPW8L`XYYXWhMTk=Os7raWo(+ zX7U~#IyS=h2+>o3dr8L=X5Q;3MapSM`EsCx9Kj@Bc4O*yjDDy%I(=)H`o^Ac-_@S{ zKde+d&D!#xQU6uk(`mlfW!pc|_aNGwbd}cU)l2eLa#*y~o?R1fPs?m>M<=vr)x;0# zy$|h9z2C0zwM<`W)wi?1t1)fj{C!^CH+l7LFR6(y;(ac4Qr*jVr>}<{KtAg}(quVr zQNLlcD?igWwU>ue`$l`L_aU#9-gqlm3GWV%Ch+4yqsRbs6YZIuHp*FSMK)B^C&=~I z(H8otcSRI={_N@==%`@Ds!0|)%8!mpLr0}o{Cc`C98J+nVdQ8DzO@}$-wsPp)C`oo zW4~;E4f?PqzKgc7PHw&QtIEHfzfI&HoZdeyz4jgIs4+krW&1~qYscIMPG@Jgk=Gs% zR}W76E!$p#FFo9i;LMr%o8UbUcRjeaK0fT50nXn&+%>|XYuhOApFN!NZF>~k9o_e) zhx@j0PlKavUiENO!NtzLBHX?M-g=6CpVPC$2UlD1T=rETWbftMJW=|+Nf0_1FZi_l zjKHVZE6#rEx`m9f9^fCw}D%oV&!(s%C@Er!;dcB3Fx1Dta#qS z+!nQiWb0u5QT!4sufxw;3blpsWhwYt)=#w29>N!=;1_F6k;Zz9m1eMI4891pQGepx z3=bJSflN0t;wK~Qt@ozroRQrB%eUf$k~-&P`hh%*{3vr}!>4o(I>My&*OB%{{+*GO zKaHd8=lGTk4`o=c{5j3lJ*|I8xapyDk7Vzpx}b)#YGmj7gLiO*iTbqI`8hNz-S39ycC5z65lt@# zFFLyhU2FIoo!@Qi+;z4+)4RX-8?>Elowlb!@q2l1gPxlNA7y+e2%X<02;JW)xDtCw zaCInthhR%6e!C#{pPE2cC!u~POrrm$oipRfi#S?F*#S>e3Ugg=s^AgV{ zo((+D@;sduZ`+h-`x+Nb`78e?o=S@!>a^mLx2?$0a{Ghj%8PeZ|g%_?Xn^((|XpUW2Z$4m+fB5J3P~^XE$5N z^yC|~Z|Dqbo&Ga?)l_D$`Ti%%wGYR9mk+cb`K;w-RipD0Y(3 zlYVQ&c>Jw({*{lxbw<`S@!g1APz5M;h z6U$dw)(8`KXO6v)xHGUdRQAnjxtYj@NS-g4X~JJAv8SfCZ)>SNnQyh>4Ekv$b(en* zIAk02{L!3GmYcdB&8UwqYb^RPeJA@^a^)bl#XGc3F#o%wuq#Kfey6I9`j3SUwC4Kf zltcaakl@YIgWUPSdw}@m*kGbN_A8jU>X*Neu6X9QqlsY&;wOq3pWduCWRITz$3?N? z8pc<8Zz_tu4V|&SFPgo)DB#OAv?_Zv`Zn!IJGW{t`=UvaR{Sk`5B`w5xqEO|{SSL; zJiIo5Tt|Q*?b8*e9W-{1Pa=9W>9VE|P8zC^Y<5l>dIrqe~R%CbeRpH%J(<2?USI1@Pk^N^= zCd&QcER~ye4A-YeKAJ=MZ%&UyN2kT3{2Q$Xz6Y!&ZZq|KAU&e?>R3jc<)lS9>(~z* zy(}%>1U@=BE#5ITExzX|;yz8B#r!*+G)DBXOQ^@;O|;ERwBZ)=d7E&`9o-6?S`$IG zRBiKL6*gm~Cpcr}E#?m?ciXd}$nw7=mbWdlT$|yXD*LG>_ym0>S*mds^#8HN80<0d zW?v}(rtl-O!TlDscEv4S=8l~;6GQ<2(q7-I+NUyVI|__U+Ftg7g<$Zp06P585; zltb_3yyx~ZJZ|=%E+Zd!Qarwt6)g4xe8b1$@uk))RdzGqjHlO`{K;4H<{H+mTt`{O zXY)^Td1n*x#AnY6dVIE?ckx*i+d_PHvlsUn@NMvw;y8@5HV95?YTdEB@) zvEJ~}-!6)mZ{@5*@zkD+Tz*7OMv1cw8{sp&_U|6A{d-Td|M5NIgw(c-PrpUln>F@Y zgnd&{W9%D^OHakdu?j+mg4jH#21Dk4^KszB_tHZeznFFzW7!vw24!78Lusa;Z`bgh z+{YVR#yMp^#wlsm@Wg7%&{<-2VZ{ZsS!z2jCOyf1Y~2X=hF3B+8qtP)uce)mX+>YP zBK`Ppqmj9N?ZGMeL`4%`pS-|5%;r7Us$aDtG@p0&Q?4v(;$CHBMA)9A~;-^!;#B$2Jnl?eUN5LU0-7)=*!|Yx6xcs6x z;ureB(D63#rp?gB>~ZNXtGQ!4dlBa(@?1!1ttwX}^5hFFN?DaP^|}dMh}7e*5aH!=d2U=Uc&v^8?nMQzTbQedFf`twmGX zkv}2dg!wV#k8pvzkcr4m-Vc?hubLv+srSG%@W>;*kM98F9v3xNf3XM z+c(g+l1a(F{y%(M$T5|%HElp9@@+(`6xCwoS(V=U>t39fFr({|jOh%on^l6#s!Bba9&?{eP<< z{e6odW0vm;KERznf@@gAX7B}$>ZoxKzma*|^!1(t=@t2OURtp{zkBWEB)jF1>T!4Rl?O@B{x^IRrrlwJ+r>RHY7gRX`(05QR%VwAwJ%(nvE@KY~pnCZ7gaZd_{Qr(Q52U%IEQ03|_AG zm6~#ipS$t&XZ51KdrkS^=jxjA?mPWky(rUtho7q#O*HTLAR_6KgFx|}$~uPpHI5Ly z=HvUrCtR;`nEDbbUa_c7U~bOO+QYz@3azs0hDRy5tfB}KBO zroxvE$cT9MlA>xaontbfq<^5tgi3rgZh3&>jS})bxa^^#_xP_nU;NYeogiP?^BQ-W zydOY!k&pa4)9FX-)@kkNd6lP(_aL&V6I)&7d6@bfE5|AQbAuBqyRef?9j!5T{R#5= zGvUd9q))kX*DW`&``^HM_BAK?7yDRpHqzc~?bYD5lggLcKA$b$$^8p@PpCYY zEMMRA?FlEyLu0)p4OjTBDHZ4o>OA7abso?^uODqsna~@?KG+8$gXdH z&livFV63tWeb(Xg=E8mPj(t3xgdtO6hkfyn4*Me?Cj60)q4AIXq?v&oyV&}gbw}w2P zAnjTHNaA{bB>FVI;%9yF-5Y%IJ)6Kcljj5A;ic#@f8?X(e%FRz91zRGX6}<7*_Vwi zy@9!oe9F^8+?D>wzSY$2QGeu{*f54pcAswt#YZQQ-Fv~yc4GbYgKfwKjr~rc&)SHq z@mLdS?7JD?CcfnU!UOvP_anPyfA;2mALFe3=mX@Lv2EK)OX>7d7(AQq{g?Qm8&_%Q zJ&&|We{EobWiO%sD(q^zgmIzHi4p(5bCN0lTI(Il&g$i+*{L*0^CLm>R9Z3KjUIdW zOq=-;qiX}fP{iWw5Y36i=#vBRo9L$sdoc-577zbB;c1Valt1~e#*)c*rSTi$NY*!@v&S>u zH9iQ+)2MM9azOrwMc68)ZDyybZPr%o_1Q*75LW7ITJe%s)~A#o?Xl7L_OhL@%7Gqz zM0l@#9_IaLvI$jhczCAl=j%UNzKVHNQ}^9z_H^P%J|g$mPsoSY{MN7u*iT^~_Vrpb zmy2GIy;VScWCKhFnlcW_bjx^M&~9IB_@Ta>J~ZWi-fy>)?{w^0m3yGtg1Wb2$7pOb z-K)nU^4DD&Mn)qaUq*%)nwZ_gz8Bh*M<^S#g?2>$8ZLq=z=@WCa+vPzUKM>A@Q0aZ)F?C3VX= znRc%C|5X0hf6>cmTOa-nV}}+Sz4|=y)lSgnYk}PFjv7$dbE@=^){M#bAiqTSf9Rvw z zAaiT7RqHrELHXxgcVYZiXyk=epDb_uRibJ=xYjo&>Fk((t#4#S603cqqN{Ulox>L0 zlxxR!@UWIHx)``j@2q#nuNw374DyGggMhVq=N|XLSMo3?G_D5@N_ZpylLz;hD#F`w{aowdyu+l{`LU%(Z2ib)S(MK zqBXLe*h4#+)6M^rD>q-e%N@&K#~HBt$HZUe##OxDUcBECuZ=#ujx{9R@s<-0eW7@M zFWz&C_khK^C6`vZkhjy=a}r%cUkg6K{E=D9Q{Zd^x1H}D_&wg=>Xzr1{C^nP5M4$) z;A5P@oSnwp*D77w>0QPK8>Kf}``Xim8_b*#ZExrmdZF#sCI7jM@Z>l#IZpS|{Mg!P z*^>#MaSpcb7GFgEwWR&uwy&2xeVJRHCgqXsi(9$Y`g`!ZCHrMo!0nfE{3Ys_Cj8Xu zm$~$d_Rtu*ryY`XFCG$3^naLfMid);AG9XfI=VmnOr4TGLS&=s7ko3*P9y(65w~6P zVMbW?A@v3^+U5hCvX_b=}_fOMV|oX7vN6F@2fBsc(su-7yfk zir;ss)pnL0utNAKtKUMNMS|6wqYHkbZ-^bJsA!o*Kg=3eXwy!M1);#T9?5*GU zP@-z*>OA{^p}AbvM2hA(p9-2g_^jwCkFzP=wj4%TB=>hN&SO1RF0@8GLu+}?t6rQl z)ZVnu&RHpOL}yQg;=l9a3?h!^SCce$h_cL8KQv=20SkEy4NUtt`z_i<^r1eE(q<*S z(Nl~aW@ug^rOl&nk7eF+Y`p!|vGFy?nK{@iyZ4QY?0I)wf>Mo+Tgj{BXVNJf{~J z2PXNAxN7TN57kX?)u0+B4nnHS`nsK;K1ESeqI@mHtoCr8kec)Vi;~?d#>P*(xD@(fy#n3F7^- zkL~m3J9CLAn*06XM1C?o%|CofZXD@t|3h=OrTBlyTQ*;yulp`}h5tQnY&*(cDw_gZ zMYfogd{^1AM^$#&zDh^`nKYi@U&dkF?NG0^BI7CZ5}qpTsp;U_!L^_6V+=%lFZ1%j z-dY6BLR%jPYAjneTQX10ff@WZpWR<&`;zpvuBsy?{pr}C+t|aH0B2}-=J~eTqiqX) zOzhk?6>*GK1ZRyU$`UFt@@|z6G zMw%uM{PfUbeaEAhS7^N0j!iKgpKbf#l|{1|<4*8ZWG2VcW^9cNg$DP={Mh{qd}izz zgHNN_BT-~p8&J02`_Nhy<<&gf1m^ZMZxWrA7Kzpp@A@=%9%cmNz|lT4zY(j325XsD zY9`JDX}>XL!^a+-YQ>}Yjqq8-qxg}c(}Aqzh+>~;O~-iIiL4L0PkA9TVkiB7%Io96 zypWrR(mc&gdkAT3PGB%PwFhmzeQ{rV$EK5P@zA!(*c4B*@3pVd-}U0d06r$kCdSbt z&ZnQ+7F$*CapqAbJZALQs!Cn-ZFwaI$l$cyh*k zt7@P6nm%aNd6Pidek%JF>R;sQ68aY(%xe{%?^nK`(n9&bL->8~ni90`o?`e_asgVb zBM;H!Z%IcqiCoBmzwp132ez8l_&iOR`dMXQP4gSkmqKnBa4lh4>!dJv=Z)wlFANy= z%BC=UFK;M6lP2&NgsH9y%O*_gKzmh+E@z~8}zO4zDGIhlYL%V z)6_{@$JR^@??-2EpOwv7X+e7@I?~Yn8rr&rHdkAV4yT4#@1N^@u$a4frehm@kXWvK zS6ycfOVWXv(`76W&0-9r{!(AL^Z!fBHs{+vqQAP4HsrXiXDw`U3CF!cY&!ugm=(f$=3--^- z2iZbK7GMt<*+9DE%iQ#9z4SBaYstH$T&z0u|Hh{LNPCXqy%X6?`9~A^ZM5&r%;EPX zzvJm?!0~O0O(c5IUa|_w@xdo(TgI!VZHuRdwRfT~?YfS&8YbyByHHD5l3Hua{r_k}ogdI=E9pGe#MQ1U#H8jlm&xVHmVbQK=_#kJdCTTae zdTRK9WJHo?q1ByQ|IXOz1L$QJb()Wj*Fc@5qa+XB)LLQjmwzZKkrzsj{Y12yh4^+n#j_TR_k`5$<% zQ#kT`Hn8Gz^R-8vkYW&MbZDU6#c)C{@kzr6koF+ zO>*NB`cipcL_W*#zbKBvH*?>*!Y}Z`4-?+$g^g#usKUNdnG`smNg>MEDc0Avl3{EyC`>x{0%l-qYvLAjqm?Hc4GiFPb`S0H|DZG=p zxDOah?t+&dN9M1Ez75Y)zNqvXP`01yCA(KVl^~8}ymZka^sm;*Mi-}rWz+0L$B1vG z-%Va`TkLi5bj)SO2NHQ-c_QO?k?*@-z%vQf@V1p$RjW8hNaKrtDbL(qXl$QHp~Y^V zxuftiJmXK{8S6`UrgYQ|#s>Old1hk+eieA8O>4fe!8`xM<(bkHp2X4rwnj;IM<4xGIJOmfSZr}-hDQqx((^^^^>_#pKX4d;M+@OP(n({Q%*9CClke~w5wx;O) z6y6Q}rO4C2qT7^Cs?2Os|3QO)A}rZ|C!#_Ab8TMHAhcmF+P1oyI%ZTY;T0gSCE=e^1}<%D*=7YID)fo4l`-%>eC)M)rGlgXl;7mO75E%d|$^ zNL#K#*Z2Mxwpj>m`Jqx}po(3UrS)wX>1xN-EC z@?ws225t3Y+7tb0+H;oIp7*7+r`p`KCpK8JJ>?@4A83txk{9u(tPN;Q=2opgu(-Fw zsz1owt>*s&zPwE4<0JX3^U^-1I(S7eUwfH|AHbI>AEa3m=*!L=M%v>~!LGx%!}_R9 z=Fv==S|`?YGHaf_b@JQrlWCkCkY5P+H*}x+`Ao{S4F6)bt@)J z=^Ec6?W}c+KAz9Cv&!tXbC7)7&>N=zNUM%C4*;dJulCZ_`;}h)x#VGFa7z14BkTjp zrnc=#{%S9^+u((PV&dv%fLrCvHJuT@iairTBYoGVXE4}WJ(D|`H^v`(^pR$62? zdF*Cvg8uE(Ho{x4f^V|p9_CCKo6Ke{w#Fy1i&#stjQGpbB5jj>N7+}&ns4w|@=Pae z6JakAc0FNp2)h~lT%Paa+iYQuPHnijfVH0akBKLwV7-m(bjhC`D7dHppOiF(H4fj*t=dyyD6;=lDX(g$px3c^D3j6)^LOO zAMeHh@e_OJTGgpjW8Q|=~ z{`Ij=K(-D%q}qW|zN*P(HQ0`1>^8 z-%H`|gV?SY&~M@k@q3cCLVFU^MzKaJx}YX9{$yx``FhdFX89z6E!b>I^K{o|WBwgV zIz|^mgGSy1ZO_h=TsL}>F;bK06nHOTNqH|i(0-ceV8+z7{g|V|H~%!>e#Y+5E^}zz zV{7;PoqXp5Xkr(3M^y)_uc1-dbjQp5rjIg)Yd_^?Pv(B!JOX2a*UHuv+Ksefw~Sz2 z&?A+s6HL)9G1^1xW;e?x2yehY+OuGA<*Zv1A13QQh5GHJjZ{aS6XohM&OU>NcTlJI z_rdQw@PBO$#ry1W?eXKwud&!w%b62r?Z8pa>iHMb^jhoduFgvLXqxfE+P9`0(^<*! z!;FWIjUT?l`&Xjr<8{`t@x$lPbdEEC{!sXLJ4p4{WQhja);&j(u%_ktGH2gU&IDg4ftV{g+(J_b_&T8tvRjJ~xnG8+WIe zHLlE&Xg};W`l%N(Il84LLjP8E{+KwdN1IOERlpI@9)?Rdoh92TFQuKbd3Wa_p=r`& zuVeR|qoKd6idnCdcz_t8J*?hwl7j(_r;U6gGua_L)?%Zvfw`RJ9@=~H@*`$oKf zcG1^m=(UN&Px?EPax#=s9_9)ntSuYGI=i@OAIVIui;}Ed3_U3y`oqYY1t%yM`CPp{ zvHTqJA)bRhGG%2s>wY=st0BZ51K{)m)^K1OX^;16U>MzBQGcdg=?g_V=;t}?hp5PP z*YeI_uTm-V)>=1m@Uq`yI-YP_fmOTJH!9eO0>U^^@1j?jl*o| zFRe*0RT%4nwk7i%Gu)PaXX1?;b}awrs=D(Z=jDHnmwz_-2GTi8{hOReKpkGD>=h69 zw@WXt>oV}U>y8h}>^0as&u#lR=VAZW>h3Z>XSk(v)?N?jp(k{a0nG;JuQXfo zBC)|5^)~I&b^~+34OaLCeAHuSjI@6ZA8PNy-$+mP?z7P4jOGhjtC#IeMqeuIEyARi zR>{UemcPRHL7>iDl-#uD)x^W#WWR3TVA-oVBkZor8mGL+|H@D2qW;p$XA|)xH%0%} z{TId)*;affd}Z$AMsB3>kM7CUd79ei_9XG2VPAsY*YIxsv-z)&72h#S|1n>U{y3z6 z`ndl*LcDXa``vnQZY6a&qOveQ@^iw}uSt1dEPIx;rF&J^7Q&U*kAZ`sFWN4Q9{9NI z&2!LW#m<@FCCk*e$EPvJOXFVR=*)*7D(26Fzdfw|6&2Ie?77?|zU8JNe}e&gxui|F$U>H7V>`S1He}d-M zTAR=RGMZm&?LP1GXnw8rlj_f>`8q2}^J}fUSTo?#KRyX7N%L#1bE=axzt);|emBjJ zPq=4*R8eQin|$i^mCi1F+@A6pa=QEM!aReur8zLH#lQsi2n&>#UQJG^W&T?TkWfF0)_{3&e`LfgTy0yD*BkUu1RYd z+56Y@m{pbC@INgLcg*<53{Uo7--C5m%dEJG^Vp=w^aiUweTlpOhw(+^2b_PSdk)t8 z09spP#b>O+XZtj3gr49j3EAt%_OV-c)QpNwW=-SSpYX33g#3@60ZlO-M;$zo$lyn0%`F0FgQFFE}e^LkSJg27G-c5T-_p&y9ZwQ@o z{{^9E9`kjL*x)OkgZ)44q|0x5=8x=UIqC8GG;)?(b>! z+s|2&d(rELhVjc;(C*<~Rv2G?9R1F?&}Sd|zIzV0%5^b$1M6f7vpD zGrrqt(}}b#XGb-slTJlHOZSxQwP?FZv}JQ5QFR8k-yC1B)8`CXc?Dzsh-@Xs`ct%D zN&N{uR*AoYmrvrHpx~U@@SehRF3*bW-(tn@C2S%#^5XYJ!fLgzpP(VQ2=``krtS9;A@--L;9`INuz84Yn1&geK7-{UBC*BDhFTkJT?*M zng`O|yRfx4UiCotOsNM3fqIXkN1kIGy@WYco$nX*S2*p@vKF|<_>Kzhf=Tv&_j>U1 zk@myh8J;cVc_C@34e!fc_@h0v*<-ZXWAwvG8ylybRq=SsA94_d59@vfV0iHdOCO)tm9|Ye!6zOv)QS}{vxVC)+lUx|c_K#ucYx(MY?pEEJyP3pm{Ov0m-0ab0DZ70x7{_H<0w zzAXM%Uuch3Bk$$(lm0KSsT`&LI>`7Aong*9Ff>$CX-{I^@Wn54z8`6<^!4tj@SUEi zb=?#HfM1%lrqkw~eB+PlrkS(ozc!C%L`$L>){A!2Ocy>LLtoE~vIh~jlszJ<&l!|w zG;ky^8D5-{7BpkhlD(d!KkX|W>9dB(?kPz5uQ&hA_$0AD0PZLJ7h~MW{xt4D$(s`! zw4A-D*6?10iH`!$k1)1_+T)#162nv%!(Tlva?d;Jk>ToGPhLey>)>iDj9uxUh)!T;3(e>T9oti3L(`b#AL-T+b{o(NjWFl3-9}jt^8MD>3=f}S@ZSU9nH91- z$L8B@K%-+g$L$92wDB9<_hm1rJ=;E#9kPE`TW#+H&K*10-p{$Z4+6Q|%sy0`VMoy! zff~*mnm4j5TCDdH^WLyvWYDEf-&Fc;)h6S)0Qp==!sG^5i)A}82 z#PE5nAEr#Q@e^~{bM+L@T+hz>hC5!5K?j^u8_BG2_y5HF_Z9{06OZBXN5$Kh^Zb4?lhT6hnXS?gnW6t=YTv_Ekv=M-giq}dY>Z6=&O1o0{#XR$&f z6QPl5q^z%TpJAfJXcwo#Qtn7F+GaP*?81A5q-Q^ql<>l+;mG1}IM@)I3 zqkd6n)uo}qcI?VfJa1mbGwL7sc8L0@v>)dU4a4?;erGUNX=JWzo<9`79Jr5f*4(Xo z2j7qnnXbA=r)sTstZ=Y(d8^vShT?a9ane48O zHhqj9O!N9!cAfUfeRzE+{?Xh}yf3mKI%;i8JMEuSVTFVK{N_%7rc>!JZl05t+q|2) z1%p<6kH03;yU>cnf>yu#h+p!&udC$ww62mGo?%vG#doamy~MkpIOT+|Bzz^|D+ymo zxMbCUZ(89j;wT;SpVA;qWmHAdyn~?> z>Cf@!;;T`=A_HC}Z}qdjw-3s&cla~xZM5gRwQiX=qW|&v%eH#0!Z+M*+Yvg7KQ^A} z%Um;Y$jZMUuj6I8p`&HxGfpYXwT^}vhg3X#nq6_p=w`*^Y@js(;*~32P0i6#;+59; zj;ifv(2xIzwpY6!p$+##Tlj7xbD2Zg2j9*`M<0MU*N+`+Cv|ft{MiNX7S!gWThXiZ z#;))QqQ->=G&dLK__j{ zMcq~BL+F4jusOKfsPgs;L(e4jj(^_m-g_TAectWec1920FI;eYR}<|K=J`EO2Tw2B z<4m62Pa}JufPS^lgnqyu6*qbX{@F#J{HErZF1N5{W9YyNbYSVcFeT)e|;#>zmZL0d0vR->EYZimhmFP0sV8sKhcamJ(%sj7jIeS*f8@&QfayN#o z3|cKyZb)Yg$9OVO5pL0brKJ7VNjr7YPKRly1<=D!{>@riu%p6G%GUsTShlXlc`u;9 zV(1o?yL?{VGbKlR)Rzml7+UU6xleOif}H8F1e-{9>8km+buljN@L>xDd(g+`JEvn2 zwmWu3eBJ_u7k2pcJq-T$bjZYc!e@5WV1ormfv3-sIQo5B;5_O^A65`gV-Fv=4fBV0 zeFr*707XYv@ZPXsIM-8QZ;);P-IPuI2xXAZ;+vvF^FF|t!ux#6BHcZi_j&jNs{I2x zO8r+HQrQ)M0{AA`_J!l(<@g6&UcNZ;GQ6EhyJs!9p{s$qhz}~Y z!$L!_DQzWIoI{msO;sj3y~|?9nqmp zE6QU%#XO^Jvm!oO(;Hdl%W%2|Ns3 zP}|2ofQ(;1_H6qQ<+%dcMtgl6@Kx9YxF_lec6S#2*N&Ys0sl+KGS*PdtLQ?W^&5{( zv0#v8uODdH@=G>y&*Wxg{eJXB9eQiRf?J=dW`1=Lx=%iU%pv$jJYAkS#ELi5FE>1S zVK?rECws>4eY=)6%3M={9$)gdKfay5pY0FD9qdurMQOC(FM`&3_Etxl23W(|2cHy< zGk)6F!nmwqKsfOp^GAcwTd!UbR@-lbMz=HOkuH@!-h7X{w&FDUDUN?ycI{3-?+0ktrxGDc$&M}xufT?IMDJA=yAOlr$2EtNAzv(q8P)&B`nC20;ebVeUy0yW&Z(l z0Et_~GXoq8yw3QL50<-F<8gy8at;Bt{z&*D@cZ~~=NwV#ChaAUuIWRaikv-OIvoZU zI5FTgX+Kr&wmVWK#!sg@!R()gXmK{C4YDTS_%3JD`PtnRa!q> z1zvnx?dJ>!d`C$?f&8Ab6!3@rbqy0#G-L{Mo$#i9V#`tduwp$Dy zukdA8hVj>w;A5JJj&tW4s_h-nyT*4qH&b(tJC|P={-Bz*fAx$KWt@*<64$|mPyKMX$wzXH+fUhtsc8}TjAK6EH z{pWYwH8ZJsNVa`d|L}PHOoy*$KNq&XvEjbn%kEEEqLWx#hTeUOa|>erVNNptM9&Ew zVf15gZ1fGCNw8zk#`83%9PXLFM(=HeRT_DaX?&2$c$!n-jPI7RCAI%)WE8%(QrhF1 z^n$BvN6tFM8hOnrvM0H(?x^;Z$7a!1gFU-)UC#z0mP9cc8X?tsAF(fd$T~)LCPgCVyk3fIQ>mIm*xZPuGoXNvqcwhI797ZyfQS z^x~z4X>Z}RggGY+D|YT8Y&Bt*KL#v4x$`sC8uV?TfiT4;`kseUQt7KcTMQ#~zd{I?=tn%-M9zfS(liYu-Oi zhHmqnyMbCKr!k89UHd2&nXnwE!N6>1p%?e)f(g*ncxdY)XzW60?E>cN$8o1)nz@hf z%kD@&%zdYw+>w4H<-XH+%6+`LBmFur?^y=sIx`K-bEbP`_>O@E&XqvUE1sgWucq+~ zrtamuknO?sKpIjFnq)%rfmn!@oZ8ILInCiZ-!G5irBI_&sp@}Bxrs&G=HBjYs-7SULCE-^f|thvgSbZbD;T&zTlP#xxr@E z^xKt3FW5S}cV=@1bQ_4db1d6QOLdVB9_ZDx)WAIF6a#ae0S4wcMFwU&1qOck9qZEb z?b*zY7qbo#zoB^KL+I!UZ_Q>OrIQa`!6(JgXIs8>4}8+w<&&P>xLia3;*-6iArp`L zf{}URiQVAF!#m#OO2a!xh%4DB{t=IC`+~TNw<9AI7r(r1XjS(T>wb04 z1taVc@UMIS>zuaKK0VT>ChYQRvGXGFUNZmduHsxrGWA{R@*Fz5wImezpDc7fdJo@X zeIvY)5B>j)@%k$IdpzU*)j#cqy&}ZH8q<>Q4J)aV<6k6zx z7yNX*r@eU3m~~0+oyy8b@ht2IEtJUk{@@4xu}?PqiXR-4Q7X#h;;XpYDS_MVcCq{=0#> z&RhfYoO%QEotq6TaBecN(3u0&x`Ji!?5#ZCOFxS0N! zM88Z#zD@|!Pe#7B4#F;mSK;Nj?y+x$4x|%Wp=J4{+WqXSg}*!C_Ybjse!avUJLHkR z^l6)DgFY(j<}2<7_xQXQIISCKBM;4)NKc;1x5fmA`-~%9mygDQ+iPe_-{SvJ{916k zyK(TU$G^M49bQ};uKOIhAbIcz|Nf15pBLva*=m3139xQijO(QebSY@ zw~|jF7u=fku+Df3a@L&Awc`F`Q$E#Y7Hu^iK8>vz7uNhp;Ix-lFh>=S4ax`~skJyC ziGG4#8~dQw{daS(G<-VCXZX~W?=GJ<&~K{q%A2hRZxo-_icd43hX8aDgg!E%lPvhO z7waSaw$_8i(W|=WUhBbhhiq4hUR_E$;?4gsFxR=yz&z((1M{6GplEj}`(km3uRFkA z)wg(_=J_&RI)L*ro8~f(fNulvq2%cne1`$GC+~B3FNO~rIHNs__aJ;&4So&!HUJ+g z&LrM-My>SFD%pY87TAMTHsqrD-d$i1;#+g3I@?aRN*3jfj;;*{7CzhZ2Xt^yd)Cmw z(zP4#y-5dG_QE=-`0wq%DEmlZ@S&i4JZ;2M5r>OFCTt#xIcPF8}5`HKtwi zoN5Deo%0OLaYh@M?Tj+;%kO}1hc;TFQQ2>XN0}Eh{5jZ)?169p*yQr(S)?ca6mOnu zXjwY1pqn>O>&CffSe#1T`*!2RgM~&1XMu}D|K;#o5j6iUdU%0s*qhGvY}jKu*dngt zrG_Q>Hg0SSPw&Ea;?e!!6hDp*{?tFzPkgzX@aSOUzp9c9OzPZEOgZ_655yxY_ztA% zGI%1!S!I$vM@GOSNnMuRc;1oIkyr4?4!?M$(3$GVOU-%ynzj?}t?yb7E)$O|PJcZ% zD2uzGd%-8Y;ggf#lRofCU-+aSW1w`qe89t(2ANNZ!!L*6mrlagHyVE5vqJskAL_?7CuX4^*S$;;{ z>l#AKN~}Of@SLU<@X+3hAFf=HS=u;-d$9JNJLKLK0bfybpojH(aqusUC9<929)10o za%e(wi4!a z9x3gkd%m$XUD>v{am@Bv$$P$;>ofeqT42gC&mE_5=i*Sa=A|LcU2hcFHV$7)a-34= z1WaESI#-*vFLbT~jy7e?b1pZyT<0T&gVDfJJ~wmNW2>-yW7?8`&FKs8zWZjg-3k! zw`AH8WLP=nGUL>HtjCUxO^cm6)91y`-kYok7kojVH+1)TvD57RvyFeG|K0lweesYbE1FnmAif_EeAn>(NRJ+V!vEs?Uy%MI^xt+*j{31( z(vigvfYbg=m&dWEkW8+iUfmF#j54 zx57#%WJLrkFX1d(D|5~S@THez=?s3`{QLY+yWMY9p8jMge1k8uxtz5h<>#zwDGyq4 z?brD#I!W{UhoH5?&{`Z?+Xt;3K_Bgh);i5zEz2=w_T8X!S%=qM#6F=KYsf(C8?Es? z0B?Nvj8B&9JsIXd1z!*I#)ob+`#+p~=u^Dor_HrTq(5z?d4z|cS^0a8@-2KXwIO`H zpYcP!3Cnjr0jGJ$1N7mc5-akM8LQ?xUEsA}ri*?ask7}hCc$2eGQSfHTJ}hMv_+kF zY+adF)G>(tYa4(&*tgc?FYMUFx{{Ut?2g62CI0L!)kCbkgLt2gEFg_(2jPniiIL$) zHu+v>4hY`R+5ECcvYo})>x&n;c2gVmW8C9Bvcva!GR_g^*5$ttEzk4fy+u5=&lv7* zW~bM>rp&*r2=mDARy710zG23%mZSxm^&Q1V#db7jL1?|nc)dSaYW%cqq+RB%JGq@S zMR!WKnLbGUC!{?^&z)k{sAw)xYa#!%pL3GEw!Q2?Vm&_JH@e%D@5ixWK9j~TOS;dr zo}v6`;}wF~ct0`u=Q(!(Rff6bBVK2W5ji}lcg6l$j2Gqv!qH{cZ*&iqXsC!g0{(~j zSoy1)`A>5nE13sty8Dl}Gd!6!~E@*8B@%vM*imTi-ZY7Nw=s@|TluqD4Z5(;_q{TXTkEhmFY|FFD z3w_#$5SO3+9oj{9A$8yY<=PAC5U-#&&m&*9* zzNjy^Vbt#(v`>J#Adl8oJfCMXf606;PZ?*+q=RE^NG9tA z&+!HA`}$dXYp^BM2h3$Qm$08<5p$rXkLVBeLpE`Be$!IxP1 z9cjp7t)CZvC&SiyVT&nKX5j4=;}_+(Y=-wzdAw16>IVzEdHj0PSNU1r6&G!+RDQ_6 zp9?}GErN`fekus9H5e#bnL^usgEsy;Z9SPbzZ9K!34SBj)*XLz#?}CJ`AptFDY9#@ z1-8+r2mHB}`&k3tddIq!;L$s_vhIC-7GvtVg}*m*@-wN&<~x4ha>8^TDYh3O170N^ z=1ccR2iJxz-6skUgsF3kvdd4zb6U6?ZEszQ9Lzxuwr%y<1LnK3>jK(Z^i+!dUYC|*?5*73 zo5&>gy5Hr-(S9k}hK6rydfGw!3u+IYDXet>wf~p6H;=Enxc>j&H<#rGRF>?E5(r8H zDheo|AXyNCwl09;R*8ZYMXMBdL;^t@pjvLlrJx0cYVKR@hZd+rYeBIUwF`n_d*$IG(mR zf_x=flhq$TgV;5y50^35!?j=IcMy8>F25Jq<39$?sZ1-V!jIgLUOl)|fLzmr%+v&3 zc#{5FdV*~$_#nFTE8=vAS#s~wF76kCV~-#@qV+Af=M|@UV0<5UanCBw(5!S{>tFkT z{m_|};Q4OwCNc8y(!9CZ%kt2f$K$<`K{n?`uW@f;Ttqo$UcevGkP1^*LE{4o`@sVh zf*!GkEjFRlUu==-RKump%>_t8aE?X)a24`g|xqf zFt20cqSV@1(Bdq=7Ws678K2a;DbNRv^@V)bIGVAEdQxX+Lr*gOaSl#&`-r-0PK-Xq zn606H$)5NetSj$<9|r7~iV1H>NHOy2&hFfSogLZ}BMf9!?R~UXr2CSb`se#BT{x3* zlB{gy)Bq;c5A&|c76)^R8_Rv*(e0eM?;B7U%OR{r7DnG+bzrR5xPY;)&Fz`L??S65 z3uSB^$$oORcl^eY+ru4S*gf-@SnYsgc~1eE?=!FJQDlORui@W~b!^sZQbM%mF59La z)F)b1hMd1VA+4@r&|^pJEz?8;>CdWio2LVLRL*OZ*&VvFBR2tBmTdab0lt68(2Fep zZbH$rGYxL=&JlPedbX2zE3@g1BumHsrZ^+B5$E=ie>UZ0`q#U1RL2hHem-SvHE9{X z?x17X1spU!o;%iCxBCS zyG}cx+G%W+&M5h)#rgT($o!-9* zx>g@Y*V@ZapX4D!LD$%ej6WNChI5X!jnK1ox#%oAc|9v4-WfCbzKY+O{3_AqinhJV zerzxF=X$l%(557R26!NPRi-$`Q0wm|7dK0BgB6$UU+?1XP#pY5YwubYH$!n6bJpp1 zT-;5Hb2PD(IDF0ySqn|=#dUenyTr)5ONYU$dwJ<~1)LeMc2~=1Ul`sBy<6?wl@|>$ z&Z14pl%>9kwtd^wRnU08!hzTYDjZzUI9B0M?8p>?Ujqomzf3-8@ZnjPGwzo${*#!8 zOPP;LplK7KX%o;Rb}}?g^sE9|$%nR;%zkqj>v8Q7?Rp|V`WMkNOI0nj){>naFE+al05>Dh#Xwa|qu#zgyU>42voBWc`I6$k$4oJq3NN@$woE%8j7 z7dzj$d>TiMd4xEPqoaXs$8lqSgRk0?`pBXUtf5G_OI;Oh`_^NP1OL;qJr|vi0>tOy2jOMY1^6yN+7a?U_N_c=hd?{}; zil-u{E%ttkA6MNuxG5((=-0LlZ;^Ke=UE46W95K?s(q6J+MAlMu{Q66c)Ss2>F;7o ze>=Uv{jEh~>;tbAng5fVmEEDhXUS`+w4qM`ML*`^l#mdNj+@p0;!O zj{(u`tckOkOO3}pz&u)%5S@n2tcM?x`wSj%F; zp6vITXXED)FMg)I_(Rah9V^8Lun%!($#*-td4RG-qqToj`VMekeBc_=Egv9`c^ggK zRf;owfH>xADshucIobXvt{hVa>t#M^7n!sy|8MbWtf^I`iNA~1d=Q_;`g)zTBGnU? zj(~c=>s`c+H1HYz+wt(w4V@#OY0`j2#u4Ax&9}h;_;IP>9ScSrIGZ(E{(}+XhdVck zccfVQ4nFFhU{}U%`?H33tS&m{G}4p(zc9wq6Eqk-L5km(G400O2v3Eh(whWx9%0Vq z$HmO`U8j;i;=g>o#c$oIxAZF6PofO&eKb~oR21vZf5~~rDWvVkm@v)nLijSWbG!|r za3PR?uk44vHSWZ31FuRKj;kv38ivN(85%l!(`nIHFSWS9T~p`?dp`OxZDC^77kn#U zd>)-&*XDMDr+xJqw&e7Br*Gd6FKdLC1^RaFyW-n8h&Fo2-r6^OfX;C}zoq<^_bixE z2)~+hEpJ;9zC>FjvDA_eR%w~UWwkGH{l(Uu{SUl;$wdV&hBN>{FhwZo5Tsn zC$f(^!|bKf{g+AWRZy_U($F;jMSb5J+Ozp+yeKj#I@ihGGbk&tt@HV>`L8r#lK+qi z(|p;!2J_v*&;G$UyD~~0{5(SS`|zx9!JE#9H=PG>8V7GGfj5nXH;uWO@wR7N@Z4Qf zgpblq=}a~iTDxNW;Ct=dEv78(OE%KKPse@v*yYUm@LNucmce(;JZBG%o(oI~@sZyS zj%uDWLO)sB#2LKsPJNn6m`y02GQ#1+J>-!tawEL330zW|csqP+&(rW9<T}Rr`Oe(9O}$nM$)A7ozD8* zk*oe@8J^OC^?E6Bn(KF|U$khLX{*k=lFeDiIa>436y?$Qj3#}!$&=>4;qutD6r)cZ zWYT~|b`w4Ll<(!(h)19mZ<2Q*VLrb48t|FvVds&EyeT^oor5Hb{$n5AdJdB1{}ddu z^ONO==-2kKj!sM)V&-iNV_@}IS$-D%g&)i~z*^I|S)9uBQ;63XCz&wS??GtoyE?y9 zHz4ii+T3*3UIsXl39e*;GuhzI@yL@Wut(-DZ}!s{-{n^c|LoPHV8%7P&UkL-I8-mN)(*2Vuy@vO!D z$hN&lJcaPB8trSr!%9N6Zv}ZpYjP#$A)7v-@3PPy$#yDr+yUvJicXZ`E+NN`W1Ms^>+9mmTuIWcX;~a&wakfJ6 zYnBPq{JRy7!l#G%PC7d6T0JB>*U93iJNOBNmUdkY{(T2Lyb65061=$02c z224U<(Am(h4ReOZHq1FQwklV0p1q52_tq=MC!lB7{HYCQZ(SS@lVR3>qto483XJA= z)s%P`ba2R&pF5au10x!iMHK}p9Wm%rwIn<`tq%O%A@};pJ4N!0DVxMqMa*y zc@qz@CS*@?5S+aOzP-86Sc5Zr@!KjqSqi*(-V)leJz+aCGh21y%61)$Q%T zZ>JsW68c9gkBcnwt{aOFZtX#`{oCkwz=KwVQ=0tplwU`@dE;xYXP-cuYbIpooAllr zM@GOK`AV-Ye{jV1@`pxj#_yN+D)zVKKN=zXLD@qHHgqEX!V(f*s2zaJKQQ#lit&Yw zPitLv@ZRY)@|Zmudyx^xZA)vml?M;;E1I|DSj&Bzx@rvs}~pjW2jkY zpD#3J3xm! zA_sIr4hTh?!^gfOIXtk9ldfF+e?RHbq{axAFM3Y^RUz$p z)`Tg%2XicUUdJ$2=Q3ua8M|V}un3-22+w-sCY@abdhKaWuk9VJ&lArwdx9OB3+Rq? zIn$86r8mfPdyJ<|KT=FT)UThqZ>OheJBHKFj`n5@|44Zo=3pW_#PxiY}%@spFd4+EZ- zCKevt&wikgalQfE$bZ%PBz}wgif3dLH$H+qBibJ+oG~1lIHXj3py2c<=Rv#?n-cvK zWwK8Yf7lY~@x9^Gyu6_*UvCX*9bL_u>(zn%@zlnqPa8e7<~&^$q8lsMPqqu*6EVQ zxPGf23GWm%UJSk2u~M`Y-8Xcl1N3DCal+Y`Xm9I21Kl)p+?W3JB|ToxJ({>)iZi+h z;GnxxiOV$QWcZb?9MvH`%Y4!#-)m2?;;gcf4ZI&=Xw`e0@bSp^%i?2+yy+|78OeSH zcw=>K$$lOEz;B6k*NcmGjU+wQe~CVeZwR&{{Zm{PV|1`Mp{j0df_<;1pL?%{x(*HG zH<%x9YxnNpU6=rmdun0ig{Lpz&dEyhJeyEeJ&CtzcsJw_?}i+j!teS}V+da%Pd(7} zg{K}k=7r`(p~i#wA*;JSR8{>V`@CDyVux-?kLjM|!G}XtdzbU>ci8)7tSH=2b4w;^ zS+RqgfPVw{&jf!waVNx@vqFteJF&$G9AuyK-KYd{$ATTbDZ_g+EcQ{>TF)%rgPJkb8OdQB*L_xCNHK0U?muUDYYlzmmjpruEg z-RS(fB%Ite!aep#;=7qOVGmxy-4ut@QOc0Kt9C}N84*qBfqemW35T?I32a6mAWnGQ zp#>kgpYCw_7UC>U6UTm7IDMmm&%`cF@`!8-vkYEOCBEjuc~vWlF0l5a?34DhugW8j zu^;V>FV@ZpFSOd)puYnzk#V=Yf9;6qYsh(O*Y_!Bhr#jIHh#y|8PwO8GE0cFYm2ry znWnfMoYvbp#MRs~mv$Ajw9D-GIwKRbYtI_m^V)fOH!Rb?hyZDp9>EjuIw$&+lefN2JA!k_2TXh6FA9;J=P5+yE>p<*z0ZX3CKKwM z|LA98!FgB>er_VjGDp;Z>!|2l>eu^4k>>lhMR-G?4tt|%v!65eMhn;@$_B~D@AJ6X zA7FEoSci--@y+|}|GE6v9iKeXD7$_6tG%gS^wo~`>@S73s(%OmR2*wypDx>|3=qmcXFSm=ac{-5W6nuO|@|u8OrTAa)zXmx?W3dmu_#W$^dS*iOty?W@J?TcKBwptz?d{o9jm<%t{{yGr z`OK9qy-z(fNc%A(cO_Z5>urolLR)$>EGnn zydR!*8FM#@IlPp)yo5QO$li7WZ!lOJhe)%f=|{B1(RBH{dJ8*^lGz`&*oxNN_o1;B zjn8BDk&aGpA&=;8iS7*0wunN}U(sQBo}th5#n5MHoT1NQ-jEbs*4;DcXDW1bomo36 z{=4)EAEt4%7CSKLaoDf|A-e_-I*N?kM{kp_<_v^~jmwU2?lUU2itmzciHny;{pv~A}9(43L-rin#5KoV<=!M+O`TAX~$;JZmOG;#Bn$hgSHyTh6NOSnfB!A@TG_zRGa>%&R?r!zh` z%zm#$K3;I2{og7dzjt)HTiVIk=)XuEyLp?euB+E5dRwCP&$vVsLT@*tuhSi#%KMTG zjTKJMW-bhkh(zi(LGLQ>OI;>?g6Oa2`*zZ0m%o#FlI)Z^JH^=T-{RtrO8-j`4IO;tmzLgL0*u|CruyUful}eH(%rnz{kZyW;ADL$n)5viFBrk^NB+pc z{z>-_CS#jb5_k79NjjELlDP{IN!+$n{>Fq;A37V`VU!^qo3Sf^R!A0;3>DDKfy8Ni zUw{S#XCeEU6T$Zdr_xM%w*RxZbkVobq;)rG*bk6a1K$gK_|T&4CgxD*SyO@O>|h=N zres#H3E`4F1Eci$q&E|<`X7uhpSo9()?m`o{Ri5lrI|CFJ;WIqe{ei~dtgL8@=kRa z8(Pj1cIHavO!2=94rokd2lXc7vVD%DZ;>vfC;N{x-ocs4&Z{kblfS)`qpiDwNc0+K zd;SEl!8+GlY)7oA?ujk?-Kc|AeQ)u9|AZp$ouvA2@c#fbQ)klVzUDW`w1=GDA+WWT zU4(q$h%d&|&n_o$E&~13-s5g;RwnX0i(e8y?o6+J5C7UP^ZPONTRO`9esD!}^f|>L zKT5`Y%*8#UI5W0M{*PVUQ;Ku+_CXh?F|qVE!>@30k0}m%d)J(G%cN`2d@mqva~y3A z^bL8)$487~NtYH|fl-vDK1s&B%hbiYPzu4>+Z4hhzpoJ9eyc)g@TG*}=O!Ox{!OxG zpg%aG9n-oRTd2Q5Go(MTG&1W58YvyZbB0EKy$&J5p7Dq}<{uLcp`-Y%p-a$6WK7AZ z6^x5y)DngCHKLGpxY^On3Wd@cKu@GI(B4_fE_?qgkCY?R3|&q06PUX>UqM%|$A7=@wfQ@i#);R~^S=f+bvq#-Ue#x@Z=Y9@du`(!nMJI!5 zKcsQLoKW;s=L%Zqfvma7q$l|oyL6Mre2Sj_+N5Rs=MjpwS$dl4*E7$$cPx6k#=#U4 zPHUm3lux?o>2oGM(;rS~={admZd_&3GWJVFBO9?#M@h$NeIv+!NUV#W?TB zR!p{CYY2VD;6}nqY{v4iD~n+-HcEF8uoqMQQ4g1XAv?0#QPyXV?8tPEQlP#VTA1RW z2hIvFFIODnDSSBB#Z6S)P{pPC1upJF#lgq4_Re;3=P8cy5A43qaPgxRZ)kItf0~OM zr8sDFM7HqUVb5kW+LAOg{C%v57U*5l?e^0u#`jow6OHugAv4 z6%%Lj{PW&zR$fW+J1U*IkS`|H&a z_Sco=J)26f-K-2J>D_?F%KE{vUzaC0RE1O4Z`hQw9=iX_n%9nLc$|Jo7F(ZtW~?%I zXbgY+gJO(RK5z4KFMGx~(tC#YMmsIntci8BW8oEa>E3o~4i2jQ%uaIrgHtheW+xQCNCEJu5n9d>Z(&iZsdY!jTW+)4-|MNh>04 zFZeCG@F&v41)gbVy>O2_yMP()>Xm#LF0<)MlO56^la}Rgj4vDfl|9k`lLp_+vGzrO z=i96`?$viaT)3@2Wf#b9EWB_;#i)XZ6OBD^B({5CVr<8hz|JVm|A}Y{a+=;7YPuad z|6)N^A9R03kN2Rh+*4qFtjwI|r!p6!^G(RiT1)b8t@Wq>l8b}?2N`qe#pAzKRwwd{ zA9OGweyRww;Rj8(yu|)B#6C8G{cM`$PB^ zmY?noKQ8^^DefQDGLNFCh5Uc*O4<&6lzjAi+O4(xk>VVUe#6DRuQ(?^zvklJQJmo! z$^I)Y?)Qo_`ie|{y^DK8anS9}3CLu6cl%4x_e(5&r!4_}M?X1H`|AHxUaT^Dj%@!K z>S5BLz4%2DZKli&v#-YgpVFX7O4}(}QEAX^;r8RCDLz;98eCqYZ`lVMdY$PncIkbN zMytCg8C`&AwZ<0apL_&8kMxZ8b~=?p#z1iO@Epnhcbjp-=chvO>~@8$ftd7=C z#z`j3@WB^o^ts^BXmE)$xyeP~Rw25P-&JbgwIQ6@u&P|POS)?=c`DsM0G*T1v4}f2 zx{k6$qjU#=uyH@@rGfESH{~Qlr=Fdg-)j$I=vY> z{TO`_U3?K5U70)VE9mrNOV@`ppwk&g(CNqi@a1E>Sv#VQhSox>&w#dWgkCpX7*DSk zj;Lkb9YCLZ0DbNO=yrR0y)lknZ-id&o(Zjfu15@?Erv!t11%7(?R8AS7anshInL7W z@L6Xrw1?8E&yn|_`F2O93tUnBlcXd&<`z$fh&;_rBw^vhDCA^JbzHH!mx0gJuD9_e##GWAJE+Um3ZYk5q#v*9ti zixT*4kYCf_E{bSD{66Ca+KFv+OFPr3PdM)#=dEi%fA z8@@buTYm2ei~2*W$FYAGznX@BzunvkvbyhV|2gJP{A!@a#qq0WTwH&}8Ge=FKjq?1 zRGjz~c=UveQ@xg7rTdS$xGdt-#`b)u793BE(_5@SXHm^w;tt1$It&OeSQu_t z0F3M=V$Yo!^Pd|UdyD)9(0;G5QBzDWL)URt&lGpE`3Cx{o#UEyf%na-~D zQQ}3TMZ2DcraVpmMFRqw`yg@gb`^c;r)2+?kl`ty=qR?vI_LO71&2; zoZhDmYVRkMQQg#R+N<~viARrUWTf{6YY}T;+Nq0j+v4VGF$GrK1tll@+%swzEifQ_WbAv z#A_bAJDF?=-V5AZFzV&dB5vv*=|cJ<5zPI7ls zF84~{X^|ZK3x+%Fso*U-`K>p)P42;7xk5<3Ti*@(WsMP_h*2Q}se%)jKa z8_-$C`yL$594w~{&D*Dh5$4vwwDYkyt^@i02R`X-QJ=0 z&alc!bYpdRRuOe@PiWF6vkz-7_udKKCCc*>>KL1~;_q4~HS8;Y=Jfaptm(pLtd7XMJy>J*D)~>QXZNO~fApPc6PB z`5Q=|(>1k@cg&AYpU*{hZ|wa&!43%90I`@TMZw>zSD3^*k#S>EElLH$(b}$zgBC4&?XcN#6Ij6M|z~ zyqhwkqyA{u)*SLX`w2t;Tj=AP8ExoeH{gVORnSsXMngvQv@y1f?~x`NsdJ$f4~O0v z^<5i3jd=N|I@pc7I?(MJ-iQr3o$WQGi6-{S^=8D7x$`KyDK{~{{ucbG4>t1EVq+VH zKlKXI-b9|Q>4#r+=tdsCY2{yi@06bTlG|$7vx^pqzFq-;v_1r|>+fpyndcH`_n(5n zR|n;tW@Lg4?uy&Cj-)+yu83ppSsAF6w)QpfS$==uj2;F0>btzB5+_}P=&Ro*uk~S? z;_Au*4*KfesVE;Gyi=jJOTO6zTx5>dBQm?=gau!EJ@RLFno!wqSWL1`3>~9XzDZ|3q zF*5Q!>+M$8&%Kwr^FPJ4>1Vp(KOYfi=ss|I$3ScK8khHzxV+c2%PaZw_b%^N;3^Wb z+`X5NGUbO`b6&$e$7$2ECRE~wNPb&Zls}HYmQZvhW1u#VV6D}|yEF&lML$LF^wg%y z*BgFRoe4im@^1z&MRNteo_ZAbD-)*qYfYHq|570`(c|VDxe5CML*u#s#+mhy!?QME zAL~U+;WxSm*IV+#X<5-@nOo5!eypz(qSupO^xH#@jSzo$)~kj_>svZAg+ZLjb4;)3 z6>)itk3Gf(xXb|Vo8J48A8OHBy^Emtess4&?}&)!h&}|g7CbsL=o4)_<1+B&4=L6M z^+xVk1ZmUw-@c#QXrpA}GW?Il@TEA8`*`xD7$Gz|VDx)6&$3TO=A6Wsh)#c(JVw68 zc4F^Ny`hopUq+hvx%4ELxH9+YT#fys&ck{^`=+3i-OC(_2aa}SO>kwEamJ?jF>&P- zyK>fQpA7ybnR1fq8kkSfS0mrMatd8JqMeE#LcEnnQ~lFjIlQ5{R=C%T_jpx?GWP~(&JOPN zNscXNJ$IljK5>%e6&KvIdqi-r$wyqfe5>gD_guc)Og?PsQWHhGiHp8sN#7y1C4)@dY^?&$r8nYv47 z_}p;LKe=cWUV5UUf{cg)3EXCHM{hS5N|a@tG7hh zTUO?K>!dR<-_FPFwaMrQ26{v8os*Ik`PRJgOXgH!W0dSaSZejj1^LLt$WRae={9rL z^aJ9J-30OCdFA>>?^(uoMd$2A-XCt;cAw%Hd+$^WbC15^y&K*49r{M*C>vU3#k$AG*X(-c z`;kz;6Y9aQsSmMUp}#6e>rU@BeL{Zk_{s}TqK@D3X26*V=+{s7hU`p$24tNZYiTQW zbu61}>RUrQ%Y+xVc=_AIqhqoO-8ldo3d;P1GK2p7i?Ry(dqb+b6~}gT9v$07U1=RC zKVRivW!pj?(UcCRe8vJCSD%(rM`@lnim_O6 z5;o!q-)S&oEx4ogbxW`4Az%wmq%HG+O^cfswFTcRpHjZsawl!^`U>vq2FX@!#v~5z zX3bw8+MIB8!>0+88>$x-#MvHlOI_Cs$$Y^MQnd6e27QFDli=*MgI>x0e zn|V|FG=Aj0sv~qH3mm3>CQmSLYIiUOz@1FHS7d_2jFXwS2jki;9HzWBI85Hp$oCHI zu1=?X#_Nc2QTgN(T(Zhfqd#fIvG#5Ml=RKCb44oUpx33vx0I8h@l62^9dVmE zZ3owIi1)mCOTH#t7>guEx`r=Kncyw%dqSxDsVD5{TofxQAw7w52DESY{bpP~%PNe0 zMj2jDZ^$js_iF6QB$I`I3yooXExkzhZ@k9J_7%uE6>+{w%-XNcjuyJRrT0y+y3A96 zYaWUXJZoOEPdY!zDUv798EwVRVk`Haw`yOdySx7>tms%b(LH$Zu_ny+yP7b|Pc&hs z-`Rv2en%7X{)P$D{E!J#{pQK!r*DT9Lfa0RFd6yr=x^iH=8namt@RoBO?}z_r*mU^ zljH09E0_b!hy0R!)tCsT;qRkkvb88>SjHj!U& z`v0WUBONBL6kUzvUh9)Fv%hpDosh3~43>otTZ!?-U{E@T9mtUe{(zqCzxR-f{!DKEo+12}yD z^*_YhlSYn5K8fkRU6gYm{87mMSh|j##hz*Z*%P@(A-T%EUDq7J_XuGYp4UWd9JjL#<4!>Gf(_AF?# zE)rN5yi3g52zztKH@SVw<&49BcU^pXv0WF2=9iohjkvWjhO)Fapi|@bvM%1^7b#=k z$NH$5%C(E51Mt&btV3syMrp0lMqgSZu}58;8gkq*(Emm;HAmzV3jbc^TKiF}i{t`pvwrp~WgC(=omI-BAj0cX|;>s;&P z5qPkr16h7GbD?#z)Xl*sd>?I{XOi)L zbpN;4irSjVTFGRs%+y+8kN9=_!@ce-SnEW#m)a-l?WFDOgEYs|`AFV)ErJak=Qxic z&m1_FGjaG!uNyJV;L(K#FTS6=z}Q{38F$k^x%q6ogWIU%8LJ+@z2%tbXm zc;G9GBUN@Uz}e?w@so7mxr?fo^@x^87u3>+)2&<_^x-qgmP}Jh zpS)=~(Zm8vpU09{IEh>|ekQgfm+%|LFO#2qN5*oC^7r=Zn!kX(UM+jQntsRRgZrzq zp$AW3-zvU$8f7TI{=eh&Q>PPGhCWHUg#*~zh}Vg~==~k(VZ!(g?m&A^GV3zSPi9?; zSLTvW>p*LCIqOF>L9|wRwC@#NNHlqH-4CqQB`Gb*(1irjT5aj<`R!qqgS=?V;JyuI zG?=xS;qRvm)!VV9-VA@Pt2d}~74ZKs@M-=RjAJ8sBVPWQ%e$XA+vW`a`|a|+PTs$| zygQW_c}RIbba^!|LEb6t^6nz{&m2ihh1FsaSr*=6~WhUMAo+LO7+)K zw`kQZ4rgBCTXfs;bZiCeJ9lNRv}#K{PajG@?q=NwIPw(vb;hvVgz5g1CQR{{nJ~%! zi3#Pm;pp;wAj>Wx^>6CRZsh>9su5Zh z_>(s4Ew6XrQO{Z_>{8K!3dlFmn_zPJ#6HB*~z$V*26H`xBz@& z&t~TMujJQ0_9GMWUg|^v*0e>3^ta!9w|4g|$|4|+A^R>H*#dCr>(DPGAe^|cU$+B_}HdR zqW|Kf=K|Ndue5w}m`O|b&va!8#>%GY{;O@uvS%+RySfLdEOZ}gm*o8U)N`UqOZ9VI zS%L}9E`Q#ptaNktKEu`B3pnw#M08@cD-&Xp!E4c(o3$M;d#`NWqyCsW;@9%1IGz^2 zM_mn^S<4-A?cBrD)?MAYPxYuQ?P|Y<3%-Jkcl&>h)J<0E-iXY@FssLo?_2BY>?=xpv(2EILT9u4U&Ix#Z`9aIMtcc-5G{Y4 z@m6`yl5X)X&B%Thhw(=Po_Ao)J!IfB{UyLjrdjCn{e05JJPXD4bLP^5WAF^wfA5$Ajv0Qzy%*|;$1(Xm4dkX#E&L+GPYZCYK{#glMK(BQ z`9+3*J@v-(i(U@bZY2I_{9?t|@Qdf-_{GKK7r(gBgyI)tnZGg2BRa>)qtS&FqX$Va zK3V?%67Q)NT(ErCBuP6w3#oWGac*Ux3TV5g950A<#f;ud(Snld*B45BOTFU-I z+p-@7riE9Wd}8#!M%ecKlX$Hs?bmf5@I8LN;ur9W;4BSWE!oYkmfh@S=u@=c`;feL zpP%mE?ewC5CawUUV)ZSY*MW_#y7he#K>+@mz=xlkLkOZ zv-MHDE7&>1oY((YaifSUurLql`)s}o?e{{ySA^3VHWu0M2b7LZsnmVX)AwM`=8LsP z7md%MJ+l>uzILPH%I-IDcM(@Qnlc8!Q@VIPZn)%;@#8L8_x;H1*s{psYqoXi>h&1M z|9t)ro*l`Wc3hHa-;Lx`IkF+pI}9^_$9WxnnM%B;^4RCsOBYGpll`-=Fl`v}XK*`bv^S(2x>r3mIQz}==Z}lG1IX|{+RHg|kvHUiWTJ@fNB*R@ zp^qN_jU51aThCCloE<=hU)IPuVTT8bMDw?@^|PH-A=k} zm@y<-QZ~vPBDp+LI(K|NW0*H~_@c;rcWjfrd}QOi@p;76uH;_v`Lt#BZN_KAk|~jB z<$(vrZ(eO}3+8i1U){MRX3yEPj17Qn3Odo=-wo*#?R~KMu>$b2y12DnpK-=q%{lcp zeDB+S(Js;@8+?y;Z3NfKb%wuetg}JLayA08LCElb%ow%Fzmfc-$j|xl)yhB28}b0> z`p}E29pz(UPw_U$PSUN7LKgNG8UF8pYf}bq3vWN$w&C0D)4JzIRfgI(rhVFwTw|M| zv{6bMLSMU$iK(oo$s=2hKkFP>@tofUbMs95F}RF!NLM+hP|k`$8Ut(&7#C|N5R3tK z62Pes7cmZ{*qGdpZGyp-Cv#hIMRpQhOJdeWpcPN{H-KN*i^R_ddd#+Ck>0muKDyAB zf>XUA^Ra2z9{!4XzNv5YS>fL)lpBYCH@7eMb;>1OecVf$>`2OG3xQ10rjOv>r@*O? zY4p+9kpL4SUU)sJeYp*kOZpCUxql;VBeo+3uTDELx})>Cu`u|pdFTjE{_OP=qj}+W z{mnTkx*h(xqZ{YO%8w06xqQCUhL(0whOyBAPVL%Lj~xv5Bk}z`UG2hF#MFJ&Nz`2u z3$qqm>i+zsXd-oQ%r&^b7GDEBdyVvFNY|GG{nt4#a#G`p+=O>RVcw^sy+u>|MN_{8 zJ`fj--Os3J1?y)?LMS@tK5q{`C8L{mHc#|=duPt?imwH&z4`rG)~T`eXtzesX1>aL zwZ(&w+K;`6Y5(W_T5wHY zXBev-Y)a-cKh;0L9*Dhy?g4y4S_f=m3a~9HXYGDkUfi0WeWz?oJ_F`a<;Sk%e$LYk zp7+jc#dFz}O7NIHZB*@N00c9$Z*xz zkTjA<@|mUO*mqL5cR6VfdsX`rp4w%v>ZDDV++zuJD-?Dd+Ebc6XNgAgh~FRCd~Gim@v!#u?g|p zK)3>$^^o~a_kU=@G=HH9Q~e4PruYviypTKX3LoYT=cDuqel&5`dG=l!x@BXd@=fjC z$9mg_?DL<(YaZSJo_+1sZz@}`2jBrO1OK1EYJOJ(iw?oq)3hJk00$0ocJT~wvH|%j z7~#p2z4@s9zi{`d{ns9JF=|@XzoG>~zo3sl!`}(SM-HpCY z`HQ7{=l~9O1Q$Djljt30MYuPRU~~`r(LHESSOAUj`koZMdbr&~J;7WX-2*oC`zIN_ z`C{oGhVZTZ_d}!y`iFr|zxN~Ja?no%`iG^+;#O~tZ(pZ3zu(B`Y5q*skM>FP$P?%! zzJxAnpQN(@t6Q)(8`GRj|2<%Jeq1ZPH22>RAhV%cSZww!=oSui_2P8V=q9Mc$i2?S z;;VEE9`zQ9Hm^RZMYqrqIWN#HTtvO`x`oJDc8n$vAE#Sr*~7%?7TWA#r0bP^QJ{-H zi~N!=hnY~ig%ajxEb}ym`8pSxF&e+wZFCF&Yx@ zK6}FW>nZR4j^4-+-e`ObjfD1@Jxh63gTbThe$ltaL6^B>5%CI!SKy!WcJM}cq505$ z`!4)G{FF0QV20dj`~#%LwF5@X|3VPqh`$fQ%Q7%5=*V_3^_U8gC{n$?S)=!eAa&?}6 zJ^%4BF=Hdo0M+mgRXuhf!u2`Cf5<(KkHHD0SBD?YtSrCs2)dsmof3RQy#J`_=*jz2 zmiHCuy?MKv@x?#EXvVylu`gl{3c-Wln)^EHW6-86DCcqf!3%$mR@cVE%@aS-`ro8; zw2qX~iLyFVCU?~?Ok_T>B`X&V&|PqQ4>EX{O}^%1UZ*;p@$aO~`JD6PbF9@?CRe)h z2z&gF=qEav^ZwJ(#c4f1M7u;cB>M#X(}ORTXm9(6eumE%@?U5D#{(-G3eIrPIWIbD zm@Q*A@xlS=dy}{?u@_x`CHH&wtZpdF$`R`+go{9_D!mj5l_c0!+Rn$_8i zGHDt9NLQ9%EM3d+H@7Lv>i7q{x`(T+$X)|ozH|7tdL7>XYLn0Ebxw5k3^4h)XW;Uk z#lh2+5q?vr_zQ$bivqC-!bgpf6KZ`w`Yh$OD$C!(@w(5Yg{R-;PxCdw3 ze~;7A#-9WH95~kImZG(JjB~0v+g`(d+Ky?q|7D5Q0j+oQ9HuQ=du8YaMHhmzZLjql zU~#+-NP6`L!0BMVcT=9$XN?Kb|C>keK&2HES}7;7P3PrLXiXW9Rw z``4xq?d;XA_>b0LN9d_N&kppo_YSqV{1ScAddfksf8feQga6{gd)+y}^Q2pySTb){ z*gP+29Dvuj?=$uwN&buAMDU(-beD;pfovyCgz}1A);NiT#&@UX@m#a5X!9{llwXdE1*B;7` z$GvzS9P$#lQb=@h)EHc(3rT{F`kk^p=WnE;xfSw75fl^y4!vE@jd;;a>sx)Ud5n z{=QLOmF6QJ2fG3j$b!#fcPCsrMq?6p7L~^Q@7U%+7wmh=I&VraID!9NV3ntNY(O*9 zeC;(vf3@c0Wx-F(JZJiKz#1Ck@_x>Da7J~L%WL!5H)uaF`O^Hq0%y*r%I0$p8j2l4 zH`aoG$dHF5Ln1R!w`uFeUxR<~=To-;XK3H5eDc}zsoRiQ0vU1x^~U4ioo;=!wAu`fF*um+4{{+$N){E!7Bqh0Q3lyA5Vkh`TyjWGull5=fIgYkTbyYf7be%OTT=~@`X(Q z%Q24syD<;bj{gq-A5I@uLf3+^-%WXj|C{15oQXeYL{(>LLR zdIhrXTz--VMtApy$Y)(ALdofG;QJa|!s!*=yon>A!4=n!Y0Se0M*d7>3uOJ-d1gI% zlBu`kMQ=RA;%6meqcyS-9Oh0*{{9J(`~#B`^OsP@Z3eHC{ht66$P#Bcyk1INDR`|p z1h>)kCi$ztt(JW~dQH*t9}%ZF#l`dQWnQ%=7LhlQr#rX7Z`lo4nR*8C%5x{Mk|%C< zdGBq>o9;J&S5_9c`GosZT)rEClROc-FEvYYYZE$w#2nr>4DqHaKH&AXQyK5vR)^>E zmIOGRYw7w&)Dw@>@p45xU2g;~K{D;P0=iz%_~j*EK4#ZhhChz_Mb|Zl3PTRJzeW6i zhpvlv>n?`M5l#;#zv%kuCQSc7$LV<4;w~dw=sdF<=b5~bdwmb~mOYWTlaRZU*;}UI z+qYvhFZ-zr63rXnoslneS4ntk{rm1k=Ft0X!bvj^?mqDa9XVG+pT75lj$1^HHH zD4%@UX8RBQ6TNwcH)IX^ZQ+LE77$m4PW-9+)^hJU)4!i@Pdak+m`b0kboAgltJ0as zY^C!(cjw*PbWU@(($SCC*Lkr@(sfo?Nt=!BGdgqiy%JrbxeJfZT=(DCpxb|9>+RbX z{N>JV74n6Hui}^fT^O4|yLIn+L-VlMhUTIC&Wv$Jv*l;E_K&XGB)fh24bomhx)<)^ zQn%(|F8MU>z^?w;dSG|P!BV$iZ*s7As1Dj-V7&oR`v$oym-Y+x8V5UFu;74!Ede$+ zp|uV4aVKXSmpa%|!O~X)dn>TTaj=ZTPR=^ccd(ZTmVJwXT>D(HCBHT| z^_1#|6QYrcwye`BOXF7t9Y4g_3OC+z`k4Nt8JUFkp9vk^u~PnUkxAU0fZoIj_m@J` zG#4khvP>Mb&(d8hbM&RX9SwYv-xWBmg&(?n-CObz*Djyv^#S74uEV1h@)oSim(Y@r zxOVxblCRd~`?tx5t)|Pjmv6}qLygarH2)TtS8pexcQSVu!|0vNxL4*!dpqA+Pmxdh zx5Kk8fmTg~W=(*0jfaL^jE>+U&Sx&<{;EBn(fll7Y8m*CGLI$ zcd3PAt<5K{j5yt8)%-?W`+peUe$qMvBV0OS9LRqwX~MTVln0um|N9+o-$R^m`&SxI z()K|ok1Qw2f1Pr!HsxT~p>m*qDrc7~XIgwYr0x8Qa&}S9w_G{n$)oW|D4A<~ett>( zzHJeEzY{x|1-jq)3v_6`xZhcfY?O=N{Y}y_X8JE1yteX?Y@v>4J#K%tpbA-Hby2~| zz-0OVm=Ng3GX0^Zt|b2qg%{$}oVglg;<)ppknulNA#-!GLguZ%Nze52Oqk@Kq!1c? zjQP&;$BIw$hF@)N5BT&~zvCD1=>y1^4e)8%3mt?{OE<8;EuY>J*bq$(bGC@S{lFCN zCUd4#1)m-s@`n5xJ`KM$eA?$6Ja3Np^vC9nxia+QN36|NC%5>$d#O0M+oby_M}PmW zHuqxAPwg9cYw>@VHamM+4gUB_jE~P`|F862bnG9LwF3WlR!&Ry|4zK+?f7>k?JtL# z*UB&P9Q=7s!4ID1Olwzp|4RDCgF)U*e@k56Kah5)v3afXt>jEp^SIIF`x9v%X|~QR z|FyV$8%R^0a07Y9iZ`eEOY7jh2b$yfIp-Y2Rqt;eG8f;a)x`ykQNBz0t{!FcJgD@2 z%|r6KWh@G9hlg`+Q9)b*alPO8{x-d9GN12_dz**UiLZO7MFxAJw_`^o>0_v`=t){M#0kGwtU^e zLQk}(zQn;!7A#}bmajWl#!-CzJO?`wSk1HHFSni&?LNrz&kIxrGKY!ZdrGv6i!V_; z@`Z_?e`>UYi!UOc^YQW9PGir1KuPa&i)6$)u*3E?pP*L(4anLp^7uY%POZ%IN6aDW9oYv9Q~&D4H%PnQl_CGh8t)#V zfFGw*p1u3@M^j#!|1NngA0@4u_2>TvX_k)?$2`4G8LuhM@KNHJvt7i!V&KFxiDUjc zz)#j2I2XrU_9bo&aay}0Tzjz*H0=%EAU>P)wtSBAXy<71t}=Pke4jk3Ct<$zNqs4E zrt=&5rmDHV3wFJWMzruKbeq-v%0^25+gU6bFx~G7eKPF?m;Q?~Ee_xV#NohFQ%|y= ztGnSMu?Z$V2w;@agKO}<-QzB^35EdPMR-B~U9 zh@-6*=QIv>{;zfUrU}l?e~rs~6LH#q8##=$&s_di`9tL-n>#fZnR1f-OI1#>#iRFK zIakD&L)v>^QO+v*agHlzw90`F3m;#1<&?yiL)xaVDCc#`IoXvnNaaBHRn9M6IcLU~ zL)tICqMTinlj+LIA&=4T>5LDYs16f?3&IuQNjy$GXx1-dlZ1Rh{`NSLLR`?M2VI+b zv}==Ozaw!s+2F`N(nMDd6*52Y9Q~i=%4sG}__bDRmNdzJN0wvHTX&jrlKfiotSpy2 zN}HkkQ|Z&kCXICg9!ut4Oq-iN>tf}$$sTpu{?|;9p0jm57}^bu)Lq11T#3)l;!D;W zd9A}h)=R2ic5!e|H{E}mdKLGk+FXv`V`xGH_5W7!tfx&1p{=hPIP|*;S^KXjgf6_S z5SmeC(o_8xO_=1bQ;6)hjPN&y@!d?hb;xn@)7jpSv+OYRW~e=5Nbrwg9+oF~bs^-e ziRjH&q!tY8K{;96ul zrK>$(Wh3(9Poi62^k4bnz`^&v7`nUpi#rxRx%F2Ut^DHsQ6GHq{!bdd$mh;Q{xyT6 zXRtrZn|*F$7v5<+UvhV-Z0mWacU?AY?zxSL^|QC-htl7k97?QvjXKG*W$>P2y?gS` zUgl>YbX+>g#6xqo$$v(#u(u~Mv|#C8>=GjQs+2wtUGYTWSBSDdDnO>gPp9xJkFq?< zYQVlB0)8yu{ZjD-%8vF5rPs-hqsv{>8qVi?4t*}BobwNl-ZuS_^WK`y8ywT`Kc6tw zgg1o}7v6l&O%02HeWAH|-HrF0-0&~v>kH~rJ(0p*(d0jPF?^^tQrC~QXHIMyy)FNd zg17Q1EC2p+gsCQ6L_6|BT^HVT&kYUBL(C)dWar}O_8?s-FG9O{`|=n4LkpMY(}wu` zIbSwUTuz!t`%k8Af-7@lnO7TbtmVDK(tGk7DzP`!oaEiysnO!Y(d&zU*JT3sQZw-b zQHovdhrc=eSPruPHT*}u+2YmYMW3htN(~#W>eL4 zOELTWqMKz~^#<+Q(7~H=joq7k9@&0sv@Ach%)SEO=WomBdF4U9C1CxGOAjjjzV=tL z-JeHWBli|OTpNzOeJ~+$;emwAx0@4^pPR>A?M_Huq&1KTE*#O{y9Swe#b2k+NT`c_ z>lA!k5YBwN7kQG=e@_l2A3g8H!RFoZb>tO)`GKPyZ^qS=#5hzeEdJ>V#(xg!UPbXw z>Hp2(<(A$hL!gOC{>ijm=aF~eD_!Sm>wq9jMhDh zj=(OZ%xdhahVc%3ug>0#?Ywz;t$ZU?_l=5oYd=^jyTs~Kqu7yA?x^=!Lv6<0W851l zPrN)8`gRuULF44zP}q2I0_VFky_hXCH+l|yL9*<1v`cg#2RxE2@geO{{AA)qQ*t_b z>*Q}N7;lepC|Fed)49Ja{%Lh**Zt zTtNHuj*M)qll&)1i*#mO6C%$=PGA0)NN2AxWdQFDf$NHgPSoWy2YKMIS(7vR57)a( zw)_*LYTs>?@j-(4SdyPln?+N`(B57nLNh9{orzzkk(1DcZ9gNL*b`f-oY*q0$vNcB zB424I{&(^k)Ahd#|GUr@{?A$HRn5r^-RHXB-;zrBd@O{}X)_9pJc54`T);!CqXbb589uzxf$R4{y8 z*c%#|Q3%|Xd$vOpOxlc7qM7H8U!V0(VPnR7g%-bmI3jw>DfS%5;CsQy=yblXfDXBX9KB&rtHx%=BrF`tNhljqk%z5E};fK(zmHw!( z;dW$!YZev6rXUaP@R+C01+i<{lj!?z!8fwn6l9OT@LeK1s;7EKGmgz%G_<%797*$6 zGG@cdi};`Jui*diO~w4r@So)W`}fV>HWb_w+!K8Nmig~B?M&n;>(5uVLLTdCn1M<6 z9|q>vEijUy@h=0+0pf-mm<)d*Fl$<1bY^E^{zaVXy_a;g%Va15pZN5^-a-t`R_)Qfxo(n5V=`#x#m|CtlKtH8qCyaxUv;H0!6BbyS32n)fl#D{7rp3@sX7i{Jiy zbZ-%U6^jb05@&avFlx9rBZ5APi=C^-C4Q@F0C97On})t=Ku52~`H9C=op+M=`FVTt zqa}CeuYU!4QE{O842{Y29{YoEK}>Jm+o#>|F-0ZGNXCWOlZ;C{C5f(Zy@Y)8t+|Q(IsoZjH^6V^ z@qVX=tgSmx^WZP*C^v`qoW!$CdoJ^$SE+sQE1U~T4*0xx zj`ZEBet+WQ-&Hg674%*>t8qzuWek3h5}(UuJ`<5!6JHUox4y#`51$`OUG#P7IkYbr z>&>*k{k}-D!wt0OI<*zKU@`03?ve20Twr;>;;$h6c4!u|>{|70JG6KxaT`5kOtra| zb&0?I+QAE!Cf2IGZTCbTV`$ppc*la&R^HJY8s8iay|aH(i1p07JF~E%-{eINJp3dT zev$$|Nsi+uA0Y=wE@3XVc<_@C_^#16^_+=c+6_e|d$#xVCN7_M-kuJLh4+;4Uup30 z8B;S0&pL*0$;*|4!2ccw|5rzPd~bLpVVKs!>clmfT}>V;^J>lXm)mXyWOoM*D*ghIVPaQBIOuNB99cavhyF;6DLSe!v}Yr0>R-^UZqT96!^h(9 z!b|=mF#qo8?ODM-sYj=A!=la2L!$S5Z~ev1&1cLIO+Cw-Q3E~v-W3DZpAW5?4jo-| z&#C*ymwC2_^1Q#)GzgMDq-t*^L|aJQ{$+2IyLtH=ec&= z|Jkw4xKHE#4B=lDJ!5$)ZI=c|S1$2&Y%T`K=l?3{$_A@DH3 z^%cC`yvA!7DqI(j66~$CzZY*PcwG%&U7UaMDgTeXH;=EXIv4);J}1LD2~)^?f+Qg# zISC4pF#>Wz2!tufps2J4#MVFm*zgpHx3)jA0)~Yb0+y9^}aEc>) zWgjAQihj)MnQ!4?PiIiOZ^prN@TL>*$oKXCy%83|z3u7n&@Dr?wNj3>MQ~5X`BM1S zmLImiIomz2DI*HoKs54C2jn2mxt!Js-0Tb;?ZQ5!Fhkb(k_qLAnKKIScx$@RZKlUAw3)Z-x$;kh8QR$o&u4`pY&WAj% zRdVkt&B|HnK@KXrJlVInFlLo!FS1`Ub_n`ojTgC3@`#)wbE|d9b<}er^(~^_6R7`q zKmUM=| z{ToH5q|8M6qan3^nD*|>qQ=g^_h-&`y?e*~*az3S&doT8Tv+-FXVx&6CSS!r{;Vgm zzYDvhQ_uS2OkeO*r`OZvg8bG8xRs?{l81&`Tvrfq-=ef0gWF4U7(=gK zuPo__eWTLXstqpbo}PJYcP(>f_ZgL1>+WzZXA|cHWB<#U0lZ7*FLgk^Hg>Z|xdPeo zS#-{2i{{lTexB_l4uuwX2;Mkp=?fcjzA8C*#Y3Ky9+O|(@QUl@4f%aXrPm*tpAyF@()*YQ9hIJ2ktwKir2o<#3iB zPW9FOC~Tug{Hi`g&-guj^hYCowY#;5(gtj0%fqx|mDnwvPVI|XT3;iqzG8Alr>8f} zy79-gBU;-<}K(xpF z&UO(xUUrso`rBu11KPsqAIcgItc3QLI8w6qHb#oPw8j-PN6AB?!^bhclBWM~R4`n| zzQLyP?8O1kcRoJ>zr}{O{$KV&?{O#_)f%sD8`K^c9gKwc3cWv@jm`9M+s;i6U!Bq~ zl3Yd(dsXTWwGBAN`ChWO#KO;u@SpcS+5x^#E&+yc`h>Fsw6CLi1-LD9xPdcgm8=9F ziybtBOMHtyt7aqnAvVF2(M_sALQaPR#1k>gpG+6!$NaoB4xFo z+v9kyM}7;I+?Kh4ytZa6bE43m+hdp$jfwTcYS4`vQY;#j4jrOi+a15nZ%zAa z!+Dydhb7(O%bd|YjZ6Ejvvmgcfd*_xCD8a9;B4soY+$P9Za&JtgrARz2g;d~JbSs9 zu^&U9c;i>CPt~L5WQK2CFY`2O|0wn-x~emgwL}(rgs~7i*(B;7;vq$HZ}SjoqtN>R zdUEQ0*!SThm3$-PBYW7}s=}~?b%6eNgb#FrA9RKCYE1s(>R}$CFKPumL^P|@kO@GGK z=jLZz3x4oSmjypfO)px${NLjK`mg$C7x$quk1Ve%VxjF=Iy7kJ zimgSju6Vj=)ACJ4jul&q4soTe*i=+S`kv*Ri#D&=T;zOkYf%I5NAONtzNILA`BvcO zBCU2uQIvK?cgD5Z2MM>HOfw3{&dmUbZy5Tbe}Z-+x|by+o>+f@@wwu z?0Dbca^}p~aO5F$C85d1JQw!VVzlJ3i}U+vQyYu5x|4rlKYp{*Puzsb{->Uryy~es zJ1*0#gTbxDJshr~|7*vd_Smux=bU}nOXMB)^5e^M7jn&1RfUbj0mve!UL4)VuJ3o!oVgk_FjZRnT2=o&< zv+B>B5fhSSdc@e(X4%a%KSV5-#O>(jL%#r6J5gSguK*npYQoLy;ErqbOrn`c22x$My>E2 z7d-Z}`*v>lb8E@Lzx*~nW#129*ziT(Z#KlBGvvTqV+M}MJ=O|*$XAz+{xKZ=qXd4s zxv*?-I_dfd;)`eOve5{ehwAmj zo`H{!fRFBRZ{Co8moTSC9??H=8FQUbv{|GHcH+aj`qN>M3i#J}T-B}u5 z?O}c>T3imlpAEkshfW%RC-{*mvaq2?b72=Qy%jvG`6#-MwPiEl?3SfyCwO1Txy57A z&BhYn$OnznR^|V!3B8lO3c-d*trgjHd-{!TW9MMxMDSerOhZ-JwuX#~VIMmkMjCi- z#eItB{JJ>e#KoF1(AZP?zF-Aw!8`ww=ih}~FScPP^Q!?_?G*Ao*Cmu#tW|E3QR-x0haJSz%Wi6Oq@7klQApw~zND z$0>bX?6M8C`y6Na%DU?nd`H|@WIP9XFCE)g5+q_Y~@MHk$)VsY*VaVVwYOwiJC2T zsi@NIoQFIji-}!oDRwEbSFMWFCW#KP75@4o`p)HgZ#_J;KzrcBgg~?vaSKgIupYa94-XEL$=KTbV1u8>q$19IT>rLA0&C< ztw8zrA^KnhHn0&5y^O)|ZB<|Fw3&axwvB9%`6n$n1pF#N*2`oqP`+*ATesjBGCl zMyXFBebvOdfkVI}`281N@W!jBRA*L3)(x$Ut{WqA0Jt@L_K;ea-YvQ?yhGh9{PPpq zT6i8mm(<=uyTnIAe4j4mA0g*si@#n2a2MZW4>*vJiEjx1)ad6;y|kTbJ)QIev_;lT zeD?lb@rn49K3DA%d*(4#Nc9WrH8LMv9nrIyTbn8SO=y7iFXH1+maLIA=4;9(*j4%q zdf5<}HxIkuEiUX*0qjx^WtY0OJ39I;HQ1$UuuBE7OS!O1$-7K#M5-<4pE(7Ms(_X> zVpqF`@;2e;6Dt4AK+2D_^!*azf9TjOmEDncx@HbCbhoe7flW(8=NFrn=>3{oYjt~l zevu7LN7S=p87uI@e>ZgO_mr2Qu@)|4Gl_Ko+M2z?SNAM?9Yyvk2ghfFue3!MdKhO8qOZOLLWh+;p50Cc>_Gij+cMy%&glP<47mQ^AOmjre?SJ@ z_P-vCWE!G!5Ggxz<*Z!2h?0j-9JY84Pf8Dejt0itHh_&$+}S(GFdosTmD^qKYOFW%mAnbBVF6dTK1QSR!w$OTba2c>tGB2Sd+ z9imgU=<|AK6MdKj&q+G$HDagFZdr)D6ma%1hle3=@s2f_HJK?RgWc|n+AyG{{9uj4 zQJt!Hdv+G@gtva%`7=!!`RqfrWGacz_Tf6s|M7#m{|I}!te6V1=9TnGNcPyvm;{ie z;D7tP&^PaxC4-x=ElS^Ikk5x~JjJO6FSGlzJ1~G= zHP1hkJgbls<&1QpQ!AbE)yQ`0EOha?mTx0e8FA)Y@VF-W_zmd2=(EzVvc`bS(o7sa z`ZY@RYodNpzv@=Mifq$_txIgF+VnXs8tV&#>BD#NzdZ#`93Q~mX=r3UvaayI#PY-q zR}uS4&U;K^ER(iEZ+jI4D;$Z{_2iX2SMXleNOUJ_CGZmsoQo6Ll>EYPLVP9e%7WlS z?Rd)|{F|<#3?1Gg`1m=`QmL=jI+9&V1MOjsJD=`)}|{+fH#wUdcZ~ zd~YS=sd!<#w2-~x|L0ZdMgJSN9MH zy&becB8Lmzu+n-PUYXC>_1r(iN8mouzqGOzd;~nXdBHK|?<#$0OnK6VT;^FkzURA_ z%{+b@o_hoS>~CXVeHHr$bA64rFWN|VS$h=j?+*MTx>Gu)8Fl26!C5CSGOFL@1>4f)2LSD^!r_}966xdYsU>1UtN=ac_(88 z`Ts`VmGI)n6)(o-L7L5r*J$wKSZCr}?fCJU^j~&@AIF>h$alW}YdT{?w(2i+fQIbz z8hw*oOnTCzYJtA-*o%*G9o6%j`1n;-n`o}It zvi+Zfd<)&P(w~l@4o{+vZdYTEy$9sG@~8Iye)59gfXE_? z7?0e;`0a`B+JS75Blg`(TGA}y(4yl?KZ+nMbkTu3`WrhDLu^%_fW^vw-VQQS)?(8Z||Ps^Dl(Im!$aW%FFT5 zXB|38ZyBV<(Lb(kHoQk-kO_}_!qM`A=;5-aQzC8sk1=(U{P}+UxD+l1 zn%l(QRwiSU`DJw7%w}an+UMHS$4JE{$2v8*qSejd-&$Ix@djm2rd;*haHa7M&&Z|7 z5@V2?3z45kBS()yo-V*1I+FQzK`#A1^0w;-dEPc+mMsG}MV3hbFC!Tj!RL~$zB<k{?E=r)JD^j3WQN}v;Gd%Un=X{pbDZ#z5%FV_rsYD``HMYt-l)&#$P3a-jNNr??7 zxEF8Z>igiT#6kWxTup9=tGl{dxH|M~+l$3c<(GA)p7k+&j|Qze>pM`)*++#uhj8hQ ztG|h-A`9O@OgzC;-5FP1<>;pNJc-Swn7$P}U9Y~o7*FHAhNr(K&;Jdc4*wtFY5$hH zY&-f4PDMS`Y|pU0lLMQ#t}hibF-ha)!emxiqd&*qdZ9ypQ10gukWM{kniQYmFTBZM(F!7=%ea; z+uvy2NX~G=mS@e|3;W|F{uc8=>|pG}u=Xa3|Ira}Ps)Wy?hLS3NW(s+WQ^BRjhKmF zt&?}e2JEluk9^Sn->xVtcA3QZa4^3zxWee$s^e|zB)`k84fy%SeE(L;6`CB11#+ab z!>`tc;Ex}}6JNNYZQUmNW+mrF;V(|VCuK>D5gEV5(thUtL3GNyxR9a4~myNXc zfLs<96ngQ-$Kn2S=&_>LzW<8||K6`6{T|ZG*?X~vwSscas;dM_y{7UHDZ?deX6!qV ze$8V2;V8Pb*bi8fT~ms^dF~?Fvm4tC&P)~iWHDndw&QqkMey(z;6u@S*B2ghZE&%7 zHIs5&*pOYUZM(56NPg_yQ(f2_yy$Y`ixky5<6tqm+d12p$%Fk~c$)BIsjt+}OZ|?r z4*%vrNA7>IZ}0C{J90Zw-#_bKL&kZp+g0smzY=wdakE!RaL6q_tKt`E)iuV=ekDm) zd*H6mW515A$6l|a>-%z7wc8Vy9>PQH%d#h~8+#a1*u$Xa3clTDsvYlB=8kWe@e7_q z^^v`C8|}SuRnqx8oW6slYLAj>>-eL zx^wLPNjc8o-VlGc{l?Y%cKer9`*Xg&f2riO+P`#Oj&O-gk(Z`?uNxTy@x9Kcy`gqU z8M2Rwz3Pe0*}uilTAvrPMU-^gS~RBHmZB`;Hb!GhQU4414udP|89&(?e4o?7LpA33 z_D-byP5AbzXMB6d$uqVY^^9-qXr4pDoJ2XrrugiYy>&B2RT1Une-97Ozbe ze{%X@`@iCcp2~Veeyle2DavO}Ics=MRx!varO1ADcS*iOrAYP zQP*x)6uo|9+Wb)~elfpmw_Zg@xF5aZ8SeWQb-6ih{&g$1bDvX`!~LWcJLZd@-C_LR z!tlfEx#H(VrFq)^v3e(LlF`lS;s?e1O7X?xO2_B!8u7V{*N!=%N8&r-iu4_Sh&~-F z{e@hVtwYD1QEw^VyVS5EwT4W*iaCwCQB7aoN?(p^)b@{W)Qo)nrgQk?O&A$obZ$9t zIqjS!SJLvLux=Sev#5L3@~!iw4&E4TN1piPQ3nroaPzMy;$pbHZvN+bE2?C6%h=Y* ziZ5ir7jD))(O&)mT{$M}vBgfk)XzG<;H&r=h|huOu!ZG|4{8&xd20@MDss55{QiTt zbB~X!TDRQ;jVj<-cuKy462n~fSmBTzbblmLq(WxTYb2;t-@75tlnKQ?9n7&@Gr^8D>n|P-4&t5&;~=| zZ)o&+*nsE_kKkke7uwqAtG1o&FF0_W-l@7v)En!&I672!dHv4E49yp;#Q%JGlKADt zo3HDg-}*r7()@bgHS2phI#<6i{@D$G@M!*CTIcFNATPY;(du@EX)zz%@9=+cHTy?z z>{$@l_a?dnPX4IPoaCa}kFV~$qWQ7>+J1b+1=)`=d5 zUul)a=PKi|QK!x9ekJv(h}T*_r%wOf#~PdW6}p@ zn=UW*Blu}eRbtD^u!u>idPGjfQ2ZnDS*f{Pn*<%p**ZjQQVz42J^qE48d(c96-6(A8+DODa7%l@O}x`?3S zE$!d8we2isO=~7JF;(x_TmckXEcr6?vP7QP9`dblDW8gO?)9nZr}h7IffoGa9!+BD z)gkkB^`^h?NzlD>(gtW#&p5kfy@U)>(^G7RF_qo2{(R<{ZsOHTg8x2ElU4YFH zc@ZD&sfCf+en0>AreBQ@L%f*{jtQO#J@|CcMh|OxkUDMZgto{*a$gA^o^s6m51p8>gO5sVQMIk%HGFI@;eX=JgkFnom$qoBtPtY))>$8;H zuut{`u_Pzn;VQ#Q0$~gLtRyPOvp^S@D=SF!#$>%3pV_Va!P z>}PVwe1xZoFI!s5vSDed%Z4=^iCmBt%YEXqVV~cQ?=WXYN`JI1^NZ9nM&Nh6;0*N)fL^GWsG%V?^)!jS)KdezWCRZk-gDEAGFwO&IEc=9Q{z~3XujUCDBt4lEOae=-D zcH>)K6={8+Q!z}dd~|4ADZ7KJ^k7=milJ#$tA@61;Q4uZUOhBzc*U@^aTUYb4wBZ) z<#Ytorm>f1N?x#SI?q$|;Gg+l!!(|!>A|<;e!BH;uc~j9(Kd^G#ay@R=-0GOced=2 zd<)38Ko9;-?iX^uP!GPuecKZ9-osU{2Ls@;rjO#Ah?bp_??LiCs0Vk*eKq&hdhl2B z&Ew>Mlxtn+n=yRjZh1!XZ6x1DJ@~ZTZ{>cg9^5S7JWKv(xSk7rGoEicw5*qWFOu&? zJ-ANpU*i5HJ-AlBd6oRTx&9FPW+LBoZh2Vp{e^sg(Sr}k{Xy>Ovj^py_sIVa*9Z1D z26W2Mf)$eQ2>Fia!TaRCnS1(pv3zra{GW53w!iW7jh{1`C0`r)+VtQZa_=0W#$=v+ z6YU7LMRIj^sI`-Ta~-zl1bo<@fL5_4ia%T;v|w<|O3%I6Gy=P^wLxc3Ku1qOJDd8j zca1%9g8!-5N{PoX)s?nM_Rcg1@O2No>Zyn*Tqiq9Q%{p%%_jRwFcVWn9sfec!tDGm;GTMA+KG>nUW&^NFSg}^uLil z(6EuOEBR`j*fHRL2b!IUS<-j-i)KaPZ+LVab1hr-!~Q^@2*2R6Yo=xdrsJcl;TIzN zQHhZ(yte~wZ<@ll{VX1)83avm2-@dUwU*g)Quf2tG+&1JH79k@dV*o}r!UneksE&(xu5kX45h zv$Ws{Xt>lNrSg`Rv)Fl5eUQyohVCniTQa#9zWEo%C{UGUXtVEbNtr#cWv@K*Z6f(a z7W{*H4}If(BJb-%?Iv=32yR>{tu_yIFj}?3m?WIi%*IGHiwttG# z7kn=PybxLfe-->oiz!^}8i0?K;x%3)neqI4JbCEfB<^qIUhriUbr(1v+NJu6e)sTR z=JIg$F4?@DcVg3)agSB+l1$?L@?DZSgLfihpYE#de28B_+N81v5Lw>kOs*c| z#@7frWi{!-|9zE}Lw%LYh8|^3k6m;}%R}U=CXbi*6}&r2y~mMWA$h1PKH&aL-ly>{ zjWji9-fugPo71WDzk-|V$lo3}ALkjo5c`(G%}06ux8P>wKgZ4Xv~4Uy^TiO2jPd*U2v=6Y+mpQ?uF;B6$rn5OyJ?X;2MGO??(i} zyH^W*D=+wvKx6^I0g(e%2z)*-xLhD|#WI1&6vU`r+pr-oc)vhol%)cZPZkTT$_w5r z5P9crfyg>{2`tSE-YF2-=?;O&P4j`4oCNQcG-R>a0;lH%Zx`67AowGJ*#*H_0`UQx zDKMiTSS&ELASmr^NGS*k4m2bd1aA@;TM(QkaAHC5dV%8#f>Q*JE(lH%m|qZ_C~$Z| zaDu=g1;KFw2jTN2uwOxNv_RzB0)eFk!F++Y7Xm_>%@6dN;_c_wV%o}M{F%f!P&Adbpv>cKJB5F- zXYqi{6*U+2@NJqN#^vO4Y&!!@)^vT_sVbLoPUfnvZL|34lG?WNF8;Q1Ew~Kc`C&$w z(J&f%5B)gIe0eij-m{d9^lgV6Fb(AfZeD0L6;Z>NG;Mwo)VjBudv^^fVZaAuX4|G^hdNv8a@P))@wb~ThyoY_AEsx5& zMxr^He`PXv;bUtu7iD2zPBec`+uY!d=tzfVg%RsJ%#gTNd1p8`fc==^(1SMG+t4{L zxSqTX(Rslq1x7Oe1v;7k0^5AS#{`~5UJ`iP7kotE314uvz&^}>f!WM|fxVdj0yCKZ z0#lj)0#lg(0u!140%Mu~0w*&61&(9>3mnb-7nsld7dV{xFK`I+U*I6-zrcQZ!MQ-0 z6DQy+!jE156*+Jzvg$U@TaoADBapSkfm$E zbZl+u>@7&g&PJYn-pxIYqxs0(6SAr$FEJxjoK9$HFg-7?^#roiDfstkWGRV5kd~*n zwtz1oJW=r-R`^<8;gUJDgRyMj-tnOIyp88H?nlToW9{b|oINJ`{rj|GDQ%D#``QWl=UVX^6gyF46EP^2^13VA1$lE%XIXAkU#^>`R|< zl@)hN${E_r_<(XYv5plgCy{ba&7v&yfzsLRce_NHjr3_lq+=d23jALy`==5sJJI|F zWxjySewzL|L0`2nkAn2u8T#rh^XdIjzS^yem)0&{ytGAVN%*3rEl0K%`6GY<`fD~< z6n&J2Uq5SP#mc!@msqwjMjzD!yHX{`UsojzRzPl23Fn8N0x1Z39FWsd=rfzk_jhGRD%6 zX_xtnMqFkVbwjJ~PX2WIPWHQtT~lo5qBBKvMRCd8mimhijl?+vM}S%0$*_&e-!9&8gB|5;V|~9_qy@AdaoI;0UP-C6@k!=PrS%V zTo=dKllT-8dtCliEqI8%e-$uR30ZRgG2cizKd|40{Gm@xd$nd-miPlo%#KUP>jMiZdJR|Ms%KL6_qkMBNBu{ckq!D(HV1mq4`Wu-5zt z<&iQaUorV4kL>N>jKTwggMmol=l8<IiG_^ zC*VJFf0lcxTTsDR^E6P-vB&1Iuf5F!AC;*W@6A28V8Z!*xi|1#+pA&w&)nV#Kef*I zt98L|jTkFSy5e6ial(lAF7j`?G5f&&XAjy`Pn=EW?HVoYTH`-S_wlUg>)av6Q#{v8 z974X=?#l~4&$Fzh^}sGOqQPr)RrnNZcA*^6kEK1r|CPMKUOnWQ;Ng7Tkl56z;Gy8d zDP$9&tG}bH>ybIsSP@&cf!MOrkF}&9%4d9LNnB0r@$|hW^<<1hW(malf=QtW3-4diZ`$2eJ*3v=18{2h_b9PO0vf&)?C5=WsJn0qT0lus33O z{C<^=*T7mt>ykY+veuT`*NR1$>e9;c#C`-nbyX%CO+c@nY{>ge){E71rN=l*`dH!+ zj^UgFbzhlc;7hizR8LWP=Mckxc$`)?R^PZ@*7Cjb+yOrV@>^vj+GQlF{1?hevdc-r z_mpp_k17Ly1XTvl+MZSGrOx!j0abU>>RFGHIyM3YXI*(YvQ||WmE)_;c0DpV`jKsS zvOZqAYTN3`?x+2C`t&i2#m~1qOg|Q-t(ojwGxkhW1wIqn*fTSdt~;|y^x@m0jFs3y z$Hw^{8ylwWD!<%XL!Z;d_t=}@aB-e?Y$fX+vx%?fzRZ{F{XP4YJK{GCt#_sSa$So& zU031@(LRQ`>em5jYA#Fc;pZ7s)gQq$LuB|g{qgnf&lXT)_HL@$0~2cR#r(^q>2l7) zPAC7B*n!2wh^k@SLuZoyC;te=lBI^Z26j?Z^@pzhHT?=sf&ycK3(qciphH=#(Db9x%hCE zRZ?fy=45+~F<$wWTWgGQ&6{Q2%`G%Iw)q5jEAoNFLI{=nKIO{%zPMf?+!w!@GUjc7 z`Qoa)n)vF(?bV5`@;5oxVNXqbrWSW`8>BBoeQ5nB)7Z^_qzrd~9p}0rBDf1#SEY@y z)5b&ucZSl2*=fTfh)p4VkZJ6&>N@V?x?cSMOZQ=>!Nz2(4p-V`Ul|d6qo?c@PcfpR zkk6!zn{$ll%QxZMNc@>}-584u2@jpxJI`0Qn%JG@Ygl6%x=*>FrMm@fH54^26 zeY}x#7DR^c-BIqNn{~O5k^3&kHnUFGJD&S=-?2RI`(Gw>l32vlX$C%zcj5Cmi2Htw zjn&sVMrf>WgoTTl+aPg??LjC{yu}V!Q3GT-j5gm7Qb`%eenQl@muf zdC<>S6isU>ei=IY^_a!&$CUjn+ux?#SFu5hFI1>bqwIgQ+3{K3{A&z84vD$Zb)h^y z^3*%RjBC|@6U=f)^KJ*R~YDrrOfJKSU0gO5NA0K8Z6u_P<^Gdn=k( z{=|=u3m)`{8N1k37KWW1o5H#u|9ZB=UG2yjbLPZ@(r0mIXX}6ew9h1toAghp??iTL zAg#U^GKP)QLGJg0gD(8u-26Mr#yuCf=M~(8HbtlV>Z-xLX#JL!avS%eS)(YoaW9%R z2k4T*y=c}K%JCBr+>56Ej`gRA8>?#1E1AUr$p)y)S3;tMQz?;WMu+C*E0R>9Aw zzC*oEA0nm(^=cY{-vfH*=^ncBGpYptDaWQGnZ}#6L-1*Zf`U)q9E(#o5bqQpoDz|> zh7qTbxcurqPwtm-CRMDtNWSlmp4y%DKkh{*2#jF;hdO#0C-m{UaoB~ke;-$y-NZSU z6`LlrAGbQYNegBNt@XIE^Et= z+%M-`iqA<9x+U@U1W%^%?P->iLMhBi|- z^vZB#M$W^%4*v|X>l05>@}JM=3unEDe0f&ByU2&k_sx7SlP`~aGp&4c$d|^vRb!6t z4t%yhb(!1`d=Nem6Y{cZDUJNsLF)O@?P^WqVlL_>IJocDTxBQ0{Jr zqfFvS2Rd1{Q_YEDjHBo<^4>L%cReq>8_B!J+P|aTq3>?voyhLzziU_Ka^5ZHo#b_NvCdZ; zOkaurP#QjuvZtcKx!juDoHtj|sbZMl8QXH`ux4znvg&@)#w%G@oJLzi|4CH;NifgR z*B>f+DL!~=&%psX(=(htlzjyM%^qx-M^cB`GG@dNx*5Ecv2RJ#g28E#-WF$aR$7?W zxgkSq^>CKyQP$t4mv`H+l6nT3!~ADI5BCSBb@Vn2!8X<$$+L^+SZ~7=d5+>a+TZq| zru<(TsvgDX$%@7=;wdQ8P`%b&FkIdj6e6A&jGMH;$)^|854$E(yE6dJ=#{I6ZmLQ?r{u%NFvFd$G5=PskAwcp|1Rf$cY&kpWv+vB zvX4%DEPEi=jG;dYIX`W<9;VJXg9yYmptr|K9nlTDs=6}&fr=(1`hp#~4`QRrEO$(C ztMc5YtPO6VePbza4CNKu<;nkb#*)2M&4v9c*0XMU%=5s;_0m^chFf#aL3yGhijItr z`r0iDNANucx55}t0t~E`*v>KsPg5uHCzaSjmy>pgcak1T8%f$fE=s$Nw0G^a)3hy= zcRA???DP;Gi=O&&RhO0-ojE^~zSZ<)L(f!$1C_~&2RZeo?lWh&a6`~(R@)L?c1fEW6kG*LZ|ho zMh8!$!-w$gMtDY&@Qj(>4dERNh`WUil({8+8b-fFa2l!n-fVGoh_Xhyi-bzky^Kt_bW>Gnu} z9`vRmGHjcRZvq+2S^8_nbNKwz*CCt}9F^EkUd{w822ZylyC5Gl7v_ME`0lxK)_)Bz z&s}TbIdv47Uh-LfH3{ZO(gxdUAw2tp z=O8ph@T|cJjxmM}RTV=I&BE^EwEpv~ntMrRf9hf5*gn@k!LdKv-(0TxILYh*yoUda zjU-j2#hI5W7;B~|=s{MybiDddocgwWCu1H7pLZccxsjowkfEZHp*mn6adMuQHRclE z?yN2Ep0VYqaQmN51>;Q}D0`Qt;zu!n>z(_Th1K#~qaJ@{)NBh1m zA0?Uti?C6{<6TZ$J__agntYUK_9ovZ;rkco`T=?>5{Ra<7FW2YIjTZJqIlPOF&wfg`N!{i|1^CnTD$G8U;?H$@lac{k6* zRy5N(VQZbmdZX~B{D^bs7RbKtSo0Pe?|w&_LZhVIL*U38;PRm=u?1c@n|Y_AC9;o1 z@GeE+-Ny52;?I?=@NPY6(&l7K)?8!Bnz^=&VA0Xr7>_k}KG!_nDH*|fx14vYcvmR> z@hLuB%)?C5WL@PPd`r$5Zm3!|^pI0G4p#}@a&EgC>qK(}?O^Y@^xY-#mOa7EBA3ZG zX$o%>&Aa(V`aMcX{gri9Cy)=`#DDQjc+}uJPs=1>?-o3dWiR3U*^&jc1;W zW4?@K-i%@X6e1^&_A{T9T}svYpUAkPtBXwg0Wz!`e)w9~F^e~|pPTdE(Y+J06kc|; z<=7g{_>X%myu4EC0Pcufl+1XD{*oza$QzROH2vKy{Ruw|e7yMJ=}0Zubn1T2d5BZ7 zWm2{HW=-r$^nDC%p2J!?{?=1dk-xpr&%0Dwa&r^oCuy=4HOCpP&YNPrYwBE{Gu1Qx zlJk*lRo_e6saq@SeDp|lK16Z}Ko-j!j=+wJG^ zcgx!e=7#;ObGbPqia96pPk{L*@^&CULdn_{!>m3-PE_j~hxq>|^p7>?h+T^PJq(`u z4r%!Mn-UvA^uqRYuU_G=q+OVMBD;s?o@+Mq$}931PGJoukiocCEg#A}g-)SQM^_9xhkkltUM84{YF^6RN-!7G zW~s}1+KjJy;*&t>_a}hAN466h-Bb9zGaTCA3Eb@r4tD{UW58+FRhGnZmQ@&MbVkDO zBjEQ|d{c>2CHq89+w}hd%EK>Z@;lJbU0iFqYWR;o)>k(cnLL}TDPODWaO}Q=!b{Ii z)ao9lf3DQ(kAIpU*768xpOuFDk7mQ~7ix89%eA^s^CMe6gBLaTar>LLlJ=ZdC;Owa z_~vMSRLf@>(f&{Rb?~dcK%RD0FkYov>u|kG*)_ySL$}!X)VR@%MV$FVX*XjwUBN{2 zCIyqtse*qRvO(Xk%LZQ#wQ%Kn@`+xyjB;i&F4E6RcZ#)eWxN_I@tKtIOHwp!kV;E5 zhuQyVFCX+(?^ypk|1R!4o_N)}II|D$XwyC&I((L~J;R(;^23ALwnNa~!x=(+FJ^QE1oqU2llOMlhCjg3$WFaOmO)cm87SDc|A- z=+gJ4rPkWbg*t@j{l;&pW2|Ze=VtI;=>Nsvr(M)0iHGrBvFA=RU(VR`<*Y3u%X|sR z$aVZj@bX0kpVYE!{G(h`toygbr#;fT zFO~ag)_t_xPqglz#n)Zp9|+IA*2e9h@ejdmp>L+58G^^iy&Wyw{*Uu%LhCHq!;~BqLBUvaoPx>Bi%Z8z3B~SX9|<_|?J=SSCq9N(3k?;VI0p@F?^{ua z&Rq-5yxRVMUj^gMs}xKydn=e|W+}+oLJBhffudhMjz7j-T-S3A;!5HQ;xklY<4h^C zP>GE*(Q-e|#tmem5*ts($-RwB&Df+Z8BNYy?xt|yL+Fm2=Y0G*e9W-l)u6}z0=#lF z&VgR|kSm!^_L8QMKf(Nv_IyMi9|5m^@awj9=s(!Vv^c4+(6E!~G7pl>;|eC4UnrPh zex_i&`H6yY=1~P>&47ZU%UpU-1Y@9RR9k)oV-XAOii3v5L(3ANX^HT&By^bw&Q^9Z zKdt!OVh8>kx;zCrGeWgl*74;(4d>c+$^H!~+t=E*3%&k;l*NMJ&M)yLf1B$? zF8JdYEBlNH7ILq6=uLU8fmbzSQ&rr#v*_nzkTKH6u*X1uXsCPSrpbP1LW@dMwyT75 z>VAT}Up>@YUUtymYp~y~Et~AW{c3Mhm3FniDp~UHOPhq8B4?B#yYRd{Z4YxC8K$A? z0j?E8B~N@R;vIQ_#beeNpJ3&@++m)i@H9 z;;;65Stpb}EC32UlX8Tw3#}U>`RVrx^6w?T)2FF5J&`A^dwcOjZE?e$xzWElsGSjAqrQFLLEW@{0=J|fg7aEpE znKg`g0NG6FfZ)#G87u3a@vwR5YorZUeI$9T{a2Us{HybM6BJz?Nt*wB8ts6N+(z2d zc3NmWZ)7|(;iGBrQ8jOFsvjnNRNeoOxj3?MzRWez6V8uwWW}%+ba=s)!}-EXJ!Yrx z7-QMr=7SGNzW(Hs`X_*Y(yziVAGG`Y0R`jDDh1=rN(DJboCmy^cE3wnh<5+Te*ayx`;U^}rrkGj|82DUm3H|=KKwS?T`T!)+FiiC)bCQX z`&YcXn08w_!3EkaI_tkmyOY4h??bzFD{WYV3?TS+f> zCY8?a_oSz={3i#mx$o|S-8j2qwbPx2E%DjrL1HKKJbM(l`o6>|9kAyis}6dTtLIIL zQyh9Carms59*3dJa?Z#dS78@MUX}mhcRKZf>4{TbV~<)WpQXRYnLj4q41AMvse_E0 z)J62Pv3C6n6^u1UDJVL@x37`^GaXUZP}8a7r*QEVfaZ4Prq%T@%s*6twVn;>nJ{o!o%4MQ;z)=9epb6Yx~kh=La_-SNsmT zB=^HdJA<+6AF-xRn$W`}o`uJ0*zh9C%MV&S&06=6b#2)@JWA0ukNHcW^wkaYRR~&r zDmsJcgR+kl{)~*@nZ6ZSZ31~E9-ND~T}iDY4pOh}X}6CK9;2*OIkSZIL-%*C=#r*}|ICC$ivQK0`#|?%rX>V7}L| z!^qxPm&8VA9tuxrME5z!y9$xHV)0Lsz4F*m({$p&!K1w1SYnuH$GrOb^%WBHJ(qaH zy;$2n;`F%ohw; zJ7vyar}{Sz(+K-7I$1b1i8R)8c9M6W>&twJDX>FFzHot0*rteYonQ=iYwV%rO4oyL zIK@A>uMx#LhjNZ%2hKb!&m#ueDz5!p$$GFvlrf5PsjYR1Yg*c9r$_bg z5^2|$ZM$Apa+t{IPUN*P`Xy2K7Xpf>rje`yMBss&;+G8v)YuhVJ6)OSyw+`{~`j+%v>8V$FQ}lEZ_IrN}2Tub$+aduCa5Z@(@58_=XuIIcL)`bZ`$6^^Co-N=huPFg`hF$v zrEXQgI5mgty#T!(^h=`YAszeAfmo{ywO=_@#*KCv_mD?;(49bM`!dp;sth;jQkUCQ zzGS7p&s1r|-c>Nk{Go!_{1uEdZ&EPUoTi}IkP5-;(ct$e@Vo$gABldFk3Gd<484p! zLjS}ELVO?OJo*q#6kHUXs#)k6Tr`^Oy7SduMekf; z#<_0Re5oL|m5gyW@5ta#%Z|UsC3D4H!2U(-L*eWxX6-GE^B?jOIM=lEF#!S`{diNfcQ(Af3?S3WA401n&AAF+x7i2cfzh_{gHZZDb@Fj z3@T}tlh#4f7FIgL78{S7+gF$dD$GM3l#Ui$qyU?Ugv$-f!9X!>9KDmru| zMftg0f|hyAo^}}{`A&Qqh6DGtFJo?wD&uX+kh56W_(-obX(Z@KzoRIz|xIu`guL<$= zQoA3I^G@hj8a(|8XeK;;&D&f08t+62jY7}ky_CE62F?49Cn8wieN^zT-6MkUkuLAw z2MQ0L-Hr~`k*0X~^YCz?2iFp(g#Uewhm$7rPudjX;TC=EV}y8kSKf>46XM}}D0dxl zqVUC+xL4nPjfW?iUqVyc^KcASYm+bH;T`A~Y3rHk1zz~~nk8HN8cU<4y~$<``R^{( zyh~Fef-_P_1sA181UHc;?=}NN{Q76y+x%Mb+60?l-^c%@&wt8$smD5?gpGQTTz_Hl9_I9&?-9^u2mCq(Lc&& zqU)nC1v9}N7jaYr;GK(Dsj0f7xe}X-s{=kc?1z{A9=bfI2@Uj=ZWg`ea=A|~&6E2~ zxhF=Y?9b~h_r$1_`@V9YP?|3H{pCKsbPsdiHJJN!{xA1KC0#EQnck)4<;r;`(8;N; zu)N%+Uh>Xe+Q3<;4ft#;`Qvfqb>vxD(-7H1WTy&Aqn=K5{{+%D5$o`L8u?;K!(ZjV z3D%0AM;8=*Q0R$#D>Q{M3|jQ0ukkyZW_;waXs_&XkTuLS(uLN(O?rqHo&T<(uOa%k z%!zmG?@lN`D%rc;&5W_X{R8jhY%~Y+s2~4dc<3ra+VoRrmeB)z5j{+7Aeuh>Z5R7L zR(5t(Yd;ztoGoK5--XM!-=43Sd?%={l<}O@gKsL3*&nBUH9VgHKjfXhNshs730z8z0mbiOg66 z-WyT||9w=I)y>TR9=f0#-#r)cg#x{7et{l1itk0X6TJ20Uf>r#@JHyNl5LYO@Qcb2 zAK7E`YH&(;h{db*vP!kya+6&XzQ2c+4V|asB%O2jFabUUE5UqJKy~e4Ur%oaZIw)U!P=xr%ZJ*}No^ zd-d%_yyRBCY0pd2$ydiW?e)P${P%0T#FJm~5_rgr40uUKM6e@i@~#sw zqn(TtqWzzOr_!GtcrX8s1U{+0v1xyeoGF!H{w(?1`G@diDdQ}8ga)4iKG?pDpK@*_ zWjsu}@Q;AXXYr4ZR2nu$1v!gC!Fcln1>?+rC>U$Lr{H(AP8-&XX#pU3v|{vtoQR`C>czH;g!V;j=>;tNIw2ejAudfGZ) z`URbDGdf?0R<%qQ{^s4IUI6_f9fw064S<)oELT032EBxye;jWO9Sd`H%GueSSr3~dqJZ?E`~Uqbg= z3BJo*3F&@=$REVVOY~#G@4h^%Z!gmS=I~v6{m)Cj>F9qduNq%V2dto8?|e-M?BqSk z816(5^rHvXpaY8j=SL5GHhZD>sOW(CH`NCEM0@{6op&+rFEZ|;gLL5^FEi$%_xTy~ z8pfP8_1u$;ujqe%^uQX?0nz{b;6M#HAo`ylJg5N=UbJ!GC2-(OlHfp!c@`)>6r%rK zPx(IdKWT%j($fFl5WNrmPhw)v6g>|e&}-{}O-DjHU`lDF=zU5D^pvKH-j^(TUvlXl zbUanwu?u=%dON*ukJ9_%&A(F*TkqR`!G`5AYoR?df8OD}^fy7m*3P1y&~?%K_J#Dm z*`$lVjcvnHWyB*7$QV9jm+?AzWUlW4PNocWK&uR;_r;qxk}meJ=Ye0+H_|`bRa&z7 zD+RItDd;gx1ry8~1!K)$C>Uoxt)Rrk_;$T-RgxC>`H&apecDeaUaW(d@&Dx9T;{mM z=H8N}^_Mk(FOiu~Li6N&0g;tuo$9Pw=Og~e()$ixM;(Ng)F+C*BlddhEC6fmCr+*X ze9U-1_g3q=O$l26P1x|~IOC!N__a4V5 z#v0%5`(_;9AHLD^@a!dzJ>tIqu}6P$|6^SoF-rR}zs@*p;Yui5)P{i(3Dtw{O`US@&G*y>vG<>0UW>1vi~LJ(yr;I@*{$?k#YWE$dTI9(3oM2h zEypzEA6f5oVE^w{v2k+ycd{lhP1B=KN7Z}o(6oX0OdOa_StqDRd8Jl%E3y&mLGJ_Hu71jq9DO%!xra8r z?O9j!cGk>JH*~wZ(>1JXzCGufb8A-|ru@T2xv^SIQmnRPGixp_&dz2VWow+<_mm?k zI@Ouje5T)np8vWz_g;y6#F)iwWM8F>jhxe$2tKW(|C_^$%n06nr=L&T%#|U`m7D2v`d-bIVrQaie+vI<|4!O3bA<6Y(2X)e^F!uLC!Xaz z>C@mz?Wlp(dZe~Zj|9J#fm6%CC%zSYs^`ocNF2V1G4_uuA6 zEaxpYXODnR=gs?k$eVK|zD!H;Z&{zPVsvC`j&V`CGrGy?$O<_1tlQY@QrwL*rx@dC z+Blwg2@{ALTGUaCnHZzRTob@-%&1&oFJ$6)tU3uMXOzeqFRl{o}MrS`urG zjN5=DZPF>m>af$9C3LM69Mv?54-wt$>!D4(M~ki=tVJ^q!b(TvOk3RY_!FL_O{w=5 z6LV-2_*iSdTNZj(&pU}ZMEsL??RQ&3?;3bVoFjZxr?%MdUJ1R^v_-|lLehA5ufzJ^ zq0qZ9-VrY;jhL8E+V31!+jVjAE{WLgj9sn$t_SZ+M+))22ySheEB4%#CZCs zV;8IKpV4k<^ETaGJ&QTUTI;b*9*upY&eFdy_nzBhwNqm(%R_06R{Lhq_Kl%5U0Wn) zw`7;y_n5C^SW#t%u%ax94HLuuGy1qUd}i6PR|yHLeNi>wm0`=7AK~HJ#JO5j^a}ES z0{{G!`F}g-a$U!{Trta^DeAg>d(pw6zn=fsp*!a9U%q4hf#tuRe|7&~&(G<0x_tyH5{5nA+9`>~)c+Sb+pE&)_PWC?=#zR!K0XXcq?CX+yafBfc^n|q(ToO|xQ z=bn4+b}r+n;AkU%8}AT>wbbT>ZFt5R-b$Mj&bvti8DkR}W7}~)buhS}3UB*93%IR^ zhK0@S1OC@0?Hzv~eSBktHb=e(oZ2`3F!i{ZGIfZs&1q@S&ib?VIJ<0k#o2YkZ=H{AeA>AJ^TftyoU1Tf zH$Ln9Ddy0~-?(>U{uX3=JK*3K|+3;iI8RcAbU}herdmJ@wI+ zHfra2oAOK=GH=yak8F-@m3`fc{gGM)<(ens6Y|h$8+9P#kMNJ3^fejhZ1mah=S+f+ zC9}`EZ-?-;x_#jXPmXN?uWJdU1Raytnt>@nY}mtLnS`qww)RB6o@3 zeq_ckxg*QDr_KA355MHhZC7N-1B7=)hCC=XGUSJ1BR778y@mcEX-8qY(D!7Xco;q| z{Uw1q>mJ^r{3}Ht=qnqhr9G)dwf>B?Fp=?kQ|4IW9fAhV)BfC>N3Lw+S;(7jJvO}R zjNR}%XU~bRyL(Nna`)I&<&NF-y8FhY*WF{1s@zY+jd8AuyU{6f*b~TCV_K~pcYfn; z=ON7MetJ9q2&`@FG4DYZ{X2Rce$aZ){ApP4)_2-^zrQBDMee%cx2-zz$j@WLa>uL~ z(^{)QcAfVWWAeWllcmka@a}hyh#uuKPT$sgzmiosf2(BGn~}-Z7=2#WZ$(!90b05Q zO)ZDtErj3Io=Dz#`=(RQahS^+PdR&IyEdJ6#$%3cJnf9ep1J8e=k=IxkOA9a{{^{p zKl107JvQR5b-#;T`c;oj&i&n|I^RVO{TDLm3q3bEKSd7xCvxjnWY({`-|qY;a_FbX zpz{4Ga_GO1Tc7Ir3+L*dckul!-`}|3L^l1`$gkc1MsEFm&tE#9>iHAr#@aF!-=b64*0c4R@YoYahW|Xa@9pDceo-N9w1zvq$+Xc@_|4CtuM_l} zujx0N={M3Rq~8P^3siq$eOda8^s6JPU&RbMO}~n}chPr4T5gN20LFdvt1noezK?iQ zwU~jO7&|)Cx6aa^Hq*Dd(znisw|F1^bs&a*C3Z*p)kemV&Gf5Q^bhfipkIB?eXtn% zQU~4dR0RF&b2-CKzl@=8b;3V}zSUW5`j+&eF1R11e~kggV~k7P8HZ&4{@p&?PP2a% zhjlJr%{$HVCYQaNmiCnNFZ2;+)4yV%%ZbqC6ZEIgjCoZUegp9nykhBhV@K^Be+oJ7 z2yNJF!|Tp#HoW1ybJNq#HJE9eo^j@2W^8)anTz@Ts4DkUm^Y!R%h>DsH6n_n3cU>M*_Tp18C3z3X0Dz3DaQZJ0^M81*(X#dh{; z7?Twq7g_(m5n8$pz9;zI7Wam8S6Ep2;#OhhJ;8A``K}(6-fnf=b?t6vth#N}tIinY zR~f5L8MI@k&+iS3Y3GL+IbVB4Wzt+M-(bj@}=; z2{~KpY_Bb*^_|qw-z|My>TNQ9GM71p{C`{T-#B|R7rk`evGXnZ@ixpfOb*6FKPkt| z!sKIqwds53LzuakBFwd;F1q`V`oX=S`+d&-qb|8ejk@f9zWZa&4c$}uKE`*A`-|B7 zoTEn7xhIVJk+J`m&gZ+|!uL0Pf9?Lb=VQ(ZqfWRdjyg`AZ*!iHz0)~t)OYS7qfWVF zVjsqRl6c>`=aVi5S?UkHGo4RP?OXoG-dWBMdoOn$=w0ZX+)dj#KgG85sZD2{t1-7> zXpauR$BvP?1M>gGur?>Y+n2mkXm9)1b!kT^x0L-N`!=uBK9gB*c(T_n=hj}oah{@m z_S(W)UrV|AQ$8u%2KxFRdS$k|?Wb!#+K_uA?erCSedU}Kqjiw>l6mPV)?D7!+MM9b zM4HrlTv%`BrH{uxd3#@Pd!d6+wbx|&`sLM5#?aY}rL!1QXEL_V;C$0`-gTeGm^+oR zcM9{2)O^Nb{m$Ha(fQJbc!BjI%UY1!0lx?yf%Lvabe8!|XHS^2yLiwvwngq>EMR>z z2_yGqE{4YryqFtaSsUJCU~NC%^&nhyNy&Y%2XG%d+M+UQO^>a|UTDdgrdE~J>+k0t z82-;j>H1)vjjsGB=L;7tm|FWqm*(q!>4i?BY2m$;&d%7VPnCY6Rf+o9&VO_=^uPf7 zOXh0&`F!rfcF@lYAH+{TGcP({eCLJVQ1d!}Jutt+ZyM#OVUN?){lfZR1KlszQwBLB zu$#Q)%rI-02Lj!@vYr{9W2AYWG?MmNY#pyIp6$x7v*(w}?&M*tAdcJ}oCht;g%;*O z3r^a2HncFyt6P~fp~z$5&_o2Z(E@oa_<7+|tOJ&7;jJr?`)Zhr9%n3hE>=qtS@xUo zh=IRmTyY|g*%?>18DmCVSbH^Q)UlQ~m@(rO#*CL4Gfp$Myv&#}Nozkai}8Z>g7?qB z=Lf@c4_~vFX9BeYNBZm?{}42A6gpwv)E*h?z#kbWzF?dXS?UYMhc6f(x-(|9f)BN4 zj5tNy_KXpyu^(n!XwNut2DiZbo_Uyz6C#7jS!Nk0WQ-8Gb#-_PCBul^%3CIB6OgUr z`sy-Rd&(#Cl|zgXqu{llfI}^L3(SiKu0Me5N$UG3yzi;ruQ>k&K5tE|b-z9Fg5p1K zPyE5XW8y`4{{7DPHoofq-;Hm&qsCRbBggG<*Tu!d{}Y@)GEUst`(Edd%;|cQe$1vH z;WPA8M2z}&5NvdG(5(0 z5_-P{S!Fb9i?t^l$ZB69t9^y6_7$?)SIBCokkw8htDQntJB6%v8d>c$vf626wbM#g z`y;Z=7d`G%vf6%RHIdt%?>@!3-;mY*r{_jLS#AGS$ZDzZ@1IZYS3U)q?RI3fJCN0$ zkJWa*QRFYHWx&TpRy(;?=i{t7pV)JO}4N3v!SO*>pq`z5k|a|7!)gIT}n$NEhN)^Bp@ zGj`T*j?iaB4wAJDnfDJ3Yt8yi^od369bT(NBlAS$rViN_+y9xk6>;cotp=a_Y|*Wm zi{@^KcdYuN)4JIHdkKqX{e`gSjj*!#qXEO)PVy#2+D~q43heZ<>t1T;I&fUTrmb&V5^1HU3w(oX2$E^fLYV zWwk~l`hjKMr>XXdJdsWv$=S%Zlu_E%x$K&bH)!o<3V%6FdBr|LdBr|Td9hD@PI=|~ z3(AWuJxAPMQReoPS$JRv$|~#6L$tQ7JF-S_n(`i@yk}Tvk@70J`9DzJfAm`Ays@Q} z_Y~zlMR`wA-c#Q4%6fv7_cXjp@}FqykUPmf_u|n<9*PZL(O1^DD~7fd9u=l|6l(|6 zOY8CvSVMSQ))1hP#_lwn7?jG`H5qv(#e2UYF2-@1vwmr;RnM0-2dmx^WL`gAWU_xH zXLUsPDjQ0hw}hdKY+k}vxnB|WpyR>=+zE+#0R7CDogYnSpL`#6pYc==^s+{9*ZcnY z?YNJ=o%?0krmjfRYMk@=x+CK+X|4V?;0*4-eGBli>Pe{!m$waCGd{OroNmi_-H!E- z_UPXU=gx)i4z!1~a<}>HSgmR#FytNA^Wj}s6Gqq3zVHi`_ul)_&+tF~LPX`U7g|)F z&DE;TE$1BBzLpn`zt9T5NZx^puB@A*)uJ!)RF^ZdZ^>D3;L3UJhmaZKv~K9cdF;t3 zJ9qSK5vL>3X)gCUdUVwdYe0Ni=YLGTW6%X$tdpv>?HD)HTt>APxKnkq5hq;!))4P7V^?e^xs72a&mYW z?ZjKQt_Z?g5U(Ze#Qm6kwu&eR=b?}_5;kr4&iZKPQe^w(gYe zUwn)Izp>d9R^^ht{vMQFVE!IDkox+RaLMaq>}*vZyd?x5?Ni;4vqwEVZv17BRrdO& zZLBa}GGM$<967(VANv+R7}qKoKfzzl2E2wXYj;ZKSAOXHQ8qe%l-{K@T zKa?%^{2J;!5q&2WG5XoplVff0z%ba^I@@26X?+-MWc~q|JKZHwL|oa?aGo`Mvi{{+5-nwR;aot;|^> zf5AOGU?d#8p<-a7BOxJ5XuR7AN1vNkEwZJpvfCD{D(mA*8%5f_qod>Q#o{3~FCEC~vI>9*kHT*%zP`0RY<^yU!i1UJJF7|=eAU8RX z&9>RJW%;&kv9s9+T7$f_9eHV66n;bTV;|@=a*OOa9)w;7CbDjJFKbNbl2&^!oWm?* ztY#l**ILHu>lx=0aX-U&zMi=a?gP&;rhfx2H{kwb&YtmN@6XrmcbDn*fl}Q*xKg)2 zyhpb`TC3ZCy-&CQ_JD35`ma6Xzok9R{k;#+rM^byIGwb#U$bXccu&;U7}j^rBQtbQ zI?uZCd9|uI-JyAcrCE@HlRdc@hHNyU0KRlZ7p@dW4 z7YOTt+$`q?yztu*o=iA2HkzxXwFd>-MLo|y3Z$UcKAx*WVFq0?EdDuHpA`@CbUeO361@W;aTI{yx|Uw*9LWcXYP{B9C5 zUvj>j!=KY$-cLiPAnOVl)GhsH*B;s_owG29!^@-}?b=IwWl-PkjJ?gdw$|^T&RO*^ z+vVl#W$wE6(!TN72l{L19im;YW8P<@T?dFwyABbXb{&c>>k4bZe>iU0|73rseJlFB ztRu+Y=g=^H|8pB(6Idf5ynZ z2KH?jIeYUYbCaMGocVf)_k3PMIF~1q+T#b1O zlb$_s=jy_BFLh^rf1Y{%AR~ z{%SHZ6N6hcFvrw9|*-sYi|eTE@MBHCot-YuQJQSkZ3EWPTWd)xF$*#0kUuN?O{#ox zP;%uDtYiMbT@zWyL}&j%9*D0LzEJiHbS(8O8_%9*<&1k`w;nG**L-eNW%Z!4@yF0t z8Hry_2KPMZ2W76QwXZO)ap&Yjc1WEEJ<)lvRp~tF${J>NcvmHZ_l!EEWbol zOeM>#1NLslJ6&I;Zw+%he&e87>0j10Op)DnKXeqa<{@(UefVX|8s-Vk8gn=09Csl^ z4{7y!^d2As9!(KBkbOv_pKc&;V+}K)3~pV+?5$*)?w%4==iNO;*yb8$f$~HCS2p*5 zlr3wR4d4D!@5AjgOm{Dbzd!aUwoI(Q;59k3>`%ONxp zh#pvX9`YZehd&O`>7hSm5ZQJeu!jbqhh9PGp=AJi;4Dj%=wZ1*4;pzll^&in(wu6m z)8V(+COw=`ew{tXl-c4>sm$VaUMY$iNZEz`O;snz`C~(O1De#KOdit-w-a z($jhBE|uJ|-ov=BuLn5BAh#t|x9>3QG2(6Ky+D~)u}<)zoc%p^9WpPntH@NG8>$dJ z52Ck$bpRzdhz<`K|6I0~r{(<6Al?(@eVL>bY6(8ew-Z`=&o;|39L2D9@KdV)W-V3=;A#ZJrChi#eFt{jwD|z!i8+@Rv zCk~tS1kW+@H+5}HL#C!)^tsW7krmcCRXB4hRj2X8EBbJEyD$Cb^6Dwnc`9{3nKnp) zKToQxV~#wwt(r$2ka?8MLk*a_`kWeG(Mjbm^Y6fEyOCCvnRj8m=^UdfT97VS8ud;a z_u>*TamaPdanoe&Mf{ERp^hq!=0G0N@2aVLY!Q58NP`-?!T=%{Qg)yZdpf- zZ*gB-`Ud5*QO+>R8xD=}?oha`0v(L=%ESh>@ey-@spvPTW&6tX{ z=$jRK`~~r(O{7i_DVUu-e^quT&xgwH3f)bn4O3{#NwjG)H2X(&UdbM%y;7HAI~b^) zMK__$qt8o!L9UO*kEZ;{XO{OHXUe!Cnhv{6D&FbYwkP@R<>xwbM?&!#r$J zm}5*E(g{7-(M=MC`?D}Dmvdri+X-vIx8PlduEH?fZxd!CY*Dmh%#W>oVL#xOFuDIJ z{ic2$Q7>v;%uLkv9+Ryva3@chVv+E&50!{?pWv ze1}n)=f z!RogD+lMj^N&IbLw%oOx|CK$v6l~=;lD7o7YcF%Fgg$?bjV{|rIil}!;3wg((S)Nv zS`Xh#_ z5wfNvX{S- zZN&PFXWKC93RrO?v}I3IS3<{XjIobHpOXH*rHny_|7I<_Z-vm3H{NSwE6!KcseF$D z-wv-4dKdoshT;D&{I`>@)Thu@(ZhQDkMWoF?lY~n#lDYQ(g^HM)Q^n!V*gS3ck%2~ zc2{}RfOrkx*gV#+inEP2cu8Pcq%J)ApuA--I`l4azC!NS$eV(qTSo5MRJUqZS+&|x z^?HG$>Xln2vDVp&@mut(i9XR-=2ksCE9&YV(;~H`_%*L@jbo2}E&XU`uA^!f>!*?T z{@>O*bb89$ufOIlam_vYcRJs{;;!&}#`j_RjOT^v2`{4;deEBKp6T~^wl2D-a%;ae zFK_L;rgCfiCEks&X{RHxwJ68MY{Q;r*uqz?|Nh>OUJAeP>W;DGa|`#Chhxvg9*kYi z84m7CM^;R)ietU(n-{|?zkR4x<*A7+D-Yk!yA-?$aDw~pU%wm1d`YW)8ami}O1~-5*;sSgUPIdkEc9kGxseP;R(Xkj$Bvch!0NI?{bZ znq+CKsmMaNIBE~|)Me${#IFFq<&N5ehTl!%my2JqqxR2+-(2xqir*YZ?YoBGO!3RW zZ>FPmui-aE{9Kglc1P{;1(d&lJNmpQSd)tzIj&|Y_aw#6;N2myU4)CBLYwiPpx_3* z^R3kmq~D0Yz!7+Ik76G@KAv?Y!Pkt(eUAq5*3&N21RiDBF7Os;wOu8i1y%-iJx$(L zA}@J&Ny;Yi%81{(Ufgi{q?A*_BI<<+zpXb0AK=yciazKqkTLkpXybh`%E~)X)EV-h z*wh6N4O7>S+LK~aSKo*|)lvJA*vPB*!&m$AeuO4EL|CVeWSv^>*jGb4$Y*NrE}F0z zHumm#t8E|rCu3X--bDK}K4(zzgF_rNC+OfEw+h;o{ghi+^e4Df0eu z{ADlX3H++TU-(^-;kQNn=rbZW4K(=XL*hr7C7(0xwc3yR3SIFYqS%yun%LkUiM^vA z?|}4l)OtG6#wjV4gLr>t*VKUtYP|$~v?r;f>U{0;iA4BZ?hOfFL+4-Lqt%{%-&S?% zFSe@VW3<}i!=e50TJ4N`VzwSn2A5Rr^3TZocw$s#9`t%*XmsVpPGMEG-NLHACBJWb zhE-jR3#;h|fV0$Ab?jBzXcYbD4bGhbhxue}bqDS8@tt9nCteP(JP#fhU$j-# z{@Pac?Qd;W7hkni{V)CN+myJ3n)MoYq3IKwq<_S0ttsbSj-Ixvx;R_aH1Mby(6KU} zwVfz=vupr*=Tm?)mHV*hk*=8utU0zS=9!x#;bY(Q3*(+_nCLM)fL4#HV}rt~jzd$& z2ZQe$qYKVN1V#fj-Gu zTs2-v`|N~2wqUG%W?$IBqy0pu3vCDea55GZ4euU=Q!cYuS)1TwaZTq z)hXSq=eli+b+tkk6+Q~z`&|^gxR`njBR}j>22L`r1uaLugZ|jb^8#rF-*5x(Gr&94 zLK}5%onB7Wo|Ln1y}Fh*J1cE=G+eJo`l|2|rK=UZzXJBDC+Z#(UfGYa0hugGa8&TH z+X3%$gFa+$U$q5kluQB+UqJU?!f(DpHhF&)?N6H=KJTd7%R0xTn3gKf0njlr*0S@+ zSW>@2t5T){$|Uge#Rjijv1#WVu@^XMmx>MjW{C~X8OrYENmq7vj|&^!JKaLcUF)Ni z+!aXAsmQWFFt3w6-8a`es;cOp)%eX+d7%G8*|yEFQGF~yjRC)-rAr#|BK&n6|He~3JKGA@X&c+s8PgZAjAG=@A399Z#V3&yxmcikDRuH@Ao>OyYEG<>ZJvas>jy6w6*5P zdq0ZD?;+&38VQflcE87%7uoW{t0&)RS^3Ryt+r-(MCA$e3LhT?e{^Y8zk|=wskcYJ(OGS@4pWY zaaz|C&oBpJJ+8C($@e~yb;desyNV1(U4YjsdyKe8We!6byNcXJT}W7!5f)C^vG88% z{mWycsQ;eGW9+|*tRwM$ZNx)_=VH%v)UFmASpT|IT{qe9BtQa@q3%(lxavkCoJWVT%vnQfOLzv~|Bb6%HyxSzU(_ki*5>FI#K(C=;d ziR>hDQX9kXCj5j4RKMt`inr->J{Lc!V<}s9Kkb#9Imzk5Vv#2y1r zWVEQ*PP=-BPp;fKHl^~{DU&LX4sg^iW1dvmJ~07$+H8+czPt$eXIW~y$~Tt|O*q`s zQJa2`R{QJ#dqUy(R+T5Aje=y(kt6fSIp~7asLIcwts(bDY+aTdU0GF}kZ>42`Sj9( z3FxC9_RRV&uf3hp^|dw0Z7v)|Za9YA@a?Fsl{6oda|F4k4*6rvJ1s5@y0GS>!vlIJeEnkk%ITy%hU`<@iSTZ=s_Ecy?AIMCv$+=% zbx*|Bnio1&){X_%MC270WjExXBgj5=$S*acY!z{*+h3@L-bFtIm1H zu!GNXmah7d8I_XPyFW+fW$l{v@K?^kSA{>{X~2+iQ~V^I&{m{x9^j>`mLfCIA6QeV zeFnJ1tEyRJOOXC8vWUn_hiOwa?$fVY0c+-?$RX&jK1ZIyJ0}_WiJUPCoW~ouHkxOF zo~OiDdAfXg9)Ye@zMUjrbRUxMSN$BdiIOj64Fatp=tlUCSN;J{Rd3iI0N+pk5g2OR<~kp80FymKmauiT3w;o)zy zU!hf8|6e*EM9%_yDevs~p{3sTENFL@v#FN~m}3PwpDJrtUm~A>&)U@m)~@7?ndow0 zkWzZ4-Yl4uaH%DGZJbSgfVUv+#(V8*4)-j2jPL2WRn~GYt!a~RoV!2q^8F%bRiBNl z6j|^a(n`NQq3j->W6JL2DYxr<=oeo99p7Fz)@d(%*+m+%0}kL4s<^C zQo#)08P$E-!SCnT*oz8dzf-G_xo79}$+exskx_&m7WS*S130qgAobf1x5SCG^{Zfw zua>pjUFNpPVc9dVo@r(egtB(_un(|B;WuqaMUuFY*JNJsTkI_ui8FwD zk@4D1xU_i&_MPNG&NJW&zuRs2&B9Oi6lCtA-cKY<-pLrD=%c%5Ea8&>7|Eag zn|H`xc)H*#Wt4Qoj5vYb6RBIT@2^OF=D@1#gNZMB3LKI99yV}Ez+d!>J;#2Ejt_59 zflo(+#=EP0x_i1}e-^EsuIA2x!2QsG+XH{8w@JXf$AHNKfXeH+XcH#qNq~8oaFH9s z70hlPo4{o6N%Wxz%-lo2eoMI2+1J>gb<|Fmozr3N zFko`0N#VH;mm|ZD5x`PBy^k=F%{RjR+E6)IPf&uey z_$fN-YQogt%YIA2HhcSxzL8V6KJuXG&=R^o17B1$m>@C-<4Sley1>zE zs^FnZT-lrj#y$gIeWU~LyN5E6s^(gHsh>L5SnejQ~GBXY?7PsX7)r24n_^JaCQJLay z$$ntNctP=cd|1V^A@Fis0bbdwx_1;=l6x`WBX6zvB()}PhXE*(Ix7CpL`qlB+?1oK=g3x5@WX<*uquiKvkCFdu~LdTBYA&pvvW z6F(pJ^CxT8IBI!g>_pAF`=7iN9`BJgqBGn9`yRJyqt&dttX%b4%DK8xYpCxq#-+Ez z!cRQQ8P#uwJ1ZZ!Cw%J;jL>=CiaRdWd|#K=-`1@>8r>J)0{c2!yYi^;mhZQ+bt;bx zZ*@ZJH|3(%FZJTOJ~J+^tC)WAnTlx_r!g+ow02ZPCSPCCV&#n&=r_2>RkZZF+f=l| z{q-mf*;w=~a+Z_1oUGmcz?s{%1L__VeTp@dQzIU8jMLe+ZMWriDUUstl2j4D^2XQD z(^4V(k>~+Gk<56Fj?coA`ZF zbj<<%w2(=lgrUWjv92y^JGr??dLK zQfI6WZuy#UwWdVdzs+|I-?IK$P2YHf@6Y)@PI}4v2iBwF$y@3~!aqr5%@Fe_M%J2k zFvcOfc9yUj+V`K=>u+2hgGO_3&%m4)8M`0wg=Xn@a!%kZ=cqf8m#hKGcMR^c10+rt zb&i1d^BALhc_eSGMI^fMI7@|})bE$woRvR7Co;ZHA#Y^y4^6-9R*r{{Enpns?KRE~ zF58=+-_NxPUtwHLWZmyLv^yiYO~njb+wv2*bD`s0++RcA4?^D$;ywvY$~gNi^!k5% zwI;5*s zx$CFxw=%cv5EdR&-Zz@JmLjzd_H}E=X@uL?tsA$DxI+l*&{?aDiPUQQv_VfEVOkk+ z%GQmWNBCbBw%^(zR;z4JdW|$1>Hm#vccfng?=?lJ{!nwU?lDu2_$~|@h|`jO%bcz_ zETQ6i`rl$+pY!gzd0J zsxaD)H^Ew$M|DZ9t+YkGm_YofE-h%|SXpcP@H}IJ$P4!i{n75?C*K>?w_1<;@F?p# zWqixssB^T%dD>-VyU6krFFGr~f&MN)bGN5DUi)aH;{tl`IZG4m5uRL)yJRG1CP*vs zBDK!tt!X<6KNIGtD!HfA)_eu9~iR|}7JM(ge@m9G}KRLs+ z+LvgP2VS1y^?TsumbD2Vr%>lndcA%eUVDsjgSxKz`WAHUuZ+I{oDS4CXGMMZSNEgt zm>X}dhnI6;&t+tF%^lQ(q5Dy~W!i%j3a) z$I8_gB00Y<<5(o_9o!p!4?7zhU6vI+k#m#Lb9%lf?Fc;xU7Uq)$XXa{LR-$P5r1=E zvkPNAJP-aO^Cq2!y2$ztGzBfu4`h#A#@ueu<2`&!+Iz)6MfdN7KkrN7-`TT5{HN;v zau;=(@{jctiT_O9KNkO7zQ6CM=P7R4>y|tuO(x%Gsav6Gahqd)7s7tRH+*ZeHf}%# z=l+y@s${H0J$>~i$7Su4^m`)@ef6Quw)B@;WP0zz^i#DiNqbkf3d^r4wpGfyqVPHI zTE{0HD~_=jFcv;1d|&wW*^g2xRUJ|{6UnQZdK)h`^)go3T|8ry-PyyNZ1~4|MqqEh zT-Pm%Iv1XK*YbNGK0%)oe&^+n$y&uX@Xej@&7HWv#r*;N^aI={>7ysHg?GllGy8)J z{H>~ty6JnzuB{Gu-1eV#*xC<%7!j_OPln%J3qN$g>oO!9e)kM?lV+IgIb6`q_cxUAJGc48jHe1LIr&bs<9-1oP+ zE*zv^oL=j?fb8(fY2eLkrR_chKV7!g^;#L>O~=)Bp>bTvD-f>aJ&%3CW$%z5?k49) z+(z>wJYarOXJ_hOkvi&do_u+8stk9N^CfPh`4S!|-(OItZJ|rn77s)&OO!Vit0HF` zZz}e1tXc$r(4aFpvu@ReAoDoYPZqVPxP|n%Uy*oL-2@8e&Y4wf)@jR*4M6uB>woqx z+hXN=A9KS)#|!#G>6zs)1-$F;^CWm7*-8E3t=X^e4#A7c*Zk!!PX@4@{U?!~x~9p`7a zYB?{?)^cb{7tWxfPa@9NGxtbmE$wW3t+u3C$&#DDurAZyL0UtW+}tO! zs|tJ92)l_ekz0`^H}|=}YsL4h>q^?UjquqIUmFMe2m-^E5nw)KZ$VRSpuV) zacKhIOZir^13c|UzB5#Kc%|U~H{MUev+0w z!7W-i<5jp;3!lP|y8zfV?e#H2#;f<0e`n91#lMwr?D-?#Nt8E=`go5uFmpYbwyjn4 z64R~1MI5OUkq_9X+#)pas@UYsekHctb(ujwm`;C~M!%R!|CoYqu2k00!_*qOq?dXW z{lW*xC&r!3+CkhQ<)#h1?v56BSBpE=;_hv6_qDjMv$&CwyfC2&uNzwMx@m8(dy>UH z#p0e}aXT&U`4;!h7WZuy_nj8^VvBo;#eJ8>?Y6j!EbhB4?o}4|y%zU6i+iKR{WFXE zzb)>EE$&Ax?k6qoUs~MHSls_(aeFN8-&owgv$$WkxZkq4_gdWhE$%;A+#gunyzA&~ zuSYHJCoS$@THMcA-2Y>7|H|V2jm7;ti~Dtp`z?!muf@IJ;{KDx{ei{(p~Zd3;{MQ5 zu0t00KP>KlTii!1?k_Fw;}-X~7Wa1+_c@FEqQ(8A#T{nBjW?vc^~}43UUxf-yQ9V3 z)#8q|xO-dNeJ$?mEbf68_YD^JFpGPn#eJj2J;CChWN}ZixMx`0PK$fK#eK8IeVfI7 zr|#D3{SL#w`b?Wtt9X}3k7L*De>IH?pA{l}Ziw)-5aA0$gx?V&d{Kz-%n;$(A;R-Q zgcpVgF9{L8B1HI4Lxh)w2;UGQ{QeN(4~7We93p&6i15clgg+G`e0zxS=R$<<2oe5L zi11w@!e0py{#uCeH$#N)2@(E>5aI8I2>)}4@B<;j{~99vlMvybh6t|?5&n6I@R|_e zUxx@k6(aom5aAa>gkK5~uB9~2zav6~M}`P*6Cyk&M0n>A;oU=o_X-hi4-tNCi17Fj z;n#-<9}*%wDMa|_5aHuOggY7yKhS5g3O^7Nczi$5XIhBxSs}vbh6qm!5xy`)_#Gj_ z7ljDV3=y6kB0Midcwva}k`Uo5LWKV`M0i<<@C_lt?++3FV2JR|A;PzW2!A|8_){Um zw}%LSE=2f_5aBO{2;UVV{FM;luZ0MIGer2F5aE9a5&m9?@IQwLKM*4PuOY%e2@(Ej zi16wV;h%>HuL%+Ub%^j&A;P~85q=>=_@xlx+GH93gY)-@5aE#_!rO!hj|mapIYfB( z;Nfjp_>N)D`r5y2+U5Ig+NICW)wTJzO?wS@;aBJCB4x7^cOLFfiL)K|X52NO*|eu{ zKZyGe#J?Z+F5L0N-+=oi+;0-65cdJxEAY?5{b$^54ik?16z)jeJ8*xEI|BC>+%@7) z{LQ#O$NfF-vAC;o9{}bk+@Iop6aVSBKfx{Kh@>2+KDD8y#-_EYIafE1xTiL;({%h? z-38oVjS|oL` z1;2$yb$FX`r{OLG#zVMQ;JyPG<+w|5SA*Ab+=aL$pJLp3xEJCs!kvvfiTG~ZnGzp& zKJG=htxF2*b*dld+5LVt<4eN*!uA*eQaSQ_UAOjd*g$@gPSUWirAiZDHF#3rFeq=rM|}xZL@ED|LGkkNZ4iD#qjejmPeLEFKgp&_ifCHjy0XX=$h4I zaZlGZcf@_#=kNUv_5WzVf`N0dpMArOA%7ZX3yX-1u{+94TidTgc9_U$oZ?qMz2wLC zy)e&eM!)jtbgOyMGjM{{Naaa+1n2(KAD-}T(jP{?Gdg*!4pY+>;mF1mVpd=rm}!`~ zn1vWSCLS{cGa4g5K~4G$=cP6D(bPp@qA{&85ttU3mY7x;4TCP1I^H_13&*suj$&pTkrnY55IA2QsQ-S18x{GZv4o>qb5ugS`ivTSBPpcAup*X=_}|i z(KsKWao0s_kLiHvgz1LqiHXDX!wkX<$4KL(U>0C9FvXblm}fEjFh?tgftwzdXHGF#ei=;ElD`k~(O8KO0JurPRLovyibj(`JI~bb{H^v4cHh|aw z!Z~uP9|Lo-L0C2j%Vx(wbT$WOk=Eb`2rn*DCbJwS{wChid&CHiW}BGpVz!OhKGH^@ zpp2A7OiYK4T4bxJmeDO*N3;oV8`jR&UR(NQ)Lr-eYUvwAXWZj%JUi$4HS^oedoa7j z)))W2KCZ|ARp0*j;Ga#Zm^^CPUw+uL`}u$FK6d-wo5!5)aQfEg)=hl)Z+G^3@o`V9 zPi(iZ@A%ezL$BR;Pn(-ce)3k)k1zkb?1h$}ZY?>z^__Qewtl{G>RTuG|KgqYe}1~n zwFTE7OZ03gnel1mt~>s5f9>7#=S_G#fB(=O?+pmw_3oNm-#*j2djHgl7ydRc<4-M@ zCM0yeFZ$I1qxfA_Av@A{&LpRRfPnVCyscNCV- z-23djymx!lJbdulmQNhIZ$!KA3%B1D)AgO_dPL@p>OE`uv;n){+?Dj(HoeBo@Oi8m$Gz08PZ20(9Bluh*dh{H<|cGVJQGOf&r zWguqyn|LlVf@N%wp9%B_#!UHL)U7(LqaJSJWzvu4(EWwC$xnEj{LJVuc3Z2&dJYgCmAF3pKu6OyL;r@ ztZ%W6vg{Rq2~c&!KNZIaRBrjF94e;aG}^)8%ah(dQM2n&r5*^)1I4{Z_6<-pjb*)V%12%QUxX8Bb`3K9n1 zlW@wYby*T2np26!V97I|9ZX-0h0Hq+?|_Bg~A z7-1M`B?*_lE^QWpOa4b_gG=%Y2WJ*1CFU#}mRMYznYgs5ptLY6GciB^hK1^zHqr>S_ZG=Ui)_WU_F39Pxv)91Su+%kRYtead@Zdx#Xj?)r% z_MBALE%T?RrMgmPCC{EKS*N5~!l%wpo;%Go`KH-(byOEx!cv{hD9q5t5;j&EqitNY zjkmUD&!!5K(-vqY)Nmr*p@gbV96nsjEG^8-$)KskGia;comG^+G|S$<*q)K@&dkXq ze)0JJd3mLL=j9pBl7ieUx8W`-SZ=zqmX+odWx1qnp#;M>KgT`3f2ln`eFavuHaB^} z0&Pn2^jUr=WEA8Vre~DcvkOWRp+E=(1hQ5XW)+o;A7a`(lZyQXXM(2?t?N&$SsLPiKC2UuKs!tY9ztXVUj>n(OnBl z-B70VRQj4LeKDzT7_-z~B)vr(c{%wxCAxdC@^C{KId1xso~qO>thpqM?g|heRZDkH z%*ZP!&dOvp!K5xBwZtV*xD?DwfL;$Xnn{ELM6>h=Us9{5nM#kHT~Jh#sJNFGY_ZSJ zBx@fe$i~`>bMl4X<&@YBva;v6OR|<0rI+LsxTV63UaQ!ZrAH6JLobzh8uD=HqtH#10f?d2U|?#pG=k54R$5tmbb88erNIVvOS+)QC1{E!v>hm2k!bSPtl@#)uSOQ6<7g|9s?eI-<@laV3Hh%iuU z$5$*<77|xsuJ{W)=Cf;=GL%s1H{(l z@J6Hl=lOoCKD$}6&XHCJ`^8sZ&2OGf_byLkI<{6wHFo@Wa_-eXi!W4 zF(|;-A%YPB!$k=3K;fA*WXwpdXtB0@u{M3$7XQ1Va|%Iykac~m>F|Zn*c|}G6AWw2YqovF4a4;k(wtee zXa>T2j(e%SFuf=}KdU6GC?Ht)Y|bj`CurT=ol`9R1vG`BkRy=XOckvDCIc`Tj~B1W zSy_b(va)jRHzx(eEyrDmc<5`9C5R$=0hi~LWaqe5dk-I-l&Cm}ndjot%%xc+h*=iY z{V2q2PaTvl)cG*?BGtSi=)iPDmKGu5df`|@3|45;E7GJsuo`6QE2@|H8UYgk=Ky(m zYrQ0OVF?tZu~^|kddN!8cQKfkxJruB-NiXs?h-Azun;MJwrP(DwhmO7o>Sy1UY@fw zuT(7;2}A`-uUS`NPGMGFj$7uj!0%sbnV@**9nQJ4l2cM$^XJS>Rci~`=vZ;|ynRSM#R7*LjcICrR(?Uz%KpXDTZ$0rbMl!C-!N>6 zeJXrmkXP+U+{tRmWJN?q<46 z$1c8l(1X3GHVtXP&#n1KE*%`PqF{kq$r3Qo2X6DGA~0La3C8!d7%Cf0iWi;48gI3im8FrA!s3Aq>sJFyylg3WD=)!9Q_eZA`Kb%0Pfm5F+^X5*jir>#1hd97rD0_i z_(n0nzdqUsCH zO6rvXoS}8`N_%4BvQie1OIA`&3wB6LWIM*|J>k`KG_GL+lA{`|b zW$QyW6>Ufn*V~KIm(!F5MS5IwpuQz}zLUM08)f;?*bgeAn$wH3WR*h*qr_fcDiQCD zbXs9~Hmr%c4s|U8jZE3R#4=Wi$Jr|vBaIq^Xi-*vdJcp7Vx7eqbm&{$_oh|yhz>U^ zbF4lMtWT`0j@Xx{7v+~0+S4ghac+(fy&p`22Bp!xO=U#TnX1gQjNLCC3u--I04s9z zgSEJzAUBr@gOt;dj%DVd!?A9IO-o+jN=tPvm@{7|FdcA`-9kC~Ef3cN-tUo*)si;inouq7pE`Dvd7azh-UWbsi~=nqedhp zNdNF<=7nigU1m|vk`gFVx&$3zX|X*!1L0q~=6Gi0?t-FxVM|L^%2)2)NI%O(qL;6t zg2F5plGZR+Ucpl3BgnV}3klY1(}f=!$aq9+1#Yci$&vttk)8xQP>XeXbIB|=+MHb= zr>R9sb1Gqq@y7nNH5fv4>x-U3QcGB`_ouf&NzBQJK`BA99y-#Bcmp`};pS#dk|t%H z1GT6jNT<{KSrgpctR+d4vR*dY#aeyi`I~W?i-b2_(=bVD0wzYBW@BQeYbqu)L)J-> zOU1bYO!Rb(VWKZ22SLFX*u=ao2^!0^*wP1Gfhn?nlqMq3AX#H*u$stQ9oKU zYM?>VP$(fAHP!@rXfqHn;5EwqRFw(j;7y>1HaFi83}5m!D@ZR`$R;qOH3O6SIfSeV zYXb6PgQ&PkczNSqA#%R-jS)#^@!gzA6A9jhbb{crlto3e;R*^xq+nvAkDTljWoH#J zhnRV@%$1j_)oo)XlEr?n0IEv>YRGK@1bsZVruPaiI&y`h1vMrp^sZH47jQTY7PX0$ z7c_aux-;e}F2u_8d`?oh%p-mcMGloou=veEG$@Ra*iw;8s82OXYOE&f$7>FD4bnCR zD{sWkpkgf)y%$&Q`a%{ticeCon1QQSygJd}2=Td7Sau{Cv zMIhn0VWa>R6^I}MBy18gArVm#nu~;lwFm`y}2Sn80tWwU$dLrx9jawsbZqfyy3lB7ApM zC%fj)USMp^PoJAQOLpO9uWEXVxn?K*M87n`wFlRV6~#e{7%*Odas}(rfzt;pkkOSJ zrckF9Ad~;b`U)Nes43nw2VY;Bt4G@c>k6vbNX<=z++i*`Gn%H7|sw z;b(SSGg*_mhZ*^bu&ke>uf+rG3K-R+Hnm5(%%rMk`A9T4&W)#QKKdcxHKEE42u&o5 zK(VhFYrj03QtOP(Ma{4C#!@vO_eP;Lp}vEr=lY4Oz@4|!C1*`n3>%VUnUB@uP_u{H zg-Ho0xx$ht$$a(KfKT~z_U0fU+tVBv@QzZA;h~~lB|6@W{3dNipYubUm{-|$0->qx zSFYXoM=ntc*SO(%o^;gHy^Y6^~N@HEZGxe-{+!`>`(lh8IJUlqPyAWcPdug)&tU9M9mV0s8ur;Gx}8`>jl#Ubj?|x4B+(!u;55HlBoHpHwLVk1uzTVG|bJ| zSBJ6}N)vE}Zgq_}W}=3RN_x0oOK|^>C0@COt*G9A%2yBMmB!`LrSC*!kQ{;`CAS1} zGMa~*H@~2X4YCMOAFNaQW9S#Pe$^tH>Bltm+X{gWUDl5hfFJ@%<_(fHfdbSy*k%qH z!P5jlw1KS7L2iAvf>s93;OfhtTvaN~mt)@gCC(!KM6~sAbHiOkW2(>yHq9%)x1pMn zl^OX;N-Q_HIWPX7D6tofCKOd)F=!TF4VuI^oo=uK6twn&1M@C!Hvs0}uY3^9atCLu z4dY%N&Ss9j1`md;+S6*%#+P3tAK*q0b~l9XD_4LuL-k>ZNc z^vm~ZaOQvymn|}l{potkgnC$_>;>BA7tBqy>l9=?yf)Tes$cu&TA7L`S8WV_8VZ`X zn^_t^AUvJz8RQ$HN?qwoP@!Wi*VMy64@xX7D9%xr2UMA4;Pcg>1r-akZ8DsO736=q zuee}IiCo<(EI<=d!_KYBW{uMv0`W`dt=j-(ED^bAu_B0=Z4GW>{(ecCnsvZK5mrd9 zq$|jtM2#W9NWl#(b%IRJrMu*kYk-Cb6s?KK1Wei#l=8EDyds3Wi5qJqRR;64?DDLfrP=5rk+X`ya0?W-Ip_sU9}+{kGKM0s5*H&! zF6zVF!O&|Eu{pRkOx$$Dno@UO_}0i*grFaQD??J9HVbsfOp!!DRP`iRjj#cxDQH{E z*+`RHqcsyfzogAYi4C7Z(UkBJVY*fTMf881~|vrTGQiCic!*ywZBGks=s`fK}FTXlR%m zl^s`;JISv!YoxHkywc*U$=(EUr6?>b;?`?}E3i$jdr2NJYhxhxtL*?i$}ilWk(bW> z`71=nNF5NHCXfN=l+l2ElqYa@Ckw@)MW~Q4R~}U) zpS}i{et$^R`dTGSO9VM^D4Kq93X+--g-M4ffO9Qfnx7Xey3!uvlG%`&F6tWdLS&(* zk*TE-F=$3!xYoFSRp31lEl2qES&4tq)GSJ7P@=cfmCh{6S~q$PvmmaPtVKTNHd}$) zB})Xwqni=cRkRqQZ8B)+FcZ}tO;5n(FcC$YoxsJSNoj^ZK}f9%M|Ty&7}Z41zshul zA1;1!d%y46b2RCMTd0Bt$%Bn-V>$g zq23fI)-XB9uP$9Q7D*2?89=Tj{fOZ#swpKbYIVj{S}2;iqh63p%Iw`u z3XoQh;3{5fEkk`EeqBoeC$$+7Yum7g!<_?uK(rbDbC(}4E)(>(-F|q`G$tTDH}9i z!_pbeCtcN#R)WRkP@$?kzm^UbS+C~CYeh#uaYLh3?#}WKri~O!XP~|$dRtsUUd?O7 zfat;zON&F%QDZ3slTiJGclw1+Ll}^%sUIOpElpY@$Sn@Pt!>}_vH~FCYIn;LsbMxj zW2q&`dJIZ71?z)sfLE|cfdESGN>Q~Oi?J3?fT#c&_#0&5MLK9&g=KI6BLHuymIfI} zctF@A0|Y3U+K>rMi9u0>St5VQ-`l;EH~q`)LY@$opPw)59{Na@3K_Bc_<3D%o}GfoH94bE+L7tZc0O075bIYM1!g_eSalds~1Cb zv!VM!#k2xyc>Lc`er?X0M6I<~wS?bfCuq_D+RH4c2ThipE(*KsQ|I1fs*%q^iB`kt z`r+xsth=+^hNk-fkql*MdSUbe>op@9*_3))L#u`n(TB5N8Lgq{!b5ASC|?z2tG+Ko z>#ZJUq;51XQJWYjmnB767O;(GV&EkQe<=?{g?7C{9+vU#J;*T?b-h^Tf@s>y$O7q1 z8WZ(vy5eXlPMejYhfGq+w-#OX;4d z?_*;XeR={xUt{nCjD~H|(`KnP6u((FDCH7q+FV!a&8f~b*VN>+)CEDXPMa%l7a#=a zD4RfiWWdQI7N>|N6+~%@l=8}wY|)3S>*OOJR9?QG(mWD~PQXMaXpo$> zhuh(HqucA#=q1L}N1QgLh?^g&f z01{d!#wFa0L?s`y1cP;%Xz8jrr4^STIve#*;NqO6I%H8CTx?I7dsCw5D3^0Yi&2G# zvQHn$T3X(GJ4e$mFQA2*R~)Dmzhat!^*mR)GqQ^c+{}J08VzTs&Ym;>R65pYI$p5 zaaN|Pw?MoC;$nf{sDHm6Q$MlH1Vy1gDMX8>($T4_i`8|2TNO~6H?z~`%uIEIy5$xBGpgb)Jhi3P{C?hhZ&%r8XJ!q;1nQz z=#p8Aqqn|aMf1v2VRnd$W}hl}T{i#_8Y3V3FrGULP51S4X=0mDgDyqWYBf}|KxxxER)`s5nR?#952!vg$*9Ccfy;cz9r8{I zi_dm+{HEKdF!rl4#G>VXET?DTrTU10s$^sa>On*qbS9%c*_gFNYYZ8Qg-Fj?n$e1h zfL8k^>7^wFXt2%6s2`$pLa%COU-kxiK3)4(@!uZbzc|4>5vTCVG)fVOt2a=do|-&= z)~#p>PM&OK#FnZ_FUnh)m{-8*J_8Mez5YDT;u}qd+;D?ey|^LJsh2)f&H9p~rx&S1 z(TPRGXplf}iY#Bc#aU?8^XZ+}v0w=0_&QYuOWkZVbe?FeO54?{bmOx(GpfF2sHc$n^{!mYN+`#**jkkL zi*&W_>eEkJ&xC35cB-z?To#ErmC7!Qy#CloJmw%vly3hlmO>hdhib;$0u)^Gu3VEM z`Y*$(b(A0o(qTA133Uqk#w5>B%dQu~|0J9duU-sO6TCfs-e~3xW|;=hR%oODp2Mk` zMd?e_`xEB;NN;%m_TrRZVu;zcwQ1x~ptns_T1z)G-#VR~nlfi{sw;W^)Y-`kU1@XX zS|0;8`(l7d7BNaHkda?-lhu>P394RX3q}aN=8MbbRA<>}V;pk9TJ)DAm$YXpc&|EGV)A`N~%8)q;GDkr4WWfSV>!x6azsCjh->3 zueW-mUodD0(bC3-kPGYg5{p^2&oWN*Z$Ij}VuINQZd)!%%OB zK%7u+-ftOb{COjoX%z&f|fGlsU7h?ghLKA|c1JuoM%h5p}R0)d1D8&SYXGheocXO9cfLx?crJYDLssQq<*C3!Bo@dXWz>QwzbZ znhH5|F`5R+S}J;)Ax;Z1Um?2<03D@hQG@`=BnY_@ycFexwpVSX?Rg3kj*kBr0K3quM)B_T_iv33m zv+5MTNm4lbK@7mgEc|P(3>OF*Ga8S!AZ-qs9p^Z`yiU~=nzoI0Q#}*ZCnVKJX=>tA&tp(C^LzZoJC0U9R07z);8zi!# z#Tvq(tHH=WeQ0E?@vV611N*YUblIyxClHvXw{w8hL2#7gjrvVo#YqB^<^N;vZNTF^ zs{7G-cUP;G>^Mr|fPoMe@=+PvvHX!>l#p7MWLvSmBFV8sJ~nIZ%33V#u6I|q41vF( z23#;;s7oDcilK(O!37tb)(~GXV5n~~tqCS|TT@&J7q|FdLJ6(^)P?&yGiTm+-mjJ9 z(C4|&edNclX6Nh7%$YMYXU?2CV~;wc1C0{AAf{eTlZT{v&{h$q-}Jbb`#pp3GR?-x zJHyjMeZ6pHGNH)}y;B;i^+Kj4Ysw}7RcQ&I5Vd$Z00Z-C6AX^Ng9u8e>Y$#^gzU?H zoVPH9KA!B+zD-#Osk8PYb|gE)JsP}Sq7*0afGoYKZ3;|{eVzYUHDL9}CdE!S(AU!4 zk9X4tXlH4pb+(^&QY7G$+6^7_HeG+4Bc3&K8apa~)+wZO?{3YyIu+>Z=!Pgq?o5o0 zi_t72=_->VAZV1@NBRM0Y|iMBLj61;?#ie)e@SF%Kch$o>UJkCq4a}CHDGKp$WP_c$*ra;vQ!~KaEvmO1O+;5PcNg$X;{^{Y zmKJSK??`UKG&oE$X*NkCm40y5k~+kd&_=VJ!yJok{SPcpuI^nyFEVNBp$7O=t$eH* zO@98oKG<4Ze}BuswBYJ|Yxz~A1<)2WfkPWmy&^^PbWnwH{s+Y=Q`%WCI1{ET8mIDP z8ShZh2FbUw10S?fEhgMdfiv(YmupfWT!36m%8XM3>rU;1%1x5Trd28$gbmqd=X6IC zG(CbVQL8Z|y8@F=8K+II;;Q4(*rci2wZPoT6nJ!_D|{fPfU;?1uM`T{Bj$G`p^8P; zoz!`lPoS8>={eIU1?7w3oNHSB*;zCurCN@wx$3 zP|p<8JGj~LL8GmaE8(UWEdTdA30 zey?Oz%RuW|#qRd3A4uWRX2|tBdb*7qnqi%K4$0RNI6Ql@#-q4spMdbm_KKt$VBlrc zt;t2K6@@K7W>gYKY?rFrq=G&oUp^1v?WfTc%{5aUm5L>C)pD}%)^GXM)K5av4TP59 zmhuR_CSOG$ZTE3VR<&_0%eTUTAY#t~lpp00o>Lgfx!htEIy{l$<~*G_%R8`P;^xq# zK6!IUN_9+15@)WFJO)uXFc)9R+t_Iw{-!)uq{DY`ag`lK9L+)r#z2lMR|Xo{d7>4C zAhN3lZsG7$WlST$Y z5*odt?y>vogWc8(q)9wk zZV=DUzz(v3v7n^tB3ejNqvfgqr`-}31Y&t8Vog4BCIwWr1N>_nYl`)?Y@i!WYuo#9 z>4Mln#ud@Hr>Y#|4K*s(zdg66bG@?!1l=-IY0PPa8FHM3RL|+83(mc4s@a!mUQ5!A zVy?iFK+UmS$wHo(-)!HG_spP1T5#5k+c)y@ioSWa&~wcJdFkU~kJ zyQiFy`P@a?N z^iu%uL0ol6Qzs&-byZm2CaAf$lw3CzoOmkjVjH|@dg+@1#0ETis_ z$g6`8I!$xIKzS)%G@xI&rmtr`gvo?xH(+nESV%Wx33Vr$taPA;OqVE2H0tyOr@3?% zLj?wc^p@)^Q&rnZ@W@Vxv)Fk7X<-J>t>Mtr+~7_=2{6S)P7kEwC?z6DxGsNaT9*NC zy0VE}3J9xrIw3=YQVSQ~BQPU;f<}tmx+RHC59lAfOpsup+By)Ncd{|)DsuAq-@odi z+Nt~3b@psX^{+96Xijij`>OS8R0lfNcd$1hBWcj2sth;7$T-ZUkvShy^(}9vzsi^x zT?*Q=jsE7Lf-X(41XV8Q6o2vGwfa7}WUHM zsZ|{<{l4uRS4%M^-a0GfTt&@Y8K+u*YO*V1aHV8*GRdw&yFRa+RQr9F-ILa1^-SRXP{jeRxk@9pdu@YdA0*{yD{sBMU&G$HT#N@XbG zkocnQ8=A&%8^`Ud%;qfa4}wu*QMYYh4e}yfx+-kXOEI@M){b!FxT{BA5%8RMP9aUI zv*kftOZ2C7ot*01i?GJ0TCFUusRvyJj*Nmrkrt=^Y1ky)p4*JuOk<_#p6D@;)gP5d&$Tq| z1{EAMdwE-p<6m`UBn${oPhdPlj0kaECZGFd6BV5*mO=IYSl(f#j80SwgE|o=PP;74 zOVM1TNHWl911nKT`*|svHh8riSd;2n-#O6H3jFh+w8-lLdhE*4n+CWEZYM`&9H=vPSccx&DFQx8fwY08V8kA;< zOw-lThAUZ}4;m@7UQ3rJg(N2|*@I)H5|Qod)!iUPc?*~BWU`*C5;~GKK$-{^snY~# z{<63N;dGpIR}!zwu)tXnB2knldytfV;uCD&5f*T%Bno3g|-z z0o?GGwl=z(oswo%j1p06ib~P+L`-u<9CRsf1N=qfQ}H{hd9bV@aM( zx9b|w^flGAY6X~TY6)4|`cYk}kY+5E(i(*{GGQ~_K5jcOIj^2$d6M#H3RuO6pMkCs zHq6GHX7kb58B4y|As@R9+G>Xr1y7d;!vWc=rs-eXLg&rWpr%2-CKw{Bz@cj< za;yY;hKXHc=UR}O0ydF-Ecx(24vUOX)~kHHqgo!Iks}3LcTHx5$Uwi7C#Y8?-?CLc z4xCj_15?pDldnVYNE>%XK6=LKj*$E_a*85l3ugyCi zS-X^D!&Ui-(6T6!>?5fVoShY20qW8vBv>*P^RgbD=dLA58Zg{R{MPnpF0-hRAK5_% zEA$pj$)}c$B4HYro)MA9R;;Y8<{kzv!iqcgyWZ4<6>pe&m)r|hs#)^O@2(se6fOx6AsBBFW$+Bm$d1(Vrq|y!zO-ri z3K~4_bXAk6>YJ7D<)?D0CGf@3g5$=bvP93X-ka%xuYgc91=-CT$r56fLa8}ejcGAJ zoJ3GC0LBbDV-hsQNqz~E!`K9iC~XtXYCa+P5KnkY+Ju=E%0FyD+N^F!dS}w*eJV*W zo0g87!c6k&(SXNKH{trJJ!P=WGi6R4^@0W)jHnAo_`>fh3VJ)0G3Ur!%zAR95<^Wc zPw_1IPPS>ufoZC{TXuCxnV)}l zWEm!?&NJJY6PU9*Qny(q-_lMg-9Sh8gG#W{zo7%Vl=_knX8Nx5xPb+|lOb$$bz$Rb z=2Dd4$Ay%QtVp8e9c?c82`!D)5TOPKjl=*PFp{N^%&5vhOGjs_zoV;7ia=7W5JLsj z|3zi?iLZ>}8GmF)hy(-!WRTsN7Bu=oAau3#ruti<4(k&6Pm)Q>!=~2ikYd|%vNIBn4;9r_b^Yu}ifmG( z295W>s-+Gn+J;yK5G`Ee<7GP0Av;@sD2P{&rv%2@*>T^S5=_?L&bW~%Lyv<<_$0|` z9o8++gEKyx0EAlEK4aNa7yZ>81_ya5iBD-P$6wMEVN5lnjp(Z=)XqEA!lvCWKVaQ` zUL;njOGp-TDRh=>eUZdGoPi{m76fCLptxq*4cW`JW9Uve`F%|O~ekod*fM_~K2K<?B=)xKXw@vo~9(`ezZ|pzS<2Uk+n|jHR%VxRR^3mW3F^cyD z-8&N*B5)A7!^MX?0i)Xl{ZAK7(lwZ4sq^Yksl%g8hegI#!+qA8gsem}q4k3FYdsep zrb*-u$?=;r#6gj{Psw07m4=`OcDN-w!Ave2`ZR&JpRRl=_1ax zv*?#{#T%($nOIk+drm21qg0)8F+0fSgXGv>+xmv(E))i2A8K!KVT{-yxWmUZ8TMNsKKlJpgV*<3Lz^*4Cvkhw1G#--wQyE z0ai5Rt+%d@jkxB&z90KN`zgz?R;32V|j?9scOn5g$-7%NGGwAs^p3z*+qq+Mau~t z@~)kWbp83KKE3+$s%d!gHOC-NKae2o2AqOuZ5C+u^eVF~8`?Y8tR3iIuJGd#!uZNr zo>blAlfKjoe_DT#*$Kcb7+Q$-X_X}ns~{XAQSd6|jVeRi|Pmm%rra0$vEw?L6QtV=c_F9FOu6KU=oWKDUeM4s;|-C!JvZbX%Ad;@ln~8 zezEaZGFc@O9?(3*6`Nty0Ppg=$TqkjRIPd-$8i4&Vj3Q3Z`I-}$tIu)o(830(g|Uc zq=MxoYLBb6{$lTEd$42#OWFOhrIf045>Ptj!+Irb5qZH%Btm6)5nX&Hd&lU=^xc$ zfi2ZVMh28Ax|Su^k3joJ$adXN+EY}B^vO3>Y~KSqjtN*b*;4^}M1|DQop#fUDXM*{ z;ks=)5j|u}m$Y#=1v?K&xv)-~R~yg{m4q%1;5aZ1eH4`i5)>$U$q-kSI63j&vKu%# zacE22*yEH0A|(ZRW9bnM-MN`&mPtUC>LhXkF=O#ESOwc(aDrSWr6{LI!Ro1cN|MtU zuyyS@u?Gd-XrsCEr=m9DO{SVDO^7u~-gSv!4|R?Hsef$IKkO*G-9E^6jA{#mLTKJW zUQlh8>M|7aIN#_t)qoeb}2wzA#~StlAWKlPQaBCCzL~%3v#Qx^imJFx}zI0|uN*G%0#U z1IM_higj%*S_Rvlr8O^M%0lfhSE1teS0>sWUJ^N8^lg zGV36T+BM3yWYTyOj|Qz>dA0jUpS2d31^AHhc z=8N0Psl2=euV5+wNI!&EyGc4Jg=Mh5oX@~Mxto(lDw}&$bup48F{cjp^y6tqJX*)X zilIWG0yPbY6}5ruh0b}66SOT?s({$-M6UB|`;BQf2fwCcm2qb1D3@|{7oz2ymP|TA zT&&&nkSm^=r=uG*L#v;`$hZB5H;GzO>1{9&pzb&&%Xo3z6);#R?r;T=Du^pU!l)D~ zX&k7SP~1Thjyp&~atBF{U)djLr87R+sX+IM+EeDDq4cJ8Q+yG279}8eRaR#TXob4S z!5T0y=g3!D-gDd z2?VR6f+5lX))b~r{B&6|CJ3I4a6?l~#;TDw>t=*Pv%8ebY8qZbk5v$oWDc_oBRHsp z!4bR(?4eK^g=wAq4A6O* zT5PzQcva<=4%I^eek?17(j(A?cQI*d#@+;TNLjp=oy}wREab_mcTyP%khe5ni{}F> zz=OP2iT*2tmBTY1(4@`N%e@e+jODlF!3(;cVKtWo7Bb&f=I0#NhGqgjkyoET?D za+~h0_lB#Qnc=b)z1b2@p>g^=nC&gm8x^^2+4AD>s5^%5t;d!gYY#2Lc)gd*7gND^ z`-6+c=4MrFs!%N7U9TFP+*ZJianM1E<0PUQ>y!LXh2Fy*LV6wxlW6WZ!Yk!_umVF_ zr?N1)^xk`uEy;%-Qp{On)BHysS$H)tRyi=1xL|Mv@RB#w++W-3BwOlAHijDE7T{*I z>7j>=Q;rn&E5K$U4@;5I9?oSRd1QX_EwYC6K6_MC(m~^ryA~$t%j`gRaOI_JsjOJOXE|^>I%0+GBv+qv|l$zBBxBPuIOe1J31D1!SdQzUbQe;Zo!i>118K$ zE9H_SP?L`=nX2O_pM{(h{X+*`_cpc$r(+z>=aL2o?vMuF8Y`^`d8=y@agAOz{^?AE z9uW30nJ(I&YzoorEKjVH6&c9b$}zgw5e}9y$6?eOGu&#}oo0*g>MEd6d4_P}JuHo) z`Ld`tZe6LRiE3X$?7$#uAh}1%m0E2yNRSYc<$(?|IvsEyra% ze^&4I)t6mW6kC^|BC#RzWsvH@^5!I?3%B8YiRjf85~aD?o*^r;!-KxW>$`C!*PRr? z?a~fkLJ1YN@%&J?Yzt1@*VBs|4rXm-CFt3*;0#?I{Xw+`A^Pi$m%?$U6}J6N_&CQP z+c`eyt-YQm4i^0lZ0+fWQmh*w-Plh1;jL*OkeANgiD}xxDZZc9aZ^1YIn+KvlJ~40 zphtaOO)INc7JN+!z|n=_Qj4kaBa_F9-FQrD~hCt09dP`TWYpXE`U z&@H*}O8J;4FPFri>u*83$bbe2uMx}rj`_*P#_{>ddy|d$`P)hQ-AupdJ1IN5&6M|q zr|jR*(#uKTl2q)-Xc3DifZi=3<0se<2>?s#Xz}jkTkT@*UxyQv}!-3(ACl=C-qv<0ogOvCyysL9k@XJCXxr zBL@GGLYiUE<%hB`O+lL-9I|J}VOxqy#8o>Cr&5+K+u?cMQn|7SyG@i%@}E0gOg78= zDuuDZ;ndg)(k4X>9afuJT+FEuts_mkL_GLw%cVihjEC^ooYBsrlD{?sF#ONX=w_;A zQF!n!IW|%N0{GwUj5_k#8QpAlNJgb!B*EAjm$~@!kc_BmiZ8<28JD{<5@Vk<+4?FR zy=o2M(HN4R`IPguOcRQu&J86SDapNWkTg@Ck4_|0IVu(vddR5M0t3z?fQmkO2$>%t zma{&ahdv?gplEGP6-sor^A@>hpm|WP!aaR$Fw)l2->No6p76GIig@eY_4UB~7#=;4 z!g=}X1_PpfS2DvD&sSEYD#@+ohvd+wIt1ig$2rx8DIjZV9dc4v0kL2NQ79H%f{6)Z zF?J~o*TvpqHK}Y~q*Z#RDA_f01Tw=?2%%Oj6yO&Zn^Id$BzUByqZ{@EaJi?erMoS) zYW?cf?R};RfHl4A5!#QtRDC_G>FSRx0ou-xVwZO$(}RQPu-oo{XlN8e7QHu-g}hMaLd`Pvc`Jx=A7ycPdvT3 zZ8NEM!8#wD6L23ZS*+Ms3Q$GEwu%{uVuEu9??w5JBvC``HnJD?iZug`TPlUgaJ zw%{BKM?TwFiiAo~$RZi!i1Y1Tl7m>EXx-IK3d!;ku9}LIbdy^N$&&J3GzMzAT#(kp zSk?zQ82Xl!8bZp@DrVaf=CN=zYOwcb32Mr?y( zv65mby3ztFhaJR1gNNflIL!F58BHaLAqLQWU?XqZ(`;p*ai-K!k-Aiky9=(jQ{-w0 zp+mYTuW#sUAGCr7KHJ3UiDATx4nYQW*Q{B;8t5q&vF5r^5RjDyCuT5QH{?ywS^4O!GY9r(_RS&_ER$15uC4Qx6=!|UUaPQrn!AxdmG_GrS)Tl<>`Fe zX;6y1afqqLo!V|WaJ4)-$ovwGeJQNl%p=r+j6MbXnBJnsI4NeAam<|0l;<0D?(V_m zTI$6OEqz_ zG*-XDs5b1P19jiuR=(qoI|O42@AqcFm)mMC9`4s@r`bp{E(!})sV?(KHYmZm0D9)S zYkR^xKhg=b>O7M~R9Zjg7=~LJQ9o+!0j2NHEBfS|!F1vnB%n*ilEW}C;KcHU^6nmG z{na5fUlLpe28+`PO?GcmO_P&m@l6PU(>7CpY@}=k!d$Qai=BcKEZ|>4;lWGnY~4Nos4i+{x>n=5>O4t!Cd$ z+7$ASNravWkZTF#+0&rHPRKl{pK<4TP-d62A(i0@v=uypqj8Q0D=XAbX6>*`$>RJ1 za}eu4$dgL7_4cS7JCv)*QE`7{k}?I$iTxe8cF{ecbS#Xapf&5KmfA<+107u*-D^@1 zEOxmqD*93~?-`I(SJ;;&6gZoO@MVyUbw_tw`$nUMq2P_wcq158ET=7cm>m2qM0si4 zsK8wQCM=tnh=gZ)qjaULeC5T?XAE$#ytE5;cc0`krM+`vKh+^;<85U~oRoM>K9QpO zm`>46{=(PR_6Is(aG|ZId!V}=o3sGd-dDV0>U9>LqLh? zsUY8z!&g_PbY*vs^a=!qDkKx6oL0&~lp^#&qY=alzM0BDE_|5mYj0_5S&Kg+q!!Y+ z*`R!PTaIg$j5RFiscp!d`Hehx6zwAea=_ROO%Lw0q}CZ;0YbnOnu4u5#DX9PwO3bj zNfN9FHa%rbY1qJQfuoQ+ic^YQ9$jo3{((F~&zpBAao`Abt1Krbh7{TcMq;(PyBh)a zd(%|v+27f+iud|h#`Born*F0i)zX=vN2&J28ZaZScWBf;$@%G)g*1tml&ZoZo z0mG!!4C#$VjGclvhH6t_ry!n{Oi0cU^UV@!M5VeQrCK+o*_I1bWWdByZEp3I2#&L*Ev1b8 zSE-mvHn5gbyX0p+R6_KE11LrRJnR@K;9){CC?)EU z;wYYq+pYFit$qH0T?bk*Ap1QIF?Zt4s=!4Y9)sbrf-#~4TSQ>Yyb zAknvoA8$LMf1tY&LAYGyx~G+1jlS>B`Adn0v&c6L>bPxTod-2?yK-20Mt0QC3C!h zc;{!0QLXuWRps(7m#fUD>PDT(;=I}dQ%WqR!9An*_R97<^k}9wwcz+eb4NJY44fI< zQ$PqRxcZZIw|k;dGuq@G?#!gjykpW#;qIEpE_uuPft~=)tx96LzE1yX^ryCxIZsk) zSL*KOjBx*Q-L>j`g$GSFd7nX~PHx5f5JJ_JYb|zgA=R7QOA9Uu17E;*Bm>R2DG34G z;1;Ml=v&I6JcX>1ffJdn8bqmxsKtC!YM5`3IsNQ(uCfOouu6K+)XY63sY%v_W;A9* z<(Ut6u0hV8W!hV)GDq0o@kY}gAQEwva7fFK0l1CqSLykyr5o206DT_gEYzK?DjiW$ zITo~pi75)i6hcRC;69$zOFx_(_U*v_2h))o;&kZAmGd`130mH=wr2qPB)#pO*pQNa zVLWwWA8qmtPclGAUm2!|D*D({$EgdZ!Eu~#xi^mr_l7bgUuqR@;!`~yuU@3HuWqT= zS;^SOU>3)O88860JsTb)t~p3uS;KgQWC9wv;HTtcuF3{Bln$0GU9kwpIbcy7%!Mtl zm#4j)0e2{Hh=XfhFhBI~jUxW=Rs$E**qb?FJKD zDm@N|i%TGD!8z&@AFg~!Bs~hVJd-@6!WN%vDz)>#+BoXQdJc8bM-Y^t^IKDdUynQN zP!?lT{awAi5b#$pQdF>0CESO12aS!SMz@jvy9(}Z*{Dwv831EUj=E8)3L~-PucV0x zWnkEk0>|JO>BKL_YrNQAk1fPNkVp6Ky*gVdR)_9>SoRp31f79krhrenz7O5Fayd6h z#&dusT3G!ggLj~69yUa;sNcOGm7xqo$RDQhZyOOMojcDdy zt%@WOCrgzQu!$6U6M$_3Y;~-o1Xh?Wr?}$`0|Dl^0w~k`s2<~i zV>Bdf;CKv4>n3xadqUE>$C;6q=c&*-a(rmLI6ky493NCZh^!iljOd`!jp(4ljp#dx zTsIBoV9ORx0ghVHprr`m4sLaU%9W5pGfBKA86y0G?4zWlNLg-F<=eV@?)pL=f%?ZBKMny^B_g>nrrOL&B`C05bQZb~AL_##SXR8%1GefeHBT)cvVd zq^Qx|-$Ccpt3eewT22l0tiz*mChvjXA)^ftPw@!=pBzC-3)MkPRCt3E=0mooHsSs( z%jD#YW$URxtz7HJ!s=rd7eTZ3)aK~!faO}A^h3eC$A)EW;jAxR2C`Vv_Ey$c!qcIV zi}6i49kXmSP3tz&d`(0yrN@LO7Xk=4i727zh|b83WT;I#t3rw^*4PPa*`^Pkl2h|7 z$Ez*k$+6cl45S75HD!HJdyy;o{XihU3W^eo47+x_|(INxX2uI=D1T4HF4bqJ^hca2>csxr3wN<^ni zmBZg(w7M0o#1NR7SF@&S z(ZeYt#=|0Dh78cs^9H=)LQD-+q>&sI zt3{Wp2d}C&P%*_MF^ap}c%(8-NBtT;i>v)KJB8*M<6CNYW25F`BJG#>!!ZdB`?CD5)k-_OwPYO`)6QO8cAW zYQ6iVJ~yNruv}!}XlTuBbq8EPl$PyLOHAA0_4RWB z>gjXnc98LqftEGq$+RB%Q%R#Gr`YqY#wR7GPH91qdZDYfdOg`?3B(G7qQmmuhTL|h zp_IEhsn($Yd-W2nZaEe0SW$uGYBE{V0`s4eJ#cn{K`bo5q#P4Bc(Sg%MQ)0Wuwq*7 zi_K(Pw&@oq%upCgFlK4fpblZ#*3zTxFr81y_gLu^QZKefnRKu~Hl2rn*vytYmn==BdVNJ zX{_Y5F@j}dA7GJ~!H0GAy0+4qA4*ayP>cto%g=1?Y#pV==0#IW@DE z(^?pD_EB^xGX~G5gc73+$I99i`3-OW$(Bc$LY&qhl(?l#k_W_@mYWGDmYxlTV6$Z# zX+f?OtU#KR1G8S-m(smG~qDP5(8t`dTCtHFsE+;P$b`%tO=_D&o*fs2HO zYA3EmtC~1R+91okec>ai??#MfOJpjDjGB^B{aci}KS&d*ny|V-!Z-w~TMcwrp+Y2e z=)(;A%7YuGd~Z;mmD1s}7HBy2k+-%&^4j&01<5+%aU#B~vIuuLbWyf>8f*^9YBy{` zqsGbMT!ro`%IPE=L*Tdj^me&Qs@2eUSnI{6tstOlWmk!7!ZhZzHyHnROK(Y{t$}Ua zwxm9&TM($qwz~x(gLukfG!Ju*>QVw?EdxIj)BwzEu9^0EdxD$#$ptf zp4LAM037$acDK%j9>#}DH{3X$XUp_;V-yxeI^fWA5I$#QQL?*6N((@(4{szO44-27}`m*0?4G zy48AMXgc&K^pI6jnLsG4p*uZO8-b8aExGDRCP-;22@P@>41mFrS%1qa zXbCx&+2s8WCLz7ol5@QmfgVT>1bs-bdOc(_oh=W#1uay0c?JRQpF!e<=fH&^sQMUw zav_CT;*_LFP@SooL5m{ANovy(0cEPy$n&9*_~e2J=}-0R9wzcYor!h`jhcSL#Kv`QwYntF$1~o`)Hs6srmiR zsg@uW!&dk7b=k$}oYd12`6Xz|&?QaG@&?W6DsWA8u~Lu}074xL;MtT>7izcBOY=BV z_nv!c=-j}rjjMZ?!-!P7z6Jq1giAzAk>2X+T4K}+C&PUIuKGQxATw7^X8?-2PNfVs zxo=WA$&6Dl&G~TQ4|KF|=r^jxDf!Z$%_mh)?TlzSl8t+Xs&)o^jjm~A@vUH_Ow$65 zA-Q%rjr4<_azd`Lxh7VF%gNnKu3YZb{4b5TOtO;lky1HHS4i!}(&WQ>F(W#hD)dFv z8fWqn(kuzoyQE<^x?O~c5LR-b@u#^GwTkQIDTKo!E|EdGxfdrry`pAAol1H7%!vPd^QzNe4CGJ^D(0aBtxICnfV)1@O5$DMshc<$U*>CD#)S7wEo-BlLdY&PFaVRz^-EPs$Xa(akv<4d46v+ScRrH_f1V{$mgEtE`N$d}em zH8oz2ljBVv2+_t^^Uxs+W$#X#!<6D=7!D=&2r9L5CnLvEz>Xn=o{0md!>`{ANfR_@ z!zM?lQ}^wqoQE}6JGsydLwJvT*wvwd8C2jDNkQ%cm`K&+;p;KYU#OeMVe^JD{WuUv ztcfW>&o4^*pb7-dr*l5VH`$XD(Umtv7%Q(KczF2$2%7x8)=@3HcCs*u| z%>d9^OJ+@{pQoeS#Yai~A<4n!z`|lrCvP`6*OWroS;_4}B05<@187LGigAWSIiw`5 z+7ukkjB`+SoMz!^)a=qG&s{&+Vka`7U?zGiyfmOSHk0hbVAakE?-y+K71!+pi2Z*^jMGo{l^=gAm4P-OU~ z3^I$5?V4&`AT-&!pqQZT+th||fF%fLreu+>n|DkGn}LCRmIY@XlQ53a9vB1B!V<-y zV2cG zSY-(;S06cgdV=Ol$r8~6sOq+~%lKz#XE@lX2HD|>Fz#(7J6)i%YE8L(mtlZls%2neU z6@BW9Odh4^X_oBh{@5fcw-DO!SCR~E+aAN73Ji*~I29^@q?^m)jBG|SVv7CpMQKx# zz6fHyLh{%WTFKC-NafI79ytC-I(ga*|iB9NspiNlzlnNWn-5jkmGj zN}(B0XCBn>u_uDa6*HY7Mb3-*_2OFesK z%5Ss~BrK&d>#S|%a}Ep>NV~MUznoY!H=Eo(j3-K!IvW`)adEX?0Fv;Tw*>%PIgqdU zI2IwK?7J!mLy+s5;HqYtsz6(|9U|97ciyp#&R(Vk5frRFEPLwoCpq}$-z}8T-KKlw+B^&w;VFw1Gs;C4$r&=R)$7SIZZ18PEX}4 zKHge6DKK}c6kI)d!dY^m$p{Lsshoufxv9-r9MpO$Yk<%-_CA?NGG<)wq&FKy8=hb3 zCy^Leiu#ofDe@Uf1HQKy+uB#HU&DH70!I7sU*62}fU&88QSGaO-Rs^8AZKX3l@fl* zMzuFD$J>Ywi#LhDS2IFtN~Q?QzNQ`2fPcUKwe=25T)Ix+YddnYID13#%+2D38wP|p zH8*zPRpRvAc|trVe_ot>s}Q?yq|l=`&OUat*!hyS(>IIb^C;xRyx7T`#Rd8Mxp}b@ zh9C0Ro)4n<4Qv9Y9bHuLNq7avB=i%p`#9Sc`CT8R3 zk;DoiE+!~+q7I>Z>nQX<9YW95QRwp&y0d;Gex9#KJCDq2qC#e^6ym__Bz~Trebdf4 z;^OSbfP-tA@bmIDC_7jHNS=SaNa z0Gw0thW&6Z#2fa(xe{;K3ukX_!yY&XYZF2ou1)O5?~}C+yWs4qJ7VpGd#Qdc0X{k_ zaTw0=S#1D6J4P`(XP=4-ar*kXmwy5XH_n|v*s&YuUV-oQjl0pPbMuEO>!qgHC4`+@ zI`1N!=aj zzTLBcmq!{;!gK18Bv7kD=MEmh8{itrVAED9v?Xh96q1Fi981iGT)&>Y(IR5iu3ltIZ>QVFkjf( zcpAR_B_Qu$NfvXs)OZR(=SnhWqKuebWf`-lEMtyQ&w#H?%2D_I&TpU zj=aZ8P_LbPF9HY+N8e|ii(=4!(2DJb_reFQ+2@gA&+j3gDtYPmtqV~CacIAlxCHm; zN3FT!zx+|FZ6^{O|0C-Fb@j)raRPktfVEaef5J*kz<=PNmAHuH&mFWLm+_ypW?vyN ze`4Kw8ScqL)@+KO_)}G}V}EKjQvO{}TFDpS9(vLO-5-Bawfy9hgom^2J1>2g*!RMd zgt6V9wq)0z{j@cY;9UAN<+$=`&VSgtU{Nbhe8!r00q*Y4TE{FZ|H=_-_8GW)|6JAM zEL@_N=RT+W6Q5V@*bkT5ag6s`LJ&cIGr>&DR<@s+}u|o*B_&3(PgK#JQR;4@!mmr<|TWco>?9>@+ z`7wC+JY(H@9PZI)ER6f(_)Fb%=@}$Ci{R(KqjDcVYh8|0`eXlS#SQ@M$@A8{L-3zJ zuM!;mp4Ch7m%nGl_9K4h_pN#R;O_svia+~(Ya${G{DIX*MO^rSl_3A#A6kjMNOYPx zz14cFbg^b%C(dKZK7ZX!XKoOeuS4>;M|hH?<==ZiMwu{dk!VudfbBN%55=t zcK=r6#RcNTZ^ce85a;J7FDwv;7Et)%1&0+5uosp4-vKF@5!Vq1CC$lPlt&vAIwrU5j!vOiHreMRT@PLWCG-mGuLkpDhXA0hU+eC7Y2Q z&Ek&1aoBW&RdRZPQ$VISg!f*l(I}%-kUfvWsX5ATQI8*@!?r`%dC};aH7u<*t3b= z%8@@AzJw|RP)wQYkG_2%J%Xp&1li=sXI5idmZof_&13X#lukx{hd}%#!fQ?AFcUYT zn+2qa6_ih1*A8h4uGfl9iii|qAZ4UD2IKM(p!IpYKbxi(Tl%v6(r{T&s$7K}qWV@h z^uSO#7;<>s5bDYUu&mIG%z21sD+CNzN$AgNJl>|N(X5OH^$ZPxq?#)Q4y9p`%hT1r zJV%k>gFxYkQ2kxCqaDxQl6^9e0u?HUVsk;_y4);NBEwh-Zzfl`m$I3DfDw3Cn%*$# zO6PHn7m5KnsT?H@_UnpE^?#{nS8nFJ0KM7LfD_z|Y}`n7RbbL&z=cqTuln(7YkH)e zGy6S46=}YOXQE=HaQ77!wC0KHU|-@WwgsTqgyJyxL4aO z@Iq^g8b1Q6`4vT^0xxmkHAo?Ch5=U@J+G1W7-kd$16_C7|K zmk6-y!7BuczwLTCu6_VG?Bz3)G%nNFxp&RjDoREYPD+S zgdncudOH)mojBL_Vr zh7d($I>2mIGZLYga>WirfH4D`9*GvYbYRk88s>?ERwhkD;X$gI&6Ly~ z6<}@+w$-}QgQbGPi`^wMk@Rs#sy5iY+K*|TAesU0%xZSbX5`?JY1O2%29m@g(O5^h z3$u3)>uk@EifSLv4vx`cu_2eCsSWT$CMpBZauViZ`1)Z;CK(GgFR*5y^i?~cxC-hg zqL%?EPr5WPTY0VP8Cs1Aw^>f>NT^4tlTa`2PKY0v!|F7Pid<#f4HdYsX5z7#1Juki zR>tb)3eu}0$I$cK>tPaMgy>W00bn*W#yMXT+~6#7T>pa)k5xpCSa4Cn7de5XBUg>_b(H%vv-TgB}la{t`apX+Tq?OYH#fn z@fW&Z6k^aY`_Vh(zZTuGznfnvKdtO+>KNFENFNoU9|A9^Je~QR~ z%P9AXi0r?D`u(?v&ifVe{z^o8;qLyGsF`h9)?CYqw88C!`#9X4aF4=04)+q=n<7>u z6R~QZi&@sCm=(FH#)LCZS$Axqr)Q7eA- zfEC^MaVxR>s8!SWloj3oly&2=W0uwM1*>M|7cH^;OP00qOIFR9FImyVmo01H%U12> zuRv7rS5{=>DeENze{CfW+!=|s-4&@ha#tjJ@vcbi^D83Ol@*crcypw7|2+}w&^?iw zrZ+@tSFDVPOD&N|+9B(^UHcT1$EZ6sn1k3`}R z7b1~8#fY`P7!gOtB9Xn@BG!Rzkx1|MNX@}FN5sy@BeTcf7KyL?F!DSR5yO8Ru`c~_ zL_Ggzk=j$AjfnGKh(zaoF(Qgziqu^BQY1e2YmvH*e;pBrz8Q%oPDf$`XChYSOeC`J zOyrt|vk~jovyteD??!6&KO3=5JsXK$hTHUaXye~Ut~-A*VmM#6zWY(Ua zM`FFdh%_9#5}7sp-;qe-*O6JrejT}{F%pfO#5g!q6OA28MC&eJ6MgBq>!Q~lxjq`b z=Z5I)k-5>D%#BfT;pI{5)>lMppMPaE+IVv`mS~JxyI&g>$CpK|lgpyw*5;_SusIq% z(Hs?-d!sdv|2ERCjEdPk(b(e;L?a6~M57OHh{n!86tylq6pfwDL~GAvqp^t*jIv$P z=-ORTt8-UWocd7II`g5Z==?&|8u&sq*7)V9xO_4iz4hy;-#4St&u5W-IF?xZ z)>!o1TVpln-xd?|-W998^6pst@bAV%W_K)p`S)Yu;U{9zTR#%3YxrnPoH`Oq9Qtx> z&aSV-#PDCmB0EpT#L=^{#N+>rbU%tEGXDygKa0hW{X7=E{PS3Bq^3q3tgne)sjm^| z7S>o77S@R5!!^-?higRT(VEzVD}GWVntoan z&HS`R#C~2AJAAn&arsJ3EEA23Q9qo@t=MKc{+8&BW_dFCAnMV9DvA8vHAU<#W4*PSg}%m54Sa6V^a7 z5ncG|gjjn^BAUE4Q9pNOB2rwL5XG)U>_AUK?CVchv4KSF{KkYO9!!XaM-o=%5yTBA zBC~Ue*v^qe{oYC|aJZv9w7tUQ#cIf}v6 z_@@am`!k8?-p?e&yrYTO3r_>)my!Rg331|EiTJ!T332i}@c&aHe$NjQV(;^bSS(U! zB_ee-J0o@RU9q~z*;rkyF;N#en5YwbuBn^daBW@9rMY$WM_*CbuSE{bsS_Jl)m_`vim=wY_`DO)n#SR}$jESA?TTDoj_MR6LZd3e`I|$@_W&}vEQ%xaQqKy_a&aF z`$+vC&e}h_X6-dK9dl~#yS8RsLrv#(HC@-&bl*_ZgG$oBYUNys{YsXz7i1~_jm&xe zDvJ>iON$YFE9NMeXK~^w|BYOA)fvEyGlPK{49s9)1_Lt~m?{PuCe*27!vyW=u?kMm z{+V30q&7_8uvPliGSx7lmcWJy?5kyX9lL7R-Y@}W8W}#DU2N^8A4)mWg?vD|YQ1fk zz$vfvlZ=7Ng^WPDYFTcWxRL!YVHdmt{xwXf6XS-7m$Col?7o6sl3P%^SF(FEyO2BJ zU&BO_-B+{w8g_4C_f~dqWA}D;e~aBlcIUJET6Pz(yO7;Q?B2odVs@L@UBd2Cc9*fc zoZUOwy^Gz~vAcrZ*R$Kq?%nJ{#DRYe6K`PmUUq+*-IeUNuuJkDO5e(E8@mvZ;9tYU zYIfJKyO!M!cJE_%9lONmP`)m9yV>nwmv}gezn|SccKg{KU{@;fA>9M)hj<468YVWf z`yjh-WcMLWzlMnvyN|J(W_J_2gY0J5&9Xbh?q+s}+0C*0CU&>5OFSNx zH_C3F-2%HscDJ%yVz>hhi#XrYx1Bc(k?nQQMA5!UVXSbc*AF&&ISjFGM z?kaYl_^PV^5q5VyrTmevxZTgbPvt)?UDTid{R@Z3-_P)+>%_10_#ct}$8`2d-M36L z{Sz{MgV-h^Cnrv+iBjyoliHIM7MJLae}9jPpBobYcO0+gMd5@06OLE&oj*R&TOIx@ zIbO}D{`@WXtNdZ*AD<-tb$!+Ie~#nB%D<;y)PZ#UJ0f zUgcNom_Pn0j#ulKKYq~zDt}n~CpliNTmJkjHdN1lWRm#CjVgZ_{HHlS8A5-LJgD-k z_0wO!Uvj)!SN-u%yiw&>>#09}%|j|ajDEkz@vjc4-#Z?zo`2pWD*h!Q`S){t7=7LT zsLCG}e}v=1@Vh3Z@+-c_-~PXtB>rStY7 z&8T?A_xSVwg5$&b?*q7-PV-+_|E(BO@p|3z&X51W@ppyv-&;4U{9*W+Gpypn+V>fb z53ApXoXW5GIe+_K{U#N!_(y;Iw>bW_A@%>@7L{M|Y5x4zkEnRXkNM+2&hcUGYZ+Df zlOg5*nBx^6>W6HTt}vqT&^w?=Sx>$A|U*#8~zGE!$LlSpNUb@nQ7&Vce3V`iJ4aeY|@4 z@g3F6U-T9gAGSUo<@hlC9(`Qp53B#j-m2om=qvp;6(2^wbN-i#52LR?rR zgvzh>Xa4x)+f}^U$NA&WaeUbNu<0Ete;EJtJ&q5XzaRY_l|O8K`5DKD;rGKkRsOL4 zd-Xe2d|3IP;P|lhx5^*Jzu*7<>ha&>_%QtM{G7@^2cw++dDq9A{*{tbgasSNIF-za+i5tj`Nt;7 z{~nGH!`~lsd>H=zlH`fu<~0u zKCJwHjt}d<(MjOHgX1G1^M7=qYF`-spK4R(jH%Dt{RM-uQ;< z@So=RF#Mk3_%QzAryL(PzZ_~&<%h+8jpM`S$A94Xu=@RUlKe+ksq(|%f38!-Cqnx7 zX^s!;-%6LtAGUt|EysuDKi*S4|Fyj;K8$~OYNLt|8^0MTr%u}XAKPR*9r^9TM^*l? z@v)lY!`APCl*<3w5d5t!R*&Dx@nPjJ+p6+!2!a1zjt|?ve1YSWA^AnQdi!6_@nQ3C zhT|86z~3_o{8v|0`S*q7|Jb;SUm6ns49ADb5B~CPDt|I0|2d8i8{hvqN&es2rQnCr z$HaT8$A5z3!{n=9oh1KrljMKfdsX>i>)(bCsCYfUo=D6FrW#;?Lb)sI$W_=|iC^yU z2C?xO%HJTy+1|SPf-dWDi?nZXU+1>gqF6ua9&HK3~rOS(9({3iuhhiiVJ7!p1a622!S z{6I+fk&y5cA>n62!Y_n`KOYhh`zzhavFffCG84S!|Ulse!Gw3|BL%?JI5yutN3qle1XG1#r}IZ|MSxC^!EysP5-{n_)-1TAWq!w z?r*%;fmBIKD*x??kUzY?|M@EVn`nAB<15GQPclBKP>?7d1fp^q#G@SVrH8$N{ox)j zRE`&a?__+wo$>V+_CMvpm)5J$<3qI*_5?iRV_(#bFA-DqzJ}rd^hwoUv$&rGhhN9- z`Y5}H+5HN;r`gr?@e}s{f?Z8V*E2oMV|M|&dY;hp!8#6qklp{q?z`CC&+gaQJFdyMyem;&$fQ{}Fb-!tUR&`(t)3rq^rPeFeLX?CSBbmi=#JH_L93 z-R=>+C+u?!U6D>G@`!2b$PzW%m(whuM7xySv%_B)eZ=_ZfD7 z#;*7?)z2ShJY3KI+u3bm_YLfJu=^0ZC`A0$7$Mf5pWmv<6H{CorPdA7L zsh`>)!qO$={L~h`zzhavFffCG84S!|Uh`zzhavFffCG84S!|Ubhj^O(ezQ4iuPxvn3`z5~FQDMCtUn9P|@U6nv zi|;Xfd3=xK+l}uN_&$g4tN702`xU-9F_eSv4t)3G+k~%z?;ZH|;yZxvOZff?-z9wW zYLE`!+cFg6~Ou$MK!PcL`r4A*}21-JTGUyYS`lJ(&>EbNJ@giRb{nC-MEVPQ>o37csmc z8aq-iYF@zinpvWz4c|6=xT9P19KLz8Ma^n_oAA8{->2|hz;^}TORhm2zW3w%9KL7p z{rfc{);34PcFhqrC-MC&zT~yS>c%TmsZ?8bC_Oe(=^M*uI}3y9ku1Wx`&$e7!BV!8 z?XRQK9lOpmd8dZS!!^2OS!Tol}VQnK>^uPsZgRHx$;mhpQ~gu zre}yeqv`UN5(<;Qv!jE>9R!AaWh#q2siAak#2vDEEM1bY%9RWXDN<~uFjCl_MRDm$ zc05-hw~!O5O>fSY3L=H3pxSV$@~O?)N+m~?hPMnSlPed~mBHcE=2Bs-m_l0t#21{+ zR|vi*@klnk&7XFAnyN6w@P`KT6*OUp5i&G1GFBdjAN451DHOB7*HE#P%U6Jvq2gEt zRV9aFQ5m!q4oVp+XS2YLbkM3HAc=~pWJd@*ITTs}Mh9gJhPIc0c0mo!=C?_=nBJT% z=iZFK(KK3IK?kJD*^!|X1t=tqq9aCuRC0?uq+8CS9Qfg;fGQv&zvM22pvSW^sUnKe zvAp`Bnp7;5bK?M;*_IPGE6_vVUT0c5$UR7htA(`6%@3@@YPG?3*=ta{51;_lb}?A9?e$D0a7-8yn~ z4CIzljdq7=ROnDbGV*bcY%Lc`s2hq>I4=(lXEUiCx$Fo!8*P)_1$Wz~G2|-eHF1>7 zgXuhlR7$ds;4)R=*9g%OT>O_V#ss@#&JX&vqdZs{*|Z~{ra>S{AeSEk>7dTJJP2$o zud6Ut9?52b+@4kUwWeA-Tl)J`w_-+Eog2w^rSqUAl;nx)8mpu?jbO3>J#Y5K>fo+{ z&Q^)&{vAX^Jn^(;x8(*=Cl^RcS{N;&$ev;)SIA>j1jVB#dUM4rT99qamIq6@Vuhw3 zZ_?Em90UcE8ir54@<4iIEE@n46I(8m%@6MA&sIX=VQ$1I@>aOJpK;JqDy2cot}57( zZ8=$wbO|X^slui=4W^3OQn>*7%vE-zwlxVAC@CbRhJZ|!9z(;jn0U(B-a=udw}Ac$ zjF7X8H>NLJ87t*wt2tXXj~Tp_CZtfk$>o?445AI$^cLhv1FaxU32n5zxj&ot`1;3C z`=y+sP^_f#+3kj}lr0xAFv|mlepG!>c8vtaE#ahCW>AvL)8aHYEIi|HRGDxDVk;00 z%7SDFwTAte#71*@IZw)X5Pfk9kk*n1f7^E zLDd9PCOUY~pq#Guffi?U^=x_*Fyx~f9h|C+Y)fq_6e{IPDP07=W~Icp+u+8;#Sb>& z@39Yo<%_hBqeChKG!{i&XL?7rWRtYGHYQFWsfgUaT--R2&0{ta%VS~>GRDLOU-q@0 z?A}3(rEokds-i3Y~5zS`1?9d&`@X@)iN2#!<=Y5pWn|nDKrMSbNXE6tG^n)PTRozKciIBBfEREQQi& zYI`=fdAO1Sd_vSWBmCpr5fNFYL%$wLiNA{guRjNeB@TR;fbDwXCHQ;lzwtt7bQo(~ zSttHbbTl`P34fDhJ6uuwqNViqpol+;#>DwA5WtDO^!Mn;Zp7c6PXKsid1_s%Jc6lH z7x0Vd3jOoHkZ=FRxk&rmXYtfzWOz`=d_2Yt`I8tr^aV=Va^4pOxI<(|2o$_SN z$@SMd*BRt`;eAxRYKC|=Rw?Qnzl^C`anzs1R;0v4jfj9tOKlZzk8KryP=k7XkkEAH z^VE_H&k(3mI*%^ndVRTue|#NjPyH>m`nf%_XUtST(y4w@BcjAn0G#fO@kU@IjX{%2V^IP_ z`^7}Al0ra=cq3vEF(Cm7#HeioV=hi4(Dz@v0eO#qkN)nuf>%00xfp-Tk*bI>tIPe5 z1fe<$+{j4o(JKl5@s7HfIC$ZDz}WvIyd)a6@w2N=&e%K@IQ5CTv0{dxvHu%&A~Gtz zkwC68#%4;K2Dp=-gb9r(7RdZqQ5;PaM#Zn{#!8r=(Gu`W=~3~4dK&L>s`wKyT^31i z1N!7l`BeQ_84KOhNU-y(w@`vFQ?esp!oE2&dS@zKERJMTYVLLeva`lR}p5 zDQwBWXuR3ZaDLVhU;rN2T}-1+#1CevAy7(fP1|$e#aUaaXgxuSU(Tu&id*DU1s=v&!O$YcNdzH<9noZ_(eq{|G~Gkrk;Xo#_8uLv7H$UteQRdOyErt9U1*1ZVzR z(*Lvc_uP*?0KPcK1aM-G3E=3Qt>PO9Nr_L)L4W-auqJjBls&&&Z>7@Z9r-~qaV>~t z69!DBumw!{FXmt*0rR6Y#KgO>`abuoYpLR&qKWU&muU>@ zN~)#N2CU#Kaa8bX!bJMr_%3zhCG5dZkyd&G)|V zzwZ0=aq{^*`@Wv%y3TcepU*khRfg(6!={})omXW?%`&@!TWw1w-5Frz#<&s|euj&us^jx-&?WmV0FXUK#*TZY%7v!|#CD z)|u2!sZ2K5fAjeL*U35}c22Q_ixp-$R_F>lT-epGm1o;o?3{FF^3fz6I+NF|48!-K zW{2)T#hsbf9A~L{Cw{qWr>taCA5x9{4yinA2A@@`Zc8=rE>bo96@UANk+x*@xubis z4|j^zvyq+bydpkG4(^n$lF4N|+u|I?{&si}rxsq%9Xp5h^R%U@{}ZBi>>b@9C7X7> zz-NARKgh!NTlx(*qRv~dL7i_iCk?wW)sJ?u{eZ>o)F$0-wn_D{Qg!T-jIsUYY+K^H zRHlVWo;I$ism+o$PuK$DeTRx^N z-Q||p7P4xo?cfWP`ktxOHcqw0aVe5=O?Y@}P}P61=)lLAg^u_CYexU2X>L5&suTr){_}r-O8@r&8pP44_hL(+*sp2$J?AlKE0bS zgZ~lQv|BQM>orI+I3>ND!A?58(kn??cS^4~ww_F-C$Y)PJ852$+jd@={CVf|Rwf_s zVs|Yc-qmhjeQ!70N1VTWbgfHX*ew}z#Nj2$KXhh&_BX4Qh z=v_C`Pnp1<=0Bk zZwdPTGTo-KX<)kLvi0Edojo|Q@9tqW`~iYES|!)*;nPU3uY%)OizRFA!KqCev9s=y zJ!L#`1knR{1S3VRpwi>C%;_1r_H_H-YQ?cX7teY?>+p|&UUbN z8@nW_x1SEWH)BFW>EHF6Y;v~yuCRM1%S&vpHF{F=G%?GKh)ee;f%TkBOg%m1%^)FXRw@{jLjTR};3?Or7{PtO$xy6hB>htX{hwU_1dm^!+`=Wn~y9NEiC`n`NB(A$`X?;C%SpPtpR5SuL8 ze}|n~TMkvr?OgkXe7i0CXZzpo&FT63Z}7MOIlB@Vlb-HwT??+5?z<&iA8~4R(=FBL zS!)oeo}ShNV#)WX+n!|uYN+jfoG@*E+kU1<_Xj-R-{ze6&FP%>pG@oh4}{j=n2=eq zbcGf7z3Kn^bcj;J-u&{Ty|GDP6~v0#wD;kMsmw1}WbhSiEd4#$ShmjKfaJeflD-$W zx1e1yYeg8|J1qgHf9*cz-(|nSBpvqG&b90cV3l1EEFZ0#`}bkeHcQ%cTbih%q9l2A zpY)`Dc@+-3O!B%Vsk;XATD=J++c<(N_$IrW#|pLe5}+WZx=ueTFhE5vtf z`r~wXe{&yfUAW)E4OW~3SNV8X<|P`le`NFXeHndZ-y{>aLz3(FNspRO&PY$?lgIb9 z^)2by`@E!nKf7+S>(SM#&#@n0(xYB_ou&)xwRQq-r?<)L)6*jP zJ+I?hcffux#P!%6GwlZ{{OYrM%|*DT#~gIcT#d7Ya1tfa<-~I{QBl9#e@(J^rfn{N z!#dXd161Fa?6)J^5dV)`;Y|l@GedspJZt&6yg{AZdVuZfe}s+Gt{d+>z>co}_tSjO zochLbR*qgPhn)sJgp2u4th=8ch#w8@h--HUtNg=G@Nr{__iXPy&@LB#jf?Qkr%{q{ zGyC8`u7@8VXa^3v}{C3-c=?(mnw`Gm-yURJ_ghHRXn(g&G^j)$9%QFhb|>3boL3Iw-OFUe%$n=c zmB)@JcIW09oCkJ4y}K>#ZO%PgX9cHt56()(~ydFQ7&NAyig_XSV zpwW96*Uws=G}&_lUFoXQW%qq-Mc(EHPV(HRD)L+Fh2k z`-zD*!JbDZW~6(mbL>|COZML#SK`0B?#Jez-YiW94#7R`waMQs+JNNA@FBLZD@$HG z1k%;BZGWE@t~>o(eIp9mXn$>KoL_d9{j_`cY;MqJTvTblg*{+K?d@3^C>#Skg?(tY zof_D=QhFE7!pWVp{f63$vy*HcLVowDvsT%;RO=zWD^D6`+ZK=Cw31tA+n&iLKAfJj zZoEG|q(6?oefAw^9qDBVmv;N1{PM)1HV4f(kpK6=;lV@GtuN_P;x{evmiv&nZ1sib zV!N_g!0A}HBDEf>)obmAE6!SVz6X;(9eR#kj)(un64W1N2iCguCMnj<*LT}xl$=#@zB%@L|3j8$%geZs@{HCdj}_>eCHZZEJ!+CXfB3TG>Vk7u z+ZMWZ`I_|BrT>VZ++ARYM%%~PgA^7lUtM`F?q)q;6F{}SVv`%ye-2s@DC2ep#q zuLXX76`-r|3MjPJugvLxv_!D`&(OI|$EE^ch9_GHL!jD&4g(ZO25~euAY4_P*zYQYfA%tJ_0I^N>@EJQg zr`6W?opgbw8|a|@9j&m`ixiWkWJrKXRD91 z)+Wz=#%nV9$7hlW+)@9}Ls@@eu=U`1gcWut-cAFPe-#EbTWLQc`1*6*QRS;`Psjyj zKS-auc7>%+Zaiwmd54uGPaKtAG$$V)bxzfp6$K@}PsVMERp(bGcOD&{ge4Ez{9%LM zHlw8LGj_{=j;;9V4?K4l+Hp6zcaE)o|3p3YUx9n?8!pA)!SC38{7n9)GIGqy74}Qj zWyhwE_tYMXgE^k@!g&Rge0a=?v;04j?;UHW@K<6YEgxXAHNEM?U&G(d^lx{XU5|a| zl$KX5E5|LIfB6`_dV&`5t}+(TQHP z!|N#W1O3xQ$)u&p4|24l=5HGJE|R{_Tx{Xc3}1s)6pNPYWA;4t~=noG3{2`^5mW4{OTlm_c*J-8bs@E$KTHM?|}UeZdUL&x8%CHR;VKmPi~y+ zOHfY|e0Q#GS8G-;vqM!$a`Rl=-}YaBKR7OZ6yZB_E0PE7A=D1L4QiLF`Ux-j&KT@< zlJ|=EoI$re8+jIf#H+M(opaoOXB$X6{;T=t_FosSsI*7^uoJ&|UXrye`M~q3nV05MKW{CbWxQ=(GB$l; zDY??hRBmhC@@474+zt!#qzUClIM;;2M_H0@%RcmSxg?BXOE$D z?3PpajgE}2acy~xV@4)+XSGgjAJd)PlhM3G&&1}8g7%ET-3PLIrnOF9)iJg|BmJMs zO=D{_2KH#0)St02t2<{nvp2JF!!AQPh3oS+OemaFKc;KU(Dn^kJ!4zOZ^*36s2kIo zQ8TeSW6Ri5yBUZD0Lg`XBSt|FEyQb7tgLR9`h_C}VJ3 zL*|mI#YNNlCTtitG^TZ2XU6({hcYVHk1fj0Z5=Z-E_Z5GVNq^#_CV%_%%-fCjFHSu zFK5hJG-Fj^ZB|y!$e8A=+}ez;F?|_*85=X_)Mxc&3})4j*_f4Im$fmoXUwYM%o9W7oqOBNOIr$}a4i zTw@Oww~x!|%gt%XXdc%%WvsL)*XPt^_fBmZH)BTD z$ds(CmhmkUT6WnoZvAdKn^kf?18ZEn^CLCk{+$%j%!7VY{l{yp9P&yY=LC zXZ2>~wB>Bc&DlJzCZ~RU%f!v&YRA@W*N`)4cq#;lPs{bRc_2C_G2Rgdf4p(ksLRY>0+eVLt^>vQUDo`$ox zSe0$c>Dsk#T)m1f*bQUb#t&z29@mq#Vcf7S_^!;Fal=_P*)?M~XSeLoH92ct zP1b_VW7iFgugvTpjZEmtuuZdne9x@@j5gcW z7TYc&qamX)V^+0o>hm^bl&{a+wEa-#$ixjM|S0tWcQA5o7gkHVTX~-&D*tRv}879k8D?)Ri9OtwWL3@I=ek{c+%jGOIoccws5l> zGwL&{cWtztvk$Kyo+R5FEwWyF@SpFHryS!vNbW6i{)ggUbj~Yqf7S`kyOC!ube=`7 z`ik@C6kqIIN-q1F^F`!>CC;_v+^;*|NNz22zC-bEIzLRVvKK@!KRx8`tDXN$u5NJt z7kPlZ!x26|Q*ZM4-sGOUoeRi$4?BOJT>7Z<68Xw42)T{?C2|LOC3zFMj=aEDVw8U)c@epf zTtgmL==s%=4PGMkynzJkgq3~k)I^5BL9h8NuK-}pZ+@XG35RmJ-^e* z^KN&pA`iAW-$t%{-1%4J@+X`>BKMMaxBZY+ZBqS=$LEq8HaTNIg?Qof&KHws{MPvg ziil^2YO=bB^`=d+@%PO@CkV`gP97lB+IuKAqfurE?v*^D5_! zc<1qRS)UV}k0H;;`+hdPbI8Nw zTgc@*c>Gu7tUTxU$y+8lPd(nJzhx)qdE}{kIaiR^CbWgJ$cHx&e>n^{JP1XCg-pAcp15o{C#p#;qex7?;7Xdkf*J6 z{+L{TiSwuCdwz?5;CvD}zrp!3@|^3PZzm7f4Rg$22f6)L=Xc44w>fWb=Yd#1k~Zgq z$%`L$UO+BtcRq`}Ws~!F$X(AlKR~YQc7C1Qve|jOooA-||CgMPAa|0Nk=uJbel@xI zHRqp_TV8kmwc;N+50VG};hgs+&oA#k&V}T{40~VJX1kO;2k+DxUrjDO!TCON^(oGq z$s1QXZ@0jwzvxoudE|}XbH13odtWBsZMr{5-khCg*>XXRdcXpxCFsnOsb+z0Knlwf2Q@|KOxmys9#!nv8;^N{oNtWor*v2{0uqoMd!bevtD-I{#4JexySina?xwfCzA93 z;Cvyu>L7bR-|FXQ=o?iobHhI;jJid@zJKMRETzaT;3whCr z&c7wMlHVg2EcEzpU-SI>Pj;S9t~=HFTypDa&exFpmN?%f{)Y3@Zz;RRRr<}&za`Il#d(OFyyiUlbT7~1 z*PV|dultMhY2=OXI$x;xd(JnIlMkF9CRhF4`7QFuN6r(Lcz*MD7@w{m`Q%RW@#L-@ zJ-&oIWs>uyTk&G~3@4|y56eJ_t+OP;m2^TXt!na%_9 z4|AUU4WEAAk|p|= z?;+QI&-owZ{O>zYFZ2B8*aHeke>}O@9zy_^lXLA61@M*RhMS!4CwJcL{0ezflk*6< z>2~KC%RRq!EzS$c-S$i-rhh)U{T}BV$c+y;x04q>PZhOZ0 zL~`A8&gYTmJnwuBxxd@_VRF@r&aaUxf9L!U@;dt|8s*>hEYH90_s++V8~dC~$V23M za?c+<{s4LLTh4Eir@Z5wSI+4VI?pAq``CFUxrclsxz>IdNBKI)c_Yqmlav2AZ@0py z-+Cp&u@sFPtINGo+(51-x00L4&yqXHZ;|`RACr?kJpY|m zGk@|CG+(G^&xsUudIoZ?mpH#v8$@Y{&x;+$;Pa&6*uOL^G ze?@L050N{_lg{($^^r@-$zGoS56SuDc5)GUh+InEZ;el{n*23#6Zryi2l)r&KJp{v zB;WIYg`7_wAQzF7wVq!oc`CV@TtIFjmykQimy!F(w~~|Tp8u2NeDWX3MdW{xOUZjw za{kH3lAFk9k~_%XCiju=Bqw`&{*RIK$#0X3$T{b8{>g`stI12qP2`Kn9poR9`^di_ zC;NE*uafi0?~{wjxfd{h@_yuM^5@A-eusV2 z$@9obmZz^E7m{x>-g6hbubboXN6BU6Ka$JIW54a`SCJ1OSCE&HE6LZAtH@7~*O7laGobV!Fj3pE6x{_FCyPgUiYlW zpCMP@b!s~Ox5a0lp7KA*Mc;JJtMT%bU*){7xY7A&@tw{mkp~}iMje#l4~H^~mkI3i ziiqzpKGRAv<=z|Y%-){~`IPxZ+xho(2PMgjfHCPAHs&C)4dCJ^4u3xVXX%UO=N^*k zqkb5NKl(h9l@Ixg<#)I~>W6XodVUGdGrIy@Vr-=KUw}9e6ax^#r9`r5Qnel!SH+-d@<5v{_Syq zbO6H-!Rc3q9w z?exX?Ka}5QD;>b_!#Mm>`gQch^4lXnjKiO1*V#z>2z{~qLEFDz03-h}4nNnlM} zjKkOSiFjTSz8LA#@ERbe`@b*_e^#SUyH(4dEWh8TfdP#C zl^zT~?}r|HjlLLhEWb9&Ka9g4p#M+KzgYf2 zwEl*1_D2c3-|wXpyw}^Ul47-VI01m z&&Bh)D8E?#!mMrUKa9iI^SpSz7rt2jFuvG;kK!l}e{qM$wy^wSq{sT-i!V0d1AZ8X zU+{#-rn3ITNDsfzMhx=9IQ&)gOX-W{Zmn^?Bsz0Si9Ddyk z?$2cYV)=3T!#Mm-`bG4`@~1`TUtt`6-|swsdu%l=zgYgP==dAP;pg_be+qrE{J8vK z9R57|1@y)8=XmvxF267izncDq^u_Y?@Wlpv6x*LL4!`~lkKJJ3vHZmLHC=z^EXB9hye^gjKkOa19+bR zz8LA@*F@V-7>BR-4eAN_CI!b1Iv z<&Q-5AI9P9{S&;80$(h@E6P8N!*3q%&QSim`-`+OuLEB!e|D6A z7>BR-dGLM@e6jpE|1b`}k@JuDf#8cj$q(c3o9N^HA^74?^20cMy@Wt}0qwOz@ z!`J&%c)tq1Sbn_x!#I4se}(t4;EUzQ>t7g$ulKd^{uX?({5bzG4qxwg;e9XoV)=3Y zVI02R2Rn?uSbm&;7>BR-$M8NG@)ygG^AF?j^}ZS2KZ7rpALk#&;p_c0ysrjdEI-aa zjKkOaYwP@Dp9f$3Nq!iIulM)xJ|BGXC;4Fc>53I z@bx|+-Yb_WhT>dZ)e~A8>`?)WcUy-|=|?~3^sBR%HbF89&_3_py+@3B|x5u0*=`(peb z{K4q@C5*%GTJL_&H21~w8}P*jd?5cY4u8py-M^3ZCq{bYzr;oi^20d%Hu|s97t615 zebf)*@MquR`9Hw*M=XDF7yYt5S$|~tR;}p(E)LRz;rBEB z_4~RnMjZ85nr(k<0K*UC@bx}4-jBxmBSw1o8=~bO#^LMzX}nJjUo5{8Uu?h!@(<(i z%kJ{}e~jf9BR%qOjq(rU@Tc74{&d#ASbnEX0|OZOhjI8l^xxji%P*F{ep0HB`e7Wt z-WSLFyiFP2~F*^lNQ#^LLIbiAJqUo3xGRDWR{{_s;?f1l^| zpIClTlz$k9U--29Ros7z<;VNaFb;nn{cE`Xi$BQ^CXZ~XOv+>0Sd=v-i!SMC| zKHld?{fQCB`ZpBSUl@m9`D>s5r&xZm{CNJuc$z=`c9y@IEPtw%0RtH2599C~nf@L6 zV)^m@FO0*_V|xW{yZii$d=J8jI^zav~v_Ce;599Fr=6ZVvO*1`zvHV%CkNRO8e(v$^C#*lQ{6&!; z#^IOJpFv+NAM=0#TpXka!>^%#G0QJT9PTO`F@WKRarlGuk7xaf<;UA!7>7S|p3nca zmKm0xSbk2l{)BP(vp?_t?exX+v(nOUEq@q?UqJt`>7Kt>eti8C#^KMQe>RW*V)^Uv z#Rhy72kF7^+rHrCFQP9-9Q8jVn*T5kzxRvoKgszQ%a8XzVI2Mz`mfR#%dfREU;v~1 zVH|$N37-G{EWcQO{QFlJhu`)k_rISeYTpL!-C+F{BR%rB|4s)m{4fr`obBU`WBJAM^P=S+ z#^I|y9kj25{KfM7Jo~NXSNl8m|M1oR4%*{^Kgt$I`75IQ!#I4k*Ms(Z;EU~R>c_vI zhH>~kr+EEm>im@~KYso|=@Eyo_JPn|5b_tx&$o0Kz{OD7J2zF2r%Kvm!r?!=FvRguYmQ)mWbZ808P+@YOyQ+N;9+i;*77FW!E_ zIQ+I1KL2x>zgYf!&wjN2!Z`foZ1+!{?)i)5XW@$t_(1u?IQ*=Y?(b;xiSmn)6y@)T zwx2K#zw;dT|IYh2V)^m%3*+#6=ocL5`HSVp$GsUo1Z^ ze;9|Kb*|5U7xNd(kMAFZarkP_4DFkt{>AcV*mi*dTpY#WtNk;yhX!Aa^jLoJ^HX6w z&A-aaU&;C3NS5Di)4%{u^AFbO09z>A~;^FLVEL`eMYf>?&=<0EQpN;g?dx2EWb4^{nqk_arpH$UVeKGK3)D|`4#wL13rp_^kDeO z748qy7bArF%fc5M@Bu%J!_T_X{TEq&F_OaXvk`;*Fb-es8KQkdtiR$>A727LjKf#^ zhiDHGzF7WHd!oXYsMn=#qzB=%X9!2 z2kF7^8}4@hboye%vHnbn?jMG6_-elr?OCGyV)->T4Gdu9AI9OUy-T!z315u#$bUmr z{xA-|$6BUG+Q(RavHT<|e;7~w2i@OYK3V?aEStCuV3fZnh{IR=oM^8TBR+Jkh=cl-xi6N#5MOM-2l5Z&@VE5%@;j}_eKFFb{G~QxkRQh3&wthZ)Aw>;EPp{X z|6v?{!|U$vPhTuQ&$A!RKa9g~rvD7Df5h?&B0r47@1g&}o?d>j{HkdE4dd_^zTxE` z&+?1q$M+AzIQ+%*EBE&N#q#6xmoN^$jQ%A0V)=d1{D*P)YQOiQBF|qezdiE9IQ-;I zpZ|CFbzdw$K7S75@O$Y`<^5x^{8>@{VH|!=zvti0`4`Kd9r8d$|84qW`SJbhFb==!Z7=^D(>;H&eDq9*0bCrU2g9HCq5F3hxi3aM-T&Ap1~B|E z4qxpv|F+0|vHV)dHpma-@Y|XHNRj(u`EmV+arkQgdFSKY7t1fO;~EAq${)tztG(zi z9Ou5+Lh1a+=kH-0zS@_*guYmQ2fo;V59A-l;j2Arv`>xwzu3N}`Ikrg?=TKu?N_5c zYxrXMarwhI{1&!%J$|nHV);$@Vgo)<{xA+-?PDK9UySrvf8zYZcv^n8r;YZt)AEz$ z$M=ub{x;(9)&BM^^u_YmTRIG2ls}BaS9{&RqA!+jn`b)6599FFzW4j|#q#6x$1tAy zY!4jmgJb^HKDhja(fyY&4!>}BZ|^$Op3{IYmfsg0f5JF?wLgyb$l;6S_xS9O)?XNh zKgjmWuiD3bvHbY_Cyc{ad*^8X9Qljo&&C%U@KGGa;j4Xgw3iNFjPzLlhHb5nS{7upN7slbwKFIUm zf%%K&$H$*A4!@lKH<-UzzBSjE4&dS-Js7^)qeuJnn13 zE2H`kjPi$Z_#;JLe;J(rQMU9be|-KG#^I}deYCfa{KfV) z^-H7s_hB4<*%v(jgP6Zqew=?8hrfz`CZAss%Wtr9U;q~f>A~=m`JVr`c>XI!2=m_@ z)n6Egul@wkzX0Z6EPpD#*nkh@AI9OU{{i$z0AGys$bW;47{KtuIDGY2fc^{Mi;*6F zW3>K+aro-r0R0`n7t1fqvj4LIjQqnmeD#Nb{t@7dkreq)iSiHQ@E6zxF47*)=U>J0 ztvT>?03-h}4qyE*pg)GR{@5Rb{P_5*^oYY>#r_(uWBy|KvpxIK`48jp)!zg9e?b0X z`PEVVg>m@PR`~ocXZ~XO)JO9V^WVjpY~1Z;iJ9Fb==tBKH&9Pa=P@{Qju^!Z`fqOWYsf{EOvJi~KO2=6{*{ z6Zrgh2U&i+{|@8u)t?9Y_dxl@@|W-e6kHs|;Wu69`TyJQ)4~@cJ?g(H+W&-c_(kkL zDFv4qyGBpg$DMzgYgd32E_1 z{V)zc$0`VE_hbHI`EmYX9DXkSi|LC$$q(c3^XUI^s+V6ZKYo8ZjKgoU0}RqWPG2m4 zX7;x8AI9PL{nY&$EMw{>Ad+<9`^3-#~w7-hUFy-yGFn7>8fJ$@9N>51)UreCxg>9l*swdNBM# z_RrIo@4gsuYz#@`rKwrSx0ri{(#^${)tzt3M<3Z-n_5%b$raHsAy0599FF{}K8_f-gpT z)PH>cGK|CD+~@OuQNH_P`MuHd599D#-**2$JG(EIzbe{)g>m@muL=D(q5NX`h0*mx z7>B>;1JC~+<}a3C7L`AY!*3gQ|6=-L`PQ9HI)ICV^kDd9|8akc&1bs)k`YJ!SJ{XG z3}5NN@Ozp5Ip!}$9KLmjkq%(^VH|#8j`w#|z~jGIeo-|4VH|$dcJ5!r{g+sNWwiea z1X*=fA=@{DytqFQhM)za(0J!Z`fF{oTKw zzF2;|{|)2thv}b3Uo1bqehB06XV3Ed|2o;{Uo5{oTK~d0{KmuFf0g-*<;Ud@m?mpZ5ImJ0sSAvHYy8ZP(v04!`PX_g~`rE0$j$`C%M>j`a|aw71e1 z%WsL+|1b`}b)NhC^88&aKc4?E4!>r;`yaFZ#PV~Z`U~Um)j!=*?*GK{?4hjI9u z=-5Jv}M*H6|4qyEvqQ6AsFP2{vm@mzwu+{FP5L5>oWi@4$_0+*Iw=UKefN-FGd{OPrZ#8!0^L3{Hfn{f4`aTi{)qG ziw*dIAI9NNt8@R$^u$X|@~D1R%y*nkiCVI02t&qRNk@Wn_Ezb#sS!#MorAA0?rp5wk) zetxw7599DV*SlX$Uo5{f%0G<5uf4_n&gq`NSbqHeLl}pz{zB1zDCS=*KYsrxjKf#| zqUdiFzF5B9UQ7pYaTJHY{thpH7LOld`CYD$F267izwA!;XVMqTABgHNjKg1em-}z> z_%D{fDDuNN{N@er-#VSkpDe#Rnt!E79R7?}_dCYAFP7gMZGT}LzWQ_B{)_I5@7aQpSn z|Fr0@7VDpQ)W;Va@PYD&aro-b_B-^&NRRS&;)@OVfFH);tN&Z{hl~8hNDn{0{~E^O ztH0dW>5JvZ`~NTwzmWaujz7Wk7t4>&f5JF?^~a0;c~O3`{NZT*3FGk9fA2E-V)u+@Y z4&(6K|LXoZ^u_Y+_DVW{k$)J6zy5FTze8UvKPS3=3FGja*&j0cM^5XHEPp|iztSTP zU;QVeKV|r0`T0@(g>m@mZyEhB!xzhs_a9*#{&2$nkS!nhV);dB?QLCtVI02tcP?c9 zV)>Oj`UJqmL3%L!4U;_ozgRsXe=*`%{)JKfhjIA*ySQ&ZW2C-V{$iU31~Bpu7Tt!1F(Y`HSVp{SSn3_*0K||CjW|^6j)F9l*#xjKj~T z|Kq7%ezE-c`ZJ8fFF4lozkt42eti8M#^LA8b^mMn{evujAXvK=Xx6`2{_H6KFb;p?N$%TYTxtGd z`K6H`#^Fyn+5OK?a9=EcRaE{k4qyGHqyKcwzgYgdsQ$w^{E@GD{y%2@i{+O^&o72? z_@yh{w{GaY{>k#=``=2BIQ#~tw{C?}Uo1aAn*T5kU;V?Qzj(~QSpF2duZIC#9L3>x zUE<|G&+>sUMtZD2by5Cd9KQOaNB{Kj#qu|B0|q1iFb+St+Vg*z`HSV(M)`+v__cNJ zA4^{>e^%s&arhg*=l(pFUo3ydL@T2WV3a?M!|(Z_`>S~UE|#Aat^Z*he(!qsf5P&M z@Bd*Oe*UjL|6N#rV)-+p<7XI$Ki_^}MB2wV|6=(q(fS+4 z;Sct>|0nul`90C~Ll}qOWiO5)?K0cg((R8de=uHtK^%VDpWR10Iq=2uIQ%Jp zb^ihSV);q5{)X|?f8TxUHa^XN3R(WT*bn0H>xbNb)Aq9{zgT|!`*#?JUu7@OBkeEg zi{;zTYv}+k4$_0+SJ(*(zTZq=jQ_*_FW!HLarn*jAGY&6lwT~rDat>L!=IOoP4mAa z&wa7{mT3JAYKf*Zt!U^tY+I|k@7t3E|*992B#Zer7bDsOJ@c1p3 z-yN;LVH|$tWcTm2?F9LY<;U&EhjIAB_5(N4UO`_hzdfqIFb;p?p6;KO>-mf2$NR4^ z4u63D)Dzto%eU?s(*ax@qzA*FdVuFYhxv;U$NIO}Mhsy1VH|!1{qNEj%g?p_DhBys z9R9k4J^zI3zgT{}{K7c=1+(3Mf%PwzAGhBg#^KLD-2Kg5|Hbl)($a4&e;9{fOaD05 zzgT{J|1XTgANaKAKa=w>mLG5bVH|$`k?t4M7t4=-{|@8uOYOoRX)EcA<1|AV>xi{)b-!~iZ1(u3j8r+=xI9~p7D<Br%{9^fAqV*??!(V3&1R?D%tUs~*c~Smh9DWD=MxMWj<8eQxBF}9i{;m$@HT)^{xA-|gZ>Zbi{%fe z_EtZP!*9FC^KYjwmfs)wVI2PYd)=SO?N2PZu<@E5-6{>k*k@>fOm7sk`_(=Sl@$)o-SJ^^sC z{b>*4@MoSjE?u^BSbt*qS@>cDKH!IO__I!Te=&VA($BE5%0>+G!#MnA`roE6mcPXH zQ9q2sPnLN8P4va`*GGOBhrgcwqx8k{iy}Xa!|$cPg}zvRPvnPj_#40O<)8RvUw_5& z=SO}Rhd)Ap4t=rwvd9nP@Jqkp`7fg{mcKFb!#MmYCGOYL7t4>!AI9M?q5l|tvHUsF z^-maw-&p4P|CPR2etiEpjKg1lmis$?#p_=zzbMK-jKl9;<$e);vHbY=+b|A)roAAD zw9Dy><=5eh4frUwKVcmH#`D~-r7uQ$Y=4_=#2`P6!>?WA{v-6o@;mV*@WVL#1=c|& zlKh#zSbn_x!g!khMegr#ve$ntS$;jf*np2>`xD0DPyLqr1@y&8kNI!05rh0N4!?u` zDfGqiE2Ha=Fb==q63_nv`eOOb(f!LX4!`G8_kTiPEWamf+wzBT_^nsC{~P*Z`JIs; z#^G0f*Zq&_i{;19Z-;UCS@rJkv&fg9SpMcH|1h4;e}nrc((fV5kAMFSBT@ zV)=8T>(?+2Ke^TYpU@Y}&x^MIFb+TO4)=dUUo5{an*T5kfBjwVe@tI2KR@!rIQ;f| z+~4OEuYa-pifH`{wg%Bzh$HQ<@Ck!%OgLG!(aFd_b;O_mLLCq z8OGuFJmh{eeX;y`QT|~Z{MMKa9g~ecb)2U-jiD zmLI==5XRxRb-I5HeX;!hX#T@E{PoYcUrJvre_B-jFrMcBtoxTMf3p1e{pYqIp636& z`*$jTvi!<8{~!)O??v~Yp)Zy{6wQAahoAMb`+uP?mfsZlVH|#QkNex7>dQ|oe|_YK zari~AxqmQyvHZSh{=+!@{6DyVB7L#^8ho(a9aL8_b2oEp_z>M3>!C}V1I0o zAI9NV9h8&CE@b{<`SJaOFb=Q!w{Atnp6UO1!&USwxeX;yF|1b`} z^icOJ>5JtLMbEE=arlc)biakZSblu{7RKSX(*G@evHbY?g)k1kV4> (}R=xc%%f z4!{3o_jmi6FF&#TmT3D6p6vHTg4AI8)9Kh6Df>9>;Q&yJRV7>D0yFPI|j zHT1>ulgJO_ss9c4?~+fJKa4Ln;G@|7^aXMFJ*DnHO<#=kSpVbu7hxR!yru5%GR5m( zEI&T~4dd{qo$3DT^u_Yy?Kh0WFQWe;%P*E6-~SHd@E4x#`CrZa#q#6)!#Mm(`oEwr zmOtC-5Cgb4NDqeJOuvl27;)qspMQsO_*JXD{72Cj%eS9r(gBS8!#MoD^WCqU?8{Fq zKVJS}9R7j}+`oywSbkNs{e*G&ofom@n zu6F-4<}a4t99=(zarpW5?te7N^B2pH+dm59@Y}9;|7hkfmT!+2Z?pWuIQ)ShxxW*A zv3yqF==vAN;SbS&k>wZ5$8&WUz{Np&F#N$=JpViBixJ2A8|NR!;SbZFn(y@|9`&R3 zH;lti?(qB;X^2zCmA%|HJ&} zM)`+v_!alL{}7)a63dVGUtt`6_0Qek`xns{%bykPKf^fubIQ+%0yFdAKU;bkG z8>0FP`=`+t%a7mR3*%}2 z@40`W@+Zr$iROP@5Qm?9;QkHt#qz77@`rKwReyK?VfteE@%eukhd=U>`)|<~%a8B> zhw-%h|8)NuZodgxe*FAN7>B?4U+yo~{(~&PCEEU#9&z~Vvd5?UrZx1%^1Gtvf5JHY zc{{k@%=V+i^11nr?*GC#{Q2}J*o7MQe`5LTtehCY#Zer7C;fc-V)Yl>5JvZ_YcE3{Pw*( z|7G;W@++d{AI9O&+S~nW>5JueMdxo}9RAQu_aCM&mfs(3KVdwb|HIrL(EO9-7e>dw zo*)iC?@0G2f5YovEPruy{uajJm(OwkNcv*=%~Ab@arn8Pb-$FpSbm+Y0~o-?L3%L! zM*3IL7bA}KXE4e?jKi;;>-j%IUo1Z_^22zV|2+5qqWsD7i=yqfF^I$O`GWgXO1%EX z^5;eQhjIARPH?}FzF7XcX!(WlwEQQzUq-)?EWgL<5Cgc_{)BP(6${<}0e!Lj`1&V| z!>^+MB7N~E`C%M>HU0PKi$BQ^${^Fgj;%opH z2kF7^J6F12QR?}N5y$%5ZzBdU{4fr`WtIEuOWhaCZ^IWG@Bu%J!!JG8{q|D##Ym6* z^P}}IjKi<4aQ{vEV)=GkGabOlKa9igp+9M<=PyQj5Jvhiu^DRznA_S^u_Y4BR`D8 zUsCDi&pgBHUo1ZtUu?ifagZJizx;gn51=nb9LsN>jTpf2!#Mm6^cT?=%iomRTm3K& zzv=?d{}TFQ`Q?!x#^LwTzk|M5{(5||0Us!T7>B>E%JbhuUySsa{{_+ca~Ox;^=;Pw zGOvHJ{J8(QFb=={Qup)ei{+0*`~NTwzwJu*Po^)HAHP2v#^Lv0<^EdwV)D2XjQji27b89V&C&H+7>8f?ockxz7t3$8XcG|3P0YKfeA7l|O!#MmN>v$KjdirAgAN=a5{9zpa;D0OeE4=>2^5g4|Fb=0HXJxv7A$_s@>Rg`y zxHw1;hF`OT`}fipBaZbiE`Jz@KP}JwKKf$$^P=-7#q+;qeOB19t>ZBF7n6s{b>zIm zJ-+X{B+0k(X!u6NKQ`WPCq#48zg?uf!evGGY zBj;o~_mLNpceuo--$@^j+$MJW%EhqpLSxZLv_9(zg} z--leahx0t~3C>H!Uva*ed=dF}^15d|{tUVLu2a+Lzb!ud^pyWeF8Zc(UX7Qh{3_>t z#f{EKi|=$ki9GnAbA~PNGW_8%#y)qLYS(KyUSC*0z=d4iSiis>jAMNQ=P{1;2V6?V z`U!4i`rm^y^6al4+y0g^j;GF(tSmd&*!Mf<0`j~*=TpeJQ?Li-6g4Mp@)GTMUcQ}`<_UX`G6Vj_a zCbXXhR{Ln(WVLsO=V%aD`%`Ey3as{^(0&$J?PH-mY_Qt9ex6)-n)erL%^;=mw(mMy zGe9X%J;Qm&vCdg5o#&9%ekR(>#Pro3Cfa`mt9{oFvf8^v`EwpHozZ_g(yM=VyR1)n>d!p> z7+L+-+i7naudng=Fgdxx8U2}Hdg`wP?E`|<{@)66>(@Q~rR0_p=bw=0F^=bp;OqGz zv{wXHdqime2(0#x&>jj{?V;fJaj<^>#_!!={hp2If53Xa2hZcA=^xqp`#e~`zvFou zu%4$u`+s1y?}zq)z-sRY?Gu63{t(*x0joV9v>yRh`w(ak0<88P&^`lL?JpFQ)t&{~ z&q&j=Jq@(S0akk(Xx{^@_B+sCNSdDQf1v#lu-X?vdrT>_eI~T;1XlY^XfFt?_JGj- z5LoRCp*WQr=O6Jr zCs@yG;`vIjo}a|?nqWPTiRVYbdOj4-(}MN9ES`@A>-kqa?+VuQtmuCRto~*2JT+L) zOXK-ou%6$=^TJ>~4~*xR!FoO!&m)8NJTjiI2J880^j88_etpP{avDe z$kb#@u=+p3^PXTm&xz+Z!FoOu{n>%lUmf}K_sP$$`~h9G*`F>-kf(w+mK#x@f-_toC`)A0Jr#?V*1^u=?*qf01DI z2Z?J7u=*E7d%P*LJzn%L2v+}r=r0kh{t%xeZ{hJA&r>6==cP04+63HliXUHatpc9K z^NqRWwiO;PA6^9;H9b@$&U_walI{p%vV`qxE&e_-|JS3+(c z@9P`-(?VSRWwnv>cs}xbaxc^8*tHMRci9hK_K~1JE3o>jLjPD`^?!x_roie?3jJ4s z)jt*ba{{ZsCiD*lR{u}vuL!LEi17XvSnp?{JqWPcdqDdVV6`8C{sO`3FA(itr0Lln z2HLX#tGx=e&jD8Z8|bePto{g(BhP33l##o5K6VXx-qT(`cwP;@o=1C%T={E{e?acx z@*KaHr(gJnFW>#i4U8X4?s(GUr<3Qu>Rd%GWd7HX7m{1ZwOoJO$<-{+JLE?4)O;_G zo+m$^to{WrBCEf>R?5l`Mf87&`cVIdhmi|D@8v(0oSf~9_Og&(?P1+YR{s>} zPXuxG7xD(V>TUNY?d|b0*5~KR!%unrSCSVqehYclM2~lqbL;^z%O=U#$EThS?-yo}h@~g<|PyQjLpXvSay+u}k?dTsJ(^LQFN0HSZ`!cfHgZ(DCFWbv|EqSIr z5QOsHPM*#BeUjY!MUTHq-a`Hxx#~oZkK5PFQ+|^3{^W`ioQue9UvNH!+_=E`Z1NoD z|81rJlE<$n_jCJsj9hk?&(FK$WEi#z;&&40+fzi^&_$erX;@+ziZM=rAK zAe66;T)NV^kz9MW^X=qWjJJv#J^l!J+A8N~$@4k=Zt|8Ndi*tV9{C;eM)LdQ;Rika zKgk0>b{;>==fC+!&XdVyjPGx3^Ob3{mJXkbczMM25w}I$=bWYKeHihSgVX%RC_dYH zH2sMYua44Re6X^i4`Mf}T%yCQzkd31VjNAV9M{&&P#hiqFP6C<7+@!k<15OGn&UyXQG#Fs{V zRmAlXH%8nP@tqOh6Y(!1el+5zBHk473lYB>@f#6uiTJ&UKZ}{9Fgor0ayjR4t zBQA`%DB}4MFO2w%h*w6uCgN{7kFJk(5#Jv1V-fd8{9(k~9h%n9==67w_>hQ?iTJdL z%OgHN;%g$lFXAU7elg-VBhEc++vPhc;^`3|8gXI7#St%!xH{s-i0_WLBjWCe{}A!J z5oa8}?fmT$@xBos9C1O!$3%QW#9xW{^oW;7d``sYNBr%GuZ;NGh;NLzDdM{#Zj1QI zh<_dNOA%kPeUg07#_MeSzKuVyvBAdcZM@OOAK7@bjX$>W78{#vyw%2^*m#?b%{Ja{ z;~h5MX=96xKeh2L8-Hfw1{?3T@g5s-FSpgk`)s`5#s_TNXyeaqe9*>U*w|*{LpDBa zW4n#NwDD0JAG5K;#>Z`Z!p0|U?6mPI8=toE85_H7{FRNrwsDh;HH+{P{*Nbzgf@nlaiC>o^X80@e7VS`Q(xrCG$=$IB|L9(p6tB_*_-R@->x< zmY%U{`6s4Xu=JwkYre94?TWJH=PtYG6Ot`je(u`UYZyH7bBm7qf0Eb#PWM0a`(H_w zXURF`CCjUpFFU_-c}e;5rDe;l9A#?@O3IcmuUNZ$`O1QlrInS-&#hds`rMK;R-b=v z+1fb1;+&;tEiYNM^rF@0SC*V{{)$y)|Hl+(u32u?wQUwYiHfC_%gUFpE&0Dn`wD<4 zm#$$z^j^C=K*Uy55EZV4lqerccVc1(c8ke>X6AWz zpW2o8|L*(V%kIpXGc#w-oH;eKLJ(?syg@|3^jfi+i$JTBM<|p^ku=JvjwY3nTBArT zlevjt^q9J+OR7+$Q|U!&jZP(slPe;l41zGccDhKFIGzt`B9BZi)yN1H^?wIbDdOca zkp#^d6TS#dbmF)oHc+T#u&6T8|I&$Cqn3(|da+Ve^nPH;wXo_+O`IsEt*EC+uT)6o zR_4xx@9=jxjFTdf^nU{9EuLwWs3F93phy|frd<&*)6h{x83qkpltCD@R(hw|S|!#; zn^qI;QftgYH$tmOV0B_O>`=?2;kWoZtll6tD5MmF{cqg}G_~ActU_-DmMjt@t3@#) zi3lwy8W7&(I*k@)E>5=Gqf&`YLJf%rsZuUh8?|O3LW79JQiCFvl3f@Q z+A0hpXfBEq!w0=wDVG{xopjKxurMG1Qzr)^uKD}{4nT(woSz_FnZ3m#S)C^8tQN$NN5C4~%(nX6@@E=VqFpxR|AMSDyh!ifVWct{) zh{v=*kT_9<0y^&5Nl~GN;0VQa z7*5h7N-#l-`%;{9PdM1|%T6OC*h(5s`# zVPlzMT*$v*Ba2=nbe8!8o#)uPB}Wl0prBN!cgrd=A{y)xq|&N1w7w|Ax{?S4s8S4i zNLvdyIjw{{HUU{$d)&K@I*}}&A7qp!IHAspL<#~#vn5~8K?EUMFZfK5^M~4j#FjAa zBBcg20--^GC^X7VJc`v6ved#h9qNwVgvlPlZ9x5kp`o)JoF5`0>Y~KT2>g{hCLv(! zFqUf>UHuKna=k!}X&-_Kh}9(lg97Hk1N$2)GG4q*gp@D}C1P{)idg_8sPqbTbd*NN z*O~wofgsK%*iCw)ifmAUK?;BhK(SCs#ae?A3>;7``ARMH&H^U*|AvxRPaD*jV3tLO z$?(Q-TrBj?vMlgf5PC`@w6O6xu(V-qsZ(se@GBJtvu+3y(rhXr{|Ofy089pzCF_<6 zLj;NtCY3lEjCqYlC5iy20Ekg+Hq-tKJ?QnIfSX4~sk4XywPJ-1E1i@>f%L&0DYt=@ zdSL^`N&`z(Zmv9nQNzHFJm5rHH1T}`!hjZf!ssv^ECC=3kW%+lS}h!3GPzO|tB~`T zl=>pfV-qy+2^+w^A>kO2K3c(W^iiH+lfdM=LBP(71|PS^06Q1FBoP7( z556I>QVGl!5up}|B^n(=ys+xP8KlrhnRhs=5^O_OL!BJ!6u|eC-eEuDV5U;BP5@GG)M{aLWNR>n0lqINl&dtl1mv7#Pe7+ZJut^0vgGJ6@6n>k7w*W3 zZ4)V1%dyLoAq$aui}|-`h{3f7J{|Dlz`+}*(M2N{h+YovOK`H8=M5B=i@}=b4g;|Y zn}9Ur2suy#A3E{m4_M*YJBCNj#H^Krw+`4!aN`1vscP<4GO0A+AqH2qaBDh{MPMwV zpyFKCaZo@!Bo=+>JmY}h)T)ZlXpdKfcA#rwca4l4zmxOmQDJwFg@R<_}Q5O$%kKzPH&48q|K93^B~_KcwmHNfnIw*|#6 z>~luQeSl*~kzm20ID%2lWJZA=q*VB=2%u^Wh$FpRhofMIC`4N8qX*RpHWrp?q9~0) z%?d9F4INTMWE91iTUKD$Oh}xEA=PM^%WJ6#O?yiNCki-b3Ob|<`j^xq5gG6(vcn1J zu;<8tr4+0pjDpe+iU3Jy9mjwq0)fYv5Oa#sY1Ax*g#d{_+0-zi7tB(6)0y#8wHyIP zl>!&SCj(;{O%=?AViXbuScAMWygsmB85CrGg(?3BoiG$OHfK_WNEq^($LSOX#>oMa z*mS5`KQaFX30%zJ84S%_ez{JsfY31L###+TnHijG1(d5lV^HF*#~OeE0>iw;fC)uu zKx|U1Xn?a)uYiu3qyQL2j4Sf#=^!;i2EKD;f(YVIkh~xOMqwh*D&YwrbR0O8aKPdQ zq=z771UZzIa^x09OQ;frVa$?t%&6lSr!*#oDn3FBsb=`(L_SKTkvq#6jy*%pY(ctO z0)<`1p$Qj~(kg`_o_LsCc%~I0$PLT}Du@_Dq8VjwF;g6PxB)a)tWyYurZFs$&J4-| z|8S;;L<#^za@T|{*OZXQLZUCi%wQ2(knj{VmtBChadI2hcDUIapr1+c zrqM84p_CiO<#lH%QpeVf>Q%IX)@ zlbOonrBQ6f76y!~VX77@DPJFj1L=Wm^if6w+9B*hqh5W`%`n0VNnD+Z zK;e^@G$k39l#v1=oEi$NnD_>m@aBX~ONpFRIk1mTBL$@b>>w(=l!*tOu~a=6G45`p zPN|7RSw04&{tws6BOyZz!bS9FI62UHfGfy2@JHk>p-Qpm1O{yb=b-|~s0I6^e0DU6u=}*WXLK^DDC<6?YqfeqZg&_*gFm!SR^$zsLsKOKAoKX45Bg96f!AFjC zFHYVs0tFEY!mt{7Sgu(pngU*&Ry~bLBO)OtlQROo*ukRM)~Fbk z8~l*~gJmDPB+>{XffPYbw+_w;5hQ3q&>GqI1fg7|1yq!`N@P%i(WKEBK$;_S9(Gkc z5|UzrMqwk8AZM2iz!Gf}whwd+X0t(o2m)WR%v6Boj?$TNYJ~{sh_|MUT@0EGb}_I^ z!TE+$E7_$)#z#>2Bcy7B(nbVS#-xnE`B3m3#sTae0bVMgbc7Z;FoB*CkeCdLG5UjQ zVK6`rvcV%zVmo|6e_&P-=;VSw2DuWUM}JVM0oDStgkXLVl+pfb1b?+)%@V^- z1^WduOd%@{ER_fmk}`xyATy-sd`5qf0u6uRETI3wL4y*I)JSz!0k6Pfzz4!`Dx;cw z!0V3D!D)?FgA_n&8D(UILcUh023iVMKpGjpCQJ^oLNyi=RHa-EY)zc3XtyDWMAaZE z*iSBwhBhEN5zFF!RQkwpIDOdP!i~V7u2h*uYY?fC_KxAvH_%OK?&!S>#-MwQ%JUtsXetH>F;vnR5Q02uEB>RHE0za zKJjo0BKb!*AZ0Xf(xr&&0fyQlF_7X6V}q{9CeXuSA`>MjaWrK0ewo)OASg%f?ABE65f*vbp-5X z7&8Rt(O-H9M!ga=5ZK!2)(F@>)F<=-wml9nfLx1`gO~%Y90>@dvnUN>QV&-pkX%46 zQvyVZThtb(J4sQM1k%i~4GE}Fk|4xqU>kr|09j1@Dls}*nHBWda6hD9+#`y8Fuszp zV6BJbdqo)|BEa89sXvr52Z|NW7DYNPqBXmgQIa5)NOf>jh;)$>3JA4D$;&uiWzu~> z%!2&FEClKh5LyX)Ni;&KfM&=|tI;8=6;>Z3LM3>Nq?nR2s1!(o(wzYcCNDL@zwEII zPDesm6Ju1!kP3jtg_#gm1YZI|1wdxRNn#a}*#yR1gtjSdeq&&CcM0Z+F3y-bHCMG1S+631lI|*!afP%M%|F}Tc{0~D3*p7f~thtxJcvU ziD)!RS%YLs8qmt&die_4qF^VBLGSTx}n8!u~H`jVImQjFenTPrXAeqNq}ocD6;~T6Y6~0)-SPe^!h@Vb`>{68Zf=;Y93_=kF!Qyy8^+VU+xWnWk1;lRP`_@6uB$)h= zvtcG5D?&SE8AE9B5aoX93WWEla~2ostIPNKzxmXz9{5+_KYUvM{pqC+z; zfisCDuqu#l!Kp_31sNQyDr#L2-{IIAif5Y}g6RN}5UAkpEnYJOXl@5QfD%=~`Gg$> z7MKCtfK*D1>CzGt1-@l%8W=11q=peO;}Iwanh{y&EV8)<+>0h_-xBR~U@)O*6DF5= zJQUkAaGC2Eg`noDAgu+gMlF>z#05h&$k+K72}Gm|j_gtr%vyk%W-9Ci3mLAUodFZV9m3WkJ^Ys$?h`^{z`3n*xJwSAlt>x#WZ4ZU>5HdB zg;*)Eq=WX1t;%kORCh7SHD){9*H@?e<nYKlR*N_pmEds|=ITF;>?II;TXkq^N#S5km!=4(=c?d94 zsgWQvS$MPrl9e5yfbgmr9IS5e^@xjIf-kLx{rj6kY1a0NKqvQEC$MaWQF0 zF$lEm$!=(#JOzS~>7w9i1jf=atI**AiYC(J;b2GiQ?PfOhX?7{Sp7o^KP2Dr0I)Ft zouPWBF+g=@vK54@k$PsKC3b$II@CSCTTrtRryEL=wn61_ms?5I{A6WMq^*SX5t@b>0b>RvBU0%8rNCfVNci>oC`APD3x1)*+!=~anvV%FV@f2~yrYZilFUaGG zf&e%8#@Ta1`Q~N{kf{lZL!27+z_LNR3WP66Y~=UA z?lXL>V7<{gO7w^hv;j{K9FZu5gkEDaH&LO7_NbK;p}Wr_6511<5R*ilG%fbbd4P~Z z3HL3*_aZ}qZib*FEDH`D<_&m3=B5M!eH{eHxXly6#ehS|ze$oH5z|ndr`AWdp$Tb5 zp=yIuK26!4Jl!AE0?I}p`4?=r0Mrt50BcJOZC(hEiF+YQ0kjAd%Z+26s}wnLG?HPL zP(4tN5`hNj+(FkRF(+9DqfnxJ7$AJnfTO_A3wPxe@M0CxXo#;yUnD!|_BR8uWCz`?HOYagLsm8# z!m)7i(DK|Vx(aN3bkdmzK~0RcT08<#KqgjEz@v0LAbj-$-@OI>i-fA!<|DOjfIG?Y zg2j-YkxotlP*xh@Y=Fl1|H$1F^SIftm@-8JrFKS<(BdIbx3H}b07G^t%1|++Yg>_m zaX<)62IXioCJt^ov_Y)TAF{fQs&+*IES{V+_ zHP&G8ka2zseP*)fO{PSc#L~2>6}Q$6x(FsXQnXa?*aeY13`N+qQ0SU8H6%T#NFWxm znr7jL;E730bp>iAlmE7)#h=KPdNhCzBKC37XOHYB4E+2Kc=#Q))O0 z-||2?qJ(`xhzD&Ar%{c9vKIXS61W#_v67MzC@;x8Ei^QWs$@cRb{ESJY=$?z1^}7} z;$&z`qLQE*!mE0WJ51o8g49q`DRr5e4sn?)(DQpp69hGnhNoVeGiAz36lPr9I^qu@ z=(u(Gp*!I!lZ9rfPl{SryOx6qgM8k?U0{75zxxs?R~mtYbdEoS%&=?3G(TW{ls?J_ zkV}l9gd6d-7K^RPbAA@94RMgWMw~_#VQ{7(4tl>o*T zoxMlTiroSSlHqEW8ZLnX>vO#CY*>;KiiAAej6?YrRs&{4{lsJBE5oiIUh6iv01GgI)yC zIt;a_BrpgAg%{@;8Z3LDOgkON6KV6@obSPIz=-^zd<%TH7oPZp$PdoEH06K5{S4&$ zj*=@OXPv5(cM!R`!g5d{Z;gd&;oU-Jj)S9?s7NM`r(x1wXtx8tMkE}fK(`C9UD4YkZ2kk)%Y+^k+Mxt&~bp4l+J&cqLE>hApKPDy&$4itWMiDPW zfFI#RcE2bxQiWUzG9xo;MyYN{52B+&W*(y|8LW#+oDlR&?vDf@--wwK7z$CG;v-^3 z3c!%x8L$FHb2%M*13BZ$IKXg6tY{4Ed0fcujEeALS^>jQaUd=X_FUlBNS-m{s~qA{ z3xcC70;>lx73MH_3L0}#&=KlPAfRXp=5nf8K1({@LEj0aM6@sr^ZGjbBq!pHq7pXL zc!38x8{hqE_L#0VJ!yc*xuq^iU|b1R^sE z((@OLS`W0yw45dL&{1!BGSo&EWtkph)3r=$3BXpCp<{%l2#Dcj5e@TsIOMVA0>P<; z(B=_cJM%Z)uIr4nBk zh2>|LgYl%6`N7X9iTU_g*KD{A$t7|&GE~+NuG4#qK~aD^&fF17;NmmP9_70G(i!V4 z47B0JE+lVtvC2S0L0AOm%Bn-d>7$10;P~>Z>3%nt4ycXkaxT4AP>2%2y{(N{5C(PX zLpdLML&%d$2k?=irBF5Z_fc@e{(pEVQ(Q85>)DsS@-r&YBe=bNYqMy%^(XUsS zFRIhih2w_swIDH|ma)n~_>EJGfEq|WAkZA{;LHK8^}|!C-Qf1FLaswQ0H{h%OJVLO zW-)8FaE*t)mPm;K0!bZPy zVzD6&fVvI_=Ys%l9tcQN(AMckQ!JtmR7iASS|QBgcyUb=vNQ|R-h#uHLkmky!tq0r zc%ySx00<65^D+oB1FI6GsZm1ScHvH+z=0^~Dd!!oBpHj_E4sGJDZ_rJAZ%N)xB!<^ zbB1UOtB&yiZ;&{?A9;co$RZ&3$t4*$8O%x$_n#$j!}oFKs`(E;dYG>umK3+1>h|9Kvhr;#lk!R9g&OL{1K9JN>EsCPVGW^ z!w8H}XrBs#F?N_IO5*ag)+^`~f#}kv0487N5oEa@-}dFtI{+R~QHYh6)JvdPpjaYT z3g8WbBMtNlo;-mj^i^ekcZ54VA-!5)yFp7yK*>FQfj;oPdVYB*7)j(EEsZWn zEY)fB0<*%h9PMOFb@Xl)s5kAMaK0r0116e{T!yU%f#&2`^#^lUi)286K8U*b8WKUO zeDM#`!XeR5A>*qP(1V3z+#+XTRf`%BmtHt^)&fESSR+>;f!zq&Rks8$Y!1OkV%YKK zjls}S;7w`pArXF{IC)|<51I)dJ_I{zCTwV#>;KlR+2-U8jXVXf_k;H>l)SUSfc0he zl5pq(P6KN!L8{5Q36VCwYR&+lZ3aO&lrh%_a>nS z2=f&c7m(4AMUGz>5~_mhMLKwnhSv=29P}WeU^`sLXhN?bEk1;fXRpQxlfr?g88LW?jM@_0Eoo(wK%T%5=dZimt`z`enH&D6!@-huo= zvs1`=8|-H@dCL5~y`gKV93G**f|3v86^rgBFGr95)vfuYCXvfVbu@BK#f?^)jLb5DDe&vx){QAAi9^ zoNPj9ShRLA$FMu|Mgfi{GP4~}?p0RW*C03g|= zoDQAg)k|k|uAxSM>zM^C&;!yk$PoSpNsWaCMkqjE0mb8i-w`1W7kC;5C(%<+Prz$6 zA6mE03){w@)FN6!U{6aUIR(&8{P~S=ezsMBX$_FGjN)ytz zf{oD%F{Qqh4dzX99SUc#!cU*jaZHuUMf!IP))AohPU)8rtqjHnZB_i1v-K{`U>a*9 z!|3p&TvXQJ5IO>;v9bA)SLTq7O`YL3Ha52KFZ!nnl-WU9UmH7n8yh?LpN&l{{M#Q@ z!aw(I%I3d!w6iHgzW(LErAjzb5Vlr{W|QZ8^o{)-m)UEYR!^Q#-+uh9T3#Mi6uAk_ zfB(2!Li8x_X!G+GYHa9uJwIwuE9s?KpSz#j5~zDxr}5J3Yhy~k?;D?Br+;^LU*w{T z(y&+gi4A-1+gfRqQwdSu?As?(J~&@|UA&9Sp~8mQ4HI`dT&?}tzMYR_yNm0qr><;M zu2h%9S7JAubARbx`{g86@vm#6Z4*1ZOa9%qM9Rc6vVPZYoTwJPG{Z6c=#qrJQoqI0 zXvwRQo4h8id|2ps_Jv3F3u}9ys5-UOUjLM=)NQ4M%3izea>Yk4t9PYZN^-+-F5{vv zPJG^d%J2lmlUFmI<;*IhiC@0_Zg6v#?K65$uTro*x?i2mon1TCj%?H~*l*p<%w2{4 z^?t5=nfv`m!IIB4mNywU`BKb@r2p!T&mZ|srE*-jT2^_Jrr5K-Lkl*RNzy;6`rUA7 za_vq&HR}#IFl4}u_crrS3>MdGVw^rX?7`5=6SJmeS9N^S{Z-ti_F*GmzLlQt`}p~; z-*ullPy4dh^P5+%UQ;&@vtK*%S?n^=WqcM&hn;m&4mn_U+gBaGjRrwTT^u zRZCvEyY{(m)346!@j389Xl{wmIkkRgeNtqfo_W&$zu*ezzsM}E$ADc0J zsE>`Nw$Gjq(f3wfS6wlFefG2A2aj8JHPaT^L`P0KRp;~E)tyBhcWpX$YWndS3A0vw z>a)i$Ja4e1VSZD2H~+jd{kk_@c)n>HX>G~=;rTyqZ(k8`XV~H?U9a3TR*s$Ywe4kH zs#Axk(oK_cLT0rd{_oe~Hcm6D!<-6_lXnXhJcj?S8j)P|?<6h>MO!SpRd#6_IKOw}m*Lzo;t0XmJTFdWF<6SS^ zI;{KR`|g#&{di74?UI)(!z!*{6McExrIdSVzh0yidcJJcJ|}d1*oobHLIWzC47<|R zKjz-jo=Z-7%u1LWe|*%n&QG1AM>HE1@=TGDGu|fB_D>~wz$51u?VjY6Iv#bsaoYF~ zKPJulR-^O0k{|bXmBy}Lt?XWR<&DjKN50l=E-~t7%wmUt<#m?+JALWxor7H(t(gDA z?NiOq+g~g=>a4o&y_j1oPFH@tRu$X%*X64Z)+A5$3v8D6(XQTcU6D)E-2O7bD4i+tBAP?|E;+^YTyxjd64gpvBe8^j!jFu zU~7}wyw0R)O}CbPG2n9JgBx2mEwM0gb*nxjdfe<+MLcllTIY$5IcqDOnONF>NK)gw zn+t~y=+rj*uz$|`YZaVM4700H_K{at$8YN3FK?c&>Df-6Fo&6UWz4lUQL6mtaX~Zw&Jrj@yYJ`6ptpg=B$`stbX$~ zPT4mK-dDZ+@==Glw=Lp=TSomC;IifT>5pm;je3Drb#3soX+soympNVQ=BO<_dRTJ# zsBhY5L-yEgXtHsBi*sJ*xAwP>e(8U&-i`T7)0WF+>W_76&Fx{=DdC~hGF#hamA;*r zI(5sqjb|&4@sR#0T_&>o`b)_-F73FyvO(V}dn%TgnDI7qepU6|}d8Khm z%79T92ClevD%YX))xBGTTU@+$MqlXT<)7MRmbTlpbB$%%o=xv}_RMp+(eW~|R{=Vvo%$_b5U?&B&R4fD1I9jsaN^S86lN{(D`rf_y<^wl|a zE7!MqU~_NB#CX58xj`-V)jYJr=KYNL62B7tUuAeJr@3{XdTdor^oy>}F3zK`W-9hO z*n~)C^-lHOojquaVdsGHs<6*GeN%PK_ikCD*2{w%9&M|Pkn~m8>r`&UcEydK>vq;^ zIrV<_(Gp!=uG;19I{(GJI@xP9>gv0PHi&=!tmfFkeJe`KyDfa#I!BUiXpk7Xb7aa( z-(K_Y%=8Gq*)_ah`<)xg`oCMU>V36W$F6Vpyu9>z%jDW0Qwj#;op37F*DLVrd}&C? za^uuqZx38{dpb37+`@LllUo1xq@M13tsyHbO-~8`G@x-}7dvG`=QAG)=d_($bMAoK ze&LN){Cct^uAHL6#~IB^cXO+=EaXgFIiH)yRK;hxjUVjiq3q_oX{hgsf9~5k#gEQv zQhj9Zkj}TahP_Ij?%-R3b zc$P8$M~U7y8%OOQ-F(525!>c`Yd3rJn$L5OFAh!o9`~~4Oy|}=Qs(qD{HYS}^73NF z&+gKWtsmJP>wo%8!Y;*`fzL-b>-ebFn`PX!dB+gDqVQ|7(#Wbxc(ce<~L`}BNE z=P~jJqqFb5UwyrAz)Hv9+8*NPA4Z0@xc9Nm)?%G2iVYi^%OrFB>b*EzvP0;?`!_p; z=3NT)4$hA*E^2;5w~G*4Q%cYo6ZrIz1}ta7>2Gv9ZH z4MT4g9!Zi`%|5C>Uu$%OtJjkc-8nNSE8?QzRloo@moCAsohyX&4#*qlvEkeA+XiO0gHQn&&<1YA-nO8^uia# zY!=i!_$GPTp_WbCuCDH}@BFo;Rh4xei~~QEk9+c79XURs%mv$0^&gMSJCgCV)v}}? zWj^%j^kr17LsRQMXnXba`r$QG7TNYJ-pF;%x{?#q604ltv3l=Z@vJ(v8%K;h74l@+ znO`6B7c{>Ap>tCI*5^rNKELu5gv_p7^a^jzNp8vdXFTd%^sLv$jjivFzP0w|hh5F??YrmQvBi)Pk8Aj;J3Wf7 z(Z8C%TeXBwUqTNb4m&f)zCsJXvv-nKr|aEXUf#QUrf2Df##trGpX-rSP~~X-{?%uv z4f>d}wcC=LU$5jH=n>@j?pIr%?z1$$4?pftJJlz(^)K<3s?84;URHJLp_!I4wux5V z^kDHD#kX&`9z7{9)ph&q3K4rn*KK9NNy&;?Yp=F;cvqur{?O>b|2uBDR;WaY! zKOYy|3ya^IXX7&MYTi2DM!&}{r1kT`4Fc^xmG4wG{n!16c0(KHi-sQ^v8atU z@5uKtv$OXf$glgomN?b*P~hPC&F@Ppj6XYc|Hg>@3;#Jh%*Su$%wfk%)GIF4|I;mN z%IHRC@7=n&vy|KCU2C6&`lN62IJ5j~bZh@!18!^^(cZ1zXh}wckg6}D9k=YBU&GL5 zT(g0Du9vvFe!|+6cuXA%g zymsBua(&*h(JvwL)IE>34DF(jomCmw0(;)V6x1hTFXMzxzy;m#?yY zo?%q}m#LbTQS;4~HnKXYOIqK$+hNw(&OW!Cp6Y#S&mAtQR<~=f%|pDOWepxy)pz{ahgA-%n3dzbsbl+-VTQP1 z+rZw3W_8f+h?-Zj{^;eOJ5BuF{jGDE>NZZVM$CM3(mN$I)_dUI7SAfHZpg2tB{X^K zZ+C9#+73}YCZ6Bv`+ZY8pQ$T{xAW}cl2&*2>)-OVrH>BG^7UCiWY>;;IZiD@T^E%~ z54%Q(kJfno!pAv!dBL2SH9Ix%+^U+e@$>!b6IxtNYtkoT+0EU{=1YpNe%`;{%;Pd~k9_Dact?I){cWjCJawEslFqAf=cH>~0?rtI{! zsh?}rTJS^sz0W@msZY|BZ@Tpuc*kez1L?BP_eRL#+-F|xvS!_#k&ks9>u2noU+dPd za=NwG#&*fNcx+L=OUMqd2dfu1yE!iTwQib?->i4qr$)_#(I1bk{PJ{m-24(Zrr+*U zKV?r{$()n{&S~X8ckVc6%E&J_yHy%GcvSbSCH?OQr0u`;QyF;1_UWzC4Q0pDKh$+! zFsf%^`Ml9-1M_@NJJf5N>AnB>k^3$atL;BHq+djpCZ~$qt@`!n($kE`&onhlwXC2W zrAcU7`=HHuNttoJA(>s<1?s zDR!XPhDj^lUYL^grN5}rfq#`Yp~>R6BeFdeWrj{|1yESn0uqtVlLtPVdrZg7q+5J|wZ2a|<{!i~kTt4t5DZF&t*iO41Y;UxDTsxl zp?8S0W?Z+Q6K4hXx_AF<%kmXhBwa|`(RtFoAoZe64*CwAj^_MVH@mq$WTj2D#WkKy z{(La>w;`&{sA}IT)Os!IU+$X!7UMn-t z^!XMmyZd$d#oAum`fey_@oC-Y-aQA^Xt&U}O8WBbE2{CCmB;_nY;c;YwEN~JhF;H> zE$-sz(couhK&vIqZ`3@hTYbCI!l{9cHI;gfAJSoT?t+uoW5>AUHtMsqRqH0PUsi?L zRaiKu>3aJB%_phuP29O2d!=u7ULV(L&e@9Uu{Bnev#s;GSi<|;)$|S?9`-k#qO*H- zIv%e7Ii$<9olT2({WbsBwbZ^jGhHgyJ)z&#Wq75>x8onKm_EooXUvrMo{7u7-b=R( zQH(G9Ja6sF)uI0Leq0Z9T(YfW8Mjg&a~lS}Iyvld-m1~v@4h@d+R!m^+@5u*SNp!` ze%LE$q-~w%73zjv_4B`5Wo0kj`C>jlKgeR&=e^E8)GXuXzj?%=gyUV4>%A;0{G8Were3Ac|QD1jZk-yAgcYYVIv8OX1g-)weaQkR{ z^9uhy9bb7kyv|r^cez%@YZp%*rgYo?yLr%&N!=4_Hf+12e5nUztH0{G<;m+qx`~F& zonvlaFW+#)n!$H(I>x^$xi)gb?dHduMV&u9cG-D*Rao>%X?BU|eNKm~Jo}Io*4}1f zu}nK>)jGedDu>G1bor-A1@ARy+8ukdG{2Ia{T`Q`@VNBk-?zG{`kbe8SY6}Vqbr2ZQrcZvpT02>%5^*9jFv4!OQ&U2H_mQ}lWEtvgvVt}#Tn9V)Xr=1XKYreU;BeI13riVUlRj&qI zSWszwa@LkI;r6Y(&m3#At>&m|nX6ySNNXU|Ub&+ywz+P#QQ1uws#=|KN~+jp(X`7y z>ok|HY2=LtxpCvE_G)3{rSeBn z9)1(+dk0nNS7zZqbBAs(Tj$C22Nm~ie0kvd9oJXgW^C$jyI{J!DMVfqst+?f+uU^yKOl4q4><64V=z>4s$PaLSA-N=EP_XqR^^c9yVLiKE7n8@9TpjTyGDn5qYqO z!_kSaK7ZKaVz;1VSkC-fSK5v#-tA(_@cFZMr)^)fy7sV}i;Gu(F{(k|YB3jnt#GcA z?Xc>l=jh|M_lD|I3v=W2V|y;`SF_}q#&(O=G;CTavcib8dPy;PZ_jk8k~?X^(75k| zul4`hW^Su@r@n1STA$Njb0Rw5t5kG5J%asr$Y59jf@Z_W#%V!wdh^jc~3uZ2q6uM+bi`^RE9u z*9PZ*pUUexFud-UCP{mG1>Al=HRQjiD;9MaYrnwzAG^9UE)=RKCXc`TS$y6pv?f~oOZ9AkqTT8T2^nj<-$+b9UcOnv!_3+(ZYjkhCe^Bt8tW81x2J69ak2fO*Ugd(pUz2fjA&dpO7-Q|+g|Ht9C>BMJNHc+ zU(5)qX;8E*-X`dUa_QZPSJigK^e5*!H<13>ey;k||LpT>tgi7UXHD*tsq4N*_gNkh z_GX{$!N_yl_m*_)6f;lvc+21qJ`=y}3s~H7Y-$aS{gAuHPdf^x`R)tqlQ3v{=TGTA zXN%{q7#_ZC#+w;Fe_F*Cgx){VyLiguuae4k-1_W&`Y`)~;twkkgH}UDWankxqE#?muFFn$;SLKn0@_Q%ew2?nN zvtUztX2~S4;Pj9Kwi`B7?W+{q4>;R&-qU3*I-ILKcKO$u-ebmAe|Pto+~G#&Q%;?` zb_ojp*?G$2n&a&g8#Ky!>bi2nqXMTIXJQ|?*c{&Wvo!ei3&abjwcPr!*EspY(ytdJ z9h@?u_kUmJ-`dwcZNbv|t89~wRZSnb&rT#B{wpwLK>f$1bNU!Ydvt7)F(;^V%Q1&9 zUGLJk{hQ=jqOc#8H;-06bS=ahIUamI8oKu9vJQzR7 zQE_#AV`==tz@L|PJ9PQ@biRM-wAhIAy{^Px&QQKteEMd+xbcb{`JPEb6l&8 zdDnL0_<}WMvp)YP{ph}XNw-lMcka8@4ezh&vHj_ufL)Ir4$9=aMEVh<@{SC%by;;X hrR=+3-OGmbcxr#_RD<*1W1PM$nv}V`^EDC${D1g$9OnQ4 literal 0 HcmV?d00001 diff --git a/tests/test_metal_q4_attn_exactn b/tests/test_metal_q4_attn_exactn new file mode 100755 index 0000000000000000000000000000000000000000..871ef14d54d32def50460258656c53937691c35b GIT binary patch literal 927576 zcmd3P3tW>&*7rPlxFm%8MIoRGQd=S_Qnf;DN&u~h8f&1nYIgyV3PeGqZdF8r;5AlF zOWUn>7ra#yrPT^8Y3qumYf-zc)?K@I0c!=KR;pF7(tQ8(BvC-?_WgbD`+lFl-#C|N z=FFKh=bSlnW+r@j>+(;YjByPALAuA*Qyt44~k5QJoe^z^3_CQY0@Q9GSV`~DLe z_bs|7M{vY01Sf6cb7_4)3+)@EeG46VSCmo!O2WVN^cS<2zUasx>JLxtWgpCh4vA2S z@-(hH|6G{W71RkUHl5>=4IUl+8>_c zwjExzH??An@-)6X|L(z0fVVLF#heAXMK9hn-Ubg30l!_2NuWG`2iU!d^z^KPe86%3 z>yO_ePdmJaeeE?U?-;tXNuL<-ie6ltn?K)C+8^HQVRm>?xpo98-}_&0T2gs>`uwb{ z^z5Zs*^6FGe=&1ze|Vp+mI~uNCALGMe190d;isny^vhb1SyYstxiGswyvj{>cu9^% zl<)noKWqm)nhyeZ&wPCCfcKgM4CQz1I6bNl%P4G#06 z=FANtgA$H^_sT_I{I)dO@!RVdC*^y?bHZ`PfcHXg!RSBU*gx9gH3!>UQ2xR2?(#7y zg8uM=|80lo0LLib51vD!?fp6UICo*@{A^;C{_sXQ;5pfZ@(<<{yWU%}D0}hZ>=%mq z!;5vmb0R_ce)u^_C&Ui?G8Zq-TsANB#mxTjk{$439Y|6Bm%;<4_wb*-*-pRb9PlXr zAb5TGxHsyac0OiLI|>~ugW8@Ym=v6Jb?PUlQnruNUf2G=9iDUjp_S%Acsbw|WiOyv zaLUD9czPE*yk-Xw%J+tcwa7U|LL(=i{3dg;BTsU17hZ$414#C z^1a~&ILe(7>=iq_xkZAJ(H~y0ryZVCZmC^wc>Yd6jyOHt0q9P??|i5o@VaX68fv8L zoc|o^%bD$E-%XvYnKE(0WV<%T_vama>>!-l&d1MPv_qi(d&3&Tm|kEU)N6+6HQJGH zp6toC46)l_G{!}Ro~#}fWA$Ea(FVq7HG3L?{<-?t!*~Ezqs~%$@qPcly26Wnb)PR} zRw)n&+vp`li=!9h&W)b8Y{5L_>0c-U;c@C*_7kJ`KdISX^pNCR*I7?KiZTU)eM~YS zA`{Z~xwz&s4Bv$DA*O$k7$3n&hJD%lAN3oHK;P_?n+lQQ?&1P`QkNCIn7wdh;z;0? zJJ;Tt+GQX(+qt;z04ojWqV1KVr{>OGoVj>e^i#AHzf`bTR}}qJ!IFG1e{MnkNMV7d z(&%Ti7Z;(VC_1Uvcfck1&-H@OL`yJK;PvzHr)9sGx!``F6QYGrfM$oA(+jR0@7Es$ zH>qH8_JeUxi0)g*(NABxeuL7!bfpz&WI}JMGWIZP(Y!5M0199xl+HZ(kp2yFi59j;@C$DjxR`FucQtjTyP0mxbT?g}E;jWP zOH4gUBBQNQeER$4qRMMLGXtkntny*wo5iBajDy@*A@3P!d<}W#%%6%h8j-I);P_st zG`@_y+L_N8W?YYa^Sh4kPed74Bd>SnKaMh%A+K=epC4{4LB8P~ZtUV%R$2WHHzsQi zt5odf#$>KkzZ!Wv*(mObH{Rg6lcC&z4Q*e)qK#x?Ll75gIKmI;sxB9u-XR*W!Jqov z&y5XWtF)=CdT}-XY}a;_FT~g=&7oYR*K4z6dhH%BR=F2t3@|v9%@w2kVZC-c$}@hS zWIX7khklE;jOn%M$X^G2G6Ktl{2b)Fh+crHMqj&;zlpqlMum0<(r5N^wY3OW`C#)> z#4{0u`iR9$|5dI?uk~x_q^;#*?bB8vnqFnXb;muWA&*meG%e`DQ z!xegM1>#F*S8Feo>{+}T>2KZnf{nm!t(RD{#f443#0QwY0NcC7W7}zzd6!5+ze2sy zVcRujA>@twY%9Tcp&NnuT=<}5rZy(|MiodBL~iTLvzzDw&2F=yYkPjd0TV#@8O!W{~ntE zdTz}Vm1{0)iV*779MIGxFyoPMbtiY*mL_&(#*N`>qkJ{%EW2e(`=C6i>p;1GS8cgp zR~YE|Jm@J#oQ#u3bcdvN}K)#yy>!NSINZ0a#U6pTf zwQZZ#-N}gK5ccvjy4tpJwYSi&8gY45lChdk1ApkhY6T6yUCk=lHf~Ha+ECe!@)?-A zRoW!ra}M9sTk(w#G26dsyUNkm-%wutCO76o-oNYGMPC!~+xz8#U3KLF0uLseS!Kg- zIa-#q`SSoD@nXa>z#oVZh(I{B@B_^vwnh5_aHnw*4lTS4WvjHAD7#H%1}Y0QuL8e3 zkFr^>a_;pnZ2;c^dFY?fOI%w4s>~GxO-j1+ZMzdaWC9n1o<@ zimOx`xiQo(68Sc?v)2zWpXWo(6`Wq{s@H2@fQ(!O4FA6p{fPzv=56So;JzuuA?7y_ zS3c@uyg5W|ob9eQrkAQa6S{3__oL;fM9VFJgYopHM5bga3uP9uTCbI0j2nP=Hsrs3Co^^M%+kyUnTc*9 z(17TG@qcv@<69Do@oyA#)dA9d9Aq%u^Ks)4z#9cw)*HC7d&{5c5|=S!+Zom^E=RnL zb&tmP=eyq3E&~p2F6=4?xo!BGD-d;6=dZ)J$gX!6^N>F=@SwB{((4>)SEQ>^mpSSt zXHC| z4}>6uPS7Uj>))J#9KRw(co#b30N`E9&8_#!%dIbUWe2Yetddo6Ub;)}RkAv_*S1!1 z-g#+$m0PFy8MdZ?HcHTr=$84DEsfv;&Q}8vlq)UUFbA)AA{;2!cA5E8PyDExTGLZB zrAF$;re7YNQsb4ESMROMtFJ;o-=Uu)oVV^W`Z??N`qm?yPu_CBty>rPZQQCuKPl)Z zo|6jwFz}pjm|5e2buNVypJd;#0_z!dw=Txdz#+{exFCoS+!0(6+z@;~-|67lEr_2+ zTsXP3CSc8nn!q(5X@b^#tQoN8uqI~Hq5Ps>eX>7hwjw|FZAJ7lZ?Ps=9;bJGWfFXHk9@ad<_i zhE;TFDk{F$WK~?(T&%dJNv!xmbFAWqW_QI+jqjR$nv?|A*@Cqu59@adCpnq!$2wOr zmKFl~T6u|i5o3YrsIvoQpz~MG@)VR$y}aU;QJ_;4=V|a=W7ha#{Qd}+x2d}~h3mB= z9(!lo72eYvh_WDr0EA$K0SE&TEV-pMp=RppkfxQWLjFQ4l9rYG!Z#IDa<2J&$j1Q6A;95f^VhoL`Lc5|kgN zJj#m^|FroJ`Ts`wIh5Ow|2NA2jaa*;TNAUU4Quf&P34-~n&7oRVr}lh`u-Evcbh+gri6^3iX}84rMR={)8*LnleJ&t>)C- zKhEb(H?X$+5W$)D@~K^&5zHj$+JCByE=xs~OH#Qy(i^X9xw?NK{0-rK>x-t@?#!4j zXPrNuQ+GSpFj^;C{vzl^ZF@w5CHJ)ZII;s4&4sOtI0rFP=(P&yQa`{Z`^1_5R~f7P z0eQ|>uN}gwwZC3Pc1}>&-=F1b>k#(x&vfM=o|U9FwqY$|%0^W()+vea7A?1$O&6_U z)A_Y*x=RH&{TkL=*NRMyTg6In2XdT`e>7z&iqXzflgospZCy-p|F_K(^C9=qyj@<*E2y16uW6 zCp_=06YFU$;Uv<$WqsEQCoEHL@Ac*0b{_rL7>E7l)ai?Wn|_+zZlSej2k@dbCJ*J! zQ>om(CiPwivhYoTZ{1k;>#&wq!xo{o&h=>yjdzNDeF}8ZYp*+Ol8?zIDH55U!My&^ z%O-gh`qF~6YbbBGOJK8En%!rZ`r9N=KW1N_%96P>TKk-E31)A2z2OjUhhQ(Yz%JVX ze&}uEJmG*B{uI````ERa7)KlU;-BFCx1xz>Q@ffc5KY8=?A)0&UZFg|QGR#Ld>Y^M zN!*zJG)h5vbK+h4w!)ru@=FrFDe#Tvm~7zv;FlWA%e&x{^|?9q9SHH9hpy7Cax2*u z@-okZ!0fVa%3E7j%Hr*l$4?j5#>P@mSd)v&kLk(^<@;-IUKS7Vatx<9u)_rg5QRsN}nr2NV_GU-dK<^=~N7znjhaL=q9^^1yqFXY? zOtf1MxVOo@7k=r=la>si4Eh3UKOBM z#SYE*ik+G#Dt2j9_t0yUgI=Th(F=KjUZeWaYm|dtqx#Wnl!IQQU>n%!RR?-iI_Pz= zq6f6Q4O;yKo8l1ogM&S0rx)?3okrhkrox8Uh_Msxnz4Q;K)ZxqwDSh-suwn?+QBnJ zVnD-Z1!yxbOrZ(irHBqkyt|}H)q%Q6V;fbwH}1^ejk<56?lJm? zI=iWE#Rbipif=W$U$)ns5Jz=)33d0N&P99Of&+*T{?1-!qN-69vuSre*~$thY*cAC z?GbFZRKP1j9lPzO&^!uw0a&vSuD92HdNSc~K)_)PaHvFG!eK1p_usVFnK7+Vb#~Li zUbdaWVcRJ%H#anMu>QZd=B8!^U?0jB+MLqPXjBz#de3P$2ykI1MlT__OYC)0pCP!1 zoNys;0^I1cR0nY10L;48_PS}a8dY_$L+U`QdcX;mD>QY$*Mho-U$fVF=2wlX!<*~! z4@3U`AlOy-CfHvJ*i${4e}R`CTl^FrbeyvR-UfZUzl-cV(m)od{+gzVd_M#GXl&DL# z;xNG7XSCOuwXjj8d)bn&o2AHKuv<~E5p{IC7127x%GFklZEd?|^6E<(@#=QXzXAI& z`gjZY7X$a%unWU0ZX?{%JPvz^_5h&>bKJ%GA1BP{AwHeWCG@bhZTXhD-|?&cl&iQuO>;G<&HO$1*hf}g@ln^fhS zcjjl|+X{RmI9d3Xg>Q)!=QUY?8M?ViHE;8-{0h|BhHuoq0(C2>eN~fc_2%7z41DZp z@06G9wHE;60`>&Au_yROAgH;drB?}XE*FaIKf^TB=L_yP{u0u1&&>hPCHkmqz^A7s6vC&9c;b>qRpQ2Z0uN5uG=xXU+sl4HJ| zya3n*#G6?gvjx7)M*Rb*;}6`kP=6890Z4<-3TmkyV&d0{NeM!qGnU!=e5Qo-bD_YS z2LaY4cUNX<2Y>^vIWUDBtO8j zuw`CCSQ<7|GYjG7;Gvqc2!9*LPyZ{zS%j|BC6d?M##jbNwXf z)D4;tow@`aUnkJ(rr-*aMpE(U7lGo}lh*f$w$|_WwJv_a_@?-kNXGj%4&a z;#N(hV~yY-qwVk;DI^nWU@M>E-)vM-?D9LZf3os<=vI;?*fr6Suz?uHbQ(NQazwmO zvh@PeB$s#AfL=1Y7VW;rH>b>ALz-l^QYf>_>?p#?YogG<%XWMJ?z0|CIE+W(0DHBn*jSFX7W=4J^>~>K7lo44;2aa&}hiu63h$X69btVhcxx&lqIs0 zXbtEsv%f_h!hvLV7Sbex{blwH^egGu(X%EAeajBo`wp!mn`yZq&&$#Nd4Y$hKk%Hu zFLz}SI0);<*-gcQ{qzsiUy1tvCV4~sbEt1a+U|F<+iMNL^$_|c|JyalTZ(7YXu_H zZQk4b!+6frR_wr{RLt=Kft)|=PUTQJgw`GqA=kMZt#*A{>E74w_{64wZ zlY3xK-e(UlvG3tCyqW|ZA-9l@I3nzkoVsBMWQ*2e=N>6*kiD+g;YL-~#u`CB=b;YS zflhr94t^ni${Jy>D>+JaP0qT}j=F-(W~=OVq<^G!df$EbT(nk@t|NXF?9plVHiM26 z+{5>W`zw1L@t+9p-k*g_FtZoh>$;t8RDBzvNBy^*^#%Lnj&2m>0(u2)XpC7U_I56R zX;gh9*8`VbPF(uXBl~50y|BL#E+5|)HjPoh<#!HT{smk{_rXQb$-qUhnQ7e|3LMIS z3(027Kv7Mm#NJla(x_^U)h~RIoCEjEz zWH-zo+oa-y@O``<{)nA@{1LQY2nQUee0&Q%Kz0M|A;_M~o+!XS#fPAM)X%_w2Ji>r zd+7b)J7LpWFZfol9tn75H>n1PHvp$UIB^p8dAotrMD*S2;Q3Er8_MJ#;48pe-K6qV61@F=;L#Yd7ZKzKI0$>vL?;ffHL1KuP@8w0ZG=4{t(^~{ z&lbQZJBH3Rb|O$e*<6##C#C^&^KKtKAmHHFqzV~J@GO1c3Hvg@BKzzfI)pc=g5#)7 zeV;Z2uP~-bH9*w>JLi3;?@8E8(B6??I$@21jp5uQ(;i#EC80^>pFnWmy9aLOj3$-e z#0Kb>gMxk;2fC2GG!S-{6Q)zwv^d5+KNI`h$pr6!0Pi}+L;KKh><@OJ&IQ1r@m*MR zLGx$KpFpd^CYAd%qSYQJtqQ(I9r~78)}(TqK{&iE;P5!$(Vpcd>>a0$bgo(39XKpE zHmO{nX}~^cx3CYI13QZR7~$BjmBFS?!#;*^x{keZBJiTUM;Y2?;@gfjt(tFq5^GkW zEE8qKQ{k|K=sb1^Y;M9S3w{~SHK8YKNIoxb!g^8T z*b6J{dIh#|5$Hhsw+cEt#h8RWFk+`KhL5Ipru=~TCfJbsg!Se{v=R37*yqEJ73}It z*uWJS18@-9I_=0T;Y{|thM!Jw4(A7?6P&#dgmXU_0R>H}A0isi_RblxU_WQ!yjNJG zfd}DSzJ~Pbd5ueHlj@qh0Xpwce!ynx^JBpWR0;fg;V!@saL1U)PH@`bM2~Bw9FeU+(a0kdCOMaZCcdsQbX133#FRhv$Ul+;82)UOjYv z6Z-yxvv0wFdJ}ss>fDd% zs}B3-S=bxOBAQgs!N+~~Tuh+}9o?k*&1U=kkAGl(s2%yLX@2ZyK=wT-?dJ!qy#YV` z4NY(o){@Qd3I49<(1!NSA*oHOn9T=;btw*Iw0{m>+kNN!f_!YW=biz7hksU+>S>%^ zkYCpyapzvJ}n8>ggp4^b%|;}Ixh`yUAI*XAF3Sbv99a4 zHsPL`7>HxDC@p|E$U)x1z_x<I;qd({Qz8QTgX0S20aE9NEF)-XW##l}{@A!>dguBa6v$1r}{-1Dq181Ed z3OLCGoN(vY@-g<(bO*9dIC~E=*W&EG6?X~lp1+5fpLgIFKFz*IZ{zQd{Tl9$-EaJV z6y5EE;J4szTz|Mv+2O{NrLeSH_%4hO_p5u4&xZNJx&8q<*RR95zIyV#WBm~{Yn%Sd z=74+$bSJexE)fn~GM~a65S@XCJ7~+EVq+DU+xKVL?{5Al8gBrN`_KE&_37s5lS^{+ ziMj2-c&iiHSlC7f=pIZP{F3C)yx(1zeYj`v-*Fd44F1M>_?^3uxqa@c+`S7)vO;p= z(f_WB^WGzHb>5BmbU9?@35GiuvU~1Y9LBeX@oY@3z2_l3xZM0Y#Lhg9VD>OJ>B$k#;oKsNNc|KYqZg1e}9?tgIb+Yf=?J`%jh z!~YEaIOQ^{LFEcResU5sxR9r$)9gO2l^3yQ@RK8gK&0E zzE`>ra(L}-tQornzrK?0`N6MG_dm$DPxlv+u+EUbmG&3C?|_is^;#O0z-1Jtb_F3O9+`l;f-bgX>}uX?EC8({VR_8Il@kX@(k#k-3Cw5B-q z>d)T^4>4aIF7SFx!Vp2P;7yFu_-E=T{4Rc{kk>o&xHtYe@(O1jZ)AGo_@`WCms|D>)L-8UCa);WXkULxrzFbIp~77KD}|sz&CZse=iQ} z9XMnYaokg?30?qBM))hPGC-C)>BTgXo0VJD%^0k&YQ zk%K><>b!wEspyYz#2YJPPw`~DyGq9Ut4Vl=^%UM?O*Gvum|$v;NWwd5rPrI5EKXxAw(h!Luf&pD8$hSqY?g!{8tE{ zBOF6`AK?JPWBB$s!a{^sk$)LsEkYSWF~S0rFG46ndLhDmgiM4v2nAG*@Df5XLMcKy z!gPF_jIbK%^#}<_k4G>dy%}LN(yt&Mj`%giZzB8-VLQTZgqwMwJ>o9JgHb0K!5@Lj zq=;P+Y)G3BY7h=0)FF5w)FXV1(17qMLKDL02&WMKjPO^4eaLpb6%gH#eC1h0OHui4Eg5GH=d%_S$go@Bz#(IBK?cY?xH@xKK94 zd%U;EI}G7Dckg^L?#)XE?bUc7ND*Y2ZW>?aWq50CmiLO8fzjL6&hvhLE$;t?GVfol zjnRbNU^UOLeN^+SwWBqm-t2JL4X!3`?NIMIYsYAwUprP4E8jCR^m|dwthJ9}KRHel z(!tjRC$ig~B4+twBJ=z1f{R~Gf(Z8}y{ws7GsSp6k@cyMA7c^&_Da^wE?ZhpaUx_m z0W#bu$Ziz)LC^&g#v4_6qCg@2*D=P&o#{nmjN_2LG4n9=0f%>(_IalJ-Ov?Z9dToY zQl6z<=Q(pc-l*&ov9xQv$eg@%u)o}u6^0|vuzy&Gei}GdyKZIOwst;Xn7Bl;E$C^c z3=lI5&I-+RcQ+jKF$(qY76$nMbB+h=-V50$oUcz$fG%p>OZsLu>QLYP%k_YNefl4O zBk+WcBo@OSaKN5M;{?o};sn6@WG`eakYHuFvRWI@oxU-XLpjD)%$s@|nK9^dwQ>Mc zTXL2U_CJR)?lmVYXQqVZ8F+I+ed9L_0>uhugw6sT)IF?BJ*}D7t7xp$mqNs9_pW?@ zTR=&`wr0LTg@>^{%q8&G`0;|nL_5NT#(NzyRE=?}fqR*Ux|`ngB?-6&0oNqpl!$RB zn1G|vu4{wLGJHR0vkkmm@FDav2mLHU9UgUEfWIro&KV_-mupW!=lXg>--_bT+T7Kr zZAp)uCfXAI$MH?6RCg<}7MOTh*IrLnE74cK(y{G@_mtol*soUJlch;GbzFQ8UsxtZ z+7;zJ=*(D(<;J2$Fadj#Y7z>S@@d9%))VwTp9watok?gr|lV4Qe99H;?5 zcHlhx95>Lq%AKVR=R6GS0ACLH;kYNg4)8S+X7uXeXB_F~V|L=+zC4>RRJgG;Ir>!~ z@8vhh|40H~*wAB3n}$2qN5I#Sc-!%^t9n_DYs2g>I4{FFl&!-%4Z7mLp5uIUV!Z1ZSqxqQ&u<5hjC2o?ei6?6&f#5$ z34Jv`f;SrA<$a)2^LTrjbaL}_dzy52a~9S|)Y*YF&Cy*C9`yb z4_m>*#|sCg4qH4ZwOtot=_m-X=&;|+l^mC~7ll|1oQE_|azfVeQivtqofYN)F1<6u zoAkgacXs*`#X(Dt%W&; zIX0#ZZDcLXnEXg-uDC?Gja^uNCINI5Vb0WRMO_V7ZMAXkcyj`NUV7CwCKZ9?pab$C zj>JY0?Zvo5MeqJ(ywteP1Mf3o*M#xBF&S`NNUuOnUcS%Xkk%JTJBGjbLd%f-kCa`NI8>;VM7A5c(>O@@0eZekz0&Hn!W+Mng7q12FG? zEG7wg-L29y3qqJ*752PMZML*ap=wJNR+H;k()^9Ywtht zCKI$(m#L?EtVTp_d5Xsxq=!6_EIx#`0-vMrk>P#$yaBNL^Lfcn_l{BgCB}$xHGP3G zCSi;iFN1y+;9^Z0sq70k1M55I$##yA0RPhlQD#~fUZvCm`M zXSOub3wsgQGzQmi!CVuLk)PSd(A%AA{)w6+rQ=N3Nq>MA28@w>Y@~C8prhYU5}p14 zrHDC>tRcVinC>@tVI1P*;G}zSWB0W?!ph1)fTxyzlZ{a_rp#) z>;Rk#?yRz9XQEOLd~!&C2UW?M1JstMdTejBp?@{(Lb4q%4^&&IPB?6kBZ#AXSeg~} zN|Dxi+xIyp{yDsJexsgj9O54(_)UIvgf$QQAfX;~8@Whi?t)ky@RXKFx4n}a z6^k{-XbsR?XfCT@w@6W!`n*E&{0Cbt`3vhX=IZ4ycD;phRbo7)qCnvtqP531kZ_AS z484wewWz0GUexsp>e0B{LiLtXk=UvO%x2Uj{H@UE=Lvt<>N=!ppJs|$YTC!o>e_xN zUwLJ?Uh5;(Yp)K_Yfs1Owas$9c1NGFiG;Cj06hd4fK5C)4`aKqEk(I|g;(gs)LF{i zo}bBn_{Vf*-Ab>}J>H+m#^x@JJ+?v|dd&0Bva>72p?~)NvutRfTj<2&*|BQ|1%|$G z{DoM4EFU_5h5FU=pGFwU8E@FlL^F;de73?nbkC5r(%Kb1p~s%~4Sjpankl^z#gD_0&l_|s9uxUHRe8b$!l^=p0Z7rfmHIM8Iv>_fPUc3xG%FoS--3S>OKGt*FrrgL_ zlBr9WGdbRIN1G7FnJ02$_aZMI5fZus>88b)Cs(mFsWH;9-;Gti zh41u|~l*rt$pSF`k`{@oaaDr%H$intN_Pc)v$`Zv5tunfOG(Xuqjf1!cGy zg)3Ar?u_ zEX(n!Y>blWEgslsJvMFFwiE9pDHlAZR^~rBQ@J6|BQym(k>d4*Y@bRRir@K|vC!*F z*_1&nW@;SsGaNL?_9Oic(w`sPC_6&>VFCSq1LPR7H|8X-gby$veEF!}vOpv?(tb`k zZNxV6Wl+7zf2@+VA8|32BArWVIqP13bhz>ABW|WxjPW-15jS1fmYX}gjG=&cyu@dl zsa)1|C7N~1sUPUDFc&tqR2yO`ecQ9Uba{wHzLJ@Azn|KjyJKqiFzBRotk9n)(Zqh?XcECc= zBp)pG<_NJ|0!^d$3Bo7#?tB%nIF-R zzSY2%K8$aqL#-#42<__J@I8o)JtC6mKF6N2_k6s9`DhO88Yjkl;3qg2t3k_n@MuCH zXpea~g7g&V(OlpT|A^L1cmjXr&=XtZIraDLlhl@B+?z7u1A;#d^jd@d<-n~4=W!lr zBgR=|6k=zc&tc~d#~NIN-?XDKk-qzm3z;&#c;wO}Scl1u!tZdWUCCP*JH7bI((k~7 zbAd}b>`E!{TLfHYxUd<8z{B_9(_0t)g$19KYB{Hzh+8{#XxKhXXF_~|Gk{+X9Em*5EeljwjQFA@p-)580iuc0o% z??hYy{z*WcO!QB-Int*oy~=lxf4Um}09waUo(J3)0Cy|+kLH7L&j;==gI)_!|4Z2a zwjMj~mr$oYT!MO@-3uWP3xRtn^d7Z6(TTkhe6UX3zjNSz1Z9N#m$0V+N5Gxtf?%gR z{3w@r4*_?APq=Re?)y=PbPwT<^l3_8cKC&yxc`9qInUa4L=*9ZUzJSfS|yuebstB3 z^Kys%_h6m{3^{y3dgwIFWgxZL8xI{s`USjr66JXUuX*S$16P{Y9OzG~7x^R`o8wJ( zFe_~Tx=QvIFPTzI_Rx_Drc&5LR~jdpxIf#{PJwPF@E&B&&k8+8elgOU>6}ET2dyvV z<;G4czO;0q%{C?jd?%-~2lSOsGzZ@mfRwQ_--WcRML(ICFZi5|8KBXSoWj_22d|OOQls!jJx7OXUL*K4U-+%3v3{si=$J30 zPg7bAUX%CYH3{^6F0D&yRu}=Dl1K6f{d@xZ&IK6PLX1lXdXP>oz?j|v{Yd9s1@Fim zI{7mE@zNnuvkqfQ0S}Vi5MaQ^PjXH+63Nb0jOTv!NGGQP{yEeozPy^7Pq2k?e&Oh^ zit`r6c^SVuMLL=KB%Qnk30x)V4< z_`e$`U|?VHpT{ZD(KsXihsQ~Ch2J%sLUVw12ESIX_seI@#+< z`qaw}KQ5chDLo=NvGgI-AzPm4-!4~opM$N@Qg2HOf~@Af03LyiTEnqk%$+6djSNV` zcGv1)N9}_Rq?-$$L@4Xl?O?`zu&1Q3@s^9EhViI#33dsc#aeM@Piv4=4jnL%333l1 zO&i|JW+Hou?56%UQzCrH_p_PGuom^UnKZTK3`fJn`CNBTPm~AKFv^yeK+=$ktK-V6)VNkV1u}`>6il;Sc{V}NA0ObdgOIW>o{upZ(R{k^Deu5d2+a4>n=lT>!bS!dw{0P6H3EhrZ1P4~d5! z-x>-1N48-ObjA9=FIfus!$|h%Y^fbKq&U}vI~udQ$oI_+9k$JBcPt6UT@b&naq1s! zV{%Bpa!i+pHJa8p9rOyVZKPKgLa!_U-Do{|Cs!AH4r?3PdN@2ZA9t*6V(66|(v<`Q z{&U!Tf?g?xUbzCja=&^cZ^?j9Yar>Bcd%{(wxCxwI{Hh8&Y^PyE7m*GE7T{AZ4>m0 z0d;f^y@K>^uBw+_$#smS1MP@zv{v75JY;7O?51!r>Unlg#dsEwp2S*BZ8t#o z5?|6uPc#Qq z%_f>W=*-&2i%;*xZ#s){E-l8n^j_#5+G9HRmLBu$y33Hv(r#d_lw{cTmlSzAhqk3N z3+bvqL083L{d`;uJ=Q2P>fqP6MzQWEJh2z|WybM=%s4KX8ApdPBXpM03jHvJ>O$wi zjy76nB1S!HDq_^NPDhOT)+vb5#ySbQk=pr)Oz@#1MxNRwA*Ocmh^gIp#MBPDK}YSN zlSn_cM%Z>#69;Xy2dr5tOava3w?9^Sdy~MeTp)2hB#CVU+4+&qyp!$l0}13eX$qu5UJmj zib~(}bW^|QC0hC3JAi|$HfA5U()WBl)bF(dr*N@fXbkGdz;n;vJ9ymc6GZ-k#x&ha`k<|E=%dhE^cQ!p`pZ_UH-!uQH_jB`qBH(A{AMpCme>moCF3tb&yYoMW z=6_YPvQ64|{=+fceV@Bj}UxVSCJ^b_m=;spj74hJtJf)}#D3wrRvr_gzS#5gNfxQ12$ zMg?HVnTwSjOj&Nlx<=`YgDJO8f)6OIM|vOmyOFL&8v0sDH&B}Ht)RS_(y&!2&E7*A zK5t4ZknX_TPXR6yz-ym2g~$F}{~3U>0X#4PeDTc#>giF>444zZo1dQid3e>RcLL*` z0A9Kiw5_7x4P?ZUXr2&;aNZeaC-25TxNbAm zb@DFmXx-u6`@rqPyRf4lRJRZB!nUHi#Jf$s{2+oY=GZqJCtD150j;Z1M{H>)9CnyB zU2VbI?`MUb0~=arJ#0&J*kM|-=S_Gw<-F5Jdl(M-3O3aP?}^c#>V9q2Sg-$(i969a zf1Qlbfpg(u(8V5#4fx4o_yX{E7cSw9SUJKoRH9fvSCcV)BSbUWt4z&Sk|Vdp+dHi;>U4xh&mne4uSu?smErb{6cz1HJ9TX6!?8 z&r`5jqHrgN?xE5-Uq^ttdrBtuzqkvDdmzU4M_Bjd5Z>7SE;G>{@JIYD5i#^G_Hx}~ z3C<>A*HAf~2huqpt!IB5DcLqUDoL3RyCw(sKciqD&^f6T@FRWnmUhVEd~MNcOE52%V&1IeYo`4m*_|tJKeP)zdO8=S zeICws{A{?dpaA_Nfm^#M%pivEpKRdfa+jLsa#sX51a|~6f&{^%<~Z)fD{H)PzcR=y z#hw1BQCQrPFb~zG#JSsD1?tEhrvdzo>XbeMa%f`u(t@78J+S>+n0|nXpZoLAw^vbrSSo z+xI1Bl%Prb_a%o?5O*NfA+{pki1-rX8pM|oe}VW4;_ncDhxqCv>h4awZ(vxLY5XxF znbda(>@%#z)mRtgoXolbdvorq^w?F)f76xF6cZbdvXj`4r6ZmK{w}3F;ws8xO+G?- z#OElFb=ihE1vn1{e6H0FC$S;M0RL@VI=*>;&fNFC%E61JDBplK9M0CLU!lGyi>spg zRPNTR+?T~2q57Cp&ZAd(Ad5Rk^{L#uS9vgtv!UD!J;V9;Di4DUKquT!7U_J$R_^vd zIX;sis_cJ9`zXDEdTxYK1Ubf@T)5A|f$tJQe@~R*zVui+JNHLgtgqI}nb_I}dea#) zl~Y|2>T>wb;Afz+X7mG{anIdrI*a!~z2={6wRFcI9PQ}M<$1KRqK%vdT90BJ1mjEO z>0Fx5n!dbldxOprDgTX7PIt8X-uI?&|Dtc8PyaDeUxd#Q+!vB!-tW1Gj{B){+*72p zR=S)0E#P23{~zz6Z$drX%M|GFbNA5eQRjZN09}9f9{N5S2Wa5DhrTD{7v4k1eamwg zqx~NGgvaM%o~UoyH~ic^bm~j-3-6)h@b8{`=-UNcfBqi&QhcX-=)LdUH=w`m4qj^l z|B16lV9&>4uEpjI)F;|I`K$NeH^E<&67H@LA59Yay-Oon8!%S*8tixXDR0AFr`vdk zeG9*N-<9qH|AQ<15N9^slruMzx?@c)R zeR5LE_ifnc4ul?}yD!AMv|l9NodH~$9#$*G65LN5&FUjdf>Uu%F)p&iH?=K-VsUEQ zBP^~xipAZGW^uHyI*#_F3k?Wz3C^y$Dj9eXcPtDt+H)my>t%oyXFJ1|+TvNMMh653H;t+?-N3&Qy#Vi$b>>khRt7lb47u{s5((c7sb9*_5w+36j7ndC0ChnPjMl2BXHi}iHbGsw?EpSRHdCg|`Za4N-pilw8sZY@nxceXuJB|9z(F+_4KF4h$` zhAsI_q=dz_@v-JeFXVfY&%}Wyk$x;r5kz-c2*+UaMf4RnkQr~{env4LY^61h&Sq&{ z8|J-5+rbVjj{2zzzVATu%R%Z`CC-mV;LTD^Pr(_}ZfPj_*SsA0z1yXsILj68Q$!|+ zhq*&OY0hfU2l4O##z8#QlowY|-*@A?FgFz=j1d9Oxfx*otuQwO%)fJfZyjSjp_m4{yN<*&wcg$1MR=YfYW3% z^_FAu4=UnVE!M7A+IPP29{rs)Ief0=c=Ii0@aIFuk3+^yknzcoar&kJO$0div6t~4 zJdGoKFS-{r^WHVhycdEef)v360k#$on~Qg^iC42uezb5uRSSDlOR`|Yn+FNaD}t|} zoAqDWN~I>;FYynXw?!NF!QsUoknJFxwZ}--YmLd~#Ui%OKVtnlZNxhB;%0X*6Ao?y z)qEiCVFZJo1I=pSL;P|TZ!C!)o870GZsEQ^{RYA{(ELWD+HyZw`$Pi`dARGpBnEF} zM4WY9G@bK%7^ZM)%S+(9C>C^5hkMQ$$_cl2j%2WFgA7w}um2JsHEd%Em-;dnmQu>G z#rwF8vIV{J2JYqm@%&WW>DFUO56@`4hKU_P?FSSy1P8_`9UKm+E!k>wTOwrPP(haKS8XiYx9p zbACG5B#tz%Ylcsb$~`x%*KRcLU0lj}>Z;+7qq2>f4cgCh%|aR7OQf>lrghrS*W2r< zao;8dGBWH!^tio{)Am3%wh6)Z7~Uh`o0wNy0)Rt~NFVlG8Amhk3%wm;e(TrvzuzkMU}I>^Ep(oVGW==L!1WT`FOlFr9%zsWAFEB|QrL!hYsNX#oxT=d){#Fk9T!%FIQX&zjN8rvD_@oW`obaT2`%#C=lhHTT5r_X>c6yov~cVx z@4WXU-_X$}?BgNFw#I-5?kjKNY?aO-2_E4~cgLtN*zp4Y!R{3JaXa?8#7padKmP7= z;E&iZ3UdhAr1Bh;6Yi95o1^Z=S#T$niCx#p$QMuL$;k7LbR^PkhnS`PB;K&B{BWCi z5Z>nWB%R4!Y1&5p2XX7S;;UG!!5%)Owk*Wm^^DQ1miBq{-h^ywDb{ql_st>TZx#wDETtt#2IO@cMP;~&hT#9hJbymDCczGRbx$qaM`hx66b!%982 zN!b7c-FshvvS6G&{7ED`No#Tn=Vd@0e?G(@NoFh7Rmq%#xBCA7o&CCQ-r zp|EYVzYYaGe}d1m^^Hj;qUH6M)E1&eGyL*x;3>M_9?XXrE`2oFw2=!jTn69PApQcO z4)%Hbe0qx|LX5uf=2b|e?{=I4(4F`U+%s!OKf_AHwh?_=z=KhN>R9OY!b=}XOqZd% zE<+AWvA3eTsioLcWsrWv9xLG?-184+W2D0T4KQEE-^;rK-9&a3Y%;AM0x$BBzQFvZ zG%_FE{Iv4fjm%e

    h*q#jFKGrN?z=|-*EBC{cG-IBEYG~OfntlBL54tLkT3}Uh5 z)5Dp#@IJH}dcPF@oKGO%K19RYwp!}r0NQ*Dx%o18W$a{>QT`y(bjMU00NKD;TIfwB zKR~w%YrrM&R{Kn~rQMf}`wnBj0zSP=zqJ)jvch%lz&OEUU%{6eIKU5gtFafNF?#ZY z4IU_a8Fw#6BKYWZv@T08yH1mO>7oQ&&xxeg46FqLp73|kT^~A&Kkm(H-}h!?diQs3 z2rJ|v+mYM=gEd0kT?$>12wJz{ehlf2R_HqFm*)8bpKSUOcLuNFJ{Qe@3+}PC!pBbh zK>XneTt)z={hX)NvllLOXJaOCmI8M=S6+ZS(`0jc>2e))m9Zo@=R^`~HO3te`nCdZ z+{M-2uMP)zNq}=s#1(#mH462hG@=LMBrUwJif2mPk(bt2U^qpBBFac+R~okmy9(7bfNmzLuwQJk+oKwF;?g0 z-sW_2w zf}VEO)yL2YTVRyP*Wd@=sj)>K>jz({u~Qz4^ZB?f*q2xN0xzr;SFvvU;QsczC?mgK zWEt*9Ovl{<_`EYPUvn{EE}D|n6J5*uZ@^1x(WVudt{5<>Am>$cu5tDhZbi>nPr=?Ku3wVkOR8 zHu&Oi){3_;hW^*(I{bd*18ZWD<7my4V9w&1#41AjZuA`=$vV%2E=|3@Wg-7x`aA38 z15O`0FFM`qE-@*Y_@vU0ofcz%o%1j=wWiWKDl)%{F_o_5ObQjN4FP|_F45l1@-%j> zVn#nb_y}vFFMc<%3BT1Mdb2@AvE+C5*kijrmhQ%HvbeZRr*B>+dzp8QJ@z@+plX4y zhrWsH8dQylv#Py~pCYbM`y2m^xN=CK@h^z$7W*2ldnTJMVGY^vIPNpY)zvmR)AS?oS6MlssVaAgj zLn-HN&5zl%^;{Y2B%dqY&7KNaCm~A(e(LY(Z%)zKOorS=Lx1PIHA1i2QL#hgBig)m z5$AJqALnb>Sz*%bsxad>zxQc2dihGHLH24AnDGif()A3SMC{#%oF=NC`fl|OsZ z&irwcKFGf@>HYk_OsdPTom89ul>BW?rhL0bFR#|P%lBwr8T_7R%tLJYfx!nfrw1R@ zTpN4{c-G?g*J|-Qowb@8$YmnpFUr`#GT3tTzI*QAJsO6$-4TP|*08XhnsB^RVtB8_ zsaZZ^Z!f&Df2Swo%txH}uvYy3@ieRf!kQWWx$Gk@blA(dKSwl&;~wjt7i3FnZ2^Bi zgZrsE{1yt8<&<)%8*q=Y3Tu20Y)svIYEu>J8OjV>=PL2;0rN(80!iYCCV`Rp@@xzs3SLWXaJC%I04G&|DJHl$Ne8AOM z_lauS4SY=}>?!)YrgB=VakkJ5-flTBGFJF8oM+(ow4fhzhBnFmREYb^cuOG$o_X)6 zO?1YWSGHm65ZEI5!1Hs*_&0|CNwyAn?uV>*;H1aj!4&;Y0=09=;mx}dd#x&+NtaW+8(o9YyJ%ZoqVe6fG5FEGKso19 z$UqPI5|_f95cJ{dGUGVpUb)@d_}Hu*ya#8e<7cpg)c^M7IrT)d)-xjGAz%EC1LnBx z1MI`O0aAgMpkwPHk+I4jzX8ec?OnY^z65VlFy3{b8~vUc(Uxppp^pTA;}gqr>dA+n z1o&l6_>CgtHo({8Zg{t3h1^My{@LOQ7mKJx!V+?&ToRb-9d_jYILq!UQe*%Py{ zND_#MEJ(#Eqnb4vo<(;tn?5Ob1mtXbNRJO=?G}65THyzb34oy|JL&0QWD z&oZuMe9Jih0iE^tvUZ_A-%cC0?Q}NFT=VrC+08N^wZD`D56Eqn`Ag<8nOkJs$^0U; zIH8vlHBnqT_Vmxy(C`Xc;>8gw3TPR zlyaqf!9(!z;Kuy7R^~q`lR0p$%z>UVO80SJmi{Ssa%3-E#+Zz?$0iwjlpRYWS?B-X z+w}8{({E@`8JD}+6Ir3!>ukG=x68i8N@QG_@B8xgV~^xZ_DIs%Bl#K{9Yi-+rgpc_ zUi5PPS@uZUn}g$Ks)6>g=Kqaby3&e=QzfCL}+w@vG4<1cW1tJNHaE(LH^Er7QX8&<&H)kp~C|acLByI?G zF!q@vTnF?zI9wm%a-n;W)Z_AkTcCL^w9jE*4Et)FXEFDgi>9%!bqk+fC*J|OeWtgm zU(P!>v-f5NzK`oJ+ED8GOK^VO(Xn+1pU_HZq{Djm22JiM%%flZnooGhN#gamWBoPJ z7YNLGmx<@QDt^X|n%D^F*dzC{1K7t41aCX|2T>31WH0E3{Y9(xgVh(_=cBj#Md-$% zSJvl>YXxs*cb=Q#j?YfkVGDOHiM?yr(%7Fk-@T&h&3gEo+vcu1vcBh==+eSn_0s0o zDf)SiN6-^YC?&mQLNeQ8)BE z*iGx3_KhXPevZ1aM@YSb*r)5CX0clIx}BhI(?gf&b+b}09|gRqo3G`u*e}ew-Alc` zP?zc9Qm@)w_`%)RV$>~1c`Vk~LVc;5MOhLn=_P(D?X{HjOYEELb9^jy&sr>Xa!wrG zM^?{oILj5H%oBNwGtG4hcR;9LXWdq_=J3>QVY!m}u%F_l->tQci_{ckgVyCrb6c*W zMQ&p6eJK5be%Qu6h5*+6r#a7dVW6VLaIS0v;c^E=_R1$)qxa8;7pEc@h>j+6z=jRG z^gHCnzIrJArwx6OO^vP-Sfa-=2x%L;?fR^3C~IfRkp1#tk>&g3=x0k(W#8y}OKMx6T(5sC|F?Mk`|y9O*S~M7 z@UlkkZ>?l}r>2T7Rpd)syRr9Q2yeI@9x(=9Q2@`#XYYS>6Ffxcb)&#Z^bk{#83c~- zzdYO~+hawkAH6(BbVJ41eoI2GL*}XrM9)yHs_l)mVGwiCD)dis#xt4!RoFSOVY{Fp z+d8f)m3x|m)ulVfk`tidLh__M_yVc34$*~ESdV|Puk zvtY!7uBjwWK}3n#jukLF@zWK|k~;YxWPA580UQwNu z2cW^x(LTEUzE`lhr~k9&+zpNRZ#e+2N0FapY%qC-cZJh>-hd-8M8*~vN69biM#(RE zH<7n7)LBVC9ybzMS{d9BEA1&^Hp=fB-mz3Y{7be&&ojr#xsO)H-)_DX_~#1tnq)r( zp0UL?DP5C!#dEIf%&$1tQ_ zzRbr`?qT9Z#tJZHEF0rp!k;7jJM{NQ`1HCX%ZRN#*(;KGsjmebwv%q-#SYF>M!7x` zKm19~Yb6sW=QzZUQWN2cN>n@N<8;_-_?P=op8d-c(BdStn2!xIL(YkIHRx#^uJ=ql ztNDNEV#n6;d^$dVAE349anY{z#65?9BXsjVlZQ^R(++MOX>@(n5~aDsQa#-JOq~{Mnc)FpTm+e6x-AUyI1^7aL5OT0&sXJAh2L2WfkaU1+=$D z`8J(ozb`=coheJow1ku=e6wN3L9J$dx|y17Fv>(mtfUykHD;FM$5}w8LoHBM;tv8*-|_yItG^_)hEwAcGXE zL1m55VT~GCa69`xo7HJL4Q4A939nG^1JLlmqL=CqP_IMOt6N~V0_22>vjf@Fs=#hE z@n?xYOZ+#)cMt4daA?u1^&@>l+TVs>zYI^2G6LY$!Uv_^Qva!ZQun2})1pffa^_7- zIK6m#eWEXWCd(UQtP7+WS@L${M3YO+9L(LoVJn#ri291t=WFZ-X?A;uGmM5 z9Pl`Mo#ADJ2ds}>vH10Rkx8q2yk1`eT^jFX%s@j~-($loZKoxQ^BoiCEBcrcIn$6m zPO!E<$(Z|`@%07cY7ll0uLt*In{RDO`MzVT%7<=eZR+5GS~I%0$2mLk5MPJLr-OA} z$j7P3erNom>IyiI_ke#~-Rkcb9}!!#qE9(XIlGP(4^#e@ym;g8#2vfNnSNDyYnSRA zu7~?7Zyn(7fy@WcTi;hg=ctK&Ky=NVRf}$~fu^qqDTCJY+2AGUbv~Xsd*Qk=|HbPb zcx3Uq+u>v7lzo7*&su}_Z{e({^1-g*;;IJ>*YdI9TK5{R&z}v~_rDF-@9kZ~*CNM- z{`6?kYZu;_BYFpTZko*Rso=w$TW3?_>J+6L>r1sgiT*v3{gkQD6yEmn31}^8(FTI!LK$?Q>7NPVu=T6gbSrTIRerNeSUl-D+q=LB+k1No|% z=cke`hjcmUjiye?D|K9}i?a z$z2&aOX#3a{(G>a{*tewUU->1)KM>IKf*68KC%j#{D=}+H-CP&`Xlrs=bp(svX|bq z!}+^m>CW7{cVt_CgMWCsbI)%GzX$&a!ix!azUBPg$aH6S=Udrv;#bn0eee$yznbnG ziN8Snmh}8%iZlk?HG?i{0xhWd_=!^?E)u$0)P< z?a4c8mPkFb(yMds-ce(bdfuE~{q}DNmwFB*yqIvQ!?5(~ZqB!AV#Pl^z506mH;I2l zdi4nW`Qjg`*K)^|Nl_tOi!8j{qNOtD`o2Kd2hG+=xJe^-jDt_+LXRLgzt^h z{u$V|ERw#xpv^zy6#Q#1clSFKd;6wSx@|yB)-NvBQlg*V{RQh(_VY(hU@gh|FcBSo zHgjxSU(N-vcOTJPNj=l&w$#AhYU&v~VGgG|u(u`kOro=8G-Eihw^d)8o#qS$`?KyK z_I!o!_645I?`;Fw|L087G0p&N!qw-ZmVsIzys(u#$zJIicN*!Y;OgnJ2Wh9#)gVKg zvV+a_EjBMbyhO%!d-=k3UvhqE7CfODp71qu$Jm~Vrq3ULGUf%$0qBT`Kd0O^vE6X6|xgAGpKhU=x~KCZx*?XGca zoz1^fl)`0-TDV-X6xJ%%!aIES47d=Yec`^%y)Bl)>n+y8P~yUg zi{O9mXUXM7e6^%G#Wy|Pr+IFkwb_Ss61JSU8~Lulu{7WBRGXhCYzmus0Xj%tS@BzS#ljUMv0YOR~*W38M1-Qpv?nJ@elpStI4&dbkZJM#r_ei59vfb;X< zteLn=nSyT!@C6?!_dVvOcb!UeZ^GUQw=~yQtJ<^@_DqwM*JO=*QlEDyAA4}eEk65~ zqMMmEEVRCsIjGjhr_RUXQ&-zsJPg>2r&lNLsQt$D@4~M*hwI_*C_8GgFTtFvpLckM z`6)@6mQcf|Ab(}!KFgOJJvG4>nd@0(F(v8Vgq)Y}P1sC)68>rUKWDG-dgNoDkdlPG zFHcYKPchP+!|y|!$o^-CT6x>Yhxb8zdDq2n)-SCEJbleKvv(It{?yR-@{fV%2*MKiA3^vC!cquJAuNUe6#nJhqR3CrD}Ia# zJF-)7dkJ5PgzTTZXiI$r@Vj$XcbTuP@N95+{e743E8n|fO8MS*78TaVn$`$lrcx zGj)8f$9)gH1}uIpaHQg2Hf+oC*i-0iM_S^`@>vHrXUTE3ge&WH`02T0$WER(2|r9fDdFA7-mdj@Q&qyzGo$KNAnR_vyi7FL<03nHC$gBGYo-@8e$J(|~<4 zk!eM~a`68Ra{?P4u3__sH>b=Q(LAEZ$mU`575plGOY;DWRSU^6WZJW&?O@-t8vACa z^nCuVv+M~TM7BN4{PKgQ)8PNSKXPCr}oWLk) z@0f5S`o|E`9#%v5`*WxETjchzy~-ug2}}AS@c*SX=INfJ82`7z2X5hwuiROlHRsRt z3L#hY3J(`?7AVZn4X{7ge3f2-bA}zca$nENI@X8vtS{?YKi0S0)mvaguVB#>=&YFg zUD<)?6VNL};bx*&_#0=x(KEC>jXU`Kq9ZBXv)*ZmZVw#rRBTF>t3Kdz;Sp>Ey3l)_ zlQS5i56DYu`&eZhBndA>50K566kR}yQkjrUTPyej@XPu$&|H7^#myV6ga&XHu@u=q z1=&9p-G4vi{TYO(5-$Jg{J#>Qgud@uRn+8i%{joD;4@_MM+p=EKgG|zxnip_=dAV7 z4P!^XQhx{kA*>$`EDDJ`peBz0nl;w}kW=)vCDidpgF9lk*Yjks~X z!39-e&I-{*RXLm$`{-K_@#ahl@&mf6Ibxe)Dd+!1p2$Jp#2weIjdJI5))3@;L+7xB z{v-QYdidW5={g79|E}z1$=;fk{R^AxMcU8MJLJ(Wx6wYMXs26guUpVN*Q)VDd%dlNd4C492>zn3)kMNdy~+1ft7jkxD* z_b0r|H~#z4P@2y7X0=5sY?bk@&}R0i9>+F8 zePDHw@QpE+iiBBwoKFdB;0(cYG0yqb&|WVuaC*Y)lsh-@-h`t0a}x&I=J1SKMZy}| z<8|7lh=0j1|8Met1ik2+d=GG5X*GRgHGKozLs!!eHqf>Y(B@skpCNp;*+0IruHJBa z#>@4E{2!2dE$SAxn*PC=**f6Jxr%XXNS}V8V_U=bz7=iglov2|rLCU*wqxs3KD|A) z!P-i~Tf3`G$C*!`@C&!^TD$HgBpvc9Naf(%<;#`kT12-UI4mOD#SH z%H`b7^rcqc&VL*E+pISB69dv@4iVdXwl9%&+dH;4(T_zpbDnS;{4{{?bIwDHe$>|c z_4<{}+g3Gzeaa~Mu-rX~aeeMkQ@sHn%S{m=E23Ef~C!2G8XBZPbf~%T)ML3)N;x`tK`Y;)L+#3t0;#zT+ z;`-n|hkJd%$mX7*OJ2_BoYNHaA#zr9BWs1Dev7{qy_CS(H*@hi-tyVMDj=|~FXh-N zXA0%ysJn&>yj&cW&OWmbal}n zbQD!XIZqGW_F&hy>So?(G2ge4G;$Z>^c+PCMBiXX-%y7hVm139x=t)CSnGza;cQWi z7J{BZ=EGCS26Dbk^g4U!bG?waKZoD!rQJ5}n!ICEHEa3WVd)#4ckkGwu$HeKp1yI{ zZwO~CUps>EV!~Oc*N#lz8036wQ?&S%^o_mn-za`Hed93vdE&RE*S3lr7F4$vIYjga znv+nmp}dk>*71~ALwO~&tm7%KhVn|lshaX?D6a&ZswuCA z@=Cy|n(}HWuLPW?QC= zMJPH(Y*TaJ!d@4GEj-aPPGJve^&Q;Tw}k8a4{fq$?VHW|l{ffisi_fDEUED`tf?`{ zN=p*FB`QL3%R4pF9~P5O_>G*i=AD|Y$B;F4A27$LXU)nue8?-^zK@J_HMsg%PR-EQ ztj{v84Sl!G#hGO6%!^#$!#v~5eB;Nw!=00{$OH=lurqJbTG>}hT&Of9+7!16pFPf5 zyA;kJMyRTNl8+KP&qoRSGdwn>DmE?5gr?9oy$@BZAia8rHaW#4l%I!cZt zJ?;FlMD`0v(~G&%w{yPC?3^z>I{HSLM?cTd=gTWN8#?-bn)Upp^HkJH-+OB9t2SMt z4<64@usyA~KZgf5EpDj~r+v=D^W@A;4zOfjt%NZ$62I&#eFy)ReX*hZ%Ndpuba0vY zWl!i&oPpt<{niGHa&CI8^T#B)rMkBf6T_!`_%9(?N8w4UB?@E7q0*P^Vi*IU$kz( z+(qkJt)cs~$v>4Z>7>&A7$z02HsEjqNK1J~;h|2A}{O8D;lKAF#` ze)da~`)r5LP*1;hC;a|?!j78te48}f9}DwNBlQ&f#sa5LRd8H~(yi}t=G%9Dlx-t9 zyXjdoFsX-s0qsvA$Nw|u`nn}{rFn^6ZN8L$eO-;e+FawW@LBmRe7<}>eE!XQ6*aWR z->>;OPBKd08eQ?#W~--=qY5r`GPgo zNZu*i#s0U{=X$H({yo&?Bz0wN7b<*J+WSw`Baw4IGN(pELs_d~*8zP|xVhIE?RtVX zlRgO@iUN>p8uBwUcXF2H1a_ErA_N|5sFG{!>+Yice`!D1pL`{ppYz475_`>@ z8TDSf*<{SZb9ccjH}lo<-OD$MZw6~K_KV8ekW<9YZ5ukDzr0v$z{8(5hQ4Q;H*SMOSd-={}Qmd zOHfvOIA^}o^vhWcJF@mla6%u^LSJZMe>+tA!zYwMTZQR(@-KVb8#()!s#wcrz$b3u zzc>FQRiCny0kyFKs?{#~uL5u%10G+o#@Y!E291K#g6EZP4^}T`>;#v$g%;GQSVg}K zjRruYHv0ES_RDv%CJ6@TS$-EYzJg9S5S9wQ_|v-2o3*`zItVT0E+YM3KO4f_7%pY) zVytzWIBWY+%9w(mG~;B=dX%{1(5lef^DdD2e~ zTo2@7iDTe~i#ySD!8QB=^q88DGS;v5wZbr8WvqYg9}6GH4XJ&-P$O)e{NoSry=V9V zMb*R>euSSgHj=wFPcD0*@Tp}l7CyafOW_9o>zBP&_{_3D7XH_=*9-5-c)q-fZv|fy z-{j@o?csZd?=8N`_irj+!1oN_TYMiR{&!)1;*|3J<(tcYOx#&GAOBDIBYA)N3H%3D z_3)XM%9yCxo68T@D2IQt=WseOUEzfgG7kmt(>EaSUx zQ~7}SZ_C5L=hccu>kituFGx9iYhNtnE{t|S);{z*DW`shSZSY7@cNc?=SlM&@j|;p&}<0!7xB>f#S( zyHDR!j||)ae4(ohntlpRcN2c9c1xkqQs_8^wTWIo#-z}3BYa8d_>M_OUvQ|6f4-BB zztj8FbA=vyz4yT8^6M$%3*gp*mxpf0OuB8P|9aC+==DBy=}-g9z8SD8c1st!Br_(F zgNnv54!8#q`U>Yp1($=+{u}f(f=}p;=xcb(ybs?NrB8zN^FQKB|3;P_EB&}ZG5Fg_ zt52G3-t_Hl%mKoSWe$MeT9xXf^S8g!1|t=pvJ&q3*d~%L=3+)4{zrqiC$ERs%b1tE z$R9ede~-LUjuYD+l2_m#i$Z4x?&0v~+em*NKDh#sP-KZj;dR^@iiC$AwOGpP@E^4V zm%Z~bybizEkounW+Ir*&$_OpRmH1<{(Q;t45!L|Dg+EVNAE<=NIr;|9;fkJ)@{0b- z`Qf>i$bxs!aXg|#+CQ0}IQ}B?LBl2Ail=W}%R6A*%CeR|iW^;)HSnp}DvMvm{F}5` zwZ;iEczk+ltV|OY_MHcV$6k7w!CJ$T{`}c`Mw*NS3$acYx{o--Z zA{1FG^b>Gtv4oS)R<@b*IYOtCHo;Xbd!KdQBItBM$G2CS;B?e4s!VX7#uyei0i^+=aJqv>xJ?a(;Lf6`!|-`?%h%kS#{rp^=75?cz>;un;=mu)Z40-vnqe=h7x`p@#R>hkN9F*jpt^Rv90 z>+-~(YxwUG|1`sYH~#45+X}OmLib$m03sVVSfa|RSz|Z&CCq6(+cDv(-zJ3pjdAgW z5^axO{(Rv!{=c_G6g&z2Eg0{%FmU{PUPj$ez5Le-FYwmPwlHPD zCgB~H=(6N5J0>heCXsfx&|WFH*Rt_-x@R_e{P!O4%$vbU&Xy+5hR$SN7`VmR)R%nN z*LU{@MmOwbtwD|mW}WSMqex*)%3aJh-aW7dDeiRcX$2=?)VN9>X3A=gG7n(umFv|6%ca-Xaot#+PuBFB*U^=YQ?Jlooma zs4E`4#hVep8yBgz)VkECvs>UHgBp`p7TCa%dkcoVM*Y0g1ax!^emPkg7ts5|4MC*q zOS)8aL}}1M$`yH4?)Z+pl{@gPtsme$lb{{WrViF1a@J7j^i?OFX6tV`?S_Z%lz3bZ zEkssDmeT2o459PBz%KQdwUej*Uz%@IILQ~-rM^MnC%CR=-V-+%+-FrL4!pq@P`5uc zyQPfryc1clALD!9o!KqaVO;9I8JTCetI){$2|Wm3fIVeb3U4(#bz8e0dGr*NL97~613*v@XVMx^~dA+}CIN4t{uMMm;A$QkHe$D0Rhc=Mni?Xk*yW8^XO z?V?9{ub}JOMS(p6U8{I6A(g(llD88C&Iin`&_jP?WVQKr(Y4aa8zXVF!BNsmKkoW= zk(4L*zeO&WbAQ)*yJ!t*JZ~3?zw6sY=&g&S-%DMP-3tWA(ezo?z+w5k*OAJS$;cI!EsfpA%#*igN-auDCV9AbANFTt z+sT`QwW0jWzGEF_<*NY&t0+s}5$enPN%?BuK1_tU=pcgK^}Hm~>R;2)e9v?pFxDn7)&iuNm31IqH5 zi&mTMCphTi{XevwyeCy>zB?tj>M-cTGL~{_o2R_qo$8uS+D^_yAC+=}{ zck2JtcJe-64dZbEbA!;N4*oCe7vZ@wey(-y{L<{bjk3P!*yD>{GZejM7g4SX_3rK8DUP)D+=Fze_iX3v zJg&2a{gc6tWM|9k;$}Nr=88K-?35{kj)FIkKek%J3nG9OoyUfCRbVF7F1c|p6Txsx4*w{+_stB832|cCQ z?Vpi@h@89?Q_w8%_aVblCZr@^*7=b<}>cK*!V`)SSo5b zrz7)~KlMNLP{R1}(D(kSO3VE@d_DLWclTQow%>1w=K~LLy(&NQmks-l>9pMe+HN*& zx0bf6pzW$@yBykXDQ!2Mw)>npP38dM5vkabPUh^ajeV%ZY@dn}^st=q;;gW(E+Zy8 zHEl{&YP%)S-j8;wToe?ypZ+o7UVGflw>wjZ_FkNtZ;i9xIodJaMZ5Gxe;04{v-eXS z_T}73|2J=P|8|iQx4hCPuJ5GKxY0Gv(3#MpFFfXfMPZHx)iOZ#Ozq&o8LljNPeB^t z#e~cG4?25z!)3K%7Jjko-Zf1f;l;$S!97iU5b4Vng^e$p67#dPA24pEzM>0CB~2=6 z0_pGW`vT);T4M_)Q-^$P#oDb7`+`N5xF;7-G5Sg>_OT~|_d+$w9zb94;5U=;De#1j zg2OES1qW~V(4?Ti8dLD$#BOo-m3NPGSmO&GS{&w>yO{F16F6gPOi{zX)rzKFD`Oih zmI0?|Gc(es4)kQO6~<;sx&PbtskrU%b97_ImCX zrmFUWzR=85rz&s}cva9)hlf3Iq5CTE_t4!#Bcbg}yXs=49ok>7i|vc8aql*;BWHTo z?P2(yr<^(X#lG8&Yu^9*6+Guoc+UiQ(0F)J5qtULxc}v=?@#O`9R#Qrh&L|eHBm<+v_@QHoofP6!5?;Ykt*irT;o@0OFBKn_X`o-2Tv1!Yj z+r+h5oY>V0SJuaoY@z+=fTaB$Bo~U2fKNLWRFB)eQMa8jc{Ec zUi7jEj&dhG%!aL18@5&%Kd!#qMQ1N8zkt2<2e2DZjGfAv*fjm${$~xR+o}A{nwI7Q z-xWenJM`>%H9fZx_e*z*ZSlNwz?wE6IWwTq*~EO=KDuV*205=P>wY`3=@?VCo;hP? zWL_~Okk8T4#k!6)2WjqEn|sy#CGfNh~?2nwqD#v?7bU# zL@luOGS0Nh{C1T-8N5=s>$QZvI@Tl~)3)J(z$(Btww*j_rN$fXiK?Z&1Q|Mr^edB< ziZj9swu()jfcBTDN0rL?IpR-)V<2^Tg}QWOZII79U`5Hl?W+&aT?pMVBe{_Kue_(P6u?;rSx-%DPEp;9|YK&!;(A!q~?-#@@{__HmB++nTXC z`Ny`NVNIv^`2Ml2kE3bhNy(=)A6u$umR{9GFL395Bf8OJ%nh=CXu&?kCc+w*-t#ja zAZw}13k)9M8T0SwS=Rq2nZILc_rq7=j!d$%BeQAe+?efwv|BOfbMk2;+sk6-#@0Rq zTS~~R?bUqQ{9s@KiNuXKIO<7?N#vTPT%>6 zzS9gIP1ryb_+O!ilk%ni_XYME%C!%&rZ&(Y1$N+eu~!n<{sH!TRs*{j8FFJ1cdLNC z8ra=|eF)eg!2Vk7n9wixQ$K<6HD?|gcxyml!-qbO4pqWZ_-0_=t8$Ug_#L6jxR?j6 zsp0-Up$_D1;zFBFsE1$T9Cw)-sWo6W75|2R6=9)vb(z)qLcxbps$oKGU`|C1#^ErEUU^oen+rT_AK~9zZvP7vX3I; zU<%_z?ox@p1+lg8(=_@O-zM;yiX4MnuznZz6h%M88E>XC2Yu$wI z;0bV$xinP`vIk6@F;Cv(Ze(1n+}AU68v3*g>{V2eX5?sWNMci$b$>+?`n{3a=zBO@ zw}(4nJ2^up?Gn&*h3CH1n|rqOlU>+VTu9$2qi+OJuiGg*n0l?!`S1SNNwksRe1>+Z z!Zww(4d>R}ss2(Y%lK#PMxB_eT6zR4p?QkcevWq-6V=eNFR`s#y{jkp+SwQPS6ZBm zU7@?^Yj$$~_m^nSJG%?pJV12B#D+oW{ty5S#>`w{J7>k-pK^$C;X< z+U!<6J@{SsAMj(H|9?$0Y&mz*EKX?VKmM6SFZ|NG;0JH=;J4vg_>o@v?YSr?dsEm+ zr0LpEcTBhdU61jJ93pn?(Sz!CV%l;t>r`JK!XpTGxY$ELWYO1u`kPDFlDpuc|Iwyy zDR_zQPR5Rm8#{b84Y!ItyfXS`H~5~&+`+ouDY*SB#*Vyezl*vFZ)5*M3x)q2VC)<~ zXV9JW*pW+nkUrSWDdQ*9Ri(cp9^xvXzqK=-K2nu1IbX=w7}u?zHI?&)X*bN9IWJx9 z#uy4N>j}@b;cqg>kj@`uzk)H;DIcIyurZEOZb_-LWjiZHK9IRkU+?Cs%z;;I*jzP- zEM@*fN2H(!lK0`uD65S(gmoLQ$Y;gFm2)ZdD?9jrU#NXv+B_S3HnKNi^Kn)tGX7G~ zGX&V2m1({Ue;Uhl$f5U(GBxhxarF&T|^bSRb8`8^JIlvN9=EHwa z{zFromDA`q?1fL5j&9uA&soXcmNFmyf5*Sijn2yF+0&?{o-gq4JIGnNSMe?L;r~7U z{W6`Er?4$sOItgJIV-FD{L5AnCv?8nJf007^2mf9J2mV`S8}JYQpOzT3hix)O6BRE zN@b}UU|%W!efaOIR5tSezWn#+e;|4I53ul`!T(V5TLSDe6#hr@pQ}`^;6219`M-_- z0;O^v|7~NHN@=(K@Rc*PTV8;39rlOxb1U>=eXlg2nlop!usduG-2WBzXz!_LqEB7J zzFwgkTzB+3<=igXQ1rm*+$DNNE6zNiC|hS&md{&zopP8y9a>R&!@L_S`_F6MRGj${ zapje><~@^eXufpUVq}#Idk{QYhHOHy5!yMl^2_pKg-LTSe@ktcgf3JS)Jtu z{5)?n?`U<#qqn{acIg9|&D_ZsoSCOXXM=A)@ZH7wY$fBhjs5LFXe4xzG2;VmQVExS zUh)k#BucS?i>-oLl@H8A?hP%k>^E;TFfNc^($B}8JWAGUzSxWkWla~x+RlMKDV+23 zzS`B}E&gIkFXPP`u`ANnM@>!XmECDKq^*yI|0{M!+WJ`ezhZZ!t&b1?SL}|o_3`EZ zirta6K7Ra<@`C5j|7~9Xyo-&U6#Apa-ME&%=rA70zI*$c>sC9OM}L^#96Tn!yry{j z!CN17oW1=({@1C6`Lna{*`CiF?m{P#4yx@%d!8SZ>t5p7i;1ZRzJ$| zcb&2=cbpouH~-Ycz4^aY9QmIpk%5mWj=)bWQTeCX3p}Mn*jIzU=+BLHv@iP^)B}Ec zMhUd^bx z@3_2R0)1dSeW8f;J?r{hV_kpfR?Y%pd+Y!7Mv|=StAVSurf84RxAM}l2Xwff7I4^d zIP>1$G~GPwo~B{rikog!v)w~{v)#!7*>2xtXKJFVRD936K4pqcv+mw`F{Q?)Md2nz zWV?4UZcpT@O`rQKP1VeEpZhzTPUL1c>98k_a^E{uX{wy5HZ7QAX?iHf+VopLpQeYq z`8GXb@oVznQhKwTLAs5)UHjdd`$rjXW_hncb@=McEIE%p1=^%BPdhnlCS&Id{Ona9 zntQyjzGj+W)b)^UlX>q%Rhku5<+Jcv`F!|%`TY3&`D}avtl1o{Jo3uiJsMi2PPs`N zdAVU9dk?y+hn?%ErCe@U;k~Yu{kdO?>uTrDQ;$+b$(U@#wj_1X!|ol$`cv7uvDmQ9 z_VAsGwAU7!daJusRkq1_IJwU~+bbM7cblY9-o3?r9c|IdnJtAh{YrHEr4{F`kH-$; zUx(uh9CR^y`uA^fhe~?j>Tv* zC*1M)B(|TfBWt4^8S9cqc;7hSwaPvKIOuJ%W{Ad^&Pq&Hvl6o{S@hK{smCIQjS^Nq+e@O)3n6@@^v@y?zJr#TVwa$=Kh}kvWqg6=ziKQw4=z#Mqdec zWr6p#>esdJOWlldnxd6he6*QnzEa$XH098?x8t=)?(l5HhUiX<;_gW}=SubW;WsXM zX4u9h|Hb!*VLO*3UfkA0)p*NOzb~ufEPSoPM`Q2HO^eTZ~RnTiL^wM>dtQYihKklcknByb; z!rib;x%hkPwT*f;0xJNTjf7^xZ%Ut7u|d{Gw%UM;D&=mR>Zf&3&c>;}+K;%<{{JL>*GW6}?i{>}y^Nyh%VB}vKB z|DB4aBDd6%x5KI){^8du^O^;@jkVGqxd*`BI_G#BxsM}#GW_0LcPVx(Q&c6xX0>Is z`GjQb^NY@Cv?XP%2u#T+4Z1NSuUlq@=q#jPM@$*TU4aPKH1LfOzJg!g z?%jT;yX1e~%>M=9CA6u=Skquz(O@2NCN-%+!3XiG)=3u!Jd$fcjoD}COZ&v}oPcO+jo#!9yERsDPN=Hg9eXBUdgvn@?pzXAB))Y(Z^np`5$EdN1J0c+8C=>!)M;yXpGgTfnUNK z1~P7+5dT!e|G4;b48I58M=AFg$2Ii6YFzW)_y5~*J&Q8E`^7Z=^>Ix*$#_aM`$Y-i zrNHkxu7A@d|M)KXyPNs%F!OgE*SB}cpW7vWhd$>-xSVGGuH$-mm;3{}}IWzz{GzmG>hHlU^*Lu==(s*Q5kyAZs1iq4O*T=2* zxYWyKUAK?*kj$0X#8}^^%d!qvBj^1#p$AAlbhFk!*^rT6q^_(9&Yfp{T94n5AFjw; zS7e7PVgGSI%zB;dhaKv)oy>li2d7fzS2=Sv-jB6cy7JB39_hLrgAz+{bJcN0m$^2A zpK?bpZQu>YTBQwJc&u+*&Zm-oL@zBIJ;~kRsow!ychAMDbqN<|@n02|h(B>&)yIk3 z=N>m?GZlTPUQT41hCG*5^~a#?yRt3o=avU*rO0I^$au16`5W?!%va5N?|%A|tR1Eh z<`i3=;AzuFxec4&HdnD3KZ$seeJ0=@=ku%=^6?9w2}a(44qI&LPE3FDwvp89%U+tw z{$mel;7>Yv+pZh5U~g;7+!qFGLkZhy4ZCRL{9;?2m3Np}i?$BYHZ8RtKE5YfV?XI* zAHK~}24?}<;&R=I$=owT7uskox|oMNqpZyRgQr}Da@jk%3~p`}SjiU$YjW(N?b&W6UfG&p(jXcuLt&grZ=r2l_C<2GHK!u0jdNQbJZiup z`JLg~ZIr)}aoJh!fV3SUcUgnV zKFSH;*r2=6dmC*nee92Y%F1l}J=^9JeVoa9)qej@!>Z?ESvwHjW#BIKA8mhBDMUsxY z4O<1)EMQ9=9|NY8u?koE3-Xrq#Y4%@_=9JpD^2i1$GA{;BV6+(9pw-9f@_w4NNKcb z$2eCj<0r#R*T*YeH02M54pRQ@X8E&8hy7gFOx%lno#mIx-c__KP!Ef7-HjUxtVYJE z%^j!TWA?^VAztw6eyubntP4&fjnpSkha2e{rMnTXTXZ+vm7}{3mlIbXYn#%w=qXC- z5A3mJGNwIavye5@}p<$uLsW$PpK zwbuFB?mB2L?T}8~Ao8xClcqgBM>$*?ne8r#RCWG$9dT8$s=KOObvM8#CEp@5ANq~; zrJs@ZGb3#n{}TVO86PF_HxYl65wFvec;U%XZg+{#CqCbZx5)p$vOCwrvd&ET{fZ5x z|5A-HP?7CEPd!VIljpdR=X?H3__ri+&z><*`bD<8jqx|1aGh`TzRjJ~+h4EC8Nw>5 zd(tD>?sq6laQK=qoo_sQlU78wixTO2DneHV2rE`d%W5j=I#>+SpIRBLR zUlIQ+BmSTn?=s{6BJsZ^{OL=e!da^h8e%k zjNc*g_Yi-N5&x`?ax;Fa8NXWMXA(cth_5o^C!6uL62FT0 zRYv?HW_*zuzfj`Ki7z+eXPfc)X8c1E|0wZ~8u9nbzl?_jWozkPr#nyF1kNRiJB0ho zywy;8fOv5~CSF|nhfQ4i!9M1?A959=UmE=TP|l$1v=Qd~Hxc;4zjfX{F5BH`;!#An z@LoOq&TO~vZ&{xUjSGmA^CPmq+gepKymcwxa=yBn;U&ni3ikw#?}^ej!~1WjskO2$PXQ#THGf$ zM^di{*HZXqty#A}QO-}Kzx1i-GvZw@%YT?L4>sYKu;+Bw;d&PL``Jp333iXVPj1wS1let2IMyl=Gl;gePH$x-5mr&Ynza>Wn-tAhVI#SgEl zf>%i$OWb~Mf zuT+IzDMkDv@Q+a3z46OhPtZK|KiD!Y8uQ1kR<(c)yPSmyEC` zT^3p!r-|J#Pu>_kZ-mQ1noWGh_=?x(TN&3OgbA;+<34S~8*sjiWjw{WUK$~NGuCxk zcjH|@>8``&#+Cg!a4}@X>G7nGa5WI_-KUH`%)7sOzGzpw?uNTg;|l-L`yB8;*TW-S zCv=zhtZ}#UrjyQNkhwqQU&kG}H_DK42j_0a$B~dz@FG;7%(;u;yM0;+e zpYEie)_|Av#~sM9J8x3ldGyORY?aODj?d)&PPfzQbgyKsmP$VQb;V5U$_*X3Ww?o) zm+ZH@egighD$;S0>kiF*HeD-c&3BZ2_8Fx4s*k1V%bse}nf^+XzUH&_)j}v!=qBr) ze}PxYzu;d-;HSLvCg<1Wy@@UGENPcZM+`nXnLHUoWGqCvevO|#aY$J+${n@F&wU5~ z)Ol@AtJF2hrIB|RX{8-+Cf($vgEXnfrGpmtiD^A%y!|F{l6-t|8M|0VE$NpXKB z@bRa3;Fk!miqd5t$t(0fOd2=wI{n`S{vLtP8-^bI-!$<5K!@*ey({ov1^-vEK`rp{ zHxBdA-+-@r;om}1t1|2puO8uGjXZ2-eofSXR87K zu$fo-&q~s4r+mHtzY6?m2L8J|_`ho4zeI;0?^-1A-vs|R7599BkAH;+eu?n@C|8RY z{uQK|WWqoH67X{k{2M*^zhvM)Q-|+x-7D~Sg8xp%Jyqc2Pw~KSHQ>wqC3P44Cz0kN z@jCu50>7t$|5^|JFBG0!Sw+s9|;J=5v7XlxD+s&T-f1Qs1N-zAgNz-D&Kff9H zRs;We9{e{O_}{Fj;y1eIE0A74v$&AuH;2h|%ldaNTd>u#&VQP5K7hQGzl zqMR)2KrIP8nD^Zkge48cKTvU(a_b|sDwqk@`&q7pIpL9Oxif{3*Hy0tE3;M z!@^dY?qVxVciI2f-FVj^-3@o$g!@CTv}dF%3cq=maqU%S9l0~oiaga}z3xAst2g^s zC#|**&{lA6)>C&+oBc@pNWY7xJZUp2SNcw8-D_{s9{0+31>QjI82x3FSDc5QpPMlK zfg|*^;I8$C`P3loWv_glFk1)fFtLm9GdvHOFu$b@1kbN<7kIdkwmopByozG%XH-^B49+#+w7 zf4f<8dgbecsSMM01GCU8&V#4S$ua8?c#6!!g6l&juCsA3^GW~I;ZCq7Tda9`S}*?WgVb%GX){ zr~Gx82fgAvcsfj&gMlM>-iZ6(-Z0q@`phd|C(LI8^t!L{iu2(4Z`wrYnFJicGY+@O z8|I2Y?MbhEoiH!iwPVopd9OGRo~KQicHjt}ez@zsVO|Q-ws__1gjw5Nn*vN(&;FUV z#|Gxd#=0m8ejDYI`Agu*ywy(p#4)DaR>ypsYsd{E=eS98zRr*vPLfXM!=t#)qsR?M z756{!=iyH&@W{0W{5UhO@aQ9?aTBl0JVu z=Ye0+Q^$X^7yfsVCfbAlT;MJ>@Ly-bbt{0qRppt#2feEe;B9{dgX zA`=Nsh5osunQOMMdp7W=34GqKG;z2z+rWF64nM|~A@F~q|NMmhPvGNkB!8al6U@kV zhZ82WXRmmk>}9u23##mRHK6iY_6|04&Y<>L-W8{R$=LzD{Rw-Hun`hg>@dRM)$b6N zC1H162|G#HAi|`c5tP$$zs0S$OGz*7WKT&aZ*)exL>`j9dFg%w_h8aW``B>ZV-rRaIk|7B6%4*Elf;=YJK55JB28vUV{US@`ww-q>N zN%QpWMt}HH@=AXwrkoRDdVdxfQt!{j25w*IX|mnpt?Vlbd@K8i$llD4c zuQb}tUfEF`P^s(D&32PDhwKXoeWWd=-403E&M+g)Y`6Uq_U4r^X}7(ENxMl|9pk;) zZKv>8A!g50PH>8P1P2+HIYrr<^so zPJ6cdF1zYpg+C8}Y2lSR&m>Ip>2)4&*13N`Wyi0>vf9RnWgXufsd@VAlTx3%1kbq6 zddv7b{%bSLtoIrT8-FEC>b*j*x5G7q@;Ywws&`ML-apZnCxH73_0appZAQIINH6Vu z7w$mM{E;jDRqE|z{0u9cM$c+S)_2j{g+eS?@_wpYx$t)q8@3oi)SEdXJT`_A6mh?|i-9F|NLp*D=(q z-g_inl-aSY!{p&hhH+zgRCU3fwLESfpYR7Mbx0!j)XBcfBq34Nr zh2T0J+3raW)g6pK4}U4NGsdbhmd2QQTY=*vWzZ+2Pn{nK{H4H$mP1Y2UK(iNbUD+& z=_0N(3fe}g?(gyE;m>0n8DsT&y*O#O@PaPPqdKs7cOTRVh(8E)Q{ze^6 z>SZwhOBr$0Ax?GwQ_8@<4;+m;7-fj;BV$DBaD+5ZoAA%~1%9qkhd@sq`WkikP>0X_ zFYptoL!#<_N8sa^abeVR~_CYjnrX$ky(d6l+n|u!#Q}Q)YaWb%7}5ju9p$- zdPU0U3BT#7x?hqq@RyGB)WIm@jG0&J@T`aD}AbSefdqO_LEmWkqwP<9oiJm#splt z0@Is$oDD3QBWH@s*9midxEAh}uM=i`q%Lp1=T)8u&rB2MWZ(#%cjBJ#h8Yy4 z9rnuC33ElXrh?~DuQ(5$Jx!Rm0!Q$4;vVpZxjIHWGJ*17|fFpQD;y&jMGbusa<(01!Ci=Hy zz?|z9XTXecoit(k0Y~suaUb)B=@c2<;o6iT?H21gkBd%ai?`0{Yr;G_m3I7wE{MHF zukt!^S&^*cB5R2kuL0{HW`2+U>Jnq*X)id@X5adgiTB?~FJt8h?i6plOM8HKv}-Qm zGS1$`%>?ggbXW@GEz!z(dcqWRPKoHS-okI_f*h_zUgddczrieT2kE4|H*lrDboS*P zB5OKazb0JDYr^eHr{@ThGD^`uaX($x&pnK8u3!PC7j&Jl)|6*27|5#kgJRG?_3@e-e5M?jpygd6n0R`*VFd zad(h*Gx`7zJv}(doGjz!cH*V&@^C-l>%^%fMaQXah|zX86W0}|!Gw8mx(UDF*d6`e2HgIbjxJ=zsxtq^nZmgCv93xdtN4;r;mK-Ri39_ zpPA)dBAwLh0`Bj~-&wDI#y#aH_3~m|r*XUX$&-Xh83QR(`p~Oh<;3arigE2S%Q;S3 zIe+s%xMkkuOc4EeoNE!`BKID|?OLx72=mnIJ^V(!VqA}VmFKCK@K2%RUeZaucH@rq zF7Lr~y}hOoE@MvU*|oi1Axz59$2GEp=-nUoDl5ipvz2C9FOgo_>;>FG-es*e^zNew zmpVR$+m(io6XqG$Yw>&9Y?4=bonyo2Mol&KHe;>RUuZ0{xyWt<`9D9@keP18?`Ewg{URNITYN*$7cA+(Oib@6r5<>d^$4m->`bR({79RdjR)WHwGXAB(lDzCGR^0Ktu z=K1Egf!P(-4}^8?-!d@uRr(@wMJ_ z*Jp%z`uL~#rEHPOH-ftjKj)3traYXMC4EhFFt3wF#;(4$t+G`f|0zUY)Ai*T#7SfZiBWC*Bb(Rq~dmmnCpQki^;=w_!QZ=^H1L--kwJr(Jv8t)Xi%=_ ziFeJwbq<6E16B8Z`1A0uAivOHD*xE|5M9=xoK|qKUcsT!q=DON)M>K7g9dlvIyeI> zbCP|`z-(YVeGXpo`0N8xu9$JwE^ws&uHw*@QPJbWq+ z-u!fsu@*}PmW;bx(#za40{26{&N}*({eySD zJ4R}8Ugb%hg+`y7`TR*M^|j!xy^`;nbd=lueb?S+ z!;yUN{eGX%?~n6Y`|SPQYp=ET+T+>hoMJrZw)F8}?|-o8y1Ly9fhW0ZKwYGT{4ZY(Ewvwjr>=5Lp|C zupQxeR(_Z|8uzXrg$=sE59sU#_QX*KuL>Ld$ji4gepWRDl=Wrdx(&V~KY$F7ZGo`C zciaX8Se{6=FQ{!eY%pBbdLe8_xET4#2HkP*IxcJg8H!qDcuL4{oXgOO_@SKkpc6&; zUy~n@4RGd3w?X6Ay!;5l!v^g@ZKGg=QL<)5*p9G8FW(RMt|P(*UEv2U<=+o}UVhVWIrFFiFx%MESm zOR1elgpge#)UVL4sZY8HJoPONpbJ>}`ugM$o!P|mD#VkH{|4Q0hrX1?VdC>M(#fa4 z2c2o;Q}3ai!TXz{KPMTeUf$939R;4+))CN&#(Y0{Dt|ZhQKD}v*Yh3F%lia$xG|s2 zOF3jHPxPfP>-ly8Px`(Knqtgn_f}3C@`>`6>G|H)%Xn^qU9jZp`^D!4H9v%#+v6{2kND=5XbV&a5$wI82=zSbitN$`c+$@k^c7sq^-xb|c&+MIrbncz zfm+ZW|Dvys`T-@k7dk_kUQPNcCG#TQ>TjY+0*8*pBd_0^SFIg?radVROh()FMNWkYOj6;dA1L za`uC+MSeCGze{~E+1#q$fSt(Sw08*~rhY_t^iR7%ZBx((Pm#6v5w;_2XXS^f@8I6G zP1vA2`rwxGX>LSy-n5OE|2E@?{Zvpk7C)iOu$B5@ZiDaiJdInq4PIw?BGq-Ew&}3J zbXijnwj*4D{A8z9xOZ(4Hh>J~78#C%AM>LvT!!a~A8b$vI#-lmMt;|Z+2BKAgEDS| z1%!tUo&>ecgbilOS^>g#gyUKHVQN0^F`wU&p3{TX|2w)4B-t#k~9m#?Pw$3Q9Jp%GYhMi0U8j zSVFc)VS`2720ybrk?QxLHpEQ?_5{LqgmaOfY*2%H7i44pRtp)nw8#)DYyg?5J`WQ= zl=FAcUZQ;1^>h)+2d^g9f1}VLQV0k5Tzy>if8N!9Tda z_ktg^lz#*KSbK-x5FeHCv#M`_vNix7%opL`r!%+>XndkPjqr8S?{6#*=Kr8J)XhZH z*((U!5w07uq4W0c414k>NY=V}5}4NiwV;ez3tZP^%~(eaLAux4~XvgX!D` z&k`Q_=7ZYcOWE+Zrx3Oyd?=6GpaAzS^o868z2OJm&3^Equ)!2w{w&7Ns?Go<8!P}1 zW&`x)rw@8FKPVM8$mTY9kmbSrAJm5SF%j)%9Kv>l6NC*Q!+I????Fd&OKlwoq+`C2z z8}xx6w3Pp}u)!!^emvu6RpUU(1`~x0!^sc04JHa34Cgk8V0keA2eqw*4bXpC5w;_2 zVKP`%Kis>92pd3#xE2}i7d9BeW$+|^uz?Bm_$=K98RQ4t21&vO8Qcc9dT<+D1GTM( z4bb;CB5X&v82QNt7jW-N6*lM#KcH_q@HJQ;VS`j&{#nKk@BV_44U&Wm1IQ1&m>+}+ z8w}t!_<`lY_z!B^02^$;_>Zt1;rd*X!K!|RdsjbU1IVzYMTQ_@gMM6w&xs#wupe|M z@-si|MSg(t!RrS4O4cX#;x_n*@bH7(ptdsDpbXL^Qf!elUY}|tJA7MMfR1Z1y z4a@C#F93U^a{~D{gV`eZ+7{l0g zOfF%pz*spZTj%phB-bN{(nF9Q!qdm?NdI(H>prYuQ-h}B4OlOYiBryI~ zrC7&wCQNr=x^qkv-NQ~5-{E~&*dmuGbdDzq-Re((LjDS(kTI7iTz$9ea!EwLnICsMJ1Fj#(-RO>c_)dKd_`@$Mh@#GO ziGp7|QQ+&3Q9F!Oe*>P6NA!FmMD42kF}1f}kk)uVZ*Q(n@bx&}-Xeiv<7FrA>puf$ zJBfCFQr7;3upMCw@)7@UaPNu}?W{lMxGn9B=1h!F9Ov@|u;h>b0A+m*cxaG={5Tyx zN^=VKoMWl^w(n5wW%Qf0y(!pPRK>`9M;c;5Z^GhkodyJ zi$HCS=;s<`Z9c+wgh^k*Jc)akRoFjCIaAak=SRBUx2(i3LVb+!!@EYHWdE@7+*Y^z zAj5QSE1KufTC~xR+iD8S6QNE5wOvos9=tAV6A-o|oI76Hy%BxU#%mrqV&1{e9 zw(^4YeEoo>^6mkxHs%Y8Qc?`%MX>V1R2qk=yaZrL{|=z<8}oIDR`4?%+S9tNJPqxW z@`}d|&DL{s~Gx)vUwH1mzokj)U1+=&(u8R}DPruorZwF<)V# z(qPDEWpx{=uF>4CkOlM_}A)lzDrFy81SUuOwg;i^m^D4Yr998%JsXgyktG!WMIkXCxV_e=Bw?c%rKNE z^y{YQ8x1V!HyrePV?O^r$|6HPVb2&nUplZ0Z;X}4|Jq4UvINgb%*lS>G?j;%X<$r)tK*)oAR-tJfYtUdcF$aNxvgBxx8fVP+q^DAD$S3rBRL{2(c+zh*Q z`Fha4CjRBng?L4%qmX88^$zqf#5JE+!Q6w+W6^n)qdqNi3VYJnOXUql9`esL(64bd z%W3yjj`OjR>DN3~l?StTK~oHA+}4q*pU$%<_>c}=KtD3_e1_gB2~)q&b%;Wm?Ju^7 zRGqLh$rg=tk}VvR{H0koDU|(JP1wF!wjN>4-z)qIWo-S6jM$UYd0W7bWHf^+M&A9y z73Upf`j?*XCis#}S3!%6`OeU{DDb@srdybL7Sw2?W*KipHp}=HFk!9UFaI8K;tbFU zgxMJ&)a4XI9Y*N$shzq^-vURzb`*57Q6`DLZxEqAfq0tZ9smtTdB(GlpCC@08T<%g zQ6BkCI?BY{^XSHtG5C&ZgevUK>pMcF{*&ryH}a7!cYqGm%cpuig)+w=j%3)vDAxa= zE+00R#`kM5m(FOJOVjw)2tK!b_+0vR%7Z!oI#7CUvQs=O@}zYXpGzM`9;%NfPs$Ui zu4LszsLMfZx3KQGC2PwNwj*r8Ga{_(k0z~ekGXWDO8<3Y_wU9jO$PmV8;MY<|0MmM zMSjw6KBz~te)yIXbPPir>6ZtJHQ<``96IL}qHfo9A^B$@jpU!}t*@KRkl)PZFV*F5 zH1oB$E4< zJy_@H@-(?rAC3mC#MRu^Kkm$Y zcwc9Vi&O)UHrmiWo7+{FyV;q~$%eTG_fsnyd#MZ`pCfKU_O844&%z_@E?+;9y~~v)1U=LS@2CR z>PLzYPk!|+XzPCdDB{E!$1f2kS!k^}{!X$iOH|(Q;_oo805%NoFq5uiPwqp!jwRFs z^C6VI2Wd(84jZlUU3kCr^cLe#BT z&~+vZ)iK$*w^8TMy0SVRjCiW!*Fe4RRL5t!HTRw4fx-8|*>@gSBFx)fGWdk3H20x? zg!FtKX{_z_)b(kEK1I+6ZDp^HYii_e;#pvAI0HBq=LF{=Y)7~l`M8}pZ!zDYwl*7> ziT7~6O&7p-I=9nX!cG^son`_@dQBt#IJ20Ivy4-SKf>{Xe@+UMeUqM-WG5YIuk=S< z_C~x5eL927PV*Vo?=^7QGg%qG;>AI2>ems zA;3`kYC6tk`kwK(s^1WQyc3s=cjb-|e}v-&|60!9U)P8H<`6KX!*QX**C=B;*WpjJ zJMx>xuelD@tc*x?FO`A!`zGSOzdck2!o|Iq4tTaLF&(b!c}a(NftjfDZ#)A28C-{7 zfnhov;W|_>{y6_j{PB)rHr`z3;wm7|4HN(?RGsdOb1}tUgJMdMjY4SOQFL* zxDKmX894t-WpsxQc>i)am4R>t+9&D(I>=0i&kZ^}2h0MUf8#;$H*+1te5UCj*I^#x zZ&l|If4nP-GdbGh#2;Zh=MNv_{NFL?@CY!ZL!;;;_Mwc^?#!1q3LW-w9VW9fBGrjh z2Htm_i1%Q}QyB>BI$&;LVmi_R5 zwI%)t#~8B(AALOE zA&u_Q{O;Z!U@#uv_8#DGh@)?V;D5u-dHEa?&*60$2t##t7WAnW8Fs|>=&39)_xt2o)6&t5&fMMktXCU*Lfb$<@^M6 zxRGa!8U*v1%`D9=AEf?C0FF>Bwws{M2G(TbQ2*H17P1*%#odqAEQ=wk|X5dMu zzk!k;>2)(L4?Rmv1h!S@wVRXYa<6 z?CHQ`&Uf3pgij+b9?yI5zuo3{gq%2gCuA448>7oU7kH}U*`Q6hnq~hf39?&NUltdp z=76@gQ#RtnobCaHN%p!slttq(`7~jutg)b{a5a}Th(F(KjHkFT^**H0|Hp*PVS3x> zpz|4wbZT2^pkEvL^hjp=8zNt`&2sVGCiu|H2A;wWemc*dI?pbk9~pTj4^%c8@-_2Z znW`ih;)RU2Q2(UsT{^!|(6?Lo9gQuNlx#yjo!?QZNoL;y!oLuk^%gSzs`IqyJk6lk zpcM8y8rx)2=pD9JdLNbMyRmn#jO|g#Nn_LXO3yp-{LO>;U#cNZ z$oa9(^Jkss_n&j z?ec~Xhp7)Cp4#ENpsm}Bia4=%u@zzRVQPC_41U}!=b$#t{l@e=@LI3uZ{7aF47`M` z3Ut}mA`jXBCD2Ht?E2nWH^h_dg`lnNvIuda{mn;M^c_x|nf;3|eCDg1(%&2FeFt8- zh@vh<7ytG&d?bItLo}RabfBhq_viP1mcA3LlLHO zzc!TBZ1dv4W}ABuAIM0$(VRL4_hh?6Z7`?aXIkKz=bluz$equtUoFfUjkaAkkG}`B z1?|5B;)WxRWb1;wu5DtjZN{Ds;qdJb^qcg}6RMv^@NU}1=Ld1XlP*!9$Jg`mzG*A4 zaWvjr)%$@bS+uQ$$Gn)~Bh^4q+YpTVLrj_rEZs7BkvkV`p)`8lFL;m3=?Igl>gbx|y--kHLm+K4pr$GL_Zq4?;2%fF&-+(x> z{};&TT5FKMNSD78mp=eFX8*NZ{)51iT|WgKi}Kn0dNr`qNq&6u7;T5ye>J!NhYTO7 zz7J{}1^Gvrw09Ax|a&)0%_i*ob_t|u%kk7S9mp{i%IdhQDZEr<$`QL*)%>IkG{L_Ia zy(fbn7xK>oc8i(W{}o~XdEEXJ7(P_45$Z)Wm@sehup#(vnBg$zuGDZrmLDRup9T5nzF#wlEfYrxXH^Af0S3f5=% z*4G~h+YydOezM1T+`F=c9OMTkh5EtMm_zmuHq7QS)Db_F^Ka0qzv<;qK>1rxK5Q5y zY&e0Je}eER{}`xkI&3)Iqk>~cd_SfXNf8h z&#)3?T9m;DJm0~$WF>49kYcn`1fHB{^LvMwO$pNicZQL9aBme{AVHu|Cu1n zDuJ1Sa}YX4Axt1JfgH1pV|GZr6BL}cWAmJ1!r;H8IruLr@mVA=mPBQOj#*5YaA3kY zW{$wTnW$vyn5PMYe`Dm}-x$PamcRscR_@a=`Gjc?OnZ*W5tx;om1G?=gD@R|>Bun? z1g3u%rI(J$CQKqQi5!zDFgLm=i8>~WFx`OZ#xeH^%*S1oSRFHzF!;ts4%T8MbF#qT z%tWY;u@MIU|H#4rKM2!HU~;-EembTfVeSRyUXDo=nA+}&hmPq+82k$)2mb;gKCuFW z_a&VAS;)HylMYNe$Ak*Zl^)6!9TQ2I!N3gW7(aoj=&3a57%O3h0W*waJOl>c2(HsH z-h{~nCX-{FcQaqzA;t7kzSl7_VX}b9;+QJ}v$2Hl~NsZfH1Rxnawe~1?Eg&MbR-;gqZ`(9FEy8 zFeCaY#X4pOVV(x&X^z<RC=2c)`<(O=NxiLWLr(>oNrWBY` zj>!_3`N>K*9W$OVn}FHGF+&CBMzV6Zju}aq&A@Et7@NS%NKqnn%pk(N4b0md(@$XP zQxvO?89*2n7?oqX3C#R~inorrhcNE|^A5+{Eig?36k*^YJCjr@(y5F|xp{la+sq^S^{S z0L%f7xzRx%zvi2iZ*)u(;^;qaIiGXP1%cUUR*vYHM#3Bh<}kc( zX_DC(mn$%be3d~u<}t!}0OP?iQv~K2KV^W9nNApQV7xhIyukSTEBEM_NrdqQ#+PG8 z3QV=XlAvS85GD|qK#mzCFdYJv7#(9LjNw0=0|aJcfD)o(QVA0dKKMT;jotSMOi-ZW zt7G~SCJLA+j!6)h@<7F1$8;r3dtlmgOpL(9Sd}J!J|F8qn2x}7U5YX{RCprn~L?((ep?r`Uld9UPe0pJ4Af~{lAIYy`AWvNGFQ+dng$cZM2vu+M%5&+M0zZ z+S8!{plCDne;{fXcB1fo3sLy@Aw0VxAEy7~ke}L#!p|(A1F$D&!Fvi3a$?fA{29|x zWDkKh;A-C2KT6-d#`(YY6c?tBLK^m8Z~K1KFvQW`pZLz)7lv{o z*uH<5+EFjZ4lF%q$pAfutGS$i1)-b>^)xFdQtg8@JZD*BJTukH-|;)d_SruIBQ-p?|{ReHC4=Fr;BkdE0l%f)Gb#i0=ZfHI&2kvZ@!+ zUa6cwU`emGpc`;CmvfZP9!98@x?V2O)_UDUoY3n!!dx$_I?qs^(2K@oD(@O_q}L_T z#m4fg={p~IUls9GUu4hL_4PC2jP*6iP>!gtkMweW0+#gp4)kGTIbYEC)g#n-h^PI5 ze}cBw>odd&z4jw4>T95(JfRnj$)r~`aHQ8>(4n`L$G?}5jd;?F?Af}$DiCL^uUJDl zqP~{t<-83n>GcL^cVjt+=-*`6dqq6ywHCCsUN0d|=v9KSs4sUzc|tE5lS!`^fg`=1 z2er18was6H$%Qhf;YCdx6kuLlq(^tvBmQD5|ZN>QHBi^gQqYaDQ-*C^0`<7zJN zB9&)VFT`@a$exWT&-mTHWW-S!>{$i&%;*{IJBG4Eoza*~WhDVmy7dMK3`yOA)goMdSgEPcb4ke!DNk4Z-KV1?`w#Q!QLHxAF>f)UJv-E+a2UQs+V_} z_@KPsK^Gaz>(HLn!z#p6xhFwe*TWBpYpw_E*--(vZM6+?>{%Un zi0^4e82E_#d_kAJf_OmoEucM(vf~^l`D+H^$xg3;wzkt6#EIu!WKWWv_EKM@JgBc? z$`fJ3oYks6kFXX^;V^Y6#mDjZ#T4I($3H{yoq7BMgz0<}=~&;LKl`Wqxws$L;0ke> z(f;_xF8(ntNmAuF$iJ>G0d*RrPDL44A^Yz1UP?trn{)0Roab&BC(!pA>g!xstt;*sq1>}ZHRl%4y z>`uR1DL2`)5v}vK$2U(OZ50IV}@vDNNa;hUdv`5{Jn?iJkA~I3E+w-({kX!+%T!uFFGu z9G>BeIF@&+p4Xq}rMO3D#Rc?E!DyCWC#YP5U$zjM8hO>2jCVnN#!81i+((HKOS9Fw#w1bv1#F!i2j?+fPL zI8Sxc5k4+UiMbu6D*FHVK2u0Tt?_S7p3EzvQ@L+^1iXOZH z+wZvTAAR_@vUAaBwB=D~)0vJNQ${+_#+7RGloQ*d@eJ#rbnMC~8SNZ>ToL}aPIZL- z#!I#(S?*5LQr)Ej$;)>04e+5gjr8|H+7r+L?_ce5E=mVq^d0CIi62Y<=O8T?=`k+X zZg&6WAZ-TW|J3h~4bpNbUEF^=NPCdd@eg{2|9+760Hurje-F~eQF@bWH=lcUV*i00 zg*XXuQ%xC_)4kIxrPpdRV{yfV7G|b_PQ#TJx&-&OOgo-s$?xq?iHXFMfIS777O7*D zQ9zfc>7z9FM}3oQ_EAi2r{QE)n5G zNjpop7j@jy<;giqU%>Tza?UWM50kW$gv~@a6aW7rY>mLWSLP%Zru@Z@@cU)(kB4!k z;$PhKuM3>(Dmdp^7LI>k6r3}!2tSW7&Jq=z+p;2@FvEh&!V_>!LHNA+#c+$fBK$>3 zs;a;hPh%$bbf}IBQE#k{pBha5sVx!r3kGZPxL+#npB=2(abG0vmkic&agX=lc>d>E zpR7GE?uQO%WxXKoM-Hz#gtB8$w(t#0T2Jz;D({}QJ)JEtW%;~RIVV;2$y-smGRx~t z#~hnHbRp^J9cxQ!h`|45WFPMY**np47krWG>+jN-RQ!9} zZo1)`inbIEyY#@?g?!Ue*;SF`>XdTxYsu1k)yd^MUrXMZfO-4C4oVREt(=!9-&p)| z)QyqkJAu&%N6%ke6TN3{ovh#o_LpRrYg`lb*cg(il$oWZvYl=A*y#DK{=JlrGT(O_ zjQliamwQONig%9KgK@Uco*Bz(9$2>H@@pH9Uw+^@?ef){l{NPdaa<0r`RC=Qwp_Zr zs4vbo#Z+G=AL6!3OfvHJ(|LCX?^>LJ*kv{yE60@vJo+)#U&d1!&KIcW2l^j&lY0B0 z-&&uD3r!FTv>%AdCUQp8;SS$L_U9i_W*ee+JDulhtU@sZvmcd?SuvZ!E zwGQ^$t=o(2FKk6?Eer0iU>@A$~ZQfPQrk_=cX2Q|SG%cuy$+`3rK)(zvi- zDe&IuA8xQ`+mfQuNAAbJ>FwypqvfCm{9AFHJAX1Y5PPBN2~!xF|g{fPDw0QsTD$ z68UI+5p}c#dcCcueU3D$!+Ok-qh)Ku)v+c=tsnkfDNEz(F^8tOG=%qI?&ub3yIB}G z_U1LSm)d8{@Tzj}jD^$>)rUw`WD6IlNFzIn@~Ga?@34ATQ6A0v$=B+wQkAHq5JMfk z22ATZ!nntL8oFnZFO%=#U8AIW%uz$&CzGJ}q=;n9#Uv?Pa#``dZl*B~eJwR2{##G$g+;Y9e)xktILdBN9){uf{C0IY}ZD`(J7 z2BDp}p`FY?J0UsDDN-QmU*}pK_zULez2N_4e@X?#9#X&+_;S%r*SOwD*Xu6}vg{CT zG)1@BL(qZNu-|13ulgtf?{B7dP(ooZ+uF%DrmT&+F#_pRtkDQZ&!2KUde5ZOvJ&da z?Uib_Hk^w~PpG#Rho{QHc|;c@&LRhSQ&{-B%On@v#69gz^anp$FZCtrhqEw5tv2l* zq9HiTLKMEyjVS!43sLBn2ulA1#(d+pe*%;L&_6kxi?YyHjzOO}8hz&|^r4xKrpGb& zILPN7tM0?QSepbs00K5Pa0u%YO~a?xj5&}ZeDFpkJl?>5N$DCSaS)Yo`Q<6eUg zHJE)86J(#m3C~(K44?H_f9i{TQ2*rH#J4YKD9Sw_WqTQ85W}4xXcV$cJv+St6AR>j=m#1yJ|zOq3_6LeMd0s zJMjG@^cVf|sPD)%^d0yP65{)_z9R>Hhbh(e@+s6sDQro7$1|+&h*Uqv{WAg5?qtYM zWefX}KKG)&=o_Q;&)L>C#z~I$m@lT+dMax%pQ5?#C&){4Tdety#+1<7Hc0J)xh182 zfV68!E63V45_UDGN`d4Hr*Q6xeEtmj&tK4&;Y>Arq@DUE@J*mIAVa+VKGzb&T?dc4 zTM|EShyNGs0)LW;*0xJ@IW_=?xwA@jEv*`?xiEg5IVdT0CuPTJl=mI_cH(oBt^LE* z6~KQ8-yq*BLYU>l+8q59&RHQJ@_d9m)V6NIuL<`Ia0A?=U0hG>gG1M2kcD(RwYYkL z(62?e2z4OSEkeyl-Z)(^+S?Dt+?;q0MHytPJfu@ukAPmm1wW>B_?p-^X^$>g^*|h* z8>X>~Y-8lnU*|Ci7~1o>AM~`5M>6fb1*=yhsVzpTLl`as??6VVgAk^3Xz4`Jo^9+t zTpdUh?K+v=w^Q$BG)V2wDBivt$JWCB;%cNclD~-O+D==RoyU~RBF3eGr=bAYI~Hy+APMAXs1*VZ;{4@ z!gp>ClmhGhY*pwR6C=@XM0@!V?dLk$5BV;>LmjZuQz|I*kjAw`yZLc)OS_51IQ*A( zBZuqlhQ`^@JQ{PU-JC-?Sc~Q@!MICp=M>rs`SuaAVUw#5`SA^ea}Ylla$iO`1mzpC z2N9QR4l0R$NJ@IjQ_>d3Nekw8k`|B-xt*m2#N(`i$0>wq?8?oSk}&Qbjfeggtc~Nx z_*4QJ%4%+>8;0RF@Jixa%IipwuOBQ;CJKl;46>Ka%@;N(U!CiacPKKGQKyv9<;K z^!eGUyAckM3dmPhB_KRYDxmqxxT`f<%Su0PtDVc^X zT+-wkw~n>@AaxdOCFW(1O(@Ii5^s=mLRn2>eEH>JzaG5oJ?re{JjhMHz0x4>O31qk z^1cIkPwMi%19?wk4YMC|y#skq>hivGclCmk`h4;o$a_+k_Z`UlF67+-dEbM)Cn0Yw zi)JNaNMj27-lxnz7iZ4@Z}T%(hV@wOYFVRk zEC3hz64q5Ukhez0g}wS3UtE5;{BbcEm-JC2cN^QUq5T+n$!E!?C4Wz*y_>YishQHU zQ<^TTmD`kfcTICVC`XnQ$N?qZ zOC)7RlqIv!G;@>fdE3Zk%W5-s%fTfQ%8-{FL|LCFn^ESpWp$Y`NRvRNJ(nhXgXbhU zyu`b5h@;I@HfYe6hNf|wrei``g;aohoD3UcwTh9Emow^jX zKbs5kaQr;lI}F)o=~?!J#cbIY2HKZ+cC#TL{+V0uT0auLQh6>#v4dKU;oAY7%vH1} zG8Fd-zo#gcfK{alxQ5|=SlfY+#ik9%{U7*NurKC=hhs(Z2BEKz45of&e(rVeXi|+ zvo)LLnVVuQ)1piz8Jps8HN;^a7cDu@VIFwO$CQtCtMmL$$-#U)W}t$$puHDvTGt19 z?S@|Uoib=Ys`nd^leL+nDav-FpD=}&lq1e^;%tqCG9<5@sQRKbO@1$9Q>~|z4}KS8 zJ4nhVU@LsE2QbQ96;J->z#c93uIljJ`_sMJD*LgQI~jJyK5if4SC4pUsLd%sehcK6 ziMM5HRNc-r=jm-}&UD15<9<5mREl3N{AOqq&t)QiljUNKDCesymxt^eV2p2ZcJ+bRN$Ij;x@tPu|F$CE<$^Hz>@uPz`vIo);7x8DmjqgV$%+SNk3NcSoK* zyuoGmX}Ey!QS1?!=-Cvm&a!1&A@Mo|~TOB{}~{b$9&M&&`2- zGiUu)lcR36oPWwkax$Op;jWzTE;*XQP=}Bu7kkpPyz$*rlsn58&p#2DhVUu0gL8Ua z=zf4c-%I%e`B0xbENT4|uUMNSQMRaw7$?xCgZ^u|_+}ZE`|}<0dt*PEWFZ+apWTtL zq@NP&Wy>dDB;SmKePS)sqv{}2J(r2v)6v*!U*)ji_g)^`FQexPOulp2H%1#O!(M1L z?T6SqDgP#&;S=osVkhN$+}BQ#cCq`7@yfTjhaT)7&`gf^IxF!`sVWiUSfVT?75CJ~ z(jMjw}A(?Z@FAAT92!KI?ix`Ztbq`FYFIfLKK@b zC~S-)Z0hLzu!%ND__H==_yYWI5j*uaK6-IaQ?J=ZO;6G_^wPA{uedg=6rk- zz8y8$=6qtBO?&teo3p?)((y=CX8u&~QI02;+MJ6euZE^xD@ym7u&3SY#nN+AWc=UP z2jP;^bJ?w=SWoMbx z7I{{co*OG``y}6n-?3IZ*XyOybE9OfFm8HO4&esK&c$ZGhTpKZJlAV=>A4{?z8^g! zY8ld|nEe|Xu;#tkYfY&w?%}An5SMBWXgH7kq)WY&(vv2s;R5DWwY>^UW8xl(`W)#} z+{~!6pk*ex;SbF1PxV?-dQvhqoHKiYlYbxORr&s1m6?x4 z?aZ9x5jy+ASW9K&UGHYT6tydJE$*LOTAlpVuTRb%|7*eQV^KXbcSZHioOs>F+>&{ExM$@Uls7!;{mi+;237V3_GMsA?!)uPMQ+Zl^Yp0f?M6K2 z&fXc-E3@!KV`gnhVM+{m)h;hg37R<0p_$EU?n|Q`x3ET@@#+}IP0TYN!THKQ=Ex&g zM@ib%DQVjM=HMe&@m|_h>_y#J6y&&pJ&`@8&?7gpaaLnOn)WYXZ%mLh*AdBiX?U8m zb^z+<7W&|fG-vgIoT%FC2&bkw%PHI_Ih)3&Y0n@JeXr>Eonek9e5e0o8Tas=+5u_K zLc(Rs&L(_2e1t(PrF>c>Ji$qo0*O6w)u50|tulRNI8FD*u%`XP9hX7a%O3#Coi z<2Xcd8TfDIHq52om7Ra=l%2nSC_8`KD?4xQleMb{W$n*@$lAFhvh)1cvh&(;*;)Uc z?8N`noPVB_onJ?Pn&~_(I~~!VWZpW9|A0WRs~E>Odsy1`cV(|q&0L|0|{MDNLbr=y?a!=;#;M17d~ z@u&Wdtu+CT5AfcH4gH`r$X_`H{x{5?YA2Jlu6`8e4UqjzQC4OBRxicv((2^;nO=%S zw9|0MIp|Oi9qK0l2R+Udjj60%8;NUj{u$^{3ms}3ZO&TgP=5g!=y0ZJbY*#hc8q>k6QRqIdnM(AE?hr(;V&FI?lj{$UojR`R1R) zxAyAI6z}J#b=jQd(Cu6U${K6a9Gm?e^;_)@$L0V>J?5enLeB@eo_}EfdjZaUoH5%g zXDpTSe=!fOe0Zshb;UTx!naM1+$c%=wP;x7BPO@}vqi%zXPVseYmeBRIRicNXWc6~ z7Y&4by{c32u9&m7ALwAv0ieUk&p|UG%a)0bSo$B=GULKX?f7&QRwi@&;E}2sGFyS%uo6&si7EWFefPtly-@Ez!vn|tnT~+ zlvDcu!|wr(-lxBYG4RBkuF4^kt?Hlcv5$py*SgJ~3Oj$b*M>;WhMltG;$F#teHrZ(`q8Ek(>lSQ_^oR% zDRZ%xy*M3v+1S%40gsjGY1*o#=~||rwd8AzchzZW&Of}ZC0`}nhdnQ;;3A$EN1MF! z0`B2+Cbw|g!XANZ5JzdRg)3Okyoa%G6)wzmE>(Nk7FJ_#Pi#(6I=}`r zFS8e=I}^>`NAk=*N7~?g!OAycl~?4Tk~h-RoO06Cs5aoe61)q+dm3zY1h%g8H0PfW zfp1vh$DL#czVpS`0rp^J2g;jp(5yXRcE|r5J+$m1Ppu*y?F6#IZUH9(C53Ew6Z@v3 zZh|mJuE%^eB=|lp^!~wG*egSDHZEM9guT9Ov_ z6vZ3$!Rqd_Fy&eBVd=5q3eB?wrovQQO3w_~oUWG6BDYh=0Za1W`Sa0@Sd$i;ZAom+ zs6{B`d-3wdQr1q|Co1FM|Lg9FRQ`$QGGh^*+dfiB1!h*@y3)hlvJ<8t{Gw@X>0;0l z&@cKvm@omj>ysmu2S7uYM=JY}*4Dhfl(nVQNTm$-+vV3wzYMe`;kn?^N&kpcvOyPZ zr^J1wt>SGvu z&<6Xtp>0^qua`F49p8Ahf|qCm5-?rnK`@Gl8#7VC82hI7vY%t}YUW;{030pfKjg6;g z6O!{Q$RyU_Qz09CYlqZkpmGH9*f&!C)ANRNaT(ZqO2-~lTKkQI`GC%yVUovipDV%=%9}w?77+ zkKM*I9=a9J>6}!anW42lk4d_-HIlw&TB5I+B&?Z?`;FE5x@k4!Dn}o|?y+tv!F@IM z_t-twP0Mjl?U~NLaK4=t3G1c=$l!%_lh~U~kZf9FkWK5HjOV?(MmqwonIG@A690S0 z-arhE2Zhziv4#7SB{xfE%%3^4V}Ht-ZNp{J`}8TWSq{#-6yhq!MPpq|kTfm}c`1#~ z5e!40Yx18RH9B-oRQooA^3yF(L`}!lA?UvRWuR~2>KHjV|8vl@xO(-++Q|Efs1nrW zT=3~VZcyc14@+jxKQZV28FPAE-A7}shq8ij`C+Xa2zfjacE@GLMSB5mxIA$A<2r&f zX0%SS;xgu44V(2IXLDXdIqSloj_UD%OT$>h#_ydT4K$^ZO>8FwT*AbuenflwSQ$qFhP^4cN@X=rP zY@LhnY%W7qTtyzo&^NL7kKo(dAo$8M%tOf!3Z=r*dr+6jSJLtaJ~Jpk9yU&Z-igq& zGxX`YV^n_S_fwy9!`l8OY38O%uh~)Ek~8yv-y7oiZLfqes9pZ8cO~p^N!lN&LHXJg zYku8&NxSxLK>no?tYbR+=U-ebX;-%T=3khL^K_#;DjTrAxwyqU|JO3F{4?t<`G2}R z^Xpc7Nj zK7>D`yuHu(ob?$Iu%)nR@m0^h?_wOzbh&m>eN&sG`X@gie;~gg{~$jhe+k6(7oWJ` z+Wj1OpxvyoU_52#*sw>1{sHIlaNZ7kRH?EhFR^EFcr@ZVV2{ckup-<`TAfAD9i^Tt z*m*O`FBSN(_RGRg@$j_?pWvB7%Q;N+XKQv%9*#3ML2CJApQ>}9IMbb1j&nx6P@i+_ z=v&EF)e~(Re2!Y)Fex;jLLXKs$ucezE;BAST<*9$aCzdg;PR@ZytiOS2g+bNUFxFD zLjU+Q<_2E-_wD&7<{7MCaF!K+eAKlb&;0ucKho!UJER-<7YokM(t85=;6=~0h~H53 zpV7!yfiTWP<>kO{gYew^1KdN0g3r8d3u*sHoO`@B;UIe^@)!P6oHEs$^EYC%f=xk9 zkAH;cE;63KnD89N4BO$^sP6O2#!Ksp6R`IskEkk!J`(2Yy%0yway}j{vAH{R-eK9_ zPVu@cO^bd@a(w40?YM5bhvooUbgZm}1{P+aW4xGjx)06g@$JC`;aZ}<6RMi%W&qjEL>hd z^ltfQh6+^e3(`rsR^73h3f3i5x8H3P|hFTy7d zWFXv5?T7F<^z($F@I&w~xo}BVWf1AfpjcC2m5Sma!n(ua^b&Z>u>Q~a5vsVXovO0iOmC)5Pa+L`hf^`hJD`)(v4fl zbJ8SLoiw>sxfb@`L$;^qb_0r8w`>Y^-OinZnD-U|Hr&|+`7YH_KDRt>+9 z&Uh&@+V)KFBKzl*7N+F3_r^KS!j#3EJha?8tm7#>Ue*>*!ZjJ!G+d9!_;y9R%EcyW z!Qx+~s@!QjPi|cohOhPjZwcQZg>Iw4dvT3B%I~JPZIn@mF!<)S@6P%E;f{COm!{s_1HOtpx-csivgaHKW#K3m zZqLFI2!rR*xtsaBKZI>Z=_YTMPJQ~7QG?K?Gtjov(ZDf>)P(%O$aFjwPe-$mcW z`cORcJ%sUFJO@5{4CgHHytf>E5R)C}k!BZJ&AWI9sF12}*pdw60_+_4pVBD*e@Y`c_RyL$t~ZlohCz<0x*Ts{z0g{YpV5b* z?@oF^$H(CNiTCUHT8w$=&@KIoE*QJ9_l>^Sc2r(RHvez3Y=#`^y?Zz3>4iLKk9}Bt za!Y(y#M3y<%IHDwxbiYOAs%J(_za7aDRi!0{#P(QH#BJI-k8r54VSG{wRy` z&H`4nYnPv-{W6`(303J`KpNLyMOk$I`8v+w5k3d_Dy(1VyzN5>)A>PR&oJ1NFiFTm zdSvTzj0Zh~3wCk9&KG0PP<@A~*8=g5Im$ps(##`V=Mk^-7>0aw2Jt@7<3=9a`+!Hd z`U`l_nZbTYBc6R24OM$F+D`4sXpq{SQT*$YQLEaS(J-8Yv7_#WpbiJ4F7HE~4nkj@ zf%l6{ioX?S2F+C0Cd8YO?gpGY@Hj8H2=#qX#`#`J#XjrN@=0mfdzUm5_C3;bVwH=q zHQ68>*0Es6*t2;c;g01=k%jF1VzBE|(iStOK_g_U_QrjSXSbgR8 zRZ5VKv$r+v3>aQlVd^L7OG($0z))TN0GbPVk)P`OXe>MH8LlowJk`|~jE1U*8O8Vi z8EvP2&SuFpM;G(wq7HmcQsoa~yhsB4p z_Dk)L{9L@BOKq|MaRZX-&KtowSG8KJ$>@xZmM5sC$vgep_77M=iFk~wM{uJb+c1F7QKpf49 zCxb3)P77ga35e?lndseK_|y)uw+@@`knnAbCHmgFKX6npwC*BG@(f~pL)COf+o?82 zgVlkI2C2!6TGjT9(pk3ueD0U#7sU4(#z8Cu{zXb4os4F<5!FhF7g08mgLpgL_;X1Ex zaJHS=JJoYA;>eF^9p!~EwSDKJH2C&`ipBtbPPw5oJ!3NE-|3t(y}wQJIUxh)C_AL` z?#=I8lOJMVq>6a^v9@hhpG6s7k>!~bHwR%0!ruL7MaAHG@@VkDc-vdLt0HqG!Zi1> z$L2&~kA58Ejd?uw9of6w*k5*iT}l{MF-n1)4YmgSq*=U4w5dN!W>HYNHFXFt?GQ;R)@i#+R>Z({pYx+ zK6gLpC~ijw^VuF9;IqN%RK(L5w3E?x>U*Hy&@(Mll7cynHVwXBxH$^zp2C_cq?P?( z)(UGfw6Yqr_BQUZ_R6C1$cx@V#eR7VwVQaBZ%=);C z{giZ8rd6G&>y@MNcV(EaERE{<_Vdzs*9CjUY58HK1m*D>mIqkBji z+dGl4XCh(m1n;slztsD-pTBagm&TiWe-B$EogS#fcfxrZyhG3#@8n`#dI@_yHwWRG zLGSj_9&+n8dnKg#TikKbMS`6+hD$+%NU!PmcMJIDpigY?R?hlFsz0hHW87TC)#gd~ z&LH)j#SAwaapbc+ZV8K{IY}+{;EZu%?#kkDrkeWQDTLdqM{Xb*gG>Thvdt5Tb&AUC0%}LhY9(NDnR^J{+a}w(J zjb+3W4*QM9xH!a7zhR7vBHT<#yZ*SO^&4Q`RiB4vDEFFoeT%DpJ)Yx$K271Rcy0z7 zN#PPamrDX&j&!0$__h?$#Wu}CbP>|~gW}9(G10Y1C%PKxpf~#=j5C`RpzCnu;HrJx zrv2oPv&LwX)Hhv+9MlfKLmc%}7SxxQMN%I1mXvJ+S7+tDRXJz-t|eJrlHZ?`|MIdd zU+MX*cVB-!%is2V7M;VDpKTiv`;7lslPnKU?;jl)>z3{l`;2L9Y#VvFEXzLo?~)Tu z!IqM#3)|0^pY;PaaBQqa_CeZkISO=s`}r&Tv>CSYz7fM#2DcfO*T-jA-hIP|Hoj|(IBL7(OEfQ_exH1( zEy5@6Ht#CO^|Y^~6=I$8^kC2-pl^es4|p1TQiU^R4c|A`3h_^}mPyL|4 z|4#8}JE4dlXox>c@n}ndi0^NRKSl9qTWt`3k0Jg?ipPA&1Myu9@!wKB+Mk5@j)wTJ z89qq82^wRF|0l)Iv}spB!wvBVanI`X6zbGHy|Q+>f2HJ}0U7{m1N8+>1NEzPb(Yy& z(jxa{XWwf*+g3_m)hV@e3sY1lpC6wDKUQa9=3P{;7a$+acfLow4RQT3W>B5ip?+!K z_&e&m;5W3+q5I?dy~&@C1CQgry_x&gH}HWv{7a7T<^aCuypQ7W428wfKOVs2BgE1D zwG2GNfnUPHDyzLB#WPc_`m7#G{?RQpCbG+#IG0e@F6qa&v+AYFC*?d zP$oBgh~huP{Yrs_|4`f+#4Q(bXm7;R4bN(qinu8}t^?v|en#~m(LB8~+THZRlx8)M!jv0$UR7uIQ0PBXR6g0ay(hO1mEYPv^C+&BeQ5s2WQSdD7;N)kb9{?k zCg^e3hmgHOn7zm@V+0m$LfB=5h(mi2HW?z~&=!PE(nXxXCMkN{HP}Sd7ulsB#V;RJ zc?$M8gmUX)6S6}*g0`zKJ#~Y$G+}!9(|# zTWFiSEbu@a`O7sCr`wC{a!JIA`a|4r6!(8=tEuESKHN5cB42CUoTIo_eshN6TG{3# zmD$QRG+$?NXk(%s{awVtk0_38Q!V0P7h#jV&2d4^Hu*r0yEaqz4c#s~1lC}eZ6Z!@ zTZ3R1n$!Q^+9d*Zp?}7a{B{awa2xGL|HalAZ_+pZtmZG1MXIH^uf~0CXW|v2zKZ+n{c!e@;<{5@xT+xTL%Q!p_d)6^++P_WY40PBje)4! zV#K|Jdz#n3amIZV`Ms&FCDV-i+-RKR0rm1;mgNCD%j8w!4(cr}W6y;>5w_x@XT{}AnxokXXf9KPGSESoSFaD!=EF5(f?uZ&Eu;ovc2zph9oCpCUZcN zfJzdam<@tCNtn^tMzj@M8Jq$%BGTFp4GD7~YI6{z1(g8O=4?e#gBn`}q_sh>wt(W) zN&v-#pfW=RFwb|@J_pK(_I>ZY&-1>%4}a`W{r1{5*Q!;kYOkuo-(&cDjBhKZA20Y* z2;YyN`=Q}i$1#jFhmq!V!rVZ*!$|k0>BkDLCoRhfH(uh%o^`z7HE6?#mpCqzIJV9d ze@=Wyz>7mTY1XlVVfYz_pTe2-1+Nl@Z#v9;Bg|@W*k5P-WwPID@Dyv`(jyH+U9)x- zw4ZgLVEN3Q1-oYMEto#@tAc;b{H7pnW_3ZonP0Ql>g$5()4##}s^F&Sy9(0Tdo_~1 zQ?lnv_FzS`2g~P*E-Om4RCLS(h0-q4e$p%p!|-G_Z6^GXO&w-ahnG-)f1<3k$%T{QkJnQ#EjV#SNgvvQI=occ zWPCB-8o=4qa|V8+SEOB-d6s;Wdc3s2w<2x4kN0f+X5n|R#%bfT`4(=gA*A#B6)i7( zkgzNH{s3Vfz#Y4y<@np7y$_alyn^_*^Pb*TzNL*H3;lcC@AdbV<8L5-$s=trei&}y zOa-s(E_i!-4g7ShAa8mtb=yEaA1_c>po>md1xd5&3s%ExTW5Y>aAam(LFbv<;mz*~ z=Fi*%KYn|%4by4E?X+9VS^IU{y*+bR!A-Q~3fiuky>jhmvR95a8$~;cJ5{&g-U8YG zcKSA)|3BJ>^;`M}zkRV^@VicD+VTHvJ5Dd$NWc7_>WgRk;Qz@!_`sRA`xVR2GyU$@ zI`&O344#phzf!&BfirFQE0&pO`rWTrX3jn%Gk>)*^Gw_QTKnDn!ZU66Yi+yfg|j_p z=H|c7^|1%ewB4^*&pgxbe!Vg?+BzdMf2}g}Oxyih``!G)Gi~?lY`cZwNPPC6`yQWiiI=6+s?J(>vMnT&`II%g{f$c)r_TqOTzell~+JP-v*e2zg za2z2!4Y5hNGE6gUFD^S7-mjFl;~{QrJ;Vm1pL`41rX2WL8=g4(Z`A)(?7fbYU&-HZ z3Fp|RU=I?ivpk{cI)fIfvp(9N)ETr;on0(x>M}a^Us6{cPSO(X!=b!Shkn*cx?|7t zB($V3&9K>t4bv_;RX@$cwA&n-)K7GnCT&O^oDI!;l6IXqqJX&VpQO8k@j@^5t6k8WLa$y-{MdKx;N2(h9^SF-`jU6=$#9-v zf`r31<`d{Y(0`nCyunzZ!&{r7A3aHr2rB9`QNB@@clow2^i9rAtHvHDNxsnzf9Kn7 zzEwv-hj-WVzKeH^ifplOe2w=Vyo;TT3p)t;*Sicm39+$|JcxZoG;w)wW2~9lPG$^c z5ZjBuZ}zNj3&#Ez z@^=5M-PnKZDe%pxD`+5&y8l8P!-!)A`S_B2yiOe006o8bU&GLK*q?~4NSd6jj7^Vg z=HY@oY)Hh`C!O~lGxrsEW*x*%WoJPi?;U1+RWNL|bPx5|qY)PNm#hg!*%l>6}DF!>`|E|5z z85@QFy*3IXIJ>Qu@#=rF&9sfzZwCg&PU!oq&e#b3@3s*d!Fh$XQU9}Tc&6X|Z|--U ztP`2+;XETJ#fI|#`}UM0F0@X}<~+fi1#W%+=2vVs3* z;57zbZ{U9z_+114)4*E{{IP*QH}EzC?=u=iq)D_8S@+}Bwak(*sie;6DwXHi+Sftwk)m4V|8oNVB;44iJ@a}1nm;PVaK z*}&ZmoNeGf2F^8bKLZak@Z|=+(!f_6_&NjMVBnDkzQw?!4Lr`k6AXNpf$uf&Bm++| z@C*abHt>T64j6cWfgd&S;|5-8;1vcA8hE9FpEvMo16LXNbpx+6@LLAnXyEq@yve{H z8TeBJe_`P52L9T>wFdsy!21k*$iT|LCr;s$eS!6{IhFmTYoD-Ha-fma*2%D}H1c%6aYGVn$Nzh~f02L8yvpBnfJ18+C**9NXN@V5ru zXW&ByRt7$C3ZLu??A<*xE`)T#>rTmvvKtIM(!jSEc(j4X8F+$$?=tYc2A*W#DF&Wl z;MoR#(7*u$FEH?<27cVYOAWljz(E7AH1P8VUTxqi1HW$Ibq0RRz#9$xo`E+R_#*>< zYTz#nyxqWG8@SfM-x_$Ife#s28TiC0e6laJIWsPt85fuvo*5U^9`?PT85hos3uR}< z1+}NcnQ`IFxNv4%&}*t4&x{MF`obw4^2~f8G*D?y*OB-KeP&!ZGcKH2N5Te!y&Y%Pk-~k!kc~GOc%*@E zG4N;uk2CND1K(xfdks9vz*7u7!@#o*{GfpY23}y`M-BYAftMP1g@JLpQ#`z#|QOi-AWQc$|SJ82By&-)rDW2A*Qz83vwh;0FyHFz^BcKWgB|4ZPIA zD-0Yo@R|P$Duz3jhlgyDns`sii6%Y+H#Kle1IHOS$-r$5oMzy&4V+=%^9+2Efx8*_ zVgvU!u;0La4Ls1mml^ms1|DkQYYqH61CKE9%?2(q@b3+Lhk^fK;Cl@GCj*xlc)Ed0 z4g7$C=Nfpvf&XIQ#|&I<;3o}SVc=&Byvo3p27blBuNio)f&Xsc4F>)X1Ak!P4-Nc@ zf&XRTY6E{|;2Hz(G4NgkA2je$10O$y!~dLPT&L_)n?#(F5lx~E+|0nO3>>yn$C6xXQq<8+e_8 z-!kw<1HWhBO$PqRz@HlU3j=RA@Ye>eHSo6v-e=%L23DuA??i5PDzbkdcG&t1Z)~-@ zvX`{l)54as-n()y)CTS^4Busz2DYk`0_)g^EO#n?ao8pIG8c+p*~iuCbpBsU4x~pp z{x^mEAD%cQC}Fs_D=r*9*}(O(x10E?KOYd3`{1jYyBs5|#opos9tk7&tjW9J!|bE= zg~H3eVR_eilKU;V_u?S?ZR-dxd&+BZC4JdHyGzsJ?61U~$Ua~Fo?=&v%f8(&^?S1Y zFLBxT`PFfE$<(ocWa+mG*0e@EY+9=^l2g1z|Gj3!mp(q8^k zccsZaX>zx%+`AR_tNj@Fsd4w26OMb(qPYhx&6Ts&viv`}3yeFoCb(KoZkN_{M7z_* zX{1Z|lj%0T>-XVR4GaC;J4Ze~u&OUOdH0&!E#R|Cs^W4=3J>vZwYzh`cXhYLRx|jf znWg?n9TmAts)Uvruv8j+(D;5|ql`a_TjGr--rdAInEPFPnVE~ZDVO_!ox1tCJ9*{a zF6vg_pU55MJq}scYT?K6@ZOs*HY^mppqW)UqM6mL7x|Mi)JVBp9RmG$m%FSaycOur zy&v(m?3EvL6n=yr-n)N+f(PK!9L)o9c9$bAeWy%c-o; ze! z-Rq18mxtmJKE8wWoHEDTC7K>*--`P;+F%6tgZ^BflyVQppT3Jk;;+sOJjXk{xytu# zzAECL;MqU)-IBRr=sWhKfz7u4`u^SZAMMZD-LOA%Qq1Da8H3&#Gw1Qm_HUN&e<$bT z{qH<_RMWGdb0^{^9nZdemF0?=)8yVUN9(w|w%8R{M?V@mm^;)4yP`FHY(vB9F#pdZ z{cyR3KgpvS32&z#y*O~_pwwOLX?ZT|kSk7jE!rxX7kzKmQQf~rI{h@0yhPJKqbK!K z;6U`G2z_SA#iS)Xy%sqUmLuoW=1%*^+2`~98v6Vz^rNJQI|Y0XyEa8<=Bi9reBI%$ zvlk^~x>O?fF^BVh6aIZ3zwYBM_%htbr7gpLhT$jP=&xT=rf`4t9SPkD>iK`rPkrqI z&*^$`WWu+UB`hCAK8Ep;Q}n{$hv-tD&%h&o+U5ID{WTtAEODHqWf-RdK&kwH{{Fx?mv|q2ap#| zdzJK&K3^Q;uPcS8q%W=4=_J@MhVuCq@6ylK;_iYbc@}?{kWZ(b@vFlm+Rx)YOkMne zvB^SqiB8rv0-oj0_NmAnA9>yGS=*-_^x{13A_b2M(VIgn`tDEj(!Ab+pgXQi?ups$ z@!9AAE7M$4pBc=3dP^G`h882kO2}(*R;uQG1AD6gAxdni9+=lv1fOY918 z$95rdrSc$cwhZ@>$4lEe_x&ED4Ym9U-RU~|`G$ohUKwLz?RuS$IQyt}W9_}9zm4{- z35~aVc=uTe@2-JYoiRAh{svmH$Ew=^4@iIBsr|Ikexhk}ofqiUp*X+PuH3`@WBs81 zwp{Mvep|PlggJuz`G`KI^Lk5vwc$iV5p>3S{C3*`4hR`nYS-uU3<;$e(`_ zztnpg<(B-s9HMW8?(kTgoktru<94hqGE%~=#XC;eR7->3U-oc(*=rTtT*Mq zjne)S-%a)S>P8b*@+SIMFIR#;pOh_UijB-IFXq}=uY&j zpvm3EYjuCOg2~9@3U~6mjQ6cmUD0*E&a)TE*;SHmH}WNI-JdYd{b+Ia8tPu!{Rnr; ze;$)ed&k;YA%DHY=|NkKznUoSDvd}k8|6tUI|g5PU6$sL?aWJe!XrnT_|@0YuOx2{ zFD2RSi9_bCX}HRrT$baBugjyZ>gyZ2*Q48QPxV)|N5{%L-VQymjs0aP>`{-O`;DzQ zDZ9{H>Ufgu!~0$HpKHWR(zdbmkyw{iDf3Y&LzMO}I+4Rq-|ctBiEQ!l{h!s&8FpU$ z$QZu%hX(0qt9*y@74wkiT^?slW1g@ob6r9pC)!du=t(*0E;Ty}n(LxKjJ3G1&J%4n zy{M$HIM-6evw1f0B)imDS9IWkR@S~;#vpwsF6TU@=DPMxkvno}AED{Ewu1IiOYn|WD_@NfsEn{(|P`L>bz zk+%MacH`~0aHS5b@>5=kZ9)HNZdHE7Q`Ow+hTQwW8rnSYhc6ozN_*)(GNo4_I^7DS zdaVf=&zCMrK(>fp@FMYCgU)<}bHk2%lCw%%SOMROye-3{tO<$FKekBxJQ?l_T}l>3 zqmS>Velk_@mft~U9iKVMXXzAJMJ884Q>i^DzEgv)6sX;X`H%8nm!2l2DlwT`ojf7;;s3LFzn*UnEgo8;-4_~XAI`6VpKB5=>Gg#;zG?n` zut$J(rtH*oS9WTyJG*gSHgLD|HsnjgdN=(qf_jRizM`l%AN3ba|BDHpnC1=~rT@to zbAzz0I98w!&)-eH6aFkLMcbJ$d4Jkm=pl z?+eX08C&0>o=|-ttTl9Su#)hhScGD>hMp+(Tk6I;@H`_mb99|AI_MQY{!7V;#(1UImX$H$G^9NrR3R>A+h#$>S70Bc9LfquSBo;2L4Q9 z>`7xCX!{jN&e7f!s~`Dly~y-t)Nga@xdrvzl6ps;-`^UZ^1@S5@K7YY6ah~;vi*o5+Y2Qv^8Op` z#@ocH>t{<7tFn@30ncINb8vZH@ad;>gU>J?AM>PS9UJcrDr9y&Jfp@&1ka}~zU>+r ztm_mN96Tvy@$T`ypc)$;+>MO%i~QvNiX~G zdmH5YVlCejkncsF1TDwgL~qry{xhE=>(3_tqR&hvZkaFaVC_rJuMD44ngp%4=FueD zOJr=Kma%T=;!oBmcOr*H#-1Q=Ma9XBYwFy=_ z{Q5KUR&=)!THYqvkA%W}PZ$aNE$$^cU-9;CO=JB}yRr5z?Mk02q;0RHjekR1UqPD} z(5Eh^PdW0|)P+r%(eK%Yg=^7^Zh;py9lfR^G!G5y2ci=;mcht^s@aq!6WN*3%va`H zH{i74beuN~sY1u?UV+T@<#v`e+o~+zLxnL9t-LeS6*D-dv;UQ{l@Am)c`|tCU{{mD z(6VCA@@EhJ$hy*(?$7q!o6;J6Xo5ewZ!as;e_u(Y)|Uz&CjE(bX9bw+X5WoYA@b^J z>f;a3`U0hHzhZumT=3bCLo1GCEeDzpS*T?&x>pXm*LGwUy6a|}ybWfI7(U4g4nNDK z=0x$1-qi%XYmXD-YHqyTvuL0@uPNuw({%mx(o<(kdb`5gB zXqZ2{LfmWodK@U)T=0qyxl&1)N|@)A)1EGPO7IfqJl9DbO|~`_yRCbR3tjgVZ*Kld z$p!t?k_#eKA>T^6fV)JfD+Kod_lQuJ3GM~%6`=-!i*C)-XSrFfl0HlNN2t}Lwc34N zpnR%lW%*PuPsGad9wmKB4);|{4trH^311Jc_o{5cKY)Mmsvd&f#OsbwT?I#iBO}yB zf@8oj5$b%w&B4thR2EpykSm!pwNJ@IJoBdZDR)~d%iS&>_sa5=l0L=rn!mCnD?$~Y zYh@Srx3U|rY5iPZn7WcPd8xN$$lq{ZD5cLbcbqHpMXATjAfrrr&3En-NM=<`rAed!?^f{qmJX`twLrw|)Drfe}0Us^T49Rr9@T z{zUj$#tXp<=nJdeS%LC%bJd!2U8)+|n)|G5`M#fgHtxSN&PyMzXznYsW(?q8POM#m zzlufuRPfdQ@Ts9EDUW)D__yAd75Ml*Yg5Cb2({=7r@#G^bZSVmTGFJiy~#Us*leNE zFLgP>^PDu9=QwkbxiZhmpijWZO7y7n&<{jslJVwrx>kwXU&a5BRp~`(fs4`IOVQPI z-qF>D!P^qH4RKJW&0!rYhrXOc8qA^Q{&(h3T5geF$!8q##gS*J*IAVH=knXIUe-)K z$YU>ZnRTN3BN=mDdc9BdSCa?6dWO5n;XALv124jB3#CnQYt9{@idY|%GX9Y=n!M5E zG`umAe1&;KaJcO@*v{GwGDuw;YNxAdqiW*c@tt1pU*(Hn-oV^P<}cQ`ekVA$`Gn6x zZcoS#Y^44(bDIZhI=X^~u5t%=cw`vi;`}Z7H;a(;^8QXU_@I9)nh$&$;WJq<<&b(?j$} zpd(ibk8`g1zgYJ=j_&@FtbbvX@e1>S+I+tnjq8`WK}QMO#-2pjmzX!DS_yUe@T1Hb z?tykt)8gvnf2qt(CPGWK;_5ocdr!X_A9`Odx?m^28mr$E>P9(hM1|EI{px`HtI&D% zt6Rk-uQzMAtvyQI9KRZ&-(`*G^m<`!hs(y*bs91*tRG65q&&6g9a7fk(BXsI^Sb5CU+o|IZ5iAurtEUoUY?nhxOj}K`MXQ-x1rt% zGsYF8-v`Qj$nOyRj+qp{csS!sscIPdZ3}eh=iI?H4LySmLoD^A$Gu5p%Xaec{nMVH z+=p=3(-fV&sa=b%Ds6p{lo9@WvV-Gy$1M6ytfh8Rmg_DWtGIY$3WWg_(q@oDS4Vso@ zZxkAO+F#Jq)`Jhz7bLu3(b*nBXL}u7@zfuKvs*`~(ggYE+r}1MwaY)G_*k_|rP!xe@A7Zyg>KiRwi;_RrG{@Co&nE2D0v#K{;!e=LR#GduOob z)xE1#m6Kyt;qQfve9zINtOq|vz11Z8RqqboU`B_CV9%TQKF#WqF<$#y6?0d=6IuC@ z$C=Z7kV)NU1e(8AbgBR0QuN3S!e*RfWoPuWvg>IJ$#a>M51n}tI`a(h0r*PzRj}yK z52HU9gQYG?goYmd2zvBk`qjPSzoX+{u;|qfqgPJ^*9`Hi(o3B>ItU+>O?<Qu3RfD*KhLdvZ24e+AtvTgoc(<}&goWf?3kWgRFk zJk(!Yc&x9ul&OX~_(YF8)~173xX^p}*E!7!WLWOHDa^fYVoj=;^_dLZBFkN->wDdh zszI#%VT-Uz=3MemXYf)h!2D@n3I2veh^(>dMzW?mBwDbm&IP^A)e)}IoWtqAjB_#9 z`Lmht#>8wYcx5;;<^Xd)bg=AhE?-Q^asCy$V`F|azwlX=jpv23F2%f2)-pfvg~sPA zY~~e_R=4rQq3P%b75t}?G2A^nPmOX}tLNi?{?u*(^dXfEf0fqftCD&Tb>_1_pd3?q zcQd9*nDAU=JhF^%d*|oH1xnsPF1=x?x5O`TpyPe8nmIxRbA;&2`>Af|`f-;>DC?YD zWOAH6kg#d+F!QzSKQK3sp`Fb((DJcq`#`$Sv8}n=smq_(#EBiv&;5&){}Lhn`ea=+ zr7lEQIgh%)KBw?dQ>|NnaD@5P0rFjQE^{h3`U(1n)zlgqF7H}&@2=#znmJbkbFMP- zBmRY#1f!pSaD?(6ro3y>L$@&pnhKr&bM%{7dp>-(f%()%V?G7H@6BL7g*^Kp+mjW@ zS&V!UK8)`WSVA7+T=N&D4e96D)?M~^;ls#7t-HvW(r69b$NVj$#i{vQOVWGEWA%`< z&mmlDoV9NXYbD`2P6+jf=9Eds&k^KR*2;T@-haottdIQ`_iG;Jh(YFDbJH(6o3S&& zK2j(BD9IkgH~CLH0GGMJ$u_izpD;>?xw%824*%uM+hqOo-AMi=GiTHJWzP03X-Zp2 zUb_>ItaCc=Kec6(f7P_PUK>9CsWYZ_Cd^^xGaG1+wV`#M4$!3Tg*NbbgXY`t+>?2< zo)bv@n{&?|O7v|9cZY@-c6eG!mFMif!WGNlVst#V$}} zUq+xwmFVi4=K^jO!XH=*9Qu#^KI+i^-M<3S-4CL>J2oUP^mxnGJdnxy7j0D2@yD{F zZxA`t%IcO#K2!O>Hn5d7bP;(QxN}T)O-rjMdHW#g@g`LR`Cbb@WCWtCdLT=LC;mpA z3opG#TQ8=(_b-nN6nW4;;RBH)Z)l!Luybke5g%LV+X?n-A=)#cu;NGfunKn@G|8{{ z>qZ#)$07I`9VXGXaWg#eWj^dVidt~b5c5|pH~w;VU>x75uYEPh&?fMcp7$<`!@kk4 zw!tHd`A$0*Zrf@Fw>7u)|6Y05a%&fI>+ZG}mlx)$!}OzyTl=ZJ}X4uE%e)=-|o<-gy>%h{Z8n2I`sRYOS~D>oy1eacqrj2#Km6`uFlJT^0MFB z6n9sIioG{NRq!rrUb5D<2L4{tnkTRM(drIXK=LMTI=TQhh=mXEPG6dvH{#+zd6!Os z?R*m(-FeVuoDjd0z_I)X+8}a=u`FcQIM+_qvuqXz`IsSLV&tkmAAKPz(qI-+ZVqhSeyAtU6r02kTQh-9lc8) zU!!jeP5Mh>t4`Uk@IC>0gU7R}wnGH-??~3=qnOwGnBPY;&*$IX{Y|g~a|d*}(|QG7 z!@soCG1}(-oml~0PTF5|UTY)&N}81PIl{Li%g#k6x3SNWvh4O(G4G#H3*XOSO_*|Z zAI$pj_B=0s5^ZU(#(w(khO|H8-H}l#Fxxve>%M0$I|DmI`$~?zv&3em~qtmTS$&-ilUWCWDdyv`G zM;>LWj$PZw`ebCm_9nRh8hN|#lRGAFh2Kl>gty_H(e(9F@}6oX)xC+Vt|0&23A-lG zKVcAaVd00c-y03TqxiNu(rUHgho~O0Mw;iGCe3Kll(zBh8=zCR_u zI^0KjDg(RvABRgOT$1)*!aDQS_?LgDZ6Z>d_&lyIIoLJCdrDJETW9I@8sDHx%A(h; zc}3bH#}z+z#G!_v0}n;@jHaG)Tzw}x#@TG3aqXre`2dcoYAgM*^+iUs@<{7MMU2=>S%Hce=EJ{xV)N`1EB9~w z`qTXzw;kUfJt=x|X7t8~V>Xm-9`l#-{c|38d;gs1>is#gPV+^>&>&?F&ud(+o1Ty| zwLg)6a?eZ$(#-SS)GjY+WIGGj!rd{=8b{x8>gC!?0^=#e=gj@q5|>q-6>!l`)cfWW z_YXnV4Mye-LiTY_)@=j$H`zZ}+o9h$WTV#WL`MFSeDv*|T{zHPerLt{lvhU34%Rch z0~1JZ1mim<3^e)Al7>Dsg}M+uX&N+{pGe=4_hQ~fzn8J96y2vi^K;R6WbE`{LkJwP1fYu%Ru}(9%QlDy~;Qw?jPYtpb>G@P#<9m| zKGJypx&?lj$apq_@k-j1ezI!yjx7DZr4n67<_On8m+!yFEk}2f@%bjO=tC7<7&q9@ zvyO3t|75Esw(b!W`8>y;ge}Hj`J8@g&7268EO9&pekejEfOY;xUf}3hcTMe+ z8XX~e&E9V4x$>_hyasUix9clH`uQqfs60!NhoYnV;!9H`jOfPVr(v3#Icfy_9tqz^ z!T&zSdhB^vhYZ`!NxBKl%M+SdBKLKi6-}%j8h_3D-ifS07WIEMeY!jOh~9VkD>Q+{z=-7)QW1EP-1)=rN zZG01bTkgfJ5j{a<>e9G?8(S$KYb}|BdX;r>iS2ZfE&R9!KVl2EELM0j=_F5XgsyGP zT9ovu*Sv|#TI`PXVSBSA`y$PQN%m``8M1{~6}COwX4syYwh&i^!aPG5nUnYjIR9ZU zhc0;d0FPQBD2i%dRnl@)YRKj|Yn2J-nnr15u;rS$iEG6I+I zJ)iGVC*k#v3rUxG#j35W9js+tU>oZKGw`<+eN+6oSLEsbyOX}vH0*cY1%VCoyqjbl z#67wnV-x#|__kKs2{~Cw_zldHQ(Y}5$bW$_{dwqSKh*8|T4rF1$LWU`@Lm3SBCA(r zqPM{-RfAo=CWFz(>X4CR(cPM&#|=LJnZ;LKX=QfuScSvScV#k0w7PndD>!_TJD3q+ zRfYSX^s}kNDgA67ZW`?@YjMIam+@}#e!9J!@Vb`a<{FpC(>%gxUWUi%NBc&!9#~&O z`6QjI>31Uka_}QEQfOVl&Ob26hW4x){a;3iDL|V8azom= z8+F-@{{)Od`d`@gSgV_Ef8^0@@KMSn_UF1@M24KzNz0I{kRe@vL58IBFRjS!>QckH zxzxp$Z%)aPXFhb~$cI8}DYovc8Bx#IMLk!mbUy4zB^-CydHm~Y(JJImxN?V zf|e!E&gU*2^r8MJ(`FgFQ*b*(Nm{ao=CNDiPjn?Klum2jCBA03 zBD3_m3g1gf$B_&8)nO9sA9lIsf5trbMVaeLe{dxPqLE+Bp#q1nSC{>e)(qA#(O3UM zTv{Gr_mkSsqrNAs9&60;W1KU7+=bsT-<(Za;ql`p{u77k>Ck_UZ)A<BI}-;6uLZ`Z+Z*TQRK*^hViXjgE^4ep?fL5;_roun!Jb^urEdpmPH zAL0G9d*iW1(oTc7(pdk~^-7(|JWynMf9j@j-+YWXOTvBg`8?hC&cj~@@rUKHgq8L; z)4BrL#yqW3+L&}!SwE9zrjzDeQyvtbN^_Mvlx7S3r5S04^vSsvYXDkrT>s<$X5q1h zwC9HY?esVZzn(?=ZC&25Fgy+nrRG}9?^g%jn`A5GF_xT@(Z>-BpKE;M@)1iii6Ma^t z=;LAAxoY?^Y&&-`bweA?ozMT11^hpGQEbTQtKFf8ZP+dZFA@Eoe<&4CM5r~`or$0G zpo`m^_~7*q8vhru*c*@lee0OCz;@*MGw8Zu9YfyZ8PCGLm43_i4|>Xj})WZCU%Iys^5DPyOc&*S+T9-*jT$W!ahVf|TcwCoT)wI(I8|x1pe0 zSsZzi_E|ul_%~Up!~DGybLlvH_7=w;|A6P=lnu?}%Pjn+lK)xw-S7ne_m=fTzjf>- z4W3z*`h{C*|NQUj9Of0`EFljP?;i;-Jkyvb#AfT4Y_%%jIbY~sPQ|p8qf-7%^lB-8 z#k65Zbssj?7BI;bf~D~q2Ybr5e4C0{$x zwRU!JQT}+_>nY3a$O1Qe3Eu5dD#cwOIq@t#d}TjByF?$FL*0*$*&FY)Hf1qi%oJXuPqSWM zpT?6*S%iO`@$Fpw?>>wAF3qE?{QI)-lMXNF?|IJm_Km(*(^t%Ky|zQ%xbXi~9d48p zE{2DRd`r9(+TO{8otNym(c<9@k&bJ2IFAbW+S;Jbp z4!6VkCS!~rntbz#iwv;Xi?$znNxgobYhB%6m7q&~D}HgMPvh>jtV;t@w`<`!-9K2r zji>#Km_9v=5W`4U+PY-kjYpTvYj<=k?@L;zf>ZBVvN`pY`8!+z*2wpX zEe!QIcYZ>a*brndd7E{Qm(QEreXI>v{vfZzmru#E%uq)jCqAOU-JW|$dN6I>WniW{T zlYS#{Js&y`h<>BrYn}6eIv}^gai(TlRn9-yW-C*-Ta?+-|LRA`TJ*qAXNSaXt9f&{s-o`IkU{Xik4vM}M&B4-fUZF*r9OS3MA&tM**z z4erkO1i$adp4UrUvVQhK`X9~@h;8_{9!pIiZP|mL#`EVV8Ww)U_tiUO?2NZh@Qro! zRbuNR>)`w!QB}}#I#{aL@+7_HihjQ`r=v%uF-EYSsh*eb*eFPSr!Ab)CvEYRJ~@~0 zo$(=uHVn%5PR@7#H22D!EBMBLu^j$KE|YH;hQdBg*yjj)6W?R`&U)k``Ho$u#O?P) ztjzf%->?hJ!EW$j`PMTOb`4?I6Sj=+*w^K-7EvbOFA1j`wK8W3-}pb2!++7~@+~J6 zb_-!YChUu{#uRU_;Tv@@H+wy8zC1FJdSc0jjP)TmbWY6_?A&y{jq1oAn#jO&gkhax z?yxPL0@r%1?12+mjdnTnMwv58yz}uRexJg<`k22e9v+ha7fJbD1J*46Q5R#clviZL z6l8?cF0>nA7edb|ZW+vJz*cZ!q_j&m^^+~%q#oWz9_sY-vjYBOtfNEozuO->DE2}h zi%`Sgja0*Xx~hgBu(F3QcV!QM(UlF4?p?hsJuuwk%GUM!O-UbpX3z4o0?9EYg~Lgs zJdgb;c`kJ^@s&@_Q){NWRd>N}=BqVtdQ=y|*qyGy?({;zdHwMpsm=pi@J}lIlPNeK zT{qvQ&J{c@Po++Cs}A7R?X5sk2P=>r#W^qhtEE2XCjBuh5dTMOQ+XumMY>fo{>pcg zkKNeh2>uTIolC_E-VfgIQY{4^1|N2*W`gU%^)3}7_y_P0E)^x%O`LAG@(PXt$GCM} zl{W`BcdH+$ukzO5)^2r7Z~{2Nt&R##0jIdtA;Imy?c8d=;P&A5ZuPz3bHV4j)gHlF z;4HVQ6`TjobE{o~3%~_#wNr2*xX`Vt1z!Wc#;v{(+#lTEtv(Yx7(CdmJ{CL-Jj|^= z6nrE2Mz`7|cocY)TfHy%5!&Vvk9t?|BJd)Q+93D|@Du3&f|r4pdDNSN{|f%AhcN_f zgKdv`P4KheXFaM)@C)DLI}|fnV~dGQsze-hJ#F7d#m}*{x;>o(i7oR#OGf1kZG{PYXN; zJjbp6B={llLvD4i;CbMAZgscdH^6VW)kMK>g5PwjI|RQCe%r0a34RCsj$4fp{66@7 zx4KR6X7FaWx>@j6@K(1PDflz+XKr<);4i^ny4Cf9cYt@e)wP0mfp@vp)q;0}ce_=g z;77oZxYZSc7l9YK)n$Ue1ApgMg9Psf?{};If)9fayH&p6dT_m4`33&~{=u#K2zFD4 zZjb6EI1(J`Q9T96fMYzWo8ac)<{s5qaBFaDkGeo`0yx2=Itfkzr+8F`;CA449(9i3 z_Tcs&)n4$q;B!4HRd5zK%cI%~euDfwLH`%L47|*(5(NJh{8zV%6>NiTw`wW)S@5%N z)lBdU;1}E~M(|7Em)t5!@T=fg>HmT+Agv2L$|blLxR*!$upOKS&hw~ag8PH}d(=_E zgTaG6>X6_r;4U7uUvLj_57z$#Ukbj|qxJ|c02g>vt>8j%p-1f!d=2;-kJ_m*vgsQ3 z;t9Thyj?*57u*Hhh5j$N2e=3QUvMvQFZ#dWOTm}Y{{_EEoNs#6`-0yFzfJ!a{0{gX z`oG}!!SB=m1#bp#rvD2b1|H^7YXsj2zLEYfcocXP{a^5{;9Kecg2#ZzFqaZ+(>Lu1 zwMy`_;AbP$O2N0|=XUzP;5)&0del>b?*`xPQBMlK4}70TEfxGK_*IX3Lhu{lH#};w z;LpIHdDLG7e+mB5qZSC>0p8(J^91h#@A9ZW3*HUh?NJX1{to<|N6iMOE=Shv=Y79N z&5&=0!G}F+s^EHXy+;)b{sH`hNBv2#8@<}?Rrd;x1V?(+-GX1BoG)Pi54J8mU)!EZ z*fp<4Ajfi5d(p#jk*8{m_>UoMj91+zxH-7FSKTbQHMq4`jTD>!PVlN51*d>hyy|+v z?ZEB4>RQ3=!R@{3YQfk*uQ}JN3I%6@v(WzqUjV+qt1c7V1>D7}1_|x~?%`Ga1!GIM z2HU%Q!PwKS!OqSvI1ilXRec2a2lw}?UV;aM2YXdd!3E#~uj(ea5M1b0odsV5zQ(IA z5IhV#%&R&Hz7c$*S7itu1s>&9=Lo(Pe5+Tr7d!?$#;Z~V-wwXrtJ(^_6MUyvB@1Tm zvgU5DN)UV>_&%?S70i5Q&1A1?DR?S)s#i4=JQF21N`k@-U2)xLvjtPDO{DfB>6}$|*%&QIw{ww&eUbSDa4Ys}Nd%@3wpY^Ie zf?oi?;8nGPUjo16Rl5Yg3VzkAb_#w2{04T8g5Lzc=~Z6{ejEI@SA8b<9q>C|^|9dh z!S8$3hk`eQH+$74!CS#wz3P3zpMgK~s&@r{3I5WnHVEDU-r-em3ElOOstLMR~`Jwq1YgHCDH&#c!F(z1o5?eyw>5P^you$KZX3Xg9 z#a0CAL|AI7NsERSZK+8ntr@gtmb%BJv3Di4m8B+{w0LOomb%@fB|}TL)M%4-7PPZ0 zb+buJhn8-s;U?`IXy;gJm`TfomT9T0P1^a;&bQQWOj>7Xoh>!Qq;-eZ-BSHcS~j$7 zOXZrhKG6DDs<%nYg_g@cC6m?OK4YJhv(87F@B{&Ltl%+a?Q)4a8mI$@M zbhC7QHbpxt7r)+TKNt*y|f~7nr z?Jj6{S?Y&v!ds#I-3#qrODU5!3ECviv@vN@piQyV_t2QT1(J3~1=h5;_NDJ>7|Qxi z)dqOccjdVOc(gThzAAXHs>C|lb1eSlTl}NZRd?yR*aZPutEpJ#-Lxhq#QWQ)_OZU^ zsh!$q4SUJf;$P02jV13Di#XTYVjYNe0r~fnD{ElCaR=@tUL7B6lP|L_S^YwPH3?e7 z3z2#rU46~8KGlzu^l2F8IvJmv_;fzg>Eq0`Etj>@a?zJvZXOShmxq4nid<=Bm-LBF z_ijpaC2>Y)GEWLm8=kg2XYsUKnWm5d-KX@iI`f>*lgV=qPdd+8Jjp!qJgs<|@kH}P z@JvZvnf5yK@pU|J@oeOIk7pClM?9bMe8IDw=WCu?o^N^f@f;$2XX@&Fo=l!|c+z>! z;z{O-=V`^$j3=5Wg2%;kf^bUWm`Yqc)O8wlo<`lLeZli7&qq9)c;4gL$nzG@I-b{g zs)*zKX{5<>4o^DISv<)+@jR_~n(;*QMDVzHP7p@%93uR;(@2x&Yo6^qU+{d&^AXP` zp7(e*^1Q{fj^}lrDxTHT`Uromd4c@}4_W92uBz%6XxA|=)sVt-y9@n=Z~VWgI5$_$ zbitmsWOL65*5TY21@3|8#NI}D_Q)~|-t*P5&ur+LA9D6#4>?r$ssVn?hIfP~s;5rr zT|Kp=w{QKBsspU?Z_9I4twk5whAwmwe%3B?bfMnhZRkR42eZeDu~oZ+U85{8; zT5-2^ajEt~`z%*|B>r%>edbbU2`wR4eIPX4Z3!-wB(z%xXq{HvZMR0LSfR}X&x}$n z1-}V?GfFiR+#lROO2r5sGeEsAap7(o6Qv@AHU~T>O1TBU4SqXHHGByk3?3Y%jtjnh zfO<~C;cmMC_g6|xlby{(^-5I57g*Fd7FG}qa z{66^oD78~?A-FI~RSUj*fO=Tc!`*gwl=@s~kANSEQlALk4Bi~2J`#Kl_?jrSS@3-W zv`#DTw)>*gdqP_TUKFL?5xf<=HA?+M@G$VODD`*2lLx3jN_x24CP$$sfS&+A5v5)e z{2BPOC{-o+M(~YM>LtNb2dL2!4tLwsDD}M1mVuW=sb>X$3H~xlJuP??cvO_C5L`f8 z7P!<3!R^8AU22(N_Uddabg3m^;pc5`SJj4Ptc7biRMKa|4p|p3>9eiBi*|R`?VW!= zKfR|wY+3I5(k17L6wa$Z{r}er=5a->txDt_|Gx9p-Pn0-=s%>2|M5N9r_{5?+p_0| z$fTYXzOKkIpM_TA){vqe~kkK@d+So`<9m+;?y z8Sx8EbY5{sLznmmo@mj7@O$HC_toETZ4~}@jlw5|!e18(Uvw%R z;U9IvpW7&W+eYC(358D%g|9i#(+U5$6F#9)_@<4*i_RV%JM6#n=Z zKW@KQLg5v3nfpJ9tPpn@^zd51e!dA$_*$JGh`zE(0DE7R`lwaK|Awl@|A`y_2cJl| zqBpTUM=oP;u(>&FA2%yB_Wwy9{Qi~A?Zs0C`TrjBa#?6w+Ga~^xHuzE%Eo`N zBF?KicQOBdu>Z2?TZ#OW;#}FvkFdp%_f${(*irnGy@vO{kT>}cCVQErPoKcn>v?FR zrWf(wN z`ctNUl&vp&6gYn+-?442@g!vVCRv<8+HZ@mm9MNV=U3F|vkx~c*6AI+C?IDbj%3X_ z)00xy0k^c+w^;r^%NdU~EiA=N4D!EbuC=FgAdhrM(AU3OE;fyA==TNhO!qBv>gv`|JaRtr zrxDcG!&cz3-A)?A`6g$(@DHZ)Xa}q6DE~sFK0kstzk!}Xn3~V6Ol;|@zMdv|O$p_d zbMxRau?gQree!S1Im@u4zsgM+^X7Neek1D~+lfP=vkcJ}_>Z4rUrf5yl(`3P4COQ1 zO4rfcE&*3)ANaw4$yxmpF3!%<;bQHU+Ksp8!J`fNvUenkHuKST(X?R|36Jtiq2&w2$Z)=6Qvipo<*X;qfc6we8FEuan^u?43>g>Sk#FfTkvTg1X&1 zSn3FTE9c!-d#3i*w){6aJkW>yOWGUB<9+b4?$@U|M|Swha}87M5uq?ugpsf>7N7WpP~Ij(|*}@T6o}S=xn2d z@WAnq4QcosRZ}imi}>fv*$KjPRgqTZBA$sn*Fd|NCz0nM^>Qb9le4gekMD&~L@t#< z6S*Yi`$LE}Q`6kRI5&2%^sf!nS6uPrEr&dbS#mD#;>q#)oYFh+pK7(KJ4RdR^M8nA ztfs})O+^;|q>p^kHLxxu*E*4gaVQAf8~nb;*&IdU|X{t#~;Y>^j~GvtwjT0R{S`vLxsZtoeAPc_}O-2IZY zkX;i*KKVlRBXwxvQ}x!yj*&dkK2nA#@CuPX|AzMt!Fx;6kQ>z5SDdZ&HGC*%!2VJj zbYvf0m?zoi&{KF;##E^%8BgVI2;to$#5MHMhK0M3*|JYm&OX{f-F(e{n6GGmu@9BJ zGX8Y)!5`%`26CNu1lW(yeKZ&o{V*JEyXk)R*wymwl(e=WgNqYRW|KgPK z5y~igq&TOsvie2#&(}EoITyN&&!kz|(8;Q*ZzFt>Xp0`G<>$w)O!_x`X{lYyCEjG( zNY0UU+DT-%@aH$wC-ow>tlhR>7~;>2Z0)~?!JqKyPshoOo`Izy8IVPOq-{oOm;HIT z(&l2<^Gm#@`@5kl>G3!1Vto#aoVo1CjAVNV@tx`mNjiQ>uU{y=PPnw8{S{%-(LtqM z)6mCKFR~_}>kMVH>V%QF*jSxvXLKoP=Yf`U|C!_^r>%9T__xTry#L65h%Ze48GjPK z$klvd(*K=aaF~86{im_6dQvaou7Fm$Y;krA?Xqu6y9ss^T+s{KkjF&!#E4$7j(+sF z4*n|935GYZCREc0hIggRxW%cI3H}uSf1n+YBKJoZbH)+6!m$p~nHiirF7+Tf#AwEb znx`Uyy8o}s4kWSfayH>=(H~M=jvTD}jC1zky9#7c4Qrty8>^-7abB{I^O9MM-sj^i zWLd{6@+9f=kfS|oKg(>j_OqmOHflX%{(gs#za%b^v%;6(Fy1(N2K)F!dd6mGB3m6j zBU$SiuZL*ekQT$3uI$z7jBT#SHqP#v;h2HZYEb{m8jYwTx$( z>A!7~$#^!{)3$DBgvdr;R))ZuXO;cc$Wnm_l;tO<=xjy@*+J0*0!M6w+j8ke3!Z?cf( zqBmjZy*ID)62_%id+s-m-c*m?B>TmGTIan2OVI_nha=Ic&sh6!KdRq0_RCTR+VFAF zq1OMh&AaGOGKZ6UNa{npD)*4c{+Q6Zg68QW))iX9ziFH&C*ucarBrU=c{Q}okmE@i zn}@Ak58mT>rjQ5cywF6u6h06hT?DO~xjJJ|MnuH^0K&gKl;n)^I$>UMG#)9lHwE*2irx(x57 z-1}4Wloed#Nja+doArbXh;sycyr0Cj+-XnIGur5R-B}@?xfwsg8zt-?EfQ>G@e7!<^cR6tjKU%DV z)^G@W136ztEY)GXHJE-+%t@Ni?xp>}ct8BAvPNjNS^JQtpM3v2>@_DP!$Bw49N(o>+Si zd6u{(z9b!AqJ0nX$UPELCeah!^uZ&{<>VfMb4X_wI`cQkL-zK}PeT?jU4Pvxx=&W- zP{ua))3k-mWe(COyPU31e$rd(ir3TM&@EQ^IG4Y|vbt*?`k+tX@lW8(?;2=7bmgb~ z!LY8p6Zs~(GUEknO>N3d9+P#Z>IF`}`47VT>1*5xRVZz68|nO{U)*u&$@Bdt^3S-gx1xx>ux_$<~2jCAWXAms3r?i`Buh4E=yG*;R5s|L+jXok*@=t$v zq3r7v*&+K`d=|Fjtd|tA7MkjE*G)w}3}+ zcS5$0x#b91+wpe}tmCYv^uPBBd`*664}AyhXzDw2DfYkWl+AaIoJYfW$A8vU>1lFS zQ@s5NVb&sd5-Gc^ugTh98u;h#73?Yg&<{lh>v+03@x1P&J>iJVYx(YJwPX=%Pphnl z<=);ndlJ03f%39nf;#otL-8-~f214(IU6$UPx5eQBfo<}erw35w6De8o1!mVM1K+a zIhgM@ zCGx8dA-4vidy8&YKRzl`jrV10y?ePgc$_{FGpX(3ny0y==cZ+>0|Q zdGR2g8qQx8xpi&6vzN?G8Zytdw9EOc$S2WXd>1XyvMfGy7u5^jJ7f28$}Vy^kMI%H znapox%=R>OWY=CTKSZvh`yvxWZhfw43HDcn6~B&-?&#tl>ThxO7kpC_-9b4c<`8oQ zeU6QXyP&4Y9xo@2LF7%se}rE-n?v*YY1Z@1vpIer3iDPd?e(~hUL0$$)ilw2ub^ED zXrIezr^{%sA*|aC4$^+kxj)l>dEG8NN0V{?IPG{Kq@x}S>B2gnq5lc37qG^I9x8I< z6LizR@hsrE7us)mdhoo8?p%*+62}z$r~2aby+d-wx5%kK35~I9DrN*XBYDwnde-{ zI9P%X7Y-|7;`CUWWUqiXov~EH#_6&2DC38m3+CLxBx7lc{T=y42A*0MjC0Nj|Aah; z$I=#typr~_@Gv3~!-6VUHb`$N5nwQ%myB^cBD@F7$?qw7`tZnpXdOe{ew4VD8VMHf? zlQ_4~4vrj4w8J{s8_?uTRMDfIbM2fpg^G|oc|Me8cn;bZzf1MEc-ubtjlJ!RO`?Bk zeVsO5&bJ!o{_>wp)(6)%m;0Ee_O93KBc{Adw3|^LN&iv8Fy{TNylXglhDf4)Q7Fvp zP#$LCcGKmEwWn*E^vwd=@p9VpGTLEGH;DcAq5l;5zx?l1?pPEVC-*KK3h5`v zIK6kUh`G(D55upC&fMmaM62>H9vLeu+;MfDMD~UzSe3=(xgx>-j(Arlk`giWEDB!VFz7DQ37C4fsHqK#VxlqAHq zfw<(cD3$b6wl0~WC~DB8ttEhMQCnqcwf%G<0j*g?iGbu`>HOa3&P*6$(0;zJ*YA(# zb(iPv_uO;OIrrRiJ4K!)_i;A@#Zvxb&p-ytFt-l6iM4R7G1tA%oARDQd= zoVpfqCNRUY5Pz(;;XR{rVfQ9$q_*KbleJYNZzx$JwGHo&StlavZ)KlqroNqjVa~80 z*@8ak2I}-O$3H%U?3@3x&-&gk_SszdkaaG!={j(P0`IqkH*5Qo_3~yTb~MEXrNSPh z@auvf&}-5>ay8|>Ab6{%ym-&r5Vihg?)NuCC+N8-<7|HFoW{64p=RvwLd}@Bu+#dd zP&46RDDNex9G5=hO*Gz;_5xebmm+J)N!jC*1=>UO`%O9Q2?qjp24xIIR@X}V(u%Fd zkI9=!`*Q{NNMEh?(l+D)TlpNw<@7jX-|1wQyY3Wl95~~DP|o=CGdmYU!v@ayi=o5H zp{4##nR3Qo%o%?%XZ%vGobdD|Y8a4)dIhBYV}?ozZ)%)WHj@usL5K=K?;M$umA1XCNbH9V=s+nadu{+mW4)zWOjZ zT<>}EM#vFpjCR`jrTXXEjTZjvH~1{c>p1`2ZtJTomqXjyf1k7oDtX;R7-^jPGdu)oWlgn#bG-1~t?<~g-{kWShe4m)hP%wQt5k=> zwsxcNUm;QRFhz%bA7EeTC{m(v_TQZ3et$(Pr-F<8<1~> zG#}FK8oL%eYG0Na^`njQ&z}9GjSfq4M=>_z+c^8l6_i+c&9zX;%>`;hUD#d>Ejvj6!^Gym2H%f4DS4~Z8SPR@ts+`SF3T&o*7Lkp{_D; z&)g2^cQ5v>y2q^5{}P!mAFdf_yvVpnU+t+Cb3U|HZSIa4>dzZlw1 zWP#Fm;oC}V=q`YF;CWBvx1<>_vBwE5{5Q%AoU@8tFiP)zycJ5krLi7{^Kbg!CIc zYnT3jYg1Ww>BRTe(Sml%dPw$6^s8iV28QQu<#RCi+SWYd8hfRu?hEjp;Pm(XdRs?G zE4s8tdg*Q5O})W-TT-sz^dI)2x2569!roPhedjA&O?Hc`3H`D%7xtgA8?myOv6ps> zY+GzpT<(p9X0b*6W}*>A-Q;PIJZ-d9!_@LTa~SbLqiUEcPB+$oQ@%@|&sawpN3F=n z>D$A^wY-8pFtnNQpkn{D7Wfo=dJ0-Z_?$76)3R);c^Z4`SL5%-KM{FJU|rZnh6`NB zF?R2x=O$_Qem}qax1<#vuF`Whp1EVc(I^Y`>47Z}%Tb7qwB z2gx^B?WzaXVi)CWTpOo#KI1$l|DK0N2+$j!5oZ4wFJ)PE`*?qv`Z8H_c8eO5Nx71? zjdaqdIIa5$FVm}>fA&#MZBIEVJ>|R~C`apnVmG!|Iq&pQ&g!0WhW3>6W}qCc3(EUH zy~=6oqnvwt%DJMaoL2(nXr0hF&&$2a`9&Y)%Z6?TJ>?|zl=E1i9IY!F=ULUOoU%U3xw5C6#GZ1NkxpO% zeNg>|L}nfD7XjyOBQ0__zDx921g48gC+qCn+7(@si*`x67v{s4Tzpf*TNQfn_}Kou z2R49r;s#=;eh~ko;9myA$3(-=#K4!vLRW2+FY>yP6rrs(zdNutroztq8g8i98$gzo-0zIlr9Ekb+qz6Ers&&IavZ;QSy zHeh<&*mm&09UI&pY;Sw8z1@NB?Fejdi*1ZF@`!EjD%>k^@4}sio2v3t7TNC{y48O7 z&|^cIr!2wV{fB&kJw|Xt=){ttsS}Hbs?5bhEB}0KRz38-6@Db`AB*oxgCAM*O2JQ- z@Ev`|vb^O7YWYVtbwT5xC!>uG^lJfl=XvnR``A5cX1{rL9?FaAQ&;rJ*@-U{&H zt@W|SI`YWBa@-2$K=7-Ce@4Do?7?HBJ!cPLJ$4_)8kPR^58^(@KBD7)GuU{Zu#CSt z<=v(6Zyxh}(mU=)v?XB_G7{t2uB^75p zyW1xx`W62-r~BxXcM|1qvP@>ZO{gy3zV|Du35c|%L` z32$2gEol!?2U9K#>;iYrjsNAnho!tY!Qb#BJ$-MEHS#H2#wG`seFgc1teaba|GBJ# zn^_NYSQoS5=f8>V@|##Mvz8wpdn2}!LNyN|b*5^0S)5Ts85QhR(#}GmQ+u@&%R{=I z(njV_x6yuK?qT-PU9o?D!+u|9X8*^+sv)bjHh_sB+rE)WO^+RVI8w1pA+r zFOB|0;`Za~&o_WC>*3YU4&-jmXM3`Z6TD&EWm(YOQu!^eo+Fo!b(v9?s2XKaH5QAd z#%i(FWJi3kQQ1G(n8r848d=j%^{cr?^{<&tc%*ee&1aQXW6)sLwFf&^!l(by(!XX6 z?m~2&H`9kmXqSG}CvQf~$lhL8S}F8h#5>Erm){*>iKr=Ze6TT}dYs4vL=KQ2tr`ls zzaV^v$Z~E2-!-F4BXbGua3H@W-FT4w)$4}~nU~G{Z-EZ}$gd+fia3#(WBiJ<2HuY& z{ci4N3T#!w$0OrE6fgIa){Hg;em9Fu2wcIsJD_d2M`O>OIj^_Hb-Qizx}njvOzWui z+uex{(UZBT=d37O!Xkf{_KN)d9BtlOU_GeQ0e>Q^3M1Xts*7SPybe7H%K_Z3?( zxPf=1SZ_OpZb}4xB#rE8LN~ojUf~t${o%I>%USU6aBtS-aVL{?k#!bc4Pwr$`Iy|(U*@3B#OAOGdttLNWC2}YWq_r1qo z#!s}H==m}3wx@mT`hIjPrOzAmzgW+6{D*UYma_`UHRU=2!x408dUJ zjo>mT?tJ~;odx-U1^i$IPki~^-&Q$yL4WX8=43xKx*gmrvUGIv56M~ILS-+#eROqF z60kZS9&iY23AunL#g?e=G@M3uT>rGITjX+L1FjkwUAvaqIXd^S_ENmLUHh5S3fi;@ z{9i2XllvUJjhz6jNS#04rVN?;C9Gd~#h5l*e3pcff6sl6e9=pP4L^tdKk^o{CLV^5 znT0=4cM0~UHm_-(TpY&w8Q_vTPiF6W^MmR=!}}e*>#hP$KhV6%rSIIHy(b^MDt1XN z8kcI=<$O95+!=WLR?k2C`^f{~3E-Q3&ZEGu#Ufj4#ty#V(NNw3fHv~IvA##!>rZ8T zg{Kks!YW^!=9=ZW_wkiWyI-{S>-?$Hb#`mith2Ju?cgoMPi>K&&yb;s4fl6>Pw_MGu8X-o zfGjq}>XXGDKo*-~^~qunh%DAJNXudmvc^YS2i5E*oQdu>I2*Y{L}w-UT3(^NSDAxg z+3a+}cG?FH({Gpr=%;k(B4uAZG>vym+|W{PXsLE+DF?Ka{L6%9aw4Y|{NzTi0pHX$ z*7CwelYTl@m2|s_--kX`$I6oKBF-9QugNDYvV&Y;7rJupS zR}(LIH^?4ip?vfja{_ZHXJ=OI184EBAnyj|lt9zK{~K|)DEGo}^>Vvy<3`7~Ebizi zWzHm>3pfqnP@x4i93kIn(}A?fA;wU~F_X56ls6~I1sG0kiJ z?= zM4wf8ZM9$gS8NfX}q^ z_O>qXM3S-Q*WL}Xhw1Q-hZw5~OMdP(l{K?Q8=$`g|K$N^nZ)aQc0@Lbt#u9iTSjWT zndnY7)ee^h-_EmlRKWA z)9{YA}Q7^Sk8fD4^zys^3>X)to-ajn}#KPK99t8cvK z6r;z}{s(OMfg2-PJGxzd-2R^@eQQ?Yghp*a)+O|28|l%n?rfW|DM@6mGyUUM&v;88 zo%AaU*;*!VYZRehmC2hfr=(s!O9kir?`k~*_JU|n9&-G_v_)X!1LE1wnp}*9PD|gv zFLUPga{q&KuqD2;>EXn+FgC3ja_$x5$sk-zxUJ6B)E+6a0Y_;oveLGjp%L_19`_On z%N($0Qg4jMLcOP?KGt(^d>&~&CB7awiUE$Q1=gVXdhiu}bI>>5hG}A~4;0-VZgk?7 z?g^Ko{2RMSTM(ezo`^lSVwCcR-J z(1vK>f1UI_P@dwff%FHZy;XN~YkbK%agt7SC50YHr9Z4Ab}iEDt*4O-&c9 zBcJ@2^{40fmKjarw-dJ+9R+lBp)t}tDItZOZ&6l&4hghJ&SNy}tE-GHe!KZ^QKrxs z%3p3~h^i5o$ZDEqLaa2Tu!l0&pm$%rnN4A)hMCHVMpl~3J;P%6l)?j|zv^auViwA~ z$D~i;&11=zNj^!RnU^(5kCU85G9M9XK3Q(NZC#Rmt@!PwS=rS|GAHuBUt!c*Fv-Y_uof9{|0|Ik0(PWl(Yn?f@z4o?mJqkeZ_?4?iA&n@&Xmwrlr18oiTH4y%? zJl#i^_28kuUS*Wh#%4dBtM=o$R@S!2eq=npzy_iI{k z>VZ_;-uJl$H#1fT?2DMwMYe;f-&?T5@vS_;`%NlD?QjfIrv0%{r@&|c7F}U#r&6Ph z)`8Juj{eEAUXOKag20672o?P~AH6GWDF-jh8Pa2M-KK)C)>&Ze4n6dpibD+#zW8~}7kfT$c=G7y?lsRGn%@4>=gtZ5 zect~2lbRptSsuv zjH*uRfL@V$wV(URtPj#u%U*rJnXTXSnPj&_WQ~Y&;YMbSI5^V4)~{L4yF;CnSz^hI z7-71aoZy0T_5+#UMW`lij@?_RreP^!Zj@mPfO>(16YM>HZEw!kl0 z;c;#7xFO)uP;hD(^a^+X;oH1bj)T=AgY2PI;=A?O!pGTJKLMQtDp1NBX#-l^1oJ?$s}m**pk%R?5o|6YODU+1ZY z>S1cOl!pzhY~gipx!`SB0sobVdoTJ@o2FRG>fvKnGOmT;VY5>#VcBwD{}JRSlJEU< z-iBq!0oLKR*|I!G!U{W=vhH=?*bDu5|9}GTOZourFQPB+vF?{LkEPr4+?~T6?zS5^ z$7ask3Y0l7G_FR*!@*v#mc2mEa`U>qTzgS5@$3T~H!Il-4mgAM0>PECkB?7NZjp5? zL|#}4erzX>eLi?kIN;wC4%oYVdxCvw-}^xQDQ^z2D*VTRfGvi7>)>N2sGO6Tyu}AC zUm|A;@SG)KQcq%MCGrVQ94W;W}Je37u6kRA`T+(lqg>i$A7x4|uoa z8H<(7V>dGz^4Z}Qqp-ZK>&VA$`Ksh6&Jr3H_zdL0UKLi{Gove@Qthl#$D~kaA>f22;)$X%FS>rJQY)vyb%qlzB+<9>af3neU0; zmdCl0#r&g`lR+8STuxa+IggM(j&jCJdm?DhfbNuEP|hF7`v-LIB=5)gKUU@&;t!{s zaEtjXDQ7rk+`xAa<*X!s66IVg?V+5#l(U_3L?7TCWxgnRJMniabG!H>DJRlm)=N1f zC}Rd+Ddnst|4_>LhO{S=_6+P!d5Ln~Bkz0YiAmni@PDSv7sNl1at2z==cSxclyMW^ z{gktY{OOdFFYTe6y_EARmp z3)9@_Z-%-%Ss(JApXjk@IwI|Zq>gfV@6VB?*N!pT=%9|Qsn*7<6(x-eLsd4kcT(nx z35|ig$m0y|pP@s7%nyB%`jy{pER64-AZyl&?qEB2oR%T~E?ldcUR}1dasTSW-1oP4 ze#|;*jbW`mcDSxSM(uiQb#z_ZW6^aT*9n~+(b+K;T@$0|?EYxvG?CNY*CFd))<3>x z!7cRZ2A4URefy4wmDx^T6Da?-=jitc=##;`shaMtllzPE4%dh9_i}Gt?z!BIKA!8W zciaT6>r-(^_i;;(wq+xo_uO+%P4>}>sa$EO8%MdLW%Cg*nKe@9K zT7iAvT!;=y8h$-)SM5)F`$G8m9pLN0_>?%}YvQM~uY=R=jBh*R%U$uTcE)w-Rp_jM zPob%zmM+Ebu-C*p!6!juTFG0>CC3;x4bzTzs|nKNN`|Z#OZ2^He5|ZpyjwD zY1~m2qU946RTY!HtTiw2RLJ>i+X!H-vSM!ujkwT4rE_t&TLTFO$5f*t$#| zPTZ@+vG!A|)=f?wOPKZI1s|nWJvBKsw_-{Q>)6X$k@EimEEZQvai^0E$EfaCXx zZvl6S%qNw44oMgsRhXG^`(;XI{mW-$|Ht;AI``#V_}N3b%#! z*xT|7{^4EZHrs|IbV*Acyx$J)E(?J#5&DMtoGtU4%DA1Y0jD$GDV8Cn=SL|spL>#W zkKBf9Pu)DHoqbt&J2&$#^^9OW1nRFZwV zqmsXx{Qn}qnz(f673hry%1~)O`K{co((9fw?2A~B*Uw7U@G-ceGVxeeDInk%KcV;B7iEdhJ^F=o(j3C$KDZ zpgDX~(gN<#7tdo&({||8^~K-y`K7(Z8}QS|BxyVKx3k6tXDx)*FN4tpP5(eUe9OzWF88kH(U4l^b@weP2N#H z9ne%V*JLAacDd&VU?+PQ6H6l1R)1u7uP~HXs9@EfQkT=UyL)C>Bxck`# zOu@Hg3yw^UDqLwFSDn-*Yd2{AWh~V$m${F&wlimXOnUu~j|GqX#bru=TSy~v_TFVA zk0Z@#m-#YjJ|N8tjKiaRa{iEt+dF;qEqP{OEuO=clC0&xTxZ(iYaF)Znj%|t%?a8v z2wFGtK3CJd;Pw7nBaJA|5APfIa^3x`kN0j@?)?tvcZZ4+T-RJ#Ik~xV&EzNfe;;e+ zY5uR}|I+>{s)G0m{yU)4?EGKKI&v_6LhCx*%B}dX_^)BJ)tmDd+7-s$V+Z%h9TMoz z0}sBEE$<686+?GkAI4gQzWo=j$Tr?0z1<1V^l-?9K;IO)Bsy+I$c06c?{f!uDaO-H zo)+d`coW9vK!T-TlZKlZhjAri*vvXDK_(oK>q#2>mTV(}y~+MF-pe^`ad&XPxD6PG zXBF99ZQ5@ljWchFtd4u^^LX<;P#avXT`lezmzgCl`%|X4e|MR~ z#D(sae#w59F77_?ytsdGnM1{W$7LpqyB)kH?n~e?abI zX%T;n_-&nA*jMcd;t$d1%=TpQhw3{b_EhnQ={q9!E5#qK?}*qHZ~{MzT+FN&*Dn9K z_kkQjcyijJWvrK;i|Dgk^EKLC43F6$F8pQSOl0Rja=Dbd*%eSN_}cJwsF z7LA?r_O=w&)J*%2DBl0^w*|h>to64C-pzapfBAnbZulK}pQy~md+WtX$Xb$=`F%+@ z6#r0VR*63ye>yy-&J*G;A4wi;!EEO4UGW(HjZx+bNjDz9nZBK{RA^Il0e3;!(ejrix_pQFsXb)E=!`M1dvjh!9nq~gW=Ta4UF(%p^! zZe`vf{$===Df4!nr@y=W5%R=yRs%0zypn$_l{rV!t;WAvnKy}l4gNKpU+O%A+~wE> z<}IYMRA8p~S^hn%Oo6@P=kY%ey(WI_NSALUoz4^OF8?|H6jfFMzhCU(pGTP!CEbho zUsUEe@xOxq73_cOJaO*w-$KWr-)e_`b~uo4L(k+Q0~7gn1avffN9l;{bx8|}6Z!U1 zl)FOCz=G@@v}EXW;yLK~Hwpdez)qk2@yE5@ zo$ucA+?;v0?0O&>`D7e?SWdX_d@~V#g>$t7^JUX?HaH zxzM<>Kex?OP0`3wT1B^kwXk|WI3vYRk8qz!=n*^gh}g|Z<-A4S6Z;iu{QLh-z5h#i zsNVl~D%t-D%l?0o^7S~e|3_a?O~MM}bPRss}Hz%9*+l6c|I|Ld*TswDY#Z7gY zxz-__7G!|gSEBPaLBdw8!;tL~|A0VbzK`0=zMlea*apn_&X5zbu4Fu9zc^|O$$BD$b95Eb$-X*Q zc#hvadS~PBA6?wITZNU{X@`aH-!`kncN5=Dd}ks2?dO#1E6-*uA$_jZR+>*4+i27M zkEZ+Dv?ATtChgLu73tkSA>Wph-nqvplYHH|%GQ}qSjv;NcMe#)nLBXpw$vXO-(Pu`kh)K1AaAr*$a(KhCvO91!c!^U zPrU*BbuDvv4%*3e#M>bBTpE2!`}1gH-BI%X%R8<={&NH~8$1#ZU6*^Ft)tip* z_Ky1}|7Gpe+ivS#3LXoVk1rbJlaF6mHJ?54cJ{{G*dyn$SKf+j{1#;6bCHeTyqvvM z>s1Bj>svD3ec+(;^IT0Q!I`^&mDhmXc=+hsse3;Cls23ThbE%ko2<(G17%dOcXP&X zzA1k8?@bZP+=G7;`{gF~%l8K>1D%jA7kkWR))Kk_*~P%^9>ygXzrr7L|Lb+>l%2sR zd{p!ID9ehj)Ol}1HRU6#Z&*v&OL0@ML(%~p3Qsyc7FX`I6L#Vk7&`;wu+-2v+x1((34dc$^MwZIGb zd@|!DI@$s|$8g{Oz1)rZ?ef4KNRe-U)p|@0T8~3}^)HDqJVa0b;)43u7o5T?`s7`f zEA-i;$Ss8bK|ke!$}AbmdG7_G?bT?=t^QW{~w@=RAg!i~n_I2nU_VZTu0y%e- zcXGeV)^tF>M&pf%V~kH2gXjp=(Be?Lo@1@lKR>;x+R^P>!~Q*5&K;!Qc<|C|?B(1Y ziw>NUyiYMzn-9W4mN4SS@ane+7db!jQ!EZQ<2y}gNE5mAD~;!N_0YhK)Lg=3snr9 zuD1@UY50%#wGHrkbAgrh)}coAFqi*cU33lSVD3P_7Ll)azbYeK7p&kt6_W66;q6_S z#Y4HP!94YC3tPWuTi~+~D>|!hGk;%a$Kc=4M;~(Tat!+X=mgaydiA;IL(4Kps=0Cw zm@?T?w*MMr(GM25UtN&zZd#b<79GG3lU!%-libpdun8Cz#-K0FO~Q^8{zW}}u(l5<<35oE3<&ho8k=CAR) zGT1vr7tp_Fl)n$&Pv@?M$Z~>cP`Qi2I*>i2Hh2$tjkVUyUbES^chshRGR8R9g8o=4 z?V(*I$dsz-dkJ#5YVdKUxX>+a;G^mc;k^>{d0jPoP&=QrTh8k~EU=(2BetcI#2$vm zw|6~(%@6p{f50Qk*+4aJJ2c{ST(K`6jBC*`yad-yrwnH-cWeXrmVRU%oMsk+b2ATy zX?s{s(&)Puz3LTRxoOn>Z{4OCPpjCMgZ?^(%uMd?e?+^#`2%0u7`L3Z?Nd2BQC;ou zlnZg?44(V44ek8f&nLQp!x(#kiOI}yD*jXSNA!xsFR*cn`A?E~`X}+?7dUx{_!#`{ z+ori?EZ!pAp6fEjhAeVKQ|_4xE|dEMZ-4`^^-BI^Pq>sz9bV+rA>yZ8FXuQG{K@oB z!+_7mRu1^7R^2(yc=LDQjlrs65x6O>cu&1(@Q{8D{Mo(A&l)b%BK{@PPtlIMFp^(A?J zcA5Grzox$RU#gFH0m!pn*B9gYO;3HixzST!jA!j->bvV}>T4UG?-sj#+;1^kfcIA6 zH-LAsN%JA|`4O^XvE%$Pdq6>TU~FSN%XNFO1+Lv8o~5`$m>+>L!KcDEHmdH|lg?om zP_Or-?MuK*@-K>i>K2&;W_U(AKYJMNFQ7_d(GM!-?|u2T>{@ikI1^c zny|oi4&yjXw|ySS{PF!+c_exz#sRmptx`N<8BLzo8*tXIVdj2A2*A*KIpoO@0R;$ z*4hA{<)`cPyX5uc-R$GDYDJFlQ^J0_u2$p-n+XT#R=p4P;y?U!-A+5_0-{s)-_muL z$oewX&Ih-S6WO-jlbfisI30hwGG7yaKK^`VzKUPuRw8dI-^v{pXn&E#?T|FX@DJnt zKk-k)KaKbQB>ml_{XXB%Ne_=8^13aOCJTQS`@i^$@E2icU(zol?K(aW>AAy#PKEiL zq#22Sq%xlszhBPwjHJJhv`_QBNczE~8LZ5;l4cD4G06YKKL`ID0n$Fp_X_Fb zh=&JyT+;aEeJjO(EAh7~^ShG%5z;=-_ZsQ_GQ~1Uq)fi$N*@qqn`_wbI@TX<=H?jF4GU7D+FRx9iAJ9+h1XK^cdXkj&{spg( zzVq`f|Bdc#$^}(c4L*+h9=dt{c4Qz&J`$nIj-iWDa(B|rG5w=ww6a#xHeY9)&Oo=D zPtJ2}_0XiW%?`d-l%d~Z;GC+{7M=Bp_Xl(OC%{XQ?_Frwfwx4|oe1S_=YQrLjF!sli) zK2g+XiFj*cdb6W$A-qglNae;f?u4el`M|TS$n{c`t;|;ORFaai2lYc%g^n_E;Sa$R zxz>>F>G(sfQSav|CrX+B;p+y-l^FP-*s565$wLtR++PDY3IXi7^q2G$}Y7LB4{`clYF;;?y1uu(U z_)_2_SY~2@|C9gTg4jOw!b1cO&^gEePQ3fn!BaP3Q)m_UGOCmz`;dm4xuM2v(rpHA zIRBUgos%QBETkP$uiQoMU}27T6}L(ZMKue^Ks1$WoPK91Oq3$#7`Yuf&s zKHA>Kx$juo4o)sB`99~){x%Px&B%wi&s{pHK17YL(0tA{#%ju-?IPEzb?))$tfH4x zX2DK=7J9314?qw10{1+4dqUD7H>oW;ILerTETFT$8dro)QxUMVuD{xK#Of&BKW43* zbyLJaN|bm&ZNw51=e{;xbe|maHD^|#3q2i z0cTc|;48G8UXh;tqiio@%izXHn*wn*)@YkUP52>wd+_Kr(B_=GQOE7HXVT#j#&ze= z8#-(`2tPVW_)B$H)s&Psl$pD4AbJkbo^Oy=o$K?*8sp&qcYSW7oEVRVc%ME= zv?r5v;G%K*9#y*B?TyyDOwyOaeIq8hng0^Ocj&UjFgDVL7Ms3byECmjRG0S@{ajC) z^^1oa6>GwbWYS1}X!uE)59VC5k4ybl{!6-8+y{YA>=_;cck*_a)2JU1Q=_KmnbO8^ z{>wXJ<=C1NU8}%6%X*Qq%4H0s|ABb9Z>9Jr{lBp4tM7Y~U&DAbFwR?%@$DH+$bBw< zpH_5@^*99ZzAdpE9A%dCiA~(%Xt_sj@uWqo9N*cW+!Ih9!M7RUrBuQFtl?tD>XJ1a z$r>);{Yw8DKFj|$c{qe3G#_f)XX$RC-v7YUN&9CAZWF!V5XOrv$Lz(l;~8UD+^YV55A^1MKt)Wx}S zOgFeo_zCV#9y(`>&+4WvuVQ025&IOAP7X3+fw8z!i!+}0c8jgaH9na2*_K*GIWAy! zKKG0hmMG)L83MOSor^e&kn+#M6aI?-a=(H()Axzuf!z({^TTd!+8-Udo&VaSbtdP` z2M$kSzp{}wDDHAfefN-_y`k)kbyJealm!O6!LNa_t|u>W#2wVU<#OLN4ZW4#vR1is zDzg7e?x&Jg8yo~=ao%(d z@TTP4m-E!SNh@@#4#Nj6CagG5)nWLcZxaT#w7gcYG5DZhd9Bnv4|%QV0it(!=qzJ5 zf7>|Y9PX-Z*@g>#G=GW9Tmr86RMtZV;VNIaL&61ww-H9>@o&Q1H5U7uP4ihRa*x?2 zYlZb4=-cV$FZWICRYm~Q(m&B@(sIly+m4nh>kjB#-QNh=dqlqJk7xdJIJY^3Fs^nI z?LEub={93yKy*_kQ9oxRhLo8?JJx_3OCl3$k~wcLiA=7stwpcs2QK$w&O}QFCf9`A zf(@I8(R+E$W!|&x2Dj6OPELl)96|rCUg9#ZsdAYkw{hl%*I(3hp1=z!KbubSB;HcimUtrdOePiQatL%ao}VqVO4 znQ|xS#cX9RUzsKM!J$_pIyckiYp!}b=_zmc_~k2?b>Haq?pp4MsCjX^tLX^30Qnu> zxyx7H*WI6cSx>FFulr9Iyt_yfA6#CbEb!RHvM6H*WpFlKn%?G}`_zi%-ODNC8|2-< z-PTr{4-VHcSI5U*zns3Dh+J{~g#hesVZYF@yr;7zmkssl{4LQsf7VLN@>G#uqCbX`*Lr2u`eQx!LiHRj8f{cyV~?_%B4=}!<1TWx zaCXyyy~FlV%AH33v^OJ+yUCx*+7%tp-Z*m)YyEO>Vzg(>1Q*W0-cAu`nIS%VJCqa4 zT!8cEXgt(wjO=H;;|}qE5OvC1P{+rHQbri~G<^Bx-io{gmvVL?v%2G>v{uT^@MLP|7%o{sMfazRMK2 z$21W*u)pfZi@>S0DG}I`vp{602Q+Ttjx%u6gZDCNS1_DiNm#{QgtJt_0%s=YRJxCh zVPeoZRdCsp*pmacc>lB;{3N<7;NH9-+$&|obMA45u`vI}xyKoA&I^Lqs(j~byL;tp zwN+^S7Ylez&Y?0ng9@Z=2uj;(qb|SRZp+8voZnb>Jom`Zo5hB`*kcg-K=g3`^^SJ~ z@>S1@pfkO#QjY&jPukkM&Mc|3z<>Tn+E$xuhx98@->v?B3mq%`_!R5`NF63PH;=Iv zoErySrIkW z5CU!xy0paVGH;`tgsM@79a^#k*_|C47n$4zHBeoM$$)NLvuCox9(VDYy|`7Axi4ng zSz`(uFLN$DP_^`#zGX6&cAIK?{UdL~!FPSO*jsGRCplTqqW6D`f8GBUH-S8zBi_*W z69asDVe-~l&g3n#BG@N=aP3<+!Q(GFDd|Du|1HW19{<}3v%Y(1gj)y)#=ozzZv##O zV`-^$Emw@851%d_dZlyO&`Qo&gK@8=QY|kQyvrC0&+og(xGz8x&Jp|9fqTBX4}QA@ ze|;>77XT+MW!yD?eT~cMbfAL--6}Rci@B?s%2(l+74+N%W{sx271;#59*RVVv>AO- z!8s+Aqq-uR0_ncVKY{5$Sl&{RJD^|8vqFDjqheJE@9SC&beC&OtnA^OOX_y|`IA=U zPJY_+g#F>`FI4z>l}8yzP2>4_6?oO1>)}-@HLnumxs(6(y?B-Th(qS2>An|#D=^~2 zJ?w)A0`|Nu{shPQjUyGxwRkTbv@%z0A22z!$AJtchw(aeAn+CWmK?MXqS10epDc{-W0*x z65>Z5y=j0lyTp&IdDCEJeky+C$f8TrDSl+co02$_6hG@{(@<=o;TPG$daDKd3KfH~ z=U@B&-XYR=vtlc0kfr}a(&y)yoANE@-^70u|51ziKK}jI=$hC?Ax-O%H#~^GPWc+E zzF#fyU`KAF^cB`>}B^I5Y(qZ^IQ_P?F|*ckcEy)i91vrv>i!a~@vLSeB%!hWf$4J$!~e6#a&5%$OCXs0Mi3Ip{>pmi_b?cV}lR>us9dwA&0 zVyg@TNUZs|&TW(v7!RRwoI|0%LS0R5SEzA1-YjaW#!uV06OP^y_MepS9_t4fMT5aB z*n$H7YlF}KMaH0%tH~l~!J`KnR`$!{WX|Z&D+9kA=+Ozd>jpCo{v@Auo9`0YQ=)&r zi}R$vja~qGc6%qecY7!DO>m2?!OdYV_tv<4=P!V%%@NI0>W3A$v*>@vHp`?AbT~RV zr}ku64{ovBvFf(&lr6{zd4s@ho8Rb+Q#oR7Z?7m1Xs?=2g^Evm)W>Ynf>r5kw)UhUr~@}Zsaa(n8cGu;>BNa zjQC>~Q{fjm2zNC^HrXY0r=e@=qCKh^dkwTlbTGEIqRT%>?A z`<|GbzprX?-o7=HUHd90-w~}2%Dh+o(PtyG>kV{H_TAL&((*;S=R?Bs{~qYMiXX_i zCUPUGW|Ksm2khUjy#9#44Y?l8}nFsMVp=&7RiNAt- z5ie{|<~tJ49es%xe+Bm>Uf^Eh@9>+@^Aa!qicQ3CQs!>_H{Wb^whh0*Jv}$9sjwid z34Fa%{b|n>m(YE(KDwifa?<&EirT=PltRYB-hUc$H=oXjz)WWb_y0$6H{~X|n__kA zbyGdqm`#4sxy)rRmo>hYu*}Ehc!>c20Ijd8_OlJdNERXiq6q zy|H(S<;l&n(lQqf$W(X#MEwX^1%8n^rCu>$mV-Fwh2q2L5J%Ya3M1?m72VFiTubJ% zH250bF3Z)1BhBT^0uL5}PjlhHuXni4=J8+a%<6IavRs+A>%Cj{5E(o5aGrUVv|^|7 zN?d8Ply5bsnw+7q&Z9khiPv$KX+3do1mZ$3#x(}w92etW4a5Pff%0Aq#BnYWh;s+x zVlT%1lsKUYs>kFt*%>=z?k9Hb9cRQR`QLURzWp1vgL21s7ke{f(7BiXZ)ML5l>0-< z*vcLs2tOlX_WAVxcy-@u2#cPj^m7n6NzSi?_S!tHz}!s#R}m-v-P3I5;ol$+X1vc> zvy9WjxX(L`yTdEhBn^Y1z*o2u{ji;C1@AXTstdIFfPMB@16~*UbAZwI3)vfDJa1CY zsd)7Lz<(mcUQ8RrJ%C%$YY()e|JF9Zr#mh5|3cEVZJcVh|4P=rqjM?nDztW{X)`l% zQ~qYv@7vc?hm`qq?Z$Y1s@-VMR@||`lHkAZ((fSp`GC;8`}O_9ney&oj)G>I0bMBc z?Q`7!J&a8V@jr|IYtJ9{h0FQFMO!DFKa6MW_J12)7T#8GE<$%IS(B%s?uzP}domSQrv6t~Sb$&(W9vqPfm z<^0Ewx3nD+Nx#K^=Q)!xK$aN8*%ogop!3$8=Z=bTnHk{;HCNKc>c(mMy~A?O6a^p5 z0%uht^$Cq4Hb*qA(uMtd zp^KIZ|KTW&;0#1?kKFMt4{4T1)Y(r*uMUE$pj zXoIvlqxb7=XFz5U_uLvrs=!O z%sPVi9cTw{=Hxi-j+%&O&W6F)av!i1+CXHES7YyF(}mD7&S&N>PxsWlxZ>VMWS!aX zW^bDR;)<&qzW|oSuJc0Mpw7ao+Kt*~1aw}TEt`ABJ~@>g*}A=dbj=ywT9vh8w>WjU zZD5GoTr$9YJ{P+pouTfo5z4&o(dCWj#&%CAx5bxk9m}~^`QNpzheg@UE&j}GwaZ>U zq|V9L@silBi0n+q#zVaSg_g+Dnb|c-<-pYS zz|^{^bxAw`mvelf;vI*ini$KVn(^4Pwxh@Ez=nprrBO1hUhDH}z25BnNp9#tM}Kq^ zunCbT?vzP8KUUkybD$%E-7c*+>$pPvah+R$D@U666FT$6f0g(X zJ6n-k3SW;+f(8e8a;dlubfs{0KPtx=OP5d8x>HM+hnf+{u|?kOaP!7H`JfMt5{u}j zq09T10^eEoUu@=!jG6BYFu^mP`@g5|7rKtM0KG7V@E4^%I#29pj3n&!4(50`if@qn zT)3r4*FF0hqXe12Qm3p-vB9u5X({*T`dXt)8Ee{tJtRlz$JhyI%Lp;#J)5=y+-TnE zh{j#QH;}k0+(_JQxaCekFO|!kjGR<>&Bhqx3G$r! z0DkU=MeeiPEbdm`t2_O%a(}e#Mtxp$9{xn;MdnfP@V^oAaRHpa=#Nwzms)8bC1vemiSo%f`^xaA7;Yei{F79YbI+*{BH60 z_xbn3&v9=%W_kMdFX8JzU6Q8H|NiKy@|Ap`xz>@H5LWdwHMc;H_j~&c#U?EoHuO!bDKb%~r-DHp49}DDF zioBlv@_-$hf_4c_aX@GaPY_K}<)bNT|No>Ztn~XL zEunbJ)0(x`;d*uk^h6f-O~t-TIrKz6uqe$T%&0T;q#=7C(9mx_8QjC0LqYir={HU-NE|V#D1@!b6&D6 zZ1hr=te=t~Sj#k#1K_cvx^6ESPiOlJ(Q)e}Db5e)|0HZRoium3-1{Rj-!Gfu*9f0eG2*ew7)ciR6C`lh+cTI#G=lN15X zn`;Rx8_!*N+5hFPd@3*^e6WUjXhg^K)EcLAY7;b0=@aW*D}@KY+!_k-*YX~1yDd8F z3^vQK&))D;))2PxvpAzX5CIJq$=xbBzn^rCr!I**RYLPZlj}46>xfGh8%4mM<5)g- z!z-S(Ah!k{2it+cC&4M5(CZ!81#N}47Wzo^uTp`#es?9+Jq0bd-0RJGD$ix^hDWqR zGaqM<_>}c8>j1jBs~H~UCDy@9*iKHd4Ak$A3Lm_DRY~J+Xvt&1O&;(qb!~$Ft0Mew z{>z=zRCG&auMs(r+)0(YN<+SDSzkS3Ds)p^slY@dX-`MEp6&Qg!Qpvbk00)^M%3Ka zy5ex}z6aK=>`m7?x7QUFV7Cc+Y4Q?_F@!PDbI_V^Zc0~WM`<%-ame8x4|tcsv2kkG ze)=TwvR(uKj)!BDM2{Kk-WE|a>Z*Km(G@D{6WajhWuQmaagk-PUdtjEtw2^fl63De zb~nL?yvNvi!JBQ&wO-?vex*{slr@?$`Ghjs;mum%okUMW&do;iZxDL&DcG$tua8fTR1_uLI-U%9}*O2bJbowVmH)VDY|#u1WW?w(pDe z!IP}*S*w;cN_`pd<&}I=hSar>^?Smp%WHPzBj`N(*X;fey>s9BFWzaG;ic}5x+%FY*+!^-yN3l6_8#YH0!PiOP&mrK? zRp@9XGk!uhj{|q^;ym;IN6~Tg+uqm<&bo$wJFvOoo(?aA&5iAgG+uZX+&Z|6vIVd1 zV7zz_YOvVgn0Yn&X~k929i0sTj@ z0n5KDx%QJTSCUCEbE8yBj6o$2_Yfjoa^j>XoPi@j9a z48Rq8%fa~h)rYlgdZ!&5?(GA8{B1pNg3A3Y^kNS@XpJ`>VvOf9#&L{sJY$UB+kphe zSnRqafuE6!?wm~8>lkYp`lxRBjDPn0`?2ZKzjPa8DYi{~IJsZvR>Ge!uPyYsW`s@ND9GN- zdXzCM#_c_3cQa=7w_Y5tYWBU^fBM!>_pKkWBYl(g zxrX}x_;^Fzll1ASrnn)`(5IKCG}WD8&)G?z-hDjs<|C7Qu<~i1tZzqW8|{CZKD~SU za@P38efcx$^0F@zBvw6uZ!pJ|K^M^EJlVHvdaJi*l;D3+RsnIc>z|Ja^lO zaJSguYR^D^hpZMpWT(6vAnQ@|97R@JNP2lQK>h{Wi7rLXHt{Z>taf4Fvf7;@tJPzH zT+IRfYhNn%7QXyWKpS#0oeo*;v4yJX!(7#LbObUUeKv3LOnk_@;phnNYA=MZN2bui z`8?@;{8MNU{{$};kjYkoS2X=s`Tw6xR$x@dy|1z6{C(~K<{`jep?8N&GsqV2aQcUR zAT3wTQ07~dBkNmqAC5!oa<+E}{rEY*9LqU-qAGJE-~MG)LU;RwQ0ESdW5#CNbvriO zo}1iKh0WHgFuqW}kR7>V+nswpZ9|ZaWmHTNIcTyaIZJfq+lGjIB&kzm9c_#ycg`7S z&LVZ4Lp;sc|EjknMQ&yOZ@sOgF@tZL>K^|~%7~*a63&Q4h6~$vlrwUB`93Yft>g?m z6MrFZ&N|9%b(wq}qprtRXk@3@>8*|u+Y9~KcPyTAY%!FuXK^>S?CrQsNkXUg3&TD? zw0r58?REFASkhQz={Ews?)l4BWHfq?Sq>JgxVEta`OsCCh#Jp6>%pljuI(0Ez{3c) zSJ{kVgjMmXW_dqn;etK~^z<7MXu*cY`~4aL7q`eIM}Uqpeb%Wm0} z6o<}W%NO2;h|#PC^zj!mPm96jj?v0|89eY0Ximv1GRgD9ZO(19>F2cLW9sfkh9z~j z*%)u?Yvr!8$gs41&JpOQK7p<-@+=u+8EYAL8UJSXVwvwxM==Ls*cCu_)y}vVAt(26 zR>WH~W;tsxpEa0;{_zn9cAMbc+HcUd0UA8ew?(%tyKGt)@ZUbepBCEnvT6JC_5wIX z?58^5?>*3^mrWbxv#F+SOiEg4&da8aRb`d5UEq30yLa3eV5of_I{3)pJgur^>P-+Ta_R?cd{ceQr* zlzMbP37=4@gNz&LPct%q9o|34m`<3pvIa$Y0UbL1@1ZCozmI=Xrdl*jpUajo=9WeI zW+~~EQiB8ezfM+g68k8puz4spQpD~d_M)+sV)cl=vdGqP?bT+{=H_P-6*cT|PevqboG zVp7TXW7eV2x4YKZjC$zUOzs-gqodn~esMiIV)eMc_4oH@{O@UNP@hNf=7Q9Hh;-8D zpOOD!x{Lk3%AamO>7?I3qx_z9L466OO9>90Jk-dOy69(q2r%ehhev?( zJn-i3B<`o;R{v+=;r+mG32rqm_3pH%g$Vto_Xm5Z%iIr+3BdQ&Ql=m0`0z$P_%jdu zxvyT^VwJi-8ET||O}b{mX}=t9Bz#S}z&LE?PETMQzCoV8>Sj;YYZ@D}Wv#5`+yqs& z1-J0LZ;hAWW-|9dIvYKBx328+Yy7iUFJ0r@JGx|zf9wnQ;9+T-j76aB#q5EBwhyH3 zXPCq4kg(a%*tO2e9DTQqcXXiD!m{tobe+920h?5OoI{^Qo_e-|dGc0?&TBvRr~d3y z0~qgtjQb#X67K00gm5QZY|w^el|1&>x)okkiCT+Nhw#=eUH?n8Wbkjy-Kjm(% zn&{qwecWwTiQT&Hox_cvkT(@rv~!oe4L$bV@J(B)EC)qCA$Qp)GA^=ba$mg@_|o@a zQ&d8y`ceK_<#eqa60#}DO54S*=k|;hZqM*X-7{%x#qD|KN!|=XZ`|F9-Sm8Tv-}pP z@!ab^ovhu|D|P6(NW4LxfsTYu7d+F*Ke@BJ8F&_2?nS~Uak+O9qthGH%=nX!AJ+F{ z>Gyu{tgJ8W5V5AHr#MUB4LM4ALQ8BR{~rXN`7iiO#!~dd%CW&I<6L4L5-I23368%f z$+_!8q?L9&PreH5jGtK$;^qw*^YqQ3?s{}w+j+O++%K@#i%sCS`okyE2587Rn}@G( zPd3XBZ?HLP#-Cn&7S7U;&K>^B+G9^8khL-Kt(c9s*}lc;mh z*Y8?}%s3AHnqcc+lVXc5-IDRB+Y0U8VH=RO8(P$py>27->APc_j|^^dH6w!ey~8Oa;r&(Qu8*8Vk7w#djd-ULpMP*FE7$nKuCAg%kR1?k-*2dJoT zE*RNe8YMb6aW$K0Umo;l3c480(ES2eQbz>kWjXD2kv4l|(Si}(axTF8Cp7|-OOO+8 z?}f89Y)`(r=S=eGD?UB>dnixzps#~U!J(8=Y8v!{Czon^Hld2!`WYfr_r?i*n;cf zv*N_Q+o741tHn_o=m2$g?53KQ%IxYGQ zqkMGGV4W`B^D1RZ+h5jhtVjANd-o3QMtj~C`E9{hzeV>2d+llHT{$xonqrH;j7|PB z{|EOqXx~)PLwxKyqnL22Z(r54cQd#~`Y5#bll&7rA@C=-}=`Fk>cp^c^8+4Z9^NtI)2-gJ*Us@ZhNrd}9i3;%)ok##}$1TTLDrOFKGS zs%#o~A<=W3`4HSAV=i<%@BB4v#FmZBRXS}st;ZR=0Jz26iH;-h`_{3>9zX5I{LlF} zx*B`3c(>G_=ct57`oj7A|CfZv_`-_`|F49{`@(Aozc1lyec`Qy4@vkNzVKec?@Bn| z7d}e(ZNgI50NNyNFQn~4|2@t432v;#HR0KX-aifeOPXANniYg)3}n7P_5Yj5KcT-L zAbei`<9&Mnzij>qt}i8=$$JjTtQ&!ybmC>rrfN6AGZa^k|8m}tz%O;=%lH>(8h6mn zH3LOQHPJI0zaD?~6mPsTk8P9tXo;Ru=0W;!E?(9NxPUTlgI3md&FIJPb)T_uM!Wa* z_+zyVTS*g5`^4QtIl$FU)q16I@?~FtWRIz#-h7)HEV?!o_*Y^-<QebSdd(*8lrrHre{1w`LS#-`psH*fp<_dDF1^*m@if__J; z-jDAhUi$mGb`w0iv^&J}s&*4SFXNijC$RLQKkk>>jrDl68{?_hZnWp;+D$-C(6`M} z@P$$HGH(e~*`Uwt&_WJqq6lcCNNA*f&}aRj&uq|Vq2P@W@P-YZ*{bb}`gp%kPc3!C z1HbitnK0)UbLS%4yp}JTk8}+(R(5Dbq05&_7gTgZ`P(FG$v2+yjKPicX(oM}K_AiU zx@S6cUmi8h(6!l7b<8WKhaoEgWe*R;d+Z4GW`q*cX z(~CWq>F5!OZR~pJtq$a#$m&az(YKJd2da^8h@80ux~*z;a@}sm+GdF+1ZbudTWqPceINP@R|l=%9(YX7(EM#Wec3=>f%)%{PTb|V zVjo8QLgR@o*m~@jNq^G8zrp?g-$uAz6Vv?GSFhH2=V`^a-X#bq`!27dY`2seSB zDZAnoA8jLJ(guA4Zz1$eZQ7k;Yclh?IB!SmV^ zzP>#}I_a11f1D>?!&#iCf_PtCtmjZq{us|o_%&YDFdy$RGlT}>o|V5`u{r)FJaOlh zMjGp0^y0qs@9L|JMc@#bvn=YA`8oe_fm_COp|pYae@nL|*|R{qiJseWKZln3+B*&f zVF$&2%z5OAXTXob!}4~W#*gQquVi1;c+Y2Pf4G0sDl z>a?5z3{>X$$3;e&csihgn53H|_oTH*a8z9BkSEZ&{H zu)8hfZbP+uKlhvLerQAm-wr=+)%XM47oVFDYKYhVWS3AQzSZH^t>3io;CwabrMJJ~ z@UVT`7ryDhe+OJVUHs{f;@O8h%(*_c4?CUwt7?F$??c@=KE(97kCD5i zC(97`%)SSrk)YLv@oo>iY3i)Y6I5#=e zTnv1wrw0Cbad?KO<`{IhuAF#-%!zj9PEs6vhWZTO{fToQsp!+tG-LZa`c={H>E&M( zUz~s}vCh|Xe*Y!h@u;&>+Xe*_NDHnT6fEr&IGW?BSX0cq{v2qGx=(=X=INfA7W{%n zkF=6fgAVOn3eLm_u#8pbQ@&v!_Xt$RhbnkJR_PBly_67=|GXKaUC^y>`G_!d8qVt) zs(s++BbpX4j!n?&VQBTBC)mxU)g0kXa8bLQ2xd1L|8d^pa$m;ZS#^r5j9FZXqK zXtlecnN0f;=DtHutPRkbX!Hay(eck?@}xV6hDE1`frow4AHP1f-YXrMS9B=4{g|@C zeWl?we1q{--jjS^^ZX>^Ii5Jl`04ETdv^ZS-)rYz^UvSArmA>o=n~%F2A%G`BCslf zF>A=}6gocGifn{t4>4cWpR1wUHt;06)E?-Y;6Trhxwv@$MQ1Ln?EmsC9^$Q0`6T=A zw9T2rNw%+tgO|PE+BeG5e&J~gWyN!clR7v=HiHKWM^Tx6NNgVSRXAA(tf-y4Gfx*C zT+ckM@};#pbM$d%j@B^##Pk1ZUz6-!W{#?#N%o`kukIdqj{Z$^w2yrgG~uk{Np_IF z3Z^~r7WDYz%u&hcQ975OPxM*1__-;cX8a@#obfILrg<*AH^nTCCRhDQEU#+Y3AGZ^-RVthyfP9zD@PdZCN-Mkh%^&q~HOr&Cb!#0mB& zTd40CG6gnqBU7A0=W0WyaCL%rqWi7Cm`8jA_U0AK{9a_poA?dq=kAdWc^A3j*Zf|g zZ3{@N(gHR4q~Q;J1zqTqGp2p)Ud(H)ce%hEnIg;X>caY}??$G`wBub^hG4lCEeU0a;LInjKc=*l+T40bU(VzS-omxmJIvTn0!~>SP|WuH{-wO;@}d$ zBM<)f7M;O>Cy5Se--rB{-0^`@+W!}&%}Z`HfGl%WQ%+=Z(B1_!;RV z`!B(to3m(1%QBaFmapNyh`F38s{xN+`gaWpTD_L7d3=pS^S`D|k|%N)^VZ_t1z}%W z|Ae0ODKDv@VRK=J;-ZSE5Cv;KuQCx(vS4$nUb&;JWNf3P>W;x(tf zO6oIf3HsqN=w5dnsjt`m&dFa(zU10ttVcSJRSDmy$A(~JJy%wF%;hCrNgdvmZVShf zZN%GtYVxz}xR`vc58`cinml3?KvU6oC5wo+m6*H?yCpV{aa&5>E##Ffwz%W%%7e%Z zlF_gQrehnbFmg*T_@ZPM|40koIQC$3ZhR=8_risi>=9dYKG-!9o*@0e8fi1ybS`*1uwC>0*`ul)n_i3_FpiEZ$q6D}W$z=w3EL2+qX;X|kR z?yK+=gRcU|-dn{PZ-Wb0-}&iWWH0oY$Sf;T1K%<8Z|kMOH=ZV56>-G-F@Ftg$k5>L zUpT{)GZ`0xFI_ScylIT||3>^uFTuu;#E<)1t3Tw&ndzbCd%KudTflv1Yg-#~w;$iV zy`8K}^xhl9Wvqyg0H!{-VY6x}wj%cr!jF$%4nOT9Dpt=9XZ!dT(0v@G(ND(s@x*C+#JeDj(rY! z&zWPXtht<>1aF55ojUJw%iip^vwDD4{S?2M{EorPYKIa>Nbw$$(}M$^ar$2D)@$I^ z>A~||`1gQ|uNwH2j9|749|yd4lv_`0Mli#Lj}{!>WXeCC8BB8F!+{HD$A_a2ir{dUeR*OcU1Lte|%yySiw zc#Y*))agVZ*RiEAy(wXjRU-EXSK)yJ2B8%2W8V53;dO6*`buB-Lrwc2+YBe zbDnuAun;iesCHl(I<|wK+M#75TNa?RfTKf&(!nzA><&C-8a!?8z*DCE6n#;jZg%@r zML(SNBh!A?^a&kc z+n{~-U7QDMIdmm^&V7jW?##yt9e7W(uZZFu9ZKI<+tnWV`Ncs?(p6;ZiN&9Ae?D;8 zVMGhn%oXA8*jH#F$$qw@tq$(;JKDNGbyje@?X&F9DP&g6>sE!>x?&$(LKoO+)=TRxQx4X2Zf_sX+KN*;Zgt3TI4Hh&(NT!%+V2i z)&rBrXIZQL+17r1X|6brpCbbzhg>;%`*`q|@ZOaX{|8qR zhb^ILr?=?j24kb3?wOYbm(wP2TrFPatWO$m$KUZjY+=&Fc2VZ^!a~|ShPIDJzt~f? z!IlpD^>;ck7H{6}=x?v^UVs?P$bu%`ReQPQy)@*#bo7%9>dBp5{k<`5{yJA%h`XCC#fxw)!dGcGmcGdl6%-emoA$ zSywXbpSiGyfoVM&j?B}C-$`Fu{}S1gS?`h;WuoV$+jmp;80(_!&xgQWc=5Ki$>=W9 zH-Ai?;15VH(^-}aowFsZl`E{)J{|Vti};S(EZUpRv(~v=O`2*Kkw*O%#5Yeb`~f(* z3EWHvN7KO7jp%YWpvz7D5jZru+)3me*_Fb`D;jU>xjO**%Po?t&t6(0sGtx zaI%qh4_H)susc}Jtl)QTr4z`Dv6Zfv6+S0ualX)VOXuLC{n!#(+K0R{qkln8oYiA} zc}iOa{i%Y#gr6T53RPZ;Usqb|G31`=8Q(2fU!LB!t@PpoaIrO+7-U2GSk-OF93d6}6?xmttql9W+_LrXm;`hO+0w%Bm8dN-w(s8Y@SDMEYae?oecaExx88UE zO4A13`#Eh2|EUPOZ(x;wQPSEldPP3vtZuBq<;zFBR-S~MvU)^4XDKRkwf^_91EN{R zM17U5`*r%;03X}I-7FK>-)zlBW}9Ty7+EEkc2RGC#=5c4GPb9H>gLY$d~mP!tIp}v zskAyodsrKy^>H`Mlvd_KGv7Lft?49X+C)G1WfzAFIlJUZNz4GhIhzBKtZwgRp1>b` zJ$3ox{QI*#{+AoznUawjauY-T05;}9Wowp|239F%wPYf5_Gb`xSK>d%olI*)2jO|p zKXlMg*)0^lhv!LwniIKw3&PmY<>RNbB7VvoFz+pB4j?J*JKJc8oppgx0;=$veTAhKT_>4O*V&8x+i755PZX*jkOL z>Ny0SG^RIER_&^TJ|fVP=;>3pPSJ?YMXR3ZyW+u4Irs(dU(Nd^;6(Gv&%CN)%%e1* zv2Q`ws`WiQVk6^!XqMB59frU3u_w~Un~t_?UBDlCaL~`1sk1QhQytA&$qsnkShH7L z`(^u-rL0%1$t%WO&^LIDvWAZ01Ci6OM)mwY_QljIJ^v&!t@1D8f6Ic=p;O@J_pEzu zPdamK5b&J$+tE{;dFDS}sPm35Ykua)hKg^Xv3=I12JE+x%qVI5rWcJoQaDU(!FAu%9zy*b%_}uqu4oob{n^a5LrCv;M05@*Hc* zJvo-^`#h0vP2ij3`F0U)7{?lLRfskj{%`7aZ5yM02(RSKrbYXUi6wZ68&B{}@Sfe% zI#OwP5;1@Dzv$AYF|;c`X01Zbc*n{a7F^=`tCZ6w`7}O9sxdp8hch0sQG8jzK35m$ zJ@fqjYj4=he8b0~XtPxln539S=zzs$KX=; zwFg=);=O>OW52zYGO_t@D4+KOysz^Tt5hH5xwFK4C(M46c+9ZEv|x_U`F=k18s=U# z)4yZGg10^4(Bq@Ln@^b>WJc9h#(z_$?DF7EZkeSj!}~eVgUZ~)f7Kgc4@z)9>7C5U z>@sMJGkjTmd*_Z&Qzwn+Z^Av~mhmIt1NKEWPd9CPJbVA78d zF;^w`T+6#{@MpTNbZp?i~f2FS-||4KI^l#oJx{S(ZZ(a=;Kd8j}L%L)j5QB&+bRI zz8TXFwNWyCY&+DpPQawAJn!~5h4wr921)i~ZeN{#rWjsoWP0%6rJUf=_HU-1nLSwd z+_o!jd2KxXIIG=rXt(zGrSE^T16$}+V+(y}OxNLzL#kcPI3k11Z-y`Cz}H3ppFV$v z{uR4t6tq_lEQkJ&g2q1LSv>T2q~Zl{f21>Y+Nahz%v7U4tN*7L{%gLFPb^H;QB56( z;a~VooAKRi>Q1$P?|$P2lP1}pipT91k4u2Rbb-$#!f(35ckr`Y*q!@|eB5#FoJ(tk z*Xg{TXfl@f(k7f7CmkOCcYwEJ;@jd zRs~FB_Ab0|8^4=MYmmLuOn$b%Pm@nPt91KX?XP4R**n#KoOiRWtfpJPN%npuyTku( z5_PM6qWz!qEF6`aG|8SrD%sn~&qDV8_jlxmx#z0Em(gE7`r*7zq2<)8eT*jj%Nq0I zLq+D<9}kBoPvuq!8DvmjI zGce)dZ_u3P>@d?0#q3KqbM`8epK8DB=IdSQ?3x#&$TRy4r=3069B4a|>qGxvs{Pj$ zmgZ%-wWaGRyvznG;#|6|_w}dRXq@;Bdx6{8D!gEnzLmKQuN{$}>EEir#t8 zedi+HQU4nL$LZf-{!52fnLM6NKeuVVXWEx_^c7o`+t*yTuN&OH3^jd~-O1^z2?$UXS0UBzS_aXm;A47YRAX)t1?i| zx5z)*>tVh9|6TsMp6|B%6uTi6-jD`wNQXCMz#B5*4a9HQ-5I-uvp0Mk`vvi&jNUFj zL2w0h0L}?X#wd`^ z508W2TcmGr7PZdjG5214`5fb&eyZkNB1B z(Tl&XrS25!)*8~nJu1;UKcvpO^3l$HH}BS6Q-pmedY^{w8w20($o^n~Z{a9RpTgjY zJ(1H(DBExLxxpjUv5>v++ub!v_q6CNO9Ai3?(+n&CvBnZf70$W+Wk#+c#yGAFPsj| zOoMiAgobW_mZl=dUXL8>*fBJpzvPv+N)1bjl_$$Xl zYt9+!HfT_1N{G>eF52Or&~YyD?pfHJIlHO7({-z$Ibt@HFcya9;;n5B^jGT=hvTX> zR+`h|(;iZ-P0v6xviUT)@KcnB_CqP0Z|w_>s9x>Ik{ceq@z)E~I|R zHnIgtU#KtF+RZ+NYdg3TSfzZNfe8na-=+au^Bs+4rY)IJYx*GKQh#k6Zx~_j4tR_? zpfRLlhQ5dgmt4!9P?_3ZG&tztUf(g~``~@QO|e$5i(hzj1Gp>oSxqIR>wNbfy*07$ zWOJT$bmWRu?8koWq{OaZe|ZY_xEeF}m-Gncd{DxfHtg%vDZRcQvA_ogHUlMtLf?jPJJ1>dKwI6>Ah{xC5UXwZU=t**cZg&*Y)j7`v&|#{;pNs$ek8d(3|WAA<}|jslmD4g-08yKOY~-yE`vDYT%Ei1m!asrO7=R4j#YQ zgdTux61p`uy#JUcC!$N?R}`+~Ow6Pd2WQv;MgPLtBQDPVO#R2>(Ji1;<^3NxaOg6n zgD&3!=Fp|1TZk^DGhCkT;BB;vxBqc@n(BWU940XbOD?{Hbu7iMVlU<_ntjK5Av37` zp?<_NIG1!jZBf~uTztVW@hivJ99qGey9Vt%z2k4@9Mzq znK}OhU?*LE-w6NKoLB5)&1>CzD&5)e2>7#P*=s3}&xzspS|3ziI!Z@5`1x$gjia32 z^X54E_bL8kgTbe0^ua3pN%JWq8}a(`r$>Yq2Uc}qPTG0c&GBQ{==1KcESH@-$==31 z6K{~e!Iy2+egCBh`$4WHgrb*C1y<1?7=4GOaOGDb}2rO>Qejr#kBv$ zf7AZ7X#1_D#(uC*?Qd86uR2&jJ)zvoLLvNk%vfPlSnqpsMCh!xKSh5u5hK{kDc}1|9e2vKp`mb`%7<%kImAs4o@>`|k zzec~+*8g$gV_iJNj_C_A{T}wUenabSzc-R6KY2gC&VGEI8}MBcuN_Z+3sRP?@t^UWaeMdNMyrM*Y|dFgvcpX07zywmI{G2^|= z^s$d!N}po+m1C1&-k-&@#=1BWJyg4V>u124p@T+z*-N-5A)7IiUt6u@jyJ9_a)`&r7!s9TWCF7ldHp_i!*jyM(m(~aJ-ZL&AdV}}GgH%@j zvp$bo)*birD7!;)fmJ>^0h&J-nLu@kH@rt)NvY=bl?M};-v!L?uZ{ON(mD1!=Ie1Y z-p>3^h#l|rQQZCR%>BP?Ju}C`S>G~W> zzm3LcYD2e>&!KAv>(DK3o{1d{zdhO9QsRuvSv`aOp3TVUPy2}1#BVA;<(Hy+)S+Kh z{S#dm`?T&(_bZNL!ofBE(_;?Gwz6?$(Mi_WDQfq7w6_FZAelY7jy-7PZ0SyaB(Dm8 zWcl#v+=uEH-v5<9aMTx=vm>D`8^Md=Z)tWl^+-1VJuvA&s{17uwhNf*)tRko?B2(i z-!n-|&;k4@)0Zcp1LQAy&&0EnO-N;zLn}%vI4di=ckZGTqa##d^KNFZM&~%r=m|QL ztvyxQsD1WLt{uFR@2>A=RWr{N>-`-+>%I0_pQhd-ejd-$Yocp?pP04YO0rh(7!x>B zQC^mLYHIg_KGF4Fvi>MnHl5@TML1tvI~92pouHMy$5Zo?3yu|c4IQ7CSwQ^<@~xdm z^R3#W;X94Iz15Ks*auze$OsE*pYc)g)5cdk!CPN);CBO;od??hZBMt)cm0{~1TGm! zv8jH>e3E~yk;z@zVHPmW-NE!t@OsvV^LYLqaM8hs%mv8;CXYFx_2?>-hy5=$58ghT zyiq1Et3y6$=083X!35+3*#&jZq8jXVO6AT&1)% zM=Aa3s}w#IM>xhvfS)h(RekYcd&#uj8tZCCp+7kA4>hE9V@MC{B|MD#6 zFN{5TM<44h*(38g!xpt8wWV1h>|~s~9TmKF19w$X?wjhZy1!Gg>h|}$GBN8%eTtu? zY%S}QY!AO+JjGjH0%rE0SYMuYVKy-FQk>T#H+qTp%CAUv5BO3Bb_Kov1Z5-eo@^_l zDVO$@aPGbZ{+FSB0P1>_eC4$;kG+mf#I)b%k^I9N{WB}L%e9GIt#*O)a;2=jOO&!k z+)m2c#{SB`w}+IJ9yENe4*%r}Z%R`ocO*oZ8w*R5){1ZE^S>EgxP!N`Rw~{^)Q|3b zY{tUH9ehvnSXn2>22%W@1OFlL9ALGKoz9z2$qpR7MPtg?_9M;C<$g+dY(P5BhymDO zJ$5GVNEVvO9_Xm_z)=r8R^|5g5U=&v>%6?fo#EL%xStU_K&B^QZdA{|6dgYsd&@43 zJN@s$_(sRS1AnVM!G7RxD|0fPGQUOUSGt2#^ZJWkiVrHf?PJTASN8y%xZ7#d`a}ym zYby7lIkM43Zd+@+u#WIf3u{T;^d#wo?-*N5v2{A~X_wHc!-=7~Ss4Yj^Q_Zr`Bp>j zn9!-`@%5-AUe~N_NB=FyW&nQm-T!rMOR3Z!0dL~V+BfJ6U#vI&5bQI!d^Q2t89wXm zA&Ad5i`PMKOGw3MC0kqvTs$j!CWJd#jU7!gp~H*GgYLv{4*_Fe?=IoCB&lM}Uax<7 zDD{ao$C>Mrg(dqo;@gylW2uMOujdKV%Cw}wO71waViG#de5>oqPv;c;l)#yq&xJ~AiKwf zU4o6I6Mmi${K)G$GM=*L{|EjcGC=Zr+3i##kY zk6lnaXA<}jJ&0H2F#j|L!gpP6QbFWS=G=6b1|=uWupFD{EEn&UT|*j!`dNLPxp&VQ ze9PYc*{>b`b?I3+zXF_e;M}!~763aN=gGnyINunHR~P5qln2h&#^$+m&XMWCx#idp zUZd<@uXDcEpSO2S!#}UW=6)9L(KR&3&c^+M7~CJGKOJ){pqMks%&}zqH(T1Dl5F=A zjRkW__Ao!|eZIS9cJNH}$`<&?CE!=*N24~e7uIPGupTfDccWh(hjuitHLmitc>gD! z)n#4TGa%oe5?FbHIouBau1DW2?HxE;NBT0hERE-f)Mad@(7n!D>O4a;y!1aE%VxgO z96O%$>ppiJ_fWUPL;Ki&0Pkmy;dWqWkD=ssjp3l!adOA-S>-W?{bTdoG1U4V)f?5< zbB*o@zh(|y#d!K%-Eq!89oR6M_MJ-U>yMNoPZS$?y8Rh?<}LhY zO5ZG(zS)yKmR{_&^k&Z`iM`T(Tej;y26V>~ ze8_g)L<|q)M9J4P&?&j6V#;20%KSuYo3n2h)vL0gE!_d0iL9B5oSA0l@lHXc{i#Xx zsfxO^m%D}fKBdnV=j2pp8~V%p{C>?ZTz<>18XmgkS7FP~Spx2WVQ(S&-3`Rl(RWMO zLn}shTVtUcE`HObFD~M@Z{*5X?7C4Lv(#UFvnJv zZg+QKIR?ggUawWn-ek3OQJv>aK_`}O)Rl7DH#>$Np}jNZHK0qR;43K^CC<&4FQ>|! zdzPG7Ci&yb+>U#qfBQ?uk+Kn`jQ0tp%-g?{I{J)s?CTi6DU4$=<2jjey_UVR?}r#? zvv*cx^y6vEQ;@H#7XR7cV+nF&iQ-406IU)yH9E2OVLIeRv&KvJ%P{w>K7%bwIH~2k zlAYJ%PjYg4&olDUI9H#=H}Moa{NyXx&WZDZJcjK&(b&$z#LjBQr^mIMZ#~;?-f{@L zIq?>D3J2KDQ(|=K3wv0rKSGzT!fu}F=+VUCl-<0$?B=dcoM}H<=Ct!w+9@4(C;X_k z*%SQ5W(U3%xbQwxaCHBfuCJC2Ts-VlqU0U?Pnfrodo&ND^4c$eiPvavLGTFjb<}6= zK{L;Z36kTixtGIBL=QcI8~Yh!-;HO@p}pw7sp!7b(03zK3qz-%57no-v>&ha{VumY zM_x}i>-vv?9s96TNM~GX%O4!!*WOlBTu8RJL%EXAIL`-tru(v%pYSFv7wsYM9SK9< z9>)7vyuRc3Vc$r<>q?Cdy>-k5ie;2+=fB|aA^D4`uA9J#;KJ9b+ee37nS4d`Y=Oah z`uXPU_;twSqQ~RVXjJFcd*W4skEbr>NteIM)XP~frO@^m^FPH7m^9h`4(SBx-Mi)Y zkpVqtLf2W)cOU4S^BD{K;u97RpNdDOa&$?Z*Jwt!l`pL9Vxm!a)jQFc08dYM&fUIV zIXVi+fwrh=I2#ajnh?Rq)&d@7&QoEadFb%a2j=G5(^= zt3}vRg|9udM?82_rS|coWiriKd-aLc_$IkowcH2jwN9Vb@Tb(~wTTIty8o+NJ? zztO%Y*BpdjEI2R>ns@GN)4p6vcgu`lw7km2=655%9sbjWvPDZ)XC{oadVfy8cZ?1k znXsfRb5y_n^LD{M8_@Am;ZJYOIA6SRt8~frzNEGy=IY_s$AqeO3=S!7lD&CI=#BD} z?K=unx1+E8s{iONt@7d9{Q9M#7hfM5`i#BBUzewEf1xmAd)v??{Bp4m?&uQI-c7hN zG1RsX{z{yJ#@$_8n_ll0`r`HOq0diQHJ@?DzyN>g_?=|D+G20;I@^Jl0GGUV9A055kF`3@wS(S5o@8~!+@H%jwsDq^F*&p6cpWhD zMn^_-;OFsQ^1dUZIpZt-cDZ20c-8y$%tz@hmjDy}?`Hlu`VM)}=2G%5GI<&H8?kxN z?=z>JoY974qtwk|JpZbhX%tBRIGtNq8E)mc#PNwhOhc6M(kye z(%2#KrBu8cNB1QL4D(F*5S=D~le(u|{+V6?EoIv8gF}Z0B-!1-750w#iXZ=td~;Kt z@vF#$&xvl2mj^=f?=sKeUiL~otKO5(>brgHR;A#o#r#jPkCArc{kA(_V_tfgpI+uE zG0i5$!6Q2%%XxzBKL38nRFdgFNlFYI@)2(w9;x#JHPDjb;j;^Z&sI1-gyz4$fADGk zYprtD*8V}98lTd7YX09aAh?cii+(;Lmf`HI(f1j`VntjynV^sVFBj2&UJMo2G)9FV0E*WuKb+1(K`|XFZ+mNaH0RP$=x2? za@QlCZWA8e^2p?Fn?iTZ<9WF!v40{l4JN#2T|1+fHArz*3hvG9J;!TZBD<;I;yy6$ zO%20;+xrHNZXjLdVgH2RUT>1^f995ze*CR!hj+n`l{QT29-Ih{zL<)CDsdcc!H;kh zwhkX>0P!Q5zl*kDdzvpit@C03f-d^g_6ZkqZzcNMJo*^$5I2y1-a$X3?VjQ3Ht%8D z-Gz3Sc_!S~$a{Ct|If3^Cex=b+_Tev9ypnY4QJl(=nH$Cwl$Z!a_M{13@3*8Zq}%V ze_na8px@NxBR;?X;9BgJ<2{M{N{OQoW(?jU-lfh1Rq)(U&b|mdekgM(zcg)aI(AmY zkDtX@)Y9HTz)Kcqm_4W6#s@gXz5v)>7d9r(X}8YA_M%@wp&OQ>qwGCQxk-V@TZp)S)$V@-WMM z$7+fnTFaQ3JgvKfIlJlRgd1KwYKtue{oonGm$UtY-Z|qhFCt{=LKJtm! z!d*M^yDCS2ob_A0(~T!F4!-V|*{3qJxt6});I2cz7mU8C4aF{OmtaPoO0lnTVQ&aF z!0d~?V_l6O8gwe(yN7!^wux%;!}PtKGlqKyLg)0OLMgZ@Qpy_QSIT;^_adwM7<{GT z`f-sGo&)@3|9*;fCA$oL_d@87wPO`DmTh^D`1oH&*~j_6{sQWN&irnSwkgc*)n4N7 zU0bl0wQR+#&$ zEhN6)I8RN(fbpTm=aWMXFImKKv$mReeXm%jo0ofPB9D`g+|aNOyJT)=fzE?Ay5|yW z7h0!VvM7^HnN{elCrb|&aF>pJmKrO2gc>VTLk-V!ul@5qL+fAh?7z0kkq2fAztHje zd~m_|{EfcW!}n_$Tao9%|QNApXv(JS$ZVxZFYKrhz-tv6=Qz{q#ic{CM zs*5?Y^E=eVd{FGR8?>hK>}9`Rc9zfhU+?P_6fEk$I0~4NAM_4mc^+-W7xpmUw-e`SV2Vx1p!mf z^6k#DH)C6p9DQ^RZJZH154j%*zIKRyY;?;sP)4$!zW39R^WMjLZ8Nfnc38hiCQ{KOB89C5)##x9TFkI{|ZqK=t8 z$cHD2-kYI!Ge)ulKX08D{=y4VurH*BPAq_y@8LY{4$Ihw8_#< z*Wg6{6Jx;32Kc}OqTg!W1CeI;ilNeHDr=7;!of5DlL`AP?J=b%#f|9MK8!{;ic-W;X5 zoPj3QId1kZ<`ctuipH!3yQ$XZl9$_u(7xVBWk;oc*=1u>@rCZHhdy;x3V-ZE`hT)# zl(PgwJ$Q7&;>giEF6C~SoSjEY7kQ50 z12W}j=sNOu8}CVYyZ@rDYZKBJvs+}6P@qY zIO%(rxZ@Ij^T7l8zkKy*xa{qt`SEo}8_<>f)OYr~$-JAwKHjVO*497Q=LR!~_eoqr z<8NBT`P^;j4KtAiGgGYI)2G+0EtwuLu`GAVrW}t9tvE59y(q~I+$&$3TlAdAbHAbU zbe^MRfSxa#w*kA>%o$eDJ1VeJdf5Zduzyah%KLp^8bGBwA8s7(N+|oM zCo#k~%IYnhb~N?NH_qH4iQYZ^e%3aH85o8?PdLmmf9;?$zO( z(57?D+V_vA`3^L$IOoDyY#r%t9hdW-c^>$qy!%-bU7?>Of1Rm|rr(G|vGZ}$dIKkZ5W zxB7T_SMJA(NB?%>qbr7BD|XB19*gWNW}dre>gQgMT*a9#)=REJ85rC}KNK(4@>|

    >n)t=?Iklzjb6caF+-+KB)jGF2f;6;y+>b))Hftm}65m=N>Oh4iQ{(-Zx zFFq8gd5l!~A><5v0*2U+6BE_by}#enrGHhNHT0#mftq%YcfS1U+I!&xf?m6IVW8&a zdFK!R34f0%>zw{G$B1oS-u2(= zZR$@8e0MWEek*j0Y(BXmadgcn+VA%)%M2qAg$LYRptk2zHlb66?#7yrY_+CoD)AS* ziB094HF`OZ?|XYS1{zFwNN^v2_Ulv}qQ)wDkB3zag(29a=JJ17-7x$?w#Y zVr$Mvzok6YBb{2jBI_IS3O^(79(Wgd^#jL+qVGSQupzm)gQy6yp!F#43qTL1PvU-p7i`e(* zZ?x|6fAHR&ZaLDA|3x|VDfg51DLYEJtGe_ue#Dt9y)+cHyUWk5&X;IQ-+R*4z5o6c z{F44{+Kp`<`mY;>PBg&LDIRspysI(;D03*L40L*EC~~V?W}nK?pG`i_fkf-+aaHJk zH-ESC(Fr!qipif!9piLX-I2{-QyJ#0@Lq*Xv?-Ue@Cef;=(z>@m5<}?Zuu8go;k9q zFs4rET4io_%RHwt%%@G|Zkfl&h3Y2;BG~}!MJbCg-sW1pkUld?!vAY44RRSc*uoaD;RQ#=Ggzau&V{@ z0Sp@cz=e$yED;#=`mPHbBUn7+3;y?Fi;$jIp%lJTq!iloD;-RJ9w{_dp)`wprAg!~ z?Lj{F2l*XWD2*qdw6Q!PShY^)K{^Ld&hpkAhCdy?-CHw}b$6!p&!L{v16U)PhI=AM zUMdVVjq*gAmRiM=1MQF=eOno6c{BC-Ob&+Q7cd zX!c|pvC|yh?X7A2legyMS9Xr9?=fXd{o_5i)c3fKv~1*|q~1Ii^W0<0m!-t^EB4lW zdBRik`J_PP^B$ha=ZSo?oOSeZ=Ke$I!EaipzsEN}-)BWW&!fzrNQaUR&15g$!(P0n zW&rK_Y+Rqvk#XmQzBuHmX*}$y=|@|>K$mW0eP~3V4ll$$)C?XF*S;Uz0F{Ua^zY=|okbLWGRPc{uS`uw(^IW+a&lZ@FzRwU5fs;-^YDfmXE z1OJoYTlZLΠpJhxWk45$8-w4LOV-$8C{oO7YV-|&t1@?+5 zC*7NOIYT-dnDl^p{2`*SrNBahEpTAv{NJP>Fh}pkp2{ZnB@3M zVmcZa^D_?F;^(HE3uCUH2dvDLo9dKf-i`&f(3Epwx1-ww`zbKZ+c;83N9SA6`cm@l zR37UovgMh)ae>Gt@=8ozAG;+sk2$f2yjx6OmVG!jk9qPjdDBc@x*d+qWA4O(uVV5d zW%e=NPVp zPvv=n){)0Y$?Zp3AIx7%)Pbp(~Q0eeMWWQ48!Jm{4 z2t@v%l=-ty={(Nuk|tp@k#D>9RKE2dsol;!QnB|w$j3vvlJ+lkmz4Cq^}EyGKws)O z|J1jbea5cV5bi?j{Wolhuk%~O?`Qn3(j0qKa5*)vd?$)+fz*Wi0nJXJu z#|k`wm5ir7z7uT-bcU%$l5T zPjg{+s6J%4W^{Y`3sou21{b8w;Hp?UKR@hG5w&9q?`8K49C7yB`qHp4 zaQM10_l1AOq9C2zhg5RKl`0E4jIJr*qfs)UoL4h3M&H(9^jGDEG}f zM1#>->PGH_-#B|+srDhp^6#$A_dM#8-s;Gc(K2bad{z}x@t|8SlX9}*7USG-f^ zZMRG>%E*qS{`@%we@1>!@C#!%NU$E5+yov&19J-cGyenf8!7MErsvUhyGCyC1xGZ} zE_=YmAp`ql*GQ#f8pvBdY)cpJbSQ-PR1*7Gvc+a!>UMi~>h=MYnT^a5hc8GIxNjJk zT%faN1;BGYhc69;FVU9grQeoT@##!|k*l;`f{GG}eMfOSZ7M85hBiO!kjFlRW#JN%u-pIKuH znUCcaXV#am`H`dhKyO+{<@fUg@Lz{~Yu1*fgTOC)#WYPH|y*z)rwJxc|?rF|-Rl>bJYO`Pt;l zzwl(9!(ZcFcm{CEuP1T^PqVdF=o?AE)sNPpR%APNIN8&m#r}FPziIq(WQ!GFW{le? zEB;!D9b9c{)qkt!k=^onfG&=?{|ogWy8crCXL6Q9*QKeGoJ<&(^7_PQT-_0Ij& zskx5~Z~%R+rU=`yV_%4t$7hUru-GlHHmIE1A)g}VL1qg&!`@75sO%1gH?<8423?=u z8p@0J>@>0NQtS};cixRVBELlT_E%J#%ieyvRqf{-Z<4u#mY8s?H_$-Hi*0#Kt@ME; z`#$ES#%>dJYP~JvJ=xupDJ%Fdfs5Z;oSPQiTG-DYpq-L=U*WsTt4(HH!QTqXzG}wJ zs&VR3JWFIg>e6|B*4KApV^jNyYuZ4a1)l!Fe9lE$x9Qty_W9Idu{QWw7d+(EKIGrx&vEub$J53F_CyPGrZ_Jt)Lx2THh6BIlvYrC z&>Nh*-r?1^(QeL6RZBllb^WJj0u%q&9*E=*XD>6kgMUjtsBaekCbl7R_IcoB5-?|f zQ|AwzyB9A9CVJlst{lEh9&|XH@`IHJ?eB`sW6mxm?*ip94_^C9UWVD{JfA##w1znP z8}-5G_wY_OaP2RO7RHh`gG)UR=ehon+gARo?0Kr6dA0E?^=H~2lh;jo(E8fgJm!}8 zerJ=X{t^Ea{GJEQ>d*5Vpq_#uSh>$*GY&~Ev2C)@wU zn4f~z2_KSAny@|9{o3W@?_W%rEc-Ra%aI9_?ZYb_+=-V%N4-s7(G?mDoLHNrx(7)# z{;`1(D?}-{sx|+)D_CVrOirD9=6-;3D_E5`I@F)`5?`n$j6WIrAhGzz;0Ns9<5u38 z7#m5%)K5mvNw+7*DL=uWDePXN=Wrz{@Whz8LL4S$ZsdcL09~@ zi5D+9z6X6&8Lcr!mPbC1z=Iza4vgHKWe1^k&GkxPMy7!VA8}#71cqI%M)$z@v8BJp z{)YJLQaAre^0h{Yr{<8i*oCi9{j4X7v8L|_NL6mB;H)_h=soh~tA^}+z`Rckw{~F1 zSyxh!#VK!WgH=kQ)e_PtT;K6K_|9>7qt2knr&0E=6xq*dYZY@tXJXXOTPYW{Ge_gR z>07j?dOCULeODfz>i(~1J%Mj_oN3-E|K`>n!Cl0sKbO9w(RTHz897OF^lGPww2fCe-}-`wu;<6TmBT(_VU)!r1lE$CVO_3x$GuexPVlW*od-#fyyY?%$TNn_AL{oz!r`YnF?e#1-X zKFI#?ALo2CIzSQo%8KDHy+ZcUM*Padw{w>HAim`vMbGYZYptNZa9(#}UiC0%cko;N z+SwiH3=QZE>)o?Eq6Z&5aFo|F`?BV{=poQ5J`DYW4QW;|=SYvXxAWF(EvHSXM>Kq^@@KK2{%fTfhV3)p+$X4|wRVoRnYYdStMT3Zy? zUUG%b3#shw(Xx%bgDUsBnd96E&Ae6H&)DL4hk3b)cc$@OZRD^UAi~0=FsTS>&!i^d5V278h$N2R^?hXuj9`z-tsYg z=@a^|vywU!`E9(iFL>DN#{a1BSwj^k==eaz=hK}Sy1P<$y6IlBbk<$Px#7GKxRKuC z=YGR$u0ii&eXKKZ<*S^^>pi`%dWhjQki9<1{k$joFLLh+JeT#!;#bGKDCr$oSq1-l zg6Dhr-M}xC{)L;@t@Pt(RO1nT6ZTZd$=dPME)-A2k6VZ8) z`QJG^-sE?>GZ@eRi&mSH~u9cjIP9+>bUS#O04+o*EV z_vrU>7xoLmr0X%K9&}+(3x@n~iZ=X}{%U<@9E@$PNGWThpR`2&ity!3#TH>47W4i| z)~RfZYfrUqBaivX%Vh0VJAO*O@^Yp3(T2O+|Hfv<_kZmE-za>>tkDKvdjYcF=p%2bPcB9ohDk~<-&)yj}x$r~K3TxQOXPs|$&kzZm1$l>fl+;y(86Zr(G| z`k;lSJnvUNv^FF5UDm)&j&{|`2R7#KLQiZ{m2CQvS)!` z?U{@x&&1<%X{h8v;FS z>RnAeTDw*%Ee?cMxc?s`&8Lp`a?k#MbuLT%UonvsZ{cdahkTP7XYEgiR?~gXJv>@3 zJ?QVBKIU2dXZ$+@@WH*EcMePAtPk}M^`FE3L?ZDGM+RyvPw(C;W8usB6ufvp?_e{j zX@~ynaxL(XYL5Z;f96{e`ZCIw+7v*Zm#p-A%KCNQX_<2`kYX^YyuQ7cykW%h)VK3} zJBOwIi*KupzTM6l(D21itZ9c=?Cc-7Z$WP-{@yp8uK_+BrqSlj9!Vr_-5YP~JtzwEABYbP?!dT*z@)-GqAS%yyeEV5+&qW4YAfL)Bw6ksRu zb*)(Jz*w87=smUP+wi2ZJelVWz09|wYwmXFQT+XFV9uJGWxwLWey4Kiz0ylxbYVLM z%L4{q3c0Y?^=`Q@G)?O(Jjt&V9<~>~TYFj+O5rz|oOjV$TgiJ`OLwd7rOCju>|YvK zva`POy*2KCv%d1JC*A+Q{igM`0e;gdgC z3VqEZeHmVH5?P?a#A&!`c^B5zlS$~yi~iaXt09d3dkDQ&^Q)3{3U>lFcvJSR!)Gs_ z|2k7mtg^G#T|Ki-CF8wjOCW_A+-I4VVy>Ue}=0h>{-r}uT!#s%UjawD- zA*wgNjEt={uvKRv;vFxqR@Q=MC8oy}S6 zzy|}DEa04HKI4ycA@HcKcqwbh*}7ssU|MH~&{m@#!8gz2IZ^Paz6h@#3#_wQgEH*Z ztlb^@BI7)pymnxsu}5R`poOL6wJHyKdoVT++TBFn5#>R9rLlR?)E@F0ft{_R9ankA zJF25SLz|4>lY7QFJkK9G##*}fifEk0&LQ0=8LOX7fzD-n5Z%0k@0GijZXI);&daCR z>%ghbAn9DQzIE_X$F_>h*%5buabYZn2A2?jdpNW$8j}BrQq}Q$-VyEZQ93CQ`mOta zC+WpnH+L&OO)u;~y|D)+VHZlqK16Jqh1|D*50B!Zp2CLM3Sa-GyB5%YI^sTLP9!!B zdb%Gz>i0N&FCV_Qf^Uf?vW(p>%YG7C5Fe@nF1{SSV_}60-)P`*p=9@tg(UiNfzu~* z$HK8v@?!Xoxnlu6KzDb3=<4H|^E#Kg5k7k6j)i1g$dFaBJ^D*C*UDBW`=0pJy{3Fh zlj3Il*0H^P6~{ZdE6t53cp2?en-t$%viy(uFP=Qlq$&2bCYAnpJ-ECMoK69^#o%}{ zwzg}r<2tcoq!-Kfr*F#lS@$d)!iKu+)S3Ggu&++ReiJzpD@Oh+;xno66rbJQ-9NaF zZ)$$+WIpPCg%N9<`8Wx<=Jw7MOR;H>^uv#$1>L7@RWg@(V{&2=EBHkV5zPko| z>m&y>W3y^|omeru6I%=Tul?>sY*{a`C-TTA-wi(6?&-Bozr+vh)$PzH@BjX&9-^WE*E zjx;{CE)#!_xyij&XZi!&N!9DgHOHE*np@yupa0mE|6TmlVxyDq!f#x=UOaHEg|ZL! zWKP%TYLDWKAKWSCsI#Afj~Kl1JlgyZU|HZ(d}aa9;z7?rpI@7o?_M6Pi!YnJfO!_B zmvZ7KuFK+#JA0%va;+mJq`I$R47ybq-HLSu8x65_PPac5)vvKeLS%b8`)>c7&i|89;V07Vp;WD>ZK`pTf!;=z1&BZbJWS!Q$k*!y_3m;4j9Yc3*CBCrHOYh`< zdKX_M)ak5=srLL1Twy_OMByh=}glp(J{C>2;UO@RYQNmRSUE|h-cX@ zMMw4UsIkC@@m`JLjTuG<(K}~sm6U^z&8FNy%DrmJ0pmNWrydyR$@(V{-<|U+`|}ex z*D#zuQ{J&dJ~Uzu>s%N3LStU15dPEL`9~~|LfIiR?KR>-v`c!9eD6}|{|4?GxRr4j z^R9D$-p9{G_to(A5U!z%G*a={SLn~5;RAb#yC?fOdn-d@=h{xmCdA&mg!V*Z?)_;w z{$Z4TPx1EPukFN`Qye|)-X^Y|SvOy}BFK7S#+FTCP z4rh#sDZ*Ge^Wn@G9|Gp^mosDhM_|*D`!&AP-SL$!9GwforM@%I9cQx+nKdEV)>!`! z??lI1y2e+>n(w${T}@sz2A^;x+?;FT@HzXcvl;iIvtqc1x$jOm(%vch<5t~OP@BeH z>Q}}*%|3@Qcg8!-eg<50jCb59qaT-ajCY#-3vi@8RK>!&G^S7ZjD{4Sbg;+2&&Y(V zvFhK0G5t$AtA7rio$2F3@{OEkIk6%|-z9FW$Oh^vMW+)FC?am8?qc{)`}c{VQAg&# zCANQ4bRR*g{oR=U-A4apSDYnUU>+-frh%o|Hy7V1o{4>>3fd8iX|5i;?lcw|k^JDSn(my;^@hgfGlA=OmNt3h<-6 z-sDNHTyOYTvhC-83pl+6+VWGc&VhdNup^V!@8BFLci=9aS~fYIy7@N!NwEdn1Wfd$ zc8C|&y5By@yPDHlXUsWJ*20h3x7S*xeS}MChy0u(250y-QGUCTeYppYXY-!UfFjcz znx*&B?F`89`IjOPmHjAxepBjEU#N6&Z~IX7CzI=7Bplo5}4gOmV1TGw?>bVWaN#xu*l z3AkwbMbZ}9q;_pKuyi|Q(lq;dlV;e@nG~FwG}+!@(jG#g)*< zSmflw5cCpU`miVY+hb_{``C0w9&pci9vO$vGPWG)N0LSB@L>yk(N`36u8Xy;3jA|M z@kl@5+Aq3AZDf2ubaj{8!G&;qFL2SA;+G5ex47^HDv$j0zTw~Lw(zg=!uhdsZ)gfS zO+I6%?_cJm2QuVyz&lTq9&7i8lG)Rd?66ll4r^Bhsp8e0a&?<)sPh=KBmG8pJlPPX zOGIJX(@-3{58U;B4CSM`P=Z@l|9A5)Hc;B1ZVz+I$nPMGtS-9pL07DkE3B(Wb?82k zWsaRfHWT@}U1a(jh+Lo)-J!oyXrZS`WvBZ0@iHA9h`kQf6KM`s}0gSqvz_Jn0J}zO^)qlhg+^QoN8<|JcbIehb?CI_f6CQ0@&`X_{I*&%Eu)95fCsC8OB}kJ zd_U=W#@afT6?~BX8NWZ)V05mn1*0zr=6CYcT;j2YRbZzihT9Q;8T$*pi%y!jTIgL3 z=v|ycom)ZQqPka7uT^V~eQ4D{H+JcFE5T z#DBX^A9S!Y)QO2PFNq}`1jNOxtVz7wT) z1n$E|G-apO&cL8xKj@?%caA-v_E^@oufX?=4Bihs2VQFEsgF1Kw+j3uDQEs~_XgkB zfAS?8%g%Lr;WXNPBkjL|eoUo5*Rz&fht0+DZK7Onb51=gP^0Hk&Y%|&ZJ}uU^P+8o#0ZQuwvBwO8AWwN0AcDR6}Sh$*8yar0>tX-{)z zpE)sq31`#S`J6NS>yoVb9{x8Dv?8ytrZfz-B2BkjHE+}QrlH7e%zu^X=6N=gbEY+{ zKjw_Q&c-j3E!v!G$!MxHDKXhd?^v|p;KSbZed{<29foHF;!V@2GQtfag97OPzq(R{{2ZCdL7AH(}`;;=iGfD4YUffKocjxeI&TA8!=gfK{Uz&7VKAw`#2DmoxyLd

    @#1?~ccwI7++jzU zd0wYpYmYliu^b`!n!lq5xT?`kY4^8 zYq-(DXiNRBpbg28KQ?KqeWOWbNBHviF0LKnb1@Ovd+UgaaBNIh*!~Y~3hTPt6dq>` zU%C&sr`LYPdDS@@osVe7wk4ko#h*UOp0LqX;%MM!AADwHJHZ9Vtg65AQ!KO%{8sXN z7F-=p8Uk+_UvYFkXBw7H^;aYc-?K`mV6W1X`|q}0;;%6D(64j~&nn-@ zU6hB=C7z!U2tSYPW}~Bik5oFV_W3-N{c}t||IwEdLk--yE}t449*K8g5u+yE>^0ZT zW>1Hh8t9)5uDziv-aIj>BVOP;XX#zmR8oCkyt92zuP>GPbylFAa{xZhHA3kq!3tARTUW@G9w-f`{}t#k);Gms8(fr!8W~ zu&o@JzbdZ-TikS>>kF1KCn zF>bxPN4gsQ-AbPPL9}<7z&Fs4*I>FF$w-BnG5+lJZr|yTA`G&ZBfct zpHVstKVPM+!S|Cs0k*V04@W+Nt|KW4Lm9ntWmoX5|&>FUzPrB@KOJPZ=T1q z{7G=!tQ+qA-hSx7wi*#3;Gf(+eV;x_c3afTEgL&2xZ5qefwFp6CP8Zs{~SD&FIXDiG3_%S$~SEupBTiy z0GoAE(CFN;{HbpKA)oQ754mm6q3vX3JH_sjFN^xUQ0K=}9?h-;zVcU18VOu4w5s-A zXpN~)1Xszwfc4nGh`GNWeEIy`dxDeQee0&d&fNb9cnDv0X2igem{e9$V5Q<1L=vrV zK5#;x!&l_Ijx*{peRu}>F_{$HUAK65LeP38f!G$kf;Hgq9L_)oI7eQIZA-LUhu(9P zIc(s2&y5%zdVp~~0v_A5$Aq36F*an+9v7--UK@s==Wn>foG%JDKoi_889vGx zX5v4EPor}L0op)u}*kLuiWK0c|$RS5=8{A{A+;`+aE?tDvXC>CQS z@;va`<_+$2?`1y?EW~>^S&h^@^O-W z8oPt?ZSurVt{|1{Ifgb=UIU*voBzAW|L?#w;VS0Z)d{BZ-tX0}p)*H5l+LNS(>_}@ zXI!hxADXE7H*tB&*G{a#@2ZwPG`%wvzMwIPPVZqHoYg9B!tcI&KZNhG`TaKY`!}7p zzskQsF$?^}O7#y(T08W_-3QHl=W`DJIwzLN+nkr(q&OseV}uK*O~A?F!D;rR{5LXk z{{Z(nNN$%;7~^o`zCU2bhTkWtU}owLbKteYl;zl-_%isCU~;1=%eGq>U&nj!t|gRx zhceA)VP9*s=HrB0NAp08s#Nv0huhcN=$~^wG0ncg^p*4B^z{d{;q;ZW-)>(Qns4I6 z*6~g1x_zB)%5v<##h1B#oovdo?Z3pAxqTf^nfmI}d2;&t$GE=sbMbIJY4`b(4j7xS zczSejhcn>7L2|ouCo279ZFrizv(%UQ&|!RESOZ?8EQ7K}@T#mA>+R~*bCRrzIYYU3 zMD|5}?@#J$hGWZ!P`?h|r}bEC9z1UC4dnTI-oM^=&g<(N3;LRK>cjxjUhmd?=^Z0h zU&NkQbgnzDMWkxC!u0_VzB&DOVw5$~N0lk&gYPP0JMynBj%bka$ReWlhRLnkSA6@Fv-zlv1r+0on>j-=St z{1-fG65&mJzdMd-{?+Hs{yMNbn{V!-EPg(wlJ}CvzSvu|6J2Ms)<6kfMkDfsgwX`}Ay^lJaC7kfjA>=F6cE8-5G*~D01)tfky2|=9& zRxGzK+CO8x*%#CI{`&4`k5{9QSd8~&WQJexyMy1i`Ek!dIF~Z*)gDAH#SbuCH7pSR zfn-$H7WsU|^fme9MDUedtiB=}b@)Oe19#gGTNkI-VDs(tg*=~k#rL&8I!o<*J%bjCc5{jf`kE9vfsA&V-`^x?%G_*& z(VJ7*pF0Vj2e79a9GxmVgyhK_yRxVgPvyUhJajqQ z)nB=^#IHLTDto(YFYkY7`jBFOK>7o3AgUOU#B z*gA@H*;~|iA8^Yzt9&+NECn{T*j}i!I@PMW*DWV#HnI`rx3K3&zkcZEKc@Tz>?tEB z7Ba`$r)!;SMc?abZNC`#OY||vNe{f+#9o~{_UX;-qoPZJ8F+Ay+Hk+?OR`4AVsG53 ze7?ut%=UJ^F&~)F-hr#5o$c(6vd3rI$+T~zeskOye8hE)m3>ELvHo+{tJ>?{=LJJ8 z+2f-=rL)luhmg)hH&nWS`6d;-1JM6e#{X_U`QX9cdpHX(ITrm|`kOZ&zi#8Fnr)A# zo?`ni&O_hPoDXInLMeNHne6>d%%x4{LA>ij#;1Kr!DO7qPFk!qHYZK(#fv;v)nMuh z$3|18J&5y6UGAw;s`1cyD)md0P9eVsTs>||Vm z!!hb94o`dBJnn61AEmn!+hg;0w0-+Jo>_AaB;DwS8gq@RD06R ziL>}9bvB(%oW)gEIEOetd3iZq;v}|mW->1)r!+_LW?9&amq6v<#@Va*>VBr zs8`9??IyFQxpncnl+IX)=%i2@Uo}AyB zmxvs+y4Whdg*7EWTFsqH6APU+_IBQ9ugvT(oMi9o2A+%2uOo{mha(qSqndNzSMcob zs@{CXY2$7yj}=|W^P}iJ_$qu_3~!3zQHhI3)4`*?30b}Imo+wvu}q&Z)?=PrCr0$+>LtXuacTZ1Cl!uQ@2GJEcOBi=ub@LGPUI!iey-7v)_q0YW z!=_WW`I~;7jZz#C?V&0L$TG%%IO9tHE-PWVPjNJO-Zg-8-kyr=NLkNdq&Ms1&!&Z% zi4S7-hZ)1_Pc7mI;Gd;fAkc*Da>uiPd)X!9WHXW-nUL&c*dIe_>&E{+z>zMA*KJ=m+>lMBQ zE#A5c`_X>(gw)=S=JxGNX+v#(LEpB@uOU7L#UA<=z%NsMI@3EHaVWsI)xbB#3#Rh_ zjJ#kNyx>`Q-F)U9`t%mJKW+`Tic9^#t;pdAV~08X;8xm*=Lg^6xyo-Dez451coeDj zV)88HJQUaHwfujW_!iaATAxZ*dK;WU-m}ltZ*Lm-qnM4faR~bC0Y1#(yPM8s9?)s9 zNft*+z}-YEdZge=Km7=UQ(K2TF;Hj7*Ewg%Gr>jfq7W{EJ0G)du8Ufau09*T3))Sk zUuu6m{ZfBMk&1qoWpb`8#WL&T@7z8WZpb?+!B@5C#98>0nhWZ6(A98fEY~s??o}&()WqXRwSDe5 zyrk-j#!`t~*~a>-wSJY`&mQz$>*h%CE&;eoE~|x4#P?nPDqrs}7cepa*38H4lh+&+d# zWv7ny@1$@VImw(ghA%6ALgg^V2Mm{yNmwUqbX{5jt9|AM@OP%vL{ur`9(^FMP9*|1o*< zKUkYTBCqm?;SCAg-F+B&PyR`@o2P|3{F9iQL&#COA0dmr=3^&`!R|}pmO9AA->!V* z9{OsD2MKO2?iQNZ`8riddY}&U5C^QKq`FSf;WUZi%m-B>v;n zzmKv_luei1Ox%a+nVjd?fQ?Mg@n?R@W4=O}cB;#J6 z+*;t^R+ho7EISCS1iJ@GTcAJn^?v^U3;rV>BK#N5D<(YiWcDd_wyu`>I}A>J%=|A% zaPc0#D4y{}>yF{u9eD53IQS=CxLbYehWFo7-hKBnFiH{%YYbMyDah1ai|>$xW5S5Bc6vP#0oO6!3?akGseWY(%D`I2CFY!Pjb(6_MMht(# z8Y6lfOkK&qX=YrR_5f03Tkcllj6O0Rdi4qR1f2NTD+1Bizr`Ih-?EDHp$#k086R6b zB8m9edx?##F@GTqLtpVU+~oS8#b`JLK8c2_3lz`Tq2V>It@dB(F8#8uodo~8)9;7m zodm8G&}6puByJktCH{kG_K(z2dH1rZlelQqAR_2rHR(l$T3qLdd)WUicAP>L=&4x&3!O@=U4oH zg89~5;%}ZRea5U$z*lj=^&BA{$m!wu0peq5`0a~MT#&j6l+j+@=9Ca;p6$pM=nOpD zi(F!K5jReY>{-KwQ}Bumcb+Qvu56q&%+mwFNN|v?%j^19-bS8$-jo;PkD8l>BRSU+&5m^BV_%fBkiE)tyS^D?*3nZ#)MDn4m= zk>awPflvMlxJ#DT8k}k1o@q}L+!^~6!5x^NX6db6n=e{%F**o+ z@#_3#-CE%OxC8s6jHBX}1ar1maR};O%nE(-wiTVhI$&b5yr-BfIiXLEFntvw zbkdjW!WjuNKJg{vZ;BhgaMj@# zXN4jz44S&ZpdLJ^^ClP7VRuv<^sDjJ;oMIbxGZ&H@=luL%Ply(%NQL#liLxW=0C_2 z9OR!UI81io0G_1^Z_T+A|?X}*Eg!&GsqL-L%*QRv^&Y6jr+)ZB$mfM zSnWb*|M;izZN%}b*Qo!KSbdki4wHAU>feGaE1iwBAJ3Xo`QLYpWr5?zdWZXd%#U@r zXj6J0_%1)z3AFdJWf;}x_+$aoa+C_F1?5kP66Jc$7T4(f2mBYcr$09 z$M7jfeiuEa8+uH)7c+OF$A2SLe_D`9CUlG?*RFBrWh!Hdtrcef`li_31kAI`dnQoF zSu47hX+DY#ENBlNs`{Fb5wX0u`6zPRILW@5Pv0-0;lViuCaHE7X-w{y?&x8!Sbdid zy!3V9!)JznGA_zh_6PNk`E4o~)Di1mYn%^xqmme+#-EP&LYv2iI@h~g`$o+(>xbgJ z?EU%|uXnE?_qeu)eJ19;ndh$S-6ZCD6Z)6tIkw)_r8?~&Ce?Z;7*+toVgrv{TlcHT zjvbln39BF9rw`(Hhp6xHyIlJ%wE?`e230feni5AJD5s4cjQzjJI}_aXj$m^(_5Pqb znm6rlkY3EQ<_%fMth>#OU;avZC(QRWN54d`();Gyzo=xpt*Q^RTo$*M*bOKefT7LzV=xt z%P=&RVgFQcqW$|x)wd?#Eqs$)p)r)J92l-q3f`3|Ee%BPCT&GlmiICP=>YW4tKe};}T=BgLx z%ka5#;IH?zI?jRrkNmZv10OM+@MZYi8TLx{_(KJqehi<;kKt|n81Sc;tSw)L#uVG_cs5y1wN2S?s zGB1*e-Y{v3Et+lj>Tdde_?Yy^*MyiV^9SO4G6=t#6#Qy>1a(JZ6Zj>2V~ZOfq0RLn z*Pa3TroqEy;NfTcIhUS=&dN81f86amT=$l~lWA4G#BT$?P(nuUNlECAnfQ`qT4T0B zFJ;7hdMd-3r~R14r1LVkmpcQ08~k`9#0%6p^$@AuKM&*)JJkoJ)`S6_2(FK zE3(jY-i+pF_hjs-M|P_#!G2ht+1v`xYGdqSV1EbW-^7|8Mc16z4r>1Gtuqf8B5%}yG{F<&5xnu z>n@Gf+3;fafv$1OZ=+nkidyTs(a~1xiOv}Ne(K2wWxMyP=3A+&cNURXxo7^4P2Ty< zYFBn>>KusWMbHa2!M7*7-@2OmI`=HzOzuJazZX6g#krf(gYXWze3s&dSW-Giz zd+DvFOm&zK<hmu7g}8vg`L=T%{#Hvnvft!$E?_&dgkm2&mG6|jPZ>fS zIG`kvR+Dm2H)k&OWaZT z_+jSL*z~c5>3+y13C{k5VjdWN^oHUnX4&(Gbox^jfnVxJ43`Bj`JL(emC!Ql$5v}c zV(|NdOSHX8QoOAD1|N98MoaQ^NoF zzNxKp^@Z`SA-!LGvOEz!$@$c!S7bC$#0<-#Wihdw37T8r!J6@^bK=AF*kDCS)= z`<42}t!{f7BYb(zLT_%VSMlxeGt@bQC%OOgJ;ocIeaK>{X^L%bw|$ zEul<)H18N6luY{)H~-(rmv8Z@kydm8_VIkirgcDVWD(m={xzJRQ*db*{gzL`IZDCt0`r_|4>4(qooCWCXy#hrehu(n zz&u>dd|U<1T*=)+iH2sT16$cBHzqpv$&FsuKAB@hA1uSJEBol=3xY|WK)A^F(>cjM z{q3#XM>`|=g20ZozkP5{@25kz&hYe}L0-{Oc-H;+1SAdSe&)BV%kd$&z>B|tZ%AOA z@2>1YmH6A4G5OMhuP~OQ@DHnpwcV_V%fB8>N(+SV=gw^(-~8jf zD$()fhZEmV^G=)}K!Q0Bvgy_%yu){keLNRhn{M@ayg}*QLpj5AiZOp-TXDAJztM(h z`Z0WwSnuXp=x(?F%d!9dO5+0eMQ8FQmMnx!azM`yQMLqn)^l}l;xY8+zEk95*G9fC zW+Si7VQr<&f<~v!3c(H@sqf51pAdbPlczQ#{n3>Qtg1!)s=b^)LZ%Uo9ETRPH!ojm zpXc-N>4+hyJqGxyV)rTTtYRg+ejNQUpS`+0p5WJ@n+E19ioCapnhLLzcr6Fy#vwttc5jv zLo#G+&u9vHE%bRh`|u_n1UTa7PFvqOMO|W#Nmm?3UDfk*enWLszk5CDf`h52Z|JUd z>4{6o%lv|OHD<@A;G9vvkGfjl3w73>y`#Vc$1Xj6l@$$)Tz%1W#@BhbTAF>Y%QJNT zI!_Jx|zr_8`vWVsICj= zJxOjnx#UoEp%wMho;iQM*dm@BF$-^DZ$@yZe=|q;LNjS2zlYU7uqUZ4b{?o+=ZZK> zxc5DGv|U)E4*RUP5nYX zXs_@te7>$jHUVyhZ}kav&O7I8I_5pgUIhH)zbL#v*M;Tdz*4%PzPXnFl1-yt#oW%a zALH9K+e8Q1b~%0N!S`3Yef=D6n|?dCq_e5}UDNMOTXLT>2a45}VNa!OA!RyGp>a%D z4nA-%puxBOyD!S;Ue8fC#N%$ZeNcRcJ}cha2!p%X_9D|)!O6L^ubMtDP#!pOl~U0$ z>3^`_{iSDuOMLrVU42e=g*N1rQ?8tH2wIYyvID(sn)zO;eF z(pv7&`#Zi4mFQ%WTdw0-F`|BGWCzae(|_@piM%Tw)3OS=*vKg9cCK4CO6}hPpGSv_ zyp8`DzpTyBfaFo!`-y(DOEl zU%m}5USR3|I@31yDn+x~+p9Fs&Ag+&4%EA6c&CEzEaE%I+TH$xr+YWmWQBLC=ssK(vDNPWesT0{$W^ARl#{nMIb^XN^q{Ugw^h4E1;5 zFW<@!`7b-v2TFPOJ*Bk!cct*Rw@pg?Uy~C5*QD9_TI$ZVOOTnSA~TgBGfhEenvBf! z$4cGBDEMd}R({hK_b6%K_lK?RRnuD&$4G~HkujfcX|FQ6IQ-1Wg0mf2uz));PR+HV zx0$oW>GpHfRUD~PbGbh<+CGo@EY=(a81plfX@8-jxxLEAJ#@Yy@2;1OcrW_bF4kj} zRW>m%r0Z38wj)D&o4A{sK1R@Mv&-<0Uf{GBqz-e?;5*&ku`fNzmH8$mBJ-WYp7aDO zYG67&BZz;~f!iA_opqfP*=6E16r(p4@6{Q2&Y{-v4bGDlRUp&hgJ}GPb>^4(pQm@Y z3o+HSm6;j56xf<}X3|IM972|(Y`gL%c?T?C#r?c|?$5?GkJ*Um6=QjZ;{zT67`{y1mzX~%qjYi96VzAwMS7M+_dmu%_W z|E;!6og1@)gX8K*zY!f5N&aP=^^+|2ldVzaTa%ZMla5?XAFaRz!6C9;f4^T)xK}?( zbHH9X{oor#MsB5#<{Le-gFW0j7BQfa=eBl{U8i-*uHz}w-MQ8c{td{4TNgKv3VQN& zCw;QLgm{+Kk@jNUcP8Cgwu+%~GN{jaXI~c|@R7tK;=TMM?eZt>kgeRhTjTFAi^-fb zsK4o>_O3D~^Cy|}jBUipniF_;ckSoDm4xqVJv#G3=tpM~E#^^vzD?W#)Q+8KJhJI- zbf6fVmm%|ZhqLa5Uk&dxFs;fCKF_-b=9*jD5MR;xxD&0&gCDj_N9Y1)@gVt3#9+Kt zx+?Fr%(cSrg4@E6O7PjVUz`(soc0%S&MolBrO_SGkm8<4PGE$?&-e%w+-X%ASy<`y^P;MmXzi1LFvA zUPg5jw140A+xx7OIl%(T^zEtcw?lkKYl-@7)@{CdihieQ%_#{)f6BU>x7s?f2K?9a z1B@%r>T$x$bL5|EAAX%*{V?P#k8f?BC#j)j*v9M_O&GYG%ne@7H+IA8WBgb&8POPM zZyEWgNX2JY(6)3a{g*B;f0Pz{`^58pC)iUa-NJW#?Ht8p&|XV*fhQCxFA23^(>40p zbAy8!d~fAviKA6lbEx+X+{${GJ*j1<6GzA9G*dhTy4@=}h5xOFpZ=YGNVm{9@RvDo z_?|N2I{@dU_(B0|*=XkHg7fGQ+LO>ayQ?v4taR-E_3`gMUT!8gpY2q#8M3kD;xW*mb2og< z7l*pQO|oDubyd%IZPzl%g6~r%_}2g%)n7v1+wt{*rOH)yA7%R}GqNH&kLBt-KIAkD zoyS)iFE^&!m%v91e0l###-(-y$Nj{D7q0$=bUbZxH*UCf*kz$sVjP&ZrT6^Aa`hfy z_6TwG@SX7m;49tyX9U}N!S>md5HSxr^&foEBhRLV>R)8N6|5IJu=b%x`H&f;16gr8 zQ0oTfY>OwH*Pa~8LyyAV$oWQan>m@_&dGE1*XXFw-S2>dGbiYSuHN%Y1It`HgVd3^ z(Ji4N&B49oMVYTc=0UNuJLl{3v9s@>uIl-&?OLX}`M&vv6H6gBC#A7G={xfsedoLJ zZJ0T9^w!sCW1;FJ(|p4Gbo6rt{)CiWNm&i+?i9=K=--a$Y41c`xts zqH&YBH&Qa5^3tv=8uyD9hj%p*(^0nBYWA?S{waR{#mqxff%=wV*HNx{Sn19~1J9xX z(ej6;3|ypMJ$zw`hFe z+cQd``OQkTJxk@o&t%aZ+V$iJ;+ef(|hy;y(ye_gP*)=Z0@10LjA*$uw) z*RMy9IDyab`lNi$zvGGbVC zDPw<&vSZ{$ysUc}tYzSSwYPj?TW|VTzGh<0W-C}*VwryEu2A(^du6-n_mjlFyPmn3 zQk_J3w*50;Xy~8v25_kzyR*(8NQQIfi@i1SAC?@sCxQK@p5S^fa6J)R_krt4;5soc z$`gWLhU2;5cs{tBOW!lVZN+nYhu_QKuI`nI;qDG#D%{;qUJQ4)xp{Ap7sK7(xp{vg zuRHFpbaD5$)Y1ILa5v;W$8cA4t#Wrys4MQaB?TwrM>~}M-pDu9Z{hA!{1@(SGHI&) zYm*YEg;X$e>g9sNGl0!yz-T(Kng+}+#lCR~el`gPe+~U}Ru@^as#<=(%-v6Equs?_ zdqLP2Q?%g&7N)%0-DRQw$5p5eBG=YjocSq4IUR3IBoxuwiU-(bC^JT>!F>R0?}k{hy3&- z^2}P|($xer2Kij1n)`TeGUeHJD}8ZfCY3pB$9GLxmVL5InM0EcO&RCAyOcSxvb4UN zbp?7|q;&;)ZTuX)=9>A{c;t6l#F!4v4ulQOzScXa{3tMVczUjV1!GlS88oYyOR@RT z9L}Q5#H1+k?yoE8g)Ef_uk^t)li;1n@K9nYt?Co{lsH3@qvY?^hAgG~C7O_>P?OwuU5s>yc!CWTEx%t@Ks>y%GOE!SDmxl#C|a zmEPd2KkREIIP1@Uk|&r=cX4vG8L#%NoV%y~V9GP>fLkuU-vnG^ds>p)hc-!Ob@ssi zO?V8>MZn|T>~)<+#uqL}KZncsYBE02nfABjuW0c2iN4UH@&eII!DUBA%C?^pY{28F zS$lHqU-P^k{ynYP!6DsCw~4h!a0IR$>(7_2J;3^z|KDp5-_+Xk6RkbvqP;|D&j)@a zfhWn}OKa;75l8?+Z%#uk`!s3vdsl+5p@RPl(c6d6mpt#}<@9Hs>Br&z!T;p@?PK84 z?C~7ltaSsrhZmbWY51P*ZEX0XqvL-@UHRdBZan+?1=pab=ngBjb*6U8u-6LapU{S6 zNUIMqSH&+bpB$YGOor_35uC8VsdJb*TGIl(t*YurJi({ao&0~0Z`Oijhfhk6YJpF1 zKSr26)_?bvCCt^i<2=uR6Iwfyv2`@Czc7Py#%0*XH^XDfh^xHpf(ori{6k{v}yG!SEZ$hA{>HW7bV>S~vGid2oQ4!{Hg+M?fD{_a}_&LwJquz7)R7Usm{}ekaHM*L{nze8J@|-jf`0 zhe>nn+eqhYj=v&4z3#|WY;O5ge*t}OjoB^=o{!;YtC6B*N);oBl&cyKRO4p@ht1sYSDLh8h&GdiT|v7c01bX z+u6=3>|omOuehh@T;Gs+y?r<3&XsOv_)3n$SHx2cZ@I+^z3lRpLG<1D61#k*@}5)^ zGdAt+B=}CcoovdvKj=(+<&A+I{)hV$UH?PAl?WUZZ?dOSaFQ6nq~axKwvWVYBgXg3 zOZ+ro=ftfIZ>R@n3hqwcP{#iDbl|eFSC8ejXM)epl%R)rT_1oipM%|wK|v4vLU%xS z`T@Iq2fFw{jzf<%^i%Upyg7$gT;cMHu|0$5U+vWSJ9Wepb|+cJxAvE5PX0mi4G$r| zGsh0GUh});34uv^9hQP zhFl|CW&pm>GQVf2c_g-4z8g8>4JjsAO^GL13m;Iub~B&4Cp^_&K;Pv9Y&Z%f^YGsp$?paF^*Z+hPV(fIKE?f%HvHp$ zZ~DxIo}|)3o;N1=N+ZOls_$=wUtEk2ti~%oi;c|>T*XtSkjJ`ibOX_@>?=dvvI(kh z_DXZ?!EWAYIq}wI+PJCGW{bkbz$fcW*?;vR5r1SJtsA`FvF>1t*p% z^`;c*+(71#w^^gI>^`)o*m~vAqVbcwd)tN#m4U;e>pUx?VKa32ICpF|-;)lnAQlRj1doVoS>SUz*>pq7iOQ&wM8vl#mJI(;>K8-AiYWEL~7Crhq&=sQ0h`p&Q~ zgx5JTZHoO<#;d%I;F#>LqH)cG`X%~~{lCxD%e3z$T`QX2(-WC}05bbPWXVCulqtxT zsmPdV$n5FZV|yBzU399mcv{n=(5Zau8aw56L!l#1>%oPgwUwdp`2@yjlE!JNt5nytQ7mnc(J|H5q?L-`#E*xZ6tK^p4S2 zXw!Fhctftc=A3lPu0In`_(6Kdy@W3luTM6xrh`*MI?HgJ=W6 zPn8N9_Y#-*Sh>FvsbYHl}>v*s3X}kpE{cw zJ^SxWaq|B_e)Rz66~Fe=z`OFjBL7RZ!;wYO>@;+h?MJ@mkKB_m_r=X$_t!C3$Rpv< z=JWiOdop(J`oFZjdwf*Y759B6lfZ;<&;0^P3`zz>q+C<6OcK-pA_^$pT2O3_L~E&4 zD~JSRZ4j+Y)!KrVK&)gAQn3VOs#X+R@m5j1)Y?`qw#+10rQwp{VxI4BpEF@VtWQ7B z`~Go0`<%V^x$V8yZ>_!d+G~4rHcI}T(cgL23ZB)DoEN`qk43HbUIFf}TA#$Xqcga% zI_@D)?dv0r>mUaQ3Y)Txj>3WjbeGFmC+Fxb!m}H64>X0}MCUzPV=XNEo_QYrzlmk^ zyRdAgp2a#P=UD~zW8__~`p{t{u|9P1GG)_L1|9B5EQ5AFq3nF~o(wyQ7?#q1H10hx z#nSkDsnr9&3-3K+<)iUJQt!P@->6@ficZvmE!5B0ZYF)NvE58hW^2{3hdS-3kxK68 z*%td)Y_F>yyK{CD*s6UTGVFEYQ{<|2r|z`z%F0U}Y;Dh>JlFn+J~U%Z%15go$us;6 zw(f@}wZ0MAxiw*2&-%vf;EUbZYXn@4^GGl1#m_a?il1w&WzOlFLY}Yn*21X*^R5Cr z&!qWw4(Vw|_ey#Dv$MM2@aga#H?$7!c?0v0V0Lf#rVsqn7d}ddpEAH~aJw=kxTWCn z3ql3Ix!|6H$1bQTXuRP3f>jp`VGg;cnL~DBcT4AL1IslZAl#PD<=S7C#>Z!Y&0joR ziyc(@;VnL^`BILyZC0p__GsQ+;r~5)Rw?!u{K)%G^cGCg+OHyiVBRO)z-!?)M^c~Z z^R6B~ab7icn75K?OLQOc+gtpb_%TABGj_|dUvu+znY^Ts<^-e@Rgs430%i4J9rGQ= zZ%cC@{>;V8GR+=8S@ve`m5;9Uo$7_n%QFqO<=D@=`L4cme4o|k2Cs7Mqj&L6{bw8a zB03H0ThVKtQ3_8zN_xC4=3?4>I&D9VKA1{hTm*()2!>6ek1n9ECg1ScS?6Pyaqw(s zV<1$GJ!W$s{K|RWLU3(1=MR8kp`@hO7bc~)M%bHeA$6KaUtrHg{2gPD)rrTY2MEW~ z?bF?Pg=KsXx}*I`qiEVG*SZ(gX_Kz%)>}$F{nxf$;1h@DZ=_dSR~al@SNuem;NLEB zaONk}l^#{wgL)4n16|z`slWH+nBI>w{yoV|;$&3}Mj>@g8+)yKAUx z#sbjiGw!oY8FX+LWv5X8PS)I-cY{;!b8YOy$ac%jpZ{d6(I2o8YaV9TW{uU-?FXSj zjSHm9ea=7aSZ(Yz>neVfPkEmG0`2VZfch@i&f|YgzH7Hp4DK)C%@6X-IQk;+LH+Bc zj?y%9Z?>JPlz-}KuG8%l(h2{A&ZqV9+IK=YaWG+Qb_ASIUqC~hI5Fxa>^Qxw5eLBB zryI}lhj)$lf1K2N?lsFC-3}}`1QwY14AS!kfD6eSFB6}^${j!NMfAA+=xY0Ei-YK3 zTH_#FZiUwwgB)TW+vm-zX=F`CE#o`cbtiK^&g-l3k%p<~jK|V#@vU&>9r7G|dx72T z=Dn$U=yKB8o^S*PvnzyyZPrpY@SkN^N*GCkGCkLZGNN_UEy-l2*on%{lSer@wl~0<%9}Q(tH%8{glK zF(LFNqw`6&D@Jwe@TAaAp0OW{i6U=yk^aNABcHE-fKDB~&#AAm-G1;)`xC$G))VcC zCm-`VI@Ug(r)Te4QXYEUty4xF(|(K%GQigTJvmBff8so7>|t>ByDrY=bPe8lv4gXN zxzErhHiu1_;BtD>+bzAFvH|)&Y#EZ1zjbY4h060v@5{D}!6oT%2fe~A_P61mgl3$rFtxEfVfId-dGM_Y%7xT;0|B;Vg83`Xt{Z?W>qR>{ofuzBSmt z@gwwL>@&^i!A+IuzIE9fS3RIMC+?DKb9>jkGtZvtzSFV2Gfx4(4d9!eBmW?DuXj#x z>*#)sAI4MOOrEi)*CnsrcLnj-&Z8~s&<9H-I}7a7;TNq(_!7o$EHHC5Bf(Ae>$%iX ze+Nj7{>D2;c++cGOFg|hDL<`goAW=5siS`i{g;1|PA`n_M=E*>+#j&NBDk7LZrp)9T|}kTfkajd+I48yJ4ByJxKabGvsj@rRE6D~ksn z+y^f@yi#DF4~?xPUIDhlbrrwtM|))0`F-O0FZTZ=eP@Qze`Q+`AN){x$iyEgg{Qx- z6neYftviQw4{P=`C-~py1LJcAN02d^D`@}nTwte+d34hk4+mH;AQ|_5SJw1%Fb6nd zKF5_D-{er)ICRj>qO-KW<}qfhk=|MF%}WEPFOs~t8jMsu?TJ2E-wO{)9~DnZZj2|- zv8!a;r@49KR8KsfWHmD%(7X@c9I&=nJE_rf7KjdeE!d1&LS6c9_beR?sNxN zdg?h~9Qu57!!*y^f9~bv`^cXqy?$4B|KT=l<-2?OHTINGpW4HsT=Kph`jkD*1HEs#$VdIR(h3eX$PQr`Jlx2t&QtcvZb}b3)^4# z;$i9Y+cQ`rNL#IMtMo7SrkmJ9w`5q&l^kbrVsYAJJ{?dF_&Dk zg>eac!;EO?xAuaMEN2a;_C9DrAHxPWy|&*I7rf`@&q5!pCw~a}v-+*M;4ShSh)Evk zXJ7CJ=Za4*-{V&~-;P~h`re;7XY4VeX?>s!`&_tE?GMdL7pa5}MUPjKr?G+TPv3I$ zW~iQPk1uocrYcV~15S=`^Cl|~J{G?Zb@Qr}2L@n^H)|71Y3Eezb%)?-v!4$7j;{+k z9dd0;3Mn*QuN0c8Qu^fh&>Z&LX+b_$f!~VX6QSMo&4;j2^y57djq}mlHVXrgBQ3~o z)y*sZ>oWaE26f~^Djl%tj_~q4qu1rz;#uW;W#?o6YQ+drS=GYy<$6b#gl7wP^z6%x z-%ys_!G9UQRL7>bF2Po@)T%G+)B)AzqOE=KoG|^c`h$ODgBD}UIjEF2*sm0xZBq)b zd`K!CQ1?oX9i4wE^gjbWxCDN<7`~X!cz7D);i>S;MU02B?NxR$wmsd$yg%{hy9N!- zm!}0^V4uWA{AUz-qCHKd`{I|d)VjSWIOo$Z%wBt+@{cFGSN862#-Ew&bDnusn%NKb z)M*Zm{aN2ln>C8l5?&Gzq` zlH-rkwnql$`b8%#C3*gikF4;g>+^@St|=H2ZR|Hhm|5Z44cGHb;g0rM5N0-mQ#;p7 z*I=HXdC_FXjw=jyF@HoV%p7jok+CUsCd~Y)%Aws%w_Nwe=OAj$y;*k8#4`Hs8t%JI zWzb&NgfjK5^u)U;pR00ct^?X}bU&U;|2{_99Lgd+o$-p+f&2PAjGq=*Oze#|bPJ#S zS2R~qSbH{mr^j|yqE~FzSUl7INPGn5%1$wwc34?ybc&@FImHHxyC82^|2g*%*sJjk zgMBpyYfr_)3G*3=K1lRGd*J28V^c zN5I}~93hT}ITm^|YO26t*2#`$%rg4mSdnprh&uRHPztorG2fDXMwvJaV%yG z+63l4%u&N}F-I%DEiJ>XSd=mGMsRr}So?byf9H2t59(Vi7k}S$@%LsoKj7kT4;O!L zB)`_h-|IOqa`E>%&NsRE`#sKQ#qoEv72d_Q!OL;dYaIMNLY|pdW<2l@H}9b8Axk9J zg$u&z|0qv5h<^R9oA;^mkS*$W$ph7kC=cvr9^^RuJpd+;67Ir-5#cPf(WDeUtX2xm zmMR^3F8JHLJdV9BtVz`v_yGO(2i_q+hED8-#(eDal~w${iLv%H<%L$3@t3Mz_{{Pw zgS+YWIyYY!r1=@&f@ha&4p`+|?#NuOm}2^`#}l6qHobpG4mx2jdmuZ{U3D_-y{5P` z4#KtkyZZ3pl!}lr_7T%Q#0gN!|Nc@b|Njf6=vNP#l=*p5VXy9$EIT^?GU)zN=zj)$ za0&cyG1xmD?41UmOa*%{!Vc+RulycTv75_Q;q&AyiSl2kAd9RM)?cfSeQu1VZj{CM zO_~iR&;HRVo%@#A#IYo{*$CmdHP`5!ON$@#1z)+y!QCI|n`y62jExU;1%JWbp*@^3 z(H=C}++?s9n~mn(F$`vIgtiw3 zpJvLjAGzg{e>BcB>d_-QR?ibvzdcGTul_`$iCzKte%=nUB(Jj=* zn70$VOTg|)V7JDEUZ*{-hh8PaH6OBt_Upv%znubh=h+X_U&8JK2D>xt{}9_t`PzRc zPQPkrVYjZ^KQ0eNsndoYA-n9qIIl9c$AoyY^l^w7arRAtvJT>cf^%iiqSicX&icW` zc(Pv?`$c#zpEHdka^z_KN5ZC%(TGV2=>F7-b`a#oC$7M zbA2iLKp96i2Xnd2f5RW`_Z)17I3mQH#{|+Q#`mo8n!5u(k|zE`=ZiS6@@7t2!8o&& z^C&X&0C*7QAD61lW(YTO?4{7AXf#Bg<{~xMB|g5#&2LoQYn7L6|HRGvwen^wFWs(n z^L}OSPYaE6W42_Ww|?XW6R*`eRnpn$nM!BE>q@7Q&mMC5@7;o(F~`1z=g8koIJbQY z<9heHQ9fPN^?|EC_7%(!UTS=2$}?IU!8(11>o zqwi$Ds}nI&I$j(jkMXI0`JNlC@cGy(QWsUUFSdxi@GE4HC#e-1i+pa5Z>NHi;7ZpY zco+NbsGZ(m?T096pp*eYjf-W&26K@gRR)JL#^go z4s-tlp0|IM=3xIe>PXKhI>TzN|Ld*WWIHf1JMbmdUV*8!m%e2SI+E!8P)UD(k>Uj` zw2mr1K$JEc^&$6N(K~byef(fapU?s3S&s}SF2gt{EBI_AmElqQ}v$8MJE|`|j@Mef$4Hyd>J2xbMWQvqI+CU{*5Ur+)tzGG_Ltypw%u z^saYK@_g-cuV;VhIbh8v^j-Wpdf$aStA%ep$T#U*qz5UEau(n8T+JZ=b9I>;U#QI5 z7*X7f&E8=kn4erInJ-@4uZi7dHB`{wq2Bc$;2D_@4s_y=og(qHnH96Yo4jRcx(*y z{{iHbWaSjjo$+hBy$?AOU(XgMKOUz+`jhY5aoxz_Bem^1>eNebqD*y?pdYdF*A%9!e}Tee8V4CzSonJTu2GNPOlM z&Roh*O<6i|aO#au%vqutXrbxaF(E(qhTu)pUuzkUkFGsCdXp13Wml!sX2Rrs@T%HY z{5q93myhr&+4;wxO1lz+7QA$9-RbrM+J6PMQDO4{c*OAMyTnWlIQ%L9o3G;0BgoB7 z$UC)}^u7}@;6xrh?pY_{(+JN~42gz!Iz~#S2p29k^pS5*QwpzLq!dh@qLhA`Od4U2 zjJt^c^@7!<2K&l~ioLE1`uSRsN`Erd* zuqkYf`_;_({TB~29y0S4zD;#gJFu-*_q3*8##rbqjy#Sd_*)ozE_34@N$EA!q#<-p z#=L!fJ*<)Ymq&&K03o;#&4RB-tOjOEHvYOY+vN(+JANP+sH=` z4{1M_kD)1HmvqyI(5d2c;~LldbWI;_k#4%WA~X_xGb}&lboTG$S%tn_GdAB|QjYC2 z)6A*7s(j{CN~BAHO_}&YYmDE-eYw(8y9Q#LOr2cE*AYBXS*Fop@#}H#cm08u#_gg3 z#hMs@Oc$%=A2d%S-#yXU2J^4T0j1#bQ%d>2CzZmN50Of5)jh(vqw}Xi9~VI<7eX&n zpqmRAlTT*tezLKF%En~fTplaty~V`1I6T?WT^{2(vL!Y!?vVfUHK|ViugRC5qjgb7 z@PFR8?F|1jZw+OSg^ExM|G#0)nf@x?9cmorf1xh7{r8o*8v~w_{??}A{+`}v^e@UO z>fhkadrtkA^>x&<7Mx z`p5@eL;KKI4erSfnO}SMKJ@mTyYRKCb9{o-zxqdEUMBx3zv&9vxT0VHm`7|b+B1%M z0sdc@r}?1xx(Izod>+X3J@<^Fp6Gi7=>jtkl=SwYYm@2M6#BOd{oEBC@GdrTd|4t=O-?qtO-HJD7kji?9UVVA3JDqhl z$J1-|fMBsZfA}u3c!cR`=3CS4yk--9L%t=;C*+P$g4SuU-mQdj)0H8%R*EY|N{ z!d#%}`Uv;ua)0|cD_qs#4dxxKV$7IjFLwP~4^ghMwfqK-(2qOYH0RI$GTf6)f9)nu z-~U~=k4KZQ^0=ISOXbY>84O~5TW6noYtLrA1ojEmyLp{wjyHXpV}HOqiM?sQ)mJ<` znCc;ZD1IZ07?1jPjA#EtU+XM0HqEiGb?Kga^06UyC&+a#ofep1MieQOzqei3j+M{5eFhfUSb@GGX8TtG$KC%;lOI{SHJwm zr0Mn&lWKiL6?AhR^m8tBbPo71KGad^#P#see&)F+#E)3~{}?~Qd_VCc^o_a1kI3~{ zv3g^d`R_4V;_~be@=S3fPNeUrH)2y^Oe%V#tob9xonek?{HvqiW!uxBHPQI>8d1+6bRyT2|j zgW-b(E+6>11%G|MgRSRrzxY6L8cyVcZ&IdsDu-)){@CnPe?7dA>+-^%T>r??)QR&# znRvk~EMng&ckSo^mlsw+lfuyxd7-<@3kR=FLO-rM#tU6p6Ve!u!^64_!`nG_Chz;d z<%8e2_oQ%-;e$s`bNB$&Z{C0AAG^}rjzgia`U8oAekSb z-F}zA3!m@|@xo4%rrTRgnr$mC(Er~wKZx%+F=mR^!0bk6GkTwuTyr(pAv!YpUfr?v zZdF?U=^4nnx43OGVT~Jl;itp9VP{WcKB_zOQD3bi{x_e1cD_Sy95igDhc>_zbnLckCcAucnm!HnNCE+6H$24S#Lowu~Td%Lsh7N6hr*)|9XZ2m4)3 z`dD-LtdrOLeJlIV>KOt2W;B1_gM7{3|I^La`gzUYPsV>n^YZDg8F48bcN0ePHI1k92YJ zN0C2^ydyW_6MY3f(X)H{56;&daWCd4dJoy+`a&oAK}Q?=4Eb+<(3$u_OBZ|se=N0M zKi(lZ6!*pH51#BadQX;J=jv>!M((BCKZWmwHO{hPrE6!h<)jZRp5 z`gL*jG-mVRlcRVP2T#I#?%ct;IPV>K8b4>oRMPQ(i5`AC#}ypk9aIA)mE6$yzV*E%3X2 z$gB}9uzx_g##sC5uR8Qj>CH7JFT;$fd<4F8A+_fgN0`A%HU0klCcQzzek)U6}k zQ+^oZ)sLnu&;CVX8SQlqWgb%o{Unsd=Q!`8JVw5rl~m!ZrCO`{@OA`wf3PG4{o-!v z7fUVKjaYMfQ1WWwyLAHt%XO_w(C-yHgD0rlX5kK_O2>wV+>@v>=?0=vR(vl#z8g0|BC#A5@kF?I6oGPjN?bNk~lQWc`tTT8#}11r@h2TI`Wy1Q7X7VnF6yNl@y;cvQq zbz)yMt1sBAsyzc1UDnU_s6`*&2cCnGU8^9nr~HP@lr zsU5yedkCM`us^u2p+AGqpK2bLSOAgptnjI73xl;e_Nnl?aQa*1#WA_a%_~zq`casi z@8*qA-eBeB+F5SiP~{D@%%1F;|LVl*50XL~T$@<6Qfw!su1yS`1|2B_{V017F{ML( zd{5-7;pj&>c2}M)jBW(K-@D_Ph*H*HFDtdI^T=zFXjn^JNjF5TOMAKrx@+2tOMATG+|zpy#oe=MKw=# z*WOWE?;X}7$bM#XqJypQqCQKOrS9k-^u5?W7~iwtvqq@G&vGy*)7}W*YQ3p2B<^d{Xm~l_E+&;sF!6d3Ii68Eq41K-6ad|Fl9M*_k^+o;8NUc#R-ITl z{(e@_&-`#?pwF3anq=mEmKGr^GT*;UYA#Ulf0%N%<@{pRRO>Gnmm zH#(@nJC#jV`-9iQu!)>sCJgJ%esu6XK4WFL)g2=!|ES$lpdsJ5p-a!In_xOYw|&j^Y_*nX-goA6`UXUWl!I3idQ-?QK1KJ$o77M&Y*- z<`EY1Zu)^Wwt4N3)oJ~IJnk9$d%E`amw2}LxdaQ<_34&D#>ZQGcf{N95$YKoc%`eS zHS`Sr{X7Q?@8)^hpGEO-=W%Y{Gv@SQIp;<6v#>e!s+zJ_*%L(bkEvG;E{nfYWAbR@ zIiYBcH?*%LDRlS>Pl)-qt;G$e2V*x`p}st$lxyEp;5DhgU(cZNSz>P)bG@=>&_*|( z_+t0qB=(-E>2uGr{mehD@DjI-BbB+Vhs(9!cl{90qrQXP1@;7F=c@E1Lla(i z{n-fe6!XboHncl{|1Ts@xU&}i?8I|uRG6J(%Chap6U(4mVRj$NHlWjNxg&Yn;gW6+ zE?$A(qsRKZXahPq{&~X5WBz%>g@$Lu19|jcyf1~p@$+KNRc?G_iE(c*c$TaG^FCp( z;vgKJhJL`hF!L_WpPb}f*YPg3jWGC+w2$VD(f`*~6rBME=h_d_#tt^ImYMw`<2t9x z_Hf-s`^Yyo%DKK{H))>}^vSP^ANA;o@uMm==b8pTcZaWgz~4RLb7JDp?+rF5o4sj; z&9a**w(*JU$DrAz@v(c4_3Sl;ol0%?G?@G_$4wlJ-$yWhH?ijSfzj>Qr#3QA>@#sC z9h^Rb{?)*Lm{=g_HVx2=a5_w$(-zqnubjMQljjYMb8&i{i__y=oc1krWbKJ@=lpJe z3a1MTow#%G*Vcrw^g8RxGlIqV>u7$y^EoHPokOruCY-*sGw$5Q!f9lqkMv?}??=!R zAHk1VdfLr=w`}f5&=-wf$k@Qlmx9^C^BYu-&UlK0)yIF&HA6esk&NKK)wY@oox{1t z_ti?Fg$qn7oW79$n?gTcKz~lAU(W}pCxO$B%}w*TQS1Pv#PryOO>;5v9*&DSN8WLm zP2HW`vm5)9`r~5WWBLd?=uBa6pmu6)Qq{a!X_fPQ?D1jMRn0T&%lOA4Pu0BoTxYEr z=VpI6^^tVUksjY1@M@iJSTEt#I{$`VLE(_mPm6m5Q^BaXj(Oj?&bvqPZed6OpLAi- z0gsb^Ci(am*qhMnv9pDJw2_IQ;KlC_`t%1Uh1;MxKX%iou#tUqwGMtmKDv9+YrBp_kQ{x+5=M=$U3{{Lx8f@7<=P6>NoKWWn+Mbm^JFa_Ht4t}M8eI_fKo z-42F$)9fjn>m7RM;nrUMeY<-5W#egC-G}%NeTli1hVGbVFYfOA*CFQ|H9LWrzlU_!3Ut>lA;zW|TFVn*{+W2t4#uKucH$m(59uSao$m5F-#r*Q zmrfzv`zim}iF?-!1NSoQOOlS^-tYMLIPN(<&KdSv^7L(MOj=-vNX1iQzY6;XSNNlA zEI<2Gh2!z_UOo~3>V$vY;gcTlOHcTw7yJYE&ByNqp3&aias1O9U0gSf$I?y2zc-&c zrkf_l+RO+04oNp<{G1qTv%SO{n$15*w--*#hK|K!x04sgyzjVqx0<}t~Y~kQNGcSKXVIw6_2&)I}!KbC#@3;xpvBI z-lKkMI05$z*6}=Hoo_t2!!w0-MQK&@q{}W`2;M!MfOp^S?ciNXpJRA;)$w?D(K#;v z^WTYh_j}99-$y=cAsxIUK0_zoA#*zMj`@Z}ygTCJUF!#xp*Y?t??k-2@&vrA_kIQ5 zy)itv61@ARi+9ZH8Jzo$o9-hX#ATf89eU@ptwa6I9cTL0K5JGF^M@P10p4|k?z`EO zU7zAVxc{qm-vm~is3Wc+?-<_Y7`*$1!4zU|k#^!8bRxXF-;`z9{SwO zE6K$>*9Y*M!a*=4K3>-x+C9{d4DK3|A1mXq&#kA>>$07Fq!X(K^`TP#ze%+&I z5Ubn4MeggwMef5VE~dShcU3yL_)kN_4lb6{7M-~G$q0jsy^(WqeeinPs1p}yC*kI` zB~y_>HXD%i-LN<-9pHO{@uQ8j2iCXC2A#K-f=ek7hk7 zYXDW}iL(OXC7y0+3q9RxDp@0VdS7dFEB&)(Tp-+fzZG7?n!w1wuA!FUJ^W9&>i}t6_~2z8WK%8l83G$Jtbc^TV0% zV(h1JGkUzSt8UR+LU@mHe{%$ku2Kp%`jnmpKinS(k7du)Chv@Pt@YFY{uWx5t}Oj{ zv)1xauV4XkqLSyZ&Kesn^|I{8s3%=RHjO&;GucEQG3&51?MEq7+{MkBXCvRYV55oY zTwvzTGVF)q<+7LD$l8kI%Jc2};^o)`u5{x<*1LJDupxX&UYxGls&d!a6?{ju2^ zd$SjN*9oD$uAO$Ko&j&w5@S(1xV6F9FV+=5*C)8P(vdrts17{WfPY=XO-XNObvS-2 z7b+i`C?fy4%0PHK*z*_H#xsF3*$H+|3dANM?+@+D_alG9k=g8R(OBS*4xa39U)|3i zfi_wi`}@`J7Cx1-10QKD^e6gOZtv=O{?jzi^Lu)Eo^MU}Jl~S-c|Mx&c|KCe0nbND zJmH-;$v2z*&fPWO{duS8=MgYg@?n(eFaDkO8^C!NQ=Vx*=$4yyp^YU!I$T`7FR={z zmi%ZpW%>5qiDl6IU6k!5udzP6N9yh!i)?b%jt{Oq`vCTion9|?U)jiW?en2$ z#oQ48%Rg^5{j;+264@P=R$M=ZI_a!cb7BBE`!(g-Z?lK4W^Eao@BrGTgBbQzK@L9a77>- zM!ujMk0|wKn%FnK)c5l8X@{_7jr=KXA-zo3zd}CT3U5r~IEABtgZR|V?{hSBEaO;% zUNnjH;0)HWBcl!?8+7kDZ*~o_ho;-`)kHUMW~%kW9r9IY9z}bCFXH@(NrCYB%+ZCW zD35)`S!20&S|GfZ`8qx4X8Emqx~Glvbg%JJ7G2FgNsLQW7HSKWUB`Kd^}o8;CqH(u zfM?ey zcWVFdhuyr>l{cC^aQWwM-azFUd75M2=jNTFJR?uD?0PrPyw{bdb#7jc>Peo0O?S9? z>EuPwD-PFrLumuBRborng#0NShQ1How=4{VYjs`B+yQp*G{y$Kp$WB{;ZG@9T(1;Nt5OPn z`ILfHTbO&3EUqW5S`b>EFPjr>s&N>8yv={+KlFU9QJqbHMc~zZ>Q!&MdE0ty-)id! zvifEHFZ`OyKYR%fwvP$!0B_4EW6tLt$!IVCJil$hHuXz@_cU|A-VxzlCCJq1YRO^e zd$5`Fo&tLfb>xfUJj1EG8Cjpd;O1?=G<9<9RVUPu%s+g`E!%!!%CqccCzd1g58rXy zwg*jlhW+ys%aQqKN#@hHODNMf$`|fS|5)_R;8pyiWb{;M)%2ssB(ze_3o zaHmrGrB>-^=te31cDqt=f1Xl!eirHN;Nr@(a>fcttzLM!h55X=%&v!5rJp^9ex^C_ z6xaTKlkQg^vVOX08}aqQasTAL!9k=Ghox)J-nr#{7B71i``7%LP^}CgN;XI$UXljwy8M{WIVa=b&UN>hpa|V@G5KNd;<<1k7@-E-S z85++n^^eRnS__?K|q|4HfSD@q=t$GXp&RxWq3X zQqiNmdVs%MfN|HrDSo^0RDUG- z=<+^v80iitm4z3c72Lr+^XNnU>spt`e?zMApq?3zq1VcDHoA2DWA~Y&_s)9I&3K-% zgR-Yxdj9^&>Op7MQ11iEnq9iqctYQJ-1DHzyQudjWuosZ+&bo)pxwtP`xANV%`?*N zX(xLIH2X63o}*0bDVtQ6wCZGaq4Q6u`xIq*N0N1qu_0<)+EOwhG(qd4OL{nC;+xEv zcx#d7V}>#&Mu*wSy1HGPrLW}MMew5fOZ%bTPM_(#((Suu_H%dUsPpY6&NT*BY}xg^ z*Xc*rc)I=gYx3g%m;a@2=05&!0N;EUWr_dS`($#ee$($!XY)_#hA$estRvepvf zOn^)O*4n>;R_I}~){p&E>;Avf*%aSR{-p8!kJ_*;?8COO-(3e7*W0RzMN$0(HrXsE zrqD8c3~%O`&M}-LpQ9byg5upjM=IT;8aqS`9W0Dp0lUM9gVWH*`2T0zxJ1&&s)(l= zqAgO%lWpNOYz%AB+3q9NT1wdxuJ`mvTjA+Zvx_m<>)06fpgTQ*4!38D6+0hWM6?kb z!+BP$72V}iVi~M)Z4KXH{j{D@gpS#SP2p45)^MqtKg+c>3?W~-)l~8uTwB8w&SiIy z&-nSA%kCgQtO=aU?jZlISCnsT4SDt?7l*ei&*(;(_PK7}N6H&29V){vck|v?p3#lc z?Xhm&+scz}1fPy}^WIRNu{9Lf-*EHHdlPI82aK(OwL)OYg{*gyk61h=-N|eXO|Gq> z+O;*5DjkR(Nxp5tE-hU~{IL{zmrRnbV(bp^ z-TTIFooO$oOfieRvPbZJQTW71tQ;qHQHH%JUM@QXwit(vi?`;VC)M@hcZr4if$vilHFk_sbhDT=-3_PcQXzhm^Pn) zj@RNnW@oRKoxRfr!CJMh6GK~FyY{zuj%@5j=&HZKSK<9TM+7$|1!DccI-h)=Q*wf- zr2bG=u!VM;=}DSf%wBU_cIF0OJtZ&bV^2EyFDx48=r@D(eem*jXhQaimP3x-F+ln7 z^(^w67(=zL#ujmHfxo4Yao6B}{ucVQb#Q-w3$~(`!{8ZxNZf_6{-?#uTKukBj}Pso zk;?ASi#7V`oI^v>NshBwV59gH8^s=M6l`$)JnJx?j}Df-!t;Cwu`b5(4_Av$L`R<7 zCBD52wq<#Wf?5|ue=UuVyYUbC+`4z1(X$^!*HB;G;?ltn-SqpUQJ!OH&Xc;tADzE# z6!Ssbr?Kak>l6G>)_og$3^Hmr?Ygthv6s}j^70)cS2FDjNu^I18wPzY8TqU!%djUT zmO&SiiOr@g-Ih+*dEY=sHm)^g`Sw|fWhIU*)OfJdR>FMFV?1l6sb63ZO{@=HzD(IN zQ3q>uH4sqBfWeEAtl?nZ7jy2v`o z+HSO=sfV9ls{Hg$RxeR!DT(VW_;*5AZM1dV#&Q8|-f3fb!$K~n+x-qZa@mRVlxzQ( z{!stiqLe=Qky6^>hot>Zw3kR8Bcs+lAQ_CkWb}FdJ)^rg_L8*Pvs-V%_m27R$Z9Jj zd&#L^vX^{atY9xTduN{t`I93@x5j-teqj?Em@&M3`l{ILGCF!i-UndtMvldd+i&Bj z^7^`g%OlU=+cj?`Ic(*Y$2`or7r${CL_bhVr z%E;3?fDMdAf5CCYn>*+-#%pISEMN{X$L>!(joF-dN$l(CUb{ZFo%|Dkm#nd#pvU!l z+tF&7xZ<)6X7di&)H6+*VW*ok-HuU5Hfq_m)OY*2*8hK9d|>T`C%Jjhe2cpuJbqtS zcLRSk=M<)T1m#n{2nxL=%(aQz3BAXl+dA@x`ZOvb`5QB zvw}~%`>21;cWE4l{iIp4?Kan@+d@8L9AbTF9Yu!SifnP#O)|~__mkm~_sLT`PA3)L zj^RCKd_a2);QSA!Jlj6sEqBVY?D3R6W6Buc$ll7cYT!TOG&=hZnc?89&k2mLx-_jRyFFhZY)^6~jc?`uNt6k^9KCr|v{ zO?l{_g}3GYQ0uPK-1(}PDA)bMWFh8;*7uJ@ba6 z9maC#27a^A^ST~ zuKFePq3m>ucX_Nl)86Bj@8sSuVL`y#si&={jSUT;cSSjuEorw>{_mek`QKeivxy<4 zv{xYZ3h7PoiqWC2%Xf6Biwc4*{JU()#U6u8pZkh?Ieu4MOOG0Qwu9Gy(Q{}|#w2Dv z<}cknwBJ!a?bd)jyNNMU)SGSMOCG!_!ylzT4iC)q%ePIa+N`jVOZaqE#V~vcG?_c1VX6|);xx^Et9^$6)O!Z0jKj&?W(a++qQrFJ* zEB?`}L!&R3gIn?$mCT6iOLrSA%dnG4g`duvC)SzK55mvyyI80AJu26|JLxOo=e4E` zpZmlz`tusfW|^`~``?LW^!Z(sU1rMg`%Em$cI-JC5AN(6FyAOl?gB;|8jtHU&}g*L zhwa&tF9Y*@*cXU3ysr}bOG!M|aF)FZdQ>}TjamV1p>?SSqh}@a>|EBMI(m$=zj~H^ zF6|^aDcX#uvM&S-?GQ(tbeiybb3(iza8qlZ9T1VE)2%Y*e#1QL zxHy#^n}zp%v5}`U=aa#lPbRkWENtl6*wSI*Q?53C@c*@8LRb z7tp#d=efj-FW$yF>Miv2n)7;}`O)f3`KEqn8ynV}#_Kpy55%DoFm^f0iSE z#_0d(SFI6NY~%I=j*jpR<c}GYB!Z44r(QN?S@zV# zGH6Y5>UmR^VNXgdgB~TPo;GFq_V~mycvN!gN%G8pTGps{RhL8KYw_lo#QOZ-W0WgtGX0yiefIpR1gIRy=CSzjKk1!ib31H^_$!^l420 zfx9MTrWyAyHFFRHnS;P4w6hTzSScBwZFfLlZM3g+LhVa+IqfVtc&ElH1N_uUxBUkl zIq1aPC-yXbpgy^iRDR1^8|>$Qz$NCZ0*6zLy|8@sjvx=)kcZlbPQH3Eq49&qL1 zgO6iR$1hC&xDPVNJfGuojxvq{4qsOvYwvu+8nN@q-n_-E#mlKVh3iuHpCwaDXSYVTpM+NH*d7^B%{zha^1XdD39^xelXx>HwFW1FwOoU=!ItgkS2Ey zFN1r0xAvbVJzqNdF7R99%bw)FM_dN^Ay=BbJUf*fr~QpYI@^i}&&~JQ$C~avx=loqO^+m{xD@%eS-K`*e@|j)iH9Oj(}YE3ph-7N*TJ zW$dYuSVlh!ziuM0(3?D0KH(boj9OH`?4|$U1EKE)i%P(z2E~B!Cav9%uBQ0SJDF$O zwM%x9411Sog@0FU*>U{;%FUNDI3fNADi$ck`9-v?=}2iVCT|KqTz3t|8eX@8E)PX)svoy{?p6ND^;Gc z8|2#E+`JOyfiZ31$2MUGGOJ!Ga;D0~5+7-*FoX7uLc=4OPZF(pc!u!7EB%l%#XdJW zCwL;hmcCiyV#xvb+ObdO+W#@vto1q>j%@83^tiDG!~b${dwZ{5;l=7`D zN|A>Vlcw8$COsZUPQDH-T0;CKcP-eiC4o?cJ#xMHmlb-vt();ZiDQT8QM`YhYiId@ z=bIQK#Dh4I-%l-bWSYJ~yt)ZY5TE~SpTp-{$gd@@{UbMK%o2Ew#vY*D;d8|tabkA=%-qZV@$S8%`JJ?% z`244)EYHqNETfIZ=e4FR*T@G)SK>ZbPq~e zWX2IGj3>G2tVbC0*$N4$rj3=;HplLxO|K@!@BW%b6+G z+&P~5JijLpE=mo|>w_&vu`I4+yDixvwsc8G7fBwL1TW(2meGei(N1DnC0H~9jOvbj zVxNiTZ5(Td24eS-{+98>1*A`o3&eVn>b@p)8pVTKgg*0|$_d2%PLW>O*OzLIY=Evd zu|8-OG;}9vBlJ>9`WXF`O!_#q`X_AktD%jblCGH+2v1_(L zHMg2`W6v!9OI~X2-BXeEb2c$AtnaYF$C+i;+$ye}I*A^OZuU(+A*9c?Y*UBmxNc`RxM|&y5)Ms3*2&h@8uonZi}#?RCB(9^J30@x2G+i$@z=;T100fGj0lmzwq`M zbol!I%QfCB>|xC%77}sdGDEGKdxid;o*w$_87sDTH~TMndxUl`$q4_%Kpn|db_ojaQ-m)6TIDC*WSBL;OvCyDobsx^6$4|f1~d=$VSfpiw+_WFz!-%KiGzB z-2M!9J@6uuULG#2Dm69~_I$u5^7Qikm8|oV9p;KaY$SCgFZWksgOSW!NK9#!tq;V` zNGLP3HtX|%h#iG{Y2|0u6>+uEH?j#j%@xN zKLVw+Q@%;jBb3f$pEmAs>38dK^xN7o{X($tOYdv%X4WPKPFj{`u(yn|BWFR2-ios5 zLHsD;RsU^;eFx;8Z({7=;}tF(jE^R?X3G88ui-g^ak28)V*KpPd7p}y_RSSun|k;c zFXA5E%lzSV;eUt_-q4<5774V?7S8}3bp==M&6D}S!)}--{dG;3$1Wuab$92aHMneUE5LUiFM?2WOJl@Vq5>roESb# z^lP|@7&zuV#5q=sg{Hcsw!6 z_?FpLcoyT!BK|ir33>yU>-XQ?`Zt(t+4xJm-h2Ill|}coA*W(Tnd$|_~MhwggVox!+VChbsklFfB~PBB-EKfohgCXX>OedRR^3A z4)34F*z=e01n(+UIvd`2m~pVyOkCAZb1m!qmz4z7Po>POnCt8M2OmI|-E?)Y;GC;H z!HJ#+mZ?8A4^v%%jly$}nSXg}l%rSu*!}JwC<9Y1c$b@%7lnuJA zdgH{5_R%wm=aqz?n-719LQhs}C0Mae8RN0x$dNc$bg!ka(;!O*lbKB_l7M^z+*HLe` z;Avb(d0toSWf5;`uq)RmqeIz^uI&N_)Op{ws8dS6`ieiuQ>>QgE?=;v4XdAL@v=ea z8C4$NAb2T!u!|M^I4QgJ#X}v__m8UxA3Uof9Kl~myd{j)9*uAE%y^mMw~f7nf7)d| z7&y<`F!DUwfTJO4V1MhKE;~5<>LUs5z#8%ObuU@L7I=OpJg;x?(;ogm_I1ADV)g=qM%7j?y3d`%Gc>Nc zS}9npf4`D*;gIke9>x2|)E%2HyFPf>JNOsmV!X`c?dWahoXvlYWIt5?!C>Us z1DyZ*uq61J?%^Hg{eQ^_?q{E>Qt1G1W;(HsZqC%YhpiLCSwR!K?z$|4w{yVJcz>C; zIyEckC$9xLHIu&Z(HD`r))<+y?<%?nJy_g;BPQG(bj&qMc z$GL}p=X~f@f^^{O_HR&2~;7PL&(9{E9LIg~B^^Vm1=(C=1 zD}B=X0(qM~&kK|N%tySYwqnja`idvq@;dmjoB5KrIA0AOuRlN3GO%ALN*jq^uY*d+7e8Y2ZOD-UpEy+Anj3!5-~#LoBdmktJ5l*xc}G zQp}jh3hMLyRhnOc?+W>k$dW+##lH1<5pX)G&x*VTu)cp+{8f-eMrS>BTb}k66Na1m z+!yiI=lRrUlRPH&i^fDR5wA&OUiD3(YfD{9A8D?1Cp`6YbjqFJuIx|V?v9Vu0`l}7 z5&HdM+F&O#Mdfvz%WiSA?#VBQKe-3FIG(&&%F8~MCws*eV%hyF%eDO~V_fh|Vp*liGVN0+Q{PoH z51H6^@40=KP2W94JM!P}tfvpPCg=8fx3%Bt6}IQuC48^O4|ihYj9;J1by-&{_(D={ zYXg1w@E08;BBy>7f{uQ91YSmFmq8=lppnR6Fcu#_r9MxftW)n^niJeX z-c0CbJ2v_<__6GPoM4kMZ3tr{FkC))Yr$6I2L}(^oU=-^Kf3zSXvz3cbT|7LGv;i6L^L5nfA|iSP%XKJfBtJV9T1%9Bla~`I0rF-w!#*hP+ok9I{7Gz_$y3 z#JOsBpU1+FYDHcXE5aY(XVAtyk|BTQKBXV%`$=Enn(@gy^vP5(GGia@GMPOJ ziHS8rI#nc{IhuuDzjTvmqZN`aCVBiXbhh2-X^bV!bK~Oz>0|#y7SCUYj{-iYk{9Tw zQ>)Uf(doQLdhQ0E`wY*JE$b$ui>0>~6*D%RcT+px+`MDnO&gQ?1VY}2&JA@*MF%{a zZ%1BjPDZc7rYt;=ZVGmUKNk(~&rM^l|A;hqj99;+>nQd0 z68g9V`6azFhOV_A`Vh`{GniQ(u16QXg*Z>$##HwgX5Pj;uy{;yk%hmnoZ{%Dx;HK1 zp6j?L&AsPW$&T(8<6C7b(%eW~cUw{#*WDbS?@#>rl5X||cQ*Q1_b!ahwI8CtXF-QE zO05z6=epwS^Ma!N&)#xutET%a%|-#e0G(t8X4p2Rq|JY2;yL~9v* zzpl3+TgLI7KE881GGzyC>hag~mw)8W6y|XGpCk@1hkSC9@ws6GqzARX?M~|I|7G)B zKwaj)9QprmSN?Bz`5r$0ob%DApr3)ab*|scKJpy9aGo9ET=rYZo;GmFkv+^exw7XU z%47VZI&pdPq4JEpapt9j`!6fc$Qvi`8uDHsPuM@1)WLq<1NPrV*+!Lt{pTi@J>blP zJ*6_Re_Uc2*#9#3*(wA3$0U{s`zdQA?|tG2ipRy@iM(xL_vz%#j;^D^@cAhQ^D_+Q zKY=Vc#bCa3-Nf}?_*!(Tn7YCw#kV?+KA@={(1+-^4H~=-e-?Ml+R3-ok9Le0&vT^f zG@BlM&6p^qBG&Xk$A(uJtDGo9wpSuQz9c&`>=t;;86RfY^0g8z-m3nVeuO@# z>r}U&=aKLDU}f0fH~oz4ofL>k{+y^|Ttgmu;V~UxwkzMe9+U3_Xp4!|Q@{NM>~{K; zXVP!uDXUZ&ee-r=8GUmNWn)#gJ`j5?vFu@o7DiIG?F3zG8}fQMdCl}={oR3fXU|4Q zK9|!sVD{c}YzjuNL+7hz9#D8I8<(&6)nm47?CjD7Bd=I7m1~V;WN*fu6DxDUHy`U7@d|oLS^Q=43^Ey}^`oM5TPiuqHBAA%janIbY?IO z4B10$u}`t1_dK(9YD3DxwX1pr)>_HqZY<`g;F!(PkTQ<^JD>P7_wV?kV|oqmxYv^t zTv+beTg|4hEFZWPy;>w}HNy7r6$z>J(DY=30h38a+rd?DSnphl&E%aEewVv`r4W5d5n>>~C zwtL3Uvy#p^RFrh?p;<}i9jZ;5a40}LIgOPp+F$#x?3?mZlisUqr48xKH~pypH0%#c zZ@P16@Tv+=a13pWt{*Ns;hqj}SK=LJF~5?{JWCGqEx9}of7$uO^xE8oJvN;@mTtA0{J{*K-@qIebW~Q&+*Q+Q)&v3( zklRb>FTN#p<>_VS8E2jm^y(SdCe8JhGlCueKvv^V+u&iX&ZYrDo9mCi<(XSe-<$TH zJf1#hUDo8}*n70K$-8JgYx1%NeM&6KHK~E0zkvPfCF*yl|JT5`_jc>PqzADqe^!bg zqGwRh%XiVodp9vgz5krR#wR;EMt6A9Ypw>1I*z8yEhW9MZ0+(s^KNV(hF;lc-q3dQ z&Wn;`|ABwho=wCl4#7u>a|JIB!e%%B3fl5=+VnEo_EOq-26I@KglKc)kEL%~>sj#g zpJ5Z~#WyYGn|db=3Q8Zk7az($FJg@1>E0T^ruepJ=Ek|6_R-_mm-H0Ab=(&zb0f5E zp!~_@o*&O`FHXkJ`{QAq-}ZL-|5$&~|5*RU|FJ&0=>PQnDgR^r)c>)5k3e|uz7Er# ztM+x6_N*rTs`h+@zJ8JQ_r<-x72IADh>eC8Yt!*z(AcSxb!0`_Lp{enL;28bK79*5 zzLUq{V}3PK$hvW8cHnf z`n~21uYzxj+Xn@o=3Rx!%r!9YEuU(Am)7paYLY@>-hn&`>%JX)kM65regCuEBVR@Q zgJUzj1+CZxr~j6^@P63W*BTk%pJpNJ8>}(MeUEre-~I%3XE7EC@ZHbT9`?>Cw4xX= zhqzyHL}E1+p&yf8?0%bkjAwn&p?OcEduXgDf29b0Yx=cxQ1Iu}l|1+mSuGkAEgCu` z#;o*|J1B3UkG}F5)u#s+QAc%eaG&)jo~7&SxQ?`Weph|*L;tB-xMjJI7{JlNm7#Gd z)>E?A8kv06DM9=(o|@>rXIZJ=3MIZL*2#5FR*6W*pdT?;? zZ?r*)-c3L8Z|^K*|G=Hhn=d?)GFNlyGshwuNTs*?nb%CkcjV~&7lU|kUcJCQtJfpH2z*MZ5fbE8$(P#*Qeof-gyPIJ1He~$@M(@zVNv*>>14q zJv~}iqNghF-bG`A_YDbze(OoI&+u3q7W2=Vr(F{668lnlM@RDuj4fq9B<^HuE$LwT zO7)hM4=Sr6tzmxUdN8x*I?WTL)!a91Nglev+`AaFujzAdUMspQa%*Xg=S*zR%co8{ z{fXt>`(h^~O*^xqtQ8$Ht-JII{394E`Fe}aJ*~w$&(L{FEBfhlUyjbZv|gq20-bkl z_2|4v=c%nLp?BW^op)<3*ExD3c-8Dn#?K9T>hs|zM;h@a4KnS%Xk>8E4d=urB9{l< zkP=Hf6JK@CyFHN-V!q7D_gFDQBOZUSl(@pF?^QZ-{5@|>`jqLrPLrrVwV{JZ zA4QK~UxS*mLi~MX8#dR`Nn!eMF2&#l0I+U+-srlvi%XsYHH z(6Q#GH7-(LZoL<5P~MX3=CrE~mRvWe{hnW)8)MFJN&T<7#Ofbq9)-Ec`bWFO67@Lv zbgF3G{MSkJUn_U~p}1Zpeb3ap$ema7gTX<~XZUI+xpQ*kp_%GF_vCd;u`cql<}Iz3 zeJ+j`&hr(}Lq0m*26KJo$l%>vZ-So8_0loHJGn+S zG>@VU~|iub|$hLmkey zt{WYk!Sy@z(^s`yVvJjj--i5I--lh8xv!kY;$ZHS;Y&tW%BG)fFVM5m4L(S+Vz+WW zs;+|kEL-OdMEujeMSAZjOQ7fG~d4^DRYT$ z$BHpqtoGyTZz86mbfNe=9_Agh&?z=xN4Sb{n%eX;@)%1S9Zlbp_`Y`cdk+2|-rhYv z%Iezxe`W^AB;lTL3zsBBO9EO&f`F*xf)YW+1}s+FFHm}FG;%1SwTc!ndJM+aJVFl@ z+5*Lv%up&lfHK7%ptKc>tsqufZA&0ro=H&PaLaIUe(%qIo(U5K+n(?D_4{ML_Veuf zW$m@rUVH7efvI4?wCA-%SO{IHNT>a_@Q3`-y^DW& zZ;G|E9(oe2{N-@=%dOgcskB>YtN0?VdFepJC4c9fX%qdJ4!+%#hUF-5_NI=^A^Lxt zy3;>oAfoB_G1jvmYin=wXVqB8CJ&#ib~6W4DxVnsWwk9}RW3)riZ*f9k=3NS3cb3L z>p9Gw?)B(1s!rJ^1g=KhPJYd&1r5B7?>&C2av5~tD)Or>b17GC^U*f7 z4^rIyxeL3u0GFeI&r;yD1b8iGpY=Pl#<{mW?AP3DE&1~mM&>C!jQa0G2jIC&bZ_S# zo#9Bgxm)-H!ddG>Nx!4y&sVjCzehN{k{kZaO)cSnB)lFt{3^JuWb??D@NI+_0gHNr z(k(j!9CYb6F8>2(bsA^pKM0KNg>D-<$~)u(kwH&p?dpu$kGvV)eO(%3&0}%{B?}de z75|`lx&4aYXT5+NwEg1`Hlt^|?mL4^Myv+c%rE9%0XRzKr#)NL+z73H3pb8<_QQpP z+E`yu+-;<*B;2H7-Q7++G;8PWtO3D%348Wb(&dw0X9yPapnPMmcq@zjaF++;ovu%o zLF5pp|Lj6*=Ix}-&bMafFwfc7qMrs_D;&OVBICWr>UpEt4`&aDuT|P2blH^l2d+Mx zNmJyeG2cf=n)K%TDmR?;Ypmg2E`+y9O25NRAIIShdkZN zf1qDF@}l|h3gRQuue3s`oMG_}UrlN%=Qa5ir1MQbKf8o=pts+)KBTc1PN`?^KN|7| zIOQaB`9Pl)caH7z<2wg24(*YB8v1t`zA87V|1V$E|L^fH8lLFC z{g3(&9Wedh(%ZJ$7SX?l`sLO-mmS(Z_uH>NNO9_-JP|FM^)B4?N%p9|XV)Z5S79*v&>)y-IrOHyAy2(U`it zk`cV$8(B5`u7VAt&`C#E*eUfzemU!of^~;(39VgqTfxS0*v(yA8hUkON$3|_i$kIN zN($gLYlAIvuwKn(?6?z`-L`y%If(=IlxYv`j1y+S8bvOR+A~C z8~Xp)jNLRTbnNwesN-(x`eNwBJ4LbBwDQn7-f25DzBtkdK9Vjabf&yI#npumuzxv& z{QFPX*jkQz8utk9OSs{<0xypaFx4~%}gk8kHG_j@^bqt5#-<~#W9T_(+5XwJ5$-0yjOS3mW86JA7_W)6|v zYtJ`D_~V`%?PBlz_HTc2?E%$sPYl^!ZL(}Hhw}Sz)~PmMDt>Q&-4o=#=++z1W4{&H zy9F2=4J?)dlO@n5twcW1&I%GN_-GP76LT(;x z%IE)Jpi^jnkB*^zojQa*sX~UcsRuCA)4-+RvuP=I2ha=OyPI>A3B^XQ*1)#64}qsF z4EAM@0X<~Qc39i|_F47^3mLCNz~|YG?wyBv!uH~HHroK~?&oagjN%D<=4~&nc>CS# zw}BI#-PFytA{$0pu{WeUyovuxE3)oT+mQ6l*RDuq9TNUJ;d!)eJ3ev}XDvQ5D`zWy z4}8wD>N4=(#79Q8@;-m7wHjhRwXfaMO>>0aYNtCGk2^yLXLJ6+ne0EBv){aJ&i-o? z`sN01M&D%KK6w~C^eLG7^3BFJdL?n&gfl$bBqw(FVomxYxzNJH^tGCOnc0If&I60} zCdH@kvjkcCVU_t_$coa(0&LJkng1yLH{WykW^81e#$n#IpN^B~4(7-3)bP7I z9rs-6>x9)WKpw~(yzPZeCCu#Uc5#m5g?*VYbEXA8`L!1|NqL`>EU{PkS6NDJ$0 zfFI`aKJ3|4#CboLxnupWA152q=fAy+^}oK1eQ2}_8R}Q+mN=hai?OY>zG)hK$|mM_ zK~e0~bI2H=r?g|;d$`wd=x0dwwodY=D@9*|y_*_X|EIYZp!Vk&n$yGFgLS51yG8d2 z=tGtlUd5Ufj8*eCrir638D3oCE?y_^-wzI|)_tR7+%VT_vAOg9pIzLb{pm&NYOQBR z*;_mu^0t>w{i&z@K8+9SAx^mX!t=(@3ZIovzhmaVTX+w?@XGv)?$1m?pLQB?rx=41 zO{t-iT5tTW%-Kal>O(R)xj^clA(kC$KkoB2kEPduVa z7daPCxJzGxYImdcLyr=8OI3(U`oKCkcA zIxv&Epa%%9+Xm16I?k-=Wj=c{_170!>mDNveWvns>D93}2p;HbNAab{-$cLXcsiaY z3|;<5ko_!tx|97p=Wb(t3l~cV(%4k$Z*hj{>hk{;J!9#1KC3!bfv=LrTy@1?#I6zg zy?d!gy8AlEHEBG3+C&;?;L%+4Y0WsJPn)7A}SoCQV zdcv=JZ4#W+vmQm?bdDiihHc!tIC~^5q_vell=b)yd=fSkqSxFLYMh16LP2mz6Q=zO zt_B}H=Hj4{gmE`0k`K>8Th?TvH~a{2@Gd$+IovUbukejViBBD>upg^c`E%vSQ z?g=#w@`oB48fbFE zi$$?nUK;4WYy^fi_sx^6*q^UtuI~eH;?Ks_4@tFuaKNqeeU-PuiWOZ!nvL|W$rm|( z1l?lBg7X1G!=2Oc!Dso_*v0X|`^bvXK}HWb#UF9f{l5}y$Sw+e)=|C1=ydW&q8Y>q z#(LlbLt4Lw8At4}7#i zL@(LQ33rvmLvR)o)4H50d_{W`y2-M$%6VJl-me*({$?ID7KfA8AA5cmM^gUW#CJkZ zS!E^klowf?kFnh zPX~N9V=WrIkG1#|bZXxU*#Nkp*Hu9nxG{E|G9`o7Jr|?TJ3gWBlNpx2&z+=GyaT*P znOBo$+=VpK`?-ZQqEow|R}IvudZZI1y`P&E2h6{n90z@ShPYuSU$5|QljER!uMl^w z;&$`yOLE)&90no9!EP3o2o?{MTlB`FvF3;BN^d6i^Z z`&hG2v!)YiuO@CEYdASf@}rZp`i2Y~9ydC@wMM6RhWl=t(;(XmDd_Y{S35KO3^?#O z<1c+yt+zes<>>pTp5AN^=0ox^rA^m5V7&HC!{#%6ap$Ici_V>oFbCX{m>WOuE;wTn zWA5HncNiCq>o;4*hw>O#&BtEFF?KVO;}|>5#|Mhr1#grbH{P8O&0hvI@C4&p$he+l zT&b^s)}QdRZ67wR`yUt;V%`cs}E~cYI02$j}~)Q7%OhTBqQp|c34w}8+#X;+d6ArTe8egw}hOw_X&siuqX0O&HW0ctA-0_yn zXB$3&`#^@)av!LXw39tP!0-p$3#vx%quTGV4=oLH|W7l+U|k zbX50y4Bhtb_eh6^{avK9bh4^=vu#EYIY%n`Mt;tAZBaKmxo9^4IsR|zdD8nG$^T^Pp)nE=J91A z>Fa&Se?9H_3;r_Nw_E@8;ZMfT2yevK`b*yHm@-HAHt)ptXKrlLXpS^)=Vwn~&dZtm zF8~8~0Sn`SlRJ@l+%X*(F|g7=x*#|tw4i9jrugvCsmI{cxzqJNa%S}}1ipG@t8b^| zC@#O#Ej%{ruK5jW7in@hv(P@tFWSt%GsudOzp$G3wgc?{UTL_H|HoC6ns4y(*O7l2 zFeKXy>#OeJ$i&>Ies>dLg1sK_sqp;kUjh%w#`Ut}zMYdB+`3;TujsJDy(8>bY?#YF z^(1TNPkyUhbJ!02c0loyB_rvU#$Ig1sbTQBLwqBS9}gDf%jVqHK=av=7Ww@o6LCw^U+)iZ;Kye-X#Y-G^ohPEc#n6&XL}j_616( zG%Dx#7BVbGH$-zMcu3&c#J|c>xyk=3SNQP`%>(Ti zhcDYA|5)ytX5_Ai7_Vk;{U2xjpJM&%y$;zIgRq2I=dsr_Xn#5~0?Iya)>KBgknd*J zkzjK$-)C8?YRBjK7Mxs#e{O=p`h*MkzL&Yt8tTHj(3%}cn9gC}GjaszwikW{;W{__ zq6zOFZtI2jBV4@j2U?$$^P=edNGpb69sV=JHT+92>5JySd-ys2wJ+A)%4TrL5Wltl zXd5e2`s)ihU+$@KJ|ADNF<*=yr>ytz`zZXA_<9S!RsJ7X>w4nO0oxxg0jJ%^y`MbZ zQ=VNsq9<^!v^@xSCa^zS4DMTOt>?bN&|TDd+kC6=9Ak2t@u_>Pc!VdA|v3qG&V-NOv1FYB_Y`~v|W*qOtc?9^UC;RvlZLFcJr`q65bYc0fUlIth zx30B@r`px;aemGEJJHGPQ`cC7i(~9{*W~uJ>}~^GTKM0j(ONr8KbE1dWB3d9OIkC! zPa)VdxB=YNz@BG6eLb+GG&F#IY=M8*T%QIOG*>^Sto_ixA?>Z90mdg$ukUY!Y0i?$ zRh=q(UrV_?kiBQJm+WoyAi|Wr47?y*;oHl;FDb9dM|)jc20g+jb_YVr>*_1?4lnWA zr9P=`YSRL2KD*62|Ya4mp+!T zfoZu#Bbs`%_wl8t=^wf52=*~ugMYHVFpB+>l{)%{69blJsV?sKMuPd)uy0cS<{OG* zs(aZ0Yh5|`DtoSVEVr*UQQ>zQ8q((2iayrF6QkTd28(BosIEaSLSCzAu)PI++}tNS z+G`opoEqNFVf-*4U}QpxeqHCnJ$EhK@h|blf59VGVZ$}poqO`zyUuYh^83hr1jkEz zS~Hthl!oR#!?~ZY&CFu_S=3i?EBo*&WPsVg>3C=acp;Sc`GPaUiXtm2uPIQ!goop+ ztjMOV-8X3be|6Z!)t~p^=c|-)Eo)P6+6oq`JUddowVwIk$Gi%^UO_wB5tfB)!i0lI z8^Fop!7aj9f(KP!1#_=@I#a&zt9aH9dLN6n>4~&CmUJ~o!s$wgYdVQbM%#87&S2_O^g$Az)tnMt@CDF@_M67*e4{Nty8R}$ z(m-=GuOD&Qp;LMAUjbweDSl)vzVp~~jTt!tG6Y99OOTfer_8NtyLxU_o7F+uhAh4| z_d8=^^Zj!p`FG?m(S5;qfIT8Gt1)c=X7&S<2k7rXU}Ugx!dHww2v zS6|gfKV^K~jA!>Sd;NDCyl?gO8yCXM5Ab~){>Y4N#^{jN%Q#>UI#PZYzHE{jIObol z`GC?hFJ|tb1>&u~M)(B6Rfb^JLC%mox6cqij(F+C{FpcR66GoUZo*Ak%{Q?93SmVp zVLihQBi()bNH5H9x$l;Ve-(~4@0QW#=q_jjHfM9@efLKAxuZFHyKD${RV&?bY#>hc zgv{E>4ByC@J%a9mp&7YDgagt4O>NS-U2yMcY@G$0rtqyiZ+m&JB9HzD;!gk`8KZYy zoiW)}w_=mH11Pw=jX2>VbXCeHLOWeN3tmHy@?yO96lHC0Yjo0TtuLqBe|r~R0z8$& zIfWTZXxWLRGRoR~x{QY?;|A)Q3s0fCtm5L>GWambaTEV@;T!&kF*Ep=vkdlyYvz^| z$3CKO+2;bs=0kt=Z|LF_zdim+mo7Hb9@*VCd;|R*PCZvsuIf`63!hH4n;8@9x#H#o zO)c{BM#*d7lD+5r{^*Y&3zFS~D73fWW%ir+V|)4R+XyqVE96!d^T2*~O>i}9lkls( zvflUF@k{P;sx9z0u~hc^O9R%i zGW=2O)22>9whtT?OhAuq0&*et2AAdmnCtPM<^eW;m3Iaye+p;VlR0yco>mih3fWj> z)qu|z)K&G^@E~%dGG7sDYpxEK+ETZ*#|8&{olIAR*N{OT6P}A^AWH;Cnt!b={u}*i z;l?}At+|7JChinGk?=&8Cs(cU#(f|C`3^YSrA53O;L)OQ5vIM<_mS5m`f>$r`KlLg zXcYKnG+|#MOzjZtD$hRJCOY*s;=V}S5c;BiJd2;mr?SR-`N}wNP0pt>%ZSr^t;%;3 z=NyJldoR`QN4urVqWh3T4#o=&|8R40Z1aKmx>65zr;depAZ(Q2)@pOC;WlgHdt)!G zv2Ei%y*^&?%9WjD@ekSTA6&e~nDDQ@453XYz+WfXSGhRKjeCZ;tJD|fw(dIY>lysz z#=Sz^mBcM%y(gEaK4udxe5mr!&9oiOoxzKYp-1ynZ(Gt`z>62!)5)u|`r6iYsy*;r z3pr!V+((<#e$^N7>T_|ai#Jui*0=20oA>9YUAV`nH@nv&Qa&4-sy;8b++TjSJkbhPuC?|OK616SW~Z&y># zln&N<^Ur?Z8MURowZ1OU;|;Bw+Z$aT?pK8C+*)P(uPljaPvEYdZU#=;fOkYc+`h>U z6}%kl!z{hmX(_Ajz?JkPXG_K4iRsn1^~%_-$6KMzdJ#*P*c)3gRad&mFEuY`^<%*B%7XCK(mp8*3p84>W z(4kqQL+2{30(ZQ4D~h=i@4bK881_j$*(dS72r|@Uz9~>F-bUlg`+&r&{|BJu2icQ< zC(PJ$rpNf!`VPQPC-Yn6gJ+8bJFhIa*s}(YzFMxm>-CI@%F|r$hZk*&|2e`AQQnZL zz$I^DG!1}{ed^o0n&3OnT!QCJL$9WBP@j-^&NI7xk@_Rn`CRTIYfm#TWA05SCuN5a z;ziC}#B(;nb2fTBr|bbdUDI~;(=}~Y*KO_oMpI?a0=<2sH*uZ3j-k9Z*7%0!QdaLB z!&`27)H8s37AZc1_{@+4j_}WKv)Vtu?dte;&X(XaZ$qyzcliarwJl?C3OI-6C-JV* zqwkIhk-6_4VoaZ=&zlWR=oS8VPe!(9hr3s}oAQ4AqcO31@QM3=0B26@IW-5{2@~#a z!Ecl)7+g=di<4a&1A@WL9zHU#O1iHRwu&(4nt74vBFoM>HvXLTN=~P_Bj?^EyRgfs za~1Fy%mHshQ-ar4FK%Q`qQK;=Gjn#$@uxNQA6#~(igOFu$5cC}7+cykHwr&Acsz2~ z7MSMFJbhDJmzut1gy(zo?m5pFuBhVM&_-{s^9tq8Gi`@PkT=2mK1tYq;$&|oIsG$) zJ=&7qU6(538-(2epZFWT-SpkU&CH8n{{i9*9|j+tYCT=vpR+-w5qxH`b`w1HaoTs1 zIbU`IJTz+rEJVZHMr(#^iE%?d6uXW`sYYU8ksDZRtx}RM#osN;1Pxv#fUO7<&z@gCbTm~yyjY6-@yJb0Jb1?Vndk$i{X=~~!SU{M=FVPu2H)*D zXOf?Vd!O(k$?{)Fvp2ls@@3N9Q63Y{Ugf_C|6p-^=qFwoiaX19yi{j19XXrnggm}8 za``UE=er`O=UirXCU@l8nzI?brL40Vy``+P8Odgo-+m~Q-m-T#)9t(L&%u@O{^jf8 zNB;!f-+&B#!;qrLxA^{IM{C_kTox_{?Oz9Pvo3?X{d}+G%~#1Dhw%M+Y4;7Uk1UGp z<~*m4GaH@Jl%cCw*8_fodze3>PxYgqS**VwF_x0$zD}6-5t9GD?1icC?)ghrc%c`z zT4Cdqw^#T@FRV^sV-%JVe%=dPp|DabCL66kKyK_Xf9&hb`Akqg^Sh0`oc00LMotWF zBX5wi8Dy%lK2P;nl5O>ATERX}ZF-V8=iBR-1UZM$8BK#ZyTLvSvfM`%$2w7Xo_D76 z4Z=)*v%k#GUCu%GTOAKYJpuG69$=2X?*y4Bf?<4)*5q?!jFB%b==AP+9^V0RE+c3$m-(tV)p67HAFHe6g)WCk_{e3QvH(G5X4Z6we z-rzh(XD8Z^DIfU@>q@`4p$?n+7tVCLhkyLsH$%Vk@)wd{=Q$So%5MN;uX$%WZ_F7M z;@sk0!E7JSbe?4Y-FFW*u;5u=1O9Zb6aA~(hCIs9zIZcpEjU{U&T!?x8DX36#sT9B z^5N&WyO$Q9;}7XvC+eN+H17@;#P_5WsGSbyI`Q7Lo4IfTzEEA&7n^mavw<@c&KQ6> z-na%n_Q7j30Fx)dZ?bbH8S!ag;vjTqWnfaX^nrdBi-T7!_!C$5qqIt+vQB`H)So%v z-V7@vP31uxn9`Igt_OwS>f4U*g}N?Pr@D8 z>0eRocKl_uf2@4^Q>Jjn;a1~khOaWd-m&?|_#+5>t{(&bD0T72Cip;|3F}5%gQ_s3vo?a4qh4Wg~SLfyFP9E9gTgJVuWE|7x@6bhLGtJPFWf}gR zJ3e&VmqvP>GZv9vIA%6DK7nJt#kXg3yu1XQ1&k!l zk!zb>d?QvS9MT4)dF+GP!nlN_`s zdjGPk$^r3u-M<7TIj`&<9*T^5BRKG^a0vL%S>lUT^SundstEs8;KR8R-7W0!Y(+Qv zHSld6)bbR|s!Ucn;y!grnEKrr>IH_8B+u zo!$dqlY3*feSmu*jNy*pQxA^N+u`B^i-C_H{*CoSpX;jbmQH-DyEw%;*cL9m3BLL^ z=CF*p9K)R6%G}-pe}Y}iyi(@8gt;%Ceq`kD&7GU*3?FwyN=B@yVtgjkf9`DU;Jn(@ zow(UzuNKFOi2H>X_j|?7CT?kQ z>;&*B92oZEwkmF=FSb!W;o|A)U-!rAR}{z2&GN-^NWaueyFqEHn72j1Wx!f5Iv)4> zG&hD`h1ZXIeS2-khSK_YqP_Y~Y9nt{uaA4}dPLu}19*G4RsHC1#1nNgp3Va9M96=U zwIaWUwIcsH+Qc|HD~Xq1MLd2zb5X=u!#C-JGozUOg7yV{nx?UKG#84Sp}6~tW0}M` zmBq2e-wv{;;4M;MPxGQQUnh;iR z`w+%oX(p&X>d_tro5pUucmu{<@#7RfuGoZoI%O^Qo>=d-)`{#E{jV;}iRi&O`dv?- zZ#DDSCp=m{VI}fsQ?C5yieop+uPKh*D4#LBK|W(TTt4%9z5H2>6aJYaC6NhxQm)Ur z3=XjB;}d_l-R+CHD*_+nP#66AIs=3BrKi`|=mh%9T_o{5e()_gymg;~x%L+9MsJ5c z!H>KYomL!6OMPqcWZpbj!PrEXke`3OfpE$ESM;}i+?n4Sa~KSJ!W>x~cwk+*;hLxK+57xD~jixFxtnxCOWuaW%N-a8Cr46YP63O5or3^xRK4Q>!_04@)gjq8o;fy=;k!lmOB5^KT_9$2PzousqD8Zi0` zC@j>D^`Wa#P_Tr3M(s4GzimwoAmdN5|AY4xz|Vzsycg_@fwrPs%N^=i_75+-M-?q) z-Q>QtI5p>`cT+oW{vYmOHP*A9;^X|0GS;;-$R;Q!*bZ>H@NX@_a^edHPGkBC=pl|H%M>MLm$CgqV0hd6l^o=LcL&yQufbUIG^ z>PJCCsr#@Oc1B?%p?fn*BGLOwIEza)JkmVynC^3wiSHUoTk)e?yV`0?BeZ5R`|1Al zMf-^?>RiCM%=*0UJhO+OeBB#Cr#l26#@OW9Rn(Ih%O9yu@TbylTibdJA15v`hX2kO zs;;q51t&b1{`O78j@t+SE9xO(c2^jy6Sm`(sU{ zX&T8t^dUNmHkHz*G1Nc6A3Hh74{yyFT|>HQwCDadt`5>L|M_alWA0-PZE!~UV`obJ zv1koxUL@~$!p8YyhOgv)(;4`DXF%J?nL%wM%|qHoPAnnKQrtDbn-ARr&ce^E;D04` zzuG3!yozt;2L9c=?PR`z%OcIU!i!z|7{Si-%AG@;|1h^YFVVMTJHgKGw`6H?!VFv| z%3|IHJ7F9K8g{o8X^f>9_q*MQN*Y6DbNry0p zFv-Q@>3jn#l1)i>#96|>H}29&QkmA4F|kzT_u7UG!ELARX<}P}G)Jwr(l@ZeZRoGY zOK8$^KNQXw zPl4w&-}GZDJgVtOs&r+o%pA?*xp<+e@3&)O4(+Hr8(&vX9otRYGQ+>q8LrpYL>q&o z*#}=_${OL0$tKdB?1jCY)A7PLklQIP>nlaE?@|vq@!fTa^ytpj_x8osl0HiMDZ1O6 zUJ|Lse!B8xPA!W4CwcS^nHl3I##nvv;5xC#-kX@Wot$Zsb}8j0(wqLEtGvyFk(VjY zjAQMU_LoT~xXD+afRQO^Zf6)B+@P%WtnZM;Jd^3O9%wI9z?7i0aM88*U41NK8 zn}AWl=i&G|Q~r~mvqw`tcaTj!D_C@9qF}ZOnuCuaN0TQGvzJuM>hGp99#ZJ#)mNo z`ue2h|IsZc$lN9HANN7sF(&PB-_*@|lfCym<<9_SSWC2VMSk2|L&`qZ#wF3~pwCkMb0~YBzUjjp`k;FV>cd`eSrdJjLm%epo3@UF-ptY3rT~;+rT-U!12(|_y8PxblH384PH#}U4qF< z(h2uIAiq;_?0)&Q{omvdD2`2&51%tt{&@N%pEHE7$RAl8yHEb8;@FquFD;2plE0!P zcDMYMC9w(k>6{_UwqTTTGn+Ty1w%9{5G`|E(fu{_^$4^c0GRv`P&Ki%>t{}WaU7~&Y*V@0NUMumx?OOY_=uDgM7o9l{ESvJJp>~eGLyH0ctkysn>5AIsN;(EJxQX8?K5W!|+p*hd4EROA?{07>3FeAPQ*=*$E z@Wl$B?TZ8$W8|n2dy9 zmR%lpM=-aIyS9)go3$l5v~btD>+Oc$d264sPPD5Uy(tra&oKK#;^C3joBux#vv((_ z&vyHf5B)FFnMbg*{SNWKo}2#5!|m<-(@(d){lo1y6Zy5bpw9-T@J;zIjIcKmA3x5V zpnn}TX|Ulo3ELFD;J`B0a{?X=ePZq>d-$f2xz~Iq@J%D*41bh%TZ%O-=joMf@ z(!MV#ef%+BCXJCp8u2*dlcMxjW!3qEX+hrEQ~vl`&gTO? z2hM@#)4o)F=n4PYvNz87zDq~p{S3{b48gX}#zi|t|5c{p|Cis)Tb{t`>lS+`cscRi zucHq|+YAg5ul3%7&;EL&edxDIw4SzLE6d&Av;TG4z^5>O?7yCH)DCYf#Gg0|pzqW# zUPNt4#hLY#8?bxR2JwNyp(ClgRh=)~WdA<7PU?%Z&-o=bJ019{K2LR9V&n~1Qhy$G zicf+EZlTwSZ|W5-H2vOtvpt+T)#rKC`K?5M3D??f>F?;9?N!NjQ*U*kR}DG>=(AWDngkzB#``QKT2L;v0PE9R!Ak3sV2G zc$6=z>Br#MPw}(+Z??Czom+l8dW`Wo9YVTmp3KuaTZh}uH0p+iU|T8XJXRbzycQiy z;1XUsqH*sDtU^m?nt9xEv;9EE!o_Lx+Sut^{Px%`HH()L*Th^;;cik!m**GP&qg`RQfqN zkXrfGn|u@bjx}y%KCrjD_eTD~myP$i|5No}P$b-}_{zN_`M+B4rFO5Jr|({*T`3!7 zX-z);_vL?u`#GL#O8$WdrJpx~{{qFk|5!Mk7<>viUuoLiG;g>MyLO!2 zUjcr^KM0N1IQ9J7m{>0NQB1#!eD+n|-0S^!$KP|HQ)$+3t+l~tUrC&Bvq^iS-_GSf zM?4j8`8cfmc#|)-x3rU8!rLDa;+&>pv#ul~3$oA4m%Pc>{JKN`nWxHg;6Cxq+P{mY zDP0f7|Mb|lyh$m#?#Z&=_S@~ecIh6F!L#>prr@^6#k+?C_W9q?o?AWq|418~hvODX z2ZFX?L$|Fh`q-n(qQyqG^^PT(o$mEqAEa)j%fiV9qQWa&c}Q(Gx*<*j@C8oOT5{sR z9yqcB*+xC{yqCG%$DB1V57L$Yu#0Fw3JZpiI37*qN1QTh0K7i%wJhOdH0%7uT@`Mvti9oE=&^v%#Y?4fAB z+s*{mFVXH?PR5n5N%?5EN&9G0+NAb(_tMrYE$ugHfwh)$llqbFrTvrAdi|)7T{zmD zFQ0xmg%|Y$KIBMOw>@uq{XirW(q2})c>GOp@NwL@3;SH1C+mIVXr4>6qN7}2^jbJi zGL@Ur>0HMBlApeZt_^;CWgt?&prifpyRP2V&!|gwC33hcra2ZrBl>4>Ci$hW_D&z} zKyYVG{5mjv%n=WR3;@_3R?XcPU+QS|sCtFJLs@fyEq(7J?epw!WrH*tFB1JtiykkE z37%B;)WL?2t_>a-nAKoB`bZVvHo?&{~WV2T4*e5i0qHh|n+^)WfJ-hluuWN%JjfvHD@vW?nb+w{`YSrJk zHAjk(CHXovZSLrssIWhF=v4F0yb@$rotkn8duNOIF5)}z?YqUKox=Cq#kZ2CeUrsE zZ+q0fu)MGR5&2(ue0uX7z0uX)H&N}H=l7YnT>1nJpJ~SE4_m>n!&;9~erc^akKD^R z>OAtNq|rK9+7-BX6&USw0Y;}@2`>Zwkn9E63J1?Wv8!U3wVw3vqMK**e8oQm4?JP? zJe1aaGsh0_zF7xKj}706q91C9-`AsNKXgR0rxW*q-;mM9 zs6%?t6L!06Rbz$S*RT_~&bA&H(kxmN-Fl6k&$lUuJ|vf;GL*m6%l|9#Yppw_gD>O{ zB;}{yeqXnic9{J1XNZ@7HTj*YYcJ$2{k*CB-mmj6GtR(C*aG zo!GA^iiHLY4mI9KzR4-*sp3YZ#Nub$#SX{Y#o{00?`cPx>q3pwNOM2&rMNLEgbfMJ zhwp5xBz#6n>_MZqMBMlka4&61=erYaX=(>8Z--njy{2(H-|w`GiH^iK(!Nc+$E0_x zJp6U=*WfRB=4AX;cgQ{GPOQyMq|rFtz!`TU-=8m$FMmwyHT?_Hh~L$^UfBh?2Rz7W z=J<5Sw)Sws#E;Z%Ou7E=@W#@yb+#58nkJbvI+*5r8T+vB@rFfm`wV{MEEBxivTuqG zv0~8~#j#TZ+Jtnzt-VVW`=i=J93Oz~QD~27Q&jdx!MUPW(Lq*-J|4|I+QW|CPkL;x z9tCfk@+B87@OZBFU1f*%Xs*@U-UrT`eGpmBCSL@-a-;8{^Dv#!M4FKiv-d2^01l5^ z|5X>R{~zHj*&gR!gwdVScQxOi!d<|u;Jxs!+yZL8yr@8brK#K_z!VdS16A5PQ0ui;HoCU_F49WQCHak6U&w4M(iS}u zU%%ii?_hbnMF8G{c+Sh(+R7ujZN(rL=Ut$UPwuL?=@PWjgDHa_|3$buo9@y@(KBZ@ zZ~;us7G2agFg1tpe0_T`we<z& zM+5bUZ#y){7detusbi@x}@I-b^2K$MwEHxohJs)m3_> zsaJh@qowZ4^ntQm8s9TK)+=KJd6L_|s-=v}v_DY>?H}Qlv64LQ8p}3qS=>?vZ@)7p zi>F)bzJ>b|?lkmWb1qu1Ii7_d_XmzmJrbYzCBZJZ(7@&Qd)wudQ@_B{y|tskzx0v) z&!YwN`q+)|bd$l?jr@ z?}&c3XRqaGL6kg4us=?^0^}ZzliYMB9zLWgUL=|0KvU;;`teqfY`r|c*m>28-4?Wl z-7T3Obq1GbyY2L4+oBVySM5`};mmInVHN)Lf!^L`plR=wxv~{=bnNmRQ?BgAnE$K! z$EJ+_KZ4FQUQ-+y>+8VzL~x+#-?3c#F!^5iUM_FjS~F{q-(>r&hx73zBm4;YK!`RY z+f(`-m8<(J{f-x~cbZ5~`yLLGUhU)TXIMTwekyt!^kqlzZ~bl4KV%ZLb4`KW(~goZ zAxm36&ze|3`hs7LiQP62IX8FkMQh8DWsb*A)++Am%thxRV<)yB@wJbv|Dy}LS(@7! z?CXK2j)IpN=qvTM))kX46}zUAb;O7l4_J2%V+-xWCUz`{KC$y*AQG*F#_kCq!$kjV zJL#4Xwlol##C{_7@9^WuZ?K_d5o3*$j_&|HxnPJ7!nD2C42hDqis|&KzYxDGi2m8ghlAg9zKZfqu zR_0_A{|h)zdCc@1J`FWbeuGLw{Ua{ z{1RchpA_o`U8}<8x0hxgX=Jaf-`@D5xldCzbR!DAKqlFgF?jxB#yVEF+A2hbh3(@W z+-I2>Qe1Ezafy8$xH_?~TMF&|)O;7VGdk(dxchbkOZkjp(b2g0-WFJ@IQnAwVdhzO zA=tC5N!A0F{#ww}1;$3vnw|mnEK>`Xq##S$$DXc%{n~!^Y4xlLz1MhPWpV5kowHwU z6((T2T4y-G_^+Vtg6&?wcB1|k*nADEOxzW8- zBky5{IZ2tR2X^y{Gj;#~}FRjLzJ>I_3j&=k4=>6=UlgBxm zaYkM{Q#dz>EX>H*(2=?01MqGitMFs&&ZL1q^bQPbje9ndnH7!LCOb`L&(`1GOIRjj zv7qL&-AwwnRy)lHDF^|ch)ZO znonNgU*~?x-4!tUqt0&d{1Ddl5g&Un?&uL8J?#U}@Sn7v<7Y{)HN6WP>fi(?9{3gb zfPG$%%ASK4WhL})ZqVC4yP(tXpiUM%C3L3OQ$3epD>4?}O1V|Y;8<(o%Sr;ln*Dju zFyKxyZOKZ+!<9ef@|6EZf3&v0f-jx9d+;SEGWi0Z&Ntf2H;Hswzn81yO6nNlrM=y& zW3*StsL!ZF`38CUZYG_0#OqDItngrb@Y}J`z0t4Y&GF+40+HrL0d&WWth&~mcTMJw z<(tT}wia_&5!?y9>+aPu(e5F>m^ov|*BwU_hySwT;+~EVzV1DmIQ*9scf^as*Ih^x zhhL+(1}_fZoWrO-@WwO531(EEWB|?zbaWMm{}gcwmwePY#GWRv9q*hJ$71Zib*|$y z0I%%hIj2RAp>Ou_FY?Wv9ewoBK6LvVz`6TbW6=eK16$@TltuKL_p;4;o6K7#%%^L2 zHY@yhbSvDu>{;PG|4qAz`x}0@>wTg*5#(9p;A_+8_`6q5I~<=FPQAc){E&1x=t(5j zn89gxx8^&(NgBcZLmhw#cv1&`>YrT$2JV6NGU$KCv-FYq!Zy2^x9b=y!RKv^m&%{0 z@{!+HaEA21Djyo4^0$q0%NI|l@;Tqml&T+wM`Dxfc7B8j$F6e|YKIj3K z4^bN#8|g|X!}NaY(cN^xne0F$!}Q;X)BZ_uoR7Vm4AbuuCpb}@WXs7g{SI-LgXwL= zDNhSbuh82)!1M~e1rMb=IMZ}SExLAly192Kz0d!q4+*-fZ)-(-ozAkI zl-lxV=!fKFW^cirodjN&Tu$eU#x6W#Hmp9rNc?Xw_k*m>U34F4xSjL(4b9O_MYIn+ z5%}8R6-A-&*1*m(!h?0UhMKkxvn%*^c(*Xs_}ANMisOy<=++T-zP@kN_fYe#w|qrU zvwc~WMg5>d@U(%)pV%{k$Kgbb|Z7k$^Dt?ndsg^qpsu~ z)5fiZ_AQjJbMX=UN4E~KNAMqor$N7>#@TwEeS*6kYFF>(_)y8??rCdXZ_X1oy0mQ{ zc1Ls;Tg2W?XP!T2?CR*BMIPOUUe3RJo-oX`UuCq;k6mB#U+d-nkI{kOE# z{n_nT8K3vcC?-#G`wLsj`0VzpjJ{qO*OEtg$^m9IcX=&kxNwWDigkD6dNmpzxK`(w z``>`Sea_&G+Qi;Ee_(BSU3)9j$P2*1tm$KtBjoUA9H{H1^kuL(lxoQh)3XFRph7#q|zvRNR2#*lS*RpDqgT6JD?IyyDmzFT7Vbg>ycp z@J_|CF#bvCjBH%#TmDb*Gum6j#E<1Lrdo>^)5$EtFGnXch`St}Y){%C12y^$y z=Uh0arfe!S@1GhF0C~(9dtlGWS#_{`e(kp`$)dXt}^aX z@lL|Ua~PtXmz%>M6L+~eTteLC=5V3ccg@ws{QGx^Ynemw?<;u!uS&cQJdfdZ;CT$M z1J7f49GB-2pK}6!$MEjROAa6}IfyLc5V8o_vPkBEO5l06wc{NR=A~fjwe~>P+(3A~ zg0|3KZ0(wG)_`zQ6Ff!`y$I>g)rc=Y3%g)_zpH4D0Uqz(frHBzWP`c5c{j2|HPdz9;zL(KI(~v%-+c zodgd|=|~uGvXQVUUHJxHHt4%M-@wg!z6-%=3x9kwZ)O3fuUoMtFCmxyH9W{P-ekGA zD0b|mIV)>$5LLwaqu>>T6i{O5w1SamaE}g?kDZ8 zz(3r(C-yBm({XNR+}|vURr3bG@oyVh#Tx6CE9^U|M|gS)vV!M-f^4C|DqMfim2rr7 zH}6O;U=Pba7W*<^$Ucv3WWdra&Wj!l{*3!*A>bcbJTz#B?zj!-KH3>@t^;m0wrijl z&O&#+9)iwkkNXO5Z#dx4GUOw=8yb`kj@*VkMLca{oyCC{BO^s-YW(Bqf&jl}+<+U6 z>w~~QeD^;Z(>1xv(CbLMux=CO4jE(((;B`P9CGox-AH>cw{F)Fce!=Dnz+lY+n*A5 zIo!LHxc}>Q%bD(h(t)hotL#Ii1)o~C-(uZ<(3WvPRx3I3w%^8wF410iEo+i{rGM=V zPsu($Xaxrvc;0ujJs)^ByoJ^=x?}T!fBicF_7m(Kv^S~i#Jxi1hx@u_A7k|8M90CG znzQzvJi7wH*bB(&tGP$jG+uJI^z9!ZBV7GfSG#EfxPbqfUzCJC-ZVB;>Fc!p1z*SQ zAG3ej!(Jz(^B$#L-PQh>zAEfv{{Q%$u69e@O_}y;^fZsqr#SKTtd|*nYv9pMMUhVa zP6N?BSkw8|4Eq@V;O8^!w)dtey@PRo?6UOUy;ZkF_B}^D-?^Qa-YfooD&>^$-83W^ar>iJ-<#eh^zo=b=;#>mzVHqEGgIF~nVdZo#XidOg|7PLXP>wH796JiV%^br;oiJ% zvY%}?G-R6dgoDgsJuo1AgKWsaMKyaM^Us(||IqATds`Eyr&ABGQPkVW5q1QRjfo*E zba}y(NxUFwWIryEcS|quAA7`V-XIlESmyb%pKaZf)6v>6xMR@zTtCs2D0%fRr>e7UXdnnLTp72<@!h40yH}t(8_}@Z!q0Y}Lk?(cIZ$c){_o!TJ z=F^?5iK+04vc;W%fjOkl>D`)d4R)+8F9jA1PqyiT3^$)SYeg^M`K#Ici1*JWt`**A z3|txNmB>*0*y7om6LM4bH3#6uj2s1iZ3Or{=K-tmTOMy*fhr88< zLV6s!&!F+<-L~Sh{@k{#yW%O= zU;B7Pb1m-+bnzF3zVLW{bC>_PH8c}{E^fL0;m3C3cfJ{1ivJPB%gD-vXI%PY@XB)V zirQ2GPB}??n(0%Ff4wOiywO_c8&kBw-=!!N_};CdcJPjRGfC&56Om`m^#eELcUYv8@0N~_w>&yb4klts?LEWGqSkbV$yX;t!W|* zy@u^&&sxnobKOF^_yWH<4>oJ=IO7kW7ks(BT_v3~=)*SALF(B{Ufy0RU%^_kz7Q|e zy3a=^NprdgUQPCPzsSBDerrwQ{R?x?u9*8Zow?uA*%~^cZAolIXY9^`J5G4GW1F{k z{}0L#?$}5fMt=Ib2c!6Nh-<~0B*Eynh`Th5_6J6L85oVVz-UhoM!SU%vA?KgzxONl zMOvrkoS8n(X5CKlH4mEt?;*Yc9I>Y5jGVFJj7j{rg=(dXo zlk5Dbz1^2O)gBB?7slbWnwOX37= zIW0U3Y}6IFFw&8Es$b=U7qT)n|B?rd;@;fUTy*_&UAapyAMbDGV4o!c8^hhaacG)g zV=H+RZM&7WStnYzt?c^R@-*6}H7LBHHDv7FGOv2CVrQH;lf7{14-E9h7sbF~lIKXy zsQtm2Lq23L$X^;{ZxJ}%<3qRB7u(AkXvpL2DA371(9jW_#2FvutDFxzx;QO%$kpMx zhqlEGuFndW>$|}XH{Zhb$a!v8_#QXhd<)luL)EX?hi58a8(&77TVxR(P}v|6$@1cp-WOnu5H+8Q`85IsxPk zn)hkQ9cu6=A$I^DIQL&TBQ$3#mC*b}girR)2D1f6yp_p39UQMSmW*(7A8X;ySTkDV z;^Q3jnxrQq9U1K*g1mp5&bj;teXYV}I1|COq|2E-?aV%yWbKonJaj-MoJ#XCtp4NSlEYb+BAHuIabpd~0 z$N$`&7tO)2+VWG#k`m1!OZNQrZfuXRY+SvUg!i|-#{T9vt>>w(Yi;=p%#(D~66BArFXYd;|S7r5P3?Ai7YBTBd zpO6)ZC+g~z-x^^6@++3(`IDyXaIMz^lAo2|mOq)i|GP`?khp_>JbmmXfCK1V!7 zLAs&A1A)sp@+iSTnRLJKZ)V)3Yte%?C158JH;g!~1;r&`ClPmTQk;REq%sSL6I_{m zlr5Z`uln%&t3LELx>IHX<`Qwei4)8v$8rDDSHjI^DwOdGCyZpAUei4J-j? z4}hzSfhE%}>NV}cKLm|af7E7k4qwWCfW4Kmr?A2r0Zol*-(zUVO6n5rsIo$goeQ_4 zwEyE>Q`JX32chqWXwNz7v)H#ad;Cm3yp8zf?OuJKXe=fKBDAL*S(jYxmHO#u( zOj@n&-AgMh&OvH{2*)mU)k?%#?IWu`@N&>{qvPj>tZ=fo4>>t2~R2AuSwU*OUL`9h3vPEChN^wR#5~QU~O*M zm3DN;CnleCj~8&}wwAmd$g8{Z^x=B&Q@P?+6W$(}SVuVPg?nM}d^PNkUJP)Sp?{x| z_aaNlbdcRh$K1IO{|Nph{Dwf})S*Dc889)F&@*(9=QwfVrx73UyY@^|;J?{ZssG}g zzs~y&*2^A`3_m_Zbsujn1F+`CCw>TDZ15ZSwNCgAe;fRQ)|KWZfiDxdZwqk=TI>98 za1yw0Ke))nN$#F^H~g4zo$!u$F=)&MebO9jSVVf9=pNt8SYA$_)bK-Z3^4knbJv68 zpc%X1uV(%C02|(GhxWhS*vrW-?;pK6mp$I?e7Djqea9bat^%JjhSDuHJX%h&ZfUph z&(~QCKlu$h^1z7h6m0*stE1AEdesl{4KdQNPS#1k^yCa*YzJd0{Zipog^hOkhaK9d zZy;^a(}sUA?KC=h@Y+vu_Cvo+npa7aGp}`;x}LS=vq&TScSFsm>9^qTQQ|MA>xs3v zjqyp)ccm>nVR%nd-va7e_EhWoDl*~!DbMg6mw@B%8{EoS!)Kp;)h71j4!pG^o~- z@hz}@5$#Iwj+exrc^x_b{Ve>M6+Sown$B6#giPidxM6Qr+YK3Wvt%Q*4Hz|iM-*OC zYfrY1gfH^&#h+`=9Lc|Sl}6rT-U`0g!OrH~#mpgiH|U$ccs;r+X1&hw_MG^CqP&(q zLC2#Xf;*tE`&M*D&!7D^?dBXJq_+;71nhIe)63Tzq~W z=T6zIohh6_aUOI2*L{C}XG7mt?|h~2>N_V9zRunAw7|3S)UWkTcUDe#tYZe_2Z%mshW}oxs zIeA}ilcp8yF!l-fc-|X(jh7imU?KqBFKuyg+}oNb`XC&)nQ!p*u^4Ba z!f_gR#?8E=ncp{bC->CGXaI4-PiIpuank z=`XyLv0win;R*Zo)^u?C3SVp)ZHeFSi}j@qx~ns%9b*hkWBUy|7V{T>pLm5Cm|6(pF)(49 zGYX9BzL0aXd;dD`smvwu?LOrCvV)My{mu(I{s$^cBe&0VbrC1icgeXX;Y(+$9RCAk z{HJ#XA}4kSA}2qDAK4Sox!0OrU9jc6oqi>)i{C)kTCEwz&8!*xUwJf7b19rET{E+0 z@LwaY1^0k=m?szSz^9n~67;n--s#1DCCim%)d=_W4F3>VF|u&Nub|zT$gDIz=(mp> z%2=Z7o*tK;J9nL#TcfWoJdv>XqCREL2j|hBZQ4iT*U~QD(h6%YZRW|?khv(&JU367 z?BtQB%F8oMb-a3EUhCSJJBw>Fu?wX-g6VGh$Gr6DCT8mGF(+xvzCm0oU5TVIdw{ss zV`iB#GyA&)o6-{i8kiCZFIag-s!dR%a>^5 zn9pcqQF0rN-kUYTG4uP+xAg_~p^s=R!RLu~wv1(g+s=IS4p6{XN8QX-mE^vcz&+0weWW8dpQqyyGXAnH^-G-2Xl!V!}#~2yCfdyTUW({4g z{jbvgKK~zU=N=zbb@lx-nE(@la!Dq44H#MyP*J%=1Z8qjf~Zu$YSpKp(8nnHSfpMG zQm)oOv@%tTf-NXKHFMBPD+nXDg4l}ERuC_>wlxsc2|)$9Wdd~G?{A-Th7l3$=Y9Uz zpS920`|RslYp=cb?KthVO{ zoQlrAg!`qWzkl$uGqwfE*Bje1@M~<_=ZzpD?^kpX7>L#Nn&6Sf&>?4!}p+eCC-K9KZc)Xult;{dk+V;Kc2> z+v-_r+Arj`-#h4utG->DY`-kG{eDK98JSA^)gRk_@1;fdov{6~?Ej(t{(wH9b1iDW zdbj=l2wrQhs%I4cbhrKXq8t3k(*?Hl4s$2ppl0-e!26uLV9cMXd%O6j$!&vu+-bV) z>8j`}l)tinsX23a$vX5S){1x=CCb>-zdvoVj_)TZ#|+`wpA`dNqcsoi{bPKMavy4x zHRY}u8CF#D-+}alH=nwW@}Y0ZE}zs<8#c{nf9laVqljZ+LrjfRw{bpin6o~+9KD@; zBcX5hgE8j0cEdPhl*X0{;v8?Sy7e9Q{ZLNx59`j754-qB{Xsh<%hylkJ4T1DrF^c= zZ0?!rL>$cxWOw^7{Bhb)>j>B4Z@fDczeaFsDE?i+4y@e^GH$3|jae1(?+E5r#IFEu zi!ryLET!%k`I=h~*;CD4Na^6RS74uzoXckJa=@Lt#5z0mc#(f3%h;2jlykMxpBVd| zcf$f+7-OCZ13p8T?1|_{Mt`VP8NmM^&Ag~!SFe)z4s)7QFNzJYx~T6g_yh4skqD2v%sNn6!vJ_G!3 z^8K>>cgEKy_KNIauZWQ+^s_T=u{J^an~JSRennaV_VMO$PC)C$hSuDEy^=K5M>_A> zLG69)oKQB{o2ZNCgtU|B&E&(pW(Ts$tTWbE#B-7NbI8Z^8D*_;`*{%QtB*Z=gR5uv z1E1yf7G96oSWp| zCf06tuy*5>Tj~6fvR_3y$buEJzg$E6hYlPOpRe%tNz|P)f0V}0-*bnA?1t(G**#`J zAN*(5YT%Xrs|fe(D;0xoJ-v77xYIXcv+?XIrSPZjF!=lDNAnv(e(S#Ya(*#vIcjU> z!{+XXAaNe#A1`g$XS}ph(|?UP!zo)0ZW!(o+{L(RTotYoCs`E2u^%$~BHV?zA-KV~ z3vh#Qx*J5gtz_f?Tz_0Y+ttTrZsF+~30W#L0GX z2CfILJ5F-*be!gBYFp{wdc#VzC?40EkK|1d2g76cu(G@1y5hRvWD_|Rr@2}uTqdp~ zPO?KZr~8T9#xT}R zZhj749r=gIk*U7Okt!>UtbMEaeBQ`l|59;x-qq0?6Q$N9#)vV+=WtIjcYP_3tRuWf z!~ci)U(kEUcKdU*!4PMTTO8uOCgK-A!T86VS28tg0QwE_KRxEZxB1V~9S)tkhG#-g zDl6aIV|_}vg707b+>u!y@t;?AFV8>nuPM{(PMO|4ChgyQ;ZZ02ug8S1B3xw)RGh1E zJK8(&vr!#?_Us+#JePJnH?*ey*#|~tE{oj3e&S1|2X#CxH0aWItSbXOcsKXQ1>Kkn zPYPvJUtpDmri`Pl;a&3{@R6x6M~?Wdu&GCvQ^R+YFZ8je5xwfj)b$N|=Qkx^-fVNy zBmcL^{|md*t6$hb3Y@zxdV1i?k^eY%Wh#jA5(@- z(=pxJl4;H8HrAKf(tS{9O2FbSOy=qL;DOU?UcX!*{;O=yA$L)hYuM6WIe&2DzmPqdYt$VobH($-K5uMNaQQz=h!VEp6 z^qE&Labdhh=fdh+W1wdp*T6@G{RD1(#E`PtzuUb?Am z;I7pj__R?eAmI=&s*I4*bCRn``^;{4)Z?uA$YU6JUBY|O7sG35hS$8de4qRO*Weak-qnWo?XN8MwLjO7fA5jR z$`>+2@fU!*{K4L}Kl$^b1b1@yI2X0i^*`yhS4Tf@{o*h77q*lbSkTgRC^2T@PdY8q z`Kmhn8UDge)3Mc+;(r}{z+bRQ_a!#8CB`)OW$%(dyZ0{HzrKyGux&s%quY#83(klv zik(+Es1Cg%#u=(&Uw%su#+VpqNQ!-VEoRRgXQ+yOMJ+eTPFf&;VT(`x)8#K{d6Kay zc82^}Efw;!*IDn|Zi9AWoUdqrM`E1gLN=_7u~)b>mp3iIC(^9=WwdD{VTtzKDJDPR ziH=tM68wpItmhW^er(p!XU!baIBOPeHf|2?KHN`-BranA$n^9`qS%V>DDiPOtnPcu zv$wN%Z-1`tyb0PD(_i1d)v-$sQXYlG^{2)yA&(-}^gj91hy@ zs}1(k2JP_2xBDW8KTC_W{gr(TC6$pC*;APdhJdkoHb4 zT}&FtVdX7)Z5z!sz0m#3EZ{*oO<|f&rJAEWnmBZTmBg{ zM|Gq8d!ZBQKpyTCaLUX3&TzknHwNc68Ie%(Jf!$8~`LE_1d zFTLVJ<`drhL1$Ww+`2v|c@M-8(o;Q#(AO&GB=enb-!4cwZv*~?G!j+5Ev;jH$TI9O zBmKz9bo3-2a?*#4baj+_a>IRyAM$h*EOh0RjWtodYzI7K~O1yc{`i!(07plKX+gWUVmM&0DS*j>g zCGr&=rplL;t6vvJTg=y7%596yRxCGs1NMV6NH6}3^~jCTul56H{PxU2H)zh&<)X0c zCT|l)+pUy7=7TOOiL(t{2tCY8)1ofKjU)Fq_joKn{08BonHPYv*&a9E-}1uMgel&; z#9PN32gcUNxAK9$M!lwV&@GPpe_mnuW6n|>H|&~%@P~vwiajZ|*Y~^Hs~-KL;U^mx z1&&m2&7gfV-pN=LX=km;W~6>RZwj#uTK z!FOuDW_-P>ntdXc-ud0!qvrJRT;gPq{;~bNba!IRaQdQ*zUWT7X^v_BIkPx?FaKER zt0(J2)81FRc{LV>m!-5rvVUJH3Rk%I5Y8msCsN{kslVz@3xA(*)$3AV7G)lHZFT%c z{L`;ZkN<=J5>JQ=(-L2mU zR2z!l78)3|9|K+|TLgRJX23hL@jMPrG)H@G)_Y%)&j+)tcmZX)I#C(Fo9{;K1!`ON zhmZPhduvrIG$el;P&N+v_u^0EUPwRRk0TFdUl|8)R-&I*Huh)C%C(<2ax2&VF=;(U z{u)Ou%X-$be9tP}$T!lEEY|m}#0yb|s}pJQ|Hd!eO)gYhDc^TlBYm5^J=#e2MP2u> z&*in#-G8of|H)ym@DLX&5XJ=nl3dys+LV`E^@9$;Xu-4D2xF={35QH@#TQ*&royj0^> z2S+#Cz`MJ3#hj(ix6U1p|1kL}&(-X|Q6H9)hQ^~x{_9N}Z(r)@2!7^ivbhJrT|irD zoZBvY5aVD^z7s05!DZR%w`y>jk2tt&gPQ74@K`B zhhP8P=l=5%|LD8Pz+8KqfkFF21G)dpz%2U%19R;64J@+X1GX{VEac9woBqUlemiS` zY$)pg%_dIJ{;Pr6_MZ*RvfnT;$KD9^+WU|AMRT$b-L}wa``5u~UD-nJ8JV=7@vxos z+YoQ)YWy(rXd8F%=3_JVxq1;aGZ`LFLvB^3cc)#7>}}`{SG-8sjwJevhF-j>BBJvB zhV=hJ{|4B1*Nm=58qq*{C|ctO1~^k9*oAqIU^cJ=Fb|jsECO~0b_ZqwdjfN<=v4Hm zht{%o5l{3NT}Y3To~3d;Kz_=1o`E@boq<97rv_%*a}3O~XBk+4UFS07(g@_!rO2t_ z$g3LW9m5zO{mvYpbWIzq&j}CBuyhWb^O3CArz59*=Y%_2i$>|3T&HtpjGA7-`Kr0N zEjZe1qAw%-0CTSNM$1WZIf`?M?YR5?AABGws@u^ot}vXZyCa-Q=T8 zX_U=R8POr8w!?|wzEu&MXS%dW7Iec$i zcyavo7cVh7&CT$N+PO6DkLQ!mAb;=h&_(Uy-A7z(_T%_3`{k&9$N|avag2qjdrG7O zpua{-I|u5m`-Txe%U(fQ267jM^7>#$P`i67tAfWHc-O(y=Lf^$k{_%qApf%Lw}~g4?6k)n{(YbC19l+1 zk$zogbdrF*2^@1l>|U~O9mChsFFn47rb|zA__{yo$Br5AQ_CBmeUf^|z8>N4lZR8L zYTZT@vB10+1{|Qe~C)FjJ^s?cl z=fz8Y-Hrr_rNA>_-XyE zgKAk94nmi!UO6)?I&PqSjm}$ik>ST&{$KJ&`?%I4JzdnFwq(>x;6t*Nt;h@iDd+P(Bfy97u*1X;>1}UU4;KxIW{mFF@U5`y?_O9^ zZzwGZuXpu^E~JNUi0)rS-7D!I`o-_`i}&A`(%jKAEc^R$!+Q1%Z=*b4mmcl3JL3n{ zycfZk@q=`C&hgcJO?%2d#atl8{|_U^ zKWAQ^W^bn+qKifYgSJB_dG>vaph?ajj5a!Kmc0nMQ)m^`L)R-6e*xbGR(}0Ew2{_- zWJ?&6NUUOve`^(VsQJmyh;tqO3vfkNZvA2ADWzIp$DRf&SsBmgouzkw zGHt8I&h@4o`Swc&=GiYAm}{?bp=3^B?Lgjk&o%m(^t5d9k^K7|akda%V8UfL%CQyR z$%Nk34&!o7ZE&f%K`?h&;MX()xFLg?pfHpF#Ll#A9C$ zGBRk_n}4(Ixdvw0_ZygVTsd|4_CJ|*ip1=6<}01($5W7Dr@||p;h8STi>{2h>0xA_ zE!r~kF!nZSE*|k0v=(3Jn-t*AU)er~7g<*hMaP}ZogEr0n}2>uybN7kbDE|sEAmkx zHUZvSKA-l_TT1(nEUgdnrj^Qk17*C5_f;GDCyhCn^x3;<6VAu1)c9uRA>#&wf6jQM z_}38sZt=G_2MNr$P3an60-bZ}E%FKa+Qq9)TLkS&U>^8jmF{f|+7)~c zI41wAm4CNn{_`sv2ZZl%^FN>XSDzsN`ELGwh-Z;Uu&RChTlke%3D9YS)55{2cR@!% zy9c;q^DkHar>5jTZeVznn|~hh%TJL158eDTiD!{VaO%L;cKDT-AL!KGv7@E&9rWtX zTwiUUN8a5T58Sq=?gPV`>*#+Bu#e=Ce(_-|g8VVKJ7oTl~8#8BIsP$6mwMA10w1$}TSd8V@ z4#h4;9s%S)DfsDrYtJ0+U*>Mpwsnjf@ICrWu%$a^kfuzzv$X(z5BWp!M*giqN3Rnk zts3BU<)QdbNvqH*tUok`JP4n0II;5j!9eguR#Cl|cj$0pmB#F2^PGw8ywts$z~##o z)-|PuFEy5YCG3ZNlH*gY+6bO%@L$_nMjB-`=@Sy z%rP+Ao@F358W%n(I_XgTEbaYlW1n>wT86i>?Wu$txpaS}6ISen1??XZ_OSV{!U@ar z!gB1}2z!9AI@+(nTK7dDU`Z#RADGEoC)n(019;c9m%_X2l@9XVo)?=D2Z&@U4F@FfhwL zAK3kC`pDXw7_;xWJy zLpP$EV>?(e{NYRDH#1i4WvtxCSa|@sdJy^P$%{Di6(ctpvs9-n$~S;Fz(f3(#(J!H zw@f?;-HUhQ{`}s^&iR$&%EMLgwEk(&KY`Bq^%iLaXQssy)WPti-+K0yyG|e7=(ob# z7~A}$S4)`wISeeMowxZz&-$?W2xt0-I2wEOy_Yz=AE!Gucwg3Ow-~nY)Vhndx3Si9 zWa=>5=@Q!OV%n`5+O9%3bJn`P!ZWv?8Ln`3x<#Zbo;hB+<5=e!N_gsi2kE$P=pX4+ zyn+5R{4>xWvKgQ1F0C?q=owdcJMU>`*-^sB@pktt!bFoV&}P!7*M^JsM5NkpSq{h2JL8z_OuCfGc|yXFb!L>V z6W4qydkjx=#_kQBtTDs=yur?T%y4*TxF9@}Aqbzmj$Lawc8s||r8%7b9gd&oWgo4P2tG91SecIxN6}j zM+r{23E+kbN0~}+!p#A9fpCwU_vrQOlegTZs+Q^G*SOD&d`p*M<1#cvHKvmYr_w(jPwzEkbim zeW+8p#d!`Nc52Ihu{;!y_6^0?_6o&Uu}-*>_FO~TfAMRzWtP1gcwm@C8|T=c;y+j= zKehn;twZJK{5Jl5gXPb*cgT<2knGr|IJBGE?qB#1VkeZX@B{pwP4;j2rQ`0$&NT$R zFbk*nankCKzm4=;xex0-+!EX&%3GV|?9a{Y%3Vphj(<7#Cf)4kJ#6;vj=+CQ2lgD` zZ(&_&U%ybizI`ZK&tB!_=o&4(toU;DftC^=VJ-Cg@;*Rp?8{4l*prvD_SHiFFNfw@ zuwiKpe&XaA(TTh}$(@Du*c-MCpBCZmpiK+)c7obcbuM@J95hQ>;9*%tdjE_BOgZUngx{H^p!8v+A{|;lFntqx9?k+E0mDa9v43A9maMZ5}9$QAd zTRj|gm3{eDa5s54*=*_O9pG+E;+(Z>@yLHAak+LAG|>~B^mFz1KKe>)=S{R#HTY5B zw?jX(uwg1(eAW&gD<$ony{aO1$)Jel6*t3M-5-Uwdd;{nHrk3mwZXTg2^!Wv>kX~v z*<)xoy)U22TQ8^ZR&kQI;BWVzYW`6gl|ap_M2F%p(ba_}EY}`vV1Yfzz^tzf6A;(u9-Fua;ixe6#>A?r~^@PCfoN|<=^fPn?}J_8HwFAU7FcLQ~1 z{A%=<{QOS{9gF>3D%yb;;zLF!dbZ5 zZZ7|#Kh)+p{t@zHQ)6vq(l+*r)WW;kH}s(V1=!2mwZ9VjiT=ryG0%P#7#poP?a$qg zEl}kR(EopM!(Rdlr+t@R`DIss!GssuD-A5Le`jF6y+RP0_>CYm@r=Re*tUT|yTQP0 z`_~5M+RF^gJ1K26z`xwr8E=4BV#uC`Vjz1`8sIPPLW)P=$6M~05v|Rc5sh(o-z{f^ zZr;gTEHUKKLFBgNZt))+8|i%VR9kBd{<#@rBwtuFa^=f34@aJ!e7VQqg7(>Nzf30$ z(U*9~jNkBWHSP3$6ED~PA5*U!`wTb!Sf|{DY)(gZYQ_XtyS0>m0%_^~xgI zL|vDGyV%1?4nfzif|K6vlm$6N+w1^0IEiEb4Ll&(G%$%n2GN$1P3O9}ZO~&S-@p4M zI$wbOEwq{LHA_RDJ_~LgXE~HE z+<)D$$bRJ3yRJ-|Mw?5fy+AzindQo~yIuSW@cX_-rq!j$w7L|TR+l2v>QZD{U5ZQ# zoATw^4*@-y_KGXhWP6zFhTjiV`8Cd|?2>1*Oc?tJ3@osJVqm^~uOR$5MUZ~G$KZ49 zy9{KWZ(y!{hk;r44-NePmuC&gGw%J5&qb!yNv8F(B6E>xdPi!nk!M+U%S(LzO?Lnq)Id(fY{zb=$KY?~V z&5bXacBc8q)xlhu7WB#@Ip@kWM{k!bJu1^Oy?Bhfo=j_>#2uAsz9jCbOgn7&FAw?S z$h2usIPy#U;mNdnz+o5bzh-I0pyw%Db4e%?K(5Jl#Ja_nmOd4cTc-71RGP7IVXO~x zQ6tmz-k&4W^lokma#1>ygJUlOxYNNspOj(FUIfiW9i8ryq)vAt`E?m>tg?Q%^gok3 z&bX6n|CRsFr!PGD^_TxJo$KBFU*bPcens&g%nBLVet_}lJjt+=*vUUCZXeNl#}4$Y zroP;To-$v5kv5OgrfV3RRx^&hK-z0F+4D#~YSSgaLq>KL+P^fo9D9+0x%T4*=GhAj z%(ovku)uy8SV+B}=RREwmMibZU4t8hE5Oyc^PhU|%3Z|#XL<)~QXl+v%zvaat;cph zm-&z8LKE@V6&l@Wk`Mn*bhjEG`)#q|H()!c1=h4<4}-qh&lzJ6PE7{kzy|hk*6RBd z!u8GGU-)@tO;^H!4ebA|)pw9^VDr`Kkp}hwZ)D!qKt4vdK}JfqF}e@3!00~kxY2#k zYmDy0JWIMy9rG*62#v2FvFS+ssBj2)_e-2k?h`xn-kmys}BJv6PR>CA?cM={9Z9u_+gL zuUysOrK=yWToVZUl&~22q?T(A_)k21O1YMS-zI!#M|RxIdyZwuj-Al-rR_S_Uxz#y zg8Wb&w!<^868|0In{RY0%`JC;+nmHPXQaK-(YZDyad~E6bWd=O-mZQ`PSQ5j;MOPO zAtwu&=Yo^oz8yW#nO8ddoNlL$B!gZ;e_G3&GAVybm{X2$=akslR5nL$2kn=iVn5k2 z`ctaChIcSr-6?2~G_pQuKgT~x<5}Qf^3ymgp42|F1{0QV|JuMjdzpc`_EQGt*h>Uy zr(c=xZ2JiVv+RWi%ARxLHGq@JqfZLkN1AfmMGm1)olF+3qHm_V<(LNa^r$KL514sl zj(xWY&$aI~FwdTBV7@)czykYrAZt9Mww}(OINSohNr+E!$ZndjPL>hXK>s0aR9@hJDeHbdK`@M#_X7Urk5@Ifj5Qush`3e-v_>1_dE zt-d9b^bL$5yJ{tqfTh4%eM=_k8z`B!vzG;)^zoMhYk;-TRsbk|(|!33l4+7#CCDXk zMt&hzVv=9*QY;t9KQYNK=vMQxq}<}}Ajz%O>=%>VDy7Y|uSasLx1sHvFUhT-heO7A z^Rq7CZlpa9ljm%pY;M)GdmA))BkeI8C)@;Z`-P)#XXAvM18%o)@ZD^jaLd582!~v9 zWK)ijO&<$K`5n2GW8~5{;o!U3IQ^RrKmS8G>N^`JTu*TCfjcz$l89vMc(07$w}6-I z8t;`6+@FP`eB-?`f_qIk$~WFCBRJ{mvnb__dlIF^?UXCog3RyPT$TIq4s3UuxBEIt2J{Vn&a!L05(!-&@ zG*PZO;2uljkY$u>8Mud&I9KNb_cL&^t(Y`Dy$^gH_*hBEoORjGnvZmL*~%`U-zC%5 zBGXo#P^J~x2Mtda+4G*X9=Z1fGHr3vPF7?W7`>v%p29!M^B&*;@{>#xUrVM`O9Bv?Meg@{+7Xyvn)C0WvO~miXCm;T~ zuD-Oq_^5oUz+VH62a3^gpzDpuI;}I;2~wUKL1-f&2t91%t+5(tvrZ6ttP!NY1A@@s zMs&BDK7&QS)Ry9!8SSB({bx6^*=0zQub!Z8FjBJz4IwG1OeyXm;y@Sbh zox_BQmU==zZwXKPl;DJ`1}9y;l6*>V!c73TQ8?PI1Si}aa2tdptr9oAW#IlG9PL?x zQ@mHfNpG*DZA);%?Etq@ING@cC!A!~bHdT)B{<ZiDZvRh2b^^E%6{N*!Yu>$3*qRi5}a_af(r{rzm?#G+X3zY;poE>oN(#Lfcu4` zKTB}J^#nIdxMARM!c~KtCfo>cIN>IMyGOW@;Bdmt0XJE=(co~xEd%#I!d(pxC)}&x zZWiu3a5&+1fSVxPSa3Mu(ph&O3ohn|W|5OgJuugio!J$J zz-;?W1GDVj215(SvzuWuhjDlkmTM3=5h9*}*ztKYM{#nc|X}eS1ygLC8&^{Vl z4xn!Y_}+{BmJDx0pEhseF=pv)cOPM`<%*kU|C{rEhx^GdyMMObrt^RN(`q30e*=T| zJ_EDuFMyge=dk8>0DbHO-lo_GUmPOOZ0z=jIJ5G1TolLtfR*>+S}AwHottjLR)30g zLCrz&2i&=-5C5si+7H1m$6gs=ZYq1evE8%R$C;-W*>Ae#`HO*^^E0r(eqE3@dQA}i z`=h}Z+8Yeaw$~e&W4~-*uKjxhWy|^weK3-~xPm^poW8k?J{!Rrs-4t`>*7e8;{JoMR}2Oa$K zgATsb*L~nT^xEDo9^Klc-#YqY<^WmtE@)+Ahth#{=(@FxDIF<8Z_1WI8N4_;A2F9Q zX6#R_>zcT8a&1F6Ar(r?8!Sov`MEiB&o;HJ7pu2dA)(Zde&%j}j)D z5Phf|KUZAJ-@M0(YXi$jw|ReJ)jWk!hYUCDjonUI8DX;(M%l~2nfyO@%LY7NyU*R1 zSXCUf?s)RE#F)=kr$vme4i6a{1H55$_0GUzXS_4xUkARCJ2Af%4`}RToRe*8(`skz z8v>okCM_9ukw2855Ut zm`ObJ9HTD6vyt{F9$JnyC03~%p3K}txsZWhX~RX--QZK&@E2-Bw{L3qb{-gj4vua6 z2lb!uDgV`_{I_vW?!W-Fa_oPy1sHtFe`=?DK&-Kem^Ce;c<7_Y2(JxNC5wDdX8$DdSmb%6L}F zcvkF=XQip*8SB_eTjOt`n{U2>Id-mrK|5$*ww+~Qmfh7ro#8sMj!{egyU@!T7<(kw zUuTbN8T30B*uc1BP*6#@C7ao%pqB z&X^Zt+%i14&*5X;!(_%2jdA4D%$Rk&afCXWab(*Ui7}rt9++_=6A1msI#}^ijT4L` zk}VZ4IOBxYmD^mq&PwEvbST*|y>Vi35{Ilu&i17L7bbCz4YL~DeMy|wTcnc}m^sq) zBp#g^{%)g>=MewL1}_~O+XM8y4BU^vi4Haftn#{yKzU8aF6Dttp`-8ylB>SlvaTmj|vzIN;}Mm9zyrNfOOj!FKRp)?41`)>fc57yDqJ6qul4w zF48H-(=M05zoPfKF=Q|0W^9R<;#c3i@rlznE#PaB1vQ^I_>L~VmVRpZ*p*+UHr}+DR^XW!{>=rXHb?J$y?G@`7~aIsHbVK1DywIKA##k$PEh| z$Q`ITernixZdlF!;(;}ijVWpNal`8N^%__sIhPXF(+yK!Y~0&-ps5e#ON~=Voa6OT z!QRBGMUsan+1uyMC3avBZsKiN?>>fxD%!0dv^*3z%!;q-=gcd#k7y0$SxZ^9kEoUP zL+y1Cjqk^Qz^ql}*k9mpW9)V271Hn6hiB+ibBmz8lW?uC#|-2ic>{UZ#X#0T4P^dj zAm@JshlS$*5Tq>9wcf|CJz84l-^bql{pF$fJHUO!+tuOX$OHHlK9u)zXLJm;#kkLc zwcVBfz^%b8!ZlOR;eymN0~ll6ZtewbbaiBM3-gV>!B8}Te6GiT7Wz)Zw>Y=0ecF7V z-R|gIq%{BNTtsW`%3FJeR4$cCZ4@!}$+n+1FxUQH1B>jX2IkpM8dzvAHn70{rGfeO zB0q<#T(rm}mdZJozEU^|h|BjB^;z)Rt)TY!Nc zR#w-RCa<`ALxWS>?4nZD;+smOMM*schJ7Yos(@RJx@+v<>8=5PoMsd zhogO@CreHa_HeYFa1+1{^l;FhbZ5!ObCWo90EHJzVI_`+RC}7I@VC7D``G18Pf}yT*q6bj=tmI z&H$I^zDpV_=gfGYP`v;8()#Vg;6MIz*0)%P=l@*d{-V9LWhL#Y^>^**lK!BuE`&9g z3@g(fqZnyy?6`4ZfAms?n|%_r&p!4UaURS5UzL-zo0*3UT zFYmz|r+onAtGkh2q92v+r<~VorQbr5MJKUqe8e6`t&?kAstKA%wcj4QM*brG@jhv5 zjr=`en~^~U_S*)RW4~n}cL5lfXaCi}eEZKpjWJuXbFRcKz}4Ah#E;)~x%w|S{ABAe`f?%p!8 zm^pP%mzN%;&WBu`V}Tiawa5MF8u%=5-hBw$p$m_n?gjr8bvp#Tj)zY)4p!54|1CUq z98Z0Pn*iMYzGaMue*UB^>C;g*5BS~%)Ep1KS7Dmd+Zdx|=br#`~%0CzdK z{iJX5ryacf#aEXoeoFr0s|w*#@~;LbT|G7b3E;|wOUZu@xbuW_^VhysQ&#Y2f{(HP zMKWr*yQZyq=(T)1>)EtpIrD7l@+$GW6JPsxjUAJ^>;UKJ?ZP1g3Y|6WoMb%Y2YuNS zTdJeCD;_clnN|(16Y(VT9R7CZ{x5@*om1=AFFI>IE4sUL-5BGoH`i4;?logVq1~DO z*Lef%oSTnl=S)3s;EX$kc4DDZmm@&wbZLxZ;v0?GttKqbK44(3z0W|#d_me{xB1St zKQj>fl7X6&oY;nTvUO|8hr`GR?QfQR&|bj(*gZ8)tcFI`Lc1^0cCve}q)%k`+-&mZ zE)bx%&v`F>{D#37*&Bhf5lB}35&u4;zhv95m~i(08OZ)W1KIy)V6OclQ1kO&z_&WD z5Cm#p^8oCfS-3d&M*JPO7PkP`O1lJH`Jyug0pyG3e4HuZPCaL@O&|OnkuPJxZ#vZ) zv%KH1NDF+woPS&3_vP@RIb#4#kVbqtv}4W~Kx;Z`pI#+#z&ow4>o>T=kX4-ZL((>|%Tt0^8XMK;{&;bh0BT-CINk^ko3EW3&R zm;mlNFJ6^nr~l+8b-H^l+ko`t4QXU6#Z#CR4W^;D#k}jLGzIIy_dH#5p|H z6WoR1wnGaJ?>OtEyTOTfnrW{H{L@VP>0Chu=hd2rS#jw)%_SKT!$T$715#u!H{(W; zeGzS>_0m)x`Y`2OLF#&`Gu9N@XB%26vd;v{J|!L!&B*T5%Y+r$-!ibkE-^6Q?jcCK z6bsU3rx|>fU1(s?&NndI&NVQ{4jL%?^ojR`h&Hz~28&kZcg_^ZR+51&yCXL3Ol;eo zuyLQlI>V`)`*H50+EK{)MQkOy%kof)o#aK@E=oJxQCKB$1b}J+`J?X zKA_EC1$Q5~?Z~-Q`aa*IZ|d7z(!tPoGqP z+CMNb+a7OVmVJYPMbPsV(BkFLiPR3m$JBgi7cTAwjFgEDERp&0X zRDDu(JKW8Cn1S5QX<&|BWnivdVPKv;6sUaxSRw+@Y-M)^Gbj(fLV9ptEf*D;CHnk(-OQJ4Lu4_#danOhBR zcM=DU`W^Y*lEguy9h|w+$KbY8UWfJ^y+rga*}4;Z(N63@&3&!7k*n~dlu=eQ{U=*vCG8@LuYL;z>lW6OY+N`Ozus=2 zq5G=O?-?G(|GG~`wv*UoD;m4o`n`0e+vtm8tDq%t&ccQ0F?#~{Enb+>U3Y({?;a?o==96z;B@J$s1m_-pjf8nS%@=W#icE7`Zw@OP1|I=Zs&xHOn5 z``&Z^Dd!)hF#z~w*_UUZW5ROnvklC#&onS-_ck!wE;TUA{#9^9fF>OpG$X;MRkSrG*9+BYOiO#8w;}kejv2{Ol?Iov*u4WC1 zb-UP+bIiN9&xW>2L|fUmWb*!hSyAZ{&n;l?fnITda!Bu9&i4WI6UA5DD)N6)bcLLJ zk}&O?S`5^9syiYSH>}SgJ@rb_8T92hk|#1!oPo^J1I`QHuec+(!LVi zKe`_o_H}DxQ%za2?H?N$wC^@B$G+3RTzj&CdG;ir=8U6}Vf}GAxPy$Bf5Vl!x}Eg4 zQg_VM_*teglQRf2<#%l@eemCmznMG8BIrEL@Nz_B3i2?*INppbEMwd(V_qKMy+HbD zOCaO*h0Yp1>(1Itj92>G5_vK*#{?al0%OD+lE##oOI>zQwRbl69J=+K;n zVZKoc?MW`&CO>0w0sDS0-y2vs~Ifg#Ksfx>pKabDxv!nA@9>n?@G0Z_ChMk$oe1ebX`d3-_N< z{3CiUg`P*^_h|Yu{F-xLYG96CV_?v}#K3G@^^$C@G*C2s;&&G`?q5LMZ(6=L-17D2 zAE!)3b}#(zoAtT>fBwid<<7Q)26F$8fjM?p13CX|V4i&nQ29R(?LUIMQ#?Q)z6DkQ=1F9U=290R%Y%Ruh@GVq)6!9B$NrsZ?Sog%xIe>^_;0e-^=C(%uG?~&&7 z+PCAJTk^A>e01NA_V-_9%AIYGHZW*kX<&{$(!gB%a-i~^jD2$6UDqUa4c;uN8y_O9d%Y44nu2 zK)gW^U7;4(0F-XhAbBfYhWG)-4!s{PIrPG_4qxqkLv<2Asx)-NI2Ovo-^68MqeV@Q=qG!fD?9 zD)^7U$Ea`W+K98aTQ&vhO1ybYybp+%Toa)Tve&+w#KDJ@LH61$;I@-@s!d@I^$Wl= z+SBdnL}lK62Vc>N3hfZ`L3wR`BxzHae=?iGc=w+T{G)%@123T5M|GlH`y~^`nSTR= z_8J4T?H3HpvR4}T4RoRZ0~&wA&3mDN-2ZQ2j{OS*bL~ffqVay*b4@f)p6I^EUK_Fe;n_8tSX z?avM5%|8SGZ+S!ZT+L07mH+R!d2cl^Xm2qv$9~hmT>CFT+1MucK>p(f;Bs&WyC3BZ zo$D$>{x6pN$M5mRlbq%9cw>b8p8P-E*$2?T8LnFBuT~JcD;1Rd=j>K3^wl5;-P8&~ zL#2YyO^p5awIzfL_99%c58;A+3FrJ)Z9l>V%Lx}8Ot|1s!gZFlig3YUgbR)!TyP}e z>}#kUO}OCIgbQ9rxZqgAIlp=`-t6Y`=4`d2%bWK}Zc?9A-uxlB8<7)LIED43t*;Y~ zcB#S%C*J(7aI{esPPhr+t`v^;s=^632b}cw8)>^LoN&v)4HFK36;8NU!AWnwkv6Tu z3AY2>g~HLkRXE|qm*)#dTUX(P6A$(ij&`rY2`3&rTR8fl3MZU+uuM4mqY5Y79B@5_ zqi?Ek!Yu<=EL=G_oN%v#D-e!8tHKGl1DtgA8|l9)oN(#b8@dQr1r8^i_60~Mzi}8i zoN(F~&|bI^;BdlC0Ou2KBsiRKbHE*je{UQO4kz3)a0i9E8XQizSHXQD+;!k^!tDSj zo&3hJ;BZOV#`&vHz6^>Ad^=>uzqUSMF3{iuPt_QM9| z*_ZX5^pz6Y<}I{yT#G zj?>xi*py+7nTUQcoiHAG8*?i%@BwTgi@7(=&-a=BQ2ful7ydv-DEkr>dM`Tm(5SU+RGr}BOVxHyydS-FS#%*;^y zOc&4k+RDY9tw}R@yYNiXzLjsqKU4TDYtlO2Fgz1Fe1?7d!ruy9oNG<8goiJ7`P}q@ ziwj9V4!)58UHmTScX2W4w}LO`f4-GYC(cNz+_JBgxqZ~jz#RKq26F$mf!THs1GDU6 z15Z9~)-rF94(^?kxn-KYXy*oBMECXVarc64W6kGD+tIJOk*+t_&Ne(MTa@X;orfP-p`&!+Cscv!oQL)Pq3#^ z@4y_Z+x(QU3zTO`MWneUJtA9Zsn(-hTWDJeb0XG|ocD(c?MgGgYyMqm|B8Alk2k~C zBYMy4q&%0&TQcsv(s`d#Z+1FoDOqcadp!3C(vuzjB?E)@iw5S{YYfb_UobGwUTI*y z{X3xgd@HoI5~sb!`Znuxna*D1R_NV%o138?1$7S&0Q26i+m)|7Ie&&|x0Rwse$3WiyF_8Cv4CMVE19|_) zz?0M5^*Y;xe4aOb+SZ8^Z*HyUoXAA_c7)E0z?ZsLPW@VB_6-*rTcP$2f4(ZQp!=aj z{}^=)P_GD3wtHj4hen(EE}P`o@k`m?s(ZdPAO0fZ(C&E3=h5yDJUZhak8Z!?;Zo>! za1saI!rP+Tfl1s^x;+wyA3h&EKC_9$oo=e;qGG}V$SI`|niu+tSAv^3g+DCbH zqkU$51Dj&%9nzjIoqC6~-_Sv}?K3cFC+3UxSqlK>Q>-XmB>g7VUiOfqwsh0f8?T2U|z00=oYtFf+ z>O;=BlW$KqeV%Vm0}i4tvWKYeMF;nou%LaHf!X$t49v3cFfjk5_VaJ@4&ukG3#9I) zXiKs6dHc@3=jMO4fkFE!19R-r2IksV8klE~G%(-39H{wU7WRmJ(Azt>7jO%3_uy(= z8(<4I>l$<*(Q7^a;pjkR=s@Ofetn|@8QF{e6j@#sG3$}=y;+ZhE{sg0ADfYB!x^84 zPs?B*Ps+MvA7}s0OAU@(JBxC8d(8S8T9Dj0Ivy7!an4={?NRPYJ<^kSWE=TcgG(5> zp#O8xA+_hy*`FOZc-iKT%C>#rhWf2N+tVIjDET$pU&orab3gj0gdL?nXOGVk@~iRI zDcL(>Y;yhFeLn0tko;~gNspMf0a?>gTGtsGEVMskjYxKfucN`~Nu9jV{)zj~zxYRK z`~&!9ojl)u&xEn&XJD?q)xaEki-AG=O#`#-zZjThZ!%Ci`ibARYc7eh*T=_xvo!Xd z`PqM#&OS8Wn48M`4YPgWgg>yQ)wM;pp{ILmmZ|ddc{lIh8pvH>z(V?_A9M0-+%~KOGjI zflO@6^hH}c`=W8^t+YdaePGYDt+B~nBlpvG#a2H1avuN6zMMAX<39R7&)uI>VYY21Xr1-$O+o}qj-W$B9KwnhI$ zT^Vp0`F~Sc_z(A=A^f8>E-;YyKY)WNkM1$Q-fgdb;JiMG`SW}4q+YR1E52xYU{Pth z)j3vg9jW7=*ymj%&HlXJO>SS@Gd*KbYg4yK+hAXG-%wwaJ93W1KF^M{HU%SX6T#gM zZY{XIxse6LJ@{c>B=NZww|L_yG_&2ap_%E=il=-J4l{2Dt37wnUO)Ax*Q+0!7&kv+ z9{CC5&2NjT2fAC^==Npo5pv2iIX!ZacP83KcPfjOlMnCHwvG?*-K{J(lJChK3D1lq zW;=N{`aAH>NBd`oGLPDh?rGYAajJlJ5RLD{pNn7bi0ASab!>D+Brn6-@;+tk@sL}_ z4U!F%jWX?^A3vfkK1N=tjCGWeHRk>Y(~9bK=II9B(pDZCf0Vw~EC0s&nso7Z4dnel z139zc4}_0Qy##)}7{0BBf2-hQ=99NGk0^QVR=v%U*r0c)zslD}E`6hYQWeR4QdSpd zpOoJ7&>PG(PdoBFN_liXaR&2R)w4Hrq&-qT-gSAGc6j&Q-r@J&wYKa#TYHkS?5Ev& zNPl?_p4yMyV$~~WnKm+WFn7 z;V%GxiIdL+aCd`Ko@w2kcVYU#KMg5+n`Xd6`%0vD&l+O%&u7_x!dn*G$9wNsWZ56P zd~g@#{@Wd=tl`}IAoY*hL23Wc)JJcke7P@Zd-Fz7OP@8jHJA91v%ER~HE-l(oIh(+ z7v5*-!ucU9ZRop+4xudxe}^pr_F8_D<%@sp_r-VQ`ZyzI*$pL?gJS*CqUOwMg84+> zk!7m}hvLunyS!}en-@o-O~cEUjR-}5v!N>T+-sGQ7e`l@IXa~;-oUsQ%k=5ZmlfXr zxd!Bip$Fa)Z7%l3W9e4;L4T$l@8FB?E%C(M}t*1wHW^sQXALme<{hr%; z5ze^K>OY0C;Z$T;XZ9a=vG?@lUu-&)F8lX_9kkJ&z7_G^$hOasA)mnmpCXq(VyxTN zs{)&<&L!%;(F*nibJxz+??c1?6=~hex!014c)UXe=R9(CPk#SWl{-8)geENwMS3hol&5E~dHTz#ySh2IhbI5xgIt*o6Vcm0fcn05UzYy;;#$; zXZ&6DF1@vG84|voG&&MC?gBRr>fQWFh%@ePI_Em+ z>?EC?dwo&vW$Mj5e^MjoixW-0Xrjdz)&Dy`@I^(ldheUPn=9{xN0m+{VX+UmcV}Ae zqS%MXW}yDxxt2FG>1WbeIo4;jG$_3^tECob%4Q7+*QmbqhxwjAAY9G2_5!!$E-uKY zPRdt#q?YRjY-=Wd={e!^i9dsTpUn5hbHe@kRy$Z_Ze4@S154d{<{xdZkk86ixm&g} z_PtD-a!xh|@0oNS7!*E3=~73%!SyNm>t6G1*Ikdy;ts~O(s^_3Pv8Idf=~H=S!?vr z8uZqGocis|Gs4~YKQTHj8sdFB(^oV7;SB5-uVaJBpx@>SLVLr3fp*v`fF|BFr;KJV zH2-%bU!|jS-pXh9bKir0u8VxMi8 ze=XkXQr`vc?*(Br1AOS>r5Jr@6k@4?mX8i!vW|aK>qsw>;5_ToW(s974d`9 z0*l7>vp55P*_0T4_p&c-Qf#y@>Ze^6V^3Nyxq*#iIX(Kw6rs;BEB=NWBpFXgq@ic@dNUArroLcdu6^wV;5VKc5=RJ zM%txQ_PTZ7;MRL*aYcM4{>juyb?k9k{f5r2{7m0eeJ_&QH#%dz z7kRDpYOIXj@b17^0y`cwiVR?EqCK1xL-LOEvbhzEhZXTpXbbDK zhAY0Q{M+$|MDBsH-69&-{)akh4$w&1zF6ewG53Sh-Pw)g^{I=S1y1KJ8()JjUEDNq z=Fa>L@Scmi2V7%kANoy^{Vx|cSvbbpM#}rXi&MUhPs{FB$k}Zt-X)5T2y z*WCBo$j&Bs>v_(7j?Ic3SZc+CoU1;-*~|BEG5Wnu5IU$4TrE2z@OjRM0pA7+Ih+o40arcfA!awt_9=v#c=#u%nB20{Z8EE1okgxJd0F*?KnrD1Nb( z)8c0@h1x>+(iD6Z`zIP{58)-Ry!f-Vr$}?X*-A5G(GYAAYNLGO+ztsFhza&=O!hKXp`JWON`g3B{c*0Dc@bqf0 zydNi=@~-!VJ0e@Zj*s5pP8P{rkDu!J7XQ?J)FCO4xVJ^}Xm_9oa=S+;GL$mv+=A-e z9sB38W73RGi7^8xlfkEywee4hRp*13ylJkgjP7iyi1HqN@10E*(UuP?qL=zIdza=*{hMcf(#>`QB`9JY|TDniK#rEGO%K9=(6q$ zWYP%aGVnr9HwV%WoJ0?cBboVO!-BZTd zjn>g|@?+XJbf&dZ|G`%)tb6-}m*4&G1v-agzKhNbKh3vfwfSCmW_T&znj7gZ^@EIi z%s z#f%B%Xj}Kv0#a1m7B2Z&kl!)I}Tk#W#5<5 za`x}^Z>0W}(1G?`|Aum?AAc=)yT(D@{JixdfW1Zj zjQe;exE*iR-`8F`@B9FADM=sfO;sJ~r?_d?x^u#3P@cqll!5tE;!O+N;f%Ku)4N6z z%dL1Ge)#n*);!-5Eq#biZ+QJZ&HvIT-AKEYfp15Bn!%M#3yj{0zC4_BJ1ywHEgz&u zcMfK}|1drJqy0bGx^Hq?{Iizy=q|?OU7x2%Ki!)iO&m;*HcwBBH}hZfY~TlsU-uI) zt6emf+b+7Z5LnzUI+uK!A13@E!WRG+rNws*ZWnDC+Ai9BKe(mfuI?Hhx~Fq1bLY*P zJI6;5HnLr`H|=mK^Xr|X+eH&&`G@&P%XRIdi>ZU;xuYWlMz_2P|FhIz76tFJ+?KpN&_!*~f5Gdi3*&l>hehXl!s=JjTDVp}-G;Rm82Qo)4u* z)m|+R6K4TwQO;V<1IMmTi`RjVjZBNTj82Px`aE%06K6L6jwOxq&f#L}acBc=^BQfq ziG1E8oN~vu0!LRwkuCkV`LBf>#xFE6Bih8^+I4UaG78$Nb;eC2uJ#e6fLUTN|tU&)&r z*|+io$|^paa;n2;Zxc^^_Gdwt&;Ep8d=^8u5TD)d#(f=pBYdU!>*R+g*8;zdZei+= zteMHTk;hw&Ub8vy&tdV>twwL!9QbsY!;i?x7&7}|bcC<)+MO=1-PzupfBcX*$d>t< zpWZ^+M!%Vfep6Oq^c&4fd!pl5d7-upbe^6WA@ja@H8}CT#v#pLO!>N85dI!%B-`z8 z<>9ON_RjG(Kg>O4KISQD*1*J4%g|Y3X@1$ov{`C9&L%y{estF;?}qnACX_cK->YaR zFRj>{RIr{)~>e(i?sdFjiJ->7{b@Yc7I#=WI{2qtblq?IN>(o*v z!)uhm@LFE~@K5Ml$t16@r|@kd$5h6Kw4#p4H;uI`0@Sm#(+>wiFT1_5wHbaAU%2|$ z{2puihcDy*#7M@~tEGn@Grm5=cgpyhV9x4|v5h0q|M^#R`jFt&=>LM~1oeWqW7`sh zF76kk|L+r|zh?_F2cIeU5O4knE)T`08GN2S)xdoF$Aa~t_}%8az`j#(YA8NgaC#^{ zNpN*#{C2^$mGRpIU#^T#6x>i5zgh6L%J@x!n=0c!0LJg;KK8@x)q=;wqvD+lw2#G; z3-GV+!-T`{@UD2V?q7+Orv2f$(@CTI_u*Up%Rf1uoJP;fB}{yG7#dMuY~0&nAa;Rh zN1ruHdav|Q(U<6WDEg`3&Uy>^*s-JfM*FRVOQjcm8~UGSP1^f#U4bb_>A>)xe#AJ6 zUCp73^6)Esvu+lx^Jf{|s|R|pbVzKjV?1wI(TUT^cSga^Ue`v( z9vO;WN%>rUYlfGH`nsEPiJy~r`g7>aE_+P*;OC(gHIASD9Xhk4MKu;mhgBhHj94BAt^O|>>ydPrhA|Kgz#?p_-pD{Zb=T)8__%o1A2hr74o+qi#(Q=&L zEtoO9cPo05sbg$V_`c)h^*-U=KN?SYbJr<1F!}H93&M9D=U?<=$=T@6dLwr_w$NVF z-F8y>QrqXNrJ zv{yQ<>!O{{`=fhS`=iq9cYf%LH#ad?`GoPT#pkYt`{FHoaR&)QrZgY&#XmpfkA9Z$ zM|VTxyZxlu0X=rMb)Mw&Sf6#rrPL|Vb>70>z)RcBTiEf8Hx@QR)0y{nU3A&B&Wj$L zcIu+tRpdL2G@$$4Pm|wr@_3H4$NQs+oBYw(YHY=8eeqo{`{JK&0AEj@4}pi5Vh{VH zpD*xBPgoIP9?+bL&iwy)d-M3Jiu?cn+*}|x348WMk`OIP(4w*g6eJ;t0jvw4RBcNT zEkRt0_^~2Ff}tA3R!&7x@KaD|%{^3&6)LG(K;*f z>9v2$1$IJrx6FW7rlpWNRo3Bqj9gk5c>FAz^%28s198Rm7H5ZOO{9@IX$9W|KSPig zL*Sw4s}G&?Abiz~(Uj5lzrA*UPI+p#Y?2{h1X(itHs_J6u05oQKYge?#6RQi_o8=v3t{+MJ!^&1?XF8O)pmn+t=j!NI_ zthX$CE_p;Bq5B)hr-EyKYpCR>AR+Sg7ZH1hja1Yb@w{Epm)>n`vG@eo6hhOT*A;=;Le95cB`#e)~Sko=_5F zOP+6BUtB*6xG`yGbam5)cCoeQrSbwEoGhfg+bPfFd$yaKud%D0?ZZF5&ytmAc(8sO z`6BSpK%cAtdJBCJuM!_DfgVSAVuQa4VYo$UR89sldU&{W3Ohke7yp_zuSZY96Q z$>41A%_FkXyG<4!lFyiI53);i{s%v5ytFtRWZ#qI+*t~NvjT<8XCvphn)~f<=@UGH zoYjs@q&+M78)J*ai$qTY3*s_Er4}{^Y=W$As_j1*n^E7_fGcK$T z^aVa=5(1~JBkXLVu8H$zwXbJ?=#A{FxRLZ5ee7w_H+s*f?2ndj_j*FvhP3{)2%jAF zvE`vJS8O?#^3X+U;VVYIjtIJ={8L;}t zE@k5O@Vyti$H%+e_PoUZN1zR%2N?%!jOEh#(7EqWxs1~X=muMa|LDha6*s`zAec1p z3cN6O8zTSACOx7jM)dT~GMArc&mg`0Y-HWnef6@hMdbg6mhSen2i^8OsyY&VVJq3% zcrSLh$h^!5xbu>ay+rd;gPmIQawqemJv0XH8HWhm3x^aZ_&s-dJ1HxlvbcXj_79^g7ZNWy zJ~9r{>2pc9=x?@_*xVnwirx1MtKqwLz$(T@Ir<&wS$*7S_R11J!FP@wC@5GpgYssC z`rFoYzwOJ7wY5&Wn7#E|9_kpfZ*7X*YG95#s|?I>J{2%`=mo)1iu2wh?pThaEu#DT zs#EaCN(R=*XJCzerCy#<>IdVqZ$?xe!Pzs#;s5dS3?Yx!S0gZXn6}K-JS;_4B24F@ z|2wSqeshOPu0WoM?ihM^4C|I-!cA|C3BQj2$!z46Pg};; zfA+!H`hQKlu>P|bx(1t`gPuGI9TClVGA5{X`4IgKJqaB_o?%fcSOMH48!yX!q^B+W z?r*7IbsKvw@DR~A1!Mp7OLtr>^%)pzz&7_U;6!xthzC1AVy>dU_1`A%zkgG<{>#D6 z(h40i>kIF~2e4O*4~Uj)T}e1CAAoCv>>1&_JJF09cvS;!@9|;BiYnqN@m&_qNiWZb z{y@`q_~JNYi#&g!9{Bt)y=#BdYv_UC+p+-oMqU@+SRZ~v9CP=8ApT>WozBMaDAK-_LEh*$5W9(9Y;_RW^)nBeCM)p{;{L2*< zq2}RAZyuJ>C&9JGXEE8EY3Jk$AJ!aNZl=)GtXzNz@; zLSsC*^wu$#S-17JecfH3t-+KpxX%lmBH!{Jw$EGdEFhm??$1X$QX}PQ{UM5OEX&=O zVECGIAAK+UxB%d~i zz+ZEpA9P9c{+{jPZD0N~__wcPV+ZrBd3h<8wo#_313NviIH&o^s|utqHX&0?!)Dtw zV0FeUblma2g7}C&ZFFmBC@^@S(U06e-)D5kM({L*Ji=Xg8yX02fWLeUtPP>PT4x*2 z+Md=;LNj9OL#5=K8RM?Q6rl&6?=$NgjiZ6VQr4B0lIMY#mrUERv4s%W@UI=#nn6+g z3q4A4*4gNgvbm0a$Fho##aY{THueeqJJ&b!S5{zpKh2G=_riVyyO;OVdQm>KOku!R zJYsvq9`Hz_y_>j$)vS%+-xtB9M8OWYJQg}ckEcQeI#7Ah*4Bns1L8b&I$Dih0BAhJ0(;CvExn>S(b>1YQWIDC|YRxZnbqVu} z4Q5$^XZw}yCnsGt@M7$~izmfd_e?VQD!KqHR#1mv@=uf_n1n7QfnV5PsRLO}K0bdW zP4lca;M2S_^lGu2M)*8w@^w-gxU(~~%}XP!^V+5~Y%e=ipDB|tNSgYpv_#V62eOX1 z5Vp}q?4+S5y9Ko#)_^ZqW8Rg)mM1yulYL(DX=-Dv8`n<`9)xG_nUTm@X>s;Gc%*^* z^^El@#$019IGkLJzkjmxNp*2}|1@NyPdZkp-Zj@-LnClt)^yPYLJ8*Pdf=?^H5V>amX02FCXm*6@2%--+;)cjA~L z6A2y~8P@{Q@d2k8Tl7^kwwaTI+B?ykaovEghABI0zPeMkY`KCt?4noM|=gwqu{uWv=iZ2GM(hG&H{1a=$OP7bz;Mnqs1 zSluW8cXX>yfS3LB=}u(4O8O)oC3^6de8rK4w)Zi5_Oih4QTTqvxHgXx&TRE9S()qu ze}MhT!0#c!Lli93dNll1+A3Hp{017H$y}+A=pSF96Pic6v^W-960=jKo;?nR} z9&qJ}FNqh7iT3=PcjWm4E&c~HTA4S&R^e&EOPI$wHQ1jGFT@TQ!EN!4#uIqI&N7h9 z6@vdaM)Cg^=JTNDQ+UmOG|`PI%%$qShw(#(*D+H+7G@Q zilY4o(b=MD{>S%D4DMqu$tUQN`@yBBq4{;dx50VZ7ZP713{ao?CAn8P)j=N7c<~}1 zY4Vc|RmTJ+)9i!C2yeyTO~-;Y%v56ossC7YLgV*S?+4$2GadNwHe_2v)^LuH z^o##h+2^`cBl~Ow7CUj~$Ehdbj6VuztZ%`YoDnw|8R%3vWAw*U%00=>->a~zfHMv9 z`9cP7`-O`$=S1Pm8BsVRToK-AZgxe%CHQRQl1Gq{1bbRzFmns7kEGe8X>OHf}q1xii z3=bA6#pg5F$Cv7S4DJ2|`n!KEXVmWRSW$i^>uV95?M=Dh-Au2Ym0sB@vtfWg*pe4x z_=9*u8#c~^d7OnPd4}@b^4TW{oEf=>dfo)ag-@dQZ!ylTl-D^Xo*Z{EbqN>M2ZLXf z`BnKu?N0nkyAw|#zl-i)0u1MT#~Ska`c4?GJB9pS_x~!t8=J#pE(~|^VE8ud3Cn+F zV7N}UpG5IF=>DW%cbK(W+2b$eo%v?{AlaTpSa5*aUGU$Fk z7ly}qFkGcE*Z4C>xAAW1K5IaNU)cjRH-h6_-hr)2dhf@(fxjqvx)4<1B2ONru> zH}%uT7o+Mw34SDlnL2?VQzz%BKm$*d8MXsEn^!V^N$fuYc08Hke-C!*T-f<9=0x=G zzkr2}zdIRrwyo_l|H(7RDG19U+A`(LeQR?&@&kN^F#++n>q!0?j(jAb!n zc{_5~JAWJ?USjYt$-~2+Fn$IX7|Waf)3N*j-R=YMSn}~+`0lsjVv;kCv3i*CY`||p z<2l0{&;O1ZPdhT6$Y7E2l#NVyARqS#F4k)QgnTl0$^XFO-VSTdA=YlS{vYtA#Iqh> zpNgNC_Aym}D+*J!mx=rVY?-n_8lOO4V*F6bF6)O}hiwP{sCd@VOquc%t2qsyXU|W5 zH+C}V*#X&w2=_tXG|y+zt_QJC*0$Xc@A3}Tk>N$!Yrl{ET%W|UM+%tru}A7!aMdbw zb&16HnFq?<;*~jq_6)|3raEh+qY~D8ZB-g$_!V91ciw-ZRYsQFcHAbgp0sHCCRyt@ z#I==LBZNOj7DmTw@MziIdnX6APO*(!rpit)RKv`=W8#>S@K1bN+2c1vev(ZyV(LGoj!)4|prO4QiadW6 zY?ICRvSxyAG7Dd9=_idB;Y;!$`B%i$H(csF#=c5?zKOqvXBuhSNPCsEnWW7o?H1za z^86Ir<|@|cG=|k__*d9;Nx?&wl2Eh;&)16DBc=o;{C^Mn&?qQagRca zdRapp&SBENeB}E7C%#trH8xFTq8-*hTw1(&VH7R?=W$vbk%@Y&J zzROLa6KU~(z}xmNu$B)-7Fv_z(qij=m$#iq`$W?;*5BnF9(JJB|4>F8_-ocx_Oc$q zI@cP`5t2Q;8F*_3-aaeUzJQbA?Ni|GQ{ZjMf*zglCf_H)+-Dxl`MBd~_xh?)4*MjU zfV(E(t{J%dEDG*E{WiG!Zx`-D@VP_4kJiiYipY4-ta-e<^ftx0o%jB<9eLaJ!Id9l zH*alOH08YA{*hf?eKv4%>b2Xth@CC#CbfaPSSd&L*#Z0~+Bwy0 zhh%-p^{0mIWandO<|ok5{lNBIaJv(>qhL7DgfFN!AGw_IsQq-WsNK!1_Y02*&-dDW zlr;GosV#aJFE#J=8JGt*_0? zUn~8H{MqCe{M=3X#!mwKwrFBb*%-PkpN_`oUAy6U%}cc1(BH$w z{dYzCICI7Dl`LmJ@~;m+kyDLLMPI9^4{O&J{vf~O?4JK+{I0VOW*AxPRmNYk)?8$) zxrZ`3Wvvt+@PN*f{6xnJ>A;d(`||!D;rAOS$XXwLo9vq6G~dM;Kgd^AtW`elZ0JE_hmz{Gy|_>gA2uYpm?bSKjMn$Kg0X+LF1>qY3H7IPw>q7&B< z&Y&tmFE_ zj@(n)qqxVN`M3u@Rm@%r!Sv+&T$uhj@rJeo(~I;D952UbNw##wC< zW8TX!v?L9DVBgh{5b@F-G=C9azQ>XCV;JWu>bZgX8n`>f__(r0qW!SDnWyg1(wui}sKiV|%t|2S}_AuIX!dYaTQliEwk#|>r0;VaGy^fu0jt2g&OUGY_`>N)9 zd#K?h6X=_K=E@Y(*Kvf;wSbpu`3-UqOmfcIv74jkH# zF7=O(J67C+wcM<%V#$M2VO zzAB5^V?a194Id6AhK7qZZv)K<(knH;Mma|tWhjnZDjU1Z# z66cutICn6Ha|bP3_dK@x1I`-x5!Myi(}C+XoL^(Umkqan$afCk$jZxfZ|7A?XK%Fm zX20SUO6MEdV@KeC>%SeCB|oCT?80;Ky#j~XUwCI(pCuV_)&tlB;${cVS(33igZ(yW4)qgtm2D)&T`W2iB;R?+{cb-vx01g2 zsb6PGKTVqEHA3$*CC^fJHD#!;za?E|JxTao#+b1S!Uw*{{l|CVW0}rH#EX_`Zci-Z zR5XQ=LU7ei zVWKn7gq#IEzl^!Qh&dm}++PR`jOC2F(jYKl{|MOlzk>Pv3%(uZFCO*vFyDT`*Tekw z(Gi&6VeNDG=NOnTA02`D9oDi7j>G(hQJpYf*5RH3av^;dy~(Fv|IyinPur8e06sg< zF3jWX!fiv}jNB2e^8t14_QA-Rfz=to-@h>~7#z%gl2mvP|3=Pys;y3K&jD5%U$xlt zXNA*g%NF^|Z??kd9^quZ%lU5EW`!&H?!&ie^u}j6+Zb5Bov%fOF2C=6+Tl$y1@HIB#1#g1vt=Pgz3}EB~;na$)(u%RSmZvnzgA z4_aZ9=c$SH(<-ewU6#1}f6y=L@8J9!-E**c9Z`9EM1A*C)@D6=za% zJ~{ZXXTA7xnp44j*P<2C`6qk%_s-lhYI?DCbMAZ7F9|?@Vv@Ix>i>^xZ}yK(ySdzp z^F6+0RLozdU$Q5!YLx7tCzgMiSH7{=kw@X0AKf!*&DA_t@=W5ngl7WJIG!;)7x0|N zGmPh4p6~J$@_dJ_xri`2ojIgg!`;W2tP8!T}D}A~R8@s+klp82XwHpZw zN(R}oCp9z9vr9(XpE91}z4)ddD2Au}{KDc*Px;!5Hv2MXBmYn7dDTsu{>EOGo>!f< z>AT}^8veI0V~YyMpEdlZtA=d)c{i)iLC%sqAo*D^?CFR{_FF-0`C<6`7r9A3JBhxi zZEDxW)D<|JH9CK{IX4cj$pEgt2%J_LF2e?Ztn@`;ZfV#74ibFvuUD2>2dZh)dcRdS zd(qkO@~WNx;#|(pIcKvsWi)iHd}>dR;-{=Gqi@OO8N)M!r=ja;;_v=#?=jYa-Rxa? zihV49^QGCFOMC}%`JRB^ZV$@eJjyyy!8rwzp_`Dk3UmFO;hkPs6Ic98+JJ59Y`YTt z8N>M!oKLv30{$=?n_KrSoGEnHZ&!`yjPEAKbOK|`*-=ZoP)QN}A6QIuYYg7kAY}&IyW}Jqz4ZdeT)1^#@~Tf>&~ftYBB=Ia~&~(*{$btCJg1* z0w-1CpS_FhH|~IbV6zKY#Ut{G&)$km#JT2yF792}+8eKazL)AU&vqJG`)Z?aM!Zt0`7v@&RlE$JN?0j53?Wl&hUJOI>%9##_;~+1;6}^F?)(J zdy0AJxrO_e3Z7o|H{KP&E*F>raa@Dwh6zTAv*{^_w^q5Ipg?- zFMHE7C0;yXXM73qi@bQk&iF;dZ}s8{JL9(!&$znf7uR>jGf&TS_2LOT4o?qWB ztULvahTDa?#leZbfWv(s8#T_->k4zNp!VZwPscRv%aROAy!L2S@t)5->Hqwa!V#LU zL+E$#40GOrfuWK@dm?_r7r(~&ew4A=*Q2?>H!xm)-4p(bU7E6{G3IT2V~^>CnK8^? zg9kH$CBY2-qMb0)j*Z8_*Y+a2h`c%M5mA56q&?>o4kwJH4~i-am@#EZUXQ?^_LUCz zSwkiFXwP)yzg+$+|FUCafVfBauMxempsU3lC@HfW^H#7I)f(2FG~rQTYmt#1G#>AL z*)ddoG5FQ>BKN$buCY2p#K5ZsuMJVwp|w^Jxl%N1NA6`Y_WqxWpB0DyMR)=Fo+)>C zk@mrQXC_(g>R;zKyn6roD}x6qtG2Wx$eADEr=KqkPUAkI8u`LKv^wLNO!;l&9}tr{ zJA^*81p8Mr_*n_=;(MJ@ajrG&=nP+d%Mq*o3-%*^{zA9lXQM6lEnD@6XK+s2TIj#d zK3@fI?Iqq?WH%66#YN^^wx841Lwvt8Cf19OHSzZn-`m@=AQw7p z#`xLUBeY(!8<;TPjT5;;w-|W>n*lJZeS_^wk$+a+k)) zr}A;bmil7ZH?I0;B5$w4sJBx`6KKazPE7hn)!ZfguR$=>|ti_ z={L2(;Ow&l?fhd_y?86WrXwnjbPNq3GeoyZ_fWA{q<;Gkt)gM`@E@j4lJPrcv*+qL zp1DTOVsEmmuQvh*anP9f0(bvUqyN5)fPHEm9(z={X$AHv=t{w&`&KqWpBv}n@9iJ% zH2%~I!;>x#ePq=Kz^f4R8-Vv)^M)SMvYtJ zM{eKS2dvI$n1LPrHr5WH1(Mx%L+cLIVOQioz5mO+b5{!S>XPz2qV?J$bqHPms+hpo--(&4-^iJC(drZ|kZJR*bF4jA3 zgLe?d+GT`|{Q*%|{z(N^FwUR4w9Oyy6#6rl&W=f5`YC;ji?hO?`Ah11q+9ik zaaN!E$)CN|*PgvKrailaXQ)-b@`qONKJxvXJo%)rCVe&Ot4Uu?x@c9_6;?2TJSxZh zr!q)W8&#J0^?Q%IbDqnbhiG4*2V)Bj2+j7!8M+|46QOqzJl(wqIzk@yQw&jgIqdWO z6J^B>ujr>Wjtc}k=nmL~SrcNf`IA%G56ZsKS6OprkNM~4ABTZ`dByhPV*JHua})Of zupfhcSP`5|_wH)Qbe{Z)x6ZvD{>NMlet`Yu=q2bN^*PMRhpc5qc&cQ;HL~kn+{K>G znD;0ByZocMdycj#e3tN3`K2R7{>kN^5H^+HkT*d5YWT3|e~9@kI}ck8VK?%;LEV~X zeecPOwfFjC?cI#$2c>SCx4{3g`Ma`}Z*^#@xhRdBG!dt)P*U zMs~%{l;}K0nVm8NeO|6U6K+HfIYd9-8Ix+aF$QgnK|6g{pAW+Wu14nIZll7x#uaah z@E!lWyS(>ac9(f~b;=oCc^^0buJ#(nBgpe-o@SozjK^6#pFR)meFpf|J`?5vdsNu) z74T<2bMjKjab9jA%QnIT3*dn{^G3GMXB>CKn>A*FO@mYDZ!PGApL4E*pD|VcbMV(5 zgU@0da3#15Ra)Ty{!XHc+gaxoFK5pRb;DPP6WwiuR>oPYCf(P?Vo%Z8cAy}*O8b=} z@>?6@)W$d+VVvdz508Au`XX&_fgPc*0r;?FUFmr*!@nBgEoyiEyoycP$GXnRS6nr) z+?RHr;jD_|Oot`NM3PI_+8g zV;?Xwm+#{G*TC&~#yw&F4egcmMR+iwt7WVG)47wdC-u*Y-@Y>cuYpw+=q1~wA3T1V z9f*rx8qZm%%J&5S1QuOXFWz7AW%;fO?gmqx=TYZX)J46^WFHGrS0sKIX?mXu{V{8) zymtX74Nu~IAnEvVZlF2UihT6-^<^}#p}uKey;q-9Z-!%0=26Pfyqwlq zZ-(>d&U!PP1EinW&%b>`y@C^s+g|dxu!7GI?}8Q8^C9mFKQLjI^S%kW|BF!kPq_9` zOrY7fzhg=t=2m6xAkEBC+{N{?zMQyg^a$%f+=7J3{U5#L&KmYp$HCVf#(aES_xjhq z!+P9-0(#XPP-8frdJ$swU$2kgG$1YNxaZx&@?Dlhf z)^cOJsa!Ov6`bA(?OSuhZS7C;A5K(nuI_I=y9l^UQo2VYmA?9G(~HdiD^5rk-L0*j z+Ws1Rwx9r>48H506<;A|{e*}#xaBJ z!?fpWXdB~I7w{F>S=8IS#?`9XZB<`uL<&-#=hQ_Rn^?2Y|7KMA{J z757YTht?m2PgKCS#?QZP(`eRL^Wc550mPq!ZN%f{@#oM^=H-TG$92Zt@NBp6eeaes zM)B(l;Nwf)^@sN`_p|)PVF!6sa#0N9_e31`gCM`wWLZO-2J{T#OMsqlmZUkU#JAAqa@u|eYXH6Z);dogLY?s;n+A8W zhT~0qhLfQ9ZEk!9@m=_C;v7-&Cha8;t?xyjGMq2xy5$@-VVZN8@LKAK$e)e-F9}BG z@+fkD(_H>#4A6gGAMZNv@8~wZIS7A@8xz{8GYR(Q zZ5gFCW%F^o-tl`)8 zlRU|Nb;pc9*$nQTA7JFly4**4*s-)p^YJv}72Q|OHD@o$PFI(k#hp*LTFV6kRgB}; z=35-FTgWFEzmRf;E2Sn(b4m#He|SFUK~qK@`Laj)=F}ql9K7_8tZn0yQ7V57%Mo0< z_?(1a8t0qv$TuzRTxdIMAv)vV2W^kgxxo0rOS22Q)*qW+!rJM}#&@ElT5ITCSU>H` zI%+r8Q)5YE{d|5L^~BiitQEGhR(Qx;D~#MtonhLWBySFL4!X4ItGIg9*uZb_^ilWW=o*FciNvZ$vBMa}T=Kef*nDV9=I-KeR>FZCC2Stklxr2=@xiZGJ;|F$+4YOv`Tn;|z zpAp_akMfVcR}jiiPCAO_UHd40Y0{FMWhP8?e&OXkHh(-YRR(Nb1dNRX)-GhNek^x7 z#+du~zU_|mBiwh|#vSQLqwYHmN8QJ(JJN6W>b}i{$*JH-mKFf>g~u-3HLFMb(gNT%(CDtQ?4d06MLc-0*UxiInBojFVX||U36q?GCS*UQ z3BUc0b@BP8MC`wr_#t996ps80IC{qO+00QnslXL@(jWM2NEPn^PkOj`(xWpj*}%W> zq+Kv%@^N1!IaT0BD)9dp`nq(NGW7o1j*bm6=(g7qstoqw?hd z3!VAUjidAZ(aX2V_$9e_DytsR7u6YKEoQG>)y^q==TEMx4;&NC9JdP_L%4Bmm*9Ei zJv<0m%fjE`S$O72_q%60Jq%r!T$$<2V$N0WO5};_=|{&lKRhE2JdOvS6Ts{4;CByr zMo)YN{rFwQI-d-{rU1Skfp;8+ckG#L)kkGGUst(1hvEU4&Y@e-NC#jpquUVLWL(9LDv3g~Q{caCm$a4o`@};fYZ=JpO+H zhx-FJ>W}osFC%=IP>d=~r_~hw)_aOdAud$Tt;?Y>*+6_$UTlilbzK*z-&bZW0 z{QVbkN2*JM702m9T{K+~Y_H?redPPPJV$7c>fTEqUm@@No%L}4IB>U*xVJjv&~t!C z-FyEAaoGNl>*2L7o#@;X6rmBw+u_s6`-<`cH`%P02CN#+dRjcGKWA2jurW7spL8Mb zwbTqMXS}wOHL7r9UTpAasm1w7 z%oF%p)eFAvzlVFJefHiNK6p9w#iQ%O(@N$|adp_nZxo)E3QuE!hX8O92Ykc>Ckfzb zcl;y$w*0}u@KxP&FMlxIA=@6sS0ASw;pS=+COiLa!W3tf2~(X%2?e`@*%ymJd_yaH zRo~%xUVBz$-#x`1E9kBn&n(WzteJ}*0p13{L($XMu^k39p1jZIJrg{v0fNugeYJpM7Zw5zM7c=-dz^eZYy!~5^ zi=U$@PxvVwEWJTIP=28i+>{>W#<^!$oKD|UJL80d>4pdQB`ys7=YwwiDNw_489x&U)nr5 zv>kkDjl!3=DO32e%Y@0!nqy50#oxX%T=B#GUZay-zdqD$tr;A>M@&dt* z$~exyN;*qzSxet5DvKY?wgSy@XVlm38=*UOdhmJ_G(kIY*{A6iJf8z&e`H83_$^ITL_ zU%`F!;z14Btm&-F>7{$Vku_b~R$Vo6&y2`D-|5b$24C<6rXBOpQ)=X!cM3gaH1{{V zdP=&}%glAUbBh`Kbmu0*^GzF5oEuDBvQuHgBIX7EL<}4a=ttqo>|$J#}0+&2(-tbB+v4o92JRoL6?vd8YGA@1K9)EV#>bz97_` z3wJb@(pQ^JT8h(XLe2;>A!i(zFwxm-Lczed=&IG?sR3Xk4j73CRuX_2`00G)-%`IV z8qo@kI0%hs^Xw*vp%LxupOMYxFVv$p4d#BH6rO|7f%l;S;mSZkzV2m3Za#|7XSg!H zKs-NO*}Wjt!P$(WZ{OqIi_z9G_-yzgaH0P0i;y9boaas3Q=E+^Omm(yVY;)9amlu_ zngeG)wi14MAOPOSPL3z z0xf=+IPK4LaU6NdgNcZ&zRauNmGeCO=O69qUecMyeU#{H*HO-`CN0Uio3x|i-&d+ zwIje<7+7lo){erD4gza!X0Mjzm^S-v(7CL`DlTH5P>FR;f8-nac(#HYH=p_C3cW|t z{Qa=?ux^~dm}u`zD)1@XiD%5UMx8 zAM2GY{c-?DX$3 zE|MFU$G;72OLO+nck!*M)V0=s6>Gc10UKk9lia?x`ukp7x+9wOJ>u|94P)ck@OQ6G z;wLK8{L7dwCSNJ}4kB-M^0Vrnjchf#-QtmZN|8bLif*Mj^}-RxWjF81jLjJ4dd!+^ z(iF#eJM+2U)#3~!){K;YG<{auC=;eT=bJFg8KIE6ixkqQp$eg+nI>KPqP|^*(Y!Y^ zJ^}jT(xVic^-I=kd2%^hrVI29-;j9xM4FKux@&(#3Hy7N@Vv>xIM4RQ+4uLc4wN8E zXbxD*T$;^(hDEG_3T6V#hvp%Xyor>18})qxjrtfGwHF!%PjMFCwQ$!(*!=c~vTg09 z@iD#~+q%GCdq97CLWg@nk9$Lxk;CWr#kSOi^@!vARW$DitSvL4U}X|xdpTo#8Do7ZV}1!d?|aydTv@m5Soy91eL1P__H?@hSztGF+Uid( zJcti??ZOSK;*Krch2Q(e1a#_(1%Ebc^3&!k-m14@II z?h^$Eg7mqOw$D;XUkV7tdn*>iui~CF^x-V08NFdPfL+CZ>veOxf~ z2EMBwB?JdANLt0+Xa_2y=wK3buwj?a&brg3T~{*JT2IPBey@m0GV)e(+*{Bj_PQ^2 z^L&pyk_`>slytM>urFvlbf&QU046|x_UskSWIt!TD}xOsPGcZhPxMl0dzfR=#^=AI(>+=jAiWitw3F6hi#nJG4pc+yW<>Pu zL}wvmtNw|u{hdDU{%LW&zGa(m7#G(+DbeXhzgOcUFPbOX+6lwt#Rq-)_Za)#(BDsx zZ$1XsUy0C*{2cob>#!DWmb`Q#T?zTS7+HdQl2&yA77dMgEXU9o&RM{I-4&g`H#&bG zV7@P~p9l^lfeXpdnH1n6FM+lUc|)gZMS758r35 zM@V1eISjpdi|2WC{1wog+SKBD&XF6E-G@B|$TdfinT|pio~FMpJ;`*0x1u|*5+^xK z@@~+Jds+F=5kyDUd2uf(PV>O{uJ+=&a4H_n}D0Rz&}>zFUne# zkIg(9?xB}E*l}V{_wK~Fj&jVr06(H3kD9vR;|d29haXZ1d_1U-`JP88ns!o|2XpML z;JSQoFC~x4l?Y>P=-fva-nA}3%$}ou$)21!*j(L*a~SYn zDkYp1lWgSGy}j83yTErKNEne-rNfys?2xG?r@p_?r3)7^PLh>fIW>|e#qo2m$qp~i z1>RV;0!RPFntSWOl3)&D1F|so{`$6ytnd=Xx-qYBLF=W)PFCkj-#Q9?vd%hf>!{uS zu1_`1?H+6#*n@K*k@@yp^$)|px4eumHF#`rH8I8x-Ql*WKlO=LRUzlEj!A8qy&a($bvr=rmyE z71CZeY4}&f=={p6ot3n;wFo`U=&1LOY(w^IK<1N7yBB#+bzT6RsQnZhJ44+@LBVsWxXWll^d+(ai_wWz!`Q)9P}KRa`7e6sVA@Gz2_A&78?oEN$;rc z22D$ZrX_(x$>357IF$-+r9souu}yUAv}4LfbABC>9MA5YqWvGwyk)fb#=pvKO zhn{gb$A&QUY-1ia%WhWRnt(NQF5f@k8OBqKEmyQ{EBe?$=+7N$r=d-W&KCZ9g(~6^DPIa8PmhYK4RGl~Pz# z9KKQ^@HLW9{LB0Y4L&yiD#rc$jQ>>T;Y#M?3TWCCXxe1#h}{fL6FsXzR&td&29Z7x@lsWAE>3{7z~JMISf&)7qua zyLx(>Bii0f-*j$!xHRVIELYAtuG6F6(Kpe`pLuy&$Rjy?oj<0tygjs2`y1}^ z@-&f0^BAK1pC)q_A!S@i*oF>Y&fSdSsgl#IKXHyLIGtOUZlC+2+lCMLcMa=VZM3m= zU~zrx)ClcODbQG(`$3?)U!}j@o?`!5_+VboVB=)g9jf_vVa(*XD%t6JABcmN#&>~! zC-`0p_42*c%>JMW*m5g<%o(~@CANnOY!R~WrDETc4DlQ0s3XQ|Ucy{QU@Fm99F{Jy znl$9mV2Rp3WT2r(^G`*44i?zE!HM&kOO3~!}fxB$Kg@NLG<1-e(U&$-0j9cNWA!&bn%Cvk$cvP58xl-ttD4^bn{Ef79WuQ zsPsKWE+42M-Q@$sF>hl@pP@Lz2Z&>?t|RV8rkpJ2Q?DFT2Kcg=w5cX7)A=|$4V+p> zn)thD%|D~lz^_+G8?So&vJp@ZaJ`SXu_k}IvoktBbVKXNqf8okk#WQ~4)ASY0DfF< zc*l~FZRdld)%T7RKis=byd&9pMl=O@)IPxk#%=eL9`8u{F6l|m-x*`s3EGXFAlVtp znD%0B1gC;g*-dI?dpb*N*z8Yr=ejPp>P-F*IB!?D@U1=dF1<=}PNxj^eS{l67!%Cp zU2wt9N*5M-Q=Zw&NvqZt^cC`(fV~`UN?B!*RNqqPH*w-?T6rHVR%`@-Y&f> zx{c@3Mhn?ndc(We95?Z-ikjPcli!%r@Y8q z`I##-M!nCr7kRSx&nPS6TlWI*n*VweCOWH4nCi&)^`tTmGR|HZw|V&=BvikT&A%Al zG!fo30p2to-c$x}x(MDh?k2|DUE_l1?wcWalx<3DvMJEoHIoM4<<8yXlqJ1n3;p|K z{Ff`PVs1zL^t*NyeAo2h^mFX1$WtbJ=wHvVHP7k3M^_$Y4PJ1kK3zwcMJS#!(u0Ym z{3BcB7I@)NU`c6n!3X%(fhXWU`cLbknb@PmZ!<50PFt+|U(UM70Q`0*TQ709oIUDx zx1B$wJjpXv$R+BF<=J)T5Pu|=y^N&E=6fsnyC+Zm%``lv3;6m&;xyMEP`_x=aMM<; zfq^f)Unrc1rsyAy&sgG$%|EHm-@Jd^v}9vX9AVPPi|i(P@EPB$@evO|D_-N@`v?m- ztFN6iGcEi);*dAxC!%$bIMILnqmQqHWIDeFhTQqdbbRz{_eF+I)Q$Pcxn|yWFb0Bu zwc$+q3qP3C=IPlkOl3IVAzovAx(QR9{)Dc7SL~b315F7zdgLt2=d*^r`taZ=gIgjI0TwCPInkBqH2Y>x= zCjA-B9tq^bP)wqcJD!8Cb?`2^qaGQgBPP9h2JhNel1JDmz06l?i zVSNF4#ILhRj$N{AJN9S6!>f#uaG(AAtRbYBzJ>CjS}E~iLQJ) z*xv2Yl^@YItpl{d@4~c8^6@WCKZ?W46w=0DCm^4|x!|YW^oQpDc7_jc16q<>iK$}C8GYU+4?$%KO;6`VqH&p+8VV5Iy9YbKS1xg>cx`rez_qq-l^7{3fZ)cm$W z_afts{FT&@=(iVmch)~opESNouj(ndNODG3x^PrSn&$Qh^CZ8`XBpQgwh4w^8}{B< z{C+P#u(^)>H=8o9bjzTxuMjs&WkA2$7>CX}p)vbNyH;t?vM-|3ppBBluQBC&Wx)$h zC+OWLJMQ-9zUg1764o8ZO8T$7!K!@#OhE!V(rB!`5V zOU`C%nFwsKw}EvoGj4wXMqE0?J{iV;_p2VSJ~-6i*~iS6a8LD?FfWzx>VQ?%H?eEY z)dA*I^DH?pLOb-XIL)2P9iq8ptQ%|MGj0OeZ{!=n9vej;??Yz0T%-AANZf&%-=-a-Q?2*SmDc_`*SBPU1l~4zqWA zYw0hMS2m7+5UQ*^!9Q^MPx>M{(x!C7Ytx?33%}+*_~iUwL&3j@~l$6o+>zq&=HVh_3_Z10S0|j9{!Z@=AU01NxJ(Kv)>{lF` zUK)M~c}DytP%>u(xHY7TcdPguoAsc&+@JKcKcr0b1j8Q!eQq96VdWR8eBCu9Jh~p7 z%kILO;tKAxraso0xFh=EjM4ZFi-$3OMJJ8Rr{{IT)_1%L z&ixVFweREYEu8Vl_Y2}-;kEqp zWK4a-RPLtX-jE~Q8**d@&rDy~$5|nd-;?mvcw#c02b=0%}O~d{d@#W;7I_qt80R9oK3jfan4y4~U zJm<6L-Ri>WbJw~0^&0G%@~^5ncjZZIH(I|g^Cu;+j(&L{?yZcF--4HH{6u<61qzV5&lbb3xJb_&sP@f6V)f9HF;6P}@QNp(02 zFnq|oCpiZR`)OUDHL!15|I(UQWKWvplNRKY!^h>5WfA!#zdvjF$Rb*E!~c3n63_oo zhj`jy?J=V-f1<78kN+fIz5sJge4?}2i+@jT16Gf+MkAf0-izO?OF&k|KOnD1u<@FnLC$32j_Y3BP0>5pL_JM;eTeTH z>{w;i#p;qbps#%R5)F8A~^>>qD*`B5+OxVVpQ$-8_mG#0J! z#kawOhPg5JpMT=!?LxYdDT&uQiq^ZL{n=BD&q1nluV>$Bzt$~VcAvx0An9X9?n-p! zu8)a3&&bXh&b7Qt_R@L3nt$1t6<$Ut9ni>9z=P=GtK^N$(-iMNjlc4#U5UPwr6xbH zBAWREd0zD9s_caP>_LCi3TTI?xll}*jH^Vvi zguIdVt|Q+n(nK=~ygViiczuPm<;1m=M)A_>nvsW!M0;41LU-M}R(yi9QGg%UpNl;+ zjPLH%p8lF;k)G-dofFYtlN_6Im;UPdv8Fmd=Uecwl&}!E>L?YBPDBq%LLW*-FG@i_ zN=28%Pr={K_;((HPHV5f{5y3V;lpnD`85q!)jM@Vw%pDA_k3*p2 zn`yevyZSSJ8f6RK>+}^3Equ@Tik{Sdk52F4AJN@1?HQnL0fnN!qQme!L!ar3q0i7b zL!bTJAt}17y=TzR6zJ$igFDI22lRzAOrvNmeqhk$2p;Y_RXRHB2J?TC^ZH5YtS@L@ zKfkM!p1*ujo|NWN=wizLJ9kBBL;Wf8d>ws$R%1{aV=n=2q}w(RU@y{u_+Zn31fK4} zrfEHbuc71R6l@)3EsA-{yr0GUX;q_|ruA`kl>&4X^qdoEa|h=w3Rh}@5$UvB1j~bC z?2235_$P>$yn}sYLkM1QA{};pztWsDJlprfI`Usi{>YsC|DcE++3-@;lhd3X%$I9ZOn1&?4n&i+ z{-?Bjz76g(Cui{9A=%k4y4)8WP6U^e!0BXg8(-M@&}RHF4}0`D+TUJqP$6G`&j;Y@ zY~*Iv*Y5x)ui_cWlgZNpjs81NYdLK;bTHj%rCz~VPeRdH(a|rwxH!d0mq7PhS4`l8;;biG{NnpIHmu?v#(0EOg(G%g1j==|#-HZ6%!H%UyMqQTDsyR+^W-$k{M{adx)Mb_{qb`$-`qJ-QdiP`UxH4*rb2abk zkNP0n%{%OmYv~0{w)pIvo8A1%qx^@EKXUMY(*A>K$f{*gdoL4ZWAP=Ly#Rr@T`P6o zm|*HX&&PH=Wyr=RnmG|#Az4f^RD@;@BTnPH0U97WqO?QIiSnNgO!YJASL^)HVupSstP z)I*Y(d^>@Z&?vTEgBu$(+f~O~8Q0M1D}OF)q6+ zJ^I!=Kzfq%2;&`DGdVcTrEfZKFZon>SJus*?)jeAkvD?px{K{3{7sAP)OUaF>et~B#^}BSG{r$j-=;(`zLw=Nu`71B(dBvHrO>`dh;(n(%kKQi# z;+|5Rp||PI172LM;-I&8RBl`)TZ7&3xVWXKxks4YTA(fOOM=@D$Etk9sI!2-2!K(CD}!QJ^kmRI9<3=S6wT7Qg77>dYyR1! zCpzEv(oGukIhK6eOj?$6IiYBqOHWgr7Uo&|j;|xnAG|zegcY6il=4YmO#1IkdWKU> z=+aEmJh^ebNlSM|M5lpAqNnSK6P$#Q7pvq`iZ7}5XMI&xOmQ-R0ny~$^yzlSq*GSB zzq^qYFT5#2OH-W38LNo?E?+(2UnEo+kCLuATS3?z*on}))4s!uIkHKUp!ay4CVPf<9rBTG5K~qN9ZsHHxSn1GnS8E zSrC7*(b|K6znK0X{b2dOy+C8&Jeq>rlxm7k9nl ziWHaPT;#=-D-J#;+#BP?U86Y0KjQZ_(u==B@rE{MIz?XGWr~9~2jmOS9`=Hcl3fMK zS1sk>hh1vtc>M6Sue=t2)F$Y#e82Lg;}$XBvNaB74M5?b;&6pR=uN&t;3fo}X++K* zY3O#c)1NwzBBy8%n9kG##-@a*Bbw%l{V5r(U_ce5t0`BG=&AHO!Nze5? zvhol9v`w|!(*6&7ZysM&b@qMllar8h66VZXH#iS zmd<&%T0^l>@!vmkH#pLIGX9P5Bi#xeH)&_SdagnrdV;2fG0&Pf?u0cBs6E%nX>}Gp zmXad#Mp;R`W6*B)Oje)pWg%z$D;dA&@+#RDHDdo??ZiJuykwh7V{?>Y4|MHuA0l4+ zHJ!IT0!@0N&i{^RiNCj_ldUCg2lHs+pv{9A-yGr`n}6<)#>>_z#vV=lb%v&;*!zJm zhh~z_I$ulL)g}%9tmGHFTn-=foisW(WIOnw0P7F~Fc zbbrXveeMVLM8|iLC*-#Kw$f{zbfv|zMkOXK%l=Jt-N5f4(#|kxX?9GmV_)KsmX!tyJ8h&7d+u+Ej? z0b4zN!q}`Ymu>3l_=8{0oyIx*rt%xf?|JCXe&B08c(a~xE%(u?dPDoUef zbieXo<-XP2KibVYik_D8zws*i4tdJQbE(6{V^4ZY5^m$~T! z9a_EAI}6N}O=NmMqYDtN*4)DUwSVfGr*GeIo@*m~33H%)`{6l~{TG{gVw?(rvxN%5 zfdvW~|9pk?vC4!QHt+(C9s>-G29`LJn^p#Fm7*JYbFKDWulO@tpRAMZlJ1&Io=Ue5 zLFeRiEa09S-9}xaQS7VM6LMyZPiy95!;F(er`Jh-T~(Ll(&^+@(c!g!FOIxem(sey zp9-B$HFSD2bb1+M5nX%{8of4ugh!{BcBRwHmT&ZDK&LZAr`JW(>1DtFa@ihmN3_wj z4Xr*C+PVpP-Fl&+*Xta5y?Eqq@a_=$+(YPd4?(w0eurMK6TRN#&}(ytcN6q_&n?jE zXZl6(*O{7gE?3Jv$zkEjQ+{1&+O&hJbm3f z4@=gc924F;(Z6T#5$9f+Q|p5Wt_s@z*qP{{m6n&KF|X` z5Fe3$h@ExPout|Wz+vf7gcEl$f7_S2eBkaR(o^j5vphbKYCpxAix0H9HWwbg%KpOp zuQDshCwlRi3DfOmgo)7m==)&NwEIsP&~EJZ%-U~G3SUD0Rlt+>3(Z>`E zRD2bB+;NH4hT)0U@ICOrp>A7iW0G6)@ntmg%Q20Gd6SnEL955Je;2>1!oS}h?gTly z?`-=S)=vB?q05*h}2z@}WfVQaao<@F4l$I%wjv>^R}8;ZLbX_j~X&@gHb9?I^$DKlJH{ zzB~C3^xI9BUu5-%_e9%{UZMUX`zfS9tqQ@*g9^dj0}8>RzYs?2j5_?1@xOt8)FI!m z5&w{UFB^7jiJ*zPLvV+y&;6J>C6|PfSnnDBcb=biV&Plxja=~!f6~!H`WUH8YL$E) znOq!M?@wucz7F}mJawaN5`(4w)vm*90!y~_?dfMtgc>zxefFm@A09^aa5 z=`%mR1^dVJE-Cgoq)Rp`F=3`XoUjD?`>%XF%U_o4@}+a}-tq9{ zZNWk`Eu59ky*&Kf--(~4EBKAzmm+)zH-d~sbY~##;=_Ku;Nl4Sx+BP1hnUlY^liUC zqeyvskXQRrjjI>=bw;u>F{x$0$*27IxS*zVke%j?KhNe#-(5e!|_u+vWdm zXJTR7^;Uz{EFGVG?-PgrP{xvOzd(NRYyBVV+WWppoM@Ny77f7pj$0%jWOm+heUi8} zbyu`H`cB!qI_F0}B3|p**Okehq8?*QOk7$gz9@f|!EZ-y#}6y{G{-S+xn+DiI^HBZ z*rl9vC-zpiodx8R+y2)cd$9P7!zM}cRHpr@){(Kw zuC$c#8oXMYN}6*f-WS|*`kZOsVEUDA&muekP1SzkdJ~swUuVK(d!h-mY~6oN9OtZo z@@zui@5&6HdwvHs=ONZ#a@kDIzoUH*maqmZsYC1bF=2qUHF;j`>SJ&Gf78dF!CQRn zsl5z*?BTbojd$GA4o-cU{St2k&57Y|j)kmWJ4yJ;8=!WCRRAoHE+rp3v0S%|5K7&U(jtm#DysX<=^Gi@ys` zn%P%A7fbZVrE{<#!j;y`R(1C4U z*^epvZTtkvx4rHo>)cH;py;0LeFYcH-#Smva7r3}0ib&yawlm$zeoAq$(*IuytG2+ z>H*Hx#b<<*!td?$r;;%`x|9t2x5OU-P91nlvbU06lbG7VJLZPQdE@j$1DNZrR%}|h z)IE>9-x(V+@i+V8I|7YMtyypSyJ$BywT;|Gm~&5Xg?uZIL;iC1){>1x-{eC+%Ra+h%T?4R z8z8|=eX7+zm}K=o+ZQOxIoX;meZ(}sHG2p0`^qHi2ipnFxNl7lk9ym|t)-Ng{e(9+ zE`8h@c!#%t9r|e2zL0!^y$14{I;`~Y>0_NbYDg1})Va{AhkWmh`koViC-L%6b+`|A zb)ehLybd=h>eACLm`hgk!3njNTUntt^+!L4%&vSeTU{{~$=rhkJ&e?w|Pl`F4A5NTLso{F3 zuNCyiS*r|t6S(KdImc+L_9Nmk=aA3nQJ}B3TXqz2(j|!IKG&tJ zD(WVA?^LMol5c)XzCexDFEBT5@}e)TeuZ;;OkR8Ph=^pJ2s&_NoTc77F}w4vwkP~C z@9cct3Wrz|txbg+P9&UcIpaAxxkq8^IaY%=KN+;)_>Vuve`1N#{0yTGC!aG%pD`BA z(f8eP#?gP#V8z|uWt{1T|LidN)9rmznGflj$T!_BdyqJ*f&D?XWs?M6c7C%Jt+p?5eGk!N&1M#Atp?6yS`sEwp zM_bF9)8R)+_Km=$Xs+_VMLUZ7jS182Uz;$+eqEvV5av7CK8k&Tq4C^*Dwt| zQT2OrtjP{!<^k?H=``#^mtqdqi0T^1>ryLLq?GVUui zXGdOPe_X}+w(%u^C)qA&9gpT~14cyBIiq6JTZQIo+q*JG8vvkFl-^jjO z1K-W)Lfe0Nsj)-eR)ufxuhebyx?bJ#%bjXJ2^_(PYvf8Vw41?0(eWOjIYX88Wm4lVzHc+qmr`*6;qGx#0Gm&phG zdUbS|vzLQw$|B7l^ePnJX}%r$xW*R_U+a|LLV2@qGUsO5_7vCV_SeK8;w(Y)DVh6C zXwDAq^+}Gc1D|8*%X7p@mRFp2&+c)>8U3gDxAWGEWO8SJt~lrILCNi{!25pS(YZUG zZ1;0{>Cb7~+oPtF&*4uhn`&$q>WGuh@gedFFZ^z~M?1@<*a_XrJ;~UA=$5L!3irlk88slub5!{>x08$#xa_ zgr@;tx=Zih&(vKy!{q$cC1-?gEg^gFv+{%4E z-fA|qn!V-PLTiI`2Ikwf@4Y4&{lIBf@!d5kS%LHCPkOj!EjC8U_5+oUJ~>o~OpFZm z&>wC!XHEAJj~vz3M7(%joxV?FKgV}{eD)IS_qR6Qqd4Z?I@QT@m%ib>o80f)^o`6> zdzzE}1AVhDxu-kdH|ZPP*u?ikzxjb_cc$Wy$8(1|d9LGoll~+7svpz&t~`Tp^b-4r zNnS{@ec+Ho!;QE&DsleZpG*^=QX3 z+Aki1j;{bd90+C`ItP8;iieW$cFM+3zi8Ml@URDERc^kIeGT)847IJE`^d7ZnCkb3 z>o+_)sd?AWe-QAUJYYZYb?uMPr$K*JkMK_KHhn~S>x8uxC(*{6ycuv-4Eps`tm2(9 z(15HlkU#R-8a{59U(Ue$IKXU<$YfQ^&qq4v%{%HPHt!6Ifop5X}TC>C38HfD& zb)Sn-%*Cpctm0iU-)%K>t$fED>-#5!d7tXx(24YAK6%rk)xUAE(VL zH2z%b>T#Yk{$2yRj-RqS7@JqO2MZm@&9$7Dti0OViy|;w&b!mO1FYiF&;YZ>7Y*oy z3k$e79z1MdUTU*hH}y~RN7<|5pd(qpF#R)SymeE*y*VJ?DfD|)CNRuAv2N=Ek978X zoq=KM>jJ}+{e*Jw(C?;n>Sw;bGB2v1a>|#i`qLOsT6v^<-#;dOGyPnZN zQP1G+{l3@C%O_c-kx!_@>Tea_41I6Hu1qqS|JTqM=GUPY=#-{AvVA>rPQ7p+eTKWw zUuQLEhD+VMr4LSVbeTiQ*8xqG&W}4T+tT?-PLVu;&S)EU7TdV@yiNNm-QE4Rc}2(i zi0;9IdzmoXPBdYb9W-I49dE)6JI;i>zhS~O+h@X5yJH&V8C$zTXxkAJCL=!{|81Pz z`RC!!*6|tmuf}rl4`U*FljEDlt2~ET5BVkex;asv*1wOA$kz6FbLOSD0Y@*9r~90G z>4!flkE|y>dcLF6jzG_TLAtjV(P>BjSr&PUG_8?zyb8Og%OB6<3HQ;%aVCr&?jcVPw@!MvHzyli%r?Q;A4(m0Z2DJ`5pcsSlBVu z=&6^cAxlViExEK8d6iH9f9mRydJ|WPu10dN2bDM@7;^>C5?f;-n`F6dw%$fI1d}}{Y%NUG|=Dmr7 z-p^{pIPX$@hQ`G8~yz@K49hlHGuyqy<;l;ki0A0m}$kf z>|lOg2OmbY=j>h71utU23*IFLH~iMTNk`p%%Pi*Me-|$-%7EiK!&zrtlpPMZxG|M@ z;RbYS(gE<|U4DUD_I=<-^Hl3)!G!SRMHfHDkY9NP1Mj-Flw*lo1%3z)gcsU3ICyc) zSLqopT;{rPTtXS={4K+N2AFd2LVNUVqk~K}`Lpel$al<#X|gT6=ws5-4Xz5ul*hw~ zN4wPJ;Dq08JBfVS&v<918;BQW+7gr?Tjbfg$^dVoidHsLK)lZcXSBpMLx`jj7+lbeRzv%rP>0$i%4VFG=h~Xt! zb{2RkJT9i3@IW~FbMQwrL9|w7wC^3n|GuV7nw?8tN0+3uBtsX1q#d)RbIxyLiQ9?1 z=+wb|8|vsVxS3(MQ-|73?QA#0u6Nt@+I*7y`(0f2ldlc95ikGDEgK(I_Rel)-=OR+ zx9ow7DT_R$vVV8Wwi4%zH^ZLQt?Vw!zU!9#h&iyvQVv}ae0>(Ow$rav z`%T&wPruoPnJs*aZg=PDwO>Q4hBJ;k!Fvxzeo1+qF|0FTy8XXQm}39Jgq$0hP<|VZ zFVB0j{J)!Sso!bPtu*M?ctf|o`AuEftsH_@wLz;qf6@lu>T|@gj%6e#hj`wkyadm^|ApH<@qmtnb%d$~krvlF5aiAyY2R zKE*9JhHr-lr`a!dDd+Iufu>xBeTFHA{ij=QINu(uMY@zrGq{>;%7OdjYv%r=bAo?F1YNk3)Y>h(MGvw(KS>Hfbf*UFEbhX>O-`E{mklMg&tT4v=j zue#HlZm(r;DH^VMj-`FsBtxSoDSXCV(|F<~i@EsF`R?N;&X-*ox68YaBPhQJcv3xT z=d+76pGo#VOqc@PfAibEF5E9I{Hm?S(*LGxyHxKXs(qA&_I={ldP`XE z2rmpSNIyggu`{es4|$h^+p&$$bty`SG&Fsa-G@aUz1>+(Dvy;OU?@#GbcR^C5Tug-te2K;>eu5$4luMKqnv{B@? z@hIiAzT%yobw|3^eFk~Nm%-(w)~>wp*{*H)BH{(N+N%tM*QM~|O_lY=uX1Ccu;W6j zL2#D=ZvWZXEN0lZQO4QZpkH-y`*z|sf&1RxQhe@q@L)%h`h9F)c@1%b!;647@vNDY zbMQ3N$bHB5m05;Qjy3tS>`TdaY+vc{$x$XP-9Fc?OL-jGG~M0|9vrL7IeQuAwq35e z(0!<1lJmt&Pcdn!c9C0`@;GPNy!G6ru5@$vKG$vgbn=O(1<{G^UK10M489wkxxwdX z*?Y~gJ?dk$5sk~EqIg>L9`!}?8C<@0ptJVyv<+7u+o$%UZSk~Uxoerqw`f;)91b1h zX;zGBrpO9C4!+|qj4j&Dodr}OlLT1W8-=|PsmC$4}`3}lZ5ub5H@ z4Sx$7eoRm3?B^VP$bv~5(LD){PGT+umz$ygkGi}f z%kT`@&0dB+Mf<(oly&y`>G+3r>f1wH2%h5TTT<;7VA0_h$Mh}l5hr|jgMMp{{z@57 z-*O>o4vjyiZ~2wUpKbq^e8==H4!?L_`3>DyerI3h$n9zNFWmZnO*v2glT3H4e#eLY z6Q=$Q`U%>o`anDSLPPpNOL)V2P7=HzHe3%r1`1_^5NcbC zjh%m_b6)@2IaaagcI`RNd3`CN&g)|dk0J}cdA974{dU#e&Y6Ab+13=ZX25lGiM8Pf zvYz>N&+LELllL3X<_@jS>`$6Esn|WUUv(L0JQrfeOufV0^ZFHvJI`ucMH`}n%k*8& z+4?Bn6^zd?=k*UNZWM7LC(n=ceGcEH&i7)zSNYRgHNlSM zh5R2jH;`4;JIVBK7UfirYzXuY!!2+Az}N&_OT4A}*yrz;E|T$0^V&c*+JB*Buda3g z=Uid>Q2b|LJ9o5ITnF82I?p@%&9WDakG2EgErA1^BbQml_aYM&=w9qmfw8%_8lMdh&j^Iq9$GkQ^E$`2 zU;$_JP4VL*&N(}8;?b^aQ?T?+bQXDsJC=oj%ck;U{rZG6<|fXm8}YsGjEjDeF4^EF z`n3sItJ4|&it}6>ge=!aKsE>&_Jhn(m-3q_KZ^34A78EVBdp?uoa;j`8g|r;jjZNv zke#GEHVRqTTV&X8lCMi0ye+)_9H$Q#bWcmq3#$(GZ*2Fpk$J{8LusRwR?JxYjE$(S zCnzIZjX&!gS@E3Td293Y?sHH{J*2CiA=I<#49x*H2h59OC*aKib`s>%7%pNSDzP!S z7uy5_D^EFlBQvM6lSmvFacl&R!O6i^;0t?^==DI4**O;$c6wIsz|QqZpf90Qt>OjP zG;H^Ob)9>V|IJ$mhMyMv4WZsB_`9)ty}zSg(ly2dq{)t?PPP!p6kWy$?0rl=jWLZe z8aop5_=p!=Pwif>mG9Vupv(OmX`8SeG2JwV*Y*|HMu_HMSnzaP%D3t2U=A9Qf3K z>_trfTTkqSfknj2j-=^i=ELBr?D*5|UC@PF7*FT~=8L&&${HPouL~R-2=e@#xyr?+ zWC82bbRYIW>=kqm;3Lvvv55&`TT%z^eq2|6tUUWp*_M1lo}Z{Zb|v?6o*rOqfx?rF zTxa?znwZ~>Cc5R$0oE(AD`_I#N8AqT+KjEqCTvJd*^MWeva)fxh%s8k$05TtVMEeJ z8OdjkjRy9ev~68Z+Cx^u!Kvfm$LDbdKQ^-e7Vd^JFY8!4_;IHn15^L!1H#i?+l~D5 z9ovny{>xjL&!vU%?Lzpr?&rRWt#JFC$?*0`@c4_V`yzP$g`AmI;4}Mz8$0}QX1^fW zHQMIm6?i1_O{dL=Xt~Z1W%tlQKOdh;e;LDa6XNgAgh}>KOqgvyY(nl_nK08{O1KJ| z^&|70Zr^W0-dr%qWb&MtmVKG}eLT^_;7 zPs#I5=|`b&50fW){k%3jIJ=*8Z*9A$2?iICu66I8);4)`WG-nHoPB*=y6#5bs`BO1 zJ;VZoalm2^U=qE_LL0gi6LvDvuUmFa89tMlXC(o1vy{SY!6x&_{XW8Z>q;ZUL#rHjt6chiRYeLP)s zv~D3DdR!*j+*fb0JG$sN!yb;{BPbFH2vato59Nm3V_-_-S%iOUDSfP@7 z{8RoAxDi}vJ+$Ax1HTVH<4l!2#kU*(0BQE0C==4X4Dw0;%cP%hCno1;pV{NkWvm7+^0L2U@+_xbe8R7H)-&0jM_%d2 zcCxpAjx^P)^Za@I$H&C%O*{kC%sW*3u>%o2{Dk=TxySJ#FroA&|3jH;>#qC?-OrUy z3BDoTf82ER5AWvoFdaPVt$Uq@r~`ZSw*9>pKL;OBU4 zZEEkB@{#cWb)BQdQAZEzilB^gzd7H)$xkIN|e3`X#y{*~jCb7JPA>`23KQ4WBRNzs?bc zlUFpfUbWn_j|s`4;VU>3j7>2=Xa_0T=F`4;Z(cV z45$1(#5JMsIi}-Jw(kMQbzUo7x<`v<5+@!1RNzN_|32kB9sdq!hoe)G?%mPxUtscQ z+21AKPUy3vmv?kFV@(?ODcri0$DwN(_8+>`<>>g&cH5qyz9M^_<(9jUZ%41g`(Itk zIeMK_+;&Eqa%pyfTTXE0oCBoU&vz;3oC6Fn<_2)NLx zr~P-*QuWi*|6lQ~a4-&f>YQhLdfK~(JFvWkF$te?(d!?>6z754h(5FO%-*#FFg` zT;1t^Y97e1dH=%LgCyDi2~2qJS$pswAx?aD8S5bUUq?AlXF1S?p)(na^oQO#_Why&fu7~SKx^&|2%`_tgzyuWa=Kk@~-mrfoI{C&~Q{;QSx zqhT-ld>~r?mmv@aD)H$83f!Fff>IpX0*p6v{ZfG2MO>_z}F1H1#@n z{Ri>?bW;ZVAM!dfhhyJ5nzTX}rcZY3QyxzS_@!H)l*+GwluJGdNIN(j zGm$NjWy_x0#CMa#8E(wpMq`E{(TaN=>wdh&GdF7Pe80Y|33 zgLsvxC$Hp*Tivn`be2uGJAf-k7I(@C_Gh`}?joP$iO4;vS(00iq7w+_^0uKbc5eV5 z@Os;+mUnKO{PTEA0+`Ns==u)YiH7NDxguIOV9|Dr==v1)n})83+P=Kx%ViFpW!P8H zzUaEvP@&I-?e7x*zeCr>yLA`C#qzoJj@--4bwsmmk<`@pzNn{EH-Q}pH;RB!Mz zDt(^P(Sz%(N@pT-mCpCPop(0soaRoYqaWYjVnx=HuCv0m^x4=xqchjo*P=_*T&_iD zuKVxLquYOc+YcKT{pI$?dilb^SMjg@SsIy7zjg2Wm5vdSS2~9CJ1fE&&DN*xC0Y>tOWRt92Pvm{H+eUacV|>y+E(5h+`RXx4fD+(arlkR+0)yJGenvqH9|GChz9c$zd z7n#Jp6Hq{$V1GF@O>1$QTbJTiLn9r!>&P5~=xTSxU+xNKTk~+(!+xK=WcP-^UcgtC(T$=rTw_F_Gk{yN{pDAhfZEjh;o#@Gc ze)LXe-q)QR&U1ZhZ5ZR|-`eL~0_{hc`V>v@ayBJCo#U-I)1UC)!$-Qd>qmgKWnI z^ZV#}Nc+y$)bj@QJnh!=B4r%E_^cOjxQnz^q;vS=>SObYq$JIMY^=Jyzj`_%vs&%1vChU8<1zAEl7B zIaeX;c8*EUv`b8wWDi#ejXuGAXW8e8PxFS~?)-l6>92ps&*RgFkTF}~)3O&j44;;6 z;9yriz0~uMcdehZMfB~5W^gx|Go=RjbcxR@ehEGezcqZ?<{Z4BMtu51^Tu2)`f+fx z^FBPdxz74UQK#R#w?}(-n{*%L`0wBC&cB%RQ|Ar5_4vO_o149&8Gn4^jE_%zx-fRp zv3BZOh5tK8PD{2wCf?!g>Gr3j{pCo*EJ#m_nC zAg<|PNAWy-mo}C2b~fLYd^e49$~>U-FFJ|~`eZEeZHI?*Zc$HMh`7Ap{Gd_qnk?XZ z(}9lS{o>`oiQ(PAO*8gXYsFKrwK6;%`O5Hgc$s+mS&ctzEcoDdBkvSQ?tzzY{CjES ze)=w++l{ZUJI2>#?4);(=H?Bn4Tq<$d#aPK zyLrJ^<*jz}E>vFn4*y;ER3~3|^FnKMwqNPyouj;r*W|s!XNGnE3lo3CFte zlZoeid{X1-?D?3J)KlU6jCb=8LdH_BkbaIM9HIYO&r=BL_gcbQe*5cqTZ6S~xMgr* zB>$8ltDeW+WRlaLbEseZsM4I<;se>$nVdK!mR;kPYo&kcqu}gX%Igg4Bg#5_l(asMKmWH$bNDE6tkWCR@s{EYA0>`8+eO?PCZBjFajah~{Nxpr z&y8a(2NCxx;)J^+-TuDg_IHE&OT1*du6&OA=-X(@K4Z$J+5bTqwG*?z@ky;cr_OKW zo2vQx1nhd3j9eri1maPgk2jrMd#>cao#m1N)9noCmgy(3^a6D{Fn|vb7Y0_Fc9QKR zw~uRS&x3(Xw|}NQ^FD}n?=|_kdu{raX~((cmUouRw0m|dH=1&{yXEdP<+AKUF6`dl zSuV@|lD;}Hr+IMJf2LdRPUUmgzu7H2mpJXejT}bWXKwjp@`vh4Hg{^KntGD$>s3#= z14r+<^;AdKL)wR5Q_qu(V~ktRC8`HHEO>mwt>^OSdPw{2*VOX{^$c_CDOWwvebw_z zx1RH&>mhC9*VMC%dL)wzhlf(8UU%%xy}czPoBY57uwc&mfRkvLc-Y_}bCVQm8%p`^ zFd@0$>(c{np9Xg8(?eg=r$O}TC~4|bPnCh^=>HtIp1#BhzSau|NSk-udYtptJ*J){ z`*4}?Qt~K$hVEZW8#_%JcmW(s=3YjhkA9Ni$ZgXs+H}U>JXw0qWB72189V2e)b-qz z`0OmdWFzw0y0V@_z?W3J_G0gzZo2&;?JDjsg!}98dkjrzrTzC54?g`-A+&Xy$p`;e z2=2eF5W4U?h0u)On)FnAvk8;zHx(lL{etj6+wt8@y)DRb^3&PfkF)GB^k%4Y#t>s? zGJh*$tQH?~))e&St5QQF3aH1>fBv7y|G&pr)jrPj@h7#{hrNMp6}tOwmVH9^xInBG z&cO!Y8e}@9t3O|7Bl6;7;cYM4FYh~a_}zWO_jK&LZShaHy>QW*eeaF>)4ul}Yu(5B z!_va(!@_5>KP#9!rY(VY8Ye>6&iB=Bn=mwS#fW)h+JgJ%HWvEQw@mW|TVA70%B&l< zw_NX@ymNr{ISo24on-JxO{4s0keKGU2z?yund@ z??l2>6W-tpF23=u8(No;_t}n)4YTe#rS&t`E67}{oj_?$IQjQh1RttxwDq9&%)z6h z8w(!^{kD+03hy0Hm}Mv5u^hvlhp<2T7p50R86e%jb%GiXb-4vO z{ucZ|RAN_q+UE9Uxyb(0`Hy__xz&17_;u!6Y2F(x&zD-s-haW4?9#YxYt?DtJlqr?K5}Vj%QPK z^yYH*`DHiCw(5QQ^-8QYdpc`guf56kQ^N_sZ2bF$b z`zzV*|Af8-|2_qzfZhq zN^YFBLH@?P`Ih}sXi53c=KZq#XHD_$_%{s;$CE#UaSWhM(?(N1_EeVr5dFRkSWRDZ zM%eO)=%3z^k&Shd{Um9Dc;+=G@JwLn%D)8St+tfGyf*}_D;_%0Qpg$<0K*0+XBU;| z-6f~~L1FbzdzwGRh>s=NXVVAKl(FHN597V4fxA$pTnKkIv;+^eDED+j$$^?r-mUx67g?EVwH3r-r;VSfptWLB z^T5aojrBvq`e9`y_4oy>FXwGN)}wHH8S@)#X->AL?8Fbe?%v`{v#3D+3QCK@L0_n3 zyx$rgm|aS~EB9`PCYZF@L&BM3CT+}mr?f5O-BQ8#R_pPz!#59c&VkJLMdyU8`Mv@= zSW`Z3uW0wEx(Rz<1&4E>FSUdlQ(K1%&Uqhn3cO)yptPK~E9GOCJv{VHYhFlS3Q7l; z96`5M`Qy^oA0i7(UlNMUKpxtGt$SsBC~^&Z5`F*8`$krqf$Z@YzNvR>V0B(NqgUpV z;pL^kNSeK#IU7+|#((4*{+GO7&i@SifARmld*(I{2lkZjS-yYAT<27sg*@f>^Odbo zJa{$2)A^7QALGg8fKy6nhpfxMHWa6>7OA?*9pN?&t;7e)QGi>qlqM!C? z8wPBcyl)W?uP%D4M_Yh(3SKg{O}smJNe^&pEch}8ycrGtl!HfQ=%sld1HGc~tbTY| z$!`4i7ovL$@T*@EY6#9voII+;njJtN#l_Cg#|O`E7))FZaaHJ>2FF?bCI)*p%sR|w>%GcTe?p@SV$78%+UY(e)l zMSO83dE^W66+#b&EBAC1j~AZ#GQ$PXOlZ;i!goS#GYJz;=e?^d60!zgK5JtO{XT_$ zKJe>_4G-}D2Ig_4uU9qqkf%J$|K6A1xbYA4zk|Cp^TDIum!Gn6@a3m&9CdloMuU(2 zP7C+)Sq*`(!C7bSd{)~q`eV|ER+lh_;abb!ka8z|h`-HA8yp@Eo~8M$lE8DJlFhr~ zhePMrd2fycBm)FT8k-Stcx&FEbHfjeVqZ;v)3y&T$)t^rI-ikUy1!*=>i_Pm32=lAnXve`doanC-E%$v(>8lPWb!kA3WKclT=p*m?_=|{(VV$ z6S%^;pyYt9d8dyud!y0BN589P=Ie0eN>kb=+Tvz|fZ*5J#6^<(d_MdvT{ zr7roV^j!Mq&Glyb-+f;s+2JnwbDR2#T(B(U+$Gol(a<3Ae*Lc|{fE%3$%by}ZNwyd z9C4d0WK8vWH+YG^{oTVBEf4Nif4jbMV=;%OA1?1$lzNPJQ+E>cP?cLtrnzC~KguSuB z(z|N;uQYi0>}xYi&+f_hne=<@8Nh!(Yq)8v-%TZfm=WfGaBp?VOz@(RGQqurOU8j8 zN${W~@Y5MbbxBT~HG4JoB}z-~H@a;gvWH08ann^S2}hjb$c+I1TE7XsI462ja=$zpHmL3mce`A zH+~0qeax}xj|=0L^@nY5^*;vZ(}B7FT{zF{2InWZaIUsg|E1VD3D$kUp$F@$c)R&| zt97_wT|7#8zfb>jctgQTbogrF#Y4gcv*C?Cy=xP!E`=rvMkWGp)$pWQ%u6-==mzeM z_W}-jBPX5)%={DCu`lo=*{}&Xx%keR8*}eToSZYlnq5yFm3PkGD0mvBJvraG-@ya) z-959cyVEE4d!tv@(C>sE9QvLAwW8nXbt{IRQC9L!dxxPPfq%h=y1+;5SK#CGz={5_ z^c)U6d^1mruB-m5u=aW2v6yi7f1$2%K8q)Nbq~=>7H=;)vD|^f6a8)0+{(s0;HEce z-C*zYz@KBf@X(jewH{VI8jE00>wG+T>nM+Xr)6;P#@+b|=mrvze|jPZ@y_Mc-oR!b z=xAT=A;pH}-}NKTb@aB3d>@F{Za>Xx(EezX6>9st#a@CoE8RWL=lbvNaX#gL3piKP zJ}-iP$B_%5$;kf=331gEB&QjqA54b2GAE26_m| za)ro!$|E^NxYa)ABKmnD{jH$i7tsInk>4hwZH|-Yw`%*p4UVzS z{rmERHYFQgRy!hm1#PTCKVrTMO2U`wJKe}ZTdhBqgeUPWJ;4{ugVX21;goi(!SE`0 z%!b1qX9n1he)#3FKGmUL5id}CvJ+SPkH{x`zSZWuT_Rs;yl>%-^T0^!;XNmxcVqON zXrE@jnUjAY-+XHJW9fFlZ=EUJ^ZH-Zt^X(K8^4_%EM>M`)~+~9Bl<0ztk zMT}!V`%?MYW7>ex4*WKZSC80l0zdFI%ylqdv=6$8U>wjhirVq#?&VKHv z+w9c!`}X|p{e21VeX_5Nw7kGI1EjC4IuZTGyimI}qG~{1!8HS{g6RV)=UMHq#92dE z@lG)MzoC_cw=x#ZCGvIn749h4B0E0Ao^$@4H*Pfie6Sxs6jt&Mc;m%)zp!HHmsO1? zEzZmuIO)X|uLWLOQGR+^N&c4|#pU4EPR`xSV?%9g6yg_lhQeI*2Ub>#k9%k3gE$LA z$6bSuA#`R_%0v3!yCuN?ou`3QG5+eEIiZH@VpnF$u4)(ij9>+sJoJ|Nerw$f>-2E!rrJsQy`Nk$o~I`mieKsQH1lL{Z%np zMVq(BOJ3R<=r_yAL)yci$oeYI_ua8=sapFcL*uswunPyy*FS#&c8g6N#b2Dvz9*(u z`@gM)u5Qq}_q?{=@LtjABZJYI?(A5}IVnt8v^>X` zFsmSL<#Mf6(K}&q{dUq605_@(hn<`>T|gHMy^uIAY*etdStl}`swyW;7<%~w6$ z_vXiQ@+uZAc&XyR}hx`xTKeNO(q3zgR`OWqRI`QvjcZduq+aVK>=W$NmCv+91BymwKyg#5~Ph&-3O|7qXk zS@GkM6>n08d^Znu(oLP|_nG<_htt+1r(IKK1OL4`KUvUF!9m;JJZRALg=;EaU-)Fj zss*bmVirD8v4dak!c`S@#J{>=b;at1t1JBXt*K}ty^J(#!4nmE3)T>FgMQ<46$#c! z16bFV5Ii_1ICmd=_97NbnB@y@dB#dyGLW^Kac}78jQjI@&v-I$ z+O<#iJ#9huz=|19_Pu!qp+((!3!bc)kY_#jDtn(?+Kp}pyqyxDu80*}GB_nPq6VCq z5QjX(UP&~08sFniwE9^Y6YeTM&6=`(n$>XN58TIZ@kj8RFe&obW0Mv=*5LYOn$g&{ z2ET`+EcXA_-X}9%S%-JdzBpO(4tM#n<+&BPW(smm3UW;moS4Lx>b#}PZy6ZQd_4chnRD`gl=(a8SWUb& z)PgUfD;)|=pThoYsALD(y~(~@{DCuea|fgYA2jvTezl@1|AiHSp=BinJ*+dUw2wj$ zsd>}doDbc*e>Lwv?dzDbcHAu^*6zcWO|;N=enS=iMW?hc)qYF#d(+;IDT{-_B@^eJ z*Dxn0v07)b6Xs29nBYrHtcgjeehv85UT+a^BNd#4PJMe}*t&dXtMz%udhw9#jL~g( zqu)7rHogL(*-flReNEO1$!2+^Syo`l7SeJ*?I`y8IGb_nzu$#9^;^Ce_M?xL%>q2W z;UIIq9e5Z*-#z+#6g&C?=Ar5)(w8*MLU$8{Hcz9UQum3}7WCL=&Y-EzbuO?)*gUD08`G177r91Y#E&4}$N%Q1$48Ly+7|I^o{H>4SVej>_&knqEnxv+j?db= zpSPuTW-2*HIIQr{&T-horP`k{9<}=q6QWj%FO^U| zc39V*7{Jk5FEG z-2s zd*_^1-R0+v^R|LJ&uRVKt^W;Z33Iy6*bAg^e^&FmIycPOnmyQOJyVTMMq<0YSu|eu zQPUZt`ZJEX+ReLx=Kx3J`7Q)*3Vm5i3g*Q(jGUJ!n-uO7XN5=2JZEFTmz_8+u4h9d zZT^F?0-swQ_?)xxYQ`lS3E4g!&ptx$W6NHziSSLzqYv1YT445h__-R}w(4YSy*W=O z{?Cj>XD0H^pgq5AA`UR;W_;3T?&XBkckH}I_!#KLKAXJ1r0#c^Yo{&z<4~8*0k@Z1 zMkiRP`2rW4;!AHtFL-qTy;L)LsTiY|x@G`-^sCpRms*Qnsu{gh0KJsb3aqjmSI$3t z5E@krE!hq}-%Ne0u=DZi@2C3Xo&7%Y&r%;cOX(d=-=_}``+}kN7<5_|dw%J(wC}fq zR(r586p?Ib$0R&6fwc-2T9MnJV;>HeZ#s1US{t1MFxJ7(g&LmWuA^kH8en`TFwTB< z>4cAh;Tp+c&@|-zZ9Rc;_(yUyjBYaLoi2P@N#PGjuaS(#-peafN1QjFztaBKj)%R9 zuh((L(~I%2J`v7E3pEy>=wY%wj=A}Q5ISu3d}bGT<+KiCZr4^)fQxnr|Tk-rr3I^xCnE44Alu zJHGG8i}1uPg}@s; zV=p}8a4z*uxc8dIiNUv)g9H6*d_7xk7<&Kmn$m)nfkP)8t`FB|HdCkSD?+YBPkiw- z+A5cPfJ`ZS0NE1Q@I(5D?k_o92_9fSyWYd4G}>DWF8zdhjQmBqWHOWgvR+|i&9-Kb z4>TuZOOR#{C6DNyaACgi3c6NnLiso8Y9C|l2GH++Z#Z|n>ts{v=iDe3nJf-Dt_Sj5 zJaQfW4(H(W40}%wzt=U#oZnD@jHpu~oTxX_MW^chDNB&;%vo%`GXI>@S>_*omU8;5EtX7Njg%Ql*``F(e1Sb5nGY}fN30YVC zFRdnR#i{td(tD5Siq@sCf!>}xrmZ$6ZOJ=lIXv{rOYgCdu9IgeTU$= zm$L795B=)v=syguHwD9a0q2gwgJ(j+h|gi)H99MFco4tU;8ZJg8vV<RJ{SBrawuOk1urlILTztB z%Zlb|pP0~6h^|Tf%tgl(ENkADNB?rOD+g#y@P22kANCFpbK7a8EOgI_U*Cs5`0Z0j z>(2XS8@oHd^hVD%*39qSeZ#N1cR;==KW%+0c|kBBS>#UEv1I3v@L7@-wqZj`cn5Lm zGw=(<9#``y+vZ2H|n+dah!CA^5Ab)yN zeWuE!9`1Niwl3FZh7Pad+~Lu-j^S?|4sFvttpdIeF}CgS!%gH}T1#i#mH~2JTKrVY*bR<6rd=FYsg`U4({qo@TwUZKN za#y3@qUEd$c4kHBo;XKl920y6tuv)t@}1p!7q&i?X;%Bnn9P3&8?k zHgV5J_e;v2FaO#0CDx5D?k>&yV|w^TXFev?Fz4VpZyru;IDlRQSX}DD;!%r!aZU}aSPhlrw{i=@f1&P4)N8(*hJ>5>MlMB` zIR3kc(Q@gxk+%a^l@@m3vur-FDOn~JxQu6A1kY9dLk*nSH>CN}5<6mo)z@I-R|TCo zGVp~JcUOlZcSOM{aJlu&$Hq6Lbc59r=2`IdIIyaFlHA8Ju$tn+>W9Fp{2+fFtY$>P z>W2Oftd2a=@!~YUvCBHViSsdRkA^Qg652M6w~xm0?ZHyhu&=_YWZ_Hki6=Pq`IDE_ z#bld1Ptw_ZpScyBK4;2xhtuRPaQYT`{yT6w>c0f1#jUrxa5~MO+@iJGgdQ;(PHW*s zrNHQXV6{7(zV6J2d2jPu!_e#B?4MdXbiHx3D=jvn_g=mCF0}_*kALjv!2|6*j%dD+ z&oPcAOtB9U>TQLNFXLuoS2rbR6nCQtAHt8)t^6+GC%=a;0b|Ynk@hI`NdFR+7#vS-4}RJoNS7Gf{otRnJTYx_3TeohgW{QW4p&&oQ}QGveCTZgf7p)+hh0R z8gx_uqXls^!(&U|WRd1MXsiY9x$KtAac6ESB)@W+o?lNT=SShk9} zsps7&?8TY)^dk8f(fTc6ZY3i$vZuV)tYNafP;uDy{E@ppwI+`HKZ?W7=Uwjk%;CRm zT0YprJwMiQcmC+eTTjG9nz=Kz`D1_N!_Q(Pl8u5VjCSsToDkb4dhyoBagn3!W3{jS z@aH`uo4<^Y$nRkdcQ0P$te}Rs>gExuU)Rxw>d;xpIPM*2UW+(?*u&mh`h!I7*Hxo$ zzTr;Y*-dHzW~N9#IgK@!?l=Wl5gcAkxHxh1@^O0uD+1hI9Y;L@bjShDwu9&ul)ns^ z2%v8$WG^RMq=fd$#%b)`j=Hu?ndtAu)5MR}U-j>7`nQL3_}`utQ}P$??fv0jFKK2qHd;L(! zqNqD@c^(|1U)G(tZ0;~*afiX+3U}h{9M|8caECh#Kj+))kM72;bnnL1bh{hZ5ZgL@ zlFc8}Ro{|sHi{x8}lb~A9Hu?D7Rhpl@t5?d>t4bE2Rw+w#$2JP>ldy|?Om(@4a_RC?s!$A1MGte~>exLB`{1tF^g?*~kb{6GxPqfzi zcn4}2^HkOUhLU1J)_S7=?%7a4y;c2BDG_bd8K-0coryfw`}_vMd~S+0MfT)oe!(lo{;llLb2yJEPqL;w zM*W;A7me!6eFfHudDtfUFq<{X?GsxuYP4f-;>DvUP7d@5?F-@`qtI8sJixuO#T~Eo zFT-z)Y{{T)b_;!#{hw!Ja=yiem2y;mvGWYq4xd@Ll;awp|pJ%;gm~C+aF#Ky5N!r zO4~OOep0;SrZ@FVzpvutvv%CnmtTD5eHGs)yqMqVnM*64BYaTbnc<55*(X)Jd0FmF zV;BDXrvBL{SA4?%^AZoPBJ!56OZ zWhNHV26Sbw&SR(febo`p`30}CHIU7L_ORn>?rO9yID5-Iz^UZ$P|ZD!bNG*qt2wuQ z6&iIF-{L7(m{2~O3~ZZJx4>ILKW#}GmEz4TZ4AOb81r_%JVtt^Y~!X21oWR zVvq->l1IkNoIf#HqW6}3oEtl#F(x+_x;HN#=_wIjP{4$3tjfWA$zs2=%noWM9%mZ@tn&2piZ&5z;@*W zxkr?;{)ilYFL#8li*g&+0P7vs=rw^wjl;kz^`C3-pJvL`yV}asyJ^q6_~JT+j4yZE z%h~_hK+f$^en{;N=+w21a_{)${@5@RmsvT$o0aDFjj%aozkU;XA)lN-N~jGoew5zY zx^W_VT9cuPDYo{F6~e=03@IlgKPyMI;e<(Voyw--$7g-k_SX6wouL`uKB$Q=y;@|x zj+tp+x?*ja_oVjK#WBM&Yc;udHFUYqhaiX&<=0vWMO?3EEOzHnsU9dsb*UGU-11W3Ds8Yx!2n zyPC4(`@?rLp9ME8xxwhJ8I{75hzmt_h+?de3wFu5fm;~+m8RUVOU@09;(cqDk{i^S zpqqFXxj}6C%`)+)XPp~h9_AQTq242(Yqi+1_1Vw8Nn2^d+3xH=PBHBYrt%-ZU4fiu zvdesgr-?0FYT~@y)RcL-b%%pzq(<=^H!t_t-PjIuSETeu>n!|x%9^4K$eERf6c_L)d)=+`|;HuxT?|JNgrk7Yv+~Hshb;9Ij_i=u1=M`Ic+0* ztZFZ8v7&SB8r{g!>q_t4yoG%)St~q_AE75or#5ea9_?rDADwe=^LKO1_sP6}R|cIR z?h`9+8@9pI%kFKSP9OgF)qio_bUwIW_MCe+mm}L(+p1mJ)bVr1Nb1@+3pT0k2U$a#Ws;}zeO$Ewx9b;`Kbo+2zsPUcWtZ}8yDzn z{T6J?tAfq%dsgIXl}`?8E#`DkmCZ=4S}-WJ>d`^1D|mlh-WLx_&8x^w9Z`|nx|g^{ zg2Qg4j^ixNn0%vk0`FsN##j8WZXEC9Y{r-JJi$!+g6Dd0v2_ahCKB$pvA(8lHb?Uw z$v1<1Gi=7E@;r;@SvKP@JhzsS_d!Cr&5-Xk+YNjZ)cl_0TS&fzHeOUXyQLA^*#SSN-4I$T#hpUy*!olJ8BM zu~wen=J{=#@sfPAgZx_v@B6Je} zQ(n#B5bM&q!{j?`GoFy=MxN>8YWe0Q`HvBrtZ#gL`s zgxQVOU_yjl?VZ#+mZ5u&#fI$}Xcc>+*u%v^3kF0ka?M7kQNIOU8+7(0bo30gv!Od@ z*Eka=_@9EV6n_k3ovAD3>`Y@lw(j+xyK6sS@A^6JdlUcleS~f>cV+%NdA_GCqj^M6 z&Jr_^NA!f=S@d!x^!Ov_z+=#Xd!Pfup#xcSrf!~Sa}{IDHuj#fdz!P*MN1ieo`@dC zXq>3|_RCtWBGdc*wdUO-YpcdJ!~51}X3E(>q5sat{28K?NYLTm+!1$WJ51RRY0UmW zt%u&&+uB$39Y>k#q9=Kk^GPAhr*FWuI@-R!fO7-z4Dp#R=fl26Ub~q)B}M*`K42}; z=QjF4Lr1=B>JQ6Aj{*PN)98rHlD@-UG%FN)!z0U>YuTzFcGq_g@(C_GCu`pN3E1fR z&!gfaS$J-rMeL$i}B^y5_3`NA`nr2fzvOyL1R#`IX`<7XagS1I@%k{87O&q;n#kBq9kq@BAg~XtR{(`4%0C7Fi&%fuPho!-`JrJP zd&1C*0`8M(V5}5fWB;Wg{YIY?J>`XTiSwtk_abwofil|WNhe<9O)2Ap_9ks?i+?rO z$Xxh&{5!7dbzgG>`*zru%@-*9 zw0>T5VNYXemUq8pgD{i23vSh&%{Qc7!gF5|2)~tf)g8dzUltQCK zU}B+hufVuM;~s%gg~n8YBMXhY1db>)?i4t*(D;?Wf;So5Y(6~|HICRzm#}tZx@1^J;;^uM| zJVf@Mkvn9)BJ}$lG{3&2a2Y$7nJdR>qg;UoLSr>b4ATXTi`m) z7C>+i?CV;f$(qf!?o5@_drs!6O2luT#8$%7$4yv4cyZqw*B?x!Xr6T z&2w2WvO~7;^~A>goW1RK9d?1(U~2T$XOt&?9?x_$+oR?z_6kD3u}j>@IU#kg`B`YN z^wSy2G@#`z(Aj$WQ0iXKza0u@c>@&e>J0=6U;lg{Mx1^-Wn?gS(P!++ zn3IKmIZl6rwuOKEB>-RF0bcQImEXd>0i4GSgdVig-nxi<;}!DOh2CAtDDa?O?iOhe2am;^# zQOtjVBbomKM=<{d4rTreEMWc%%wzrw9LW3^*q`|?uy?-k5K!jCN%)HJW9Q$4`^`mG zUB`WkGH+x)D48%&>s$Y%+o-E{b8bNVG4xaCAwq5D*?}yZW~8BOOXF-o8hSP*53K6q zJyL+oeN$F-mdFL<6+fLBk~cs9{7Gb~Gw|;wWGV4OkeY8h-weL^@kIG|Sm18nPLMgY znX#m+^E%seva zw-);9Ec5Bh8{C`Ml+N8ezjSW1(2~G8bDIyZ@%Vy(_4L;?LMVNdO}LRTo$&3OcHfa# z`QOW$ub#d`r^5QB$S3sbyC&|u_^@?j!)}4~Eb_0=-Djm7@;?sTlB_9TwM)^9JwOmT7Mu2&OY%pNKShhZ zXBE22so=J4@?dZ=TA#~z=+XO0p21d`Rm91;B}tdHY-gDhz~2xr`Osl5wQI3i@_dwM z$v53f^V>t88MkBY)NHW_6rUYe_SfNWtD0-9$w|@#&xZF!>3^bZ_Cvg1+3}waEt9gOk5X)Ljazw_ay0Usg#XBMJD#O(p$c+u z6j1K5N9VDmto(Sh?G_*g0Hh<&~I z31ja|=wq8PJI*&!=SD|v&ndacl(g9SFP_@=%ZIxl^H2H8aD>Zg=-3o?fq zD}2k=;agVf^da#F3K*X$a*q`Hc=|qA>d6?XwPTdqi1&}tW%`V66IqHnXwVmU=f-@3k_Fgj z&%%Bs`?EqL^NT_`6E%}l!7ZuUuofxzeAyQK%xPY5;kfo}h&PKmh`d`m}uOLn|C zGC}3N2Os`<(b~-6wwmR#m!B!`?XV*tzgb3%RYr`;f3cictDIPDPx*%Ws4}oeP-XD0 zZF_XN)R}(Rqv}rFR`#Q$j{AY&>~Uv)Pt*3Gr+ag@^NG=6PprG2{qf>Q*DW3$@}lp4 zw{2Ld*!h+R*iM9MOGdkw3~vdoz-B@l-ZClv=9ZNrPpox#7oi6o9_@Z=c!0L4ytBE7 zeoqJYQy+oD6Z5qbi`e&=hJUq?tK2!6?{QwaJ$A#;dS{wD$2rH3zt5Pe{s>9&%9y0uo?hPeZUHrB^;fAgF#h&l&cB?RP40Qv=-|KNJ8&XCqSi9* z{yRxG@DIOFejT6uuh>KJxoi65FSbSFb2p^%L-;-AKh>6xNZ&ebVHbSyPvbk;tJARE zx;)?Ib@KZRHf>Fq$3l1Nvb`F(r^W}Hu8_cr(I1G9^R)3f*l^9Pq|VM&3DzECjIu2^ z_ZXuaSINFxh|u7u#*^T!$OqyJ!C&rC+9>n;@_PAkU+iXPGjD~SIV*eNUmd@_Hhimm zCigmQTZ{jk=*!z6ed+H*^FJBhUDB775mIRRxh@PcHX-Y(xM5b@upncjKQ7mb%MCL8 zWo$R=I^y!WUjF|p_hE+jRev3>vC6(C$Y_v#)BTCw&`@lMrH!k4dc!)e#I_OtGif&O zaAZh$=-6)g?%KuphV2G@tFgs4ik(F__Hk=$F^${cZQbbO8tz#T8NOS4c@Ar|$#bMU zcQ~<%eY$QjJg2!&rYGgBDvTH~-P8o;c}9_Es~!6x_1{>1 zxb@8y@M3+)kMUxCNE^KHmx*5q!C~;?qVL+~iSMemwCxX(1IAL)w93m$Ze9A4_28GxP~ox-xaUYlkQskZkV)^c*8^jWl?YX0xv_L=x`lm7Ad zoybmg#O=;N#<2PfJ@(QMx;s#EX`e84eo{6?r1Kza4(EK zigFA0!q{_wE-Bm#V}GF>I|0GHFzR1!;a(WHT5jQ9H1-R^v+I8tzqcN_RP{mT4XUjT zv{h(T1~lb!>eX}rpBmJwVK8p_sOhNliB~{S=I#ggSA@t_4-W4iS>Bx2=v0) z??rFUZa~*kv2rx$ajUZ%G$Y$E_v5k+%@AH59ka;wH{_UeGu}cTU@v2$9lUmc=K;uu z+(&_2)>;s}Tkg3On-c@NCI0pVPp0zkXOR0umbsZR5?N*%ak8g4i+H!BC2C66k@S?J zzKv7hq37^rwupNhL(V_Ew}CrE;_($>_R&TAY?05L*k_02Rg9hh9+WwHdY#iVR~$qR zm{Mo=YLjhVTV)_JBX{K9jD3da_3zXY6ht2#Rxg+Bcvg(6>Wvs9- zbp}+7-c;3L^V)*Q&B&*(okCByXuD0Vr6ySV(7#Z==tEohzu?1vBL}QL=-4g$Mh?~m zVuQb#IQf6P!jAzf?8xCNowUx3<4P;tIi0jF7t@B2_FUUE>h1sTPST!go7Sex`J`2o zCVA~0%=^{y=qs@wO2y_;&Q#Pn=9_by`{pXbD{_5~sOAF)HSd}#v+idtypnyzakN$H zFZGL4|KY9>%0Hy&rP$!9GY5O*PR~I4P|gwji!<0VkE9NzGG_P>8V}ye*f+;%hA}QU zv)Pf5l^UQ$)TL|ZUEF1Qg#EV(eTfj(ng`^>t5=*Aj@d3W+2m034N z-a~m0^R+J2l>JLx)sxtqJVAJzuy{~Y1o}n)oXpHz5x0AYPsVw?s!N=Hn}X5$tw7OV z&*#jW$YP4;TX{06@2N1&Xo}+A&T*YH6)nsxIKQ&0ii&(^XxW|3F0(;@n0u z&jvcGjB@*!5LI4?E_;KkY2R?l8%B8(t@7mm)N3hcsTzxVS1f1W^n~k?n&r}0tMkk` z7e#rpMwB&j4CB69;Rv?J;8p^1l%Zf8+Ik zU9BzU+?~lo(|Rizyqxol(3P>y{X?)#G4E56Z%V9hKH(dA|BHgr`bP@J=^qHBeLK}V z_eKJRPTQUvYBaHi_v71b@Qirj8IvL<4@85 zo#gk=FOhw&G{3~>q0BGzifVq%hkvSZr@r%e7T@eY=aXITjuu~=a<`fARIxwy>&>24 z?TXgD3dZP90>wtXcNjj)2&eI3)WV$2z%JR~T#)!w^V5{NsEXXj$a^F`(C2(8VMTp< zkgwtHU|&A;rY<;Oos)0s(?flU$ncY1MGl~^{WvE$D!!dExf5_Ac)AAJ1^J+{s3-V{ z?Vhve@=NgY-1R12PN#l?m%pOC8T8kI9n!}a@p3$Ie!LXi6}$e<)NzP~Q>I-_tUiRe zJS)zRXD4|#pdo^1bq;WhF|4bq7<6C?dLM`RpV!pfi`NUO$A5}rpIP7ZRec<<_Xb|a z|3ydAL&Zhw*D4sL<3~u*&7bmD@Ap&RTHs*JgW>Z|WT+5is8D36Fl4B9=tmsf=Vgw$ z__sT2$-6C<92IN*Gg`qIy(3W0E{(;GqA%g|hs%B>A4#A4<)dGK>(2|W&%Ba+6sPy| zpreMzI~|sMzca(mIg?FD)rqC!U_W(Gu3tT=>CAz?icQe1OXo;L75xh%Oc=v}3abnMv zpzv-Lank05OJvO)OGYs1=$(v*$nBEPIh`~mBbaIPNn1ji;!Q{KEzUg5AWrsG&cV0j zp5eNxd4mo(Y~F)ag12#cfWpBzeHrcG?Dule`a0pF1np0{^FSr`S!ml{?qM3m2h}QseFn@pk7Qla9A;QtwlP zz1h5rp6M#3;pLLFZ*#oAd(gzo>!l9hj>tu+ z%sE-Ve>X~h!Vl}eDc#!?tQiew9_F5hXyscbMT=?FM6X2ON7Ckd*h@$M zFg6AGI}`f(8x@z(*ueNnoa{y2;|NptO|joK_FmpI)I0W+%W7N=j8{WD^z4MS_|{Oe z8*$XZ(4d7PS4*F63j-gBKTDncap@8#dPU^z4?COkcH4RU^YV7AzG64~Tp`>M#herQ zr=IyH@^*bekdn14a?L(FLD{lKKfwP_(m&>$Ll(8dA@*%iN09OKG#z7Q$~t=}Il2xxwSxbKPs zeMhq4_p`Lxv*lXt(SqRS@8Cs^-9vm0YlwSQs}1FRR2JVHDF|);ESUaC*>oD^=e556Dp6|I4Lm7)`y+qp0nBA*joPLjj3Hk)VKMmQS$EC8t=|Lv0 z+(kZFm(8P`sf>&CvsybvnKJ7bHCAFXDdQKfXxQ~CE>8DY|7a^8^jB%@|6NGK$CZUA zJycq>?j}vzWP=W$Wo%oRvr2wgsI5By?LC+-^fy6&QPH0Sy+8T!St9h8GblfnP4h0u zrV0A{);C?P|91hJvJH1gsJKMv*%-!tG&nE{T(}vW7zsUNpT!ZV=-K7EHumi{iVw># z)0OQUs$KDQc2UnLvh9Dm-w=w9U^x2({{4YG=;Gjzf>P}{cWE7T30jJs^v~-HgqE6n zHy7)m=nC}y^`EF?lxjni{tD?r|1bYO^|CgJKa8LAJ=es1X|d+ZSxZKi`Qn$6KjJ@v zmmes|c{T-O^>-DF)3*b~Rxb*98C}}OeS|j&PKUBdotce&lyi*v{IS@y2b<5u@;uIb z4wL7R=JPAqx{Ln<;knmaxcw^s5Zo5}wn5Ph!DHm!_9kv~K889l!E|xf# zK7=~XyAUUHAW0XR_mCB5*XE;V!appy?F_D{)piUy*MKc(8uCPH2PCzC(iuX-#5aJN z#~YA69G2_>4^#N=F!8g^PiC_#AjcP3jVtz(JyP~0CG0?JDXj&ZnES|MY5O*s(n4hM9ZqWmOL%C1W zSIjTv`+@mnIOUzTeZXF(@U~j?v&(sQ(xxJy)ME$r6`AWxz88A|jqkRS_Gc@e^9{-t zK%<;V#J_39?;>7g5RLz7#BZ?TKO$aaCehExKczq4Ax><{&zIds8^+O=TWQlRv~4VO zc?@!9kZQB+W|?SbfmGTb>3$13hQvCUlckh^vpw=8b^Fm{KHC@eM0HOLZHwyDM#p{ z(7M5rpMI|(|EJ`4xHYw>C-S73FIJyVz5#9Wy(js&BUa{u16kFd??v)mEBP2ZDerCT z|0>^w|HkR>lCLxQOkD--L>@UroUCW0EI-}d%lov}!o6yI|0%R}o+&$8bao)$ob77! zUDQV;=ud!)KPO92ud(uvjpUQ>Xq+CYV5}aYV2s{g!Du}Uc$3JPJAU2=kkH7p z7T-K)@lDyU@$=1dY0|!Ey+y$oy;;Fn{fvTf`Y9l~;n92Hl^Y4q5FR9CSTwK^o|s|L zz-oE6XkE5ETYU3lcx8q~4~u!0IXDyBVwvX$C|_t;DrK%^%^8Lo*D4bRQRZxH@9!g z6+Wt-f5}`NvcE*;nyeEp@YCRm+-BDBf-46LgqOPXn4cMAIS=fH4@ka&!Fdlj}9C|ejdQ}9y8VbF-0eXcVbB9Tz&GKCOeDaIm zyZ6z%@7O}<+t-%rr)Et1co z-8b?q^}7=7-bC8vw0kRYZE5$N^o2#ccWD=C_b%;cXm<*@Sf7EN!B4CM{j|G~e16*f zVygT%PG7BHtiDn~Z2lCC)}IFo?f$v_^)t|I@$1lxY%BZhq92w$0l$6F6IPjvtuh}2 ziu|48NGx_@Pgmw}6@Q3Kx{-M+@#T*A;%U8~brqHWcJFl$J+L>4yDJtuLbA{$zS7uV z^klAAjsO=w6gQ?F`aERS{vQ##+#WZ^Zi~YYpXt-%Aaq&o8M*IT^uoxi^1n>hk&jG> z8}lw_)cpC(^?S5_7x^Y(o0LNxWZa}K!fz&6^}k&~?3@*pHNj8sk^j3jqU@ox)u*)G zLm8~fM;BoI_j;hzr8!TleH|S%M&lcQOPGz__3{Ahq}kurp6}&-uiX#JSijC}FE)yT z2b_f|NB_zieQW~yfz+V|#!BRh_n3R~oHx{AbXNbsE|)l=hnc(!kJHfM1(lcYmAPi> zz~s)B7SRIY%n@yu|Af^sUVIo5(Bv;GFpFia$Si zFOJ2Jr`|o(5MG`lcb2e!81l1wIq~e}JbPg;r^pd6cYTgZ4EejVby_gMq%Tb==DtoR z%9eG*fs?}f;`J)(8-Ofzh&ZXU#QEDNy!jL2%{bxL=DyB{#GQqQrpi7JdT`On)Hx~# zUDU_K>u%~V-=SPp`#3)}58Lh)PO-k(!8h{lHlWl=XjX!%n@fL3rN!uPDHyG9RxnC` zL&5)GUCO>rLwXRt@_%}7N7g{1w;9Zy#Z>InoWN%0bW zVJ)=3z`R>HPU;g}_&>fwMB%P>o%cq**U-br*;uFeMrR%hJ=~9cw4byJk-4I{_e0Lg zqeo4(;SUZTm6;QX4-@S~rfvE13i0`#gTLXf?Cl?R_?+&F(e($h2l@&-x6jp?nDuFE zuk$uIG@~PBF2Zl}fwfWQ{A6hlunS`m6mH_wuZUwm$JDi*E)bsr z8*IoIPVfoc)RN3?vEIB8jWe`_G@HSHmSl35O(?!2r$@7sj~OSdNatMLD@1!*a3lqPQ4Fxs=@X^;_!#i zQit8lPwh!Gq~E~3)aJg#bl4x_7CuejFLMX z%`?VAlhisRKVA0IbDV*4&R=Y^Zz*N}U@Yxiuic)n?j5Kfr};YJ7k2QZYrHZ)g-*5d zKf&7;#?4)Q13K_1U1T;%YbH%(x|P`d#^HnU40-k)7~uV!K9Vs#Lb|l!gI{Ik5WM&i zp8ZuxR?cTHgw}pOI?&j9YoYPiTLX=Ml27vg3)liI0T)!grsW%N6DNIfV2kEmgWu5k z?eMF^{|~GYIuoO>aa4?cF@n3(h?Dkfk8@4{8lu{P|J_4XvIiWahmtPxWHY)Hp#eK6 zw;#G2dHyrc>RUNufbQtv6b*Z^Sp6t8T4-rA^H1`{-^6|}sQ?hcVPBGhGJ|ioqv?jE6?wCMh66aOtj0-e=Paetp z0`O|uF8H#7=N?u+$XT>F##8EWA9a$xe~xsi+frbnnnUU=!1}a1ucu$)^ghJ9!0V_h zlrhmNV-b0T2UP(>+m>-#o+=}qc&W>8RlWqZe!pME!Rr;o4nsk7{t8CxcoyWFC>@W2 z>f7^WMd0;N@cRbvybydJ!uq5DeTv_FzlqzU;(^FIPi`nHC-df4u3ZMFH-yj>O#?}&S?ejfgWLT$BY!fmx@mj@YjFWg|9ej&&R=NtJp0*Jk&{?|5h=i94H zoL_ynZMV#wfL;}&5+`FJv}ZH_ z6a4>zwuxFf7wQ-sjYKC+e#yU&gz>>&VOoJL4Qg=<3OA2nB}A^G%rz z;6T0LK-U||W2>zXB6J-=9w4}I&ccPW;6gR!$vSv3a42{n=adifyt09FlHi(iva-$2 z$!y?W*g$tqLp?CWo#U$y?jO?Loij~fq&p`M|2nq%R`H?6b3*Z|!#vyaYq3k4na{H` zFdw^j=gj@za4t>CtyDhQNRKbhCr)^b_z&9uwVYE+(x*~~UO%EkEOn}MIVE#o)cRT;;Vq;Hb+!73j< zXXvLqo`vq_Dj2U1RM4g8C>X2vS1?BJt6;R=TS2i!xEY!@5}M|LrriWhyAhf;0-6@! zjlW80+P~-RLaXZ11BFhl7drI~^%MRi{psp?k*}{b`T8!W@b$=xe0@LqLGp6Og1+b8 z0vQ|4c1!6>_V?kPON6H@xyj&3e>-~Z`A^janJ1m~Q%g@vAxBAgRn$WLQc=|KY zOnCaPeXDzTKMxfe6|JA9o>J}yw`!T6yMm0Z$u}5(O%5`?Rs9>S9|Z~zpJtBt5)B^y z5pjx#zYY%DueT2!|=Zq%6%*uY5EGSvYprY=uJ7SKW0v5t&UaRm(mFwk~y#9sHg0QTt(bUd>vkhBVRmmLTgX5SNu9_L0Jz9 zJ&|vPrZ9#gTSHH}d*8EY#+gKu_R1Lt*~9EgywKXi#QSN{h3_Wy@V;q%$9}Y;*(a49 zm7Lv9(&Mdf|3;eJjb>*aarSD-tOM72rA^N}vb_9Xt)E5C)NFbCoSgqy6ydDaem&H< zPsUol3zTnvI^SL7J4t<|jIB}+zNtWFf138y@_rKhkTiL&KD$s;3Vh zO}F`$-yM+o?8Ly#XT}9(770%OvfzA0igtdf!v@dBk6N%Tr)HrxM`XsS;QeYTga58n zWhLna|G`=?1lv6){)OthDt@s<)`8>s{sOyMGb`18lS_Yya-ZzIi+|MNdl1^c+4a->~{%vFE2?y#Bp{G5T=@qxJ6;jMBeV@aOr*Zr6XZ-ZS;v5GeSC10A;=8xHdvnam+s z_pP$leXCjbtzq5wQdIBEKNKWv_RCUFQy2f5Pw09~;Si&*x#lzHMHg$$mv(W@w~95N zpH?+bkoztjzEu&@Zi}_Y))8p2EPw~G`+svD3#0$OfuLb9K zsTln`>e_ZK_#$yjB(BX`@Ll3$UD0+ec!)TeKW*270asUyu9UboYr%Jkn=5f`)`9^U z6{8=NxHfCS&cxj>ac$OuLx`JBoPSJYUpL3<_wlsFTKg$k@$$9bli<6|7600=i2MdN zUa~$G{Lbh7!u&3{xc-~TcWu{y*O6}m>%WWRYpwwoQm@Z1Sp$Y=o?;9)vJUjI4qVF` zP}YAw)`72N&&oU^YrulrH`jL$%k+=?n~b}xLDKlgJB+!k`+SV~TE?6`^_)|TudM%Y z#+|cP)_|=4eBi)Za6s07KJZ{Ic<`o$18;)^VU#6hwgZX{g{=Rk@a#6%f1Eit*MA?% zx{vjr_{5$p>pa$gnbsPx;jn)Vm{?pX>pryxbQPz`x-UW2eF??e%yl34s9#+7rL|f2 zZBy&M82w1Xg>~NrKS{yqr2hk8QSeaJJ%{p-Fe;>F&^(qXAGVvq-< zj6Ydrd`=#j>wAE=QwD25vkbNFi_z~VUi4w_0#DO7(m&f&T!OwuLG*tLy7addjMZxu zjMCp!Fk0WFp!mi3>2=@o1TFg5z&EBJ?QO$ftR0{6KYwp-9CKWJbFa?Q`p6!@X=LV8 z&^)#tE&y}yCtB_Oe8YIZ`tkW?D`T}j zE79TK5uG)GxJKIkts^n(aqbl;kJ4ry?^@%kvA56ujy=9D4^8@hcVLa{ z@o8mGJrVNoQ&0Z(;io#-BePD>*T_eGY>dNd?i46M2A`C)Jy+o5)a|)bI%?h*?Xh9= zwP?KjmG7PRa&W{qj);#J(!PW3oSwtw6&-IuE_(5Yt+_GC5if;1Ju%qq{6QdV!Iy!O zzTUnq*ZXqt$Qb0(*N8jqh{Og?i%tvByqoNS)p_`ns7uwx_M02JqF+#Kb^qBZV`h(k zaI=BFzlC`%wti0X&%N!z&E<}y;&T->u3vQ3W|wiFJw94aXvjaZ-)TqxpHxvZx^0^5 z35?ThVNIc1UHE(Kht0&E36yn`dX!gcGk=9_#6HoU!NfH~!wpA!y$^S~#NZn(HhH~g zS<;K1<^^ASz91|oyiGW_;2Y0B2`><~5b75k_IyZqjPMHK-hQV_ruREtvN+*!&-8w0 zN=o`Qm8?s6!m~KxCQn(vvn8edT1v(zuJFw1XOztD*IY6n@mbGpiMR3IO4<)4wg7_-P4&Q;3T$US{B zUl!5-jo_pBeV-2A{RzDL#1=hbICysqJll$YgkNin~RJh^*SK6jYedJD@ zw*N?FALakzAF{rkOgmT8&cV#D0@`_qxfMvADwt=L%$ILy_hDOyL`Ub^R$1+zQ9RI-PX4)_Jzllwz%$0$+J+OHqd-eZdt_)$?AflquZ__UQf z{{)vLUkP&AGg^H0GH{^z;qtvJ?-_6ozr2yyPj09rf0KnLso>%>_PDTRwz!X98sBgB zK4?)>kal7fdA2hi{lO2Bf4D(G(fgmWBbNIX8?y&Pr}L*D8~D+^;$NnD;-AswnLau) zH2a89nPjw*T}f_jn_tH%69^B|yjaL;k2%}+n$ieH&Bdm=uER)UY4t+aXmw5_Cx&msJu{LM;R?N6&C z4L?WNsE%#6(zg54G;PjAd?9J1&9srbZv)=IPYvg+a_Egs+QjNN7{ ztrKZY)HAM~Hs>#lft1?`TR!nAaU*@yzJuBJ?`XHQd7Uk!dJ1!lz19;eUEr9_QT!=$ z@6~N)J2l3#+#k2!Y~Li>UgM9mX>;W6mh71iJ>_m6;HhjE;K>r-Fp-=;qmOTdC(S#t zHMbiwL{*pVx$~JHfq~k{d$rK81?2x0{&|%7e>eAX-ORmQk@H{nbezB5vv<&IC4U{X zp=9^`4JCW#zgE(#&ub+;`)nxjCinE*pWMsS(>2=TO`7TP238I_JpUz62ktvsm!_Q% zzaYb{ahwRP5#NY4v{Ue;8+=siXygAj?jf=VX(R3PS9=`6+DHfYCS}26uYt#gW8>7F zx*vD6{}=^sSWmlsL>l$)+=*4+K{;}Y5w8DJa9nPP=QpV%U* zH7x@FN!%}jf9g9K@2BXOi4`W_3ZgC<9)LeZ%&%`$5{(Dz*jbSc2+R|rGL{IB!1bF&uw!ShlFZ9^J3@-=j03*~6|qrKW581|JG zP}~(77CgJl?3bW{JHfF$TcF|{Z-OtvFRndbu=mrMSB<%edAiOPG2%Y@I>H_iCV2D7 z(Ybq%+3X{3rl0NKZ_P!Id)#sLMf^EVzt7a#g|>Vvcmnxx zI(R<=+&;!0{0vKmjHHj>x7kKKO`p%QhmSY~4?9ht_M@M-CvVRcxTBNm?}%rhtA;)1t?<_`@au3^OCB-To*Z_EE&1beM_}>1-s4LTJpDVDy*OufPMGFJ zb{+LRJo#VnWEt}u?)^>(NT?P*Jw9xwl2x(4RkG@>$YkZ#oEQDA$f{?-rE}oagV4L% zpm#=N{<;YZk9!6Y&M!FbNd`__c*2uGXjpK<6AB!$@Ow`;!ck!hyD{8^yS2bo-dI@|AE}P0-5!2`~=TGkVC&j29@_OkwgE9 z-1>at?>%LSzu^5l-j9~NgKYZGe&3e-3%T{ri7$AbPyD54LGoRm>xT}kUYLBhXGL(i2Z;T5!@+hMYQ{=jSiTsbNZ%l=hw_9AKYeHMC^`{qi|& z{~LXU1c%b+1T%9HilJ$jG#;BY#z4?sM-O!uIz^_K;&9}^(CCnR{6Ebgp ziWjK)f__=%i_EJ7YF>57KEb?7d3e(IJ%VanUdnicdG$5==|{*nRO^rx1@DMvZk=X6 zEn#lOGPjx>fgcO~>cG4b*pYd)06wyWc@@n35E>W2ygI~runx?nNPisj>yX&6GcP+Z zx1xyez}$)!$lQ`S6vOi~%ET?Su@hY!j6{rerZb^iG^%N|`_#y!n)Cs%5mHuia$ zU-%=O%KYj8Ugm+9&oQ6AvgAfPaaqJky-*N0q!9jF4<~;a`MSLec|y>yjUy_s6n@n?2(RgDoDl1({+cdo}Q6RgV_*|JQ*_rSQn9P5 z##LI@V|EApkPA)6ub9G}9?b8gfDY9++d{&ULwC9sB4 zmVBPL%yWJJ#*)1LN9gk!Pm}8oPp|&pm-Og=yrhHcDV~p!@4J%GeAfY4>fPi!Jz@ZU|aY6!qc8I!gvDX5&36e2U$BH|L57;HGaP%f1Tjo z%5Pi89-!UQ_7?VS-eP=)pl?{7wBEBK={3)B#%H_D9@d$5b)kLIw)xEUca!c6A3wKz z@BHHH7^lPh>#%2V2Q5;@OV*{w(Yb8V+BIS`F;@CL#h%Q%^jX*P37syC7d{Bpcnx8$ zpD**kLvMnY-Uv?}0dF0SEx<5eLwXTBcPPB~2G$vc)8WPDnYk+Q`O=PjSN4l6x*$0N z-aHW91?BAEC~)CsaA734;9-n!0vB$);9D7+P~Ukyb%lRQ)2@l&kB;1jz<1`}Im;S<6mL~bo}1S%Ou zq2mRPh!J_LNe`sm z(Q^9fr(M4->puG!?P6^^YN9Pd`7?d;p)hE9MD?f020w)CbRl2%leURoaUcJyv1B#b zn-^K_preC+jCnl{S!DpaMWfM;tacb#?J%<1VPv(#$ZE%t)s7>p9Yj;wY9S?vU} z+6iQ}6G~Ql583AHgh!ODwi8)R9w|E03p6zI6fYR4)~I*!h{aqF?xv0_grZ9WZu-30yqHT3&U+P;}S zIEEg}VRwW{dv+qLy%bM7px=)q{W|=&(C1^HP6*$J87ZlUr2k&N~_4oFa5fTv;(AtqkjS4-n)=C*i5r)b5c5@ ze_;*%^@X$oGmSmMlvsRcvj+S%x(@kYMaB>_EtE8uHnV~?ci@Gz!qP+huXDII=M(xs z)^&=G(3X7kh9UF?GTHHb<}h&AV00<+UT|K$A0J}drs4~kkCJ`^^Dz{;k{rhTt?SQ}ynTI>(q+GW{mGJd zu0N%8*S}b>x#X7%YD=yf^yiYKLGP55r3~_nPs#O^vd(g$!!Bk1?qhx|CCnl;5#kqa zU_Nh9IvVj0EbBf^jgPZ&GJPbrk?m=tjH_qnRUNO@!bb?r+E06h&K#h<@_vx^3Oq!6 zCH`yLi!41-o)6RJ2-++(Fp{>4{<(+NKCB};f)lj&0PQ`Ao<-WLA(%uu$D*1n&EwXrU>dh?&pMKKinB7VA+uoi*LZj@8Mxi^P zU&5+CL5HwKbO_+c&z)&#%r1m?4MCnMxNyE9rGxtfwti#LRgV^(LtDQk$h^BW><*pS zO^Kb3>|cqkj`&_>LuuDiJHE(9Wv-C(6(Nh=&5v>>B;-;2GoSbD9mYQS4*Gt|Ut{pg z8o(LuMWe$xkG_)gWmEmWBKcOs&gWzlJeqIY#@i_8hK`)Kpk8hLq_jHZZi6uRb36ES zd-!!Y`i}_w?>IPfaq$c^d+tl+Z1d@XT5UhdI5SkMZF0n*6UNuk4o9=^;fMD=M*NYN z1AGlH2l`GIYqe({#Exu7Q1g+OgNbu;4^*hHbtHD%8QS`old^9at-nOMVz2!SGDC_M zhfkbk%R^E*qh|{^;l!u8oaac0saalcPf+qxT*wzne#*)=<);;VFZM5X%KJUC-+_bX zIS2=5AmqH&B1An}j~kJfu%UGX zm~yhi&Nw-U4wf}3LBL^)EP*f(J>c6{)pUfu{~KiQvllUHz7VTfZ+7*^_p4Bdw@+Pp@Wk8__n+=)$|iUkx;X$O7j7 zv1f7+C*ygbjcsgq8P_#!&bMG2n?;RQ+9f<4n{>LvCa;?q<1yTd5=4eUf0KMaL~;&Y?3s4l*lgO-N9i~6--h(U%%ZMZrpUpk$>VQZ zJZ<}qcZrApE={-ECH6}Rv|a7HgFDh+M@g4|)dOcxA8cZbZOreq_*86<67)sH%O0!j z^~>0_E#q~oj4#O}_Di1w@3^FlG*!lf#EWgfhd|NYDVblz;qym9eEujHqi|o0Ku`}V_!-c znY*v`W?kdX%nT76k85_*r#oNDQ5{~M?j=Z!Id8sCZxSqtZ4|D>#MfMzbgD+Xvpf`IM9TUE!)!qyIxdL9zKG3d8_;fe; z{59a(YWR5-Ya4lf5uSdOx?Icid(*b%3fwte!CmjqG~uq_nsE0V6YhD)gnKJZ_}52F z_}QZ--1qpl-0v7q|Nh=5_);&IHBOW^_Eq-G?t3U?MF;ddO~?%KeVWi4H>rL7cyvec zDe2+ycJ2H`&M4MlYkV7N&t0A#L^}IUb=VhgChd#M(?dz`Njm-AOj-hRv)Bz>C_kKZ z?hB~{#|Dsg`{n5!NgqWzYn<7nJ$HF}Ea{U;hn8+8?Q=qt>se0|`!r3->EUrM?fh}} zGA3GkGpt7&H?Y6Adyc8=!e?CLWX86ZvCB>?d#ap#s~9sSd&`(HK5EPu8{qnR<`}N8 zVhr0MYt5(M+Oh6@L?G+V1p=Yp3k9MJ&9r3CSM8x;B5!`jK8>tR???ALhBbxXX5);7BiP7f@^8)iG3jU@M+oJnB7-p zMxEAbGvT`-HZ7;^Tot+^^s%yoNx!eOU*3>D1Ugp$y&H_omp@(X@F#VU`_u3#h+bg| zeapOAzm;*Cj4jN5e2U5YcE)Q8{T*)YZQg7P`}oAP(x=!iuTEo}JD=N{E3k`(-yz2J zYSw)=#KzcQ( zUCS}TCkan_BKn^$Nh9n}sPt@0$VdNt8vQdt_BDXx31WM*oVCfozNbr)2(tfij`)B2 z^P#JLhxp$UWdGwK;2i}2{zqC;FV;7`JY|IE36rPhtt-24-XG&x-#4+&-$%&dc|N*o z8T*5@^Cs5L2U+(XLWl4GxZdR0n!AlUHxtt0pY^1%Zca<;6FwQ;wCtViBh2rho!Ade zJ!s9>3eF9bYxc&@?4t(EPM?h~!;6kX&DWQhuiXEH-m*P1+LiAG`*(WF!x4gK*r_Rq zObHadVCK#mR$b7#Dia3|DB>>j6NAC zcyJm#_&eiYOL+D_#e?Urj0d8Va0`xH2_Miq_y7Nd5ASs~@d3r2i4T-L;0pMV{uB7n z{tEcu{15TrVT*1@^6&o?A68l4aBqjg!`ASN_;8MAnV&5RM(NE8{wMfgW1VJ426i9= z2OtA;7fcyzwJPyf!5V9AruS9KQuw;+UUimA&RB1S@0P_sgEy~m~s{^?g0vhIEk8#TJZp^1>Yyx6*J3J?dz7SrNmL%uqv6rR}Ztf8jxtw+T zlK3f|yw0v}Up;;ow@JRb9^PM)=QHNubDcEr=fpKF=;h5|4=OXeJoCsJ%_n{@BdOijPY+GtnWJH5QtnrgkQha5Jyq570F zk!hUW?!>$~Uv>k1UP#{$VGIhO&x3uftdU2Ct96w6PS&9p%FO8Dy+-|C*56nD_FDN? z+RVMM7rvX_+ly{;=}?rGy(TqYp}xsEmNSkb*myrm!NsPbs|79o(QSe;`F;GH1v zjCn_A-$mo=9E)6L-inSI@ABMo_uI73Mmz1a*8z@k?~udh#Ruc)ltH$cMc~sH(EAL| zywzi~k4<xHN-v}h8|R51tY{rSEK4_SFW?#fpFBT}4rwfF z=FgnkCUkt!sgz}}s8qgHM>BWa;3n(tU4tj}#!tVdPgnFcR`1QXqroE^d4288v6cN< z2k(oCcSgijYx(&`Y`)FtMBZrpMYb(_AZ1VDT%jtv&Mf;@tL!B5546gD>aw!&O=Ol` zIoOD;v>RdMJ&oS1ZP}eEy8`*Ztal$(?>wvQK=OCC$}YRC>^uBrZ!0ijw>gYcq>COb zROTW0dI>%j<#;RcH!JwKfqXJ1(x-=2necoCIg74fEO#2{(rD$fb+7lpfi2H z3fE3-V;wfh?#@Z)J1Hv_-z3~`u=FdtR*aq4*p;Lu@hd35;xhfO<6`tK@O#VJdfW<#A7%^W3Dp z8u?ezNAhl`Jz4mc5n48JlQvV@AiAF3(B-yeecGn1t$ZW-HR>s4tVNa$JwIvhJqGs# zAn%Cpd>=H#NxdbXyszb*Gp~(PfMVAtdVTeOctHmLPwB!Q2yGC3U#dXbS4rNJ*gZ?W zDXb6Vf0yH*zi%(U`%w6h_l7m$g-9+FxCv6$@xb=sj-Y zBz@ul((xZ{rf({RthPbLIU~2J^uDM}=+X4)&%#gpOs- z98o*Xsr40|l6+f4o2Rm01@Bmrnby&pdi&Jo9WCdT|4hN9I?nY*3T{J({%pm!BfbK@ zRg=a36JvyaMz85bUr|;{Uv1_p`bzLv@tBH%_){|Pw-g>^#V^rTCVK^!F64_J=xy@0 zs{fv9hrS(JBls@#^=&KuJK|UJU+GW5tNWia^FybVAMlgGHLfptmT#o&DEdeEy}*4c z9=|3E#_D?%{P7)|4e7PmHrk*iSC&Ou9ihABE_?BzcMkgsIa?!l3W{$TIcrlF9PX^++cK_!SHYM4s7wkEk2oNx3(_RoLZlZ^0!u`u*Y7>JX%-mu3eA*)cNp# zuV}@mr`-MeDrbr7A2Q!3^ZqB!3V&d|?`O{FFAvMyfM4kBa#!Nyhx8Sb9`db7FW<1D zQ@L+N#yReduxTfpKrO`GGSz}b78JVD?bO42|LACbbL~L>Gfp$s_6Cjsb_Z66_%gr= zr*~Lw3i`66uQ+_)JsIpfo)_fXKS47ZxD%j}^X}iiZ)d%v8LPm9wGY|GC>;C!h+5s| z9Fd0}G6daQ1N#|8wneV`?wZk_@esVFAGx#c$8y8D8Tr_}yszx(TlwxN-;9>A8j375 z&TZ^VG-c&tiJL*(gKlHD6?d1!6%#khZT!WGn=Em66F1Uryl=(*LgJx+3tj;p_$bCLq5xQ(d-Cz39(fHC7fL8%+~&bwyxWPV7z zlq2QIIf@<7_zd()QeS_*u2;xc#kdqnd9-1rls85*u9AFh%9=u77s=g9{7dd#lD0{{ z3i8XGZJXD@oRoG-TG(Z2LT};0OD^HTzj1ie@LM1}cvqNpzZh)|QhkQcGS#1)i5E!U z;CGO5a2r36w{EE0I3y5xbrE#66Zc0z6PC2yyQ>3tkk8cKT^4D>ZS37~SKAKc)G5yg za4(|6`~RqW`}nAedw={SJcWn~@}ywvK|~THJcb7WHA@m8kc5x~!KyuMvYTYhW_Q`$ zK;lE8Vq06hw{B}|uXw9fdrM+%>8-ZXdVPpq>utTQ*4BF4-fMiQ2JxX*MS|M=-k*6m zXU^HPdp4opKYn>JoY{H!%x6CHnVHXgX67?_dE-)ZO7SM*$3vyYDca6gP=|pFP$uXy z0D9uSw|o8xWAIKR^3NKVno9Xk;GM1AWq4PlVBXOS*7Y>i6db7PC^&Eg{CeaPwBOEx z14Oqs7nTh+UpL{vz`il5EATz%lZibT`v+IUf6wE2&r*F}!d&v%b2#rkjXKhrgX{#! zxR-$++0)-D(makdWJl(nQTbLHv?19Kt`=#2hBS1ZWjbhm9OK#Kxwu<;KI~Y-(f&j7 zX1ou>T7zj2{9_{LK3{zcm0#_J{?c*6%=Dz#6(g0kNz z8Q&8IzaCv&*mGiX{^7yt`G?;|d2i3mKYZex{KId-4*9#yfYX|H_{dMuMk~;NevWfz zfOF(jz?r0d^2UAnJx6~us^>%CapK;*!-GG}JN)*K^A4Z*Y2M-gLjQWZ{G8%}eH!jU zqfdN^`p3ir1N-qV$IQILL+9ij-T*uX=AO}01l`Vfdb4aU?9P`1&U)O3g&paEO@OsI z?=aSx&lF%Bduw(+?#bqp9mAVp)#LDy3-S-Y0iJqeA@Kb<@;m}uA4k1^k$3p52K?TJ z-uhxwTI7rN6-Yb&z7_bTxGOMBTMy|MSY0Xl!`VC|{#p(Nn${s^06$V>Q+5_rD{c+Zc~Rvf#Y*ORp; z>RIfdHMQ9X)Ml@Y;&en`rE!GWY6ae}1NLA4W$0EKD`#VFfJ|0OI5K$n()!?4!5_4@ z&)Ncc*cnLR@E7p?YZz}{hivkztI_^wlb1d$JKTpcd2Nih-CXc7(c?qNSVUjqRjL#0 zx-fa`bMysX*U}g5d^LR=u;)l$@b51A0_P4YmVZ7b&K+ol6hdUsr9mMaaCc$>!z1rdLH3dSD)={r9k+w;~^ZE#@e=$@mnEBwX<7#|Jo@J}$#6%ziN zp!2;%8+@Mn0qR$P@rLGzS%@PUo6XhOlg}67Gbv9qWWA-}P4K|LdTsDd=qyekobpY^ zcVIo@AQRoW0^f3Nu!x-lDCm7xq+#V^KJPDwe1ZOORD_Sl_t?JiJIV%TMZMYm zeUyEUHjTaSHw$G_82x^n)`_cO4~1kf^cmoFKlNpbdxh38sN*z}yFd$ydsM`YLfny2 zXS4S&kE{UwXKI7fu>VT34yF5vNC)|)k?25SKN4ZrLq<=NuU;LKnnmxKgCsN47Hi@NsG7j-{K-_3Zpg}$5025+J-WRbh*8-^xs={ z7;6{C$9{wn|54rAW^4Cf4z~(UgX^D;J)%U};h%h+WW=-P9c>U&1)~oz{89ut&Y*u6MimpbR&(~tLq-^o4%=E?f~C0Mg!Ug`hwsMIslxJ*+> z?XnH)4y?)1Zii1y)&@VCi*X3F?gt$;gzv(5L-Gyj*{B^3i|+&YGC8L?|Jl27mahNn z8+)j%Uwi{HFLc+?hu?n;V>6A>s2O$wuY#9IYo9Z-n#R&pINmShI>2k2A=?*d$; zJ3O!_ubBEb$s!~#y@WPp^FI33IKbNUO~@gzUwsT^(%4xg$|E^r1#rGd;F`5e^hGL@ z(z7xRtIU_dSFGI0R4#1XqukeLmkpLsxu|O!jcs5^K~$~~`-;xy)R58itM4u5GTxo1W%$vn?Qxy-84KOX3m2pW5~~=tw-v z=m?ojw%yB*L;j)mn@)6ut$Km`>u9q-f^TTtG36iN$>@gf?*ZQ?{}2o|Z{s}|;PV&s zf!8qRypH*Mf0vMd-eL03p|$fNgCqUx;CUtw3i*e4nZ_Q-&|Mve z?uyQsku3*o3^6-X4@K7&p9;87;cV)c@fM^m-fL%TxVvG;__~<~NSE{Ro(aWo;O$O4~JO_R=2suUOpl-anafc!zf!f1BqC7B{8$ zcZmD=o&25;&VgJ*XH{_*U0vEQksXze-0Uokjk6yy-%hX>D1#Q=wN z4n*IJ;HNYNd9zc{@eLlqnKNl)A9nZ#i;<`JNApv(4|Nsv^9Ihq73Y2ZHnI13Xjb_A z)Wr<1wb(!LVV_SEcgkoFHV=C#u))7=6ymTqb`ADnv1g#3Y5pSe5*|0;%kVq6FtvgF zkk@Ek@MC;$ful4FKo^>?_aL6yJc;i&Y3+o#xgxF$VI-&B3!bAi#5-3Z?8O<{yCISz zP(RX9X?f6p0{$_qg&sp4XWh9VH38|U%rntPo)>T@3AjxHE{%5&iLfgXMtcghc46-) zB97k4Si|_^%-(9mQ~4Y5eG&UNkD+`T(+S^77B{)KT%>V(Ph@Bx-(R8h?@-@F-OG@k z$|N`>_x+2&Wfj87zSuq3PvQ9BEh^x1mf-PXR?eBdbMbw#PFp^%dfQ zM-WH2-H-3B7*i;Y`UJ&2DB}7tR?%8=HtKYUwb%6CgNUQFSBSK@AkE<1rNG%L;{MGA z&bI}eZz2x#T7mKEui6VMQdi*j4va4Zd#j)yr5P9nnF9T22OQQef6W_Tfj2OzzrfC| zgpcu#{JZnC<&m{Kj4g!(+B+ z+a$t>R^LOtIK1h-EeOMY>iu-Kdj@QYepIfdJ_c;;?fmu**fqHEOLyF+y{bj~_nm$p zY}>v&W$|sdJ(Yjfulo0$#(q!7@BV$K-q*ix0^BKZg>d8H3gE`UjfEQnHyUmfTs~YL zoOU18k>)IFs~X@ze0%Pw$@{-2@NGocj%${JC(i6$13&GL-jP4~s1N7jUIw1juCJh7 zY5rOZnp}wU2DgvOd-jr1(~ge9T^WME0Qo50eENdcFf?(?!BL+%&c=4|T`~CX5;~WO z@d&&LIUw^}M*loUy9HyqZBC)N<58MRiWaB-7xA=SFVM;^+yI$=2t51JYz^;5V=Td5 zEzy4fxX>PH8Op=j56`a@;Eu_Sx19sKA^ktT5pR~*=Z*pcdy)C0cNSp3XMBMCW3eBY zo-QL^#}}u5>Va472=HpV`ph#(mc+dn;6rb%*zwNxPS*HWzgYiQ^p}aJVeZMl-re|J zhWHbka1Q|S32JBj{&=%Ccn5w*@tgEKS|0J7hU1xhy#fE>{_@c&IuG-g(cCUA&gEkt zed^I~EcwQ51AEE_@y6KEf$ML)?c-5Jy`&?04|l-cho804H8|_kef5Lo$A(ty0lo7v zFFle!>ge4#qx#m;+MX|8H|oGaIO6kjQ&*fAcz>vIPTutWg|L0`HejEhciR5(qsIPf zT;AmU1*66t)n>0dq0O#5as8}~C$3LbowzHt;lu{aO9Q8rr3!rWQ)9YU9Y?=`e@$wv z>7S4q2mhNZG|0weZxLrXv6ds?F$6!t>=Qw{RHpKUcZ2+0UnL#EN#;6X;8&w9a}JX-8pvps|_cgns-ULb=52G`{{Bzdyon zDvQ?jG>_1|4_cEF&3=dY_Ylu?N@)9M@Edor*`9De`o`1veH6dnKz=IwBj}@wP&VO5 z@h_BU_aBD)CLHOS4q}dj>^gUk)Obb$1GF8m+Nr8HC7IfB{vP32SB+L!{^#>4yE2qXHwHodmzBk&2$uj^1Y?8QeZ~BKZXWQ5jB$Sn^nP!EcQ^Vbq&DWAy8kHr?cn2f_}>J7 z-vs`?3I2D$lQhr14SxM^{5}ok9~nP>Kb?QtI16{2;ckK}o}{G;V4v+#=v{yKg|g!d zhlUqG?*sp=(b+fVJhM{81D4pIZof@CDi={rbDCHQ2`Q%t`J4+;z`w%sXX& zHSj6On{aOd(r&;w=JWYJ(UG>ym-k5#;tu8&usF0G-UK^k|M;nugFSiU?=439@l(g3 zjp^;!y}$hsa{|c=UnTxSyHgnb-p_tBecW$ffnMiJ_)T}Cj-f3+M7wmKR|T9*$5_T=_a|4C4L*r91cSx^iYNXqqBMmf&BbMd|KUjU2I_Ql zH|UP?-=2##9n!PCv+ipiq_ZZp-w)o|5?X?{$_0HwOSHkK(I#L1(K<8i%Rd@BSp0`_ z(0n|n>mNo99%)7S;IlWk!Pb6v(Q&|;1bXAFsD*$3Y$;##<+E}Ry`-R41SVNA!9uw*Zuup zD*Hrx!T-5}=U@MAUS8|BwF3XAUqU|}KzkRUz5B=IcMK%*dPuKG#K_7hwUm80nV$7TaTrj>J-V55C zd&QvxWfWFJX+8ts%+>5v!NV5a+ zeTaW{`=kT;XKFo@5Km(yjg{|?DI45^GqE%#w-YX(*HZVv-30e*I0NUb`+tM`{&~jn zXQ4BC_w&Z_N!TlV7x1=>(;j#p^msQH>@1--6%QBG zh&L6_D!Y0I#s>|2MrYPlTL`p{WBnvHD%Fns@ZV4ARNDm54c=UE-=@QRuGe-RnG4%* z(EsUE@0dZqAIF;X;NORqJxXiAiBnSNz4HmvYxn5m7+9c5{sXD{v~-=S%P-Gm?J9Gq1i+y(!m@V^Lu z0)EJj^=;$EZaF7!?BepNID-m%66fU2Y=3!**7(8c+F&vvWXb+1n8T;^{`w^@({>=Q zDND9ZWpTd{aV?1Z0Q6z9&qV8HR1S*(l3=Ae=&b%`wgqgPW0pK?~O~=UU(4q zm-d2YJ1{5Qh&i{2zL-~`8^8Zn%;B#C7vhJ#i0cO*+>ZJ9WrUrroxOiA(%lI;@CuAG zXJgEw-#YRq@~7{=6?5_nNJn*`w|Q=dE6}Fz|L_!)0bR@y)S5XlWU;YhYm=ViMa~()dusRSD1Cng{eor@yV?^@>Bx8+}qJe zM+bWbf5karTiCUF>ZmiUUr)W}47*}h>Lq8`@pDsuc80B)o%(|_>|3)_zjcO9JTLX% z&anRTQcpR<*33yg?hNanlX}=0mOMX&yW94qXkXce zD-}Q5_!;f?`KjGXS{C;4{8Zc-hIe#3oMBIuq}rTehZdxo5jF(fqO?Q)$u+p+^tgt% zMY&Cy4>0aOumEcjj4Of&agGh zQnQ?4cP>kv?F>76d1|^d?8fD(Gn`@XFHfE74BN0Gh5I~a+dh1Y)y@xO@IR)6|6K7i zS{*ts_5R;2cr5JN^HT3P!(Nz`dIMq5OCFl|Ss(P0=RD8+7~f6*8H#_J;-8`TKdt!B zRs82GeyqPtn3!)&Kjsh-@wX`cs}z4k z@y8YaF2#Sf;{TlDzh3cwQSpC8@&B{p|C-|ersBU%@qb(K-=+BfP4WLg@&CKx|B2%N znc{y~@%JhICl&wo3U7Q-@qb0}HntU|F+`4OY#4k;_p@bKT!PtuK0hV z_jL#eYokpHTdN zSN!=3&Ul~Bq-UYxKTYwUq4=jM{uzq@(~AFG#ecrypQrd2DE=jif4SmcrT8ya{A(5e zI>o( z>=7UHh);UNcX`BL>k+@#BmM@D_!~XqZ}N!0*(3fIkN9tS#NX}_f2T+MJs$A~J>tLb z5r4=d{(g`62R-5+@`!)bBmN1G_@_MLpZ17<#v}fB9`P@D#Q)hNzTYGMRgd@qkN7t| z;{WOq|Gr23agX?qJ>s?U?Bn-nkN5(Q_z52I6FuUmc*LLS5r4KvyzUWyo=1Fu7R#Fu%*S9rv4@QAPWh_CaAZ}f=Y?h${5NBj{y~rUhdkmR^@xAMBmOCm_@_PMpYe$Qok#o&9`S$ni0}7^f7K&? zz$5-mkNCfO#J}$mf7~PfV~=>Pg699ssW$#GjctegZbdCPImJ z{O@_%)UV`e{r$&=s)t~7>C4!hf92TFhP-_3Y542lKO5l>!k>h{2;uj`AA^6imap9j z|1I#hA^djuZ-)P4gx?7N{qR4C@EhPi1b=mYzV-zCzk`1R{Ex!_4Ez=Fe;@v*DIDn! z!v7Tfr3i0;{|WdfB78IakHY^k{C@Z!f}iS9fO;%KJ!%m4SA<=Q@(bV(ymoA8CDM+D ze+T@f2=9RZ2KdiJcnJQz@J~VbeE6@0|2X_b@OQy~3(^eUhzXJZ-0po7?x5KX^{a4{{grCaU4}Tr}-$MGYz+Vl265(Hje*>jQ znO}gv0)7-{`^&@MX;XPn;pzN)HZJPT(-v!GI2zEeEz)8)9ekoL&~?))L~m1bk1m)n zamtxz>*p2CUsSqsjW%av^o}QvoOW^ZjraX;{@m(ozw_kLnPtJR{OrZi#g~2V?x&7_ zYHjPS54|*Y`Q>}R`}-5;R3!i9;a4WFyz=^g|HH?#w(L6a=wGL)dbKx$5y8;gTP1-l%ehl{v+*@#kI1n-yt^&>v z_c^!&a1X-$9`0{&r{hAzg zE+1|b+-SHlaAV=d!4<%bhbx3T1#SY|sc@&koenn8!w3*lD5t%h3zcM;shaF@W9>1CP^ZY^9nTv^#;dwg(f;mYAE z;40zP!L5he09OUK5pEM)HCzo`E!<|fI=C%x^>7Vvjc{AxE`{3$w;k>>xX-{{4tE9I zm2jVhGvIc>`Qe)2n&AR)LAVyUR=75}5ZqO8?QmhZ4!8(h6fOq06D|&yfJ?%4!tH|F z4c7(N4Ra@IxQ?p%47f2Y%=SKlFhg z`oIr;;Da@IxQ?p%47f2Y%=SKlFhg`oIr;;DxpWyx(?yGP&!=>Q92KRNiTj0I{_f5E4 z;l2g;FL1ZP9e}$X?qA`)4R;6Jop9fQy9@4axO?Ef3-@nu--9~{*8|rJcQ4%c;qHU` z0o)Ja4#E8h?%&~l40k`=18_fq`zhRmaEIZ32KRHghv5DL?qRq`;2wp046YCEakwYo z{uAy=xL?3M1@~WYzl8f0+|zLX4fkuf-@rWs_bl9R;huy09o+BXo`?Gb+zW931NTR` zKf(PO?tkI_5AH>{ez=$5UWR)G?p3(Iz`X|dI@|!<5x6(tj>5eO_ZHmSaPPqV74BWQ zzrno+_deVQaL3?2ggXv52zLVRBe;*@K7so?+z{LjjEHS;F}Ng|0Cp=Ul}I9GM)U5qS>GPP4N_P(SqojIj?lC+3-=@8v6Yw0s9Xk zUeo@*R&`<6hI;;fz8puVCW%{85FI&eYF331g{nK;iFI}~|wB-DA<}O&a=Az{bS6qAv%{4f8ha=#chC8je3dZR~ zvIgwJX*heT;U2L@)BPm4$#B!*X2P8VHyiE(xTSEb;b^XEfNO?J!0m&(8}4zqSK#v1 z2@k*B{wRs+^6^`BqwrgGgum*jE~>*%29ID+K3QL?E7gh17qcZvGQ~{FkN05+zu`64=s z??u+E`e5^S_tc-Z<)*eV2k!l!ednC@-~E?=YvEVcrYctK{LM#?KJeW?J#ggmzDqBB zchb9;-E;jVU;F)M&%XCtz2jcUyL{gn4}W3td5>Q=VQccrhvR?$(f{rJ-q=4MNWOdE zvByFOUj5?whu?YfUmrXD>F-Q9FFOB7N$(BGjeqVrbj2TU9NbmE<>GI3Jh}MbQ*%ch z`o*5h9(nJS{wLR`zW4hr&A%MmT3kHk3xz+OyK>gd1s`|JubF)6%7(^7VesdLSM6Hx zUwxnd!usA5S>@!i~Kz)NcOe?U%n7UC{RS^9sA1z$Bo8_F;UaCGHov#Hz%YE$}? z;nb$|Cx6R$7FhPxvVVH(fjd8Z^3cUHo`s9_W%%xeqd%Sf&v}=;d&O^=PZz(5N9fNi zM`u6Tz(;9idWjb*$NtXcAKnqjPXi?q%k{7N|Tw=7SF zOT1A-9wZa)6#z|A?7hScA&{-`bKPqsg`G5wv!KWttiFO7fnM`IuTN$@mZ(VxVR#(Daa{7iEh z{Y~K?O)PIU*e+v#G|$nWY$uu{_#cfBjJl7~Z%J?Zin{bsI7KiT;Xjr}L^419$2=^h z@QQXQv&uwozeLk{QlbN*wDw>;5)2z1op8I1P{8QuYKfH^ElZbVi{gAtG$@9<4DQQt zAHlIQv>j-!GB}-@4en7~2j=C<`ZGcj5(4kF@Y0_y8G+592mrxl_2_Fxm*EHfm9hU> z!?M1l^WQR^051~ZgEG~0nn*9dG9SJ8Mm`+162((rr#2f6AN@aCYYqqfaXk=< z2b+`lB>YX`pdO6uvPZ;&J3Em?iHLS4W1Y#8wfefn%U$x>qSf3r!K6Q|ha#b*#=Tog zb&=R7(TK+VKud4fR8_Z17nu{fFBU^!&Ek@hn$kk8BN_;Xb$>V@Nj=)ABdeG3we8G&=853A82bb1wsOPB${Mc?v96& z;055@t^1pkpwFCjI+e6k67!{i2f2QgL3L0ZfmyMr|NCvLq*ZOGsMo40e2rRNbz`NLtgmg*sw!(M zv`~k?HK>OYdaxsgA~0eHyPAVR%uC6(puW6RuL-TioZO7L5R*Zu1Cy$$6$wQ!9%Il( zrp`#BGZu@+X}*He1kDPtS|=aal2JJ!2K7LYgQ7&SP**UV&@zD8SXY??9F|G~Vs&Bc z-xSnY3{XQ!qL!F%Fjk1FYI73A6>d1n>T=pNVdxLWO)S?}*XYX@Eaf$%ejAHNn}dl& zDAKCO{BeIrFd2-yf{h0gp{qfkjI`MmNq6;9=K%|E2VShLAh&g0Q>Qw}T zv4&umQYfpgY?h_;egtE>B`cQWVAgPk!3b@23mtj&4#*-Lu7k$ zS}_JuFgKAJUKKbnj!0`f6wpjKYK)+SI7oK9ufk;7tDZC$mmywa%OTvut-`Ksz&()iY`tEyF#)GXC%mT5CDU%I3NipI+q zuO`pT%NKQ^TEJnI5NA7r9npCAoCNijcxNOT>Imv{7A$Ge*JE6`K<9NQ-Vb)gf^jr~ z5(&B6pG-!KokkOr;mxRv(DBS4AX#%e8(ek;X_VH9sH=3{ERM^}35-6F0$Da1lVU-C zyZNh7=IMX{Ex2HJ+#drgKf&?4!BcVz{~WE4w<`4-L{{w}?UWYFel4AO702{+gU zDO3O^;T=LE`Y52f0PT!01`GxOm`V-q3PFO60*R9bz0gE8H`vbR5oq&X%>*X1-N3W) zP%uFv8*j>sOj0NPaSQ_dH#Ivog!yZ}-iDsXVPQ~85D2O@LX9}2SR`E9qElmt2$LK< z9d8qZEwe-l5fl%0n$TD#BM^-wBbc1b!l^?hjNMoSxddZu_A5!*Poo2603ows(;@CdphO-h6ro*3IpZ z)2VZtWY9@2b^h+e^un2eh8Na8dIv@V);l#U!jpbwL~CfQuk_Va)vh-xx7StHHyYKx z%PQ-QhDKjwgB%9ZT9p$S!$ApV4H^`Z{$yg6SqGh_)nJ^hSIA=|!ck)CPSN<}%Z1Gy zyv?{XI~!WUX10166_wSEzMN~vNak8Q$+`cpsw0MOHe8m(A=H|LPlOT}$oc9|H>XM7 z>@Db8O_&7)7Z8tFJb@8C77Q3nWx|tVlr}mep`D%Jl}@t_`P@&EN&o^JNy|u(pcx6n z5t~?w5{W>G$r{B9Pt|TV>MI+nDk^KsFVplQp`;2FONs}mVKLk%=8I{u)SR6tti|6E z3U~A9#_CcX1$Xk)Lf_-7tKzZC^hmHX4w=s1+zcfT21puV2)m-rNP8r@J5sE5L-vbA zHLev1X#QOw2d$8>I%T@EfEEt52gBV+1brIB&1Oj55>pc{W>mf~XHqXs;FwyQy7iKh zot=={lijGNf*qtKS&mt241sKUZB7J5En$D6P4Dn0_|!WevTA@$N%Qq&M@;yd!ZNmf zmk4T(cAy(1NO4BM=5y7S{K1y*=#)IFBg$v_`TEY%5|WNe;%$7&2BDz|6af6Y(UeiF zg6OxLsJHp*YcVpc!u;JC=8BIv$V@6Z!5B&1DU}evH2cvCyW21{VXXsNlR(qZP+^HT ztC@J**4+eYRPb~>*x?UhQg7lzoZv%Fb6Z{}HgLGXz-qmvGaMF@gvG4T`RKd-@s7@z z?nj*x?IB`%b;(+_Wr?wUdxHVcMqPd7x~l4Gqozh3_IS~PRf#vu78SVhu~)}(tQk`y z^lLl^8mEKGXIyKmMpHD}-i~zzk;Br1iZNV8ZiBDEXsoPl*j&%~kOMB2%2Hx~!Ol!6 z*vNILT)hsyv>!6aipfEugv%?aj zdh>cUqQ=*_!D#TUgZ9%BaT9I+0Jtw4YKlW42=2xzkp>g}B0b@63F<`{G9ba}Rh5;M zB`cPd7E?E}3Nv8}(got7mLwRL6jSIgtqHxY8EXWlJ7L6=96&(V% zWT^t8hg&pd$*Jg_D~&~7RF`Rni-egkH%v-%z(k}OHYPG(u9$4{q)Dm4(u@ETo-Z3F zP~u=b%z%OwnPb^XhHQl%Z5gCG=l9dm+rtLax79L&RS5cn<$)t z%FR}6u0<+2aw^w#yfMqodALz?c$EvXBtxyd+Rz%8x{(>xa3&iD1=6y5p#%{sTNCiu zVIUyjWgYz($p~`hCg8EdE!Shfr*b8Mc*Q)MKqeaoCe9K(Nug#ydDw+YlHwOq@q)_u0O6@5}F1C_T!#W1N>uO-Sk4#_EBzff8eqsX zBk3XsD7Fh=}b2q?9%e)f)x{nb44vZ@i6go0pN)wR>z?z z^#5%T__LADnQl0!rRB|rmE1;v;GY2>M~WOVapuZ~17m5d;f+LRN2Zx3ElrMir02^O zla(3hp+aP7Mt}*=mkpDqP{e3~GMkW)&P5b(OXh-)GMMsFSS-O@vu8ug9|&X;$7GaC z@ouX>Gr31`OEWYB^0TPii{r^%i%TD7T`7Y#v`cZrA})bANfb&F*jK>uxb&s&a50gP zP6nYsXcyDko+LIBj4LXFTjL!GgXu!(bS3o*H{m!vkqe>R7}6zUPRMv7!V(M@37#n% z0_XrnI?gU*gnPmq5z3G`8)9sL+=&U?#4ZzwGNjBAwM@A^aieoG7IJJFr!!L-1{zjO zmMp`;HX9gMv@M;X+_{G$*Qi-$Y^d~A81*#`Vh6vfuCkgo>uF1?s$7b`)KB;+^Ssald1^g?ZwB!}z}`LB-uH4tW@SCg zV$sYvXU{i$^gZC^Kym>gN49WFJz}hFW#&S4%+2xP(>z4*HpG$gB-@T4tfa5Jvn6*y*=;G86+|{v zmDyfw9=afyp2!6)Z&7)t+@2f;LQ-V)#!99fIcg?^;o_Ie#E-$9)nDyiBO1dp0U8F< zQd;sRs!Ta@qM8tLz^@gT`C_hQ%8Y5|Jj2I13#{Q`FDsoB+A{G;;cEiP718A*p|g9Y z5h9)e*f1*~EAByRQJ749awwZna>aG!NQmv8X@r<(05;4D$clRqTNEZ!pB&02lw5IL zQp%}~fikN&$$Gmz@dz<>02wZ#vW6a1m8Hn!CyA^HCI@uos*fgc#{`xc=`5Lsk9sy> z!>oX;cy5@>)F+3s2_*+y!MEImg3OezRO0crmcTVZsCaSfjc@hamwXP8BTb7#xz>ru z3?*bjiY^m`npuy`kds$e#my`)V`jk?F7$w&(q4`&sgtx2&G<15eyPFj(4{kp3q!a` z=4sh--~e_Gc9>H}<~%Nlrpf9!@6A^$B@7N%f8s_{X9un$;8-%SAVmi(w9x?Maje?sc`_;3UI9aA5miyjU1}VH3q{PS2Jd@meXk(pp4YZ49rw+ zrvbC~D+_|GcjkdLJ?+WiEK9T+JQK2PPb;U5FIyrDFx?f%c(6Mix>YY19dded12>Pu z{^lT?U0vwskz>SY+R8l{oN>ShcQOKEf7+oYne55LF(bP)p{qt9gf>) zEFIm*5d(}DG_#v2jqeK2rpWgoso{q)6WEhsJ;#9ok1B~p6CrkafYpg6K8u11D#~D+ zGn{k+*-!T+qAf|f;T4NU!%zvA5Em`FGKkq~bdKfQvgBH{>!Ao0 zl2JN>;*lvE0+=aqQHz}*qjTv7-Gy~&2)AT8BIBAh7nC}JxQR?{k_%$#iHC`v3xI4$ zV#GE@$V`0WNaBK!(xq(rY$x=rO~?j-WHiWV7Qa?%J3Fu9H-+NG0?8O#tC=%jVIGLl*4QiGoPf)jX4QA zE&#GfH~<+puR*!a?v50CBGlRu#ie6&&0QJ zKyuH3%V8vn4m*PZrAcSAogs*-G@=^`42&)$V_N>W!3=QH)jlUfxx_<6?g-eBb0!iq zJLBR^iJ3(j7_9BE%FZPlx6}*cGzpN@Xd7g1F>R!1R2DryW3%Yv4) zU*wO#0;PhQOSHYgP&z8e2S{E&-Z1EtDjZ1xn=;icfjM+$jdYBwL;_g?1CU5V(ixv5 zi&|yj-V3=>leH%BWY`GX3*Kj${ZRW z1=T{cr!-)~Mon#T2`6SiFkCKa#LAoPp|6D2I-U$!!zIxu6L8mz{B#_y|AoVBWFI~T z_F*)41b9AcYH-hK8?&+`G5DE+?CA4`#-T!%*w)hNL=#*=BY-*GWkZ0ChN4w=1E8n-|q zgo(9mxtKv(eiDJzrp+I2F~YDXtSsGXwy&!z)hVW=dTS$$Rl!O-EYZNC8Q)(i)Ep^b z_^Jf9FIY+yQ2OzI@mhmiHQ~(!%d~`Tv6C^Yi}f-E^^Dot{A938UthPCT}XfhyCAG? zr6=cyr%Ui&0UlXkPfNHY64v%aJS(14hAgr%E4ad{hDd1PY^$Rh3*&b%u1e-`g>hW* zOBl!HzeMh=Ws!A7w^CZ-L9(!yOCbU;I{1tFfT_rcZnxU7bWDX^FQ&C14DvMxVMsQB z#>4>(ZW>z{=q=Q2Ni}Jqj(3J#ijz+PW$>Y%IO$r6`NNkFDe z(LG`14;!nnr$-R@29FIe3~`fTH-$N2_)XoQq)ViYbw=f-m9>q=dS7FuFnVe=ufV#o zPFEpIpjJAZJc4ox$%sIxO`O@dOtz8PTyB#O<-juLrpn4XUv<@`!e|~ckWIi67?FfM zX0?awMa;5Iq?77LR>e_YX_-+L!_V8uSly5Q5<{o4wXj~qjkLn(Dno5frw0@=U2dy{ z2>=f7`h}{++ruSVhcoPL4QOxg<3gevM-#_%j>q5kSTL|!KMkO z@L&_vVp*l-iG*a(RO6QyV5}N%dVnalYRX7QV0WasEgp?v^{ZGktf{QoTz?sil-6yo zZ`AxSPz|e=k zUd8aSyA52n=ySE*>M-oeA9xTcn?v#DPTJwZ={cA;v|AP2f*y${37UcdMmM*y0?fsp z=U`pm)?=KRX(dQ@|B-`?eKJ#?+^Q_M!>JmelsB^*H*cz}H7aY?R#w2yu&=z)xU{Nr zo7yiOrEqKYWb)R%BzA19P@|!SR}7;h4DF{uWMdpg#q5sEiOL+N3!SpfM`<{@(cb*;T`v@13RO*e0N$!FZZ!Lp23TdPm@qE+o#ei8AK*CP*4y zwkDFJ2EOQ`+~PDuL`+md0AQwa#kA|zM|l={l7KQfMvC?8x2}Wv*%$$_4QQE*g-PhE zqIf7oPW7Tpn3s-66AUpSma$h5Y(V+cM6(hW3b@P%xr3fFp(n{;G}f=L!`#p25XBh& zYMh?MSjuM%STBam06LH;1Do40vMg3Dg&Jhnk{G0CdaR3HGYOOcG}|}vcP68-U>j<7 z#_*BQwBT$NFVM40@~h&%UNk3BEKkHSyaJ*YZlijdjrB@jef4Fq66~u`2V#XZ{&={% zgdS*t50Fp@eTs_kzc8OJMCBuoaw2q6>o>;mKG+YB*v*!?6RJY$#ACIsHVvH` zG~0ybRl1pc=&Yi$d~-#m;j3R?DB~NcKgSM2Z+i1jNiwxRLfGJ3~1VD}_Pm zmDcd(QfKE%F%KCSTKJ0*p$Pv%cH$lgEzFY@l7W>|W2MccJ(IEJScb;LltvFd=ot&( z$`g9cgk|OL2=UErrV7DSNV;r=Ff1KyDf{vA0hSZo+`B7%*vpu-r5>@Cp*cYEmV#Vx z0G2jUh?K#Y3Rp^BiIJe_2E`ja(fJ+P8-Bqc9ir04LdeC?Z!!&@%W`?qPx7#LO4%j6 zQz|g=&`5)Ls!MGzoIMj#64ip<%qqb{gP<#W8W%6YLVu*FKCOpJuzc2A!_g*;j+7I7 ziu|Pz3I|A|V;T7rCS7Fqhs9=xfCR_Hn6$)9z~j*A>Zv3%qKHF@hpl;Asp%Fx z&``d)2IOwQ>k!7etqqm7F?lD!rfq44Hx@{EXvusmgr6$WK-x_@0P(9^7myYOm13`n zu)S&?@2s$|1Viz0qJROf)H7B-; z9+r_nMbd>lsQ_tF1l^?OQb;h31|Kk?Txd``Jp;`<6DhI?D>^hXk}j1Ily|?%kX1!g zsua1Lsti*+uZg_NO3h=m$`!JAHFATb)(SmMh|?!A+`0hq#?_vQx{(^fP5uSDl9D@F zkHi@jI!uOxMIEu2I6uI4a3YJBm=cdi%H`(nQPzYEn_Ay?J9XT#v0={MC~=Zl3>qV! zE{HY`Al@Xl&lZw&Y`iIDINKn)3}edpMXxK8PIY7UWDSj*VcBtWtvS|-k)$4FCh)f*o9kPnbl{$S9^MIjd;-i%|<#QLU!i`6&n>Cn_F z-$P@^9|Un7%grT|nar6NWX?6enN)>=yO+=(z_=)TdB_zBw}IK7O$7){2^u@}2;qXw zkR*Y+tq9#9D@~gT6jvkZ>o8f4jF z6G^Fx3lMJxOUtC91s%c#Cxekazc(^!dKC{ovF}`nE_*WQxPi&Non3Ngz>$tO@|(D9 zBynX+v)3sNsFdIYkvp2sanL*>3;~y&7qq`;khx5?a_r9V@JL^%*)pln*oEGl`f8!Z z(vmf06#z$?!6!m391cL&ywU`Nb>2a9OUH5GFKj~hMbG}RM~I^>Xk+&1W0tr5nu1VL z2hB(9NOy*NGW#FrRMJ|@ zPctbJ7<{L8TNS-chu8f~^{kRpSy82D%^{ury?PZ_3jw^T%C~>ir|mN&U8`~g6pifk zk$mSFi!*x5P`+Y_yE2{ZpkIVg#RIjaS* z?iWwo?4)IKi*=S$o$&baEozWVZ(EWMfLeso(CJn7B)0*WfXGj3<3b-jdyD7AosB7o zq=FDT!+gJp;1WSPbLfRdvWmsfMWDVoR@9MLZEt(o@T>n(O@XI7v$n))p`jJSk7N<+ zscj$v`4rY-0`wv$wWjrmtj?bVl>)G67T#LcF-lvoXfDv1`U~uC#7lPjyY(gvgKeaf z7A9%fp&wkeqz17iGzU8~huRk_8ZIr>*VV107nvmYPzBPCTBV6*H29_GO@l4f^;>+6 z!-6aGZ4j>_U4YYqDsY?z-NsGUbwPo@W=xi_ zf?dt*tQ(#+#yfAc%;%iqAQ-yk9lj&9Qt^e}rMJ%enYu_|oXah6*;l5!a&i8xOGwI{;#n0yvxmf3b0vhoFK zf;|kJB-dZ3{BIzbc8lXxInu!MD z;sS4Pwk~ob8xA;|JxHD_%&i_uxo$wR4uz_cge^wE!gOKL!QpsZEN++J)^3dU3_ ziw(fE7h$tPcgdi_vxrBQkgy@@t;poZUrd<@!mkB{Gy(|?hv3A4MVbJy5~9^DKbZE1~d0t=7xRNG|!Uocy4X6m7{c113z;+pznjTmi$z z4RJ&*V9+@z7?+?ahxpM5(5{$3Ot}Hft#lw8P;^Cbz{y-i4pc>q zGgLvWVRvYK^;T;NaJxOm)2PFU5VWz!kv*rC&OP_iT(d8gvKHeTE?hw*!2z8_G6+3k zq}9CjPE)&5K_xAuah^7Nsctz`h&0@hJ|o*6pqt3)%*UsjTo@@ME7WeTuTk#PG_avq z!Oe*_3|%O@b}AVxfl#1D0*-C!C-gJtPrq}hw1HdJVRauD06TH%Es$&@UlaJ&yE z>FMY^?P@X`4mzJ{3mH-Dm^4jH z92vKyCl;nz?6qXc7$P{L>}PAF>?om*Rvx~RrnR-1+fiW}W%xA#b|nzXq+w1ND9;d+ z2F?wxuiv~CN@60k9k8=lOr))tLhXsXN(X7kBnh`L!*@lnOG|q!RG=$JXSvoim9?G( zkL-U~lbsWgR8oksH8V6mHrSJU0OL~R@IZ2jk`aQG=FB5`odwwW%4Bk;_{-ktL<|Xv zPh8^UzZ#LIK%~f>?K&1c;D5+6Zi<0y^MHF^uRdri(tG?*pY?#Yv1L>B=50p9dPO;A z1y@wA-MXGPpk*`%YZEelh7(fBwpB154l`(EriYI3CR*u#ZX}E@`?T+(|0B3VOOs;y33Rivmu5;S6HM51+c(Ab8R^J+n8Zi5;Bz+^faR21yZOq&3S3B*)4* z@(-tewye%7@_wYQ9b7vGhv6z8Wrg5byY$?=D07a=tem74OwZ04-&swz#?skJ4WoQ| zuHjaC81Otv;00?~1n~mqP98}R-XUD)BuiM%ffusC@gzqkT!ux43_w)2plvC(idWh` zD{^LrL^F0ensLusY;&nj`a-gI>5T8uzOs zO*&r0{33IPx-uZn^2GgxT&ql_h%8MZ?1tpfE7NB%xrE2YOl7^Xw#wI#wtN$1aG}JB zGbhdnFt=sQ6@P19$chw;4Cpn{O1Tf|akJATCC_BGKnSk~L5#*So$L*WI}xy>f!+DE zrD%GgIW&la;}jSs4vGRK?Mw%-u$mN=Fn1lyGiLHuEjxqeY&@knQ{)u2%;6fegJ@0& zS?>MH%|pUC4bz6Y>Z(R3rpk?N-U@sfVjYHH_jlPLCz6oF1=@=ib@=>~%#Qv$_AzrvV^sSt!*#>-l(rpFc`Ffi*?6rv zmfX6LhAktvP^4uy3WX6QZRaw6;fh&Bv$E1n2qYy_X)@4W3emz@2BQzNs|u8qAJCZ+ z<#^08^#?4_nlV(Qvo#gIg8^8+7PGFvGlLU}ix!Ix$1cFry`zh8AdTSCm!QhusA;l| ze#}8qe&$Rpqs)8yeBw1vR&kuDs&*b$@nks0xl}?o$>?w+R6DfWNG2n797|2MA)o@b zk{W`+b{%J{l!t%#ey{+-yGHgoLm3G~E zYmIuQ*{f7aj^iQK)hKzJda~FvW23LJs@!O-uc~vwv}TzBGkmNb_^*7EItWY1tg00} zO~bY-Y#U@gr&6<+!K*ui>H_Q5)q<_rO-8o{QV!AmfR-~eBcPGG@Ew3Ispcv2a2$OQEQ zWL~Z+4b)lRI&;p{Y0R8eU1fcvTT@^UdRU3pZr)m3;j34vrGVha+R1E|T9yN!N5^oNWZGDayd5ZkVQP*u1sA9OekE5ef0!S_>kp zaOb`JGNYkV9q1_ImaUarS$5YH3LAJJlZLemwj7&G=qhFLgjs3BcQlQL>dhJZ65I34 zmZG7uy0W~{?2~EDE-|8^F4wetw#YR#gUsv!@uuWNQ>I;Jv4R<*Fa`KD$=x-%n29?h zv||v6Vn!s{2oyp)9`tu@wJ zLfbqSxL(Nq$1-=}&QguHZV6i<${$op342*l#EQ*w-hv{G zwWf^s!v>ycbjAT{JDKDqBg_elAadDC`@Xb%4F#}%O^BMp8<=nZCt{e*6K=hg3`29m zum!gjNbD@v!mx20v3@Y@49m_d|4ocWCShUlgm3T~*oTYR0{F#DXv0`4APuDEg zsl$jy67O+$gaReeNVuCeY3RL}c-qlz6bbXW^wbF30uu+i`Pf4t1dqP_kQ7Tp57v-} z>jlh3m_@wKg4n45~Rh^gkz^7w^q;QCwm0w zrE8-s!Ny(PX_|57;|b}S#DK>& zR~T7wkVsgWR?*Fn-WqfzyF(MYXi1l<7^R=GH{gNPCR|Z9hYX=VHPoS_P@~QUL+hHQ z$PsTZYV^LUpq!<@QSlU!T&3l$n{iC}R<>d3l`^VwEONgPXB^djq@04c!gQ z$5k6OHHyF}bqSscRlz?i#IzwtDJ?B7ZNOSz@2lHjRBWxO;|9^TE4j1QNiG(us*Q5! zc9d+{F?Z%;lf7e>5X52ErsbxuoHA-^m=ZO8iV#X+bUnwAua4r@5|Ll~@m|;Gl&-O= z_A)mu6Q;Wwf0qkws9|bquzFW>@$QgL3#K4isiWpq6*eUj$p_g8!AA%hj$CLUInaSh zqYo|GP*qdG^gv#1JEF{KO3gG%FoSWVC21tf_14%ZSt|pC8eg5!P>%D(wmlS&Or!{f zk|LK{XDGH}^ETK`IB5tQeAVj=Je(r;pv0YV%%re*BcHZ5$SQ#m2`*V=m@ob*3AvucM~-tnqYn(+i8Evuh0_j8K~{8ETTJcH4bSrtkzDMuK&W;6*sf0)ZMNim-&^Y5??ydlF*b{qzq=;{TU`)wrrI` zkQBM_n=M~)36T~NflfNjFBYQAJi*D7Xzf_XgDx+lz}_zwNPN+oUKW%&oJ>ME|OYmWkM-^zXvRK*p^FvOHhMF18xPB9yJjanzF5M5KeV7E@RQUceTDoiV;3n;1{wGM@ml*9`)IIE%V%oY4CNcPi8}AWVi~K|Cm8Cc1 zJ=Nw;GU9CO6S&FF*LK5N((oN=GcRo-b_8k27~WYU6965XL-)f*CdLhUn$y|q8eAzd_Ui^d}q!jEHPsHVlAVk!AZafH-H=WeS$9{(p;anIJRx8&L~VCoZQl6mwQjr?X__j;byRi> z#tk`U@DB+@=N{&$VsfwvCf9C|z9zu=511Wc*9mK?w&NVl)&{Km%%_amtxK0$8O?Zn z9OK~rkb*!k8p;$Or5^X6dC33J!j5gLy&!7A7nx2;r!`^d_!k+0s+^Fla zB_!2xI1*LI?bvLqtXjXJv7waH$Dxc#E2mWF&6B2uDb&)_rU$8w0LntJCGl0{bfX~> zxzAEo?h%mB%B&2dnP0iFlDkn^0bvr{%t14~s3jD7?$zf)k0tWcYcG73XBTDZRYh3h zvoW9r)gduS+Zytn9Q&bcXSS0gJTLmB$j?D26wIX&998x>4X*SWXZ6iF3UZP1$V663 zTg?=0!h)Zk4Rfx-9L2adI=DedyC>HO7Mzo62n)lBCigqkN_)G5a>LW?xa#D?v-5-7%Hz^x?n!fi7G0c+843n?$mk^NV1tkqbytaD z%SsX&F0Cw=YAhxupbCx(88EGcFppF)X+`z1)i%A@Ey6A^SvKQ_(oAh7FP#*XR{7k> zlM^f>BFtM><<#|v!+L_H~Pbh0ccS)D>rx@vNRimkeUwL+c%{!~DT&IaF z_d<~>9_q4}$aJ&2SaIyFWx-q=9ig}-Tv((_nOeI|Llx9Uw#%<%g0ip#pmMD+46-G0ZVAFMo}?ZW%B>YWjgl_#Ma8Q2^3 z<8Tho0!<1|y|EA*N+LPQ-mDo!a+1&%+o>l~5(tAS^g8`v3EjH#rZLOiGdPcRli6ll#AMgSn)pFY zBfZkoE30EN(qMLaibmdG+V@kG9kaLm|kwnqRLI`9f&d#P+}en5kt z%1Fb6ANZDF2@>>qvT4&m5#N}t>qM}ox{fQisp%PNFxTu%#Gd3(8x;xs5adV`3u4;7m#eDXZHIZ8`jGKVKhv<31)WLGrI^j{i< zY)T7(O$kRDAI!zIveW_C_M?7AIfdxJnA+CM+5>((UqzkPM4qqyEHu;6m-`Nx@lPv5 zWTB-P){5q1TBs;-DB@0ltAFMSntMj}uV28y*g2_wSH3viem0MfVMnRE8U938iVx#~y&2HowF zG*X$;BgaKaqGL>L-rRt>A<7Y%?0DI-xku40OxL35M3!qrEZ!Ljy2fCv zjJrnc_TyD~wY?-_EW(Z$Lr7_B_FME{8w?rYz@G)Ve^ft zc#y1nDCN6Ft5tD}x7Wx_!FIu1Ve=SQusq5gB6VQNVSMLLmc?S+;IRlhG+t%Q8W9yY zB^1Zp?u+1cW)fjDmy35!w%iYNa#M9)UDy zd%0Gf6oi_2PC~qZu^z{a>wPsgO@WyPPtojNfMK6>8Or!9kL9ql*`*q;6#5olsQH#G zQVYTZvHZ5Duda&puuZu2hHdL$KtA{#4z~KSug&Yt=VSZKWP>1I#bmohsCh}vGNuqk z$wGCG%(HQ)KimoHJ8L?E;2rE82g0GIiwm9M95XeXVNn;1V?P@E(9OZRI6dnT+7(PJ z%pA3#?czc#>M{G!G>iw2$ucw*ygEHN7F)H77i&agiPeR?!uqZ#F5`m_3=zi&RaB^p zKOVY9Gzgven6IO{T?kJm)?g0CJA%pi`jU$;)_wZrmvix~Xi@Q%SI!>^jI|bw#WolM z0X%+=Q~&eItz=7V$vS}%E=9JYPM2S<>~k1cv4EP<2#j#z)O09t<(0+yH4H;~QeIE` z^ow-g0%lvgXwiJV2yluC%!2OX)k?CpNVZlav(j|U*B6OodOspvdO*=Z2(wiGmdPm0s*c%|I=z{DJ z2Oc88Sr4+nzAGqN5$)9+ri;WyNgu#lzt-Xxqv6FxYP}Lv{iITVSsg~41l2kzs}1VN zSm1&QwkWZ7zMk;m6`n>FX0n`eu^uSdhm}m|q_fX_kreGiTU{3ymAj`CG%OY+H8(C` zw)2XTt5=b(qAlyA3oZ3v0%II{tun%` zgGp~;B%j}m)Oem2?0L5_dvj@7__M&a)Xe&KT0*SBpll$`hha<2ml~u>$dKjIDzek> zmWeO)(y~=lR@OCCR&FZA-Nf{)&IN2*c8(|(FK$I*NtBj>omplVNd>uJXdo>SJyb)= zG+W&RWT3aLIW6(lTHH{!Cxv=D-kp|^L511Dbg1pS+!NPtuET{76F`02ckOa3m&aquHU^3CfS=^bQS)e`(+inOZLSJqWJvBm-$0!&&`y5VnwiAy$V ziP)a8Efr}D(vq#kYiPDqSPi8m!fj@^dfUmgU?CrntW`S6p+abpM+pBK8^?8IaZG1wpLgnGBdQQ=wM4aG~yx(d`hN#+g%Ah<{ncoR&baMYr_W z8({H?-!}h$DFV#~a-o)Vk%vn{vqPDYN)w)(Qj!2&I2-jM+azGTB24$)#d=XuSFwJv zUIgzYI{8Hea1cdX3J-k&YN9PXJxbAj1nWgaN>m+7Vl=UuPDx zWfS&L^rcHc-xf4c3!XYRHYu@q^kK=e<%OaivKrt{9Mj)cRnfSCDS(Jrta;FJ7vK!P3Ci+WO7a)kGwtF79vb@E4uijJNPw zVUjZ%!+v2r8V(oHE!N_5i5`=@ zOP9A45`WgCOWW)F73Y!{V?n}_I5@gUtj#EZu9M6v|y zO_WX=panJ|TWI`}(az>JqjM#lD+ORn)D|Kx##AWRk;We<8GK$M-YkXj4Dz+eXlCKb z=LG;p{Fxc;LNzN2Zv^U{;V1|o{>~Sqqn~C*yO^CJBR^rpz?d1A+T`;L8G))H?gg6} zOKllRu-8?wK8r-pYXf-HhuDu|$~jN)1ed5oEqW0px%eW+Gl^m}B3U8gu|Vi@1*r=Z zI9CEH+T?O%zLG@F!e9g^3TXvJbE^@J)9%hSY|TL9Ae)6Z*H^&)ov)#sFNz%D6_pfm z-Ri$-K7^{PdL^>tej+JIYC^_y{vP8`9-xo7CH6WzMMxfu;MX91K$9q7Wi zD$m~wZEnH>-kQ2#OQF2d@KslDE^kmLNxr1fTjD_s3ayy^F)h*|a;f)d!!aBzG3LZn zU>H;JPEW0{;|rTuu>awBr3x?M(4k@sE$I3YvhL256?zd<+|v`8EE?ZLa3{l9PTnFX zfA$owmF{Y#L*957l06HGYIZ``LZK#TR{2g*U0|~ba=Nd&>e5Q1aXqJ`EfBV*KwvR` zF(EY>Gup9}g-xGbLW=|>D5Qi1Sqgn~m1Ge%O}TDsCN7DvJIQ<~*6Ct9SCi@Nnm9VD zKM`ey+Jv?bnlQ929dG11HBu=tGo0p6Q!S;oqq>i^cg$DyS@R5EBjF+5_VV8BSOz<(e200PR~g%R^~3J!8eQeWpdnN zusut|WLFp?8W)J z)-rW@buiLuz3Y(Z`2W~@8?d;lYj1p?GiQbwLLxDVi8e;mSfhzb3`WEl^Fd;w~4Qbk3JiWMtuQK?dkmMT@Ov~o38YO&>3v{6?X^GGUVH7ezfql1ouIu~~ft>Qj_6^nU)Nm^tuQ?wb?=~uG`*N~w z54X^8%&1eCkNJC58>g+=0W36^7Y0tVNgR3Ryqi}wvwoz#H>&ciBBK1pa*pGO z+>U6z0d7XfgPYxMhPQv#y&~amFVOTg2rt;D9t=rVax?w?!@nT*S5bM>H$-@ST zQ3!F>HcNmkWJF_C#a3ZR6wW-;Y>Q@RQ9#-d0=Ao;IhS&z!btIqh|AjTf4W@6hKy3d zva^cmJ7R-{{E@Y!G?yjMhskZImk%nPYO#T^?x zh!$!Uav5A|LrQ~{qjC&w@rX~xIvy^ap?b3B4x-NTmxZM%n0xDw%y*24=WX7ydFw@9Oe}7RykIiMC)=KJyqXz~ z31S;LUl0)!KpNJYw`Om!UXWoMy~Q`4RfZHxwMB2a!@k8HntYt8fG)op-6j-~h*EFl z&+n5D=!C*q8aPZ|8U;sIpR_XNRq|wh%4J*_jRDL!sp&EK`i!b$T1U6q7cphsaQS9j zy^y_aYu?rk7^KB9?PHQxM!gw?l{9xQAp<@Q7X>Q>!X4fH+A%&dGZyi(WqwqMn7p#2 zt7`~pXMsRd#moe4PxF^xDn*zJjTR^F@U6aL+eHi~cWhXjy>=7+2%B8UyWR@qQ{}Q< zD{s8Vf?g-boHIW^7)g>RZb=hFI;%6dQI52+(>?>lfFmdhhU%CWggB_Ny3!?SFg-Bn z!4Y800^(RkA&>FOuu4%}Vj_W_o*>u@{>`VVHMopA=PQ^T|9tftP896y6Mp~;z# zi&Xzc#X|@T?&O#S6FJid-FneFR1)3SKdlZUF(A{^qBldCF}pkxsaSguJf=|?xNRE4 zJSaEIVg^TH@QgR+XL#@yrNtpK)|kRgEJjPH7UkW7S*lBpNZWFPN(@+7rqiu4DZ=7h zo9$KaC`^+vB>8u^kH*a=|Q~m0={8Ysw7YxzM_De z3C%&NsSY_E#i~gDj+!Vt0NF$7SI}%Io~?bEw98nvNSAtl=h^&=~v|d zdV$7ClrY|gSBo=p!-H=bNN1G8mLbJx-nv0Os(G;~9rlil1T$1<+*BE~f+OtGQr>pc zE@LdwA3^SLFQi-_!KHHcglaoB~PhZj4GAcZBcxFT4JWj2%&lEMOk3OfZ;8R2!gbc{D6 z55wC;pxSw8W{x||X#H$*)ZF|0nbPG^E@zpS;w^cm3;5L#n0%r$wF*dByP0EqbrnAL z(Hxm-!SaV`9bsiNu4af-0XD4Q?9;(@`ILZ7bw?^1rO^&w;bBQ+;~kRc#c8*wYueXd zmbWdA1ipXEOgGCix?j&xak-?^wBl0TSf;q#)5(Qzhk)z1SV{FoB`C;$4 z=?><9d^w7Y@zcmtM9BXINic`}o3`a)KgsqDIT(=Ab<21qCbYE4hfHxHmb{KIfXj%2 z=CY?rQ%6j_acLYbBRY(7k9OslVtgxiD?j&O6$DRzV|!0E%pmAtxl}2ixuxqi3Vc{5 zEQA3Ny3vN)NQ4ejht|+ELJ@HU)QGU@VENgZXW)7cT(Az~A~atwS9>{SoT9)Y4i0+Z zT2kv`a1>d-Ms7W%8!ry&3Fc(HB3Tu_vX-)GUPi8$Z{L<1c|oW0fJG8}KUjR{r;jX4 z&$Jvyr@F$r!jzEQ%eHT)t&{PBMT}(V?VaUmP`k7cHf=RnxuyJLakz3h=3202@OEQUhepUpyjbF`-&&{elW~e2o5lF$f-T#(V}d`3 zmZF@)S;C3`*tA$GgMWDt)q`Dz)(CuCdA&yUg=8U*z2g0o{m> zaQ(LJU!WFFte1Dh61$6VJ2=$^DOZyerrpPVl4T12d`!rd;@H$4?9swn1U4k|(MBa* z*4YIt#h}IJ((`Zf^75@a;^vPnS@aR{cu#DbhV>TmVIn^#j!wbJoE=;63Ig5CO;f2C z$MM52YRMBc>$h##xKZ(pXLM119=n*E;&xPQ-+&orbp(*DA2phx(>9^4!~siE&@8wN zeyjIV?>gG3u{C!yty6D=RA6b@o44%}yrE~cJy1JzZ3CvKcm;r0jxbA$t%Fchc-|Bj zi0t$34$sM@A-c5CJ{4>$H}}Wl-bWwyl=?!dExdOJCRavUY%I`we6>tyayG_aA?9Lf zx3{`~#oixsbr$99Nd2+NMrztDBkAiR@Jh}e6PqF!Ad8Km26j54GD=DdsZ3^2#gquD zWh89QZN|i>=BfDL&p(ic8;0a*$GH6zPcy2apAv4W0o+H7 z+ax$@iqMOgyfPX^vnE2LLmU#4=dQwu#N53?v6$X)V6(8Kz!$JaL&q#pQP>ZkwiXQ! zL-}xIw8IbwobjYtsa(3oP&1wt5YkeJfS&y@ZIVJdrx%W24N1M&7$dC$Yi}CgQ^b4Q z=7`Ut6n9`kU|R-WI^Th(6}@svY1-_ue|kjLd>)p#V@aGAws3a~y-p<`Sc{>_Q8iq# zc_3a;KG_zBOHkxq++=FWJ+CTk$i8enim}zqQMd$o8*-^sn{z>&CS2}%lxzJaY;Fiu zBsTG>cN}q$Io_gwY-UVWZdi})i|B2_917>htby3jswh+eUKLRs2E;4;^dUGMo(`a4 zeuNSa?TI$815z3zWEPq&VP7V8%3w1Y-RKp$JY*Atc{eIOHfNqFlB)m_Gry>Qb2Fh$ zJfWP#o9u~YtL(w(?zo(7SFlXSVPnYG=ImVFi4~K)n!2OS;oQdHqnvh*<1(cKYJ868 zHzEc_F~bemwPete=@2IuTs5K0P?j;=S5tIms~0;_2xs_qBteUQvEkVjlHhYO0JZj~fs!Wf%^N=tD42zG2Np{YKaQ>Vi0_R{1C zVdf?<{EnO&u|^)*LO!*8+ebC6h##)VoU1d|g*tJY5}Re^j#177$!905r6=>tTXqid zL_Y9G?1(G=G^b)O{*m%ZewBI7twr^m)6}C`DBOyMK`*9i<Y4G-x47fT?sH$l5PvQ)zD>&W~cE&)rfY$O%woM8^+!faKW3bsyL zu_1}2rzWe?V9Mi_N~$1BomyJ1If=6{t z(!Tis5u%OiOFwPIQs_fVuNXba{8@_A?&&UP#%HfH{tukMZP_5qf?l)h5u)5;P0*jWmUTs5M4(*qt zfhnyN$%kTTiPP+CEosseKi#if1jZKbJ!kpZ-opL(i;)U0QA7lo7%mhc#VVr+PCn^K5(d&s~J5N@<<9REp4=5Sy)W0k|50#g zaLk{btCl$8GS^ZqA(@pNIRiQ;N~q&?UDh23)~3I^bg!Uxji>C91sm2yH)eJJ0T!CW zEm`Uw&Y7%LHMK~`J4sWS6eJfW#0tz8H(-~UG(LBRTKkr@xtH*$H$M9*&hmYgK_vpy zHxDYVg_X7ue)ft)3ZZFYIfd)v8+AXGwC^r}dLj9aC+F4@@X|i@AZi5pDP_S_6daGd zxbpulrjU5bVLs+{t1;eH$ulW2G zp}ya&%aA$pzm}hf?rJ?wW@CYA2hC!}Gr=sGr%6TfZ0W1bE**1&mh**CrHo|^d6$tX zhDct-`ua36@){LvN%X++GdNALS#wBh+EO2+#&OFhPD_D9+}p?{tY{S)&bzwD1m&A1 zH7R2%-n5Ntadro03J%=U9TxJfFm#>tCYkSmf_rcytNcYr`{_q$#A3;+X{lLr`KdZa ztU^>qY%7efO^GFzG{X0t9ihDu9sY3LBQ7QURE<#bmI6r|5UWd(MmU-Dd}Rj)TcPVx z5Xy`D7_G{IO|NKY>T=?9M#%(j|AIk+zalp0vbjekV_N!VYlppR-{D8>Afx!r9et{^ zIFc=jcA>0_FYJfjlcm=%LPkB-SN-|ZCzs$UX(7lKUr;-MfuHgX0|Qdjn*vcx%I84H&;KL0}{Jvj|! zja$b3%ah|(3mjG2p=yCk0iKm8E62r(>UaPjh9GwE14=l;kHHxJCpM+WWU>ISKjTOM zb2?O}P%dSC9Pn4rxq&#9ZZGhc)vF?y_rmb%pfc$9~#%4x}Tw%XzW4M9FG zU0odzc=Ud+z=R+EoI`(B;*WePK3XeAKR0ad6g|v=6jF&XK|^XDmzYCA;pi1wpt-jU zMaP~DQ~4++#*Kw{{eHUZJL1M<-FY|t`I@-FL&dR7Z&{<1*Ds+!;Rb;HqPPI>W4-Z^$@GnDQ!(wg73wkK43QaW z9N+vZ=#i!%H=AwTwqr{u88al+v}C>?yKm^=C;BUk%$Zr>oSMlhwHcmWHlt7r9C*Q{ zEP}QB=^cA4(?_r1nXz*Lx9r@wJp&h0Z7`2k;BLd^Vr?b80JdehC0AIBV?{ju*{gV0 zXN4;Rv8W?j>Jq5P(^+9}mi0W-hU3I9-@M_9TuZjtCm$pXS$xy`4!3C8l4CoVa868i zeTpD4-7#&@p;N+#?IDlYBj-4NhaH<`_8w^uSUOR8G0b6lQeGSjq|vd|P-khiD7Ql+ zhRp%&K~a62)hd!#onXtDJgJ1YMyjC*aq})VPn!0X>c!3O$;MVBdzYh%w8O&w>20=9 ztViOX&U~7=I0FBEA5G~WK7@UgX~lSLB{X87WBL zR5P`aUvqjU9NbKRTJE$#&9a#RqH~1ZJ?ca`U+t%^li-u_n53c?1@iaf3J^N?8&RjB zka)oehi5T}rpvKt?$EptFBC3~9&MvZ5U8$jw;Af^abFZhlv~$t+7@;5FqyJ^I)*J6 zi*Z;;yMtqVSmFTQF14gv|vkM!eUG(PfUc5KG~SF(%Qw` z?%_OIBDG0Y8P1p+w@Jy|q!pjchI72&SeL_ttG=_-qs|Y?5{HR#5)Kn1&xJ&Vj69AL zMT|2L2uB-z3J<1in7BMWmK7H2v=N^jr!*lWIfjGNR*rqWV5qOG?wANM{nU`M zD7KVp*KNE(k}5+yN~jF+i1vdr%vsnB*Hr1^#wJS_j}mYAc4SRB&g~LmP05qfk+x%6 z{HtX!)67$VQP<3bF9N~^Bj#j@vQVHSx$_}S91LxnqwCVF+i#_BuR5wiTRZ3i5wAS_ zYjF(S>l3*b_i&|&xrrlg$Ei5MQ(*sZZhDEEuDN&`RTHEyrAb83s;U#%5tpjiRz{c` z)5RRNiO}vt+Xc-8)osb2p}H-1KUM9RZ z%nq0a3=h)7*M*YpDV38v#uc`0y{70%W?6hN6Kv@rE=2e+4bZS2G~>sZ2ve@mbe37< z!YaD(ke(@EWy9K3T5GDCwwPQrue4>0cXPJ)vRqo0Q%`AG=Woo(3Ok<`t+Q!8uh^iO zV7ALVxF8jaY0ah`D8{p=YGWQpQ-T>n zy$py~sV-MNb! z?8&uSe*~K>PWCyKT8NnI-Q&ZOb5v$6QJ6W7euj}Ih;d|;-hmX^c+w@8ro=c?RIfDa zQ(h=#Am)9?><#NKyNGw#2|NUV|K(|~sAxjcz^snRLa5fGGk}n``BujX{2^9Hhh=(* z(ROsJ5X9t+nCg{2?1Epk0n&hf|NggjttQ513w+rfi^SlZw1GupVoshAeRGYL6GZ>q z1wtH>KS$>-7NUM0xpvM==~^UekIC*|B)S)nOV0wMcaa#8zYi@idO$sv-0O}tx)+JA zW1F?UMWW_&>79#2H^>5!C9hBUnaz$Ua#Z#h%p2& zU9JV~v&4XFVbd%z=33Y@OVlM*3Ne|q06!btbA@Pir{HIYJ5z{JH@Vh$;M(XR*A@?4 zhdksuPOi1dJMnWk8RhJlkxq%sSR+JBN*aFlr!1_UB}P+rfd@0w@pEz}5(2NjTRT(x zFs0h)7}G?z!x#bRar7Y1HLlb)AWg2+R)7vyY70Q0E43M5#Fg3vFy%^Z1ZYf3Z2)LZ zatqO(-lWtzfI3fyRtq?myon<2oZ)T<=$?^{=u-@$)TZ>ggy=td?&Mz(!@RjQ zaO;{kcM4elym}OB=(J)AI+kvX!ENa51)~7tXD>Je(6SFhKL>YQh>XZb$+%FRg4PWJmoB@a4-(dV%zPISs5;zZK5kaG^n1yU*PNFj{1^05liZY2)uA^CexV(c!B{G$MAFc7liF z8f_IM>A$qajqus|Uz(JNKE@{*ufJBilIRoHDtg`5mH)1t(E8V#ACbx}}Ihh+cD_%2?NZ+7b$1cfXc40oZoG2D$IPUzNP~e&WL*W5W^~V{GDn;#mCy znymW42ebtg&)5SLV(I}7->!{lREnO5v;`x8^*`3SG)jM}LrWO|Y6<71C%O){dc$FyaK5K+xfw1t#H^G~#l2Ef|K6=z!>R}pj*g!>TV z6HIUDQuKC$p!YF80!TG8)urW821cGx^qQY)I|-lqsiHSMsodM1)C$ReQ@55f1lZH9 zZR!DR{F!D90=E8KyOQD``Gscm1J?FvONidlqZxgG-96e|!iRwO0uJ|R&<9gJT8fP2 zDb47HYdc^UVAIoDYA5-68f>8KOg^I-9dM}qrM93Qc=Inc*}_Mj(;6M^AT;)Cz0&9K z3!2dehtU_c1+9QJzfnHB04b*4-)OZEu)YB;qYG%mOWNXYz|NO6wEJ%SrD_^`37!Vw zJpQr@-94yHy2yXm>zdJmXnTjX1#Q5Ghn0ucKWN*DKKTdDXa>FZ4Q)XaVDlS_KKO=K zqss)})UqjwkvBCr;f-%;?nZd*cuQ-L9{S&AQF&Wy)yPNFJM06H@ZNW{ObSsqs;we? zY*bs^0DNFfGls@A=inGgF8+N@`b%gZKs?&#;ikhRB)GQNiN9PDJakK&V$)gvdlbl08o95tWr~ElOCuMA@ z=$*T%cA03H?%I z{X=OK^=Mi;^wo*@UVn)m+3s5>-nz2D% zvYRLG0iA1c&&faN^Y1X%4}|cVchs(vn{Ncxgg2BI=Hd9K;P*d8-kOtEZOO|~9#)^U z?DNaEF1y^^CB?yTZTFT6y!d-?2VJn0WfMlNfrxUIIe6l@_zIbVgSoz-DEA#$kEt5m zFTTB`5(neUeYscw^j(3sVY7YMEmKm7g~qIkO4@4?$imH=yze5*=l6pk^481z!Or*j z1&;s&8}T|$>A@^rsr-*1P&Plm5>U&?tvF3g}jFhyec&H8too~ zMzL^U3*MXZALM)ysRGYfD|#T3R{|$M;B_5jw#4VBxjUQ2gRE9$yr_<>htIDNhnCcsen@*Jc zR^(Ym9^*(;nL%8hrkb-QcuvV~Kje%;=Sl{I^1(d>l0bi!!3|An8p>g)p3w_YV6#?G zx^Tc;xzj`nk5J8EZIIa(fua>u?N+^27JG!yUE+fZVI8ZKV^}xJUz~@apsgHbP)OL= zvn9X!iVFW;NFy>P+m?ET4DBtf7y}4FPm{q#37P3&EmamgA(=ohBD>5!uid1C? z)oL!(JjG~rhzzW>qpwgl9_d#pm5VTwSV=OrIj{xVy9E90wjxE^Q0*(IqR!%ql0s5# zh~Gw0S@|kbV++>45Ykq=vbk`?_sK8=-Ksq*{kZ zvz3NDnw@*tB{s6KsgoswvX_Z0sKC)XD<8AiL1~t%0D89wC$m>%3O)IK87_AyB{`Kw z3!aN*r^rg6H`K6ju_ZT4tXaNtO?ph%yJnnl6uu;U4)`M)zBxn(6mUcMJkatcpPnJs%*3b+k=5hd^>+{cDqQ~HKMSAI zMbB4eh>SnzV$)Y=h+RK-i7~(?hergjogq@wJ)#~Ec@Qb$RjoJ)IYALHNOm`zE3~n5 zg|i`3IC?XMv*tXE+VX|AeYX$|KH*%jM>ul!2&cPPIIckf`}PXw_EI5MeO0)xtQF2R z*NLRy4MOzZEVS-#2v^fLg=65GLR|A*arF52g=0yJ5S0%K$Ds#>wyHzS>3Cddy-&dZ zQzEJI8KF%*BOG17M0~#%T1vlg1b-uvntzK~x0i$?`(>p2s&K4%75N(ydiM~*4hhEu zVESvq+3~v2x?UHKQNVG)v|*tw85WM6fV%)20DA!4e-O^PH-y&khH&%%j=drDoDrez z91)Jb5g|7DH0P=!Eos@+nxkj0rfn+2+{yvXRsB^>Uv-V<9=b(y_J32;*EDEzul$y# zb$&~89>VB)s7ceNnlxwd4oz>rL(?Yj(2|;)HIes2%`x>uZQdj%xUy4qeKb{f=FQgi z`q_F?;~ZUUnWMWF&eN0D%-6M?`MPs(zMeFEoGuy`>W=<}x)@%hJGxKM#j=z1q*W*D zqVg17Z#YF4&8O;)?WgI^w$pXT+@*Tbm1pVtHD~Es<5_xg%1Yf)c&_d&&eUB?&eyf# z^L4S{Lfu%E4Y*Nvj&9Vol#6tixL9{&T%v1hF40BacHNPAsjg*Tsyikw)t#Fz)5Vf2 z^pu5P&|QZr5hkFEl>gARhX2qpO{gc8-lU6~MqTf2)WzI8bZ5&Qx~uCh-80pyi=2CO zz5O2Dn7m)t+z;rERS)PhJ0H@uzK3+ZutRsQd05vI)r7!8HzJ@kI{Ny4#_T z?RIG6yB(sm%%KI#9Af-i4sG&V4x|4LhiLx3L+|^(!^pYEq4(b7FoxS5F5@AGmidsw znA?f8`yKk47vS%A4)^fy(YQZwXjgvVFsAw8^*+WhE|hhxT?=Lk}_5qj@ng*v+x2#tXX5|HEZC1z2P3a!qBhTX*g@X zV2JKrMpDae!?nH85biyOtGN;?DQM_@LBrE|z!0T38}6Jtjakc@4UzIg!?EOUL+t#q z;ZEy@zaGQw{w3n;+iCx$5ZKqiFnA2F;Pj9gc4_q`E=T6o(3zDkeaP=}3~J~K9(Em9e`=DWVM&r+cv_M}oSvljEK3qo%afAC zIZ0woR+6j!f+XjH3zPJ*3zNjy#w6D@o01%hHz$eC+9YGu4M|BgHztYJwj`srJt@ik zP*U>nWRi34Ka=#qe@fCC>R+`p3a-cOQ}yN+`^=6=pC<}Ps?*{8e3sx#eM+nH{o zW`$d8UEvm;=ejj_CTQ#3j*j(iW64GCvrz^x?Au6x?6PAx%JE&-J<&zx6$-%#CZq8H@ih)o7>fWzgrYP1iZ`b z8tHb6%wM~Wwqdu{KJ0cb`GecF>&#bB09;0TyM@%jB95r}6+>ZCS22S$KUUHttnUdvM z*n7Umb?5?*zG}5cn_TU2j9%z*&Ry%#*R1t8``39Ksp~yS!|Oec>}*fc$OeztbcM%} zv(qCci#^7Yy@~RgZc+!UM^@#BYJg%t+ zJ=)HPJg!w8o}}(aJkIJ)k1_Hr(tXY&#`-*t;IBNc!hVlf^Ba$&c)%k@hdqv(KX^p% z2>iU|aW=i>(XlGwI3XbPRP^(Z2;(U9^&0i7G{Rt!|4L_# zf5v14rke#!2C>ZZiHvg)E1B|N>1WP5N00ys4kS2`;6Q={2@WJUkl;Xq0|^c!IFR5# zf&&Q-Bsh@ZK!O7a4kS2`;6Q={2@WJUkl;Xq0|^c!IFR5#f&&Q-Bsh@ZK!O7a4kS2` z;6Q={2@ZT#4y4wo)5obbwD%vo>S}18JwdfaFSQ1TvL&vzy`|QuEp@3i*ncnG)vo^3 znq4tPqVTdCD6368o63X5Ch+pmBORd3aaOqBG5hz`pC6G{U^GmIn$M`V}vF{N7 zQft)7^3~`T zYKD6m(w>jGaf}QbkIZg3B1@FkHp-VupF5`;qX5G8Ip&gvdYtdy3uv$nfJR`hs!_mu7kw=SX~42tVA9+)wF$ zpY)$9{4(OSnm(ln#mZx;JZVDX1f%J1oT2DSpNW|8OU_hurT1d#tC+6zSuFhvOt*`_ zoastm#)iM`tm)IAiN}|ye0KS(I}F{iT!ptw{~L#)&p&(m@IPR>>PKSBcgzYEUiBNX z^t+g@`l(pDD?^1p+NOU0<1q9OnXdY!*!XWeXZrZHmDAG?GTkozlyg=1G@JZC#B|j^ z#^&#mOch@Bo3ZqFnXdZFSbEcWD!iTio|&cSC)(uiWv1JOFF9X@SN1_{`tLE_PW}$8 zQsM3B+69Vk$KNKVE4wB({iM|@yd8bs8bw!jOlPD7vyUW82~37%bPy@>dU6j-#bjVHEBjuEuGx^wV}Jx?TV9DAVoiqcihWcsu?5W2W2nFY9)z@OJ!thv|0p=P#H({r8z} z*MHO(s_=I8b0*#>q5i`z{BumV%YSK+3U4PLC+|^oJO2EH>2~s2S**g_>7N-Tif(5= zEZnQ;cJ}oTOQ)xwS*GZA_5ZsDMW1D(zsux!DNWe5|67@Er=P0cQ}HL+#Q)&OicX_R z`WM|l?{TSdR;nV$&|hb|o&3Gebi4XHDoMp}$N$eChJFFl?d<<;Ot(va?_uJT?~epa&Lza9ThWx5^z&S$z^`E!|W zSNcRW+?vaHv0btrrYuV zjgwUWVi*401uA~K_FuqsyZ-5m$Eomk{QKj=>Eplfcty9X-#n(<*$)Ax+v$g8C#v}E z=oc~FPCs77bi4co4ii4}B$d8h{419!y4$9HFJ!u1{Z2C7uK&tCL&a|we$82mZWsPr zOt0e>G-TdmEhY8<(nDCeHRq5OH ze`i%Fx~X4#+$rExDjuRxFw;X&ac>QNMY^Yo%$F#9s@Tb}nqd>eE`~!4CmAkyS%u4F zxRYTu!zPAZ42Kv_GF&jo;Ti5^Sk17BVHd+8hLa2z{Eovj+{v(-VH3kHhC>V|87_E* z!!z8;u$o~L!!CwH3?~^bc$LF5+{v(-VH3kHhC>V|8P<%b@J$Rm81^w7VK~V!^}m(> zB@EXv+{v(#VI9L3hFuH?7>+U&Z>snfFidB-iQz7W)eIXLwlNGqY9LoNnuNF;=#MHs z)7`Wew%fSxvT?7paj&s)Z?JK1v2pLPaqqElAFy#Bv2h=_aTjmI=TEAQdzy`Vx{do9 z8~5!t?z?QUNzAkQ0Dn&@F+8hjY|cJY3Zv#wX`)c1XpAW_wk2ug+Rq3t^4})BN@$jo90nx5M%eN92cLnzw0FOf%YPM@|2NDZ+9yHf z4l@1oHuP$yzs%){=KnQJpJ2X3(`%TX%<=y=s(xyjzMbiBMbYb+ek{}D)gP7fb*8_^ z_4gR#A9B83H>!F(iSzj~*WWhwKf?I;n0^tv{|Do9IeeeQ!}XnswCUgDoQ|reR8ixM ztnW50vc4O1!tLw(iO*2q)F$egUu(Jilgv-bBkh@KVERg?N6SNF-1>-YFO*I+|Gv!p z`vUVTm+@~z@ypzIVYUxd&Q!4}s(my$BKai@!c)cDTrbbvr0Vl6=KB!4|C!UT;_}rp zyn|sYLsLGUVf+P#rX0P?_&7rcw|`SlnEK#kc3;l$3WirP3^M#K!&Zj(G5iz5F^2Q& zRJoQhJfGolT)vHr`x)NB@E(T0VE7utcNo6UP_P`E?IDfvvl(8%a1+CA43B2M7Bc>y z4DVw23x>T6-(k3r>uEEG_i;JSWA{Ra2N~YY@O6fQ<>)Aeix_4yH1)Kpk4?EV<#8|j zt7CWo+qV6Gq3kU8Y-U(4&G>1k&m9fk6vS?oU44v5Mf4{`-IJ|K11gH@9LRX_g}V zCBpyT@$Z1-W_nEoQ-MUglTAqfqTQ(-(m&-cY9xX`)K(M!5*$cyAi;qI2NE1ea3H~f z1P2lvNN^y*fdmH<97u2=!GQz^5*$cyAi;qI2NE1ea3H~f1P2lvNN^y*fdmH<97u2= z!GQz^5*$cyAi;qI2NE1ea3H~f1P2lvNN^y*fdmH<97u2=!GQz^5*$cy;Qu=uNJCbN z@omB(@@hajgJ%A7nm+*P3?H4-qkr<;?;4j7J#-FH{iARpcX`UsbYCDe#I50R6m2KI zYJ7L%YsdF2z7c$r_~vRt%fxpTz97Dv@ZE{;9(&2q=J%SGpj%&k7LLbAo*eZsE8E-}CsA zJ;JdIUxx>G0VE4!89uzL=)51_5WaunTQ);Dcj3De-;4OB@GVFY&NcY1#P@A{kKpUa zH-#^Irf?SHtHakeQyBlmw|bUveg$6(zE@@m&A_WyUT?OqD8H&SxTC7vms3%YU+RO~ z*4*_Ks65|h7uWd)T7DH_58lolc0qWqH5NS8fT`F3DFgG%z(|6X2PF*n5IJx_FW7^^moHxl z7ghgdRpsi3%6DZ&prjhH749z~jxy>)^d);xtV&gk%3v}5K-~oW-a=m>=&wLp5w0XC zQBIVBeH8)JR(>GhEk`cR5Cs*Gz}^0e{KA6#fF+o8446DLF=-0tOL9lf9!SZ~Pb6Mjh>Eo;eOvO&Au>oIingUHn7_LedH_PYCx&V| zZ^_G9FBzMAFc9>Wu};bM?Jp@nULv9-uN7sLNOD_cu%x0K4Iv&KoZntj=_@TM_htJ6 z1^$xCAZeCpuN$k%3n&)oD>RMr;N|(HRlYb;LT8l}`pOFq=K6xR@j>mOu|#KhYcBI( zt>2#y*^S6Th?rd>^O5g|53jdk_tgd7N}oSa0eLJ59`x={7s^pehF6Ka^p#(Qg886R z0>14P6{XuNP(N`AQguXAcKCu-{&HDr4(2O|a`xvFQz+jAnNo^IbA>N|FT&)5tq@5W zTUlUFuCF`_%dJB0&t@MLl|gU0?|=pK`vR3{h=II{Tx7jKR*j5|OTu2!gCL~{WNo;( zcJ76-g$fXlAZ!&RgEArMLZx9Gx~ikG|&fZ|@e(&yzieMn< z&#zRb7mO;e_-!^2LyX>E!GzN{RHIgcdDI|7EUmt_N z?Jv&5-?2yL3E?P4#}+U>-{vSQsfM=SEzODuQj^1Vu)9aV5J%2pN7g6lO8lOy|`dW=&-`P`^0}4 z`^0yg$k(&Prj|cb(zSJ9lkU$iN0o8De&OUFzl7hhTJl?m5K+F~HhulYDI8=mg0YId z5sZDpNsLua;@3_xj!Ft5Jc|E8GE<|-hO5w5P*DicX?4Je^-)#XR@k2%<*n*#z2gUu@*aGHWAZg#;If<0j$b3X|Lg`Mhx=&+(ftwqZo3(80_yo_ zG~RslOfXn~>MjX-;ov2Mh>SsKB`|@kv)wSfMUNYG|IyJ1J8~OIO#8!ll@pSS_Ll)) zm1whOx*xiU)jse>61hgF-2CHOk0JVQrOgYRRMte2hR&9AI1^?3s&Xl&>!hm(=8H^%wR zWZ{tV1D#Z^>Zb_Ndc{-8zDhFEGrXvKQdQ_VAZOy0WHNp*5b}yq%G!5G74_agnq>Hw zN8xYZ=xqEQ{VV-#f0R^I^ZlvF(cb+WePt%fhJMdv%wK2tD-MM8&BP2u@yJXeoCPqa z5afUtLoQewUkL>mo>7E25Fcz@sL?Rt%^9i*_`Un`LppGD#y(2gRFLAIGlCVBd*yh| z%)yURgkG8NCrAI@iv3>jt4NwvKJUJea(ZRP{tUE=6yflR#u+7kh#A;|MghyvM^+?; z1{KsLnz0Vd2#7moqM7~>17Fd0E&c6kz{PEPrZ?U6^wCTzgNglnrd8>^KXaeBZk8b? z9-*M!H`Cv#=BNmMI?EbC&n#;MowN3dXW`-%Kb(d7dJA#2K1FS*^XDGTn;$q>ULb0Y zg0SpHg9%pbg+>0sEVLwWzKoifxE{T4T_@FQ*XyMC`X{Ij4vxDuD(oJTWbt$O8T}h> zaM44-dXG}V;uo(V_WVFGRTi2S)TKpJcO3=O&nt%E1v&zCDHArEhY}I8BE4aef8?lA z@jlXPY$AE@dxg3_%6uWQe>dV5&!$$R*DS=atpYV8l>rGax555YNa?RgV*4J)+>kDP zMK;rGX9o%(oPo&3TsPYglN~1`U`x-*2&j~+_SGMYI3~YCe_QUs zYrhWe9oWw;bNEN&oc(Bmp%m_%1A)F?hW!Uc&*{!NSV@AUuz@+$h5Av*mN`N33PLo& zFLcGBaEPgh5ZBH{Ufv=<&98?2w9K`}de2;xwi06xHN%0{DeC41MLV3i|NiS7vq$Wm zi_Qg^@sU7RqPxJ57x{@SV(-yK;vYwwU8i^*d1!t|npeZb*&@j~E>i0}{&D|2LyX@} zIc;g6oT{Xm_`K&Ci`$+l$=e?gT)ZtNsiBxA}OxL>|f(U zl(*)4C~u0-J;zx2^DO*yevhc=dqWCI^dGZd6+WZ~IkSi0f1sc8IQVo-NgC&~uRG^M z4?IJ@#$>yRDDO{AUp@1M4t<%98lNAOnG(+;;OJvAJ20nUG41t=k@@?Tt*V;1SvOD- zoZ9;cH1=J|x1St`G~X3N;*k1}6Q1xGM2yds!x>W2a=7Oe9rNUfBWz4bQ&+q)&n%0$ z`*9P&+T|C9N;UO4uElE?LPh+Jyr^a_>J~EPN1!wyFqk(Iih^msFg^Veo?4y<*+G$u z2NrTHzcxJ_f`{54P%Pv4Yqh7YJDwxE`FL=+jfm7+iCd3XamewNrT=P!x500aX(K7H zy1n9|<4HclJg|8G4aNG8G$h>INXhrzL@CIioNnzR{&CYHl)j{VZ*he`)OCGwyc*g; z!D5=)D;gHTFt|}N6cM43{1+9vAiok5Cq6H-J$VUfx|NB**z%4>^+0imPbQoOs6 z6odF37ggN1C^EG-kXq+!WXuizoJt&{qnQfW?|fSlfvh75fdW}>ciw6H31G1DT1RBK{<^dw=U>YF#wn!rp5Z zhYP|^*Dv0yCL2W4Vr#)3UMv&j&V)TbZl?P)q}wi2Y=45X*r9%d5yIfq%TUNzZrpbY zC&w}V_>?%wJa!6GpFBl4XgWmPezNQ}pIIzTdGXXKkT)@O(mv7fd5oJd9<8V-!?KF( z_2f9s99RcnfWy2Me>hPZk)rMMus>fXxz87md|p--q)*Kddjoq*D#e@dG1)~5V`_wK z`2jhq9Jm1C@NDk}Jw)YQk{r1sD^PD2)S*pXRBt_t$~0rB3cfvU<%q^bBR-bX^Rze5P_ z6XjHWK)iD*25IOpKSBaix6zQep%+>N#feJ6S9#@|R!2 zAEK8MBU5AtPJ9eI&=FyKb}X})ct2St^K8xlRqV%N6QoRZFGH(%kt%oOU5cmqeX=Jx zlb38fvI^01 zCd5OE${6N-AjlCDIO4@KD|wv>%gzBB`c&-29Md2|jXp&JHu^FRWJyPe+Osel3T~AY z9h8kw+;kS~w0@dyDiaN7shKD>!=a|4i)b8)i8Cy$-4Cx5|BzD?OF2bW=L?z0BN%{M(8wKzL+Ec2^bd z!7}cH>D-`q9}Hqa>`{bh|Gh(_%rSvY*eB9q8o*YPvuGfTd(+i~+RN#}X?7vMKXXqh zY^jd3lqwgE>1gq^uqA$w4$Xwfo2AV<^bSeS>48w7&6X&4e zY;>$o(R7TIP4VrsE2>Hhy)>tXIUP*F6o%}4@izRn-bku{LJsEpZXvU%eS+q3BbwP= zD>(MMSAZqg&VaAFNvhV^VTyQWg{&zl30x&ojr@v&%4(g0xIpqS$F>KyD$V*-l?TNq zD|q5lp|x1#8Td8n!1jNU7}Y`5)A%k$^X{pvLKv*=KLJ1W^<+q@gaYCh8RnQp{5C^4 zcZ=VwC=fSglvkjI27G=wb*WbF#e*5>8lmH`Q3Ax`ieNd-W<81skR|adqHg>^7W8^b zxbL6zd+=uZ-FqF9bgCg*K?OQKEQ7V51MSIEYLA=)g-l(Q=sL$-IQ;oJDy;*e_Z)Pj zw5sM6f6Y+#k&=t&L7)L48JOrS#C$CFTT+u^vf^zZWSELXBmADrk~VynHSOH;5HmKVCQ&b@duTHDX&3+D9)* zC#`6~yZG0+v?1i{nP}%P5_Lo-INT3ENou>nPtH|s{kd}sD#Ux|7Ks}(sfgmcnPiBH zA7rX+BKIPSx|>L0)P3O8(3FDWk8o=Jn53r$G6dyXfw}O5;(ricnh5^@z42$1Y7>km zsH8)(=fy@TKlTK~i{~p&ihrIjl37v5?aB^iT3QcnN+`ih zJj_7xuS^SPgIGjR@^kBX#TC$mTv1q+E)SHzzxd91CHv0yil@($gJ$v3d1ci_l^I^8 zlWCfwv??g>&9WL{;t7Y4++AZ!~O*`Nq zVb86$mhFYra^J|ytM_6T54v9nNF^32ed4v%0WokPWJCPv0*om?M%uMMlW9LgMKcY( zD23OvMs@Vht`SG)i*LD2Ma^p(54j7(?a76rccxF=epHe8b}DQDG5)z?@%9PW)G?k$ zy0`1ZGBNsjUJ6CWtPIo8B313FmEzXZ_K6Qp^NT0W42b7f2E}7*)B>dFUjvbUOh-M` z{D}4nU@FZg-dU?gC*r-eVBix(?ZC!Rm|^m7!%xX)A<8e;+{CTxkg0Q4i0`aZHE8Z6 zxOpAgmA|3@U6ohdwT|Yu)#~s2Yvopi>(*6@N7p$-4-8lgRm~+lah*%zbv962$gCQsQ@I3!^~1uhOZ_mgzKP2WYS2-Pyt=w=Ibq5J@ql&Q~Bib94_dO&`_@ z0~s)@*Iixd+v624t}mlqJ8!P9gs~`oLfcS!j&s*lq^N)HRxP7j8~@yxF|@E_PHB(p zkS1M%{jNG~?D&?W4`GMxfP2z0?8t1NKjF@7PHj)lT-BiW=@YXWjUiW?r`=Jn)$1Kv z?Ti6!)YWDj!XDu%XP=fipf7W$yXC)>8Yh0L|D?kv>42Xps}`qJ*4(I1XyZwZjw`Eo zu30#e+@3U{cO>;{&8JLg!DiQ*l#~vAA}M7-b>^CsR_B z48uL8w;CyRTAx0mjcA>YRSm|7Hf}WNokm){(dih{OD7!zM%r9sGNnP^RIB$m=cc60 z?MP}(9(D|9izl4J+Rl^#XD{|hq;zQGdM!odm~ix+Rqw2BaMl`AMx(11`-*E^L+&9* zpJUPquDDhWx$CtickiqLeOJFbHDl7{P8rdRu{ zUgI$)W>0AyGY@&1-L=l)1uaR77aLP^4WrG|mfSXfG^zPG_u!0y8Ov&C<}7PUY11=? zXN=A5G7crT&#WF!?Ma?EZYXuY7&hEp?$H$YU{bBS!P7QlFsaT}JG0TfuwLt3TCYta z``fFFhn;ICoI|NC3mUXqy?a)l)~t6px|}VpUT3Ei6o z#L??$b~m6*lg?4Ftl!;t>_}2wa=)v^J?dC9aZ*?E&XIGcX1E)X_LyhP)oOIOMqFdY zl>5-4j25Hu^Yza5*>%o#qxG0t*Vw#9XWgpIS|`djd+7La=di2BHRT#jnoP=QNot$X z?;b;i)z0Xik=Z_XXT5vCIpth)$d%EPoZjeJJer)*=gA!LWDKX|3?99!V^+?fvt!1Z z(V5xp?gsZnMwiyEb3L4uv1Xcz2t^ z&t!bOSK;eT|IaD>a>lQGUg7yB{Wl8tGu|zVEhHf(-{8;<8^+VMMC?ScRrWr zKNWs9sOBi3(q44V&A2%r+8DF?e;ZHI?#rW?TFFi}qKVrP&Y=x&@pz<@o z`1y=CtW@+bGv2&P;oo39XT8GjWqfCj!k=dRnoAY_2IFh4Q1~R{3wJ7f-fES8-EM_% zWW2ag;pL1k{ttzJgYk|Ug|{)@^>u|m%XrSs3V(y~0meUJeDHQfKmJ0Me*1S6zLxRQ zW`$qHc+U?MUe5Rw->t?+!t zdj=GKGvh;y{{;9Vtw!v^zGae!0j4)FuB}z^_hRrw^i0NsuPfZkc*Y+Teku{Yl}cu2bkV$8E<3!0mkd!Q}mZid|ct5Fy8%PKeM;e9V|?r=oclw19gJ5Vt?<_vKQv$A4)oj9{@OmL@Ut1;MCS|<{wl_I zovQFU##5Fk{65B8&rmqo4YEBiSNNaJ@T(R6Z^qYLsBr4{$$wXl!Z$HK>Q(q&#?vbm z{td>LH7NWc#@!Dp{8h#m{#4V!#GcHnikntYI zn;7rfr0Bn3Jh)lmJ%{mD#&>H%@U%`Pj?&;&g@I!uZaO z3V)9A$uBAV6O&$|@N;ui{Ng%=A7p&&eudx7c;llAe~Iyij}-nfQL&{h741@yu|>(2wPaMROZ9obuJ`M8&ih>F zoRPlYKfd4l!C}txyszhdUH5hWzVGY4Zs%Lb6IVL#AU{C<33>G@kB@Kl@lE?%=kv&u zMx9?zo_M442zf2}W8|H0^Z2*O8{h8y82R{3&M&^g^V@&3^L%p0I_H(-CAT==L!M{X z>oNWN$-8fN{vG*&nf83K#h{3GO1@>j@G_*U}lzj6Ksx#c3~$H=3XIY0j@AO8cFJ6}XzJkNQU zJZ--77V?o+=YLgvk@K&}JFj+r`a&On$2HE=$@>>OFCmXCalVgQqdMsm{|JpUu__xS1L{U3F{ zfPC~$=PSs~cR63D^!GTwi@f(!&i9bFeA@Xz^6}3)|AyTBRp*JIltQ& z=OdHJA16;C?_v6BAx~J~{IsP$z6Io2gZn|zP+`^l3&?Yy1b{8{Jkkf(pn`S;}6c2^tI^W5b=zK!I|$s>067V%#4@h>{x zM&4mhry>4z^7L;wA162ObADl$kAH-GC3)Jvd;A9ScJgNOmhXA|Yvj=b96x!|51nUp z`}nt#yUA03?(q+jcaa|?kNcI!zeAo$K1iNR{v~+{`7!bU`IPHC|Fz_a+2l=%B_VN#sr;ta&RavZzAs_ zZzKPhyqkQ4yr29>@+0I4E4;p%j(GXc$areTFDFl6`Z*b2m2n$+64Tpr_-TEaMjj!z zkl#sOMBYg5BHu|KA#W$|C;tn1!moXL_L7^)50Ph*A0f{rKSo|c9^d2T9Uwo4yp}wL zypeo9`EGIxc^A2rypOzue3*P)#;Y^FiF}mlZ_W548E?z@3*@Fpy?!3Z`1=_j&iHra z3G~PHdi{u>nQ=3D64SpVcrWX7W49E#!O1ZRDSjN6C+oH$(NCLk*(pa zFaP9^lAHeE`G1DoOn#W$LO!p==_g-J9wpn;iD~?s$zLMxB0okxNIrAe(_3#0>3HO7 zax?j6atrzMQASN^T=RbHwu-CBK}!nY@U+i@b__ko*C1(~~^^&yt(TKO(n~ zo8IF2wUMVATW1&6jgR|n2E##5%Oem z*J+;qLh?cK8^}k=L*$9i@bov5cRk1XbL3;>Uy@Hd-Q#D!)$%{Z+ArSll(c-joILl< z&XEB0gBY%Osm3)M};fbF9xi|Xw$Dic7 zXRh_}Z6LRjZzaFY_-W5=YFc-yPv1832J$ZO8Fqj6DIWhOc`x}<@?r9`-r?z+CV2W8 z;J>i`bjfqclgY#6X7cUispPMMCpR@&^VPKc>?b!(c0MFN;QSl$lBcKilWuaqY4$S` zzf`<)QR4aJV~3qrh)=#c#n+NopXq#u_;Tle5ntu}5P1#xjKf#^OSH#?FK*<=IDEC&MEgzn;zoXq!&m!GwD*KBmY**_F%DnX#~!rn zUGT;7r)Tp&#^LLF+IQ%SBRxZ@3BRxb-$!9Zsf-}d|lsrjJ~*$ zALH-r(CC&Cvu@?#vnt~Z`VU);!#arnAEiR+cfU);!#arnBP ziR+v2#f|(Jhp+3OxE=~$+{lk{__|(->!$juy#f|(Jhp+3sxc-a$#f|(Jhp+3yxLyoj+{lk{_`05q>&x)Pjr-s#d$HN!P&-ec^4qw;naeW@XxRD>@@OAwj*Yn|v<>&j47>BRx z{kZ-QU);!#arnAFfcpjT#f|(Jhp+nyxW52j+{lk{_`3gq`w{TPjrBR>Gq_&^U);!#arnBQgZn%1#f|(Jhp+oTL;m7MevHG{{WIJz zgD-C6$2ff5Ps9B+_+t6_`XA%)b^i_b)~4!*dNALAWJ#G&2Je>(*B z_bnOUVvO_mzskN+U{$l07h2sRb+J7;Q^6#a82Ys>peES>Y@W);1`LCxh zmcKL0KgQwjqJJNKvHT-X^$~zk{uqa^=LzwAA?ja@^w|FQJh9r3ark=v5YHpR7t3G# zv}!-b;p=%tJiiEEEPvan)qae_*Yk~d-Vwf7zO9yN0Jmmw_<9}^&qu-+BR!_S<)oyS zub;#?{JnOc4r#Y({*&b&a=q*;J>u}U+kH2D{{elm{CxR~arlc%?mtXlEZ-(84dB+0 z9t?jy{k`-keWkBa&iBR%rB$6wL_ zh9Beb$J>1%#Gb|c#rO@rt@ddE!;f+J8|eQpr(Z1pk!<;mark;(7SGS3{9^gd+439X z@Hf5J%l~WUFP48OD}Ri`AAO(ue>%_e7t5cSt-mo2U(fgAd0&)YEPuhc>iUau_<9}~ z&j-U7%Rh!MHsDWd7KeZQgC0A~@{5rk^X~w@*nmIqV;uheO&*)X`V%8P{8{$FAV0?8 zpLUza+USer-~E(oKgQwf`DZ*2otEFNK0f)Stp1c9aro0e>FK93f3f_${4ox{i+&4z zvHZ#7tIHqb@YjCY^M9s&NBPC_r#-3Kk8$|hKJWh3^u_Y?^2a#*!}Mp;7t5dR*_Y>E zjKiODpXa}dzF7Wg_+kV8v?2}DgW=D#4II9I0I|UM4a?75e6az4;43{C{>c3vd(;@d z7;*Ua_(mGQ@M9eQk=^d&`FZ$aq=!E-TYqC5{BR^4ba~Kd@<4^|E_HQ$2ffThk*VO;EUx?&6YpKaex0D zo3?m=9~|%RgX8^uaJ;_{j`#P$@%}zI-roo7{yxgrf&VycvmJeh!2Ujv@qS|*|6i0H z{~vhB>vrH=pMJ5X{}9IsM*YV)>TlPN-2V`LvHabcALH=X9(4Z(n^~xTv3z@cISpXs zALH=Ve+BxpfG7UCCJ<>0b}t&fSbo0!j`5T~{fFp}lI1VJ7aQ;g z<&Sardw=S&+t2p=#Ym6x+hJWAu}y9|-yffiISy=O5$n)qe>36M-*o3TK1=kq_t;j8}^^ydO!EI(iVVjTX||MKM*{lCB$%g^(Varo2eqrVvVV)=Rg zF%G|tKKhq|FP5L@ALH=5=%YUx_+t5a{xJ@JBYpH=179pZ&p*cDZ>Ep_Zs3dM=lRDt z{Jr$iKMs7c{5<~{hrgdb`qP0gZsf-}{Dbt-{|jKf!dLFhjSzF7X;Z2gaM`08Ki6ZFOM^YuT*;j2F)^iPER z#q#s{ALH=Ve-ZjKf-jbzFMlzfrvI0|{-gh+H2q}xdH(7j332%99|`>>!5263V;sKv zQ$qhr@WqY%7>BR^m(U**d~qW`#^I~KCiLF~Uo1c0{>M1{X{@iU&v#!eKR^GBarj;I zPqp(r^go zvHa1sJ_0bxALH=VKNkAS!t{%g9@Bo*_44$`IQ)?z@2_g&U%D@remVYb0*nmGMe~iO#TI2qwSbt)KQ2r(M!5}}z;V+^8J^Et#t6eYqF%Ewpee1>` z)t^}Y;>?e6_|tCi@>@3!$rsCS$^00H-$TFSd8|LOe5=+pfLlX)F#L5)|Na-cFGd{o z*JkHz7{Ksj9R6YUr-uI3u>6RT9{%0g{EuVFOWvB4M1FX4*~_=Ego9RB!sc>R5m zg=j&gL!&m=w=&ugGSboW~FXtcQ@YSCk`nQ8GmOnYGzZi$VX|qrNr9A%=%Wui@ zk8$`1=&$DXTP#1{e#SWb!}Qq}7<&Sarvzh*3`eOO{_Akca zZ>PWGFMRsN@@J*oEBVJb{72}&n$s_q-;((;4u2Z^TeNP2QvHkN=k*uk@YP=<`ftSi z6U*O{oj=AneD&{${vP3r%JK2;WuaHk8$|we-iysB7d>`tu_t}VB{a;@YP=>`mcm9MtbCb9A9j}ANVm2f7U;H z`QOR?uNdj!&s$;t*&sj0;jcZ@`v=@S)$xM1`eOOxP(qsDb^Pd=pe~|ul-2aQ^kKl_9 z_=Ego9R8BCz5Ffo#Ym6*r)1L~KZWHN%Reof z|1l0<{qdrIUgR&9f6%k9l%M_eA`V~u_o6>v_+t43S^hB&U;X`}|6llG`T70R7>7SH z-KYN~9lw(0=j{)a9&z~UUl{!jBY&~{=4|?79KQM^M*qa{#quX)(;wsT4_E<__TQf5 zzF2;~{KPnX^?!{1kdePw{$hNw0e@PvIQ+w}@ccjWWcS5LkL54le#bccMYG)RrZ1MC zx4(#S`14-r{*Cm-@)uNDVV)+x3UGZZa z{$Bd$aQPF-=D`j5x-i{&5i?92Jb zIDGXlkN)Q2i{&rQ{1}H{QvdXvezE*~`HgY-qx4_O=@-l2n@xX=!(VW%*MB#Cv3%P+ zqygL-(u3jO`bPKPPG5{T=3U7?7{Ksj9RBz>xj&!2Sbl!}ImY3yrhglKvHZOLVjTWj z`k!R|i{)=irLUAf#^KK_c=>;Ps?UG1`~iHi0e@OUdNBMgZSGq)OKJHLBaZqThc7nZ z5BwO1zqQ@{@38!0q=$ddJ{aW3IDEYi0q;k^@+&U;_!9gWhp+c1;C%}4#qy74evHFk z(&^=YnEP+B{5_{v=O5$n^}YtYzXAD+<*(0{-x!Cl_dDQy5AenE^XpGB4u9&6UjA2d z{)y#p$nuYI_q(MnW9E-tU3;eW3he`TOz32K;Hw;_&r8kTd_reKFEw{oR8vHsBBZ7>8f_ z2QU9ySbj0m!#|eQUyQ@o`$zCT5|m$D_VL9A{6YRP4!`5Wp8sF-`mGr0k^hpc{$m`z z-gkocpCEs+d^@d60~qz(Wz!$y@MrDt<>y@bV)>_e_T~Iz9R6JT zU*q|YSpLk+k8$`T^ndd_FTYs+>TLOqarj5?^YWj{@{8r?*AHVH{&D)Hvps*Y{QUSO z#^I0O>G?mCzF7XjZ2DsyzTW?HL5t@vmcKLeV;ugL`#t|(zR-QK{QUSi#^H~C!Tm|R zek_(hEz3W~;onVvFQ;EDe|qM}IQ#?jcTDy2i{>neFo5C5IDEa|>0K@Ei{-C_Y=it5 zhp+cReWt~IvHZOLV;sKTAGMdhSpH1guVDbA{4ow+@0-H=r?CAiv!%!M=g03c4qxx5 znnho1q2%wv7aQ;g`NueXz0V5ow?h76`BR-W#Ro< z@Wt{sG)y;{viJthp+dm;eBiH#Ym6*kJ$$U7=DbypZfD$}Y6bkiS^|oRoVd{}_iq{v`K5$Na_eAIO%!7>B>_N$x+L`HSV} z`=1zxKjCEe-^BdI@(*U)&lrcV_u1k7c9?##{Iyy6V;sKTe~0(s!57OfW%VE9@b$ht zygv`VSpLqFd{P-)z;p_c;C)tcf{$l%@{I=}+eT>5& zf40~EiC&mVc|20|U4EDvoUyQ@o`v~!V zLQKC{{v>>{0e_HxjKkOa3-LZf_+q3-{&(940~mgc!`J%`*U=Zt-i{($u@{e)&M_%sp|6;a(70VyU_Wv;szm4y2#QPjm z{gLJ8`+ucJ9R6v1-{X6kzgT{L{1xNy^}a~FKN9(i<&S3d7vu1EUF_3eWd36L)XViB z%OgT7ck#(@Eh{9_#c_yPCt zpf7IZ$2k0H^zUT-iRH8U%lXGRe7!#v?^DI}i{+0zITgR`$2k1EZ}9S;!~Dha^Za8R z{xKk-)2|E1jjises!YW4h!aro2!(fw(hezE*~{~zP< zXVHHSuRn?9@5|~h#^F!+nCJhNGkyBS^7m$bjKe=*4djvbV6*#T`T6yi7>BR-1K-2? z7t3FpZ9iii{`z}8|EHYg`HSVx^Xf0JKQRt}KmChYezE+#{4oxH+;-1@8-20-Nm==0 z9KPO%jQ1mB`4h{ZiZ3?cPiq#3ulFb8eai5~NRMq_e*H4W;jiZVm)A7AFP48GoBuHm zf6yivh7!l!`J(s|H$%-<dzimien(dR7>B>)JMO=Q zzF7X<+5C%f_~XCt{zRKj)W2B%YMTcbz^z#v{wV!#GJmoBCE4*yjKe?hW6ysk_y1z~ zE!p(PIQ+@BfkWC`xcw5#FJ;@W7>B>%x9*?8?XOt=;;j5J4*&2U+<)CE{C&`~=!@kq$^00Hf9n(7e?NV({CxWxb zhV)?ghv>I*`z1yk`=9mp!2pIIcHX`h!^hZksO{{`~{gGB*_-!NZ-@xe?%g>)bigEY{-s1kp=!@ldWYZtx@LO(le;oH8V)=W#`pfe##^H~D zoBLDgi{(F(O@EBTKf2capK$*#mT%32tLA@B>*eeUmJ{fp%<&dML-@Q>f>{=@Xe@(<&S4fxX<(u3h2`hfcmwkM(0e?{b7>D2Uargg*z8L9I z{@wUu1OC8|aro0e;eIcDG19}&ufN7P{CWT6{{NvbmY;9`V;ueg`d_9mmY*O0#5nw} zZJz&s(-+G>mMuRq4!`YQ_s_iC=f7C~;mnV5_;Www{v7&Z`QtJ_#^KM~;r=rEV)=Rd zu^5NHai{yUx%`Rc@5%PxF`lOXU)}Ge-$s^yFkAm)9Dd7#?mtXlEPs4<{u1Nx=YHG$ zcc}iz^7H|CyXcGMH;=P~Hh^10dNBO$-*f-#^u>r{`Omi>F%Eyz58R(* z`QZ3VEWgFZfdP#CV;ueg`j0YyvHa3Al3w;>9R7k|dj7w#`Gow%@@HlBALH=XJ?j2@ z>5Jtr_Uy~~$2k0h^v`&$=P#Cj(5}m10JnzpVEDV9_M{YB%KXKMqyCR)<&SarJM9J~ z(%w&BEPuSkFo2PNjKkke|HG5K{9^g}`Dcv7-#^LoUrApqKR^GDark#ncK`Ld|3Q}j zNVfbcJ>u{;p5^{UT>ixJ?e(?s9((r(Z08YqtExIQ(1hb^oLE#q#%O=MOOszvo}vUuH8at$$?s*4(nH z{*)eZ_)EUxKKjXlFGd{mKYxBD#^LYS*zjFVdY&}8#V)<*czv(yb-$Y+5e`n^$ zIQ+%`r)Lr)Ask7>B>}q?1$K&Sw5%`HNHSSL#2;;ZGdr z{=4Xl)@?`hF&H5M1Kc3D17>EDJQ{CUk z9^7s%g^tB z$2k0Pr+fY-`eOOhG4VElTSIy<{F(Hd=ll92MjZ3cF0Z8l3_r%<@A(VQ|HWMY#qu!@ z4Dw?f{v-5n)chmM@5q3_r%b{^_hgvHUq% z{xJ@}gZ@Syzlh~e%hsP5hri)`(OaWk8$`9(BDB{EdTCgSNs@&ox#yI>*qwb$aUo8K4R{j`=Kk-KQUr%2we<1T?9R6DRBlN}cXW@$t_|qEF zgW>Odo9F*A`eMW}{|?v(0~mgc!{7LJ_rFD7EI&W~h;jJGZ*u=J`eONc{xJ@J|IO~d z_zItYV)^S+=_}=rarhnU+@DWhEdN;M$2j~Yx46HOzF7XD%#U&SE$?)HD%U@;eA}$0 z0o)qWgW=Dke-HB)BaZoZ%=YUT!0=-n{+tha{*QA1FP1+F*#`MB4u3cOV{E@Hmftk4 z+K=(%-|qSEXZd%N6_5 zq%W4=Czd}BUu?ji){q_yfA7oO zf93*Tf5eDm`B`lr3}E;%4u9^;-M^5&SpJe^SNs@Huv z{Fcm*armS3KSEzDe}Cr3IQ+%4JpZrI7t6QX`e^{ShV)?go9X|Kz8G=Te}{cAfZ@kD z{Bak0{%2n4^G_^)d$KEjjKkkS|8n|b`FZ(c9DdU)J^wZI#qwuo$8RwXfA-(Fzm>jN zet!Kp#^JYI_Z zugae+e?y*sh{NA`wfn!KFP6X6i(j7p7>D0+jr&hu==Cp_U&_uuVjTYd#qLk1FP6V0 zyM7tt@JE)ozl6S6{{C@30pQk<9t?j`yZblO7bA}OzdQ3|9R8$E_wS`ImcJqMV;uh0 z>)ih#eX;y~nIGfuNBZ4AWs%pvSbl#0H^$+2mE8YJ`eONc`^OlEzva#DFQG4%e_FQv ziE;SbN8P`fzF7YHZ2Dsye#<-Dzn8vPesku>IQ+I--2Wkcv3z^HF%96>kRA+w?|S!7 zy4vesjQAA$?97(`7>B>!UL1q8XVVwU@5=lbhkx|F?$4$#mY?6hjB)rQx4Qo(`eONW zvixHl{;m(We+zxF{Cxe7ariU;!To#ai{&?G^&jK#7j1I?+w{fq^XCs@9R8Bq-Tys( zvHU~X^v5{-mXEl9#x*|w#quX-<&W`{{~hlC75yb-`T6tDF`n|j%l&rcPnKWG^AB$dV)>ghKgQwDwHGfT?H>AK`I|C7#^E3OwEGX!7t24G zO@EBT-~BoFPkEisKe7C^_+kV8w1)KHRQ~(ie-_Ul=8_S|`oHgT`_BgXF%Ez7$>URO z2J;uoÎIQ&JYxL>3%mY*L##yI><^f%BK%g@h0VjTWN>v#ZZKSf_GA9a8M+#1q@ z;m@M~P5NTQ;pWHxF%JKzi6ua@1if3pSNF#arpbsbN}P?#qzgh>rafsU-uIC z_t6*2-yX%=6{UCU;7I8FQYG(-<0_= zp8N~l?~qTHe+*x2z@Jw8Z*7RfA9X`6FP5LLzcCK~5dCc|zgT{L{X53tAD!d*zmxfk<>&dwIQ&VMxc^!DV)@gp4l#gR zLwYd$x%4~eixEfO`SEv*!=HS)=YJu6vHbk`)fk7rc7gk)XZidS%g^V3jKe=d|3mb} z^7H;TV;p|hBG3Pe^u_W|8*i0h1GqJ$2g6@&4`}22e?HgCFUD`Ezr6iYjKgm(y8i~| zFP6V0JAa6A_`9!j|M$=I{KfL~{*PiD{*nRrU&;K%@~wGh8o;d~JsAFmVfUX+UyL}W zpVe2czZi$Vk^XmBezAPCtHS_B{xJ^!)*C$kJL!w%=lRDt{7v*HHGBPu%YJtMCdT1! zu?M%1cEKd~#qy72^%vu*{5QG(veVtal`Owx@@F%?2^M5znPl)B`=MOOs z|H%8@Z+e5*zgYgdRQr|sk8${`Kj1$4nZf;cvHZOKRE)zv{vr1trZ1MCKmQfu@b`bz z{nJ@~vHbk}E5_j;z0>^*=!@k~%Z@){9Deg%?q5M)EPqF~{>FGJ|2^(sr}C5K=g%MQ z4{`W=Kjr?r=!@m&&kw{n{4Jk${~r2c`8ChaF+Jk&kAK$v2kDFD-<{Q8jKkl0pZk4m zzbTf#I%~fci%z-zgYgP?D#Fl;V<6f{={p2`4!90pWln|l>gV=pFzKwEPrh_ z{V@)I%Y*JOpf8rcIxByS!=Jp@{Stk#{QUSo#^G=Nw)?l!7t7DD|HpVL|99R08rR<~ zWchjflNg7;dcXUNwf!K=-(jKe?tU+!;V|50N3Tz$*? zpBRV#2>soxKe7CgZ2KAG@VkEQ`G23jSpK+d{>M1{o=4pO1AVdld0GB34u1#z=h%rR zw%=m;i?Zc6#^Fym;`zUdzF2-f{V@)I9sMr)V)^;?>llY$`mN{xLHc6(qxfP2{B?88SXzwUo3xd zcKjCO@F$$^{;%naQ4 z{}_iq?M(NF>5Ju`miaNB@_)YjcPM|d{FZF}ogL!vN6vQtoAkx<=VbZEIQ(5x+<%n5 zSpG;h|6)9q|2+4fSMd6uO_smk>JS6C)&7fd_!D2`{(Smk`T6-zjKiNy|K0S(jr5CirF%Ewk{XO)>^7G^O7>9q_bf5lT&=<>J{G3#M<@$?p_+6K}e_or{zgYeu z*UNs4!=HDV``6GH%in=7HsDWdNDqcT?ltbOZ1eoZh-3X}&X&Izhd<@D?%zgVEZ;6m zrU8unV;uen{Ri4Sf3f_Xu9wRnU{T)(HG0VHS=Q}{!aQ+ z+P(Z@`5QAo#^En&_581*FP1+o^J5(TDE$x77t0^b{1}JdbcN@CKYg+M3HV|I{B=Psrye|=JhX@pFckv~sGI^u_WwXZgoC{Lw-8|3F_XKYxEkjKkkH>a2j1-dd+Cei=l2g{9R8e9_di2lEdM~Z{KYu@=C`~5 z5Ph-y{QND(;g@c5|H(zKf3f^WvixHle%meXzm&dM{(*7T%WsUspZy;97tj~W-;r%U zVjTX_zjuEneX;!f{#T5{e_*5gAEYmqZ_Q)U0B#NG!SLHQyT6OR81X6gIgT$j;1B#5 zhri?_?*E9s80q2f%g*0o9RBo=yMIcj&p)yJZ8i=JVB{a;@DI{|5q+`zt@vUC{=koM z`0GC5`Cmp~jPxk~t=aMu<0SpM;oZ6r2;k$;TCpY&<>-%npGe|(mIjKgpK ztoz&Pi{?SiW6WNCUVvqzA*_NdLLZeg2CP zNBtkd7aQ;gevHE(x!?1@oW2<8;g4qJk8$|N?Zu%;+e=?8eECew>-5F)?ebEUALHklywf6Mng|IPHp^7H*qjKd#2;QrU> zi{%#(arnFF ze~`Xd{+#UiN%0&1wyEiyCz=18C#QM)Ir1^`FUhB!?(v)6YRf9?0)M`k@t=&(c{U;w z()gx~I?p9vLEcNgo;>LZp8kF0Hu4w9TggYr8=mOtpL?T^fBZ?#uO=@b50E#K?;u-) zsZ_pik!O+bZ>pX?dz`ST**+Su8EO{T-|I3yC*${J{I<1LUTdSU?o`iz8+ikH7x)a@ z9zDh5-z4uPKT1AKe%3oYebWR_KLh+1wmg&Pk|&dg$<5^3$y3Q+1y62jY9a3@H%<2O z9}*vM{*8Fa(^Gv;y2<^f+0RJ)Qt{43iRY7#9d=$JKKbetUrSznrt=-*%bov4e3kP< zlWvm#2cMIB>tH5Hu90rI6uMW zcL)CCfaT@slkEP^c(1Q>$g?=Vv3w!Ei*YPZ;L{k#@&j%oWBCMcWcr7|PdLr~_kG)= ziLpO*ex{Y>>GoOsY3G^b!=HD)n!Js?iag~$kAHwX(=I^S*CzCjgYl{V8{C%!tG{_X zCks~pI%u~B*87ssKM`2{523#*u==Ay|65@7uZ8}g!0HbQ{bPaE{}uXM1FJtZ^xp9=0ujQ%w-KJ}l8{xZSp4-@^rg4Mt4F0%T&MgN$HtN+VWPjT-0r7s_6k$0?nLRwxf zCr|sBGw!d$*Zp&Q>>%;98$52!uoLfFMSu@?ldcPRnw}H6cr*SP={X1K; zpcGes&3Hcn((C;LcAGe{-e-XKcOkC#bK!mRV7)K?mE@^6dVO9?o@EyZELqcA$a+8f zZRDNr@VFhO`uO<1`yZ2M7Ce58yrs<)fz=-o`ac4z|0DE=0#<(~xIYfo{cqgw2J3z{+W&ynz6b4bQhK(} z!Tou#?!TkG4Os1I(ElG;{rjOmAh7!TLH|Tx^*@CEe!%MQ2mMEY)j!0~$m;I_{WBo0 z{uj{S1z7!Ap#Kc8`o}-YUjzMnfYpBo`U^?v*?$Q79|5a>5%kBDnEf-MeR$%^$$-^g4EnbLtN$AG*9BI8T8xRKMehSrSu2A|0=wH7p(W|qCXk1`invTG+^~lgZ7GG zwMRtzN3hyAqCF>A?KRQ960G) zzytoGA*UkO<6BfMO-yvAv-tPz2`}^>|NU+`qiE|6E-YGghHyzdXJ_xTmba}Rm>@jflY^}ehfgj`spC| z@c8WA^*%+s{}A<| z_Zyy1-rwxye;s)X*AMiUh4ktV>mSK_KLy?=g1Fun@}K0%U-I!i^K6fg-|2iQc@yiS zL_W^?zMZ`Hr@nsdCExu8_n+_rAD`Z@jrUd>z9h^qy$|WPWWB%aoT(ny`;M+8>-|Ms;sWk=K!bOYR~+`#djSiRGC|9%1}S@~z|&dGSIY|1IRH%>R?*wJhIv$XnYz{bBO) z5BmK16M6J~&XZo`<=MZM34jGyq5RGxDD8P4VMT#&`*WIR9PH)Om#nU#`kBuC*$up zm&fq#&>1>wT%BW<6{}0JR?niIlnV9o|^F`886JZE#uW0Z_4@xbC-QS zVV}G0^GW;MW1oMr&o=vf%0B;WpL^}I-9DeT&u8rOS^Mm;&*$uOpM7@P2iH@-V4pAA zXP14xWS@Vr&zJ49+df~h&sXj9HT(RleZFp=Z`kJn`#fl$Z`$YI?6cQC-?GoQ?eiV` z?6c2z?ep*U`49W-x6k+N^L_jLz&;1;^F#YQWS<||=b(N5(>_1853cDRvd>TL^E3NA zY@fsS`7itY+&;gs&m;EvrG0*7pZ~Vc5&QhwK9AbxH}*MdpWoVNy?x$oAIyUd_GvU9 zHd^c-?DPNC{P?`(J=Q$=|4V(o!^*kNKJT;-mhHFNXVgA7+UITddAogXvd_)-`G9>I zElYbW_U9~5%}s?XF1q~U!o~Amy>MY+N@32zne&UK_MR(eUb1?iI9OWLzO<*>KJIWN`i^i(Y*qdDTx>H@|vGm1m*1t595BTsB-P7P^Y< z9Yrfg$I#3|N3l3CR4lHTS!genihZT-{=UM}{^7okp*-H*+rGS5=xJZmKU^v-9qzV< zx7iq~QgjX$t-7kS@B{FcCRjW6qaI|Q^lWvGT6TA1cmOt4x6bRg+C{gzW%;t?ZZRuJ%tm`$7cC}&GnxC zRfX$k7G7Hz>girq9AnyJ`dN>kE{&5_wCvAHUNiH`bBRT`eio~;bH>Y0V9ZM0bt1)3 z;u9%EX<&@pwOIGI53MNAO{}G3Ei95S7d`D@d!MbLjZ0H&Jbv1tQhTX;85giWFPl1Q zFNlmwxX?@Ny{s&PWmyD43+u^i(LN4NH#psfh=oM3=a<4 z9-%N)vI*_2)CaeJw!i2rq>mIZ@XSvvtHdHDW2fO>0+l=bJ z&VH?#{0`dz4X!Bk+JF5kZ2{`*Ogr~t!8R#*Rdo!loQY*@z$#=_p|jgE=)R%0MpLrR zfmwwWD+|{Tx7*&fQ6k^44;HR(uU9b3?0OqTdGChrSOG90|n7`o~H!^f*}zVMdIZhowLz*JfgCvdye) zgHQ~=7TSl_^ew9_gxJ>DxRRz7+$*|$xYS>7+3)OWAL_z@uVv{+HIY$Ud1aBZZz{!_qLaMhkNQyPzYh2$jzZ; zIIJDyA!S#XrJiYnjgAGw@vt4!+XfD+x9uA2AZW0?z$2(S6&E~T2GRmxYfW}|TAhU_ zeVjDIcH}#D5*-ivvDAr|w2t>@XSC9`d5<@l)%pB6S(MfcVAC^ZC2H*4?+gv}*xq{i zVE?df7XM$Ee}2w?R%VT-ES+Rw8?vn3PM8Xfmys-Cz3kIzQ{~7d9Vz-!a-ym0rSlpe z^u15r!!Io6N|iPbh5o+U9b-+ZF*YKF6?X8~Xs3-4RmB_5Jge5Uew25j^``Io6*%Xy z5rzSW zDH#uEu>cOVcMqn6PF_&(Jo1v}!Wi-TCop3OjY1ckUsGW$M!Jw^OQJSSwbumdCv1>Y z8eOV~MhS2R$iuzffdSinbrgFFE4z!evs4*p#tDo#ohcVr6rE10+ z(+a=5TK5b+P6pGBHLAz0yp#!d@?(vtQ7oNc|9NvDDUVb3l;%XrR`&$o7Xw>w{B}%X zXhnCp(8nvoVsHQ88eE*Lo`AuAFQW2c^vaV(XQ6nVpVs=wK`@vUzN&-G+jJbtB8uG#mx*bB#R?CaM`p z!)xd@CkM6an1}emPjf^U(EC>w2M4XjrlZJX}c&nlA zG&DT7(1^Nh?dvEk$My2zf%;b>9%v?P8`+s^!KlbHzkk| zrw3_^R_&*s?Y5HDVRz@6l%h6)pT+oPZohTfiK|FvEXcQNdaRCTF5A9L(rxxFuJurNYf)9sp#-9OhY1pGKOWJl8J9)wLt7Rqi- zq~uDf?U!$1mJW9;FNTfu7)iO^Te_wcH@!nX>pS{arHkJ@#Ho7?nn|^k46W*3-ZShA zi%)*tBa|_$v_3c3OFy2LFXv;VZ;-UtuIt5Vs7=;C+|yCOT_C%rZkM||$``G|?CM^g zZo6pR*UhtZ7cI_l?5w(NkDI2seW>@vUQ3bQ>Jzp%lZf6>}{l|?i&@{55Utxpsh-A=mIqiTPzoL&oIpyi>s2A8ZxCu0OuOgQHQ3)*b-?0@3U;vB zAMRenU$xRz#?NTB0pqW>6xz2^n2qH?oKfLNKLGB>RhgPR(sgA!K&}mx+y7t>!?9H_ znfwtO!y0|nV0S6(Ic!T@-l>kg$9%k$wiJhDCZ&$Ezc@J5ZMTN)*m$7d?lOlZcZ{U@ z@pDS*m+R9JK*??l*USu?pss$~Y;sX7*}ih9+cJ(-VQIRrKOfih2CZpChh01GSyQn4 zPu6&$PBM=k$~{F~7RD^>t(zuHr7q0$tqObTR299e zI|r<(S=w{r8fDLLK9$#J=QF;~u3N8-lG<99)k&U`N0s&Nc&1|J>El^M-K8N4y>`dY z8k+HRD>iYc8Mh@}**@4^@78om>s#kgv$p(XlZH)-C0L%%lj`kSkF{g*yD#+@hK8x@ zRyCo{Hq+(Ps(camDJ`yE)>VCAYXweWo79FuUhCsH za2!}Y`mW&;){yEShld8R_tGJT9fIQ4ekD!)21)rbm8VL3URS+^B~Map6V9Sk{p{BU zV){yh%|;D5`MWJXgZ<0wpu*0B*t|532m4sY?Igw;=%>T|p8n-%&W zZWrmh?K!YNNC7WL`ijdX{54%Uu|hZ5{;=Dsalp?1Ig0d^W59PG0lvAfZ@3=f;!rzU zfZ=z1E3E1+b=f`)dye^w=3SgdwQ!B?Cwk`;JKKkQN^^=h_R8A*BB^act$ME*K&vBL zS*w~DGj3cPQwn-5jRZC!Y&P0IXqgrAqMb>aNsbgins$l#oAZ+$gbJ8Gs_ zSVEhunvAbliY8h$KUbxWt)E*L$|!IKT{YZSoY#-LJWYi{n$Bf|wrg8hvP*pz*`bv+ z;wxO;x61A{6s!V=dy%Yg@ujK{&*@^@Ee>5(>?~TlILz&WHAG%9R9aE!Xdg5cKiCBg z%)Ejv%DD1pJI@H609-s8wBn@i#olEDYcLv6|I+K073_ZF5L&3&d2nInbo^|y+pb4i znzZe!{$=^FZ4&$^Ytubw`-y@z(6XD>IKTH17JCP5RJ>YMDD~J$Q-6QSw&pmUx3y|D zHj?e7{_dv2Qe5n6vb27*X{(QAY^TkoZd8b2+Gcj!nLC;@r=}GJtH-ozc2xgtM>6RW zL-ntA@hvs2tp3%v@m`Ql=F+*Wuhi32u)0na+L>BK8Cz*=q3^V7RaW1f1GtQ7wc2UT zlku~a#cIRX?@qL9H^3j8s7~y->`$rKgYn`Ie&vCtjkefl&4=vofStkm$y~3kBi5?a zTEf}cNoN6D7OxoFPAtPd8GmumY=3S0ufMi)L8GHS94`0zvei4>=U>t+PGhkhGb-MH zuvTO|pIBf`vljH*xa$V2#aM0Rs-oQ%>`U7N28(^R1pD<=tXSCiFoF%<>x%6w%(1N` zmd=z<@6hsvwxg-0g~PTip3mR<2TFxL9H^)CAUjMM=(jeti~1L)lX@)oDOf%;A|a=8 z2=MC4v{4cJ6KOu#1Cc6N@&#?XrcjFINM8ZjHl)LEE+2D%5W?zv@??sZ~GW$l&_^AzOWH z>rv>lS*x&}irJ$9gEmW++3|Mv`%t=IlKtpwO4;YCCobapS;e@nKkWxu=w92;thOB< zHgyvGH!Yj`6$^V);d*PuY-O@6e3>7z{Y6J%O?MFwe%Lgn)y4j-{*R-jB9$*XkG4`*CKyA757QM_cJ@ z$N35#qOJnTrz|I@xN?Th!zJni@c4JRu?aQ&{^)!{Au-56sIVHwZM>d^! zNF3*j3TORdY#@dOX|cm1>!j}8nrz4%%^ zBDN^CN^G|E!1h*etPR)_dr^wLvQDtS({@ny60e%k*{@fmSUoFUO{gzjgv-|T96Pya zRJ9#>C^P4!hF;R%lK%HiU$ z9gYvJ7_+2t#ARcQ9Qu6H#vEs>rx{}`tVt}2V~&aSShL7h*UuPBVfBNM^%uk}P2+iz zB^-OUWQniG++byAYf6^VMw@1Y5@*$zrB36moy%D&jS~sS9$!|MgS6*6 zu`TmWV|up0j$zOQzt?Wb^IWwO)DEobPNr`vyKQ$JTR8Zo)^z*J6ZCZB*{bZ~DEA1y z5Ebm?12-`3q|BbhSz`|fp-qGxgJ>gNH+4E1X_O{x#&Kc2lFW{2aB>uWLA7wgOUv=C z)Qv}wYAeBZUv++kj%EpHsF~Mkx+>Esn9hQ9y(v8s73M}G$*-^#a7KwXE7)Vr2o>cA z2o1|)M|5~_EiYz1QXBCFcGY^&T07dwy0x~bR9{uw7F*4%N^sI;M{tGm+-mT2i|={u zAZ4XJT~ryPwY^rWuIyfNy5v^>j+cch7y9zsVODo`u2|%AW}#AXovmRo%`04r!;)BC z{nkFnYRwnIO2Pd;uh%6H8+zN?LXjQlXyNsA}j5`aO8D2Xy-q;Pn4>wQXVe$VO*71_|Xn?d#yo= zodgcBxlf)95nQEwoGp-VUD7>SZo$U7HLQgyUB1j_arKtX$7=^swh^>ufOv*Jtynk@ zwWBjz2<_>=>4hQtKi;UTH=_$N)=X>ljc;>dQG)%^=SgLH>KU*IcVSiU$$XMBr zcZ0sFnsqK(e;$S#t|zd>vvmU>x;~-i3In*lTGPs@PSXQVxV)!-DNd8?m)4ijs&)mxOotl3mlr%jj48>FlqV@h4gwvwO+&AHNPdOFv@)tP$9 zGT&;l&Jj!2+P;4KnzUQC9S)EDxFN1y(PK;Hp5pTMWwlSgD@oc{p-yWWPh%&-!!Ps< zQhV4nMr)?(nS?{T;g`>>)QcU;Vw8h321?$7vC%CC8jX=xhjA08a5CcR4hd?GJY zRh5??O{vT#9@$ne5d6{V2N#rv3TTYl6I(c1F|S`5&WUh1t>YpdCgB=atV=E+X1=ys ztV}$UU}S?`)~P|5buomUhaG6*NZxkrc(Np2?5-IPj9}f7f*<4M6bTVL>As$Z# zm|6K;p?(;LXAMlX{7Ko@W^0GpSkiE4tRmXB54Z8^R2YuCl^V~G`wiFjfw0hoB4)hu^)ZbY` ze?ypzN{TQA-j=c6laqA{Q#F#>_)u4Or!6n3Z;?u7xLf5tKk?BB+p_z1zj}?SO^4;Y zUP|Vi)=NASlR6Hv-9>t)w30ECc}o{-%xT9W-d93{B$WcBwbmLkS31A3ZjDwJLkqd8 zaaTm{vEX^ZvAGBbn`3J=-P4?KL2(|O^`mowB`#0m~dFer39Lmu!jZ9wemO!Iry^x>DNe>|IrlSm_aLJAA=Hl6%#MqQ3o4$AX-P zsUMQ+Y^dl^HJ!&5ajP09Je8z)%DJ4z4ug8E$)6oQ*_J0<%&3?BK-C_(Iz>EI_QIwc zewAFSll$|TI0>*@h}y?jC-Jdh^O>6T;en*Flcr>K&TOz&*~S+QeRT4gb}Q+)aqLl| z`x|rYN{{U#aSI9ks8y6+bdkq7SHM%t1;1lge}ZBaWeu#Vr@SViwSKfGCGFZv2ku*k z4W%!#c0ngTyIyzU=9;YsInpib%7mn!Z5;KkZ+i1ZEHqs5`i7RzP(@Zo(K{nG)-1Qv ztU30#!vdZoc*}|E%K{el(}8rf6prTWjiP!!Ng!2;SzzZdH z1nhV4pkrE+8Ykl@(UQd~{=^b@*>zrf%DvnAn!*u{U#qE8ga&1{Ek2%n3)LN0V!Lu} zokUp@Tcj&VsuwC>omhm*1|TH&M-+U&fD`$2)vvhH7XM13s%?j|9c#;xYRBj73cDR^ zj|?`RcGhd16(4)LN+CF5RdcJRf|#hhsB~L*ay5z4LmTO-Sv&snTUFJkk6dkR>Ga)8 zdM0fd;Zi&rQFqNQ_Y-MQmz>%G}=@IK0iHn0aD6YCsNdu^8A^F z?y!TXGcjmg9VXl^(_sY}d-|iR7&Sg5T=iL^wGm8QHXjg^SY(Bmet zYpr7xM@!9EX;P7G&W=4vzK}FZ!#O@yHh6@qZb>n#teXno@`MB7@{Ga*&YTrPc8Fw~ z#dMfnDGRqVIwbeaXL+lWrmtofrroU_nBv(6?9&U&@ML1`MVBmTZGyDrv13x(d$$iv z&kjH;cC8q$^aoT-+=rX8xJH1>5H+P+YELvb400QRc1aZxMPse{ z1fee;dqTx}Qb|v_`E}q*`to!9b|iu&*<8OXX@`c^A6|I+$gWRW1JClWd<KuSZ~JK}2?F0Hb*Yre2FOp_X3 zrinp$Vw17{=pkYF44jFgQc9xVH;nh54 z5^hBE?gg%PTfc(wMgSMvnj_e{D|fvuhbQz~e3jOFnyP9e-AJ~Fs`~8lO^7c#{+FOiy!^{sqQ#JW@&Po+lkJb}fBX_%^WXJSl-ktKaRW}~T9&Ji$T^?a; zhFgu*ealuCBsEdBBh&V!>F~7Ch&|cZ6JH()D=92h)p@3b)_(`~B4f5+@hP_{u`%Kq zbTr+}FC15o+6}v|Zx<_(GA)gETfWhB*$&4VN~Q~;xE>!H?o^N16ONsRGu3YlT0yO^ z_;3VUJ-&FJp6(~=^t{sEqgn`e{?OhcJ(ZhQqOARgJz0Tk-(AHXYl6;LaaQ4i^KEw6 zxV`Ch{c!!Y#iu;o9BovKKH+IH%11A{?2sM+s(*_D&j_Rqt1YEEgR(hXd(Wd1w*8u) ziC2nZr$Ia(@Z-};;;Nf@>dmEc@2%3JYN+;-gpHNAxvDSA;CJhqEH$oHec5le3##<3 z?IZSS&KGb?TXSoqy(3kYU0_dFBYZ}wFMf@<(!vb;*v7YNQajw5)JK!NZdKj?9WbC_Re%Eao9^4JBhjEaRkJPiSy(d;bT}?wdKPjrYMUcAB z#Cd;V`SM;|O6VAlcg=X{hT|aJ6{;*_42F9Py48_7WqYtp;^Qlr}!N@Sb`Ix3$@p)|%W>B9B$wS@R3*DHFe;p-Zo+MTi|OSH^_B zK}bG;t$598NR|kv%#WMMmx~7JPP9j(EaRm;D;kV2PC`xb(|tlazRbJ#$F-44O{7$) zaV|%G7Fj2a-+~z1lAtGV%@<=*#>emtHUu2p8cou}A_D!Y82E}Pg^rA`l^W2Vnn*P?F?guV(epFUO2;HV^z(@E5_vqDme@6T)6|gk*WiMcIy+QKX&Yf zE4Fze+iRuf6Cqd5EiF~`uF9kqN_OwLGCNLK!h>bn2UIT?mLaawhvpCEgQ@zLK~zFc zb*158%`T)mBOaghwGRxTB~$exh)x3R;Y)kg@Fmz+zr=P~6|ie<6}TY6%n0^$S0$Nk z{wrWlb5&qggbO3gu=co>bTg{TFr%svGpb54qpBD)s>+d`DXmOaRY_)46=geKmBR&9 z(}0#QmAby5dK#+Ac|lcKE~qNY1yyCapsFnYx3uekr}}&U*OpCL5h7(?l1(;c%a-gF z*(+qP?3IW48lM}8?2-?9fJ-Of|!2QMrr!^N`OK`*M0++`HS3hpJHsglVnhP$E3r=fp zcs=3dalzFH;#2oqjo>73!|MVs0Vo9;rF$5AeBqjsnxwU{Dm2vmc!mlDC=)@JnyM79 z${}U|91k!Zh(h94p<#^t$0Pxn;=mRfifwZpxahM4g5OS(C_+I85M(GHo%2*o91&zg zju2vp1{QdoEG^X!K^x%H0UL+4u@vx5nZe2O;G*GqM< zxQ2!RvH(zfNJxt0QD6yN2}kx3_(T{O9$Xr5kbsneu&!`<5Ovh70lWoZ;xYlkuMYtf z9FA27!t*mE{-{>~iI8{=@xN(|kOm@D;?PE@3V0?QZwKHi1G1|-sRSgT|5!b3U8U@8 ztY8)8hb^^-v6+rGhp7;o1SIwx@Gv{ff?V&P6bi+CJ61f9Cgv)66tiz_22ma8@s1|^ z03qNp=fDqj5I<2E3?c5YpPLw~K!L;H$~K0gOZ8;Z^?h>{9l=XZ+Ps(z;d zz!}Lw)KLj=>^NQ#z|0^D^DaNe2!ZQTJj6(WlLo}wpR5aHT79sq9mnbbU}ykX#9{^jBCLr$WCQ@M3ft5o?)@<9 z6<>rc7z>|(wA4UgEC9){7|9Cqbx-tDEIZ?o`(D!3mfJyss9y?q-5#+<|dyb2S6aY^EYj^~7 zf0S^TamZ+ZxK99rV0a2AHH4)fP85G;KOBWe0|tSHh{h>GvfoA+Kg8FP|8G)(J%t=3 zwi-c6lD{uOf6j*n<)5s#Gyq#ySz zXfz|p|2cM4PM#zHg<(4z6JzM90RcN~S-l2Wt!yE4fV6}BVSKuxDJ)A8`rdAGChu|4ZzzGv^7_U~errNC7Z3mM7K*N9cYT z{*xSQ7r?gnw=Mz8hOL{xh4{&>`ELYAX9k3H5XV+EAmqamGWS4_!N(N&NdVskf%{$) zz*;M>0@NG^4tPONT0uosR6$%)4*=l1t|+1Mrq6*lbA+Tdmlp!~B zGdBakV+R5V^gGxQ7?`7hU=Jv+7osd63eesF0UL16fgpY&Nwd3V~w49$s)9S)Kt806-bo>i~d#*bxlT>(~V@ln>%P z1(-KZzy*VUT44b<;9s`ezw=tLSX=ZQ2<4(=0a zT!8KAI4>8(TBLV)*$*7250NnSIC=F#K^7( z^BMS0{7|gRUm%joih37*cIBgfcA`V7iz-Wtipfd->h;G-m6R2g^yH*fRR2(zxSXhp zir!zO{Xr!-gd7B;@uX&d;5exuLP7+3BBcAHy8j&9h&28a6C$M%(<0RSI4ShoO9Kj= z_~u|eC~Ovh=_V#4uJ;EL!$WJ|-)N3b27jYDbR$AU0@v2yU#=#=#^L7%1hJCGI1lli z{u3?W-1Hl2*MDPy*(YF|>;J?7yZ3=H05jA7So#0Na3}(1!bB_pzDIyz?mI_2w?7F% z$OJpo{WmH0zgd!hP0ElbgpS_(Iq@1I7<%yv-2Ljwn*hFN#@C?O1wg|AZkdpE<`CuC z3<%1CtQe4G@oC zP=Z8R0UkKOBZkn2y&(R3$JPDC2z%)4$RQCi9RyGp7We}x3GPDsr#k`UMH4^?_-+a2 z;NSypwN54iS60UpudoC6Q703DtEZERz%A3s#48{+HNdOafz1YZ^304)0+_iW-h2*h zz`$UGbV>&{5VUW?q85+70mK^#c$6IKD+D(BcNzaG0zfG}2>{Cew{k&{kqgC6|GWHO zWH$dS4q8XB)!*nK#lgsIK%njKI3GbE$}kWJgarIR{+t2QkbyK85HczVgbe%zfn0ze zO-Ls2X9$G5yGVl!!h!wzkAF{%(Hv4B{i6jzDN-2GXR*i4qU){f&!knqtB`R{auGg_ zN*7S26+XM+EXfqxStQo`$@F=?qsydEm-l7m^TstPg>GC8RP_y69zIly_*a6f;)L=qmCtGqNY{L} z8z5O6Xfn%6^4*o3fA`(F6|1xD z_VQmn4MM~Ju=bgmZ@g%i!LFP%igrtl*Gg5GGq5UsGAvOuyIY9Ld#ebKwAiGAIiArc z<@?k|qrp3erqUm>=mt2E1i@@qQwD=A4+L?U#EK+lxjH2!qG$2n38Q=W7DY3ue047y zPdfAIb?sxjzzZ2cMcMY~xn>x>;6*1hn}GHbb=pQ{7N3HRcOF*bH;LO!1+rWsrLT!r zYkaw@pB?j1p|p5i$D`rLwXG_j9L!E~jr01=IaH%FJfaGhYGn7qb8hcqcZfb&O$=d` zy82?~>__uA_v;4@xw2~A9{J!Lj1Z9yVBO-?qoA6%rzxOSn`mFxx#k!@uu^vF)icbI zzLGQMn-*_tZ`rEeFzlTB{H|v?u7j0-m(!y9X)4ne{RA=Z3$#L(2ccp~an(PP8ERHe zf0lVDRR01qZ3!u>MnElbO4!s+Lq{jDvoyTpy5zimT~N|h{}2iG`S>=L{_FYTh9275 zvCLaB!p*35%F6wbT~F1;2`!#jh3h)>RW3Xf?#f%)E~R3Sq#7X%Kx_PgB*~N{Ui{7a z?#u7Bfn^WI46ClDJS|JMcdotHRmCo}k0}&n|J7NxT&&wyxj}D;$PZ;dYnNR|J`F=%lJST3(1mLkXD}ms z#2GN8SH)H;_4F{o_75vig1DUC;J0(-d%g2GAE3Ky{f41KZI~{%Zx73H@r z+!Y-ZZ3E=@m3XOmluE8Kv#uUw+h;2|q0Y!jibN1*vu&ZXJz~X3T6#f+&1(~;E;_Wh zMA4ycq%p^9LZ6m-Vf}{UGlCxqpQcBarQ`zBZnK*VoqsS!PPb&R%rhgxec_XHKhx_o zUmv!=UbW?E69JoI`d=VXtL*xW?r&fA^t{WEZQQu~vx`i~?*iC1gm7uzF*N36DfK94 zb5=wRGnkiq0yeFkWim zGkDdyq~ft$0b4p+_muX!R^UGQt{fnJ5T*wcz7Wyr8#>Sbxz7Alp z)Fo~oO|@R9ZRWN+uSwaLZJkLf#1OkR6OQXin?bxWjxspfVD?hrg>u+Hh>0arfoZo0 z$-*AV%Y1%?_0JkJxMvcCc!2xUb99b|DR-iZT(uLhs_nb)3UPhdnPX1ENMdK2kubbJ z<62D~TXJ@CGPe(t$dOh zYp6#mRj|6#!ahG;I9HY}Su{y(KXZ-EGOwl28`no~NMEC@-=mHvH_6urJbCF$8VklN z`984c>*59c>6|+Ij(j(bY8xiOyasjcOcIR{yz!Or6uY}^gjjW9Z_26yel)h;8MQSW zJ|CSMM6dE5&3ff3pF%U86LK5|xhQQ=a4dSXFTs7`hfcwml8lHxVylV*Nq4JX}1vQLN)xCZZl)=bDD7rvrPCYq8A2!GppJ;16 zbn{k3JMQG`FoNa5y1NAT40&WgZKLM*5+1~_EyaGci{qMQ9LKv9!9N)36sAPS>GUFy zv=w*Y$$fMclAhbF2M4yEy}gB}LOFLIletn*^JB4v2uK?51jmlxa0cRV;=kVM-R#EW zZ*R4GQoX;He44(!!_#M@M~U|SG~al2JThZqI)R*Nr2o!}?Kv^i#?D8ZgzS&md%%)m zi=i}&swe{+yXj6WpTm{DWsN%NBsAqQe2v?qIU`4e7laR9X#1${KZcFJ!j_e!)8M;e zVwG|m`6g*sx(#WWeN<)S6{X^Cf2>7Sq8+UyXBjV(;Lt@siPL1#yJTyx!ENV?#L(wm zFPEv$evMP^T8`k^ynX6J8%M7YY2wY+Yw`8ib>7aS-Zj9=z|(H${=cfs?BIOWnvax z>Gjb_H!@B3@f#ILb~<@ zOFScevLAimqJ~TXuHRcuM_&p#ue^G+qBYG&Wwh@XI?|#m>4pWDpG}Z>M!yMgX;Vvh zCwO&VFd|hsrUK|MFC^1qc3Fs)q}!i&es%m&I_IeAbk}RM8js_q7yWoIpG1q_SL!03 z0K!T>_p4m@L&s?f5x&%Fe~P&IO!Llagbsl-R&@WS`nu1#*a7~>1Fl#716lEA z9*qV4Am+@+oUKd6##p5B!BBn{%aV#rq~)z|!c=`;KOvZ8bDbpZl0^Vnc=qrX=!?wh zcqFf(23L z+Z{YgpPp|MdBrXTXqt4kg%vr|8+B?fLIKvF%<&ht9)K}6nz{M~De=@z*j^SiNO4vW zT>pN*LVBneEmUCASZC;|JDCBw8(wz3?KfN}bH1)nuN!gYwNe&(vL*eZw2}v9k?w?^ z6&VKFePY3**Nv;*XnMM554i2IrbWEncjpN#teQt^)&t=%BaSI+9Q?r*+vs8x2MXeb7M+bS9F&zi1FT2 z>oa|DLBaF#hZ(_#=Sxt+(K;TxOIw;m;F;V-`%WZu52L-?LDIWDLZxU5b1uAwEnA&{ zd5_-u);1%rwqNO3b{>~DDZ2E%B^eRb#|*{sb>edB+P5<7b9J!_FJhEeT$vk1Awa>whKK(QI<|HhXApdAOYccGZ5wcft~TveR1F6$-L3#H+8S3oPy1 z&sTD?>oF62xI3bIHfWGsSe;^$eD<_xcaN9o{^u^14-E5|Nmoo0o(pGIzyBeWeU&UD zh%dW*w1(ZyuWDYJ+JfCeiRu9(QW#bijb!uUC};h)+k8>m&78#8aa$bOj|?O0i?5JD zLZIjQQo++|(RXf(l*kZ}tL>`xV)jja)%9*>){Iki1iyMpU85apHB|pOd_oI#iNvan z--{!U@#_+5{Tw*<3+HEPeZ#7(UaTSTHBxubZI6rW!|yU^9wy6~vwrp{3XyBtu&6gp zeE*)r4Q!S6dC=0l%kNQ`3|g!qrYO&(EAxJ^z>TN@&(79+?;WXk_Jc55&;wg@trs!` zS_2|Vsa6?`M7g%=GQ{v%2lJ6NDjxB7vYopqpoo=$hTgBHu&xkZ^+elM(&z4%Y9_>S%+)+@A) z=DJp3!NOduy$DJTuS>RX^~P6fiuvQ&KU6cUfTSDNO7a(;YN5C;adq&kdGm6r(S)0N ziK*r{?!d$SdcAv?W+7XhGCjcCVw7ZqSe! z$BKg^l;T>*axE5jyqDB$tea#wpL7eH$8P zzJdPRDX%bBc3ln9JlU30zj;)a*LL{0&9yna*&=|{=3m;b^MAY{DzeJQM%=L&dJfzb75<`XiRNsHJsmU0#cmHCwO!H{eJ|m zs+n1D^m4Yov==b%LJ@t%etSRaAiKbfK?$iQf@G@L;HBowBU&*+g;fiw>{g-mP*YnM zVga)h6Bb;I-GSr=IWfLZg>S4#(3V~j1^egYd>p-1_Bgp?2?d*FYvUPR7F5R4@Cs#h7t0~eJRr!MP#SEShv@Mr({Q`ORL~O^KIz`*^FN#eM?KFRUK(Bo@Yp$mc`<_#TYPZLKS&Yio!6oMNWxiV*1&L09d@3 zM7$h)5QdKP)%gy;Yj`Kemz9Zi;Uty9%^by?K!==2(}0DU#r;fBDPaw%Yg-4hce!u6 zchRP&h3~v97E3|ny$08}EB2*e-EZ`(dwgeymNd;5pI@y=cn+4>(I-Q~dt5LX$3qT2 zGs{ciny;Xvxff8-^u5Fxt1|TtrYw$xugwTyrqfh`K?iaD?yb%^cN%GGN-Cz(XUy?x zT-GTyGkjO_sE66~-Yin>DZ{Q!g&AIGoj+keI&ZLTC2v)#g+JqCROyE%OfCau?GG2iuly&Le>yb z@u19yd*5ceu90u=xTS9%4T$_?5SM7z{_-?l-5(TR(MGp5ZR2T8|>s1VNn*j|k z)oMfO1}?iskfGdq;bN0=rz)8AxvgM%^0)NUw9FPwNh-P=A7sS8icQA1*T)Z6yI*>s z5*{z2(~j%7P?hj_!N4n3av}Fpv;LZ-Ba8AP=M{M#xlU{w{1h=`nD zD`=nolr@D+Mk;0PdHH`+ccKzLDiLUnGeK6&ROsr>oC^Anv+YjeLrLbG;VLQCLcMU1vq9QrNPO}iee?`#q?qW5h*zvxa6 zOfTH>x1g@iEG4cWzA-F9WcW!aHX`6LK9;um`74S1^udBqxE4!ShD;jM2ARDDD4C#IEQ?olI3ulhdYwplCkX#II>e(?kzD0!}B*XrUy z*67(v>adTOS3c^ln~i<&-)nWYdCSIav!L;rh~M8RqzXmo!_ezSQM(cR@zgqA&fUf@ z`jZ}*OWjdS7uEwM(-@jhd5x}eqz>d>T@38s;iwgJtav-{G0T347$w+0CCm`rWB>b2 z6UvO*xzPG}jMK|{>$I1_PaB7J15#Y3TH~{*6SrO(m_*4^uCwOuc2mEk3u0YK^Dt7h zV|6VIQ*feONq(=MG%c$vNE_MAt+5iUsmacMRo}nfHLl*#+Q!d~NXXZW<-E_kXF_rH z4|cvf%7)-@OcV6%&D16_rz!kk{leUhGgUBx_C81+pU!)p?pEo@52d?>Hy#X|MUGLV zCg{%q7EG2LCk(9X)Hb}rH_*kSt?H7kdLH@AzirQ}QLA;ec;Kpvv%2RN+qwCs*|+I{ zuzFYO;yUuRIQ*N>{V0p*1DPZkyfd12c(nqa;ol=7qnmYD<8?Uu>Z8ROnci~bzHqc? zKl(I0XI{TO_SLt^W5{RNM=;dkhjWs{7?q9!T z%@vF4nyW{neJw z<7_PRD4f;K$kFZ3FHJmsRyA}xB6z}2Wbr|I2)&$lyYac=bJrglwqwlF5Ovz2UL+dk zE|y?CZ|UQkp{bVAI%fXBavnVXy{&-$mkc!FBtFtf+~0;9lbH7SLB7dOesW`(OE*#ocX;%Z?nxGgt~Wyer-sgb55z+gTyak(sNr zHFnng^g9$ zCD=>uvh50F;;ZLizwOvc{g?doB|);&qO3LKkC1F97q%&GkWN%?pMh=0NHNDL9EF+nBCZQJwV1tJ(Vj5G zt=zQPXO8KLw74=dE^L72 zn>LuJwd%*-7@c*NV{>jjkNp+5LZ(yJK|x*$p~udajDVKdrY^&6bsUhK%V1}$!<*i8 zA@>J#C9}9Wil_79*@b(EYS%6&fT+hoIU(p(*S$7}swqB^reT5~85dxRq^sFaad$iJ zT*ZIL<7A_%vb3Ver*Lk-iXNLRete2OrAwgbfOkKcCtBixOkn%mnJM$UEtT2pMTKa) z0+*yN@bmy*qfgfOTuWh9xA#cm>95y^)-3%Z)@#@;n#Q6tk`LJ`BGKA^_cn>t6UtB2x%;YSYuE5vdg9mwl46zO zHN{TWi9AwYGm252#QXMq!!j4|Gbj?}$quIYz0zIVsf)v^SRDjkZJwBxJWcA`qtNtN z(1f7Y3D>+=j^W9T#MHcJYs%f7GC{MIvde~+vOgSckO+0}hkS8&kzg&AtbdUxrHVJV z#x?XtC(z;%hurvl`FvezAgjJ~2Vjb)`|=>-VNE`7p^|kM{Vg|Cl36j`H*Vt;^;gP- zA~KV@17gJD?lYjrraGrbHeWeo_(E_~%Z)WXm Date: Wed, 2 Sep 2026 11:39:37 +0200 Subject: [PATCH 185/189] Optimize ROCm Q4 prefill with K128 staging --- ENVIRONMENT_VARIABLES.md | 20 +- rocm/ds4_rocm_q4.cuh | 400 ++++++++++++++++++++++++-- scripts/environment_variables.tsv | 9 +- speed-bench/README.md | 40 ++- speed-bench/rocm_q4_prefill_bench.cpp | 59 +++- tests/test_rocm_q4_dense_pair.cpp | 170 +++++++++-- 6 files changed, 625 insertions(+), 73 deletions(-) diff --git a/ENVIRONMENT_VARIABLES.md b/ENVIRONMENT_VARIABLES.md index 2c5d3c980..3e54a4e72 100644 --- a/ENVIRONMENT_VARIABLES.md +++ b/ENVIRONMENT_VARIABLES.md @@ -98,13 +98,14 @@ The detailed Metal A/B contracts and expected oracle counters live in | `DS4_ROCM_ENABLE_Q4_PREFILL_Q8_K_WAVE32=1` | On gfx1151 wave32, opt into the no-LDS Q8_K activation quantizer that assigns one 256-value block to each wave before the exact Q4 prefill matmul. This exact path takes precedence over automatic direct-Q4 WMMA; `REQUIRE_Q4_PREFILL_WMMA` overrides an optional request, while dual REQUIRE fails closed. | | `DS4_ROCM_DISABLE_Q4_PREFILL_Q8_K_WAVE32=1` | Dominant rollback to the canonical one-workgroup-per-Q8_K-block quantizer. | | `DS4_ROCM_REQUIRE_Q4_PREFILL_Q8_K_WAVE32=1` | Require the wave32 quantizer for strict prefill A/B runs; unsupported scope/device, rollback, or an incompatible required F16/WMMA path fails closed. | -| `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA=0/1` | Compatibility control for the resident gfx1151 wave32 direct-Q4 WMMA prefill kernel at 256–4096 tokens. Unset keeps the automatic standalone and attention-output-A paths, while attention-output B stays on Q8_K+TILE8. A true value explicitly retains those eligible A paths but no longer opts B into direct WMMA; an explicit false value opts out unless `REQUIRE` is also set. Use `DISABLE=1` for an authoritative rollback. The kernel uses transient F16 register dequantization and F32 accumulation, with 64 rows below output dimension 1024, 128 rows below 8192, and 256 rows otherwise. | -| `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_K64=0/1` | K64/P80 is the default staging mode for an otherwise eligible direct-Q4 WMMA launch. It stages two adjacent 32-value Q4_K groups in one padded LDS tile, halving workgroup barriers while preserving activation traffic and K32 accumulation order. Unset or true keeps K64/P80; `0`/`false`/`no`/`off` rolls back selectively to K32, while `DS4_ROCM_DISABLE_Q4_PREFILL_WMMA=1` rolls back direct WMMA entirely. | +| `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA=0/1` | Compatibility control for the resident gfx1151 wave32 direct-Q4 WMMA prefill kernel at 256–4096 tokens. Unset keeps the automatic standalone and attention-output-A paths, while attention-output B stays on Q8_K+TILE8. A true value explicitly retains those eligible A paths but no longer opts B into direct WMMA; an explicit false value opts out unless `REQUIRE` is also set. Use `DISABLE=1` for an authoritative rollback. The kernel uses transient F16 register dequantization and F32 accumulation, with 64 rows below output dimension 1024, 128 rows below 8192, and 256 rows otherwise; aligned 256-row launches use K128/P144 and float4 activation staging by default. | +| `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_K64=0/1` | Base staging control for an otherwise eligible direct-Q4 WMMA launch. Unset or true uses K64/P80 on 64/128-row or K128-incompatible launches and permits the K128/P144 default on aligned 256-row launches. `0`/`false`/`no`/`off` suppresses both wider stages and rolls back selectively to K32, while `DS4_ROCM_DISABLE_Q4_PREFILL_WMMA=1` rolls back direct WMMA entirely. | +| `DS4_ROCM_DISABLE_Q4_PREFILL_WMMA_K128=1` | Value-aware opt-out for the default K128/P144 stage on aligned 256-row direct-Q4 WMMA launches, targeting resident `attn_q_b`. Unset or `0`/`false`/`no`/`off` keeps K128; empty or any other value rolls the same launch back to K64/P80. Incompatible alignment or row geometry also uses K64, while `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_K64=0` still rolls back to K32. This setting has no effect when the persistent or automatic transient F16 `attn_q_b` path owns the projection. | | `DS4_ROCM_Q4_PREFILL_WMMA_ROW_TILE=64|128|256` | Override the direct-Q4 WMMA output-row tile. The default uses 64 rows below output dimension 1024, 128 below 8192, and 256 otherwise; `64` also retains the prior kernel geometry as an A/B control. | | `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_SSD=1` | Explicitly allow direct-Q4 WMMA during SSD streaming only when each complete projection weight range is already backed by physical device storage. It opts attention-output A into WMMA while B remains on Q8_K+TILE8, and never treats mapped/registered host memory as resident. | | `DS4_ROCM_DISABLE_Q4_PREFILL_WMMA=1` | Dominant value-aware opt-out for the automatic direct-Q4 WMMA path, including explicit resident or SSD requests. | | `DS4_ROCM_REQUIRE_Q4_PREFILL_WMMA=1` | Require direct-Q4 WMMA for every selected projection and fail closed on an unsupported device/shape, quality mode, rollback, or SSD weight range that is not physically device-resident. In an all-Q4 attention-output batch this is the only control that selects the numerically compounded direct-WMMA B stage; it is a diagnostic assertion for strict kernel A/B oracles, not a quality-sensitive runtime setting. | -| `DS4_ROCM_Q4_PREFILL_TILE8_STATS=1` | Report dense, pair, attention-batch, and token counters at process exit. | +| `DS4_ROCM_Q4_PREFILL_TILE8_STATS=1` | Report dense, pair, attention-batch, token, and direct-WMMA K32/K64/K128 dispatch counters at process exit. | | `DS4_ROCM_ENABLE_Q4_DENSE_PAIR=1` | Share one Q8_K activation quantization between the two Q4 dense projections. This pair remains opt-in. | | `DS4_ROCM_DISABLE_Q4_DENSE_PAIR=1` | Dominant rollback for the ROCm Q4 dense pair. | | `DS4_ROCM_ENABLE_Q4_GROUPED_ATTN_A=1` | Extend the two-launch grouped attention-A path outside its default scope. The exact caller-marked resident decode shape `groups=8, N=1, K=4096, M=1024` is automatic; row-at-a-time batch fallbacks are not. | @@ -153,7 +154,7 @@ above, it is an unstable internal diagnostic or tuning interface. The linked sou remains normative for exact eligibility gates, bounds, and architecture-specific defaults. -Inventory totals: **1080 `DS4_*` runtime variables** and +Inventory totals: **1081 `DS4_*` runtime variables** and **6 external runtime variables**. The auxiliary inventories contain **118 test/test-fixture entries** and **19 tool/wrapper entries**. @@ -959,7 +960,7 @@ and **19 tool/wrapper entries**.

    ec@Y=n8nUpaw2$A0(`S)IIF1~Y#e ziCNZ6%raxA$n(_1t}(0`W{pW;jp-7zZnQq+tQ(54=#;-8n83P`#JW*63Y~+sqk%SR zZj@1n^UVC<`DwnH5Ri|Pm3)Jqb66vx(bin{Db%k8+>7=Kw9A|b|Jq$srufXc*AsIb z`b`5*lEZSkTh+Czyup9EcW?IsuJziCdsp%phn@I5iRQbIhuzocSxNA6$1j;!!mi)i zKjF6{Xit5mY=OXC-|AN2&U%9!&pehcdPp#HUKJQ)q&4OL#IK?2J4m%os1EspX&w2C z@|bJ4#pXeOT07n`d71Xjv3bm!J>@bxSuXI z5Bis{^s6S1dY}!hPjkV6<`cSOqMtiT{KVT$@a>#d&}-*3?k~wK;|}?@Tt8UbZgtO^o zp^+W<_dMH|k&Yk9L7elC407%s#J}fg)+>M78FShyRut+y^(%eNM6kzUjnuu=yS?*o zt&j4LoxDCaQD?28uTx*?n$;Is!<#P&wX7S*-Nyc$L9*o&{f`$~li$bwVm;5pJU8=9 z;`tg+A7IUCqCO#7G6uFJ5&kUd~*hU}X9&XQdG_ny^bk&U;?iPK6;q zZMzbiUN7uJg~1oJT@Z5q-ay&LrWF4{YWE9#$A^M5_(v%>Ve*X;wAU#e_iIgMO;)>q zVZQtB+HTh3T>EM7d#d@)vSTU-JSk|Oi{Dsy71n6Ad9_zgxxQ)d3VjDdyH;zBrmf4p z@4NW7+Nw4B$9JnQnE!&bxmIb|!?`?xmnIMTi#^eCUYco2)6EljX^Px5*siSlJLAW!~X81k2zxRyHgdwW?w1DDfPn@Eq`N zPVC83Vk;XS9A!7;u9q*>SR`+~$%pRx%GTUN+Bb_QX%|ldPp}uB#Khmhp8Foorm}o3 zd-8nRYc96drWn3p*Ol&=^rMZEflZM8exA?Si;97 zeZ7FNw@o_vx*#3>{y1T;n{=|jC5$on4Pn|RXzpGJbomM5!1Eo%wJMHzd|qG~ z7XVR(U%YUZu_PD&F|qJv9`edqk@uo$WKHyJk06<=TreRrk`nE zfD@Wi*MJvWQ;mO!#o_6)Q_QhXTawsk7u(nWz};gfYIVujT~&QMV6=r^D!MaQ6># zxQTPptmaF3PUCr&y6z5I&FiRpDRpe}=XaFj*Kz~8hTA!hj`_1+8|^FI)xdj+FSsiL zJT(>ikinTJJpAUKywgtUUE?paS^Md@&S`lUjnleWOn)P*GI!aRl!bPc7jWz7A3pgjFuf9yTqrLoG#%}`SIR4sCPy6<@30;p3T_adWM6;#8 zfgK>cUjY8;r{2E#L(0`Y=JD0ZY4Ytr2w#+5n!k|-9(m&1?EhBsXiYb?FQsR6%`BIW zZ6S^5+eY*gVynkg!56l4%}b4SRrIZj&+AvUb6)?dPpY$qd^|LJ$j*wKA?&YS>SAxz z#$C40MBOr8RvXd#ZbQyF+mn&LM40xplF$8?HQSZXQ9pFuQ`kd@KM?*$m?NU6zaYJ< zA5dryB2IMc0C^<`6>lZk*zvrJr&pgmN_y8fjpE$%ulr0~vHf+gOr>$lMCL_Yz{*>c zC~F#J)q7@vJeexTH-5WUO_FukT zV*ZEDop<&tIj!(p@&Lg+6Q5(}2r?d7<~!T&CkTDo-=KeG*~hUb{K~yqYlgX(L-+IQ zuC-2RqWmEw{D5k{B-6y{HJlOQS5S7#^80xJS>~TptmYSamh;@kGnr=?Pv@VKDt2N^*d0C}@hJD^z z_JZ@I55zs68`%p=j^FO7(&QD!bUn%JuFyI>M`2<45C*w1K_tj5T#;Wi?iY{G8P;sQw_s)pxk94|p5%~x5?T#St?mjuk z{_K8VZ0oQ}74Z=V4lzClbxT_=-;$aVHHeaQk$8sHzk zAdTvO7xZH7UB(|r|4w+zFZMH<4^Nl@FM^*@@aT5rb6U4s$tO9#yHC|!lIYT8vA?Xw z|EHJcS4snq=gO|VGpX);lMC|%f4!V3v_5vFKB5!`}7&uJG7Pf4xhtW^2g=) z)2#40*WQu4{GWsIb2$`0ms9brIn&?rPtLBtjStHoL1RC`59enA{NLb*^D}&KeufXu zPuZ{UTY~S-75GEzLp`}xAsYZe5pM6 z();T@g_f@3=e z=f&iYv+I799q9ZdQV{zrQW)!a*4Ogs2G`Ffiry_dq5V6@AUha`4B>@POZx;Mc-B6Q zxN5>rcIOuCRdoY`S5`)87A4 z7-N_St0n9&CY=|?es>ySZ=3QYdmt}i3kZ8%Vb8i@4SYY&^D1Gk>;XO%9QzGn>l9|r zEeKEkI%{Te* z`ET+S+htz9`}r1sr}anc!;$~x*uNp)Y?Cj?zTM<2va`H=^SkpE+2Gny`F4=+3NPPv zCSQTw!^=0jJ70nQ@iF-%!??i9H-&V&Z*lu(-~DdiO!M-h>ulDRzj^s}?}d1{`#rb_ z7+}w3`k(%g{G-Mz_iU9r+by@vD|ftCuF*>%?zdjKNAw&RN6M+CoS|Mh-yo0cKa@D= z%rxREiR<2@L30)mR<~+pl>48s_~=>+Dmw^{2laX-2fcCv z$>k6?_e;u=uhJb}Isc>_jYVSL^bqxo53OlQ(2|``o)hC9kVQxNlv|`V4?9l$mIlS= zN*;wjK;)p#wnz%+YLZFi+v8Y6c0*58U+t-Wi=O4y!8I*mXjAwp(TyzoL1<87PxUHw z(S9n(dXgujcZu!xo%? zd>*(BJIw#2mv<03lXXg8f63SbSHb5f{b|gHTU0lA&%gPsX3h0G7)S9l_Y*Fh zy2ykV*tdA$3r#rpH=;{YM1J(Y-YV$oS|@!pt*RTpzo5Fy%7ugi;XN(+AZV=*sqPZ*6 zkvCyeRPjTXPJNT`DTK|#o@hS&yKwKo)XXY_$I^$3SjGn86aE)}AsE~LL0as<8c!dt zHT_sSqQ&>!bW-%&o*hvgc0}I zequl4s`;~@r^S=81&CLBWc%BT`K>!il_s%X?`J-`vVGmJnrmcO+X%be$jgfDnY?R_ zK_4jlb7NhbX7uoWfV?6x&gs~-B;I@SUWaXv&WSd$r*qw=P#W6NQ8ple{MkskaNrxSf zH4bw=qa*A84PY)n^o5pnJD^XM9cCfaf+$yVgsU)FZ`JdN`@;^P7K zlb*fE3jBpgX4h>mk{N2b>T~uYN7~Oh>>H)B7s&4qtIJ4{_i$?)^eeQ$#!7>tQvw@I1`-%{((HUo=QEq?^Gf z@dk~AnK=qyHOmWYAWZv(W*>h3cn;htUBm(tFFyv`;W^@g+(8!`{|5Q?4Du-)Jl5LY zZ8OZ;{)Tm8Mzjn&64}`grd-ua`<)O>W~i&)>k{C-?`rZ)<&IMbOE zu$pz=a$a}21@;^CyZ9>cjeq2Qhk4iezw{KPf0$!`jXJpgt#a)=e7~0BlaIQ;5#Eyl zPAYAY?PGjocYYB4sQ%O|^%{5G7{$Lt-|OiY!@KQ`$X4PPd{2fhL}T!CRa8Qds|8 zznIcx_W06noAxWfsnVX-@M6j==m|c6Q{dL{=XoCD`O6&%yut>C_6e{4MVO0M*ztH_ z|0GO!^)`LCjwgQS`sGDN=85kr(v(<5-MBT^^PTaw7ygB~RopAO%EPUtl&$&+w_fF4 zxV729Ec@p`;g%aO|0-XNTSeg3T!UNqI_)mE&>jgsnYA4}I<*_OTtBJXz#HM#hu~cT zx9ag{a0It{fm=$;UG|Gy+&bjp*7crkz|Z*C!T$@~8s*`Z+pZ$}KJx!>;FkVd3~q53 zpTVtizJ2s>SD*(za(Z)S-v^!ZOmxrtqJ!QKUG)Cw&}YRCx271MtcJh#^a90OYd!}( zz4=$aiQYW2`a;Saood}sgWMWlKg08QM$$d4s+=@6IikvUhpUa?2lKaIe_-PdH*T zRNMg%Hu)WcAG!94i8NnG`+7&qJbwuv-|qSo6Mhb)Uc%2(1GDU&2Fi!rB>HM1eRdvw zcP@Q60loe4_$B2GJ)gbCM`)*l<= zq=WC+hj~s=MkY-r;z{ z)|omS2~)k+5~lb>TGeNb>T|Gb-H~wg>Iwgum*3A>qT(*1j}m;~LG7y=?iwX~q0#K; zDu*>LlkL(@XnGa$XX$FLhJM%i3OdG~{NrW#H#ovW2Ii02)?h`qm%II)0gkCY57X|8 z-F4@UVA(L~_Bz3>wb*s1;ah9$fhpjn(hc>}H3 zas8C#*|+n3;s3(gluSRS^1uJ~ZWj4r(fRcuPp4-$Yt}xGj%dw#9^NmRmOMWaKUerZ z=t5U_fgPH&peQs)eyUP<^w-Hq0_(4{I}nABOB9BI&~#Hkag*R=)dN&4@xs}p9OPjoF?`! zeMs-Fi}(Owj){I%5GFb$9;>exHjpr*3(Ne;@WPH$I_?ty5zTqXbirYqX$l642gRBH z<_^Ge{5G3&KJ#pz9?c;GB?bLqc;81vd>a&w?$V z!bZ28M~MqjPCSqjT?qY0pRJ`m6}`|2wXwag_l@nopEaP|_tJj!M70K3^Y0pE&2JdB z_uXvbb7Fhn&5iATH!rsLca#sljPi7P?xtL=#oO2m-w*xRM){(@30gnLqbZ%>lS|9- zzrlD*MkHE#xrxuWKTC?=1}=!EUSi_7bHt;Qnb6HD!leV-7N|dcKj)dUs|d{Ew)t!=8+93~hJ!MY;BA>U;2PAF=?+w$4#sP!A*j@_dq16Xx2XWZCRDNmp$8 zpv<(h(0-=74LewmNg%{vOO6{oLGtG&4FMoj0FZQ$Ig{@5FBpa^vC0&f6XQrw8MZn7;%6 z>B>5!=aQfw>UZt8lE2l*vbRbOJN8>?{-8QD7Ox6|$D7S}uKio!Uy&J{K))D1wLyGp zCVXmNc&vW#TK(aX>TIXB%I?B-oHpGET@gQg$hU2~4&UGGSwfp^^uy-JeF zueF-{b<5;Wr%tMe^h{^--j*72W(l&6TwtJVPQO?l-(EfepZdwXEB*v!=N`y2y0rLY z?tu3}4qwpe>RZUhw}^RzEDSu%^W^Q{<3Hl}rlfLT0Cexb%=D_=@Aio8t%lyswq9~> z@p#^bdq&A8`0m&7N%U@dZ1?N<2z;IUMrJZshWVn^_qcTQY|0bQP-^(s0{eic_gPE0 z$M>%F|7?tN4s%kved3oAw6L16^@jh?x0{()3Ep-${gTk>96%cPY*}lYdxum3VOsB= z^z!60uAl4o&Hash{4wU1`h1Ls0}-I=koZS}ht>Li?0>o_{|K*|Wn^6k2y=ZJ<=Xdn z|JRsE5IBQ{BLxbUK=fT%b|5CoT2)=gtGU+}i^0x)< z8nvwzzILk1*Pajlxiq5Cev|&+Iw7Ppy|v-ZS@d^-y^^tV>2jg{=iA&fJ^9B`+?#xh zr+poGF@4n)kbl;G%<2Bj=`7}SHgh_MIgJm$xq0k8-8n5DGWkA`#JooxFJKJ5!&scpm`rADCb6fT$e#8*_Oz}&x_pfWIKSBdE!d21-Vy#t za$|I_4BwYyXCsqS-(JE$L{meV^v!Btbdu=7gx*zqFG!8;W!;M}vR*R#-`2)aZLJGO ziPu<(?L#|$C9L^BfFD}Ga}UqmqxQmQv@P*PPrlnNyOy#|TaCRe=NFz%!?y_UrVCGx zan59(Y~$a_bm3$(cZ%(2z-!S2@!7JMQJRF0w|tYP$bQ00W8&yX@d*8hbNPxw))?u> zyK?Fblc&J`adIATLGv}m#O2#sKN56E_F0b;cbKr`x%>;#$_7O8YVM|Tv^SmSXAYn8 zWqZ>s=J1kbad_4vbGTvDwgBfDtKB&~4P4p-UC>$}KK=vxYb&MpOb2OdklDL;>gInf&Y27ZsZos5IhkJk5_lq0-US%2dFBGG`>6lg#% zXh3i9tPl8>3EuSu|N21#kU=%L@~4C!hl8x`x)W$OYkRV72t0f@4LIRc_97WB4H(Ip zRzA-G&VB#J^Fq2c{JEk~%gw;15_G7T$M_l;zS&}o^D|?|Q>);PWCtMqlLi-Nl5_IFwo{az2M0)61BK>NQZfMRc{lG7x8z<#PH+izI zlc}S|Xx{|R@Tt%DO*`Oq1n`c`ee(?4k+`w{VF z@e1}LtFbA05}T6UzG&eO-26kxFZu4q-d3~d^xr*wn+n1;KZJMwUyN~vGH>N0*X%iZ zxU#lFGnb3)N0@K!Jco~mZY0j%{lvR`fc6|Nzj2tfn*UlK5_z`K|I;)!CXV}DlH+|Wm zB6~6Yu5ndAzH*Ci=?ahL^!p-ub535=$DZa~kKPD(MVHimy$8K_gV*%C%3n*FyV+;# zp}#e6pW<8e$OeW*k0Pnqvt?lim5n`A4)$!h&?fXYugnLp3&8I}@Vp4xRE+(8YV=Ro zvo&GQwh()^tFdPrhdtYg*t7L!FCsl6?MI|Xr2UBO+4g#RM6%gTrdiJLu7~TnXAJ*l z6MscrFJ>(}ji-R;5cF#c&w6N@?(M?1t>t{)v28On3qL9&_CQ~nt3xd-{5dnGBU7E6 zZdFfF+YBwrvabYRH6LyxOnV{GHqB{;-Kcbox%%h3Uf4{94RvYTgJ9CIpi&|ay^<>Y>)N6SHt`4gC0L;=vt8- z_QECO(pbrNtLC$KE9s%X{6#cvV}7*8qiKh3HvZz=yT&T;FKX%^a)z3=g`6RR)OoNV zbF4}byvqcNwrMOx^A69wlrg`AvA-A`xCmUh5ZZPDwCy|K$ob&PWNhZRhb)q6=-hL| zkZFQrvKv%i^;MmrF|@6vq~_F)eNWcJ&N{$3Aa|mu9ow-d*~y&5Cu~P6b{ZeRQ#AtD zkf#niigMn^`ZBOvgxA7GL-&<++vV|}t$}nyKekiX^$T6P)kWL2_bNrd$JplG6OM_; zkiGRYUYdQRsh8d_xPy#rt<|e2`a4fw_*3F6#qqEG;9RVHOjRCqbA88{yD5@NXOvfA?@YI4cWUTy9)Z! z5{IX2Yw$-mHMn(sn}6v{9G<#a>*fMaruHYoMGI8#ccB}PMFMYXt_D2%`Z8f7M8{f1 zcL`_QkLUYE!nfg5MPUu-W7QJ&yrDZ;_6VR$S2VhO=+6bDUvAPD+f~Ww z!Ij5})B3NuGB7y~oc|4RKP1ex#iK07bO&LLCVhdOo1A{ItEW8Q#1-0^$#LN7@zi^c ziOaVAoF{9~_jTxX+>fmC73gv~c5LHZey|JL9RE_`WEZu(iysLKLbF4_-lEh0<+-Kqf)}5- z_lOK=abmCd=&-Jbr{uXd#HZ$2%{jd9&$WiX#}j0q*qCdL$mD+sIb$yUBYIg$nBhG* z*TFV)kj^2w18dOOc`qz4zM}wJom@MMv?amf4%yy*u5X4HckkD>+@yA8+v%o#S@vgq zOV0iveNu@IX907r^UBn!H1s&*6)CYn_=?{-nEld`Gs8@1_u;)WJXuQX-J_&G)-ewq zIegz3zNS1qwzHgj`=*+6#Kk$BBjzBJ(O&OU{wE!6y+7f}ZBz%%#}~1;zH7}GrxW_+ zo+F~K@0}yMdjjr+CeEEt(#3Yq5dT1!`rzVY;%fQND_)$|kK}yQi2Jn{r}ZH@ZUJ%6 zn>cipJzix7bK~F)zpK{-{i>&)Q`ldxqu(WqZ1m{ZOSEMQdw=nNX1(>!9WCs#Jl&7A z$+{oa{Ocv~%c|EIpw&@1to}>R04?TKJ-*5cI)a=9cJn^);79gd$!CK3c2DXi{C1oZ+yEbJb)O05+k?FH4Wx%hS{xpJ9P`k7mdFZSVnJnXyBeQfDPKf1hWft|&C6)pH0VZtp7+~eLnXu$gA2l6cU z{t@Wae#XO`-CN`mg$-CpAHThcNRTe{EDh-;3EZu0bGUcFJ{nPdCRIA+@) z@vS=Rj>uuabAaw$eTCfn6CBhYC>*r=Nnh-0-5Qdf6tad_+i@L9TBS@r{Wxn~)Y6>9!I za=FVJ{E`1ihU4jOtqpqfcM9_FFPgvVkN3eP^^eMK_WJIZ24>rj7%18Im(PWc!RNr! zjqBFm`2Tw@sLol;1$@srPf_=;wh8b5W+ynl4;=rnLU`utfo+3N+rt{Cwdw`N(BM-Q z{I@UUGzAJfZ}Qqd8+hn)bLPG_7{woZIqL+vGnV#ykB*9-LOePN>*C+XM`4*OL&4w9 zLhsXJ((_Rc+ooomG z?^oDOsEj!HT9;|fnr_ZL>fe%Hu$7rIey87hqb_32ddv$iC0xFzC0|dw_Vx!Z3sejU zf35ZO!3|clfpgQ89@c~(@bv6yP52m{jWG*Wk3V<8X~yV{rb+Te_gMWzmvS4;WRddwdc=Kp8+vwm?IIwLB}f)^N+k+x{Y)U=`? zdB&Y#t)Jm5lJB}*tE5*F=sPQo|IC-3!a1ePjbeMrM0Xx!(0420$2AxBvBsqEJqUUr zzjZpxd}FrG53Fp(p)>WCbQjwzxPP{{>;nChHn2YA_FHJq37_(0DEr8xb)NfePGxOe zT=d|XW)HZNaII512X&xV+P~{t{B;Ut?;z}W%9nk!&bgE~$UXG!@NDuum$s#TV*1gv(!+t1DV;E9?crrUBiuz>6=9Nj>{|^@&*593 z>Xj9RJ84rLZQ6<4vZ~*`h3Td61^A$=I{Ds0wM}i5u7RsZj+_-a$9B$3^xt$tuk-DP zXz#7ZtoM?6$!~ykfw&u{!_@=Iw}*In?j}#1IWn5KI5hFcw3+!aT=aIh>AxcT4zC~Q z8dzxG2Go8?YyZtAEX%$T*c7<9^Y`FX74)DAdLUffii}NbN>^+Duk^u?sDtK#{+&a4 zn!_JKrv&2z!y`H8sBWD1VmkZQ8B?{E`Geymg9-Mt`j7pet`WjX@m{&;<2=WlM;~Xc zr;oE(`Z&L!%u@PiB;{$3pmJZQd?Ux=3>f-5F-Z7P`S7v3ntTcW3Y_82aKHX(ahnPRPb2j1YQdeI(@%Rb4up`%B|QD=<{=N!$@ z)VDNGHzVuNJZ%r84N7Bej=?ic!5+-bdb=cr3^!QQg zBAsFL4P8v3&-C91=`)3i#wv|?t#;8_@+kiCuZaH%aTn9hu7(ukU_sWwH2AS}_^}M+ zV7-uiaGy?HA7mk!$VB=g2SZLDNr}F8j2tZ4uhae{IhfX~#QB}<>AZ9KZl0|AB=%(R zWE#ht^T7Ez)HA-# zK-Dvvpn*4C?fNxmUJHM+){TLlX4wys4%^su*OLxhdc$h^M)cY`m##lZoYw48uHa#P)%9)y8wf_>UrQ?N+A^&`~*N~l0_8t6rc>_e@3XnU){fGr_wEUtJo?o{4j+?GcxT zwlFqobMih(x|7-`=`1NhH$NcVK-rd_sC83k0R{Hqc}MhP2GI9`DQ=&CgT7e_PI$Vg zixWI$J@cNmSoQ#mllO$0^D)gW`bOie|MsCgeX|y=GylD>r2D@g@ZUr_^W8;}?-$u@O4synE%)zsni)SHc9E)y-d!7xKN0ADIU(!pM#*MdG@N4OMU`uD6T6zy#bG#1LoY*)bd z=DrJj)ZY(H2yk|9MZxQ}v$s{!FIlGTW*px-_*qTSM;?LyYfn@-?8DU%YW8Qt$1@U{Ih&2 z;qV0cyz5g*dqmL-Qpw zq!fhaOy^!SrJs&YR@IxSQLWqhZU=7%-LZMO_HU+)Swo_OPaYPXv1vo+Jl6H;CpRwZ z3Z!?Ce;Mm=$9{A@^p7BYOrNf`+P)qQS|P{sjeaMK`+VmOux2fUUUhA;mS)yS+SDu`eOJ|V*FK~!ON_wR_-@z3IvXiT!Iv<3-M4lC_&)A7h);b*mRGk^mBc)NFb=O2cdfxDp{U6=h^>CbjXsk zd87MTC$|oT-jw0D`L@FT(}!9kOuMcc*V5*VQySw`$~d)q^%?thYeah;w36{G;V$IO zL%g~PzmPvxmRQ5{p$T)LoyEtmoH(6*V9Ck&NIsFXRp@nzukS9O-l1ReTexa4?+taG zCBEX08s62O+X8i+^6wHvuGKn-`W-%hOUc))T~m4gIenLYduT~Kkp3FJ2k#vmNPDf< z@hcyg=j+wc$e7RY6)zGFBF~u&ZF-=@m*0^;|B+>Lq30LN=2~|NfCu!)qO-Bb9P10- zyOV$6W8gLUI|`n>5P90sdA4#k@^|t)3;r5 zW&XYA!5~53aJJjMJ|9_*mP$ z$eV{Jo)Gob&o{P#<<`oHZ?s|0{nRyE+R){Fj=MXhgX!Y$iP5Eufp}ce74f?WRo`;z z(CwYUxdn9P(wV{KORfyX9;vIrzo}?JW%M46Yb9&J)}m+{cf(3|Bv|H)R#A^qc<%I? zP|LktUBlC9SDg>K!@fD4c6HEhot3R(-=3bP_pFW@y+em&ud$Ujr1#T%euvtVo}>4? zjxye}j&s+d6Zv+Fz&p`hsf) zp2=OPgKJ_{ccd?OE(tj!Q_~jB?rHVQ^W`mK9z^8}OTHjh?eCgAyU-ebeu}lU9@vK8 zpECN&&jT%*NPCw)!#_pO+Ok$${1oqru~vI&PfMNVH)H7b2lHF+^hcfMH)EL2IM*@G zHM|R-(;4fl@x!(q8;P?S>pJFk1MeDZ&C$!~57CfP&5g;w-z5Cv zhficpFSLDLUDxk$(`+M+7)I^i+i6qBgfM?Pwo?4cuCj8W^T;?W;CtLii&nu z{-SYN5#uTu!S6{A4Vic-(`x<;&#!ov^GGK0BI`wo<=Z7caN3iu7yn*Q-M#N-6Yt-( zoxV_A)+XZlb}rGn!<}DVn%2?Cu^6`kUp;g;S8aOhvVi5Ck-O`Y+GXZ0YhbW z{0dgH_SY~5hcDg|tY0~ib$eIPw_=%D@3?o?tj*BnL8MEownlUj_G@T~#$Xa_dVC9O zvK35^<2Moc@!ECJHPNJ%(5~O=e_ue`*rPg%-$cB4CyRZO=|le02|hHk7j^OT$Qqx+ z8qXeU?K_mek-X24cMD}F-Z!xRQ!f4Fw)IKkH8&G$eiLhc+K32id3J|Sbe;9zUFVrc z(8w&-awE&1Z;ecp@ngy;rJYUqY5G0oR>4bEF|Lw3teOI^(OQAb;Y|3sGvMcH;dLYM zx?yysYbf%h@{BwVT-Px6GaizjbTE18Hj@ZxQmBHQUOV zJHCQNKcSrZGHZBeUC)^CsMMPy2f$^`k%6jD#B#pD9xwkH-KUt}Jx4y>?xrbFn$f~R zH%@aT?#1<29CG+Lel{hCkjybrU+`oE<$jiL=H8KV|KjD%G<|WT%q?Dc8sQ76x9WG` zI(JXFhcJEb?GE4Vhcyt1Mu zCVhxPYOe`4ZpiAB#H)jXaPw{O{0Z z(U~}Pvu>N+S$Ny9&Sy#2#XYLB9b^1-*0R>RsD`$oix9Bn-{y^QThHj-v`PP2>-|S` zLV8(oXodD~s|bgVNAt^};n0i{))@H$RewD1^{LY6|D`2Xv&pYMCBN3bL-a={^@=0c zwtBpLdfVXo)3wLiO<8V#++zA8NPnc2v3?w`-O`?iueU57`qqEjEu9T>Z$XBNjTq(K zK)HVJTe48?)vqON+a%$0q20iDEzhkAiyR4?M%Zk^;(7RbIxM@CGq<#7)wIN)%#1nr zfSZGnH(}4?%*7sMJN76IvgawZ-=U5BY5zQbUdMd&R}|*cI!OCPkFF!0J=+^)`Clh} zmVF*;M;kJl?Tqyf+O0ADF!kFp$(8@XytKi~P{`1%#a7#i(M9y#P3*PeteK*9OR!^| zeZ!FGjr~)j_0X(`^nZ;%@7wvWw^uX13v!?@@PjQ?{J-kQr$lvjaCZCYQO)y7zJ<$F zX4=BYPk)FXD(NZtcz>CFruB{K1Ge8EIz08g%kB)m{Lxz}zAJw^H8^d6d>5vW=0nol z&GX^%lxw#y30%8%k^frxF_dqm_JHqN)80qE9u;oyF1Ntj9J}>echPsfH&b_T(9}Cfy{Axq*`ml#{XDI_@1Q;ls81U8`5N_E5WHf5>XF9&`vlc# zxu3fDsEg(P|5EL-t;225%g6b5G9MA49(|x|lItb@Y4Oie{$<*c!FZ=yhpC(F->{E7 zA-d$4GRi6AF!xE{M;XkwH_}*dDhZ3TrfKa<_vNoS`{WgW9Zg>cefKU4`f}G;e4odE zr4vVZ%0_ZoPNec{zD zUOs&r>1+OW*>z>_U4N%ldD#H9fz|Y_IPI8E-y*@VX1ustaImgD~d&A_@ym&}`m>F@F3T8x{2Lw|i z&SJqp#QBk6SJ=5v@L<@vS8#vWX%yTYcJ3DZH0&%8>IJulojV0T2s^h6 zz8`kx2)-M3ZWY`Tc5W7YJ?z{l_*&SxLGX8B=X$}`uydW@hOl#u;0s~rYQg8j&UC?N z!_M~wp9wqD1lNR}%LShdJL=E2)nVs*f-Az#rGm@D&c%Wchn))rmxP_~2;LudCJQbK zI}-)(2|MQsHiVt=g7d=8If8S;&e?*ugq^X1v%}7rf-}QTt>BEX6BfKO?1TiThMjK- z)`gu>f|rDyZwg)zcD^AvDeRmoI3er|7aSLMP7xdvc1{wEgq^Pojt)B~3Z52r1_@S& zol3!BVdn(Fp<(BE!NFl?pkPJVDHSXaI|Bqu!cMVZQP?RI%nLjDf>~iFS1>c|WD915 zo&JKUVW+QPAnf!J?5c5k2_CF*(gpX|I6=YPHBL{#PiveM!HycoFZfZ7V+n4rasGV` z@PiuXkl_0@&H=%9Yn)EOEj7+Q!3nj_9>H<7&c6i5)H+BS) zu65#q!)l#>2o9}vb_foxb+!ps)H-c~<+aZL2$s}3e-kXKb^a=tSL?hdm{se%BbZt1 zye*hf>%1kHTI;+a7^rprB-j;k{vdcT;=C%jKjLf_+#PX#EBI-|`Hf&l#CcipqlmLn zaC^l0wcrO4=S9KyBhIe`-;Fry1-C?;=LBDmIL(5uMVwy>{x0II6>N<-KNs8(ah?)< zA>!DA&qtgl!Dl1R6N1k~oW}*%M4ZP2pNu#w1y@I$p9-#sIFAZ0k2uQ&AC5SW2rh{@ z4+-AS{1;rr{1?23`7hYO{1==TaqbhG8*%Oxyd~l^3eJu=cMHypI12=4M4Y<>uZ%eL zf>R^Toq}}{=XSwMBF-Ga3nI>~f|DZ7&4Lpm&W(cOBF+thV z5$9^b>WDL4a9G6ozTnV^Gfi-C#JOCsBH~ODERQ(f6D)~1mkJg|oQnnXBF=?^SrO+u zg75{C1>qwn3XTpr=L*6TjTeMR*pENPU(evZ^~m1$X=ri{Yrtp=nJZ^yT62P|IZdwI z^o{={H%-V(we}xh5IWGtzU3&H>8jFb74lN;M=CveseH$g9vM{qGX1+!|F(j!X)ki% z$`{z9_}HuX*|Q+qo|}U1w~zgrd#~tTWXK8sPO>$1_rO|fDx;HVC-RKNUnI|1$(d^U zQSyv}Zh3~}6q~wb8F?L-b;~kxJA8UKvW%RLRo$|T?2a1VvyChx+s5J0(kbvsH?oHA z=3V@VpEV0UoUI6SwOc137kn0D5%7ki~i?e465k+7p>`g=TGO0AFVD;K|m ze7_ZWnEb;e%FU%*jhTEn=zJz^6rYsSgP*`&^q-1MK3 zo_+ZnmDHys4fz;#8%>?G*V>J&Igzg2Etm6{8v5mBQ{KJ{UgDkd=5oH#ptB7>X9YfV zyN>jO= zyzW#}M*ktvrc;JQ{nSHoyNEM(hc&Y;;Cq^OJ{>2G_JxCh8t-iZ&L^4w zDfr~QiawWqsm`&b<1hJ@_|4+7cMF5ew?ta;qDuc=(ktyo^lv2_RXg)&r_!oELH;4x zKu@4#g&%OYcpLLtWvlN?d_^6aQ!4M1tKpwhLr(r}bBs^xLwu`@1Hkp%f1z@s^mB-N zf%X1t-qj8SfJRRDPvC9bi=nuA#FbdZS8qi|uQH4cC-GAbrk<+!XS}@aUS6eHJplAiYa2@EWoh&4=vE&UB>nt$M#s+611coYzcV?mq`=%o4bxxZe@x z>8~tKmLt_b(-nVs?ibsg$>wbJ)xPWs@+a=vH1m>mBo0kEgE87%;r2(fNwcZxVbV{( z!c4T8^U3L<*k|0c(9e^@_Z^7MZBJ|0M&_qAFw|0y9NzMOud~F; zH|v&lm)`Ttx@Gw}SF-w;b<4U-@0n)ZvRIQ!xJyLqmeo`5z0JCnLpx&!x|;ja4z;;2 zat7}I?68nC{9?}a>rcMeUS4YQ_A&n|HF-15ze>~HGPOpY+y^gwt1$JYc}B@0bPg|aBOXG z1UQDR&jH5qAp7g4GefRi=c3s1v&Mwhr(+jk#)vbHVMFZYw|>uxS-y>@Pw82Iy5v4r z4MeY<^W7k0GriE_Uda&iyTDGN4O0e$oZALi&cCm6_qEf*R?A%E_FMVC`M0<3aNj@+ zXO0D*4{!iGs&?!sOnKfpepUtN`1lQ&B0M_gthnD_yB?OlyzUJRf=4EwcTT)8)jcQ1 z-$1p}5ckG^drlndar8N{?w1RC=fs;+tPv_NUU6O(XQng8awawm9$9y3u$CB^ZIj^r z+|dNT9KVP45SUNcit^B|P0U^SfSB*fme&S3Z?5!PBb2_Kw81-K%U5tVH6Qw_|6Ih~ zPLfB%PgGaVwI;UD)V;jH8I91Q%6YkyDmjBtI`J^Nb3}L)M=!a(!5Yaqa!Y)0pbEbM zRq+acmFnV@`>NmvkU5P(4w#Q#X12Y9HQ__@{{y+;N1X3$JnrY?R~`5C_`8mKYW&pW z){L(`&L01@<6`4`;%Dl5WGw%;ci3s4@!BmeT^|W<9|UK#7x}FhHk>fwk@hK@ys)8! zsa^YrSxyoCOT2sc`%ud{_4OOKsLVp5J+uH*HawIVLQx<1WHXzYKNlo(>Mi zZ_^7ix9A_eOq$}30DXLrJUZhz;xi%FM#e0C$WxX7VBm~cfV#*|(I-gSuIb$4V$L|c ze3ZGJe!GHv%Cpd)(-DCdf)C8QGh)Gir&ez?bnpnjrgW9w*?udytFyg4&WgG?4{D>% z(-ZWBdBxZd-&)@Fuxpbe-744CXXLl2&w;MtmH63;kG5K5e`4kz=Z5Q;f8sU%PF>Zv zCVc6M=vrW}$5^lVyBq&!;&rF1nO~d>HcU3>f@_1voC{j;J{q@H(yQ+K%ei|An~Qh_ zcn{C}K2KXYKL1VM`gWCzKOpWG@LM_?T*Y|k`)L#J-*vf%-|EjbrViZeDo7bm0_Dp^ z{eWI}-lKu~mW`(W21h?2&dg=b0&{y8FWNkib2t9IIShZsSzsD+OX%sYYR&@9`au7o z4>2O>x7OtmC%;LCP7tnd@gm}5WGmr6e&xgsr~72%wfM{4${%NSb0JLmj{v zuf$GxBmb4%<~-_?sM`y?ZwTae9=RV#JX($&P`Q+&@wkmN;=4;TEPUhp-<)!j8$O$G z@wQfftNGLmawjF~7&P+RZ2Jo0A7*XafB7lVw!em)kFRvspLYNB&h~*;6j}afXLIgc z>6_lUa~$i6AHL?cGY&TbgXggaLvPiry_@<(;}z+#J(oZOX!iul-4>CpCj2b#ZI^_c zF@nUMVPKZMnK9EILi6ht-d+E|*g*|;efMlYb>7Y6+H{hRF_mrF zjv{kqHda3Q^~WBdIeQS@0{Vw>Nl49Q6Kl8!LOOVm5ogc{X?DlTf~VDne)*l=T>q4 z(ehE=x2ygEAM{nU_uw8^pYola53x0|vQ3*dmv@)H$}OL@Ajf_O{LmP_El3}}1^gO) z($y;a`W$o_bJ1zcL$5d=z2X974u!}birDuSv;RGgef9wM*{MVVnOnHu}a^zuM(5o`om))g@&emuanZnHuqt!ux;C&!pjRDOvALckd?nnM(Mic*XgK zm)XpHcJXIIEsglCjz1Y{0gs1^XYmm(f2^C}gG?RZS*A1pl(wC8;#o@2ci99l)SLpp zvIgEKh5Dj9-VW~~eucGg@*&O~-(?KU{baz~c|=z$nX}9ObmJEJL)TiaxlvBol3&dv zEURN6|80YJDWkvrJcs9g4;=k2xOyo#dkMIEF}e#Ep}TM)IDG-Q{T=jl&&TgcYIM~o zU-lNpCZPvA!5f=k*(uSqs!*(oy&XDMyzA^R%RZZUcR%vq|3k96=a|!tcjiu-9-guft@4jX6s!WkzYHU%94beW8;@=Wq6Dw z@4M^TOG*RJc4z|nt@M%Z3-HpewLE^L(M$VF(rT_wVZ5ZHGzHr$gKx+1BFtH5H~ic@ zWJZ!%$!1ag+eO$R`lqlbIL4Z^8QT&0!e|8UW39doK6|6e=B)dV{A*?4JDqC(BHo`! z?gRQ%Yj#S#&Mg;DoN@pjfHj1?Zz%63(mzk$ci~C3hR>Ysw)HN;jch3wAA_uIOTLpk zN%tOgg3GQGz&TKNK`&fRCC=kZVB-K0;tMhr{Bo$x2@saYFFGVanBNS<|XU~vjPp9n~Gtsw^l)pr}RTGY0)Aw|*>Et&&gL_|w*>_OS#JWC5dU)_- ztZQ$C^Q)pL%w=zuzlHhuCcV-+Y~It*+X(y+pE&StvpJtuynL75;>Gt?{QaZi@Eq~- zzWAV7JW=`^d!)y~{Vwcnp~boOYo44bhdP%AUHkNH;L36E_}j{*Zvo%t`7R9-rv6fx z^ijl*NY~?EjG4x_9_aGsig)ebIuu9S#P4;-fs@kx*g;sLkAhai#58Nc#DScDn153C zQvTQI`73P-WOtf2_IvsCl?_WrnRodt9a?|-?x~jZSRnA`_EwjF8*gZr#z_ABndiW_ zukbB3{kgI#x(eP*ZGV_C8$msg^N!GdF^zs!oJluDYgn%RBhnxfo~1owFo*Srbu}3; z$0gyN(*KJ8?54f@ir|YGd~{-u>Ks=h{nw55CIZ3xbs`~ zo;45_0N#9)wnRC*Dg`Fy zPnlQGMTE=WSK-s>%0eSvCVVCGa;?FVzXTs&IZ=CuQtloOFyBMOl}a|qUg~ns2J~Ll z85^$pepYiA{PPae@7j++H$9s13VFrn&r{kg^acA^;yD&eN6Jt60d%C6GESlg>W3ES z&kWMu*4t`+4B8Y%KSHu5?THRLZdun5FS_!5a8+eb=UcQx{eG$5GcD&<_7f)mloO+g zJHVGwMjAGwngdN0gR7MOTCa=?yfV_+!vMg*Omi%kwRTtv(VaTvz^B`HZTBo>TnEUa@^My?Kic*2=)rQO^Z(IyL83 zgUc%KQR*o8DlpNOYThrCe$1^W82l`p#Fz)Q9vUafXI|crIV~@`2k|44r z_?IT~28YP*0>8)T76})Slf+D!pPdkWZkliD5RbMd_zcNJlWAdu_81-HB>G%V*r9KF z>o$GXNjdVPyALRueHwH%dEO}fXI^^65A)&^|5W)}lY|2wQpQnhGdT7`@Tj`;V|MKmpFj6A^2Wu9Jwe7mOZf)YZt2dW_Wyjil2pzDEz`m_R-ae`F!1Z z?!0=4wrlM&;T|s{{&L!1-1r|6uf4j;-h!;7nMX9}c5qSp&o2L#ZQlxxOU5CYo%UQe zD*ljfm+V?KHp(wreC(N@8=h^?&{$CJRR(@OT~|X-&OZ>(Uzr2Hn+xBY2mhPT9<6}$ z)t*uLIg^in`CZ<}c*&kh=cD*JH|L|KKRkKrhBDXhvhYGak(cv~EeSm^7Wqg^R;Z=G z&t3BVLOXc4Zc7t3lS|>z=kQG88OdJd&=|{kF^_#<7JAF@>f`1WET4DBlz&a3?Jln6 z(awvg--Xok0_yu6>U}=^`Q#YwFk`B8O+1a*`>LFjf|@xyIsXj?aytUZ&CcpNV@p-p z#m1lfiFaf#UxA*$Smga3!%}1I)t0l9wdfY_-tPkFn9e-Pk(V9164~*c1AWs?x$gDs8K$Q|j_6_?d^sT!-A# za#E|6Z)@fm$WC+Ys|Zthq0ZFmhf_=X@0iLs^1lsTBV%3Z)~qu4`zIQF&@}fBPRZT2 zBSYB%ox(@I{px<#)=B-blCzQNRo3uO72|Pm-j+D+l&tFU)t}G5;${~=|H1!Oauz%V zK2-f!DI0Ngn+>g--!E!ibbIHD+|a)yTN#avFssLkLeU8MhF$ZXTgIO$Lu-rj@1A6} z)G&^TdGh$VZkj)kM)O4HLaJ*S{2+Uew60@Z>=#!UYi#3AN9raJ*hw8KGKPAKZSCu`PVmr$@E9GXQSCSe4YELLg);P zen)kR%Hy|HS%TF z2<;y}WPZ#ft&z|39U{H_H%b<$_tgy-b*jI0o^gxj)u)k|xu3z{c@cPC$hktcoyT06 zf?pr??J49_{vp5s{}jF-1aEiH*X@(w12sm>)u1mgJ&tb-g-ah;aT?3^h&9qd2B&-D z-SwY)E-3Fq=A>k3AMw8{HD8g_4uppbUpgF~0X_lQ(|cZ9>iCDo`eF8IhoG0@gAbG! zREamJgy!J?D0b}rqLDOb>0Z~0OwI=jmTP~phrNZ9ytklE&2wCw{P*R)rS+*+v+zZ8 zzbClj?k%_%)Z1I^0=I=Xng`}Sc*gU1zTYQYeRUPk-9N}jX}&E!OFX^SlgpFiD%`!r z9|^<0$>nK}+FS5X`uYy)P&v=N3t{2Et!Ndx=kbb+*#4;=Po}*^1@}u#vK;yK+iqmT zS#~vR(~3ifZ6Sw!U znvq{MH}QOtUX|9%a0+p(nbM`&V{Bi7_HE#(c9TobzcH3Dd|wTE2-@+Ezbg9s@xvEYK^KrAH2=O& z&Z4Rs*#!AIwi(|dL8B8P9#4A|@g&0Awg7$R*=z7G_M5WRI5AoT9;si>pv{T4NydN2 z73zy@`&82>;0^F?{#DO;$&!JVa}wVVhb>1wzP0bvUhP=@H`;3W1nwL&^P=38C0&!F z_2d%ix+4BJ4_ui7uh9dZqbIyaD!fq;9YO3+llUTsah3nqgrBCqUb%e?%&~hJm}{p4 zg+Ffx@n^^L5YHT*3wab?w_?h_wn~QDHDyavstU#; z)$Hz_vhALxZ*+FYy+QEzqPJV==W5eVg(DLr%*=nnx9#TKCTV~54~1<%5;l!|%HNJ} z(C+Gf7fkk%*59g5L@1LG4@BlXKqy9@y_xt&fqnVm0ow_w!;hH2iWiiH-aC+{k!?E z&TZ8%Grh3+UR@5cwp@x1QYnw_D!7z2?#J`;@OM&VUqf2Sfptc@`OduM=KiTvm7Q;2 zL0mjl`ZU3g%f0_k;Q#LW)T^qUb-D76+~w`7@fioLG~?{uoxQIv$GuM(*~D1t9p4>{ zIUl7Nxpt<9mlsh#*+;d3moGMiMoJcqqKc;u3fC{64WRJ(&`7;HOO+ zwJv316IBi_mnVEY71~kyOY=^C^n24r;i|?=bGQ$C zZ}6VS^MOBa#%A#FGR9eSa4vP zFWXo5h#lzY8Eapi3Jk{f&C9A%+j8-N^G>?eY-j=Hx%45|zUfw%{%s&k`vIl(UpvP6 z9r%!-U4h=N@0(u|rtvt*!xhQzGzSuXOqY4_m0o-zPIx9>R^R?o*EjDN@F$cR9Pcns zb%$Uw{X1|B{Yx=8lW#Akj<@=(=44vWXrT7^)vLAZ0+ zn+sKhX)drvHRGRbkj5Zi^WgY`FP;Y%p4#n?{JlDNEHWv>@M3)Hzw+Z5#INn%@}5<} z5OkBhq3S3tyct{&lK0A1r0`rk2llWX4s z6s4R>2cr4n-I=*e{ku`i0ZPtH2pbf|3FnaD^ z=-cjQFVE}bF*rP}n!4(|D9?W2+9UpKe@FZW;IPiX72nFYaCnn}S@x3#%Af6*uknrF zc5(P6(h7$~6aQNreiB^LdHw%34lfiAH~66OKIWvKxfx)NVrw$D2XZ!FRQkgS9#Z=R z&C6t5ju&%ofUcr&Di^%zgkStKGHQK480@Sb9mtsxM%QaNVHeRaMvic2v+;p;CAd@) z%`riKZBmEYw>-Fv@t%%1vZl*L#MH@NeMZ+Pigzrme0 zqeqoze}j5xZVdpIQlFmGRpp)Hh35m+A76BzG<|RApL33jJ6|X5lz+Q-)+XbQ^(FLA z>&M3b1-hsI6hik9ra96B{4)C_@vQs66OHrU@1d6jK8RlJpj}$m_Yo#qHU$~MdUNMO zuKgi#(&uiKE(Y{1F>VQ8M3TdxZ{Cv5*!_jE8yPq`T6M*s<$h#AQ~92{ad5QM7l@W^ z>KU!(4xxJKF`?71u?}8QwcMo1KQU^R1){#KJ)@E-3?W_U|FHJ%@ljRh{{Nm#LM93K z+!JULqmltFDj*_g1|kwbMInln_JCqbkXA*lCsR~TLs9XdYbPC0xljNB|;Ci2|&?tIED z!Q{JxgJX$>cLsBMJM$6KiT|4aYV%o>YQH>L_KTg7>MW5g!5z(WdV;gPz}ep5Y#(rz z*ofYEqZ^l>((X)8$hiaL#G0%$b*I@sBh7$DY5a@bwRxC4(X3i@h{NbR{n|bnf0#7* z=0nKU9niy8`gf4Ih`awj^vH29Yh?chx_bsi=*Wi@)IBZHEOL!Vh2elEf2!Smy- zKE$gyCSL6bGDGwJ#L$sCA8{PeP1PlZpb^_F&H#!2PJzDY{0H-I&b^CIWd2uL<>v$o z-1xwKH4oG!dB>`Rmmrql3StQ+qYohFlsyR;Q=F(eV*V&*)kN?R9exeG$UDBIe?6JR zn2S00)QsIDiI`H{uWM_(;wx1?`~0V#EGCZ77~N6BxbhPE&fiTPH`0%S^Uj5y&VB~$*r{WCC$wyV=j$_<|1>s=9cDeC&%N9@|I>^-yiuh_M3P>*8Q#IhJOK6M;hL@z#|Gh8HK}v2S*Cpz3d9^mx$dMU#smUk)Dtf@~^udzdV)&>o-or=z z>SHUgFS+^>wFe%JRth~UAQir=jQGZ5i^`bOY0T|Z=J*=s`fA{73VwQ5u?AP-r^nr7 zB?)HFuP+IVdNp#c|Fx3LjfdXr5jrxjk3UKreBn(ycZ7X9tpB-@A*sZ}Pdit45B7KO zwLgnC#e+97x8jHYj-RgPhZu*($6^_CSTqa2glTc_$fmCQXtXz_k|&=Y=RQLG8r-{U zdXX=Fr3SnbzUv!}SM);oe#qckhF#U|{VwL>X5M{f-eq^mq|Z>-Ecfm}`Kv+G*O|C8 zDfsT7qf;zc2miC}nXG~OFW;>X8N*KOH1Jv5#tpw1`jBPYjMtHcvh9z)>)ib!8Pn^| zw^8LP`ERt-zGdmvx}8)fr3O4=b8zXo}xGyIhsi-knC)W+OYKIxzH7 zYWv7lsaC_i{H`N!1V0ST!+X=d;OTuMD`naP1P8vXn5@*r&lq~OZ*=f!a9RB1zh-zI zyPt7v>fxJT0-v!)JT`e+Lkqtr@zQxM?i)xZF9Kht zI}Lusna^93tbVr<*Z=mUf8_N2O-Am2k)HcfM(#VL-+m<{_v#VF{!jV;!s{8iSB+R! zG_Rl4@1cGt?lcI{7fuG&O!g+F=pKT(l=Z)mkvmcM5l}z;)PWmmTk~XMO@nvRXYQfz zxs2->=3(Z2v%Xkr>^3=uW_*O-4C9xaxAH6Gyp`SMJk=+;U-kyY_;PHgx+5gb9>Q4_ z;cA$1=fP{d!TgBtkv%4sH;X*M)qMI?1`RN@>fK9#Db<^6F9ByNjZBeaucwUov?oZl z9~V+rj`Cr|8xu_MpH27x(j+ zIY76japodp<|1P*XH0*;L1#k0K1P7-CW@;pJUj><#_rydACTzNNqC6gNOwGJ%z{_J zUamR$h%*X0Gd&9adjLP|M-A`uThEWX5gQkB_u;Gf0FBlih&m^V4gGLoLg-A*Eq*H% z_yZiqpGWm*|Mn$RKWWayq5GIz_+HK}i(jq1p>}O@|FM5$1v+Zwz=h`dI zqaL%TBR^!jc|%n9FwU0@vZ5Qk;O;Q_y^{__?!oq4w79-j_iSgGy@LCJ_)cK3|Tfn`!^g;f?MpnaKmPy<@`KQRS%TucQV(oio!X!ROL%TV-E& z?q0mK?j!1yPsMKSL$2Q}`-yxcA4XrJdHx4&@rW#TIvaUo=zY(hwh)wv@^L=Xbm)=KBgmxmEB0$ zrz!hJ0{VwkyBFnVLA$h;dE>45TcB$e>-b~(XY?apYr`q?|G9N$TyQ6HtMm%p=U(@Z zeyQwtt>DuxZ!n!YyO$RYt0C`EaE#b0!S$SR68wth?V~@d`&skjfEnTY)zp1}ykho@ zGu*s^{%nH)4DPHd7Q9t6BKYi8m0cTvb zJk6SNS-;{JF6+1K`2z0BnKzbpjBMXC!rGUu=H8uI;7<$RRI?9`<=&k$sIP!?T@yXN zw;xPSu2H>J{7$NqyLVKdpSyR)y7g(#i}kyjZ)?Dp$qCl5Q~TYt!xPM%`5Q@7w+s2+6m)9h+;Z zH+?d4_q+westxVaL(l`b^`dtpqN6!qc{21CptiQ)ZE3^-s z!?z9(EIqW&(4sSCt>yEhGbgRkC;5~eX07*9M`s7;bEN023WQGAR@?~I7<%sTI`X|t zH9l$jR`9i-?=_c-L8P^q*^TFsj2O#v)+fgv&ibGuyHIq*$6W7R;PTY*IyaJO7X!bh zK4U-0wC|++C-^6+F3nM_KE>ep47&3_*R!{OgHJ@1Jze@s#V=9phL{h_OzM_=^F5QM z+cQYTw>OsLY@7y#DrvkE%K;u>o$n1|K#x5U13Cjf=&(0? zaJaYp#K!H&{;yS>P06q?Gi_wpW7vy}|IND@bJ6(~d2@|EY(Ub7)R$_vxIT!QPw^*O z=Sx`onfM6&1-c{NK{~U;AH{``omYBv?`l=mn4Q?C!yWj@Vw(99Ch2QSC`L$S^vuIKPON28}0i>2MoSyy(`Rm$9%JZ zb@CRHCw~I@709RH2>8?FJ^4=20|qCu?8UU}+F{ql^tRn~wvwY_GExy`cY zkve`h>2|rvOS6Aq(p39qlm0J!Q>vuziARr-fIfpetfnWT-@pfM5jvlexZokyS@ui$ zsfd0k{*h$rDbycZSNU@^@cTJydx&@vi&&$3AJrX7!~!t=$goRHnrV-9`!$NOXdjfn zQqp0P)v zm^{fr2L4&67;dJSwaT;ynm*{vonx!in)Nd8GVF}5cic-2p5M&79_C$|-G_G_tm`ay zU87%G*K~W4S=V$sk+MJj8a&?AwQu8x=6-W{x^NSIgw$ErRQn5)r*)jjx|Fg$6IiFq zS+DWvo$!sTa`aC7N56z#)PsIB4jr`@9rahQY3G)n@0c_7oULEM8T(r}Yd;y@>>SSC z_u>q`bk#>)|3}G#!g0k9jK#jHW-st{>1EO-4|bikACZnef%C?>%-fyBq_~#faDG|* z4&!6@0l(0oK;sI^tQ`_)oJ{)sus~x^QX4;O?mU=R&%GHB_O(W;j+c-d#dqm0je24X z;rBG61Nyx0Wvj8}Ig9xDR`gYPh~}ZaLys_5vin4aTosxL|J~kAmP|sHJTif}0N&h4 z2{P(xoqZ<#VZr-34=%QAXB52k$i#<*z>T^HyrD=27l8&TqPTHOiZ#Z!_(?+`OMCZ?^K%?AzSDrOJc8bxelFVf#>A z>LR6cwC|J7R{TEFnXdf$GH2*ZB#VZ+{JMnm^6Q#yWY<)CmisJQS$F|)v>B^n#T)tc zxgMb`$*=V5I#VvokzeW4RQLH!Yzt4mcSuD32#&=1g2LK8cNverAi z!Gp6M`qhK-v3bicWlayG3z=YzAcjd){EXJ*UTA{&U?-kow!M?}Kf>DWX1seiTk)D= zbmI@wUdVZrp?uHTly=4?SvYBNR&H4Ek!Qg}Cru8E_h4ObcI)(gzGE(-`ER+7V*xSO7N%53q&tM-YrC~6kc3O-TF@F zEpirT0avMZDS6=v=6YnHF$taJgRFamzPAjG_eZd?NBUUNo4q+TJ*=FX7UE0^p4$d_ z{J^_83+%*J!rlt(d<-1zE#$r1zX2{B3HA% zYi%~Xex~f_ar2wOkuSmDeF5^ zzSqPwBxZk3qzbuR@E|@g1`k??EPM+D6I;lS!Gs@}XaOENhQ#}EUU;Vkco2--?9Bou zvi{dFG2exW*xG2VwBFLYn0f4V-y@pC7GQ4DY35M6jqY<8NH2cj@`1~q*F45_8?kX) z?i}X3`i;^|XAZm1pI|#Sf0{eZ;i>btnfc?4Y_O=?+{r&ebGPs-=1%*^>E=%3(%i-7 zt(kkwTNVJP;8XKjkH3vGvtNf(1>NRP^CtM#92Rr8T|Kx{%lRwebsTHB#yykf<4l_F zq7&T%ZjGFN1X|L_d-)d0X9)e?y4l@yb_K*1*sM4az=8Nc(YN-|&?v9-z4>MYxDL&r zy~gl<%YTr*z;8B6&$<&Dum>9O34GgL#&$FP*4f>sp+S|{$D8g1mWGN(=h!zBzhZMb z{id96@hi0%pWe4aw7rma#lLEv^Z9=vHiR&`gfP5X7&_C;zNLNtMuV%VZ#VPK@Bt63 z&Y8(#l?m@Yx%NoPEw=Lx)b>}6?o9RE8xi`HIu zWr!HH{-!Q^ha5p4bbd}TF7~dP9MV|tVJyO>7<^{}*Oky7wY#6aL^==6)1JD^L-*g8 zTHC$-9%_Hc_>gI9ZJhsLs@1qB)8ee1(^euh=R5dXIW!;H+Fkc3baX3oz`Y5yBl|Mt zr>k!RY@Mt7n7RIhxmMh%2R`UrrSD7~^h0n{%9=zdqj*4JbUQOxpD=J58qK|l+6RX8 z4YjNvA8M(wqB#3}9zgzUnPUY52Ob_1Ddt|V7yE}q3-+x|4Mko|3+;Q6^Sjj5GTYPG zImgpTtmnqg+dPe*-syRL-vZC;kwu=@TPi%SH&=OH4?o~}9euFbb2Hrg=>6Ba>kb?Z z<}FN1GnUH*%-{FoD;V-|hC zo>)nz_R%vCI4>L@N}E4$ns}x-;CeUh?V+#p(L>HUO01Am=EqY2uDZOC`qBNHoB2k1 z#m9-;DSq+rNO+GWx_6lK;f(#%y1cY6oM_m0-wws}jI(?-;lEiS=974T<3pMk^+mcv z&i-&`0{uyu(4}`2t?)3xb&e&Bk7Q?OQmAvf1zodVzX!QDV?S`moTV68MW4XjE3Sa=yj5NI zBq!y|k1_s0Huq$Pm^}Dq<}F-Y*%$_Hb?;GAVc?kcYs=5fLc9pC9t z;bs0`V-x&U3hN1flY$*%()WhcmVo2ogOcHjds#%oX&6vK@Dpcv(YqC3Ae}sFKiC5ifK~3cAFe z_(pnz9j;#KW9I#Iy5oy!r&VwYK8-YOrrRS-nqdzkRX=s7<{^09@?5KITCUYFf?rN9 z>z6`oI_A5P`#3Mc_bac$i7kSRXX5vGJ;WW*nl3*#Sm4@pcTIKb+{zp@OdqH7Y3t-K zTJHFZu5?kEnraVo5cH;D1XUxJlx_U`WiE3-KF z5ypQll>Bav|L z&Kg|XWrGAJ1eD|+qFPx4$`y_vkZl&cvS)Qa&-&j6Pjc+J^lKq>yHq|=r?DT&MmbA*E@ULj>{&VXclbun(@8I) z?GrR1*Pd$L;VZ0^{#~V%@myi@iF;{M(S|F)^~vD;ByfKsG@umy^aN;wb2hP%d81ug1>_clJ<@a6L1UMZX`08hJrD8SGZJBF;BI ziL-b&_n){>J{aioq>EeW>o*qNhjpL{n`-xaVNTnA_`G1#HSn5lo7iFx#3uz7KJE6q z`}e2z<<_t0i|#fkJ;=)j8(pm;QRJiULm@7&K);EcN=!_+sIEg z%m(OfZAC+EIq?j9oP+OR&Q4$LoI#-XohPEAhGJFKMv1_*Di|h3|JlJCf zU$X7pHDg1<dEhr~2$%2f%EJ~#+snzDPu?r;d>yCnyXm{|PW4ES zsk0)_Qf?0A%Dv8c5S^6?7edd;H)leyXB)dRHf7@zUgy}XKVbe<#}62fY{{ixK?}Ou zl8?FHT*o*1wv2QX?HF0l*Fa-?zK+B*UXeV-Eh%l_y(jwk`O5gS;s4 zUBq6T1K;o}^S4NPzaL#(8z)(uwgk(9;g=|@a^@ZL7kl?R-o3~>E}7B2vD>V4#bCup z!wYW02cCfQWn)g9c^Y7!VPJKD{Ts%tSkPYu->bX!x&iih_nTkxjoN>TbPVlluI1mM zwf4H{&rO+3`%#l-*g=z~+Yg&G&HkxLQ|%g)4gk(41Gkfa--*C+Deydjz3*~j5juO{ zH~Hwa1Lsam^waxP+~u}=he^}z+ez_Dp86=TaTmX9`He!B4b_be*{n$iX9_#!c|$dn z(HX-CGS&0UZ>4-JGAl1>AJQW<30-c>yu?rhUcHI9P0fYL&iTI?R&)mRVitZ7+VkqfImCHi{Y^~)=z z-`Z<#qHf6)U1#21AB!>Mjnh|*wZa;rV`d)vh{`GD@e@~uZ;T5IOWwqVFyF4HE>wY@JoNbp@aEJXdVu|#%Hn=kR zNwm9(b&tte-RXA;v}kty`9aQS20cp?II|RI#}RvBO>bl^cPz2bVc#s^o3YZB zX4!{`muTVt_O;(U#y4a6=0@k6FyG7q7bEn46!3Xb1v=cXXw!RcAR65b{K88e;l8eF z=K83)Bf|B&9^}N*{aCtGXz?K4_xTSee(hJ*d6rXWOD<>lC?8(Sy%;qm(q|i9NBV|% z*0Ot@9p6JIY;?eCS3cN^U1{zrpPG2KBj+f-cz7Ohf$mHSnR;(a<~&|XsAXQyP?VT= z1{d#i+K6-U{R-ArGLrVfxF@(%6ne7DtC-WA{dZ%z6dSti>Nps$@O^Ms9{7$<{4(;+ zBJX7vMt=)_ieE~mPl8YJx<#}l+&J5$srH$qXYt+oa4b+6Ygq=zc3Xb9R?>PZ@mUbH`VQI1f38{oS(={$++_k;h>h~}Z0er!b12KMY# znyW>`Om)wS$iGK-(`=1Pixgs;|0pgivK#-NTNdU`n~X2dUHJFB6_@q)YV3DK_y zm+X35(Fr$C#^nX} zIrc&~{~_hyMn1GaJoQ~}{!f%Y51wx}Kl%79U~ZM(iEoe6+u*B|&Vvt<&(9p}lG6jx zQQR%3{@p@7iUa0F_oKf3zS?x-hjq5%`K-h#;kqU89DSipIhJcb?MGh8H3j8;3D`r`dMnm8i-!5lyO|9 zlyQwyIvd>^sm@sF8>L^WJCN8Fck1Ys-RfpP$>v+v|L3IhEIY5%_|xo8Z{ok@|8od= zvhx88Te9qXPRtT>HlMowG0Wkn2hyh2zLJ;?(P6|z<4j%5ujhQxOKd}jh5!et@xfy^ zIN$WuSVvpLV6zT{6UR1M+==SMbAqmxU~fCZS@&u9LSTz-hbQY83C|+B%e*g!Ug*q< z)^{QQWnV%6VQkZ-N}0oE=2bBJ25ZQ>ND^txKJDj!mOH;%lfpP+6A>@6d8qSEkzj*% zX2;r@LL3LRLw|gV$3c5rp|8!@q!p_{_HVTzc|^QXGbuWOsq(w&UT?X47O3~iSiK_y z(MkOGQMQ@7o6)5T?su?j81^_~J*;7_+vBW}@(CEl`HSk<`J&i2 zC)riA?L(Yp)| z8_j)A;K`mL*xtCyU@vp_ZFH9!oAd#~mnPQg5O}E=%{t>HK2Um!L(Ka|Yz3{u)b;>- zANkrBPo1aO9Kr9bjLEq1^r?utp#h^lshbcwSm^N!&Z#%rj-6CIv$0J8cib@*nhxyB zC*VT*Bp&ZPJyZT&cq;K#YX8r)ud)Ac+SiXJJ$*1Aqkba>Em&6`82Fg54F5(MR(tcFT(%0$=V{;2&By;?#L>p-u7cvH4%iy3A8NsB^Jn z4`km0*3<6IGq(69V!#;NI`b%d_D|cfTYc*!yHyuldJ3RNu1!7$58c0A!ncM_76%$z z!Dsl^FZE6L>&5!j9p+`Dm+icjwP>xg;9KF9fp-HBw_$gOPC2}Bs(m)|)jnP4gtF|u zoRMBboNRN}(bHvn=<2g<_TifGAycR71ApYJ)V(g%bKHex;t)3uS3HCsTB8B>arODGM7Yi00qS}*pI#`;~I zW?Wkt*XiiNsjw3poBCtc2)IncM!ttW$tNrPXtzEAOY(*7I*Y})>gm%OWG}7s zOLq|u?b%MOe$L7ckHNGl2W)6hij{ebG8V90uR9lGKDHAoFEKdO&$>o1@?+*wdVnzF zO()K{_L|!THxB><;O>ez<6pZy*|YhJ2sj>D>oNXQ@~gc$E**c$^qOY&m*$zSf30Z1 z6#6leerP@yu*VD$yrkYBezc zTa6D-mi^{r_ATn%OnNsyJSz87=5{F0ii>*mwj^{Vr;m*q(}_qnk^c7xHvgl@CW;Rs zxw0CXb3!)Bk!-RpcE3kz5_>ZHlFr)Yp`)mV=U<54^*j8=@XO_gUt~iAaz+ENQ>A{w zXQ*ru-{|alv+iT%e&HHqh+B{$rg8RMXLIT~j~;=>Xk992IS$`EGk51Ye*K2GzdbqVaeeOx&>ziPEBI$( zx6_BKO~0}KDW#8BCtlCSKIiSN+xS8IBx`6Yjv2XS62+xW;v43IJ^{Mn~EQT`Bhi)h@D51|Ia~a zf%uT_xS!^OE<dVm-dhM-=%o~=88EF@i}LX+OW<1 zrH9q89v`nh8?tjdJ=XlYGi!4sWBEMkk!Qfua-E?%fiK&vyHdK&9*m(r^?eHSA4`AJ zhx6+6K{~6!2`iqIOy&3Vj%?v9@D+vIa%vVg*4|R^rzfSGFCwoLzZA)FTe;t6e@PGj zk;1spXDL?n(>$wjU!LesmMxqXKHNlm@_i8QO?LCXr!g7Ym1bY==3Q^f#fN6Nbjs(_ zsjZ?Fpq1JlyGW)MeRbWC*3+#}wBO`CEvxGM@;y3X1d7yK*z+l2KLKClbNP$UI#)?IqVmx?kIvWhlNR10_zs{j&AJaeWSF@6>hF>3++@rg! z=@)C_^h@X6v+XlEUmt)TzI-GwYUcxi#$DKv#(lRJv^C9sbh;D2dKGufbJh|1CjGJQ z8{zIz;B!xk__Q3uOG|b-_E)Do_q~j~7J2#CqnVeyKx2X@d$PVQV6L=ZnETjf1R9ae zrmo>mzop#icN6!?mvE=wncV4@z@2`x-*xI5>9s}*@36C${arFRV|>CLpEb5je4(BCC4(PktmulI}7x%O_7@L7bZ`yCb8UE*pY)+WhNiVb z2W1o2xqj(ucO`m4yAvJ#lyxi+eM4t?9zQ$S{G_wa^XP->XHRL^$etp3IFI=n;CVb} zwW~W7ujBV-MzEWdC0A^yjqy3)by*Yg#7ic|Y>s^e_+;Q+dXD6H zd+-~qmuT5g@(irgzt`Nn!Q>e@pnosBd1sL)9ZC~@{H>dpPoC(Y`gvapF&Fub`%v4OQd&A>Fxb%c@s6uUMJmfOl8l zy?y$z&JE~hh!a;iRXmmctBk{!XWDnW-<$!y8QK9|x{LNTe|MNP)xO=N+RILV&NX%p zT>aE}s6(t%8}s@K=TAQ(F1{~NIrVq_OBY5N!$f#q|Jd_`MIQ84=La?p@%?OS%FkZ7 zBhQmEulj|Druv=@-4XDVE-WhG+{(Q<{pC;R9~cm|XJ8lc`8tcyy#^PnJq)!tJ zc5gosozfwA8^wSHUmFvwWY@P>bIzG#=Kd4XcG|h-40xNC&f%KVDYPSfoBnHmIH^wb zVkhd$GWG3BEP?M)eI8TarPMVmeDYjKf2%cW>!!awG{;@1)R;eey5UcxzYQIB*5?AV z*M8ldPR^MD=?}C={Bi<&fSJQ=`=`LA=)r38L=P62{7m}+H~$IpCI5+TagX4Lsmvof zk?n_%cWxkV?RMpFgim@biMYA^Zorqkv|>Wk$NzTpsNxOA%yjmPpHP=*;yB(vtaUQ+ zD~JtC9nu*tB=4VYzYaqiv@c{W_|`P-nZkiHDR(1f;JctXnf7OHnI!roo%X3^ucPdH z^wY70s}3hV=+)$(Y6~~#L^vPPMe~Ub+Kk+A;vT3B`%OcOxLcla=!^(>eA?J0-Oq@8 z?0$18-)J0TNH1g@k`+gpJbeC5nre@5%bf1K!HKg5x^I>{7QCKd{UH3FkjvJt9~XK7 zp0X2o{B(%b7+LQLwSeRJPKU5BJ*HUjvAe&T3mcI|J;4 zy`;xGu{anX&lFBLu%o+3T38P=X6)c$;x#KqMk{!6cRc4HSs%&OYmnhX^yQWP<3j7{ z+s0{QL)CFvw$GzGQfDEXSqILX zP0;l-f~6hR#H!mj(ie(w#v_WH9G!vgWJzPKPySfN_!1fUDA(T1I_fM`9ebhlxS8e* zc)I~HwjpgtZM@R9cvbPg^+SO&v zb;}A*9^k+5khR5Ld^zhqo^?leIkN;f7>B>s*idufA5Dyi;tFg2TlB9u)4F38{ZFPn z==;>Z%%7Rpheijlr9S1!-m80Pg?}n%VB_g?gGubK#mN4?q=2yni+_-A>Yv1!YMBgN z&aoN?BMWI9Td{9a?>6hqF~MT$UBg-y@LaHQOb~-I^flW)ex^e|?&iIC*UiK?d<&YV z?-jSNQr~ymlSE_apRtcV$h(;BuMoZ4?BVg83ymk1Vj{jy+4d{Mc0^7buC}(icG2$N zWG|J@C)>Wp%we`YmGmCgtD?6xKgyWO(Ua2F_PLBL0z5De+mr7Ztk{CVin!#+)7Xje z?+98}QVo1Y#KW`V0&MbTw?DfD9)B7(6X4RwjuVi}T)o?9+R-?$l{9=8Kceb6mq){e zw1-Y=3jB8fy;>W3iZwet@8oi~w`lmc)ko=aS(o}P*Kd>^zKyvzviZAxf_ujcF0BKq zD;``s3a|1JvXJik{ujK--=Nd8NJmn?boZ(|-mN>6x-~~p_EjT$3?<$MdHC+0)RyiK zKFWM)j0LQX;5CX);uGxh<-~WaxRmyoCm;8>)?lxGh+m#3HL~@CMs1vXGkQ$+t)swV%X1UBpERv@(&8nLv`@+MH`n#V zCp5J-e47-vNarg$17eLRi2)AXQwr)eiP&~PsH z%sfw{1J{=XKYt4v%=-L+`PUv>@BL1Dc!FR%*S-e1S?@OS&XL)3?Tgqm9Qh6(bNBtr zycaxZ-#E|gDeO)7+3xmo##Xc!*wLBePgtM5>@#2F$Lh$iSM#mtz<-k7muU@`oF&|8 zXAO0~t@!O<@D3Yl*Lt=}z)0mU6dI3H%58LD7-Lj7RfW*$e$wF>A@Zt)D%_ zzrNxe?%PPMt$*&q(Ap_O{aQ=go7MiOC26&q*S57i@xM*2ZE@%ILHf~B*PFR#PX8D4 zdBp|JT-`-qMF-6M5<~fMcYbdrUvspc`PJNZxUgGJer#@cg1a%E_6zs@EZ(=q;F|e; zpEBs{I5+AE)z!tl!+q|-BX?S%kC_ukk0qHrgSp0j%)AaLi0P(<8);mR<+IZS-S1AW@V?>Q-Z!^({NCpZ2pF0+5<1W)64&jKH<0fV=Kn|kL< zpEoKm0U2(PRrb^%i%{n9NYW|h=LMJY|L6KP=Q`JAA5v4Uhr|MnDh|KfskMhV7$ zx7B(~>nHqJ^DWEVz3t%hnBdPTS4m!MpVIztZK2b@zwk~l*a|*M7rR+?=JP%9vW5T6 ztiNJ+w6S;i=;t%T@yTQE4uDU)9zZub=$rPCer6Ae;oH+S7l!@|Tq~!JdT^}?er?(L zPG4-=>y9UXWlW+q&$@k#;mLU}9{iI07@lNy!;>QS{V#YQqf34MO&pm9Ju>;+cWUT_ZZ*VEUtrV?7OJpdbZ+0q35vj@QI zl`TuEyksAK*+0DZoQ)5ey||~@i~TNM=pNsl*bBZzTPd{l!yZc>DIg}0+A09QdVz<% zJxOm*`O(5h`dEExmJt((xC2v`^GhK9knmA@Q1+tW3*gy)eipK01@-2?(OLGnx0ihY zT{BchF;Lj=dL-?N0y?ywmw|zG8hr}7NmKnMb=D|fKK;W_T>G5pakx{muod0iMTZsV7F!;$BpH1#u+@xyArfVRy%ZW|e_;Qdi(-cyBn!Aj037#hf!HdCfMEquhC>))z;xBc!? zcP^hN-{BQm{8J=y8s#c23t9ft2A7AtC?048ThDi6M zxz@gy$y{^yRPWea=h{D^PSLpKCLLfeBaP9|^0)!XhPET$T!<}N{9qgPt$~OAJN$PN zI1;1HmWTLC7nOgjDaATgggz;0peGoHHcFO--hN&%#tLRq_s-|WQB(Dq$mg?_JM z++R6n&Q+W`<`2es9=k?krjHu)x7;zG&o~{uA#2V5Jnsp9T=AKu_fnrV=DUy`{z3cj zbje!RY|3lRG@shT(n;?++L>v16X@YQ`1PL9!5=|aKJ%u${V;ra(`k5Eb;e z^7Mip_JaTD7g_VC&a#Se&N=A5(As{K*-u z9%OA2h#~(g;O1|hzBPS)eQOe+m%oB1+D{C1$z1#4l`8md&5_QsKKBgWxZmsneGI*P zA9~pa3<@u=+tRsV0M7;o$6OlxCG(t@7&u`00v*0ed>-OTJ7y)9a-I3|sq#6?_mp-> zo=wVMMl7e&P=zNcl62XcsR7T#j%M_9tzpE4x+!8 zjbHb{=+4Xz^!G=QeT2t`KXT6=K>yLvuveNm3fT5dnHz!6dh8hNr}I7he{XQ|v`xno z=ZAl>I){5-?zjVP zo5`2lFZc|{Wkj}c_Ct3ptnY(wMOY``vtvJ*>9rlk_6sT*LY$ckI|neR)=Dqz)Ke zzvOoOGIH&Ixp|IXMvncy`z*hV`pZYuDCU^@wB60$!uqUVGM~IO`!8iod*wk_wkYM>H%LW)oAL85-`Ba}`Z%MfT-VM2WuD!+}Gp9W>79)@PEUve9L9AH07AI&<@zh~7paQlo>`nN_YefyO^gqouuIn+hDH*X&VCMmfY92pp|$ufJwLJ}^qUe#@BJC= z?|a!3iqK{=ZSH}Vehf|h1itxH9r)MY%e5t$d$|(rPgeC0?O6@1KAaZXg-%B?phL{> zdg$g$&{X+DgBOHYTmszB)Fe4WOuG!aGS@nP z&EP=Cjo@fogK*7H$;ewHyJuR4;ov6owl64YZ7#L5;ojq-bu&#G`_3F zjQd7X-gn`knb$zSpvTn#)Y8vSSgZ16=9Rj_{jD-n=Bc<~wL4bXk}qOSAE8`sf_0&E z-J<=XvC@ZYe&^A~ymQgn@I1?-eV4i@Gnui@W2`@7thX}Op^UW;W8LKGGw-L2wHIUE z?74E`rwG3;kA`KzZ-@(Z5cgXd|C>?IoKChEH>>9U29L925P&a^`N*6NN_k9g#5 z1-?E(@F2RL{^{(J?9lSlIXq)Z==j{M-OMxbl8t=^-fe@*PE24g ze35*3^a!-~PyE7bIme^)IqWD(7x7+c9dr=-dpmPSJj|&1Zt;n%CnO zGcfSQ8fdffUQOg))NX4bST%FKynoP>;^<^X3f|#W@ek;K0d<&h&~6)iATh(zRF8OZ zwRt_~1g}5V+2C6}>k%Km2NQ{XShP4{ny+ADO3IH0OeDJUB-F$mU)LPP^RZ1>JKkK2ft*i$-j|(qLgpK*;a%4fzIjl;|Zq*#{yq} zhNj8Sp^!0g_IX5XZSbw)+}!FMKk=|whr!)(a`my}t19tjXdZH1h_j;D(K5_Bg_Eoe zJJ4Uh#2&1^VTTw#iT{2*vrb|5V69U1bG#bd8_PV!aGHaphOZDEIrY1_#}h}IcJ_FKuh5QY z#P5>FE^KFxcbn>_=#&C-MYkCubr$*0k*gn;H`9E;1T~*j-3!4V8ibyFP@-tOER`Fblg)e z!xmP8Eo>aNu(8;}{!p(tO`EDR+wJ4n=fSt~t@ezAj=O}Q}Jak&<7UY9{^y4Pxw>zwxxf#%-%U67{`n4bsm2A?yDiEE^ zcU96YFr}Gx~s;$5tS&uxFt^1=Dx1ax8mpE`Noz^cHd-;-oI52A7 zrJ8sGN#Mk%Lwklq-&^O=KkD1gMM$0|~5R~%omBXp#5gRxJ7+YbS+ zhi6)te_Wg9o!B&Gy95vK*Si_cJMm4ckQJMNy9)AE{tV6^d}-XJiN-ejrp8Kt_R$xe zBW^C72Jes*S;kzAPCglz%zO<`4~})|!uc*d%oOf3UoER;+x10Cn6nn(Au^A6;)|1O zTc%mWLQjfRGY?4>3Df)q*QX3-E}gL_*ESDH3h8WVHF~us#`@`%jxJ#a^&6gwF^1i| z>$FD9gT}Yp&AUeBs**!x(7*6Z=%LaXz#_b!&Nrtqzp8h%nU6H%8*J>w=%&p(>L2dD zD-@k!P2}qm9__xTt#jP>H>DAi+IyOLS(6cb!`;6kmpXWo%DjlyC}y(GpB7=u_G5o% zuR7rKtbS5)Pr{GVPs?g(B2A?Yow3oqByZ9W#SYDP>u6#;z0KGI(Vj{vm#lHSdrA}a zzbzdV_57zhFDd3d`#(6R{GAU^3hiR7<^O`efqyW0)U!)-Nj*0yf1q`t+7*s{!8nY3 z^u5WAMVxgD=Zin)W11(u|KvQ`5Yh&ly%qY=%$gUU&tA{|UyXmP`F`mw zql(}G(7E5CGrv|+z>(iQnaS0efiGI1N5o=o+~mz@m+V!5jMZEP{^dAwpLIU6ZH*Ou zhyJvZU)dX89~k%x&(-~*uYC7E%u^?Mos{cbZ#Ax_UeR;xB&Oc*dSp51T&dTZTI$## z4F8c1ozof!W;7qaWzNK(rK`S2t;X`>+)wxYoc8w5X5QH?3 z2OWLewt}mDLGky-UnK@x@!;GXU;vm3Pr=_@>3YU-tYk-0XUUEhWWPxsUrj4Im~zhO z$v3}+n6jlytDlsgg!nMdl9bVhZPF3^P_S35@mP_q*rLJpb=J|$;MMFK(kb6VyGeBA$`ISP+tjhLf~ z4C{`~&@R>jJC`-0a|-J?-HI+zU8H(eUwy7`Lz3cQC2=PU^112j^~u2nZ#uS>E7e!Z zTRp7??Wej^>xZg`{%fCG>g)q6B?D%#{;@e2;?67}2=>{Fh!l&yyDMEq%<)ZCk$8&_-KEU%n$Dq;dn?`MQy|^_->ovU~@2Gv-+P zDQ^2(gNavJWitv4o$2all<%~+B~i}NH)h*Cs8{>q0mj#b&voE4bnw;>J6Az_ruJg~ zsq?@t@L{v`5myWgdYG5@e(S8QhcSrGMlT@+e}wx7`Ts2bu1)YUJ6A&^R*6Pr+b_7X ze=F~WgW(zjGY3R><B+_duiP!dT=zX7e+j{Rp7d(`FOTzGljpf;KJrXi8h6p3 z1%9ax>9?!Zw)8imEx@PFEryX@3_L-bsyRDh*4UR4d~C2YC$G^Tt#NE^o)ul7z6|Ju z=DY1-two0Y17u}Kx1V9x@n1M0KDLSXtNBLyG3hvp<>O9TIUFAwxr$ zuQ0r7izADyD|j{~co18H=!c(iiHGaNrvOFAh4>FnRX<*&ZHM1pZ+wc)8sQH#n)3PV zAMk+d3W|CMRmYKBXO8aVo#fywzOB@ncJu$y95F8A|MM8{VzwJEveC)@Klo(9$IY5y z4KIrMq++|VmNLK8T}B(4i}D+wJEYTizE?OWn2>D9oYKzrW}YQ8L`YBa|A9|C+5aci z*lzEj&-$i_J3@;u;oK!~gD=cF1j&ascG$fh1 z=s>qE9^)wZrhZQ3-WJBNQhv}+pXJO=GGoxZ%wr6dr?DOE!+-GK@o7M>icKNa_%w(= z(RkE{59xz!#;2bz$8P*7{n*Fv9DMx#9oz9H&#AWK)W{p~lY-wP;EwcYMt=sq+Lbsh zw0lk~ojIP`RLdSR z9@|ybcQ$Ul%o-6XzAp6XY~&pHf9+GkJNDIEwCDHi_h?Qz^P7wxPUMKE@iWHwSts!n zyq;)tq9;0JaqrqliYNM6ZXA9|ag8&wq$yulX)kf$4IV+GjfzEiEG?9A+B-% zzyf)sXkF z$HayBjQ+Na&JH#2pv+p{y~sD~J&i3ddm2~JhvvFm?9RyHd+}S{!Ta|-jrEL!`%fG{ zlHK@`yzF)SNLD#%=V4DH`v^YhX9dSn#}sI-c!b-SH_<}rC8dvRWgpUB5}o9{pTm2{ zHprPKch6~qFA!XR?w0uv${eB&<-JFlPHdv3v>`dvhks0yY@%Bdf=#uKo>Y6%9>(p& z6x00y>GmComLoUte=PM$)+{z@rY#+a_7xvGtC#6#tdHB=_Al*fKh1vmls^8^Ei-~L z&Nwpc-%zH>YejeMPIUS-#n~sFc{Y5Pc3pz$eqil3i(a{KGty1&(} z{Z*&5|D0PUy<7V~JGK3XPv8E)Y-f+xTpS&(KBwCqq{9c}hX-$r&-L(Dvlhh2aN9Y; zduJ`s3Db_o-1?}qrpw(jUr_bSC%nZg|%@|8=q|NeOAsw?U|X+rjB z1D+l`6aIkjOG%ARig{^<7PLSMw09|%%RyqfM4r1M6d9Biiil<{Kpr0i9hktrGAJk1 z4vi8o9)?C03BSB?jRzCs8pA^;h9c0bFtn<(53~y!)d`J~O%9(*vqxn?KktEd{tOt% z&xdxd$NpdHjOAAKa{+YoThPrC=;lDqsnhlV*MI2;ycaE=H4u2we3S&DKLTb9PV?-= zcUtEVt|zY!{4`~l51l(GAy0DFJM4SAsl&VjmoMYp7`+32|J3c>5qMIaL%Esv!}VV6 z4Ry|Ngn1S(DLc9E*w`I&FSUZhk6pH-e#gj=c#Ha~toCa8z6~TU?8wSX)_8w`n_K17 zdT(aCbWnBTjj8XK;G)6(Zhek9O+QcVl)&wjHdmE zz?-JJYeP-IP#73$0)7+=sAXtY=-{I~uktkdTGi|qk38L!|zwg=qimiceW#P)#eC?oz)exSRd z`+K0Xt<&LUYaROEmU|sOGTb}N*hFWa%!tvsbo))}5xv`N(hR!-{YX5p<$S9%U5YiC zW$umHJpouoml>`?eh2qW8UF8jRP;2}F3a*fCY^xv;Ln@-(rufx0~@?}AKhPG=e-Vl z-1PS58e*N4SfdQjt$x-}53sn0x;|!|KLPLLClz@=cD5_aUgg&LbJB8j(9%)hs|OE! zf?n={p6<<@5|V%Ma_FVzQ91zeF=Nfy-`wO1FFY}OOfK|RV_0hH&9*C)GLMUuKFqmt zrBk>oR4IL1pp<_8m~>M|XXZud;Lo8>(bUJO-_L)YHD;{+1&{Mo9+;Y|6d0eQly&$% z>38IBQS$XMXJy~hohrJ^S?48n&uc99qkI(M)7Hh}scbrYd{qE>f5YD{3~iW!&5C#O zV>PmDWxrs5+7#a*e~S|2yL#-BdFY^a%EuxX8Xfb!$P>MGeJ`+CcKKcma?0E&-wV#a z$9yk#d^^SXUX*{8@5QvF5OJ}i7fFXQY@G44*af`G|3ZC~US$oir2SfOt}>M-K0G`# z_6GT0jAu^ed(lx zrkVV?@z7!K^!6gs9DdMV_*vjoGA8>=IcXWc(fnQ=!abkpOzF!uANH~TSH2dkQ`t*# z*N>b|-KF@C#cc0p{VwX?Wqbe6Q8C;5*T*1@+1@p0-F-La#(X!}kA2wo&?k>ze4lsu zZjf)jL3XY`k~q?|j~~Y}+RaDTUMYHy{dWn^YdShNtWV_}cZ%!7fp5n>{hqAk=Jw9vE4&$w|HfI?f!W;SYWz3&@8$m*)?$RaSJqMAtizoR zjxHz19_r?sdFX95G)(~i!8N~|zl!n}{{GU#tziud-TWZ=CO%&Z?^x4;ZhnpO(P48B zY=ijlOgI02<&PvEJC*2TKR3TZ`NOCi8&v~-9O~<>;8yEU3!QXYgtPu-4aT1%$1Yd? z5GUWchc(-tO`c*1Nq=31JT1SDH2gZ$*YB$QAg6pCINBiJ4*GhX@(bPi<=+7VgdF=f}(PJK_`R+nekMRmW&7DkM?E$^9>Gi?3*B2XK zKWu%(+gX SiY6-K(X$zO@H*xIT9W@wq$1dWxPKT=h795tp_dH?}?>dt+X|z!w40 zL)tmEXc~1+rS5BJ<7(QPg8lF+;@>Bjd!bb3D0iS#b5^k)+N$Sj?s!$)_#*67TjQKF zkz29Ly_$lrj>dtkSqrTaZv2{g)O>DKek$h+O@5}m#?AjN`Qg++RQD*rFE-5L*U*P} zG~yxn{+E=K-$^ImPN)4&%Eiv%L%VCF0~hUXO%$$Y+K*5^&zBLo37YvOS~&|FwtIin zRN9s9qm@24(cS~5f7k+*g6s24nr?4oJ<8*aC~BEK0v zqf38+TH9>T*?cZ3)chQ0OVM$(y^7z^AFaltGgAE6 zp+A>yvlySdiIgqi&ZMK*axCc?w~Y#>1A}Lf? zu_fi`8tEz46$Ew0{;V9Q&*SN{&dgibGL8bXg4`os{_8U&5gBn zDoq%5==iEx#6T4PT_2ZbKAyV zi2u!bR`gMgkv$A0sM)Wd0ER>}){-xr(b_!b=KY#H)v5c|pGborgC`A;nzLKj|8jLs z8J=||d@=amNyi+J&o3N6X5l@2`5N{T2pS{1P|R~*;b z>N0bVdidgWgY(2WFFS(o&Iz1Xj0>I3XWhX0;jV6H5q%Y&Y=-C9%6j{DW(2ucZJU2o zM({%KqA!ZT$zh3$zA$+|O%Li`Guf5m_ywMKaP1cAkR9$|m1X4AqlNwaU36l)bqu@5 z1v=v?dT?k?j{gWebt^n|cm}`cI1^o$5|XY~v6aSM=G1vDb&BT5_tl5KAlH-ew!u*s zhp*(l_Sapkz4(h=g^K&1W_$5lko-56yi;*_4DG9o=0JE{B)nlAUIO;LKE;pV9ejfD ztw8UX{}*}Z;2ZClZ{b^xdB;7=)^dYyf91K{yZHlOzs(;s^S&ifI%DgA=B{Ya)+a;9 zFNX&92pugp`^QS_pUxcsN8!Ux+yjtiFK4YaMx8UY7^h-J863PP)8Oden46ef9nbi+ z=Lj}@ZmO7>dR84rFQhK!rb8*?ZC6VF+mtfKR;BPhADH)w`||btcxRRN`M&JE{n&&1 zvlpkbC#SK`r-zP}Bm`gPZkU&7Z*@L?)$H3ZBg>!g4NXLDS3IU``KF9-r%}gL>biz~ z{A%tnNeF(1JQ;-+gu%5KPf!Y9Cm2x79m$eMp%c1$RsQ-!J)a zgp#cd#l!H?XP?f=CpLM1_OnFm9iOL%jW0_JeCL<<@6B0VG30JyZl#MZ=H4^eHFKc7 z-z6?n_c}cAC#{U-lLxGj_@Z{kxEDS`wDxYTO{PN!3f#T;Ve$k6x}#G1-CQ?+1^N2s z2=wVy_>!L+-Xzmr0Gx<-X@&+7!@=+(b&cEWVUryeRl9hmk&9Q zx^>3;;Er*|zW9Chi+P@H(o}mQsqPn5e1cd#mIs;)&+!KGRzWy37*83^c@t~=b#Ne_ zvYfjWs*oSF7fl4OMuS%?&i?SpVfbO)wIsvPw`%&_McaV8SbY8CuADQT_5#Sl;`7AI zoKxb^m15Ip%gAG9ZGlzI59bFSsYeG=DZT1+^< zRzJft`uOL<@8pXIE3u9>Ge^U;HqZG&tvixJ+{e<``7%5fJXrIVUC5+Sr~eicGibaHt)3>H6GjbEoxuv8C=U1G)>bT^Tx%~r=toCyM zBe7S`RQwhEDt?!2VzrpKEX#@664&qAh2_ZD9gI1Lxe00g<5KZWb$p1660Hpp&XJeH zr@-%RXnrw1)B=xEuCWYdZyhO~hC30#mAQsTm{v3R|6}d#(B=J`Pq z2udbsQ3(P9G7uC53l*@a=nEA31&t3yS`|4g+SU+ICJ9<756L9K{NA5^&Iyx<+E;(~k9qAmd+)Q)-fOS5_F8MNy*55Xm*1PU z_G09Ox%drNqi5@GCCPZVfQRO6D?BGaoQ4>DNP7N!Xhg7DK)KQVU42HdoU1(66X`gw z)0Rgs9dvMhf5S(_8#IQ~(n|hW;rrcqDjD=$^>lV(m^(Komwy}hU&TDkVm@XvFEdy- zt|VTsqyKgA0eH|p#_ZU9bM02_y^_%;(2jU!Au@`1L2!bTKgRu5AvTq2MY1G48<@|2{Btf{AIN zcv-E`fODqO*{jPXo{{stG`qf3G{U)GQtl@zJ#b&f+E~HF$jA7=#Vh1Zo2(}-sKMG_eo`2Hgii`Z}a>^=?Xg)Z?8LZ+Jm`GW{%-w(^H@|?rE$_ zGPEXLL3fi%K9CJL@s7s-QTvQ#r%5yH*G!seZ#QX{{VJ)x>D^`bZRT1r7Z*W?@)g>I zU*96)R_gy${@>*1j2QnL@e8>*ITW4C|CRjTv>`O=xufGFhgf%i37@-VNhNm=$rR-)CI=CT0C@VN1kFX zlcw8OnKaFwX;R%sHHA69oVlM24om_UE`z==g}%LezrT8)L9v^KBZ}iL98oMSo$G9a zAB)}+`SG(yK7V2d=V2r#$C7d!AAk7h`{YyMGwk!gcgOybZeQY-KS#O4hlx2)xyN(GYw@jtFG6Q!l1Xxbn|t32 z^s9YLCnmFrIpFGa`SxS9C;t7YNwM{i3XWySFnLy9>QUxQZ9PQ3-<@wKFVn6x?{H`6 zzhHlHbScOF(geNe-UCm5(HfFSFMm1AxfssTi-%SxWf^)|0lj<=dbtYv7{M=x-%-w% zZiO#yC7n!P+WUw8X*v2)hhDJ@*ax-@yBserAxakB4ayM5O;2DxiQ5ozo`ohh2# zGnI1$o1m$fKfff!s{Bb19;>qiRp>Fq0V)|sjJX^6ulHq(6TL2i)(U)in_egXCiI3p zU*5`HNqLO}2TX2>bm^DDiETw;pL;&^*oCeP0=&;Oef#=5^BOk7W3}!-pA^aEds-8- z;Tj9hby=8Q}+-IIB^*KTHJuoZf{mNr{XM{lkV*?*@y^!9i9AiliC&|9{B z^LQu!eey+zg58zyLubz{&A!nse~WU*{()Xdc{}orS<@O5X-#laOi{%wQ4CMTEXg*0 z10Qn_lOvbp*)_B!`_6Annr?3<72FoFR_9u|siMP8Ps8Cs-T#-p#v@JcqC4*$dsgrQ``V#|o@CgRWW1 z?^=GGJtRi*>}%XIS0~tvtNex93GT!zr=rshl3ORZ*TWS6yH<4ynUv%v;~`HbF$gf7^UwvV@KHsK4~vb z=V@g3_!vC%U-TsSG`tzPrcW0}U7zRB3G zV2o3d`=+q3=iHOfT4Y5PQ}`%tE6zFhWW=|Wg+^J}s;o!O3de?6(MM>j0bdaHp|~Pv zCjRt`xmzXLnD*1$pMs}N$m{J%8JlzlMe)I#nK$&`nGd(L4+qxK$t}qvTJbd!?(2@+ za(qU$&SZ=)Sy}!0HG5->F<|0HOpUJKe3SWhe*f@oe3$=D*-ewo9Xf`t$1ff=8JHBp zi>=TF;Ty2E4A?q0{JF*~zMPo<#@MjtVh8Y~t(2Rz^??HR3BpgwYhq?o_bT6|6Q%^!_~)MBH!-wPmI(- z6UIju*vi*aakCGCcda&3JuNztz`mgNFqp2Zbj+S8OP$3ZV$`|_FK26@ywsCY2*7x)1F+Dm-e$Y#x9DTBjx-RYf=jc8J96;_ zKUx#;<+?iE2WuLMiG7^DiZ<(98{@gaPkdD0OyZlR@Q$TRXWHYo8dv)9L6 z!!6jvYO5X^b83Gf$G*kzQtkcb*fGXX@5i?b+~fUdvCo3P<=CH{@91&8fc;m;Z%E#) zy?xy&?&)2};5~|mFTG3aoS_?cZ-3HfjvQ&x$Lm~Qq#V*F@bg{h*vE8 z$}R3G^uwQ}+c@{a-|E^_YG|jDbM9IQI<;>D%dz0R+k zQ8AwLhY$1H%`e9nFZZQ{A0hVMc>Im!%c|I{1>=23?q}~n?V3Eb&E8!x^_fe93o-iF zwh%e399o7)987is-`<{!DDnmbY?Q}~cGY;fVDq}+1FZ@3D%aPCZNaT5O< zo{()g$DRYuORl(;_eIx|p}y_rEh5j+k%+hI=FK;GsgaAB>kNMK%SR6|_ss_GebMN1 zy4N^&_xF(-a_p(R*D^`Eac*PFbncUdU%hjm%ChW$DO~NtuVe6wS0d#eUiVz>cxc_nN8I!AW(z;NMsD?H$a zZs8=~cYjmNT`K4;r7gr$pYYzDd#l}8(%9Vor@MvgkmquFPj;UM zzNh$9fdPS)8GpNbZ)}NkW>_}tfX_M076=BNZ@1CL4YYATZSY1CF1q8%nU`FdB?(@GZUZZ5PsONT!MEDX>FB1ykd+4qqVf*;m2e8 zkNlmH3w`Y8+$(+yx{E?{yi#Q@9z4W<-O;+fDyL0$AK{4n$wV(VbLLOa`oEC>ZM3Jd z`*`OW@P;$|QTWA`9}$ZpiTlQY`<{l1$}>MJ;@sgd&KUE22tHKc>s5Z^fXiDxUHzln z-gghVTk$s^(R;q0jRW9AL7pw*b&lhXup8h*`0_=e>(c5CzTL58_$qW_aN_Nva1!%4 z;aE{vv?9H%8~rnWX7Hbb^yd#};3LVsp~s;KjrCjfwU%*D)tKqO1@2DoxbJx@Dp{TE~^Zq!!Z*+3x8gRUxHdy~n?7;gu zlkVa9M?HwY2CjdURJd4po`@eegKJ)_bAxN0@Fd%=_%?Gxj4!V*0s7B=Ks2#W-N^f^ zJEmN>PuIG0skv+7oQ&q?F!Pdn$Gt!4dB<62Hu`%`y#?QlO5$~sZoZ#$GkSht!0RTh zI@_e1A3V$CZ+>XN&apq(m^{4~`-#b%x9Lkv)o|~?v_s7I5$0O*lrxY06zjX+IA7n! z9Y@+{T}Yizp~=rUGx$ROv(p~We`eZn-oG<{!?fA?Pfr_3+|(S->s))uiQDE}(sbLk zd26T5x#Z|=*Ijb#w&i(GPTTf_t+&1UgXnE1$Gm!5+n8;)EiV`|?b?E|)0X93JMDw~ zWz$~ne*c(ve{f{leOf5~@CED_Tws+N-W3|!aVJZRcHo(;i&^&BtVMOe z>`U<^gSchaX3Z2{?_=$O2KHjjWS}uigV^ir2RE<>rC5vge)h2j_hG-PZ=`;3zsLz$Evc$jF_-`kOMze8RdJah%;d&S4^!iTR(v0Se&S)0vs65}`Z z8TTOId^z=1H*o#PwTWdDDVso<>gv8^?PDw;Pr6D+{ghK}JxJL_lr557H~9M482z$F zRgMjkA0hUCo1QPI92?P@9=~^{C&P}2<}??4=WNFKC3CQOfWyBwPl$c_96ZS!6mnO0 zli^=kcA+~5d8AF^zkDN`{|)fF6GQTYrmxz+zY8Bq3(oSbWjKH(){iw&%=it=6@OZaINkv zx@nX<-?T$tG~d6au9@2|Wt#Wx<{Px@;)3S;WiQW_xt<|!OJW-aKb-d8pzK$a8G0th zlH{I?c~*bxkOh0XJa;vz`qPDeS9yFsskd+rdk^ic>G;0xJF46>4g9NzPsfV$*X(mQZ4%YG1;e=QC+e>o1e_Cn@k&GFXI3g%98xr|hN zK;vHoFC0$)zs*?N%80rCkrkCaOSYTy+&RCC`i^Zd%Pv~I%$G4?6C$RDTZ_x!WJl!rbC>^O&0dcx3e zrhUE(yCI|+!wKLApHCMZpF^J339TbaJsG`;eh5GM>s{cLuN1n;RSM6{Ce^y$CU}F( z={*097{KjS7O@w7_|RG57`k+1$b?}15MS_<0xQ^q_7Ad8(!UK^art=syf*{g`k zW`LZzM>u}U7OeB&X*TBt9|tFG`CZEX_B6J@3B^IP|FsW#oOAIhd$8;lj~SSu7b=Cf zNj^I&`NKS?*|(Dl{-?XQstq3t@9wIvy?d%EuseYKrhUK@;7O~yZ!X_M`5?A{C(b@h zl6@=RJf0LX@k)gMH}O0QJzjKuH8?C=z|HC(x=HhFd=Js>#bd8@%jT;LJIl7aLZh}W z35`1Tk`=w!)XB9cyLGNrow-(Ad)M-hDRY_QMM`F)+Spnd^1J;`5;lTiA&WVjH-=qT;T-=>y(gqwx*keREz4Tf{Q?8Dd8e&Fjpz-kIR+ z4#mqV4G?ed-fe3YTWZn1rF#SShr-yI*T}zaMY|PF zIeFRMMQ6$Hu552?C~;@tfyMV$uMONw-tOf;X(&pLY(JVk5_wY}P6f$RyZsxNOJi-0EJU?^>@)3NqvLCP< zN=!EF1EPUH0;lSuSC7Pww)nM7*?rRO-;;N&%!(fdE}lNSuE1KnVgTnOEopmvC5By;~C6yCtE$TWIh1!0-|OrnQY~lk+r< zRV#CeZ?N=7wE3{wblaJC0r!T|?xP89J(|$gqqNn#SN=-rh>E>)sOpj2=V)gW?O>!? z8lj!-?z`4db}VS?R|)U`iub>XzWJE{muo*&8^q%Ntm@I+&uHTiZ8XwG18vmPM!Wmo zj&rS9eQD<~?>=VUeSU30hR!%Ynj7Q2&v@?;?=|vX1Mk)IUORQKM%Tl}u-Vw1&=vob z;@DI!13%i)w*$~Zt#|}pX3ep|4LXp z#DlO~ZRfwSTS*RJZB|S#?MZIey*Aj`H*(IP5?fgx)~66@?M{FAuOl4T{)2bTng)M5 zj%-rL^Ki~s#g4*fx)BG0{95vASu>6&S@93}e-Zbbt@mfQ8M(Npb zc|hqf*@-qAI}vi!e)ttKCUzqBlaQNaCz_CooFF+(-==+i>wokuY)U&w#d9X-AU=!1SVy&h>yyV`WSlBh>KjP!OjPM2f+2--sS4dY)z`mmA zGVCi#Ct_bw{-xMglwO8?B^7*~guUcPtrdG$VQcA!jpa<98^J5t(e~#5VpmZvSP|mKpoWLF^|7v7a2oesU1|$*-}W{Ce6W zuKh%N0=g%sfU(_A-0?@jw_0TWdw|E&(1*u2AEw{yimb)2=bAeR_xv7N?+?Ue{^J?e zm47&jZ_86T9A1IYY3Ychpz^i4`xxz0*$oQK_Lp!r@>x&M4@ z*H0JREk0a}ZE(IXwP%6PUp}!ov}}J`Wv>5q%HYWsWmRrjsCfLc{Z*9JQ3fw9#0IpX zqiljxmgMl(5bN5`jJw4?ZI%j=o>u zJEOdAz?7E5s~^qnd-u@0bx#L2v@6&7`xyI~=F+lyuEfUMPj_|>09Ga9OYkGvQtIJP zsVTlu!l6yX*pQqdzXHyCM~@>jAaj`Y{zSssuRV+fy&XE-7j*nk;6babdBei{UI;}U zA6M+bp~@NE`Sk#PX~+Sh+jppM&PrhGt?6Uc9EX<@GrdMgKW)Ai?c*H(rK>5ugXY6?_03W;ro7-S`1$s{xKD5f^L#rmp=5`uB zbPPVEGe!j|@RiY>dC}AKS-j{!NyWcVmQ1XpmGBwi)gO`^yjt(#)l)8BJ?-Mv1{bfM z0j~^xx%OM(m1n>GH8>^Q+6N6vKXB$P#ho|q#CoTF)}O%fJ>c0tfX`vIt}SdT*Q8nh9PH0_7B4zHjbD{+>cj`4Q&N9t~()P!_tZ6E?mm6 zKXZLkM>22nrPCR}@xm|GJCM&V zcjFah+IPU?4c^mtjZL_Y?(^liF54!!-U^&`7N8s7^wwPu-{*^Nh-aofEpP4pq*&j< z21BZTYfaF4B3p^t(z}A+UntkSsqOkn=m5ZV!;*7?8>-T_ZZAvUs`;DzTCe7M<|Xn% zY2>-8jAqFSyNLU3C;UAe zTh*)Nl`<~nd9WYE^G5Pb9{8sDm`I-1$NCNUJr?ze9IImg$%S(b_B-K(6BjDa&IZoS z^g;d59`I_xntsz))9>`&zEK8uftkjC99U^C0$(*3&BM%G$Y(Hd9>jB!0`Q#7&U5e( z%+6W;3hU&e4m)$-i1b6@LXZdC==D=+%+aN4xY&+wOdyLtWJiT-mv70qqn} zrnWttd62wZ^6FW?@Znf|=b1lUBcBF8enslr!-_A0ZR9Dpf975A;~C!T#e4GA@$99# zzs{j`r!Q|%$M1a)d{nF?_3;4tCu#2(H1)EvU*y^kcNzb2=G=U1E_+L?7e^@9bCUIA zbB^v=eY=R5XR|MgTts^>g3AU!{>B+PVxOH@Vi6}LY4`pz@w{yN2CZfAU)i_*hTr1X z@{qz(gSR?c-i`JhKQMeC-~qFeu}veldwk>l!A{$IN!1tGQnl9W?lkdz*;Mr`{;$5g zNj>ov(b;eLuQJus8N%1q7Bs&@DSZ3Cl>+~5O5w3DlWuJN`g2A{U0)`hGm=ldcQ=@P znOvJ`N35#!kFlBJ2LL~s@qNqpJeaaOcu`h?&LWNF0?pMd`lF{+S{ z(#q@5uUgTYD%8Is&M5eE96u*~5j{Vrdg>H|ub%B_J$t-~eoj~LuD&gw%A5Hw8Zghm z@9X?x>+ywXM{YY%q;m=0d5#?Wd~`j@U3bwQvUd%WAUpBqT=YYHUTS3D(zl3XWq3n1 zd@Ohxj4`LI2 zYr@tbeNo+fJ9D^W6KzFyJd<(dyJs}60W0Y`*m2)V^ncP>|96om z1Q*ArHrF0O8^Vp@CQY+{O}TVF^KKWvtorV>=h~OW3;!+OEa}(SmerSh{Ic?WR{S@7 z^DF)8%93}>e*ei`$kpJYIbRCjBjz*yI-E%a23x_E7eAU1l>NQC84Ga_NHsU@Ntxw4 zF1&VGw;`cr*ac>3o@ARJz<Xd7Y0uOr6U*_t2VH(d5uufG1fb+*aw3Joq&3cZ5UHQqkJZrc1Q_E(Ir#{PO0 zec&DkY_D65?Nv4#*-@*o&;8Z-e`VNj@Q%(*?3e!+eZxjqvzzgdQ>Szla z@yl!R+S|Kge}YXFI>%16%Gjx(zd_v{T6>aj6!x@gq(@D_S2+aDC&t&7p9}c;1>FYQ z)T&OlDeO^ooDrJGzKhQ1dh5#+{BAsbXg@k?C)*Uf;~n1Ff$eCrYwulAV1@Is_tw+j zVr;{*_2Tmz5Bi)v{i(F)`DU5+utgPOV;X=hDhZqLYqS^Q9D!D*qy#hdBZKo%eI4 z-_3CR@QI1klldeDSf+!cGn7_yhEw%dAfN4{-QS$;#L#fg)hI5C_*90Sh0cl}NOTcv zmTXDq_i*UteEOh$ksaWz!6W)Lki1)%fO6r2{QSjxqD;?1vAc&%3GX)$3Y5k#foFYFoON zWSj-$z0vg@qX#){K1lgC$|nnUMi2Vw7{|vWv)Sa~`?22t^Dzw9?z{aR= zd;`2OxW<2*u_~{|j5QOS;Y_6qllXfRB6|0w@{-_7RT=0enMOCM2M)#PC+wletKTC& zWmOjE1k>z!F5GO|GJ0bg_tw!zeOt1Mr+fStd1L6m^1Sg1Mk_SlFQM0@exeUY2Nb;S zrEUW>E5G@MqMnhq`N$4}g*)zTj9c^Nj5*Uj0`53t&a`i(pE?(5>dWtBu5~i7*Yi8+ z6%qf8m0R%^VnYN%*nAGU{z5x@8=sTd!80P9^NwH6_l&k=RB8-o8QE8Hr_PULBl1Kv}Zdn#8^rYuV#n#?zGSj7N5U z$@Faok25-C`ps$=e5js3G9vsE4wyNXOh34CY}-s^dT3yOnM237bl?O&u=t;9T>l)z z_ks0crQq*pN}>Nlq`R>5{VV75b^b`Q%83NI*@@9(WXM&Xufb(?+Wh+tzf^x&uOvtB zBxcO~`<$}^i}-&^&c)Z6wMD+pce?u+ukv1~yXDEbt0d=M%(LXs&6G)Q{A|ReV4{3m zOkNBb_vq-P;HMS7VAGPWGOl+PEM4o^j31n29|ipY8F%x9?+tNf+&ugG3mqBvC~L!6 z@E>HBw-RK1&H)u>KC_QAm+Wf^zee}4YO?t? zG57EC+XikuiR|%0RnFEI_D&2gGWeKYUdMjE!N;Lm*VD^yiW08vw zqtQwDFFty5`7~hhO<-~bu$c;srf`n#a^ffWjlWR+ki^)Jr)c4YiyT~Pp`CfI&Zo1& z`_6arkCAV1teaJ{sSJH@L?~*bdnjf*@;-N5SlmGazkj}aF!uZvLE_H@V@1d#*uahz z`GXD9lY+7N(DV{t(5x|j%i!JVbi$2Q*K9@K(>*k$PjM#M&`kchhGxPSoT3-$x(0_C zzi{|<1A`oUhe^}zSH4!iO4)so7?hv^hPvVE?k9rXj-ReBt}lq^$w7i$vpJ{Me-Y`9qCMgLU%{X+7u zar4WSU!{4?vS+z@|4(^kR{XEPV|l6tPtCKx>6Xn^**NMip?$wqGs(@rhJ4A!u~!_q zS-dtIzaH5Y;g4IU(YJQ*``n+1FP!=22>1QV-ETfgJm}`C5M|l+CFC``-`c@Dx{K#@ z?X%|8C?>_OX@EQknWGdRBi1(x1e&#poT`qr=`k*e?%s;*4tz zILv?P#h)n!28WaalLn>CK};!l`H9jWa?Z&-XQLmdeB~Ltmif@ZhAEL;UwTiYBQJzM z-1qX@Y;@!p=Q2C|jz(DVeaM%NFGsdL$LH{%y?kd!Z>y$lHndb^nYef@@KRS_ek)O5 z7O!G$GWv3&Zen!RL|=}<-0?;H%k{_*liAPF-l24qFX+g~lf9@n4;{Ic{xuiC?~s)P zC#`TE{!y|1RyZ4dc~J#63v}hnX-mHE!Nny`TV3?yIOl*Y^kc<-G5B$|qa!EwL3;7N zIp0D)MlS@H4#3|(M$VL8obCe_tW8IG_V8GK8=Pl|cRx*A+5=Ub+3(zU=U(BS&fThA zN9Rtrub@3_4yByuiZ{^Ko(X{so zeB|K9Q1oxfv_q^$-486jq&CE-_5WJhS_H1@zvv*&f9LM=eCNCzZG21{pYV;_(R;C_ zD?VMRVwY8e%d*Y<3H+BWL2cqUw&y|gwNIXy64|KxDl{L$B|m)!FZY4N(x=Dw!nVR% zf*+vHwi-SDk|c+hT|%E~(RYx!x2Rtiv&X1&pt9rRJ9UTTGQB7LUGROOyYt@pyk~f$ z?6DpCJ30U`PSoFT?BUcOMEy4UZN|tb}2LqC4{ zAe&)bIk+qwAttwE22ZDN0=6f>pIzub?!M;c#C^?Cz{k`F?t}RMA@x=F)ZQj-Q8tmX zw<%Ly?ZJ3D{ht%-8@{~+~uQHH+qmZ#50@$)v%u1*_8pE;$+r`cn`NsUc$A-6I< zee3lBPAnYZK_2ip!5lX6|HZi_!53W}p79fpx?4C69ljYjy|VwJ$X0ZC4_3Fiu-Zyn z#{?^I`!)LJ=<(QA-95zT$aCPK7_z5q9X}%M~_dp+l8xU%<1-W`gsC;X{g}c5x%H_mGt>_R>C=wboBX&iUS#M zn&a;8Ei`w&gwxzJB(di+BaPdQKL3`{=XavdKhGUPe`f990FFIReZdQTzIA?HqguNB-PZmp=<#zCXMw*FMXI;X+_2e3LD84*wl|$hU80+#R^*>hf0$Zg;tKPWm?U zo2bv5cPBaLNybt)5l?61E9RUp^7h|P;o*hUH8^^G=duOh+z`qVaa4GCj+fWxipVoO z4@`VN0cYQ!EHAOl==<+G7(`FgyagHAsI<~ zTOED=Elc|Rq{^|)t37?*q2arQ8{&D;!LgAJJ$zoM@#WiXO2Ok6rOm{-GlWzy9L{}`-7kD#fGrS7d+BaexF)*zqnWS3&y*#?3%5AKvqWgavyGIkKKy%s&h0tAJL3$OFkKjKYffnVWX=g(7 zY4)1yr?aO+ObztUM%UiZ8E+PRqa$A6ug$xm_*|z<`#bax0|Ov)D9zWYA7Cc z^M0>9Xug(pzsAk`t=g;V7FpdLS!+Zn4o|6x4P$*)ifpi}8|e_EgV#vE6g;HIDc)@w zx}5qJp)F#^u&{-UDCP(U_ z59xAg=yE?*dyHG}?v$=Ze^-$we-Q0mCh-k)wa)W!&(IF~oz9p9|6?;larrwuXvWR` zr%D-HrBcRvpVGnj`6^`%zL9h_xPB^zV>(sqXZ8OJ2mrc*==4Ax2*CT;T>+-1j-s*nFOsl`ZMrQzF--A$F$FU zDBrX>^4u`~1=y@p!ba!z@*f%Eq2WpLo&EZem`U*10&}CM)2i}bANt#yt{AxR)1&i9|9i2SDhI#a9lh! zYNdphDULxb*@_kdC-gZJUn9Wz8Ot~BT~8mLLcS-Hg1f8cuecy=y_Q65iyq-RaCio1 zphKJ^ug10|+O0?LIm8?`^1bJVUKIHe<5~?K+tWuxo*OzcVox6xX<%L(hx8A&Ei4E& z-et}gMH`_B?v{)m;tVtKpQ6XpNSB~La+bLsf3p(>$gR*A_rb???zs@3RN|_HLq{GP zBe}TY&zw8om>%(Bti+xNUN88=+uVEE{|qd|d)HggaI$m0pL3XdbhnCNFI`S+(Nfp8 zxs&>4PZOB`0hr6jN&2aG2jxqWQQ;@UNhN!ZpbeGR!6ydse;xV%3QUu(VXj@BU?T5@ z)vlp4M?Sn1Sxs}NeYRT8xYkxZI!5zv;__6j98-ti)qeKS^iC3dL1Pe|-p4pNt5w>B z-+kA92;aT={Wd-U@9Mn$HNmxtSr8;vYH&cx%7I7j-CKoSWqyqLzJYTi#4>r0^Rnv| zhlFnob>XxWI5|8x!@iaOMn>)vidQfW#}|)rxN+ZaHey39g(=ImKS(TtcP*goQpz-+#l5YIG#|U&I+}-@m|OL=o7>lE^v^k;m|;&aedT;O zeVs)cPG33u?e>)zON=GkezoJ9)OGt>Y|8TN7Zc0ez78;Dx%TskWo}<{DN|nqI!{ht zB?k%)>gyRU9+>Q%LL! zYrq?nHIa7|ysGHIdRtjJBgI-VV<7jA$iArWZB$>g99u?=`t|TWt;bsP;BhN&A}`qe z;WdG?-&}LFsJA(%P7EOJ^=>Yd-Z8XtEPGz@neMpek*eJlt`C6l&FQ}rqiiqzQJG>s z1g;^rLvPWALjxMi?mhFVY9BZ)N82lU&`M)#o)!+G~O{WwJ@ zY4%91LxxV$>>B*W^nW?2*0cXP%fpd0JIa5-qb@lV*Y~^Ph~{5?KGk0bc7yolI?58~ zV=khb&vas}FYiel$)vE(0xOo=m+hai-n`}MdvAa6`4hG1BNpR* z1)1UJ{O;oSZGPNy5Y4Add$oI!OYsAY)(j3szb6@$wM9N(p1vlZoEW~6W7Jnpqsf=w3W4xTPPG-!L*blpmxRUOE7_zA1 zOdpu<1IDNJ!~QYU!JUg}$J-CnxQ!2_Wz|T=3sNSX?^ERN&seX5jNxdi^IkD!p#b)G zc!61;T^~iSu4HrQe@{KKRoA`KH~){$vzxXP&a+EjtCoIUgg%-UiYhMPe)!a98@dJS z_xgiJUrGwrmifXBM}5I|bW!}OjL)z3$~*aNqBp0rKX(i~4`EL=IND3T>XIk(?CO$J zcq;#0b#$?xyiBLHED+Z)34LN%2IYKuDk9I)IA#qmecRK@_)n+@^eW%YxdUt47a`{f;<~j1yaEv>N=0-DP0|oEWTLSQgYB4Ani|wU_s|nm(l2FOq)G zABro+?s)X8QJi0FpJ6S=);#N23h@YLL%;O>IqGT7Wt*8ij5wNp^sHgX@YqsnneU%5 z@8UVt)KkCKkSAPO<>o)F?=8eGaTk1!wfsr!BC^q}aLdAQEh`41+17h6YZK6{J$?#*ubJu08h7|Vgperzw) zsZ6(OZgk7Pr}C+k-_D*N{kqQ0zfJjb*i%MMECw#xr)!;Q#XszBZMg*bOR$>Zquzqg8m0F z{tpVt2M>1L$60vEvFO**-~5I6bsIm`TsxO~itRtY0DVVueir)>O4<9%X76uIK5a4& zeW~{m{)<9GE#?uzD z0`k;Wrrn}c@I2mLbI5JubLE`|7Hh#D!D2G$D1MoG1_ryBdk+TtY2#X+1%tr7FRhIM zUjql`Kya9kj_1KaG%q;pgtj#P724NHfWtP%B{=+vdWyr-o-mKQ8rz5K?!<5yk{c}5pKrE_(xE47j} z=3Nrr=f;3v0*r*mkJB%;pY6-)Sx{vyu2y@hOPs}9sk43%aTb?b(LCb(6cprjij&yN znaP5@yz)H7k0VZ^Vk=npX0HvT;VZnvipGeC7$ByhiMs$Eq31eh*@s&3yq3?69J88l zguubpN!SU?*ryv}#giOg;ZCxp_VPX1at^ZPa{0R5YW6fY&tE_8R4hdF_VCA0LffXbo@9gI~e3$En_I>A=L=@o|4Q&yS<`;Hz-56yD_FQJITJ zlfk21NjW|7mo+w$)G%4{v+%(ywd0)HdE__V^UrREqJ9ed7 zXU4FF@47Ey_S|mdgw6dPbqo0yES|fHl6wn-wf((l;VJB4^=Th76FF?Syuda^z~HYw6fd=Rrg%or+9TEr2+KTEMdpb6=5Zvx9$#igTU zGm;(T2JY-p{%^@Y1blqt*C8jHeQMhHIPdD<*zUf^RbRHh6wWaJ9A5b==(>a6cz;~n zp|_W4N3@heeaUKC53!*}V|-sRfn@Vby4U#XccMS7`yTxz>UR6V`%b?4;Aa!}8SFs+ zZj}wqrN1w{lSqf?oejKG$Q}zg7X{ZYY92>l7*Ak;6>de|m(5K4>V2h(H(7!$kyx*> zbYi`Nv-_rVHjZ^n_h|2X$&K|2-+~tJSdRT@H+w>AZ)X*hXE*R+2H#zO4)cIcvrFq)8MvEl#Sa!;9i$&oaB9_n z=f;NXNS!m}+2A5~Q3w~ooljUdSH-QzD+l3sLA&YnOYNu8FZJgPQqk|CY|fRXS!R8d z9H>4%;m)z)BjD`yJUjf3J7L^9&Em-kJpFWI7{C}f8&n!wi46n2O#O)YyM1vSv2h*a zIHtLvUI$$bamI2zW8q%4(#K6a4&apIjN`AQ>Wjuwja=Er`m42mj@!>}^j+)bFz_x3 zxJoYD51&ZvyZl!^eiz>r+HibVq-*5Z|J4lq=(qa6>tE@A+t{%5^~ApG{!q=udFns? z(^`?vTA?v+cE_+uDRZXtiJ~q2{{?xE(P#Z{1_sqU|A{#}9p5SF#$3|J>VKt=#K-Ll zW1Sb){if=tpMKV|&*{PWhi)GqAeEik>)$bO=sn~lbJiHXd<=P^dNAVyhKoogAFW5$ z#j)1lR2pkxBK%KB2dA9}_{kXNA;i9|XdwnI=-!L3qzBCdQr>Qt>?2=MnNxyMpICfVc3#IXi2I#xmaTnV<^$WiKp{m6Z4N@dy@Z_bbD`&Z*tf8X40G5oc2HAT@%w< ze7?g^O7ou8$xkYq^fcpB*@p23QMl1f#wN}k~e(F1WF{KUAT7!Gu93K0d`X_wG z|H=4C9`#gC~1@JNKgNkE1oVC!2Gw#GO)wv5(ADGBqD0y9V16O%* zPEVaPe}ppC^-IB$2f(RMnEyFRF5bfz#WTKg-RZ!4m&UH~9KAjeE}=rz~SYp#8t z=8yK5lA8Nq_|BdDf0FsuT;gw@E`7$VPrz4k!1Wv>9>|Fy_yOW$X!z|*PF#@s(Uj3% z{f4v%XP)iYM(7MY+l5?WbP+dBi|koLgj4W}EO(x6;JdPMYMyQeMuLNEU4GZM@+$J= z^QJtHKWc8qk#F+gAA-N;e@kzyHISrruPv9}HhWcg4rS@I&E z9%TOZGG`z8qAPR`Q**VXf26I|iZ`}0pY2_IH~qeNdosVC{QCIJn)n?4WQP7T9r_pC z8~AVVA3ARXUpwf__2E27p3X$XWb-DS&$IT#CNlqK9RRPI{)O?&F6WK^_?L}eeBvv{ z-;^+Z;i|(g`bT0e44S&YpaDFn_otTBV|Pp;4*IqD>TvF-6I>R$F!?obkS})~aCo@` z4*4DNX`XfAApbc9{XVX=Q{hxwiY3~*2u~_?d+D|U49NW6m^3A=) zVPxdgdY5lsp?PNgP@I=t=Y9ElXXY8(gPCW=yf^dQdA%FQJg-Op(mZ?X-Rl|??Y~N@ z^-eHc0SrqGJo0VbuOd5kY^E=&df%lF;&(4oUpj!s`zy5pytD@GVBB?Ojy_OD8{HWD zCh|@PcfBLn45HpoR7dls{SDIoJd1vih0MC!%y;Ckq<4P8_cTZUfnKHe&9{O5MDi-t zj{3XWifas&@+-hs%EDKQ_M-t48$Y`_#u{M}AO94rv+X0odB!N3y48jCWZ6uhxkP(a zCs->+{%K%+;23(o_E{*)GBlNCFBF_;|3*^vtqFJw-y~OP4A-d~7+$Lsyt`Uyc_{vE z(pKTJODq^Xzk9*G2BBEWM4Y^=O*TB`2^0j zT237NukmS+5Bpd6GvMQQ^C)=S2%TSFx99UGK2g6I=TUqXw($KIW|jrF^$ba#U%VhU zv}{3MsHGdf1G%A=H}uO6wLHl4GnC!HzL3T)pF(2y$CEYQBIFV;zSEG-J`I6{GvAfk z8%a3h`*yl>C#rlJ{>gvQ>A&mK@Um+s_0gXMp9XW@{R3l@%e9a34dLLYd1d(jSn1o7 z9J)|`Eqj2jKSReEbJa`mW%$B5@He|#9p}LREq`t3AV5qfd>OuQhP|3S{z%a&KZZ}` z$M7D04EWPa)|M~B(KPP8Gy4d-&+n8U!@=pCUjv7ZgLlGbd>A%Q*eV|eVuZqr?;7a% zFzC#ed>G_cs5$lSWZh`yBFBE#q-nNjw%xD0>CeE&qz}F(#8jEx58snB@vBM0ucljA zcO*7}U$Qs0xbYF%Tpx1n8K7?(JX{1Ge!iP?={e}Ed{g+x-M&L~Z|VElR?W-&*7A!a zW%V4Fg6^1&FG;pF;sxlXf|yTFXIZnfA2XkHRu=biXW?&yA8(9!fjXxiA+`I2;#1Il zx9FVuFwUuGbv~#59Aj=p7JAN~)x2qE*474OxB4>dhgI3lt?;ZiU>ODW-(vjhS<~a_ zn&bS|;y7i3eJXwci~RYG^YNdER9)Td`Iw{A%xVL^4I8^hj?UJ3hg|z|@KL;Xj$kYN|C;7I zsH=BIkypKQ_SW_O+0ANKc4+GC@$zEm1?%D41Ke*7qrT2Pi+2;(jWga?o7gXd)d%vA z!sEpE%-LkdnC8BBw%((@_;)olRLy&fNM~wKeG)pCQr_LAy>xhCJ@QxEV9t#L<2bUy zQQ8{K9p-{n587;nmuN4&)s(3Y^MRbTQ+w;YZ~PJ1|5j`t!4iX@?!4ihK;s7HV~hAV z^#b>9SzAApR6gs#c0D{mX9z!4eZDD}7=H%lOfZoRrJ8%o1uyI-_|M~mr@FL({h@J} z_QQGaZ_0yT>D>wRN#{<#FgGiPcih#vHtp2hyeOCgi$9ZB3vB+Vl=1!Eq?z`cN`d3= z%yWkQOOwjaYbLZd1Dd-M+WQtXI2}K)Y4~|{H|s-ZpLfYG+`|f-zkiOy-?q_?>^Fs+ z3)q4zq1Xpc7dmBID3iXbaTMYUBp*Pd8*|>i8r+VtM>R& zuQ+^CG4FKNMltW2*{{?$rn>EEjPT_dbN%_{e#N)L&rs(Kp5p$`4;i0u@j`G>yk#PF zWrsci&R%26a_k{)*;vZtNAte%LCLm7uX<+$`SLA3KFo^G!9HHd*t8C)jT~az$-jp4 zbKDzJxkq*Kkj;<>`MisvPf=@r{jO5#sIai{5 zb0!43?Mc3DEXsRY{(=c|0`s@ZI@b!1Pj>tY{`XAxPE&BnqTli<_?*02@e-xLzs00! zcC$${pqcA|`*py74)bs=^KlI{b2WDhB^#QV3~XhiT$k+FC)fF1`(&OKf20DtuJ)D3 zUl305g`y>aAJ0hr@h|V-KH4d%7lgL1{N*DvdOj1mV~Vfm6!JxMpv$mKuaq)TKl#EdHVeZ@x@U6+D z#ezln@~HOxkz0XdFT3?~%e)t2t-${HR$@GW+B9JA#ses!%-h4Zy0f0ZCtu{7;*rJB zcWoMHGMH;459=ON+P?UR;}h^3>J?(&_@!XTdwYQOKKeRP`CY#G0KPe=`{qt`R`~%4 zA3N?P#h>69=Vo`X4^)l)QGPgy{WR|+_yHuD^C0W*ILJGEw=}?Wv9LR%eYK4|N$L|gK^v>}@QDZWUoce5;Xw>$so*njWTxWIkUnS6;Q3n7#2(ewYK zYytGF=h~jcW9Y+ur^v^$k?+u1Zr4V>KaaJQHj9orZQdZ*!6WsZndlRuwd=@Jo3TFV zN<~)9Jbtx)&L1Juh(_Lm7PRjzU+RFb13n#N>WCqzJqGxyV)rTTtYRg+c^Lh$kiEK{ zzVNy9tC9JNBk%C7XaL(n8~#`A6_&AY#<(j_zVoq#fk?b4g}8aCk;?PfbHVStacHkd zW4YGUH2VPl2kQ4X=(pyvu4gDdg0;DhZ%Bsp_KXIS*Fv8svk!0LL2&ja$ep%pNcHAmOX-y>bh{=nc~KiOO3?mTJa$LG3U?cTf~ziX5sDZ%?R%F zZ|cxMWGZbGcC-3~cBZs>=Ybk@u86Zl0qk}4jI#keo2;{YL%2`-WNKvG_0C)^b?0g) z{nk3J^C5o_KAE|~?(EtxUovpbvxk#PM;G4J!9O)O>&a_l9#=3wqRr9Fk>^*Zy#y!b zp>w~T=IC+itFHDDx|VCc9x>mdeRmEuFF*A1;MMZWeulgU654U*(fP(3lr2vz>kS{H zzru$Dlzq=@i}gc!#o(4;)CzAHe(M>L0JK-U6rZmfkWGMF@jJaDr{*0!Ni+(c%T7KI z_{o1!&*!+X{4TJRE~sxV;J;+kxF5eFp1;qx>s}BYgsc{E3_e}9CziEebAER zl&$D(h30$dc7M`b#*pLB8ZwZ3^(y>X<@>oq?=Sc|RHKtgZn=SH#fZAc$PU;5>A!eP z2Jec;v@FMFXJnL2yNx`RouT&cg3qJF#oj~yWE?u9bGOeLu2^I6)IECdPQ~IqO}%FL z{w}pSpL=!OHsgJ16Z*RrzR#WC@t4u_){9@h2QQvu>Ha#?HuoyUbKBeD)4BHh?z;_o z_cZUU;5+m9&Y#?RwW_y}IFrOIor~UB3~X}I88i5%&P~Zr{4`}X@?AmCkUv0t1?k=L zo46hPMOHvQ+U%VTVf{I5O->5m;GUa&fbR+Z@~zxz`bb<5rMz3Cly-Fwu=v|%lM?^e zq{RO)ubJGM zJVH9m6O8>tOM8vc#o=d07M$+Lf<@ekaeSr~|Gqg}oM}HoUBzuWJ~I?Ai?`2WK1(%6 zA;$a=W!h(1(cE4W;2yfbfDhJ4MtlJMYdh<)%Bq`~7t&>_JKd2X{Y~7>O&??EwYe4e zN6&HETTC71pfPZwy<=Z`oGbH3TU6-*hqrmlzxi256#e~WW#a1>*GK2A?*|%c?wvVy4O!JbJBoJ z(okhZe|?y<`LtuczoT22{VwBo*uq>9_Y2w5x&K>jnL77$51&Y=BmG8rR3iD8aW;&z z*iW{Gn{SO@JW4upEq$~?7la3Bk7DcCuyC(ooaTVNa{9qHN{rk}AI&$?dxZbSH&n+W z1~l^A=1#Kfq*Jo%#gyspT%qka@epxfnQC!aEI2-{=uu;m(^m6HOc9 zD>@%{q!oGaqju>Co!~4UB%cWn#+#+9@?OhKEBXPrE&QkkpH2G{lf&Po{dt^o3$4B^ z{&Q$ZanEC@JDx8F^^6Rx*jE{kYa_JkDOC&8LL)LCa68ApioGOC-T{rk?}K5TUhgo`NCw9gXEAE)0LT64-m@!ME;3o5N6_k;g>zL`1&R<|R5o@4)5`I9&KH4H}1@F_);2M z2CvKYXu`myTS~ZuZ|s2Ad;C~58PgbOZxQ*&NyTSp>ig(W`u{DyD}R(0eEY=nekIsb zCfy=%c;yVmW6)kpZILe$t164MVAD1F*|wDMVeW}q_IdI}s;fEF`vz_m-OQfUq7%s% zd2^aA9s=F&5}m^TD&eP5`XSvy+JTU zIU}WG+q^5=fq1jp`UTGi-L|BA&=;c66=$5&nt~Hh13tv-6qNcK@j2!^XN z_H{Em^r63#*UH!zAsb6B9svzHv70?8$a$}}_wps5b`)1ltF)pP`;ZnSmq<&jom9GJGH_x#Xu^&VihnmBs+&ICg6l`j4>f^CCfyD2R~%!5<<55DNJ zO&O7fmsoEF>$wiB1L#o!WCrO#R)P-Hx|TWH=!+J#r$!3Uqp&w}z7gDJPDZ|3LSMOPBV3}_>F>XiZMz@59GzT|Q7H7VSnFqzrX1-QA^Yz8p*|P;> zGhdy{G&kQe-*93nXudQjQ@uRtJF^{q=d#2$%p5v;>l@S`r~1e=e@yJ>68s4%8%f#g z@aPGa?hoM|W zki0`)UIOe7CbnT<@3j90^(C)&wBP#_Z5^QOFCA?;bM+|w))|`=_QC@28_`){Zm)=J z1u30ioM}(#fHC*^0JGM1d<21W4)R{Xl_jIbac`t#JmqEFP%`T0Ee`K`i+dSlo2_LJ zOY7fpX!H{1p{YoH%d%gmT=THZorgDg77YjvTTL0bNWK51?{!CY&rwE)uDXT1L^%dK z+$zsrD8CAWU*OUpzPmW3x&i}GUiGMun?u-%G zM05DDKGl3kTvO|;>S?=(yUBV&C~RyEg zJ^l1C&gzRUzm*t!#-_gh1>f#o?p^lqkQ zaH_ktJCO3%HOv7x%lW234o~v&UF|oky(HR80T=81ve)Ew#KJjd=rYH?gH(7u%D}+c zw-Vm})s$!3H@oF}S8KlT{vA`6WzX+a#{L#%eT<9(GH})m+}quTe!QN|H9p+CQY~RGHIH9J*i;k)XN8lrvRJFfzf1OH3^tqhJE8w z{A`j8{u=t{tS+)-O|ATXnY)Fw(eC0dJ_Ck-Tj5FZIfll*8c!2(v<2In=C-$sr?Y9d z15ashF1S-S%)!$t#?=iT?5rO-_Ab74Iy}A5;9|DD!!1w5(;=oT%Z_#`bMSPKho{Km z22ZQN(>|i32gcOlbF_cHgQshl3lC2->9@hrRig}!?t=zD&N+prxq=gT+6JELolKtP zH}Z*l-ZYQr%73=k0OP}6>_zRX6vD%RQ}|8t6Yp`5t`h_9Pt!&KSeWwHGQ;Ciuz&mb z4e-Sy|371in7$B0q=awbt4NF!bOOGaO8XD~3$!J>sCR#!cmIOHA$JX#_$KjVSVOdD z{66c)EBq8QV=2Gu$TxR4i9bN++kua0Y&Z8gIW(4QzwGAyfjo!Ca_lW`-tWjWx*7a` zvzzxidEy&2tQq)})ZEE$How(|_VVmsQP0HmGwVjKp=Uiiw3uW6!YeoHNVfg7`<#TI zLm=VYw^>KFjSAo3(&CMLS7U58IG$xc!L#V{rzXv?SDQ554x2RFzKc|Jsdu#Ie;r-c zLKk;tJN*4&-rWIx9`)y!v`uyB5#O!2_0vi5I}PthW9?Y|g|#DvwPO)?csn%trdc60{dT7^hbG6FvTXa0oywe8Sz6!Cx&pn<)4Bq^9{mEn;?v1|X*}{f zEn!UiriY@2W*@mYto%4Iba;Bc{SD0pc@@yCVlH{}p*b8(nTbhJ=HFdk)B{;68D1HH zXQseAQ{kb+Qd-_Ca*{YhlB4AB)rKsk`z4x?rQ~CpNCyw>Vx0lsIWI9I8{BVVt@$hK z&daXcG>>?-$BBuqzB#ycaromVXHU8}eX&@RymjX|&!UrLQg7Wk?B*qriYBfC=h10v zV*cEc{{Z)6A6k)%D&%97ZGTD~b5{Z5^g{=WlC6GD8TN&QveV-HcSncscYP9`rN0`F z@ccvmYyJ6)Nz?83OqyoDN-DoA;rPq^ck1VW?^giBslajyFufeuPG%jNgdB9)t?gw_ z>?Xm_VlAp=&t7BtAKu3|l)F;%OXTnH3UeJOL@vU96*!+bXRKF&ljDhXn$uX_-acy_ zvePJJr?$qFVC%uuU~I1SYAbzJe{aG6PcVFuHYKA8cejIg&icc?R+6*+*yIUjk_mL;c=%Pw@w#rKx8xEtuL%S!1~!!YtLI<<5;owyye{i^zZn7ntpU$d-$f-o*!!M zsS@oaLwf=6BLzH31z&oCH_+X3?gIi(M0?Hb)5YD`QSv7e9V&K|=+GVSW-p%E`rkQp z$iCT(OqUJ=$<~?G$=2}I(Bv}FqZ_9;37V9i^Wzl9cR~7$beyT^IH|zXp~oz{AM+$! zDk9IJ$4tAIo7dmu`6AyJE%t%Wl3pcx^yicugJ;*=Cz{N%1BUL>8;^k-qRE&)yTmEW zw0#L>k_ltT$=*H~yy7iu>zMF9cOBZ{_Ek8jJOt{3jB>sxWb>;xSxHR4Enu2DbTor|8vpX2hf)S|ME6c1-#R+}d(@R5&KJh>^ReOkvG3>(E46jH zb}F#f3g*A14atyJFJi8WUtBppJ|371=vx{dJ;$l@Ds{A`5x1hIcC|13Os13n68UB= zNOkz6^r#m21ovY^*<*b*P?0oK=Z*_}eUE7EOvTpG$o|3<&KXx=8{YtrsUWWMq6=1- zwKf}Gqw!vaubld}j`oEQb&S6nURC%1(f01~QB~L9|CyNp6T&_Bi;xhM3`mgzVhWZa zK@A|H21JW~L7_F8T8mVzAP@|-L9{YTO9d^LV96Y$(gMnODIm6@_JUHit!)W`$|M1W zhFiFp=l$8|Oc)UB^L;(P-yi3-&)IvQ+umz^*4k^Yy>_YE0~?BqEl*2U(KlU5g)M{J zK8fLn#$hv6Hbi=}>>1f+{B^&%58I7GfNxaYDg4)Nuts}d zieBX}D|*s*w~ss5ev7euJ^M!PlP++*Nwe+iNGGa~zal=p_Q+LiZuwPz6@ST`x|91Pj}WGovQYP)AE`uS zUZpqOBgvXx3(fZ!XicwxS9|1J)8~;^%~4#Pt9JRj`WKpYmgu#E+0RHmo%$ZF16khP zy02FJeLfE_9&X}4YoFcL=X5;#oTJ#mG~Zt}r%kCpZ+w#fo}5yR%?z$&JGdfDG1zji z6@JylmH+U~#+TT|m6|yzCT48v>_l)U&2BK|>>u<+TwzA(tpB0ycl{4}S1sp?H~BB6 z(BzvY6_#AsJ`%Hy7~d}+@zZEKCvI(ISpzhaKfC?1xy)~u&@Ri{CQRS^#r8R5&G#B# z?|(us=e6A-tHKlVfEU^W`m7(Yi#zbe53(J8e3kE1KND}x!HNkkR#XfMUG^=f&TptA zOxT)e8Ql9v11Hg|)K8i8_t9k5w|vwo5anN@dpieOTDst4n`4){ ze!0(auVh6XJn+jTaA1RQfPF&16OC~-CXlbLvnE#ijbs}=dKUS@heT50gYK!L{YOGe zKQLv)go`huPlN&YkSCt^Dn^>#!#i@pg_9H8gij2_R?B;%`+Z@>1gk6dg!Y01s<#_j z(0A|Qd=lR-A7G<9*T{Z~J`HX<7?5I5=AU91pm{yfx8F^r@9A3iOrV~4ztp7J_6?+` zpvCWr?<=(y$b%fWLQ(kp6Zks}f9}Uexed1c{pfSepVR)cM&Pp=TL+YYt>??_ny0~v zCZ2VKXB|ZsIO*!-@-3Ec7&4+&7kKV>=mP97A$wApH#O$3WA>?B)&X6BV?6#l136ye zyWVDhz~P>p@>T3lX@egNd}(7QdlJhFxL)q{mq&?D)zI0BygW5n=+*xUXR%chhst0| zHhIW(Gd2+K%D&=t%TiR|%$2gQJLkfn&dNhil6X4H8$mjLHSE37 zd z;<}da{DV@y`yHiV(hjBY)UQaTt2p+)FSozO@WsggTU?$#g#5q7;oaAHj>cmae7|Cb z=e^Y_PX5c}i_eel>J&KQOX<_p)Hx9CJtA;yO_xBlrE5UC^&LH2{->^Rp^pCBggmI* z<>)ps9$rD7+DP(4ygJy>Kl>x`f4w-bGUeEW+;Z_m9c>_9o@dH3?apy!Y9r14ik}}+ zIsMWuu3UbF^C`QJyrZRU!UgiJKEs+@-=p%GTckO!+T@BW>!!lrKIdEa(@(+^?IAOj z|5{o6Er-9On!`=EzYf+pI&HFjC;zLw<l2*WD?2!ob$v5OvYu~RCMoCma?O0iGiv4?q0uv|@ZGDM=m{T$j}P>=B8~a@ zPtaZ)_67?(>R$bM@l=q42!YA@ID? zf#?OKT6>&r>|e?Dr#w$g$g&()=KRGiyW>KbFsx1MzJ!b0I(D<;yGDk!mf>x#Yx(}Sl=45n zQ_BBsR|*YmGii#whBRic5igyYbrUo;0a_am&6PrXH!_a7fpN_B_dt_o9CK7W)t@%< zdK;_RZ*wlXNEzvCxIyHDiJ;tbbd*-SgftXifMeU+KNH z>HlE+RUbXiw;27go3(KZ>*rSQ^}H9WC*6N9vbfZ(yOg@pBWk<4I>>DU^{F zybyJyd*)JSMU&^fA0|8b3(2qTLcikIevJ06`M6KumA!q@MN;il#wwfkUmA$c@s_?^ z-7ip2U!jjg!qs03)ND_GZHq5sv*h2`x;xMMA-z3-omDIqoYWtuV&4i(M@O*;n`T-15JG#caPRs3(KxH&%^&WzKnhs zmJQdlSf}JXE5}|<-ZxYqI@}RoAG&yxvVkgt4!6aZK|A{>yVSfV#r6`zQu@@a zxiF^5q}ldi(o2l))#km=hIY8;;J}XeH1_Ru5A%;;b{F`jEBw<9K1zn4Qow9*ySPng zN6s&=3+Ft4-D5ecuPe`~zV6zbRoC@n4!M(=Lv~_!OXq3=%QYV$+?LMe+Fx4Ny#}{` zzN7*>sPw}J{MN>sIhuA^;U?N+5?(Ge&E_as$Dc``Uxo!fg(3 z*YPv&7&v-n8FrW&{TI5A`0Yj7Rs6V(JZJ2dVcTxrYbMVd)|`NJq7u?@Ww5XctYf~z z_-$$KBapgaX{y=dC(W+rUis)s-zl5BaapRtwha3@H{aEF&h4`jf9Q7CK6)GP)PFXC zFQU_@^sVSMi4O{TiyOeO>%p*b^wD+n)!2JJ8+t8v83)hm zs)OM&>@nNBVmsz}bHTN#oIeeQg}vT)=Xw(xqwLK#mpV11TiA0Ef5+z4mEv*f0m89l z`%-saVFurW?r86;7EL?lTKA$lP3WpU-FkOXPye;6Gx)@zIV|bb*6jw%*5@4=9Qwyi z4$k}!b)`pDbfn(X-q2Ovp%?)c8tw$bRu8API{KwZW?tH z&9l<%CGN9K8FVn8GK~`+Va=_1H#qeY*Tz1KY`4t(`HzMf{Q(=X=3(}3*H|st{sA@Bpj!vpHOOuH$;slUaw+b9P27x3naL^F=Q0en#Z`lzFH zpShQPFO*6Kn(JiyU!u85NB0WY-0 zFI5j+PCDC@Zr)1ep~p!s3xgN9`OB1#E+>6vzMJA7RQuh!?*f~&`ob5UUI(&LJCo@jKscfdnk*LxG+WxU>4CcVzv!JHGHef-;VW}E%-YPvx)>G=M(ToFcJ zGCH4RyJA!~4)liWc*apMrXG2-m-Jt*9reNKIi?T&(HvU&f-ttZ+OPpZ4bC49qzb~KI2+*VZyDC!5Had zi#a#>{F5;21Lgaqr#XA!tR`=#!J}mRe@GoYQO`-W-=xgw^w8jG>faP!4jt^H+|lW? z?Q!vC(2D#!Um;I4>YXRtbN1@FhwmkJL8Ppmb>9?pfGV$ljP_Mb9`H=otAFkr*!(ei zF!q^^=)pC`=)RTdn^!%pHfPP1YjfMkbA^*%ci-vQ-kGO>R(tVH&ys%}y4O1cZXMmP z@xw^UHEuEX^vZ;FMY^Z+246F6Vz1P)Gk1{xAPDmv7QPoe^LAl98wI6>%j7bFa@zUTD7G z{l96ZZo2&+>b91-|8^T?Rh_J^{@LErXesdwnKv-}T>02A{b#Lj0r7{<{_A_t-Vu1w z;T2*(L1Qb4S77wtDII8!6g#_XO#j9HZ)h*Y=)bZph!4gnFFDveN+~>jjZ*0CDz|Pi z=>gX4X-@FJ%?HNj3Qi+qG*@uqi*tdkGUmChpDzirUO+PL!*;Cc=U@(S&@7HyIljT6 zvf=2U+eK$df6rpfST4P@%9oV{PTwGTaR(TwdfF2`LEj4xN*_(4T_iUyBG0j_q}y%X zybRS7kE4e&AF%NVyg6v@24jfvWTkajit=bTqv{S!MF_ew73}I`MTHDKWSIB<$t(VUmP-Nvi+e+zkCibHveZZXP%2W`wBzf9`E4Fe%%koq0etz zQR;c`Po16oJ>*XzuW4_Gz^Nu|<@-AYH1?EFpW5SZluO>9fIekU+XD@@>R&#`CjJNV z#CxMj&#|ZRAL7rAraa9qa?6cQ#lML+*O{^mdjst#+V$t+(}ZlO?@gP!z5=zr9}UwS zyyo8H^{Pv_#~H^Z+e`W0mD`PtY-iCQIzq>p_FZ6VOt1QdX}?T6rc*W1&M_Of&h>w> zIk4Du$lQ}*KS{bwx>Ir6_YMy1f_~qXF<3XoV9DtHDd_*Hj04gb52P~&%fQd1ZK#Iv z*8;|0cQgJP!}#k`#$O#7e`)Vp=~c3&9fRiOgA&`fwnn~YZtSZjc%gR7=S!r|*QT&W zkha>;R2*2~OE$5G9!Rk^7IO^c$mBSZY;A1uW|UVWBcDySh9EbG9Kpvjva2XiVv)LJJ{^TFqd4ugK-Ia!wgz+ z(Yot4dYm&gE|L!)i5>@&r?G+blIfdSa!&Dz8Q+Bp$>-AQ=b?5Bgi<8O;jhg{pyh7_8v zQVPwKD1C8c_-^*wX+S=gfZvMW6Q$ks%_p%@oZNL`%?YK z5$ec?R61bIoXD~)qt|8IpSt-z+4cZBa%W}*e7uw z|5=Yb(ViyKeep|JZ;k96y8GbgX0N@2{Nsi0mA(5Lv1eX8#ChiJNoGITi%T6GTchu$ z%~qgWtr+Ni?4qUx2wU2CwCq7CI9Dmu0NMAjEXL{hsxQswOZF{qU`acxe&`s$1N2{VgayW!(JQ@Epj7KE9L!Ku0p(lwapXCI>k z#*T9hb}@eh9SSoCns#JtN}CHa?^HRod(SfqpBfVJ|Tqz9h3QctbgOkDF6lg6KnoEQB(xE}{cUC5LurJ48?WuSw zZayR42Z{b?&%CoZd=ws@^V`ptKpQrD&I}^X(2biKivy#5sRoCIy{Ez6T^wPKB^+~o zDdi>LFzaN8FlHHYd?@oEFBSz#d`aMNdU+W*d~-Kz@KA6!C5FX}L2JO=B^>1(6F3_2 zZD|;2HPSl`i@(>o`9T+dJG%H=LVksdzgKad>*8+_ z=UZL;9nSfb82%2iBCm06@NzgdecE66`vG}oUU_J+`L}M~PSrz}NUlfRymyr+97MmC ztkAu`S01uO{r;kxw@rCqH}fFp;O{Xo`EubdJQx+uLK`(o;lnbe&}@Oy{#S#)8<)kf zw}CaO8Ur7r-+sY6JR!2`j&x+cf|AH!hj52D#;!LG9yjNb-rFAQ#L%CR50<&uAO(BDYP zT3oz)BEF11oJQFRl|h4##+4lhpEZ7;Pr1?Q;h#Bi<#qJ)YRc-#YiiO?ETc$XA`zZT}GUH1&seQPKF>fn&=Y!p2z;2BReNKB^ z3%yE)Yd&NL?bnLktGj~TS@sh8OW1wPV0ViBZVP+fkgxrBV)Uza7Iy3U#O@K{H>lHu z9wEEz7S2nI?J+K%EPWg%Mw~q(SlB{bP;jmgTGX0n%~>ywk0<+8v0sGe@;TEuQhp@T zoi(SSb#n40h~aq|bJR(%JANek&dEu^=~3*5lYObiwm2EwF5~)U^npT-bPncnH~s^E zv{yLT3~^AHIge4KHH`0B<28K`ek3*ghtB75UgAp~vz&2e0q6C|%wym|gnztCZ8lN3 z!5$#crfBpL@-!Ezxi0bX-ERKFs(YvM((P}%c@HXYs`8TUDQ@0<=KiGca5rX48hY!; zJ}~i4ty3kPik_)-GQ6&|lzjG(%l_a2?2H-qO*}{bX2QAJaqKnkURTSfi@JVr)z7|y zDZ)#Q?@W10qp>sc9ap>8bMRv-U#s}sV zy#D=N{)^HU#`blT9^&}BX4!j@(F+bZzOMQPE5X`$3j9$F)EBTvKf-=j=VPRFyckBF z<;Ma(mr*`@Sdp{|C zm-KwRy^gidTjS!=X)noY+Iwn3myk7redKQ^{>U|86!j_;R~j7hvzK7vnX7kcFTu1k z#k=Ym(-mNk_RqFLN3XG(!^F6257q?k>Ek?WN|#VBHoUx*2_f$vXFWB5XXT=I9K{x` zxEp@XEqwTP{>xqe%XbrRK`|I_e9qm&@tDSA$p7U@?5PQEV4I5QUd7`Oj*P5x?%l?{ z;sHZ5uXRy88}su2_XNzSI+HhnkNDhmjNO zWBV9w!k1#zcJF)6ejUlSFkSzqd8}{pe|moj|2F{K7e2fL9)JaQ{Ce$U{D*%+V3{u! zJ&txwp{Fw6{pJGC z*FN`p_7|Q5*6gG2V$ae0uIE_|eCu((N#7zpNO6?Y_@>v&`vhLAOx?VtIBj!OaW}U6 zLIvA8oMmj=8O#E-!snXEy`doLPDYI`4-|6%Z zFhkRc%|9tw+?g7)ME`1k#^uR7NC)>MASUO;IA#-2Zo(KRHG4&R}2TwbE z5dFAY_#|z3a=H87Cn&47&O|?E%=IGkzw95nmAG&S#2K$VOH}u~xX-`r5SYuq-#NJY0^GxN%;xnlKk0GZd zEBkXUolWIOkR!46Y|oG%i_;+e$$wi+H*)w$ZM%p%RnnU%Qynk#qvt4A+aFP_KVq)a+}>|IVDJKwR7vhSH^X4t3cx7KG)a^_O* zG-b)e!KpGnG1<_OXrboLE5ZTp4a1wJzdpcsa7e|K^)sBfDSL~ZHv33)39qVc#jiKg z=JF9Oj&xOQ$2{ihwNCJ8CY5w*ZWJugz z28x%UcycY<<;&cWu~jF=Se+Sbbz#ia6x( zk1YESw4L4<%eEWMh|F5c#&TxwHo93gpy7})bA32Qei~LOc6*vD6<~a#r z?dP%^ni6(NH~lF(Rcvlt<9ffY>BAk;P1h8K2cvIBk%a1AL|EiS2m)|3m-l}_qac5@T2z}fDom>yS zjDv2jV@y7lwfhOi2C6ZMb^nOwJao6b?_Lo)HP+ExX7e1`5?3(pkpJ_vL?{12@}=i! zUDRp(pEvLNTHxhB_h*lVqHqKMziI7dffC*wt{xEBQki+;mEz3JK~H{nYinM2PnWNC z&&|l~zQULFn))xp^{KvtXBypvzAJ}sS1~ph565KJO_C=`*o46n;gd7(3P$t)y?EC* zRZe^V!7VqKNxzPyY>3Ky_=Lun(T>w7yG&*D@o(eG)XviT=2KokdGvwqrCv{$uI78t z2RV4mVCJn7Ur4q_#x#3@>)*PIa*eI!H}DVov93vT{u!<7 z&fg+W-+z_c$3u9Q%42f+MU^w(XE2EMZLNLktGJT&64)nL@8)x!*~|1PaRYcKu{X`P z`tt@V)_L=8{6^+sd-LxZ$^M7_#wncZJ=eJRtl%ETU5kn`Qusdk183S(p#jG?D8v3a z7lP25KOse$}CD6?^(9hM-(N*BX$Z$)s6W7BJy_@Hr7e8Xb|6}|J^Zl&h)i-7m zKO)m(HCGwC%zux`5|d{qk!OkBTUV^Yx@Wg9l~d*ZK=ahZKoz`@>GHzwUH`}~)QRyz zrg*_8EMng&ckO6`%L|V{lfu#SdEtO)6<#=grx*RW@+>d3V@*hPEDjHQy3#&+?x+0! z|1BT<$i3%Z+++A)TB^eb;Py)YSw8rX@_0UYpKJL!uZlW+@DbNrFTe-EFX02p{3z}A zn>b#0ooA@uUNvd5z0#!Vw&DW)|4s9K_@47)rf3b!esngY_gM+$cYqzDBctzCo?Y)& zCft2Hm3411+hp7tH}t}T1KVR~Phvi*1M^W|t|R_8pMVp5hxXWzPe5!PT+G)Jnp_{# z9Y1~()vq!Ev_3GDs{{@TQC8ARNcLHKMBn(WIg&u0$~_PZLh zTXXoVlh^$Ho9rp8X9V$^(foZ!@-=_|TQ^_p=QV#n7XKN|-)|;=D{+N2f4_eVlJa6tiryy13gKy&jRyHbWVRDyAS&bouGbMiQOX4>K59`H)#I4 zr=cxu52X4}(cPEnWcB&i(tl%^CmGH9lUDs~r}ZO)6+833|BcYoW1N3T)pMb1`oPR1 zpKj~qcP4)dd8hBiCwdY-(NjAGj?dB@acAZyy7b%O`a;M1LDyGz?f2jOpi}XKmM-`L z{#a_iWBi-sP|O$Sb9lSX=sjt6rE71fH~h|;Fj8TSGq0RxAE!-X>p*r=R*z0yU!4$c zLa#r{*kKCzbFP0)nte5WAiYPiH{akJG%pa}_DnID^@&fmi}{PC*`h1mGZ&ev{kqhD zn&VX;=o?~wHu2Xln|oNRXVP?ggGp2EXGl%VusB|f@vHfcgAPwBPC@q;+UUHsr(YFU zPh&PeJ~@gzNHGVFVy7xn2p@*I6M$G+Rmdzd_jce3qoyLk`lUfEO`uT63D?o&P1 zxT~#&w|>T0vZe97jUQ-C#kizODdVIP(j&e!bjl3-X6{!!3;C>7O4j;_Z-L+KPt6)p zeEulc80#qgRf*myy?KnuOEF_A`5wzhE5-OQXW1j&vYz;w#Q4+)pCTI$S(l}?>Zl2P zuiE6qt`TARmK2(QVy_ORylb#hzHgvOQ|wIAbH`K{Ub7w_)9=`B{MY`>+Rz_cueUtH zj3ndU;Xdzl52<9k^@PjQWkW;1813*?F7J^}GbIf@WVF5J=0(n0swea<@OBh=e>}ep`o*KtFBVy{ z8?ol}IJkIr-D#RVioVppisygF|L&O)j1v@(ljwOB^ z{~ufL8|#nNrcH9}o^G2p^Us55EB#NbKNgxg*>gDv1d{8(0ernMu6=dck-2HvW6r_W+zT*HgB!%gH%2Om$GsWCwu=E}Yl zE)5pe=UavQ^Q}N{+RmAyjnjiFwU+q5uS*lw#@74`Z;!Zkve>xde7s$;iI}FY&o4Hx zzHeXy+l*ao8OKbHTQ~;cuaeH8HUA&V_GRW}y=MQ8GWKL7=4Mt8PfGb+=yqy{+h`Br z^Zeru7D0aopATvtmskMNYplrUlsQ|n=CMO5{ ztUW!JpAukRCeoCj8aO>SEwF<&O$>s^Ym&lm(}wb$sju!3CO+HiExWCV@arnqkNQcT zcqZz>Fl`I=N3T+OyUq>>2+UtBxdcTHKAeGonnN4tq~ts91}A_YF{a?N|`z21r+nK^~FU{igyY^zQzoWMm^D8T%1 zw71`xZyICfeHP^+D^lPa-LHP{?VgLD^gi(|{voE^Jl|c%t8r>{4SFW=(na&p*z=gb zlI>3WnAG|3LoqsRvTZ|y!cy7pE}>l&XLc3zSM)+MZJA*orQaM~DcSBrd!zpwyi?gF zYJc!r7?#iZEyA!a>_-RR<1<#sJ_&&+_|-tXk`Gv$&^pd5);_VFvA=Y|4#Pa}#BSJS z*dulp>uG)LH{$vVo&>L&!K;|BV4R(=RX_Z}#|?I45i>IuC2iw`|M^ZFBu0e#k%T+m8&PUwO|u zwbjsI^IE0Qb;zW`mhtpiDSdY%eRu&9#Y-64V~Haq3D$F>%lW>IoCVY*W2V< zqx@fc4&Tl>ZM4&xpA{O!_h}rhdfMBXKHV9!4VyZPGL4V7_U?$a;j`2e##$?lp0++O zdS&RJ*Em=>ndfOw9L2*e<=i~O$_{05o=ZP--=f6Z%M0ITPY}&NCf?q+F!oN3$?L1H z3fGtW!bkGG;Zu`5Vdmd<=9OiKnrB$yM)rg+fRFvn;5BLB+gFB+&k}pfnCtp0LpI+v zdRvFk81|ki@A}x%qs%`o_Yt>_IwknE_`!F*=hyD{4&=KOo1ubrK+S(n2(Q5ZBh%;n z%cYcQ+-yc{;9r(|cKpEj)|r(4_!aRT=ULe-F`H_47wF&A0}@TI?awb38km<+dH^8 z3BN~=btw0r=;Zk42`A6`=V3F3XT$?p^k1wmg~74&JkC{ad}N7nZ!ma@tN-)u!d}He zI8_SHurAEJOY5djp>{!`yJO! zw2ypa-{4%|v6ZyzdHUp+#gBUK{P zOtFp6Uq2?AjoHmQZeXt|>{M#Am%!vD95Xl=zYk*kZeq8RBI!iHc7Ot6dhErfxqx^N=fs>N?;Omg zZXNgR$Nr@L_!{pqeS{r!vamN;abtzIWagBl;+cN-_^>KVW}5Y7{9~@CWM);Sv(}7r zvp<~rNIK?VkN<9iR}X#D!K*Xh3<-yfewx=Yln6$}bj&BNcHZ5YcMC&;_@oPqj(ME? zMDp=3u(zVuV`q!_X(JOq!H3@+^ce_^i8MiT0qmyr@L7O;bhQqixZ%?q_|BEIyY5k* z8?fSu5H7NO;fa%C(8UY@t_@yMb~V_J?tLRM`SzQ>vO(42|AZfA>8{h z|JaIqm*#?dDfUg?v$(gAe~;mw_epHU_NR)( z;^)12KK@k-|2n`Y9pRTw@J(m<2ke`L-w8aUy}4ufr#ZTqZW@cF8;^f~{PkJgG(Og5 zHrRJkx+&x5_*k1K@_pf9{DX9R;e>cvJT`&680LM`&AZ;@C44F7wd$0ySewwhbkhsP z+WZ3TR^*!avKP~*t*-rrGIydk=p`;|b_p zxL2j|U1#=QLq8PmiJ!j|_ZH{1&PD#2eUuIE@vb(sgM5{IN*R;XQ|9=B+(5rwPydah zAFrc7$AWv;5+~8cJ?OlOx$EuVTRqN~YT5%rk z8LZ=Z!aDy*aEE6K>vEGyW=fZxI~TlL5{Gxs4;S9;42;G1H->k&pNn@lT;=jV?~KR0 zS1c$0kL0r!(!o38GqmCzGN%>qm~V*3yVEY-HGWhaj^Um1&d0l3&%?Va-r=eM z{a>~F3b5jQ9dRLfXYnq>;N1fTQ_}4O(pJ2KPK0;&nljezKxeIGjIV`vWt8E!qAB9Phk@ER%&;7bb&mdN} zgNxkPii_NbPh7M1V%}Tq;NtHLEjze4g0^VI#TW7oE_OlA#q`0eXroqKq@9GDLGtts z-!y54J&aU5H|Wc75gdw-DZ1r+Oe}84d}|8xb*ap^rZL}|4nJkUSDElv7JQZsKIWhw zw)@gG*or}}^&-+66?fnW>qU;@+i>ixZ$qozICnk1da2HyGcOaX<7XWAbByO0%8|*j zf%e_SIzZyonAkNlStob`9ey)w_A)p(V>w?&c@1j9pJ0u)=5eoe^R+&(iuot457a#F)#Ph!pyqLlIj?os0R}l|EtZ+Px{~u;&R?T1 zUR53VRhal4SEu=l@}LvRh1=b{KPwNLfbi~CH}4(gnQ?!LJ<-kko$|1S%1(Kso3~YY z`N~VS$GUmuz07xQ@ui1ZJyXp?DC4~1F_iv1X(Ira3M!(Xb@WYeA$T0Ryt?^Afp|yVc-=9FM(v_tjZ`V3% z>gCKvP89PT)>&htr5-+O)RV3un?@!2nQS7@nswNz_H4=&t8u&L*~s_L-fUtz=a{*( z6#M>Ix$GtPvbN%!@@)H_SUEOO3=^;XKop_TZA$zCVAP@A2Z zbt`-^3I6yNd~yr?ax?O0B6hi(;G+r5dyi*rb~|G$`NH!$^fF@(*?v>#C-Y3&ZM5k# z+R^kUu~C#V_8X~`b}v#&KZve07Py4e#Div!M(OvW?}jzOa7@PSAQsw|%Cr-!r)uoY zUhM5gg%7)S+AH)7c&maKi_&SVO|JbS!4q0n?8u$|ssqohz`t(A4DWksEsozxK>5%_ zF8Qw&vu8Hg^Hp7_Ud^3KxfbEjme(Z8`3?m*JpcPkM`hz=cDiOY_};(vG0j5gT5s{{%Xpy z?eD~wLG$w|dyl;8O2tCsJ>#4;hkr2jlkIQC*Qbx)q`rgy>2_&+8U4DCvfq+d;>%bl z`yRfU&c7jNzE4|j)tn>z5}gWN7 zqh?)%9{=TXO8ULx?rKl_v--W`e^+(R8g;Mz!LJl|7yCXkV$-BxB!YZFHy%{rOEs}? z{D~iAWz!B3%NqP++CqAnuAf0ZJP2=;a`fcL;UGTs#t%6*axCRoi(WK_^!P;9u_L37 zBO7$@a9?^kv4_Un@YQHHZ*rn_-yZp@Gmk=C*6H&&e_>28axHUo;c?1iUvbu0t}6{j z)-hkF=iD#9bx((+;hqlVKFaFXuul@>5|xFUSUbsinDxK9*DpVIuz+WmS7SpzFr;&! zG07ip5bxvH^b;cC>ptdnj9=3e=+2UdRc^jTzLAIcL`fb#LVgfGDapgRoa5VL{F)x- z9NVIihYxX{D?h1x_?UI^$5f9oM(sq#=Pu54hKA&yEn0on&1*0EV%(|yzn8dqZIm~J zJaG9bH_uX@k*68<6K>uqy~D^;e9YZE^Ilh;R=RnIRZsF1Y?|Zd9Uw1?UU90@7f$NM zo*38?wjzIe3_#z9?i=O?BNe*NW9|SucoJiSF3^P9O>+6ntlOZ^TcOiQ(CfFL+gp$q zHzO}5LeDoLFD9^_+>sZt=QV<{@`Vx4(??EBJp3soi>s7^X(dX*FTYZ-Y6o*~lEqb| zC9}iJvSo9kO*Ia~k9Xs*Xcs+SYgDJwUr~59n|fut?%%Zm+qc>}imYBuJz;Vp|L_Go zsOcTrL(Gjr%9!(cUozUqKhJ8Ky-WQP_^V4Bbk3{&I7xCX3Ep-ndg@y^H0rrXxCy>o?_p3emOFK zsAN8UTTYq2(e;HpSLR0xLbvmelF@zXd()4!>+Oa%g3Xha((bn@r5`3KrC)AQIwaUU zQ7Qd4K`FRjsuZ3tBz+iMT$wb2v4XeJ2QN1;pBIzaRq(3xv(@Nlngeg++TX9${pv&3 zPuJ`U78)D>Pd%YN;Q8o>yxOyO`iQOzmcEKCpS8VXs4QRh{bk0!9~t4u^PxOL>t?oi zz0=k2R^s(D{nor2bA8U(wH_MQ{E6&!cTZ)`px6q533r#c^8^Lbi=JceRMNog1T&sp z-;W>U#8`q9``ZEVh>e^vbKsK&=+DH4cA1LvK6QJQk=q)+eHAI*9a4y~9g0e89 zPiPPK%%l(Xuj^bMf1Xt1K|M1TL+@Rlv)QHNo84!M-dpQIHzRq*Hp*Ud>G`?~)q~Ea zQSW8SHo9~zolxI+&hy~^`T8cxMBhW*I_8_8-PM%+lDrM(8Oip*3q1pxeUp06Ql|Bk zHL6QmaG|=;`9A7~DAPN<)?>zos4;6p{;2RMt&7g@=!}VHm@)BAGau8RF)=z!9qa1$ zZkN82Z5zC%zqB9f!}OW@uGsCnC+It8jyl_($GOJ9iY@yz?{)f-HJ)xi&LJ=Mf2ZK@ zbK8jf_`hC!^EB$k|KFht)#d+2Quh|>Rx^e%@9pgL#dzI6(QO~n+6&ze-_Pg%QPkDF zUiQ#d-=E>SfPBp#iT3Nuku{Z1Xbx#-uI7-w&^{^lX7RsiAN;InmzCRHUG9f{X+QQ> zgU6)Hoib}KQtTn7owDq~N@=TsN}>G$O8JNWq<=8+1f`#~L1#ig>pdcDi{J2Vcws}Z za2@*2ufUpLR&F1*rt-~UpKh=UJ4sht?pb#b|BwSuTZN7J#F+q>KGoX4-d1>tS?kAs zs+IrW>1>McCV$e{{zpyN7LH(BAdZt+2N=`a%7{f#_5wE9G$*FeQhW^W=NQj1kRzMp z1hxgmyMK*Txq&lPc=+iB$6lF!aLX))}gaK zL8`TsvL$@i(=lner(^kE#$fMaV>p2B^a48EfpJ#zwb&x+tFbX$V>LITyBs8z!CKeW zFyvDwK0z)zW(_ulgRZS%fSW(XwKeo3U%FLa@>jUFh62uIcaYEc#hlCTAU~|0oXhSY z|E(3uH@1c>dyI?25#<@(DAm5&&D*HF{?egR>=AC>i^?;)QL;VE&9jv!-3UG%;^sZC zJY#FfvA^!-nfJ!o8jcxT18arAlIvOTBp))VQ{W zGEy7ftgpT%T!5{ifi>@nDJ1!}1G}_z8S%&Wp?Aq7=_#6TOk5)rq!hDe7rV&+Mvp{qJNJ3LE)DHLu1cpl#=3p;Ofc$F z(|@f0HGK=#EA5MYL1};RQYpHP=t_16KdEDP812{{mbo7ohigy5CPa(gCF;wFk zY!P?n1R8oU?&^C{pn*PZ?AtxifUT(E6nI7-HnfoPKMg+C;Rp4VH5bupZOxI=UzI`U*L^dFwRE6tM| z(z>88^w;>W89jS3x`z7d0hbQ$bJOpU*7F=gbDqS7f%;jyE@wWdwv;`;T%X|I^6$nT zgN%9$8B$m2*h?y1dHEY7S5ocENF^(c4MTm-`<9t9)`Z2EK^KyVi%nUw-7~(dw<8;$ zFlFqy5?_|@$ihd-YqgaypR<~0%`^3L?9TD^q02WZn`z3jY=3+iw78G58Kw-I3w$BD z+8Y@x{vs~S^gG0B(4?RBgo*5ls(kqwO72GQHM+=p$=dd`p{a+TUZVW;E>urGBr%zs zAJbXz?}Xl(XzQ4btiLRZhwfQ#OH4hE)QEy@eJuNq{8+lp>u!*tg&p1x|GW*=Z zcx~w19Oe)+?9VCJn9Yfoly0ANuU#M8bMY?WvXIC1dwYfW2hRG@H2VhZEA(CZr(vz zkpIC)t1%JU`~t7xJ3s~U)daIuHIBjfb0KMxC&Sgv5s+9lxty2E?HKpmn=2w(tE|Rv?pT{vmW#3?jG7}l~22AKB$H< zQoS$T#FsoiBPCEzf1K)_8jx?l-qX-KEzpRKNAZ>#`=$r#^D_eQz(qq?`U-^ z@K}TE$q&r-l$v{8UoP=Pfrq$hJX3v={_mN)n(1fpSAlD1yN`b~>(J=SS>Tp@Ml~N9 z)0e(uu#A`x$Q|LQv*w9)X7q#b^Ewyn{^6GE-a7h9_<4;f%dkI)FQY%FQ5H01srH}Z z%jom@logt?Wc#i7vUJCu^L5JVx&_TQ3X?zK+R%7RpMgf}tNqxXE%`F6x1`GuYxqbp z_LuxvtYP8;L62$&tx?ONEwnDxV06Jio}FnoLIbidIQy%o**$0{$w|>>ER}sB)6foW zOIj+t-X0e(2;9_~r(afX8)id4Ew29ZNoB>b`edt+x!(xSIwwwL%XZ;?H*Dm|%=x4+ z=aY)_a<2ND#*GY=dsFc>NG-3#FQEtP z#7pF#0Pg$cmjLd+gDpVu;v0p}!|9U;t;j&^1=!wA4ETW!)?z)Y!nFsCWe!(!E&bg5 zt*$*lJ_${X;bjxJgfei~V0Av{6|PMnm-AfCh1C~vz7?Cm2HJm_>ZgO#*D%NI>J%Hv zlTIP~=Lk3NMb$Glfi!!Vn`bM}*aWaKxp~hk&)|8s{dG5Qh4O^w#jIO*^YmWU;fI@$ z6?2jC!fq26USoOq(TwRCUuaA(94}A`??hc3uTlD7Fp?$wmTo^gYgs+=CRrHGzdQ)d z5nB{YPHt=h2lF+a2M0><8*t*k=h)w)j&!tQa9+H5H`g({fYyCE&m~@b-Y(W*r`UJ6 z^|uROz$=a2ow96ul3TVHnH`<8*?9;5JHgGbF zfJWMTE;6H$^T*L;AK%s~^aQ@TKVkjAIO@&ybiVuDu1_rO$bYWe<_~ou{nPV_p+Ay0 znewu(k1uUc-sq$4LMvH=5JTn}FxS;I4r zA}6DrgOMtD+|U=}bpQPiJ!RS&85FqxhfiJM@h{X|oBC4gzsfl`dWh)U9ot`_u|0BW zw(HOHR=u+hSiWdG!Eep~%f}`rt6nvjo?{o1N>-^X<_EFbl%?5Uk1rFg@&4ybS&H2^ zz6^Slocft5V}I`WGI&&S>Phm;1csy!REhz!=dS0RS%l3sBhWS%JcNwSn@qzAt(Ftyx3SjURD;G z@qQ8U&n(A3!k6&gSH)-0dSS`K)5yan z+{Mf>&*JzNM3TajuJ`?$>u5Vix9@o#hUbC_){cRp5w2Zf zb=2`yK3+=vh2&uK-?)}8{*hAX`~#)X(k_#x*b!1;w()Of%zb9oSI1w7dXpQAkxd$t zu+Gu^i}{Yn2Zw&l^J4q+TIS!~{nwz?z0|)S`;*%9HR{P9AzmJtXDl9M><)i~1|{!A z_jT^QiUG3qd8a>K;C_vbHUBKkw#FJe&(6GO2Zr)o8n32~>08>o0)GYSH?P>`d9UiI z^9{?~XVgMBCqC{N0GnTLA)br)s>HPkKSdqciR5>)3^`i2K|Zt0@40ax9wkrxn?Ne8 z7|b`S{5U)qi^XyeW!0^F@;aDSX70XAah+*u(k%mD_LL+!OjLy8d#5HJy-q4zHxx#k5Pz4_JF*$p=d`B)SvEDvib4 zGqgyXUT)eOJcAAfa;|p`AiYU=Q!K1#k3Rk7Yr+1vSh3)FbZ6#u^j+~-;KjeF6(^W0 zxDbxiq=VVuV-3DU(&7HIpYJvNpJO-k&7$EX^FNNA=!l!wmejEu(0^{8M|s9>kZCu# zd8hR}Fs2Fo*d@$BW>qOg&Xl-V;wMcMX3)O%(C}d9lSFHKc!u!7C;gBz#XdJWCwTIA zQhidE8ioPw{X9hs(Y5U*|p6U65~IpXm7O7bhnJMpm_GiD(?PM?l*eW{+ITv#F-Dw+E09b zrzy*_kH(kL#^UoSrYzIQht~UCJ>?e4;`w|qe7?%?xf6Fd5gZU*$j@K#M`|k{9d^`= zhkk?|6n@Wl{Qlp97Njd^JybDm(aP_SUlCG%H*nIiktf^N&_>Ez&N%4^ZJ`(>(l0KE z;SUcn^Mv1nuVXYMT{3pA@p^9(Wly_!vJVFpg-$ zc%m)iYhrP3_k>!ucYH5~3EFQ>_EY)#E4Eq_`U7(iwvjIZ>8yqNzOz#)!GXfue_PCtf#+rV& zrz$Jp2}W`ggEPBg%TX+gTiI?)c8DEq6VOEx26*8`eBDyIk|)}UFDnL%27yr>kWYJ% zL%TTE^$#{bLHaAk57&{tI2>J+RQJ`O(DMw#2Hr^gXtm$Q( z8+&HnU$YV`9=#D+fA?1Ah4mda_&B-HnqI=SQ|BG({KV8*fsW$0DxS(rd~4}&AMvEr zeh>d;Sqtl*R?-$KOH6!Y>2Obq!Kob&?=)qK<51?F-_E)J;dhsnVI%#5j&R_L&^%Y) zDf`5+N7k_aX_!6>(AV^bd`BB|c~^)3Ql7`2^3t!MqtKaI*FalWLt|G#Ya^k#V#caP zj8#W4R^9Pk#S7f=aF=B*=x+0{p_FmHobx=+{SPNCo6Pw(d@bsyA~R+LBcJ=a_Br+4 z?#ndZ>(SAgPTU9L#HEHCw|5SInw%W|>}9L@@O~?@*Vi$;e_=}a-;B#Xxs$n)YO8q! z->n#&tY&WhxQ(@#nXl&k3&)2K48JLS@I~<}F>XIA4Uh6=H~O%V=?CwC@b+LzPp9}$ajFfr+53q zN7p8X>nCc?ri1ZkIqvHS-gUOWPCM2zUJ(xd)5X;>y7asA9Qti+8Gk)k_=WdvEi`KrgBL9OhQZ!K%1#f37JWs9 z^~dp}gjWL(_2||s^I8*Q2OqCUkG}Y5LTjcxfc+YtGZ@!AqIm&+cILcmQM2~V6<(Wq z_!rOP9^K3Q;dtSMMQZALikic*`%;VGLF=&OM9W;+9;_i??;`(gV_Ymh;F&1hnz3<8ACc0)kb2JvQ(KMFMvg;mXA6KwgWBPwr2b(|XF*^LU zHhjxeD>8-gWiJ0Ke~JQd**x!|y9;-aU(j3f80vV&BK%xVWaRoX69f1e7U1n-Qa%rD5YS7XtZKF2Cj=k~l=fwHK4{jCxQ45c;2h980*W}kqPgIDeLo4+2+wH zCx%QWo|hLtH$VOoJv?cR#bEVH%8$}-dcSn8_I(+n=y~q7`ZJn^_X)#JmXX8YJr$G2>_C$rHTy7gqu1my>tFZQ%w*x*YFUG>nA6Ae7?60Ylg?L(Jv zUC;B{VK0mN5<~5{z7QSCZggi`Frd=+o<*Gk`qiKJQI_WH>-YLY4NX}6JPVfgLC+}h z`1`<1k>hQx&~9&fCo*64M{8lkMw0xhnICzb< zY4A0)0mlk&@9x&kxt*-d13LMwOS(%|6<=+QKf}GbS6hP&U3?e$ExqZ?h~^#TjHAo5 z!q~K$qZ>StBQJX*^|V1l|Iy+4wb`tx8Xs<4lM`mH3?I_AA@z^?s*$~9G+(1N$;KW9 zyL z@=gE57d3{vvYR!OTH!p=_2XBDE~5=&^sd;e&{m{uj5XNc#TdpOO^!`Y|LXT7_o*3e z4XLIbn)uJy_*!iu>^a#dM#CC2q}ipHwk(Oq6T`<{yEuIOcBhc~qm_?mcMf68-Z^?e zCqo~Gr{935d*Um9hXr3RTv`r4mwOj3J+KTP0sh~EPkzjQ{-ePTf8WCY2$yb9>hkv( z{Ug}Y0v5G^HR9{L-mpRq@cd+WUf&RqzV(F1`G)7ZhDJc6YO8JTbBlR~##KX=g2npx z!JG@X#AgNffmh%|bULy;*&ph8*cq>>j{HHVbaSy6EcYY#jeV%h8=@Z$<(=?`gyBb^ zQ}mKZrm#kMgnaTKk7E5}>Lz$)*I)0yy-Vn?$i-Nh$=gHQGJjzHYcTtv@(%_hf0XR} zS6*`H9o@q_%==gPLVwz8`~eU7f104V>_dSo6SVH((CB$>Ld32+R6d}M!P~pR(O7?( zdIzabUITJ!GJWBvFQSzRVfkAWaG(F;w$AfDz`xLKXT|!6;jDZyOuYqdLmkL>?lC@z zhXRM&w*EtF-@M*7G@16$bu^ziZ^+l^WabH&w|)Y@*j(oQj{XPP$=sgW{|&HMc8w4C zKlPh@EZ^pQAoCl&*h@DNTdn5!q;Ey5@i$yEE?mEcbS*gWqS*)N#$#YY6ij$~PGWZb ztDZT3zjQptMP0#bmc4;9 ze7ETCKVVLbPHJgSb56fk#Lm%^R_a{$chvpfQt+TT)(6pY+Anpv!5-~#LoBeR(S=sC zvAN;Xq?j?$<<#fq6>qOZQZJ}qTsZ*>+-A>V14&?_^Tj`jLv%Tt}N{< zCJZ)`lU z-A10iBTBz7p$+PgDJrkzTy~2Ix+i-C{K-AY#gXJmU$09)n>UR->FY;H|HknehhkN! zO`lVna(@&5ta(()RIg~e^N8lXw2z*v_w?m{l}CB5WQM7ebheJ(+mkvUkVZJ3<#>=o zdj1RO8;%_W-EFAmV$$t&>PnY*KECW}mF3u7RK_^z7x86Jsw~q^R2k!fCGlm&DoeG! zl&SB^n1_t-yARyHOQ-LCN;~r3?{A~jUOJt#u>YA!*yXh zE40O%*|>r}T=IF#py(xk%PRT2-H?-DOH}mxaYne1dz=4kHSbW2hc?7dEUvh+c5=m) z&D*il=C@^SN*mK}!fW-vbj%O1kBO&CprfCkhL@4qh0sWQXe8PfjK#-KsoxVUboK5- z*3Iogd&s*2x~auRUkLUTPGXHJe5r9wDi|)Gymf6In_DeBY;(>k&F<*xXX^7uhU@ox z!?o3(@Uh)SF0#K(yO7Zde8Txu`%d=q`xkgVrO3gSwVyfI@>}xF*pTo44d>X956g!` z_UKXgcHxgWUG47oSol#b&#GbEaJ*enZ`4=bTP@}Pte)+qo*;JG|!EV3#5R=wLJm73Y)UJfP zzU-SJ=^i6od+jB(K}@dQh2N{I``+N{zLA1gTLyjo|Kd3*U-6viX8i8ro}+ikCMLOG zfS%O{J@B_qoYJWvpcVXs3%ma(Z6c<_e z`&Lg!CvE25q_}(T;+`b;o@Wvq-L08#m90qo?!|Psg#|I)&GGr(7r>WvyFXM{?PuM) zFgDZvDg8YKI-FQw4RZc7F(lso?9YyE)yPElYbm%m7}hxCF7QEl1$(^B(tCUS)5|z^ zM5Kggh}Kg0eqHZCwhZSx{d{LFGGz~K>Isy0mw#kk8|HBNA1{ZGLq0hP_}s7o(qipz zTSr~}zighfsmuJABmX~je z$&o$GH@ULsx5{JuqB=2o6H%U#H_p72aDTP(jJ$F3rje(yim<;Qse}E_db#tv2{x`YrF_nS+8S!Ppe#$Dz`;hp7;&JhJJa1dreOh_5rQPKb z_`IjV{1k)vk0VQZ8q9aDtGGT4UyDu^Q&*Uz_*Uo82Q<|&LAEJtdJ{BwA$l)+u4T|j zo+Djn#ot3khyBzfqCXLW$7u6%EIR=)S5E&5PT{k9qG zZk6Zs+epgtRYu?ZI=+m)nMPTT$~LerH@<9%Ggp#I*{<_+tzF3Lf#hwZAFCb>o^bYT zbmVg$eFJ769)V54=ym9PWy}K#Z)M~1=e>Q_mW`cVx?uDzt6Ak*BN@HM8?p{RIV`%I z{8!Q0_Iu2Ayo~<7+L^YLOgVtv#^~M1=ogXKTfzV9iL)e3`Vtwv(|Rq@$mnrucVzTn zWb>xqCQ3#dIgkwAN=Cn#;PN8hO`O@qR=kWR_PCMJyWFwsIWjugo>Pk~MlJ`Td87Xy zbornBR{dY+&_*)juRX|N=7V<@?Q}5gG4R4*FY;Tu;7MpIgH(D?-9jfG)pzKd^)4R_ zC*3Nav)ZqP{FZ#6hJR>T!AJ}juSA~}t{8m#JNwORJt*>Two=h2?XPFA=9|>EtCT{I zE0lsizfcO@J*%`j*t}F}X|Va{N}JQPkU8R&t(w!o7Uke|MySAJH3!-dKLk5;dr#uKlSf#GtMOU8JAtvyyvssK zV8{Vtiyg#{-s!T68&|ZMyKYs-;5sW|_`P`?MI2K(Ru?!9@|t6<_xa@$vnVmo*Q|L7g9UrWM5@qPkBs{*^J5;? z&$4eB`@cL!z0oWChsF#DhQ}5b1;%9tBiCP56fV7?I6OKp*gV%`HCA{=oLu24nz_|e zJhRp_a;D|I>SV6>>XTEv*PN{Ijyf46o}9)?7VWS7SB{K(qek!5wbB*I%s2g@dnxvZ zMKd1hAG*EB6Y5IaqU%R;&%39^*N%9HY0R&rGtZL2d`l+J!(Vn5F}=38WsglKPM6j` z$nV+Ed%)z92Rs`SjOB{%_Y_i#JU)8hx?sFXeae&;>zz9 z)}upePF{AW#Fdv8nr8&FLjTY+uuYolzh;I0%JmfdX;*lxMb_57p-QfIf73I)jJ`MR zJ$590&bq9z3C$nS)+XvND;lxq`%Pi?_|^#=7j(En@U+sE5?SlE$RmOm-L z57E;nOn!|MJ;FAOfMk4zHr^Lt~2jFF#x@?>&*Tq z%sX#LX#NlUqxP&JPH`AMik~ak)(4y2tV#cew|9?^vby&FpP3E-+Vy?%e}*WS%-B5N z(>iJ#?~)Fpt>ZpPnIC3si%ZrucpjSHn%4n2??c1e+P1soU(CPwU(CP#U(63L`Y*LV z+S3V^X|IRZpuf`4lGt@V#6T0Oqs{_ ze1*Xo?&(ejJq<3@!i=GOUd&|Mfk zdZ3RQ8K&Ji-*t3EacmC6-%DpWi*=jl&$6GRy*VAwHK6yFO|{ykv%9e>Z{T~>0X+%n zzx}jF|CMsSe;NPCRuTK)*bHC)D00E6Pm!0kAM*FGMi$ejnb7)G*64Qa(OOg6zfInm z>;;Nxche{Fkq^1`ICW&3Iu=q#jPl3%vZFK6n@X?w4Y!VP7i(nhKd9qAw~k!5jv4Lh z=xgfucj{1|bI>0YbFYect2^8}c4k;3Te{M}8$2ic;E$E@loK)VY9%z~ZfK03^}2-h zS_lp)zue!D_VA;v6MSiF3x}_KKkeb`TP0rzXUTTRzc!mVz4^#}WbWS99sjRn*~1cT zU=AWV@OsEQBfyI8fH}+mx<@2dRT_8*|0=g_vN4|dUR-)l!+U72XYAmaTQjetd)rmy z6+QS9v|2DISTu0R9kb$7rjfpiIl5#S5B0I{CXe#o;+FL*%F=f!-{F&<|E;{^`+qB6 zw6np_9l(*HWr1-i)=wp`H8gqFARBv(pHB2Wy`rGd3MAJPi?Ziu4TTf!E$?fOqs;G6 zesVtLImdZ@)2A1+>_0OG=l??8%oBavvxxTt51=<+bUtOibm=p$hc@8Dx9=)MubGDJ z$c2Slxm(P#l|PWd`E%@*g3N0=e$+Q4I4)&UbOQL4dg~MW`C@n~%i>Ny*QQ|vZJQ44 zPD)9uyN|NJ8@eTiJY(Y`Pv_`r_*BI`vv{=q-7AX&Kk;M)uku)%me6PEY3o8AW1mTn z$2V?eZz=g9_fAHu@rN>3%2!u1WK0!)75bI?z?oHZq$kLz`tI<$Yb%FE@1Rft?b5#?_@`qVSLzoy1RIAHiP9-%W7tiRLNXQ{j{- z{OMGGAB8(cXDQrY;nb){;ar8&qN{;-|78lNM@tljPXu3W^mo9{4SMSLV<(3n_IZbx zabG;r9y0gl*hJ{^khv+bjH|I#C!GFrN&tPC6A!OOUe->bUBJDRX==kq_G4WHH zs|yRa8=fYae>w>dB7PJ;VweA?Rbz6n_mOPad`BjQn8T4*@xPhB^1=2_UCT}c9%~G1y{P(A`nZ`%-)ZRQRw6%5 z|Iy-#j29PINWUO?YQ~zyW-gxrHz=-d&b_S~gSt6GTA%*mEirV4b+te27_0qn^eE^e zYyZ1rESZmEomL3eO}{>e|F!zwbT+|PiQhB%zU1oF3c)!a?W_6T!#rJT+>-CO)&A|u->H4bh0%TWS(9h?ojjs$RGDP@S;0G1HoU=m zUMqGx;cIJ3iSHd$cop%l5rN%*Y;! zpJ~sliL$=Re)|PwB}_45tsZOJ=0eDF&BKhHJzKUz}Z{7Fo)RxZS76}kb#J%PiL$b zjw`Zvdb8`S6EjASR=b&lnN^RB9;vqZtg6-6SJ5VP9obRURpi!{T+dO?bZ^3*QFZEW zLf~r5<@~Stv_u05@AFz!tDp;4@W0wpO}T2DhqkFLAHW+g9MQk6ExnT3(j(CpaCLH9 zy1QlVr>yt+|Lza2u;H7(cOLlOcJIQ`9o#aCvx-N1+%gt3w&IDOPj&X^4(#3xT;2qH zmI0@wz-tNntY4Wm&bjSTZ*Z=)w9|D)<|#XjrXN5D;JHf~ztY^Iw)ppnXRVJU|MAjX zy}crv=U%~giHBEm;(v2RTm0{cZvqa#W@IqBGu6qzllWp_QEyN>ahc$t&u`=E-=eG2 zI5Yo#U~E5h+t5bdAs>PadIoD(I%+TSW_b6F>5MfGf)0@^R5VungXZOnm;E~X8RVdy z9=^8~JKK%l9bP)-6>!ax63!KXqf~zSw~Jd_pw(~TCXvp5xM*kx>q|~0w!D1ft-^9z_%4R>@{oZ)jYm;RVxuoekr^uTBMe^np zSo3q4=bWptPXn$M4_`Nx@!n+>K#P@kG<>b{7Guk%yw5xKa3;^~P977slET+JVe-SX zS*9G5r)N^0rokpYlK=C$rzRtdS?J|Fuw@tYUwxa=q5l(nMImEg>VU8K-w)dJ6+h88 z@;H+YUlHW{@9`DzO-0Ze&e!*|llcqr6UqEVI^Vp9Jl86iGcW^r(GqwC@evu9TY)rm zSiHkmmzIXUrl691p1Btmma-1?_S=s8G@imKP0am=BVNa5)`R`Fmpst*9hP0&|Ha$p z^j~w^#QrbbHk5JbgzVGMzf15{c}e}h|5N?{2H&FLiT>aFkNOWCF#X@&&$d#E>EC?= z^9rg4g6HO1DeCXNe9z(AGuP@~e8E!xx?2}6EG(mMrSzqQv#P&n>*EpnSU_LH&7LtH z+8N0YVW;i^4|u@`KJY>Z#)UiV7GkSjBRlo$jGek@Ok;lO7~byt$Q?AQxZ z(2pW#JTlo6I?tPSvDrNWA5Q5TIGvguIC`vi;P{qafn(#jmv}2WP-x-Dyzg?#H!z8( z>WIy}^S07iUs~|>AAGVr0uLk|`XO{Ehm&}rPrx@(Q${cB|G6`E>$Jd$*Y2W@JE-f< zz^QkNW3ky4f%Cl6c4Tr%s0Dl^TT19mMQy5M3-4q9au)gb@3~`ZHSP)A1Gq2aM&mL_ zQ~tP>R<)Y@U@N`-jD35#-x_%ZHtd~oA0hX92e%%FfMVml$p5C`#^P4`va7<-6jKIk zzLv1&%AfaUn+8?1$#;e9Q?iCS3d2uT6Y0WturezwTpSVsJ)fft}FbzWTm%A5bIb z#E|XPC(HJ7DSsfkPPO^-()SP2S)VNI*ssHm{T5*FW?=9pV6hCCEQK~{jlSu?@8g*b z$?$|PC#eipR`2P3Mqf99=cC`k!-F*l=Ws!dXdkpBI8M8lLWuIezFqiQ<0(_p^ z;+%PCB5prUy4hx6_aM5Nvq~rK$=|qn#oNEfeZatpbT^F)tOzIs3(LklCrci*U2vnp$YyKDi$}^qI8#hM2q2 zt4IUh+Rlrgdao~Q(i6&q79ORqwd~8F`xno%cs_fc#jm?b-m#1wR=MYy>))p&OY9r`xf?fBadp5Cb2**+Y%1ftpUd2_{x?n19nu#b-^=>n zRL(v$Qo|eMUu|p%f5KgiDb}XwZ1|L|%HLO{|SS@d3nl$=y(oI_fUQj=d zx!+!J&}Zhc{^H$;>;B~62JKHjlW*Bjd%J4`@*_8&`qM=F-{)I9z-X(XaUiR7W zz2Sl2l@xvxc0P(JbBQjJnU)B zIAc$nI2)q>OK)a>EOFi(nLC$`3E- zRLD5QbPu0s)X6pe!j_lr)Vok2(WmfGU!Y|J_gm%P6^IV?23poB&J$>1F0@9lFD(rF zIQt6kCfZZ?TgT2vV;+yE2PVT~Xv}3hcW`#PeR|5R0nseS<|!-K+)y0*cGSV;zxkuH z_di&c&OX%5^C0I`78S=Hb@S{XPt$aZvn)w@pvle86~`91d45J7o&D0>w@$NSf4rQz zo(|r`Ux;fOk!C-C$fydI6inE zSur-q*deESL*WeX&jlO07X?1+xZYw6clCxMnWPEEvhaZ+t>2@JBloZv8XxAIQ6+6m z!xniW{={BSXCLMK(NV_xC})w5GQKMfyg!}2lJ!@FU9y=I&MJwAKo=9!x|}6^MSByr z$+~A1eOu`6ubVsl%{*uSCbCO@+iJ5* z9wobE`X!vRe_ClsIND1+7tZm{QeWWm3-4yjW+L6S0iViPiv}NHEq)bRcVJDbkvAml zy2^UQM1!7xT#O=W_ChPHli5IyOPr zc`bG9eg-NHn14Ar4f^&Z|7Dr~`UZcLoCe){k+g0~dk>qCT zU$!z9*-sU>minNZ>~Zq>zYp@C($tIJJ^B6v@+!%+4zOmQU`;3Tb|&orYdAU1Pu%GR z0|EvPPa2!vdSlZ&&v~~k+^oA7QnBfkt#;4g*TI1&8GqTcYQ62lE=S=XyLPk3-%H4y zIkQGs<8@#*ybJwv=B9U>&E0F>l=*yGR9H1R|mj( z&X_rATSybl@efDl!#N~}PoiDmHXk@@3h9QwV{J$B2L!Zljqw?w7Bl3ZZhvWk@-;&R~JRqO-aKC)k(0|GYFMdNl zYvexp(~4vF;-8)}A#j$lue*RQ6dbc(YmzZa>~U^=%i*&PpTKz_Lu)w?)I#1FE+1g{ z1I`82V)s$&b?#RDBetvJ2kur~@B_66ux*0=i&p4A_Y6tB&i5F)?Vj(E4GsIdP&e6R z)$nH9JU?=dH0+JM+}B@PxSG54Uf^!vHQdo39L(MQoH2i~=9k_XG`XoaS-1~(dU4}gz?-=!_pRe~T4)rIzp(d+!LsoHU18*2D=N^eia9_ra#s#xD z8wRfuSYAA4Z6>(T)32%w99@X5cw-K91fAhd2xqGO$Iis)oeob60_Z#PMo_#j7O7)3f`UUaf z5%9*uJL|Gfa4T`Tr+(0<_Aj~ch*oY!n?8FHZPIwQPA(6~UT>pmH}?Pdt7zYPeba}Z z7(X+(3}5Rnd9P#2?38Zai5)~)_mW3tYTPa?oWh(}F!x^q25tuyCIct8A@jI(E-+$X zrI~zwa7bWj@tCdg(SeU2giq&8*B_8GtA7FT)r&hkyCp|)_#Jd!5ogWYY8QEO(OGDp zGjwqQha%9dUof=4$q-r&%+<_gWR2!^6<&LrR|^ z8A-2n_F`i`9tEE}!ZYUNNq=F1?ws4M{WIXFjlW7-!epYo`jljGsd>5J@2ds)5G zRpw~=fIhKDSMm#|$~Ghc4+%V*_*OY8H~CxT3O{~9^FTW;!`Iy+|5)ytZse{<7_U}$ z{hwt0f6V&VdmXx848js*oyT6wr2QG#SyA>$v!*hGoe8(Hjs%S5O~*S}J!QYX4E=H+jq`>0CXM+D z{5WO3i{D@IpTsv=`0f6G-rCp)cOKaOpaGmVopV3=*lI5<8Pf+iSKch*!&BIwtpN9} zur_gCVdP%woUp_yI?tG#VSE}NEEyA?2D~l6A7zEM^qW}Nh)i4OaKNL;$$+~ID^^5# z`>YOzCmz>VdscKj-Np^H3UhhGAa`KnneSBWT>st=znUr`}tc-IB##Z?F|x z#2xVGpcyB-qK^Ro^kE-=s)IFB>&ZVKTUf$P4ZZ+->w2qvq&T33>>#gA> zG4{Ia^JWgSdkuDI;d1h5t(~JEtFYHG`~~|Ztr?xC5bPP;04{B2&vTHz9%?8H45lC3 z;omjaXMhFG)k4ZT2>lz;$r|Znd=m9~{z9DQEU8@8sj?5Wm79g^y(fFge#QWiRr_OH`l%L40a}g`zLxbCK5L!J+F97v>$sPPe!`Fk;{%@AJfDC zd+ST%*e_XWH(hsX@XBn}#rfWlzrY&xE!w>8x{{dcUNzX-cn$nmPPKI+Z-6yb@mCod z(&5CK{?^n}Zdpq{HdB3>9eugp4t>f(+#*YgGj7%ueuWKB* z=d6V@{{7tXU;2Pm)I5xP=N^50?|IHe{s6g;;CN*pYkupRvcTde(ffHi%rC)TKz)_B zun(_62ABh!PKHK+7XtZL6`mbc99mN~tWft(d#Fmc()CQLkdv>BW%9=wV8m%xLnuadb}J&|{T zG4QK+*29E_gAU@iLQ{pK_7hg$7TjcQRJ#jrLI%Nl3w!-R{|I28xvz!ZpW^K9DbDSt z@up1prR+dtA2L+NsAUK5h;Sx%31vjy>mNA1Wgpd?UgHPy_t2$(RTMv(;c;agAcpov4i#qw!QrK8~W3BHYme6m-<>_Jm^z1 zxL4!xf*Ut&jI~jBXf=0r#^ZV7-(t?y*8|`rXjiC!dyd15S>?a;4#eZ$hE>wZCY>!% z-Ozk^EpISwTy@0;(ieD#+x}Y%8)kzSW=|b$Tl=x4^>nmlOM9I22i9z>XeTn>j~SyV zV-jVYkOOvQJ}#JX$?u0PVgG<^wm#m=ndN;rpY^{_m-$*F`2$JI34ELn|K&sGkm^O& z;<>F|o5wjqnJ&-?E9*b;AbsNmLu2Avm-``9A_ zvl`Q8VCEn&d5HcV21bSpCw$e|Q|{utzzK2OX)n9bb2n8aU=#usK2bnHK}I&;s#RUnhP9@hU?w z8%EAx(%rdzlJudZ%P!^x-r!4=r}&Y?o4lHDVEaYla@*qi1e?b?`}TfroY!*REffDL z9Btk$qs@`M&<5_D&0YNL_3(4Y&D&)oxL38x8OLVQRL>IDsr+7y*#p=f7@CncLO2ln z-?XUocK-c0ao3qYI+L*SzU=;!!aw@<;ZFe`8KXU-)jCUefO((B89>3^1k!|yuvMv; z3hi|8EO-q&%1`69J1J{him^$nx9%Hdzq%J*0z8$Ap2CbJwCq$;8RZ>5TgJ_laUFG4 z!&9g(tE42h3O-D7+{AY^e8V2b%-~;i8SD$!SC^K=eoNnS&ihU*f&S{-(8b2#_T`}sy>ym?1?nHl`*lNDrrs7)MEGFH~81UC40{$gRmb#7NmO* zBGBH#{lm1kTt9Kv)%FD9jO+@zmBl=;pIz@i&3h@tU+I>$&TYrfILE2Bz~5A;4bqF4 zK2C7ko;$+Mb^m#Rf5bB>?^)o!1$-zRiG9(DH1_)|eb$L`{Bi8lW=%o14;&Rv!H#VT zav}BxpU(p@*ONcX18n;m?+jA@Omx^Y&^gFXD+-=MHWpet_^QIjnyk(DA~!1c6l1pL z*kGwGjXSb7hr!p$bVYa#8RQA!xkx6mMAnh{*4pB`#hV^%xfR=*TiIvgK87a}p2&9P zs$X}<{Q&)$3(j_E5$^`LwCFD4w0HUe@|r|nE~71_ZoHvU;G3I>yMZ{hL$IrV4$wBy zsjriEEomd@i~8|x{KS7MYnc0AIr`S*|5RoHX=B{~Za~jr__Tu~?K68sU#t^44>{s+ zyvXnmHR!K42N^J-IyfF;4r0 z4^`n?hVB(N$z=BI=o;M}|DU*YyEIalZE&Y1s}uG-TMNf&Q2Bh8xM zj6KLsc%j0%H^r8~%O`Luw)pLF>gF44k8lM%yn(C6(e^Crnc3OeWWL!Cd{u4fWNm8n zWxcL-^Tig2hx;$$rMFhu-pfm4|HIfhYp0iilMdh=(GRC@x`zs0j&*VweO*Aj(eb-u zuQ~ZfA`Td69^uT#95>F}v)a7BIRN<4J)N4PFf^uQ~XzJ`a5O=X?j3|C`%Ci??u9Mi;l=cM-4s zi|#qrUP*J4=$ra{J#kK-74N{7`h0C7uiHQMc{p*2_E-C8Kl0hn_9HWB@VV(nW-y<2 zsLm^C2l&0`D(OT&hnI8txb+D>t_&T{?3rV1%j2~@dE(=G1s~`9@-}$Gvme|XII`fT z!1*ex&>1h@iej$Bdmo%Vo_$gu_DQ@if($j8Z}OFhx6%0WJ|OAp{~>7kVd(sKgNzM2 zJ&mvD5Nk7;-{L=bwvfNuOBEJ-R{!yJ723OA%b2J<$*>Q?i*k?ko-Yu0gz`qr0xo$I zBRUv9_OZwJM&Ucpeh$x>j$KX5(Eb7OoM+$jgqn_77xFlZtUb-*%<3CXPs<4)#0#DM z6wlcL&)MSgoVpL-iMo_mo~Y~aO5={+uScu;6zc68y@?yn?-I!GU`=j*D)p89<9W+1 zpLzyU&vK<_lHM~A21j_8ba=(PB;}R(+vt|yGbdnIn78^O-jJ^OJa2+0pmr z3X$qPM;Oy5=<_y16Z!_f;mXK1?{@YIcTnC7|1&OzMmN?<=HvC4J zg2C?-@8D$Tjsd~oDix6I%Ym)>7lgjTy`H8lvPpbFN zu1(Mv80)jlnQ)o<)R%7um-P=0rOr>|wT{HK;WcQr#``kH%xcSV% z$))hx!)SlU9%l;f(3GZLA34ZkV(EZKG&^M1L z4n0Kp&Mwx*vAAqp4BEdD-ezMaXZs0n;LTUb9!C&<-A%U9-Q1xx)`{ZtU7gPD#F_ui{x&oCMSc6T z%>B0LUDhY;4K2x3S~Kifu8mnI+O6?FJ!O1AuvcRG#=coTuzG`h`Z88N_+yOmdj@mx zU7b$*vXgW=r{8hP*4ntRa4N7e1(>M-cD@7*-HuL&`{wd*u62wb{^)cbW&b^3 zA9rBEv%U)aNv{+6v(tw2^i})fZOpac>;Z6wpkeh_atCLbFH|kN5 zXajuYUg>|45kCzc`FnbcxvVki$?m5A(V?k<$UH0bZ|bwb2|a`V4m=6RJV=~lGutmX z(~Y~IxK+TIWK7fDxVef0o`hrWbmJ;j-V_JNe94WQtvGa9x(j;~a3)=FrF>wdKsaWK zUm8eaj6eLq=ce)Ck1_{;Y=sY$PFQ-e?}JC?wBZoxgvF2b5BjRBD}J%x zS;HA>gR2vkE?79_H2?U0$j_G-hw^!^T7iWImSP_munU0wvus7hrdD>kbw00{{K7HQ z!SM+ka~EOv&hd&C@ZSVaTDoM~E#R5c;F$>e)mJZ{YDp&gmYrxCf&Pe^ise`LXTYE>fjsQWty04+4?5J zJ(9w*GZkN=vQ9x8g^RudZ{^^kKEajr^_$=#;gRPF3qP%pPkleYp9fwUm#h0k(J@kv z&MRR{w`msgn8Y`-%XatY4B}23?eqMr^RWXJZuY~gcz}hp;Fl>=?`9;ub;jB&v z>hgitoHMWDJyEb5ln6d(L~MC9Kn(ylcT8ZkKZ}tACQ@pnb6W*S)GJh&So{B`}G;vUhMK zGU_ehz;nVO;J%6?|1O{yN|zd^xsT+{3d2+vr!pw~dTHcg31FZ5oFWUWpC_ zIdphSiLpa@Ug51Jv2yvZmBjq=H#ldD8wFEqvAaY@9~eKOu)v!(Bxitie>M7E_>%S3 zw$64G8`dV$!#{ei!khti;|~#^OMETy*tOReURfN=^-v!0$vK|&c^$gg{||Z07(RCI z7`+{iK5Kpn@bTY&VLj33#+o~16W{JEPDw6z3zyvhUp;|2EN3pqGpDyOw>QI|a4%+l z8FOCB+?UKfHul%%%uQsThchCjW7gL&J~QY)XSQ~suQqij&bc7(Puw|0JAS$i{D@A? z*8aM1|E2Gnn%DvP`Tr01P0`llx^Jp!wkLMt9#8E2B2Vo2G*9eUl_$2AaWuT~+LGAW zW8P4%&StSbMAP2T{4D280CT(zJQQ@}UswE6;+gZA#J}jqzpD7>hzAD1X&e8?jbE?$ zI^q|h*8mQ+rd57W@lSbTr>2v(t|V4Wn(d~oQQAV%R+hw00iVKwPrGT)EA1suY?*xG zdqSI2zA&kHBj8J{lqdf|D8awIY z4H$E!4^{dkz(xbcr6Er{7KVImgUn|KI>KuO+d5@)wrG z`pRdFvgFs5#CpkR3^V03rrqT;uU+LYD2bh>+_T3@LsRypUYmOf9AGuYr+$lk>Weum z0v{BnF8KA01_tR%E9EIYGKKzf7D+sh7kq1QIJn3ME;2Z5S66#G_eQ@BeS#nPUSxJj zEIsY56*GAAU=3pvY2bgp^#;P{{(rQa?cvP)7XFuwx@3K)MxbNf;El!C;nsSMu6b3> zm{(TSjDDqQOU(^Uui-Y}*5TITUc#-xt;99pmgAPguc&BImU z?!nE*&A?5^O~XyWO~y^ajmMSY#^J`|M&U-_hSl5xaw2 zW#YQxGH|IlA1?mH4Nd28XK*KR$8b^H5nMBFAMOL(d$_&0cW`gx-o)*|ZN+WDy@uO> zTZdbVdkMD&w-VQYTaH_bdk$BJdkR;BTZCJPdl0t(HxE~Zy9YNLHv=~vHw`xhHyJkx zHy&4p8;2W<8-*Kz8-^Q-8;r}x<>30^vT&KWuDA?bD$a+CFQ)&vGq{tuW4I{p2(B5o z5BCA?J=|X0JGi%TZ{l{~w&J$nUc+s`t;4Owy@XqXTZwDHEypdzJ%_8qJ%y{mEy69t zJ&0R?n}@5y-GiHro3R+ZyR$xaxa$Mh-;}&2yx@UV(AQ)+KWo6)FJQ1xKhcA&MqyzC z`;7Io*LSzASw3X^srGkxUjh7F)X00m;qlN`Y->3~9n1Oei|>nn{obCm zZrlFG8LXBj)>C|vH&o8L4o~y)9xwhp=oR=};kJLOt1AkvRrl+kZM41iQ~AS`p>rDv z9^}_=ySxT%7QT}Hy7qsct4QWG){rNGuNaS>aq3lD;I7HG{3_S1vti}n-Q)VY*#S#TA4 zaot5n`8qd(O?SZI!+O|Xp`OH8KBzjupUS&)L;EqTAT2S5-(UGb=@4qE^2W~1^Tt}h zxh)IG7v;Y*#ADAhWV^5RY4vB3H+H@T_mnqwG&3a>%}NQij3EDW{5P#M)RLbPijPVO zjoY5ux{`h)&z~xq>Ze_e={r{xstnrHGPFY|G7LwXBK^FvD0!k|`G!73#?huS+BBZ} z2YX|uhkD_y8KYt3n@xM}>EPHPjq+Zor99?77N!m1ao*V3GH)zWN1o^ScQSF4yfMR9 za=z&-e13RvO6cs+lu+x4l+dXL@~p%S1KvE?7N841yN2(VsC!+9Q0rR4%nkgzdE3c^ zfy-jex8h5j`!W387FUFip#NcRr7uxfvK@anCoEZ7oHzs5iL#h?!MG_);pDPDNGgkP zTUmrnS@?-|mDB$vxS^bXY@wVPlyBa*Sl2gTV93g}5C0k%%J;+`UF+Zp6JMQaHxX~v zTR!unvz?5C_&I~8kO$2IW)o}CqpCOb^6B`ZI zBxlsaGnj+!)+m#QyQpf>6{=3j)8pielTZEAS--RFC&R>riIZF`oS&hzJ&h;wp;I~V72vHf<5$a_bBS>%px-9% zqkZhB$SXZTIHx4mA+7kVc~3|GXwErT($eP#nrNe+JO|*5Oj%=`G5G=cPWQEz z6`zS0y^h>YY1v;bjy+2~;KV%}6Zx^7YwG8TJwyHo`Df~EZ$@dTmiyE7PtRG!vH#*9 zy+dZkILa8SFD_gs_SpNKd0TJo9&gHfi1HHoDQCTGm3O)@@@`@!$)`7$QwcMFIm}-PVfJ2|`e5Hn*}ME-0^cTJRPcE;zR{F_ zQ!je}<#Ptv{Ac-#&(``^RfyNj<*r|UE&5Y$`B_uO9lRZxm`BQBuRYb2admII6J?mW z9xK}5*dBzDL74x(?Aj}VV?=AsI>tBu_RO*`pmWb;G)pUdVWu zGqLQA71xt8WiymmJJ*n3YsS>a915qMr#@3&gEzF*+x^7q-!VRnf#1_Vz2J9FIez9Y zf&VxU>Wne@uXo24D`iic9bWqpi;_K=^@ zCsQ{tsQcdY^?xQX!&)-!XDrRyO_aHvGIz8uv&fX$Gx!l@>5l41qDy;%2LyaudFH_vFnpsOu={5&9+h z)X~gmvOffOZ)Ls}KHBUFRe6fXEFx_`_q~|5@4k1;QQ&(e^&AD4)?u^0^@;Qo(agH+ zqrmyf#S=|9$7jtqakPCUZ8u@o@k-k6g0D-|k0kgm!Q@SXO>l3i{I2ZX<cm1tW~3xo0H9 z6BB+i_l%?>FA<(x-`|ez9#T9h36**>%}%OgCs-v95qkJD}My94YkN$q9-9x^<7oy+&|ZTfZJ zK>MfaXAQJ%9cS9llo))Q=$GI{eN!Jzn7$PNBm2A^op1UO@TR2;rbZZe$~1iZtOPH! z+A_A!5!&U4cW6ws@AyD_DfL>3@A@43x#afJ7K7LEk%yUfMb?#s`gziZRQjy8asNl{ z1ZmwOG+#cFs+JhfYh2AY^J#c44c)I6b8!yI50vJ6s6rqr<2nh+i<1ph|O8p=o)Y#JO^=d2UyArfl?{=r$xM=!Ha zy?JphbALjJI~$9PKD(+Uc7XDmY3t$4_;O^kp-;!JQ2atq$j2BXM-3rwt+8fp^>bj^;bC|A^Ex|7&kSpAAgmoARd?*gqgWev&zXe~*|v+~GEjyD2>WA?2*+1Uwk}#N5wt z@l6YJulY>in-<0y{wRGysx`{*>6eb2+VDUA#XdVNtvI#>`E*)$?#7l6Z;>XS_4F39 zdw*xv)k)@Q9QYFXwrSfR3+(x{EjMW_<|g{0vHP5{_;=Ei$6^-WGwYGh>TQPyy4zpn zo4bolU(H^Nes5zhn1Dm@ms2<6UXiGaF-g=_>9MZW|60Rg+8ZWp+IqCmzJ>hMvq`i^ z>)phErO+-;Do6RvTBhE_f1^pC`H9P;|K||9C@FvZK~Lx->Y3o{G9(hW7~hNYVR$Cm z_91h!l{wM3*5e##k>y5m-BZKO6`Qp->#i(=hjD0F-G`;{ zuNlF__wy%8V>&0HwcN?jH|7VPg>^hc_&y4rrHmoM5AwCXBF7x~^m}M8YfEuQyJ~H9 zBHgiTQCkH|`@G)y`S3!b$-)gg*=wC+t&5+5?JP< zTe@-4PSJmrY54!uH`;~3ZsawKJrum0`0m%R;iYW`hDg_XZ^LIV4z;%>x0SYVSC+HC zXa5`S0H4DAvHyA`tai9#A^s%16#7p6;ziV!G#qt+TYPpu+8{nqICLy^x2tpN7519s zI;k(tKIdoL*%^kf>i<};1|x5{oci;rQ+yIUa2vf&gsE4w(DZx%jrM5jRG$}9=R=A9 z60fz}*56K7+D|6eO}(|gzIE6LV9%Ca7hYo-8JGW7+R$LdD%GFU6GP#1mYMT8d>5V= zYLDDLxwW8kai}k{;_E!v9r#8D3)B9zVw@*C`T{uiGyLq!SK8ZCsw=*T9b6|JY3qv&$3@9vcZRxDnWl zP_OO-E4k4#l|HZc?;k4q5PItdPf;5EoZ(BW`q~Yisf1%KTbK{-+ueUX-{8xZ>CX2o zeHRuBH!Hnr|5(1S)O)GDs}?KVm%K0Oj;3nzx6hDp6*O97Qhl}k$uAw;Y`!}Tx6ksOE1rtCe8Q~zc+?ZyU)I$wkGe$P zbPmYGFRBFt^RjpUQd5&C{ej?4k7bYGW_t_L+>SeE6sW ze7%dcpE$!;!6W6tKS{r?zH^2(b`5JL$e~|9AEU>;CW;!H)7Mdit{BueivIC{P;`0P}9;b_WgSt zyQ^BYSpEo}ZM4GjUz0a{#C_KPt} z=-SE;K38!y{gVwv!L;lK#xp7zn3Y!Yh-kZ&cKaie&01*_AJN!}zG=MjdU&Sx>ERK* zuJ?a9KGxXXv#}DolQt4QtGlO2;q>9$qXfR*+08Sx3b?<|b4%>LZk~+_&rP`{*4YhL zrQH%++tsu2He|!UTb*Yg12?^M>#(|GCCHLIU8CE&c%~}ucb&V|y|cI!*;UtQE^+T{ z7vDvCXTqMFP2QP=e^YV`c{)Wc!o1T`|IF$E_J{oc%)@hA7wL_zPM)c1*J7{7yyep0 z@8Vv@=*NuFs!{F7sGzLg=p*-o!({ukh&);cD|-MJ>wwX&7h!bP?JMk8n0281+~GS_ z^h53Nda~*cLPsQfIyD{qhKx2w9kPR-@}9F+HCEjF8Vvicv8_2HT19IjJBHZ>giSg0 zA-Notq5sR=|No2swbsLB!!Q2tOZuOFdp*6{+F|~uKO@}#zr+9GnyWAVTXvPH`|khX zU1pqx)3}`R`s*|foA4Lt9Q7OAq46>AS2Az4_ku?>AK2#hjV;<+Et~yIoSozSj$NB{ z2f`hN4cO)!pS^9QZLLPHJ}Z^8bA5yBI6vI9fIE#6=h_na`dn*U+sU`s$@e4jt-7Cj zHTTr(rQXl{Q^`O3`Db;i{X_ES&Ieci6YD)0AFIrx^lb^{5Ak%Wi}+GwEvc!oNOQ+n z%aM+;DD}@VX&H5q9mKuaF=p;RpxtSK+ql1i`#S~?5423@zZt36sp7_^#^UEX#*W52 z#^N8~@9RjOYXU8^$#W0sWw`OF#El3nf$waoB7R|SHHMB3z3a4&7iAl#Lk$b>{7(VC^Bka+{iBDdKK5u85eE^)d@G!ER zt)38e<;LDY`Y`EeLaoS%*?X2}0*5E9|7{F5{Y^MacaL)}!q`qJTub;fxC@w-Jh_lJ zw)`JHYVcTr{QnCaR$IY1x;QLfVHbz({1Rc-u5j3FgC=@lUu_xX;C5Ufwtbhh3Hg=V#vd?D{B(wqGB+$n$ADazC&kxQ}+bxMmKLZ|2bU zYpwjI`ik4hD>;|sH)eeyv-uNwiyw$@T6&Imuw33kHhL!giXvP8NN!s>)WLZdY2%}N zD{uH5+UUZR!H@qUT%F5s=%VOZcp-2BOf3{$R2Z0AM0|n5E==usgfK9r{FMp=QwsYP z2Bwt1@nILH`a$P8E8x&amo9B__8-mEBfjm(B8Ly$%a}NDb-A5AXH@I2_%`&htUvQR zDv3URemTE)%h}FC;g++G|AuKFVraWl zXN-66{YvTpgRd%_lSAF?@TiuuF17$fhzE5zXTZYX)$?bo#t&B^w zKT!tlALEu$%|Ff>%Q0PEFTjs`eJ5rei%96y;BscVJ(O~qmRdTucHF-@!@2)+LWbP}Pd5X6-9r4t57TYwts`ejkP#psB(CtQ zo$M%nRV>3UC#?n8R{lB2G>|h~{MXajcJFIBUiee~ImZ3t1h&>THWns9v>C`9?FpQQ|7S8AIH? z%@EVx54$?|kk099%GG@_rk>OAvfL@7?+>9fEyGGe6Fr^LPxyzJ{)M{PN2&9fuXf{Y zTWfwD@|zrwb$OpZ4AFC%@W8>TNm z>Ta9z+vIB?Zly0Yjr~OI8}Q@EZ@5Ft$Z@8^haP5q{10UUTf4-o zEThhu4zIF4vDa!WZ{s<3X&*6{Ji@zmT^;uyWgaL8UmGL^kS_|HeIqEFx}H5Nvep6Lx~Zv4Cnna*2QEia`7 z(pHypcRTmUpW|$AIb}5%n~91rx~EF*`?e#0>x?Xou_}Bl*x*k~t83cP3qIFlA7JhI zkju7iqs?2bP<$_rvwx~X@*Gnqa)j_wZ)hy-cI5eHE$;FUA8Uu_S+R8HI|J8Q^PcJ0 zg6wwNJh$-Pf$<&Wr){+lVmr2jIoZnhQuHYgntsD);?o|@WjgOgPkgF%6t>}2-lu>o z?q8e5eFXbSZ-P%cNglmhcvC9;5^*}86zc_DtKrUXH_rj`=)SIj`{T>2pP+2$Mg)3+ zOfs4|e8~#NI@b7#RfG(SyU4RR&oVWjH2-4K68k!Ebz)z)659Ql`95D{Y|@`}_U#6i zJj{FX@woWjHdv}W{#?aT=2`bbuxGjCeIMLWw7!pzJPOoiNNs)CZ7X}jgX{^D+xjqVtuDCO)=JI-s;&3C zZ7u)2w$_g_<9b!nxUQu=)!dDg=$CAd!)KqX_%Fs)a-LI$KK2ZroUU==?iOh0nGC@| zR`7d!9X#_V%Fww}Bky5`|GFjoP4f=yHX6iv>Rvkcn8Y5 z*v;&t53+ww9_Jj!8F}q|;aoqmFxlA@U?X$u`{3RFR?$b?JChFn&^s`!HO|>cW>$u&ES?nBzM3K_N8PrcFIPKu#dBmHM|$W zdP3I`FnhZ&w1RaS_E`p}52VcRzXRUyH^uQ^ z1vXaoX6*v61^g?V8NP>d_xgwAK9i}i!YnG;rNmhng4vB{qH9CzpKfo^?Rv0 zE~kz$Zr;n?ItI9P^!Yn==)a-ve_7-ckJ#1xmmN&U2fv-TsUP-Lyg7bysV~&J+=uPB zvBRr3`mPzAv3wJG){YX+DuO$Kcb&Zwzt=Rv6Eiw?e4TMLY4{H-E$-U*;OpF@NyEQi zX~*0&e4T|fY4}x2Yj)G{jUGnzfj7QRnqWrtNd^#JgN?4z@V`o$;w{A=VNa9ak#|l? zVlnpL((8nqfminN=xLE-D9k?oIl}DOu}2Raz_z~`oO_Tp7FkL>uw~vtSx&!sFWaoQ z8N6k}d^-2ea&I}d70y}q?BM3Dw41b7@q1nC5zPr9&zb~Zo3Y5-yK45)_|#z9MZV)( z|C!&WkIWZ$vzvMA!B`1C zCooe1PB!I|!XNQUX3lcxQX($J6XNrvfHNfVqXO|s==m|jQP zrC_>|w01DPMsN23(`$IQ2bey+hBteF=hiizHuwz#(`UfNI!kGAu;Cptg9o#K-ArtW zfafE?X&-1t3iFVFzcyZ>V2qsyuW(xuoqGHKf&K>C)2VV_eZ{N5jls(skb4=P6Pt(! zv8Q2wf)0NOx|<;yPoopUzU%?uSNVr9<`c2YC`TteA-Q~pl8*d3jcZ{+t35R*NFwOYa+UZK;jrYioF?NB%*DE~Ig!PuM z=xL58yQ+8~bO@f-_wXZ|7kPcrHTOGoDeW;I_5{eJ@mrWz>9kvzTQv7)Yv*Ho3yr#* zcT8J$6xlaZzVzZ__>SxtVUOWE0#Ac|MO}EuHTEgacBozbTH_-nkGm_yy4L6uwm7uy z0QZha7hBBUO*+q~7`sOLXYr5DLqEc|qfZ!R+OIO&|IfX?{C~Ck|AWfUSm=MPiF@1r zzxERUql=)gx$gh>kpJIUM|#Hy9;c{%T~1p?@8&IZXz_INtDS0df9kxgt-inCUX}50 zZW$B#M{QNRdr`&>ZDsuZHmi(V-7?1T&(*X$vcrXy9aiky@VKAX{uKTj=l(a8|B~B3 zu&wUD-+q;Gm0LzO|0K7+b6XjIzx^s>fLlfy{|HZofmt*6KTpiN1Gn5&vGERE-xk9I zuh-uC=j7S;l))SIiM@5fkot zt)h4DwniQ&tbK7VvJcAL<;MnOC;c>G^r^^M%o#KGv}WII-u8h9=gs+^ZF^nqeXpY* z@G#1+dtb{%cRW|yX~S<0MHX1Go{WX~hX7+H`|wko8D2wJ`TC%P)qNu`C=DC()-~SP z^KM!;I$6=!2Bl$Ves-ld_I)?4UuUKD3ocXI;F8#LZhZglititMR`L1ho!t1oy%gU! zSgZK1C9xX()6f}nho9ou^E{59*~uCue#~M_wH7~3CwYhWl63MUX_umtM@hRBojgR^ zrRd~eNo%8%*cgOod&U?Z@c%=4`2@V)m{h~R8QAWq@%Yd7u-(|dzT!j%^PXB73IkUo zn7aqQ;J{(Q9S$6BaQpFN-~)Qqc4o9?y4gRi&s%oAz58{_W-Vw99VNVKu0yNJ`_)(c ziSotco*T^l6r5p*bwzf@oes+eAX~-VjSDY;m&tZ$t&!s`1Bkup`DlFb)F#YQggVFv`fw518(0nSD)tJ=aSYohvMJY@cv(ocpZ2i!|TBF z7+wdS$M84~&m%tP6#S0i-I13ZLSAwhS;P@!5xUDFnFlI`=h@kjcRZMvLhj=p!kQZb z&sUfN{pGG*6VDnDPKv@~__2$S{al^+V#btlD&kzk@rm27whI}@cw%Swr}_G zw)=0ONB2?Yt*eW%ap)W^;k?RylzpqQkLVoTjxApmw!stkqu+pLN5KOvlsm{X@hoyK zyM?ooH5cMj>pEE1PNAL!-mC3zikD&CtpN^V_@}`O-|Xnb{SR@ctFZS3AAB?2iTk1A zkjb3}56tXB9B{IXxS2f&124A1y$J(1&k`;Ir!9NoM&8T~~;Ysi|vRZgHWD$-%(Dp0gTkaw6 zUf)05yC?Q7(&>c1$hf~*9INFGfRm3KS;a2vzD#>C^$1VTL{{+hkB}`iTSc1=J2DRO z?&clIrR-tZ$8x{SmvSy38yUPZ8-3Be{+BtA76AT{#Y2O3>5SWG&ZC_L=Z3+p=I$Ek zMR=LBUf+c7X^;CNZ*PRbq2b2a?L(~F_fr@LWVMnb@BC$aWP|p?8(5Q^EB$jfcoFvT ze#<|^!1I9{?Ipmo;Vra|u^n3i{OddHV?V*(L3@+NuAD1memJjd_A$m@PIMf6sX6Q9 z%CjqxPY(dTI*pz)kh6x8&GR1aw7^gPf;ej_;pTHkp;kDnQOvwtn^srY*r)1LK_t;%}JiFpQfyr*{pDf1iIQ`5?r0?+MnM`_h;M9%{`4i z;V^U91Pln@AR97pQOh34d^6^hKbkSJYSh z%Gv)LJml{G*IUorU~kv{A4^5uiBUO-Hvfyb`_zWg*vWO^Qt+tkAjN-w6Mo6yC2*E- zWdfJ=Vul}G(g9#^FazoU;D#awKAFYf#+`3?Mj`Fn9^E?oJ} z&Oao5cPE|iPtX5=zA0Jzk>K8>CdW=@AbfsA`>S(F3+0(L|1ES z8oZ+J;!eQ8UF6U0*Pd_ncd4%^0~QQVw)LV6*Uy}_qZjb}wd{Sw`+G=hhxZu+M}~Si zGSvRIc(&Gr+?0LIA$Tz(M}c1(13u54V--E*^2U|OQUoL7bwz`yqo>xH)=H)9ICP#t zhHOa>nhYMP}f9ICKeEe$MYJJ0x?Z)qRBe)d*Lx`7=l?l%{^vB?p)!-GisS=!W zn)bBPrx@RQQ`UdIwb3)ac(b>AalrTeTLK;79rb3C^q^CbXI7&ZVNY^mHShJl&6pWE z+1$szr?-MyC~M+k%Z?)7DfWzerP%Yt%T;;gMVBf)uw+ISXIo6Z&S`Z~;;?IYyZqZ$ zt8}j0$roShHTqz)=1wyHoGbIs8f(|cCJp+qQ*@Ae_VX`qFIB8zEm>cR7irxWV3VXd zeULfTeYe=u8oPCkjsnIjj z$AzrhnV!~BGvPhNH-ID7x9P|kD|Ae$mGs3gLMq{A^ zBOgJ(o7Q^Zg{+>Mf60T!ac*u_9=85@j@+fMhxa#gxt}Eg8>5|n_wlb_V+a2x+LlAx ztW)jVR(@@LMLKQM8Wi5p8Z!6YGOv2CVt1T3lihgP4-E0dm&d?i&}}1U)c)Y?5f8E# zD(8bP4o-_5acsDT)3%tw_1VFp z3O76PCM;Z!oab`I?{nf!ShyY>s(!^jaN-pn2_GSP(4TMb(Bwp2*PuH%Px{sP8ow}fdr}&5j5p#VcO!E%x|6LJ>C88nL+K>7R%Mfs ztXH{*w07%zgZm%;Owum3XZbQ|?P%8;mv-S-ke0v;kz>#leVw)61TQ1J zA=)Q=5_aR?Cth>^8{!k=n9Vrm!dE8a;PB`+eQUk!yiUI6!1c}Kt^fEU{?6dLdiSU1 zU{rm@$HGG(K|BdCJp}wq-kE2W_UZ|Dpe-F?Uv~$EuoX#sWWmbOJYxx_Hv@Ur1kcsTb|aA z{yF&?csvPK9wohPUF9<$gK;mhu9D<=Cz0o!KG|ND7faG#8r{3*@+Ek=<@n;~<_TA4 zW4puKkPCF5zut|_3hIulca!k`YR0$t7wzY%u}6KyGt850)Dr7%2yYz79_E1XJ^l62 zm*z>xyLa%WQZY1-_1Rcz73Gr8oSlQNXNKKy-Q%OGuKxcs znLs84xh0c3CJ-tKXsP8A5tW2Q31YPZ7H@q(p^s6t7SVb^3JA8r*vfdLf-P8R%^aj+ z1!2UeqSA^*TM#d;`qV&BCj>9ZEyKn6y+8Y$GYlc1{e7Q5_G_K9_C9;>v-VnRuf6sS zJ#BaiKD!=WT?sFldQq;a7w}ce(Eq5;W^G9o;{nE2<~)U^)=>J?n8rP(4|$TZ)OXCW z!u+Shx5G95<6cvhM>!wUzkfn~4pW}RxUJQ-XG*bcWG^pw%lk^S80(KvpE3BlOuY`c z@@8Zhxm!Y7$#$|V)^cBi*26qT+%;aEljQFJ^2g=MN7unTnPS$#B(Hz@2z$n?i{XC~ zYhjL)N8KxyZ+|j@c_7X%P`j#c%jBQNaF)OL{`ml1i$0_vR>zexPK!S?Wb`7 z{!H|Vr%T$d)!q2JuLkGhA5%V(CG^LbuC&osXxZ5K)dC+nY1<^<5w&Nc{dn&}njFq} zZ_k$g3O{JFEj|9LuNJ6nUMIgU^;Z&0IXT@aqnXn8~uiUZIFHw#HNs3sa-sM>FK^F zh)d|c|35kj-M0f>>mA)^W1-_)2JblJ%tXt-0A| zJ?RzCq2uT?_Rv<-ex4n|_S&rRZ;-K^&gK2P+vYltcN5RYtS$XVO1O0n`V<*239utpvp(Vl`$FFR{g{taILA-myCo{^JPT3@>FW?yVGwAA`i=~dyX zY47#2y&0#6NLz7_u^&u5&6+%H?Yo)#@zT_hrg-+TX&Q3pkC{dq>A$nf@yffA@|Mgvw!GRb?0@nzHb*;r{A;6InQJ)t+}C_>OdiFy zj>@KxeXYKpe#r13FbzJFjV)gD><1I~B6^=V$*kf$JHX@J7`E6kyh~qd;(%Wgr}oiz z;(4!Ms}tYTckLuT!P{;6F3)ysSAUjtVgC1I`ZM4Dcpx?fb46pbXlwX}vDy3}KIT^W zMyMM+YV3~J-1_V6Bk7A0pnL~vv#9)ETdN|k(QaiwJSSAdyo+f=^lrd6Z{cdzt{8cp z>5e&ptCRnUe!Tg8bO-(G_NCoe=g)YXdNU6Z)?Eit&vszG*WBM(ukj@JS_K`tO%uN} z|I|34^2luFor;j1iOfMUA9Lj8o`1Z!spr!dFYCGD;_={DIb)s#KkJ*uXMHzCpB#^^ zLV40{lxOtmvaHabR#8TK^52*?Nx0Ue9v>>S?&NUN6(2jhwF@_Bpuhln)zAmODe|a+>7%s0`yZSSTjH0TIT4u zV&pTSY0vP*?58t)9^eib+Gvm&PeGFfE*)Gtk4c9%(zj|;`MY)BLzFYV z+V;QLQs(^n8_?(8`SsRK==7z&*qziRew{DYlR9Xx&diR`7@p?rH_nWow{R-)!Wo`H zE&`G_;HUHzjeMuOKlMH5H5h&}bs*nTja$Jrx#@(LJz~;IW=InqTNNqgj+94TsDqmT5I}|U2}(dbc#P>`g83Sc#->rv{!&N+(o|3DOqPe#6E$X ze%{%f&G5MP3q{X!_FtFG$XXQNZ-fu}41iYu;;RUU$xN-o$?;$;C4M{TP{nZboK+zacJJr)sU4 zks08l#3l48dWUv$^bU54886|lIacrFFXyN|P%0yk;iMn$re|$pmhK*lCeiHs#2vF%B8g_B zh&z^MmZ6y$-+6ilKZMSU*#V6{(mG-IP=1Neq19sVpIF1_(IM*9v6oZFs9VQxyLG&} zojUeCNga!m>uA=!Swo|yeZP5K?`Ir(t7wTn_v)FTWtmgYQr5|f_Ni_?FK(xvN6>>O zqUC<#j-e%Ea?|g*dznPjqvLbt3ZbX&to{Ugp4QWP_#(=CtOR~*v#zLDvE#cA^dfE~ zGRb_>w$rR}rcZAkX?*mC7dSgf^?!?cd;-7y9=*_(7v4BY{r`HR`g0z@j;c~4Lno^L z)6~DP9eO=;lJeMC);_(eiYPDpq~%?Csw1b1+t}~Sqn5w+I`%tr1~+i`*NOTzV|Tug zZtVB#k#qHJTd{Md{W(2hzi0Uz`~BbV#21)82-?5TbKAac-%0KFEPKJ9tcSZ`voGO% zDcSGuzv%RBpMm%Kw(UHtZ`e&KY`h_@?Z)h|2K;U2OyP(gXt#iBhrpcwjUd}Y#_GDG$W!itl z;cT<#@PhT&N6Zy*H%f%QXV4&MvYz+HsmCngS)UaJU!#4;ocl-r8sR+D2y@Ea)6=bp z#=k>o2l#BW#$1D_ANH2~@+-Z$(S}X)S)Y0|O@Go@_z;uR)NP#49p=o>F2`=?+(@%d z02%lEhX0~(Ro_xcn)c?ZTi<5g5A~FOsAJw)`r&oHQF}m#MEyFeevS>Ds9!Dhlg<2+ z);%$A@X<5QJ}r%1hLAo64K+VDkmttRD`IB~POpd!5KOC#ogqlSq5o>kWN(XL4*Un) z7G-QfT}s_P@=5nUU%V%@qD1h zE{9G=-iE38V$d=5T*&JHqm{s8`4^#sm<&$Qp91=`By5pFW|kjumW3)&*(a^;OFH6Lm3bDxp=K#xuYR zDEHX@%jsX6SSzxFwIb&KXlJM2Vs3)`Hx*eA|CYSeM%nBW(0sAsHMd>QBu}-G_Isv@ z-fhQ(^1)umzi3Pdoy2dZ9L6;}&{bxhv7R|!^!;4QF>OX&Yut9uBY(BAhi`E0>`d^P zUOToe6ML}zaVTvj`-bvl-;C{HJ$xNy8eSleon5i-X@}e6S8wI>7wUc)`FOhEOz2;+|H!d&{+;<_)PMdDXGq9zsCJOwV;20ucV?~zSsApN zc+bC5IrPR;`;|^SbtCfW`Bh3~?=yGv-seZl3sTb%(YC;uZ1VOr}g1~yntn&!x63!tEBb-ebN;r!k+g3Vq zFkuj3AVFtSo=MOiBApNOZNeFZ(+T|v{Rm}*QbJ#X#@yc`^dZQ1(VNhVP)v~CJe8m^ znrJKgTX$HA7bWAG^O3#@5)gR&9#&QlLU%$pf_x&nt5##R&V&p?CxUc`cuwaNcO+;H zmBylyS+$$)%~Zc6nb26Y`E%n#FzcXRqVKtbHdT7)G-y9`H+>lMCL?&>d&!@}N2dG2 zN2;t4ewa6l&f<;?)-M$mbFYr>m?*WT(npLd`VQv=bJmy2$UMS*G<^RS-wS&8*sfa% z4bE}KxJ4D**F^fF$LaqV^OkDA>|pF0(*L#1cPseL(iskK4G+zRpHx@gzp&O04OQ~~ z+Xo$;_2;Czdu9HXZ%v(EbL#YTo4gtxE&FxHtFC{2q*jr&gp_pKX;@+%*we zBfI8);3Lyt3?K1ZA@d)_XNSH|x$wvCM(nC1(_d=PJ-9ALYME`CrH1hOK8{WDd8h?tdQE=)WsSWI)l2% z{tR$t?nZ2jcp84H5{o^HmaTJhUnRad4L>L6mkjr1wa}lBE5oMglxl6sux9m`;LB(! z9$Il%z~U@S#_<;8_#@LtaHnl$m~unF!~nfZBzAL2YSWb3tqt z&$aM`@0t;@L8`B1_3-{pX#$)nR|`KEyYx1=+Z?>;h{hBT_b&2vFcG{g={b1WSY1%HDxX76MCYZ zvQAjW>AgdbkiNnzW0w^Qwe!7K*HXz|&#G8acWd`OuKajyW6HSn3?rAF$~c23#2MGV zz?9Kvitntc)$CcW?S5Z=C;ah&8NRU@vu2H%aHMEz;LQ4b&hDF%ayV|}@tFMJo8#zv zW3N!HO9w-j2486MN$TwVdktqU#a~I;a`@&B+~M4jyPVUw(>Z_~q$78okh>J*F3sM@ zUEF(F3)jm1-?H}(b3gYX_WvH_zHUQf+PB6n-@Mbl)x6JwhN9Dv=@*jcDDtkMjFFUe z0ry3pkF2E_S@UT5FV6p8OIUoKZ)z=TR~tHZytK&I@k~F@dyd3cJ(p1tdk(nEAM98A zv)32JIg`W3zNn4v^Aj$;I{CTlm**mXK}(5&`7KR{;^U_Ltn(u6ud3rY-CwY2Cce5- zo?inW@aJ#Rd5H}NN&sV3HqjJZ#`$WRO*?v&br@r z8@v-`e?#b+v#M#NsC66!MS+Zzh zOUc58>C;PF`w=qkC~aj)HBn*!@!{ZQ$gr=t9h@8-6BZ4{{*;t>xl5$_bauxVx>omOlUC? zTKt%J`uf;JXfcK7_3*|-Xi>}a5}p?jB$FFIh!3hIeccD_VIXadAnD}CmtFB8?Kj=+YUmJJO1yaH0ib-g?WT*ShN_8Ex~ECEo;wbVua9??1AmQK zqI|F|+Wo#XE%Y&aDcX&z?HKxyxJU3OMfdprRD0PYUpD+~aFR}H~sDOMd1$C z>Ky2P;CH=4QFuyx|Cq`{=SWZWqI@%VNPTVtUt|SaeC*>@eXrm>xm?q~URKRI5li>{ ze$nglPN8c_(_VWo{USbY6nt66`iElZrZJ}Z=A4YsWqe~{uYO${8hVd+%laZMv@D4Z z30{9a5UO;~A-s}&A5ThiOnd!0JycG-{;LF-2~8$mSsfe3^UNz#V;}HcyvmQwkVZcq zfZx*LyH4<52C~o@S@4GrU^BF08yv<);1I!t{U909+4_w@(NOX>&A^~N6?m0=5v+-u zi0sJ6a|<}}9Q53*`@W=~4`f=geCjklJ|cEI?~V8iL|ffE@q>=m>Q;D2&j*3>ap-vu z&ncV>>F51U^ppH66Oqjk*yke}2hnHc*bkwb1;>z=be8&2%d%EjmT!gdM!%8AGsNlr ze9~1=hw<^0*l?bOt8k%crE=e4j`S_c_IM-F7QgNs^0~5h;3?Zb!-Gb<23o)_ffaAp_Tf%?eYiF5BA~xL(`84muair zs=;Zj;-t$mxXIw&PT+zDHy7OI1TM$mmVw&@uA*z;;>Mg==N=pgUAh8Gtk`yRWfNmQ z1nOLPtN8YugnJD}2AW@DoTwpGW5V|PyES>OD~{pQc69rNrL-1dCl zz@WX>K+gX%Fw zz0|;L`w5^&@89w)o|Av*riBjee+N$U$`*Rh=%ju0haIeED!4;e{fE&<+c<+a&nN$? zV;8|Q(~$8L^wx;1&d{aM-iGaP@$=N}NPLiZ==tj^!>Zpe$Y1?IfOU7x*m~p<38Yp; zYW%L;U{_!&JlY93(=0eVgGrHAfk%vZ^NIMEW{ zN_|nK&)7_c*3h*d_3uPl^`p(uwG;UlN`KDrZ8>#c z&_eH}=mn$8FQ_uM>KfW@BF{4~@W;B6hQ4p}fxG&J_TOb~*?O96&`i5C`1L%0c$zQP zk>`yUpC1eIuWw)6A@=fx=bJPd3*1j$#->iOhUz=%k&O$_kG=N%1;(bi9$68cOJn|6 z9_0-6_X`bQ)FDng{rcfjRaoK>0Vl^3;E}Ess26CXdM_p-H4)lET_l z`nxJ*yn%Zi%>VpoWK8;lc?I-eroDo6^2yG4%#q(e@xI>y1qku$O^jEQsGr z{;f80E&I}w>z9!0o{n4(BL8Tc{yw?B0qB#kJD9gBrwpe~S;)}&@MkrAS_Qw3K!&g{ z=Q#FdavnbN&_8aZ9E~?T-7m}<(v3rFnHLVim#besJtRJEfWF4& zoqUFo$1iN!?{6Jrnve8sQGd#kF^`aMh0fzySrl4B8rd@%1CZAjORv=(Mw!eR&KmPF z;Rx}Z$GThof+fVg&3lFJ_IAf4;vw;jvHjL|56S=T#U<>9v$}>}a_xp@zQ;Di_OIgK zN62K#f-^#0h8yac;N4|t{@%U=`_%~O>hZ>*! zm^4@Md^Vxb%BerhIHlAbLkGThR((z-cM8oO5zFJArFVWdW2^ekg{B^P_9F)7+7B6+ zV~1QAfu{>PmS+EjByrPv#q=XZI1vo$d@)C+sNH}T&l9qV$?kwJU1`8LbG*1$}A zqJi1%>Z!vw|JBS>#OI_kUg=Cbo`MeRf~<5!X1bv-y3^;ThR}Vsc&m&$l(Du+WAU&* zzqROG-_!tS{>t|`s?fT0I5zGa&g`JCT-p5K1+g-0b&YA7GOh4O1^5KGZ}}|z!MRI& zg#wq9)(5%MN_9S)I$p;8s*QY;!Wc~U?0DLa{V}W5zcCJ(KVstGkj6Voek#W{Xc4pvfVtp(9h7FZ@=r7`&ZIfRh9g=FSed%mHTIvL*B;! zIwy5S;~YOBh2UGU7hm!Le@dto`z|(zu|~$Mb2VQoyS8xp;MNMpJQjWVmBTAycTz?G zJx~gMrr+8?z*itA9=6vSHv3mwgwx$PLRB6fLE1Q z#QvMS3aoEps-$^W)^QHp|GR`jX?~ zX5}RMr%KTXnQHJ~*_td@^08N*VvRd8{ZeG``^e%Y$mGSy=J&A8FT%D+HMaT2FK28m zLzc?$jdF%+{XlqpP$e=4Z?N z4c7WE0|84mId>~?CmKFGXn^dx{!(Q336+6<_hk7u=KZK_KW5&uop^Z8(bJjs>68-{vR3O5*g*+rUH6Vw=BW zg%6*PaHf4oGuUmP*GcmTHaKhMVz2Nlx<&DYx8?0^%=H|ZJ`y@z0KLwKZq@L16}Fi( z*L9rC+?^Y$bZxp@$rpZUH{XfObLA4Byxu`J?oxfDwFlgRejU%Vupe0TvAOQTDzk>Z zhCa=?Pczegkobw*-F**n;>ky$ne1sj|B5*AhPN)k%O04+K~6ZmF&03TrrzGTNoxX<+w%639W!WWM1ev_N$6a$0y^#*3! z`bV7|{V;vUZG;~X^d4nA6g`DIW=^SiFno%&y15f)NV6ucxeIFydpdphE1j)zqx{U# zFds7t*%>8>%%lqcTdu8Y;EIH!P9+54=7P%? zj{21lgj)tKC>-@HAqcktTsLrfcKp@3_6~3v2^=;Xa+Hc)))8D?+#d=Yso5$zX^o!o zRUpICVOHcNbl0qNoADiPpJcHgiN1?`?=bqKz)t1=r^9d9{VP2`+02xqezKVfKUsmj z$o*#TlTO{f1P-HK?mQVhm~DS*;+XR@Fw1T>Fw_3Tzyf%)1|Av-FI@mnoeytS(`O)C zRgP@MUy+@63ivL>cO_jn(qrRI@4;Mls_{#I{1Chd&o%YupUN%vbNKL6TlVwi6|u;G zirBio6|vRK6Rv`uYoYxnw+^oX_m8xoakl+eo)1*%8D9X;t;6-4Yp>^d?=U@Q+5e+w z^oVrH^GXBVM7uRSAHZ)Y-{C5rJ)i6|Jj=%2ho9>l?7~cf(#Oba5YGq6zm@Z_-X$y{ z9HPFpna=v$jP9J3l;b=v=iH?0{oIGmy4}${-;llBK zEq$%na_oVY5+HFcwEOb@Kz!`WOMv*3moxX(Li;a==UVV#X%2qMv{{iU+&jsch4uIw zwv3t)=I)?P3w3vbXsLfLch?-xHvcXe!T(}(sm2Wjw!gb0Bi@)`JLUT6ANxRqpCr=Z z=S7yP!Aa?}FbB|J;KQQ~V$~`v_bqdGG97 z6}C%;hBdCZ9@#4XA7rcVtaGDdt=R8g@oi~>hxN@9M$hHiXFxaIFP|)1FDA)WQ9`zI z?G5fXJ^4m?6aY1@5+6#w#8=rSF2~L?FyHQBV4mGg5V~{`gl3%up=T#SXdMuw?K&D5 zv{Mb#cxw!@bty9TePrztWbR^Q@q4W4aO^knu6S1byA3~$o;3&ebu!wtxI(#`(3VYqQ2i(axW~v|ru}c8_Ze9Y+VAt+YGgIn{wL2d zBddATE1&umFdtQj{Ab%+h?7kI!@zv|4Fe18O$Kt8I8b}W$786WGY!nKXBe3KwY<@Q{BmAr ztN~exqI(*OfUHSrK)yH&DHcW^Z@6Vvq&9n2B+A)+H}tN!ekXUau#P=;0KF}}Tl8ng zM|wSFiq`6bZ!vw0^b2!Fu6}XG0pKXp(=V5sbj)+P?Q%JJh`%H|rvFB6tD)0*CS8s_ z()<@Txtso?cG6EK{de8;2gv7a^9}hywXer$!n_k~;gCDgP@=w3P<_oaC1)AwxvlR5{3UKS$ z!|@#T;!dIPSw@%V)@P#2m$0Y9Dn2dr*6%e3kFBmg61kHOklies0=+io^53eQOk4bH z@W_MkW572rtqhmVC|z_%r++ToZ*-ySOM=9dU36*6Gu^Q`#Lelz8bo# z{!!|*z|o2GJ$qv(b(_Gt32$yL&Y6r8&D>m${WJHwclqut%FUH_K^+Wo?cr){wilV(6cETc;099SJ3`F&#gwsasEHg zF{9&-*K3Q2lU{qwz24>q20rgJ<(PLcVwrby1Bg6a9Tkp6!?T7HOblL-? zlbl(uP8;vy=YijQ0-aWuq|@q>bXr}KPOD4OX?00D?FLi7T)P(N>9m(!ohIMIwQl@G zpz5!FPIZ?)yUN5B*jE^sZ;v-H&%R6$c^oT9yIpGV+4dy{GR`+J#~y89raj8Q|95@X zfIj2g|JXcqTAg%SUn@Khou+%F_85JZX}7%K=(E9;DLQMNgLKikZr?8b=IOLj50|9V zN(?S&@8)~8R|&8^otAFW<=8#Uzi|etn?9$V^pm0M5tsL+(>j=M&}q=l(`oxmysPJ2 zo%V%?OVVkMO)vd=RHuEMNOx4HZA;*e>a-7BoTt-fJnrZ($%m)Y?f{2hY|z@Jl|!FJ zuQiucgaha``Hq;k*wWI!GJL~~ev3-eH!h6!XDn)Tn(q5^beitXEkQ3DTf*R2ivaFa zaL*=mn6nn)C(zKb=`KjvbSKkar<=YY$6mYaKhs|qyWc&@cggze5C364=eYSi!grqj zdXVP>*ih2#`{|FqD;@SVe)5lsI)*jhu>(7+X#i)TCym#ihvpITT1(%whJNfh@?Muw z5t~jqqUoK$Lq>NM*grKm>>>km>{|@XwSQt@p8aD3^X(gf8izm2dAc|(SKUsyoG_G- zPpEUpKlPlIyNL16%rt9if1c|Y|Hx)~3E%xZ#y=VhP2st&z}QApeLU~PcB}C*Z;KDV z0pCF_u%-iR81&A1&M0eeYSM`ZHn4`XR_~_}uXon|BG0R8x)Tpv zPrx7U>(&k0cy*Iq)2MP(FX7#K$+me=UlP8nqdTtWK1b<}o$&O99Xi!tg+4h4{h@!@j?8Qz{c6&icWf(-Eq8!hnZPk- zq+Y4mT+0)7!2yI)?dy znDmwpnrU5+^j0Us+u6tHt$l{iz@cNj@!1#Pu7w_lDRT}`KDTP<{+{sAV-7*M$>81= zj<%gc5NInWB@X}qAygGt=N;v8_$*Uu{CxxSale{{DlO2B#b)01M zCv^n(8{w$qB!bG`0q)nrQNKxE9nlf838M>e~BW%=-jblN&}+G@j%TR)(kW{ zW>rl&yn)WEf!?)((7RNyig{&O<`PE-){G?{T~{-nc)_b29auBL(Se?> zYM`#s40M+MA<5RdFXHH|x1OMl(QVRM|0`RR^vSl?MsOpbhvsz-6DMBk1ONOID&AK!CfxgCEy6c%>_4BxUt{}!Yu=Lv2f$T5ro?SZj^9Wfg=dF16;Lm6TlIK zOJ&}DIJl@Eo<&b4?7$pHcU}SRY@<8n|K~h8#)*BjK7?=dy;;YehOPj22Dnym4jvtv zfnEjQSNV5M)`8A^!N0&u2OiXUyaSj^XYY;Cfvi_of4sfKTd$sLKWgMX*X|9!YOdf! zIxu;zAmV=0Lp)Bsy8(~2>vHT26PInL8yK|H49v1S7|0mdK-q7j(Onm!!$zUYYS3vT znFF|hxo&5Db-YURaT(w{6MqWnx{$srx{kG1{?yPx=79Gzp8O~?HQa>%cYA;BQE+@> ztKhRW@Z@UvH&TG#Ka;T~wEJX<>dkrj!2Qriean9AjcvT|L4Qk!AH?rq?!=?d(%tUw z5!YI-w7K@XJRcsYXZiiJ?5(`(n=J<7|2HsbzhPjOy$Pr>b2f8s`ziN*?xxs_TpXgz zEd2I|*t7B;Awpn%z^c0mt<*cY$|`g z@!hl5#~G&=+6``f{$L>c{0z*ueswSi^i+q^Hn`aZ?+ z59w~)-@b6-%7~kzIO<*o&XoVTTQ^{Pbf333zPc!A-SWg|@o}H6NeLTU9T_q{24ut7 z>Ro|FPJd_mzck*_JFIz(1=RP^&&iLrX^qqO1>qC`r?$z0uz%l=ey*HP1A9*-$-F+FYT6FzOj38x#pYJ(w8;(N{1YvZc+R?DYX9{@>O0X)Zumh-{5Iu5BI8Y{YMxd zWHb89^FzvyFqZzKoc?32{EFmb=nFlr=LN^IG(nNy}Pp;Y=RzbOZg2Ny|RW1Rj2l@-M=(koG7ZUXC`!SF0YL&fG-3 z(1FL%P(A~LPom*ZMMJl3YWH*<5`Yid(*7oWCw$U(wMpM?+?_Kd0I#(DZi2oOKIuEr z-z%^5i=rU`ASkm`VK+&ANO6Az7sy_yYrI1^Y|>fYe4xe=mGlB zla3P(yM3qT$#>+a-}KYx9bG@Qma^9He+Am#jtu4*7_@T?%(jCD=Gd7A=GxtXtSuk2 zhjxF9u$u5F;da91gwmw`?2M%TtTd@VE2Tdxa{IH=V#me=h?! z4xIR4W56n}OAnOSbm~?f$UuK*6qi4b?yc)wTF%*X$IJ)G_wl@$*HK>irawAJKBZ~1 zB8RZO&nDl4^cOYS-yZkrnegZ?U{auG>Jf9qw=f(y0=M2#pKRK?48&|WhXh@B8U{anL zZd~2ozC&uH=aS-5-8i+y#ytaunE#=E$!QKj`}W!>e@}e%BI(1gS=;B0C3fHsZsKlO z?>vTvD(E&4ULFn{X~k9#bjB50N3@putfQ`4N3?}`KCN{SkN=(L{bsHz+up?ULHb^2 zTp|08b$EtPHMR)auMn^K^_L9Hw_h}{!2TZtbM18o7TOU5+5aOrl5ve7b&;*5H5yu@ zrFs6ntliiA#@oQXq}!Ete)xW#6+fK&a%Xj_I2h$T3+8rLeLz@CSVU;%KSv4jpXtCT z{dRL-c%!?clUo>X31_iuMNIu^%-(?_cMGefie`;W!eXAgKy+x3^{>0$f%Op6gB6g#B z|N8Z1Pw;;MY^JiiXKcNXd%$+0pGPBS%YhdnS2q9yy{wS-8~E6xD_St-8r10TthL=^ zzFN<{5{yHdpREpyE@up_DEoX}$Ffg{TM=VN)2A9cTKbT0rTfR)CqK`g9_-=ZN6()Aj)#LjvXiAJ3p^aO6K*oNY!3(j z$##~0?4H13Bd0m~v2y~)JUhI;fplpuuC*`YZs}6wc?Wo3A}#ap$cF6c!$!}^Z;viU zZ^@oM04_QJd4{H~?0c$X4pp|2#^cg4@)^`|SE*y~IJgvWx$eEB(Q@{T_pgWzda1Nt z>j_%<&Y9n09-i+HP>)ADT3c2@PtCt;O_%Hk#eG3sbIHgutucy{$Hq<@7Y@QMRlGOe z*vmR2_G8)ik~T`-&DcOg__pCHXANdTud@bo|JY;KU{=WXpj@4e^aAaueE-V&f>zqC zLb~W{{2CvzhEen6nwM&VCzAcQZRg0Jr#=2e-m=kF0uLG;ly9#vxNLj5ft&?kV6Oe7 zfqC}tf$C$n;^$mNSU|Xga0Q{xv(>mOdLg#j1m?x3@LcEFYCO;L{Jc5N+*7UQd}zZ` zU@d$T)!a4n;0?f9>R79H=7LN0F1vs`;WQ840IY>iYW2=Mc&Xmuk!Tk0jFlP$(R0ja zJMvTlgpOuTsR*b!C8d|Vk#F--UsyibeVn}|J+qH7bstxje#}1~a&3;=4Nq!~`_Vb@ zYr%QvA#8^)JbAhr{O|a;L-6Y)Zw`|0~>NaFxRGpOg4!;pT!nM>zg< z67>^q8Mt!c_}@wVyKo!8eOEaCc@qC4+zxPOfZIp@rhMq&l`pwEP3e=$mt6G{E~)%# zaD~Dpl|LC=j&Mok&jr^*IJbPQYc+KR-x++A^)J#pZj^&KA`Iil( zw@5D^oAG1vFFU{;GWNJ|=zs!ePJ4d>hyI`~``}C6oxq`!&}r4+WYbIMIr8m{{TEY) z{G6J`weEg-~5wrl*coxNW0D*7*;`CpuKf=KbL=+^q|XaOeP~}dZ!P_B82zC2 z&C(BA3%C!zr}~LC@W?uN_jzb1zvn92M1Ie(DVwuEfZjUiJ+$#sgD;DX7{hxuHdjr(?{88jq`xSyft!o~PzcZ5%-4q%&A({>f%<@( z4Iet=1h22X(!(X`hRZyh{P@(X8d@0r4{g=wG|?WD!431$RXKjDx!{I)IQnU?uRYVl zsZXZeHjwW01dcwLf7=1BZvsc3OdF>nW4#kNN5=Yq>j`cWbzU|KVxKCl8p$q$dob#x56tI60Uy07L zJe1@oc^=wDpriKsYu%ltW?+uJ(Lihg1M}<+2Iku@0}G+YL+nMnm2eqh z2qBMfkUeSik48qOu6y&4rKAePXV7IFOyzEboY%HdZo@_K2-6;M(x)0}}1P(c%+&=t|XVaU0fI3x! zyEBo_8JAB6XKZ>SFV2{JF1VW$=^Vf8GH^E}aL55P-vI7M;I^aZlKDH=<8S1mxg^c- zcQd+el(%n&Gn_Q0+m0`^sY+*47TQ}~9-Y8{y8OMey*?tDzxTV}Tw>a>(Ec9q9O@+g zR(pz{YfN08eSv|wcC~>y_6P&B?Fs{f_IUBm!=o26 z-!h7QGVUDN*ZBE##sr28eS^+hbtwkO56z1^~l4Gh|+8i-yrFvrd}FxSon7Vux0 z@Z?_hJibkMjjqW+t zzT26=Y0fp@%(=gvz{!77?cl7tB1P&febH+-t>9F7_4wmgbVcO()Zn0nX9)&h8VxY5XF6ne12o zgr{4a~7;0wp_R(S3smLBf9c?Og)?pU6V?Ik!kUte3+M5yyW5v&AFGuaD&~96bJ#84NpzYOiQaDgk8Gvy(H2EkeoNpx z3m0O?><-+!cwu_6&i+o_oqy&$b8b|s%bypMhoeJt?eXHB5=ZxFoRP|ye1zdK=~dnT z zkISB1>Ar`Jd>7jK$8W0p*0|qv;~VAC1$eCP%eB)@T#lV)V7A@CKzz3bX4yUiGwt}V zojgvY>zYeyZaS*(QaC^L__G8&e_cR(&dVI1sm6#pD{?3KWAm>J%y90e>2z7e(2Ok? zukGA{Gv|LF;BLi=*l=vkYMnEmS8>-|pw7&g)uZO1_5x{b*Cceh#@L~i4nMt5o;s_k z0)7&HEc7T!wRXRRUOPZp2VU+M+Wv2y;gn^60FUfv%<)dj>Ciii_GJ%s7pKiNCfL5c zI@WZsI;OQ`itht_TKJE_f92#+4bB#BC%6=y3zKEWo4*(CGjJUpoZ5Yfa9@J!NJP<9R!U7qA3+ai_pD+3i`6{i{W8s4p#iR@mhRUoZ}QY zQn_^>W9=nf#8>y66YX)nqvsfRZ=VBim58@E4}dn__irmATcQq{?ZK|tPd#LJ&*gnT z_KDIfZ58Fu7GI$!?;=j?rtSc$KUEtj&8@{Z|>M@ygz68#(@8*JYY9!D5VkW5@lyK7DLQ^a4+olfP1Y~AfdAM5d9 zJ>zYU5BuvGJ~Vq_7;luqd(sOx>6yMbpY=xb$!mhYzB_@-9e!|G2rqJ(MNhK7{>e__}WrUvr+5{FvLD(3?gV7TQlk1NE!J zDC?W{$#1&f^yeG#b1D2>%CpDQ-{M(g?h*sD?Op~3?IHuSZ2gyXYk`5{>672Pp!#K! z_M6tvcI)To8>dc%w#D3N{yL^1{sOq0(e`X-({}`BU?=+D8zXs;o9|KkXv+(}I zgxe$o@POn<^09;XUn3K@6LdaIaguy&67RcuTH|p~K72{?u|m4sm5+_&CHW{hiG1As zM@KLJm9(CG{KM5z$#de*8(h$?bL;s$>D!YJW78uaVdC49k6$P9J}Ms%C2&XO<5w=u zlaB$&#}f8}FI`$OR5H@sW{;$^AK<7=d`SAXbL3~XBL`*P-tBhe;8Xrpe0C@Q;_})2 zuN#+lbNTFgzVYPXzj^lLV3MIrj(v@RocU#7(Eg!;ocU!SXMP#@&E()x(tgwWMcsd@ z8&g0q0!PnR(I`>H9d9B-V_AU9DPd>VCN9+5)ZR(w6pKf5#?q^`OU20&CeHu{Z zPQyPtjv!yH)~6nzp9pxqCeD7zhtDw!{@9yow|*_qK>h0 z1F}-bn50$^9xN54PEl+g`~$HDL2QLuU;|LLNrUvQY#GuA=sR?Oxb)C-D;&A7RBzdd z$hH|%@SR!n=B_E+OMBl#*=BBpIQ0=93P(LA5oAxQUwBVA$}n>?;3k86TR7@x=4QaD zU)U@hbux1^;Ff{gBplC^2!{w7cW(gy7w}QaNS+&U)^^MHAX|w$k4d+Vbo$2l!l;A% zwa+DR$RTx*zxEk$+bKKQr!W~g4A6IIO}A$gC1mC}o2bApHvCm!KlMn$r_ipNPq z+;1M|8`X0m@NDXR)F#TYA2D(4`8P0VhYZZJA22Y}{-uH6z!v&H;PKnsvZooy`TquH z+cz7SW8VZ6j}PRmYc85wwU4tZ-yy6acs$;}$59^N#2H8)k6)u_m&bKBlE>puAP3^{ zTI57LUMfgi!n=mY8w6>)T0wZER1iBoDj8s`BH8#lo{v4{@O+j2!{vE1jv&8eo-c<_ z+VXsF;o9(gPvP3|e6Db9cs^6OHay=&ILZ~zONPYn>B8|(lL*3zw^N1V|0WSU*${to z;``UihIoFj%X^+|RD=6GfpcVIGPs@K+RDanDEmK=4M(T5{{P``E*p0ny;)$t%{P*b zt-!Xj@pluKZNF|{(Egi&S@uQ)x%1D!|6AFRKUZVZw)+2%ZrM*67_@Bzv+V{0bL>9= z;sqxV&;He~lg%EkY+fxoy0ZBr@hSh4 zESuj4cP)CNilDeY&{}rqlzHhTyUwv(Kb~C;g*4mBQw{QgChvH0o+01XtOGUa67>56OQ()A_$j? zzd?5VwN>Bf~##zp4LCHm>dpjJ26__131} z>Xv_tfkFEx24>qoHZaG&(ZF2$2B2(#AK-5rOvohc#cx}e#`?vc*3>%uYPa)#IpKQj zU-^PJ;m5rm{io+CJl}x*JDT;5Ggr)N~ePIvLludQ0#)tWksy9-Yz?;Ckn`qPEaw5G1-4#U&o z!>3rcFZ_+b#W~hgOL*jBm(R@~xVV7)W8e$;-o@{Ne-{^#e=GPRzUN)}bY_o~>Mj3T zirYq(f!X#~4~p-C_F)6F?1Kho+N}nDeZN`DxIs3!w@>DV8TO)`ulT|`uV=Tr7Hk`H zK2O+=ebqv~-dKCDkr&6`sq-}b?zrOL3B0r3ybpDMErD};dtlnyHX5^@O`7JC%5ZZ_YFNI|Qk}2X?t1b9ySM4z zHU2KJf6ITWj0XNw_j!G-%w=$wj61G$?&s8G%+F;wYmr;wuXhR05wr)Ay+ea`!e=$^+T%_5 z^lO67m~#rxqgZ<^-?;Wbj$+-hWK??~N3rg>3^^Q*9QJnQO}gwlw@yDYko$iO<_lZ;@^BZFT~8lyAF(JII*a@f#iG-3;*O@E-@!NwppilaF|~ z5I()q<>4rLrT~4UwffrI9zpiyzmXqHX}hbBL+4D-IDY?xO&NwJ9gf{Y0na=8*xK1c z@gt)@9G}xR=%cb)LX1bgflo2{3~A4nPCi5W1M-nS@jU~B_CF0|E&!OvzwBXr_ZDF_ z;ZZ`=J+CeUzsOOrMw+!5D;izvFL9(|2+}+^Xx0O^s z`AD2Irf(sgzVXI^sn~}%f!j{Gjz7e)hh*PGJIWt&)R&Gw#M$S5oDVV2{=mp{o?Q!l zbT8Y+6P)#tY!AiVZ!R}&o@f66IFx^pKSXV>`O8a9T+qJ6z%2WF24>o$4b1ym|M@q$ z2k~R(1(MfN98B`{dF#%;>z4l=1B3Rr4a~MrH!#QUXJD>fYG9sy8c^fEO#Bgh;kUO5 z&k+_7?jY2-KEM`y)-~8b;@5hfM_~h%VFQ`7`Sp$sWOOg~Q+Rn**vv;F_hvp4zA!os zK59m%jiP@ZH6xvMJW2DC{hjqYFEluMtt0jF)|hoQydb@Cv_C#*WL-FCErix6_u)Tw zB=G1q%C83ZVFH(f4XHJk&id?s8NB>(M|Im<;D-CH-P=sw{t%F8^j&u zKWB~4Eb3O{%~P^=#Q5X}y6b#cb0GcQT#_0#cLOq~qr8S0ek!nEo$vS^PQ-)BKDqyL zzgfdK%40R~Serc0UTNZ(^D{8Ve#*dX+cq$0HyD^@|G~gadzpc<(NBK2U2{oRMw$!$J})8ZW2tYdwxXzh9KQ?S_|FQ@L+p+;D8nYtWbK18*mfcah)5 zpO1{qLMI-~@I_j?`XVv-tu!sKKCpYn*66hEVfFb%Rvzne9y@Mb&O!9!UfMs`U7u6v z^51`xx6z;EU4WdcysNzO?i(4i(%Y2RGToGQ(SJ~u&n;`TR~Gq>6OZxl(tUo$fvMOO z!@-q9E8(=K(VTk;Uy0vKiF5R%DWx)`8oT6J9{t10UvwV;O6Nocc^l4Ikj1iGyQ1mqp7dhx=(;Ck1%#Q5L;~_i3Go z&j`ooIAu2a)41oO-RGe@m}{PN^!bC-NBa|JF|O5rc7l(zM#{&%F7H5xci!n2diNb`%ieEmO;V;k z%l!}8FB(Ve!*8+r&wtx5hSkrmj>0@Y-Y5IX{g&zWb#QCwyUA+2& ztA#%s`~^-qlfjJ#r!rHDoqJ(2ke`O6wN0~-p}i%tyYD&2*q?8+hnKr7womfzvB0P6 z%E2Yn`=7U*vX*`CL)AW_gYq6_{zrGB9NQMq-rP~t(tqtu%_TncEO*YI;EtU1v+fzw zjr&=;v46)(8U9W@tzt{upSC5yTFXx|eX)=IzSxc&AA7_syP;&n(CEOFh}p9mXFM_B zlCssqDq<@Jey?oZKh6(Fnnsl^8(k6k<11C+r(Ycretv9qnPXG>Vh!|r(F~vNd|BzO zpKCyW7=GX`(dHswEShSSAMj_`u{2+7Pl+#fsHtc8a0}}wx1Ji-p2dCree6SV)_ZR2 zOFaETtN#@GhA!x^uB<=qX73)rxA=6XUi8(19nfg^fXdhx=(o?&A)g@wpQ4vPqOaT5 zw-TSK_9g1P(Mr|?;}h8WU-0k`!>xPR_gYdJi=|bv&m%|Ymu@Mh48=G2FK=bN+RCyr zo#(-tL-v-g=WK&(Q>?7$j5ih`w=LJ8OE&Uc$M>c!&R|?e+F2U+S93oWYejpL*PYC* zpD53lMyapNvEdc^=`?h~@%ITGzIgJX&lZF~ z{lto?9k;(~#agyns>5b$Wnt)tl)WB1+Pqiv3|+yy=vTr0j8XX-Pl1;h_nGJJx|iPC z-ORdK$%fK3SEWQQ`FDIl1%58_SZTHP3)PTEC*mfaZLKyu$A33}Qo$Z~*Iqx<$!90| z?A+sva4u6n#`#kl*j5E8N@~3=iHqcIg6qn zqML#Ge&;&w%%q*kXVnCs)zYB+QmmF*psCx&l29-GFYRI8FYgoT$-CA9x8y9&&*Ps| zuF6QR*ERUoO!_ZQ4Rt5|EY5v0@AHd7r|>R1SY__N1{nvIy8oGXlwK7+D@*lm*-GE{ zA~a>6Y!ulu`HU+K9r`8t@{h%&`IPc?uKBjBu106E*LIz3-W>bWe|@#!Q{G?H96h`S zzYUrwn%~+z)WY}iu_=)X?zb~EpY0E&z*hk@>1H@}G;5*x zz7yptAMNv2IbS?|1^jbW_@hmX0bjflzN%y|tV_?kzZH5z^(HU#A5ZlSy~ew~wYaOx zAl}j!f7Z=6{=?cAiB~z8`Gz9Ve!(=%*xo#luq?K=@WKlRL1t}xhr(%zuztMEt;^{n!1zyU9(ayylap9 z@2|N3y|bt?Hk;>Z{FDB@7tgfy;C<+4-AC%xbAs9pn_cCZwyC-zoZL3r@3seht^8`N z^nS?hkZ2q~9z2Q;pl^bnnJGiW4~-pCH-M#>=S5V^=!EKo9e$E ze~#zupU@+$er+WGs4+kzb^CIWW5-Mcr?aygDeF@gcNIA8w`_bBxpZ+q0B6q3e+Aid zahHN??CQh5DYXCX;w}=7zP6G2{>#Ow+{P#6cPp^pa&aSsTLTW-{KLha3$A&M&PsTb!R_y?F`0P5AY4+as8)s zNC7%Yc{Xq5eC&EFR%8WRCS_XV2IEJU>;&}9FRWPhjNl^CLAtdI-za^NmEGcJErn{#5#rlax=plSc5`K=>6lttCM|q|%ItO2bXcXiBZbpVoKY>m+yf>&gwEq4?pR>|> zt2p!m?@CiapJQ}JPuhVp%yTbCHyqRF&_|fO?&uX-!M77V{ZrUG6y{wzydurb>swCW z)9^;z$&db8eVFNclKPk8H{z>r;5@2?{!falcs;&)5^<(XWO}|=-;d)?eP8m0I-y%n zBu9VaOcv=}PoC;{m;BUyl$Ov(oZBLO^hKZ-db?LecsO;`z6JevG5*iCebTH=@o|Hx zlffs|wehd<)n|d1zG zi-4{jHb2=8oAq3L+~#`rkGVGO-a}R_cDogeznc<^ZzcSLu!-<0;T6J*gmr{9DY3>^ zvO~UEv&X#7_wi>_Vh0acG3na|^k}JQdI0(B^&0zv9(*&s+xjojT|1GvaaRzhJs&z} z>6mo0xaZGWa*LPlF*lvgP|-dJ`5&W);K|=4mBD*^Wn4vCom-J8!%H{o*fJKm=~RaL zQZvS+-e*3o|QMd({`y0uE* zAy=EN>ip31+rL_%eK_WQM}Fu@-leO}dtE_jDeoE^=`8gF^m~jy&Gtf|pYD)39xLrhJD48B-9{Ht%68 zTsiHGP!wINb@SJVCi=J8)>OZqpA6)RbGR? zwbpOOHuKHc1{%wTlTQ}jO4tu~Qcu}V{PRi+U2_}nJnNmi(F;T7e;zLkZKq$p{g3el zODwD0#P#VJx`DJ4u{BipjVZxS=!QtPFTInAf4FC8Y%+a+a%$)j-bKTSw9{h#UH&=X zncLv!&!>E~!2Iixw5idDX5~Lh+sQtbUfGXr@lR-zp7Z0O*p=n1-x<`%|BrwVwC4I3 z)I;q!Q}AZ>gWUPSbAa@v*kIy2_A8jQcY}YAd?hnKJrW-rCw;tt@#)Q?A$#=vkLEWQ zR5HHOb6tM)UHFXsebLMX`2kcIFZ{tpY^;jA~~GrNc5%dJ>0&&cbW%z3^k zUiuK5-pKm98vmtEy%xHaf$zZoG=nRf5g5A@dwCT5c3QB1Ti#EN>>Nga|6yw6)_p(Q zx_4Si?6a2C$S(TiU7x2$KHZZVi62OfG|x1JoS!l1&*zZ zpj!rQ^Ish{W2L7#W91#@52<(Kx{Anx_u>m0A9D3fV{vE+Z7dr`x>VyVUA|~Kn6ZQUUyVI|__U*)twGUs=ND`uw1QnILLYWYR0E#V8t!O2PRuHYFSQQWx3~G?JvWu1qJq5*9 zW>b__s7$qjVvne8Me)|!9>YbMNxZ-bkPH{|ygzI22_vBP{C>}Q{@Aaz_gZVO>-WCC zYb|tyI_%j|+M)Ln-ZML$!0{UDDRgn#;P_p9gU2q8-&JJa$v67+MpHlair!p@-^vZN zRd_b9i;Fw!$tOJfwL%ZiR`D)8iy~VH&u;bdK1O^!c%}T0=pCGVi12J=3)6pS%@n>3 zJ>F^LnjL|Ejft1+G;-69z$ar|e1uL$q1nrj5l+Ij4Z$CIxb~wCX8+>{S4-$*Y#6B)l(|zS?ikI(n}b#9b`k`g)#B;M1JIjBJLAW8D?CE z;**m|qx=`RN5sU#3*~>=+L~LMRMrm3qtWTm=IPv9N&i}x)7~|V3A8;*T-3D*ZLpB3 z-&*YAi}HB*V)jx0jd(L=@M88sw6)dTv7Nq=Ju(3!kHUksj@w3WXp<>t=6KVNp!UW> zv$y7j<`w3aGZyZ-6v_D$c}9RYl4ly8VT~7gUOYPB%P*@0H)4$E2w+X)DOwgl)|o+@ z46e}zaBYKC-`l>Dx%Ft0nd?b>ThKAJacxS^H0Yan?S=sTEa~)vLBPxY1aFOjPr?gN z{@PGfQ*58b|E-t9SFe;DeiDCuknbe^+KSGa;IZ|WBmeWS;BGY{^g&n1f53C+fSPC?ietlx!B&6 zItaT!B+X|{l-w&hRPZG@9)WzSuwjV>eN0Q!+{oWbamnzaZv+1etl3S=>herG_Md70 z=WXy&>}oDtoMperH|u7RI)A#6y^4^7B|~C!oh*0~92K5;zre@)H8YVVjZEB*dTa6; z`b-H=Jw5`tlJzd|Jlc$dzyMB&r9z#bw3F2q8{0IrZSJvpUDmId9|mA_aL;X8ChNJd6fQ~ zXh*LenZdDTEyzixj~hztYfn?x-$_sWBYw)AyKcJ)+y32LYF~buf02(xXCnw|$LG}cozWww^VqV(iS}(Rvj?4~4(WOc7%uf&6HDP2^m*{<``oifR(Hmp zHo-THe6T%e4f<@%HQ@ur?ING{MHcUaOe*YLMIyYkwXiRSu2of#~|zW zL6!#gN)~iq+3>VKa^M+%L~?z@2flc054y@e_*tXRTMPHa8=H8VNrR@uj{4%C9rZ^( zZS_a?1LOPsl$nYgTWeh?`aIQV{cs$83Uq&9MH%6^_7AK`>;3AAdSJTaUENn+x}fXI zpDgIIa(@N&j-d?Te*Y8Hw~jiVqU`DZNb8OMNc0(O#n1ZUA8+x+KiNk766$=AcyKAY z%pdt|x!;u`&;w!}k(s-;i!`MpOK)MVBb)ZDB=0JJq-hO(d%_?2HZqKXlVA3=gTkZJ z(C!1oOLoHl`k{L0f^@$Cb5>7Y>0@=2(R3@eO>D`>?kAq`2;o6!x8%=e-kZ>~4#E$h zXGXSdpe&VhSZUzwr`~@tFR#kbdlqF={)RyO9D6ZgY56&J0eWE}=fnvA-|OPFzwTVS zV<)%FU0#`gS4f>IYbWo9k3D*>&H9MpwSizLVsUne)wl;|#(1l-?WH@*)DHOQPZjTt&l28$DVb3H28X9fe!lVZ<*Qjob>X9*eJOcFAEEo3 z#%6OPE75r?bzu8>n|G z@~qn3TVp}r*CNMAx0&koV;1%6E)7GYp^vXXL(KoV7upX18{>i(*plaC>q3#4z>P`! zq`#YX{6br6UMer(!O4}BcRS^od~*l7`F=ORF7V+W-+l3lvpiV8jC=`rsHaa>0KJ7i zh*yaZmP3ydJh48(Tj~edncn}y`CtDpf?$5M!ouc&O^~%U$*MDg#)h&X+C0wR={N1su#{4|R{&2LIutM+Liw4LEqoHc>*<$70zH7w8wr6^))C%p zpsw+AXSS?kf9TEZtGJo;n|yoAZ@ZDXd?FODy8*mrttWat_He?j%;8yn;FA0^I^tV^UNGk#zkLH|%pRsM zTE9I+f3)v@H+^Vqo|yhx_4)+s*eL>>eNP zbldZD{yz$Bh%RFsTpOd&xxb}y8K?KLS8SE8fPQ?b;`*@e#MlF`O9Q|QW4EzNeh%pA zqexHai3vSjWt#MBO6_5!SDlZn`^o%GndK0iF+In#h@EJR6LXUwq z$}YIHXwhK%(7o0}Z}v$9|nT=BNFV z;}hc`oxX^4i~bf^>5aXitJr;aSoP=G0V{-!a>P5(vq*46%|v9(9pii#*n!g0mD4G2 z`iAbdHO+7Pic)Q@(=K9f{nm$D2kc#wWgjvymx)iLV2<;tfVsoZ3XZaz_kQ7yWdUsw z-QQcAg+Ep%utq)uYgx|gUY;zCH{-K+dQu+2*;AqTe|mX3lSk{T2^c#G(?`pFG<>-0(E?1Xr_=m_!*i&Ehkz`dEfs^PZmyZ2JR z;LEfDcu44*g0cVnnL93j*W3YP_1NY<1x`dKk9x3kEpwIpt^YQA|Gnn@SBRbE5Om0_ zFT4vMz+NprAX=_Ju0ImzOXN2?aL^JB(RrR#J^S@e`*AQ2Y@3L@CdU*--2b#9S z7vzjB^8A{5;PZp@uJfG2=$Q$8Bk#r<$+7Ml!O|jcjo|xhv3>eeTp6Cwe~# zSn*H#PbppN>r`fV-B~`D*S%objnIG2X)Id>&l?NRgO4s9=j&{5@xB|7smJ+xmT`Z; zhFO31**_-y@dBSc4jD=I`6b<#tbnKeu#Wh^w^QuFe49FoQtZYx?6;#F^Y6cWw)lN) zxhs3k;C(9pMSIk(h`x^>Ha?+I|43t(`ZH$_6|MSwc?j8K@k8uLC)7M#;myNL`Xsp4 z_ZkrPiao9}S126WO^^MpFV6cL9r6uHfFy&9CZP?hOldO0YyAd{vcoaKQbSfdf98u&G z`E>M?oQQwWk5v~mBi7~rQ(YSZb3<+&lv#O}TXqg*YfYdJJheSzy}P!Xy=Pk&TR61q za%75U*!SAa@bB5e!vHoW(I)ih!QWt>>T9h5A8_VT8aQV7*Xn*r{0luwJk2^E9a1*e z5${-5=y=fD{${Lu^uLRIH~z{BOzWk&@pZeqSKm%my|i9b0xeS*@P&qMPuK&hQtVyC z{gt@}|J!&*x?l%f9tE8<07((G}5}(9oH^>Yr+Ni=ju; zQf$LZ-lv>3$fJ80qxXH`7;^~!*iiHX(t#K4&l2AfFA;w~P|LkJ$R>q%xin~?2aA=o zFF}Kb@&0rnG9@rM_6%o&aI$KiHK0jz%^a-Nd6R_Cz(>{gDVksC>SE>>8_d#D&-N?Z zPhqZX;34e3izWuG`z9KE6a|U2*k0aLeWpyp#iXgPN=qkAejtm8i((s%VJD3~ z(IKq$uzGyKV#Qa4o1WmTPxg7qr>U8-j;)&%J_yg=Jw2VX(t`G0c%*^*b&T~&#$019 zIGhy1-#^p&sFpiFry?7D)Vf^tuD-zqDygN844#p=w{LbQZN?cLE2EB|+NtB-(}ee`J_GF~-(5|0u+ zcuT(G$U@t@8$EkzVApVbzfxSAM>%J<`WCOq^npLX{yD(!VZlQZEUfit_!G2MuvYdJ zG(4ZVQXkPjzCg#S`jJ75C0#W!Ln@qV3UAek!)|Bof{|0d@1pypF}&3-h|jRDN1 z>K;Qq%do#FkJ7hu-@4Mz@zRfy-t47S;xDSSv%IvYNIMyCdXe@8+(@GT*`$2|H7;_L6*rF1Zg} zdJ>wy0r)mJPy3?cYYO49>X+nR;Z!SmMB~Mac9AAO*=TJ_STfCCc#QB?{N2>`uEky# zkH=hTY#@;jRVOrlAN9Wf6*$w14{v>eHDER82uZ&{&e>p{D*J5szaaad({(Vi&rg9x z4`-G>bY2JH%m&#i5;)^e!WqlOnWdf#ge>tz8K`jRO-2U#5}a9jdbuald3qss6>z3r zK3~Y-%|CT2-e@DSy-jfp46okLy1MfJhp z7iE4?K2f{VztHaV)5!0l`xAiS!n3UbpRN0U0K?b2F#Iq2)ds`gI|0M#;&agbiN9_& zYqhe+U(P$Q@IQm$ciUw5FM(n93DSnSOQHL{To@kh!SGcYbB#ZA|1#bU46_C#_?10C zb0at|;vLwUsP|sH8~972r`zCdDkqs{)@lBL!RJUz%-`u?kpEnnS1<@{7#J*=6#fW# zvjzB&KfUTU{0H5-uCq0`$%DtWssngjLmel}`lf!`_{*gFPk|rFV5Uys$JCkOEP@7} zEHi8ecDAfw{4&^o1nhV+!~Y)ryVHf82_Eb`3`}gAcPi{`d%V^74-0m-^8Qt@lh6U9 z?7MQ^Vd0I)F;@-IrJ;&oE86F<~fblc9z*ye$ z?~djB=yvae$C8itz<0kE7c-m!+VOM7vmU<%jpuZ4Ja11L&zlnCi42w)Pua+X2l8=G z;Nn{CpO8=HPWc~L+}mNzKFr#!*8c;(tTfi+BiZ#*(MAC<;BnkiF$Vs&TW^X&P_@4`+dJv$(~5aCPkLCy2Iv}+mm$+gY* zP29fWH$Sv|eR^5=ntg(uj`g#pPk+StkiU|(m}Wxl@!Z6}28MY5J@3-xTm4of&~0@O z{qOP)&i;oNX|MfW_H%ud${wi-?lWbN)OFyhRp#mv>3?S)D0j72=1|(xA3K`rtdovP z_=wk5r7?zI(4~In{U=&wWXWwOYywM2OQvs$}jj<<#+I{q)l=^;5Q~sr7hXqu zG=7pD5`Eh)IRF}Xwb!?N(p0XoEi%sXTlDdqGR|tVH_k!ot%q-z`J=2ElzE6yJo^%_ zT)mI<>d!pQ_+3oeN3>02+kyHuUK+PP3j+D7`{eOzq0R|jIcl#}UertD)z;2cr-z6C z!l#uzegotu*)TmN@-cOMjBWz|?OH$B^H;$(*?2!|Cg>(J@x_*Y5*vdr$ujaUPl?oD z?mN!DN_@VFznW(%Y1>G9nY0^8n?>5K#LwZm1KZ|G*61{bwK@2DX0I1crayv>lesbc ziSQ_N{AB)Q(iq2s9uIEgo#Z#Xi|`ZD61-EewHtXpF$Q~>BLiQIg@G^ju9q=xg3~Rc zx$sKS1sA`w%A#r3AQSIDVbes98XSBCTGZ7V;BXF;_T~TomFvsC#HMLz@#8Kn{zY{X zE&k03TAYxHIOz+mTR(PrTRQC% zP19JX@?H&ZJLG>jF9`mcb(KA=N3hPdnsbC?4{rqC8iBV@Dzq=)RCxOsc>5T5TfCri z8@$Q)Nig?`2Xj8|ING(YX1K#Xi3Z@V0k~@f?mkI^yN|yP?*7AtyC{6_Fz}=G@@qW! z63vpl=hE9O=L+6?({|)-*9TXAj2*nSWzm!;Lh;uaAG7u^TF)BuSpJb+UVS!ja{9H~ z4T-hep0}tC-2Y8Evd{M6Kec19*AB`0lIu?o+nLUX(9DmZq5FXCIpB61Y$w5RpaEY{ zZ$64R<5BzR{zL6~jN4f${d;LC@$VS`}&MC+t; zcjSN3R4-h!+6~8QUXtyG{?0D$zbo3unJb2`^mO(i|N8I~Io;UI=wS`+&f2ww zKgjPnyT`-trKi!`m;SqY`&Z!i2J5+?k+oJc{*txkAY;vW9vg&o8ua!oAMk+AllfTd za_PX5TYK=%c;Uq4CK`oNW4r}59}b@wdir3IYvgM3xPT9st2z!Dd~ z^J$;PUHILV_Yacr`!KTWCCr=fLinD5t-^q490w(q$cMNF6`x+RPOm{NPulp=@koHq1=EP!7PPdL=#&uB#dk$mR zMkDYGdb|uj!6e=iV?5+HyInRxa3g>mD%nwSCHSog4-e%UKi0ci=c@_53fx^bYG|muEFH4xaw7J>m^*2d1a! z9XNgnpC#Ge(PuN>1_ySHsEAw#zKmS$$`Lu>1N*K9M2VN~p!rMq@;!>2pTaoTP|r=& zSI^xk#>bU4677fG#XNO_CP!Z@k1)Rjntx6n{L!Y8cQxT)U=O2B8=NJ^DJyB5(s_6F zCt#W~+3VQ0=4i&?oB;kh+*fs%w}%>DQcU0EGgqXLz7`NZ-2`5$?Vq;}&^k@~ApIAv zq6bA&U0NOZGJjWS&?KKfa0mZ>!Ty{?1AahSLgsVVW6;;21NGR8MF(6O zV7WFE_`seAv6+DPW_=DE+J`Rn&ktLd--UlgqRmEEgGOCN*`j;W)lP2H#II=6_@p)& zTX()?Xj7}~eYB~*n)!n^H7D$S%#XP{g!Z}mPnNTfw#|VqT|v8y4gk(aucl95;A`AB z;{CIaxh{gQjU#`;-kG42eUjS4T0sQgvZ46d#m)GLX38&0w6Yd>Qa#Lvp*8bQ(=O_{ zc6aOY^Qnh?4)VyvRVDcSa?V$Eh&={`Q*-d)KsM4I?=^&Bcz@~abL}!;DAJL=rL))% zQJU%c@>a1|sgQMR`HdXD@=u&&=HuMK6wV#AY~AyC$RBW4v-V?cEw(QLu2*w@jrm@E zrag%7LcWoev7@Kk7b=~-(dL`|isvbvZ^<5k1FrveV5a9bx0uhqxn^GDLQa?V)WwpM4VnKtb0ZD&$P6>?J{ zd76P;t^H5>kw;s2zn%9jDH*d?|131Ge(Y=RtR$T$dNm&`@ z{k>kC}Tdr@q`hI-=3v17~%eC$16MEzSX2Ec4b|a&T%i4Z_zRk{ z&M&gXzvPo%s5Jb7b|vckZILbc&g9!zc%uGu3ft-*<<+0()qi|GenagzOa0rNM?fE9 z$SS2pJ?z4(X0&Kr7^^oGwzX}bHTzQ4d#*M69NHv5Bj(Vwbs=+Z>MrSH7psl*<74F= z%6=i`<$D@FS0>M;XWMzoL*J^DM!5?q$D}RoV{6}|`dIJv@zF&8t4eI`dnDgd>n+w# z-S_=gOP@sFu}gM2(f5sKxAh$xlG$tU{(N`rpQrDw6}r33&lzs*IBTyx@X!IcNCjpC z%vXvndeOSY8rn>K^;1{_t_Ej;&yn?m?f(D|wRhoV%9FhNEO1%1WHi2d>CPbJDy3~D zO?+v!WN~QuANW2@s52KuH?5iF@nytGe%-yrve$4%*rF?ICjKY?t3I8lyUeR+J^4g8 z1^?DhM#o#zt@vK>O6LSiCXT@ibWg6%)6{txE6Kl^eF=Jhn0ND^&3|33_@3$dFUR={ z{~XajUEP0vO1_Jb``vzUS2umxqqeX6D?q-b=d5sv+zGd@*)9 zk6~%i<7dFfxZ8+$(K5~L$z@FR%D9a@I`iRHLSXYm8N+&+GP)8coobp%%VX^%&$-sb zG$dEr02J~|xkABu0v zhWGGq;>@SDwb?C&z)I|8i#>l<{J-bnLnD9rEmj=eBmODhReU#Xv*Oi!H}EYQz3C~= zHU^e&C!CSm;ai-sae@CSXhpF#d;61|Ejz+DyN)p2hjY%JOZcmzPswbt329&V|)CrmRWI=XZ`rd)M{&XyT$JQAM}gJot%H8dk%gXfa+>u;>#OiJ!i1|N2>6JO48W^muNXhm}VbG-a}ZrnP2TFCl-(RIyH zDLhy3OyrrsGmd98&q$tc@La?*i049{^LWa5&gSXO(~BpMCyS>CPZypvo(?=I_YJQv z4%t>H+xBhc%=Nj*%h>DS(~&4PA%%zK({_1zq;*X0?wI-??9i48!qQdv<& z+yM?c`qEyjF1HTU(x!EO;l{FHM!KjWOuZj@2qzIy%c&Y|^IyAih*@r>k| z&Qsrh1o8L$w%bVSz%KT#tY;s~-+VdtmU7>LBEHArx7(TWw+y!q%;1~?$JJY%_}wRONb$bPeYo%+tYVAT|K{)l8Hbp46iucY~mq7yL=8-iDj z=bWJ6teN1R(lf5=7&-Ww6?aG*hYVRsIb(}BrySWWivR7{#d;6UN<(ggR?KdPJa=XH zy1W4LTx&{T)}aeI6Nd8F0w*<+6W=?_UdvomVY3TZp`j(j7rcN>#JT2ycJ5u++8eKa zz9 z#&2i94#LlT-THpUIKJ*H*!)zv7f;w0Urzi&FP^Y1ej)KMc=3d7@h=e1xVq(sB5m=^ z(^Kuec*3^$_QbOX(&Q&>iw_dtITY!7q;E65xA7S!8tlRpgrj zUs~g8<~cF8S@gXm(jl%qr3;7HWksRzcwfNbzK=~BXX$liMOIk*@wBI7s`h0`1|?p5 zv}$-SVV?AVNqN~&&DUXM9e9R0@4&!Nd6_*Pzu`+?;e0>JSmo>7SnBJWCco}+f5k3M zSyLJFX1=kbh_unGC}ka9V}+3`U7A&Op1qa&bRNRX zgx^5lGv)3YtbMTFnMqcQ`q%aiuim%rits_oT3b;b=FE@ylh0Iyr*fZAoqXXQUX}M$ zzWlcF4@k+M6-6IfjQy(-{Hz9d@x9KQaiKNn*mPf{>8KUi&wj+up6w9+WQ4`OWh-)I zI_I>lf&T04^Of+{uHvnO?Rr8hG}xTW_C4BqnD2K+rh4(ICccXJ=8hq|d1SU-PiS}y z=eSKFo-uxt`@Zb?75!}PF|mJHF~V*loHMeIeUNi?A0p&(Gy6zIsvU)A1j;#YXzq}f zXujSH%zO3xAuX$Ts^<;ayn5c9EnDUdX?cmVxAOi0?=SFP&wDHH&-1>Da0L0M^5mpi z>`m@)pa?o_#`t%$hHAZJ7cgPIW8=9?I)prd%>bCyzQL9y$UiGt51S6{gvWJQrwwh?!F>)r>fC|miSWHhobs#MBZQzZdd%A4J=mnv2P$A9Nc8S_xG`<@x7IE*Ua}X zOYHCRjXlilJ^iXS7@YlGU%TYE6%lVOW8Hkl(bjSKwiK;y$+8(D&DXh`xJDgbm0Rl zV$kQ_0F8MyjIA=I^HDj}2!Oj?XQ(8XQ?(YjMaom7V3E$?IZ z)jK#*X2MjPb<-u$r-I=IY+MajqAxNA(?cWd=6=3#)2-Ovf!|&9O?O!D1%5vSj@@8bqVX9q881o0q8lOlDN1qOc8);)(`}%)<`#O2;yHop!X%BGJJqoP4 zFx1D6UCmm}+|tdOAMo}9%~4@@&Knwt>;e2%p{vxe)-~6UO_8vPZ+z}HzIX@nE%>Fr zM<>bG#MO_VBYrpW2&InpaJzalcO(=4PvS+>zjLOwaVU>jcP&Pbx!m>hG;<6eO!4Me za-H_beR^Xk{@I*Ryc@J2I&{Oz2F5?5)Cvdv*-M)JX-=6xf61(r%q1Vww_wnUf8sBX zbk4ORv7puc$K)?~!PipoLP|?PInO{Vvf{f|_yO|$gghmruOfXF>8nUzMY?EJ&u?4d zj^t4}=0BA|n%bzc#IN6b)SdGp<~&OK0-YIKXh3w9H_p%n(VYanOW^6Q-Ov&8u%BXp z$}41_@1H3vIAlgIt#N!qu!HV^O_((y_L@I&CHq0y7y2@5&g?P&?BWwJu(vp5FA8Pb zk}n##2Y~$;?88dnWUhBtL%#FG54?5mb)k3)bJ71n_LrlVpo2sTnUjC9mX+YCk^xuC zZh6i5_B6)4H|eSTqq%#UwkiA_;py^AM@am0gii!vC@PyRwyUkX^qgbR2tZJk6K3ZrlZ{{sFy? z7iETy7nM}?D$2BuL(lh@K6;j2dfCNGl#jE4)&bGa2Z1eYw~;xlp)`TFbKud3z|Bn~``8KI+zfuUfV(*r+3;5QDsy=RT2ysksO2#p z`7aq>#QSdVTfE5PKC<3Cy5r|6zI7MbH1rv@13or+6zxd!U_h{+#+XEbmTbTqJ7zb|y$dT-j@ffx6^>GSrXv4(TJJ~ibQ?#c@v zonwWKZg=Sg*qPFuA5mtT%s`(XQlAMoA%`5MpYM#!wwoD)X2zg}zN^ni-~m&RIk?-X z?5@$F%?ZBapL>`0-qmh5_pUZMqdo7V=ik**$9RN!Ugc@z>BM-P%k%Lw(B7wjU+ptt z9_iUEIz(uXs6oR;U}kLY(Ms3|bkqR!)4N9XcHPWS}&> zQu~z>@>?_G)XX>?Wt`>%55G9u+L5!T)K1XX0DM@ouJpVY;a@R$i`rc>cgE&|t|V^KNfJFKpB~Q^EGkvH8wuT!`$BToIo;U+KAxK79`a z|2-ZuZm#0f8q1NvfZlRrWmWV09{I5h7)ongGsqb=gyLwTQzEntMdXJes%$`Q3)`TG;UR_9MWNg8LND&J%L6IeJVBHlma^QxUQxEoA$rc>ut)J45ZWgm-DS0cVEX?hbk$xL+Hi^fTPq@2}^^mRDY1Hsdh?vW^Z(nXUhasb zKW;zs9H*_`Jm=NZ)ceF&)GIjAxIIT67gq53;a#wz`Tqm&3OAatr}KLga{m{h_@8j? zgOos{Z(r-=?#!*q3X^8$D0pdP=I7&gju>hk2rlS2srMrj=GC#EItX8P81u2gPLWr@ zuhIjhcK^90oBxGgE}juWAHR$;zD&l~SmV!0AoUzSTF~)>6ZNI8nW^s<-vKg}`No(mfif^gF*aeX#le2PdVA?yjx9 zvgH-%dTA*-Dfq5?R(xfgjRQ}|{)}Dw>+eHp!cU!jyE z!#$JRq4fvh6EomjW9Q$tc?9dL#qd7a0MahNHsbN}vyPhd?q~W#aR+%+a#0H7_gK){#NO&iT~BLZL!U0;IQmo5O7ybop5fLH zSU)O;Z@oSxtg+t(jP6F~5ib=#-hRL9ThWzyiesOaT)Wqg9-hdHT%~-^d->XvPir@O z_jEXs2Uy+%JZ|#x97Dg+8qs&Si((iLm#{!fa-99dH__%Q+I}Z%05QHr8yX08#)m~; zxQjKu$HcSeMe*C*_&nnO#dibeh>AC9FL`ubSNfFayle9Jblx^$jm{(*T}UNC&A6#V;Wg4)q~)b)gI=Z8_(B zKu?FJTHTZ9`i1@A2Jk4@5Wd|=8w8)iDcQpVz)H}UTNc~XIzaxz1Be&ij_`Aa1Gb}t zoj`U^$%<>?g^%zUTn&2mpI|Cy#Bru2WpCV-Iyh0TWX~A?Ek<^Wf#apV^s+GangVQ0 z)8KLLTEhr?5AZJiPUmK7&2jJY(czCqa9;0h^ij?$N3Pn@W()Tlo6?4&o`dYZ)OP?G zR&`&%`&<7?9W2|R=*F%kqr-bBqX*-%^WvP&A$Ig6XW%(@47xS$#saf z{k!(zO6*KWZ)86gvOc_gL($g3c6-_q9glAreD`V2Er|ICI*IyQJB)1%!ykhqqi^a= zf<47shigqa+#!3N-s?##GxQ+M*dP=6w5Grr-z$q2SNzJ*D17<~fs51aWO^@a z$aTFWPjX+~aqTIOO~?1SkC7`k6s;X<*Uv%6U_PE?ypsE>x#sL8+37YEozvfbhkSy8 z8piQU^F1`c{tNj8*w=>)RSVjuvU18wZg;RT4CaD>L%0v4CjCeGo8-}wYPmI=eTU- zd6?&wao9(&oppp>zB&OP6W(<%?{@Dl-toMTT@YTG$K8=-tYb7OM7jqzhfgZ z&*X8wypH3#Tivgoz2IluFaBCdW?)HRfn^7dkK7sfaKX=J_`zT45%}^3y)!FB!&oDT zq;VEgBWbfZi>dV7vXyIipZ*KKUAWM17nP^lDMglDq&;a2doQfx&Q!OKA;>|} z7cBg-IqPG+?xfUz_>skT*K&UzJWP2H8(7PB?xC-DAP6d$TUuMEgXS@lsoUvXT#+We2 zxrC7OiYLOyH%{gm$dkp>%-L26SPB43$!B^sFsBK>@E~(K6S`FD_h1`+pmdV^&br{_ zvhgmQkLSJb<(!=a%r}C&rKxV7CA{Cn-9G0rf8&AqkAe9g`#Qe%fv;2JT4?$#Uzd)v zfcaU#{5W6mwXvDOC8fZ8+3`zu&g`7Fq!hRf#N0KO-IS%ihzED_`k7|JEGJ;XOs9hh zGn{rNWIv?|zy6MO@%e^y?7#W=A!0Waj(iFnJ>~gq#(xMrWCK^=Neb{;pDo@4o^*Ec z#K*cNel*8PH3#xK#=rMpz3_pfbj-I#)IdmW+psQ&*w|7(p^`Kp++wtV70 zN%>?8N#rX87TWR!znJe=UcSx7FUh@AS@np%sLm8?5qs@w-W04s(73{b=3h zhi3%A<23NOBY52j{O%0T=z_1HAHS4n3PSPC% zL-A>XBVcH%2{WDVnlQ_`+JxE8RVK`Ft~6n;^KC+{-7f=YZ{ztk&mf*09uDIY4s#~@ zsps-B-)(z45;#0I35Ulf;qbU593G#9!(;y!aJV;cqy9*5%=h}5YeLSOG+~yLX~Ik= z-GmuVcN3;NT}`Mvr6(}{moX2QG9TlamvPY7v0>)P(ATxa$i?6)xE$9#_G^Iy@r1R& zvg}d~e)iRZ-;Ln=r^r5kSnTpbi}J-!>jfLkQJ@X4xEtKV^CozS{08c&Lu)4DlV|e% z3Gqj}j-p%_k46#qp5Z(C7XF9g*Apk1-pzv_oQJ=E)4#P9;Tb3B!iHqJAlQDKe_tZs zm*qK1dsOZo`uGxgBW?9?|2S~Bm$;2>ap*a~qwc+bia2b4$o24Amrk_7O@c-sZ^zFh z@8ik~+!U~08nEg(>uJ%%-kez##l{@tKIt;v*HTX)lel&5!aCzE$XRnb*NXd(P5acB zaiUA$X>8r7u+~QcXT7q5HL7^5I5m8%!s2`+<_UbQ=?Y)>-^abu;OTUq!Bc+{o>nt& z>hogc{+orT6~fb0;2{891c8q<;G`pX+6n(izb${TIDA$2+{+(KcgVIR@zr^hBiy{x zgqhCmCd_hfGa<47p!NZ=Mk6AYdJp#N9fQO=|uVFh3 zXgqnJ#d|(@Sj`#j9eEFeha-qz2fqz~hsraacb!oyKD1i(z2&zv)Hdj%`M#!|-HmUp znd)pi$toRbZ}j4daA3i+EB^`)4r3g$*7v&QU<{BRS z-^9g%{}S*m512p7Sk9LWd&@n&%%FJH>5f{b-qRq_*7~vEq*01>Ffe=?tE90Z+PuC*8o4?&v`6?2?|3 zPSCkAaq#5`_|ojjp)KIcp(K2Hk}`!azcpc|1y-Ojcz)dqaOgnky{lHF71m7T9;^cw zUGTt)fG=-JpuP1*e()dY68JxR@Kr;*weBhSTFgA{HnIUSURPJfyNx)Fe`4Me`p~t+ zyK#cCYy)FgX`TnyL}qXwy?9W40c$#s7k1S>-^iLSZL6&rwtITwp6^`eF9u)m1*RQy z(NpSlX25Cmlo8zD?CL4G&S%~~m1gX7oza9Bn>J=S6(%mzDK}w;Gs1-F&Ttdv!$-fx zynmDVzZ^KgwshlV@X<>d%kj4WC$6qi`_=mL*U{9oyt<1`nCTRmFv}@0VYYJyq2_-M zdvzxB4CKk;Y39tS{OK+|oP~c|;6|4QcH}+5$2H%n%y)pfEwI`h2r%EJly%G>bV|_~ zNBHs^@9AH8q!yci=J`3#uGfJ&v0<2b?rP@Fl_~xC?mQnc_~Y_%>8qk6AGXDvpdnvg{3#^`vf%AX40)BZQ0Nw`+fer9J|ANIU>hLoV-uDc?Z}5JI2M_n~ zzwrJs%3s3#?e^%XAK9fHwD@-7v_I3uapWlvCaTQ*=QuOH`duF5;lD}Ca;|Gj<338{ zz3V9l=U_uOuO{u7czGAeIkm%%B8%47E*!Rd4K$%bdP%PHvcZ#FXA*h;U7FC~{Zrx9 zT}~+Y5Dt99#A)6yV;(PMKF2e!}G3Ri|Uw1@S%0Jhl0T8}R}*DiV6taC{I zJ79GiJMTJZx6%s7c8n-2yNt7Jt+ZKVi7&jYqt4*B&A%`7w;TLcS>NA)|!B|WALMcz*@7}t7SQ+&Ayv-F6*EfW7sED zZe7qD`9?mThro^RpY!>0y(iNAy|DGLZoK#VX77fR4SWiB8W%|`?o-@Yc2hkt0O&Vu$k*4*zL(Jik0xR;c(N!~@4~W-(nHJ`Gh|ac`PJ+A`Wql_Y zwCo|+X!Dxy-nl9zud$f@Yg-8Suy3u-pWC<%zmiq{^u}7k#s2iyMqFSWDCT`Cw16@u z9|kYBvUQKN+1cf9GDCYP7Irpv z+Iq?^^88L_YOH9F%3Z=7B>z*zKh}>s8Xpy{CCV4_?Sq_??2YY}hgvsb^L?{zO!*mO9V2qIxO3^j>F5iy0^#T~>rc7| z3mDpw#~lHGWIa~)>L2l+);?CT4pw*H33`$#dVX-9!E!w5#+Qw~T8kqY54(f4jxa{=rF)orDA4+>0+Lq(&rtjiggQ;tc|0>pYi32u95qF{DYQN*f zh8w=9=t{QwJ+-HWf;wSBjXdGFD^aGvRS`my_Tnlvt`;r-|!7d!%w6U*`bs6N0hU_ zXEDzkJdE=!U(o(>ck4hovV`V}OcW8mO5g=0o$4PTq9Ny^Z=lf<}D^joJf^ zf~Po(?z(&D7{+1WhAy`D()bwP)@|+JubrX4U7*8Vp~u~z%gEvLdth5?$9lvG{wkVx zl=6;I9!Q65Zs21_0Gc<0_3%M`KVKfgA5;BZ0j;@<=W3n-><7=}`NQ3J@06dZ=(h66 zH~iO(qr3N~r}JCJRxtd$#@5rz2+z#%6<4Y5?1>JV3)vkRTj!&k1 zwIBbkxL{+k>Vy8>uMilyS0VaRtwLzeOcM%LCNj3)VvN7ZSYOVVPk`rLhTX`Obt{io z?F`VDQ|bx6=6&FiGl-R`W&O}GZoU8QbO_G84J=@a?ct1a8IWZywa`U;~nwlX80G9c#zb7@yU##-=H zi2ObyCBw*DnZdW9N$hoBJdzC!-jsK+gV+}|9y(K4egI|ApWS;zGt(V^nk$24 z8kwf2vkbZ-da1PC%&}-+TzJzm%XC`@7hOS%GJ7zl?JR@oQ8czQbG(mA=Mv1bXup&ok)w%b_`IvqKTiksDCZojnD}HOG*djzJgh zr@t;e$#;GVJc#bBCQf^pCHD%a6}M9P&=EvO7I|?$SDfa7@vZjaY{eOxmE+t)ob-Uh z(3v&Bc>}l;4E%FN$-4f&y|D-{2M!akw+ z_Z0#kH!EbmClZRLol@p=UF{dZb@|?wkVj>@{E9s)mw%h_pUgZ=Vm>B9gT95%{momL zC!=#8yZf%4CD0%;gthBg$7#rUg@YS??ZFl=kV5R~4LKgG_yHb2Raz(TlUjl`+ zzml+|eQ-%uY(BI&-_bee7n$+NYMc&z&{z-RJL=JlZPb&speOVs&pF@AC%Sz~|2Frq zA7#v*rhdtuoH^K7+nsY5@L#GRoSBkocIDJWo-D`Dy(T-nJlWn@9s-Vj#G3n|zUARU!g^$3?ER7EF;;vrV;w8*QF`cd zV<+3-8~);O^vMlY`HRDM`P;8=nA0g7>)V-gAd&g@S&^S3t8IPJl_?CargP@^oo<_s zvbR&Tss=fKRZ4ba-G$3f(Jiw@1L@D>weCNk@{h`Skusa14ZDj|pk*1RAMN1#yA8d_ zcd7_QuLc>|;GQGkO7u+fx+}BkjwF|k{aJBFW+TqiBPFkEo+fzZsE*yt{UXYUn6zAH zOmZ6V@i=L}H)%OeC^-#Sd5N@jCXF@D6rEpL^X631z`^KgMn}DWSTnL;Ju;tU+C9j7 zs!6SUu_j*1$P@U&AwqXDzl|(Y6=R#|}b& zzNK~=+LZ2m2UryiyGe13q3~~#7k7i=`YW!dGr^0iQXKq7b2`?GyIOG?bMSPO7k7o? zJeoL$xJ+cejnLErw#$p&1p{xb7y_>@uyPthtQnkwyOo)?k8u`l@==!hD%v*K)D?;k zRM?NZ_7wKVS4!dFP`sZ);470*{LB0Y4L&~qD#ragjQ^F)!xhZOx1nj1p=pz_BX%$} zP4uh|S;>L6RW5jCCHT1Uly=>mW&cC;3^_AOeRcR9M5(`0A?*n$>i>t$%_l~cPSm-;j zjlI9C@H?p^6n)&{&uNi9@9OC}j%a%$ebd^$Xys2AxHRVNn_M~TgieosN83d!@AdNh zg*=kOANQxUmA9LAYJbCBUY>2_(L6>e|Bg(~BBYEf2%FL2tGJs{JXLa<^=Hm;g;y4} z1ndi+bK9_ye^;}f)l3`L_6b#(16zdbd)e6cBtzWC ze7B}pjf_AYSZV&+og zF^@byADLp)_YD^3kla8yT36fJBXorO#V6MDN7cMo>?KqcueA0Uo-8%27B;tU@kj=8#? zxNn$pdOCmc$}wetFN;VUY|`?bKPRVwQ;(A-{w`Ycd~zE2^%7~Ns>d%I0rddadx<;K z^alOd5I7$;CI`;M>3e{J6^Sj>W^8F9t_z?;j?9xM!PqN2c>2_~pt2 znNCN>ZPycqckCGX&z_`bIR9XbWhZDcc7jZ&3uD@qxe=TSMrAii%w>Q%*Eud`t}FBT zKj1t$(}i#CsdwpBhI5>_X2!Gr{gL4#yi3k&QApbln=sosNcb$WbFvSjU?HJvU)|^P z<=9FF>;#Ug9vB;`v04Tv`xzRT8g_>L!WAwo#3)bjBVBLC1Hs5Y`PP5Q|7iU>R@?!e z_T_8XlGB?!d;4K{SsY%Lu(wO^N^awYw9!KLmfr9#Hpk68D|lA*2+b~sUsYen-ByH; z(-z4%CH}0&Vy#IZpSZIiXdQS3-XR(LH;lRX*klKx}LBnp?JzL4<@eXAK4`~&k`QL<2 zTi8M#VO^vTe!G*bKXJF5J^XgJo!_K9$ul*`CF+ai*>%See>9c7jHJotE1hL`vHF{D zcuG6)ReF!+`dR81EgE9lsx`0-v;K9Ka2}eXe>6U$NSC}WJ@-lPA2%)2*b{S28hMf3 zL=XPT_bPnE1JH_B`1b+AQqJmY;mk}6KaU{tru;;-4iXgo$3ObSI!L~AFEHfJPrl=$ zU%SS5bYe)6nYSH`fosRgciPim_`&REa82Xp!c?BqIzwgTI>$|zMi~GNpVpweHmd9%?N@cz6`M zoM0sx55IXfKI7b@BzU}?d-Ja5xrnDHPYZi--{M&d|19Vpnmr9YX>C8=r|I4~-ur9a zGc@~Ae2Ui&c4f_S-k*lQ{z3g1!5#_Z!)QvnkvpD-uC?+mxg&xM(wdUnIGuOxD=8+7 zNiXwN7`Y>sl56CV>Bt_j6xJ7zNBq18j!)RR9s9H3;R(i4xF4aesOTeoixxx2p{=XE zwBM^8(Dq|S|L*Cm@Zx`~c;w!r$RL6Z;k3&7jpC7gkHEV%r$6=LS1TS|JdAAn2g!IB zqgN_~AC@SDZ$+io0*7k})xP`rSG1;Bavrj2t-fn~2PEH#u2*~C#rlS4%-44}k2b-V z@8COdY{JfHdu!7vbY)|2dzVL72GcgJ12n_$;cAl zwp0%eZ>b&>e!N(6p1T*{)vZ@gOu^2sF;^RmZe5X_C)ePA+_T+<$fNmve0p*oY;eew z`@K9G2ho_vYm@RIGx+fb+vnvuk31Ty%-+a`-gw`5%ppuY~pq zHuCvLdj$)xA%t6j=6`U+2zj)q5wBw=2IG2NQYrN4NqldWX`XKLHz@3G_%_{H- znL>8{9X$`I?w-zPq@PcF51Tshxn&GK;9D?cT~!|E68Z3p*jGnZrP|L!A2kP2{!!nz z(iYLx@6ZOt?7HzRHG8(i!*-JQ=``V2Pv<4xSF?}kIrb_&&%65i9AQ(j-`;p}C<0&Gf3goLtlV9`TP0)w0dHBe6jGdo%Ke+ z@JE%ywh4w^8n_$Yg!YzZozC_$ul>z;FH@OUG%wEz)Dh*n;BRLJ) zC^`IMQ@&Rgyx>gY1}Q(Z?xp1X(63RX^;4Sq)6wPA*ArLf#id;5!r*`LZRT#^i^J>t z)(mgD8h#@=TG5)(=_IUMkXBs^Fkogkssop&1 zr5auxuxfgwx38NLU`{p9lJgR@L+^^y+^O8onoGvIo;Igx&grMhJu9XMa9VU^htdtN&2=JP+-r)1&t8qMjd;_u#1~lk zo}K9Z6$2OKV<*aN4B?l3YTakIogZ=Lbb{TQ3+Rq)GBc39Wj84CbPVCM+MTe| z-0OYIPIFQ}HdsSS>>|eSZU7ho-r^}K6@y*HK0a(AasGvdeG9M&BN_KP$qhU_`}jb_wNs#VU-M4 z`MPUJcyx*QLOb-!<=kmaeXBSpKj!~=9{!KB#lslC!Hmm}B%8+ucJ;KuR_hIJpKjnR zk3(7y!Uh`H|HQ;-Bj+D}hMuv_XBW?L$7m33k(^LdCK}6Kvz{&Ke6Qc?N6H(a_z>vL z?lq#N*uJ4F?Vz#6#0h2}rM)Ngj1#u+j-)5sxku5*4vI6j2=Xx(*AwS6<>WdKdgZ7N z*;y8mc7!D%74>sH{#wY9-6GbzJ}GeVx{~F7d^Wa1L4H zjlM|zGw6A@W`~d7niJMO$s@~rk%Oza_uFs%K0MOj61_E#wEXaqZRCHM{QH#u4B~o) zTl0PKNAGJ-neEn3!Kd~}zcWhf=ac?f*YMHLx`mJA^G^Z)EK^($;?nV@%?!6bfo$Hd zOR0SCqWJzD)!vtW8Kw1C>boP~ox)~J{{uaA7IE*hjxvXR+U!Lad|KRX@+jjPC_Lep z8C!EtL}v*y#2-F6|F(tZEU#mp{ckGvzl1NR;LCa&`!N3q zS4G!4Gw0H8>!0@7bARZ<>5rK&(c;zEGv!}Zcj1at)^4QWPqJ9fr zlIvXH!SsB}@YV>H2Br^tr+dTl9O(-#4WJ#KA9e4O%2zuyf6AX|o5m=hIzOw49P*>@ z-Hji0U*=|h$n~RU4eFruP1WyTl;nGq=gd?6*Shs*tA1#gnJcgUl#}XG{hu&zC)Y1| zJ$gq?q^<4|Utjmt)GheY`2_D`KiYMxM^CK2v?15|4Y29LY@YM}^{zj_A?y^QYbguJL;u z3p6wL*@1UfMDNL3xt6wY_C>sD`lzsEMfC$6N%r11n>f3!m3G~Al#q5DRY<*akF{<> z{(JKjo>_Okx35ZU==g~;4DDFlCKm@wO!U_$vCoqkO$u^txX zoSOi$%qjJU&aoF#zwR#zwBEflz#RgO_#4ewu+I1!Ek;L_50b;V&tn(7i_cN85gB3f zD|fr^3whTbpAyn2`_%FWoNK?(-d+1kp{?p)^XnDi7W8!a{=|?UjUNH>sq8Iv3h8g9 zLgep&LUfz$jCu6#OmNZ{4p3hsGUh*wE}!Q_(>bS$Ig9oi8BU=e*fQSKT3j*q-{{>r zE0vzZKb%Ddtl%3;-bU!)iM%;nr-jTy9{gP6ISaB^{?g>_>2xG-a=uz*8_9>tldvVE zoRlYqejwR#H)C_kc<(>gew=%g?f@Tby|S})sCbaRINy-hAZijv30{Y>Yd zyvyf8V{r(+_%?V@KPSb0^#^X=f01rvO5(MSaScOZp#8 z-aMzrNqH0PeVlyrNfXTodU;G5@cI&Iw-MJ=k;F@D>xLa3EZW1G6uRr4HR2N)PJ@Tb z@@M!SN5l7?M|!q1aCSm}&2VmF+@-&|eyrKf_xKh(Tu)dAT(wq+MyI0(WuOmb zq8DYMA7!IU;-}#6VEj7|L#MUZU;dpsqh3B&hfer+w$bV53pu|5IxTzl&)Gv!fNoKa zy>@`=1dbjyG&$1|-4Y$WRPoTdd=sDV4DjMdD;}ErkfF&L&Us#ZNb%6!bQ7QMlz8zM zDIR+JOTj3#y|)){>V;n9nfN@Xrx$;o@s(^B{~SWm;j2ZD zY0Fy*se2N1xs|&9sgSa#94DkK(U3$r()n`m70s>AMt99MJ`ShgbrK!lex19n#2I;J zP53z9vY#AP2;Yb+1V)+_0{4d$0(VUcp-+D|VKy}VJIvjc%;6Qx<+qvB$>_F|xP!s< zaR{`2HBHxfSFhryQMur~Heb=`-R~J+(No&*(dixhBf49uJp;5YpiuNzbeOgq`b=L8 zeTK#v`t0WpNzrBPJ%fH`K}Rxd;gQqi`7-+aJB>j_ioF=Pk#5`AhrLLB(!vdW zI`VW1H%#puegz$`@Uj<&TMJXxoA+<=en!pkhN<19tL(65qpP6jK%19ZE^Xc*+T6-{ zi^7$K%&~OZt%BwLDfWz8-T3*$OWwggvM~xTIGGN+zF#@cS)T3tR`N^d&>GpD@S)wb zTlC29(c$UDx$C5vPCdAhpu@t6D?Qwpc0dXHu->wQRKUr zdU zF#0%obIF^a*S|R_ugM3jzC^yRlIp&ySzC#;?XCEsLIwv~1)s5SsIPGQF1HR$L{Y7!mKehigH|{;+Boka*_1wxo zxbJ@ahBRn^k&72W7n894s$Z12vIUwCP3gR!gKlb>l{mc9&b5ZJjQ*2(5Kr4^>f=mN zA-J(#AvpC5h48_L33W&3OJ$Be-PpOt94~F$FeA&#&g@$9t3J`wqw{Al))zD87cur_ z%)xZ%=(I3%V$QSbkIBYe+kes-#tvUJamyFTxX8wP{ON;BxknYiPG0u-l8^WK69!i@ zK64hl(x!d`FOg?_cqq|+}IDLe%(SLzD_V6}YYdrEW<4~&g&$t8>LT|UAuhSi# z`nwYhjTKH`&0H875eT$yhThfRoxHNe@6cb(_vNI^E`KNUB-tr>L6WiCzu1j;>0j6O zL>GiMt1W|y!utjVjEwquu6$iX|BzMpOz#zLrL2}=y~86UFHb!IAJct^?;Gx@GcxM3 zWFw;{+r`X*^pP()GHMq(_~I`ddRGXHBcmqUBlxfWs1MTJyvzN#eSN^meZElk_Z@h_ z2>t-_M>h6Px_>Ye+pN-vyO#;lvG@|qU4TGr(+c?;6HdM7+Sm@H4C&ZJGlxJcB#TLg z@@Qr7sArNIPiKupc07f$#Y(d}xvX2y>|O ztgC_fhYNETVd?y=>Hg9@1EciCq<>`6)9jlGz4}%6TGIYz(o*f~yQFddmbCYXGcx{& zNcwi~*nP-5&3kw@h(nZ1HdA#?>j#rMEY`arY|D(AzZo1~+b@;-I&;S8Z4+ zU4!uL2c&I@psk+1ArJZZOXHY2s?%2BRLWAHBx7D>>O%jo5S+bIAw2SOh4A)E6+(l9 zgyQEWA7lPavSx(-pewx?J=)kp{S}%a{eeRxGrmM4r9*hy(8#aXAp{&9Lf1OV#)N$6 zC|)&m2^xuvDH*kvagmH#s*t`06tWJtxHPj?p>zh&6X^`J_Z6-sGtOp@+i9m1m>k}g@JnL7u2pzYo1sqUVZ`jT9F`pybx98>MZu087S$uC(} z`rJQ4R~#7>y`n3FzU=?V5|H{GN$pW}dAW70D1;e?`X4n0k_ z_c719cYHN4D_od7!ir9MO8KNOCjEYso^EFoI`ot@S8l8`X=!#^WE$&8^mH+C!jl%{ z#cJ7PFb<+Y$Gd*igXfpBulz%2WvfYJ_&mP&4 z=^SOS`eJBdl0BR;5?+=lj`7qw6x~wXxr!U5xMX{f8#hLA@G-5ufo@!Z;uwF=?km%c zKTYw5HfPu=Zd|_Npv?i96vOvofEOc*S+$3>{0hXhh_Vfr#JB$#zBoR=8!Hunl7H$QTv<8x{s>rZ^+_k4?Q{7xZb@mqzA%@&1>)oThF zyH^x4SGEZ=?T(U|Z#*+8(A~lEpUW zo*Ays9TmnO{~52dSIOsXUhZYjn@D<&@8K1{^rvpD-<-OUy~|cCZbKRmog@3+`r;wk zLz#X3E$r)G?sMeOOiSmyTdj+*N%7w|b~kIJZ4myA@FU#@9Vcy_b;--;DD`JmB2d&N3&%YwdI`9cY{vRsF%d2Hu)P((ir4xT6 z@se%Ijm=S-Juy{#{0#WAo4b z(HPk}McLzsA5XmG#3pdtp_!yJ&sURnu1UjxYh)U@u$Z(_CN0^1H8Kr+SxegKCN07K z&&V`z>J`%RN!t&8i!MA*x<6={cJ2fBM8|gllkL{~?@F(B(v>DVq%@PpyCsojgTJyz z>Tl9g?5J$VzUVc+&06DLef-g)CcWVll--zr>Db!B-~*05a3H*ASZsL5Y|qXp#ja+p zNlw#yLq~3b&c6_B7|5IzSDZ^*)9g2xA4g_Ru`T98bp8l3v(}RQTWkI4zvN=?{~^X) zdhy6_mG$Wrsw>t0hY8c|zY}J{502dMOZKlm_OVgyXZ`GJqtPK?D^!A=&}MWD;sMY| zox{maSbl{MGN-Z=*10k~V5_H3n2_R~s41QzzJqJzM2Y#(*-K;09 z=RR7+3DAD-pNaPS@GUGq-J9+&|Lk$@AMIuyMNfD}!wMBguR=sRuk=sWt! zGVQBHziYpRyx3s$9GUiysE0{|_Tm>sw3#x~kkLeQ?^PN!NohMJD=H1TE&5kQn&NXs zufgT{`j&mLq1Wm5ZEpHNhgL81>|@N@Z!R>t0MTlVEzDp0r>-#Xog2z=ZG=x_3324A4j4fTU#UBCEYcbJe6u6gwDz5Sin6uno3!sQMv;_*nWWZ(#Cjfn0<=q^g8K5SJx!C zbULw3bh!R+1>qNJlG---lcCeehE8vRPCrCnL>FIxM%U+#_UQDou5|jL6&w9&(CIYM z>2;BG`k^;;?h9G{mak~@>pig~d)v7V{r`+|1b-&?K@!nF zdF7eCybH72g;_3He|l7C>l`QVPU5vM$Tq%*lI#auTh_l2@5u1H#YW$rC=u^KhIe_# z+r)_$nmCtt{EoOa{!6hVb#P-GeR+X6t^4tWow7E#cQtA2Oj?GK_d3(S&&8yLOA;c&}qQ02A{NFh`i%v z()0WwAN`;HvV#Lx0gJuDUg>i#HuXtIddz42*YX&DjNvi5i{klhkY7{pE{bSDWILas zo!CZqwlkXgg!9%3)`m9pcklE2CBH}c>Hf$igu2@rM<|}N1^(pmoLSr_QQoQn!OwjB zm%m5(P=A~=NyS&7$1RPwHjIk5M(u$I#<_LXM&2x7vr1qbEek~hE`8x|1N%2 zfq%a}+zE1Y-l4Q-TBaNa6C3bZ-Jld_08-hZgu%kEBU0uT>}r2|E&WiKEwWy_ydMNB^%xEfq#hq zq}hLS;SK+xP4DZwlmDdJe{|F37g_CTW~|@2OzjP}Hz}k&Z!2V7zNL`0`=&zH&~t>5 zI-?H1Wc;t;A2rDLYs5b!-^+$Q(cTYD)E$D`U48B*%9LCZOkloe``>)Bt z-|#0K$)}CsnuIpV*Wu{};r0Hcw&!Y)?@N+5$|f)WBa-!Bs6nnTN!xgEdIS%sSvqzP__q6L>@S|i{^IH0;RCn1=@ERO`G+yb;-(5G{$Aomb62}G_e|HWwVHU*anY_N$OTW(f6)Ms z=H5(Pq+P{H^z#Pd*3`(qmTXhgjjVntaiX~`&?SeiI(7_K13%rQ@!m*enoD!fH)(11 zpCi*;nmeAfcJN0ywT*N~9#6CnfG5&Li~f#v^Z!=qMjlt2cprtdktQwG-W*vrxF$M$ znn}aw1A8~wb?o4~e@{#E9M;v#edBBi>GOn{zUv%xPMIVl4K{6F4?HR3DfPqgkzw;-{j+2 z{<373FP)3`j)y023l}13;oMa2<>BZ44*V=#%5OBkB&~Pa%DbdSPCk4odtT0Ygcpa= z*BwUII>?wFpl$p7X@dp(5wO~is$U-guQQTW@d>T_44mM(AH;eohGs;{j|OM|?(cl3 z`!Vu5{Nng`R=xYG`;K<3zr3npWi9&MTI?en{@JI-^;c$|32 zZyDghPq}+|oBZGHjL&bs)@sn4rQ(zCJ>u{mN?%g#p8zj@t^X5Td*5ZmiFQeE@d$Xn z<0i=m>79367ZbOp=F&Dt-zi&D=ltmV#A_b=x-!|_lw)j(i8}~RJ8M|*_}HS(s|hCEMqk zkUPC5%&>L;HNMoD1Hm++@9&fuT)%@FbLPwVy_>^aM ztfR(y+k2NN&x&egY}SjvYn`;PuYA(gEg_Lo{475e{DY-kg5 zU%+?I3ayb@#A%HzBp-BOo0oSJdB2UHK>4=UePo@xNd^?%lfAF-g7I7D=^0K^bJzdIPSn<)VX?qb9M2VdUwozNqfraqoYenvu)xJgQpI@CD>uotKySe zdB@z)IIo|6`pH;-YW=Q9sK`B!yvFGpGV#~@;yVnDORidP+T+kT-Uf2p^Ha(YjdR+= z{$K4ebC_V?>$YVl^zHkCgJx^r+`|3mvykij*g75XXAa(tO>Gl*5$4|=yHdWDOOd~v zy|rW`(Kq>!&#=#M=kgB9k`0jXrZ(B?ADdwHKg$;woHfXrCw;_Bzcp_M^82a;>&ESb zrr%>@LWRF~*47;I%YMQe8{t5naNv=ihCX_89|fFnuK`$7#+II;(QX~fGv)uBGz)q!re@J6i9)!9xaO*Anp*P0hb=FX$+Be}8p`)HIn^ykS_s{O1FS)H{s%WosPoN0;=J*i6u0P(_j{<$Q-Mp#9NtYm+8|sqR z@nOoFhNi9_z(HT#cPeT+->FdBCEwW85va2I1s3#}zT^w5U;cug)9VL~4olVvqXS3A zS?0YHvpd&nf5IR2w%&;eGAEjwDQ-BK5L+$NpCi+I=C_?~HF)FGPFo!R@yGbjF5)yk zBPhdxbH?cRZo7Zr_Vcgbb3Xie_l55srDPd=^X=&?<_a(TM>CnyXBSq zsXc>u#;<{^jmmKEy}X|;Kir!07VbG#RAx-C#}ASGwydss7=JCk&>F@-ZO&z_?Spq| z4#bNdL+|wX=1VpjezeYcZ}EEYQZ!faPf(9wSDO%DASO(*A61A8w{&I- zy*QKSuDH;75qXS{J;nvNbPw*E-usasYSCJ~i=g*@bhkq9h=}KiK6tdYUVf;h-=$Bq zZRB^smp79fAJmQ9vGCF=_}{&szod=z!7hwLHZ>8UA!G z_%7ad0eOskjqSw#o!oOLt&lYFbLmM6-7-Jdxf=UNorh&X`(~q)-On6}2M%$|I>#-m znlm=V4~-~is9Vl@?UTX31XE5zYa8<^`l@=<+;WDv<%o7FK7)8i9_1dLTMln%t{3iQ z@gA?rko+ke4Y*~byJbKRDI?X6i)c$6X<7KlzKT3%|4cu>>3yJ<8BQBoo+=LKEvR3a zjEuXf>a6f4_Qw^RPd*H8rLx!hfcn-@uHL(?Aavy4Oz>VlP;@6W*|uGM!d}W0ZI-Us z^1HPC)jb7I)TYkBd?FZbl|4cs7y)I>6(Y5TGfk7Uo?DQ{} zZT9?L+3BY2yWFxjlTZAiOW6;0E&B>!(kYwqhnELJ_l3(P7SB4y-8-x4hpwUwo(KHt6FTUnq37lYcGw&A!Q;n`PQ%uFdUZ#2@4=LE|Zz z`)AOc9o*}a99zSBj;1Xk;v~x}&bw!Km*R~6Q~cX`>*Y2Br~SF&oVN#WB(4p7-v>TA zcgJ~S%H^e1)a~t2+rTr2_9}0(v0a!=Uah6;fYZA0yZPpI=1a2obj!DvzF*|#yVB%K zus^@X8H3q;dwYfty5vhTd;0TCz9jns;2ay)1p6a5Zy9k8olUUc?2YC#;QW)jbOC6--e}T1nR|v^?$o8&PqNOW(~xYG!#r)TbkEu@ z&^^dZ`%~uB88`O3(3X>^^O$Zb)6V8wd&dDLg; z!iwP@{#U;}QDH@4=WdOXjqaKuR)OB8xr$KqE_;Z4*!k@3+)X1^AC)`$Aw=RrBdqK?(Frwgk=@6K| zf5CWkd`5HKLmw`W3vDNF6y=MC?P4AFB(L%<^0BXBJdvTc)p8$Mb`>-H{!s0P zho`md`q_;E-=G2ez^|)+%zB0XsvNC5z1#F5`K=S{rYtN+U<=2aHrDl)#>0c<7DRTir&ZO zO*l+>U2vGZACvEG+TEN=`Ha_>#zp0mPjHDUKZX9Jl!Uvt{Ug%1(9YG#l!Jb^%bc2a z-qTr5l*Ts+ICR8a=Cm7J+hN}G<}LXae^EFP8;JK`m^9s5cG3V}-_r){h$#-2mXe-8 zIYYX)`yMkcA7>PWKc)<;zg2Jp^t~CoGRb8ASD`VCuR||V?YXlY*}fJzr&jomKEvJT zZ?)FNhKk&~rIi;sy3C=#b&SFWo;5GoM|W~1r%0YaXS5AFi*4L{-llz(?(TkDSkbXQ zqI>Y*-X`R3s0lOdSQDn(F(%|bhzWUr!-Ofe&xFZ#$4v6mx1$Q7ZHG-Leb4dV#@U^F z9{y|{pMl@hmjiE&59>{iuj{X14l*C|OY&7?BAB+njSI`x_IP9FrMH7ezXqoJn0o1l zJ}wEbCp~h!BhwB;&wft2Hy4peeJ*pjm*%^P%ny(!fb+lCly90KUoV$rX~p~K^?#?4XYJyr2)?Wy$6h2$gI zzv%wzE3OqhEW-!B?f7u?BcteTn;!6uSo>vnUmOcQN>-AskB7Ie z9&W4jaGVLFhkL-&!)=uwZdIAl#cUIfy;bDMYX^x_d!+AN&biVp{0_Yzo8K|R=&6^b zAWKMhEqU`JULNab#C9jwXV-Xw5vBDh!zPM(Lo%>?cuIqN(CZ8Bwl zS)a`tt^Xr^_PPN2Y_&53eRhVAGiKWRZG6DW|7!sMQ+mf#>qGLc*2Wwwx^)NRvzhf! zcrjHUitCbob3b_t&bPn^)Uc=!3qc7acwEHh+ED25FTh<3<3w8b#csBX@3_l zEfxV=kR z&YIY1;M45C1E>9rcV_w^ZPFU)JTFB?Z?35siw@{h%Uvh(O@3^hWY`zb{;ygm(n*)Q zHpRCBXVwYpTJx(0BU&O|P-oeX zcP;zRlr5R2TxQJR0eU?=#!*dIEcNCc%Ar*-rtcP#*g1%>4SQk zb;&t9@LI>c3AGN4eJJZkG(oghd9?36iT|Iv>#ChTIl3gJB^bJ}o3vxLbk6zhE|r74 z=#;^I8_M{bDI?APi(BtMDa)xh&8`rVil5Tll zA@3`0-ZzyOc}RITxq07?$UCW9-d*I~;O6~L;95BQk`G@)n>N71dC$oF&qo&&Uww}J z^_sKi8u9aF`xIznB%NA?&zXhLroi@b?b3}pbSl|CA2{@|TZ_-{MLu*z@b$UK+Rk_+ z+pDNswCV;IXV&s9y4{_pSAPYq8bv?uVBLE-Qb&HBF)T45zJN`bWZz>#^e-lq--hGM z^PVj4(k*=Qed)}4+5eEoFB6YfyHlWBDbTHnhHicRo4T@FIS8$4hgNz1q*-t4tcQDr zh88;O?IQYS&OgOt8=so?d2UR8DRH;i z=L}0353Lp106YSZ7Vj+g8y>wbaEqh&fk&^?o^K$q;?aWr6XojsM|Hr@*Y7G3&++O& z_fH*z-8$xxPxC9@*;#f&*Rtb*(f-=3<^8=wXSufFR}n9~)m~);ye^3!Z>p>}ewCX7 z`5jZO2H{;AYx_^eW--m4N*-r#gMQUr+ZPbm$lCYzmf~|W!1)~sUH6s6W*zXpm%DC? z$>*%obo*D}uCuCjr$z2j9|F#-<(u_(DRU1`+i>Nved?YI9G><|cP{_Qw`f=QH5@#~ z)4Hr-$s|8vZFqC}F8M{H{$N7x7??1@-fF^cy~D&}70g-CbsF2#=P#1;I}KyRa2bGI0Hd<6g}nMUhX`D*y~ zc*YUtTDX5Eebiiga^m;gx^4hge8a&rU-$rLE}b|A&yfB1j@jUt;TLJP=y@cLMe>Wn zPJWSQe*v7qv5G#O{2~(^bNB_i9_o$c7g;W@l@WhDezE#%_{Gx^{Nil#i(iZ}q4>pl z%-;m&5uM}Aap*!y(1S!7pDh1>@`^>@idQ(kEp?yH(-UeQ#V4c(Sphx26h1MKJr=y8 zEFT*FIyC&4p3vE6Ir@;r(>9`e5}p^-dBvlBL;o?>Y2P1-*Lu=^UH1Xs;rBd0k5_nS zY1nERo*}#0@1jo;jeeQD&OSfYzQfgvZYC}WPjU1u$@Uwp0f%24)3-cNoYuoy@I+(u zJb65Q%P@D%h(CDp;0gnuX|Dn9n7+l~7Y_*D(0#!>`>G!s_!Rp-xBSP*=gEI5UCMWS z=r1zx=z4&Q)WH<6PQ?d29n9`r*~b_i%vMha6Zxjhr_>j@KC>ctK&1X9*;}7^{aBxx z({#JwieoyMIkZFjnlUbI`##^|Ti=!kNUyOHocHGDBGa#Ad%6je>OYRlP6Uo3r{8pc#Fj9etr8{h%ehVLd+qUJxBZ=e{nGFB^nldp$OG z{;|$^{mW-t1)|&4XFKQhMT9!9k0v~VEcn`4vPbsY6?Zvj_C;q|WoFL6>y|OrhQr8u z=G#59zoi%NH=e~ETAkUSvT#~~duG4-yPWY%#g3VBN4V$pD;0O1)xMfKL$ND~-?;_`WDc`I8DQ%6#&iB1aN2gToelODZ2+rn9v_|Wu zWz(JoibG%9sJQBTOx*3nm5-y0A@GzqtKXc99-KDuq763&7KB#@j?QX|i?{ks$LaP0un?CN&T8H+f#CEZuedmGS}-kiag2d#pERkt>o{C_yX*n*pl&xYl*1EKnZ_fFfg z&ao|6%o%-iOljCTXXnlq^}03%%U+A_6UsT%@lX)FY%V$0u8%omZswf23E%s)YngR< zpU`&FB^!L7b~S=)H9Eszd7f*7km1@0$Oa+JzMnDbl7AWb3(3#<@s-Ly+A6r0bA9MV z!;YE>;m3I!WGCs4jY033y|Godj^| z!v%~(IW{KuV4DE0EDK!k@H;c6vXh7}4Lddh$MEDp8~BC2NaTE=$Lu;5Ewsg*k2u;A zJk2Utj7`IK|5wcO*nXj>gnvURHv<2z?_Tcmeg^;4$Ni+qj-*Dm5Xcl=`UviQL^vjyRa28b)T9*-KAka zYq7KL#)METbvNc3++UBcf&N+NSo5Uo%ZC2z92oh%eRXcs+de=3q-k$)L1O6WcfbeY zwDvW2Pg2in*3a@NU#RMCYcD<}Lz{PYlvz2}nTz9%t;e$0$`U)*sj>CwwnoopzN)jj z;(<@?$6mz2pTtfaSVFw)NSX&R9tJOD$DeBNf-c-de}X46UW{FH#<+;}y4bOS0On_m zRW>#yi?M4&bd}o8}<=B-plkOvK2W4%+)}#>|64S@(B$HP*E*H>8i})VMaLw3| zw3A2jnM2F5@1$<)64D;98V<}Tg&&{C8T^FszMHrk%DAj!?%>Cfehijb8+(Ulxwace z&Zm6lxZZzB8{@ewAHJOr-`4%ym#`H+I)6I6eHuLeLdw1Xo}$7u8`y*JfL{UsZD2LO>wravVC-qSkFAFThd8_V8E~=z`6?LU$-e>f zb?HZ-U(11soIkG)56|u+T{bP6+wN(?!NsH>FRgw0xbOnfrf~N4Rq47LeY5hHNcRv8 z4)y>SdxDeb9p(qPHxOlX4+qdaXir!U4s-h^bmbUl4|OkdZFCRV&>xs#^yXWoH!t8@ z`iI*|_w*0LT>ai1#ATzO@bnK0k;NUoIlg^ez4>)UK2NbPg$8P$bRBs-oy3Pz9NmJ> z0vz3fW3%DO^xp+m=f}IHm*)QaL1Z>`3+v3j1>M5Icq>8|oo2sI9mu^~kEe@{)Gd4r z+*P8@+5J0p3q6qYJl(=s)ElW=2%P1N(K*C>x&!cy^cIV-OB(@`X`50C&F7&><6Ju+S9ksu)=1p z5)(qrv#pTiuYAT2-OIF}awjJ1NS}G7=rSJXoFeDyp$6tA%Ec%AdS^Zp?S;TfKem&- z^-|JQuIyeeuWCi65Y?1P6@st z-hbS5^yKN3W%$N%^5^h&H{*+cf^m#_31eT(929{EubTTh>Z8}DNtE+2{@{f_$E&OH zXh+$HTK}7Mj@E-RdQw&lWpY<-YAo}KEm@6dfbN1j_aMD@+2m_JZ1rr_8UIe&oX7fcbAFd>Ukt3H7f!Z&&vx?9 zAg&pG&oLc;qJ1T_Md!8BrF*n!B5^uv8U_BS?Gwr8>G)rUCUn>F4>j-^_L;!#gg!fZ zc}Hh+ib+eewWd{;U>v%ZX8%W*vK$?Mx?6Xy+KTLz?B>hk+tKUr{#TcLj$WsiTThb7 zhfgCnpYX~#2S~9W=#tMl2l#?G)e{4p=%=BZI@5D>KkLky9(HPZjJas*F&$8*eFJr9 z+&wz_w}>;n$a8=Pfir9Kq(sI!*_;DB$9~!wQ+z#4a&$l&-Fg0mZ>_y*^n!*Cb+cFN zPCunrzZabL=KBxi*ZSOMLiGP8l>UDbI6e_vcjUT5*G@q88XsmYbn0pUm9$jt^z{Fi zeycUu1A6M5XM1|uyGA*@?_e#s1ER{!p(ap z-yYuX0}q_FBmJdw$LLD8u4{p7@u$GUd4FM$Km0km7jSWzr+fUooBh{&ln-vK^KdWn zd>~T)cmO!zg7l6%VuI~S;GIM7Gwey!8Ogs>UHtnF@e%xcYxlEcaAa#F|CUZqvZ2Sn z3&=0}dxiY4v}q0yPIi^RdtCmxdpe~#UqaOnVX!a+|K`~sZV(PTjvocEM< z-n5%IolpIdI+SNOc|2M0Di?>emLg@r-$Q9oC)ab9lzepKUe5jbQD^5(JkB&XV&B!dC z3>l{0NF2P)UEe<^{=b5QlG7q_@B#7*2SXSgufHru{;%L*q#S8*@Ehe1*&RB2 z5Xl{p_PIKbVsE|PoJSd%Bho&1@wCLt9bCJNzI!qUaz>~Bh*JH3Fv-XqMf{h{v9HvT zISif$l0ySsJfGyw1N*UQHz8Y;@iXT|GmOl!PBO>E;InXU46uUJ|4*^q67HTwT#z#) z;jer~6@zb%ym8ECXsE%#R6E1P*`efdcw?&lGw??=bF3*>c0-O`L0^+6!%hR%kvSau z)^Whax;P!@|BsX( zNrNN#|IE%a+I0It;LI9WmhA9<*7|c7fANsR7kKlGvDX>UMt96V;X8u=chXU_1Iw>@Fq=TRL zGd5Zyjo>hMO7ag(59A-55u3l9GHy0_&AS1>c(TMDK1%IHu$AAaSM4pdHNsV#xeYs-GGDN-!(XyVb20qxLM}r{a$C@RQqpk zUWcALc5&MOsIKY2NuCJbotz=L^$0qF*lgZ5^hNIt-~(Q7J5}?}ZL@zNZ%KgDxegC{ znR+5|I#RBPr0dPVMKPAenZjw&b?)C@^u@owG4 zP&umSMDmNSXP7Yc|2R%Z$`-d9*+S=;eK^nLjofSdvA67xyq$pDoyguY3E#dwLV1~w zPl+{efX5(T=&q9R)ba1TADKh%w+ScBJh*3*Q+jZ&hCY4&AA5xNaYnfQvS7n0Q>^=T&BAPay7NxV5p)W*9d|SZvaJoZeKPBGfBUDLNfB?} zQM)+Hq2Iq^Oog9so`>FjfFmc1mcK~6+9SMrk+eUZNO|~6m*0;nYVHH zTMO*Y2w3VC>~$_|x$2+|26ij3{;0RrZ{u%*_6v5F3wx1ZX}5t*O%1Khjew<(J2~UH z*oB=WSo-R;7ub>rSjJ%|XC0GW*h0awZ!xex0Csi+EMwyj;#b#&EdW+~DnpOgrG|PI zIP~~5;^XlzHPX_d(cQWICdvYkp^{w?|@=5=8bpA!q zsxoNSbZFN!XxN452rl4!W-9kro%0#Z&q~UZKO{$va{OC7ZQ7J8>DkbegGg>U7`15MKZ11@e~O`P!Rd5tG&9~`%w z1p5`r8E?u-u&1aT=%32j<(4xovK-QW`xWKvqMQ+KIb+GA@rWv2XncNtNc`tb0q1@v zb}~zJzw=M%(6YGSS%Peoi{Jgt(lO%G$*dzs9+FPtMAqZ>r-BX05?hP=rU8?|{WwoI zmTu>mx)SUxg;Viq&Rk`hIPUx?Wc*VVGB?QznYTof&Ref0OtAYZghn42Pe}RaiBI!} z-|pOg@aeC9$Is)_2az$`;M1}fIs~7VZs0&yKE2HIk9W17vqkjn2WN9PnKPvZ`1BZ` zRq$*0H2l`^X`6HKyejeO56l~L)#%4ro1ORJS({s}o6|b|-aSRToqH>~k8=F??{?>2 z$oZ-B2Htx7-=!?bT-ky@zEb1kGtqv9F%TVlm$Fvl|IU%q67AQBcjyyu1(NpX!yW78 zm$(Xlp0n|Tr#ai;=KT%njfcFv>Glf|d0!yyaC^skL1q<#O3_v#wNXMvY7A2{T&7S#LK}G!@I$o7VN3&#Z$1gGCUpm%J6h} znRxn{O}8}Vy?>jLck(3nz{@xOttfmAZ5Pk&#@DwVD44o}_sWG7#DVOg(&t#n~$3YNB;ay$9D3k$8$*?zeTTP9fg zYhZ&OUw2`lC)!hAU)O6KS!$!WDXPm zN_r^HjXy*2$QLHQEF%={#-B>OaK7nu_I!*<@@epW`nzQ)A$_S;NIOdjN9(`l^Hf6G zT~AoeZ(j{>YcN+0Hx0=T=bk!r^>f&pOmo_kNcrNEyurC%^zao|XL17Z(wB+;r$7sz zSR)yRcUN4wAc{EcLzY9MG?!7pJL`w{A)sxJT;R<4NwohX;v^URbDY9-`(8KSe!iW2 z>Gt*A@{J?kdv3n%CSQhqyPNN$&U_j6OgCQ}?Nb|tXIGQ=bvN%e!Lp`Qe}$X>UExVR|74AKKcB~sQz_56`?SH7mty}Xc^y7VS|7)szvOj?j}pf`y+Rpb#Thx1CsX~98kLw%pI6(419w9ebcUVyN8?arp|o43DYg# zINCeI&4+C#?a8nYy108yXTA(u@?KFBzvDu`tF2G~S+_%w>BOk{(a@$OcI-UNvOqZVX z*m@Xg`p&r}Wj%WtK08Y;+GympKV-08lI`jXy?eT;_6F)z+;eJk4StWI32oH>ABtx^ zJ);oX`m}*dwZjTo`%fu^EDC)5ZUh@!v8#q?`FzvMUIo7&hCDk zWrv|RL!C1Qd~1?tRg~50L(VEgZ@xM?INC>?q5u5KikCv8^vtCwbP5*ju7^Pu||o{0xJROD7q7xT;D1GqU{F-dNv~75lME2;i$y`aE>S zWx_8XWxqk+8}ZXA{3@j^i?Z6VZwP=NOL)Ijd_ng|vwW$ovg3%my`pV0-&ORvgmNYy z9oJO(;H2MH@&-rcJ(CHOO?aIzcIowZUe{I!>@yu58|L15YTG}UuN~B zFg{e(f6-C4iZqM%pGw;VSM82v-fn;UZr(dAzcatB9(z;GN#6B6 z+Z}v3etnVey4-|JcN2af%CW26^qZp(Wh48~;y>Xst1&6`DE(KO_eRTo#a5#CU$`T? zH13|TFY&YYKE8b#veNpCXxDT1PiPk%zl8r!Mp+~CW=v?W!2fpSUc%H{F(8z~o2vuC z&*jw5*ljzWP1TVbO4#QYUoYFLXKB}_Xlvdq=Db#WlkKO4;=s$@;E8Npg1*9c3-Nog z=151`LiQ8#quY$nq=2uw!QioPa44^Lpe}IDc{)Sc@IG^2#NI-A{Ol|6eg1}gp4T7J zTLO-sap^&&-`D<1w);2G*1%oC2X^}dZybt>U3xGo{f&;O#HSZAS9_uo>$C=9!G$mN z_kt|*uJ|j|8Suq9Z=HgVOZ@3?WRWKk{r5~?;_>sInr+@4-#}jR7x8S-j@O-f*2!mC z0^?A-wB)hXjQ{sYw`xlsqyN`~m!IZj=RgzLuhMp%N8XOFblLK30G4_C)cSj$(3$F2 zHft0A?YsQ7D~kOY=FMo`qv!#wZUd0>~m!A|8?|vM5T&;T& z&8LO1Bct5HcUeOX`Le5X$K7Jw+bK`H{3!LE#d=_zc*n4){m^vIcW*NGVw9O1IvKto zS#~OLq669Bkz|)mv_tU|i5E@D?qO|^zcH1gaghB|u&(5>g}*3ytU1Q*f9=3fbCKgu zb^vvnb~oo@Pi5IR(eBZ-F?G!uAag7Q*9T>jq&w&`LJ!uH< z4T0;5hfcKSGY5I#u=<%dZ}1qsyX2IAQb_H)j57WhB|es5C(vfmlnJyqYpicxJ!fZ; z>ojnRaC&4Ywm))3c6g=MWEFXPkgvQa|9e{PQTiXp|2TH9^r>p8)lil0yTjUL-B7-0 z;n?y;i}=;>Tg>kse)o>8I@ju4+gi`6JAjV613mAN8+@$kDD-N6^lQA|fWPdc^SSd{ zLF^2`4h+EGpWR?)sc>p$&VMf?89f7$J6 z-d26#3$IGH+Vi5ZEyD&f)>=8OWng%v`uYK3?TF$rwfF_BE#YlF@tp0&jBjjfOQKb_ z6F=~}dy6m4!Fh~xplEO?))yQz(Ql0k%qs%!vc21(2_|jc&`|pLX&W=%E^1GEr^vzY ztMfxQ40Xy)Utbv!E~4gd3CFMhVZ$lg<6CWr3m+ z-ma97UH0(Mx6U~)8ZzcEy0!A36t&%gEHJAs7@mzhv;$lB@|a-wD)uD${;T(mtTr3j z1xZ>CQPxfA}qt6=>X&KAnQf@pK{`a6AywDm-7EX)`sHsUwH8#cu}VKcqd-} zg(DAwBcD^>cl2HJ+SqFFm-jK-d5@sInzC2Y4HyA3y7va`sV!^T#*LMaMO%hjroL{2qoL4ZdjdvAB!kow$FS z&_2nR)VgQH(iMaMt!MiPa6@Z)@ax3Gs|UZ{vpv8(#a=X_UA#N?qMod&39Og#tebJH zpAyzlF?wm<$3U;Bbyhp7c+776_UEH}3-GJ03pT_qh@W0K#+nyEAH~JaPbbEn-!O!@ zD&i{8Hx21w^_v{qt6}ab*1Ac1^FyU~xO`7Xfmvt1^iUo&6I!%B|LtJ=9KyKM zdGG4dxQro}%-z^ZyHBN^KYDd?!;koX9pkvl*SnH?$Ysy)|Ab4f-}n~o@8B-YBG%Cf zmz=tB$R($3EWBj!MzbFK4GZ=5Sq*`ZS+maA`KfLWYc6 z`deV+_wmf(OIN&$+`9>WvxxUQEo5!ofm#HASw*?oyyqmIW!iK4(9lG+51#CeNwO;g z%n|Qn{r!>jX4VSlf{q-}hK)7lok)D-yJ|+hg5C>fH7>C)PQVXR?9;i-XDo7S?2E$n zWACs%J~MoA!BQy@mF7-xo=Cm`;19s;$Tcx3R9BJrZyFK!e2l6+e#jTcBCf z4c*e4TnTm#ag7!-rrNxlb&0?I-6NK)h~2IBcHI+MjG<|V%R81NALAW4(D)9&@9hIK ze5_~Q-IdXqO`<*OCX`JxR2zpG=d@{oF!l>@TM5ccaonjnH;4Z8m9|x5jPw3g-^M z&HN(yq2_CQpM*0m1}=c?7_UAMir&q9HNK%2yniTiz%#?o2Nw-`%#e`rQ3u|sU+QBS;j!`N z{Tg4-_QJP&wh#Vfyff~f^yo6~6}+Dz9Oy@XzBK-`hIFB01(faKYyz}QW1zaSkTwz3GCC>)s#zE#4L z=7N`%@T2RvH{KgOH~~3v7&!A!WXHbXk7UDU@Z`cf=4{NqJAQiBXlq_AWt87BZ=>+3 zP9R?`QIq3)MuTkv1Sn`ink9wxx~T4ll|@1g7T&u z@a6>4zJz;^5AT9|51wl+S2^m7a8L7mJbddYiGI6vNbL2ybK}qr#3TRoLJs1c%NZwt zn|+|8eYuAe9g=_74>{M-+cNTfAYQwDnAM>DQKJ=X|C?pC|BgD#-966h`tR;>UgLi& zYp&|(!Z7+BM=pRSBmXzVV zp1si-!8>)|2wM;5Z7TR{b+>)5&W;i&|5kAKVtgK?gTE=@YytSYJnHmNCb%s9NUi*{ zz0R3*A8eU9hcdYHtV`F~8|pi0_vD5RJ*R^+Zw>APPmi&x`~RLb z#yt1$%MaR=2rt`el@1DxppMn(N6h!9gF?gfooeKut=6M?p&@){p(ps9aiEP`1J4c# zB^|XI46lO6Y&g_0GQfWHgD;M?sRsRuc!BDZow(|MP(Imnt#;?_68TEweG9jrZ=Pc< z-!tgE>z(qQG0{HFd^0A0N51*Q?8kUl$8U|)+Vk4)@78`~)8(u2+iB8i|6#t>e$As- z4*0P(pMK=gkHNHWF#Xuaz9$cvWeDR~`BGY_;^bg>Pp<9`$JGV@H0#PZYs+j~@yv zaRYdIsA)?rc;)tXAGS7!pfHezg}5#dhwXtFFFcJSX(;d?Fm&8i_!vTGR#p&&p=K8I5v#rxZ(T&y9a!**ha`tz>x3S9VTlq3PVCt#C z_D@;++o1&?`}^6i!;^jNfr^QrY4vG+g1yf_p$i9z-{a37jE+oiyNHg@JIp-2^N)^^ z9np*rWfc-Gg!WfNWenc3Jx21<){1J^lJOrKT&W`n~qk;|K-qxH?3qr%WtJ1im zW8_}m_tM=G2S2;vKj}gG0lrVGAoOs0pT8HhueY5{o3w_Td2`mtO5m~dpt<~HTeOSs zhwxkB<%Y4Ys({AV6V8FY&mqjRVq5nxzvAZ;lL+B-Kf?H&(NzdRl?lK-OjE$oxC7NFmUk1QmQ z=0|tfJ8GiQgY|;`_l6Ig06*vhUx0Vb?}u${wDtj810PQ2u06g9%pFYUt+aivURb&* z?H#wn&kP;uajN#zW44qa*S}a03P5KoN{rmTHMYn34Y4IwK>#^hyeJQTbuxSXCwIk$ z3UjP^S}V}*ZJZZOTgu#tkCa4NiG@DjyhRadm841E5u0=8{W;OrR6qX*OJ{IRTquhF zt?0d@348EM;MbR5GQWQOdh$!*m&h-MUlPCm{8IT{dQH_cm;U6eDVIGJ7!e%cdG}tv z=e_*%F{V5RzVE$)b@Ku{MrHow>?t?h+c4!0%6QV0)%OOKy&Tv($vXyk!5swV68As# z8<>?p89U`Q@&I?ka3|fAnfgOh9{q6Yn&#AN@@(M0SLVly8>Vp3wl@bFH0$0qQ#Rkb zcFO9-tEWWW`^1zT{Ic&|J*9^Dmli)hzHT6MH~XI8k=gg< zo-liDVA$1b`wm;2IdIDCwS8}xO=wYe&f>LGCgoVqzQo=qn|dSbVcnJmC@XBm)(uGt zj;>c_!bbCtLljv`KfC4719%&$Jr$zr}t0R(}}33Dd%lJTh&`BMq)! zrrAyHYw&wmXtDpd_O4BLWgXr*`+Sh(9q#gD%X2ewO&M}c5^_y4a!m^N+7BYvq$Afz zm)-~5lrN&f$s&2vutyet1^3$SUuI$I2VK}_C-PqXK^V zc+ImG*l@h@WaYTW`K{hKeka{Fz;hFo~)cJ!OFKiLcq&4q_nS$!&3BO4XFJk;07 znHYSu7(V(^?BgqQzI(&QLaTS>PI%=kcx_+$k`)!-TA8zK#Z3c4>5u0ABz=DFkJEn# z9jl75hFkDObfv?g>1FJ{hD&ykordho#UD6hH+Mif@Ig~M^Oq|ta-Uxr7+yRkuctM# zLi;H6kQz75&H2#1`yM|r)bd$JS$*kEqw7DzmQA$KcYZ?!|3#;?FV%ia^t*9yN7>TY z*t*FJ&uf?;6i@i9H77vb~#Y3RQl`bv*J7>559a?ni{qii z%%h>jg)S}DTnvt&#j~Krb-uVaes1+Lw7BM5(BjH&wAiahw3xlY8@3f!cRsXu)rr=P zl|AaxSs#WLSHbV+!0#usCvAZzgpnx*qeGA9hhDhyD)6lS!}tcymhBdQ=3wc|CV}^* zyjwh%v-(M#Zw8@p){>IPcC+u~u0nfrjCIt=rl+47dg?6XMDSdErnx5indaQ;b3XD% zg|fkO$L~`T@7H0^gM8T!8hfB5*j~+9@cOrk!W)q5r5pZ)GMbUq4j@l`%>TojEgd{h zX}tf#cMutBhU(>i+hoP3+WMbO{Al9;%>S0BCWevm+L!QanSty>SV4LV>$#M$o-mIv z%V+J~$J^36GnJg9HLUR9&Qk2*lI`D6mg;@ggsJvs6DDA9d?E7L1;}Yrk=Ldmw|$3w z`}twyII~}uUbdNjAK@)uox3KJ55KR-cu~L}-?y*iI_W8NKTrPZ53pC!dB5(v>Q08} zUMA-Szh>^3=dIS&BSUj27g^fqJ}uhKx!Z?yVD<LV-DRH=^rXR|$Wa>x-)spFqa1e1db2CA_~o3AjnXu{Qcmo|C_k z?{$-ao#>0L%ZI+8C^PMzFRm$)7-_<4%XwkiJZ+H8<)I|Ptni9qyBld`{8M%2lhI;Db6m@OD#!{ znWY7ejm{(22#7YMw+0-oAmyj1rgtNd1L;6e66 zl$+1KVTnI>viN5#d`abJ`j(`B@M%YZ+NCw5c5*g4@6jsX^jvtXXvmzZEykWd2VQ(t zRbXW>G=cp#daTsop=$W?4(JFn!M1+lw|(&82@aR)T+OwmXn`(R&+0tMr8B>?LO&WK zSoC&;s(S}xa252axO`9upL)wpd(qS8y~)`EvO(UP?9ITh3S_-J&H|OUiM+AGFJyr1 zJLb3PES#Qj;}&g0o3&erT9KI`d9Y%=1H+FL~9Wgj(* zKB_&XjMZ-54Lln>ny9e@Z}NQ^b$JV88pbY+mn{%}9Wp|NbI#rv@MXrA_UP5{bL#v% zeLaSs^EX~ezhomJ+o$8%N9cWQ+3PhEt|CABfNjZnW}mmtVw_hGveujPbmAN6i_T2s zn?ZYi*+l$-F*p5_K65W8q`qV4HCm5>-t4o1t)uK`p#e@^_{WiaL|NNQETa>Q(s)5H zn`4TuM=y9~0KHTTdZ{R*m%3^Id-N;o(M#2%muf*T6+kbgv^=Xg%a!vF9e_qvLrb=! zS1YBw)!6xX=v@G`g(rIbmZ^c?iWAlSy z$%b}R+|!eotJr)id@FQp1LY-K=>9b~ItQSyL!J#bJk4E4$zE09_#AMY{p_+yAM%!+ zWH4wN^8U77;5hsvF%m}`&3UJbpIdyP_ermkjKK(8Ll$bj*0u;;0pto=AL z;C(&r>y!akLGPC!1D@~7fMyN6G%_@JICoZ$UfDEr;jGX0uomHoZ{&k-@Ql6ij6>O! zJL#UQnkL7-zJfK-zslFE^}6Brt*9!>YaKXz(xKW=ZF&o3s=UF-mFS5toJn0Jk`It6 zWe*@*0vmov8@Hmd3yonNWFvQYYbk~L>RC%ar5q!FkuI4G_;ptH*buU2dyB^h*7>m| z;B5k6MEA57F4MY#u3ch6`8VlmA7ktW(C@QXPL#i!-MN0wjiQmsdLYO3M4pR5uEXEq ze0-i^@5$l!x|XQ(8}g7*wbxt^edauR-Mdem7>e9?O2_g>Tx{KS$OUm$FSGBgM4qVh z^@`83;yd@9yBWh&cuwlh{7}MWbK0+x3tY{^*Ck@WWy^Ksak(h9kQKy3tcwXs^?Q+HD_Z!YpCo;`X$>qI_7J2qp#KGg?4 zViQthh0MAMTA@mf9k>&~Ut&GF@DA3+_FRi|PTsV`zRsJkEbLKMGuQC)G3<5bMF)b1 zbUu-7HFBQn*r!m3*|P+GeqB@Sk?WcYS?`u_)t|eke~DMRdWz%nO6u_V9{k$jg%NR{ zzT*G!s+=kR8@l8F$E*H#d6l_yVB~J!^SpvG zhUe9f4&?-#I|`L&Lc@s9V&63`BY5Z}evh-JxL4Ya{$*Xxb!Qn{rfh7_rIYE5UpFRH zMBa^GT-l`jYNNG#!cpY(y8|zOyv^sH9XC6CbzRPpvqB?*e+SrFc<}><7bn=eh;w=I zGV$UBf65yX{CHW;;lR zLrQo%ajCNhg|x@jILfwp9_^GpxBhE>w?qZUynbVWx$A^XG-^{+g1Oh@tLaIkNas<+wrAIYeSL6ZgW})ywLSj<@g~nRvVTD@0j_5ewh(6eV&@7z z0DNjfZMyO#AL@8Pwk}tv2M?|0+~MK&j!~~43U1RqtvtRD(zosK!$x3dk92JOnq!Ku z53tY7=XVGh;}Ps(!K<=aQR#Ih{$%cw$o@Rd-Uoc6)%JXW$%pM%KYa9}htQk^tn|9< zFFH1OWuFg>%GI7$d)S=Oa|Rr5A1hhG+w=O(dYU~To4qYMk{bixgBDbv=kHg$BKF$q zX>oJ7i`s9=3NtUpR!(qcNcg&Uj3#V$50Vym>geVLy6}oXS-;m-aZ{cY{+s!VX&h#g?YOWg5BSzwBHN2<@99;yicE{64 zo$)a5ZGLMU+WhtYsii~L8%MiRqQiRc)qC$!d!Y6B$9|4=puNW*G+xN(+f0~bZza@Q z1sz}Xn1@|mSymx;qX-|skJ8QjF5)M@hrb5LTKr?}LiFR{SZ81TtVzR9tdm2JyS{^} z+%j85SEA9&N{7f^VahgZ*R?6i;d?jb zDenKH?aafYDzb)uyE{vF0$Ir3iCG9sIzic&0Fi_c2q29t3W6gbI--#vGAbl$5)uK8 z3$2bQI1?5b>FY3}f)aIxO&Qb~Wlq{`e(xXkoO`S4 z)~)5#Ij2sYIwdwmpYpslOzSJ(%XdpypGp}c&F@DVwtGwc_D0KwafcJSJY(LTKOZOl zx0ny22V2_K{^)&hPs*iD8|yhM1kDlqqYY`#w(~@e{c)M3;Ty1zsz35U+ke|b4Butq zf6MFH^SApKMtYc+e`}v=O zyt{r3_KM%bGR|IXW3Ql$yXq={Qm-rdhm>KLJu}W7NWW&X|8SJGwdfB*IbTW^|bkR`T4tPH?Gnz zJioNm-t)rqOS77S`?safcILD_!MIab$<>@+>PLJ1?T|i{b4;97k88~OD|XhF*?wKb ztfV#Wkx8rFnfPrCMVF%b+iyDzuBdnHWY=N)oCY4MF~+ucFy*f_?EjW(*!B*PcXTr< z9oyJGy!-9{6kWQ=CUzHiA#JtKNb+b8kMC1EYa1=x z1J#>;n$_X<>^?xoy1&>Fi!UsL2dOhU8??;Pvs(AT7-@%++* z&Lv^(YBMNzR)=ocLK|g|Q)B_z6L}`=s#?MM)EI4|*psXNWnR(tUuB1$!hS@4ls54h z%4bhGb4Uc|6&NS_VXx4Kxr|Y2cz}DzFvH%&ACI0m+8&$8!4vTWfOt>tsr8jWmHF+&hR^^Vl=8?RX zO~5044i@i}5Wm>nwco+w2tu%PvHK3-I6@ESQuj;1$K~DWad$}S;@&wqwRrfVKNWXK z>gqni^N2;y^W4K7F(tM5>O~uP?&_Punxf5{6DP zv@uQWph&L}TRcJi zQ!I6GMQShQiA^4La8d^c{|dn`hQsCHfA;r6X5HF6s%5<53z_hRDHdm_i+?~@PRM@j zG@GT!%Ray0tJoTd&4H|83(HFOX=4VyHwioyIXt)Q{(Td9#>Q3c+irtK_2pf7N^b?l z*R7nhmU|C{R{qo{#9MQI6nodbtdGYx*b*``IeQpNeZOv9JpLQ_w|pnxw+G+;;(lR8 zc|h*^t%2J5a{Jbtt&Z}X$KrE+A0!~JOzyjRQ;-%(8$9BFL!-|F`i8D}6dUsoXlwT$ zTQ_pPVDHtIudTcV@|Ud~487cusA~eM7u<`aC|^u)&c%ix}jAS>%!NGUiWArs%yTALqtS zXpG5K1-Y9a9_L-rp7}k7Gln%iwc7TUY0Zzqul~tC(IfCHtzz2QiWqd%sWXxy#DeKS3;kR=~==!K!-{ru1 z+vR#qU}57(@JjuMy?>FZk&ljRZDeYl_Pm8JuJ4iYj()7wV#n6!0QV+sr41kUVE=KZVOKDVvrvA!0y)oQmH7xy6I-^_#DzJjDGPJz zjs(w1jp8|OVa~BzupQ>ENa>H(dE9{{Yl>1JXI2_kXtmya+a8_us@TP3EXYtcy(t!F zBQ$N|dtG)ptI^#_A7$Zd=Y^74n;TO(ugI9LPL;hmZ6kZEYA@_zMd#Wz<|0e4FS&j5 z7WTblt?&qbgdQ!K)w~6Ibbz`4&4SyTzg=Lw&*1&rQs@M6@0)SkunnGFdVBM1`tYA! z{^1G5e7(fsPx?$(&3Br$ zaC5%q`(cDu+d3!E*E*KaJH*;jSKX(ehaqcmBGgMJptQd2{x-QbrNT3Cgk~Ar}93* z;@iys>L&3%$>Q52&r^-GmsNd3JgqayH;r(Mh4nRUv)GznkbHBdyRarS$q}p{07f&SbX=(H#^9` zh46R(H)HuGy!jr<_W}7nu=su_&--|$&*sWEpOOD#!WZT@9_W-u^W7%-j*#z&#dnK5 zH}Xs$-z49hB>yo&llhI8Z@ilCH-z}ZVD8HNxAS~kX-4zto}49S9*^z`y))_MO6c*s z(1C}b1Ghm3MnMNM7tGo`&EhP=mTlr~rMER_qKlR?{5%mojIVK;=2dI8!g$Yn-Hp3N z)>TewhWBl7xz67!nm0#u5(zr|n>*qzZHFoQA&ptP)OzUEeXV^(-|-D|UGyX`az4qy zeEJGptE24)@;NsE&k&#Kaz5-!ygheJ0Zf8anc2vwmDAdJOpA-bPzo zrt}^5qM0Gs8y;Q8T+34Zu&2IzkXLZoK11`=PsK*pe;ySd$-;Zv(e{Q3eB0aLVOqAK zFR7oJ>$7h&zS-XqSfgy!BI|3k+J*`6Ug&NV=dx={=53boY*;2d(q7cmm2h0scSv7Tw7qgKPm6fPW8_CFO_%d(9qwg!^xSNZzwce>X2A*L-QH*JgPp( zW=s6f71NqCcox1{%NW&HW_q++?`}@KwO{kg^3Jz$Y;rOV4;J0Li;j!MvjS=ahav~okQ`vcvTj3F3sS1 za;8u0=o}`$lkt|i#^X~%Xs#XDP?^CVU2OGH{xu5N$iIq!U-O@b<-NDn+mMA%*>ufY z2aX&7=MI7s;&catVMZ5j6{l@_meRB7@0 z2+~BxZt9?I{EWapuwD)no+$qm{4ZqvQ9qOYUF3&`N$d$j%ksHTrh&0ibdCL&hV-kv zcJ!3z(k0HH&fbg6kp{|WneM|t#k}tJ}T!gE54x~g~;-DTSE0n2ewAYDGw7b{6Dv%VsLK7!of$G)1wyL z(fknks>$Oby_~e8)O$4X<&uZGqJQ&dke*6fDsgJgeBOEvH%I>uakGm2ZE>@lcktqz zoO&x?u>q5@u1m)@OJGWYZ-l_a0$+i^xB_3kz^DRWp1`pM zz99lf7x;1njwtXA5}04$%N97Kz}H{kpaNfCf&B}7y#@9z@bwgkeA``MQGxGrfwvU+ zvINd7@VNv|EAVv{IJLmnS>U7sU%J2v1->*O`iHoCIScNPy=UYOS+5BFJ`2sS?>fQN z+o46amPL9q@Noc5RcFsLzH8km{EIV-du6Vuxo8PouUP^JHiC8iX=t)$v8?~0((XAc zGO14 za>jwVXobHXwq5wmi{>|yN9fenlG5gXl?-ZbkZ+i$>YGE@dHVmud|hpRBY7~S(98MH zGyLas-dhd-7|J#VJNJq8S=`Zq?L;Ov{SDmHA-4VXWx^vlQ_XW}FtS6I@b$#T1Dw6> zHW0f&`by)R{wCTXejY!dTxpM*v)C&L{l+eFBj<$FJ>_Sh!O~AZP^J%Bei}MkPajI% z>-o1$!Awtpf?Yjui2Z_BG=4Z1_V4a~FNa?u-SQ=$GU4$7q`aypc81 z!I=U0`VR1jU#q;++#A4o%s}WtEA6cd&+}E0w=Oi#w@_d(^IxEi`7f|F*LSbLGr7JJ zflax-`2tVo`tBCko%t^?i}^3GEAwAqI`dy(3iDrJBJ*Eh9P?jb6!Tx;SmwXL(ae8= zBbfgJ^O^qwhcN#I4r2Zb?9coc*gMa6El}pfN%)HJWBaGU{q99pUC(`sGH+x)D48%& z>s$XAI;ZMf&JD;NoBh;zh!Bf$b|BND`O?s}rE#_(4LzHZ2Ud0V9L-1O9+O#}DRKdM z#ZRZm25EVDEhmwset>^BAxnuLg48@qOEdW5#}nn>VNq`Lc7n{Iry0vSo~?_G_a}Kz z<$0*QGuBIa2WL;n`u=m;a4&5TALJn>4Uql55azYSNt8mr$r_YDy zZeCMz@8+tKdz*!p1TMI@`N$f#Hwaiyf89z5p^vf%R}p3tzJAT>J^CX5dtUR_(|71p zSicl{g8#Tp>TImx-bK%Heht^YX#39AiSI5j)^NQIUk^&3BeJ7>yA?gxgM9lS>%VvS zb|mXR7x`ojCSzCsR%>68Mb^F5+Q-T`+Zbc%$JEQb?xC0I?j%fiu@8rC@=ru)m zE^DSxLI^?Tw$xv2XvEK%=%PetCwLU0=Fb<%2(}^=*8|P2px+}`^YKzPckP`wAkBLp{tw) zZd+yy1s9|BYxoX5dOyh{vaytTJ#l{DezKMg5E=m-L%igxqb#cyn<>x#NOp`iEowx8Vbxu_%-})^gl~Xz>KF|93TM<@CRdOMR%UVU79W&m(0@zTxDPJaV>!I|}y-4%PoZ8agAGfmvTMNJ zSlU+SfnH?d#b#z*ZBHp*+$40)_X6O0-z`N|FVW+WS=yaFI z2X0B*n`pbNkEK1r|CPMKS-n>1&!K#aM|@KU5ifoJ1G0(G)peA04KjxsD}2k=;agV9 zT21`He8y*{+#`iPp1waq`CAzyky+}axChEVMi=G#H~A#bbjApo-Wa2B8KVV^QH(D5 zl+75mS`&;hqAX*K;>|IN((mB^GDc$an#LGQ9ZHCk?;qiNfl4L@-a~u`@~Hl0eId9b zZJS`?+g+rK3@By(sLJ3>gYcj4GZc@AP;v44jSAxPSHW2Q1_h(_>lKXBrvNX78)@K% z)TgC%JYznNvA-G|7|YooH~w-1JOi_|=)YwmOHl_6`U3CVm`6~u0Q>Bj*so-Lglzsv zft-n&%c7W~X<9)%0j2Y77MLFC0c_99wJx2==Cwv--*FTxbNHa8Fb zkzMfYx*)MjS?c;E&?D&?>=&!|>w@ts&)LIIf;eyd;BxhRo!t{kT+t`ND(`Lh@E;Pb z%^hW_Sw4ile3!hp!;XObMi~KS88(&wd^v$;IRV(7@(uM-Wnhn>%HUnw_TX}_ddYkjg?Xeq%*4xu^v+WC<9hP7Vae)tW>DTt{)m#?e!z&q6 z)gOD>dt^*fEl(_Od$)jE<8ElD&cOKFdolmAYZkfZVWW-zitoT__=sA^xcl!UeTaYf zee&!0c~JJ|cZnzn!1D3C#K(Eslx%Fc=2lQ=`>F(Uk1m{yxkVpT~6>9}G>c~x zG9)~7Vz<29+K2HC+YS0wV~K4PJBx1Yu zII)U-x^6K%r{$i=}SlP6W5MC$?OzwLXvDVkUI)Nj8NJm?rXs>D7w06jT6 zg=N2e`Btl=+S+sE>6444&!Tmo%Rgs+xzEIpoAi&r??iU0BTnage}5d}c^5b+c5e>; z9b)319o%EgPt-x1Lep|ViqdDw z_sOhNliB~{S=I#gL$zA?^#&K?#Cp7TG_HRSr*m9HGndEDx(2F;h{Gxp=M ze40;qd34MY=f99+&dzxqd4Ro)X;$#s2A&5X8*(27a#?GB@E*D6Qfy9q&@J(|CwP*_ zzn?d^3!Ew~;TEd8@`8+a36<*gA~w4L%5;l=jIw zQea(TuJ3Gyf$OIgjMM2a_8rgrY;T^%J*Btg;U{OOnr}+3LWam-oH%=aPKHQ9hFF3O zVOKIlVM&I`=eNP1Ty124Lf&UlKUEGmnQ4D?Z^)zTzth{j^_}kh>N~N45&46?jZDsA z^i{H1KyS~p)KBU1;Bm4x8IrEe-NxR=>*7b+YM(oua<^EmbH$%@eVCy;)f{qJKJ+yI7kpTc47&P|ZIA35`S0j^nmGA?yuyzGE3C-jDxH6JVjRoObo*@5 zI-gG)M%se5Y1G^Q-A$z3**2|BnN_3}ktTVq5yt&$DfE@t52a%BC}%3_Y*ohG=DxY| zu<{(QEvot8A;$)nC3>^%fN2GWOe zjzI1`k$EI_m>^?@|DY-0t&DwhoaXaQ3U)Qy5;9W*wD7ugt;NY*rbpR-n_8B%VhQ!E zZw&CBITq;kO=|C|8-#AGF_?Ed?@_M03GyDod#JZ{v8L=_>M9?@=HyYrBZP+sH-)2L z^pCmAwIXiMFt4v?AniI0R5T$j*Vn|e4;@uTnRS9gmFLi9Z*Vp38%22|DQ}ut zp8UU_vF0pQV`1;|oY`SVs0k5gv#y&sQfgV%3XMl??k z=l)FkR@;~+$)pxH^cg}zsJT$GhlEKS3&j?+aXg@Fv+Z5wI z75T<(e)Bxv$opCaqxCfk#_6jB(!Lt?9;N>QD0JHL%m`l-Yj{7tO@?R03(uJ0S`pa3 z0KZ%4K$%<8rvda!5a9^?Z0c*=8;n0i|96t#Kfgk0_oe2S_&k*PgEZ(&Sqej z?BiUJ_*C=Ll)9+$oQKGJG(FI3zb|1$eR`0$;g(=;9`vRzIAFb_aN5!|3i#q|Qfv0PbU62nN3wwf(*zVbTF24XT&t7TZ<#6gJczGk`jitX1?vOq{ zkCy|9^W&x9uGsZ=qK=(RoHFccV)bFfrI>MkJln^+4;mtPR%Zjp7{j{C^1%mZqW7^G z|9M=|iFiGkdi+`(+hl$drusNuZwDO6|3ydQP;t?Eu!2!~pn^_hwM+Y}_xq`D&9^b; z!SH!IGL!=uDg+rS6d9@=`VkxVc^P9a{_W0~^6qIc^vx4h$FC`zv>HXa3sNwN;n<*dp^Iaex#p&J1w^I22#rZCf zkK*)>KQcOz>&X_uJmo;4oylpl2V>>N4Q9i`u3;@vvR6dEPv9t1}= zfXfFfMHhJfZsrw=mdH60!Mj9-cZ<%&i9J_>!n+FMq|FHz$eP)vj9}2wn;4IK%zXCQ zq$wG}NUI`EWcotsj~L{1=3xeLvafO$z9shz*HtbYe9&g`9I6z&#ebi|t2q4*+QHd# z={szoe@YuU6Wl0rnS2wh@HS4L&NtE*;t$XYub1-!GN-~@T9--RN#04)=g>psJ#`LF zpjsdN59Cgt}%LV z1*7$z3P$PO6-;7YUBx^Z&3qZfycx;-DMU^l;blH4y_Bl+f01!ztuB7czd(j{zz^T* zFtTJ7=efD>owavtrtk-?y~4{4X?oHzcl_{%(~1 zgdf&_RkE)sSo1afa6k7vL@VDiDOyaUCVD0MK9V-y#$G!5hlwf3-!ACq?^IkuV*}$S zak3Y6n=Mq`H^qL}#M^n#Q194NE~{}iFkTJq(6bZP;afw=Zp2XsUmf@*a<%m7wovea z_|K@bKQ3M3lwPs7XLV;o-flaOe_7s+)mQ9cpUc4=QOr4!f9jcUB5&8{2Ps**Jjdv> zcPU%e=m+`#KKjR)bKF~p{yhMm`ZwaR_1DEWf~*VM&b{Xq{z}~Wxwo1)|J<|R%Di&r z()W}{Su*FAo9Qa;^7Cmj=bkuMo}9fg?jL{DjPvslS&x>BoWdSVeLDSESv6SAQ@P_g zv^?i5>(le|GFJc5#82_pRYIGkE>*M{TlKhwKS{LZdqizqbWK`@_K9aBw&R zT#f{%*;grz;x4NI?&u7L-v`0(4gaR%r%KL=G@0~&4&`B&GX7&|=qAFGgmwJK83h`9stk(LBK zdS>(p#v)oDA?;?&3KWdf^A$|c2MPXZ$Ob(wlnqV|HgIJK`D9(Tka7wc7wKoUc8W4^ z#aIW3&7_Q9yrN;rYW(8#uI4}5$_H^Ojs3rKY0>BKBtoS{>j|Vun=H`bGmPzN=B$z* z7HjJdLVFLT3;j*dtJF9r=n>?{XUWgf-y!E@(**r_^Bc|l|Bs#gvQ3nJhG(%|Y$=_< z*pCMX#(@i0gA-$+XY8}s0u?>GSl7nB-A3_Yxry&ra*h}M0=~}9>&%39{o4J85Of5i z*eCGs4-A1W4t3<0XwT%9)IpbiEiE`n$sD0!;u}EC<4?HHh;Vr`_--@s9sMcznV?Uk-oF;#8_jQq zseVn+hXTb$Lv-%azru@hR9dV)NWmCATfu0(zk*SEUj-AG7nk;v;`iP6-+V+1PJ9Kg z78)uzaTXfd*0!RSHTPy{W|H}Tr-Cthyn?ZMtb%cRw1V+^2L+k`KqqZ^0(*?#5w0Qh zC&Uwc*bL1wai$1aXqJgHq4GT1#0_MkStg#0mS+=}8qrA`GMf0CO;R{;nEsdhoWFY! z8#DBG>sZJB3A}PJ&h=e2r5iEMk|vTrRzFO8zNC+jfLGVQ+`5eQ4>~d}TIwrf`JZ$d z(|G-J1>^Kj6^zyYrC^M{U%_boLj|Msy$Z@&=F(>(7z0J4TJwXLpHa}RXlPgrv@8~y z76(6zXDt)N-O4uRr{SMl^uSvv_X+xn`K5e6FrR#Od8ci=*vk~&R*QZXrwkkIw5bp% z@|DbEk-0YUz1RzAd?)+&GLI$xMdHO4K%<=L#4j`By~K+QqVYeCc#j#shIo;gL_Z_{ zl>YxcaRyzUOdBTAmTPI#HMDIaba?`DW{_&L?BmOS>dv-qlJgradx2m9}euw5+Ss8uy4D0ie@SxP$+qwJap}~$}E600n zvCS$@>8@hW8qbpc@L*Tj+nQo@_j2;TNPZhW9tFomo;30ye~}O0q380gm3-x9 zK4evYzURmnCixgUDQ|`Oe^uVO|HkQWkx$kthOUBkB99y(o+O`qFVA0hl>f%*M-`0K>lMVe zyModBVc-~%Gk5&54xia~)$| zk8CD%KyYU(V`V%u9wrZcfw%#xk0g(A{;Cu2zdx5(_-LZSyQSyi;G@uyo5)vf#`(u{ z2;-RnA5Ddis(EwY_8j4(>bZisIP5^N%r#jjoEzuh@|q}fXZ(r1Mx&9(z&z7C1f?%YOOE=9Zh z&<8T6!zrgN?Y_c{&n3Ps?anac2NK_wc6T)6`w)LI?e0R{{}Jt8%ANOq+Pz!*Dec~! z`hZEh*=sgw_wH1ac3*-{?@oQtPs{IAv>cq;oq8ed-kthW+P%9C?cSYso_6m}`x)B( z$82LR)u*+g-LysM$T9HkV%q&Facyb$PTmDKCju{~-CKzB)9$Ox^k1UguSkBAcIWc^ zbF|yrCSO0v_cOFx1kV!W(ZzaCW7GHF0@28!GWq0ixc;DUo zlDNC#VVffpUE&Lk{Y6jaeBmhY;rrqyv_qeVtlIxwLg(w^CRiLBALbrJnRAG7|w6pYe)C@5=! zpWh?@Z)-%^Luso|X}gD#sLDqdVE#7_D0OKbqSd~F4jQBJjdu~|V|Tqg06S^XoBcW2 z^LD!*m$81G+g{GN3Vr7+Od0xD*60%x(1)as$oH*8u2>76$n%g9HeXAsjDM6a^HJ!b z6(~GTLx&esR<_UJX~w>X>}$)};mZ|WbLvk3rLV4~ul&&HQ&}^}dQi@h#_~SU$8T4B%g}u|7C)YP%Lw1UC@V$oEMfo9@r!#o@$BV1eQqzO&=xOueU3|X ze5!1n77Z}y%Wm2wdkh^Z`xEf};7Q?q@p>io4M3K9i#VyX#2Gq1V_)Yl#2ayfKgPa} z$m?g|p{cTugC1OTGIh3!!RPfc@p`K2lX(5l{8#pIer_J7oYTj|>-pw4&#Ezq*OvpO zPC~O1RNb8V?^Rli{)B?8)fJ4=A5-vGtV`L~X-E&kSN_lM?Z_HP^fp7;vzUdQnjP58 zoNj7^qu(>9MHZ7ipG@STx^`NhH(3iE$T#j5KAu9q1sD8}?+{VA8_~tnnf7S(o1Bfc zi*I!1q0qwv$Vab}RxUDE6!(6}S$XuRsTTafv3B4cA12xfmu30#a`E||jlbcp?Cl@1 zdF{F7_NgFLW0FtWN~;Nj?_-%p>GF|J#s&AN}lCJ1^RUeobwmk>}yKcKbS~6H)z-8 zse1?NCu!b}_<9{W{c?}YPoY!);(vm-*eo5-tsagJJW3zIe3G<1q=`(o61(3xd@%k% zp8W?0cs{0&WNdekE^XL#V`etNgCF7fUleC%fAq(Y+K;cE&jnNMY&@Fs^^WXloy=hBs_G$bXouLk;hX#1=Wo;?*Y9Zg;Gcz-L)=Z=Q zWpgsKOKL)D@0}3no6dRF1(O1Ocaul*-UIAH+XY{K$8!&}ALJ}r9OEfhO!G69WbH!2M3!!hdK+Y<&_a^s0cSw>8Yw zBUW!~tHrD4o2cu_r-FPLsmyz;R(rYwKAB>*?0Q?=%k>NKClq3-{UOX!duDl%ukMe- zeW(5yRT*(WbOp?LbhS-R~K3K$bOZ?bt3Mh#LYP$ zH;lM%C2rpNxSNP;khpu!$5jz`SmH|0$2~{f7ZUe`8OQvr>Qg@cQ;A!1KJGcvueB(^-{{0 z@viSW*41EToP!8m-L8fZV3<6QaW#Mg^@0OkuOg47wmyi^bu@W^;KErG7tVkSH&LFf zgKq|o01xDx@|!%bY~Y+E|F+Liw%OUP2JVFo%*}472Rd@Iz4gKU9qn_oZxtAsn>_^o zI+prY@u9|ZLeZ)tJX`W=u}hnq$Fn^!54(5!+yh^6E=|g8+LhV&t0bI(5IIvI~PAKyP%AA zW*JxTo!B(EfUmVJV{o!6;~mP7x+JQ6iO{k{J)UQwyRiz!>(L52^$rTg>JbXY=wS** z>+KX2TZEst@xiD8n)cs$yU?n-^gy9g8#L(D7V0N+Rq)c;^E_W)Y4CMy5usI)=lMFc z-st;P^gZ_$$k=F>Yf4tKzYp(RDm-1uWd={*FtqLpPc67B{e6gknLK^BpQn#E`|)$q zgnp&M)1QK7!qa!}U){s=afr~UX#IWCrQBWDYOar+LB6fY!+jqn2l?J6Ueb30g@@m2 zj<*I6UqhVGy@Rj7!-XDP$(;xM?*bl9oXkJLr#3vCbAjDGejbjnqiDIGhrd9%%a9X= zFRtZTeR}~9kJCTro3=dskK{|Zh=-r#ztYyzQwv=1@7<+qdU)>T9KYbkagtUI^z-W_Je&Ml;b$y7Me21E|Cc^rM7q?Y0vN9FNb&0pX&btO zpRsxx@oo6^1!YVy%ead?!msZD{zx!+{)V)zJyaPHD$c3jsPY;7dYX!h(|@C2ygpUI z82vg0qxDG&M(Nim_{;oypx}1pFYn2U??Ta0oHFee{XXV#R>_)1)89b;DzhQe3f@Qe z^;r0CF8r8v(a+wI;NSaG-xQ{MzE5JTDNDYYh5s*^Gxof>*>>*ymbFqH?^{KKDL?5RKJ-mBHp7Oj$XODB0wD9V%P z40*;!rJT>}CeQe&l;^;d|@hhwaWj!eLM7|Z8!WfEd4L#}ZS!>da4;T++Plipo>|q8IFSK?m@qSu#?z`zd zJj=}Qwwd3ZRCZKyb~{OLrk?WcOESjn)moWHoV{8)@8A_4Y16Z|Ob`E8>u1q{XqF-S z?VSHu5^k^7ZXV&gTE<$w3zTnvKHqTious}}#xqh6z9~m$e}eWs&ihI5L(=5A9G&+Q z=pr8GS>nY0N_^r!m~Qbdza_x+^t3?NQ;UrV`<`f0cD5dJIwZ@|}uG44G)UCr?uN8XG1$X4)2 z=-nvNgpW|4jjuvmPf@2$+k1Fk4Kn!1VAAc_8QJYyzgx!|`&D?z%8uv-ItKZAktV!E z+930lJp>76LFH( z3`}n$Bl-Eqhv2F7=MSXIzfS2nt5&Z!7rA{6kOtHS0Y?zYT$cU)a!b zTe0CV&XLI+l6BuIbKSR^b>AA+eQTq7yZ)4)u-Pw5-A-NnYd)dtF$KeXeT_ArF)uos zYreGeYra*i`TVr1d8*uZY4f&n7x~#TW4&kKeov41e7S`&mNAh&7aUi(@}THOqIJ#0 z|A$CB1+9?z!B}l%4TqkO^#kj^?^&m*`LM8?XWgTDzWO1WqBZyM&Bnj5{*!jH9(a@W zz?L9w!5gdrUS|zZ>$KEvPPWv(+7a1*h};9poOyM)@0C}Bd^hv0(5By-|NWl-eb8p! z3?*LZg?}wr)wz89UDUPhTJSmI?vS`PYr(gOE0VZ2Yr(_B$^2=%77XZ8K7NM8wOI?k zMcnlg*JdpkkWoHVWW|fuesjQg znJfOaUmE#+*m%kMSn%7)`?>j@e}4Tpj_=y8|Dwn@mG$5G@io?fw@|N-FIWSHxxQx% zH?j`&vJPCw8c^1MUe2HWae{8%!H= z{p-I9WBvDztovC1iBIeqvd&`-=rY%U4M+TIz{H{oS@)?mptC4V)_n=G?n@}z=3n=n zIKS>oYqRd#rq+Ei`VQ)GsdZlsv`6O8HqxcP-vo}Lp3wCR)_vC#FZMR34ok_zF~|cl zhL4(MY#@)!^%sEIlz|*#l%dvrG5ToYMIZJD;3@h>`e(U{OVFQI5dEKmP8|YD-dO!{ z1*7z(3P$UXDky$2etz9ITp8}#bzZ+cts7i-05{4d{|8^;_M-`uM+wLY>3a0;3E zduX2AFCenA>{Fdl`+WE5zTTRs^|3GxtGQDE9(8uG)Njfkon@<%EX(Bm13J^#4iaHRIc+ z$)3O@%@W!avekKqruD;SV((PSI!QgsDzv#bA{((!w09_R&Cqb4t-aodJ6&S%jTW1{ z!M!Z$Id}7-FWsLImJ{9}oL%&l``?5=61EWP7aeiGLwJbr0^#<4-xtsBcdGc|gh$-7 z`~6T{+^?y4eZr&ehZDxQOZ%NEF6no=cuL|5_kw=D;`#lWiw7h=?Vg-CnfF%Gek{H& z@fmkXzn0?CerHv_?8GU&H}Zb8xFm5oWi%Au+poU3H1S#Yl*Fm-LCLqfKOH`xdO-4g zcXsju_rT;5_tcr%`s<1;>-R7E&b^87Bq6?1i);dBSK>pBdpx$_@4xP3>qKdDGvP1D zbA8#L+nBu4y(xK>`(Ii@^)uFl(7S2NJ+x`RbD4X8=8UjwlkN^1$iC+O+XkL}a?v5m zKjhAi(jw!dw3k+~=hAEo*IOxDeekzM|bn%(+6{Jx4TKbqaH( zJXo6;!(8bb6jL3`T)Ah;-J6?fzuf;OHkmS422H+u^ECGAKVz<7qd$KNeJ*!!$XuCb zi&O1S&mr2R5S7>~V4l;NKrGH1egm;0ofz?03x`&C9Sw%bwEWtCxWT&G(n>TY1}nv-stW#D4On3i3CZc#;Y(K4pyyU1o`UZ|#(R z^Y=rGnu4?wtH`sR@#qhJi2U=fc!S>moE@>;x7e696gr(Z``Dm&Zx{bE&C~wM{seoM zW3A}+t)oLMw$KKfHM8Dk$-IfPF4K~@bBZwzrHxnNFJTORL*4DQ$gz=Htff&b#14O>m8wu>l3Yw)8g4{WZe43YvX=ktPa_1nL^i!z)?*T{}7># zxgE8M_h_Nj1GG@)K|s;ao|8(NpLog{zcOY1G<*)N1Rpn>X$$>nTS*h2L-;@WmzlQO zpH@d2evYtFo!D%qZTF{X+Jb5LLefZ^Z#DjR(4Q7S8vc?}@rn7gnP%-}*2PX*Jigr- zyUk`=N79<8XIwjN!9N%SDYqlGeBx8$D*C8>gwghIX}7d_y~R;IlR3s->xq?4aLi&W z`hdCj;x?n58e>`Jk2_$rZ#r$S@yA)T1#)*w*4+CZ&ut&zu4otF&J^D;k(@uHkFSD{ zEWB=OPB(Z{W#=6^Rm_jTKyB>pT1e<3@_!Bg{D%2|3-@we&AnWaRnNOSRBdqY8~k$d zhl5`#-c$8b@!qPJi+lBXxwvPamx?{fJ>7RE_j31ij(2;K=DIzB6@!mdt#wCm-_iOs z?S%LR8D);+G-!?ZMy#Qof+yYJqf$o;|F>`tku^vgYpq)CwgqcrZQPrb36H%T9vg;@ zQ+Miq+}8d*6u6;1t=7?L)W1`w9XSs$#}@@?WAkWN*U4LR4$zNFX;Wl?Wo(d%L+@L{ zt2Heg|4H00f`96p81KjFmuck&-;(po8RY#En)j8ho&E&8>ieYU-7!@g-LX}#xR)(j z<(^M?X3=VQ31P*eHST)}`}@6IyovCU^BQ+-|CfuKc;4@vl%8n_c28)6AmyyG?SMfk-PE&2OCn0wiTG0fBTmhjPc(AVMC@KC{< z_rJM!-!Y4I^wspUb;{Np^ti`uUC!grar%9()-JS5Sb+NgICSuTfkQn5&g0O~R0D?& z*g_~Xoc?If7=`IuXwT3d<4TS^vD6tntIPbIcFjY(#tDBy9y(#64}|{+{aDXj6Fz5Q z&VE03D0D23ebyc8gs!#juRtig{NDPOoCT%9!gSKYn3ppLNWXA5S1Z z&Ia%2fZNB|gP&u{kdgH9-z}EWPtfP{tYM?Shlib_Py5l&+mm_GF(DApC zyRdayx)Yi4E6&KOXXL}Lu(^#zhTKhhEHdOCfyj`11tK@@13t<8ko^5AGlsb*>%;@l zahWeJ`YhfSS$#y|2mZ=>X%m-gj!^b1m5i56o1Ns_4-Paj{+ydfu571I=FYc-s@L4k zs#o2KLtZaV8d6)Fu(-C^x%lTsuK6~*=cRZnK(MfkHa_SVGpFJn1hALT7+Ixm@kbQ>L(3VB?%g40+ zU-ZEW?hgLkk`yZK`Mb1>yYY5r%m`bQ^xLpMBCB>Oy7FujvU~jEFWt`~tCpIw>KP@2 zQ2w|06t*OWmf4(*2YdhC*>UP%SzmZ$jLJCEo5vX40=;<%{Ay(0e9gRB%Dj;|A@kqb&C14 zl(`kl+-kA~z9;l6f_Wvd1M_MTd}Jx}Dwz2pG%kR7b(r&D5zM7Xe;o7cu-LFOFC&;+ zQN%|ux1t3yw`2~*@cb0>E1Pnc!I$FUL$ZGVZii*Pe}2ugMpu_|PqW;~l^Ul_d{*Wc z{s?EWhKc|$hk%#QFrU6K*Hu>HGKrIVIhl8Zy6(t1jvRN8F-)p@-F;cr8}1txuX2|W z3Ky?--%1#@c#XS=@Ynve#m^Gn1gBbn{~`P}@do#meQS$T2>(f(<}L#k5?T`r-Or}< zbN`j_A3`f3>Fy!xlkZ>n=hDTS-BSodO&+xcnPMe-HSlCrkJIS?2ZBpepnFokDJgHb zH(0IJHwIg)6RBe<|1Hg$5muVgC2T6ZYRcl*+!4sH!mEy(xRc79-)@Zvou(xfIV)?N zC8a&)N8pECXgYqy6z=q3ekTP)R9|gzgeHgVbS_5DmOk5Vi3q)cKKi819G89@Mx3l= zjwAn1Nq*U#$XfJlQ||h|F^_8qg@juPI`gEOa1~)T;V+B7cRxnBns6tfQ~%S&o%^3D zu8M!a-MRnS;{N?xir2<3b63R|@ZP|Ceesvh2i*Ppw-#U7|3`TLAKh!?C-MFb?_U>x zoVd(=W&g(FA^nfi=QZvo=MC;&{l6>j(f@dHg!6HpkCE@&;_-YJfh_e_@=fmLBRW>U zoqV%jhs9~@$LCwtKfCypyOc16z<5Od6&N9F2ju@D)^?5G?Z{g%xVQ4_ z)`&`#9sX-C_;xM7uiEK51JObN#KPo5H5tTeh#N zXdvTsgnu1z4~@_wWxQlvdK{h07Oh<)HWL%2-&3r~tV^GEE}z=*+<4)GP>t6x=6Xx1 z8y-3aUV0Tgbu_$n6!uLcy$$Jw@Z1sb-r=k>3TDHLjWctV;`5~)`7Z4jS#&{i2K+Sj zxRl?QZ)0SAgI`<)v1_8`v< zhXTEhvV;{4&j(@C6zr|5T*5gp;@3C~e=uwC*OZ|xojzfN?;mmJ{SE!Hz3_S8FG=_$ zDzqgjpPf#}=gL1($>oM^=UdcQc=C;u{ZH0vhJ8NgVIvKD;myPuHuD$S3$Iq|I*B`P zFZ>7M3Ta0@dz^mXFX;b%;`=3vHppIl4gWT6hA-(uqpi&_3taWhW2&Fy^+$k4y<+r- zR9v*aM8PP1k%DrjdmOlMHMlSqTyQhSW59*0&iPixCKP$h22KQk8-d7UKW{I596eyQ zW()Nq_tmo&Jqj;*&Z+egS@s)SK<10^6*ux&Dtx8JssfhVk+!uvtv--b_o37-&I z>Pz^;m+*&pct$XEC>$PfoV?-ih!eoa;S1sLiIY52-tq5QhruUyiwq{V%)%#xM~K{7 zY70~{jL5C!-L#2UB3q|)G-R-F+9&Ik{qTtX(As}d2Os~II-WM`dWO0lLzj67+V^bo zM)$v|&)WhMb1xFLCc$^jh&di{31D4E7e=2d^t`P04`%yWBs* zCvHf--~A(Nx@5l3Ui>38=SS$lkMRE=kuf&8pIyAkJrfx{jWzZ${`Cd2TP}1w-xfZ4 z2y4=VX*Zlc^~4j-yGpyyKSsM)+m4%N30MA1AGJA8CVTTDs~xgM=*O7XlaN&gpj-4c<|3;dK~_70tab!h?Fh2kab&gQ$ZE%t)s7>p zoj_JQfvk1{S?z?9)!s(7`7+@FC9Ca3Ruj2xZTxWePE%ID9xL*`^|^U5~7`)~T(37<3$+bK}-ytrNwbPTG75{<;bJeKYj? zHQK(JJ~)OR%VxEON_%!9tF4Wv9nkLwl5P&W3;O*)QD0=WLyXlS_e|*bG4A;l`h5(3 zFht6SKiF8`wOW!JPxXGn`TMEq{xOQ&!WfCH_8aJT3V!qL=qy4Qhi;7D<>+s&K!4L6 z{Y^Uhn@IFGMa-F0^fw2YGa?6xE<@J+J*}bWZ$cWUvv=4@3qj@yC@SbtWL>A|2yMwjZ|I;ekjakcF^7S>hoVc7_x!Z&3J&Y6;O+wEqofaKK88RSMlknp zqpy^Xf_Z-hvfp0lllaE@hPBVWq(8YgCaraUM82ocL-^Mv*^K$?SN>4^#+6M_@_VJbzJAf>;@>Q)Exv5pZW15VIHB0 z5Wn~(=JQKRMhYp;XzRBGnRmB_ z-Ju=3DY4U${VTE65#Ot9DD8gIiZ8Npt`%~=!tro!^MjlTaXg5B<`(z9k?fNnVofmf zu=r&S;H>wO@nM`tU&;BhS$}9jWZYL&pXoJXY?!qC+zq%m-8G6F*VEUtqDqgigSFTDGgs{6O8-*k zSn|rf8SIB&QhwUP8Mx`-g5>?m{D!l^>f0P(AO0&k`hhA={2Gs?|J{tk7{=o&qpWSf<7vJoFZdLa3N7$?rh?67zox!6M?YluA*b=ix;H%z8^bCA4Kb|KpzlU z!1zD*OrH}cF8<#n-^b(|Aoffy=%e&*{I?;!z*X2)bBPT7K6(6Y zi>GaW=Uw7=0ofB)?UM5<3AA0xd=5O2{@O{p{A(Lf@W`}ZTAG&84%?#yoqaAjd#U#N zWo+7(@q}5%Ch~~=(gxs_7nIRnm2oxkVjHj;D7rf(^Q$;~{wRpg9|dFdXB3Rqmnn$< z4+Z6%-%ov>h`*E44u|Rfr z^hCz@Q|yom(~Gj9i5r9_I-rTE&_o%tY~r%<%P4lH7VP(S`RUuut`+eK+!H1^_6N@4 zQHE>J6`st?b6qZn;CNhPZrU{^(=8KAQY{lo%G1t&>)_jte7l-&)pwRmPeyygMlzVb zYx@t4d?^W)Lm!k3WpPxq9#H#1(`7%1eV{Vrrd(vR8jCh} zwxz~7hJBzil_j&ShSX-t2yKO!$&kdoS2wmcy&r2ii~p zpY8^qzZ{;u8h&2M+D4wAgQtH(U9RBy?MJqAJ)yRHr^kSQ|GfcsKV!f>e=y+QRR-L* z)_@;+4fxS|1MYumd(O9vr+k0oA@GoW_R4@SP_AKrwN%MzE2Z+<0iGQ zACK-RJ|#UY-m0}s8l`Gije(oE}1YPtxh{X3`Rno5gP6 zT=`+7b6-duI5vQ^>n=|3K>9e+S>wzn?U{?yV@aPuI<#~%X&)1soKL%(*r#bqP7jN7 zYAwgv%a~^F&9EMAe2M+NJqrw77e3<}XE3(4j9pe*>EmVOTg8|u*_-|^XM7m5-4%?D zjNL+W3@aJKcF0;)^jkaDoev0P-ML60^n0;DbfGR&_I%MA5-RfMx9ro%+VoCzzY|zf z_-!^`U38dzufy)igaw4x(fO_*Tu-=<@Hgx=Y6yjdTM483ohU9Mj7q3>R|Een?+FjP zM;&A=As06xX!eLLedll~4f_cpJ#guZv;){@8BF0W>9b9blI?KuKF zcf#)w+y}gY;NSm9OX|h?rkA^v@GN1*tRd@5?^yWfc-Hq#tn>F1 zvU#pTS1n_Ih<1))?R3B6@|WVB1)3-;gimRlnP z&!Y9eDHx-_2^78MCUlm|L$y8&2$KmvAb)z(hk9Sg%JZHP_orACWB8Rswt30Oxf0cIP>SuD!gLzOsZG=yFK4(rEXZ=^7 z2Z{JRSfP9##G=ETWs6lZc%oy!lEFu27&7=1)Lmqmg_O-ck?F7Uybg0Qaf4}_%&)dS z59ShY#G%7P=OJ?V?ZnLz9cCl8#+*(0fwK_ehqSH|zXxmlzJw^poTtV7TjT*_j{npT za9bT_ppt3g^&3>5;~Ng>*I{0#;*kFpU4@=tuh{-JbP?YZnI4g;up7c}Ll2MmZ9os85@-^Fc`?|702MedUyF$bUNsChmnuIZ^{PX>EXuBWz_agD8eyD@Wqv2bGK%OFX zjxB(9g1j^49qfIVOsTUiaTy2A@*xZ31K&q*8)Gbig)jNQ2aGJqwrt6VBx4gu+9SkO&zaZy?Ryk>Q&V%+tP@P7q5q-w00|0_p(3_8B?I$&k(yM@_SU5LIzZ&_m9 z{ru`3i(sc8Hg;J5o!B)Jc^6^~@FDH3pb@n#?ayZ8cN4;MMo&7R6&3YOEb{ezg!u8W z7pcOY2G~1ruaLp+;joto*dHMMGy%KO0h`8b2K$!PeG_lV>U#t6A3$%W!X5(H`yd~1 zybBrL&?7S({|D)Z2-wvQ*fdTv*iRStO?*1L@7IVYJyXfH|ogm!@sdckpk+kw80wKe+Agsvl5`>x3A@BdM6Zq9+6RsL?xcfkcLz8`NK zfd@V&JfP>GHY8b6!to*84;VOG`ksb0*yCB4|6jjvS&jo1#+k*hjMCnF8tbqfS^g`| zp3ikJI&72l!T(B@)`)Xr)rS%H0e%zS`(Uf^AMihkI3MD6T@Y<%g|V6Jf^i#9cVsWgmGDgbwq*n6!K7d4R{iaBamftKi=`;^?i6J0CN-B z&hNw+k^{Uc9sNFn-?;O7q848|*GGDNRzBu}Jd{6UHugYJ2h#WH^hNz{LE86m?wQin zVtqj6rJ}9#?L(S}DE)m|zQ$W{{+IUbit%M(OYoKe?%LDZDu3onQ*-xy7WK%z-j6Vf zH_HGl>5pk4G5A@7<1g*u&)U&rbctclT9ZnsYcxFwhTp8T))~&th*1`kUwT z+ctEEbF{^zQ=+`1WS;%!vBKs zeVDiIyA=CRXd~!ny6#&DS^?IK#ai3_pcVBAHpd*C1A9vReM_2yL^$5{9hD$nGSlVZ z?e;`JEBDucuVbtsey8#ECnEeggdaw^L{H+Y=$Co=rx8y2?)PC!3HIzUCgB}HQ6|mz z<3OkHvG569kFf8=u7}w-`Hjuvv$}D%(TB0bT^8xj&V7O2vL_pQXK}uQ?$*$of@E8U z?%MPWAJy4?qrdyd5r6m5ZL6W{9FF;$?5ojvgQ-|sP3hX*-+#B3qZQ`u{qdn0*kiv1 z{pgWKfA^!%pXS7WeW)Kc$LQ_X@8K?S@6G&o0KdPByTZrB?^n@hy1reO|9#knzHIN* zX@Q%&4(+(P^U$ol-#>KG-p)gLXYp=?PkS#1Uu}f{OoRBY6JHuv=AMkd_?_(2KRU7k zOWWf&1=n zJ(q>`lGb-W_~6LRzADCJ-`vpM<;&S{74(oJpnE%x{fu?KJEr!|(=eaap;9!XgOUL0-!eY_{w+Xz48xZZl) zlcaC0zpsJ52IA>kj5fo2f`l9R9ly1{u^68yoZt{Vx<~OW#`rwwl?Y#%Zn#Uj{b-kU z1P^sMOz^6-zK?-huL7$Uv|dMVE1@iUcZupo>GmN#%?)bWZ1hR0C&f{JR^w>A#TGVz`^3NQem_Yf@;GL~qe!MG^bLol8)^yfX&SCEFzvrGc5Ph&hA zHwSl1FVT+v36Az3+7IHbus%K(-yzB*nQ#}vX)k0y!g_!|jdxKI7N;=u8IqeW731a} z3PYW#ocBj-eJ@@_e1-QA>5ICrqc8B!!S~24yaO`L-`6$H-}hc|apz@tf9BD(m*$fm zbsy}}{v9;xY1TeIQGju-@$&q)z~^t>to6P3OJDc9zwvdyd4<;Z<|6R^Dy?t*&0`L| zSp-~4wU0lKvfnHi(b)ukJ+UCS^US!c?!HM`-S42hccx`^pP7->{Wk27zgG)5^}g=o zKSCQVMgRE;&Yc0y@$&&^toHGnk7RY8_{mt&f*yMKnSdmZrTopV8F9&|e+=*_Y@usdH2IBRhq z7Ivh2HvrZqUpLm7&*xwqdwW(E?#X769m9jL>d}4tvaIem!BcO}1HL~&o+p9pv#9rT zzV5fH@OvA6UyI*{ue;~P?8MM1oM$GZ4cNRzeR>mc!O#qv36T!KlUhJldu_2PR6`6uIn!mf8VF?>AzbaiZVWHFC#d>GJ1z=ePllaG8f*3 z0MAf8=J7rv%PK}$PeD#jh3`1vi~h@Lg>Nx%Aen&Xc9Ki!j6JRGdpCfWi5B=SM;f97 z$t!u(*2NmLY0?Wm>z(7ne9Zd`-D#$^6|=nuJpa<@@1r;N{$_yApE$Dd{AfS3&3bCm zeN&C$kVR=6#kdE%NO$$dK*ubUhwp^~Cz{t%)}z^v{y45HfxLuow!r&+!28`$ZRx37 zc|BQsqMjohw5B%ufZFW!Y)(h?RT@W_tybXuH^6@P8~u0FSUC%G17xy7!jZwlm(~t% z2>zhGebyGp!(k0E2VeFzzg z=u5mxb-D(1B6!>A3%s_{7wwF5q!^q0eOu@Y{@p}h;9SAJ)4MjX@8qsBd|`WMlBe9Y ze*}}e-0Yl^Ec+4Gb+o7Z#D0HwH~MD}!WOeKVE==CCv+{q_f(A9e`43=@P8N~KXfZB zesWhnzO?3~w}MG7>m_+|JnZ+v#o*5TePn0ecx~~IUs;_G`$H&W8s-JE6;HNvr=UHi zclq!p8e6wspcQs)!yD0ir%Y>sY<3K95995I$2za|cQ->$JA&WG#=>^r!&>)us{Gw| z@BPl9-hakl%tP3hAiwoe{21-AW0>=DhMoS=-+wZ!^X)}iU+8WYo(tM;fX??St?zf#4^Tg>DcFAb8N`u{O>?z|wd!jk{3PY6g{-#-ya^uYU90sS zhRy=IQ7#P_a69Au)yOYtq%`tsO0fSj%uL>g8u=JTFn$QS4jzZBvBO=a#M(YXXV z1V4C%=uw4lzq35NebKMNhd!ep@NgILu~YrXPx2MUy_l?XND+1w!kAr}hyC4HyD&apjxeGz)vaNccJx}f`vNy>eGksj^IKO9@BBM>BT|HOB~;{EP?F%~lRshyjkXH=UUC2MLLH6l~{L;JBmzeS1=+iymcd`$G zd9w0A0oJUTS9*Ssop^pCmuX%ffp*!3bqChuXt(Y&tG3|}Va)MmZ#AkNbDd}VznmG#_LA@f3a4So30Qy5oid@dC*Xx^kS%16AF zBh~>bUkxLf0sR5vU*7|Oi*((+dwu!TzeyG$dFfTODVz7vuZ9EGhOa>mf&J=JD3iv{ zd{G|B8B2llDuHX-GSL^QOiItnG^{dT17ERn$5FYkagTEUHp|~vK;@#Yso+kex?rqg z^EkfQz>D~s=3lbW)QjjDr&b0nzG7rA5qAVH{n9P%+9i}pw2k$;0FWQ~SJoG&ScuR+A zOVxV)8t{lm2!CqZ)1V{qETbc2HraNsJq!7V+HVrk5w_|D?tep@{Skaa;dRoBTsC*u0JRT!7DC&*_|nNIp$bjFNqIbdUm*_nDGvO52-IoNB%+0@VBEl6Fw*Ur{( z55kV|&C?E%F6Zptk@;`p?oS^5ei&y}AI#|_S@8GBOZ^u6ANWq`dXas{bsf<8xIy8> zKUiZunw7t{@4wML{H?}`<{MDQ5Vn}nnC#P#jqIsibOx2?cY^s722#Z}Y&uTB6+~mVvR2KF-wFIp_ z#|KLK#%JSgR>l z_0Z27I0KjO`^r6H@A3JWH(ijJ#PC{;{gWc>^J(Hv8STOPu$KZG{M)h-hrO{Iu@8$q z1NBTZ>_vb+?4vN@*1rRet_>?aFk{g=tA@LcEnSgSK#|KT00?bxQN?= zFp|>_fR`x^@lG+qUYV-B7bZCZ^&=gX=7atd@K0ea^fc-?^U$dhzY}oZ zqdtK$R|vQ?-rXz8T7odzQ=qjAdp{9z^iIZf#vhZrW+R@;pGD&D=|ep6cor*X zau;;PcfFFUz1M>~2L!iUz|BTD(QP$g1_jK2f*w@XL%9+r&XZ6b8O+IDZzG=ShsS~M zqP-`whe9wP5-{IEIKiv|%uNF3aS3yj9p-NY%$Ehs-x16UwD z9;3RyBw@lXjoKDJ7BHU`FrOxv6H#~A4kA2n5HNp+aN?B*0CTp0`6R*!_lG7*nEbu$ zZ!wsYy1vU`PVD*)zS!GO`$kUx{+t)ch8FSt`xqA)59X81fq5l+Dr|vkecxp8#&sQH zUz`QT_kE13Um1({Uf@U`uR|MAeZP)4!tD$A-iLtem$Rl@QKlLb`?Vbu-q8}A&iL-!>y`A6Q z2fGG$+*B;0v$xo{)k za^Qx;4TBpBHv}#lE(^{Fr#(V-{Ij6(CBTFF*G1Xm4%{yAEkM|g8yA5mCU;@a^e);T zy)SFri6WfScnx?`yS|QgrTOb3w8s@VZ*Xt6?}e+fvA)Dz8G=6?`6%5``hwQ5?{e3X z>`$I%V>|dRAAENeoy){{1m1)kkoqm7`eE8#7}ITY3e6o)(Ok0Z6R@3%_MrCF{8y}l zOy3WleRYBNS&DT+ zJ-+kst=0N|{C)|&?cMj2-xPNq!5X)7G<5OM)pVYR?_Q0~#bj@k4g-o5@lyfJp7_trb^Ih&o=MLMGQaR=-q{H%>` zL_gYd!{f!L`j_qny|XYcJ(-n#;z68IeS6Wy&d=SPedq`r@%crG>(2C^?602fn{*%- zwlCfR>`Q#34vfei_RHbEaR+j;ho8`9tvRF3Dm`=S%=Kq(O_ZH^AhGVuI?PMG=lK&k zMROBF_bfk+egpr?#4yu8GBF(fx0cd-yJT+>XE3prBi-#sICFc;oc_DXo?K`0?lZ>$l^}&xSK@WNyJfqXK~wCBHxs*4iWb@;z-YqwtDZjss1xV zCgV&8c$H|{?SoDYIvUbrkEgL1j@E4y_Hy(SRSe$G2;StbfPnW0z@zb;#jRV#;yx+j zev3F7<9R+FS0&(u~sAf1o~1Tt+W0OXN4w#^21mN4BbehLyN;*0UbOF@=JGc#kAp|U@UMsa zkYwyxfKNP&en;m7KEOHZY?MVhK>Dr0|G^weGl88WI6H*q=;>Wlwl*{ew(<5=oUQsUcp!_12YbNIM^KW5;WqV{){p?3-0^lrRM{Onf{Mw=Iyv}=1lG5UtkTk z{{GpC1M!<*T<<&YKsoTq@r`^q2Wi(~94jg+`dCL=f06IwJj5OG<*+!k9o_^x@4$!& zrG1^g5fA4h{fG%e(Z*9r*Y?{FF(;6`5F!3UyHgnbZe+jtnZVbf*P(fy?na$LTYQLi z*)uBVz=?-9cD@b%I}M(@w$%Uli=XwMUWECD_=?8l9{Agr;O%ncrF2ji95@ecNAd4x z`McY19(U-EZ0$C+>w8L?g-C<>>9{?aT3=IMNUd8UrYv_&bl%=XnpZoqA zGwgHUAJ&)u`(n_11gGoov-^%?-Ta{Fq)BlA3@93j{$Ct*=F&Hyv0~d^M z-TOeBi>`b0kRM~*;hV-DLb+pyWNQaXFy38^@z9U4u9o64-aUY{I^t@Oc83@b_aSZv z((FL|Gl+j-``ANSleNyVh^H}<#>)4GVjmV~VrfinBwTLM5|6;$3HP6H2F_Xc{08^^ zea7h*pfh^!7UT3->=nKTcw2^RkNpngY1=Kve&74rAP#quZ_I^`G7IOw`P)3CKkgHEl%;*1eYlf+pQiQU!24Z?G3Vf{a^EKS zpMw7t_{-sk>{!__eAt#5zF`ZBC*TY!>`Bb2B@yjK0eXsxe3#$?IoXTD3i9K09x zDwk>3Ag_=mpP#wg$Ksw7aaSRZ|gu=}nfC zh4^6~;(EXb_hLSN24T~+=?C^9 z-C@Xq*I}HQjxmdV>&PF=nsnf9%*l@<9o2!}=7DbXXpT1Nz=!9d4CrExqYn9dcAgFc zCY{&51!qr>Pg29CpP=)n^iGkLF6TrMg+08wvue+h(<|X9J`Z2Iv+^{~Oq+8S=tFbX zQojC;O{ zekS&q?o#{=pVlRbUn*%?*wH14pF6{Pmn5EchD~3VzWK ztksFb8!HlBN<632@&xXd+Tr|adE&4$?DWdSA%r>Fh224s?c#&JXXZ&7b8@==6K4nj z|6L7Z$rtRf-qmi)O5lvETIRdj&$1Hxond3L6LDwQ;h~8=&aj+eiFRk$Q@M$VGpz8u z#P!awqvs_;&am+#6FZz?`$i_Nb%ydiVK0qMtaXNAZ)vqN z?D27lRnD-S35jLSu+|BQMb0p7V&ZaV*wKlJd}o+{QsQD~*s)288P2e~rX;31!^)>8 zCOE@hnw}Wr3~TsA;yh>ATc1b_bA|;kOyCZToi`@WNSytrBkWf*5+6FlcIb(JIKwWU znRwe7);lxN>kQj|QQ~!H*c}%o{>K^i#zl!gIKv8OC0=rd-8(Ds8)w+bS&3gb!%8kr z{JS&kJ5v)+Im4=_CVt`!J3A%uLuXjwl*IR)VJ}Tie8(9UoSgW!Gpu2J;sIyam~n}3 zI>YoaiLX1u4v$WJ#ToYQ`H3$&!y3*{e9jrRE+>I~eS4e#d3a*4Gpu!Z;s$3}&hSKs zGwkrNM5{Aw?$AV|Gwkspi8^Q4x*>^xGwkK;#C6WFo3azz>|xJq2UjI3ona-b5*5y{ zhgK%mIm0%uOcXoAZn`3Ir88{)6^SdHVNWef6gtDMTb7vT3_H0raj7%xp{0r0&G z3Edf{FHKByhWQsKCfdW^)y~dMj71o9|BsIObP;s_Gk(YXkfBWf1&V*7;-9McFI4;& zDgH|oKXe=>OvrSmAF`b3hrDL`S1bNCihsT0->CR275~+W|C5UU(~7@F@z*K->lJ^C z;*To+or?bk#UEGvw<`Y6D*n$a{x2&2FDw49DgJvD|Gy~y2NeJRQT*Rk{Qs)>f2jC> ztoVPb_@7byKUe&>D!lPo#s7K5kN3IEw!?ePrvGb-|6dgU1B(CuDE=6Ke<=P_ivNt_ z|EJ>5QgFr_kS0BI75^y3e}UqksQ9NU{tFfVMT-9t#eb>dzg+PzRQyX6|8m8DrQ%<$ z_}3`@^@@KZ_e1tf`G@@&=(bk(v2p1ho?h4Vm!xCysi9;Ji`W5x?6b{w9z3eID_*dBor05r3yg{6UZSyFB8*?h${lNBm)r_=h~= zk9fp?$0PnxkNBe=@sE4NKj9JoRNDB1Gk?b74~}u~-v?(N^N9bINBr|1@h^GAzw8nJ zXOH+EkN7t{;(I;f-|~om*CYObJ>n;O z#83B#*FEAd_K45(h@a~bKi?z1&?A1CNBl~Uc)v${iAVf8kN9$r_{|>i)gJNNJ>swP zh~ME6AM}WC@Q82nh;Q|XZ}*7b?Gb;INBlmI_}e_<@9>Df(sA6h=0l>{%0QX$2{VH_2-<0K?V4qhyTw=a})e0;qU#(r|pLSUHE^7 z^g;N?_nhj_L;4-?kAeRQq$!8L5dPf=UkCqu_(y&WD_-zd!=D5HZSZe~e+c}o@Rw6K z(l^1s4*rwyPlCS${^tR69Q=OxpFsF*_*cSD^~gaz-aYNp#v-g0VJngLS6}pLZ@hM@ z|1|u^;O~L|B=B8;G(%oL)&DZW3*gt_e+l95BF$N(c^u(y!G9Y5YLxSD@SlXA%E8{G zmIGR}BJ7Djad=JeKMwysz-Wa3&+uOd81?YK41W*ss)GL|_^F&t@IMd#cKA2K|10_rW#5h2gG;YlPbkw+HS9 zxEtZ_fcqTW=iv^(eI4!_aQDC+f(ych;OgM&;o9Lk;C908g1ZgwGjN}UyB+Q>xUa%} z4eoBZ9dH4-8n{}xR=75}C|nF~AKWc)x5Dj*I|!G6`!d{D;Iz0eo)yoI4~Y+r4~q|v z=fp?EbK~d5N5;>OkBX0ukBN_sUl1P`A0M9(pBSGMpB$eOpBkSQpC122{KEK*xE`Mw zzbHN{esO$u{F3;bcwRg|erf!&_}q9w{POs``26^S_`>+2cwu~Td`Wz1d|CX8`11IQ z_{#XI_?7Xi;w74J+K;y1_R@xAepIsd-* ze7I3?qv58*eFE-6xEXMB;R@g`hnokt5^fdTm2g+VmBUrQZG_tdcOBfP;64p!z%{`& z!?nOg;BJDu87>aD7w%5DFTi~f?n`j@!hI9&U*PV8%YhpKmkW0u++?^Za8u!?!R5i_ z!(9q@8Qe0sE8v#Ht$tjBBHSdni{WO&T>>`;t`Kf9+!DB@a3yf1 zaBJY!!d1gdNc zEVyjAF>qtyE`S>cr^C&Jy9jO;+>)1@2b3{cs225^!IJ`wAS+0%GLSFm`D)dTH8_oRMS3PoAz{oHuuV z;j)$5?DdiRes+A+m9=*~^50o=%5S>==O?E5L!bZgD?{?Hi9dMk^e0x=-~Gg^!V zrfhiAUc>zY_O}E7H^8;R?S}Kit%KVPw;fK0%Y&N_w+xQ{2p+-0T}utNEc-{m<-%c4 zynhJXP&n*`_Ya5D;IIeYp9Plx4?5}$MQ4-Z>BRUv*R$)!`?DM=&U#tS|K& zswdTt>NW*#CfowJBDetDEpSi6VLw8<1p429?U1ZzH_MtWc3d~Vm$c-6$ftE9T#IW3<=M5Q| zeSX#`-)OD=uOqJi%y;X55`DjA!zdrBts}m(l z+kW%WQ;&V?PmdkH_L-}%cyH``*F1FVRbT$EPfvgN>s`ZN_FcRGf}eh7!Nt$sJaTLM zr+ym!=lB0--?xYT`B3|Nhn{{qeCUnOuKnrXfBwynkrzki9xv#+t$qEUJ0HF7 zk9YL#tlVR4{*C%M<}y!=0&x#cr+qhG$i_sIt~)J;9odSJsd4{m9CZc6W$U$}VK zH-7h-#iLHP9=?9e#HSyclGC*G!t!0~<~;VqqlG^hIsJ$yA6^#AnWiKfjDefVJCPyeyI?dPSt!*w7X`y=Ua2q->A(>1@g zkBu7?LG8!>gh&3?MKHA~{mF1@Q~Hy?WjqV?f64!aw;wzF;m;qvQpU4zk=~E*J~;Z* z+5en($-7Vdmict?n|Or&%yM-0lMQ^7R;HJDp>pi+BL3kWf&4U3BC$+AQN+vNAP)PJ zzh!yyH;eRt=_ygapZa-n6CPt`9>yS<_)xj@Cjd%#vR>+M*$;^Sb;^T3nVtF${js?b zUkRW37X9h$f2)23S2~KKK1qKRMt>5X{4Lv!MWP*PPNBaY?0@Sk%a!?LItf|E%df=W zFW|^<`IT_wZ&{uUmw2OwJV++oCjgqH*mso~LLQR=bVhlZP1`3@3B+W$#B+y;mh-X% z`Vri{|IyG?vjHwI%(@_NF$bzft_d<{|RZct?LU*3q8?PxBM~ zN&IMBr$5QRGgur_>y!59_Mqo200zhy| zBAprSMg#fiPb>W) zT5VG(5YRvTyv6*Pz11A20DB{Df@ZBrl?3)kw6 z;ie{~&@~H|=#}fLT%gGq6;uTZZqmao;dYICw-oACRV4+j(NGgQT4IrCOn{fa z8bX0!Oz&uIiUfih$_hn8Ewv%NDY8pv#8ko}9ql5F7tkDNRbqFBL%U+iZ-USoja(n9 zWhhvn0j3p@T;nKH6c{ORs8kjd4AliXn%WB@b#;1upgjaJPBe6Fpd}a%QjiFUH3Xue zpw3!NrzS;Y05Ex*yT@|Yag9UyEN645rCp1KnouRZDO4Y*-6M+VXlV`9HfkmwjC`$; za16A!0;wiRf#9{)KznV2lFlT8Ju4$ZI^I|{yMyCnP?cBBju9HGW;Zo;%%lHwl&}0k zoPh!Y(LF4XV%6Wd8YhLuE2`;wT{sFZBIKIGv1Y;<>>3R1=Go2mT{SzP=gr?ed;YTB z`MOyHH8j@*Mig-Yma9=I7K7+D;r3nOSjbGPA)}p#l9LdK_CUCaa6lzx7Bxur!&q&U zRY&PD&M47tGPouHHVC)yev&GVYUhYE>7jCn1ZqXPAqrM{$4f2ewfju;u8JzYkELW3-QB9KANk($iQpZzfHwF^<+@d5kKrrG$ zS(4Id=t`kzIM5Wnfe+R)NnJ-%Q$c$e#VAb7fcqgH^0@0R$*Q(mXy-1jcOttEkU#nxL?kHoQw#FwMU{Mv)NoyiX<*Q zE*Yv!%LpT4=Z4f-7xt zgrr+In$$IX4Mqu*Et4QIlG;KEPI9Qkh{c$*smthOgOy^462Qa|W6R+>&e&3Xl&f-$A;Dyu*~Y3EV^mt{rk>2;AP6Hp6cyTUYe11cmIM`GFdDH113 z$jMwH#Hi$?fjSIxE+`~rkQk@WASAt-5O58dJWBPE{3_vun_HX0p%^4{>I1CGAt6kQ z`kMVCRdt-{sTe8sOBnJuTLmNoGFc7Hn@XtPTvS!1M6)_ga-*oadZV#vYqe3jy{Nc)V@4~FHAQ9R%k|l?-)uoIDXYo^yp=(;jTb&v2CovT zii=P^s;h#QS1GQUl#(j0Hj2y9CS}D6P$k|Z-)aN>)4*a+Nv9OSwnn1uNQG?% zoq^SW60MrsBXZ>!(t{xmiW0SkcZZr{S_&}hHPF7)i5C$_^1-011t%tKoW6U=K}0a*bo+RT2udR)s>1`qhQ5 zxP@CFA}eUsh1*3sjNT357S`U2mK7GXM;b#dvdo%}V11~a<_8OP8;8jDuS9XgH{uaMTz<3CXWvNGe>5z}eKL1=4`AsGTN}MfLX5?d{+oEf^~_xS(QS zfW%TF8fb}yX;ClqC-S>EmCmg-5RMv{LF=13m}-t7GEkhdMr*j0tqYC10N~H=P{so} zZj^1@TwYXMYE*98Txx7CDywu!Q?zDH*+w-<#Uia@u{Q16MGKp;HFxcT737(A?fhm` z3pgy-`Sj8pYK}zr%#KlSiFRPQ)g01iU%s$TUyE_!GD}DUQ?!PnXaXe?`X)$0Ek>JB z!=w!}Y85)3`2!@YZDfPX&JfK3IuUibuA9XPy%I*BouX_SlUhT8MqU_x<;pzG&>#se z*cA=50wo=*R>+-9UBSDuf@6CGDzdp2&dq_{dR|KeQ*op<917-}aM^S%0Fzn@Gb6+u zK<&_zfOfPn1`GuOn3N=;onfZ1ik&;?c_ylv!8T=R(L4fe-mCf2R<;{>HVT=BG(J3N z6>A|Pb$cL+L4g0JW~YWQf6dh!(DOJf3`#KqLA6Gx0ZE+_E~?Y1F+_w(4xWya&tQrY zkwOGTLmehGmdOZ4TH0GMIhlo1hm0A!pv`m%#@HNClCqyhGs*x$X2T{?h$X``1<^R% zij`W7(<}&0R#THekTt#nx&>NMu!EXJ1eutoE|7E5|94H2xo}+%9KRu^2*T3v!rDh~#z??=rv}9{G_980R#jbDT2xWCaV=nB8Wh?C?J*c;7DJ6r(`qP6%+EYVtSLfF-60yEe3`JhgSQ!X zrf0*<>SdIamO~B|*m~BEk<7Jrl5_uGRYzdRS2tL9hye$LkA-8TuH&8FoF@5Cxwscx zKs;jc1V;4MP|#qT0X#WIX``bh+|~hJ!Q(HYOg{IMq!NSxN17KTh|!FM;fPHvd9hfq zz$9M2!c!YJ8I`3~WhJE>i?7l2Jh1=^=1bT?YFG^SvAJSeEHq~)3af*jqG=D0t}ZXs zQE&%O&F3z?Xmc5lU97i+I--#20=2c!`eT5k5r(kK>u70giR^00*Y-g6Yl(=BhoBbN z339M?rbJ6?3O9zD_8?I^2C8=GXhK1WsRzdM?OIuR%eGDEeFFJp{$td8tb2?f^%&( zRPb~()Epp6f92a999tFO{yfx3{M2QGjFrIu&0lJTn(Fn*t>M840Sk+8f~+0vm=8)?Cj4wvr0BQh*fBu-mU>mH(T3M$uHv6&XBuG4v{q!|(~Wp+>6iRFv|u`yzWfj=GToaqLF zT2kJ0SjlY!2>vPXaiqu)6KAe;I53vR8s3O?G^d(rlG0>|M{>SQFp4_##lZl} zii%imN1I~?(}mFe1nL)V!f|pU7u4Js(j{X?$ao^c5)2dxo+%vy=m17C&Msqwd%_G6 zN|8AoVr+nH!vt<(mx@FwQf7!+s@$Hq(QN|@IW~>cfy@*G4J#&1mVscK4vZ_>md;S_ z+~d@gQL)%qS6WnJR8~}p9sIJ*rRB6)Pg`1L#ZvU8e!}nZgxeMxyLYEj-npi8saLA; z#XY}kg+y0Q&cO~ffKgl+ICbK4!cms!J|3q>REjqP!Pm-j?r1Bp20><#o34v2H0XG3 zQ=rx2e#dePy9n5`2}kiCl3T<|TB{|`0Lgp^HcX|rvszgu8Wfq#%$d=JB~K%FIK-!jps;k7XtlV@u7Ds9+xuQ($gHeqSuC0w=k)moj=l%H3`i~@ zWXKk7sRxa|$`Pt! zflh-Ifx_vYU;$;b=$AB3*+=$)AV9m=I9FiKQjU?*h6hV}j_8;L*%oc0&)E=1%5!Wx zf{3diCVmX=to~~E8q^q;3eW(MmeP_pQDw@(6V-%}0e`37KO>w z=Z3NgB~x6N4Tjk6nFfh@3Sa}QfULL&u|;7r^|_&JLdg`@g@v5j7%0<q z6@1H$5Xnr*N+lj|YYAKvgo>9=3^T5`FZmoG2b&h=0?gA`14W816NGCAhMQSl%FKc- zT<8HkrJV-0q)yU8G~>rK_@xH7Lzm7ZE)3x&nI~n-fCJb$*a1!%sq?rXnk1{^yfis|)=+ats)GH&KOLfjc>fyB&dI!H9j@xG}9o@(g1B@3mvzsZ6?+T9(dxCvENNNP? z>R@S#w}k@(9#zm9iG|ta0ahoPjV%f)s3?PN#&D7eWIx>(i`2E#4X@Tnq$w4CqIhIVhX7^@T-0JG$mm?UL3d$Y z8p17EhRC>P%>%dk{Gc~5i%9uIFh&^q;x5pKHCXBZ4Ea!}{0Fw#YYCY35xteT%=-ILk6eVo2IV@tJ5uPeaD8(GmyXRf ziz%(AnkiC2a3xvt>M1!pta+SE>Bv6Vq9KK~Hg&|#rFaQquqd=eu{cV)0-G`IDZ6XY zLR7a=+oea@;>kiBuGtR~9g*7=n+({1?C%Y>Xh@vga`GSxXZETGS#1)L$=tPT(e_rL z%U`Zn)I=-rcn)sB*r=J);UcFaD6McS--RXY30#^arR;{mkV8#N38!W(*p97=#6o%roF}7>S|-&R{@k(ot(?2%;*D=tc|!qYKHn zT@VB639??y7|&L&OFTs6=3v@HVrFMtoGLN1NCU&4%~si&WaFmWVVougk{UJ4?7G?# z0$&A`#I(MNR*Pf>#SlQ0&O?znbKeBh{N~wqHDjYuV+dpB2HpBs<9JV`t_k!0fTG6H zL4J1WB3&XLCkKE=CI8|@P^!T`J8))QsH2sc$QKnRq-#3DO+iAPoeWa4I%3Rj2NJZJ zy(Sp~zlf^Bu{sJFQc+`Qlozz5{h~liy~%q7*d^NDU`T`t@&S_9k2ef<1=M0B1#HSx zw*=>gR7_oY_ z4o)C8ws!-<5(rT3w${ecu?}d4`J)aDTV7p03aW*sPieq}jhfox5>CvZV7N@uh?O_n zLth51b+jF()CZlBpN_-zzosTOvJV`PeHbPBrmCGCbDqzd8r*XxHC{4s(R|cZZM>2y zB@Pub;@Mg{Rbo!&^i<&pu+dPo%Fd8EnWm{0A3&`vye$S%*1#Ik6-5}OnMJKlNH+M4@X*}aUCQ=m{?1fiy5TlClOd}8UjsqMicA_E9=IJ?Q1p{ z>J(E@zO@?0s$hm0mS|wnOm3+=QjiHZ*uG#XRY1wd|HTVMa@B;#g)D1zw#81$tS;8e z6x35@YY33RE`9Cht?WVqEZBu$bt^eJKRjI!+6l|a99fq{!rGpQXT@{MkVQ6T1y@+r z5D6`uZFN**Vf+rpRmn!HFpevJ3FEl@m&l#AEV8cXR!UtoL>BfkDMa8!2Y*o?Fclfm z-J>=v9aCY~i)k$ggM76i7?KU5F=2j<+m{x$bn#MUd|^xnPO^s`>tlR#sG( z7jxTntC@}ND=)GyoA}-=oqh=!ID9LPGOO~kIAo zhJpoT0-fKx1LEv0CtDzgN~TTGJz?b!8>_IVM-cc1j}0&kag$*;g*jpPjbp@hPIQU1 zdb3e_b?L@xV{K7&sW5t)gmv|1`J4koSpv1v;p7pNQ%FVxLT#eV#$|g0na$-k`A`ll zV{Rxd-CR^&cC|2?hYVyBumB4**ke|ExSq!>+q860{m7~~>MJcX%3`=}bs4J%&|g~7 zX>2X5=W!#gFuKZ6o8937g-q4kDq#XZMs+#Xj+>D%$%j>f7`h}{++ruSVjVD@rRiRbB%lrJFZZR%-$NYFbN2tCD0(A%A>fi^1OxC@v~q2Xo++oA54! z&EBEY04BGMtJr_8*@MB`GTEybK6bZ(%NBjEwp$&BUB&Y*+=^>$I9l65J6t$D2lIw@ ztAbmwsKt{6HK8D*o7-3c=As9?IQ?xs#+jK`f@Jp}Imp;2Gv&#x%5sa|ssT!QGrM}z zhSH5jX~pW&68@BhadlbgHnm?mO5xV($#{Z$N$l8Iu0}%*uNX#27}`&Q$i_H~is>tF za-A|zFoayPdsp9y6tinDZ9+0LOCIQ?Zi}Sc-qOzZG^6ejr0Z&LSvsQHLa5Xcw=OlBoNQJc=NrplnmPh6f_u5Gi|7*KuPWh zJko{488%VM{N4mf!pqh~a@4>VU6dQ~g@}lWN(cbVRIZrN-TElcLQfJSAFM`uYeS3XxO2C==$TqmdXxOo(Od6$BelJ~h#-goOew^Fi*Q=S=AB zP8@>`=xZ?dvpGaDhQ9)*XEB!Y83PuTAv1suB+9_%HjFF_J!h^4*|ne*(sM`?t!O?_ z6zFJ=z=CbK))~V`Leqk?RlGpYGRd!s|9al+SiU?F$M6b@TDXnsNw(uliz>^nftBE* z5_KR}NE3)Q?J1zQ|0IQhn-$X9gI&D5hE!B zZwe(}O$b)?Ec5i77|b+1Z&~J*w$8~;Js038*o^{sz6N$h92WH5sQ_kR8)v-80^aB` z$U&y$I{=!b29`vaq%jFH3_a^4%>6U-&_>bL>P?F9Bd6^FBy~Vk+uuAgj;Ak3)C2O2 z1xPT$hWrg7_hz)Rk=6ibXbGbu_pjRP=(l}e>*&wW^9bWz$G3m%qXE<4a@W>c7P^= z=rA}x3GO8H#Uf9bW#?(^C*edoM+#{$UeDXI4C@A2r+F0&?YWpIb{oMcUbLg0F4srA z;qC2(V}J!7t8KMu=+vOuCM>Vg&EyMYC8foiN=l8Q%C!|m+l}f?n-?gFWM6biq=->O zK+ODv8);9{Gn6B-QW%6@`ShGz>a;Br^N@j|g|9WjE&L1FiF+WlFi%=Y23Ahhr8blH zOvaXD85$D)z?CQTnhDFw-x1=Q+e{ULsgQKp3Sn3}+ENbS$}ZuZQh|wwMykY9U21#bT(+2VsTTBRRslWtZoaWvgY~YW z`m`P@!SY$JZ;I4lbflcvQ)I8R;+-8P4H?ag$)_;se3M-lm>mKV9P?W`6G>pw5;Fmh zL#L~!lFW!A4kaG8=53{>Tkt?b@umt8u?nw47;CmxmDHJROf)KD%`}JuD-EilpiM-27&11C66tZ_UGJ~Yn3O!A}RWkVD z)&+<+uJ%mSjnois@-MWzU9vdqkvPjhhskiTsJXROoF8C2I4uhnni7vk$|dIRQQCwI zn~g==?bLC{#)dh2qr^#KF=z~ax**y(fOwPGK3hoAvGJym;cSEGGK?wX7rm}XI@R^* zlT}r3f@Q}|8_ls!jHI^0tzk@@o^!#*KGrckJ$pUey-u#fCr%j?*(gqo)E+>CWB{I! zBTbEU9EwJL3uSs#7zZKwF)hgtENA+`oSpH|kVX#X3@-&PAJLUraSd%K@dty>p>HMw z@n#%*$*gZGV647rPlu*f`5qcO{ve3!SZ*$v%w*2IAakbq&7>*}+`WYU0LDex%R{b6 zxDCwqY$`xtO3>J$N60H|h9n8pZAIt?S!vo-ptu@IXGgeHRSWI&NL?6Chc=m;JB~4o z;TF_v;HeK-*s2xcr9qY*Fp-q1xB&5Hu%t{XTF@a}b}ks%^LrzsrdRRc6MNe{blG!3 z#|=#8?d*~}1&(yQk>A8+BZ(_pn!QeGK&1pHh|JMcwGuQAnn#4mPmg=q-!n*Erdm06 zXLw+w&(&<1RA}r%Z$^DJS7T|(nz9OjBhBCwq2>()ple>fB^*^=nA6CvdETp59g!rpmgGJ+^M!R&|;Eu}Cq~ zRaX{mtito^)wHry(pt+;Gbs`ne5ZC>8NE$cRbr`Um7L0oDmiNg>Ff{5tGHSS;7wJi za@fj5S-2?GQj)G#IRc7CcKS%Z^NhtAy=5q0F~qrr4)#zlsVvRc7HLD>Y{a=@OM$sc*l*GcE)q;4zR6KFRmdSP2Sx$ArpErXZ3ELhKCl{UU-(1nJ127v{+-=0g{O`r=qob8LmZ?PbHS z_}^*@Jl&bLB~}OxtvGHaS;Ts38;C$Yg|(OHTU(;(_b z4JOT4yE~GhAxCr4N(!;2z@QVuZj-IJ^mJ4fX&k#Em^;ZpMpuNaBoG;pYyjEYBN40* zTK4ETibd9)_u7K-L>qj8=f`BJ8!hi=bYjo7`p(3+eL1{3K{vttrUpGEan=y zq91gX1(?Ct7`>I$Dd)t)g6;y;AaFU?hUXWsT*qOv-4)t1e&qKhb!J{iiQ`u@G2$>K z6Fq6Bh?PHWEpgGNbUvVR1}u{7*R`uwlnpZsB0cWsoG| zIQ`ccYof>G(@*hGLeEzGvOjxBww zRc-C(nt#A`bcK?(k_v&+XO>l9jhmtpe9gs2GTBZdjm2juBKz3u&9DO<$^84i$YG?nqyeZjaARq;%%v(@iFfl#&%TZmO(M?#ooM zp;*Dqi8c&bD7$hh87zTMphW_XZR#iVQ|C{ZYRTh;+}9~S^^xYQPGZy@D0=^=1@ z4=3g6Xguv&G8+v#pJ{U$QS6vI=_2PXkS+ri31Q>R;e^PwU~C(Y?QhBS8Gv^XEWXTvJID#BtYoPQfp^jD_zEY;OvzgmbVH#!lH2`+y56YqeP8cZ9 z5R(SZ1+J~!v=vHVBD5W_vsg@|^_W8KiM&b&X~^UVw;;oJJ+KQ)dni<(D@bR#)-;v2 zo&=BVdsvg56OdFwh_N*_G(I-ilY0Q;Qslrua*2`=f|TOSBYB+#*!jw2@(j-Y6EP$x zK5>bY{c1##f{-HH8g(ps!2ghC+!p@n<^lJ-UVYG3r1$urJnI2%W6Or}P1}sBwTkk~ z3N9&Ky>%^bK+9?l)+S`!3@4!48^@PTbTJ9y3EtKlm2hP(E<8m z`ADB=jX1N584ttzDNXF)%;#P$W)?__5QD8*N+20l z&XIp0^|NJlR+0B3b?xBVF*poY`6w*}&)OyD=0%xvRBGiTwP12~&iKx1vNe{@R%#gK zlXDHU(gT3!Ndhle10skQFn97ug76OEIwx7eat6GR2975=QsFWnGNb^avIT8Bu~od% z_GytbGsJ4KlTnL%)M6WpUx(ysfaXwh1m`i>aSNF0AT0sRGtwRxRK<29_r$+K+vA`Y zZi7Yx>PVA}my$U}T^SIy!jwhaPsp^&N|u^1b~`fYmB}-hT*71HrL@vmT~<_;w0sk2 zaiPSCGbhdxSztA#d&>#*uOF`VkLD7XJo!I~uR+9n=Q!|rv#h0Bib2gq*oGEhh+rD}R z*PI=dc0x#VuUG1i2;(G7t2URHRXZ_NZe;VDCwv)VZPK&*x$F=VNl4;??5g>@XYa<< zxL|z<_x>O#2~{^UYevd#=!x9WEZ9AUw z)*-Cu@mg~%x%DCqTSji7NXu>v3L{5a+hTs9idj9gveHclBqdX5GSFTM(ZX2 zsPZ>z+TK7vW+N#-Qzn*C<~@8q@tP;6I8IbmI}fXPG905^Dj^d$oZ6u47~L_S}b8oX+%CT@Eu>0v}4RH6pb~UC2htQXoT2S zSyo+Y6qgrOY$mOEakbL#<@hHKM=-m124xbnarIiGVrzMI*=87SvPKK9Tx-gKIq7zmy0xjhZ z0iIOG95O+@44IdxN&|J)x6YI^bs95QwYjvi+N~+D2R)!ft2b@kSW;A}QcD5BkFAqg zEVV2LK1*z7I~20ZMw9?kCUgK?mMOrL2^|3O_N;1IBn^tGQ_4!P6zJ>BSeQx`CBf<< zT>VZ>wbJdBzZ&<0ab3Es#2uXhBywp(ivXhHMI|M4KigoYRg@BeHANY?v<=g9Rhza} z7Q@`2H6kINTVp|F8SbQJgRlX@@Ut)Wn=~7gcmX{V+n|(5=*(F9a)Mc8M&lZ`cW{{b^AKsLlXv(z9EK)E- z6Q%&4B)PjL7c+533+)&LBN!A(Ho_4qEWVX7&6LTp%4RaS#Xe>oq~#7P3Y?WD))|!~ zSi59To2F`A5$#;(}~6{SS87! ze0sp?9;U}wqI)c774lno+**~%KyB=S;1GWvn2MI6WHS|ZSlJ;M8O8-B87gfAx)g{b zn(3`2x#n?0GEE^1aUgTRr65yMF{Z&zh1Jn@=F!A%tv!X_-jo6YVEL8;+%CZ#9?>0G z1LkVHSFygK)wgyO(DqOPtu{4&a9JaJaimAkpg9ZU6@^H&Ky3IaA+(HDe!}`@3eGUQ zZsx0Zh5XLkNz_q@VhtCz{@)ad(IG3cMnkrtV&b+FtrzHG5y{E2|0}>!Qn)tK(S&Uf zT5Ew`bc{)qMkFM|TDC&sB)(?iAVcBuzzlqVvM){_#CzM#;b1|erD+ds($IS_(WIlz(9hp@r+Ld{drVL8uuB*Wi ziVg*%kV5UUD_jrTFf_7K@DkThdRqdvdZZ#oq=g8{cu|_JAt0>NIa3o=oMHTJawk}J zWK+EUZp(pAVbdlNf$<(u5HIC4?J)}HU$JEO!uf^EsPov=B@sOQ;HwZ09wFSnDv-sGU_A7F|?R1O*+SL*X;Qf~smNSsP^uHty<9 z(@ZlTPe|7!20X60!pMq)M8b-+if)SZ)}SlasiX;=zi_vz7^R=GH{b!(8eBm&hYX=V zHPoRaSEJ4bL+YBP$PsTVYV>}ppq!<@QSlU!T&3l$n{iC}R<;4@l~Ss5EO;3zbi~>N zgPXB^dlkE-4c!gQ!<8Eq6^g(pbqSscRlz?k#H1ldDJ>~3X~0@rS+sebQL?pSGdE+l zUCEucPI9qOR&EqSx1(fBkGV4+o9rF4gdh&P7A;0ibIPcwU`o{FDMBcP(e)fdzB-Cq zOGJLH$9r9$QM&4~jn}wonK0c|1a`a7h8m`#0;_j57w-#6R2QK|tI8@$m>$ThZ3mS(NvWAc31$$EX*tL;y)`yU)=B}PqG+>G zRgCk+wmlS&Or!{fk|L8@rzo~$(>B;eIA;j!iptj*co;?QL5Vxzm`P#LMm}9_kktXB zCA4t9VZP|6B-~xs%J1yU_g4%u&SBIl522+??NYP}F%2M^y?5`9NrPj_P!P{wk%2j~ z#^984lM?yTBjmUp~blw=%%NXzP{%>CTr0lySt5zk=sWLATc zOUe*NrWs8_528>rZ>EI}xL(PCW%D_Sm=SyXwzl3V1>9Cfz~>Tr^7m8BA2I?$5P zlv$(6RgDGooD{1w}j>|q}(l`FszE=8>NqpMjkW1vf77P5K%YT z3V8iSoT0}e9GClnE_>`Jm#s=zwd`w?X7>oK84oMkmuDLYHbHyQn&R9+*Idly8$g}_>Q!hmo^cbLo{Rz?5vRqfR4?f`(Yy! zM;YX4PHO|4LCqyYdJNZjbkVFK5^bRnejFP^H7))ezvtBkT5(_?gm#IwJ0m$PmFj$} z(<-H-nb3)~hH8a9B{l}ED_&J-BcV$+aV~(_=m!YWAO`eZ4G~!BW{gf+NG}+f(L`ZU z7d!^hs4h?j)VwzGF<|LQJda(3`ClLy2lT%{H37&v`){7Cv@{Nu<~MBva8LuHUk30@ z#s6$fhT@$KAOtOSxHZHM#)z&%_RFv;8z@NH%Ro1zlSQP#Ws75799pGf#1jGbLezHW z)#i<_UhBqdkftd@Tt{WMVBC;n2K|sgbnXF;DkcY;U^49n$!h|f|A1K$cAc=IY&*`; zY^}n&&wPHC-MVzCl~Iew#W5Z()RK@1$9Du~n-6cRbuB}ND z(FS zLXRc#(`zn#mS-1b>D5G7+OsjB1=S%jN!uFo9US|iY-hG}BRntqoXF2XC=|@35gb+a zI1R4!8fW#*ISO)-@`yxQN?XkoslkGuo&|HR!W_l8H#)dMNW16O2o{`kY6uI=02{-~ zIN-iP8jWPhmp@fdz?MUxBql97O_TW@YNfs1LAl{+c3gGx;o14YZRPQ3GUuc@K#MNU z#S8@lJXCa!b+AE5i@K}Cuw@wu4Of>IOEne~6Ho<5g$&rjLYPM?n5?4u*lL?x>=t1l zmn@fYLujV9l9x`3N~?TsNPM`?(+Pn9PP~JvnQ0bp}VAu_*0AY zcB)ZR$FDrQz~-IRR<6?omV2Q{6%FsUm&kOpJ6UneXpmqoj*d{=5-u##rA)12k-oJF zN1-%UuPtfGm_r*Z`u&;;FQDEPo*hiZUrhEYJ*5kK@T$PEx?U#2^woNZ^cCs>{5b z+cak~UQK3uZ9bD-7ii)KHI4L2PowmhQD(i9Ae%9~E&>!nTP)(aVY5({!HFmGH8-^y zyUg|o;z9>rIczMHjm-~e@KYH{nD7JN5-dT2K2J7n5-8#uvvr*aR+Mk%%57?Th8oN< zI}@=dH`IpZivnjdZt5;7g-ym8!u*{?j(1GKci8)YDa9xaijLkOV(df3hLR$A9^D+J zB`=x76D8ULd7))zq>1UjGz!^}6at$PjxWS0hN)1=|GF%c$I*{KbjJG-9F zdT@hf9B)g5YufbXia_(=s8Ea0Q*?PudCyb{c*!fTqooBZ!eG8OJ7%;n^8y?u2Y~c- zc*dMPAxFwGn4@k9@`vV?i5;m->5=22B+>sbdv60CRdwwT?{nrOlMpcqYE;BgK|ulr z2#OJrK!P9yB8eZZ+HsOh$jD@7n3+JZ_W!0f+SHFvl+%`AECFUuda}%8%XRRyNS4pXzjF0*)7)Wd zr7feYftFeJdQ-+}^xbr`Z35@3V+sewFxKMX)bmh?*SC?W#pmCGlNZ$HF>f4Kuv;7& zBGGLY2U-_nagcE&A_+exR?M+jon$;#CmEB~NoxG+K=xz~)ov3k1f%lAIq3yBK3&}e znu*{^A>3UiSN2#kYXr(t3QmUuS@UHHmK{aAFta8_E3&d8qOsO+AS(v4GL{vw)sI`_ z&GltuOd_lqlNmDAwfyGxYl*`^q~%{aV}WL{$;trIJs8FWP$*=H$B0`^OvM6px`!$K zY-n>)vf-^IO8StqU|Hi`TA`ng#Iq)CI9g*jTJ4GqbzQMNJ@v`Dn~THw0>8bZtj)OCWxc zJ!)R_(gFx)F=mldc2d$&Ze86SM$W1J+^tO76p zDc2=jA++M7#Eas#!95XQf<74c1SU$n3zjbRR(Nl?L8)i6N@w46Q^|*kV|gl$`Ib01 z1GwBAtNt%sk@B{{;;j`q!luZLDANr$nC3Zr7_oqwkuVP7#H#6F<4rfs_I_OEkgkmP zCMvv_dn@M2v$JQFmUw3&p4k-5ylt~DGkIGMZ_C+R%1>K~x0Jo95#WNxCB_SMM;saw zjA9SQtoaw0c~Ogzq_27i7zPwOsta`i>R-L5N#x$m8d}Usjubuk~Ca8%UrHF6+dArzfl}XoH&&_ zVH6wGF|f!BI@nq9Gq#&9w0TL?XmcXj!yQlw?-bR<_ZLQs%)p zWE@(pNy4pMTT#o0;H#aHLWOCU7+ z*aW$DHJ!tsEsz@MWdy6Ns;a52s#;Tqy@{DY(+98_!Bs{vc*&L|hC~?wZpiR_%{uH*wt9tmJGL#uql^{D4ra!>X-js`8`jrg!-qMy zG81%@TlN6et82668jP5kZ(N>^Exx$!z#KnjGU!;MjM!Jyq!qCtnpwaV>(^mGxb4j13!|Y7PuR_zt=v{J&AUhmq|_*#^iWn> zjHQHe(}=m6BnghUq8q_zP?{WgduPep#^?Z9udzW8V|dd@JWHSgQE<^kUVoH!hhR0% z%w+}mKaQHy)EH>;rtH|OaN?8NZT`Qh0-cA<1)DNe9;FGL7qrh*hVqoIC2`aR=b>D5 zHVGJS7P|Xwv%RxswaxY}_0GcU6<&HTr}x<@pR3oIeJ-*0S$kbY4SW5#SE(Z{QFNY& z^fDPUUZIA_2+*arL@)DxBALw9YcPl6UAO@BZ9)|_;VN_A8WR?mJ}g*taRHaZCazLK`A8jl8%dg6VbK z7ATLuoh#>macMmce2rr*LZTUVGS~;B+en4Ru^1GW^f}L4N5}8rKYYj=P=ev608Z+l z!49U`18q3Ai&DhSLo}yYfc75Z#^P8!F&E=aq)rx~c@`y`Eq;keYeTcIbuq0gMGjM{ zHmkUhsW7g?k3U8__=WLUgQ1Mu=+_)D8AN%%uo2PlUoxOosg_CM4nS{fC;|fT-}y{B z`b-A2s@XOH)e0jSO)}s@i+*ks5Lxx{zTjlQGD|>G?7e2SejJTnl?L%q8~zR@BK22z(Q%oCSd}Rus|*iuzVx zBu2A4AD3eWl7q4rUcaFd=igP-u26%bH22CXLVV(~f&x&!757O<^Sr#!gd?JHm$#8K z9!@OwCA^#CH^`<>dB_rS4Tn?%rYu2yYY>vE3WS9sh(yuhQk)PN7h#mb@ml%^^d^ao zbB#qk$7$O&hsdf7n>rvQkZCYB+if98a7V!EbvRA|_oi1@tgG}bzh>pistx9YL$0j3 z2ClW(RkdL~cG2+?T&#PB9Xr0w>u+d4h0UA?<4_BlFt*C8`#>A&EeGD=EQJ+SzKXSL z*RQBGcamyI<86us0$$wsgX;hrrN}bu9?g&s7-Gzfn)3~rT79i=9iQWiGqG^~N7^k^ zxCw_A6+^V3>4)ibcWJEf&XR_Ex&l+80po$qGRH>e%^}sbrnrrC%SKw{je8&EwVxH7WH)?nzdn*h?GRm6^;dw~3>n`r{FKP#atOU7DG29x($xbj;LWDroy0sOB*n*|S%8I4<$}mu+ z)IoGKxJwU=!-^lDP^v63nNv3M-~p0W$yrXHnk-VA$+3&UaxIPQ4PkWfN99HiV*?2F zpjmIcV*jKt;Psz1jD%osRYR)nKfLSK`Ge5!jw^Y zNdn!FfKRMNRC!`*lAq&<~& zv3x_dJ2l)&#cSB3vzQGPQi$8lJfBO3R7r$uEPC!qL~LuTib)cFjw-B@ZECJLmmytRDY zdUddFiqSH>7zYJpdNp`*!XOy}A*YkG*(q?6^2-0-aMl%8l6Q0NkcGLJzZli z`ACJ4>}iS1-0gp`Tx5n!6TxzQN=Dz287%3Kv?V3EOnIIlx5<*H!dk_=;i14-XkSk2 zDddlt3DX21=MuzAia|vk8#!p)Y|nB~2A9&1(y%xtmw04HV;&EO&QLv>bB9q%0P`1+ zLA(M&&Q+?RciN)k6z3Z3sI)&i&C`d2Bx_e=bh z4~E5R>j9G)K3Vt7;?>A-MiAS``GAOw08+7Dy{@v#d_ab6^d{f9HW^$j#TMPw4*eEu zX#Chy0a<pVTttQSzjI%4uAz^NGXaq|9UT z_8C>jG>>l9FEYwnb?s^#Fj={NUER7W4AQcw_8G}5rQQg_LYh05qyq27LBZ++%$?r- z+7aHG854Wi51U1U&lLsdm( z#VY&=n_S4V-WudplE2fk6+p0K*E1Z5fR`#%sYWMKFviPM(7VdxRl3FY{u;5eu#Y?3ol& ztTr%tR=vBmWKh5Bk0+nCYb%!XSRdVZ7!s~Lv$ZG|NJ;1x$~~zDtcV*UG%BCe{EU*> z6p6c(alI$~Fs0%VG?Q34tTzT>l0JA~s0IZ|AEdKUlP(Y{yQVD~Ms28dcBGNYu~uuC z(_$H~B7`ANIKjxN>>2ya<88ogA~kkTZVJbyqG&CDD2PtrSR=0|LTkK~Uq=e}O6I$|OIkjTLJHcqnCQCKz*RLJ7 z&XPn|DRSYJ-8p+lgC;DGI#hK!F4I(1Y*_md-`e${&GB@4DhuP4cRcRH;)NK0OcbfjsR z@+g%|dD5@S0ptRWlPF<%ra4W{$OR995}@XWq05k9G)~>19@RM5ls0=?BY}kqjhiZi zW^jaETFP5Jtun?G{c+?D=R(T)QQoDCRbuLlQCb~J2Z2h#W(aiElx@;Z^#`zPap#06 z)Jdgg#;L5Wso1cFufb7ju-QsIZX&6J_oUU4EWsgZJ)zo89>N2Bh)%*%Kn(^#?CMHdVR=N)n6t|h& ziW||{WNjg8y7zQmuvi#)3x3KnP(D*x2mrH}K;^;Mr);ZJ$Sj#?B7=<=qdFpDyoQrr zunfPcdQMfYIvzhuFPRowIoFWnCYcwKXsn3JIUlZE6FW=Ohkc^zV8YtfA2pv0h@5ej zFiFdIBUtYo7^mhxEZpayQd+BW6Kx^*5IHv_TC*w=1VA5SUeZUbKrn=C>Ke4y`1gkl(CBflQ`Jug=0z0 zlfh|dc^kQTlWw*+q$jA8*@|Qq_)1%>(Fy@w}#5cAhmispZCo47Kn_wpg;w-bLY=pqNrKdoW3(pzB3yKg z;dN^X8iJg;Zm^_dx;%Jy+3o!x7; zq_g?VOj>e87EyV18H^q3r1ETM7@N6I7F4B{I@~o=_+piL-RdiSE5V>E(Gjj#U-J=a z;Y50QMl6xL2}e%Q2l_l@U`?_^L;&xGRkf+T$i8q}G-Fe;I&Q>J{#CC?7I8YWGMf;e zpq-B!-NfzP+%xbYDRe~wp9-)!bv{bg4D;D3R-EaG0CT%SROW+7jW(XOjoSFl+8%BE zTG^askBwhzJ2UxZIc1wi#@ptL@wRzke75w#WaE*@gwB?_37su*6M7M`Ykflyr`iHU z0922LRf;g~U{@EoTp1}0y^s4OlL`Ma8013OHnkCJv@jQe13U=mfsKzY_bTT)YQm@M5hi}x9J7`v{ zuUffM(Tpv+C_j%~j74!9A~jX8Fsm(qto^9b4DGf_E+vkel7eKxVespGSNoRJLXCB` zt7)EkCAb1p%f7nxYjB62S@%Hg(6J3LPw@-@&m6%@i=~4QRJh(02Z(I;)norG+vL26 z&YN7Pf@S5#`dC~+6u?H%KvJ}&*6u*%D)VN>0=dUW%Os7n8U7k!i>1}x>iiXZf5_EE zl(XaY$0{3%X`_rJubV(CX+0(~SqLDDjiClsI-)Xyp++i`5mXVyf;EkV4clf$e9BJE zyO6!J)HIJ;*bAEg7OR1T>QG*Jf`wX}3C+)(vW%lqW*|k+n88dm+n(LGq!v@b&IfLS zPtl2TpIL_nCny6Z?3`x)iut#Cal(+??U=QnVl$%}`YGn78o+tPI8B11rU-qA$tR;x zICCO2I>aU+x$i2KNXFSK6pP^v8#Wt*4S~2h8ro)wio$w$Zv`42hVrS%XoVp*IO9sQ zkes^4P&1nrkd#u0fUf;8bdpBer^AyceZ7r5(K#q zCz+aZ&&y@1Dz90AVyrWAlqx}8RV|fjbuEzNgsWYFa;;c}#SO`d#QcrA#}ONuvrYPE zX2u|8)e0=w79pNo6Kxy^q$EaC zEi_uf=EYbkgT-WYqL+1g$R-BkY*e~z&e%~TX8|mhUsS)bn9wGkWKOafdosx?doVgX zZteQ(n5VOlG3je{Wi7A7%1BuirnFU!&vE^RWl)qc+<;z73N5J) zS#rTulgtcd8N+>LqBByJ%|A0~wJTbgAt2kZ6}~~H!g*yi*qr`&q?T40Xi>&metGZ( z&yP}R(U@-$RCM{0qPnqY%}^bP@D=uCpTJwT*yLS7@>ZcN1T@E&)nn=cqa=|{{v<_s zy4FxRRcD6TL-6@rlo&_W;kY9RZ`Cefe#R1`P!NZiVExA|G8~eHo=8*gBU1}~8B5CW zZZ))+Be%A<58PVpqZ+rtOjKmf)!gMtmB6ONMp?O=l~yHriG{iJq|EtR77_j%KhY5Df`^^R6jgUJsO3=8EY6q!|*FFYOpC7hNM)Bm_cY$KbHD5l?WSx zaJx`Q-hzm(jITAPvZ5peh;0-LWXKlOUWRy)O;+Bi0Zmex4eSb5TU$BT7#gLbR@3zB z#w}d3*68xaRxgxe-1|tGG?mN5y@pv%mBNO1#?K9wPokl;U|%xsqx91>wh=#@Vl(S) z{0L>;nRr~TUdA9Kqj&nC5zC5CPG({}A}2WIj#Z<>`=C*NfL8rMqfRG7(n!loV3G~Z z+J8vaLR(B8ttYuAOs7SK&{R=PvKRTEsA^+mM|Ex*xnoa0O##Rj2DP}2IxqQY5)Q|g zXX;cxR7P!cmYmAR<~n(9J8}upWuM>7o>1L#j&%!kFh*U)mF8JjL-jizKZaEhDCZ87dsFuURq0Fwq_J}ic)oLqrM zLp_>%kyo$Mq@mH_nDRkJb2_oO350Fjv}pbUU*i`1Lifk9gops9)<%&OtTMXP7YrWQ z_-@LIQ5Ut<*g2at4y)mm$(vn{eb}!{H}IKAo=a4G z{Nbr;iX$s=P1Ta*SxJ$(kU42gZR)|3U6*+d-`w=;Lz@J(Yg`VGEZC4Py)mm31u%7+ zYROXeaL#0{Dq|gOm?WuWV31r2k;yP0t-`u3G?A6KMnwX;w6>yl4UZ$Ulb_-YZ;mDu z3yj}9sJJFl+D7={D-tOr4X@HD8sIZ(KM=RqH$c1)e`n)!a|w8|oO%#7iu!=OU|d|meMw8kkq?@#pZ&?VDb4t3=7dr18^CS!KSh|A=#|@B6@?#D{5J! zYnzV`NFj4;!BOCr>`;?2u@jZ@gT$WM0JwW#i_(UQ>wL0F$PqR*1lgFHlBc%$ZVkjD z$-dvH%cMH;Kb9X$ceMh0oiSyzf$W0WR4`NK$xKI{O?|b|rNi20+Od~b%1pwLcNv)? zh@?fPuMZL<&-y@@L=T)jgX0vNC5P0eP4z)y9Oq_Y{}Cv}y^Wj{N*AH2ysMKzP`+_e zlRT#4^{%YIZWyR3*uqYyLde^<&~?%+UxBR(Zp0a<@{Kmj(@S#1V#=zashM;6fjUOm z^`n-BdfBBZk;Ifn>IAXl)Hl+@Pt-l)n7r8`rXlbte(Cvct?6cYT3%$&>So|cTE=^L#b`l@}0Z&{N?@f&O8RA+HK zi~j6FnFU|U58cp8_e&&|ddyhH{Nv;h!UhYswViA2v+OWDDiScZrbCCDHJN3~*~4DrfEa))84 zPcG-T#wl;plkwMQ%tX*AMQMZ1BspqouPn{)N3#BohZ$sghW3*HwOABW7->EG>A2yt6a|Rb((p;HF zQrm90W-TkYO_S)XW6S!5y$fxv#n`1m>#*qh8=5>;Q_5)`TBGBlX}7Y;0~&(7t{L~s z3tS_=Nx*-Q4{Q_TO?d1epJV+#F1Wk&0919Ksh0RQ8fyQbr6dkJ^ z4B?}Qm^Bt&NBhAJ#K;Ur7QmPW7!30<>H4{41rK&=n z%OH1QFn=4PM%kyN6EH2NR15|Oue=81&9xODv6_r1^KwQ3;~0(-H!)}J(z0bAOf6Lc z93JeG90{&7FQB0Aop6%d3`CYRRlk~c2eirGT%(12h&+xHO)6PdVp4M{l%NkS0TU`% zLurGS^DVY$*=A($R4p4dya6+w*g~Uvd&B&i>5Yd>hHo513bWrjh)2t=93#>!zWJWb z<4r*>-&(nTLv=D4BP7+dBz_F*S!gRI`YRL98Cl?*8p$ego3bt&QK$(H+=R2V~ey5bT)SPirhvUTQ@|;>zw%8|c%{xxtsTM6;a%KmU%85}KA0S9Z_ex7%DzL6+ z!XEOtJ#vn-ci5R(X75&gz|@J#qgE#9NqKTC?xl^M$QH^0Su2u< zhhPzw-1~#uJK7)!ah5HXBO3aZ>cx%j$wpTseV3!M>R~DWblX`n*5mO{yC{uZ9Ebl{ zfK2)mhp>;*+l>25k|PF6o;1)5{WC+YHf7GV>aLvi8(M~I8Aq7BTWLl6_|a2IwO4fo zbE&&BXP6atb?#bnz+(21tk7nh&q0fZ{Y|uIUPhW!`!hI8t(wIBE>YRSz#Q1h>jP74p-!bU4y}}iaUAbS>Jb*c$}yR2`dP+4HvCMwJWb>M&#*Mb zmKu7>7MjZY)>trYkfFw;wgH-OZ+Nq%LbC=?j>pLgay-LKdR!W~o!l^qpH!Wbffgdc zkS}T-#W*n-dOC+wdQ(z35nrYIuv6uY7}9k4EHN1DpMGYkeJNBkwc%gGJd+A;BtR{9 z+@MC;i~#95!U`9)N1G4JQwKEgUUL|!=!|~<796TUJ83O-nhc3Mfv_DFgJ?Q#ifo6* z5p&6KUb+U2j3AbS9+Le$PGG`_a@~qm>(kEiB~_O9o3I9BQVttwy={gMQ@5Y$jt!kW z%V?AiAN}BAR;8&ug8a(-zh zDNNXLBVLfDG)W~ngM#B$j(xpgsIRo{j0iFQx51?9DY-Exvy^HbYqmj>Dnm9*vJBaf z_JcCmENq5rs&rXnlcmds$u@jDz9yXIyoQvdWaG3|cl2gItOb^4HUXv`+>$z42S;`o z#uBBWKt^)sLy|a=TsB7skeQdDO5Z-U`Gb~r(2*QIxt-SJm^>-Rx*m0+(qx=%k+tJg zlwcFs|IXPdS<^KpPt$6G;!r~x?UQ3SNb7M@vuLN;~3C`+VIu!6(xC&5l*&PgEGmZ(#_?QUIu1MeVLJ5$$9}Up6^j{Zr7fE? z0*z7;Q|ea+$~f>b13{{nO_-#GRx0#9T0Ch@kJUOmbW*G10fEw7vB*`eD>oDoZtS63gU+iC~LQQo-WGP(V|1(1;&nBA8qu=}cDS##TB)kPiN5VZ+i?YHO;SHmO`> zSE{b|t*-Q4Q%low>T)Wx{l+j>*!eU`omDG%#sw`(KOsKo|aL?1V>h^Nu8>5I=i-BFEzF6;3 zH~aBz?3=NqR4xQC3S>mE=2}D1vG`9;43mjoQh)=b5bQw0!ICFzsh1b5XUhf;IV3(U zV!rl1TPSKdaX1^gV6k)Wba7C=A1ZbZ0(&~Scb@JXm@fKH->D5v7wu=3?wKySr+Is)i^J0fG~jnn zZ`V{l9@0dIF7V9QKUwtXB|_}ix9fO6=sW-~{qEx4$zs?&ZO3GB#64|rvgpi<3Ne~j zjMpyDR3UbI3h~7;rcb0t`9^k>_@IQ4cehbqrtoPT7t^9hlurL64L= zkHGD~g5tvfM;8+-*<=B2O=H3Q+Yddr-p_& z#$WB9rmr2JR=Ay?){4MRpH`XNBfs|Y*FO0*!e56ztrddQ{51wh)zRIljc7!JzPmIJ z)!Kp2!hy)J=WgwQ1C9P(&AA<)2k+GiM-ZUv3&2w*kKCsnbWjvMJ2lS{z&#IXQ|Wv3 zA+53l9tIxPc2QM-S!<(+5AV`eN%Vhco_2ii+O2sG!~3D#+HDg5RjqK0qIp!CF$%c1 zM=K()$@e}+blA^dBl7DAe;s>_D7O7^O;-K> z$F*XL=g8v}V(f7a->V(es1$?W)QS%RZhu1S*C_q5KCN&Vu=`1skNtqeEr*^`-|gR0 z<=6>G<>=?{{rr98TiOm8QUABKl0%57{b_9)rLgm9t*i^M<3AN;cm1b|V1OXp5AgRf z#&`8Ad@n)Zhxq#-Al1xRzg9;XIQWdhw?C_0Pv2wDD*TS;lzY!}S|j=2F`yM503009 zRt*Ane@Aof2i*N#?M90K;P*7=KERGaZ5HAC1~ump;J~0ZmA(()doSR~payv`HmDWK zSf1CM190sH><8TOeXVE@`T9P{K-n4nf#&RkL&pnRaWB4izM#n#e(;A{x1$$`?tR)` z>2u^KnzIKEhhNf)cLTP+tbFzZQcQba);hpoL&I8GKR&yDs?8Vx-1Acn?S25iR82>I z3QzmteDqZndSJgc>L&mFztWt$5bfR(t+)r@BO}Vg?$@;%!jHbLId=lz@upV1190b? z3cvqNtzDN1zNJ-C5(nSXJoMfD8_m-VkA1(Fm@=i=`Z+9xDLpM8=TL6<#p(zM|zV)P^=Pk(S9 zLyjSeM_t2?{Uu^3znj`#K_`9eo7}07%n|J;&uT9bJtwaf;@HVfxR0JZ4V~l^dfhPv zuY2U{;VFej=7_yhS9O$#uG2h(Jk2vYNA$|~J*Rn&%@ITLefTub(K+I9@t`&~M+{7B zr?dvocJ4>$v#IGH@>0}?y`_*>=g{lUbMU%PzIL7K9GW9`o?AFPM~t3Za|rRA=fV5f zd6Z)3jHw5Z;*0?eA7e9}_}G5ItiyA};04ZobHvDO@4-2uXAZgd&goS&z*tm{f9s@S zMw~<%pmeyptV$kRTPIJ}oN97k>)#fLZ7}8!gzy=szAl%GZv^IqtHO((j{OV2Gizhqs>wZQEO9# zG;(18CaS&M4kHJJW=e1ld@5YYwA`m zuEp}Q@J29*T28034w6alyx`6#kMXH+Mh<J-&f&Peo^G^}uN=RORXQMx{0;+Gh2g&&id*xOw$U`@u0zrq@OR#({=g1Cba3$N>NPC_{Xhhd&gP-NH?Lq98V z#hS`SxjNcseNz)ysyq>4S84{qFl}NCGl~o!6cwfj)$j6cRk)m%P7wnuP@=LamPaJ5 z;Tx*J2&w^>>Mhv!)bxZ7z z_ar+BDPe@QDVrM+Ez78qH#=7(YOq7NT;Lwn3e|oDHW?%olM39Ifjb+8JlPAhYBb~; ztsa9!F;SozH#o(%alQx@!Nt%DAGgv_vnlx6V0{dHMxkpZ0h0OP9s)_AKP%&gCM6B!Fj>#&g($F5 zD=1wmV69wfBAG|1W>Aq}vROXS3R>$`Z&l77!E|c^5FspMm2gbSM)}5hNC;ZW(E@>l zg+0~&w(BCXP2fgkOtvj`uNT@|N-)L|f-d`li()dv!Cb0xctSGqMA8usCJAi*kYtfv z2ZRRmFbfXqnG7DXcT&YHC(kBN{K`nbK|At9jHbNHH)b}o7t|O@NP`umXLhsb3XCxE zM>(C-+Ho>r|umM*Kq!9>s>N{N2tajm~T5R%wxG;e4;;7vHMQ=#R% zqJB9RA<4=H*5YE}3qmTPOtGvdjYY;$gT+{}2o6QV324>+hFC<=C0QlXll=R^MYyyPG7za3s=GY#Preu=}AZpa|QQx>zh_W;Fe3V4v$7F zO?fmr_moR!WGPjrTqI;KV=bt_);lvD%h^Fmmex3WH;a?eD>8;I(Y^+UJA{Z&rP6{6 zSlP*133MwNCN8RL%f+(!7cVOt+I zt8aXY)$o<3t%_x0xz!^+#d3_dXvBd5lQQY!jR%T z5f7|u&a+R%lZVFve}uy$@IHn7;2q)XfU6t*{v@&N1dPfM*>*e~f6vFS`j#*JGxB|1 z?7d@>IQE7v*4{Zucn96$DB$(3e9?CIBr!ZcUvvQ?528@KrZwgvCny32$)3HJh!Z+5 z6hLdqw2k_^J@g_6vveXGr$|vi=L?@0UWq?UxAq zOX1i9c=VUT6@6W3ZLbSQ4`3hQLBPX+#cv4B`-X7T0JZ}Ty&+tyel4_`UkgV&VDGPm zUig;Kir*5B&bL5_jhbt0qn3AMljdk|(KJs4gYs>fd+{eUee4sOXGfRjf+b8BpVy{N zyHC>+_i3)(_hUf+fTj&Rpt;H()b!AUn%4iImUrXBnwYvpdsy zuBlUWeeD!IukI9GyZIE|J#wm^Cr;P3!qatE_vw1x&S|=+nXWswPuImxukN_*99f>2bj?|-JNDG-F3)wk zIDEZcIPy{5y*q|5t-9Fv|L9sxyDsYP((_8T>tfmcx_;aJy4e4K?z;Iw-QD(OJ%8Y9 zx+v_?^-z!Q?C;gJppRUh%N_UBG=~~IRbp1xa?r))t-_}oB_JXb* zd;w*8Q7@=`S)X+5W!<^w6}@QMFZ4d?AQb?7rrbBLic9Ik_BI`r++93nagHvI(-N7q7!KDf}~T(;PuRW5co7gsp)%9cBv zhgLa6W1~awZFFdTjSf)~acE@`hv>V{q4nSAaBhFlA#Qxcp?5ywa2EDB^o|~fbLTf5 z?!8Yqw9zLV&i&s;+AleD@iP4V+~L_d;t*5+?$D@G@*NiJ5YL+|oWy|5e z%IWF-h*MwkQKxI!4Nh@ey)*CTMyI=|$ti}Ko$ecBPBGZ()H}C0^Ap>gqU3I;r*Nn9 z#3K(m#lEjN9fx;2Me&nP&%y7&-}6q-(2I!kC8vAZuv5Qr*y$YpwNrTBcIr33?G#J$ zU0P+nOLQ!D>HUjc;y}5}d8FDUI=|@B2fyeNN1k>$N1kzs#e**A&L6r&_m5rr(2rfB z`DK^0=w~j^jlXa?hu(CF9dEhv5B$NU7rpBeOGaIKH%wT&#$4{``z{gE-FmO?&M$Ji z^-8zf*)hqj@0sLwt}1eCp(3}~e~Mf8oaz?CX8@k*7KhJv>(jjM{I>Jm`rh;1`NgGf zy{*)p-*bsuKX!>*43z`7%&qq?bLa26(ybr5((UxtxC^>(acgTgxgDdM+@d$;)^{Y_ zj_y{s=G@|T7q_{^Z69}Qhi-G9Hu4E~@!+qw61?$ax~+|I;PZpXo=+-Izv zo#&{zFi)Q`C(kiBCr@vmmnQ}m=H(4u42i!a&%O4FJlBC`d3tX}p6Feb=U#GEp5xF} zc_MLVo^z}-FK^jr^F-rgdCtmj=H(4Nkyo(uy*$_cf9C1kV|iNtSf1YXFNAwPPwx<( zg0^WM$Np&^v46G)Oydz_7kRYiQjc@lB9GR%$RiS$dbFWSfvfU3qE#N};Z>f3(Hf6) z*GD{}=oXJwe2Yg+jd-+a5s!05E8aiuaddv%BX)OqT*Y^I^zC#Jnq|iJz`cLzMu8DcYW6*M)!G~&2MJ4t;Ws?)BL_W>#lQAA zw;%F2JiqmbW54qhCVua6)%?j*P(11>8q@McnLGc??UVAIrIYg=1C#T$-jniA9GH^t zTy}cC7?_@aLigElJ3HUqH6#C|!{zy|eV64=>$p7Mz59xMee8;St$$g*qo*R@wSRfO zE>`5bwpZpmhAZ>)c2?y(oGbJ5c3qh-JRiw-6n->c^xu;2JRCy2x958jo%v_2y*q#U z!R~x9a)18WeGlb3Zhko5z4Jfvy*s|1FZ#Zb?;dy}Un}m*caJ@ppLg4yeAnWq^PRhf zknT{v=>1W?qwL4|?inwkFMcK8G3%%KqUVi#$Feu`MaNtDI%XssXUBzJgg!nGefo*$ z=QZlz0WsS^eScjZzE2}vpcMTX{14+f5&o%Dh9CK--MRrC+jOm{@N7{77qS9-^8Wfh z{UxMNIOy$@hqZ^bhx9Kvx}EpA?stDNZ-?hg`41F4IB930Yt;#^)stOUo#JB z90hU|$Wb6ifgA;L6v$B^M}Zs#aumo>AV+~51#%R~Q6NWw90hU|$Wb6ifgA;L6v$B^ zM}Zs#auoQVr9e@;+Idlg)gkm()J|)R2&yT{qIPUZk>6_Cuc%#3DHgS3?U8g>E1HVh z3-}uw4yAh`Lo9!i-`Kb)poX7AYhNTSWGhD!MA;U6;ix^(a@DhfL8D7e;oFSHe;jgHD3B$`7E@gNH z!(|LB7%peHf?*}YDuyc=UdeD3!_^G0Vz`DO%^Oobsu`|hxSk=+i4*>6h8q~xGOS}L zH#)=Lwfub@!|NG-gyBaS-oWrihBq<%7(*Y!n;H5U)-!Bi*vK%zu!-SDhRqCv46&>Z ze?{$^7}A_MeYY^gay$7QVHjn&S;EqG3Gk!%LWo(N)S`;SYxhu#GF;)#9!E;3w03Tnf!mEBHvwVjculkKl zeB=rhUiDL%_ydeT*`|JLmZ|V|_#wutekn8l8!E;R|LO$zH8|;r_|Go>HyH1=$$$F_ z6<+m^nfW`bQsGs&tG^p+RCuG`N|%qPU#;*;zsXF07vt^dGk=2$Z&$uo z7;l%qTWeK#rLSeCUs(2^$M@_%*^|6R!rRr)FBxx_|66ZR;gz198UIB$D!g6z zmlf%%7x~A}-!qK2qknV#`03AX z7(e|QoUcp$hh6{pbH>}z_vcOH=l}Z~72b}&I+_*Uj(_I|72b}&zRP$uKFh4X*exo& z8h>TtS8P&vHBQUK|C8}{{lgt072ZyN5-kdEC%-?*c)R{(YFLH0%?fW9{!5Iv%m31t3U9|Be`UNKeL8U>Kk=^}e=bibyq)}bo$+@1 z!@FBlcsu<%(KbF_Y*To<`hRq%!cRsi=r6s0yh4tbyuz;iH!$8#zFhDR6@Q+JJ|li# zo*GBdXp;WY(?9)Wg)dU*4E)`Ux8uLBGTyHKo@cxr{SQol|0Cn=^nY!N%D-LuXEEL` z{W8Ye@%IMC+sTJbjJMZnW#7uIpBET!$6vo*_H2f z#@orixs11?ZzbdH>hIbK(vM7#{-+pkN56*|Z%4m96NGwg+m&xFo)TL($f_E z?CAgGJk|f$h5y}oD!g6$KV^o(+x1U{=PSG&{kmq2AOD{iZ&$zW*($u9{xFmAcJk$o z5*6MK{}0C7$;YC(D!g6(W-{I`{5uoi4_~6<_t@0$pBQgfzmF|e;qCe_*QE+?7yi%X zH1CmY8BqDecZ-)yZ-yGt5o<4ZQ{TFqvPX0 z#dy2)t8P%?ud|8&A;#N{Uw+JZuTA)qZyLY+C5*R|zgrkT$0q*&m>~XzA5-aHWfT6% zdWBzLgFnQ0JNv;Gf-1b%Cj1`8+qLgkCkXG0tN88sFC zuc~m18D7t@jo}W4{R|H<9A#L%pTje}o?#oq9Sr*!9$+}iu=r;jp5gTj+ZgU(*w63) z!%>FCuW@*W*E4KmxPxIo!vhRQ85aMX!!x{|VH?984Eq@#U^vRK{h$iJgJB=TA%+JT zjxsFzwemlU;WCETGmJ9qWVnlAKf_^$hZ%~uRD8t@OBt?Wcr(K`hFuJM7{#4Sk`;#{Fg+_o$6~yN!F7jr%Se_dXlAV+~51#%R~Q6NWw90hU|$Wb6ifgAlxui{;k4|EJiuWVOV}I6L$3d>W82&t=*4u`Rlp- zuQGjTodlIT!T1Yp@NJC$IhQA${x`9hVNwjS&YwC ze^kyl8UH7)zi0USJJU3)2oWhKgRT=M8TrCG@a5d66K6f z=ZEgB^TNpmX@G*uzVE8)2-!uF>LnrgG(H>^- z_r(k=7_Mbl%kWgDYcqd;is2&+f532v;qMuGxt^*ye2~j=IlDJA`~<^C8NSKT$$T`0 z;kgW#Ff`<}A&(8dGx#yW{Ge|Nrv|^ z{3gSnFdShRV>-Oc-}(-fP9ek78O~&Q3Bx*uK88Vt?F{c>xQpSB8NSZ&q%Wy-7BO7G z(9f`$;U^hB#&Ce)OAMdp`a8tm|6o`klYL7F z9`E1(w*N;Tf6Q2leY{z+_{)X=zoXw)NzKxB3sLcr?oK)({iVB8JETA5F52Y>eyFYH z{&Ez^Q6NWw90hU|$Wb6ifgA;L6v$B^M}Zs#aumo>AV+~51#%R~Q6NWw90hU|$Wb6i zfgA;L6v$B^M}Zs#aumo>AV+~51#%R~Q6NWw90hU|$Wb6ifgA;L6v$B^M}Zs#aumo> zAV+~51#%R~QQ-eO6!0P|&3IN}6L}jT?LjmCT*eC^?ct++dh{pv{VsD0F-ZFm)gOgR zy31XDhI_Hl5VwZQQMBvvwBdOGPcNPq@f^f6if5`Ow8eNnh9`mNE<6w5`5K-*c=qCX z3(rwJ({-U;g=Y(%U3hx&4B+_@o?qg52hS)Tw?k;fc+SUDhG!X`b$C98Cxqu#Ja^%F z0MBE1_Tu?(Jb%OEb_(rOJm=%N3eWX;Zo#t!PdA=Dc)pM4T|84=Lfe4nc09d!ev0SM zc#7OYE5TETryb8uJdfcS!1EfO|Hd2m3f( zFX0)(Q(P!q%kbQY=YBj-;n|003{T|=!qtqY6Hm_x!ucMarIUs0Ry@1#yf#^APTYFs z^Hm0#{H>wHhSqRkZKS~;3czh$?TSdaAr?pkY7_p3P0K@(hD~t3CcHHmZuD&k#9Ld) zo3EjHQ#`TB*XWM{K@NdfEE1!aV7w_94kiMP=8q=&Xz|B4#gLeM541Eyw^20o6{oc5 z!`I{whO90dTm3N^S3J>(M52UEL_(3R0VL;71locLf{~!`)!!P3MT8GULAC)Y^S+IN zL?TF;#%CO$F&K~f6AjJ2jj>2;)Q7Snk__iSI6?8Jc@72qTQdD_^-~s9Mw&R4riO3= z1!-ckG&O}<-!yadQ~YYe0oRLDShi-b}C#y~uHD;!(=C}jdf_r(LDCLcK{!nA<+Eg&Sp=r#%C z0Yr{(Kp)71!iVQy3>Q`ZEv;eoqVgS$#Di^!t#M0`D9WD!L?7IUVntOkqKRgDLER)` zzQ#a25sM%#iz_ioniFMUb0m)1^2g)8Fmh>xXo!FV>thjrV}n0#3MSp-NZAjzH7M2L|MWJ@bJ%clx$bu&m|xDsuJZ*x2nL!OX?B5}N-Ine0a77T>??zzK(#*%mO%<>xa!t~zdi&x z04Cj-fi;|~>(;K2gst5cPXt<6rc?&D1RIbSOO)g_(h^0I>!XQaB#efT4G+rK1fzja zFdV21#2aG4Xo4h5y4RJh;RcEY@(N8OoVeB>Y7JzG5;7~;7zj6Ps|_S<}||MJ(nA?^?2uL{0>ofml2OehenI`L>h_P&60NbYEHwuUgdv<`ej*Cxn;>ICXf)Ra{F@NQ53+(K zWo#|+jkSSr+E;BWa=(CmM4|~_IIz|96$`|pXo&H;NG-D7Age}3#wB5|=s}Q@<)b21 zTs!y1%tFP9MqsuG@t{mdx=?BO8?ve;7?wg@;=%O(rof7jKiFbQ(i*kTw_>HiIEs_u zKOWx@fWQW?WHN%`nK?n_B+wE?1vi*Dr@E@HBGtOq*WUsPW$=yR>`R2U`068(L_87m zN0sUYrOGE>sl{zx|(w3a3}Kp;i)g)F4IvTK~2{EXmj6M5h>pSD{~BCQhjf zgdvJVnNxHjpi>;o2)-&U_&TT9VFa(^;BlcF#i*<;t43vpM7gqu|twG4X=QP=yJ+C>%QD}EU5Bax6S`$9;LrwO=fw=1K4#2}9gma2x|AaRu zbwOd`U=XMk>?Xdg1zRAdeBuo)9B51U)EBz)u=t0D9;Yc73PFQt4Q~?fBd*cz(-7Bz zuNL8V=RKz(;aCg0lSr(^w>1#l*qrboej?P5b$)q87y3fO^#$E0ey%&k?p|n5qT}UL z@q6TzQ}EmS^eI9(n$fYv4bPupu~${SopzbQo7N>dg%X<`E|sNqUF5w`J z5rkFbwGj3h7ZFxDi66U2ISMvdJc>UdnSmq7hPyG)5NQPK>~=tj4Nz6tR@m5kC*nHw~m%(`Avu@}@&?oF;-iN`-vE8^XRcFQ)KXr$Fn|<{@sv0VZ zk61JY6$@Q>Fl3PJS9!wG*eL#~$2PY3+YHz3jzmKIIuFeK2M-9jbC^aDV`KDw_#WH| zsQb}qe17yyP*|V!1QR|u_(&llWe`#cL?G>~9*VaZ^q}s?PDa>W_Y=n)*@Ig-!MSLE zW$+b6n>Et?n}yq^8%$H)Dn*#eD%x{f0Qf&PF zV!qH@#ETw;ilc4%#6CpV^9GK)a-f$Cw?@StPozcsJHIssDUFgqS@O4t&lOO6cT>h+ z!2nnHZvpwFr2KY4YaG4ZbMSEZy>rRK-(C3a#xGJ@l4ox1fry*m-$flB&yJJfX~Eh1*H67AmkH=DQgdqDC+nuNs^ua zbs~N{kDP?xor3|g2P&fjjdr~k4HUnAED4-bzNQ=bK zAVOWD5$o`zxcJfuXr^Doz*ii;gMLT9fP>rg#lBL*(?3q2G8kX)pI{bxe>q{ZxMQ+Y z?0K4kj@?7Q1G~~9cy_Wmg2Bn=2=+|gEMA0*PyEMZ)Yoqi*P*@CmX1B2ulfA(ZQ%yd zej=Eq9t|cD*#wRJuanV|K=~GGV&YEpzC-^>wL1C+3BK*4)CRkcc{D2QM&e}gUHIww z3r=v+lfm|$sF)=tUPJ8ucr#TNnij;RNmBQp2-VLgM&Jc90(B`9HkyZG5z-=kDVBfg z#E|$4(%b(a@q6blsq3T6Hxl{l5x01;s13blBZh4e)Ql7c#Jt=F_Z5Lle?lDF`5bIR zy7XmjrnjFIZvb<~t&O?!Boy0OgJWn3_C|CQwdamXJ%ZU*< z9r`AP8F=no2HTt0_oYdk^w6um)=HEiva(K!LKy!om<4M-$_g6$Z zitY#ea91&Ycl?cLAZd##RK5O>PUe^4lR>J_QbD2rfz+>^+zLH{k$(cRv+H!k(f=U* z9{f6P`*m>dz<%zT!Y|!Zwx9_nQ+Qws82U3Z>^~`bPIt<-C^3@y8lFO3XbgqiH6~E)}@!8lZ&=}G?$)l%8 zmK3+c`nUf{%G=1-DQ}9-gQuG5^CJ8ldxWs)dy@i5>^pUfDtuB7a%K;~f7dYOuKUG| zl60TOz8*LYasYU!HnQDV%KNP0Yw$FoLtd7m#!pMgOo9jH^#m^S&u!PB-V zT~#%4qi!G~IJLhZQ16!|-JU%IX}0gcJ>`0%6uCaMkdPk&zRd9b&0-F=rx?}oEST8f zglCIQq<$0F;irgYcR!Z$!$JRL#PO-qk&v$?f)4Pv#W5BjJ^@zO(=?Z!VhrWQS58MG zdWx!KcrTSQvoNooZnXJLMvi}hKs)~kOSMkDpm_6i>gZu&qd|}waQW&38-wALGDAaL zBXs8(ASv1hY6SJh6jv2uQ6gVFL)AeR)V?!BergONj!u=s84}WRxaSjnr^pdUN|}RtN8y*h9!`|H#OW#*1mQ?3*$5|ZNXU_tKdkIOsmALmT6^9&Onfk9D zcfiG9Eh zy$RJCDYONkYyZ>>TkJb<-E+6p-|wW~p-*EZpi6Z(#RYesqr?)8k2tpN=fGD|Yc(O- z&yjQ>ocR1XkSxEIBl5>cVe0=frXi#}V13`B^Zw=>j{A2AHrP)Ay6;2)N7NrgcK>)z zc6EpYb)U;G51dO)f)XM|-F5E51xB8K3e5g4(np5wh4Nuy^CL#!Kf}x3?URATh}mF- z(Q{=EIR4L_hw#54-F`QIp=)6T5W;{UkfhC1=dss*cKLc(3b_#H8RA%D4h(FKh-hXo*wJx%^ zu=n;Ese-W6oijEmV}sZ+!(6Z@XUGJ(GhxsFX{7rdqU` z;lxX`h*00TASU+DLUVbXIIVLZk#7y=iO>+oZG{icwiHNpof`vTwo;2nW<%7!N{tia z#z$wPcbz!SKO)SrVJg}n@&PpnleXi~?&}xQq#$}+SP0t`fPznkB{tkXaUq5apQ2jm zd7pA(Zf1YEkooW*7oy|9kPp^!^e$T?u}w6b!sH5!2bgO6%7xO3?-Sp?FpB9!9~zl> zAMuWKQoQ@|YmN8Eb2#2t=Ad;yOrFLbO80cs@btGiP_1Cj2IKsE4qGq97(yQYFUdUb zp_dZ7+Rw!Ap7)^#IxM60UWBYl!H z`R@|8ZSi-ozV5)v=Xi}E}Zrk$Y-gd&kmurcjNsGb)nRL{?8AWJerbX){$D5w=| z+9n&Jxa%V5X)lwxsYP^Mq%2X&!l6vjO*D>#;S3XNkHPDne@fHDu>ws>sTSD#e+K8> zgXG-%dusmDSSpTOL^kcX_%pDhe}Q)~dJ$w@qxjWD6zHx}$a|T&J@WSuvH<3h8L4k= z+=yx1Zha`jWWjwR$-qhg=zp@Nm|i>7LS%HgW9X5!ew+Jznr%* z1YN4{A|=X2cPUytO>BurN+FpL`H0k6cl@5Xr~f_r-HUh9I^-~fYkOcmzkGi_O7J>7 zI5Injr{~MoCx#5qzk%m{?~v!l$ks5mD-?m$F*76e%{SO9zBRZp)T)r;ck^2?95*|0 z4!RbgV|{_lF_JgM{R<+kp++Cs^kCD038pcr=ZoLM?;$Dr_uNGzqt4HhT6E;^WXH8M zvxgRP>|a?3lH4%~zJ^feXbBm^6!C+FvZf>_aFs|g@?R8mFIDJ(3`#b(jnGxe>eCud zh<`6+!>9V}ph=#WNd_Kim+3*&)A+7X^KFc_A`IsCpMjr!pFdu7I$MZ8ib zT=nARg$?4avTy_~G#-da)1{iZ7vCsD*9aMhg%UtEM-pMOW_=40AWP!shcachKtJu2;9VyMK z`NX?rN15$%U z!}2Xn;M>H#3#o4nqPLdG_~Xe*l`k$#h%aBPxQgrdyBEVI*o^r|68w9SVmDX`cE9Ig z@niN8eV0)0M9%xv@6YLdC(OG8sih&SML=-(uS2EjuRt`XS6w~0R?yi^hX4*dQ>O7#GgCWxfB zDfr-D$&9}9ZzNPX#8)n*iGjHI`lV=Ajg2u(gfzmA6WZp3L8$?=mJh*$tSyCwOnKr` z=#Af{xc7dK%1f4k5SB`yy8!#PK)PJ2Y)evZ8$+J(){WvX2)XAC%E`{>DOc@3AdN(3 z_#5TmBJr(qG>V^)eZ3O{4uL@*aTD0>nR4!OOeW8va&w2y<#`D)jJ-rLw7)`)-H(|= z*1|BOaMxv9{IQ_PEZub(gfxvX_LsMyX!l=+-Wi2L(PRO4ev;b8e*7j0w*N9F@h{mPZhWNqGpvT6zci6vmEJ)~a`eUE;-o~7UY z-^DzZgXdJnF5eWyOx5j6<(i((rPP_z8ZOc+*vsE84{lTr;-01GiUui??Y;DSxSM{5 zNy>ou#nQMKz5=`<{`+!_DgTMI_sW7F#$+0t)28}Z z;0-QQ9sP^T#L0f~1&<-9c~0Y-o(6GWL8I7vLO|SiVw1SP2s(f`dS6>4-M{!jq|euuC}U#H)mH|Td5E@Z6WIX7|da%5`JLh;4rss@df1fN}w zb`^^>psVtUhnAClTh0Fdr9v)6xMO)#d~3Nw3_^j$P}P{i6L;t&b;KW+@j8QHtbN=_ zlXwZ}I&UurT|cKG8%OTekc|*@YOzBtd=d|@0Lw~b`@Q6)>vtNufa2UPobJ;cl&LKs$Qo5$AzveN#e%?n9b% zN$hiXYDdo6b@CxBuO0S`Iz}9edrv#&S-i8Tw_x#-E`3NpHo4n*z}=JI>)5Vs*ZZ`N zNyFMARhUo!XFoP&=sYaV+U_9@LIHyYxLy?{?=N#{oSw>KJx~?oxQE|Kb zfaic?$T8{yRoqJsc(!XhJbNb(>o@Q76qSv-J%tCg0c}JpDMh|^pIzLkmF_vYPaAX` zp4_MHav*N!PVEq)87?@OcR<^&_35M9SV84J*W!ayI$XyJ2J$=y3q9T1?z|m(ueM`S zzo)z4D8iPu=R1#`G^X{Pa43JLr^7W;yen_U4CmNXr?V%&r=aJw!+AT;@a&&7JgKDP zgta9*3VZajkx56U_B#(1^q$Z*QZ!g_?2H3N!_E<>r{8n9(6c|U!_$@DGiiTbr@P~X zZqKys+TJHs`w2D)74%IwSt>P%>xC)JOeWUulU-$d< z8t+}~gZB9S9*^Jmemw5&^W6J--q&?s_n*)G=h}7c8C7v$RPTh&NgX5FkJ|q7o)H_5 z>KeIiROjgKQCo`-9<_IL&xrm}yNlb#>^Y{lc%M~B-${KVyGL#;>9BPg7`4x;Y-dT& ztNO;QJ$mQp&XRp2D-OJF+tKs;&OA7-WWD9xKem7LrsA&AeWUw}50>nGO<8C0`cu}8 z+H&mLQCo^Py`p_|{|W0ytu3F?KFZeZ*xr-&kJ>Z3ZS=v>`^F56DeD}wdECyDe%oN} z~9UE?bD zy=>-|l8%xCW!pw<8L{>V+F;N-RuZ6zDuw7sOaq`tr3 z*1mPY#t{|0BW6`^9WiVB(e0ym*cz=Hx!YQ)Z$G+Y?UskKR6NU-4Sor4AH#j$S)@@2HKV_l`X<`oO5pQ5#3~jcFgb`>qk&j+)gv zrFX=};@%Ml&Rozws%y;VF`Gtr6}J@+jI8e+JEfy!%Jki1_l>U~7~L^p^Rb)8Z5X%p z=#qnDOV*CxKQ6!S=#G+ne(%_|cJ%EzX5;8BqqmJ67`1y$Z}FBf1GeFNMz)U`C~hCs zK6>}4&Byc{U%aNhc=qnmYx>8wja+Lx;r`;Dm+d`%TS@QHn?|i0v42!Y@s82m#Rre> z9bt#&-m$&Y_Kw(Q$J%^5MT}TKV#A1OZFZ>7+&Q9lS8W*4J-TdSHres%zwvX1fBygEXVJDZii%!t z=jh_|in9N+=ppx<@B9n$Zu0ZwKJu6{_nWTp^e2+{%y&MET(;Etz2v=3&hIDBS>fD7 zUbD*iL*(+?o$n&&Kk57cx%+PCC*(iq{1bB5zd8S}()T$3jXZg$bIIF$eY$r!zlz-S zAI{Ur2g&D?OLluaN8U!ho?QN49{(tL{Lh{5Bj?Y*w?E_j9J!nP^0#|= z50FnKA0*Et7w>a_0eL*Rl{}H$L7q(BLY_k2NuEyr9l7g&efcMx>E+o*KAXIcTua_h z{sg)7S@*v|o=X02@_h2R8NU35;implvqk7U;`+sT`6a(;?D z*9Q7Ty!f5I{N8(<-$>qhuk(Cz#b=$@koSJx`Sav~&CZXKH-5?am*llycD5V2?0ho! z5$6duN*K;B2gz?Hcf85|7s|?a5qZy>ofna}zs30`^0d>OJIPID&X18h$iE@anc?x` zcX|E`-tAmQ9)F&5C3#Y%a~-+-a_3vfMelR|EO`%kJ9$>M$A3qjy2SZ}cYFSOZ*(pr zH{I-f3Ay#d&dbT;KjM4~xr2N^dHtt6-c7FmwDUglw0oULy~p!!`@HiB56NawoZ!ybb@)X7TmpgI{%ioILwc=l>#a{#WOc3g-VG&Zm<%kgqX*&4{+5(*N}M zD#quKHGu7GP`v`c}Ab@%RgW@QQ#Tm1y4I)OCJA>^S$IP`<#EM^#9{rbdE1? z=d;dlB-i#kUq+t&2j|t|=bi5-ud!y{Bdk0>Cifk5K1iPMcjqbRdVX7uwg2z4^jDIn zPjqf0Z+eyUgXG$ioS!0>Om!Y*H#IokO5fo8R`O2r9CGzrJ-&*(@eJn<`S`y?E6*3m zMQ1zzfZU4zS+w{c$eXL2kF)Ct zOPt?AZX;hs-g%YB?<8+;bnYUzH#z^7+_}Pe^ekWgoEGPk$)z83t|V`~(YcXaz25nw z@PY$O`i58=ZP0``)zT4H+lRc&exJRl0QkF`W26NlS?0UewJLa z&H3nyeEH+Q;XI98vfKGm^4y;}uOR1t?)+);#$Px;N$x!0{4BZq6#HLS+dqzbujjY- zH0QUGC(UwhAaA?I`F_QhIscNp;}+-A*}l9j_@7!^{zc@uk2%+sQLlJ3mbBztQ=}Oio1I@zp7|l? zO7fa^=T`Fk4?EvS-oDm(H+jQt&Iia-Zg)QE63=h)9nSN}~`N6zK; zxWV}?+*&JU71KI;5E^6ZZ}|CYS<6V9XNaeeUr=eGRm_m6k}3Ay`a z&SU3$e%s#Qd^&mJ8=dEnH%xQ>0D0DQ=P#1?pW*x?`DZ%+o;>p`=W*}z{ARw>`BZZG zyPU5iZ#u`hmAv<3&L1Vu|G4u5awDo@D(WV1MXU{0Gi2kgI>>eB#xf{~mkwiS*}_7w&dm zM&9zQ^G)Qb{mz@n(|+i6*`8}iQQJbugq&u`ZA&S#U$UT|Jc-fI8nf%1HYT>cm5 zZgQ#pFBana$sK=pKJgk~{`^s6v-rj2@?)IyFZ;3mX+=ePAP=d;P{?sm2} zwtheNIp|9Rn{hIUD-oMlN^W?stIPWBv z|I~Sa-2XG@lWTqX`CjJ>$kji0ZXj3u()sh`UUDzF_E#RCQ0L3r_pI~TQUG zkvEW6k++aPOx{8M1bH|4KJq^D!{meHC&&~2=PbJ?;E+^kZo{p1tL#eeekc>{S8`AqUO@_FP6 z^5x{YWc&Z{tbS_A_W$DiUWr<4Cmo=G0t;N_i9ekHk{{4eA-@)_jy|}$(N9~ldmD~A}=HFC9ftAkZ&cA zf5F%1E^;aP^W-w}Bjj1+Z;}^~|DD`K-fR4tkwrzN2YvZ}BUg|oHF|s@`80AHxsu#T zUP0bLzMb4h{vx^PZ=T=vSp8s#jrR3vR zGJoL|5|b>`6J{C@^{D!$uE%G$ZuTb z%j+axK;A*FCijuok&B8w|IOr5@)P6=@{h?2$^S!cBadoj{^aT89puZ&edK0x(J0S< zJ-L+pRdNOShvbFi-;>+O$E;@l)rE{|32@{B!bp z@&WQz@=Oqe<>&fHV z-EShFN6wR1kXy-{$ZNmvTmM%T6|EiX z>vtNtgM2pl)kQ_y#(De_@-A`qnXVJ@RDopUI`<(px?KRPqJj z$#%Sv-$yQ*?8~na?{UtH7aWuAZ=V*IADeNPc>COpe@GtK=iDzIJui!oTI>0@p5%Or z_#)?X#g{qf$g9b>lGl9S<6k7VJ$Pld{2uYOS7-bSa>ez|&y#C!a~^Y>m#21v^JMV@ z&Zm+0f6aM>9eLIG!RdU9R}1WSZNlxwOROXlAO560+3+@LZ!y2(W&U~1n~I921jeGL z*j$3dHi0XXIQ&`kzfWH*fBTu4KIq3de6=rq(#nVY#qzuH!6y7d{xJ?;?NQM_6}}kh zQU38ZV*5B{bF%DnPpYc2z`HKtr zF%DnPtIwq`F675J{P=tu`HSVJ+dszP>-jgHhr<`kPq$x;!`Jh2JU@poF675Jd_7;k zjlQ^$ALH=#JRZ;Ik-xZ*ALH=#{QiFWV)-+yN-=>eLwYcLJ@3c!f8;Mle7N&hjKkOa zfhXyU3;8h)U+)v}egXN53;8h)U+*9AJ_5eDkRRjl^}ga+`r<-G7FD~TA zIDEao!uu@v;zE9m!`J&Ry#InPF675Je7zsT`!e|ALVk?H*ZVZQUxP0$FD~TAIDEZ7#QQ|};zE9m z!_V`5;7EJF247srk8${VKZ*C1@WqAv7>BR-nRvemUtGwKark=wiT9!K#fAJBhp+dg zcz+6CT*!}c_{1}I?_q}-k3twEw zk8${VKaBUq@WqAv7>BR-$#>Eh7xH5qzTQ9MeKhhH7xH5qzTQ`ViN3gyALH=#ejD$* zk-xZ*ALH=#J{<4I;fo9TF%Dnv&+$GTzPOMd0AF0lk8${FUx4-o@WqAv7>BR+2xy-G zUtGwKarkP#fc6aV#fAJBhp+YyX#W6TT*!}c_-Y@4_7d>Lh5Q(Yul5vZUjbiS$d7UO zYJY+D81Th~{1}I?_8Mrv0bg9mk8${F-+}fX@WqAv7>BR+AZQ-~UtGwKarkOKg7zfv z#fAJBhp+Y~Xnz7BR+IB1^(UtGwKarkP#gZ4b|#fAJBhp+ZNX#WFW zT*!}c_-Y@7_CoN*h5Q(Yul7V}Uj$!V$d7UOYJY_GNbtpl{1}I?_DX2K1Ycapk8${F z--Px~@WqAv7>BR+P-q_oUtGwKarkOKh4xhN#fAJBhp+ZlXnzG?T*!}c_-dbp_FC}8 zh5Q(Yul8JM-vwV>$d7UOYX61yVDQC-e8us;`jk=jeaZLLNMDT~oPK=U(Dy;VOZYFw zcpvnxhlain>ObF~wQYElmq)DfY~u34mG&dXu|6FaxPL!=vHbLXMvTMnroWE9SpN1T z{}_kgHp}yWoW5B8{+Iazz$kx=!&iSL=)WY}9#{DC!0p_F%G}P+GQc_*lAvWV)<4b*#t)ZF%Dn-+n~Q2)V~<%k^k(Z z{$m_|kF`re+T)qOSiZe>&L%MOk8${OtX&aef93j%@i*k(pX|Re4qyH8pno2eUo7AL z9V?r_$Uny6FZhJ#{~Yrd%irsIRz~f(`~P^O=P#CT-GL0rKgQv!{~+`ygz}5! z&nX`2$2ffTH-!F&;EUxC;Db&0h4RNZ{Oub(wvXi(BR%SW4?ft0U+`lb{)T%!Hi`8o zMtb<^`;-`mzl;7t`eON8#@Rw_0wezzhri}QkDtp29icP=I{{1}JdWG4v3=GhPU zV*G8m`j2t=o9UO)7t1fVH55GM*eqtQH`a?tiXz<1Itva&_jQnF9zWPr?e`@f>NRRw$lkFel z@YUZM`d@=DmOnMw{}e}iQCyG5_M+g}UKAYLi-Kc&QE+T83XbhX!Lhw4SnWkozH0p7 zRC@c+>wkRDRx4<~XBp%A|D5Fde`B9-+on@}{l!}Uy<7$u^&jJ?zqVhy{~7vXq(}XC zCw`2>pZOd2ueZdgf3bY)E-9P9$Uny6tA9N7mj_>r^vK^%57`8UALH;#p7#86C%P|| zpML+uc$Po?U(&B2%b$Y}HsM#6e~82Hc*bL&eVykomT#AJ*(5*4;cusp{sFQ6V)^{~ z8_Yk(;j8~Z^d|^kEPr~Ee~iOde}m|M5WZM`ntzPLKgjZ< z^3(id9KQOWM1Pd<#q!hqV;p|zvtIw`zY@M!ewu%b!&m>7=>Q?PvMK@r`tcq;j4dC^p^@>EI-}_-}~{0{#CQ} zC(BRs-x1>Q)&DB`V}&m+W(HF~a&Fn*djKg0~|Et`8#Pa7SevHHKr++7XvHXg}k8$|Z)_D1=U&Hz%%eQLH zCU9j)4~9RB>F@d%_r-{#{ubIj8zwOP7>BR^n$dqV_8&3Q!{3r@{}_j_{+-d^Gkmdp z>mE9r@C*6JIQ*VleEq-4@{5rk`FADx$2k1f+uc8z^)Hs+o#lRL{bL+{`5o>*{c112 zSpLT2d;wsTKgQv!KWp@FjrA8JJ<6XRe=!bU{a>R$Z1`gNdCz`u`^7kX^_PwQv*C;7 zPfqGD#^EpcoUi}+y#Eu+uSoKbarhhQxAOcgmY<$KV;ufw`XAx`FD~TAIQ*^hnZH>6 zbbPQ0zbZp|Fnskdj{e3`e`3V5?U&SFjKlAJ!0YcjEWcQOy8bbqarg(B{u%mW`RVyD#^LALpZc~Fef`DqXJok_%0I^8Z=ruS*Iz8ZBJpD!en0(- zbp9mEPwP+V5r?n-;L$%kwx3x3rsV!H#^I~~c=RU^Uo5{LA8f*}$|Me7{mrBQdH7;>`C}Zu`nN}a_sCx?f3uYV6By->aro*FAN}LQ7b88& ze-IyR!Y}wS4u9aQUjEzp{VPU#__LSU51Zu2IQ*Hf_5Q~@r+WTk`O{n<^kW=;6a6CA zpIH9f#E)_K-Snr>7t6;wU;=d%99_#6E6 z_=|D)>*(KUiLw2}@=KEaC&uBgf1T%lKYg+MVk-kCFv=g}@Hfyu{bbKyjPxjfdjAvS z@H^>W%kO`&e7n5KCj3JFF%Ez76fb`TeKFD_e+y@m{1}H{afqjdA$pr+fX6)b%S_e%k&(=@Eyo@k=_HzgT{0vi>m+U*nx%{F7|^ar_hc87>BR%RWRNP@)ygWj}JECS7j21zxgcB|8qyXFGhOof9d%< z#^FzVhx>K(#q!hk7cmZhe7XBK(HF~~lhj{~!`FB+7+(hKFP2|w>wpPdnZ)6*dAH|} zfxF?0ksj+m(Pm6w_%RN@gZ?Sp|HSf(Gy9Mq9KOcS!FW0- zzZmIJ{`B~Xarl$Z_w_%A`HSVx%5pzc{uqZ}P5;W1J%6$MVtlX(zfk@dhhKYv=YOoN zC(18IdX#@hvi)Nm{?v=y{}Z3zh~=l-FUH}Q(=U6Y=P#C@e*eWd{8{v$V*cVnevHGP zOaD^lFP5K{KgQwL((hsZV)^OwgBXXe@s=?D66#+pe~KL!n81}u9KOb9!gx*a#Ym6s zw+A0=!Y}wSp5;H!w_l#?KZz`VS7smbV;ufIroWB8SU$>t35@(>9RBz#JpV4%pIH8Z zWdDnC_!^H2<5OY%#q#&qpD=-ue~iP|_*EFs3ceUgQTF*ZV*TT7NMPe2v%zgT_~KG=j`l_5PC ze)S?R{{Ve4;@E$R@xdnif*<4XYZtr!pDe!^>EZX;j7ffs!`FCg7=I1>uXxbMhv3IJ ze2veB@!H^v7T(*3174zrV%udtNy-{}_j_@!&8%9P$^-UzhB^F%Dnj$6-7< z_+t6#^QRbxzwc%*|2w$-#PU0m{9_!x#;e2lb;w^Vzc}$@9R8YHJ^y={zgYg7#E)_K z1Gl+h7S_!^H7NRRv%B=sNT@HL(y##coCV)=GklucmdALH;f{vyU>gfB*V zBR%8Zmw&e6jp>S?-6{KgQu}d`FD;2wyCJQn4=pjPl1ge2oX$N?$DBhQrDx z`7sV(<40mVN#rk5Ju0aD6cU7>8f-4fnsx`ya9Vvc!*Z_~rEf_!=+2Sbl4=|He4{ ztxtIVV_AN&{Pg)@jKkkfKmR(@HIZ|D*9siW%j*>35@c` zIDCy~ySc*i7t8O0Y?J&Lhp+K(e^ueWSpE)tut|Q5!`FDZqtA9u%@h6s_mOsYfYdqd->5Jue;)6~2RcSwB9KOcyz4vU-UySr9f0}=cXXWR3zdxkE zfGoc$+5cl4zQzYGI>+-D%im~azywD5V;sK56MiFovHZPB{xJ?;;}2g(Uo1bpevI+V z=Xk{!zc~B;B+H+ZJb#%T;_x@V;K%<+Yo`HUEWa=L{)ut;8Xx(7<}a4to7LW-`ipV+ zb2*;!ZLjzI#q!hZpBRU)@t83_Gs-WPKOG-z!mr9C4qxLpAEYlvdhCA#He-?>c#Qe=*`%|MdGO#^LXz|9<8#mfx40KVuxe#>>X|*;s$E{Pv{$F%DnjYh%1^_+t6_ zr2bPxSo7^4D8AFo7#WdNBOzDW3lac>OCz9P7U+ zslOP9ukp??{yElPEPoO{*o0rmKgQu}d~}SL4quG)$bXB?n85I39KOa=$N1{-#Yhi- zL$d$GIDC!2K8L=6Sy*@2gBER^cbHW>o3Mftbbdv{jdoP zKgQv=o#*ARVg6$I)Cc_-hu==W#10hXFP49R2M`$f$2k1fO3(iXEWcR(LGv)lk8$|< zOWfba@{8qnCC7h^!=G}6`$cx1MES+?_a^lh1WG{e$_(IQ;$ek2}xTUo3yk(OK~a{TPQ| zzsA@96y`6MpXMLq@SEu0Kwn(Qk8${U`oFOoMQlH@{Ph2~V;uhEo4ou_(ih90nw&pl z9RAE(+;8Xh6U*N`ZfO0*IQ;&PxnIWm7t7z5?0+#1fA?MPzm?y=V)>Jk{%>L&{{GLo zKaJ}zmY;tA$2j}}`mf;gC$aq9N&Uq*{GFRU{~J#7?I)JMEAe9-{>F#g?<;j*EI)nz z665fD=+9&Qi{-Z`=g%02Kl@S7f85EQzgYfk-~5BePmIIgK>u8pUo1Z@e~iQLrr$+h zEPql`{uqb9=W#Fp2f6-Y`BU-1Cj6=l>A~=epK$+9`eMX!?n|Fv#yI>L^jDX9{fXu8 zNw$BC!=LzV&;K8OS1i99DdKw-M3+=vh$x1XFCy?=;t`0GnN z|KA_)>o1mHo9urv4*#GHx`wn*F@LfAwEQs+zxWvUx6&8O-<^E_#W?)lX&;PS_p2+TB$nw+gFQrEu{?4o1zl!;b<(H246#!R; z^kDe?OWgksk3TWu*nc~0#sr2R{I**6FQqR=dgNa| z#(vl&KgQv2Z*>1ouD@7*kL!bejKgnla=(kdSpH;uunE6V{uqbfxx)S5(ibB=%HM?# zHsKfi7>7Tn#r@HB>{M)xb}i{)1*>mTFri$3IjF~2{= z@_Q@^CU9j)kB{(s=uf3DMjYF(KdHYMhrjg}&;Pgl{uj%ioA@yfzvMRe?>m{>pDe#E zS%0NR9RAvM?(Z1wzF7XAq1QU1wY2&*Wc~_JzRe=(!=k2x&5$7evHGP{YCeCn7???$A{p@ zIQ(f}a{p=iV)@;PALH=%Z*hO(g}(j7^3&(9F%EzHBksSOzF2;G{*Q6^8|hz5Uo1bp z{)ut;Q@`T*f0Djfe%k&o#^IMf>V7wUvHX3>{ukr$OSZZHEPb*3;>3?}_~XCf{?QkC z{fp(N?Z;vq{``M+znuG@Sbk6P{T<`k`v1WF<@8I*^81qGKgQt~{mA`i=!@l-B=;{d z4!>l#`_s7oV)<$R#W4oS=3?6?m4nO~M_gByt%TLcAF%EyhFWjGG z`QZ9XEWg6DAIv|-;cuk>Y347MpFehJ`C}aZgg|(e%ag7bbp;!*6}7`{y6++fOXNJ}G~U!{2+F`^UZ4eX;yCN&Uw- z{Dp6K|6bO=SblZVelf=3cc16}F8X5m>GSUxhd)5yhN;i?f3f`1Wc_0t{-jx+|J%6! zV)+y8xgI8PWk?T(KkaJwS6Dtc|A-OC{Lu>~ zl=+M0uTAogarphU?%ThyX5|;lpO*MB4u8@z_wQHzk>yVrX9c$jjPffz7=HP6?$`7F zU5q&DuQ=KNV;p|PYWF|K@{8s7+i`&jjQnF9{xs`g8L@-h|HSwk@?W2URA|GWD?xBEoo-%6Hmujhx< ze~iOFU@v%)_P2JPg)f$$zWG?ax;g>w+ z{x|G859Jrj?@aQKarm3<1t-#eVS@W&`J0pdH^$-5`HlO(V*XHSBH!{7L{`y=f< zhw_W%*V}yoCU9jEhhOqL_kYIkZ?XJc$^IMT@O%H@{=;^hAb+v^_GJBI9R7koxql0N zvHb0cALH;R{nh=Y{QecoPtRX54u1~))ja;i@~0)|?-+-_X4L4cZp)d!SpNL1_7By6 zjKkke{|@?M`Qz=piV0j9(u3jmjrIJCxc`e0$M;9N{bC$`*Le5;gY_?#Z_Tr_35@(> z9R8ML+~3XpUo5}E^}+JTIQ)M4pJV-t<)_d8VjTXQ<30bWTz|3r^!Sf)`0ey7>5Ju8 zB>P{C!|%2R8c3U`FP1+&S^pS^e}MjXx&C7L>yzyt%A4u2N?E9i^m_ay5dyupwRfogVjTVe(+}8+V*82Z?@RWd7>D0C*YkfR>rX6y zW|Du5!=EzG{SCZ+5zC*J96vD*f6n{dZ_@P-S^k8?S9-+ZFTBS6i|LEy=dE1;CU9j) z4~D;cmHX2#_WBbej{Sd|&6vROV;ufE>tG479DOnV27imqn85I39R7^e?%z&dEPrdV z{bC$`|BdcHOkXTN?SC`I;rHI;{*UR4bv15?|>Nhd=W} z?!TVCSpE!runE5^LwYd$HP*l-i_sV3Z>ax0He<@tg!Ewe^FQoB=o zt^4=U7t2rck8$`LZgYP(eX;y?Rt8Mq%8(umf6DFdAD}Nr9OWOd850UH{D~2QZ>P0v0>h7S_~Yr%WBy|K1NOa+Nq&sOFTTg~ zf1cm}V)<>EeaMe-`0exu*nU|ozo^(307m&^JoE4K{BLCW+sX1bChd=69DeCz?thW_ zi{+Om`Nuf?1&_P`@ASp;=O=!Q!{7Q%_b;a}mS3OLUyNtvw-?VyTc+}p2mR}Y)?aCe z!(aFIF`0i4>rX7d7$0oHugZ`f48P+{_rFeGj5zkcR+}+_;m0`qk{Ryz(HF~KklBa) z7>8d?f5aR=eg@e*5G+rMdHUe{0jOP(ih9`P5c;#Kj|Hwe?5J% ze0!~(P2kFq9t?jW{d?$(5y$#h+l&beKgQvA)9{H{5k|9k0+kskS{+b_nm{O7sfr2NV9JMh6K{L1nVarkSlbpI~;V)>gh z`;Z^w@Tbgo|7-Nc^7G04M~uVYaFzQ%qc4`fDS3VwKoi2IoIo7EZ<(wXA`(GqzA*#-|YTd=!+3Y{jW>b zKgQt~-Rk~T^u_Yi_8Tz{zw{3G+v$ttHzoUDjKlBvsQVAn7t7zC?Ef(ifA+`R{~mp@ z{My8iarj$5;r?&wi{+>9FJl~j`6t~UHIK&+S^mr=E*|6clH`FoS~k8$`#o7{hrzF7X`r2H|S z<^O>DdzC*~{@!H&pB&;@{#)EHxx%+!5m|md%|FE9=O1=|Dt)p11IhZwIQ-hLxIde| zSbk^X$2k0wN8N9vFP6VC@nanRrpMg>D1EW~zGVGl9De)b?mt9dEWaHeY{IX~kRF_s z{|Wbx=lw$o8F8Gyc3)&aY?2@2@b`=<$zr|CUo1a;eh}mECysT0{FT1_#q!hZ#~6pd zfc}~E#q!hpj~Iu)oBn+IV)>{8OyJ6p9t?kg{!02{#Nnpb|1l1K*U`TIAEPgpKRMZd zVjO<|c=sQnFP5L?ALH=5U*`T#=!@kaNZPN(IQ(sIaDVK4-+p5G>HS-b!=Ly@_fMxU zmY=p?h;jHErnx_dzF2lI0KJgH8BVX+L5d ze)+rHKb5{1>9PLl^NSdVznT6kCwl&3`RVmFP+o9JK3@{8rC z&%a|F{?_w7|J#|rSbmy+jKlAx{}_F-{OMMQn81}GJs5t;1)hI3eKF$5JH7snark@Q z>;75v#qw*D=if07f97TG=a1+9LzbUzf2Btpeize!hQ3&S+W%&Z!=HME=l@OmV)+wF zEF+u1l_5PC{*3p#|7*5C6(f%NOWQBSIQ+gP?q9|H#qu{L_YW}+zrEi57moA#6U$Hg zKZ;Btb>Hb`@ z{JgCLCUB+wh;jHmO#goRV)^5f{9_#c?vHr>UuXLvvHbM>731*te$4$=p8v%1)AGkS z{P`bu|D*K9h5Q(Yzmxt)*nUDRKfQm5arj-I^86oQ{$lxSv)VsY|1l1K#s>G%&kWwb zh~=m4r(zub_IuoahQ3&S`u|@s4u8Y_?*A*xFP5L)f5kZbt#*P!+W(?2mOm}I{)ln- zeP43_&-BIewllE&d4!`fa?%&P)#qwt)*KaWn zf718ef1JKpe)|7=F`niB1NVECKUsczvi^M`4!`B>}InVz$^u_Yi^^bA*vz~YV z7`xHM_lH=1`usY^;g`MO{@L`!^4sviCj6=l>A~=~4!FOZz8G;_Kc(M)F%G}{FYbSa zzF2WIQ;pe#%B4Sc#YS;SpMGR z_=)js{f}|~V*2G|`7@I5zZi#~Ki>U3eX;!c$@N=|!{0f{{SEZR@;4>*7vu1oY~T{4 zeUiRd{yK|c0#}CgVE6~=|1W(p;@E%oC;7)X{QlQ?{*&MD^)Hq`A@O58%fHn9+4K*R z;G~3V)^r5kyYPd{lz%^sk7YwF@3T8y{-@XF%EzHh3@~2zF7V?e6R_>DnoiO{O*g~ zKWU-YzZh{GKc&h37vu2v&USw$eX)Fdtd~t-S{&;+_3BM}sM~uVYd8wEG_A1X`jP%%lX4;Gi3_r%Mw9eKFF*ueBKy7=DbypLx0GKW35pV)+~K!6y8IALH=* ztbsdXZ>BHC-;n?8lu{B-=e7>8f_e)n&pFP48WIseBv{K?hsZ>2Am zpZ@=BjKiO`#Qoj$#qx{s!6y8w4C%q}XD@aC@ASopqy8oyWj|~J!;f+J3+vo}?PA}4 zV)<$NqZo(Zf35px(HG0#kbHl~IQ;eo_phcemY=@=igEazP43@JUo5{f$v?*7SLEH_ zL|-gF9ls*R;Ww>x|9kYs^3(PUF%ExVmHYeXi{+>9e_|Z|)a%_pzS_5+Sbo}mAjaWO zw}Io3b_RX1{Pg`njKeR!+5Kzii{-z`;X8U%kNL} zk8${=cewvk`eONeigENDI(}jt{=qxle}TSO{5Ju;B>BfU{Jw4OUrk>uKfV8oarpbc;r{jX#q!4|`Nuf?gWKJ| zm%doO-PUIlxH6;%!=L|z`#b215l8*+#Rr@43x15lFSmg|5qpll7=MG`mXtrn;ctJ^ z{fSF``-$am#|NA63;D-5{MK)~Ka;)~>5+eH^86vj;qTw+{tEhH`TKbT2uA)f4!`dw z?th-XSbkdnF%G}{r|$2hFP5M7zZm22`+w&C0DZCiwN`FSV3a?`;pcnZKiMuc@%&7T z^r*k|`9qAuul~9F7tj~WPrrX+9Dct<3r7uQ2+keN~2b;j~V;ugzXWgGr=k+I+pO!zy;kWg>e>Q!w{F%x1lj29U6%|?6 zNkwx`JUT1eJIHIv4dhPpn45}mD;Sy{+(|K zAC>U637?Yi#}giTi{<|+d!9Si^FNK;K|UM&YHN=*&f}MmcadAj`^f9bMdLmFSHLIQ z`y}%B$dk!`CYO>+Z}s$3$rpep+wnwxAGv6_>-o2ygOUnI9ZcxATy9`UtTXZ#Ct z#r4k5lWT8t9&?+Qr*?z$Wbp&er;+!6%^CYtHGXiy{&LJDJHAW2zD^;};P%G;h4>D} zu|I()Fpm8PypW9j6TE@xe+eEj!G8L=<@t>9ed>IimE{hB5tV}aHG75ZBPt3NgL-v(CyZ0L^+O|AS!lFF2{# zS^XWNe>%j~KOOpO0;@kJ^q&S+|7g3&>hBEgXb@NbQ|K=Wtp1?Te->E%W1&B6u=>0H z5qaa=eZ0hH$dj#sz9lOfAg{5C1s^-w{n`th(Z42^r~WgSlhq%l{TpW%SO2a%$m;JF z{bRyc|CeLOI8Xh9?;j_VTW=qc<$n>m-!9;x;r%*%y+6NG@eg?XtK{|MpOe*JQb~#X z8b1u-gmRt z=hfta+nh0eJJM@>_It={ZuL0Ex6hW>;rtu&frZWksp|{h{FfI9Tt$@xB|Z_t|Lw z16KPUw8zQPvwaTU&x7^;9qnzvYEOgy|G?_s5B&jw)!z^LCjzVgA@uhHR)0U}KLV`& zADhk>`X2$S ze-ZS@lrj5fLjO)+_1}d4g23tz2>lO%)xQw>;{mI`9rUjQR{uHZF9)ptaL|7jSp9RM zKQOTR`$GR>VD%q{{$9Z9&jtO@fYrYY`jY{xzZmpy16Kbv=&uW`{M~h|KASQ|F5IJ39$N`K>rnB^-qERFu>~X0{x4D)qfcJ`^wU@|1XTc3)c9$=uZZ$ z{$kKS4OsotpuHkk?Ge%b5v=x)XwL~&drh>j1grfd+G~Q<9uw_H!D=6h_OxKNmqq(n zu-d<(y(?JlT`~R)SmVo}JvCVErP00@toFNTFAP?DV6%sBZJi*8SSgVYCnzf zO28VA1mmxOHNFbQ!vt%*ONNCgAeZ{rd#>D&P)YZ=6G(e7?tXaPO*bAZ+V2F5D{YdpfU$y-=I)#Pbwygoic z-uyXdv{!?#_GsTB_dek9Ka$J2J;%P*({Fvkx9_Rsfv-Ein>?k{`D*f(N1a>A8=3z{ z$Xm&q$^F0g?Z2J8m*shee2_e;)XSsxP1ETmU|Soe`NJ_W`TL0sd7{F=PyTfV&GUgz;1*5~=;1+0%ec{}5u zC3igI^|Omyzr&X|;`P2fjjxUIu#vyUyT|J3mF9%=@XLQ+@eOul4+E(SZx#B8czl+H|Ip;d^gln8XM4rp|J>(e+J>Eqge}!`ox%MLGr^vI&eB-bwX^!v!$H~RMZ2f1Rs^Q1R< zc{beZJe9ovA?FJ6)Q6oHl6Nuv8ghl*2Vs9(NAAABc>}rseCPYg9gKI0^B#Yk+&0Vk z`{Y?%{x0(D>pcE*a(<=rGvrB(KSy5h36KAkJm&+>W2gE0m#lI=p4@Z2^Qp$RUL$SQ zvgy2pYZC5Acw54K&c#~ZpA(+=<}AOX6rb)qnEv8~8?n(GRod=iqbQ1q_ z!hcV=_^m_hV_d?=C;YmE-;i)c!dE0*pYTly-?*y$Sav{7k~XPx#LX|2^T+(}!-K;}f2e@U(=>5-v}8X2Mq_d`-ejoCo)}RS9<_ z{9wZ0P59Rd4aVhyArOLV9#c3zTD=yHqWye&mga~`F%Do zusLV*LYu2>US#uPo2zZEv3ZHjOKq;Txz6TmZC++`z0J#QZm_x0<|dn0*xYP$i_LkP zSK54?&8uu~wRyG8*W3I7n{TlBgErr2bDPb04tlfAAF{dK<`3I^i_N#%yw>L1Y`(+h zbvA#*=8xKZr_CKUf6V5O+x!We*V}xT&7ZXSQ#Nm~`O`MvZS!Yr-e~hZHs5RWXKn7Z z`97OJXY=Q6-emLrHsd+$12%8A`HMC`X!DnB-eU8YZGOn+hi&e%`4OAHV)IvR-fHus zHh<0L$84^&`4XG44KA~}&~~`WVprSz|JU|dV|gEGoBZEWpBGp;FS2>I&Dgikw)q^J z&$W4`&F9%X%jOGho^NxZed%_K{WJShX;JQyb1phJckb-7FTXrDB{%c(vWsi-RrQya zo!8n_)104MwWz+PV4B%gt81Dst7)mLu4!1jxOH)-t zx`3wUnk9Ah^|{5hD;t)1aB0)ZTvc`T896I`R=dcOf6O&6Z^<<@HZRYus;OIAn;$mK zVP5mnYBxk)n1D&FP?09U29EsZV}cwtN53oG*_*9i9%gNwQbbu+&_~^ zLu13@s+BEO^|_axk8ScM+v@d=t8y#Ka_`Nx)YmPpIl{Wf^fO*OUA9bC(c*t9`C%Jx za4WG0D`v1Nmz?(Ym#}1|u6-%RP~w+ThY$*gu!eFlyoauB&TVX}jc0+L0S_D{_l+*q|s-c91tWHrbk2 z)zu%i`ip9o)-~i7H!fdZHF#3P>0xnwO;y9nrb8zYCCODS&evVXb61wq+`4?u^xUc{ z`_NKTU$Z!G+o{=dUsSa?oQP~0R>>`P9?Bi6Y&)lxS=uH$bsko_VFXlLjUAhrmR8s> z>5q`4CEwUw!~OpV$yPR1Bg?~5tZZ(!Z-iV+-d1$^p^D=9&%Q4ja#_7ES=myPtGdo= z-@hYu1|B}4CFVB=D)-R$kL>~Y^0DtvJG;i*uEV*et|hnpZMmwZruv#(OP!s58u{hm z3sD-Y^&_RLYql+Cx!9i8lCP<0u4`Co8>;bI`|Dw=Uu|DN&C7Dj?MLHvwgWXR$-dlc za&}5dtE#%?x-#rzO;#bRa!cwggSzVvuhA^olBOBCW!L3atgNzc+d_%_#NM1+Q8iq_ zEVC81h{10+e8vHQMf)}%wtzz%E9l{8m#l#K#wM(G%d*2(vu61s+wJp>%W4{O!<->w z*eYrI5&dlakCdY2MY3qYhB#s#mI9T0cqVov+r~;x2sPoaxvG}c4T}%$ggDpOvNBBv zxL0)5%6#K+`~H&ps+L-8XFHtjq>Kfm+hj@oN=tHhL5BVv`{z=eQ$lGn(GL_}#^ESf zTEF~lXXLVtP>mgXk!5;FVzkw+s;YBMjg58$Walv)b7{#ut@gnzae;Hh(3CBU?R?Wx z;DqBzmsMBUF+cPWKNR6fGcEPs+7fG|yu6&Q2MAw6UE%nk% zI*j*EXLOyN^ImK)aJcNV+onTTF4>i$?FFD#+&`yXV}Xx?B+_9ogQ+H z4Tpa*9+v6|Cz0GTyZ9^grHus*i5J{>R;}6bG594q-1@CphI<~nu3qL}Hic^}P5B~A z^Ns$J7FT%x!1sA}ck5q8IzicAhs(Y2B{C|_juKqbE^pNJVs&<{l(Q3PeU)8@loeQJ zVhx}1V4_0XKFhtC-$;eOKfIKK>4wYfB~^td(r$V9bq*@TuX^p72_F9N2=sf2!K0|Y z(XI%FD}5$z}CGH->J9hbHMZyxCzg+(j4{iQW9r7Aug@7u8bN zu&lPR`S7bITa>5phfQ`gwX9t32k0<`946^7sfH|SaaB`(rQP7zrRCulwU+yk#J>C& zTjXJJ-B4#0R``@Lc;lU=OP8V}tMCbbNQxF-5f(V`leD_AE|h6@`*L`&E`R9S(9V#D zZk6!-Bu#D!25+be-QC7CenJ^;Nz1F2+0A@ovahe|n)=*zbv1`?Qe~VNFJZyiopMbpuKkac8r#giM30!}P}Ll1 zNz454>acg{aW$CkNQ-*W%F8n0m;8|yR4A6+VE=PlAX8qX>{*(ZQnq1l@O?M11g$I3nGCX*~Jp3_;-RCW{XWWf>J96zQ$&z7qczZTfRbOvA+ma;>xvE8t&EfPi zBxVnT>RM_K9pOWQc5gUDY_7?!DQv&zbw?gg)WWU5o&0P8_6#aLk=U9pu4*18Wy{K@ zCM%sE8lf2W?92XGv%Il+H6Bh5U4iCCDW^o`{G^I*t`snKw4_-@ufXvJG9fvp@$nrD=7s2;j zM_vXK9k!5y&(IIMIXGNB;w66Yvn`?r=#AIaG&ftthf2h42p*pMGg-gX&fzQg@GI?? za?>ze8op{6b{z`dTWCSGcJx)}mg0H&%BJC;Kw=enMjZN#{LpHKBu#ZqHT89NPrL+I zb@h#n%i>E3B*g7Oc19cOXMeWWN>+z;OIBwo4o~2BG5(m_U!5+&QzUy8p%=h-a)?~~ zoVg@CnvZ|Yu3nZ`g|q0PXyHHGTMT<6HMCTok2!X+T-6XxSXiJP>GsUV-k<9g0{+<2 zVpr1HI|y5kB$T|G$dW6m&R>3pS+ufxX-zmuA0a8vdy7`*0$WATt3rr0rM)(&HF zV^iF`g%s6&p@RDbc(zmMm9AZXEiPDzAwAygBrkM^u472c3#^w=3SJ+)gp^*_t3paY zAsn`2+B>J(=EjC07c8DAXBV4|;q68Is}8zD%V#t>f$^`l7uu(M*o=dLxS_%y{Q|fV zPh}3vBYUoF7s!VP2HXGO8-`08x@Yo_I2j(MuWGK#hi?u$6Ayl=9{C&d#ZuZ{9QK(k zb=>?l%`J8IYS^xgn;PvcbJ%l_kTku1&XW4$`s@lIZ!d-q+ZeV&wT*V#v@5+`s z%Q#krrKwwSI-cn@ThoYYdv;#GI%n^ntntDy$+#p>y?po;z~Yj0XmBuOZ@_g3E=KS` zxxNOEg|P{j51S^er5?;QtP0=KSye1=UD9Mt&9ZM!Jfo~%nXcuE6nr@S!=e2rk$Kr2ahTjV-iZ7R5UyY(LGF0p#t-x8>R<&S|&-%CwTn7$aeC^6Sj*y{$T-nluZ!cY9*d-`l z?H@`re1oL)n#xmU-@LAR3rpUl*eRSv8S-a;HW1SvYOq~ph4 zUB24BCzj8wSyHvKK0mVt*Ir4xUnI3ts8#PJO=xvw2kVd~#*Et??;PF)&a7!!++5dW z&9)9tdj86WtnJddwxGP7JFdV>rePL&nI3yxUSq{7u)NC4&&50CBp3TD7k+kgbCqp} zA;qn{{Cr!Li?b{Iw0xD9U%WDVRyDlH;LJL2ebs#UqM73IJlbR(mhmNv&_wI7KMzSA zTR&G`E+fYcblJ*=n%Rwb%TtugW$U@P*}mE?&)cKEbL`T}8u8`kHLS9C4LPg8mCKPV zckcPB56|gb`&w+dtY%4#wTr{n&RIj`Wi9z-x$3HBQ}G9Tpn;8-vt1ca{_M*$!V&-; z9yMEWvd=Zk7dNfOVnB_Hu3en7_l+%Rp=S5Nx$CCm&$hYkd8DPu&V57wvUH`L68ulr zrn}j`Cvw(6%U)XJ{@xc@v%JX`#iv!de7)T?H8$q$Y>wM`JE~f7lB~)%))nOz;bB*i zrS+>#JA5o-yKT|s4dsb!jy`%|`F|AgYSo383*~((I;m7w9 zv}-rOFI%Z4_~No(`I>qx7r*dVUTE54YZ|Qiko|75J6OM&TW-gQwJNoiaCUdHB!@GL zSB#w}7UMe^zj4uQzwP|5-}dE#Mn?^}Twd<`*7B7N{vq4M*;4F_85Qq;uvTQepO|A! zv*t9~vWE>=i?PEa|4(U80#IYqb)_h>MG;A54efiDv}xbVTCVEeHm$ePhAi2YtO=n+ zN()&+mXairk`!7fv`N_&=|A&4&+XplF1`N$|K9I?xie?ZoS8W@bLMQ1Jq^r){@8gy zRMPx`gK>Q-+F0oDA!^Wpw~Xex7TUlmiMSK%$1m8^0WKQyFNaXzMOC6KATWgDj|6oL z9{mC_B`^SdY8?X{utz=Od#rNcWkmc!Jq=W$ANtJU@+zlt4hnnJ!$u512Ly&1D?y_n ze#4~uf$t@{D6sla|6o@S8thtcKb9`pP-Bb~7!V9*XFpdeC0ba)7O-3nVc0NiFcrgP_&;`!( z!Ej$tDG}Z@6!-yS!nOD+%NX4|9PS7w zM@utnV7TGaGA)GPgn%ZiT??m{@^3RC$GZcz{BSC?ljLkX$f#Oe932k&g9_Zd(S-qS z;OKzFVVFf|b1>ew7%&PBf^nkAlz~o}6;!mU5K};^B7P8xbG!#e8MYeuT!8XYRYQ|P zP|eU1v%bTS85Vy%A`gE$pTa6eB$p%RJSYR~3YUrtX9X~p0amBWr82++?h*Lb6$d&f zwZZDRwZb0#+#4An?J~DE@CM*To(-zv)<)4iJRCoeLfuJ3lA_mSsQ6xHC5Q~*O810| z!=IQ3-f%EzeG({%C^np+ynv?wKDC3RWQ80ABmNq~5e5&6(HQgIA|MCBh_((Ex#vJ$ zWAaE~)b=cZ|A@~nlokq7bMV?hDP=`G&|;7t{PCd;iqjg083YS4lZX`u#YD^_2g{P@ zXAmwV$18GYL^c@SYr+ON__DA82cgvA#s^ zAbOxT3`+@`7GmHbp6G8O-Y8Ota}wbC;!=v5Bkuv^Psu)~nDT^ICH7chsD-e|C6?9T z)d*LFQP9H=xt5?CONC{kZ~=iQJs^0Y_%e1f*^vTbHBc_o!6^~sB5;aeP9Mny1-2PX z33?lWphX#2EgnHA_{|GK$~p)FVqydrX@>I^o5>f`jBz(iK^0RU$8NQPxq~MijR*&0 znN*|><5`KV71Ed?P9#r1xbeWO;Y`RR95=_}b4iE|ey$O2&@bq7K+s_@FAg_wB!MiC zWDB-xv|o^zPpTr;1@Rqbyg~+JW2>7GiyM92dWJSx?y1s_PL|OGXJEoeuh*OEcYFmNY=^zFFQJEni+3V?rX?m7ySp0C!7$7*yw-qDhNbsP!4A#E*=6n zpr>zu8+s;lrxxc?lKX)-*e{vj@cKr20%EfxD=D&2V1A7pM+F%c(`@sA?3s?eHINd88yH)Am`N@Y3!76scQ+X9T80aUko<^)n7q+h9g2Nvp04hkLFx<; z?5;v{%HiXTE)hcfLJ2%L#jZh|j>wCIM7vN;b>_T)D2q_iRRTj;>|O&Mf|;8`&?W9p zX7qx$Ke$aIg*}6FU`;SnAHxry24XZ8^-uB~MWPX%H4^}PGvZjp)Xcn4W4^)YrV0b| zFmDP701C43VMb)coUF$Z_rAScgF$+L=mLxZc%x@EWi#3^q4A6d;C%;gBW5Xkj~F)v z2ZYiGn@N01@b?5eQKTs!h#+Au6Cv4>8GluSVh-kCqGv9jtrKZb8gOoQ6bW28nR?A!{3#2}5!> z0~&>T<91QkKo+K90U=UY5P^i)#fVNICJijGn8!WQ0K9#Xiv}5%F%L`%IMFjPb3(Ip zzzN29h+%Igj%q4O?hqW{5rWbJpk_ z;R%|E2nIOW@pGTNV>saupL2sGddBu83SGhS`@nU9g?=(SCctoq6!5tOSp-jMzypLy z0JdvEJJT-C6H0PBJKwO0c%xVk0>Wo#7G?C{VS=JyXC-%bPx%maW09FHWx?*;Ub$FzR5b-?~i zAkx{v=ytG^1DhRuW!Hi^79#t2kDZuTE*l@+bW9?Mfss~=V}b~v0ao#btZy7ZSZoCr z+zRp+ZmJ?bABk-P+=*W=SomTvi@!DjK)lj$&IV{Kzzz@YVA4&7#b)N;K#864PiQd` zh%o2D3BVw=|AkZxQdOPe&2m9-84PlQW*IoB>CgrleEi@x9qOm~FQ9DLd7(x3MDRb2 z9~{3~sO5!^fZ|=vS%bDF9*Tg61TrB8Oavff!t)DS|;nf@2XA)0wmo=L|z6$B)l}QwJL##}&r7Zb2|bAqePBr?{g4LQbU% z8z?6LcJhEs3b(gwpfZ~P!fegikVV&nc$`Ds!)(iVt#iU~yJCn#mMLOlQ^GM}q{AN5 zgR*NZS$6hBMS38D)`<0I4B3sB?qB zW1f}78G-zgm<4wj{#4la~p6)b@OK8Vz0yJ0Y}GPhh>uEUN!JSI-85KQgo+jAsW@_JW`Xxsqr>%u?c6Km@NK z#{|r4k@bRxR}$ug#}N#|Vu&WJ7f9O;>Hk;_0R-D}NU&TtH@A}m2gFT$Oz%C$sK_la zLo&oZ`9qW?us)mRlnjeIVxmJHLeG%D#h?Z=T>QkQ3DJQNn+tDa-SF=q}f2X z$e{i!_{d=-fDwg?BiBuK#W5K*-U9UcpsYlY7&n4hp4YhG!U0D~Oc9Fo@vM$JWW+e& z;4s9Z;)X#W6jHjgNU&sltnYNpp2&EfGyOft4OlFH$ln4Bp2ap1oBsy{Kp@?>7tI$u z#fdtaHbqSpmIH9(9@5;ax$hQyRI=C>7^V&^Z!WwAQBV$`<mZu)tCqP1tA9_4vC-W9AhztNdPiyF;THgog6g-P0`cQ z5AxyghQRC%5E#te-oQkJZ38Zdu)_x_-myg9u$zHfIeNo@@H&!Z4Og&Zo!|xyhUyjb z2mxuCi3!~XEZ~9Y6z6z`umPbb^M3;2e8B_mUlR|pb?oN5mIp%CA#jk7*$F|uWY5Bn zbR*1}z@X5K6MTffBDuieCYV8>7+lV!7%5s5W9g(atu;;Nta%Y5v7n>_sOTs|L z4ufefu-522W3{icO&+lzn01BA>VZv#l?)C6jdf7a5$a5hWl*fu@DgLMCh060_6|nS z9dSf+hhdEmC*S1w@kY!N_E6(6JTUkkza4RT9CKV_E<3za;Gkkt-(zYMI90eL>KqiF z-GDSGFh(+a3kE5KS^~`@GSYJ{7%|~!ks0SKJ`OsOER#iSB+V?V#&{s^kyrwFD^t-m zLJ|aAA$3OpYyLH)v1JE>TMKiaVCj`uEe2O{)5>5TN??0tF2_GNLh)aWJAlj=&>_;( zv$#JD2vgkEAPfqLQ8OI}Ij#_LM}&Oh?;044JekNWh+_F6MiN2-713R-2v-*K2k6?E zKh#)%sI&giVEut2E(frwlQ3wJsz@lbS*vtdf2e@3E)$^&i9Qt)dKD6dDkS_GWGp0# zu^`a_l2u4ls*ng%AyKAE!l_E4OqE2LDv2^x5@o6+T2x8Xr%Iwll|+jwi54{yEovlM zkgp4K2GvNksF7$JeNWE*4MypApMUzB}CW#j0VaKFQlSGRq zi55)~Et(`+G)c5*k!aB((V|77MT7Ks)u5-nOJTC_;CXpv~qBGIBvqD7lTi#CZC zZ4xcoBwDmdv}luP(I(NNO`=7cM2ilI79A2T@HI}Bbpk7X*2&nP>WGE*vsXBv#H3uRoE(>EI{?ao5NrlAIM?-2 zoG=!Ts!POQvs4f$mJ5o8+@K$Ifo5{mT) zS3#l*`Dt)=isjj3Ll0#J5yp%h3I3jlbl~_9nl1o>>tnDk=IPIY4A00%@Hvg@fF)`p z;ils0V5}HPQplAo=}*`ys8#_~$l_^_wAJk0GPM1USYkg+>JHY$BNBpbX>^w2!CU@s z8^Ic~EetMBJ7E47#>n477ke$RcZ-89gTzg+$Nm$JYDx3=3}K|TV6Y)2_ynn0Z{md%0-S-h|@5;yT|$N)fHTZ6Kf3vLb&I88x2!pH)}6Sbwc8y!?x2(tsW zxB~;?HHv|~1(z+`T<~h*fMv$THago1z=Xr3>4$J-22v%QrlD?lvMToB<8W}4SkO2w zSKRrPy;qbbfvpVF^IKuMED{%B<;0jn*xImU6t{8>Uxw!thiTz*?&Cp;a5FH4Aojy% z0K^8rV>miPVU4V_&pr=?KiU8Y2Lfge1GXx#O%Uf8zWdlQqk59u^WOmY#B+h;*#l8Q zG<4<86F>=`7=v>JcsX%U@NQ1*LdJ&SLLs+($`y=dhv_1xS>3^U1)0JU1<7;4WK7CJ z!8tOb9?P!4S!)JqqP$&wd5M=WW0tONG+!?LTEUeDas|hE0!LJKL(;Rz%1B`NHk#Mfma!0U)Y9AVwpFUQ=EVvcwFN=ISUiD z2!hz>3ky}?fshMqDEol~dODD<3?lTg=MelAf7o%18v{dEfgvHm+juMi!ATbr^Pm~` z%Lm7fnsI+>|%PsTW%VrK2&sBdj(Oo4=l zrZz?nyuMWb5Ac?3U#FpWm3cgq<)e~zPD@8c8A&du0 ztj4Vh;ydZCT$;hd2c=x)+77E@X^3a+VoNQh$Z$#m%{`Gh3gu^0ME`*!71xau6)Z+B zED0}Rx?HTB9&3mE;xHxmK!5?BuUT<#EFs%7S(jK1N-_}~)9=O`61lR8-SDONP_Dx{iVSt)>Pz~L6% zGenwgPYAa|HwL;kSRF42F9s}0KIET9)$R&m>?jJs`0IC*i*!30g>C_zW;&@G*MeOG&r}26A&@QA|YcA4nI*CEEvn>w&cx z|1JqvAcU6OmNl^ns^xltV{3oXLq`2O)(IZB#5*{otK4!2$-O4MT?wNzrdGID`%d3*a00 zUvGnG0Uw{#IVj*K3dcj40F)iz6ABgiWd!&{iQj|&A<}L{ z0PypiKECSg&Ry~JIY%lQ4f=Av7S+z$Gc7(iO(_3E%Y^XL&!T@_v$7lVSVyZ~lV)ma z>nE4DDqBIgqfN(r`;8~kWA4l8c~e#n-JMq{yrLyyvuN7M+Qfj9?H8%(FaO?aIo5Mr zEJzRiUtA*_NcX%TZ6-)NESaykA{)Gq)_8dSK1tTjN)+TYaSdo3*cwjiz?{ zx1)2tqAgR_*$jQD(0)Put9fCX$nAMHGI6f1^HR^b{hd3zf_ixlwfMM(K)A7`p5d|O zQ-<&OAdxQOr_v?MXS?8zvT%XVj~MsDjeDvN7bS?;6)nmapHNu3#r$SUnoHx9y=P|8 zOjR4*)M6K@PmP`=a>6}rnD+PR!$RGA@|#R`A6sQ_Y)G0NRQY`I7~0S78^iWezb2hO zYZ3Y9pjr5Og@77Ledgt3Z~0g4IV2;f=AH7e^J(XlWtUFh8!G)oc0ta^BSu0h*So|m zIu3LuO!})i?Dc}BFWx5TrKe4>RG+f0U@g7e#d2EGr`r!l)m&TtJ;~}xN944F<3&$> z*S6%>cC`0#Zw=W?*}ZqDeDTBRpVImQvb4(K!p|NEZCN+!+X5r+cRqKcF4)s7Q|q;LV|ZuC6kC_OIW+}UiI3kMq1-mcW_TqblmywjNKT=~e)yZx{6!zh^v4=1L? zOwM%tFj^u>$|ob2@9LT8yUBf>^`DaYWl9e?zmhJ|k?6QQaa=>dx!rrT?r+!^Fm9p$ zi1lY~e(6p8d$DEzwWE@u0x~}mpKniev$k{Tep>E*_M%F+T<@1nhkIMU2yHR%`tU$z z`HthSo@UN`eRckr7elmBY zkl~ew3dc=bW*(j(>CzmUzf#R_Zs)_bKX%N#A~NJrNr2_Pb6+;y*l2L%?smzsCXX|| z1}vLAWkKRSHN$b~Z>)TNo78Wh-uwL6KKarLYl)HZNl*MX8t632TJF6&{Bc*MTi%E< zWz9>*_sV_!xozU-j;bl0ml7&VMbuTcZy%PieK=pnK_e*v@#}_}|b7JrF5naEhmUUh+d6j*s(_c!eqVmI*L$AzY zClqe|zDLsVvZR1(e6`KvKj~@TT@@M*E*#r9%Ozn*mTJgip+9mZqRqvI0bwF>Qn~#> zO(hN!MqKiKVv(5G|L4-tjQSfN#~^QDZG4tq z{TT03rejxDxWjrz?|awnBYG|5hrBocc=_+avE|*T?!PE)@S65>oVvs``l;Q~jvd=K z&)$(LGXDA1*;kL%G`(9xyLdXvZso{b7EZC@R`ZNvB6fGn$Iauf9=A2m{&$mvd1>U+ zx{3RL3TmI0+cC4}t%duri&E(_IX}PeFm7$zFs?|?#3H;b;AP?aDr)>>y{ZpUUtCx1 zKlR}DxSR{-owgl~PIDxy51c+aDwH5^3iAyj2+Fc+1TUtb9 z>mReZTXTl^lt|^I!~{M~uN!yhNteXRRnr;_Mpva2{0#pyL3nno{R#6{(Q%X=X`xFl zB-ftEs`|8bp1|YbPsG~hv%0R$ zxhZlvZ??mvzZbnf`YxQdr&j311T`5u1wGdUfumEE1T@1dzRoyVsj945C;Z82(TB{L zF+LUv5{suuUFm=K<$I3Fh#=*RBll9QlnUjgv!|qeh(p}9nh(|=BzNar7*^Gk zY9v0^CN9xp#!IgTtK11QRF}qlet2N+gX`1-aiTgyKb(?_m-w7`;OAoZgq~?Dq!m>b zUORj;>}0Ng-{RH)`F$=m{0;Bg+LOy7~ zrc9*8>8|~D-`4d|bN$dLdbfYms#!PfvmIA_(eAdIpWa2?AN_t}>DtRVucut2u5dTs zm{NZ0u*H#>v*S))@;E>4Ui7ckm_ZLe({1r6a@#(syxGbbe~N$j&ns0P^VmC#zWjlMeCy;p!>wj}A6zuoySw{j zL`mAV{8it)K1-|&xcahLjbHlx&dm+|(tR`ZW_iWZ^y810NZcG=!zYznqLk%TpnTi* zL${`A>1OfowT4d77D{!Uix(BVzuUm~B~B+&MX)bMOo-BHVzT14j^(`2=XG80JoD1# z<_OfCs?J}xw>|H^%Dt4&cFq$H)1PWxU2o(ev?ct3@Z-Y9T@ie86lUBU9hN)L68Cy#f(D|>(V zfz^8oh92CJUfd!%@}hyr=Yk=Z!|k&3d^^oxJb0IQPF5t>!n^j< zh|b6A1<@0R46k@s6Ye5NUA*q~SD}6Kleb*h|LU=E=dizdvSr$%+r#N2EsmN7&6bWD z8sb%^KkWXt@#|hoN(S$q)zdY})ij|_{c~Tl6m?lytl(mm8x|T`rx!e)P*vNFjlE9MrV?(ksWcfVBVntI&WlS0Ic4PPtAJ-p4 zY@#34=UuW6?x=LVp4#B_=uquW3Uy8B!CTk!du|x_C|dBvn0@zu+jx4-tjxzBcfC8g zx+!C~|J26K7iFU_NjO)HEb}{Tb=k;MraEQ!W7qniqeMDDb%MtbU6>en@PBp5Gy(M48AE{m>`oi0+B6a+U zx}PFPwf%gR6Mq$o&Kp{A=gp|-9aEG2))<>it`3rk7nytZWO9ycMRwHbttx*)JuZE{ zwNEho^_A{sqs?Ipk0~#af3q^ZLhg@K*Yjja+l0c}jWhM`Z{Im`ymXGi8t3*^kE<6b zZ=1x|aB=ZE_3){JvX8W?iz%Iljm#uA1`9i^F|VGp*wmu0BGs$z__XX{nho}A{IViW zZDFJmS=69==`eqn+&G??ZxqSTx zPu)gp!Pgf>yS`oGKTuSje9UcZYxKBxzs|S46ndV}kg+29T&rF?eQuoRta%1bDk=^B ztu8g0!j~m>#5=_nZ<%{Nu5IVOoF8`GN}s(<56w9cts|EjeACT%%5Z^~ivDJCA3IDR z2Fz-b_RRXKG1nw^DrJV(>wK}K(gUrxH)JagvHvLk=<|hV>!%cTtU9me8~q~DDJfB! zp7o;W>=&!VUUToRLg)NPm&6Y(ruIKxm1uR)_{5i=8R>~z7AnXKe5EhuJ0@7Z!AV5F zv@fWsImaQ=NwH+PgX#UKQT)%8%X;GlH|MVzVY_u)zju?>?}x@_3(_9uE0@RZZ|%NV zVs!UG_5uy1u|b(~^^$M$UA!NLNj(_iVj>{lvSg&?>P{Pq#IFl;M1>^w$-MPAv}|Og zYQm%;HWn%SCf}jA=NPT*T2YbwbmYO<)o=U7I#;&5`S4KQIiM_WaG$56-~+tFX*pDv@!}- zywdQzefY-JnJQaPwT#U)Ge4)`s@bQYSv~pd`C$T?6VlHg%N?n?|j zxwe<*4v9=zc8=~`eq7B$UUjMM*B7non!;jlEjFf|KbSRk&otlYw)vl0p6|-{zHv2H zdeMR2gSA6ldiH)(OUlyR^l6)0Q`)7D^4X1vd>Q?dO8uYkl_xY-tqs@htE;+`KDE(s zVyUD{=l5L~5=LALeG~s@;*iYCLgpXBw7TeJT{-6t&K~{eb?|9jXPp4g(E=Z*=c&$~ zu>Yd4!|yfHvz*#S)XS|d@*eMgLG(leO{be8Bd~4R*<<6+g*pmCongL}&S$TThcQth9Ys~66y)^Ie(Tg6U^Ov7odb29^Z={Puq>qEe zi{tw~Nj6MevNAp4_!;LtT4P#Dq@6VP@2kC|>oauJ*&P~vQ!iY|pJaS~r+Hh0gVM^R zXoV3T70F{)mCT69`tho@+Phn>PgYEzS);H+{b7C8+hLwBxAx!Mzo&VjLSlvXB`yA6 zzwZe;qzcQ&T)ycaDY)Rzu<-R^I<^b$R!N7Nl%Iu=NUi@j*HPhAvn)ZLo_bj+JLhfjB`16kMZ&ZSB*YjmG(dKNB6Fa^hvh+j4phe(L|}cGfDZ`{z=2O-PoKybokH4niWACrp?GZSnPD+%AK1! zSzWv6t5#mnO`1HU*(YsfUQD*^^4}X0SNROherhpQYQ6ice48~r;xvoCZejacv%a4S z<}EH~cfQ|uwcg9?P0wsO|JrX!qnBtXpL%xwk;2UKYjva2j-_ln@-X9K3E{F}plyn41hKE9Hwe8j}; z>f6@-cP(F(Rgyw3RDZrkxxLd!K4EefzeDL>qd%i-k~BYma9$n~BwKB?!m!yaWA)3h z;qD$jUF&DYOspCD@zR(-sSVoS1VWwS;)XUlPqNyZ9UnJScdD)Yk!5~MRRk5@tTe34 zpL5sTaKom$Ip^|PZ+e+1rkz&WtzoV`o^Nh--rX+`Mo-Rtp^=`ke3jgLgK0`R)#suv z7E)V^G{v2N+*qS26%(Byscz-o zO^1trto=1>+?pG*0wPu2ucBHs@}r~YMFzMT_*OIBlHcBRb-@^!CFY358V|-tK}i;ohn>AFf2omvqe5 zr9^xDYTmhWZPdA+GG2F`q#VlhHq74ZmT9)V^NPGe<-zrT)-`@wu|P7aX@PL^ z9(&4pr&D6f%q|?fH0sNNg9jg4U%4*tIilxIzPh8|ShF7*En*@Ik0r%-eB3VPGu=JB zY&qZMa{FBk>k{t2Z#?#SyT{akvGqYEm-m(@wk6HJx>GxT?sCtRo`k&obCJ(CUu@2t zrMFMWUw6-a+xoUICf1IJk1qQ&RP0F8!AxDTzMo6Fsz#fG=HE(9`kZ607Z%*MrPc(kI<(nuNPf&X_ZMPPDh9o`uQ<9jXTIn^+tnP~ zrmpqE<#E2auW&u>meHwERo3#J3eS70Mk_2T*iOKFK{VGq~tvqw$g42Flrl6^MgnmYHliJ6r@&;-T_Ky4T-#S(K8jDy zXgamxZ2P^kK=~8pYw5IIvJsmT4eu@V>X>&WI_&uyyEc*B=ii)9)>nB5%@z?JpX{d+ zP%-bEhW&QqsvMO|ciFB+N7h$l)7&j1OhfXH=#)@mb5FlA%x*feakxR! z!KiC#8-GX{jlX!!c-{Hh>kr!1ibjn6duP!?ExTcl$MDY(dU@)^_SmD{@26{6(e+mj zdAizaWBS~xZQ0`sG;i{?t0XO$&S^D zsP?`&y?xW@+(N0ub?3#uuTy;Z#^=wNV_KOfAEe({R}|hn>8Y-_^%{*CEn-(ng(XbQ zcHF+1Q+m(TNcl>b(x3UMqnDbF0vDkUNb*2`sLYC zW2ZTkmJ9ebZc7@|wa@L5#N(`OB_>@To}Np!{xz%b#*+R|yW~Hf*jh6?_xfJvmwv+{ zG-^{GZIx8{mpzdcQK?4)h}!#*jTOAMcAA*S1FKUrM&b=h@&`IZdz#yRD`>Zvcp ze@UFT3g7b0X;aIDf}4>VQyy#n`7-%c*Zr%3xs-YQdYdeaFG$tw%!pe*d(DFF7AJZ- zc6BLiUXbkaYVL5y!&WJAMopG8l`;jcC*PQ7m*O_x)Xjd%#W|~XKAX}I8Kk(e@%km7 z@qsRPFDV&T)E3s6{yObzzp_j{ZFbZyN13^+?um{IZ}@X1mi%&l*!AGmcoWkgDtZPBK{*0uEJ-m*tm_{454Tw`l@G0$#t z*Fqf&p<~-tIXl}Mk2Rd~vnen2kVq3PQa|H)_>7rKyX3z|oDhs%QnIpFAV&92@tE+h z^0e3GJyPF3tb6>~Pg?T0@(Njb$z|_u$yt>N?rRyf&|=>Pp|bODBg8MSeKzd=skn76 zBANP|QsW);{FFur%bY$~d!xH5R-~HAl0xeR1C7em(tbcYNXXmZQcA+P$V{ z7cN`Ukyi9&(gt56>9*zGS>M;$yd9zbSa8n%GyDFmz2Ox0`GUB>K?%MDlaAC^1+k|u z+Ll`hOjL?j8_8e4iRNr`LZxN>hqF)Yi()U3;|bj-gVi>d7|ef4pp;gzVI!z*`Y+&J0u_r#A`Vg1%4 zXeUg?oGs~5f;NsRk6Lb|1*X1e{_}R++_s$?U9*hmmid<$Y!W+VG-c-QoU8fweaGKk zE~qpwU$v}G?d9nZYNX`Cd6RtU4fn4peR)*<<*K&0daviYEydQS@4kG>XJNf;(q|w(tJ(@kj{oH$f#xRe}%ds1>Ry0kkS;tbx{QrvqrMKr|H8Rz)NTRcol4 z)=sC|GvM1q(r0|Y(RS`V=gvL%UVgvL zey_du-fOSD_F8*?`Re-lAH5ml82&_Xp>Q2Yc#%~j#8?E}WVo!X$C4*aoIO!9ol@`q z6Dr@G^phCD5xWtb%!yBC-kmMfuan-L=#0A~l^jqK{<5+b&s)0K8G+Xup32A38TlEB zP>ABxuiJlaOn*C!KvveG#Y^Tax}DS;UiKx4FgSYPPJa~d2@ieJQ~DkJ3g+a`y#us2 zyymM8c-6jCi!q8*|8D>N1U~`Z!g-5x7vwEk{FDCb-Ms|-b~q=2;{0u3KMiDM%`MOY zj_a>Ceh+#&;GOVyl%Tk?>#izy`GB`*aZ#RbzB9EqJdbb(ybtF%5Ttm|zn)`B=~-Fx z=g!TVw{-5j=N4xz&Y9C2Ufpu3(BETX2Na6;hS3v#R+cb+a~I?+TBOTaIIlOnjpYt_ zsm@9i@A=mowi6!B2Z8&^e0=SM_nH$7#e4qU2j1dk&;1l$?gj^buQ-8FyyxFf;Vsd< zkf(blD^K@q0Sfl+?@cGXC!OO*@!l|c!q3W@mz(u$5p`Ag>kaS8-#PG;JL!+&J?RhF z=Jtd~e4r5xW@SxC&P>iqovzMsa09G&0ffp~+Qko?ae_&X&?eFZofO=m;(No}=t2hY zTtQT49&p(daQ^*tUEGb|#s&v|dz}5Gcu#mPI4&RXp6$6XdiOWsQwO}}a7PV_-yhx` zIwnTY8(#Rg4tP#*jN-lEIajo!JtrOKEzFrek4U9Aya%1|oMgf%et$Y~@VzC^%_}OJ z_w1tH@DiNxTu4y77k;jx6MQFrIYmV|%bv+uoYNcLWGB1?CsGvurSO31Pv}qI>=?f% zo$x4rKX`Z3aZl8}9dyhdb0#`>29+a9Fv+>b)y1D&Oxe*-M_I>z2RzsQLp#m=@N&Xi zG;aaTf@@vefv0zKz-x8_p?FVt*o$0KBvf+I$@4iy&N!`$JMgl_4tNR9B#Qq6I-x1; z6W;Hf@H(9EDBcslJ1sjxSy{J1^$BmS#DU*~PNXP)e|U>?i{|A#lZ6hGRM2}q4oDsF zvK`GciuZ&Uz3;Egcs-nhy^drZ~7U@_+55W9PuB9CWPzn*##B)a74-o z;FUMWQJd;z!@26Yxo-n04ddb+h2y8@%_+($S{DB(ZN)DX6loX5KU%Ov2jb5w(2Wo_ zXiANLVqVcAq%4Y0?U4_-1pmn%@R?`{x(dAR4S(9a#W@T50-YQ$WC5B3Zf*~_4!nPR zKe(v{Mf2{Ddvg5UWt{EYJ+9v&_3m+{9cV;y&rxM;FiO$9J-5J{{S|_&!Wkzzq`&^o zc*jj2wy~d(^iTfNYwGx{6BoyQu<-mSZlP_cXkl9nzwl;(o9X&|cT-oEhw17}Pt)b; zV$-c+iRo6V$Y^f}YceepZTf*{W(fc9u&xB-W6MRGRPS%+#$ zi0fVPrQ?hf5SJq!n7}r%cev5~Q*4udA2&K@rRr7jKWYPGnb^?Ab4P_IQ+sh z4!@|);V(Mx@E3PF{3TZ${`1`q|Akxd^%WYaetXeA;OE2i8eiack(Zd?;lybjoWC?+ zrFqN9A9KDA7Y!MZFo0Kw4+u~v3dijO-&Uf~jfxhxk77#vk zwr_Ypsrq3dEyj18ugJF_+>@TZItTGD@y+6zt!C(B{#jhwIH%{9;Xu9z7dzh%ul z-(RnJP+c6!e4k!3Qr+(=v;2C^uhdVk8KsW&WuNrB%2^WE4EB9$&1m(n*NjmoMC~3C zc}ZlMwdNu9>@{Q65$(JsJcZqqikS7wWRdaW8Mgq-L>6$8d!RF?+nyF{a0ZA@(1fRQdXhy#<-Tfn|h1W1dZu7w%BW6AIhI;dAt8P|U zQbD5U!v&*X-=*tD-gRf*^&WX+k@q*`edEmgrbnJ^%{%IckRQ3`U3I@T@2UH*d0%~? zVTaDzuw%p@P$%K3->I!@4(JlljurL8(N^M9zgPE1o!_C{?@{irDDwxv{2R*r9%cUu ze=oT<}z)fUx0%1Hbik; zXY>_&=KJM=UHQ-HkDC|0b;;{${UvkdsY}|2^v9+0x=ZIU?ui_$*@L-m!&;=cSBNTM z9p`mqYKe4<0&vUh_RMo1in@bTdd(<#by2upuX&PH7cs>3`s$+JE){LMLVN{rIr3jY z{3^v!PK9#o5x+)pwo${7zYg)sh|7>a8~Kd)Zh8@E)qGIbYcF#(&%@1x8x6OI&**Bc zUw>Xr=NsF5K44~nl+8BM%@SDwhD$*b#ajhB**GB1xkK(w92 ziETOPgFw?FH)A_!Pu2gRlKkZ8gUWo7USo^aYliu6)Oh>9Q?v^7SIP96av!mp=(J)G zXxv?|8T36XIONF}%%9Iq-{{7sgEt)a#TYfAoNtL&vUwSGI)>mLdB z-5?q~;9`k)g0YvfjUs!zZj@OY#kJnC1vtFqr>4Sb;f z%fW5BzaHE+BJ8Qx)z>@{3$Fflr|!h+UFs9xM5|AHJ4E+To@M-|)u+{q;P$V6M{P-F z#@aGfCwJ4iw#!sl*W%8y>-NkVKCp|%ArJIm0zF7T52haU;H}bYD)rSx=KS1*1kiw_ zRJw)6u*x}xt~o!riaGWYl&3lW1L8ENt~vh*@#^xBu2H32O)%V^^2fUxS8_GYe2|&L zJg(ALYaS|Rn{J>^GaoFZ4U|`FMk1}659vzb#ZA?yLuH2{?PGql>Bb78hsV42kYAqJ zl?z{hhyI-6L(DvIDF^PQ6>QT-<(e*iF*o}C@_;Tm(`#gN*(UM_z<2FC=k{R#TF#B8 z@rd+atBDj>7yW?w_R{M$X{@?P0XmX^jt)ENsFc4$M^lj3mJj+00Pl2X0a+LyS|>tZ z1bjjTd}1&*PXV9o7r4(MBfns-NdZV-Ma)dol+<`mYZd2Ljh*tcm6& z!hXDwVSnzK_b~G0$lHN+8>`Z5T1PRfSghBW_{^@>AtIBQLs}%WO3;oMTcv5=%}lL4 zvsSNJY!U$n9_|d*y9fNUyv(c|%WaBsV+ zJ65m?>w|Y^(<|WSea{sf`U_lFwN8P44Z(Wx_h8c{7;_26JOJf`;DX@-;lkiT;6mZT zE4Ha4Dz>X-6+6_Cg{7AM%MR{~>NiLo9yLgv2sazf9<>YcU5M{Oe3!a-L#?j(DY>q6 zw>;h+HAG#!TV7BM|92Y>=)#fS59x0r9**>I_|^?pT_VzzNdE-!M5HIe|7OESy4gto z4bs0wd^XZ&!ymc&hB~^!4j$ZvHFp#1wp*=PeG6;%I@aHhYF6=%T9FKXjP;hixB-x&1VDtR3A z(K1${k%7i9gRki0us#mD{qpsJy8qPr=v-;(-(70?@320^9A~68!u`TE62-}EEov^4 z#<0~nh7$3A@;dpyc>HLrqBx1o%sW*w#PLi}q-dVCF~%Tg?#rKActg(xw{fZ>v9R9=~0 zrrcS%Ux4j1#Q}RiVC)|tz>ePu7>6p~)*S-u6G%US^h0R-5Yo?9zNI^h^lOM;L;6|d zcOl(#y+ugxKs>^c9)BI_k5!ye52?7KUS09MdS1m<^#t(wnAO+RtH9rL!0VgT6D)uJ z%|rp$z-bN~gJ(GW(8nEq_^gA|(l#M)6a1^`Wy<~QKhS-Eygwt2`gj6ms83xPWy)*o59qETuM276QF8S) zl%an8E&ovgr^u%rINV4oQzmZsSlHXf345TN_O|#$)M>-`j?X!$yg9B+IeWvu3;W!! zQRYqLVV{ei4gWuuPqIYKJE*jeEmMAiJ@pgpg|*1DBkvQG{RBSElQ<9UjVe=qyJ3&+ zTa;-;p18kU{VmGYq0C5(nLqZx8Q3?^R&=ThF^-4e8?krpg#Rw~&Izlv}3*XEy>K_1}tUz}N{ zJU66F**>&P*%n`>w0)e?o$ce`6I|gzWeZ;hUa|FOd4=1OgUauI(1Ty>xXmx-MclR4S>PMTLdpcXKNk9;!w-82 zeB&&^7x>1ph?o238&_?5<)O+C`s5wg=0{o1R(^O7{$c%};va?6CRmm{E%1+Hfkp?e z!LK=Sf5YL2{~P#6*nRj%|2v^Tbawm;V2Z*M#%K9BZ3)c;HO#>Ojbzslz^ zk`0b=ouKW~Ur5_mK-=^0rtOIlGkVH>$2_~8v^}>6ZQr^FZENpM+Y=)+mPyGO0*)bE z1Id5KLfsvHn502@<*6RD?b9I0mc8Y_t6BG>ZJ+;i+MY2j)AIP!j|(^l1~_mH4sqfh z;qb%zHz+S<1X%7R2i{W#oEvDlmkf9>`EOR<69O)g(T=untoa!lAJm|{S$=OCe|p(W zOO&EPX)pd68Xu1KUicT&__u8T#c>gq^P^-I+t5f$+iAU0<@2P2=0`RtTW0=3njh1k z{4TvW&Cj}<<`Wx)v(DMn9yG6X&}8rPPRFGC(fn-C{8J8^Pj=GHM28;i4P z)Vu(;B5&A=d{lbP%X)`xAP%;HDJ9;5ZJ^d^8_43qZ7l~`W-IDMz@|j@Bnsz;suIK~ zFGhKnJ*n-(ly1^Tl+`oAF7v;lbFVJXaQ5kr&V42NRQCtgr_TKa>M!z|ab6^y`+AH` z^(SnM5Qi?Y1?eswd^Yl$|DFA(=-GF}CNX!UL$8?g5zFig@3p|K+S)hXWyotjz()U9 z=n>xn{j~3X%3|9-+j9Q>S(Xb&>y#H~)+_JQzyDvXA0z!@^jqxz>ldB&i+@%>>(Vbq z?_inT@vK`6eYgZJ2riiEU-5!FDiRw$&&xfa<&v#g?(E(3VO6p!PO7zeR>>B5yt=uR z^T{6_uxWE#z`D)g6GpPFC1GBYf3#=%L!aXhJvzaa!L@Ih9+CSkFQroSMJh9LytHZ<(iNm4Sgc%NxvUbudF~`1@cxS?Ksk)7cC%t z==VeGp?}!br|!a~^Sm;h%Ip`)^pl%#C+&92ScYhX!TX-+q%@L zJ$0IC6D(hACR(nf*5fR;UpEr^#WBz?lJ0RN^oyjsygI2~sexWngS;o4`bd{P^ICd6 z^qC*(79sBir#^BK%8*{OE2Cby2|CM7$orjBXX(;)lFst)MUyPIlIn3D`^0636LgC% zUFDDC>TwSHgUc3I@GW$TE*+(PY`yYsd!O8C?}Kh~RK4;|=p^5SPI9eNC;29Hl6utN zFX$w%KsR|y&`CBqb&@kc|D==bgszfwl7+o=lB<59PSOi@J?L29iC*Z?M@m4y_tHmt z!WQZeoh8{m$qq{TNV0vBJ(TRAW1)jgfDSSdI>^lzS6be@R%&?!I>>7S?$|-Uh28Vo zfF5?xJL44}1%3XvQAtkwWs>@Z4Ic_;?4LJa|Lnqe-W0}m6UH^VVz>HD-eKoy!963gf99<25bU3Fg!OP+VGHG8 z3vGk0Q7%zgPV@E9yKL8;-tx3%0@lsIkYr0L;-^0Qjb&TqHr*iT`_7)N6Lf}WZFSJw z?bZ!}{_j%dySlUXI%UxM-8xUKL)SWnZa>~1G)OjwcM&E!4uCG2YyfAmKFKaYwu27n zmTCRIh%$mL0pZUOrZT&se|D|sPo3pgqx@WydB(ZUX-!N7-BaBcDGzmthfKg26CZe{ z;yZN;%7e!U=|w1?;anecD}EqeQ$V&B*Lq5+xTO9S^n{ng>XnbfPH-u*Ub&cT39}|z zeu%19W~_f-cPXY`c^qjEJ^iSqE2v(10&A?xuU?Ro6Xs2_T$f`1h7J3=C#|iIbo9b12xG;1k_65gsVB3uP$X6a8}0<1ygsqP_2( z^>zbiCk+a8MC11?`bc$mQ6AtBEzQGt-KI0Z2Mr48uR6wC7|RNbFKDoUXb|Hpj1!IB z51I*<=hwfbdu{z*-4D>qEruTL2hy3%dDPOaebRDm{YScR+=uASpJci5!xNT7vJJt8 z)RSJw_H+|CkgceA%}vMJch0TL7Igx&F%t7kG>3T?Y*%5JOP9@xc!fXZqna3}iB0R6i7kw5SvesziP zz}&ic(1R!+fbv9loxs&aYn~`iw04Ym6w0r5(h{xNHO{=xD9=eFs{w=PsRI4mh4SEM z0xi{`ylb89#Q3`4yVf?X=itXPE!Y44nB{fI0GBt^DL+8pf^#0XTzR!l`5I(~%dgid zgE!RZ!sk6<5%^Adoxpcw3ujtxUQ=6UV}CCOJ@up?x{uQzdotM)-&}Lcai0cw6Mu2s z&w+gtw4%Yh6Aij}PXg*ipdQ&W>Hf_~%11roFJaj0sXo!5Kx3%q;yp7_4>X8-Ka}5v zdB3WT$2}Z1+(Njm7$1`9R-w*mglYXmg9coD+r_6B15O3@tunwGgEH`idO1kj3YUU0 zJP5wy4w{O={1P2~2YPakc+%pTJj>EKllHK81(^@~k|1A!9|&`CDV_GU_XJ*Z9C?qU zFPINu-Y%uqDTz0^Xz2;$xoGFoq&lTP=!|G95O~c*9_T4Poct-XEkQG$vUHE7GFG8X z1j^9(Q5mAS>q&Kx?>-XbyV=O2eFD5z;BPmk*FnDfSdj0Ekyea6T1zWX?}v;!<*T51 z7r*~Eq#-YUCHxxD@hHsCaq#DNvBz{&+*E%CS;1q?kLoj!5khAD+7g!YGvG`)`mg>|JP==K02`I_z~HyJQ*M@e*v|!rpXqTpjklfA4YM>kd8j zU-lbIRCJc5eaHmMCmT$HUE2hjQNU&myS6}!-)=Aqw(KKFPjlL`N$xxspK7sh*sHT+ z{%t6$MV=jH$R;j+d7n-^ORn?YDKDr*x_GBNp21H9U2nn|>|fmm`o5xm9DUco|MTj0 z(CroVk~MbF=@s>r)i#0m>0$q_gZ)ni#NWoP_${(7`A^i+`q$B^T)>1 z9k$Eys%|Uv8?i`h&Z<{ZTI*JDsgb|Fd^GgPnIeBg~D%9CteF6@BZP&J!+*zH#}UaEWF2z0!SB$WYftdI_n}{| z_80WS!n-LE(o~2WW@7z7UsMji8u!s#UZA_>k9VCE{LHTZARl)_{|=w-ssByzBg}t+ ze_}0_&FC7ofZjc5y5itB^I_&#!4EYL7W@$N148~2T?65RuWuy2&V#25_`NySSns9R z3P(oBQ_at#U|m+*rRwmHqmsmV1lVF zCKYeWQt-AcdEv!zM;6dqv+8`&rX6s5;P%07(~35|4gY<(58*zB`yJ9=fm;u^4vx}l z;Tqt|5if=N6yd{gU%)lNl^|^y+}8;I1@7-~C*e-RorSZ(U4&bRGMC_AhI=02A~*_n z!QF(@B0eAPM}$T2=^lOy?zPiCn8V>e02d244DLa=(QpZHkHdX~y8nRt4Z^eG z#yZnyz)yrz!llCOIMEfZRvJ@W;m4H5 zBv&|mym7oMoIT!{NMT-Vep4$7zB=<0%n^rod?M4Wa%L3X*+BnU>%j`8Jj=YybLJ#B zR=8KhGJoJj=E+M31V*{D!f3=9uh+Z{{OYjhtzEf)OB){)CoYj}33-eug2c>9Zvm;D zXwbtV6vl;q>w($l&SS)8o;jIZ%-C|{`Y0W%ryXY&mB^Ti>)5;IF1eFABY3A#c zcvyLhxds0QkKaF`u_IilzmX_gjee?tdzqK2`xMgi1zbabYbtO`LBEqtz|lzfHsgIs zGV)K_?O`_yKEhhzux3Ok!=tPl@OMYwIiuv^a?LTUQ-5DoHxnhDuzRYS?5RVVXlx1p z!^l%8RNV@!VG}Rw+T+b?B>L)C+P6IWo`OjXNzhC4-YiqXsS=ZJ@r7klgxwL&;R~Y? zzKr>5#u#70ymJqhYrY2^e(1rP?7pmXyO?FRf$nX<^(xAwp`UmN9IS@EyPfwnpX9=9 zt2|leFwV=c7Vx70KN@dV)&jm-!i+w*_!+g`{K)Oli$~4l3*{axGYaj>5%&q`A6T2r z7uMafXHJ7&zLt~PV%=Hh%kHXW7WcZ@UvfT%lSo_Z#xni5s9)J7d|?jWA4S2pK^Jd} zVbgU4%M@Le*D^nR><8q3678ZnpPn-ekT6EOFw zpn(+7LNfT~Az_Wk-I!nVe8|)oe~XbB)m+{2BV}HO5txetR(HGu{Ai?eUMyQNN1UGl zZ(%z1z;$5RI)jR-5>wq&i8(9WGOAyEM3L>y0#n4Q#Ojxl6mwPFj8evpoC{qlAIxTy z`m>QKv8l^j@m}*J$7v6u|1C$E^#bt(5z~q=S9nVvOyO;qi{`JGwG(uB3N(DUuzyBe zQU8oKZG^SGAi}EUB+@*|VOiUv2&;kflIBZ}$l70suqJu3LcF!klww@)ra5@1Cu=%~ zzE##U<9XP2*MCx?__9P2x$)B!h5t%P*MZ!AkK|{lM z=}{lNqq~6m+Qi{ov`x&oFtJ2oLH#dLZ=p3wp~aX~;>}?)=#pq>E9UnNj*V_b9a#%A zP99R4CoWNJVP}?qos4l*V9r!)L|t_q_L@XbyeB}tr5*Os8E~`?+OZzQu{fBZzL*yq zT0L2}jF%eMdNCu`WOqNFH%G|>Gr z-pi20OKp2%Smq7TMO7YOxDRW%Du6EpzsjU|*#MTA6~PJ{+U%L5ur{=znD+oSG8J*{ z_0q2wM6iG=&P&_aiZ_~(Dr*(iffiwUoA_XeDse=FD)A!v-Wj2?CW-vD&HgOYf_`4a zI9G8#HtO40(8F3$kiiT6o#Mj+vr1Uuz&DbYYaeCCImTJnThG`tTfleZ{p!YZJH!P8 zclgJ%9pZQ%e)JCi0tV;Cw(91wZMq#egEF>tL=M|FLXI;mqd3l=-Z=NY%1I5Sq6k~= zX?tc`37?V1MFmdbBynEv>oxK`^A^!WW*j8Kx$l&HG?kls1Md|vFI7u0o?aM_g?u3w z$}%bMUV*&GyVJB`aKuwY12C!+1y7#7JWSs+)?IWnUA{daY@^gqIkY zQFW8zTtM>8X!!CMJuJ+a>`Y5>rg*APB|@L%93V2tI! zDF^->_}k&{fPaS42UM&52OLnZjS?GJGP5)zea?Vw>f`}i)jJ04P?rzbu5LydkvnH` z^ATC<2kcZcci!?o@XQJT|NVPQ;A!~G0cBwvb*9WXdw0k z)}R+FJoRULriBkbP>TJy4E$Yzc^nGbvK2>Ihl0Lrr4d#|e^ug8)VGx*3>a-|BCNRc zf$ss(UWe}jZ1?VN^Hg;w-D0LBj&&zt-85hg;hP=75+AHhj9U`+r-Rs2UV@!CPNGW8 za_&zGA5~)8kM_|DtfwUGL3*qwT8j^$u0ZE#dqnizbY2J8z3IH<$Dj65{1y6$el>oH zKBl6N=r03*72sk|8lkuwZZ`I%-f-sw?hVN9xqQUY*4^IStuc^a`r&>F*4LNz$F*Hr5w2Uvcl@iPF#iqmyvhl(pMiR0PZo&!{?ZTj**Ue zZ2iohNqk`sd`m<4fsL4J!ZG$U`)GPAQO%FH)RvAlT_*m4u`r;IWVa`t8v-8vL8_?f zL-b?M!K?!k>>G!dk2+vlopJ!OWTbeWEQ6J(UL~38P%ie;<)garp{trj9^fycAX^UR z$~6w#0OyP++tji>MG*yja#2SVQYC8+Qdu9pWq-XD?W-UclI(asOl74q(U3uE;Scp= znKqOwMOf?W(9f9oC%er2>jy~2A^K5(-lFP<+wy(DBd8pB8}0=hUkLebMBah~E%21$ zz4lvqLldya7;Qm%E6pYDAlRfROKo1D_525W4el%)-;X|5FJIi%fPUe8h&3K8yz#Z& zvWF3FLq7puN4Xl5(=T7tWkosaH|~U4OGRQEwcm`ggue~^{50VYSzU{;81+m;mzwtS zv%0pvuTxwYrq}pM^_q@Qy{0KauW63bYqs6hH<8e{br=r;24EA7K7+oU*^;K%xxy#% zY{o3bPVdiTSN=X-v45pc3WMlFcCLCHJjy&Z3C)tUW;>bVw{z*0{*dub{;du$G z`v*rpd-&M|eheQue}(GR(|?RHlr!G2lZj>=g8OWRZ{+TQYos+R{2~uM<{!Ch;Og|9 zD*_^S4t!C1cExTl`%S*^>^piHzQdPzXvHqClZYQe{8=}axO2r$uUf=+B0f*d5^0=I zW4voQ-e$m?KN{;JM2DDj8cU+No@4&3bIfThtDIv=<dD`-*U&kG z*5P*G*Sb4Rae(ST&JmY{-!3agS)$q2-P0A;+xZc<^BMB#>`^LWoo?Z(>-FGG@u1ld z^dlT=0QXolL`SqvS|G0wzc>xqL z#!`Ya1K_2Vs$TtEq_Wxolhz2`snKGMB-O{-@=3lx9ima9#q*#eU0!ZNCDzEWG2UC& z=fx(_nmUI$i{iX(Ls703b9NDPMrZ3E=VPq{&zpiDV!jT4kaJxaz{A@>>x9!#ghQRU zv_tl~fx7rMRPYI`lXmO}#gKtc9*niMW6!4Y>1d}P%3lXQt$WiH;*x%mwo$VbVxOb3 z^B+%#tk^GdFT&Z{6$xo2p^<4mpUb|0KRG%)GUrHc!XCuM!y_WMA>3Gmd2$y^QyXFp z`#jjDH$9oL4gIO&CAOiEjj2E1I{S0P*`G#de+~(LnEBSt`=9sd%zYK|HqnWk(Rov^ z49WH|(s_~7>oqF@r%qI-jQ);I7lGG`HkIjA2i{eS*x&(K%&kxXdPW`1{mvBn6rzhD z^FhGETxl@3nj<*#8Rkfg!w?1^2=v8!|9H7x^C04+2LfEF+?~yk2XfXKdC%egWq?$% zEGh$M4P(@!IB(#LsY1HZ%ijPx8XfxtXQg;H_SvX`tWylWUB~&SmqpjfPH|lN+=H*$mvf zQTGiN5f?Fu9hi-CM;X$;LHY|4D(`{v4_i)ojDs5wr-XY1E(wmoah7YizlQGx3Fo5E zQBRI}D}`?_=bLU8h;gQr;B4uIGo}=0O>gKIe4t<8jl^ff-YTo)fpU#M^aZqkv&pNK zQA;DNlH6I=!&BL41=Cx-aL#&YTHKZ+Z>1_0Jfu?S9+|0Fm*^Fl2AW9o`BJu5DUH-3 zywK+>S$cmqa%v(AFuZG!?L+u2gg<|`QdUd+VFAt`w2tBXVovf)_#gw^SHIF*7l@=r zI?pMl4c|h#3@SI}(<)h8t(&P7;XDdQvF?}ZhZ#GDc$gB<$D24u`~~-GuMHe$j0C*H zC4O5><+82|xHA|EY|+opEMfhT2Qzdkje zWqJ)R*W~BtccC9mQ}gl?=&Tlv^r@W0P>J&z#aqB@hoR1bZQk9)|4KQDZ7b+6o@c@9 za`+ip(QIV-Jjk?=C-YE-@M|0Z-AwuHt~{I*Wc?+E+@e`l6LbQK;d*QPSXH+a=atcl zp<5>7T$=m*ENeteqijk3Ea6m{vX_y>$|^=ILfx^Fgna9>uZ3N}ThAa!9Wg ziM3M^7o!i^kavmiUzNe;GN)^-1ZRIQZ&rkF+GXaVtbz-XLIuYd207p=wku;5sq6O*H2kcGhgqvZ|(an(C0xP4B$h6!r zFQVD}K}Td+7{m6kN?95wm1;5WY0*bzwj+57X~jdAjscIMF?|{9|Bt{m3FkmAj%Dso zVF5ITlxKk~{R#4jhuV%T5$f&tKz;}tQ!A2aKgXG}=X|_^`DhOA8Y{+p;1iBTDvV_k zXf!z(V~=^LMK~RNG!M8#KcX=cp1@x*=*Z?IPIakmlFAy#y&)qyAo$ZTUaQf56mV=a6%UVGp+8n`+c2;&&Ihi1g{jBbL@;A0{~p-*L{okiRftdhvy& z7eRw_fJ+wSN-6Mr4!F#4V>1eYhyUQF&CmVl`Ot&BpVxyw+Q-;2eirO|fG0hNF(;ZF z4*apt&uBxu5cHG-dTPd=MsV;s#NfH0se$0#b1-Mm+3q zk(>q0L3>gC*hs4Dg*i=P0k#;>4bjzipsOo@pM^A{pJ#IC5FCMiQk<}pL?VHHTKE9- z4=79UJKIk-^MU&U;BEu`(R>i@I^g~?#%m$U ze+Bv9e#?RTIh1LOmY|$>_d=|Pg}}WOe2?lL>BLzHI#?I(7oE7*B8_nW3i34I2)NT+ z5bP|c9)-T0MYt1u!hHj9--j~9dkA-gn<#wVsTXqLeg)-opK$PqMxu#;Dw)>3N|tW( z97}Zba=Rd-X`TR#DCmOp;Axo4V5+kx2|S4S3uy5u((?sc^U|INt~9T?;Ga}3_7OHF z*B7!j=Dh7MRkAmENqRBKL$wo3rI3d%G)y#cf3jyD!?>A1dsuS;Ht;dhixJ<<;v`x< z#`;`-Uc$8Eb4wT6?W40ncTr?}KwCN*bI@G@#`4QSUv1Wb?ndxFUL!zvwPzkQOr-6uYtMQ1V@-J209|x7(0R1I>Do8z6d_e*DB0c0Lm0P z=L_K`3adbCQ9Wo)0)C%I`;v+k#(<~f)A|E{K7w=S0`zMk`lZEq5Kk^ZpWedw5zp%Y z?L;|w@_Fdvr30mAE&7xO8YI3Uz<`dQ);Y;Yw01hspT5cwPtE}RlPF7c*^#Fs*g`+Q zbhcN;`3n6!&wC3znc5_tyb=BU0%eGHY3(4~L}6d>ZaXQTb_L5_WzIn^vGJlfTM%?-fFq3p4+vwz zy2m2T9M(f-B6*4Arrt7B3UtYR$xLO~i+ajT9O$X1%!ED#lH5e^B~D>3Tyj%GWs4_Dfi5I)L+cw*+!xXpCaZKX$2vv+CNt$;;cpuV|czzqD*m zhF%cgS3?`5^QFD9QKquCW-u$A(+s6JcjHVMTLS-Wu|g~jH;6k+zHS|hcH)_p_R03A zR950!ZSg9sC5cuJeia5>+Mfap?8TEYM{OA@YkRuN8ljLx;yb{CEHI^b=>eL*$;Al= z@*r1;F{f+48ME{nc#&Py&!z&LKLx6~WBD*?2=rYf4>n@oU4V68g}E@0oCX?t3H&w> zG$bB$cylcHAIXNf;1w@@vt%jY$I;p&+fo~3NO7L&0Lhr7`{oA4ZE?vROTuBF59k`J zx?vxkOZ=5%+I;NMw7+S=S7>h|zOoQ}WdX*G_LH~rv`zBxue5KOaUKV%`*$r&i??~38HmPsx z!B-3@qjmBXgqtY*Irh7%9(*Ow*_U?Iqj96Xy089_oI$V~qs1ua-8~ikSwMUedo|Tv z2i{9`Nqe=cPb6nlA?;qcC*8(fBJh5KPrSb;?zC6;#@)47)A+;Ym`>vwJE(CpXgyGY zv!@t5zzh3de#{YB6lCkR-7m?Art$!*=e~RH!(>GzCrNJyO%tsWFKEX)CVK4{rRp}c zfX{+g5Fdej5dgVEYk&=d_F+mBIsCv|#nW&uoQm^dI?jn{I4@3t4RW#x=SRU_?UE;& zgQ{lJnB321ZO6A3_qelTV%Vj{uuJa&@1ZlM>ul-ujDvR>CbLZXuBRm1!M~)4lO5Wg z#jM1uz5uUE#QyoP7<{ZjWYj{hZyU_ zgB)$N&4iC~whZ_vYnu)qO;!bhCyrNXCrN${!OIQUc#yg^I# zz>|nSor-brPHi%HCwKzMOg88;KIhq(lbFwwpt0H#W;zP`JdCv#2U@1H=C%};c?9cC z3wm~)GdZO7=I5aIp65&t(H(v{SLNWBhf#*`IXuvzBhy0W$C*>WM~bk0IVeSwOEIZ zUg#%ml~@;*L(h>F>iBd{)gR}|Isx6 zt0pU2rFYMNH0ED{`3%7P$H5H+4GaSf3o4O&P6EzAWi=s^pA1kd{v{j6Bw9$5hx z6@U@N+-&UKwB z=S^Wb!tI#*bl@@pwD!lw=!AQ4== zLRrx8nMsbaWIsGy)Pr_)_n}>s`=@Ca@Osm3GU)C7Q0T`(@6hgxpyk7>rv2B`Ze)t- zN57w;UG(!`O1piupADM$3GMb#PLFawq1`^pRTGW5Xg9S7-Sknmj>@`dH|hRmo2jge zc5z4RHtqHS_b%Fn9DTpCchN3nD=JI0+t@=7BFJLSbHibh#UK~ZzB;tlo_WM6huN}J zR_y%&Hpn@Up|!S8?3qqEOhfX#39^&xP9L3NIPfdTRQJ6nMrW$N>Z-6`pUK7@RoGvr zz_r6J90y+PC0{3%!afZ9b>unNh!w-VBPH^e(vQMcEJm7GzE+x6V)n8@uFrzJo(5e$ zbkB*35|dXF;tIs`V3!q_?D2|1T#R@w?6P#X@xHQQxku9}$XUJR!)BaAanDnbS%%_H z4&6f~J70T{syjUg=U?1~#61vW+Yr`0C4x7$z0FK?2E2jaq7Z}c;w;xKmcTX%xrWlo z9!PdT+Ry$rLb7Gl&{RbhoFQTidV}A71vsfi~C=$MhE)CNF~h z7H7bA$dLrQJwDZh`7(Cl%-0&PvWD|wDdx>ax@I~LlH9oh_d~m&qbIv4o%3MZ39#e7 zf*j)?3*6d7{S0F0{z(RIE_bsum%GDxz(8z5kJPVVkp;!W4m?*p{6Kc`hy(i|M=dBGc|aS$I&&bKG-K>qFs_rp2U{vRH%hdmoBAL{GAmlUb z#nsps=nc#|oXxr8SqZC_Ki`$yI5L60!*CSmu`KxMpl`Y-e-wTd#jz*XQXKwCieq24 z!%qXwg8-j9<$#k?H_`z8ZDJPkyfDt(r5@>^#ZsiNLmdvbHELHV@68gcs63^6^hoz- ziM3Q7bIN)3NDpR-C#gK8`}Rl=XNh*Co55$ez#i%Sum->roNMu&p6i(G8}@RK`>x~b zS@^AkUU#(X>8+IiBSr<*G0x<|eHIRMmxA#hg*4ok9z(YCM1;lqYE2Xq+gdT+WJ9KO zDl01!-7uJ^o$eq8ALoqOnCX8po@=(ulr5`A>sL!a>Q z9Ly87P3MMt-9x9g8haOg(fotHj?-@d;1`L6yDLOTI-%V=V?=uc`U+ix_BWucn0n%CdQWdXSBwUFV1Kk z!V=qtvczlgERoJthf$w+p#d&R0^2oLB?ArOj)g%+XRgWIOESPpw13T(+LKsfJ+0H& zM~v9tFjhh4?PyyD-nt+6YBs^q+PDNfx&(R*HeW=T?%T+CW>CR)@+9IajCnWGT2S^B zcPQ;kR2CzhV2;F5N9K+4NXe%*{8L#1fMlm05qdksnv$U_p z`EJy-vv-Py{#XUwcd+?o{3eY8_M_n?<}H?61z($XN+U_X=Hra-*(!~MEmyct5t}TI z^Thfjn*P&o1YQw{-xv;#1zhR}(Ntr8;sMJ49QgtrjDN%!6XcqkQ1kDExd}D@(UrfH z)>F{!b+ZxqSR0yF$Rd#cezI@eDxQRO@hH~EM68nuSTCv2&!yl!iAd1TEdbo!JXH>w zr-z!0sT}a_&C?z0`R;c;_m%$| zf|^KX>bZ_dKPXRRHQ2jeY1{tnd-S_sQP8=TC&1>&pwGt|Ka4eQ!Wy4~HBNbQ(5(Q+ zAur!GUiz^tGnVi@>sipu`&ycLA2@F~DV!G^WGx;t7w>6FQn5~clyE;)19?+JYr&2; z4-%SJIDg<80cS6jnsC1)u-`KqHT^#Pq{s_vI|R1&k&>4*#>wU)5nCG=^U_*P%vy6% zv!{;_^L*e#^wNPhWJHh6p3_X%ao_(2#`FiQud59zYhSSTioy)} zxa+@UB;Lq~INREIvh#Zx(m9p&1<>747IIXJd(PR43D>ueV32D=4C%Pne~up-S6RYk zyv+4WE9F?xUanHMphw)mz5HJupNc!(TFyJ|e|NjR+nb6z<^MlgS^YN)*GF#o(>a5WGW^vciSX zAC~&DLi{vuW@#kuUXUFvh!xi39nEC$1+u~C`Qu$O;EK73qfuDfr=i2Y+Q6(=1ztfs zLk4~xVNT@33>~m>H1qzD7b46JPm4BP2OppL5Bunc|6$)h53pRnbJ&jeQ??)2uV8=B z=0e9m#RqpZeQ>u7@8F_tNede>PB@EVFLm%MK3lK347gwaTC~Z<*NDftcvJ`W2;yPH zBlGZEify>VtSAY{Xv<>V#NUn!Zy4BU>T?U(Gm(a$rVM^bg8L;B+{eQheG%I!lLN}r6j zsf;-K8`)759bl-F3HP}|k4oUC`U%ER+Z9$F~=nrzIK#yB-&Lvt} z3;glBy}>tdUKHjKYm?G*kxsZ%xb-PjH*CS3lqPmxD8&c+u|klS(@NuHhIX7?%#xB<+C5qK`FMmo5gx^xguF1;orkw= zZ8MqmM_zW+y8`cP|HzE%Q7?p-8Kz*I+vcMT`Eyx!7i^PlyjxtGmUW+c9lzcB0_wqj zNo!E`K))??zD~h-{s^7tsn;i&Xe=+kpt90fG(#`n3Ywz(?cscc;oQelOqE=O;XLTp z0{=_6{gBVw=F?j&5qz|TH?Kk%ZMVS&KzHJ^anB6D-)fF4?YD)-rv)@PG+31YzFv6l zV~OcJc-MKX!&02B=x%B$&Q#gNA92P?egOCU!`W!5Fn^(D$8UOun@O&MOr{Bd<3)bb zXIVgc1M}0)&n%za!2Gpy;UB`4)I*u8XO}V$?Src}%Ixq@ZA@L>g!hPkt2W3k;_mua zAuNG(da#KL??bD>_e-JAIf(V`M`L)?UPEoXgF4@1-F%g|GGPkRDE=*53|>fk8Fw#6!1-ylv@c7~yHAt)Xom{8o)k%K+1LvNJfZKRyFO%#KkUnDKJaCu zd$xCSAS>jtwqv_hU$>d^`^g#6C4P1r;r+u8a)Vl{RbZ28GaFzmhvMVpZooSLeeYAPbvWf&+Hz!3BTQ&Nf zgz-HEym1#-(^nY|@KOQiqzJzbgFOo6jvEkf9g=3ULC474OD!V&e#`@}t+WjOw^EC~ zLXK}^MdQ0z1J$Q1V(>083Hht`(QnmgG1hdyh{nAQb?MCTOZu9PaiQ|I_f;n7BWrB1 zF;?g2-Q=`+H{}&_-|BqDO+~ch-Z%CCTo~4hFt^fQ1|M@G;k^wHHgJ{u6v*PoAXej>O)c6wW#uXii~&g*+7)6%F2nxEMB8Nwg%cy?BGB{gV%i z6tKCh^IxS&6K^d7|JUX@^?sxSYhanwmW&CIu58RRpjmG0xYygPG}62JNFF^GE1Y=}OKdSF##9>q0KkT$}4{ z>{`W)0ea98_C$X-y{6HPZ4?zYD9M*Barj=#9KN)qLFw)0Hl6Z(${cCFFFJg`m+*TD z(DhKB_~iy=1N^!2@y0*Guc%8h{s}gZP1`u*pAp}GNoln0o?<$OJ!IX(xbK|$1bu(G zP>b>Un1ABH0?>Kx2P#u7c-Wq@wVP9L)~ED*5%wJ1<1*B*#~pv~bZviZ0(cXCfgf(h zlN>`S=WD}96gQtNW1XaPrMuZv0qZE%QbB;~68**~*=Dk%+|)1PcXvuM&|=pM!Qd2^z+s`XLTYR{Q@H5ryl(PoBqy#chpS- z-c|oF;C-C)oB_Ml3~#$*@SR`QZ##aYZyWrr zYEH#;@O^#o#{R8a*#{D@Ob13ey@A+VDNo_5l z&&P2;RSO+7rRA1#8S8M5u?l;9E@VvYdn!{E${ETGo98I-?g8^gcLIs;eF>SX5%(I^ zfQxSw2yp2;BRJa$aFYOc0pL=av_zDFZ;Z^?J8qal?9KvPA*Yf~wr()?xLRhp@F8ch z?G;(t47{Zi@)VtmqG+#%ZJ`;o-EvxFtO#JRXW$!0;E%b38f9M;;=VH8Qiy?P{#z;& z+4%Cy)@>dLSwsgsKX>-OGWrYITHv`4YrP$|N(pH8T9TQ zAE3#EToGK2b=ZQoPoZri+IH9_ATMOVb{Q>vi;8TL^!`rs*3bot=P*|B2p6L5EZlit zh;eQ~-$^d7;{0u^$~JC34VjA44nhWbAm1^L74imI8RO!fk+XnOyrH1^InBG7vzTKX zZK&Ul_&~-2N37s7u%FDB#(d&VEiYJ_!-=G|`1S|M4q}9dE{)2t;GG}6%dsaz9}3!` z`ZN!34aL}V(sVKOsN$z`mkM@Z)JuZww@St`$3uP;=O!j}<2Uh=y6vO)qh1u}R>&|O zQA^wkb0g4(yW5PzSoezP@y3T{<>Ea!Yf75I-lg`pF3&wcV|MCmk@0F`ql zzsv={L1f$l_<9jwK>q2+^^>}IQWbRMQj~4w`^62z?=^1+Y(>LEhIYUuIsP#6dbUM; zv$4}YdMVoMo$i9!1>R0JprN2e$buS@cVZB(;<>`ppgkMTLU}pMUx~wev8j*|&U-rK z?hDw5^zJIlDxYLaHTKSJj=j_PCH7B}MUp^+^}tUInBAa@zoQ)r)7waj9}!8?X-v|2 z@lhHV!j0sAiHY&hmq|s^xQ)aQuqGV*z~7ATNys zJjHv(jps#%Rs59edmw{cgC3#_dTUy@G}p;EhjimD53Off*R;NAonM9R;0m2xK+n@K zhpn|L3+*)*K1{aIeq`I50v?cRq5X^YG1^;b-O>I+V{uw&C(KM^(F(oTcLFSe8EWpt zS(e5ppJ#zY!)uS)M+qgbg5%n=Ix$zS{gT?H_6ZNd$A#N=*luY5qc(RS zOnacKjoe1Omj!*&n;f#G(;B0-_Q8`58>PKG6zBY3{+%_~I_jNO=K-6HHho8 z4v^jACCIq6-w%Zw1{=va*hq%MMsfjT^c8f2TFy^eP_l2!Mc7De7N5v`&PzJo8E-fF zL_)q7Y$SQfUU4)Y3c$hnLQB65|7Wz>+a>|q$VRedKac)oqiwPo(;1y?*?Oeuown@N z2$K%%UbgHdG#=mwqzA~PFktkyO^`jN9C^)s*s`;b-qV(yO65RHbPhh&?Xr7>n4c8V ze`@!jvNI?TeYB@?Of6rIg7XNC(KW1vtDtd*4OV2{4IJfCRue47omg*`!VPz4$iCy! ziwwee;yVhrZLx!${d)ku9C@#b*_hLiXMvYyteDk&M|u`dR?`SsOw2Lk7|bQ*!M<4Y zfFm8U+atz!q-Tj?GjZT7eDiB7aEyh#OW*cNMwo1iw05rnUwX?t z108n-LXn1Xgr4UN>MVi%agMJ_fp1V)Y3?ZRA@spvXU3YGNpl2y34A>09!>qYRXhP> zo{F(gfn5yWSHpc4r=3|d7j~@)a6+Fx;e_uq9Yp_9R!Cuc;{o5zyb9w%{k#{PnRa{4 zLvS=!G)926QCyN_q<0ESL02>3h=+WKe4(8;n341a1as*v;~?4uPyy}3;0{NxV!Z0O)(!smlkY3i1Cv5XTp7s7oaD260VgGk86WY ztep>w%fcBG@5qI-<*<@HTYT`HG+(Tzi?AVG#F?=JHvdk@a%5X>;YY-AJSQdn!odK8 zY;HiA?BoE{=T^yLMCG!R(1Cu(t(ScY8~MplR`?88f$)n$dP-e|Y;GOGfm8rA_AM!HBEvS3$UyhRq5DXT>A+nsf3}q5E!=4~ zFuVal|8DoK5@!xq-&W|DVu=Uq7HG1GH!{cu&1;yYHI*4dw=(0+ai9m#VJqG-NO10N z!+o}EqnI%Q_sTXSo!)?ut$a2gesno_u>x`d>1Yy0Zr)rcydihk)e}HJt?)=OC5J@xx^ zXY5w78oXf|c*IojigfUdG}!*9n7~5>Ubhf9kse|WWCnsm{BJURlX!zh@x{I((hX(d z+i!Bnb&$CXUeGgSahz=r=Fl5^(K_g#=#FPJ;_L9812Mi`keo3@qh1gxEwg3zyakJ zm(&E5%(Rfa9nccNu+W**F5;y^QA}+n(G*a}8``+dt~3K?7uiKwX-V0rS>|MUYo?}V zKI65dsMM(Rqz%9KvoCNWoA3Ac`{VQbW53qgYpv(9p7pF}-PfvTJ%zfvCfwV&8F^-s z7nxr)(gJ^sJ$I5;(pBM?@zPq-?q^KoS{GSoERHd*%JWoqOk6x6E`X*M30$MMoQ0b7x$7HiftsRB!oyWN>6; zfN_7{3*5P9{4?gMq$eQA${?s?*+ zj}>C|vFOW;mGFIpf5-g&hkQof>C4Dnd$Lv}@seK$JRBii-T7AbQ+CzA6F>Gz_G_gQ zC;K?$9;JhXCo5f>*&k=%-on3}hqBi%Paunv$YMTskeU6QSbdL?#-qP$@%b$O_nvRv zGKLR+HoG*3)@a1V>TeMD6#hEo=D#P8ImNbna9<{kF<-S24b1D7b;u{d)ZG05B zf;)MJF(ylET2fS@xJr++MicUOWl`S!^)=J(4@n4}#Uc#zF?#ovMfvNVnp4Uis#4~S zW=u;M^I(X(pt(%u;v8fA!`BmDx@b#HDgBfR`og)4e-(AgpH=6Sq~ne$N!`*WB@J^; zNm63@c1}uCU6Yb}DbtclXZ{kG&`?~HliYss)8Qc>9tz;$-jL9O>Ch@LXoWN%?HX8r zlJ&k2S$Bq(&~!wUCFRCWO3I*|9pHM9J*Ll*p91o;9{*SL@#4{E(bK7v%k{QS_jPvA z=>YT~Q2$EGhqON}7=_#mkbgeqFp~1f!*-9LpK97}opS)+$-M#eK_;qUrJIn$8a1@w zM%H~cs#6RZ%u;42y+FSABg6d*pR3tVz7CSFPNAI&=qJoR-H$b`*-GST;!hKQn)oxs zcMk1baB$&^HA4d!ADY3;#x$~wZeX(d;O>!V> zCd+H%oyf+ULKycv$U6Dt#K}oHlt&tTd0oXxGzm#Ex90i-Yn!;~xXIj))}`-*taV10 zUekAd{E9`d)JUJS^72<|s*uZpvD6u4DC2wX@XFZX=*s?%aq|_UPDwx0?0cMGY<-eC z_Zju;bLv%p?mfH$-j8j(zCP{y)-5WZF}L%zQ)^)E?LqcV+{@Q0{nKlVxsb!@^!>gJ z>RMgEe!LmM3Dv8=U-Y5eH7oNer=hd+*u+7~KT;Rf?MmLh^Q`4pl{a^)-oZwApz`K^ z&K^j6fV_=$CFUG8xgU_ZX7;MZHdi6jSHhJ3>-k*R66SRdPoH)Fy3*i9>t@`yXx)w2 zu`+1yhxTb_*uKr|6;+rtAE(=?1Jb-N1)UH|X)*gV)lJi@fka(M#uE znAi6T^TNUVQwzVvCe_ksFQK0ydwo;D zRhvtHfd2sgBl%|@2OAl#-xWQ$;m(Mm4O3%>G~5-f;CJ9x8>T2uEtfra(&kN66PmrW z`!a`llCK(lIIae8XP~Qk%KZ^F!<^3QJJ%`=cLgjNlpCSEv;mkC^wVpBtE8QuOuAgs zZI5ro$=b21I;)IDGGC;pM)rP0pIh|dD*EIfDqX7Q&+k<8A$sK9Jz;y!l3TZX|1c=i zJM6aYInLYh56<-NzMb$p@DCw;BH`XQy?+>*>Fw-&Gbcg(N~X6b{(jvo!_V_s%hCR=zc1d(~pe zXLe>~?rqzv9FouLGArM@op8zLK*A>yE_oP~S=q_^W>viS2WM7Zf&Xgp56P??flF5D4`=sV6?gN7Q>|mg4$AmNXDlW2^SeH0oXUFsU1N-UJAKK_;pfoCHuYj( z0BiR#J(Tn>dyYsC?V+ZB=_btM^@a9uq<@+0Z5&A*4(;JI#%5ozhl2H4Ul?n?Vt0$c zC+&MvKi2=*lXQ$d02^?PwrF;M7K$xA3QV3~x=lOGbdzw6biM%X3+8I*L$5XGq+I&9 z+<9r+5~>gl}j&Mt4&*qy2cmY!}qUrt4T{y3SR*2VX^-r%}g^ zHiEX#1wYK~GhY&3o$XYs(`hGMYS(IcW82I9z^#@Y7rWleu;X#qaRs~V)^A?xZTN$t z6fRTL!sUviaH--fyeVM!psO89VLyjjxaS|KWl!_n6&={n!{I2r!r?58Brck`82;ye znp#%Gx0E!e_@*WXG|bI&HUyAP!j=A!y(ic-w?o7&k{?4S0#HZk&g8wtt3a_Al91u~QwCDM$ zNx^Amy0iELh?Bnm>48q(TM6LZmB5)_bPJmKYhwXh*8(GEBt1LeuB204m7m^CyEAF? zfZ0iXGww|K(y3-NJA*QUymuvqDgQeBRe004>(smdyl38=q`!J6Cw=V<%=pF`ka1w< z{p(J%=j}k?qIIWPZ$0q4MeD8(3@E%4Io=cKt`1_HeVRRRr=6Ymou+-|K6-6TU}&|6 zd`3siN{U%~Pg0kN`;)RRt1rw73@ghDj4q2^Tbz^?m{^ugSTl8U5NhMcGKKRN015amtt{-93Hb@Z7&r>`!&y+{8$ zC}X<8&lR}u!dKr#FNF?Of=dT&ULJpnIoqL*#L|4m!3_hZCyj7MR-Xz8t`1g$tKG;X zl<#}?T881ucpZCs_85I9J5It6p{JaM(M~Cz>K%?Q&2`klXP|#YI(#xlJFv4j>1gWQ zBw6?S<~L)L_qgs#+CQq0<3&nzC*Y5!PEA@x+~<>io4luFbn^a)(OfMSOus);XFwHw z+Jf0LYBaEANe50$9B^9gnRK89_p-3U)4Y3k`jbgz&5?UXA8>jH<6rb%+UVr>(`F>e zopb-@yj86;q4_9sxr;H)GX`Ii+xT)*e}gPOu_ojxn`gCG-+E|_pIc8 zvr`6Kuzs2~!2|SdPt(4fQG=_`Vl(=7imu7JOKBJwS~&MVWI0tj{(4Y+iwVhONWoyq=SBYyjigK*qH}jBh!sx4^}`fuR!Vl9pXT8G_+Z@{W(fG6qy(Z+R!VkG4 zP-oujtn9&%`GCBXro$@rAVq8;^8h*2Ntp{sQ_7Q4DQg9P2!0uVhFas#UbuPJD3KxT zMJ%E3pGMz5ow@(s^!KL`o=&*@XY&6-h!Xi%eN|DtuFu)enBY_TW-xuzz7h!hSV*%r}g=_S4@uK-yV>%GM*kz}c6xHnw7B;uue0 zSivUTguw8EiYV`FnTx9McxUfLx9;W5nKb$j%vH^iyD65i|6lqOxy(0l#?_po>Jdyo z-<)$;jQ+@a)?QhE(gKY+2gAQm*0N-6&B^+OOW#lVnez^Ll*EVo#FMqqL`(MA!|KZfPxyXA1b0CZPWbA(@Y3_=hnxwm$ z4v!%2Dc9Xe@9<6MJYP`6^rT?yb_8uv0q5>#Q%<9hUu_I`8M5bdHG6c{L8mDG=H#Nc zCM6YR+@0jTd`i-n$Y2!Zu=;n&oR?C@jsdo2p(i1U{Ei7NPAb0qo}^IZWu#%=LiP`B zWR2=U?k1?=jWV$tqa3r7X7aH=C90M^1W(0z=T{X>%HtKvrHFrlm;X2T|B!jnH~40-UuiYEu^Qcg_sG@g!Q+(e49dJ?_!ES$w)Ep` z=jz9A%zD12kpKOXuZ5iwR-+&6nXLwo?5ntW4e2w_wQjBbK5%vubIJ>-yHZw9e%rcb z3Ew960=Py7X)6go+F7kXPJ8-rP_%pJqD3Dby2+S_-Ex$D;D2MU%UiUk8<;mt-Fcn% z5phT9%Sv7Pg7w#8Y_rVmNS|UdZa2PMnMa#9*{D0I-n*$k^Zu;J{2Klc{Fj=)2{7hw z5=wjYt%)yj1QaMg4QoqZ;tXv2cLCqzbg`Zok|Aw~+`Z@eioR}h>y~=*l+N5t3*j#8 zX$aqE?1z^5QCE*wYF5&2JJk@@DZ8S>a`q%n|IDMNe@=WO^QW@Ec73qYU-tD+LT3a= zCb*Ozf=Ab~9w&BM!en2@-pTi`TUn)O84+uXLMxx0lfyo~FR2rkhgURokMTAHCDs+* z@O~=yan}`2#&zN@!41HD3ipbTp$*+47eAlRKBq~{hsa*hI>rh|f);%%^HPFq@AO6M zc*|$ss*up?UeIwvXA*RB)t!R{-!SkD3#=N%KFgOTVV?y@C3^fF& zh#^T1#<}zdlO5M9+v$TCHoau1I;NhvM|nRWmNPXaLu9@pBq0WWTt!TR>xwF4zG9T< zM5K9Vzl{G^jI1o`&m2X?K=#ukx82<9TX7w4v{>(3NE$f{@x>fP3uV5+&3r>O^AM|9 z2QlWvqQbRK%r%@Yiqj&PXOQ;r6nz8PUncW9yV1Gs^tV65-t3{=>UK`pzM+z_{L(?0 zb>7>yZ%`P^FCCm&xAS(w8OtvnLij|&8K*BDnpqd-eRD&s_?67M?)a|~znWP$2!EdV z9hpmyNDdJp{&nLpSFy_2j_yaT;T=v6_lcq!v}=v6_lcq!v}=v6_lcq!v}=v6_l z7@jJjR|UOdc&db674(YXsS7+KC^?NCEX z#ZHn};9397PZ@A_LdrG;xu>G{(vOt=Vf*t_2hh)OAD@saaoSL)V%$|aiLhiPwAo3z z1Eg_4(?5+$xa@D)aCs}ua6NhuuE<)>pRD)}okAaTa1EXMK16#JzWnoe zCGW!ZxqJS)tK19M^_{zL-BD-cz8v5u^QD|rIv?X~_vO5S(T#uRo}=EOZM~Enjxq;s zAWf*7~mtp>Yy=J&jKF~2AH zl!jC6^WDQdg|s)HGo~8KJ7qgr|Can*;SAcho4lMPuZ-;?#jZ+uze+xm+4m!DYAiC8 zu^RU}FkcjHt#!ugZx%9VhE5`fq7eEuwfO@E>|ig;3G)02dnK8lGVV~4_=)V39mQIa zi@n0kzeRqH-br~F`^8n4oBNR6vX5J#-hM;eY1)%BtP@BY_U=Tot`eCp{chU8V#?xB z{;UC7emZ(OYujw)eAes;`c^PcD0^wezQ)W}(o=wo z{y?e!-kAT3){6{SkA4%bt=D`1$JFa8Ttly43jKwxM(24m zx3!W7k)@nPME{MwA+(LrLTe{=t<$)f+m1kE5`NO$EMwLq#2rUgoyL_Jan<;bAltW* zMGAgt59D9=lTdD@PeQK&II)HG(5hFR1(MG%;IV`_%A!c-NhZMo1}b$ z&3!1z{EM9towwyko6(;(qCaJ@D9;Ur{x=3qK}E8Qq1mHIFQNrtr~a&lWzmY;)n`{MRgd zsql$qFBkr4*(-&2WIbJ0!MB31o^Qf(&i3##U7s2db1qKjJ^2I<${g){eTC`OQ~J zlSSM|gv;A0>z8dUY+zioi#amtOtZZCQ4zT4ThSHP6!bmFZU%FboMNLwve+h+I|`P zpm##2n6m;OaF_A}v|I+v?w=utBv=6cuOYq)cmD5sq3t0M8VIbubxb*45Ww=4`DS42xp~vO9>(u7&?1zIl9$`L5yY)A!^f3%4&{XKXCH0vex#cPV_?ayw?ptq%S5 zmz&7zugIlU4J|#>cUAo64suDQPSOu58bv+e97N;`>>Cwc4j}t8%+m-zkykNa(}}mQ z_%K;Bw~8erJBzf%T7m4MP> z&iS~;kuL6hR!{y%!nX};+j^<<0!#nHu=Vc(D|Ecv?IEy&|5#V%%-}s5`#gg5E!fEw zbP1&|ku0{3Gecdlp+_8!(rWxi9O0#JAI8?Ir7-h5^98HAP zVso+2W7mf&k+P4zmVLM~PY1oCzq5aMuA@uA+stv?r*v_DG(UOFdHM&n7XxQMcGX(m z0qaznz2pJh*wXBNkH%Lxf@aUZT8sBN;tC#f#JOu6UETAY@ovsnkZxuC4n>)r%{x64 zouQ=@`mKupWZbpeejI=8HsO!;;+v60Bz>*OkKm=z5e=NHbR+w7L{2AN!mC>PSH^h@ zk<&SY-|iX0(~+RArNa9Z>ae(3KelE)jJ@khImF@ z{kZJ)vI(=E#a6vhcssTve)$$7{T-zLH|hU8^)F?M`uwHr-`J}m9nw!U(!XS+&m+BS z<}+n0rfw=L>9eWKb?1h%v<~S<8tLl_6G*!N+%;3*D7&@K8-`8HNNz05NL*0nTeht% z8-B8vzh2mj^q=NsSLatNqpstw%}?{LtIiYuFw=jB_@|iu+wjLO-&&Zx1i2663?O~u zT1VH?O2*i=K}mCto^Boc=+A$Stl-ZkOhB zAbIlVaS%Lyl9yFI(4hZP;W^%#*&3zv-5|EZ5nG!2RqNO#^hu=L9h6rZ?xpVd`l4G7 z^Z4)1u-nb>6mMfadqbx)E)3o5t?vaK_v`z5fTI)lWv!tf5zaW|mWdmoy#UA`%=ERr5_}d5b85w1P+8i})k=p|GD!3nPup z`Qbk#e)}yV>CXfbF8zyI@XP*RcXy>x-aqQd2XFCah498jx+}dp{jr=zY)Jo2sVfUy z@W{CZv%g0E{L_TAwqEmms&aEkkM|!BBV8}jr87sAfh>fs^jGDK@6hWx1JBrc2Je}K zZTHroh9al0+vGIMc*|)QHhhP~#^B{a7?zEk0yw&VA?%K8cv;XbRHF;A~-c^t}2p9A0Y09RWklU}H7T=??F;;PF zal9FlIw`wGLC3vHaD!I~;tS*p;B)dh_*6cnp$qTJDeUK&$lY)$+@0OzjLCQ?DZW}^ zj&>#Qiwxy$kS~#UHE$lQ;mw2Il*cOTjgdcEZx=nldj%ceE(*OoRA0q=3F+wOO5RQo zJnzxAA`jz@k=54QMVCq^Z;T{R21iIMdff5tBB3YezolO;`~GCF@Yl>~${QnVt+$KT zkj8$yNc?}hD>hST*+@W@5l|M9KU>a72%=o zbCa`cR)n(eK6O009pVm2jceT0Y1BM_6ilffHpLEd#h_T?*)ap^-oLY$sGgv`$`5(o1d{Qfqqs6a}w_{ zuBmi%sXq2rZ|&Rvv_4Yl@_T;{`@w!eck)ll?Dz4nqWmVRA*K1WMXRmy6CRBE{$I*Y z-jk}f-klO&4IJpO)TLpR&7*$rPIXKtWhZ;0k4U}ym9ndCE4xd-JN17mJ9!_kiu$;K zwn1c4js2JLi`ZPLKbP8fesR{GO)|b|-5tohW+d~PQOs+4nAePEUNeSy%`TkN3e;8w zDr@sOkCo4PESKWYawE6 zk1dHB=3N!F^dHNk?)+m_)D3D#b+9^Nn@7EE+joj5V?E~}J?b6X*gKEwZDjrA8c(XX z@fC4%yp40koh0{^Dg9+XR5W1^IXv!O8Hp+~0;L#Hi!8by2;IA3I{) z=kh&cw$jhpiu(g;TO6K(*XVz2aYPryfGakYcZ{I8C<sb>j90?05oC#0#2}s!2Coo~_^v(&fvmz3*XGJEQDp3+XazsR)QrzxOfuB&}_I#?u zZhO}eTyW}sx911q-Yr%ddpZ-{9}|CWk=ygJ65kg8u2*SXP8zqxuuHlDMr| zVW0Ryw`U*rvXuB|9f|I_%zw=#{zD~cTd5js#w)aEoNICM4Xtv_u412#v{%98|J1!n zW5yugyC*A+cjxk5&PTnw+mW>GZbu>?e8B6^;OT$4xZg38vfEGD&7$nqQg*W`yGqI~ zm$F+z*-fSFKBG;OHb87dI`>GYvUk?SI#hB_!0ck?VcFxwUSU^tR$NYc#-xh$W=E*I zH|12mFf3so`Z4xScfxfydeaB?Sd^acOmP2Zq-Tsyx%6WGF3}m}?yY*<%hg2pe|VGo z_Y0MT<>dhhy~al-jI8oTPDd8KurV_hMtN#gM_*Ypb;AdHxU#W51sQ}-BwY4?pzP7L zKdG~4;+K2fJEo~7d?N8{aK9ivjP#`oqsEj@iu*;{_o%m$UzrO^CrvtOLeY2MU7-on zo$&<|$U{DN#k!py_kx9vghv*TF?1!J`?1Ht_x);DcL=&*^EaLPDfmQ=!oy7dg$IB9 z$fTga8CUTBxK0Uom32<=I1>x*T@>Y+y9oN637j@LuBi4u>g@WRE8}Y&j=o>KvNArM zd%CAVZ=VwEzCv}o-@QM|v*-S(F?)*RidrwKv!CJ2UaR8h`}r%&;;-OrVY=!r=!MMe zJXOGp;HyA}20rcs7rC#3e_QUhj6}B2?W~TMa%g^~I=&Zojr*5@n|`K$-hPMOv+2yi zFZa8x#Gd@}{MT=>Ilsa7jKv0x!4?&woPG^7qVH{XqMFj`m;H z9X5q>|1sx8k8%z$6?re_llzgc8fnhk`-k-+>-n{;@Mxro2m2-EJK8bjlm!;z< zXVRlw+_mcBu2t%f-ix#7tcB$lu(m#fdjlqNPvvy(H2vTHXAEcDQ~8}UBjXTqUm@~z zBhPL>m*>BT`?WL0u0-BBU`(4&KQm;Lx1RR0d1Tef$7R2&jQid6O-EUM>*>>`56I{3 zKzA(r##MKQi~dn)iZfl_D-#^0)I)Kl{k(Ro|J7T+xcf|52BUg9MBlg%ue?%>C zbvO64O8fS+`DFM?=#Db1{ABFw(C~Rc3!F zwqT3g$rIB29QmkF**{187w{NLUS1$CofsSB^Il~>cgKVoc?rwtEqS@h$@YLw`ZUG> zm!flwzDNrC^s~9_r@ZwUs)N0&u4b9blzjWg3r*&*^&D)z^m%36Bz@pzMtR9OR!{W` z=EpnYmA2l$ZI?U8OMZ6VI_bRfnKGL5a@-y4;B)c?G^9!x>o~_)yE(=>&aq%u19wio z+|(_q{)^p#FE{n{)YqL9IHlp(5=C=#uPl0oGw*fGjUJjY}1lE&QI%b z22{pUKP@nAfL-U$jBu>~Z+-sOC7lobjCcAZJ6Z?S@0c66EtGPb$o`ys%EXoY&7wDz-PrS!qeSLIN5*I`!bfLx z=L2-70Y2)v15xmQ%{-jY7ya)A?lsVL_jjh(qK|?*ber5)659M8_xG#@_eA=Tbt#;! z0{3cgcLw)Ca7Tdq8@a~>z1&Cs1jje*d8p;B0l|$OIvg9RM5Xaf<9@I5g#lx>M=CeR z-Q!G;4i1R)(9b3=vi^j6=sEiDrBzO?misk@hcJVOu;%xxRzEI%*t$e{pOd|SR-PK5 z+mQON_(nA@V@)?DGR~mgS+DJ?^u36Dg}(Grzh*6%^XpOB70xl+!%f;Vu?xMYWe$*a z6sZT3s3&rkO72^byB01?L9h5Wz}IB@G4u=8@8mv3nIB@0H+Kp}yNl7qlP5x08;{qe z?Io|Y`O+__EUI;y_YDbcxo;@^O&L3fH-E$ZFGtkQ&0gl+!s+v#5gCRkzJ_VDex8?W zoav`Uj>+wLxsrQSve{p?o$*WDY-jrWYFNgs1*P)}Rpx-zPR5->>Ga=dso-uZDI4~!`Oek z(O~+W)LoIg%-8JT{O_;v#Qh%s_dLb;@nq@B6P??6N`gi2SQ(bPSkskgo`!D=4&i}ub8Fx9i$t*!+7Ch#OWIz5gJMf2XviaL}DgH<=dV993 zm$fPGNu=nSzi1tM4!IuVlYWTYW6wOOaZgNB?tp4FFo1A*SHYvRhCr9a82=e>E_JlI z($~R;{?|Ko{|H|)cPDj6>Wv$_nt@xv8eS>7*$KNReeQ5$-YLBKYwC`?Yrm7ciEU&3 zLyN@z?5FPRXU?EA>A6QP;~sRdlUM3bq+VgXBOaj_pl{98rw>#mO7<60H*W6K+nLV( z!i+2DO`n&kcA^f2mv)117yf#y4jJ}A)+?w(ZT$o06kOD!wBc#ht{iW<^be#hG{(Eb zRNBDyJ8XW|hAg4|V~$8+9!TDYFNIbUZwMQAywX3LI9NHGhF-bh2mT`6^D^eSShJC} z30Hu(Jem5J#ymra%Uhli=yF#>llz29m!d}n@EaQDE$3d{(g2ySVx{D;SK2A@1LpFZ{g|D6)O<)^VD0sNogzjKPWJi^hrbSd(VWDYl? zySKcrBce2b|8D$8rg_V!pf{|AkDbcgxU;vnoU<*Z0sQ}g|A4E!nQ zonv32yD3>I|Du~xUZRG$SIU1+{(C9qoB02${P*F%A29rfIQY-ve<1LV5cf2N|DpU3 zQ_5HH9%8-xkKn&RDc{R~(`cn!%55KZtp0k)WBo zNaP}QCIH!_6E1pQe1tdI0w9>pO0I0y^Plaxicz~ zFKVX)`@JJgJp=h~zjvgmXAu84_~8rYe}vyZ?_zUL3i_yVHmgpbNEtD3m&!1WJ!PT#mc|C{u}{8>47Y|Ez&*O`;Z#|}+b8GA4%V~$q>&dQn` zw(@XjZ}LT6UZT&`w)(&LDfIP8fam5Ithhu=MZg>-~W3JBQAF$5`Pr z?7G~=AmhgM@#{V7uY4ze{ax?m|KJD=%~E=V{^016ku`nBHs-oyZp-~p$&78R<+vZ> z+u~r}i!t#DXApFP^;53po>TqzbRg>);p_$C?ydjR8%Z**uVl=i98J?+MYr-Y zxliIyZ!P4I`AA6Lry+^@xTyyuR5taXu(5=F_H;N)&e z@?eDBc?085WlP;e^KQ1^{YEj%OTFG#qAFWuKb)NBp5+%#KXUca_D$xo~(!4 zc$&DoOz?{ABmHRExZ3#oV7RX*+s0S;^}e?BhlcyyR{CPn|A;KOhxOF0pP(XlQq|B)amTx zR5d#}$B~V$ZcabeMN8}9txr}%nx|2pBrNTi!$*4bGeb-MQ=qS&cduRH+%fvNp#Ws_=C^@se&Bt_OL|MPAX!s~8?Ckk@~PvB%SrS26M`Mqb6ps{(n= zMP9}nCF2Ey?&03ria7zI7rxqM%K3-L*H-ei30xt_Y$!4ldsFi8ipOPaX~7QqLW&k*(p~F{%%ClE zX({9(9bTDN%KmA=E@OR8;r`2x^O!Q%XU_XM^jDEpWvJKppx=Jj7lt{a74{PdoP+%k z;wqehH2%34$@ot(*Vi2>O1AOuRWy};%X7fCI@Loz+@jL1Iq0`BR@yD+09adRAMYm4 ze$y0V%ABC_@d#b#}CrDUxLP0K0?yDBTM(||0Qvk<+GnREkZ z1!DA7@Eap`1;5-yS}FWj4)C8@@R=5TQ%jZ!|0feZqXT?O2l&5R z@R!-}>Hrgdg9#ts0Y0Jw{5uwWIN`;V=}YLi(dWNxgs-KWQZKrr=HybAigqDa))0oF zXSwtZx!crf-EI1R(0Lzl9d-U}ZoAI=+d2Qb!p4{N+bXF>e@1y?s;slTH8Y;?FhxHose;`>XXDdH<|l^WOLW+j_kon*Q~A75_%P z#x{vQ54H4SIpHPX?^v&wc7UJX0ltp~f1d^4v0mTX0e*T1_+$(IZVSF+y`I_uenJQM z&KCSF7JSEgJ+=dUeh2tg>Ve2%gazNRUJvg8e_aRo(-wS|1>dn=_wN9o(Ee(# z?@+J1cYu%Y0Dr)O?@G9w!+1mNgRC8;{<2;NlIH(Qy><%C4sF?m9O_HQwyI-&vD&VS zq&50bj6w3~OH19d+uGzrWp>k&%6PjU8bUubg?^}uxk0Q|&Z@U&(PB z_13>$8gv;C?PWY9Z6$YNtZy>w0{^yjiGUJl!K%IjD^w=7gEp%h4wp zJFFng%UzxZ@7fJM^UiOVUSY*o5-)w91-K6M#U3xr=3nYpIQ(WxUVx^AVZP*4&Y3Y6y2)8|K9By4vU1p;ZMq6{Sv&a&-h3*!Qm+cuWZ#E> zd|TYkVC@@ZW8^oL@rsOZehRPWj{NN3(C^ui+c#UeF~_GQDqDtFG8hUUB7^I2ck$UW zxC*}qUc92C-oDaj^Rd=#oMTnlw^czrdE2ZEcWI58%6b{^r)Mf>(o!?BWqdl5e!4=K z%!j5Ct|0&1*^7%@*uH-J&t06!ct>e0KF%3O@Tk}4`Xazd9g1R4dLw1qxVH;?TXSb; zH?eQa=A#N8Qt$7im65iNdf7&|Z$`EUT6Q{QwK;W@OPea`7}NhOe2YuFgYcuw75psx zZI?EV@M6xIX2E0Ax*L2=_GuepqsG{m zds!tdbXw7?6X=q>HGL7imAxU~qep`89n!o;df6vA+SHTHE=zxn^7>bZR>d97w_D|s zyfx5AL5V&Y>AIkAoNd@5xYmg6C6BLyQ)s+|J3!=S>f+|qzs15QV{74yImV5K8>4T) zm2;BU_~DiMW6(buryXNot<;}8taLs7(#4YQ8ss4KXIk_hCLQYAkYr(YKj{aC#( z;d0;l65L2|t&sE0FZE`v)~GXWeD)N0=ytwT8VgsFl}7S2%fQ=3pJ});da2(dO^ zqu-4yy6xJKsl`rG(tlu$Z2)!JmStO>|K`$Cv3Zi`RA8{t=;^1tF5Wy^h($t;FXOpKr#e%m2S~ z+Q!5(&P;n~#p5OaR?RvvJI5C@#b4mR6 zem?Q@&G>&?@h@BPCnWw3;_ooyKeFO=EB>DnzliunX8e9D{z)tTLy4bE{A4qJj}`y8 z6~9m77ZbnOjQ@)jzs`!^E%A2|f0r5msujP=iho1mml40rjDNw3Uu?y{BJtCSpKiuK zW5v(6;-8cFRm86{WtK}L4=7cVwe-Z3>(S(_F=(w!HVxsyx{uVPpw;Hh!fb0gb8e>1^YdH2JBb1^ivJl zA#>TU=&)qhjdW%|%tMSkwAukl;}6?l!3N?Q zWwC%VxQ}lx-z+|OSBX2myNf36rTj~I3?)BDW0Wly$xG9TTwem^5ijL{U&x(I$kNnNOm(K!UiJhOoeQLGN_~snu4?EJhf;58Pi>%gL?VI!;T0eI9 zPHyZ%zGCzx*v+S`y!{V!E|C7>$1^lut2ccmL{wBCq&PTmn7!ToCvSF_65DqCM|y%RZo4?erC zlD4RYcOj079)H=oRp{#A`40C{D}Q!BqmuSlNxN3dvD*2mq_g{g$y$)_#!IFD+qGJ% zm3|!Qi>bR6*zgC$PkpYy-dBqs+gyQdULk($cm;O+58}t>R$y}%iy!-3fqkwJKen_2 zTYA6vvAY%6-E#3`BP+0xzZXCDv;uoNSNzz%3T)p!;>S)_U?*paADdQzO`9%$>|X`; z?{4v9t17TnlE-4~S_O7(lK8Pn75f$69pcAMRa~q1ZWTYat0GD98F|E>RqRoGzmafk zoXl4jiJy6;3g(qYi+>3IA&PGlei>`DIAq_jN6+Km?q|fHv(gV5Nto!0^p(c4FIDRM zaQ;Pa2&*^x#Cv;da&MRo8)v}A=+}`(^v0}TiFzli&Sw!Ow(e?N(Hr7TJl%U!pW^hx zgGD#v_3p%pj$UrK9z6wD*5{xrR0a|5-xt?VFpBbRt&%gAEjsL0H+C2Wn zHX8rivbB5p7r7b#iPvfO@Gs?(%$^cCJ0P+U+nsnR9%fvpRk4O4{l$mip@p@XpYeax z!k&&Thtz{T$Ye??QYH^>o35gEhI2q?e;Zy5g<|o-ooG{pmjp z)$xXo4BiNKJuN=$LsSty}p&;PX`XYo;}^U^6^&O zQru+rOZMJX^Eh|r%+AC$`*oRG8DqX9tg}xe&DT90^%Z)y)Ei^IoddNjXo}oq z-18swv-scGuMe@Oyz?ge*Npcjuvt2-9sy}AIN?jdCnokBWz6O8GKJX6{{JbSk8liu2pUMBt27Zq| zO7PcuedB5s-$=oa-<56QKWgHyvS6iNdPy^2sEPmLyWsC;@_&!b|GOst*#`bZy}#h! z2>%-uUthtGziF_6e{F7YoWcJzKmOB5(>TQ9e>eD@CjaAX{&$=FryBS@dZOTe7XF`A zd~t#w{|XyFV?vS7&3^nxkY=2P|KbktpNKK!FZ~xo{yR+mLk#?hdVt`64*s9R-UxpD zX*T|187Fqtv;Fu#KL|X;8}fex{JRBzoStU$|Axu`S<*>4e}(IP5&mCPd|%+t!@p^e zC7+`v{!~BykCJAdh5zDf;NNKSA7S(Vn#uns2L43-Z-W07_7Pf1F-o^Iv1~-`BwJ z(R&O21Mq)s`O^m}air;z4R3;)GOz`xPt|0bLNM@;_X4E%|DwBY{` z{y$WFk%Aw8iH*NF+2CLLH#Yw+(q!BGKM4LMCjSF${vR~?R}K6g{o+9Ie+>U0D?T6o zJp5umrO)y`|7JZf>GT1IjDtjvz9H^G(utf}{|KEaLdT;g*>wJB(m5sg{|WwoD!yZa zAAi#TWh?!QwbCz+rGFu`wqcuOyd-HPFaN@wV(_T=jCQWS@DQutPq@(EkGqw(TE;+M z;HBLa_(Oz?tl!5S(++?<01h%@rw(U)61Y$mPrah}7y+MkE1L*XG#FCbj{hJV7% zhEBG#pOyk2wEMnC2}|jRzn|iJ2*1$02;W!w(QE6IEmhihzWEZk{$Rlmulk)7M55X^I8K-qP%Xo}4BS+~^wK^~RulwrlJhsW| z9*4GqeY1ABU1srC-iJX?%1r3K!zXjOhTaWwY7hFswc`uWj-f9b{NikR23k0;1dqrw z9anUs4d+dP+Vg&JZ8#qdGH`M)!Y}wdLwSgP$AL%qjK;0;$2l)po8|}ChI6M&lRL}o zeA_(!%fc534&hP3UG9&sE<}@iMQpe>eD8;9D*5I;)3!KUmhW0P&s=Bn*o-^JALm-P z=JkVX!?`U?+Xc=-zc`!Emo1$C1efsn5w58>R{5+C*T(z7wc&ielUBvJIm|E4=JQV$ z&OH{NyKwXT`7G|Nb@791!#OoVONGzwesMORD=eHZS$sZ^+s_|o??|nWA6y&G;ZX+8 zN28=Clo7XuO-WuYh+#kj@>!FqZFpt&@tPR(OH!oVtVLij=7iaT&yM=QZxJ0H^ zxGsNQ`^RVjm*VxFE(Wi!`Qfzr%(HOLw)mWZ+rrnT^8>nSyZqqV_`E6Bz2%op${@WjCY@GI~A6y&GvGGRU*Z9TRd?r~qM}SAs|&kzge)!-36GjX^0nK^==+E_`aR3R-_7hdWLP|0TyD1KEu<5EzQOgLpj|(q_?q$O;oqBY_X|@E z{=2pDAEzHD&7&6ni%Y=oH2IIR`Cnr4|4##dqW+QKKL!7%6yJw}AAgCB|EP)I?Z^Ln zq{#-q@b<32(w??fK<9+V=)avuzgv*g#R@YYyChAs*EgnF@x3AV@w;sN#g`lWw*X_x zKTdy{H2)x8@LpU1-ra(iyT(al$bW&!|8u01{>U?e|1|ucR(u-;KmJW4ZTXw{|7pRB z{2wPxIrxSDM+8>*zYjVaP5$@V{NHEtuSuE~_-|2s)q)>?o{hh_o5BBXKmHe!Cf4SE zE_j!i{7b(^tvc;oa1q!a#sFZj=a|D58REBNs@<=On3_@#d%u)_az(#*BW*Eb9N zQv|<9f865X;w+QzDF*&H{SLu@0sXna{GZ^*zX^D;W6HD~-$24d_N*1pleO%oDPiTU zFNTyq$=bn2_8BaFva(3}hq8A-=u6mBgxw)w6Fp`aw)$yv2e&3Z5>cdF)10A3Opp`Y;hX zC!!2}9!^?ApC_8Ug&S#dd}ExdFIez9Sw{rk%vTQs|4<7?%K1M7fF}=QZZiA&Kk{$T zxD^_^p#cxQY#O(kG`^EG!O#d+ec#~E!(TF*eI&BBLz(RtoX9Ir%Ik!?+$=YDdF#ZG za$_FdDmNKd$hv^Y$1b-s61F4C4719uS;F3E50i5HoG>Xjq18IZuiTbMx~_Ulfg)w0 zhmp=GyD=v3?~q>Px)V1Wxfta(TJ+SEr}SB5ouOs4S#FJz9y(iaz3v>}t!~x#68=2= zC57#I)(8_gBhO>3JogDHZ@ncdyJ<{R_VJBfG+SR^llh*gq^m+tbEJ3;&^j=nB-f=6~b2$^jZh{<$JfJ>#AQEXy$t@`7p|R zpqcNHq!)P&$Mvx`6KC0`i&^A-W2AO`gx5FMf@#Sz%Y2Xllc;CmdObP5@gCLJAAcVH z5@cuARkJQh|3>H?1y3)bfli1{we$o367a+Gt(I&r_A`0vZs7Om$%4NtvhAw+5(Pj0 zJnE5ISFbSgH5ORYw#Mlm(zK8V>Z?KHVjt04GY`^#5k6Y_n0as;G-&^YMgn9J zfqySNnt3p34D`#xk6G9o3x7*5@DDTdkYVScmzjqa(urKZ!SyDShh){)j6V;*)C)5Y zJ&inc^UK3=(nuc06j^!b35{-M9wM!DzMeuOPXE-Pk*NP&XmrEgbW?qYg$DkTo9#T9 zG(v3}l*3`BZHe{kXFiTD#hDamLu%qf5Ih zNa ztUu0CJ+x>)xHg=ByTa&izUxQN=5w5dv&!Q0eq7NTtN(s5_3fV8AwRe_oZfUzh0h~? zaW(phXrrXUtOe*Jm!Zd*3zwyD09)*R&YpN`4LxiqOHtQ`oTB-RfF$1{VUu7@Eyw> zmO_0?b}GJ8gelB9B{PS062CbY4e_Daqsk}cW-|~PKyYadVLVL zW8U@=CNxTzf8u<)F+cY^<_0^~#afG&$Xwd4-MIPwwC=eET5HT;`Nr4b)?D;TPrf!< z|1fENca4O_={FG9kyZ|2g1_9Fr(~a9n~ua=_PUEjXDDe!Mz^8Q z67=50u_n5C0{SoQ%|!ISCt+U7bRy;19lxz3@B7iS^Cj)B)VFS=lYAxNKF!xoZ=fN= zzgYQ-Ag-fMx(O2+{h%p2^r9b~1S4O}!&-DgNGtnq0&uJO+USJI{CI-?ltCv^{{gpS zzRnV6=j$8%X1?O|2mR>T`4amn`TCl4lCLIQ(T_HIX+w;Btsq>=OXS(Hy#7v@&@k#X zeFvF$|D7MLIIGO+En0_3FJ<-tuINV_t>Nap`%JJx&TOMLRMfML^I9K9|-dOTDle)#JZTG9sZ8!r0jXlus`Ni3M-eln{w{YGo zE?*nY^MOWrMjK%X`dzpk%X1Q8c3r;%zsOkn=F*3rz<iEf()V#G&JB{o5{;Fk8p}LptGI`s8)+?!wQ0ObbfCBC;!A{Ef_$$)1CK%%~_UET*CwE7{M^|0S z^#uH6AalWSF~RI}o3i!j{}$XsS#ylA#sgjHr*DkVjt}+v!Y!DVt^(uHKO(KrI)v-( zMSs1Q>f4V$5C10MCCz*Mw|Xo-xWnS)5|fV;=vD~$T7(CW$-`TM4?1t+&b8@xhW-@j zgX?Q#Ei(3Xmh^G@tCAl2FX4LoFy`r_`d+}Fhd<4rAE)d5x4LaPq-!UKwbTCse1;6% zCjBQ3d>;KzxFUm3I$JUbmNAhjgZC|%mS9r`4;e6t`dVCXKV;BP^{v65hkpg|B7>Fu zw>oVZz=N}$2kG;O417*APgQ~s8QhQSVGpddUA|V8`K8W=44$`SaIrN986@hzlk~{q z9$ar0GRRVWGx7U~WAD&f*E=J8P2tK`(H*G+3E1Yx!pyOlw7a6a(@8IF&z-oUH*LCm zUiNJz=z&I9qJ9hRZKho_aM?DfkA-I(@gmPLxT2$NcvgpNSuVp?`qdqeo@&A6lUB<0 zdffMa2G>c84QYq7>3J-;L8KLWS-5}khjT?}34Zh>&myBh3+_tNN`BLEH~YgSMrv34 z!P)fAQ9ngaDWny83Am=NSYyS#QCe$oJD+h@`}c(f7e#u>UpVf1zBaj>XHFA(D_G}t zeci=4|H0bk!Bk7`&-OFR)`#1AL6x~s>T(8kxwBE1r49`sPr}p13#LAtHDK8PgDW~5 zcbTQb=g^^z=n%SU5e9+m4`ylY5!xv9Oe~PKY(mshie5WOSoc^|?M_+f~dWWLJ zLsj2){CW8E4Ek~UHvU_iZ5i~U9=Fq%@-bx4Y|`Is;PdFu;TpCq!Q$Z*JajWUJu+B?>m7j%MyS38`1A0m8T8}y`}lADr!9lt=s~Bp zGI-UN!9PvgiK=n<)pND@1 z@FIiX^56QQEdzL1($2#}HV+?~JQNB(=#0c2Z__^@dLS~0)9pS>%K?*quB1l>gK@p1 zk-=!ycOCvb{4R_Bwfwj4vt^Kp9<-W44%%V50C z@4F_ymmBF5^viI)W61v))fbQdf^AEWjqsJkw%byv=Lua!E(!WK)N8R3T}dytB?|X3 zzP7qvnMPT$UUjvZ#{l9sU&5CD%fb^#yy&NbyWAg7U03ZS=QnM8E<8xNNV_Za&w*d` z>Ra4$f4KFr+B<$W%C>DgEx0d8FZ530mioh;kJI-0(Mzyw>B|<}KS?k7{Rmg|rY*lp zf_B6Y&ZhU21^0nPZ!hj>f4HTI+Vg&JcK#l=;C7H+^7lG!jz8R^muYJ*k-rrd+)Ea{ z=W+Y_!;MYW*80)2^Ecmu+emuJ-=A@N_`|JF(Vq2#v-3B@f_uoKw-#4)qD?MiFW2t% zgR}E@hXuEs^pd~DxFP;H~sE^L}`lvxebn3!xvp(h!Z$j0^F!hG{Bs_$C-ZT0ap>LYhv*>yq6Ai+2{l%R)N z`Rhk|$={W@Tlw1LQkOw}q~7$EutYtDI2ZNt7x}f)y=|mR(7TYX^$pwpT!{@1Z?nO_ zvH5?)8iE#x@SHWYqGSz;OP@fJxLZLeG0EHt1H>=fj_e zfA4J527k|g>npa*;bBQT5B+T(UNL$2TKI-e6Yhh+8*TB6VuMBI9zB-yqHn$zC4HQJ zOwuEdf8ly3VuL5DzEAMy;m?ewFq{9Js_r2m0|kNH$wqbTlqBryzqVs&5 z_0A%XvsB+i{CW7(4En5-@!$GKyPW%KC#SU2f85IN#Xp+##~AoLdLgdS-zc+qsD_7O zX5JTCFfG*r!+fd%lc?w5dgoHkb5-9E{CW6S0555-GAxxR#^0l zujWo|8PBs1fqw6JTLu*-{U`&UM-Rsp88ptcWUxTWpEH*5HprI20#gQo227%^;Ccxg zPuicZf(QI1z>5rg{I|jz`i2bPVPiWF18f<icJs9=M~pUh;7(dHo;!dH9=V2>m$y6aHJ#4}*SjKlGrTeylA6^hWR;GVpoy_i>Fn z01iVJ(eD$(Oc_Y~WYl+bUGn#~0Ym>E*Gt)qr_8qF&%>W*@Ziz6@!v{4G-Ut}-P(DO zK98XX)KB4Iv*1Gp&*6G(`q+>YPE!WbXEF3(iYbGqBt3AoxL$PWR`l%&{CW8ImYOnn zl>b(2p(%s@=s{?k9(-)&=i)?@{#pZ{N3X^e8I*&=kO8**#O_c-52XKM$lz8}21^VW z`v16I>c@EM%>w*+_|ps?Jo>Tn?bf{VrEizF}oNgE6uWvq>I&Faxx4 zhEo4%>IbMFJ}*{%Fk1G(6vD#?6F`j^)8jGj$0MDB^j?&welP~-_7SQNdZ8cOtN(AR z4@SuKk6`k=#Z*x8!B|y?Vbl*~AE;}V#$mD#5?L9n|3QtL-~-HG{g6&U+Q4-17JK8| zK1B5abcnjA!*i++hR8Z}A$jmY4CtBZiVu>gAILr!tNI{G_CX}!;e)oI#?9~n=H4Kr zQ;;q|dGdiD&g}`R4|<~?(D%sY`QTC22MKch?o6Jy=mJVUh*NdwPyN7!^@F}jna2LI z4|rAv*Z;22VJm#F71w{HQ;@c1k`CVD4V>His6K!WyYA`GQ}sa~S%)jYk`Jt)hftpN z!yeQRP(Ne^s6OZ+``|p`;e#JQjV16w39kQ0ry!ll>c{mT=k~6u5Bi`V+^gS5mDg34 z_c@b?>p$pS_#jx(LF!2TK=#2c%zbS9c9eClWMu-ycR-E1aBbX$>p#*dNK-px(0c(h zg9Q`zM(29T-xR7qet+zDE!o8jnb>d0&~umz45A4BJiOO8uQy#cyhSSWlu~9t@-m%) zb(XOuIDZ+kjtcwj4i@B$%}xzQ3_@YVcG!GM#dB}Od!LgD3}d| z@dn0Q#w=IlycZ(dv5NBjk?$|(FIHq6R`PQwKN$JJa{gSLv-c#{h70=@%zVOx0TU); zrmL98BZOTFCYvx(z(mQI3>9OD6p9thRKj!srh|+bqhhKf1wp}#Crn3RI?5Qcia8!7 z6eyT9!gK|utBe^$=kU|sZxB}qUt|)6&Cx_*oAoRx^xsPqI%X1u9??Wmm-TB<$lD9b zY=5^M>kC&sWgqq?`FQ4*foE>i2Vzv6G9O~NpA=jt!u0{JkBkeWbMzf+738B|>?Mje z&m;QsU})ZN>LB-Z zN8pU-FwW2MlD!N1IMN1`Bl&l6ZjVyOSzoMi?~OBB7cn_evYbX>sXzV-%H|r#ke~~A?73iI z^Or%4?TB5Pw4d{@a&rJPjUmr zJtP-C-U(`K#5~uC@6aHff;8Dnn5{Usd#nDB6E5c6)AKvXBmdv^CV9c)CMM5YTn|eA z4}47a)!jDGVT$al_fUrJMH}15zFNu31dGc+jkiqF6SsKj1*B7u&U}nJybW{Fwkh3& zyU@Qxj&C|hd-&dL4<a47e<2P2X@31cKIptgSikexXG*ynz*D(vpbNC+lKTl?$#ERa*QyQgE9HI$ zo@}TAovtm{x4%%QDW~?qgG#x7D)oH@I$m3D+W_H+rkvVFJC$-Lfu}b51oRPYxmV(a zmo+|9>!a%~wb6S@eMdkOwB`CH2-`IEsrIc@%Do0W*(ZW_zgO;5#1ouwNK;O&Z=q6d zhf?1*&?s%WYrK%FDW}>uLn*fjc(QLjsIRu%ZG3zB0rowilv|+33X($uHg=cklQ0G90Q2l~Oiat9-3x(P{g zxsBR6(x8;<4lLEz3G|S*TzG52LsL$*?+$!U{)_^a>ODedQ&5 ztEo@5??a_r9q?q|CD3ebxnI16iU-tJu9W+iQs1|r6Sd`XeT8>5^{Mu~s+9W@c(U&V z=qPQuB0u4brkv`}ElRl$l=|KUP12U@6d;sp%Bl9PQOX?xp6ojS+Dlt5B~V!Q0Q;6I z<#sFem4bHAmcw%m{1b@g-TT^amQrpr@MPbMpn=+QJwt@8nsTyzfntVIZZ+_vXFjN# zww#sTwf7d=flup<`Jmn{-hrM29>t8XconQYC?1RARZh9x(^K_lvQl3*%25Bz1igx* z`P$Eg31{SMBeSphTJ?B)_AY3=CQtTt2)n-0Sa}?B$cE9N-)Uv`q?qtP5%KHHhC#?P z{>~S;?^bjhjC|6~1bSR6=SVdBubS%nX5FkY&EG4$fjaj5PDgLiNs-+Z@<_*6(ATxH zuf+o3=rCKqNWXlmG#saMW-gw2`2pp~m*0R+Q0k|4K94#Vf=4=>U=;WN zpmsO5md5uNu$E4`x0a^sTO;J$b(7c9l~e|6{&zs>xyd2*S&=i{N6BmH%g}+^qsf`d zgouY&J;CAuP~%0gLE(tn<-a^Iwf{(4#e zw-x;x_42(y9+QXtKTx9``rEPphja?kDK7H9&jOsYx~k>6URPvLUC$uzWjE!1(~cOg z3(mOj2^32eeVXiaFBl?DC;5mq8jX0QDI_21z0i;HCaUr?D7F>f|50s$oX3&Z*h-O) zcLOgRBl-BgA28%23GWF~+m9moh>aSLn5khTAL*1<96h7EZ@!E~Ir1@Ks1FYY{TxU0 zJoM}U)`u^}5-&vTi@cGV@!34CF5>&fh%cNVuQga5azDksn^MQaC`&qa1})dtalAL` z2oyhObp(rH$ZI(+gTPbAr9aX%PAC?9k?Xy=wON@?c!!y6C4b6& zNN(eB_=v_4)qM_marh1!-Q(Nwe(8l>avy4=v~lAtYGcF#0cXVi$aw6VRUw^%bbx0^OTSD{P`8+WAJ~;c%;~}Kwu}6Ky zU%Z0)*w}-e`;o`SUU$WwM%a@FdoWhMS8z>@@|bu9SR-NpM-sx=_=$-k^S^BFx0k2BVLXue$(a|DL1gG`P+T#NLt<)2$JEkM#cOOw>&>n!cCoeh7H- zM>xrbzM~a=o4x^lmlNwRBNTlbzma_sKxI&uH>fcT`iAKw52RC&EKbAv3URT0M&u?UW`w)18m@#4VRR%)DF9W zS)|BsJO=rC*@mxyVeN2CwqZMy?=8Me^6{?dc)UaUHTSt*zhbcacW zj4@0)5Qf^@9rPs}&DVP?FD%lOYpydy&%PZ+@0O_gsCIp=$ovhukX;fe^_zP#^#Xmv zUM<%w(*y61DDR}Gd8(e3ip*NbB0VpHW@%+!bQI{j{A#&onVp>3w_Ip1ehbR9^v!4B z(fUMtPudBw&Z->Mrc$N;Pk|?!J^~%4tsnoQpthO{KJ}wFL2)m0-+NuJgQwoh96(xK z7tuNm|CevBvsw3zF6`Y{(tSGcSo7WYF5%z7i^lUF{BO7U9U&WH?^NAY-_22UF9x1` z_Y!D`R`()T=ZHjO_zf->0a}Ix*k*NA`I2F7?ju6)kN_S-eM5= zRM$*U`u~`!bD%Od=vqttG8_3c*JXlUWo4AHVQt0sH`H>?J`2Zpo6v_|)W}r*&{dK7 zxFT~j=yzI~*WHEfnsUuDlRbnO4PVtUSdo{k$V&kIL@Tear!Zbqu328W7yA|v{)O0V zx2mI)BD1R^GZvKkP4n0(_ZH|Kw%^&^&6mA{+FQ1Q}z5ok?E+&b{L)5WL0q;FJFsg0|Fs z9(bzm&mc|peyORe+2_WnW}kN^Ina@8qc!yqoRja4x5AqGsBV#cfn!|F5=VJm{kPmS zy055NK<^OZ+b9_S?ZC?dk8~T2vi5!IT3e4j9m3(;ADB1kn_Z09>NET0~tOExwFz7!_CpjUVg7k5X{+HUz`e(XB|B2B59fxNB z(|tut|GO#vZ$vr!CXN1iivAsB{kyCF-z4jQ8F=#RPoSewKU-gK0Co!L54%+VZ;<`} zBg2P?{{%ITfc_(N(pjWakS-{`=l?gsV||G3%eq4U@%QwvhRl}we-9q%aSY|`t2FvY zDf&mr`qTA|`G1wH|J%Tm{||!(qJE}-9${tw|Ag^FdNk(A{y)I*A)*D;I2!to)=94- zoq}}A)_eM|1dqxUkp6Za_m%o){~vsS{`;8z-r^>dvoBHf&u|bf9+TI$cf(};PeUKp z|Ch-6uK}LyUIBVW)qer7yY$TehZX&$1+xENVE7Pm5vXwt^dF;><|Cbgv|-yl{T~C5 z^?%a;OX$D#p8m8xZ0Y|cOn-b&0p;woHU6KX=pQWWzXSR(|Ie279|t`7|54DF;Xmg8 zEMSvJf7qq^KTGz%nc+jkp`ga`(0{y68jN%b(pHWB?ZG4cGfDpv=->aI{v#itzq~&5 z7JHzaeTqhZy`p~)S^qhz|EI|M#{y6Oe+aa(5B6oD1G8Zw@E3GKO&_C_uGri-QMREi zD-$9HfEp*_{tVyx@1dQEe|X~DK3>&>`hiZMdGG?(kRw$ej+b?CA$h3B0kq=p zO8sL{|1Q)I-Fm7%93$6ndx&%n5pRPUr@)6(bkYr^Q;;^4sCy0ayxUnROE&%j%w5>H zb%$IBzQ2R}D%r+hRj1LiP8V4n*#DzCroqN(I_U?h1L@-^j~J_i+`GMn4c>Bm)sM)- zwFv(%cD^r`=u7Y!c4QwRU%{RMPa9M8iHiBEkFZ3+oFEMTeVBp&_)@&qdnzWQuP|4^ zd_WldXCedtnIKHLiV2AmrYo2ugz*B#OUCR|F~7tKh__>Fh1Up!|B`0lzoaB*r;4fU zCyY@ruM#E*m>?PRvWj`4zhG7{e~6Nh*dC)2-5|aE;44Oig|XR5TRh6 zAq>8;k%8xfr1KONgP4f`1v8B>`2R-+{{KOkaWV$~1muJ^3T6Uf`T^5V#*9)iTR6c< z!8}SB{0k%l{{kU7DJmv_7i>x_(87UbwDck_a;xn87lpw~RT+W$1)D z1=Ei(slcSln9eE&-w3WzFx?4b2F5I7+Nqdt^};_DOf+HAfJu`v!73)*K{%yg!U*#i zFptR?FBNm#LHLJ)@gvMQV8+Q97ZuajQFvd$v?k01U?#{IPQ~nX6b>sG2f|DSX0nXA z)lL~-0Zzhx1#=g?%%_2QTE_gUV)kQiRKYY5W(F`bWXw-0Ccs%JRxq`MnFY)&8S{gR z+2$+=3Z|McbAg#FW6r1;2ZK9wm=h}I6@!qYV7?&CVqg}_m=9D;I~QTT zg87&*F97p`j5(rW*18DU3g%tHEC*(}jCoDP7+i&^3g!@DRsyq9#=NRxwz&%96^uxj zHNdQqF@IMv(QZPTg4scs^}wu`F-0n7znd^r!E7PSi@>}nV+vJFcq_rEU^WouZ@~Ob z#w=Gc2U`h!6wFG(6a!N%V-~5HZmoqb3g&sjYzJn$jCn@I9BeH-q+p&U%uZl-%9v>? zCd6F`Q7~DAc@>yfWy}N>Q|c~wE11cI5rGk9%%du%TN}Yu!Hgx$0bmZunBgj>yp6ys zn305e1DH2tOp=Ou!b7;N#N-Yk%$vZxDP#Jn7za;5jqe>un74s>TgG%(G4FZ`mlQd@ z2=hKL@5`8I6*JsR_?H_0OPCLV`B27$shDow!dD6=iZGu5^NEb{Q!(j2!bt@aM3@u6 zoRBfCRSdq@{Go#JB+M7Ud?8~TR7{AUP@!N9gsB3iO2*ucQpWpUKjEN)!FThDGS32Y zR>m}_n34X%9tCrgFy8_5os6kfF%AL3b_H{VFh2tGql~FmG5Z6AmlVuJ!qfm$BV)c% zF$)8QwF>5Y!dwF8l8pI6#XJ!tF`1#lV+v+B zVSWSVHyN`<#aKgyM-W$s&#-SMF~k3~MwF7o%FX&q*u^WMl$0quu;q63jn)Qo(hX~-V}`WW(wjzd1t3CJfp z8To@jpGH2>8OSF(3;9InB7YF*eB={djC`UmAfM=R4 zkWaK2`9!xPpXg5HCxX6;e4--qi5@^c(KnE90(}$tMBhd}(f5&0^h4wuK|eu0(G$og z`UUcdRv{n%g`zpHfarI~C;B7uiPj(=|6QUvsDS8Y z^f%-a#rQow02E`ifGEac3Q>$T15u2ptFDc+Rp>8#6WrJoPzCI?_}p#fu%()Ti!m#wzuV29EX+%%ET6Xs&N9ed`wQt3=3l z8Igy3%KN_Q)gL^nL;Wt`CQUuEUEZRHQcoPPWLHnncW^Y<#z z0iJ4CDAKZBc;8Y}pK8~2v<>>_`*Y%fRfwCy!cQ;*tS z-zoLzfhD_sgS-{mdh%nKUE3AAu7S3+s}4NXE-TV%dkxgor`q+tQs2+OkzF;Q)9cehrv_r zdIM>-y&N_5sdg18^}P-p*|i_Ehqk_tMi;)9{!UB+OoZ-fJb$(XBF5pqqyM%n!40B>!Z|_0X*3@9+djWy}G(G+wO+T zwmpKpmbRsUr`k3MX{uL3o%EbY(A25gN7r2%XGy@4eFH&1#zA#X!uy;N+q$vuFTyVe zX+8~P^Ble}uaxTr>|K=W2Ktt^+?MXp6|vDw*I+Rcc~U3tn)Z8qZNUr2-W`1(G6-q8 z9lXV-WIb7Zs-D--#-wK;$w7TSpj&Y?>-odOtR2c(eSxAA@)}W}_S=O#c+KsAJv(ZL zR83uKJJ9?|b^Qie)D9A8uC}gI^bO!(@nu$5h*$@D2X$%Z@JryS?Qj8U)()tX+M%yeVxjnyrtP;6<$MEO-D}_wfWa>Bx_IM@mhPvqP1;v?OTm|8l0arS3utakj?HL<*t}xRH-@(LxkZFD+ zU+pa(EERBAUj-Q}433;`{#D|1Zol%JDmz&& z`{V=4KK`G|E|<%iA5eDae=56EE}QUxvi<&3*$?HiG{4>FE7;rdKb5VK%SJq)Y{-8q zt835v?Dc@Mt-(XMO`!#TUJXA_M;pbUji#fGT+rV1uXscE)y0{9Tm_96gJ);K#Z!Me zHo_!z96ChG;ko)@7}It7!k~#r&u}aZ>iCGvFI4!QMh%flkk<;&ihj|r4a(wM#isIZ zu^k_kac(-fe5bJr4*xb$ah;x2czA}d@>tnPN?8xNEb(T9auw*`6`7G-MR<3UlVttyu<>Mdp4FAt$ zX(8pS=l@EUo~8UI`(b%4(}Dd5ayEDzc$0KVWm8-e%eXDo<_H|&0l8)q=wuwGfMqy0 znp5yjHNUztK0E|Z0*=fv8@TqNhPIrS;f}o*lexTC^S>p7;l~WjLphj(0+7y0L(ey*Q%jnu$FfYNK4V8*Ag|22=l(mZ|59220U6U#^}% zKUhk^d7gT{Y_ODxbG!#9mtVo=WNC$ZK6DtXYo&Vr$gqmzs5=~Wt9`>@>Q4Qt!nM2c zNJqo^G`IC-a}#*Ctkq>}(_CJ$%r){u7n6;y5yrT>;M_EfN!J+OHP+A;eUaMhV{TLe z{=KbpNnTvtp(LnqXyA(sUd-Pts{8`$h!=A2Hw>*nm5FTK}I!v zO6BSL8tpqHj{HhKjrEOY3>*|Z7_a3(BC~VR=hA)T3i$LLASMI z$91Sla(e{(0i)zGl#DSnHPCp+0mmear6BmF8}41GZyK_L1&*(bFV$}uz`b5Mp!Cp| z0f%C+UL4p?@WH&5@#2KrIWLCZeiq|3$14o!u!T8QVMpfG@B)5dU(eg^=}oX>n}4iO zqUYjD4z)UBgk1G|DeZZA-z^*E>6%^Y#FZBu8h!-V*~(_^Sl|DuTZbEB+$?tG=RpEd$Q4V;y%>Z-^xs{%sJKfO+)@zGRx%5C^&a2;)x3;E?H^zlwj{E@srcc2<9>)9B;opksDN)8nVJ_U9 zNtEtTxhO2c*qx_zNAP*StR0xsjT1Rx2Yf~Uia*sB^O?=wD_u`(Qy;Mj=Q`Bs17Edl zBbASKxW@Ht#lCM+#>u|Ej&gK;QQK%4?0QwnyMjDwLo3$EVZ3+UjnO(wbsPK(o9EK4 zSVI%fg!ECY9UTIUcXGW(-?^!G5qpgqR#EDjw3y~0t3OvkzOaL;dE`g6K5BQ&JFMML zqdr>qQ(v=sa}{bE`D@zfC@?MC2-iKua@om)N1a%ZXU@pV}F6 zUvoiU8~ShhT03B!l)iEi<0Kj5!~x@ED#i)vp^xXh$o?Ap2Ctv6&hCN!U-BC_r@)Ex zypFz{cgLRI6ZuN}rD=T@ulOt<{-Ar5to+R1^?{^XP#&_!rMbur?N5E1)S2=hgsESftqHFF2Pg9LtGHg{xT?$9L|Up$Yt_yE47xnn7t zJ3_=OIA7|?9UcJvsczMOWKTc5BSqgBwZ357)ELECBC%dftacU-VLe4_+lwem_guK= zKNVg`_qIObaI9GqFMj!DLPXgZr zIu$xZEAMmd1@9JQ)ZFFdcsu;R;48={o#@_nhoZ-az+vqyQd@KD21|BaKQ12Qgqm}_ zD2FoNp$v_!JLuPh+XY;I2d-SU zGZ_Dw^bzZ!3)yx)r*e^M-#y!c#evMWV6g~gqZGSnFUl9|ZIU@1b&#)KL_XEE4%7!@ z1^w8vgZ8j~pglTYu`~D-H%!+q@{Lx;bD|AgdI0*)oq?Q}wT-a?b7O1>#*I2&-op5~h4Dju7vG`w+~&;9$#vq= z{V;C6pKx#7MBqC7yK%z@DdUE&vjJIj&82Zui+XS`nzanqT^c**F;=K=Hw42kP4-^Y zkHes62Ke)!cOcUKs9%e{3~Z*}r!eeEF77#JPFftrEn3)tTSPWwcH|b340nwT1JZQu z${f$d;ktV&8ulA-ZyY_!t%9(&C4=F^LGWcVe3}H`Cc@`*KgA39sIkwTY43o~BVPv` z_MetGU@wNJd$d#PJ=%X-XW||+jlQvgYYjdZj`xxc}G{U;5#PX5wRv3^~Zw6Cn; zpTC}Lh+Fqd!}WJwsw#>pZAicLNkjRJpBrw&h5)Qr!<6g0L0&^N>$n_xC+Tz3E~1U- zTAE?EZ?1dirL@AB(zyoo6&K8{GX}3r!!`2MIAwj71>eHIbI|Y7v(Q&&PPc4bH`W+; z6g(4n>l}^JQDuF$F2cCzsIop=_Yl&a+#KpF>tc|e&ds6q*}6HV+w1F1w>Qj5yuGnL zap6(idsp&f?^N?+?r^%)#mk!P=|ybZ`-mIiD|KBqb4*EEr)Z6yV@j%Gqe~l7FCA-e zo&Iw}7WAgRy;h^|TIjnD`W}G3=M;SpK;Lt?hxq_{9e}>)6nzgoRJrJ!vYtEuea|WS z9)P~DLEnSW_jTxd4*FI@UrsmXPPlFu)Aumb*H@gX=u6ktnIl)Fwd$zRcjT%n*N&v` zKO0hK{&(mL9V2A_{_pDhCeznX?5pVO2Yvmqo(w~OsX*JuVSN;Y{t%A!Q3BRS9f3>0 z`X~YGqXevv;;=p%r>u`kbl!EuUcuv#<2vv+FwQvTLLC z+5c{RW>4}yUA+PSO+}k{;-J2S`>HDFTgBtRUVW82jy5QwDW-kA{nO~h;rk*?CJcX{05xuZ?kelW@%4jL%p5EU2^+XHy2MR~? ztqNT`n=DW8A%$~z&qCK_oG>-iV9wP|+iqN8d?Y`=+I*PzE#y!Kzw8+5`ec9}bxzK& zF^40M1Lcm?PjH3IaePprYuOM>tLKbTD)ynV&sbS&E$Td~o3b-W}yqEhG3gh0pLFg@*Ia0@lj)JnjK5ewiCzkMGQ0UBXM( z7Vwf4`M>7y*xz$6JU^Q#UsyelC|77L zbBjD)qPzphqke1w?aS7Jayohe?H&5>GjuQc&Y(B!3k2;=GP~d(%sQj=Y^iw5WSJ0lwP@Ko?b{7nu>xyG=q~bjFV|VCcl!oE_S9~kj9qYjpyfJP# z^45OsAxy;nipZ}m-p{Wu)?=Q#tZyy;g5ye|NqQ5$xdEH*tipdr^uC2w-IVRsYm&BK zflWL9KG+uV?W7l4fB19NPJY_<2*c!1U18GpC>(WBSjUBNwpy$M&%5ceao=jYe2BBK zKK`vc`;X4If%x7N>^cm)tR0eQKdR>r=*h-RfxB=9`Db-Og{9yb&i-7*p$^U^Bh;E_ zlK9t?wpTlI*-YLK9>T}K?sdZ+zzBUsH1$6V_GqzpRfF%|U+B?V_yBvkE8u7BHHS>L3;RsAMDP=FJ_U3V@mF!#xK7!2(3+6zCRk9Pb+bu2 zt1~eDZowb7c0Mdvv2R%Il$uR)T_87-$(Mk zqqHNVmYO6}u1Qk+54GtB=o5PMO*`swPqbEj1mEX}ObI=T^Ssqb+n0w-4jrCvk{&5A zN#?aCX~aeoKHJWV9(Zoh4&@#yNZO9)nWyIM?2Ww&Z*iqlV$)B-&e?W6H@(<{v;CUj zX!)g&g9ZC$HtQaprDg-4ecp|;v9`4O3YWWbmMHj>bjie?^mJFW9qOI#j(-e+XF~cs z#zC#p7R~+OnIOTJ`V;ClnED7V5k^ZaZ@_0}xWgFJKL55}e6x(|b$md1SL{cVE~Epk zXJeN25h7fS+0+-QZ$`mC5r!$DHPFc_>m-k-g<(Q{fTGXqS!}-y`vCq<0@it9gF3; zxPtD=^>kT~K+m*&eev#1JpNg=dxWKS3NJaYFiIC7|7#qt;Mk1AIt4LYpnJM1`wJfg z3jTu8<`X!|5;$pOcHmf}CFpshEoc$`H}ag(7Cgr&g=8CTj?;}&=#8P-23tzD>+T`h zZgECiz@ovH)}JS5v$n74E3`SClsz4On{gn~GBXs<*qsJvPwr>5We>!^8`5x3_L$K& zdmO&cGQntjX0lOw>S?2Gj_wi5)1l_x-&6dF$HB;HmUH96Es=>*Tu6{t;fbT2~)Voi_b#nq9|gnAs96FO)42LrB*a4CDEZJBvy z=ppl5r+`^kM;po-+rDOAA6jnSg!5;YR}OgY(zCN3yEJFk>Co=x^3a~)qC3uHEHU)&tbr5oLfCfSulUpSfEOR~Y3{5yLLW9SJKJE+ zVC6l_^h1Yd&k5OOzA((WY!vDn7W#&HUTSh#PhejJR_8b@J3VBlxyIS4tfvFXm^bTC zXb*Gl*+z49VQzdlWL2-qjrSRwZjtnQF?0P$%U#?fPyO2{%N@*>Pb0pvmplVRPTH82Hs1%fxW2POMEQ1u_tmw7jW|Sc*JUqF-hM7dwUEg*-vt|`e7zpb$_(a zUChBrCR=6yjL_;^NGF(VrIc>uY)zw0(o&S6?-gA+6liI}clzJuagM%I-QQ%(CER%4 z)`V~8ZzDeXPDcN!q1B^#Tiw1O3)U?suc0p(`#;6{(MaC&N~-ft2}q| zY7#Gn_kWu8C$9fA)?dEl`IA?jd8sR}J9$-)y81m6T5ZP;mCjIS?NLx{)frCK?h3J7 z#hA16dhqg!t-Rr+bu?__9B`grY~^r1jFU=qjyPXYT!1pxA)Ith=Y;c>#ZB1HIZnJJ z{I_x+)>5zWwqFnNwkvP(wqM@iZFi3H(v4%h^xHppsrDpqyZj|@yLpDUS-;_J_@A2X zw{yJh%dq#&whO$?684_??$5k*3wGVWb$q9jA$wOBm+Z1MPI~t-&Q=iiu6h4BPI_ko zXPXuFj(Oi?PI~)k&h|*y+h#GbwWVURdv>R=3bR|-5%YoeZ7gps$J!+HE%UqYdsy~V zd0O7Ydml!c+nU0K^N@dA?<{uEaYfbfejb)MwXR&Ih0w^pY}rk>hb6g zu)&J{vllj0kH$Idu%a*Qg&o!CN2Rc(7Ja~)WRfhAtt}VPhp2zNqI1u##kcmXdg8aS zRNIZVQrK2ohq^`^CCg3^i*-+m#j?}WV#Qi?uWF}H1hey3?0+vp+{Z zk9Wmv)qOw*gZ2karG5@-hAz9tTBgI!OxTzS`)0wm+I{0J(_vdS+4b6Z%Z&D|Ei=2g zXIpvyTo;j@}cVXUUZFd|tDgO^2 z3mm;qe-qc9vvWHO$92YvPb0C9h5N3eoz4P_zuNVXaAJ+Ig016pF+xDM+_bUi1963s z0`2+nn4jrdT7zq8O`bmc3axAXIa}Q!-g507&Vqdz={)AqDAc9OCwWDYoG=f2**S^W z%f_BYA!MvgG)e20Craiv-i2S{dRJ*O*?x8PF8m^95ca&dIoI&KI85i7<@vC@W^xEJ zF7D>F5j@J<5+vY$=5<{A*5SZfr@qp~xVRE~8})bW<+N6c#2P&!U~=eJh&5ooSldnr zR(utX?-XLszB&nC*Es95FZ-i!c%vV8;4S#h z7u^ROjNsY}-}vc@#vId2kLw-rKSw8Ne4ewkHxc6m?Fzqnp7r7c^5Grqo2qT&gEg`h z>s5c>K~lh@gQdWih9EXBNF0Z~zVR3nDaVo|nzM%L2T7?(gQa14LnIfy%gIi>p%Ckq z`fz>foecf3JDB0pXG->RT4OF#)|kiPGv5ILW-IQKYO_pI50t^Z&cU)torS%RU|iw( zu=uD++J`u?{ZE>tGOT@lU~>`r-f^^t1qYs|9nQir97k*1-{+#uf9a#MT<*$StQ{2F zk9QEZ!1fzkY3+^sDy*~r1{_--VND%f?<#o1htZG^xjk{8chC?QD?AE%c&EGIiuPb_ zw=`CG1#(#aKe0cDb(V_Rt1Bp`X9mpITPV)UPh0{l>4WFbr?%mqv_Nl+!#yL`WSs=+ zd(r&1Vm3}9V}*3^i!OH(KE-pH(MZqxu9J`e%yh4!;uBrQ$4o?ejc!wM4rn3hXT6_@ z83Wuczs|zrpaBV;g`>!8t>0YC#?q}$LJ7|I^Dh^F?q!U_bHRh-aykp+L6=OCvDx0m z#j2mkXH0iPN8u6ZeY^(a_Oh`8_p}F77jU+W*8%7UO}r!^!ps@t7D( z1#`3FZlM2J9l1URU1il?vtBC2UIdrYU8qNWM%=d`rZUceXTkVquXH|&!}AJGOvHZD zdic)Dp1pb3ad>uVz&-}XM=wMEKdJ}q;Vvbu9{p>Dl zDW(j2g>*g0tsD@M`@sOt!C(&mEn`;14;izJI1I|1J`q04K)g#Xj#31T zXzPo-vq1|(BQp`34*rMWKh90r9{0l|*|)%t`+m|3als2iqZZ;?iSPFI%{AH>FpFJ8LiY9I-dJ4{e$51)ch)RK^D=`2onk+W+m} z?b+Iv;MrUfFWB>(POEbh`nHsexh@~;Q0fP{TyF8hXv+cDP1ysNCTB;($1$)w7It=o zJ)IAZ$S(Wmq!%1;Z@-?Kw!O?{R%n+2=Ikr)_*;H?hr>0DL6GY-yr* zcFkr^y7`)Cc6}l4V>)_dU(4a7>wDa@ug>G7`VmfLb-2H|w#zm9Qi)6U#m$E7-|WuW zH5;6=@8qo*aJeJya~3#cpC5*JOXuA99ndwx_^@y}$_e|CoaJ#oH=bg%e)OT(0Vm8M z@-xc&=ZsI-oDmFPsy;2a;oSQ*T!+nedpWf?jX7$6>Ic*xs9#Y3pngLA#S6#p`ovZH z;TIqS<7T4)*Hae9hCM3G4~WM@ydCza5_m&aZ1;koF!0)8k1EA;b&w0UA&s6pa@|+6 zcrz-`&2eMnm!+SR(>KSwhi3}+;xI9vZ7iQK3^6u7V(A38ids;_bZ3R>k2cM_&! zew>fBfy)O+k9>-C2AdabB?a%EvTw#S|6b@vr!?!ljod-}iv{ts^qxQw+Jv5Ik-VXp zKf_RNFVcvI%E~~$^}%!VH*pRd=6vL8Tul2vYTV<^F~`_5k>ANb(nFZ!D$Cbm7y0OX znr6R^=Po>+zv%EBMi1ZN*{IUz^B?1i3SuyC^1~|%U=N42dJpjE`P{oBIkt9(%?Ax1 z_z5m;O;Xry&hm{j))~5oX$>HSMetI9S8f_6#%pmGdeM3w-yVz+?qYqvWLE@!gS9oy z4YFwcvjKL~zUMkdeZ)dgTA$y-UMH>5a}nF-gZgjae)Xu6%Wd-ed9Zv{45a%cq|bhs zgtVVH0qJzi^Ms-F^XMad?y|J6yva_cca|?kx@Xy|~ zL!O_Q3fy!BSIKaG&A46Q?^nlH_l|-~cm8z3li1^0%~ga+IB`MJNpa%_}1gC4})0@`_Yx$?MUcx zPRCW8(>YYw7xz3uzNhkZZ%~3gieqR4k~vS;Nyu4_d&`|Y1@!0RIj(#~PO+0ziGIOd z#P=jIw%4P6@_$BgZhU5>E8;kF<8!trv7Lx9ZbD4NCwqH(>wMUJxk#Qb4K%NE zhAcz3f*{=EY>a;1t>W%)cK??1D6R8wU%s*6?ZFc8cs+_sr@8fTVysYlXONVRx=T+C zmeN3v-f@(Uo^X&HgIuAck*#Z-@u{cWh^ywgg0vG}IyML4`q8t%DK{>~Zhz~ztD(>kZ>->W zFE1J+M#RU_9B>%2a`FEFiZi9>L8^=j$S|Tj#fXF>{;~k~p@#aBzJ?aEILJ8)4ElQd zQ7^8(6fv1c-MRV*#OC2ydll~WHU@m(NyweCpl=yuQp_2bwVW)zAhL8Tx4`(sg9j_T0NBUW>JNHqq(R81C~Agh_#NwLfA>T${H8kOWZ$>)%LI_QOu_vU9G3DeC*y ziQFQpi@sO66>_N$*SvEtX7C@lUa)-Zo6!4rn{dxXF-_&bQv4ztZxlD^js3?wivROL ze4n>ii@Bfr(=Eip(f8VR0*`ar!#c%1g?D~%ew*q7{w(UK#$G(dXKz)~kY}n6 zfmO$~y$vV*G==I35b0e&x~{*8x+wnn7UJ*-zY6#Y+`mw~Z9dWzKdAaM5dI|0Sd<|< zmMeNZ59)yN2EWMhgCST8qkLQN15pPil4cn}ii{LR#vEWM25}~6BaUVn7fD8t*a|#~ z8Jxgqp!fu%0pd7D{lv!^^${Oq)L%?z)LTqrG!StxDQLSPXv4v1%Ry+pPSRoDBQYaF@KD;)gnFpm zvHsdr zT-$~hSR49@Bal!1Kw@ndC$fNgP)wZOuLH1x=T#G1m7T)mG z6(RPv%1?ul+bU4}7uuHE+X*tLt$0wHUr?Uf`(Ol%^$ZesKnAr{BclOgJ)`*kKcjx) z6-IroJCpt1Qed`0r9-a90R@q^g-rSV7o zTzx;6#-spVf7l}P{n%J-hjUuDN29&Cb(QheZn^Pyu^-RQ1)S?~?#4+qd3fJXPya6n z678s$XlLk3de9s-iP~!beG4K$oB-W3^oYemUt0^^_96c~^rNv$wvPdi*2F77Uun+s zXL%#R>jRzW-Cng%Deq<<L*TP)K`3pQ6F&% zqu%0BMk$u;+l4S#QPYxxIax|@ogIo%d+AMcoyHb(dX-774N<(zw?m} zxzzr++3IPAUDP%sz@uj#rR3jegU|}+wEu7&F&eB7E$b-w+_<+7Wl?0A6j@&(w!McU zYXErEk7(`K4QU$twk0O??GN_`Q9nkUa$QGy#-z)BP@FQozfJnJLs_g*4sxYko8Pyl zeh9rPNOo5?w!Ou_1LqP_Y9`)`NE?uL?K?d*9M6*%cwkM1Yd6<+ulZS|Y3-8|kr9eL z`gA56>v-%tvUj<$ziy-J+Fsmuzz?UmcMZY`xqLa5!TUSuY#utFWaFaoChB>;jjd~c zN|MkAgy^62Qkbu?f!0-|%WLSbVLn{_d90z3&%RIK%jSP?af{-sddR17^pSFY1?M#9 zT0v*S7WmPEKD#l(+7o@&S1bUZu0h{1>L-2+`V~FX(!~i_(@4wF*K>D<;=U)hsv3DE z-|3~?swAnTN-uqmbKHBS(e=oM-a*BFc{ubT8^U3e3wOM#*3;S$p~OX5u@74<$3LBl zh;|XCvpVsQCB?1`Nq$#`Y<-jEQ9IusFO7Fyuvcu#rg&-UtHyEq^8c;)#XOhOH0OqO z<0Q6s!eP&Z!`=zrWj_wP)w%7*FYKGS$Mik_30~ZVfkJc##M9s%f{u767x$(0*z38I zjAJUj+edrIN_!|{b|bexy7^n&sj!8EpSA^YK3Qbf6#Tmdax*X|Mmm(TIg#3r+DXgX z1YUI(hwlv1+*!bI>%pTwEAy5y-fF^O4^GRYwFbM^QCY-P)4WUT)9O76PT?&EkKQ+6 zWj(eq+;fD(UY?fs4DnXo&&wiS-u=9%iHH3^ZP`i0TXH{-tveRn&l^LyP51NIx?{ur zyhnhed0$&c3YEovqn4Kh9?ctC-ax`l#nT(kKp>f4+5a#8QvZI-Vbv*f-28!XeQuVySL)L@Rz6&+~2{ zwB=)UzJ|g{iz65E&$j{AYjlKxcSGJVJ`{9e!X-UT&#bgNCJM z`MRYJpWc08M;zUuOM;HCr+Y7&pJH6_{RkX*&$0geU3x#=SNsC{)0|_-MLy=d@Ccd* zmZvSt=i2C2aBcE%h|lGvCAZeO@6PAjw0GiWyFRlgZAfbev&TxVjc5KOcdz`(?%w%R z+W(7~$2=T9gA&TT9$CJh%YVi_#>CYxBfHQSw!p5p zU{|zfZdGPjWrM-M)(WQ#=jHc!F}|^82ocu<*M$A<{e*@622%+3UMlgbUv9j?1^of# z8qv>aj>`jfCvZHsEKPDpzr@<$1@Qi^@UAl+=11_hfJgm=Fwf!q=6Fup1RjlD>PvK8 zr14IDs58=MAJUhX;+XHwNx8UBnLikG2fxPbzmBPv04ra`8{7`M+Xbzp9t2 z(bu1Z{H2h+!w2szVovCU^J?_{F`)A`{Px7hSQrKRjD{ahe2k4@pffc5AmU@J3<901 z;rkFDVB_&Va#`V4fqhTlYf#Q5s~{$LINCiAb4 z7y-Uf!@oxSX^8m)zn_MG8GP2JJJ6<%iDlKRJjyu7Bv4OKBd9y53A9a_y(3R+$y0_8 zLRsv4wR7t-&IRkrdAaeTOYbT79A|rTxam}DMuR;6LzVR)Z zyU=gwK8Mcf{*KOd9`bcy1I{D$a^Jd+K2U>y$x*)EQ{HnvM?5@3VLbZB17v&$9-ZGz z!ZRH7OSG6KXE8x+6&^Mq| zM-BQA@jt@(eie)UL%fUN?NNCcZzR(J&uVw7yooZe9eA`pBj4f+6!Fy^G47`1#@{+A zk4Nq$@H&7uJ~#e0o>$fAodo*N6xC1ujdYg%L-n`x&t~Ga@DHv3nC|e)ZH;f1HuLZK zWr4!GHH7@-&-_JxnWJJcCRD%7RCyQ=s!!x~JN$&Pp!#I8iqrVyNriV4K2h6?{4$34 ztCGvk!ym^{uN6KaKSWcyg)crsAKQy~cI#`Ll!AK;>po73^p|}@_YEz5L-!{weKU-B zQ)J)jQGYV(8B97*TPu4aQ^D)r%#(dZKIzoVll}AxcpVg8Q{gDfEsWzk7_+Kx!kh7s zq4byFW*%gKNBzZ58rt}TPFQ-(j#xEyT zo-($Q;g?T{_h0l&fB1#|8Ap0$QaVZY(bt$K**(S+^7G0qP4>;-=XNAn{^EN$ztso-#Ux%=;suFE!25>Id(gR$SdR1S{qbK?@YppF zZTlK{U*Me9^*b&)j-Y<8Yi%&=ah@55I37?J*Zedm(CIptLPt>F)Y$NZ{=_1 z7jV99k4M+{_JntR54@lAZRWotpVA&HZRUR|=}*940^By>wjpnKev^5rl3oJL67Ya! zGRC3Kai}vNWu8R6<52IE{JZAysEd=!%_bRftD4O3f;SFwA)_nFIJ=T?vb>7u1%wMq4uM8>4^5DHlz0Qz(LRKsqLuUs9mVdDDH#WjoOIXj$$*Y{iy9| zO??x6G7fD<{UHuMjDrvR!+%R*E83*%3+Ny3g$*>@M&!n$9pJ-()F!iYkY`6a4t|aX z)@65i*O<1HM}8b=HtY_cZNPaPu(80VvUK?DIOM5qH4OF4-0jhI9m&7b63(Vy;` z6Z31~TRZ&RWR^x?t(h;G16JKJA4Xq0yYi~}*2+3__m$_-H-9x3t^5W3@k(1a=U*qo2dhN?kv-~w@{z=y@ zYyWDyKk=OT*SP!h&Y7$Knlt~jbLL-d_h%hc%6wG@ViwexlsPI- z6MHeUxx7)5SR4;zBi4grFrK13B?hDM4`X z_zySpbrxUZ=NP5d;CBMQdMo53zRLyY(R8lIIbypUao(=EoZjL`_|u>AU`qq?Je559c3L&!aRMk0?NEn2SL9VYgO9%ZGtQlGj-Vou z;u|mET!(Xtli?8uLC<=3B2I#0EXY=huW*4ZJ*X?UlEul)fDIITG5vdE_MA3G+tgru zFA?Vye?c)96njB&GV}bCvqL~H+c&>R-(?8HVZcG(yJ(9e6vsZDYvBz3T6~t{wfK`r z&qO+P-TqmrW5Z_+M?A#Zwh`vk-QlxSL9>C$T30qJ3whZro+3htm54wb!ew*J3W{r} zLrlaK^DzBAxvA@R{!UgFcr9g2fsC_=jYvbRgvZJ&=0c5b|5X`9D=#C?;upkT)FI~L zirG!0M@L?^kL>s(8~@0@!;qVw|BHDW1O4^Qcu9&H1ZtAZ7(IK{&sN_#vkicVXitb3V>@ulmXS3F23F<9s*H!||QPjJ(_d zlkttwDZ!lW6`W7?&W*3#itp2yIQE{rE8>*@Z{vmjic$EVjZw(Jx7%uQz54HLGZy1D zCdEQ=LRUxs6(jUN9V3*1?-kZM|M#}xU*qn7c-(d5Y%zF;lf`oVHBbJ3A5WRlm9wqJ z_XHDO(y{+Pu5mWTO+-4zadq(;H%+>YChe_B`)ks{nsk^Z9i>Tk(4;$R(p@#_9-4F? zO}f7(ouEl4Y0^V9>0z4m2u*slCOt-z#{aUKZJ($~Ptl~OY0_Dm^lVLfo+iCOlU|}p z=V;P-n)E77x=@qeph<7iq&I8QTQ%ttO?sCmy+@PYr%Au2NgvXr-_oSt(WH-R(#JIE ze`wMtHR&%k=`))2H=6YKn)EqM`hq6?vnE}qNnh2Z8#U=$n)F>w+I}xRqlZx%inxI3 zi1+^m?_R9oO{^c#|2hjDO>sFG52=o8iVIyd>DHRGrzY*INe61uZ8hl#P5L2CI!2T3 ztVws*q1stN`Zu+-si` zmJ#jmeV;$x>v|vm*w^~*d(CUz>t6So2N`&nfv+&|RR+Gsz}Fl2CIgQ%@T~@(VBpCH zo?_s83_Q)iGYmY-zz-OBu7Mvhux;SS4g8dWpEmF^13zcrkbzek_+sB6|7PF~ z27c4PZyWeM18+6(4g-H=;7<))XW%akTyNm741CbQM-8kDeEJM-ZVSwh&EvjOk%?|w z;NGF;wqW4a25x8I6a!}%nD=&@!^tslR|Dr6xQBsz8TcXt7Z|wEzyl3D$iTx4e1(Cp zGVnD9=H0C3G~Z<4aR$ECz!MBS*}zi_e2;;r8F+?)XBqeb1J5<^BL=n&{J4RiGVs#| zUS{Cu3>-4>N&~-a;57!WG4S6EyurY48u)Dkzh~gB2Hs)dj|}{&f$I$Xg@Nl0{FQ+Z z8u+Mzm4Q#6!Od+U>BoMdng6dpBP%LyGVnM9-)i6q2A*u-DF(jBz|#yo!@#o){D6Vy z8u$?d+XjBzz)u1J@Y%ZwB6A;5QBYwt?R>@KyuwFz`nP z{?x#A2L8gp^#=aRzy}R{)WFKXr_bQ#w!qolANz%fPI&zpSy6G5fyWv6Rs&Bk@MHr| zG4MSGo@U?~2A*Z$2Mj#dz>gT%Ht^#He#*d48+e(4pEGdCz$*>>vVqqaxW>SLGw=oj zziHsN4g8*gw;Fhdfj=_vrv|Pw@D~QIH}F>mK4{>h237_>eFitTg$_UV3qSS?j17P6 z7u0^vz5m!R{Mav4{Maw3{at?S7k=y)e(V?YoNBip`-L-Y;fxOX<9H#`SN`8LUf^z# zAJ-S6I?|cBg&*gUqHVz_lmEZZBc1G_V&HoWJk7u}3_Q!g4;Xl^ zfgdrjZQ#cZ{FH&8Ht;e7KWE^Ofqz_IPzk)TJSMUiDbasMP9z2l+{(b`7&zI$=?3m- z;A{i`#K5@*{;7d4FmN9OUu@t@3>-A@Wd^k_+JM8(7^vTaIJwqH*na%`we`+z()*x!oa7_;OLrjg6GWs)Wn!G zG9oeFz^x73&cG=K&M@$~2F@{XR|Dr6xQBsz8TcXt7Z|wEzyl3D$iTx4e1(CpGVnD9 zzTUt$8F-w5Z#D1)15Y;a6a(L5;AsY)Vc=N?e!#$U4g83KZ391U;HM1yw1Jly_&EcI z47}37FB^D`folx>Hv?}l@S6sH+raM`c&mYT82BRte`?@51Ak%QdINuD;DZJ}YG8E+ z2Tl(x$U^oHWgoV_!<)U@7jl-g*4NgNyWTJ4U8v2x{}p{dI$Pdscrwr4z&T`jQ_+9S zBkwYg61Q@WtNYp9-<@yg#JTQI9Ch6vn>r#Set5SlIqE;dz>RXYoA7Es84{BB;A z^G`u_SpTOw2gO_zRR7WcDb82gOmiBNoHMgicB`?S&Kjl>FaDe3ZTZ&kW2>4L1$lQ)r=zxK^V@6kwm`rtt4SVE zHtHz(SmW(w2d?R}#A*%Sw6@eUlu?PdtVU>AwxzP+gO>06ek+eg*b;6$;qD{c;U2=x z%Ui-rxx5eTmd*FQ$t&-6N!iL9W4uw`_o!v95q`V_-rM*}(;~r-x3;Rswzm3=BYl#G zu+)L4i~Sq^gQE%H#gy7-w5e7pJTDy+%!MqQ5F7m(Y-ULT1oMaqphCEm8W zJd&KF+Afth>dyY|?M&WKw1`vMOXIKq${U;>-s1G~ZlRBN6?wbf#~XIMFM7=9b&f~g zC_JgdOmg-UN0ZlT7|Ppn^1eox|A3YD?k?UcucC~FHi^2&%?m4VBW0l7;a}docmE}A zml0nv@fH#PK;8g-Q@3Andn5kkjm@|4BfJH#Z4qAE%D?ccxY{u=lgW^3{httWX`PxC=`PrJ_V_6K)I!Vx~6Pke5kQ=C6(db0Bf=3l9UvAhrZ zeQi?m{UGl2TO<+w+sVJ?&99183GW2Y{pPamd5@2N$9c-$<~V;lw6Af;q5OSKhw^45 zEXjLd*gF&FJ-yBO`HDmD4EW&CJ5Qa^^nB>NiMUmVbFf(DdlKd)PMdh5fwyZ*J;@EU zqtU~8Lv6SxUehNvHLaSkl14QS-p;uo!S0Bi)XUj;?y4i6WZ|`Vt8791 zwEPpqJwGtcZKrvpC7$*fKVy&r+wn7E^qnDNba?PJav~~6uAt7{`cHO7V}C7eel208 zKi1t2Jm%RNpEpqDc~TmVT{w4fYMw`>@g8$D?elOS@C9`n_rRAr4YYBo%cz@KxJfbE zYYhL<_8K@Ic@xz4|Dc_&?`iKmd`2d;*1YG+2a%6aS&?=Ik9s(Qr95ARM}pJ~?OK;# z%Ra_}2ZTS8ajVN7Q`$r-rtnWY%nO>!EXjGGxh#_$KX$KpQ)wT`jtA`4-M8P7M+ebu zw`Kp6&3o%j#qz%Ucjd-mhENIKnmPIW%ke$t!|F(0EW?xt_DkX@paT^Iw;@@D&NKc7k47#9O4` z@ez6(XeD3YzaZQ19~tr{SI9dt`+NZh9bjd)XZDN3c~5UyQ`6{DWLOz#EsdRds0*Wm*Kefa&!5etDiZ|-Zy@3kPAnSE_`JmWm zlh&59j^zSi*WaJA$&mpmLr9v}1hwESD7 z%O%Mvr!1Euk0-c!9VPvZ)a7jG0V`R{<3IYQZp+q*Pf-a{=W4@Klg2Ww<(f85BU?hF8NE<@ufMJVw(Buujyyc->ExZ zqu=t7UoZLgk$?Km`(k*P*kkj?_9)~^^%3f9Ip$HHpSpA3`#nh=YWWj+uX@7EO^eF> z(#Iq@Ig$9Xw3+0bL%MfSzu`!K+e+s<_1$&ws@n%AJ8{rTeOAL}ctF~-PrK=$-NaMp zIxW!SB4Ng8Q{LhJwtmp~>w&z({g$pf@pBycvx7FK(|YS*wfS^Y^#!~ot^cRx+j+P_ zb{7Of0p5}fsvU$Y?fTdux7@SIx1{HT2>k$bm&cNw@zjCaZznnbjM#^*G_V(GvB+Z*zk#92vsNu7_!bnD!w z-*^^1%B=JKj1R`fxOM)lPJgnq95V%(UiHn)?a0hEQg-tx8>#<*ZoA2jwAp}_`fgTy zO2aJ?o{;v7zSR(>{?|5^ZkO_#t=*-)o8=2MOu(1x(=r@vWUEiJ{_98UA$hmcPo^EEUxlqyi0%IKFbr|5a=~`vD{rH@k*N% z9+`|E_kFZvXBTBJ^?sZ;kv%k&8QgQOKB*gtQs4eE^0)#tIG=3bVjc( zsy-h*u!Hki#P11TkoS$PWXZeG`{{7fopUk&+=7>+Zj)#uNgk_O#-oylp4z?WL@qz2 zz)vDu0@(kv*1f~7J8q;8U;j;$w6j%#qs5AG$jcs|+ov&3Se3US)gBOUsR8Io19CiS zZX7hv1$Kh9WK@GM-f49~*{ITimMWdgx0Nr$qb7Oc?T6c02M5vz={Iq?=P7HT=in@P zBbWLSnhv`z)y~CzE@4ZWqiNlD(39LW$G`Q|9j%aYt&wqUka2C1apxf8+9Biow(`YL zXMW})e7r&FyShq_(t=HdgfxzM3{xALQOD>)|Z> z?$4SQNqy-yGONEGpJUlserrnZ%jJtxku9PZd`vjkqB9@o-mp`?jQsMpmK``-ers9(Z}~uK6$Ei`xsB4VFhI|#>3d9SJ~n$Pa1r=Xfb7du)_mO z@tfpali$`#a;)TzZiWv32% zL%2EMd5X6_=ghgJZqDy7DL+1Qdw4}mD1129wR>9Z9>T8tlJb+K*u{s!zu>OE6&Ae5 zoDBEl!5^CEM8}bGlQHP>sh+K3KZk!m^|yfdkJA?V^Y5l9S@Zu-_`eT3mzDx8&AANP zz0!8z=Wv>(f^7r5AwTDmlhP*V%(_J4-orgC;%7hnsOuod)z3@4@eQ*?x2U^J4M3KZ z={_dIX-_#vs@Z6-UDw3*~g(5A?ktKgrX!$()bPb1;0E9h@755Z?z zA3B>ZGz%GdSnEBObAt{i)e)IctH&f~=2b*rw3v^nqaN3^vYK^C`R_zqi;m|86UGNV zt4}lUcMXO_Mv49;wA?zc()st~+gVS?sF2LXwB~-^TZgDG$g}WCN^48=_t|uwKDs{6 zt}i5Gr^9}vugyGDK~|2ZAZwtvfOzNUNjr?BWixL(??S#bZSvCoVkoCr$}5g?3s8RX zw7-PV={a8e1np1ynB%lR8LKza{)BJKjrZb5{kdroW!0wxaqse`Ppa};qo0DWr}JG4 zy&qo+A2Pkqz5|i*CVeaIU(0D9rmL4{IA26)-{9`#%$eKked+ngk_FN>qzpGvhWD4o zFFD~$$)8*L;gTj_d_$1FN8%3SM`XfYZ6-PYz!cdkb|PC(-#47R4I__3$?FjEJQ&^> z6rv0?ZyfGosie|Zmxzuob2;MgaF?LkM>*BLq^)K;>q&E% zb9Y%*K&Nl&%nnP0&tHR{WhFJtrVrhW3^L||=UeAoU@wcz1GKvuVW{&yOKUzzwZoxxP4BBlMntJyAslJlDipC{l^NGOF3v6lH}A;7Q0E`9?~rR zmFP8JB3IJsd$O4as=HG6N%M=s7tk(~oPwsNMf59EPBM3NrreSoDFa)|lyLqe=|h)^ z&JRwD)Yk&+We)J31IYAEBGX$_zHKPywv_idlsoeLzV`5xAD)VXhhpKS7H8oI}YhR?`cvhR*SNKJ|l?eke%f+9b8Ke@h3Vkm%J_!dk35uPr)a)`Q| zwlBJF$%X$-zIQ;rFVXTn75QG`OVx6`L;OxH>pu>-vVH{V7ky?nVP8R=?Pl&vbjs*G zr58iHMDu7m^(8Vk%@G;f8M?TW`N=)VVUe-tLMtiFSQ2jVhEBng$B?gOB5S`TW7D0- zs2A~j9&!Ip_*L5LS>&zgZez8)O?O_6_(|2}p6(=L{!FJU#c8K$%>QXK$!V=kX;Y)9 z+pDPKpHtUYQs*OSQ&-TYTzPBi!luj^^itEJ_2@;n!i$=YUQ-nrheq`S(Fu_&tE?9D z&vVI39IM@J<*y7I0(Pr~qoUct2$ zD<2+}_-yE|;hx0d(DD<`4HgXFVO@v0Vm|xu@{u{xKel=F`H{fsl>L$Tn z!u7_e8wAIKV`J1W1SfzKV$^8CZNP0})Ky@)L#}Mz?834~`4-GBtn^wdE4>~*@5;)| zvcl2@ZPqT$k5Q%PSp}tot%8hnM(P8nBhNAH!kVht)*7V)npPF?q%FSN(* zzD$+w_N%tUJr#bI{zCBMw1qX^e7o|zfok1(9u)^|-E^x!?AwcsXwiP9pAWExRMjR> zVLdQ}wVWiU52J=aYn6F}HLQos z&&lp(UySZvj;^NDj;=Nx-WIMt#tH}eOzsTF<@SWe_fmh(QMN+4j;q!(l z+VvXA<8R5M$s66z!W-qJE6N*!qjk60ap!K3L8>%TPuEaKwS*tf|7OvRVi-4|xA1MT zx)z`2-saN*3%Na|xBWKuc>~+n;clML(W||o-98ydBsmH2jhMd^^LzL#%AY2``G0`l zqI_mW`0SUY&*ZZ!Bk32Kv{7A0k@2+Xq2I@6T`514m!6CNHZOfidH#RFOC89M%S(gd zmqGB&W%T34^xy9Maa7L>SVbX=^=#oEtLqTG-uv`do@C8W$c||LB7OFq-^jSazqO%z z;^g`+ttS7FE*R)oIr0?x;8FC!W9Wm2(Fad3){yxv#aKghT^&b6ZuJwMWsUgg5KCF` z6pe=yL%(;9)<2Hzl9>^Y-X*2mKv_jla(fHvRYp@7TqW^VpFmD_r|$9A z=u@A>gk(%nL;30cA9wCtQ>IfLp}hyqU8l%!{v4rghIV95EPj(6CqmlOD)@$MYasM>Dq z#X!#fL8mpS9ubqYR%o-MGgr(3LG_USmpPuZ>xK1QE}z`ceZ=Ibekgg8{M4g&NM8Sp zF5hq=`SP@FkT`ES)iin^=dwyY2@O8**FyxiX;{Jd)%j-7_TlXkDz0Ge<;59kOD1~S zyt@>4n;Ts}6Fmv~f2jON+>XHQ#2G0|#?a4{tESOkwMB>ivp2M^sb8pRgr%PKdAEv8 zsUr>ldBGQw_YjWxTA`D-a_Z4lrLKP=d4&I-?c%!K{SfUY$x?gBOKrr>NZd%>e_rg> zvP#0+{fFYvhY|b1*vtN#u!MiUrG{o$)qVLazFPJYh3Rtw_OP0)cxy^I?uPN@c*+-# z6*(73v}K=ykGBGTceAC(hvP1_k}TN+@Q+^h-haA%WRFN1G`*y^Jqtg(B6jCtC*^%C z*{WX8C*#{T$i*zn8(r%fCF8xN$gRs8n-=BamUg%$KpTsXu?mjFSt@ykMOmdftqDu? zttZg8;v`PSB*(~?*u{z+`qvZaUzXTWmqHgTy4VxwVvYNu!#hG3EPB}!=w(O1dM}XB z(9iyee)b+x1qW1V^n#ntoe3uwBDB~ zIMdO3UnjvCj@J7+$EcIE1#x#ZWl?rsjCu;WQsDckebE>z4>_y5B?+N7ETMQOSiBL~`nB^_nx+WmgfCNFP*b$S?i=T2p{k2#O}EqWw~Ku##jVfe{VLDz25fD!7Tv5s@+$IX zDru9vOc9g3-XSJDG+9h|Y?7GdDNGrBsQVpj>yaxx=sm1;&av!V%iA!EvDYokNtH4` zlZ#nmc`I~zZx~TCjJZGdBCL`zm#pawUuM~ipAMGcZbXd88mnO(bGjqq1$!Di;LF)Z zHNtAuu!4E+E7}Fsi=I{$S)Nu6x%^jHmb#k$N9tsVqzyAV4rtnB***4qk~d4+?!g-% zxYO;qvWGn`G8gnsgS%JH^plMrbfEcp_{qjku)O#g&)ni?(2*vyH)2)JojvUxXYtn_ ze=WuwZu_O5OSnE*z<4(yVe81XV~{b28T+Aw74-2063R}oR_IMi_}2I$U{$vq7s|X8 z<3^?BLZpAb(qUW?YxTKBGHqi`00JwPqo_zHoLQ8+}M!Pd`)MSggt# zeUw?iIZi%i^Y5ik6F>!x*88F+%(mgH#`M z{p2fRl+|@0GCA2P!EZJ^%y_NfZpP*b)U#O!T0Z{0x1AGk_tre^mgRT$#K}II?^}zN zb%}_6y+!u8$#}RGWg)uCPbmxb=ZreqO6%5Jjx(M*OuED8F{bjOpP+wOt*p_}{H{m$ zzM3@GGUjSx%(a^IhU_e0#u>7@3ne5P0uGdnsv1_AFeSJz|i%x9;+%N4-M3*SZUQXU@M!9s`Wu za@(F6zg<9lullUM689kdWhGk&XE9e2E#uTkd+14?r2l+~w8~uh4Uzvj{LB29$8L{(CF3@k|9m%=wPeO@ zI=zh9TElNr7n0Ur;!fr{-T&|IWs|jP>RitapZdt{Q?J9%F~&2SsgL!Md7k0Wr0j(@ z^i-4P+vwPnakL&2Nco#%&%UHV=HDsHZQ71Gpy;@9LY&NR{l8y(6Fv$sZj?Au88^z_ z#J>}l%YC?luZrL;*5fYyTW+6nY5#4lKy>#b=POnPq(7ZlGZg!J_%PRw zujz{{5uW&jG8bMtvd^<{3HiNmMY3JuL!QD1B1gV}F7C#`V`D$C(6>{aJrUYJB7Vh< zUbDe$f+pz|ccbwmYaD{NYd>kuR?J*qN=1PE93^deXNd8umK#66$X&7ZE1ed z*j{#J@9uUTcCtse2Dd0?n;M{6}@_XYdc5M zSjN6pWbl@H>dkLm6+b%l>MT$8lnZ-3mTC4~cOk3F`(LKY`ulZ#4VAE{bD7hUypI?A zOR>MywGU!1?n`7o9y)qXbs4;Cm8II0-KwGkF}4SP;&&2m$1*qbow5q{vn3DFwWC9% z@pam^(4@Vjwd-E6mj5TAH~D;9>$}7-{*Gl{K8|sHfbo4i<9yck?n`7JnAg_%zAn+; zgL|o`lhn<9d-83aPwHQEUTYa^C5dH)$FV!U{5)iG2j_Ch%f4U@mrt zCXlDvr1gc?hvP=pC1U=2-0gu6@0__4elNcZ-iCK%+*(fBv#j)njmYXM(tiQ=_2(R|EiY#dG7NAN;C;eRXML|3*3 zJd+B_!8?T?rO&X7RCxk>e!$TUkS7+$LulNaHxt{SoQa{&p0p z-1wIMGVqtK-Ip=Xd=2hp?X*KoW@5nS={}M768`iCrJPh!p zOd5N%Y4p&easA>crvaYJrbu~9o%;yqb@DCrX20SZx`FYoQv zb6c}~4q<=DEz5Yqi@K4qs>Cm4C;VU)1gGcl#+UFyJ-Yof#Cuf2l(Jewo+-C)l@qk;dbFeUeB+e$>G21<)^n*rB)vbfA-;_z|JoZ z`Jg`=OBm0N&&sxnv#i`-b*hZDPFE$!ekseYN?<(v%V)ODJ-zbK+h2Ti=L9_G1Z~%ePH@vhvWphu=ChFTVEB0GX%xv}tsRJV(bh9?vb$NS->ME^a>H9_?>xLuqh9UcS zC+pWkSeqOis_!yrGO|(Ybs{5wNInJ*%pEnXL8bPGs)bfK~mZ?8`blu^=Cc68Y$p7>Fi+(SCRTa8VXU6BE??~U{=M$ZL`PH%?C{^zVlJ)3Ep&jpn zeiav7Df^D3PdLdQ2wBhGGe_p+Q=OxPFJ>3yrtEvQ^?!=92!0fLDe($@hBlL(TQDX4 zxs7psr7cF=KxAw@-+H!Rr>O6rt$vTDule7)XMjjkhOgfi%2{}^T^x|8(J^T47HRrRLd;5^R;`VH2}R!wc+ z-M**2XKUpG*PZw+#a-pRL2BK+7!{N-9tA%dqYA+~edBt%I@Ue23$x;5M6Wr}2R&ET zN}_WB$9}P?Dx#k^=Vux6P;~S_N_nRE5#3naG|llcMvZ~rW8wQa_&-2j&wd`}A)|Ze zB;HiU<*A95$bB7VRif2b<1d)sJDqRmBS)^GOm z(fwxW&d)Uu(vN-Fycc3sbT7m%WAB-{7h-C}&vyLCm?SvFU5A|vUGSJ8KJ^dmBrWe~ zll{NiOm*JE%rz14g3heRN7h^AF5}p6Jl6VQ#B@ZSYFXa8Dp{IQm!v zGIA2STPyUq;XPhla`jbKUU#20YD^DL9(_c+Yi4*tV`g|mxiMBvwEamt3lXNYvl`57 z>RIOEgkPreZ}ES&vz+j{mf_|cm&nrz_|d!!kJFA0j%`1*v5fplJomtNA`frFjmSu$ z-2isiz}!8wmwl<%wN?ebwf~&6pwy%2#Y5RQt;3(iIF+-E5{Ag9+ah6KNq8ckg?@d6 zHeAxgT7DV(AY^P;$p1qAC0@be_F`?yI<%XP&ZOhpdGj#7mNa{u3vfHt)!UkRS?`}D zzh&)*HIDTTZKRLVd0tN%B>ZT&o!tCwv39uQevuic*gN%oazpC54`tbhbprYzy%tuN zWcAVYk9?gAJ|S~=q)V5J$dGfpYZ-DiGUUP^kRds&rImO+y~E6#OIbATJtIe6{Ff_7 zP73WD+3RjOA_H!WOV1a#vst&6x;%`%+m~@>8UB{`@NHT0vxqE7)w1LtxY>Q!?XTt# zp7d=WK+|%BKCMyj>%&Y$j?k~|@wsj<)#aG#Y$d*XTa^7$LypKfMYrske~9?mfFDzi z+z34?M_$8DczvxlQ=C_|ne0?!p3_1OT!BA$aMeQOKpApiDRRKU-NL9GSSfNq_K+nx z`BAsXf%2#vm?Cl@$?1+=w7q|XfArq($bp^Ee@G5ojeW=ep&XF+X|L_3qgkOg-E{pK%HouI~n0sjES?~{?t>IB~ z=C3?@?|G4|ud*I3XKG}h@KvNk+EahppzxoRgS5+$n3JIipNjoP%2@bP@IdTdi=2hK z6ugJ=>L={2D5U(z(>Ce5GcmiwNnA3A=5zYtPIRU0NIV7nOL%=SuZE`QRj{uj9#<}; zINh|LRHrNE$Bc7dk+H6{2T!W(f79n5kFsA~&O=%cFo%i0x|XoSjr28HgM2C#zq+q6 z`;WnmcN!Gac9F@ z+06gxa;3~<94Inf`jnP!a})U~i?+>|i*(z&9Cx{dAC<@ASL)x4>q=xBH+_5EOwL#ePf6U7^^Cqp;Ma4hznv?Z7DfAk z(d3={_tE^r`i}6E>|yn>hOk_84f;nnUiaLmtbs{<0d&Q{nxEO|#|HzR#}=F8K*E-} z{4F`m?zl0v$!*(rrsp$8tp;SE+qt>zSOx#=zUAu)(Twed~*??5iy|Z&KyAFB&BD${I zE@dxvs*^&07L^BugasCPw8h+)KY>15cudX+kf-^9vz!t5H~Fz-zLGs@i5@v8kmTg) zw2AKU?en#=r%3vPv(a_kwm2uC+ZJmPxsh`MRm>%QMtf*k-i(#lcgcJ)^Ccn5P{u(& zi4QhzApMr+vu~$^^5TAjGv!75();LS|L$gZ=4^h1_iv&;rM}7{`yd~o>;=ny(*2)% z7FO%}OLhcD>u)dE-Gi9y9Hd<2t_!{Yi@1_E#Hr7)Z@J%4mpSWWQYKj+dCz($M$0nD z>d4$D`AyPgd}f_DI$nc^SLFmQVccaodGLbd=cq3^AK5y8CuO&JWS@#;(j@isI%#5U zvReDO{}RU1$8e;#_Z_H^-({}jjX zYpYt?O_)6l8T4Tn5?_FDTe^M8ZBtGejvnikL z@hsHs#*@Z5H@P1`!isNTAL?k$Pw>?%a(BpC%4i_{rdt+dF-4Ca{%Ztb>YhM;_a^MARU*y^swXmek{GQ`$7<0qf$nwv_F9cuuzu=5JG||Do6g3fk%Z zEs6e(Fy~v>4?fL2>&(E$FSmJ^*9-*4b$aN%rbQO}Il?~AWSJuvh`*e_x7GYVmT>9E z=1*PT+aBu;s+-|E_T1~a){+}6wRu!P&UnchBYVxy=u2j~O1%2coz_<0ADSB)ooCHo zcGbY|KY6gP0YVV=?DqmSH}GS&8`o=6cLIn6;RkX_fsf^8>R+)cl2f9q$Prq4(5E z`h=d-!?q@@FO1K-F_c&rRIPG?d13Yy?t8@(s%IQ2bGEu}uWM)T+fBO>KQBk_1ESsN z|9ba6pf1R*XqZ_AR*ky`+j?bIokgB4y;dI--YdJi_4YUV~_V8%o&7t`*1J%Rv1J(ZX{h@uuzR-WVapv`B9@!JJ zD(9z{+OjwNE1#vFByKr_pUwA&XPOr6pkCJO?ylR;Pe>>8=&NL}i_C+wKB8)IKcI`H z`madWbFS$3D+hG*sciZP<}=lYV$U80Devq>vkJ49%qkr4GwkUf22h7v#J;<0ADrV| zIp9j{SQi_>dgR+;cYeh03;6vres94(347)vH;6s^IwkC&FJ|R{-(ttUzya(Vd`;~7 zMf|SA?y}Ny#&ng%?mDy;I zJ8qOQqlEic+=$!vF|RoptVw}~Wc?z&_(I!yD0sp{-z)hQ88HhP;nqtJT`zAyA5eNu zXh0Kt1sBCiy%bPB1rn~5!&}Hh9e;7Y9X!dN1!%!{2dnci$DE8&W8RHbWBPe&#vHZ^ z#;ouZjCsXV0FNG6vpmNhmiS`5Du}zveWYWbM->YG8vM0K^%r~y ze8{8v2|fls=23kFH-Z~Is+Ztzz~6XOPr+Wo^m~HsCg1)m7;2!R@`O zv*1*4s#j$R&ID(ARY$>{z@5A*LvUwsXRk^Xd>;5buSyb}56<_ha|9QGi@d6};E~{w zUX>tt6nK{pZbsBXTZ;({|jCYUhY%%f}aOJ?^AmPJ7CAB_6U9n{1R@J!f*h ztKe6CYPI0$#5dim{w#PVc&1mqD0nt_wpWD&KL~!%tDYA;4?NGSRtSC+{HRw|3SIzS z;8jlxegpi5S1lI25xmi>772a}{FYZO6#NeO9j~$lzYl)js~#1+4ZO{(Dg^HY?_|6x z_+#+LUiEr>+d$AV*h>SnljpSnhHDmc}rMhVUYXZqBYf;)jb`PAitJA*s>)G)#4fzR`)!GiO_ z`94)F_!-jk4DDa=a`1AmDir)Y_<67DFW3P)Ue!Ehs*m8+;MHE$OYp1USG}sI z;Mc&f(f$SZB(9!5l`FVExW7+z6f3;Nd>iQE+c?Z=cE#+!x%J z`9Hyzf-m){B*7!WBYo-|!K1*Ve5$qJYr)t0RD#CHrfWI>C%7kR>q+|;+#B4R_Aj_E zxG(L$7Th1)pY|{KQt+j;f596GbE8ik75od?O*T> z;2UWFf^P=jO#2r+9z34*FZed_ZM1*E6TuUG>SMtUZPSTS9|(R4{8Ehim*CrRb35%{ z@Lk}$eCmC{_k!>BsdoiW2T%8@&4OP8zvfeK3Vs9phEHu2{4w}rpIRsQGw^3V^}68Q z;N3n|BX}=(uTQ-ycprEl<2=D%gTMBvRp6`@$eKg^AM&Xe#qJpRm`{ZSH-Z~|>UqK6 zfWPsn6@tC!)n31<6dVhV^{b}^uO^?X+5Zo=&hMf3o{8Ueuf?d!-2+t-I+U2}c6*d9 z?i28v;8%|dZUb)PR~3TWgWLO6x!_cAs$V@II1`-dSF;6o0(bJOQo)_Uo&D;5!R&!v zcb;EO6PypuNBsONnvzKlid+#O+W(ywk5JPJI@udWk(E%;i$x<>E~;2ZpEl;E4eH~ZC< zg2#i$`_<)wZv)@vSHlEP1W)vsN(>r-P^aRe!;Z zXV%U1tA2uKgJ=6yAHffTAM~qUg6Dzf`BhKBkAffdtL}mqfEV~xuHYxYPxw_=!HdC* z{i?IzXTZ<+RhHo8;N^bRQSkHN=lv=}umg7dDpl}H;FtU=N$_g$YQH*1@T=fg{i?O# z*TAp&Rf6C*z;AH=NAO1QM!)h4ehd7TUwH(-1AfP^zS#wSAN;;wofNzcyv?sp2;K?a z=~qVue+>TEuMP?R4E&j2{YUU_@NU1_FL*C_uV2**-Ur_2S9=A24gT7%_6R-%KExhO zu-41wTx3Uex=)~4=>P0LRv&`1iX-DK=Bh0A+*sYPV@$AB+;JWz3;q9naaR#>$2~jN zoCoUHeQ@v6)zq@X*0 z$T%ncmJ96+?%8DxXVne9UiRZ&!kBaIiL5!PCtlM2xHD!#!4fNmcw)Ela>N4#Zr%(v}IF$iL=*rwhrd(ZyL?~PR(X`F>uv+Hayy%F<%Y5S5s!4 z=(pfA=A*Goc`Cl<9z7Q8Y0F$q)pGyVbqNvPubW+HeZf~hyKo(6$=2gu?mO;JTvdyM zdJRD41!V1qd6F$T=-O-N6xKdbR@iidr#ZYl z!qe%@p^Y=vwmjBK%R^iCc=>#Mem>fvCw8S(P*xb9EnW@eO=$^1aRX9^Y2J z9ef}0eactI_XS@)-&cGG`Hteh7iHChFORP)Uk=~7d>MQxeC_yJ^TqSU@Ok)7<4;K# zvk8ljvd*T=vnl)RPx(IL+rhV$?>)Y^`QGH)!1p)48p7x?hdBAV^5yWI%a_5I!q<+k zHD5em44;SZG=3D{QT%^3hdBAZ;H%^NlN!48RA}?S^Ws#V;J3hU#i{cI4+jsAQ(Xk#K1BUf;=|l^dz|Vdv`4{@#;Fd1 z-vPf9r_u$F1dohUDT41BqBe*>%w2cIsdI$30K6bhwHEw7`29GQAb1pbRGf+veD4sg zw~D#z-Z;#AY8;BDY-aq5)dYr)sXDJ6LN5cQnIhq-HdoH`=3#o)zp>Y(7A z;GJ>mYr!{wZ-`S3f@cmuxSah$3Zd^7mwIQ6OE*+bM^ z@rSu@ z#r;ZYD>_R9%hXhSI@G3P^<@Q0pmD|7FOK*xi})`&ndbU`#PvU;h5xB7{EJ>I@!uKoU#_xT|F-LYatr^r zwD6xE@jo`=zv_55*ZT{JlU)aL`g)RJtBmOUl`uAMiSJ&S%*ME-|{=2mBFM6%SpAB8c{(q-Ein$Sb zbS@wRJHp%&XxGDzzbesYzpu)A%Bo?#p{C_Jam#h^srW0|$li10GW!j-UBr3nHOOU= z$qSLmYmmuHBQkkuL?%}vlacA_~D@`e{&rBo7v;+^V-L_LnL}%VI%vel-{!?d$_nGPx9t*&*h%CgtZ^`e_6DxG}ffJ zSGIZwdokoc%a<}~JZrMo^1qI>$vT*vWm-r1oMx}rhtNb%ue7Xcp>KemHMDQTM%G6y zIUn#k=O8en=euV`-ft<#pVWbbDQ*5aOo{LGeM8B^5b`pZJPjgmmvKgc`&WwHy{%zi zYJOmb+(9~Md!SvQq9gZLg!SEro0sVLR(Wi>192R4&UwDfhAx=prGX`~{w#MqhTB?- zml$NdhQ0J&djj!}rLBLyLiRLvpuNjoqudeLHmG_XaQ9O##f>75Nz`AMyYwWydhGK! zXE%p^^h=3vD5lqwSz(4F=Y(R&)AYq|Sv?X7NA5@dD2DQS%(5@v=f?3Mc5=51YcSO( zx>z+QSPPNzJPL1q2|X7-;ZLkQ_R`gSF-Ovx8A&Vm=D}mKCwwpE$=a5CmtnVHjh8$Y zT`p(OGM(54ckjf}$X$l$3#{X3I%9~pmOS5tnLz%`y3%Dd)MtAl=fFGQ#`Tx%T&4Xb zIsLSm;w*qin~LS^NE~$*pzh+S!vyM*b9MK%f**bMaqhK=@}!E~b9fS-jPApKJuO&s zsSYpMxg-*9e{H5Z7h|qr-Hd$8J{&*F}n6Th}X#}m=5I|OS&Nx_+Ckq{c$|8Z^4&qZFz+io z2wmjBZeLKzUfav~PDR|x9k+WDgX&k%zJ#Wx`a-(iyIATtd@J|e*7{~&qWALO;_`sV zYKiB-C!U4V;bYyd&vLKfnC5#8Gac!7#7`}L#QY4iKXJ;Q+fOtt#ra5^+#{|{?t#{( z+$VcEbvlB&9Znq&qppY2K8MggBj<%e}lyW~S(SO6TJ~%j(c@lDg3M{}4unrX@GbMizdjjm-Dk8zORT z6mdv;kFd^JOB@gJFLLeyZFX=bVRE)L*={+zgf72%C`MeE2bbcS_&PuI>hAQ^+G-X-p^F6{pvJbDLbNp+U z&u=AdBHy#=18TQhcGr5BJ_fqJYaH2hG)v@CrXzZvvUwH2*ytgzPxj~tI&fQvHz=v`N><`sJH_p*Td6IJu{e)+wPnB|#{#4$E z5Z*mbSfih6TC^9LE$2k#?xWq5%@>@9`JDQf{h^Xp`ky`l_@lC|RXvOE*2w=ZD`t|n ztu^{6_jAi#u(H=$`s`D>&vx&H6@LEgW4Dfu6Q=Ohr{rDu{TOt!AKnE$Yv@G{yV+y( z9exeR+SgFV!UxjkclEZWWK$+t0c*-Y_NYmJ+T6#=oucqvEbiBJvhEtlx$>%+F;@q2 zu1O3`?s#=Li8BH4lDt`QD}7J7^to~L{hWoZ_jSnMM;#7>_vB1^34LVPmyy2*9xAy% zutfKtgjZ_Oe_}RuusCDe(b4_rCgIO?-G4eyx_SH?d6Y9!+|yWH`-)}H2)q3G7wFPI z6K8c(cdMqcgYZR~a~^!D<>v>UJlZ#WX{o&{B-{+@NbZq!>q%s|@aLD5C*>l0S^LzT zAK}m1cqUDdRBNV|AvU z(WRuGhg$CYXOfly9j&{>y+zvP|6A@Od||rJySAyQ#+$$o&bW+;N1iaI#B$UM}~JOF4)RF@e4z{HK_ZZvPt!>~zju&c$Cn`a`D2 zm4oexKNG&ILKcOY3l-UDNngQz$pP+5W-j_*fV+@o9GyO$)RqkOFIrJCw&2_XTIqxd6O2!UupOVVG;&S)K_xJ6uo<7sB zR53qs82*?0zombXb2V~j%=$%5i*%imSLs8(AfJ2bLxRXw(St9B?=tvyqZ7S_F2r4k zeHQWUMkc#`hu_(|C?@YJ^3J${{vzEG*(CkNL};#ll;K<+q1_5i_-{1&(a-p{b4TJy z>hC0RA3>hnGcyhSDBT%~`z$M^fxX+%J2)rNLtH&6!x^aUlJq@^-;rgx33gpXMh1vW z@*wqiDOlv!CE84JI%A4_bL~av{Qh#(a_K_OYV~4ouE;j-?g}HjYw35E(a!79pQLT) z(XRD4fN-1W$D(fJUZ#5bv%H*NB<9hd4fl0y*b^hNF_7P7MnLOKCHzNqCif1Aj*he6 zZL{g5Rm!YV#&hUQAMSPO?V*c~Fcw|Py{j(6nGOF+KP6>Y3O=^WlNbKOpuBLTZ*ui9 zY2TTV`z11**hs(hXY?iuSuT1L`@9bn1zXWCB{}oIboHi2^rm?7RxbN&d$@P0(cc}g zaue-k=z_e%k>-{sdfA6&`SSJ(|5Arfiw?EvhxfdT4kcqad55Gi!mIKQiJXs#%qv9C z$Mib~{>|n-Iq5&RE2Vln-)oV1h5^3JNk#0n>&t%%UlnO^?+Z%v;4STqYY3|I|mIx1NU55X1-u(&x$qKFWWuDMw&3wW&ggKTo-p{hP+^tX1 zGdk#T-MJB-slbi!Mj7WvD`nrRaSlY}O2Bg86yd5qlB-tRG6^CT@pg!8!a`ReT>2Lho}*{T_-mLE?B7hv<&*Ra^Cgf z{Mgpy_>btv;M>T2$UEW+<-czyRAkLFijqB~|D0Bwcu2FfkTIfabbO?~`? zbmT!_zQQfLXkXI@TEJIU(~*TnOkc9Gk*`F_9WaL<{QZE`kB@+o(AMX~7pf~9>)S#QVfVfaJxxdkk=_qCbk z{6m`^oz>b*cJhf=+D4(smEyC%T`1=|MRv$}mVm|Happ@(m zT~5m)`@_-W^nV?5@7y1gMjEw_HgHNofU)IRncE5WvNv#7Q_lTu>{jq2^`YN@oj`f# zEo1+$I>M+G=A`k14J89%y=GSEIFB|-Q?+W%4cj$*AgLOE+ zal`qXaD*3*%eWSMU%RD?nR{AgJtpt=vep1EZYIB+m!M1o&V9I-|KF02q1+7_bth?9 z(8BGV5w~H|DfMgd_NM3yKc^jt{G5Wl^!w`xPw30xYsuSq=#m$i?~C$H~FN$NMIf)k@=uj%nP+r=%ctKqX^iw2@@|gY zhbM2FMBn8rA|BDjUu913N$yc1z3M1(Ybd(6=yr{F#O0|w0(n~RUf~a&qD>^s=(r^O z0&n!((i*!qdG#yeLdwUxI5RSq4C4!P|EkEX>x$j8WO>9PvTre6WTw{91nV^$c+!tP5;XGINvmAzu@c3_Zqr$Bf5bj=|f+YbUT!f@a-mO zu5Q}F5!ocPH^d#ffXw$Df!}5R4mnu8dFEf1pg*b6%*jg_%W$6+NY?KT$sOM!r~W83 z?)IL|++CF4eR}*xo2fphStq@PI3&#{J{TC1xw|JNPQrXbn<>uyn4*`88_`$)n_gPt z?btAfIck}+%bQWNqzk%(+_@WOjwPy3N_&)kNcz8&$XdW0;t^e_ES`Kap7S1^xxKQ5 zo*?BD_#fyA%h4NUydb)1ug_ino<#W4FHlF3`8>|v!$Z>N$sQhg&-|~%cc1jLlk=9u z&@a04dhUFo9+OZ$dl~P@IOlx&!7_BXs9*7utozb*=gmmC(oe{lLoYJr1pSBH3+BGT zBzn;$fBdJGU1Rss`UtM%v zc64$uACi7)ldM_9(;gCN7m2iwR=a{gur7$^4v@VSBvvj=M%~z^SKA{BRcs;!aPhpxN z9IPdr5wmVSQA%9nSQ8lG83RtycX*GOHRi-pZ=$}#`*LJsg}kBUNz`|Ee@Q!$x&Au( zsrAJ7!M~_8?vJcxKj>y6W&uIx0=ICbLRT3g)eiye?58vb-`OGtj!M^wKjD_ zj7oVkMkT+^J+1G>sPwHdyq940$rhRUlkWBbNiV#Gepx`D(I;zzI~N%6>-U@b&?jt( z-ZLm=D06jnyt|TJ3Z^Gas;=FP`1$N$?J zebXFk>*@QrbG}&C1zesPuKl`oUnTE#8g(!Ce&EM#mz$(re*gQ6GEa6KSuF45$-5{b zi)Efz^xSF0k$Hdcu^Z!fQ_GjK&w}5hbu0aa=7(^KT}QoTN_~1W^qU}}O9d#$Q@+f1 z^L!oW<@(~^rG33HoA#CAN$2g_jtzOP?x6XkIjn_gb{+S$$Q>@QHW@3Z#B`{pi|@t42dFSv@1`+eYj(W9)2;%7F@ zD}E-Lb{T`fFOzdd*Q*wb($_kN133ol}R%fEh`PS=HVk)wf$x$T}@EOvojPcDx4 zblf+VJLAKw{p7ONByXfIUBH4f(uRt7XlG5j9yc!70d|(fiQfo4DWVMDMxUE`$1VWlfH~ zW4j*j9Smfbjr!bIzF6Y)dQ!Q!&%OUu)|t|&+p*|&G8ZU$mvLK(kG%`X9mc!|ifVK0 zf6~W^-fK~g(REgt7wiRH+Tuat5kDd$GG~PJy7hmcl}(y{;;BjfPt+Us@l!%eF4}5^ z_L1M=q$R)3qwoDGVXd~L+jWBNJovn%TY9MEUCNvGz&T|ZW6w_y#j5Zv#UbuX(s_{b z-i=J#O8E6%28NDH`JoH$#1wwLj`;r0c|H*z*0`9aoZ)9yM$Z@EVc-7Wua z@>kCOD(Rcy;nB;-J8_UY%6;osgHM1jX1*laE&?)#3ty*Fc01YUCT=%ASajku z+{zy0FmbJ;+=|gT@UzMqR;-dE>6Csqg?g2EGKp7sz8HTpA9N|^n|!sj3(YUpIeKp9 zvu#34buPf}Il>WL^=Gt`Y2>XSsfV3=A$JcD=3c@Z0FUdo`TNfH*W$0ewWbPQOBq+B zWo@e7^O@>o6V?;x2vL0_m4Et=z7m#Kr;qoiq2rstXFa5^=I(+~|u?*--kpBCZNo+X@z zo5MM`Ih>E8;pjb~sm_Nj!g>EJ;Y@E1r(<(C|0^1f-W$qYpe@3A{Vd^>G>6lnIh-}o zaP%J0RHwQ{IL=wZxwbi+%;s=bM8na0MN^&SEy8*7Ea6<%98N}aI5zHt7uXM~-;l^% z!uv(=dAO%X&c+{(KcmFwGw2u+l5OZ-!r|gTzcCvEEcbfIDJtM2oNxkT#Isg9Ow7q$JmBsb= zKhM2MxHn-7S-C`#5S84ZKqPDma+3&#MU+KRuqBACfrvJBsX!$O(T~Bn@K~&s^wY4^ zntKsN4VtvI1n7dcRghM#T};5`CUF7T^00V+?=#Q6a7ENlzpvNtkLNWr&unLA&YU@O z=FFL!Zp&srvhs!EUo7O?_o%U??E$sq1E)HrY0&S_H-AT)=RtRV3LSY1yCRg6)KL-(?Io8^h_> z;y%tgqW>SxF<&7p{jUy(wrcuY$T%MgjrOF+iso_k|H7h8i zinU7MED=8SEI50I;S?AdKMkYn)a<>iqgzw{cK6Q(mmmN3TIG6sZ8qN!M?zgs)wAwm)vIm>;h~P+b)Qu`%)b59 z!R^?wl70GLj9zssap$AsypA>`z`OLMK6x`@YW}AB@@nDl;{Uw({-W-9Bff5m`|Y(w z)Z;}aAaa1BBzrBFeTT?$?t$L5qDv!VY4f9l%%8BnhQfFudnFdV9Rt`4>f7gjj2wEZYj-B85wj}JiGv5n7qMdYpV67H5n>}zj}vWqKxb}6_c>FX(1+!MRS zO$qeS;gmoxco+%42#k?;q?m78gm3B({z%-l__Jtkon8LI63G{%abtj@7a%& zFKN+l*w11AkG!SKiM!ax+=V|Ftv{PuAJ>E{dx681HwSU3U$5 z`nKMieEFT*$`_KTETDDxqyPcmWphx6+8Gs zM`L*l0N%)cV|}}}*PliIN}1xGT5kUnon7P#@H$^_Z(GisL@(+X@*dGOiO6F$Y~eGf zyB4a1DAvT-A|uJ`26>o2VZxN(~(omn&L3;;=b8D z@TE*>8Ec&zT({uTv?oXC2>VDSIv+I{$-|-EJ(~rwKx7p}f_o*`> z@EI~RvElwY?!yF&(=v%jqa4x#r z&}`%q@m!yM7j)@;$$`3Eq0Wal- zmy)<#cqT7$YN1bl9X@~17?hm*~T^kXjLI-5R5 z?z|=PCJVSOGO8`v8Oi22(x5jn^D5F@3w|^*w;Pr-6A9Y$q^=I#NEkGjlR+@fySA&YQN=kY#e0`SF$G zC*;RkiNG$pQ_ldy>wMDIEbfL$KG7#w3ryn}9|@ytu^PFOzVDgEy>oaaOVUT=$+^H_ z%ahNo-xryGLNo58+zY><+%Eb?_^)?Kg2mf%xrGP)Zj%>92Q{lwk>b=>{` z$?JOl_45;#%l&`Bi)}9?@U}zu=q$Z%4l(~i8gNkaq{tHVzDV97lzz|kCr}RaivAPc zSa36wf3jY^^}~zJ_pZb40Q*cAZ*S}JaxXGh{wDNHlH5(Q^v_@;aSrWZrqcu~Yu8X3S4i`3s?i z-fZ4jUFzwFTzO>sM!A=k6u8&!ubHLj@pSwN8-CEn1m=#0D@@zJc*vf!@=tiwHe_8_ z!_NdskA8JmaCC!5{`guZ_NG&yjPlK#Bn#e9zbUr3r1{x^b0$>33~;2J#N zX?jK5+_cT*_O;O0Tc#wYn_al&OVed6E6R)q*XsWrX)=#1T!c^4u98%2R;c9P)6QM! zhPsq$*^WPnHVaJq@JpYZjT;ddlEDAxrR}sq)57L-J(k_%Ypl7kThmMCiI;SuD=GXy z7VTjknWvT)Gq;MEV}BG2*07Oo!TOP7ys6wqkxVsJVtgPGp3~LRmzI+ zArW}O=P|q0m{I2Xu-*Ivlqo!h3YVK3qv`}F@>(WYFe^=&tf7oGXeKppL)et)Fz zrA^Y#^|WsS?UeRLV2!jj68@?@-R8?q)1khj&BuVTHB9Gf!*s5lxh=9E>5rq>V9gSm zxNEfe0BND4Tcu9~H$^WY5?4hWb5GODUEy*TfCrIsgm%gpamI5x&mH7J&a(Am)&*&o zKTge>?xyv^?K6bjvMQ88+> zyRWibKaTYZjz)0N7e_nPaI?Kn(j{){E@SO;wWkYCSngQSpR@U0fu$0Ose%v@Rev6npV-qf2k*{FL#@YzgcI2zI*e|A5^{BbjP!wCm-GZ zdDBDtKKHMD^v%f~&wcJ4{l@1Ve?0X0;!)TE&uaZVLDkQk)&u*q`InfzhF8t}ls7Ow zbM_1zh@CuUkCFUZ{)+heOoysi<1p904v)7Px}1%yK={3fkTrM>?4}kP2eaYR`qCyn zcM9{&asQrZjc%W4jrxEyf{uyS$WJC(V-HTWE0ub$R<{ zxxMi}j`x1s*?QNIW@N(#`y~f^Tqk?n7-(rMG&K%>g}eXk+d?(&J+*S@{xq+W)=l5; z7CmDZ^CzOyBC-K!_-b{szxniEWUr~3#?U8?HFJc9ul7dK@ans4IR#}y!>^>iNz^-$ z`mX?vVrY30vbaKIal396eEm(KYO1|Z&5-i2ft4?N-D^*UnwGNv>W_OX`ce(!jfzJ0 zG0W)Jk{)q0GL5)=xvzgWaudn-)`?KlV&ni1;s%|0f%oD{x)w3-b=z1A!*qX((0xfC zq5T!Ko>qa+X`z9pc)HN={=P z=$xfwE!gdiS__0$$~rzOTlq!SF&}whHT1EAf2{M-Yr^jEn!x*}_L|^YbnbPa@o=aB zT$TODX%u%Cp5kNc_p=+o8K!|phg`6uH)*<|vIh29T<;;bYb2s@@DQA?x6AwJSyEC7r zoIjEGPs;j8@_vN>Bkp{QzX#>?FsybdXE0@4$#)CoEF-^%axN8kC}#)dY@!^|2l%tH z{w{gD@OLTeP4OpCPJ&^*A>|CAjH!I(l(T~T11aZnfhPfY`gCVLM>%hh_YLg-N#4)! ze}-P5`1?>!AH#Y@${9u(SM%LYIV;IOh;oVq9?IE4IWJPqUh?i$*7K705dK5TdRF{N zl#^swo1~l(lrhV&NE=UwdC-K>+H;B-m0AN|c(e;4yZ-t!YZ7R^UwzwPO)l=uGJd3x@c!_7|W z$eZYB&Rbg6JU>?D!+U#jmyT|ZHeKnn#<6bSc$cy|Xlpv-qdYeU!ZN-2ruBxW`9gOvY;t}Wwh5wDlddj z$TP0*o^K4$a-lrqM*JCF)$j_e`__DPNV4(ke!F0o=e1KkMs0>(NBXDComQ7NnROkS z?xKG?=wI%N=XKDpi!MNC1$qikmAGgTez&VG%?mw=>eFi8N)9qku0_*YYF`Z%X!<(i zVl&NWOL&@PerWKoYLVQ5eY8gvwY$^NK3%-=TV-vLR({Ly*l zRE_^w!pwsZ^PmI2-2W-r7<<~@R!Ha%?;;1CPk7KJEqC*NJG8qZhJA_fH;m^D8Q(1W z?L-|ko&L@=29%#1rmQ0FNy5XE%AwMZ~95PCNHvgID@mcCj{m%3DWn)&?0%1x&I=$lKlR z%*PLy3m-BDwr+AVILkQ&dW;-k1tfnh`Ts+HHD=Mk^UxcOl%cY1`K{co((|4&Tnm_w zm2Q6{`)B>OJ7?d$sY~)AH(w+3#S5-SaH^BFZYp@+ITd}X?YP6-e(8%r1DSWgDzd(g zV(`e#85a09863TI6>D^zucZrI7Cz7#+~AqV9s1Ha%xPeUPhC^`Bik?VmOg=>HhKhh z>hEBV3(dL-K3UFH(~%vgvv(A~3)yj_>>su%Q2^0$Et_MKb>?ZV~(@(X#F{Ud1EW%OMH9~zM_ zN8orGIJ$;RU83ni_{|aF<#d=gM})TPeM8k4vz@f-L=GVP2;LLpJLn#*xw6d#UrxT;_<-*jz(dgJO9b z1boa!UqRk1&x%#2p5*Rl5S(J)k}oteE3ssmYh?g*)#F;LSB`s#_}iE>j}X6#_(i=`Vio_Zh0L|0?GHe8@iMn=q67a<1c#?>6>(;r|6>XR?|G@RQj-$c^aXaO6Z}C$@+XVp0&yK z6z}EiHT<32FAjp^>{&(DwmSRH409lP1xG`3n8%U&m}C1lI?OfDD`YMOZF9gEc%Exq z_B(4UV$~Ynoh^veA+la%FxdJ{)q5?uCvz8Rga*snasze9+A>F6)|6|-Wu2NO?lGS= zLtNIUtHu4N&$>!n_+Dw3tap>e-RZNg6!%X)YofS+_F2W^Zt_`$;y%Y3BJPVmYrMEW z@mZIN%b1N7x87%s7I(SN8YyliZ=H!-gKai(7y7JW;@;x3E*7`kXAQx<=42@C=ez7P zX<2hbKU?mx%DIc2!AV;M4-Gz%5ws1VUhXHf!IO=}<&L6RwZmzSl6y@KC-0x*7CB7r z@~){`X_&*s-J#5(xV%kjUZmZmz!K`bm5)B&?J}mdl(7cM7+_c}d0?!Zn;`QaUNm-> zoR{;x&>{`#d>Pi=Xj6P>}Wr#mk-w|}QdSS&iblB#wI@$RT7;4lG*6`sH&GeRga88@Nl^WByZI_Lq?}k+*jl$)TNb zd%A`sA$Led?vTQMKb3ud8vFlrt~$kRtvC~OmzP9biJvVJ1zrsJQE{crKl#6MG659>Vf{>tx? zCkZ<{@JXdN5qA^zza`!G@qb@gHR4~4e=+-Cou`+-@^12^aaO}#zH}LJ%apZL(yhS1 zLRoiFck=E--H)Rb+uPrH>K!n6d=-N`H#~r_e(2V@JAjt+Kwa^CbBz ze?^{5RZ+!$zcfHx0GXPkdlvt*%DPefFW`Rx``)Ga2_5_$B16uJV-^s@aDJupT+n^A2rP{wi=I+TKB1 zjy@-ziJpIp@Skq%^ttZ6SKHnB(Y24yoOA8gA7&t*Ol2Qd(8E68?9YCMbG6-#BD+Fo zuPvc!{E5L~IK%6?!Tb@plUUD%$CdRtI7hW4Axmi&-3I2uQ-6YHWQO?>?lTEL;({L$ zyE$3NzvMl!cG85`|J8c^m+(Nn{;yWD{u7q<{}AQteq#MkI;C1Hde7*9@02{R@o)MncAA4CN?=K5{7Vm@Qz0Q1;Gg?V^2>mrVGrkOe1^)zIWW44! z_}Dh?O>BaGtX{gLc?$mmc}`2{QlAe^%l{%& zkk9-*h-@!EhVvnDT|TQq+$^6p!7-rAKn9q9K00rsCG5~T4Ea9s_wI5b_dCd*q5|2w zxcd^&)4+aH$Z(c#hd0ZawYG*VIw$WpCPB` zolk$rda=(LlXqVX=jbY?i*^c5$&&|z$yyvFoZ7QzZ1sn$7hfat5@8*9u|GP@q z-+oTHc6m;7A?YVLoaIH7u@RVVzh{sQ)6zjUOxgverGvVELB92eLbLZ%Ci%K2C}-Co z!cv~hy%XTtEbhQ{IJ5HL?Gln0qX?vIZw!Hyhzn$fyVylKq3+SKzM-X)~& zLpjJB9aVDPyT!}fz`5+H6z`|r0{yy_F+2h9Y@9JbZRfhxy<>^8P0@vKRe; zPD#O|xT43=m;Mz#P?vQ!AKx7s`ETN7?ld~D?_LBQiIE!BEek@32Z@j4d`IyuMJatNBa6!7S5^>z1M6i2>*ZVhl!;ErK_6?(I_46(0r{oi z?RNTQ0)B-*`SzFV2T^topX{Sr@24z>&uTjvYO1AtWc5v}D0>lZCU!_V!9&@TPENs< zd+mh1_ytG3X-}@@jCFBtY{S~gdnd`Ath$oDahA+C>Bo~tdW??rK{0TijkDEJI2+!N zX?lDwX`}IVDPh_q`!P+A?;;$**D3VX6vnY6Ophl+k8?i{6-4%8XX`;lbkGNZt2vQ< znrq=e@NT!`mEteqrh3JBYK7nn^n4usB|6%IJNt1@{XyZM9s&E zXSJ`8Fnfs8?HeD}zH{LzdqrE`Wt?Zq*`vrUWdDPH$|;pwHjwk)Q^MPIP>?#y~JA1-9hdpD7`lSjy`M)-xc1z;mq2c?CgH&!l?eTNEeL{ zM+lcj(XWGq1s{$@t$BARqwu<%?C5?uU2W-S0mE_hALcvLyXKAf+1kSi*g%7a zSw|b-TditzLZ7AFx!8p&1y9#F2Gliu5qjwf_Ik6yl{JomX6=Q(@V&aEI?loTk#^OR z@9cI}$NNq>pnEDN{kyWa_vMxj&nFU1cJWc>mEd$>oif3gj? zf%LMMhtJuJteyQ5_r}l%7kPVGj=x@H(l_GPjwl+Ni>_a7F?&k-*4tjho`$=r&@qj( ziHpa|xvlUBGS>3X_zt#p@cVLDI|#eNYewZ;(EUN&wK$6hmAe?sgRiV1Z-K+DtTpQ* zYsdL9=7~1+$FhJ2xXO?z)zbE|SoZPI<6Lp!TY|E$&yl@Xx<0R~Wew`!6S(EP?)_o| zeHpPo-rn%;izK5Txl5B(Q=L^&I%#qEGcoQx~>#iMC0I)-1OwUa5so5CI22)(7v zd3z>VCD7d5J#pF|mX|d8uEkmPimu!w>i$r}lpJUm`*QGK`;pzr-Te=M`?!Bt${9TOWt%#P+r=lkfmhM@f)nEy<16tWrahupB!0n-!;F8S{HJ~L zU;Kh6ck=%-{2d!7`K2#jCEPIqxm6M}MuYV%&6efJ{ef4Y0oZyae@0-aluI2UW|m1pe|h^)-G=eQUl_A9f(f^BY}Xa^PR5>q`!-I$d9KV9VFk_t3Z07aUyV7rTAj zZ?W3I_jcKDfbU|H=6%NV17ycy$N3}Hfa2OnTayE8G(4WbW7-`MSdBY?@ev#odMf+I zX4U<&=LB{E^?dhiS_oZ|xJ2Su8&=nT2oKH}{x>iTIt@cc;0~!9T(1xpJilGTBKM=d z+1`oZx15{E-5jx5^*(JAIr)c~MgE!x!MhJaBL%jS6QPkd-d|u!8+=Yz-Ksi!UVMkT z1iytJ5qzCRSn#@le!NPbSkXhN}8eghbpT@{9!rUOOpOJ(mulXEb04^rk}F5NSaIVUxNHk{4?>-ME)n~ ze@NQL_+B7=D*xF7{Zi6|<$deLe;xm?Q`Rpe{oSPfDc?(^56cvPCTYSl!NKUkzWM(vK3ztc z(fBl)a5O%RApC#Ar|SO>pU$ZtU!0=#_-=)?E9Ws$-R=M zwli0<*Ii~F$w9Z9PtJ3kja4&ifXxNHSCpaOV&I&r%bAq-ap-}Wz0%oBk?(bQ**@3C z*B^-GZs!*>_a-{j*36TRiXbxKzGDLNPEBGWXI$<5eU|YEXZf+d%~^43bDO($d{%tb zco%!NOl&W47Ct+l{z;@hBmUL3gIeA7^V!Q}$5gM)=1%CK-~aHjcI0}Q%30y8df20+ z?7lsbRiUGdT=;!MWNR^-2H}r&B>s7ua&$d4eC65?-9GE1{#qVl-lr1lVtfO;e~%m6 zIBIghVzfH@Hquy%R=CgdgU&1KkzEpL-_9`yeezF#=W1&IXJIaE3&xxzy^PIT8ErFbKiLP@F;iJ z#6FJLj*GxP`CDM$PyE@i2RZj00qoG^in9OV+&K*M0ANNw#C`7av5hfmRF#}Hwq0yK zMH#>@a;??g?Y7P;dPx-qcJlMkTYarJdbp>!=fT?(k`B4a>M47MnNyilUA@%EDd;p! z0hb=^rMAB3aF_48WR;wCQ`RtZbV{5x=3c{_o2|ULiIeQ}>clup*Ihc;ypy_fDf2s# zc|I=P?8(2<@OX($0Ko&!tj4mh&~kc3de)DM9rP`on=CLz{yCYWLAM(H6WVsizKh|_ zId`MH8-Qo*-XZ2?C(s+(YwTe^I#%|V>U%ZgGhb2Ggq?lRb4Ut&kF+B9hBj?kQ#8U% z;r{W~&&N?ta=_rftxu8^xSDj(qLKO@)gZaso1}G_q%9>ohm7?z{$)b%&}B)cZv=)m z_@HcPS@!TWU0&06W6c`Uths5hS+z3GEGCWQhld}Vdq=^S<==eY zll&UTlfZG_icIUsX+iGu_4jE-*I4(1H*((BzZ)85Rq~AnmQwDPi>=q}BvoLa{mDH6 z6%cxx16|4z+Rq$*fWG?D97g6MG836EGKaH)KgixX6Prw>;DyZLTIRu*=Wr@>_@}4m z1MfI_B662J%5UpYxWvW%M=xK9xl;n(9ig71$ab(_#oE&PW^kv`-A27%u%{FFrwVP8 zeG|O0ynn!3I#0097U=!hcH18D0?rL0ef=8s%mNq7j@r7`PcZ(+2IZRVJ7U3!w7^Tm zOI$7X)*^XY`6qR8uAJNr?UMZjcPHOG;Y`cx29_7Gu{#zy)!0LB!B;D_-0-IH-fpQQ zqt3=z+qTpO%JG4-bGc`nzEGJz&k?-!bS>a4LdrkRp77Vi%l!&HX5elbxcfWuoyOgm z0UFL&{-u#;E_iqd`<2bWpt#E^^*uryYeU5`M}tRX%7TO4(AP*`ztL*1k<#CC-!vP& zm9u57a_3ZJ|6jVFN?MWstC*rpSz7~~u?r2Vgf1LIXHD!CmKm;1uKb2~r?jJch+eXs zz0^98C1IbV(RqWcFdzWcz)P@%Kg6pU9fjk2NXAq))d*_#d$&a5Pt~-nDpI%dYdGtH^+5Jwi8N zwQI>VeV;V)wZpkex3%0z-gIZUt*&re(N`a-$kWfzc4*(!<@KM| z;+6Yu#wb(tD<5>kZ<<2BJoZ&tDYo1YjtVR`dYT zJACsveK&XGNb>~l@{ReXkNs%TLZ7t|TG1Pt;maXhV~67{Z5t)6zcCY0`4c8t3M=8pme%T;1MyS$jmj8UD}s6>x6z zCc?PYV}bWLeWzi@#(?OijHQ0gMocL)6F62v8_N>X>oPcRFH6X%bFM*1xhb^rBLXe(*bqRWev1syw67G-Rv49=#@2L(g3A6~kodkJM+ zPTnWD+uH85@$mU=#+pw?T(N|<97tGt#iG4b0pFCTN!npq}PoF8D=5pO3~*tj)#3qigL>bF64~H z@vvIrc({=8;(@l#-$JeP=cqQ8Fi#GM{@8DTOF6LHA~Xj&u9(Mq3>OSHtFW<0*)0h( z(5diG@wRbx(}}&qj$z7gh`jUU0P|t;XEAq02lQ;3xt+QGb#G#HG7er6PAgkIr$xe`2Fzc?|FC8Ya5StIHg$;halq zn8N#$cH~ZB-t&O#t}9Me$r(Lzfd=|f^LXLC3VYQpYuKw)>%B^HppN*)GkX=C=W+fa zbJBd@-(L|Nv1t$M;O>Y`XCr);oyB}U3`}B^qyZgjnYR~V!$n}3d)$`MH*gQGVHD@V z+zpI=V?@S)cIx#-`sC~4N+?JArqa=q{I|fMGt*Q;AL6+{6^hJeMLy=)A@$bXG&#+F>m%AKEbtwyCTGk+M zxC4Ei%9Rd%zgqCYh1^EVGYrROWf+???`OVdJ5M3cl=E0Cv;SA$C6~R+H|+3(B(Xcjs=%R!t-MbXxF!7w6%P^krF=YHI8U z-m}keWuZ%uDmtC^z22Hk!t_~5rfOnuI}@FV8M2=4=gv%-2Y-snufg9)UzQA1O(&?c zY(?^b)Vl`G5?f`P=&MYX+O?5#BK;vej&~sZSFEokc%B-m|65a9YVm8>SpRad|D=Rx z`b(xcr5|(!TTnLs4>ge&!Z%HLOJod6`C2q@RF-8rST9R6IHN$@}8BO~Mu0>5)^vo}>0q{4F? zOHobl@C6tvt-H5A+br)DUw3O^q4gg2tJ=6@cx!bLIvhQ_s>Hv&n0sSAyRyV@Ow?&w zxno#fIEgd~UC4N5Q_g|%_2-$3moDzk zg}qq*lSlrGziL1K_Z!w={2~Y8u7=1a4@%wH=$iU~SG8iV0eD0QV?#T-{C!mcVdh)5;ycN++4i#zHvo6YsMAsTsh9SvwGZ(NotRbd(B^L8=2{pU$XP+Zl9Jfx&jG= zCH?{UxvB@`Tr)M`7I^A>)=%r0|2jS`5F`H{^;u6m%K86qxo=Y}u%`#aU-c7gmj6Uq z-S}J3HI(wiU&XzMC!bK(=klLB`to1=Ros(!l6#3Cweoa#B_Uo zUf1SA?}RWf;80VSso^EdO2fQ>+{@5u+oA8V*s2D0g*_?xFXsv8vSIig>Vv+aGiB3h zSx>hvi_mH8_P~2end+4tAWvHYM;wBimi?i9+Xt<1vOn0`=n@4?9pr;eqgJ0qH9WNwabY@5Ee)_6h8QAsO zEo+GMow}>g@__SVr*bNJ1ZFAUVNJ9+Lt&o7)AL{dGbWw>_wUHR*fam$iTrb)`L{Rn z4_u9u_j=?X=Ms^BuSWi*ocZ^A{s~V|dr4u7i@rnVeqig4k!G4F{I&!CJ1%$bkvqm) zS)1vDt{t?018ZKS+!rZh18aOF{G5bY=LdZe(rw#BSoAEVox|w|IlmI#Yu%(`YaQ*c z=b!kuO>$bte}g=j{yyf&Gml)zeclVXJG@Md)i@Xne)UkIAGSs1^L}H3It9$TT{A8+ z+3Ui8_BMl0%i53}=%Sp%Y3Tbw|3rqp5*WlifLkSF67}w+3;nlXZ(Da-`2R}M1lLZq zI({v4-`%wce3i8@*K%69xS9WO==bg4pbjbXui8xx{8_t6fmd-yfFDBtenh*Y_~+fi z^X`(nZrqV-4BwHO3STJv?c?14-HS~K@jr(DThAYM_K@?3)e*UCqqBeaDEe;KchP0x zZS~eE=uTy*83n)skCeY~ike->zYNy?4BVu?$d@lSDjFwalQ&W)IsAiLMvfOPFqHJ0y~RefZAv)${?f#AMF4cte4EF0F+jXdN3WDRnr`2T|nGZ$V&ZLK7Sr|s1~C)(tNXRhiU>kg$qb4 zwm_$;$|b=YzW&Lno_9Do|NA(UL7psEL!gKNuU24d3Y!1TzT0}?n*lH zg2mWyiBU6hsb6$UC!D7$97A~D0XTRwr@-rS*TuJTb`QOl`+()}1|oC32zw_Dr(!EO zpP9X6P@w+VrMEUC>&$;WzhUmPOD}3Z3NDLX=lRaQT_rWE*J_&)@OeRJKKIUTIh709 zx~o@G-7(%;mAT?FygD506XUlQ_V%BgpsWL3vHpWY(09CNN%M&j-J>g=Y2_P6aIRJP zPi^aAK|W)PKQ~`(byW_i_wscXk972nl<3HNw%eg0dG_`t_u9O@ayvB>{hv8z=X#;N&XoD%+Y2;;sO+h*dHgHwya zsd8{?zL8KqEx*oF2~J%BPCc0Tpoa(G3O*1D)sEa!_Vw5#XmUd*7m4dew-Hyjqk5#dXvsva6TfIltQC(O zTjb4dKX1H~5B|_BGekEHUEaSJ+h^H-cUsTVXZ9IjdSDdye-Ga-d>wNEe&G_rN6T$K zPwZz5B^(O%<9Ilcudn|^52Hob&HcGDWCDx4GB3pj!z#}r?$4cTjxM6FfdzX=?(&bY z6A;XaG37m*U@>kI?{p;LF68UOzZ%>G+>N-b3&EYZPJu_>+mpAXll1$PwmpyjfxCh4 z_$@R2C*c`OGgZNj?8nB+o{Ud)X&3yOwE?;%d!ober@~9#$iLjhb?aM?c47(_wRI?3&77E=>BC5 zPzA>-eO7R}@06x}oPR<4u)S)Z5#}hFmyGRx)~yw^ zPuhQ&akk&bOARd5|2%|KNRtt`i?HTdr(Iya)xsLi+>5LWslZYStXH$1waL3T$ZX(y zSeucP3eSAt0&^324!_NQ?k7|H$2S`OcHXNy@{#g?u<6?#! z#CLyK7n8C0lrl8WuicbDCvE}f?w`!}S#qZ@SS;t}y}FJ-w}pPo`gnZ)Bxuoe|H0yk z{{4e}meBbK&0p|O(eX*r_jKDiJGI*?Ha1cMBeAhz;}&)z2|t2;7@@!SGDn2A=J}MF z&?@hws^lrv+CI$lReuZ3o@UI~bhU#yI<3(5uVU^_tG4}K=KZwow*OV}GY5nYFM>Wy zV}CDxH*&0L%pvjn#ox>J?_xj4z3t>BgEoDIUPtPZH0O*D*Pr`!@nM*1Pkc@OFq}c| zC7!eVFzm>w&K)0aPb2oY&zV2m#zt(~oGbom)}si%2_J%-bhT1hr|n=41s75@{qh8E z4&%v9+RX^a{jo@1IX^lvU-`F_Kl?8O%~;aN9!lo=b>V!o#bu7n#Fe!teBQp=>lOWR zp~0@M1+1Y$8$&yLpwsou=j|^xBh%sD33O}rcJAzKKwhucOL&SBo2S?L=c ziq-#r@)Qo*eTJ7%yyfZ0TjlmWHWhv%kNc)#-=z|Mq6l1;w;no}^Fv&`(*WKJFClsf z!b>cHm)I8OC3<_Gsegm}wAlX6{|sK@jincBUPAVHQpQ6S_da$pbEh|D$i36@Z1^7L zPXp_{g3o!*nBN?tESW#g721B8KhJ@`@?Yl9bKt7@W&S(|UW#Al&vW3Q_+|b)2fm44 z=FfBBmiW2Xr{_;!@3ZxZ%WiMZ$0lO_B@J^Emkn!<)Ft!hobloMW&WHqJ`9u0pL52C z;gtDv&iF9wGJnn)A8wD#pL52C+gQu|IamCzm_LGpjr5c3x4%i(N$eJYpIiPLebWib zQSPl;>4}HuonXXOjN-1mtp9RXJ_{U?eK2Lhk9$q`0idb z46#uJ{<-%TaW}l`F$1|Z_}I?{4nB0T+#l~1=)^8)JG`~W`io~Qm{c)2B^ zP{G56K5H9$L>D~sC#(sdGXG^Bz&9UkWsmY4^WZsbCubUc^t+?74_>mota%%}J-^?)vtnd(UorWZuf!bg6e!{gh(tHo-5ATWFXA=mR|l?L}6@AXTvsnCXi* z-QoUV@6s?JWvs~(zvWC+qA5%sLd$V@-PNF9w=Vrr+ z>x-UzCUz^)N1Oe6#T8?3qbw=+^;Ou=2$%P4;r!gL;d_3P4d0$9_++j}l{fqq%EKm41ja~t8ou8}!FRCa41E2OF-PEoHhc-b*g|Uo zHhj(rA8}uTFMp~H-vap`=|AXMG<;W4UgFolH@z^b|E@X%pFavdXwR46dz|u8z6L(x zz69U%SKII%jDk;SRWyA6ro8@N1K)q5;5#^D?CYO|?C~92r2DVrlhEvEp=|#MQles^&VpF}$Pd9jUJL`3%pWAtpNcKVHg$`}!w-sEx zZI5s4ZOrZ4Qfz#ZxjlXP;%2EYhkbc9pOhhW?PUHQ2+!H7cfUGkhu)3OV|dQ)dOtM# z&;N&aS~Q(y&&2-i|7>$aF^bL4t# zj`W9Kd!U~Kpr6ap(aNCzgl`@R?cB+cQF>#x*x#6G8UIv>Rn-*xg z@EEkU-$BY2y1JSE;ytK-VuNGaMLz3OK6o)l zuP%{~9Mt;;^dH3rEOF;^_rC!<8dbo&Fv^a`!E0^%8Q9_|&ta{u<|_im`Ko8vApGz^ zfxj`vVr$IDTB>aZ;EKKFX!`u(U0OE1#f1&`jz0GOwvjhM<$l%(`sfZvnt3OEJexjF zrH|9-WAxs3r_;w`*Tn;UMlQN#9BD73uchykqx$-$gQ3}rkh^BH&Mm%)cdCaJ3;oso zEPae@M$_V+(8o`7|MsDOdk1=l`?vL8-;2n4%=e5w<#I1#2K_s9MN|Fj*fe?L52*uo zFL=5BKw(S0zI$yXQjAj_u2R9v&QSr^j9tG-VE}}dMx#5t;6Wo!u$^2YN{eQW)ss16_^l(e+fJbT5bK_g;53uHJp-r#fn=tFWu{N%JS}60| z-4z7>M`+XQH!NX}pIMhbqplF^@;=J!1W)$?7whm$12{s(J>Zwp(1%S&AGSZ|?jFwH z2cVCTfj&YHvxIZkGjdr^x1Q6S7w00E5U_+Qr|vA6KmcLIXQ$#go-dG^m&E$>fIE&GNb=uB`@}tu=9cv3xO`Cx~r#?)d}QygBQxbk^tcbq>1%TcHVEVyCw@QEV^tV%;$UmDpk^W6k1j zY{hG-4IbfBFM|Hb8-C@NY^uL?>B8nIM$aMWbU^wQ=|D>rS zzXFbrsJj~(med(^(%;nA&Ru1ZVQKrEL(omV4_#g4S<=VS*V6CO|E;XWGTxsKV+`W3 zD}d~(gMOcaoIJo;5pT^{mCV5+=3pNB$M3nZ+r-|j<4SECpeX=!gGFY+7wk$4eWx~n>#`yF9C-- z=AeU*96kVVDf#l?yCUt}0iPx9JWPK>-}HT>)AX(B%o}`7O~x9*#qgZl{U!QFJ947( zi>zlqZ(SjST%EmTu=(IOPCm% zmOs)K;4KjNo&fH<`L5?H;7ihTEGf@jxeJ=Y`$6E&sujEHCleQq54t@~gUs#Wv&Z2Y zUTN>SWpEMs!3p-j8I`X3`V}tT983yq2L^dhO=Qhm&^zn^R)MQ5G=Fay`-nY#4;_q^ITH3~UZ zK{j!s4`#muC-<;Cz=y2{;Oz!R(J2^38nMf9CXIb3O4585PIED7#IEh>H0<5?gQHr1 zDxBtg(uhn(7?CHQU2+4QEeX(PABz7+I}fxoz8cNF+RKPM!4<&AU$;N zt8M@51!vkG*3Ef{v?3RlwdF82W47yXQec~Q(*v()*Aw`Sb_WF3Yd1M?O%&Y{x~K8w zZ|9rqxpzH@xX4^B{WK)!E_$v$P2SsyJ6rGb5OXl{_KBQjDbazrbC_8sbSj&Wa~d16746LB z3F)e0J#NWKdybdk<}&tCIvc0yZhghq&+%01i=N~C2uIKH-gfvj9TwPx7Dr$&WetqL zK8m!*7{l6_xEb)+tG(3)`feNV=)kMR<=>p^JAQsTHmUeHhdz!x^>`KI6si%O*Pg6T zy;!Gu)8BpQ_rB~&xTjYf!<}%kK^v1-_TzWzOBlzJv)9z~FVi$s&PJenZ^pi8BN4J^}v=_=ODX?|cd zKYPJQ-cU2fzaIOz8*BP^YkUZfG5 zn_k2osHn|rZhgtt$@-9br4Bt7{jbz#pwr>gh0eT3+!5j!OK3q#;O~SF;d1XHS*JHA zS!svv-K+1%((WKMQRWwRh?rBd1<{@SAy`LciCe@At_FrQb7dgKyl&dQ&X$;MP-z14Qm&_ z(Q`nhGroN3hzC6rfM?_b4W7l0^r^gsQ9e7r-qYz!&&#_vzP<~-c0T)nT;Tm|G&USZ z^!Dc5n_MsBU1YfGe#w|8{?XPktpujFp0-Z(So+pQ`vTyA*cq1dunuR+)J&(l+>dS} z^H!fp$sVdF?~XU#z2dQ6D!y(gZ-hSz{0AJp>Jps^3E8{}JSbizUNtYjd-}ZW?yKhw z>K@u#C0;XcXm@#{=-i~%H2_~B{AVV*7_IRAf>%;UJmuwiUG)i0SHhHeL%QW$fcH=8 z1Sc0FC){)v&DOYm=lIiSl5W}~dh)kWp13#TB2S;$$=Iyxvy?e;eYe==jEv2{r2ULd z0j`XPAN-iW`xU&wYo}*CR>IfMrjM_qkB>Ord5eCWQr}4*PhfmpLdWUXLE(O#NL?vE zuyHDvGL`0IRA3)CaQDejfy^%#@3EIGAKCrhx!UsE%TK>={nqn#TaL_{=s!A`^%Z;? z4($8qUE$X_Py24AT-^uBqs_mP{s{Y>W29?~qCfpvM`UdtOWH>ISUr!v0QYrH)F?k| zvZZC@{g7qy~NsZipU$%)>s+UE7}&C#F9xTcI`5M&Z3*Bu?mr;GgLB3jD6>5u%r>ZHy!L zJs1Q1&%(C1UN7GnZQem1eMbmh!)18#N?dA`rZdpN!*APtYSTb$RM2jrFHez2`cn5# z1vUifpI-E-#7UnE-_ARKO>41bBV#p4@F?78IixG)PBb#&)o(dQnA^j=n}akTqN}kz zk9SMMdG<+os2wgM{y!4F#17v?_+1H)vcoG0za`;I?eGS|Z%X)bJG_JN>k=-q!}|!o zMp)_^8ixILU>E-H5&BPPBMJh^7JGK#_fx<(Y2Ww5X`W<0NFT^}%X%$w^NADw`!V89 z>Nwt~564|koY49@;&OSa_}?yJJ(d}PBI3&>tj97Va1-HMB&^3WBe0V2LJ8}!%m{2CJYT|kEHf~7P&IR& zg!NdW^Fw$JVcj1ZAK8~BaX$w5q%DWG7;E}6mh|fu(bpG!Bk3DiUuE155-;uh7+2;) z8*~JI??Cpne7zn&#D8h;f3%w(_?LDE1m4wd|G+!A7WD}({XP6|uXa-cd$gMzctg8M zfxl`u9XY|dZI(hW%on!smOzaY{>%k0HkCG_*LBNe_`VYOKBrkhALu&a^`?;bOT1nUyxu}%kIX0G_1-7Gu|{~k z)N}HBX<=S(GbZRj|x zV=X`Vk<)L9+z@^2W60^np37wPh{QH_Bm7n;a!+LSK-%XtBm=_?ySc zb28IsHPZ)S{w9hxr3VbsR=r^JHqs|S_#5^XHeLGpXib;CPZ?!`6Yw|B3k=k?kGdWd zoB~$Clee`$J@C?fcH3SeowUo2PYsOLc$OM?n*a8{l)&ew^Fz1rpTYUGfR!UWQ2)Te zaJgc0{3~?g=OfK*=Dp~}eI@SZQRV_@h>Y0`>Xh+0`BAZ7`t*MU2H?L>!@@eC-Tr|p z+|S{qzV(hnaoirUA9E6U;xXu>>|uGkPSeK|@K>@fYPuJejlQr1*(mP=h&=db$gY<2 z-N1J#A2zuQj`4Q(IFHziTC3jngl8PLiR=9m{y^ikHRdqqa+3FPYd)UKo8*-mJMz#wy0?A(fdwn3~5(c`mD?#djv z&|DDSpLaZK%gRseE%LVk*FfDC?t_R+nI3WJlXUGS2fiS^@Sf72Lf=Bgao?)j7<4A( z9pSx`ki)1j*yA-g9TnTLrJf6as4SmVE$i@aN15M2pH})kj=BWT#TIXCB67Foylp75 zA3yJ#?0jHcEp-PURV^-G)y%^=ZhuD)IiHO8ccNPz{Ed1gw2k}K+?Q_ttxdx|2WH{B z)nL^mb9VWk?EfG0-{yJJ0-rJ$_w6^Ba~T0Aayrqk`iGv&0|Ik!WsVoq9hV;PMdzdc?;~!Lj)TW45*a>1 zU&{u@zNl?5ZA=PGe8sjGA!*ms4~IGT`P*WM+;QBSS`QD;J@+$rgJmwbxEtl3 zP&IQgaZ`+Z)8+NnZ=~FX1-YimcJRh%oVg3z;-%>5klyNoXDUP9aPsPbUU-n*SzhjG zRjN~~XlF-GUw}IV914GRJ5+HrQxsdqSP+$Vu|MfFYQZ>_0S%|8tBM=;I~9 z`9{lO@UEc-dxOl05OXIfhBve+Pwy+AG4y++f}a|vp^<-}U9}-2(RFW+S#EfV24~-j zylUQfl)F-&I7~OL_hE-w8SDEz%cxyDi@dHZaE!8#(btFO87)WA3o0D1l2T0@cfLcP z2^~N()|5;7j$ynbu(5}~7XRUmE`Qf^ZolZy>oIx~ymi)%3xcP?!9D$LEAJoIbqC|v z1+IPuuI@F=USV9ll5(s1fu}vND;t)16!A~y!x84a=$RLTA2QZL%bmS#Tpg-$#uIpk zakp`ewE>s~6wvLIy7g7&+D0mw~TIu_Zn$}PU z<12A5=`xb?jx(Nsq05vSMC z>|6DF0`1H*9Zu9X%nD>bJgwfI2lY}WxG8l$ieKU)Hhq_FHj?$6>}!}aiHpb}-0;fY z?2F7?x6vZ=HjlSR(9aqvWmN0(l5}~?&noYGlozT|dm?X%1mOYvyq*5hVC9XYoR2tP z;;zLSsh@QEEH+deyQd?kH@52fu=nW89wd>yNI&)@N$j(d(ani9g`YUa8Rb#RI}A^O zOkDF6C)jhHgr|t?2_omM|67iI1J35LYcX>vJmighm+^(q$O`@jU-2^Ei@P7sAaGIm-$ZG<<|#aZ!Qp>C`In;@zy5(i;eVIu zyiWgJ@EF2hcN=k!pwo2}KIHBdDew##fdS;rbip%#6Y|c}R^C{aH9kfD ztCTJ$|57Sf?0tzhm4(OXqyO9KbHD|8A3d0$td}&;(m&86Dn86_td>2v(-2%^PkGX5 zR=aF|@UPFv9gM1(#{>_7&!O9deu-|c&-#LRncF9|%lRg*@Eejx@b=X5*^J#R#&9NM zIfF61hW+H#$nvLSBPCXkd8NZqf$pr}>~7b3Tdzp#LyG(h9m&x%CpL-Tg!V9i@8(QLO&jGS+S-{T($PmwKmTG2bhGP$YRX9XZ7<2 zooRXQzO-p@lDA`1vA1(@ve*5OAMK6zLE|>X`(3Pm!A%MNBey{F_d@gEhUSlOn5%zn zm-iaw>9vIY;bG>0ymcgPo$&`deh2Zw*B)j)lKWUYp&R>ui9BLU*7L37p$W2o_%goqjny*c zJCQd>aAgSTq)((xgYoOJtw2wZ^1bINv*1)QIGi3pAELnV|MB+b@ljRx|NottKqiDe z`=SX3B>^o(P^4g)5R?E`DxfIXf(2_7`>{wD6*0lk62vM4s0g+!!J4_03L2D%EhzR= zTxwO++G;;$p>-y43kl0WXui+axp%^V*7ouHx}cpeV(Ahe=_aYgope`>)VC&f5U_L-hcz+?oXt@`9EW( zvDd$*F)roMW5ExDTW#>dD}jlQT|rt({p*c=I1w6@4h{PE{(*(Wg;M;)4(QNPmkvdt zL)z1zxHO8Be1iWz15GjgRpi)vwa44^g{$xUJUF9Q#UQ@cFgGt0Ip0 z0OqfO-Rcj1vh@^A&SqQ)zI4&0^qa;=&)4BsdJ#5;6fVwht$m-1J<}u2|6X8TZKLnI zTRS_DyZ!j??eAt?r1zdDE@M?<0(&+Sh85tC^r1Ivej|E_@;0Pcw=dy7l*d`n*oXxy z>}U07o?Y}z0pIA(UQhgjz@_Ph1M<(lfYj>ywX^fNe+{}6L8cO|YNmhlto&BdDviZI zdHx18tF%0Qz4V?E(@yvZaQPJNzbCXl&+j+(tis|kE0T!yAioj)UwgNx%V~SJ8b;~6 zir4YwzCMwHYoIrdzt8b0&bf_|S@@C9#eWIg%3OR%H?TiMF~T>n7Nor$E6Te~yQSOp z$&>EPn|49Dz5YH=>Z@8a^OJXzh8$Ym5+7dbE*{VA- znK(L(TRpTu{V;I1LnG>dNtaPyEg1wB#*IOI{M z?RQaD-!<@Ci-PC6@E-veT{ZBYgM)c4{C|Mgk9EJZeQ+?#g_j8qZ8GJ*&DoVM{CmLF zXNO0l4-{Ip?{FO^&ghTN#eT|_&&B#wbmKJdn>lq(ov!8`jfdXbTfn^8cg-&LE!UM5 zSR zR`i?j;LRl#G%CZcaI{h(FHZW8OJzpRJzc13z}a)yy8-GFS)o+#icbL{WPRK-L<~dU25ZM zw~Yz3QEOfL|1q1f0hby_BUhNV`lwg7?L&`HTR$~*BYy(81Yc7d|LVey4t4mTY+B=i zABdG5Nm=SW57_I#oIY~)GcN`9GBEW~{m=?@Y^Q(fM^;>VWFa~WebiJU9W2`(+tp9m zrk`HrS?!P=^%2^l`|rAK`ZMit_(!(A&$J00gL$R@tM4`bu{PD2dXhg)^Dfhjo8)$t z(b&h@vmpi>PPq)HJ?nru?NQk@TkUyB<%Y{%;I^d?{r`pbq}a(_?Q#6O9_eaNiapnD z&tSJb>uG~I z@o-^ReKPE^)I;&#!-FPFX}FMLKi^eXr|(L;>Y8GwQWtD6E5{uh z0^Wch@aLvo#2}#m_Tt|nzM6E^{82X(gQST6C0#XSR7^IqkfrwngHJ{`Xb;ue`>#C1<%K3KaY&=L00FCibcd_+@~`r z9NkLmSy~T|`G3|UUs)agpr^vo5q#FYQzzzFYy5fEoA}c1kS)~zn_CZT@N(V+_eWW` zdY8+Q0g*#4o4RWv{g?F4Wl{fMFCz|HQp;X%>G4g*M!~n&CIz3OPV{lDXoBd<6@g_%q`y1~Ch{233XyRRUR!ZK> zK;FwlKgr@d*?cdD@3I$U(OInRB?h&wcQj76b9GGizB+!e31#@f;_oT`QHR{MmW%lE zBW7?RgCSd`%x^@t`X&E%czmY)OU6U(cm|lmSF-H2E^G%d@uShmJh@!QeHnwxWKV|Q zr7q4!&&#wQqHH^SQTFF1`YwFW&h@G2F48w2AW!gnk;h^?Wz-F~hHiQWFx!Lr|X6ij;aruG4$XMm_Ps0c2I6g3C-fXjmHqp0t zW>LSrGn3DKIeU*c8^4(}d*_AgvdeVd_MftIh_y1mXvag@{&UlE%JHXfX02=bj;ieC z_uR9(>Yl}`Tab|ij}%zZ!W~uXY^Cgz@3NJyoV)RBL5ux` zo*TLc7r%)u;YjC*r)Li?%1^NRY^+S{sG>bJ(3kKN6GEZti}CBqIM$BbQ#BwNu}%-qVdCDW;FBQz!n-9fgrFmxqm zOPqvXL#*xIiNsYHuDa&h<7s0n?|$#DyI1L1?>(-6{J9jnZ(y~5amxBIdPO1StX}Zo z$`xatt4u*oSu>`Qy%g02;+47f9N{cuqPEJ`y_@z%pksSDn`IK~o5%8y*``=^Mph}H zUVL{jW8GY08QW7p-{#EpLi%3y*EbjOO{MSGP#^eO?0cLIGyNEIp^bmFV{1B2nGWI4 zU3t^OCG1`Dq$Ow3zxi7O(VSlIWFN&Jd@tYfC-~pY^Z0i}pqY}9A_d7Ie*hcvu!?od z%LA(wvsyBd+50n$vn%nR<4mS?!h`U9@E<&gRQC#nf6D!oK;6-Tvx~ym(BN<$8BRXB;j(IKhcXWbmyN3I7sF&LHH2tnV)V(j+k)M7AO!k;ucRZEXh&&;_ zw+z@(#xzU}(3yp9J^ri4CXxiuF4t_1zPFaITiGk?JXJUGO zu6-Tfm7aebnO6B%^L%7sap(m7^S|)Dj)$E&b}jJy_d3y2oO$LyT%!GsJ2XG@Wkba` z5Zfo`wx$OQvpmG#aqwHQic+YDpE+RiUq}y*O|qyr*8op6(w%qM`=ovRE#Op7_KFfy zELy&t>rF1cHQ$pIyuY7w{)qm=8I~*Zh!dv0$(r+i{wX@ygR+t>BK)%l-#;w-gin4b z-J!pU&|yFHn6-;3J>XTz@NI8UXEhymeb*|)t8<42|L+M$?)$KNV8MK~O?|#(a4_jj z_KaaipzlZ3;L~QxZyy?bi}D-czbd~X-#yC)e8b0~bgNYtn4*|Q=z!DA zdhX8I;O6UmE5P$?bn92ShaPOtUwt{g0rwhP)IsQQDenag9{cSVDHEUnf%17j!28-S z@tnR#dCn{`{|U3+BpNfSBqNybbN*ilzJ@th4LRx!>-^;4jSo5aSj)SGl*#uw-+F>) zQ)X9k@Orn*R+ZuXeDFd4`xVdnZh$o?!TqEUGAHvYz%BOhK>E-;;4jw`&P)ICH(c$;`#%Itk{HD$Le#W!v^@&MS?E@xF zu|E=x+b0^A1byiNok@n?^n~us|i zdUvb#XgA_UvnRQZYaQ2a{^jWP8TM}4D13YynCOb?^o$GpGcb+W+t9+DT-TS^A$xNc z7WsMp+!mi`R{5^=T3^XAvUj@ur1GqsmhZ_X8b{A6M2p80x7npYuGLb_w z`&zZ)^+En48C3TJz!~+!Dqzw_L?^sE0@2mrYGaA!a*pj|P9-p>?g6HL=m6(5XGfWK zC}v-(nX}(F`RR7Mo3D3eL(sgKO`ch2IO(jx=7ZbO0w4PSQmwzPWF_#@`bukCV-s_- zQgbrL7C%yYBXtvv9#t?rIGXVgz0{dQOW-AiP2;2G+(YB1tL$O-ovV3A?Th^0Y2Qqq z9XnNyeI@s%ojWz(v+ZlU+KR2pZEK0!))(Bi$Y)VyV{IMcwzaOSt&UAfZ5>EnmAlq_ zRBe=figwm>mWbroQ*F$)A2s#Ow$GGNBDw(1vVi1MwU7b;oYutPLN=enC7bqqmDru)v89fRDW9WRY|pt+!@t zJ!X4MkMBQ^^%%~Ql+3bW)to~fbcix}!)2zUc z@i1q69%XN`8ISjd1y5+saCzXR$@DXGDhf^5KJM;h;64NI2Hbq>{tJUYew=&8y+LEa z9xCL`5h|y5r8{>phN^20^Y;T}wjR)jG1M~-HexsO&H3==fZrOi26}IiKF?m%2A{{A zdy&CE>QkI^FT_teeby-9vf_WrH__SeoXYvyX~gAEH|L07#u~lo>w3PO#<#^oj&P1j z?3-S`*-%;RoOknf!0$Czn7I2jmV9X7Y_itcBm? z@+h6tqP;9dlIIw!BHoo=5Wt@F_7-T?e^KuY>iuQk@E~KKTr>-unF;P(2M%2eF3muW zy#_hfv15n_eA&AJ-mNz{*c}|~&wDAnpUQXA_+C0Vm;nxU!(TZOTyyqNcYuT1Q$ma$ zbkQ#Vgsy#wKc0)tnZ27@JKeAvoFisK8Dn8^F45ZAm%b5SVsl)r#;OnaAUa(LT)gQ~ za7H$tBo}^;fp-g~vA^|fa75qLdMxQ%f5q-~xSTxs8w|xp$v3NxVpkeO4BW*z>;J|W z>mUE(8}coXeMbA4m=AC1S?hFul|SGMrBeQ{d`JI2WXh+uv=;=Tn;pLQMw}eK-MXV^ zaFgp(u#53io$C4jr`*dn`LR-X$6F?q-@pv|^BVf~YWjCN{XC8FnacQdH+qIsALoCH zVH5Upz6R&3F7GTpP?UE4iXztH3KnOsSA6O)dut*&td%j>wKvU?UnF0q8`|)FY`TX3 z^KapKAjzuTxZ3mPnx`E2>#CEV{o{$mb!KcHJG*dXO-W!BXMK!d?yZdZEWFJ59P(Gt z86Fy!jpAi%2L(N})ys8rw&O4P9N(LhZPkj$76R`m=vkyV9^ntWhmK;;E0vrezq=Ca zWEJ^}%`9J$Fus$_o9Hog8GP%W@>?4Zh4=M1*7+>{3C!1Fu3dC9-gn?L_Xy(}xk4nz?)WIIiqv&JK8hIiNA5VFtg5 z2bWsUnvkJoRzlFjxxVAb_d)w!OtaQ(NL+Mi6Ma|ivs%hXH~8*6^u6Sg<86baqa#&w%z$JH_S-zZwrCz$`&GWN7#U+0_B>jw}Ed|0y6mp9SYjeV@zwgRh8 zc`CP@|8~sn$(g=Y>lA0Wt3O9tCreq|2xIGW`a3anl)hyi7u8R(PO6XPmwK3e9mj6V zEHeI!H<3?Y)U!_WSJ&rnQdj@x+E>K)uj|`4@=N;n9AMuV?DX%G%v1I6mGrOtA-ix0 z+`)#hpgE&r>Kxysmec<3_ zS{GhMfH`>S=oZ3D$>iV2bo%WEx8Hv6(lmX42Yr~r94x!=R`^(&UBg<;7jX9Akf7Fw z1`x~O9MZMaMP+|<;rXLPuQU&ZFaHrwR$FzQ8;p<9Q{j@NW15>0&C5YOLXknqp$Pu2 z*iYY>6r5N$8@uZy@{8%~WBaVS$e(pC2J_mV983X6RS)UKCsM67c_o3?vU@AC)9_a{ zb+|xtzh?(?X}eW+VNkl(yq`O?h%Cc?= zi-8??>3t8kZx#Dk^IGSgN_RFi0{Sdjb^&GZIWhEJ{6RP@9i^)r^n4!WZlavt^X5DH zcMiCZ4F;d0;sZ7KlNM4&HsXzyYsZA{39RnHoU{jFH^+}*v(Nixb*0v`Q|x!ZNzn%R zTMnb0oAzC|t5NdP=e?n4#NUBF8$2WbdNZaO_DqxJ*h9bz)l2bt^exrDIIjM$e?|RA z#_De^HTHuRp8cJw|K$gY_)e(cl28ag9y39{;QIb=Ei3KDABb7aN;L zzR)(xxnpCQwxuy{PTOjE)^}sRzY)IGj7~M1eW{`wW#{c*m(+Rrfto>=7+)~?FHHoG z^WAozK4xilnZ}HHDV%)?AC^TrdmNdeYwXtc3AT~1F!KUY1V1xE`+-!trG z^!1rC$L+w*q(9#Q7NbAS^q=fcqCM)zQQS+$I|^=A`ZBP&FqST^PvN~sT{`pu?}-Mf zto&zv9=EJJ?&B%DM{J=JjO1_B-b5 zVKd&&{7#A=?{D|z?00AG|7H8BIX;|zQeDg#!FPu8EWEhDr0Mn$($Uz;C|8AMu*FW^gIL9NnV< z{i^0)=(^aab#}U6aU7Eltn;57cR;q4&8td}!(*qb-iN7g8M;6!Yjj;}(8$@+ojxV6 z27hGv@M+(NzOR1&cmBXJUtso*gx=mnzZm+KVZY6HB%6N%OgfOh{bv^z2Bz<7&sHsV z?{?<*9MUp$0Ds!76-np-g^S-Y@$6(1QrQ*Yiqb0f%F6CtusF@=2sPNe+gPj7K8{m* zg7#!$J|gmhYBwc|2>^#dv*g9Iv-htTlVa z1zM{rE3!|_=v|Z>$fM{4$5?wjF+a7aeOk}Z;rZD`eE+RN zYww{#tNu{$AHVugKdDiXWHkw{>%>p zmkgxXRBM?}@~<^Axhp$V0@K_bPTK@;gg*@A{s+K?2Pc>dk_Aj2b3**+`z8)?Q~ zcX2>Bn_{Ke{KhrNz{LOdrSIdJ?XVi|4W z8tBF~I!GEoAHNa1#)jeeTvPuI#K@5>(7+nM6U)kH*XJ?T)af{N3H!PY_AhqmvA>sp zMGo^9!k)S(*ScNy$U^q8$);do3+l+QLfFYx4tT6@aQQ~gs-oPNeYfWBZqxK_|A5Pq zbAFIJ?RS)|hfm4&@H@s+wB=o3W(^Advde|N2~4yU=k@4yUgEuSmCEh`UCP3)p!auD zHVW;@v$9&W&!UWd_eY?ASy~6+TieK2-Vx@p*RhG1`unIqv}2{UyqxRFIRcfTBsAo%|%H=LHVkpfP1^2ax6!a6Tn8 zwn92i>G{}TJ$5ngNEVvI8tB-}z#$JbR^|5h5v}#u8@#;3nc;bTIG+(aK(;68mYAM@ zF*<%8_Ldhk?zF!T;~N|QuKs(ie{cZ(x1BkeNtqXs`IYV=)x7?+pW=fGZ*y(=^6DIb zqd#`)v@zMj&YI47XpU@jwOiNv9`F&~IRY&nP#1ge$*p$;vJkD zZNqlb=<~8JBOZ}$-wKYaox+LzwAGb)SB@K)3J&C&zVD#D?=gPi^uVf&l)x&^I5Pf6 z{5r*$G_&2zu(ATmIz!O#8Y2R-z~#_rD8zq`S~iTdWw zS|i75>}Q`o_QJvN8RK6i7(7@wz@D{`_WB+L2TJ34G`P@ia3R&6=;Fe2{7-n$!Q4)?V+^ejzr9a+jDKc)9{8>CuT&o6?i~p}gnNt5 zjyn*Z>kmn%jhr#gku%2m5cn(mtj77u_;G$aB{-XY)qFPnf7Mw||A%r<@BfoI+ONgg zGRwPcPv>G66wUbw{UKi?(TaTLpTZ9uGzN`x zbDg>O(^LH|d;7=FI`r#1XY~1Z>65NLckQARft}gsBh+{F`HSGWBjcQ!b4G5b&v(b? zxpVHkSf5*tp8YqT_j{fFwf;f-*G2wyIX3q*`W{_FbL`B%7fp`!{b#hNYmVKaIhM*C zOSONut@9Ddc0bZsFqdQx^TY28U7p!RGtn!LKtC>`f3<%!W)nN`E6oA;0psvv^vlEG zj^?$-RlXMQ{fM=xo~#*=?@tS?I?5dGgnl=oZ7}oY%@n-XUQ6v~ zXoHsiw`1AHKicBQlXl(Zj$;e;bZBU<(Fx8R!}l1AGskeBXcS|3Rs1-)^X7TwF@}@k z^V~5Me~;;nYU?>hcZ6OuhrZ8v`d!`e=6^Z5qsCMFeGI*H$MVP-^C&-VEbopV%hyhg zvl1{DhJ%s8Q|H-n(4@y z8TKIFDT;PJGKDtP@GY(7ZsU6&(PoQ%a{6Wm`pbJ<&vJz;Z+I%Q?1rbpmY=-@oB_ky zLhQerh^eFhma&F54cTp-g>HDyAB~)}33=#c_R&5JtZ=ad^8kw=-+ffTS>5!Lqi<%| z!@+0a^%cMzTUDn0H5WF)z}V00wQ5uV(zFGmg_3 z>-Rji$TJH$AfwX;&AAJ1HohJ0Of&%35SmLNBlDSi|>arHgvMkm%fOqaZ9c)WDK zEOXB4qu8?4CyD%5vhzm#NsiCzdrDrK;OevZCZ2$XAAcI#IdMLa$FQ9z8{2u9*ja7( z^tg8O?PuD}k2GO7C*Hzd$za&c)8cgL3;I}VK17$U!ET=I=+SRj#%|tQcJq~v-8|cV zxWcLD-|08$xO<^T^=+QuFSk1IzW`Ui&k-Eme~#;`^(Ju9uoKCWckn-9-b(J#JdDX} z(j`P|w6-946!|*lv-WH91oP%QJa?w(4EXSM;LiS=O#1@vHHY@2`=+D&&P3mh&L|0; z06+AuzNPhe@%L5k_Z)dW)9`i43Q@vD)^g^!29(U{Jy_e84%{~_N}o^<&inD1uU^Ob_zH=E})`zDj7 z+TS9bB)xl|{64b4=WOsg2mHaJ|I+JW~A7r&?LX#|LQ5QuZ-uN}&h28Gn!} z=t=~As8IE!Pt}$}=7Z)%F))XIWZ4BSY>dhcRk=)C^H$|Xs$6A4=xu0+*5j*`!jI6^ zYu7-t^vxW~w&zICOl_%zhP0dg|5{pRfb$h7;o@XZb_Ss7c0H}hir$gXwx zcOP@kLKOe&Y3fF77p3qEzf$n!Q_9G`s(#SepIr1^`usce{q>B&EXHCcwB@>x&P-H1 zWcF>eG*cH}7Hhreq5F`{V{7t5pf__DXRN>3oAzqC#knmR#K2peyk7f+s<>}~cJApM z6>`?-<;N)b82yxawHQ09`m2Tdhz4(|);fNyOt#r;FW>ZqZn*}xZ5L(Wx6E~zSY21M z=c8toW2<|Z_RQoe_C37r05o=C4}5a?Uu<12t+!>yFIHaV;`4iv-wFNc0Zl4hvL-v} zQmg+bw0lo+pmoxcitMoi2G4&1`WZpTONTx^Kl@zK#_iH2H~LaKN|~#l?H(7Z*)u$( zxJmZb5uxWR({}ACN#BLO^3>qs9>?Uvw{`c$q2KNv8Ty#D#b+xscm1X$Ygfm}6#R0r z5ANv^(%Mb9IyuzwD)f~&1$H4p-+L&M5t0TX-yF(x6 zjAC6e-M+Hi>DPzoFVQ>6c=gk~!K-Zt{vdG4TZf?)mh#}$8Ll0)iag2cin)IubY}Bh zA7gTA&GE;;L>nC$&4CZ(`8ME=jOL85=-W)ei1DiT8<~&NTdoEs{BHuE9od^aaC0en zSDL&m`-AvA=Fb-LCYZcT`BZhvi9QVd3a4Fkn|~W71_`^&9%dofsp*W%sqWCd!;i6v+V(VOY}HTDSdU8c}}x) zNPF>q$8FCsFFnjpFY}a`W>XTNk=>BxJi$(%|4qqMlIi}Dk{oLC5pNtCsr>?V;F6&c zx!(vrR^|8*n&-XW2(IN>yvjV6jSk|}_=xzadG0$V_`mBN{P~brhVycY?=p5B?RW8E z$H8~4J*6re`uj%A&+d|$tV^U?Bc2&u9GXttg78hyoJ8hncdPbqT)*WitYkmo5>IO> za# zSt+_RyZ_Bz>mu1r{TAneac*iD`rCPS;Ls-0)gIPQxb}NfZ2zM-tn%Y;RX@5%Xs>Kw z_vZyC(?_4qKwl@0!wvWmj>XpDV-FyHL!bC-br4%>M&z zVU6>9N#pNo9<(9*$VcAL(LdvVH^>U%ceirD$}z!a?){K>6(Nd9?-IupAv_|7Vn&5{SNhPty9x1k3m}76Mb7 zPt`fCBbVtDV|+=)9k1_lmvGT8hfFk7U2O zIm4~D-B1#UHu4Se%&DU+^B=1vab!JXX7a>$hXAj{ImH zv7vHxpUZ98Y54BGVTLhvVT7-`Kkha_X6+^-tjd3oM(AkeLPoC_DP;M zp3gVHGrt?7Z8~#%jh8*8R~4-X=Rco2Fh6KDu%~oW_nR&>M#%tOu565{Jk@YF?yObj(Yks69TX%WZG+HT_YJ#C$ATj8mTK1e=t zL*!NLk_FjC+7H_7?n|s+WSu;cLzz6vtVU-&UVfm6vvlOM)Lh*s)Lfk&iaf!&_D}Q; zZG77E=2bP0Jg`Fj3m$JQq%Rns4%*rXAE;-H8y>kl)I|Lgu#YUJe8nL4K9^j#E4;$0 zE5%=V+Y_CSs9fUH&bRK+x0oY)&*NLn2gPo?Lp+sxFYEQPv-G5|8-3k^g2nt7XH&+= z4|<2O97vrf>Ae*bv4vE;HJ<0o^$f1OP5q4Qpt3_9xneANBRVfTP=*bzwtA$oe};(H zVdNy)ZSpEk`ScXH{z$yP1DerTLJSG+V|9^CZ{!wzA6!yh2I=3-pPljl2A}4Umrve) z_rDs)8pR38i}NXo``<(SuRU?1p?R(NI=u+qjQdeOvIRQZia_07`dD@%KeXH5t!ic% zoIVH*KL$-XZ5}ph9?G{n$KHxJ=vou)*zA_+OBu<2`rl7N&U?vv zZ>}>B7r3zQf`Joi^Q|t-157-E{jSmD*l)|&A2yRK4BhmTDkjZL*6Q=QL=TJ{aqyeQ zE|1@j(TzTTRQsODhb9W&+rW1-MzRAxVVzX}g%_q_Uq}xfT?j7!l>M}OEaNj9K}QM? z$_xqT!v$HPwrcX9;C=9;Ft&dF7c1I0pYcwdysds2XYM6lv8{eW#ib3&N!(8Y)_vQN z;lz?eu4W%E-|1uK!UN#3`fxw>lP4?aS|^50+_}|a-YF*{fquxO5V?v2mYtV z=OL4BA+OxL=kSNF_n1d7k#~p5!*1cO6`T*gmg8GpcJIUM{JoqtgPM;6i(15@}C$-zifgI+>6XH?pB=xkzx0X!`)19*YC<6ii;@RP%8Y9+^bam zHkW?YyEiDMPiHHo-rq6sYp05!A2$A=y#+zU;E`sc6MT z`VMUxt`zz>jP(CxTn6bZ-{-Hsci+AX7w(Ko~^y!d8sVARc=E0;{!GHgoryrpkVTDtwveUBYDH0hq` zp<6G8p62g8RKD2LiVw*2JJ5CH?>5ns^vdAHJ=Z7QJpYmw=tNRy=S8KS^0$g^eC(}> z_?Rc{eeKZA_?u{dx5i2TyT~1v@E?EvR^hLnITWsVFq zOrf>?-Nu4o7V$obOK59+OWB{h6TM*$vS49C-J*`oY0RnH!q-uka1oAfoht{tBJ7 zk!D{AtybPX^5i$L5xs!(Bu4ngTK%Qd7V~}i#+frD(Yt5e4KF0!LHQ2STd0el|64Z~ zM;`zty3%;U3RW`@6DC(oeNMcmO};M~wqOH+?F6R!Y7OyTw{F$E+sM1Z4eJ-2LxJ7s z;6BcwNU@OhGVJ%5m(tsg=cqrj?I7h(^s%D5!6!Z6$Fq6&v@83bwv_ohfq8s6 z^Z7D#h4I+iy}^NNJ*i)*k5~5Oe5^$DZzn#wVhA3?ZW&u+k$uI?b9YbuE!QAdv8Ri< z^0~EZY%oka6i3$bTRWe_e)~6c>8H`v9^+cXbuE`-0;Y0pq)o)Asr?PK=zdbYx2-Zz zcL6a1OY?~7M?Anku{ZX&%K~)|kSaffoPkfk2>U@|qI!A{_Ir8^u1T;)KDj-OG$gEe=n(DSwy6o+0R#v|YAk5E_p7p|>b=ySE2C#@xC z7k!rOsr&TVKE%so4gEI!SqJqC9iUwwKL~9uFF)WnK8Sx>m*?qe;sHihh#n-h;O{a0 zf1UO-$B1oS+4C#)HnrynbayK>emi)KY(6!TTwFJn`ujc0v%|I`Lx%o6Fhg5oCuv;xhSkKi9dhZg=jJyh*P% z!(V4J27bXP8xJdM16Jbf*Yk6auMD(?vn)p@gjz>;4>gq}h8h#Qg>Gd2gu#LO`P7eo z$xBLVsYHj?y!ST?QW&2UC49H_|!%5zDF-tI20U#ViQMXzIixbQXN!-0 zAhGZCxjb~Yo8PQ_bb>8&9 zYo&afIJmWn+f<{J_LM25zJ*HpNBBZg+E_y>JnjB#M`u~DIvQP#Jt;2iVZopuns*(K zJLQ6cK^HW~j=He>1sf|^Ci^{|a?1o8E?AcRi3__|u%Uu+CbKD)_VHyIsfj`Yuw zo|8l15iO%V(bgwRLM>xG(H7p~!Ofz`C%DD!91 zk)$KDS&R3u7VoJWLcKnokQ-{9a8~HkCQn`SXP&wN)a6rj>1OytGx~IR5%!@r`k|b3 zh4=`0Hvyl|bu>^HzKw5r`pyq8S()v>)Z)yBglS>Ud@%a_^_J1+ci!RPO23&+Qg`zS@Bw5xmlhTG_zq=W^E$_&IRn>$8lrE}&D5jEUs2nEw2C!B}^U!3NUa z_XK0iVzBYR4w!P%y?K{Cr1OBiZoYR4wiMXQf-Q7ll{|0ZdI^}LcVo|F9=-(ZNx_Ig ztnZ1w9Ry~Za@RUA=4S%3#p9-&3uCSh1h&zXo8gpW-i`;h-js7;H=)}Ddl;DJE%KwQ z+w(7QeJOblDi3~oR(#%sKy(XvRVFXjPLI!HPP|0k5|fu>_leJAo*X3aE|Z5(cYGdm zCxQN22rO#&QI>7F`O=*vPu)l=o15ti=GZ{W-$a>bUH;V$Pmqm4e6Fh;b8tN6zC*cc zXr6g@2JHrRtzbSE2K`zJY#K1pjRyByrabd^3wc+Vyj1&d@p;VYm&lXcE}mH9^29g4 ztS-#^gOvS-%HHOdC6!$Hg?cB_AHIsIL(M<&r_6WO5T##)XBT6Oy((11+F?<*Nh??z zSRud4eD*nptD#eaJVC4V!7+uB+jnAfnv9>{dhAcwHk&w8S8aL}n$`yXYi#9b*^V8q zVODY7_hk3Wl|M&{y^lE~oj~RDm>=b4Z&OA8bZjV^JK1KA;RmXe`lc(Tjj2lMm;OrW zyS_>phu%tuvi`4>@$R8iv{NZ~(_JaJm7o;-^D1RrETuOQ|9=hXT znBmEp_FXP)wZ4Z8*M@E{e<5@|b1p?0X&ZetO*%gp_NS=o@gv^L>l7Nv{j+bKgP^f;h^-U|KrP{ooY0q+>z1WB9 zefWk_k2$J4^WnRuUHF+QrM@$jf@jw%1<${!6xpPcxG*OdU5=j4S+_IBqof9@r~_Rf>lKqyF=N{lS!TVf58f zV0(cHZ?7bEY~PdzU$&4ZSzR*lg!nvg>Lv1?GI=?6AU-e8;r&}pUba0dJ`Y?-fUj;b zd71Wwr{y`az%PMqKH&H7)~Y*J*^Rbj+TQ?=g^$yCUw$s4PZzncD}fz_hH(C$;W5+;I_kGCaPu!G zU;c&12RZb0kP8n0m;8FPK=2HEfD8WyaJA#uNGrMvJDlw4k70kklWQhdzHG6g%Zza@ z?})yZUdebJ$Y@?d$+|h0n<@>|W@n{wu!UmpX{n z_BLsj-P5F*b`PZ^!7Zi81D~#9?F4%LF{yc%cKoN zvEpkO7<@r%o2DGR!4F^XkXOIVzs;ZTtc6abjzz4A7HLoMpp;N&IeyvnbLW(dqWS~g z;M9!{t-hanvuCPS`gywRKfMf?=)cxLB!4(-nW8nKZoNjkh855IJ`<^);pwbG~3@Q!{roO}%ste;!hCWCv{&DyLyXUx7b|=P03NiIl zk+af}x6+ZjGLXMAS>McJeY3kc$4%!aY5%XzhE=RS?MIG})wi)E>oj?O@VA{f`)U09 z8SDsaxuU~azoJh@a~~qEhiJX>pF`%5{ilpn&+^~)aL!mAvOr-cF%Ej-zfHV&$?+El zotR|eF-De0K953!cK~KGtZxvhf3b3%HLeEF&&JHKV#Cx%-mu*2|`G-Pqg z8{1%wQgF45^dZ-GTyyp?v{8FdaUVQoqN&0il7e1Y>=&!H`NbF5LD+K`hpM;E$nTA-AA-lh~D z^#cP>v8S0-vEQx(Pp$=DW`H->fIn9wr%y*tpT;=@-Ho5QczH>hqf_}v<9*B{wDA_! zo4a(VR9A;e!S67cwGL>kfbrT<5sME61VhyFPG zo6!MESyxsJf9Vynk2d319=?ga%m?r-|1h?9rKby@RY^bl}$rTnBUr`<7J+U9x4bngP-zURjjflW37}e z3!VOGS?RPfAU=eivGrFD{2#!@hxTfZI^+63t{>8ez{PttHVx2DN0)#myL+x*16Ivh zEaJb%=>kc-Iz}oyXCT&x6G04 zadTv)b?Z51Ki5cPB#q6L?%bb5Dw;AV{vXhQrIc9(>>c`M=c<9kmdr%fbbMqaUns7< zQ-7!IvRK)31_o7bx0&NPc0c#Kr)+VwfqA)w@83n9{w=w$t9`WNCGr*m+l&vO{vkc( zAlE&nJTa;Rj*N2-JW#a7k55T5HZbur$sp26mXCE}d^JDOJrsVDwLc&HtXljm!~Q_{ z&3Gy9#3hWGbP_#Fp73vo#m>pF=P{0ZVzG0`pTo1!|4F5<))TvBmVu?)*O79TJ^O`D z@hf6Wus=D0eTF`KFucrD5C5=i;_ub{GtYu$RJ?_+=v)8fJ35#2+y4Tu;`v$Jx6b5! zmz`Pmzu&oA0?5W4ga+ONHm z+7tPoXyw`T!+tmZN0rYSsW?H0hblgw&cx8!l{(W+=aOZ@cNOO*$Jo-OxA-~V@X9OE zyWo!v2CjUSQ+d6o_w^lOcnxK(PjWx+3I9vovjWe7KUrK2%!{)AfmJoozlXTLlj~Zp zY}yxY+px-ypHaEj-@J)7`}vplSCt}5%ciHbV%6na4?1VnRKT~yqbH;DAoIUroe^*H zJkF%cBhKV<^7Vby=Rft6>^~>qlRNo`V-w4=cY;Ua_pbv}e@NEb?!sPHIq7?}d#ek3 zNigYp%&A|yuswodcRN8H{ziMnpBV>ZTPsxxZ}gLv$zKt=oUPa*jKi;Z|2TXq&*Ioq z@onTWKY7{kZq?&q@|9O0y^lJqa-WUOjQ>C2K5tfk#?c?sUw0=2i(MXlHT6>;wXu(Q zg{eR1Au6Sw%ak$}KQn2HJ)d+NG;2S0%<6QV6PemFm-Oa;Ddy_56@%H|6``Ip;k_?= zQ(v8p4}TfFdA8z@kZP~NDZVEiPxc%9!|9uD?pn}7{vkSk$_D4yc#cv=l9F-4EF*Fm)GQ)JRuOi|87m@b*E9bTG5e-wH8o)%Yhw$F(x znsdbY{#ATmdZX&Xx1(xT8|TCpehI(S+F|&^%1phR;CGx6h+wWXE;n#qxed?>PBYVZ$!xErb zIl#oj01%1-9+F$jCJlL+91MIrcotM-45`wU@bhkH)?SE-dA~ zRr%mrdHlQZz%Ar8D-XQ8BR&t@eu=y$U=IIC@8Tby0#l#+kqHWA&!T^|W-^iTCLW&~ zmn7m#8{F(Y6^~DQ?2k>hqGlZ2m?XzWYaJyPlO)}q0Pd^*gkRP4p=;-QW87Y?3#Qn& z|I+cBcFu=MxBv8jXHDNT__UcPHQmf!$&0~xhmNG%(i;ul(Z}lRME6GnAK z{v_Xu(w4El^p*hfykwO^4@lo(4SbpN#QG(BA*V zONu*A=ehFbJA#g?xEqIgzr*lPb8fWDa}^iJ!}vQqcfl1go-25a=c>H;u5*T!;&U89 z?$g{2^PccdJQu##s=dnzZY-|xq}n&Aea+zX^wQ3I4ls9iR;`{x93E?D<1%TWk6dJZ4y<2NqRZ5{Z+3a@_E!@X@;-&jk_wrO=Iri@jEY;z!{O>vU+3;8X^^E)c z;+OH){KVh@V#-M8SxjBjAF`cG*L}*=7ym=0)a|!Qp_9K+3Vy93-2tsQjx11R;xt^p zq6d8ScnW?gi~rUYt09d3+k{@L`BhCioihO=-n3UY;ImiAv-VUItLzM(D?L?uj$(eX zzu#S>d6<7nKN9abrC*}=Qby&@cFWyQIkhDv#j36E&o#vgbr74-&|>+-#OaWRC;r7b zLtXk~j3kYv3Vg>Wam{#AioP zSEFx1HwSV*MDUot2(2Cu>^#GRvg}%Tcb7iJIL{+5&*bIU+v4-Ug{9x_ip<%yznO8_ZdLh%ONhTc z8r&8R$$vztzVQj~2=`l*P6>n}?(;{a7m9E0Q+%3!*n#?E4@$uRvx*s~~_c(N~5W4p){}N8*7`t7L{S3GuI`n7YqRX)} z7Bp`Kf7!qjLaFW<3n{eae5Xz3jD_}c^5W=@Ib#7mKxcQp@9N`^^DXVm+zcH(b;d#} zE@a56*dG1m+3RJilYLKg>QPfZtwnJ&UUY14r}yz4oR#Lr6P!)`R42tZmn^@AXVK(U zCQY;NFsby%Yv{|X>C@@-?KJv$Dz>((u;V(hVx$+#_NRZ!_gUvGG+{$se&W>m3fNbt zW50=>iWMW-OLQh3n&Pv2yXyyc^H0sMy*=?o=X`}R>zw)c6X2TLd($k%rfnU7AH@-L zpN_fxL|aVE1bh}xmeL;fxJ5;~!<=^)!M9FwKpQrzj@`tH*_V8*h-a;LCu7U{4QnFz zpXCkyqSMoF+v3D&8+?6Rdn5CF#ykw~=^6Udl3!&1mNKtD?hAglr&s7n?mxTC7u=8- zIMn$^tL{_J1M8;ny$zLCYsIh!)*TykR#DIVimAhg{UW>hs8#o)VGm{>8Giq|m%*>1 z1(WNJLbGr7SQq8_{H;k9;CRZ!;}Pxe?i}0N{K$rE{5ft(?YAb|AK*-?eh;r}Z?o!d zfQEhYKd$`m_D?-FI{7aA&b8}(4Y+ur?1Nut?l%@_jpCFaTn;d2Jp~^zXyZWY+#OgB z{V86ynmH^Q^f>tWg?V{@aa+EW6K*rOvinPoVG?+uy+L+7kX5ueO1x} zmQ!yt<~p|lo$IT{-i-Nt)~DuLC#6G2hy&MtpjYVF{Ii@maEp+)z#(G~1rKj=`|^GM zD_m=1E{f-UK;Jn1dCKSDkHDP%OgFNc`dxjx1NaGiM%Aa^bo+D+eHH7s;oKj6-xGR7 zu?jK*t0s~T#&4*Ucrns_{r#++v%#f(#KAN)DHiXrFZ3vheG`4Hi^{-v|DC;qH4fl@zG{PR3IA%rKlRlSaC;c{vRw*~8lh3+fnUdab%r)(+5dFkIc2M)9CU0R z<)%>X8B-1z|Iv56;5+-t1}72Uo&73r7ACQ;VKi-~ykm!4Hs)sdTo33%^Pp}a{HHnd zk60chvO{Lu&xr<6FX=h*y-TD0n>cUadyK=l`TB@go1~(#Pt%@v zp#%GgyC?fOYbzt;=h{YSi(xIb3uKn|IByC+3+F56H;xB_5blsY^>*hVXXO&JJxTL7mLBCzEa#9qsTj?1w+{2uAr#{l!Df;7fomEhu!CLBRW1eAO%9uOjonb#pUv!Q4^NQD>ZkKh9 zcZU5EW2H4zot=1bT$}J24Jkh9aF730BNM`7)xKZHwQu+t?Q`(#R2$cmuXkf-G6=uR z+*pwjzEzG+CmK*n+(@0p@ZYwtTU`5Yh;Ls{r+r_KYv28}Pj33 zoA`wov}cT^Y(8HY%XIq{;jo#%+H;(4pBvxaw>5t=?Dn;>`Rmxcr>K3yoBmPWDDos%ZZvc()gH?85pd@QaLdnkwGXr>Fv+Bi zd)NocnUPCpR7|~uKIGrDC(RaY3ozlA>LFTax&I!(yQ16TGiDzsyzn6F_Tpt)N4SQ1 z$j>Qi`Yg-tL-}1s_T?Ng?#+AJ1By)3G*|Cs+M{?!a!AzRN0#lS+&7R-owmt7n__=k zb%B1KSl+eA)5xef9^q7${RZQt_D9Lrx7tY;@g4dvv;-RI?D5R8*8s=AJhsR4v~TQj zzY(U4XiO97E%DzldpvXOO5nojcSw&=C)MjM1Ix7EFlmPUx=FL_*G!sbzhcr%49%~OO=si*caLZ51bmjUNi=-Z|Hx?OGBi|?{hbcd^yf(s)}Dm&Fz$IEndAl5q2e-z)YfPPnO zyC&q~`_O}?yD{x#dnuKyjP1owYSufxES??n{AJhXuR3Us$M)4ZG4Ha`n;hHA9=F^D zlvA6umsjn|&!A7R*kmOE!rNb?HNi&KEGwjec8$R#C>{__wWahUeq>x z#kz^-ZpbdeHY>k^B;>23GqQ_D46Dipr?>lyZ#Db)8_;RZbJ`Vk@&^wy*JEqq$T=&m z%zj1>SqTkR`<6I(H}!7PjaqX#|AOGQDr3WL4a0-cxwaP-pC2sj=Bc~LV~wi9PDu>6 zR(}QS)BQ`2o48u&T@myy;z!+5McZP!S4zLt>)PM9>YV@f4>mBL?w{}U;Q{`k@!+4e zl3~RMXSw#T*T_@<%6|4bG{`w8GtLHm*RaOJFgsfe}D3G?X*YYc`y3}=u7c`$8JR27}M{L+;KDa+PiX-Qu_Kv z(%!}nq_Z;8puOqP;0$PSCNvpevPGQzieFF^JJ2!gK=EtpiUBU$`U(2j;Gaul*Itev z88p_w(A|v=U%X(HbkTuHfxEB~P2Vft85kBEfDAd{hrNR#)yJ}So(8|Wcd#0GKD5-} z)7$-nKAz$0JI(VCQ-Yn#cqU)6vFuzY7tN&J*HQm#X~zuOa}B)YYHTi!ZxiMEn|WOegy8_xV`CT3Y1G0TjdVvwgUp2xs5439~I$MlHf8+p`I{^E+U=#)P{m;~QQ zfp64|MdyHbM5v?YMg`w+?wKFlujQXf0r@ytsn_a0A07#g9xGs-LhV|}xoA&=yUdC3 zvo25RnQHdE9=+MY-*57N$zl1ut=jt4-r&F7v$rn+E`DOhy(f8$!(M!zg!9A5!|u!2 zrP{-hdmO*ytS-N`vw@AFK8@9~1p;?{t22Q+{D$+KnaATP;}gv6R|UoxiKqNa^cuXr zpHzH8-;ghu_{dR1%kle&&jbI&J3cpg+4ep0dCZ%a$cq4TeChZWIB*cyK~p}%zCFG? z^GiO`e>Zt)_Wbxf@L#^t@0vWm18#^vNiWrWLU&B|b4H1uc)Llyy)%pY?VZW_CD|36 zA>UEpXD`Qc?SWZo?F5hTpO!7d@tw}HikFx%sT%lKkM#(;(LyTUwM-*yQ_ZChHR~;er2qg687W3BXuryo4fuk z{;2ra@%*udI*W(?nfl7ttRlx6Q9CP8w{tRQ8++3RYcHSZe>mHk{yzIJc5ywz^%JgX zT<3CSb9H1Pt72Ew`uaNV?a`dGKw2|g`b(ho2KHcRKVWFS_BHigAiMa$ex-K}0A9m+ zUU~4W7l;d8LtL3WVzk`A{Xd2f>v)D0tpit|r~e~!t!OEI-ZD23-9rp1$t>bY8@SgV zx*pb&so&zvk>$j;FTe-sx8UjqU+wVmCm8nA`kU~3 za$!9MLoTSF!M-3DmPjgmP~6J0L}>mLXg_IWGGo)7bOQWLX$f)Nm5znSDg_rJ*fK>! zL!=e*+eM$zp5YIF?(o`u#1B%t-{(0z6r3SGN`ZkXHiqARN98zQYc4!l?S9id_ib%5 zycoY8_j#^)PO)FscfgaZ$Rgs#BCEio)#iQfcM9}Od-v$sA80je{j_zr`+P6|R$IlR z_x?km_I6Wo?B*( z%AmtJGJ#tr1NzH;qO;sGD^#XSCUDE-IAv1ZGWV)X4&U(+Ck2_v)G3dDF_)6vGIy#B za+$M_)+y7JJU--e^A{)|`Oe@LV_yY5kz7%#lsfs9^56Z?6WJlElnx?a^rf7$teE@F zjCE+pfG)XXqpx2_OzzlN8FXX}w}<(e^)Ir5P?klmu8CoT+atbAyU zU75a1J`16ej9um%_^xlrKv>TtXXV1Dhj%ij;)lu~iQILg3Otb(W=M#NGI-X$;Q?Zv0{C)33qHHaGd` z-&64n;B}LiZeJgt$Gn?I-XDQE{utB&+*k_iMZwJeU|`_M7GS?K-$=11 z$A4oi?IG_O<;~(>@p+?Z4|z{2@1{WX()c`RK?40F+cG-bEmcWi_d`16<^|8Uyp|; z!9&EG_}}0&;0}+;V7{@Dr@Y(2Z{UGM+v8VGQ_=sq>{5Lo=9NEu@{PZhVZ|=!= zrN?ZkjxUpG`vMhHA9Bn1D1(eV^=M-AkBcTIuKh&N^}RbF>dz_Y`CZ@dv{H2x~<2^gYtM_5u0! z<;00@wUbwRQ1MpMjh)T+^Clm@S4i*rrcs=G&i9OoE407vm8mptnfOR0E@0&>PLy>s zWv%kc+CVzd#J*m>jh*=l?EWTSUT40Ao-y|Nx9UE<99pWfcHG%0VFQ}XU5@10)W8s!DwwMIVk@-9rudl>vo zt~Uj`In~QMTX|U%%G>Pay(=m2N2laHo4gafyf=|feNXsDNP*58TKNZPCFg>NhOmA` zJ9U4yC;!n09b(SS+rdpbH_Np*gQLv>{L}GW0PgHU)-W`5R1WdndjrSZ@yoR@GxcHf z?w$QgPb>VEK0xprCO*p^A;^4OWd3K`7Yah3&NBb;)r3s=m9kk=x_OsF@AK-dwKiy? z{2?U#fQlbTrin8uIU~ZapzbZp?`J!@%(qjlhFAF2^ShVdWPU^VwcU-M0ch5H0juG_ z`#oao*~INpzkqbFlwK-}wx`I)0yz z%n5zk#QQfd@@3UbM0U}b9|5TcFJ|9_e+Z0HUPh*YmjYM zv!{>57pd;WK3Sq~zp)Ws!yZ2pU!=9jwokEFj`XLrmhe5U+Ba!t)(z=1?=712+Wd9( z=lWCL_z;$|RPruiW%w!m7@2P+{zvPOZM)C!L6{%8H@#JNYRd3Us_(VPwd?MG z*7Ze-oT2wHB5!WB-4A~-52m0;SAm6T>oMnIh zs4uo_NFcU%H1DIaw`*OSeSHFQWoow3TZ_M{AiQNJd;sfB@>)?UzMIfbAAv6@H+QcJ z?DC|%yp@8l&z#mH>DWhd<|jScprrT~XoB|bHSi_kCrgv!#&TZT!v1CA;Z5SC&EYIj zcABCaoo%4}HN3MT|A)DR?;bQNCZDS(#lx2%PY*S|QE5}0by24?9$-3ov=()asXB}= zQ2G{tXG!)M>Pt@dbVV$Yt_k_sZCh%+d`p^gfrr57M7c#)POHXi8f#y=0Fwsz@oCbi z{SQDd-d$k)ar9|}w>-U{(Oh`Kba)Z`jDkn^p`X*fy^nm-^Sg4Y-jc+YCIk7hoIB=T znq5i*kLT*Hy)-!$&Josv?pNRHrBfS>FHs|_lCD2@Dp(^*GsR2uN2Os6pyPG6L1$f= zm9~*qb?-?Z6T8Ms+oZIksW&1X$*UI}-2;AJ;iY*=X;=%!e`z8Oa>t%Qyb3|`5E&1 z;idTQTeZ*1cGDx%5q}z z$Jz0y$_})B8p(@&9?6fjKJRP%Y_scU6GLy_YpZD7HyYi+ICKav<7aaM5Ik!cLR>lF z7xUZ9*}_O);|KnXncE&%njh>UKbyWu@j*tfu-$qp$LJN_pXRQQe$1(4s2b@N)}%yt z|JV&LB)oYVel`_8jdLYW*OKkw!fyDa+=b$sgN9E^+>sYgzdIm5oAA=yn?IZW{!?^_ zIZT9&BFs@(zzaj(y_v9s3Tu+?fxLt*Bu3TTMDI4E(&AuvZ8>aHG~t zhJDErcbwj;ac!p*Hj4NYd~A-rkUY;4rZu*ibdtFTdHH_Bf1T$j?1Gc>W!q1YZ?(yX z&wrDz&@T4!y}*C*ciMlnKb)v9%l-}d#FMMcH6~wyo#EwM+nKMx&OasJUh+NY<$K)Z z%d@+9`Brr1!|&56`J}_R+spS5=?>iOj?Ljm-LYBh<;B+7>@CNj?`p5!y$}!gs0SAV z1MD|u{As)NA5*<@XRF*fZn@1~xp#Tx8oLDI{_d5l{Uy1aQIs>mE2oM)uK!Tx7&>z^ zabt+t;%R7#+gJ!O{ zpLa43zhCV<;}F8_VSKUdE{-d7(3$DpSF(~_Fl@#Hsxg4=ct@XZaI&8 z{} z21oj+k@yF!#dbQ9!nvAsQn~g_=*I!*iK{orwSSA9<*qWLm)ah_IhQmU_T$i?giN)+ z+SN-1*-vuhx0Jje6DHYwpCI!V7i3QVF34KlE66(DEyx;g7G(YZuV6-n^H;$V{6Gm# zKtB)M%{|P2WtVppJ(S*zZA0eL`_}S5Bt4Yw;p+T#t>-&o24`aF-#`zA+>r&IA4TtE z?oIk!Uj;LGe}Voy7EFDAg?#nt&bxoUVc+_->>H4261G|BZnj(T+)mrNv9wA3adw7d z(jCrMpTP0^49v4{GEhEz{vDe-x6Z`9BH50#Fqb9xIjH8Wr&Yw(Zrk8pqJP+4JU0r!Afr@C!FpSJ5f zwlDW5jxeuUKS%gAdOEfN#H&BL`+F{HfA=2QV7c$BrZCSrgt@wXy|0>WbXb1E?lbzb zLitlJw4(~zq6^VIH<6cYRyDQ&S+zrw>(N<)d>Jsi^1FN_nbkj-q zg9_{)e}3YAP$}n~YBTcAhLErKNxm2MCEX8t`9pUM_Rse%tHEC4Ku>HdPPrqbc`Wtr z2qn)a>9_~v&E>4D5We`2x8TF*;Lh%`wpj)4-Ih6)Bi=~5TG>b6?2YN$)aQ_@%y`gkSV8 z8NcR$U)Ln?tAAlg_sk}F__cz*TIq}}GOgLm7>j=1I4t$X;UIV?9p|#sj79Zn#^N@O z1vF7(aRhoI{E9FZ25)jIobsgaIefc&I`~=O+Nj(>{)CO=M_&HK-tUlK_^q>?n;EYO zq*Y(d`K-d3ca`S@$Af_^_AqNljQbPVhWrKFkQKW3QA$4d9?%a*Eb6=mgdJlU{j5#c zF=p7CSX<)lWQUu{nCMOjy20(?WBIHxWW9$IHX`nQ{BJN`(<*9;7d7r))Tr`+aI( zdDFJDuh0C8zGj}}OM#CRlh<-6$Xv81~q zw|`mR;k`7^ZxbI6AWwStMONZ3L^`|9`yxFn8gKo=eUTIW=Nxu>-^bxuGPwJ5O5eYc zG|wxN`#nniMn90$@7GM<^6e%AbL`jtg)TS2L*Nt-zT#vaqWxmt{X$PuKtJB=Vl_1K zTh9L<@cRkniv~%D^b^Kfyuq`CnKcTZ74^bw!Xz&=`0(?`uYJC35%WyE{21(k=ZFWg z2i#!%8|2!H$)|Ae_=|h{-hyZ`_eiAsoJhH9mw4YgzBT_pG%&;dE>P>yjW1w-|MK^7 z#WUsKa+xzXU^R%RyQj0-e~ z$!FH!Glk5J(iYf0=12F=kAjPRXjixIy8Fg7>N2t~h=(G&D}l%b-^0Ck?0lDm zc4bFP^L+T5v7C#k>o3fw+b_<6$iIdE@N#dNv|F>IBU3H(iTsLvvSnEk@|_n=v+5_d z;w!BCd7P(IT2A#MemnS4?o3~5^kHS8He?W#)ASAHXSaYv_x@DSj4xa`7tPwmcllBh1CC zJo_l$!nI7G@aj*D-9~=#`J3tsjLs83B*Yb21)aDx&-0ye&;FHttw{$-#3qLNop6|45y9R#-9&T+2KXWFyrL=|iZHrypI_BZlo!;Gm|Di7NW#CeS zna5LbE5_U=ae*w2$cfmzc=Tqj$ zZr1lI(Ocu|XJ`(;|Db=uvs}N{eXf0rhgb6n7q0z;anl}A<%QovxbRIr6!K`F$~*@s zIp-bj8L@}H^?k~^hTqXZ&VY-m=kNR$vdepxTmCf$_X>^wgcEm$3R~g9Cck6wBip_r zk;bKcC*$YdGo!_xzl03hqWVSq#_}!v9Bp8RJKOVoN zoS~!llKrJCXBD=t!REfS+CRxD=@yc$TKk-o*fI7X?p(im1fR2gQ>+Qifx!B9d~meS zLH3xLhA)-$SUdVTjdu<>&c48S<=Cav@8VF7y~(p34iP3?S6Y8;v?G~d_a2>Z;uoNg zd)}Qz7;k)^;9OC2d7l^G zk9qsmZ7vP=)3#NIIvT`ZC$AIU!!`8qbnE1GXrS7zwuyH*n|j2PCx@wB+X+*AB3$ix zL+v@*vGGJW{^|*T&CBoSEKzayG9C#&@Tlaf+6PAHzR*bIxzZtZD|B~hA2dCL{#mw~ zYoOm%zP#2kgP&M|e}fY|WMILF-L+P9Pl-F;W#E|F^WXD)%WiOK!}ehD5ZU%R!Cj-c z*PV)QtugIWz)Pi@;H7_-^w7->;g8Q^-o$ej+T(uY`ngfMr-}ane#`Y!#=D>V|E9M$ z{okS=)2Q#?eVavoSag1U%(Lk^z@By3qa)h$UxfEdrX?>9$IlhM4?3{bU1(R#U06^t zSAMEe_{oRWDBu=!03GL8vzve`iz>|Bm6aJ4vn%!gChn{N4UTblCyw9eRnTDF+2Krk zyYB3~!kA&xztPm&`;Ostspnl#qx0TngQ9uFS*a%O^I%q`vljVf7-8chnV|v;0TUKWP5`SN+1)RQ5>16^pqp7Jhu|;^?`5 zaMQf4G?L}f&u!NInLYA%tGfcyq42&fvfa15bClmJt>_2k9YPPX>VR*@qB^&nCgLh6 zCmu+NE`t6a3viy(4ZF|@qhg2N>=irkD0@JO@6{vNiE0n97CbP*T2MRU(7Tz$XT=V^ zn;kpwZcgmb?sNR*B^T(@a{O;F-)g&P=>sM{ z*Y1@RzZ+Z-O`T`ruo3s@x)>esJmhBGt*Dg_ znEQXiA?C;2|D(;Kv+_TpdxLStuB9Z^*kPD5JYQjTq={db&RtY&GtgDXt4{bIQCY`5 z9p4@F-Q{1i?VB|A%*kPN0n%+rE^+-;p#SoGl3Yiadk-bUM&2Y{p&5fp)6abSP1cS( zM(*5d3@$Qp-1|t5L(hCOaYIa8uDvxmj%%r{VXPEW6oZz-98PyJ-x6@P-d=N`y0wzT+UZiV+j51-fO+FR(3ZviwB zT^M+n3-7c|1<WP+W9VPQnr2KmlCw_ zI>I&?{y*3L6YDC$+a6$C5;mRRAdP#rti8>BLuxo-+V6hn<;i7Uzp%DyN{oK|N!FIe ze6)uH5un(eOP@O^5tWa7?r%&q86FVjZe9`tDS zZ}^|!X{9TRvllkk)*8OH8oqYMm-4j*@U`_TWc!@R-yC>g#O@~e+G#Fddl&fU(ujO} z2jjnMg0Yia60SL$@y@eXF;@w_!m)eYGd=moQCu7U#nT=J-oRLO1mvH!H*2~NYdV89 zoynTcVol@2Z(a_vr@N-bLnglml33SCy0i^J?+$nWa@I9=;i5a6um^vF-+laUuVeXTaZy-+J2nud_JSwI&g7d?*xD3`t8@6l5Qmvot(z@ z0qPJ@%!P3L+}~ROMTJ7KXl8UN!g~a=DsZF7oJVSErfe?VS|7DXy-pz!@H?FnJ!$d zJ(l@+6TH^?5}&R6GD?&1@php}Q(*trOJm{~NAU=QiF5gieD)aG$Gdv!GLt9IZcNSt zE@-`;ZQ^om?H>s`r2AP<5qGAEJF%8`@LzX8q_5^}Do3(uwVyS7{+G$78LZ)@E8_61 zC)RN7h}{9sGuF6kcp_+;Au$=D{=aT?>OFxfkc z*Ek4|CEt`^#KW&eEAbon9oBX-4$3}S|J&|{X9F)))*-&H7Y%4ifd+Jk2J`^W&H&$f zf_J^ZzuwRQbWpXf{wd+d;V65%-UK?p-ky9n1RlPV2Ap#NvPim11BP>^mCLW4bKk%6 zdpXS-`a(fP;}3vqi?E?$9ph_Y=r(`B0Cd59)}~s+8flw&e&IzUeHZ?|MVLz)^6dsM z><>zZ+@|&VtQWRTVaOGt4Yn87q%dgkA?OHYxNm3*!Z*zjgcmLa-sRDTr^puvR}Mi> zHo_l>PPlhK^6j4!m!KPOdpdrtYm**&mPo(GqZ?5#eZSMujrcjyt)8yyo3v4Lba(=1 z__XH{(+}?d2$pdFN051c$iNKycHqf0Vs&@I3>YT|-ZxE3$#rJw}(maL?YP2eQz^YCi6uenU@s zSq-jSmPjKz%`E#cIuoV4nRNJzXI;Cr=LltdAKG(tQm@dq`Tj|Vkw?Ur#rq?Rtl>_{ z)7&XJ;EU!z=;ohHe(86&^spL4r~l^J+l(h%>qB_we{Qrhfwe3jx#rBhi>qt9h%n*M zCf1v~&f(*s8;SGxe#EgmlPA|+o}7pI{S9$`6NjAQ@eMEVUwEXm(IB+IdRubJ4-Mb25}H%@r8FlK znzL*LXDa)H53XFn+h0kv=Rx#h(4G=krV&kcb?O=RR>pD{_Y@4>DGf1p1@;ogUGu7b zeEb97vXvgq>3uqSlTsS>A=6yt(Hr5e=#u)c@1XZ>@S1T~`R`KZ0pyH>jJMY9PX3D? z?Er>Fk0RZ;&z8YGs7&rdWpSS^8`^~3=JZ_fIuHEL2hR(jO@-Xw?-u2-L+c@sCoy&c;9>^lHBa%EKJ0i&=y3cmVvm?@-&19P84DDE6!#iX6H=FnpZM}iL z>|%a-{Ek7tcJkWYmWY4b!u*4R~o;N1m4 z(Kh83%{xBtTg>^5%>51E!1dt5H=%9ULEEkcN3H=^CUa+wcgP~$44r#n2s%x0EDrtF zSe>UfLu2S$V^QS=t%skkjE!yQ9FR9r)Q>&fC)vlE#3yWP6ZbSefTyYht|Lzs_b5vE z9^*^r-XgpfcQo`~S?9ex>e&@YGxTE*ZQZoUrCaCF2gzQ2vF|Z=^BxMv#AE2b^|M}@ zGf7h;yI*hz9oZ7=H)YY^dG^BD#94}?t|Q=FtYma32fDeb>YR`b-P~B#KZK2r^FrBB z>`$^@*&NB*v=2WFhbHB6UoS5t-lL@?KlI6vf`P4Ng#%kk&K!7XXJ!cc(in%QYp(T2 zx7NCCJ$jdGcMeb8pnY?pr&BvbxM+deeFVDkWF)XdYc=4}*WH8-qn_PO?)!`Z=KXB` z|CR9F_*7BaTI{h#5%#X3I~n#hK$os^w(QYW$pWr@C}E7}Lejr%(ihrelhcDMPZ6j6 zUuz|t90xxAhPd^Fxp(m>i#gp(*jkf5&%PizeVJ>gyxPR&+k=whz}2&9x8d9Aga17D ztYSPg+|cEpLzhdq$2QL82Roq6@h_##Ind_iCunndGHreczA*9bR;K+==3L{iv(X!v zgI#M9vfa6iWv(4snBWOB?FX3~@q>rG^U!Ul$|MU1M0Y`(&w{2V)}Y=~Nqo!xF!3!P zv59Zha|dM~VIHKrn8~;Jky{0!+0%hNM5q7Fb4$GiFFx_m2^~-;pZK#O9m}WWxOa#z z$gvu-_&$ z^lqV4>^S57Q(^<~6~C_xdFjNNVNYoH@k28`T}smrN63DxwHg~aeBT(prX(%4uY`B| zrkQiZC0U#!W}%ajtd~uFveDM}uROht+8`U1SFnqCVBKgZ7u-Avs7dU4txlH(Q<_pXV@m-FTW`lE*~lzO=!$-JKDr`oVWhLWm%ZnPp4QL{`DOE48OWZw zjk|=eoWc8e-0#Bnv2h6F=<=p{_7K*HXu%bP3AZe8k9YH+0h{XEb1e7$5$M$s=EI!b zakof3%s}!L1>Li|L&W2^q_H1&>ApZC&i;CIt7j*(e~#ue%PuwZm}#E@RGali<6&FLs;Wk7Ms{^;@4$d4fG7P&;BQJjXuhNa_8-`o-jx&&GZI zd4us{e1lf^48dD99xds`zTKOByAS(z#=wqAF5Qs*)(!A;;%S0j8`P&7@N_q0AiR$U ztj2_{Q2qk;(m&U}Cfl66>-VQOxxB$a`Ys)g zXJ4}+VY@X2{rBnCug2qja7p8%vfuZ{?#~8h+M5lO?)%HvLTlL-@O0xk?Kl4aUJGh- z25SM|bIvog{j2YU_u0_~jvofcKkP3&bM3%(!>1i&kJDcDGIMC~DFpxRt8mr^^4osq z^?wEM*iGileM2ycjb;h^1hzAl_`k_LqSq6TO~S_bf8e9A*wvxnZ)c(d9(~K4TjUR@ z>XtG)p!=Wj>XIWSl1F;^Eu4QO<8h771{=!ut{~-_z2sr`~?wV^ahD&kBF-*O!#lTG3k0 zO;fs96CUK(wW~GZV{A4?FI+SJs)bLDzk1;>$M-=Vn`kYVl>s~e>;~-SUofi|@I2u6 zt{l03dgTMpgcts5{KJ*_QN}J${I>9{sLT9+aJD^JwtcC0&428s9x3BhcZ9mb$b7f? zQr{>`PhV`>KzXAm|K=_WMue&V^4`{SQ?VIQSr)v&fb`VGGpD5%1j#e*Qft#pUx9qr z?O!dsl0dK7snoN8aVuUpn6*)8FP-SFgYy}?RT<(pczcjNri}lWvj*k2PG^}%zNhm8 zD^qdUEWIV$#g_iOKifn10{xRVvp;0_Uc~;8VL#*PP|lZ;*HBTj1H`doFb+Zx8*6yY#&mtx&yb zZ@b?Gx890=qQG`&YodIgx5r8Uk>D*phDN&anYQYXje+WW#|!@p_^Nw}ItLj=_;$kO zFUz$REwmN(K4CWl6XmFk-zJr-{|kxVN|s&X*mjf^Bt{sW$%&#aH3qPI7i@hh;O^2YH=1M}@wK*>Ye z`+si2GVFTb+Q1EMufv0epa&u7fpBpbIyUVo9Zh|{(g#DL4O$1Pdj#ca4Sxik5{&l? z59getysr9{H00Kq)3lfQgX5%w3HG-7jQQV=VZurAUfI~=yug~r9%qARkF!MfIPXzr zU&d!R!LB>bd&_}KkD`8fYJzMnb6o$X}Q%di?gywt^MBS;KpXsgb!Ai>u}y!XNZnzcsNm5KM!Qo4t#B^9H9`4|SY1 zTB416dqn$?Uu{%fThTqNO0hP+18;CryXcSR<*@YJ(9x6PXtU;pbB+dR>RVc;+t77r zowfv02c)t$$JvVy&p9L1d_jfr&)qhwSE%_c>ESpR2@^KKI&Sb3di*omBAa3JAG(;r zn5o{!88d~6#wv|?trpQ)@+khPuZaISaSLYqmUYyopa%=G52nJ8rNNJ-qX+Ab?t}Mq zs?I_fuDuT235 zc=LP!?<{iWUI2e&;kSBY{SbH{$tl}NUlhzZy{uL`s8dJ&sOn&C{npgQ`+PvPGn$}* zciratHD_H5e=;_XhMsaKlyuyU-FPSIz@_chl+@_$RW4nBg*ffmebFx;n-vVrlWuxu zYRH+E7CLgH)!6!S`oK?~?>=zf(>(^p@3vl5x#GiKLZ%Ua^DFXef7V;Ll8=XZdT)h` zwl{O;;nKhi`&ZBwodfBAfODD@@GolUO}5<=D4DCB_KF`#(9N^?zQ?0U3EAsh(aZqn z6Kni2y_uAFN8tzT8@x->I^$o;Oa;hH^(SN|co*TTOGA;FIH%eZacStEnH#efPqCep zoTRg)1l{Z)o#83>o=5+57LaEjuRdWPa~5MCnBtCk6=SmsobYT>J9)|));)Xi7i2=M z`Iy!gW25<2y=PFK{<9ZtH1&=;QSS%Tn@Fc|yombLrV9MGXO1lHJi*4mZm8LvRkIF2=VIcsq&dd4x>X1M1mLFNw`ZAs{hGTQ&?eS{^F(bVre z_|iQ1Qu&b%cCUCN=<~IP`{GOgp_Tcfz0HBTeA(l51bjOVU+1I!erQ5~vwJHFUT>JQ zyOjCJFnzDnJ%=6VMzda)=(4n z?QxCmv(TeLYosS%i~O@l_X740jF#f-cfhUkYnwJR9iBqGMoM1A+!?%!rt~wg$qMc0 z7S+D3|1IF{fcv-AOMWwD%pMpm8$2XBbL-}|YWDRRgX>mw1kzf`zk+?Z^$4~esw2o4 zGo~A?=C4PCmd1JHI~ly^TYZ)_dlB@iW2Ys*@)NaA+;nBpHyHOIx>kc@zEJembJo_E zuFi`NDZXKNHTJm+;Ac|}Kats5tnV)3E%IBpieK-c@0?b>b=N}K(xi7 z%=GKlagEL1Jf$*EeVL~guRUYFZVhXxf>tu0MZAT)ZJ^gS;TQVH(jseUE;L~tw6pN+ zRTF0*2Nn&+NAkIxtwOJhe7*Mj^bP%z-@?^peAiaB75NHVEBRJ`?hRD6$-hewy;joz z+IRe#okd@>_D|#cZN@I`zKW&sK-wGl9{f>RAoY#zXRms!+Sk3cjya#{D_krbM4vMm z+VohFFSj*!!B1DrgPz}@JJ)(k06btk7GKVN%rU;;kM>a)J_g>9zoX#bMd;H`&a;cN zk-w4WdGOcJlLyX;PPxe+JsX^v;m2dk*4^IwE(?OW-O#$$r+HiJja_`XWu-5AJ;^KJ+vnGoIaCAgo?`mAQ&cj5Oco%LDGyf4uAf-2}mYHP8+ zeN}CXIcI9(jm??NcXhF~|5m<(tF0*Ww0{9U)($N8*5O{pt7d_@8(3nknz+50``pib zcV{!Uyf5%}r))4?{5?1NGWa4MS9C@EE=tw6oGNU4XL4=`K|*hL@Gdh0v4RsBiJ(sxd4G2a6knx-x_*NinXi;*o@zwV#dnP4_Y*l{w{l#Iz`W#GgjX4 z4Bv^lR)6VFW0lr7bLfr->s#N9N0rt$bC||FS253(d<&n`nCsi{!?uSz5|=aARjln= zzBSicqf;3V(U87c8_gr4P2jV5=|1#VbonB}Zs%3`Mt!i zo}Y9ge`ReKS-$=911FhO@?%jAZTJ5F$;A8j?_n&|mhFjn{yR5n-^sVNw-s6b{Y@j$ zV=-@ez8dImw)*te)PUukk-Pho`eoKGdtk6f-s03^{0f${_g69p$8XpftXVaYeS3e< zw{nHq?|66C?9I^S0i;VUw}y2P_TSJF&A}x0^!QHpWGk2!$8RF~;|&|3YobZ3pk1HP z_qC^?ZOEvO;^V}Nce0R^%otKn8~9L%Eb8Lti9J4xJsufr!#k9}g}m>OcPC{hzBjY~ zQ!eA=_Vte%_oO|4Eqi|Iun2p3W~)zho&De4=UGS4$PD&!qsw1l4NsJ@g);im&$akz zdYy7Z@KPb>ReFcjQ{Xk4`lEBW41VrX__Om^M^Wy>CDEhpR-EyEq}G>#wJuaQRcI6DfCox_{@L5L! z%U5(fYB{^6x%Vda_$?zd;kRyGxF3BDUQ)~@wtLhpP9`*Is zNIST!H8NK1iCE5sgy&Azdy2W8Yvi*%Zkh_E87UleVa?RR1QSP6;ycd|UI8o+KFZ?{h7twCDub6Migqh&8{-3ElqusXj<~triSKQw+ ze%WRSUW|TEa5Qx89IN5i^ea52Yg9gWB9d9)seD(}Ok8j~w4wHlXfr?kzn}kY$SBSH z^#5M|vj@Bt7ro?sA`TBDx*5E8&ZbmwE${YNeQE#7nCv0)xy#e9X9fEKwu#{MUdE{x zbWrWxPdSp=YUa&pTRv|{+g|=_d@SB1H2OgH@ZDbrC&F$fY$;*!9DF?;*S(a>cD7_xHpZXs8FL;2AIfMO_j#Oo+(+5NeUw_= z=fS>}J|3a})&88;1=z1BteEjWK>tOLeoVZP*|vX!^cnWm>>bVMX!bDId+E34^uumf z#-vyNJL}R0FDoh%H0#{xe=&AQLXbyzC|lkX6mBIFCM`UmF$#!eE%9b)4FiR zS$iI>I6m#Ysq=%s{^+d~-}GNh3*LN|d>5vW=0no_kl%;(DYx%g8n}JeV*l;(V<_KB zEdk%{roUH>j0(38lvvXQ>^j$lvFvP4#}-;-f8=>fcbDIh?VzjyFRu z-=OYfJ|aRp&Va5-ua~IPqRzh5W%`lMe0Q^s(>C3I<4v>j=u4-RQ9>EVc~AOb%3!^1 zPi4O;B`nUKroAuCm%Hxr!7KkVlCci@ezYR!%U);k|7z-;V)mpw+ef-t>DDsZIh5ZL z$$b|v-*NmK1{F4B%O%!(Q^!j#6Aq|c;XtCzn%}cDcYbSw@6TH<*-iS&zfQfQ_`N&l zTcuOaQXkk&--^?Zso+5I!{;+5S@uKpMQzpjK%(AU>ec=t-fTL3(s+rN5iM3zK&1-nI@7Xas5q6#s{9V{tF4z=y9v9pkc9sgh9Co6DFNU2*1)mQ)j|e^+b`}e+3pAFl+#ib%JB*2aIbwPFk_T+T(IXT=cr)%C1{9VL(SFkDKye+sn;_MWBIpXXPd@tAp&>tFC8*1upa>tC=s z;yf=nFXB8Wcz48kR&Y+lSuZ#%;;a*#8F6BQ(<9E)g3}_-F9oY2&KkiRBhG5U>mtrd z!ATM4=YkU=PQBo`i1UQt=!mmiFcNVd7aSRJmI_`RaiW6d5$93CAra>h!9fvcv0z!m zStQs$;yff+5^)|BEQ&a_f&~$0fnZL=sTRzLIQI*}7t9lckGxlKWQB9LAUx4sg764O z@W=S!ZG1PKkbT!flPlQ+Mq21xIV;ni6J*a>>*`In|0}&|LSL%A|LnYq_GaXklXRx5 z`$j|POC^t#diqlNjw3xfsG1e3yHs^s!8as}v`>E-8O4XJ;zwpdw>>Wf+ixH8n)|Nk zAdL^{<4U7*=_mS(C8yJ8tl~^H?Ie9hUZ*}odWx-`x{RFGshzrv>{g$? zjV>drb#-O!)Xw%ovgXs~B+apt9?orArtjlvbRdE@s6MPK+{mmOOsRvNy<@b77t zIWhQloh5PZ(aw6EE4epLeZHw-<~q}!&e(8XcY!IR&%o%~^9M%#v_o;d>9gVc=@ah; zx0xG+3sr6YQA>|IG<$wr{I%UbW!%DI>(lczw}q)Hw%m3%@4BP z5^2SYD!u5U(r&^2R=QF3GlzaEt=cn)I;0yI2$Zhy1Kt*IW?idnjeU`?pjB&1E-R^EL_@=Gu>t&TkbCP@cKq<{Ri@v>r02 zUgpT=TkSr4t8ZBXPgKr7_%HZ5P;-{R9ff^Lm}kGTBw3GC2~Ai0@p-4eccwdMzrWR& zIiCE9w>HhXWFLt`Q!ZtWw)J<%<9(B6>)QV&{fzrvJ@ zPjxxzPl47yj{1Jb?4L(ob2zc~W!Fz55+(>|fTr-BcXOZGuuX0m6@+*KHr z{#&$gB);OdFz?@eaMY%UJ35Aqy}oo4XVtwO##i|Y`5g^p3`j$tV)9N+$_vg|+zC=U zpP75>rbn;0>pvb=8T;}5J?j66z9l;M);8Y1T=zM&D2lJRJG)xtf8{La(@QF1pYzT_ zZ%+^3s~>l6yIT9Vus*GR6^%9M;Vu7n+KQ}Pvu{}s=sU;kTb7@5CF=~cZ&?rMyQkT= zEcT=#-V)KiWp&ke53_G&(a%_WM?)|Ap+5IQ&%pbitrmKQ=jPtIY48npeP5IJ3{zKM zleed-t8bcHruN9eXP{^F_b_#-zX_UN&-k7Wk5=QSR`S}H-SJ7>=}}qeduGg-apk05@cdVG@+j7a2t~HlP?{jNE?Akfs4Kg<~@-63yA6Brw^XxMEFy$T8*sQd^{dEnJ-1VmYY}?+`#sL zK@Zmx+6$kgvhuU)@*BXX1vAi>HT{40uAAPMl})zJ+r=yC9-M5lo4fI*L>W4`SytoP z{y@%{)|LPt1=9ZbLsnyBL50qLx#NC*^uBA{^I!9y^Wb~)OO|9z2`4 z#CdS6sbaW2sG?DCn47aa)`EDGRlqv#$$fA>HecKaXy+bP3->5YdEPmGMt{!n@f$Ej zy53XHihKX1W4Y|*^=@boJTm#bbK*ta+;d|54U{VlaohjhbK+Q+lh29uzFg2dC*GD~ z4O4mX{#S=MGo3kxGqEA?$a+hIy~OBj*9tz$8%^-b@rT$Cfw_dOEUDPPm9;A$5DQ%0 z@`fPi&82>8n9{e9Hh6!mekEs93!ty6=X&0Dl0F)KqP%pTHL+zDwvoQz%sOaMX?6Cb zQqCZhPCSg>91$MHu}f~LwT5$!+!!wlgzy^>iud=2)E1}27lI!^=QJ8UU@kraGVQ(W z2_KUG@8|_T;(TYzng2O{^_lC(KXB$V!5PUSzxBdyBuscDIc2LCb}eD**O4KXQ^5EV@4o#$$Z{@t^ZPqh=0&VQ z-D7NGEXsG@wNqsdGhsQc4-jU?Wl)79o5rJM_-%T5)=t%Nu1Qna8eoi%l1FD8=Dixq z)Ez!_%(92PK>3dbE{zSKExM;zDoEOMfKQn-4lf^N?qS?+A)oRr@@KV1poQQA>+aH6 z@Sokvx9HwclI@4mm3n9UP2jH1_HsBY>fk)6nKsWz&=b}bb3b%fNyl>cPL6D=+`B%* zzd?K2JBF6xXDdF^YSjG`v;H_Y+{pS9uaQDqHMS=Fic7Gy;JzMxT=zFO{@8TN=+BwG z))(i3wUf=c;D+ES=YkfzkLGP3>DBflCA_`For`#X@E)G`eSXa)70%mcY<>Gn#2*kR zxn5_3tChMB8@r zy*ZHGcH(^?@n~6gK;=@7=3^CU#CP{ixA2Yc-!bJbH+%)*;%%)yR>K9?Wlu`9F=+I+ znfAlPFQ-38ZbGm6mkK9?|61!U{uym8{j4au{Le4v+_}^@qix?f_7y)gdvN6V4D|59 ztIlE6!5g6g^!o>tvpb@@npyT4!tWa^oI40p+;6la9P1{?5FRPlGiMZ>Xn87&0 z6AguKn*C2U)~UQ}>GZc^@rcec$p^mZ4rc(Kg` z(K~1Ovi6OkeYD4|v*H3Xwz^}}$oSBvJ{EDJL*{&R=~W@lKN>&Exia*3_@J+%y+;qa z_LT2*{RyIpm1+8feT^9_%3tl4&t8Bn`W+hp*n9X{pon>|B)J-+1HNhf#>@ht7wYM402W&&}-UE#vxlFh}V ziAQ*q_ozh|d=-ChmyE%MifVAZ(BF|IAshM;H5v=(0irY-PLeG~lx z|Kh2ZY?(I5mZ=UODSZFi{7fqTmXht>^!9FopDBe;iub?9@G{$Y&o2IKMPnU)tK&~s zG=j%N#k2Sbmp|66@Ij^x@GLV}e@fdzI`J$;*t=|n7ivg>Us(t5lR|s39dChm5x>G- zIQbZ7j_)!D=6y2Yef&gM$FOEs{PFuc0X21lWx(~BYc@VnVW<_%T2^OCp zO$}AVLdbU5Sn;j1!wh>i@vc1b@7Ez+-3zSgy7}3YW(2HZvd4bnt-E$UjviBMu@`-j z?$3gbiC5J)v_mg-hfDDby*(h0x^;Hl?Bz4`f$}^Z0&7P!pk8;^pU7d6m_t)ICzaXt>J;8h z`m|%%kR;p*e~L`a-Y}ebm%TvpT4V2>IGJ-;**i$akUwPio>wk1Lxw$rzH81z-=u>&G#mndgfeSwMfQ|z^KG2!y}mA@WaS!l$s315Z2Tzjze zFTtl)O_c1=m$!!ltoI7yEJG_sL~ru$fIg=-bBC*@x7E-A|Gd|XyW}zGrbjbgBd_@U zUny+{_JU_v;yIScM#@k50c@lim?zN#jYA{!=P}aW+rw&jlKR5fWk}cb26%JSam!jt zyy(gg!Bv(0DE~z}H14(f?rAyqAWxY5Q_hVh-T;4zGE%uSsx`2-e_2TBAMwhV@0Edl zeuLKTZthvm7p~sO{ONxN_BY_-^@Lr(c$zTUC0<;(KZ)=sp?wOk<^L#t-%;3_6Jf%2 z<&Txj3?=NG;!k&v9iD|-fg5WJB?W za)Y1wlbG|M_QMn7zlV9y*IzMzRxHAJjLH-+pUON*wuR7lK>cH;J%$-C3 zntNg+El4{L3DV93!0TS`7^b%Ll6(;G?MU1oilZ-VL0`BR9)Az|Lhi2F9sRAwj#_JY z2fEgdh_5Zps+;(urnTRz@by2-KJvqfzu)ky?_I`sTCnD=w2CeN@b&uoS$@uP0>Du> z`+M$dvf8@v{ZYEr`1gS2(B`)u^vS<*D!Qkj{oXNfp1Vf6V={6^v> ztE=pt=sFtsi3a@)T$KH@%fDsXKLy97@3pl18u;_cG5TTV zRO#08tK+_}%1OzqoV$@%{B zsNuXrvPdC%{{Bh}|ERQ0o=vICtKerI8gmDFPs{06u6(W$t$ca6 zqCV-GKkD1uF+A3hX3Z{!zyDRO51OWZOFFx3dk;F4z0fIq^xMBb;@)-Ac&y@%_>7P> zv?9cO9If6Nr=QYQJ+jgW8Hb?>=l9J|TgTa^F6B&)HKc}%R6r>=6-93YL>iOz-8)?)a< z4A!l5ch`K+O|y?Q?w*uoU*&~;NSM;UP;52)l;0G7sz>AUJZ+ZVQDd{s%%P_;4mC+b9`m?X-&FM(6HJHTTYawCe!-yZTke8rJ@Og!$GUQ{eS!gEtm0B4a4L zm~`UX&!^mS>beM+On*dswn&~q_nT%_U^6iC9knTzYi+E{waN?8`K~Nt?szl7O;;BE z19c?I&>I+I@<{8^G?llFd}FFJ>wVy@cq$BKmi5gwwj1b5bVs6au~kgFODtzTWhKge ziErVn#x3lP+kJ#hIoz@F1!%wad%YiOaFsnm^23L$k6ENO`g#6OC%ybPN*Ac_HMQ5b zX}oowaktjhXOWnBpTXdH0eGIzxk9F$!&;ewUmuO__2g6j$-n@03g3@{w|g1umPzn| znj_X~(3g`I$2W$;We==4&1Fl(8t$Nj)4TER{-15@E%gsslhUDmM19k>UeVL`Lq`CtuW&W~|l`dHE$cB;3%q z(wecqQ+Ds}*x$?83->kmZ<9~Bwo{OP?f~k1m~#xroD-PvT>A|JGwjz5%(j0gNd5!J z7QgZ0Y~W4s_X#|=7*T~LR;(22X&341DdtsSScfm6AvYZv`msk=n3j&4a1 zd1DM=_`VwO6ZD4f_(Rdx#}8c`f-ay#Xn6gMti_>9-3juw?l!(fg2pC7Jf38f#mu+x zwi!CTR`iSe8q|fnDH-M5=*73VhrhFm}J=B0se`)YB(=h+Rt*X z>&nb+k=6^pFah8bC2Io_|4__O@0cmS~=w( zyQD+yn6h(iH*4dVj!`??cvDRCyYXIk?Vm##+7BztKH;9%onyvE-@F?He=mCLBdpx? z)8!AeM$P(X{C6La?c53Zsw-&|egJF9b2C5XZ^1Wcr+n4lA3K7si?!_DanV~b>8#~i z*-W|R-38qnVG(n3qf80Y}A{Co`||ZSqyVOb)}(wli`y zw@w^_T%O7AP%q9S`EBC2g5MALy_n8>w?L(w4mA3&>sGjWOh- z*Db4|^!|71J2->aI#znUjoSh*fFEGP8!TgPMHALjug+~XE{}R)YrVD{V{iEuHb{N> z>8*lqvBy18orAxV0{daoN)N0v(rxo|>dpI8-Bfn2{UCAiZnCEdw%+8`KY{w){i%DX zg?+j7{_OgeHTaB!R+@SC-p)Q;mF2#tjBa8K?T#M^#+;8*jb6K_hnF?9&**W%%U5bE zhD#Tsbz3cbtCepO{5pAhuiFS4PC0HG=9)1SZcZj#{n!nDxO)?2K=(%xccqETwck&U zV_b!+mk}rZ@5wqZ*N>Fs(!?7*uP64Ur&D4M{Iuz#_NAWOi7Ek?OAU?_iT{s;zmJZl0CXk|d`<_i zyMy07pfhK1P8NtJ-j~(>p?$)vFHh#W=`4IpOP@r$t^o%xc0277o@D?HjkUO!Q~hFnz9-)(US}q|!6Rt2@!t{f;afs_t5sb({pM_? zOwnZa4>MN(3shUg1Ice{JdiUW{!t*Nd!*)lsoI4j7V#~-Y+2JK*52AR*0QD>Fc>>r zoe@&svhjiQPMXzVXaVK9^dZ~6>mHZ>{hct;Kc)5GKHB*Rd`Qr)Ko8gV&0BcCZtcGM-_`#R>1sm<+@aPKgX&m(N@)-DK=;QG5?ifB!TI_Bc#ADQn|3Boun_Izr2RXCC zzTu>Ib49CH!t+S4um>1qFEMFO@lSK>|KWSNrY-oVQGGf1r-8S37Es?;?Ze`)5VGNx$BTRp{jDMcNQ*`d&k7(ryU8?J=*O$QK8W)c@ z@_!5OjU1ltgKm3xEZWCDzI)nKroo%?Bh+x^7LbC_{3I9wQ{tvWBt zu^+qrgg@I)iT?l`))~0s_j%(MH!#Egqk-~g`{jFl-IXp5?vZaDs;vo!7cd#^rb+=LXm+3a7Hcn>P5x zx6x7S|Kq{7@{xh8nPF_bh7wl8xCpmjoZn!4pgqJq76o%!!>6O=3xl2hs>hS>>)@;E z)qAuFJ;Xx7^Qp(_iEcykmPbdgM>k+$Gl6WhA=&;6+R&IfvS#A%3BTMoI2sIv>ixH% zn;IhgResK=!C&{??<^xz|B|wp%b^B$7N&aX*uTM@W@AT{V^`4*?Z=k_`_i7Sv{mI@ z?}d*9YCKN&o-|``=$~_ii#yklcFI59H*1q|$NCcbr~PBg{{_0Idh(%r`OrPt0e;#2 zB=M|=!4u7MA$`dPA4IQGm}~9pXA&k_HU%BQCiCV(w%tW|fIWAUY%!p3iFr%-B9a~k zd-KLL=I+nT-SEKRXy}#!^?r0g)6jKHYa0>m>kCASw|0$|^M+83?3l3W*IWm03Dui4 zS6vddiUU#KuC7t(6b6#60vu7>ZihzfrjF!1fvFYnrtaCBXqxma`*?qQmCkyT%J%iR z!tk2t(6J$R(_vd-}n@=2ND~ey970XeGYz`Z1=UNs1GQhC zEc+$yNOeRcOYlbX^zN+L9<13jShGD@v-m~~q!`_}?3DK8_+suGASdSJMN@XBy%CrV zjZ*)A&YWm&ULZ^~s~#QV5Zcb3`qstR!(_rYA40Bfr;P_`-vP$L3FHkphF?SZF%902 zeh6MZ_x{5A(&|_0v*4kAV!37in6ky6g8y&D;ib_5tczpwulIxwnX*(SZ=$11gMS&; z#=2>t%p>@wtV8c~w0|0Y=yU4hPj@kT@V+TlPyAIJ7q4~{nW5#gZn2|lg7|SjH?<}; zCK_RJ2T1gH7I>xmA72>%RhLAIJ^#Sv8VAafykphCOW;dz5xxY|&4W*X%u%F%>*0o*|zF1@?<0+_ir#Z9#v>ICBr2?^%qUOVbMMLjJ2<=-pewk@q~C zim`;tMpgax)9=fL%g}9=n~-USnLSW;E}5P3OkXeT2F5AbuWdfz*Lrv%zEAtN?Cy-+ zG~+#reif4@yCd<=waRthA4?}Z?k_ZBlgijgelz|R0}DqubNsvu5QvE%-cu`jTVPky zKfylqp`JSFL+5+dl1YO4iU7oYRa$PIvtq3Ku?yz9sJOm^Q(;t+lKR%=f*cu{X6g_S75~obcOL z2QBH0Px1PKFIVz4aCc?9{9Lc34K@6C_kyi!b=GI%@bmgp;^K_8;z~J}qO8w@_^#c? z-ZMow&0Er2DM!9*L+Up^AWsU%&ua>hj4 z=U()^H)GI;vB-dK@SbU)OZ3b9+dJqc5;~=XZbH1R=)7oiVvJ^BQ@A%D8j0PI`0?e? z_#g0_q9RI86 z0ih4ZFGj$XBez>0r*`rlgD78ZJP7VfuD)6Iv5syMgr1E7YF#Rg_{QV&CNrj!7~6@A z@zsp=Rp8eI?DWPn2UlXJ$J=C)E@sbP8>txna_hj*E0LT{hu-ZPJ36yx$iWZ3)=e94 zgncrk&%oBQ4E*6|4%FL&eY|(=ub@ux;LVJ!_~Ey))7AJ~M!0M&9$*Z|uy&uE1RKgD}HQ1)uj^bN-EOggqZ=;-7N)?NR3b~SUL_RDtbefqGAa~k-p?ctBppbxxd zM}J*eD9`Rn->`i%GG@RVZ!;&P>$uUk>~3_;e}TVHe7$Tf0{Ju1IZu9_Hn?rg zv0taHnvdOtxq4#G_q?#b5hfgN07qq;i|rorT#kv?+bsr9t6!=w@bL7%$ya0*wDvn4 zo?ek2-RkiX4^pmZmvHc9zJ;TW24>hR4HWIWobedTxQt>f+RM-3a=s_UXAH) z@G29$k{qM9+(|zYa-Y_*JBGRTZG^7}r@r!Cgh<~V2f-=bFKb6%t2>r@A67Kc(3FsO zceysdEty8v=OH^}xH$AeMqByv46ET@e%BHn~*&Ts3TD z=o7xzznWbze%J%0GkaUTAMSnPO@j*B!p*?N;{vJadWYav(uUS&7mU?=1e6aywSOvg zYn+U)Y1Wifvs1khl4+N5S4C?zPQQ!bHMTN7;(K%+lL(tanDFYSH~5xKh6Wf~^)NojOt}U2 zeAY~jktzNk*4{lns_NYT-;)6{33qZ&z$8Q^16o8tM6e7*B!G%?5i9n9VrvjvOZ9jG zA%V~ujIE59Qi(mFXvu7}a!L@QwxHM+wLL1>T5G?@a8V~26fglY;WEGXXYD;silj<=X$CjQF&zq}q>*sHMN(XHCb8(TxnnIHJ{v*#%$ z#nafa$4UymHZH%9c{6=GO|~|^6aD?axSz+&0kSoXGmkT79%sy@!oBavC-m!M1W0e9 zxXQxAX7Dh6_m=E{M4wKP1;cmqG z`P_Z@3N}C^bO)mRMA4xiDe4%SsJX>%r5t;JBiQq(9_`=0j#Vp-PaLw3iA5je-oEga zni~tZubk;kiJ(85+WMlGb7PMEZQxsb#dPX1dphTb;cnh!)jb&hl73cnvlrYQEW3Bo zft-8LHy1Cgt z69j<9HE?fa3I70RZbbIro8zzX!l%sc(ko~! z_5?jocO*7P7W-P3oyDC(tDyn4aT!6p?2jn>G-Y4wi2NbV?n*gi0YkKw`D3kl+n{R} z>-ZG?Gx8CywegJk|7Wk@F3zozDf6W?eg2ay!E7AU&_lHIB1mr&-{M4 zJ}>XWK}(9czosYqhq(_Y-_tW*at^*?@nLpE5xsByvO(UL#e6Ns<)EolsdV4NA>x+duOy;pZ2_X zzpMDR8hn}9(HeYuzZ-UXf_b+&@Hvfj5&yuLMT4ScsUPL~{*srs5BraMy}pUsliOI2 zSW@oXCSd!}#Vd0~3!Yn)k|%o5GQbLr_c*dL|AqZ)iU#T4*)F!|(qeFWNk?LnaBh)Y zL1%(h$Twy4zPs0r^Ox}c@H>H!Y^z(*xt4k}CUWkcd+*>XLpwfM&(OwfY#Il$PGR6~6|v%-JvH6lcEhPKdM9O; zq@3E!h zV+|yKJY{}*UJ1=d&J3;m1lmKazsE%@v=3axx6av1a%lO`W@3Y!Ys=yvBGV11C0T_QT-W3G4IYk2DQV_r>z)@IqGfL~Le(Vt}5D=Gge_DQNsa}=*nF*v?} z?sVk+H`zo)+0!MzRQwXfZiw5k{FHBXzFBHgY{E#zx7U~CZk_^z(ou^`+TOt_`B*Q~Zh6`Ma$BOl$=H0^JesAeq^bkCQ^2otJxb z?`mb`$X)2C!)@5eqMHtX+{t;T`}HaxfCRoKXaGt zX*risw{U(C?IoVY_CO2H$6oJ#)5FYfj@`|j=Psm|(5}w0DJC!7PBv+pEtt`oe51|6 zH`v-p4j`Lx!Ik`Y<8XCgx&v2Vm^8!QM=HCDW_134<=G6+C))uK(A?Xlbuw#Y)?q2@ z@RN}~-B%|&mCuqxx<6@G<+73CHK`%RAD$1~1tyJ)zRB4CZ(u3Mg{A2(EN!Qs;%CNC z_glbeyUl@F|B4+O$&>y@`@WF@gRffeRfa!{+h%3jTJM$Q$(}%V1+pnP3jQ>BPrXyL zr&+sf`&rs`^|0%|%C0JYW*Xvt^SGJYZ2M7C$Id3he#GRZ+dngDnq6(uf5A4TQu3Y- z$T2!1&)^QL>B-19uz_2E%%>zNc!+hD{!(@-q92NXq%-wB)E{40*>lwKJi*!?BA&zo z*69AnbcYhL08BqJ?b}V7WtX}A`VM2!J}7&ol#8m2eBTeQrxxIQ@9A)G_#nlj%`teN zoARN-`+avh`}=pPLp4wmadU`St1SBp(+9nCbd_4O z3(dPs`||d8+*b{r-^#o5&AW8_65h43uCv^Ajh?Zt8Fr1quMGQJl>N`I!Q(CM`*w!? zd-t1j+_rm@I_sKd=a@XL<9ODkl=T_MI$gzd-#m%B&j@sGmss{NO-+7g8HL zYwkRlTg$x}5B0Eysg9R8H;V7lT^hB-7{cypXdCo--zKZRX{|;4d@K42JVfKbZlU_? zSs&mcGT@rfO!)8CgtKG{XUU`EhzsD&i3yJkk=n=6-jbKWlJ zY*~t*{T%dhI@5DTo!Yno8>UIZ;S75X>!CByt8QD`kN(Te+eBW&T*}>v|E8PwTjkxR zZ?kOSt-gIhd9#(5ZtzxlwaSCOwM~S^q5Du=>SCq0Y2PQEt@wSUGu`v+CVc2ibQTS@ z+jT9ae%W=+HfPr~yTbi1U0L{E;%GBg#fmrQ*R`EO**d?{FX6JzyypB$pMLEAe;wU| zS?deofgHL1daE>}UGF-aK5LEICIzAm^y`OaT!H8vO6m9QN*PC)(uslS_eph*)i+Ad zsJl2lxXYEn^`)%JNiG!NcWvdg)@8t5qP%7FOr6ouU2W@v%;oI((O6UhPK*IJM&sN6 zmbs%qbJ58BU6!&vw}~^s4cyJGm`t@gzq581yW=lvjZJMGY(-<`IuGO9J6$|~+>bfW ze%qDrY96$A5TC*9+d0@*+>dUl-}jMGLKE8tverAj!RFZx{kqT{o93;clr=qqEM%ND zlo%#a@iSVJC!q=AgPnMWIrc8r|0rv>hw<*kx8haB=*AwTwFrNdfqai|N-N{iSvX~3 zc3xQUk#E66r%Vis_i%YV)#*F1WwM2@U37r*shhQn*ZXVgl`gcAeU7`1=3)Cj^%Z#J zTE&AI-8X1?;Fn6Ur_9GLlJjm6=Stzl59qJHlfOmo!ffCQA8+!)<;?Z4Kz#}_%ZFIc z2z_rF*uft`#~$f!MQ`=yR(G;;tDA^3C3rsE&*KN)jW4hhTM2zDu=5FU^jQ(_-Tw7< z;Yf3M1ivB&R&wlUzd&>!|C<=U`p6wJb5~<4*~+@sd=TfxPgnUlCHx00s==W?B~J8~}X zwDg&4k~uq{mow@Gi@S6Cd6qms!9AC&&U{a91RkEdwVXR~^uBrtl+nbJuOWt_E;LeM&{n(EolD(~y|`xsgiF?Sh9!a3~HB zT8C_G3j`B;$dAK>ADCzY9@++U@MFC2UK8*j7`fG(4NPSJ%P=v|g^BpuXsxu~lDn9B z?E1i?n!_ewZo*mSP_m80ISgcsdg01GOP<#}#$_Avaa-;j7P#__(ky2V6X#E`9iKnV zo#ycL`P<6;;UgO?PMAB{M`-R=e#P8r|2W&+X~z4OiS=k%k{l6v0Dwn#QZUSxf< z6J&OU#1`19I1#|X&5T8~sC5K1%Ikb@z8MOxLo;ZvKD^)ZH`5pR&3egMcR>U8LIXa9 zZ~Kg~-AcdZyZbaWr~>_X!(G7AK+))2`&QytY|WtGl=CfoxhC`T2X>0Km(Z^GSIu*P z-fg%GmY$9+V^iZxSIBMBkv3!@ZhT4Ir!Pg&bN`@=usoZ_}tX?vMy0w z7rOPzFElPo(0DQ!kLdY%ym$0v>2_a!$M8jKU2}CPJlErIXs36aBj|(t=M>}OvlSCV z8tZ+GMYt4)?=0ZD0@|Z?yHl5B9-61UYpx9a{Q+Y`rmb~J{^m5Res7kA zubtCYGBoEq*jhO>pR={Q?xLgHnFH=kpdIO#DL-9(>ut-g?i1$vQ|4N6ryl%AY=yov zb zT~oQ|ca4>v--RFa{0@1r*>f{*arWG6r@HG799_Y0?HBUR4j&u=D`q7mS|XW}XYfck^CyFYyxd`288b zV@1SV1HRPf@Bqud3;0!^#7}(s*u>B$!2VBx-(HMMZ9YSrnk&I_E51F#>kr|Tgp>2I zhc8ElbJSab{V;d5PJwS^UH(p5cHD{2F^j(6K&+(G`{?NdoEMD^rO)d#MLbgyaJ`52 z_NuSQA!i*UR>*1d<0%AJU0z82Nc`qjzL8wh?XbQ!zc0EMIl_9V^6q67O$pNb{n;NOp+t4|gWepU&gj<&L5i9_HCgpLQpFr*Bj* za2Nf8?`BC}3H&Gyx)H7nWk@$Ow-zPm;!kQD?5__7~wO5B4p-J4yQ@?kT zLz8somYh{|X)tkpUPu1<_GHn^4)5tM>#wrcRP4BT41+|Uud9F^+27|UTR&dRx$Z7_ z^r<|TqJzxj$>QGTBQuiyk~tN~F0#n6i)`dxdC3Wn%;@Tut)%=|!q`a0Vkx26bPKv> z{q8=_y_x%gJLc?S1FM;K5qzWK3ivKs(SA>IN`dScI~>g6p3HDkPG=wUpOx~Vf$>e) z!|ME3%=f~*BGf+lI+5g9OD$Z6tY@qqv|`soZsWfFP5K8F7c~dZ>_(F(zrHhV26aDj+-6WdAjj_XBG)*(Kc;llibX7W)G=fdibX8Bqwf9{ z?bu=3Nw?oMX@>njCgo0X(ro4=4ZHd_?5IEBsYmzxBxk3iS;!8c=Og%iq}daC6*+`# zZrqJm0aM4i#PD|1~Lu4hl`H)eiAgdx?=%mib61!j<=?%8Ia-~n0_p`~4Z>F6V!72FkAEwO= z`zDiS+BcG_pSn}?5WH?#o;7t!o>e!LCpVAv>ug08bGn}UI4{NaE5F={Ey5Y^UB&P5 zdWfMQUxrt|9V~Qpx+!j*+nIy9>0{)dwqEw4Wsbe*de>fbx$H%MMcLKZg_=5^9Thx& z9d@Gl9VJ0GPKOoo263MF3|L9fcYhaHnT6j+82hzQ>ifC=zozF!SmP|<@1dlWi8^mi z8}wxEbBl8GBaz^_9+h)UQZO(Mz^t+_$v#S?uhxd4&_dYl}^htFfG@;Fb z@9QXEDj4{S6&$Z|4!44uH^u9kKgyxKZ?cwK560@WZf#S#_z&II)&Jk3=je}p1<-D* z!@+c9)EetV*M9U>zLBg^cjEmX`q%CQ>-ukn$Qx?#3j76k_do~E<^N-bRtl%#k(gW2 zYlnW4=i=(Euvd{jE770zxWSWLdpZ4@|21@5 zI?7p+b8$v0Fnd<6UBNf{zliiw+CE7W^6dM~yG;9DrS$JVl`@{YOg?chO)A=O6}UbT zoSy*hkB0`7BA*@yZE$=OM-fl!7TUF_U$i3eo)^(?@akP_mM7Sc`ry5d4L5wbY`6_h z76Vtc1Q`#f7BXLors_ zK7#K%;(0~L-tjxGgYMStzD-c@fp6I^e(4 zS@#{vlYHnfG)ePeu{PM_A1K^Gl+pX4u!6G}E4B(k%OGQtnroyrCz$NuGOo*sIU#&&UF+i z=E^$>zmfJlo#BzAF$bTv=Si2EV+($FkrxHNi`k2F;Tv9I{uW5?_k+u7l5`fQEy1#2 z_-~X|IrEPBi@$q=ckl3yOJ;O$?7Pj-2!wYW02cCrUB_mJzJoUE682IjO|80DWxs6FOnb9QGwhd4nr_#b zlvqY4?G2ny1a2n)zvF@9Qs8+Ud*7ABB6Rk?Z?e&81A^}r?73VH&jg-`4~nxQ$5f8R>;O8tK#z30i8k4NiN2n2?egxV-`Z<#q3(ICd%MrOYhxi^e}vXVvPS5R z;^&>q|0~IN@??kV*jS7vuT(L$KJ$chjuVbJdijcxyJw&Dci&`aLAE`CwqJ>ckxH{=*n~4wzEviJz}K8%zn=Kn%~;bGf5>!5Pa0* zgSr=(#;$9J{txesPg!rP4*s!jC(lbfFQbc>4v|<4b&^vz^SwDg;U}>hTBPrGQit~M zc>ZkGUb^_ufYG5*@Y18v5j5eK7)6hw_1=Gt)+^oq+@u+Hg!F@6PW?OGe(obru&_%h zbXje`&%1md`s<$ZVa~Bj%elk;II%=}SR36l`6;x!fpw3cvl8id3AAW-LsHPX8M)`; zj`%Dk*-6A+Slx}Y7Cd)c9(Bey*Lv}9=9|%ym1f(Ah?i(#|Ms=t{F!e?^Ucl9H(|b+ z1ujPD|8U^*(sE?DU(u$Qb7^!B@Cz?>l>54>nCoNajtJN8x}OtE_Y=udp~d}p-~B(F z__beI=Xp+@O?mk6Q9itmdoij@B+oXyj^qs;Sj)sZ+rEoT*vNp@u57RsyVBfMJ~{b( z=bWSX;^Dc(1-dIGWa_;$6@R?Wp{BWALQ!Jg8C<-}X(P$S_q$kIosqN`COyfWme7;k zUd5co_uq}>GSkpyorl18#w&cE(4GgrBNM-Zys_kMa$$4__$huVl|BhR#p_O@Ey3K? zCgq$)dLHi`e#$v--AdWP)Z>eJu}>`B>F0j|_GgVnovLnetFNLViD#=f-EaJ+Kdc?;Wu~v0_PHi6!|rQR?))@qnthH*MN7`^{}}gq z{I_gm#~z7`k%=x)b+Hjk;$G*;pYzS?}`;XW*GDGg{5q-54yFbwS2}-H!Zw z^3vc_!0``xHu9|GS%*ArVlQjh253wkoS34i0pfGH_i5>^hgo9 z`Hz#bBYUv#xqW{Al!@5#+>L$Dn@QPkuR?!UO#U2vx}?|Jj!d|5BD$r4si8xUh^Ojp z+pL3V={DNX9LUDxIXBO=doOT~j&ik|SFgSR`;LuJuD#OD-=O?E$%ht*r+(bce?j?k z@wc7LBO9N4nOmiIVJoBbPWURNbK!$z^K%<|$?1XUaPF2<{~n?q#R2mo`%&M1>i(~k z-Orlwp^SwKb}lw~`0SeVQLeqv{Xd)ks?WDDX-c>R`nhmJjpBhR-hjp=oRzPye2mT7 zL!-Q`J#w7`oTl1kXO}SstyA*?-qFuH%~%7`A1GxUbCoi#IZ9_ELn4)rg}zaGM%}Gh z?Qy4yDBE5)`$>TRuKmyYMEjpzS8M#~cFddnZ`uDG8i-BLgBH4EH^wA9h4Ia&uD{K4 z_~|QXQ)^#A%!cS-;-cYG7q{yfBzlQ%=+FS*Agx32_>Imt!}tbW0WsLDgW=@S^%i%c zI`N#Kt0m~$j^gV+1zQMo(XH@gZNuPMM0c6@UqLVAv!eA~$#3abkbfB6bg5G2u#tHc z%)Z7N@-C7>8rM(z`JL^~uhyg}iP%I|`rC~Ioo|W-8?;jyZ)XxV_iBg!_!N(W_O?UY z8qrBBR)h54YD4D{@kWiL$OIBi6%e=DIb>8YY{7;rL%v#r=!o-<+aX&9M*R_iC}uXOufdr6?9#m@e&ZdAZty|I4&GI1US7dAYn|l>x9iP)PT|4k# zH8#luga-|*m3TeHXqJyxFMdl-afo@}jIN+%u-fizcOhT<;_34gpCkC46 zoyPvJw68fa?V~^b#I$eu+uNnhPt&drNgr~Ab~G1id&mW}bs7wuZEVspYK@!MB>AOh zlz#CO@L=ym9&8`tV-s&6J!1}jtE&sFA$5Bc*HCe-;&#LMkIZ2081JXlCtHWBc5t8G zmO%aNd}40%giEk#K!+rohUNi1LlMeF9y2x#>y1srTF0hgA8qg3kQRDqVdom1%bH}< zupuMVw8@I@xeHr{d$DO)Kw6GX12zo%uwjT)W7F^mHVuzq)3Cy|X@K7B`i?_y@~(1t zeT}C(zv0m*XNp!pd%$IE8Zzt$-8HGwTFa(^{z`_n3tZewU!6EjY4&Hp@}ecqdBQ~Y~;{@1ZCa}^KjLiE^u*tdZ7^!xLTF1~>nFh;k|JW8Ma zvsUz0=bWOqYKKctA@s=A$;aU#@!LgwYv|;tKz$4N4BvW2-*mrTykChhFCD#f=Pj&7 z%Nh&56<98Y}S{#j>GO}jOBl6e^#%P^DbK1uFt0~*&gBd za!Ng?*YOv345B?+?8HHyJKctVfMqb(Sy@rCq6dy$E*=>nU0Qp zFMSHrr|@G5eFBzb3)}9C#kgwe(`wFKK36Z4IIgqRkqB3b_531XYp1{44C5WN%_{%7vjL&~V8vIzW4N5UU?w*&%V5Hsyc~?Md-6 zZ&JnrmTU1a(Nyff*N2+J* zml1G0vd&}dr({=qYf=XGlo{2H>@SToUHe+mfJyXY82!+E-iwdn0Kt2P-N(hTR?3K% z&^b$U5&y4p9o;;=kA84PNOSrmc03i|i>+8ijFA>&!;@_{PGH}n&d*5i!G=fWe#YEh z&ws^5J$7dbvXZmMMvcovbT*OvS9b;;<7}e%5IR>@L32)?O>%WMc{hINUs?)#GW(Kz z?edXPRKfGlNACI^o{>CxJlI9n)p5?K19oin6Fx&_i}^;r=Z(6LmHUOOIYZpe8Da{) z=ZdFPi$8h<8l!btHPJaMZ037ImsppTZr%pfDISrt;}dS)Z=R(eBs zi`Qpvi86`;Gc=zP9N?y{YL+%ls?|26d0dJYVJnWS|rwY zEGIac`?C8|&p^t=>l43z75^1KVHw{f^7GiCGM+Tfzh}lcJ$C2#_Bf}i>ulc_pljPY z9=IB2VGF6VyAx;C_~KFT$MEes*IiBfTd@NXe=nF(Oj?ce9^m&fc=@g9e{+>X=Nvi!ePHdHSufdfiUz!Qm^#Jhv*v5uvM1>Q z<(c1j**{Wt7JF_T`i4{JL4F0Y@!s}hd@nj+@Soq@g=1{)m%+!Rfcs6`Ji(^5o)G`x zzmxo*tsN6;joZ|VcF|ww9=0s@V(hh8%R=an>XaR0e4R8O!h8KE4&BCwZpAhaylw`s z1sB5UFPUq_M?OG$FaD_UZ}sjd{}p#)|A-WPFH-Tr=!{=d7yOdCg74jsk#?MQouC!LRd4I^@d8k3kFK{%ErQX@(YPpG?I4bRTpXnkzfQ^d8`OckF(6q_=#G z=K&tcJugF_6NRUiZ1qEEh4{Epp4_=*-K?ocILG*%` zc;ChuR{cx+$NFnD55U|g{2>nDb95Np%wIZLJ!`S?>b@~2FXpl4-IG<5t236*lM#7F zXP?vY<$~Vrz6Z0ZPko=n{KwPZbjM$vK1gQOzvHr%I#c;Q-6GrY1-`22-Q4Ph^)@Vr$KU$O&`l7QH{XE~Q-0f? zc}q1WL%Y)L@4I>Brd)^643|#%TspN~^a*&$gGQ>}TPUx1OI{bB|HJ)XDf+}57Z+{M zUBLNEeA0CHouN_eiBsJF@C|2~qdx8Ohg^4#8o;${OpId``eOXgv3RI6)&d+A;On;BWhSnIn(lMdY4@g#o-|tjt&*4~| zp&4mO!5!B-^sEbWqy1`Ds#W*vBRco!E^GS5nmGNEzk7~-F8=ia=;5ZLf#JIz4Ak!? z7TK8ebD*v1_G8nX_|+@8V;)~e=$qunx^INLM}g11oyDi+8eUpwr{n+Ql;^&eVbeG- z|9S-Tk{_t==*gL=Zwr|#?HA@gwi$tX&SsNWbEn^8?)1Bb`{YZw)9+mF^y|o-ezV_q z>Kf*?h6(R>0c(HN8Jsab>5g$HYY?|r{u9490jJutb}>f7e=)x;jIV(2+Q+t3W6QE{ za>u4K_z}jc{-rUO*i6qmiY-YCW!%_CJ5)yhRVI}(!dcCaO`A7}4wihqU7Y;xV;!t{ zW$er1YYNz3JUqzY>WJrB_biX(q5la@Yk>|*CoaE!$!m8fdqR7X9r=`XJRp7Z`oe36 z1shj7>-;EvQ2p#Fb(`5!bRN!UzIuCpnY+rBor>4-d$S@X;9P3?#+o>v171&U;5_k? zi7}gN-vP|zTfn>c4(L&t`@e+$e_PbMrh)Yq?NS+o&&l{kx&KA;)Q<))HWBnmd-FHh z2W%nc*R!+bcb^Ab$){fW=QR8C8P3>dP>*!p+Jj$Xy+q5dC(pn-{d?8TyN)~q2lQ`~ zn|Bp?lA$!v$KSenoR6if+Ow2`|VLUlJN5PZYyg>b-*`57iY{VL-c!IHMPMgE| z&frpt;49sJ-IXB@CSS2w)%I@2J!B$$!ff#+#Lsf=#PZ2g|9=H7k?iI$;~Q+|NqQJ` z1^n)$GK}YOw~W4*yz&a(#mHL&FXiO*wr?e`jXbSI)hXp2y?wLZG3LdmyvsATCi}>% zJ>lp^W?^gAT%`C)OQjd#9!m5gia$u7PU_-`+syhpv?I%Yg>mni9;olDIQLx?=RVzj z2waptQvFkGp-;v+u^t^*fadUD>-(`ubL{_P(k%NUlcw1pnw0xLN#_ZdD!<-eUH5+o zH)JEM`*@`Li^o!alRAFMo+TRm->i+`FG<8{`X*-zOj0N1bR)n7#cd zv~3yt*GQg5uQybM?T2LN(k-jrHGD7m`555cm3Qx$emu4j*$i>wDkh7k(r=Y<_;TpJ z`%Pc)&Cm|$QVs2E{+={xn!UoL+RM)F=Nk6|S3l(sb%=F3%)Gvg|LMoX#rFj&Ccn|E zbbgdEjECp-j~)~(_8_+!6xck#_w&h}fBwRc^F5vCR=x1>WZ$!)9|t_8^NS1dTe&~C zm+a~MeFDS%f3U6_g>Pdi|FOFoV(Rf<8SU@;`Hs||zwo8q<&7`x(5GKYzS{_mEcW>N zQ+7xW<1PRutRHm>4rClT(6NS&)<(m#bsTN(S^lN5h2s7X;7R1#&z9N~N zYy!~BGC%Tx`Y=ewiG28q#7-!{X! zzdNyh=bpVx(~kD$o}7mr{s`YZVDQ653wBm$JPCeqqT6wq(^I^AO13cbj4XQn?9=4R zGl^p$--yzGZh;=@OT;yTK z=kzn%@O0ASHUY2k{v2g2Ugk>jG{InE`^m_Z4#C?f1}ykm-_c5SZF@E6_#89$pOUuH z&UJm^ZJJ_3G^cmcrsQq!B6zDMf}Ew%f$7;7(rZkj!u;c^|s{%Hp^7h3tFJnW8;YIG{aY zG5PRa(3~v0n_DJ@K1rs1df9t;{{j7Ubm6MQi4XcC@=w==8$S{JL)vLRu|XR-H=Mi& ziti0A;?5SvAs-R&c>USz3ln`rIzr>r|L^gQ(%Gb!Fb(H zUjyAY%N+||Pq4Nhc2Atk)@>LQdI6p?20VT~z^adI@PwMcacrkU=$9T>Ecp1{UyVie zoJBpHMUyy-dO3@B;4I304GTJP7G-ZQ>GWPa4#p=lg%b|!=q{2b*29b$J$RUS&5DuH z0$$wH0Y6CAN9XF*oZ&mcjLL2DY<|(5?RY}>l&!an1XK^;O4xWo`@fCT2wp()p z^OxoZ+B)!iX?CFP3ht(Vmj8Qs-r%{Fe0bEUdk0{1F$3M@ScjjfUWBh_THw(+!C8+& zYhgFBd4~4H=R-(YE56|w5}4fBN}_;^G)Cr4)>J6Tj; z)TPwT8V=a@;KA1D;|3L;2iUv4?%6lzU+1@6a$k5Pjb*-dtmr0yo@Ox0~yM#Dw4hfHb`{C5Dk+F|4rt9N?d%j0fu(eP>Nqhz_P zOYOEBHcJnGn7QX{zTSGcAlQxfdZ)TNfNRI#RX*k{r2D@A32*Wabb1!)FzT1=UUheH z>;5+1YL24ptLE%6Fc7_nJZyJQX-oG9A7j2W#zNLc@ES!X@hN)wGU7Xyf1CE0Cm;8> zR%2W7Fi*ZGEwcC`aIwM)-p)M!3_bR0;uhR|aO>p}=+2TuTTR>z>B{|{)atp!`?U_o zrWSbqYh*t08D=r3g`QO-^9#x6cVAD%<%fX}?M*&(6!`5789}@&AHI@Wqr>>U8965V z)-hnQY3(@fCrz)JuyE0%t&_6-jcdAK6Pi{NzSD{}m-qHpQBQbXx6pXuH#J~ikCN!A!`TPoJkFE87r!_oI zu$^aL$GKVWKINTrX3w*4V4rc$cNzA0_x=C!UhtrO<65&1u{U97yT^-BcJ0> zS)b3?XTB_m*O6)ek#9u@-XMJ-%Nn9{mT;$)HPrpK;vecGdOhT`cac}V zn7fTi;6IQLijFK~JetRfuE@Vev6jr+hS>xB8_F->zKyh++O?O2)=e7d*IL@%?AEsy zrPpX)53lQj{cT!JlRK{u(T}Dz-I#mk^k10It1fots)oLb4w(5R7VIzG`F(H^{zfn(U&6=e5xX(R!^e!v(33KAe zv2-TSWUkR4Gq1f1v=3X!&#^`B2ps>K*cxkzeX^d|CqE(f$%CVpJpUYVJ6>>OpNQtn z1y}O9uXHr}0qOMw>+DDMwdkOm2WIYXoR?(qDMoZhmU?fGX$GF=I$W) zwEID1qy4^V59w+4kT||QU42RDzrnRK>Zk?R8sOKK408JN9_=OK$^S4W(VDm2KF0Cn zA{P(-Kz3`z+V?%m+A^=z4icf)KeFC%d(sc)Kib{uUcH2v zK*SxGw3MeK@rQ(ul7q4r1z!Nq_VZYr9m}b=;I-J)1KzIoL2$Jbb^QrEeHT1^5B%(O z&Zrl9T)E`=KQ4Xlnn7cn_`wG#cL#jW0dIG{>1Ll#JfcpW7LWWR-wo>J#2-4qShP=8 zpx5Ye-$k2ybP|8j!;I-;_B!FSU;y6gYjJuw@GhK=g44g`xA3`jO%G?k`#11eIPB2h zo(4ZZ2cHF7Ls-K^8vNvBC;wYoTkt}3c{Ml?hpnLd{(RoYVe6OTHP%?eeFgLnJ8|uE zqQ~Kw&car7Pdgo!9syk*u%t8kXTVln$ZCf(A9cp*a7Xi-xB{dzT)sp!M0gqi1}d~y zC&)E42JqAHkdO23aoN~ihn)uer^%wptst@K^I%}C%tdGY>C;oVW>Y)|tJJIy-HpV6Vy{!jCr7=Ef# z-(JYK4XY-=d!^XBXQDeA!TfO6lio+Nv#O^Y-7U*;ny9-+PiDSA1s4z0_xo`EJe*f1`bPx>T)efbv>1 z&8POTk)(GYi)9(!1bR3Ze!UBH@CVS9FT9=KegwX};VisK;@%})x_FO=Jzb%PUEx1^ zMpnNSn_51`@q_LGt?fyf{lpa)-8-F^`2=3(Gnbd?;nGL(N3)>SD!-rSA=aiNG31{E zZr<_qsP5tGQQZ-G`5ZjaeqyNW%(Wk0shsatAB|1ze&4{&`^_HE-O$UwLoW{lgTl+} zx5YO0=D)$gySoSfFq%0`4ji<6fi_pmFWnc0T? z{wQZ3;j!V5T;GE{i&z8jZ-+keg*HyX7Q^MM41G9uPVfo(FL>Cp#G1DUe5=h0aBp1T zApGRi+B|-HdK@g&Z)7y=l_rh?x_wjTX5h0HJqG*fJP*G==$|@e%ZcQ9;a{xE<=&Sc z-;69~OQ*oRVp3m{b;&XAji18Vbk?`VL>pKWomoYXT1t#w4&BL>(XHe^4$YNpQRRT& zjk2W{oHlq@-6)$&@oS0SR3NjHj8WwalB~LisW-pEI`|s$2<25#Ul(9H-%2`Ilz~h> zsduE2eay@I{5S9Vav$ef*}=kFZba9RuCjWsJmh>P-dQba4bOa@?6)bBr(kAS2gEP0$;RLT|7Py+Ml68;qYX*?%g!gGe5}Jf6;}oK^e#&>#5v zSiOAc5Hzm^!vpihpS;xl+Dpv3&O=_3;mAuGGgsw~f(L$$bCwSoN)Z~Ko=+c|2ib+A&g8Tn02v4chYRxH!!T)J}^ zbylQUby`=^$yILt5b_IrR-NoZq~Cke&HtA2lgY<^M6~8HH@{f<$T%9NbPP2kUod+m zXQ*oTeF5)fH-=mhdfeOoiJPynht)QHl3WCwtJ|>X$2&zQ^R2YV8enw8q90+GfsTN( z%5&^8a_v_4eQNXhbnV5~P#*n6 zudlM(h}&sUX7Uke)`mrQ(#PKR8}!kv1O0njZPULumD0aIDWz|JP)gsnDW$)!k#6Bm zn^jtWXymh%$fCdx#}5uZ37Qs{O>CiTL0inM^(&EKq58^8L$%%;TWc$GT5F4v-qSrY z_=ME^tR<7f#1m5Nf>dZ{qh!$BDY2`vG_xRC6w4X>6jdrM;1WA6`Ka=(2=ue6R; zH)_sm)IVTz3*$p(-7f16LnDbxXRpN`Ahd3DXdSjo&krjJ{iei`dw)Uu`!;z(5!!5| z&ArglPoSxv!Z)8T1OM84xwfTpFITet>55*Vy{mxLN76&Pk?AM~bcp%g0Ns2Enksv! z@QYTo790w1BJE7N0XS650nWK$WJ37EN9R~po&4XlFIEU2?;8<3d;#Oe?x78Q6ip7V zv7+YP-!BX9=UoHxBlCacWx+z`L;4rO^<2mj80?I-!zdUnVjJzx7+p2te7^4>nY40-<26DL*_ zU>kOX^S^9A%y+8`f^WI?sLw5YLp;A>=H0D@!EL-#euvJZ=j)xxKeuo2cjOyctzf+L zg5ds9p2z+4*$=%dhPL^kdBw;WQ?Lb}z#cXM`Z|&PtDvtJqL=N8UbfuRb#68EwKMd! z!ZUGx1o|4g*qV1UxESbQZA5lBxeNM+2yyWyLTBefW6ROgf@4!BK!+!w!%gL#WSmc9 zyE=+--%Of_gQmY1ofq`DGJx0V=clYySt|2NUEyBVR8!`uUcoANtkNZ4%9=h(xx9|n zC6aZE_KU_!9LeCpE1_1 zjCHH$>iM5j_J@pNKYPjlc)F*);OW+XuExk-qH%7azFShRm>=o4Dz|y2723B>ccgm6 zBd;%P?HdFSq8sR+e3zt$mYvR#8IwZaBkyo|AUcb*Y2fvtGEb`A$baG`8~qHt+eVcg z-;urWMe^a%BhcQrc*5(jtx&oaJ&Mu=yjQx0Jrnx7g1I9eX4HJQ@U5Dd>szjNOK7ig z13plF0$;9%HY@LyWbQ>xpoi_SdiVK3PiIFa^B>eHUKRU*UKdk`83*kih7TlWSi0&F zFRnIkz)$do(RP!6Gy6m---L6lI`spY)7i)K_kO|A zz}KIlX|i*ez?krT9vWX8Y^(5_Tb1i49v17+KM^Na9Y3+60$Ya00oRA{6-AGhY1S#6 zVr|@s{Q4#KVD$|>#E=R6_Uk|E6lM?BI@OXQXftb6N_q^sYv4JM7JPqzLl0K=TA8hBos9(;|q1Oqxd zlrM5@F|Y#|qCOmE{KPenZeQfA*Dltj5M5jo@K)L<@TmVOM^A_hur5M*@dUALsp!Ix zaZkDeU04aaurcVuMxzV+W3A#eZK=#^wNId*2j9w0#DZfm1@ab3WKdKW<@u6Jgz~LGR3VTk&VruX_VgolP27 z;6KWDmB?+Yf#11>S9SgY@EZY7=PvA8Q;t5emh(`K?vGa7etvIX=F`vZ<$N~sLziat?@2weu3kZW3i2om#5V%JF#eI=xA)C(NBWg z&jYVVW?HC!T%G1!=rpCf1P`CnyBW?q@l7i@D>eXktH@XRzW5)UG49eVqnmwQW2Ha) z=!^V_8;hpEJETOGFjpf|PsJrOU+4ORqg}dig9{Hch5O7`(<;^)8{iV=tOMnB`-y2@) zyA$h+@pTL5jjoprX)AkwIBj%&x+kO8#C+)x()*je75dT0nvWX9UeEqtg?+5~e(~+Y zi{Sx~x&K%`zg9}XIlp_dQme88Up7IHh{asL#hclxvsWQ!tj0?4FV~#=mRo~3+g2m1 zyIkvx&Z`@|J}}V2|EgZlSHAmS<|#&AjB>FJR{bjK6+K5!V(JZV;4BB7EA?8FOC3Fg zIRj)s=d=cb8O_HI#w`9UL-jpo)t8;%e!B1F_9{PN4e||oEH?}+mnU_Yr*oOcf;@Px zU*|I5Ya8DxZ#%IK^L@U^E$}(2V?1@C2i*~$Kk;bC&xx&w&uyG9A?8X*zV_Th5Yjw0 zJMuPcYO{ml?~T0*V-^ig>j2K(hMoeL3Qr0|J1gD5IF6U>Oo^53Y~t)U!Q-oLK?YNX zkDhGvn}{h}y0~hk>?Fj8;Y%`AeUOacVZq)gjmL^?M;C2yecyoKwd@;`DX*g4h7$S! zPogtQmCj_y0~QkpSpSy_KaM3IOyR%$))LnOSx}bf)B?>9xFva?kNZC~N8w?t5pz_Y zY5jOBw2QSs&t(mbO=2CVThT?Ti&X#3e7Uv_DT;@c!ksLf&rM%%$PM26x}#gUTYaUx z)y1mQeyTgQ?od7SU;EtRgnb~B^^ecNM0XCtUL)sRU--`pg3trmToAkSTfwgGT--$c zf)S1TT7F9|p6^MI_!d8I=Js%bRd<-SjJ$kj$B@d6bm!}4+SdO|HD8wR;2y>tZ+}3- z8jQcjnmVJ<(3#~rtE!(-zSG|3r<@~i%(2g>UhRtq8DBd-uK}N-gFpNzwgTESxhwO} zx*psOKEOA;Yjwc(f_eGix6axYGX~Mw0;S-Oa6gyd&tmV|03Wk!6*OXnXau%u?%6+& z_rk$&wSk#~qPwzZFgOSPolBcTu=SMQul8WF^nPX>UdCa`Rs_AD^dN;U&HTeie5c$s ze-iy)>=Eh)7t%f1{l!~JPBt2NP8=?!; zmkFKFd>?*9YmsUHl(V2C+t0Lht`|;-k8R-n7QT^uOfrs9vT-LZ_GZ29JAU=fFf?Cr zAQY$52R?@@t@?KaqqH$i`xkxGnzgWI@wpSN+TX{}4b$&s^x4n2nwY1i*&VPS@`elz zVZOrfs!h&WWO?DUdBJ9M3C3=LafyD!uqi;$aS8T=lhuzGY1`qqHyE2@GauLk-9q^S z_78Z#<%PxaPsp&3<~eh;l6N`>XY*}^<}X2aqd8(+aox@G1l`U0Q*}2P_RH{bvu0RB zisLq^=x(gV%rA8>pbgDM*^SU0()s*<64)~^p|c@#N;^9m8MDp|5zc%E!`AI!!n$d$M+QZ-n z{EhZ7uXM655gmf}R*cIX@wF9x$6?JcIys6p^m;>*>( zT~&T(^Y$yOp^;J7hd!Up{ssT9eM)%8zIwa%{4PBo%PnJmQ?bK|9QD+H!5F`Y5l_ME zi8dyCq5~FotBG{>M8C*O!Y(PPzM6hV(mc_oEKhVI?_~2B$@AbNN12kO`hDY)>i17f zs{djVzbmbaV*@?WeN*^$dQyEhXVJ)pX`u+Q8=8sT5Zhu!_f^sswDI!?v4^TA?-7rQ z3-Ja0Z5ojiYTQYgb-a6#Z#H=9n>Kmsm(hpDHF@ZrIfs9S-Re%>f8eRFWgOgp;@FYw z!H#5;*Rdm6;iR!6o_e#7WDg6DrjAL_TJZ?KVBSOvC6|;uu7!O_dr5SH^Zw_&cXWg4 zwgrr9&p8ZVAhHdHW`^U+asav$FjQVue{Ju%EY{@{hulSHzZK9v?KECU=|GoD1)9nvvKe3O0a?9LA z8D|`s_Wx)f^jgu~dy<_#O>*{0CtkYz1k|TI`_3%rC~Y6D4AegbPw~XLr?$V|t$S=j z`5m;AyfHS{Lt4yQ5F^8Fr*A@E z9;Y3Rx#cluO_#c5awsESWa@d)XWl(TxxM`F#~2kuO8d4Cp4-=PRc@7^GslBdL%ZQ8 z_N-8xl}X&uAzQgL`>(%@t+G5;n z5BR>6)X1cmmquto6SP2kmtwgz6U!yC_Nq{%Uv?-WnsqPd@qW;OaqKJoazm}qDDmQ9 zXjHNA%bQf+oSal29ymS}fnJ57Rk7~SE@)H?8YP_^HkD?N%7%X42krbhFi=nc?c9L= zztkDa6YA%^(9Lt8n1VtbEuPf}c+q^65R(CzF*wbCFZSZ{L%5H; zHQ=Wy!+gl^U^aO=XB}kUlV5^)2QFX1JMrW?V>BhaI|@%KKa^W}KUMG5-ayBP!>2uV z6LbK0@f{z%bI!M|;E>~2?5y26EF|8dwlcf5N;bPue9wkeT)x`-3(VXqCf9niS|x*8 zBi@+$eh>T`+)wEHc=f$f_K&p3I;rnN-2T0u&_DG}eZNre)&`=lB)*&B#BV-7-tWOq zzyHC1BcE{lt^S&R(cgw)TTMSt@8c8nQEMPS#)H@|Xzey))4H0tDI;kAVeqD5&9qPh zFcb!c8h{_g0%{tV9cq4z|0_K8z83a`cQ`-&-g^A)e$HCwvX-LvhF4sDPB3eWbN)&v zUwc49Q6_O|9C~q|ySH}bU3?EHXS|8@^-i};2g=0vfO{w-{!ezGd!YM!p|dU1;bm(a z`hPg@dTeAejX#d)?8?kIo#P(3;aYQSl1!Rumm?qP0Bkwm%BM@QCh-fr3wksTSVoo^ zuH^gwqhtW$V@4a_-@MdvFFY}OOdj-BW2iOt=Gebh$~-=+^bzhtQaTCyAEor| zzm(F?r%1Q7#j-9%27du{il+XO`u+TtuQ6loCAfZ6<$*;|x+ zeau~V4bGElyt+e>C|1y**`(| zzTP^}w_W%C#-(xH``5=HjqBd&=Xzg)?Z%wA?FRd?58WQ}Bam-Pzc17~hRmPAOZhaHFdD!gtPXd#m(<`43hI%Cg~YgZ|MfRpds!^-{i z^q<(^Hk43r=Ak>>Wqy= zo;{a*^J&XtodJ~{7KnaZ=?LV6N~!ZQrPO(;QtG@!DRmAZ)mgXhljLFM zygNzzpyKGK+1_(kPyPevS)IiOQ?6pKV=waZXTw4U=fBQoWdE5qBU=Am-N%Q-!zfmW zV|SWnKZtF~a{OMUPfW8`8Jm%ArhjR6PsS_!GRv6~^{qXi&9%8}#^z3bj-uxVSKU3}+lNmWU7wG=F~4Wv%Yf)1?Hpe)g*qow z_jR;!Ep1Ige|Qb??>m}%p;YD=cc4_^t5^$d)&DB)cval^V)RtolN_JO?dauR>5Q$; zFy@)FW(~ATxba8ErTP3=`DyqUn*1z#tDFBJ`QfxcRQD*rFV@ZCsq0QW8u1W(FPxR# zNsMo&(|(L{aX)-$ceP~TqTMaY!u2d$IGgXwjNAgvJcCwdpBo(Q-XC>8?Mn91LZ2IG z?>DA@=n0jA>yMf=!`{q#lqKav_8^-Y9~-+f?o;{xqLdo$M~W1a-w2=4EJ#6N9=vY#W1jbBj&S?O zTA*2AQhnHK{WiSK*IGs2$IEAPZAz$dExx75I1ayp-O!(``eQRX`_V%mkZf}lHh1GG zTgaVB$I#_iMo!x~m;ns-CErW>YHe)A$72EqYnN2yW?YUfjrwsR{n*y|*lNkU%PtPe z$Ns!rr_XoLXZg%q=rWFhH?nnVab-o{bH^k4aT|Q-ZWA*D-0J3>OD4O#Z3=TafiV^p zrsChPxHH77p+A@L{v#K5{sg^BXYJ2e#}4NO7qgB(XKmtZIE-@f-|Ew#)BX?OKd`$a zyfYB;`kcG3b^~`hKlGtZ!Ol9`bIwKd2bj2(cbVp0nw?ajJ0>c2yv2X?JHgJJd30`w z-{lfo6nHF^@ieW)hUG;s^phCmYpkHg_7h-IHg1B8UCh(d+rG2;b@t9Nt>^K+x4j-% zaQO8!y9@Ito5NI-_O^W{&9OU?Ue9>Hi7z|`O=yKC=&n7*+G>L@JnY8WI-MpAKXhWn zEMg#v|E^6+x3S@TG!;A_kB!FoTx;XtT<+TgKcYFr;NtPaFZj_P^@lf@zKS;nnBR^0R`lgd z7&ChqLQu0`*E3epj2+|)XS6nTZr)$WQ=PhR{mFFrF?iAdsqx)H|Cc8}Wq8*0lvf<9 zXP^x#4-cNM^eNJ#&=A$RmNLqNhDE&CNQD;V)JO)NZ>2;O8?BPqn9EMcw|c7aqaLy_ z!{9t|&Zi#5cIPC{E5?O<^K%W(4{>EX&(K%l$yRud?X0)&$$mlZRa@^rcz*B_?}9Ij z!O6kN3%)dYW6uxjUNh;Hl6V5oJGd644(Z{VD=l+QJyz7y-%cl{TgTCRTr3|?(St*` z<@%4pQ@6lVhiCAt#V2}A=a6Kzimf!}3a8Eos8cjYwy!?q1$my#w*{wO7l-fWz4q7L ztiAY)-9?J~pKg1xThRIMX7Y4CO~m1mw68Ln1L1KgWwZ`|1NOZ>#gE_}e1h<8wB9lQ z@9@sSH{LPd!ne!JyL4jCE;ab}1^>&uTmRwfx%D57yl+aD%-A}pxhw9seP!sxmC(RW zp<|=W{;}NpZ|4qxWAI@o?*T}+pJlBzM){dqj8ieA3=UpYY;g1)<|clw9>Dmu=Lj}@ zZmO7>`mZ{Qc;CR>6e?xB`AX@3AEk^jPbs{Q$GlhEm#^o?W0l(Hd$9NRWDoAeUYy3B zoX$R<5jtMdF}R7lVP2xWRR!2pvu|(WEPv8AG#UAy;xSF*o2h&|g*qlv*LCdU*K&tR z$KV&7C!^4UFt`@y2}qqF2O=!EWGmA$_BiHloKtoQ_eX%~C26N6E5M#T}y zwb_FV%`f8~>yJ5y?IrHPvpnbWbkM^*DgHqPdt=z@1h3H1?2}=uqn(27muxsfsn*6( zgR#+PpUy2HHhC}hvt;W%pQn?JElU%8=b8KWeg5_sa?6Io8?q*&xc5wY&0J{jO~hqN ztiuC;(!y9ieb5StFKT6spTS3n*8Ym`bq9&yu+ZI$Um{O1pgSrh-_3LL*ORYrjzXVa zfiHQ&@FrRI3f5P=OCvOh7!HOPSp&ThKe3#5;zRJcJ98Yf`Uh`z`H<<)=2W^I90%@6(qkJcgs*-C$9N85*T6`YsPI-aTvgGYuqb0U7dOTxIljN8<4?SSCZ z%-?$U!}#0@U-i2Mn$rv}UZ%d#_e)7_%JJQDYx@PTypx@ip|SYeAN~k?7ugjW z-l^*`@nm7>Rr+}!t!(_vs@HgI*S08+erazLz3c-W+`UJ!#gN~+e3Fkm(b&qqihqP0 zl(C#kdtVwlmStCvwiK1PybSMwnJakTr1!wzeWVU;)w>M)R^II~-{#m!;EH&e6|{l< z2lCr7I~6l549#q){OL$Dh9{t-X~YcUjH>%FIU}0>O*S-5bRoz74P$Y6ndM_fUda1w zdm*rJm~pAkAF__3j~|#c+r(drZ4<5Q34Q7Xol1jVr9-!%a|<$w-O@3r`x50x)CS%v zM&2HHVDT@C`=fI{^lN!MPH;*yaxLN@yvlmpJIDAHXRj(Lwey@_fk#()x&M*aE9WZy z3U(E5q?%YQCN9fTVzwmpoHoCVGjwTK}XpY*QT@;^Jg$V+23)GWZnuy^W18 zb_g}WqdcOq3}kN|CZ2{n5y6!?hDVrE-5(pG@e8wRE`v|deKNJ2vvs$Ucs${?@ok0X z_=wXGh7ReRKN}p;TrK6jIs3ck8O>$6^4L#w#(AB(z?XLuWIxNTvVn$#LcWsU@xSsi+#yU)8J*KcOli4?}BVMm_{%fZL(4bw6*^&9? z+DDQ1ibtD6J))Tf@F=1M{xMGeY`3ifWGc1ryWv!8sQBfUqHg{edv){79{!f{o{rD{ zMCNlYasMMJq2?(W{sP_QJ;ORKo+dn#cm2j-n}N+Ie*R(k2gC1_>fIjjaa*_6SP9P& z;@#5^gXgZleKgbQOMt!zmm1lVON&x!(AAhVS>w{JOL;FEpmnl=Z^<9ECN=K+i+HcS zQhDO(G>$NORntFcps#>AHF{M~u!VcDMZfn>8DnBvC|=f4aKQ1YbaZvO#4~c*OS5}W zx6ZlUNV%VA$lgU6HR1d#LZ4s*7cG~~iEs^lSLZ83pAJC2NSt%$+jQHbZ*_}-40vmo$&eH8Xxv9V8cUN& zGi+ia@E_k#*x9BW|UBPvrL;FFs=Yj$jvZdvc(D zCcmrsy>3ll`18%9L;KiwUx3ctaaRR*4)HE|)ylc(^?le&pl5o|m-6;X>9I?>Px*6W zkHh_+S2=6^C;Fm!5gmNf&?v=9&a=Jl{d9HYji9fhM-D&ZoHzfgy)1LgT#JT2*`BAE zl@4F}csvi9=;Wzgr&jUPDd4}?gR3DkUQE7oM3!$T7Z+8D?;C{{t_(S3&jYlpl`r4oP zRueM5Q#}FtI(Jbjw&TP`7Hx%=`jLS&a^4?eI&j^SdbFRyOB>ipLw9@^yi^Q2i(Gn$Xzrf_PT5YB6%MGc1)S{+-Ss(ix3wtM zA7kH&&g|lENl!$;&hXZsBMYT(_Gf28Bb z?R>YB_YNISx5x56PIEs_S|3z3zHDd71WF^G%vv{V&L0oU@c8e>noa=-z`k|Dru4 z5nnz$fL{!L^up=dq%4CkRp86r;L9}dV;D~kPcy!yo1u%FN$;dDv2y}nTF&{XeXrln z-1|Nsj8CohDW9#lagy(ia{I0}&UNWW5ovgle2S)bPQ{O49XJ*C_U@lzRXpm4#>$tV zl5-4kfclRj#@sFZ*7uUd317>=wR}(By4T5{!?_{PlUKbxDKFBuu(U07TE7fTvsEK`eSGq^F7@3?VIn+Yg!A9)xQ5?QYe@9v?pFFST{UWu8BqWGVgCPu|kpk@h;!$ z9Z7<>Is>P(FEKc6Y)j%cZ@`5kzd@f3?!Doj`!1$`hL=S)t99!Q?h%J;V*xN40Qa=7 z3;*`HchR3udBL>CSAh;l_=mre%lA0=cVv1O_@1tQ`9=u08rLMl*PO&T!&6L(^E}*Z zHzm`56nwjpI@`{MZ>|m5@6#Q;=|~?$mp2-G%eHSH?c~P>Ie4hKn+H8~bY^MxE$;gx zym#auX@;kZ!QYrYEs}_9nkU5+6|O0Ur(%|58~dzJxQEH%OY-cGs7vSS4^4_47^&v1 zjJ-P7%1sp>?jY~veuf(h<u zI;qw|WySA(6VK2v*b^M+!u%Y*U5zZjnTIs{AK;!tgNXagZ-<8G+3)i^7W0G@-&3?a zHpRNX4Vh+Zve9V_SG%pqQFZ}O(#6T2Mskl&fU{5Ov+O;fKkO6fHu}dQ+cOXhkL5Q3is7&_V?)DtbVn1sg9#>J6#^W7{a&GG1y6 zwxDRqJlaZ2P)0c~D4rJ2v6f4%)=Lcu$|OMw4UkL{&HMfB=b2;@QP1bR=kxwCpFPjs z``ORlYp=ETT5GSp_LOyFSp(AS59yE2VawlJwt(2UwXTSt>$wryR^5^Gd8csA+XMMH z8JVazvJv|G{JzLa{g99Qv%lsI9_cNAa3`jPbWz2$(47_AY3Ie-DIx7$iysc9J$zLR zZshS_V;OAHbUVkS8FrRQGwlr0EAU#0%euR%&Evf5qt^_ zuLLH&+;)1HG~G@xDYi_d^yjnlN#Ut&N}20VSIuTDvl!Di8QV7)<4okf8SLvh_aw9y zSYgE!ZlrC+Ip>~?==PGpI7??Wet1DJGTaJpqpf;;LDYxBvaGrI(=X?4m2gAKW7&^^ zr%lM~?Fnhm>I{nFgEup8=)ZICZ)qO|ti#h=634XSYb4y)9l53WjB1@pn_RTE^0TWB zMi`?H-w)sNnc)h~H<@quxiI(`-{rqkcEdDthmN7^$;-!02PXOOVk>Y#@H%WQL%!cH zc#dEoz8s(b!+t@{#ctq9TkEU`^MaS*`!5{%Gw@!BePEdvzsj_Y*?T#^z&f!trdykS zyfd=nPMw?S&#%E3lmEnp!7;Qm7W&m*W+nH|?FApTmrG2Sd8SVB$lwy*3D3UJk4_)n zSuoNs8c_@z!P3XC2%JEMq7HoaQnLd>mh`xKQy|~v^8FM3)_w=N#6+Vbx*I*N?1*b$$=MHdqQ7PjQ zf0uo0JU`J)tZ(YK`nq%2NPp-EvBN#t$(6&3{hBL_`M3)H6KLRG3UHya=B|T`V|h}K zd*}EEZd~6m=;!O(_ zh>3lQzKS+$TpQzD;3qz+Zzk}~dGL>?_Bj5XUazwcF^N3!vs&&*mThdf`)p#3GiK>> z=XT^tzjI>j4~gZ0^YTr*k9xVWJou~fo+dAYJR=`tgA&g&dwtwB+=5N4y8K5IJNFl| z><0}!YVSA8jxdHgFX!*TJ>Cx&dMx-`mYv3!1nWM){>$SxB<|PVzU~x{>0Kw`J&K1f zy-Vwyp&NH^f7&rejZK6_lZ7~W87aZjNa{w&?bxflK> z*QW9w?X2dUyVi|N<~Wylt~-I3>b^-CKKc07TB$2s-}t|fr?rGT@Qf_xJ5_>Tn)Q1R zbOJpcU~h7r|CRJ0``yt=9)3eSQQ|VJ^Lx$@CR-bb7nfxBIe+z>vdNr3yr18Gep#Mq zsV6D;AhGu*<8LfqR>fw`o9sEUhP?x|Yx30gq4Z!O^_fe93laL)wiG$76k1#$-p{&3 zAJzX~lh!R}{*fyxxg$j~g$Fsq1{Xd|$Szg{Z~0UKBpXXH6LQjWdc&3nM)C5Of{*J=FZmyaG`?wj@9{escwbgyyt{@ainvg}gc zYndk9IJ==`HurtNuim;xWf`{kmF_jz171kRSj2N=8n#DpLw)=le|*2V4)~- ztNH#l{FlGZ)k;~1t|ZN59tD>xOkS!zmsItKl5h069P6(7Eta1#hnM{tIqWxEZnZqa z&Z;i(;&-6^CuGhA@}bFcWlr6>lWzaYD|x_?zb`8UzfIT#TIPZ~=zuH%A2siVNg*|63KFX(NUoFX%t=cN#toky_5Z z;#H=j^vCn&qv`yd7eI{*AKa@<&#Gq$nJaR@H-WM z^Fh7m>D@2{KIG@wB3|bt?g+aMK7=n{7`iU5-0ayONrbOLCk7{88xu@mZl)X_6BMmT zFY7`7jGr0&=Lr3IqaQw!+#7ldn$TG9ps&@8`&LrH!vc3__jmZt?J8q!+cW?eY3?}R z7{lwrgu*dqZu*Q34&fUK{LEcXw*u7`Z-^31F!9i*;^rtqY#8C-QCPh8SP5zWC@r`vKwjKJ_5)eRoWg-9BCI z&ZXwAiE}cVo8!z&@~wA2-22wSgBrZOXWodf(`w>%lWtwZxfwm*`>A=}SMGVm~|11EU~H0(anKJP;6 zd;(1#(BsZ*H5o^X64~6K7qWchRhsIakkmH+SW%mwK+5 z__rUQnDr>X75tj`W&ULAta1F_9Kt%snZHkv#r6G#^r@Qr&+Dzu$t7c?tatKE4SL6y84r++wNlqZ0gR$Bt1*8NLN!`MOtw4}Wm+q2}x01Lf_M z={=o+L5DaIVNKL|rn;0s^!^LjFSx)eHoPk^BItv6S=^r+p&fW8>tcp|32RXeF#AG0 zNh3Dc)fsbz*N0eppn(IsCU71h7k%TaXr~JO@e63@)4s?$*`2ggsqyfQ&7`88)3w+? z|4Hn1_Jix$gHo);IxqWJ!}_ydH6TnC-#bsuHlrfivf0DHc2QG|Y3<5pkf zCqG2&|292eu==8q&h&U=XL{1?*F|%h3%+v>WBkfFc;}1f;0fj+pS!x74FAfor@3=5 zg|tchmv2nse?9!}^zdB2>8tkd@4$x=xhKh5gAKcXNU{vRQqj}IyY~ga#SBmQsC+5M zSiwHbi{`EYogwl(_kcfcMezgJntu`PY{19r`IJz@7T%r9TrTv4A1q01j%>MvIZh8n zUbdq3ocB4+c^~NDz_MTO{#WI1j{K_`Kii~DYvx?R9aE9A%ust-R!A_hUE5-g`mZ^M z1_lFTXU@4>EYq2DpP8pD`^T&+&OSc(hQYfuzbD9x;X@YlzY_iSYTa3M!#H=oX@|aO zzCWg}!G&&Rn)eSW6Me;SLG%4?EYFp>o+j_D_%;lFIPJeinPhcC&%{`g+#|V9{jEV3 z?CtX0M@ZG5ZuGk@#^)3I3g@u*(B4ZO-`9Ods}D%`u7giUnBP#r*pT=_#Oo`n1pmbQ zD;JM;&Tn>*2OOH(zjG!Fglz6HyXHcUjXBF@P)RSgQ=8#hq9f-Ro{ev%-_m7Axv-0{Pti^O=!9K9GIqh4<&Wq3&=#`zX+yv1 z_I!;SxXdBd7)}E}_G$x}r(GRl@=w0Adq7=HBtQ4L(iB#))o8Uc( zyS8}#fEd8- zQG`vB5%f`L6Q;0F8dSwZ|Ques#NoRivoRck3 z`r~BscA~H6l6P#l#t+>1Rt|d-FB`o%$F>-^*4jeg_l)bS?ICYB_U;awK{vfQ3EjK9 z-dxv{y#EEg*|gnVZ>E2(8N`b87Mx}69&(mvzzoX=AV zZL0pE^L%^{(e1@!m%C*Ts0=&H&N~9* zcB}}DJNcp&9&hSo+soWKYgA{U71iFg{NqYU&7B!)k24$W^K^Y*XDq0nhfPm%U20kd z_oiDX+O}b*PkADHI<``sne<`L6i?nOyTd}sCisdQJj}8$=i3{w$8wM*6__4Szh!IU zyl~iu?}m?cxo&e(NVbJN*dTU->lI~p984YZ*Yz5o=-Ql@!WOYoeumf)CX%N!+j?h; zvyZ>7i@mn0XU8660%O)#buM$Pff46_l+y2FrNH_^lcw0`n>5{SFZW^3`tp0FG?rK& zyqtv-Z6eo&4>2w_YKrh}zg6+Fihab}yL;yb#g@=JoQgt7x}^NuOSN zaM@t_-IW}Sj3Dj|Jh1TY$_>7|$=ko`;rfEa(5}YJvG{R^Pp4R;CRw?qPo`M&HYQlh zk6^nXW)M8VTK+NrEB}N}*qb#7zK4HWD4%gVx|zp5{5bdX^8BNI$Vc$a)dPX$2x77! zV~Yk(7I_}6Y`k(TcC_WMq|5G;YX6J8lO#}i9QV*W-zunCoCi(c^-&(#%|NHQtyDTmJAHe?%YxySTGAWh+Io9%k{%2ZuCHKf( zTZg`+vG_gQZRxS@!Y9bIX^pYw_2wHhlX!O9o!OWFZo4xF^51QDW*Yz9c4uZ;clGw= zu9e=can+YUko`T{y@7Vi+_v($ucQxc-x$~4jdAVW7}ws7wD(`Y@Imjh8yeIm=V={weiLQr@VV$u%p)0#p=#sfEm) z>;uAWl^;Zx)jfYFSvO+e>99$?(<^ALq5UUmA9`A%_SrX%)r%j4k9w9z@&@~Vzyy25 zIt#ni$9~stB{_h#Suwq|C%H@a+F)bf!a0N0*vk5|J_Sgt_jrTvk8xm|PCI5zgFl@@ zHc8@n6lbg=jqsTs#DO3`k-Tcwj8h3#)WiSL+;g_co7raM;@;NizsgSJC7yM>o#^aD z*@@N}I}vhrE&K`@6FU+6NytsI6HQ4*PLQ0YZ_~cMb(FEl-|q;ic+T`3WEuQQ_E1-A zQ$6{+IVWUzE%0i=CZ_+yT1~xw#oZ5E*fr!o;^DlE@CEzX*2&mcNY_olzM|(6>?=yA zVqa1ICD>P#UW$Ds8GN0Fz2t$`vV-fewG70@GKl8}@Je>HgSo$%RfP>Djqm&%JIWo{ zP_D#=G7cNcSJ_VnV?W95WD0o&%G#9G+1^(vgC|>*mAhqu!pSRZ%PFg&3|^X#4QO*m*%YTN z!Qrg|*0ntyWhG8oFNfDIVyv|tWmBE9WQX^z#*S2g@3s08D7<844c|-XXWb>+kz&%U z1r`sk7(D2xx6jO*nC~aS1JZR*fd_it3?3+b8az<`GvI;jYx=(hT<8lfY-R3+1LA#i z_b({l??BzzIRsc0i7&yAWJ{@oKP4x5#tDa> zCB}y24EYss-aC8>nE{!@toNtm)_(0_Ea~gep%1(huQfdATZzF#+X7+7#}#{UVD+4y z{CWYu6yyNWZCjxuAL;ygRe!7M6ugv}=~X&|EBWgNWFpzyTH%q)up=d5W0=8t(wM&9 zBaORc7GQ6KM;t;1l;13J#(|&t;DZ-qa~lS)K=0|sht>^&57lT-MmD!I@S&6NA)PVG zOMp)8qLM{D6T!mBqD3|_6<1YUiUZ#?PX)w-t~yjr&zyi)$t z;FZ#6T>Gu?DrUd^Jvb%YIs^?$KXB$P$(=Xu#CofJ-kad~0r2dfz~?x2)R+zTE#B>J z!+i-G?#r$X_hoFjJ6#*@HP~>k{Sq5)%a_=22Ml#=xX#@(x*sODVYjj2_AgBsI;-Ub zI5_am^Y7H%THql#IKbfGpP|DjJQ}$rRjKB@g8zc0;C?4`b`%*=e10!$YFi_=&30sd z*~m)zHz%#;Y)KAhorVt}F5< z!_b<+d%mNw3D?nmz8KeK+XUBJfwRs6^x&JZbvK6Zi$yoYGt-_{Z0+r9@SQt{N!4$y z30hBND^Xi|S7Ws(*Sx9ix@qVDz;*MAbN!pkQ+DhsN!_9OoBm3l<~rsjw5>SwTzOiv zWQD!N{WkJa-^sc+K{0p^oKM?k{f7qCc9}bO$A>w-fwE6aKX?jwh);NXIku_~$SY=C z%8SANY@Yu_zR3gMG#^vR)B0Gq8NbJZ{-KlQ>_541uEKsNoN(eo<=A6^b2EKVKePw@ zh+s{>>8t5?YG2PdgS(Pb82>4Fkmkbo1#@BKp9nGLm<#y~#?OOzP6BhVJH6{1Jjb|Y z2irpGz&PD*KwhdHahB}N&RP93>(gZ&cIE*g>4&;ovMtB8GY`Y|&m7d<;n0Za=o8V= zi-SU+Y_znexGo?&^JdPQy_g#M<@DL{@yxOBaL1uD?e_wIhxRf%Y|WDCwa&!!_c~xN z8>058HE)7}Y@b?FG_RLa|FrBb$mu!I>pbYyk<%x*^h(?Ad|ygk)$?86wQLFPOrlI} z$8hF8^2U=_$NGg2$MV|;J+@vx4PN|;)VKQ;Uj$p=D{lYHyWq#uy!SQUldn$9UaI@+ z99nn!@)~stV&4NF6)Q=7JWT#z@=rojFB$tqwjJs={!`4k`PM@AmRK(ay6q%bKQ-s* z{zKm`AXe1;(V@|__X4`)f{}n;xAKlmh=+rSRB8q+43Q`kYat>&v8bM)HY|-3=yRCgCV{ z%&jq-Y3fJVOz{JNAI-Uq^cr-`vHS=;R>k^J)i}utSIwT{uRxYo?C>f0{}7`J`6#6n z*`nG&3gZvi_m}lVk zRerH`_(HTJw;e9fxrEqxjx76pbUn#kzotE8?hJpPljEx+Dze5#1w2~Trk%YYM;pdD3rthZhl#HJ!#LiFN+uck9@PFUt?QVUvlxw%Jo>$Kk&^j z^{XpM+%Nn6$9EuCgNx>TDSVHZ&-m+bCJ`9y09Ri4aEf2{_ph0;q}oNKnw$27^wQlI zUcIu%@W4v!0`oLavds_Sf6O+U46oH(7x0e!)kYb6Y?|G}%^$2bOW@V;^i7-x`;2GR z&rm+LgtjqO^crHEoul~yexqcMjn&tGx6V%4U7^8cN}*S9x+=C0aDuiT!~Tl#)!1LJ zpby;RfbDgMvAxP>BRgt2_PO_s|5uuQn0It$qE`N2^bH$b)raIsMo`_SptE?oF}g;i zZ}5A^@k{&e$U*!NaA0J;^elFnW=X9G| z+UjDP!X8z_8KFh&yXbswY<-zQdog{;yc2Jmf_J>dJG-$RO?U0R6?s-L7kh6V{Vl{c zEL$%=zfr%(+0&m%dvU&5>lR~+%E!hu1Y1-BHsM!jFTgnj$(|n8j~a06i~7?G{Y-lv zhnNHBtcLhQww(w*z1ss>lbG(?;V&;hd*bW20jo!u>(k)lQxgJV`4{Ot%xU27yq_)o zZjR%JPfVoV%qKCx(j6R~qqLGUoT^`ee0GR-|9-X;L&G^&qqr#IQ)zYvIxBu4;bp8@ zvL&70%fbKo^g;U~yTM!0n)-Dac{f7S+^2D%4c+v^Wb0Pxy?S3ZW9h@2lbrXbQCEC5 zrUT0#NO3J|`DVNhJSW+yKY{xCP88mnRREq0#b1{52&QkG?P+1ZLq4bxqyO|a`cL#d z&LJ=D6N(P&8;UON7a}HXxGttAp(9O~{*#CPvw&D#$bg!I-{Es8`p)Bf#>WF*Jw)@z z`Af}ZO#kUaUK98bac!U1aMt??=|IboW6XRy`{8HP=M(UfSiSD$Qz@^bTy0C&l8m#2 zyu;n!F?x{G=6#gEPx*9k-_e5}o9Os>q&J&9d_OjMe=#w{o_iBxergZC9@rT5jalH0 z!8QKdj8%E>G57dPf-{_{bYT*GdrC;}zF1o1f3Z9b-6Y-UCUwA}5dDNb^l0VV#HTFJ z;GAHJy~u^ze%dm6V+yug`lxS9R*C5z|A)MZ^j~?g@d`$j8gEy6-4~Gnzduj=#x~0io#6IByyA9pc-` z(gT1|gxD~rHnSf%(-Ur*-PPwLQkI6FNjhhJI9Kk%Z;hD??Kh^|xAEWLMQ`?k7;^^> zx;`hRZ}IF1RY#UP@izYWK1wlue?{ z^vA{1`^Xy~%VWK$MMiy^yixIO#Nq90l${q}X6Dtw>BE!_Q(My*D{EO*xA8P3I^&U@ zUow5J``wsKzfbLg50#6@hJ;_j0W-&v>4&YpsBJDXJv2~T;?VJf(07auBKV(bTv>uQ zu+C5l{-zoDXV@vEd$IHVch2YQ{E=jp({XaM6Qjq-kR7owF0J9-L)X{fB=(#4`fz5H z-_QBo#!q{!5hF9_me$})fQ@mE<6DqhdOgqEkfBGi#!2q1ga6ePB>MkaK)+e9BuDQd zX3XMyoU;PU_}?k#;_J-XB46j*-F*z?-$w&It(csE%jS~|5pOkt0O)I*~xUsWfsanV8{P=*YN@tPO+V zKgcZm;$(f!0Zr(TS?;7SI_Dx;Uq0$r4sv*P75Q=U@YdK_17zx$JiIkt9-crOx))Dv z>5goTb3JWH9{z<%Q|)_9n(~!8gex;|J%dli)|kvJ*!>S>=B<*M)!!UsW@0QxH`4F+ zJzjiXlJI#+#^o;5J+<_Dokqi*EvxZvdN_z-R{N=q@9Eg4g&9)eVo2{n$wh+eSOMG=z2*xjLWD3LiS( z$e&+Xy)l8XjqahC?a2Gwaba->4gCK3p8m-5-|!QE#vds_9>E57vcT)F zpPk^3EQY36c>HFK@mdD&&ZHA=DZgq5`kwBgDSnbO$%ba;U1Vq`ctIz%5c0I?vLE&qdP~bh!Dn z&swcm&6B*TrF*~|#dfa%ZziFaRWHWZ`xgAfxL;d!aAK}%WF0I^$SQq`d+1&Q$5y5i zKM1^1?9v?09JaB4s=B`d7H#O>I%6)~Ti^buU^c$J3I5Xy(ZQFBKj9;nGK*M`oEck6 zTyl8+k%PoF!KX}PmhE#b?WiBvu2Q4mrDU1vJ6MyL_cCxmvf(c@Ui44J-_IxiE;s*S z<(F$-GwdI@c@HVC#EQNTJXR%J@YEdpCb#UzDw{<86}0cQs+PL>tI3ya9C_K1o5gE0 z@#~RY5&pP+7JX}veV_Xi@r5(robSH>efOJB5D&V!JV05d{T=d}-EZyY9o@xqruJEL zsuUBxQfUeLs#4b6LeeaLf`xR=seCKK_*q-3G>5v!O+GPc*j9BeX|v`QyiC8F^Gehk zOZ_}6z2P*l-2-3NeTBZe?>Lw~_Pri`S8nC;L!u5jt`!{cFyH-ytje zKDB~5_(w&~vVxiD%gf5JS)eOlMqBcI_b)F}TgSWW$5GA!S?I@#{bKL~8B02Hd>^D2 zA6oD&?m5%Ws484AFDW(3bW<6=(K4_uPI^xTkZs zYS+=ZQ|)ii9yW(!&T~bZ7ZO)xm1ru>UJuN}%rEEHqfaJ)>*C?)+cl3%*VdVL;zZNl z$MBIOTLR$^5^0B6jk+ILd`WGHPwW30+FAy#>c8k9%75qX^IYe=9Bq6=8z1wHThM#4 zr7J#Nv0|51g3Gea90Q*PC$)*+*n#`d*FJuHMre!ftI&K1m%Q{HygUTmN}rzG2ipp3 z34VY&+iLXqixV7P_8t0EjlP4-yI3()`QOT1sBY)pCT&qR zm9mqRsjl{5VmkdlGK<{xN{__=<`z*2Qu2U zz}??lYVLdqrnqNFBG0FX8g?0dexK3j_n^-|&mBU4VeQ`xjy+F(!3%x9b#YFpxz>u- z{}<;+{?c8SAB--47QBl1kS+{=3JisBvZdb3e+M6O?S~k52kyDLymTAvrw*NyO16#H z=gqs*obx2tQa2t?=XabhitWF5;^7?X8XUd0YuOTTZYE{%I4V56G?v%@8=HcQp11w{x780e??r8 zuhOR?MeI+xXZyeP590O@aCDuj71R71k#mRQPnR>ms%pk(=WTv3@Y`OV)lB@$=oS3m zR)T&-Jc&m9DMRQ~AyOOtV!F=sp6@De8{G`myRsvx%J7wJYfj&%6v zyS$GNq_God%*IY2nGl@+Gi8pQV3z;*q;L99z}F9XF9}tmM}(%KD~)t?rABlmoxwS@ zxu?HwPY-|FF0cRCL2L-mdi*DP%kNX`?&tT)e#3aTlw7sr4P

    Y-8wT6F--bZEDC+Szw zj8}n+=0|1L6)yZ`!ON{s3Nowcq`sNwwB5+8N7}8>7Ub1%BKsjory#S2eHnwoJ}dIG zh5ioNbqU`S%|BIkO*ut&ogXcO?D~~4J{H$ulb=AHR1f)?F5kPifrn|Xy?qYeHD`8E0)-yv;d7Dh-(U-LKtu(izvU^n3TN#y{)kEjeQ z{(ky@PV)UuSvASnx{d*JVj88|zi|IAow%bMV=H=dfIR&tN~@pc`%#%zwz4q1u(rJ1 z$hBq2wWa@2uFXQOUA;=a&(Zg5efN#5+l5?P;>fjAz(0pZWZLi1|9dAHKgktiy2}?J z!(Pi+Idqvmywh3JQ+yod{gY?Ow4abpr?1+5iqG1IIo+2zoy43@W=^Ltr-|V=KMmc} znbVRXW6uML&g&RkTF1N|Siro-FI;qIEB@e5bKT2zJ=b@E7sp$D!+z_c->5Fg@1i}w z0)I5Ww*zxzI`ncE_L|CxuT%fWUDy`E=p*69au>EyutX=GYl>t$Xq$;G3a>M8;Ft{c|brne#J{?<8;i!hP0Soc^^qns4-jbbEpOJO=-)ui)R*c!Phk*IiCM)OO*V zId@I--_)1+uatV-tCVrL$D~R2Oj6;M-jl9-a{e^N;2Or_YQ|(LV>1Pvb}~BcRp_+N zesske^|5}l9a>O>Z(gT-B)hR<9y#Y;Bx93P-W z@V)Rd>kXs-?y4AByKCu4$r@|f`_MpK32X5$kcXCVJ-~JU$Ro%ZwU2p%7vArD`zpR| z+G_U8vVP(EG)x5EK^HcRi;QKS)ba0Fx?p5>#$y9`Et()Xdzk45hXxGbKGn>xEPK6M zM(=BVVJvyQ$aCZh{AB1m`I*lGo?za|uvf;u11@O3dQD!sE&dUuL)xFUp1kA0V(0Qp zl+_*(*{fTmD=fmc=w%Ln=RfJDNzCELR)vvSJLhou$T}bE8I{f){t>v;0$mU3Y&?`h6DqL7dFlg`* z=m_6%&d^kf+*GO*SvZgMXD)5HkN3jh$`R(0-A9-dr9Jq9juf+I7gNTLc8z*)>?jre1XCHGHfBa+?V3B zYv>7dMLw|F$LPox&f1&oKo)jbjmK2}Z|F&HYqO)5Ma#%fGsSMjW}^5)BkY& zZH58Y`~dG(cwZhDxqx}A7`aC0=;GMg(#>4Xwx49aIrAJj{{Qjz?(tDocfbFhnE;b; zC-;jcAzCs-N)Zt8a!6(nf+#4UR;f)CEnsRXT8n~6AgDpwM@IC)5?erQ&1|&N$DqW^ z0ZRK6OP`jDRa@I*AgB|fRmv@!OLN|zwfBU8V0(Vg`JF%J_1&}f+UxpV-}Sw%?}Bb5 z?C%!uT|PiEhs$pqC$HAO_J_nbdl`SxltboSw!NtHJ8(!ehM%{N*0 zeVxByet*TgCi4zC#gp-#q&>nr*+%1j&dbk~obrg_8&*McYR^e?(x5p{uEeHtAo%dA zmE8T+iS|5PN6?~if}`B-sjrO7Sax_?-iu=R%wEIKSU5`g_i^)pMS0 z3Fq1F;yl}U&a+*_dA6R&BJv}WJR(0L$s;~enD{7yi>cJtc^P1C(yoNa5ofoIOP8Ja~LmEnh=FAXC~8&~<$t7f26oswdW zxLbWQv?$fS6MWTrc!V^`LZWSzUfM$CW6m``Q@ykYl{VO=Z8v&pbCkv&B>cSIOS{LE zmlTt3U3#rr!NKr4f+grW1qZmiP==vzsb)>)TNxfr%d^K&mj17W_c;PRe#OwWT)Wgu zmySzwrP!^PlP2Cue(1k+Y1-*Hk<=qv=Fzld+NSX|ca718|6$r+>Rcg68!r>2&6f(Y z#x4;A@A816ZTeO;@8rC1Fy~X5`&+<)o56*zL)&hGw%rJh+yJgj;mjQOkVU!~I`_&@ zbeiB;9Qv)X>ZLYAW9VCBe%U3hN1iW>jXQ=NkULS-k7mx3>}O396Sj3L=QKWmr>X_6 zqLl9wkyo@)91f;(nCUDqr68_!?(#P`sH>K%0*1?OT#V?&wH&5adb3E9xi4J89Y z`zh;{l3}r*2h!iz6iMH>pEwMMCueb9FFPdOqopV(^y$#tL9HcugIbEt8+3SgS_t~m z7>B28s_{p+)VOWkLtV0o!&5hC-(2kJ)ZQmuv_S3tBXr~0NMNVdYQUqfn@Jl^J$tu` z?vl>DU%-Fy?0bo+qO=+yc^4B~9ZI^*)#m1bv}{-KlMzx3=+F($?M zaAvdS+NO64CF92#ACMFqNUZq%CG5|qY=*s|-6s!MdAgLXk6a=DvDO9n$PxR-@HIs# zvHeBd+c(YF5tpT7M@&a2BU$eR^~pzD&p-3@Hfn>`;~VTxi`I>GjzGU$J0kx2o*i)y zY3gSM&|OdXThosVDC@W0yRV<|ZZz+g4+|g1X)W zzpQH60Ik;Iq1ETG0a~o9YGRdTw+68VcJMy%;HTuS&Njg;JIPyLt7y~LU7O&t&NjhK z@WEDxO)$$Y_VO|iCoLc`EqbtG}MmoC(*n4j2Z4JAWUk1NbfsCqcoF#mt z7x&|Fz6;;S#%#vXwA}kgpjStk4`aLIY>{}F z?tGUYbZvKsc~9Ju#{Qh8`vQ$v-zU1o^OJe^9?fUE-OJ1)ckA(AZPp!;LxI---Mjj7 zxc4VGuqBW)@Zj^l*j>6mj=j4TUcPb=dq$w#>eDyD)4hy=@ID@}8WXxg z#S7R+|6Ko?3}gQrN8ECFjbR3#f6y0vmhV=o{lp^ImLXlC*6-7|yS%}lsYf~-&%b7U z&|ANi=)cdlel;HNgG(A8mA%CqyG;hB**`TgyR=K6u{g-<)g9;dzPb>`6EQwaXsSL)OUa@rPo{jUa| zxZSh4M~TN?#6BV48OeL^4UUfDJw6E=;@=`hVWF!-A>Pg;2R!85y{6RYZ5|$9NIuD8`0AuN@PDsyZbD_m!Pko3)@*b) z`0P~gl3%dZJ88mxzqP(1V$FWmOD`Z@v8Sb9Prm!XCvFQ2I6r*l)+77;l)3j@PjhqDC3tWep`5!-{qmdm(pMP_9fr<&=a@!Oq!s&Bh(#6=DW+6yuGA* z_ob!{lsB64@947diZJzG-PiiXZTO6+EDK&>VE5#uRnwAlgM2goDr;kvFITba4y=)1 zNubZ{Wa?QcKZW&atc^T-`6PE8WHENDQ^jv^_aJ*rU;YiImi@p=(>r{o zcFA|KWdQfj_SCsR|Kv^V4;g)z7(3w$o(?5I8QSN$-{z89&Zp--HHP`jwU1Jk_%PW) zk3z2`zw5vF>mZB(L7se zjilWHbjw!UZ5!OOiWyCEPK&&G-EGOy+K2-v+T9>_kQ+!GftilkaS*( zk04(l?uKb~{eZIU67QSkd=qDljN+Z*2mFXWvp$B2-jFT zlv%*|jHEot2rBmvlyCG{*npwGlY*q5_8l>HZN9_)ZDK!T!=3G<*5>HjM(R*~!KWls zTMdQ|U6&dSlApMJv#nq$ch+zpVsd0|>C%e1hlvMW@3$ru2ZAZ^akKYuZr;ol>!DV* zM%ro3_*%5>yWDd~`KoIRx`)+C)`oZB4Nhwp{aH*sN5GdR=;&$hX!Csa6+by>>MpI* zZRk3*PFn)W1C!aC>^>(CR!&Ao4P2&J$)o8y8R42Sm(G-M?2}kv|lCccREie9@dPtr@J5VHDoe) zGR@=nk;!JvwI;DXs)EG8s0id%ZR2j)myyq&;Wv-p4CJ(88NVkYt#jkv~6}nvm5x_OAl@gf;Z-(2?J#I^e5c zFm-W1A5iU#CTQS&ce!!RS=Z`Q>V~n<(^Pvc`8XT9;k)Fsv?qR261}^^rR&e~PJ4C% z`sEX|f}wfRO;;s{oM|bcqf@QM*4Mia`t;=;;m!j?rH6BMG{B0nayj zG$|o_rHEz*uurV@$8={>;vR)5l)=3utuy|q%#@4FRCh{df_D+Vx-=A-2|HDD#HFFH zGB*Z4&hVX-oFrRPf^KU6gQwgP{%$J$lPw_IKDpqOf6V!eeW21E^Q#z})!>BZi@Geq zQ&zL?*^6}^U|DCGP-{M>wZ+(IzEy88%F}=Lq7A0r9sN4${eXHC`819N)TcI`Z(xR< zXJDG0V_>Rn87RB!^{lDuSX&cWV-r|w*P>^<20i0=*5K8w#c}8v%kj-{?I}U#&$RW6 zGTO5Lri`{sGMf6G4PTlKU#d9L!5*dCgFattxPVytkFLrQ?QIIw<|xK&d%(Bz$W1=l z?}sJ?u-#ix@Ou5ddy5&DR6CbGnt9xLQ8fM6Q#PixKp#oZBK?oh>jd9uu|HKm7Wqk2 zz&csTd=~yHHTvKz*7LjwWrrD?vKVzmw_ntrSwQ>FOjld}xo_%#ormF{6;p|TC+O$h zm`ainMK4TSI(X;owlVCz?DdZHGNWp&*^zP9lwsrW3*{Fa>${$F97EQYdwYgQ$GQy0 zkG0$yX8L^V_{L+i(4#_Yq$jUO{#n904jmX2Ehg6Qz?q6`n^M&solx5#PTB7VK6o-w*{9Ao7;wUlyYEMaY1@#n&)=o)}25xO%A=^lP60wB&Hog_TmfjsokDmBO`wH>D zi80*tN_j zA#s}@$mus@ur=KDYv%aICU2gSnWqBgsl{tg`B$ysEfvs8<};tWkhcx;+9v!$|5%)F z4a= zMVsQg1kr149Z365-mp9WLhHaZo?mC|QXVW_9uK5!C-&fDC4uDaJuX=N!~$QB)>`Jg z%9poPIEX%H3bg5od|y^;*21S(&V!!cqBGaJO8`7zJeFR~dCYQO@Ui{WMT~*%igy$o zvIKqF>EG&4M~e*&j| zkQx1shc^aSF3O7D#ecKs3a@^Z74?S3XPd#JCSqSD@UEP; znCAl*MT1*5Z49Pb!xMNoIe~}p?B)I7izORzmqfg;HN3tPt_VNQ72kUyyt%;>#q#c{ zxM!#Eq`vnlJSpdSKl46=`A%lO_oJuzgz@a&mvzm&mx24;S)U7-_dUeD>dyLHz`QTi z^TG<~MRIGQo_!TerL45^vG(?1DfFwt z2i-w#PN83|^jo&FHOTEL$$Cz0Ez>h}Nar>7(1(=1dd_N9e^S!*oY`8)bLx5SUNlm) z!E47o<~h|K0(2_I)8}6(r!-?{9fZ%$4@$u*JerX^BqteILV~1h<~r9?cV?YFz@{bni&hV<>!g_ z{CB2m-{IV@muKrJ^jOSWwyzqxo1s4Kx-DRNHge%)N%RxiV%9NxV6bQQ(&R$o3XWj! zFJlf)-m*Jby?PS+_JN>p)k?G9aqld9^NtX7c_8_cM_9w#N&BDB63xM6_VoB}_GBxV z5+`mV`r~CAplhN@tD#*xsIUGkv<(^6(fjSZ7w=>tCz&y%o;L8I7FpEA&r^GRI(s}a z*0Ogfe>2}U^6hTQPCRd7|EFBW$?fYec(1i-cnkVl&z_$=Ji=a{*6I^oXa9HidDam$ zGL^mD=<*j@BNJt;rHlglSx=m%w;@N-we&y9xHjlk=M z;dRU4b$|11@o<{^!<>;5E?j5&n`2KfFx$Syz&!hE1Jmu754R7O&Bm>tJE+aG;?AHO zn^@{Zna_e^_7CVzV!ZO;BTZSQ2^bB z$)9Rp&)(65j;5Kp-bcSRryq8^HYUCDN32U5yeutE(5#E1Pce4)A#26iGezr`Gq?BN zGbsAKe%+$g(5xp_zviFkJB&u~j*|%h+ z%1mAo`N3ztTI;^()O=Y6ziG3`mFJr~lR#lbtzS38oCzrUh3 z)%j_QkG5ED{i|~sTkB*K^zs+f-IvesQaWYwQXG1fufV=IHcjCB_OuczKh zvnOTSKJv}#Zn2IM^Ia0nX1?RZHw-Fm=;kY}_imdYxlA~qa)kqlHfw&**W59$zPmSH zxtIK9f4S{Dh3|d$A*=Yd^VJ7-(_L};!T$Yu;l^~vB;CH7zNoFT4^X$+&ySO){YAXl zbo!+65-%fKtiJw|zG|+nW4}K0-Rs`FWR-Wucf{T>-)UW})fo1UIBNvEN1RoH-6GD9 z1p^VMPOv@f{7CS4*jXWXH0(SjcrffN7yK;jL z3ceI}W(%$hJF^6z4?7yqrnOMynM8CRB*&-=dj?= z(au4^!K0l6f+eG!&jbgIc0Ls>8tt?S=8tw-1an6_9}8xVc0LqL9qlv=_8#r*5$rzN z`9QGSXy>njfzi(Ug6$FKPlCrI&bxv~BhDWM4@R8bf}cg4or0|q=MRD(N1Xo>Y>qhF z1wV*5ZwbC1aegECZp8Vu;O>aCRqziHXN%zWh_gxXHxcJe!L1SJb-_&$r%~|ri1Vu8 zs}W~|;L8!`WxjYOtoS0x;#CcwDMa223;PQyGR`7Aw zzu;2Vzu=>+f595ozuq#QKFFR^@9It0|1-U5 zLSL%A|AOq&V@=2{oph!6@8fE z!z9Wrpj^$FVmQb?lRR>KBYr=_Ge>rBOytQS&t`v8G|RLh!%hWmK%cAclE;m4(Dy`> z$8~iY5@}a@X;*f25@jTtZ&Sy2wgI$N{K~Zb(8T}AJo&IqTKHtUb8!qjL#|z4OndT^ z(T~x#QM5_2)sKSdu_!$+243&@9u5+AJpzhgXxkyhJl3U(T+_4fYd$C|LyoSb)hnbJX=nWl4C zUv>tIwfA`tIbLm-Kao3Dr}PZ)Q;pnTH|%NJmK7Kn4fYN$O}lEWbEvPiLAateZ@6c0 z^p$(M?A$!5%iI}{id-5-p z9a}#B(qD<+%qx61C&+qBng5`+Q$+tJajtJ3a&cxm&4|^}t+YLSl}XZ`15VC0p|r=@|4X?SSkG_ptbTOi zy=c@izJGvwG4xJ+VZN1j*B*5AYKP8ne#ZOCG2hkG zyy2y-B28z4zU&MT^N6nN^!4#<53AuI_Qg%5zS-sIUUd#g=N1n`Cl&wZD6~Vmm;}w) zO?~oxlU%YN`ZAt9qiRoHRQhkx!coME+swRw`{B_Wf6(4OeB8~&8?jaQbr@fLm&fmT zAa!60`V{l+ZJoXaXDrSHshuy(ow@Pxo9((!hL^?uZGO+X-=lAd&YjuD{g>-LhZaSN z75Ck))`-7g%lX@trLoVsXQ8jBhwsy$Gq+u>1Djc&R{zq*YV`1y|C??3R+ibftVMdx zH2aq2$F5}cGW(XbNYA~^zGbl|<#U&a_ARTco_m^oE1iDEjzU^h|Dlp&nHgy%GxMgaO9MTItqra!AOZ`oZ??a65+3;u$acU*6 zo$HKG;!KaqLf=zWL%WLud94wk_L5L&jL~b>6CVLzh(?2B!@)7m`W#~(k0W2#k12Ka zIycAa#*HoAn8LXTcYbrDLkHP)_kYuhS-#CzR(7qvQhJ}6{qbwZz8hq2X5?7Tj~*#y zedBXXA1co$w)gp#)17D0nHgcLaUOd6J=AaN_Vyj_8)(Gl*gg5zX&aMUq<_s@n%rVJ z$qx4rKuZ*VFxf#5*BjakpQN&K(ra@Xz^A*Wqc7X~fA3y5-7hPjY}wnzE9o4Ze6pLm zai>HHI=ER@WBmXiHm3R_;Nw8rAAi(pY|Jf{{dbvc!w=rz+JDV|?7{c*U;3cLy57dR zHt(Q}Pdtq791$MH@k?&0u|{G?Zj6@%Lc|RS#RvF9YKv3k3&9Vda~g{t zFbluTG*ol|hKF6i&W5Dg>+P@gD|Mb$zNE04OPTAt6T|t`qb#$oZefr}20RahqPBwOe(hm^^u{0mk?^ z-^j*sDrQ24jgDFVkeBHDU}&N14rx+ns!;Zlz80LHB*dTaQ&HfK?~kT^R|QhYWvY5?q1@|MSK8w56}BPzow#6=Vddtz5_+# z4|pfJUADnB%!mH3Gw=NeZujt8Eup6eMqmpx`vc>LKg95$-`bGLJH<^hbb@sK7cU|{ zMrS4b7p$JN=}Mo@_^r!Ce=3`UMO-F)1Id$Tbg1Vu$E!IfyqS7+Zu1AUXS12xlIX8_ z-W14aJ9R&hc(imopmHfk^YI9I#CI2Tw;EP2_wTH{&rPo;UA(Q;&uX~jri{snHU^FU zmYCnXUxEL^(c3SMHvO4%19!UnPm6y>TT6c{iY|Z0)!3bjeKXqjk4JB7&X0#r&Hx5y zpC6UqDm>hdP+6~Nd_YR<&{Svu{r(2!?v3cIX1e_e&rMTHohgF6n`B_B{h!R4WC*RV ztvtK&fjI|N;>PZ2W^IUm+(}d&JR(~r-+?bW!x?}l`7m{CtSjGk<6HQnCkvp_x65wyyR2v_ zbIKg-kgfO*+ARFM3%n6Thmm$KF{Mmid{Kt;|GVF7U-4ya`=pEYq6K+8b8jwHuMpE(3jEP z-%6Gc{JEDn|X8MFbjTtM-U*nd~UO-$z@WbG>AY=Fe@IuCyFtUa5n!%&gh%7`XgNa;$?V(*QQ<-GJMB6kMH1Ivf(?P z2f{1FCmMV4vcwtvJ{RJf3|<|BPdW03-XX^EbMYlNPCmtJh-W#5uZDTYd{5+^a96nS zlw@=H0EtIWe3|NrC1C1> zhbl<)i?+m*E%`DnlP^;(F;WbD_!s@&l8Lv}+3!tv?y-n~g zg^ahK-^qF31V?WLSHA(yP6c;w!FS6hBPTvG>--w^?4a7a^7F~0NFKst-lkkJR z*_)eS;l1W-u*Ih3=tj~$_O1qZ7@9t~ARUUY@ z6BF@oWsG!RfR~r^ZZ1Dk>*d`EADmi=QuGiCjlvuOQLHM}^=!~SZ(wRk# zZ#!ux7@wT3;23+-wk|+;h+5zg_UZ@Vvp1W4;-cug)U`TrW1IT_G|xZlJO+%Z_Uxo; z*)5k%syqe{z#fwLc02iB<=c1RNwtU1y4UUN`?|4tz=;?W+SsPr!AJq3odtE_ZRl__H(iu`}Mcbb7y9?3hMITzLDvMe?_RZ}Vc8b|p>Yr8N1Yh#!%!$LGwM=Jz3> z%bV-Hd;U#)s_599&hNlU`F@D!PK;5|s+p8*EuPdL`-iEMvY(*7S}$H{eITvP^n3p1 zD_7S%d4+ja%+kTtS00>ZInM?HJDaz<{M$9q8F!A-?cdTrgA3DqPnz*eIzPGw-c0XT zFlWPQ2YTM&k{6R1XT3A|ew9bxGwgpO4?5x5k{N^P>_6*jxky$1Ny+wH?7{|3RG|3vouDYEBcPxcHWdln*l zD#o$sv$>OE`IJyId(VgLEg!9#7~1f6_yX|eGWrt5c2xjOte--!ol8kqysw-W@s))} z{F3z5=*zVSOaBslZuKO|4h7sj9ALed^3D=}fh={qcLsEY+RPcQ>b_P(JN)xLGwzbd zpqn1e_%+{(&;Nn)rs6Ny%jG%bBju<306tPrFi)Zf8iz*c&;8_mpr_UFEcJyeOBnxb zI|hy%cigfb;JxU|9pI|UzK{Q+9UAwUdhTsG_ajf3@8N+HcYr@l8OfX()f%WDP!dx9 zDzA(?yfX05hyEZ>a?Vn+K=V}QPybW#zX2CVkah{1O ztv!_{T-W!pqN-5B&nfQ@EOvHcBX|9`eS^MJJIJN(oS6F?c0o57sgd_bOS)q?8~96 zo!5=>f9~bi`=Q?ZM4c*Mdy;VA@3iT(y%`+)A>Tshht-2SGvE^xzai+gGlBD}<6W-~ z{NBvmeLz`Fy-S_H3ewK^1!?D>fH%F>K3r|-Bl#fU+nG2&6h~j!g1&GcJbp9!Le8$) z?E|dF_8MzsJG$2Ph_5Zhs-5)M*7|!&eFM(7k3KT#x9fj)&lsLlg4MfHN;m)A*XOI} z`?2K&fTQp5_ujwNYU{%D$K9>Qj{=rMn|D3zQ+(rObWcJ1y%XR(XN`2mNPN{7?S4kz z*dY^i?~i_vxmNoZ^mEUJx;z6sJ~73|aYht=VI*?2%j2`2zjM92uGZ3b@f&8Hd%TGF z%RBz$zOUoGWObFj+ix{A@DmN13ogq4+2!BT?C*o)(s4*EB{_e-(6hGq@Op_zOU2Hn`rNi zwEG75^C>aiN}j-dE)$WtYv}kNr0o$Y>3qHyhV})$UN?Ehe7)Mf20@R^ca5 zj=sNjXt!9)2=4M_FPh`s`<=-;lg*S47j&C!x zZhg?#H)`GdVB4yU(!Wc$G724GYL`_xq7jM>yY4-=jJ=ehy+z+2oNP6gF^{Z=9oqMv zyUxw?C-P{W$S$O|7QzoAd+d_#?uL8ZJUhta?n&wPbza(b(v<&|LaX6vewF-GkH+O? z+AO`J#^z^?z1EY_88SxCvOjj{4Eg^P^WJ|Tj2x))+Gt06ASt%`W#7(a-*xL>PhHT( zhWg0O8&$_1aC4h{9>{+jUhpHY?K=O&*fvTAs-IRm(&*fM$>!X-k9Hkoe^1|zg|8a7us3eElU8}8eZwoze(m?VKh)qVdxYeN z4_P0x$ZPcT{J)m`ir*+*pq|&(+}x(|mOW#R*41Z`n7N<9;CU{1o`YS1_*$%$O5*xx zY)A2(zP}6@pibfYaqxB@W8E?tK2USSS`GR#Q{u$NP`dnq^-goy60t@)=-_m3yu1H% z&jsau$eNT6?PKbjuJwwZ)(Q<;cAeylL&z3RXW4=_HOzH!vOE2&?q)RzU$pj< zz$sU@;9gKqw)hPE5#DGWnET+F&kOkfC(<=m(}AvRp%|rEw)m`F=3Tm7+40z^ea-#rd?#FM6r`W8 z0%ae@j^W5o@UltIvVUP(VqXpXcl3tu_(RdRCJb8|f-ay# zXn3nv`qEID&II{d_nO!uLE{r49#1mLOxh{DZ3-}EqFGw0(QNQY<1&dp zC;BEG|2%`6X?DbnNvb^>_%?M_V=r0W-*T?x|BA5XD8{$sPRVL#`oGav!zXa(m{}KJ zGiAxwzR=JNYGKc!c!s{H#N=}_A%ch`5bHk7xI-rdHXVw&F#54dZ; z2W4nKEH(RtYp+W(W20yE^YHhgx8lo3n0~taf!3&5|D^9dXt8!q$yfW7)_f}M4!+a( zEyM=xkgo>#V@J_-ZBxt?opI4!G2L0qHS(EqzYT-G21m{j3um)*u+%$)d}^C?YwEYs z{;NW`j{d~Jbo(s>)9l|Em}>vpK=FX(%)?mbV+`|h6+GbR81rQ2Rk2OJESAX;_}OF3 zo8tB!ho2RF?mTZVd3C;MV7k2y$obYuLy^nV_#N(pJ(Ax>ek=KXpWmz9x$hRJd=+bN z`}-DR=5PB)#qQeP#M%OKmUlx8`RFanYABxnPF*`Tc&%gQ*WI`+@B+jEHoU<-%&l;L zIrYkJt8uy8OZzvkEhpGpzJU)?0YBYU@D293A1%lv-bt=~7kQ-z7QeIYq0BmS|5P`X zon=qsUA&w8X@ae{d-YGGes_QB5o%#yE}oxJ*RqzFanMRL&)(hHM=H|Y`;^g5l+*6` z!C=h!IN9j6dwY2KE!wB^QBC0G8#SdPr3=xzy;ZnXqu3o~OtpUnR9nOYDQ;>!kU22^aUiC1q~?C9 zn#ETv;aPauvbIa?SZmi<%i3BhbhpR4)!s)sAm-)Rjb;+Q_|z5j~$KJiX?7B%mPM_S;<=DiC1 zDeWDc;IK}0hhS&=ckB%M=QlW$W&fBq-tV&-I@7`r&Jh>vLhqX&@{QqZ%0sLb^x5#0 zk?$n#pU$!$=KUt(lDT(6W!MY(ud!cXAaTGAl#D+C{J9o9x(0k24}H8k27Vd(IC6qJ zhV#ga-))0<4AaKL?!CFC%y$r*)l9~=bDSK}>Q(SO(knCrgX|?H&l&M)#{D<%%QbBw zK8@Bf0c*<%=bGI`iQ#o&`023A$o zisKn1uI=HXuAyKlbQ9T7ZB(4zKPC&u)9rrbb#Xl1e#7%oN+r$1@g4rzW1Sx0r0(hw zo(AwWPGbCz8oXg$07WZL=~CTTy&rgZ<*#<}csXTk=Dv|5(|yowgU9PF(LVO^z0*$Z z;n&k=)$<8`@^E-ZB9^V<*e-;ooEh8hd)_yH;Tv#xeYk!gZI!(!(|+RaQ}Jxy=KTlY zux#LZzr!21-y4`}HyEgRw&&jCYp->2__yR04tLPRe*uS|2bW~8|9_3cON7HUK4`p; zHR)$<23Vt<6a-G*dckB;7qZotB40@-L;Xa6&3Lu2x&>Pf#P{c7KkXfPD2^WTYXYN+s6-(#N! zf8Be((~V61Q_5m4B~z~t_u-ts$;bW;?lc)cs!aPT+M)fp1Xw_Ox@u1&-zYErLZHUu zZ1+hs_J;mB*SNTIIe9Dp;oezmXpVZ65)Iusu+EPzD1!ZH+64vC0$(6nxTR}!1a}Bk%Z~}4e$938&QP7n^I$>LDhxz@ zd%8xYQy4_PQgB3V8xM`xOC6oR3EWl+Z|d6KMAM{a+0Xskt7Yr$RJO0@HHOzjhmH@q zn-AX#BR5Nzi9ELpKBWVHKfFHWTfaFY~NAe_^2Z%hP4Q#2Kmf zh-3-wXrA7KHQSRl+lw{Zn>9;p#6XhKjmuA|Inx(&?*KV9CqFi2r`dJD3}}@4KhK++ zXGjyxszZl3l(w^{?z$X*m^Aq2!^qXgXyYN;caX7g0@(vk5Z6#~OoQ{$55dc4&CjbV zUhqa;Iy}_BvsbC?Z&SATQ}BOR99|k7z`8g-|GH1;uqjJrawj^vH29a{ZLFIX$~;PJ z%3AbJ#|NYkhd#3|{(Kjs2QNsndK0hWq> zAv*k}@FF{d$$wdyOrJ+_?x_X4M=~*`xL?;+f5lfSfA)}PR*oW$&=tC)hJNLD=`*{T zGRnY%!Y2!%C+LB$aM%77v<3YgT&x}nnV;#5?gd-wW$QEVi1Ydo@8XQL-W6k);``5am-IIFo=Tn- zyLJoZD0Xc!bI#tg;k0z(yaq@wrThZQI62QPzdh1r=chw^;3L)(zsFwqJ@%&0edv2% z#-JZ#kqX`5KGQ&#=(*zCJLo17I;DheLcFc$ylAuOt0(vE%YjDXHza<11vLKq{3d(< zR|Qh5_Cn*|ho&g5YAw&oH@R!uh|4(t^F8LjcJ5f?!!%X%Wpodj_GJ2{z2keNx%x&^?uN-H{`lb@AonmckoWUxbi1t($$nz>D?VD ze>G^j8+WEVzB}mX(6xH&Kij^QIZ*rMyY)VO*n^$M@LBf{f<9!~HvM&Fp=|q;Tks?G zWXym!-eyk7_aX0oBtCYlpM4S8eR@~Mz8iB8WG<4y?-cYE(u;OM$L8|0$IpSk!@%Fo z;HTE%3iiG4^PB4Z9|g|tMc4di_zS(SldnY}rwX0(l$U6O+ty6`dFDa$v5hoWPmF!f zOWR7CaJT^+m2WP-d&qN{=DqH2F?hP*wYppnPoLqtd@HxL|Jm^Ly+P3}9v@LfxuRXd z!8JS!M^_n`YCmkCXy4V0$2i8NobegUIE?`}uR;cQb(F`NgCPf>Mb4l`=gf3*Y6ag4 z*GfHHyUfG2DCxpAt$qJ}W1YvqgT1NRcTVBkeFir&?Sss-@Jh0?-Y0Y&F8$B2TfO)0 zyL5wdH=co4DZ)|V6?o+7dY7GD*PCtshA~Uv)m6f)h{3C6;kUnP@G1?wk{qM@ZloUx zxlilZ9m6d9I?`VRr@r)Fgh;{8L*SIommNc2t236mA67K6mi;{#a?j7!*W|P@PS~g2 zyCUndksVT99C{_SZN!RHtKk8DHee`!SktUK=`#<|_FVe)EaOn|VCCNz8oN!7p&1|GHv_#Mwpu)Orx>$APtHS_`@FN9 zr}8BCOU6@-FV}XeJ3`Xz63(hj#z%aQ>@kV7O45W^57DM6&;V%FvT&AS zuBqHy`ytj$wUH?}(?A~aX+HyMKQ5xI>T&Lx`Xy=AewWYF`bors#-}HtC=Z_*O|~Sj}aidiQ+129Uf#I zChp#nACTzNY3mTbkv6kj~d?N_cA~3Mr@qR-G{H^ z19Z9WK-4)=Z0JXey2OTPZ1G!JfIq-d{CQMPG35oj)UGsV;?RAJFM2oUuKBOm-BP%1 zWkn#l75mwgwwD6PjoEe{>s))qm6T)lbmWIjFYQv5J(Tk$11x7FHc9z+&k!7p+=IQj zcz%7I?%B>Vdj)58`A%|l1X)Tx5fER(o*@>xwXc`{qq3b(OM zTi*X?!No7&+jAFp>^Fa4Zt#bhbQs@(BLm=#Xj?$-MdtGvi2j5;S_6{7zEIZFLppSn zZzvz-ybm<^iB0@W{9O1|Ys#0MyB9C5`-s}*Q_-w_$n%?JKa`K;ki!D$;#cuU9pE@D~Yq8pGix!S10JnYpj`D&^-iahUx;W{b^4Ri&_hPQQw*_#OBcb z)0uWAcM7eB2Gl2X1c|&S$h(HTTe_fsNVR*CuM*m&xy--Hn!OdeW-*V8XrIxK1n`xi zzE1XyW!8-0(LKnm(kpZxd)+_!g}keWM%Q?}!5x&@IlpMw)1)nDjS*WV`Z8yngukMB zpU|FFeXZGka7OF>I?6sk+9>vn{$AR7q*eAxnh@ih-KsKc_U+IP<(1B$`1=n%v4pdE z`{`pV{sh-R+plHsn84mKG5P(R&tkc`%YZ%pBR6+Fu;(kexr2v{3jL1fA2jCX4jQtk znEPw`vVWNSaPob9TP@bhl9b@myZpiK#s1*-@#L+doD%j9bS)iwhjfIs+|g6W9XXFJ zvZAzWVt{*j)Q+UZYbGqt>px*}nl*8B-%&4)?z`xPLhj0`Dx)4F+xKYQ%p9!Z-knO; z&wjqCWgjf#-ktuGSID`palYX8$5K+3s@xiWrve2M`bwxvT?`#iSbZH@LdQlf*lOVT9uOOLV75Yv2 zyzlLI%iKl0Kk|ptn0%|-u(_58(#IorSIrq(YiP%Z4+V3@CnPNh)@}3fyYn6!pf0oS zdb_eiEoG~%hyCf@RmZc`F%6%_!OT+}yjw-Acy~`tvtu`%j-#{4vncuWwx;0UlxVBy zm7zVmN5_r@uyd0C|7P1iJ4f3)*V#rtbyrX|@`32>NN6YWq#Z|g3Ni=MKb|%|eaAra z(KACUKZN!W>+ea?3he^}`PP-aq=(iS+6;V=#zZPjxyIDQ%3tS&gV$a zSyLK2TV4L)%rW%b<#ptHnQDB}^sVsgGrreYDh83}qM{SeBN;J~=FU%!UB>*NBfCU& zB*<9rnd9-)<(EKf(RG8rraWUm$+WA<{}KL4DobOOC{Hms{s!Io+l}n)|HLQ4VNaL- zQt?X^yCLDj@@>kNd~=t9>Glku`1Zy~&c?~`xWo9R^K-DP9YrrtD%&t;tCFvBV>uWe zU<>D~%1*_A&VUa(8ps|P?b@JKcaQR}_KLXf-9!HwzX=(8t@LZi_kAVzTWQbUD}P$#GRoGP?@fK3WwB;x z!3Df?%)Tx8j6D+^9!=UI(&X1+WH0XWQ*4o@n$Gv7WZPBHES2AtK0Sa;YsU91em0{z zba>hJzj@yr0T1-;Apt+rV^t7f^l`2eJA8k>5smKKTx?0L{H!nkO?yW*!zZ4-a1v)O~gGQ~9`CO!p`4 zsabMGd~Hfh@rUOk+mucm>pV~2|2w#p?cvf*9xnZyc8Z?~Q+7S`)!}ns=D&K^a?)hK z(Y|kVz^qrz_k1(o3E!*?Tl0NCY4RtKUx9oIj)Bz~C-k4kcm% zn092?*BO{;Pw?6`g1%@Ul)qB)*J_P^Kg3#3Dd6nAFX_O@A&N(vZPtBG@}JDQ|74cC zzegxTJmu}wA)bG)@pnt_SewP9Ne&YJi=Rxa&3CLP2g?UZ4A29QpZl{s=K6f22JKMZ&>e#o_?Pt7ilDxY8 zKzCkK?e-howS98lIOe6C`5DVRjbXm7LhpocT#c)D`t0&o(2M%ekNVM32hdTUdrsT8 z_Sk98)N{6e31{r@;;j96c(aQ*d*72Y_|jFM@cbVo4{D7oeqbW@6@2oJV255NUGgCQ z z=Rs9H_hvlS#~PtBUO{dY-=(`W>WMLg-_!78(C1HHwHo(7ZxKJ=a$bjrXc^oqwu-Tm z-KVu=LaYM*yRDNfnT#xXd@OMR0=cabWYkqU`waYc;d?oc&A02O7QVG|Ng#Xs0%Xf_ z&e_kz9w(U|8Ffm_%lI%&)EZ8=7cw`Jfu8m1(th-ym-b`LRTcSW!Do7DPb+P@zRk4n z@zRznZJN^3?7O|Re^VOt?bvu|9JUX|r7jknu6-Xk&5I9I;mNPBa)v%4Sv1z+*L6E( z$**ggkzG^mZ+ib_D~rz|jy8Q&tau~8KHoK#CHa+hO)>ehT=|tYX{|~I{w1~rGuNJ; ztVyv!-X76mcg?5GnxkVAOPwa#HO2I+)R`nmyRQ?Z9}@(}mpaz~CCBO;PoCY8kj+fl z4VOgscsjTsd6iFkp|9{&)Yd+wZK&n6FOWRU+1{JP^C!+R=i1vn z{jSDAdk66uv~MrV#=qhLY)b>ag^m)Mcx*6py*m&+IL)PBsb1eSZUyDc?NM|gW3A!D zFmc4sXinxp6T}C*@eH%=J+s0+ z_yq^I42@dEJ8O#IPnlopT#CF~gj}h0F`lyZoz7e2%+CU^QtdIM#TPKvBZ!%S&hjzl zvz4~*ADk3w#m3&+n{y_CoTXi@oTdATGbMaJGQbxC-_2QIH?|V?R&eJ-@aW?r-h1uK z^YBPxc$9NRF0N$T&VW+q6aMd~|7s(5$W*PySF(+Ht;>ej&y@Y#Kf8rB@&(-9d})Wx zYQ_{7|6Zq_%W1dDQ(3C7g>kmlo$f4&0z8*I!QleD3}!_WtYrBc$m*+}k^) z6_AGPzr(Z#`Tv*D8Oi_3e;4nR9z+gTo&sd&ShIg;XrUEdh|KKguG|`aNyz1rvAZCb zcU}DKgl@>r7iN!o(cBAcrY$@>=c zKiQYEeI0e(m7KCvby<{Mid>FNZtl8Gl-0yqQJYfGGYovd#55#ke@<%+a=Y;1H>{xq zK4>1Y@GTHdY#}{?6CrS7KlpI0Bq@aR!p{BRgK*@IKo&TW^)KVZY!4?AbECP^d`s_Q z#7!m8bKasCG>!?~Mxx)AH--hCexp3o9mCG!C)`eq zpTvEdm;6T zf7Ljj&+{eN5aQ?(;_zy5=u8XymiGPI%vw#|-oiV>2RyneXC~)tKEOw^g=g%jt;L+V zsqbf9qO!VsZ&KUxY@Mt7kg@)Vu~yuv zNB`WuLf@G(Xov8ooH=PFkKzHv(d|rSe&XP1>~ii+)ILzsC$|6Pt77|?S`N-WpO+&4 z?VoN%OAkDLMe8W;1$(JqOtj#Wb*Zt|m(pUNyu|rk%Gy88*VsPY*Vs}8oaJl$^j_ba zpUm;S**een=Kck~H(P3aZ^j?>y@@{9?711YyL;{>lf8Kdj|TFr{X%E6<41{GgwK@v zxRS9HPJUE0A;x-XY^J@!)jimLCOBBaSpm=sh&1)`%21)^Debz-tO-|lv8&mKAAPOd1TbuV?}*F z>A~^6d(hKkoq>BMZ{B6>+33s*-Mppb-Hgs$Yh@GbN9(tg{av_p3;h?4o>^zEhr{CQ zr1ugpF_Gtwd2TNv<{J2=Hpfe>&>rwtZ4y86(em-J55fIMz~6rKOLaa=of<3QavNuR zw66aIucS3O8-Mr(=x~k)s_`Et2F)b+M&{*z0%x3b<8xHf_OBBw>C86z&I8Yju8O72 zK5w#kCO>%HOnv)k>umIpl_!W5a>n@h3R$ZjFQj&K{^kz8kzVm>;&zH(JURm2<00KU z%=vKoer8z#>erfR_~gOeis|XMf=lCnuwslS@&2xgXE#?ymS7>ca_ptf>CpDV^y_qK%!uPmJ9@<}6VLwwNb3e@-8Xzt{PD|BoWfIk7GA zr)his+E@!Ti92~3_f2$Zl4Ne_Sw)wI64&RK=sz2-7rji{sk^Mt<*%vOafuiPozHw- z4DQJPKEIoF&qCz7d*RWi@VgWnWCp)X?rlCgwOdF!rvmv!7P)?rE!-k)*Jwx)9 z)H#+oK9cS2$+7mEEa;l`=Kb*c8J~f7j9He!RqS?*z2XW4zqX>|p5){L`7tIP$mX8R zxIeplknzt<{*%G+SMi6HJlTuywf6j!C)vlI}$bsntIagR}4(Ig)?f)e*m)>lT`fbkKsrCF26=>&p$$T zI-ZH{0D68o&qojXVsD^_kngP@8{%sDu$_a?d6joZ{qQ)q8XjjyXC5a$*Y%x_6^#!4 z44dFjyAwAS{-!&2jH_=gsf)12;|Hx!i+CFFT=7HqW{Nh*ha%g?uJdFyH11W>-1qJ| zyXI8o$;1Yc=f&I2w*PPlx{oh)f`39`wZ6kwgL=&Oy%Y9})4$!!x1#;>*U|WBtnTo} z?sfyy>`DXCBN&*)9DecK#)(6c%c)!qf6|8Z)6~P z%+o7<$he2}z_47(JlcIr;e!|=K@a;+(obFGHq{Bm-czwVZ! znA45i$9XBfU-=8%*doYy>k~SLQtZuEyf9km*>pd<&Mk8rW6*F@Sm)E0$zODa>o2-Y z{-SFn8{scHpZZth7i!Ab(LdVEx22pr@{%f(6 zw{t>&Ov`O$jx)i($Nb5-)Mr=SFyu!$KYb`CS9+eA;F0|I6E;bW#TMj|7I3A%Y?20_ zh!+6uoa=yx@Lg@J^J31A%tV)it+48ui)(WpUXWvKko%wuGbl607=<{?p|XP)4y^02 zL4p&)&FPe-v41L%-hK7S_TdjQUZzgj7CY5>kbLNII_i828R`FPolR%dS>fU7tw6Qe zvkP|Md!o(z9xaP~xU@8OHMTk7_x0p2*VzH5C^`-py0a*%aZ|jmxuaa#`x0}x`9OPv z=Iz+z9-+h2dxjntJ;#0=EP!@fNe9xvrn3C||Uq^F1%3 z-)3F?;DTsR%7gbdKHTu-^5HgXvKYL=SNpUz`E;ke3;e;^$(+Tzqu;o>^1(otXLNDF zzGdh>tOHHhR6E}bbJqH=xiI?aM0icFPHeFUl9Eg3uJPL4`TH~5a^E@HqPq>sIg4b< z^qsve|BP-P_4d9*9R6nLpY-vGz3&b7vo_kDpq+<#7N2_1zzq9Rv>(>jqYXH|FfQ1n&4kF57yiL@eg@0>esfbKTz=C_fbY?uwu+qwk}bu)-(5ab;E zF~(eFZ>H@0U~a3*QW@~yY0UdBrAa?@1e&Dru$UYC@joxT=Dg^YYu&Z@JK7^Ys1(_( zo^$5O@QE#h>F>QgLh%LgVyv5b@HYQM;J_iCYmu+CU#ulfXX=l1L#N8w0@bUvFQ|M< zzn1@b1=W-ET+e!p(+;ia8$CVli`4Dflhf?yS^s;dxxTo8fXjnDVR*c3yLo9@Ol$ce z@4sxl;#;+rALvM9eN%_l^7lH@utib#V$x=j_L?_dCuqB5c&$5?Bb}_yiabZYJIOaA z;GPH3S($hd^qh2aCIox7u`6R!Ha_8NU7PiLjK9jbm;T6>Tz(E7uCp!qxcAL8zR|b0 z0Y_4g#`c>gEzACfftmI#24>h_H!$74Q4pFi#r#jTCmAT)uF#5Z&>$-iesU&XV5vF!Jq2>`xnew~dHpmNw2r4;Dl|EB4+M&p&VSwqC@7 zUJ$Db9+BkI&C=%l*d$H^26fHt@+kzm(T; zp17Gb`PaCzL#~k>e&3OH3V+|~NRtfDp1>IVsv}Ld)NEV$yNNUh{4QoM&Vg@u9Ufwy z^nQ0=R_B*2PF=!f;jnmdW+Ix;#7LeY1{lRDTS31@&vJ<=>&X4tVfclPA+&Z6I;P z48%@rV4A(sz*PHb1M|T1@!;*%;O{u_xEy>Q%icGJScLB0_s@KE+Q4%+Ci>ZZDi(V6 zE-*0No)5$?dD3!l<9qyW;5QOkHnz4bW-}+pI8%76DiB*r9-T34MW%XzGbGjWvB<2x zvaO_R>}qtm`>VReTH)23h}+asl;WQMn`${z@lUSA4`KpihEGjvIcNEjIm@>nKlTu3 z`6|$lN>^RMnZ0uE-ILxwIAo=fck{5}mwWUo2YMy?`VY@9?*`hfz2H@7h_qj3QfPAozd-82F{9eN5F8-(yTY3An>I?CV4CEd$;0Uvy zXWH-64(;bBNYnl){AlD1>OOE9zph>SKQfOqWqDQu{A0szey{L*4O_fyh{R%Oke+%c z5VH*4n}9#wVq|1%jNlZ1Qj7fE7NGN47}CW0~$lmJ#L zL{YH?#nvEJU!<)FLV}?+7_AK0V!^kd(ULjTN=v9rwLh@*Le*Aby zGu9pEjtJN8dZ6P=_kGDyp~VA9@AEasf9+?Md7)EgQ!Z!t$RF8cg(5X2l4l!UNAiYl z%w^Xy+rNrT*vNoYuWYd8yVBfMJ}vPgXP+bg;*ojy1zM04GUeWv96}x!YMR$G6vgMA z!Nmnm9SJVJ-@@E#k0kn>@FVO3peOsh@;S}ff7h4GRfaCRG7kDHd@t^d1K*K}Pav*{ zxR+cQ-3)$;UrMG;f=}_f!>LQSG1P>q_GN?@lJ4+R&VK7o^7f+~U(AbrV(A_~-v!v8 zH5T>2ru309tU;Tt3mqD%^L_hE#Q){k%YjRU`IPDItv;28blqERbpPpN+JlcF!YfT% zGwfs&mu~w^$eo`iOtrh2P_*RJ&p*b`JpM0;l zpRDxxg){KXRq1Vpe!8|kSN254fZdMH_vEI+r-0*s=h@1$mS+?4v?=|qkz1fK72LrV z0k8iGuEUQF)4g7I6CSUyq9JHosV8X=a?$?WMLAsMZH4dJuJa_W-4FIZ!yAWW`q2?Z z>shm(yN)?ojL%f}tcdJ;bT`eegtX=&bo1{eWHs-{zUS^m`O~Lh%kwSld;XY^_4<1B zcg4idvfG$c86{0Z??d3H7PAX*xt4vm3qOsd>CQ||-7IXcRvZrpmc z1=x3NgmUbKZv2lFe;@JC0`b)MyYZ_PKacaab9iLq^8jP3Z~?YG3h#rjQaBGjNVY$B zp_iNyh>qfJIkm5xa^wfhi|j{jyVL!ylHE`3q)_^j#d~I%IL_>v{ZWoR)BT>qx61P^ zNthlffqpL8vQGZM3PILmn{9B>6(3!b{w`R6?TL05neHtT34aa`EqPE^OA6Lhr%ecMUSx=+Uz0$p?)JX!ll zcoxxJ#(gpLLT6SqzYBSneg*l5(M^{sWDFY_SHbM7%pqybNrZ9zw4e7ZcYHM`MG5#O z!e3LzC-SWkQcP%%9FNiFU zHSNKs#e1U6*E`(D$KLI%^D6#wTey=u07oO7=h^Zbc?XX#Y4?Pl`rU<`$eN%UR1x32#fn}$c|J9t;E zad{crtWB05+zy-joWPR`TZl)i=olga-}G)iLl=KALsL>j>|XQygd9 zx1lR&8LGPT>?Y#1E}lD1@iFSk?{pgujiGF4z^D&uCx(s{dHjNN%8j<6Cl$|ZbQ8cG zcT9z50K2jY7(tuF<6Ww6^8W>%N_>^-e~tRp_y0}(8WU4L`s4Ra{g%J8Uh4cH^~)jY zLrzeS#zJ)uyO_Gpfq_r!n{4F%S) z`u*~2D8E*5yJ7TTGtukP-w!BHwhq_r=03gcf$*Gsd~WkZO0a1_ha{VZV}tvJn#tF^ z+SoK~4#=iqqhr%>h`JALNew-^BxRlUWlgea*pePX;|ahG(d0m-r~?3&lHE(SAULAcFz4}h*m&*z-4S2((SLib5gCj zmQ4fgl?-h!xOjlJI)0i`?SsJNSMcqvzWt6lHgkb3cMI?jts8#sxHnO!`1knuZ(?5N z$sg1u=&{dZ-2&Fr9?Uno_y&By7~MMKD1G+hZRo8AoTazw*(p!J#@BNI{Qe?H6c@`$^(C7tJJkDm2=vKW&99_hshs856w}Y z-NCxS+;rki68^pm@b|6pO>%uU#p#>>M$V(rk8d4*XiqbJX{0X+o}`+_7vb~i59K)Z z&tZNU>p1Klq%Z$N{j>X@8F$gjPJKRQ$@YkIFK3l=ZW(`a`ykq*InGbF=4-!J1}~@i zVjT(B?TeXy{hoe_9#pa)>#9pV7j}L?dYpEfIRY-z(2*aYO%d7@S>2^gz>;iXJI`X# zuR7YafjyVc)eCj)*Qb@y6|TCL^TRHEJAmDRXzv7Yp_RG&3${Dx|Dbcw2im9er^(Iu zzrwCb_9h0WTo^h54VUbq8?FtUKLlpG*Jf79_ZuC?Y~!ObJU0Jys%!PwXCPWJ5hxEUO8-sCa%Q?jeQGa(&& z%JiB>)|bXvu6?a&z*O2Xl6Gi3AK;APV8MI3eV&VBZR8Ozp?#LdBL1y>-Q75)NB=q@ zq%r*wc083|kF8mckC7H*!;@vdIf-?PGXF&Q9c*}%?{UUypme z3P0w5LzkGBg>KxBRHk@D_KpkOxF;2dY)AWnd)>Gv^lz0n^eyrFjBT+(*1L#kc@glf zFc+Lyf(;{emh+$P65J0DW%z&hdk)`DzQyFrvA^cVn{(;ZcdPrIfNh=`yEDA~N~`{= zpvSen_tPGYTMPJSe7DnvubOtF|5HdCZ&3)0&mh!U>Q9tcJtjDg`?AlcoOJTU%M-tT zIp6Z1Fqir3%FknmN`F$>|9;ZPX~%t&I{logtWW#D09{+C6u5$~KEVFn;F^(VNN$*F zzX#v0xf)9SJFo*0e=nGkPg?c!JHYRi@bWv*$sf;Tj_?=M<7Vfdi}*+RRX4V#+LMJ# zxdTz-{21%7=$u0bpbyO5G3HBkiUz#hN}1yGnS*6+-ZS)o{ETlr@9)Vwn>Du{eZyJw zAiomXc%D6<|3wE3{`1aVIL79F1$;~rxZkwX6KvY(3Gog8o!~#XaeSyPZc{JXMSGol z*fLp*vDacQ3!y(MQ+AB;dD3_Y@AZuzx~-4iiESQueGI%7TnMNC##qZg^1Fl&a2_@O zuhLHPEx!{VjZWh1MKWhFQaG2?lXFSE!1vzBNc$iw&e^JudFIR-7zI@uNX(|pinXs+xG)B1wveX#rCk=}AO&qF+tdtQk?Ckjt3+3Me+ z72@N@dUEEK_qNJTu#fZicFrKZ1dWIxXQ)H2P>0;Jycc&h!uvMPw8Fpq=oyYRKOlFb(rSBhPV_Hnzo z-{zx|9{!U>38BL&R`kPsD|{$l^e4;aFutK(_fem0AB208+_<~dCquii#c|`lZt`^t z&2;IM&!tnlM4y0{Txg`q4Uk{{mb@-Lk8!`NM4uSr;-XzSi`jpPPa5u~85+f!c$xbR zK7GmB;6CTEfU|1_{TFjl^qZ#-2foaKIGw`D>I@ZIqm3ze7(P@)!CU-@#4?JRZ3=jTu zA~c3QujVT<3ma%~u>CFz{k8U0+{uwSF(h5a0dV>Q@I^M4KRg^;BiV@ZM{C(1GNut@ zC-&L@uzxmlXWDxrHhi72D;*yi_mE^oX?9{6w5K&zZ)iqpLU8v@4n6yr_G!JEoovF4_GZ&IaHrog?)1BZ`{YZw)9(WA^y|)@esliflr_?8jTGMP1=ilt9-Kb@$n9g8 zKE~~pU*P>E;MB-_=%eAk7~dB9SHORree0`rBh$Xk?VI-CC+Mr%m&#aTGd=$#wj?d& zaeW)@RvvvTPcnIgvl^e3)OoY$V9Dp(#p&FAteZ8zoOM}zO+lhn@8LlPSC2o}`sa8g z5B)YYtpz$Low&~ROJ2J#(G%LA=*Xw6Q{0=#{1pBxJJ`6^ndgPHLFKci)Nf-=(SA6e z@yhc&m9yTJor>4-do!C$z`5j#t?S}^4tQPGz<%PF#>Z@qJsEt;w}5x?9nhm|-R}~< z-(8xwu7UX;=f)X)PQ;$w{T9trI~u(B|D#P>o4?3DU^_m)o|~g{_qo88Y$Bw8PPIRr z>GW+JYUe{K z_*~=}$&<}<5dfaqgG)(*uQdBNt_(4oc==*g-TUZw zfA}xQriAk#uASH^Xo%Xsm{78tR{A&8jFa>*8AktkKzZoT{caw?r{tB=i~O6oT6ifZ zF3*;Z<8R7OU%zoyen)Trw$kYHth3T`jjhRR#MPa4bR)B|H9J-$|4PfH7g=faBJw{- zo6hLsGi}YULpw6kU<7`ytu6bjvDt8UM>y z=6K-Um3Qx+aVoaefKqV!=BJg=WSz>7V|tu6~}8|?f3w3P3^ zaDTohWnT3Qk4^JE7rH;-DP2@t$hnmVbNb7k&VOEDl>c|u4P!amSjsncSHny>{%fOy zeBa-l{QVa`wtN2eV>|T0$CB?hLL-YkzCq+2mQBA4un&<>{dD?~4IOLfZf!L@TlaD1 zp5>>EEfn{E08irQ1kbe(Z-G{wUpE3k?2!>Eb=e zG~Vhe(-7T`%bcDj?P=M<%=fR`2|qCB9J%r={21tLL}^@J5zlvyT=_=wRer=73U9`s zM*26J{^@Rwyi1SU1iZ%EbCSMz87s-t1cP1c zPei749NtDgV6h(vcej#V+g^=1XO0>B4+z_+=Z5p)ZJJ`kG^Sss9?9GEuJz%pGLeg& zDKE>Ew=b~-zDMPGOnFyR*6hgHVjq$iW`o6B=6_FttH6Qe%f+mNFQ#>UKY){E7YB^!lFkrgYOyR%<)HRDd@LkZHO#6_VCy6#mrhRVSGV=bFb~?In zmErgYy`K1Ub>Zfm2j~Bk#O{fG*`_VyLodKn#(>8U z2V3FhEuK&lIF9Xf2>sHN@&zBi`>U}i%wE*PUNnKdsF%HHH}<03*RZ$;dr{W*k{)lz z{b0O5OE}@cj_x99Vm_cfGbgZUc}}3c8}G}r0__vHoBlbz5Agh!=T73` zQOgbt#^z$C6@7n#!_U+#<*a9FVAb5<>{ZZOyA*g}U)kwX!O+|{E)EtV*U`B`?au>* zO|#I^d9s^J(DBZ~zdHWi%93K^_W09puDKZduDPeK+4IdMJ=aaDNLuHI&dBC#vM2BL z?a=k}gQe}&_^LZJ(idvxj7OAxa&#uLlcnKxKG|c9#g@qIk8-|w8B5-GqCZ@CBY=t*RkyVeMteM3l{$%+0@_hGu1Q& zxV*~>4`MH*e(XZOY09m>G=TEnu3fHPwCg|F z%Ovy3w!db^Fx#F*_#Nh}qPI0aN}tM+lTz32x%90WcwiiMCodc%--5x4gyiOD&=X_d z5wxtN8u*N658v_&u-%*8_S{Z*{ORaSfJ?J?oXEb+mAjQvkNSbGr2aN=uKKQPqmd%& zLnbv9{yTtNtra=NhCQCQbGh4FG<>w$C|NG^Qn&NwZPLTHGWHGPW3#-$mJ-3GbyQ__ z1J^p>Ro-JSr2D@A1#j{ubb2=7NXnP&US)T4%f5iJHAYd^RkQaPf;@mYYmjUtoy0KI%U{v9i>ras2W$NjA}(Avj%@;#}|%Ps{ME3M$&jN{|j#BRWE z!R_DOaaA*P=iwDQjNc9E%Ke_?ntAy9wOV6k1)d*|$;Us#Y{scnxt(F7@pt9ptfc0sm2+=Kj>)>!0W3CcoXGv8Y3n90S-PriYL>sTwkI~B zsp}&5VV_))=dY%m$fn+*NyHVS$86lhy~?$HLdQ}(;pRS`u!}x2Bt@GZ}7MUp6&b4n~->kGZNOSh=x%LRw3}=6rZclR4 ze@nXHLF>ks%$ma5gq`hvFK28;dx0IDIsSn8ImkNmaY4L{4Ev}2D>`5kK9p$<(>_bM z)5aX?ep~U|wWOg#?VPXHIq3C}DJ>wbVi|WEmB4=>9~2!~LVq-lmA#OEjb$zww=Hu9 z`?pkF%zYcF>*_X+2yL1=#IL!uy;*IqElpdeac$ky6Z_lLbxrQLK1w^9YI`&GjOqVk zJg>Xl8LMTqRdm3N?-iE^pK`}{G4UFsEsU?mw%vu@ZxbIM+xNiTI8XbroBmDGTjFrd z_`XXXO!jC7cXiE{W zKah0hIfMV;1{hS{mdE?$G2x{MudJ2u9Z_p9k|v2zxMEDPFr50 zzOH!kXZj>s^K-Y2aXh)y#e+@6$MGb+3!W6a>9wTC=~A!%5J#p%kBoc@JgcT0;m7w( zm|-tBVYdA+p~n25z-kg>6UUD;YXSFDxNE`1z+X>a&zeeTz19G9)Md-M^UfLouUGbP zQsq^Lu*<&Z!Hc&&X4c}KW-a!+c%gfI_o6QtKwT-+bx)6_s|xW6q`C^huU_C`Z%@+e zQ@^=rRUfNQ&BOQv!tcP;kr`RTj1&2;Af8kV_)cd?ZZF){qpCpzihnYKlljw?t$+);_bsfz3q$eN7SRovN6Bs zzsveN{)djx7p;?(=r#H-ymVXN9^x4dE-i6aqaQZ3Uh0krZ zeVz60Kfq_nB00t^0r|EL;U3~yQ4G;MwX-~?=?gqbncW*wIhxFsipz=IxSe$=Uz6`@R zg7efDrvhgE+eb(?V_N6W@ zT;-1ScJgb?;&^s}^f&D~SI<71s`ucl1VxS$F1>?a80^s_u6FWJy`_BSKYMT;M7 zrMwOBuMs zH0(5?+0Ok``>FqX`Hl}im8pL(;opY!li|IR?0vJ)9gSvu*y~B}BiUK?(~fSLHO};1 z|LjH2(0UuNLc7<{@6YV>^PlYV!GPz<8`Njos6G#H`+PP1bmWG)_T{|id(ZIW^3N=} zm)fj8e~Z1tyVMU)m#lfci2Rx}ji=VI^9jGz5z92Z3G{Ft{CZF5;5VTwhrKDUKY@;= z;Zt~%u4|Wc>Eb;e^Ynrq_JaTD*Sz7iSXsq*=NxojXl+08e1u$kbX{f@J)SNV7!?b{BQhOuuLp{E`1-HK2hX$>FPp^F& z+n+DGheEooW61BNvVL=q=~OG z^r6}lTul1~58EHM=I;mJ>M{e|8y7eVKUr3n%e$xV(L%i=qhYNyeiYE{n>@DzpLOUl zSWoABc>n95z+_|CKX`_Lh1Tnk187KuOkWuLE%F!MSec}cn>FKNtJ zpEDL7_*M2{g3m|-dQ8rK=#GUgeXy-aNNq0Ro3XrnlVqoy^acrSpkP zvtM`Po&{%QWAS&5F>vu$g@XgpcL-Zp#~X$QqPrCbU3p6(|Nfa!^tTZ^@A5;jtv3V9 zZW}! zDx`fcD5Pzh6w z{d8nW=w~I4-1{)~A9~3XYNpOc>O24~eIJ_o0ethhGVss6muqJ-_i`oLAFSygIWgq2R)7;6XHC4bgpI9>-?M6 zMYG228Xj!Dn0{mT&<;L|CP!*52NoXc9{h;32INO3Z~2H|BI6Px~-*Wv+GEhCzY$+riP+3M)$9%~sED!GBSf)&leWR=42a_||$NThDG;;}&Th z%k@~P)n3Pk_43oF*A!qIc7pxCY(LC@&vg&J=9Z&2x9|`C{6?Cz@*csRq$$2z%D6>J zGx5cV!Cw<^WVPa3ztt!B&efhL{j}K+y(@;c`Js8m$QYBb1)mIkoeX`QLi}~m*GtgL z_ChaP;psK62Kt%;eXaCNS=0=Dja_cdza3l*bhEZ1JDk=NeM2*T@uom$=RspD(9?or zWs{-9lhNTOlO`GGGuW<k1GSHqn#fxSLMl!D`iFcTV>=~X>Ax8 ztakeN&cDF`I)-BpE8Y_9Y#&;fbFz*s%Hhj+p*U@{-kC0~yeVs>Nze!*3 zrLRNiYajZ$-P32@t@}dvPdoO+Xh_&R;o<7MhczO?-r#`ZlsGr*@?~bGi zi<$?n&p9^B3LV;{J5oL3kvA81BnH8Q=oZ?ivrE!L%TDLS%&DQT6W3Z1h|VT#8gf&p z+>>lK@{PY_qo09y+p4^iy0aF(NIX1xGqm?Lp2#N7@hIGg9!23|(iPUSWm+1wP&YZC2dNiQJ3Yg&uao>cW(uC&iJ;OrT8hs@Mng zznn5mKd84AJ`kT_X(~s&xaz!_bAmUYiq-qp&tB!j_Fxje4~v&{pYAK1l#=qzyy>M& zUN>!ehWf8V=IFHZ4HM@fu9!2|yWmg!=w+O=Kbkc50%iIxru@hQ$l#amGV!;wPL%Rb zB-^T2JCHf4Kb`0M-2SYz{?#)$4)}TlnkGAkOX(A5pNGfi2HPsm&8^Sz;}45@7}OOf z*PlAQrV?9*#=$pX;>#R)GR!4>q?|ywVPZ8E&%~KsAf;KZp zrGy>OT}$T@CR`Ex%V38dtUuqHuRa#jHlNp*?DrNA63(=Ft8*IZ6HXn67cs``!M$;e z6YJJn!s&*q9KJ$yWN$t{1x?xM*KE;+@d!6DP5BKj;?puHso@uOJ&Su z?EK(N0r+xkI%gl)i^7K`f9AuuOz)9nf>7l#X z9~`0`cQC$PVZ)3~!RSs~@kiCJ2Le&;O&Zq(qI3DL3b}0!@H?;Yx|DAMzs=z3yd}NX zRiKZoV?UIw`=jNzpZ8r$9XOUu>xcBceCfLmjGDAm<4+(7ocM4^&yeVQ%RJggd4A+* zjSH~(1&)(CVq2fANL{z$^wK?{ld-KvKM8I>3cQ|}Wug9Yb(#y%X-anq9#$!BrjsVV zX$^bD2H;M#K>5$-{J|&secc$NoBfUYN_!5`7M&w*ESe7QkktGzV>LSYY+Nc34KD5- z9Ou%7t6g}QCERDcn$}CV>uWAy%$k6Q=6U!NUy{78X}X0k^rYr$#v!Sq`*eTd%_)Nz zOQ-M2>lz0q!4mnJtC6cU(AN*Y?8p+zDc|r^^fBVb&Cwh&4(i{2H|}f7SCt$pgZ@Qk zK@Szq1Qy}-biO%_@m0B_%y^_3+hC(FMmBBIDF1Rdtw?l+IgzbPWVD-3U4z{8JJRq; z?fsN-d9qLNRd@Yr8{^&U27&VRe(l4a640|m}0e(w{L zL;L7!`M=-hWEIbO>pK%-Rb5Xk^aEUdCF_`d^KGtoeW0-J^=(0g$=huQR_^Qoz~2doq)& zGXoztL67jo9NzBDXw%-SkUdso75JB9)${)m>oWGXHCFV0Xip3AmA&Ehfq{SUUELr0 z%76dEIK_yIkuSEz3a_VJ(R1`9rrgLD_HxjAs%Rzv8rYneVdIIT2vFBB9%S?aR~`b0_y5Ff@_ zk}|bHGJ?AWdt=oft9ci?XoKrV`Ubzkx*?hJcd56bgf_sFXpiy@?a7b_EW;15zV8rz zbR-^4;#=p|y3Pf%piI%JTQol4mgIdt?*GsjMMg45j8R2~b^i`%7juE0%Nib=$~?}n zqDxg4p}x&{xwZ{S@`shgoh)f&~ z>p%wcA0Gq1I|dQ2E9Y$J7la<9nX%ZB8tmnc#dyjWjHuruc$Zu}-;>ttTlSMy_2@qE!N?<4UKI2&E`R-nGq=6zgXnAz zh2W2H-{So_>|Gn+WA?6xMywHy$hKc__x>Kzg@chA12acOcV*9Da1Q*tfI5c(Thja0 z9Zi(puahS;^M&3|dXPewW^NkI*(rCCmRR6a<>R$-S?>M zB?RyJlB@Avp5VJ7-*d%$_A_N^+(myO_@y!=->z2OlHZ870G~Ry7-8>X;0fAP&DjYv z$33m!lY^Ww`6KPo9LMM8S8+XEDZ|3X1Q(xW_v8;Oefsmh0A3POy3*k2fqttPu)-T$qIcs6g;$tUT_0f4Q z-Jsno#H-P-CdR30PB-j_ydgtF7_SJtYLl}USy@f$#q0diGuRc$ z9!9*(WXi@SX5|?@XsXeJcD0A$9Pk&~!@S(X8X-Ca?=2sfyW?{!{Eov~qUhvU=Fsa2 zwPAZ|#(sGkW1nIF{z_;5L_;zei*{t&;xRhFH??yT_qNc7m9m3==0ay|lIerSWgdN~ z{1n~6A?yeL6PpI)s^}C_jZK626ZJ=J_$O_U&iK>E%h4NuNIMSkT#Sw1f1*3y?m1U? zoZ9><{G{ObB)B6vnvtJDul6NQ5ADAz5Pf*61E=pYW`f5Z6AeG{mX9-+($SV9oBsC{ zV2}ISH`IFGwkJ9HUvtKITEjZlkZaIgRo${}*92>L^Vpk0AI@R@g8$b#CA?!@y<2O3 z&wi_O${F8e>~NY-dcue4{j=X{#Q@9j(%|eiDO5yA3KtlypA2o8YhgM@Py6&b3#sV9A!*})`~}Xh;b7w zlw4BsxE9tSttHXPPWrb>g}b=g3L5CLN&Pj!!Y&ACPX}pJ+Mz2HvlyJnc2FGhwDJ8Hm;uA2O?#XlJ~Q zZ@KkP>8wA^-b(#i7vpXGgPUhOd7OS=*QIsPYeo0%PjuQe)mbMUf9X0Wpf=^&_hmvy zsrzJAApA5u#SbnxyZ+5?*@L^(|ATYt-{|H!zf1j(om>ACpI-lY*-l?H7D@asI-W>4 zY!G&M@WyqVLu)Z}fsYKg9$%NX+)q8~bIWRHPM5oRVxt^hr0hcIGil}IJ3zjH^ie*f zv~K(0xqaQ&=T!UIb39TO+6O-a~sTQ(q~6Pvm(T_t)Y zTacd6uor+!*@L|J8TKHu1(D6C^qwENy3bkAsfPcN-m>R^WD8Qmn)gMvAV+DVbgQ$w z*n=o;EH)v_(0~3Ldbm9{DkQoTg)WI-+c$GM`sxgOKQ!ak>m9uK8Dl6odkz|;JW4nG z))ahMxp|%^PXl|8R^aK$3*ZmzLxg5inOY_F-Ld^rSLd~LC z53nB}2pyQnx-u{))CP?bFCKwL6$`(-3E^Xj3E{|)Nug%wRRmfU>jUkAM#Z2}(#c^{ zsWobI7W8u=wDbGGKtTbta|`B)Vk#_gGVwG|w8YQmnnJtITnE$XVW+Nx#y_BcNI zMpj<6!TUqZ+$yKlc{AH2gQ^v8OnFa(BL?@ow0$6D*H+2?k@}b?wY|UFz71X4r?#o> zDN5Uj4~wp8GadiU-IS(shC1!u#$f<_A5Bcr@1T2z%PwMFBZCM>oL;eYrt3OZ&&(ypPQ$NJn=Q4 zj6CB1WCyw*x_GqqHBYO8I6K2>I$Va*X zTh6~a(NS;BH; z(2`MLs|OE!hF%_ko*vAc%Dr59_HyW@#!)f=@iF7f+27pc3NJh{YfLWmR(<%sDL31G zOd;d=sKO`EAt;=RO^rg@wp=0Ye3)>1do1$`WbhYLrfBL@l<((VXN~D=f5GEIJ%Kx-lBy4T^)MKd}L62Wn+;GjgH%1Gyt!%zfk)nSJ?n8rGkHg zbLFWt{^8-7(KpEU;u^+OwiliGUAqhAZQ$RY;JcBtu+LGP)?~@;wXgINXY4NaObSWv zE4vHH_Pvy;JdMQ9?FJq8&S)zp%;ACd!p{Py+GDc5loOWmjOKZHF!y{SGo>wW{a@Nz zz$=#hGU4WtGbp=MeseC7orr52k=LVB_im4j>)t=#2WedQZeXN~?Z(`=?FQ?y58WQ} zJq^du;2LRL*gyxb4KYA4JUvjt&%wdrm|6}5fe?HEN%NFB2 zH~uNbBg5t%*n08dnQr_l#g8N&J(cKVKR5m{#Sf)ybX4`&aj32LfLkp?EM(I7{6YRw zZ|pg8>~AQ3uoLgx!;0U2;^ad}^6M)0)3WPG!>&VZy;b=KI{9nC(R$f-(AF}=7rEuj zz5`k^RpktE$|)hP-q={=+LMS^J8HpGb7qG;&(g;I*ad8P7uhxEnH0Z2@fy~I9Q$X) z1@3r)H)(b?Wr`lC%!`zFBs_w`(cGn@kTMGtQsw}K zl$ocHGII#E*R6j)airPrPSrXn-}2=U$~xOtsbGeUk*B2dMKXiTg+gY5-9Shxpx>rkgeQOP9cWv&DVRNT*j-uxVS3Qnh#MP~*jjqqf z+L+%j@Nq!&ka|uno=%z5DEkKL_zHDRMSplb{_ndR-{;EH!5t{ooK>uYw(7f@J6`2C zz8F2#t_0^y(i zv3@pBeIIl+;vx9|Y4XYLB*wops6R%&_&I!Nca3D=qTMZt!u3phCHeDx8O?V2(B+QA?F>LkMe}<=KaW~CdIDV6F*b=m!(PT zxF4yxnD|Ecj86Fp`$ET$3U`fA&Sfq};A_N;3hIIAFD>vaL~hF`vY z{3&S2sZyO~fyP9<>>H*hhFYLmU{W~ZwSE!V>1(T|?UQt7b7NAdaU*9-k#V%XjNQ;5 ztZ>K76hC_Cw-NYeBfx^!<9Md zS6Qj$vUmA9?bw;pu|YJxB`v5k{1@gpZJtJ(b!Of|m(c;t3T|6mSu4YPonQ)k9_<_Ace#X?239B2pQer2u)OGne&U0?R=##p?K!}t zY}^DFdl{!^cHXk>H|&kZUyww4p1m1ZaQO99`{Uutn`0j_VV-^1gxU5X!kg&t7x9H1 z(1bQ>#`(fg$4{@BjSoce-*pLTHa48AlEL#y*l0}3v9=D) z;l4faBbtp5E*?L2KYsK_gWwG&b04=K{ZUf3#Wik~(^fze#-hWi=1k;6SyuguOl%gi ztgT85F_v-Loa#(#>)cFuq<(=_#k@D>vtFu?tYHX2&3e5C7!qyRM!axFb0fS|+!o?g zrtVw+Q5yUhJZXT?oZUkIm#cHi@T}h_zx-G|3vEz-cVi$mK%oV3~wz|xmqaL;--QYZa&dW|>yK@HT<%2?J^K%T&4|8QZ z)wET3vICxD7xV4=U2>3n)mHk?OAe0kF8;U}oE)0C_+t|{KPjkt&7@aK;0gTH!LmLc}<=16g@b8SC0QAJar2^bz~;bM$Sanri3J`m2ai- z6Pz;dpiI#m*}nRa7vy>}UN<=E;_$7cYkl3v+>5{1S0umxX|@-;1?~UF5_c{RkD-3$ z(HIDiFB0A`5B0#l*C+oGq`@Z$-?ZP*_`g7!gKwlU-om#mla_|h+2sb`-r&33yW=0e zemnlrNP1JEWX9G}ja~7;U28+9uZ0Ho2z89LqGnvKaqa-YS-X zKjkyZ;NWd321nmyY~uUse)L~!j$p&*hVq%IZ?Zm(S69^zvTk4<{|a}QbPpb8KN*D&cSj#&>~xm}YmlM&<=kWa9{aEZ_&s=z=K`K?dKjkz z_&%s)ZH!nw;1#-?buwagx0A5_k_|^F+1fgGC^q`6(>VqBChyOBmT0~0^YpN>WoZgH z>-X7o|GA!>KIHCWY$b~x%e`mPYvw?EZ^JKB*D^ftCoS~lgGa28_@XxYco059w6;of zlj+cbLU%2GnmECL?x>V}H`k5-A@Ta>B=qTJ_>vzO-XzohHuEgrr4bs04+q1G)Ix8> zPuxqI_z=$Ai4QruACL74-tO`tS5dakcpuv{-sl(aSGySJ|1u%>n-S`MQTZo`mt%RL z$?zPnLQ4wYND1~MkH)-#IsQC25Kmdo-3nFgAG8)t0wa-|n7wrS(wn;4_ThX4b>_*a=_t-U7`z1}+w;Ewp_A zp-nzhcS|4c-G+ue?X=hAIT@=LjaEA&GyC}G!uu46ww7>jFk>`SbF?L?Cc(BHu$!#l0lbw^Hv7EPW{U`P=oxD@84)J6W=vCT<|6JR65Z;Ua*sg7nhjwXg z6TLhMZQ8eAd=m0omrweDIMLXu^X2~t`6Ye1fcoAtbS%^U8evOOiOb874$Mp-eV5XK zzncgh+N!j4`%cncHUDPY3E+x&nQudXvHw7RJARLRhDD&64OQP8WBTBSj;7)>j6JIE z$7GLa+IOjFo1qKY_Rr``Cofapm-H-q39!&gztraK%%kYzuT97u?c!f{iq`dmKJ|xA zr9!XLpj*(n#Toc+=^oU5i8@Er4&KT~-hOys@h=EGz+bbc>EvIMjv?2=55gLj=UT`1%7X9!btFpuP0jk@*6%$a}?)O{E;s%mR26(E|T?Cw`J!R{=7WI{4j4vNc@% za!XNfe~h*I*sQ+(mWqDPnf)n@=RExWHz$RTO;7h1=q~S>)+zBcky)e-9FJ`VHlLjH zkLY|b{7$LT_JfZ*d$+|(cuwH&o^}{Kch8tcGn}?uL|cSQjjYL~MM>+>)tEV{b?Mds z(nSL_PvY4mf7G0;bJH_O*IKE#O48Mj2zpi1K4_q?fH5_CRZp;md$2{n4@@6#d|Jp~ z)=6-{IaBHA>T>XB4(Im?C0jOgOIdsQe#GFK{OxsTPJ1x7iOex{Y-$p?#yyQy2?p1+ zSJ2(0;twQ4?s`Y#Zk118-ZEjj{bv(q*l(CH(|(Omb;j?qyNS7$&&6`^P_{ywu8R+@IKee88P0Qu?x97F%X``dkycKY6GKwdTd(7AgSRC4DKX^HFC&O@*7 z!(IYC)7!qJ*VjsqUCMpR9~yfc*2n9dIevk*Xk0`GHyRoxU&*<)*S()^6>+0!tLTx# z&p7+dhdT2z#*DRSsCaHCpL|w2eChr1IB2VA=S9#`^jx9lkGz~&00y9`kp&4M@d#&p zEaun;fE|@pLHl(7l;OWVku{JsYKj^F$J`FaNVZWfHlkGo(*zO;0%vjci8 zeDh&n8sASofAsX49h`>|pBzcZc5M8iqkm(M`4Z>me#G+-PZ`f}p7Yg4#%^qC;NwE* zXvCW}=ooN6_cZ(={*fjj7Z!aTgnz5a>9^EOhQ7{QnvCr@zL7;+p{0IgAdT#|N1OYV z((S&Cx8^{5Kf{Xw4=Kdu`%;^A{tX(sIW9}sptbsf3?oaBJ^v@tdLs9L&k%i8pIrKS z_WwQ8gU`71^|i~K{iOB)5#Ikdy#E>cD*1qO*O}y|wbwdueS&hdp2ADlYphez9WQ{F zia}?QOAiyxeIwxHJw#sNfZAHX-pHsk5i-nHpY zoBlwXBzM>*eI|U!weI@xD)G+VKHVM+ygTxbH2Z2dJxsbohtupL(&IE&v=-YM=puMl zBAz6NaTA`(j{>?#Xtd*#+4vlA_jLGCq&(5@wIR( z-%ebH{kZuj)vggu{a@(iB$Ji=P*p$lw{5kYyzhu-n>CcR^=)`G*)K`s@TWi2WZe(e9Ya!yZ)CfPWW05 zuH}1jH~ohAx$GNqJ-Icz5^|f*FDz{jozpG@69rj8k9$7ULwi(~a3tTf?ThcsYuX5n z)w=&;LMVs&v?gXiJK7BomD9YRcKnL;+l;T!RJ#vp`Bq+Y0=(5HaC$kX3-LO@yyGgL8(bn5s1KJltzHJ;Q$ze7lu0+dmE8TpP0g zm*(JG^QDgLW~;%sEc@$%94C%y9v|mUlVHF%2}&(tej-w;WpyVtY;Z^ zJ>QG}7d%V+zTLQ=0j)>zy9Ct!uK48vzh-l$jC1(YW)O~sCZ_UqFgJ>OFClZ@!<;er zHhTzt*NJcO_~!3Lp73{Ya69SXW7$I9A5dT5|0Vyqk9ijFyOKDqfrh_#<7$X==vuD* zbvN#N#A$3=@~tSHu78l{Rvylt3{rV>$d`XHdO6@Vt>Qc8JoMKThptXKYORaDap(fy zQChA&!%cJd1f4X%;MyBGow9|Cs8{`Jd$R<3$vuL6YwT|^VU|6^gsJvbgqjQG6~FgI zJVV1!PjH9}^HceE1F`^T98&FV;GRQ+^6ag=J2X7k{sr%`m?tFvo}%Tk>DGho$TZs$ zjZR~f>TN@gvKM%gE>7oZB=>k9IQxJ$%iaU}!#a^>qkkO6{Wsxfp?Tw9$z=^lx8I{Z zI)^QLZ^;7U{nol7damzd;I{H+)8fo8v;C~|66dGE|e}~NfnRb&2Gwg^7k;f^dJqHv* zQ}>y8(bTE*<$C(`W%_m#>CNO*(@j|KM%R8~fksE81hj z8S8Lqd*aBG*cu7;bw}md^Q+N87JrVk7@OA!_a5d+fOx=AmgNvz`cPDM` zM01BuvE-KH=8r06ObU3n0+$7EN0(LDa%s?vx#)7&cJ+LjQUgzdjz zvHsOBl)H<|GQ4c zhO3P)z~AomP6+J-Cyb3QV=G%v`OQ8E+(r5rc|llX>%-6V@x(y%ABSVZdfbvZJwkeS zPkN2uLG*ebGO_)@lrF2o7%0mzA|vAKXee^;hvo2x`D&| znk$R>7zzCeG;=Qnuu!+}!9Db2ep0uGr}_J>U)$XOhif~i|5)nNYjab7xYo!;k#i(n zTD>;8pmN$+Pq(92UiRo(pC`xN)BR&@Gd{78(^lc;K3B##it!U2Rh{4p{p(JZ6u9@NWP1FVH|1K}d>eW_r6R(u-Y)-%t@S;a@L=3lKa z2<_{*3qEZgxHwa^Kh6FgZB+Z06E@Ca{^2X?xFbb8g$F*v1{U5+$eAbq4L8CU&YEt` zPvE_2jby{D%fP(&ibeb{ycQ27+M~E{6X)!aa_y;Z+<%$4-k5$g6W z`K8&<>0aZUx4#YFfZtC3YoDmSaZYpl)Ie0S$9I-0Ez=%Pn(j5&30#QBxSH?CM0j*y zLv4gUu8i-S4Lxnh?!5E2C+(2#y@C42^DcXxfI@hVs|mB1M~%y96PIR>B2@nKi8uSX zT{r26 z?j;yx`dmIB_!r_Ufz#LmD=eLpbuE`7VA{+AH>0$dZBam4zKS!3NHs z-qYLR=a15k4afxArvp3e-|IeDv}d+^hAhdwgR-6WYkb((g_FltPMb%%?)mgzGDtoE zSI)UEH&`&v3KrhpEtuf>uU`~%mkRrqVeR-+AOF|y?5T5oNq?KXop(Iyzo&A8)-x9V zr@ICB!O!LJpX5GG)F=N`zCvG3`o9UX^EhYOe15-E_YumtoidhD2LB~d z25aOnGtOi3f{W-MV|CQ$`gu0zB8PBJ34cB9(*xl){;`nfOvmOqfF1VBS7RMR*ihvS zR*!MUsE#t?@bR?X;lI1c<-P**_zex&$GY{XZXX=iyPj<7exa1jWO=Kn<^^%A1>nD;ZV*P^L+?9 zl<(;|?~cMr?H{gwG^f|X!3*Vo^D+JBNog*G4*B`E@Ygw(JHl><4q?j|2Cs+J)q38J zBtlog69W^w@`4G>$M{HIP`ILfSvT5e?98A62Wih6y|Izx-q7RVg!=kb+S)+BZ&sga zzXj}0?c>mx+my!IwxKU$q`BjKV;ruF6N=>vW2M#iykH@9B=DHKp6&~TJ@6EwQP=5z zvnPkI0md6CgZ1C|4qV2W^f;Wq*8~4+!1{X$1@i^xUE$+y>geL@eC3?8;3UhgxQV&J z$5*^9KH85yAl$W0-H7{_+oyqUn{IXIQge5Lb26HnBg{+k{YzG*+<#vGW^c-rd$4t? z#9uez%4M9J(f7ka^S!z12Tr*BM^5=L z7C+_8V?X)&?l;!cw|U3WSlQ*|`4F5u%$dP0dCy<}WZrYv59j~yw7!KymIg2 z{4v+xnt#>xi*j$h{vUaZuK!i{Wnd&DW3O`Ew!b9B2MMgcn!+ zvuINT_n+l^dd>R{@c7Z{$8!2C{L;cd1Ai|7kNF9`4LtrC8YsAbo^gwZf{!}jBZeHK zm^5q)!m@R503Lqf!o#kMfd|rKr0GAMfng7EG{Tyw^GuDYf#{0M&=*{04KuVWFeK=M zc3Iq?8=)L%ChKCRUBp_nk1_j9I7tsge|Kx*hF`IXoA)hpI@O`AYkfHdXReaX^e+)3Q%uKAPB zl(n3+VWgGI2Jo33S4C)-HLCI|Kk*@a|99y7vdXJMI@9BgpXo`rHwx!87u0hhef+{X z_}6F8!PCq^0e5wuF!U?azQmn_V!{)mzp8`xCg@%3;5^wzZMLMpe*haw_?{$d88YlX zA@MTUN<~}ockc@TioO8EWmNVxc7(OK1zLU_=<8yBqv`h2*CUJ3i z$Y%Z*0S~w8&Z4=a-1(*)+M@aXEqM(roRg+`|Fx-ua$Q)^d~b`#xjff%#BJ$XhJg>K z{NIxHQ_>7R<6}vD&lP;Dz5Cz=Q(T(6no#ZeEc{LAC746rLwQei*01}HDtBK8{53+S z#nXn0t_X=PM7+L1b-RN#=8jJLk*E@5lx-?S(1>es&);aWQ?~ zw?H_b-Fc7DH#Dx`e4KCUx-M$GRG0E=j4t4PvKhBr`)~ByffveV4knORLfR|%Ro{S( zMu)kJX^4zYwW?jY}{Kr9v@T6SCQbGhmN~Ud-h)e zbCLyW|2U4gSJObSgo+B5X#LH&ioNIqTyw=)c#_xI8R{J1vJCJvG$_&og zHz%?8_MfwFZtPCn|G~c5lzr~Lnf9H`w4#;Xq5;;9!U3MXUm8%5RWJZw;-jKNk(1O~ z!C159hc<>KGl0+BEf_x|3)Xq?RGagHPXd#+>@FpLGkd+q&NKS2z2M`_E6>P-CAWCm zj2Y+i6hhm?pB)2+erUc^?S}|8{-1VlRR=Z}@w=q*dDUC7mc0jru~=F zkh8q$xDrBhXNJn-%m#X%v;Wr_3-ae9 z(-U8pmR`-h>DJMXEy(FppUx>oR;n|TKIECA$-5+Xm?ho>TX6%2+4d0XUWYuEgDh!` z>29@KvL?<8hke*?_*j=4Ym-8fE$l=F@d~hBUGc!4w8FovRsZt&-<+317O_ZnhR6|e zh|`&E{WG3@UvZana1GMqZyVI9V~qG+`$5_@z>E>+f6O@IM@S)Kon^vQJHvz-cC5;W zJnQrCl~P~gZSZmyPPhqQ7v4+1P}B^c%65`%oysD`vF|_?y))m-shg(yv;bX15 zdC#O;W$P2H`3I3*;4=uCV9o!4_qso@Crn`tg6?6T7Aj!QoV}UHKCp)SdHMcsZ}=nV zW@SIdatJ=z;IV}R|6rWzj@^6(aRj^Q#LvA8A<+ zuE+0H!IkpsvVK}pH@>mI-N76t`FQVR&7ZCJp1gCHrKR`2yk}bTH!zn;X}srJ^8_KX7+k}!7C$o zVz2n;BuAz)9r%c`-}ZqE4Wa?SQv>kwF!G`RIN8;Qkz~8%Gd_$alNR6C@2A{zWk@SH zS6{$A?k%hh)vnyBjDO;OkIX;Ft1RSI9^_Rz&;JnrRB8={C!7oa36F>D$qHg$XK3Bc zUMsDbxs!ZAu&wkx>}7S&UkmF-ydKGYsugL6V!SHyRCbmFa3wj~p1hx2UylqW zoqB$V9OVIIC^sWR8HEhxi{vNgAwS7GM}CqsVA1u9jQr#v@{@zePYxnKIf(q^=g3cf ze*I&v{6snd-IJ3~-k+zRCXlVg5 zpxVx~@lINTLt6u^YdbsBN}RMF4y~O{U-x&WO>okZ9ok!o9H|J~Yqcd%e9fYL)R)lP zdQh?>`J|~~EFN2MUjIYho>RWb{5}I5d>J@+7C6v%EpVXlIp9Fy^T2`RYkJ=ZEc5~v zHZ%8v0nxr`ydMS*beG1lyp0A9Zu~!j1If|;Vc_5haBu`TI076T>4bxiTsSz4PaM^s z@98~nU*Y8TBdZ_J>AP^qLfzAW4DE*X-abZtrn$7Nlp18b{d8w%A!9X2bP0MSSxO`H zDLKhAN-(qu9~@9=SW26zs$-nVvH`=!IU6WNIH@bEy_4u$cw$}SG>-4X~p zHm=Bn1C`Ud^Ymc+QsD!Xe;@hHSqWsl^?j`Rz$5X9QPoBsg%j!G)`5T)2AHg{xW@uAT#~41Brr zTftRae*1G^O0cyT9Mt~6nYSc&-nbL%omknQf$`nI*+-1e5#*?G8SXp$d#()kSIBT* zc4fFPBg1{gmEqoo4EL*_Bg1X~92suk^BozkbN7tyhskT+VPv>{<|Ukeefv>hu;0Q< z7wT>;;1C$>YhdtC;9(RR4d0TcP;*|*yT(%Ez7RY+1dk{>zl$}s;~28d7(BmZWF>vt zk}5fSm&;kF!F}P~>4_@tMT^c*V%js=;yH2^krhhA(%)Qo4QEwN1T- zd6O-j&H#=Te6j9nKI1aZLBt8x(ttJA?IT%|XWc%`m1mI;_>!EXubc1OG|4%VNK={n zz$4|qlelhO@~e*tMtPn z;rH9{OTEU8Do1vO&G7C^Dcj&2IHsiJ_bi&)ikyXA-+%Wp3 zxcJy#!1wdSn>gT2^D%)qt&feh*gY2Y3AI$Ae{#pU9{HVM!to20YiBUdZL~q{kPdjY z#+r81R@3gZUY=0~b{RAE|2Si%x$u47TpYUC%!O7s?H@8ga^{sr&Rk)gD({pt_YG-(sJkUQa$Py|KxF^SLE{4sjtGxF6dt|QKlI^x zOFG4s0m+$bIdAq-TIj2KK(6+Pao~#D`mU$eF1ru&v(PwX?If2d8DcAILs_3 zE{nKE)-P;0=6|RE57x@2!HZpy+O|UeMUaiGciU(F1wNkRKQ1aVwmNZnsqU|HaNTLk zZ^`3{*9SbxSCZOzfcTS?*8)!c%E&Kr>_^V&|8eHr)HMs;66=Nda(yRQ-#6#zYE^d; zKF?;12#uh;UjoaeT8k$+zRy}0Soldvczb_|XkL~*OKTbQSMsfYW4HJjG~}&9hi`vt zeA*=jZskv+8|6E8VAw!F1IiMSO(Xw`(~b8pbjofZR9hrV)mpE+(?s_rQ`NWVzuNLH z`9xcUXM1>8n)2xk;hQQ8od1(T==SRh8UH^jgvS1XaN|jx>HGY1M#o%RCY>{qO?>=r zFxfJ>GSkj8eQEC_GsO-7dNl3ZgtxKByn+YEWA&^b^~c6q;rglL{nhZ&@*O@N`yYH% z!5^j0V~V(=Gg~x@A-O0Pn&9 z^Ue5ukzH&fwh%G+wgW{vmyieD(s_<-`%?CL;=7hp9=vxwlOQ?q${hBG*u2zBzNNbG zV`XSV9ds;&3{!q}V%TBI_X%>kQ6Cm1_)iY>_*B_<8F!df6`h1*O4a#7ssbI)1E>b!i~u$O|t)oa_M~L-Lvh>TRi)+c;SEK zo27nYTUKAP@yp7Nv!cJ}o1g1fSD3J0_WSqmL9PZD&G}OJ9x&-$>bGz54p^)_GobS7@+UDf9|X zS9<#ZqT7eDzhZnf_Sb9a1NS&!d);Adud><5j#`R+?rr1$m14idJ32FQME+m&4I5qM z3*<>gP~FF&vsk*pma)$|{*};S*GG0O{TGa62Y69qp>I2tjt)j&PzvvQ&ZJ3pl}WSh zw!+W%Ayl7R+3PEkU3Hv=jriFOc@#*M=x=aWht|S;Be#cDDLra3 zzRK99_Ql$!VcB}|`HcqRoIU;NwCDL|VT;rV_(s`Iuz~6bF7#GZ~^N7`j z45&Hy9X^NR2VMD|@$rCHw`u-3f2q0j^q)HN8o`H%Yx}$reE6kwpryz$Wc_abwT&m=g*nMxNX(LvnDoL_a}MywM@Z%pD|K>Da}OIGo8k6)5Ep8hM(8?RvW z1C94H=rz8d=)=(g1+V4Qt%GLeH(!_6Biy z>RhC$FTa!d)@lDi&+nvXSo|}x#ELc(8^RyN=5x&T7uwy+_?$$J^$v5+J9;_aGyW!} z_?sN-8;;)3c}vfCh;Mh79srCY#D+Op&VJzZxT?k(9eqwBB`Nrsq;l3L$?ogIZ?TyR z`N*c)^Z0M@qDL_LHOAbIgRalX<>Yy5$e)phhq9irWoBL-oIXlf zSG6^jeJ|Fs2RefE)wtRQ@qJ)@S}FM3q7?d%lJ3RM_eFBEVqxg~kz|!qF>uYcV`^{VU7wq8oGk*WgPkXEpBQs|&slk^38{}qw8b1&gp za_9?`NpAdb^wdDCd^?o92r_QNxcI;aC2@hqyE@Cb-dV6@tz$EPaGHG-^aCT~zLDd~ zxahiHab(;E)`s)oKd~~4k@a^BXqQ>OMqhN!MY6tp)aUkhc=aRXpGzJ_raq@UJdQSW zFP_@c9oZV^D%y}dywaq}_9BxeeW4EF%FOG|;givRmO-%l4`t?cl9|=tEM#V4EJh!t z-)*~n_`D?I^U@ukmmc`M^u*_-7d|hXg)8;pQ{kLN>8!tbco@k)69Zt=Va{B#uO<8% z*UhR-=hw*Gzs~P@aO;=I9$QN@c5FR3B~WbeF?mT1`}qbRhihF=UUCc1Vb=Hn^J|X? zR$YrsU~{L@6X58R(9@53UV@yVcQkLuk!S0N^$gVJ^$PS7oX5rJe&|DU*zP+k#BIUi&gnVA^P6vV3m#Tp_uK+``mG1aR&|j{^_oP$kSH@ zh(8mE13QuD3)Ibs4@4G1(|5%M%o^jf4BnkfC)`|m{SNd!-9uCG7-y32W89Nh z^7|L+SBd+N5rgvmANC3#UEiBKM69UdiAcwP zeirr>_-yWr3=Bmj)_kO$kH}T2+!uE6w*fGfLVpkG1ZH)I*=G z^-ZeMd6rguE*dwZ!_A?6)@sFSp6E+nvKzcnZ1*zoW+Hl7^+J5TZx2Q%alf|e;KW?n zz&cnGpRwd|?xA}Y99x!5{2=g3u}iZ!bJ)uMsp|d;ShS*h>x{W{Z+-i{ycziR#s^N# zM+aXl{)CTQ(pAKI(3j$e=Litxv6SJAgN@B7@Jh%cP^=1BMb>)dbtl6cTfr9sNl z?djw-x!>BwJGzVKTwR4OKXh0;RwRi&)CLrF9ERRas@nuGXOgz>YsRB8@&kDGjA zQn0P+TvEB_7Q9Sd!FeU>rBXk~O07QyY#XSd82fl$1QJUR5wj8?8W?mzk zr*T(Hcta{NyS(_aU8twM9_iJB+k#VV+q72#Z#h70JJUY4^4Tg=KgAc+pF#8o*tPQ# z^bYPTE@+R%{c^4oXIyK*Y5q$uZcz#hxEz#u1viy42Pc(+mkmmP7>s^mp3~8f6F>Ki zUGqZdpnO_5GcLJ@(UBL!A67oQAsrn#!nw?LzoXGcN9Iml&JZzo^Wq#HRLysG;g8rl z2U^OrOkBKXc&V!|Z;92H#j9AGjJ}L+vc=O)&{el|(3gjB#~1N0HzGq6v7e*8L+L1{ zUaXFcJlT_a3(%2U=wDL~{0>>!f7%LV;U5(lV1?4rmy1iVS)eOlPFwPQ4=l}h+Uleq zM>z*%p&u*ui@^_MUFpcNeUM&!c;3~>$LNLN(oy*Pd&rs6i<9Gk1#43S&mJDjZ-es; zQQtY*(jKVd%zl04-3NtxI(Msf9i2PbzJm6!ITUc7E7~-lxGKv6M?-|}GI%?P7(E)&Qtp0vWH>du^)NiHVX6&pH*X3E2ErHgYoHGujM$gaD)e0z~dBi*vS89=H~~VadmjcPdw@_ zp(J$pCgAkkkx}6t=p!QOh4yXKbS7Mvvd;>G2w~=E%|GiE|-bHDgY;m($Nv;7eTz z?~abEGO&_9|ELvnjwBgC|xjPNUE7Gy42)^!cZ` zL+CZu{&H~aY3d7J=<_WLv%*bBtZ3bzIY07RXI=h0bol}Bs!aPl7lz*khQc@5Qj7WT z;6t{(gmJgyo~z4WC%E0?(mCk?<~LTKH}6h$&XbIzZY-Y8fxkQFi@g2!GkACrbq$W* z)Uj+4I2WKS7Dt7LgT1^ySA;k5oE!Uo49@PMEF-qe=bI503w70|2=l7XCx~Z!sG(G6)^9~K)E!+^#qn#6D9D2aH ziFUK?M5W+yf>LO|E2-hXeE;Xt>%Snb$p4K#b@e}q+uzsGbt>q z!e{3Yzi0SuE6r#k{$=!9{%OxBW(W@n}X5(JJ}T0GltLIhuht4zieANN2Bu*P1v^NlcD(2C)g7D_ZQ_rLS0k?jN*9J4C_#!s=(Ht<`@HLU#Y%)=L9rh8}t^nas1wALTB}!{`!Ej|)~kjqGNlqke-_I;-~iICDMw zqtU$s$Dd0G*K_B(d}=;9kNLqCSP`FU_L^&Fu%|;z4fM}?*WS<`HvzoaXYC!-9xw0@ z=G|cQ9a4Q?JgaSYx6hUNbylE_a{zwMHAQ0*~py}Mhw8vXq#d66_{?=qfm zpsRJ9hw8j``n{1c3H~Q$1*7tJ2$^vQqxUOiY(G)TSl22Yf-bL=HTZVY2f_6-F&vYT zpV)VievuO-W{tz=en7niz(euIx50-S!0o$Ux-hihmcr=G+Pj=PAawi=r|xpK%^VlU zgUi5gB4-$O0gsA!Yvj)O_@y_)SEWB2eAIv7n-dqe^f7Q8THNP%@AnQs2mX7rLdL%| z&8jlr8J`;(3Ln>4_am39?ZdRam$vEC$9GLm3@yO-+tjrNhF*62bRB(??6#nrTefFl zXqQ_ygR(kTCP8bi`WbjAU$7LuW7=mvlyBPnUQWod6()v^&h6#z8{p(0_8Xu2u-o<+ z+U^b?Q|vDJvZ&wlb$&eQq0Cy~D}U9*5y15;R^{HWSflF_z*X}5upS#2v2T3~_;U8# z4-QRo_pRT_cIN*3z(e?|Gb09$#H6wkgUb}hAd+BJ55w@?i@Jd-ofLC%p^VA~Sy)}r?uXAbN6-jl;eg@44j9t4l=8Kc8b4j&V? zXN(QkF|YN*1_W9c=LG8SG3Se_>Y)klmaIC?8D`=?Rh>*Cy$k)3v&^;lo1Mx*ZiU9U z4?e1M&$;-d5?3V@{N(=el8ftJ-*s)9`>1dBG=cf6z+65~(oelRD7Q&If}e~amFzj1HdI~%pBT*l zUy=Xsz%>4P=GxT>rtseVYS+-2BOewbt7-1E&sNPD*Xq)rjMw~|xICq6$JgL@b%Z@M zy<@=_GzQV>O2)xit%64U?mPEG`0mZ`b*RHxP6^R z|D5xQDfSH0SI&pi*O~2oGpH&-=wbF*YizTmR%WJ=Jqwmlx5ma#g@5! zO{YwK_3J!2eccq(*WNB3K0?}gzN79q#Nt)D#ZaV!wg$y?2G!|Z`9W`$CeSHel5ID>#^26;>fSPmApXL zHIMke^vWX*dA-azbz%T%uXjtXd;^ECn!uh{be21=1*B@X%=G~fzB&DOVwCNtKPpqq z2mkfNcIYL#aA-hdc|~I}dQX}0OB8--eej>>oHf*X!X8Pukv^c`ZZWzKeHh8RC+Wu- zI!UreX&o|jl4MunH>Ur~NhJdu$ntO`$^H}nWpAxX2uAgN2cOU2h~{5?KGR<(cEDi1 z`76p|=VJ{9xE z_@iJIa{j)_)#t+tGVGb~JPl}+j-V-SCMz>FuuUPoHTn5sGZ_7mkn18bGS>((d* ze||{Xpu0MK+CS^Y-cSO2M1J;)xPxZ~G1iy&AdX~wNN0f+%k6Xa&)Q|(zrFeC`fBtM zi}5~(%tS6`8h z+I=CBfjjMob(D)@_QO(nSA1XlqXlXQ+&aB{I%Av07%yk6MT~hW`(c+6SJK@NLl#w> z>7xtdfbp6Au#bj2xKl_w-hPprAdLtQc6lK!+K0xmNko79S7#g}e?~SA^=(j4R zPZ=4)^-=WdN;Zf7zfkA-Pjno$i+me0`V_j37Y9!IT7~rMJoM3|V3pzm9vRNP_2pdx zwFiBHhVAiz>cY5CT|-=;4P6w!D&zC3z48t|o9NBS?9ZJ5&x6=g4UXP11o%mw%(5%; z&)}*2caet`p?z8q4{`wwcyaY|XPzBodEc4)jaktEi_rmu;qK1aUNd=vgC>;jyJuGv5y} z@8UVzsHc8CLY{Etr*8h!`rcyf68FI8Sj&HjT|_pTAGqb4DVI!B0^ADF5xwtji1}{W zaOb;|ICHD-awl5l-R}GA-S1|Sr|;h3=7(awi>;#|o4rMSx7aOTqw*PyaS5S+uHcX9AH9w2X2U-X=iVgJwDS; zx;>NnO)+Ee6W7%%o31ji|J?Pe_B!`@-XKf%_^4m$40OYRq|?z2mCj?nNd@m9^goF4 zznM!uc(8XRXW=EsqF>AR)R&82xA9ZWw9~1l*!~N1(04TF=dll=l)b-n_Ws6a(JNwo)&Z0e@keUxf`@W-u$rtpoYtyH_G zTb|&SZ#>ynS;v31(L;IXfW-#zN3bX&9m_9O&%j_WbML|6Fz?>LvtZy~v3)}X_!>Ac z2ZF;wbUY6ZuTxiW*bQxI{AJqLiGjoOj7xC%3-uI-r!8h4_tv+K)ZK|~-uxYJ+qRBp z)|>-5RyyHWX%n2)-j3RgFdymYCxV;iYP-Jev+5(#wSkZFeovm!MO~fC&lxT+UL^V} z=MqPU4!benS7kUf{}BCB`{{9MJ#tE|r4_Vi>Jn#h8Fe-eCeGq=t162)KRG#B9pWUm zaAqQsuQLz=Q73mxNN%#uiWmQFphv+A!p^3Wy9=%8(%>L(aE1K2(sgYwg z@{J%k*fJG6VIli;!>njLzQVcmJ$CO?d-<#ZW-oIbWy|I3c3WV>ZrWV<(3mr^5YgKY z5gUHhSk4g*VeZ(oH2a@k-PrxlsqDQ`H+Kv$h9(pU^2YKcb*2ng<=O%9ATGpOY_T&QA>;&YXRRvbT?W`$5(rWHp8b8!o zV{hbr_R7ru!U^`iX7F5qejQmjsVednYh+Uv{0g3Zh3d^woHp*ZinF4Z^865b555Yg z3*b#29u>NHR0JOFjnC+TzpSy@#NyImgU9KP-?@uRqE(}dgG={;-)G@c!N|}g?jn@! z^@DT=mrgCdJS?01Ao?O2%{6I;JwPdR#~A?=mo(GlC)?>JCH{w-CYvmOJ6Y)XJ{xD-$b=p$Qg35Ho0)ZxEmwO#p zMoKOnE1Qw*AUAVokMa+aUk7~R$ge?8Hv81H@m|*4zy$VhRbRHhM9wh(9A5c5=(?TW zczBU(zNzGNz`huBc7bRI!5fn@WGUvW15Y2EkeFCY5zY%PZVI>jPw>aUx4W@k;akw+oy)Nw?PpI&?d@o4+qRfC)aGaQty2t&-SjPp zU#9wWu6NqwP=Ig$1$;eTkk9{f@`54of+yg0bC`GN(^r7s#4xL1NdUOzJN#hG5QiVk zrHxpAFrDYhfMxhWaX|4XlI_dMvyk&pT%(2je}MQF)lXQbB`Zm0lW)Pf`t3^re-yKk zHV#9dUBHK#e0Ssd%mX?NHpzlWA-J1BoQu4<0s2v8aBAbI&`a`PGiS)t!A0((5H5l{ z@3U^Mi&_t@8jRls?IzPNwVzAB)SoO;(Qk1&=gN{Svp)XV?c;iPjtw6HXK&)!;dhyK zsavN>JUND^pKA;Q83Sj73UrPdzGeCm@pUC(O4>wD_dEAwbozn_OlCp*Sa|Zyo(2}lFN?3Ct~|9|CRS%$9IJ` z9N!h`8X5Mln}8qvHs`?qo&JA3DkOb9w(q(>RC95m`cMC~RwT1lXpAqoW7wgTIn((> z(U$&iCGURvtp82GU>nbWVb0FQcXGNgm%zB<-{~W9PddX`=Y@5@sru=opVjPhdT@To z?c@EVvQvBgI{^+ILQXPgjp55DkQXY3Fg{>dOe* zGM;$|vTrL|h(HUv_u_NuLG!?PI@7~nXlEBVBzpJg;YKsQ6nhS-cpmHXX!)mI&+{F? zTX^7{oxvZ8v5C*`_EFcR%P;JUi8FI{U&8cd0Xmx3k4xNsj8s2>r`9*ot>8J7|DL>A z4m{r@ui|@E8{)aU`zZ3B{F9E9PYt*GCowmNk)w1!LI!=!#ZKbE?sMUmKHA0Kj(lV# zeYM1c1UDCVdz=1d+KHs5pV;0dck3L}r_?v{KV>JF`)bxf;f+NMBBkOhO~6?DNzv4} zhmL{gvH2&kJ;{Gdy1lo?H@a(l6X~t3PWuhKE1SFF^X-08f^}L4KdE%qclGBO_4S>T zODy~wt?1uxuqrq5)4l;`JqzlfwL15_IXw3Fwd>=I|C8~P%EDeaJPki7{8xr&ggyV2 zT4ctdle#orj!#hSIMzyNwKgv+j4c2f&$O3b+FS?%d12`VPkTZ{e0=KY%}-Y4TVj zxHZqgtqg-(8TLKEO0c_=v>EzSUvKCCzu-UOA;N#*ykf#LPiCJ|XX}nIe@DTo_nH5B z@h;xO7sWHqvF_{}=&U>YCb#3gOXJ|5c;PPftrOl~t-dkG(@DkiggZm!_lGPLbn*U5 z)d9y=VS^ODy-N%z`TZS%zZji{7!$}%>m)Z-5KH_ha;)Un!{9c!yiak(8RyIo-S)=n zTj0*cY8U*uK+oXsqr@UKvBlAoyx8J5qazgY?izlIK|jB%6JLCOT<%i8^h3#*TD#*q z=u`VT#1;qFi#dy*ORRK_uXEr0lKap()klp|1^pWwU$HdY==z{}G#mz>M8nm2if8Q5@V&0B_6_=G=$Ccv0P8|$ z`h9`C6Tr0$n#|Om#BG;!i2op(-9#OgcP^Vk*{>)wGz$)E-$OL}^Vs^GXf{OIP}vfo zQPHg68-Z?hHnBp!eKGXfP4o)Rie6VU9?1${NE7Y!+L0z&P9eukap*PE&}*i>R`W;u zKOi;t!SJ0@{{NEs)?DImo-BRFtWUsKalrK)As)!7VfX>!V`%v8b|)@K?KsM4ueLlX z%$a99vKcx9&-Nmh7+u7T(;|D;FyR!uBF&wrZ}DB(IBS@vVqhdV$kyd^eJf{?C!aUv zdHhjxGl6`Q2mcWKH78@pt4&is#nV&6P0yjnx-b`<?aL<%D*bZ&KI2N zbFtg!bYir84xijN(2?c$O;%i%bMVPu0e8vrT7%OK+=-8E@)w_7nWS<=AkqOkmUb^fw;EpT7&!2USnD0?y9ob6Q{g4*pF;SUa3(R{{hVzRuY zm@HZ04~{W^2br^Xz-eW-BEGSZ@^cFb3Yy6ve<>mAAo~= zxdjL51r8jt+vC%eyKs7Ty{d&T{t1%P8w4Q|ybCDOA_P^%ZLO9P)(t z(A|`ob|*Ttv6{T4ULN~kN9H^G$KQ-?BZgn?q5jvs`YwGPC2xW1-;OLRosIN3^rJbI z|9$&d<~e?>*Sha}eyoY2P3e8$yZl(k(cW{=qrrRaCl{`o(6ZL`V@2mw|M$4`B0e}7 zc#9s3@sB@;OninO6>sJo^jLM$k>5p+sfHd??aK`g6aRx${b@!f8P`6RY`eytmqCoh zTPw`|^=;nW1kAI`dorlwtQ8&0G#~v9ENIVND>NUyyu6tC@VjlCVBgHIZ zC^u&=B#!|9GM_UKKXT_G!|i(qc|oy+df|&knPi(LXjk*4|AKK>!2sO#DTN2e8JHy7 zZAt7WxN^UA$2j(iHCFk+OJ5g0ylwa=(oEyw=u7Od#rn{aem~D3Sx*Fe>&a^ zmyZdbS?{v#X_{x&kHfUP_ria3y;ID4GtV8@yNS&6M)WVuv$x*8##q$;tE5`*1j905 zSYY6hZR>s&*|8(D;;K~d4*DQ|_cHYzewS@Or#66>)}TKy?wUeJA1I}bE{r`)-nrne zcLbZk)LW-Inm6rlkY@8N`au>l>uwX{m%ozUd6e&Ij(&k&rT5LZf&CQnR;nHK_dzSF zF|3kb0lrcezEZSbfNixa}P;?ed(kw)CK9&BFC^Crq}W?x8Smro(F`=iMkZyx%Z7vE`EN1p~iva+}Lw5n5lr^Ef7 zJ5lA+(8ho17XPVF!?UiPv^V{U@o6w|o!&GyIcT0T;ot{Zh4}wi$vfj6x=?;Kdw{M# zLwn36)r;|E*m4g4?e^G7|Bt>59b!6-qmLEr@rU!y_%VDSKZZm2G2l-xSzEpg4N2U4 zXZ8`GWB8!w$8c;0=hwiYli;2386O5>PYq=a(w&G|_C13f9|oQIk`IIY3N@$Rovf8+ zE;8)jn>5K5&9?b;H+^q>O#0w!LQIu8{qQ~Mk6%p^el=Y}x+Ad>{F1$~*^Q6T>iUpt z&%o%Lt`GUA`#G1MfzHY|g@4@bJ52YMzL9QKKFej`&uRshRlxox#=ns@J&LY5(Pu4-ZURn%Gs9c}UGXn}Nqy+hk$?DwIbd{DOeu5Y@Nx_aj_@+x-E*|E_# zr%CO~4o#f{US0&fU?Y4x$Ng52`a1V4-c4LL&Unu?v0n<*2l5->apHUCY%*i)=f3w9 zy+?iV?+R$Bg7=C^XK7D;Dms@^-rcLcba-Jc@>lB+&W!`(D6&EWZH?m&bHS<$ZMMKm zw3psu%2bE>K+f8&y>;F<{s`=UE4Ghd*~s@JImn^xPnI(u+r+=A=U=gHL+x-<`K$xm zjqm`SA*@$@zA2a(e+K4EFp&+Vf_uvaFYG4x&(pr@(gya2`n}p;=e@U-2fxz0lj)Pr zot`x}#{=zmbyoB{GdIr&roiHD@~VN&n@Snq8zxP)Usnno|7M<3>}O3XKd)KP+DvHf z8ffooXmAF8URUAg)zz#I9ev&wEJf7>6y+oPx zRgEJTUm*Da8r_)l_7&iEggq+3NAV6SfK8+ibIqC@nM2G2;Hov2^-}sX_%2^w;*P?{ zk20UerjIR5_d_mzqCgqmQB~;eTx3 z)Yh%)3*%i(x<-7mGyy)z`P9W3%vC0Hmc`sg-^y!zC3fjFMEkk@$K+4)H#F4xc~Dl#wT372wW6zxrVy3L!Sa?uQz2GcEBy0 zMw$F*-Y`BW>2{u*KZShx7M~npMdx83&t+^{2h>IevF+qv!}&Sx4Ow+Sb+VAnPtrDb z*j8?Lzn@N9`d$UTkl+b4ho2Dgk!Yh7Uw^?zylD*IF5+B?^39nL=r)mj*;thKxcmiU z_}eb|!{&|+O-gY53J!6HonRr~)?V~mJ_S9Lg5!zinRDYNO|rY1GzFTu3Ao<~{O2(b zH!vU9Lo;)^TPVTMOcCwLM)_!hW1oD~=h`Q;tmuzRuoJYI{NwPfHTVQ1p2z*nf9DRNe%8f4{0002gJb>Q z&Fo)+znvLV&r3p^8Ow3_htepR zr`__YW!?+2R$zbZ5D%a_3D~NyyBIsGq)jf#E(1-g@k&j&)`4-Pc zek6;vl{WJloHoBD*uf+9omuD;qR(5&Q=5@K=t_C`9rCO8asCLIMl|wQXhD0F@}>62 zoeiJ%7=qekfUhccpW@CcR>CVEqaWt7SGPMZbOE;edgd#Nyu-Ju{MZ&+@xN*-v5b8) z!d-dtosTT`hogCk#Lep-UiBNtU%P&I&v5+`t*J@&QTz|o?{V~7^H|e^JI+{}Yxss_ zNN>+5m%L{BT*N-Si3b6W1h~`I|E2s6vB#t8ES-?^T2!4_blZ|JUd>4}TT zJ4N2P-qn~Ln}TyjeKmEpzGH(i=gfoS9J_SUaw{4fv1&pQ<2!S=T8h2E=C5W(S>(4 z@K4RnugPm=9?O^?(dIbj$n&eyUV;}KQiBj_ zSS#h1y@|Y4G3_|>=zL=jW#5f0>jfX9zru&3l-=pI#rmPV^T92_s0H3I^0wY#KeRXW zyV%NaMm7O%L;v10d}iM1muM6`mz{h8@RR?dp3iq-xdK>97t}YG@?WxP)Tfx+8TMU# zyJoBC0ACdP(uMDT&F$-1xNZ9F*pdcQcZ%tEx_tn=bLK#?+S2SPlue*a=P5LfaZAAm z?gcdX_Vry8a=F)Y~t5@PnTXKXu z^!|abLj^jSTe8 z$wX&N;hQ=)B|q_VlvT>t0zE_i0MRni@5pcBcJLQj;Vk)R{J7A~?zzeP`JUh}-^xFl zKB6xv<=q#R((X>B@V6ICO8hXB693nvnfO}j&b3RCnWi8!6(TcDMrN9X%=D*AQXz4HyGxk%>ZIwnBho2c)aE2ob=5Z&+$yrwPzs=d= zRQpHNRoteNvx3pWXxnV&vp{ncWXz?MX`iL6sjbq_J#_woZ$2U!@jmpgovgF_}`{=z!*%lyyQJKTks zOk3-$>fG_wkT0f=^c&$(zT{uVSvS#QKiL{-zBOs-Sn0_4C4OuLFAfcq{CMJ`&>G&a zo2WTpubh7HjeH}w(ns?R|HM!oafVgLA_g?_T!(YJvN`x&+4U02ba$?GtMTVvw~!cF zak;vazPr7Mc$U?XwgTOECf!-@4T_OL{oXsVKHwvWMZ|l#$J*pi+AdqQ*FA^KIgR=o z-)rkAb25LDInUWfjI241cXu5*`&)_luGXP5&xd|=CedOZ<>&hjaB0I%bP2NQE_5Id z&c(>Qo#8wjIIn_t8kp{w7IVCLN&zoW+CWGvUE_i*!}q zYo29Qy$Nm$KPtdy+TUW0u|g|oe*x#*f)8F6{TVc*xaSenofuyXbN{?zcS-(Th#V@O z{X5>%+9~<>!B3pLu=0jj(Mu*`c2l|oK@B*_k#a= zE~ZY7)#Vc(&ykPTKJW^^x*^C}aenR-ORR4m@@S?<69z6#aiP(CV;8(0er(R_i6$c& zgW4niB&qo9HTpg}l>T4Ecjb@LjBlTK-tPo^%A{NPKVCai@fft%Qk@qUj+7RLo3ZH{ z{cKBo=rr-YmVKHqN_91ddf&jUq>I^;Dn6Ak%A3=4XkT=@SNs(Iw+eo`oqkBS&^Yjy zIdF7k3Gp3(^J09VfVFHibF#sCbO`N9Xr0~JkTFK_7`Oj;mK8ldCs=iSvo)6eje3t^59vX&MMk(ksh0h{+965dUB5*p zpT#*~cvU1XB^=qR`Af1lx^}ci=0fmLOsr1~T(z&8+O7}1Ojc{;lETD1kpAP`-;OVO^XphsbE+IOs*9yjFzB-m^Zmu%laAGNFzBDJ7dwJ4# z<~aJ!l-M@R96EaI9_mXjmrV2L*nZxHKOtqKD0_)@cd`|5^l#)o#mNr3--w5Y=(|UG z4)*b1m30?$UuyJ+E!LO73Gg7Z10JN>9RpZ}Wt5d3{?W?sw~`kt$6$wRkC!693f1*^)Xz0`aPoZ8cG>|ar_iSm zc`RTBo$+!JWsguM9wJ=SzK}l_AJJ#eg^yb2JbYZvxHQfO+_9|F_`sWL#{U!Lfp2S- zLi1~s!Y6-7O6*$`@2xg3@wV36yQ9DKK#%E(KGO@mhWOdb`>EYADBC! zo)|#6=Y_A@PuVKYT3kt=q$|sga4F-HeDq)BX)i{5S3x)LX7Zf*OS8whc{h@$wT4Bz z@+5M_>-@eay@s|fp^ol{$&(yIJEPp^a?oU%Z6v8yi#z1Y>U ztH<#T?E5o?H>3HlxgTlLWP5~3lk5zWreRM%cZ{>zyvuJR#-6dMZ`>NUzo&cG{RO-y zIdD~X_T5VSf#w8sUSC$e_WoK*x(8a3e_Kj=23p&?vHteCP_Vb|of`E)GdWgfz5k*| z9zl=z1fStY5_35N7uV&J?2FetQsm2C+nfMSb+z{U6W@M>IRIxf?d^jdp46M~YQI_S z#nWB_xLD(py@osT#5duS|1xx$Vc$V2ydG;{;Otus=Kq_fJl!sK%k{3-eBu2;QLlYCRf5-6~7#9t9<(;^TLfx$mGyfC+8`BqJh~)FF?%ctk zb-k8vcBlQ-{|0TzF6!N%=iR>`{P1EWPI-m+F{~llGk$~h<2incneknIH~{HSA$gjw!+cA0C0{L-%jZFB$8=MkY5&75cgmc#V~Q!uun%-7b7*pc zDNDEi+@Z{gm8JFFtSiv#0hnnF#Of4i6=k((<0+)5IB)93_9R zR%9vNFVToBB_GpRI=F8y>kRnLd5M|n;C>@(&D*Ry&$@Ec0^-%i^G)>)83o)L9(u3Q z*^|yB-(pSj)}02PMJLIm-n#Rlo7aa_G;u9Bk4{?|@nz=!0^E?Xm_VlAppv??{G1H6xKD0ii1 z=gZ%u3flDNA{Sx5@?S)pGuA8r=}E*o&8V+vYnwe0*=a1YQ)_)yenLeo`uWow=oxn&Ujb;iUyDOUS#B_!^l{d zfy<7Jlxdd7s7zDESkK4i!5}bm)$^eLdRv zvO|aLo6Su1J9Owzu=-adSR)^VCYOmG-8jAR(4_R7Une-e3({w#<4i}#Nduk^J*L^B zIpNYp=1YIk?@hMBy$@PCUI`@m;OuN6J|GV)KrvnyAMCe!S0)X%Y!>ra3iqREIa zJ>MxywcBXJE0auk);g4Zap+P6%aY`;&r_?*TsdOf}TO5k|~ zaGef(rvc~7p+n-?=1si~_#2t9%$HSvgngP6`n@yWUth-m`RMHf=}V4p(o*^}+wiQu zBSKqfzil);nmwMQx1J8Afy?rp69lcVDwrLO#N&Kgha#i4uAQ*?)w+B#P| zCD>~P^MkY@8Pe)W%vI@6*G!5|f`1O2JtQ=4o>S*l>S#?1_OL3eAB+oaN_F!0kZ;xk z)>83F=~2z_3GT-&k;&hFSc>e*kI#yR5>Y~$tdm=fYD7hhav z*4lJSY)#ICzkhe;abB`eB zmP*dW_ru_a<|@U$k^W;Fch=1vBg`BQOXEHQ`l!0MF|K#vHM;v!_$q%{;gkB^J?6je zTlDe;m-)OWS>Ps$`U|-*fnJk6dLY zHd8uRP>FrNQugvR=iE_Cd0cof{SXdmUC0Iwn!5?)d26d^U3p`aR~jGA>0vFc2Iq4| zSW8zyt2uer(tAnERw}N}r3d}}{A^jRoaCQ&>ZP~M=n;E{6MY$3F7x!@BSWqiHg0bkC7-O-VuIQWI` zfIi~~?D8Gx;@epcJ?^ESnrGt8IlSUpmsji=61wOHr_N65h$rkyw2W`e9NKUfe2%X9O)BiYwL@TZMHqj^~<$+S8GKLG$6ge@txtG z5cr97-22IsudZ_@R`-o$89Dm9oy5%_@W>24qO`!N6K(RbGec-`clV<<O^kzEo#{;#kM6Pz3sZ zANmeMpU3b~?uu>y81h{7Q{caR5m;&3vn!L!w^+Vm ztP$ocP%&qLe)UIW0q&QOJ*hZ8#q-xO_f&3RJz2(YHvT&!_&r0vUg3VgiE-IW9_M~a z8~(AzmpXlZT;h_UJU<%mUlJicRb3yeYWu?A*ghJs_$;={1mG(EF_b*kb)y@Ic4c3& z+_HhHZ}v*@%jdu9XDbgmQL^JHH!n?j*wz~H>6W}F8$lX=HQc=s9m8I=d|X+_D&_N4 zp%k1bR_aU2*SUf8frnV5GVD6qQEa_ZXwmpduGqREO=aNl5mS~{Uk)9v=Z?*$m8tNG zEc+ezog?VR5oGX2!()g+nEyX8<(YOesYA<@0Rz!;i7Csld&QIqMxL$e9+fjMU1Q4S zSGbyY?;!8wtgd0d&cocwexmOO$POjFz2k^YWM8kBGTNXIm?db z2bT@b3NGVZ-?Ay3=UbFP%71*hmfaV3`LbI=)0UOtyH_(OF8l#>e0+#iRiCGO=#uQM zM}lKBHP&IG#}vCiuoR7sATI*WiZ43nar|}L$($vd9J4ae^bhM z4k%@O`;>wMe>G{cT}kTMYqY+cUVa@ob}hIz8=RX3?#)DxxduJv>f69cqsN>SO$`S| z@$vQL+_zbREK*GRJo!8083+Gs;dd4B*?-3OtiCLXXLN#{oPXQ2J8u6ea83M4GV&2% z`Za97nz!H67b8FRw<>pXes0r|xcy!|$^J)Ji)XoYH&a(~M0GzSgKV*Ga{1M!p`ku= zo%ViB9m$@#)Y;e&xBojyPX5oyukOpd;@5r(cvrlYA9(i2Fl3PwI|W^3+p!A*k(KeY zwwGTRsAaB@N2K&U>$5EqZ)wJbg!I zaAS288<`-UOpN-K=z=`={W$)>X+)J%RCKg>xO=I(R(g8TMPuq0u!dKTq>W zo}p)W>q&4@=Npl0JJQMCOF}Pp$M*rgDm$@gq!0DH|I)R*|I)S0f99qu?d!a?_*A_4 zRyMw9`ev5>sW0xK^NsA)b^k}>dfxWIh+enV59@s!`;YMKzR*oS=%+t)lmtB`!?WSr zg@WC{_n+p=p02ZBxq;xaqut@x z;@kY<`^{c<%ZO>qim}6NV>~)*B6@p@F^V4lN}i*;rP~#5-YX_AKCC?f$wWn@;nLvP zGI$;P9ma1$27uUX1>qk8U%?Eo0qUGpQ3GlGC zK7kLn_r37=)BP^2+uZ-cGWeTpGLc8E7h>%)ZRM=hpJSUz#x^56?}x74=u*D#$a87- zMEFMyb|KA)w?Cdwp6tFA(6xArqnkN0+hE47^}pPAojD&I1&-d{cu(=7Iifx40=q@j zO)zbx+7G*JnKJNTHDwlM_i^Udd>cMh$+NK!v$k7i|NQ&ojr@R(So<(Xc1u@FvhM^3 zr58w+`n3mfD&?8>PT=g&faWg4wivI;ckMQc!F?9L`Q-$oM_&Vf(ER$S zqx6$_y_;r#q?9pzV4jogdeY1PFJwNQkJr5u;uDQAx>+lHLUR$Tn~i!2J5C>~;1oRf znes~mRY#@--i`0ObkQS@YzHrBh8LLl43hH(!xy^n+e3T?D`U#C7m?#mBCDMQ7LCYY zIWgpp@CLi>#cxJ;+=F{U_X@+kl*>}-)j+Iuw8(Hqq%I|?& z+4i^Gygc&M*G6R0Tgj82sIvtN(e;LwJhLIq@VHF-7Po%6_Ce|I*W71Up4;N-ctNeF zAAhx7$Ged62)-wgPvnzyiy5>0ZUhtBQOX!^R0?cvP>QS&B<-l{eX%_E1bnrTbuJR$ zqkcEC+mqmCTYS&@g_7e6^*_^;{dD_-FL>`eMm-@jcCd;n|zLpb%OTP_K~%d!hd$_e(zdU@8iSr=cLo{N2eNuEF za38oS5W4SLhtKxoJ%gLr9CoF^my_c6*Y$PEx~qTKGPF+q&b5VgRi00BUz*(pFOdw_ z=o8=KejCOtzV#2~Bac1Af0NIc#Ip{X`jXR}yKpv<=g8$r_KlP|vY6UQvG-8+GI_$m zR_f1=Ee8*dQohraXW3WAmVqns@7zJ2a5Vm2@jd6Ro+a>6Vi!~uCs=nbKn5s__fOZo zib)s$BJ;V|h6Q%Kg&d51rV=^0q7d1)G;PP@4+`eQU2<)1lG()%FLJ->*xuQvfZhhv zr$@+d1n>3DA#NSLFa2Q(n%3E$MbUPc|wcaYS`Z+x@Bm$Zbl)U%7@vwBnbH=oSK50z&VX%%T zv7U0*8F3{>@~+QHT5J06j_(#zH_iT-y6t7|*yd4I*4ygmUlBhmGK+YI>>C(*F8$GX zjkDUfhWJBg#`+q#bOKs*XeHae0vvmccm>9`ut)QjY-jcJZUSNgJo4fvvzuNWbdia-- zMPEVD=y6H*BlP#N-L8%7BGEOmF5s!2 zT=j&3UxuwUSbJd&z}d5b>%Ye5?SXmEzc%lr+rK2;Ala#~+x`zm^hLh!hYr>s9V`jC zKN)0+Rp&i<)gRpa1;~SC zHY}$$L-pAKjy%Vry_L_PMHn;1AA9 z78wW~2_GkuC*45yr))QGg6g^U_%t_fjPist@X7ve-U#JE$D;RMZr+!b2M@p&Z_XwT z1zbiybDs|Kj=vi+9qZa&$$H>)nNo13Na@p4!r$P&ojTU%BKWuB_e6l3`rM3- zqJ{P&==qY{c8dqFj?}Sst8V7NHy_bB{zV=6lu8DySXs3p)5vvM_Hj4gCp#baS1U%4 z%8E*oHs~8iW7-zqQQJ2dzoAt79rxYG9h+W9zEc_BE(1OZ%!OMgpgHmM!)bYP?qTkO{n9Llk5#JQz-%M3)y5I=`+w@*v;8~r(J3lz;=tAVQu1X(Y&w(@Xxwg9 zNnUMSoEA7SBRxx2&oUwq{*e|3vnMwdLnsD4sdlwHy9|HpO>z z&w_a71MsPut>Q`S^K*|;7j(xJhIg@l1RjcKjxcaUHwEV6nO|2qxZB{C>s`sI;+aM+ z2ZxWxmN9pWD4V1Az`gfk$~3o<6IW9{L*?MyJ2BeC<`_teU{x>Cu`(>B)g2Uk~_j+LB`U@ZA2^$Z_!9 zWRDl4gI2(E*YjJ#?^=HK__oxIu%fl-#5>^2JK(j?y8L$$@paS}%jLiS>+;{Px%okt z|MqhE?_Ba%x%~H9{^z>)YMDHa9pbl_9W+b{Ws{21DKFF5Asp0Ct_ubUWa4^b}P z$|LZa;*#wfQVritvVZF4iw9|c#=qjZ4cY@%`QDW&8x&Ja<60ftKD_DmmFdWY8TLx| z{d4l(bzR!yAY94VHHVEeCWpmi?=$d0e^knNS1M)v-&2ZwwaldK&y$Mx>Rqj6rX9=RgP7K@Zo$duPLYXF(@3;l0;jhje(a{2mgqo6A<=k4s-$%eV%y7FnInUu%y2 z?m74FF=ws{(PbBI?Am^BnN1u^Vw)9+W~`+~?%X%Dgpir z?;YODDHFbdla&>Q_hM6o_twt4EG$2h%kg=s1^3%Ng8#k+|D7MKYTTR=I9Z<=Xe!SN z2wx&?*?|wXo(B&e01wW&P;$Xxe4m7CV`y7)d=v989(*zUuBJgenEeu$2TOhs&T_y# zh6l4Z0_+C^+b*UY`;l9&cWc1EDU>;Kd4|0xVzsT49J0AWD&f?vAZhCi8_%-S@uwKb7`xpKf8QWt_JXz*AOpG}DuHe`< z;)25G#)6AF^BiT*`k~l(vR@SYMf_YoXVN3(M^`S9&xp64PDjOCZckG-?|Fx`{r{D)wjPY*4Y>xOwy8SJ1Q#g7*dD@HAUYF>&*v(&~ zx{H;UW-oN}Rx59z@{(-vc)ed@-tQ5f=*DbGMQ(k|2TxqAbE>2Zku#Ofht`$OBA+|t zvi^QIcE)sjHtopYOnk0-MlgD{`&=%cF6#Q>tA6ekOcuYCerL*)-Lp?g_7wNI#?fup zd3MH)!nJ~gc7L=a`e}F-{kYtWBN&~klzv{SbRKb4l+Fo81EjIM?Tg~P)Y8ua_PNVG zoqI6S+P3YKFB{*`%=~F-)}3|47>TfU%9q9m&l~!}9S8mArLOhvb(9?9_`7D>M_8lR z9CzTVK3H9>$`<&KVxT^WJ^DWGyXwp%CF6NK@(i5@HXOg*s=5MOMdH1ao7PywUibxT zP+WXHHWv93c7#)|KlGUE5Bv)E^axIW;qE7;e1z~x=S$ibm75Tbh@F~qx~za(8;M*G)#=U?(mwx zZTU`HvlBzP*zg8E)+H4Gzsn!En6`3}J5FMYR@@Cg|1Es@_Wjd=*E{r^cngZbIP+KT z9ge4@kFowg+Jiea;Tzbds`Renafpvhsd3)@EANU149;vxR6HB*2_$aZEbd9kB@P?E zACTAJ)_=)`QOhtZ`rL4F?{Z-VFoRRxwtFr9Pdk)HWU}ARQORBp9 z6M>idvKJXic;1{hAdstg0VURH#RsSbW~2VWd-M8+85O^_-FLR>|G7vemVH8bH_P*SMEJV%gOW3 z^d~b1{%Z>ipKk#-Jv)kbzIg$49e(fB5iii)O;do+^@fMDk2U>F42Icm{YK`^e;e;4 z1`YpEd#icp3d$9$u)>$pBsf$Y8Y$l-6PH2vb!@|z;_=<_`vuZv6ugX#;re9w%)B5sxbrd?xPQnvVqxUJi8})NX>OdgeF}BnqfXs$V3s4i z>O=f*;8g11ZZ2^%+V8EQpU(Uw+jBKP^iAvd>+~Wnq%p> zvWfjsb4{Dd^WrmT{!g(^X|24F|I(XPeu8zxJI}VBd@oLef=Q(>RUpHk*#K9>uJ~0{Kk#M16@tAOccf-)8nXgIc4Z&&Z=X)4vxvRkmCCI~U{kti8}@ zfWtxJc1Y)LPO`v{O#4+}Cwa1-e91H>1_1i)9yk9z)kkJ%L4VVJ^dH>(ca_gNjO~m3 z410&0{}0nnm$2@0c^8}#?~-i#Ffx_5H!i*2uV?0PuVmBBlfxsCH>>2QoD_^cMO!)k z45OP@=S{-)nPT=-HY=a~6v?cz$)?~7Egipt_cA1>CIq9uHgz%_Uq|?f%2JFB8;q`Z z-*^3i+GVkz_Qvn%)-H?v7h}*qk$m@rXB*5|gVA3og)gsH%J?2u3SBNImE5X##N$pc zp9y|k1D;$BzRUn`u0kijlC%3=j1AQ2%m3}_%UgzpTCQ|tmmkrNY>6AuJLLboDACDZ zO}^wDor`M4|9Qv3F9){2Hk>;aCWq@7|F$PC3Ka3}aQVf7ouwH~&lYCv2*%|Nuyzd` z5ZCw119H=I2W<3ZKCk&pcYUgFr%fZ9Fn6~?w~wQni-vvf+BIA2Ne}!w;U(fv&b})= zn(+_jTjNy@ykB+84bNm=r%*OhWj^9M#+CucMU;J6Wz6yOF=c|YmStMI*b#txaaYQhQR_L?L3+LSszL7HEUQ9Lh&amsHRU3TB( zq6ByzvAKY!$Meo-{Nj1q5Ax1Ms2|=wkePel$)=w0JCk&U*$0Z>-@LdB^V*g9?Z!MO zzz6*7`^1OjN2k4g=68#ybAGD|KRWDm;w5wN2_3`#e{rXuv-YLgb+jw`>p`l%?LueZ zezJVYRHDfoe`b9RI@w-u)YGxR*L~JQ388`R{^7H_hZbJ{iAj^}L!|oNe}R+YJ=S*( zu5B5ZenDt~i)$ZINA#{UHtKIG=XbATFHm@0LstZC5RG56T=I{lQ&+3zzvi1TgjbLv|)f%6jB zCphosbJ`qa<}}@Yfo~Ff)AZFpaD-x=N8iP7JU{DCvia~AivJa6QivWE$d-dWsg_>Z4B z5MlCS<3NmMoLj*W`3Z;*{Dk*4FKbPjWG^+T&PNo1H&el%%fO>c;SW>7ZG}!;4?plT z?RJVEvF86VeuU{i@gvm74B|&*#97fYW0(2-m@M9U*35dQxDlQ4`@e`5HGE^C0vx<1gkV$vRusX`I>V!EQp_e`}qBSf1JmhGc#w- zoSFA+-t&IH$8UV@`tpr-&d$`BUgnNzKB>O_pW-qYJb1{(gC!S-etwNZTLcFxa69X#02^Okeq0d(k`cp#o1W!!#~gbT0HhURU%Ni*zUnUuI$)cOA# z<_FN9vty>n2j&1Wo00phl$zV29fBhx@7105cdIhC-f8%~C3KTXK5oc`9}i7K&z{bH zR9E(+zF0>5-)sWf=!eeOkWE0s4=!PADVR*~k`Fe*OE@DwQ{(F!Jph_lQ^I*0S;Ql4 zg?9fPdu`&j3?pvKFl@GmP50&06mtd#=Uq+Mtv!7FAnwOQx1Vx9_LjRx-+P@xbU<+RhQvIjk?u%rySn)N?-vsta#^Zm|A)oEA zzUR`4^@BoBPH_IAui6E!nFF(re6q8XpHKb_@=o51P4q3;M9(}gaBQyjh`X~tasHs~ zt}S%39dxXr$Dses4!Re1(2@n8#~w@L=i?jVLkU})li>DVBll$4b*{{Iz~Fm^{Z&%Y z8fRZQ%Ra`KB>X`BLRk!%I@XX9Zbhy?!rEa5^yf_bnk;(^b0E1#XGOn4AG9x!JoaoC z8~?;7x+Ltyvh1B2f8Ln~Pt|!{nm_IFY7W$ggq=~4Q-d_?o@Ybnj?8zhukT-d7R11G;1o^9?M26)7UWQ z+7)hDUu;bhcEm;yUN3|ExFqK6YB+~e5*n!{R^5j)4quG%r(`y zeb$p>`W@Yk{o0%OhW@~Mx&0A#Bw6>C`Mk3{q=x@L?&9?GUkd$vyn|PR_>N?n8Cl37 z+dQEzE>8C+U-HWqj@*!8FJWvP z*yQ{#VEi!n{msN0W1%~?Bfk_H>yEAJXl!f25smHIPR^L`(7WUpl3(&)#_q^6u{$#D zuetN_GHp5QSmMX={|SHJ#C$w!#w6dCET{S4u+`IRS$94bnmT!Qwp&O4D!UlX$0MdJ z*FHbF%$*O-pI|mMsVsRuW|RL)lFvwg=0kH4h3?ieFNdL(nvJCDs)# zyx&J|GUzCnSEKWIlrTtGZ~4w znOoq=thp1}PUG+u#zXXZ3Fp-68Td2w`D5+l5(^+Y){3NSEDWthhGz~$r~8qYpvj+c zue<`)V;)75<8EHA^3aKCeh#~Ny_DDAGH0@D|Eq&mzvm5aaCKtUO3|H^xH>Up8f2tQ zSjLGgrd$}77{N>tz zb>H!6f46@{*uUq2U5@`;j=j?@(_St#d`CjhP%FLIGwH_HNNcYZ?zkPDR2i0D>>Zw^ zkKLsdz0gjj(2h4vnqhA+sq|rA>@Sz_k%K-j_HX5UF;6hk-p88N{M7PXVkgc@#SdVM zcify9X9o;K&a1g%u+ENJ{MgHbL!4)JS8Sjancrj4($qbDLjJ9NLWwgAE*j_18udYQ zXJQJh^Xz;g#mRqx{4L0Aho@%*{P><8EzS(EFB54k?iDyWFDtN}F-;93XRk^RzseZO zb|%))HB5ZAm)mz+5z()!Ts!J#Xv?8Nz3iVrL*;KO8j`RznQ!Q7p4|>Si3S<^;_A`v zGG)1TTXGp=D%vCdFIYO1T*kPI7u;&fdfUxOWk;b)2|e0W%H=l9SFJ&wovv=CmA3Et>^&3k|MUihQ$32(6v zG3BOx*N^ug-;b_B&Lm#CU_P2?Zza43-JQ-c+3Uj&#mKPu76OB!rPAG9#JDQX>?5*6 zS<#iT>}?-m-W*ve!ydtSBmWzEr?R0Mf9SQ*BXIv^(XjJ5j}E-YW~`KR5&}`^S2OWS z-oZDaW1Uy2b7H%re>opL4Ewy(dZL%%jM%yO)A~A@b7EyHcpQ2ahh8OY1(WoA9rWtk z>rclsQY@p9Tp3&^kOAN2THhaAY~qp>5Qn5f_8~*9_@|t?CjT?_J4(MrS5iqGTGQL! z=jNrDye{GCZakyuZakyu#a|H52)ZLtdlNsjD{5$BZ+j!igjl%q!pdoyKZZFuqACaQ$mk*9TalT~912e9Ujc12aj$P`uD8mzM)|+?9J-BrGe*(#Lgn;N>uA-}+18(%)5G12 zjknJ3NQ~h#)RQj3T5068#YGnn3H@WNLksVwJ)O^@c(^mTH{ZFVd#FG6h0L>PbLwq1 zrLS@(i1r^-ZyQ*e_@>t6v4*R{u^M0aaIrUh;ucSs{kOG6f9@WN&*q#A&KfL%Ui+`O zB*fVYiv!_HLdIr^vt`WlpdlffevRMSH8g>l zx4%L9rPvI$_yNV=ObM^R{v*fd{7WfiS~pviz4*c4zzOLYn5XrtE(tWYus>3ngFjr3 z{jO_=IEeZV?asF)^FN#6H87FzKkG@JVm=v~4ea*g|4+HJY7uD%JqJcbvyYlGZ2gkU zfLqb*PsrPVOtbyrlqn~Q(;T{Zi%Vz6K^Kdmi_po__Ib#D;EZq}m-$P~rD$;CzKDC3 z8yi_-+#4D^!9oAPjSuWV*Ngh^}C5R zcNiMohJI=j`^0_|SJI)=Cy~Ft&wrR$Ajmck123Y}%gJ-bBHMnz&0A{neBm;ePM5iK zy3D21{&^0sJv;7Pz@1Og>4E|$?py-De9l^WtySM6RD`{b_UAj=IVV4V>zKbu%;UAp=S1f98tC){=(MAA(>`ttJwOTm zz5CEwuq7{zzY9H9rDF4`O9%tmk3oEhnrbNrm~VbxX5 zG5%%zW1**NPJNE!YsS4fA5L>58FRSDf0vmB8Z9~AREuDjKfy2Edn=b2(;=ATV4XYV$8mva=#Ie5W>j8Hjn z^AndB+(aGC70PahhWOI$N!;rj`sRt2?t#Pm&JRe()4Zw&@f~^+b1NO$G2LF+)%mX` z&ZIRolR0=Fnz6UljmhBp?EZng4w?xJh-R)ev?bfVlC*!*}-nKn`GAtWY^AN)~1>A<%zQYOgw0Z#v*HW&^`1X zl1HRF-RE=qoeG>wrV!ox0sq)R_ZDYD_cHApy{GBk)BJma?m0Hjnf8<9sc#{Z=G#w@ z3a3VY5$zjT5s0m}0-R43NyN|l{n_-dPV}!Uc+w4gIS;()4*o#<=3;jO&gg9J1pU(< zT|zcZ#L}%#@}GU<*QaIE>%>cx-5%;;4!SZZbN~EV~zdFYxznX#?Ix_v*F2>(1G0$cLhP-Bf;>?w#4Ewq6~3 zk$>mRI79dNR!_!3wo2zKWleIFGRGF=dgkpq=5G@7crEie5xRE`aS~m+2b|ZlcfAez z7Nd^=?3tUvt3<3#|Jig8e3GA7*ws^>;yaqB6=%^sL+fZyw9Y>Ux_dCnUf1iAOAsu>0e1;Bs z2hUjuJQ{k(euGQz){5Spbm?8odzIk?y;I)V^zK&Zoq=hW-qrg$=-pZKlNDMCy?ffF zckJsKI`=VZHk|B|5xL# zb;sE<;zIIH)4Se=-hJE9lx*8e+ClGt6VbbSOj(wF0yyg^V|^`p_ch9*vi0)1^v<;b z>@7M7O-ZcRwTJd7^`is3gk{IdI&6n^6>?p+bB=U3{!uhZJW{rge%{wRYJ*ta4qfCu z=;Dq-$%DLyO7yGSQ?6+ovPrbpb9Pld_Jj;VV z<|7|=`9dFT#UPizh~!4a9XO1?$PsKCj=HuD$#iqxT5R=tIcLsnBv!{yxbEYc$~B5B zhie_pTNe!@Uaa1NQMR@0aAMe{-pQ- zs?PJHf{{g@wDftNw3AEdBBPI!fkHe-<5|>K=kfbH}5^=nRS1rJkypP^F~ogd167i&MQ zn~~#C3Bf(q{p+L9=qjbqM!(Wg;KLKa$Y{vytzAXp@QQoNxBd zGVR%ka_LL%#kb;&@;tjHQI1aFRyQu>*WA2k(II?}Uw$_yv=W;z>FWd+8naV#Zv`)I z0Y7dAPreMk+ywubhF88I6+f1>eo9g5d;@+fFRB&2?FA&&|}@n={Y5 zj0+!f^|V*14REWL7>km3tqtXI4}7YqYFKDZrNehds}4A~0{glZv%NdB+8w)<%ajjH z6q5f^WiTSU!@s&Zp23t!Pq23aJQq5Ad|zGw{u_zT4QzpH{-$>i+}vXlx5mA$z{N|_{Y1Z zEYH3pxeSSx$DCD&(;e@}gf{%70QCzmm=`zd>gyeePs zMbhtKtLgk3eC8XB7{h1FMNU_{+*yMnAvt>cGK`odR8n zbM(=+`J-EA;;%T+OWZrohed|J+3+dZ@Oca#wbzMH^#Z@L;8WS~sovmS4)~V~9_E3M z@Tj>LAjf|(pOSp9xVt*j{N~l4j6=k-hW~)EkX)wcpTZyR2REj0_2tUvB0lx{ce&PcE#+E`Tr`37*fjju z;Zeun4SKiCmt8~bp{X`_HQvpeo@#w-k8IW1MohCRCQB;|3gI6jtZ zrUWBv*soJN_sMSE(>1-!)3wG&S!@;OB(W}0S-3S=dMEc`{D1YXUv})!0@|)=K!<*C zME5{Tx_t^Fr zyQasuN4IGB;X>{UWhYe(9^)796Wp-IXqv|Q+@&-5#FtOa{R;4TD{y)X@Om?F z`(^mWP4J6p!1ImpiyQDKclbr3y%uP!Y+;1+%#jll4|_`S;(Ddfv?`_0FTYY~)pqvY z#Ea`ms~!q3%ahKDG1WQ@JKpurGX`p3KB_aBuPC^hN4@G@_w8DT?ptFWg;zgIJ<;S; z{^4`?zP=IVTgC9y*eda1r$6Y-`A)vQ zkUFwOaoTX|ZiCn7J#^o$r%j#ScI{bp#Pd%){GDAtG38nIch4?|=bw1^fn7^Xd8Yl% zv&-T6qr~%>+dC;!AD`u$pZmvaE(qPmKZ-|>WbVy8GOph+u)#UvN*VXNlrj(1N|~3h zDIEdNP|CdBt`xdILn%0aHR*%U#g*yhtQEX1K5)62{k#Ovt_N2opFM|srakaZuKxYY zdS7#h|8(Q7U@2?L9f7|M3#E~ce|fmh?44EKW8ulUgmmVSSk(eFpf9e!R# z8}ggk?Dfvlyj!W)&GB3FCl~shwQCF**8Yj~b$88V&!Ex@K@;w(a`y@R+QZ&+nbxyA z0$WFz_3X25+e6fLofXzNEzJi8ft@PS6;r%B{ihtcp~U?@c982=PlKNBE)K|sRPbo8 z>K8~0vhM0XDPT8T7)WESX|K8{5QVp_Zs;F~0zXDhpx+xBO3siIR%{Fmdj5q0gX~oU z2Q^k^y(BzstjpS@@%xRd$4uqhq9uE~ux^7lKKrDP@#Q>F;tQt$!~a5V$w#jtxUAt` zvV+SD7bkQl?Ad&3J=iC-hj->Mhx*qwE{^|#RO>;tnTVnHD(!4?;aF<~wJCV-s0ZAP zp^eRy{nmx&YtB^Vf|YsJEIj!FQ=!$MgxgruDXJ~8gPw-qqzV?p<`>`5$P2J<#Lt1Ne2A>Cs}Q|XU)ghha7y;Ds3qy z&IEMnU-JFyZ-t*SzJ8piTKE5*%%=EmvL{WPf7FU@;V`;|Bd#AHa@&rCeN*-G=w!2; zm_kdjF}#m!D%Vi1Jgzo$3yOFD5~*a5YV;6sWUvT&1@sQXj!i)xaCHrX$d_z2iu@I>u3;qi(mTjz zd^q>gJID^}QtqX9kp0#gKaOv_D7C!bq$TKuA!RL1~+33W5Xrr8k+HWS4<)Cx9#YqCCdmuz5%?8CrMT@dI#|C zU8A?|Wq+MA#VqnkA3^_O;EA7DIZo`NOnZ8wTzUw~{lun=fhSpRyy0nya_J)^tDRAv zXHQC$OCJGU);SSQ+xd2tTlTD@cQ}*RG!~h-MhHlm#%CA3$bTb8BGa7NUVUL`4}4WJ zjgL7qZ9=0;&HUm2Yvwi>H?TAi{eqqc2jfGOBHIjBnumWVsiSuo@8}(5cT5cR{aS${O%*eLR-A-l`$qp)(Bnh$Qr{- zL(PoabdPsd5ogV9-^p(0R>oT6ARWuKjJb3yhA+1j!W5uF`l$Pi$fXvxb*VUlnIv#zrj9eGeMEiEAP2_6NACeEu}(^6*Uhu6--%PpllH zN8C|#Q=ibXD;>NXKzS{D_>K&R{V(IDeb0X6C-yx*<6e0Mr1AsUz*_VtTqk`w1HR09 zZPdJc_7Hp9Cn?vOt$}r%^vP}R^Im5ScP727?icd7c5i=bwoF`c>6TA%FP-`clV;j+ zlV;evO-hVnU_x{E0nhsXFFGSM5g%A*;fZg)!+$B}sQ;3sOJ(^X7dnMw=);<)S&>Fp&Ri37e9u>$BCa5E<~^s{jSg>>o8NS@-RSVv zoNPC8=1TGl$&(K6=iF~O(Qf3-W!%r;UUKHs+*{l?PGQfezZL1p97#tZ+*#}L-ht$4 zKS21g+ReL2^^6ReYyZ;CD^i}3A$!{^-MoC|8J~e(_A_o?w%Wr#3t7;_y%Ie)K3d3m ze^cyAcu>7k_)V2k_y|0}i8U-gfDw|vHZF?-r#Us6$TsDPwG40J-Z;?xjn5Ln;6Fx>El4Ri)X%_@9(^C(Z}y zY;Xk`YOVjs6_TOW2L5qnsF{D4F1g6#&?SFScgOCEXUS26uW;z~|4}=PCu_DuzcK{q% zY|SU$(buh0M|Z)u80XK)PaAVN>n4x0Zt@33Kfx!U)rvqv*ZQ&Ps+NtX#tQoe>0kF~ zZ26Y%>)d;PH>+of#~L0ZKk$&TZCLAfZ5xCWB_86Y(Wd4k`@=cA;>@$~tHjl_&Ep@9 z9~yJ{0CY<>qv9C}dFc*A%g`ghcSJvj8T`Ztjd>9LyvC(<@4MxCcQ11#`gxTp>uv8! zE@M9LpzKOh*315Lav5{JfHEVO1E+sTF3WcGIhXL>-kw3zN73Z{JR2BK$TPrbtig}& z*^({8VoS0Nv4#&x&sfMW6&k&x;<5g;onyBE1JW-z=c{Mg z7c)-clY-4eD*eJm26h+=(kY_X+mhl1K{w^|^vk-fqix`)-IZTHs;eCx%dkq>`;E}n z8F4Dxw~5~OL`R;%o=+xwKE2SLXQ4ySMwi~3{~|8&++5-Xb^c#%6BI}2NRn;BQRscb z7u>*?dnUq7jrfA22gv2WW}_pxmFqGtj9u2Bz#icpF6joI<~lOTACB-(rQECjhDl=r zHQrv>8uY5E!!Dr!Kk+KrCqVc8vP*#OzlJVA@#0%VpUaq&`>n`O^abeNO$_+Ts1rN9 z*3}10WDi$+EmyetTU>pBY!X^o!%HV{8D-F2L#u~!U+d}wF5$kAd(rAa+;2fAu#WL> zRQ+t|^jP%Ku1v9&JjoQ&f0ny>FR7l<31r!$-Mo#;Gdh7xdxV?!8|4{#o@Za`=B-hl z=y_!@KFH0}ck#myx56vt!Q(}{OeBH>rS}ITxuV~a z?H|fr7K7hph(_}-_W^UQ(5(V1qoox(SgiFtbf5~m0Vn=@zI``!B%@VA=Y^ZqJSX%5 z^80eyC0=~dF8r`F?HO+UZK5yGE3Ms~vON3CZkdS_|L`W~8~pDSH-9hRPp0kc4=uJ* zx|wsFwFd9YcnEG@nTI9@A9tX9{~vlkvsxZ1z%H8NcJu?B5~V%)^J`HpZjGU)wXr zKL*hGOE=TZK0piOJrADI!u?~&vX5;&FZ4LJxYz$b_zpOR=|H^ClP6w~yzJu3r|2bX*UK?IRn@8Z2 z(eA-WJveUQi*>sHz6X9}#u^?JxbM3^8tL&b(q5b9QvP2x+#5MWaPF?{M`~>kUwX*3 z=Xv{p;|DBTw7)>Vwg0a*QG!=R3{B6sN05qFsVrd!@vJG!vM)(41J=Z+mYK3lJCIxk zJc>^d>=78uv->8OfurJ6kCSKq)3Ppmo9EwhX?-o+%u24$|2;?9eDV&HzRM*#(Ji?S zd~rXp{$29)e&?jJgg@SX@bjB0*MG655&zSgQZyp!^AGUD1N~alzvuddOgHQP#bytp zKYI}9g!VST11rVjv+aKXU#*Ok3kMO1J{7>;2#KUM zj`}3`s&|3%#G{Zsj=6cgmB)JX2sGe6HwFVfnCASDsBDun;34iF-oL5qzrW8E$@!Ae z_d&k}w+G38hqw%~L#{M=x%L6_^t{dJ6msl+JV(1YvVGsPXgGU-JFJUKLglVr;nyEI zw#vt*5PyO1f5fw7@%>7H^Sw%er6!YR+J7V!%{KPUthrCk{m=0iqTY;_N_dmjB=|X+ ze=+^Q@^=aCCC=xy%)h(muK}z3sDB^&CynQ;)RR3zGCwkHw3QgW!yCY$_?_T>ulufI zfNc4NGanJ&*VsK|x!wbU_l%b3?gDcvcKoXfmlGE;K@rWnl_w1s{9x@ zn25zvLs>(|nY<27`Myj&18rKN7%)EXnj^?+ir>7KeYSo3q!-Dww*qGl zt;w{@`2UsLZj#KswqpGNLu<~1?>V@VXYx+s6`YHX zG-gAyp^uH&5=n+@|B(I~{LitUyz`W2^aeS0)Xhs( z9yF#E`mswi1D;i{6h2es(h@&us%Qq|8v}-iv!5hb+s(V84?f9G(p zsQMkfz}xP#qo2&NcewAmbYw@;I)ANoazM!A#u{AC|7zTh>f8$8@@@S;eS1qOechoH ze)t!YX4vaV&!i*g_5+I*6MxC|1>3hM7>;sAt`GaN0*|j{8@4A2+97xp?%(O^Sze}n z6JvyU5aI_ezAqNOUs&q!H1$Eax&@jbJpbWg2hV>+el2-zAGk4N7J=i;=_J>d>X(#@ zmPkjr0=(+*b8>Nf5qTQ-R2S!n&DjxKs9g#J$$%iTf+nF5@}cq}lf6q<8RM%fL;nuRYKM zFY5*$>xUH95uI31bY^``EY5A7Q2VxSI}KFM-uRqN860Z2W+9^^{;_0{bSHaCPF^ z?aiz~UNSn~f)UoNYVM6bv*@q6skPsq46nay3;V+AhYfw4UTV#%;@PS58g+hX>a0LU z@msY&%1Qmx(lQ_Mq%?jHzP+p+|EHCVh00P>Us+n_$ux9o`-3}7nc_H9yZ5(o?|<<1 zW!30N%^VJr4;Q{o@FVNGL81AsymQ}vM<3bfyQwYAoCTO`=0molEron5|9>ej;!JtT zbHGvP)ZDSa*44n+RlwR9V6Kw2Y6WZ6a@MNb?^e9P?GK*6tR2~HK01_Y?w504#J&H) z^kvhz--@k8Y$iNob};gpug8EBclTMQ^pAbEXRD^kj=Z>WsGAJ>lM+#f4Fq@ zCi3Kqd*l}7_p;w}^Cy#^kg4Bt^Cs$j{ztOs8*W~e@}4Ab8GG!s^CvgILitZv@xL?o z8>A!W{{;upC(*$ueFEABZ*1C#uLkrYno%Aps46i!6wZ7=C-U26M=J62lOE=lU_665 z;+ID%(ZPsk&ckLvW$S|Ru1RG^{$_l;koO+Gf;zL=;hWf!nR?kRw*&8=gU4Z)bofu~ z+~?}Y>0=l3f5c%j{|@{rWt==F&9~dB+cceX+(^#}zb)-kuY(qT?tA^dWPB5Y=PVm+ zXm2THCr1H`zKYV=G3+S8)xZMa$&X37KTufaKJ+}c3AJiZV+ zJ9FQoBCd0DMXyag?2G5~j^1VeaH{BpMQZALD&pb9d%Y^aLFj^<^Rd8=U~WL6=Q?58PF{o&1vi=>9#|M5?)80Drumv1#v4 zKhn9nc;fqKDh9Fp4cgr|HW(l2?pHje&LLI&{*sW8i@qh;GFHWj6jXIOi zWxI9es19^Sboj^=)}Bv;6MU;g=}d6rDb~TVwfVTr*=zZ*Q%Lhv!oG^xdpTl-o@Ab9 z-`+iR*X^Frc+ZndHJ{ptsjfgr;d#{Tzr1;wBUk;8+wU8dLK_6b*Kr>OuJDkVRo;*I z^~BRH$V&~?^UZTL0)lvzqUP<{8=1t$1%#}8fHHzAEpEaM+6m-n! zwt>gRC2vfK@P3byH`4LLXrNd9^k56z>*ZZ4EPUaX0*1dtRr? zpYRe8$FR2W6(T!d^ol`51*>3;g(hTVfML z*T^T?=%aua;(KKZ@6}d8Ik9(&f1JC!uQhA??snqmbe>gDTD_ZnV99KUoS0$$J7p8@ zuNg3;o|tdetbRGcIaNIta-UVR?;r5X+?SuyAj``x&DP; z)CT>7UPbBCKd?nj&|T@xnx|OdBEj{iLqb{VbZQa!;MZs^4X)*h{nPEP;o z_hbxc9B++iU>sWcPlq{{K)sa2ujGor|YQ;BzmiyuR z!0=-0&zFUmhs8y=jtGf{9|lg5OCmXU=&$g_M48Fk zbH3SgHvcu8^HBK*XyoF+tDT+y`fBIUYkG%onD3{I41KW2jdc_s>BJ6MIWjbWcgBBM z8Zxo#eq3tk?Oo8(#C(~ty1q0NAg>udHJ!QeGZ)dil(6hAN_gMD@N%cU(#t!>S+PD6 zbXK+)rryVwhq{vQyfeyq=Y~LoOrpL3kiv7tGX?`90ae)1j zH@RO0JzjTBxVirY;TU5ie7zG`8PEwl>QtZm@$z8&n6t$XrsiKV9_ykW&}-b@pfmJaaQAO$P6AGv7*BK0nAIlkk&{;HUhj9)``#(ggLq;N zqBV?PujPjJ=!_d;fh~pZ-^Ae+9fN;6I{^f|0E~>vN;fX>X6^ zxhtUceY#+;0xvQ$>xH{=b*`9bxT(*3QD1$oUvoCWV`9H(P4o)!nzZKC+!VOF)bB7y z+AG})PJI*EaxZjO`)|Ikj*ZnVy(^2)(a-hnTUA@54% zWuMNwgS=Amj*$M1>m@G5s?wPLOk>LXt^Bk0QN>feg6;0*@qLVs+SPXgysz>o?TTlZ zI_anD=(~l~d51K@^$gejT$1x`95ugkD%jBJo|Vq?P7&CdvjV=Fo#cl);=tH(cf~bKI<~#IJ6}y`2C=FxRiI} zZ(8x~it#WDyNJr#E1IU)UJ>7hp0>C%zA2r|yiGIoQR~25`yKRS!s#mD=;@Q-GCaE! z7)b+0q649^*!U^+dxE8|+`ZU(X;f$rd1b&&6FT}*Xiw?pQK3fh8rIpppy9H~TLW!1 zc5vXZ%{^Y4eUQ~p#fry-V+Xw9rUp;==x)Oo7Z>dq6*4k`PjtSQ{p}`e$v>gzGb$X~ zvicK;w!BWhSsMbso4Maie0SM!NFO~8+b+e7)42OR7Isw2a~oMV9P3gMc@H~-R^Aa0 zSm%lYj|dT@|Iu`8ku>Rahb>&g~Y@fCYdUl!5+;#UqG@+ticLP785`I2$}5w zavEz%({5s2AbIRVc=6n|*eGCgDt>`{I=L#{8j-Ajl|rzI*`Ry+od7N9_6mSb|_HkGL zMgO6l=qBv$lG-se9T`h}zXUmJ0CJY0U&vVjo#<`Omn@6IWC5O89N|kts}sP=zN-?ndOmsWZ~}+ z=P7Mubsy2p2iOM|jwvp(=d1s`s&ytotE7el_X?0 z$L4!~09(>+{?OhAKYs6`u{rjSneQ3E;j|KKnDd`kghabPdDGFYE<`RdXUkj|3~L>7 zC-gyiC40Pa$tMN>@Uo6Ak5tiyU@epW>v=o8rHp?1>1Pu>We;QO3DoqFedJ!&vFI4? z^>X>RWRsJE%?$@2Ez$Y5d#S7cm(KGct$A6?8vg$;m;W;l2JgY+cex+YS9ZPZ&t{a4-F>c+XDglEZu0Z*qChOUh&YqB;qFvr&15-#GhHqWjM(&+r>3?+)^Q zMxJQ@KvIYH^Brja0?K}(GH8Fld)}*88MHquxlFX5vian_ zOZ-6LxbQm}w=MKO9k|)v<+2EP-q+ClOhfY@g_ra-G~aptKF^21Yr&~v>WU^QzSSA< z0ZerRJ_Nt5z~H&yz4~YE!^Y5#WStd%Yd89q8Kg#b?NkwOe6oGRpIED$%|n_h;UAyl z9h}QzXs6bPnYL`L1dCtMd`mt;9@KNHJI^!7cWkgS?Wty-;k^@x3&J>^En`S#M=m@q z1I%>!dzaJvy&q#yLOspfZ=u~C{G53kLs^l^n41@p%b1%xD9csZI&}ESWluS^#F<@UCG%)vj~gDn>s&lK z!(Q}HcrkoA2+SM&|JcQU@;l^z-2)pLtiSfahuIHaTruoQL&F}0UKrX7|CTIx9GL1& zD!FIxA}1cz9n8&I7Z1uvw;*F!OI_F>!Q4+{T# zNU7kH@mJfwrcWB%7nA~zYm`ENRx1VW!b%&0@l{Hv1mnL@3T!qgg~qH@3jJA5dJ=jf zz3`Ib?b+2@Yr6UZ|Ez)N!L*NH%}U8B|J@etX`qX8=ymTGz%q zk$Pm-!;}6G`(qybXF0cw^Iz_w-uRV+LlcGs!xKv@0+Vurk?XFi2v51bGCaO07@y~{ zT53Jz$5(hN=4|m)&S~0=hp|Q0j})HuPP?xQ@eZ@tU&&^lr8oO6Ikbnp>|A1c zZR^Y#n@*fA`98?*8M$Zi>HWLhHhAe;>-qCS$NzyX1AAD&QE4@MSB)20*Awev$n9=F zM=(x|%Z}8QcbCSHA+;wjy;JJSi%U%#!!klKwSjKZJpVZ(^dZkPu%}%iyX2)hpER%KWbeZy!tR zx~LnmEPq&H?f8plK*&eGnBzlRSff61RdCY_?d>DlJsCB(LyOu^b(&Q|dR^(7Wj*HH z+cpHbvd5gkZRVTTr^Npa{%AZKiBlYAT$1;?whlmNH}@9C@@B^L%Z%+!jPW$~ux<=9 z=EfdNeOlvr=tn<9C)AxjEv8TBdk2Ih4?Ttr6yN1mZxn*8RwGr zrLSe5bz*y)*wp1OEc1MKR$EaDdfx92>FC?e^8d2_{Qt84*8j3Tvgm)+|4#p9{nY=m zez#!c(BXD7p3fd`H{)4N`bFdU40F8||NEjPg`uY6VEh7Lu{Hx62I*xgJt0q_&QR}d zXDT0<&0}t%k8fei_m-di)o1~J=CtNb0GO%dxp1xw?!wrhknAh3R2i>VBb61AO1Eqk8n+W|1ouEuoei? z@21b%hu6E`alS!J0>1GG--y$GnJ>3x2K%Pk*L>LhM)W*uc;SEe#)IxR3f*tqd&W0% z%s1}l8~W!0_78%@RgrA)Nw+@(j}SODalRkRj8%N}j+zg0+(We8IV=rAYUg zy6@DYa}fN!b>F$=Hr?mzzDtWo_l3GoZCMGt`}^rWt)*P|$cfOa_5Kv>+~B8vKX!7Y zQJ;5!8Ta|aLj&%)Dn1^*Jm8*A@$^fuRp&nK`A%W>%ba{Cerdto;S)Q_~yfo4I@p+Mv8eciz>eF<5lxfVM|}dUc#V!$tK!?HsTF z8T%;gMb`hUb39p&15YOl*3EyNL;kgL#vD(`Rj$0b!+*WIuNHs?hqRyJubJTP$&CSK zs(U<|o7Ty?-jB~)dM!4srwV5Ifhq0XHH}!Ft9Fim{YB&ybcoZu$Spo(KqoHWQS8AJ z_FOtYDiQM^10Za>;D6Y5tz#^uQ~0$>d)Y6Ln*M`_jnJ7HMo!ELeUp4-ybb1gMsDbC zp0@x`=6T5lp=zGt4eKvs3}h>>^@V)rjm}vO`x82vVDW;?~uWvRAi5ZC)$Vm!M`d} zQ}3^Np>pbNIo|H{b$x#53ZCC$p1x?@l4IOz{WjK}Sx_HKJO6oqmo4P7*%Qab%? zd#2h(Hh9l##lOP+Wpx$gXW6=6LH=*apT_<0ZI$HrvUOieKDzJt_1u@x&UWdReHjD% z-NC8(6>;d=djHl+aMwE2#G3Gn#zB1Ii^@*k`#Z+V!?<}FKh7GN1WdT`fMnCBxaTL{ zr|-x@S|3E{LSHi zj82;VZ};|Eb52a)B%71rr7i(t| z{KQ}LyOFGy&*<~H>Tad4l8c1%vVmxq{+)fM&HTqb(A&*v)*Vlez(`ESpi?TnD^C}TH00LJoHU{`G7r=x%YPO=*xvk zed(L%3$!}9FFoA0_R!Y*ynk~+VFrEqBcwnd!C>%npw1;kGfKv6W2xOc!T; z?!fLXlGmdk7Zup!df2do`E!aF4s%yI+>9gD9j-&jrJ*S=Ugv zpv0P0$UGNZf_)lrt$gIVsf_nxE5FFBhYLm`*Q#z1TQ=2w#<7Ppb;dY#%>Cg)Q{UWQ z;^Y@Hm)0G`2jZJ1-=9?fZKr;$_~m;JPeKv!df z0AmnTUy&;||E)u=Sd2w$=tVuFk2B@S6&ra@k}K8*&XOyTo20AQ;V;SyCCe9*Cz9oh zG@dyR`5vp}&OYhrixwg)NRCLq&QrA3t63l{NLSuib`twKTLu48j<4$PJO1$i|5(DmM)!G2J@hkH9KlZA10C=}4}8#t6vl-Zc5|^+ualkn z<;G53Jf^X@qLlM{Bdh0L7uZyWJ{w!%Xr(9eyV(;1>knKNUblK&;Kd2VNn2MLe&NN6 z@UOR(hr_pUrkvso1k64zHk~NlRS}=oM{)I*)#m{l$`L9F1g`@ zpJatUeX&>gld;4l-pn2-yznUJyBzlQNs_5LV>9oZt#sU%YQOaKXHUhDfwYJIA$uqv zCCNgcL5EsQ8@;gqCuZ#CY2iZ~Z{iy_@U0uehu;jw<1;6RPjXJ%fl1|&X6TV@Dd97d z>$^C%@IKZr$I*YkLyWDJT#s;lo9oM5Be~KkQ~h=;wPq#pV5_}(#=gDEZwfs?xPPm_bZ_rjqZIp_x^`>nL2ymIols{?&ot~ z|IlkDKS-Nq4p~2I&1Y!Ftwo`7cmHZpM73zIzA6gn6@N*#!<%`ou_EgabPCJfd|g8-cu4+6@{8%)>!j#O zqU%V}Sw**zW|Oj))tEv0CsK4&t8Ve0fvaKWQ|sETy);MIt#+Ttcw8I)XfApb_GJIl zoc;dK={1r z_sL-V=ntT)@KgG+{%x+`a$%n#-P?NUpM2sk{=Alb;Quk5vrGR!V$V>(kHGtM3=u z>e7(kxb^fuP4xc&&sx)cXwpo3pGk!SJaCy@s-*jKSfS)bNoB z*n&>N&V2&cSguO0GOkfvBe;fg4fe#F>e50U^p)dyw|$@|4sJyX*kkqMe~^Oz!Ex3v zQTT(g9oIOi9J@WyPQ$11Kfrcly}8H#K=yma?m_jiXWXiH-1jBFnR{&bBqN%4JnV}n z-p3C?>(RrkNACyTk9})UDC%XMjocgU16}EKny!46{%X#o@2rGg7p7PnqLuX1*d{N@ zU0w{o0DofOPnoCltP<}1$h=KIxGDV0wDzY~AZMxVSYP@nn?K*e+}NYKKhp+gS`X|1 zp>^Al**9_-y7t^fp{w|QQ_xy}KY7?QO-`3x9cu~yEGrZv4PeLL!hg?mZ9GjLw*22l zwp;Q@_t2y4yN&lOSt=Vy6H}?L#U7?(%l~)mjAh$dqi?K+UL}pWzSaH&agDI=-NSce zyRUs*Q^&QZP1J!0?kK{Z){HauvIdo_O!G=oqkF`Yx;>jZ8L3;$DTG}C%n<^ z)79z|_$dCSeGJ($Y{&27#3yNC;a16T@bOLLBw{GU2HzBJo*j(eAMh`0S$%CdFbH~d zze9sYk*9M!N{~6|%i2urhQAFQyoHTWAwCA}4W82I%+!$lBBOPwp;!Y~G4#HYbc*g% z!cky1woUiGaC9Fj_F>4e;pnQH@E`FwcJKG%N77|tcrUQA54;wy`4E`+2)=UwKHU5j zAi_eqeBUJUXo+60L(K z(T8XmeWcy!Y%6kf1oyNbo!vFSIK&l)Pduu%!5?T|b)(LOiikf&2l~Rz>xgevd{ek( zpf}w7g7Q4!X68aTf_-Tq>cjUH*-gA>(HoAPkH);cJuN&58AD?(+qwNS(?Un4Obm-> z$yP-EtoxP)X!+mXb=Lm-E7Mq?x^?cyPi0;({-9fD2X&fmvG8R{ssm5n_e3y0 z+pY5_>d5y?bKf@2ivRUO=K2=sCh1(RrlF~!XFhbk`I6deu;Rgf)Pe6uTRf50Pp~a! zEZ83~JUn^~Irs$k8oPE<=svn)Y>=@-?&6I^)4eYXHWY)4JyaqVjKj~{8;NC5CK$^m z1%`yb2N_4=uoxa6#m}gkKBi)eJf3uXuSata;(v6I@ji$z(m}>|xq~&> zbAqpuWC-?R;=;?@M6c*?!ZulPR@rZh+2KyiV{tGE|5)=oG?Mmf6ZZ)_Wwn*C zQ+~=~e+-#Ek74y&pRn$?~`e4elMn;%A>%0jCnP6 zCY-J#yPsm}h)?|*ez=csN{6xcNyqLdUuD3&om>WgTflq0%zHU@Lvk5>?>WlOQ`vi* z(Ue>^#jyeFLfKI0Hu~AfZ;>zC7>nE=1luZn@J+rg8{RDRKNVe0KlkMMUG!DbY3&7P z9|5Nmb<-)^3l1mebtdn~>+4SD)oZA-Nr`QW!u<4bpcBXwhG_aNN zmp!ZSZ8vr~y8oqXH+$-xgx)#RKE%DoYwt|_ANW6KZn8RT?sk(Gg|;N-#>=@2(Xoj! zcYM|VNF9yq)Q<6?J;qh@u~}t|-NfWF#_kTvHmPhEak`SrCOPw=`OAO@9%fvJF|J1# z*TxRpt1L6FTNwK#jH7I?_CoWVF>}fyl!@o~hoJMp56O{}=ohrj2aUR(awFe?+p*#c z!dkZxJ1>6h11oOqvJNk_(ze9tTY3!jv*5BJV2v3DGk1LP4SfmtuS*VnM`+-v6;(JPgnL4G+ z%Y3Ea!#t(n&_hb0i*uENBM&H@29F~>GG$!&IAg!)6nml2m_5Qt#wfAInfR(BXCo)9 z_2Ul&uU#9!AE=qS(_J|L`C#o5{DSJS`>6Liu@(P{?W*K~uj^aL1K34|7Q+9-0xOB7_bjo28;H?w_s1p+ zAM@wx{y?l??6lBz{HhGQt;*{H?dVQEsXQ<6FL>L>Gl|_l7(P+AU~~(-teJJzKH|6T zt(!l353%_0+YGFNPMw-Nj8K)*ZDJnY;GOT7*h5*yM_=FDPF_9n zjSKK+p9!zh_ara9LVk28vN8D%UiP+MCQotd`#S5F9f>unrMc;YE{X*egkef2?)RZ1bB}nsZ|N($@XdQJWgKQ*);<=aZTH z>w$smfQ3oG$+hS_Cf);#7+BdyJwG%g{A93nbNk5f(fg6p@#%UOJ+uBV483}8n`fu= zD2}|-%N`$daQ^4&7j+8Rv(P%pE8fg=bf6XI{bBW-x9wy7_uRhI?>}2Nt?hF6{U7rF z3SdaF88+12#FmNir+#ZQd4j!c+} zI`s%R^N!b=tU25dpZ-wg)1@QnmBw1E^ymoW+@YS*)>eO@L^0>K`Pxp5vUuOSQ$E(R zxoN7i)+i30;!(XwUy;FpBkeO_yUCpIV?Np{X`eLCyh{&wV4$P3=xaGNM|NXc7pR`< zsGZia&@KK&cBQ?nUTJFcD*l1}#Gc)$FPbXbkOVv==xpLy?Wo=4XSFN(I8^gMKZcMh zw#a|hyQUev>jB2A&Bgy#@c$_Iuk$(-Ukt`#gXi&$8T3CLJ1g35H8_=F=W^c$9tk$H zxjzA}svnu$3r>2Ho}8k*e7igMH#0ZFp-;FM9-mLDeb~2+9wF0?x%p|N+Bdt=)4r2u*(Kd*p0n()@~m~Sd@I|aAw#{^hEG$hOxdq5 zVSl-|#`#qH295br(stT1hfpS5qWNY8lx8EbuSu9Lv_2g{&ox8V0v%z4Ul%S(F$ z=c=1cesl`!v!&3!rPc=g6^8HPo8uN*!%i|L#~7c+`^!tC(}1_xq$8}zi+SS%jp(%H zhXWl&PX^qjTk&DEx7$iFGI3ds)~xLDoL|<*3KVjNL1CZ9W8a^=eeEy5LJXFme^)0=ht;S_7Qq2;r%h2_3!nJ*lK&s(Fv6#B+4 z_OHRe!`;m~b*(j|JkDBoanW;qL%sStyzn{d2-i;VA1koeG4ciLCE<+xDFl0lHb6`F zvF6#&zkax^GTfj4*oyqFxjqIgXs+sMYd`#NXjf~vkMT)-*YgkZG-pZe>YHkNZ%4b? z=-x9~OXe9n2%EN7Ko_)b@$6yUm-McAkN!Hb46^OR@A<-d*RfZ~vzNL3(m$zh>eG|N z$p5F$z{zB_=`SB*e542Xe>geqhiksYKF|2HA;C?H);xxtN%&jiJ?Z1g8<18MEN#hS z?c+&L(=&S6Qr0m&{qI=Um$6>5Qpa3=xc~B8eGC8Ih`+=d@g3UVa(Q`N-(Jz*T7L=h zSV66IsOSP~s`4*3JS62%L%ucjaGCRu{_8qL z_w%I8DksH$d2RL8ti$Wj0TuwKli(51g>dng0>?)LBMmi!1Ntw~;r7*5Wb?MHO&b4~ z4?46u(}kbgY2!R_Q)k)=7V6wMQgQxO&wH6y(bs z)c2~Hdwu8VTfi9fRWfTU_o6|ENZa74qEY|iUjH_GjJ00<4vawu0lr1Oe%n73*k|tR z;rEB}-93!oZYpQWL|5g8W4qCzGDgkYI7bAZ+=a9edoMqHq~U_FpM8O7L!a<>;ZxSo zvGFa?r~)gKGX<6`MYcJV@xA3s>%AF^(4+0lN>z;79Qy%xJa*6@!M2z8UgLi{e1p;* zztjzh@!+5KL3=eGE8M(krPg|*bLj4j$5QfNWzO}ld!b42u1E=Sj-!lO^;bCu;$iPH z^lv+oeGBw$cs{b0*G?^~oxG0n+1?=`|HQyD^tPYQtQr!s_Fzlv=^TZO^P$95t#QV2A7%Q^55QBUeHpWFV|!4A z{iwfasAwSezo{+SxAX5AL##7@%M9*S*LL6O$vb*JpL7cF$QZpPUM*j`z0CVGd;kS^ z<0um?!d7MSRCuT1OQLJoQ9{=iyL4@b=-O9kYfC4ypS{DHaY<;~PGkw_R3ZBmW-Q@l zhm+c|o3c{bk>Rb>ZULit_mD{9D0E z-=T%@Up;RXy*=13H0eTzFaDGMDDJkA8~ERmd}k2t>U(Nq$s?(uHpayIL3x{b7oHk) z-+hyJ4P3JJT-X=;5p+R{djKwf8o01H^cv~-UlfPNk!N&Q=&dZ~VLSQ$QqD^ue~{bO zOKv}YhaacQr+XgeKYh>G!pxwL#eFym;0$A3VI>d2IrOmnCnTOmjSli&N+j$KZ8B&>FhbkPOAkvg>Ede zy8o8~jdj_Z?m}-=HMVEM_wqkA&}@enac+Rii*6)OYp18t*ChU>AAJeB`G!Y9 zZ^n=}iahm0u&a0W(l_y`J184Q*--vP|M5N2#CvLMko#U0`>o0Esm+Th(|N6W?+W%g zkkfWpfBjM@g?`JHMgEXOKWZO_{IGb-mF4j*AGWVAa$)zj@yHJ3T_Cu%QV#7KXH9*3 z{AnE9KH>BDvA}ItagrrJ6tI49=o(|fv;L(AeL4*NI>Nfjp;1oR0?H(_i(LT%j-M=Mr$gDbC; zxis$m9sl?A@pku}*LX+Y-%dJ=Z%Z!ugUQRbw<+bnx01F3tCC~>AKuQE^R*~?mk_VIkJ_ane+FPKsHHwxZwJH<|V6SLw%@(}0U4e(%rhy~7EDZSVKKet*ns_MYcivu3R|Yu3#C zn(uFO?tP4G_%~m|xjJv}jPY;ktTk)wMZ?0`}Ww=#@Yu{RB2QMgh zo&W z0)KQ*r^YA-5351vg}!*ZwFZ4L{Od>?{>43e>KkpAP2kP6`yb4sNzkvb?o42RqOiR+ zx66lqI-@fnfB>EBDZhq^Kio4J}cbCE%mv?PwVwh zeZG`1zx_2K+K+zrv;E-ShSq*$M*7hX;lps+0e#O}AstfEIZ#(FQZXgmB13EM|`1E+wOyou4=A31i;13Q|LJCC15=1j+~rs<;I4U##JzZ-}) z9nt7QuS00_1*MJf)di)HTvk95A$&)#CAHe+Dc5CL>rmU&o(*2d@X+4Yd z_Kn`ejrHr)&@aWhVb4RUYj%z1Ew_Hu)1P{lC_a<;tcDmgBKUC1n&89j)+DyEw}hOj zd0VjZwA|Vb7<>Yr!}F75SJ}~b=L(UUclH6(`Sf{{;R!jBZ+bei*WPm13O7^U(@%|d zehocw-w)tE0M?uug9gGxyIbiSWeNuu5$@7t_l^PK;8PwwGPp{*uMzezVX;f+MdM2> zJO9uPC#`4lJJ*b;xl;GSuAt6`!N+huyaApPzHH&rCdMQVPR=|&cgNgddUM|amB$yb zZ=w4!)sD&LF74VYL?8BeGV($2*TtPI(>JyCd!}!hk%v8g_mJm{RxIG#@J7kLo!z~w zNv7@a2=e-}?=6JwrB2;9lbrrR!p66zcjx6Zgk4YAaOA|_@a?A0jQR7(^=fZh#oxaT=o1t zsh*mDq5P#u<@cm~zisN1;QhX56EqN5AJ_UATBbf-$iL{yZsu~%sw@=N%?ZkT748@4q-lB6HLxU*Y0g?YbJNNf4bxo>NoJEEvoAi@Jc$vgwAI` z^G}&`8r>r&jNdEb$cLUDlP~!}{&@60CZ09FeE8@A`N(1)$-jZMs)^5u#EqYYJz)Aj zwFeX)zkAZ0*{jOry94`7@^f*2CfxALx=Zb+C{uEoYn^rni%w{SBBzF z@SUj8-b^RwEBg6v=;_&)nVrQMxpro6MsF!=Z$@t^Yi~xn+2pq$Dz!HA z-a^X$0qb*UC9;3j2ISE{!S`Q6hyK#Q(&&ABf3cIbegrNT=fL~dBipRch#lBV+X+?qTk?W#6SPX)mYF`m$wD@oda;Xt&^hbo}TB z;okM8Z>7$)^1;=y^3@ml(2r}3pB1?n-`mq^ExVC5o|Uwx6C{nw*4#KXdjhyJ9-OHH zcWwfQZe&k~`{w#x&;9Z3%$^Qu->JW?bXdbwcTXqYFGFv-4Uztu^`*C;(><~>H9Br~c_Zo$isRANo$iZ@v7v!{%Kz){=VCWG|JGrcoSnS zJi8m3AsI(FGM(?H{^N=Yk>@zOm!6m#Y|y?=+}qb_c{f~?*qK_ScE;G(N%W%Kj72OI zXsBQC#TM=9yu_Xfdko+l`-Gf?zJ>kJcfEZb*4OW_AO4Py8yahc`Tr_M~JMn>o8%KU(Vz z_8|CfsNmfs=*Y93p{3A?cX2P{mg4Tmt?yOpl;9s;Vnv@vj@+UBUv$KeKu11r-{LOj znD)tTq5sVnr8dN;ThVFiv!MxDk*VNs<`T979FvaeOJ3M4g@I3^ zF%!J7dsN}4H=&TIhBv=>}09~>zZjTtYT!DqhKdo)J#HpW&p3h2wqWLrXU~}Sm6-mOI+vTif3f8<`Sj-t3I`4s8$T=3&G>r9=AYA# zF!;G)H1wmwr5_uS1GOiteX%E?Be%EG5bX&|9_t+m)znn|W|upMd#VlIp0M_UMN^KF zCmce5zN9qTkM}xG?+Gh^PwuSq=0AD!p&z0FFM0Gsv_o^Ik$JZ=qrBl6_G?vNy_e@C zFwwnSD>%26Ok-014qrq!(*iG9kr{k@`+II32T8Ae#!}LY#(W7H@6(ts^6lL@UeyHs z8xu%NmrlD0I&%~{6KDT8Ex+cSH>^z`$174=n!t(pbaYYArWUDhap+Dg!Tmxz0}XM; zCq|p~?z7~6&FN&C<3O8q@1oh40!RFPwkAk`j(xky5fc?ORe(ZBI z&ti|e@0FG@Pq*@}1#7tN?!BzxanghK#O`1BsgEILB?$Asq-zL0Rv)6^AR!rKwR4SMHb!(3NT9kjQR#=vmS9mDi?IQv;M%D|5w z|Bd-XpX(Rgtg|?6?%65J=WgMOE0L?mFou{r2=S4 zZ)tp%a3+8;-UJ=0_rl*$_#(m?^T~ui?S*eq_+x~F1JJbfPkQ0MQg|)lbJ^Db5C7W> ze^KEN1)L+d61TR@DJAYPFK(UUW)ru(%sB#niUuz9;u;jUI^e97Pk1+YQ{#(4r*T!8 zb8=?D$tV3BFYQxGvw(410$zr!4dUYouTM1&b`@Sf8ujh9oja5^B>eX3JE@JlQN1DI zwd>pZrXAqhJ8kO6ek0-62|QzqI1?fNG3JW=TIP!UhiDUUimfJI{sQ9h8ySmI_8KP8 zhuHKo)(ct}^lqNY+|gJl?mETYR_0_87n@e*EPWu%nu51T!99(O(u^XF!d5Y#H6{uh z>4k+9#&{@fIAOfI9p!8z?{84v4FMwym~{y7SDFE;k9xF5;Z9>WUa|o&SA3D;$Ca6I z&!()ce2^b%wf{QyJ8VsvqbKQiBYnQujAQReu^HDgr%?XvGUr11!03GWwPj9U`M@w= zJ}^B`KI7V3{>(Bbh=2S*d35~F)WP}ZpaE86V#0q>zxra%is%dwbs?{>H#kUNKA{}N z$H&uO<UO>zzx9-#9g}J%EW5gxC*zNxpxs^{c-(pdAMG<9=J?gXIus@6&J!K=3m)( z0(T5|7E`&?WqyM;LxWl*uxMtiw+#cLc+y!ip{Z%CdKLFhn@?6Z>e{LBGkzwij*V*}V~6csIlh8&!l)!w$IgwXM) z+W*D-3ee}0dfp3;jfS^kTgw@0C-29n-=m6GFmDRpSelmq_&aG`Hhs()tfofhQ({~& zTFJbQO$_oLFaC7+74%%+t$Av&Rj^jvtvs7(`@%EQ$0);`+mQYOS>s*y4Aln`rw4+F{y!D|to) zqmq@|q)%?Q`bwI`NqKaKLxQ{t&mvs5=ZA7#KK%jhYa9s=rSAP+Sfj#5!1tz?N8`7a zvlo|UWTbh}F`ehAl-xCfw&KUPbhFi#CV0&x*3*6Ii`Eml)HxHl%>07RJhO(Ne4QJ? zrn|v~O?&%x)#JhPL489%m3Hg%ZDBZxI3I@J1cs_>?40oUJ2HN^^jX#?&6ft8W^k_g z`4rCWrbPFTqc4+!&PSyCXbbRQn^s&3x;}_~>jW zaF9G_=oI>(Dd4O>7X$iMpK0?-XaIC-!z64@%lJ0%#l|$w`0LID#uFP2<|Jp-W0M$z zuGSEfhP$X1vR9}&rB6?gE^_()HzM=Rv^uGtDWo(tVk366}8P)V7O}4UDR=&pZWTM2> z_u6PDMmy?HB-S@l$IngMvLesno4G+>{WgY4vm3d{lr_wS$xldkG>7|gjwMQ7L2swH z-2W_fen36Y#CO*F>9L(_>=kf+MEW@CC+lo)MtO80_opjQ)|67`zsaL_$PA2|fwB7H z;kCcU-sO+m+oMf+-=jP~z3C5lwAI6rA5orx<11mioOHsQLX96dGB?!sP#N|Q8lM2Z zXiOgCw=$J)#xIZYE90BB*M^?h_o|KIo1nKojtW2b12c_jeVScN`J6#E`K)m1@rB_P zRgyLHx$8H)5If9Z<#AKS&Ac7yk0WJ}e}XAvak`yD8D^|Uh&Q;l2QhRICf}DldnIU$ zc&(Yo_$Kdt8Fo5(^I7W;r+DkG<x}7)anv(yPO)A6y(~M-T7|b8 zj*GUX@g_(0+v8pmOqzi8m_Si_)41{|_TcGF&?N4`o)OP{IM>i-=DYc(or)Lknt$+h z68LJq!Q)BbvA*Xn=yJ$}O;tIS=KGGfZ+K}Eb?qlUPQRp|+MoGo?nlt>jf}Ux_wNZr zrv*xf%_VLZ_q~|5zkSEB{owa<>e&x1t;J@2?1Gjx(}%hAVV=He z>p1w$T+LnnkuL_RA1mQ8f}`N7w@#Y+2^P>!+4q67&B$GM0Jku%GH4AJ9ybxDJ)sz9 zoK=Q+nDv*^<11)e>;(6x$o9vRL$$^< zH>Waod>`cv6T#cyHn5jp#2Krx(4w(ZEPGs0X~SH`E6zO+8aMfpk#z<=kFnV-T8ZBD z2R&(HkAUdZI=y99DLsMOKeU^@HL1O<-=juGul3{}{!O~=-Rxhhp9|n^YdOpYwp-@+dDHOc*Yu7_Qo+%M9a ze&HKlKw02Yon>ER^k(DWJ2CJVTGAW?cY}86@suBy9zP5&Q~q}A;-2;wympxXS9;pT zN#*BQ8Vq{e`{HESs7muX#vJ>N*F0JUto?Q^ z#BR#OKbUK8CLS4SgZWSCW&bKUeV*HoLij&v4q6=s*&B!l_uTXgd)YtdpMJXab?I%d z@$+kKL7xpy;hXYb>TUmo_{3qx1o=H~(r|~{MDC^tgo`Vg&ptjF{>0c%^5{(yW3Tb_ z=}i-GMm|a(lWGkK2YRKWr(t^Xf3; zG!lA=e%rL|iaz!Ov@IDHv;4jYc4vXb-NYxu;+yh<$oP5k6fjf}qC`r~Vx6*Fz@Usf8u%e=R-Y<=OXiIU8+9xM1E~u8)yE|<)g@chG$WRa9exh;+^9E zDiis?u4v^|c9D4Iixz7rWI4&*FJi+>+YAm7ule3e&%*ikAClWjTevIBUEj0*jin%` zFn+ARX2sMF4;GS7VvFGK)Gt{?ZArtK`BV_Hd(j5Tfuf-!sJl&_FXh|olIx_t1nZn% za%X1@xvKY^>}5vZZ~^uAqfW_5$iS`q+W)3r@j}z@U02ydsZ)KPN1gZi{Uuy;x3$0D zDzNS3x~X?zD5n-10qogwYh$Y{qvHx+M;n$|PPOR6(Xr9k3CoQ6?fe%Vy2u{1`G%Il zj-}BYbj8C1*d2rhMT*k?vUFr1xA|#kEb{Dj-&W1cqb_~z&FyNcuEmZqF}Gub&YCC7 zw6QHi?buZ6hKF#sk`uePEV}=B?r#Gxk)@-8drxo`UOK~!xo?QWhoG{C)f?A@OaeZ>C_JX&z-`M1$d0q0RnzwbQX?&giX-hYn; zdltv0d30{?`E~|zt+cuN0y~v|X#4swZ~4TS_lf3!v#X-BUCz54KO!#HTxRB#bYx-H zd4=ektU$|)G5XIqO*;wgliaNJyLg(?9fQsv`%WQmQi`t|UDkH1ek$h;FU?0v!x~C&uhwUyx6cHo@{ywo@%1j& zF2amlg^W~y{G|PR^_?@U&gJyY@Hy^7VZ7J1n*pw$rQHSWj4R*UUb{`&dy>*7wZFTU z_P1X9OSW*jj%#=I zICVMbU-LODrZJX0BmQS-Ci!Kr_EvAsKyYSF@;W#yeW`g@7~CGR5SkW98-*QJPGlx! z)qq?2-c8zS*2cPnG@d9G|4ok{E_H-YDtpQRPmcazv|HxuinFS+ITQ3f=B8xadhVdx zyCBh}5WUeH&mQ}tq@G=4ZCcyKvjn6Uy&D>&HiSu&Cq=dUR1gF zv8v1HpKK@!C+02#p3TyMS!rdn#M`a38)r#3Yo(2yCD@6-30?)=0uy?63%LCHqtQ-% z*TDK}*7j+Gkh8i5O7xvRfP0jn*IT;;Ch$(mx;q2cICpjltk?Hl?XGb;df(I1u5q62 z99Vxnb~10SEU*tio8G$a(%J)M=#m1Rn>Td|Oi)HLo1qRp;h>!rs~}xr_LY zd zkp3NP^NgLZD=9{tQeOvdN`F0ZgXaSc_KtD8Z z6P&Y--9???M$b^Vd4ax3vyt#ZeS<4c629e5zM)e}U#)NEzP`iyW*#U#clb^a|4=)E zfgZJc;Um&L9k~_yhK|;ujy=q!@$b5GRj}gT*GMeH8M504wuslnw_Iu$@@>kY56R`I z4CSxz@~{M|hQ=P<#_Rju9dnfT8{?7KKxxAri zDrs&bz5+Kom9T*g46&)ZE0=~FK>@tFQc|;8{coW zcf>~$FVnt_yvL+>thhr>wgSK34E;i8Ms8i_+X~e0Yu)Ewr@8qP(g;q&+2i)}{rW8V z3P-ojX_IdWX(aDzUa#(o-UAr~IcVLn;f3~4!X%H>znnVw@5siov5jphGdxW?X>2gf z_X^fwPw<9Ca{CH(28X>&=xXb_DL&A0;?vom=%3P{{cWvX;@m%~HN@fm+&v2K5pRm? z{!wVI_*MKOv**9Ap!WznejDkzd-Wi6+mw&)r?SY%T#5Z$SG;FHAJsK9)w@1V3Z{*vfVU>LU zD>SU{pIsU@pK&)dtm2X+I#T>kX;?Qe|J~$wY1qX{`FShLr-A37VL>ndSIHl%xF{)a zp~-uEthN3xxD~jw(y$Z)!et@K>d-XB1 zhZx@O*6EN|va!$KTj+z%xZe6FcW$gzT@@FYdexVwTkAeYA1KS^@jWABy)vF6PjdV1 z)-uk~e!mRbKg=uR`{Z%wSe|K1O=}t54GfmvWv#yt_a)ph_`Akjyk28G6F(6Q9h!0= zG2u(XU1*`f+uxvzp`6A=md>qhm%TW9IO~emjI^7O=_Wz1oA@7l(@0zU*70Ly=m^jc zlD25GWjEtb`%lZRB(4eER{Gn~X`pBD^KxHzz}9^&CQo+2Za%>M@6j~TO=sef zLz)w%(m57W=k~GD0B;59u9rua#@1TSn6NeEX6f|Q&OexYO1BqX612r9RIl2nbVC{6 zX2Pn28O7osS~r=xZ%T3R5C22Rl&kwC&?3*g!}2C&I<1e|_57 z`>FHM%_+QXYt5)df0Gxm?k>caj_@P&0}a%HZqH5s7cc$1cCG1Y-`!!-t9|VK3=zLd zV-Mc+C9S<}`iD+}cGeaB3f@LKUzfIWo;9I}^hLiK?Tnd+o|`lH;9XtYu`vd3|N0Du!Z+= zC$oyzkKGwUhl%~!HqtF4Yp&v3oo}x@& z=2%Iclh2e@D%;8&vq&SlTie;CzYpVohJK!Zkv+wG+wx)3e?|s`=bpLGd_UzF{fT%x za*OV^lnsaMHKbG75kdZ(WtDu2Tq#%>xgawbHL?V6LZ2q9J1Rd_8U~~j9aP|3SF0Kndaj6}fZaL|UcLuJb#y!)u1$o#2wBSSF75A@A;XZ<0#5W=* z9VU(5EgY4KyhNDJCpp>hwFTVy?WNgG8r|1*-mb)wn)#Fs--yF6&`CCD4tRJeuy*R# zSS9GNxQo08=UFB+C@wsYIDcISt@hV-%i-Oh8SlwO#wPticinE}-cG=<^kAZm+*^I{ zv8w%yv+jlXyfq7V6s_wSV$Cw8Xjv+{q}{CP_OM>t%Q~%*IidF&KUiJnJfnT~ORN$f zw-;&;2OM99Od;IP0k=QX)|W1}=YgAFqOICjuWrx2&KYZznaBkGoOu7&)bUK)ek{DU zuIfBwYW1Ua4Wzc-@3ob+;a=8+$!)!#w$>D$ZfiAX0oB%Td2OvcTU(D@WZ?RRB)Gmr zduq5F$?uo+A+h6+RsA<`m7eE_;g4C7!?y}f+-CsqJeDCG=n;8*hf8NZqzs)qHToXb zwB@X}|;NdTP8he^eUg@(Iotb9T(ge9Tp&>)HCPg!Uy3F4U6F z`bFC|Z@s#%>LFn0r-_fiF2at`o)ze* zjb8b}_O=i29N=App8BhM6WjoO`+1Fd)~VQ9#$YFBBNy;q1oMf#4%s?yFSr(7!90zH zEJM>TqRc1Wg6{Wp>Hb&$;K}oZOIA=nzQgyJK9TE1=dlmmkidTd+*r|_xeLA)l2`OM zb{pmH2pRj)*t^jAfz0az0oGug(IY;7EC8M1KWRQEPLN)6dIxu?Lla_&(668etn+$I z>p5Uau5W)cTyOjAfKMZX#&VG<;WIgNdM@LxNGGv{au=Y3W3EXqD-VTh_x6K_fp^kr zOIIQpu4;}eQ;wrQvQ4ePm(AQ&_|g-Ze4)?g8|CG@l60EC=c?lZ>KNvwz1*v#*sJ4$ zQCF7oo7Pddqny-mK{NDqAI+o4gtu&?6H@xzNk(Uv74Y{vtR;WHf>=SiHgd>wt( zmNL#NLOa2CoxOTMynA54F?;OzI^$^K@b6Vz!n5(g*SSX%hyN|b9q{7tbr#aZ;oq*f zJzgBX*@sbm(2cJWC!A4z(gDO)VWX=!{9B1rxb&m3edyQwwdb9aGRI;4t$m%?9{RyL zo_$*M82V-%{}|t_*|A4&*o|%f9%$}f=2(0Y;oz2e3uOuY=Dln)-zM>v3FGPBJDVH% zJGK>~DLS|Q+OKFgaa-`S2M5G+qUf{6A=hTi4R)V4b$?<)B+bYjb<&Y6AYDFo63nqW zUq=$VxmE6{laA!eq!HfV*%6#TCXFFa{i}1p;61pmG0glPePq12o863C7hom)90R;m z{sfhe{=S+$r2kd<@Bo#+b);K<3w%)Jv%j4s*gW9D=DUIo^}z>>J~lkye_3C)TVFM6 zZsFnkUVS%ejXjk)xs0_WZ+xf?Mi8K+++PU|Pdu|M`sGET22PI#g?>6VjmdJS>s!s(w9 z*9NCo>FpkHdKK^XfYV1;@n#SB+_EarioY>%`WUoWXDJO0HnKxzWN!~}HxpYT@OdA2 z+7q79j&bnu&(j0cS&v>~7a%KK@A4^wtN-%~{I|jIsa9OA4AoV=0Nxn7{5*OuBXhEj zxfgpH)+g-Y7qfR$EO?qdA?(Y(4gM;9F)$yCT}CB)!ea`&yYR5P81t~c+rA^kx-D07 zVNsxa@krJ-*yzqWq5ak*88=p74S5g88Js$u!I`Q(YSt;%eP1x=4rTZG41F+R%sqWu zs}k!clP;E8SM?zMke+#1t_`1G+7Q_idb^VFaQ!t6&0B`p)qKZzw=m84gY9(1@y2_6%P_l8-&g2+ zkoneIzT&5Of!t}O=fQ`NX+!tVdTDMj)V%6$moKHw31LrwP8z?7an+u76JyKH{qcn} zu)T#xUBEl0OV&vP-btEYb!d2}B7A^zR{gdwK=Dx+f-k0WWf{xINTda~l z-eL_p$hX$T`RG0jufPUmEB!Rz?5CnP;k~;mymw@o!uypu|Luk6WGg%;@`S=W zmpOI#N8vNN<4WK1Ywavd(~C_h4Llip6Jj0 z6r5pjI-@(|PKU*P(5>Qb!Nq1G%jCMe*64Jw0eO#cl3qvujF)#FDG=RJ$vG|*9zF*09KJNPem z?V51rfM`-PGDaA?2-(loN-hSbz^R0D5eLU^zRWHHhB4kZ%HaL?3~b--PFolCa6$_y%8|(sy^h!JB1#mmrHRe)=lj%mPnew47y+qnCaa8DuJNvfRRZu1C#Y z(P_Dlc7;!2!uZTY%d{4Lo;k_6()YU{ zQ?iZ^Tj65xdEMgOSJ@AP&qlV;JjQnHVenu7u@LJC)(%>m)OY4w;bF_+yslZt7<)PK zapOYbf13@8M2t_}SmkhoQHc(&yRh=l6N;`plD`F46hf zdd|-pS>}Sl_Qka4QFQeSIY-regY<40+y21Vt~ol$ZXORU;J^0g*s@ASU*|7?jZfl z^Wz5(u{(DA{?dcYNrfGpd3E&Ay;nzfO+{{7SMij=)s!LjC0)M1ls4S`hpyjW`j6RH zM-R*;&7$(?Nb2j-ZTT$&aBp^9e#@=}S4Ve0L_FUGU6$V>`F;xJl=Iy@Fx(L1{=kn0 zw!f<}wcTC2GEy4;F*4L}a5QvZ^oI4BsqcqF*n8k!*nWYAZqd)*Z+Q(GruAa|!9>Zf ze%ZP|+upM$%j_q7$QU+)1EM$Rh74XTWDR8gfw}At&HA;MHQ}xd>H#-Odl@~#y6~aV z4!T0gg4lH@$%3TO{c(QY%{kCN)`$yvgH$qMrRVGZZ0kSrJ6SIc=oGdtJWo6&PF}qW zIkl%X!IX>5NVumt)4^G~LH5y?SiAB*m7xWsje@)588?63yNPiIJ?)ig?s>-#NpHfC zCG=f-kt<77UtCx9Th{+R9n$*Wq&qmk-mLXMmWsL)qq;9`{x@m&k>|^u!)u|X&{5ez zO8%aSykzJSG)uJ7r)62Zhh+K*ELlfKxwjg)VKb>TXTnu+qVL$X4q+@dy_<0JN}q$g zU-@pr9Scvtv-QtO|6Pga|IG3K2IGG?pFIlbe_Zq*8CrLRt8VRoX+7&?1_`^aA>KKN zEgud&q2Bq5RdliAQ*q#6Y(W>>@I-Lg$Q=Iq8JyK#)CZXp8a`y*y^lGQOWVc24ri9b z+cP6k#^o@4U9$J{`tIz8n{Rk~<22rtB;3hRcuvH8!`~ah|J8(-X#Z>)`n_)W&FIAW z9$8?`nBUo&kcO*0)#y@$BTbYk z9z-2knrE6T)w1Kzc?Q9o|JdA6k@k9}yV!$n>>zpbur0xMf5JiP z?{eYR+#3_AgD<%**J? zL}y(7W9Z6C=!)7@4NW;pds^s|!@u5?4PRlc4~#B-DcH5NA@syG4egN~^=6XxK_{Tk ztYKe-HOZlsyw|%8m>E6UU2`7STR}~fHFh5M59oJF10&az2IfCEt$?)brD`8oIwOm- zEhb&ZwAyCEuxr>>`8}&ed#;;Fmsk`u`@v?;9k$%}YF@k4ULcz^_`_E5LF(B>Ufy1+ zTE$$lZc3DB-WOt%q%pmRG1Yy$Uu4~FEmqX?c_kqkjcm%k{f&)Vc-L-*kg(!4|9s%4eXg!M%o+W%-%d>GA#g}v2I*9voCw8;KG#oC9Qk(d6c7i;oB}9Os+HM zGP@6Tie_=abV&kPJ0TgkHWb|jZQBiP+XHRm4Fo%`Jr4T)A?5FbmT5lw^CZ-Pwhm&R z)K)|#cbooPiQLMX;v;Cc->31krLm=LpJJESRSlp|8<7#htC6$o+sfIxOX3J^IVL&_ zZqygKII9kFAP;^6c$lQ0!xaz%%w-daX?1jsIpg53N;y}Zs&yk){>x1L_ z0_a}QzwFU{i{R(TREuJE01xcL^XhlZ+O&UL4^aXjw$QkfZ$u*n7H_18+!D;7R^#$2zRVRDzDi=oTg9EM6JUIET>h2xs zsQRcQjL&9AiK&C~{JJh@@8Bfym*ES3G5GelH1rs+CI-EY&duzdY&^|p?qm$LC#ktA zn~darm9G=mW_~~K<-`99apzjIOeC%i?^@;YF8mvb^XWqT06YbKL#)5MUll->q@}mD_&Mf`kztDN>;FUZk3*A&(!?uV-50g`TiGx+uz#a^49FSs)^{L zz~wgkQ@LmM`Nh+8GLyJ-!Lb8zZQz&$t9D7SO4g|z)gBb^akK{oeEdBq!RD~1k9h{( z`T@N4LwM^xcxxMd%xrX2y{yGw13s}-WV&qu>^Y$uZP({@TA%B_m}8oM9^KfQs^SfmkNSK-%v zV(7QY+mrvAx6K%|=G{7^uIdwXNq+gA&sP5Mw&lk*Feg)?85y`tTo2qh+zqGiA34H0 z*PkB?X|L`vea7og8vRMoU+qcyaX%(b&IQ;)6PUEULBij*HtupZ+^U4ep z=kG~w)fu_vhheV{Y$g=uCZK=EVl$-^4xUM>RfDhcpGx2?(^5Xu{|QXt6dQN~F&YW}<{=8Xf{!|WElr@sOEvS%D`*KgrXrBZkw^RvF3w+TsS&d$NtGlt%A zp3mo1Rxe<14|`IZNT+{aS0I_Fn^%4p{0`JJcG$G@&aA7Nx=OEb>#DrUDrraBjG=d) z_t&l1X2IiShv(O~75og}`~|~GE<2YUYUH9Et*`Y`tU8)huubZ%Jgy0k8|J&!nx!)&VQPG9}}lC3(05j z5!jnJ{AS{c)JD%9WFh)k-NpNXm#2(4A5Ry0YmmPa=i{5fv3}qcwEJj3msT6Pq_r6| z=~%0r=e;uuef|JCZEy)ZyB%6x1}>R)QLkwi{%h2s{;18ywxojf0Bb9ApTa6@7(CU{ zy2tR4)zl^4vA}AepDx~x)cTKiO;sQDdU|H+c1ZDp-sU zMQKkJx-QeMW8Qo-bC`Me6lpcLljmYJ?={FCW(jdq{5B`ezhlfl9#=kX2eUe8>|m1Z zUp_#dF?KQZr_~nbGxJgJiseT3O#&afyW=X}mPwzgxXWL7e%}Mpj|-`ULPC0)D9PnS93%?22N=_ zTgQ+8PNeC<9q(t8r9VXvnk-9C{Pfc$8k^r!UMI>^S!e2DQvQ-yB08mXuad5_myY*I zOIUAlm&_OJu!U|iF!IlqRT?!qz;B1``1VgtKG_~GV$bb)@^&Pz&dSq=!O*8F#jPQ{ z130msaOMl=!jSoDL(#-zA@(x#A29k}bSYUebT_gwkKKxY0DmI>o>26YeW7Tq|AYqL z&M<~PCzc?7D)FJ9d!K14@;7TL^Y z){DO3{}h=*^it#E(@UTBEhny(_Wf^Y60~nGw8*7N?wa>qnUlh;Gz;2Apzo7t3Z`?6OyYdPJ^`=&SMx{r4o-)(G5 z-wHOgEP$Q@L)n&^@or;VnjKmEytVk#-(VvTj_6FmwpU#nm1e@#56KNX`DUK1mwoBc z=>cauu#|nN=&Hg-CCLv@k+yVxTlpb|to>c~etbFa3DV@xYn!INXI<4y(un>Iul+3l z7XIBs{F!{+pNq--U1>{>oR&TBr@j?)+SXT{h5S!>M&>vR9{)}HN2l3m1XOgjnYrgzo;x@AXkG$1pjg^GE$B1i#yYNzz5C0%>8XuJ@ zocH;f!EJb#Vlm=>uuW2K174wI*9qQ1N*(^{mxpgCwbQ@+lAXq;XC_Jjg!hw z&12swkGV6MJt+2LPQBXaXV>rPv-bLD`mDKrBH`=LSo0`P_Tsx zlh^grNz(>*fW40&)x@>I9dN?OjVZ+WxRIzp{t|t-4c9-w`=_4FwMb_~vLkyQ^JpCK zGrH#m$SS-6HUv7S@<-~9q$C6cQgz8G+fVlI)mq7-hg3@O3Zo+m@M(&Ab2W$8n6+))tE}(t@j>c-0^iL|3#KE z_t)P6U-s{>x9)BnKn>vq1LU0?efwoT=IxXt2slO*!YTQA#dDN!}m+*&q!#y8RKK8 z`B-I!HAFIw=Co}3nY-o<^Vqad)bMki71+poLOLse4R>B3b6VDbMVu4pQS9H%83)I8 zUMO~zd;YrLoUCPuZJawm9tZ}~INy2N#{Yw9713*FxVDH_(|75)uEdwk)=>PvqT@fd zBNRRIZYX;6J>-#{A?9I}B(%WOPB$!=KTpL@7B$$mRt}V=+ z+i2|GtYPl>{_Aahly&I01xx6;-_BN87P;*##7Q^{XrrSyxWL^>YX3X5V;}hT6X?R3xUlVW+W(t#wV(R{_EvEJ*5|Z;E$#1p7I>}w zocij|SzjLYb^qM@ZaL4Lr_0Zf-`PhU^2YCx-`O*`gSWpvhqu*P`Bxd<_N||{^0vB} zeXkO4TPNLVR=}0t|NiI162pUl{d;_$$=i;$mEX@e8?w6AM{@zBH zx1Au|=WPf07H>P^@wQeTm1fO&u())V^f2b$!0{EjyU^UrSjhS18o~!z~obS0F z{0<`dIn^CSFrvF_v>7DpZk|AliG@cenYw~IV`JQy6|PSahhE26K`{_@Eg=FH(W zn~;y#E8=aGD7>en1ek2%|4G_Wqj2oAVuV-f+%fn5!C#}?hZ<#1xocXQ71jE8Fynv> zwN85ZGTMi{CB3|XjRkD#x7A^vdOFQe(pcyalhZ8RTE`pa?9V=n+|Ip`;5YVQ@H|g% z_%1w3yrqmZXWOf8`4e_OwA18a-&yqGcjVD{0EgE0<*0qG4BgtkYTD2QGRytQL`3c@FB+lrxz}Gf9(6`WmvW^k&(VE8aNOYh^E)RJ!`3Uf8 zoaFH&`VPE)*#jLucLyPVzN35I3)N2iIpCSjo3sF1$vg`CD(#TWC!Mp}5pSkGQ>Uvd zFl|_k%(TyHp#QG^wbxndeTey%?~b+Vtj8`$`>ubau3yj=V^ayN7HU0%KZ<(W@-LUa z?!i`MFSa6vo-ocX-(qiq^0(z#i=Uz_jZt^b3249A;F>qCmr|z2NasCM)%P>kgwny@ zMqjih1Wv*?QxEH!z0fMN&sbaL^nl(!NIhoEXltc6&OIn!W9*05dvbPH!ZZDGJhM*Z z!LzSJ88gW@)F=67d^coi4RfVr1M$9orEJ)p z=M@)DK5r{D#@{z7l)R6<#9t4buCFikZTh%veKB-7f;H=5Gq+Ft%v$_?^7v&-KjW8` zT>k5%8AIDDaiek9;I77v!d2kPaiT?~IP4*_uEJf3y8<@?Hyk$%r@KKU+lodG!j<3# z;&eCV0G!Sc>3*0m;4Z=G?b3^I#ke9|A+A48Ywinh=i{Wi=!+}B<>N#*&%aMDu5t59d9R7|c~8Rs zSo0Rbw=xGI-;jRC8To!mK8rJk*3c4r9{8lT^1t1B?sB_~|EC^zY1TIK`EB>>e3rbX zO>ekudgY9=SNZW#H-7yY@ym!;+rni7#J8ix;U83`|6tAC$UI}y2b8{4yXMiVjAt6| z#-4brCErNhSl$+{_Az`MEk=XPT)Jfk#i)Nre)bk<~G&D=Hb0iT+^vEfw6vQ0m- z2HO8gz2L{8M&znfvp3i4o!_K-d9%$ekNV%C{*NOae)Dmp=);eVJz3h>%Qo|y_gIRb z{B%iP3O35fa(>x~E!NvdZU{7-njLI7wZO6kH`=n!Ix^56PFti~33F%eR%DArDsRp8 zvpBP8MRafOo5aUc(Q|Tt$;d!fGyHr)5i(7BiuHDeRnu)!Afq{dSm~^=#a)=J<1Nn!fJ|HrxF zxQBn=S>jeG-`7HR<5}Vwk@>6@-R!>*XYe6u%=~^q|4Muqnf8T;*nc4XCfaWPw-2%3 z@bf3(li%-imh#Q5pC)~&Uq|CG z`>C_!PipJIaGwe%ud8)vzBhmTu`zYrbcval=|$YZV{^xKAUti{0O6fAV-#nWtGhmu zn~pv{JSQ+dqo$^6(y6=|;Q_U|+}$@Vcrsz;W1I58o6o>^a;|-k4+qVWCx_cJKEKTd z|0=n2DX}H^_Q~n(c*D6pZ#k#(rgNBikjA`AXWj*wcd3!1yv2P4Teue9|CYRWlJ~h! zaQ^o=?{)j*dvkLAjPFhS*9eAYT*gx77|O1sj?vV04ev!?&0Gtbx#q+2FWmqC5^nib zff?1IuxpT=)~%*$N#BNLvCGScEkv$}afT`{kkecMkBM=HBrnjj+1Si+hAJ=6 zyLpQAq`CU;)f~|GdHT+6UI}lC_0@M~bD6%eb=Lc~yTF|o=PT-&BQef#K^xY`uoW)s z!J8I@CxVu9IWTP`F45j{F4Z^jiFC`knD4|R?C0hN?l=4B^XFaBIDbBF0q#NE_i+Dp zMdB*#N9Lw9B=Rh0Z@&O{!|J}bo{`L2 zcYE|3)e!qB_C|S>4-AZ)c6o4^Y$sPOrwr(@>K4AX4CZY+=yh+aXRYKGrFrpA^^ZEK z_ifcl-Lr_}uJv~uh10$jj91I>Qs*J^_9Cz38oeE*_LX9L(Sp5q4RI&Y?Y)P~CCyU& zLY-yc&g3|_orS&i6XjYXW=DQWJwKXX&Ua4aMZOomTETZ#WW87a2K?UxPqPLh&CmJA z(8qb5FwIe=eU7j#{D*1Be)wS?cPGdDJ-M{0-?F96{gy6Gn_bvajLW#Mu!TKAdv++Z z_DS3=q3*TJyF>GscWMhZ-0y^H=2ab}ti#}mWFSARnK0Gm|BIjK{~^=P%*gkQpB-5s zANb8P|J=(cpK+gqzhOst!%Od#nX?r>EHZ(`WMDCyczC@t8CcxM_a<;-GO)OZ?-hJ6 z!D&uzeLqoBP5OoJbM}_BmGVg^J-+0MovbG+y*pKOrp3^$$s=0tf#^$lYWEd)y6v23 z{&ONX%cq^=@&7;>iHgA6Et3M!GW0MvgrJit$Vma{WB?lJ$td5w!tP4?QeQ@aUd+wK zZrPPTx2OI5&SB_12B#s57Om`S7h=DUd_r5(nxEF$425s*YtEr&gxFgw%2-w!*mUJs z(mevM*92>>)OZ!PvoL*@EHH|;RM4h!=qoZzMWB_go-6J%u-JcvJq7(iU&`D0thM;7 z;IHg~(|&&Ouqj&eEY7#3H~A%Tz;3g2cPMX&VD@6|T%e3eVX&va^x`HRKveywW zoOu#oI@`0Rd;dy%6md%T3h6fS#(~lG@vnNouc|dF2if8*`L`GJw(Ji%OL3OCbp>A@ zY40TN3G_*^!+~E^-~7bK^$%=a7CtpD$B(|zq2I3>LQsp^B~ zh)xwyzu7yaJ@X0Av$C55oa0q{zrlZUz3}~m;hRTcCt@`K1N56Adpc>(Hr}faCMJvl zFN@Hh<^wmaF-@M!QtYpi$3kBHXJcsKJ<+RcuVp`z1P2e-w_BS6cA0k%;cb-rWKx=> z@!DSSXwV)-y!v$+ekOgHe9I`OlJB{hj7-rCRt4(a^pIidZ>JhP_&GIKv)_161eAKe6HI@}vqi{ptC}S;g`mZ8gDQ%dT2s+pDt+4An zU$9cWe`b&LkJRnsM(bES%DeU6TubrtjPUYw$5#0m=lWFM6@-cIoU1L566efpS0wo4 zVn==)UOHSlOZMW5$*Xmd;JH*ZIm6;^x?Et-+nlD(?vVoHXGZdjpB3q2{Om|Cd*o;MKD?jh zTR12E&|OPi*smo_`^pw{&(Nf!@WXc2&r5kjSNy}!qg~v=n-h?J)s>6DnVHP-Aatud z+zz<(j_g8qxc-;4?Np*fIP}Zg${N(Z#gzXn;~U2AE{?268PRY`X|ytgALdMn{4T6} ziw4SyGOoh z{Orh`#?Ok}Vf@TUwefS&>s$+68V7wE3!NGRy{cs0F&h3Da@Y8j3knp?*Cs)uoJ9$Y4rozao8yGiz&%Gw(^d=3wZ{4w3#r7w&thaYuC>b;}l0*0=zIR?2a5DMcdi~W- zHhulm_3fNj$6jsHXf5yvWf_?|Xbn?7&H1fMuXf(}q7Jt;j;5=_B#({1r{$=+J&@BG%4QT@__Z;{|EdVIU0WwWitj1 z^T=QD?H=HN_ec%l+6Q>Y_&p-q@ulDN>l6Pxvp&jjOd0jv?QfBOMX)rgyR|Erq2`^jAjHTJqzn?X(FwM08T;++LrG0%A??u2fzMkT(IX+vjfv5CStOb(v ze|6S#IrblDZ$7k&wL_hSzUmGBS8z{@`A6sWn&1J?HUH@BHuHa`zFF-vx>?%uIXbs> z;r~OO+sWy_1n19At0m_iMb34QbB`eB9!F1h7(Ll1;GerL4@S1q58=gcjh`L)jq!U% zzPAjV-9>15C6GVPVJw7k@kP2OPG*IY=FnVvjIHR`s7a1 z+{*WGTyLvK?Mc=th1v^1pAf!oXl;)&-V~Zw?&R>!(w`rgvqQXdwW&WR^0e2E6~^xo zdCK!&1W$Wa59V$69!4IMoc1wcqJKXm&04;5O}z9*-6M)`GVwW)I^tLJ-P^>w@|@y# zd-3y49X%s8UR@9157yjB!pl}%pU}ZBK;G_4{0P!vmjjK=j?6H5vm(=tpBcH!_}$M+ zr%q1)t=XqYEJ$I!(t&a82o3ARTomz3eQckp?wkIRuTN82-_yD#T!DoEqPZ4 zW`w!(SGvzJy{(%@BI7RL&JOX)_~X|&Mab$})9lH#8b0WSPJs88hXVhetAPIvg|*qd zX{9!grj0lAzG@?Rf~>(L&)&?~aXx0f_#5kxy7I;n`&;lMrN2t)fmy5EAFjDl;bsj& z{vQ~4W=95)F5J0$twlZAfp$)@fkk$t5Wgqk*%i9CEj!Ym|3PQeKS=d=OR9hJAp1tI z{_~XnbL#)5SASR1S=5nT(ca16TXm)31A}!vSM{;8XMYDg%8sO}yff<0R{fol>Teum zYrU!TpRVRU*w3l|dawRNq_e0ad-mX#X1x})uUm{yg1vho3*}R{|Wi?;Rmg- zpFG&^>eaoI^rEeKC#?qPkAa_oJ9#hj>fJ#atD=m42b@3ft$Meq9?CZQ*99qS8W)6c zL0n+X5p>DNLP5I)`OaCuS|g+8O6`|QuI)X0P)jLm9t&Q6%g9paUg`)#2MP)2t=2;i za{qElPU-PY@D1iY@=SJfK4*|-&AO*0m-${ADs>vkTZxRmP(Ec<;@>)?)cG1^^|E@^ zo|r`)#MhimtpDj7a&yUNF+(Mk8JE>)ET&5q_Ju;rSJIJ>X_|;pR zJ`RU1$>i+E=GYcGJ7B=v_4K98-Q}tS`tHx=W#<3%++Je-ySwq=+#>Uz85u@BF~&L* zxkvd2>7Ox`93=c3pnt|b+x&NrT*Ci_3S({A-AmJ#f91_LerDu6{QPr_k##sR;nANW zYlecXRYHtW3S-rdF>4RqOhwKLoRx0I(Ls+nA6i*p4G4j!qM=)n&CwmK8?)#d=XQA2 zVR+>cc;zS1)nm|4pI$huR}9^RXQ@vP^$+3=@KW*x*^kxSP1PI(?=^RwP)@N+I|thZ z&=cj^Po8k+oLY-AvgZYz5BN88GGwiJ^*iT{Zwy)XE_hpr@@^tdd3NJFz+zXZbWH%A zkHXCOkY=DaK7Sz%@5kwm4QD&wf?Et-_?fu9i@lyxvquA`Yk=3)z-<(`U4d-o?sa`; zK7FgJUFOMj^C%a5IcvF-+2<-GKH1(uGVVI%kxa!K=-=kM2KfP_kL?S`Ru~)lN_d+4 zo@QobIq{QuyL&!y!bz>)B~R;n8F9i5-!8%I?|bX1*7^NPH@m=V>laDyt`RaL!jm)e zCMvYA7^3efz@QzlXb()lo7rJ>gh6!4u8#1irz4c?1dRkQT-kk=SLS`j&yIY@_}wGw zqwbDg4DYxH_eGrkW2}c_9eHD>W9j1!9jy)VbncMGCNACy8^b;>@7~hEnlL8B8|>`I zjA8DKkh6bA*eJsKD(wH`?akw(s_y^)JDC6z7Fn_{ znqaUbprw{2qP9#3N`O{cz}*TOtWmTUsah8b2-P538BuA$7A&-8E?W5%5JzfJp%qJ8 z7phooYakJ35?nAqG9b?H`8xOB$$%jC_x*mpf1Jnr-1oWntncl-&vL0pDM7fY;7%5f zI+YTHn+GmWIO z^)~gB_UIX31w34>^LnSffb5$6jTpYeoi!HckY-M`TDlg&!& z@RQ9-`pF9IU%21+%s3X>iQiZcpF_Rebuwr$$36&7yw`4EmVLm$OnaY!h0ta-G&CAo zIv<)k584`q&VaY79NtQ7lAYHB{7IzusTjTOMgB~-@onF;Vs~S(H4E2V{aqgR;4(;0OIW<6wkB& zMtt8nif7q>QJl7@JumAW`pt|F@izQ~@)f>F-1Eu)k+^K!{rI`g#V*Vw=>0f(4IzGz z{M)$?Ydc{%;Slw$&2;wXcI(YuNx4pZ7564x=jVUe?Askj{Q55JIUwG`y3)SE!T5sC z!3h5=8nOyoqouzUUxhu;QVJxkg>hds5QvX`RVfgE@+#K8S{VOT&|C{XEUm#$n=v~w zjsH$^XW;_;4X=-x8Rp+XTbJnH3F=F=xx(FZ{4>*b@d*AGBTF@JD75{(9UhtC;V9RS ze(Xmd{J_J}cJb0EaMyXb(X3N5PE)~6@^DpJyUsB@GTy_{zD9U#CGU;(aI{ta<;~#E z^KgtqBjelzE||nQYuDlt%{?@>;_F6eq93?nE^Z%VrM2@$`f3#TuYunQ{oIWYQ{RZs z8o`y3cXR)$uw6Plta-&4c&nrw-s(U58_|hY{MVa&uQx)&`bIKZ@ND``|CdkYt&J(X zRh;ClJo_d0n*n^IZ_9w1SBVbAU!tp0la_1uHL$=gHZb2lS&)7y6r|7c1?kURLHasd zkg>}&Fx&2Jpypc>;H~e%W8Z<-#=~D8zPlsc_iFQS^qTe0(X%yEQ+=+O!aapIZ z){dkXWG9G6r4zJf^9(fd2l{#gH2EC#7-1|wUa9(J*&hO14ew>zEyVX5Ud^_n#M=$8 za;BMh-0*5X^(vr#g{(&v!T&k-e@GKg{>#7u`yB%d?Y9lgvEKsf%=pFFFGC0i=$rlY zTejlZBck~q(>L-jzRtUv|Cbnij(xL>-wM7JzI}|fr#lIk5rz{iLTw+Hf3Y8GbDa1% z#qp`JwlaMOdqryDUF{osK=A_n<()cz7y5}jY3i6~qYxgBPSiV{&)kVGQ1uNk{wv+| zUjv2HzDuwE@~b~)(pe`mu)zMgf%*2Mg3!dng3!c62A^XuG%(x#nSojM{RZaR_ZgUX zLfWW@f4Q$SUJtKCkv;XrK=!26!(ZHm6c59X*WWxlQkye766Nl`>rV|{*UZ0I*vB4k zLvBm%7XQ)lk={y~>TBuX9~vDaUS-Y5l`ofjILh?o%S8s4ZFhIaI<<-ULEG2Wlb`(Jb4ZzG@4<{R>(-x`sv^GGlA>LS@hTUUZB^>C6y z(Di0;vdf*iAcyFiCUE&l9Q$wJ0m-JUBn}xwUrILhba6YN$B{h${+HN%0rt1hXS&xc z4SD($xEDCX@eK9ipF-iYj4aPvkcli`uKj)U3PL+pXbm1)T{;rJlMImEESmzkHsO+A zsg9Yp=-J@m2jM4xkFBT-m(MI)`s1!|FF9ajq3cVcUt3vE`4x4H@vY?IvnP~^x%OrB zyXwE6IxRj*CN@*I$?Ti(*5(r2$vDNV&E?v6x!*PM-C2~Sy~@g`@xRRHdN-dp_)h$z zJYFT-aF@W+yC&yl~%c^2r|6zhoZH}Wgn{yp(_BjdRLpLpEJ zxX;VAdeS7T&T)0S_zwiedyS9=YI*CF4rDlbOGmDe*NFcuWz~Et>wE^`L*W1%;#)3pI`HxC%={vZ^MR? zY(IcL`nqJ;3H;>m7k3V8y`u>`t8ox_p{LB(pQFzsuH4WA_Cs88Irba_bM09M=Gik1%(w3}u)w|@sCoDr?$gC#x%M`~C4}LG0z#cT|5?CY zxl5V<%<5uIA4t58`HyU-7xCTCXZ|yS`Oh@sb%n+@n(iasjO|wKW4|pv{Ca!`wZQ65 z>|xL|`#GcR!Kuz59azsE&RRY9AYISw{e_>`R`(_ySkM07T0Liz4vbx#9cZx^2Wt)Ts%ntYsZR+6@tv?yhz(I;~md*!tmylioUr*GyWAB1lLFI)W6 z{%+mq8?SD%Yh;U?dI|5=OSaA1!mD23y?TuT|Az3#sMl1|WOGL;C$(Pl!2ij^r_^gD z_>IE%bY#bM{LfLcqZyhW+o|h<%aJGNB0toIo$$l%2MW(tVem2#utC8E;_HQ1y9=-+pDO%3{M0a0Ps=tQ+V7RtZwta^C%_DrH zJQe}Zp*-o?LZJ4M{mi80+xHuoXWwUFu3cwfj(v|H{WQ-!XW4ffm}$>8Q2v}RUIRFh zJlb8@Ioz1rDSQZf>O`_=9b~qFN%y(_m<7Z=dBon; zNwwlp?tzU#+hy=+9q|_Cr?v2B8SygsK>rk|l}ysV1%S1BmQ2z!FpBJ|l}rMb0c-Uv znWSf+WLk573q1J~F9TKsYoV(OgxdhJ0FXT#8@(W&y<^uU9D)|N7 zYF?Iyt{*tr;)iCOAC_#b@#+X(wzypvlP!KXb*wS+lRAQ1DjapJA*gK07uno*Q@w3RaMZ8Ht0TDk!9^c*WLY@P zm1X@MS#}|^>>MM@==%|l?3vB`Dr4t4*mu-Rvgb|@hn>_&z24)2EV6&N z+Etz(yNrUGN%(E{vkh9H#^vibz>96qy zpJjjBz)X9bf!QaPX@2dU?u@^z3x2Zz{<94HXq>gWll{}1j=e6DD%;jGE`!~&&veTi zWFTvP2IkrWfs)Z{`eHW`ZY5ky@Z{6hk`8-m8gWlP`H0VV?WI-49rCG?cr`R0D8|Nt zuD2lTw9Z^7NPVgWp^bnb^st40ja5UNb%Km}wIJgi5QP4=V7pZhBwcV2=|HU;Ru48Z zW^Hu^w1Ld4roU?i>F+YZD$hX>Yte!|Zvab4K(giPfWMK7VM+SPb zs-C(=yCJjGh7?=tzKA2UcIZqHvQ0AUUD=|%pK5FU8Qe(vL+d)9k|tW}M}Mytp8hE% z2saAcD&Z)nlpx$xaK9Cfek&yiHxJz7!jV_0o8L-sj|oSAmJ;;dW^jvzqi;(I!Zm?= zNI3erlpvgB)&s)P=cNSUB(r3TkEH)g3Brv6CtG~vKyU=%rh@y4aEwbSLAZI~WQ&g+ z42~e&N^n0CjMwn+onC;l_g_2saPhcZHh>jv(AhaNiQ{VsHfEHiH{2+~wd1!Zm>#Dcodm z1mV(IcmD>ss2`d|PA2WZTt{|Z3hrzpJLUi9J~`%z{j@)XZ}h#{$KC~*3T_~{c5n_J z8QKlG3jQ?Z-!oMPI_m}7!Ak}n)P1~zSWCwrXY4xmt4km6EcN!Q=h;wJS#}=-Gwogm%6=P%>>7&<8-pyX zMy8Es4d8s%x}E*ii7Ktfbpzj>^d7u-67Tm!*0J}>pB_5M8t?(;lkaDyha2(#?i{Ey z3XV@~Ep)aXnp_9{Mhfx!XEL{>@AetLW1;;C@BsZI-EshX<3pZXk>8Tx2eGHkKk?96 z`nUUT(%LKZE^DF0KOL;N{Qg;X1JC;AKL+CeH!$0N$G|N6ZJ_4NIjp%Iz#e;-e^cy( zFAh;=7JmCfoLPB?5FxNXVC`LmcIq8)=cZfn)%TDss5vO{fIBz!5kCo8`yTjJ_$vd< zP36xwzI*oiIP>%(d!1XKH3o9d&%grvX+iqvDM9#emBAO-y~05G zvcAI@jAtw^U`)Qv*nEpI8^;=|vu?A8vyuTHYr))WRNo~~Q5)!0QJ;}lQQNhoqO4n4 z1^apzYQOw|(}Lm2oUJSO-M0jJK16HhTex5EK5W+k2NP?@@!!K1_VACXwD>P-VAy@w zuh+*N{J*RcDrW9@baKds$+KX2H5*t5-T4t`aegD>-y47(4zc7Th=wl?{!}V!lW2?hM z#>W6}7+bw3u-MUeM*nr;8MzbnTk(K&A9_xHw5{tM-FGT<(rWDW9J|QaOPRN9u69A^auGjl(R1KZx0WR zHzG@=``-3&@T=sgblrKz1%j$h)!`i4@6u(Xy5ohJMt;8aHN(Seuz#y{7natEi)VjB zYnW!8n0;4X*^>D; z*Ejf-@6`WZd1W6L4-3F2M}Ie7-wB`c-Pn}x>OSm04B7v=?<)13@G0M&oARAUXVE=_ zDsDy&phLfKp75!wJGD;Ulqx*rSTFT#pT~?3okz9X;J+9@@?|fiAddJ9f7Z*DuT(4o0 z&PXnnSY zx=8UX*Fc$f38$phINecv75_fzoDXqk&|Zc4!;d+R?+ ztUHyuR7>8Huj(g#bqYGq^w)u&!!(~yP0MxD0ta%3RZDKBeAmlOtKMHctXeWKB`w2E ztJ~LqSheI_N?Mwmrm@)4I%t?_5A{oZCr;lVYm5q76YG{r9-d%tpEs9i!XMnozhS-m z80xF&x53cz2;gWdzHYEHuh2fCXQ|Hy>Z*N2|KU82_Bx2h-z9#)tX1XMZxcU=?seuB zvhUc3XXsROi){Nf(zU++cLNLTzZzI*|HZ&Ody9cZ_MZ)8O+;{XF#eJtb&;*Lk+}9~ zX`O!`d-t`z@g{H|@9pVwUig0EN*}@ha%Xo99*lCI1#7!&|3!F~u#^y^onr)PX9h5e z-j4N$HhMcUxrO=0pzL5IfqY&-{0!`!`qMbKt$o@&f3;J`T%;_&V=kgKca^O@L#mhR zq&|Akv?t4cz`$I4fq_N#d;|0Bdkri!=bY4cbIo(UJx7qb&Jv`qGYvl5zEkj=VElIT zeB%9OkJG*YHdFasGq>HxzY3a>&*R{;Rlu?E)%CzYUn`{Z20qT{sxO#x4QcRq*WT_4 zpS{3;C76fA){F|PU(OgFEdOXj=kgCmSP^4Kqf?C?EqTbdl7Gjuqsykxj>u0C9SEJ! zhJBRLw$_o87tkKZ|DA1DyK}Og^>Fl$>}1Kwd=E$82`3qu<>8<| z+0K%WJ(D>4QlevKYA~ zdwM^(=pgu+K5ggRQypulvX!LMC1X0l&vpE()UkIQoCPk={V!>>f-~a-gYh9RmMzeJ zf_A=h*0)%P=llKC@6pcI>uc#xt-otem+S|nX|5kD9bK+HMp5$E(sj#{A=sr#_vRb> z*k{CfEPF2RM#($IJY*Q()?e=I!A#0^_Fx{Ec;p_;Z_4(dT-}ZI2gXtPZd-y6fpH5; z7M;Ma@jiPPwN9>esYYlb)qi{R8u@dK$42tj8u|0UgGL5%#@^s^>}L(+E&v1b?6n5w z+rI}&$85vTxt6e)@MFTIggVbw<6qHBu+=8BE7CNcbGwa}GdWJ@#Sv)gWsux7gv7YVl zQz?*sG;2!5K&>h1eeoOl##Z>k^2xH-%d8{qXHMPE<)u4m^C8#fxW&+<_PBSfflmSF z-G{Ify72hvGVs5qZHJ)O8u&zda1?zvN_g5>Lwkjr3T}jOw6lgb3pWqk*}~D*8tNz9 zN^nDkqrEk>UAWEQz9t-PuAx1`HGvxlZa?{(^63Y!eDRfR@znCgSH;4mls^hwfp971 zPX(7PTuS-#!1WT&EnoXuOLZQ9?7tVPD{G6+gwSJv?-oVi-{CAUYlt&{_HeDKe zOnf7q{U4K-XaCEBjZ(fNY($6Mw(%l?Ofnf4n7YEJUSKC~09TT4ECihR)iX2}Qb z1>BF{Q+i@OG_nEOeU849-*YWvBERP|rflv40ebtKTN&fk247_V4k#aiWYsF-`;7gP zW!IZ@_Wv2k{yziR|7T#Xy&S0d`J?cy&MRaCwXb<7{?1H7oO>hQA#5NlCbZKp0aw20 zOhEwoqB$RD3b<3x*=sY9cvs}hWbj*0vL>w>JUZM0->>4^7WjP?d}z)XKojPS0kmVz z7(i<}W3UQ7Yw=t0ZpeW>$c+UvyDd#UYY@X%Y|a}XGo){d);PL$25rL=Sp4IMi3 z1i$OMy2Qh!$cBj?PJVpqHHyA4@}Ity&S_*krh*&py;tSJ3j6ntTUX%xgX~~s-{BDFXAiFU6zMZ{3OrO zcMly z72M59oHHk%2kyos&hg8x1Xr8H!3XsDW^gs&b`~B<-#H$A!xypAE{48i$hI+@t2H`; zJDfD9+leoGx+=17pq)q4cj_LeRQlfMeshHx$0GYO;JMUE^sVvKyyZfZ zmTylqFwg$3fw}g019R+e8<=g6Gcd~@V_>E|+Q1^{`2uM1+tB2h>?^lvIXtawnSK172C-rq?t7Dglz9n0m@fS7Y z2Z{}{;zq8*kCL}B{%_r}23u$kHZ)Xd|9$b7w@of`zxgBIsP2CN9=QgaXRkMDx%N5( zbL=$+X50T`V3z%~ftmJG27ZaP*iHV|W2J!`zer97xJTc+yHE6{`HSRbs$cnGxBQ0; z%(fRAm}CFUz+C%&pm=8@vTq0>n{WVn+fKm$6IsHU+&J_SDGo-46K{uR{xCNU-YZ?m44tu&6YD7Fe(0$*FQ1UqJL z;HS%$WR&Rc@ASO|XU;eGMy0#-c{zDFGL(HmqMcGl_Gq4w&YV1GXiRcd|9^61=)fdS zH0zxA{YnytT!j{=QrEsooI|toz!iauLw9PI)q}h1=FUA=cl=|uCeGt>CReiW zA;aHAw%Yh*W#2mYo0IuQc@zSVlzn-2j!Db4vkc6!`xuyQ_cAccKFPpLySstMmvyny z>Ki-cT^jeNe*P{2&tDhNne%eTXR0}(?uu-Cl=+Kn{+Z6dX}Vq*9Nz8q3!d%Xi96?i z6X4&9!T1Pl%~85%K0kQJJfQB(nBAxPpw0qmZ&wX6UG_$3wL?!)^5kwLI|w}qFS%Zv zZtZ;$xzr#+8S_oh&(58r;*AGK#kH4A z>HUCD3BL>cXHFiYz{wWh*$ggC_rheE`Q{4YJ_Og<80dOZd zxXs`m7Vc{9X(z9_JU0=Rfs3XEhwGeRAc!vOy$l+5)?h`STdu}e#XU}e! zZ9tv?|#%hKZ%nZa{M3Z zNt`p!pUQiVU9N8(`)(e%gGN>e=h$~E!R-gPlR7)P(^)?eeV4I*5|iDA?v&h$qCd;M z^Uj~!!&G3`7}-%^f5y1p;p|~rc&xeoPhWSPrv?sZ6qvGh|0`?pIlDsDSJD=GI{hIYYXi_v-mdl!-dj@5cOLt3o*Q2{f zk~Gn=<~Q5mvh87RJs;%#oyb_-J*#o@qa#I&-`9Ke$4^Li=#=q+CbjlAiu5a!c}o{M z_hDR`#6g45o#=9si_^KVU-R7dGv`h0Oh`5*8ZHe zJMT>EiU*NNj@@P;=YI{%vs-~Ge+{(% zFyR*Q05q_gx`;palKyje;x>Zrhbc+nkFBD8S59j_?(v5&g+HE>EO+_iP4W_dlzst! z-1R$0E^p&qk3atA%Ba*e@n;P#+rG!G=gYi*EdKbF-h)4$A^lkV@n|yd4*m!waUJ|2 zTU_6J{4ogrSk77S6)S?n#Urt!&PY1v0Xlf%J>EZhja+lh|Ar4*XshV#9@_F{>Fgwz z&Ti!!j}K-L_xRw4reAXH9~j7;Uj}B|*Bi*4Uj}mLmw{i753c0hFI&G|ZksOP8;=jh z5jT8r0^3CQ9%(+WeLK#%B|q!Q9s72)zyBOl?<{+`f!w8HV2(Z1z+8JUQ030RKRbya zU#<41wxK5ip0A0!Ukadet^IT(9&q)*jkk{YJv4Y*(UXpx3Xn;KY;Gw&OI})f95HNFM6onvJ>I$E}}`k zGke}#zw9TB_uZ6j)<#H^j`+85)T4%=_oWx!7LGE^+6=g<;I;`z{mj}7IO&DI3rC&I z+6=gr;9e1qcn#qYLG$j-;5UJfQby|9h_km_J_XrI{PURiBD|OMGf)TlYu6@m@F8`O zzxHWxJ1INWr*JiG4xl@K*yWO&98_50t2Ikmz8JKI|0Thi7 z=B{ggG`Ds?cU8VcSWob1d|;mr8sEwtNFI$}rMOGux*N%(@yFo<(ReLC2p(^UzCVSgPQ)9c`A=Nh zGd?)#IttteNu0wQQ^7^S9nBj{Df_?Sjf;&8WdHv|U!FJaaKG8fH{y*3;L*JCu1U+W z|7l>hz1_ep`%MG+=bwT9cX>npT+L07mj4^vvLgm&+v^R?vDX=xYp(&y$2Oxc@}Dr2 zkV9xI>EI2W>ncM2FO&Qy?(xRsoaOR(W1Qli{6EFn2T;!$u3BiYR*{c!G zRWAtL)CxjFWrEO6l>PR#rKAh?CtYwL>4Jku=loXfVA2IENEbYZbiom%>nv*(>4Kw4 z7aT{r;CRy6*HAl=bis>B7rdNw!O5g^e)UAW*~jJ0>(q}fZ`O!TX-_I|UI6ZD5Nun!ptbNB>q4gcD!p2uEL65rh*D_7;wQuObL19_%h0V^Bp9PCVE}IL4!j zAly7~X~Ho!RRrNyf|D(Nbp<$raGSxk3&)sM5rk_3CtLh##;=MXTsr=S4~44&M-Wc? z0%F3A21gK1`vRJT8wZXc+*EMy2{#@bLAZI~-VtshID&91!O1Sa`eJYd;WmSNO}NX! z5rk_3w^g{w;0Q_C#`&v1f%jzFe0NX8(RyEJx;F4$s@~U`F4^UA$q|C_lk}wS;8!cp zOlZ{XSJJsw=Elr)p7t}EX@#pu9%I}&pKL$c*K=k-S<^?$xy>3$RD82uujXBn7n&onT{ zzSF>5`*s8K?Aw5{1unwhHk6P_*oWV?t_%AYPqwDl;a9tj=Sv9JVgJe(ycIw0b;v)( zrxCv%`*$4s9cQuMu`$D%G!6S;7HJEJ-+&CfA798a?oIRae7Zjv|119szds`wzaJ*1=;O&sD;_1Uzu~L?fIF0w*VDZFclIa-@bmRdB!&)PzJGcb*3a1Qsj{C0F6+ks zS-FS#^lriU=`No2wYAH7TGMCq@50l``v#u%{^`PJTGL+S{H!J5zGwqswlFYK6We`09g z@ptMzO}{&@_*W9|>^JX6-Cs`P9G~YXa4#isnom1&{P`rVW4-WMaAo(hr;)x8@0aqe zsK4a~8BWnixTn1Olr5(D$? zp93}K+n}wr1no7}vss_(=IljohrYHGo+0QABxi?)G(%_A?%w0A`1Gr@ojK<;;$zr* zEZ?}!K#pPGv3OKxAjh!pxEwwl0Uw^~@|$GYkK8)lY#{&tF_8cN7|4GR4CMbm2J-(O z15ZqISLkdL^7+15Gq+8fc3t}d&WTK8Y{%)m2z;q~3&rj;k$qV9cl-SCTi%6)Z` z`>XJau%2do=#`4=zB=f&1%F{JvO@k4V=oKRcV&X~PZS+x>}5g5wiZ|~yz3{4au=NE zC%M6;i%ryv&E51hdx4EC&_6V|Pui7d_280rDdQn|wn#X1hOtY*i7tL79Ca~vDY&WN z<_kyp#x4an58OQApn)oa_-rM(S;A4@DuQsE!QBDw*tU}DCm)G($Mg?*Pv3a+z;x`x zY2bEJuHz4J>>=4V(a!RR9N|lM?4i&3((~khz;*&)7gl_F_MUS5<}0dL(>r)+3<_Bh#Rx7&2`P`gzRE4EFJ)tV<4b_V4_m z-jQpW)XUpr*2B<(sq49v3E7?^4QkAbq$zxdyFvC;^8eSGXUOJm=epZ#a)>_g)pb9eIphPl2_ z!XJ3O-StHu#7_6tEK}v@FWj;h8<=fB0xV=~1~VtmBJ5}G{Z76$WCS+qT>gu5s5Cuv zh4$n43kF2_x7~W;hxd)*~r9$-F%Vup1w#NdMoRa zzaX%8=CjJfAN>(6VS;{Tc1>$Ae2_PV}V?8<soy#y;VL=lCM~M))Gp71rVCM_J+a#_aIHY2a=Gw*g#h zZg?^8w!N1ZPJCp=^>2>goK8;#=cGRcPrc@QV6^#nu==x!{<_DXzCh#Hh~9jkdE{WQ}=9TUyZqoq4*7e`%`>=^y2<^~yg!pz+AE|6m~h|1*$( zBlrWM!*`w!zn%x*j)H%y;A7^KH!+VWefb9cnM=odD zS5WV_Z|?Cd=iY~FeAExh`y$gG{S)QLv7qnGKZ;rgK6_)V)Q6nqpYzB0M^46BcTecW z|51FeV)~SLH^HZ?3y0X67xP2 z@2&sRTYF>dn-y>9y;xORWc+^;i-Y*NOdeMhhb|5jrjt)I z`82osBHYU~fO-D(2F@2J8hw#Oi!Y+@o8R?CM6>$eH+whNPDCFmpKhc@-{sz&nYl}& z?;)Fk`o4Js|IB2Z$!G0kpVd;Y{L-wJTA-=h>3N~As=bVddHzFQXaLXJ3*3^stRSB@ zsa%zjTCc0{t(o^HWrj}S{n_06WS;kBh6;FAKUn2%TeF!5mbvZB@93|f&&pE0TehM5 zHqxh@la0cACZA`ELILGV8%ucS1IpLE<~uIG0-446+YPdLbL|iQ`Pt$Rc;2WrdT0%L z8!|=X_H25neF1wRC#FS${NK)u)f|5)1OLS<_+T;^xA}t5-WXt@6TS+dd2gmuN3$22 z@4Hg2^3geOmGkk_mqI_6hu`1I9B|`h&{ZX8Vcq_0$Pc|kxwFa3w4+y1=q;Z0t;N5( zO#VORzZ~STr{W5e$4BQ}Zt`f@PCt3J^L(cc2cU-lu10TmMWW`m)qk~6Ac&-pV1^6#%HEAY3eO#oB zev~f-|A@Uk0IpJR+e`Qgdi!n8N`v=di<^8HZLe)g6>O8b}dpHB1K&^`2!o=fR- z@>O2PYnSA$FrtrdB#{Hm>t z0r2jyXaYYTG>QyBH_@M&X~WrU9N9R6F+r}^pJSn)dXz_z=NndZLPuqkFCS;2qx|To zG;~yY`7@J!Vd|6AOJU?_F}}6NZkL3mCn`^syyLuVZYBD#GQOL>uuiVO;LFOtravZf z4@~Y8mR_4g8#M=Lpl+Wmb?lfM!RhYo2Fm)t#npn-dCP{E;Y$}+1J2x;zX{%RaaV$C z=;_10DYF0L;w}{qUE4r?|LNjXZo?DU?&!XMxVZ7ctp`Wnyz1h{f{P8hEZp1(Z>{0n z=j6=rffZIfn{(9%ID5IB5M|u!1fhd!!S(Vp0@rZXo%7VSGnr#O$T!T#)lSKfd}N98 zjBVq7>;+c5*vf9H$+RX7#g8uD3Fw=9t$5DN?4{}l$=2?CqxXxgoEATODbyFjcTT}q zv45h0{t(_u!Ozv6BF**YD$k6i=i-Y{AGOiGo8Td%Cy?oe_I{of`rCaeI_IqH&?`La zouH?4PG%e^!^n@mj%+wW=b$4@UjNDtJ;%3GJoz&_C-e-@lHtKFmLq>ob97Jr-x5xK z=xgaPqkB^HOUd67>#pZMs-*l+Nelipv95+RQzkt9ORv7W5>6Q}`a)fit;ge|zi=mu zWUj|gbv%oI>fY~?ltJHUe$v*N*FAKV`~Ti? z^qp*-hBd+X9O4bo^DMz9xc@^CI=@p8y1!j;8UH~QToH`lBDgddpCO1nc_~nKSd2a8 zsdm`xXA+Yd=5v0`wQ2Vqvf}aEtaxI3T0F6h@G4;|;bp=m!bZXd!uqs$!={{&Z}yxC z|Kj_^nzZ=AHY+Z9TaO$qQ=hiMe|=x!T+l;rWb}IB6J%F2JU8i5(sbrS_beUp-faFQ zY%Rapd+!(Sd%8nK=OE;Nj2?m}f0MN_22Z!v z>N~?%^(Tc^-S*jHox?HD)jdN`@GMzvp7-<&t>9U6Bi*ImhTdcTY32uNBLtZhCGP-q zXTginC$0=@&-xtfqrMOK2t7o3qPzRB&o`!7)1$kC_$BHJmMQM9D-fS4-(lIzDF_;y zcR2fV*^DznQDmw1&0nQHQQPKN)BTFmf5qk00h*7}zr~fT4J(fQR*dZw#ZEGM(r;B% z<8Q6?b5D)VG-+-_AIpZ5PZrur_RBrgQ?`@oCuGgtJkx&1o>!>zSK+=pAN)JyE1tRaaAIhJ_Y;N8Pj6Bmaz@YpXhE#7 zlKGY5bp_Gw&>83ZqM3^e0={fRtFlL<+v!L885z1dFlSt(9)FAC89(Je!JYV5{ZBi` zKSv+zO;sBir?~0YwV9y+^+{}}4$PktuUhyHXTOn{)jOP6WySM|!>@0!=J|$b={;@?K`ANHn)oq`44S+$l1ja$<`} ze~|RWz@=&NJ?C_aw2bH!iQNls1-OfQheqt}+0NW~wdT(8aiop!6zK{Lj)exBCw7V? zCi4yRk(SFlMV8S9$#cg>2uy5w75-<=|)Ic+bT2NDKYda&dZO-yrHly+5o` zz1hcbZF=ORY1IFw^horaw0M+nqa%Rt0jqd-0quM+J)-_TyPkLE^6g~usOT9grX7bi(Kj#Chg&IUJL%Lrx(zt7QnF>p4*wNl zGgo?=J67Ie{g8S$YzRgczmr(pu*j7&qq0L!)xjs|Gs#lTv!MTvE#_cPfH&tt`6j2+ zvVz1mE8_9K=)}sXyhCM07u6N0PlqDYq<=N{)R_U+`G!O{AiJ3()Zx#LQV+$;h-Y{2 z;PD42rv#oh<-+6hd4`YOIYV~nr#v&CUTeyyT*;fO*|%~Hbrql8agxJl&+(r4?Dv8$ zpFK%jd=|yF5TD)TzWX%z2KY+v|5kB$@^Ro-ur0iLT+6eO$J>luvpVqa(ebiv#%@|2 z_+YfdkI2a=GJ6p=!smGH7MIs<>1@tFzQ;R3)3@oN?~pg|zZRH-eN$d(>>JHX`(fi) zdBKAj*gX9*g699`qri#pB_lO|G4(t5%V^yyYc^&QPCt_%k=zH^6_G4_stb@v3x)e>Lz9U0b~3C+l#pl{3_G31YMfqBS8 z>^!awL>HCSx zF7P>H^ID*C`tSfY%?t8N1M3V)STD0I?NW$kID^AWo#vr4DT`4Q+T$J zW2$3)T2WWzo1<%KXIb|j4uf8jwpR>(5?>_kucqA4*ZDp%9=&?8?C_)X>w`R}=+^{u zR!_$^jK}`xThZw)f)``|3t|&Y7rY4>CkS2KD9HFv6J)%v6J*YEt>Aqo{f6x=W}K2dPv$oO{!H;s&s7kqhS z{M&+CN5;niuV|(?*4wemx&vlaM zNpw`w5kJ#M{EF+aC5=s-PPwxSn)_cCo_u%&b|v+5`7H)7kMNb4dWoNtc*b+YoL+lP z{ov;jmDNt1@f|Uzs}qNxN6Z;(;`ksU>5_v$@tx{=F6C<;AsL#B?+>5wbk)PO_i*v_ zcIr{g{vO%Ark!UMhE}?LEE%DE0?>VI6}-c^#WEi&m~7zu0@+d%;mcXbh$ z9WtP}PoUG70r+T~`T)fn#guzs(IW-#@m;dqU{UCvW0WgQo`Gy?!&X;)9;ZDW^*E(ZcE*?i?bu1Cjaes$rX8cKf0FKfBYnz0cb$3z zliwDd61w6T-(nw2&PJLS7`fB6h5nl5_LJ(D+CQJK-^qRQdW;!xz^h-s;!xEw%FtZT zqv3MDHK81RL7OX%YV*Q!BY%tfEN;d zt{Qo`Avf)C3v!~R5IeLOo3#p?b~Ltbe{5-JuWV-TrOj*nk-h8v5!v<4@A=}fM&>HJ z(Pu3_cP-o(Z)qj8k%ml(9rDFLI^>UhnD9qFhQ>eklV=y~*tyo%C7&nzteeNura?*%X0iT&3PHy{@@_seF?8hBUpu{!c- zy$RnYzT^zyfn9+6klnIB(}}k-&)SDRz>YAsZ8Lc(AHUMz*+V~dzB>TayUIiH9P+M} z&iP*7&~?D_-;{)knHTDw81etxC%N_C(>K(uyOZaK%7Z+22~wu=Y9Ma<*$=;oeuiK#M&U!rS08iEedtxwN3%~2ed5;pO}7mBB%{y>wq*QG z?ju)OTS$|B+N5{DKjI4*{7WwWdEseeXX@{LtGT2XR~~D5N3y;Son6g**Z3f)&uq=x zkOT5Z%)wSMeKWJv>6=caAtNJ5EAiDWdD*S2t@7xP7mRN&(_XH6phuSq@Al8ni9aTr zQ0<0?uafU~%AJfot9qZRzM$>(*fE;hOm^GxW6IaRG>nW! zKE8qsG2hSpO6YoMW9*Hmg=EjiRs|#AhnckRmN{ux^$%&yOZf#{I+;p-x00WE@89Ll zd)eioVjuhCd(U5TnoH}K@t#Ky4YbJ$FmIs^(pA!fmB?dHCpIuXroToF3}w6DkL7!P z7rmIi_2J*3f3VYxUVV}G)lY^tvwtwWD8KVW=^^bIlkY)(iR5?aquBUhJj}T#*}2yV zLTBZIjAx8{T+RROZ|WUBgq_unO{6m`W$U_BNEb<-24-dyMyo7*4)_FFTa&F?n_+w? z3$v~{KmH?V_UBlT~L)7>$>>)+@WNv!Y*ue4YH5i`1i6jD4M9P2Y4aWBEidUUv>c2C^4y5ucYW~q!!U);*J*4>NW%^R1RfHN*-_)9b{b@-_@E|)MaIzwaVo__G? zUOXh6=>HJ&j3_pGE3_urdQKnsnRa?UWMpIfD%#jLIxv(*`ET-WGwEZ`;O=MWDaKh8 zeVcr*`ixNfT2EvF(^@%z4JHGDM` zu!8uoN4$kRi)4)8&R+0ieP@OO<>gCj$gd`=bI7{NAM*9>64E;DJkHj?@JM1v(~6wX zenWHF?1>c3aX%F_cVL6)h`Vfm?)2s9)J1Z?X>JbtvA73E?-^Rl;r>eJol^BT{nJ#F z@{Z{2>0tcN?mGp%qxDsf#tu@K85)NL*h;{Bf@y!hw9sqxi|9jR9Hq~Sd!VP7J515K zLQ0=Uw~u7qa%8;u)sgY7$eC%_D|=c;MLu|UROG{P=S4o)&?(%!26^%XazrxYiL|iR z<)gGS`UG+Wdxk}+a4mFi#%}YOA^CS7qI{KY>Hs}>^G(s%pC5ACyh~#TjWyt7`Vcyi zoIK>x&XtT+>a)Ik*8T2n_q!7OEc=l|W_@8hdH{d5^nhf!)|JH5+5>QPm@^~XcPE)q zhpuX%?mhmUSUeltEcPyo=QJ-bL;gsA`ZBmyO!8 z7BcO2*9d-4kMGl;=GgE+Gaj428M;Are$R@3F!-|adSCYeM%T?B>gc-DhK5>@|J>6! zU>Q1Z3_8!~(QAC6jqY&f+gs*n`^-b zBEv!zJex9pIxN((g7bFdW4^5%7LvY?Ep}|LTH=%WF50JVN%URzVY4SR>K|$RQh(y^ zp}xyLTO7ppnE&u+i!GqW;Uae&Zlz74YxU3d#Mf|+<5|ARV1HdJ<7#wuU(Up7e%j{P zGs$=*;{-m?Yt97rrm{B|8ROEW)-g@rE*~87b$9k`<&(eYzBF)*_g?r~$mgzi_T)X$ z++PkQa=rYtK6QlL=rt(xiPmgO@c&S@Y`#D*CoXw~|2=MOJL+B{n*v)!wwUF`Rd?)B z)m^r)^3iuDkEi*Tc^Ln8n6AAd)zo=DVF>orWN^*kng{yuh0@=P+;XtD=0LO1*6Ki| z6_U;7m8mr_gWur`^- zZb}U4#TZK;u(xVR8Fh}rzvh2_$R&;Y+aZT<+wXSxwxlVsF2Oi!T%PYj-N@6Ffu9~) zoZj-prRAD0He*vv#%J5ye_6qG%yGx~%DZ~=X*0J*hC+jTV}9)ZJAG#E7=urv*dzQ4 zZ*2px8vXJd^-t5{cBz&j77 zJ#XrUk3BlkibwGq;j@TG@gqei1KG z%(aHR$DK!A;W4AXmJdqNU(BPV)2y#Chm_BC#9NjXJe*-|c_Y?4`bl5kwZF0gSDm7< z@%8%2DFeGtIYsM5WymtYfG;?F3;k|*Z}YIw4sicqtl@v@jMdW9nW7zdc@%Qa?BRho zDv8&Mcb`8?bBou?|8`cWnR$ccieyc2_;sf=Je_`{N&C~VkkKXYlFth4(MI~{U0*oH z7$Qes>-!JVp%ve`tF$b~HU3;!!+V5@15&ts%%oK*+*G`|sjnf-T413p5U_BtsI-gzUs z$xQ?P%&nW!@V&gD^2~R@dr4DUm6l1G_JQ0DE{bn7hMzS0MCY*9!y4EN7Aw6d-1-D} zeR9rAdz#wl>)5J^;eF`roi&-S66jhDuX^?x&l|2ix55aZPsJKeM)rC%aq?lO?g6-EEYQuKc-k-|48xp^k}EroBokajfRq=A!-$+@fe(XxMks_chf4y4HbeazWXW&Zp3 zjte(&mSh)m$=&eM?~wVcp>MA*>Rl8*Oil-91BN;DU^d4#2CmWrc7M4xZ zgpLv4O23=3wp*Naadpha#s?C4Uu7cWcT?`WU%)d7_V6|oTSJy}kC5gU*g5}uJhSj@ zWBWV;Ehc&9p|d~7GyW8wu};7k4bQWOww5i^8#O(8FAdJ2PBbm?b@5z&Sqd3`TbE zLjLYv5sd$QcVh9B(^+5hc$Pck#CPYp^_=DAtvnn0`@^lJX+}RtH?-m7+*it7h_YwM z&&i*2g3y_`>2iRztsM7=YQVZTf6 zUjPl4e8n2_(W?IkG`z*3;U?OZq~X7G&~T>o9CClcZxUv$w&74PKAt$^eOx>Km^8fo z#I}1+>Tvr)VL-QGU7(=vj!ykl|MjZBRcL&9NLIka=3OFOqf6YKu-dD_{uGGX=)i*{Zh{zbIo%>kp^Q)~w0z=`I4vwy~p0ooCb>~rk~(T~P0bsk;Y)vCCGzFdy3@A3blFQZ+J zE=i*=gVFcTMV&ESC>|NdFmLpf7LyvVHk3bGvupW7)@z=-m_YVwQb6ef22) z*}%R9^=FOSpL0|C^ER(PvBA9ll#fh2s6Fl;FJeQ#rhVq}k(A@7kYC{dYqwhe5BPGr zvK}AFWuKSMG1bB=g1I`&#QOn!nesuJJ%PT=u0zSY`V{Osd^_xq>dHEr$y58p>P}|Q zv%61z2Yxcmvjg%A0bfG*X`IiXUW@Qg*0;wFC>#+6L?4o~o zznZm}HlWUUuHjolL&R4T*IYi~w<3XF%lqhiM|W`dKe|X~?VC8ywW|wfq^kIzDQBdv zhF7ft&Rin%pNs?f{>sgBIQ1EfA5CS}X^sk9?ABFj^x+Y6sfF%$Ng1-K!zVDGv{d;f zTWcP;wkm75_{Z47%<&prS=M;(#IV*WnyB;p*k4AUVwdS$*Y}WViz)wj?etSGwNt*S z)VB7gt;3062yEsQ+X;cx~GvI{+DYsoOT4?^C|Uw@5!*_?mr$ z&-JtF?Dlg8Fg>uU&VuN`Pr0VWN=FV4JU0EbyMGVrhN66`mO)W zK%vS$y1&{f^8z;?)z_-*dy4uiS^WLL8Qn!cenMI*ZJ9BZUgP@~(mdTM z+S-Xd@AW|=V`S)yzA*H~+4TYRoAN@BPuJSIL;0|zRn_! z4%@rm|42aw{A<=#8d;BEoohMw2n~TwTcEcV=ik>?F{b;>hcAaQKGV5ZO z4uzjY{8Z|Wz3uG5)jr0~?%J|s$`ir(+w_lF`cIcM$w&U#{(Pr;40Y=vTVHnlv1yw-%aED7kfFPw?HTZPlD1Q5IMB>qPt|2 z)7h$b3+w&jBVfK;?@vk7UL(~-ap_VMr>)t@4ej9ypqKY=!UsYBN!E!EGx%Pz)vOCi zKk|+aZ zI%4Y`zu_2-ORC?{-^1bk?UH@mxnlH6k-Z!H*T+7QWA#mKAFHA_Yu6V0L4MEOJ^!2h z9n@NE$u|vuKSBS?)|v=z#;M;-@^_98dSK3z{p-YH&4FdN_96bI`1{qPY_0c_w(D{D zJI8+gPVV@@zM9Qim29oR0*AlLsGs^>{9Q!+y%hdFfbDu7<0ifkzk9S5{2(!DID4d` zcUDHKPliTVuNRGMkxvpRJ6U-SbbL1E+ri{xbTKq&pto- zOy?=1uFSOpVMBmS>#!FbOP9DDre`fDzB#bVaP**kEU zyL$dNX?lw8E%}b4vp%68-1)=mw-!!r$4}Rd> z)sQH7%^fs;-d?`PvGddD=h>8VJ>@m7(EK{_AOFMsSEvGoq!W?1tk zbY;zPEb9^U%gw{stL*W6SyzYO=V2Gfrj?y~1TRs3)Nl4tt?pa>&Cu)A!}v>5_}~`4 zJ;MK-A_IOvnkNIC^%&-B$bkm@#gYS#46q!Z33{OML3}3gy;+}whjufU`q%r3#dop4 z!mG2Ht0AKf`TM9-!z{)R+0^Fw`xqbd?-1(e z%ztw1-PCOca_J)KW#$0zeDreKbc9~xeKbH4D7>fZAUGOP#j2V_hOoH@T>V*%&4**jD0%bfNS ze62PfzH=mXtK^QgP4&7{jXEr}-YyKCN*PnoMoJa%h{W*wjaLp#rgqv zIKJI7pxgg%6o12--d*)&?+o8hD%SZv_6wT4`W1wvzqBX4L}}~`+ToSCydWg|&b(Kj z-%)-^ezN>gZu$M)^1D)QAf3C^FXTP~+VBcCOZnryLnW8iwyR(0t2YFbbsK6;|CY)< z!+9yMg861oiD^6dkFAydj<(kqC)$(%Ka z-tP4M*4wdvtMum8bi3LvA_G@t)+S=-+za{CFbMiZ{Vm=060;jos)1{U=xV zY3ja=dftDQa|w$7nz;Ec#CJWdcw>#e%d`K&HwX1iPv@JZymvNsztaxxt)wkGR2SAq z9wtrWs(Nd_S}1##yssh;we=y=mDdBn?$8%~=j_J_^g)jm+Gl{5EYsK?oyT|FJm&C@ z?tHilc!-e9W4V(@5jf4MeqhoHSUV}Orx{$HeVu{1_O%A)+t(PFZPyr>WnX1rrajrf z9Pak3W~@gu=I1l^=RpIbxTCHr3{8YaK^y-S&7WR+Vw(TXsmG`JPfL$a^MCDUXx`9y zv~LH^Pwm&i?+=%D(EPKfCTYGp;oJc-kTy%+oJ706)ZK-@3r%>9aZcV{Sjydnn})vb z{UciU1M1%Gecqjca|^=Hyn22(JUAF%m&?3BG*WK%bj9oE=C+qWE3ua?&it_#_s8s^ z(SG@jR-Cy<{1DGm{y*B@JU*)GjQ_ur31pJ62-z1&LR2zAMP(N#S&-EgP!tz}&>E%I zB34C&1VS~4R;Hp@VhajwGZ(2?p^~Zv#I~rlvecz&YX~lrq^(kxWFVOD`*ZG{Ff3|+ z%lD7_I`^Lap7U(yInQ|x-v_o^;R?Q6_!f=c^c=p8f#tgi%i}sk)GMrguy=eIr|Owv+_uu3a!WAws-_*|Ekwp1ClC!UsJK5?7w9$?Z2fHc2|#C zVUuV5gt}=J*6i3N&iNnK7j^gI|3>#5{E~ji_P%T#I@=eq5qgd%rzo(gqb@e4&WrfWg4LLiojIw5p2QKZUOkPpZv@}@J!^noM$}GXr2w?1fI@3F%OKW&o2sCMQH)gR&@9LO9*`(e7Y`#PDtTl z*|c3zT-P$ek{#gSWbF$;Q-;q1MmX1~^T+5@C-&#LlYhDk9lO3m)axrw3p5h?i~9wn zPikhIXBCeO{EP7v@5MHKe-S+8kxPm;ulKYL-r~uch5SFc+jX~Z{wrr$x?OkP=JUth zKIE@o#SJbTcixcOuN$!Wko>5}$-^xi{3j!73PYM1d z@K3nA9R4s1om-c!_!K(t*_v_q_-S@J%ahlQte1 zaxC?X%fnAOvRerI+i^?u9zQDqxeZz|I~I9vN{{MHAM#v#jBnPVi|`3U{k6bJrDV3@ znSo8rMHxCfpH(!pfcTtk$VB)x_r*GQVe4$X{sZ1KaW-K*q25FAk(XIVFJVtr{{2E; zzuoi_w!odnHv72?CNlrB2RCia%m_T~`tW?5Hb+yJ#_-{k`9J+PW44|#ThBan+gdsC zeE)B1{>r=Lu~~UmVEsboi)WQ5vAOJE`{d_5(>7zbGhip-7oIcve!)1t?aA5vT(KKZ z*b!e${6aULup@pU@!Q;Z!jAZD#51l={Y7;h@yye6o!ofBj`&W*a|Y7nC+vuiC%$V@ zUH2pH19lR2#CNso-ekTDIwdRxzBd%sjSjzoe3RozXkN`cN5(dle=n%(99ACx!Xbge zyrSR)kI&}5k4+k9t?LT&tf0>0=}gBooy(%U=0az*DtRwpo>ab|xNxZE>oDs(c!u$J zU|^`YFfakT;Y(k~zaMq1@^o$Xd-^8Gu6z7j=%uM^8e`tVH~N?km?>cX>Rp%-ED2_? z7wv$VR&+dO&JSh=&LnRxXGHWr=hB~HghL1;=_yfl0W+p9$?FmL)49?i9&4cFp7f}4 zag;0js(q7>xF;#s$hy(r2_OBQ)LD)BD>#d44eCOg@W{7qu#p`!9`AqEK2ZN+@T=1p z$KO$>IEUxD^eRMMhu2s^u&KgY^n*r!+OuuyLYh8!MZ+^tXBPB$2YwC z^L5t*4^mfcX>kyrAK~A;SQ?zheL~f;g?oHe=5tx%@z@8%WX%e(9$JF_s~P;P0C%yy z&Md#k8q_w!Q+ME~Rre+55x;n;bMW6IEzT`lbw_64r)>@NU%t<4;H}-oTL%Z~39X{R z#xL95^z|^`@0P^5@o^^phs3ue76n>L(gO8_hR5K??Ka{Wr7C2HG7YM;Ke8u=1nmeR5l%@9^^IkD;Nb4$|iu;CaUVY!a zty}IJ(z=VfxAOim?|IQx9xK*2Gq zPP`Rc)1l=@+XwoP8KV27d#E@oQn%wqYw)1C*bmbu$@uNFkn>*Pnd8b?S2^qTM&JNH zwRH)8=loBj_n}N*;Ph*F^ikoa73im+EB=KKt!#uoH_pS}+dIT=yvGW|ldcMVY}NU| zs}S?+gZEo=2eoP(-|cJ#Dq>Qb%kY`w%<;%T#@xF%hZg+Z_-ZcBV=j32TufXeu;K?+E+#J9lfEhBO+~kp#rg@I&ZDv)mw)(7_Hwdq`)8c}I?}#Ln2k z=vko!lHK+~>-KLzuShw)|Hr&@R}O2vX5M{yWxdc(PM-ISW6veq$42}NH(t-!%%8uf z5xnsi4-WMA_?9~Fae-la2VeZ~s=;~(AN<9A0)6!kP86CjF2KI&($I79!6tNEO;cDe zG6pk>Mh05?d4dP-K=%&(?&aTfhxI4G@29}Avk#kY4|x8o6z%$y_L!n~`Zk`vU8;Bb2Jaw@3ydXf^u`x9 z|2iQUdLcg8OdmV?*Zx;svbN!n&Kj7_!nxotv z_!}CC>;e3iu~w;MuWPOs+XBJ^d}DLB@ojWNpMhWc_s}HSnmFs@*NLwu9--9M8E$9Y zoMdk&{sZDg)2}_t+BlTQ?7QZ(j=6$;BFdXNh7ZQLb1b<|XXO5KOHufXIYr?!par3! z8)}*u|75=vjQ6H3ZSf}9h2E^Cvtm+~{)>N$kGI1A_7>N5&9Lek9uYAcvX9@S&YsSeWAN7W^M{r=O= zoaZs;A^PX*%Gg2!LbKd)hAxQiMCe@vPxm%JN61r)zFPI=a?baU)D=IZyqAxC`R@pJ zSUaE-W>1K-=D(i8c~H)U{+T^z&X|8O>^Ka3l3x^9T$C1&e9^=`0G!9*999G;Gu*ox zI8XUQcb|J5`z_2x|3^7r&bov(NL?;-@(=d1;O9G~>yQk%TJq+V{=js`yf^8|l+oP1 zK;INTPk6fgS|dct#8D=MPUTnR^%1`cJ}mklVm`-SjIM_8U(kTJXwW zU@znOVX4#St?++z{*tYh`8|UI_4|sBp^ptGcoNo)zi`zD(CctsO3|^rg0fzDDb_La z?esr=cEEqxu%*g}Z=iKP@)amwaq+QS^5qtLj%n=6n8)ui_8Rv##_%Aph3>X)4tppE zz}q?S=tJP2pFR+R6YNNeM)-sXt zPvRdBGG3yiqk+Zx&x?*(?@wRgd-e0(9`_s?dpO7H<6>^-uDsyUIaZLh-Lkx*LUdJ0 z_9E)+kQw;r<@#sBO~@gK`OkMt(gH1vK?`Hh%D?NMkH7<_B6D!JQQ`fgi#A92j(6_; z?tAw@?A-f1UTz`FHo^n_@W9-;!&~Puj(g$F8Z*JB!70{n2Urt+fnNtNW2*np#a_Dvo5gtG zN^lvfu);p_+>yChS;Q@RqNpoOQ_#`T2SJ%z!UGVQB)s zP?hf)%J>$JsT1!n|Eg?HId_Ap&0N~Nj<#rbne<~J+KR+ylcx89&?vK)%6kLvh9~jf zgLKaC$gY99B%dVN|C!_HY1H?E#?jD~q~=in>XikJox}RyGUnliQ`*b6W2wtB?Pc2; z9qnN&)6rhGeTceGZg2NDwwKwwn)ar-?M^+VUCw$?zJW*ca+LO+|ID;&JKD{(-#bma z8^57l!HLH0b@Di{!hi5CSW!FMc~`j2gxU5M6LS9-q4=M0?c*3KUkNG zXB4p>pGqBHC*#Z4^N!B=7FbslzLI?SQP4j2V6_<^wWgHbezwP2Zge*l3nv}|r#C_S zR^N74>m!uIh-%lW-q!OAfy-p2yEIbiKmOYE!KVEDlhQ?ZYpbTTz7Egz`&pBM@49Ej zQ;2UIcslxL^xEHkAIbsj)Hz2!(X2n(y9e82Qkp}H9%ucKRoL_>xbICUY+jI1(~K;# zP9bZSvGaGBb%D7TgEhrrXz9Bp$h**{7Q&;1^Gdr14ngBLmGlc7p+8fhZH(6jpT{4_ z=ANiFu-gg^%#pxF)!Z=Z0cE89kMiMLZ%++s?Dqnr z4Xk;@OT~|OJm}a~WHV1;^wW}SKk>2-kK{$JQocX9`FfI1dpDo#>wG*9u)Gg=+~nqo zCy(}suH`O@;XGWz0xe0mkKGBKq0eRX{a*F}4)bm96d;t359wgJi#7bEiO;n6D}K8Z zpGkZZ-%a=t6>rj6^3b~O{1azmOnz)^Oqg!h6W&Z45&5(6^UH&gxoku3Z<<3{#(?$D zo8vs`Z#J)+5}Zlfqj{RZ(^9QLs*1vw5DJI-5ISq2WIJI6em^6+CZg!c9$!>aA?@cy@d#0{2iP;_JO z($T?v)X|f1+4>J;YxiG$o{U8KtS$g24lrK7xY4nhiEcx*?SIt|k44XP^cK!@A?q(> z-zjh71%aOQB{TusGWhNb_$_Gk4zwffw|5@b9E3l{mxOl9C&9k_ts}Ij9PFI7PVe=k z6&iYwV04g?eA-jM$9GNMlG0y5qc-?i7vusLH^-*mR61lvFKfuny(CX^U)?dCDQ}#? zzCj=3lm5%Rr!EToo3Ye<{D$%3UpHu7VaAiW#`ltZl7E?(dv0JK`2+)%jN{klt3EIA zck&6w`|_{C6~777?LLGmAD)Lz9d-PSd^y8Cvulxk4qovG_O`LfD3v{iWeYAHd``wL z4gV%QvQ10B2-?nGhpNdzJRkW7$9L#6D_g z_EY0XWB+_!Jnh5;TG=Z+#9rZHcdsyVH??K@n{0n&{v*Y1Bh=aUJ@|3i$n!YQ>*LXn zpgYsOyl-5NjS26%mv@JI7w-h#$ITDcWO8?8A^RB3R^?A;c@o~7=1puy=9xU+lbJZa zht=cy+4FzO{o-#Hr1+Nl=34>Zv64N$Pv`%%+zbB79tB(8c-LozXc&70bqV-lY9?(K zzL@;y71mTSKQkU49>`tj4dfNa1!D57K%UOjB}j%!vgf+?l{@(l?fvO4yf=)Dlg4|7 z?Pc#kW2^Ow&VNMi&pnHOUQJxJx3VssHp8?z)QkV8?+&SnnXNSpe!eI-ihNJF`J&Tw zws0nCkDfLy%U(=cHEBDzclY)2_?VjSukmw6S@&&a@diD8>+ZJ4<4qx2hH52*^lzC_aF~e>pXj2f1^vTh;j7zzP+Qu?AbJzfln%v z2JKf^Vdgu{o=+${IjvDN?>a|uuSrX`?>1qQeV3c}*t~JT)L39^3@|ntSi6M1`cd5J z7-R0^`?fpMk83wNZqMcsEAj=GOmcckCowmsE^DfYD{Otr6Z`!Lyr={B24=3aF9 zzM9N4kSCR=1>aT?Sn>f&(LTMJnA3<|csz4@D|E^4bzz(JfPa$n&c5K}!U+zXPvE`p z75Gj9=9|G?f1HzNDet#&x6k>^-vnU(U%>psp2Rml_H=2kg{IH)bW5BC%+CVm$9v-6 z9G4Qm)DO%T9=l}Etz8qA`hi*DiGN$7vGutP*|C>;3@aP*vOv)NhoqybmpiCGJziT8jfT^&4W7QC3a<9DII z+Y!fo!O%S6#4h5-f;+<3M#kAM-)UzT8r*54ylA8FM>rz6{Oj^6-@dq_u<+$&wZpe# zTZ3OP-q#W@xm@Mtr!9I;+qxGA9bC<_UnAeMro8SdPO(K(w}O8!i)ZH))%`vZo)6zc zw>Y~J+(-ldpJKhfn)x2fy1%x)ePax3+nWi+M^%0U<+aDEd}Yj8M?UeNsC>VQ%9jf) zbmWUaG2c^ezRkui$+=Tm?TEgp%@}JjXYDF?U&MF8q{=$qG11J?d(bh28`s1JFDCDi ze#lxDc379dGrL!gIFip?=6kT!7LCZVn_YO>%loa&x#B9(xqX-aXy5LIXT*cY3E*=g zc-;m3?h4Q7hOM9%yQ?_+vp(n)z}KVjjwA4nhDlakgm={C6oof4w=Z@t3jcsQwI2Px z2~+IbOqgnyn=sA(o(a?KnI_DzZzhyo!DHa;T|8Iw4B|=W!5P>8BOD$Vg~Q{baCm$a z4o`@};c@>LaJV;cqyN#mvB3S;b4-|KpJl>SySE8b>|7Hj+r3PfWM`XDcS>K*_+Q36 zT*`b*U|z;UU&jTRCqrLr^O1|eRd6}1d+ck01M!4fU|D*pCNJk|!S819{XfV)zgyz0 z9lBD#_-VahgE{KlfmhrO?&7&*dD#ur(}wm;#3#?uK@C-uSEvp<{03KIuZ< zYiY-qLfpF8pnSZ=<7-ZSt+@Z#^iTgXR&)tGZCp1hsQnS&*{`o;k1E`l9~W#ZweXL` zJb|y3-Qnxr2e?-nJe}b&c0jSD7%?UT8vO0YbrUf6m3C5Z`!+v#Rg% zyr?s)(DKvyGK$dMZ2V(Z&tV+_-ul2p(bG549r`q$ywBo23p}jAM|&dg@!;V|;@81% zec++;OyFHUYQ={tk?*V>!vlHh8+6fpUpXRhCg0jKm2W%AD%gBX3=Jy{`sTk>^AGq~CrJIpE6X3zs@L(T2cuBLP-*^Uk?%;2l zJ;aPlsy)brDfR#pCfgU8Fv-5agx`J#d=s!y3yeyBGdRk=n8D9JR^7kB+rL&j_<1(< z2|tCKIR=)+^U^zTGod5S@nLZm|9;58i}DEvGYk*@lDIJNUjV*k!W)k=ck?8}-hQzw z!yf0spORPkqSGRL_n#eggUjHZa8&CY}`vNjLDMJ9u&ic+!J4P;8(e`{~Q&Hzo|e906ZiTsgEAd^r?_ zFE3H2@TJa#DfWvdOtm)=;#YFw0er~)jb{_j6Fm3fgZMfg=&?D6GA+)!b2T!B@?z+o ze3Z3;VD|com`P)~v+?LHo&C8w56s=^OUPqDDPDxxd+2r)1dw zcFSCC#y-QIL^#a!G1b1(#HHAmn=skF)Pza)coSy9N3UYuuVntO01nVC-EtXx^isxh z!tKC`vsS74=JoQo(bS9Fwl6SYihaHbQ|)t2m}VCeYX0YNR%bHLK%P{d7JN=+&2Z@9 zEbQBSw>UH~k@pB6*L-(kzJ0m6A2)WtkNNgfSDQE9_OoUj>B(wd)PL-eDs%#x=a*f* zUJ`Ah!!YyQm3eY_HgqJ*nder6KMo()dR240uOsd_9rJ5lS z9D+t1ghsTudXpp2h*r+eNN4j~+EJhSb3aci&q3(G-=P8F3ZK6~_p%~4w_)=cu1N5U z=Z7o0_(SdZX0+g^Kjz+xk=8NzZ1{2ZgzFZKL54`Se{TAoYS)-B-Ts*gGwel-OOBP@ z?7QHpmGH~`KJY$17uW#rvo2h+vKl)B;eB@ehX(J5xbW~O<%RdpQ2!F3pVx_sCN&Ox$GVKi$6DZT|%PH)*N%9UW=hN6C6^CiPrt(vt0R(%RtVtGh|g zsTzJ1S+u@t;qZnv(1cR0OET;~8a&Cc%gFn`(u9y(=1RBi%LxS^!hs1UPV;^l^LQ!q zIe~c{&-{)9_s626aAat7B;1FFAANMOOSB)KcV3|29kb6N`ES2<#<=@#hIT70cU)qf z!oth&WosqO8b^HYWr^~^A24ON^$#?8t-`*~7X@$gBrGk!_M_l})ink2R#@kC{uiF4 z{rw}r+EHLF46Gdh*4p4l2Z6N~b5_f;O`kot$uH}m@-dtfDz+}{jeH{;&qLtG56=B+ zh2A4+-d^Z>*f-w)eRFohP6IxLJHkbs5s4?%KEmU`tn@t%ZoF4x2wop#{g7tTz<=Vj zKX-^ZJd$J8J#N;jDfT~z$F9+CWuAv1GlsM#L0$~8zY`yC1%{xb&1_k)XH`sQb3W(S zwh->)+*-9aqj@`aC9Awi%~ga;yh(44ywKX8&-*lJ0d-713|?$$9}--<-Ls25AaFyz z`6cTl*;UBvRSO-t={^3Bb&tJvpJ!JjPaAvd(q9Oc=eqgcBA>=*ICnR5(raUN!UrpZ zJkq-jxDXk>qU^UzVth;W9YRLkfenK6!m>9RE8CM#Jw{JkPu+R0-O1h5DY#R;OPPb{ zGDVV^zZ_|7RJ4~UTgZ0~;wRZ1+bM_IH=*<0-7%(o{{|W6q&jw5^;U;Z>#1swF%Bq1 z#(UJXpKAYzP<@y~JHmC=7CpS%uvslF|Tkh7&NJ2jpQ=;i>p&DBko0W z)+x>Y2e`VA{5|QH{|2XyS=3Pm50p+y_4p2zhM{NATiC(7IQ6A}Z)%_*!=rtPu=Mor zGcJ-FmnXafZA-Ts_;>NGfwZ;8dmVeb!~q+lh?88trs`TZF2fd0>PsB9sbO?H8#lXs z5=`Nl+5EH8@=cgw=b13u&QVC)XDZ~MvJ^r`Jtke}qP|^*(Y!Y^ zK0f{hdi14#TWWy)OZIDd^6)JaOB}W#3D}7=BRh1_`G{i9_blPr#ltwy^27%o?qTgO zMwZYVu$Q?shw}^z*#k9m#C&KTlE|Axy?4>xXV9omp;7ywQScOd@%;<-jA0x;Pv(9E z`r=`H+qcKUU%NtoyFrJ$LyymZE+dD}>xphDmi>t1{8cpXDD|~bA4rF6ew^l&vmZXF zFK5KD$JGC>gx1{8b3M-h&V#4${BFU5J+dOJUUg^faq`ctp3ffctr-;w7(wbwS#j>9len-+C zchjoqQ-bfkm17sC)$9QGqH&zH?M|z8LwEH;|a9~{adaO82PC})=SG2 z0&91hP_QzQvAv2hzLK%Nf-%1wo_85~BS+R9d#r4akAFF(?Y8d(ijf8OGN*^UDTN2I z0k2)Ku_pf5f<4&1Z%SlMT|WN}vnM~D|JbqMjhfTc^QGmp0t=x5Z&MHZrTasDN`sc} z69or?{Bt9Hzf~ds;wKdEEuWuI!#!uLhqLWw)(ztruf{b+!5k~T*~?lSSlpF*T~dwY zz!K7$ysQ(!W8IJLWxVU$xfymsT3U01`WZ#PjsD;xEAS`!cJRHpU^V00xW*T3Sr;D+ zy^Zba$BDuI^OI}18*P7i6dg>44%YAS1hVgQXqV1H37&G1-^*i?jl7i-|2Jq7XWbV& zdHRz_vZ2A7;?9A1^a~mf`4pBNKmqipVV`JblI=}!WH8C>8iN3IMf6f>4a~7=xj3oIHMv%{e=_*p=I-HA2-8Latx0P{V8 z{UmT88C*z#&ZI(TIt3(OwYjwB0PP>7ed)1}y0pgSX*+$$SE4mph4>l7u2KJREpt6o z>ouMu(3`*UyvQ1V1vIBNt*8z^aszUDaHascrVW{?4Z5(L|8?j|mi;jBAiA@jIGtgZ z-20##SF3!i5kyB8xp8Y0r+Hv}=eu#gP@JJz>2@V?S_d44&a45>n}C~m-$yG87G~EJ zpfit#d+6m(t1UJ#-MtfIJoT7)0e(b7Dok5N;rR;t6ouz01U}{}WWJ{nil&`Z=LRb` zHn0s`m+kF9@~BS9UdQHfM&*jD8UM-5!zAWoA~fhK*4$U#&ODhlciV#d_Y^>jBxhGw zyk28*_b@uX$+hVGBJvjf-e7g<9M}!aG@)NegW_(hcXFwk`)&uwsxin)t?WE4nhMr{F=ezktw@+z1J1+1vWA*~=OZLR) zU~^Ru{4ij@R7!YjOp1|L_npBR*!iCQLBfcvsx_Q&MY=D^sefPK(1r6DC&|i=oEpiK zYJ0iYWT%_wEO#sq0Y^V#&;3x};$SXeJ+d(R{<@YiR(J_x-I(9gf9MLMCtK#(v~>jQ z$z@jE))9NXoz^$a=@M-0+Z8{M$b6q$bx$DkZG9F0Zv3yo)ufm}=w7E!KH3wlszlCT z6_eInebI_j)|P3af&9`%^j%X-PS&G^KFGpE4R0AHXuzH(wOV9LRlBzqNkCGY)Q zA!AWPm_d4bRcB~g5;QFt97+L~Qo*S-a4Q{}mVs`fbB7*NI;w-N-oFjHb|8wbxiS=X zaSmPkv;Y|jy2e^$;xOnL&N()Qp=X=&(OGu3dRF_afphr2foBj;ExKIMwr#A(4nlu! zQa=rCO0us3c0||iQ5<6^{JYAHyHj!f6_;&a>c-u!IQWflZ;TsPt~iZ3cv|Ad%}|_6 z6E7hy1(|OnG&P6g@}hU~zQ3&;0S@3+A0c{ zDC~z_ro#T%N+}#%6dtY+`07n4{$5orgUuc|5>#7Mw+kDVA z>DdHpjnD;)G0{3(I^Y?|NE&zHBsSe)?K4SsS_4g!yd|FLl+CvPt2}Nwjibi=-^6Jg zPfG(2CK`Cvn$$)XZ3TyX-h}3SYVV&@+b2h8q-kj$XUnN^sMjbyJ+Q)-8_FM zkL2*R-k6U18tA9aH{9>$*-0MFV~G0iNx>H(bzDQ(!WzDeyBWn(k<)fse`L+2GY7+m z2QGTq>BBb4u4X^0g+A8yEvh>-B|>{s{Tgd?KL~W!a{~SCyq@!C;e+|zf{l~dcc`N5 zf|yD1mD1Clu|FPKnh*>9PV~GI>h5`^ne##8(dAZnm^0R173dzy(M3qVmxg{%GQ?u$ zyFJEgUcy{QU@FN|6xLcmd>eT*SPVVNE934UU}WCuXisj}z+P}-7;~xdxQ{$fl*9!1 z_YD?)NM=xv_SLrbEIPvd;*;wiiwR7-#~FtT#zB0;GIs09cJD<_{9NM2&$Jew1C4B0 zBR+tAh`X13lS?;0qHgg4tsjk^X|TfwMw9OF0d!By+bGhD6=(PWam>|B;v{p6j=bU4 zW9k537Lzv6q-ELLqtn2twWNu^i`Kjxod$lrMp_@Wv0q55vZG}}Y?60&o&4WeKn!Y8ipgTFP-#P2~SD|={MU8U7J zINHw8$Y-m21h!q{z`|krA^6c+ul%9-x<u#%(1|Fg}@e)IhVAO1D>&mQz%;-=U#t_EY-4 z-?M+m>3Grbq`*Q~_MSyu5!+o`XtX(BA4i2ELX2Ph4`a!oMj|UI^Qe7--dksZ1Yh=#yW%1_0*V=iP+);-N(jJr1JcDN zjA4HPdBiJUB*!k_vjhFJ;Nh2yk#PS{v=w6B^>3kk=s2|X7jD`X^#j`8X4c=?_LFXW zo#K&uk0OHzHh${Hzo2+z-y`sD&FRB#{CdTMi-(bIepzg1itPxVVeCzg(H!t%y&|RcI}K>!!NTw zNeL`;W%0gl{_%tk?Yas0yAe3N0eHL~xSR^@x(?cP?d{Edr=Ty0GxTdq#o*wUib27( z`I7UTv-r;1diBH@^z52H^}(#IOQZ8-82k^ry1UEBqxoGsBRUT{IAqE%-8>ox(U`SW zQF)LVyx4<%?&cXr9*tE>Z)8Jvyzb8Dj2BNXk9f#q@X~pom4s`ceS(bw%IK^h^mLgu zBt39oveR$Hmy@sTvls(&S$wOU%IN*2UQT)OZMBKazkf}RHTftwA$yV|z%1+gWjlJ0 zF)(uwzpa83$!y|z%jw7RXYk9xxV3)Q8KdRgbM2w-E#S^V;AR#0giIkl|Ay@SYCGF* zBku+D_poV$^#x<_G2g(@PHS9oIDCZ_d?_24M=;}4}LGfP@ zZ|n{^A4wa_ByTR_Uqm+#0iTtb@CV8@kS|QX;4gJf9oPViK2!hW^lj53&bo1COFXQe zye}jOzq0LJyszdQ(aW4wdXsnk>+6IE^1XqL!;0$QYwV$&c(*h4E!z}R7^v;$TWWoG z3_iHE2FbRsqx~HgwrB95)!vjgyDH^npS5UW=oZ!q^f@#s({IvW**d}p+{jmYebpnw z_g4L6xQV|S``fBVhs%CY_7IBmY{OqzOw9VmzR3LjJg=;tSR5A5Wxcm6XGE9%8sk^t zhnn9*&^>c+Hsg)_m0X|X4a|4%tp7Rxr14dHWjDb^vOTiGfuko#6Z~tQWViV!WB<%{ z!LXx)5)ZtOxc;=!z_=)0a{5_(-$lG=&41~Wc%#y^9up6|k+e0O12l2;Z6ubb~Q612)ccSZn#(Y9rkr6nY+G~BMw~8 zISawSBYfkV?Z53nbD42_5*TslP>P+(`0xF*;nmAZ@a?JiPnj>_p4#ohyi~xe zeO6`9q)ydSeaxxmS#n;4cIaJknmd)x)Lb&w_4GMSbIyN?9whUw^qwZnw9hbMs@=_m zDRvjapCZ#MPdx9c^Wb~8@*3tj>K+QJdE)he9HW$_v(PHF77{l7hfKSv>${H z)c57DCx)8wfA|IKjP0I4{v2nFO35oZp|Vgkmb+$MUD62mzx5w6yNbetpf?R`L`%_q zLsw#?YQqxIaQ_{Uhq8C?YVn2VXjwHq3fVL#&5qc)^xSxj0h zaavQXzPNHkEB8kjT6ODId_3~~lK5CWclvfR{kk2vadd6Tb~FEh-xBGrmzAXFke+J) zo_`kKQ2lK)oV;0#(UH=ay5=!4&V4m~-1};1>u5io{yf~Rofm6e5d)8V`XS%?XD;W= z$r{T1I;O6E3U||RZ^%*Z4LLf4=N3=cgRhXMA4pvP^aEYiw=eaCkKl)_`4&%I{fn&g z?nnzBy(2xSbCO3M^VA(&#l7EN>$kxYZ)@m|OwzJ~N4Asy&*cAH`OhS-SFkzVSd)59;PWvfpAKIn5Bkfx!wWao(nYWYM|BSfMJ(YDGZIAr+ zwy&dY!H@hCyodg1?;S2ZnJIdbVgCx)bYM2q4&Ch70~|t6Av!LeVswxl?-};T}H^2q_@lOv~*PYQaomyax>JvZ#H2PE@6OB>>8 zM|8%Fe^~<_h(EqYylesHnD`|7=WhHT)Hh(YjXfHzIaavwn-$NRex8ZXvIB1XZxzpc zf}a~(`#3oFkejwnY3LZ9T(F6K=nVT|;#0Aa+|il!0RD8e7HDDa(|qr)3@u8nsiiOY zz6c+fF)AooQUAdjN&4R12k>3jPQUIyN=QGBDx}@HZS9+o|8}3k6Q9iqJnhO`W9Ua@ zA9B9wFZ=%rk)7u%MCQInA^7_v6Qi7*yH04foAND%IB{$_C`xsBgzKJ#_#jE`R`$K6yJ=^$mG`-IPVL2*BPGz z(y05?`k%}TZ0qFg{dK3W`oEU9OM|Vf(`Eb9i2P{m2#`;uZ>d(u|Bh9N{Owc7+GYo1 z9$JtBPI`hq+G|F}{J^Z`f$JnIoWq=j`jrG@_z!d$yR{dWkN!8bAiYNE>6F128L*OM zCzH1sI(R&9I@f6-vycZn*Kqp$w3??(UhFu?8=bET*+%lA@*eHjNFyv$X(lsyV%Ii znRXHH;!*PVtNE9XS>ct0(m5De3V0A*{3UrK^K_A0PUEk9>Q|B{b*ae@tcYg+j65&9 zb9K&1`8k8W%j5-~W|H?Oq`zeHqRTlsZ^FCif|2Qfxy9s_%>8?lmv!Aqc_aN@OTM|J ziDtyPc}yDc`Wk6>5_h09ikDVZ4?jFuw1+(@)~@^3h)*ONy6o6BkR{>`yR$!>sQB)HO-#Gx8PwmVIgqUUMd=$#5yRM^-v1yqEyyLX{;r& zQ}A{+_ML~J(>m)f`%d|&m(A57*S@n{_P?%e!XI++-vFJKKAZDY19Dhf6r-;#R-3@l zijKci=IkrsBhTy!C#Zkm zvd{ciQMj{0U?f%{a37-(xU&>OpFX3!=Ka{bYni(#%;7c6<<-pTWY)HmxP!s5aqzW& zGfkJjt2eOI7(4&{4qMUCg7=NB=qYr`rPDhpBf2|QX9nn-Poe0q=rBCb&}aU|&}V3z zq0e6KkQ80k*)!;8Ds*&{!JQPlj(=)`9v-Km*nvTpBY61y>C#!VZZzeS?cgct>@R3u zzqr4Hp07S7Pik{1bTRew`}Ra=!-~`7`8xXiyvCq3Ca?s!(b~4T4`-43Bm|rKB=U3# zHcjgqe4X=y3j`k&-U zgI-fFurrhV=aD}$C%-=_zeBId3w$jmZ*THO;N-cJ^Cr9py(TX(x|Y0Y0Eke<+%qZtrBi9Gzl@-IF;G zP1gRO(z@|&aGyCzQ2Dd0A?u=AkJ*kK-V>2b8Zz2G349#73d zZf1Y|UU2d{p5Z)MJO`lBAMkuQjXoPXm|^dsUBTHw#X;vpNB`=^eW5t5Wmp5g?#4AK z4*IA5zvjk$N}ObZYbsu-`I~$1#~(?91{k?`5p*#H+ppS1$u%Kp{wXx&-m^RGT$fRo z(SI@zqHTXP?Xjn*5ZKtJ5SZGc5T5u;LY>h$rOv0$GIp+Ur%S8NH?pkk%$$&H+eG)$8wI!hn>9a z@s}VUAMhp>&SHFSo&QdUe7t0V^L|`De&_UbcX=OUqyHjpG;udsb0YFE<1kkE&$##$ zf)6{;*XayT?SdpjV+GSUGZ%(N_O|^@7*Z=4rq`P^S^Kl2d1Cs|lf!rTB z`ISfc4DvRt4K z4tK!7swz7+mFC6j3X!@DZi+vI3r`8^Gv00hSb1_d64`-$~zJq`5mw1-Y zzC%Yj-w&*ajy|I}NsW?n%YD^!83S?lHv~dYfV2?#BH@anRd) zD>l_g*C2TN0cks;Xlq2@P=I`V$~fNNx5HLoD0S(dBxBxT+A0d)t`L}=r4Sx@t3r7D zEefH*ml2Aen{tf#x5=7O`h#Ur`h%U;Q|B66sDDB;q(5+IWY#G(QaXfnhDLs~4#DT> z5Ke4k^LYUeI*LCUx&)0x#*~a&&A3QL9jlOk^(h1ocepgOTA_3X&=ctlwDuLOB{R-u zjoV?bn`w8CiW4o3)*Fb2#E~w$TtiF2Ypwr9V?A!XOG{7e_fzKw>I_{k3LZj+tY(f{ zB6O5Bf7A6{gU!?x>enskSO5BS4{HCZIiI!v^+t}|5RoGV$7zPHrrF^e!AF;_D&Au4 zk51;xJl6iV;J;t+dYC-UK0=z^#Cr%jssC>Q-|x}6V`R=ZzwhL2CS9^dJ!cM@pzSBq zQ=L5>>q&9x={qZ(aZI-tyY{FbQeLvG^toF5I5H@~q-ERv2u0f*dYWn}UD`iuPEve3DohU1&+O5JVBP$|T5QG& zuqz8|h`H_#6{a1EmjUyeOJ+dRyKFVPIi=l-nc0a~QaCxQT7*FA0p&K_r zaf20?YM<@KjaD3dOt_cp#uY1$@sHSjWxDah6mMvAmYw3p4N@Gm*(Y0g&anI2i}(0b zu3L)Zhdt`&IPCCsuDlj|)F$Y#Y`+S$#vROjOV`++JphIMio)dzp*IBzftwI?rV%-N zxS`uAwndw5$SFDlCZBr1*yIpxMAMuW`V*l!`(!)Dzy3EWu9P@a=HK`4apaXG2XC1R z=`yvRm7M%3Wn^!*9hpvY#$MIo=2?U-Eqvlc8Z_WS!*6x|MRS$m@Z=No_%1N#3M4oG z>ITQ>)B*5MeB(pD#W()0kg<4AA!GB7LdI&3LdI^FLgwm46K31(rE%Z7H^0T3;hsHk z&TTvU-r4qS^j(6LKT=m&DzplE-?A?zaD*|fn{}4#eKz=5U)NUkY#WR1X010#_XdP( z5A+YdT$S8f=S|tXWqZnI=>Bg*uXbsDlK+w{wmE-Lur_~i5P$p^z0NrKxtn(+ex=8e zp67XT9Tjc!zkNJPm<<+t++JpW7EGNE#c*!WN#v_fpIi((B`xFzw3x|Z2mbv8Yf$)7<&})rNm24+y!hqG?R4Zc_wMY zO&b1Nqtk$e#iR{1X{q-1=rrJEEotYQH2mU3rvX#1k=BQ_gTS}w!bZ})MV9I30boyb z{1fu@a@&1Y>6K2p(qfsTG?SKP|1!F6;P))j&M;|dc1*5gU-TN^2G=-OpLnczkM3|P zlHHj1q2bjdiym|AfqlWIe(}ME84)|9GhFTcmOm~`*5-omS5o(=2UjV+EM^8z&G0PwXDyxB-t%Xzf&ZqR&o~H|cPq}N(a*YZZz;}|pMUMf?NFTI8OioDZrpao8TmQW{<$0X2gO0R zcf=r*>E7+HMBlG==sSIh(0BBcleMlE{jUBWiw{#rks zNUMkU59GNv!owH?u*@n*KJt&;7M&YDY}O1E2}bMiUnbN7vIpf1rU)>Rt`!-v6_ zR>ot~jB^Z~mi)T9D#@kO$*rQpwI2@%zF3vgy2+agolZ4$dIxlR1^*(t_#!mAHh;+1 z(CHN`H+wUn(;27G=@oB%wW0~!5p6WI7FsKDPyZZVPn# zWO}_die7JpUN_BwRXiMGc&!U^jqjlp`!Uy+wTXB~hEKLX;oqJq74JZX zcX`L##EBM~IG1<4LEIYeRP0C{*cipXY$8s0UrN{^YXf^TN&AgS%d&?@rvaafN&B@) zOR@(>rvbxjNn5SB64CqPa{p_@b1&EOeT$s<2#~*^f@z3d(x2}_gVjqJjNSmc#O`XMEo|$uW95g zif94-+z?r3tf!yYMysD2Ec)4l_5|}*H)~TX`nwN#e$Vq1kIs)=N2s%{354P~JK#?d zo->{EBr03ctLVQT-sSI6KGdIPPg3#a=yAs;TAKzZT7#S5foX1AwK2)f`S>!r^{Y$v z`12+$EreE&WBo3ERgQnZCe8#oy6TVLIPojs z=qWc&?K=DlzeH|aPvX?allf31FdiSJw^)tNqMo(H-7X($q?~lPYv4ihzjfHe<9C<% z!-hYl8r|>V|A_x&*#B_z8~#I|{;qGGPci%_-G0YSmtSP{r=GEHxK90rhbW{!|5ONG zexwlG{Xii&w3RSgXVmVMjQCV{^+S!vmHXnZ4?v;Z49><2vjw)?t6Ku50k{ zJ#M-qw=Xkt`-5?k+l?=W`Z4eU*A^ohujX%aVCHB1i*%;aOUt%mPPr5R81bUHt6iEq z$hB*Igm}?$(XI!P3!dTsMFS!<_fF!X?JCaVKW`^)O_ls>$u>39$m-V;Cz|^ibjhK) zjvd2H@?T}rvh0te(_ET6-lS#N??xlh{_^n|7xZWkj z&Lds2QH}|*BOn|K{rxr{&+?ZgyL{opX8kxu1)lrKvnacv6J# z>MQMu?)V7tVZTvuaTI;sQDm(a#`G|KJK)VIRNha>qxGo%>%g_Z9Q0yUVp8(~lTZ0M z9|XRXKr^bbBZ<=2Zm_N#(s57sI?6iy;`C=$Bj;5Q9BbcrLq%OpHTvCZ>?1TzFS-5w zh&t-q+D(5I{|fQw5si%W7t)MwB0jr$JNNCiv3K@c;`DD@sxn(c_;AVMUnN5~deaM+ zR0M1Cb@m>8$5`Ul&JgWD$H5o~u4bV}(->``4$Z69RFVI)Jm_a++vXn{kiK_1@>5>O z6PQ9?=6bim*O@~DbH_M!{DyeRZ&|>>6P!J~NB-~jCHlj+Saq7SbbRu?PaOV3`ImJ2 zDe{Y7>wUCq@B4G&M7yN7SOJ_j%#nPM*>T2o5pip(rnWlzPT99Q`$tADXCBXRWwNEz zV{D0uI|NKSIIR3x2EWUwN8@lW`838cZn^n<4@$2svy z%?AfNXNFaeMVfH>*t`?>-CKMI^W z@Rnr%j`WJe)MoB6KRu1R;Nn2ByB~R{^KZz+-|UO;C^Rm$Vx#GgL*ugSMQ(rAP={!o z(;wFV>W`VjB>OS9FZ-Zx^OO&ot#xyV^UoI}*LksZI_%9ZY{aH^4`&hPEr_p?Z{@Mb zU(VWEvXSVUe8^)n=g#E=)Fm4r!A*6l)iXZH>UpupSD4esnk{|AG_N(g0r`DZl6B`^ zLSSrzwR%`!Tb)A_aC?eKkEHm`AW7? z_*1VY?RDhYP#^rNLpKWWO)LNE2WRy3OKuCXW*03IeH{sXb9@M3SD)zUGYg4x)}P9g zV)o`!h!ZT;-Qx6h0R3_1D#QK-H0@;C8gKGv*?HtMdKBoZ?Up^8IO!5Zb00mStmDJ< zh@p|#|Bw&*>fTdPH8FBeh59b}rj|B*6;==5{7#dWd}a0U&+j~`w$G5DWSt;7a3kYf zf{fE08HcsRGv1h8x+kWHInmr)?uHWxvDGsF(>AHIzjc^Z7a5-x`r`PHKhA$7*K2(G zQ-_n!86(Nu8lxY&|Ji(vyZ=Yr)ubICqjbZ6B=?K%zfV5hW1zCr-LfA=l^uOjS;?Qz zyJcS|Uv*5Dd+z1KO!?u~oQF8)SUxLjQZ0UnTc3i|>><2nDv zp7nrZ^R{3gYX!>Wdx&#zdEist%;ABfBygM!{YV_wlL@$)AGn&Tbuj+&Z9#)RAucqWY3XS`I$4Z>Ee{Kl7j8c0bUe ztDQcCeq9nQm|t6yij2FZ;^N>I*2m@SPd*83rL)%hi1yY{ukO2rU%T>eHgGQ=C^{3G zYTK?p;VbGCZI-Us@*3H9nQzJP2yNdW+4o8EVCT8J9N*rj)NS;-k-FuVJJo*7t^0Gz ziSC_H_qG%3UPvC~Vd~C2wQjTKkJO!M>b~Etdne_@A5N${d}7_VktdV78UJP01tS6v zk8)`FHsVFgHSUAikIvvZf-jQ~dAhc@8+)b0Yf6Hl54-xscS83-AJ=#SftgPE=P7U2 zO=jOL+n(&&+^!?Og}nrgr)2K6(3}R&^+}Gc0-s~)%cI0emRFqYQ_rIQ`xIyNpW@%n zT`xZ}`LsS)oOAc!4~c69-VXqe&e?J9i*tGD!?Yb)qaFs%9NMe0sm69;I%S1RbIB*X z@Vey|bd*c6LnoD6OW!8B-RxVvp{Y zolcxXXOrwVPbiDdm^KHQvdQ-47Y+vTJON;I7{b)=If3o&67E2 zc*2>wGKGwePlOta5c2n|Ri?2cUDUL{{xah${d`nxB}tr+awt--R3|Vkupc0T;xY4MuvLq?Yqog(}Tnty9wgO^Q!cX-m{YL>bUHs z)?0V&S)e$^-a6mObDzH9y<6SyyY-FCQQ6N)zf<4LOYQ~E_Yd?9ZfxcIu|MBw+PzhA z$m6*eI(fd!_g1|l`>G$)`7XPNZ}bue`b%C&vgLo=p<(cdn>klGSmIbpZI9`4jLJ`m1_`ce=Of zQ_5Rs)?R)N?fr>60|v#QUq9CxurCH0kabD0qp!5psdAxd?*;l zmlg;2Qg5V>s_Sw6f1c_ho%%cKq8i=`8tMeG=|A%@`{C~$)nPe?HtC#1D^LsVKl?2C*B|V9H`kvhHADeOcJgYePIdxb)tpRsH z-|Mj}lT7CQBQ%Eb^%#1QZr?WDk?pIIbE>7oIFCH&`V8(T<0s!eTYB##N0)gX`Pv6# z15bI$KIuG?QzTEIGun-v#cs|$@7B6XXLtYCyrN^>Mfc#rT}_zH*$#bY+3_aKwBt;e zVRtei_ivaG-;O3swcDqWpMN{15ZZRsgwpq%{%)Mc{4w~mb$kYX+rJ!s`;wsUJo2N1-n2ufmhl<&Wnn^3=1Bq;$c!8F~HL ziAEQ*TQK%^u_Ld=Qjg>{={uLRuk<6HBOk{5+ou>k^|CZ%3F)pSZyun$^6CBOt{&+i zab@UgB=lF8TMQmO z{odryw0D!w@zv<~w;4qrULdX2T|aGb>)S;>N1u{z4?CegN1yU5Q(uPt2lC?^fQ|8pl}_p3Fqd&hQr)E(QaI_@#cw!cF@jm=)< z2<^E|Lk@9tL)rGzv?<@NGfSLt{}ta_4^;9G)V(31o3_x~xq9gh(R%4&{PQ!wV`Tgv zp}gjCsR@(qB_`B8SQCJ~@xb6XU~w!kIR<;1(VRtc@Z1M&x|Q}$zB2}WHg`AwzrQoa zj1T&3^)m~7Ha=y-chleh#RshXzxwb$rF%?;ACh;48@F1q%?*stcJN{3v7G%&PQZ&8 z@Pd1Z!40pqaAKRgZn=_ih{B5vqUSDNEISQeEYmvnv#T7uu;j1c{*eKniyI@TOSl1@ zns^Yrc!$SV$+{2xU`_POSlP^jA1}K2(U<(nD;RjkwWU0lxYgi?;6Qkxb%BEy$9crHA793Je@fh_wy`XM}7>CVANd`IEQvIw5!9>* zMdo3<`L7gvn+cQcEhg0c>8Ib5jP|4J;tIa4PP|_UTv_)22d=2Enczw$xH3n$!W!|L z)`th(yXuAX#y?qqi4DXTIb-o1$_gftizd#& zcI0ZFAv~Ep@*Nq>FYzDjljvWSxqs6*xTzRT556%bd6 zK1sTT7VK@r>%?Dl|Bm!9Ui=2f(w{_wmsticMH4b9Cp-|2E(U)@6GUrOM(f_Qc>j-! zSDnf0=#rF{Waz@@q#d`VbM|i=R1fl^QwQg5sN)}|jtu)l>QK9FjE~c9hW$O|H+^v0 zTuc64CV!gUNWL&|BVPU&x9q>8%1$|{>}!;L)h+vh$|4V`>?>~Be@2xZds5j?DEoq2 z_D%AI*!z+XUqhSnr2kp$(5a_+Ppljvf4ye!xk~(;J#uJbG@V+7&zXhLCg0vsVd+L4 zI+bdVCm(v)WhF)Ot%j}$zJ43Bwlf~7_OrAtT6KpDGtcuay6y7x4q8?DHMD9l|1lT5 zkHE;2l-C}^6DCZzmzyxf{;3I*>}4jD--gr6^ATD8|2ExHztf;wY0#~4hHicHow~AH zX@ORSp;Zxo(gxoi?i4t0q=Roa^KWMVQ#^L-uO}YLk2Cm`3_eLNm7Z!6dB9a*o;8hY zqcE8`?Elb(Id)E$0T&JXG|%jfxw_RM23E4{b6nUOLm7v^X4%g(#|})37diYj+vLx- z2ar!NDLN-y7|M5KuW!u>rA_vdeyqR{>6_TGEe_smMxpVq&;w6inj1ODcUv_2O z6L|L!`nUvma{PuR+kd0H*0H-ym;&5?^WDDxx8Yv>O9$@Lf%|EK`)^tIiC^n3VcjFF zdkwVyll_8de|tqI&TDpRu7O9-L&p+X1Nx3#wx<@E*ID~HXF;^~^ID51mb&{4D;N*q zifjOW29Fl+Ec3cN`qs`T@aTTz6^~Zlzf-UFf7Ay2eB&qohrPFttE$Q#|IfJ>?a6sJ68IqA%(Srhk`pRq{`W~PtW z(^$@f%H-74XM(f>=}9f+s&v2awa>YDC7WqJpU=F$ukY*k`{TU!Is5tSz4lsbueJ8t zc|vmnZ6N!njWnx``zR;#E3~t#?(OH-9Y-G7U#qp;+QYcQvJGEEyuhvORYpSVVtBYy zWvBA1+~jcir)v!YcLP}4?tOQ^B+W=zpr$w zb-?{z*1DNaIdh%HdmDgVbA8L+F+tHu!T&->$CK~ezS5+VgH&3aH^Hh)@|e77ocEXK z)n)2lBCWPvQWvrhp%cRA3oQ*-Y53*0>XJMG-Q^?a)y16@N!gRq3{%D|UA*{K@Du2AU zmVCm~$eNRK>-Y}PjML1u!2W3dQPz2YPrSuy>t6B--7w)y_nbuM(gkDC46*-20 z=*0l<-)Vz?+aK89iQm%z-;~`&F9vwek}p^Mi{0qxq89^!F_T^l@ZLnb!So{0g0;EC z2k3=@W$T{cEf&2fWL$&ydV-Ii38EJTloxt2Nrgf$CNqDNm`7xeb0#7S$wLleQ$AUQ zR;c>A?d@Fu!@F};zmbCk(~1?}iB_1tE#*92Ku#$0D0D*PARFOBuZK#1)`XMa~u?r*$;Gqj?!zocqrrlqsraxFTZ>DVz-??_8u zze(aIYrZYCA$V|ueCMHC&(?OPh7C~q`iCVhn>e?b=Rx@%$9InTy@u~C_SlZbT=RRC zq$5)*vA$Qx_egZ}d9p@pXC~2~ss;maK*RWn_TVd~_pTE|;IZW4QWN8^f_sl>Hy<6QAn`m+2^ z<+EY^Jcm(tYSqlWJ51YxmFVc3!wNj6o;^X?0EkUN588XZ)NUl7_HS?lm(6+S`t>n7 z=4SNNP59nd{c5CNqzi9wJN;?|)~ZE^ziF~%gOFg^2#5^=KC2j`^U8lo`E1IgKfX!I zkJU0)q1OjrG<;A!$+MNaK@O8{+9)JoZ!y68I{D74gS&-W#+iM%vU}RZ5JT#a{!Qwh zR!TYKSCW=3X_@?MpGh96>j}z;t;Rc|N0xZ>cY(S2W%n^yOFg7ZJ;SMI%jGf#*c>n} zrky}w46u_RpZvpBj6(@FCii2TpkU>IDOXfuDs~bP1s>Bz;2fNs>;S&77YUvZk?n|}ImDn`2*uQ0-w}cu`3;YeI-XQo}*uCBk>Lp$N@k`Rgj-*;_A>b*_ z`$u5!Bl5{V#`2HKj)Xiq@dDSgyVpCchiManEcfrEHDWuW#^9D7#s^{f9y{SHGx3*=UeY`gQCCb}-nF1pjY_^b1=N z)%LnD+Ai?eS&Ln5U*s^HwB4AZV1FmR2KpsV(8@*DmjwP7JurNdZ%c~pfNtl0I{KUY zQ-={U4R|0fFm_MT&KB0sdYf)k+@~GGr=;=9VSllftc_k7fv*Kwd-dnmsj~IxwnoP> zUzLgH!+|dS$6myQA7>W~tR`OUNScQ-9%`M69eq^rG zf;_)utdg)PS;_n~KY%?Ddj&ZM@F8g-*u=Q8EvaVhepH=zt~~oru`T(SJda6v>`LxO zpRVBiYLl;{pC(RB>4p=na^ryY66{KvN!N+{fV%c#Yto1fiTcMQ{3B^%<8l@Ms1Y9u z57&$hiH~^U&rBN)>^o^&yN83YtG1ooNw1v#(s} zw}-0zg7B`vHXpm8k?=QNHXnk^u@g|XAb$G!v)S~Qf7qx({N1TA%KMlK2YT15kUQj5 z81G$2xCNZ`p!$yUu2msCqze0cYg8EHT_xdk&a_MT2zNMN;GdvJ#mlCc=hBcZD;t%6 z*59vKZ%yz%-!;F?!|%z@zV_fht6TD(f(HDS{NE+7%-YrbXtqdz!%DO42Ws=9@Lqvz)XdbYI_= zE@z|fl=68Zdk6srLxIH}z$9{qWe&~_*p%$yB(ewD6S@VyYr>72vdlfyBh0muJzztB za<-D2?-%()Cf_1|xS#ZZ{9%M8-&;jo67q?F{GkF~+?1Q++t-qtFHrn>toK^hkL;6{ zP$nRgI1FAC*@EZ-Oxc2IvvH-x)Bk|HqCajExishBPrg?L;GW#*h%VVeD12T(wlJA?gJlbjE6g#PLVS>Hq3bMQkZfUpciDp2 z7X@U|8I%|Pa)b&+wot(QOlF=YF<<%MjEVToK2Nr=zq{TT*~0(m%%IvAbdxRA(#O-c z{+2!A8DYFy2wvulg+p^^ z)!?7t!dztwdXJQi1_9-twma!GMNZyj9@>EhUKH+zo^BL`}BCp8D z4zsu3K$_Gmb}zT`j*p4*My>&BtrdeVR=@KgA!s zz|V!+YV7nEe<W#}7q5(L?F`Gmyo}dcL223EmLiN5Ka4Q-dxR02hvoo&ovI zevu6gBCp_31vBe=8rfsbI_@N1U_j)(QJk0f5?Oy8=X<_NaftqIM}Oewq|)E@_s%1) zDHrbV?KRIVe+zNV$a~Jo_@liyf?GsiE3)(eE}Bf7==4SbKhpPWC>N0NH-Zxc7n!nm zQ^udJ@+Wxn$afh0Y^Iqqn~PK$e$A}9B#(*L26(rhSC=W{kF(moO!^A%6>XIp#kVQf z;r_4l%9(PV@IX7NT&%Z;RcHYn)fk)B~I)PPIcm!Ji6mCZo$XWk2h*t-NEW zmdcn5ZapUh8t7d}8^L(={U8}_upU5YhFY6(y%^{IO1J$y`)PAb2YSCMFl9jX);yo) zTh?BsDFf=lpPHoyKt6LG{uH_TDquP=-z}7v_4&REk^ieu}sbNrL6rzGV1r*3d6_!m0-rKJbh zM!G2zi}qe=$xfe?aUj2p`&Y^yB+C0U=09+sbpZY!P%tsTdlxVv@c$U)0y0a@f}v6T zOR(H>fhxl}1!)7g{h?L%0loupdki=**N(_9%`-+fT5a7)zE*oIG@SbvhT1(}r3BG% z;P0z$_FspnKN$9c^#j53$1lkzupn~B55nBO7~syt_X*zFv>8ml<1F~Qp7;yV?|?43 zJN$`EPk6%s{m!Ah;NNT&3jO}S3wOaX$9o0ts{gqSh1{hgFc;84DO_CI%Xn|J34f9N zo4}vwl{%VwxW39(GV>k5FlZxBfERogyE}m;yNpSYF6t~W@j;UYUNG-d7Jbvl#EBm7 zE%Hj4lam8DGuB&T!H}$_U|#S|Rfahxui%*1t+F5U9newDu*#a{%sXg*XSMYg@+tjP z1b^p@lD*dj{*mu6&$=RULg>E=@k9E584pW81O8X;{tonC za3{94Wv}3adJDX66_4-(6Z>kJ;&@Df*8 zaBUDLXE9_Bfo){HQQpF#7Y2AeR(Z*@WniHE0PizF z<$puk+X>Jv_K8w5BE2!)J-Ay&(3kP4tzxh1J zQ%b+(-1Tn%SkG?%7@pA{bTm7jdNRUz@|A#3McEUG?1i(qv@P>+59MUsRo+j?dpIaB zZA;$Uth@`Q4f>$+X7)79JIKW!5%to4$vfA|d$Z)F-zx9Dvu1(2W`e_JAS1X6{mgXEubTQ9nV(J6r~ExCKFaiO;Zc3+@7*JPk1*Hb6supa zS^aumW4=hc)asY`c?hoON$TEU)$=QC>iIckOuzWd z7jSrlwEIZwx>p-G6JM4~Bu?EGPd;GoBI52;`DWt-ggD@DB5{jUJ}VAbyoI<0D&I9` zKGxbw;(kC}z-NNP>VI``|G_<hgQHGcRzqNzS8O`2M@Llo{wx%{%aR z;{Pso`M^!B_~R>3K0c$pFZ1t$#|~517X06td|I^kRpJA9rirw_o$>D!zr+>z^PGnt zJef0Db5iy%q&J=pl#Tc93M%_5X=i-?olt_Xlcteowwd64I;h+Wq)8cj z2W2J;ZI1PBY=`!q@(0m#^c=)BpY&%|;k&ds&+Rk#F5$a5+br`FN&m{9nc8PSt=<9+ zM{iL>oSV4h-`~+B_nNHayYWkZ<_V$Yz=@*Wz)dUmRCPjAu(eV&9sWwubZD8-^wCXs zHKo3Lx8iqFh3|ou*T0|Rxr4q7&HZ+|F7|%n|3~P0e{U7-%Q&u}oQ%1mlU;P(%1b{a z?~kp#cS{=qn%YIzt-P#P$y;XSy-o7c_w(txl^0wiy8RL>?_9~t|DI3Rt-Rn9*;8L_ z<-JDoGDhdqbt^C9D0F>_m3Jz6Wu6s%d9t_B_i~ed=1U#$94bDwj}dOgXG=W%g^I85 zV}w}oS;V6spV@R7dp^db|E171P4qSb3NE97agL>j*1(PE>QZ26NT0 zINjw*88&>&^VplrH2afG{X!>8l-?E}$d=3`fq0RZ3I2}(7e28~co^?V_94Hyez+e3+-C9x=A2(d|BonsAj*3-SIWhES6Susu5$6-vTo%jQtpsd?lV;m zf3a4%6MUQPC3wGYmFu8?(no=_TPXW`tL*!dmo+7PNvTyH-}~zSP6*u~O?Woxv-nSz z@s893^f-?C%(G9=tNLQS`zUMDQPTRD{`{XM&7`BmF;A~j$I}w0=qPc_*%9KNQ2B&r z636_7Ku>y}r>t;)uFw^Byh zv8^4&EKi)jFa)R)Ghx3F7f?j77l45daGMDO(yu6W24q#p3Fz~ie{J=wwakXH9C^}I?wkybq}sRz6- z_59kZXLxWuq&@a6^&FudKWVatV<}T3XY57C2TU~E34sNr>jO@LVM5`0f!!#4$Iy@N zFmakX1AY35)h9={J_*nEP1ucAF!DF6PsekZA83yBZ<$rkN#X>)cFGzcO?b9%)?>;( z_o{l>>riG}weX|#8N4rWA$+-77r?Rb+;{OGXFm=%`L;P4ZJPhzI!ol7=hnkWwI0x0 zQrB}g;IlLD>U#LKb-82vvR?XoD`y1G>Bf0qrd^3^ls;GE_ZXbeLHoaxc-GTzBm}qC zt9)_Z7bIlu@01X{@M{Ud89P*ZfA4>&Fv|O^gz$b36aHr>zMHAH9X?L{bawaSEOr=j zXQ-)T;2a=jHrupz9X_iVx%rmOJ^ zhaT@Z?vgV%b4MDZboRN+^L^pm(>N8pHchWQI3*)u)7YwfpYz1>CYK(!Z;tM4f0;JX zS*#m*EKlw|Iq)U(GXgv=GD+u|iYD=&k!aVBIrY^Wzr-%Vfv-xD=OHUD7WmSs`&~CY z9ez3mz6z*Iqpl9@8yvvLYVI!;x*+FA6ZN?EB*riNp3;t~d{^+#dDJttb7E83qf_22 z;|`9p`==82SK)2CbIrn~w{_H#ce~%eYr)cC9iK5@pVOYS^wWM$@QrFO&4{!?k^*WS-2qxUfA)IQ5CW33p#Um!_}T z=%Nq7<&(bl7jGs_qyNL`o8+ss#xk|d?rY<|!;+=0jymj3WlmBT_VAhTaN+iX?{!&> zJYz9_AWE>S{jaw=HzdLP&*dHdX1{h(xZ&adB`t7A%j0e>I`A&ABX((=Jz-yBXYYMs z`!wp>If{Ng|L`QA;PEWppR#GAQfE){mEwP!y@c|`Vh4^?yMO3sBy;EL5a4qw?K5@< zFJx17c2OSt{M?0NTlJsxYj=oNK9@PKk@{LLHNvT{4LA`S7q^~$C>y^Q+s^untJqJ7 zAKhkrCOP!V1_j4@nvvSeQS11@WYMAQdY8G+VQ(R2?CdM>eZEh8p4Xk0y97)><01zY z`M&J0#CHE)`s%pX{YabLvG26ax#pBDexKhK{mcsH>ZmQcR@Q(MSop^O?u=6RioZ&m z4n5qw>lAogZI9oVNSSEl-*fcn3zuCNt?nJ)MOmRQw}Hb2cf1kQP88!%vnKBsTNwXa zNY`ree!>4Q1TOa{uZ#gFvR|d|q7%OdU+JP(+eKbA7gvQCdGcRLtWEs4AF zU2bT0H~o`)WW>fgO6`dpVT`NI@r)y5^WPj{nlC2(>ka(pPKgIkw7ZytRA5-G$?~); zxp&E|{~|;BcOCWr)h2X|dq(NA;FL-9H*vgPUWe^W@H%x|f-J0Ml;P}$Eme|dldQ=K z%J!pNNe|w8Xt>zmJ)HM&&DWFniZxn8MZCU5JEAQrSy45CO4cI0c^-+;gD&Sji=t=92drhBu`hL-P*6MxEL^8!x} z9aTV|k~4o9W*X-oB9Qa`qNqnrW%bl_Mw3aO&=? zLc2D`QJzD-8;-Sr6I5FHa3el{W_`ke9N&Pqa!mN%JIGix+|&apc)xeBQO5W6;K7Q# zf@6Zav#X~Z+rt`81iw@g*7xriBXG|Bpv5L_$m8xx@v+Mu9{kod=Q-(FXOOLxJe1RM z7remSTDNB&{Llv)^AzUx+{~UtzW*NhhF6;h@9{Ujsdv9)YqBw*SA6Z5yc}QzIRj%h zwmO%0?snom>ye(1jCO=EyP$@d7~UgkfsbTs^w>Cabeg+3&1tjZJTT|*wh zt!5tK{qZkDp7)4D{zE+jyjA4+X;&W6*_nBoh?Dfiq)T7QRT%4CL|BYH(@DnRQPzv> ze-;uaF!3lbaVzhSvNj}M-UTim1uo`Fe1sV9)x78~!|9B#tvKAHL#=^o+xC)bok!bV!agd^ zmEW)c8P}9hZAO|aRN=*Tr2(|+E=Hxk&8HJ$bb@zCnD zH+uLS%#-u#Nj{<7&Z~Q{rY5mo@>w?%SwDHKqg>?D+>e1=QPx?_nB1&3{Pw$$y*YSl zYTXUaR$q1 z)SU8c7m+p6&s`|e_wV?oC^(Vk#qT>wMNh=)jY>IIob-96c_5GuZ zjHP@-)OPOTsdQ+&_HyrA9&~Ud-=a4}e_ebhd7SVENA1piHMa0IwKm7T3O(u>8);>F zRBZ+R9pT}M;r(iXBY4R0HE)tf{63D(ShMjEeD7}P%?j@C)Zn${4Act7;92TT;yx## zS*kzllZ|Vnf6(N>nDn=JfcZlESbu*fy_vOwUeM$NZXBZaMrp(c->athEAYL*tc;8E zrAhcfaz2y7d^+Jfmo#X2`7QA|SM=l?etCc0 z<-mWWHb%7-d3%<_HdehmkCkQ3XI;1`<2;t0Mfw*}IeES))~ER&Wm$=#TKQJ&OC&8i za-wetyoXB5NH?4rml|ongTh_1-l!+aT1WT+l6AB$;2V*3w8K%JXn;?mHSM99Rpebd z!5sV1)P2YnZ(s}I3#2V3P3EofAilym18^sgMGyC zwPRb@Cq?!s9BcUfna0xF>u31=qsj%RUZIt@f)8)MKD~Y_xT*|1TD$a8-Xj{0fd|w$ zGoCV5-S7;y?&-#P<72zP_(Ucd%10e=H;@04f1FErZoIj_M(^RvKG4IL_G!L3?w^L9 zH}0j}pCK?H_vU^<;V>#3vfY~T;ne2I`twhE*i8?x$rZG-CVmm zW32jP!MLXV`QRJ<&cXRyVD4Xq^W<)DKFES|X-n$A20JH#bsabi!1@;MZhl_t7$dMQ zG)nURi2f&Xhk_QN=<0sgjNwLVIkZuidu^O$Ip9Qrk#y##44Sk6*e!z|-Nw1`Ucfp&jjt> zxLJ{}_Dab3p8Mg9@BPQ7$cZ~PWn7+{b)eI)@Q1^qL*EHLLM{LwGcGYi&SAoXNz=Yd zT{Rbiwbw2-2L3ndD$q5q?3HtftSQDZ?czKW4llO*wB;pD$-qr-(!K$Ep_c?`sDiyG z#%t@P9{Cr6J(=eV!J9uXvf zy^h>1BfbxW*0zk$8f1UesJVUbYntyJ+AOj5IEUoj+T%3w-p-n<=&bS}-x0oGzXO~M z|KAXv%w6HSR^Db`6Yp3p?~byxjGB1(pz@0bxR(}0u8!Xe?>7KB1mm&a1>Yxmgin#R z)w%2{`Z=Bc7SZo3>HjqNx2eb*rYvkvFGS`LqG)3Kbj|Z+lqP$gV<8FdrE=Z~TMzSY zD(Gt^xHNd*E4m~0*E4?%>|TSiNw1MI+B%NQ@;af%P(sO=zQEZ-t0_=<5b`vMgoM!q7nK-v>KacTci@yVW|`OLdZ#8(>kTUh`3U*oj(M~6;c7(6DjPgCEF$tUnP zpQ`;B_vhHPQL^>|{qNhY|2^d!znvX`i zg_HISFiJ0Wdyb~a+2Qb7_d9cM3fJ}?JQ~@0JV(Fv7n)=4;AG;w&;9(MxBt%LNB{oz z@$f?*AI~K%*>Uq=kyn;pjC`ZY-KmW&9h{tc^I$D??%z6*jUmLqQqg)B^_zZi_6-yS@D|()8#}9=T{Q5?_^2}5SSxanob z?>6OKmYbFGwLdeDwRIT1dtQj!w@pI)!p@U03HgDR(5`dtEbk(8VaT{E@G*qUtT@jt z@4ed{ydNIHnzGr;4kx-Beh{)bUhJxlu+Ml6I{Jf&?)u$YkFsWDWh+CpV^zp4Q|#J~ zdD>-0NF(}{-cN3tckK`BE4034FGB;S4|DszVC}bn3qH0-df$L1>+FGYiJznOX@7#f z&*$KUQ^X&&C#4}Hle=96kC&fep5FS*Kgu7%_)u3iVHLQ))RvI8w0$rQEjD)pu(u<**kxpJR;BR+|EOc!?U&MIFKuw!~IW z2&|1l8IPc~ozHh8{99mc;IBrlZCM+_uT5c16j*D!kTuZ~+c=?)y>WAbiGz~CL-gwl z+e>+!Nq=eX!by5a+)eJ(lEFzl+DrD%666ET$c{?E@pXjr!SC}46E$c1QQ8uEUKm3t zcTV!{;$7yxkTG$^tlpTYhc8GC-MmrOY1)BFoKbYtrow9pFZ3XDA#$<;`W~Pm@?QZO zlFPf``&RbJiOZ4iLr1D8Bl9C?*!|TZ$iaGo|9e3PdP5KTKo_80%ObIj4KeotA5P`0 zJ-!Ll8BFu8v=f$GSY%VOcWi;4DLfK7O!m}Sd-LEoU&=Hb;MvkVlW%v1PHS-HX_*fA zY@tP|(5s8t>pz7*gzRLkT-FM>`yl#)nQNFkp(A-VEjnA*%U1-Ym60a$4rlVxhm%9J z>2}`JL}u`VaKpxXJ96(3!cd+lp1wT&c_Mjw@Wk>&^Mvul@bu$}bpqlUQgbol+7Z) z#TR$S9vx)G`{F{%8+kShMBJFjC(-U$A6e@%{1FpWp?o%sPp5M4Miy0jwOSG z=C0aS^vbFyi?*!XQe<28M9~L4NvpOLRTIBw<<_FDtF{)|AJ|sZOnNS9+R7)2l2>jc z+*+j7KUWm4T{4(?ZFfV16P?SCvu7`sxo{3&{;Bh;undF7KuQhfsL z+VmLr*b3Inlu-B~_DX`2=kQ%{u@pOqY9s;Z=QSp*ued(;{2JvzzUX zvRw8ja%a__xFaUO%M6LsiqFD3B7@q$7G9CZ34B<%nG3!A>k`OT(C_6!-&*2vMhS;smF*SH&!*?(lS z|0so?ZY?Mun@qf(i+>TVXZv30==*v|SmZ}`xWBlrVDZ?xto_E-veaiv#TlE@p4A&#Q&kP6HQj z9;Dq-7Frw6`cSyI0(w6mdOwvtX)83r15c5L3_XGexp3Ldz**gg5e?{;y{-0vX^Q^> z?hClLcnbb2rr@8!4UW@R=l!CMeJ5uXe9d86C%oypD+IN%*NRc|U`0 z>C|LNbUFd;>$!li zjxd!lQP+;0;BINrnF^mHYgoclhYPTW>+k&wbxFGiRT$^pufizojc350T?L;u9e%9{ zzU@l(?bAH)acaLVa@l77`z&|)ir)1S%Hj7F9?wSJ3;XsJzE0#6az0P|)t_XqBKm$g z?1-;DA6 zYfVJYhC*<(3mw$B>q_g9)iBpWM>uo9oQt0q)2Cky_YOw0r;GAN*p%H)rodYZcv% zd{;f+JEoAgP_hkGiv|0!-1}Ceub%j6FuCJ$BUNkMPKG%&Ps_F z+E>bVO$0o;$dvhRMTaJH=cFIP%M_xUJ2x(=^-uGk*`4DDzlue<#L@u>D zK778&rNYb7GuFflFD7!Sdyz|tylQorHbeFR+n}#MVC)=@cQ-;)`w14Yvx{R8#Y@$Y@%&y;@28j^mZ8!i7?g+40<8Y?(te#Ks8&z}q} zzPZA&$!$zxzl|I#&V9NPdi()+1fJkvAECE>@Zkvz7l@xAaE;hfw1OA3=km}eS$O8M zQO3qGl9zKLgwpm0i~%+e>vEefF$S^MQvF3voBBt5#=#q;{!#Od1HMY(^-|FVN?B~e z*EQwOI?h*|zHAQ5S%_TlCI@nX1v-A(v`JF6BTjC26TzZlcBK zpFRnWssxv`AXmGZ`nF)_6R7|6XzC9$_xtca>vUw6B6n1MpF76To$gK>GA)fgzsR&? z->*5fPN&Q55#G>i3x8$`bLDhto*#q9UZcML8nSz#0CE6^?&p~ z>{aw$7xq2Z_8M_u_7n6eMrWX(=|p-d@FTIeQEHO;E@?~ zXe&?n19(cY2M}8VFZ7T;uG5Og7?W5B*w5|^tfg4mt79!aMm>U0{zSU)WbhteXe%!> z;5B`%(18}dlcUFCOAzagC6C}eSqr7IuE1+QRH67cIp01;*$p7ypEjB^-qm7LYDaGr z0#6nSAJ+qZE)2d7e}~KPd4|0w%I|fpwrLHi@Tjua+zEcZhPXpd3^IcEog!n|7w)XR z4Za{;>#6pgW$+Vade4YNEuw4R*~S>gVPh9}*kwfBFu!9l{8FoZh&MX~ev348GCNWe zCi(_058pJZqoS$KW~)uq2R<{8G@-5EcK=LUN*?D>O@CG5pY6y7&GXR%y60ogbe&Q3 zfioA3Nr3m*%bc{rODXyXesdKr8{39#QN}Kwd~SH-S$54AZH;FPVJmWBp~vX(eZ8&t zJ`ykV*@}1Y-L@^qGoC!F;S=SKbiq?K_Ly3DJ9QVj*j!6Houg!~qP_c|aczw88{m7{ z&&s$84T5KB#}6Ol8m`7QR_`{hx;d`G+q5C;5?QKt!-5VC{lZwr@NMjGPf~vy?P%c0 zMtEJJf3X#@n=ZxomE8A;tY}@_Ht_Axd|#z4w)RExN}fwd7afUiM^^$p(SW&VcvI-j zItTP7Kv$xN=lj+K(UvjTH(g2{I*D-_)Dxq7Vc+Vw4`Y0Il0pg`#Lo* z%vU1zC|j6oXn7WUo$?Te`?TmMlC(zjsiu7jZKyqq_aMw5liN|9@JgWXu1u?D+pfv}&E}f0tIR zyAZ8XXATtKtuOpG+hnz#_1m)c_SD7*A1-*qj2mXSWIZF-bABHif#tIQ(#kuq5s1&q z@fB9Qt@cBO6>*!gSZ6Wwb9}p3%sqYzntKiQgolu?z9PCRp?N)r8_5pyjKV#c;4tD7 z*>_D$aG$=2XDe%pbEQ5-5BFcHY?+dGwHa z{`jD7>5iKNbhTuJkxTx!$Xf#~enipYC~rG)!L&Ha9=q>6dfYk!dK}{&NxrWB|3PHP zX8#L_S2)BNq4@1o@Tn`K2m2zo?+tKS+H%<^hPS(rHAz2{kTE%PTMs7FzodaBgXN!~ z{pPbT@rGb@L@g^NKK|n6f{f<@PLw zzLzGr8!9TW(MJy*r+17|^XQq{Fdy0@KFEZ|J!b28LH6OI(xqOSml~)$7f5~w`QxH$;-yUg)BYF5*5#IX_vtO@9e(Qbk9p&?`=FfD zO6B_$|JDLMY$WgEk*1Abb6D;|2m4GH&uMs!$FYY6u8QZ{;%oEl{W(h__UHKfB7dXS z@xnw^4%@FteDorRkU4Q^@wG```*#KEzL7jquk2}M51Tx8{@@GkV}(};?0M~KJ@Eh*J_iMnS}uljBt zZtUY5z7$^K!uKKu^QGT~-*()Xyi9-KGK_f5Ua;V_-)Rz zo3ZgL1y7tA^1`Nj%iNxOg5VUm+&}8^!iJb`uqwXR1ipR-tjamb0lW+B#aOWV93r;gT?zG@^jy<|v=Bg1n zVlbRmLW^>M(G|dIcR2N!w4cvDrd!U{8$9`Vq ze(XJ3WW3q2<`s6(?rv<#mLQ;793Bo~wDp@8R!&u~z$dZ#MFA zV65w3?a(kqPi`4tY_)s`t+ncG7resV_JK1;jui9E(Mir6?M0VchirOpuG^<%yDhms zaC5tdy4Bi^G;NBK`948?5+^oAM|f}St&NiMQm%&msnk(ul`k}H_trTqd!r2#xx)!r zp1E$%osSd$TdW6>gRKj;Kl%f(C-rKwKjo~DhI~xvcr@LJoc{F&Ny9f_rs_ZZL2%oS zaMO30`1r7~zEXHX7~AUO{tc4f;nYSwwK>nTjd}$q2Yi8i+{flox*_P}N6^FSXABB3(vX7O0 z?R&rM;o19jm?xh23eH~aL03@0U3FE2(yytsq2^6=W}G{4iVZCKhokJRMSc*0eM}kh z=G&IYncb*%V5V5)lXIAJksZeXD*}f%5w3~YyRqPyW0QlktJL|00~xXd-L@0Cg5=Ky zCLG8cTHz0J^Y)aZCQWg+}=Bv+p@6H zT=ys4Wyn15bvkODoL8bxkxtGkIcV1@HmhP6X!bSI$$2G-S7+d^&f&a{wU518XRq(h zS{-yIE*Usfd66&6nYe+RVMyQ%1OL5W&MOTosFyLb~dgd zq+?9BoQ-4O&)BSUa4vy;rqYASn9CUe)$Uf#GO{*hjGf3K+W5yedO2&;Vl4akm{6ZH zlJs7rM+$7kc>kJV!qZzSjPf?Au>U{xsaV53HG#e-o%M?xj5TGRZQ6^BOU^c-k5p%y z!snQ0o62-;R0RFKjq?IOLEp1m^gqmN_>y7urNlv-BU3rYRLU6w`A=ttbv`M>?%NBU zQvZ0)Zv4FSOJ>^n=a))Pg^d9}?+EY_t?V%z%XMXY%^qUE*!}16zk&adI^-M^XVqid z(_g~QT3;BDMU)QQR#Z6fiJ~<8Hbx*zQSAk6hk+IKjh$>Aw$I7Hp_+4Sdkd+53-*8N z8{6J|`9?OQzOjwX;alk}fn%9VS2vs3y}(7P%=z4bhIYSup_W=UICzeeQx^=@?$;J- z_oItQosXR>dV%|;-zNXxp8m>hvcCiG&1~hrw7zcNy34tTf$)Y5Waxy~5PqA#0(4i{ zr)s{@lux=?+o^L8R22Oz?RQ%iMzdcq*JH$vSbSlbGzk5S!1ur`Bb=EiqVS2o?MNu=xxBw z-^C6+5q(5nlveyW^`lcx%kIm01(_%Cjo^oJ<|wI8NKy7g)7~TykDNH#(Z_w-TiX$8zZ)4;5KhNeHHt^fQ`vCT{o`WV>61XLX7D-J&db#hcRQ1)0kKl zVdUrZGA4z(J6GNBzH;&ecjwcDSKa0A+)p@TySwukA%{pi`>k?cIcrc(=Safht8+Tn zuXJB|^-pp-cM*P^`N8e4%M^VluiSRK^uZOWJ(nXkdGsNkJ~(MB9KRS&my`A!`@&1-cTDM=spvu) z>*;zuKEg#C;FV*dkDX)J%RK1$1zyG0Kx__V4_i>Nv`M@2ihZ{Lr^1K3EADHW%R4r% zN^iRd9F@zr&=j`{#n-KzvyKI31y_F4C&a6Zz7~7egY1uIw%G@yrE&H!g8qKue_-aP z&~GUx<$D2d|55bJrpgfa_Qp_cd!?gsLZh>C!b5Rx-=7D-uUtE-epe4Il7BG7|Axkx zhm4BY^dL6of979DeC^-K`GSL2>Ah?FhQG0~udQco-`DSW*wEa*D(uf!#)(~CjQ4fD z&%XDyzU{AHwr=B4Tc6q&raiOi_wkx%sMe?U_wWm^#%m3`LbS;D@3VQ}6B``YUR?iY z(y-NQm-2ST{eAFW=f;j{%*nKTcm3KKo=v@2-&b9bwMRJ1V^JdJHJ@?;18r^T_$50S!HT-$?iScg{hlH5Z(i zsP}5GBotgM`BHPzOn%~+iEkasrsA7ted#hY! zqqO4F_JL`?6+Y`?=ofrp%WQa#$(nDk>;WVWTNt5b$Vl3^qWf@$%DdjK&Zb3*95Jq} z1zyt`Bk!?g!go3c$os&uWO>hW`;Oi%Wy0IHicVw#eCr})hVYBnXcrfRX$Rh*9ar+@ z*f7L+(}6L8Gra#V!)ljm#c9cV#+C+-w4fDB{Ye+aCnC(=1S2{_=a)J{gxoUVGQ4mY3KNck$el^Ah!Hwo#{rnb#B0k zEY?>YZXErq*02Enp@n;s8u^DKm!f~1XWA8%LZbtA1#+IrChHNJCbn!z2`eTfC9asz z{BhW#q$u8FS4{Z)R&0m4D^kY8znD9aWKXe-kTWZ7o3sW`p8bHA{i@i-q%KKSHob{@ zd^MYv-AzrS>*%>xEeaUV074Y`8dTo3Gg7kwVi)EKzysj;ZOR^P|bIm zy>LgK<~uoAYw#}$_4%jsj0p9qv9dEhC#ohS9p-7`d4=bBo-pE15q`|`SDxSVyvP$l z{29V_o_Bfn@%*-GLi3`0Us9hH6HYKkt<~7+R^@dhF1LF=SKp~!-|gyqsL@t^i6QN4 zdiD6$>WhseLV0gyuJyXgne)h};9|ybBhHB&$>nb7Z(!pBer?^2ZFzN=Sw5q3f>!m= zcz+qEgQ|63QuV6wN!6>z`#16ZynL@6pOjrWA!%ym1b-88?L2myFKISsX=dg4{5SAD zOZUA-d(E@?o~`>{miHUXwB4$|;f8-6`R4H4sD;X{#)LEtNUuD%q!&I z&GY*}ndy}2)3H+W{h54!)_qm--o!g&cAu1am;8U_dEY8yfTs-2cemvGn0z1WzB}c; zop;8#T*{mv|K~iXtTG%4ZU-#WA@Ah0ZC$~tM2%FCz#?!~9bSJI$4alDR zW5f0sxC)&p_HeP_g0Yb`@%JFpXx)vh4Lo}SJbDt`**1c+Yn+J__)kPuia&;8N75EK zJJa5Zt$XWV-3@!uT|dr!Z{oi`Q}70BSJuCS_XW#RJEmrEmY8)sH3NKS;ma-H<2S(r zYrq2wzynjj18GZ2>*wh4W!SP6FIcvqBMn)!)B#QGIxF8kNAtAEUhUbQ#@oZqyG81% zXLmsR*14e1E=4~a?Kz7?CNaPZ{pOCi3)^9eKBPT;ui6j2(BvN_@{Uhg>mn!l73Y(j ztf!BFwPyakC699h&2%WhZFNOUoJRY`jAj{Q^y(PJgPtN zW;KELsyQ90ybIlYojGc)PBXOmKki7FKe}VLd{Z`-{K5i-C0Nkn|T+ zI{3~?{}tayDPO=@(Dv$~Mk!@l!F^7`0w?_h_oea<9}_L}QmW{kQ_;IrRTp?J*`#@L zrcdnXoEE*4`If%M;Zs9!u7j|xIu#vV|JtLpHHEO9w#o=Up`EqzJ<{fBOUI{disoqs zMp}Tm4}b~ryL1Ft`H(UGh;jUwf7#Ah2H-{3OiQ)dFB|=8ADPp6bWBbsIC_78AFz0V z{f=|Izzz*w0N=I&UO=4>&g8psfZ_$z`QXd=4#*I~#g4bz8Nya{k0qRy4d`qFI?x>A zE~MX)^RtRy%o${)vc6;ury@@p!ai2uJ#dybDOBs^0^7_4yW96}f8auJ3G`LqFDbHM zsbds2Qi|5Ni~-E&J2B*8eB*e(mUn@d3G`jUS>P@;R*ZW*>9UqbtF!^$X{3ovTjo7V zrQzdHrNw#kNfRFXR6lL!yFBQDy_JMQ6Qxao{{r?Ot@F_D!aualMkfp|%i}(oHs(s< zHS{lSDbqX-wmx&vL;jdowSsmD!_ed$U*i99lkQ!hdY~lztSz%RB4?Y73p%8o{TmC0^*iyQ*ruyK2Sw zqpayEOMcX`hJ3Z;agkn0+EMyFmH0}@LtoLuc~VJFA}xtHwPxP)cfsZj^!ZQXbJz^VrF-7;7yk$x0j-)j2AnUU=xGZqTGO@vYV2jS zST+c=>AS#I^O+o<{Fl(&S_z@I@?Xt|b9@g-2>rfaLTGojg!^)QRT9DrR7eOPuw26D zb9~Drgs)g4Aw0$15^l=z-6bJB%3=xOPs$~%&hagh5PoN&gz!3L5|$x{mJr_QRte#o zen4pQNp9cG5(h6C(eO_OjcvSbM-=lcpI%**#pk}x~pH&McI`Mz8U$K?Cm5{}IGeNRI8 z+wl^X<@>IX@Ya0aSPAFl`$kJRC*PMT;SKq|%O#wh@4HOGS^2(836Vd<-pg5Vr|6#H zJ7m8i`1>q4zjf#=*GQ)p>92_Nq~ha1;Z$@%smJ|0g?@2n@t~{~wHEcz?V29KW9PAL zKMhXSbbb5DYKL)F)~c>;H|goJdVfVF~trbEeIaAI1vM_ju zbfN1B?JbR$6R;9wc2lho-0 zm!AgDwlap&_g31rt1!(7QQ=S{lu+pUM~vA~85_pg%o}Rub*a2RSavR7Gx=8V{R-oL z5n+BHjq!L#>KFkY*5jUu!!CR={#ng`ifZ4(xz3Jx#4rTpIn8Ny(Fp>2yVFK%4!dTY7gi)-2 z38%CEC7jComvA!cU&1`rzl7PWe+kF2{v{m4`j>EIj&B~JtcerQ6`{wDzlDvy7hZKc z_btl0k@cW>!cc8g>qBl|bFG_m19DIHXd?|;AFi8c2hw!SmyE0}nX?7S$k|+?-`RSx zaWoH}`^vOh$&1ej<)>44gXEl?&J*xbC!yb`;HAV5K~j$1*#W!+;6(X%Smo|`nMc;p zW6WhU@3sfb?}zzL;yqWsnd=An24;`R{{B7w;a>hhe2|A*I6(A$;jC+klYc6O&r$M| z=H5no8g;{y2JkyNU0?G<@Gsr%z#_1-TI}|nfS*(QjGy%}{Z5Gv;~t0c4)ts?ZMM~( zB$j$k&Z92&17-6$?{=Nu4jiv!7zTPB2y-tRo-e zcABv|!+LselDmG}(tGPyF1@!ya7pNrdpkbfR^;hH*vfd#=Lu(w(s`!wEads*MVsg7 zuW0XS&C|-*AyZ-hQs5DM_1P)zym-UEvu)c%Ptua{9U}jdJ;@%s9vh>;B?5mzaCFk*2Q)CC?GwQOY(V z=UPqK)$IS?pll)gKNtCA4<>Wh`l^4F@FI1u`ZI0JvzEgHzKelYoy z89O=eE^k3I2EA->0@trY@? z$one^ql7P_oCDm%emGO!NAWIr_de3)ywPI9-Gepdt9Ctdu^;mY9_yd{R=ngt#+pdf z`Y+gutPWcQ}SfR8p|0?m)ud#&UV^G>X z!@H!7x6-6f(&v}>mOc+xVYFA)qvX5X$~TlSP{%%u8#8>|xFI5y&HTO1_EYlRR>^BMv$#FxVO< z^!y%ZUzB$(b)ye4-nYfICxOePF63~~;n?;@zNMazfTI)8A9+8{yY%e~6-IeKBb0mW zU5u|QNalf!N^$qIxn~z$*>x_Lvm3~&LJpk1wKwNI`f%Q(FXuh*v9hcm_VwZ?3>)5V zb9TsTGmL*~#Lp(Y{TEtDg0X>kH{Xh{uN!DQ!gr(iA*8(a{T!c{Z_%Y;r@^_zRO1w| zCI9{v|1SGu`5&SG@Ehu^o*(>kI8QgkH+2Z{0uLwQO$4v*qOPmqIn-R?TecbBvQpQt zi2opu`I#s8NFk4B>_4RbM&?L(mewe@FD@`g-SWLlKFL$U9Kq9@bJR!XXbE!^<4qww zi#hV!2AFe1UFICcS#uQST~7NlM`H7u%p6M}9w1K2*HT`>EcM?p-j&4nBaa&2;U?_J zzs<1VZ3XGV14^A|RUOKvcSrKIgA|Pzq~hYdi&PlnU8q86jtZl_^Hmt-y_N7nupu^7 z(x1*{Gnw-l%>7lsz;w?36yYy7#F*^TqW_c%FGU|T@C&qaXAX~(`Ej9VPs4sC{jGeT z>z#Z#6IITs;L}N3;c2P2bJ=eE%&GGkJG57a8GY%4@QcmpB087tsS{mW;$@~U!bFEw zo+I)j_JfX>t}xmNT@$V_Bs~>pRsM7JWLfoOVS7p$`lIT=9zoT?x3*{XM(H!- za8UJ~xJL9*(#IA;;gKCV8K&()hPyu9@!-se2e;pWe!Oh;_O&yePkQcf>xE0j&bJ~& zKNha7o9SLR<#c!@HWS*E(hT?}0l>H_%4F zC%+e;{Lk3J@wsdIDrryGZ*FET&abU8ySXYLgr=gHS+VZ&8kMV}p82Ut4B7-d^->KLQjw~F2k z`JtQ{Isx1Ye;~dP0`=~vURmGW`xStFWjDi`7kuWZx*Y%N`0ds4t@4)K>#(OTCRK~> z{tq&ifiX1ONj07~`@h4PZ~3{-@8R18uPgOB3$3`q9=@G{xCvI=gdX^&kTFO#o-+G7 zwR>N?w}0U=Of?=3^kKMF_wXLRH!ha5;t580I5x!ck6Sa0h>Nyh+lc>}WZjqo4+#w| z9+u;7Sc`AiVc^}(djEE@vlxaRce5VTz6aVij4|HKJqyCa59=lG5$(FXN6LHOV_VVF z4U6GD*?laB_sl^;1ERf|^r-}!$2+lkyp;Em%#AtL8Af2PM0f0hzeL~JH5NCUWATl7 z3tls`7`MeAq$(OR%ldammYgZj%D-d{%e-Hu>WQYF9PsDM3a53<`5k!le4oX^bIN&^ zFAh=fE6AY57Ai15S6FTME&r@e+A74xAvPR*D#L->GV`o3O`Nwl}O)wbinFop9f9{b^OfrBBDQ~_9Ezig#~*N zz-omBd(qf02+eN&`ue6;_)-OLu1V@&+xS<(RnYU|ztXQ$AK+7iezoOd_rSjM)DT_S z8C3!Q)MMe1RO3JS4}quUDinD7=3Jb-27jm6;FJoDcn3d)_~lpc|0VA$xRWZ%`y(ko zn0@MC^nbj|o}e{XYf$u>d161_FcrCQ`fJhk>1}Rb&&n+`IgeYL-lqA|eWpGx-KY74 zmPg0b#J>$6b9T{h;0MrU%&`I2cHle&-jMq!;LH4ZVF%=%OR+iefw#opp1?^FZ9fLz zC%lZTOX0<*6DK;w#l*WMEkRR!oTMifjB1|;4Lyr5vl{Mgbavj?)W#hmarlZb=lYy| zw(w^T?6aNOl{0Su4wl(7497y(ro+&zc|Fm0&(#gRDiofPJ94kWK11aC_>+|UUHR@M zA26clnECD`A3WbT^L3CfhkSF*d<)2z#JW{;j_nR~Hj}>090@!Kos|EReWZlV37qql z`CwkcR2b`JywE$I^V!}phkHtI&B0I3>uS9*Py1c)5UI=)XV1HMh(vgZ8h8kY;vou_ zrdo7<5A?|u!~+!YT}uB{J-}p|`k&$wDLXZHhv?1q-~D!66)P;vz3cK>DJz+>{N3izk>Hlu1hc`FE!mAUVT)qK9$VXhAQfV89#txt1`))~)Trbp4g-B2-bQw{xWZ4dFB`8?F) zo88ORJPz5||Do+oz@sX%#_!wRSvmehPdoKvT&PSu6--_HLSXZ0BQAI|>>Z}S38oqwq=dxkS7Pvd@%d#Zm!6#7N~ zoOC)X5_S&s$~e1JT@v)`6pYpT14Vy5k2`N7iz%LOrg2i=6A_xPA%=H5Z|UF!_m624 z^UhY5wLSCR)b8H*ZLy72w>X`ZwzxCz9qs9Tzpj_}{iD6TlFqVc%YeS2+JiOVjAhOf z{VU6o&Zs%PC&&|)F8i1Zm&HQ=rMzUydk7p|CUYH}lY4Y>#`3% z&L9G@)vVjO(}13q`LF8A{0AzUkidJE_XiLbs+3h;w0Q3HjuDw=Y`!x*oS6V18wuCok#e)X1E`ZW!?EfS)0a!DBjPc zZ#B!@{!QiJz?{L+Uw0*gmvEmEx-!~%tW zf;YG` z6@YVsqhj01$(w)^z|+;pF31OUL%V{HoZYi`UG@vSJae6amt&}(=q~OgzZ>YUz1yXa z&*5bs!u)tCxGU%SJ5a~=CQcdWYU1^Qgk52V`SENg|9#L9!Lw=`IK~)OmzDP4I|;py z%_wJ)ntO?QXX^25aqMmLn`G6;iFy*SFXfAlBwmHZ>ahyO=xr5rAgf*2UcKK=eRIBz zF%N~$+mWHdkfFknp(2o>TA?4Y@t&74=3?LOv?=d4nR3*b4ub2kdXs{2dLvNoE{)|J zMNix}kIfSKXh)!(d+Bq(d~`c;%LUhqm3*}Rm-119-ph@S8Xj-Anevf8-7n;$1pO+~ zRSMrfKix0nqXfM@=?cU?7IJ}hUuVx^>YIi@yYs@&_2(eE(v zZX$d9EadcO`^Lo0AM#bB7^o zW}7mCK}YXmJbq`Uv)@mgk`au!dBhbFr+CvGmV}c&6@HRoepKqiu#2%m(UN83tWKKn$ZC)yUC;8tc zeGWZD@1yR)2}E98|38sC<=k|j+_@HBxDb7m8uuPodEzLe;8^oU^&cCU(fk)Z(-mf! zQja`U*Erp!V61+ff-!o31yh(;BbX<{nJ>ebH`gu0XyJ~!{Zv-Xb9Qh3?g#LMX^p64Dh@Unx{0o)O}=x6-0eo2)u>~`H`2*xfjfM6?gTyIGJ;vmi*-Ijq(2YQ)ZZ-hq&msQjt^GgQ>}2T+8P5 zSMwB^yE>wDz!}!3=jLU+ZqQG9=INSzYjs~aMSD6*VTUJ55(HRQA4}sqs_D#i3mE03)FzNrJ4i)##J>8nY=TuHeai1RX=-|@Y2&GwW{CKKUZs8emasL)c7=EKNbdi4|j&&Pt&SS zmuOW-@5_vjzV&WP>ihlnqYyH*jSH>116tmwW~@ zF4E6x?G$6+N>?>ja%NJ-FHzC36cv`B_b|%{ln>%m9Q%JQap;*$Jc(3svHF$7$@p5J z!>1YBCg!Y?9~Nlq_CkC2WeELE(&sArlcZlkdTf?ll>X+l$fim9%jP$&&GI9FhHS$= zt_n+ro{eGbM}q^Sz=a#ZiILDV_E~Jfik_XXYh&MTgV?a#)?S^RL$xcm&d%u>g?9bg z{f2OK1jE=Tu$y{=9OzPfO zr=8a(v4?Rf+j9-fmnL(*oHk`-nJ<1Bc|GL_UcRnioc@}E*w<4KnF1(h^{R4P_@Z9T6+ z^r;tEiv{9)t@+I;)vrnVNT8h25S_d9ukfN_DlT3hs$iTxSix9*pn@@ao`Tq(xrm(< zzwN%u*@zaLI0&y68Y(z(1{xZ8wxWtP_a^M$1rzji1rzmD1)2Xq zSzj&YJjO$~n{d5xi8vo;h9;XhQ-~}y*~FO$@eem~1DR;Di6_IwZ{kuNI%z{j6MM5M z3J3Nx&hnn~Pj7I>4E^0&*0FyCufiDTnogV{S2CU4B~2!MyuP3I9H5U6fmgS`)x4DT z4>~d}R_ZG_Qm1?_&Cj`K4?>FrR#O`A^$EV=q&9TNV0Q zj52Jn)25+7k*`+L_abv`<9j(Tpz)pT-&dI79}s?0!pWzAaG7(G{vEsxYOyia(?LwG#hSe`iVYE^ntk-Y{+MXzxAUoV8_IH*Z;zk^TsC zTHYu_c2VEl#D8R%>axdi^ZQF!`qnK2_LMC!{8d{9Smr!#`0K?#zrQwTLH|sxk~{wq z3ddUaGx+me^o`K4H2QEZZ4sQh9Uix8pLM7BZ4m}78$y`%nOwe2Qarl5;)4!-EA)YN zxY92Q9UMqMT}7N4M{H75nennuD1A5xDD+J75xy?8E=|(Y@1>+)M|zt}Q+s+MPa5gs z^?9U=4@kFK(v_O&kX8NZ){w51q+{$PzZcE&WzL){H$nf1bXL+Cx(eEfJhGp#)`Ur3 ze!BZ6|EDw$+NH*~SZM29ksVv;Y(Ks^-O1uTr;kX|?*kVvMO&%YSSjOK(#iMY=OHD^ zWi6{D>vYt$323}G-+TRJTcRxf#u>iXzi!jL7Q zBX{xL6K0ryJO?nInefpx_^6sUC$|g`KC1ja=HkFTMKaf9ozODQp``;FS;Gsi?8_Hk z>d=ofZ!R{*av#_QACPoOjIq=|9{iJjl`($I?DIzzjMHZ-7^_cLFh+k^!9?iQFzD6w z(5s=)t0B;5nB?f#aqK-&Eg{{=Tk z1J9@39}?!L-6GdX{H193o08t7-Phs280~%~AYBhhcM;nC2T5nr?kxOL{)K4wV&cxH z-7gUqNW1fx`~OqgeHMJX*t{0G*iXANz*#@-ev)yJdA(4Ei4ON;;VJNMNj5<^)T?M*$HD>q0d8B?fqw5$J-Ld zSS<Db4|Cye=sJ8J%P#`-;0|1IeXIGdDB9nP&0 zIP+`PKS#kBeSm_pCb;+>`Q_G#vWF6=PYK*Z$yE8H3oy&=0F-`d%+adeMhDHI@eRMj zP3PS8vLMb$i{7jN|udLC>CL!;o4axUaB3G;iPsE=y z#O4cEWyI(QfkF?XfWqT6ba){pCA$osX6$>&zP8*Q?yTsVL-zosuWp7v_@U9KvStvS zh}>q|*axW*5y_^?X z_Hu^W66LMWQORN7sk2V=`xx})-|dWjo%ZDY4fwwIxbVJ2y^Q(>A;XExE_IeLL&s%2y|5hncF*w=Z5u+#9+G}*^N4=y^HYTKOt=kzg&dOOu8iFy^~%0AA;=3#1!J|#8@x2{c1Bu=y zk3EaYoKv#{8=2D$0XTYsIW4l7?D=FN4^_9)dVIiIXivWJwy-Ak2_5u5enSM@U8Du2 zdb;twM!(73Si9IpXC4YY+=G0ymAF!oxng+tL+;9>M@_R}4~}o0clW@CiB|8lEL&D8 zHs7sv{f^u^xInrAh3L+7Qi zs&w=g!leD$?+5#up&_ar1y_6am&qP*oL&RH6?w7|U5d~EFZuRDcO(AQ_|><(2h@!2 zXx}7_^J4M(7kneMw2}EI>6VkOif@d5NP=#0--L2|)AokVuIwi48J(sMv-b7zJc@iG z^XeJCd1O*n_T))M`%9)~WzSp@UiIjhU|#|ERcG81>?<6($ zn80{S9o#1W;LQ?Mx0wQQ)y%a$;7I!94EiNOcMyjD=c||t!bG~Ex%I=Ont1=vYgbuCE z=@F*b0usL_ZQI*1rLIrgxAUY54rK$ z*nZ550`5{a{xis+Z>yQBhpgV_W{X$N%hmq5arAG!1uNDW>w!uH=H&7W8%VTaG{v|WF7ngZ3Pddul0M#MG8_`!71=(39zJnKL7Zmwq1t`~=5*@~&iWF5TW%(V;`h z>YU33XqiI~H}mMncXFno7jR=>9+gHOyU9c9lBv=qL(7u&j`)S{W+<4brz_~tQx%NY zuTU^fPgXEiPf}3MB3%58&(Fz1(=N~3g;rH(1h>$szfeD!tMunFN7r+Fy)uxmN1x;C z(0ZfqSJL;qTOebjS#FwH$^O2drz^S4&(jO8@w^M}N`D`ttn+#Ljb=aA5-0R44W9ly zG*j{PRb4#agbR&|)&EJn1{ zOyXVu`uX*9_yhTMJUm6}btmOZpD!j}>hUzNwZbElU$2onC7g>V{Cs}x&tsIC$6V3~ zzn%@`JOXtv^Vr~2d0e5w9Qst1&fwSgsjvk7UIi2N$qL5lzf~|+ze~Xw{Z0ih&98?D zZkJtpPgZOfijLx>d2Z2fW8TMlM(X}P@>husnO5Xk`!$|O%5}kySr=XOjRgPRpZcbM zipRn?HRL7VOve6~%o%%bakiZ|zh$jd&410+)srjd(NA>oln11j^VI+1-K$mTqvir9 zc zSnKAXQ?a+=Ob++q<-UhS{-+5IbQG?XbxQ~FCl%(3KU4hJsFeG8SBW1RmE!Lr{`kUl z@%IpaT;W#czP%6rbjlZhe+jn~i%f6Va}hD=kj(i~38S7i*8Zo#!%A!&wuF&R5$_?Np>t}6;mtYG<_(dzyD_#N*Dae4AWQ6#F2$Q&AV20UFS?~F2 zzwlM*PaE-4_RmS$lG*ATY;lN;v|rwoiq~Hv+|NH`>;m)HYUc41X@mwJ0p1sw$G2Tn z9=|7CRWMH9tzfMFwSv5ptKg;i$Gb_tX1!pw5+z*jp@a~_s8VE%2JYOalN`p5lU#$DDRp_K6fV=n7HFJr!z zF=tOb`vl`F>pw5+z_qdlWc}v_2iAfEvi|de2W!EDcTF649~`Jp6dXv_e*(%G3R(Z< zlfTPY|8eKsSpR)4>ps?hViS9!tn*j{I?Xj;?IHgfFu8D!tozg&&{3E!>%Jse_azl> zHP(H+qkeAPmmaY0+p5-marzGGA?rRrJu-Ay4*d_%9+^L15-$JkL!rHp? zkL>Yno?Y9(%TpH9D)FzTT3o^{_Awt9Vl&AGy8Yg)IXF zPF%iaz@)aCXN@&JVxAU@mA{f*b6*LKI%tdfVgc>j*UIibR8rdZCep5XcFTY`^EUdvfOJLw<2^^r{_wv@s%r7Ik_chaU~3oIHNE%h4m zkL-6^(f_BEt{5E{Cwl_7XqJeE@GXu9G_4nBCcYX^UdO3N$sDctPGlqYiN4AstPvXS zv$fWH@TN-~w$b9-t#>a?S>tY;f580@+%nwzxHIz)y8n%P3AY(nGyjnLbKH}-S8@0C zI#G0guaiYjCH>xgf3H(TMZFq|)+Igdek$n(_pDy0i)Qv}DjJtu;hxdUS2Vp>V^N>v z7u>fd-^%}H;(ji=E%`f_C0rOl1F*OBLXvOl+>U8Q?tyOr*5wWRVFtw|9N)0RhQ({9I7_wKBTkvFG2 z9NCwB&E5C*J+ox~KJwq^&W_Qd6JxYDSF-2QXp7RD$y?*yz89^D5vjI>x~A?69N&%0 zo-THe7_;aV+^dwak$3uJzAU8w>%hlWjK%%n-Rt1pmzLP!!@#?v;2HeCXFJk}J@`WQ zcUEve&BYz%MKe;z6pwpgQ#H73O_mp#^~} z)Ng_ot#HK#t^~$OedJA?z%tU#o1Dp%xzg{}hc``Nul}FR zm43{Xar8NTuja}GTY_qTGGztEN&97vFdnFvdAHb{A2Mem`7iHDH-IOb2KOqrglg+7 zq2SkCaB432#J7S^TX^$Na7oe?A(uU`C6+G*2O1wM*;RROpEKCyjpqF1n{!CtVB$#{ zxcIy^A!4Z|;ft5Y^_spLTGSAt)vqMYHpZhj_#yJofp~-7U+f&Qyti1_IS)FWd;igX zf4)!b%QQ~-ko^h6Mn|S*9UgA6Mbz4?Sv59G)?M6nL9QvD%os<|#u3;{xBDZxb;lbMxA1;_StM% zLe~nxQB4#35D|5*_S)D-w21OPS_JbTs4%bVEi)SzKkrDaOr1Ufn?sf0<0dn1u0L)I zabj}_`zPO;ajX1s)x=@vh%>5V8_l?F{y0sWF#%gh8gbLDMtOVvaY4jkFDVV1m@k-d z)^289?8GHv+nur7WX828u7P?cw9;n$g)xwP+jEvrY)Xuvk6O1e+WsT$mNu`mgq2TX zjV8*}fCtwuXF#~Kn)2V-l{@VvU&K#c~q>ap_U7hA}c9(u! zM4O_6EF(iq9Qx7{Rjz4K*iYhp5$sdn#dtqUzf34K_?FyX&Lr&tXx>3vD}6D%>O{&b z?znjy-0}0?b}ya3(mfsb;`~+anYfDitKE;{cK3R#Xd~|Lj+@-^z27Qoz`xsZi~AP* zedNF6R`(0Q8Mwsc+u#LV+`DlNxWv>h?v1zRtb4a$?rYQLzX@M?)BOo|(k4p(pdS+) zlDZtgg~LyFQQv9p*0_VMA>|=KA?2~uYYF8ofsZU{*D>waX$5zM=4aC#-&{0_`LQ1P@qX}rD!6@=J@~1n3>i%ye`>J|Ure7*vqla-0S`M#pZ21^w=LT; zK;ZV5w#a;qdLBCdF>;rLeS*w*kUO%HLGq>@`kUNNvM~2)f5E=3-fyj-!fJ>Mk zlD;=%7{}a`b>bfAxXc$PeU@m8EgIEZFZ1uH#pG1_;YU_ zxw4ghD{sCf&3o7FnD>r5Ip@8il$@%fqy<$)js@=(_3iOqQFf22q8C%M-7{1Bx%)%c=CVX$uj2Iy!)LLlvFN!dR)XON>=6kt&&x5MkXsU=e+1| zMOHlxE}a3V9)aH73cd5y<*pmQ;F!BV?(FlA%n{Qe<6qd8@Y9P@*mu@l5gk#kNiJU^Z~NzzkB^q^dIEb50hVVFHioBdw#p$ zy0073uY5tfd)yW6X1SNQd({0&s0V(DrZXg9_db{+A^sLfZ4Gw2Qa#KFORIIX~sMkuM>uURikEnFeI{ z#03Z3%aK)QnX>9>C4-RvkJuEpBuA9k9CdrUKj&yazQ3$5JTgWl-096_jP8NnJPCf) zF>iig-YjC?$efUQb1`0^<_r2|nJ+T0_NsZ+rgJ^>D)q5~pSpysaCpe$apu(l^wW=% zZiv<HCflX^Lrcl~m<4>*P#x0f+Y znfIRiih1w5?^v+XU4k3BV3m6cZrFm=?n2y$y{n3r<30eV&I12~`!M+q_jNt1ic)d^ zNuJ;?0S?7ACl7TmPwnOY5ceNkGcM)foOSIUoBP_V1)JRCaCs(=+Kfz5$zBaSS=FNn z{eNF@X&iJ<>NhU+efN5+wfxRdYk4wtoJF~_I!}z8m3n35czD&g1@F4sAioN)I%eWd z8gqV|wN1nXEv3*=w!$%UR+s5*utP319lK%*ce*gYQ-a!*-)ISoXczv8V*zru^w~B` zn}|E;qkjaqh5W)-7Lf(me||4##*ze+XGxD?JJzKy zIF^lX-!fj=1H5zuJass{br|QHuJ_hv429$G8(d2K{CaoZOdbvN^Jv3TF!Mh&JeX%7a_radPU%M~Z8&;B3QTs{=XPQF2)nyC02S)hnFvA|qYV0*-qURYi z#P{d8bM}TVeO~yS?Uxj65)HMbD4U&*uzxB0L^H28&UWskzQU95B=0X+s~P9>xeps{ zoELtFFyqYpFV733)1uuHcJ91z1z|&JM-6+Ne%mkT|NhJNOEhheZ(pNq^UUyN?F;oV zzkv^^ZVg=>y;3#n6MsQ&yxZq}tZvYoYwAfbWOepf04V(x9 zH-eGJE`DD47<#~R%@*NB?yF%fdKg}^#-a5PS@wu6DC-UQiW_+>4ZgC%X`|o~^}uK03sLZi^WQD{eEtVTVg0>D_Wo;& zno~2O|4#SM@QFLxJ?8$IHC;Qt&tC8|H0NjN!O!sjpOG;(x|c85=$?d(p3WNkC}n+* z?B;@w=i8!&=ddQ-n|?>r$;FEuzngXK^rN(kwe6@0mMCS<^qJWa(DJDAzabm^9QKLh zPH4HbP4tSpDQ|@-tI6KH$ZGp+ZSsydj`k-6%)wz(>4k4=@LRLG3tab=l?HID! zF=Vx4$ZE%s)#{Pe>XFszk=5#ztoAXo&4HxHm8|v&vYNSipkc{%d$ZX@0)ow>td)c9_d;h^-%4(CK<07jaonz2(bk22Kjy8{#^K{bY zlknG#(C@pU-|y1)P4vN0^jJ2lEkfG!39{PDiL?Xy{dmgVk-vj}KVH}qS#2L$_%4ySkIz7dxICpWz>)=*^6g$ZEfVey3tL-;T~AoN>5{ z@w*!R%{Az6u0?;7f&L~M{Y@crCJp_~UgnI*L88l$b$?fD1p1rsx&rnNJ80p^JVAwn zx~y5V+7|r_Yv=AAm{yXWOM-Sm&mt-^M?_GDQ z=>6*&lrH=I>y8(FaNP-|yS{z?rlQ}>uPVBt|A$2>{XZz0mD=AuE_Hx=ChIH*I_#Ou z-`&iQCAew023+ETH<{0GDjkj32bOi8rpCu!H<3P)vyrW7ql~M&_=>jIXpzIAFFW?o zUf}w@v{(M`qrC$6(_VoGXfLw#Nbw({%~7;jXkavL75#G;t#w3ObOiOZcQ5Tdj-Ex@ ztK{bY2kreXWv07th_v?@?L9_&kI~*^E$tP3g0#0DS|#P@SfUH_(r#?pxA+-{ZF+mr zZ+p6i2#vBT8inqFeu*gm5*@;3(IJ2%m%7tX*Le`UYasGWe#`xa)Hbeq&iai-S3O#E z4uN({ka^b!+9H$vD>vfoxD-)fxm+1VBz&9{N!7V^0^n)?>iE6`3#vrXPM zh=4!0f={=GUq_<~}U?JL3FlZ9HQ$pB3l>C&{*^r;+@V%UWv6A1c zSmdXD#ytodcOc}x)o%%t{N*fy@RknLanyNX0|^_U!Z`m&equ9o3u`;MbLB%;_~o6e zIDM=1FL~ZhT6s6)GdU}F0h=J3xdT@KE=bz_<~JkE_6`H~pj^?>4^d%a*LWoT?`9ls zU_3@3FAc~3O^_ic+pLU}jqo7igGm>{IB`E_yTudc;ye_xhMb3DZ+=nI^#gt1QKxY+ zT|Ku^zoBvuUCt*{m!Wd^oVDg0Rj(wyGiA?b{CP*S5xOZV)Xfpo5XRKMVBMJ@lw{8z#CM5sPjuZ(o=#t zdz7RX5iWbI=3f8$z&t!=9@|JG=a)VN_Wwm5Au5mG5H4o}-T;d3PRaZ#44Xd+V)IAA zIQ@?b#_DSn#QukZa?kI-ZJvm|lZ9=Jv#&?{S)hSde2zILh6v zSz*}1wtD2;>v5Bw$b1$29GX#>?Wc)hoFz@;?4pcWHn^CLT}C-)YT^9eXMXy2x3eNK z$>ozZi~SGW!y^yp&TBkbSG%0fFv0PJI#>ElGYc$ZXQo-k%q&em_iY&8w&&Ybe5<~* zWO*`M8)qbg>AS!(G}5IeX}*c{L11|5A2i>DjLzB6;T@Bf@SZa7$y%_9D|b#`<$gK) zjXeU>OFl*99h0KpgQ>E56}lCLs`PgSr4fFpaR(sVjrjk zxyglWw!)$n-)~vrxPg7366B>y??&R1 zkelV)Kui9S#PeQAH8|FXxZBQ;Z%h0r;#uQNC+@}bhD1zl=K?8FwS@OV(2>(TmAiz4ryF6sR-23Gm{@^Hi zKMk9L=oKc>w=!?GFisOW3$q8CV)B0*<28x?jx_f+Z?r^wQ9otovuu}_v$wgU!=fz% z1a@44-66*HO4fZA#JoA~Bt>XAU*HX(<fp_5i`yc5k-B{mrbI-yp$4#7^vu@S{b6-njec!-3 ze>W~0|2%ZnGWPpu=MAi#_p$EXj}GBsaJ|8{Wx!VI+=xq0e8HX0x;Z_iN906w)3SH6 z8#k|wR=)?Dy3d@irQ91R(X4eH*hdYTo-rL=h6f#ony)W2UwQuty=7};vrXoqvG&XY|R&jd8<2@4yeUmrJb;vN!;^b9+XZP;(HOk#+{S8 z$(OZxkc`cP3T5*k9v$XnTfCCNlf!l^8T|T8Lk7p9o|0welJ_$Bonf!?oDTDG!ur!T znO}i652g}sgrUPk=OJ?Vy@X8`9cCS8jk%k0in|bEhqSs3y9cZNwuF#n%y~MCvPB*+ z=J=&;q}!BClc*P}K2OxA0R1}5->NX=e+9Y!qoC+8|NHGP^*-EoQ!f8s;=ypU-0KyL z(}w^B4^Dyy-!cAGxEFpM9;`3D`oegCPGY@FaOA)7VMi%?=eYkz`0#Nj10PaogUGgX z$vgf6_`vy;3*v*w<>%wWLBf6wKHP5NLqqEpeAscJHCdn?j1u#kLwqCU9sv6BVXq2{ z)vFccoks{r&*DKZOFhu$iTb>GmEuanb@mfjrCfj=Q;8MUw7D7-ly(T$sOx0 z@O?uc;L1jB>rovQ-OWR~O5O{UGxg{RzLK-QwO7hJvsoUIsW>;}5jziJw*ftXk{iT^ zhw#4%mXLZmKh&A`M0ww5!&7bJ$~UsVw$Q0L%Y%6r0vcxH9L=OQavv01hsdB-cup;4 z!i(0YSEdmjqhHT|7w-s)T>dTeBXLrDkG+%2TZ7%jt&(nkoF|_&e`gM2=hX8JVGU0v zcrw|8a&|6p9$u|^#qNdN$&j};`j9p|pEInSRTFwEW%nmc;!}a>6LffUi>-~J$kgNub?~3)Lb9cKv^XBZV!Swkc`hFl|kPm&%^ER_a?%Dd> zcd`y`$@8i>PmC&G*54QY7Ufv1YBTS`wtV+!q9>T|F7}Ojr;U4YPFyN-9c$dNqI;2W zQ$N&JrO~i0f+J0#y2lp8f1Lbh%-h;~E*w{FTj(&-mbTUSFaD-`-lu&Q+G(Y|HgJr0 zhin!PHW){z_O}!d1)u%_j%9Mct%fuEoC(iD@3KB|b-E{RWLo*#d2-%9M(<1;Hp~?} zSh0F0{*Bx6@nioJ|6+7VV_7r*-L7p!#}}JMUe?k%%C>43=8g;8WZjK@jBeQJ*Yx3P z{>JM`eA@*YU?J_Zt&Fji{aG9T?whbl6NWcfl+TG?<>r_>qu`k^S8n7=Sdr91BY->R;m_fe3W3IEd zX70r4Yptc(na$1b)`Z*3?E_p@nx$=972nJoN8msZeaae2&2f<>{rNVpPVgZO+azK5zqe|IoD&;cNn9}hrQW63 zDzxI?Oq_+df^b*%&k^V1PUDw2x&JBi=KMaQUzA?VKc_C0CN5OlDPxu3D*dXrw(X}A ztiy#)%Dd+!*0{R8==dj7x70Va;?LTDw#;tf$jXbZCJXel8To5ZhNfs?9RgMg zlEa&aX}rmm!#ipAGX=Xe(yyeC1mv}aZzR1& zJtdE|$g<&Q3wGV>O|H__SK2U}A;w=I0+RNI? znf|pSymUV82*1}wn8Z)$Lp=7QjreWEKW)SlCh;Y=5|2GhBYwN%&##w`&aziND0wgs zj510s#oO7NLVvTvSvH3S zX=`jA!KIdTDQTVtPqQi)`E5xWv_|k<=b-CGScpzlS-azjD3vP3%H< zE^#DJoUKgvIuh|;kUE7ax zZqa;ey90*e?QeRvLf$S`zowd`jMwJdbhT&wxy-E_fQ`jI#5{#$N1Gc}hp zFQ2OO^v!&Cgm3!DSPelIy2a(&oovX;qay!-h!MJ%o*P z`93wn@+52$VZ&X%ZD!a237bH>#=Crn$I$+%+|lPf!J0z+$Z<9Ia8FX;B$scpzzM_) z%xBDaPf+RxzVl!6MM0k=T=J3pgWm}^ z-xs5;Kj5#X?S}=@7tkd7+T}YakiOb4aEQzIXMxD83!$s+c|QW0uqbfvt{U7yK2v*l zS;P&quy@B>ZQGGkCsjIm7t!XiSVBD=d+;XWndTl7BDCUtw4tUm_webL8v2v_-WUG9 zO&0Go@_l-!Cr-X^;+?HWT)Zn{&#LP@*n8&<_KNH8w^!7HueCXhU$MPH`t65nUA{@P zTUFGQ1$%A>rk-u%h4%aU;a{`evQyf#2VQbVs)2iN(MDNwh@BvR++OM@_JVIR(|AcE zc4XGD|J0I(F%=M13G{U%^emw`e0q$_SD&Bn?acc#>xX1H(U&YL#g_O{`l$MT?QGrE(7D2EoIil)Kg`yA^X=k6L?88@wdG7;X>$-+}n_{e0z65L4kCgXg zvbCxy)mn7~JLL6~$mbqQRqeZsQE%qY`S*ok;dNGT-J3RV1NCTn)l%hq z!&3F*+m@=PcP&+)GrxY!Pj%LmY21YdM;FNaXj4&B&bu7Rma67dOVv>7QPZidHxu1X zn7mom3A^+8D*W00EM>4G3leJdCACFQ5K4MKNBoAv^EB1vyJK1N(zIa8nE%T%?))E?Q4^~9 z#C`@c7wKj9SnZHI~0duK?$OO_?BA3b;ds*Ap3L8?$YedFjw(MO zc2Sn>H}OOE_SIPMjgm>I!vXOAAoS)CvdI@S8GpuPPlKy!8*@JIp)6HqC-4{JE`#dN1hbSW%z96>Z#ddBINh zCO_^XZame@_aa|G_iE5-h`hB{ApC!_KyWJ)cnbbhO&@Hb-Ex0_J#|Rc;??_p>69s9 z@_)Om6Z>HgMPxAgf_iOu*-X1n)-bd&UgR$3gT%>PmG5lC)!I_j`HlQS7tj8y z$U2g4iJ1=hWvujrggs-1%|%A9;k;>p&TWxaljTX5_sr>&JLo5cE3{>aj3I4%Rv>L% zB#^d0EpQ}!Lf~+h??HjcA`1mh#r}`LX)fP96&|P0RWMdBRWL?>Ou=})xh(9m^0xGk z6`4&Fn?2Y|vvamU>>>&N_+_?0enOiU1E@;B4TkJ{S#wG(_=lva>;wgCC3cu=JGgL}F<_awV~6K89_)t%Cu4-5$P9tAh1 z=5mf4nMck+PaPEI-4AYcc`T@+I5*r|HO=YV1D#xXPnHvV)ZJE<#ckT0AHS(2w^idl zkyj4#$^_bO%0GLN zeVUP9YI<8dsr6Bf)!@6>hk#FxF29;JD}1H;ZJTFDydl#Z3uRnxVco%+oN=paieatN z2|7eySJRK+(FT<3O1=?2n~X!12{!_joHNO~b2Vq_suvIQN?D&ihs=xa8h!YNQ_xkR z&-rE^!Z#&Mz7yQCo9lo;yb2MSf%$+=#kY!lNmo@jaY%RPr|Mjsn;5AQuu zhXa#D?s4cM^X!;v$UF_mJUM=u2U_$1<1Fi6CG!Bkr!1ifO6Ix9I5?EdY(9oVif$WLToslSXZdMBUYtm;Q(Hvib|-H!Yt!!!POqYd*%P=Drop-_A4~cRlhgrAcq*(OWC^irw~KtJqDDssb{9L z#=6M)RMA}>L_R-(?y3>pm7FmXTMldtDLYdiOwDs14`Hv3v#C$;7G#?FUb|Yut;UY= z?Boj3<(w&LbPm$L8TciQ-IC!!$LFg>#_-$N#}1#*$v4iPQ6AZcx)=TY1kS)YEsK}6>^(jZ z;pw32mB;=`F8h3%d8bVFU@h#WV1xe_8*%K7&0-&xJ%hkA&G+%G)Z<>D!pqlUJ=cgI zc}>;@Zv&U$Buy*&MfmmI#LJkE13oWnC*p$5xH||FIqeB>S<(pZ+(_8>4y}HQ$Pws~ zMTXKW=s(H-6li8_dR$3*DO2(h zx$hCPF6@IS`(kU@PciDlTU6Aifq233C{<3P-WvFQxK>}yode1DT{GV{giF8Wk>{Oe zo)LseS!=@md2*hFJol1Lig2^U;hMV=XEo_`@s>b@r4pQrI&_G*=9g8shBGhWB`{u1`~FZo7JbD4do z*w7NZKMq|~IOr6a1HNK&U<+LHA&AhIF*@rKAZLMr$Dyl>qj~QICt-In9)gQ66DM_B z23!hFk+|QOaW9#1)zB(gOJ>lfr%hV+G;xx)ubGy2BUL^N1M=x^#&MQS#RcZ`sF}~B z#L-{9p;tS!pkz-T{|lfml6QBr|0GR~4Vi*@RDiQ{j@=SAkT)=8zNmGVkw5fqGHH27 zyqA3M@BjOZGE`c5_uwW~cD()@DND-je}$(5{z~?g)c<)754xr$?1iX$Sj@2bX;D2>l>vh#-t#A>za9kM9jtj+w;DT{MI2+E2v*5IK(#G%1K4vdc;ca_c zO!?zxeftttFsnN_k*K%FKaKgk!WvVT%Q=m`)KkV|ALAHT}PqMyuTuXZeD*H9PPBXv<8YrnR#hhgusa9JvLL#p3i z$eSg<_u9GitX(8gxw|n5cUKAJKaU=btc~Phs)IKn*Sg5nmjYHt;mwxi=jU9 z*2*vIeW`b{!uHP}eT@EZ6AACJ=Im~ca_wv`OW7&s%+99=Y0RZy-W-)UFY=$4`6D{? zdx59eC;VOuDYm0an7|p6&*;<5T72O`;H${b!=VR1*n+d@=VxQX{=94j_tVReE8@aF zU$z;49^Xm67s7`UK%IkoOuJp=CbCd2jdD-UHW$dk;So>D`acWo>WP z3HJc61y0g@?;Rf~Z+rXR|69IyEo)qF6uNkHHQoqdiKck5*c%o3<8H>S0Gwq`icN0c zMETFzisM80k4@g+@ZUvz({Szq5PzGDGyl(w)O;)We-r;j&!brc=QQ6a`G1e{S5@Z+ zdE`9Imo~#LE$8ytM~|y}uG@3VYD!!_-WaQ^nY(b=8C#|Q zd6+Hai%?5Ux!o38r)3Oo(lQ1$%}pQHG}m)|(<;xS5KlW=Fqqa3_b^3bC_;H3$A*?vQBVy7Qvy{O`9 zjJQY5I16zy-&Nd*o_v?2|IUo#8>MGwtm@}ETuniVoaq2prBACY=+w~Bh#osuXfaOK zZ4&nFgIA1E`E8Q?67`8@e(X&sdamO7_fm1=%((A~6B=)PXT*&$w_qcSO5>L>BvUd{WnxaV=A zYg!ABLw1doxEjXypH~@gT-Ji4Q}7SNHHeIzLH>fX%sV+JaFTP>F_a}bK>6PW|H)30 zCQh9r;Qc&!H1AJPwiay1HXdiG2orvJFv0CT4W0;p9ZcEC;$Qj0CWpG7tsMg&;qA2? z?5J#W8uxQ8)PF5}^=kBfhr!)pxvf0IEUnAy@E3x|h4_B}zn=oXpTd6>oD@F$Bl!AP z{zp=NZCF^joPQdY&K+mmQ#fa|=CNa+Z3}wWHy(F2UfbNf@mh+P>sbcxrz z>mQA(U~L&~wY4d4AC5hEyB3``cTRte__Vom`xle83vto0nzxNz^QE`KP9AYuDQQaQ z_8&$3-)@bnh<0e+D8ASDM&tYcAlvQD2;#k_AT=Lqb~Z2d%MnMd;6SR@V;5`BX;!D_ z1at8S{dww}d@uJ{d(l6O!u&yh<%W6#>(p_iMGYi_K zr)y{b%o=Ri%8s7$$FQqvi6|dIee9N2ui8mF6grlhn|n5mw60vs*-YZrTI?#0vExm! zi1M(wK|Ze~>{Tb}!{UM&V~6P4zG{Fch`eyW;2+~IVe)^9`mgkHU+qJ$BXn2pMxA0T z8W@)uk@oVsSKZzt;9ny+H-3<7)2{ih#_sSJ!7HK3)%YLk1%C6bq_bf1 zsvephQ-Q3rLF%$DAWb*Yz@NTaJELD?L5RY!4G;Bo`PPs|@(>yz@q*u(k|x|tbDhgq zabB9kw5je9`knHB?8KUm^lI;H#;i?p)R+8j?nxFuauYJvdjGn=dV%Y# z_U**S?7I(rWwG4zl4hUSZV&TR=_MKO>QL+bHPb9!(JKm_6J5tj@=NdE+f&P4KtJf5 z(0!rTCwJw0RUgtf>`^VLrr&x9q+hxz7^inpFjl`tL5F@7u<~qkLKuB6H1pm^9(%Tq zIVbe4g+AtLo+HrBb zc%hL(E9-+@zEPZs6`EWqb(y1i*5RJQeTkdEIqT}bao^uEp^6%Z{8q% z6x~3%yg5~h|5ukw+Dn#8{DsQ>1AW>Wyj&}H?250JHx;YwH<)iKCb?!7Kp!;lOwOza z+Cq?Z9P?y{w){%ZD88>`yd+(qZ330&y5UtNbG72yPS}1!|CbiG!Xf{+vnE~pb8}C= zJH{)po(@OPD!q2~H?lU$Md!C|t(3{T-{_Eo(IJPR4-7>gXeW+4$+N=IQCc}$ZoJJS z`eUDYN7-+mr<8re6&kX>)DND+d1T-1__yHy9zS>M*;3~nlhBY+sg{tg`Ei^<#hyf} zCAn~KtTy&!6!d7?6=T%=-JcB)kJZ22W5~3;;i6<66?b1>75AwbH-b1hAEM%Z=<7Lw zURS=|Y{m~X%dO^qNkbdX=i}s_SSe8EKKG}W+{ipv=T6~EJ^4R^|4Md%rgh~%>t*6? zUa9}zd3W_4+$r`Em;RA8i z;K6eE@fN~Tw3PBv(p4e{-VU7+JC5={jqj&f6Uvvuliwkov_anHS&p-73FQqDlz}d$ zmNqzNJlr^iJmtLh9L}EBCIp71)XDi%d8g=HI(uENguR;Q9W$d>V}G2)X9DHU%2v)y zx9}|bM|f6GsXzT3$g_m*`uT^SXLTk_zWtJZ(`@i78*;z&C;aMjU`>?4V}xIAQ{l1t zb_owX$9q2De-GLlMt=nI6~@-5@a3U?o(ZIpJ`wp~z2rmPUKdE&uPG>ZU4}6au4g_B zWnK(nehkJo*C2H0R;5Fi@1-BbzA(Cj^=;g_=nhhMD?ek<;%^(^j}P!W0{ra){Otq$ zR|fczky`QuCtCdALW`gAZt>>@_y-60hXwfE0shee{+k2*;{yD51o-a^@c%Zze{X=l zD8T<`{wD+cPY3v)4e&o7;9nNte<{GfD!{J?_+Jn3zZKwr zC&2$+fd5Yc{%ryN2Ltfo;Q;^40RLkF{<#7E`2l|3{b(6e-hpiKKOf*<7T|v=z`rWM z|HlBo9^ijH!2ec&|D6E;CjssHbAbQL0RLA3{@nro?*jb)3GnX?@E;8D9}e*U7~uaY zz<(;h-xT2gIlylXs6TH=we%bB61Mmw1N?0R{P6*PM}WUwfWLi!|H=S=R)GJS0Drdt zf3E<4-vIx0hF{}{9%g;|&^(`_Cr=vTX**mx=%XsC)Y;IgaB_uo?tOa0pS5NLiG{qe>KE4Fn862M{#; z5E?T*zzFk&nE?qdC6{`pYi3&L>2CA~Fe6*HkOWy+n|`K!^n1J$eMj4rO-X!v+%vuA zt+5g$b8X%}`pmPrci1)dj9b%Zdd92N7fBeI*ar)z1?DFOD4`0qmzmk!DH6#7? zjPy4$(%;NT|9(dLTN&wZXQW@tNOy)wey8(yV@CR>jP%VJ>02_=@5o5MGb8=3jC406 z{oai9{*3e;8R@$-(g!or4`!s_pOJnjBYh+z{b)w|ct-kkM*3Vv`pJy+PiLfmHY2^1 zk-nUfUdc$WXQa0>(pNLme=;NeTt@o&jPwf`>5pZkKc12PL`M3P8R<`Dq(7aJ{!B*t zvl;2nWu!l!k^VwP`imLq7c2GDEznzhOEhF6-$(a8$(l=$KZ_Y^Hl97H#M*5xU>6>AU-hy@N!N1Kp zzxZF~oaI0J!8?!P_cVTA`STyVbKzRf8OQI-`2A_TKZxJg@Ov5W2l4xD{2uwcoa5s6 zU2pv0ooDg>F8sa|zwgKIP56C3e!q?18}a)fen0mA<(v`x{xp7H!0$u&eG4G@5k?H{BGg>AAJFI;P*7%{{em<#qay^{{O)55&WilY(hQif0J{5AMaN2?jq>A zfZuQa`yafsihSqs`wjeV;r)NW@6CVtgLh8i{a@hsCj6em`{(d`BYq#mdz_DV9Q=L_ zzt`~l8uC&(ut_=>P|gm#d+ARFUq60d!fyvSK8oM3U@#)8hvK_>S{)kUo@PQAmgBU08+=5RZKDXkt5uY3Jxe1?}@p14$e2{l=f95+k z;IkQ@cj0pzKJUio^4Wj>oh!?K_MI0;u6~Dp>u>!|e``4XjH3BZKT|rrcwlGk{;hx7 zzW?aoE#FVif2?r-Zz`SMp;a90!#X$6gY%ybo1cF)J$d9^eLvdg;3qJMfoazERsFK8Axmcn9YxoOk2%9(-=c=l%G6 z5T6g>^AUWu+N^F#ET zAF7M_!DED%y1rCbswdTt>h=MAK8(*EeDe4d@%bV?-^B;^E9Z;&+47$EI-733<)*$HZ{4{0hIehaE%$C`>Hoatb6@$#OTXRt>+1bO|L@e-|KZGS z#~)k1@oV4u*I)k72Y&zRqfhSo>BB!i^6}Gu@OPJf^Xq^7n}6}>_a6GhU%ltA9(m?V zpZtY?_2YMa>&b84d^z{%m*4yCuk5+^cfPRsMC&u(Zv3Zz^1q+^^_#AIt@T%5`|fwk zUwi$hj(q$7{@q{t?z><4<<0ljcKpS_H_x{|b>$n+fBN5C_?t5`#~*xh<#+cy`|?L` zc>eeP;UnMq>szn>?vbDW^?!AI;a}dgv~AlRU+Md;j~@K+2X|gu*)ehZyAICIHT(SE z?)%)Co&VGKzW9|LjbHfXH@@>L#};LsD`)_-z z{`BXz{K$8o`M{>i$L}6Ld-S8fdGYzd-`ITDCqDJf%ac?8@~KDvy0&xq|N7F;wQ37r zU%g}X<@1|1{N7`K`_8u?|I5nsGym~d|K{8O&+6&lEv%Lo!8-oSp~r2+NZ8`I&LQU< z%MnVT_TxYGLqEIfHMJ@I>G#yG^rxS7I=?*hbBBKRKm6v?|M7R9e^96Mdj)?8-{

X04u#P{8~x(PUN?`tF$y2`fh8P+CV6W@BJUBtI3 zhdv~iqcW7g(#yY={F>{r%AsfThm!Ks?_eOiy&Wb${Tb%vUrzqm!Xan#R$gK1zVn}W zmlqWqFTm-=`p)oDphb>jOvJ1qw~ zJFV1zkBQ5uk8dUH&CZUw|A2OmXOef9V#8=|Rr4lx*X%TW~%M8M2raJc;yCvdoONI8*mJGhT)0WoG@bb>+ z^)l*Pw)6c?XGeS_@jC6>#CuG7$I7E$?`(m7Au}Vlw$rc11MU0VW^Vq7G=kGL>~Z_~ zwwxzl(YTItdI@PH?`mGJ?up(58N|pzox9st6DB#O@%7XZe?>NC9#_Y z=6eO}u;+QhBDsBrKC+hyU2R`C#fMo=d}g`x>7bM*?Qd)C66gL=ts#yM;_gv+k9bpD z_m4tz#joOn&7S{`!dRglznk>jy?PkBZOWHkwAhomQu1^U?T*5>i|xJ8ytyBt%h?o& zVpneL9kd^&J(_46I%3wIRhi)7QR`=ok><^N~$$0`RWuxKL^tLsFU7B~6 zH-39p&2{JDjUG-J`uN|(t5X>+Ulc!!%>^&Osk!2d`Ua=w6JDfm52v=y;TxP%`Wk(M zQ~D0;8=O-5#z#Dy>Ia|atbofOJ-+n1yZ+cmJ(AlF%y;F$gTTbat4r+Mdq%eH=HKwg z$~?w*WDv}Y<@||!I)@Oe8~~1tZ2N6{IepKTV{k0f<&%DU=%3oN)hlNk`L5JD z#PD{vPKUITjeY*!Mjv#>^{)5ax$(N{s=UP1tG@iCz3vP2fwEj4-zPHODGm>s`yH4pz2DmKE!z)cN9NzsojE1HwAj#!vFaGm)hF5j-Mz; zM}U4%X}&YaZpEKbJjkvht_9py`g_o6pl9&&a$k3m?cLY1u6Pf54sriD>59>Nv`lu> znRw)o)Gw-axwdSC&V|2fDI95iX&B z>t@6U+XwNV{?1_DwzX!}qrb@ySPvB8OGo%I`hh0ejBd|O|D(&?^tW8ro}TtS5GK9a z$KH?D^E=Y8+c153?=su;51j<bs7*QKpoU`;9}eeo~GIX5jp&&?Tp@!BeM znYVFI)=Qk#sm10Yb0>E{;%gn*{5uzSyJ&1@vaSc8EaA&c?3Mai8_LO-#=WM}bvVRJ z25h_%*uwj`6Wa-6PaOLo6phb-$L>F z9QqOS;|G)pZZ&AF_2MSzrz@*8_-n1ks&<*9LF z)4aDWpC$r48u4~~-EAow4%ur+r?M-8{5i)e{S3KMurRvx%wSYw z6Xs3Gbl$RRS)JCDwz7h|+qp;n6lZ&@C~KLqnW&Dj_f(~|@)q=OUD2fhtKxZ)W#P26 z`sS_K$hiS~FLN)1UN)>YZ?dA?)51QdW$KVV$JB`)A+|Ue9SsazeZHBC>%!A|+p(FJ zlg@Z&;JRwuGhJJdonD*oFTVGp#8%SNwz`M09ox#7Y~p`0`za5bej{h%(;kgwI`2h~ zf3j^Pw&62^Pl8w6zc!8g2=)-)jGT0oGyFWMH zm-R3<>5seXc7sbjfnnLoja zX{+|tYdW*9bIuxNHZp-fCwBjmI$rG9kGh*1sxLyORzKR;Kx*qFURzlk?q^My+}1~E zYi-flw$^YKP;GtMYirf{+Pb-yf$J4XaDAQj)N(hH-!JJyVke)i{x{$%JCdn>!y`&dWsXZ@TE=X~IdzIK*qZWvvd>}-m#k-6nh(A_+%^b_u#Nr!&u z9T?^s=WL`iD;u@VwSRzUQLm zeCteXEn~2ovylsVFM|2RUWaU**A@2R?Gff_EMys)KA1i}{|JX6++87KKN@=hj5B(~ z$4>;HGyEsb=fo+}YfkUt4s~ckED`zz^ni6<&Wt`om*o2PH`nNGpIz{2WYAbHG9`Sb z@4P5ptvZ^M_(+<1KH ziA=uG=ktyA^4&x_&EE^vaS3&d^3q=K)iJ`WW9S#up?rhAd_zbl8L`OZ!+sAR`gUY& zKkTb`bNuMyP_%7H2;1?1W7e!)8RsdSv3wJK*4A>)DndKKcb&ajD&9RT;Fvvje4TML zarlcBm+)+S@OAFd#NkUWyHC~?9W)h8W5Y!x=T zio<`HIE70;8ase~ePCzaIVpD>*5BIKiS45wtmE0IMUSCx*748s&6*v1^rpSo_V0t{ z?q`n07ZVO{nYU1u&~M($HuG%?Z<#Qj?!B|Qk-uVF;htsBjqLmd?I!LW{Ol0{@ti37 ztO>}q8S{g^XG}ksm=sAnD|h@W=?buu@aLGJX?Jv#JHAF5;r)GG!3kv281mHLI|mHj zgXIGlfS1ajr1H_<*RY55KPn#{pz^nkam&y0%4dH&OSYp= zd9bMyY^V=DVDz!!q5s4BvfcV>SaS;x+n`(O&zH2up3aAzBs&ZY~`bPq%_PQOo_)=!FK zf9&04oZdm4@I-OaEhppjTf|)mr{5s115U5f+dbg)D&Fk@r;o4V%^vW%ZB?Khe`DbE z323p-QW_d;WQWYi{v2>O6I&wi`2cv@2cFT1aq#i4T~_!?mR*RfaBC8u`rzmA-=>ay zsw&h_{W5rC=<<5>UPk7`CgNf2X;`1IhhM_pO^M)X_Jpu6dl39p`VwG19=nVx_JnUL z^zOpL?&79J`tJCS6zjHJ$%Vy%-X&vL+hC(R=alwalVsdjp*8XY9A|JEbOvX-_Nd`S z*4S)w?of80tLcN!clB+pN^HUpJ(*csaAwD+>(a|#7G_zLYjEggpT| zY5W$(ReRbkj4eC&C+lWmdkc@cgm+9^wwBsAP`>uXNAVxuI?Nu$e;kw`jUT$m%l;jfravGPAqKCKjH%U*^8jB1z!GI(*FbVNbeXSYOC6vO&MQqFXM~btTJx#%BUvK5ZWEz>fy>( z%XtDB_k8W|M7}XzzOm#>ZvV*ky1%&nD&q>TjA7(SZvUn2Wqfh_RYrfWi~-~kor;08 z8oTWFGF-gnu8IwJ;QF>CHg5LLx&8Ka+lB&f;-JB@NY$U92plFW{d!eMEYM z0^ZDfi+s{8=&aMrtVg8xW*`55!jpDn_IG(im_0)Ah?&Sa;s+b0TVf3@9mN&cU(e0K z4r_N}iTK2}QoS$D+XP*q6?a;tzrWKOewc5qiwn?wz$?~;u>sjeKg~D$spwhE88gN!h*_M+9EF?d?0nB6{evC82 zKj&NN`mhJ9`$k?@95&=_tAfr(FD{ooS@GBnio?$Qci3-^%gw7 z*%7xJ^BSs;WH9cj710=YHH@))@KP5Ki|%mo@V8z+{sa7&m~>{e+YAF0wj{Voy zm^a=WI?eZr`(0jD)vuxYkCZPN_tYTnr{D~O(;eLzcRD=RAKfbMPF!p@vP`baYmH6^ z8;}neC+T(cuXC}!UT0So=csrm;oLD?MLRDvhA$9zp)q`(xC@QpGhW{{R_Ds^PZHNY zhLYb`@&4aJ$vVh9M%F>*F|rOakCAa)nMZQYG2|U1yQ44p5Piu<=pqiFi_l#b$udv{ zGS9ZoyyL;R6i*vsmoVo_kok%`!GF1H*Mu_%M3Y*PF~ZnI$bPO~axpLkPNkfS*fD;~ z5L;(nW4v#a!TawS*uLGz+wO0}kM5()TUM4~iJs z5BnSN>{jSN3*`<7j6aE<%WmPU6rQJ2=FCNU{v`?&sivgMS>l@K9$r z>=nX}&%oXjdhk%X8}>7Wp_4lf9hlmUFz{pzVN-kY4ZcM5-J5UlW;NfX(6r}%cs*}s zfv2xvks>L1nE__+{Hxzy^%^pH|#Ydsr z=xUMK&_%fRKwBMZyF&lv+TCBbXiq10GjM;i%&FrIfTNEZ`?dz_g)8il)FV1Q z6~7wXT+AAlbu9PG+@601-N>Njx$GC+8~!op(VD=2bn)<@ zbvomAHRsVzLUUu#R&#d^{37<8J74#}=d{Lsk+(Nu&`@BpL1#n5@}ZI2(5FbI_2*dv zeDUX&@sDB)0{$Affj5Hdz2N`f2h+ZGbNbnN>z6xhur*S1_)ciZx$|}-?Y+>v{Rwdw znzxdPFEno@^IeGcexJDi=XuMX?uV5n%-hTC1MJD3J8!?my!}%r;DD}HdgN`tP7GhB zweWi8BV5-5xjmB6|&Xnuq99g7{|U z%gmrva(Gi&w0p373AP6fJ+AL>AHg5`SbzI1%9O6KTQ7U5_LNL~dVjlHuVlT z^^*nc4%5$kKR$So-L=Ha+#DNUb@2{j!a2i+IFaqm=UDE!7n>^+n@9}f&P^@@J+e#@_+ zVOlRX9!`|*8JMm6v+aHRvdn(MM~q=JI3Rk1Zph$89cv)-56oqMXx6X&tV#E0P!G6K z*3ak>)`gFZbI=vKvf%L~S&%fkKhDp)r7!f4HDVoakV+=3@_gN&ZGEMnoAv6@Zei=v zi^Nmn7)B9MHiX?AhGZO9tEn@sEY)0B%W$nuQRE8FiHVW?ga)h_;ZLr?XwO6LO z=N;pu_hkuvmkoAhiJHqAs(-`!-={;~`oF>Yc^`X=*8f;4>Q0QB0krwQNxP4&uW*j8 zg_c4`8_Qfe`YiI2p-a##(Mq3|W$_-8=_jyc9UbM~YT$;=q|&JW$#7Mi=sWvSBN>a$ z?Mm{{2Wm)P(66K18t&FmT?sB2nQW7%i^U(nn04S6 z$ozGzeI)z$Bd!D82L`SV^%8WbdA4M>HeYYby5>V5k5WQWO{k8KII z`;!h+e~(MI<$fuVI%3EpHEj*NFVHhs)^z(LMQuI*^~R=I__er|`bQqyiQnUTXes{3 zFfXGk6P53z_ENPE zES-_X*%p(oYg&CPVc0cnuX@62)1K=V(j^uL&3>?%b4M-7I)}q{Z+oF^(%=u<#0RNo z4|#cesd^Q2$+|sJs(D|8O_IjcMpn~(yI*GAjl9+1zkgxQ*_Cs?rU&O+dRW6pb*gYi z_2AxFXvZ;+c5L(3?zNO5+VMJN82#xh9**KaLR<&gBnd|!AnyD)IshE)YjD(Q$I(6> zj%G&=u)b(uz4r^&MVhB(pP4?+W!_E=w2hpK>>;@U8qv_cM-HsmV~QFWz17>k1{g8t zc4)0w{|aN3_)L0`9lni0RriMKf5M(wWu(0|!R*a5CNl!y8SBP|*#p>11sA40R@%Nt zpHDfe7ryP%!Q?tW>}~g_PPK;%rb`pZ+6l?P^`YnmZAt&lUbg|6PEZ-6`C3%H-f z$BnDqyf8q68(Ycix9v*WW*zI;wyF^g)#o8yQdA97F4KBV+Zw&kZ-^QxW0!YN2tu< zd^gk9A@~V&0SdcZ-_THh?rOfqchMK*86#)FLnYU20pBF+)Pd8^`|1m_(W-9t{#7oF z)CUJzqj_-pebt>8>8tvvBMYC+j#8f|`*mH#-oa_&hu{l-G5Gd`H1rs6B!=%q=Vta! zHl5`&b&R3*BsEuMlaaiyvVgb_^LxFQ4}T7E7h1E-CaweTTIKOB{QHRW=|cPvJOzD2 zY>>NO7z?3y(6~=W?@*6F8NCDaAa?iJJwmg$G6SB!gzzce-eA7)h_^Bsr;l#a9!qAV zEzf%H$IKbcamjHp?3!dJBO4j5A;P?Wo58;PpZZ&+D?}^0bh2k_{`I$p`?&n=mx0?~ zI^y!S?1t*e=%T>o4*OFh&h7InXX)gX#9auEMZ|T0V-l-)&ua?4(qzP=&1Tx&wT^<#8Q#zwg<51gl=?PpBuWN`(jRL z{&{p`TVHqXpG1}s-4O2+J&AeY#|YQhzfHIg#}eRJfLxhOgJV~_`>k7mz3jX!(g?5r z0l(qXv-GSbl6Hb21g0k%7y^<=`gZZaaJb$T8Nr z{`{z;z2;6?uRpq1CqaL;C+Wu-S&+Ue&g?rf<|YpRXXMkkD$dAugw?1_{Fl5k!^HV} zlG}7fZaFqa#QoSWFKNF0z$;I4M|qCE0v%7nm473?eO?V@JO<%bGp~~Lc}LOb9Y5Mp zm**tyUz)vljpb@&xh43L=Vppl^KSQa-iDm5`~3B8Y)(XXT>U#DlmD(yttjH~av z=ug|Q&4S0v4$rS|o8+#-Y{P>~z{>>sDB(brY`^fY2JW)8$e~R>?)Y&-iPKzAoR2$x zTuD-#!JVWs2NEZGVe(P7XmXM2!_QNF*lqNtOdsd`xNPEtbIEa>|1|l!d1V%n&)_4l zH*xr>#1*TJo;^q%`dHn?8}Ra!6X)Y;owo)#3IFl&&EVKT@Cw>}e4tCK4PDaO44QPJ zUC#5~8HGN72%R>#1fJaktu6iFYitnt+{)w_~*a<6TqLM?D|G-w)89)6{3NZfo=8nIdEx$;-d?>ibNvm>7!E zo@#Vmrd=nz`DW%Y^X@g$YHlab#TwpgkUh+L;->j+PMUuwn14L3eAW(TbLu86l~(x+`wP=R*&gLU`*I_zuz7tRc#Z@=)b~ujV~Z=JW)JoD9Gmwy zoLGt00q8TKCq6AX*H(Ai4?i2;2mP4J(OiOlRCi~Lb^*)g=FgS{phH**rQ?0lQr27C zgLQ=+wsaPD%pWYPEDFy)Tv&Fc9pCYp$tT<6#q7DQCvR8s>a09{7y*5%R@@rGyMPlL z2xq=>C$NuTz;PFy?f`=8JxXy1NlkxP@@HShb#W1@AUJCeoVF=y>b3#^e**>Pff zd?&EHkUgo9hc*l~_M{JNfX2Zyb|GKQetD>k?6pqo-)z=$x|jEN-k9q?-tByMuq}Nj z*wnTVdI}6>TWVyq4z{J)k>}Q1&wchQY~;ZaohjJ@g!hn06YQJY?-}v-d;4Oq%bJreHzGG>v^4s%Mi% z^zWMb&+~8L-$TTo%h&z62!G$<^LM2!J!WK2Q{T6#Z^gWh_0?n{|5KikInIN}-!rt8 zy@oH?`%1$u&|j0skga2qDI{Nu*TaVl4&oPrXOgjnYrgzo;x?hjB(B36D+zZq&&6GM zsmX`mg*c6m$`sD~e9hoCyeozHcHBOPclom8dFkiCZtVYl5&g`Kd^C);8GA(&vlwge zCJY?1(J{A4H$vOMQ6qQkLzdLs)7?j+7X$I7JJ6WLr2pDwXf0=qa_oHeUCbClcf)~2 zOE+S>V&sVV-h9P>hw|F{1Rsxo0PTRk?p@UbJAc;Ow3~g1CcSkK^JNG2d(Hcu^;%Ez zt`#(2aGNcBXaA{rO8J=u>^tQ%cc!uj#eU40H~RnN)_wig-uhzyHMdSCe1p5@X~$>f zXq*lAcYwW*ACD8)f$o75K5on-&WB^768TH?;cncZ z0PmlAGS_0A5y_71d)A|Iz|ZKO7b2_h2G~gGoXQ`gJCc%UTq8E~wVX>TK;Kiy{TE^Q z)cI`Y4UPM~OXEf{Kig^CA3jgxp8vltUlWb{F5gzN{rulSI#_a^6nAlIF!P>3fGc z1KiAnHY43adPPna*A0m!CyPloxZGBa$Q|ky=YIL$iIUt zMi)+aZ`z%O&PwpXzJ0=QV2Q1JMnZP(oONbwjlH_)ggJ}CeJ|=$)*@&g{n@5q#h zi?_5QT1%U8GIz+FlV^dOr%Lzak!PNlXQb*_dv;tKQ@lF37fN-6Gu-sEy!6;6X6fy* zSQ5B))^#ket+zz?YwQ>>`yLkm*2Ajj3M&-U;mI;Vt5d+|A6mvdE4eph+$dk^`3<3(4s!!Of=-!s6>&{W`8e=hvq4>le9LilAyCIOpY zA`j?Xi{Mx9!S5BqwdbmSzQp%@4}M3H4IcDmft@8bck+#jBM*fC$hix6{sP_GMV@^g z432T9>7Lb<(Kl#+#bsBSGlw^BMm}P%h__Lq@Sf6AV6vJ27idSV!m-ba5niP|5AOYg zzec$aHOiiH&$Kivs`c*(#=&1tjir6aThhz7%czap8?aA3n`Sg=EOdy;Y3jB$@P;}2 zvo9gHb8jU0jXfAV&(j+og-40El#}Lsd)004VfRBjO&<20MISyQkH!NysC^Au6XvOX zt_nWK2K&g=fnTD);*^sz1wZ1cYvYx5pL$Y`C)9lE1#Udus;&)q@DpYQ6P_ae0ue;#XX_=<^n`JC`4)Q1a(UY?;Cp^<1$8+mM9z6d#lrfWhLw%BOCib$PyPG-K2GcjSA?RQX)KiA5Ef7z#JG|$`TTHTO_YVprsaEFBSh8hRy zJ!-)Z@|nFF=1S>C;(h%}`KbFYDk++J(Kd88zFwtB@;-Yve>!xwzDuZY^WQqw7ekjL zShF5BbNkd!tmnQ%9=~kqXZ*5~%YTzJH_^5#+&J8gxEpX|ah13VoM=%Q4tvO~>v7lN zuEmYUU4t8i)7>DFZABx8<4SSEa981m;&g^c_rqL)yBv2JZZNI{SBxvd4Z>;7eF^Sj zoOBlhaD})6oap97IIYnHTgl&g!%DcQIj%h)(VJ`>6CS;XmDLN^6W0SLok%yF)@ogF z8Mt(uXoql4_Y-%<$%aa6QO#M6o8HY7U(%e=S~Pyh=n#w@)E0Qpw-{5U2TrBk85_gH z*f*(W436FQ%ce7P15IZtEgL<|yZKl1Mh5mv`31bIqcnP+_Z(6U$S*F0UF@nX{h*b|SJ9F*R_Y}EMotlPtdyqo(-ZZFot zv&z!OUSk!P&6x_UnY-pa;4^bKHJu4rw&};)IrdcQ1wW27BUhc7yQNX@{0b&sy}a4x zmPh^XQvcr~U4HeqNbyI1GxlU@=P%pLZ{A}mdEv7q`6<{aBg^?^C$?HUkKM+&&J8x5 zS!mgUn`z6B``Fjf7U@>P+?l%#*&>n3yS4)@&MaCH-J9D?d^{CBC-;|(31qdx&nFcl z)1;?ZJ2R}>UQ+@Ytp%gX=7cTo!ekw9v5r48w}Llq%bTdzHhm-Sh2nhJtl%yd)25=n zb{~xkd_iMrO}o*l<+}!)2u!PRZc%$JtC!z5r3rJVTn+eK;KAF3J>-U+cMPWH+KDyT z8&IzD_+^>;!~N{G^TYw)bryHqAGuH*_wcuzCvJsgoFOIlzVpO2BlB5b8EOBSID-#K zW9Ih@`d8}1$h2>BKYIu1H`8|WUzcaU?dMOzC%@kpEN4=G`n5)H-G%GF?HO2 zxtW*g#oWPTbH{ZcJY&L8;hi;WEN7N$dOnq#jy^s-FEBBqwzhi8nfzJdq4l}k-8Un6 zI$`Exhw{LibKtwq>NCdvmJbKfok{ujgJ0ZcgMU@rxs=!%+yy+Zf z9;7kv(wTQb=3Q#!IB#(u!xpZM_rE3Yo#uV+Q=I=j$$Q=Y_t zc@t$#yZ(yzk&0Ome-*d<`oOFjY*!mQcfLJ8(0OQx@1tiD8(z*R zb6&KZ^&QRqC@>-`!PnxUmeyst0U!?Ec)|K$4*Z_TJ zwwCJ~TW7s*y9eBfalWFFITGU>7qnqR3|rx%9Nx4bJQ1{<>w#%Aaf!~BGg{xoC(^k$ zjPJx#?C0hN9x?mqk1e>i`LV}v3vrL*p1}R2gRH>xXR4n(aN@3fX^jM}8$NU3Jk<6Hi}5`lp^0`Sh9co#XWT%#W$7yZCT+MBjqJ zabR$QZ)b0y>GUVTrjx(Jj$vR$QxW%Ee)>c1_82&}DfS=O8|70zFfel3Rl!lRom{n? zGN8k%Tlm^GqRjaj^g7S#Q!lwiXQ8?{e!FaV@Q|7E9 zZ$I)%uF>03YF`<)7j4*kzd+n+bbBA*a!Iokzer~pxHCBpZf9X{{d|Sih}n_fP|x=t ztKhqDWGmm#y-~?`R%EkR{}%ihurX^O()@yd41FBYY|T-neU-4?{D*1BLHJ=lcPGaO zzOb}q;IgHy1D7sMn_JXYg3I_;Q5$=L_S{fr{R_A|L%r*ncSja5@6;A-xZe%cE~xH8 z`;USrl7alNRKir3{~z-H7n^owMr`9}N1l}r{JvxU%ba@ojQb+|jk_wEUj3lloUQO- zkqIoO0*fb!hu1q(fyLu|Zw5D}0*eKFui$$LPIGeGpA)4uq_6uk_fe3xN)0i-vq)d&%P7!``MKCFyYlC{w4dHL3cbe&_F8uouN+_(VZV=j zLR-^Xf2gw=3g0rooI}kBvA0;9v8*hx`MU9>dkS2y4c1<#@ha+MVfrjtU@UE^q)ipj zS7ew<-pfHgv%Vm%$UD2~!D5~DVxc_^{lNh8?0w04?(5*M?19sMdfBLH$PtA#skZbc zuMr3AHb@=|fEN{{d6Td*H)v1*Qgw1i}U9H{l)f2oTWH#T-L?*ZsMLtpAa}a!uuxG2h9AZVhmbSMB{L z{*&u9eE*KI*ojz81_mib_Ds^8Z@gC>PE0ChEQ+yzC;)C+W12j%BKzy)VGNNQ&p*D& zb$&Y8tLueJ>=%>Z&_2H3y3{WB?jgLFa$iVF(_y^E4zkA*uYL`|&jcn@?;Pt?@jd^} z6z9+6OH_uC8B*cnVel;tyh{iFGMEcpmG&yGBWf0uL-*u>3Y?nuW|Pnd8Hc&^iXU!u<^GA$>UHcd`cI1lsRjJ{wl>GbZz zmpfY<+rS}xpTw7rL*GaF4stJKi2udVL+MwhGB+!b&nu>)r=g5BhHm9VW>D65sb755 zvaB_h6!UoueC8LlrI9#icBf*&Cl@>Nlkn1Oq_bo%ZUlL?E)qOnrq9h8 z7I)L-0{h&^m8Q<#k;{#r85wN+tVpr(vm=9ypA)$Re-}Jz1MYe8Eb6H`HO?s#-%535 zv#q?lTbIvS>b};{>3pB2KGpdO_BR^CT*?qXsvzIRzWXVy-3)Hgq3v<-`bp0 zd)>)lz@K+D++t=z|&PwBsA#FXLxMerx>Rk?qFMi@b$@627>UJHPJ!4g2|> ztl`q3X#C$WX|f}mjh_|yrSYMU#_t_@4Zj`Ud<*BKAG&|33;Wjz)4s9=-7_@lIQ+1a z_0uxm&=vnM^k@%v@b(Qzzv{|G;LL31co5oMvAGzy@_<8u<@eA-X@h`^Dv7&R4 zqn>@8y$dH%D!h;!B{@s&c$E57@3)QLJ2J=k*^zG z7U6c{fY*#bn*B6uzH-jP36|(q>hsG3MrJaw2CgNvKb^GVqjBJxPWgGFp9=#!Pu?A% zO+nfgqK(K9b338K3K$)R!Mnh{qgVrsTyhVv;2zjlfQbobo)=|$ z4>lDUb{Y6sbF$KzdNRb^4@ZUq3;h>C7Yr@GvC_z@I~ljBe9ylze|G-&d=g5Sa`8dXL~bW>tq|>wUf<$=aH zMcO5rbiB#s0Xw|^Bm7H_K8Qb;vXSGO7NF-SEV~~3?;WWnT>AijHGWRy&-l`B`t^x_ zo?9PfIHrsTud}~N`W3;lsP5LTWR5rTu7l~%KaO)mKiF4*{$)mfLpteX=Y8LuzuWo$ zG!nMAF|M19OcIX#f-u&C=)I)hI%i%>zVzocI9=4=o!6z5A3KNNC$~2YeA?xXb%X7( z)Zw-%i#c=y_&F9ltpr~ym_x{y3tjm#xr_j1sE^yIN9zrrwv3r)HI1Pi|0{EfKB+HR zl$XU^TCcej3hexJa)36XC!C8s4lO^1eBtQ}*I5rC2*_;EL*>P;QxYYz=w# zo21eDL2CfY+Og{Edc!D_J;U1SH*j4^%YCdzq%T-O++Na`@%~jZ4hd(B?Dvf;ZRy|r zxOTZ=|7G?TPi{a_AlVSvzmmRJFg}b+$Q>6ypYXE1%oSJI(!ZZKuK9A){tJ~SdY1N& zV|gzEp7D<<-kRg{^%{6eKgC)gN&nYiedBt22kk9@RmcVIL(V;kp6n=k zvQNQ3cU>Ng{GNUYFWxnNc0_OxEqP)YILY~giAILaj64tB>1XBEgV!6B-sV5o>RbOF zFw*{ybP1CZiH-31cQ=BETA$RB<}SXk!R1*w^`}{<6ls4QeM0z_tLt;hc~fXXh0~XJ zmj3YQyj|j*>rMTABR}!l@nhrXL?WKQ3q0*tGlI9>bBsJDIjtM@sLd~uMl>nc#7l3~ zJEC~BfW-HWyg>X`;`2SPC#_Urh7N)RX>B2a6g@$!wu5@S4^nhOU zgy*K%(7uRpi*?wJVr-kV7Hn-ZZ4s>!x7+7RSO7Z&; zo?WSX+p;6W_#b{w{kN(%=3C^*`m+-=B0Ab!1m|c5?Vu-faA< zflr@R{q5|z{{kLmM=}XJxBj83zgtrM%_Hoaz53HhFPeR!{@?4>e`Ymls3Uvs2+o1J z^&iJ~`|j$|j`5%E_no!AVBd?n3*ZOsu%9~8cGp7Lkyg_8qWt{RRulBcz|X*)yf=IG z?k0^@Sx&zL&L8&*+Aoz{ zn>TlOTN!H}3toQbm@;QEb%dbLoI#p1=b^S- z=6hMF%xNZX6*79Ae9EfAziTA=Ov>tK^{YQMhdPL_J)PL_wb5nH9K!Rgyn4UxveW+` zZ)YAKRdxRVJDETxL1f9EAPELb0$Nm-h)bCelmLFE1>9{x!5T$t5v#3AS=As~8BuAW z--3nK%*BNk5J%dALMztx>q0ASZA(a4W)cuE5ExMB_x_xF?=T>Uef_?_-yi4o-20q+ z&-!fVIa^}2boPmP&cJsLU%spLEiPZqW5(I=rK?Mhk2`ljQa^n|Z3Iu%`)@d$%2)ES zS9P?;e)`}y;K8rMi=*JltKiN5!ZyDW+alfA=9@mAx)~c{Uj?#^GgRmHgU0(;!E?}7 z1^2z z-$ne>ihFd36IbHJ<=WpS?g{f>l@pik#pT)Kh$zzJTx2kcH2V zHh_0se<{4XR_P$$Jzid8-jDINBcJn}cxdir6Q6BgPCilkIvcx3|5xdqK9wCL`nyc; z^nI>*&$9>dexY!TE%V%OzRJ7)J=Z|&bYSTz`p9ZcjD7r<*qZ({#wvdLD4o7)N1wGv zZf0O-`A#Y~WB8zl_e56qurBaJr;?$Yu+8xu%o{%ciump5s#bL60d!>>a`h1M(~}o* z#w$i{qO(+|c9d@bcYp`^FOB(F@ot%T5V{xd#{Gr89obnKx(uGyKZhxEpi|*oi!^d) zq{a91Zg|pfz4*p`-N!Wetf#x`^2x<1EGF+`@UL^9X12YY`0?D`J(D=m z%6KT+ zkvnEO2A>Ldv{uKmI76B>aj`R5V|bRMyEmL+jUDc1&Vl)u;qcCIL3k!p5I%Vezt(X4 z7;}J1b2$AwoM+Y#@Ar4G#&*MBthTb;HE0tki~O05wA)>xOZ;AqqOCN?w{;!3A>>y^ zP+U*atQL;E%Lu{^1@|@KC`TDVxQXBf2}haA2*S+<*IziwS4I$S8MwZ}QO+`ga2vt( z2B&AoUtM5t2X{^qhs_2brDKl<$%yQW_j!*|#? z!D2rWx{Gw5GV-I?K8yN42)$wV&-46b)6$Ol$)+X!WX1Ms_n(fYAB*h{z)L8XJ5L4; z=GpB{9CLmKA|nl?y$vjeHfx}vVbIbQ(A4G7)=+c?yjAV+R$_zfypG_{B)${hb>@3) zya&57mz{3>(mUrvi_lzCAL>+Tv7f_-pPG4%O3u>i8;q~*6^yTDo^TcI`6_L{{TFIW z?hXOA4YO$DJbNq8hpP3AFM#L6gY{fs|DERpm*_di{y@*rt!Q_%zMy&OJ2F^vz;g@pN(cHe@76vT;l85&%ds_DdRg)1*aIzPK;l~H_vL+n z_}G`10r4jpy(<76(cak#;=i+a8Z}`+OcL%+_P#ymCsG!fi) zJzTZsuJa6!O!9EluL&Mo#&_TLaMV@)<&EHO_Hgt=llmFlO-YdHi^r(o1lrF z;6}T+1N4>V&YNhfq2R9s-w6H8#D}T!i_a3^%1OJqS9RDf8x+>KVmQ22dNsV&Yx-r; zF;@KN8+`9ILBslIt)cY-`wH4k_sgg9*18nlDoOGdeku2#f&8O1`U5qt5*>=aL|1)H zT)ustfkk#70}Jg6LE5F4AZ>PzAnjQuNL%+1r0+@$%(c%lP~)x9@YXlrv9H5xqu{x# z;Kl!9O^0K@iFQS^qTg-!Y4of)xYO}y)8ckv%^gWE$W9QCN+)Q}<`rn-4ew>!hj>0{cs18P!1H0ls|EIcp5umB3n^C-DXEzfk zo{SpEeozBBpTWR9yAh~8oth~}TBZRB747vF09Uuy7q_K#fr z7Vxd`?N6C|dXR7fVGzM0)OL0G7yF?$&v_oHXMAeRtxVp=T9I0KSL=qJ&~p*~@^6Kr8^?DOuZ2!{0B72#Eh4#+{ zp@}7e(8SLSKF|KCfw}fV19R*j8<=lDYhb}?X`>$g<-E>#J-iY{_SBaES(8!^e{mL4 zJPbeHanJNfZQk@ql(YNpI6HWIGk3ADjy-+|xh=U{@;k>zI+Hxr){+&E8yzFL!JLsR zU+(vCbu;)i-=k!+%_%fJoxaFRpN^+s?PCvnIj+Gabr0ZAO|Z{PvRrt_0H zWDspB+0@6yZG#?%@c!%n#^wvKzJ)f^xn^m|(-*<5XAj3Kl#4rs!WS7?UNAQsS-wQ; z`@ZhuoG;D6W2;L?!grDZvYTa7AlF7;_cQTJwk>)#c=$p1(coiCtHKpi%NKn=>u(F& zj4X70Nyx)i=2Kpyj4}R|TzvJIOnmp`GBMx2k9Jr2eYEKU$`q~8xi`+dTr*`G$GQn` zZZ5%@jN{GRT)zEd_rHDA^{lAEogj&xgoceAM?plxy-8 zO0H?_BOCr?a_tYKuY5KFJ)7cno(~%Nm23Zw=fg(Ep)+`n8yWX`xweWp$+ce_SY)p- zu-Jadz&!f}pz72QImRV!s}9~cEPN2T^?@tX?x)Qq)9U$7d}g^aO*|v~&%qx!g-okU zk!f`)GOaE}rq!j$w7L|THp`T+z@7p0WZE09Oq1{7K{x&fK$Ty5PGy%oyVt}O+xHk) zWPjJdLi;;{@Z+6=^xGW*SS1a?X`$SspG$rnO7rj>$B~ zu9v(%Cew}@UK8$^Olxy-o=ltiQ%8P@KRlWCeQ>47v{#o_4O)p@i0INc^Ai7g@@qNIhp?d}+uP7b7fObm z#!tSZqvtg7K%xM{W8P@Tiep#rBg1 zmuEj=V7@)qzyf=Yfra*?1{T>512qm`$$7dsELYu6xQ;N0P(-M6$3Js9D|Zp&pJ^G^ zKI>1Mo7QzVBTN+SwmYKL@r1Uj5hp`Yk0)7b1w66 zDKgOh0+kKl0(qPCbas8A`jB5t(;-ci0Mw`r{@0Hd@@EIPSwwZ-|5Pmy&ANWstxn-kmyt2uzku7h^ zCA?cM**3D}O}W5(itvg;eM1D$cDv(D*u+D|g*H`q^WoiU{&f65qBj&#SA_}Nsp zuNb+VYyavw>*?>EU_Yh$Yq$r)wViV9;qE`r@sHA23_RXnQ)DkRafSAe4J@#qH89_P z+Q2;fDM8w4o_Wu)e`H{`{kVbh=X~)T!0F`CuHyFLru=r{qu5iYlSQlPo5^lD?gn~x z)SW!HnQ>#DeY=Uzw{J7Bz`oVMLVJRNMfS}==6FVL?#`Mx!UDqg3D*-YCV28_`B~Q3 zXOKthIL~Ax&z?Nm*3BCGnCp+3%d;nsdhlE;9_1X^7_?mupVsl*f?lkJKg)S8hYxh8 zK&@nw?iK*n>RmEP@4zUst5z}zSPrb!yJV8yfs$#>y)5vg&vQAj23QMi1%OfV)_M8$ zl4+7#WymFPMt&hzqLN?mQZygPKT*jq=vL#hq}<}{Ajz#YtQV8qDx%G_u19jKr=e}e z(&%+T#=QoCTE6Z7A)20Ghmo_LxNwZX!6@^0&~pvk1b?2G=Yc zd^d|A+%j-GghMVlvMJBVrUv0Czay9Oj9ijke+zs!i=cng;pY#8qrS5U!u16A9=M}Z zt_VxEPVmYI{%!D*T@$=Ag8PGTly8DpMsTv_C9@`YWd!#d;V9z-BR?r4xHZC2#t8)F zE%_o_ekSFc;FS>>@uF~)Z-Q4waI)E>Pdc(JoaV~1UXCog8d-LUk!AG9U`O`+l<$>C z<0aT~luNQ_frrC>X`)=S!9A74A+4i*tvM1$qGR?2G z)9vw>W#Bgp;6Ka6kH%iB2U$P8;pFoosj_Vi{W93i`!YArOAKVr&%k`U5@_tE9@tHU z`v}((Jo)r?=`nk063?D|^6@;!wU?Hc9FtE~Jl8xlf|RF55ZVX` zLJym`Ype#^tP`ZqYXs@Y^vSB0ZJ%Lw{zBe<2q(Y9p-;kJX5T|b0&E+YshnH3g}HZLOx zCz-WOINHCAAly)Ji-qe0jv(Aba0`W_U&;u=%?9_3aQ(m$gj)t~o^bS489}&>;AG1W zq2I~~!fgllh;Z~_89}&oWWWsJs=*P2>j~~5;f8@D2saem{lbj|M-XlzxO;>f1&$!x zY;bo8HwGL*xMkpKg}W9ULAZ_JCJ1*UID&B7!F@}(ao`BTr8Dn-J-DbJnng|~?ZA9T zc3uzeVk0}{|JR<70=uWyhwzX7H|y9lkg4E?fjbP&!6QREAXmXxDg92VGSHbXXaz4B zctq#%_GK=e^@YYS#(H(>UQ;5NpSJF?TmP56Hs`)H4X;}cs2ovnc;S3|#% zV*LKuj4f$9%gy^U{ETh1k912L_Qnz3Tan+A;YaX0m^<;%S-RW3mAJ!|`nJH{$MYxs z^en%Bj=h_A{nKn9{(l2=?VSeZ*xP{`Gv_hq)`mUyA$L<8fG>`cXAXY*qwHDv2O&aW zeZZ=R2!|rz}Qs&eB--kt&cNK?`FU5 zmS>%T?DI3Q$X+W*8$|@+zcmKWJgb2@_DTcu>|Ys}Z@+Ayd|6+o4@S`!SJ5Z`Mc-UW zpN(V=)tR^1&0fiXkGWvZHLA}DRMrMMSJr11RMuvdR+e`tuZ-b)tzaE)?{k9TaqO+D z@IAH=dEQ@h=bJcR?=fuG-bWIvMsn|A3v2j?S6SSP8W{K(_Uj#S2j3@&?>z7^>{$D- zgP(NR`@Ue{W7xCJhaCL!Lk_;&S32-9?AqQg9^2Zae|XI0i~+Lk-O$RWjPikX*t)go zlq|~7o3dq62Jai~kC;OlGY=+K^#=a&K*HSf7(Sd>y%ZnF9LiY%PI1kxPF(CzVzu^5 z&7rKr!6`1wjjJH;8RA3}q7RkhDSb=%WBZ+NZD0lI#ttS{Kc+b9km<(#ai0@cLEJ3G zQT7UOCjY%|*?=c&_c;d=t4ngNd!G9!G4`W1X<=il!$Zc$0B;yuy%Vs+(RW7wW$=#N ziTbU0K)Me-C*RcDYaHG8Rp>-MY00pQjE%+IH+(1<4S>Cyz za#NSmO3gRDiY}}7l@B~b*`oM$(&+zI(p6ezlwl|JGI;vf!)+6;&k*B-j3K`~KcxH! zW6&R!=#N+BS0o+NzO=`^{3Ac#nhAr#E3toTw6(uc}_6+X-T>{XQviTx|Vy_BYY@j9LjPy8!kctZSAN15x{D=YjR?4cE})IsO9 z$T9ORYq^CpX+YEU=o#}Z`!JJu=s8MVgl8e`F+8*!ZAz?GIXs#9HswMFey$Clq3#Bs z(uVRIczsjb+G$__IykZIAJu=tr~EfF<-bk)^9Kf?l@tH_f&LRdye{~=A8i?%w+<)KHf5NBycU{VV9-T#Z_pQ7KIe-rR!g0bUuI|)4 z`SyJ2O+PyC*!rng$!iVu>+kxsFElXMKHtDR`>O`#+vgfsVD|=U|5z*i{ujb(!n1_? z3D*(IQ}o$+Df+BDMW2V(6|VC@$?-Wn>$~9wS1{1XYxI_I zVPL&gI#6%3%avLf_EVNy^0B#A@TBWHWk{~Ug5Y%O;V z+3e|@mc(hkMK)QH86*85iN|J!zmL$z(ic+~~#S-A1`Dq+MiFUQN4P0so5L=R}dclpEa=FXvf(^T%CI-?V_QMHbZT za`0I$zLtKf-|5QZozAx=4PZ@tDe$et@;4mw` zx}P(y&^n@5DbHHUs&z!o?8nhs2hsR0p4-e^75+A!kDz;)*3C%^B-XC{=v#%{I9?R ze78H}^6=w4D}FHdXLKVx8lJ>S4$J7i#y{iK0~_7j4Xb*>;~on!F1_M?KA1moHlr7_*<)|Wj;{Q}rb z6%S3_{22FuH6x!#!e`5YBjBq$fPo%XNc#*3-K_hQ^FSqeXI2Vb2078@DfkUgE3#3753Te7FSfQ$BppJ~&> z?0c$X4pp|2`c5*Y2Qs;iyGk8<$HB>OP~hH68m(l{cpvPO*URT>J;7oAbLO|0hv)w; zeEV#B>%CR9r{>?arc3sN;|h<+UEf4jM$H5FIG88 zJH|LR%nlslk&-ghR z>qgPH-dI=VxXV%tamYrg@0&J`!~bEfV$aP(5Kz5gdpUG@TH)1{%u#5dB}%_gqE zjvAP6?=%pdFGzbdn)e*L!N6?$-v(+-^2I*1)6H8;K74|F(E4V{2dxD>h~HCsVhuF1 z7TSG{wv*p;6@4PV=Nl$(&H@2?>zrHZ2r__xor#xCzSEZhR$FX!JD_Z1Jz(gAZM!)c#X-_^YyE=4xn z?cwCdr(8p63nTx{zu9&Z{V@@o?0S=4wd1Fn4esk64n6JZ+L0bk^iRKS+an7UKUd;YQd?h-|@@R^mYYh#=7BH~T{+EG8_J=_B@{XR*UbK4&R}%&j3JFKplZJjYJTi%W zujoee+=1uYU4B{4zO*}Bei_O0a`A=ugnn;<#?2la>`$`?rvnhW5}!$btCZ2>jW>)u6>l8dhx1|*2Op4cPwMmIB#t(qOhZ-2B+eO^PXza)B+eO= z&j#n%^~%HX%Ps>qJNX@aK$~v_Hw|1PaxRs=FZAdezKE4&82XMO+lDh1H#&kdoHV9u z#24CBt+Oe+*?SEAbhE!tJx`?X)VZ)!`tIz~cdhBiZuYlr14D1FyUj;4x7c_Y#v^f$Q9l?CdaQ4Z#b7ZIS z^XZHU3>kET&RccPQcJZbMYor@c@Hv>y;lb2*#ivBxBD4bU|#^#x`1qG@&J1t|4Mj; zuz>J=f@e1(i^tY^a-HWno?PeoF;A{@*4Of;;Gh@Q3<>XsCSRbhc0-ddK%2Xf-H$_? z)7Y!Byxro3lcsXMs=HUr$Zcr3hCO4^$~0r+z3?lCMmteH&%P@(v~s+Cm!8CF&b7$Q zx$iZ7=-MjK+ED7(oWwz+en)=qNaCQ;3}>v=0IrepI<)85C8BT1)@J-g&G><0eXY2W ztMH@bZH)V^kIlgr+oKHK72E$>U0)<81DvDpo!uvT)A&X5GS#oV#LfR_2Iks7 zH89U!Xdrtofa0Ao$iDuBTtXZ4wuONIC$f+|xpC+vQWA^|;`uN%^P5@0K~Z=+aM#75 z7_ef;pwMFOU@PrXw{X}a4sFztuGYnuGS65~pK1Q;$ZeSwHPe2$Ia<~(tnbxtf!w-< zHD#L?&f!^iw@=r8mh-lc6!mJl+4~$k1U)oM_hB z?>i)kL#{%L6DjLuNt{Eov%y^iE)LzPURFoWuA4RMQl0VF;GcPE;|BKQvL{!v?&y{^ArNLC$_n!Mt1^+0GUclpJUx9tLiOaW34a~E<8<=Z%Gcd<4GBDdNF!0o} zE>>1^*D-mQ#`&qAKTE*#*9COmM1|us)fiD{Mfyn}n}1$ls&h9@*44p59p1a@)ic_0 z=KL=L+^xu(C~VE4I%mEx_`qzS&diwJwdTlp=GL{gYXUM|x;?bQp{E1SYJV>0jX_Vs z-$i?rq+9!6N3IdoePs=#+&Pe`v_cn2d93QE#LStxGV>^4BTqr z+Q6OZ;5LF=A>1u~Q>IzGZzrq;7flNeDna%Jg6Og?i=km>4p#KJ=@xudoZ}Suv}*G~ z#@b8H6kW}{G}`rYN6sem5H`;?0nkg;J>YiY>DS+vsUbiHp(Hp`@eZ_!#>gX z`nH<<7m2QrlkP+d+3h7MtAk5IS>uc%5^X)vxID;&V%@c!{!_q0p!?5g6WMKt* zvx0GXfcpaJr}qMxZ!L7@=$Q+D*`eLbDUUbz;^?2_=Uy&`24uS!`;PHKS94`kefu*lCAcnrl0W-qaZWJvZ-n49vCv zY#?WW8kldtWnh8*CQ$X5!gzZuVE{oq@epmKHPyc-{yOe-swCv;ZYOlCM~8D6Z+mpu zN6*lq*$cyXqa50kT)0cm=;9*Q8-+=0oWK1AP0+8I?}H{qgXQ_m`MG;ghPrh3Jl}eB z_ba16j;Fhy8eFbD+AZfIzHdavHbCF%CqFt;v^Z1W(H@Tx@6ajq2Tf}3Zz%B(B-55I zbk4*0UJ?flLU*Fe@47hc3;Q|mhknFfSI$(^-tcIzVE7WlN5zJZEl&PCt$n)RL{M`RNe`K0+=hztra{iBj=vo8W|7&1@op{>Gek z7x`DGwyei``X_}~t{Rrxlh>@7c)-yqE`-wpiZ z@xiq`8$LLVZK89JG@jSG9cSN?pZVlt>vpuhf0!wEj(xd-xpuXId3Ke7`SxI-@|}Wz zb}T`@TCGn#gq{d^z9!CoDT2;5_tSyrfU5_dlh0Oq0DF0)o}qKroKD_`uBcF7GbzUeg1(ntkj*}mJj~n-xQXEQ3P<_O+zdGBg=XO> zlbM?Vw+!44;dq`vI7-mCdn5R*;G^V`Iyd61?UqkLwi0(9^W9(hE;%Pc8RV~hH;ID} zDTDmAZ-Z+j?^K_{52;@O-JvzzDK?R_U+;4^QL!u6i|yafcYF$`qRDAVo2c0Srjd=T z|K%T*b0zR%%6-fx%C}!KaqRgwFxR#X%(3eY%(j1N;Fqw4{ugNcM{eGa8_4FE})Rlj`Bp{I^9Xq?bzb-vDQYA{CUEiK=b9ok*{c8`HSAmgriOq2*QcBy9-DCCJ;Q{5PcVb zKOJv~=FjB2RNfd0E;EU9cw-{CbZ{r~#u{+{32#g>GO*bG?5Qu$8_&A`e8fL0=WgJM zys^u~<=Hz7gzpXHEC2(!^UuKlUEYvCS7Xx?<^P**-oH06*WO@Yp8dLk`Sv=Xd~8#C zApZ#i2zi7V7YMgK{Y`* z@nD&7^hY&8xY^*k3rF8n6NFm^PImn*mEZ`%Z3LGi9DPwe!rcgtAl!CvyM!ADj*ygX?7xy-@5#0~?wW`b^}hCWZQ;A)WE;5m!Nrjy4&Ie- z;QypFv!PM5UP=2}Sr5yY_lMN;%geYz`(mR53+>;BoO?)4rT+&f^?#xLP4}PG{G)tV z0+09SqW?`C`rknGzk%p~1JVBm%8&cSYp%bn|4lZo?h1^bFz4#6O@G48f3AVK_8bHA z>_-jEw;wjJz@7z^EpQF~wgH4}!U6oYbs4N*JjL}z2h|2J2qunV<%xBOe2m<-p1aE41644$YRb-^YebLKN$ZL_k};68H_&; zlnkw6PnOEC8{hEb?0 zwaf}V*C`l(p~{L!N$bz}s`qe)lG1vaZ~uoiiUIt5of3%wy&3PHo5B1U>phkC3&6!4 zxSy4Cn9uDHjGyb`nO|GAxRW(`I(HYIOWJqxuJ6wkKHHkSo;wWBg$`d}-M;X50vG37 zlP%%li`_mqec<9^(vO2L=6@Hz8~R;bLi&fnm+(LDO6LsrNU7ZNuNAv}RA^wHoo^uP zM-0rdvklC)yBK(Sy;;k+K{mLzPv(xP_M+wuzOc^g+3&6e+s2&FbGBn&rIW5V);`nl zsN?U{d76HAuhL<|6T&-ZX7r@&O-Y>N^BfB9-$|Ut(~cbfdlGkSzVI*L${%G-BW)qx zFXLaym!GnxQTM={XxmID?qa@)l~sjfE$LzTLd!KDb<*|Z#rCB}zia$mZ2ydUDv#Ia zSx@Ueuha5e2kw$_$Cb|goVv5q*-Ob>TioNh6{IIW{7VMr+AkQGXWItm+w}$(*uONe z&|U^qpKpe?RuQz;SnptUDHuY7gXa)*V;ChlAn6vt53ZESv6@>4ye#|Br#(|6?Hc{}{;qKL&FD zkAbJBxtp}N3Hkikw5gjXO}hQ?T=t1fqHjlPzX*J(bLG^p-ORe-V&f~++Tp#c6AMa@ zCi+LIV}N>vf%4rOA3ii1lx6wYz zE1mY4`6Yacsb@%gwsh(l(g*S9iq=~V%(eF$$Xo!hkh-)ozWWPdHQ`x8)IG1R1AdWD zIIm7}e>r{;=F^N1y-LqIuMT=`!CzR5tdKv%*vo>nUAZ9b6GcZEds&datp(N#@A^ri zoCW9kN$zy%;vLGx;%?eH$NsyK1-e6n^C(?;wh>&?E~P&t&t4V|onh=!aH0!aILcz| zQg9Q&{X#hMH+Ctw+2EE4M;})c#AnOEEfS9MRuhEV2<};MC%2VUKKV$TGo~NsJN;wk zlgK|E`|we4jpXb2LmYcZ_D!_C{2|Br(w%+opYx>`+HHoP3+)-SkM3o|9(wm=Yb;Xj zAxAG2+LKM67ut6N2T>RKL)7=0zr4f5<=V9d=GfmhFx#GJVBu-~=U?U?#GT9wq^_kn zlH%+0)}4)X^B-J=LXGPK zY{6$;gAF8loy+rZY@iBkAagdq-m!s!}kJ_0w`Z|!eP`^iGduUY;&=DeNr(RUGdjQ*T8J_{*FjW*ucXVa7IX?sQ?cW=iXKyet*M8l=9DALC+4gS@l#TwyciY9vBCPfCvED3=b!UFopQW=7 zjXUNZC(7w2eMdgvyt$MF~Sj&ir%;4*76_brSajE#9S@=`i*9%*cWe;Xbd7N3qxJkr4z zIo!z?i9>JY8HIBL`=@S>PU#Y!L3@{2g{;f@$>-MP96>%Fp#KZp^*P;K`kO}DMt+iZ zF?_E4?)CC}Y*^gNJR!f92Tfkz{txo^uh{mY9WS^gLU^+I%_24RLE8#S+ zDL!a|u0-!ch;!tm_`o^a>N0S;a~wZOyuqx=n04u^{({5f{Xa5meU|#u-qSxTT^aB- z^8d23@B{asukw%5IM+b#{{UV>d328XO>TRg1J3J{sK2mxGxdsgu;PoR1s0X3Tb-hF ztxxOtC%U&wIOZ?t-Q@Pg_ornpI^5JXeB=^e1yTPpm z*P0(*z_*9C7K9Uft+?*y2+nBtVsJ+Ki{dHY6T{5i!D`R#wAUm4^ttNCCiLbG#*w?w zH@_;O9@uV2#*v)_0gRu$M*$oZ|*2+>GSGcu`(ZWmOJN9aYs((MKect=6;sW z?BB7{27i#q2)>u_XS^3+t>vz4Uwo(E7vG-mV~?0+*Ov_$6z!K5F?&`Mj3@ezs#twV zF#ck{|EgH~&gJ1q)9{LABZHBbHdKdSesf6pwJ}309GlV?uSf4iJNR_x%RF!WTs`u` z&;xgg#!7tgXu4H-$lt+^XZYf+Wxn{)rnADIw6KnHbN8_JEFSFRV;_pM-g8?o;?WC- z{Tk)r+x8+uK7t4KAeVQb>$df(!l$Zz zi8^nziZ#KU(ZBgy(C|0Ihg;eAT2>X0XH>DzgY|{ZJ&5xHq2ZB|Ol$IUj0@|h`o_Ek zjaB^j$cXI=jFg>njjygK)3>;Yd!0wuxqDaB__i;7#yDqSLweMz!2eX!c~NFe|m5HOZSwLhvMt~*BxfP+PsPi#&xO8Q{4XvQ3Gmr(Jzfb6syN4hPs_ts(v&qPV?dyud>YE+ah2hwf`QeXN7Kiuj zvf}E;#y72a%Vx9wWuDbg8M=$S*JHy_rg_$PFACkxyV@_v{fy8H_ohNijQh-U7u`#5 z?T@i;R=lC_V%2GpQU6XX2;%25^(t>4x{frmh#P;gn+Elc?FzET-A(5LC!J=}X>RpJ zIG3q6@Q9<`67uHUqt^mf9Q*dW_8~;Yd2Tj4Ua0F4#Y)2t=jw^x{wtzMsyyPv(97#i6r!S36i0rmpYe1Blm`yEb9rvGxl3tQ?iQWiz^O z9c{`!*(khc$}!s?>Y#L~V=3S4A%C4~zU{`FkXf9;xK=iAzP;z~pDozK`#R0hLu=4m zf70D&&F&LAI+wMOW6~l)?zc01KEof%#DDP?KA24UZH^$cHyjvfhpz%?zMJZl(X55$ z|19!VI@;&0eD=M3J@j*9c*om}0oUCCT~)Cc)@{!htkA#6cRFdAdi*mj^mpF%uf<(m zCjEzlaiy`l@+OnU-b-#YX*6u1ojlulj#Gv<=pjJ<`u|k_9D3Q}9Ez&=p{apI>9pnglc-f7q?=kwJ;?r@P!_+tSvh%F=C;@#eIwxy|T=<_=ZyHa&Nu-KqDM3g4n}i>=Ac?C+YMHsXO+ zx9%IpoJioOi3=u-TQL>6_~7!>N6vJ=U$rYo%9X zW%h=52SyY4@t{#;0J@3x%uX9*&$1%x2Gb|V_4-RJ^i#)*DDr&m(st;m%!(x=Ep(J0 z9hHWTO0RfjoG%=utvtOHMvj)?TWjiYU08ae>TJn7_RHp1p%1I#yJ!pRtop0Ito+;l zyF`B5xUOO8wei$ZV}J(A_VFUej`;yNot@o4UVB{Jz2LOpvf)km(#5HL%$fNc;5`?2 z2e^h#KJ1%r_P<@+t-_&e8z}GJU7Yf5cplpw-S-z4ccXA?z|l7ExVUS;#rob5Zf=6N zRMt(-%O7^<5pSpG`W2`6nhw-@TDH$>V zS)w#!n>n|8t`#q_a$6>3TVn^{M;Gq|^v|PKJa205BDI5LYdQbu`w}a!#m`y_wT19! zr{JquKhZ#Y2;VIQKTB(hG}fD?G&2`niZ4QKl+8DH!$U?-Akz))O}r%Z?qiP5G4JV@ zhW^C6z6m-yhx>f#2l6oTqlY6Kj?+2l2=j0KrJ>*Q?|697ykB`)D8jpBcre3qm1n-YsK>9K*F% zMb75iE&OjWZWS#oMRu-k`{CwA^oQvAXV9#4zZ0L^vJ@LfH2q`nqO)7jwT8dZ`L3-q z&-!}5(9Q1sy;sq8vUM6(2IEig+yFh#6?`5YB?z5ADhS;_EVvjSj^NT@e1_nnV0@Y& z_T;TV*DMf~5-#q|^JK`UOLmX}q6-gpi>Dmu-v*`g2PkHK~` zG|T?ETn{Z?fS z{?=MQ=hSe&vl-ja#?Dcfnh>{HZe3h+e+` zrNn|ImQ`uu#$FV9fN#fRYpCp-(sHwq4Us%wW|oQHcu{CfYWpr36dJ|5+HgGmw3xcf zKL@;E8}z()(q{`yT|dp39GySC=vn$s_Oawj8@9#YXq(LZ?+n7OtYrO8{|4$m1Uk@~ z>m`&!{rF?SyQK%Y^MmIA-K!WcR#f(qyRvWTM&;M*u zthkEtm7eR0qFbOd_V-1z7Ze42xrSC{k4Cr9jK|?& z{~Ud=HdS?`pW>!nr(YZjP@cpV%E0(3@s5S>kbC>4bqObyTk!&(;n(+>^L$^lv=y7) z@cI^w|I#PlLc3LfZ%2J%;3}pD#x!Fu4`<&_3-)izhv|{#OVIaQ(``z_!Cl)WG&_(3dh%?vEy@ z+^l1`Ej_Y#66L=;JrcbnEgt3H=wRShU^U;)rJhfwN7P;|^Z8~0X;IEv_5(++O^ero zkB&-Tz@fZSy8=_%``$A)azaHv`91MUXB1xA|`h zo3YZ%oU!r&^M{nXVQnz7;2((v4f9<&)A`cSiaPiNeI{9|ahBl^#$YyhvoDl?^4cvf zN^G_w9`B1ztjwwhlvi|qU6I;!0AmN~UyVJrXMlOW{?WC_ZpH|8__L#wL(kqad?jygVcp8DlvRB8z?lx8{ekbqXTKA4 z`RrAm#b;4$3-Q_A?zd~fH^5i={?~ejCtn191>3?a#~r*IdA!-!H7f%D8Wu0#Z0x2L zfjz?LEA&pYdSFRkc1R-_C6tyjL>rQ9FzuI5`$c=* z6dpVyeX^BV^FZ^`Z-s-IUw^>L9PvQFnmJl>wbWPhK&CZgbTjfN=o|h(4EZBmU^X%l zxykd9^7JL6B|G&T7z-YGr01G;;E_psmfs-IqqKWW5Iw2!&){}JbW2^4=10*vlMnJN zyTIr4&5A(P-Hkh$F z((wDNp{?IRU*V7RXyiF`u;y`-@`f^*bZ#GM%8{wPvB>Q8MZpJ33#(`g=Uj@^@4asZ z!#A>Ln%6MLi#;zL9q<+Ru7NjVwC7-GP3Oh4v)Dxxx7D=Do;P-5VYZ#%~pz##(s6HACVz3$7g!A1AnO zNc@|E8-~Ph6nt|?{CdH+hs3W1#_tct-~5ENTJV^7RJ?Ps*0FeU0si%EO*lLX?}``e z{+&qjTn1^Beha?UzxPxcET1)6cCYME(U<6WF!rfn z^IQx0n3biz(RwT4Qt3tShW@8olUwK46-}4yE44y@xEDQ&U(LLC_J!Wyoq4lJoj=>y zUOljbWkcd~9V>bg9hDw?zQD)xO}Ar98k;zse5V&R_qriG?$g28m6Xrrw-~%U*jH-G zC4Nrg>CeG4I`22-gP#Xi)i}@e@8B6(&NKWxc*Y3xj1MA`E;$Gk->IyZlE20gqSt(U zfB1was~o1jpO#EMOgTze-y^%%)H6LT^!y{oWSi0nK=-lb@DBYJ%l>K6I0NStVN1O` z2ELqzjEHB?DH`mib4&)5bPco{-Wwl{Qy!prqlA3h=08)kmH(3Yp5YJ8IZ3{<=QZv$ zc|VEnA|Ls8#?g=1tz(h6hp^REp695~v2t|pnwvSi_hIZLQ^y8>=sPFL z>+i&S|45&5=dM$3VDjH+dT889{>46)oQ*WkHF77bh4z}}wv)=2+CHBz-&tJ?Iu7sM z=9O<>yU-OU$wOm3kA^G!*60fK1$C}Gd7Zm;&FezjQzrC=u@5$7S_AhEyD7ZCEIsm3 zA8hen>EMxhkt;rJ$WQyU1v$}Dj2&8n&0393I}BU57q&FCS3b4NqUM$U$o@6{i0u02 ztv+NOW0hU#vlgE_7w(IM%)pMFWnC!wJkDp` zGlDt=x;(b9H*iF|#};Ot{l|q3&~%4~x-7bKYNtg{Og(eazH0IvMjFulzURqrIeENH z+7tYd#BKgabPc}ZwZ8c7b-wtX4dCaJ=ab;!rRaQrWbXpMV?!_wh;_hb?vx&B&Bm5q z$6QA~II-L{;&t|y)rMcgld0w9L zNSpMT-wcH204pYEgi06}>YNzy|KHAZ%l~R1)Zq*#%_9#xX?`q7o=R&s&qj|ucV3A3 z5u|G;-sJ6(Kq0q@dXV2e_Z@q!qXn@DZlrx#**H%(pb+olJ#}y>>9?q#s@)p zrfb}W9FRX^2DXZ6o8ehb+q{<{ zJpWWSq3R6}-z@w2w$BzUVII|?kF3y5d?WdY++RIBA71lY17tsif!NpYdj1aCTLsid zHoz#LDPxzeP8l!n5^A1h_+fH6eQ5ghxelRb@*Rggt8$;MwxI6y*fAR0jC1QToBVZ` zhLO?8$G4Cn=KtXxLytilBL|8I{sSlBa>GnZ?m+3!ei%LFU$EtJZRU zg7VL~^@{kt(8wE0K3g#T*NOgr1y}$6NSz(irT+a6k;GEppy<-vkj`O?ZpaPAnh30= zi_QYh*E8$g@vFxCgff3PS_Z7vGxxanx{<(~P_NO(U)R7q<2L4fr89%hTG`N=Y#ROTj6Ww zdZH8YhXemXAD-ifF8RK;18WNy7fhMy58cWcvu)Hx^S1}7kM`X+QisFn5v`FuggvyK zIo1-zm>r<&o`+Te;TiNAbIP{W2lo^h-JZ67@?Rerom0z4VLr&=|U> z9Xz@h4+$swXU;qx#YS(1)+Adm=?XuyHp3goARFUm{IaTRD2@F8%D2tLk2sIBpP{E1 zdr|an^1Gaz(BWB>leEf7i~A?!TUnub;cWhw9q+Y+#`Haix2SK4mECeSauvVt3ajC( zp@5}v)Zh=0XOYao?4bic!goO^P*JgH0_ja?I5%Y7><{^RWP~(NJDa`r>z_&V-@Y_2 zbkNXTE^8u1bDU2F&9$u+9p%}7`?1rO7f}|;{q3{zSdWzpt?`|qwLJSB_nW?IZ`x=3 zgp_YYXDKT0=If6aIq*S;Tx;K3{uOKApZ6oj05YnMhAZ5 zTebDO7IOsD``sSy9Km<$@qPN!92*{JzL|GU=x&vHs}nP>6m{zo)wb5k1cR)uUekR@n5uS-NNXH zti#5a6!i}=eyQJc_E3+-pDhStd(3(IvjrAV{cw%j4>PHg=vwV_AI~}y=2ia5WPM#L z{c3b|5B9`qe0s>SXOhoV^b`0%=NY3}n;P#)|9f<)c}&`EpLJ(Z$ajXbW~&?Ni|)$; zC;9I2{E*L`@9fHVqPagFP2_v&$wzXW-1uj1=wr>Gikiczl_7U+hMZSiquf%IfVY$Q{%ujgKO^N!xu_> z&v)~|-kJf;8d+Xtg=DjNd1?;K;J5igT~)Tzq_1^VEh*`b!v@{P9>xSXL%TPBEu{8n zc-xAfL0f4aul{Gqv%3@hJJZL~2du5?Urw2$@UOYg54og%|69o6+jRaD-;|U zL7y2r#^BQ^_6T?3t!e;HM8E8S*7{Rk&9e<>ZcprRHfUM<+VxMSD$8gz+tPgrzc_A}m zXa47TnLhhL(8+@|YtC`fzL2yvC(sL>+K#qvoYgthwBgKGAPd+luW_hj*5 z03VZN6XWR0uhCBpv#kDGIrAtB9y9uDN#7Lx#W+ej%{rekq#Hcg+`-Bg+H>zTluuw7cQGd1r;Tf!jHgxe?a)dAyV@ z+JTpcBInE+9(bdQ=UT~~IVB+@bAD11YG&LZxguE;9CUm4hL_XtGI61^LPnQ-NIFZg zN1JG)4}IYneQ5N@65^p3?az|lk}i>c|7-^L=3twYKIF)t0WK}xO!+(+bT!W_OR*`T z$>C?&SBWRbKVbE5RbSHw^*V17_#S#xWxqlFi(H*U|Avw46|V1BzMs-U`M`tteP@r( zv>q94_*HTNTC61x(d0VP5ltc&a^NrgujGNPrZqmV5~qGv8CcW&e)P?t69@biaa!x7 zIC$s%=ms|q$mTFpHpSt4d0+XNG=WQrQ(YC8O`O(&{0v+a-)Ia!Y4rK_Va?KMVH@pCEKt5nZnXw@bRZd`u|Tu~V;J?9`pWY^*O!EzqQ81zr<*#Yw97q|cb8j- z4AqaaD~*HGd=(nAs0@uBKfEJ5Dw}(NRGvJ0=h9&O+wjKvN%X1sR(&d)8Yr7s?R_zA z6GgXe;$E=-R6fWSGO_@B$jAoLt?A*UKh#Y>gT9u$^WuPU)}q zq%B{rXb!*VadeEY_jp=Op<~(Q$I*BS9q%XZL^|FLPIg#y0%Kc4!<_$YXxJYX?TUum zI5X9w-PqDG;e(P99?e3l+qM3kvDHrKWfygN02^-_b&`&fJot;&ien3H>SFabx?tO7 zto=%Jd>&Pt+3K6KFc-R(jea@w+a`KQp@n)^hQCf(MQgpkK!z97SE?i9kK^QcJ#zd{ zKgFMOwblPF#u5L&MMkbnJy4eiPmGJ}Z0`{W!S}|FhAB z_yIlMmfnb+p!;=}fo!fQ`ahPU|6A$L1L{xlHT%&dH?E>DmG?06nUDWP-za_)_pK}b z3OD{J@rT^F8rF*{Zm=8oGI1yJ%|*mX$0X%=iu}KTxZ`*ug>QPhX(stCg>TA;JCSd? zgOiQPzN`3&vj1SJ?1x|4Qe^)D#%!rFf5)RE!`s1))T@PdXJP$3pJoE0;-|8WrnI~U` z$20yEp0OOBdDpdp-mN&n1}YtNhp~Z9!87mnIH}#koio$$tH3i2TJwbse&|OI&y=O` zOnwT_h*!in>YL3ebXkXuC%fb+`be~wNu23ha9-S8;?%c_Gj<6yiyT;`waK%I6YT|v z^X!ru#X+A_i4$HniLsB^?;-sC;I)R$0^hpS8YI~reg2woY%}yQ%i_!omlmc;&lj(<)7-R`WLhvp2io}{=_T95wd1ZJFY%)4VEBtPkGD@$?(E~r zoqZDfU2^{_Xt?w%R{y=r|3A?1(+&-{QLiKoKYNUZv!&;d`=fu6Fmtu?$B*Kf{{GKs zcuUfDKLrhE+np%G59T5FyE`;|g-gTtsLj>>$o+5gY~((3K%!sy1JpO7;~qRiTch>d zoo7RTDf0AR=r*O3Dl_ZUf6(9u#CiR9G8*JR$L19cLK}t#OU8tEVs9RXKD3@*c^m!1 z*t)KxRoUv&W4-c#9)C+7C))ZZf6DkuO8%dtAK73gPw2T&W`K4?BL`f&LG+`3OC3koW?7YY(w0lm z^&S7;x8(|J)dfbE($+2 zt=S&@bgLQLuiQ!d@O=$)F^7QKglIks;7KYsdeo|U6-6~^;g+6c2a%ix2Tg10DOq@sQqL+cy-$(I{+E@ zJ-2S!-=}nqZ;^J^x<#K^yQsO7ly+8`-FD6--v;!C=|9q{CC!6C>FjZCx_Z9J%|G`f z?RPbCJ1Lvmwmtc)z0_{KrUr`D$0xSeVe-7nO-JRms(N%+dnM^%@yY4o->hk6k6(YS zlWd-l7THZ6yBV9He>*i)y6aW&O}0GBoC#x-+gXdP@kwkLYf0ww{erYe!zkaU?5kwW zH~1R~7_0mtkT(_N) zv_<=wIN+JYrP7w^Q}@3N;ym3c+G@m}_u8O|J~H%0TNwIc?|N_AO=+RWCu?q9a>3#6 zyx!3)YoNjVkNGqiM^$G21XTh!nllJBR-`e%PPwmrOVg1FC#lM=GB8xvbCW}3r zsN>ROdM%6RFVbt4e4ESD?~0x%i@$_!d+Q3-NkOG+JCY(4JiwsR?;WSZK#lxK9< z{)7H!iZbC}Gq2Lbd<64cOE^bJ{_qy)tp$49Q>}dgpQpFo(A#e4ZO+t=NqW=TC(+y< zm*#xjakOpu^vi7aNi;)u&Cp#7bhjsk?slJ!?*7}MyC{0D4Z72O`F);^hs>J7vmf?Jkvp z`@cy?{@GsqC)xQ8w;ZzdW!Imawz-24nYj}gx(nK#0&gd2JB5Y=&8!7=`=bYEJZeAP zD=K#j^Znu@-~hMWpAe_DMkneL9Ik#z^&9#-I=sI{vX3)Yj9%$x@527|u})x1sVg(Ztco5?YY7o zx34&H+@9p`cTTXiwkB=Y6(=9N=h^Q)$QeJ_SJRoRlC2e(>+pAf%BOZ0f0y&THHE+1 zuw5^w-^3TS`L`eG7+nkv z8hHoq${JOmv1GS&h>l2GUImJA}iXN+agr?JmNg2Vqpk=*TCzzsJ zVzh_W&2ExU5Z(x2hst&ouAFsi;zO_QW2oPD+DLUw7vIzV<_;QYxQROLXocUK@PBRQ z&eT1QJ$_>OH5R*R0dwN49r%>9dPLKPeoyd4{cq6pc&+#8cA=r^beE6v?LwXjDcp$dMq1#Mr_UJ6;97ix8LA%^DkhRJlzn5n@{JsRc zKsK%H)Z=)G{G)!ej%r1ZoO43&Q4ZrTN#TPK{~qUmPLTnR5$DMOXFi7U8gifkf3f6% zBLghQXM!GRdJ;Vi-<$b4cxV@6sekTBEO?0Z6<(RmSPdC9g|sE3G*&yYOm}`knQl!f zlks&ITShh|H7@ z{SXzo&RX8_>{Tjd-dgKM+OGXA=a~68cQB1}2d$9qc|7P3*h`>o>rP*21a!TG^J~oe zm1&_XcrWEWC|$}Lx*>{ZZ?t)5Z}is`&pWoniokwn{dVAXts@Fd>U{xgui#yw5Y5qNEXC^)&)=$tpuSpiq=xBiU4@I#;B~=T8ZBc7w(W+HHhDE(0R;4V-CBb~3uQ}(2%cA!C`#rvYoX4CqGiTQK zHt%`Qd+sgkwJ0-a{Q`YJa7N(VMVT8jIlqnVOpY&U#%AzZeLOm+gtisqV{J>Fe5#o? zEV1k}sG}TyluMpwU{`zp6Mhnv8OI?fUC( z)yd9R8|lYq%6mTNg_M`?sn}eZJS~1ZQ+eoHxzZ?iKINFS5uNQMua6C0AD?jhzs>5} z#n!pUD#6H?sphPL`DayU({~epLqW9f+pO|V9eqcKWX>8Kob8VNJF}60OLcddA0KX= z@U_<&c<2IL!~?Sd<}1z?y=dQPo&OH?HQdS`a3wek4=gDkZa)tm>g+-T1^>-s>;l-k;{({AcrD z4=b`~n*K|7K7oH6(LX)hf1V=W1<3twKk!>gUyiCR?2oJDSIwusIR{v zU1dE^SO9!6b~+EZE$^-2@UblCY~n@BG`A;~ahq4hL*!|rj2gnU*fMTA-;|L}yw+6r z5+0^aYS-N+F5UTw3DcZeCdB`*2~(UqOvpX2CQNc>m@pOJer3${CCvF4=Kf+}U^G7J zN<+YeeI>B*e+Bb<U)5)1`sZ{B4%E`4B8L# zuUX@POS||duc(K;i9VGPzanna$cCG%@JX5M_`rwVtHqbooeJ)|7OjZQKf=qu=jKf# zrxsa1%zJn0|wd$&<>{o2NTZ z0#6s7xSx(}$S<<3BJ5)~p}VIqVf1zI>H07_A%%x!({@F1ee;Q))YhM7GGGYcW z!nww5#tfbB_(42BrcZaGW7l_>as$O_b|Yb7@jzSpq$b9BM)4^7GsaWA7u)pxMevk| zE-u=z#@8}zqc3X)^8du{*WJG1ADm_Be%;v{zBBgr;s5w5epvspXAi&qy1^SB>SFae zh%d?g@O5CgxiNf`~DE7$DqR8rg>wp>OE}zVrzh1HuYyI&fcr>43 z)B-k6)*pP`ia4Z=LxwD)oUwWMDMxk-V}Cn#k=}zd5|G=V z6*KXPb?>BJHJJhAx%Rlgj6>()6Nd8ZfRie%PXj)C19MT1&MsgTonJtF&dbO|_%#o7 za__>{*?9E>-ZOqCVUSSoVfe_4tfLpPrz-z`VSk~s?*(juJC`*Tau-Z={yY3yox^=D zo)6E*sB;WuX$*gsGWVDNX3W+wW^0&-?whK{f2Z*2+JEpad2B|WWv`jfeDN&zB{r2G zY@hgyZ^{Pjb_Q=F{KD6Bz!!|;o4%Y4&lG#{gdOq4#LxHQ2|MEF6aTUoPuLOvGVzS7 zTYgb}M?CZNOlL2iup_=R@tlD)`3XDXgT!|$s_${6eegEIj`(g?{p-wkLFa_U!1t2= z^ZWq5}H;p&(X0>rSApxT_Va;IDfd^Kd&fssW0Gg-^Y55v(|O}^Q=$;`5HCu zI+w-&nhTxLs^YzXdD8y{#r@CMd>uyCfoB+h2L^_U``eddH+VJ4q|Ru} zU&>ijYiL)}ghzpwhZ)&HfAy>LIt7}*L zjl0CpiXv|jUc`FOl)Gz~&cS*>ny>yqt5yFc=Mlenp-brB zqb$xXTlGh#;iqjS^k2TuYvHXu#9N2i4TM(FFyoi)F4}sS?|+uWd-3rm{zt?&Cl=Yw zC24j8q2V$3al4gx#&`$!ec5wL2in|YV*k2ylzo73R!KkmAbxcpA>?v1`$%cL9foHF ziaF~!dw5GYOYb@6y>ia*mgPK^_YU8%;@*2&Hr_kDoeGXQ3FZm?xB@=q;itEWLb z;c;D7y1G5Vdf5Mnd#;GJQ~8*oi+%C%L)CvX@&;#cdtm3>V6m3C?IQap#DjzD4Q$>M zXV2t&6Moms_vwE7cD~VvnX{+g)CPmI&knE)j#~BNt=O8LU(wn=B!J8i+a}#Z6>i0* z^?TN^p|i0crcILZ+h=g*>N%cSM$TFixXN9xHv$LvsjW{abkF}Z`X9^;*r#5@qmPR0 zTZ(=Px>7j*!DWrm=f*kMd;5nwjdxoSc+yqjkF5Ftcok-T1Mq%p_RtoM<3GDtc4b^@ zQ#n3!+&La)XUx89LwMeM##eK39&^EaAkSLgI4`5Nj%VL9m*b}>u_?|TcVsK+oFRw| z_Th^a7#RnQOr&hJ=K|syft5mFHI?W z-098h)(qWbXi4AlYpfv~K@e^osna_s`8c zcjd6wYvMhSSKbHxoKWH|X2j*VSv^ zJvv8Bdw`=}VPMsTp?-Gb_3YKmF5IB`0dLRM9Od@H-_Q_b58$_)wMrFxU9YFS`a>eZS6kBKe^Bf1^sD@oBat+e}C5E z8F49#KcjEKpcVPIzqr0zhE?Afw0ixF{5dcCT5?{FYso3*8DiBh`<@kgkbDo3r-1b3 zq%S9ZIqAzu7p=;^+6pC-N9CCRR0e5kqskJ$e)kD?&hwb_FzpL;V{D-T;ThgILl;DM zqVz6`r@MATN65o@ioq%`mvg>kIvtft$ai6hCM|` z(Z@y-d#`7<-L-8)JA7*g|((KZ`w-1K{l}c=RD~bA3raJIb4z!Os?OH@!3s z-U?r3E{{Nq%70qa@+6P!m*$8U@xI&pzMF50`^ft8=#HOReCsZ$asm4jsX@MJ}o+Gy*qVYVDqOt zeBL=U_Hd3i#K+yvU3sC_Sysrb?OHes$NVgy%nq4>K0l&96Rt-NIZQwQS(0WqGX~9! zK?{9XpO3%;CL?pS`m_7rH>PMql<)Xw-{-yeushAZuS3r0%=?%*_qEh89wDCBc$#>+ zG9G91eD*xF_Zi?<=S-Lf^idJRSHPc7nUj}_kMVK~S+)@#SO^cyojsyu4&%5B-mEbb zY#N+m{dRyg;TQOI@H3|Be=hdgCD<$mfh)mfxYCLQuy+z&+{!+$csXZQs2jdQoak;N zv@&Sbj(@O|&Z(bc2MR;AIJa(qFEqza+9i^`U_^@PMt@Ad+ zzZ&5!YInixiVZnOJI^doTs5%Vn|7b!)CTeCun3t*a_NeD?`mOP*d#ww!OqOF`A%+{ zkL-?I5t%(l=^0HveGdWuJsvV{w&D|-ijl#B=M&GIMdO(FiGh*yjX5kNpVl5e;x^tp zw51$4XeSgLeUJCbIYV1)#xzX10eDjq`Ri$ebQaeN4$XVEb2abx(iZXV@x0GQ7ck17 z-IVK}d_-+m{>zB3lWd+!N>A+EPhhga_j~TejLil{*Q$ zQ~xaqTbC8Q5vZ+TT|yaO2A(+04g?bxC*TWJ`JUvT!2C<<#rrG1D&JYb-C(NoEb6?D zx@zD6^N!`&Yahp2YiUq;q~pb`6vz`6S6{p5^Lk)b@hLG0>Hy zrtqMeWd)60BI<8B^KipS^<_IfDXXigFWVW?Q6Kk!b<~&b_)byZ``=h!X44Alo8r|w z`J{R?o&WIv=RBGh(Qmh(na(R6^`i48{ltF0^bPe2PBdY<_qbtf-zhc~|&` z3A3HoP00OUgyMg~wU6QgO}eBGv?Mzd<(2Did;!P z{3vK2dzk7B#;hr&x1Z^=9x=L`%K76Dfz#`ueJgIgv*mIA!-#78^1jxy^MT7`rF%3| z>Gs#A4>SKSIw4(jx2}3p%d7C&!a~-h;JfZw@%6_y4m=(GGkWcBzYpaAcIuoXA8*zl z?L9*6aVbsV1&^|R$m+lE7vR1>p?}l7gxV%#k<|)Wvy{!Er63!{@VIP9VuP+&BAE7;yp>5#al7O$!&gPz|HstO^=5HTz%2@O-O%G#3HM_6{ zde*B9nPN`9Wv}mR+0si^anIydX#GL>LK^nd)A`cm z=YAp+raPYz-b5Wy`LprU%R|w*Y(wtfH;aE61J*yUkM*U$-n4pBXgYO_;n@eCmTC=B zT@<;PP&m|&&|M29I|)ni^8r0QKi=vUJJ&CN32p$7f(_x@owPylDV&l%JOHc&eHr~5 z_p}d|{qSJog}0;p_;5gX6x9<*?IJf_bwe1`eYRTdS|j8#a}sc z)!L|^oL0ZlDJ{v{nq&{BzWvCss{10|-}*3qm~?}p8@m>d3GJbb-i*tp50S0Cz6N|5 ziH#4KbN}@R7_Z-a&$XF}twXf!ztsa13mo7dhW0!`16CT;7rJo0FXD>uP{(aE)D4h$89ke*7uyg&< zImPUqZZ@_P?bX^t@5KITXZBIMu%8-F8vEySg47ddx3E`uh`qwY-dX$Qo39nD_CpIDTOdRLSOdQwC>UI6hxxeIo@z)Dd0*eE4Ejw_uWM|;xxxcLNgTGqW zVaprzd{&5tu}4s!fG;NOTh`9N7gOQc{cBeOThm78*}3!mc3yG39hYa>c{)>vj zneEwEe!zHZ?@xE(ePU#sG~P2DKYIrn+wC23Iv1GkoKBx#B(BC^Ri93s5$ZhOkN>Cd z53h}zsWl9KzW8r6`TpSLi%rwn!s(=~K4n@KE*+}IuOe+L_wK$r4j)r<3u_BGqpbTj z#b5i*ui?&Aw~q11L0T_Z*kj}CV@){ALQbfX9a&^|HTUPi!<4tDq0ebfEq%Qkc~G?I zRQK|-XVX*;KIxx9-am^C+@?8;2xTXyHHzk4=O`AMv}9+236q>EFYnPgV}YqMVCxcK zYz(k=F?;o+xzjPu+{gEAcciy+-)S>*z=SEzbQ7jJH+gL+ zH(|PSBO(5Z$IJKCM4ll$sXWd2wu-`109cCk>9vnJjoO6=nbTXKOND+9wpkApPH^8> zBfs{))P?g)c^_~EzLS9YCUCbf-p#X^_glH!=R3^brNI1W!2HjAiLZa;>)KQYO`qZG zo;U-Tp8?E|^95fYn-W}H2+a3CdhyO%x+N?w1a1S3?jFl-%2Hp%gU|5#nQy{WC(nc_ zPL2taozqO1zxB^eIfzO6C@gDG` zn~Nv$9dX#U0RO@h$?tAH?hA(I2`BzdTp73{d~IZ$3*|fQjQ$39+9=b=5&SP4+5HW9 zm2XcxXN}4Cu7OqEODx+O{DP6Tig?NOs!M*_V&}B2M{&r-)hy=~@@+N$>#pJyM>O>V z`tl+?yDq1w{!fYUeE1%^#hF#$MjG({Yu4*pyOgo+uWN5#7suN6CPML1{r`9V*B-0# zl{04@`NV%>@@CJGO#;?5^USPx_%#0AE|-9Y^3DyC+!nQQlFXQxw_2+&(P5mnBv@J!c^x@6Q(&oHetH+BNJvgKO~f0!NcI}ojh0b4CP7Z!5Noun7a~B z_RGh7cbx5r;_%oQ93C5k!{cIb_|h009{c|Qhx-CI>W|iqgT20(%b1T#nU`_U*Rdhy$>(5eKe%_!}m!wYoG^af~i3iKPpI?N$8yzvTP6 zJgu}xb?>2%uaI|VM?Ks>4&3b}?jId-taE@z-FyEkap?Y#>pSQ~2i!zy1oC#|bn^aL zd4Zc8_Dcg+4ZfZhj_-@lsxUg{M(&gD&wCy91X74w-6z0y8=&om#rJgbAQ3r{PV zH^n`9hxN#9!qZaWX*}={04{>SM*?t?2%dJuKGJW?9xMW1)jjvJ2h$z0EirucVagG1 z+9phKeqq8?=OGiOIg1GeyMs6vi$Z+eAi6v%teje6%O>9t00Z5x*LK8vqZL=ThF~qgH&VO7@sldG_gQ8+6fp2m0E% zd~45CzU?HdB+}mSg{7gu+!t!!fCmS4W(^)JUb_+9n|N@cuUjH^Gri7&2M2uJUk{{o zH*#kJJh%oP9DoNeYI5}(PeRXK{7rL4n{i2XN=%sI6qzvDxyXb`&Il8J`yKH6fQ>p} zRPvj_QTD|Qe)hBK{|(;$qsGP0b16^wDIR>Gfo1W$^bXuSvm?&+VR1Tr@6{0}9Lz90 zSh9Qs_%8t8GU1J_%-tNxu(w~}$*`>D1*iP)Bd_wsrbY4gh>>@lWyQickrYKc+wL* z>BSnTlU}i1$ky^o`+S{S8yM_cu+$QdpfIfd+45TWKEa0RacGJJuQ0A zcZTzc!53_SX~%4Lox+`qgN)5fWt?ko6xcNyT}yPGlHml+TxU2_&Ddu+QwT3KZA^7; zFmWl)WD_Pk*P1ZNxyFQ9@X@Q7_bZwID}V!ZOE-TPK6)8rdFk!IiMv**{^s@ax6#y# zyt+r2FvYpRgsIMO6Q(&s2{r$-IIA;}X9!O!PcuHJvZlH8a0d2mfty_#n82milech(pkbgV2a(Pj7Mr8qvb}8R=~P zOg(DTAnxZ$?tJ2(WelSZjkH z9R${z%~>tZVW`c%TjiH^XvHO*6DqdO?TdUP8_z@F#t+Z>YN_6%Y5qRwde}EUdYd`_ z;iLhd!ktcxxz32BFjm?}codkGzUM2x72iW+2wop#{g7tT(wuhUv_E%9irY+c& zEcYihRTD1qC%r!ETx)+m?^B=!lrixzc(Jj4cxcrY-*)zZzzzB4mpqc>R3oof&v)gf zJ@k)tkF#ozZ+kRP8++^0UkH|Gd->iYpT=hdcQ^#rzWHx0A?o{q#<{9H4R3BsfKMIlYYEAvA&U`|(VHWiW z*I8rKx8`>%JUET@!i+#D{IK;BC$Z}VLz(z2{}cPM(pUeS|C0D`Is0HWKRrfIQbbR% zS;83rW18ul!FdJ2Mh*D|GnDgX+)G6fttED^lw5|K>giPPBkp-~)+x=20Bd{5-_I8N_HbxV7nc}Li_2M!d(WFa=!!|X7j%VFlUYo>ERHpftFopw zyxGCes_!3h$gJ%aj@Vs_47x{jE8TfcIKsH>;ys12DM8m&@>n5hio@T|9PW3u@PWjh zk@BBQpOrSygc;7+Cd_sQD5UN}h4iVPLg;8Wldf}7-!8*w-kTVo0DW=kQL4@UCHu8J zdH9y;1bxFcBmp~-CS-@MIv-KY`JP2Q+j$u08NQ(XvtHKzVq^);0ehK?b2!g1pFL2) zOn~{&JS34fiE{6xzE7Z0A48+|K%?L(&cgfV?Yx9>__U$W)>#@K;eY1rihK4a+xaVFDj0r;IAm_KUVqt(+o3pUnC9q3 zFYXQfTkQ)yZfq=AYk2X0Q(ka*DfKD;2I6z9w5B*{vFzuC-_f*Zy|ilDln{7lS=s!w z+O6PToz7%=w1u_pHfwF)4ji9O`D#D*T@k^?ld2E;w?ZK>@*9P$mwv4fSX*F1!OD2X z_A18sO2+yM#{6=4-gnU(xw3BA(ej-E`f^g;>+&E2E0i*?3eBj_bUxqx=$1w2+`+8+J1{d z`cgUBxAk^^rfZJ(cYB6zI( z(fy2fy*D?Lj!R2x+D$u8>gOY8*ng#M2j7Vg)iAz|D+8hC)xl8sO>9>`N(>E}n_SD? zX!|Q-=wLE*uwkdq&c4^BU1f~5;3*gRy&^8z$XhAFx1dR!bzkV_8ATq+h6ZnnIU|UE zLE|Bx!mI3Z@h7-4*g)bm25X=zqL)hB%^ZuyrJJ-Y=XbGbjHPH@FO!z( zERRiNd`0sTNXziXXK8G@N9*E9KZH!$LF>>(9n1#@s-bn$qU-G>=N`sZ{S#gL7k%7y zcTv5*rJHaV7uRkl$!WRO_}()#PqeiIhRM61wC^(ZyP&_HAm4lpuD=qc7rFiI!|cOa zv{~}fiF76G?_^{N?n$cc1S}dFgC|tc82l`tzwXSMzb9+{Uch{BU_S{QNCp>Dpfjn^ zna;N4t2U3;9H9P#)Gs}Dt4C`*p4LHYvijp^5W7b8;acYUe681bjzDkT;(4Am{!(a8 zU0P8+e&hz{^x{kba!ngDQyX+)1^spDNtROwJc#bp6Q?uGlCxKOaqE-Rz7Mzw20mC;Fh9Gt0G)X(+(R$dT3ZL$Q@uMe zzDqe~UVtCbkj191qR1kJ{fZ(FC*yTL zs9bS1<3Evkn819DhX!56n)}MznJ2U6Zku=C&H`wWZZ`BJ(;4UG6Wug>*LKMclkT7**B|esBu6y{6He} zeQMSJ3jV!mGydJcV}q+nad!9~w@t~^Ct6j7oWDFSt*Pd`r6;W|(?kP-g;mw=Kk@vd zayCj(}0y%Nc)RP!@eR;{wu3? z%%=T^hOtgFYt#ouG$Z>pAoEG4-GjWRI)?)%>c8%4TE-acdZ4J@$a`m`cFfHh#>8-pIg;VLz|ME9|H%XSJjGR426HUdvWs=H%M{W zPK6gYTXFCk;odYa?oP#N%)!$ey|_CR=h4Ke#HAqft%IiKa9m#WE*N-g*>HGuj+Ne2 z#Gb)PxO>kzwvTZZZR$l?>Z@qmm8Pzu$mI$L7DX;oIH)KxPT??Ui9+D(JA~q2=09lg z(K**K?$tG)?lBc&7Vrw$nZ4KaHctJV>0z@su=h(|HD7wI+3tMO(n3fIp!rA6uT)qNmvP zY{Bnfjz6ubnvnA@?i${_C%!&vzHk2)&i*dP?xcoL^l_s z`$3?)$J5^~YdC)vIhfx))Hs2Chid+v7dIhTB|Tlw{XuAHLMP~VqVJ_}58q2ooDUj@ zF1OOhoU!(*ME6jEE<*afH1vCtAtXz*$5~B_nCmD^CHaaXS_?c*8uDnUSZzORh@nSw zPDOh@8)WYSCoW_zH6Hhp=T{|hHho`f;fJKsz=5@YQ}3c9+%G<{;o&%Y%H8fbEM^=; z?=54up6v8p;Ktuiy!e^c;=h1K?p`TAfPIL!mwc~BHzgN}4`}_U^xeZ;K5!H1E*~I{ zc^ggo_Y`ON0CCLKbmFcxZfGBQm`NiqGLHBL_R0p|!tmpA!#fs@Xuc2}t$tvH z_~D){;vFfDq3gh-&Iu+mZo8f`ykkj8SNUN{cK*c}OHa^Z^aLqRKgP5Nb0atvj7o39 zoNv=wF~FS5j|+3XB<~*18Ik_N%`SZFOub95lATn_;M_-~;r)_O67Q1pdMTvsJxs`* zmV_@LJIC4}0t?tryKC+1`+c<(UCH2`)Ls7I*!n7~Wmv48A#&vs>$c(c%h$ND5J!3D zEGzw}craLhh<53}*nhNt-I(76p7!;7*pkx^c>4Ck@UjTJEUItUx+}Jg=g~$B*<0&| zchEU*;917Aym!&eV)#|%P26oo_!4cAj8ouGZOYf4^wIG_^&5#JimSw_;N^SurH-JP%gW*MH+34EPIoaXv%>K82FZpV6dWVE##SwtD}#Y1kr?Hprxr7uikp;8VVrVTAJgriGnH z5P4H}BH9NDivD9CeQX~j%UKQ#x$~3d_~_TJOFTMJF~ZE-Hpal!V`Vvg=r8B%bWe= z<%R5x>rR0~$dt0t)V@~>c&NPy;bAL!Il)RS9$tASHsjo*BzU}wd-Ja6xqv5|r-ie) zZ}HT@KXZB&&78_QscsMlKxw2+4@6W+s-~0jn8O0e1e*wUG0Fjx0&^KwzJNQ|BK?0ds~q~1RGC# z@f#J7?0W>>tvRjp;_DR;E*?g<{i9^O3t3kxgdY|tgl~nlt_2S32-UtP_*b+hUveI@ z>Hp}v%C}$go#^`G-gmyf;TezWyQfE+;LD;bfws$chC5s9PogV(N7}nQx^f9^(>_2m z{4PSfBp)v{{V0koP)HlA6aruKOqk}}r*IUujOIJ()U<2u2z$OKi+{(p%UY_ZRxO|K~s-G(V!9D|4)gt>A?0Nsa)stnZg>?R$xVnS=Ok6`a(PUp(&-+VRMf z_~l^SI#Hg+=n?L@?#BB@aA!VnvmAUvrjVY0ZT5cEjcpI<=g?mHbu;%kGX@{=Ef}(H zDUNW7eCSDZ*d-0|_PfwW&4G`9)b~xaMRfJYv_bK$-}meeIUgyyA$fBFeG%Q1TvC+@ zf8f8*$(JJdwXAx#3~YcYr-<^|A78tE0cYJfvn3w(8F`;e5PoGlAM(C}b3`w4R_Pty z)z`NP59Ispbr%-Z!`HT*XhTYQOV-Erx9hqG7F*xH1Rva5gJe5Dp$}0Ww8Ed#=2WM= z7_b(M58uo>fi{OHWEPtA#?2!Gz>R#RH&j0~VpsLgN0|8e*eq55Vub7mWe=e^-@iAH zoEjImrf~o=|3KeME5;W`RthgWS=)0)cHOTr-V8s~{2qetMaLWYE4d-bZ_oAatPjyA zjjz(Hx(hCnol%u896d>z=C+l2lHKNN#`Vc9f?-z&B_8-I;v|FbW?U36IsJ6Lmk=*n z)16SfQE6I_i3iRmZYAdcO&o0#@4Hj^ficN^$MR>GvkkW^zu(IbY_1}Ig(>42w+#CF z3USj^2J~w`~ooMTL+A| zbSTA1W&C&j&EwUdab}A-`Iz|7UIl!E1o+al+X@}kwr@7Pr zXKF4P>jv7KuQ{imq6fLWE4_~iGo5S`raGA>OmWf)e~C=5JSn`Z%!BXXS2YHQouI>=kpsFQ2l(uEf4A+D!=tuw(v^$Y1yYxkL zq*3XH*Je07y|}%KgU?=%t&Mn-@IJ@N^7KUSFB>v93q4UvQxSIAC)e%k@E`GjSEu0U zucjW>i3({C0v}-t_rY}I%SD%hG-FrF7?!}Zis4y*yG?r+QN4DBr`NVl)#r(E zW=*hLa{=9vE~f(7TY7^6Z;i3i^drUeL;YIreS3PE*fqvltAaC7jNv~hZ(}9)5__Wh z71>H8JJpQkaCDR!!^h~0Xvi_0iD%yld=2#S{DM3he~87pa-M_8){;HpU2Cl?JK3YW zGJxsvQ<69j1Duuzi;o;;Jy6Uzvj@GT@MYI0@oSt{9FkEQc@%j@{3TF4^ZYPtfhyju zBJSWof0DN~&3=BOGmQtq^-zUBDIZ}k5v za};*N;$e*6Fy?n{bWhdr*}a`QV5^7Mr|;v-?eZwT~rs3X@R_+aH zoyK#sFXF>j$P+(JT=T?FyRK4p%Q;f_>N4{vO-6;kpFMw|5W);Bd$-VJ4|K#w`!;0%oToSglDWUeKkj)2nFO=k1;#ayd(1Agapi(~`4WEQNF}yT!KTgUxptW*|3jBV zo;fl9mifk)SDUB*n}Yr?YRf73vfgIjpMQj_!hgwvTHiK2=d)-3*oD*IPIK4SE6``k zzN+TDWhd?3X#cv*pPa}(`c;A8TbW_M1uw~P&h=pW5z3Ift9AyajH*BnoFdnbnmwq4T5qa;|H2sCqfF-! z)qkT~|GBCk+GXa-t3UgMx>SGs32~}FPW9hiRo_wfsBf?PCx%93um;O?-a&t~>kf~e zoFhL28P4y3O&4Y}or5>I_5g>_Q;3d>r-;7zJKi&#Ci2LZ0AGNSuzBa4E@2<->$3;; zP5WQk^NOBHlYDXj`Q(TvpBQ-dbeRQx*~>>3(ViRj*MpOJHc*Fn+7X>Gqc0o51M$ag z#LE_7mWfYt>b>~a)HYzXjXfHzIiB<4Ur{`3`Z*>(%UR>a|5@?OC$iGImVPeI{mx5! zUTNqU9-FtGedr9Qj`&n;B)4{9J%B$Qtp%Ey`?SD6mxULk*4EJ$d|yNkOdB1Ntf+pl zMv}gF#{qoTwbQQqS_x@Kt3v9X-PXPy`ESQbJn^0`_7k4GHI8;f_aPUV_Ok!45ZU>D zg~;613c=s`CQNf~Hlg#Ms z+SnT{VvQ&pBnQ9GW9Pnu%~7xk8DZk9^W69Oyz7il0cn(da{1E(_RF2!y}#bHRsCyz zvozGgI$gFujmVG2jsW>o`j#4n^tVhQ@^?TXYn!c%d3at5IOz)osILhb^Iv8ypXr2? ztjLATS$JScD2{%h%h;j4xP0`#;d$w`N>Aq>e31bwNzOUsZGsLS%bU)1TF5Nq!Ok_3 zJ~yrQ*Cubaa|U^1^Hn3;NIq1as4gMM5Ey=s@JN6IOyZp#U9vAmhoI_I?ALKNR#Uc3O+u%XNtT_9%AGvuClWt^6;38;#Uu8@0a}XM&^_Y>nl3cm#BjPSFvU8?$1MiZ(!B3ZMX9Wh(pXDkr{M2m>^l!br*+m}_MP%k zFPp1Fo_%L5o&Iw!{u`jv(r0s?YH$u~i(>S(-&LK!(W8bYr#KUVGtuhrD;`>xW#Y4( z%f0w1iihStVrX)*Q|85Ar+DaYl8H}pO1$_9iih6*S}+Q2AK}HDdZ8DYCO*>{>cwB8 z{LlmOjq{0nhdqDUf8GR6%P+Coe-@$W@b#j{wB;>@)I9;Z+)iB|D5UJkM+qrQG$dM% z_Lb7H70swjW9^y|wQ)FpZ4*Ps-z|6dmGF^g_JmW_K5#kN^b4Oa3W1TH3W59X3W2+> z3ZYN`;eXBh(K**LcaxaIYnaQcnbV1^Z6|OCgKOguX#ZxKE`L|AVW&|x_uUR#(eS)? zqqd^Qx8I}F+xSOxw@haSXj?#`=&$H7JkQW)`eNuaG|tdxKX*uqF6-U_nP5n5F)Gr~luU{fh*U-Kx-9oRj#>?Hmd89QzZjE_A z#QSMgBlk_|<*rrgS*x(lIgvKE<8M*8vWz*_ns$?5c~G2PafciK2=S74(2uMO!wXKN z!>;WY{wO@%cMbWa|Ir@VFQFs5X}9Q+-=o8GiF5Z!Q=B8`W*BSn@~biz zCUqfsFCuRgPF_DTZ^Ap!Yw`l4tH?Wuyit0+>4dx{AF%og`3lK*;KEp%jExy`+VSVU z(eD5+lCz}S_A%DGeo<84SNio4qRHvbHs;IKDP}l01vNBT`+rKy-T_@*YS+t$>KQx zjs6$Ur&DONp@SLD7t|{_vj|0FMMw8}ajnXywG3;(kG;6Vii7^C{S98+e&Qq(TvPd6 z?OWcxAG=6{1{k?`0dz41+poF>$+clUD+i#AgY0M(3n5D-w;JYy7Fw?o2SUtnAG0=6}^EdU|xuB*yw0#{6o= zej;-)0XjN9#GIJ@th!^e@$v1S*v;7CizaUU3K@bKmZej~C5z-;c@1Z%66w@pY7UH#YjuQ^!8;CTmJW9%dZMg#V07Kq2&YEBZQ} z;i;RKWN55ldLDCOXha~;v;}%sH!rnT`UKHm&G%iT%PxNp^Ca0Rb#98W+y9Xl@6o^G z+cUU_y+v*5R~$OfFJNTUL;3P`4gEt_-8Z3YsEM+|1G|L=NnXC9J3gj+5Z^P@zSxRL zen>r}IZk!1XAY!~l#Hr)3mKKQ!2YLXL z2Yhz!58eFAqx^@FKXS2u()oia$f{*AXD^eaWAP=KvjBnM&Smm9CYXBHv$4I9GNfY@ z&Ab>|Az4f^RFr0(L!8F+xcBgy6D?z(t4RR><35-!}t6a zKC~#ijXBhQ)^zd&yga`mPuZND3I4JIlSkWEe;!*tb*~~VLY#2)f#cHBoL5Nu z4{=7u9~4X9emdd+@=k*v8(Q`f_T)?EOmS`o1~ewJgL;*5*;VP$w?J>wlby#H@93V% zCpWtEP5$;$PK9?xJ?*KU?fK*6jpDiPVmk?cd-m#B@BZA&tNPyH{qTem&Yh$>ukqdt z&D5T>Ij{LGGHt7;cZk~B$}U2_aKsm5>1RS8_C=tdT6@gKW@RGJ1w2VSoS9ztHvY9= z;CYn#T{_D7eqcp(^l8N*KT5`Y%!>;t&Wvr6Bl@O%s}$$a+n;-J%N1wnZHBYRi(95R z=uaTJ5WGD|+SVA_8r3%xARnJJj%B?&Yz0P8miispuZ67*{zZkj%jYP%_)0Y~?MKWrcLi!p|2p(?rXl9K<=?tJJ(iv#&D_A>+aW-q* z4tw29XFyDxXlbn8Ks+R$blK$^S_)ok{Vy7OniqdcT3V4}*BiQC6gq?qS;HJPN9ibQ z{(aYX3pG(zcwqNXq1xB4M@aimP5G?-uQzh!TFa9oML*LFT}^X3Gk29=LsxIcf4|@` zjy&!@LYi~Mv(uMC@s`pMJz4+ zTQ;x*YH&k&wJGR}vw6_ah(bA%3Ka4TUQHe&_Ym4&bu8>KS{*o*1^ zQ4g2@M|Nb5qgmiMbB4XJy?AF~%5{rz{IFB)9E%;k z&Xw0;kGc;!EZeUFt#OAj-_kV>Vh=#!z@kWnLg-C_Lf|G0ooPhQ9%1Nqijz&9ZOAD) z116t(z}Unvb;Qz~B=SUQ&K}v0(bw)VaizqWfBt>nPFG$@a=Iv;xsWbX>siUkKH_9= zwgs6^^7~Qx8O^f*TiTQOgq3diZ4g+~TxFb!C%=_y&J{>*4&C7ToH_vhiEn($xA?}# z3K@$%3K^SS3K^@96f$-nDrByhn(ytD zRh|m1g5EdpiL;L|ru8#Um%Y#00Ccggx_4U{wwrbSB;6YjsXH(z^kQ{#OT9m3!^SNs z8=(7t2ygD%@)-S+EVdzkXs9lKSO|ao=e^2WrI5RMIhQ?iEa`c^$CmxcpT41POZo=( zs@Gapimma`KCu>;ok^9(k;+& zlXi6TNQFN11PwopTVvul6V}kDYNV0Vsx5phC57gXvXZ#R;HX(MwWQnE7PA&dm z3fUIz#QuMV8^4%%$u{N2<|xDI>)GSpPrTM^+SB?4H0jA||8~(5e|J|WdoOXj8AlTb zZ9bj;{e(Ez<{ulrc-cC|Iirc6VrW{5^KanIrJ1BN&(leh%r5+uT-cEYEG#7LDwCG# zd=#4oysRQ^oJmV^-i=KIrd}bfl(d7ux60i{y1&RW?K}YNiH`3j&jnt+Ta{krrYlW$ zNP|pTmh*CK*}$*tke_7mv*^G0UgR`OT|ZOw3AXMS9nInBvnE=1?skeP)` z@^3Bt)4Sy2==%}ITzc`?ZkobzXBy}5OyU{A^Bi>N z0PwXAyje$B$9c4h?$CbDpNaPS@GUGq-5Y;d{+}l}f7Hl4ik=qp{+H`%JM>ZV(Hpc| zxctA0^Jw%|FYZ0XdGfR9hVs3oIKwlNoj-eVZz|5n&zVlW7x$XtpxayHkjZrK_E)0s zSG)9`wnXVW`pJn}SBrkv{1@_Kz0q@IJ5Ny$lLqa@FN$b0WoDRlb(XV2Y0xC4?UAgg zH0ZYI-{YhyK40`2SbkLBvJW=&I@1wOE4?qYx+ccw0Jy)+>X~J90ix9!TbO_HNqGX9 zcAjS=d=X=ydz-D!r-$>&(W+c&D9EYT>P0U(SV247kjkM+~e z6rElo`E^Bgl1HbLTSSNJJ{lZ)zB;94y+0K?ooeXxR_OFn`XajcJT$s4f4HI3A;#!< zI=yt+27d;0Izx1NNi3aS`o>pF_kla2jizpB^$=+5Cg^p`6^33faq0DgBO1ZGX7stu z=yRK)+a|wDua}5kZ*u9iIm5dNdcAKJw0d=~5I$QBjd}`NAX=N#wdg+2fQP`;Y}@Mf$g&o{?h{LK zc3kAr|3t=I`G;~IBpLlvv^=wx_wvMfc^;9hKOxTEc8mLO8*R|KAlLXFN^u_cY*`Nz z@5=DWjs*>UvQ)eS8Q$X^P4ri^(8PJX;}ham`X^&Y>cYlo`m){7$ZTf{VTY^@>`f=_ zHItU*TpODPd@dwyi%Cmzu8K_qhF6jHqT)(K?~lp-uMq!7#V>H+zp(>2WmzXbnt;L@PYV{{6p-imhL3gNe73eLlI8g%lPeD>hXa) zPa{3W8GCD#52QLv8CUUvh-Y&V#aG#1MBnw#L*x^^SZu;{XAxl{^gi}H81j1u?S3Z% z+Kt_wnfpzr*%y(21@NTxLhKnYlYVU<+eiN=Y<6K_4tcRx*e`v~9i~3%NRRof|3)6; zk2gF!r<*SQ#7UoS@+#gy!mi4>%RR#Y2_Hic2)qQ6>tC>6Tq4P8@9>4mX7k9Se48O{7 zp77!d6eoTK9R1phQ@t*~N_UoeaeatW8&Bjzjlg&?MsIvavRCGPV0P-pO1I^31; zAo<@qY~r(=cFH?!_*1IU{T}{Y{0G`cyz(3VLz@oiyMzCvJNvzK`9)THxKA|l`E_b9 zcdIF+J^K`bmtlqA?k5Vtp}!Nx>Wtd`lJUQRe^eviuN41~d@mdJWal6>QD+G5@$|Xx zQl{jRq9o>fnt%IqQ_d)S6~2)xzTr=5E2NE~>ZBIQ*P#i6L+ku0EzeaW-Rj*;dS|aO0N~FPgi; zqq##pyVl2v7abSvS_w^glKzVZL}~8NiHo(XIGuhjB5q~1{A?Zgqf zG}pCbm`?uNO&UJhfuoKzkLLcsq-8h?O?D zU(K2+t7>E;dIMuOVdyY^9e`z*4ktV5jH&D<*rVH4vbB$)!=Ez#;%OSIA&hm@{zUv% zuzyVNlH#0Ax@4mPCd_mS3D1ZAew&YH`OA`BzH~0pdpx4@wqPNa7T%i9xjg*b&&JQv zWS-$XDZ+Pf!%tsCcao_WANK167p>^)T9LJy8PmhG?SMa{zw%n-)p}I@YQGkkgI+98 zOlmq{@+m*_4Zf5>Gft8pvvWG`>3*7jU4C)uGpo__st1m?ue+hLzP1MaZVmPk8mD(? zr`j8)jE1&$(_Y2Dr}2X>scr9)W^@z5?4~W;x7Wtr*`JA1-!@ifwuJHFl0{!7LpS=< z`!A{t)#mH$J^GF^;#N%)?clB)#z1g213jARe4R2huUb<@`)B#k&q!uElw(WZJC*mm z6=&L$$je;sF!*{-u08t_w~Rj#FZnGCIQSK35AT-$yFH18k(-T8@!E8J^1VwO{zK_Y zx>HAf@oT-0@$7v=#EEuEZ}A*(zI&GBgUpUIuD>B}W%cA1SKldHQ+NNUhIq|mPfsRW zN;$@sn7CdY_@ew-2EUD5&-+61X^i8%{}%D>>UcRbcieyOnb;qC_1r~1$=z-FoGXE+ z1#+=(@^{)_!(DXpTW@rmoP)(@95zXkr?`t(^GIK1S6WPe!K<|;D{@J5_r!aGTW*^( zof}QN(w!R!4?p=0(iE1#?t#juo+37{R)&QFSoM2MZ!C~&1 zVU^R>8>^#pN~nW#LX)pnTiwJhJxPBm;`> z$=+9R!T2qS+AW-whF<{a-UpmXTF3Jk&uqplwerPU?W+gaR~MhztoDQ7J84fjeROpx z8O|%jw*sdwyd^oCNv}*yZQ>sDD2+4y^wUqqdac!#YZrU_kw0VvFEojJ#c4i$guHKe_YJR9zZUv=n40lsPFU;W^;-i4Ce!dHm) zy5|J0hv&II1hDH*boH5Ih;!GU%9CRD=FcTguvCAu+two5UfsO^$(ULjwg(&`nM z+j+vGudH5$bGuBa>o+_kSto=J+>D8Q>vK;`W4;x6(jT{d&lbxrVoo$SmwVx*gxG4C z{DQZ+ByEMT>aW zS~dOb>HIJHZ6U`7h(9{#WkaL&EuERdXq@?{punCK^N;bd$GDI$Gn(()?)#A+YSCKV zi=g{{bhbkGh=}KiK169Pa5Oa9C)zgTTHxi46xRp!CeBzy(<*pBv7b9>qh#VL{Evn3 zr8t)Jc=DwfAT&B)^n2l_SSQ2FuVqX`r*GjOBVS`Xad40B&`5T!BTf8VdXj6sG7o8A zjrF7U!*ZZ~)6mHtWRAoGFZRm1#w)9eJvPN(5>w6?ubg#SCj);;rktdv7Uom*RrLA;i1#XsR>VD~CHY*9rD=xQ|z5Nd6Rj_w~vcGFJlaE28(^+eMKz%DISNGjk z5W4bjHgGQ=DB2rKbsSHhkU*KD&C(THek1#?4P2dn9BqH>YGa4IqXOUFCzWmVy3w-b zmz%w5uj~W-C%SiB*$*9G_I&an4^wvL$z_{0f3$4w45sY+ys|&wKkbBB27Kury?-}TXXy-|^KHQ&!~G)y z(0tv|quJ6|%V?zpkug|>{O&SSc% zZ0CHwwRRk4!c1p~39<7qVVZNULTEr5VGrc`_GTI;1VFyG#} zxSNvE4-B*h-&dKE6}W7{_(v-1urW$@9x8YB$wh_8#K=$&zj>$GYkGiqV>dy(cwV)> z2eO{yyCyz+vGvBCJLf5mvA4eC=DAnj@ZL?{_g(r%=BOIzrvF&q%uDV$?)MM%4Q_1W z`{BR+*wlNA;*iI4&vo;BpYKh2NA^`art)2W9^dFC4h)jKkmSfWuuH>O3*N-Jl5?%W z?*ms2=U9VFpaBXm|YR&|d0F>qPm5D*pzzE&L;z z(#e$1SODYd(=zHPFR%uGyx@iwGZwS^9UF@;&$M^Z5Bc*uaV$nL7AwxQ2JemgUW*xP z)#4u<*nQ~ujd zyBpFepYb|rTvR^)DPOY6PoqC+rJ)ns{t4+@Y3GVm%0a(-+?<+r-rrG9oW?hWeCUXe zo6{5WwX|}dH+RX0{l%d`Fp%iKDrJJTF^i^0fTI`5bK;mr(~o>w8d^ts?0Cnf zwL;H+PkMANV$<5bC<*x|OSp;_3H zgOlbCI=}9e>Vj=UJT?vi^lCx$YYFJ^xT|qXoVia`yjp82-E$%NNcJx}zx|-qS@#|o z9rf5s-SL&s1RPGp2fpL_aP%Xi=x&?N@Qt9e*;^O)1s)|U$<`+dx1JtuiS%&n38RO5 zII4$RB0b#i#~WSD4#C))#jd>8lXBD^={p}`U+E`2N8S$>wokI^%VZObEFs;se zS3bS}kEchn_`e)ojpSa}Cu3%R=}O{}uXYcTjGNsd%0AwCs|$V{TZaW zvZDOcxc5>0)8x-|{+E2NuSVCu&1l;2CTT6+`swdpc^{I`)u*I8la4FT)u(JWb|#6-?fFc#6f>)K=Pu3O%5k6E@e znS2_XUC0sIbDM%3;_8OD=Ycxq+jV-0JMMqrTkC-;`e0kMqCgaj6Nh12UoR!MYUK8wU)I1s2PI$xE=e8N*p57taIGrdy1^o|E+1+|l~~ z{mvNE2lUx$XBPTwe9A=bqP_o(4_Nts4d8!D_m~PlB<~70Zm~Kw?Ph$ofDfaN=Imc| z9A3nM7u-t>ZuqVF~hZ|!lOSl1@8h;SHc$+6s z#kvpt2v4$JisHxf9)1iXzw!zO-u7%MhZDB~{16-nFN%Rd7cY+aDm}}C%S;cB2k?)( z|CZtW7MODJLTmJFql4^e@@G5gR_)Y7>gWg`S@Iy z-h?S+PS7MoB{u;v5{3vgHwi<+5Coz)v=YSDKtPIU6%dmU#6TQ44z)rnAX>?>{E8Jc zskH>L7NxB~s@CeO30O^t6CjWa#QVN$pL4?{qV+fQz0do6-XG_)&)M_Yd+oK>UTf{O zMdl;?#w#vt;Lzhd20wH1Qgq_b7gy(`CeFv5!rsl6`&j1*oLcqt!S>p@pRo}gL_TzF zkHgVkkL_0ybJ2_so-*Nu-?Dg0bf1cz#FL-1SO)_6<83;Y+{yhI(oVJLSM7GVJ2}d3 zXEEhuUFAb#1MP@^kgj%OJc{QIwDZ-K%0JnGp#0mbsbAVDBK!~;OmI6sTb^(&?FcRr zSx{HqQ_ijXW9k;3rc~DcN{`|8TKIV)Wd$bTi>5EecH}Cakvxe!;ybb~E#LFyzzEN3 z_IhpX^;!q^^a$+thhF|1`&OZQL#ac`%lm#?K9x;eCGsSZEu6&OMrfVT7rB2&tC9DTrz&~jO` zQr1VD)#n6raksLsQuZHq*^i_w{E(D=*DiZFsO(MM%6>@MH|?_TkgpZpm-z5ixM>PB zocoN_`=5D_s2n5ydX?_ETIhMK>19lV@ze%<&QyV$8Sg@04}Poc4m&rCm_7AEUyUo`7U_)?Km%_0wL6_{sFW7{Z{5{LaCvM|feX)bVa z@Q}G3kt($fwj@y-sU*dL0FpoQuKCT9yEWcsVW;5kwANvm##sK%i64}MvOj3uxKbahT? z+QN8Qdq3+ei0u8G+REt#ww_@#;~{HBYycjIMhopMb#hKs?EOE{2CYRkka?%1thhf2!JB?n_{8zpHE(6U;KoSbH1ftM=NSOI#ysKd`qH zI=6^*a4_oJeWmdH0*4b>2eRI-r<}D;6V11P-Lw13Bt<6${|g;0B;VP6r9~%mR9d`R zvr?Djv3S#X^ObYzvUD$l?Y4(WUC2I!P6(ebwDb~{hE0~;FUb?oUHdGF0<_oxrkM(|qB4Wenmd(`L1r`B@HAZzZSX*+H> zyH9n}w$QZS+H>jRTX0wRHSD!%TK6?9JjsKsjlf)fUMyoBXMU=}7_(i4+-s!5@3vq4 zcdlvaZ(zSVTdirw{!=$Q**k}@hYn>gjgRDIVaRcGL-aACgOT;CVE)cQ#&XV72%P+$ z|B*FWihf(>LFiTq?}9I%BP~F;rrI{Uzah@czEQ?jc(j**2N|cw8E>i2B;A5Lc)KoG z!5SV?P~}fFA0waeG_`iQpYt7{8L@V`;ryel^8lY%X}48FUZEQnoauEZ(7ANM z7&JrdzYop<#uU9sFi+7&FpLG$i`*`HkzmFLU@UrY7rp2Yj9K&|!JJ3C!So{8hP4I6 z%ig^^tZn=jdhu)!y(po)&i3e6swXq!lZ^6Rog( zTgrL5fSgd~QRsxoLDoShu7getV2=f@DDi;9Uk8Vul@nU~EK44;a(X?oCxN4Q#zNq7 z5BUFQHm&dmXvHrEpF=C2nqtuk$@{nS(uzPE7OmJ|x8bK;fL3(X-F9x>_miiKRuo=p z{Cc$2zb}ZF^(6asIS=?2&kH;OS`pBtwc0d8>}Ib)o+A6b_b6-a^W)7Wwp{dm;=Irl zOWuNB3|O@2#aVgFKH_9O$eNWgdXq8%c}o##7QHwtZ`rQ$_cwn>zO(Wci(YJ%{0i?& zersQ~Mdgn(e`(kMBIN@7U)DMGTR!wHzCOW}@eh{46thl+4g_Q{ZE0d3qhv7K0y3E3 zJ7vPVt)~C#AqM7}Jvoe?!^h0Psj*Z*yw)>e+}xu6E7qK^!WeU&3Zu+26^bn8U&}22oisrD8<4{+P-|80m+ghFzBf3d54fW* zI3yBW!X4JjqM!vKMh)xHwA@?BU381zgNijP;~PJvPNsCr_i6}5{JCDQQ|7^QE_(?S2~tDGNCEqTIAfT z9-Urv)s9by4;mi);mh|Zyx2O9idl(kKhQ5w}xrum9 z>SLdOOk|P7%?h*uZ***`@LuQI0gSp%^&$IRU^`{3mR$|rYn~9${bKK46l@2OV7~Vy zdgOd9`yO~Am+;b$Te2dJcV!1a*|R#-B-;)k!L0P5XUs!yOS)dkeH-AGUg+RUy;}Bw zik~#8@*x>U3+ZCRj3LP*mAP8B@a3-3s_7oaFnvN!t?R9yG>N^ut8vBjbmH1JbFO$2 zeOZ2+^4ajn9G9`>Lz^eTmMDBNEhDV7W&l)tW}E+fBgj81|i9|5fB@M1oI)r=$!J6l+UF+ z`r{j<{75Z(6?%Q}MZ>}B@pYTH8{`n_mW@IZ_7(}|Yvc>6W1q&|!Y!k$K1}JJ);H6T zI;4N&yQgJmTKag>awRRBf9*5APU?D+GGeRouIP~^9{pWlZhm?080^!op&rtujltBj z@p2ghYz`O~%T6FL2G~iEPyXRb#-S7&lY6jD09N+7eqrs67}I^qP9UPN&ax3W3nwQ! zfG_Mtg69J{=DA}LKd5Uy!s(0mQZ0KWHVrM#^Um|W*X9p0o)P#POua$yx1f8yuTn4R z@{eDVCUzv%VhaIJan3&idmoWc{xOb!RCXlf(TNwhp4q+L4(cWSAhO(lkk*Lph#G^- z1{(*%#??82Z<&Ws#^vc(2OH_mZvUHdsnG)cJlF@lv6RP#q*{Ew(}%8pQHQe8AfNQ> z$T93-upbHj-z@1Dwj!$S?_WyWg>_EWVprQ=Txz&!yD?3{{&su~^h+M2EfQH@3ix02 z!0<`FjcJbkx)WdN^f&*eAx7+E;DNZn*gZ`<8(BY(ICP`pZtVy@C5_#O{3TkdHhg6S zz7{C=+R7na>r~l#bX%jNn6Jv@bKyXj{$npBEv>`KzQ;Y7RKC}6!5yOL(ob>a?E*B)$5 z8nGc!Ww%|X%8HH4mHeYdd?-9zGd3hX$_RgE*=S(jN!!}Bq&=)PoS0b%J)VFLetg}r z#heXgTyUlmT_*T?Fme>=v8ul{-I#6LZuFdF*=}rcUfVI3d>-hwhkSBAcNez8oy%rG z+owb0r&0Hn(EO?BOpEcEJ!OI48LIXR!n+3BeC&iq!rye+dtv}JOU@bAkVj@p8;W746El_Ee4EH0viTPIgF$*g{xHOr?>$6Z3i647{9!%3 zxFt8ox34WXU#0l-IP*5vkL;7~p-ey~;Ri3uK1p-|mTbYY*_dnd^oz(V`r|f{OLPAH zBs?3kg|%wmf^6Yrgcc-=#y*@j;Cr{7PZk|4ThM7YUvTrEhIGjmLgDiQvV|G68!TIJ zU15#U)x-zM7C6UQAij^E3z9AD>n@8H`=Wp>dJN@-zZ|7Pku4N5KNFaz@yypaaK>2t zW}hQl*ys7W9D3h>Qx3h))lCjvOCL|&{1SaYelycSpPax9K6{+_>#6R$U+%~dxRLQu zI1=2a_AJ#&9SV*{US|BU2)xW03zz22slh+x?ZAz|h0KTSx9`O7!xQLK$&-DD@(&Pa zHd4kb=Q7AA@+S*dM?+iU%tygZvZwdW)auk+HC|>k&(RFwzw#MBWG~Zy%9)ttGkq2n zBFor>o+9YTA3Ma}dL3y}ugG?8<{cjsiyFBGsFi!Dj$sEP zuznZuf8!j-`@n>xH#;9r+){o0*YJLd$dsTPLi^8~j+{J$y0mXdPv!1z#uxttV;S=T z#y+1p$O8`kpw8>aKL+|VlX`xJKX`$k^R?C3=`Z=GtpDAjM+>En9@G^^ot#yh>SjK% zC94)3AZNj?bC7|1*~Hg;oz|mWbo__tvj;stKF7}5%A|>`+{GS06!}CCWc=0BvyjEf zdN$~n;0@t@0`yaZE*7%BTx}Me&*NS62p5o7aHxWr$yrA3NUIK+FM$D(_eODE;!9-x zTR7iyJk2HgyKS++&k3c!i!~RL*OCjzn!V;&<$p|EGxDCZGJbqFu*OASE3)(eE}BD} z$oQuLKhpObDHo9OzYp%PWHusuw`BYiRQ@D$D)|n9pDnq(CA0CWv;^~ec3qOk!fOfU z>*v&E$@sJFwns}};k_=i%jNNH$#uB@>zs0yT;~$Iol8}@ICFqqZYbZD9w5%#dQLe@ z4-l`)#hWSQ6a1<0rs(u6+0R;~)5A_JoiP{OdR7M1-&{=_!Fcq8Ae~;Y9^lvHQ)_cy z7ULYNblcCfpSH%dzp2x<$bjnYdG5itti4L)g4ExIKee-ZfOGJt$kkT?(}DT^v{1&5 zJIPdt{9lD4|DOnq7Xj-QUw7)}@$g>b>R1cGa@v0zE|q=;{cnSYY?l!9FU+37YJ2lC6fA6NDuQD!|b5xCDf z0RNkb6FMtvR^b0h$^~SW{cRW;#=nUCFrdd?rONa-ACNYH+ZA@%hxrb`Z7;j5Rn9tN zbf?``CHY#NanNw?Ul{1DJDwIq!-2o!-R!@PQhzY)1?vZb<&S-VJ%I(0J01-4`eJ}P z3*RT1b7(V|e#hJJ_aowi==Zh&{SMM4Zwsd1BGVJzFhIYjP+suwBozw%{!VbW?L0L6 zzjf|kjSn0skiO=LMcRntFJS=PEtx z+F%&$O`ZTR_$%!01eW^9m;~vf;+g-0O&WN?x=&g3O-=<9_}?Y3lyMLSc)`1vPr<*k zmV$Y~kJLOTnx9ROvBwt4E_;mcfR1W`UDhgR-9h^|RW8m%mI?f!pQ>Dm9#pS8jclw& z&UcFr6dv-l@R0Bow5|F&-GaZh|B8ncngj0Jmfv_e9z^(#VEbIrMqjl4F`u?;D=F~ElNImF96upgVg z7~Z0UN9l`ZDxPDl@ElhI&jNGP$t(Hf{R?cj1a@Z-=S7Dk@GCx}z7M=v{Ki?Ep$Q5G z=TGeZv^}mSs3{?gn3*@zU4$HoEENN*rOsCrQNuB@?@GIca z;$I|xym=G(!=go#9eCu9DGJ_@aBLW0RA7T9~{%Q27>ATtS%jGqA9dU zt%1?m7X4?fKY#UCk6Cmf(bQ>I(SLi)d+;4Z|GV&S5dA*~|K{)yo55>=vHx_uz&`X} zh0uQ$Cj4K=!_v=y|CKww1N|4=iEVB9ZhTO0gx9U&5q@B7UoBgF*7YD1KK+mQzQ&fY zyrz#<@_pu`=H~IfbZlV6pNZH4S^n%awVpI!U)y(?F>ja!pU*NjvPK$#Va}9zPRww5 zPR?|D9-)rg6kJD}KPOLsm$<@)>t7HjXE9_Bfo){HQRdUYR@c5Bxu)RqwZzGt;zIM6 zGq18H9-wT1pLW^sXu-GG4Ol$&t;9>2rQ{WU;ugDXMOWE)(+6BxuxynR*uTjxH=lgM zPt@HVnM?7bC76?GU+}ujp@g~(+f#}E&){{T-EtN~>XCMaQ(o|TwhH6_ zufudOZ*iyMEkvK$2Yn`Y~-m)M3b`*SfG<(Y!eEWtP>HRkqyVV`wVel7nR!QL0 z^6&d4Jcr!hCNQby!PZR{hoV@a55pKWU+wcfyEO*TEHe5xRo|g!-mDKQ`yy$9a+?#RocOYZ=l&GAd4iTL=ez_q zByKfvmB_?5-MyW2*NNsmd}|^jM~*4!Rg#VzTy#|}nO@&=N$0!j(2^$6(=3s6`1BeO9*5$js&6w;)SYXW@~V%lY@`k?sHd!0k<||9(ePjrhXBSMhH@&#SwQ ze#^P*o&J$^JN+YghS#B^+4l60FE^gr9q_3rdkK-fa2A)gWgb>iPR3p3ttaoHpuDs# zdFR`Cmq;7*LFLtk8rB`;;*W@W>A&QiZRfp3@&W@YZy|Zp9Q$RAR9^n^5IT;l?Ywg& zFaK)wm%If*c^QX8=sG6Zd9RVY>|0dcjpUsZl$Wt_dhyL|=bb`c*;6Te+&t9ik!|7Q z3B=1jwkZXC%s>8?u$BIoNl4$)C6w_8X5Z)8D6l&fS=`JRPg`2-;O2)NhO5M?E1SAx z{3^lYrx{y;jW=w0%wWN(7yPlV%o-nJPk#;>hgX#3uRS+)O~kEI`L4F|vDQ`+ zS4CXFXM)S_|AWE(2lu$hBe3+faiIJn(qufBNEvXFyq~aPdpU6e+b_s?l6C?-`Sp6D z%vY&rmZ~Spyjki2|4BU`+VzwN*F)Ne-%!to)HB7d=PJs`csL5Hl+RB;@yDB7*7;8C zWLC@h&PD7SlR4j60B@9r-~HVpV@x!Er{LP+heRfE0qe2l8E*r;#M*qX@W)B!Co=-F zG46R$ZAFc=H3&{(joO zUHlSP;Lmdoe(+?@-n7d$liqkLP&Uz&H619skF?W1|8^<28J(!i<1V}00n#+mtTvO( z-9hDEAx+9SJ18?jXft;+v_pGO`h(~>dJf{6Px!N|@Lk$m;Pn}Nm-5}5Yn8cQ(vSPI z)B7aU>MhW4^cFS5d5KH?3{ugOZj8^82t9}`**oG982+_Yj(wMA$OwpNO!!(VL? znhq@!nm)Yg_NMgr?oj+ry6`>F^7;?*>h7lRLUX&(^|fc|y4d@P{~w|2v1S$R%N#sF zIT<(Td#B{3@2cJ|x^CwM*NASv z)Xuv=^76kb?~em?-OdX>kv;WQcHSE$FJq+gcF}b^FXJe5eWIOrCV6F^6%7A)n9=ug zi+)a%I^a1}{FlRwa65jI#KT{x_?5$r5IcSx@#x2=H(kb_k1>h86uQsIOg&q;KYegaF|sMCT0e1O<6uwJzjZARIBTt|BW7>Ks}r^++#7twBo${%IktNO+L zFLt^6y2>S*J-U?}OSwgMxh1Mxl6lgG-MhQWC7B)c)q*)02W$Qn&(3?f!?ycy?KP!=xVYzSQ#@yPlljdPw`tH`MbX^(5K#q)`U>-ahNx zUhU{+XQP4zrRxJuf?+~%wwlu@uP>AG-C-h@xImxoxBC>`txxxVL!U09PbWx|fBPzr z`GMxh`!c&8hx{*dwq4c$Y0J)AkEL(@P}LJH$X{-z2Sa@!O zKA-tG+~V72X|!qmf9nj9bDmufQ`Nttx1_G;uE%F*!BzE&Uke$*dWkhFrv=XG#+!ek zU5Wb>;jwD`9)lA)Xn&8yvz}g)5ZwBz%Evh`30eD%5`q_gFCjSNcPc&B{H+Sn(@6;L z_XOeZJMrC2z3uRE;-|B_A7`<{kUK*y9fQN{$@pz>Xze*)0HCv`-}-au>>y8CXn_DARXb9RMjMlv=4H^S3Ny7cFpY(!ps+&J)}`SQ_| zr`|d`;;{edPuBe6zzbJyKKgdy-y4 zW<{(YSvAh*KDNBcqsQ-^rMuf-p-pV{*A6{WAorf^|C0F`0v;Ecr2BM5llad_c4|l5 z`s#IGVwd2;SEb1FkQJ8*eCgEvo)?}DKb-J?kn^L-dVG5d z;}?EsS;r*4EBNOE>Y3CzwyFHliEos12S@omlL%u~IA3?KS+I0|M=g1``u#iREgjVH z8T0iy?MXYXyksN#&srTmRDHDdknEY=XT~;p9`(NAp)SuoMTD^`tfe0wJz~xLrE@zr z=*%PYWX;9-?Loq+&qcqv`|`OAea$)#eF!d}@|C}218ExlA4K0IU!^^k>1|G58}}WS zF7>m&Ne$i}3?die2rIrq0Jw;QeRw4u6BK@?}OH|1W8Q zJ6az1YSDprfgQ0+-mSq`vi~Y@cy(z8=^sC&g|81%D zV~kI8Eq}5X(xz&o^dju3H1mG?JraAC_|2Can$t`F z-4&+!VlugJ2w0bR@I}Ns7Tb8 zXdh||OIK8lE?u#Lrss5go~3OUGHyR|-ZKkz*0ck;niKgN_c!1# zyK@<5UaNKdmg#2hnb1Xh;>Dk`*t}?_*qK$*FKCxfXqx!tKf%~m(!UM#@0h&PzYX;7 z3*NJVa?F|oErQB))ZiyEi|VkpYe6K zw?=CvhwuY0XK(SPnUOBGM|l~BTleM^Ikge4MS0}A{zwZrL8UDkY$T4GUZ1o-&zJC4 zp084TApAAgSUA|y1F7%+xkfqP*MSEs3JQ-1?#`{Acw`rAI2rsxo|5|5ju8UqrLn5s zFI;&A++8U?cG<&&-@4{JFEi&fvbE9&^Ez&a7noh^t(yZsbWmfS!n}1ivL})6e+0ha z)#ku^{GD%T&NkPkR3o8RV(o~6JYa+~gN)h8>U`eg&CR^$>@MIv!Q9CE+jlQ-8UgG{ zz72f8#{4Ij4Tqny{P~Klkj}aqsq(~|KPAu0U3r9u#=i`CI*3F5Lp{iU$+NvHkLc{I zJVFyCy@GV<%W@UQnac>}T=NOW;ZfF$?0=RJCou6SFtM2TM_C&ZFYf{uj{+A9BtF85 zmv@1YM}d*!wD%+Vu6}KFCGgArm_F_!@KsXx1}An^@QiDihm31ts5ULb6Ds&&$6zR?OM%< z{G2xY_Ir@Mxp-=7y$$Z=5i@dgv_&rDQ5@{tSLB}DkV#wxab?JxGDEe)ob(TqG5M9o{=n(2lRJA$fm7(%BNAW3xClP-p758fR=COCj_j#K z=;8+Qh%dyQgaH^XJ?xkN`@*9q8tLFnaM5@^_(|n&xc_ng7Lgd zbO)b0mCcBvbFNTh@%I#5?2Bv3%t@pTf3>c7m+p6&s``I0si6>G^~bwMNo&Fv z-Q+T=sLKnitM7ETv7B#++V&khl`d_^9`1cBfDR7jTl9wLuS@PAj~o8ru+zP_#u2`z z*5TS)p+`OIAgx@Fs;$7kBRpIQyk9MF1P>X$<_+?Q-^bxuYu3FD-@6lfvx56OHF#|~ z1GR!Nc!_#bxX(#wmg>*s9>%rOKWK7bOk!;wV6M+$n|7>@Jxe=M!SQ~lzW>Bb9s0+;JIk=4->n%cKlCIPdJ+RYi4LMC z@4*KMUjnVzra@2M;k#AlhIWSImv(1<;gObpTFHhL6OV+r^OjcfE@{y4MK>kpUD1LMUpmGb z`{C66wj!H{{MV!KYjGp0zH$hmo77A8BMweFom@10MRq*$aOcTJ&GY^LeOtWFz~e z$R0%_4ZlCzSb9tSG{1k?BEhLwXp36GhqqjpSw9I}RSq7lU3w|+5e-Mc18SVV1<#x_ zK92K@&+8$E@=*ue&Edb~A7>Mu9dGWh(R=uE_xJE+d^*k=_fJF58TT^o&kz`pdvgM7 zR`^wg$CihjgU9lyI{>p$;4&ElX)76iBZWFTmlXi3`S5Jq$K%{@5_CX>T5Q zz28|lpAF3YXW=}x8=PNY!@0C2^7HuXT(NSQi>4d4Ej*let4d zi?HabXBuZg7eO0!x!1;Bo(E187|8_Q%ArZ~7?*PB(R|L0_W}-j!zcCuWx@1wkz*Da5$@xzC9v-0Y>Y1e76+a{L)m}+i-}63{ z^}YXCA31j0`mD?IbM|-o75;Gj3p#WTe1x0_KHL$8$T=){_;#8WTvyHcU~PH0(f_|t zSD~(PWv`q=WKA)S85b8=aCouPr!6mSN(F9ullC>(dpZ0Z*n4uc_K4IY|01v_^L##d z^B08dZ_jiuXiE!6HW0x+q9^+Z?zx=V8`$gv9_`CHq!2^=yZ#fsj@&IHz7K@fwhYl4 zWPjACd3_&fn(v>qS!(Zb4$8Z|$N7NwcGg@)XH^~Y9g8mjC&T|Ygr{;>xUMZ~bFN8r zt(JFJc}7-EB7D%IixRv`3nN!2?t%A9Kn@|ioCm&7@(7 z>8jP|$L#VvgZ-=BSp>u*248~Na4t}~@g@P*~CaAyr`Y5JwK@fSVL zc%ulpq*KrM(`>KrQ>T{E2bvzHRrdP}Ym9mB*Ov#}6s>64{O=hBw6W3h53oC_q&t%AnvIOQMaVn6!+S7-ZFjeJFDfwU)f;?n-3 z;*&j1^I3P7h_5v6x3K^7e~r=}IXrN}0_xx9vd2XBY3iFX`2_yvQ?(z9H(gF`n5;ee zzgDCFQD^(VP5R$QzVX`$-?UAm{}FcoWgY|d03X{t&t)3v{6_};%iuqbvF}NTX8|Ye zE8mr1lwIttJDetGhr?^V@6NscKgx8|M=U{ z@V7rcnonA)>&6R3URicA@{KBQr#7tAvG zZhgUJ`8jD{`LhdHTZhoQ7le3yn>LSGkRNDC?K@NELF+G^e(9 zj&_+5(ulot@2A$!x#map6!&4pb!aK8QdDcCJG`?HS^WZ&aZGOBGJTQ;cOGr{MA_6m+Z zor%o!kbnC|hqnRP+m_n1w~@tJmBxeqVMn;%OU{;9@YxOiN%!&}!29$v!T?MkbM^%H z^)fF3wq*@BbLXt$m4IWBgQoF_ZPAB(KZf6mK)FQnw~*gz%fb({gVuu3ie~IVy4SIa zI)o=|iK`qFSR0e6YooQc)!HcJyAl2^ur|i>Eo)oW2K?H#iL8l2du{o3bvxu7%UugzUJNe_v?!JA%sVG57-va_=k`9L$WqcU*(7Q(sU_ql}0 zn!EilZ3#UuiXoIcC;9g9E^}YRn0R7VuS?d$=cR{kSSRZ=WB+*0D7tFX;kASpdVskQ zIavvP573Zl@@~_RV%`Pcx3W)8UXFYpI#NX$nIAdB?yn9(4%QR=-wQg>8+yOT za~{ko&R_Z4;w#8MXXQ(jwbC`pyuFI=MX&rSN7ZNL@4HI!+GWX`BXvGFs(A6LhTL~ z#q)Ic-eRod(Km?AvZnMcgV{A1B&N7)py|>LXEmpS3XrdF;#nR z7ki%++6``pbz9=1t~$+Kn;GLBS;3l_7z#hcUP*BBEWQgb)*`ipiAxKHXeBMPw1zL= zabi9tv6p`*Lpo7SgZv#>r_>s5XTS~(k9+n0Yyc0{z7r>++=PhgZJ*+bb--&)L*itaPi13N3mrST&PcODC1r5 zln!AzEb$sD! z@fg~*b33#z{M&_w^iLS;y9@aae$XE21I}({K58nG)(h{RN}8s*YJW>w%BTM9Kpz+J z-}3IXVNUujz8H?pVt9U9+of-!Fa9tP9*0DeA$9ep}ukq%8u&l*?n=@laKl7@N6 zZrtGJS@g3gYjxr*aI)ZC@UfC(z_UlsvT<=BE`WvM$iL4KWiz}e_bD{T>*ps$G6YAh8GLWH1@E{j1zY#dw^3R9{ zbjxO|vww!j%O(Q%h1^>_5x+$f(QkUeaoXyFC)(I|a#q3D9HwMBrTL zOmlU}*5)V#6_8r;MfxdUSB1;;O%eb)$M?<7uj$R)>kvU z+6nk6jrY^&mQGHPH17YQelI-KOxpGF-Z4qyIrGJP3h^U}KgN6OGevdqc)ryXLT`M5wsqkyX@NHArw@N=fIE}*B)PxcmT!FB|LvKD`CUG66)wl0S5geREpQ+0 zXG8m@jpko#B6>CyfulX>phjI=R*$TPxfVLYnFHor{JdB`{bIOxFq%DGlo{brc01Vu zZ!O?usoxE)D8g^dAnIMS9NTZ{>#iijHFsY9By4WVp)d3F@LHi0@c89VqW4(M{ksz> zJCS^>jmSxJJ@tIAok-py-CgT}zVPo6-=$9<>FCE!tNq_v7rM5&6_)-@^l$?nFJ8I_ zeVL0nD0DIvCH5m? zLv+NFO`*F)F10!_e6Gl)!pk$W)+7oqCUU8}kV}cYYIT@4P4)nrp|3w=>|Cz*)YgPZIj+hAv6{{q@y}?|{6~I^@e0T!Gg`!upZ7Ev8 z3)*u9=#vzlS)1CLV5}PIhP^Nj+& z%HZ|V(FID`Pblk_afAm*S+cA{&hm4|xi5XmsE&hn{l5qQs4*3Lff&xu0u!!HDF(VV zGgH@|EypG!qSM?XI9}|dX7i8IpF+l}je7$}0Y^nLcEF8CPpVC?3Tqf$712N(XX=xT z+_|IbU3&kB!qA=#f1u4z_*dX_n+2cC>Tlq`#707FpU!6=A@^g8y`)H9EeR< z`@FRppBkR%Ci+whEpQ7%) z-~g*F{NqrU=*U_MG$j-4C*vjfGAwTaa={y1$fa75OF5KW>c$J%qu;OvxzrZqQmx3P zT*##)EnUk`w)y;1C%{pa;F1>PYS&ZWM(lh7^`9C}{bAOAzYPBiIxIy9md2i6WLmQC*W6mC+vBYh-q3V}KRc1Ra(lG8pMb~yKz*?qvVWNy(F5?Wna_C} zp5?5g@Lm^AVSeh+i+bc=bis$w<`s<9aYFF0+K*=j@qj()|9v(O z_^C7Ezr+Ks`#11_>;E6%0k{4S@qj(y0fDD;ctDwp%pe}H3LfxRdc@u-dN0KTZvGBD z;CQoA!23e)>*4`7fbUns15UPiK(z*T4Kp&bIJ4S$L({CP z*+&nv7NLoIJ-{0@;|Mh4R0{P@yywQIN$%Izu?G58=snx#XWh51A}_stK-R=lHAYQh zD|JeJ8Ss_JiKoq?tpec>;3>r(Kx_$2=plVvtM%+}6tNDlpWPlQVTF zbm7TV{?Rc8yr!=;KnMQRA6tSrGmboh_hc>HChH2kcB=}-zsb4wG0JWL`TjM-IpbX| zHl+pBD44-G%d!qba+v=FykPeS3d(G|O=c|c(`^gK8;C-ja zSoVgyYv;okglj$3zOx*DqFnD8k*q~@?K|5T!+2;;{2`AKb^Y89>|(JI9bo2$z;BU; zPG(zr(pX>r<>BjxbyPHMaX4y|_5RP!Ax&uOH{CzemR7(yRLftL_-Fg@F|F>Sd-b}H zIn!nNRERP!V@v|P#~$XS6vJKgy;#JA#g*Tqz#K*Qho~eYb$b~iL z2OYjI(T-1&c%jdBd^F!}oAc^MlV>%2qTG=#c&f%3QwwjW?m`!vYiY-=>fk;+Xj~g( z{5tqv_OmjsqGNz(X~z#A;~K8UHBRp~uDUg@!rQbV>k?V2cKy5#4gJDM#_(_0-<|*_ zjt|ow=?{)`n8*XRAs^Kg2Tg+L%~OO;c<}z z1m+^WXw=9rStsH%)V~}1^{G1Wh)qbfW~g=W{RCqfZ8QTrF8n2KK^E?3U9_ZW=sCI5 z4*R;md?jO#vXQxlmglh7Srp>(o)Y~;iq?oe)v`~a4Yg-+{c3)b`^@~NT-Lj$Z}@xn z^smuMTTXFaT1guLx>v(|Dq0v67m!!{e_EAV{68T({{OV#s%{sV?h`!m?~|81c#MyT&GYPhG^bi8aN!QXle{ zwW&2x%9bew+jEi0bjAOvpHWQN`mb(ilJe3=t!;cKeEQw4S3W+V+q&cC09}obHuA~; z7I|x+#Sbf5jLd_$U|JmIjN5w-J>D1%J&rMllCP`(D`Su$Tm7Fzyuu;I5XEn&gHJtK zJ=hm{eXoPdGM39eF}&S_tV#Nrf{e+X-+CaG{-yLUy-@xM+Hd`9W2}*2x6?#f@SYX_ zvpD+TH2aa(o%Wx)IUl=kV*mKMKO5XQ4yg9Gmnyt<{!4r+x>4;M2di3fZ z{8zlWuv;FNKWzAxC~a8lQITClmu~?c4nsG}hO)<{PRSoZ+vP1ojJc^^PdGA!d7&D4 z-Q@$Yms0c5YpgGvF-W@DTz?0BJbB|KM)Qg=V-971GQ(T982Vn8&Wlt-6*wm48FFfBqR(OTLp4X|?)0_)a*xMo_xz+U}a6uVz z{>Ykj?wh$Ib#CSKqR7?j)V!ecpQoco=07It>G%jnwxnOtu@qaM(m1VigCnus2aj8c zUITbgz7RClOC#rOc% zt?ho1w=I&nNk1OZFuNUDl$6;u#7IUahn^Z((Hcr=TT4kd{zTo=saJhB4>tbJH+(6) z#QE<<4CYI}3%~99exApV&$1Q3rtmVcz-1WoBJf<+&)a~`z9CMJi|{+#Z4)ca?kV?hJxc;Bwor$BPW+REtd2hIe{q&m*=3z-L_db@(TLTjy$5D-_fa9=0a$wd@~^|G@WNN)6Hnk&cShG% zJNm0LPa?Cqk+BsxU8l-*htueD;Pf}-`R~AK?tckRvpa6H;dGWWx?Sd~5jkQooK`}M z@_^A5z-o6mt+U2M-M9IjVQBZa`==Hey4-Q}VO&U^-1i!|?^5PW#wOc$(1UiJ?SfZAkjsh;5nZ(_%{p^*f-^^Z(B*DHHoYg`>r=AbmV6($xxJ3M z)jG6oicTc|<4U_Uxu7>@o)KO%YFJeEc%G>rvkB#LHCuDinx;=Y7 zPW*4N9vT$92iYIt^Fzs1;X$^ya#jeO(_rD3i;c*sUp*#m;u|nq^&kEqxNTRs<-1IL zd^lKNX*?l}ZS_(AW0K$H)`mU3p`dOv^$Jc7_yYO3kE2fMhMY9C`En zC30prsvVdq5&7gS=3HdQF~Ewz;SGdqBKE8+JmOmK;_NDQe&Irf>_WHgMy??Fk$0E4 zkT-bP%ZV*gcxP$TEcR|^Y+I&8UZA-CA9id~@9*GM<#l_XxBfx9Y?^E&oE_9~sdzBgxe z(3!Z@030G;mNRkvIm3{|83yXwCuic!WXs>D`sNJ7FZmAiN6yA=u+PR-bUPc@5YjOs zSI)+fgFzr|5r}*YG8S>dT0OHbPzf{2g2L40pkaJ9&RgY`Wd>K1yeL+AL zQPzKRaZ&##i!<=s7=bKBwHL4*23FKJcCuTreNF`q)f{6RJDK`7V*l4H{5Q6-h4PJT zMtx%&JC^T&{hz{1-E3m_02k@9#%A|7w0pb@wDj@|gXcIkecpxIJ=y~89&{1ubFp(p zFL00a+v5M*Gk4FI{T+C3dMp2>_2o?4<=n$ScykuAM#38jzsX+#x-0BcHQ#W`r(CRU z*SQBOig7CIH$MlX*)N#u5n@LyzOXDBgnmZ;^{*Egz4$l5sV`r~KXT5~_3DO1H}aa~ zRNkLkomkVmXG1#mmh~HyBe+p?oWcu;PUP|4lN$u)Q)09du_ssKD|#Dn^LMdBPevb6 z5T%toPW|YVGjjWKUP0ywd?WZ_5p$H%C#0Bj&&u8;5RaTV+SSK<)Qx`(k6yFR#ksOI z{+<2u@f#zyWZ*WloxY0wU%vaTCbb~%jVst{vpUJv6+-DOPM@lKgA#@qP};gz?0JNFSz+v@E+LOA`O-p+ojyi;aekk>hsu;i+|&PP^yr(AV^ zUgr+NkFyWn@|ryH_ZANve(;vQJYk9V7XO%V8qZ~kYm1*Fd`P|%jpBa&FDZWQnv`3{ zulnUJ{rV3q{+Ra(tDfQgvf{qirQCAms%^Yy6=(52ZPjzPh@IUb?A}7K!@GFZ)5Ya^ z+Wv`pZ)B1Y?Wtl1MS7Lk;_;+nb9aT<+{I`|oZus|op6PDkFH@%C(3xi7iH?;acB5m zWqX$#Qmk~y_^X)Lq8qi0<&BKxq!w-eHbDp~Yub60YPw=g1SkvN)uF zTJaqEUcGYjEz$>1r1o5%*yPcNMEc;St#JHexIJ##bL|Z;o7*w5bGo7n8PJ96^u!1c zZGcyfh(30fQ!lSW&oA&QwgzHzAbZ%tilt53lq>e$44eud?yb1H=_cN>aaDTTUErvE zzJ;cERVY5p<(zdKFe|w7>pme~UHFyQyB=VFJiX1CkdeXJ!wCBOiT~c|pF+Q-oRsec zy!}V9DSpNVv z=5O+^L%#BF=X}9|EA`&BeZya0*Voarw(o1VK4fTKUlsP}E91p3FUEXL@3Z#a|OG zC*%GB_(RU5%Q(pY*Dkc~ovC>R9~t~7G^ZhXd2XGHF^>K9ac=C0#+*zZ=dHhQTHX3ytnVqDG2D=)HT2SFbvyvQ`ZIc>`=M7_)vPmBG03P> zW?wj%{#3?jouAXE_l8(=^R5cdKOcev+cWDGPX9K$6u)!ue%U8MH_qNSS{>I>6VhW^ zvfd*rZ8Y|g*sN^1NSg*8%i26jWKs@u7H9mi?Jh~-eo%1LJ{Pttd*mKb>iQLY_=}PxBSnm_~QB!9$)UXm$Uy3p`6>Jes{xo zWa`>>xp#c_2yB>%^HhyAGE%*DlXREp>vs$qrq#*$qlAVYzOSYG{o7}u)0z!VOxAm~ zR}u;?mVD`X8NyHO3)oh;lugCg&-!c`s%3w=Q1g9qhbF%C8sPc*c~aj`jMY8!Qif?I zr=0yWUV;bNaxrulzOZE`JjVphw+B8~#bFC0vC9Ix;29a1K|eUs=!#=y5OL}m!Th>dnhVVJhRj4;vxxRlwmf_EchblXELR90!Trc;x|sZLzmeFf05k zdsyvKtt2CLPvQrT@Oh=+mhxp~9d9@?!YklO-*&7?JR!7}vQpnA)FtIVzM1tb<7)E_ zz9E{gN@x;s;qVSo%vGtJQG##y9&^9Z;v3YNpey)p%usy8J~>k}oNwV9#FpQxGt&sS z&ke8+b9~j|#^HCghI#N0E!>;b$Ul5|Df-7bU3LZA0(J#*p2;EW5t=5pY$-`A#-t># z7}NZ5*us=3-s4t``TS;Vhq)_K#>2muJCI~gv5b&2D{bqwhPndhev|#G*u|tTNmn+# z$$DZtIIU#wC3_NUk=@A{W#Vh+*`;Om?J1mBWKN$<5#5}&9UZIEh25v{T+endy!6(k z^Xqq__maKB{rC}jU};&$PVi9+YyZ3F7`@$L+#4c)_yy9Sa!4 zKVR~v#3`L&!!wu6udjf&uhnZ4GwI_y%#rl9eKB?@*Jjqs8P<0GZ9nm?5{EzO(*rf% zDfYr01)A@~1g*ipFx2Ot$}=R?r^d?3_#CSmlX8fsiDx&@^E_e1pCtU4=N+Cu^1R3s zLHudLcAocm_VT<`HKuvtIA2Pi6=VF&QEN4Jx>W@o$;+K}r`30Q*Y`>FJYzQmCB zHNARtYxTuO3Zc9=GuQf-%IR~+r{Ds6k9aq7B#*Zt*1*OE{Mxz`+w$r#t9(}F7_I8T z(f)Ey2UY97ls^w_OIvrdHKF?bV_dJn3PGCWBg6TwevU~zLc4qrJ0fE^Iy;R z4BfY%_L^t%JyZAnMc%Ks()OzUh8zAl z-}CNYM&3JkDs*2fu&n9hDbu6lMag$B`R>(y+vUBM_gdZeoRoQ({15Ux7AR9h8Fz;% z`8JSmgYJ7q-Z%5US@%69Wu7JfGd#Zzl$lJKo*loCd@qvkMcubf-e2PVCEfR!l-W)G zojiXGl$lDIJ{@Z%-<#xnQ}^93?@hcjW^1I(d*pwI=L5Tp0iH56U$x}>n0z1WzDjv- z=bbTLE@h69|8t&`c9}ZL)M>t-NIpOL{JL+kygT#NocvhIL^yo@FrGdRr8{YLK8EZ$ z78|yof~(MpVh`cyRzFKmEWZ&tIB(g!jtpecQb&L$B8Ty{&(i8z*gvP&hy4?N?Mm*H6#hrXfW1WaHH?9VjQp{(uO1UQ2K4Vh zyE86B#twVYjBxA?4?o6Q%T(jAzja8DI)P=^Y|UuB9vj`jc~pEP3+?U6zqiex>`;q_ zVGpe2C9T(ceXdEJs%{UyE+Q@)V1pzYNIjWWu#g8STrg>L!@?n~z#J|IM+qkR-KNHF1GeCZA~O>r>%0rPiW^p`5x-1Ys>mrE-=zrU$TbNktYpcA1m-4I7^!ns&#UKZFZ8=>w7O2xDZ?deHHjii7Z^|8itLO zqBR~Pf%$wlhCGaKJnz@=F7Q%B-z8iK?owmLxF?b>YdK$~C75NTiA-DOJxZm;n=&RM zGnh@9@YpB&Y1`l9K@V(J5(-U}HU<6**?+XoLB9+C&^8mDF!-W?`()afD}~q4zqF-I zu5%%$>`IroKsve?StD)K5u7KDc;Pptjy=6B+!zdhHG^a=d^`Moq5AdXjyCjmpRlgi zU`JN&jH>O;Ug>ja`NT2*vHnx~Q3x;ZawgOkxv@2ZPq~kHq5s~hs?px66{8QcrYA1> zamO0+)sn|UdL?Ox>Gve!DF9yb1mP% zg~+!QpL!qP{}$M+`q!`-j7xXD*%|{ zR`cmR-{ZUs&3#Nl=NobPH0CyeuzNLVn=H(kQqalR`h95v2YEa8Z8zR3~}MfNNq{B5Cx z<>P$gCA@i@Z>)rK#`*FkoHfqpmGJs;zVAsmbDVFqgfqtZt{_DI5O)`6!QG;JhVPL5 zis0`v;QZEsGdx4xTBN@svMwDT2jEn7_B{Qlf4k5x&MY2~wW8Ld9=cW2LwKA#j;*J_ z$(pWjJyGp4&d6HTwXGICT~_a}=v(Kn(0nVPogb!!7|j#F_u!91tjFJZE#Ke$>6_p! z@3LhbGnb9(=)u2&$h#QebO!? zc`&3fD{1F(+Bu#HuV?wkP_{AHxtFwNaz_WY6B*d_w{cI0*!H(p2#w@SHSf#9;2koB zt|zs(aQ1f471#wbRvLPqFzJi@D}|50ledj-MY*d=b~oRGTL{4_XN#_0rg`oQI< zz_YE4q4d3#ww)@>FhW!~&{w4I| zOHabnUf;tKp5!h936FVw4@x+M^)F#2>tDiwtbYm9SpO0xv;HMaV*N`P$NHBriuEty zRMx+QlUV-}PGJ2@Sit(1Fqic&;V9O>gd(Ccfi^>S{2^BMYZbsi#Ix6Te^=$bDTSz9V+3sRA@DSlwn#m3ptWmjIk7{|>9X9k1}n8u|@$ z+047+UhDfm_)g(HU%r`ZlW$=5i0tp*=O6CkAH)ZFxQzou-xtohmN@yRGWZ>J(mqO-lv|8=&=Iz z#PQ!J<}hyT2g>Ji-t9bfwlJp6VU7j(D5yK9?zO|PYr^Lwb-n=4ev*@XR-ulf;@2X$9^sWxUC8101>iBqbaa|9>R>o^CPdHRK5)WGd`m3hM-4eRh&NFJAX=Z`(Y!E@jE+4v~M!o@5tirDfmt$pml1r|5-0 z8_1k8rf)3y`W|j4JP!Kwk~J4wd$>1{vz*<-^(uemHGjz)ORPQIP|MeYlIIBTC}kUw zbFHTAYW9DxQ?`ixpND+12a~yLebqlqc#$oy`m-I(vy(ZNaZI_Ot~mb!v$#LI`U}aQ z%Gk+ycadv~>|FLt5j^2MvbLrFVnZW-&O{a^GCP42p%*%j$oIcw4_28UYV5=AxpGB_ z@fQhSaT{02u@tR84j8_SpDf^0q;Ehi_@Jc-0ee8*U3h_NIZ9O1x zh`j$k!YJX3D0dOKiT!Z4yiem@@a`JY<-Ab^;m!*+<*W7)U zGE&bhJ1t-jePZVBE!Sm;J)rpPIKRJ+z`v?B!k!!%lWz<6-A`3@ard$vO8fG!5-

-ROCm (146) +ROCm (147) | Variable | Accepted value and default | Effect | Source | | --- | --- | --- | --- | @@ -974,7 +975,8 @@ and **19 tool/wrapper entries**. | `DS4_ROCM_DISABLE_Q4_DENSE_PAIR` | presence rollback; unset leaves opt-in policy unchanged | Disable/roll back rocm disable q4 dense pair. | [rocm/ds4_rocm_q4.cuh:435](rocm/ds4_rocm_q4.cuh#L435) | | `DS4_ROCM_DISABLE_Q4_GROUPED_ATTN_A` | presence rollback; unset permits the caller-marked resident decode production-shape default and explicit ENABLE/REQUIRE; any defined value including empty or 0 disables all grouped attention-A paths and wins over ENABLE/REQUIRE | Restore eight standalone Q4 attention-A projections instead of the two-dispatch grouped path. | [rocm/ds4_rocm_q4.cuh:868](rocm/ds4_rocm_q4.cuh#L868) | | `DS4_ROCM_DISABLE_Q4_PREFILL_TILE8` | presence rollback; TILE8 is default for 9..4096 tokens | Disable/roll back rocm disable q4 prefill tile8. | [rocm/ds4_rocm_q4.cuh:448](rocm/ds4_rocm_q4.cuh#L448) | -| `DS4_ROCM_DISABLE_Q4_PREFILL_WMMA` | value-aware authoritative opt-out for the automatic resident path and explicit SSD/REQUIRE requests; unset/0/false/no/off leaves policy unchanged, while empty or any other value disables; REQUIRE then fails closed | Prevent the gfx1151 direct-Q4 WMMA prefill path from dispatching and retain the Q8_K-plus-TILE8/TILE4 path. | [rocm/ds4_rocm_q4.cuh:1358](rocm/ds4_rocm_q4.cuh#L1358) | +| `DS4_ROCM_DISABLE_Q4_PREFILL_WMMA` | value-aware authoritative opt-out for the automatic resident path and explicit SSD/REQUIRE requests; unset/0/false/no/off leaves policy unchanged, while empty or any other value disables; REQUIRE then fails closed | Prevent the gfx1151 direct-Q4 WMMA prefill path from dispatching and retain the Q8_K-plus-TILE8/TILE4 path. | [rocm/ds4_rocm_q4.cuh:1626](rocm/ds4_rocm_q4.cuh#L1626) | +| `DS4_ROCM_DISABLE_Q4_PREFILL_WMMA_K128` | value-aware rollback for the default K128/P144 stage; unset/0/false/no/off keeps K128 after the normal direct-Q4 WMMA gates, K64 control, 256-row geometry, and 16-byte activation alignment pass; empty or any other value restores K64; incompatible launches also retain K64 and K64=0 retains the K32 rollback | Roll aligned resident q_b-shaped 256-row direct-WMMA launches back from four-qgroup K128/P144 staging and float4 activation loads to K64/P80. | [rocm/ds4_rocm_q4.cuh:1874](rocm/ds4_rocm_q4.cuh#L1874) | | `DS4_ROCM_DISABLE_Q4_SELECTED_EXPERT_VIEWS` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable q4 selected expert views. | [ds4.c:21150](ds4.c#L21150) | | `DS4_ROCM_DISABLE_RESIDENT_IQ2_SORTED` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable resident iq2 sorted. | [rocm/ds4_rocm_moe_launch.cuh:751](rocm/ds4_rocm_moe_launch.cuh#L751) | | `DS4_ROCM_DISABLE_ROUTED_PAIR_SWIGLU_FUSION` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable routed pair swiglu fusion. | [ds4.c:18543](ds4.c#L18543) | @@ -1011,8 +1013,8 @@ and **19 tool/wrapper entries**. | `DS4_ROCM_ENABLE_MXFP4_TILE4` | presence opt-in; unset=off; any defined value including empty or 0 enables the candidate when the MXFP4 sorted-tile path has at least 5 tokens and neither TILE32 nor LDSB is selected | Select the ROCm MXFP4 gate/up tile4 occupancy variant, reducing staged-activation LDS per block. | [rocm/ds4_rocm_moe_launch.cuh:794](rocm/ds4_rocm_moe_launch.cuh#L794) | | `DS4_ROCM_ENABLE_Q4_DENSE_PAIR` | presence opt-in; unset=off; DISABLE takes precedence | Enable rocm enable q4 dense pair. | [rocm/ds4_rocm_q4.cuh:434](rocm/ds4_rocm_q4.cuh#L434) | | `DS4_ROCM_ENABLE_Q4_GROUPED_ATTN_A` | presence opt-in outside the default scope; the exact caller-marked resident decode shape groups=8, N=1, K=4096, M=1024 is automatic, while row-at-a-time batch fallbacks are not; DISABLE wins | Enable grouped Q4 attention-A for eligible slices, non-production shapes, or explicit experiments in addition to the resident decode default. | [rocm/ds4_rocm_q4.cuh:872](rocm/ds4_rocm_q4.cuh#L872) | -| `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA` | value-aware compatibility control; unset keeps automatic resident direct-Q4 WMMA for standalone dense and attention-output A while attention-output B remains on Q8_K+TILE8; empty or any value other than 0/false/no/off explicitly retains the same eligible A paths but no longer opts B into direct WMMA; explicit 0/false/no/off opts out unless REQUIRE is set, while DISABLE is the authoritative rollback; N=256..4096, K a positive multiple of 256, resident non-quality gfx1151 wave32 only; SSD has a separate gate | Use compressed Q4_K-to-F16 register dequantization plus shape-selected 64-token by 64/128/256-row WMMA tiles and two-wide activation staging on the wider tiles, without Q8_K activation scratch or an F16 weight sidecar. | [rocm/ds4_rocm_q4.cuh:1354](rocm/ds4_rocm_q4.cuh#L1354) | -| `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_K64` | value-aware compatibility control for the default K64/P80 stage; unset, empty, or any value other than 0/false/no/off selects K64/P80 after the normal direct-Q4 WMMA device, shape, residency, and quality gates pass; 0/false/no/off rolls back only to the established K32 stage; DS4_ROCM_DISABLE_Q4_PREFILL_WMMA wins | Stage two adjacent 32-value Q4_K groups and a 64-value activation slice in one padded P80 LDS tile, halving workgroup barriers while preserving activation traffic and the K32 accumulation order. | [rocm/ds4_rocm_q4.cuh:1566](rocm/ds4_rocm_q4.cuh#L1566) | +| `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA` | value-aware compatibility control; unset keeps automatic resident direct-Q4 WMMA for standalone dense and attention-output A while attention-output B remains on Q8_K+TILE8; empty or any value other than 0/false/no/off explicitly retains the same eligible A paths but no longer opts B into direct WMMA; explicit 0/false/no/off opts out unless REQUIRE is set, while DISABLE is the authoritative rollback; N=256..4096, K a positive multiple of 256, resident non-quality gfx1151 wave32 only; SSD has a separate gate | Use compressed Q4_K-to-F16 register dequantization plus shape-selected 64-token by 64/128/256-row WMMA tiles, K64/P80 staging on 64/128 rows, and default K128/P144 float4 staging on aligned 256 rows, without Q8_K activation scratch or an F16 weight sidecar. | [rocm/ds4_rocm_q4.cuh:1622](rocm/ds4_rocm_q4.cuh#L1622) | +| `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_K64` | value-aware base staging control; unset, empty, or any value other than 0/false/no/off uses K64/P80 on 64/128-row or K128-incompatible launches and permits default K128/P144 on aligned 256-row launches; 0/false/no/off suppresses both wider stages and rolls back to K32; DS4_ROCM_DISABLE_Q4_PREFILL_WMMA wins | Stage two adjacent 32-value Q4_K groups and a 64-value activation slice in one padded P80 LDS tile as the narrower geometry and K128 fallback, halving K32 workgroup barriers while preserving its activation traffic and accumulation order. | [rocm/ds4_rocm_q4.cuh:1870](rocm/ds4_rocm_q4.cuh#L1870) | | `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_SSD` | value-aware SSD-only opt-in, default off; unset/0/false/no/off retains TILE8/TILE4, while empty or any other value requests direct-Q4 WMMA for eligible standalone projections and attention-output A but leaves attention-output B on Q8_K+TILE8; eligibility additionally requires each complete projection weight range in physical device storage rather than mapped/registered host memory; DISABLE wins | Allow the compressed direct-Q4 WMMA kernel to consume an already device-resident/cache-backed Q4_K projection during SSD streaming without changing model I/O. | [rocm/ds4_rocm_q4.cuh:1356](rocm/ds4_rocm_q4.cuh#L1356) | | `DS4_ROCM_ENABLE_STREAMING_FULL_EXPERT_ADDR_TABLE` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming full expert addr table. | [ds4.c:18255](ds4.c#L18255) | | `DS4_ROCM_ENABLE_STREAMING_MADVISE_WILLNEED` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming madvise willneed. | [ds4.c:18224](ds4.c#L18224) | @@ -1066,7 +1068,7 @@ and **19 tool/wrapper entries**. | `DS4_ROCM_MOE_WRITE_CLAMPED_ACT` | Pure presence sentinel: any defined value, including empty or "0", is active; DS4_METAL_MOE_WRITE_CLAMPED_ACT is an accepted fallback alias. On ROCm the variable is only consumed as a path-admission veto: it disables selected-expert cache/address-table, selected-slot and CPU-router/fused optimized paths. No ROCm call site parses a clamp amount or directly enables a write-clamped kernel. | Force shared graph selection away from optimizations incompatible with the clamped-intermediate MoE diagnostic; on ROCm this is a compatibility/rollback gate, not itself a clamped-write implementation. | [ds4.c:18526](ds4.c#L18526) | | `DS4_ROCM_MXFP4_DOWN_RGROUP` | nonempty value is parsed by strtol and a numeric prefix is sufficient; integers 1..8 are accepted; unset, empty, invalid, or out-of-range values use 1 | Set how many 32-row output blocks each ROCm MXFP4 tiled down-projection block computes, reducing the first launch-grid dimension as the value increases. | [rocm/ds4_rocm_moe_launch.cuh:801](rocm/ds4_rocm_moe_launch.cuh#L801) | | `DS4_ROCM_Q4_GROUPED_ATTN_A_STATS` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Print counters for rocm q4 grouped attn a stats. | [rocm/ds4_rocm_q4.cuh:614](rocm/ds4_rocm_q4.cuh#L614) | -| `DS4_ROCM_Q4_PREFILL_TILE8_STATS` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Print counters for rocm q4 prefill tile8 stats. | [rocm/ds4_rocm_q4.cuh:514](rocm/ds4_rocm_q4.cuh#L514) | +| `DS4_ROCM_Q4_PREFILL_TILE8_STATS` | presence diagnostic; unset=off; any defined value including empty or 0 prints at exit | Print tiled-prefill dense/pair/attention counters, total and SSD-specific K=1024 TILE4 dispatches, and direct-WMMA total/K32/K64/K128 launch counts. | [rocm/ds4_rocm_q4.cuh:1454](rocm/ds4_rocm_q4.cuh#L1454) | | `DS4_ROCM_Q4_PREFILL_WMMA_ROW_TILE` | unsigned integer; unset, empty, malformed, negative, or values other than 64/128/256 use shape selection (64 rows when M<1024, 128 when M<8192, otherwise 256); 64 retains the previous geometry | Override the number of output rows sharing each direct-Q4 64x32 activation tile for controlled 64/128/256-row ROCm WMMA A/B measurements. | [rocm/ds4_rocm_q4.cuh:1517](rocm/ds4_rocm_q4.cuh#L1517) | | `DS4_ROCM_Q8_DECODE_SHAREDX_64K` | sampled once; unset: enabled; present empty or exact 0: disabled; every other present value: enabled; effective only for one-token non-prequant Q8_0 matmul with 8192 < in_dim <= 16384 | Allows the ROCm shared-input Q8 decode kernel to use up to 64 KiB dynamic LDS for wide inputs; an unsupported/failed LDS launch automatically falls back to the regular kernel. | [rocm/ds4_rocm_runtime.cuh:4805](rocm/ds4_rocm_runtime.cuh#L4805) | | `DS4_ROCM_Q_STAGE_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm q stage profile. | [ds4.c:29284](ds4.c#L29284) | diff --git a/rocm/ds4_rocm_q4.cuh b/rocm/ds4_rocm_q4.cuh index 8760961f6..d1bf027e5 100644 --- a/rocm/ds4_rocm_q4.cuh +++ b/rocm/ds4_rocm_q4.cuh @@ -132,8 +132,9 @@ static_assert((sizeof(cuda_block_q8_K) % sizeof(uint32_t)) == 0u, * are written straight to the output. The row tile is shape-selected to 64, * 128, or 256 so every activation tile is reused as broadly as the output * shape permits; the retained 64-row instantiation is also an A/B control. - * The wider candidates can stage two adjacent F32 activations per iteration, - * matching the established Q8 WMMA load/conversion pattern. Each lane + * K64 can stage two adjacent F32 activations per loader iteration, matching + * the established Q8 WMMA load/conversion pattern; aligned 256-row K128 + * launches stage four. Each lane * dequantizes one Q4_K row/group directly into two F16 register vectors, so * there is no Q8_K activation scratch and no persistent F16 weight sidecar. * @@ -148,6 +149,8 @@ enum { ROCM_Q4_WMMA_K_TILE = 32u, ROCM_Q4_WMMA_K64_TILE = 64u, ROCM_Q4_WMMA_K64_LDS_PITCH = 80u, + ROCM_Q4_WMMA_K128_TILE = 128u, + ROCM_Q4_WMMA_K128_LDS_PITCH = 144u, ROCM_Q4_WMMA_FRAGMENT = 16u, }; static_assert(ROCM_Q4_WMMA_K64_TILE == 2u * ROCM_Q4_WMMA_K_TILE, @@ -155,6 +158,11 @@ static_assert(ROCM_Q4_WMMA_K64_TILE == 2u * ROCM_Q4_WMMA_K_TILE, static_assert(ROCM_Q4_WMMA_K64_LDS_PITCH >= ROCM_Q4_WMMA_K64_TILE && (ROCM_Q4_WMMA_K64_LDS_PITCH % ROCM_Q4_WMMA_FRAGMENT) == 0u, "K64 LDS rows must preserve aligned half16 WMMA loads"); +static_assert(ROCM_Q4_WMMA_K128_TILE == 2u * ROCM_Q4_WMMA_K64_TILE, + "K128 must combine exactly four adjacent Q4_K qgroups"); +static_assert(ROCM_Q4_WMMA_K128_LDS_PITCH >= ROCM_Q4_WMMA_K128_TILE && + (ROCM_Q4_WMMA_K128_LDS_PITCH % ROCM_Q4_WMMA_FRAGMENT) == 0u, + "K128 LDS rows must preserve aligned half16 WMMA loads"); #if defined(__HIP_DEVICE_COMPILE__) && __HIP_DEVICE_COMPILE__ && \ defined(__gfx1151__) && \ @@ -546,6 +554,225 @@ rocm_matmul_q4_K_prefill_wmma_k64_p80_rowtile_strided_kernel( } } +#pragma unroll + for (uint32_t token_tile = 0u; token_tile < 4u; token_tile++) { + const uint32_t tok = + tok0 + token_tile * ROCM_Q4_WMMA_FRAGMENT + lane16; + if (tok >= n_tok) continue; + const ds4_q4_float8_t acc = token_tile == 0u + ? acc0 + : (token_tile == 1u ? acc1 + : (token_tile == 2u ? acc2 : acc3)); +#pragma unroll + for (uint32_t j = 0u; j < 8u; j++) { + const uint32_t row = wave_row0 + 2u * j + (lane >> 4u); + if (row < out_dim) { + out[(uint64_t)tok * out_token_stride + + (uint64_t)group * out_dim + row] = acc[j]; + } + } + } +#else + (void)out; + (void)w_base; + (void)x; + (void)n_tok; + (void)n_groups; + (void)in_dim; + (void)out_dim; + (void)row_bytes; + (void)x_token_stride; + (void)x_group_stride; + (void)out_token_stride; +#endif +} + +/* Long-prefill q_b K128 stage. Four adjacent 32-value activation groups are + * staged behind one barrier pair, reducing the K64/P80 synchronization count + * by another 2x. P144 retains a 32-byte-aligned half16 base while rotating + * consecutive token rows across LDS banks. float4 loads also halve the + * activation-load instruction count relative to K64's float2 loader without + * changing any F32->F16 rounding or the qgroup accumulation order. + * + * Keep this as a separate kernel and instantiate it only for the 256-row + * geometry: its 18 KiB LDS tile is appropriate for q_b's 32768 output rows, + * but could reduce occupancy on the smaller projections. */ +template +__launch_bounds__(WAVES * 32u, MIN_BLOCKS) +__global__ static void +rocm_matmul_q4_K_prefill_wmma_k128_p144_rowtile_strided_kernel( + float *out, + const char *w_base, + const float *x, + uint32_t n_tok, + uint32_t n_groups, + uint32_t in_dim, + uint32_t out_dim, + uint64_t row_bytes, + uint64_t x_token_stride, + uint64_t x_group_stride, + uint64_t out_token_stride) { +#if DS4_ROCM_Q4_GFX1151_WMMA_ROWTILE_DEVICE + if (warpSize != 32) return; + + const uint32_t tid = threadIdx.x; + const uint32_t wave = tid >> 5u; + const uint32_t lane = tid & 31u; + const uint32_t lane16 = lane & 15u; + const uint32_t group = blockIdx.z; + static_assert(ROW_TILE == WAVES * ROCM_Q4_WMMA_FRAGMENT, + "one Q4 WMMA wave must own exactly 16 output rows"); + const uint32_t row0 = blockIdx.x * ROW_TILE; + const uint32_t tok0 = blockIdx.y * ROCM_Q4_WMMA_TOKEN_TILE; + if (group >= n_groups) return; + + const uint32_t wave_row0 = row0 + wave * ROCM_Q4_WMMA_FRAGMENT; + const uint32_t my_row = wave_row0 + lane16; + const uint32_t safe_row = my_row < out_dim ? my_row : out_dim - 1u; + const cuda_block_q4_K *row_blocks = + reinterpret_cast( + w_base + ((uint64_t)group * out_dim + safe_row) * row_bytes); + const uint32_t q4_blocks = in_dim / CUDA_QK_K; + + ds4_q4_float8_t acc0 = {0.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 0.0f}; + ds4_q4_float8_t acc1 = acc0; + ds4_q4_float8_t acc2 = acc0; + ds4_q4_float8_t acc3 = acc0; + __shared__ __align__(32) _Float16 + lds_x[ROCM_Q4_WMMA_TOKEN_TILE * ROCM_Q4_WMMA_K128_LDS_PITCH]; + + for (uint32_t block_index = 0u; block_index < q4_blocks; + block_index++) { + const cuda_block_q4_K *block = row_blocks + block_index; + const float block_d = dev_f16_to_f32(block->d); + const float block_dm = dev_f16_to_f32(block->dmin); + + /* Each Q4_K block contains eight qgroups. Stage groups 0..3 and + * 4..7 in two passes; the nested loops still consume qgroups in the + * exact 0,1,...,7 order used by K32 and K64. */ +#pragma unroll + for (uint32_t qpair_base = 0u; qpair_base < 4u; + qpair_base += 2u) { + const uint32_t group32_base = + block_index * 8u + qpair_base * 2u; + for (uint32_t j = tid * 4u; + j < ROCM_Q4_WMMA_TOKEN_TILE * ROCM_Q4_WMMA_K128_TILE; + j += blockDim.x * 4u) { + const uint32_t tok_local = j >> 7u; + const uint32_t kk = j & 127u; + const uint32_t tok = tok0 + tok_local; + half2 value01 = __floats2half2_rn(0.0f, 0.0f); + half2 value23 = value01; + if (tok < n_tok) { + const float4 values = + *reinterpret_cast( + x + (uint64_t)tok * x_token_stride + + (uint64_t)group * x_group_stride + + (uint64_t)group32_base * + ROCM_Q4_WMMA_K_TILE + + kk); + value01 = __floats2half2_rn(values.x, values.y); + value23 = __floats2half2_rn(values.z, values.w); + } + _Float16 *const dst = + lds_x + + tok_local * ROCM_Q4_WMMA_K128_LDS_PITCH + kk; + *reinterpret_cast(dst) = value01; + *reinterpret_cast(dst + 2u) = value23; + } + __syncthreads(); + + /* Do not retain both packed qpair payloads at once. Their + * lifetime would increase VGPR pressure in the 16-wave q_b + * workgroup and erase the synchronization win. */ +#pragma unroll 1 + for (uint32_t qpair_offset = 0u; qpair_offset < 2u; + qpair_offset++) { + const uint32_t qpair = qpair_base + qpair_offset; + ds4_q4_uchar16_t packed0; + ds4_q4_uchar16_t packed1; + __builtin_memcpy( + &packed0, block->qs + qpair * 32u, sizeof(packed0)); + __builtin_memcpy( + &packed1, + block->qs + qpair * 32u + ROCM_Q4_WMMA_FRAGMENT, + sizeof(packed1)); + +#pragma unroll 1 + for (uint32_t nibble = 0u; nibble < 2u; nibble++) { + const uint32_t qgroup = qpair * 2u + nibble; + uint8_t scale = 0u; + uint8_t minimum = 0u; + dev_q4_K_get_scale_min( + qgroup, block->scales, &scale, &minimum); + const float d = block_d * (float)scale; + const float dm = block_dm * (float)minimum; + const uint32_t shift = nibble * 4u; + ds4_q4_half16_t weights0; + ds4_q4_half16_t weights1; +#pragma unroll + for (uint32_t i = 0u; i < ROCM_Q4_WMMA_FRAGMENT; i++) { + const uint8_t q0 = (packed0[i] >> shift) & 0x0fu; + const uint8_t q1 = (packed1[i] >> shift) & 0x0fu; + weights0[i] = (_Float16)(d * (float)q0 - dm); + weights1[i] = (_Float16)(d * (float)q1 - dm); + } + +#pragma unroll + for (uint32_t token_tile = 0u; token_tile < 4u; + token_tile++) { + const uint32_t token_local = + token_tile * ROCM_Q4_WMMA_FRAGMENT + lane16; + const uint32_t activation_offset = + (qpair_offset * 2u + nibble) * + ROCM_Q4_WMMA_K_TILE; + const _Float16 *activation = + lds_x + + token_local * ROCM_Q4_WMMA_K128_LDS_PITCH + + activation_offset; + const ds4_q4_half16_t activation0 = + *reinterpret_cast( + activation); + const ds4_q4_half16_t activation1 = + *reinterpret_cast( + activation + ROCM_Q4_WMMA_FRAGMENT); + if (token_tile == 0u) { + acc0 = + __builtin_amdgcn_wmma_f32_16x16x16_f16_w32( + weights0, activation0, acc0); + acc0 = + __builtin_amdgcn_wmma_f32_16x16x16_f16_w32( + weights1, activation1, acc0); + } else if (token_tile == 1u) { + acc1 = + __builtin_amdgcn_wmma_f32_16x16x16_f16_w32( + weights0, activation0, acc1); + acc1 = + __builtin_amdgcn_wmma_f32_16x16x16_f16_w32( + weights1, activation1, acc1); + } else if (token_tile == 2u) { + acc2 = + __builtin_amdgcn_wmma_f32_16x16x16_f16_w32( + weights0, activation0, acc2); + acc2 = + __builtin_amdgcn_wmma_f32_16x16x16_f16_w32( + weights1, activation1, acc2); + } else { + acc3 = + __builtin_amdgcn_wmma_f32_16x16x16_f16_w32( + weights0, activation0, acc3); + acc3 = + __builtin_amdgcn_wmma_f32_16x16x16_f16_w32( + weights1, activation1, acc3); + } + } + } + } + __syncthreads(); + } + } + #pragma unroll for (uint32_t token_tile = 0u; token_tile < 4u; token_tile++) { const uint32_t tok = @@ -1219,12 +1446,26 @@ enum { * not merely produce output through a canonical fallback. */ static uint64_t g_rocm_q4_prefill_wmma_launches; static uint64_t g_rocm_q4_prefill_wmma_k64_launches; +static uint64_t g_rocm_q4_prefill_wmma_k128_launches; +static int g_rocm_q4_prefill_tile8_report_registered; + +static void rocm_q4_K_prefill_tile8_report(void); + +static void rocm_q4_K_prefill_stats_register(void) { + if (getenv("DS4_ROCM_Q4_PREFILL_TILE8_STATS") == NULL) return; + if (__atomic_exchange_n(&g_rocm_q4_prefill_tile8_report_registered, 1, + __ATOMIC_ACQ_REL) == 0) { + (void)atexit(rocm_q4_K_prefill_tile8_report); + } +} extern "C" void ds4_rocm_test_q4_prefill_wmma_reset(void) { __atomic_store_n(&g_rocm_q4_prefill_wmma_launches, 0u, __ATOMIC_RELAXED); __atomic_store_n(&g_rocm_q4_prefill_wmma_k64_launches, 0u, __ATOMIC_RELAXED); + __atomic_store_n(&g_rocm_q4_prefill_wmma_k128_launches, 0u, + __ATOMIC_RELAXED); } extern "C" uint64_t ds4_rocm_test_q4_prefill_wmma_get_calls(void) { @@ -1237,6 +1478,11 @@ extern "C" uint64_t ds4_rocm_test_q4_prefill_wmma_k64_get_calls(void) { __ATOMIC_RELAXED); } +extern "C" uint64_t ds4_rocm_test_q4_prefill_wmma_k128_get_calls(void) { + return __atomic_load_n(&g_rocm_q4_prefill_wmma_k128_launches, + __ATOMIC_RELAXED); +} + static int rocm_q4_K_prefill_wmma_k64_control_policy(int control) { return control != 0; } @@ -1246,6 +1492,28 @@ extern "C" int ds4_rocm_test_q4_prefill_wmma_k64_control_policy( return rocm_q4_K_prefill_wmma_k64_control_policy(control); } +/* K128 is the production default for its aligned q_b-style 256-row scope. + * Keep a value-aware opt-out for tester rollback, layer it on top of the K64 + * control so K64=0 remains a reliable K32 rollback, and retain K64 for every + * incompatible launch. */ +static int rocm_q4_K_prefill_wmma_k128_policy( + int disabled, + int k64_enabled, + uint32_t row_tile, + int load4_compatible) { + return disabled != 1 && k64_enabled && row_tile == 256u && + load4_compatible; +} + +extern "C" int ds4_rocm_test_q4_prefill_wmma_k128_policy( + int disabled, + int k64_enabled, + uint32_t row_tile, + int load4_compatible) { + return rocm_q4_K_prefill_wmma_k128_policy( + disabled, k64_enabled, row_tile, load4_compatible); +} + /* Resident execution is automatic after validation. Preserve an explicit * ENABLE=0 as a compatibility opt-out, while REQUIRE remains a strict * assertion rather than the switch that turns the candidate on. SSD @@ -1402,6 +1670,15 @@ static int rocm_q4_K_prefill_wmma_load2_compatible( ((x_group_stride & 1u) == 0u); } +static int rocm_q4_K_prefill_wmma_load4_compatible( + const float *x, + uint64_t x_token_stride, + uint64_t x_group_stride) { + return x && (((uintptr_t)x & 15u) == 0u) && + ((x_token_stride & 3u) == 0u) && + ((x_group_stride & 3u) == 0u); +} + static void rocm_q4_K_prefill_wmma_enqueue( float *out, const char *w, @@ -1510,6 +1787,32 @@ static void rocm_q4_K_prefill_wmma_k64_enqueue( } } +/* The K128 path is deliberately a single q_b-oriented instantiation. + * Keeping it out of the generic K32/K64 template family preserves the codegen + * of both established benchmark arms. */ +static void rocm_q4_K_prefill_wmma_k128_enqueue( + float *out, + const char *w, + const float *x, + uint32_t n_tok, + uint32_t n_groups, + uint32_t in_dim, + uint32_t out_dim, + uint64_t row_bytes, + uint64_t x_token_stride, + uint64_t x_group_stride, + uint64_t out_token_stride) { + const dim3 grid( + (unsigned)(((uint64_t)out_dim + 255u) / 256u), + (unsigned)(((uint64_t)n_tok + ROCM_Q4_WMMA_TOKEN_TILE - 1u) / + ROCM_Q4_WMMA_TOKEN_TILE), + n_groups); + rocm_matmul_q4_K_prefill_wmma_k128_p144_rowtile_strided_kernel< + 256u, 16u, 1u><<>>( + out, w, x, n_tok, n_groups, in_dim, out_dim, row_bytes, + x_token_stride, x_group_stride, out_token_stride); +} + static uint32_t rocm_q4_K_prefill_wmma_row_tile(uint32_t out_dim) { const uint32_t shape_tile = out_dim >= 8192u ? 256u @@ -1559,14 +1862,25 @@ static int rocm_q4_K_prefill_wmma_launch( const int load2 = row_tile != 64u && rocm_q4_K_prefill_wmma_load2_compatible( x, x_token_stride, x_group_stride); - /* K64 is the automatic geometry once the normal direct-Q4 WMMA selector - * has accepted this launch. Preserve a value-aware, selective K32 - * rollback: unset/true selects K64 and 0/false/no/off selects K32. */ + /* K64 is the base staging mode once the normal direct-Q4 WMMA selector + * has accepted this launch; eligible 256-row launches layer K128 on it. + * Preserve a value-aware K32 rollback: unset/true permits the wider + * stages and 0/false/no/off selects K32. */ const int k64_control = rocm_q4_attn_q_b_env_bool( "DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_K64"); const int use_k64 = rocm_q4_K_prefill_wmma_k64_control_policy(k64_control); - if (use_k64) { + const int k128_disabled = rocm_q4_attn_q_b_env_bool( + "DS4_ROCM_DISABLE_Q4_PREFILL_WMMA_K128"); + const int load4 = rocm_q4_K_prefill_wmma_load4_compatible( + x, x_token_stride, x_group_stride); + const int use_k128 = rocm_q4_K_prefill_wmma_k128_policy( + k128_disabled, use_k64, row_tile, load4); + if (use_k128) { + rocm_q4_K_prefill_wmma_k128_enqueue( + out, w, x, n_tok, n_groups, in_dim, out_dim, row_bytes, + x_token_stride, x_group_stride, out_token_stride); + } else if (use_k64) { rocm_q4_K_prefill_wmma_k64_enqueue( out, w, x, n_tok, n_groups, in_dim, out_dim, row_bytes, x_token_stride, x_group_stride, out_token_stride, row_tile, load2); @@ -1575,14 +1889,20 @@ static int rocm_q4_K_prefill_wmma_launch( out, w, x, n_tok, n_groups, in_dim, out_dim, row_bytes, x_token_stride, x_group_stride, out_token_stride, row_tile, load2); } - const char *launch_label = use_k64 - ? "q4_K prefill WMMA K64/P80 rowtile launch" - : (label ? label : "q4_K prefill WMMA rowtile launch"); + const char *launch_label = use_k128 + ? "q4_K prefill WMMA K128/P144 rowtile launch" + : (use_k64 + ? "q4_K prefill WMMA K64/P80 rowtile launch" + : (label ? label : "q4_K prefill WMMA rowtile launch")); const int ok = cuda_ok(cudaGetLastError(), launch_label); if (ok) { + rocm_q4_K_prefill_stats_register(); __atomic_fetch_add(&g_rocm_q4_prefill_wmma_launches, 1u, __ATOMIC_RELAXED); - if (use_k64) { + if (use_k128) { + __atomic_fetch_add(&g_rocm_q4_prefill_wmma_k128_launches, 1u, + __ATOMIC_RELAXED); + } else if (use_k64) { __atomic_fetch_add(&g_rocm_q4_prefill_wmma_k64_launches, 1u, __ATOMIC_RELAXED); } @@ -1701,6 +2021,41 @@ extern "C" int ds4_rocm_bench_q4_K_wmma_variant_enqueue( return 1; } +/* Strict direct hook for the q_b K128/P144 candidate. Unlike production + * dispatch, incompatibility fails instead of silently falling back to K64 so + * benchmark output cannot be mislabeled. */ +extern "C" int ds4_rocm_bench_q4_K_wmma_k128_enqueue( + void *out, + const void *w, + const void *x, + uint32_t n_tok, + uint32_t n_groups, + uint32_t in_dim, + uint32_t out_dim, + uint64_t row_bytes, + uint64_t x_token_stride, + uint64_t x_group_stride, + uint64_t out_token_stride) { + if (!out || !w || !x || n_tok == 0u || n_groups == 0u || + in_dim == 0u || out_dim == 0u || + (in_dim % CUDA_QK_K) != 0u || out_dim < 8192u || + !rocm_q4_K_prefill_wmma_load4_compatible( + reinterpret_cast(x), + x_token_stride, x_group_stride)) { + return 0; + } + const uint64_t minimum_row_bytes = + ((uint64_t)in_dim / CUDA_QK_K) * sizeof(cuda_block_q4_K); + if (row_bytes < minimum_row_bytes) return 0; + rocm_q4_K_prefill_wmma_k128_enqueue( + reinterpret_cast(out), + reinterpret_cast(w), + reinterpret_cast(x), + n_tok, n_groups, in_dim, out_dim, row_bytes, + x_token_stride, x_group_stride, out_token_stride); + return 1; +} + enum { ROCM_Q4_PREFILL_K1024_TILE4_REQUIRED_FAILURE = -1, ROCM_Q4_PREFILL_K1024_TILE4_FALLBACK = 0, @@ -1799,7 +2154,6 @@ static uint64_t g_rocm_q4_prefill_tile8_attention_batch_calls; static uint64_t g_rocm_q4_prefill_k1024_tile4_calls; static uint64_t g_rocm_q4_prefill_k1024_tile4_ssd_calls; static uint64_t g_rocm_q4_prefill_tile8_tokens; -static int g_rocm_q4_prefill_tile8_report_registered; static pthread_mutex_t g_rocm_q4_prefill_tile8_stats_mutex = PTHREAD_MUTEX_INITIALIZER; @@ -1852,6 +2206,18 @@ static int rocm_q4_K_grouped_attn_a_result(int rc, uint32_t n_groups) { } static void rocm_q4_K_prefill_tile8_report(void) { + const uint64_t wmma_calls = + __atomic_load_n(&g_rocm_q4_prefill_wmma_launches, + __ATOMIC_RELAXED); + const uint64_t wmma_k64_calls = + __atomic_load_n(&g_rocm_q4_prefill_wmma_k64_launches, + __ATOMIC_RELAXED); + const uint64_t wmma_k128_calls = + __atomic_load_n(&g_rocm_q4_prefill_wmma_k128_launches, + __ATOMIC_RELAXED); + const uint64_t staged_calls = wmma_k64_calls + wmma_k128_calls; + const uint64_t wmma_k32_calls = + wmma_calls >= staged_calls ? wmma_calls - staged_calls : 0u; pthread_mutex_lock(&g_rocm_q4_prefill_tile8_stats_mutex); const uint64_t dense_calls = g_rocm_q4_prefill_tile8_dense_calls; const uint64_t pair_calls = g_rocm_q4_prefill_tile8_pair_calls; @@ -1867,12 +2233,17 @@ static void rocm_q4_K_prefill_tile8_report(void) { "ds4: ROCm Q4_K tiled-prefill stats: " "dense_calls=%llu pair_calls=%llu attention_batch_calls=%llu " "k1024_tile4_calls=%llu k1024_tile4_ssd_calls=%llu " - "tokens=%llu\n", + "wmma_calls=%llu wmma_k32_calls=%llu wmma_k64_calls=%llu " + "wmma_k128_calls=%llu tokens=%llu\n", (unsigned long long)dense_calls, (unsigned long long)pair_calls, (unsigned long long)attention_batch_calls, (unsigned long long)k1024_tile4_calls, (unsigned long long)k1024_tile4_ssd_calls, + (unsigned long long)wmma_calls, + (unsigned long long)wmma_k32_calls, + (unsigned long long)wmma_k64_calls, + (unsigned long long)wmma_k128_calls, (unsigned long long)tokens); } @@ -1883,11 +2254,8 @@ static void rocm_q4_K_prefill_tile8_note( uint32_t k1024_tile4_calls, uint64_t tokens) { if (getenv("DS4_ROCM_Q4_PREFILL_TILE8_STATS") == NULL) return; + rocm_q4_K_prefill_stats_register(); pthread_mutex_lock(&g_rocm_q4_prefill_tile8_stats_mutex); - if (!g_rocm_q4_prefill_tile8_report_registered) { - g_rocm_q4_prefill_tile8_report_registered = 1; - (void)atexit(rocm_q4_K_prefill_tile8_report); - } g_rocm_q4_prefill_tile8_dense_calls += dense_calls; g_rocm_q4_prefill_tile8_pair_calls += pair_calls; g_rocm_q4_prefill_tile8_attention_batch_calls += attention_batch_calls; diff --git a/scripts/environment_variables.tsv b/scripts/environment_variables.tsv index 1c2e11cc9..2bdf900fd 100644 --- a/scripts/environment_variables.tsv +++ b/scripts/environment_variables.tsv @@ -980,7 +980,8 @@ runtime/rocm DS4_ROCM_DISABLE_Q4_GROUPED_ATTN_A presence rollback; unset permits runtime/rocm DS4_ROCM_DISABLE_Q4_PREFILL_K1024_TILE4 value-aware authoritative rollback; unset/0/false/no/off preserves the resident automatic default and any explicit SSD request; empty or any other value disables; overrides ENABLE and causes REQUIRE to fail closed Restore the generic eight-block Q4_K tiled-prefill kernel for K=1024 in both resident and SSD-streaming execution. rocm/ds4_rocm_q4.cuh:624 runtime/rocm DS4_ROCM_DISABLE_Q4_PREFILL_Q8_K_WAVE32 value-aware authoritative rollback; unset/0/false/no/off permits ENABLE or REQUIRE, while empty or any other value disables; REQUIRE then fails closed Restore the canonical one-workgroup-per-Q8_K-block activation quantizer for Q4 prefill. rocm/ds4_rocm_q4.cuh:803 runtime/rocm DS4_ROCM_DISABLE_Q4_PREFILL_TILE8 presence rollback; TILE8 is default for 9..4096 tokens Disable/roll back rocm disable q4 prefill tile8. rocm/ds4_rocm_q4.cuh:448 -runtime/rocm DS4_ROCM_DISABLE_Q4_PREFILL_WMMA value-aware authoritative opt-out for the automatic resident path and explicit SSD/REQUIRE requests; unset/0/false/no/off leaves policy unchanged, while empty or any other value disables; REQUIRE then fails closed Prevent the gfx1151 direct-Q4 WMMA prefill path from dispatching and retain the Q8_K-plus-TILE8/TILE4 path. rocm/ds4_rocm_q4.cuh:1358 +runtime/rocm DS4_ROCM_DISABLE_Q4_PREFILL_WMMA value-aware authoritative opt-out for the automatic resident path and explicit SSD/REQUIRE requests; unset/0/false/no/off leaves policy unchanged, while empty or any other value disables; REQUIRE then fails closed Prevent the gfx1151 direct-Q4 WMMA prefill path from dispatching and retain the Q8_K-plus-TILE8/TILE4 path. rocm/ds4_rocm_q4.cuh:1626 +runtime/rocm DS4_ROCM_DISABLE_Q4_PREFILL_WMMA_K128 value-aware rollback for the default K128/P144 stage; unset/0/false/no/off keeps K128 after the normal direct-Q4 WMMA gates, K64 control, 256-row geometry, and 16-byte activation alignment pass; empty or any other value restores K64; incompatible launches also retain K64 and K64=0 retains the K32 rollback Roll aligned resident q_b-shaped 256-row direct-WMMA launches back from four-qgroup K128/P144 staging and float4 activation loads to K64/P80. rocm/ds4_rocm_q4.cuh:1874 runtime/rocm DS4_ROCM_DISABLE_Q4_SELECTED_EXPERT_VIEWS presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable q4 selected expert views. ds4.c:21150 runtime/rocm DS4_ROCM_DISABLE_RESIDENT_IQ2_SORTED presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable resident iq2 sorted. rocm/ds4_rocm_moe_launch.cuh:751 runtime/rocm DS4_ROCM_DISABLE_ROUTED_PAIR_SWIGLU_FUSION presence rollback flag; unset keeps automatic/default path Disable/roll back rocm disable routed pair swiglu fusion. ds4.c:18543 @@ -1024,8 +1025,8 @@ runtime/rocm DS4_ROCM_ENABLE_Q4_DENSE_PAIR presence opt-in; unset=off; DISABLE t runtime/rocm DS4_ROCM_ENABLE_Q4_GROUPED_ATTN_A presence opt-in outside the default scope; the exact caller-marked resident decode shape groups=8, N=1, K=4096, M=1024 is automatic, while row-at-a-time batch fallbacks are not; DISABLE wins Enable grouped Q4 attention-A for eligible slices, non-production shapes, or explicit experiments in addition to the resident decode default. rocm/ds4_rocm_q4.cuh:872 runtime/rocm DS4_ROCM_ENABLE_Q4_PREFILL_K1024_TILE4_SSD value-aware SSD-only opt-in, default off; unset/0/false/no/off retains TILE8, while empty or any other value requests TILE4; eligibility additionally requires N=9..4096, K=1024, M=32768, TILE8 enabled, and the complete weight range in device storage rather than mapped/registered host memory; DISABLE wins Allow the four-lane K=1024 Q4_K prefill specialization to consume an already device-resident/cache-backed attn_q_b weight range during SSD streaming without changing model I/O. rocm/ds4_rocm_q4.cuh:624 runtime/rocm DS4_ROCM_ENABLE_Q4_PREFILL_Q8_K_WAVE32 value-aware opt-in, default off; unset/0/false/no/off retains the canonical quantizer, while empty or any other value requests the candidate for N=9..4096 on gfx1151 wave32; DISABLE wins; a selected exact Q8 path takes precedence over automatic direct-Q4 WMMA, REQUIRE_WMMA overrides an optional request, and dual REQUIRE fails closed Quantize eight independent Q8_K activation blocks per 256-thread workgroup using one wave32 per block, without LDS or workgroup barriers, before the exact Q4 prefill matmul (TILE8 or its legacy rollback). rocm/ds4_rocm_q4.cuh:858 -runtime/rocm DS4_ROCM_ENABLE_Q4_PREFILL_WMMA value-aware compatibility control; unset keeps automatic resident direct-Q4 WMMA for standalone dense and attention-output A while attention-output B remains on Q8_K+TILE8; empty or any value other than 0/false/no/off explicitly retains the same eligible A paths but no longer opts B into direct WMMA; explicit 0/false/no/off opts out unless REQUIRE is set, while DISABLE is the authoritative rollback; N=256..4096, K a positive multiple of 256, resident non-quality gfx1151 wave32 only; SSD has a separate gate Use compressed Q4_K-to-F16 register dequantization plus shape-selected 64-token by 64/128/256-row WMMA tiles and two-wide activation staging on the wider tiles, without Q8_K activation scratch or an F16 weight sidecar. rocm/ds4_rocm_q4.cuh:1354 -runtime/rocm DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_K64 value-aware compatibility control for the default K64/P80 stage; unset, empty, or any value other than 0/false/no/off selects K64/P80 after the normal direct-Q4 WMMA device, shape, residency, and quality gates pass; 0/false/no/off rolls back only to the established K32 stage; DS4_ROCM_DISABLE_Q4_PREFILL_WMMA wins Stage two adjacent 32-value Q4_K groups and a 64-value activation slice in one padded P80 LDS tile, halving workgroup barriers while preserving activation traffic and the K32 accumulation order. rocm/ds4_rocm_q4.cuh:1566 +runtime/rocm DS4_ROCM_ENABLE_Q4_PREFILL_WMMA value-aware compatibility control; unset keeps automatic resident direct-Q4 WMMA for standalone dense and attention-output A while attention-output B remains on Q8_K+TILE8; empty or any value other than 0/false/no/off explicitly retains the same eligible A paths but no longer opts B into direct WMMA; explicit 0/false/no/off opts out unless REQUIRE is set, while DISABLE is the authoritative rollback; N=256..4096, K a positive multiple of 256, resident non-quality gfx1151 wave32 only; SSD has a separate gate Use compressed Q4_K-to-F16 register dequantization plus shape-selected 64-token by 64/128/256-row WMMA tiles, K64/P80 staging on 64/128 rows, and default K128/P144 float4 staging on aligned 256 rows, without Q8_K activation scratch or an F16 weight sidecar. rocm/ds4_rocm_q4.cuh:1622 +runtime/rocm DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_K64 value-aware base staging control; unset, empty, or any value other than 0/false/no/off uses K64/P80 on 64/128-row or K128-incompatible launches and permits default K128/P144 on aligned 256-row launches; 0/false/no/off suppresses both wider stages and rolls back to K32; DS4_ROCM_DISABLE_Q4_PREFILL_WMMA wins Stage two adjacent 32-value Q4_K groups and a 64-value activation slice in one padded P80 LDS tile as the narrower geometry and K128 fallback, halving K32 workgroup barriers while preserving its activation traffic and accumulation order. rocm/ds4_rocm_q4.cuh:1870 runtime/rocm DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_SSD value-aware SSD-only opt-in, default off; unset/0/false/no/off retains TILE8/TILE4, while empty or any other value requests direct-Q4 WMMA for eligible standalone projections and attention-output A but leaves attention-output B on Q8_K+TILE8; eligibility additionally requires each complete projection weight range in physical device storage rather than mapped/registered host memory; DISABLE wins Allow the compressed direct-Q4 WMMA kernel to consume an already device-resident/cache-backed Q4_K projection during SSD streaming without changing model I/O. rocm/ds4_rocm_q4.cuh:1356 runtime/rocm DS4_ROCM_ENABLE_STREAMING_FULL_EXPERT_ADDR_TABLE presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming full expert addr table. ds4.c:18255 runtime/rocm DS4_ROCM_ENABLE_STREAMING_MADVISE_WILLNEED presence opt-in flag; unset=off unless paired policy is automatic Enable rocm enable streaming madvise willneed. ds4.c:18224 @@ -1083,7 +1084,7 @@ runtime/rocm DS4_ROCM_Q4_ATTN_Q_B_F16_CACHE_MB integer MiB with a full-string pa runtime/rocm DS4_ROCM_Q4_ATTN_Q_B_F16_CACHE_MIN_TOKENS integer token count with a full-string parse; default 512; accepted range 32..UINT32_MAX, values above clamp and invalid or smaller values restore the default Set the minimum prefill batch eligible to prepare or use the ROCm Q4 attn_q_b F16 sidecars. rocm/ds4_rocm_q4_qb_sidecar.cuh:156 runtime/rocm DS4_ROCM_Q4_ATTN_Q_B_TRANSIENT_F16_MIN_TOKENS full-string unsigned token count; default 4096; accepted range 32..UINT32_MAX; values above clamp and invalid or smaller values restore 4096 Set the minimum device-resident, non-SSD prefill batch eligible for per-layer transient ROCm Q4_K attn_q_b-to-F16 expansion. rocm/ds4_rocm_q4_qb_sidecar.cuh:150 runtime/rocm DS4_ROCM_Q4_GROUPED_ATTN_A_STATS presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Print counters for rocm q4 grouped attn a stats. rocm/ds4_rocm_q4.cuh:614 -runtime/rocm DS4_ROCM_Q4_PREFILL_TILE8_STATS presence diagnostic; unset=off; any defined value including empty or 0 prints at exit Print tiled-prefill dense/pair/attention counters plus total and SSD-specific K=1024 TILE4 dispatch counts. rocm/ds4_rocm_q4.cuh:741 +runtime/rocm DS4_ROCM_Q4_PREFILL_TILE8_STATS presence diagnostic; unset=off; any defined value including empty or 0 prints at exit Print tiled-prefill dense/pair/attention counters, total and SSD-specific K=1024 TILE4 dispatches, and direct-WMMA total/K32/K64/K128 launch counts. rocm/ds4_rocm_q4.cuh:1454 runtime/rocm DS4_ROCM_Q4_PREFILL_WMMA_ROW_TILE unsigned integer; unset, empty, malformed, negative, or values other than 64/128/256 use shape selection (64 rows when M<1024, 128 when M<8192, otherwise 256); 64 retains the previous geometry Override the number of output rows sharing each direct-Q4 64x32 activation tile for controlled 64/128/256-row ROCm WMMA A/B measurements. rocm/ds4_rocm_q4.cuh:1517 runtime/rocm DS4_ROCM_Q8_DECODE_SHAREDX_64K sampled once; unset: enabled; present empty or exact 0: disabled; every other present value: enabled; effective only for one-token non-prequant Q8_0 matmul with 8192 < in_dim <= 16384 Allows the ROCm shared-input Q8 decode kernel to use up to 64 KiB dynamic LDS for wide inputs; an unsupported/failed LDS launch automatically falls back to the regular kernel. rocm/ds4_rocm_runtime.cuh:4805 runtime/rocm DS4_ROCM_Q_STAGE_PROFILE presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) Collect timing/profile diagnostics for rocm q stage profile. ds4.c:30038 diff --git a/speed-bench/README.md b/speed-bench/README.md index 98e38b1f1..3217c7fe1 100644 --- a/speed-bench/README.md +++ b/speed-bench/README.md @@ -221,17 +221,35 @@ diagnostic because it includes A's deliberate F16 boundary rather than isolating B correctness. Raw two-stage direct-WMMA row-geometry and K32/K64 comparisons remain diagnostic-only kernel measurements. -Eligible direct-Q4 WMMA launches use K64/P80 staging by default. It stages two -adjacent 32-value Q4_K groups and a 64-value activation slice in one padded -P80 LDS tile, halving workgroup barriers while retaining the K32 activation -traffic and accumulation order. Leave -`DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_K64` unset or set it to a true value for this -default; set it to `0`, `false`, `no`, or `off` to restore K32 staging. -`DS4_ROCM_DISABLE_Q4_PREFILL_WMMA=1` rolls the whole direct-WMMA path back to -Q8_K plus TILE8/TILE4. - -Use the production 2048-token sweep to isolate default K64/P80 from the K32 -rollback without SSD-streaming or policy-selection noise: +Eligible aligned 256-row direct-Q4 WMMA launches use the q_b-focused K128/P144 +stage by default. It groups four adjacent Q4_K qgroups behind one barrier pair +and uses float4 activation loads, targeting a 2x reduction in synchronization +and activation-load instructions over K64 while retaining the same F16 +conversions and accumulation order. Set +`DS4_ROCM_DISABLE_Q4_PREFILL_WMMA_K128=1` to roll the same launch back to +K64/P80; unset or explicitly false keeps K128. Incompatible alignment or +64/128-row geometry automatically uses K64. + +K64/P80 stages two adjacent 32-value Q4_K groups and a 64-value activation +slice in one padded P80 LDS tile. Leave +`DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_K64` unset or true to permit both K64 and the +eligible K128 default; set it to `0`, `false`, `no`, or `off` to restore K32 +staging. `DS4_ROCM_DISABLE_Q4_PREFILL_WMMA=1` rolls the whole direct-WMMA path +back to Q8_K plus TILE8/TILE4. A persistent or automatic transient F16 q_b +projection bypasses the direct-Q4 candidates. + +The q_b microbenchmark contains a strict, same-process +`q_b_wmma_k64_k128` A/B with bitwise output and canary checks: + +``` +./speed-bench/rocm_q4_prefill_bench \ + --case qb --tokens 256,512,1024,2048,2049,4096 \ + --sets 4 --warmup 4 --samples 12 +``` + +Use the production 2048-token sweep to exercise default K128/P144 without +SSD-streaming or policy-selection noise. The same process also reports the +strict K64/P80 versus K128/P144 A/B above: ``` ./speed-bench/rocm_q4_prefill_bench \ diff --git a/speed-bench/rocm_q4_prefill_bench.cpp b/speed-bench/rocm_q4_prefill_bench.cpp index 2e6f13956..9f4cbf03d 100644 --- a/speed-bench/rocm_q4_prefill_bench.cpp +++ b/speed-bench/rocm_q4_prefill_bench.cpp @@ -39,6 +39,11 @@ extern "C" int ds4_rocm_bench_q4_K_wmma_variant_enqueue( uint64_t row_bytes, uint64_t x_token_stride, uint64_t x_group_stride, uint64_t out_token_stride, uint32_t row_tile, uint32_t k_tile, int load2); +extern "C" int ds4_rocm_bench_q4_K_wmma_k128_enqueue( + void *out, const void *w, const void *x, uint32_t n_tok, + uint32_t n_groups, uint32_t in_dim, uint32_t out_dim, + uint64_t row_bytes, uint64_t x_token_stride, + uint64_t x_group_stride, uint64_t out_token_stride); extern "C" void ds4_rocm_test_q4_prefill_wmma_reset(void); extern "C" uint64_t ds4_rocm_test_q4_prefill_wmma_get_calls(void); extern "C" uint64_t ds4_rocm_test_q4_prefill_wmma_k64_get_calls(void); @@ -89,6 +94,8 @@ constexpr const char *kWmmaRowTile = "DS4_ROCM_Q4_PREFILL_WMMA_ROW_TILE"; constexpr const char *kWmmaK64 = "DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_K64"; +constexpr const char *kWmmaK128Disable = + "DS4_ROCM_DISABLE_Q4_PREFILL_WMMA_K128"; constexpr const char *kQ8Wave32Enable = "DS4_ROCM_ENABLE_Q4_PREFILL_Q8_K_WAVE32"; constexpr const char *kQ8Wave32Disable = @@ -546,6 +553,7 @@ void select_legacy() { (void)unsetenv(kWmmaRequire); (void)unsetenv(kWmmaRowTile); (void)unsetenv(kWmmaK64); + (void)unsetenv(kWmmaK128Disable); (void)unsetenv(kQ8Wave32Enable); (void)unsetenv(kQ8Wave32Disable); (void)unsetenv(kQ8Wave32Require); @@ -563,6 +571,7 @@ void select_tile8(bool disable_k1024_tile4) { (void)unsetenv(kWmmaRequire); (void)unsetenv(kWmmaRowTile); (void)unsetenv(kWmmaK64); + (void)unsetenv(kWmmaK128Disable); (void)unsetenv(kQ8Wave32Enable); (void)unsetenv(kQ8Wave32Disable); (void)unsetenv(kQ8Wave32Require); @@ -590,6 +599,7 @@ void select_wmma_shape() { (void)unsetenv(kK1024Tile4Require); (void)unsetenv(kWmmaRowTile); (void)unsetenv(kWmmaK64); + (void)unsetenv(kWmmaK128Disable); } void select_wmma_attention_a_tile8_b() { @@ -605,6 +615,7 @@ void select_wmma_attention_a_tile8_b() { (void)unsetenv(kK1024Tile4Require); (void)unsetenv(kWmmaRowTile); (void)unsetenv(kWmmaK64); + (void)unsetenv(kWmmaK128Disable); ds4_rocm_test_q4_prefill_wmma_reset(); } @@ -1364,7 +1375,7 @@ bool run_qb(const model_fixture &model, const config &cfg, n_tokens, 1u, kQbK, kQbM, row_bytes, kQbK, 0u, kQbM, 256u, 64u, 1) != 0; }}; - return benchmark_arms( + if (!benchmark_arms( "q_b_wmma_k32_k64", n_tokens, kQbK, kQbM, cfg, k32_baseline, k64_candidate, [&]() { @@ -1378,6 +1389,38 @@ bool run_qb(const model_fixture &model, const config &cfg, "q_b WMMA K32 oracle") && check_guard(tile4.ptr, logical_bytes, "q_b WMMA K64/P80 oracle"); + })) return false; + + const arm k64_baseline = { + "wmma_k64p80_rows256_load2", []() {}, + [&](uint32_t set) { + return ds4_rocm_bench_q4_K_wmma_variant_enqueue( + rows64_device, qb_weights[set], x_device, + n_tokens, 1u, kQbK, kQbM, row_bytes, + kQbK, 0u, kQbM, 256u, 64u, 1) != 0; + }}; + const arm k128_candidate = { + "wmma_k128p144_rows256_load4", []() {}, + [&](uint32_t set) { + return ds4_rocm_bench_q4_K_wmma_k128_enqueue( + shape_device, qb_weights[set], x_device, + n_tokens, 1u, kQbK, kQbM, row_bytes, + kQbK, 0u, kQbM) != 0; + }}; + return benchmark_arms( + "q_b_wmma_k64_k128", n_tokens, kQbK, kQbM, cfg, + k64_baseline, k128_candidate, + [&]() { + return poison_output(tile8.ptr, logical_bytes, 0x7fc10001u) && + poison_output(tile4.ptr, logical_bytes, 0x7fc20002u); + }, + [&]() { + return bitwise_equal(tile8.ptr, tile4.ptr, logical_bytes, + "q_b WMMA K64/P80 vs K128/P144") && + check_guard(tile8.ptr, logical_bytes, + "q_b WMMA K64/P80 oracle") && + check_guard(tile4.ptr, logical_bytes, + "q_b WMMA K128/P144 oracle"); }); } @@ -1808,8 +1851,9 @@ void usage(FILE *stream, const char *argv0) { "composed all-TILE8 delta is diagnostic only. Other WMMA comparisons\n" "use a finite/toleranced oracle because their F16 boundary is not\n" "bit-identical to the Q8_K activation path. At N>=256 the raw direct\n" - "arms also compare rollback K32 with default K64/P80 at fixed\n" - "production geometry and load2, requiring bitwise-identical output.\n", + "arms also compare K32 with K64/P80 at fixed production geometry and\n" + "load2; q_b additionally compares K64/P80 with the default K128/P144\n" + "load4 stage. Both staged-kernel checks require bitwise output.\n", argv0, kDefaultSets, kDefaultSamples, kDefaultWarmup); } @@ -1925,12 +1969,14 @@ int main(int argc, char **argv) { env_snapshot wmma_require_guard(kWmmaRequire); env_snapshot wmma_row_tile_guard(kWmmaRowTile); env_snapshot wmma_k64_guard(kWmmaK64); + env_snapshot wmma_k128_disable_guard(kWmmaK128Disable); env_snapshot q8_wave32_enable_guard(kQ8Wave32Enable); env_snapshot q8_wave32_disable_guard(kQ8Wave32Disable); env_snapshot q8_wave32_require_guard(kQ8Wave32Require); (void)unsetenv(kQ8Wave32Enable); (void)unsetenv(kQ8Wave32Disable); (void)unsetenv(kQ8Wave32Require); + (void)unsetenv(kWmmaK128Disable); int device_count = 0; hipError_t hip_rc = hipGetDeviceCount(&device_count); @@ -1998,9 +2044,10 @@ int main(int argc, char **argv) { properties.name, properties.gcnArchName, properties.warpSize, cfg.sets, static_cast(model.resident_bytes) / 1048576.0, cfg.wmma_supported ? "64,128,256" : "skipped", - cfg.wmma_supported ? "scalar,load2" : "skipped", - cfg.wmma_supported ? "32,64p80" : "skipped", - cfg.wmma_supported ? "64p80" : "skipped", + cfg.wmma_supported ? "scalar,load2,load4" : "skipped", + cfg.wmma_supported ? "32,64p80,128p144" : "skipped", + cfg.wmma_supported ? "128p144@rows256,64p80@rows64/128" + : "skipped", cfg.wmma_supported ? "available" : "skipped"); std::fflush(stdout); for (uint32_t n_tokens : cfg.tokens) { diff --git a/tests/test_rocm_q4_dense_pair.cpp b/tests/test_rocm_q4_dense_pair.cpp index e082059cc..abd6a2d38 100644 --- a/tests/test_rocm_q4_dense_pair.cpp +++ b/tests/test_rocm_q4_dense_pair.cpp @@ -37,8 +37,12 @@ extern "C" int ds4_rocm_test_q4_prefill_k1024_tile4_policy( extern "C" void ds4_rocm_test_q4_prefill_wmma_reset(void); extern "C" uint64_t ds4_rocm_test_q4_prefill_wmma_get_calls(void); extern "C" uint64_t ds4_rocm_test_q4_prefill_wmma_k64_get_calls(void); +extern "C" uint64_t ds4_rocm_test_q4_prefill_wmma_k128_get_calls(void); extern "C" int ds4_rocm_test_q4_prefill_wmma_k64_control_policy( int control); +extern "C" int ds4_rocm_test_q4_prefill_wmma_k128_policy( + int disabled, int k64_enabled, uint32_t row_tile, + int load4_compatible); extern "C" int ds4_rocm_test_q4_prefill_wmma_requested_policy( int ssd_streaming, int enabled, int ssd_enabled, int disabled, int required); @@ -121,6 +125,8 @@ constexpr const char *kPrefillWmmaRowTile = "DS4_ROCM_Q4_PREFILL_WMMA_ROW_TILE"; constexpr const char *kPrefillWmmaK64 = "DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_K64"; +constexpr const char *kPrefillWmmaK128Disable = + "DS4_ROCM_DISABLE_Q4_PREFILL_WMMA_K128"; constexpr const char *kPrefillQ8Wave32Enable = "DS4_ROCM_ENABLE_Q4_PREFILL_Q8_K_WAVE32"; constexpr const char *kPrefillQ8Wave32Disable = @@ -1368,12 +1374,43 @@ bool run_prefill_wmma_requested_policy_oracle() { k64_unset, k64_false, k64_true); ok = false; } + const int k128_unset = + ds4_rocm_test_q4_prefill_wmma_k128_policy( + -1, 1, 256u, 1); + const int k128_disabled = + ds4_rocm_test_q4_prefill_wmma_k128_policy( + 1, 1, 256u, 1); + const int k128_false = + ds4_rocm_test_q4_prefill_wmma_k128_policy( + 0, 1, 256u, 1); + const int k128_k32_rollback = + ds4_rocm_test_q4_prefill_wmma_k128_policy( + -1, 0, 256u, 1); + const int k128_rows128 = + ds4_rocm_test_q4_prefill_wmma_k128_policy( + -1, 1, 128u, 1); + const int k128_unaligned = + ds4_rocm_test_q4_prefill_wmma_k128_policy( + -1, 1, 256u, 0); + if (k128_unset != 1 || k128_disabled != 0 || k128_false != 1 || + k128_k32_rollback != 0 || k128_rows128 != 0 || + k128_unaligned != 0) { + std::fprintf(stderr, + "Q4 WMMA K128 opt-out policy: unset=%d disabled=%d " + "false=%d k32=%d rows128=%d unaligned=%d FAIL\n", + k128_unset, k128_disabled, k128_false, + k128_k32_rollback, k128_rows128, k128_unaligned); + ok = false; + } std::fprintf(stderr, "ROCm Q4 WMMA request policy oracle: cases=%zu " - "q8_yield=%d/%d/%d k64=%d/%d/%d %s\n", + "q8_yield=%d/%d/%d k64=%d/%d/%d " + "k128=%d/%d/%d/%d/%d/%d %s\n", sizeof(cases) / sizeof(cases[0]), optional_q8_yield, absent_q8_yield, strict_wmma_yield, k64_unset, k64_false, k64_true, + k128_unset, k128_disabled, k128_false, + k128_k32_rollback, k128_rows128, k128_unaligned, ok ? "PASS" : "FAIL"); return ok; } @@ -2831,11 +2868,11 @@ bool run_prefill_wmma_smoke(const aligned_model &model) { constexpr uint32_t n_tokens = 257u; const size_t logical_count = (size_t)n_tokens * kM0; /* A broken N-tail store could write the remaining 63 tokens of the final - * tile, while a simultaneous M-tail failure could address the remaining - * 63 rows of this 64-row launch. Cover both predicates failing together, - * not just the normal API canary. */ + * tile, while a simultaneous M-tail failure in the forced K128 rowtile + * could address the remaining 255 rows. Cover both predicates failing + * together, not just the normal API canary. */ constexpr size_t wmma_guard_floats = - (64u - 1u) * kM0 + (64u - 1u); + (64u - 1u) * kM0 + (256u - 1u); const size_t allocation_count = logical_count + wmma_guard_floats; const std::vector sentinel = sentinel_values(allocation_count); std::vector x; @@ -2844,10 +2881,15 @@ bool run_prefill_wmma_smoke(const aligned_model &model) { tensor_owner tile8_gpu(allocation_count * sizeof(float)); tensor_owner wmma_gpu(allocation_count * sizeof(float)); tensor_owner k64_gpu(allocation_count * sizeof(float)); + tensor_owner k128_gpu(allocation_count * sizeof(float)); + tensor_owner k128_rollback_gpu(allocation_count * sizeof(float)); if (!x_gpu.ptr || !tile8_gpu.ptr || !wmma_gpu.ptr || !k64_gpu.ptr || + !k128_gpu.ptr || !k128_rollback_gpu.ptr || !write_tensor(x_gpu.ptr, x) || !write_tensor(tile8_gpu.ptr, sentinel) || !write_tensor(wmma_gpu.ptr, sentinel) || - !write_tensor(k64_gpu.ptr, sentinel)) { + !write_tensor(k64_gpu.ptr, sentinel) || + !write_tensor(k128_gpu.ptr, sentinel) || + !write_tensor(k128_rollback_gpu.ptr, sentinel)) { std::fprintf(stderr, "ROCm Q4 direct-WMMA prefill: setup FAIL\n"); return false; } @@ -2862,6 +2904,7 @@ bool run_prefill_wmma_smoke(const aligned_model &model) { env_snapshot wmma_require(kPrefillWmmaRequire); env_snapshot wmma_row_tile(kPrefillWmmaRowTile); env_snapshot wmma_k64(kPrefillWmmaK64); + env_snapshot wmma_k128_disable(kPrefillWmmaK128Disable); env_snapshot q8_wave32_enable(kPrefillQ8Wave32Enable); env_snapshot q8_wave32_disable(kPrefillQ8Wave32Disable); env_snapshot q8_wave32_require(kPrefillQ8Wave32Require); @@ -2876,6 +2919,7 @@ bool run_prefill_wmma_smoke(const aligned_model &model) { (void)unsetenv(kPrefillWmmaRequire); (void)unsetenv(kPrefillWmmaRowTile); (void)setenv(kPrefillWmmaK64, "0", 1); + (void)unsetenv(kPrefillWmmaK128Disable); (void)unsetenv(kPrefillQ8Wave32Enable); (void)unsetenv(kPrefillQ8Wave32Disable); (void)unsetenv(kPrefillQ8Wave32Require); @@ -2902,53 +2946,113 @@ bool run_prefill_wmma_smoke(const aligned_model &model) { const uint64_t wmma_k64_calls = ds4_rocm_test_q4_prefill_wmma_k64_get_calls(); - /* The production default must select K64 when no override is present. */ + /* K64 remains the automatic fallback for row geometries outside K128's + * 256-row scope. */ (void)unsetenv(kPrefillWmmaK64); ds4_rocm_test_q4_prefill_wmma_reset(); - const int k64_default_rc = ds4_gpu_matmul_quant_tensor( + const int k64_fallback_rc = ds4_gpu_matmul_quant_tensor( k64_gpu.ptr, model.data, model.size, model.weight0_offset, kQ4Type, kK, kM0, x_gpu.ptr, n_tokens); - const uint64_t k64_default_wmma_calls = + const uint64_t k64_fallback_wmma_calls = ds4_rocm_test_q4_prefill_wmma_get_calls(); - const uint64_t k64_default_calls = + const uint64_t k64_fallback_calls = ds4_rocm_test_q4_prefill_wmma_k64_get_calls(); + /* Force the 256-row geometry on this compact M-tail fixture so default-on + * K128 is covered without allocating the full q_b output. The direct q_b + * benchmark exercises the natural shape. */ + (void)setenv(kPrefillWmmaRowTile, "256", 1); + (void)unsetenv(kPrefillWmmaK128Disable); + ds4_rocm_test_q4_prefill_wmma_reset(); + const int k128_rc = ds4_gpu_matmul_quant_tensor( + k128_gpu.ptr, model.data, model.size, model.weight0_offset, kQ4Type, + kK, kM0, x_gpu.ptr, n_tokens); + const uint64_t k128_wmma_calls = + ds4_rocm_test_q4_prefill_wmma_get_calls(); + const uint64_t k128_k64_calls = + ds4_rocm_test_q4_prefill_wmma_k64_get_calls(); + const uint64_t k128_calls = + ds4_rocm_test_q4_prefill_wmma_k128_get_calls(); + + /* A single opt-out must restore K64 on the same otherwise-eligible + * production geometry. */ + (void)setenv(kPrefillWmmaK128Disable, "1", 1); + ds4_rocm_test_q4_prefill_wmma_reset(); + const int k128_rollback_rc = ds4_gpu_matmul_quant_tensor( + k128_rollback_gpu.ptr, model.data, model.size, model.weight0_offset, + kQ4Type, kK, kM0, x_gpu.ptr, n_tokens); + const uint64_t k128_rollback_wmma_calls = + ds4_rocm_test_q4_prefill_wmma_get_calls(); + const uint64_t k128_rollback_k64_calls = + ds4_rocm_test_q4_prefill_wmma_k64_get_calls(); + const uint64_t k128_rollback_k128_calls = + ds4_rocm_test_q4_prefill_wmma_k128_get_calls(); + std::vector tile8(allocation_count); std::vector wmma(allocation_count); - std::vector k64_default(allocation_count); - bool ok = tile8_rc != 0 && wmma_rc != 0 && k64_default_rc != 0 && + std::vector k64_fallback(allocation_count); + std::vector k128(allocation_count); + std::vector k128_rollback(allocation_count); + bool ok = tile8_rc != 0 && wmma_rc != 0 && k64_fallback_rc != 0 && + k128_rc != 0 && k128_rollback_rc != 0 && tile8_wmma_calls == 0u && tile8_k64_calls == 0u && wmma_calls == 1u && wmma_k64_calls == 0u && - k64_default_wmma_calls == 1u && k64_default_calls == 1u && + k64_fallback_wmma_calls == 1u && k64_fallback_calls == 1u && + k128_wmma_calls == 1u && k128_k64_calls == 0u && + k128_calls == 1u && k128_rollback_wmma_calls == 1u && + k128_rollback_k64_calls == 1u && + k128_rollback_k128_calls == 0u && read_tensor(tile8_gpu.ptr, &tile8) && read_tensor(wmma_gpu.ptr, &wmma) && - read_tensor(k64_gpu.ptr, &k64_default); + read_tensor(k64_gpu.ptr, &k64_fallback) && + read_tensor(k128_gpu.ptr, &k128) && + read_tensor(k128_rollback_gpu.ptr, &k128_rollback); if (ok) { ok = output_body_overwritten(tile8, sentinel, logical_count, "direct-WMMA TILE8 output body") && ok; ok = output_body_overwritten(wmma, sentinel, logical_count, "direct-WMMA candidate output body") && ok; ok = output_body_overwritten( - k64_default, sentinel, logical_count, - "direct-WMMA default K64 output body") && ok; + k64_fallback, sentinel, logical_count, + "direct-WMMA K64 fallback output body") && ok; + ok = output_body_overwritten( + k128, sentinel, logical_count, + "direct-WMMA default K128 output body") && ok; + ok = output_body_overwritten( + k128_rollback, sentinel, logical_count, + "direct-WMMA K128 opt-out output body") && ok; ok = output_guard_unchanged(tile8, sentinel, logical_count, "direct-WMMA TILE8 output canary") && ok; ok = output_guard_unchanged(wmma, sentinel, logical_count, "direct-WMMA candidate output canary") && ok; ok = output_guard_unchanged( - k64_default, sentinel, logical_count, - "direct-WMMA default K64 output canary") && ok; + k64_fallback, sentinel, logical_count, + "direct-WMMA K64 fallback output canary") && ok; + ok = output_guard_unchanged( + k128, sentinel, logical_count, + "direct-WMMA default K128 output canary") && ok; + ok = output_guard_unchanged( + k128_rollback, sentinel, logical_count, + "direct-WMMA K128 opt-out output canary") && ok; tile8.resize(logical_count); wmma.resize(logical_count); - k64_default.resize(logical_count); + k64_fallback.resize(logical_count); + k128.resize(logical_count); + k128_rollback.resize(logical_count); ok = close_with_tolerance(wmma, tile8, 2.0f, 3.0e-2f, "direct-WMMA vs TILE8 N/M tail") && ok; - ok = bitwise_equal(k64_default, wmma, - "direct-WMMA default K64 vs K32 N/M tail") && ok; + ok = bitwise_equal(k64_fallback, wmma, + "direct-WMMA K64 fallback vs K32 N/M tail") && ok; + ok = bitwise_equal(k128, k64_fallback, + "direct-WMMA K128 vs K64 N/M tail") && ok; + ok = bitwise_equal(k128_rollback, k128, + "direct-WMMA K128 opt-out vs default") && ok; } /* Return to the neutral K32 setting for the remaining policy cases. */ (void)setenv(kPrefillWmmaK64, "0", 1); + (void)unsetenv(kPrefillWmmaK128Disable); + (void)unsetenv(kPrefillWmmaRowTile); (void)setenv(kPrefillWmmaDisable, "1", 1); (void)unsetenv(kPrefillWmmaRequire); if (!write_tensor(wmma_gpu.ptr, sentinel)) return false; @@ -2991,15 +3095,25 @@ bool run_prefill_wmma_smoke(const aligned_model &model) { "direct-WMMA DISABLE+REQUIRE preserves output") && ok; std::fprintf(stderr, "ROCm Q4 direct-WMMA prefill: tile8=%d/%llu/%llu " - "K32=%d/%llu/%llu K64-default=%d/%llu/%llu " - "opt_out=%d/%llu/%llu rejected=%d/%llu/%llu %s\n", + "K32=%d/%llu/%llu K64-fallback=%d/%llu/%llu " + "K128-default=%d/%llu/%llu/%llu " + "K128-optout=%d/%llu/%llu/%llu " + "opt_out=%d/%llu/%llu " + "rejected=%d/%llu/%llu %s\n", tile8_rc, (unsigned long long)tile8_wmma_calls, (unsigned long long)tile8_k64_calls, wmma_rc, (unsigned long long)wmma_calls, (unsigned long long)wmma_k64_calls, - k64_default_rc, - (unsigned long long)k64_default_wmma_calls, - (unsigned long long)k64_default_calls, + k64_fallback_rc, + (unsigned long long)k64_fallback_wmma_calls, + (unsigned long long)k64_fallback_calls, + k128_rc, (unsigned long long)k128_wmma_calls, + (unsigned long long)k128_k64_calls, + (unsigned long long)k128_calls, + k128_rollback_rc, + (unsigned long long)k128_rollback_wmma_calls, + (unsigned long long)k128_rollback_k64_calls, + (unsigned long long)k128_rollback_k128_calls, opt_out_rc, (unsigned long long)opt_out_wmma_calls, (unsigned long long)opt_out_k64_calls, rejected_rc, (unsigned long long)rejected_wmma_calls, @@ -3526,6 +3640,7 @@ int main(int argc, char **argv) { env_snapshot wmma_require(kPrefillWmmaRequire); env_snapshot wmma_row_tile(kPrefillWmmaRowTile); env_snapshot wmma_k64_global(kPrefillWmmaK64); + env_snapshot wmma_k128_disable_global(kPrefillWmmaK128Disable); env_snapshot q8_wave32_enable(kPrefillQ8Wave32Enable); env_snapshot q8_wave32_disable(kPrefillQ8Wave32Disable); env_snapshot q8_wave32_require(kPrefillQ8Wave32Require); @@ -3544,8 +3659,9 @@ int main(int argc, char **argv) { (void)unsetenv(kPrefillWmmaDisable); (void)unsetenv(kPrefillWmmaRequire); (void)unsetenv(kPrefillWmmaRowTile); - /* Neutralize the new K64 default for every non-K64-specific oracle. */ + /* Neutralize wider K64/K128 staging for every non-staging oracle. */ (void)setenv(kPrefillWmmaK64, "0", 1); + (void)unsetenv(kPrefillWmmaK128Disable); (void)unsetenv(kPrefillQ8Wave32Enable); (void)unsetenv(kPrefillQ8Wave32Disable); (void)unsetenv(kPrefillQ8Wave32Require); From 35b87dda22421dbf20271444ece635412117a09c Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Wed, 2 Sep 2026 12:35:48 +0200 Subject: [PATCH 186/189] Enable grouped CUDA Q4 prefill on GB10 --- ENVIRONMENT_VARIABLES.md | 8 ++-- QA_BEFORE_RELEASES.md | 9 ++-- README.md | 15 +++--- ds4_cuda.cu | 12 +++-- scripts/environment_variables.tsv | 4 +- speed-bench/README.md | 6 ++- speed-bench/cuda_q4_prefill_bench.cu | 70 ++++++++++++++++++++++------ tests/cuda_q4_gb10_fast_matrix.sh | 25 +++++++++- 8 files changed, 111 insertions(+), 38 deletions(-) diff --git a/ENVIRONMENT_VARIABLES.md b/ENVIRONMENT_VARIABLES.md index 3e54a4e72..8a5cb7aa8 100644 --- a/ENVIRONMENT_VARIABLES.md +++ b/ENVIRONMENT_VARIABLES.md @@ -68,8 +68,8 @@ The detailed Metal A/B contracts and expected oracle counters live in | `DS4_CUDA_NO_Q4_GB10_FAST=1` | Umbrella rollback for the GB10-specific Q4 choices; it does not disable the older cross-CUDA dense pair. | | `DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_BATCH=1` | Enable grouped attention-A for two-to-eight-token GB10 verifier batches. | | `DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_BATCH=1` | Fail closed if that grouped batch path is unavailable. | -| `DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_PREFILL=1` | Enable the GB10 Q4 attention-A prefill candidate for more than eight tokens. It quantizes the token-major grouped input directly and removes the eight pack/unpack copies while preserving each group’s MMQ reduction tree. | -| `DS4_CUDA_NO_Q4_GROUPED_ATTN_A_PREFILL=1` | Dominant rollback from the grouped Q4 attention-A prefill candidate to eight pack/MMQ/unpack projections. Any defined value disables the candidate. | +| `DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_PREFILL=0` | Compatibility opt-out for the default GB10 Q4 attention-A grouped prefill path above eight tokens; any other nonempty value explicitly requests it. | +| `DS4_CUDA_NO_Q4_GROUPED_ATTN_A_PREFILL=1` | Dominant rollback from the default grouped Q4 attention-A prefill path to eight pack/MMQ/unpack projections. Any defined value disables the path. | | `DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_PREFILL=1` | Request the grouped Q4 attention-A prefill candidate and fail before enqueue when its GB10, shape, residency, or buffer contract is unavailable. | | `DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_SINGLE_GRID=1` | On eligible GB10 prefills, submit the eight grouped attention-A MMQs as one grid.z launch while retaining a separate stream-K coordinate and fixup slice per group. | | `DS4_CUDA_DISABLE_Q4_GROUPED_ATTN_A_SINGLE_GRID=1` | Dominant rollback from the single-grid experiment to the established one-MMQ-grid-per-group path. | @@ -665,7 +665,7 @@ and **19 tool/wrapper entries**. | `DS4_CUDA_ENABLE_IQ2_XXS_SSD_PREFILL_MMQ` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Enable the CUDA IQ2 XXS SSD prefill MMQ experimental path. | [ds4_cuda.cu:4625](ds4_cuda.cu#L4625) | | `DS4_CUDA_ENABLE_Q4_ATTN_OUT_HC_FUSE` | value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on | Opt in to the fused Q4 attention-output/HC expansion path. | [ds4_cuda.cu:37375](ds4_cuda.cu#L37375) | | `DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_BATCH` | value-aware opt-in, default off; nonempty value other than exact 0 enables; rollback wins | Enable flattened grouped attention-A MMQ for two-to-eight-token GB10 batches. | [cuda/mmq/ds4_mmq.cu:4303](cuda/mmq/ds4_mmq.cu#L4303) | -| `DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_PREFILL` | value-aware opt-in, default off; unset/empty/exact 0 is off, any other nonempty value requests the path; REQUIRE also requests it; local/global rollback wins | Enable direct-strided grouped Q4_K attention-A MMQ for GB10 prefill widths above eight tokens, removing per-group pack/unpack copies while preserving the per-group reduction tree. | [ds4_cuda.cu:41928](ds4_cuda.cu#L41928) | +| `DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_PREFILL` | value-aware compatibility switch, default on; unset/empty uses the default, exact 0 opts out, any other nonempty value requests the path; REQUIRE also requests it; local/global rollback wins | Control direct-strided grouped Q4_K attention-A MMQ for GB10 prefill widths above eight tokens, removing per-group pack/unpack copies while preserving the per-group reduction tree. | [ds4_cuda.cu:41928](ds4_cuda.cu#L41928) | | `DS4_CUDA_ENABLE_Q4_K1024_PERSISTENT` | presence flag, default off; any defined value including 0 requests the path; rollback wins | Enable the GB10 persistent-CTA kernel for M=32768, N=1, K=1024 Q4. | [cuda/mmq/ds4_mmq.cu:3905](cuda/mmq/ds4_mmq.cu#L3905) | | `DS4_CUDA_ENABLE_Q8_FOLD` | strict flag, default off; only exact value 1 enables; overridden by DS4_CUDA_NO_Q8_FOLD | Enable one-shot producer-to-consumer reuse of freshly quantized Q8_1 data. | [ds4_cuda.cu:785](ds4_cuda.cu#L785) | | `DS4_CUDA_ENABLE_STREAMING_EXPERT_PERSISTENT_CACHE` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Enable streaming expert persistent cache in CUDA SSD streaming. | [ds4_cuda.cu:4111](ds4_cuda.cu#L4111) | @@ -813,7 +813,7 @@ and **19 tool/wrapper entries**. | `DS4_CUDA_NO_Q4_GB10_FAST` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the GB10-specific Q4 fast-path family. | [cuda/mmq/ds4_mmq.cu:3908](cuda/mmq/ds4_mmq.cu#L3908) | | `DS4_CUDA_NO_Q4_GROUPED_ATTN_A` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q4 grouped attn a CUDA Q4 optimization. | [cuda/mmq/ds4_mmq.cu:4295](cuda/mmq/ds4_mmq.cu#L4295) | | `DS4_CUDA_NO_Q4_GROUPED_ATTN_A_BATCH` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q4 grouped attn a batch CUDA Q4 optimization. | [cuda/mmq/ds4_mmq.cu:4305](cuda/mmq/ds4_mmq.cu#L4305) | -| `DS4_CUDA_NO_Q4_GROUPED_ATTN_A_PREFILL` | presence kill switch; default unset; any defined value including empty or 0 disables and dominates ENABLE/REQUIRE | Restore the eight pack/MMQ/unpack Q4 attention-A prefill projections. | [ds4_cuda.cu:41930](ds4_cuda.cu#L41930) | +| `DS4_CUDA_NO_Q4_GROUPED_ATTN_A_PREFILL` | presence kill switch for the default-on GB10 path; default unset; any defined value including empty or 0 disables and dominates ENABLE/REQUIRE | Restore the eight pack/MMQ/unpack Q4 attention-A prefill projections. | [ds4_cuda.cu:41930](ds4_cuda.cu#L41930) | | `DS4_CUDA_NO_Q4_K1024_PERSISTENT` | presence kill switch, default off; any defined value including 0 disables | Disable the Q4 K1024 persistent CUDA Q4 optimization. | [cuda/mmq/ds4_mmq.cu:3907](cuda/mmq/ds4_mmq.cu#L3907) | | `DS4_CUDA_NO_Q4_MMQ_16WARP` | value-aware rollback, default off; unset/empty/exact 0 permits the experiment, every other nonempty value disables it and overrides REQUEST/REQUIRE | Disable the experimental Stream-K-compatible CUDA Q4_K m128n128 16-warp prefill kernel. | [cuda/mmq/ds4_mmq.cu:1133](cuda/mmq/ds4_mmq.cu#L1133) | | `DS4_CUDA_NO_Q8_ALIGNED_DENSE_SCRATCH` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q8 aligned dense scratch CUDA Q8 optimization. | [cuda/mmq/ds4_mmq.cu:5578](cuda/mmq/ds4_mmq.cu#L5578) | diff --git a/QA_BEFORE_RELEASES.md b/QA_BEFORE_RELEASES.md index 98fc130f7..54857b460 100644 --- a/QA_BEFORE_RELEASES.md +++ b/QA_BEFORE_RELEASES.md @@ -1006,12 +1006,11 @@ Do not use high-performance Hugging Face Xet mode while vLLM is resident. byte-identical stdout. Build the resident prefill harness and run `./speed-bench/cuda_q4_prefill_bench --path mmq --case outa --tokens 127,128,129,257,512,2048,4096 --samples 16 --warmup 4`; require bitwise - equality between `pack8_mmq_unpack` and `grouped_dense`, finite/canary/CPU + equality between `pack8_mmq_unpack` and `grouped_8_grids`, finite/canary/CPU oracle success, and record the paired median. Then compare full-model - prefills with - `DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_PREFILL=1` plus - `DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_PREFILL=1` against the dominant - `DS4_CUDA_NO_Q4_GROUPED_ATTN_A_PREFILL=1` rollback. Benchmark the K1024 + prefills with the default environment against the dominant + `DS4_CUDA_NO_Q4_GROUPED_ATTN_A_PREFILL=1` rollback. Keep the single-grid and + 16-warp experiments unset in this promotion comparison. Benchmark the K1024 persistent kernel as a separate fail-closed arm with both `DS4_CUDA_ENABLE_Q4_K1024_PERSISTENT=1` and diff --git a/README.md b/README.md index a3ab3b8b6..fbfaf0fc6 100644 --- a/README.md +++ b/README.md @@ -788,13 +788,14 @@ arithmetic: dispatch while keeping `ncols_dst=1`; `DS4_CUDA_NO_Q4_GROUPED_ATTN_A_BATCH=1` restores the per-token grouped loop and `DS4_CUDA_NO_Q4_GROUPED_ATTN_A=1` restores the per-group loop; -- for prefill widths above eight, the experimental - `DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_PREFILL=1` path quantizes the strided - `[token][group][K]` input in one launch and writes each group directly into - `[token][group][rank]`. It removes the eight F32 pack/unpack copies while - keeping one established stream-K MMQ reduction per group. Add - `DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_PREFILL=1` for fail-closed tests; - `DS4_CUDA_NO_Q4_GROUPED_ATTN_A_PREFILL=1` is the dominant local rollback; +- for prefill widths above eight, the default eligible GB10 path quantizes the + strided `[token][group][K]` input in one launch and writes each group directly + into `[token][group][rank]`. It removes the eight F32 pack/unpack copies while + keeping one established stream-K MMQ reduction per group. Set + `DS4_CUDA_NO_Q4_GROUPED_ATTN_A_PREFILL=1` for the dominant local rollback; + exact `DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_PREFILL=0` is also a compatibility + opt-out. Add `DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_PREFILL=1` for fail-closed + tests. The separate single-grid/grid.z submission remains opt-in; - attention-output B keeps its canonical MMVQ result and the ordinary HC epilogue inside the graph-compatible fused call. The row-packed epilogue remains oracle-only until a GB10 device run proves it bit-exact; diff --git a/ds4_cuda.cu b/ds4_cuda.cu index 54bd9a353..89e037e09 100644 --- a/ds4_cuda.cu +++ b/ds4_cuda.cu @@ -44124,8 +44124,12 @@ extern "C" int ds4_gpu_attention_output_q4_K_batch_tensor( "DS4_CUDA_Q4_GROUPED_ATTN_A_ORACLE", 0); const int grouped_batch_enable = cuda_env_flag_enabled( "DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_BATCH", 0); + /* The direct-strided eight-grid path preserves the canonical MMQ + * reduction tree and is the GB10 prefill default. Exact ENABLE=0 is a + * compatibility opt-out; the presence-based NO switch remains the + * authoritative rollback. The separate grid.z candidate stays opt-in. */ const int grouped_prefill_enable = cuda_env_flag_enabled( - "DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_PREFILL", 0); + "DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_PREFILL", 1); const int grouped_prefill_disable = getenv("DS4_CUDA_NO_Q4_GROUPED_ATTN_A_PREFILL") != NULL; const int grouped_single_grid_enable = cuda_env_flag_enabled( @@ -44183,9 +44187,9 @@ extern "C" int ds4_gpu_attention_output_q4_K_batch_tensor( (int)rank, (int)n_tokens, (int)group_dim, (int)n_groups, cuda_decode_stream()); /* NOT_APPLICABLE is returned before allocation or enqueue, so the - * opt-in candidate can safely fall back to the established grouped - * implementation. Every other failure may follow an enqueue and - * remains fail-closed. */ + * opt-in single-grid candidate can safely fall back to the default + * eight-grid grouped implementation. Every other failure may follow + * an enqueue and remains fail-closed. */ if (rc == DS4_MMQ_NOT_APPLICABLE && grouped_single_grid_selected) { if (grouped_single_grid_require) { fprintf(stderr, diff --git a/scripts/environment_variables.tsv b/scripts/environment_variables.tsv index 2bdf900fd..75ab33069 100644 --- a/scripts/environment_variables.tsv +++ b/scripts/environment_variables.tsv @@ -86,7 +86,7 @@ runtime/cuda DS4_CUDA_ENABLE_Q4_ATTN_OUT_HC_FUSE value-aware flag, default off; runtime/cuda DS4_CUDA_ENABLE_Q4_ATTN_Q_B_F16_CACHE value-aware persistent-cache opt-in, default off; unset/empty/0/false/no/off is off; DISABLE cancels an optional persistent request but leaves the automatic transient path independent; REQUIRE plus DISABLE fails closed Prewarm and use persistent resident F16 sidecars for eligible single-GPU Q4_K attn_q_b prefills. ds4_cuda.cu:2490 runtime/cuda DS4_CUDA_ENABLE_Q4_ATTN_Q_B_F16_OUTPUT value-aware experimental opt-in; unset/empty/0/false/no/off keeps the release F32 projection boundary; other nonempty values enable Write eligible resident Q4_K attn_q_b GEMM output in F16 and run the half-input norm/RoPE epilogue; SSD remains excluded. ds4_cuda.cu:2523 runtime/cuda DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_BATCH value-aware opt-in, default off; nonempty value other than exact 0 enables; rollback wins Enable flattened grouped attention-A MMQ for two-to-eight-token GB10 batches. cuda/mmq/ds4_mmq.cu:4303 -runtime/cuda DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_PREFILL value-aware opt-in, default off; unset/empty/exact 0 is off, any other nonempty value requests the path; REQUIRE also requests it; local/global rollback wins Enable direct-strided grouped Q4_K attention-A MMQ for GB10 prefill widths above eight tokens, removing per-group pack/unpack copies while preserving the per-group reduction tree. ds4_cuda.cu:41928 +runtime/cuda DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_PREFILL value-aware compatibility switch, default on; unset/empty uses the default, exact 0 opts out, any other nonempty value requests the path; REQUIRE also requests it; local/global rollback wins Control direct-strided grouped Q4_K attention-A MMQ for GB10 prefill widths above eight tokens, removing per-group pack/unpack copies while preserving the per-group reduction tree. ds4_cuda.cu:41928 runtime/cuda DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_SINGLE_GRID value-aware opt-in, default off; unset/empty/exact 0 is off, every other nonempty value requests the candidate; REQUIRE also requests it; DISABLE wins Submit all eligible GB10 grouped Q4_K attention-A prefill projections in one grid.z launch while isolating each group's stream-K coordinates and fixup storage. ds4_cuda.cu:41941 runtime/cuda DS4_CUDA_ENABLE_Q4_K1024_PERSISTENT presence flag, default off; any defined value including 0 requests the path; rollback wins Enable the GB10 persistent-CTA kernel for M=32768, N=1, K=1024 Q4. cuda/mmq/ds4_mmq.cu:3905 runtime/cuda DS4_CUDA_ENABLE_Q8_FOLD strict flag, default off; only exact value 1 enables; overridden by DS4_CUDA_NO_Q8_FOLD Enable one-shot producer-to-consumer reuse of freshly quantized Q8_1 data. ds4_cuda.cu:785 @@ -235,7 +235,7 @@ runtime/cuda DS4_CUDA_NO_Q4_DENSE_SCRATCH presence kill switch; default unset (e runtime/cuda DS4_CUDA_NO_Q4_GB10_FAST presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the GB10-specific Q4 fast-path family. cuda/mmq/ds4_mmq.cu:3908 runtime/cuda DS4_CUDA_NO_Q4_GROUPED_ATTN_A presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q4 grouped attn a CUDA Q4 optimization. cuda/mmq/ds4_mmq.cu:4295 runtime/cuda DS4_CUDA_NO_Q4_GROUPED_ATTN_A_BATCH presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q4 grouped attn a batch CUDA Q4 optimization. cuda/mmq/ds4_mmq.cu:4305 -runtime/cuda DS4_CUDA_NO_Q4_GROUPED_ATTN_A_PREFILL presence kill switch; default unset; any defined value including empty or 0 disables and dominates ENABLE/REQUIRE Restore the eight pack/MMQ/unpack Q4 attention-A prefill projections. ds4_cuda.cu:41930 +runtime/cuda DS4_CUDA_NO_Q4_GROUPED_ATTN_A_PREFILL presence kill switch for the default-on GB10 path; default unset; any defined value including empty or 0 disables and dominates ENABLE/REQUIRE Restore the eight pack/MMQ/unpack Q4 attention-A prefill projections. ds4_cuda.cu:41930 runtime/cuda DS4_CUDA_NO_Q4_K1024_PERSISTENT presence kill switch, default off; any defined value including 0 disables Disable the Q4 K1024 persistent CUDA Q4 optimization. cuda/mmq/ds4_mmq.cu:3907 runtime/cuda DS4_CUDA_NO_Q4_MMQ_16WARP value-aware rollback, default off; unset/empty/exact 0 permits the experiment, every other nonempty value disables it and overrides REQUEST/REQUIRE Disable the experimental Stream-K-compatible CUDA Q4_K m128n128 16-warp prefill kernel. cuda/mmq/ds4_mmq.cu:1133 runtime/cuda DS4_CUDA_NO_Q8_ALIGNED_DENSE_SCRATCH presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q8 aligned dense scratch CUDA Q8 optimization. cuda/mmq/ds4_mmq.cu:5578 diff --git a/speed-bench/README.md b/speed-bench/README.md index 3217c7fe1..f3235b16d 100644 --- a/speed-bench/README.md +++ b/speed-bench/README.md @@ -413,7 +413,11 @@ On GB10, isolate the production Flash attention-output A geometry ``` This is an in-process ABBA/BAAB comparison between the current eight-group -pack/MMQ/unpack sequence and the strictly required grouped-prefill dispatch. +pack/MMQ/unpack rollback and the default, strictly required direct-strided +grouped-prefill dispatch with one canonical MMQ grid per group. Add +`--grouped-single-grid` to instead compare the grouped eight-grid path with the +experimental grid.z submission; do not mix that experiment into the default +promotion measurement. The public API must also execute output-B, so the fixture uses a valid Q4_K `K=8192,M=256` output-B common to both arms. It is 6.25% of output-A's MACs; the result prints `focus_macs_per_token` and `common_macs_per_token` separately diff --git a/speed-bench/cuda_q4_prefill_bench.cu b/speed-bench/cuda_q4_prefill_bench.cu index 254f8c554..f9704055c 100644 --- a/speed-bench/cuda_q4_prefill_bench.cu +++ b/speed-bench/cuda_q4_prefill_bench.cu @@ -89,6 +89,7 @@ struct config { uint32_t samples = kDefaultSamples; uint32_t warmup = kDefaultWarmup; bool kernel_16warp = false; + bool grouped_single_grid = false; }; struct weight_set { @@ -699,7 +700,16 @@ bool sampled_grouped_cpu_oracle( return true; } -bool select_grouped_prefill_baseline() { +bool select_grouped_prefill_legacy() { + return unsetenv(kGroupedPrefillEnable) == 0 && + setenv(kGroupedPrefillDisable, "1", 1) == 0 && + unsetenv(kGroupedPrefillRequire) == 0 && + unsetenv(kGroupedSingleGridEnable) == 0 && + setenv(kGroupedSingleGridDisable, "1", 1) == 0 && + unsetenv(kGroupedSingleGridRequire) == 0; +} + +bool select_grouped_prefill_grid8() { return setenv(kGroupedPrefillEnable, "1", 1) == 0 && unsetenv(kGroupedPrefillDisable) == 0 && setenv(kGroupedPrefillRequire, "1", 1) == 0 && @@ -708,7 +718,7 @@ bool select_grouped_prefill_baseline() { unsetenv(kGroupedSingleGridRequire) == 0; } -bool select_grouped_prefill_candidate() { +bool select_grouped_prefill_single_grid() { return setenv(kGroupedPrefillEnable, "1", 1) == 0 && unsetenv(kGroupedPrefillDisable) == 0 && setenv(kGroupedPrefillRequire, "1", 1) == 0 && @@ -1607,8 +1617,9 @@ bool run_output_a(const model_fixture &model, const config &cfg, return false; } + const bool compare_single_grid = cfg.grouped_single_grid; const arm baseline = { - "grouped_8_grids", + compare_single_grid ? "grouped_8_grids" : "pack8_mmq_unpack", [&](uint32_t set) { return ds4_gpu_attention_output_q4_K_batch_tensor( baseline_out.ptr, baseline_low.ptr, group_tmp.ptr, @@ -1618,9 +1629,12 @@ bool run_output_a(const model_fixture &model, const config &cfg, kDenseK, kOutputRank, kOutputGroups, kOutputMinB, heads.ptr, n_tokens) > 0; }, - select_grouped_prefill_baseline}; + [=]() { + return compare_single_grid ? select_grouped_prefill_grid8() + : select_grouped_prefill_legacy(); + }}; const arm candidate = { - "grouped_single_grid", + compare_single_grid ? "grouped_single_grid" : "grouped_8_grids", [&](uint32_t set) { return ds4_gpu_attention_output_q4_K_batch_tensor( candidate_out.ptr, candidate_low.ptr, group_tmp.ptr, @@ -1630,7 +1644,11 @@ bool run_output_a(const model_fixture &model, const config &cfg, kDenseK, kOutputRank, kOutputGroups, kOutputMinB, heads.ptr, n_tokens) > 0; }, - select_grouped_prefill_candidate}; + [=]() { + return compare_single_grid + ? select_grouped_prefill_single_grid() + : select_grouped_prefill_grid8(); + }}; const uint64_t output_a_macs_per_token = static_cast(kOutputGroups) * kDenseK * kOutputRank; const uint64_t output_b_macs_per_token = @@ -1654,7 +1672,9 @@ bool run_output_a(const model_fixture &model, const config &cfg, [&](uint32_t set) { return bitwise_equal(baseline_low.ptr, candidate_low.ptr, low_bytes, - "output_a single-grid vs eight-grid") && + compare_single_grid + ? "output_a single-grid vs eight-grid" + : "output_a grouped vs pack/unpack") && bitwise_equal(baseline_out.ptr, candidate_out.ptr, out_bytes, "output_a minimal-B final output") && @@ -1712,16 +1732,18 @@ void usage(FILE *stream, const char *argv0) { " --sets N rotating resident weight sets (default: %u)\n" " --samples N samples/arm, multiple of 4 (default: %u)\n" " --warmup N untimed dispatches/arm (default: %u)\n" + " --grouped-single-grid compare grouped 8-grid vs grid.z outa\n" " --kernel-16warp prequantized canonical-vs-16-warp A/B\n" " -h, --help show this help\n\n" "Dense and q_b measure one immutable process path. Run separate " "legacy/MMQ\nprocesses (preferably ABBA/BAAB) to compare them because " "the CUDA backend\ncaches DS4_CUDA_MMQ on its first dispatch. Pair is " "an in-process ABBA/BAAB\ncomparison of two MMQ projections against " - "the fused public pair API. outa\ncompares the existing grouped path " - "with one MMQ grid per group against the strict\nsingle-grid candidate " - "at the production attention-A shape, isolating grid.z submission. It " - "includes\na common minimal " + "the fused public pair API. outa\ncompares the rollback eight-group " + "pack/MMQ/unpack sequence against the default\ndirect-strided grouped " + "path with one canonical MMQ grid per group. Add\n" + "--grouped-single-grid to compare that default with the experimental " + "grid.z\nsubmission instead. It includes a common minimal " "Q4 output-B (M=256) whose MACs are reported separately. " "DS4_CUDA_MMQ_X_MAX\nmay explicitly select an 8..128 multiple-of-8 " "sweep point; the setup line\nattests it, or prints auto when the " @@ -1838,6 +1860,8 @@ config parse_options(int argc, char **argv) { 0u, 100u); } else if (!std::strcmp(argv[i], "--kernel-16warp")) { cfg.kernel_16warp = true; + } else if (!std::strcmp(argv[i], "--grouped-single-grid")) { + cfg.grouped_single_grid = true; } else { std::fprintf(stderr, "unknown option: %s\n", argv[i]); usage(stderr, argv[0]); @@ -1869,6 +1893,23 @@ config parse_options(int argc, char **argv) { "--kernel-16warp requires --path mmq\n"); std::exit(2); } + if (cfg.grouped_single_grid && cfg.path != cuda_path::mmq) { + std::fprintf(stderr, + "--grouped-single-grid requires --path mmq\n"); + std::exit(2); + } + if (cfg.grouped_single_grid && cfg.kernel_16warp) { + std::fprintf(stderr, + "--grouped-single-grid cannot be combined with " + "--kernel-16warp\n"); + std::exit(2); + } + if (cfg.grouped_single_grid && cfg.selected != bench_case::all && + cfg.selected != bench_case::outa) { + std::fprintf(stderr, + "--grouped-single-grid requires --case outa or all\n"); + std::exit(2); + } if (cfg.kernel_16warp && cfg.selected == bench_case::outa) { std::fprintf(stderr, "--kernel-16warp supports only --case dense, pair, qb, " @@ -2162,7 +2203,8 @@ int main(int argc, char **argv) { "cases=%s " "ssd_streaming=off model_storage=cudaMalloc " "residency=backend_provenance strict_mmq=%d " - "grouped_attn_a_prefill=%s dispatch_stream=legacy_default\n", + "grouped_attn_a_prefill=%s grouped_attn_a_ab=%s " + "dispatch_stream=legacy_default\n", properties.name, properties.major, properties.minor, properties.warpSize, path_name(cfg.path), mmq_x_max.c_str(), cfg.sets, @@ -2173,7 +2215,9 @@ int main(int argc, char **argv) { cfg.kernel_16warp ? 1 : 0, case_scope(cfg), cfg.path == cuda_path::mmq ? 1 : 0, - grouped_prefill_supported ? "available" : "skipped"); + grouped_prefill_supported ? "available" : "skipped", + cfg.grouped_single_grid ? "grid8_vs_single_grid" + : "pack8_vs_grid8"); std::fflush(stdout); for (uint32_t n_tokens : cfg.tokens) { if (includes(cfg.selected, bench_case::dense)) { diff --git a/tests/cuda_q4_gb10_fast_matrix.sh b/tests/cuda_q4_gb10_fast_matrix.sh index 86404a227..4f8046245 100755 --- a/tests/cuda_q4_gb10_fast_matrix.sh +++ b/tests/cuda_q4_gb10_fast_matrix.sh @@ -135,6 +135,12 @@ clean_env() { -u DS4_CUDA_NO_Q4_GROUPED_ATTN_A_BATCH \ -u DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_BATCH \ -u DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_BATCH \ + -u DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_PREFILL \ + -u DS4_CUDA_NO_Q4_GROUPED_ATTN_A_PREFILL \ + -u DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_PREFILL \ + -u DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_SINGLE_GRID \ + -u DS4_CUDA_DISABLE_Q4_GROUPED_ATTN_A_SINGLE_GRID \ + -u DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_SINGLE_GRID \ -u DS4_CUDA_Q4_GROUPED_ATTN_A_ORACLE \ -u DS4_CUDA_DISABLE_Q4_ATTN_OUT_HC_FUSE \ -u DS4_CUDA_ENABLE_Q4_ATTN_OUT_HC_FUSE \ @@ -143,6 +149,9 @@ clean_env() { -u DS4_CUDA_NO_Q4_K1024_PERSISTENT \ -u DS4_CUDA_ENABLE_Q4_K1024_PERSISTENT \ -u DS4_CUDA_REQUIRE_Q4_K1024_PERSISTENT \ + -u DS4_CUDA_Q4_MMQ_16WARP \ + -u DS4_CUDA_NO_Q4_MMQ_16WARP \ + -u DS4_CUDA_REQUIRE_Q4_MMQ_16WARP \ -u DS4_CUDA_DECODE_GRAPHS \ -u DS4_CUDA_DECODE_GRAPH_LOG \ "$@" @@ -313,6 +322,7 @@ check_hc_oracle() { LOCAL_ROLLBACK="DS4_CUDA_NO_Q4_DENSE_SCRATCH=1 DS4_CUDA_NO_Q4_GROUPED_ATTN_A=1 DS4_CUDA_NO_Q4_GROUPED_ATTN_A_BATCH=1 +DS4_CUDA_NO_Q4_GROUPED_ATTN_A_PREFILL=1 DS4_CUDA_DISABLE_Q4_ATTN_OUT_HC_FUSE=1 DS4_CUDA_NO_Q4_K1024_PERSISTENT=1" @@ -335,6 +345,7 @@ run_smoke local_control "$@" run_smoke scratch_only \ DS4_CUDA_NO_Q4_GROUPED_ATTN_A=1 \ DS4_CUDA_NO_Q4_GROUPED_ATTN_A_BATCH=1 \ + DS4_CUDA_NO_Q4_GROUPED_ATTN_A_PREFILL=1 \ DS4_CUDA_DISABLE_Q4_ATTN_OUT_HC_FUSE=1 \ DS4_CUDA_NO_Q4_K1024_PERSISTENT=1 run_smoke grouped_only \ @@ -346,8 +357,12 @@ run_smoke hc_only \ DS4_CUDA_NO_Q4_DENSE_SCRATCH=1 \ DS4_CUDA_NO_Q4_GROUPED_ATTN_A=1 \ DS4_CUDA_NO_Q4_GROUPED_ATTN_A_BATCH=1 \ + DS4_CUDA_NO_Q4_GROUPED_ATTN_A_PREFILL=1 \ DS4_CUDA_NO_Q4_K1024_PERSISTENT=1 run_smoke default_fast DS4_CUDA_NO_Q4_K1024_PERSISTENT=1 +run_smoke grouped_prefill_rollback \ + DS4_CUDA_NO_Q4_GROUPED_ATTN_A_PREFILL=1 \ + DS4_CUDA_NO_Q4_K1024_PERSISTENT=1 status=0 oracle_mismatch_arms= @@ -385,7 +400,7 @@ compare_smoke() { } for arm in local_control scratch_only grouped_only hc_only default_fast \ - grouped_oracle hc_oracle; do + grouped_prefill_rollback grouped_oracle hc_oracle; do compare_smoke "$arm" done @@ -396,6 +411,7 @@ run_score local_control "$@" run_score scratch_only \ DS4_CUDA_NO_Q4_GROUPED_ATTN_A=1 \ DS4_CUDA_NO_Q4_GROUPED_ATTN_A_BATCH=1 \ + DS4_CUDA_NO_Q4_GROUPED_ATTN_A_PREFILL=1 \ DS4_CUDA_DISABLE_Q4_ATTN_OUT_HC_FUSE=1 \ DS4_CUDA_NO_Q4_K1024_PERSISTENT=1 run_score grouped_only \ @@ -407,10 +423,15 @@ run_score hc_only \ DS4_CUDA_NO_Q4_DENSE_SCRATCH=1 \ DS4_CUDA_NO_Q4_GROUPED_ATTN_A=1 \ DS4_CUDA_NO_Q4_GROUPED_ATTN_A_BATCH=1 \ + DS4_CUDA_NO_Q4_GROUPED_ATTN_A_PREFILL=1 \ DS4_CUDA_NO_Q4_K1024_PERSISTENT=1 run_score default_fast DS4_CUDA_NO_Q4_K1024_PERSISTENT=1 +run_score grouped_prefill_rollback \ + DS4_CUDA_NO_Q4_GROUPED_ATTN_A_PREFILL=1 \ + DS4_CUDA_NO_Q4_K1024_PERSISTENT=1 -for arm in local_control scratch_only grouped_only hc_only default_fast; do +for arm in local_control scratch_only grouped_only hc_only default_fast \ + grouped_prefill_rollback; do if cmp -s "$OUT_DIR/umbrella_control.tsv" "$OUT_DIR/$arm.tsv"; then echo "q4-gb10-matrix: quality $arm: EXACT" else From 6a20b131e18f6a1650706d8f76e15f60ac2bc91f Mon Sep 17 00:00:00 2001 From: GiorgioOppo <111227665+GiorgioOppo@users.noreply.github.com> Date: Wed, 2 Sep 2026 15:18:22 +0200 Subject: [PATCH 187/189] Add specialized CUDA grouped Q4 quantizer --- ENVIRONMENT_VARIABLES.md | 8 +- Makefile | 6 +- QA_BEFORE_RELEASES.md | 14 ++- README.md | 13 ++- cuda/mmq/ds4_mmq.cu | 87 ++++++++++++-- cuda/mmq/ds4_mmq.h | 14 +++ cuda/mmq/quantize.cu | 87 ++++++++++++++ cuda/mmq/quantize.cuh | 7 ++ cuda/mmq/test/test_mmq_parity.cu | 162 +++++++++++++++++++++++++++ ds4_cuda.cu | 19 +++- scripts/environment_variables.tsv | 2 + speed-bench/README.md | 11 ++ speed-bench/cuda_q4_prefill_bench.cu | 104 ++++++++++++++--- tests/cuda_q4_gb10_fast_matrix.sh | 14 ++- 14 files changed, 512 insertions(+), 36 deletions(-) diff --git a/ENVIRONMENT_VARIABLES.md b/ENVIRONMENT_VARIABLES.md index 8a5cb7aa8..f169704b6 100644 --- a/ENVIRONMENT_VARIABLES.md +++ b/ENVIRONMENT_VARIABLES.md @@ -71,6 +71,8 @@ The detailed Metal A/B contracts and expected oracle counters live in | `DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_PREFILL=0` | Compatibility opt-out for the default GB10 Q4 attention-A grouped prefill path above eight tokens; any other nonempty value explicitly requests it. | | `DS4_CUDA_NO_Q4_GROUPED_ATTN_A_PREFILL=1` | Dominant rollback from the default grouped Q4 attention-A prefill path to eight pack/MMQ/unpack projections. Any defined value disables the path. | | `DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_PREFILL=1` | Request the grouped Q4 attention-A prefill candidate and fail before enqueue when its GB10, shape, residency, or buffer contract is unavailable. | +| `DS4_CUDA_NO_Q4_GROUPED_ATTN_A_Q81=1` | Narrow rollback from the default fixed-layout `K=4096`, `groups=8` Q8_1 producer to the canonical strided producer while retaining grouped prefill. Any defined value disables the specialized kernel. | +| `DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_Q81=1` | Request the fixed-layout eight-warp Q8_1 producer and fail before enqueue on rollback or ineligibility. | | `DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_SINGLE_GRID=1` | On eligible GB10 prefills, submit the eight grouped attention-A MMQs as one grid.z launch while retaining a separate stream-K coordinate and fixup slice per group. | | `DS4_CUDA_DISABLE_Q4_GROUPED_ATTN_A_SINGLE_GRID=1` | Dominant rollback from the single-grid experiment to the established one-MMQ-grid-per-group path. | | `DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_SINGLE_GRID=1` | Require the single-grid grouped attention-A submission and fail closed on rollback, ineligibility, or dispatch failure. | @@ -154,7 +156,7 @@ above, it is an unstable internal diagnostic or tuning interface. The linked sou remains normative for exact eligibility gates, bounds, and architecture-specific defaults. -Inventory totals: **1081 `DS4_*` runtime variables** and +Inventory totals: **1083 `DS4_*` runtime variables** and **6 external runtime variables**. The auxiliary inventories contain **118 test/test-fixture entries** and **19 tool/wrapper entries**. @@ -615,7 +617,7 @@ and **19 tool/wrapper entries**.
-CUDA (334) +CUDA (336) | Variable | Accepted value and default | Effect | Source | | --- | --- | --- | --- | @@ -814,6 +816,7 @@ and **19 tool/wrapper entries**. | `DS4_CUDA_NO_Q4_GROUPED_ATTN_A` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q4 grouped attn a CUDA Q4 optimization. | [cuda/mmq/ds4_mmq.cu:4295](cuda/mmq/ds4_mmq.cu#L4295) | | `DS4_CUDA_NO_Q4_GROUPED_ATTN_A_BATCH` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q4 grouped attn a batch CUDA Q4 optimization. | [cuda/mmq/ds4_mmq.cu:4305](cuda/mmq/ds4_mmq.cu#L4305) | | `DS4_CUDA_NO_Q4_GROUPED_ATTN_A_PREFILL` | presence kill switch for the default-on GB10 path; default unset; any defined value including empty or 0 disables and dominates ENABLE/REQUIRE | Restore the eight pack/MMQ/unpack Q4 attention-A prefill projections. | [ds4_cuda.cu:41930](ds4_cuda.cu#L41930) | +| `DS4_CUDA_NO_Q4_GROUPED_ATTN_A_Q81` | presence rollback for the default-on fixed-shape quantizer; default unset; any defined value including empty or 0 disables; REQUIRE then fails closed | Restore the canonical strided Q8_1 producer while retaining grouped Q4 attention-A prefill and its eight MMQ grids. | [cuda/mmq/ds4_mmq.cu:1762](cuda/mmq/ds4_mmq.cu#L1762); [ds4_cuda.cu:44143](ds4_cuda.cu#L44143) | | `DS4_CUDA_NO_Q4_K1024_PERSISTENT` | presence kill switch, default off; any defined value including 0 disables | Disable the Q4 K1024 persistent CUDA Q4 optimization. | [cuda/mmq/ds4_mmq.cu:3907](cuda/mmq/ds4_mmq.cu#L3907) | | `DS4_CUDA_NO_Q4_MMQ_16WARP` | value-aware rollback, default off; unset/empty/exact 0 permits the experiment, every other nonempty value disables it and overrides REQUEST/REQUIRE | Disable the experimental Stream-K-compatible CUDA Q4_K m128n128 16-warp prefill kernel. | [cuda/mmq/ds4_mmq.cu:1133](cuda/mmq/ds4_mmq.cu#L1133) | | `DS4_CUDA_NO_Q8_ALIGNED_DENSE_SCRATCH` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q8 aligned dense scratch CUDA Q8 optimization. | [cuda/mmq/ds4_mmq.cu:5578](cuda/mmq/ds4_mmq.cu#L5578) | @@ -883,6 +886,7 @@ and **19 tool/wrapper entries**. | `DS4_CUDA_REQUIRE_IQ2_XXS_SSD_PREFILL_MMQ` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Require the CUDA IQ2 XXS SSD prefill MMQ path; fail closed when unavailable. | [ds4_cuda.cu:4631](ds4_cuda.cu#L4631) | | `DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_BATCH` | value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on | Fail if grouped batched attention-A cannot be used. | [ds4_cuda.cu:40198](ds4_cuda.cu#L40198) | | `DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_PREFILL` | value-aware fail-closed assertion, default off; unset/empty/exact 0 is off, any other nonempty value requests the candidate and rejects ineligibility before enqueue | Require the GB10 grouped Q4_K attention-A prefill path instead of silently using pack/MMQ/unpack. | [ds4_cuda.cu:41889](ds4_cuda.cu#L41889) | +| `DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_Q81` | value-aware fail-closed assertion, default off; unset/empty/exact 0 is off, any other nonempty value requests grouped prefill and the fixed K=4096, groups=8, rank=1024 Q8_1 producer; NO wins | Require the eight-warp K4096/G8x2 Q8_1 producer instead of silently using the generic strided quantizer. | [cuda/mmq/ds4_mmq.cu:1767](cuda/mmq/ds4_mmq.cu#L1767); [ds4_cuda.cu:44089](ds4_cuda.cu#L44089) | | `DS4_CUDA_REQUIRE_Q4_K1024_PERSISTENT` | presence flag, default off; any defined value including 0 makes ineligible candidate fail closed | Fail when the exact Q4 K1024 persistent candidate is unavailable instead of using MMVQ. | [cuda/mmq/ds4_mmq.cu:3929](cuda/mmq/ds4_mmq.cu#L3929) | | `DS4_CUDA_REQUIRE_Q4_MMQ_16WARP` | value-aware fail-closed prefill opt-in cached on the first Q4_K dense or dense-pair MMQ call; unset/empty/exact 0 is off, every other nonempty value requests and requires the candidate for N>8; a dense-pair is rejected before allocation unless both legs are eligible; rollback, disabled MMQ, ineligibility, or preflight failure prevents fallback; decode/speculative N<=8 remains on MMVQ | Require the experimental CUDA Q4_K 16-warp prefill kernel so benchmark runs cannot silently measure another path. | [cuda/mmq/ds4_mmq.cu:1130](cuda/mmq/ds4_mmq.cu#L1130); [ds4_cuda.cu:38208](ds4_cuda.cu#L38208); [ds4_cuda.cu:38358](ds4_cuda.cu#L38358) | | `DS4_CUDA_REQUIRE_STREAMING_EXPERT_PERSISTENT_CACHE` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Require streaming expert persistent cache in CUDA SSD streaming; fail closed when unavailable. | [ds4_cuda.cu:4117](ds4_cuda.cu#L4117) | diff --git a/Makefile b/Makefile index 4d03f13e1..d3c4a7ddb 100644 --- a/Makefile +++ b/Makefile @@ -69,7 +69,7 @@ DS4_LINK_LIBS ?= $(CUDA_LDLIBS) METAL_LDLIBS := $(LDLIBS) endif -.PHONY: all help clean test test-ssd environment-docs test-quantizer-indexer-q4 test-rocm test-glm53-kda-rocm test-metal-session-batch test-metal-session-batch-ssd test-metal-q4-streams test-metal-q4-prefill-pair test-metal-indexer-q4 test-metal-q4-attn-exactn test-metal-q4-attn-out-a-direct test-metal-q4-qb-f16-cache test-metal-q4-qb-f16-cache-timing test-metal-exactn-oracle test-metal-dspark-capture test-metal-argmax-top1 bench-metal-argmax-top1 test-metal-iq2-midonly test-metal-iq2-ssd-grouped-mm test-metal-iq2-live-index test-mxfp4-metal test-mxfp4-cuda test-mxfp4-rocm test-mmq-parity-cuda test-mmq-q4-16warp-cuda test-rocm-q4-parity test-rocm-q4-dense test-rocm-q4-pair test-rocm-q4-prefill test-strix-rocm-q4-parity test-strix-rocm-q4-prefill test-strix-rocm-q4-prefill-long test-cuda-session-batch test-cuda-mixed-batch dspark-acceptance dspark-verify-depth rocm-dspark-acceptance rocm-dspark-verify-depth mtp-verify-depth cpu cuda cuda-spark cuda-generic cuda-regression strix-halo rocm cuda-iq2-moe-prefill-bench cuda-q4-prefill-bench rocm-iq2-moe-prefill-bench rocm-q4-prefill-bench +.PHONY: all help clean test test-ssd environment-docs test-quantizer-indexer-q4 test-rocm test-glm53-kda-rocm test-metal-session-batch test-metal-session-batch-ssd test-metal-q4-streams test-metal-q4-prefill-pair test-metal-indexer-q4 test-metal-q4-attn-exactn test-metal-q4-attn-out-a-direct test-metal-q4-qb-f16-cache test-metal-q4-qb-f16-cache-timing test-metal-exactn-oracle test-metal-dspark-capture test-metal-argmax-top1 bench-metal-argmax-top1 test-metal-iq2-midonly test-metal-iq2-ssd-grouped-mm test-metal-iq2-live-index test-mxfp4-metal test-mxfp4-cuda test-mxfp4-rocm test-mmq-parity-cuda test-mmq-q4-grouped-q81-cuda test-mmq-q4-16warp-cuda test-rocm-q4-parity test-rocm-q4-dense test-rocm-q4-pair test-rocm-q4-prefill test-strix-rocm-q4-parity test-strix-rocm-q4-prefill test-strix-rocm-q4-prefill-long test-cuda-session-batch test-cuda-mixed-batch dspark-acceptance dspark-verify-depth rocm-dspark-acceptance rocm-dspark-verify-depth mtp-verify-depth cpu cuda cuda-spark cuda-generic cuda-regression strix-halo rocm cuda-iq2-moe-prefill-bench cuda-q4-prefill-bench rocm-iq2-moe-prefill-bench rocm-q4-prefill-bench gguf-tools/deepseek4-quantize: gguf-tools/deepseek4-quantize.c gguf-tools/quants.c gguf-tools/quants.h $(MAKE) -C gguf-tools deepseek4-quantize @@ -433,6 +433,7 @@ help: @echo " make cuda-generic Build CUDA for a generic local CUDA GPU" @echo " make cuda CUDA_ARCH=sm_N Build CUDA with an explicit nvcc -arch value" @echo " make test-mmq-parity-cuda CUDA_ARCH=sm_N Run quantized CUDA kernel parity tests" + @echo " make test-mmq-q4-grouped-q81-cuda CUDA_ARCH=sm_N Run focused grouped Q8_1 byte-parity tests" @echo " make test-mmq-q4-16warp-cuda CUDA_ARCH=sm_N Run focused Q4 16-warp bitwise/canary oracle" @echo " make test-rocm-q4-parity Run ROCm Q4_K dense/pair/prefill oracle" @echo " make test-strix-rocm-q4-prefill Require gfx1151 and run tiled-prefill oracle" @@ -576,6 +577,9 @@ cuda/mmq/test/test_mmq_parity: cuda/mmq/test/test_mmq_parity.cu cuda/mmq/ds4_mmq test-mmq-parity-cuda: cuda/mmq/test/test_mmq_parity ./cuda/mmq/test/test_mmq_parity +test-mmq-q4-grouped-q81-cuda: cuda/mmq/test/test_mmq_parity + ./cuda/mmq/test/test_mmq_parity --q4-grouped-q81 + test-mmq-q4-16warp-cuda: cuda/mmq/test/test_mmq_parity ./cuda/mmq/test/test_mmq_parity --q4-16warp diff --git a/QA_BEFORE_RELEASES.md b/QA_BEFORE_RELEASES.md index 54857b460..c25d39b3f 100644 --- a/QA_BEFORE_RELEASES.md +++ b/QA_BEFORE_RELEASES.md @@ -1007,10 +1007,18 @@ Do not use high-performance Hugging Face Xet mode while vLLM is resident. `./speed-bench/cuda_q4_prefill_bench --path mmq --case outa --tokens 127,128,129,257,512,2048,4096 --samples 16 --warmup 4`; require bitwise equality between `pack8_mmq_unpack` and `grouped_8_grids`, finite/canary/CPU - oracle success, and record the paired median. Then compare full-model + oracle success, and record the paired median. Run + `make test-mmq-q4-grouped-q81-cuda CUDA_ARCH=sm_121`, then isolate the new + Q8_1 front-end with + `--grouped-q81-kernel --tokens 512,1024,2048,4096,6144,8192`; require + byte-identical direct parity, bitwise final output, and a repeatable paired + median win between `grouped_generic_q81` and + `grouped_k4096_g8x2_q81`. Then compare full-model prefills with the default environment against the dominant - `DS4_CUDA_NO_Q4_GROUPED_ATTN_A_PREFILL=1` rollback. Keep the single-grid and - 16-warp experiments unset in this promotion comparison. Benchmark the K1024 + `DS4_CUDA_NO_Q4_GROUPED_ATTN_A_PREFILL=1` rollback, and separately against + the narrow `DS4_CUDA_NO_Q4_GROUPED_ATTN_A_Q81=1` rollback. Keep the + single-grid and 16-warp experiments unset in this promotion comparison. + Benchmark the K1024 persistent kernel as a separate fail-closed arm with both `DS4_CUDA_ENABLE_Q4_K1024_PERSISTENT=1` and diff --git a/README.md b/README.md index fbfaf0fc6..0e2f9db5c 100644 --- a/README.md +++ b/README.md @@ -791,11 +791,20 @@ arithmetic: - for prefill widths above eight, the default eligible GB10 path quantizes the strided `[token][group][K]` input in one launch and writes each group directly into `[token][group][rank]`. It removes the eight F32 pack/unpack copies while - keeping one established stream-K MMQ reduction per group. Set + keeping one established stream-K MMQ reduction per group. On the production + `groups=8`, `K=4096`, `rank=1024` shape, a fixed-layout eight-warp Q8_1 + producer is also the default: each warp emits two canonical 128-value DS4 + records, reducing quantizer CTA count by four while preserving every output + byte. `DS4_CUDA_NO_Q4_GROUPED_ATTN_A_Q81=1` restores only the generic + strided Q8_1 producer, and + `DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_Q81=1` fails closed if the specialized + producer is not selected. Set `DS4_CUDA_NO_Q4_GROUPED_ATTN_A_PREFILL=1` for the dominant local rollback; exact `DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_PREFILL=0` is also a compatibility opt-out. Add `DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_PREFILL=1` for fail-closed - tests. The separate single-grid/grid.z submission remains opt-in; + tests. The separate single-grid/grid.z submission remains opt-in. The + unrelated 16-warp MMQ experiment also remains opt-in and is not used by + this producer; - attention-output B keeps its canonical MMVQ result and the ordinary HC epilogue inside the graph-compatible fused call. The row-packed epilogue remains oracle-only until a GB10 device run proves it bit-exact; diff --git a/cuda/mmq/ds4_mmq.cu b/cuda/mmq/ds4_mmq.cu index 6e90605a2..90838bf16 100644 --- a/cuda/mmq/ds4_mmq.cu +++ b/cuda/mmq/ds4_mmq.cu @@ -1712,6 +1712,13 @@ int ds4_mmq_q4_K_dense_pair_impl( return 0; } +#if !defined(GGML_USE_HIP) +static bool ds4_q4_grouped_q81_env_enabled(const char *name) { + const char *value = getenv(name); + return value && value[0] && !(value[0] == '0' && value[1] == '\0'); +} +#endif + /* Token-batched grouped Q4_K projection for attention output-A. The source * is token-major [N][G][K], while MMQ stores directly into token-major * [N][G][M]. Quantizing the strided source as G channels removes the old @@ -1750,6 +1757,28 @@ int ds4_mmq_q4_K_grouped_dense_impl( const int dev = ggml_cuda_get_device(); const int cc = ggml_cuda_info().devices[dev].cc; +#if !defined(GGML_USE_HIP) + const bool q81_disable = + getenv("DS4_CUDA_NO_Q4_GROUPED_ATTN_A_Q81") != nullptr || + getenv("DS4_CUDA_NO_Q4_GROUPED_ATTN_A_PREFILL") != nullptr || + getenv("DS4_CUDA_NO_Q4_GROUPED_ATTN_A") != nullptr || + getenv("DS4_CUDA_NO_Q4_GB10_FAST") != nullptr; + const bool q81_require = ds4_q4_grouped_q81_env_enabled( + "DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_Q81"); + const bool q81_eligible = + gb10_optimizations_enabled() && + cc == GGML_CUDA_CC_DGX_SPARK && M == 1024 && N > 8 && + N <= INT32_MAX / (8*4096) && K == 4096 && n_groups == 8 && + (((uintptr_t)X & 15u) == 0u); + if (q81_require && (q81_disable || !q81_eligible)) { + fprintf(stderr, + "%s: required grouped K4096/G8 Q8_1 quantizer is not " + "eligible\n", + tag); + return DS4_MMQ_NOT_APPLICABLE; + } + const bool use_specialized_q81 = q81_eligible && !q81_disable; +#endif ggml_backend_cuda_context *ctx = get_ctx_for_device(dev); if (!ctx) { fprintf(stderr, "%s: failed to get cuda context for device %d\n", @@ -1800,14 +1829,22 @@ int ds4_mmq_q4_K_grouped_dense_impl( ggml_cuda_pool_alloc y_q8_1( ctx->pool(), payload_bytes + slack_bytes); ybuf_memset(y_q8_1.get(), payload_bytes + slack_bytes, stream); - quantize_mmq_q8_1_cuda( - X, /*ids=*/nullptr, (void *)y_q8_1.get(), GGML_TYPE_Q4_K, - /*ne00=*/K, - /*s01=*/(int64_t)n_groups * K, - /*s02=*/(int64_t)K, - /*s03=*/(int64_t)n_groups * N * K, - /*ne0=*/ne10_padded, /*ne1=*/N, - /*ne2=*/n_groups, /*ne3=*/1, stream); +#if !defined(GGML_USE_HIP) + if (use_specialized_q81) { + quantize_mmq_q8_1_q4_grouped_k4096_g8x2_cuda( + X, (void *)y_q8_1.get(), N, stream); + } else +#endif + { + quantize_mmq_q8_1_cuda( + X, /*ids=*/nullptr, (void *)y_q8_1.get(), GGML_TYPE_Q4_K, + /*ne00=*/K, + /*s01=*/(int64_t)n_groups * K, + /*s02=*/(int64_t)K, + /*s03=*/(int64_t)n_groups * N * K, + /*ne0=*/ne10_padded, /*ne1=*/N, + /*ne2=*/n_groups, /*ne3=*/1, stream); + } cudaError_t err = cudaGetLastError(); if (err != cudaSuccess) { fprintf(stderr, "%s: quantize failed: %s\n", @@ -2147,6 +2184,40 @@ extern "C" int ds4_mmq_q4_K_grouped_dense_single_grid( W, X, out, M, N, K, n_groups, true, stream); } +#if !defined(GGML_USE_HIP) +extern "C" size_t ds4_mmq_q4_K_grouped_q8_1_scratch_bytes_for_test(int N) { + if (N <= 0 || N > INT32_MAX / (8*4096)) return 0u; + constexpr size_t blocks_per_token = 8u * 4096u / (4u * QK8_1); + if ((size_t)N > SIZE_MAX / blocks_per_token / + sizeof(block_q8_1_mmq)) { + return 0u; + } + return (size_t)N * blocks_per_token * sizeof(block_q8_1_mmq); +} + +extern "C" int ds4_mmq_q4_K_grouped_quantize_q8_1_for_test( + const float *X, void *q8, size_t q8_bytes, int N, + int use_specialized, cudaStream_t stream) { + const size_t required = + ds4_mmq_q4_K_grouped_q8_1_scratch_bytes_for_test(N); + if (!X || !q8 || required == 0u || q8_bytes < required || + (((uintptr_t)X & 15u) != 0u)) { + return -1; + } + if (use_specialized) { + quantize_mmq_q8_1_q4_grouped_k4096_g8x2_cuda( + X, q8, N, stream); + } else { + quantize_mmq_q8_1_cuda( + X, /*ids=*/nullptr, q8, GGML_TYPE_Q4_K, + /*ne00=*/4096, /*s01=*/8*4096, /*s02=*/4096, + /*s03=*/(int64_t)N*8*4096, + /*ne0=*/4096, /*ne1=*/N, /*ne2=*/8, /*ne3=*/1, stream); + } + return cudaGetLastError() == cudaSuccess ? 0 : -2; +} +#endif + extern "C" int ds4_mmq_mxfp4_dense( const void * W, const float * X, float * out, int M, int N, int K, cudaStream_t stream) { diff --git a/cuda/mmq/ds4_mmq.h b/cuda/mmq/ds4_mmq.h index a69c8435c..425ce62ef 100644 --- a/cuda/mmq/ds4_mmq.h +++ b/cuda/mmq/ds4_mmq.h @@ -236,6 +236,20 @@ int ds4_mmq_q4_K_dense_preq_16warp_for_test( int N, int K, cudaStream_t stream); + +// Byte-parity boundary for the fixed [N][8][4096] grouped attention-A Q8_1 +// producer. use_specialized=0 launches the canonical strided quantizer; +// nonzero launches the K4096/G8x2 candidate. Both write the same canonical +// group-major block_q8_1_mmq DS4 payload and never synchronize the stream. +size_t ds4_mmq_q4_K_grouped_q8_1_scratch_bytes_for_test(int N); + +int ds4_mmq_q4_K_grouped_quantize_q8_1_for_test( + const float * X_f32, + void * q8_ds4, + size_t q8_bytes, + int N, + int use_specialized, + cudaStream_t stream); #endif // Two dense Q4_K MMQ projections that share one token-tiled Q8_1 diff --git a/cuda/mmq/quantize.cu b/cuda/mmq/quantize.cu index 52f664719..a8066c484 100644 --- a/cuda/mmq/quantize.cu +++ b/cuda/mmq/quantize.cu @@ -366,6 +366,77 @@ static __global__ void quantize_mmq_q8_1( } } +#if !defined(GGML_USE_HIP) +// Q4 grouped attention-A has one hot, fixed source geometry: +// [token][group=8][K=4096]. The generic MMQ quantizer launches four-warp +// CTAs, each producing four 128-value block_q8_1_mmq records. This kernel +// keeps the exact DS4 arithmetic and byte layout, but uses eight warps and +// lets each warp produce two consecutive records. One CTA therefore covers +// 2048 source values and the launch uses four times fewer CTAs, without the +// 16-warp geometry that regressed on GB10. +__launch_bounds__(8*WARP_SIZE) +static __global__ void quantize_mmq_q8_1_q4_grouped_k4096_g8x2( + const float * __restrict__ x, void * __restrict__ vy, const int ne1) { + constexpr int k_groups = 8; + constexpr int k_values = 4096; + constexpr int k_q8_blocks = k_values / (4*QK8_1); + constexpr int k_warps = 8; + constexpr int k_blocks_per_warp = 2; + constexpr int k_blocks_per_cta = k_warps * k_blocks_per_warp; + static_assert(WARP_SIZE == 32, "CUDA Q8_1 quantizer requires 32-lane warps"); + static_assert(k_q8_blocks == 32, "unexpected grouped Q8_1 block count"); + static_assert(2*k_blocks_per_cta == k_q8_blocks, + "two CTAs must cover one grouped activation row"); + + const int token = (int)blockIdx.x; + const int group = (int)blockIdx.z; + const int warp = (int)threadIdx.x / WARP_SIZE; + const int lane = (int)threadIdx.x % WARP_SIZE; + const uint32_t input_base = + ((uint32_t)token * k_groups + (uint32_t)group) * k_values; + const float4 * __restrict__ x4 = (const float4 *)x; + block_q8_1_mmq * __restrict__ y = (block_q8_1_mmq *)vy; + +#pragma unroll + for (int j = 0; j < k_blocks_per_warp; ++j) { + const int q8_block = + (int)blockIdx.y * k_blocks_per_cta + j * k_warps + warp; + const uint32_t input = input_base + + (uint32_t)q8_block * (4*QK8_1) + (uint32_t)lane * 4u; + const float4 xi = x4[input / 4u]; + + float amax = fabsf(xi.x); + amax = fmaxf(amax, fabsf(xi.y)); + amax = fmaxf(amax, fabsf(xi.z)); + amax = fmaxf(amax, fabsf(xi.w)); +#pragma unroll + for (int offset = 4; offset > 0; offset >>= 1) { + amax = fmaxf( + amax, __shfl_xor_sync(0xFFFFFFFF, amax, offset, WARP_SIZE)); + } + + float sum = xi.x + xi.y + xi.z + xi.w; +#pragma unroll + for (int offset = 4; offset > 0; offset >>= 1) { + sum += __shfl_xor_sync(0xFFFFFFFF, sum, offset, WARP_SIZE); + } + + const float d_inv = 127.0f / amax; + char4 q; + q.x = roundf(xi.x*d_inv); + q.y = roundf(xi.y*d_inv); + q.z = roundf(xi.z*d_inv); + q.w = roundf(xi.w*d_inv); + + const int ib = (group*k_q8_blocks + q8_block)*ne1 + token; + ((char4 *)y[ib].qs)[lane] = q; + if ((lane % 8) == 0) { + y[ib].ds4[lane / 8] = make_half2(1.0f / d_inv, sum); + } + } +} +#endif + void quantize_row_q8_1_cuda( const float * x, const int32_t * ids, void * vy, const ggml_type type_src0, const int64_t ne00, const int64_t s01, const int64_t s02, const int64_t s03, @@ -412,6 +483,22 @@ void quantize_mmq_q8_1_cuda( } } +#if !defined(GGML_USE_HIP) +void quantize_mmq_q8_1_q4_grouped_k4096_g8x2_cuda( + const float * x, void * vy, int ne1, cudaStream_t stream) { + GGML_ASSERT(x); + GGML_ASSERT(vy); + GGML_ASSERT(ne1 > 0 && ne1 <= INT32_MAX / (8*4096)); + + constexpr int k_warps = 8; + constexpr int k_grid_y = 2; + const dim3 num_blocks(ne1, k_grid_y, 8); + const dim3 block_size(k_warps*WARP_SIZE, 1, 1); + quantize_mmq_q8_1_q4_grouped_k4096_g8x2<<< + num_blocks, block_size, 0, stream>>>(x, vy, ne1); +} +#endif + void quantize_mmq_fp4_cuda( const float * x, const int32_t * ids, void * vy, const ggml_type type_src0, const int64_t ne00, const int64_t s01, const int64_t s02, const int64_t s03, diff --git a/cuda/mmq/quantize.cuh b/cuda/mmq/quantize.cuh index 768a3ae6d..ecc1ca6df 100644 --- a/cuda/mmq/quantize.cuh +++ b/cuda/mmq/quantize.cuh @@ -26,6 +26,13 @@ void quantize_mmq_q8_1_cuda( ggml_type type_src0, int64_t ne00, int64_t s01, int64_t s02, int64_t s03, int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne3, cudaStream_t stream); +#if !defined(GGML_USE_HIP) +// Fixed-layout Q4 attention-A prefill producer. X is [N][8][4096] and the +// output is canonical block_q8_1_mmq DS4, group-major [8][32][N]. +void quantize_mmq_q8_1_q4_grouped_k4096_g8x2_cuda( + const float * x, void * vy, int ne1, cudaStream_t stream); +#endif + void quantize_mmq_fp4_cuda(const float * x, const int32_t * ids, void * vy, diff --git a/cuda/mmq/test/test_mmq_parity.cu b/cuda/mmq/test/test_mmq_parity.cu index 7a25c0a5a..cdf4aa341 100644 --- a/cuda/mmq/test/test_mmq_parity.cu +++ b/cuda/mmq/test/test_mmq_parity.cu @@ -1612,6 +1612,149 @@ bool run_q4_K_grouped_dense_parity( return ok; } +#if !defined(GGML_USE_HIP) +// The production selector is deliberately GB10-only, but the specialized +// quantizer uses baseline CUDA operations. Exercise it directly on every +// CUDA test device and require the complete block_q8_1_mmq payload to match +// the canonical strided producer byte-for-byte. +bool run_q4_K_grouped_q8_1_kernel_parity(int N, uint32_t seed) { + fprintf(stderr, + "=== Q4_K/GROUPED_Q8_1_K4096_G8X2 N=%d seed=%u ===\n", + N, seed); + + constexpr int K = 4096; + constexpr int groups = 8; + constexpr size_t guard_bytes = 256u; + constexpr uint8_t guard_byte = 0xa5u; + const size_t q8_bytes = + ds4_mmq_q4_K_grouped_q8_1_scratch_bytes_for_test(N); + const size_t x_count = (size_t)N * groups * K; + if (q8_bytes == 0u || + q8_bytes > SIZE_MAX - 2u * guard_bytes) { + fprintf(stderr, "invalid grouped Q8_1 parity shape\n\n"); + return false; + } + + std::mt19937 rng(seed); + std::normal_distribution nd(0.0f, 1.0f); + std::vector X(x_count); + for (float &value : X) value = nd(rng); + + cudaStream_t stream = nullptr; + float *dX = nullptr; + void *dReferenceStorage = nullptr; + void *dCandidateStorage = nullptr; + const bool allocated = cudaStreamCreate(&stream) == cudaSuccess && + cudaMalloc(&dX, X.size() * sizeof(float)) == cudaSuccess && + cudaMalloc(&dReferenceStorage, q8_bytes + 2u * guard_bytes) == + cudaSuccess && + cudaMalloc(&dCandidateStorage, q8_bytes + 2u * guard_bytes) == + cudaSuccess; + const auto cleanup = [&]() { + if (dCandidateStorage) cudaFree(dCandidateStorage); + if (dReferenceStorage) cudaFree(dReferenceStorage); + if (dX) cudaFree(dX); + if (stream) cudaStreamDestroy(stream); + }; + if (!allocated) { + fprintf(stderr, + "grouped Q8_1 parity allocation failed: %s\n\n", + cudaGetErrorString(cudaGetLastError())); + cleanup(); + return false; + } + + auto *dReference = + static_cast(dReferenceStorage) + guard_bytes; + auto *dCandidate = + static_cast(dCandidateStorage) + guard_bytes; + cudaError_t enqueue_err = cudaMemcpyAsync( + dX, X.data(), X.size() * sizeof(float), + cudaMemcpyHostToDevice, stream); + if (enqueue_err == cudaSuccess) { + enqueue_err = cudaMemsetAsync( + dReferenceStorage, guard_byte, + q8_bytes + 2u * guard_bytes, stream); + } + if (enqueue_err == cudaSuccess) { + enqueue_err = cudaMemsetAsync( + dCandidateStorage, guard_byte, + q8_bytes + 2u * guard_bytes, stream); + } + + const int rc_reference = enqueue_err == cudaSuccess + ? ds4_mmq_q4_K_grouped_quantize_q8_1_for_test( + dX, dReference, q8_bytes, N, /*use_specialized=*/0, stream) + : -100; + const int rc_candidate = rc_reference == 0 + ? ds4_mmq_q4_K_grouped_quantize_q8_1_for_test( + dX, dCandidate, q8_bytes, N, /*use_specialized=*/1, stream) + : -100; + + std::vector reference(q8_bytes + 2u * guard_bytes); + std::vector candidate(q8_bytes + 2u * guard_bytes); + if (rc_candidate == 0) { + enqueue_err = cudaMemcpyAsync( + reference.data(), dReferenceStorage, reference.size(), + cudaMemcpyDeviceToHost, stream); + } + if (enqueue_err == cudaSuccess && rc_candidate == 0) { + enqueue_err = cudaMemcpyAsync( + candidate.data(), dCandidateStorage, candidate.size(), + cudaMemcpyDeviceToHost, stream); + } + const cudaError_t sync_err = cudaStreamSynchronize(stream); + + size_t mismatches = 0; + size_t first_mismatch = SIZE_MAX; + for (size_t i = 0; i < q8_bytes; ++i) { + const size_t offset = guard_bytes + i; + if (reference[offset] != candidate[offset]) { + if (first_mismatch == SIZE_MAX) first_mismatch = i; + mismatches++; + } + } + size_t canary_mismatches = 0; + for (size_t i = 0; i < guard_bytes; ++i) { + if (reference[i] != guard_byte || candidate[i] != guard_byte) { + canary_mismatches++; + } + const size_t suffix = guard_bytes + q8_bytes + i; + if (reference[suffix] != guard_byte || + candidate[suffix] != guard_byte) { + canary_mismatches++; + } + } + + const bool ok = rc_reference == 0 && rc_candidate == 0 && + enqueue_err == cudaSuccess && sync_err == cudaSuccess && + mismatches == 0 && canary_mismatches == 0; + const std::string first = first_mismatch == SIZE_MAX + ? "none" : std::to_string(first_mismatch); + fprintf(stderr, + "rc=%d/%d enqueue=%s sync=%s bytes=%zu mismatches=%zu " + "first=%s canary=%zu: %s\n\n", + rc_reference, rc_candidate, cudaGetErrorString(enqueue_err), + cudaGetErrorString(sync_err), q8_bytes, mismatches, + first.c_str(), canary_mismatches, ok ? "PASS" : "FAIL"); + cleanup(); + return ok; +} + +bool run_q4_K_grouped_q8_1_kernel_suite() { + bool ok = true; + ok &= run_q4_K_grouped_q8_1_kernel_parity( + /*N=*/9, 0xC4810009u); + ok &= run_q4_K_grouped_q8_1_kernel_parity( + /*N=*/127, 0xC481007Fu); + ok &= run_q4_K_grouped_q8_1_kernel_parity( + /*N=*/128, 0xC4810080u); + ok &= run_q4_K_grouped_q8_1_kernel_parity( + /*N=*/129, 0xC4810081u); + return ok; +} +#endif + // IQ2_XXS internally accumulates in int8 via SIMD intrinsics // (__vsub4 / __vcmpne4 in vec_dot_iq2_xxs_q8_1) and applies the scale // post-accumulation, while the CPU reference does per-element float @@ -3252,6 +3395,10 @@ bool run_q4_K_grouped_vec_parity( int main(int argc, char ** argv) { const bool q4_16warp_oracle = argc == 2 && std::strcmp(argv[1], "--q4-16warp") == 0; +#if !defined(GGML_USE_HIP) + const bool q4_grouped_q81_oracle = + argc == 2 && std::strcmp(argv[1], "--q4-grouped-q81") == 0; +#endif scoped_env_override require_16warp( "DS4_CUDA_REQUIRE_Q4_MMQ_16WARP"); scoped_env_override disable_16warp( @@ -3270,6 +3417,16 @@ int main(int argc, char ** argv) { bool all_ok = true; +#if !defined(GGML_USE_HIP) + if (q4_grouped_q81_oracle) { + all_ok &= run_q4_K_grouped_q8_1_kernel_suite(); + fprintf(stderr, "===================\n"); + fprintf(stderr, "Q4 GROUPED Q8_1 %s\n", + all_ok ? "PASS" : "FAILED"); + return all_ok ? 0 : 1; + } +#endif + if (q4_16warp_oracle) { // The canonical production baseline must select mmq_x=128. Mirror // get_mmq_x_max_host's numeric-prefix parsing and reject a narrower @@ -3478,6 +3635,11 @@ int main(int argc, char ** argv) { all_ok &= run_q4_K_grouped_dense_parity( /*M=*/31, /*N=*/129, /*K=*/512, /*groups=*/3, 0xC4D081, /*inject_nonfinite=*/true); +#if !defined(GGML_USE_HIP) + // Fixed production Q8_1 front-end: cover the first eligible width and + // both sides of the canonical 128-token tile boundary. + all_ok &= run_q4_K_grouped_q8_1_kernel_suite(); +#endif // MoE (_id) path. Small expert counts + small shapes for fast verification. // Per-token-distinct routing with top_k=2 or 6. diff --git a/ds4_cuda.cu b/ds4_cuda.cu index 89e037e09..b59fad5ec 100644 --- a/ds4_cuda.cu +++ b/ds4_cuda.cu @@ -44085,8 +44085,11 @@ extern "C" int ds4_gpu_attention_output_q4_K_batch_tensor( "DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_PREFILL", 0); const int grouped_single_grid_require = cuda_env_flag_enabled( "DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_SINGLE_GRID", 0); + const int grouped_q81_require = cuda_env_flag_enabled( + "DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_Q81", 0); const int any_grouped_require = grouped_batch_require || - grouped_prefill_require || grouped_single_grid_require; + grouped_prefill_require || grouped_single_grid_require || + grouped_q81_require; if (!out || !low || !group_tmp || !low_tmp || !heads || !model_map || group_dim == 0 || rank == 0 || n_groups == 0 || out_dim == 0 || n_tokens < 2u || group_dim > INT_MAX || rank > INT_MAX || @@ -44136,11 +44139,14 @@ extern "C" int ds4_gpu_attention_output_q4_K_batch_tensor( "DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_SINGLE_GRID", 0); const int grouped_single_grid_disable = cuda_env_flag_enabled( "DS4_CUDA_DISABLE_Q4_GROUPED_ATTN_A_SINGLE_GRID", 0); + const int grouped_q81_disable = + getenv("DS4_CUDA_NO_Q4_GROUPED_ATTN_A_Q81") != NULL; const int grouped_single_grid_selected = (grouped_single_grid_enable || grouped_single_grid_require) && !grouped_single_grid_disable; const int grouped_prefill_selected = grouped_prefill_enable || - grouped_prefill_require || grouped_single_grid_selected; + grouped_prefill_require || grouped_single_grid_selected || + grouped_q81_require; if (grouped_oracle) cuda_q4_grouped_attn_a_oracle_register_report(); const int grouped_gb10 = @@ -44176,6 +44182,15 @@ extern "C" int ds4_gpu_attention_output_q4_K_batch_tensor( "path is not eligible\n"); return -1; } + if (grouped_q81_require && + (grouped_q81_disable || !grouped_prefill_gb10 || + rank != 1024u || group_dim != 4096u || n_groups != 8u || + n_tokens > (uint32_t)(INT32_MAX / (8*4096)))) { + fprintf(stderr, + "ds4: required CUDA Q4 grouped attention-A K4096/G8 " + "Q8_1 quantizer is not eligible\n"); + return -1; + } if (grouped_prefill_selected && grouped_prefill_gb10) { int rc = grouped_single_grid_selected ? ds4_mmq_q4_K_grouped_dense_single_grid( diff --git a/scripts/environment_variables.tsv b/scripts/environment_variables.tsv index 75ab33069..225ef25d6 100644 --- a/scripts/environment_variables.tsv +++ b/scripts/environment_variables.tsv @@ -236,6 +236,7 @@ runtime/cuda DS4_CUDA_NO_Q4_GB10_FAST presence kill switch; default unset (eligi runtime/cuda DS4_CUDA_NO_Q4_GROUPED_ATTN_A presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q4 grouped attn a CUDA Q4 optimization. cuda/mmq/ds4_mmq.cu:4295 runtime/cuda DS4_CUDA_NO_Q4_GROUPED_ATTN_A_BATCH presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q4 grouped attn a batch CUDA Q4 optimization. cuda/mmq/ds4_mmq.cu:4305 runtime/cuda DS4_CUDA_NO_Q4_GROUPED_ATTN_A_PREFILL presence kill switch for the default-on GB10 path; default unset; any defined value including empty or 0 disables and dominates ENABLE/REQUIRE Restore the eight pack/MMQ/unpack Q4 attention-A prefill projections. ds4_cuda.cu:41930 +runtime/cuda DS4_CUDA_NO_Q4_GROUPED_ATTN_A_Q81 presence rollback for the default-on fixed-shape quantizer; default unset; any defined value including empty or 0 disables; REQUIRE then fails closed Restore the canonical strided Q8_1 producer while retaining grouped Q4 attention-A prefill and its eight MMQ grids. cuda/mmq/ds4_mmq.cu:1762; ds4_cuda.cu:44143 runtime/cuda DS4_CUDA_NO_Q4_K1024_PERSISTENT presence kill switch, default off; any defined value including 0 disables Disable the Q4 K1024 persistent CUDA Q4 optimization. cuda/mmq/ds4_mmq.cu:3907 runtime/cuda DS4_CUDA_NO_Q4_MMQ_16WARP value-aware rollback, default off; unset/empty/exact 0 permits the experiment, every other nonempty value disables it and overrides REQUEST/REQUIRE Disable the experimental Stream-K-compatible CUDA Q4_K m128n128 16-warp prefill kernel. cuda/mmq/ds4_mmq.cu:1133 runtime/cuda DS4_CUDA_NO_Q8_ALIGNED_DENSE_SCRATCH presence kill switch; default unset (eligible path remains available); any defined value including 0 disables Disable the Q8 aligned dense scratch CUDA Q8 optimization. cuda/mmq/ds4_mmq.cu:5578 @@ -309,6 +310,7 @@ runtime/cuda DS4_CUDA_REQUIRE_IQ2_XXS_SSD_PREFILL_MMQ false-like-aware flag, def runtime/cuda DS4_CUDA_REQUIRE_Q4_ATTN_Q_B_F16_CACHE value-aware strict opt-in, default off; unset/empty/0/false/no/off is off, any other nonempty value requires eligible batches to use the cache; DISABLE wins Fail an eligible CUDA prefill instead of falling back when the resident Q4_K attn_q_b F16 specialization cannot be prepared or dispatched. ds4_cuda.cu:2492; ds4_cuda.cu:2497 runtime/cuda DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_BATCH value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on Fail if grouped batched attention-A cannot be used. ds4_cuda.cu:40198 runtime/cuda DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_PREFILL value-aware fail-closed assertion, default off; unset/empty/exact 0 is off, any other nonempty value requests the candidate and rejects ineligibility before enqueue Require the GB10 grouped Q4_K attention-A prefill path instead of silently using pack/MMQ/unpack. ds4_cuda.cu:41889 +runtime/cuda DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_Q81 value-aware fail-closed assertion, default off; unset/empty/exact 0 is off, any other nonempty value requests grouped prefill and the fixed K=4096, groups=8, rank=1024 Q8_1 producer; NO wins Require the eight-warp K4096/G8x2 Q8_1 producer instead of silently using the generic strided quantizer. cuda/mmq/ds4_mmq.cu:1767; ds4_cuda.cu:44089 runtime/cuda DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_SINGLE_GRID value-aware fail-closed assertion, default off; unset/empty/exact 0 is off, every other nonempty value requests the candidate; DISABLE or ineligibility fails before enqueue Require one grid.z MMQ submission for eligible GB10 grouped Q4_K attention-A prefill instead of falling back to one launch per group. ds4_cuda.cu:41896 runtime/cuda DS4_CUDA_REQUIRE_Q4_K1024_PERSISTENT presence flag, default off; any defined value including 0 makes ineligible candidate fail closed Fail when the exact Q4 K1024 persistent candidate is unavailable instead of using MMVQ. cuda/mmq/ds4_mmq.cu:3929 runtime/cuda DS4_CUDA_REQUIRE_Q4_MMQ_16WARP value-aware fail-closed prefill opt-in cached on the first Q4_K dense or dense-pair MMQ call; unset/empty/exact 0 is off, every other nonempty value requests and requires the candidate for N>8; a dense-pair is rejected before allocation unless both legs are eligible; rollback, disabled MMQ, ineligibility, or preflight failure prevents fallback; decode/speculative N<=8 remains on MMVQ Require the experimental CUDA Q4_K 16-warp prefill kernel so benchmark runs cannot silently measure another path. cuda/mmq/ds4_mmq.cu:1130; ds4_cuda.cu:38208; ds4_cuda.cu:38358 diff --git a/speed-bench/README.md b/speed-bench/README.md index f3235b16d..86ac1b211 100644 --- a/speed-bench/README.md +++ b/speed-bench/README.md @@ -410,6 +410,10 @@ On GB10, isolate the production Flash attention-output A geometry ./speed-bench/cuda_q4_prefill_bench \ --path mmq --case outa --tokens 127,128,129,257,512,2048 \ --samples 16 --warmup 4 +./speed-bench/cuda_q4_prefill_bench \ + --path mmq --case outa --grouped-q81-kernel \ + --tokens 512,1024,2048,4096,6144,8192 \ + --sets 4 --samples 16 --warmup 4 ``` This is an in-process ABBA/BAAB comparison between the current eight-group @@ -418,6 +422,13 @@ grouped-prefill dispatch with one canonical MMQ grid per group. Add `--grouped-single-grid` to instead compare the grouped eight-grid path with the experimental grid.z submission; do not mix that experiment into the default promotion measurement. +`--grouped-q81-kernel` holds those same eight MMQ grids constant and compares +the canonical strided Q8_1 producer with the default `K=4096`, `groups=8` +eight-warp kernel. The candidate is required and the reference is forced with +its narrow rollback, so neither arm can silently time the other quantizer. The +complete output remains bitwise checked; run +`make test-mmq-q4-grouped-q81-cuda CUDA_ARCH=sm_121` first for direct +byte-level Q8_1 parity and canary coverage. The public API must also execute output-B, so the fixture uses a valid Q4_K `K=8192,M=256` output-B common to both arms. It is 6.25% of output-A's MACs; the result prints `focus_macs_per_token` and `common_macs_per_token` separately diff --git a/speed-bench/cuda_q4_prefill_bench.cu b/speed-bench/cuda_q4_prefill_bench.cu index f9704055c..a8c6e3dde 100644 --- a/speed-bench/cuda_q4_prefill_bench.cu +++ b/speed-bench/cuda_q4_prefill_bench.cu @@ -52,6 +52,10 @@ constexpr const char *kGroupedSingleGridDisable = "DS4_CUDA_DISABLE_Q4_GROUPED_ATTN_A_SINGLE_GRID"; constexpr const char *kGroupedSingleGridRequire = "DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_SINGLE_GRID"; +constexpr const char *kGroupedQ81Disable = + "DS4_CUDA_NO_Q4_GROUPED_ATTN_A_Q81"; +constexpr const char *kGroupedQ81Require = + "DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_Q81"; constexpr const char *kGroupedGlobalDisable = "DS4_CUDA_NO_Q4_GROUPED_ATTN_A"; constexpr const char *kGb10GlobalDisable = "DS4_CUDA_NO_Q4_GB10_FAST"; @@ -90,6 +94,7 @@ struct config { uint32_t warmup = kDefaultWarmup; bool kernel_16warp = false; bool grouped_single_grid = false; + bool grouped_q81_kernel = false; }; struct weight_set { @@ -706,7 +711,9 @@ bool select_grouped_prefill_legacy() { unsetenv(kGroupedPrefillRequire) == 0 && unsetenv(kGroupedSingleGridEnable) == 0 && setenv(kGroupedSingleGridDisable, "1", 1) == 0 && - unsetenv(kGroupedSingleGridRequire) == 0; + unsetenv(kGroupedSingleGridRequire) == 0 && + unsetenv(kGroupedQ81Disable) == 0 && + unsetenv(kGroupedQ81Require) == 0; } bool select_grouped_prefill_grid8() { @@ -715,7 +722,9 @@ bool select_grouped_prefill_grid8() { setenv(kGroupedPrefillRequire, "1", 1) == 0 && unsetenv(kGroupedSingleGridEnable) == 0 && setenv(kGroupedSingleGridDisable, "1", 1) == 0 && - unsetenv(kGroupedSingleGridRequire) == 0; + unsetenv(kGroupedSingleGridRequire) == 0 && + unsetenv(kGroupedQ81Disable) == 0 && + unsetenv(kGroupedQ81Require) == 0; } bool select_grouped_prefill_single_grid() { @@ -724,7 +733,21 @@ bool select_grouped_prefill_single_grid() { setenv(kGroupedPrefillRequire, "1", 1) == 0 && setenv(kGroupedSingleGridEnable, "1", 1) == 0 && unsetenv(kGroupedSingleGridDisable) == 0 && - setenv(kGroupedSingleGridRequire, "1", 1) == 0; + setenv(kGroupedSingleGridRequire, "1", 1) == 0 && + unsetenv(kGroupedQ81Disable) == 0 && + unsetenv(kGroupedQ81Require) == 0; +} + +bool select_grouped_q81_reference() { + return select_grouped_prefill_grid8() && + setenv(kGroupedQ81Disable, "1", 1) == 0 && + unsetenv(kGroupedQ81Require) == 0; +} + +bool select_grouped_q81_candidate() { + return select_grouped_prefill_grid8() && + unsetenv(kGroupedQ81Disable) == 0 && + setenv(kGroupedQ81Require, "1", 1) == 0; } double percentile(std::vector sorted, double fraction) { @@ -1618,8 +1641,12 @@ bool run_output_a(const model_fixture &model, const config &cfg, } const bool compare_single_grid = cfg.grouped_single_grid; + const bool compare_q81_kernel = cfg.grouped_q81_kernel; const arm baseline = { - compare_single_grid ? "grouped_8_grids" : "pack8_mmq_unpack", + compare_q81_kernel + ? "grouped_generic_q81" + : (compare_single_grid + ? "grouped_8_grids" : "pack8_mmq_unpack"), [&](uint32_t set) { return ds4_gpu_attention_output_q4_K_batch_tensor( baseline_out.ptr, baseline_low.ptr, group_tmp.ptr, @@ -1630,11 +1657,17 @@ bool run_output_a(const model_fixture &model, const config &cfg, heads.ptr, n_tokens) > 0; }, [=]() { - return compare_single_grid ? select_grouped_prefill_grid8() - : select_grouped_prefill_legacy(); + return compare_q81_kernel + ? select_grouped_q81_reference() + : (compare_single_grid + ? select_grouped_prefill_grid8() + : select_grouped_prefill_legacy()); }}; const arm candidate = { - compare_single_grid ? "grouped_single_grid" : "grouped_8_grids", + compare_q81_kernel + ? "grouped_k4096_g8x2_q81" + : (compare_single_grid + ? "grouped_single_grid" : "grouped_8_grids"), [&](uint32_t set) { return ds4_gpu_attention_output_q4_K_batch_tensor( candidate_out.ptr, candidate_low.ptr, group_tmp.ptr, @@ -1645,9 +1678,11 @@ bool run_output_a(const model_fixture &model, const config &cfg, heads.ptr, n_tokens) > 0; }, [=]() { - return compare_single_grid - ? select_grouped_prefill_single_grid() - : select_grouped_prefill_grid8(); + return compare_q81_kernel + ? select_grouped_q81_candidate() + : (compare_single_grid + ? select_grouped_prefill_single_grid() + : select_grouped_prefill_grid8()); }}; const uint64_t output_a_macs_per_token = static_cast(kOutputGroups) * kDenseK * kOutputRank; @@ -1672,9 +1707,11 @@ bool run_output_a(const model_fixture &model, const config &cfg, [&](uint32_t set) { return bitwise_equal(baseline_low.ptr, candidate_low.ptr, low_bytes, - compare_single_grid - ? "output_a single-grid vs eight-grid" - : "output_a grouped vs pack/unpack") && + compare_q81_kernel + ? "output_a specialized vs generic Q8_1" + : (compare_single_grid + ? "output_a single-grid vs eight-grid" + : "output_a grouped vs pack/unpack")) && bitwise_equal(baseline_out.ptr, candidate_out.ptr, out_bytes, "output_a minimal-B final output") && @@ -1733,6 +1770,7 @@ void usage(FILE *stream, const char *argv0) { " --samples N samples/arm, multiple of 4 (default: %u)\n" " --warmup N untimed dispatches/arm (default: %u)\n" " --grouped-single-grid compare grouped 8-grid vs grid.z outa\n" + " --grouped-q81-kernel compare generic vs K4096/G8x2 Q8_1 outa\n" " --kernel-16warp prequantized canonical-vs-16-warp A/B\n" " -h, --help show this help\n\n" "Dense and q_b measure one immutable process path. Run separate " @@ -1743,7 +1781,10 @@ void usage(FILE *stream, const char *argv0) { "pack/MMQ/unpack sequence against the default\ndirect-strided grouped " "path with one canonical MMQ grid per group. Add\n" "--grouped-single-grid to compare that default with the experimental " - "grid.z\nsubmission instead. It includes a common minimal " + "grid.z\nsubmission instead. Add\n" + "--grouped-q81-kernel to isolate the default fixed-shape Q8_1 " + "front-end against\nthe canonical strided quantizer while retaining " + "the same eight MMQ grids. Both outa comparisons include a common minimal " "Q4 output-B (M=256) whose MACs are reported separately. " "DS4_CUDA_MMQ_X_MAX\nmay explicitly select an 8..128 multiple-of-8 " "sweep point; the setup line\nattests it, or prints auto when the " @@ -1862,6 +1903,8 @@ config parse_options(int argc, char **argv) { cfg.kernel_16warp = true; } else if (!std::strcmp(argv[i], "--grouped-single-grid")) { cfg.grouped_single_grid = true; + } else if (!std::strcmp(argv[i], "--grouped-q81-kernel")) { + cfg.grouped_q81_kernel = true; } else { std::fprintf(stderr, "unknown option: %s\n", argv[i]); usage(stderr, argv[0]); @@ -1876,6 +1919,9 @@ config parse_options(int argc, char **argv) { if (cfg.kernel_16warp && !tokens_explicit) { cfg.tokens = {512u, 1024u, 2048u, 2049u, 4096u, 6144u, 8192u}; } + if (cfg.grouped_q81_kernel && !tokens_explicit) { + cfg.tokens = {512u, 1024u, 2048u, 4096u, 6144u, 8192u}; + } if (cfg.kernel_16warp) { const auto below_minimum = std::find_if( cfg.tokens.begin(), cfg.tokens.end(), @@ -1898,18 +1944,36 @@ config parse_options(int argc, char **argv) { "--grouped-single-grid requires --path mmq\n"); std::exit(2); } + if (cfg.grouped_q81_kernel && cfg.path != cuda_path::mmq) { + std::fprintf(stderr, + "--grouped-q81-kernel requires --path mmq\n"); + std::exit(2); + } if (cfg.grouped_single_grid && cfg.kernel_16warp) { std::fprintf(stderr, "--grouped-single-grid cannot be combined with " "--kernel-16warp\n"); std::exit(2); } + if (cfg.grouped_q81_kernel && + (cfg.grouped_single_grid || cfg.kernel_16warp)) { + std::fprintf(stderr, + "--grouped-q81-kernel cannot be combined with " + "--grouped-single-grid or --kernel-16warp\n"); + std::exit(2); + } if (cfg.grouped_single_grid && cfg.selected != bench_case::all && cfg.selected != bench_case::outa) { std::fprintf(stderr, "--grouped-single-grid requires --case outa or all\n"); std::exit(2); } + if (cfg.grouped_q81_kernel && cfg.selected != bench_case::all && + cfg.selected != bench_case::outa) { + std::fprintf(stderr, + "--grouped-q81-kernel requires --case outa or all\n"); + std::exit(2); + } if (cfg.kernel_16warp && cfg.selected == bench_case::outa) { std::fprintf(stderr, "--kernel-16warp supports only --case dense, pair, qb, " @@ -2069,6 +2133,8 @@ int main(int argc, char **argv) { env_snapshot grouped_single_grid_enable_guard(kGroupedSingleGridEnable); env_snapshot grouped_single_grid_disable_guard(kGroupedSingleGridDisable); env_snapshot grouped_single_grid_require_guard(kGroupedSingleGridRequire); + env_snapshot grouped_q81_disable_guard(kGroupedQ81Disable); + env_snapshot grouped_q81_require_guard(kGroupedQ81Require); env_snapshot grouped_global_disable_guard(kGroupedGlobalDisable); env_snapshot gb10_global_disable_guard(kGb10GlobalDisable); if (setenv("DS4_CUDA_MMQ", @@ -2082,6 +2148,8 @@ int main(int argc, char **argv) { unsetenv(kGroupedSingleGridEnable) != 0 || unsetenv(kGroupedSingleGridDisable) != 0 || unsetenv(kGroupedSingleGridRequire) != 0 || + unsetenv(kGroupedQ81Disable) != 0 || + unsetenv(kGroupedQ81Require) != 0 || unsetenv(kGroupedGlobalDisable) != 0 || unsetenv(kGb10GlobalDisable) != 0) { std::fprintf(stderr, @@ -2200,6 +2268,7 @@ int main(int argc, char **argv) { "mmq_x_max=%s sets=%u resident_payload_mib=%.2f " "device_free_delta_mib=%.2f " "device_free_delta_valid=%d timing=%s kernel_16warp=%d " + "grouped_q81_kernel=%d " "cases=%s " "ssd_streaming=off model_storage=cudaMalloc " "residency=backend_provenance strict_mmq=%d " @@ -2213,11 +2282,14 @@ int main(int argc, char **argv) { resident_delta_valid ? 1 : 0, cfg.kernel_16warp ? "kernel_only_prequant" : "cuda_events", cfg.kernel_16warp ? 1 : 0, + cfg.grouped_q81_kernel ? 1 : 0, case_scope(cfg), cfg.path == cuda_path::mmq ? 1 : 0, grouped_prefill_supported ? "available" : "skipped", - cfg.grouped_single_grid ? "grid8_vs_single_grid" - : "pack8_vs_grid8"); + cfg.grouped_q81_kernel + ? "q81_generic_vs_k4096_g8x2" + : (cfg.grouped_single_grid + ? "grid8_vs_single_grid" : "pack8_vs_grid8")); std::fflush(stdout); for (uint32_t n_tokens : cfg.tokens) { if (includes(cfg.selected, bench_case::dense)) { diff --git a/tests/cuda_q4_gb10_fast_matrix.sh b/tests/cuda_q4_gb10_fast_matrix.sh index 4f8046245..41458dcb0 100755 --- a/tests/cuda_q4_gb10_fast_matrix.sh +++ b/tests/cuda_q4_gb10_fast_matrix.sh @@ -141,6 +141,8 @@ clean_env() { -u DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_SINGLE_GRID \ -u DS4_CUDA_DISABLE_Q4_GROUPED_ATTN_A_SINGLE_GRID \ -u DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_SINGLE_GRID \ + -u DS4_CUDA_NO_Q4_GROUPED_ATTN_A_Q81 \ + -u DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_Q81 \ -u DS4_CUDA_Q4_GROUPED_ATTN_A_ORACLE \ -u DS4_CUDA_DISABLE_Q4_ATTN_OUT_HC_FUSE \ -u DS4_CUDA_ENABLE_Q4_ATTN_OUT_HC_FUSE \ @@ -323,6 +325,7 @@ LOCAL_ROLLBACK="DS4_CUDA_NO_Q4_DENSE_SCRATCH=1 DS4_CUDA_NO_Q4_GROUPED_ATTN_A=1 DS4_CUDA_NO_Q4_GROUPED_ATTN_A_BATCH=1 DS4_CUDA_NO_Q4_GROUPED_ATTN_A_PREFILL=1 +DS4_CUDA_NO_Q4_GROUPED_ATTN_A_Q81=1 DS4_CUDA_DISABLE_Q4_ATTN_OUT_HC_FUSE=1 DS4_CUDA_NO_Q4_K1024_PERSISTENT=1" @@ -360,6 +363,9 @@ run_smoke hc_only \ DS4_CUDA_NO_Q4_GROUPED_ATTN_A_PREFILL=1 \ DS4_CUDA_NO_Q4_K1024_PERSISTENT=1 run_smoke default_fast DS4_CUDA_NO_Q4_K1024_PERSISTENT=1 +run_smoke grouped_q81_rollback \ + DS4_CUDA_NO_Q4_GROUPED_ATTN_A_Q81=1 \ + DS4_CUDA_NO_Q4_K1024_PERSISTENT=1 run_smoke grouped_prefill_rollback \ DS4_CUDA_NO_Q4_GROUPED_ATTN_A_PREFILL=1 \ DS4_CUDA_NO_Q4_K1024_PERSISTENT=1 @@ -400,7 +406,8 @@ compare_smoke() { } for arm in local_control scratch_only grouped_only hc_only default_fast \ - grouped_prefill_rollback grouped_oracle hc_oracle; do + grouped_q81_rollback grouped_prefill_rollback \ + grouped_oracle hc_oracle; do compare_smoke "$arm" done @@ -426,12 +433,15 @@ run_score hc_only \ DS4_CUDA_NO_Q4_GROUPED_ATTN_A_PREFILL=1 \ DS4_CUDA_NO_Q4_K1024_PERSISTENT=1 run_score default_fast DS4_CUDA_NO_Q4_K1024_PERSISTENT=1 +run_score grouped_q81_rollback \ + DS4_CUDA_NO_Q4_GROUPED_ATTN_A_Q81=1 \ + DS4_CUDA_NO_Q4_K1024_PERSISTENT=1 run_score grouped_prefill_rollback \ DS4_CUDA_NO_Q4_GROUPED_ATTN_A_PREFILL=1 \ DS4_CUDA_NO_Q4_K1024_PERSISTENT=1 for arm in local_control scratch_only grouped_only hc_only default_fast \ - grouped_prefill_rollback; do + grouped_q81_rollback grouped_prefill_rollback; do if cmp -s "$OUT_DIR/umbrella_control.tsv" "$OUT_DIR/$arm.tsv"; then echo "q4-gb10-matrix: quality $arm: EXACT" else From 9f005f4cfb5deb2fe47fa5ad35e1b6bff4ef3c26 Mon Sep 17 00:00:00 2001 From: Ian M Date: Fri, 4 Sep 2026 17:23:27 +0100 Subject: [PATCH 188/189] rocm: fix build break from Metal-only TP/kv-norm symbols ds4_gpu_add_tensor_tp_flag, ds4_gpu_dsv4_qkv_norm_defer_kv_next and g_tp_block_ctx are declared only under __APPLE__ but are referenced from code every backend compiles, so ROCm (and CUDA) fail to build. Declare g_tp_block_ctx on all GPU backends, move the TP-flag / deferred kv-norm declarations out of the Apple block in ds4_gpu.h, and add no-op / plain-add fallback stubs to the ROCm backend (TP is Metal-only there). Co-Authored-By: Claude Code --- ds4.c | 2 +- ds4_gpu.h | 2 +- ds4_rocm.cu | 36 ++++++++++++++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/ds4.c b/ds4.c index e4741db81..75548fb89 100644 --- a/ds4.c +++ b/ds4.c @@ -46,7 +46,7 @@ #include "ds4_tp.h" /* TP context for the verify-block RDMA window (set with the gate callbacks). */ -#if !defined(DS4_NO_GPU) && defined(__APPLE__) +#if !defined(DS4_NO_GPU) static ds4_tp *g_tp_block_ctx; #endif diff --git a/ds4_gpu.h b/ds4_gpu.h index ab7aec969..895912a72 100644 --- a/ds4_gpu.h +++ b/ds4_gpu.h @@ -148,6 +148,7 @@ int ds4_gpu_parallel_ffn_start_split( uint32_t n_expert, uint32_t n_expert_used, uint32_t shift_q16); +#endif /* out = a + b into this rank's TP slab slot for (layer, gate), publishing the * gate's checked flag from the same kernel; falls back to ds4_gpu_add_tensor @@ -173,7 +174,6 @@ int ds4_gpu_kv_norm_task_pending(void); int ds4_gpu_kv_norm_task_flush(void); int ds4_gpu_kv_norm_task_begin_concurrent(void); void ds4_gpu_kv_norm_task_end_concurrent(void); -#endif int ds4_gpu_signal_selected_readback_ready(uint64_t *event_value); int ds4_gpu_commit_and_wait_selected_readback(uint64_t event_value, const char *label); int ds4_gpu_wait_selected_readback_ready(uint64_t event_value, const char *label); diff --git a/ds4_rocm.cu b/ds4_rocm.cu index 42b04b018..425a3a2a0 100644 --- a/ds4_rocm.cu +++ b/ds4_rocm.cu @@ -178,6 +178,42 @@ extern "C" int ds4_gpu_tp_gate_encode(uint32_t layer, uint32_t gate) { return 0; } +/* The TP flag-fold and deferred kv-norm paths are Metal-only optimizations; + * non-Apple backends always take the plain fallback (add without the checked + * flag, kv norm always standalone). */ +extern "C" int ds4_gpu_add_tensor_tp_flag( + ds4_gpu_tensor *out, + const ds4_gpu_tensor *a, + const ds4_gpu_tensor *b, + uint32_t n, + uint32_t layer, + uint32_t gate) { + (void)layer; (void)gate; + return ds4_gpu_add_tensor(out, a, b, n); +} + +extern "C" void ds4_gpu_tp_flag_fold_request(uint32_t layer, uint32_t gate) { + (void)layer; (void)gate; +} + +extern "C" void ds4_gpu_dsv4_qkv_norm_defer_kv_next(void) { +} + +extern "C" int ds4_gpu_kv_norm_task_pending(void) { + return 0; +} + +extern "C" int ds4_gpu_kv_norm_task_flush(void) { + return 0; +} + +extern "C" int ds4_gpu_kv_norm_task_begin_concurrent(void) { + return 0; +} + +extern "C" void ds4_gpu_kv_norm_task_end_concurrent(void) { +} + extern "C" void ds4_gpu_tp_set_batch_exchange(ds4_gpu_tp_batch_exchange_fn fn) { (void)fn; } From 30db975898765dd8b72e03f4f79c522176b6e2a0 Mon Sep 17 00:00:00 2001 From: adamlawi Date: Fri, 4 Sep 2026 20:54:08 +0200 Subject: [PATCH 189/189] cuda: add TP flag-fold and kv-norm task fallbacks ds4.c:27325 calls ds4_gpu_add_tensor_tp_flag() under #ifndef DS4_NO_GPU, so every GPU backend has to provide it. Metal implements it; ROCm got fallbacks in 9f005f4c ("rocm: fix build break from Metal-only TP/kv-norm symbols"), merged here in 79907e02. CUDA had neither, so linking ds4, ds4-server, ds4-bench, ds4-eval and ds4-agent fails on CUDA: ds4.o: in function `metal_graph_encode_decode_layer_phase': ds4.c:27325: undefined reference to `ds4_gpu_add_tensor_tp_flag' collect2: error: ld returned 1 exit status This mirrors the ROCm block verbatim: the TP flag-fold and deferred kv-norm paths are Metal-only optimizations, so the plain fallback is the correct behaviour elsewhere - add without the checked flag, kv norm always standalone. No functional change on any path that computes: the only non-empty function delegates to the existing ds4_gpu_add_tensor(). Verified on GB10 / sm_121 (DGX Spark), CUDA 13, make cuda-spark and make quality-score both exit 0 with no unresolved symbols. --- ds4_cuda.cu | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/ds4_cuda.cu b/ds4_cuda.cu index b59fad5ec..28a0161c9 100644 --- a/ds4_cuda.cu +++ b/ds4_cuda.cu @@ -44674,3 +44674,39 @@ extern "C" int ds4_gpu_tp_batch_gate_encode(uint32_t layer, uint32_t rows) { #define DS4_GLM53_VISION_STREAM cuda_decode_stream() #include "ds4_glm53_vision_gpu.cuh" #include "ds4_deepseek4_vision_gpu.cuh" + +/* The TP flag-fold and deferred kv-norm paths are Metal-only optimizations; + * non-Apple backends always take the plain fallback (add without the checked + * flag, kv norm always standalone). */ +extern "C" int ds4_gpu_add_tensor_tp_flag( + ds4_gpu_tensor *out, + const ds4_gpu_tensor *a, + const ds4_gpu_tensor *b, + uint32_t n, + uint32_t layer, + uint32_t gate) { + (void)layer; (void)gate; + return ds4_gpu_add_tensor(out, a, b, n); +} + +extern "C" void ds4_gpu_tp_flag_fold_request(uint32_t layer, uint32_t gate) { + (void)layer; (void)gate; +} + +extern "C" void ds4_gpu_dsv4_qkv_norm_defer_kv_next(void) { +} + +extern "C" int ds4_gpu_kv_norm_task_pending(void) { + return 0; +} + +extern "C" int ds4_gpu_kv_norm_task_flush(void) { + return 0; +} + +extern "C" int ds4_gpu_kv_norm_task_begin_concurrent(void) { + return 0; +} + +extern "C" void ds4_gpu_kv_norm_task_end_concurrent(void) { +}